From 970972d242e26d0cdf635d3af646df1a519dc677 Mon Sep 17 00:00:00 2001
From: ThisIsPIRI <34787507+ThisIsPIRI@users.noreply.github.com>
Date: Mon, 6 Aug 2018 14:41:37 +0900
Subject: [PATCH 0001/1095] Clarify what happens when a new value is input to
 some methods

---
 tensorflow/contrib/training/python/training/hparam.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/training/python/training/hparam.py b/tensorflow/contrib/training/python/training/hparam.py
index 3beb7bfe30..9f5059b4b1 100644
--- a/tensorflow/contrib/training/python/training/hparam.py
+++ b/tensorflow/contrib/training/python/training/hparam.py
@@ -494,6 +494,7 @@ class HParams(object):
       value: New value of the hyperparameter.
 
     Raises:
+      KeyError: If the hyperparameter doesn't exist.
       ValueError: If there is a type mismatch.
     """
     param_type, is_list = self._hparam_types[name]
@@ -510,7 +511,7 @@ class HParams(object):
       setattr(self, name, _cast_to_type_if_compatible(name, param_type, value))
 
   def del_hparam(self, name):
-    """Removes the hyperparameter with key 'name'.
+    """Removes the hyperparameter with key 'name'. Does nothing if it isn't present.
 
     Args:
       name: Name of the hyperparameter.
@@ -532,7 +533,7 @@ class HParams(object):
       The `HParams` instance.
 
     Raises:
-      ValueError: If `values` cannot be parsed.
+      ValueError: If `values` cannot be parsed or a hyperparameter in `values` doesn't exist.
     """
     type_map = dict()
     for name, t in self._hparam_types.items():
@@ -543,7 +544,7 @@ class HParams(object):
     return self.override_from_dict(values_map)
 
   def override_from_dict(self, values_dict):
-    """Override hyperparameter values, parsing new values from a dictionary.
+    """Override existing hyperparameter values, parsing new values from a dictionary.
 
     Args:
       values_dict: Dictionary of name:value pairs.
@@ -552,6 +553,7 @@ class HParams(object):
       The `HParams` instance.
 
     Raises:
+      KeyError: If a hyperparameter in `values_dict` doesn't exist.
       ValueError: If `values_dict` cannot be parsed.
     """
     for name, value in values_dict.items():
@@ -591,7 +593,7 @@ class HParams(object):
         sort_keys=sort_keys)
 
   def parse_json(self, values_json):
-    """Override hyperparameter values, parsing new values from a json object.
+    """Override existing hyperparameter values, parsing new values from a json object.
 
     Args:
       values_json: String containing a json object of name:value pairs.
@@ -600,6 +602,7 @@ class HParams(object):
       The `HParams` instance.
 
     Raises:
+      KeyError: If a hyperparameter in `values_json` doesn't exist.
       ValueError: If `values_json` cannot be parsed.
     """
     values_map = json.loads(values_json)
-- 
GitLab


From 26368188c018cdcc1bbb80ce8205fb04305816c2 Mon Sep 17 00:00:00 2001
From: ThisIsPIRI <34787507+ThisIsPIRI@users.noreply.github.com>
Date: Mon, 6 Aug 2018 14:56:38 +0900
Subject: [PATCH 0002/1095] Add 'existing' to parse's docstring

---
 tensorflow/contrib/training/python/training/hparam.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/training/python/training/hparam.py b/tensorflow/contrib/training/python/training/hparam.py
index 9f5059b4b1..372630df81 100644
--- a/tensorflow/contrib/training/python/training/hparam.py
+++ b/tensorflow/contrib/training/python/training/hparam.py
@@ -521,7 +521,7 @@ class HParams(object):
       del self._hparam_types[name]
 
   def parse(self, values):
-    """Override hyperparameter values, parsing new values from a string.
+    """Override existing hyperparameter values, parsing new values from a string.
 
     See parse_values for more detail on the allowed format for values.
 
-- 
GitLab


From bd6c11f878e1820417d1ceff1b02222178f60c16 Mon Sep 17 00:00:00 2001
From: Guozhong Zhuang <guozhong.zhuang@intel.com>
Date: Fri, 12 Oct 2018 10:10:21 -0700
Subject: [PATCH 0003/1095] Clean out MKL_ML code from batchnorm ops

---
 .../core/kernels/mkl_fused_batch_norm_op.cc   | 658 +-----------------
 1 file changed, 2 insertions(+), 656 deletions(-)

diff --git a/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc b/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc
index 2ec6c8fa89..4b8c066902 100644
--- a/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc
+++ b/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc
@@ -20,671 +20,19 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/util/tensor_format.h"
-
-#ifndef INTEL_MKL_ML_ONLY
+#include "tensorflow/core/util/mkl_util.h"
 #include "mkldnn.hpp"
+
 using mkldnn::batch_normalization_backward;
 using mkldnn::batch_normalization_forward;
 using mkldnn::prop_kind;
 using mkldnn::stream;
 using mkldnn::use_global_stats;
 using mkldnn::use_scale_shift;
-#else
-#include "mkl_dnn.h"
-#include "mkl_dnn_types.h"
-#endif
-
-#include "tensorflow/core/util/mkl_util.h"
-// TODO(inteltf) Address comments from PR 8968.
 
 namespace tensorflow {
 using CPUDevice = Eigen::ThreadPoolDevice;
 
-#ifdef INTEL_MKL_ML_ONLY
-
-template <typename Device, typename T>
-class MklFusedBatchNormOp : public OpKernel {
- public:
-  explicit MklFusedBatchNormOp(OpKernelConstruction* context)
-      : OpKernel(context) {
-    float epsilon;
-    OP_REQUIRES_OK(context, context->GetAttr("epsilon", &epsilon));
-    epsilon_ = T(epsilon);
-    string tensor_format;
-    OP_REQUIRES_OK(context, context->GetAttr("data_format", &tensor_format));
-    OP_REQUIRES(context, FormatFromString(tensor_format, &tensor_format_),
-                errors::InvalidArgument("Invalid data format"));
-    OP_REQUIRES_OK(context, context->GetAttr("is_training", &is_training_));
-  }
-
-  void Compute(OpKernelContext* context) override {
-    MklFusedBatchNormOpContext mkl_context;
-    const Tensor& input = MklGetInput(context, 0);
-    const Tensor& scale = MklGetInput(context, 1);
-    const Tensor& shift = MklGetInput(context, 2);
-    const Tensor& est_mean = MklGetInput(context, 3);
-    const Tensor& est_variance = MklGetInput(context, 4);
-
-    GetMklShape(context, 0, &(mkl_context.mkl_shape_input_shape));
-    bool input_in_mkl_format = mkl_context.mkl_shape_input_shape.IsMklTensor();
-
-    if (!input_in_mkl_format) {
-      OP_REQUIRES(context, input.dims() == 4,
-                  errors::InvalidArgument("input must be 4-dimensional",
-                                          input.shape().DebugString()));
-    }
-    OP_REQUIRES(context, scale.dims() == 1,
-                errors::InvalidArgument("scale must be 1-dimensional",
-                                        scale.shape().DebugString()));
-    OP_REQUIRES(context, shift.dims() == 1,
-                errors::InvalidArgument("offset must be 1-dimensional",
-                                        shift.shape().DebugString()));
-    OP_REQUIRES(context, est_mean.dims() == 1,
-                errors::InvalidArgument("estimated_mean must be 1-dimensional",
-                                        est_mean.shape().DebugString()));
-
-    OP_REQUIRES(
-        context, est_variance.dims() == 1,
-        errors::InvalidArgument("estimated_variance must be 1-dimensional",
-                                est_variance.shape().DebugString()));
-
-    if (is_training_) {
-      OP_REQUIRES(context, est_mean.dim_size(0) == 0,
-                  errors::InvalidArgument("estimated_mean empty for training",
-                                          est_mean.shape().DebugString()));
-      OP_REQUIRES(context, est_variance.dim_size(0) == 0,
-                  errors::InvalidArgument(
-                      "estimated_variance must be empty for training",
-                      est_variance.shape().DebugString()));
-    }
-
-    unsigned int flag_batch_norm =
-        is_training_ ? dnnUseScaleShift
-                     : (dnnUseInputMeanVariance | dnnUseScaleShift);
-
-    mkl_context.MklExtractParams(context, tensor_format_);
-
-    // Create layout only for input data as it is used in Op primitive.
-    mkl_context.MklCreateInputLayout(context);
-
-    // Create Op primitive.
-    CHECK_EQ(dnnBatchNormalizationCreateForward_v2_F32(
-                 &(mkl_context.mkl_prim_batchnorm), nullptr,
-                 mkl_context.mkl_lt_input, static_cast<float>(epsilon_),
-                 flag_batch_norm),
-             E_SUCCESS);
-
-    // Temporary tensors with buffers for the context inputs, if
-    // conversion to MKL-Op specific layouts are required. It is assumed here
-    // that TF's 1D tensors (scale, shift, est_mean, and est_variance) won't
-    // require any conversion.
-    // Since scale-shift is combined in MKL, a buffer is required.
-    Tensor mkl_tmp_input_buf_tensor, mkl_tmp_scale_shift_buf_tensor;
-    mkl_context.MklPrepareContextInputs(context, &mkl_tmp_input_buf_tensor,
-                                        &mkl_tmp_scale_shift_buf_tensor);
-
-    // Output data in MKL layout
-    Tensor* output = nullptr;
-    TensorShape tf_shape_output;
-    MklShape mkl_shape_output;
-    mkl_shape_output.SetMklTensor(true);
-    mkl_shape_output.SetMklLayout(mkl_context.mkl_prim_batchnorm,
-                                  dnnResourceDst);
-    mkl_shape_output.SetTfLayout(mkl_context.mkl_params.in_dim,
-                                 mkl_context.mkl_params.in_sizes,
-                                 mkl_context.mkl_params.in_strides);
-    mkl_shape_output.SetTfDimOrder(mkl_context.mkl_params.in_dim,
-                                   tensor_format_);
-    tf_shape_output.AddDim(dnnLayoutGetMemorySize_F32(static_cast<dnnLayout_t>(
-                               mkl_shape_output.GetMklLayout())) /
-                           sizeof(T));
-    AllocateOutputSetMklShape(context, 0, &output, tf_shape_output,
-                              mkl_shape_output);
-    mkl_context.mkl_res_batchnorm[dnnResourceDst] =
-        static_cast<void*>(output->flat<T>().data());
-
-    // Batch mean in TF layout
-    Tensor* batch_mean = nullptr;
-    MklShape mkl_shape_batch_mean;
-    mkl_shape_batch_mean.SetMklTensor(false);
-    AllocateOutputSetMklShape(context, 1, &batch_mean, scale.shape(),
-                              mkl_shape_batch_mean);
-    // Batch variance in TF layout
-    Tensor* batch_variance = nullptr;
-    MklShape mkl_shape_batch_variance;
-    mkl_shape_batch_variance.SetMklTensor(false);
-    AllocateOutputSetMklShape(context, 2, &batch_variance, scale.shape(),
-                              mkl_shape_batch_variance);
-    // If training mode, set dnnResourceMean and dnnResourceVariance to
-    // output tensors for batch mean and variance.
-    // Otherwise, set dnnResourceMean and dnnResourceVariance to
-    // estimated mean and variance.
-    if (is_training_)
-      mkl_context.MklSetMeanVariance(*batch_mean, *batch_variance);
-    else
-      mkl_context.MklSetMeanVariance(est_mean, est_variance);
-
-    // Now that all resources are set, it is ready for dnnExecute
-    CHECK_EQ(dnnExecute_F32(mkl_context.mkl_prim_batchnorm,
-                            mkl_context.mkl_res_batchnorm),
-             E_SUCCESS);
-
-    // Mean and variance (without Bessel's correction) saved for backward
-    // computation to serve as pre-computed mean and variance.
-    Tensor* saved_mean = nullptr;
-    MklShape mkl_shape_saved_mean;
-    mkl_shape_saved_mean.SetMklTensor(false);
-    AllocateOutputSetMklShape(context, 3, &saved_mean, scale.shape(),
-                              mkl_shape_saved_mean);
-    std::memcpy(
-        reinterpret_cast<char*>(saved_mean->flat<float>().data()),
-        reinterpret_cast<char*>(mkl_context.mkl_res_batchnorm[dnnResourceMean]),
-        scale.NumElements() * sizeof(float));
-    Tensor* saved_variance = nullptr;
-    MklShape mkl_shape_saved_variance;
-    mkl_shape_saved_variance.SetMklTensor(false);
-    AllocateOutputSetMklShape(context, 4, &saved_variance, scale.shape(),
-                              mkl_shape_saved_variance);
-    std::memcpy(reinterpret_cast<char*>(saved_variance->flat<float>().data()),
-                reinterpret_cast<char*>(
-                    mkl_context.mkl_res_batchnorm[dnnResourceVariance]),
-                scale.NumElements() * sizeof(float));
-
-    // Bessel's correction on variance, if training mode is on
-    if (is_training_) {
-      float* p_var = static_cast<float*>(batch_variance->flat<T>().data());
-      auto depth = mkl_context.mkl_params.depth;
-      size_t orig_size = mkl_context.mkl_params.in_sizes[0] *
-                         mkl_context.mkl_params.in_sizes[1] *
-                         mkl_context.mkl_params.in_sizes[3];
-      size_t adjust_size = orig_size - 1;
-      float adjust_factor = (static_cast<float>(orig_size)) / adjust_size;
-      for (int i = 0; i < depth; i++) p_var[i] = adjust_factor * p_var[i];
-    }
-
-    mkl_context.MklCleanup();
-  }
-
- private:
-  T epsilon_;
-  TensorFormat tensor_format_;
-  bool is_training_;
-
-  // Structure containing all info for MklOp
-  typedef struct {
-    // Parameters used for input and output layouts
-    struct MklBatchNormParams {
-      // BatchNormOp src and
-      size_t in_dim;
-      size_t in_sizes[4];
-      size_t in_strides[4];
-      size_t depth;  // Batch normalization is done for per channel.
-    } mkl_params;
-
-    MklShape mkl_shape_input_shape;
-
-    // MKL primitive and resources for BatchNormOp
-    dnnPrimitive_t mkl_prim_batchnorm = nullptr;
-    void* mkl_res_batchnorm[dnnResourceNumber];
-
-    // MKL layouts for inputs in the context
-    dnnLayout_t mkl_lt_input = nullptr;
-
-    void MklCleanup() {
-      bool input_in_mkl_format = mkl_shape_input_shape.IsMklTensor();
-      if (!input_in_mkl_format) dnnLayoutDelete_F32(mkl_lt_input);
-      if (mkl_prim_batchnorm != nullptr) dnnDelete_F32(mkl_prim_batchnorm);
-    }
-
-    void MklExtractParams(OpKernelContext* context,
-                          const TensorFormat& tensor_format) {
-      const Tensor& input = MklGetInput(context, 0);
-      bool input_in_mkl_format = mkl_shape_input_shape.IsMklTensor();
-      mkl_params.in_dim = input_in_mkl_format
-                              ? mkl_shape_input_shape.GetDimension()
-                              : input.dims();
-      mkl_params.in_sizes[0] = static_cast<size_t>(
-          input_in_mkl_format ? mkl_shape_input_shape.GetSizes()[0]
-                              : GetTensorDim(input, tensor_format, 'W'));
-      mkl_params.in_sizes[1] = static_cast<size_t>(
-          input_in_mkl_format ? mkl_shape_input_shape.GetSizes()[1]
-                              : GetTensorDim(input, tensor_format, 'H'));
-      mkl_params.in_sizes[2] = static_cast<size_t>(
-          input_in_mkl_format ? mkl_shape_input_shape.GetSizes()[2]
-                              : GetTensorDim(input, tensor_format, 'C'));
-      mkl_params.in_sizes[3] = static_cast<size_t>(
-          input_in_mkl_format ? mkl_shape_input_shape.GetSizes()[3]
-                              : GetTensorDim(input, tensor_format, 'N'));
-      mkl_params.depth = mkl_params.in_sizes[2];
-      GetStridesFromSizes(tensor_format, mkl_params.in_strides,
-                          mkl_params.in_sizes);
-    }
-
-    void MklCreateInputLayout(OpKernelContext* context) {
-      const Tensor& input = MklGetInput(context, 0);
-      bool input_in_mkl_format = mkl_shape_input_shape.IsMklTensor();
-      if (input_in_mkl_format) {
-        mkl_lt_input =
-            static_cast<dnnLayout_t>(mkl_shape_input_shape.GetCurLayout());
-      } else {
-        CHECK_EQ(
-            dnnLayoutCreate_F32(&mkl_lt_input, mkl_params.in_dim,
-                                mkl_params.in_sizes, mkl_params.in_strides),
-            E_SUCCESS);
-      }
-    }
-    void MklPrepareContextInputs(OpKernelContext* context,
-                                 Tensor* mkl_tmp_input_buf_tensor,
-                                 Tensor* mkl_tmp_scale_shift_buf_tensor) {
-      bool mkl_convert_input;
-      dnnPrimitive_t mkl_prim_convert_input = nullptr;
-      dnnLayout_t mkl_lt_internal_input = nullptr;
-      void* mkl_buf_converted_input = nullptr;
-      // Compare with internal layouts and convert if needed
-      const Tensor& input = MklGetInput(context, 0);
-      void* mkl_buf_input =
-          const_cast<void*>(static_cast<const void*>(input.flat<T>().data()));
-      CHECK_EQ(dnnLayoutCreateFromPrimitive_F32(
-                   &mkl_lt_internal_input, mkl_prim_batchnorm, dnnResourceSrc),
-               E_SUCCESS);
-      mkl_convert_input =
-          !dnnLayoutCompare_F32(mkl_lt_internal_input, mkl_lt_input);
-      if (mkl_convert_input) {
-        CHECK_EQ(dnnConversionCreate_F32(&mkl_prim_convert_input, mkl_lt_input,
-                                         mkl_lt_internal_input),
-                 E_SUCCESS);
-        AllocTmpBuffer(context, mkl_tmp_input_buf_tensor, mkl_lt_internal_input,
-                       &mkl_buf_converted_input);
-        CHECK_EQ(dnnConversionExecute_F32(mkl_prim_convert_input, mkl_buf_input,
-                                          mkl_buf_converted_input),
-                 E_SUCCESS);
-        dnnDelete_F32(mkl_prim_convert_input);
-      }
-      dnnLayoutDelete_F32(mkl_lt_internal_input);
-      mkl_res_batchnorm[dnnResourceSrc] =
-          (mkl_convert_input) ? mkl_buf_converted_input : mkl_buf_input;
-
-      // scale-shift layout is created from primitive. So no conversion
-      // is needed, however, a buffer has to be allocated.
-      dnnLayout_t mkl_lt_scale_shift = nullptr;
-      void* mkl_buf_scale_shift = nullptr;
-      CHECK_EQ(
-          dnnLayoutCreateFromPrimitive_F32(
-              &mkl_lt_scale_shift, mkl_prim_batchnorm, dnnResourceScaleShift),
-          E_SUCCESS);
-      AllocTmpBuffer(context, mkl_tmp_scale_shift_buf_tensor,
-                     mkl_lt_scale_shift, &mkl_buf_scale_shift);
-      // Fill the scale-shift buffer with data, presumably buffer is 2D array
-      const Tensor& scale = MklGetInput(context, 1);
-      const Tensor& shift = MklGetInput(context, 2);
-      float* buf_scale_shift = static_cast<float*>(mkl_buf_scale_shift);
-      float* buf_scale = const_cast<float*>(
-          static_cast<const float*>(scale.flat<float>().data()));
-      float* buf_shift = const_cast<float*>(
-          static_cast<const float*>(shift.flat<float>().data()));
-      auto depth = mkl_params.depth;
-      for (int i = 0; i < depth; i++) {
-        buf_scale_shift[i] = buf_scale[i];
-        buf_scale_shift[i + depth] = buf_shift[i];
-      }
-      mkl_res_batchnorm[dnnResourceScaleShift] = mkl_buf_scale_shift;
-    }
-
-    inline void MklSetMeanVariance(const Tensor& mean, const Tensor& variance) {
-      mkl_res_batchnorm[dnnResourceMean] = const_cast<void*>(
-          static_cast<const void*>(mean.flat<float>().data()));
-      mkl_res_batchnorm[dnnResourceVariance] = const_cast<void*>(
-          static_cast<const void*>(variance.flat<float>().data()));
-    }
-  } MklFusedBatchNormOpContext;
-};
-
-template <typename Device, typename T>
-class MklFusedBatchNormGradOp : public OpKernel {
- public:
-  explicit MklFusedBatchNormGradOp(OpKernelConstruction* context)
-      : OpKernel(context) {
-    float epsilon;
-    OP_REQUIRES_OK(context, context->GetAttr("epsilon", &epsilon));
-    epsilon_ = T(epsilon);
-    string tensor_format;
-    OP_REQUIRES_OK(context, context->GetAttr("data_format", &tensor_format));
-    OP_REQUIRES(context, FormatFromString(tensor_format, &tensor_format_),
-                errors::InvalidArgument("Invalid data format"));
-  }
-
-  void Compute(OpKernelContext* context) override {
-    MklFusedBatchNormGradOpContext mkl_context;
-
-    const Tensor& out_backprop = MklGetInput(context, 0);
-    const Tensor& input = MklGetInput(context, 1);
-    const Tensor& scale = MklGetInput(context, 2);
-    const Tensor& saved_mean = MklGetInput(context, 3);
-    const Tensor& saved_var = MklGetInput(context, 4);
-
-    // Here scale, mean, and variance are 1D and considered
-    // those having same layout in MKL and TF
-    GetMklShape(context, 0, &(mkl_context.mkl_shape_out_backprop));
-    GetMklShape(context, 1, &(mkl_context.mkl_shape_input_shape));
-
-    bool input_in_mkl_format = mkl_context.mkl_shape_input_shape.IsMklTensor();
-    bool out_backprop_in_mkl_format =
-        mkl_context.mkl_shape_out_backprop.IsMklTensor();
-    if (!out_backprop_in_mkl_format) {
-      OP_REQUIRES(context, out_backprop.dims() == 4,
-                  errors::InvalidArgument("input must be 4-dimensional",
-                                          out_backprop.shape().DebugString()));
-    }
-    if (!input_in_mkl_format) {
-      OP_REQUIRES(context, input.dims() == 4,
-                  errors::InvalidArgument("input must be 4-dimensional",
-                                          input.shape().DebugString()));
-    }
-    OP_REQUIRES(context, scale.dims() == 1,
-                errors::InvalidArgument("scale must be 1-dimensional",
-                                        scale.shape().DebugString()));
-    OP_REQUIRES(context, saved_mean.dims() == 1,
-                errors::InvalidArgument("saved mean must be 1-dimensional",
-                                        saved_mean.shape().DebugString()));
-    OP_REQUIRES(context, saved_var.dims() == 1,
-                errors::InvalidArgument("saved variance must be 1-dimensional",
-                                        saved_var.shape().DebugString()));
-
-    mkl_context.MklExtractParams(context, tensor_format_);
-
-    mkl_context.MklCreateInputLayout(context);
-
-    unsigned int flag_batch_norm_grad = dnnUseScaleShift;
-
-    // Create Backward Op primitive.
-    CHECK_EQ(dnnBatchNormalizationCreateBackward_v2_F32(
-                 &(mkl_context.mkl_prim_batchnorm_bwd), nullptr,
-                 mkl_context.mkl_lt_input, static_cast<float>(epsilon_),
-                 flag_batch_norm_grad),
-             E_SUCCESS);
-
-    // Temporary tensors and their buffers if conversion is required
-    Tensor mkl_tmp_input_buf_tensor, mkl_tmp_outbackprop_buf_tensor,
-        mkl_tmp_scaleshift_buf_tensor;
-    mkl_context.MklPrepareContextInputs(context, &mkl_tmp_input_buf_tensor,
-                                        &mkl_tmp_outbackprop_buf_tensor,
-                                        &mkl_tmp_scaleshift_buf_tensor);
-
-    // Allocate tensor for grad w.r.t. input(x)
-    Tensor* in_backprop = nullptr;
-    TensorShape tf_shape_in_backprop;
-    MklShape mkl_shape_in_backprop;
-    mkl_shape_in_backprop.SetMklTensor(true);
-    mkl_shape_in_backprop.SetMklLayout(mkl_context.mkl_prim_batchnorm_bwd,
-                                       dnnResourceDiffSrc);
-    mkl_shape_in_backprop.SetTfLayout(mkl_context.mkl_params.in_dims,
-                                      mkl_context.mkl_params.in_sizes,
-                                      mkl_context.mkl_params.in_strides);
-    mkl_shape_in_backprop.SetTfDimOrder(mkl_context.mkl_params.in_dims,
-                                        tensor_format_);
-    tf_shape_in_backprop.AddDim(
-        dnnLayoutGetMemorySize_F32(
-            static_cast<dnnLayout_t>(mkl_shape_in_backprop.GetMklLayout())) /
-        sizeof(T));
-    AllocateOutputSetMklShape(context, 0, &in_backprop, tf_shape_in_backprop,
-                              mkl_shape_in_backprop);
-    mkl_context.mkl_res_batchnorm_bwd[dnnResourceDiffSrc] =
-        static_cast<void*>(in_backprop->flat<T>().data());
-
-    // grad_scale and grad_shift are combined together in MKL
-    // So create a single temporary buffer for those.
-    // Also set dnnResourceDiffScaleShift to the temporary buffer
-    Tensor mkl_tmp_grad_scale_shift_buf_tensor;
-    mkl_context.MklPrepareGradScaleShift(context,
-                                         &mkl_tmp_grad_scale_shift_buf_tensor);
-
-    // All dnn resources are set now, ready to execute
-    CHECK_EQ(dnnExecute_F32(mkl_context.mkl_prim_batchnorm_bwd,
-                            mkl_context.mkl_res_batchnorm_bwd),
-             E_SUCCESS);
-
-    // Now separate out scale and shift grad and copy to individual tensors
-    const TensorShape& tf_shape_scale_shift = scale.shape();
-    // Allocate tensor for grad w.r.t. scale (beta)
-    Tensor* scale_backprop = nullptr;
-    MklShape mkl_shape_scale_backprop;
-    AllocateOutputSetMklShape(context, 1, &scale_backprop, tf_shape_scale_shift,
-                              mkl_shape_scale_backprop);
-
-    // Allocate tensor for grad w.r.t. shift(gamma)
-    Tensor* shift_backprop = nullptr;
-    MklShape mkl_shape_shift_backprop;
-    AllocateOutputSetMklShape(context, 2, &shift_backprop, tf_shape_scale_shift,
-                              mkl_shape_shift_backprop);
-
-    // copy scale and shift grads to tensors
-    float* mkl_buf_scale_shift = const_cast<float*>(static_cast<const float*>(
-        mkl_tmp_grad_scale_shift_buf_tensor.flat<T>().data()));
-    float* tf_buf_scale = const_cast<float*>(
-        static_cast<const float*>(scale_backprop->flat<T>().data()));
-    float* tf_buf_shift = const_cast<float*>(
-        static_cast<const float*>(shift_backprop->flat<T>().data()));
-    auto depth = mkl_context.mkl_params.depth;
-    for (int i = 0; i < depth; i++) {
-      tf_buf_scale[i] = mkl_buf_scale_shift[i];
-      tf_buf_shift[i] = mkl_buf_scale_shift[i + depth];
-    }
-
-    // Two placeholders for estimated_mean and estimated_variance, which are
-    // used for inference and thus not needed here for gradient computation.
-    Tensor* placeholder_1 = nullptr;
-    MklShape mkl_shape_placeholder_1;
-    AllocateOutputSetMklShape(context, 3, &placeholder_1, TensorShape({}),
-                              mkl_shape_placeholder_1);
-    Tensor* placeholder_2 = nullptr;
-    MklShape mkl_shape_placeholder_2;
-    AllocateOutputSetMklShape(context, 4, &placeholder_2, TensorShape({}),
-                              mkl_shape_placeholder_2);
-
-    mkl_context.MklCleanup();
-  }
-
- private:
-  T epsilon_;
-  TensorFormat tensor_format_;
-
-  // Structure containing all info for MklOp
-  typedef struct {
-    // Parameters used for input and output layouts
-    struct MklBatchNormParams {
-      // BatchNormOp src and
-      size_t in_dims;
-      size_t in_sizes[4];
-      size_t in_strides[4];
-      size_t depth;  // Batch normalization is done for per channel.
-    } mkl_params;
-
-    MklShape mkl_shape_out_backprop;
-    MklShape mkl_shape_input_shape;
-
-    // MKL primitive and resources for BatchNormOp
-    dnnPrimitive_t mkl_prim_batchnorm_bwd = nullptr;
-    void* mkl_res_batchnorm_bwd[dnnResourceNumber];
-
-    // MKL layouts for inputs in the context
-    dnnLayout_t mkl_lt_out_backprop = nullptr;
-    dnnLayout_t mkl_lt_input = nullptr;
-
-    void MklCleanup() {
-      bool input_in_mkl_format = mkl_shape_input_shape.IsMklTensor();
-      bool out_backprop_in_mkl_format = mkl_shape_out_backprop.IsMklTensor();
-      if (!input_in_mkl_format) dnnLayoutDelete_F32(mkl_lt_input);
-      if (!out_backprop_in_mkl_format) dnnLayoutDelete_F32(mkl_lt_out_backprop);
-
-      dnnDelete_F32(mkl_prim_batchnorm_bwd);
-    }
-
-    void MklExtractParams(OpKernelContext* context,
-                          const TensorFormat& tensor_format) {
-      const Tensor& input = MklGetInput(context, 1);
-      bool input_in_mkl_format = mkl_shape_input_shape.IsMklTensor();
-      mkl_params.in_dims = input_in_mkl_format
-                               ? mkl_shape_input_shape.GetDimension()
-                               : input.dims();
-      mkl_params.in_sizes[0] = static_cast<size_t>(
-          input_in_mkl_format ? mkl_shape_input_shape.GetSizes()[0]
-                              : GetTensorDim(input, tensor_format, 'W'));
-      mkl_params.in_sizes[1] = static_cast<size_t>(
-          input_in_mkl_format ? mkl_shape_input_shape.GetSizes()[1]
-                              : GetTensorDim(input, tensor_format, 'H'));
-      mkl_params.in_sizes[2] = static_cast<size_t>(
-          input_in_mkl_format ? mkl_shape_input_shape.GetSizes()[2]
-                              : GetTensorDim(input, tensor_format, 'C'));
-      mkl_params.in_sizes[3] = static_cast<size_t>(
-          input_in_mkl_format ? mkl_shape_input_shape.GetSizes()[3]
-                              : GetTensorDim(input, tensor_format, 'N'));
-      mkl_params.depth = mkl_params.in_sizes[2];
-      GetStridesFromSizes(tensor_format, mkl_params.in_strides,
-                          mkl_params.in_sizes);
-    }
-
-    void MklCreateInputLayout(OpKernelContext* context) {
-      const Tensor& input = MklGetInput(context, 0);
-      bool input_in_mkl_format = mkl_shape_input_shape.IsMklTensor();
-      if (input_in_mkl_format) {
-        mkl_lt_input =
-            static_cast<dnnLayout_t>(mkl_shape_input_shape.GetCurLayout());
-      } else {
-        CHECK_EQ(
-            dnnLayoutCreate_F32(&mkl_lt_input, mkl_params.in_dims,
-                                mkl_params.in_sizes, mkl_params.in_strides),
-            E_SUCCESS);
-      }
-
-      bool out_backprop_in_mkl_format = mkl_shape_out_backprop.IsMklTensor();
-      if (out_backprop_in_mkl_format) {
-        mkl_lt_out_backprop =
-            static_cast<dnnLayout_t>(mkl_shape_out_backprop.GetCurLayout());
-      } else {
-        CHECK_EQ(
-            dnnLayoutCreate_F32(&mkl_lt_out_backprop, mkl_params.in_dims,
-                                mkl_params.in_sizes, mkl_params.in_strides),
-            E_SUCCESS);
-      }
-    }
-
-    void MklPrepareContextInputs(OpKernelContext* context,
-                                 Tensor* mkl_tmp_input_buf_tensor,
-                                 Tensor* mkl_tmp_outbackprop_buf_tensor,
-                                 Tensor* mkl_tmp_scaleshift_buf_tensor) {
-      bool mkl_convert_input;
-      dnnPrimitive_t mkl_prim_convert_input = nullptr;
-      dnnLayout_t mkl_lt_internal_input = nullptr;
-      void* mkl_buf_converted_input = nullptr;
-      // Compare with internal layouts and convert if needed
-      const Tensor& input = MklGetInput(context, 1);
-      void* mkl_buf_input =
-          const_cast<void*>(static_cast<const void*>(input.flat<T>().data()));
-      CHECK_EQ(
-          dnnLayoutCreateFromPrimitive_F32(
-              &mkl_lt_internal_input, mkl_prim_batchnorm_bwd, dnnResourceSrc),
-          E_SUCCESS);
-      mkl_convert_input =
-          !dnnLayoutCompare_F32(mkl_lt_internal_input, mkl_lt_input);
-      if (mkl_convert_input) {
-        CHECK_EQ(dnnConversionCreate_F32(&mkl_prim_convert_input, mkl_lt_input,
-                                         mkl_lt_internal_input),
-                 E_SUCCESS);
-        AllocTmpBuffer(context, mkl_tmp_input_buf_tensor, mkl_lt_internal_input,
-                       &mkl_buf_converted_input);
-        CHECK_EQ(dnnConversionExecute_F32(mkl_prim_convert_input, mkl_buf_input,
-                                          mkl_buf_converted_input),
-                 E_SUCCESS);
-        dnnDelete_F32(mkl_prim_convert_input);
-      }
-      dnnLayoutDelete_F32(mkl_lt_internal_input);
-      mkl_res_batchnorm_bwd[dnnResourceSrc] =
-          (mkl_convert_input) ? mkl_buf_converted_input : mkl_buf_input;
-
-      bool mkl_convert_out_backprop;
-      dnnPrimitive_t mkl_prim_convert_out_backprop = nullptr;
-      dnnLayout_t mkl_lt_internal_out_backprop = nullptr;
-      void* mkl_buf_converted_out_backprop = nullptr;
-      // Compare with internal layouts and convert if needed
-      const Tensor& out_backprop = MklGetInput(context, 0);
-      void* mkl_buf_out_backprop = const_cast<void*>(
-          static_cast<const void*>(out_backprop.flat<T>().data()));
-      CHECK_EQ(dnnLayoutCreateFromPrimitive_F32(&mkl_lt_internal_out_backprop,
-                                                mkl_prim_batchnorm_bwd,
-                                                dnnResourceDiffDst),
-               E_SUCCESS);
-      mkl_convert_out_backprop = !dnnLayoutCompare_F32(
-          mkl_lt_internal_out_backprop, mkl_lt_out_backprop);
-      if (mkl_convert_out_backprop) {
-        CHECK_EQ(dnnConversionCreate_F32(&mkl_prim_convert_out_backprop,
-                                         mkl_lt_out_backprop,
-                                         mkl_lt_internal_out_backprop),
-                 E_SUCCESS);
-        AllocTmpBuffer(context, mkl_tmp_outbackprop_buf_tensor,
-                       mkl_lt_internal_out_backprop,
-                       &mkl_buf_converted_out_backprop);
-        CHECK_EQ(dnnConversionExecute_F32(mkl_prim_convert_out_backprop,
-                                          mkl_buf_out_backprop,
-                                          mkl_buf_converted_out_backprop),
-                 E_SUCCESS);
-        dnnDelete_F32(mkl_prim_convert_out_backprop);
-      }
-      dnnLayoutDelete_F32(mkl_lt_internal_out_backprop);
-      mkl_res_batchnorm_bwd[dnnResourceDiffDst] =
-          (mkl_convert_out_backprop) ? mkl_buf_converted_out_backprop
-                                     : mkl_buf_out_backprop;
-
-      // Set dnnResourceMean and dnnResourceVariance
-      const Tensor& saved_mean = MklGetInput(context, 3);
-      const Tensor& saved_var = MklGetInput(context, 4);
-      void* mkl_buf_saved_mean = const_cast<void*>(
-          static_cast<const void*>(saved_mean.flat<T>().data()));
-      void* mkl_buf_saved_var = const_cast<void*>(
-          static_cast<const void*>(saved_var.flat<T>().data()));
-      mkl_res_batchnorm_bwd[dnnResourceMean] = mkl_buf_saved_mean;
-      mkl_res_batchnorm_bwd[dnnResourceVariance] = mkl_buf_saved_var;
-
-      // Set dnnResourceScaleShift
-      // Note backward Op needs only current values of scale parameters,
-      // shift parameters could be garbage and won't be used
-      const Tensor& scale = MklGetInput(context, 2);
-      dnnLayout_t mkl_lt_scale_shift = nullptr;
-      void* mkl_buf_scale_shift = nullptr;
-      CHECK_EQ(dnnLayoutCreateFromPrimitive_F32(&mkl_lt_scale_shift,
-                                                mkl_prim_batchnorm_bwd,
-                                                dnnResourceScaleShift),
-               E_SUCCESS);
-      AllocTmpBuffer(context, mkl_tmp_scaleshift_buf_tensor, mkl_lt_scale_shift,
-                     &mkl_buf_scale_shift);
-      float* pscale =
-          const_cast<float*>(static_cast<const float*>(scale.flat<T>().data()));
-      float* pscale_shift = static_cast<float*>(mkl_buf_scale_shift);
-      auto depth = mkl_params.depth;
-      for (int i = 0; i < depth; i++) pscale_shift[i] = pscale[i];
-      mkl_res_batchnorm_bwd[dnnResourceScaleShift] = mkl_buf_scale_shift;
-      dnnLayoutDelete_F32(mkl_lt_scale_shift);
-    }
-
-    void MklPrepareGradScaleShift(OpKernelContext* context,
-                                  Tensor* mkl_tmp_grad_scale_shift_buf_tensor) {
-      dnnLayout_t mkl_lt_grad_scaleshift = nullptr;
-      void* mkl_buf_grad_scaleshift = nullptr;
-      CHECK_EQ(dnnLayoutCreateFromPrimitive_F32(&mkl_lt_grad_scaleshift,
-                                                mkl_prim_batchnorm_bwd,
-                                                dnnResourceDiffScaleShift),
-               E_SUCCESS);
-      AllocTmpBuffer(context, mkl_tmp_grad_scale_shift_buf_tensor,
-                     mkl_lt_grad_scaleshift, &mkl_buf_grad_scaleshift);
-      mkl_res_batchnorm_bwd[dnnResourceDiffScaleShift] =
-          mkl_buf_grad_scaleshift;
-      dnnLayoutDelete_F32(mkl_lt_grad_scaleshift);
-    }
-  } MklFusedBatchNormGradOpContext;
-};
-#endif
-
-#ifndef INTEL_MKL_ML_ONLY
-
 struct MklBatchNormFwdParams {
   memory::dims src_dims;
   int depth;
@@ -1765,8 +1113,6 @@ class MklFusedBatchNormGradOp : public OpKernel {
   memory::dims GetMeanVarianceDims() { return memory::dims({1, depth_}); }
 };
 
-#endif
-
 #define REGISTER_MKL_CPU(T)                                         \
   REGISTER_KERNEL_BUILDER(Name("_MklFusedBatchNorm")                \
                               .Device(DEVICE_CPU)                   \
-- 
GitLab


From cb9dccae23f34edb15cdbe58ad6fceab702138fb Mon Sep 17 00:00:00 2001
From: Guozhong Zhuang <guozhong.zhuang@intel.com>
Date: Fri, 12 Oct 2018 10:41:32 -0700
Subject: [PATCH 0004/1095] adjust headers inclusion order per review
 suggestion of another PR

---
 tensorflow/core/kernels/mkl_fused_batch_norm_op.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc b/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc
index 4b8c066902..ff46e75a36 100644
--- a/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc
+++ b/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc
@@ -19,8 +19,8 @@ limitations under the License.
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_types.h"
-#include "tensorflow/core/util/tensor_format.h"
 #include "tensorflow/core/util/mkl_util.h"
+#include "tensorflow/core/util/tensor_format.h"
 #include "mkldnn.hpp"
 
 using mkldnn::batch_normalization_backward;
-- 
GitLab


From bae74d26f93872374b48c60a73d189df148a6f99 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?=
 <facai.yan@gmail.com>
Date: Tue, 16 Oct 2018 09:45:45 +0800
Subject: [PATCH 0005/1095] CLN: remove reshape in conv3d, becasue bias_add has
 supported 5-dim

---
 .../python/keras/layers/convolutional.py      | 33 ++-----------------
 .../python/keras/layers/convolutional_test.py | 16 +++++++++
 2 files changed, 19 insertions(+), 30 deletions(-)

diff --git a/tensorflow/python/keras/layers/convolutional.py b/tensorflow/python/keras/layers/convolutional.py
index 58024677ee..f8afa0d430 100644
--- a/tensorflow/python/keras/layers/convolutional.py
+++ b/tensorflow/python/keras/layers/convolutional.py
@@ -199,21 +199,8 @@ class Conv(Layer):
           # nn.bias_add does not accept a 1D input tensor.
           bias = array_ops.reshape(self.bias, (1, self.filters, 1))
           outputs += bias
-        if self.rank == 2:
+        else:
           outputs = nn.bias_add(outputs, self.bias, data_format='NCHW')
-        if self.rank == 3:
-          # As of Mar 2017, direct addition is significantly slower than
-          # bias_add when computing gradients. To use bias_add, we collapse Z
-          # and Y into a single dimension to obtain a 4D input tensor.
-          outputs_shape = outputs.shape.as_list()
-          if outputs_shape[0] is None:
-            outputs_shape[0] = -1
-          outputs_4d = array_ops.reshape(outputs,
-                                         [outputs_shape[0], outputs_shape[1],
-                                          outputs_shape[2] * outputs_shape[3],
-                                          outputs_shape[4]])
-          outputs_4d = nn.bias_add(outputs_4d, self.bias, data_format='NCHW')
-          outputs = array_ops.reshape(outputs_4d, outputs_shape)
       else:
         outputs = nn.bias_add(outputs, self.bias, data_format='NHWC')
 
@@ -1127,24 +1114,10 @@ class Conv3DTranspose(Conv3D):
       outputs.set_shape(out_shape)
 
     if self.use_bias:
-      outputs_shape = outputs.shape.as_list()
-      if outputs_shape[0] is None:
-        outputs_shape[0] = -1
-      if self.data_format == 'channels_first':
-        outputs_4d = array_ops.reshape(outputs, [
-            outputs_shape[0], outputs_shape[1],
-            outputs_shape[2] * outputs_shape[3], outputs_shape[4]
-        ])
-      else:
-        outputs_4d = array_ops.reshape(outputs, [
-            outputs_shape[0], outputs_shape[1] * outputs_shape[2],
-            outputs_shape[3], outputs_shape[4]
-        ])
-      outputs_4d = nn.bias_add(
-          outputs_4d,
+      outputs = nn.bias_add(
+          outputs,
           self.bias,
           data_format=conv_utils.convert_data_format(self.data_format, ndim=4))
-      outputs = array_ops.reshape(outputs_4d, outputs_shape)
 
     if self.activation is not None:
       return self.activation(outputs)
diff --git a/tensorflow/python/keras/layers/convolutional_test.py b/tensorflow/python/keras/layers/convolutional_test.py
index 4afddbc8cc..63fb60ebaf 100644
--- a/tensorflow/python/keras/layers/convolutional_test.py
+++ b/tensorflow/python/keras/layers/convolutional_test.py
@@ -336,6 +336,14 @@ class Conv3DTransposeTest(test.TestCase):
       self.assertEqual(layer.kernel.constraint, k_constraint)
       self.assertEqual(layer.bias.constraint, b_constraint)
 
+  def test_conv3dtranspose_dynamic_shape(self):
+    with self.session(use_gpu=True):
+      # Won't raise error here.
+      layer = keras.layers.Conv3DTranspose(3, 3, data_format='channels_last')
+      layer.build((None, None, None, None, 1))
+      layer1 = keras.layers.Conv3DTranspose(3, 3, data_format='channels_first')
+      layer1.build((None, 1, None, None, None))
+
 
 class SeparableConv1DTest(test.TestCase):
 
@@ -557,6 +565,14 @@ class Conv3DTest(test.TestCase):
       self.assertEqual(layer.kernel.constraint, k_constraint)
       self.assertEqual(layer.bias.constraint, b_constraint)
 
+  def test_conv3d_dynamic_shape(self):
+    with self.session(use_gpu=True):
+      # Won't raise error here.
+      layer = keras.layers.Conv3D(3, 3, data_format='channels_last')
+      layer.build((None, None, None, None, 1))
+      layer1 = keras.layers.Conv3D(3, 3, data_format='channels_first')
+      layer1.build((None, 1, None, None, None))
+
 
 class ZeroPaddingTest(test.TestCase):
 
-- 
GitLab


From 65344d0a7ebd21c71b3f3ed7cb091541504b659a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?=
 <facai.yan@gmail.com>
Date: Thu, 1 Nov 2018 09:36:09 +0800
Subject: [PATCH 0006/1095] TST: use testing_utils.layer_test instead

---
 .../python/keras/layers/convolutional_test.py | 28 +++++++++++++------
 1 file changed, 20 insertions(+), 8 deletions(-)

diff --git a/tensorflow/python/keras/layers/convolutional_test.py b/tensorflow/python/keras/layers/convolutional_test.py
index 63fb60ebaf..7d8051e596 100644
--- a/tensorflow/python/keras/layers/convolutional_test.py
+++ b/tensorflow/python/keras/layers/convolutional_test.py
@@ -339,10 +339,16 @@ class Conv3DTransposeTest(test.TestCase):
   def test_conv3dtranspose_dynamic_shape(self):
     with self.session(use_gpu=True):
       # Won't raise error here.
-      layer = keras.layers.Conv3DTranspose(3, 3, data_format='channels_last')
-      layer.build((None, None, None, None, 1))
-      layer1 = keras.layers.Conv3DTranspose(3, 3, data_format='channels_first')
-      layer1.build((None, 1, None, None, None))
+      testing_utils.layer_test(
+          keras.layers.Conv3DTranspose,
+          kwargs={'data_format': 'channels_first',
+                  'filters': 3, 'kernel_size': 3},
+          input_shape=(None, 1, None, None, None))
+      testing_utils.layer_test(
+          keras.layers.Conv3DTranspose,
+          kwargs={'data_format': 'channels_last',
+                  'filters': 3, 'kernel_size': 3},
+          input_shape=(None, None, None, None, 1))
 
 
 class SeparableConv1DTest(test.TestCase):
@@ -568,10 +574,16 @@ class Conv3DTest(test.TestCase):
   def test_conv3d_dynamic_shape(self):
     with self.session(use_gpu=True):
       # Won't raise error here.
-      layer = keras.layers.Conv3D(3, 3, data_format='channels_last')
-      layer.build((None, None, None, None, 1))
-      layer1 = keras.layers.Conv3D(3, 3, data_format='channels_first')
-      layer1.build((None, 1, None, None, None))
+      testing_utils.layer_test(
+          keras.layers.Conv3D,
+          kwargs={'data_format': 'channels_first',
+                  'filters': 3, 'kernel_size': 3},
+          input_shape=(None, 1, None, None, None))
+      testing_utils.layer_test(
+          keras.layers.Conv3D,
+          kwargs={'data_format': 'channels_last',
+                  'filters': 3, 'kernel_size': 3},
+          input_shape=(None, None, None, None, 1))
 
 
 class ZeroPaddingTest(test.TestCase):
-- 
GitLab


From 35f20ca9e794b455777a52bf70b5f7d79ae60455 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?=
 <facai.yan@gmail.com>
Date: Thu, 1 Nov 2018 10:11:14 +0800
Subject: [PATCH 0007/1095] TST: add input_data

---
 .../python/keras/layers/convolutional_test.py      | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/keras/layers/convolutional_test.py b/tensorflow/python/keras/layers/convolutional_test.py
index 7d8051e596..737e12a2bd 100644
--- a/tensorflow/python/keras/layers/convolutional_test.py
+++ b/tensorflow/python/keras/layers/convolutional_test.py
@@ -337,18 +337,21 @@ class Conv3DTransposeTest(test.TestCase):
       self.assertEqual(layer.bias.constraint, b_constraint)
 
   def test_conv3dtranspose_dynamic_shape(self):
+    input_data = np.random.random((1, 3, 3, 3, 3))
     with self.session(use_gpu=True):
       # Won't raise error here.
       testing_utils.layer_test(
           keras.layers.Conv3DTranspose,
           kwargs={'data_format': 'channels_first',
                   'filters': 3, 'kernel_size': 3},
-          input_shape=(None, 1, None, None, None))
+          input_shape=(None, 3, None, None, None),
+          input_data=input_data)
       testing_utils.layer_test(
           keras.layers.Conv3DTranspose,
           kwargs={'data_format': 'channels_last',
                   'filters': 3, 'kernel_size': 3},
-          input_shape=(None, None, None, None, 1))
+          input_shape=(None, None, None, None, 3),
+          input_data=input_data)
 
 
 class SeparableConv1DTest(test.TestCase):
@@ -572,18 +575,21 @@ class Conv3DTest(test.TestCase):
       self.assertEqual(layer.bias.constraint, b_constraint)
 
   def test_conv3d_dynamic_shape(self):
+    input_data = np.random.random((1, 3, 3, 3, 3))
     with self.session(use_gpu=True):
       # Won't raise error here.
       testing_utils.layer_test(
           keras.layers.Conv3D,
           kwargs={'data_format': 'channels_first',
                   'filters': 3, 'kernel_size': 3},
-          input_shape=(None, 1, None, None, None))
+          input_shape=(None, 3, None, None, None),
+          input_data=input_data)
       testing_utils.layer_test(
           keras.layers.Conv3D,
           kwargs={'data_format': 'channels_last',
                   'filters': 3, 'kernel_size': 3},
-          input_shape=(None, None, None, None, 1))
+          input_shape=(None, None, None, None, 3),
+          input_data=input_data)
 
 
 class ZeroPaddingTest(test.TestCase):
-- 
GitLab


From 8de3a6ee6c7b498e32e8c22c6631a3c0a7a4af86 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?=
 <facai.yan@gmail.com>
Date: Thu, 1 Nov 2018 10:35:48 +0800
Subject: [PATCH 0008/1095] TST: data_format=first only run for gpu

---
 .../python/keras/layers/convolutional_test.py | 30 ++++++++++---------
 1 file changed, 16 insertions(+), 14 deletions(-)

diff --git a/tensorflow/python/keras/layers/convolutional_test.py b/tensorflow/python/keras/layers/convolutional_test.py
index 737e12a2bd..ecee080937 100644
--- a/tensorflow/python/keras/layers/convolutional_test.py
+++ b/tensorflow/python/keras/layers/convolutional_test.py
@@ -337,21 +337,22 @@ class Conv3DTransposeTest(test.TestCase):
       self.assertEqual(layer.bias.constraint, b_constraint)
 
   def test_conv3dtranspose_dynamic_shape(self):
-    input_data = np.random.random((1, 3, 3, 3, 3))
+    input_data = np.random.random((1, 3, 3, 3, 3)).astype(np.float32)
     with self.session(use_gpu=True):
       # Won't raise error here.
-      testing_utils.layer_test(
-          keras.layers.Conv3DTranspose,
-          kwargs={'data_format': 'channels_first',
-                  'filters': 3, 'kernel_size': 3},
-          input_shape=(None, 3, None, None, None),
-          input_data=input_data)
       testing_utils.layer_test(
           keras.layers.Conv3DTranspose,
           kwargs={'data_format': 'channels_last',
                   'filters': 3, 'kernel_size': 3},
           input_shape=(None, None, None, None, 3),
           input_data=input_data)
+      if test.is_gpu_available(cuda_only=True):
+        testing_utils.layer_test(
+            keras.layers.Conv3DTranspose,
+            kwargs={'data_format': 'channels_first',
+                    'filters': 3, 'kernel_size': 3},
+            input_shape=(None, 3, None, None, None),
+            input_data=input_data)
 
 
 class SeparableConv1DTest(test.TestCase):
@@ -575,21 +576,22 @@ class Conv3DTest(test.TestCase):
       self.assertEqual(layer.bias.constraint, b_constraint)
 
   def test_conv3d_dynamic_shape(self):
-    input_data = np.random.random((1, 3, 3, 3, 3))
+    input_data = np.random.random((1, 3, 3, 3, 3)).astype(np.float32)
     with self.session(use_gpu=True):
       # Won't raise error here.
-      testing_utils.layer_test(
-          keras.layers.Conv3D,
-          kwargs={'data_format': 'channels_first',
-                  'filters': 3, 'kernel_size': 3},
-          input_shape=(None, 3, None, None, None),
-          input_data=input_data)
       testing_utils.layer_test(
           keras.layers.Conv3D,
           kwargs={'data_format': 'channels_last',
                   'filters': 3, 'kernel_size': 3},
           input_shape=(None, None, None, None, 3),
           input_data=input_data)
+      if test.is_gpu_available(cuda_only=True):
+        testing_utils.layer_test(
+            keras.layers.Conv3D,
+            kwargs={'data_format': 'channels_first',
+                    'filters': 3, 'kernel_size': 3},
+            input_shape=(None, 3, None, None, None),
+            input_data=input_data)
 
 
 class ZeroPaddingTest(test.TestCase):
-- 
GitLab


From 42321707242771cf28deb1d577dfdd6a17e9eae9 Mon Sep 17 00:00:00 2001
From: Guozhong Zhuang <guozhong.zhuang@intel.com>
Date: Mon, 12 Nov 2018 15:24:22 -0800
Subject: [PATCH 0009/1095] fix issues related to clang format check

---
 .../core/kernels/mkl_fused_batch_norm_op.cc     | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc b/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc
index ff46e75a36..685db657e2 100644
--- a/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc
+++ b/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc
@@ -13,15 +13,14 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 #ifdef INTEL_MKL
-
-#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "mkldnn.hpp"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/util/mkl_util.h"
 #include "tensorflow/core/util/tensor_format.h"
-#include "mkldnn.hpp"
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 
 using mkldnn::batch_normalization_backward;
 using mkldnn::batch_normalization_forward;
@@ -705,9 +704,9 @@ class MklFusedBatchNormOp : public OpKernel {
         std::memcpy(batch_variance_data, variance_data, depth_ * sizeof(T));
       }
     } catch (mkldnn::error& e) {
-      string error_msg = "Status: " + std::to_string(e.status) +
-                         ", message: " + string(e.message) + ", in file " +
-                         string(__FILE__) + ":" + std::to_string(__LINE__);
+      string error_msg = "Status: " + std::to_string(e.status) + ", message: " +
+                         string(e.message) + ", in file " + string(__FILE__) +
+                         ":" + std::to_string(__LINE__);
       OP_REQUIRES_OK(
           context,
           errors::Aborted("Operation received an exception:", error_msg));
@@ -1029,9 +1028,9 @@ class MklFusedBatchNormGradOp : public OpKernel {
                   reinterpret_cast<char*>(diff_weights_data + depth_),
                   depth_ * sizeof(T));
     } catch (mkldnn::error& e) {
-      string error_msg = "Status: " + std::to_string(e.status) +
-                         ", message: " + string(e.message) + ", in file " +
-                         string(__FILE__) + ":" + std::to_string(__LINE__);
+      string error_msg = "Status: " + std::to_string(e.status) + ", message: " +
+                         string(e.message) + ", in file " + string(__FILE__) +
+                         ":" + std::to_string(__LINE__);
       OP_REQUIRES_OK(
           context,
           errors::Aborted("Operation received an exception:", error_msg));
-- 
GitLab


From d45cd461242e3a27e505a67c17d5ddb7f4a84641 Mon Sep 17 00:00:00 2001
From: Yves-Noel Weweler <y.weweler@gmail.com>
Date: Fri, 30 Nov 2018 14:46:12 +0100
Subject: [PATCH 0010/1095]  Add drop_remainder argument to
 bucket_by_sequence_length

`tf.data.experimental.bucket_by_sequence_length` does not allow to drop
the last batch in case it has fewer than `batch_size` elements.
This patch does implement `drop_remainder` for `bucket_by_sequence_length`
to enable thhis behaviour.

`drop_remainder` is optinal and set to `False` by default to maintain
compatibility.
---
 tensorflow/python/data/experimental/ops/grouping.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/data/experimental/ops/grouping.py b/tensorflow/python/data/experimental/ops/grouping.py
index db10ea3b7f..71e4b3391f 100644
--- a/tensorflow/python/data/experimental/ops/grouping.py
+++ b/tensorflow/python/data/experimental/ops/grouping.py
@@ -130,7 +130,8 @@ def bucket_by_sequence_length(element_length_func,
                               padded_shapes=None,
                               padding_values=None,
                               pad_to_bucket_boundary=False,
-                              no_padding=False):
+                              no_padding=False,
+                              drop_remainder=False):
   """A transformation that buckets elements in a `Dataset` by length.
 
   Elements of the `Dataset` are grouped together by length and then are padded
@@ -160,6 +161,10 @@ def bucket_by_sequence_length(element_length_func,
       any elements with length longer than `max(bucket_boundaries)`.
     no_padding: `bool`, indicates whether to pad the batch features (features
       need to be either of type `tf.SparseTensor` or of same shape).
+    drop_remainder: (Optional.) A `tf.bool` scalar `tf.Tensor`, representing
+      whether the last batch should be dropped in the case its has fewer than
+      batch_size` elements; the default behavior is not to drop the smaller
+      batch.
 
   Returns:
     A `Dataset` transformation function, which can be passed to
@@ -209,7 +214,7 @@ def bucket_by_sequence_length(element_length_func,
       """Batch elements in dataset."""
       batch_size = window_size_fn(bucket_id)
       if no_padding:
-        return grouped_dataset.batch(batch_size)
+        return grouped_dataset.batch(batch_size, drop_remainder=drop_remainder)
       none_filler = None
       if pad_to_bucket_boundary:
         err_msg = ("When pad_to_bucket_boundary=True, elements must have "
@@ -227,7 +232,8 @@ def bucket_by_sequence_length(element_length_func,
       shapes = make_padded_shapes(
           padded_shapes or grouped_dataset.output_shapes,
           none_filler=none_filler)
-      return grouped_dataset.padded_batch(batch_size, shapes, padding_values)
+      return grouped_dataset.padded_batch(batch_size, shapes, padding_values,
+                                          drop_remainder=drop_remainder)
 
     def _apply_fn(dataset):
       return dataset.apply(
-- 
GitLab


From 916c5238d864e04861a58b3f77b9d29cca03b0e0 Mon Sep 17 00:00:00 2001
From: Yves-Noel Weweler <y.weweler@gmail.com>
Date: Sun, 2 Dec 2018 18:27:55 +0100
Subject: [PATCH 0011/1095] Add drop_reminder support to the test of bucketing
 of sparse tensors

---
 .../bucket_by_sequence_length_test.py         | 31 ++++++++++++-------
 1 file changed, 19 insertions(+), 12 deletions(-)

diff --git a/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py b/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
index af20e50fb9..fb2d1cf63c 100644
--- a/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
@@ -274,11 +274,16 @@ class BucketBySequenceLengthTest(test_base.DatasetTestBase):
       dataset = dataset.map(_to_sparse_tensor)
       return dataset
 
-    def _compute_expected_batches():
+    def _compute_expected_batches(drop_remainder):
       """Computes expected batch outputs and stores in a set."""
       all_expected_sparse_tensors = set()
       for bucket_start_len in range(min_len, max_len, bucket_size):
-        for batch_offset in range(0, bucket_size, batch_size):
+        if drop_remainder:
+          batch_offsets = [0]
+        else:
+          batch_offsets = range(0, bucket_size, batch_size)
+
+        for batch_offset in batch_offsets:
           batch_start_len = bucket_start_len + batch_offset
           batch_end_len = min(batch_start_len + batch_size,
                               bucket_start_len + bucket_size)
@@ -306,16 +311,18 @@ class BucketBySequenceLengthTest(test_base.DatasetTestBase):
             all_sparse_tensors.add(sprs_tensor)
       return all_sparse_tensors
 
-    dataset = _build_dataset()
-    boundaries = range(min_len + bucket_size + 1, max_len, bucket_size)
-    dataset = dataset.apply(grouping.bucket_by_sequence_length(
-        _element_length_fn,
-        boundaries,
-        [batch_size] * (len(boundaries) + 1),
-        no_padding=True))
-    batches = _compute_batches(dataset)
-    expected_batches = _compute_expected_batches()
-    self.assertEqual(batches, expected_batches)
+    for drop_remainder in (True, False):
+      dataset = _build_dataset()
+      boundaries = range(min_len + bucket_size + 1, max_len, bucket_size)
+      dataset = dataset.apply(grouping.bucket_by_sequence_length(
+          _element_length_fn,
+          boundaries,
+          [batch_size] * (len(boundaries) + 1),
+          no_padding=True,
+          drop_remainder=drop_remainder))
+      batches = _compute_batches(dataset)
+      expected_batches = _compute_expected_batches()
+      self.assertEqual(batches, expected_batches)
 
 
 if __name__ == "__main__":
-- 
GitLab


From d372f19f132e8f037390a9a2a6fde7bafa2620b9 Mon Sep 17 00:00:00 2001
From: Yves-Noel Weweler <y.weweler@gmail.com>
Date: Sun, 2 Dec 2018 18:29:03 +0100
Subject: [PATCH 0012/1095] Add a drop_reminder version of the testBucket test

Squeezing both cases into a single test would make it way to
complicated. Therefore, I created a separate test.
---
 .../bucket_by_sequence_length_test.py         | 111 ++++++++++++++++++
 1 file changed, 111 insertions(+)

diff --git a/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py b/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
index fb2d1cf63c..7b2e922d55 100644
--- a/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
@@ -71,6 +71,117 @@ def _get_record_shape(sparse):
 
 class BucketBySequenceLengthTest(test_base.DatasetTestBase):
 
+  def testBucketDropReminder(self):
+
+    boundaries = [10, 20, 30]
+    batch_sizes = [10, 8, 4, 2]
+    lengths = [8, 13, 25, 35]
+
+    n_bucket_elements = [28, 7, 6, 5]
+    n_expected_batches = 5
+
+    # Expected sequence lengths of the individual batches.
+    expected_lengths = []
+
+    # Expected sum of all batches with an equal sequence length.
+    # <seq-length>: <expected-total-sum>
+    expected_sums = dict()
+
+    # Expected batch sizes of batches depending on the sequence length.
+    # <seq-length>: [batch1_size, ..., batchN_size]
+    expected_batch_sizes = dict()
+
+    for length, batch_size, bucket_elements in zip(lengths, batch_sizes, n_bucket_elements):
+      # Calculate the expected sum across all batches of a specific sequence length.
+      expected_sums[length] = (bucket_elements - bucket_elements % batch_size) * length
+      # Calculate the expected occurrence of individual batch sizes.
+      expected_batch_sizes[length] = [batch_size] * (bucket_elements // batch_size)
+      # Calculate the expected occurence of individual sequence lengths.
+      expected_lengths.extend([length] * (bucket_elements // batch_size))
+
+    def build_dataset(sparse):
+      def _generator():
+        # Produce 1 batch for each bucket
+        elements = []
+        for bucket_elements, length in zip(n_bucket_elements, lengths):
+          # Using only full sequences (opposed to the strategy employed in `testBucket`) makes
+          # checking the sum a lot easier.
+          record_len = length
+          for _ in range(bucket_elements):
+            elements.append([1] * record_len)
+        random.shuffle(elements)
+        for el in elements:
+          yield (_format_record(el, sparse),)
+      dataset = dataset_ops.Dataset.from_generator(
+          _generator,
+          (_get_record_type(sparse),),
+          (_get_record_shape(sparse),))
+      if sparse:
+        dataset = dataset.map(lambda x: (_to_sparse_tensor(x),))
+      return dataset
+
+    def _test_bucket_by_padding(no_padding):
+      dataset = build_dataset(sparse=no_padding)
+      dataset = dataset.apply(
+          grouping.bucket_by_sequence_length(
+              _element_length_fn,
+              boundaries,
+              batch_sizes,
+              no_padding=no_padding,
+              drop_remainder=True))
+      batch, = dataset.make_one_shot_iterator().get_next()
+
+      with self.cached_session() as sess:
+        batches = []
+        for _ in range(n_expected_batches):
+          batches.append(self.evaluate(batch))
+        with self.assertRaises(errors.OutOfRangeError):
+          self.evaluate(batch)
+
+      generated_lengths = []
+
+      # <seq-length>: <total-sum>
+      generated_sums = dict()
+
+      # <seq-length>: [<batch_size>, ...]
+      generated_batch_sizes = dict()
+
+      for length, batch_size, bucket_elements in zip(lengths, batch_sizes, n_bucket_elements):
+          # Initialize the sum across all batches.
+          expected_sums[length] = 0
+          # Initialize the individual batch sizes.
+          expected_batch_sizes[length] = []
+
+      for batch in batches:
+        shape = batch.dense_shape if no_padding else batch.shape
+        length = shape[1]
+        generated_lengths.append(length)
+
+        batch_size = shape[0]
+        generated_batch_sizes[length].append(batch_size)
+
+        batch_sum = batch.values.sum() if no_padding else batch.sum()
+        generated_sums[length] += batch_sum
+
+      for l in lengths:
+        # Make sure the sum of the batch contents is correct for the individual sequence lengths.
+        self.assertEqual(generated_sums[l], expected_sums[l],
+                         'Tensor sums did not match! expected: {}, generated: {}'
+                         .format(expected_sums, generated_sums))
+
+        # Make sure the individual batch sizes are generated as expected.
+        self.assertEqual(sorted(generated_batch_sizes[l]), sorted(expected_batch_sizes[l]),
+                         'Batch-sizes did not match! expected: {}, generated: {}'
+                         .format(sorted(expected_batch_sizes[l]), sorted(generated_batch_sizes[l])))
+
+      # Make sure the generated sequence lengths appear as often as expected.
+      self.assertEqual(sorted(generated_lengths), sorted(expected_lengths),
+                       'The generated sequence lengths did not match! expected: {}, generated: {}'
+                       .format(sorted(expected_lengths), sorted(generated_lengths)))
+
+    for no_padding in (True, False):
+      _test_bucket_by_padding(no_padding)
+
   def testBucket(self):
 
     boundaries = [10, 20, 30]
-- 
GitLab


From 68426789549cb9d2edc8726fc5edabf4f221bd9b Mon Sep 17 00:00:00 2001
From: Yves-Noel Weweler <y.weweler@gmail.com>
Date: Sun, 2 Dec 2018 18:40:12 +0100
Subject: [PATCH 0013/1095] Fix pylint warnings

* Missing argument drop_remainder in the testBucketSparse test.
* Insert line breaks
---
 .../bucket_by_sequence_length_test.py         | 45 ++++++++++++-------
 1 file changed, 29 insertions(+), 16 deletions(-)

diff --git a/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py b/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
index 7b2e922d55..5b96101b7c 100644
--- a/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
@@ -91,11 +91,15 @@ class BucketBySequenceLengthTest(test_base.DatasetTestBase):
     # <seq-length>: [batch1_size, ..., batchN_size]
     expected_batch_sizes = dict()
 
-    for length, batch_size, bucket_elements in zip(lengths, batch_sizes, n_bucket_elements):
+    for length, batch_size, bucket_elements in zip(lengths,
+                                                   batch_sizes,
+                                                   n_bucket_elements):
       # Calculate the expected sum across all batches of a specific sequence length.
-      expected_sums[length] = (bucket_elements - bucket_elements % batch_size) * length
+      expected_sums[length] = \
+          (bucket_elements - bucket_elements % batch_size) * length
       # Calculate the expected occurrence of individual batch sizes.
-      expected_batch_sizes[length] = [batch_size] * (bucket_elements // batch_size)
+      expected_batch_sizes[length] = \
+          [batch_size] * (bucket_elements // batch_size)
       # Calculate the expected occurence of individual sequence lengths.
       expected_lengths.extend([length] * (bucket_elements // batch_size))
 
@@ -146,11 +150,13 @@ class BucketBySequenceLengthTest(test_base.DatasetTestBase):
       # <seq-length>: [<batch_size>, ...]
       generated_batch_sizes = dict()
 
-      for length, batch_size, bucket_elements in zip(lengths, batch_sizes, n_bucket_elements):
-          # Initialize the sum across all batches.
-          expected_sums[length] = 0
-          # Initialize the individual batch sizes.
-          expected_batch_sizes[length] = []
+      for length, batch_size, bucket_elements in zip(lengths,
+                                                     batch_sizes,
+                                                     n_bucket_elements):
+        # Initialize the sum across all batches.
+        expected_sums[length] = 0
+        # Initialize the individual batch sizes.
+        expected_batch_sizes[length] = []
 
       for batch in batches:
         shape = batch.dense_shape if no_padding else batch.shape
@@ -165,19 +171,26 @@ class BucketBySequenceLengthTest(test_base.DatasetTestBase):
 
       for l in lengths:
         # Make sure the sum of the batch contents is correct for the individual sequence lengths.
-        self.assertEqual(generated_sums[l], expected_sums[l],
-                         'Tensor sums did not match! expected: {}, generated: {}'
+        self.assertEqual(generated_sums[l],
+                         expected_sums[l],
+                         'Tensor sums did not match! '
+                         'expected: {}, generated: {}'
                          .format(expected_sums, generated_sums))
 
         # Make sure the individual batch sizes are generated as expected.
-        self.assertEqual(sorted(generated_batch_sizes[l]), sorted(expected_batch_sizes[l]),
-                         'Batch-sizes did not match! expected: {}, generated: {}'
-                         .format(sorted(expected_batch_sizes[l]), sorted(generated_batch_sizes[l])))
+        self.assertEqual(sorted(generated_batch_sizes[l]),
+                         sorted(expected_batch_sizes[l]),
+                         'Batch-sizes did not match! '
+                         'expected: {}, generated: {}'
+                         .format(sorted(expected_batch_sizes[l]),
+                                 sorted(generated_batch_sizes[l])))
 
       # Make sure the generated sequence lengths appear as often as expected.
       self.assertEqual(sorted(generated_lengths), sorted(expected_lengths),
-                       'The generated sequence lengths did not match! expected: {}, generated: {}'
-                       .format(sorted(expected_lengths), sorted(generated_lengths)))
+                       'The generated sequence lengths did not match! '
+                       'expected: {}, generated: {}'
+                       .format(sorted(expected_lengths),
+                               sorted(generated_lengths)))
 
     for no_padding in (True, False):
       _test_bucket_by_padding(no_padding)
@@ -432,7 +445,7 @@ class BucketBySequenceLengthTest(test_base.DatasetTestBase):
           no_padding=True,
           drop_remainder=drop_remainder))
       batches = _compute_batches(dataset)
-      expected_batches = _compute_expected_batches()
+      expected_batches = _compute_expected_batches(drop_remainder)
       self.assertEqual(batches, expected_batches)
 
 
-- 
GitLab


From 4ddd2ab07fdfea81a53d110fee19bb88c41251a0 Mon Sep 17 00:00:00 2001
From: Yves-Noel Weweler <y.weweler@gmail.com>
Date: Mon, 3 Dec 2018 13:58:58 +0100
Subject: [PATCH 0014/1095] Fix copy paste bug

Pasted the wrong variable names from the docker test image.
---
 .../kernel_tests/bucket_by_sequence_length_test.py            | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py b/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
index 5b96101b7c..fab79619a0 100644
--- a/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
@@ -154,9 +154,9 @@ class BucketBySequenceLengthTest(test_base.DatasetTestBase):
                                                      batch_sizes,
                                                      n_bucket_elements):
         # Initialize the sum across all batches.
-        expected_sums[length] = 0
+        generated_sums[length] = 0
         # Initialize the individual batch sizes.
-        expected_batch_sizes[length] = []
+        generated_batch_sizes[length] = []
 
       for batch in batches:
         shape = batch.dense_shape if no_padding else batch.shape
-- 
GitLab


From 562b078b836b215b761fc91a177937cfcbdd0ea0 Mon Sep 17 00:00:00 2001
From: Nayana-ibm <nayana.thorat@gmail.com>
Date: Thu, 6 Dec 2018 11:13:12 -0500
Subject: [PATCH 0015/1095] resolve ImportError: cannot import name cloud on
 s390x - skip import clound

---
 tensorflow/contrib/__init__.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/__init__.py b/tensorflow/contrib/__init__.py
index 4f1a2a5693..a23e331f4f 100644
--- a/tensorflow/contrib/__init__.py
+++ b/tensorflow/contrib/__init__.py
@@ -20,13 +20,14 @@ from __future__ import division
 from __future__ import print_function
 
 import os
+import platform
 
 # Add projects here, they will show up under tf.contrib.
 from tensorflow.contrib import autograph
 from tensorflow.contrib import batching
 from tensorflow.contrib import bayesflow
 from tensorflow.contrib import checkpoint
-if os.name != "nt":
+if os.name != "nt" and platform.machine() != "s390x":
   from tensorflow.contrib import cloud
 from tensorflow.contrib import cluster_resolver
 from tensorflow.contrib import coder
-- 
GitLab


From aff52125a0d172f4c3ae8ea7c170ca0f6d97e97d Mon Sep 17 00:00:00 2001
From: Andy Craze <accraze@gmail.com>
Date: Fri, 7 Dec 2018 20:37:37 -0800
Subject: [PATCH 0016/1095] Update tables_initializer to link to guide

Resolves  #20629
---
 tensorflow/python/ops/lookup_ops.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/python/ops/lookup_ops.py b/tensorflow/python/ops/lookup_ops.py
index 758cb8041d..9302696a45 100644
--- a/tensorflow/python/ops/lookup_ops.py
+++ b/tensorflow/python/ops/lookup_ops.py
@@ -64,6 +64,8 @@ def initialize_all_tables(name="init_all_tables"):
 @tf_export(v1=["initializers.tables_initializer", "tables_initializer"])
 def tables_initializer(name="init_all_tables"):
   """Returns an Op that initializes all tables of the default graph.
+  See the [Low Level Intro](https://www.tensorflow.org/guide/low_level_intro#feature_columns)
+  guide, for an example of usage.
 
   Args:
     name: Optional name for the initialization op.
-- 
GitLab


From 035955852708fba07565ead298cd54ad64ab1a55 Mon Sep 17 00:00:00 2001
From: Jacky Ko <jackkykokoko@gmail.com>
Date: Sun, 9 Dec 2018 18:33:50 +0800
Subject: [PATCH 0017/1095] static lib name change

---
 tensorflow/contrib/cmake/external/abseil_cpp.cmake | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/cmake/external/abseil_cpp.cmake b/tensorflow/contrib/cmake/external/abseil_cpp.cmake
index 8b76f37858..539c5cbb76 100644
--- a/tensorflow/contrib/cmake/external/abseil_cpp.cmake
+++ b/tensorflow/contrib/cmake/external/abseil_cpp.cmake
@@ -48,7 +48,7 @@ else (systemlib_ABSEIL_CPP)
       set(abseil_cpp_STATIC_LIBRARIES
           ${abseil_cpp_BUILD}/absl/base/Release/absl_base.lib
           ${abseil_cpp_BUILD}/absl/base/Release/absl_dynamic_annotations.lib
-          ${abseil_cpp_BUILD}/absl/base/Release/absl_internal_malloc_internal.lib
+          ${abseil_cpp_BUILD}/absl/base/Release/absl_malloc_internal.lib
           ${abseil_cpp_BUILD}/absl/strings/Release/absl_strings.lib
           ${abseil_cpp_BUILD}/absl/strings/Release/str_format_internal.lib
           ${abseil_cpp_BUILD}/absl/types/Release/absl_bad_optional_access.lib)
@@ -94,6 +94,8 @@ else (systemlib_ABSEIL_CPP)
   )
 
   include_directories(${abseil_cpp_INCLUDE_DIR})
+  message(STATUS ${abseil_cpp_INCLUDE_DIR})
+
   list(APPEND tensorflow_EXTERNAL_LIBRARIES ${abseil_cpp_STATIC_LIBRARIES})
 
   list(APPEND tensorflow_EXTERNAL_DEPENDENCIES abseil_cpp_build)
-- 
GitLab


From 5a4871af05392c2e5744515a4c67bc6f0f420943 Mon Sep 17 00:00:00 2001
From: fo40225 <fo40225@hotmail.com>
Date: Sat, 1 Sep 2018 00:58:14 +0800
Subject: [PATCH 0018/1095] fix AttributeError: 'module' object has no
 attribute '???'on windows python 2.7

---
 tensorflow/api_template_v1.__init__.py      | 3 ++-
 tensorflow/contrib/cmake/python_modules.txt | 3 +++
 tensorflow/contrib/cmake/tf_python.cmake    | 1 +
 3 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/tensorflow/api_template_v1.__init__.py b/tensorflow/api_template_v1.__init__.py
index 65bdb6cb1b..b9b21bad50 100644
--- a/tensorflow/api_template_v1.__init__.py
+++ b/tensorflow/api_template_v1.__init__.py
@@ -40,7 +40,8 @@ if '__all__' in vars():
   vars()['__all__'].append('contrib')
 
 from tensorflow.python.platform import flags  # pylint: disable=g-import-not-at-top
-app.flags = flags  # pylint: disable=undefined-variable
+from tensorflow.python.platform import app  # pylint: disable=g-import-not-at-top
+app.flags = flags
 
 # Make sure directory containing top level submodules is in
 # the __path__ so that "from tensorflow.foo import bar" works.
diff --git a/tensorflow/contrib/cmake/python_modules.txt b/tensorflow/contrib/cmake/python_modules.txt
index 96160568fa..21ae9a08a6 100644
--- a/tensorflow/contrib/cmake/python_modules.txt
+++ b/tensorflow/contrib/cmake/python_modules.txt
@@ -1,6 +1,9 @@
 # python_sanity_test.py will complain about invalid or missing entries
 # problematic entries can be commented for temporary whitelisting
 tensorflow
+tensorflow/compiler
+tensorflow/compiler/xla
+tensorflow/compiler/xla/service
 tensorflow/core
 tensorflow/core/example
 tensorflow/core/framework
diff --git a/tensorflow/contrib/cmake/tf_python.cmake b/tensorflow/contrib/cmake/tf_python.cmake
index 8faccf8d55..1fe8795ddf 100755
--- a/tensorflow/contrib/cmake/tf_python.cmake
+++ b/tensorflow/contrib/cmake/tf_python.cmake
@@ -802,6 +802,7 @@ add_custom_command(
       # tensorflow/__init__.py depends on files generated in this step. So, remove it while
       # this step is running since the files aren't there yet.
       COMMAND ${CMAKE_COMMAND} -E remove -f ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/__init__.py
+      COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/__init__.py
 
       # Run create_python_api.py to generate API init files.
       COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR}/tf_python "${PY_RUNTIME_ENV}" ${PYTHON_EXECUTABLE}
-- 
GitLab


From cc69b82bbcc68c21db839cca4f6fec043f6005aa Mon Sep 17 00:00:00 2001
From: Jacky Ko <jackkykokoko@gmail.com>
Date: Sun, 9 Dec 2018 19:08:32 +0800
Subject: [PATCH 0019/1095] abseil use master branch

---
 tensorflow/contrib/cmake/external/abseil_cpp.cmake | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/tensorflow/contrib/cmake/external/abseil_cpp.cmake b/tensorflow/contrib/cmake/external/abseil_cpp.cmake
index 539c5cbb76..f64b60fd5e 100644
--- a/tensorflow/contrib/cmake/external/abseil_cpp.cmake
+++ b/tensorflow/contrib/cmake/external/abseil_cpp.cmake
@@ -39,8 +39,8 @@ else (systemlib_ABSEIL_CPP)
   include (ExternalProject)
 
   set(abseil_cpp_INCLUDE_DIR ${CMAKE_BINARY_DIR}/abseil_cpp/src/abseil_cpp_build)
-  set(abseil_cpp_URL https://github.com/abseil/abseil-cpp/archive/e01d95528ea2137a4a27a88d1f57c6cb260aafed.tar.gz)
-  set(abseil_cpp_HASH SHA256=84043ed402d2a2a6ba4cdddb7e85118b1158fd81fe4ac3a14adc343d054c1e2e)
+  set(abseil_cpp_URL https://github.com/abseil/abseil-cpp.git)
+  set(abseil_cpp_TAG master)
   set(abseil_cpp_BUILD ${CMAKE_BINARY_DIR}/abseil_cpp/src/abseil_cpp_build)
 
   if(WIN32)
@@ -48,7 +48,7 @@ else (systemlib_ABSEIL_CPP)
       set(abseil_cpp_STATIC_LIBRARIES
           ${abseil_cpp_BUILD}/absl/base/Release/absl_base.lib
           ${abseil_cpp_BUILD}/absl/base/Release/absl_dynamic_annotations.lib
-          ${abseil_cpp_BUILD}/absl/base/Release/absl_malloc_internal.lib
+          ${abseil_cpp_BUILD}/absl/base/Release/absl_internal_malloc_internal.lib
           ${abseil_cpp_BUILD}/absl/strings/Release/absl_strings.lib
           ${abseil_cpp_BUILD}/absl/strings/Release/str_format_internal.lib
           ${abseil_cpp_BUILD}/absl/types/Release/absl_bad_optional_access.lib)
@@ -79,8 +79,7 @@ else (systemlib_ABSEIL_CPP)
 
   ExternalProject_Add(abseil_cpp_build
       PREFIX abseil_cpp
-      URL ${abseil_cpp_URL}
-      URL_HASH ${abseil_cpp_HASH}
+      GIT_REPOSITORY ${abseil_cpp_URL}
       DOWNLOAD_DIR "${DOWNLOAD_LOCATION}"
       BUILD_IN_SOURCE 1
       BUILD_BYPRODUCTS ${abseil_cpp_STATIC_LIBRARIES}
-- 
GitLab


From 956b77a2a8f8db7a57b818bc0c06669fd395d56d Mon Sep 17 00:00:00 2001
From: Jacky Ko <jackkykokoko@gmail.com>
Date: Sun, 9 Dec 2018 19:31:05 +0800
Subject: [PATCH 0020/1095] abseil lib linkage update

---
 tensorflow/contrib/cmake/external/abseil_cpp.cmake | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/contrib/cmake/external/abseil_cpp.cmake b/tensorflow/contrib/cmake/external/abseil_cpp.cmake
index f64b60fd5e..eefa7d3f03 100644
--- a/tensorflow/contrib/cmake/external/abseil_cpp.cmake
+++ b/tensorflow/contrib/cmake/external/abseil_cpp.cmake
@@ -49,6 +49,8 @@ else (systemlib_ABSEIL_CPP)
           ${abseil_cpp_BUILD}/absl/base/Release/absl_base.lib
           ${abseil_cpp_BUILD}/absl/base/Release/absl_dynamic_annotations.lib
           ${abseil_cpp_BUILD}/absl/base/Release/absl_internal_malloc_internal.lib
+          ${abseil_cpp_BUILD}/absl/base/Release/absl_internal_throw_delegate.lib
+          ${abseil_cpp_BUILD}/absl/numeric/Release/absl_int128.lib
           ${abseil_cpp_BUILD}/absl/strings/Release/absl_strings.lib
           ${abseil_cpp_BUILD}/absl/strings/Release/str_format_internal.lib
           ${abseil_cpp_BUILD}/absl/types/Release/absl_bad_optional_access.lib)
-- 
GitLab


From 7578e120de2a3a5282ced8d41881f19363f83466 Mon Sep 17 00:00:00 2001
From: Dan Jarvis <daj@users.noreply.github.com>
Date: Thu, 23 Nov 2017 13:06:02 -0500
Subject: [PATCH 0021/1095] Fix crash on closing the app when classifier failed
 to initialize

When testing on an API 21 emulator, the classifier fails to initialize.
`E/TfLiteCameraDemo: Failed to initialize an image classifier.`

In this situation, the app crashes when pressing Back to exit.  Here's the cause:
```
java.lang.NullPointerException: Attempt to invoke virtual method 'void com.example.android.tflitecamerademo.ImageClassifier.close()' on a null object reference
                                                                                        at com.example.android.tflitecamerademo.Camera2BasicFragment.onDestroy(Camera2BasicFragment.java:331)
                                                                                        at android.app.Fragment.performDestroy(Fragment.java:2266)
```
The fix is to check for null before calling `.close()`.

I'll investigate why the classifier is failing to initialize separately. :-)
---
 .../android/tflitecamerademo/Camera2BasicFragment.java        | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java b/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java
index 165d335101..a7b3440536 100644
--- a/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java
+++ b/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java
@@ -476,7 +476,9 @@ public class Camera2BasicFragment extends Fragment
 
   @Override
   public void onDestroy() {
-    classifier.close();
+    if (classifier != null) {
+      classifier.close();
+    }
     super.onDestroy();
   }
 
-- 
GitLab


From 16380e05087b41c39547e3f05e1ce85cea44efb2 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Tue, 11 Dec 2018 00:40:26 +0000
Subject: [PATCH 0022/1095] Fix warning caused by to_int32

While running tf.keras I noticed the following warning:
```
WARNING:tensorflow:From /usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/math_ops.py:3064: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.cast instead.
```

This fix fixes the warning caused by deprecated to_int32.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/ops/math_ops.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py
index e2b634ee8f..1d85fb0193 100644
--- a/tensorflow/python/ops/math_ops.py
+++ b/tensorflow/python/ops/math_ops.py
@@ -3061,8 +3061,8 @@ def reduced_shape(input_shape, axes):
     input_shape[axes] = 1
     return input_shape
 
-  input_shape = to_int32(input_shape)  # [2, 3, 5, 7]
-  axes = to_int32(axes)  # [1, 2]
+  input_shape = cast(input_shape, dtypes.int32)  # [2, 3, 5, 7]
+  axes = cast(axes, dtypes.int32)  # [1, 2]
 
   input_rank = array_ops.size(input_shape)  # 4
   axes = (axes + input_rank) % input_rank
-- 
GitLab


From 16f454f95e9d9823145d6b00a7e007afd0ea569b Mon Sep 17 00:00:00 2001
From: Luke Han <sungyuk@gmail.com>
Date: Tue, 11 Dec 2018 11:03:05 +0900
Subject: [PATCH 0023/1095] fix typo in scatter_nd_add docstring

---
 tensorflow/python/ops/state_ops.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/ops/state_ops.py b/tensorflow/python/ops/state_ops.py
index 3ac69c1c20..25b31c698e 100644
--- a/tensorflow/python/ops/state_ops.py
+++ b/tensorflow/python/ops/state_ops.py
@@ -462,9 +462,8 @@ def scatter_nd_add(ref, indices, updates, use_locking=False, name=None):
     updates: A `Tensor`. Must have the same type as `ref`.
       A tensor of updated values to add to ref.
     use_locking: An optional `bool`. Defaults to `False`.
-      An optional bool. Defaults to True. If True, the assignment will
-      be protected by a lock; otherwise the behavior is undefined,
-      but may exhibit less contention.
+      If True, the assignment will be protected by a lock;
+      otherwise the behavior is undefined, but may exhibit less contention.
     name: A name for the operation (optional).
 
   Returns:
-- 
GitLab


From f0f7ed323983ab2cb157bf782950b6e2238b9f6b Mon Sep 17 00:00:00 2001
From: Jacky Ko <jackkykokoko@gmail.com>
Date: Tue, 11 Dec 2018 20:35:50 +0800
Subject: [PATCH 0024/1095] add abseil time library linking

---
 tensorflow/contrib/cmake/external/abseil_cpp.cmake | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/cmake/external/abseil_cpp.cmake b/tensorflow/contrib/cmake/external/abseil_cpp.cmake
index b85fd48f0f..6c6a5df7f7 100644
--- a/tensorflow/contrib/cmake/external/abseil_cpp.cmake
+++ b/tensorflow/contrib/cmake/external/abseil_cpp.cmake
@@ -31,8 +31,8 @@ if (systemlib_ABSEIL_CPP)
   message(STATUS "  abseil_cpp includes: ${ABSEIL_CPP_INCLUDE_DIR}")
   message(STATUS "  abseil_cpp libraries: ${ABSEIL_CPP_LIBRARIES}")
 
-  add_custom_target(abseil_cpp)
-  list(APPEND tensorflow_EXTERNAL_DEPENDENCIES abseil_cpp)
+  add_custom_target(abseil_cpp_build)
+  list(APPEND tensorflow_EXTERNAL_DEPENDENCIES abseil_cpp_build)
 
 else (systemlib_ABSEIL_CPP)
 
@@ -53,6 +53,7 @@ else (systemlib_ABSEIL_CPP)
           ${abseil_cpp_BUILD}/absl/numeric/Release/absl_int128.lib
           ${abseil_cpp_BUILD}/absl/strings/Release/absl_strings.lib
           ${abseil_cpp_BUILD}/absl/strings/Release/str_format_internal.lib
+          ${abseil_cpp_BUILD}/absl/time/Release/absl_time.lib
           ${abseil_cpp_BUILD}/absl/types/Release/absl_bad_optional_access.lib)
     else()
       set(abseil_cpp_STATIC_LIBRARIES
@@ -64,6 +65,7 @@ else (systemlib_ABSEIL_CPP)
           ${abseil_cpp_BUILD}/absl/numeric/absl_int128.lib
           ${abseil_cpp_BUILD}/absl/strings/absl_strings.lib
           ${abseil_cpp_BUILD}/absl/strings/str_format_internal.lib
+          ${abseil_cpp_BUILD}/absl/time/absl_time.lib
           ${abseil_cpp_BUILD}/absl/types/absl_bad_optional_access.lib)
     endif()
   else()
@@ -76,14 +78,18 @@ else (systemlib_ABSEIL_CPP)
         ${abseil_cpp_BUILD}/absl/numeric/libabsl_int128.a
         ${abseil_cpp_BUILD}/absl/strings/libabsl_strings.a
         ${abseil_cpp_BUILD}/absl/strings/libstr_format_internal.a
+        ${abseil_cpp_BUILD}/absl/time/libabsl_time.a
         ${abseil_cpp_BUILD}/absl/types/libabsl_bad_optional_access.a)
   endif()
 
-  ExternalProject_Add(abseil_cpp
+  ExternalProject_Add(abseil_cpp_build
       PREFIX abseil_cpp
       GIT_REPOSITORY ${abseil_cpp_URL}
       DOWNLOAD_DIR "${DOWNLOAD_LOCATION}"
+      BUILD_IN_SOURCE 1
       BUILD_BYPRODUCTS ${abseil_cpp_STATIC_LIBRARIES}
+      BUILD_COMMAND ${CMAKE_COMMAND} --build . --config Release
+      COMMAND ${CMAKE_COMMAND} --build . --config Release
       INSTALL_COMMAND ""
       CMAKE_CACHE_ARGS
           -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=${tensorflow_ENABLE_POSITION_INDEPENDENT_CODE}
@@ -96,6 +102,6 @@ else (systemlib_ABSEIL_CPP)
 
   list(APPEND tensorflow_EXTERNAL_LIBRARIES ${abseil_cpp_STATIC_LIBRARIES})
 
-  list(APPEND tensorflow_EXTERNAL_DEPENDENCIES abseil_cpp)
+  list(APPEND tensorflow_EXTERNAL_DEPENDENCIES abseil_cpp_build)
 
 endif (systemlib_ABSEIL_CPP)
\ No newline at end of file
-- 
GitLab


From 568db5fb164092eaf2863f0e4a2fdb48533d76f9 Mon Sep 17 00:00:00 2001
From: Siju Samuel <siju.samuel@huawei.com>
Date: Mon, 19 Nov 2018 07:20:36 +0530
Subject: [PATCH 0025/1095] Abs, Ceil ops initial

---
 tensorflow/lite/build_def.bzl                 |   1 +
 tensorflow/lite/builtin_ops.h                 |   1 +
 .../lite/core/api/flatbuffer_conversions.cc   |   2 +
 .../lite/delegates/nnapi/nnapi_delegate.cc    |  20 ++++
 .../delegates/nnapi/nnapi_delegate_test.cc    | 109 ++++++++++++++++++
 .../writer/option_writer_generator.cc         |   2 +
 tensorflow/lite/g3doc/tf_ops_compatibility.md |  22 ++++
 tensorflow/lite/kernels/BUILD                 |  34 ++++++
 tensorflow/lite/kernels/abs.cc                |  59 ++++++++++
 tensorflow/lite/kernels/abs_test.cc           |  94 +++++++++++++++
 tensorflow/lite/kernels/ceil.cc               |  59 ++++++++++
 tensorflow/lite/kernels/ceil_test.cc          |  83 +++++++++++++
 .../internal/optimized/legacy_optimized_ops.h |  12 ++
 .../internal/optimized/optimized_ops.h        |  16 +++
 .../internal/reference/legacy_reference_ops.h |  12 ++
 .../internal/reference/reference_ops.h        |  20 ++++
 tensorflow/lite/kernels/register.cc           |   4 +
 tensorflow/lite/nnapi/NeuralNetworksShim.h    |   2 +
 tensorflow/lite/nnapi_delegate.cc             |   6 +
 tensorflow/lite/schema/schema.fbs             |   1 +
 tensorflow/lite/schema/schema_generated.h     |  10 +-
 tensorflow/lite/testing/generate_examples.py  |  52 +++++++++
 tensorflow/lite/toco/export_tensorflow.cc     |  26 +++++
 .../propagate_fixed_sizes.cc                  |   2 +
 .../reorder_elementwise_unary.cc              |   2 +
 tensorflow/lite/toco/import_tensorflow.cc     |  29 +++++
 tensorflow/lite/toco/model.h                  |  22 ++++
 tensorflow/lite/toco/tflite/operator.cc       |   4 +
 tensorflow/lite/toco/tflite/operator_test.cc  |   2 +
 .../lite/toco/tflite/whitelisted_flex_ops.cc  |   1 +
 tensorflow/lite/toco/tooling_util.cc          |   2 +
 31 files changed, 708 insertions(+), 3 deletions(-)
 create mode 100644 tensorflow/lite/kernels/abs.cc
 create mode 100644 tensorflow/lite/kernels/abs_test.cc
 create mode 100644 tensorflow/lite/kernels/ceil.cc
 create mode 100644 tensorflow/lite/kernels/ceil_test.cc

diff --git a/tensorflow/lite/build_def.bzl b/tensorflow/lite/build_def.bzl
index c17eddf47b..1539ac788a 100644
--- a/tensorflow/lite/build_def.bzl
+++ b/tensorflow/lite/build_def.bzl
@@ -228,6 +228,7 @@ def generated_test_models():
         "arg_min_max",
         "avg_pool",
         "batch_to_space_nd",
+        "ceil",
         "concat",
         "constant",
         "control_dep",
diff --git a/tensorflow/lite/builtin_ops.h b/tensorflow/lite/builtin_ops.h
index f97d3ac4bf..0077fade40 100644
--- a/tensorflow/lite/builtin_ops.h
+++ b/tensorflow/lite/builtin_ops.h
@@ -128,6 +128,7 @@ typedef enum {
   kTfLiteBuiltinMirrorPad = 100,
   kTfLiteBuiltinAbs = 101,
   kTfLiteBuiltinSplitV = 102,
+  kTfLiteBuiltinCeil = 103,
 } TfLiteBuiltinOperator;
 
 #ifdef __cplusplus
diff --git a/tensorflow/lite/core/api/flatbuffer_conversions.cc b/tensorflow/lite/core/api/flatbuffer_conversions.cc
index c00a0a3a54..8a436c440f 100644
--- a/tensorflow/lite/core/api/flatbuffer_conversions.cc
+++ b/tensorflow/lite/core/api/flatbuffer_conversions.cc
@@ -664,6 +664,8 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type,
     case BuiltinOperator_EQUAL:
     case BuiltinOperator_EXP:
     case BuiltinOperator_EXPAND_DIMS:
+    case BuiltinOperator_ABS:
+    case BuiltinOperator_CEIL:
     case BuiltinOperator_FLOOR:
     case BuiltinOperator_GREATER:
     case BuiltinOperator_GREATER_EQUAL:
diff --git a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
index 4fe07004a8..1a95ac1891 100644
--- a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
+++ b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
@@ -638,6 +638,26 @@ class NNAPIDelegateKernel {
           return nullptr;
         }
         break;
+      case kTfLiteBuiltinCeil:
+        if (version == 1) {
+          return [](const NNAPIOpMappingArgs& mapping_args)
+                     -> ANeuralNetworksOperationType {
+            return ANEURALNETWORKS_CEIL;
+          };
+        } else {
+          return nullptr;
+        }
+        break;
+      case kTfLiteBuiltinAbs:
+        if (version == 1) {
+          return [](const NNAPIOpMappingArgs& mapping_args)
+                     -> ANeuralNetworksOperationType {
+            return ANEURALNETWORKS_ABS;
+          };
+        } else {
+          return nullptr;
+        }
+        break;
       case kTfLiteBuiltinRelu:
         if (version == 1) {
           return [](const NNAPIOpMappingArgs& mapping_args)
diff --git a/tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc b/tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc
index ca48af0c95..3050b69028 100644
--- a/tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc
+++ b/tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc
@@ -1024,6 +1024,115 @@ TEST(NNAPIDelegate, FloorMultiDims) {
   EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2, 1, 1, 5}));
 }
 
+class CeilOpModel : public SingleOpModelWithNNAPI {
+ public:
+  CeilOpModel(std::initializer_list<int> input_shape, TensorType input_type) {
+    input_ = AddInput(TensorType_FLOAT32);
+    output_ = AddOutput(TensorType_FLOAT32);
+    SetBuiltinOp(BuiltinOperator_CEIL, BuiltinOptions_NONE, 0);
+    BuildInterpreter({
+        input_shape,
+    });
+  }
+
+  int input() { return input_; }
+
+  std::vector<float> GetOutput() { return ExtractVector<float>(output_); }
+  std::vector<int> GetOutputShape() { return GetTensorShape(output_); }
+
+ private:
+  int input_;
+  int output_;
+};
+
+TEST(NNAPIDelegate, CeilSingleDim) {
+  CeilOpModel model({2}, TensorType_FLOAT32);
+  model.PopulateTensor<float>(model.input(), {8.5, 0.0});
+  model.Invoke();
+  EXPECT_THAT(model.GetOutput(), ElementsAreArray({8, 0}));
+  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2}));
+}
+
+TEST(NNAPIDelegate, CeilMultiDims) {
+  CeilOpModel model({2, 1, 1, 5}, TensorType_FLOAT32);
+  model.PopulateTensor<float>(model.input(), {
+                                                 0.0001,
+                                                 8.0001,
+                                                 0.9999,
+                                                 9.9999,
+                                                 0.5,
+                                                 -0.0001,
+                                                 -8.0001,
+                                                 -0.9999,
+                                                 -9.9999,
+                                                 -0.5,
+                                             });
+  model.Invoke();
+  EXPECT_THAT(model.GetOutput(),
+              ElementsAreArray({1, 9, 1, 10, 1, 0, -8, 0, -9, 0}));
+  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2, 1, 1, 5}));
+}
+
+class AbsOpModel : public SingleOpModelWithNNAPI {
+ public:
+  AbsOpModel(std::initializer_list<int> input_shape, TensorType input_type) {
+    input_ = AddInput(TensorType_FLOAT32);
+    output_ = AddOutput(TensorType_FLOAT32);
+    SetBuiltinOp(BuiltinOperator_ABS, BuiltinOptions_NONE, 0);
+    BuildInterpreter({
+        input_shape,
+    });
+  }
+
+  int input() { return input_; }
+
+  std::vector<float> GetOutput() { return ExtractVector<float>(output_); }
+  std::vector<int> GetOutputShape() { return GetTensorShape(output_); }
+
+ private:
+  int input_;
+  int output_;
+};
+
+TEST(NNAPIDelegate, AbsSingleDim) {
+  AbsOpModel model({2}, TensorType_FLOAT32);
+  model.PopulateTensor<float>(model.input(), {8.5, -2.0});
+  model.Invoke();
+  EXPECT_THAT(model.GetOutput(), ElementsAreArray({8.5, 2.0}));
+  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2}));
+}
+
+TEST(NNAPIDelegate, AbsMultiDims) {
+  AbsOpModel model({2, 1, 1, 5}, TensorType_FLOAT32);
+  model.PopulateTensor<float>(model.input(), {
+                                                 0.0001,
+                                                 8.0001,
+                                                 0.9999,
+                                                 9.9999,
+                                                 0.5,
+                                                 -0.0001,
+                                                 -8.0001,
+                                                 -0.9999,
+                                                 -9.9999,
+                                                 -0.5,
+                                             });
+  model.Invoke();
+  EXPECT_THAT(model.GetOutput(),
+              ElementsAreArray({
+                                                 0.0001,
+                                                 8.0001,
+                                                 0.9999,
+                                                 9.9999,
+                                                 0.5,
+                                                 0.0001,
+                                                 8.0001,
+                                                 0.9999,
+                                                 9.9999,
+                                                 0.5,
+                                             }));
+  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2, 1, 1, 5}));
+}
+
 class LocalResponseNormOpModel : public SingleOpModelWithNNAPI {
  public:
   LocalResponseNormOpModel(std::initializer_list<int> input_shape, int radius,
diff --git a/tensorflow/lite/experimental/writer/option_writer_generator.cc b/tensorflow/lite/experimental/writer/option_writer_generator.cc
index fa360a2f47..d79b6e8ca9 100644
--- a/tensorflow/lite/experimental/writer/option_writer_generator.cc
+++ b/tensorflow/lite/experimental/writer/option_writer_generator.cc
@@ -160,6 +160,8 @@ class OpOptionData {
     op_to_option_["EMBEDDING_LOOKUP"] =
         "";  // TODO(aselle): maybe something else.
     op_to_option_["FLOOR"] = "";
+    op_to_option_["CEIL"] = "";
+    op_to_option_["ABS"] = "";
     op_to_option_["HASHTABLE_LOOKUP"] =
         "";  // TODO(aselle): maybe something else.
     op_to_option_["LOGISTIC"] = "";
diff --git a/tensorflow/lite/g3doc/tf_ops_compatibility.md b/tensorflow/lite/g3doc/tf_ops_compatibility.md
index dcfda72137..dc6ba8a463 100644
--- a/tensorflow/lite/g3doc/tf_ops_compatibility.md
+++ b/tensorflow/lite/g3doc/tf_ops_compatibility.md
@@ -362,6 +362,28 @@ Outputs {
 }
 ```
 
+**CEIL**
+
+```
+inputs {
+  0: tensor
+}
+outputs: {
+  0: result of computing element-wise ceil of the input tensor
+}
+```
+
+**ABS**
+
+```
+inputs {
+  0: tensor
+}
+outputs: {
+  0: result of computing element-wise absolute value of the input tensor
+}
+```
+
 **FULLY_CONNECTED**
 
 ```
diff --git a/tensorflow/lite/kernels/BUILD b/tensorflow/lite/kernels/BUILD
index bad1c4aebf..f66f69870f 100644
--- a/tensorflow/lite/kernels/BUILD
+++ b/tensorflow/lite/kernels/BUILD
@@ -158,6 +158,7 @@ cc_library(
 cc_library(
     name = "builtin_op_kernels",
     srcs = [
+        "abs.cc",
         "activations.cc",
         "add.cc",
         "arg_min_max.cc",
@@ -167,6 +168,7 @@ cc_library(
         "bidirectional_sequence_lstm.cc",
         "bidirectional_sequence_rnn.cc",
         "cast.cc",
+        "ceil.cc",
         "comparisons.cc",
         "concatenation.cc",
         "conv.cc",
@@ -599,6 +601,38 @@ tf_cc_test(
     ],
 )
 
+tf_cc_test(
+    name = "abs_test",
+    size = "small",
+    srcs = ["abs_test.cc"],
+    tags = [
+        "no_oss",
+        "tflite_not_portable_ios",
+    ],
+    deps = [
+        ":builtin_ops",
+        "//tensorflow/lite:framework",
+        "//tensorflow/lite/kernels:test_util",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
+tf_cc_test(
+    name = "ceil_test",
+    size = "small",
+    srcs = ["ceil_test.cc"],
+    tags = [
+        "no_oss",
+        "tflite_not_portable_ios",
+    ],
+    deps = [
+        ":builtin_ops",
+        "//tensorflow/lite:framework",
+        "//tensorflow/lite/kernels:test_util",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
 tf_cc_test(
     name = "elementwise_test",
     size = "small",
diff --git a/tensorflow/lite/kernels/abs.cc b/tensorflow/lite/kernels/abs.cc
new file mode 100644
index 0000000000..b44f9de630
--- /dev/null
+++ b/tensorflow/lite/kernels/abs.cc
@@ -0,0 +1,59 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/c/c_api_internal.h"
+#include "tensorflow/lite/kernels/internal/optimized/optimized_ops.h"
+#include "tensorflow/lite/kernels/internal/tensor.h"
+#include "tensorflow/lite/kernels/kernel_util.h"
+
+namespace tflite {
+namespace ops {
+namespace builtin {
+namespace abs {
+
+constexpr int kInputTensor = 0;
+constexpr int kOutputTensor = 0;
+
+TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
+  const TfLiteTensor* input = GetInput(context, node, kInputTensor);
+  TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
+  TF_LITE_ENSURE_EQ(context, NumInputs(node), 1);
+  TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1);
+  TF_LITE_ENSURE_EQ(context, input->type, kTfLiteFloat32);
+  output->type = input->type;
+  TfLiteIntArray* output_size = TfLiteIntArrayCopy(input->dims);
+  return context->ResizeTensor(context, output, output_size);
+}
+
+TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
+  const TfLiteTensor* input = GetInput(context, node, kInputTensor);
+  TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
+
+  optimized_ops::Abs(GetTensorShape(input), GetTensorData<float>(input),
+                       GetTensorShape(output), GetTensorData<float>(output));
+
+  return kTfLiteOk;
+}
+}  // namespace abs
+
+TfLiteRegistration* Register_ABS() {
+  static TfLiteRegistration r = {/*init=*/nullptr,
+                                 /*free=*/nullptr, abs::Prepare, abs::Eval};
+  return &r;
+}
+
+}  // namespace builtin
+}  // namespace ops
+}  // namespace tflite
diff --git a/tensorflow/lite/kernels/abs_test.cc b/tensorflow/lite/kernels/abs_test.cc
new file mode 100644
index 0000000000..5e3f95c43a
--- /dev/null
+++ b/tensorflow/lite/kernels/abs_test.cc
@@ -0,0 +1,94 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <gtest/gtest.h>
+#include "tensorflow/lite/interpreter.h"
+#include "tensorflow/lite/kernels/register.h"
+#include "tensorflow/lite/kernels/test_util.h"
+#include "tensorflow/lite/model.h"
+
+namespace tflite {
+namespace {
+
+using ::testing::ElementsAreArray;
+
+class AbsOpModel : public SingleOpModel {
+ public:
+  AbsOpModel(std::initializer_list<int> input_shape, TensorType input_type) {
+    input_ = AddInput(TensorType_FLOAT32);
+    output_ = AddOutput(TensorType_FLOAT32);
+    SetBuiltinOp(BuiltinOperator_ABS, BuiltinOptions_NONE, 0);
+    BuildInterpreter({
+        input_shape,
+    });
+  }
+
+  int input() { return input_; }
+
+  std::vector<float> GetOutput() { return ExtractVector<float>(output_); }
+  std::vector<int> GetOutputShape() { return GetTensorShape(output_); }
+
+ private:
+  int input_;
+  int output_;
+};
+
+TEST(AbsOpTest, SingleDim) {
+  AbsOpModel model({2}, TensorType_FLOAT32);
+  model.PopulateTensor<float>(model.input(), {8.5, -2.0});
+  model.Invoke();
+  EXPECT_THAT(model.GetOutput(), ElementsAreArray({8.5, 2.0}));
+  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2}));
+}
+
+TEST(AbsOpTest, MultiDims) {
+  AbsOpModel model({2, 1, 1, 5}, TensorType_FLOAT32);
+  model.PopulateTensor<float>(model.input(), {
+                                                 0.0001,
+                                                 8.0001,
+                                                 0.9999,
+                                                 9.9999,
+                                                 0.5,
+                                                 -0.0001,
+                                                 -8.0001,
+                                                 -0.9999,
+                                                 -9.9999,
+                                                 -0.5,
+                                             });
+  model.Invoke();
+  EXPECT_THAT(model.GetOutput(),
+              ElementsAreArray({
+                                                 0.0001,
+                                                 8.0001,
+                                                 0.9999,
+                                                 9.9999,
+                                                 0.5,
+                                                 0.0001,
+                                                 8.0001,
+                                                 0.9999,
+                                                 9.9999,
+                                                 0.5,
+                                             }));
+  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2, 1, 1, 5}));
+}
+
+}  // namespace
+}  // namespace tflite
+
+int main(int argc, char** argv) {
+  ::tflite::LogToStderr();
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/tensorflow/lite/kernels/ceil.cc b/tensorflow/lite/kernels/ceil.cc
new file mode 100644
index 0000000000..e0ea061f25
--- /dev/null
+++ b/tensorflow/lite/kernels/ceil.cc
@@ -0,0 +1,59 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/c/c_api_internal.h"
+#include "tensorflow/lite/kernels/internal/optimized/optimized_ops.h"
+#include "tensorflow/lite/kernels/internal/tensor.h"
+#include "tensorflow/lite/kernels/kernel_util.h"
+
+namespace tflite {
+namespace ops {
+namespace builtin {
+namespace ceil {
+
+constexpr int kInputTensor = 0;
+constexpr int kOutputTensor = 0;
+
+TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
+  const TfLiteTensor* input = GetInput(context, node, kInputTensor);
+  TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
+  TF_LITE_ENSURE_EQ(context, NumInputs(node), 1);
+  TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1);
+  TF_LITE_ENSURE_EQ(context, input->type, kTfLiteFloat32);
+  output->type = input->type;
+  TfLiteIntArray* output_size = TfLiteIntArrayCopy(input->dims);
+  return context->ResizeTensor(context, output, output_size);
+}
+
+TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
+  const TfLiteTensor* input = GetInput(context, node, kInputTensor);
+  TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
+
+  optimized_ops::Ceil(GetTensorShape(input), GetTensorData<float>(input),
+                       GetTensorShape(output), GetTensorData<float>(output));
+
+  return kTfLiteOk;
+}
+}  // namespace ceil
+
+TfLiteRegistration* Register_CEIL() {
+  static TfLiteRegistration r = {/*init=*/nullptr,
+                                 /*free=*/nullptr, ceil::Prepare, ceil::Eval};
+  return &r;
+}
+
+}  // namespace builtin
+}  // namespace ops
+}  // namespace tflite
diff --git a/tensorflow/lite/kernels/ceil_test.cc b/tensorflow/lite/kernels/ceil_test.cc
new file mode 100644
index 0000000000..e120105082
--- /dev/null
+++ b/tensorflow/lite/kernels/ceil_test.cc
@@ -0,0 +1,83 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <gtest/gtest.h>
+#include "tensorflow/lite/interpreter.h"
+#include "tensorflow/lite/kernels/register.h"
+#include "tensorflow/lite/kernels/test_util.h"
+#include "tensorflow/lite/model.h"
+
+namespace tflite {
+namespace {
+
+using ::testing::ElementsAreArray;
+
+class CeilOpModel : public SingleOpModel {
+ public:
+  CeilOpModel(std::initializer_list<int> input_shape, TensorType input_type) {
+    input_ = AddInput(TensorType_FLOAT32);
+    output_ = AddOutput(TensorType_FLOAT32);
+    SetBuiltinOp(BuiltinOperator_CEIL, BuiltinOptions_NONE, 0);
+    BuildInterpreter({
+        input_shape,
+    });
+  }
+
+  int input() { return input_; }
+
+  std::vector<float> GetOutput() { return ExtractVector<float>(output_); }
+  std::vector<int> GetOutputShape() { return GetTensorShape(output_); }
+
+ private:
+  int input_;
+  int output_;
+};
+
+TEST(CeilOpTest, SingleDim) {
+  CeilOpModel model({2}, TensorType_FLOAT32);
+  model.PopulateTensor<float>(model.input(), {8.5, 0.0});
+  model.Invoke();
+  EXPECT_THAT(model.GetOutput(), ElementsAreArray({9, 0}));
+  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2}));
+}
+
+TEST(CeilOpTest, MultiDims) {
+  CeilOpModel model({2, 1, 1, 5}, TensorType_FLOAT32);
+  model.PopulateTensor<float>(model.input(), {
+                                                 0.0001,
+                                                 8.0001,
+                                                 0.9999,
+                                                 9.9999,
+                                                 0.5,
+                                                 -0.0001,
+                                                 -8.0001,
+                                                 -0.9999,
+                                                 -9.9999,
+                                                 -0.5,
+                                             });
+  model.Invoke();
+  EXPECT_THAT(model.GetOutput(),
+              ElementsAreArray({1, 9, 1, 10, 1, 0, -8, 0, -9, 0}));
+  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2, 1, 1, 5}));
+}
+
+}  // namespace
+}  // namespace tflite
+
+int main(int argc, char** argv) {
+  ::tflite::LogToStderr();
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/tensorflow/lite/kernels/internal/optimized/legacy_optimized_ops.h b/tensorflow/lite/kernels/internal/optimized/legacy_optimized_ops.h
index 5485d907c2..c24b7faa21 100644
--- a/tensorflow/lite/kernels/internal/optimized/legacy_optimized_ops.h
+++ b/tensorflow/lite/kernels/internal/optimized/legacy_optimized_ops.h
@@ -1740,6 +1740,18 @@ inline void Floor(const float* input_data, const Dims<4>& input_dims,
         output_data);
 }
 
+inline void Ceil(const float* input_data, const Dims<4>& input_dims,
+                 float* output_data, const Dims<4>& output_dims) {
+  Ceil(DimsToShape(input_dims), input_data, DimsToShape(output_dims),
+       output_data);
+}
+
+inline void Abs(const float* input_data, const Dims<4>& input_dims,
+                float* output_data, const Dims<4>& output_dims) {
+  Abs(DimsToShape(input_dims), input_data, DimsToShape(output_dims),
+      output_data);
+}
+
 inline void ResizeBilinear(const float* input_data, const Dims<4>& input_dims,
                            const int32* output_size_data,
                            const Dims<4>& output_size_dims, float* output_data,
diff --git a/tensorflow/lite/kernels/internal/optimized/optimized_ops.h b/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
index c79b69a22e..7f7296ae97 100644
--- a/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
+++ b/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
@@ -4898,6 +4898,22 @@ inline void Floor(const RuntimeShape& input_shape, const float* input_data,
   output_map.array() = Eigen::floor(input_map.array());
 }
 
+inline void Ceil(const RuntimeShape& input_shape, const float* input_data,
+                 const RuntimeShape& output_shape, float* output_data) {
+  gemmlowp::ScopedProfilingLabel label("Ceil");
+  auto input_map = MapAsVector(input_data, input_shape);
+  auto output_map = MapAsVector(output_data, output_shape);
+  output_map.array() = Eigen::ceil(input_map.array());
+}
+
+inline void Abs(const RuntimeShape& input_shape, const float* input_data,
+                const RuntimeShape& output_shape, float* output_data) {
+  gemmlowp::ScopedProfilingLabel label("Abs");
+  auto input_map = MapAsVector(input_data, input_shape);
+  auto output_map = MapAsVector(output_data, output_shape);
+  output_map.array() = Eigen::abs(input_map.array());
+}
+
 #ifdef USE_NEON
 inline void ResizeBilinearKernel(const float* input_ptr, int32 depth,
                                  float scale, float* output_ptr) {
diff --git a/tensorflow/lite/kernels/internal/reference/legacy_reference_ops.h b/tensorflow/lite/kernels/internal/reference/legacy_reference_ops.h
index 380fc8f98e..30bb92c8b3 100644
--- a/tensorflow/lite/kernels/internal/reference/legacy_reference_ops.h
+++ b/tensorflow/lite/kernels/internal/reference/legacy_reference_ops.h
@@ -1883,6 +1883,18 @@ inline void Floor(const float* input_data, const Dims<4>& input_dims,
         output_data);
 }
 
+inline void Ceil(const float* input_data, const Dims<4>& input_dims,
+                 float* output_data, const Dims<4>& output_dims) {
+  Ceil(DimsToShape(input_dims), input_data, DimsToShape(output_dims),
+       output_data);
+}
+
+inline void Abs(const float* input_data, const Dims<4>& input_dims,
+                float* output_data, const Dims<4>& output_dims) {
+  Abs(DimsToShape(input_dims), input_data, DimsToShape(output_dims),
+      output_data);
+}
+
 template <typename T>
 inline void ResizeBilinear(const T* input_data, const Dims<4>& input_dims,
                            const int32* output_size_data,
diff --git a/tensorflow/lite/kernels/internal/reference/reference_ops.h b/tensorflow/lite/kernels/internal/reference/reference_ops.h
index ea3ab06da1..133a455e9e 100644
--- a/tensorflow/lite/kernels/internal/reference/reference_ops.h
+++ b/tensorflow/lite/kernels/internal/reference/reference_ops.h
@@ -3040,6 +3040,26 @@ inline void Floor(const RuntimeShape& input_shape, const float* input_data,
   }
 }
 
+inline void Ceil(const RuntimeShape& input_shape, const float* input_data,
+                 const RuntimeShape& output_shape, float* output_data) {
+  const int flat_size = MatchingFlatSize(input_shape, output_shape);
+
+  for (int i = 0; i < flat_size; i++) {
+    int offset = i;
+    output_data[offset] = std::ceil(input_data[offset]);
+  }
+}
+
+inline void Abs(const RuntimeShape& input_shape, const float* input_data,
+                const RuntimeShape& output_shape, float* output_data) {
+  const int flat_size = MatchingFlatSize(input_shape, output_shape);
+
+  for (int i = 0; i < flat_size; i++) {
+    int offset = i;
+    output_data[offset] = std::fabs(input_data[offset]);
+  }
+}
+
 template <typename T, typename CoordsT = int32>
 inline void Gather(const tflite::GatherParams& op_params,
                    const RuntimeShape& input_shape, const T* input_data,
diff --git a/tensorflow/lite/kernels/register.cc b/tensorflow/lite/kernels/register.cc
index c0e6f6994f..caf61e9003 100644
--- a/tensorflow/lite/kernels/register.cc
+++ b/tensorflow/lite/kernels/register.cc
@@ -94,6 +94,8 @@ TfLiteRegistration* Register_GREATER_EQUAL();
 TfLiteRegistration* Register_LESS();
 TfLiteRegistration* Register_LESS_EQUAL();
 TfLiteRegistration* Register_FLOOR();
+TfLiteRegistration* Register_CEIL();
+TfLiteRegistration* Register_ABS();
 TfLiteRegistration* Register_TILE();
 TfLiteRegistration* Register_NEG();
 TfLiteRegistration* Register_SUM();
@@ -235,6 +237,8 @@ BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_LESS, Register_LESS());
   AddBuiltin(BuiltinOperator_LESS_EQUAL, Register_LESS_EQUAL());
   AddBuiltin(BuiltinOperator_FLOOR, Register_FLOOR());
+  AddBuiltin(BuiltinOperator_CEIL, Register_CEIL());
+  AddBuiltin(BuiltinOperator_ABS, Register_ABS());
   AddBuiltin(BuiltinOperator_NEG, Register_NEG());
   AddBuiltin(BuiltinOperator_SELECT, Register_SELECT());
   AddBuiltin(BuiltinOperator_SLICE, Register_SLICE());
diff --git a/tensorflow/lite/nnapi/NeuralNetworksShim.h b/tensorflow/lite/nnapi/NeuralNetworksShim.h
index c39502f4ac..037d364c3b 100644
--- a/tensorflow/lite/nnapi/NeuralNetworksShim.h
+++ b/tensorflow/lite/nnapi/NeuralNetworksShim.h
@@ -143,6 +143,8 @@ enum {
   ANEURALNETWORKS_STRIDED_SLICE = 35,
   ANEURALNETWORKS_SUB = 36,
   ANEURALNETWORKS_TRANSPOSE = 37,
+  ANEURALNETWORKS_CEIL = 38,
+  ANEURALNETWORKS_ABS = 39,
 };
 
 /**
diff --git a/tensorflow/lite/nnapi_delegate.cc b/tensorflow/lite/nnapi_delegate.cc
index 26d75696a1..6da803ce7b 100644
--- a/tensorflow/lite/nnapi_delegate.cc
+++ b/tensorflow/lite/nnapi_delegate.cc
@@ -489,6 +489,12 @@ TfLiteStatus AddOpsAndParams(
       case tflite::BuiltinOperator_FLOOR:
         nn_op_type = ANEURALNETWORKS_FLOOR;
         break;
+      case tflite::BuiltinOperator_CEIL:
+        nn_op_type = ANEURALNETWORKS_CEIL;
+        break;
+      case tflite::BuiltinOperator_ABS:
+        nn_op_type = ANEURALNETWORKS_ABS;
+        break;
       case tflite::BuiltinOperator_LOGISTIC:
         nn_op_type = ANEURALNETWORKS_LOGISTIC;
         break;
diff --git a/tensorflow/lite/schema/schema.fbs b/tensorflow/lite/schema/schema.fbs
index 980f13b19b..2d27722415 100644
--- a/tensorflow/lite/schema/schema.fbs
+++ b/tensorflow/lite/schema/schema.fbs
@@ -205,6 +205,7 @@ enum BuiltinOperator : byte {
   MIRROR_PAD = 100,
   ABS = 101,
   SPLIT_V = 102,
+  CEIL = 103,
 }
 
 // Options for the builtin operators.
diff --git a/tensorflow/lite/schema/schema_generated.h b/tensorflow/lite/schema/schema_generated.h
index 637cbafabd..a605c19b52 100755
--- a/tensorflow/lite/schema/schema_generated.h
+++ b/tensorflow/lite/schema/schema_generated.h
@@ -520,11 +520,12 @@ enum BuiltinOperator {
   BuiltinOperator_MIRROR_PAD = 100,
   BuiltinOperator_ABS = 101,
   BuiltinOperator_SPLIT_V = 102,
+  BuiltinOperator_CEIL = 103,
   BuiltinOperator_MIN = BuiltinOperator_ADD,
-  BuiltinOperator_MAX = BuiltinOperator_SPLIT_V
+  BuiltinOperator_MAX = BuiltinOperator_CEIL
 };
 
-inline const BuiltinOperator (&EnumValuesBuiltinOperator())[102] {
+inline const BuiltinOperator (&EnumValuesBuiltinOperator())[103] {
   static const BuiltinOperator values[] = {
     BuiltinOperator_ADD,
     BuiltinOperator_AVERAGE_POOL_2D,
@@ -627,7 +628,9 @@ inline const BuiltinOperator (&EnumValuesBuiltinOperator())[102] {
     BuiltinOperator_SQUARED_DIFFERENCE,
     BuiltinOperator_MIRROR_PAD,
     BuiltinOperator_ABS,
-    BuiltinOperator_SPLIT_V
+    BuiltinOperator_SPLIT_V,
+    BuiltinOperator_CEIL
+
   };
   return values;
 }
@@ -737,6 +740,7 @@ inline const char * const *EnumNamesBuiltinOperator() {
     "MIRROR_PAD",
     "ABS",
     "SPLIT_V",
+    "CEIL",
     nullptr
   };
   return names;
diff --git a/tensorflow/lite/testing/generate_examples.py b/tensorflow/lite/testing/generate_examples.py
index dd7b3d0745..825873a884 100644
--- a/tensorflow/lite/testing/generate_examples.py
+++ b/tensorflow/lite/testing/generate_examples.py
@@ -3021,6 +3021,58 @@ def make_floor_tests(zip_path):
   make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
 
 
+def make_ceil_tests(zip_path):
+  """Make a set of tests to do ceil."""
+
+  test_parameters = [{
+      "input_dtype": [tf.float32],
+      "input_shape": [[1], [1, 2], [5, 6, 7, 8], [3, 4, 5, 6]],
+  }]
+
+  def build_graph(parameters):
+    """Build the ceil op testing graph."""
+    input_value = tf.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input1",
+        shape=parameters["input_shape"])
+    out = tf.ceil(input_value)
+    return [input_value], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_value = create_tensor_data(parameters["input_dtype"],
+                                     parameters["input_shape"])
+    return [input_value], sess.run(
+        outputs, feed_dict={inputs[0]: input_value})
+
+  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+
+
+def make_abs_tests(zip_path):
+  """Make a set of tests to do abs."""
+
+  test_parameters = [{
+      "input_dtype": [tf.float32],
+      "input_shape": [[1], [1, 2], [5, 6, 7, 8], [3, 4, 5, 6]],
+  }]
+
+  def build_graph(parameters):
+    """Build the abs op testing graph."""
+    input_value = tf.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input1",
+        shape=parameters["input_shape"])
+    out = tf.abs(input_value)
+    return [input_value], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_value = create_tensor_data(parameters["input_dtype"],
+                                     parameters["input_shape"])
+    return [input_value], sess.run(
+        outputs, feed_dict={inputs[0]: input_value})
+
+  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+
+
 def make_neg_tests(zip_path):
   """Make a set of tests to do neg."""
 
diff --git a/tensorflow/lite/toco/export_tensorflow.cc b/tensorflow/lite/toco/export_tensorflow.cc
index 9fff001552..9dfeaad164 100644
--- a/tensorflow/lite/toco/export_tensorflow.cc
+++ b/tensorflow/lite/toco/export_tensorflow.cc
@@ -1205,6 +1205,26 @@ void ConvertFloorOperator(const Model& model, const FloorOperator& src_op,
   (*floor_op->mutable_attr())["T"].set_type(DT_FLOAT);
 }
 
+void ConvertCeilOperator(const Model& model, const CeilOperator& src_op,
+                         GraphDef* tensorflow_graph) {
+  tensorflow::NodeDef* ceil_op = tensorflow_graph->add_node();
+  ceil_op->set_op("Ceil");
+  ceil_op->set_name(src_op.outputs[0]);
+  CHECK_EQ(src_op.inputs.size(), 1);
+  *ceil_op->add_input() = src_op.inputs[0];
+  (*ceil_op->mutable_attr())["T"].set_type(DT_FLOAT);
+}
+
+void ConvertAbsOperator(const Model& model, const AbsOperator& src_op,
+                        GraphDef* tensorflow_graph) {
+  tensorflow::NodeDef* abs_op = tensorflow_graph->add_node();
+  abs_op->set_op("Abs");
+  abs_op->set_name(src_op.outputs[0]);
+  CHECK_EQ(src_op.inputs.size(), 1);
+  *abs_op->add_input() = src_op.inputs[0];
+  (*abs_op->mutable_attr())["T"].set_type(DT_FLOAT);
+}
+
 void ConvertGatherOperator(const Model& model, const GatherOperator& src_op,
                            GraphDef* tensorflow_graph) {
   tensorflow::NodeDef* gather_op = tensorflow_graph->add_node();
@@ -2169,6 +2189,12 @@ void ConvertOperator(const Model& model, const Operator& src_op,
   } else if (src_op.type == OperatorType::kFloor) {
     ConvertFloorOperator(model, static_cast<const FloorOperator&>(src_op),
                          tensorflow_graph);
+  } else if (src_op.type == OperatorType::kCeil) {
+    ConvertCeilOperator(model, static_cast<const CeilOperator&>(src_op),
+                        tensorflow_graph);
+  } else if (src_op.type == OperatorType::kAbs) {
+    ConvertAbsOperator(model, static_cast<const AbsOperator&>(src_op),
+                       tensorflow_graph);
   } else if (src_op.type == OperatorType::kGather) {
     ConvertGatherOperator(model, static_cast<const GatherOperator&>(src_op),
                           tensorflow_graph);
diff --git a/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc b/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
index 0e653f08a0..c6c7681c03 100644
--- a/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
+++ b/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
@@ -1869,6 +1869,8 @@ void ProcessMirrorPadOperator(Model* model, MirrorPadOperator* op) {
     case OperatorType::kAssert:
     case OperatorType::kCast:
     case OperatorType::kFloor:
+    case OperatorType::kCeil:
+    case OperatorType::kAbs:
     case OperatorType::kExp:
     case OperatorType::kSin:
     case OperatorType::kLogicalAnd:
diff --git a/tensorflow/lite/toco/graph_transformations/reorder_elementwise_unary.cc b/tensorflow/lite/toco/graph_transformations/reorder_elementwise_unary.cc
index 6a4b919854..90e64816c8 100644
--- a/tensorflow/lite/toco/graph_transformations/reorder_elementwise_unary.cc
+++ b/tensorflow/lite/toco/graph_transformations/reorder_elementwise_unary.cc
@@ -29,7 +29,9 @@ namespace {
 
 bool IsElementwiseOperator(OperatorType optype) {
   switch (optype) {
+    case OperatorType::kAbs:
     case OperatorType::kCast:
+    case OperatorType::kCeil:
     case OperatorType::kExp:
     case OperatorType::kFloor:
     case OperatorType::kNeg:
diff --git a/tensorflow/lite/toco/import_tensorflow.cc b/tensorflow/lite/toco/import_tensorflow.cc
index 0b2f810394..de8ae19a71 100644
--- a/tensorflow/lite/toco/import_tensorflow.cc
+++ b/tensorflow/lite/toco/import_tensorflow.cc
@@ -1491,6 +1491,34 @@ tensorflow::Status ConvertFloorOperator(
   return tensorflow::Status::OK();
 }
 
+tensorflow::Status ConvertCeilOperator(
+    const NodeDef& node, const TensorFlowImportFlags& tf_import_flags,
+    Model* model) {
+  CHECK_EQ(node.op(), "Ceil");
+  TF_QCHECK_OK(CheckInputsCount(node, tf_import_flags, 1));
+  const auto data_type = GetDataTypeAttr(node, "T");
+  CHECK(data_type == DT_FLOAT);
+  auto* op = new CeilOperator;
+  op->inputs.push_back(node.input(0));
+  op->outputs.push_back(node.name());
+  model->operators.emplace_back(op);
+  return tensorflow::Status::OK();
+}
+
+tensorflow::Status ConvertAbsOperator(
+    const NodeDef& node, const TensorFlowImportFlags& tf_import_flags,
+    Model* model) {
+  CHECK_EQ(node.op(), "Abs");
+  TF_QCHECK_OK(CheckInputsCount(node, tf_import_flags, 1));
+  const auto data_type = GetDataTypeAttr(node, "T");
+  CHECK(data_type == DT_FLOAT);
+  auto* op = new AbsOperator;
+  op->inputs.push_back(node.input(0));
+  op->outputs.push_back(node.name());
+  model->operators.emplace_back(op);
+  return tensorflow::Status::OK();
+}
+
 tensorflow::Status ConvertGatherOperator(
     const NodeDef& node, const TensorFlowImportFlags& tf_import_flags,
     Model* model) {
@@ -2314,6 +2342,7 @@ ConverterMapType GetTensorFlowNodeConverterMap() {
       {"BatchToSpaceND", ConvertBatchToSpaceNDOperator},
       {"BiasAdd", ConvertBiasAddOperator},
       {"Cast", ConvertCastOperator},
+      {"Ceil", ConvertCeilOperator},
       {"CheckNumerics", ConvertIdentityOperator},
       {"Concat", ConvertConcatOperator},
       {"ConcatV2", ConvertConcatOperator},
diff --git a/tensorflow/lite/toco/model.h b/tensorflow/lite/toco/model.h
index d392535f5c..69c6d7608a 100644
--- a/tensorflow/lite/toco/model.h
+++ b/tensorflow/lite/toco/model.h
@@ -37,11 +37,13 @@ using tflite::QuantizationParams;
 enum class OperatorType : uint8 {
   kNone,
   // General-purpose neural network operators.
+  kAbs,
   kAdd,
   kAddN,
   kAveragePool,
   kBatchMatMul,
   kBatchNormalization,
+  kCeil,
   kConv,
   kConcatenation,
   kDepthwiseConv,
@@ -1658,6 +1660,26 @@ struct FloorOperator : Operator {
   FloorOperator() : Operator(OperatorType::kFloor) {}
 };
 
+// Ceil operator.
+//
+// Inputs:
+//   inputs[0]: required: the input array
+//
+// TensorFlow equivalent: Ceil
+struct CeilOperator : Operator {
+  CeilOperator() : Operator(OperatorType::kCeil) {}
+};
+
+// Abs operator.
+//
+// Inputs:
+//   inputs[0]: required: the input array
+//
+// TensorFlow equivalent: Abs
+struct AbsOperator : Operator {
+  AbsOperator() : Operator(OperatorType::kAbs) {}
+};
+
 // Gather operator. It gathers slices from params according to indices.
 // Only 1-D indices are supported at the moment.
 //
diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index 205af23da5..2122d04958 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -1649,6 +1649,10 @@ std::vector<std::unique_ptr<BaseOperator>> BuildOperatorList(
       "DEQUANTIZE", OperatorType::kDequantize));
   ops.push_back(
       MakeUnique<SimpleOperator<FloorOperator>>("FLOOR", OperatorType::kFloor));
+  ops.push_back(
+      MakeUnique<SimpleOperator<CeilOperator>>("CEIL", OperatorType::kCeil));
+  ops.push_back(
+      MakeUnique<SimpleOperator<AbsOperator>>("ABS", OperatorType::kAbs));
   ops.push_back(
       MakeUnique<SimpleOperator<ReluOperator>>("RELU", OperatorType::kRelu));
   ops.push_back(MakeUnique<SimpleOperator<Relu1Operator>>(
diff --git a/tensorflow/lite/toco/tflite/operator_test.cc b/tensorflow/lite/toco/tflite/operator_test.cc
index 14ec89cd73..e284619903 100644
--- a/tensorflow/lite/toco/tflite/operator_test.cc
+++ b/tensorflow/lite/toco/tflite/operator_test.cc
@@ -114,6 +114,8 @@ TEST_F(OperatorTest, SimpleOperators) {
   CheckSimpleOperator<DequantizeOperator>("DEQUANTIZE",
                                           OperatorType::kDequantize);
   CheckSimpleOperator<FloorOperator>("FLOOR", OperatorType::kFloor);
+  CheckSimpleOperator<CeilOperator>("CEIL", OperatorType::kCeil);
+  CheckSimpleOperator<AbsOperator>("ABS", OperatorType::kAbs);
   CheckSimpleOperator<ReluOperator>("RELU", OperatorType::kRelu);
   CheckSimpleOperator<Relu1Operator>("RELU_N1_TO_1", OperatorType::kRelu1);
   CheckSimpleOperator<Relu6Operator>("RELU6", OperatorType::kRelu6);
diff --git a/tensorflow/lite/toco/tflite/whitelisted_flex_ops.cc b/tensorflow/lite/toco/tflite/whitelisted_flex_ops.cc
index 039a918af1..a96222a8a2 100644
--- a/tensorflow/lite/toco/tflite/whitelisted_flex_ops.cc
+++ b/tensorflow/lite/toco/tflite/whitelisted_flex_ops.cc
@@ -68,6 +68,7 @@ bool IsWhitelistedFlexOp(const std::string& tensorflow_op_name) {
           "BroadcastArgs",
           "BroadcastGradientArgs",
           "Cast",
+          "Ceil",
           "CheckNumerics",
           "ComplexAbs",
           "Concat",
diff --git a/tensorflow/lite/toco/tooling_util.cc b/tensorflow/lite/toco/tooling_util.cc
index af4cd386a2..0887dca9b5 100644
--- a/tensorflow/lite/toco/tooling_util.cc
+++ b/tensorflow/lite/toco/tooling_util.cc
@@ -385,6 +385,8 @@ const char* OperatorTypeName(OperatorType type) {
     HANDLE_OPERATORTYPENAME_CASE(ConcatV2)
     HANDLE_OPERATORTYPENAME_CASE(Cast)
     HANDLE_OPERATORTYPENAME_CASE(Floor)
+    HANDLE_OPERATORTYPENAME_CASE(Ceil)
+    HANDLE_OPERATORTYPENAME_CASE(Abs)
     HANDLE_OPERATORTYPENAME_CASE(Gather)
     HANDLE_OPERATORTYPENAME_CASE(ResizeBilinear)
     HANDLE_OPERATORTYPENAME_CASE(SpaceToBatchND)
-- 
GitLab


From 9e91a0899f116e8bfec46922221c45765346ce2e Mon Sep 17 00:00:00 2001
From: Yves-Noel Weweler <y.weweler@gmail.com>
Date: Wed, 12 Dec 2018 15:27:09 +0100
Subject: [PATCH 0026/1095] Update tensorflow.data.experimental goldens for v1

Add the drop_remainder argument for bucket_by_sequence_length to the goldens.
---
 .../tools/api/golden/v1/tensorflow.data.experimental.pbtxt      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt
index ad10b82283..b8ba3e341f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt
@@ -54,7 +54,7 @@ tf_module {
   }
   member_method {
     name: "bucket_by_sequence_length"
-    argspec: "args=[\'element_length_func\', \'bucket_boundaries\', \'bucket_batch_sizes\', \'padded_shapes\', \'padding_values\', \'pad_to_bucket_boundary\', \'no_padding\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'False\', \'False\'], "
+    argspec: "args=[\'element_length_func\', \'bucket_boundaries\', \'bucket_batch_sizes\', \'padded_shapes\', \'padding_values\', \'pad_to_bucket_boundary\', \'no_padding\', \'drop_remainder\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'False\', \'False\', \'False\'], "
   }
   member_method {
     name: "choose_from_datasets"
-- 
GitLab


From c266087d4d086913194bd621cad125853eca8fd6 Mon Sep 17 00:00:00 2001
From: Yves-Noel Weweler <y.weweler@gmail.com>
Date: Wed, 12 Dec 2018 15:29:25 +0100
Subject: [PATCH 0027/1095] Update tensorflow.data.experimental goldens for v2

Add the drop_remainder argument for bucket_by_sequence_length to the goldens.
---
 .../tools/api/golden/v2/tensorflow.data.experimental.pbtxt      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt
index ad10b82283..b8ba3e341f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt
@@ -54,7 +54,7 @@ tf_module {
   }
   member_method {
     name: "bucket_by_sequence_length"
-    argspec: "args=[\'element_length_func\', \'bucket_boundaries\', \'bucket_batch_sizes\', \'padded_shapes\', \'padding_values\', \'pad_to_bucket_boundary\', \'no_padding\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'False\', \'False\'], "
+    argspec: "args=[\'element_length_func\', \'bucket_boundaries\', \'bucket_batch_sizes\', \'padded_shapes\', \'padding_values\', \'pad_to_bucket_boundary\', \'no_padding\', \'drop_remainder\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'False\', \'False\', \'False\'], "
   }
   member_method {
     name: "choose_from_datasets"
-- 
GitLab


From e30a0a1de0b0b51da69b3d4bf5c545142accb3c2 Mon Sep 17 00:00:00 2001
From: Siju Samuel <siju.samuel@huawei.com>
Date: Wed, 12 Dec 2018 20:03:36 +0530
Subject: [PATCH 0028/1095] Removed abs since already committed in
 5916a9f0e4b5b2c4f80767ff83a001a6f86b4395

---
 .../lite/core/api/flatbuffer_conversions.cc   |  1 -
 .../lite/delegates/nnapi/nnapi_delegate.cc    | 10 --
 .../delegates/nnapi/nnapi_delegate_test.cc    | 60 ------------
 .../writer/option_writer_generator.cc         |  1 -
 tensorflow/lite/g3doc/tf_ops_compatibility.md | 11 ---
 tensorflow/lite/kernels/BUILD                 | 17 ----
 tensorflow/lite/kernels/abs.cc                | 59 ------------
 tensorflow/lite/kernels/abs_test.cc           | 94 -------------------
 .../internal/optimized/legacy_optimized_ops.h |  6 --
 .../internal/optimized/optimized_ops.h        |  8 --
 .../internal/reference/legacy_reference_ops.h |  6 --
 .../internal/reference/reference_ops.h        | 10 --
 tensorflow/lite/kernels/register.cc           |  2 -
 tensorflow/lite/nnapi/NeuralNetworksShim.h    |  1 -
 tensorflow/lite/nnapi_delegate.cc             |  3 -
 tensorflow/lite/testing/generate_examples.py  | 26 -----
 tensorflow/lite/toco/export_tensorflow.cc     | 13 ---
 .../propagate_fixed_sizes.cc                  |  1 -
 .../reorder_elementwise_unary.cc              |  1 -
 tensorflow/lite/toco/import_tensorflow.cc     | 14 ---
 tensorflow/lite/toco/model.h                  | 11 ---
 tensorflow/lite/toco/tflite/operator.cc       |  2 -
 tensorflow/lite/toco/tflite/operator_test.cc  |  1 -
 tensorflow/lite/toco/tooling_util.cc          |  1 -
 24 files changed, 359 deletions(-)
 delete mode 100644 tensorflow/lite/kernels/abs.cc
 delete mode 100644 tensorflow/lite/kernels/abs_test.cc

diff --git a/tensorflow/lite/core/api/flatbuffer_conversions.cc b/tensorflow/lite/core/api/flatbuffer_conversions.cc
index 8a436c440f..9c8eb5a2d8 100644
--- a/tensorflow/lite/core/api/flatbuffer_conversions.cc
+++ b/tensorflow/lite/core/api/flatbuffer_conversions.cc
@@ -664,7 +664,6 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type,
     case BuiltinOperator_EQUAL:
     case BuiltinOperator_EXP:
     case BuiltinOperator_EXPAND_DIMS:
-    case BuiltinOperator_ABS:
     case BuiltinOperator_CEIL:
     case BuiltinOperator_FLOOR:
     case BuiltinOperator_GREATER:
diff --git a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
index 1a95ac1891..cac98ae3da 100644
--- a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
+++ b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
@@ -648,16 +648,6 @@ class NNAPIDelegateKernel {
           return nullptr;
         }
         break;
-      case kTfLiteBuiltinAbs:
-        if (version == 1) {
-          return [](const NNAPIOpMappingArgs& mapping_args)
-                     -> ANeuralNetworksOperationType {
-            return ANEURALNETWORKS_ABS;
-          };
-        } else {
-          return nullptr;
-        }
-        break;
       case kTfLiteBuiltinRelu:
         if (version == 1) {
           return [](const NNAPIOpMappingArgs& mapping_args)
diff --git a/tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc b/tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc
index 3050b69028..f6a04e36cd 100644
--- a/tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc
+++ b/tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc
@@ -1073,66 +1073,6 @@ TEST(NNAPIDelegate, CeilMultiDims) {
   EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2, 1, 1, 5}));
 }
 
-class AbsOpModel : public SingleOpModelWithNNAPI {
- public:
-  AbsOpModel(std::initializer_list<int> input_shape, TensorType input_type) {
-    input_ = AddInput(TensorType_FLOAT32);
-    output_ = AddOutput(TensorType_FLOAT32);
-    SetBuiltinOp(BuiltinOperator_ABS, BuiltinOptions_NONE, 0);
-    BuildInterpreter({
-        input_shape,
-    });
-  }
-
-  int input() { return input_; }
-
-  std::vector<float> GetOutput() { return ExtractVector<float>(output_); }
-  std::vector<int> GetOutputShape() { return GetTensorShape(output_); }
-
- private:
-  int input_;
-  int output_;
-};
-
-TEST(NNAPIDelegate, AbsSingleDim) {
-  AbsOpModel model({2}, TensorType_FLOAT32);
-  model.PopulateTensor<float>(model.input(), {8.5, -2.0});
-  model.Invoke();
-  EXPECT_THAT(model.GetOutput(), ElementsAreArray({8.5, 2.0}));
-  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2}));
-}
-
-TEST(NNAPIDelegate, AbsMultiDims) {
-  AbsOpModel model({2, 1, 1, 5}, TensorType_FLOAT32);
-  model.PopulateTensor<float>(model.input(), {
-                                                 0.0001,
-                                                 8.0001,
-                                                 0.9999,
-                                                 9.9999,
-                                                 0.5,
-                                                 -0.0001,
-                                                 -8.0001,
-                                                 -0.9999,
-                                                 -9.9999,
-                                                 -0.5,
-                                             });
-  model.Invoke();
-  EXPECT_THAT(model.GetOutput(),
-              ElementsAreArray({
-                                                 0.0001,
-                                                 8.0001,
-                                                 0.9999,
-                                                 9.9999,
-                                                 0.5,
-                                                 0.0001,
-                                                 8.0001,
-                                                 0.9999,
-                                                 9.9999,
-                                                 0.5,
-                                             }));
-  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2, 1, 1, 5}));
-}
-
 class LocalResponseNormOpModel : public SingleOpModelWithNNAPI {
  public:
   LocalResponseNormOpModel(std::initializer_list<int> input_shape, int radius,
diff --git a/tensorflow/lite/experimental/writer/option_writer_generator.cc b/tensorflow/lite/experimental/writer/option_writer_generator.cc
index d79b6e8ca9..ba0303f9db 100644
--- a/tensorflow/lite/experimental/writer/option_writer_generator.cc
+++ b/tensorflow/lite/experimental/writer/option_writer_generator.cc
@@ -161,7 +161,6 @@ class OpOptionData {
         "";  // TODO(aselle): maybe something else.
     op_to_option_["FLOOR"] = "";
     op_to_option_["CEIL"] = "";
-    op_to_option_["ABS"] = "";
     op_to_option_["HASHTABLE_LOOKUP"] =
         "";  // TODO(aselle): maybe something else.
     op_to_option_["LOGISTIC"] = "";
diff --git a/tensorflow/lite/g3doc/tf_ops_compatibility.md b/tensorflow/lite/g3doc/tf_ops_compatibility.md
index dc6ba8a463..d20320a3b5 100644
--- a/tensorflow/lite/g3doc/tf_ops_compatibility.md
+++ b/tensorflow/lite/g3doc/tf_ops_compatibility.md
@@ -373,17 +373,6 @@ outputs: {
 }
 ```
 
-**ABS**
-
-```
-inputs {
-  0: tensor
-}
-outputs: {
-  0: result of computing element-wise absolute value of the input tensor
-}
-```
-
 **FULLY_CONNECTED**
 
 ```
diff --git a/tensorflow/lite/kernels/BUILD b/tensorflow/lite/kernels/BUILD
index f66f69870f..71d06ba4d7 100644
--- a/tensorflow/lite/kernels/BUILD
+++ b/tensorflow/lite/kernels/BUILD
@@ -158,7 +158,6 @@ cc_library(
 cc_library(
     name = "builtin_op_kernels",
     srcs = [
-        "abs.cc",
         "activations.cc",
         "add.cc",
         "arg_min_max.cc",
@@ -601,22 +600,6 @@ tf_cc_test(
     ],
 )
 
-tf_cc_test(
-    name = "abs_test",
-    size = "small",
-    srcs = ["abs_test.cc"],
-    tags = [
-        "no_oss",
-        "tflite_not_portable_ios",
-    ],
-    deps = [
-        ":builtin_ops",
-        "//tensorflow/lite:framework",
-        "//tensorflow/lite/kernels:test_util",
-        "@com_google_googletest//:gtest",
-    ],
-)
-
 tf_cc_test(
     name = "ceil_test",
     size = "small",
diff --git a/tensorflow/lite/kernels/abs.cc b/tensorflow/lite/kernels/abs.cc
deleted file mode 100644
index b44f9de630..0000000000
--- a/tensorflow/lite/kernels/abs.cc
+++ /dev/null
@@ -1,59 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/lite/c/c_api_internal.h"
-#include "tensorflow/lite/kernels/internal/optimized/optimized_ops.h"
-#include "tensorflow/lite/kernels/internal/tensor.h"
-#include "tensorflow/lite/kernels/kernel_util.h"
-
-namespace tflite {
-namespace ops {
-namespace builtin {
-namespace abs {
-
-constexpr int kInputTensor = 0;
-constexpr int kOutputTensor = 0;
-
-TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
-  const TfLiteTensor* input = GetInput(context, node, kInputTensor);
-  TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
-  TF_LITE_ENSURE_EQ(context, NumInputs(node), 1);
-  TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1);
-  TF_LITE_ENSURE_EQ(context, input->type, kTfLiteFloat32);
-  output->type = input->type;
-  TfLiteIntArray* output_size = TfLiteIntArrayCopy(input->dims);
-  return context->ResizeTensor(context, output, output_size);
-}
-
-TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
-  const TfLiteTensor* input = GetInput(context, node, kInputTensor);
-  TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
-
-  optimized_ops::Abs(GetTensorShape(input), GetTensorData<float>(input),
-                       GetTensorShape(output), GetTensorData<float>(output));
-
-  return kTfLiteOk;
-}
-}  // namespace abs
-
-TfLiteRegistration* Register_ABS() {
-  static TfLiteRegistration r = {/*init=*/nullptr,
-                                 /*free=*/nullptr, abs::Prepare, abs::Eval};
-  return &r;
-}
-
-}  // namespace builtin
-}  // namespace ops
-}  // namespace tflite
diff --git a/tensorflow/lite/kernels/abs_test.cc b/tensorflow/lite/kernels/abs_test.cc
deleted file mode 100644
index 5e3f95c43a..0000000000
--- a/tensorflow/lite/kernels/abs_test.cc
+++ /dev/null
@@ -1,94 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include <gtest/gtest.h>
-#include "tensorflow/lite/interpreter.h"
-#include "tensorflow/lite/kernels/register.h"
-#include "tensorflow/lite/kernels/test_util.h"
-#include "tensorflow/lite/model.h"
-
-namespace tflite {
-namespace {
-
-using ::testing::ElementsAreArray;
-
-class AbsOpModel : public SingleOpModel {
- public:
-  AbsOpModel(std::initializer_list<int> input_shape, TensorType input_type) {
-    input_ = AddInput(TensorType_FLOAT32);
-    output_ = AddOutput(TensorType_FLOAT32);
-    SetBuiltinOp(BuiltinOperator_ABS, BuiltinOptions_NONE, 0);
-    BuildInterpreter({
-        input_shape,
-    });
-  }
-
-  int input() { return input_; }
-
-  std::vector<float> GetOutput() { return ExtractVector<float>(output_); }
-  std::vector<int> GetOutputShape() { return GetTensorShape(output_); }
-
- private:
-  int input_;
-  int output_;
-};
-
-TEST(AbsOpTest, SingleDim) {
-  AbsOpModel model({2}, TensorType_FLOAT32);
-  model.PopulateTensor<float>(model.input(), {8.5, -2.0});
-  model.Invoke();
-  EXPECT_THAT(model.GetOutput(), ElementsAreArray({8.5, 2.0}));
-  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2}));
-}
-
-TEST(AbsOpTest, MultiDims) {
-  AbsOpModel model({2, 1, 1, 5}, TensorType_FLOAT32);
-  model.PopulateTensor<float>(model.input(), {
-                                                 0.0001,
-                                                 8.0001,
-                                                 0.9999,
-                                                 9.9999,
-                                                 0.5,
-                                                 -0.0001,
-                                                 -8.0001,
-                                                 -0.9999,
-                                                 -9.9999,
-                                                 -0.5,
-                                             });
-  model.Invoke();
-  EXPECT_THAT(model.GetOutput(),
-              ElementsAreArray({
-                                                 0.0001,
-                                                 8.0001,
-                                                 0.9999,
-                                                 9.9999,
-                                                 0.5,
-                                                 0.0001,
-                                                 8.0001,
-                                                 0.9999,
-                                                 9.9999,
-                                                 0.5,
-                                             }));
-  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2, 1, 1, 5}));
-}
-
-}  // namespace
-}  // namespace tflite
-
-int main(int argc, char** argv) {
-  ::tflite::LogToStderr();
-  ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}
diff --git a/tensorflow/lite/kernels/internal/optimized/legacy_optimized_ops.h b/tensorflow/lite/kernels/internal/optimized/legacy_optimized_ops.h
index c24b7faa21..a76649f934 100644
--- a/tensorflow/lite/kernels/internal/optimized/legacy_optimized_ops.h
+++ b/tensorflow/lite/kernels/internal/optimized/legacy_optimized_ops.h
@@ -1746,12 +1746,6 @@ inline void Ceil(const float* input_data, const Dims<4>& input_dims,
        output_data);
 }
 
-inline void Abs(const float* input_data, const Dims<4>& input_dims,
-                float* output_data, const Dims<4>& output_dims) {
-  Abs(DimsToShape(input_dims), input_data, DimsToShape(output_dims),
-      output_data);
-}
-
 inline void ResizeBilinear(const float* input_data, const Dims<4>& input_dims,
                            const int32* output_size_data,
                            const Dims<4>& output_size_dims, float* output_data,
diff --git a/tensorflow/lite/kernels/internal/optimized/optimized_ops.h b/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
index 7f7296ae97..0bd779bf3a 100644
--- a/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
+++ b/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
@@ -4906,14 +4906,6 @@ inline void Ceil(const RuntimeShape& input_shape, const float* input_data,
   output_map.array() = Eigen::ceil(input_map.array());
 }
 
-inline void Abs(const RuntimeShape& input_shape, const float* input_data,
-                const RuntimeShape& output_shape, float* output_data) {
-  gemmlowp::ScopedProfilingLabel label("Abs");
-  auto input_map = MapAsVector(input_data, input_shape);
-  auto output_map = MapAsVector(output_data, output_shape);
-  output_map.array() = Eigen::abs(input_map.array());
-}
-
 #ifdef USE_NEON
 inline void ResizeBilinearKernel(const float* input_ptr, int32 depth,
                                  float scale, float* output_ptr) {
diff --git a/tensorflow/lite/kernels/internal/reference/legacy_reference_ops.h b/tensorflow/lite/kernels/internal/reference/legacy_reference_ops.h
index 30bb92c8b3..431e2413e8 100644
--- a/tensorflow/lite/kernels/internal/reference/legacy_reference_ops.h
+++ b/tensorflow/lite/kernels/internal/reference/legacy_reference_ops.h
@@ -1889,12 +1889,6 @@ inline void Ceil(const float* input_data, const Dims<4>& input_dims,
        output_data);
 }
 
-inline void Abs(const float* input_data, const Dims<4>& input_dims,
-                float* output_data, const Dims<4>& output_dims) {
-  Abs(DimsToShape(input_dims), input_data, DimsToShape(output_dims),
-      output_data);
-}
-
 template <typename T>
 inline void ResizeBilinear(const T* input_data, const Dims<4>& input_dims,
                            const int32* output_size_data,
diff --git a/tensorflow/lite/kernels/internal/reference/reference_ops.h b/tensorflow/lite/kernels/internal/reference/reference_ops.h
index 133a455e9e..6b8d817ca1 100644
--- a/tensorflow/lite/kernels/internal/reference/reference_ops.h
+++ b/tensorflow/lite/kernels/internal/reference/reference_ops.h
@@ -3050,16 +3050,6 @@ inline void Ceil(const RuntimeShape& input_shape, const float* input_data,
   }
 }
 
-inline void Abs(const RuntimeShape& input_shape, const float* input_data,
-                const RuntimeShape& output_shape, float* output_data) {
-  const int flat_size = MatchingFlatSize(input_shape, output_shape);
-
-  for (int i = 0; i < flat_size; i++) {
-    int offset = i;
-    output_data[offset] = std::fabs(input_data[offset]);
-  }
-}
-
 template <typename T, typename CoordsT = int32>
 inline void Gather(const tflite::GatherParams& op_params,
                    const RuntimeShape& input_shape, const T* input_data,
diff --git a/tensorflow/lite/kernels/register.cc b/tensorflow/lite/kernels/register.cc
index caf61e9003..47ae934c4b 100644
--- a/tensorflow/lite/kernels/register.cc
+++ b/tensorflow/lite/kernels/register.cc
@@ -95,7 +95,6 @@ TfLiteRegistration* Register_LESS();
 TfLiteRegistration* Register_LESS_EQUAL();
 TfLiteRegistration* Register_FLOOR();
 TfLiteRegistration* Register_CEIL();
-TfLiteRegistration* Register_ABS();
 TfLiteRegistration* Register_TILE();
 TfLiteRegistration* Register_NEG();
 TfLiteRegistration* Register_SUM();
@@ -238,7 +237,6 @@ BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_LESS_EQUAL, Register_LESS_EQUAL());
   AddBuiltin(BuiltinOperator_FLOOR, Register_FLOOR());
   AddBuiltin(BuiltinOperator_CEIL, Register_CEIL());
-  AddBuiltin(BuiltinOperator_ABS, Register_ABS());
   AddBuiltin(BuiltinOperator_NEG, Register_NEG());
   AddBuiltin(BuiltinOperator_SELECT, Register_SELECT());
   AddBuiltin(BuiltinOperator_SLICE, Register_SLICE());
diff --git a/tensorflow/lite/nnapi/NeuralNetworksShim.h b/tensorflow/lite/nnapi/NeuralNetworksShim.h
index 037d364c3b..82c5840952 100644
--- a/tensorflow/lite/nnapi/NeuralNetworksShim.h
+++ b/tensorflow/lite/nnapi/NeuralNetworksShim.h
@@ -144,7 +144,6 @@ enum {
   ANEURALNETWORKS_SUB = 36,
   ANEURALNETWORKS_TRANSPOSE = 37,
   ANEURALNETWORKS_CEIL = 38,
-  ANEURALNETWORKS_ABS = 39,
 };
 
 /**
diff --git a/tensorflow/lite/nnapi_delegate.cc b/tensorflow/lite/nnapi_delegate.cc
index 6da803ce7b..dfbb4813ad 100644
--- a/tensorflow/lite/nnapi_delegate.cc
+++ b/tensorflow/lite/nnapi_delegate.cc
@@ -492,9 +492,6 @@ TfLiteStatus AddOpsAndParams(
       case tflite::BuiltinOperator_CEIL:
         nn_op_type = ANEURALNETWORKS_CEIL;
         break;
-      case tflite::BuiltinOperator_ABS:
-        nn_op_type = ANEURALNETWORKS_ABS;
-        break;
       case tflite::BuiltinOperator_LOGISTIC:
         nn_op_type = ANEURALNETWORKS_LOGISTIC;
         break;
diff --git a/tensorflow/lite/testing/generate_examples.py b/tensorflow/lite/testing/generate_examples.py
index 825873a884..1dea01d59b 100644
--- a/tensorflow/lite/testing/generate_examples.py
+++ b/tensorflow/lite/testing/generate_examples.py
@@ -3047,32 +3047,6 @@ def make_ceil_tests(zip_path):
   make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
 
 
-def make_abs_tests(zip_path):
-  """Make a set of tests to do abs."""
-
-  test_parameters = [{
-      "input_dtype": [tf.float32],
-      "input_shape": [[1], [1, 2], [5, 6, 7, 8], [3, 4, 5, 6]],
-  }]
-
-  def build_graph(parameters):
-    """Build the abs op testing graph."""
-    input_value = tf.placeholder(
-        dtype=parameters["input_dtype"],
-        name="input1",
-        shape=parameters["input_shape"])
-    out = tf.abs(input_value)
-    return [input_value], [out]
-
-  def build_inputs(parameters, sess, inputs, outputs):
-    input_value = create_tensor_data(parameters["input_dtype"],
-                                     parameters["input_shape"])
-    return [input_value], sess.run(
-        outputs, feed_dict={inputs[0]: input_value})
-
-  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
-
-
 def make_neg_tests(zip_path):
   """Make a set of tests to do neg."""
 
diff --git a/tensorflow/lite/toco/export_tensorflow.cc b/tensorflow/lite/toco/export_tensorflow.cc
index 9dfeaad164..816cb9bc28 100644
--- a/tensorflow/lite/toco/export_tensorflow.cc
+++ b/tensorflow/lite/toco/export_tensorflow.cc
@@ -1215,16 +1215,6 @@ void ConvertCeilOperator(const Model& model, const CeilOperator& src_op,
   (*ceil_op->mutable_attr())["T"].set_type(DT_FLOAT);
 }
 
-void ConvertAbsOperator(const Model& model, const AbsOperator& src_op,
-                        GraphDef* tensorflow_graph) {
-  tensorflow::NodeDef* abs_op = tensorflow_graph->add_node();
-  abs_op->set_op("Abs");
-  abs_op->set_name(src_op.outputs[0]);
-  CHECK_EQ(src_op.inputs.size(), 1);
-  *abs_op->add_input() = src_op.inputs[0];
-  (*abs_op->mutable_attr())["T"].set_type(DT_FLOAT);
-}
-
 void ConvertGatherOperator(const Model& model, const GatherOperator& src_op,
                            GraphDef* tensorflow_graph) {
   tensorflow::NodeDef* gather_op = tensorflow_graph->add_node();
@@ -2192,9 +2182,6 @@ void ConvertOperator(const Model& model, const Operator& src_op,
   } else if (src_op.type == OperatorType::kCeil) {
     ConvertCeilOperator(model, static_cast<const CeilOperator&>(src_op),
                         tensorflow_graph);
-  } else if (src_op.type == OperatorType::kAbs) {
-    ConvertAbsOperator(model, static_cast<const AbsOperator&>(src_op),
-                       tensorflow_graph);
   } else if (src_op.type == OperatorType::kGather) {
     ConvertGatherOperator(model, static_cast<const GatherOperator&>(src_op),
                           tensorflow_graph);
diff --git a/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc b/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
index c6c7681c03..0aae68c9c4 100644
--- a/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
+++ b/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
@@ -1870,7 +1870,6 @@ void ProcessMirrorPadOperator(Model* model, MirrorPadOperator* op) {
     case OperatorType::kCast:
     case OperatorType::kFloor:
     case OperatorType::kCeil:
-    case OperatorType::kAbs:
     case OperatorType::kExp:
     case OperatorType::kSin:
     case OperatorType::kLogicalAnd:
diff --git a/tensorflow/lite/toco/graph_transformations/reorder_elementwise_unary.cc b/tensorflow/lite/toco/graph_transformations/reorder_elementwise_unary.cc
index 90e64816c8..98105d384e 100644
--- a/tensorflow/lite/toco/graph_transformations/reorder_elementwise_unary.cc
+++ b/tensorflow/lite/toco/graph_transformations/reorder_elementwise_unary.cc
@@ -29,7 +29,6 @@ namespace {
 
 bool IsElementwiseOperator(OperatorType optype) {
   switch (optype) {
-    case OperatorType::kAbs:
     case OperatorType::kCast:
     case OperatorType::kCeil:
     case OperatorType::kExp:
diff --git a/tensorflow/lite/toco/import_tensorflow.cc b/tensorflow/lite/toco/import_tensorflow.cc
index de8ae19a71..53defed6cb 100644
--- a/tensorflow/lite/toco/import_tensorflow.cc
+++ b/tensorflow/lite/toco/import_tensorflow.cc
@@ -1505,20 +1505,6 @@ tensorflow::Status ConvertCeilOperator(
   return tensorflow::Status::OK();
 }
 
-tensorflow::Status ConvertAbsOperator(
-    const NodeDef& node, const TensorFlowImportFlags& tf_import_flags,
-    Model* model) {
-  CHECK_EQ(node.op(), "Abs");
-  TF_QCHECK_OK(CheckInputsCount(node, tf_import_flags, 1));
-  const auto data_type = GetDataTypeAttr(node, "T");
-  CHECK(data_type == DT_FLOAT);
-  auto* op = new AbsOperator;
-  op->inputs.push_back(node.input(0));
-  op->outputs.push_back(node.name());
-  model->operators.emplace_back(op);
-  return tensorflow::Status::OK();
-}
-
 tensorflow::Status ConvertGatherOperator(
     const NodeDef& node, const TensorFlowImportFlags& tf_import_flags,
     Model* model) {
diff --git a/tensorflow/lite/toco/model.h b/tensorflow/lite/toco/model.h
index 69c6d7608a..c5cb215114 100644
--- a/tensorflow/lite/toco/model.h
+++ b/tensorflow/lite/toco/model.h
@@ -37,7 +37,6 @@ using tflite::QuantizationParams;
 enum class OperatorType : uint8 {
   kNone,
   // General-purpose neural network operators.
-  kAbs,
   kAdd,
   kAddN,
   kAveragePool,
@@ -1670,16 +1669,6 @@ struct CeilOperator : Operator {
   CeilOperator() : Operator(OperatorType::kCeil) {}
 };
 
-// Abs operator.
-//
-// Inputs:
-//   inputs[0]: required: the input array
-//
-// TensorFlow equivalent: Abs
-struct AbsOperator : Operator {
-  AbsOperator() : Operator(OperatorType::kAbs) {}
-};
-
 // Gather operator. It gathers slices from params according to indices.
 // Only 1-D indices are supported at the moment.
 //
diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index 2122d04958..dd47808046 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -1651,8 +1651,6 @@ std::vector<std::unique_ptr<BaseOperator>> BuildOperatorList(
       MakeUnique<SimpleOperator<FloorOperator>>("FLOOR", OperatorType::kFloor));
   ops.push_back(
       MakeUnique<SimpleOperator<CeilOperator>>("CEIL", OperatorType::kCeil));
-  ops.push_back(
-      MakeUnique<SimpleOperator<AbsOperator>>("ABS", OperatorType::kAbs));
   ops.push_back(
       MakeUnique<SimpleOperator<ReluOperator>>("RELU", OperatorType::kRelu));
   ops.push_back(MakeUnique<SimpleOperator<Relu1Operator>>(
diff --git a/tensorflow/lite/toco/tflite/operator_test.cc b/tensorflow/lite/toco/tflite/operator_test.cc
index e284619903..ba713edcb6 100644
--- a/tensorflow/lite/toco/tflite/operator_test.cc
+++ b/tensorflow/lite/toco/tflite/operator_test.cc
@@ -115,7 +115,6 @@ TEST_F(OperatorTest, SimpleOperators) {
                                           OperatorType::kDequantize);
   CheckSimpleOperator<FloorOperator>("FLOOR", OperatorType::kFloor);
   CheckSimpleOperator<CeilOperator>("CEIL", OperatorType::kCeil);
-  CheckSimpleOperator<AbsOperator>("ABS", OperatorType::kAbs);
   CheckSimpleOperator<ReluOperator>("RELU", OperatorType::kRelu);
   CheckSimpleOperator<Relu1Operator>("RELU_N1_TO_1", OperatorType::kRelu1);
   CheckSimpleOperator<Relu6Operator>("RELU6", OperatorType::kRelu6);
diff --git a/tensorflow/lite/toco/tooling_util.cc b/tensorflow/lite/toco/tooling_util.cc
index 0887dca9b5..9d6f554c59 100644
--- a/tensorflow/lite/toco/tooling_util.cc
+++ b/tensorflow/lite/toco/tooling_util.cc
@@ -386,7 +386,6 @@ const char* OperatorTypeName(OperatorType type) {
     HANDLE_OPERATORTYPENAME_CASE(Cast)
     HANDLE_OPERATORTYPENAME_CASE(Floor)
     HANDLE_OPERATORTYPENAME_CASE(Ceil)
-    HANDLE_OPERATORTYPENAME_CASE(Abs)
     HANDLE_OPERATORTYPENAME_CASE(Gather)
     HANDLE_OPERATORTYPENAME_CASE(ResizeBilinear)
     HANDLE_OPERATORTYPENAME_CASE(SpaceToBatchND)
-- 
GitLab


From 9d5ac5c511dca4a959014b6f9882309bc9bd703f Mon Sep 17 00:00:00 2001
From: Yves-Noel Weweler <y.weweler@gmail.com>
Date: Thu, 13 Dec 2018 22:40:14 +0100
Subject: [PATCH 0029/1095] Fix typos

---
 tensorflow/python/data/experimental/ops/grouping.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/data/experimental/ops/grouping.py b/tensorflow/python/data/experimental/ops/grouping.py
index 71e4b3391f..2eef36d0c8 100644
--- a/tensorflow/python/data/experimental/ops/grouping.py
+++ b/tensorflow/python/data/experimental/ops/grouping.py
@@ -162,8 +162,8 @@ def bucket_by_sequence_length(element_length_func,
     no_padding: `bool`, indicates whether to pad the batch features (features
       need to be either of type `tf.SparseTensor` or of same shape).
     drop_remainder: (Optional.) A `tf.bool` scalar `tf.Tensor`, representing
-      whether the last batch should be dropped in the case its has fewer than
-      batch_size` elements; the default behavior is not to drop the smaller
+      whether the last batch should be dropped in the case it has fewer than
+      `batch_size` elements; the default behavior is not to drop the smaller
       batch.
 
   Returns:
-- 
GitLab


From a06497bce4e91b267850b8b82ee4fe684c01e94b Mon Sep 17 00:00:00 2001
From: Steve Lang <stevelang789@users.noreply.github.com>
Date: Fri, 14 Dec 2018 09:01:27 +1100
Subject: [PATCH 0030/1095] .numpy() was missing in code example: tf.add(1,
 2).numpy()

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 68d7e180d1..c512aeb06c 100644
--- a/README.md
+++ b/README.md
@@ -65,7 +65,7 @@ $ python
 ```python
 >>> import tensorflow as tf
 >>> tf.enable_eager_execution()
->>> tf.add(1, 2)
+>>> tf.add(1, 2).numpy()
 3
 >>> hello = tf.constant('Hello, TensorFlow!')
 >>> hello.numpy()
-- 
GitLab


From c8c5f69e9a4d2424ba4cc603daebd4c24e5128d4 Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Thu, 13 Dec 2018 14:45:48 -0800
Subject: [PATCH 0031/1095] [Go]: Fixup paths to protocol buffers.

Without this: go generate github.com/tensorflow/tensorflow/tensorflow/go/op
would fail with:
../genop/internal/api_def_map.go:34:2: cannot find package "github.com/tensorflow/tensorflow/tensorflow/go/genop/internal/proto/tensorflow/core/framework_go_proto" in any of:
        /usr/local/go/src/github.com/tensorflow/tensorflow/tensorflow/go/genop/internal/proto/tensorflow/core/framework_go_proto (from $GOROOT)
        /go/src/github.com/tensorflow/tensorflow/tensorflow/go/genop/internal/proto/tensorflow/core/framework_go_proto (from $GOPATH)

This breakage was probably introduced by https://github.com/tensorflow/tensorflow/pull/17262
---
 tensorflow/go/genop/internal/api_def_map.go | 2 +-
 tensorflow/go/genop/internal/genop.go       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/go/genop/internal/api_def_map.go b/tensorflow/go/genop/internal/api_def_map.go
index 8600452b47..0bbd88b61c 100644
--- a/tensorflow/go/genop/internal/api_def_map.go
+++ b/tensorflow/go/genop/internal/api_def_map.go
@@ -31,7 +31,7 @@ import (
 	"unsafe"
 
 	"github.com/golang/protobuf/proto"
-	pb "github.com/tensorflow/tensorflow/tensorflow/go/genop/internal/proto/tensorflow/core/framework_go_proto"
+	pb "github.com/tensorflow/tensorflow/tensorflow/go/genop/internal/proto/github.com/tensorflow/tensorflow/tensorflow/go/core/framework"
 )
 
 // Encapsulates a collection of API definitions.
diff --git a/tensorflow/go/genop/internal/genop.go b/tensorflow/go/genop/internal/genop.go
index fb81631218..1c05715a1a 100644
--- a/tensorflow/go/genop/internal/genop.go
+++ b/tensorflow/go/genop/internal/genop.go
@@ -47,7 +47,7 @@ import (
 	"unsafe"
 
 	"github.com/golang/protobuf/proto"
-	pb "github.com/tensorflow/tensorflow/tensorflow/go/genop/internal/proto/tensorflow/core/framework_go_proto"
+	pb "github.com/tensorflow/tensorflow/tensorflow/go/genop/internal/proto/github.com/tensorflow/tensorflow/tensorflow/go/core/framework"
 )
 
 // GenerateFunctionsForRegisteredOps writes a Go source code file to w
-- 
GitLab


From c1fc5d5036d850be852ec473f32292ca973f5bed Mon Sep 17 00:00:00 2001
From: Yves-Noel Weweler <y.weweler@gmail.com>
Date: Thu, 13 Dec 2018 22:57:22 +0100
Subject: [PATCH 0032/1095] Rewrite grouping tests to be @parameterized tests

---
 .../data/experimental/kernel_tests/BUILD      |  1 +
 .../bucket_by_sequence_length_test.py         | 60 ++++++++++++-------
 2 files changed, 38 insertions(+), 23 deletions(-)

diff --git a/tensorflow/python/data/experimental/kernel_tests/BUILD b/tensorflow/python/data/experimental/kernel_tests/BUILD
index d7ca5a70e4..897e949b0f 100644
--- a/tensorflow/python/data/experimental/kernel_tests/BUILD
+++ b/tensorflow/python/data/experimental/kernel_tests/BUILD
@@ -23,6 +23,7 @@ py_test(
         "//tensorflow/python/data/kernel_tests:test_base",
         "//tensorflow/python/data/ops:dataset_ops",
         "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py b/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
index fab79619a0..bcb7ef9496 100644
--- a/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
@@ -19,6 +19,8 @@ from __future__ import print_function
 
 import random
 
+from absl.testing import parameterized
+
 from tensorflow.python.data.experimental.ops import grouping
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
@@ -69,9 +71,13 @@ def _get_record_shape(sparse):
   return tensor_shape.TensorShape([None])
 
 
-class BucketBySequenceLengthTest(test_base.DatasetTestBase):
+class BucketBySequenceLengthTest(test_base.DatasetTestBase, parameterized.TestCase):
 
-  def testBucketDropReminder(self):
+  @parameterized.named_parameters(
+    ("WithoutPadding", True),
+    ("WithPadding", False),
+  )
+  def testBucketDropReminder(self, param_no_padding):
 
     boundaries = [10, 20, 30]
     batch_sizes = [10, 8, 4, 2]
@@ -192,10 +198,13 @@ class BucketBySequenceLengthTest(test_base.DatasetTestBase):
                        .format(sorted(expected_lengths),
                                sorted(generated_lengths)))
 
-    for no_padding in (True, False):
-      _test_bucket_by_padding(no_padding)
+    _test_bucket_by_padding(param_no_padding)
 
-  def testBucket(self):
+  @parameterized.named_parameters(
+      ("WithoutPadding", True),
+      ("WithPadding", False),
+  )
+  def testBucket(self, param_no_padding):
 
     boundaries = [10, 20, 30]
     batch_sizes = [10, 8, 4, 2]
@@ -251,8 +260,7 @@ class BucketBySequenceLengthTest(test_base.DatasetTestBase):
       self.assertEqual(sorted(batch_sizes), sorted(batch_sizes_val))
       self.assertEqual(sorted(lengths), sorted(lengths_val))
 
-    for no_padding in (True, False):
-      _test_bucket_by_padding(no_padding)
+    _test_bucket_by_padding(param_no_padding)
 
   def testPadToBoundary(self):
 
@@ -336,7 +344,11 @@ class BucketBySequenceLengthTest(test_base.DatasetTestBase):
     self.assertAllEqual(batches[4], [[1, 1, 1, 1, 1, 1, 1, 1, 1, 0],
                                      [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])
 
-  def testTupleElements(self):
+  @parameterized.named_parameters(
+      ("WithoutPadding", True),
+      ("WithPadding", False),
+  )
+  def testTupleElements(self, param_no_padding):
 
     def build_dataset(sparse):
       def _generator():
@@ -364,10 +376,13 @@ class BucketBySequenceLengthTest(test_base.DatasetTestBase):
       self.assertEqual([None, None], shapes[0].as_list())
       self.assertEqual([None], shapes[1].as_list())
 
-    for no_padding in (True, False):
-      _test_tuple_elements_by_padding(no_padding)
+    _test_tuple_elements_by_padding(param_no_padding)
 
-  def testBucketSparse(self):
+  @parameterized.named_parameters(
+      ("DoDropRemainder", True),
+      ("DoNotDropRemainder", False),
+  )
+  def testBucketSparse(self, param_drop_remainder):
     """Tests bucketing of sparse tensors (case where `no_padding` == True).
 
     Test runs on following dataset:
@@ -435,18 +450,17 @@ class BucketBySequenceLengthTest(test_base.DatasetTestBase):
             all_sparse_tensors.add(sprs_tensor)
       return all_sparse_tensors
 
-    for drop_remainder in (True, False):
-      dataset = _build_dataset()
-      boundaries = range(min_len + bucket_size + 1, max_len, bucket_size)
-      dataset = dataset.apply(grouping.bucket_by_sequence_length(
-          _element_length_fn,
-          boundaries,
-          [batch_size] * (len(boundaries) + 1),
-          no_padding=True,
-          drop_remainder=drop_remainder))
-      batches = _compute_batches(dataset)
-      expected_batches = _compute_expected_batches(drop_remainder)
-      self.assertEqual(batches, expected_batches)
+    dataset = _build_dataset()
+    boundaries = range(min_len + bucket_size + 1, max_len, bucket_size)
+    dataset = dataset.apply(grouping.bucket_by_sequence_length(
+        _element_length_fn,
+        boundaries,
+        [batch_size] * (len(boundaries) + 1),
+        no_padding=True,
+        drop_remainder=param_drop_remainder))
+    batches = _compute_batches(dataset)
+    expected_batches = _compute_expected_batches(param_drop_remainder)
+    self.assertEqual(batches, expected_batches)
 
 
 if __name__ == "__main__":
-- 
GitLab


From 8db73e896985d043f24592abf405fed4c867aee8 Mon Sep 17 00:00:00 2001
From: Pavel Samolysov <samolisov@gmail.com>
Date: Mon, 17 Dec 2018 18:32:45 +0300
Subject: [PATCH 0033/1095] [OpenMP] Fix undeclared identifier in
 eigen_support.cc

When OpenMP is enabled, the following error occurs during a
compilation of the `tensorflow/lite/kernels/eigen_support.cc` unit:

tensorflow/lite/kernels/eigen_support.cc:42:23: error: use of undeclared
identifier 'context'
  Eigen::setNbThreads(context->recommended_num_threads);

The `SetEigenNbThreads` method already gets the number of threads as the
`threads` parameter and doesn't need to calculate it using a method of
the `context` variable, so the invocation of the `Eigen::setNbThreads`
member must be changed a little bit.

Signed-off-by: Pavel Samolysov <samolisov@gmail.com>
---
 tensorflow/lite/kernels/eigen_support.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/kernels/eigen_support.cc b/tensorflow/lite/kernels/eigen_support.cc
index bad5975a7c..e2a2c4aac9 100644
--- a/tensorflow/lite/kernels/eigen_support.cc
+++ b/tensorflow/lite/kernels/eigen_support.cc
@@ -39,7 +39,7 @@ void SetEigenNbThreads(int threads) {
 #if defined(EIGEN_HAS_OPENMP)
   // The global Eigen thread count is only used when OpenMP is enabled. As this
   // call causes problems with tsan, make it only when OpenMP is available.
-  Eigen::setNbThreads(context->recommended_num_threads);
+  Eigen::setNbThreads(threads);
 #endif  // defined(EIGEN_HAS_OPENMP)
 }
 
-- 
GitLab


From c5720d1626c3c7e85069b0436bffacc8c6ca8122 Mon Sep 17 00:00:00 2001
From: Yves-Noel Weweler <y.weweler@gmail.com>
Date: Wed, 19 Dec 2018 11:54:28 +0100
Subject: [PATCH 0034/1095] Update testBucketDropReminder test case

See #24071
---
 .../bucket_by_sequence_length_test.py         | 20 ++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py b/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
index e0978676fd..a95d8e1049 100644
--- a/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
@@ -76,11 +76,12 @@ def _get_record_shape(sparse):
 @test_util.run_all_in_graph_and_eager_modes
 class BucketBySequenceLengthTest(test_base.DatasetTestBase, parameterized.TestCase):
 
+  # TODO(b/117581999): add eager coverage.
   @parameterized.named_parameters(
     ("WithoutPadding", True),
     ("WithPadding", False),
   )
-  def testBucketDropReminder(self, param_no_padding):
+  def testSkipEagerBucketDropReminder(self, param_no_padding):
 
     boundaries = [10, 20, 30]
     batch_sizes = [10, 8, 4, 2]
@@ -142,14 +143,15 @@ class BucketBySequenceLengthTest(test_base.DatasetTestBase, parameterized.TestCa
               batch_sizes,
               no_padding=no_padding,
               drop_remainder=True))
-      batch, = dataset.make_one_shot_iterator().get_next()
-
-      with self.cached_session() as sess:
-        batches = []
-        for _ in range(n_expected_batches):
-          batches.append(self.evaluate(batch))
-        with self.assertRaises(errors.OutOfRangeError):
-          self.evaluate(batch)
+
+      get_next = self.getNext(dataset)
+      batches = []
+      for _ in range(n_expected_batches):
+        batch, = self.evaluate(get_next())
+        batches.append(batch)
+
+      with self.assertRaises(errors.OutOfRangeError):
+        self.evaluate(get_next())
 
       generated_lengths = []
 
-- 
GitLab


From 5bbdac505efbc4decaf7f15b247a26060fe25d46 Mon Sep 17 00:00:00 2001
From: Mark Daoust <markdaoust@google.com>
Date: Wed, 19 Dec 2018 12:39:03 -0800
Subject: [PATCH 0035/1095] Update lookup_ops.py

---
 tensorflow/python/ops/lookup_ops.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/python/ops/lookup_ops.py b/tensorflow/python/ops/lookup_ops.py
index 9302696a45..70bbbb72e6 100644
--- a/tensorflow/python/ops/lookup_ops.py
+++ b/tensorflow/python/ops/lookup_ops.py
@@ -64,6 +64,7 @@ def initialize_all_tables(name="init_all_tables"):
 @tf_export(v1=["initializers.tables_initializer", "tables_initializer"])
 def tables_initializer(name="init_all_tables"):
   """Returns an Op that initializes all tables of the default graph.
+  
   See the [Low Level Intro](https://www.tensorflow.org/guide/low_level_intro#feature_columns)
   guide, for an example of usage.
 
-- 
GitLab


From 9e92c2fe264debad1548554182dcc9af06d2e6ee Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Wed, 19 Dec 2018 13:48:25 -0800
Subject: [PATCH 0036/1095] Fix bug with renaming output bindings

---
 tensorflow/contrib/tensorrt/BUILD             |  1 +
 .../contrib/tensorrt/convert/convert_nodes.cc | 17 +++++
 .../tensorrt/test/identity_output_test.py     | 72 +++++++++++++++++++
 3 files changed, 90 insertions(+)
 create mode 100644 tensorflow/contrib/tensorrt/test/identity_output_test.py

diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index 784acce444..3e8c486d8c 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -491,6 +491,7 @@ cuda_py_tests(
         "test/binary_tensor_weight_broadcast_test.py",
         "test/concatenation_test.py",
         "test/const_broadcast_test.py",
+        "test/identity_output_test.py",
         "test/manual_test.py",
         "test/memory_alignment_test.py",
         "test/multi_connection_neighbor_engine_test.py",
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index adf8831b96..729afe6c64 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -879,6 +879,8 @@ Status Converter::ConvertNode(const NodeDef& node_def) {
     // We need to check the name before setting it. If the input is one of the
     // engine input, setting the name here will overwrite engine input
     // bindings which will cause runtime error.
+    // TODO(tmorris): Remove this work-around once we use TRT's IIdentityLayer
+    // in ConvertIdentity.
     if (output.is_tensor()) {
       const char* tensor_name = output.tensor()->getName();
       if (!tensorflow::str_util::StartsWith(tensor_name, kInputPHName)) {
@@ -939,6 +941,21 @@ Status Converter::RenameAndMarkOutputTensors(
     if (tensor == nullptr) {
       return errors::NotFound("Output tensor not found: ", output.first);
     }
+    // Check if this tensor has already been marked as an output.
+    // ConvertIdentity can cause the same tensor to be repeated in
+    // output_tensors, which can cause us to overwrite the name of the output
+    // tensor binding. For example, if we rename OutputPH_0 to OutputPH_1 then
+    // we won't be able to locate OutputPH_0 during runtime. To fix this,
+    // duplicate the tensor using no-op shuffle.
+    // TODO(tmorris): Remove this work-around once we use TRT's IIdentityLayer
+    // in ConvertIdentity.
+    if (tensorflow::str_util::StartsWith(tensor->getName(), kOutputPHName)) {
+      nvinfer1::IShuffleLayer* layer = network()->addShuffle(*tensor);
+      TFTRT_RETURN_ERROR_IF_NULLPTR(
+          layer, StrCat("Output Copy for ", tensor->getName()));
+      MarkQuantizationRangesAsInferrable(tensor, layer->getOutput(0));
+      tensor = layer->getOutput(0);
+    }
     tensor->setName(output.second.c_str());
     VLOG(1) << "Marking output tensor " << output.first << ", as output tensor "
             << output.second;
diff --git a/tensorflow/contrib/tensorrt/test/identity_output_test.py b/tensorflow/contrib/tensorrt/test/identity_output_test.py
new file mode 100644
index 0000000000..391434ba83
--- /dev/null
+++ b/tensorflow/contrib/tensorrt/test/identity_output_test.py
@@ -0,0 +1,72 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Model script to test TF-TensorRT integration."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import gen_array_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import nn
+from tensorflow.python.platform import test
+
+
+class IdentityTest(trt_test.TfTrtIntegrationTestBase):
+
+  def _ConstOp(self, shape):
+    return constant_op.constant(np.random.randn(*shape), dtype=dtypes.float32)
+
+  def GetParams(self):
+    """Testing conversion of BiasAdd MatMul in TF-TRT conversion."""
+    input_name = "input"
+    input_dims = [100, 32]
+    g = ops.Graph()
+    with g.as_default():
+      x = array_ops.placeholder(
+          dtype=dtypes.float32, shape=input_dims, name=input_name)
+
+      b = self._ConstOp((32, 4))
+      x1 = math_ops.matmul(x, b)
+      b = self._ConstOp((1, 4))
+      x1 = x1 + b
+
+      out1 = array_ops.identity(x1, name='output1')
+      out2 = array_ops.identity(x1, name='output2')
+      iden1 = array_ops.identity(x1)
+      out3 = array_ops.identity(iden1, name='output3')
+
+    return trt_test.TfTrtIntegrationTestParams(
+        gdef=g.as_graph_def(),
+        input_names=[input_name],
+        input_dims=[input_dims],
+        output_names=['output1', 'output2', 'output3'],
+        expected_output_dims=[(100, 4), (100, 4), (100, 4)])
+
+
+  def ExpectedEnginesToBuild(self, run_params):
+    """Return the expected engines to build."""
+    return ["TRTEngineOp_0"]
+
+
+if __name__ == "__main__":
+  test.main()
-- 
GitLab


From cfe7df2cf064f9eacdf8ab30574ebe9ec6438544 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Wed, 19 Dec 2018 14:14:28 -0800
Subject: [PATCH 0037/1095] Update comment in test

---
 tensorflow/contrib/tensorrt/test/identity_output_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/tensorrt/test/identity_output_test.py b/tensorflow/contrib/tensorrt/test/identity_output_test.py
index 391434ba83..1aa05999bd 100644
--- a/tensorflow/contrib/tensorrt/test/identity_output_test.py
+++ b/tensorflow/contrib/tensorrt/test/identity_output_test.py
@@ -37,7 +37,7 @@ class IdentityTest(trt_test.TfTrtIntegrationTestBase):
     return constant_op.constant(np.random.randn(*shape), dtype=dtypes.float32)
 
   def GetParams(self):
-    """Testing conversion of BiasAdd MatMul in TF-TRT conversion."""
+    """Testing engine with the same tensor repeated as output via identity."""
     input_name = "input"
     input_dims = [100, 32]
     g = ops.Graph()
-- 
GitLab


From f03cf64b1295ea4e9e431a7a524cc23b2c1f3306 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Wed, 19 Dec 2018 15:10:58 -0800
Subject: [PATCH 0038/1095] Explain usage of shuffle layer

---
 tensorflow/contrib/tensorrt/convert/convert_nodes.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 729afe6c64..2a402fe699 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -950,6 +950,7 @@ Status Converter::RenameAndMarkOutputTensors(
     // TODO(tmorris): Remove this work-around once we use TRT's IIdentityLayer
     // in ConvertIdentity.
     if (tensorflow::str_util::StartsWith(tensor->getName(), kOutputPHName)) {
+      // Using shuffle layer for identity by not setting reshape or transpose.
       nvinfer1::IShuffleLayer* layer = network()->addShuffle(*tensor);
       TFTRT_RETURN_ERROR_IF_NULLPTR(
           layer, StrCat("Output Copy for ", tensor->getName()));
-- 
GitLab


From d489a943a0fbb995d48ee70558270c4a5f0f9c39 Mon Sep 17 00:00:00 2001
From: Taylor Thornton <thornton.tn@gmail.com>
Date: Tue, 18 Dec 2018 22:19:59 -0800
Subject: [PATCH 0039/1095] upgrade aws-sdk-cpp to 1.5.8 in order to pick up
 the auth retry changeset that landed in that version

---
 third_party/aws/BUILD.bazel   |  5 +++++
 third_party/aws/workspace.bzl | 11 +++++++----
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/third_party/aws/BUILD.bazel b/third_party/aws/BUILD.bazel
index 5426f79e46..1f61858777 100644
--- a/third_party/aws/BUILD.bazel
+++ b/third_party/aws/BUILD.bazel
@@ -80,6 +80,11 @@ cc_library(
     deps = [
         "@curl",
     ],
+    copts = [
+        "-DAWS_SDK_VERSION_MAJOR=1", 
+        "-DAWS_SDK_VERSION_MINOR=5", 
+        "-DAWS_SDK_VERSION_PATCH=8"
+    ],
 )
 
 template_rule(
diff --git a/third_party/aws/workspace.bzl b/third_party/aws/workspace.bzl
index c216638154..10799b5153 100644
--- a/third_party/aws/workspace.bzl
+++ b/third_party/aws/workspace.bzl
@@ -2,14 +2,17 @@
 
 load("//third_party:repo.bzl", "third_party_http_archive")
 
+# NOTE: version updates here should also update the major, minor, and patch variables declared in 
+# the  copts field of the //third_party/aws:aws target
+
 def repo():
     third_party_http_archive(
         name = "aws",
         urls = [
-            "https://mirror.bazel.build/github.com/aws/aws-sdk-cpp/archive/1.3.15.tar.gz",
-            "https://github.com/aws/aws-sdk-cpp/archive/1.3.15.tar.gz",
+            "https://mirror.bazel.build/github.com/aws/aws-sdk-cpp/archive/1.5.8.tar.gz",
+            "https://github.com/aws/aws-sdk-cpp/archive/1.5.8.tar.gz",
         ],
-        sha256 = "b888d8ce5fc10254c3dd6c9020c7764dd53cf39cf011249d0b4deda895de1b7c",
-        strip_prefix = "aws-sdk-cpp-1.3.15",
+        sha256 = "89905075fe50aa13e0337ff905c2e8c1ce9caf77a3504484a7cda39179120ffc",
+        strip_prefix = "aws-sdk-cpp-1.5.8",
         build_file = "//third_party/aws:BUILD.bazel",
     )
-- 
GitLab


From c75023169590ef0534579758285d6a1d8ad54ad2 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Wed, 19 Dec 2018 23:38:51 +0000
Subject: [PATCH 0040/1095] Replace deprecated FastGFile with GFile

FastGFile has been deprecated and replaced with GFile, though
the example in speech_commands still uses FastGFile. This fix
fix the issue to remove the deprecated warning:
```
WARNING:tensorflow:From <stdin>:1: __init__ (from tensorflow.python.platform.gfile) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.gfile.GFile.
```

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/examples/speech_commands/label_wav.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/examples/speech_commands/label_wav.py b/tensorflow/examples/speech_commands/label_wav.py
index 0017aec3a5..eb8323454c 100644
--- a/tensorflow/examples/speech_commands/label_wav.py
+++ b/tensorflow/examples/speech_commands/label_wav.py
@@ -45,7 +45,7 @@ FLAGS = None
 
 def load_graph(filename):
   """Unpersists graph from file as default graph."""
-  with tf.gfile.FastGFile(filename, 'rb') as f:
+  with tf.gfile.GFile(filename, 'rb') as f:
     graph_def = tf.GraphDef()
     graph_def.ParseFromString(f.read())
     tf.import_graph_def(graph_def, name='')
-- 
GitLab


From 4696da4bf9a586cf250b32a062da833af3ffd4a9 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Wed, 19 Dec 2018 23:41:52 +0000
Subject: [PATCH 0041/1095] Also replace FastGFile to GFile in label_wav_dir.py

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/examples/speech_commands/label_wav_dir.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/examples/speech_commands/label_wav_dir.py b/tensorflow/examples/speech_commands/label_wav_dir.py
index a34db512dd..2e1890c3e8 100644
--- a/tensorflow/examples/speech_commands/label_wav_dir.py
+++ b/tensorflow/examples/speech_commands/label_wav_dir.py
@@ -46,7 +46,7 @@ FLAGS = None
 
 def load_graph(filename):
   """Unpersists graph from file as default graph."""
-  with tf.gfile.FastGFile(filename, 'rb') as f:
+  with tf.gfile.GFile(filename, 'rb') as f:
     graph_def = tf.GraphDef()
     graph_def.ParseFromString(f.read())
     tf.import_graph_def(graph_def, name='')
-- 
GitLab


From 0f10d9fcd0c47b497d1bd5069f28d21f94fedc60 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Wed, 19 Dec 2018 15:56:19 -0800
Subject: [PATCH 0042/1095] Improve comment

---
 tensorflow/contrib/tensorrt/test/identity_output_test.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/tensorrt/test/identity_output_test.py b/tensorflow/contrib/tensorrt/test/identity_output_test.py
index 1aa05999bd..2be9da9ede 100644
--- a/tensorflow/contrib/tensorrt/test/identity_output_test.py
+++ b/tensorflow/contrib/tensorrt/test/identity_output_test.py
@@ -12,7 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Model script to test TF-TensorRT integration."""
+"""This test checks a situation where the same tensor is considered as an output
+multiple times because it has been duplicated by 2+ indentity ops. Previously,
+the tensor would be renamed multiple times, overwriting the output binding name
+which resulted in a runtime error when the binding would not be found.
+"""
 
 from __future__ import absolute_import
 from __future__ import division
-- 
GitLab


From 915b8783a05b0da7c30ba36531ce03c811930852 Mon Sep 17 00:00:00 2001
From: Clayne Robison <clayne.b.robison@intel.com>
Date: Wed, 19 Dec 2018 19:13:27 -0700
Subject: [PATCH 0043/1095] [Intel MKL] Remove TensorFlow lite test from the
 public CI. MKL does not support TF lite.

---
 tensorflow/tools/ci_build/linux/cpu/run_mkl.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/ci_build/linux/cpu/run_mkl.sh b/tensorflow/tools/ci_build/linux/cpu/run_mkl.sh
index 7be5f454ec..a8b73cbe0c 100755
--- a/tensorflow/tools/ci_build/linux/cpu/run_mkl.sh
+++ b/tensorflow/tools/ci_build/linux/cpu/run_mkl.sh
@@ -36,4 +36,4 @@ yes "" | $PYTHON_BIN_PATH configure.py
 bazel test --test_tag_filters=-no_oss,-oss_serial,-gpu,-benchmark-test --test_lang_filters=cc,py -k \
     --jobs=${N_JOBS} --test_timeout 300,450,1200,3600 --build_tests_only \
     --config=mkl --test_env=KMP_BLOCKTIME=0 --config=opt --test_output=errors -- \
-    //tensorflow/... -//tensorflow/compiler/... -//tensorflow/contrib/...
+    //tensorflow/... -//tensorflow/compiler/... -//tensorflow/contrib/... -//tensorflow/lite/...
-- 
GitLab


From f4867d3e4fecdb1ca8a445addad383aa0292a3b3 Mon Sep 17 00:00:00 2001
From: Sami Kama <skama@nvidia.com>
Date: Wed, 19 Dec 2018 19:40:22 -0800
Subject: [PATCH 0044/1095] Initialize only CPU devices in optimize_dataset_op

---
 tensorflow/core/grappler/grappler_item_builder.cc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/grappler/grappler_item_builder.cc b/tensorflow/core/grappler/grappler_item_builder.cc
index 9224ee7849..a984efd10d 100644
--- a/tensorflow/core/grappler/grappler_item_builder.cc
+++ b/tensorflow/core/grappler/grappler_item_builder.cc
@@ -103,7 +103,11 @@ Status OptimizeGraph(const GraphDef& graph_def_arg, GraphDef* output_graph_def,
 
   // Instantiate all variables for function library runtime creation.
   std::vector<std::unique_ptr<Device>> devices;
-  TF_RETURN_IF_ERROR(DeviceFactory::AddDevices(
+  // Only CPU device is used so instead of calling DeviceFactory::AddDevices()
+  // with dummy session config, which will conflict with user defined options and
+  // create unwanted devices, call cpu_factory->CreateDevices() to get CPU only devices.
+  DeviceFactory* cpu_factory = DeviceFactory::GetFactory("CPU");
+  TF_RETURN_IF_ERROR(cpu_factory->CreateDevices(
       options, "/job:localhost/replica:0/task:0", &devices));
   Device* cpu_device = devices[0].get();
   std::unique_ptr<DeviceMgr> dvc_mgr(new DeviceMgr(std::move(devices)));
-- 
GitLab


From 85ef6de9fcb5977bd738e10264f0641869594b83 Mon Sep 17 00:00:00 2001
From: Sami Kama <skama@nvidia.com>
Date: Wed, 19 Dec 2018 19:52:47 -0800
Subject: [PATCH 0045/1095] Fix clang format

---
 tensorflow/core/grappler/grappler_item_builder.cc | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/grappler/grappler_item_builder.cc b/tensorflow/core/grappler/grappler_item_builder.cc
index a984efd10d..fc55fb5b3d 100644
--- a/tensorflow/core/grappler/grappler_item_builder.cc
+++ b/tensorflow/core/grappler/grappler_item_builder.cc
@@ -104,8 +104,9 @@ Status OptimizeGraph(const GraphDef& graph_def_arg, GraphDef* output_graph_def,
   // Instantiate all variables for function library runtime creation.
   std::vector<std::unique_ptr<Device>> devices;
   // Only CPU device is used so instead of calling DeviceFactory::AddDevices()
-  // with dummy session config, which will conflict with user defined options and
-  // create unwanted devices, call cpu_factory->CreateDevices() to get CPU only devices.
+  // with dummy session config, which will conflict with user defined options
+  // and create unwanted devices, call cpu_factory->CreateDevices() to get CPU
+  // only devices.
   DeviceFactory* cpu_factory = DeviceFactory::GetFactory("CPU");
   TF_RETURN_IF_ERROR(cpu_factory->CreateDevices(
       options, "/job:localhost/replica:0/task:0", &devices));
-- 
GitLab


From 2565842e71e1665d8a4fc126edc40b37becfed71 Mon Sep 17 00:00:00 2001
From: Innovimax <innovimax@gmail.com>
Date: Thu, 20 Dec 2018 14:11:27 +0100
Subject: [PATCH 0046/1095] fix typo

---
 tensorflow/java/src/gen/cc/op_gen_main.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/java/src/gen/cc/op_gen_main.cc b/tensorflow/java/src/gen/cc/op_gen_main.cc
index 0d9e0883af..cf4bb03dad 100644
--- a/tensorflow/java/src/gen/cc/op_gen_main.cc
+++ b/tensorflow/java/src/gen/cc/op_gen_main.cc
@@ -35,7 +35,7 @@ const char kUsageHeader[] =
     "graph.\n\n"
     "Operation wrappers are generated under the path specified by the "
     "'--output_dir' argument. This path can be absolute or relative to the\n"
-    "current working directory and will be created if it does not exists.\n\n"
+    "current working directory and will be created if it does not exist.\n\n"
     "Note that the operations will not be available through the "
     "'org.tensorflow.op.Ops' API until the generated classes are compiled\n"
     "using an appropriate annotation processor.\n\n"
-- 
GitLab


From 6cbd27b8ffb0ee33ae0f3151a202ba8dd0fa22a9 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Thu, 20 Dec 2018 10:26:17 -0800
Subject: [PATCH 0047/1095] Only apply size limit if size of tensor is
 increasing

---
 tensorflow/core/grappler/optimizers/constant_folding.cc | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/constant_folding.cc b/tensorflow/core/grappler/optimizers/constant_folding.cc
index 3882e3b3a9..bb7ed05e33 100644
--- a/tensorflow/core/grappler/optimizers/constant_folding.cc
+++ b/tensorflow/core/grappler/optimizers/constant_folding.cc
@@ -913,6 +913,7 @@ Status ConstantFolding::CreateNodeDef(const string& name,
   AttrValue attr_tensor;
   TensorProto* t = attr_tensor.mutable_tensor();
   bool optimized = false;
+  const size_t original_size = tensor->TotalBytes();
   size_t encoded_size;
   // Use the packed representation whenever possible to avoid generating large
   // graphdefs. Moreover, avoid repeating the last values if they're equal.
@@ -980,11 +981,11 @@ Status ConstantFolding::CreateNodeDef(const string& name,
   }
   node->mutable_attr()->insert({"value", attr_tensor});
 
-  if (encoded_size < 10 * 1024 * 1024) {
-    return Status::OK();
+  if (encoded_size > original_size && encoded_size >= 10 * 1024 * 1024) {
+    return errors::InvalidArgument(
+        strings::StrCat("Can't fold ", name, ", its size would be too large"));
   }
-  return errors::InvalidArgument(
-      strings::StrCat("Can't fold ", name, ", its size would be too large"));
+  return Status::OK();
 }
 
 Status ConstantFolding::EvaluateNode(const NodeDef& node,
-- 
GitLab


From a9ba4a47efb80d2951ee6deea8e2d8de56dbaaf9 Mon Sep 17 00:00:00 2001
From: vitor-alves <vitoralves842@gmail.com>
Date: Thu, 20 Dec 2018 17:26:45 -0200
Subject: [PATCH 0048/1095] Typo

---
 tensorflow/lite/examples/label_image/label_image.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/examples/label_image/label_image.md b/tensorflow/lite/examples/label_image/label_image.md
index fd9f49918b..debe8ffbc5 100644
--- a/tensorflow/lite/examples/label_image/label_image.md
+++ b/tensorflow/lite/examples/label_image/label_image.md
@@ -51,7 +51,7 @@ average time: 100.986 ms
 0.0235294: 514 cornet
 0.0196078: 835 suit
 ```
-Run `interpreter->Invoker()` 100 times:
+Run `interpreter->Invoke()` 100 times:
 ```
 > ./label_image   -c 100                               
 Loaded model ./mobilenet_quant_v1_224.tflite
-- 
GitLab


From beed5ef6fc98f513f368e68ae08357a8db623ef0 Mon Sep 17 00:00:00 2001
From: Pooya Davoodi <pdavoodi@nvidia.com>
Date: Thu, 20 Dec 2018 11:38:58 -0800
Subject: [PATCH 0049/1095] TFTRT: Convert between str and unicode in py2

Also check for types before doing conversion.
---
 tensorflow/contrib/tensorrt/python/trt_convert.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/python/trt_convert.py b/tensorflow/contrib/tensorrt/python/trt_convert.py
index 203b2697ba..3d2b6b499d 100644
--- a/tensorflow/contrib/tensorrt/python/trt_convert.py
+++ b/tensorflow/contrib/tensorrt/python/trt_convert.py
@@ -46,11 +46,13 @@ from tensorflow.python.saved_model import tag_constants
 from tensorflow.python.training import saver
 
 if _six.PY2:
-  _to_bytes = lambda s: s
-  _to_string = lambda s: s
+  _to_bytes = lambda s: s.encode("utf-8", errors="surrogateescape") \
+    if isinstance(s, unicode) else s
+  _to_string = lambda s: s.decode("utf-8") if isinstance(s, str) else s
 else:
-  _to_bytes = lambda s: s.encode("utf-8", errors="surrogateescape")
-  _to_string = lambda s: s.decode("utf-8")
+  _to_bytes = lambda s: s.encode("utf-8", errors="surrogateescape") \
+    if isinstance(s, str) else s
+  _to_string = lambda s: s.decode("utf-8") if isinstance(s, bytes) else s
 
 
 class TrtPrecisionMode(object):
-- 
GitLab


From 58621c1db6ee0ac52aab0ff868f666454d2aa5a3 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Thu, 20 Dec 2018 11:47:58 -0800
Subject: [PATCH 0050/1095] Calculate input size correctly

---
 .../core/grappler/optimizers/constant_folding.cc      | 11 ++++++-----
 .../core/grappler/optimizers/constant_folding.h       |  2 +-
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/constant_folding.cc b/tensorflow/core/grappler/optimizers/constant_folding.cc
index bb7ed05e33..5a053d3a89 100644
--- a/tensorflow/core/grappler/optimizers/constant_folding.cc
+++ b/tensorflow/core/grappler/optimizers/constant_folding.cc
@@ -902,7 +902,8 @@ DataType GetDataTypeFromNodeOrProps(const NodeDef& node,
 // static
 Status ConstantFolding::CreateNodeDef(const string& name,
                                       const TensorValue& tensor,
-                                      NodeDef* node) {
+                                      NodeDef* node,
+                                      size_t input_size) {
   node->set_name(name);
   node->set_op("Const");
 
@@ -913,7 +914,6 @@ Status ConstantFolding::CreateNodeDef(const string& name,
   AttrValue attr_tensor;
   TensorProto* t = attr_tensor.mutable_tensor();
   bool optimized = false;
-  const size_t original_size = tensor->TotalBytes();
   size_t encoded_size;
   // Use the packed representation whenever possible to avoid generating large
   // graphdefs. Moreover, avoid repeating the last values if they're equal.
@@ -980,8 +980,7 @@ Status ConstantFolding::CreateNodeDef(const string& name,
     encoded_size = t->tensor_content().size();
   }
   node->mutable_attr()->insert({"value", attr_tensor});
-
-  if (encoded_size > original_size && encoded_size >= 10 * 1024 * 1024) {
+  if (encoded_size > input_size && encoded_size >= 10 * 1024 * 1024) {
     return errors::InvalidArgument(
         strings::StrCat("Can't fold ", name, ", its size would be too large"));
   }
@@ -1011,6 +1010,7 @@ Status ConstantFolding::EvaluateOneFoldable(const NodeDef& node,
     }
   });
 
+  size_t input_size = 0;
   for (const auto& input : node.input()) {
     const TensorId input_tensor = ParseTensorName(input);
     if (input_tensor.index() < 0) {
@@ -1025,6 +1025,7 @@ Status ConstantFolding::EvaluateOneFoldable(const NodeDef& node,
     }
     TF_RETURN_IF_ERROR(CheckAttrExists(*input_node, "value"));
     const TensorProto& raw_val = input_node->attr().at("value").tensor();
+    input_size += raw_val.tensor_content().size();
     Tensor* value = new Tensor(raw_val.dtype(), raw_val.tensor_shape());
     CHECK(value->FromProto(raw_val));
     inputs.emplace_back(value);
@@ -1042,7 +1043,7 @@ Status ConstantFolding::EvaluateOneFoldable(const NodeDef& node,
       node_name = strings::StrCat(node_name, "-", i);
     }
     if (output_tensors[i].tensor) {
-      Status s = CreateNodeDef(node_name, output_tensors[i], &outputs->at(i));
+      Status s = CreateNodeDef(node_name, output_tensors[i], &outputs->at(i), input_size);
       if (!s.ok()) {
         *result_too_large = true;
         return s;
diff --git a/tensorflow/core/grappler/optimizers/constant_folding.h b/tensorflow/core/grappler/optimizers/constant_folding.h
index 0b778882d7..8843ac161f 100644
--- a/tensorflow/core/grappler/optimizers/constant_folding.h
+++ b/tensorflow/core/grappler/optimizers/constant_folding.h
@@ -36,7 +36,7 @@ const char kConstantFoldingCtrl[] = "ConstantFoldingCtrl";
 class ConstantFolding : public GraphOptimizer {
  public:
   static Status CreateNodeDef(const string& name, const TensorValue& tensor,
-                              NodeDef* node);
+                              NodeDef* node, size_t input_size = 0);
   static string AddControlDependency(const string& input_name, GraphDef* graph,
                                      NodeMap* node_map);
 
-- 
GitLab


From b71a6b031adf0da61db59312847cccb675771b84 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Thu, 20 Dec 2018 14:51:38 -0800
Subject: [PATCH 0051/1095] Fix input size, add comment

---
 tensorflow/core/grappler/optimizers/constant_folding.cc | 7 ++++---
 tensorflow/core/grappler/optimizers/constant_folding.h  | 2 ++
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/constant_folding.cc b/tensorflow/core/grappler/optimizers/constant_folding.cc
index 5a053d3a89..a5f6f57747 100644
--- a/tensorflow/core/grappler/optimizers/constant_folding.cc
+++ b/tensorflow/core/grappler/optimizers/constant_folding.cc
@@ -1010,7 +1010,7 @@ Status ConstantFolding::EvaluateOneFoldable(const NodeDef& node,
     }
   });
 
-  size_t input_size = 0;
+  size_t total_inputs_size = 0;
   for (const auto& input : node.input()) {
     const TensorId input_tensor = ParseTensorName(input);
     if (input_tensor.index() < 0) {
@@ -1025,10 +1025,10 @@ Status ConstantFolding::EvaluateOneFoldable(const NodeDef& node,
     }
     TF_RETURN_IF_ERROR(CheckAttrExists(*input_node, "value"));
     const TensorProto& raw_val = input_node->attr().at("value").tensor();
-    input_size += raw_val.tensor_content().size();
     Tensor* value = new Tensor(raw_val.dtype(), raw_val.tensor_shape());
     CHECK(value->FromProto(raw_val));
     inputs.emplace_back(value);
+    total_inputs_size += value->TotalBytes();
   }
 
   TF_RETURN_IF_ERROR(EvaluateNode(node, inputs, &output_tensors));
@@ -1043,7 +1043,8 @@ Status ConstantFolding::EvaluateOneFoldable(const NodeDef& node,
       node_name = strings::StrCat(node_name, "-", i);
     }
     if (output_tensors[i].tensor) {
-      Status s = CreateNodeDef(node_name, output_tensors[i], &outputs->at(i), input_size);
+      Status s = CreateNodeDef(node_name, output_tensors[i], &outputs->at(i),
+                               total_inputs_size);
       if (!s.ok()) {
         *result_too_large = true;
         return s;
diff --git a/tensorflow/core/grappler/optimizers/constant_folding.h b/tensorflow/core/grappler/optimizers/constant_folding.h
index 8843ac161f..58a962ba9b 100644
--- a/tensorflow/core/grappler/optimizers/constant_folding.h
+++ b/tensorflow/core/grappler/optimizers/constant_folding.h
@@ -35,6 +35,8 @@ const char kConstantFoldingCtrl[] = "ConstantFoldingCtrl";
 // Constant folding optimization for a graph.
 class ConstantFolding : public GraphOptimizer {
  public:
+  // The size limit will only be considered if the newly created node is greater
+  // than input_size (optional).
   static Status CreateNodeDef(const string& name, const TensorValue& tensor,
                               NodeDef* node, size_t input_size = 0);
   static string AddControlDependency(const string& input_name, GraphDef* graph,
-- 
GitLab


From 676e0be5b166d08e4af5c3044e889c66de6837c9 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Thu, 20 Dec 2018 15:16:11 -0800
Subject: [PATCH 0052/1095] Rename input_size -> original_size

---
 tensorflow/core/grappler/optimizers/constant_folding.cc | 5 +++--
 tensorflow/core/grappler/optimizers/constant_folding.h  | 4 ++--
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/constant_folding.cc b/tensorflow/core/grappler/optimizers/constant_folding.cc
index a5f6f57747..89be08800d 100644
--- a/tensorflow/core/grappler/optimizers/constant_folding.cc
+++ b/tensorflow/core/grappler/optimizers/constant_folding.cc
@@ -903,7 +903,7 @@ DataType GetDataTypeFromNodeOrProps(const NodeDef& node,
 Status ConstantFolding::CreateNodeDef(const string& name,
                                       const TensorValue& tensor,
                                       NodeDef* node,
-                                      size_t input_size) {
+                                      size_t original_size) {
   node->set_name(name);
   node->set_op("Const");
 
@@ -980,7 +980,8 @@ Status ConstantFolding::CreateNodeDef(const string& name,
     encoded_size = t->tensor_content().size();
   }
   node->mutable_attr()->insert({"value", attr_tensor});
-  if (encoded_size > input_size && encoded_size >= 10 * 1024 * 1024) {
+
+  if (encoded_size > original_size && encoded_size >= 10 * 1024 * 1024) {
     return errors::InvalidArgument(
         strings::StrCat("Can't fold ", name, ", its size would be too large"));
   }
diff --git a/tensorflow/core/grappler/optimizers/constant_folding.h b/tensorflow/core/grappler/optimizers/constant_folding.h
index 58a962ba9b..bb86c4aeb3 100644
--- a/tensorflow/core/grappler/optimizers/constant_folding.h
+++ b/tensorflow/core/grappler/optimizers/constant_folding.h
@@ -36,9 +36,9 @@ const char kConstantFoldingCtrl[] = "ConstantFoldingCtrl";
 class ConstantFolding : public GraphOptimizer {
  public:
   // The size limit will only be considered if the newly created node is greater
-  // than input_size (optional).
+  // than original_size (optional).
   static Status CreateNodeDef(const string& name, const TensorValue& tensor,
-                              NodeDef* node, size_t input_size = 0);
+                              NodeDef* node, size_t original_size = 0);
   static string AddControlDependency(const string& input_name, GraphDef* graph,
                                      NodeMap* node_map);
 
-- 
GitLab


From 79bc4f4bf914d4b94eef89e8df2e19ff54c2db71 Mon Sep 17 00:00:00 2001
From: Siju Samuel <siju.samuel@huawei.com>
Date: Fri, 21 Dec 2018 10:31:19 +0530
Subject: [PATCH 0053/1095] Review comments updated, nnapi removed for ceil

---
 .../lite/delegates/nnapi/nnapi_delegate.cc    | 10 ----
 .../delegates/nnapi/nnapi_delegate_test.cc    | 49 -------------------
 tensorflow/lite/kernels/BUILD                 |  1 -
 .../internal/optimized/legacy_optimized_ops.h |  6 ---
 .../internal/reference/legacy_reference_ops.h |  6 ---
 tensorflow/lite/nnapi/NeuralNetworksShim.h    |  1 -
 tensorflow/lite/nnapi_delegate.cc             |  3 --
 7 files changed, 76 deletions(-)

diff --git a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
index cac98ae3da..4fe07004a8 100644
--- a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
+++ b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
@@ -638,16 +638,6 @@ class NNAPIDelegateKernel {
           return nullptr;
         }
         break;
-      case kTfLiteBuiltinCeil:
-        if (version == 1) {
-          return [](const NNAPIOpMappingArgs& mapping_args)
-                     -> ANeuralNetworksOperationType {
-            return ANEURALNETWORKS_CEIL;
-          };
-        } else {
-          return nullptr;
-        }
-        break;
       case kTfLiteBuiltinRelu:
         if (version == 1) {
           return [](const NNAPIOpMappingArgs& mapping_args)
diff --git a/tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc b/tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc
index f6a04e36cd..ca48af0c95 100644
--- a/tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc
+++ b/tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc
@@ -1024,55 +1024,6 @@ TEST(NNAPIDelegate, FloorMultiDims) {
   EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2, 1, 1, 5}));
 }
 
-class CeilOpModel : public SingleOpModelWithNNAPI {
- public:
-  CeilOpModel(std::initializer_list<int> input_shape, TensorType input_type) {
-    input_ = AddInput(TensorType_FLOAT32);
-    output_ = AddOutput(TensorType_FLOAT32);
-    SetBuiltinOp(BuiltinOperator_CEIL, BuiltinOptions_NONE, 0);
-    BuildInterpreter({
-        input_shape,
-    });
-  }
-
-  int input() { return input_; }
-
-  std::vector<float> GetOutput() { return ExtractVector<float>(output_); }
-  std::vector<int> GetOutputShape() { return GetTensorShape(output_); }
-
- private:
-  int input_;
-  int output_;
-};
-
-TEST(NNAPIDelegate, CeilSingleDim) {
-  CeilOpModel model({2}, TensorType_FLOAT32);
-  model.PopulateTensor<float>(model.input(), {8.5, 0.0});
-  model.Invoke();
-  EXPECT_THAT(model.GetOutput(), ElementsAreArray({8, 0}));
-  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2}));
-}
-
-TEST(NNAPIDelegate, CeilMultiDims) {
-  CeilOpModel model({2, 1, 1, 5}, TensorType_FLOAT32);
-  model.PopulateTensor<float>(model.input(), {
-                                                 0.0001,
-                                                 8.0001,
-                                                 0.9999,
-                                                 9.9999,
-                                                 0.5,
-                                                 -0.0001,
-                                                 -8.0001,
-                                                 -0.9999,
-                                                 -9.9999,
-                                                 -0.5,
-                                             });
-  model.Invoke();
-  EXPECT_THAT(model.GetOutput(),
-              ElementsAreArray({1, 9, 1, 10, 1, 0, -8, 0, -9, 0}));
-  EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2, 1, 1, 5}));
-}
-
 class LocalResponseNormOpModel : public SingleOpModelWithNNAPI {
  public:
   LocalResponseNormOpModel(std::initializer_list<int> input_shape, int radius,
diff --git a/tensorflow/lite/kernels/BUILD b/tensorflow/lite/kernels/BUILD
index 71d06ba4d7..1d53022e03 100644
--- a/tensorflow/lite/kernels/BUILD
+++ b/tensorflow/lite/kernels/BUILD
@@ -605,7 +605,6 @@ tf_cc_test(
     size = "small",
     srcs = ["ceil_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
diff --git a/tensorflow/lite/kernels/internal/optimized/legacy_optimized_ops.h b/tensorflow/lite/kernels/internal/optimized/legacy_optimized_ops.h
index a76649f934..5485d907c2 100644
--- a/tensorflow/lite/kernels/internal/optimized/legacy_optimized_ops.h
+++ b/tensorflow/lite/kernels/internal/optimized/legacy_optimized_ops.h
@@ -1740,12 +1740,6 @@ inline void Floor(const float* input_data, const Dims<4>& input_dims,
         output_data);
 }
 
-inline void Ceil(const float* input_data, const Dims<4>& input_dims,
-                 float* output_data, const Dims<4>& output_dims) {
-  Ceil(DimsToShape(input_dims), input_data, DimsToShape(output_dims),
-       output_data);
-}
-
 inline void ResizeBilinear(const float* input_data, const Dims<4>& input_dims,
                            const int32* output_size_data,
                            const Dims<4>& output_size_dims, float* output_data,
diff --git a/tensorflow/lite/kernels/internal/reference/legacy_reference_ops.h b/tensorflow/lite/kernels/internal/reference/legacy_reference_ops.h
index 431e2413e8..380fc8f98e 100644
--- a/tensorflow/lite/kernels/internal/reference/legacy_reference_ops.h
+++ b/tensorflow/lite/kernels/internal/reference/legacy_reference_ops.h
@@ -1883,12 +1883,6 @@ inline void Floor(const float* input_data, const Dims<4>& input_dims,
         output_data);
 }
 
-inline void Ceil(const float* input_data, const Dims<4>& input_dims,
-                 float* output_data, const Dims<4>& output_dims) {
-  Ceil(DimsToShape(input_dims), input_data, DimsToShape(output_dims),
-       output_data);
-}
-
 template <typename T>
 inline void ResizeBilinear(const T* input_data, const Dims<4>& input_dims,
                            const int32* output_size_data,
diff --git a/tensorflow/lite/nnapi/NeuralNetworksShim.h b/tensorflow/lite/nnapi/NeuralNetworksShim.h
index 82c5840952..c39502f4ac 100644
--- a/tensorflow/lite/nnapi/NeuralNetworksShim.h
+++ b/tensorflow/lite/nnapi/NeuralNetworksShim.h
@@ -143,7 +143,6 @@ enum {
   ANEURALNETWORKS_STRIDED_SLICE = 35,
   ANEURALNETWORKS_SUB = 36,
   ANEURALNETWORKS_TRANSPOSE = 37,
-  ANEURALNETWORKS_CEIL = 38,
 };
 
 /**
diff --git a/tensorflow/lite/nnapi_delegate.cc b/tensorflow/lite/nnapi_delegate.cc
index dfbb4813ad..26d75696a1 100644
--- a/tensorflow/lite/nnapi_delegate.cc
+++ b/tensorflow/lite/nnapi_delegate.cc
@@ -489,9 +489,6 @@ TfLiteStatus AddOpsAndParams(
       case tflite::BuiltinOperator_FLOOR:
         nn_op_type = ANEURALNETWORKS_FLOOR;
         break;
-      case tflite::BuiltinOperator_CEIL:
-        nn_op_type = ANEURALNETWORKS_CEIL;
-        break;
       case tflite::BuiltinOperator_LOGISTIC:
         nn_op_type = ANEURALNETWORKS_LOGISTIC;
         break;
-- 
GitLab


From 8da29925c65dc72b49f693942923519b38dd0242 Mon Sep 17 00:00:00 2001
From: Yves-Noel Weweler <y.weweler@gmail.com>
Date: Fri, 21 Dec 2018 15:58:22 +0100
Subject: [PATCH 0054/1095] Fix pylint warnings from Ubuntu Sanity test

---
 .../kernel_tests/bucket_by_sequence_length_test.py         | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py b/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
index a95d8e1049..4f1ea9f2ee 100644
--- a/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
@@ -74,12 +74,13 @@ def _get_record_shape(sparse):
 
 
 @test_util.run_all_in_graph_and_eager_modes
-class BucketBySequenceLengthTest(test_base.DatasetTestBase, parameterized.TestCase):
+class BucketBySequenceLengthTest(test_base.DatasetTestBase,
+                                 parameterized.TestCase):
 
   # TODO(b/117581999): add eager coverage.
   @parameterized.named_parameters(
-    ("WithoutPadding", True),
-    ("WithPadding", False),
+      ("WithoutPadding", True),
+      ("WithPadding", False),
   )
   def testSkipEagerBucketDropReminder(self, param_no_padding):
 
-- 
GitLab


From 7859702e8a7093f242c167ea027cc227f1f9d048 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Fri, 21 Dec 2018 08:57:29 -0800
Subject: [PATCH 0055/1095] Fix failing test in TRT4.0 - TRT bug with tensors
 rank < 3 in INT8 mode

---
 tensorflow/contrib/tensorrt/test/identity_output_test.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tensorflow/contrib/tensorrt/test/identity_output_test.py b/tensorflow/contrib/tensorrt/test/identity_output_test.py
index 2be9da9ede..da7d70876d 100644
--- a/tensorflow/contrib/tensorrt/test/identity_output_test.py
+++ b/tensorflow/contrib/tensorrt/test/identity_output_test.py
@@ -71,6 +71,11 @@ class IdentityTest(trt_test.TfTrtIntegrationTestBase):
     """Return the expected engines to build."""
     return ["TRTEngineOp_0"]
 
+  def ShouldRunTest(self, run_params):
+    """Whether to run the test."""
+    # TODO(aaroey): Trt 4.0 forbids conversion for tensors with rank <3 in int8
+    # mode, which is a bug. Re-enable this when trt library is fixed.
+    return not trt_test.IsQuantizationMode(run_params.precision_mode)
 
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From 753a3a3d7fa79e390ac8c068dc52acd03e0e4187 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Fri, 21 Dec 2018 09:07:40 -0800
Subject: [PATCH 0056/1095] Report size and size limit with error message

---
 tensorflow/core/grappler/optimizers/constant_folding.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/grappler/optimizers/constant_folding.cc b/tensorflow/core/grappler/optimizers/constant_folding.cc
index 89be08800d..8804dedad1 100644
--- a/tensorflow/core/grappler/optimizers/constant_folding.cc
+++ b/tensorflow/core/grappler/optimizers/constant_folding.cc
@@ -983,7 +983,8 @@ Status ConstantFolding::CreateNodeDef(const string& name,
 
   if (encoded_size > original_size && encoded_size >= 10 * 1024 * 1024) {
     return errors::InvalidArgument(
-        strings::StrCat("Can't fold ", name, ", its size would be too large"));
+        strings::StrCat("Can't fold ", name, ", its size would be too large (",
+                        encoded_size, " >= ", 10 * 1024 * 1024, " bytes)"));
   }
   return Status::OK();
 }
-- 
GitLab


From ab72f61de52f59887d262b23543cc5d5508e753e Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Fri, 21 Dec 2018 14:18:09 -0800
Subject: [PATCH 0057/1095] Fix clang-format

---
 tensorflow/core/grappler/optimizers/constant_folding.cc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/constant_folding.cc b/tensorflow/core/grappler/optimizers/constant_folding.cc
index 8804dedad1..adaf3cd07a 100644
--- a/tensorflow/core/grappler/optimizers/constant_folding.cc
+++ b/tensorflow/core/grappler/optimizers/constant_folding.cc
@@ -901,8 +901,7 @@ DataType GetDataTypeFromNodeOrProps(const NodeDef& node,
 
 // static
 Status ConstantFolding::CreateNodeDef(const string& name,
-                                      const TensorValue& tensor,
-                                      NodeDef* node,
+                                      const TensorValue& tensor, NodeDef* node,
                                       size_t original_size) {
   node->set_name(name);
   node->set_op("Const");
-- 
GitLab


From 6bd27704c472b8d1cbd64b600fe00f223e74b4f6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E7=8E=8B=E6=8C=AF=E5=8D=8E=20=28WANG=20Zhenhua=29?=
 <i@jackwish.net>
Date: Sat, 22 Dec 2018 10:35:47 +0800
Subject: [PATCH 0058/1095] lite: remove memset in resize bilinear opt op

memset() is not necessarily needed here since every element
in output_data memory will be updated.
---
 tensorflow/lite/kernels/internal/optimized/optimized_ops.h | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tensorflow/lite/kernels/internal/optimized/optimized_ops.h b/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
index 7bc6b324c5..a5374827e1 100644
--- a/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
+++ b/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
@@ -5387,9 +5387,6 @@ inline void ResizeBilinearGenericSmallChannel(
     int32 output_height, int32 output_width, float height_scale,
     float width_scale, const RuntimeShape& input_shape, const T* input_data,
     const RuntimeShape& output_shape, T* output_data) {
-  memset(output_data, 0,
-         batches * output_height * output_width * depth * sizeof(T));
-
   T* output_ptr = &output_data[0];
   for (int b = 0; b < batches; ++b) {
     for (int y = 0; y < output_height; ++y) {
-- 
GitLab


From 315e009f917aebe04a754a046a57cb3ac1d10557 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E7=8E=8B=E6=8C=AF=E5=8D=8E=20=28WANG=20Zhenhua=29?=
 <i@jackwish.net>
Date: Sat, 22 Dec 2018 10:36:50 +0800
Subject: [PATCH 0059/1095] lite: perform std::floor before cast float to int

This won't change the resulted value, but to have same
code style in the context and reference_ops.
---
 tensorflow/lite/kernels/internal/optimized/optimized_ops.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/kernels/internal/optimized/optimized_ops.h b/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
index a5374827e1..8f09fab4ca 100644
--- a/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
+++ b/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
@@ -5395,7 +5395,7 @@ inline void ResizeBilinearGenericSmallChannel(
       int32 y1 = std::min(y0 + 1, input_height - 1);
       for (int x = 0; x < output_width; ++x) {
         float input_x = x * width_scale;
-        int32 x0 = static_cast<int32>(input_x);
+        int32 x0 = static_cast<int32>(std::floor((input_x)));
         int32 x1 = std::min(x0 + 1, input_width - 1);
 
         int32 input_offset[4] = {Offset(input_shape, b, y0, x0, 0),
-- 
GitLab


From 6ecb77b9bd800e2beda7a7fcefaf0ac3a6363e1f Mon Sep 17 00:00:00 2001
From: Samantha Andow <samdow@fb.com>
Date: Wed, 26 Dec 2018 08:58:44 -0800
Subject: [PATCH 0060/1095] Fix iterable bug

---
 tensorflow/java/src/gen/cc/op_specs.cc | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tensorflow/java/src/gen/cc/op_specs.cc b/tensorflow/java/src/gen/cc/op_specs.cc
index 4f5a491d25..4024efedef 100644
--- a/tensorflow/java/src/gen/cc/op_specs.cc
+++ b/tensorflow/java/src/gen/cc/op_specs.cc
@@ -91,11 +91,6 @@ class TypeResolver {
 
 Type TypeResolver::TypeOf(const OpDef_ArgDef& arg_def, bool* iterable_out) {
   *iterable_out = false;
-  if (!arg_def.number_attr().empty()) {
-    // when number_attr is set, argument has to be a list of tensors
-    *iterable_out = true;
-    visited_attrs_.insert(std::make_pair(arg_def.number_attr(), Type::Int()));
-  }
   Type type = Type::Wildcard();
   if (arg_def.type() != DataType::DT_INVALID) {
     type = Type::ForDataType(arg_def.type());
@@ -122,6 +117,11 @@ Type TypeResolver::TypeOf(const OpDef_ArgDef& arg_def, bool* iterable_out) {
     LOG(FATAL) << "Cannot resolve data type of argument \"" << arg_def.name()
                << "\" in operation \"" << op_def_.name() << "\"";
   }
+  if (!arg_def.number_attr().empty()) {
+    // when number_attr is set, argument has to be a list of tensors
+    *iterable_out = true;
+    visited_attrs_.insert(std::make_pair(arg_def.number_attr(), Type::Int()));
+  }
   return type;
 }
 
-- 
GitLab


From 17bc7e61e566db5b3288b2afe6dd8cbc1152c2f0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?=
 <facai.yan@gmail.com>
Date: Tue, 18 Dec 2018 08:51:25 +0800
Subject: [PATCH 0061/1095] ENH: The gradient op of bias_add supports 3/4/5D
 NCHW format

---
 tensorflow/core/kernels/bias_op.cc            | 46 +++++++++----------
 tensorflow/core/kernels/bias_op_gpu.cu.cc     |  4 +-
 tensorflow/core/kernels/bias_op_gpu.h         |  2 +-
 .../python/kernel_tests/bias_op_test.py       | 38 ++++++++++++---
 tensorflow/python/ops/nn_grad.py              |  6 +--
 5 files changed, 58 insertions(+), 38 deletions(-)

diff --git a/tensorflow/core/kernels/bias_op.cc b/tensorflow/core/kernels/bias_op.cc
index d4f4b43d63..df12dafdeb 100644
--- a/tensorflow/core/kernels/bias_op.cc
+++ b/tensorflow/core/kernels/bias_op.cc
@@ -18,13 +18,13 @@ limitations under the License.
 #define EIGEN_USE_THREADS
 
 #include "tensorflow/core/kernels/bias_op.h"
-#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/util/tensor_format.h"
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 
 #if GOOGLE_CUDA
 #include "tensorflow/core/kernels/bias_op_gpu.h"
@@ -153,13 +153,13 @@ class BiasOp : public BinaryOp<T> {
               bias.tensor<T, 1>().reshape(four_dims).broadcast(broad_cast_dims);
         } break;
         case 5: {
-          Eigen::DSizes<int32, 5> four_dims(1, channel, 1, 1, 1);
+          Eigen::DSizes<int32, 5> five_dims(1, channel, 1, 1, 1);
           Eigen::DSizes<int32, 5> broad_cast_dims(batch, 1, height, width,
                                                   depth);
           const Device& d = context->eigen_device<Device>();
           output->tensor<T, 5>().device(d) =
               input.tensor<T, 5>() +
-              bias.tensor<T, 1>().reshape(four_dims).broadcast(broad_cast_dims);
+              bias.tensor<T, 1>().reshape(five_dims).broadcast(broad_cast_dims);
         } break;
         default:
           OP_REQUIRES(context, false,
@@ -269,28 +269,24 @@ class BiasGradOp : public OpKernel {
       output->template flat<T>().setZero();
     } else {
       // Added by intel_tf to support NCHW on CPU regardless of MKL used or not.
-      // TODO(yongtang): Add 3/4/5 dimensional data support for NCHW format.
       if (data_format_ == FORMAT_NCHW) {
-        OP_REQUIRES(context, output_backprop.dims() == 4,
-                    errors::InvalidArgument(
-                        "NCHW format supports only 4D input/output tensor."));
-        Eigen::DSizes<Eigen::Index, 4> four_dims(batch, channel, height, width);
+        Eigen::DSizes<Eigen::Index, 3> three_dims(batch, channel,
+                                                  height * width * depth);
 #ifdef EIGEN_HAS_INDEX_LIST
         using idx0 = Eigen::type2index<0>;
         using idx2 = Eigen::type2index<2>;
-        using idx3 = Eigen::type2index<3>;
-        Eigen::IndexList<idx0, idx2, idx3> reduction_axes;
+        Eigen::IndexList<idx0, idx2> reduction_axes;
 #else
-        Eigen::array<Eigen::Index, 3> reduction_axes = {0, 2, 3};
+        Eigen::array<Eigen::Index, 2> reduction_axes = {0, 2};
 #endif
         output->template flat<T>().device(context->eigen_device<Device>()) =
             output_backprop.flat<T>()
                 .template cast<typename AccumulatorType<T>::type>()
-                .reshape(four_dims)
+                .reshape(three_dims)
                 .sum(reduction_axes)
                 .template cast<T>();  // End of code by intel_tf.
       } else {
-        Eigen::DSizes<Eigen::Index, 2> two_dims(batch * height * width,
+        Eigen::DSizes<Eigen::Index, 2> two_dims(batch * height * width * depth,
                                                 channel);
 #ifdef EIGEN_HAS_INDEX_LIST
         Eigen::IndexList<Eigen::type2index<0> > reduction_axis;
@@ -496,21 +492,21 @@ class BiasGradOp<GPUDevice, T> : public OpKernel {
 
   void ComputeWithCustomKernel(OpKernelContext* context,
                                const Tensor& output_backprop, int32 batch,
-                               int32 width, int32 height, int32 channel,
-                               Tensor* output) {
+                               int32 width, int32 height, int32 depth,
+                               int32 channel, Tensor* output) {
     BiasGradGPU<T>::compute(context->template eigen_device<Device>(),
                             output_backprop.template flat<T>().data(),
                             output->flat<T>().data(), batch, width, height,
-                            channel, data_format_);
+                            depth, channel, data_format_);
   }
 
   void ComputeWithReduceSum(OpKernelContext* context,
                             const Tensor& output_backprop, int32 batch,
-                            int32 width, int32 height, int32 channel,
-                            Tensor* output) {
+                            int32 width, int32 height, int32 depth,
+                            int32 channel, Tensor* output) {
     if (data_format_ == FORMAT_NCHW) {
       int32 row_count = batch * channel;
-      int32 col_count = height * width;
+      int32 col_count = height * width * depth;
       Tensor temp_grad_outputs;
       // For 'NCHW' format, we perform reduction twice: first HW, then N.
       TensorShape temp_grad_output_shape{row_count, col_count};
@@ -528,7 +524,7 @@ class BiasGradOp<GPUDevice, T> : public OpKernel {
                                      row_count, col_count);
     } else {
       // For 'NHWC', we simply apply reduction once on NHW.
-      int32 row_count = batch * height * width;
+      int32 row_count = batch * height * width * depth;
       int32 col_count = channel;
       BiasGradGPU<T>::DoColReduction(
           context, const_cast<T*>(output->flat<T>().data()),
@@ -561,7 +557,7 @@ class BiasGradOp<GPUDevice, T> : public OpKernel {
     int device_id = stream->parent()->device_ordinal();
     DataType dtype = output_backprop.dtype();
     BiasAddParams bias_parameters = {
-        {batch, height * width, channel},
+        {batch, height * width * depth, channel},
         data_format_,
         dtype,
         device_id,
@@ -576,7 +572,7 @@ class BiasGradOp<GPUDevice, T> : public OpKernel {
       stream->InitTimer(&timer);
       stream->ThenStartTimer(&timer);
       ComputeWithCustomKernel(context, output_backprop, batch, width, height,
-                              channel, output);
+                              depth, channel, output);
       stream->ThenStopTimer(&timer);
       uint64 elapsed_microseconds = timer.Microseconds();
       VLOG(1) << "BiasAddGrad " << bias_parameters.ToString()
@@ -589,7 +585,7 @@ class BiasGradOp<GPUDevice, T> : public OpKernel {
       // Try reduction and profile.
       stream->ThenStartTimer(&timer);
       ComputeWithReduceSum(context, output_backprop, batch, width, height,
-                           channel, output);
+                           depth, channel, output);
       stream->ThenStopTimer(&timer);
 
       elapsed_microseconds = timer.Microseconds();
@@ -610,11 +606,11 @@ class BiasGradOp<GPUDevice, T> : public OpKernel {
     // Choose the best algorithm based on autotune results.
     if (algo_config.get_mode() == BiasAddGradGPUMode::kReduction) {
       ComputeWithReduceSum(context, output_backprop, batch, width, height,
-                           channel, output);
+                           depth, channel, output);
     } else {
       // Default to the customized kernel.
       ComputeWithCustomKernel(context, output_backprop, batch, width, height,
-                              channel, output);
+                              depth, channel, output);
     }
   }
 
diff --git a/tensorflow/core/kernels/bias_op_gpu.cu.cc b/tensorflow/core/kernels/bias_op_gpu.cu.cc
index 24fea8a8e6..006fa1dc71 100644
--- a/tensorflow/core/kernels/bias_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/bias_op_gpu.cu.cc
@@ -195,10 +195,10 @@ __global__ void BiasGradNCHW_SharedAtomics(const T* output_backprop,
 template <typename T>
 void BiasGradGPU<T>::compute(const GPUDevice& d, const T* output_backprop,
                              T* bias_backprop, int32 batch, int32 height,
-                             int32 width, int32 channel,
+                             int32 width, int32 depth, int32 channel,
                              TensorFormat data_format) {
   const int32 bias_size = channel;
-  const int32 image_size = height * width;
+  const int32 image_size = height * width * depth;
   const int32 total_count = batch * bias_size * image_size;
   if (total_count == 0) {
     return;
diff --git a/tensorflow/core/kernels/bias_op_gpu.h b/tensorflow/core/kernels/bias_op_gpu.h
index a0b2ce4f9b..372a403e68 100644
--- a/tensorflow/core/kernels/bias_op_gpu.h
+++ b/tensorflow/core/kernels/bias_op_gpu.h
@@ -39,7 +39,7 @@ template <typename T>
 struct BiasGradGPU {
   static void compute(const GPUDevice& device, const T* output_backprop,
                       T* bias_backprop, int32 batch, int32 height, int32 width,
-                      int32 channel, TensorFormat data_format);
+                      int32 depth, int32 channel, TensorFormat data_format);
 
   static void DoRowReduction(OpKernelContext* context, T* output,
                              const T* input, int rows, int cols);
diff --git a/tensorflow/python/kernel_tests/bias_op_test.py b/tensorflow/python/kernel_tests/bias_op_test.py
index 66f442dbdd..c3976194a0 100644
--- a/tensorflow/python/kernel_tests/bias_op_test.py
+++ b/tensorflow/python/kernel_tests/bias_op_test.py
@@ -196,9 +196,7 @@ class BiasAddTest(test.TestCase):
       self.assertAllClose(grad_jacob_t, grad_jacob_n, threshold, threshold)
 
   @test_util.run_deprecated_v1
-  def testGradientTensor(self):
-    # TODO(yongtang): BiasAddGrad with NCHW only works 4D. Reenable once
-    # all dimensions are supported.
+  def testGradientTensor2D(self):
     for (data_format, use_gpu) in ("NHWC", False), ("NHWC", True):
       for dtype in (dtypes.float16, dtypes.float32, dtypes.float64):
         np_input = np.array(
@@ -207,9 +205,19 @@ class BiasAddTest(test.TestCase):
         bias = np.array([1.3, 2.4], dtype=dtype.as_numpy_dtype)
         self._testGradient(np_input, bias, dtype, data_format, use_gpu)
 
+  @test_util.run_deprecated_v1
+  def testGradientTensor3D(self):
+    for (data_format, use_gpu) in [("NHWC", False), ("NHWC", True),
+                                   ("NCHW", False), ("NCHW", True)]:
+      for dtype in (dtypes.float16, dtypes.float32, dtypes.float64):
+        np_input = np.array(
+            [1.0, 2.0, 3.0, 4.0, 5.0, 6.0],
+            dtype=dtype.as_numpy_dtype).reshape(1, 3, 2)
+        bias = np.array([1.3, 2.4], dtype=dtype.as_numpy_dtype)
+        self._testGradient(np_input, bias, dtype, data_format, use_gpu)
+
   @test_util.run_deprecated_v1
   def testGradientTensor4D(self):
-    # BiasAddGrad with NCHW support 4D so all are enabled.
     for (data_format, use_gpu) in [("NHWC", False), ("NHWC", True),
                                    ("NCHW", False), ("NCHW", True)]:
       for dtype in (dtypes.float16, dtypes.float32, dtypes.float64):
@@ -219,6 +227,17 @@ class BiasAddTest(test.TestCase):
         bias = np.array([1.3, 2.4], dtype=dtype.as_numpy_dtype)
         self._testGradient(np_input, bias, dtype, data_format, use_gpu)
 
+  @test_util.run_deprecated_v1
+  def testGradientTensor5D(self):
+    for (data_format, use_gpu) in [("NHWC", False), ("NHWC", True),
+                                   ("NCHW", False), ("NCHW", True)]:
+      for dtype in (dtypes.float16, dtypes.float32, dtypes.float64):
+        np_input = np.arange(
+            1.0, 49.0, dtype=dtype.as_numpy_dtype).reshape(
+                [1, 2, 3, 4, 2]).astype(np.float32)
+        bias = np.array([1.3, 2.4], dtype=dtype.as_numpy_dtype)
+        self._testGradient(np_input, bias, dtype, data_format, use_gpu)
+
   @test_util.run_deprecated_v1
   def testEmpty(self):
     np.random.seed(7)
@@ -227,10 +246,15 @@ class BiasAddTest(test.TestCase):
 
   @test_util.run_deprecated_v1
   def testEmptyGradient(self):
-    # TODO(yongtang): BiasAddGrad with NCHW only works 4D. Reenable once
-    # all dimensions are supported.
     for (data_format, use_gpu) in ("NHWC", False), ("NHWC", True):
-      for shape in (0, 0), (2, 0), (0, 2), (4, 3, 0), (4, 0, 3), (0, 4, 3):
+      for shape in (0, 0), (2, 0), (0, 2):
+        self._testGradient(
+            np.random.randn(*shape),
+            np.random.randn(shape[-1]), dtypes.float64, data_format, use_gpu)
+
+    for (data_format, use_gpu) in [("NHWC", False), ("NHWC", True),
+                                   ("NCHW", False), ("NCHW", True)]:
+      for shape in (4, 3, 0), (4, 0, 3), (0, 4, 3):
         self._testGradient(
             np.random.randn(*shape),
             np.random.randn(shape[-1]), dtypes.float64, data_format, use_gpu)
diff --git a/tensorflow/python/ops/nn_grad.py b/tensorflow/python/ops/nn_grad.py
index 34404edc9a..7131e4abc4 100644
--- a/tensorflow/python/ops/nn_grad.py
+++ b/tensorflow/python/ops/nn_grad.py
@@ -314,10 +314,10 @@ def _BiasAddGradGrad(op, received_grad):
 
   if data_format == b"NCHW":
     expanded_shape = array_ops.concat([
-        array_ops.ones_like(shape[:-3]), bias_shape,
-        array_ops.ones_like(shape[-2:])
+        array_ops.ones_like(shape[:1]), bias_shape,
+        array_ops.ones_like(shape[2:])
     ], 0)
-    tile_mults = array_ops.concat([shape[:-3], [1], shape[-2:]], 0)
+    tile_mults = array_ops.concat([shape[:1], [1], shape[2:]], 0)
   else:
     expanded_shape = array_ops.concat(
         [array_ops.ones_like(shape[:-1]), bias_shape], 0)
-- 
GitLab


From 96f41397357f96c41459cf351f8853caa8d724c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aur=C3=A9lien=20Geron?= <ageron@users.noreply.github.com>
Date: Fri, 28 Dec 2018 15:18:06 +0800
Subject: [PATCH 0062/1095] Fix typo in the documentation of tf.function

"`add_noise()` will return a different output every time it is invoked. However, `add_noise` will return the same value every time it is called..."
=> the second `add_noise` should be `traced`
---
 tensorflow/python/eager/def_function.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/eager/def_function.py b/tensorflow/python/eager/def_function.py
index ebc47d1566..b25df5a47c 100644
--- a/tensorflow/python/eager/def_function.py
+++ b/tensorflow/python/eager/def_function.py
@@ -765,7 +765,7 @@ def function(func=None,
   ```
 
   `add_noise()` will return a different output every time it is invoked.
-  However, `add_noise` will return the same value every time it is called,
+  However, `traced()` will return the same value every time it is called,
   since a particular random value generated by the `np.random.randn` call will
   be inserted in the traced/staged TensorFlow graph as a constant. In this
   particular example, replacing `np.random.randn(5, 5)` with
-- 
GitLab


From 844ad0da4bb38192f8ef17a631c0f3a37b85ce5f Mon Sep 17 00:00:00 2001
From: mdfaijul <md.faijul.amin@intel.com>
Date: Fri, 28 Dec 2018 10:23:33 -0800
Subject: [PATCH 0063/1095] fix for conv_ops_test

---
 tensorflow/core/graph/mkl_layout_pass.cc |  18 +++
 tensorflow/core/kernels/conv_ops_test.cc | 134 ++++++++++++-----------
 2 files changed, 90 insertions(+), 62 deletions(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index 9495132f4a..41898eadf9 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -843,6 +843,12 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     CHECK_NOTNULL(m);
     Node* n = nullptr;
 
+    DataType T_m;
+    TF_CHECK_OK(GetNodeAttr(m->def(), "T", &T_m));
+
+    // Don't try to merge if datatype is not DT_FLOAT
+    if (T_m != DT_FLOAT) return n;
+
     if (m->type_string() == csinfo_.bias_add) {
       // If a is BiasAdd, then Conv2D is 0th input of BiasAdd.
       TF_CHECK_OK(m->input_node(0, &n));
@@ -877,6 +883,12 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     DCHECK(m);
     Node* n = nullptr;
 
+    DataType T_m;
+    TF_CHECK_OK(GetNodeAttr(m->def(), "T", &T_m));
+
+    // Don't try to merge if datatype is not DT_FLOAT
+    if (T_m != DT_FLOAT) return n;
+
     const Node* conv_node;
     if (m->type_string() == csinfo_.pad) {
       // If m is Pad, then Conv2D is the output of Pad.
@@ -935,6 +947,12 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     CHECK_NOTNULL(m);
     Node* n = nullptr;
 
+    DataType T_m;
+    TF_CHECK_OK(GetNodeAttr(m->def(), "T", &T_m));
+
+    // Don't try to merge if datatype is not DT_FLOAT
+    if (T_m != DT_FLOAT) return n;
+
     if (m->type_string() == csinfo_.bias_add_grad) {
       // Get 1st input 'g' of BiasAddGrad.
       Node* g = nullptr;
diff --git a/tensorflow/core/kernels/conv_ops_test.cc b/tensorflow/core/kernels/conv_ops_test.cc
index ae4132bb0a..d144576249 100644
--- a/tensorflow/core/kernels/conv_ops_test.cc
+++ b/tensorflow/core/kernels/conv_ops_test.cc
@@ -770,7 +770,15 @@ class FusedConv2DOpTest : public OpsTestBase {
     ASSERT_EQ(conv_2d.dtype(), fused_conv_2d.dtype());
     ASSERT_EQ(conv_2d.shape(), fused_conv_2d.shape());
 
-    test::ExpectClose(conv_2d, fused_conv_2d, /*atol=*/1e-6);
+    // NOTE(intel-tf): When filter_size is equal to the input image size,
+    // conv2d essentially is element-wise multiplication followed by
+    // a full sum reduction, which causes larger numerical error
+    // than usual cases.
+    if (image_width == filter_size && image_height == filter_size) {
+      test::ExpectTensorNear<T>(conv_2d, fused_conv_2d, 1e-4);
+    } else {
+      test::ExpectClose(conv_2d, fused_conv_2d, /*atol=*/1e-6);
+    }
   }
 
   void VerifyFusedBatchNormTensorsNear(int depth, int image_width,
@@ -812,7 +820,15 @@ class FusedConv2DOpTest : public OpsTestBase {
     ASSERT_EQ(conv_2d.dtype(), fused_conv_2d.dtype());
     ASSERT_EQ(conv_2d.shape(), fused_conv_2d.shape());
 
-    test::ExpectClose(conv_2d, fused_conv_2d, /*atol=*/1e-6);
+    // NOTE(intel-tf): When filter_size is equal to the input image size,
+    // conv2d essentially is element-wise multiplication followed by
+    // a full sum reduction, which causes larger numerical error
+    // than usual cases.
+    if (image_width == filter_size && image_height == filter_size) {
+      test::ExpectTensorNear<T>(conv_2d, fused_conv_2d, 1e-4);
+    } else {
+      test::ExpectClose(conv_2d, fused_conv_2d, /*atol=*/1e-6);
+    }
   }
 
   // Verifies that computing Conv2D+BiasAdd in a graph is identical to
@@ -821,16 +837,15 @@ class FusedConv2DOpTest : public OpsTestBase {
                             int depth = kDepth, int image_width = kImageWidth,
                             int image_height = kImageHeight,
                             int image_batch_count = kImageBatchCount) {
-    const BiasAddGraphRunner run_default =
-        [this](const Tensor& input_data, const Tensor& filter_data,
-               const Tensor& bias_data, Tensor* out) {
-          RunConv2DWithBias(input_data, filter_data, bias_data, out);
-        };
-
-    const BiasAddGraphRunner run_fused = [this](const Tensor& input_data,
-                                                const Tensor& filter_data,
-                                                const Tensor& bias_data,
-                                                Tensor* out) {
+    const BiasAddGraphRunner run_default = [this](
+        const Tensor& input_data, const Tensor& filter_data,
+        const Tensor& bias_data, Tensor* out) {
+      RunConv2DWithBias(input_data, filter_data, bias_data, out);
+    };
+
+    const BiasAddGraphRunner run_fused = [this](
+        const Tensor& input_data, const Tensor& filter_data,
+        const Tensor& bias_data, Tensor* out) {
       RunFusedConv2DOp(input_data, filter_data, {bias_data}, {"BiasAdd"}, out);
     };
 
@@ -846,19 +861,19 @@ class FusedConv2DOpTest : public OpsTestBase {
                                    int image_width = kImageWidth,
                                    int image_height = kImageHeight,
                                    int image_batch_count = kImageBatchCount) {
-    const BiasAddGraphRunner run_default =
-        [this](const Tensor& input_data, const Tensor& filter_data,
-               const Tensor& bias_data, Tensor* out) {
-          RunConv2DWithBiasAndRelu(input_data, filter_data, bias_data, out,
-                                   /*allow_gpu_device=*/true);
-        };
-
-    const BiasAddGraphRunner run_fused =
-        [this](const Tensor& input_data, const Tensor& filter_data,
-               const Tensor& bias_data, Tensor* out) {
-          RunFusedConv2DOp(input_data, filter_data, {bias_data},
-                           {"BiasAdd", "Relu"}, out, /*allow_gpu_device=*/true);
-        };
+    const BiasAddGraphRunner run_default = [this](
+        const Tensor& input_data, const Tensor& filter_data,
+        const Tensor& bias_data, Tensor* out) {
+      RunConv2DWithBiasAndRelu(input_data, filter_data, bias_data, out,
+                               /*allow_gpu_device=*/true);
+    };
+
+    const BiasAddGraphRunner run_fused = [this](
+        const Tensor& input_data, const Tensor& filter_data,
+        const Tensor& bias_data, Tensor* out) {
+      RunFusedConv2DOp(input_data, filter_data, {bias_data},
+                       {"BiasAdd", "Relu"}, out, /*allow_gpu_device=*/true);
+    };
 
     VerifyBiasAddTensorsNear(depth, image_width, image_height,
                              image_batch_count, filter_size, filter_count,
@@ -872,24 +887,22 @@ class FusedConv2DOpTest : public OpsTestBase {
                                  int image_width = kImageWidth,
                                  int image_height = kImageHeight,
                                  int image_batch_count = kImageBatchCount) {
-    const BatchNormGraphRunner run_default =
-        [this](const Tensor& input_data, const Tensor& filter_data,
-               const Tensor& scale_data, const Tensor& offset_data,
-               const Tensor& mean_data, const Tensor& variance_data,
-               Tensor* out) {
-          RunConv2DWithBatchNorm(input_data, filter_data, scale_data,
-                                 offset_data, mean_data, variance_data, out);
-        };
-
-    const BatchNormGraphRunner run_fused =
-        [this](const Tensor& input_data, const Tensor& filter_data,
-               const Tensor& scale_data, const Tensor& offset_data,
-               const Tensor& mean_data, const Tensor& variance_data,
-               Tensor* out) {
-          RunFusedConv2DOp(input_data, filter_data,
-                           {scale_data, offset_data, mean_data, variance_data},
-                           {"FusedBatchNorm"}, out);
-        };
+    const BatchNormGraphRunner run_default = [this](
+        const Tensor& input_data, const Tensor& filter_data,
+        const Tensor& scale_data, const Tensor& offset_data,
+        const Tensor& mean_data, const Tensor& variance_data, Tensor* out) {
+      RunConv2DWithBatchNorm(input_data, filter_data, scale_data, offset_data,
+                             mean_data, variance_data, out);
+    };
+
+    const BatchNormGraphRunner run_fused = [this](
+        const Tensor& input_data, const Tensor& filter_data,
+        const Tensor& scale_data, const Tensor& offset_data,
+        const Tensor& mean_data, const Tensor& variance_data, Tensor* out) {
+      RunFusedConv2DOp(input_data, filter_data,
+                       {scale_data, offset_data, mean_data, variance_data},
+                       {"FusedBatchNorm"}, out);
+    };
 
     VerifyFusedBatchNormTensorsNear(depth, image_width, image_height,
                                     image_batch_count, filter_size,
@@ -902,25 +915,22 @@ class FusedConv2DOpTest : public OpsTestBase {
       int filter_size, int filter_count, int depth = kDepth,
       int image_width = kImageWidth, int image_height = kImageHeight,
       int image_batch_count = kImageBatchCount) {
-    const BatchNormGraphRunner run_default =
-        [this](const Tensor& input_data, const Tensor& filter_data,
-               const Tensor& scale_data, const Tensor& offset_data,
-               const Tensor& mean_data, const Tensor& variance_data,
-               Tensor* out) {
-          RunConv2DWithBatchNormAndRelu(input_data, filter_data, scale_data,
-                                        offset_data, mean_data, variance_data,
-                                        out);
-        };
-
-    const BatchNormGraphRunner run_fused =
-        [this](const Tensor& input_data, const Tensor& filter_data,
-               const Tensor& scale_data, const Tensor& offset_data,
-               const Tensor& mean_data, const Tensor& variance_data,
-               Tensor* out) {
-          RunFusedConv2DOp(input_data, filter_data,
-                           {scale_data, offset_data, mean_data, variance_data},
-                           {"FusedBatchNorm", "Relu"}, out);
-        };
+    const BatchNormGraphRunner run_default = [this](
+        const Tensor& input_data, const Tensor& filter_data,
+        const Tensor& scale_data, const Tensor& offset_data,
+        const Tensor& mean_data, const Tensor& variance_data, Tensor* out) {
+      RunConv2DWithBatchNormAndRelu(input_data, filter_data, scale_data,
+                                    offset_data, mean_data, variance_data, out);
+    };
+
+    const BatchNormGraphRunner run_fused = [this](
+        const Tensor& input_data, const Tensor& filter_data,
+        const Tensor& scale_data, const Tensor& offset_data,
+        const Tensor& mean_data, const Tensor& variance_data, Tensor* out) {
+      RunFusedConv2DOp(input_data, filter_data,
+                       {scale_data, offset_data, mean_data, variance_data},
+                       {"FusedBatchNorm", "Relu"}, out);
+    };
 
     VerifyFusedBatchNormTensorsNear(depth, image_width, image_height,
                                     image_batch_count, filter_size,
-- 
GitLab


From 00fd6acd0b80cbc2be73e8b89cfebc01cb4cf9e8 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Fri, 28 Dec 2018 12:11:49 -0800
Subject: [PATCH 0064/1095] Disable optimizer_v2_test everywhere.

PiperOrigin-RevId: 227157901
---
 tensorflow/python/keras/optimizer_v2/BUILD | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tensorflow/python/keras/optimizer_v2/BUILD b/tensorflow/python/keras/optimizer_v2/BUILD
index b6be6a90af..05f6e5ac48 100644
--- a/tensorflow/python/keras/optimizer_v2/BUILD
+++ b/tensorflow/python/keras/optimizer_v2/BUILD
@@ -176,7 +176,11 @@ py_test(
     srcs = ["optimizer_v2_test.py"],
     shard_count = 8,
     tags = [
+        "manual",
+        "no_gpu",
+        "no_oss",
         "no_windows",
+        "notap",
     ],
     deps = [
         ":optimizer_v2",
-- 
GitLab


From cd2f18ff27fc2a9b502c80bf1d4a0abfc3577460 Mon Sep 17 00:00:00 2001
From: Michael Kuperstein <mkuper@google.com>
Date: Fri, 28 Dec 2018 13:34:54 -0800
Subject: [PATCH 0065/1095] [XLA] Add parser, proto and verifier support for
 DS/DUS with scalar indices.

Since this form is not yet documented, supported by HLO passes, or any
backends, the verifier will only accept it under a flag.

PiperOrigin-RevId: 227165366
---
 tensorflow/compiler/xla/client/xla_builder.cc |   8 +-
 tensorflow/compiler/xla/service/BUILD         |  19 +-
 .../xla/service/hlo_creation_utils.cc         |   4 +-
 .../xla/service/hlo_evaluator_typed_visitor.h |  14 +-
 .../compiler/xla/service/hlo_instruction.cc   |  69 ++++++-
 .../compiler/xla/service/hlo_instruction.h    |   9 +
 .../compiler/xla/service/hlo_instructions.cc  |  34 +++-
 .../compiler/xla/service/hlo_instructions.h   |  28 ++-
 tensorflow/compiler/xla/service/hlo_parser.cc |  27 ++-
 .../compiler/xla/service/hlo_parser_test.cc   |  33 ++-
 .../compiler/xla/service/hlo_verifier.cc      |  39 +++-
 .../compiler/xla/service/hlo_verifier_test.cc |  51 +++++
 .../compiler/xla/service/shape_inference.cc   | 190 +++++++++++++-----
 .../compiler/xla/service/shape_inference.h    |   7 +-
 tensorflow/compiler/xla/xla.proto             |   6 +-
 15 files changed, 433 insertions(+), 105 deletions(-)

diff --git a/tensorflow/compiler/xla/client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_builder.cc
index 1c6fe0ab5d..4b752def1a 100644
--- a/tensorflow/compiler/xla/client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_builder.cc
@@ -691,7 +691,7 @@ XlaOp XlaBuilder::DynamicSlice(const XlaOp& operand, const XlaOp& start_indices,
                         GetShape(start_indices));
     TF_ASSIGN_OR_RETURN(Shape shape,
                         ShapeInference::InferDynamicSliceShape(
-                            operand_shape, start_indices_shape, slice_sizes));
+                            operand_shape, {start_indices_shape}, slice_sizes));
     *instr.mutable_shape() = shape.ToProto();
 
     for (int64 size : slice_sizes) {
@@ -712,9 +712,9 @@ XlaOp XlaBuilder::DynamicUpdateSlice(const XlaOp& operand, const XlaOp& update,
     TF_ASSIGN_OR_RETURN(const Shape& update_shape, GetShape(update));
     TF_ASSIGN_OR_RETURN(const Shape& start_indices_shape,
                         GetShape(start_indices));
-    TF_ASSIGN_OR_RETURN(Shape shape,
-                        ShapeInference::InferDynamicUpdateSliceShape(
-                            operand_shape, update_shape, start_indices_shape));
+    TF_ASSIGN_OR_RETURN(
+        Shape shape, ShapeInference::InferDynamicUpdateSliceShape(
+                         operand_shape, update_shape, {start_indices_shape}));
     *instr.mutable_shape() = shape.ToProto();
 
     return AddInstruction(std::move(instr), HloOpcode::kDynamicUpdateSlice,
diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index d8736c8196..201646e70d 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -1,6 +1,14 @@
 # Description:
 #   XLA service implementation.
 
+load("//tensorflow/compiler/xla/tests:build_defs.bzl", "xla_test")
+load("//tensorflow/compiler/xla:xla.bzl", "xla_proto_library")
+load(
+    "//tensorflow/core:platform/default/build_config.bzl",
+    "tf_proto_library_py",
+)
+load("//tensorflow:tensorflow.bzl", "tf_cc_binary", "tf_cc_test")
+
 licenses(["notice"])  # Apache 2.0
 
 package(default_visibility = [":friends"])
@@ -12,15 +20,6 @@ package_group(
     ],
 )
 
-load("//tensorflow/compiler/xla/tests:build_defs.bzl", "xla_test")
-load("//tensorflow/compiler/xla:xla.bzl", "xla_proto_library")
-load("//tensorflow:tensorflow.bzl", "tf_cc_test")
-load("//tensorflow:tensorflow.bzl", "tf_cc_binary")
-load(
-    "//tensorflow/core:platform/default/build_config.bzl",
-    "tf_proto_library_py",
-)
-
 xla_proto_library(
     name = "hlo_proto",
     srcs = ["hlo.proto"],
@@ -2592,6 +2591,7 @@ tf_cc_test(
     srcs = ["hlo_verifier_test.cc"],
     deps = [
         ":hlo",
+        ":hlo_module_config",
         ":hlo_parser",
         ":hlo_verifier",
         ":layout_assignment",
@@ -2599,6 +2599,7 @@ tf_cc_test(
         "//tensorflow/compiler/xla:test",
         "//tensorflow/compiler/xla:types",
         "//tensorflow/compiler/xla:xla_data_proto",
+        "//tensorflow/compiler/xla:xla_proto",
         "//tensorflow/compiler/xla/tests:hlo_test_base",
         "//tensorflow/compiler/xla/tests:xla_internal_test_main",
         "//tensorflow/core:test",
diff --git a/tensorflow/compiler/xla/service/hlo_creation_utils.cc b/tensorflow/compiler/xla/service/hlo_creation_utils.cc
index 1678fba172..8cea95a73e 100644
--- a/tensorflow/compiler/xla/service/hlo_creation_utils.cc
+++ b/tensorflow/compiler/xla/service/hlo_creation_utils.cc
@@ -108,7 +108,7 @@ StatusOr<HloInstruction*> MakeDynamicSliceHlo(
   TF_ASSIGN_OR_RETURN(
       Shape dynamic_slice_shape,
       ShapeInference::InferDynamicSliceShape(
-          operand->shape(), start_indices->shape(), slice_sizes));
+          operand->shape(), {start_indices->shape()}, slice_sizes));
   return computation->AddInstruction(HloInstruction::CreateDynamicSlice(
       dynamic_slice_shape, operand, start_indices, slice_sizes));
 }
@@ -122,7 +122,7 @@ StatusOr<HloInstruction*> MakeDynamicUpdateSliceHlo(
   TF_ASSIGN_OR_RETURN(
       Shape dynamic_update_slice_shape,
       ShapeInference::InferDynamicUpdateSliceShape(
-          operand->shape(), update->shape(), start_indices->shape()));
+          operand->shape(), update->shape(), {start_indices->shape()}));
   return computation->AddInstruction(HloInstruction::CreateDynamicUpdateSlice(
       dynamic_update_slice_shape, operand, update, start_indices));
 }
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
index 783f87ea64..70ddedf62b 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
@@ -1406,10 +1406,12 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
     auto operand = dynamic_slice->operand(0);
     auto start_indices = dynamic_slice->operand(1);
     auto result_shape = dynamic_slice->shape();
-    TF_ASSIGN_OR_RETURN(auto inferred_return_shape,
-                        ShapeInference::InferDynamicSliceShape(
-                            operand->shape(), start_indices->shape(),
-                            dynamic_slice->dynamic_slice_sizes()));
+    TF_ASSIGN_OR_RETURN(
+        auto inferred_return_shape,
+        ShapeInference::InferDynamicSliceShape(
+            operand->shape(),
+            Cast<HloDynamicSliceInstruction>(dynamic_slice)->index_shapes(),
+            dynamic_slice->dynamic_slice_sizes()));
     TF_RET_CHECK(ShapeUtil::Compatible(result_shape, inferred_return_shape))
         << "return shape is set to: " << ShapeUtil::HumanString(result_shape)
         << " but is inferred to be: "
@@ -1464,7 +1466,9 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
     TF_ASSIGN_OR_RETURN(
         auto inferred_return_shape,
         ShapeInference::InferDynamicUpdateSliceShape(
-            operand->shape(), update->shape(), start_indices->shape()));
+            operand->shape(), update->shape(),
+            Cast<HloDynamicUpdateSliceInstruction>(dynamic_update_slice)
+                ->index_shapes()));
     TF_RET_CHECK(ShapeUtil::Compatible(result_shape, inferred_return_shape))
         << "return shape is set to: " << ShapeUtil::HumanString(result_shape)
         << " but is inferred to be: "
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc
index 2915535987..543fb66c01 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.cc
+++ b/tensorflow/compiler/xla/service/hlo_instruction.cc
@@ -31,6 +31,7 @@ limitations under the License.
 #include "absl/strings/numbers.h"
 #include "absl/strings/str_cat.h"
 #include "absl/strings/str_join.h"
+#include "absl/types/span.h"
 #include "tensorflow/compiler/xla/layout_util.h"
 #include "tensorflow/compiler/xla/literal.h"
 #include "tensorflow/compiler/xla/protobuf_util.h"
@@ -452,13 +453,50 @@ StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
           CreatePad(shape, operands(0), operands(1), proto.padding_config());
       break;
     case HloOpcode::kDynamicSlice: {
-      TF_RET_CHECK(proto.operand_ids_size() == 2)
-          << "DynamicSlice instruction should have 2 operands but sees "
-          << proto.operand_ids_size();
       std::vector<int64> slice_sizes(proto.dynamic_slice_sizes_size());
       absl::c_copy(proto.dynamic_slice_sizes(), slice_sizes.begin());
-      instruction =
-          CreateDynamicSlice(shape, operands(0), operands(1), slice_sizes);
+      TF_RET_CHECK(proto.operand_ids_size() >= 1)
+          << "DynamicSlice instruction should have at least 1 operands but "
+             "sees "
+          << proto.operand_ids_size();
+      if (proto.operand_ids_size() == 2 && operands(1)->shape().rank() == 1) {
+        // TODO(b/118437727): Old form, remove this path.
+        instruction =
+            CreateDynamicSlice(shape, operands(0), operands(1), slice_sizes);
+      } else {
+        // New form
+        auto expected_operands = 1 + operands(0)->shape().rank();
+        TF_RET_CHECK(proto.operand_ids_size() == expected_operands)
+            << "DynamicSlice instruction should have " << expected_operands
+            << " operands, but has " << proto.operand_ids_size();
+        const auto& operand_vector = all_operands();
+        instruction = CreateDynamicSlice(
+            shape, operands(0), absl::MakeSpan(operand_vector).subspan(1),
+            slice_sizes);
+      }
+      break;
+    }
+    case HloOpcode::kDynamicUpdateSlice: {
+      TF_RET_CHECK(proto.operand_ids_size() >= 2)
+          << "DynamicUpdateSlice instruction should have at least 2 operands "
+             "but sees "
+          << proto.operand_ids_size();
+      if (proto.operand_ids_size() == 3 && operands(2)->shape().rank() == 1) {
+        // TODO(b/118437727): Old form, remove this path.
+        instruction = CreateDynamicUpdateSlice(shape, operands(0), operands(1),
+                                               operands(2));
+      } else {
+        // New form
+        auto expected_operands = 2 + operands(0)->shape().rank();
+        TF_RET_CHECK(proto.operand_ids_size() == expected_operands)
+            << "DynamicUpdateSlice instruction should have "
+            << expected_operands << " operands, but has "
+            << proto.operand_ids_size();
+        const auto& operand_vector = all_operands();
+        instruction =
+            CreateDynamicUpdateSlice(shape, operands(0), operands(1),
+                                     absl::MakeSpan(operand_vector).subspan(2));
+      }
       break;
     }
     case HloOpcode::kGather: {
@@ -917,6 +955,14 @@ HloInstruction::CreateAddDependency(HloInstruction* data_operand,
       shape, operand, start_indices, slice_sizes);
 }
 
+/* static */ std::unique_ptr<HloInstruction> HloInstruction::CreateDynamicSlice(
+    const Shape& shape, HloInstruction* operand,
+    absl::Span<HloInstruction* const> start_indices,
+    absl::Span<const int64> slice_sizes) {
+  return absl::make_unique<HloDynamicSliceInstruction>(
+      shape, operand, start_indices, slice_sizes);
+}
+
 /* static */ std::unique_ptr<HloInstruction>
 HloInstruction::CreateDynamicUpdateSlice(const Shape& shape,
                                          HloInstruction* operand,
@@ -926,6 +972,14 @@ HloInstruction::CreateDynamicUpdateSlice(const Shape& shape,
       shape, operand, update, start_indices);
 }
 
+/* static */ std::unique_ptr<HloInstruction>
+HloInstruction::CreateDynamicUpdateSlice(
+    const Shape& shape, HloInstruction* operand, HloInstruction* update,
+    absl::Span<HloInstruction* const> start_indices) {
+  return absl::make_unique<HloDynamicUpdateSliceInstruction>(
+      shape, operand, update, start_indices);
+}
+
 /* static */ std::unique_ptr<HloInstruction> HloInstruction::CreateConcatenate(
     const Shape& shape, absl::Span<HloInstruction* const> operands,
     int64 dimension) {
@@ -1382,9 +1436,8 @@ std::unique_ptr<HloInstruction> HloInstruction::CloneWithNewOperands(
       clone = CreateReshape(shape, new_operands[0]);
       break;
     case HloOpcode::kDynamicUpdateSlice:
-      CHECK_EQ(new_operands.size(), 3);
       clone = CreateDynamicUpdateSlice(shape, new_operands[0], new_operands[1],
-                                       new_operands[2]);
+                                       new_operands.subspan(2));
       break;
     case HloOpcode::kTuple:
       clone = CreateTuple(new_operands);
@@ -2824,7 +2877,7 @@ HloInstruction::UseKind HloInstruction::OperandElementUse(int64 i) const {
       }
       return UseKind::kReuse;
     case HloOpcode::kDynamicUpdateSlice:
-      // Dynamic-update-slice reuses only operand 2 (start_indices).
+      // Dynamic-update-slice reuses only start_indices.
       if (i == 0 || i == 1) {
         return UseKind::kUse;
       }
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.h b/tensorflow/compiler/xla/service/hlo_instruction.h
index 36e1ab4931..362d07e64f 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.h
+++ b/tensorflow/compiler/xla/service/hlo_instruction.h
@@ -559,12 +559,21 @@ class HloInstruction {
   static std::unique_ptr<HloInstruction> CreateDynamicSlice(
       const Shape& shape, HloInstruction* operand,
       HloInstruction* start_indices, absl::Span<const int64> slice_sizes);
+  // Same as above, but expects a span of scalar start indices.
+  static std::unique_ptr<HloInstruction> CreateDynamicSlice(
+      const Shape& shape, HloInstruction* operand,
+      absl::Span<HloInstruction* const> start_indices,
+      absl::Span<const int64> slice_sizes);
 
   // Creates a dynamic update slice instruction, which updates a slice
   // of 'operand' with 'update' and 'start_indices'.
   static std::unique_ptr<HloInstruction> CreateDynamicUpdateSlice(
       const Shape& shape, HloInstruction* operand, HloInstruction* update,
       HloInstruction* start_indices);
+  // Same as above, but expects a span of scalar start indices.
+  static std::unique_ptr<HloInstruction> CreateDynamicUpdateSlice(
+      const Shape& shape, HloInstruction* operand, HloInstruction* update,
+      absl::Span<HloInstruction* const> start_indices);
 
   // Creates a concatenate instruction, where the operands are concatenated on
   // the provided dimension.
diff --git a/tensorflow/compiler/xla/service/hlo_instructions.cc b/tensorflow/compiler/xla/service/hlo_instructions.cc
index 1f37b284a2..7a3fb6d9a2 100644
--- a/tensorflow/compiler/xla/service/hlo_instructions.cc
+++ b/tensorflow/compiler/xla/service/hlo_instructions.cc
@@ -2007,6 +2007,18 @@ HloDynamicSliceInstruction::HloDynamicSliceInstruction(
   AppendOperand(start_indices);
 }
 
+HloDynamicSliceInstruction::HloDynamicSliceInstruction(
+    const Shape& shape, HloInstruction* operand,
+    absl::Span<HloInstruction* const> start_indices,
+    absl::Span<const int64> slice_sizes)
+    : HloDynamicIndexInstruction(HloOpcode::kDynamicSlice, shape),
+      dynamic_slice_sizes_(slice_sizes.begin(), slice_sizes.end()) {
+  AppendOperand(operand);
+  for (HloInstruction* index : start_indices) {
+    AppendOperand(index);
+  }
+}
+
 HloDynamicUpdateSliceInstruction::HloDynamicUpdateSliceInstruction(
     const Shape& shape, HloInstruction* operand, HloInstruction* update,
     HloInstruction* start_indices)
@@ -2016,6 +2028,17 @@ HloDynamicUpdateSliceInstruction::HloDynamicUpdateSliceInstruction(
   AppendOperand(start_indices);
 }
 
+HloDynamicUpdateSliceInstruction::HloDynamicUpdateSliceInstruction(
+    const Shape& shape, HloInstruction* operand, HloInstruction* update,
+    absl::Span<HloInstruction* const> start_indices)
+    : HloDynamicIndexInstruction(HloOpcode::kDynamicUpdateSlice, shape) {
+  AppendOperand(operand);
+  AppendOperand(update);
+  for (HloInstruction* index : start_indices) {
+    AppendOperand(index);
+  }
+}
+
 HloInstructionProto HloDynamicSliceInstruction::ToProto() const {
   HloInstructionProto proto = HloInstruction::ToProto();
   for (int64 slice_size : dynamic_slice_sizes_) {
@@ -2041,9 +2064,14 @@ std::unique_ptr<HloInstruction>
 HloDynamicSliceInstruction::CloneWithNewOperandsImpl(
     const Shape& shape, absl::Span<HloInstruction* const> new_operands,
     HloCloneContext* context) const {
-  CHECK_EQ(new_operands.size(), 2);
-  return absl::make_unique<HloDynamicSliceInstruction>(
-      shape, new_operands[0], new_operands[1], dynamic_slice_sizes_);
+  if (new_operands.size() == 2 && new_operands[1]->shape().rank() == 1) {
+    // TODO(b/118437727): Old form, remove this path.
+    return absl::make_unique<HloDynamicSliceInstruction>(
+        shape, new_operands[0], new_operands[1], dynamic_slice_sizes_);
+  } else {
+    return absl::make_unique<HloDynamicSliceInstruction>(
+        shape, new_operands[0], new_operands.subspan(1), dynamic_slice_sizes_);
+  }
 }
 
 HloGatherInstruction::HloGatherInstruction(
diff --git a/tensorflow/compiler/xla/service/hlo_instructions.h b/tensorflow/compiler/xla/service/hlo_instructions.h
index ca212c7f2c..83b6760683 100644
--- a/tensorflow/compiler/xla/service/hlo_instructions.h
+++ b/tensorflow/compiler/xla/service/hlo_instructions.h
@@ -1183,7 +1183,22 @@ class HloDynamicIndexInstruction : public HloInstruction {
  public:
   explicit HloDynamicIndexInstruction(HloOpcode opcode, const Shape& shape)
       : HloInstruction(opcode, shape) {}
-  virtual int64 index_operand_number() const = 0;
+  virtual int64 first_index_operand_number() const = 0;
+
+  // Returns a subspan of operands which represent the start indices.
+  absl::Span<HloInstruction* const> index_operands() {
+    return absl::MakeSpan(operands()).subspan(first_index_operand_number());
+  }
+
+  // Returns the shapes of the index operands.
+  std::vector<Shape> index_shapes() {
+    std::vector<Shape> shapes;
+    auto indices = index_operands();
+    for (const HloInstruction* index : indices) {
+      shapes.push_back(index->shape());
+    }
+    return shapes;
+  }
 };
 
 class HloDynamicSliceInstruction : public HloDynamicIndexInstruction {
@@ -1192,6 +1207,10 @@ class HloDynamicSliceInstruction : public HloDynamicIndexInstruction {
                                       HloInstruction* operand,
                                       HloInstruction* start_indices,
                                       absl::Span<const int64> slice_sizes);
+  explicit HloDynamicSliceInstruction(
+      const Shape& shape, HloInstruction* operand,
+      absl::Span<HloInstruction* const> start_indices,
+      absl::Span<const int64> slice_sizes);
   // Old methods kept for smooth subclassing transition END.
   // Returns the size of the slice in the given dimension for a dynamic
   // slice node.
@@ -1204,7 +1223,7 @@ class HloDynamicSliceInstruction : public HloDynamicIndexInstruction {
   // Returns a serialized representation of this instruction.
   HloInstructionProto ToProto() const override;
 
-  int64 index_operand_number() const override { return 1; }
+  int64 first_index_operand_number() const override { return 1; }
 
  private:
   std::vector<string> ExtraAttributesToStringImpl(
@@ -1229,8 +1248,11 @@ class HloDynamicUpdateSliceInstruction : public HloDynamicIndexInstruction {
                                             HloInstruction* operand,
                                             HloInstruction* update,
                                             HloInstruction* start_indices);
+  explicit HloDynamicUpdateSliceInstruction(
+      const Shape& shape, HloInstruction* operand, HloInstruction* update,
+      absl::Span<HloInstruction* const> start_indices);
 
-  int64 index_operand_number() const override { return 2; }
+  int64 first_index_operand_number() const override { return 2; }
 };
 
 class HloGatherInstruction : public HloInstruction {
diff --git a/tensorflow/compiler/xla/service/hlo_parser.cc b/tensorflow/compiler/xla/service/hlo_parser.cc
index 56848ce0e8..730d39500b 100644
--- a/tensorflow/compiler/xla/service/hlo_parser.cc
+++ b/tensorflow/compiler/xla/service/hlo_parser.cc
@@ -1171,24 +1171,39 @@ bool HloParser::ParseInstructionRhs(HloComputation::Builder* builder,
       optional<std::vector<tensorflow::int64>> dynamic_slice_sizes;
       attrs["dynamic_slice_sizes"] = {
           /*required=*/true, AttrTy::kBracedInt64List, &dynamic_slice_sizes};
-      if (!ParseOperands(&operands, /*expected_size=*/2) ||
-          !ParseAttributes(attrs)) {
+      LocTy loc = lexer_.GetLoc();
+      if (!ParseOperands(&operands) || !ParseAttributes(attrs)) {
         return false;
       }
+      if (operands.empty()) {
+        return Error(loc, "Expected at least one operand.");
+      }
+      if (!(operands.size() == 2 && operands[1]->shape().rank() == 1) &&
+          operands.size() != 1 + operands[0]->shape().rank()) {
+        return Error(loc, "Wrong number of operands.");
+      }
       instruction = builder->AddInstruction(HloInstruction::CreateDynamicSlice(
-          shape, /*operand=*/operands[0], /*start_indices=*/operands[1],
+          shape, /*operand=*/operands[0],
+          /*start_indices=*/absl::MakeSpan(operands).subspan(1),
           *dynamic_slice_sizes));
       break;
     }
     case HloOpcode::kDynamicUpdateSlice: {
-      if (!ParseOperands(&operands, /*expected_size=*/3) ||
-          !ParseAttributes(attrs)) {
+      LocTy loc = lexer_.GetLoc();
+      if (!ParseOperands(&operands) || !ParseAttributes(attrs)) {
         return false;
       }
+      if (operands.size() < 2) {
+        return Error(loc, "Expected at least two operands.");
+      }
+      if (!(operands.size() == 3 && operands[2]->shape().rank() == 1) &&
+          operands.size() != 2 + operands[0]->shape().rank()) {
+        return Error(loc, "Wrong number of operands.");
+      }
       instruction =
           builder->AddInstruction(HloInstruction::CreateDynamicUpdateSlice(
               shape, /*operand=*/operands[0], /*update=*/operands[1],
-              /*start_indices=*/operands[2]));
+              /*start_indices=*/absl::MakeSpan(operands).subspan(2)));
       break;
     }
     case HloOpcode::kTranspose: {
diff --git a/tensorflow/compiler/xla/service/hlo_parser_test.cc b/tensorflow/compiler/xla/service/hlo_parser_test.cc
index bc1a736766..76b8a5bc11 100644
--- a/tensorflow/compiler/xla/service/hlo_parser_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_parser_test.cc
@@ -566,12 +566,26 @@ ENTRY %DynamicSlice.v5 (original_parameter: s32[2,2,258], start_index: s32[1]) -
   ROOT %dynamic-slice = s32[2,2,258]{2,1,0} dynamic-slice(s32[2,2,258]{2,1,0} %original_parameter, s32[3]{0} %concatenate), dynamic_slice_sizes={2,2,258}
 }
 
+)"
+},
+// Dynamic slice with scalar indices
+{
+"DynamicSliceScalarIndices",
+R"(HloModule DynamicSlice_module
+
+ENTRY %DynamicSlice.v5 (original_parameter: s32[2,2,258], start_index: s32[]) -> s32[2,2,258] {
+  %original_parameter = s32[2,2,258]{2,1,0} parameter(0)
+  %constant = s32[] constant(0)
+  %start_index = s32[] parameter(1)
+  ROOT %dynamic-slice = s32[2,2,258]{2,1,0} dynamic-slice(s32[2,2,258]{2,1,0} %original_parameter, s32[] %constant, s32[] %constant, s32[] %start_index), dynamic_slice_sizes={2,2,258}
+}
+
 )"
 },
 // Dynamic update slice
 {
 "DynamicUpdateSlice",
-R"(HloModule DynamicUpdateSlice_module
+R"(HloModule DynamicSlice_module
 
 ENTRY %DynamicUpdateSlice.v4 (input: s32[1,1,25,1], update: s32[1,1,2,1], start_indices: s32[4]) -> s32[1,1,25,1] {
   %input = s32[1,1,25,1]{3,2,1,0} parameter(0)
@@ -580,6 +594,23 @@ ENTRY %DynamicUpdateSlice.v4 (input: s32[1,1,25,1], update: s32[1,1,2,1], start_
   ROOT %dynamic-update-slice = s32[1,1,25,1]{3,2,1,0} dynamic-update-slice(s32[1,1,25,1]{3,2,1,0} %input, s32[1,1,2,1]{3,2,1,0} %update, s32[4]{0} %start_indices)
 }
 
+)"
+},
+// Dynamic update slice with scalar indices
+{
+"DynamicUpdateSliceScalarIndex",
+R"(HloModule DynamicUpdateSlice_module
+
+ENTRY %DynamicUpdateSlice.v4 (input: s32[1,1,25,1], update: s32[1,1,2,1], start_index.0: s32[], start_index.1: s32[], start_index.2: s32[], start_index.3: s32[]) -> s32[1,1,25,1] {
+  %input = s32[1,1,25,1]{3,2,1,0} parameter(0)
+  %update = s32[1,1,2,1]{3,2,1,0} parameter(1)
+  %start_index.0 = s32[] parameter(2)
+  %start_index.1 = s32[] parameter(3)
+  %start_index.2 = s32[] parameter(4)
+  %start_index.3 = s32[] parameter(5)
+  ROOT %dynamic-update-slice = s32[1,1,25,1]{3,2,1,0} dynamic-update-slice(s32[1,1,25,1]{3,2,1,0} %input, s32[1,1,2,1]{3,2,1,0} %update, s32[] %start_index.0, s32[] %start_index.1, s32[] %start_index.2, s32[] %start_index.3)
+}
+
 )"
 },
 // batch norm training
diff --git a/tensorflow/compiler/xla/service/hlo_verifier.cc b/tensorflow/compiler/xla/service/hlo_verifier.cc
index 8d8720d7be..14e29533c2 100644
--- a/tensorflow/compiler/xla/service/hlo_verifier.cc
+++ b/tensorflow/compiler/xla/service/hlo_verifier.cc
@@ -496,21 +496,38 @@ Status ShapeVerifier::HandleSlice(HloInstruction* slice) {
 }
 
 Status ShapeVerifier::HandleDynamicSlice(HloInstruction* dynamic_slice) {
-  TF_RETURN_IF_ERROR(CheckOperandCount(dynamic_slice, 2));
-  return CheckShape(dynamic_slice, ShapeInference::InferDynamicSliceShape(
-                                       dynamic_slice->operand(0)->shape(),
-                                       dynamic_slice->operand(1)->shape(),
-                                       dynamic_slice->dynamic_slice_sizes()));
+  const DebugOptions& debug_options =
+      dynamic_slice->GetModule()->config().debug_options();
+  const bool allow_scalar_indices =
+      debug_options.xla_allow_scalar_index_dynamic_ops();
+  if (!allow_scalar_indices) {
+    TF_RETURN_IF_ERROR(CheckOperandCount(dynamic_slice, 2));
+  }
+  return CheckShape(
+      dynamic_slice,
+      ShapeInference::InferDynamicSliceShape(
+          dynamic_slice->operand(0)->shape(),
+          Cast<HloDynamicSliceInstruction>(dynamic_slice)->index_shapes(),
+          dynamic_slice->dynamic_slice_sizes(), allow_scalar_indices));
 }
 
 Status ShapeVerifier::HandleDynamicUpdateSlice(
     HloInstruction* dynamic_update_slice) {
-  TF_RETURN_IF_ERROR(CheckOperandCount(dynamic_update_slice, 3));
-  return CheckShape(dynamic_update_slice,
-                    ShapeInference::InferDynamicUpdateSliceShape(
-                        dynamic_update_slice->operand(0)->shape(),
-                        dynamic_update_slice->operand(1)->shape(),
-                        dynamic_update_slice->operand(2)->shape()));
+  const DebugOptions& debug_options =
+      dynamic_update_slice->GetModule()->config().debug_options();
+  const bool allow_scalar_indices =
+      debug_options.xla_allow_scalar_index_dynamic_ops();
+  if (!allow_scalar_indices) {
+    TF_RETURN_IF_ERROR(CheckOperandCount(dynamic_update_slice, 3));
+  }
+  return CheckShape(
+      dynamic_update_slice,
+      ShapeInference::InferDynamicUpdateSliceShape(
+          dynamic_update_slice->operand(0)->shape(),
+          dynamic_update_slice->operand(1)->shape(),
+          Cast<HloDynamicUpdateSliceInstruction>(dynamic_update_slice)
+              ->index_shapes(),
+          allow_scalar_indices));
 }
 
 Status ShapeVerifier::HandleTuple(HloInstruction* tuple) {
diff --git a/tensorflow/compiler/xla/service/hlo_verifier_test.cc b/tensorflow/compiler/xla/service/hlo_verifier_test.cc
index 4bc557e4e6..91f247a9bb 100644
--- a/tensorflow/compiler/xla/service/hlo_verifier_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_verifier_test.cc
@@ -20,6 +20,7 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/service/hlo_computation.h"
 #include "tensorflow/compiler/xla/service/hlo_instruction.h"
+#include "tensorflow/compiler/xla/service/hlo_module_config.h"
 #include "tensorflow/compiler/xla/service/hlo_opcode.h"
 #include "tensorflow/compiler/xla/service/hlo_parser.h"
 #include "tensorflow/compiler/xla/service/layout_assignment.h"
@@ -27,6 +28,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/test.h"
 #include "tensorflow/compiler/xla/tests/hlo_test_base.h"
 #include "tensorflow/compiler/xla/types.h"
+#include "tensorflow/compiler/xla/xla.pb.h"
 #include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 
@@ -386,6 +388,55 @@ TEST_F(HloVerifierTest, AddWithLayoutChange) {
   ASSERT_TRUE(status.ok());
 }
 
+TEST_F(HloVerifierTest, ScalarIndexDynamicSlice) {
+  const char* const kScalarIndexDynamicSlice = R"(
+    HloModule DynamicSlice_module
+
+    ENTRY %DynamicSlice.v5 (original_parameter: s32[2,2,258], start_index: s32[]) -> s32[2,2,258] {
+      %original_parameter = s32[2,2,258] parameter(0)
+      %constant = s32[] constant(0)
+      %start_index = s32[] parameter(1)
+      ROOT %dynamic-slice = s32[2,2,258] dynamic-slice(s32[2,2,258] %original_parameter, s32[] %constant, s32[] %constant, s32[] %start_index), dynamic_slice_sizes={2,2,258}
+    }
+  )";
+
+  HloModuleConfig config;
+  DebugOptions debug_options = config.debug_options();
+  debug_options.set_xla_allow_scalar_index_dynamic_ops(true);
+  config.set_debug_options(debug_options);
+
+  TF_ASSERT_OK_AND_ASSIGN(auto module,
+                          ParseHloString(kScalarIndexDynamicSlice, config));
+  auto status = verifier().Run(module.get()).status();
+  ASSERT_TRUE(status.ok());
+}
+
+TEST_F(HloVerifierTest, ScalarIndexDynamicUpdateSlice) {
+  const char* const kScalarIndexDynamicSlice = R"(
+    HloModule DynamicUpdateSlice_module
+
+    ENTRY %DynamicUpdateSlice.v4 (input: s32[1,1,25,1], update: s32[1,1,2,1], start_index.0: s32[], start_index.1: s32[], start_index.2: s32[], start_index.3: s32[]) -> s32[1,1,25,1] {
+      %input = s32[1,1,25,1]{3,2,1,0} parameter(0)
+      %update = s32[1,1,2,1]{3,2,1,0} parameter(1)
+      %start_index.0 = s32[] parameter(2)
+      %start_index.1 = s32[] parameter(3)
+      %start_index.2 = s32[] parameter(4)
+      %start_index.3 = s32[] parameter(5)
+      ROOT %dynamic-update-slice = s32[1,1,25,1]{3,2,1,0} dynamic-update-slice(s32[1,1,25,1]{3,2,1,0} %input, s32[1,1,2,1]{3,2,1,0} %update, s32[] %start_index.0, s32[] %start_index.1, s32[] %start_index.2, s32[] %start_index.3)
+    }
+  )";
+
+  HloModuleConfig config;
+  DebugOptions debug_options = config.debug_options();
+  debug_options.set_xla_allow_scalar_index_dynamic_ops(true);
+  config.set_debug_options(debug_options);
+
+  TF_ASSERT_OK_AND_ASSIGN(auto module,
+                          ParseHloString(kScalarIndexDynamicSlice, config));
+  auto status = verifier().Run(module.get()).status();
+  ASSERT_TRUE(status.ok());
+}
+
 TEST_F(HloVerifierTestLayoutSensitive, AddWithLayoutChangeNotAllowed) {
   TF_ASSERT_OK_AND_ASSIGN(auto module, ParseHloString(kAddWithLayoutChangeHlo));
   auto status = verifier().Run(module.get()).status();
diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc
index b0e241d216..4a2dd09742 100644
--- a/tensorflow/compiler/xla/service/shape_inference.cc
+++ b/tensorflow/compiler/xla/service/shape_inference.cc
@@ -2087,35 +2087,81 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
 }
 
 /* static */ StatusOr<Shape> ShapeInference::InferDynamicSliceShape(
-    const Shape& operand_shape, const Shape& start_indices_shape,
-    absl::Span<const int64> slice_sizes) {
+    const Shape& operand_shape, absl::Span<const Shape> start_index_shapes,
+    absl::Span<const int64> slice_sizes, bool allow_scalar_indices) {
   TF_RETURN_IF_ERROR(ExpectArray(operand_shape, "operand of dynamic slice"));
-  TF_RETURN_IF_ERROR(
-      ExpectArray(start_indices_shape, "start indices of dynamic slice"));
+  auto number_of_indices = start_index_shapes.size();
+  // TODO(b/118437727): Remove this path.
+  if (!allow_scalar_indices ||
+      (number_of_indices >= 1 && start_index_shapes[0].rank() == 1)) {
+    if (number_of_indices != 1) {
+      return InvalidArgument(
+          "Dynamic slice should have exactly 1 index operand, has %d.",
+          number_of_indices);
+    }
 
-  VLOG(2) << StrFormat(
-      "slicing shape %s at dynamic start_indices %s with slice_sizes={%s}",
-      ShapeUtil::HumanString(operand_shape),
-      ShapeUtil::HumanString(start_indices_shape), StrJoin(slice_sizes, ", "));
+    const Shape& start_indices_shape = start_index_shapes[0];
+    TF_RETURN_IF_ERROR(
+        ExpectArray(start_indices_shape, "start indices of dynamic slice"));
 
-  if (start_indices_shape.rank() != 1) {
-    return InvalidArgument(
-        "Dynamic slice start indices of rank %d must be rank1.",
-        start_indices_shape.rank());
-  }
+    VLOG(2) << StrFormat(
+        "slicing shape %s at dynamic start_indices %s with slice_sizes={%s}",
+        ShapeUtil::HumanString(operand_shape),
+        ShapeUtil::HumanString(start_indices_shape),
+        StrJoin(slice_sizes, ", "));
 
-  if (!ShapeUtil::ElementIsIntegral(start_indices_shape)) {
-    return InvalidArgument(
-        "Dynamic slice start indices must be of integral type.");
-  }
+    if (start_indices_shape.rank() != 1) {
+      return InvalidArgument(
+          "Dynamic slice start indices of rank %d must be rank1.",
+          start_indices_shape.rank());
+    }
 
-  const int64 start_num_dims = start_indices_shape.dimensions(0);
-  if (operand_shape.rank() != start_num_dims) {
-    return InvalidArgument(
-        "Dynamic slice start number of dimensions %d (%s) must match rank "
-        "%d of slice input (%s).",
-        start_num_dims, ShapeUtil::HumanString(start_indices_shape),
-        operand_shape.rank(), ShapeUtil::HumanString(operand_shape));
+    if (!ShapeUtil::ElementIsIntegral(start_indices_shape)) {
+      return InvalidArgument(
+          "Dynamic slice start indices must be of integral type.");
+    }
+
+    const int64 start_num_dims = start_indices_shape.dimensions(0);
+    if (operand_shape.rank() != start_num_dims) {
+      return InvalidArgument(
+          "Dynamic slice start number of dimensions %d (%s) must match rank "
+          "%d of slice input (%s).",
+          start_num_dims, ShapeUtil::HumanString(start_indices_shape),
+          operand_shape.rank(), ShapeUtil::HumanString(operand_shape));
+    }
+  } else {
+    VLOG(2) << StrFormat("slicing shape %s a with slice_sizes={%s}",
+                         ShapeUtil::HumanString(operand_shape),
+                         StrJoin(slice_sizes, ", "));
+
+    if (operand_shape.rank() != number_of_indices) {
+      return InvalidArgument(
+          "Dynamic slice start number of dimensions %d must match rank "
+          "%d of slice input (%s).",
+          number_of_indices, operand_shape.rank(),
+          ShapeUtil::HumanString(operand_shape));
+    }
+
+    if (number_of_indices > 0) {
+      const Shape& first_index_shape = start_index_shapes[0];
+      if (!ShapeUtil::IsScalar(first_index_shape)) {
+        return InvalidArgument("Dynamic slice indices must be scalar, not %s.",
+                               ShapeUtil::HumanString(first_index_shape));
+      }
+      if (!ShapeUtil::ElementIsIntegral(first_index_shape)) {
+        return InvalidArgument(
+            "Dynamic slice start indices must be of integral type.");
+      }
+      for (const Shape& index_shape : start_index_shapes) {
+        if (!ShapeUtil::Equal(first_index_shape, index_shape)) {
+          return InvalidArgument(
+              "Dynamic slice start indices must all have the same shape, got "
+              "mismatching indices with shapes %s and %s.",
+              ShapeUtil::HumanString(first_index_shape),
+              ShapeUtil::HumanString(index_shape));
+        }
+      }
+    }
   }
 
   if (slice_sizes.size() != operand_shape.rank()) {
@@ -2144,39 +2190,85 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
 
 /* static */ StatusOr<Shape> ShapeInference::InferDynamicUpdateSliceShape(
     const Shape& operand_shape, const Shape& update_shape,
-    const Shape& start_indices_shape) {
+    absl::Span<const Shape> start_index_shapes, bool allow_scalar_indices) {
   TF_RETURN_IF_ERROR(
       ExpectArray(operand_shape, "operand of dynamic update slice"));
   TF_RETURN_IF_ERROR(
       ExpectArray(update_shape, "update of dynamic update slice"));
-  TF_RETURN_IF_ERROR(ExpectArray(start_indices_shape,
-                                 "start indices of dynamic update slice"));
 
-  VLOG(2) << StrFormat(
-      "updating slice of shape %s at dynamic start_indices %s with update "
-      "shape %s",
-      ShapeUtil::HumanString(operand_shape),
-      ShapeUtil::HumanString(start_indices_shape),
-      ShapeUtil::HumanString(update_shape));
+  auto number_of_indices = start_index_shapes.size();
+  // TODO(b/118437727): Remove this path.
+  if (!allow_scalar_indices ||
+      (number_of_indices >= 1 && start_index_shapes[0].rank() == 1)) {
+    if (number_of_indices != 1) {
+      return InvalidArgument(
+          "Dynamic update slice should have exactly 1 index operand, has %d.",
+          number_of_indices);
+    }
+    const Shape& start_indices_shape = start_index_shapes[0];
+    TF_RETURN_IF_ERROR(ExpectArray(start_indices_shape,
+                                   "start indices of dynamic update slice"));
 
-  if (start_indices_shape.rank() != 1) {
-    return InvalidArgument(
-        "Dynamic update slice start indices of rank %d must be rank1.",
-        start_indices_shape.rank());
-  }
+    VLOG(2) << StrFormat(
+        "updating slice of shape %s at dynamic start_indices %s with update "
+        "shape %s",
+        ShapeUtil::HumanString(operand_shape),
+        ShapeUtil::HumanString(start_indices_shape),
+        ShapeUtil::HumanString(update_shape));
 
-  if (!ShapeUtil::ElementIsIntegral(start_indices_shape)) {
-    return InvalidArgument(
-        "Dynamic update slice start indices must be of integral type.");
-  }
+    if (start_indices_shape.rank() != 1) {
+      return InvalidArgument(
+          "Dynamic update slice start indices of rank %d must be rank1.",
+          start_indices_shape.rank());
+    }
 
-  const int64 start_num_dims = start_indices_shape.dimensions(0);
-  if (operand_shape.rank() != start_num_dims) {
-    return InvalidArgument(
-        "Dynamic update slice start number of dimensions %d (%s) must match "
-        "rank %d of slice input (%s).",
-        start_num_dims, ShapeUtil::HumanString(start_indices_shape),
-        operand_shape.rank(), ShapeUtil::HumanString(operand_shape));
+    if (!ShapeUtil::ElementIsIntegral(start_indices_shape)) {
+      return InvalidArgument(
+          "Dynamic update slice start indices must be of integral type.");
+    }
+
+    const int64 start_num_dims = start_indices_shape.dimensions(0);
+    if (operand_shape.rank() != start_num_dims) {
+      return InvalidArgument(
+          "Dynamic update slice start number of dimensions %d (%s) must match "
+          "rank %d of slice input (%s).",
+          start_num_dims, ShapeUtil::HumanString(start_indices_shape),
+          operand_shape.rank(), ShapeUtil::HumanString(operand_shape));
+    }
+  } else {
+    VLOG(2) << StrFormat("updating slice of shape %s with update shape %s",
+                         ShapeUtil::HumanString(operand_shape),
+                         ShapeUtil::HumanString(update_shape));
+
+    if (operand_shape.rank() != number_of_indices) {
+      return InvalidArgument(
+          "Dynamic update slice start number of dimensions %d must match rank "
+          "%d of slice input (%s).",
+          number_of_indices, operand_shape.rank(),
+          ShapeUtil::HumanString(operand_shape));
+    }
+
+    if (number_of_indices > 0) {
+      const Shape& first_index_shape = start_index_shapes[0];
+      if (!ShapeUtil::IsScalar(first_index_shape)) {
+        return InvalidArgument(
+            "Dynamic update slice indices must be scalar, not %s.",
+            ShapeUtil::HumanString(first_index_shape));
+      }
+      if (!ShapeUtil::ElementIsIntegral(first_index_shape)) {
+        return InvalidArgument(
+            "Dynamic update slice start indices must be of integral type.");
+      }
+      for (const Shape& index_shape : start_index_shapes) {
+        if (!ShapeUtil::Equal(first_index_shape, index_shape)) {
+          return InvalidArgument(
+              "Dynamic update slice start indices must all have the same "
+              "shape, got mismatching indices with shapes %s and %s.",
+              ShapeUtil::HumanString(first_index_shape),
+              ShapeUtil::HumanString(index_shape));
+        }
+      }
+    }
   }
 
   if (update_shape.rank() != operand_shape.rank()) {
diff --git a/tensorflow/compiler/xla/service/shape_inference.h b/tensorflow/compiler/xla/service/shape_inference.h
index 1b8fd10d69..e440e364c8 100644
--- a/tensorflow/compiler/xla/service/shape_inference.h
+++ b/tensorflow/compiler/xla/service/shape_inference.h
@@ -176,14 +176,15 @@ class ShapeInference {
   // Infers the shape produced by a dynamic slice operation of size specified
   // in 'slice_sizes', with dynamic start indices shape 'start_indices_shape'.
   static StatusOr<Shape> InferDynamicSliceShape(
-      const Shape& operand_shape, const Shape& start_indices_shape,
-      absl::Span<const int64> slice_sizes);
+      const Shape& operand_shape, absl::Span<const Shape> start_index_shapes,
+      absl::Span<const int64> slice_sizes, bool allow_scalar_indices = false);
 
   // Infers the shape produced by a dynamic update slice operation based
   // on the shape of operand and update.
   static StatusOr<Shape> InferDynamicUpdateSliceShape(
       const Shape& operand_shape, const Shape& update_shape,
-      const Shape& start_indices_shape);
+      absl::Span<const Shape> start_index_shapes,
+      bool allow_scalar_indices = false);
 
   // Infers the shape produced by doing a compile-time-constant indexing into
   // the given input shape. This is essential for operations on tuples, because
diff --git a/tensorflow/compiler/xla/xla.proto b/tensorflow/compiler/xla/xla.proto
index 0e8fa73f81..e2d7b6ef46 100644
--- a/tensorflow/compiler/xla/xla.proto
+++ b/tensorflow/compiler/xla/xla.proto
@@ -230,7 +230,11 @@ message DebugOptions {
   // Enable fast math with eigen in the HLO evaluator.
   bool xla_hlo_evaluator_use_fast_path = 106;
 
-  // Next id: 107
+  // Temporary option to allow support for both the R1 and the scalar index
+  // versions of DynamicSlice and DynamicUpdateSlice. Only used for testing.
+  bool xla_allow_scalar_index_dynamic_ops = 107;
+
+  // Next id: 108
 
   // Extra options to pass to the compilation backend (e.g. LLVM); specific
   // interpretation of these values is left to the backend.
-- 
GitLab


From c2c069b9f42da12a825c940a1d19fba713577127 Mon Sep 17 00:00:00 2001
From: Michael Kuperstein <mkuper@google.com>
Date: Fri, 28 Dec 2018 13:40:47 -0800
Subject: [PATCH 0066/1095] [XLA] Implement scalar index form of DS/DUS on
 Evaluator.

This form is not yet documented, unsupposed by HLO passes, and only
passes verification under a flag.

PiperOrigin-RevId: 227165947
---
 .../xla/service/hlo_evaluator_typed_visitor.h | 127 +++++++++++++-----
 .../compiler/xla/service/hlo_instructions.h   |   4 +-
 2 files changed, 95 insertions(+), 36 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
index 70ddedf62b..6f3208be13 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
@@ -1406,12 +1406,22 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
     auto operand = dynamic_slice->operand(0);
     auto start_indices = dynamic_slice->operand(1);
     auto result_shape = dynamic_slice->shape();
+    // TODO(b/118437727): Remove all of this nonsense.
+    // We may get an instruction without a parent module. In this case, assume
+    // scalar indices are not allowed.
+    bool allow_scalar_index = false;
+    if (dynamic_slice->GetModule() != nullptr) {
+      allow_scalar_index = dynamic_slice->GetModule()
+                               ->config()
+                               .debug_options()
+                               .xla_allow_scalar_index_dynamic_ops();
+    }
     TF_ASSIGN_OR_RETURN(
         auto inferred_return_shape,
         ShapeInference::InferDynamicSliceShape(
             operand->shape(),
             Cast<HloDynamicSliceInstruction>(dynamic_slice)->index_shapes(),
-            dynamic_slice->dynamic_slice_sizes()));
+            dynamic_slice->dynamic_slice_sizes(), allow_scalar_index));
     TF_RET_CHECK(ShapeUtil::Compatible(result_shape, inferred_return_shape))
         << "return shape is set to: " << ShapeUtil::HumanString(result_shape)
         << " but is inferred to be: "
@@ -1420,33 +1430,39 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
         primitive_util::IsIntegralType(start_indices->shape().element_type()));
 
     const Literal& operand_literal = parent_->GetEvaluatedLiteralFor(operand);
-    const Literal& start_indices_literal =
-        parent_->GetEvaluatedLiteralFor(start_indices);
 
     switch (start_indices->shape().element_type()) {
       case S32: {
         TF_ASSIGN_OR_RETURN(
             parent_->evaluated_[dynamic_slice],
-            DynamicSlice<int32>(operand_literal, start_indices_literal,
-                                result_shape));
+            DynamicSlice<int32>(
+                operand_literal,
+                absl::MakeConstSpan(dynamic_slice->operands()).subspan(1),
+                result_shape));
       } break;
       case S64: {
         TF_ASSIGN_OR_RETURN(
             parent_->evaluated_[dynamic_slice],
-            DynamicSlice<int64>(operand_literal, start_indices_literal,
-                                result_shape));
+            DynamicSlice<int64>(
+                operand_literal,
+                absl::MakeConstSpan(dynamic_slice->operands()).subspan(1),
+                result_shape));
       } break;
       case U32: {
         TF_ASSIGN_OR_RETURN(
             parent_->evaluated_[dynamic_slice],
-            DynamicSlice<uint32>(operand_literal, start_indices_literal,
-                                 result_shape));
+            DynamicSlice<uint32>(
+                operand_literal,
+                absl::MakeConstSpan(dynamic_slice->operands()).subspan(1),
+                result_shape));
       } break;
       case U64: {
         TF_ASSIGN_OR_RETURN(
             parent_->evaluated_[dynamic_slice],
-            DynamicSlice<uint64>(operand_literal, start_indices_literal,
-                                 result_shape));
+            DynamicSlice<uint64>(
+                operand_literal,
+                absl::MakeConstSpan(dynamic_slice->operands()).subspan(1),
+                result_shape));
       } break;
       default:
         LOG(FATAL) << "HandleDynamicSlice: unhandled primitive type for "
@@ -1463,12 +1479,20 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
     auto update = dynamic_update_slice->operand(1);
     auto start_indices = dynamic_update_slice->operand(2);
     auto result_shape = dynamic_update_slice->shape();
+    bool allow_scalar_index = false;
+    if (dynamic_update_slice->GetModule() != nullptr) {
+      allow_scalar_index = dynamic_update_slice->GetModule()
+                               ->config()
+                               .debug_options()
+                               .xla_allow_scalar_index_dynamic_ops();
+    }
     TF_ASSIGN_OR_RETURN(
         auto inferred_return_shape,
         ShapeInference::InferDynamicUpdateSliceShape(
             operand->shape(), update->shape(),
             Cast<HloDynamicUpdateSliceInstruction>(dynamic_update_slice)
-                ->index_shapes()));
+                ->index_shapes(),
+            allow_scalar_index));
     TF_RET_CHECK(ShapeUtil::Compatible(result_shape, inferred_return_shape))
         << "return shape is set to: " << ShapeUtil::HumanString(result_shape)
         << " but is inferred to be: "
@@ -1479,33 +1503,39 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
 
     const Literal& operand_literal = parent_->GetEvaluatedLiteralFor(operand);
     const Literal& update_literal = parent_->GetEvaluatedLiteralFor(update);
-    const Literal& start_indices_literal =
-        parent_->GetEvaluatedLiteralFor(start_indices);
 
     switch (start_indices->shape().element_type()) {
       case S32: {
         TF_ASSIGN_OR_RETURN(
             parent_->evaluated_[dynamic_update_slice],
-            DynamicUpdateSlice<int32>(operand_literal, update_literal,
-                                      start_indices_literal));
+            DynamicUpdateSlice<int32>(
+                operand_literal, update_literal,
+                absl::MakeConstSpan(dynamic_update_slice->operands())
+                    .subspan(2)));
       } break;
       case S64: {
         TF_ASSIGN_OR_RETURN(
             parent_->evaluated_[dynamic_update_slice],
-            DynamicUpdateSlice<int64>(operand_literal, update_literal,
-                                      start_indices_literal));
+            DynamicUpdateSlice<int64>(
+                operand_literal, update_literal,
+                absl::MakeConstSpan(dynamic_update_slice->operands())
+                    .subspan(2)));
       } break;
       case U32: {
         TF_ASSIGN_OR_RETURN(
             parent_->evaluated_[dynamic_update_slice],
-            DynamicUpdateSlice<uint32>(operand_literal, update_literal,
-                                       start_indices_literal));
+            DynamicUpdateSlice<uint32>(
+                operand_literal, update_literal,
+                absl::MakeConstSpan(dynamic_update_slice->operands())
+                    .subspan(2)));
       } break;
       case U64: {
         TF_ASSIGN_OR_RETURN(
             parent_->evaluated_[dynamic_update_slice],
-            DynamicUpdateSlice<uint64>(operand_literal, update_literal,
-                                       start_indices_literal));
+            DynamicUpdateSlice<uint64>(
+                operand_literal, update_literal,
+                absl::MakeConstSpan(dynamic_update_slice->operands())
+                    .subspan(2)));
       } break;
       default:
         LOG(FATAL) << "HandleDynamicUpdateSlice: unhandled primitive type for "
@@ -2742,12 +2772,27 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
   }
 
   template <typename IndexT>
-  StatusOr<Literal> DynamicSlice(const Literal& operand_literal,
-                                 const Literal& start_indices_literal,
-                                 const Shape& result_shape) {
-    auto start_indices_typed = start_indices_literal.data<IndexT>();
-    std::vector<int64> start(start_indices_typed.begin(),
-                             start_indices_typed.end());
+  StatusOr<Literal> DynamicSlice(
+      const Literal& operand_literal,
+      absl::Span<HloInstruction* const> start_indices,
+      const Shape& result_shape) {
+    std::vector<int64> start;
+    // TODO(b/118437727): Remove the R1 code-path. Note that to distinguish
+    // between the cases, this currently assumes there is at least 1 index. That
+    // is wrong in the general case, because for scalar indices, if the operand
+    // is scalar, then there are no indices. This problem with resolve itself.
+    const HloInstruction* first_index = start_indices[0];
+    if (first_index->shape().rank() == 1) {
+      auto start_indices_typed =
+          parent_->GetEvaluatedLiteralFor(first_index).data<IndexT>();
+      start = std::vector<int64>(start_indices_typed.begin(),
+                                 start_indices_typed.end());
+    } else {
+      for (HloInstruction* index : start_indices) {
+        start.push_back(
+            parent_->GetEvaluatedLiteralFor(index).GetFirstElement<IndexT>());
+      }
+    }
 
     // Clamp the start indices so the slice is in-bounds w.r.t the operand.
     for (int64 i = 0; i < start.size(); ++i) {
@@ -2773,14 +2818,28 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
   }
 
   template <typename IndexT>
-  StatusOr<Literal> DynamicUpdateSlice(const Literal& operand_literal,
-                                       const Literal& update_literal,
-                                       const Literal& start_indices_literal) {
+  StatusOr<Literal> DynamicUpdateSlice(
+      const Literal& operand_literal, const Literal& update_literal,
+      absl::Span<HloInstruction* const> start_indices) {
     auto result = operand_literal.Clone();
-    auto start_indices_typed = start_indices_literal.data<IndexT>();
     const auto rank = result.shape().rank();
-    std::vector<int64> start(start_indices_typed.begin(),
-                             start_indices_typed.end());
+    std::vector<int64> start;
+    // TODO(b/118437727): Remove the R1 code-path. Note that to distinguish
+    // between the cases, this currently assumes there is at least 1 index. That
+    // is wrong in the general case, because for scalar indices, if the operand
+    // is scalar, then there are no indices. This problem with resolve itself.
+    const HloInstruction* first_index = start_indices[0];
+    if (first_index->shape().rank() == 1) {
+      auto start_indices_typed =
+          parent_->GetEvaluatedLiteralFor(first_index).data<IndexT>();
+      start = std::vector<int64>(start_indices_typed.begin(),
+                                 start_indices_typed.end());
+    } else {
+      for (HloInstruction* index : start_indices) {
+        start.push_back(
+            parent_->GetEvaluatedLiteralFor(index).GetFirstElement<IndexT>());
+      }
+    }
     // Clamp the update start indices so the slice is in-bounds w.r.t the
     // operand.
     for (int64 i = 0; i < rank; ++i) {
diff --git a/tensorflow/compiler/xla/service/hlo_instructions.h b/tensorflow/compiler/xla/service/hlo_instructions.h
index 83b6760683..e6111cfb57 100644
--- a/tensorflow/compiler/xla/service/hlo_instructions.h
+++ b/tensorflow/compiler/xla/service/hlo_instructions.h
@@ -1186,12 +1186,12 @@ class HloDynamicIndexInstruction : public HloInstruction {
   virtual int64 first_index_operand_number() const = 0;
 
   // Returns a subspan of operands which represent the start indices.
-  absl::Span<HloInstruction* const> index_operands() {
+  absl::Span<HloInstruction* const> index_operands() const {
     return absl::MakeSpan(operands()).subspan(first_index_operand_number());
   }
 
   // Returns the shapes of the index operands.
-  std::vector<Shape> index_shapes() {
+  std::vector<Shape> index_shapes() const {
     std::vector<Shape> shapes;
     auto indices = index_operands();
     for (const HloInstruction* index : indices) {
-- 
GitLab


From d2ef9baf6e3551628b597ab626e84035af4dd3bb Mon Sep 17 00:00:00 2001
From: mdfaijul <md.faijul.amin@intel.com>
Date: Fri, 28 Dec 2018 14:18:53 -0800
Subject: [PATCH 0067/1095] changed ExpectTensorNear to ExpectClose

---
 tensorflow/core/kernels/conv_ops_test.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/kernels/conv_ops_test.cc b/tensorflow/core/kernels/conv_ops_test.cc
index d144576249..09e300dd1c 100644
--- a/tensorflow/core/kernels/conv_ops_test.cc
+++ b/tensorflow/core/kernels/conv_ops_test.cc
@@ -775,7 +775,7 @@ class FusedConv2DOpTest : public OpsTestBase {
     // a full sum reduction, which causes larger numerical error
     // than usual cases.
     if (image_width == filter_size && image_height == filter_size) {
-      test::ExpectTensorNear<T>(conv_2d, fused_conv_2d, 1e-4);
+      test::ExpectClose(conv_2d, fused_conv_2d, /*atol=*/1e-4);
     } else {
       test::ExpectClose(conv_2d, fused_conv_2d, /*atol=*/1e-6);
     }
@@ -825,7 +825,7 @@ class FusedConv2DOpTest : public OpsTestBase {
     // a full sum reduction, which causes larger numerical error
     // than usual cases.
     if (image_width == filter_size && image_height == filter_size) {
-      test::ExpectTensorNear<T>(conv_2d, fused_conv_2d, 1e-4);
+      test::ExpectClose(conv_2d, fused_conv_2d, /*atol=*/1e-4);
     } else {
       test::ExpectClose(conv_2d, fused_conv_2d, /*atol=*/1e-6);
     }
-- 
GitLab


From 24ac448941c15acc4e035bf16427258f61463b09 Mon Sep 17 00:00:00 2001
From: Jianwei Xie <xiejw@google.com>
Date: Fri, 28 Dec 2018 14:19:50 -0800
Subject: [PATCH 0068/1095] Enabled batch norm in correctness test.

PiperOrigin-RevId: 227169465
---
 .../python/keras_correctness_test.py          | 32 ++++++++++++++-----
 1 file changed, 24 insertions(+), 8 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/keras_correctness_test.py b/tensorflow/contrib/distribute/python/keras_correctness_test.py
index 6dcc9a1f8d..3abdee2c0e 100644
--- a/tensorflow/contrib/distribute/python/keras_correctness_test.py
+++ b/tensorflow/contrib/distribute/python/keras_correctness_test.py
@@ -63,6 +63,9 @@ def all_strategy_combinations_with_graph_mode():
 
 
 def strategy_and_input_combinations():
+  def cnn_model_with_batch_norm(**kwargs):
+    return _create_cnn_model(with_batch_norm=True, **kwargs)
+
   return (
       combinations.times(
           combinations.combine(distribution=all_strategies),
@@ -72,6 +75,10 @@ def strategy_and_input_combinations():
           combinations.combine(model_with_data=[
               ModelWithData('dnn', _create_dnn_model, _dnn_training_data),
               ModelWithData('cnn', _create_cnn_model, _cnn_training_data),
+              ModelWithData('cnn_batch_norm',
+                            cnn_model_with_batch_norm,
+                            _cnn_training_data,
+                            with_batch_norm=True),
           ])))
 
 
@@ -99,10 +106,11 @@ class ModelWithData(object):
   The model_fn must take two arguments: initial_weights and distribution.
   """
 
-  def __init__(self, name, model_fn, data_fn):
+  def __init__(self, name, model_fn, data_fn, with_batch_norm=False):
     self.name = name
     self.model_fn = model_fn
     self.data_fn = data_fn
+    self.with_batch_norm = with_batch_norm
 
   def __repr__(self):
     return self.name
@@ -158,16 +166,15 @@ def _cnn_training_data(count=_GLOBAL_BATCH_SIZE * _EVAL_STEPS,
   return x_train, y_train, x_predict
 
 
-def _create_cnn_model(initial_weights=None, distribution=None):
+def _create_cnn_model(initial_weights=None, distribution=None,
+                      with_batch_norm=False):
   with MaybeDistributionScope(distribution):
     image = keras.layers.Input(shape=(28, 28, 3), name='image')
     c1 = keras.layers.Conv2D(
         name='conv1', filters=16, kernel_size=(3, 3), strides=(4, 4))(
             image)
-    # TODO(xiejw): Consider to enable the batch norm layer even it is not easy
-    # to test with TPU.
-    #
-    #   c1 = keras.layers.BatchNormalization(name='bn1')(image)
+    if with_batch_norm:
+      c1 = keras.layers.BatchNormalization(name='bn1')(c1)
     c1 = keras.layers.MaxPooling2D(pool_size=(2, 2))(c1)
     logits = keras.layers.Dense(
         10, activation='softmax', name='pred')(
@@ -470,8 +477,17 @@ class TestDistributionStrategyCorrectness(test.TestCase,
       results_without_ds = fit_eval_and_predict(
           initial_weights, input_fn=input_fn, model_fn=model_fn,
           distribution=None)
-      compare_results(results_with_ds, results_without_ds, distribution,
-                      testcase=self)
+
+      # First, special case, for multi-replica distributed training, batch norm
+      # is not aggregated globally. So it is expected to have different weights.
+      if (model_with_data.with_batch_norm and
+          distribution.num_replicas_in_sync > 1):
+        with self.assertRaises(AssertionError):
+          compare_results(results_with_ds, results_without_ds, distribution,
+                          testcase=self)
+      else:
+        compare_results(results_with_ds, results_without_ds, distribution,
+                        testcase=self)
 
   @combinations.generate(all_strategy_combinations_with_graph_mode())
   def test_dynamic_lr(self, distribution):
-- 
GitLab


From 35f2d889bc136094c6f6b200f3b2913950ed9bbe Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Fri, 28 Dec 2018 14:57:13 -0800
Subject: [PATCH 0069/1095] Tidy up work for dynamic kernels:  - rename
 all_kernels_statically_linked to all_kernels_impl  - For now, exclude dataset
 ops kernels and list_kernels from dynamic loading    kernels.

PiperOrigin-RevId: 227172592
---
 tensorflow/core/BUILD | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 640f881e5f..326f9587e8 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -1371,7 +1371,7 @@ cc_library(
 
 # This includes implementations of all kernels built into TensorFlow.
 cc_library(
-    name = "all_kernels_statically_linked",
+    name = "all_kernels_impl",
     visibility = ["//visibility:private"],
     deps = [
         "//tensorflow/core/kernels:array",
@@ -1387,7 +1387,6 @@ cc_library(
         "//tensorflow/core/kernels:ctc_ops",
         "//tensorflow/core/kernels:cudnn_rnn_kernels",
         "//tensorflow/core/kernels:data_flow",
-        "//tensorflow/core/kernels:dataset_ops",
         "//tensorflow/core/kernels:decode_proto_op",
         "//tensorflow/core/kernels:encode_proto_op",
         "//tensorflow/core/kernels:fake_quant_ops",
@@ -1398,7 +1397,6 @@ cc_library(
         "//tensorflow/core/kernels:image",
         "//tensorflow/core/kernels:io",
         "//tensorflow/core/kernels:linalg",
-        "//tensorflow/core/kernels:list_kernels",
         "//tensorflow/core/kernels:lookup",
         "//tensorflow/core/kernels:logging",
         "//tensorflow/core/kernels:manip",
@@ -1461,8 +1459,13 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = if_dynamic_kernels(
         [],
-        otherwise = [":all_kernels_statically_linked"],
-    ),
+        otherwise = [":all_kernels_impl"],
+    ) + [
+        # TODO(gunan): Work on the API between these and rest of TF and make
+        # these also dynamically loading.
+        "//tensorflow/core/kernels:dataset_ops",  # Depends on grappler
+        "//tensorflow/core/kernels:list_kernels",  # Depends on variant_op_registry.h
+    ],
 )
 
 tf_cuda_library(
-- 
GitLab


From 0a9a87fa1a47efbd97a2c6741643edd9ca81bb5a Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Fri, 28 Dec 2018 15:31:24 -0800
Subject: [PATCH 0070/1095] Change GetExecutablePath to return correct
 directory for python.

PiperOrigin-RevId: 227175352
---
 tensorflow/core/BUILD                         | 23 +++++++
 tensorflow/core/platform/env.cc               | 29 ++++++++-
 .../core/platform/fake_python_env_test.cc     | 65 +++++++++++++++++++
 3 files changed, 116 insertions(+), 1 deletion(-)
 create mode 100644 tensorflow/core/platform/fake_python_env_test.cc

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 326f9587e8..4862c90aca 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -3496,6 +3496,29 @@ tf_cc_test(
     ],
 )
 
+tf_cc_test(
+    name = "platform_fake_python_env_test",
+    size = "small",
+    srcs = ["platform/fake_python_env_test.cc"],
+    args = [
+        "/some/path/to/pythontest.runfiles/org_tensorflow/stuff/to/run.py",
+    ],
+    tags = [
+        "local",
+        "no_windows",
+        "nogpu",
+        "nomac",
+        "notap",
+    ],
+    deps = [
+        ":lib",
+        ":lib_internal",
+        ":lib_test_internal",
+        ":test",
+        ":test_main",
+    ],
+)
+
 tf_cc_test(
     name = "platform_abi_test",
     size = "small",
diff --git a/tensorflow/core/platform/env.cc b/tensorflow/core/platform/env.cc
index afc4201e53..6339417445 100644
--- a/tensorflow/core/platform/env.cc
+++ b/tensorflow/core/platform/env.cc
@@ -29,6 +29,9 @@ limitations under the License.
 #include "tensorflow/core/platform/windows/wide_char.h"
 #define PATH_MAX MAX_PATH
 #else
+#include <fcntl.h>
+#include <string.h>
+#include <sys/types.h>
 #include <unistd.h>
 #endif
 
@@ -314,7 +317,31 @@ string Env::GetExecutablePath() {
   string file_path = WideCharToUtf8(wc_file_path);
   std::copy(file_path.begin(), file_path.end(), exe_path);
 #else
-  CHECK_NE(-1, readlink("/proc/self/exe", exe_path, sizeof(exe_path) - 1));
+  char buf[PATH_MAX] = {0};
+  int path_length = readlink("/proc/self/exe", buf, sizeof(buf) - 1);
+  CHECK_NE(-1, path_length);
+
+  if (strstr(buf, "python") != nullptr) {
+    // Discard the path of the python binary, and any flags.
+    int fd = open("/proc/self/cmdline", O_RDONLY);
+    int cmd_length = read(fd, buf, PATH_MAX - 1);
+    CHECK_NE(-1, cmd_length);
+    int token_pos = 0;
+    for (bool token_is_first_or_flag = true; token_is_first_or_flag;) {
+      // Get token length, including null
+      int token_len = strlen(&buf[token_pos]) + 1;
+      token_is_first_or_flag = false;
+      // Check if we can skip without overshooting
+      if (token_pos + token_len < cmd_length) {
+        token_pos += token_len;
+        token_is_first_or_flag = (buf[token_pos] == '-');  // token is a flag
+      }
+    }
+    snprintf(exe_path, sizeof(exe_path), "%s", &buf[token_pos]);
+  } else {
+    snprintf(exe_path, sizeof(exe_path), "%s", buf);
+  }
+
 #endif
   // Make sure it's null-terminated:
   exe_path[sizeof(exe_path) - 1] = 0;
diff --git a/tensorflow/core/platform/fake_python_env_test.cc b/tensorflow/core/platform/fake_python_env_test.cc
new file mode 100644
index 0000000000..b521db3c05
--- /dev/null
+++ b/tensorflow/core/platform/fake_python_env_test.cc
@@ -0,0 +1,65 @@
+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// This file has "python" in its name. Thus, it should trigger the python
+// specific code paths.
+
+#include <sys/stat.h>
+#include <unistd.h>
+#include <iostream>
+#include <string>
+
+#include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/platform/env.h"
+#include "tensorflow/core/platform/test.h"
+
+int myargc;
+char** myargv;
+
+char kMagicBazelDirSubstring[] = ".runfiles/org_tensorflow";
+char kPythonFile[] =
+    "/some/path/to/pythontest.runfiles/org_tensorflow/stuff/to/run.py";
+
+namespace tensorflow {
+
+TEST(FakePythonEnvTest, GetExecutablePath) {
+  // See if argc is greater than 1 and first arg is kPythonFile
+  // If not, rerun the executable with proper args.
+  if (myargc <= 1 || strstr(myargv[1], kMagicBazelDirSubstring) == nullptr) {
+    const char* filename = myargv[0];
+    char* new_argv[] = {
+        myargv[0],
+        kPythonFile,
+        nullptr,
+    };
+
+    execv(filename, new_argv);
+  }
+
+  Env* env = Env::Default();
+  // We depend on the file/executable name to include python and fool the
+  // library to think this is running under the python interpreter.
+  string path = env->GetExecutablePath();
+  EXPECT_TRUE(strstr(path.c_str(), kMagicBazelDirSubstring) != nullptr);
+}
+
+}  // namespace tensorflow
+
+int main(int argc, char** argv) {
+  ::testing::InitGoogleTest(&argc, argv);
+  myargc = argc;
+  myargv = argv;
+  return RUN_ALL_TESTS();
+}
-- 
GitLab


From a4781da6b85f9b1e327dcdf8cd35b6a6385c600f Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Fri, 28 Dec 2018 16:14:19 -0800
Subject: [PATCH 0071/1095] Fix a bug in GetRunFilesDir in posix systems.

Correctly handle what is returned by GetExecutablePath if the path returned is
already a path under the runfiles directory.

PiperOrigin-RevId: 227178949
---
 tensorflow/core/platform/posix/env.cc | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/tensorflow/core/platform/posix/env.cc b/tensorflow/core/platform/posix/env.cc
index 0a939aef25..d87e5dcfe7 100644
--- a/tensorflow/core/platform/posix/env.cc
+++ b/tensorflow/core/platform/posix/env.cc
@@ -121,13 +121,25 @@ class PosixEnv : public Env {
 
   string GetRunfilesDir() override {
     string bin_path = this->GetExecutablePath();
-    string runfiles_path = bin_path + ".runfiles/org_tensorflow";
+    string runfiles_suffix = ".runfiles/org_tensorflow";
+    std::size_t pos = bin_path.find(runfiles_suffix);
+
+    // Sometimes (when executing under python) bin_path returns the full path to
+    // the python scripts under runfiles. Get the substring.
+    if (pos != std::string::npos) {
+      return bin_path.substr(0, pos + runfiles_suffix.length());
+    }
+
+    // See if we have the executable path. if executable.runfiles exists, return
+    // that folder.
+    string runfiles_path = bin_path + runfiles_suffix;
     Status s = this->IsDirectory(runfiles_path);
     if (s.ok()) {
       return runfiles_path;
-    } else {
-      return bin_path.substr(0, bin_path.find_last_of("/\\"));
     }
+
+    // If nothing can be found, return something close.
+    return bin_path.substr(0, bin_path.find_last_of("/\\"));
   }
 
  private:
-- 
GitLab


From 719309ad0d593f2977c422575248018314e3160d Mon Sep 17 00:00:00 2001
From: Davide Libenzi <dlibenzi@google.com>
Date: Fri, 28 Dec 2018 16:19:33 -0800
Subject: [PATCH 0072/1095] Make user buffer alias pass to check with the
 existing module alias configuration, when populating aliases.

PiperOrigin-RevId: 227179340
---
 .../xla/service/hlo_input_output_alias_config.cc         | 9 +++++++++
 .../compiler/xla/service/hlo_input_output_alias_config.h | 7 +++++++
 2 files changed, 16 insertions(+)

diff --git a/tensorflow/compiler/xla/service/hlo_input_output_alias_config.cc b/tensorflow/compiler/xla/service/hlo_input_output_alias_config.cc
index 6e1597fd03..70d4df5d1c 100644
--- a/tensorflow/compiler/xla/service/hlo_input_output_alias_config.cc
+++ b/tensorflow/compiler/xla/service/hlo_input_output_alias_config.cc
@@ -17,9 +17,17 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/hlo_module.h"
 
 namespace xla {
+
+bool HloInputOutputAliasConfig::OutputHasAlias(
+    const ShapeIndex& output_index) const {
+  return aliased_output_indices_.count(output_index) > 0;
+}
+
 Status HloInputOutputAliasConfig::SetUpAlias(const ShapeIndex& output_index,
                                              int64 param_number,
                                              const ShapeIndex& param_index) {
+  TF_RET_CHECK(!OutputHasAlias(output_index))
+      << "Output index " << output_index << " already has an alias setup";
   TF_RET_CHECK(ShapeUtil::IndexIsValid(alias_.shape(), output_index))
       << absl::StrCat("Tring to set up alias at ", output_index.ToString(),
                       " which is an invalid index for shape ",
@@ -33,6 +41,7 @@ Status HloInputOutputAliasConfig::SetUpAlias(const ShapeIndex& output_index,
       alias_.element(output_index)->second.ToString());
   (*alias_.mutable_element(output_index)) =
       std::make_pair(param_number, param_index);
+  aliased_output_indices_.insert(output_index);
   VLOG(4) << "Set up alias between output index " << output_index.ToString()
           << " and parameter " << param_index << " at index "
           << param_index.ToString();
diff --git a/tensorflow/compiler/xla/service/hlo_input_output_alias_config.h b/tensorflow/compiler/xla/service/hlo_input_output_alias_config.h
index 439676b154..3967743d53 100644
--- a/tensorflow/compiler/xla/service/hlo_input_output_alias_config.h
+++ b/tensorflow/compiler/xla/service/hlo_input_output_alias_config.h
@@ -18,6 +18,7 @@ limitations under the License.
 
 #include <utility>
 
+#include "absl/container/flat_hash_set.h"
 #include "absl/types/optional.h"
 #include "tensorflow/compiler/xla/service/hlo.pb.h"
 #include "tensorflow/compiler/xla/shape_tree.h"
@@ -47,6 +48,9 @@ class HloInputOutputAliasConfig {
   bool ParameterHasAlias(int64 param_number,
                          const ShapeIndex& param_index) const;
 
+  // Checks whether the provided output index has already been aliased.
+  bool OutputHasAlias(const ShapeIndex& output_index) const;
+
   // (De)Serializes an HloInputOutoutAliasConfig to/from an
   // HloInputOutoutAliasProto.
   HloInputOutputAliasProto ToProto() const;
@@ -93,6 +97,9 @@ class HloInputOutputAliasConfig {
   // is a pair of parameter number and index into the buffer. If the value is
   // nullopt, it means there is no parameter aliasing for this output.
   ShapeTree<absl::optional<std::pair<int64, ShapeIndex>>> alias_;
+
+  // The indices of the output which have been aliased.
+  absl::flat_hash_set<ShapeIndex> aliased_output_indices_;
 };
 
 std::ostream& operator<<(std::ostream& out,
-- 
GitLab


From c59db932b0334c9c1e2e385e6435ee31ea3d7087 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Fri, 28 Dec 2018 16:32:01 -0800
Subject: [PATCH 0073/1095] Upgrade default TF cuda version to cuda 10.

PiperOrigin-RevId: 227180147
---
 configure.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configure.py b/configure.py
index c588381d40..bb59063f79 100644
--- a/configure.py
+++ b/configure.py
@@ -33,7 +33,7 @@ except ImportError:
   from distutils.spawn import find_executable as which
 # pylint: enable=g-import-not-at-top
 
-_DEFAULT_CUDA_VERSION = '9.0'
+_DEFAULT_CUDA_VERSION = '10.0'
 _DEFAULT_CUDNN_VERSION = '7'
 _DEFAULT_CUDA_COMPUTE_CAPABILITIES = '3.5,7.0'
 _DEFAULT_CUDA_PATH = '/usr/local/cuda'
-- 
GitLab


From 7966a07f756730650b29736958de53e99d284f68 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Fri, 28 Dec 2018 17:05:41 -0800
Subject: [PATCH 0074/1095] Remove dependency from cloud kernels to cloud op
 libraries.

PiperOrigin-RevId: 227182645
---
 tensorflow/contrib/cloud/kernels/BUILD | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tensorflow/contrib/cloud/kernels/BUILD b/tensorflow/contrib/cloud/kernels/BUILD
index 1311063ec0..20f8c2b245 100644
--- a/tensorflow/contrib/cloud/kernels/BUILD
+++ b/tensorflow/contrib/cloud/kernels/BUILD
@@ -27,7 +27,6 @@ tf_kernel_library(
     deps = [
         ":bigquery_table_accessor",
         ":bigquery_table_partition_proto_cc",
-        "//tensorflow/contrib/cloud:bigquery_reader_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:reader_base",
@@ -79,7 +78,6 @@ tf_kernel_library(
     srcs = ["gcs_config_ops.cc"],
     visibility = ["//tensorflow:internal"],
     deps = [
-        "//tensorflow/contrib/cloud:gcs_config_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core/platform/cloud:curl_http_request",
-- 
GitLab


From 447b5edf2acb49a0b4fbaa4d37b099f0304894a1 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Fri, 28 Dec 2018 17:11:07 -0800
Subject: [PATCH 0075/1095] Add windows_and_api_version_2 option to select
 statements in build_config.bzl to allow matching both on windows and
 api_version_2.

PiperOrigin-RevId: 227183002
---
 tensorflow/BUILD                                  | 9 +++++++++
 tensorflow/core/platform/default/build_config.bzl | 6 ++++--
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index f07e7365d3..0b4498ef12 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -370,6 +370,15 @@ config_setting(
     define_values = {"tf_api_version": "2"},
 )
 
+# This flag is defined for select statements that match both
+# on 'windows' and 'api_version_2'. In this case, bazel requires
+# having a flag which is a superset of these two.
+config_setting(
+    name = "windows_and_api_version_2",
+    define_values = {"tf_api_version": "2"},
+    values = {"cpu": "x64_windows"},
+)
+
 package_group(
     name = "internal",
     packages = [
diff --git a/tensorflow/core/platform/default/build_config.bzl b/tensorflow/core/platform/default/build_config.bzl
index 58fd0e2712..e26828c75e 100644
--- a/tensorflow/core/platform/default/build_config.bzl
+++ b/tensorflow/core/platform/default/build_config.bzl
@@ -663,8 +663,9 @@ def tf_additional_cloud_op_deps():
         "//tensorflow:ios": [],
         "//tensorflow:linux_s390x": [],
         "//tensorflow:windows": [],
-        "//tensorflow:no_gcp_support": [],
         "//tensorflow:api_version_2": [],
+        "//tensorflow:windows_and_api_version_2": [],
+        "//tensorflow:no_gcp_support": [],
         "//conditions:default": [
             "//tensorflow/contrib/cloud:bigquery_reader_ops_op_lib",
             "//tensorflow/contrib/cloud:gcs_config_ops_op_lib",
@@ -678,8 +679,9 @@ def tf_additional_cloud_kernel_deps():
         "//tensorflow:ios": [],
         "//tensorflow:linux_s390x": [],
         "//tensorflow:windows": [],
-        "//tensorflow:no_gcp_support": [],
         "//tensorflow:api_version_2": [],
+        "//tensorflow:windows_and_api_version_2": [],
+        "//tensorflow:no_gcp_support": [],
         "//conditions:default": [
             "//tensorflow/contrib/cloud/kernels:bigquery_reader_ops",
             "//tensorflow/contrib/cloud/kernels:gcs_config_ops",
-- 
GitLab


From 817504f1e487fe91b8f0442ad945b5397188f981 Mon Sep 17 00:00:00 2001
From: Yuanzhong Xu <yuanzx@google.com>
Date: Fri, 28 Dec 2018 18:05:12 -0800
Subject: [PATCH 0076/1095] Allow ReplaceOperandWith and ReplaceAllUsesWith to
 accept different shapes if specified.

PiperOrigin-RevId: 227186196
---
 .../compiler/xla/service/hlo_instruction.cc   | 25 +++++++++++++++----
 .../compiler/xla/service/hlo_instruction.h    | 15 +++++++++--
 .../compiler/xla/service/hlo_instructions.cc  |  2 +-
 3 files changed, 34 insertions(+), 8 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc
index 543fb66c01..66bc73740f 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.cc
+++ b/tensorflow/compiler/xla/service/hlo_instruction.cc
@@ -1885,6 +1885,16 @@ Status HloInstruction::ReplaceUseWith(HloInstruction* user,
 
 Status HloInstruction::ReplaceOperandWith(int64 operand_num,
                                           HloInstruction* new_operand) {
+  auto old_operand = operand(operand_num);
+  TF_RET_CHECK(ShapeUtil::CompatibleIgnoringFpPrecision(old_operand->shape(),
+                                                        new_operand->shape()))
+      << old_operand->shape() << " is not compatible with "
+      << new_operand->shape();
+  return ReplaceOperandWithDifferentShape(operand_num, new_operand);
+}
+
+Status HloInstruction::ReplaceOperandWithDifferentShape(
+    int64 operand_num, HloInstruction* new_operand) {
   TF_RET_CHECK(operand_num >= 0);
   TF_RET_CHECK(operand_num < operand_count());
   HloInstruction* old_operand = mutable_operand(operand_num);
@@ -1892,10 +1902,6 @@ Status HloInstruction::ReplaceOperandWith(int64 operand_num,
     return Status::OK();
   }
 
-  TF_RET_CHECK(ShapeUtil::CompatibleIgnoringFpPrecision(old_operand->shape(),
-                                                        new_operand->shape()))
-      << old_operand->shape() << " is not compatible with "
-      << new_operand->shape();
   operands_[operand_num] = new_operand;
 
   VLOG(3) << "Replacing operand " << operand_num << " of " << name() << " with "
@@ -1910,6 +1916,14 @@ Status HloInstruction::ReplaceOperandWith(int64 operand_num,
 }
 
 Status HloInstruction::ReplaceAllUsesWith(HloInstruction* new_producer) {
+  TF_RET_CHECK(
+      ShapeUtil::CompatibleIgnoringFpPrecision(shape(), new_producer->shape()))
+      << shape() << " is not compatible with " << new_producer->shape();
+  return ReplaceAllUsesWithDifferentShape(new_producer);
+}
+
+Status HloInstruction::ReplaceAllUsesWithDifferentShape(
+    HloInstruction* new_producer) {
   bool new_producer_is_user = false;
   for (HloInstruction* user : users()) {
     if (user == new_producer) {
@@ -1934,7 +1948,8 @@ Status HloInstruction::ReplaceAllUsesWith(HloInstruction* new_producer) {
     AddUser(new_producer);
   }
   if (parent_ && parent_->root_instruction() == this) {
-    parent_->set_root_instruction(new_producer);
+    parent_->set_root_instruction(new_producer,
+                                  /*accept_different_shape=*/true);
   }
 
   return Status::OK();
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.h b/tensorflow/compiler/xla/service/hlo_instruction.h
index 362d07e64f..2827eed0df 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.h
+++ b/tensorflow/compiler/xla/service/hlo_instruction.h
@@ -937,11 +937,16 @@ class HloInstruction {
   // operands of it which could be created due to this replacement.
   Status ReplaceUseWith(HloInstruction* user, HloInstruction* new_producer);
 
-  // Replaces the specified operand with new_operand.
+  // Replaces the specified operand with new_operand. The old and new operands
+  // must have compatible shapes ignoring floating-point precision.
   //
   // This function does NOT remove duplicated operands even if this instruction
   // is a fusion, so that the existing operand numbers do not change.
-  Status ReplaceOperandWith(int64 operand_no, HloInstruction* new_operand);
+  Status ReplaceOperandWith(int64 operand_num, HloInstruction* new_operand);
+
+  // Same as ReplaceOperandWith(), but new_operand can have a different shape.
+  Status ReplaceOperandWithDifferentShape(int64 operand_num,
+                                          HloInstruction* new_operand);
 
   // Replaces all uses of this instruction with the new producer. If
   // new_producer is a user of this instruction then new_producer remains a use
@@ -950,10 +955,16 @@ class HloInstruction {
   // If this instruction is the root of its computation, sets the computation's
   // root to new_producer.
   //
+  // The new producer must have a compatible shape ignoring floating-point
+  // precision.
+  //
   // If a user is a fusion instruction, this function will remove any duplicated
   // operands of it which could be created due to this replacement.
   Status ReplaceAllUsesWith(HloInstruction* new_producer);
 
+  // Same as ReplaceAllUsesWith, but new_producer can have a different shape.
+  Status ReplaceAllUsesWithDifferentShape(HloInstruction* new_producer);
+
   // Performs a postorder DFS visit using this node as the root. If
   // call_finish_visit is true, then DfsHloVisitor::FinishVisit is called when
   // complete. If ignore_control_predecessors is true, instructions only
diff --git a/tensorflow/compiler/xla/service/hlo_instructions.cc b/tensorflow/compiler/xla/service/hlo_instructions.cc
index 7a3fb6d9a2..7170dd7d81 100644
--- a/tensorflow/compiler/xla/service/hlo_instructions.cc
+++ b/tensorflow/compiler/xla/service/hlo_instructions.cc
@@ -1324,7 +1324,7 @@ HloInstruction* HloFusionInstruction::CloneAndFuseInternal(
     if (newly_created_tuple_instr) {
       HloInstruction* new_instr = parent()->AddInstruction(
           HloInstruction::CreateGetTupleElement(fused_root->shape(), this, 0));
-      TF_CHECK_OK(ReplaceAllUsesWith(new_instr));
+      TF_CHECK_OK(ReplaceAllUsesWithDifferentShape(new_instr));
     }
     int64 index = tuple_elements.size();
     if (instruction_to_fuse->opcode() == HloOpcode::kTuple) {
-- 
GitLab


From 3aaebcbcada0bb861ebfee0eea6068de0c7f37be Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Fri, 28 Dec 2018 18:31:41 -0800
Subject: [PATCH 0077/1095] Move default TF builds to cuda 10.

PiperOrigin-RevId: 227187422
---
 third_party/toolchains/gpus/cuda/BUILD        | 42 +++++++++----------
 .../toolchains/gpus/cuda/cuda/cuda_config.h   |  4 +-
 .../bin/crosstool_wrapper_driver_is_not_gcc   |  4 +-
 3 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/third_party/toolchains/gpus/cuda/BUILD b/third_party/toolchains/gpus/cuda/BUILD
index f63a0ea819..8bb22c0269 100644
--- a/third_party/toolchains/gpus/cuda/BUILD
+++ b/third_party/toolchains/gpus/cuda/BUILD
@@ -85,8 +85,8 @@ cc_library(
 
 cc_library(
     name = "cudart",
-    srcs = ["cuda/lib/libcudart.so.9.0"],
-    data = ["cuda/lib/libcudart.so.9.0"],
+    srcs = ["cuda/lib/libcudart.so.10.0"],
+    data = ["cuda/lib/libcudart.so.10.0"],
     includes = [
         ".",
         "cuda/include",
@@ -97,8 +97,8 @@ cc_library(
 
 cc_library(
     name = "cublas",
-    srcs = ["cuda/lib/libcublas.so.9.0"],
-    data = ["cuda/lib/libcublas.so.9.0"],
+    srcs = ["cuda/lib/libcublas.so.10.0"],
+    data = ["cuda/lib/libcublas.so.10.0"],
     includes = [
         ".",
         "cuda/include",
@@ -109,8 +109,8 @@ cc_library(
 
 cc_library(
     name = "cusolver",
-    srcs = ["cuda/lib/libcusolver.so.9.0"],
-    data = ["cuda/lib/libcusolver.so.9.0"],
+    srcs = ["cuda/lib/libcusolver.so.10.0"],
+    data = ["cuda/lib/libcusolver.so.10.0"],
     includes = [
         ".",
         "cuda/include",
@@ -143,8 +143,8 @@ cc_library(
 
 cc_library(
     name = "cufft",
-    srcs = ["cuda/lib/libcufft.so.9.0"],
-    data = ["cuda/lib/libcufft.so.9.0"],
+    srcs = ["cuda/lib/libcufft.so.10.0"],
+    data = ["cuda/lib/libcufft.so.10.0"],
     includes = [
         ".",
         "cuda/include",
@@ -155,8 +155,8 @@ cc_library(
 
 cc_library(
     name = "curand",
-    srcs = ["cuda/lib/libcurand.so.9.0"],
-    data = ["cuda/lib/libcurand.so.9.0"],
+    srcs = ["cuda/lib/libcurand.so.10.0"],
+    data = ["cuda/lib/libcurand.so.10.0"],
     includes = [
         ".",
         "cuda/include",
@@ -193,7 +193,7 @@ cc_library(
 
 cc_library(
     name = "cupti_dsos",
-    data = ["cuda/lib/libcupti.so.9.0"],
+    data = ["cuda/lib/libcupti.so.10.0"],
     includes = [
         ".",
         "cuda/include",
@@ -1193,7 +1193,7 @@ genrule(
         "cuda/include/vector_types.h",
     ],
     cmd = """
-if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp "/usr/local/cuda-9.0/include/CL/cl.h" "$(@D)/cuda/include/CL/cl.h" && cp "/usr/local/cuda-9.0/include/CL/cl.hpp" "$(@D)/cuda/include/CL/cl.hpp" && cp "/usr/local/cuda-9.0/include/CL/cl_egl.h" "$(@D)/cuda/include/CL/cl_egl.h" && cp "/usr/local/cuda-9.0/include/CL/cl_ext.h" "$(@D)/cuda/include/CL/cl_ext.h" && cp "/usr/local/cuda-9.0/include/CL/cl_gl.h" "$(@D)/cuda/include/CL/cl_gl.h" && cp "/usr/local/cuda-9.0/include/CL/cl_gl_ext.h" "$(@D)/cuda/include/CL/cl_gl_ext.h" && cp "/usr/local/cuda-9.0/include/CL/cl_platform.h" "$(@D)/cuda/include/CL/cl_platform.h" && cp "/usr/local/cuda-9.0/include/CL/opencl.h" "$(@D)/cuda/include/CL/opencl.h" && cp "/usr/local/cuda-9.0/include/builtin_types.h" "$(@D)/cuda/include/builtin_types.h" && cp "/usr/local/cuda-9.0/include/channel_descriptor.h" "$(@D)/cuda/include/channel_descriptor.h" && cp "/usr/local/cuda-9.0/include/common_functions.h" "$(@D)/cuda/include/common_functions.h" && cp "/usr/local/cuda-9.0/include/cooperative_groups.h" "$(@D)/cuda/include/cooperative_groups.h" && cp "/usr/local/cuda-9.0/include/cooperative_groups_helpers.h" "$(@D)/cuda/include/cooperative_groups_helpers.h" && cp "/usr/local/cuda-9.0/include/crt/common_functions.h" "$(@D)/cuda/include/crt/common_functions.h" && cp "/usr/local/cuda-9.0/include/crt/device_double_functions.h" "$(@D)/cuda/include/crt/device_double_functions.h" && cp "/usr/local/cuda-9.0/include/crt/device_double_functions.hpp" "$(@D)/cuda/include/crt/device_double_functions.hpp" && cp "/usr/local/cuda-9.0/include/crt/device_functions.h" "$(@D)/cuda/include/crt/device_functions.h" && cp "/usr/local/cuda-9.0/include/crt/device_functions.hpp" "$(@D)/cuda/include/crt/device_functions.hpp" && cp "/usr/local/cuda-9.0/include/crt/func_macro.h" "$(@D)/cuda/include/crt/func_macro.h" && cp "/usr/local/cuda-9.0/include/crt/host_config.h" "$(@D)/cuda/include/crt/host_config.h" && cp "/usr/local/cuda-9.0/include/crt/host_defines.h" "$(@D)/cuda/include/crt/host_defines.h" && cp "/usr/local/cuda-9.0/include/crt/host_runtime.h" "$(@D)/cuda/include/crt/host_runtime.h" && cp "/usr/local/cuda-9.0/include/crt/math_functions.h" "$(@D)/cuda/include/crt/math_functions.h" && cp "/usr/local/cuda-9.0/include/crt/math_functions.hpp" "$(@D)/cuda/include/crt/math_functions.hpp" && cp "/usr/local/cuda-9.0/include/crt/mma.h" "$(@D)/cuda/include/crt/mma.h" && cp "/usr/local/cuda-9.0/include/crt/mma.hpp" "$(@D)/cuda/include/crt/mma.hpp" && cp "/usr/local/cuda-9.0/include/crt/nvfunctional" "$(@D)/cuda/include/crt/nvfunctional" && cp "/usr/local/cuda-9.0/include/crt/sm_70_rt.h" "$(@D)/cuda/include/crt/sm_70_rt.h" && cp "/usr/local/cuda-9.0/include/crt/sm_70_rt.hpp" "$(@D)/cuda/include/crt/sm_70_rt.hpp" && cp "/usr/local/cuda-9.0/include/crt/storage_class.h" "$(@D)/cuda/include/crt/storage_class.h" && cp "/usr/local/cuda-9.0/include/cuComplex.h" "$(@D)/cuda/include/cuComplex.h" && cp "/usr/local/cuda-9.0/include/cublas.h" "$(@D)/cuda/include/cublas.h" && cp "/usr/local/cuda-9.0/include/cublasXt.h" "$(@D)/cuda/include/cublasXt.h" && cp "/usr/local/cuda-9.0/include/cublas_api.h" "$(@D)/cuda/include/cublas_api.h" && cp "/usr/local/cuda-9.0/include/cublas_v2.h" "$(@D)/cuda/include/cublas_v2.h" && cp "/usr/local/cuda-9.0/include/cuda.h" "$(@D)/cuda/include/cuda.h" && cp "/usr/local/cuda-9.0/include/cudaEGL.h" "$(@D)/cuda/include/cudaEGL.h" && cp "/usr/local/cuda-9.0/include/cudaGL.h" "$(@D)/cuda/include/cudaGL.h" && cp "/usr/local/cuda-9.0/include/cudaProfiler.h" "$(@D)/cuda/include/cudaProfiler.h" && cp "/usr/local/cuda-9.0/include/cudaVDPAU.h" "$(@D)/cuda/include/cudaVDPAU.h" && cp "/usr/local/cuda-9.0/include/cuda_device_runtime_api.h" "$(@D)/cuda/include/cuda_device_runtime_api.h" && cp "/usr/local/cuda-9.0/include/cuda_fp16.h" "$(@D)/cuda/include/cuda_fp16.h" && cp "/usr/local/cuda-9.0/include/cuda_fp16.hpp" "$(@D)/cuda/include/cuda_fp16.hpp" && cp "/usr/local/cuda-9.0/include/cuda_gl_interop.h" "$(@D)/cuda/include/cuda_gl_interop.h" && cp "/usr/local/cuda-9.0/include/cuda_occupancy.h" "$(@D)/cuda/include/cuda_occupancy.h" && cp "/usr/local/cuda-9.0/include/cuda_profiler_api.h" "$(@D)/cuda/include/cuda_profiler_api.h" && cp "/usr/local/cuda-9.0/include/cuda_runtime.h" "$(@D)/cuda/include/cuda_runtime.h" && cp "/usr/local/cuda-9.0/include/cuda_runtime_api.h" "$(@D)/cuda/include/cuda_runtime_api.h" && cp "/usr/local/cuda-9.0/include/cuda_surface_types.h" "$(@D)/cuda/include/cuda_surface_types.h" && cp "/usr/local/cuda-9.0/include/cuda_texture_types.h" "$(@D)/cuda/include/cuda_texture_types.h" && cp "/usr/local/cuda-9.0/include/cuda_vdpau_interop.h" "$(@D)/cuda/include/cuda_vdpau_interop.h" && cp "/usr/local/cuda-9.0/include/cudalibxt.h" "$(@D)/cuda/include/cudalibxt.h" && cp "/usr/local/cuda-9.0/include/cudnn.h" "$(@D)/cuda/include/cudnn.h" && cp "/usr/local/cuda-9.0/include/cufft.h" "$(@D)/cuda/include/cufft.h" && cp "/usr/local/cuda-9.0/include/cufftXt.h" "$(@D)/cuda/include/cufftXt.h" && cp "/usr/local/cuda-9.0/include/cufftw.h" "$(@D)/cuda/include/cufftw.h" && cp "/usr/local/cuda-9.0/include/curand.h" "$(@D)/cuda/include/curand.h" && cp "/usr/local/cuda-9.0/include/curand_discrete.h" "$(@D)/cuda/include/curand_discrete.h" && cp "/usr/local/cuda-9.0/include/curand_discrete2.h" "$(@D)/cuda/include/curand_discrete2.h" && cp "/usr/local/cuda-9.0/include/curand_globals.h" "$(@D)/cuda/include/curand_globals.h" && cp "/usr/local/cuda-9.0/include/curand_kernel.h" "$(@D)/cuda/include/curand_kernel.h" && cp "/usr/local/cuda-9.0/include/curand_lognormal.h" "$(@D)/cuda/include/curand_lognormal.h" && cp "/usr/local/cuda-9.0/include/curand_mrg32k3a.h" "$(@D)/cuda/include/curand_mrg32k3a.h" && cp "/usr/local/cuda-9.0/include/curand_mtgp32.h" "$(@D)/cuda/include/curand_mtgp32.h" && cp "/usr/local/cuda-9.0/include/curand_mtgp32_host.h" "$(@D)/cuda/include/curand_mtgp32_host.h" && cp "/usr/local/cuda-9.0/include/curand_mtgp32_kernel.h" "$(@D)/cuda/include/curand_mtgp32_kernel.h" && cp "/usr/local/cuda-9.0/include/curand_mtgp32dc_p_11213.h" "$(@D)/cuda/include/curand_mtgp32dc_p_11213.h" && cp "/usr/local/cuda-9.0/include/curand_normal.h" "$(@D)/cuda/include/curand_normal.h" && cp "/usr/local/cuda-9.0/include/curand_normal_static.h" "$(@D)/cuda/include/curand_normal_static.h" && cp "/usr/local/cuda-9.0/include/curand_philox4x32_x.h" "$(@D)/cuda/include/curand_philox4x32_x.h" && cp "/usr/local/cuda-9.0/include/curand_poisson.h" "$(@D)/cuda/include/curand_poisson.h" && cp "/usr/local/cuda-9.0/include/curand_precalc.h" "$(@D)/cuda/include/curand_precalc.h" && cp "/usr/local/cuda-9.0/include/curand_uniform.h" "$(@D)/cuda/include/curand_uniform.h" && cp "/usr/local/cuda-9.0/include/cusolverDn.h" "$(@D)/cuda/include/cusolverDn.h" && cp "/usr/local/cuda-9.0/include/cusolverRf.h" "$(@D)/cuda/include/cusolverRf.h" && cp "/usr/local/cuda-9.0/include/cusolverSp.h" "$(@D)/cuda/include/cusolverSp.h" && cp "/usr/local/cuda-9.0/include/cusolverSp_LOWLEVEL_PREVIEW.h" "$(@D)/cuda/include/cusolverSp_LOWLEVEL_PREVIEW.h" && cp "/usr/local/cuda-9.0/include/cusolver_common.h" "$(@D)/cuda/include/cusolver_common.h" && cp "/usr/local/cuda-9.0/include/cusparse.h" "$(@D)/cuda/include/cusparse.h" && cp "/usr/local/cuda-9.0/include/cusparse_v2.h" "$(@D)/cuda/include/cusparse_v2.h" && cp "/usr/local/cuda-9.0/include/device_atomic_functions.h" "$(@D)/cuda/include/device_atomic_functions.h" && cp "/usr/local/cuda-9.0/include/device_atomic_functions.hpp" "$(@D)/cuda/include/device_atomic_functions.hpp" && cp "/usr/local/cuda-9.0/include/device_double_functions.h" "$(@D)/cuda/include/device_double_functions.h" && cp "/usr/local/cuda-9.0/include/device_double_functions.hpp" "$(@D)/cuda/include/device_double_functions.hpp" && cp "/usr/local/cuda-9.0/include/device_functions.h" "$(@D)/cuda/include/device_functions.h" && cp "/usr/local/cuda-9.0/include/device_functions.hpp" "$(@D)/cuda/include/device_functions.hpp" && cp "/usr/local/cuda-9.0/include/device_functions_decls.h" "$(@D)/cuda/include/device_functions_decls.h" && cp "/usr/local/cuda-9.0/include/device_launch_parameters.h" "$(@D)/cuda/include/device_launch_parameters.h" && cp "/usr/local/cuda-9.0/include/device_types.h" "$(@D)/cuda/include/device_types.h" && cp "/usr/local/cuda-9.0/include/driver_functions.h" "$(@D)/cuda/include/driver_functions.h" && cp "/usr/local/cuda-9.0/include/driver_types.h" "$(@D)/cuda/include/driver_types.h" && cp "/usr/local/cuda-9.0/include/dynlink_cuda.h" "$(@D)/cuda/include/dynlink_cuda.h" && cp "/usr/local/cuda-9.0/include/dynlink_cuda_cuda.h" "$(@D)/cuda/include/dynlink_cuda_cuda.h" && cp "/usr/local/cuda-9.0/include/dynlink_cuviddec.h" "$(@D)/cuda/include/dynlink_cuviddec.h" && cp "/usr/local/cuda-9.0/include/dynlink_nvcuvid.h" "$(@D)/cuda/include/dynlink_nvcuvid.h" && cp "/usr/local/cuda-9.0/include/fatBinaryCtl.h" "$(@D)/cuda/include/fatBinaryCtl.h" && cp "/usr/local/cuda-9.0/include/fatbinary.h" "$(@D)/cuda/include/fatbinary.h" && cp "/usr/local/cuda-9.0/include/host_config.h" "$(@D)/cuda/include/host_config.h" && cp "/usr/local/cuda-9.0/include/host_defines.h" "$(@D)/cuda/include/host_defines.h" && cp "/usr/local/cuda-9.0/include/library_types.h" "$(@D)/cuda/include/library_types.h" && cp "/usr/local/cuda-9.0/include/math_constants.h" "$(@D)/cuda/include/math_constants.h" && cp "/usr/local/cuda-9.0/include/math_functions.h" "$(@D)/cuda/include/math_functions.h" && cp "/usr/local/cuda-9.0/include/math_functions.hpp" "$(@D)/cuda/include/math_functions.hpp" && cp "/usr/local/cuda-9.0/include/math_functions_dbl_ptx3.h" "$(@D)/cuda/include/math_functions_dbl_ptx3.h" && cp "/usr/local/cuda-9.0/include/math_functions_dbl_ptx3.hpp" "$(@D)/cuda/include/math_functions_dbl_ptx3.hpp" && cp "/usr/local/cuda-9.0/include/mma.h" "$(@D)/cuda/include/mma.h" && cp "/usr/local/cuda-9.0/include/npp.h" "$(@D)/cuda/include/npp.h" && cp "/usr/local/cuda-9.0/include/nppcore.h" "$(@D)/cuda/include/nppcore.h" && cp "/usr/local/cuda-9.0/include/nppdefs.h" "$(@D)/cuda/include/nppdefs.h" && cp "/usr/local/cuda-9.0/include/nppi.h" "$(@D)/cuda/include/nppi.h" && cp "/usr/local/cuda-9.0/include/nppi_arithmetic_and_logical_operations.h" "$(@D)/cuda/include/nppi_arithmetic_and_logical_operations.h" && cp "/usr/local/cuda-9.0/include/nppi_color_conversion.h" "$(@D)/cuda/include/nppi_color_conversion.h" && cp "/usr/local/cuda-9.0/include/nppi_compression_functions.h" "$(@D)/cuda/include/nppi_compression_functions.h" && cp "/usr/local/cuda-9.0/include/nppi_computer_vision.h" "$(@D)/cuda/include/nppi_computer_vision.h" && cp "/usr/local/cuda-9.0/include/nppi_data_exchange_and_initialization.h" "$(@D)/cuda/include/nppi_data_exchange_and_initialization.h" && cp "/usr/local/cuda-9.0/include/nppi_filtering_functions.h" "$(@D)/cuda/include/nppi_filtering_functions.h" && cp "/usr/local/cuda-9.0/include/nppi_geometry_transforms.h" "$(@D)/cuda/include/nppi_geometry_transforms.h" && cp "/usr/local/cuda-9.0/include/nppi_linear_transforms.h" "$(@D)/cuda/include/nppi_linear_transforms.h" && cp "/usr/local/cuda-9.0/include/nppi_morphological_operations.h" "$(@D)/cuda/include/nppi_morphological_operations.h" && cp "/usr/local/cuda-9.0/include/nppi_statistics_functions.h" "$(@D)/cuda/include/nppi_statistics_functions.h" && cp "/usr/local/cuda-9.0/include/nppi_support_functions.h" "$(@D)/cuda/include/nppi_support_functions.h" && cp "/usr/local/cuda-9.0/include/nppi_threshold_and_compare_operations.h" "$(@D)/cuda/include/nppi_threshold_and_compare_operations.h" && cp "/usr/local/cuda-9.0/include/npps.h" "$(@D)/cuda/include/npps.h" && cp "/usr/local/cuda-9.0/include/npps_arithmetic_and_logical_operations.h" "$(@D)/cuda/include/npps_arithmetic_and_logical_operations.h" && cp "/usr/local/cuda-9.0/include/npps_conversion_functions.h" "$(@D)/cuda/include/npps_conversion_functions.h" && cp "/usr/local/cuda-9.0/include/npps_filtering_functions.h" "$(@D)/cuda/include/npps_filtering_functions.h" && cp "/usr/local/cuda-9.0/include/npps_initialization.h" "$(@D)/cuda/include/npps_initialization.h" && cp "/usr/local/cuda-9.0/include/npps_statistics_functions.h" "$(@D)/cuda/include/npps_statistics_functions.h" && cp "/usr/local/cuda-9.0/include/npps_support_functions.h" "$(@D)/cuda/include/npps_support_functions.h" && cp "/usr/local/cuda-9.0/include/nppversion.h" "$(@D)/cuda/include/nppversion.h" && cp "/usr/local/cuda-9.0/include/nvToolsExt.h" "$(@D)/cuda/include/nvToolsExt.h" && cp "/usr/local/cuda-9.0/include/nvToolsExtCuda.h" "$(@D)/cuda/include/nvToolsExtCuda.h" && cp "/usr/local/cuda-9.0/include/nvToolsExtCudaRt.h" "$(@D)/cuda/include/nvToolsExtCudaRt.h" && cp "/usr/local/cuda-9.0/include/nvToolsExtMeta.h" "$(@D)/cuda/include/nvToolsExtMeta.h" && cp "/usr/local/cuda-9.0/include/nvToolsExtSync.h" "$(@D)/cuda/include/nvToolsExtSync.h" && cp "/usr/local/cuda-9.0/include/nvblas.h" "$(@D)/cuda/include/nvblas.h" && cp "/usr/local/cuda-9.0/include/nvfunctional" "$(@D)/cuda/include/nvfunctional" && cp "/usr/local/cuda-9.0/include/nvgraph.h" "$(@D)/cuda/include/nvgraph.h" && cp "/usr/local/cuda-9.0/include/nvml.h" "$(@D)/cuda/include/nvml.h" && cp "/usr/local/cuda-9.0/include/nvrtc.h" "$(@D)/cuda/include/nvrtc.h" && cp "/usr/local/cuda-9.0/include/sm_20_atomic_functions.h" "$(@D)/cuda/include/sm_20_atomic_functions.h" && cp "/usr/local/cuda-9.0/include/sm_20_atomic_functions.hpp" "$(@D)/cuda/include/sm_20_atomic_functions.hpp" && cp "/usr/local/cuda-9.0/include/sm_20_intrinsics.h" "$(@D)/cuda/include/sm_20_intrinsics.h" && cp "/usr/local/cuda-9.0/include/sm_20_intrinsics.hpp" "$(@D)/cuda/include/sm_20_intrinsics.hpp" && cp "/usr/local/cuda-9.0/include/sm_30_intrinsics.h" "$(@D)/cuda/include/sm_30_intrinsics.h" && cp "/usr/local/cuda-9.0/include/sm_30_intrinsics.hpp" "$(@D)/cuda/include/sm_30_intrinsics.hpp" && cp "/usr/local/cuda-9.0/include/sm_32_atomic_functions.h" "$(@D)/cuda/include/sm_32_atomic_functions.h" && cp "/usr/local/cuda-9.0/include/sm_32_atomic_functions.hpp" "$(@D)/cuda/include/sm_32_atomic_functions.hpp" && cp "/usr/local/cuda-9.0/include/sm_32_intrinsics.h" "$(@D)/cuda/include/sm_32_intrinsics.h" && cp "/usr/local/cuda-9.0/include/sm_32_intrinsics.hpp" "$(@D)/cuda/include/sm_32_intrinsics.hpp" && cp "/usr/local/cuda-9.0/include/sm_35_atomic_functions.h" "$(@D)/cuda/include/sm_35_atomic_functions.h" && cp "/usr/local/cuda-9.0/include/sm_35_intrinsics.h" "$(@D)/cuda/include/sm_35_intrinsics.h" && cp "/usr/local/cuda-9.0/include/sm_60_atomic_functions.h" "$(@D)/cuda/include/sm_60_atomic_functions.h" && cp "/usr/local/cuda-9.0/include/sm_60_atomic_functions.hpp" "$(@D)/cuda/include/sm_60_atomic_functions.hpp" && cp "/usr/local/cuda-9.0/include/sm_61_intrinsics.h" "$(@D)/cuda/include/sm_61_intrinsics.h" && cp "/usr/local/cuda-9.0/include/sm_61_intrinsics.hpp" "$(@D)/cuda/include/sm_61_intrinsics.hpp" && cp "/usr/local/cuda-9.0/include/sobol_direction_vectors.h" "$(@D)/cuda/include/sobol_direction_vectors.h" && cp "/usr/local/cuda-9.0/include/surface_functions.h" "$(@D)/cuda/include/surface_functions.h" && cp "/usr/local/cuda-9.0/include/surface_functions.hpp" "$(@D)/cuda/include/surface_functions.hpp" && cp "/usr/local/cuda-9.0/include/surface_indirect_functions.h" "$(@D)/cuda/include/surface_indirect_functions.h" && cp "/usr/local/cuda-9.0/include/surface_indirect_functions.hpp" "$(@D)/cuda/include/surface_indirect_functions.hpp" && cp "/usr/local/cuda-9.0/include/surface_types.h" "$(@D)/cuda/include/surface_types.h" && cp "/usr/local/cuda-9.0/include/texture_fetch_functions.h" "$(@D)/cuda/include/texture_fetch_functions.h" && cp "/usr/local/cuda-9.0/include/texture_fetch_functions.hpp" "$(@D)/cuda/include/texture_fetch_functions.hpp" && cp "/usr/local/cuda-9.0/include/texture_indirect_functions.h" "$(@D)/cuda/include/texture_indirect_functions.h" && cp "/usr/local/cuda-9.0/include/texture_indirect_functions.hpp" "$(@D)/cuda/include/texture_indirect_functions.hpp" && cp "/usr/local/cuda-9.0/include/texture_types.h" "$(@D)/cuda/include/texture_types.h" && cp "/usr/local/cuda-9.0/include/thrust/adjacent_difference.h" "$(@D)/cuda/include/thrust/adjacent_difference.h" && cp "/usr/local/cuda-9.0/include/thrust/advance.h" "$(@D)/cuda/include/thrust/advance.h" && cp "/usr/local/cuda-9.0/include/thrust/binary_search.h" "$(@D)/cuda/include/thrust/binary_search.h" && cp "/usr/local/cuda-9.0/include/thrust/complex.h" "$(@D)/cuda/include/thrust/complex.h" && cp "/usr/local/cuda-9.0/include/thrust/copy.h" "$(@D)/cuda/include/thrust/copy.h" && cp "/usr/local/cuda-9.0/include/thrust/count.h" "$(@D)/cuda/include/thrust/count.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/adjacent_difference.inl" "$(@D)/cuda/include/thrust/detail/adjacent_difference.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/advance.inl" "$(@D)/cuda/include/thrust/detail/advance.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/allocator/allocator_traits.h" "$(@D)/cuda/include/thrust/detail/allocator/allocator_traits.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/allocator/allocator_traits.inl" "$(@D)/cuda/include/thrust/detail/allocator/allocator_traits.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/allocator/copy_construct_range.h" "$(@D)/cuda/include/thrust/detail/allocator/copy_construct_range.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/allocator/copy_construct_range.inl" "$(@D)/cuda/include/thrust/detail/allocator/copy_construct_range.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/allocator/default_construct_range.h" "$(@D)/cuda/include/thrust/detail/allocator/default_construct_range.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/allocator/default_construct_range.inl" "$(@D)/cuda/include/thrust/detail/allocator/default_construct_range.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/allocator/destroy_range.h" "$(@D)/cuda/include/thrust/detail/allocator/destroy_range.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/allocator/destroy_range.inl" "$(@D)/cuda/include/thrust/detail/allocator/destroy_range.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/allocator/fill_construct_range.h" "$(@D)/cuda/include/thrust/detail/allocator/fill_construct_range.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/allocator/fill_construct_range.inl" "$(@D)/cuda/include/thrust/detail/allocator/fill_construct_range.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/allocator/malloc_allocator.h" "$(@D)/cuda/include/thrust/detail/allocator/malloc_allocator.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/allocator/malloc_allocator.inl" "$(@D)/cuda/include/thrust/detail/allocator/malloc_allocator.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/allocator/no_throw_allocator.h" "$(@D)/cuda/include/thrust/detail/allocator/no_throw_allocator.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/allocator/tagged_allocator.h" "$(@D)/cuda/include/thrust/detail/allocator/tagged_allocator.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/allocator/tagged_allocator.inl" "$(@D)/cuda/include/thrust/detail/allocator/tagged_allocator.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/allocator/temporary_allocator.h" "$(@D)/cuda/include/thrust/detail/allocator/temporary_allocator.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/allocator/temporary_allocator.inl" "$(@D)/cuda/include/thrust/detail/allocator/temporary_allocator.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/binary_search.inl" "$(@D)/cuda/include/thrust/detail/binary_search.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/complex/arithmetic.h" "$(@D)/cuda/include/thrust/detail/complex/arithmetic.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/complex/c99math.h" "$(@D)/cuda/include/thrust/detail/complex/c99math.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/complex/catrig.h" "$(@D)/cuda/include/thrust/detail/complex/catrig.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/complex/catrigf.h" "$(@D)/cuda/include/thrust/detail/complex/catrigf.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/complex/ccosh.h" "$(@D)/cuda/include/thrust/detail/complex/ccosh.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/complex/ccoshf.h" "$(@D)/cuda/include/thrust/detail/complex/ccoshf.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/complex/cexp.h" "$(@D)/cuda/include/thrust/detail/complex/cexp.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/complex/cexpf.h" "$(@D)/cuda/include/thrust/detail/complex/cexpf.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/complex/clog.h" "$(@D)/cuda/include/thrust/detail/complex/clog.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/complex/clogf.h" "$(@D)/cuda/include/thrust/detail/complex/clogf.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/complex/complex.inl" "$(@D)/cuda/include/thrust/detail/complex/complex.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/complex/cpow.h" "$(@D)/cuda/include/thrust/detail/complex/cpow.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/complex/cpowf.h" "$(@D)/cuda/include/thrust/detail/complex/cpowf.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/complex/cproj.h" "$(@D)/cuda/include/thrust/detail/complex/cproj.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/complex/csinh.h" "$(@D)/cuda/include/thrust/detail/complex/csinh.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/complex/csinhf.h" "$(@D)/cuda/include/thrust/detail/complex/csinhf.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/complex/csqrt.h" "$(@D)/cuda/include/thrust/detail/complex/csqrt.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/complex/csqrtf.h" "$(@D)/cuda/include/thrust/detail/complex/csqrtf.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/complex/ctanh.h" "$(@D)/cuda/include/thrust/detail/complex/ctanh.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/complex/ctanhf.h" "$(@D)/cuda/include/thrust/detail/complex/ctanhf.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/complex/math_private.h" "$(@D)/cuda/include/thrust/detail/complex/math_private.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/complex/stream.h" "$(@D)/cuda/include/thrust/detail/complex/stream.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/config.h" "$(@D)/cuda/include/thrust/detail/config.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/config/compiler.h" "$(@D)/cuda/include/thrust/detail/config/compiler.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/config/compiler_fence.h" "$(@D)/cuda/include/thrust/detail/config/compiler_fence.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/config/config.h" "$(@D)/cuda/include/thrust/detail/config/config.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/config/debug.h" "$(@D)/cuda/include/thrust/detail/config/debug.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/config/device_system.h" "$(@D)/cuda/include/thrust/detail/config/device_system.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/config/exec_check_disable.h" "$(@D)/cuda/include/thrust/detail/config/exec_check_disable.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/config/forceinline.h" "$(@D)/cuda/include/thrust/detail/config/forceinline.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/config/global_workarounds.h" "$(@D)/cuda/include/thrust/detail/config/global_workarounds.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/config/host_device.h" "$(@D)/cuda/include/thrust/detail/config/host_device.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/config/host_system.h" "$(@D)/cuda/include/thrust/detail/config/host_system.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/config/simple_defines.h" "$(@D)/cuda/include/thrust/detail/config/simple_defines.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/contiguous_storage.h" "$(@D)/cuda/include/thrust/detail/contiguous_storage.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/contiguous_storage.inl" "$(@D)/cuda/include/thrust/detail/contiguous_storage.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/copy.h" "$(@D)/cuda/include/thrust/detail/copy.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/copy.inl" "$(@D)/cuda/include/thrust/detail/copy.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/copy_if.h" "$(@D)/cuda/include/thrust/detail/copy_if.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/copy_if.inl" "$(@D)/cuda/include/thrust/detail/copy_if.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/count.inl" "$(@D)/cuda/include/thrust/detail/count.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/cstdint.h" "$(@D)/cuda/include/thrust/detail/cstdint.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/device_delete.inl" "$(@D)/cuda/include/thrust/detail/device_delete.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/device_free.inl" "$(@D)/cuda/include/thrust/detail/device_free.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/device_malloc.inl" "$(@D)/cuda/include/thrust/detail/device_malloc.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/device_new.inl" "$(@D)/cuda/include/thrust/detail/device_new.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/device_ptr.inl" "$(@D)/cuda/include/thrust/detail/device_ptr.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/device_reference.inl" "$(@D)/cuda/include/thrust/detail/device_reference.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/device_vector.inl" "$(@D)/cuda/include/thrust/detail/device_vector.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/dispatch/is_trivial_copy.h" "$(@D)/cuda/include/thrust/detail/dispatch/is_trivial_copy.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/distance.inl" "$(@D)/cuda/include/thrust/detail/distance.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/equal.inl" "$(@D)/cuda/include/thrust/detail/equal.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/execute_with_allocator.h" "$(@D)/cuda/include/thrust/detail/execute_with_allocator.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/execution_policy.h" "$(@D)/cuda/include/thrust/detail/execution_policy.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/extrema.inl" "$(@D)/cuda/include/thrust/detail/extrema.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/fill.inl" "$(@D)/cuda/include/thrust/detail/fill.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/find.inl" "$(@D)/cuda/include/thrust/detail/find.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/for_each.inl" "$(@D)/cuda/include/thrust/detail/for_each.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/function.h" "$(@D)/cuda/include/thrust/detail/function.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/functional.inl" "$(@D)/cuda/include/thrust/detail/functional.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/functional/actor.h" "$(@D)/cuda/include/thrust/detail/functional/actor.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/functional/actor.inl" "$(@D)/cuda/include/thrust/detail/functional/actor.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/functional/argument.h" "$(@D)/cuda/include/thrust/detail/functional/argument.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/functional/composite.h" "$(@D)/cuda/include/thrust/detail/functional/composite.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/functional/operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/functional/operators/arithmetic_operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators/arithmetic_operators.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/functional/operators/assignment_operator.h" "$(@D)/cuda/include/thrust/detail/functional/operators/assignment_operator.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/functional/operators/bitwise_operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators/bitwise_operators.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/functional/operators/compound_assignment_operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators/compound_assignment_operators.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/functional/operators/logical_operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators/logical_operators.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/functional/operators/operator_adaptors.h" "$(@D)/cuda/include/thrust/detail/functional/operators/operator_adaptors.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/functional/operators/relational_operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators/relational_operators.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/functional/placeholder.h" "$(@D)/cuda/include/thrust/detail/functional/placeholder.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/functional/value.h" "$(@D)/cuda/include/thrust/detail/functional/value.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/gather.inl" "$(@D)/cuda/include/thrust/detail/gather.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/generate.inl" "$(@D)/cuda/include/thrust/detail/generate.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/get_iterator_value.h" "$(@D)/cuda/include/thrust/detail/get_iterator_value.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/host_vector.inl" "$(@D)/cuda/include/thrust/detail/host_vector.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/inner_product.inl" "$(@D)/cuda/include/thrust/detail/inner_product.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/integer_math.h" "$(@D)/cuda/include/thrust/detail/integer_math.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/integer_traits.h" "$(@D)/cuda/include/thrust/detail/integer_traits.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/internal_functional.h" "$(@D)/cuda/include/thrust/detail/internal_functional.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/logical.inl" "$(@D)/cuda/include/thrust/detail/logical.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/malloc_and_free.h" "$(@D)/cuda/include/thrust/detail/malloc_and_free.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/merge.inl" "$(@D)/cuda/include/thrust/detail/merge.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/minmax.h" "$(@D)/cuda/include/thrust/detail/minmax.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/mismatch.inl" "$(@D)/cuda/include/thrust/detail/mismatch.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/mpl/math.h" "$(@D)/cuda/include/thrust/detail/mpl/math.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/numeric_traits.h" "$(@D)/cuda/include/thrust/detail/numeric_traits.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/overlapped_copy.h" "$(@D)/cuda/include/thrust/detail/overlapped_copy.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/pair.inl" "$(@D)/cuda/include/thrust/detail/pair.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/partition.inl" "$(@D)/cuda/include/thrust/detail/partition.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/pointer.h" "$(@D)/cuda/include/thrust/detail/pointer.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/pointer.inl" "$(@D)/cuda/include/thrust/detail/pointer.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/range/head_flags.h" "$(@D)/cuda/include/thrust/detail/range/head_flags.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/range/tail_flags.h" "$(@D)/cuda/include/thrust/detail/range/tail_flags.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/raw_pointer_cast.h" "$(@D)/cuda/include/thrust/detail/raw_pointer_cast.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/raw_reference_cast.h" "$(@D)/cuda/include/thrust/detail/raw_reference_cast.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/reduce.inl" "$(@D)/cuda/include/thrust/detail/reduce.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/reference.h" "$(@D)/cuda/include/thrust/detail/reference.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/reference.inl" "$(@D)/cuda/include/thrust/detail/reference.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/reference_forward_declaration.h" "$(@D)/cuda/include/thrust/detail/reference_forward_declaration.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/remove.inl" "$(@D)/cuda/include/thrust/detail/remove.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/replace.inl" "$(@D)/cuda/include/thrust/detail/replace.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/reverse.inl" "$(@D)/cuda/include/thrust/detail/reverse.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/scan.inl" "$(@D)/cuda/include/thrust/detail/scan.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/scatter.inl" "$(@D)/cuda/include/thrust/detail/scatter.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/seq.h" "$(@D)/cuda/include/thrust/detail/seq.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/sequence.inl" "$(@D)/cuda/include/thrust/detail/sequence.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/set_operations.inl" "$(@D)/cuda/include/thrust/detail/set_operations.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/sort.inl" "$(@D)/cuda/include/thrust/detail/sort.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/static_assert.h" "$(@D)/cuda/include/thrust/detail/static_assert.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/static_map.h" "$(@D)/cuda/include/thrust/detail/static_map.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/swap.h" "$(@D)/cuda/include/thrust/detail/swap.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/swap.inl" "$(@D)/cuda/include/thrust/detail/swap.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/swap_ranges.inl" "$(@D)/cuda/include/thrust/detail/swap_ranges.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/tabulate.inl" "$(@D)/cuda/include/thrust/detail/tabulate.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/temporary_array.h" "$(@D)/cuda/include/thrust/detail/temporary_array.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/temporary_array.inl" "$(@D)/cuda/include/thrust/detail/temporary_array.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/temporary_buffer.h" "$(@D)/cuda/include/thrust/detail/temporary_buffer.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/transform.inl" "$(@D)/cuda/include/thrust/detail/transform.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/transform_reduce.inl" "$(@D)/cuda/include/thrust/detail/transform_reduce.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/transform_scan.inl" "$(@D)/cuda/include/thrust/detail/transform_scan.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/trivial_sequence.h" "$(@D)/cuda/include/thrust/detail/trivial_sequence.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/tuple.inl" "$(@D)/cuda/include/thrust/detail/tuple.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/tuple_meta_transform.h" "$(@D)/cuda/include/thrust/detail/tuple_meta_transform.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/tuple_transform.h" "$(@D)/cuda/include/thrust/detail/tuple_transform.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/type_traits.h" "$(@D)/cuda/include/thrust/detail/type_traits.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/type_traits/algorithm/intermediate_type_from_function_and_iterators.h" "$(@D)/cuda/include/thrust/detail/type_traits/algorithm/intermediate_type_from_function_and_iterators.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/type_traits/function_traits.h" "$(@D)/cuda/include/thrust/detail/type_traits/function_traits.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/type_traits/has_member_function.h" "$(@D)/cuda/include/thrust/detail/type_traits/has_member_function.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/type_traits/has_nested_type.h" "$(@D)/cuda/include/thrust/detail/type_traits/has_nested_type.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/type_traits/has_trivial_assign.h" "$(@D)/cuda/include/thrust/detail/type_traits/has_trivial_assign.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/type_traits/is_call_possible.h" "$(@D)/cuda/include/thrust/detail/type_traits/is_call_possible.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/type_traits/is_metafunction_defined.h" "$(@D)/cuda/include/thrust/detail/type_traits/is_metafunction_defined.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/type_traits/iterator/is_discard_iterator.h" "$(@D)/cuda/include/thrust/detail/type_traits/iterator/is_discard_iterator.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/type_traits/iterator/is_output_iterator.h" "$(@D)/cuda/include/thrust/detail/type_traits/iterator/is_output_iterator.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/type_traits/minimum_type.h" "$(@D)/cuda/include/thrust/detail/type_traits/minimum_type.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/type_traits/pointer_traits.h" "$(@D)/cuda/include/thrust/detail/type_traits/pointer_traits.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/type_traits/result_of_adaptable_function.h" "$(@D)/cuda/include/thrust/detail/type_traits/result_of_adaptable_function.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/uninitialized_copy.inl" "$(@D)/cuda/include/thrust/detail/uninitialized_copy.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/uninitialized_fill.inl" "$(@D)/cuda/include/thrust/detail/uninitialized_fill.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/unique.inl" "$(@D)/cuda/include/thrust/detail/unique.inl" && cp "/usr/local/cuda-9.0/include/thrust/detail/use_default.h" "$(@D)/cuda/include/thrust/detail/use_default.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/util/align.h" "$(@D)/cuda/include/thrust/detail/util/align.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/util/blocking.h" "$(@D)/cuda/include/thrust/detail/util/blocking.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/vector_base.h" "$(@D)/cuda/include/thrust/detail/vector_base.h" && cp "/usr/local/cuda-9.0/include/thrust/detail/vector_base.inl" "$(@D)/cuda/include/thrust/detail/vector_base.inl" && cp "/usr/local/cuda-9.0/include/thrust/device_allocator.h" "$(@D)/cuda/include/thrust/device_allocator.h" && cp "/usr/local/cuda-9.0/include/thrust/device_delete.h" "$(@D)/cuda/include/thrust/device_delete.h" && cp "/usr/local/cuda-9.0/include/thrust/device_free.h" "$(@D)/cuda/include/thrust/device_free.h" && cp "/usr/local/cuda-9.0/include/thrust/device_malloc.h" "$(@D)/cuda/include/thrust/device_malloc.h" && cp "/usr/local/cuda-9.0/include/thrust/device_malloc_allocator.h" "$(@D)/cuda/include/thrust/device_malloc_allocator.h" && cp "/usr/local/cuda-9.0/include/thrust/device_new.h" "$(@D)/cuda/include/thrust/device_new.h" && cp "/usr/local/cuda-9.0/include/thrust/device_new_allocator.h" "$(@D)/cuda/include/thrust/device_new_allocator.h" && cp "/usr/local/cuda-9.0/include/thrust/device_ptr.h" "$(@D)/cuda/include/thrust/device_ptr.h" && cp "/usr/local/cuda-9.0/include/thrust/device_reference.h" "$(@D)/cuda/include/thrust/device_reference.h" && cp "/usr/local/cuda-9.0/include/thrust/device_vector.h" "$(@D)/cuda/include/thrust/device_vector.h" && cp "/usr/local/cuda-9.0/include/thrust/distance.h" "$(@D)/cuda/include/thrust/distance.h" && cp "/usr/local/cuda-9.0/include/thrust/equal.h" "$(@D)/cuda/include/thrust/equal.h" && cp "/usr/local/cuda-9.0/include/thrust/execution_policy.h" "$(@D)/cuda/include/thrust/execution_policy.h" && cp "/usr/local/cuda-9.0/include/thrust/extrema.h" "$(@D)/cuda/include/thrust/extrema.h" && cp "/usr/local/cuda-9.0/include/thrust/fill.h" "$(@D)/cuda/include/thrust/fill.h" && cp "/usr/local/cuda-9.0/include/thrust/find.h" "$(@D)/cuda/include/thrust/find.h" && cp "/usr/local/cuda-9.0/include/thrust/for_each.h" "$(@D)/cuda/include/thrust/for_each.h" && cp "/usr/local/cuda-9.0/include/thrust/functional.h" "$(@D)/cuda/include/thrust/functional.h" && cp "/usr/local/cuda-9.0/include/thrust/gather.h" "$(@D)/cuda/include/thrust/gather.h" && cp "/usr/local/cuda-9.0/include/thrust/generate.h" "$(@D)/cuda/include/thrust/generate.h" && cp "/usr/local/cuda-9.0/include/thrust/host_vector.h" "$(@D)/cuda/include/thrust/host_vector.h" && cp "/usr/local/cuda-9.0/include/thrust/inner_product.h" "$(@D)/cuda/include/thrust/inner_product.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/constant_iterator.h" "$(@D)/cuda/include/thrust/iterator/constant_iterator.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/counting_iterator.h" "$(@D)/cuda/include/thrust/iterator/counting_iterator.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/any_assign.h" "$(@D)/cuda/include/thrust/iterator/detail/any_assign.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/any_system_tag.h" "$(@D)/cuda/include/thrust/iterator/detail/any_system_tag.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/constant_iterator_base.h" "$(@D)/cuda/include/thrust/iterator/detail/constant_iterator_base.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/counting_iterator.inl" "$(@D)/cuda/include/thrust/iterator/detail/counting_iterator.inl" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/device_system_tag.h" "$(@D)/cuda/include/thrust/iterator/detail/device_system_tag.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/discard_iterator_base.h" "$(@D)/cuda/include/thrust/iterator/detail/discard_iterator_base.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/distance_from_result.h" "$(@D)/cuda/include/thrust/iterator/detail/distance_from_result.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/host_system_tag.h" "$(@D)/cuda/include/thrust/iterator/detail/host_system_tag.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/is_iterator_category.h" "$(@D)/cuda/include/thrust/iterator/detail/is_iterator_category.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/is_trivial_iterator.h" "$(@D)/cuda/include/thrust/iterator/detail/is_trivial_iterator.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/iterator_adaptor_base.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_adaptor_base.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/iterator_category_to_system.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_category_to_system.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/iterator_category_to_traversal.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_category_to_traversal.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/iterator_category_with_system_and_traversal.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_category_with_system_and_traversal.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/iterator_facade_category.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_facade_category.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/iterator_traits.inl" "$(@D)/cuda/include/thrust/iterator/detail/iterator_traits.inl" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/iterator_traversal_tags.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_traversal_tags.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/join_iterator.h" "$(@D)/cuda/include/thrust/iterator/detail/join_iterator.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/minimum_category.h" "$(@D)/cuda/include/thrust/iterator/detail/minimum_category.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/minimum_system.h" "$(@D)/cuda/include/thrust/iterator/detail/minimum_system.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/normal_iterator.h" "$(@D)/cuda/include/thrust/iterator/detail/normal_iterator.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/permutation_iterator_base.h" "$(@D)/cuda/include/thrust/iterator/detail/permutation_iterator_base.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/retag.h" "$(@D)/cuda/include/thrust/iterator/detail/retag.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/reverse_iterator.inl" "$(@D)/cuda/include/thrust/iterator/detail/reverse_iterator.inl" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/reverse_iterator_base.h" "$(@D)/cuda/include/thrust/iterator/detail/reverse_iterator_base.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/tagged_iterator.h" "$(@D)/cuda/include/thrust/iterator/detail/tagged_iterator.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/transform_iterator.inl" "$(@D)/cuda/include/thrust/iterator/detail/transform_iterator.inl" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/transform_output_iterator.inl" "$(@D)/cuda/include/thrust/iterator/detail/transform_output_iterator.inl" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/tuple_of_iterator_references.h" "$(@D)/cuda/include/thrust/iterator/detail/tuple_of_iterator_references.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/universal_categories.h" "$(@D)/cuda/include/thrust/iterator/detail/universal_categories.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/zip_iterator.inl" "$(@D)/cuda/include/thrust/iterator/detail/zip_iterator.inl" && cp "/usr/local/cuda-9.0/include/thrust/iterator/detail/zip_iterator_base.h" "$(@D)/cuda/include/thrust/iterator/detail/zip_iterator_base.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/discard_iterator.h" "$(@D)/cuda/include/thrust/iterator/discard_iterator.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/iterator_adaptor.h" "$(@D)/cuda/include/thrust/iterator/iterator_adaptor.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/iterator_categories.h" "$(@D)/cuda/include/thrust/iterator/iterator_categories.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/iterator_facade.h" "$(@D)/cuda/include/thrust/iterator/iterator_facade.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/iterator_traits.h" "$(@D)/cuda/include/thrust/iterator/iterator_traits.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/permutation_iterator.h" "$(@D)/cuda/include/thrust/iterator/permutation_iterator.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/retag.h" "$(@D)/cuda/include/thrust/iterator/retag.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/reverse_iterator.h" "$(@D)/cuda/include/thrust/iterator/reverse_iterator.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/transform_iterator.h" "$(@D)/cuda/include/thrust/iterator/transform_iterator.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/transform_output_iterator.h" "$(@D)/cuda/include/thrust/iterator/transform_output_iterator.h" && cp "/usr/local/cuda-9.0/include/thrust/iterator/zip_iterator.h" "$(@D)/cuda/include/thrust/iterator/zip_iterator.h" && cp "/usr/local/cuda-9.0/include/thrust/logical.h" "$(@D)/cuda/include/thrust/logical.h" && cp "/usr/local/cuda-9.0/include/thrust/memory.h" "$(@D)/cuda/include/thrust/memory.h" && cp "/usr/local/cuda-9.0/include/thrust/merge.h" "$(@D)/cuda/include/thrust/merge.h" && cp "/usr/local/cuda-9.0/include/thrust/mismatch.h" "$(@D)/cuda/include/thrust/mismatch.h" && cp "/usr/local/cuda-9.0/include/thrust/pair.h" "$(@D)/cuda/include/thrust/pair.h" && cp "/usr/local/cuda-9.0/include/thrust/partition.h" "$(@D)/cuda/include/thrust/partition.h" && cp "/usr/local/cuda-9.0/include/thrust/random.h" "$(@D)/cuda/include/thrust/random.h" && cp "/usr/local/cuda-9.0/include/thrust/random/detail/discard_block_engine.inl" "$(@D)/cuda/include/thrust/random/detail/discard_block_engine.inl" && cp "/usr/local/cuda-9.0/include/thrust/random/detail/linear_congruential_engine.inl" "$(@D)/cuda/include/thrust/random/detail/linear_congruential_engine.inl" && cp "/usr/local/cuda-9.0/include/thrust/random/detail/linear_congruential_engine_discard.h" "$(@D)/cuda/include/thrust/random/detail/linear_congruential_engine_discard.h" && cp "/usr/local/cuda-9.0/include/thrust/random/detail/linear_feedback_shift_engine.inl" "$(@D)/cuda/include/thrust/random/detail/linear_feedback_shift_engine.inl" && cp "/usr/local/cuda-9.0/include/thrust/random/detail/linear_feedback_shift_engine_wordmask.h" "$(@D)/cuda/include/thrust/random/detail/linear_feedback_shift_engine_wordmask.h" && cp "/usr/local/cuda-9.0/include/thrust/random/detail/mod.h" "$(@D)/cuda/include/thrust/random/detail/mod.h" && cp "/usr/local/cuda-9.0/include/thrust/random/detail/normal_distribution.inl" "$(@D)/cuda/include/thrust/random/detail/normal_distribution.inl" && cp "/usr/local/cuda-9.0/include/thrust/random/detail/normal_distribution_base.h" "$(@D)/cuda/include/thrust/random/detail/normal_distribution_base.h" && cp "/usr/local/cuda-9.0/include/thrust/random/detail/random_core_access.h" "$(@D)/cuda/include/thrust/random/detail/random_core_access.h" && cp "/usr/local/cuda-9.0/include/thrust/random/detail/subtract_with_carry_engine.inl" "$(@D)/cuda/include/thrust/random/detail/subtract_with_carry_engine.inl" && cp "/usr/local/cuda-9.0/include/thrust/random/detail/uniform_int_distribution.inl" "$(@D)/cuda/include/thrust/random/detail/uniform_int_distribution.inl" && cp "/usr/local/cuda-9.0/include/thrust/random/detail/uniform_real_distribution.inl" "$(@D)/cuda/include/thrust/random/detail/uniform_real_distribution.inl" && cp "/usr/local/cuda-9.0/include/thrust/random/detail/xor_combine_engine.inl" "$(@D)/cuda/include/thrust/random/detail/xor_combine_engine.inl" && cp "/usr/local/cuda-9.0/include/thrust/random/detail/xor_combine_engine_max.h" "$(@D)/cuda/include/thrust/random/detail/xor_combine_engine_max.h" && cp "/usr/local/cuda-9.0/include/thrust/random/discard_block_engine.h" "$(@D)/cuda/include/thrust/random/discard_block_engine.h" && cp "/usr/local/cuda-9.0/include/thrust/random/linear_congruential_engine.h" "$(@D)/cuda/include/thrust/random/linear_congruential_engine.h" && cp "/usr/local/cuda-9.0/include/thrust/random/linear_feedback_shift_engine.h" "$(@D)/cuda/include/thrust/random/linear_feedback_shift_engine.h" && cp "/usr/local/cuda-9.0/include/thrust/random/normal_distribution.h" "$(@D)/cuda/include/thrust/random/normal_distribution.h" && cp "/usr/local/cuda-9.0/include/thrust/random/subtract_with_carry_engine.h" "$(@D)/cuda/include/thrust/random/subtract_with_carry_engine.h" && cp "/usr/local/cuda-9.0/include/thrust/random/uniform_int_distribution.h" "$(@D)/cuda/include/thrust/random/uniform_int_distribution.h" && cp "/usr/local/cuda-9.0/include/thrust/random/uniform_real_distribution.h" "$(@D)/cuda/include/thrust/random/uniform_real_distribution.h" && cp "/usr/local/cuda-9.0/include/thrust/random/xor_combine_engine.h" "$(@D)/cuda/include/thrust/random/xor_combine_engine.h" && cp "/usr/local/cuda-9.0/include/thrust/reduce.h" "$(@D)/cuda/include/thrust/reduce.h" && cp "/usr/local/cuda-9.0/include/thrust/remove.h" "$(@D)/cuda/include/thrust/remove.h" && cp "/usr/local/cuda-9.0/include/thrust/replace.h" "$(@D)/cuda/include/thrust/replace.h" && cp "/usr/local/cuda-9.0/include/thrust/reverse.h" "$(@D)/cuda/include/thrust/reverse.h" && cp "/usr/local/cuda-9.0/include/thrust/scan.h" "$(@D)/cuda/include/thrust/scan.h" && cp "/usr/local/cuda-9.0/include/thrust/scatter.h" "$(@D)/cuda/include/thrust/scatter.h" && cp "/usr/local/cuda-9.0/include/thrust/sequence.h" "$(@D)/cuda/include/thrust/sequence.h" && cp "/usr/local/cuda-9.0/include/thrust/set_operations.h" "$(@D)/cuda/include/thrust/set_operations.h" && cp "/usr/local/cuda-9.0/include/thrust/sort.h" "$(@D)/cuda/include/thrust/sort.h" && cp "/usr/local/cuda-9.0/include/thrust/swap.h" "$(@D)/cuda/include/thrust/swap.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/cpp/detail/adjacent_difference.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/assign_value.h" "$(@D)/cuda/include/thrust/system/cpp/detail/assign_value.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/binary_search.h" "$(@D)/cuda/include/thrust/system/cpp/detail/binary_search.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/copy.h" "$(@D)/cuda/include/thrust/system/cpp/detail/copy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/copy_if.h" "$(@D)/cuda/include/thrust/system/cpp/detail/copy_if.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/count.h" "$(@D)/cuda/include/thrust/system/cpp/detail/count.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/equal.h" "$(@D)/cuda/include/thrust/system/cpp/detail/equal.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/execution_policy.h" "$(@D)/cuda/include/thrust/system/cpp/detail/execution_policy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/extrema.h" "$(@D)/cuda/include/thrust/system/cpp/detail/extrema.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/fill.h" "$(@D)/cuda/include/thrust/system/cpp/detail/fill.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/find.h" "$(@D)/cuda/include/thrust/system/cpp/detail/find.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/for_each.h" "$(@D)/cuda/include/thrust/system/cpp/detail/for_each.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/gather.h" "$(@D)/cuda/include/thrust/system/cpp/detail/gather.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/generate.h" "$(@D)/cuda/include/thrust/system/cpp/detail/generate.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/get_value.h" "$(@D)/cuda/include/thrust/system/cpp/detail/get_value.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/inner_product.h" "$(@D)/cuda/include/thrust/system/cpp/detail/inner_product.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/iter_swap.h" "$(@D)/cuda/include/thrust/system/cpp/detail/iter_swap.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/logical.h" "$(@D)/cuda/include/thrust/system/cpp/detail/logical.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/cpp/detail/malloc_and_free.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/memory.inl" "$(@D)/cuda/include/thrust/system/cpp/detail/memory.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/merge.h" "$(@D)/cuda/include/thrust/system/cpp/detail/merge.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/mismatch.h" "$(@D)/cuda/include/thrust/system/cpp/detail/mismatch.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/par.h" "$(@D)/cuda/include/thrust/system/cpp/detail/par.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/partition.h" "$(@D)/cuda/include/thrust/system/cpp/detail/partition.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/reduce.h" "$(@D)/cuda/include/thrust/system/cpp/detail/reduce.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/cpp/detail/reduce_by_key.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/remove.h" "$(@D)/cuda/include/thrust/system/cpp/detail/remove.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/replace.h" "$(@D)/cuda/include/thrust/system/cpp/detail/replace.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/reverse.h" "$(@D)/cuda/include/thrust/system/cpp/detail/reverse.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/scan.h" "$(@D)/cuda/include/thrust/system/cpp/detail/scan.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/scan_by_key.h" "$(@D)/cuda/include/thrust/system/cpp/detail/scan_by_key.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/scatter.h" "$(@D)/cuda/include/thrust/system/cpp/detail/scatter.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/sequence.h" "$(@D)/cuda/include/thrust/system/cpp/detail/sequence.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/set_operations.h" "$(@D)/cuda/include/thrust/system/cpp/detail/set_operations.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/sort.h" "$(@D)/cuda/include/thrust/system/cpp/detail/sort.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/swap_ranges.h" "$(@D)/cuda/include/thrust/system/cpp/detail/swap_ranges.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/tabulate.h" "$(@D)/cuda/include/thrust/system/cpp/detail/tabulate.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/cpp/detail/temporary_buffer.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/transform.h" "$(@D)/cuda/include/thrust/system/cpp/detail/transform.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/transform_reduce.h" "$(@D)/cuda/include/thrust/system/cpp/detail/transform_reduce.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/transform_scan.h" "$(@D)/cuda/include/thrust/system/cpp/detail/transform_scan.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/cpp/detail/uninitialized_copy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/cpp/detail/uninitialized_fill.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/unique.h" "$(@D)/cuda/include/thrust/system/cpp/detail/unique.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/unique_by_key.h" "$(@D)/cuda/include/thrust/system/cpp/detail/unique_by_key.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/vector.inl" "$(@D)/cuda/include/thrust/system/cpp/detail/vector.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/execution_policy.h" "$(@D)/cuda/include/thrust/system/cpp/execution_policy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/memory.h" "$(@D)/cuda/include/thrust/system/cpp/memory.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cpp/vector.h" "$(@D)/cuda/include/thrust/system/cpp/vector.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/config.h" "$(@D)/cuda/include/thrust/system/cuda/config.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/cuda/detail/adjacent_difference.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/assign_value.h" "$(@D)/cuda/include/thrust/system/cuda/detail/assign_value.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/binary_search.h" "$(@D)/cuda/include/thrust/system/cuda/detail/binary_search.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/copy.h" "$(@D)/cuda/include/thrust/system/cuda/detail/copy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/copy_if.h" "$(@D)/cuda/include/thrust/system/cuda/detail/copy_if.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/core/agent_launcher.h" "$(@D)/cuda/include/thrust/system/cuda/detail/core/agent_launcher.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/core/alignment.h" "$(@D)/cuda/include/thrust/system/cuda/detail/core/alignment.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/core/triple_chevron_launch.h" "$(@D)/cuda/include/thrust/system/cuda/detail/core/triple_chevron_launch.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/core/util.h" "$(@D)/cuda/include/thrust/system/cuda/detail/core/util.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/count.h" "$(@D)/cuda/include/thrust/system/cuda/detail/count.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cross_system.h" "$(@D)/cuda/include/thrust/system/cuda/detail/cross_system.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_histogram.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_histogram.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_radix_sort_downsweep.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_radix_sort_downsweep.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_radix_sort_upsweep.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_radix_sort_upsweep.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_reduce.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_reduce_by_key.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_reduce_by_key.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_rle.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_rle.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_scan.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_segment_fixup.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_segment_fixup.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_select_if.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_select_if.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_spmv_csrt.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_spmv_csrt.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_spmv_orig.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_spmv_orig.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_spmv_row_based.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_spmv_row_based.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/single_pass_scan_operators.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/single_pass_scan_operators.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_adjacent_difference.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_adjacent_difference.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_discontinuity.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_discontinuity.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_exchange.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_exchange.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_histogram.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_histogram.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_load.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_load.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_radix_rank.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_radix_rank.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_radix_sort.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_radix_sort.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_raking_layout.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_raking_layout.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_reduce.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_scan.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_shuffle.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_shuffle.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_store.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_store.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/specializations/block_histogram_atomic.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_histogram_atomic.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/specializations/block_histogram_sort.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_histogram_sort.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_raking.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_raking.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_raking_commutative_only.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_raking_commutative_only.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_warp_reductions.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_warp_reductions.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_raking.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_raking.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans2.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans2.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans3.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans3.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/cub.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/cub.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_histogram.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_histogram.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_partition.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_partition.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_radix_sort.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_radix_sort.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_reduce.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_run_length_encode.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_run_length_encode.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_scan.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_segmented_radix_sort.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_segmented_radix_sort.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_segmented_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_segmented_reduce.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_select.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_select.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_spmv.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_spmv.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_histogram.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_histogram.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_radix_sort.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_radix_sort.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_reduce.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_reduce_by_key.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_reduce_by_key.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_rle.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_rle.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_scan.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_select_if.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_select_if.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_spmv_csrt.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_spmv_csrt.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_spmv_orig.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_spmv_orig.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_spmv_row_based.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_spmv_row_based.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/grid/grid_barrier.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/grid/grid_barrier.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/grid/grid_even_share.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/grid/grid_even_share.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/grid/grid_mapping.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/grid/grid_mapping.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/grid/grid_queue.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/grid/grid_queue.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/host/mutex.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/host/mutex.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/iterator/arg_index_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/arg_index_input_iterator.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/iterator/cache_modified_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/cache_modified_input_iterator.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/iterator/cache_modified_output_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/cache_modified_output_iterator.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/iterator/constant_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/constant_input_iterator.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/iterator/counting_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/counting_input_iterator.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/iterator/discard_output_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/discard_output_iterator.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/iterator/tex_obj_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/tex_obj_input_iterator.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/iterator/tex_ref_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/tex_ref_input_iterator.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/iterator/transform_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/transform_input_iterator.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/thread/thread_load.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_load.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/thread/thread_operators.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_operators.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/thread/thread_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_reduce.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/thread/thread_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_scan.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/thread/thread_search.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_search.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/thread/thread_store.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_store.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/util_allocator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_allocator.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/util_arch.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_arch.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/util_debug.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_debug.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/util_device.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_device.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/util_macro.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_macro.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/util_namespace.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_namespace.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/util_ptx.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_ptx.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/util_type.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_type.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/warp/specializations/warp_reduce_shfl.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/specializations/warp_reduce_shfl.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/warp/specializations/warp_reduce_smem.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/specializations/warp_reduce_smem.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/warp/specializations/warp_scan_shfl.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/specializations/warp_scan_shfl.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/warp/specializations/warp_scan_smem.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/specializations/warp_scan_smem.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/warp/warp_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/warp_reduce.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/warp/warp_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/warp_scan.cuh" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/equal.h" "$(@D)/cuda/include/thrust/system/cuda/detail/equal.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/error.inl" "$(@D)/cuda/include/thrust/system/cuda/detail/error.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/execution_policy.h" "$(@D)/cuda/include/thrust/system/cuda/detail/execution_policy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/extrema.h" "$(@D)/cuda/include/thrust/system/cuda/detail/extrema.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/fill.h" "$(@D)/cuda/include/thrust/system/cuda/detail/fill.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/find.h" "$(@D)/cuda/include/thrust/system/cuda/detail/find.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/for_each.h" "$(@D)/cuda/include/thrust/system/cuda/detail/for_each.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/gather.h" "$(@D)/cuda/include/thrust/system/cuda/detail/gather.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/generate.h" "$(@D)/cuda/include/thrust/system/cuda/detail/generate.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/get_value.h" "$(@D)/cuda/include/thrust/system/cuda/detail/get_value.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/guarded_cuda_runtime_api.h" "$(@D)/cuda/include/thrust/system/cuda/detail/guarded_cuda_runtime_api.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/guarded_driver_types.h" "$(@D)/cuda/include/thrust/system/cuda/detail/guarded_driver_types.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/inner_product.h" "$(@D)/cuda/include/thrust/system/cuda/detail/inner_product.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/internal/copy_cross_system.h" "$(@D)/cuda/include/thrust/system/cuda/detail/internal/copy_cross_system.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/internal/copy_device_to_device.h" "$(@D)/cuda/include/thrust/system/cuda/detail/internal/copy_device_to_device.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/iter_swap.h" "$(@D)/cuda/include/thrust/system/cuda/detail/iter_swap.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/logical.h" "$(@D)/cuda/include/thrust/system/cuda/detail/logical.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/cuda/detail/malloc_and_free.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/memory.inl" "$(@D)/cuda/include/thrust/system/cuda/detail/memory.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/memory_buffer.h" "$(@D)/cuda/include/thrust/system/cuda/detail/memory_buffer.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/merge.h" "$(@D)/cuda/include/thrust/system/cuda/detail/merge.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/mismatch.h" "$(@D)/cuda/include/thrust/system/cuda/detail/mismatch.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/par.h" "$(@D)/cuda/include/thrust/system/cuda/detail/par.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/par_to_seq.h" "$(@D)/cuda/include/thrust/system/cuda/detail/par_to_seq.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/parallel_for.h" "$(@D)/cuda/include/thrust/system/cuda/detail/parallel_for.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/partition.h" "$(@D)/cuda/include/thrust/system/cuda/detail/partition.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/reduce.h" "$(@D)/cuda/include/thrust/system/cuda/detail/reduce.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/cuda/detail/reduce_by_key.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/remove.h" "$(@D)/cuda/include/thrust/system/cuda/detail/remove.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/replace.h" "$(@D)/cuda/include/thrust/system/cuda/detail/replace.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/reverse.h" "$(@D)/cuda/include/thrust/system/cuda/detail/reverse.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/scan.h" "$(@D)/cuda/include/thrust/system/cuda/detail/scan.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/scan_by_key.h" "$(@D)/cuda/include/thrust/system/cuda/detail/scan_by_key.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/scatter.h" "$(@D)/cuda/include/thrust/system/cuda/detail/scatter.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/sequence.h" "$(@D)/cuda/include/thrust/system/cuda/detail/sequence.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/set_operations.h" "$(@D)/cuda/include/thrust/system/cuda/detail/set_operations.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/sort.h" "$(@D)/cuda/include/thrust/system/cuda/detail/sort.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/swap_ranges.h" "$(@D)/cuda/include/thrust/system/cuda/detail/swap_ranges.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/tabulate.h" "$(@D)/cuda/include/thrust/system/cuda/detail/tabulate.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/cuda/detail/temporary_buffer.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/terminate.h" "$(@D)/cuda/include/thrust/system/cuda/detail/terminate.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/transform.h" "$(@D)/cuda/include/thrust/system/cuda/detail/transform.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/transform_reduce.h" "$(@D)/cuda/include/thrust/system/cuda/detail/transform_reduce.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/transform_scan.h" "$(@D)/cuda/include/thrust/system/cuda/detail/transform_scan.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/cuda/detail/uninitialized_copy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/cuda/detail/uninitialized_fill.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/unique.h" "$(@D)/cuda/include/thrust/system/cuda/detail/unique.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/unique_by_key.h" "$(@D)/cuda/include/thrust/system/cuda/detail/unique_by_key.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/util.h" "$(@D)/cuda/include/thrust/system/cuda/detail/util.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/vector.inl" "$(@D)/cuda/include/thrust/system/cuda/detail/vector.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/error.h" "$(@D)/cuda/include/thrust/system/cuda/error.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/execution_policy.h" "$(@D)/cuda/include/thrust/system/cuda/execution_policy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/experimental/pinned_allocator.h" "$(@D)/cuda/include/thrust/system/cuda/experimental/pinned_allocator.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/memory.h" "$(@D)/cuda/include/thrust/system/cuda/memory.h" && cp "/usr/local/cuda-9.0/include/thrust/system/cuda/vector.h" "$(@D)/cuda/include/thrust/system/cuda/vector.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/detail/adl/adjacent_difference.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/assign_value.h" "$(@D)/cuda/include/thrust/system/detail/adl/assign_value.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/binary_search.h" "$(@D)/cuda/include/thrust/system/detail/adl/binary_search.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/copy.h" "$(@D)/cuda/include/thrust/system/detail/adl/copy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/copy_if.h" "$(@D)/cuda/include/thrust/system/detail/adl/copy_if.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/count.h" "$(@D)/cuda/include/thrust/system/detail/adl/count.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/equal.h" "$(@D)/cuda/include/thrust/system/detail/adl/equal.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/extrema.h" "$(@D)/cuda/include/thrust/system/detail/adl/extrema.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/fill.h" "$(@D)/cuda/include/thrust/system/detail/adl/fill.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/find.h" "$(@D)/cuda/include/thrust/system/detail/adl/find.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/for_each.h" "$(@D)/cuda/include/thrust/system/detail/adl/for_each.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/gather.h" "$(@D)/cuda/include/thrust/system/detail/adl/gather.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/generate.h" "$(@D)/cuda/include/thrust/system/detail/adl/generate.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/get_value.h" "$(@D)/cuda/include/thrust/system/detail/adl/get_value.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/inner_product.h" "$(@D)/cuda/include/thrust/system/detail/adl/inner_product.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/iter_swap.h" "$(@D)/cuda/include/thrust/system/detail/adl/iter_swap.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/logical.h" "$(@D)/cuda/include/thrust/system/detail/adl/logical.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/detail/adl/malloc_and_free.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/merge.h" "$(@D)/cuda/include/thrust/system/detail/adl/merge.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/mismatch.h" "$(@D)/cuda/include/thrust/system/detail/adl/mismatch.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/partition.h" "$(@D)/cuda/include/thrust/system/detail/adl/partition.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/reduce.h" "$(@D)/cuda/include/thrust/system/detail/adl/reduce.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/detail/adl/reduce_by_key.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/remove.h" "$(@D)/cuda/include/thrust/system/detail/adl/remove.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/replace.h" "$(@D)/cuda/include/thrust/system/detail/adl/replace.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/reverse.h" "$(@D)/cuda/include/thrust/system/detail/adl/reverse.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/scan.h" "$(@D)/cuda/include/thrust/system/detail/adl/scan.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/scan_by_key.h" "$(@D)/cuda/include/thrust/system/detail/adl/scan_by_key.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/scatter.h" "$(@D)/cuda/include/thrust/system/detail/adl/scatter.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/sequence.h" "$(@D)/cuda/include/thrust/system/detail/adl/sequence.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/set_operations.h" "$(@D)/cuda/include/thrust/system/detail/adl/set_operations.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/sort.h" "$(@D)/cuda/include/thrust/system/detail/adl/sort.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/swap_ranges.h" "$(@D)/cuda/include/thrust/system/detail/adl/swap_ranges.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/tabulate.h" "$(@D)/cuda/include/thrust/system/detail/adl/tabulate.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/detail/adl/temporary_buffer.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/transform.h" "$(@D)/cuda/include/thrust/system/detail/adl/transform.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/transform_reduce.h" "$(@D)/cuda/include/thrust/system/detail/adl/transform_reduce.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/transform_scan.h" "$(@D)/cuda/include/thrust/system/detail/adl/transform_scan.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/detail/adl/uninitialized_copy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/detail/adl/uninitialized_fill.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/unique.h" "$(@D)/cuda/include/thrust/system/detail/adl/unique.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/adl/unique_by_key.h" "$(@D)/cuda/include/thrust/system/detail/adl/unique_by_key.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/bad_alloc.h" "$(@D)/cuda/include/thrust/system/detail/bad_alloc.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/errno.h" "$(@D)/cuda/include/thrust/system/detail/errno.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/error_category.inl" "$(@D)/cuda/include/thrust/system/detail/error_category.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/error_code.inl" "$(@D)/cuda/include/thrust/system/detail/error_code.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/error_condition.inl" "$(@D)/cuda/include/thrust/system/detail/error_condition.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/detail/generic/adjacent_difference.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/adjacent_difference.inl" "$(@D)/cuda/include/thrust/system/detail/generic/adjacent_difference.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/advance.h" "$(@D)/cuda/include/thrust/system/detail/generic/advance.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/advance.inl" "$(@D)/cuda/include/thrust/system/detail/generic/advance.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/binary_search.h" "$(@D)/cuda/include/thrust/system/detail/generic/binary_search.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/binary_search.inl" "$(@D)/cuda/include/thrust/system/detail/generic/binary_search.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/copy.h" "$(@D)/cuda/include/thrust/system/detail/generic/copy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/copy.inl" "$(@D)/cuda/include/thrust/system/detail/generic/copy.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/copy_if.h" "$(@D)/cuda/include/thrust/system/detail/generic/copy_if.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/copy_if.inl" "$(@D)/cuda/include/thrust/system/detail/generic/copy_if.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/count.h" "$(@D)/cuda/include/thrust/system/detail/generic/count.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/count.inl" "$(@D)/cuda/include/thrust/system/detail/generic/count.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/distance.h" "$(@D)/cuda/include/thrust/system/detail/generic/distance.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/distance.inl" "$(@D)/cuda/include/thrust/system/detail/generic/distance.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/equal.h" "$(@D)/cuda/include/thrust/system/detail/generic/equal.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/equal.inl" "$(@D)/cuda/include/thrust/system/detail/generic/equal.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/extrema.h" "$(@D)/cuda/include/thrust/system/detail/generic/extrema.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/extrema.inl" "$(@D)/cuda/include/thrust/system/detail/generic/extrema.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/fill.h" "$(@D)/cuda/include/thrust/system/detail/generic/fill.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/find.h" "$(@D)/cuda/include/thrust/system/detail/generic/find.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/find.inl" "$(@D)/cuda/include/thrust/system/detail/generic/find.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/for_each.h" "$(@D)/cuda/include/thrust/system/detail/generic/for_each.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/gather.h" "$(@D)/cuda/include/thrust/system/detail/generic/gather.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/gather.inl" "$(@D)/cuda/include/thrust/system/detail/generic/gather.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/generate.h" "$(@D)/cuda/include/thrust/system/detail/generic/generate.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/generate.inl" "$(@D)/cuda/include/thrust/system/detail/generic/generate.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/inner_product.h" "$(@D)/cuda/include/thrust/system/detail/generic/inner_product.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/inner_product.inl" "$(@D)/cuda/include/thrust/system/detail/generic/inner_product.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/logical.h" "$(@D)/cuda/include/thrust/system/detail/generic/logical.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/memory.h" "$(@D)/cuda/include/thrust/system/detail/generic/memory.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/memory.inl" "$(@D)/cuda/include/thrust/system/detail/generic/memory.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/merge.h" "$(@D)/cuda/include/thrust/system/detail/generic/merge.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/merge.inl" "$(@D)/cuda/include/thrust/system/detail/generic/merge.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/mismatch.h" "$(@D)/cuda/include/thrust/system/detail/generic/mismatch.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/mismatch.inl" "$(@D)/cuda/include/thrust/system/detail/generic/mismatch.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/partition.h" "$(@D)/cuda/include/thrust/system/detail/generic/partition.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/partition.inl" "$(@D)/cuda/include/thrust/system/detail/generic/partition.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/reduce.h" "$(@D)/cuda/include/thrust/system/detail/generic/reduce.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/reduce.inl" "$(@D)/cuda/include/thrust/system/detail/generic/reduce.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/detail/generic/reduce_by_key.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/reduce_by_key.inl" "$(@D)/cuda/include/thrust/system/detail/generic/reduce_by_key.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/remove.h" "$(@D)/cuda/include/thrust/system/detail/generic/remove.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/remove.inl" "$(@D)/cuda/include/thrust/system/detail/generic/remove.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/replace.h" "$(@D)/cuda/include/thrust/system/detail/generic/replace.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/replace.inl" "$(@D)/cuda/include/thrust/system/detail/generic/replace.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/reverse.h" "$(@D)/cuda/include/thrust/system/detail/generic/reverse.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/reverse.inl" "$(@D)/cuda/include/thrust/system/detail/generic/reverse.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/scalar/binary_search.h" "$(@D)/cuda/include/thrust/system/detail/generic/scalar/binary_search.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/scalar/binary_search.inl" "$(@D)/cuda/include/thrust/system/detail/generic/scalar/binary_search.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/scan.h" "$(@D)/cuda/include/thrust/system/detail/generic/scan.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/scan.inl" "$(@D)/cuda/include/thrust/system/detail/generic/scan.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/scan_by_key.h" "$(@D)/cuda/include/thrust/system/detail/generic/scan_by_key.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/scan_by_key.inl" "$(@D)/cuda/include/thrust/system/detail/generic/scan_by_key.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/scatter.h" "$(@D)/cuda/include/thrust/system/detail/generic/scatter.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/scatter.inl" "$(@D)/cuda/include/thrust/system/detail/generic/scatter.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/select_system.h" "$(@D)/cuda/include/thrust/system/detail/generic/select_system.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/sequence.h" "$(@D)/cuda/include/thrust/system/detail/generic/sequence.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/sequence.inl" "$(@D)/cuda/include/thrust/system/detail/generic/sequence.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/set_operations.h" "$(@D)/cuda/include/thrust/system/detail/generic/set_operations.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/set_operations.inl" "$(@D)/cuda/include/thrust/system/detail/generic/set_operations.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/sort.h" "$(@D)/cuda/include/thrust/system/detail/generic/sort.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/sort.inl" "$(@D)/cuda/include/thrust/system/detail/generic/sort.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/swap_ranges.h" "$(@D)/cuda/include/thrust/system/detail/generic/swap_ranges.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/swap_ranges.inl" "$(@D)/cuda/include/thrust/system/detail/generic/swap_ranges.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/tabulate.h" "$(@D)/cuda/include/thrust/system/detail/generic/tabulate.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/tabulate.inl" "$(@D)/cuda/include/thrust/system/detail/generic/tabulate.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/tag.h" "$(@D)/cuda/include/thrust/system/detail/generic/tag.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/detail/generic/temporary_buffer.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/temporary_buffer.inl" "$(@D)/cuda/include/thrust/system/detail/generic/temporary_buffer.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/transform.h" "$(@D)/cuda/include/thrust/system/detail/generic/transform.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/transform.inl" "$(@D)/cuda/include/thrust/system/detail/generic/transform.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/transform_reduce.h" "$(@D)/cuda/include/thrust/system/detail/generic/transform_reduce.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/transform_reduce.inl" "$(@D)/cuda/include/thrust/system/detail/generic/transform_reduce.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/transform_scan.h" "$(@D)/cuda/include/thrust/system/detail/generic/transform_scan.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/transform_scan.inl" "$(@D)/cuda/include/thrust/system/detail/generic/transform_scan.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/type_traits.h" "$(@D)/cuda/include/thrust/system/detail/generic/type_traits.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/detail/generic/uninitialized_copy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/uninitialized_copy.inl" "$(@D)/cuda/include/thrust/system/detail/generic/uninitialized_copy.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/detail/generic/uninitialized_fill.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/uninitialized_fill.inl" "$(@D)/cuda/include/thrust/system/detail/generic/uninitialized_fill.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/unique.h" "$(@D)/cuda/include/thrust/system/detail/generic/unique.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/unique.inl" "$(@D)/cuda/include/thrust/system/detail/generic/unique.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/unique_by_key.h" "$(@D)/cuda/include/thrust/system/detail/generic/unique_by_key.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/generic/unique_by_key.inl" "$(@D)/cuda/include/thrust/system/detail/generic/unique_by_key.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/internal/decompose.h" "$(@D)/cuda/include/thrust/system/detail/internal/decompose.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/detail/sequential/adjacent_difference.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/assign_value.h" "$(@D)/cuda/include/thrust/system/detail/sequential/assign_value.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/binary_search.h" "$(@D)/cuda/include/thrust/system/detail/sequential/binary_search.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/copy.h" "$(@D)/cuda/include/thrust/system/detail/sequential/copy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/copy.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/copy.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/copy_backward.h" "$(@D)/cuda/include/thrust/system/detail/sequential/copy_backward.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/copy_if.h" "$(@D)/cuda/include/thrust/system/detail/sequential/copy_if.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/count.h" "$(@D)/cuda/include/thrust/system/detail/sequential/count.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/equal.h" "$(@D)/cuda/include/thrust/system/detail/sequential/equal.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/execution_policy.h" "$(@D)/cuda/include/thrust/system/detail/sequential/execution_policy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/extrema.h" "$(@D)/cuda/include/thrust/system/detail/sequential/extrema.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/fill.h" "$(@D)/cuda/include/thrust/system/detail/sequential/fill.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/find.h" "$(@D)/cuda/include/thrust/system/detail/sequential/find.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/for_each.h" "$(@D)/cuda/include/thrust/system/detail/sequential/for_each.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/gather.h" "$(@D)/cuda/include/thrust/system/detail/sequential/gather.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/general_copy.h" "$(@D)/cuda/include/thrust/system/detail/sequential/general_copy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/generate.h" "$(@D)/cuda/include/thrust/system/detail/sequential/generate.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/get_value.h" "$(@D)/cuda/include/thrust/system/detail/sequential/get_value.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/inner_product.h" "$(@D)/cuda/include/thrust/system/detail/sequential/inner_product.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/insertion_sort.h" "$(@D)/cuda/include/thrust/system/detail/sequential/insertion_sort.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/iter_swap.h" "$(@D)/cuda/include/thrust/system/detail/sequential/iter_swap.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/logical.h" "$(@D)/cuda/include/thrust/system/detail/sequential/logical.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/detail/sequential/malloc_and_free.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/merge.h" "$(@D)/cuda/include/thrust/system/detail/sequential/merge.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/merge.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/merge.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/mismatch.h" "$(@D)/cuda/include/thrust/system/detail/sequential/mismatch.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/partition.h" "$(@D)/cuda/include/thrust/system/detail/sequential/partition.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/reduce.h" "$(@D)/cuda/include/thrust/system/detail/sequential/reduce.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/detail/sequential/reduce_by_key.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/remove.h" "$(@D)/cuda/include/thrust/system/detail/sequential/remove.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/replace.h" "$(@D)/cuda/include/thrust/system/detail/sequential/replace.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/reverse.h" "$(@D)/cuda/include/thrust/system/detail/sequential/reverse.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/scan.h" "$(@D)/cuda/include/thrust/system/detail/sequential/scan.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/scan_by_key.h" "$(@D)/cuda/include/thrust/system/detail/sequential/scan_by_key.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/scatter.h" "$(@D)/cuda/include/thrust/system/detail/sequential/scatter.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/sequence.h" "$(@D)/cuda/include/thrust/system/detail/sequential/sequence.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/set_operations.h" "$(@D)/cuda/include/thrust/system/detail/sequential/set_operations.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/sort.h" "$(@D)/cuda/include/thrust/system/detail/sequential/sort.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/sort.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/sort.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/stable_merge_sort.h" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_merge_sort.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/stable_merge_sort.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_merge_sort.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/stable_primitive_sort.h" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_primitive_sort.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/stable_primitive_sort.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_primitive_sort.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/stable_radix_sort.h" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_radix_sort.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/stable_radix_sort.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_radix_sort.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/swap_ranges.h" "$(@D)/cuda/include/thrust/system/detail/sequential/swap_ranges.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/tabulate.h" "$(@D)/cuda/include/thrust/system/detail/sequential/tabulate.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/detail/sequential/temporary_buffer.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/transform.h" "$(@D)/cuda/include/thrust/system/detail/sequential/transform.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/transform_reduce.h" "$(@D)/cuda/include/thrust/system/detail/sequential/transform_reduce.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/transform_scan.h" "$(@D)/cuda/include/thrust/system/detail/sequential/transform_scan.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/trivial_copy.h" "$(@D)/cuda/include/thrust/system/detail/sequential/trivial_copy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/detail/sequential/uninitialized_copy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/detail/sequential/uninitialized_fill.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/unique.h" "$(@D)/cuda/include/thrust/system/detail/sequential/unique.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/unique_by_key.h" "$(@D)/cuda/include/thrust/system/detail/sequential/unique_by_key.h" && cp "/usr/local/cuda-9.0/include/thrust/system/detail/system_error.inl" "$(@D)/cuda/include/thrust/system/detail/system_error.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/error_code.h" "$(@D)/cuda/include/thrust/system/error_code.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/omp/detail/adjacent_difference.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/assign_value.h" "$(@D)/cuda/include/thrust/system/omp/detail/assign_value.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/binary_search.h" "$(@D)/cuda/include/thrust/system/omp/detail/binary_search.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/copy.h" "$(@D)/cuda/include/thrust/system/omp/detail/copy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/copy.inl" "$(@D)/cuda/include/thrust/system/omp/detail/copy.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/copy_if.h" "$(@D)/cuda/include/thrust/system/omp/detail/copy_if.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/copy_if.inl" "$(@D)/cuda/include/thrust/system/omp/detail/copy_if.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/count.h" "$(@D)/cuda/include/thrust/system/omp/detail/count.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/default_decomposition.h" "$(@D)/cuda/include/thrust/system/omp/detail/default_decomposition.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/default_decomposition.inl" "$(@D)/cuda/include/thrust/system/omp/detail/default_decomposition.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/equal.h" "$(@D)/cuda/include/thrust/system/omp/detail/equal.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/execution_policy.h" "$(@D)/cuda/include/thrust/system/omp/detail/execution_policy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/extrema.h" "$(@D)/cuda/include/thrust/system/omp/detail/extrema.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/fill.h" "$(@D)/cuda/include/thrust/system/omp/detail/fill.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/find.h" "$(@D)/cuda/include/thrust/system/omp/detail/find.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/for_each.h" "$(@D)/cuda/include/thrust/system/omp/detail/for_each.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/for_each.inl" "$(@D)/cuda/include/thrust/system/omp/detail/for_each.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/gather.h" "$(@D)/cuda/include/thrust/system/omp/detail/gather.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/generate.h" "$(@D)/cuda/include/thrust/system/omp/detail/generate.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/get_value.h" "$(@D)/cuda/include/thrust/system/omp/detail/get_value.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/inner_product.h" "$(@D)/cuda/include/thrust/system/omp/detail/inner_product.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/iter_swap.h" "$(@D)/cuda/include/thrust/system/omp/detail/iter_swap.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/logical.h" "$(@D)/cuda/include/thrust/system/omp/detail/logical.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/omp/detail/malloc_and_free.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/memory.inl" "$(@D)/cuda/include/thrust/system/omp/detail/memory.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/merge.h" "$(@D)/cuda/include/thrust/system/omp/detail/merge.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/mismatch.h" "$(@D)/cuda/include/thrust/system/omp/detail/mismatch.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/par.h" "$(@D)/cuda/include/thrust/system/omp/detail/par.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/partition.h" "$(@D)/cuda/include/thrust/system/omp/detail/partition.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/partition.inl" "$(@D)/cuda/include/thrust/system/omp/detail/partition.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/reduce.h" "$(@D)/cuda/include/thrust/system/omp/detail/reduce.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/reduce.inl" "$(@D)/cuda/include/thrust/system/omp/detail/reduce.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/omp/detail/reduce_by_key.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/reduce_by_key.inl" "$(@D)/cuda/include/thrust/system/omp/detail/reduce_by_key.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/reduce_intervals.h" "$(@D)/cuda/include/thrust/system/omp/detail/reduce_intervals.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/reduce_intervals.inl" "$(@D)/cuda/include/thrust/system/omp/detail/reduce_intervals.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/remove.h" "$(@D)/cuda/include/thrust/system/omp/detail/remove.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/remove.inl" "$(@D)/cuda/include/thrust/system/omp/detail/remove.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/replace.h" "$(@D)/cuda/include/thrust/system/omp/detail/replace.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/reverse.h" "$(@D)/cuda/include/thrust/system/omp/detail/reverse.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/scan.h" "$(@D)/cuda/include/thrust/system/omp/detail/scan.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/scan_by_key.h" "$(@D)/cuda/include/thrust/system/omp/detail/scan_by_key.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/scatter.h" "$(@D)/cuda/include/thrust/system/omp/detail/scatter.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/sequence.h" "$(@D)/cuda/include/thrust/system/omp/detail/sequence.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/set_operations.h" "$(@D)/cuda/include/thrust/system/omp/detail/set_operations.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/sort.h" "$(@D)/cuda/include/thrust/system/omp/detail/sort.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/sort.inl" "$(@D)/cuda/include/thrust/system/omp/detail/sort.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/swap_ranges.h" "$(@D)/cuda/include/thrust/system/omp/detail/swap_ranges.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/tabulate.h" "$(@D)/cuda/include/thrust/system/omp/detail/tabulate.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/omp/detail/temporary_buffer.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/transform.h" "$(@D)/cuda/include/thrust/system/omp/detail/transform.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/transform_reduce.h" "$(@D)/cuda/include/thrust/system/omp/detail/transform_reduce.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/transform_scan.h" "$(@D)/cuda/include/thrust/system/omp/detail/transform_scan.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/omp/detail/uninitialized_copy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/omp/detail/uninitialized_fill.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/unique.h" "$(@D)/cuda/include/thrust/system/omp/detail/unique.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/unique.inl" "$(@D)/cuda/include/thrust/system/omp/detail/unique.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/unique_by_key.h" "$(@D)/cuda/include/thrust/system/omp/detail/unique_by_key.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/unique_by_key.inl" "$(@D)/cuda/include/thrust/system/omp/detail/unique_by_key.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/detail/vector.inl" "$(@D)/cuda/include/thrust/system/omp/detail/vector.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/execution_policy.h" "$(@D)/cuda/include/thrust/system/omp/execution_policy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/memory.h" "$(@D)/cuda/include/thrust/system/omp/memory.h" && cp "/usr/local/cuda-9.0/include/thrust/system/omp/vector.h" "$(@D)/cuda/include/thrust/system/omp/vector.h" && cp "/usr/local/cuda-9.0/include/thrust/system/system_error.h" "$(@D)/cuda/include/thrust/system/system_error.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/tbb/detail/adjacent_difference.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/assign_value.h" "$(@D)/cuda/include/thrust/system/tbb/detail/assign_value.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/binary_search.h" "$(@D)/cuda/include/thrust/system/tbb/detail/binary_search.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/copy.h" "$(@D)/cuda/include/thrust/system/tbb/detail/copy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/copy.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/copy.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/copy_if.h" "$(@D)/cuda/include/thrust/system/tbb/detail/copy_if.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/copy_if.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/copy_if.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/count.h" "$(@D)/cuda/include/thrust/system/tbb/detail/count.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/equal.h" "$(@D)/cuda/include/thrust/system/tbb/detail/equal.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/execution_policy.h" "$(@D)/cuda/include/thrust/system/tbb/detail/execution_policy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/extrema.h" "$(@D)/cuda/include/thrust/system/tbb/detail/extrema.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/fill.h" "$(@D)/cuda/include/thrust/system/tbb/detail/fill.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/find.h" "$(@D)/cuda/include/thrust/system/tbb/detail/find.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/for_each.h" "$(@D)/cuda/include/thrust/system/tbb/detail/for_each.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/for_each.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/for_each.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/gather.h" "$(@D)/cuda/include/thrust/system/tbb/detail/gather.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/generate.h" "$(@D)/cuda/include/thrust/system/tbb/detail/generate.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/get_value.h" "$(@D)/cuda/include/thrust/system/tbb/detail/get_value.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/inner_product.h" "$(@D)/cuda/include/thrust/system/tbb/detail/inner_product.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/iter_swap.h" "$(@D)/cuda/include/thrust/system/tbb/detail/iter_swap.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/logical.h" "$(@D)/cuda/include/thrust/system/tbb/detail/logical.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/tbb/detail/malloc_and_free.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/memory.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/memory.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/merge.h" "$(@D)/cuda/include/thrust/system/tbb/detail/merge.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/merge.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/merge.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/mismatch.h" "$(@D)/cuda/include/thrust/system/tbb/detail/mismatch.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/par.h" "$(@D)/cuda/include/thrust/system/tbb/detail/par.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/partition.h" "$(@D)/cuda/include/thrust/system/tbb/detail/partition.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/partition.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/partition.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/reduce.h" "$(@D)/cuda/include/thrust/system/tbb/detail/reduce.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/reduce.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/reduce.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/tbb/detail/reduce_by_key.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/reduce_by_key.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/reduce_by_key.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/reduce_intervals.h" "$(@D)/cuda/include/thrust/system/tbb/detail/reduce_intervals.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/remove.h" "$(@D)/cuda/include/thrust/system/tbb/detail/remove.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/remove.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/remove.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/replace.h" "$(@D)/cuda/include/thrust/system/tbb/detail/replace.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/reverse.h" "$(@D)/cuda/include/thrust/system/tbb/detail/reverse.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/scan.h" "$(@D)/cuda/include/thrust/system/tbb/detail/scan.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/scan.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/scan.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/scan_by_key.h" "$(@D)/cuda/include/thrust/system/tbb/detail/scan_by_key.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/scatter.h" "$(@D)/cuda/include/thrust/system/tbb/detail/scatter.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/sequence.h" "$(@D)/cuda/include/thrust/system/tbb/detail/sequence.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/set_operations.h" "$(@D)/cuda/include/thrust/system/tbb/detail/set_operations.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/sort.h" "$(@D)/cuda/include/thrust/system/tbb/detail/sort.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/sort.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/sort.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/swap_ranges.h" "$(@D)/cuda/include/thrust/system/tbb/detail/swap_ranges.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/tabulate.h" "$(@D)/cuda/include/thrust/system/tbb/detail/tabulate.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/tbb/detail/temporary_buffer.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/transform.h" "$(@D)/cuda/include/thrust/system/tbb/detail/transform.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/transform_reduce.h" "$(@D)/cuda/include/thrust/system/tbb/detail/transform_reduce.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/transform_scan.h" "$(@D)/cuda/include/thrust/system/tbb/detail/transform_scan.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/tbb/detail/uninitialized_copy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/tbb/detail/uninitialized_fill.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/unique.h" "$(@D)/cuda/include/thrust/system/tbb/detail/unique.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/unique.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/unique.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/unique_by_key.h" "$(@D)/cuda/include/thrust/system/tbb/detail/unique_by_key.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/unique_by_key.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/unique_by_key.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/vector.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/vector.inl" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/execution_policy.h" "$(@D)/cuda/include/thrust/system/tbb/execution_policy.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/memory.h" "$(@D)/cuda/include/thrust/system/tbb/memory.h" && cp "/usr/local/cuda-9.0/include/thrust/system/tbb/vector.h" "$(@D)/cuda/include/thrust/system/tbb/vector.h" && cp "/usr/local/cuda-9.0/include/thrust/system_error.h" "$(@D)/cuda/include/thrust/system_error.h" && cp "/usr/local/cuda-9.0/include/thrust/tabulate.h" "$(@D)/cuda/include/thrust/tabulate.h" && cp "/usr/local/cuda-9.0/include/thrust/transform.h" "$(@D)/cuda/include/thrust/transform.h" && cp "/usr/local/cuda-9.0/include/thrust/transform_reduce.h" "$(@D)/cuda/include/thrust/transform_reduce.h" && cp "/usr/local/cuda-9.0/include/thrust/transform_scan.h" "$(@D)/cuda/include/thrust/transform_scan.h" && cp "/usr/local/cuda-9.0/include/thrust/tuple.h" "$(@D)/cuda/include/thrust/tuple.h" && cp "/usr/local/cuda-9.0/include/thrust/uninitialized_copy.h" "$(@D)/cuda/include/thrust/uninitialized_copy.h" && cp "/usr/local/cuda-9.0/include/thrust/uninitialized_fill.h" "$(@D)/cuda/include/thrust/uninitialized_fill.h" && cp "/usr/local/cuda-9.0/include/thrust/unique.h" "$(@D)/cuda/include/thrust/unique.h" && cp "/usr/local/cuda-9.0/include/thrust/version.h" "$(@D)/cuda/include/thrust/version.h" && cp "/usr/local/cuda-9.0/include/vector_functions.h" "$(@D)/cuda/include/vector_functions.h" && cp "/usr/local/cuda-9.0/include/vector_functions.hpp" "$(@D)/cuda/include/vector_functions.hpp" && cp "/usr/local/cuda-9.0/include/vector_types.h" "$(@D)/cuda/include/vector_types.h"
+if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp "/usr/local/cuda-10.0/include/CL/cl.h" "$(@D)/cuda/include/CL/cl.h" && cp "/usr/local/cuda-10.0/include/CL/cl.hpp" "$(@D)/cuda/include/CL/cl.hpp" && cp "/usr/local/cuda-10.0/include/CL/cl_egl.h" "$(@D)/cuda/include/CL/cl_egl.h" && cp "/usr/local/cuda-10.0/include/CL/cl_ext.h" "$(@D)/cuda/include/CL/cl_ext.h" && cp "/usr/local/cuda-10.0/include/CL/cl_gl.h" "$(@D)/cuda/include/CL/cl_gl.h" && cp "/usr/local/cuda-10.0/include/CL/cl_gl_ext.h" "$(@D)/cuda/include/CL/cl_gl_ext.h" && cp "/usr/local/cuda-10.0/include/CL/cl_platform.h" "$(@D)/cuda/include/CL/cl_platform.h" && cp "/usr/local/cuda-10.0/include/CL/opencl.h" "$(@D)/cuda/include/CL/opencl.h" && cp "/usr/local/cuda-10.0/include/builtin_types.h" "$(@D)/cuda/include/builtin_types.h" && cp "/usr/local/cuda-10.0/include/channel_descriptor.h" "$(@D)/cuda/include/channel_descriptor.h" && cp "/usr/local/cuda-10.0/include/common_functions.h" "$(@D)/cuda/include/common_functions.h" && cp "/usr/local/cuda-10.0/include/cooperative_groups.h" "$(@D)/cuda/include/cooperative_groups.h" && cp "/usr/local/cuda-10.0/include/cooperative_groups_helpers.h" "$(@D)/cuda/include/cooperative_groups_helpers.h" && cp "/usr/local/cuda-10.0/include/crt/common_functions.h" "$(@D)/cuda/include/crt/common_functions.h" && cp "/usr/local/cuda-10.0/include/crt/device_double_functions.h" "$(@D)/cuda/include/crt/device_double_functions.h" && cp "/usr/local/cuda-10.0/include/crt/device_double_functions.hpp" "$(@D)/cuda/include/crt/device_double_functions.hpp" && cp "/usr/local/cuda-10.0/include/crt/device_functions.h" "$(@D)/cuda/include/crt/device_functions.h" && cp "/usr/local/cuda-10.0/include/crt/device_functions.hpp" "$(@D)/cuda/include/crt/device_functions.hpp" && cp "/usr/local/cuda-10.0/include/crt/func_macro.h" "$(@D)/cuda/include/crt/func_macro.h" && cp "/usr/local/cuda-10.0/include/crt/host_config.h" "$(@D)/cuda/include/crt/host_config.h" && cp "/usr/local/cuda-10.0/include/crt/host_defines.h" "$(@D)/cuda/include/crt/host_defines.h" && cp "/usr/local/cuda-10.0/include/crt/host_runtime.h" "$(@D)/cuda/include/crt/host_runtime.h" && cp "/usr/local/cuda-10.0/include/crt/math_functions.h" "$(@D)/cuda/include/crt/math_functions.h" && cp "/usr/local/cuda-10.0/include/crt/math_functions.hpp" "$(@D)/cuda/include/crt/math_functions.hpp" && cp "/usr/local/cuda-10.0/include/crt/mma.h" "$(@D)/cuda/include/crt/mma.h" && cp "/usr/local/cuda-10.0/include/crt/mma.hpp" "$(@D)/cuda/include/crt/mma.hpp" && cp "/usr/local/cuda-10.0/include/crt/nvfunctional" "$(@D)/cuda/include/crt/nvfunctional" && cp "/usr/local/cuda-10.0/include/crt/sm_70_rt.h" "$(@D)/cuda/include/crt/sm_70_rt.h" && cp "/usr/local/cuda-10.0/include/crt/sm_70_rt.hpp" "$(@D)/cuda/include/crt/sm_70_rt.hpp" && cp "/usr/local/cuda-10.0/include/crt/storage_class.h" "$(@D)/cuda/include/crt/storage_class.h" && cp "/usr/local/cuda-10.0/include/cuComplex.h" "$(@D)/cuda/include/cuComplex.h" && cp "/usr/local/cuda-10.0/include/cublas.h" "$(@D)/cuda/include/cublas.h" && cp "/usr/local/cuda-10.0/include/cublasXt.h" "$(@D)/cuda/include/cublasXt.h" && cp "/usr/local/cuda-10.0/include/cublas_api.h" "$(@D)/cuda/include/cublas_api.h" && cp "/usr/local/cuda-10.0/include/cublas_v2.h" "$(@D)/cuda/include/cublas_v2.h" && cp "/usr/local/cuda-10.0/include/cuda.h" "$(@D)/cuda/include/cuda.h" && cp "/usr/local/cuda-10.0/include/cudaEGL.h" "$(@D)/cuda/include/cudaEGL.h" && cp "/usr/local/cuda-10.0/include/cudaGL.h" "$(@D)/cuda/include/cudaGL.h" && cp "/usr/local/cuda-10.0/include/cudaProfiler.h" "$(@D)/cuda/include/cudaProfiler.h" && cp "/usr/local/cuda-10.0/include/cudaVDPAU.h" "$(@D)/cuda/include/cudaVDPAU.h" && cp "/usr/local/cuda-10.0/include/cuda_device_runtime_api.h" "$(@D)/cuda/include/cuda_device_runtime_api.h" && cp "/usr/local/cuda-10.0/include/cuda_fp16.h" "$(@D)/cuda/include/cuda_fp16.h" && cp "/usr/local/cuda-10.0/include/cuda_fp16.hpp" "$(@D)/cuda/include/cuda_fp16.hpp" && cp "/usr/local/cuda-10.0/include/cuda_gl_interop.h" "$(@D)/cuda/include/cuda_gl_interop.h" && cp "/usr/local/cuda-10.0/include/cuda_occupancy.h" "$(@D)/cuda/include/cuda_occupancy.h" && cp "/usr/local/cuda-10.0/include/cuda_profiler_api.h" "$(@D)/cuda/include/cuda_profiler_api.h" && cp "/usr/local/cuda-10.0/include/cuda_runtime.h" "$(@D)/cuda/include/cuda_runtime.h" && cp "/usr/local/cuda-10.0/include/cuda_runtime_api.h" "$(@D)/cuda/include/cuda_runtime_api.h" && cp "/usr/local/cuda-10.0/include/cuda_surface_types.h" "$(@D)/cuda/include/cuda_surface_types.h" && cp "/usr/local/cuda-10.0/include/cuda_texture_types.h" "$(@D)/cuda/include/cuda_texture_types.h" && cp "/usr/local/cuda-10.0/include/cuda_vdpau_interop.h" "$(@D)/cuda/include/cuda_vdpau_interop.h" && cp "/usr/local/cuda-10.0/include/cudalibxt.h" "$(@D)/cuda/include/cudalibxt.h" && cp "/usr/local/cuda-10.0/include/cudnn.h" "$(@D)/cuda/include/cudnn.h" && cp "/usr/local/cuda-10.0/include/cufft.h" "$(@D)/cuda/include/cufft.h" && cp "/usr/local/cuda-10.0/include/cufftXt.h" "$(@D)/cuda/include/cufftXt.h" && cp "/usr/local/cuda-10.0/include/cufftw.h" "$(@D)/cuda/include/cufftw.h" && cp "/usr/local/cuda-10.0/include/curand.h" "$(@D)/cuda/include/curand.h" && cp "/usr/local/cuda-10.0/include/curand_discrete.h" "$(@D)/cuda/include/curand_discrete.h" && cp "/usr/local/cuda-10.0/include/curand_discrete2.h" "$(@D)/cuda/include/curand_discrete2.h" && cp "/usr/local/cuda-10.0/include/curand_globals.h" "$(@D)/cuda/include/curand_globals.h" && cp "/usr/local/cuda-10.0/include/curand_kernel.h" "$(@D)/cuda/include/curand_kernel.h" && cp "/usr/local/cuda-10.0/include/curand_lognormal.h" "$(@D)/cuda/include/curand_lognormal.h" && cp "/usr/local/cuda-10.0/include/curand_mrg32k3a.h" "$(@D)/cuda/include/curand_mrg32k3a.h" && cp "/usr/local/cuda-10.0/include/curand_mtgp32.h" "$(@D)/cuda/include/curand_mtgp32.h" && cp "/usr/local/cuda-10.0/include/curand_mtgp32_host.h" "$(@D)/cuda/include/curand_mtgp32_host.h" && cp "/usr/local/cuda-10.0/include/curand_mtgp32_kernel.h" "$(@D)/cuda/include/curand_mtgp32_kernel.h" && cp "/usr/local/cuda-10.0/include/curand_mtgp32dc_p_11213.h" "$(@D)/cuda/include/curand_mtgp32dc_p_11213.h" && cp "/usr/local/cuda-10.0/include/curand_normal.h" "$(@D)/cuda/include/curand_normal.h" && cp "/usr/local/cuda-10.0/include/curand_normal_static.h" "$(@D)/cuda/include/curand_normal_static.h" && cp "/usr/local/cuda-10.0/include/curand_philox4x32_x.h" "$(@D)/cuda/include/curand_philox4x32_x.h" && cp "/usr/local/cuda-10.0/include/curand_poisson.h" "$(@D)/cuda/include/curand_poisson.h" && cp "/usr/local/cuda-10.0/include/curand_precalc.h" "$(@D)/cuda/include/curand_precalc.h" && cp "/usr/local/cuda-10.0/include/curand_uniform.h" "$(@D)/cuda/include/curand_uniform.h" && cp "/usr/local/cuda-10.0/include/cusolverDn.h" "$(@D)/cuda/include/cusolverDn.h" && cp "/usr/local/cuda-10.0/include/cusolverRf.h" "$(@D)/cuda/include/cusolverRf.h" && cp "/usr/local/cuda-10.0/include/cusolverSp.h" "$(@D)/cuda/include/cusolverSp.h" && cp "/usr/local/cuda-10.0/include/cusolverSp_LOWLEVEL_PREVIEW.h" "$(@D)/cuda/include/cusolverSp_LOWLEVEL_PREVIEW.h" && cp "/usr/local/cuda-10.0/include/cusolver_common.h" "$(@D)/cuda/include/cusolver_common.h" && cp "/usr/local/cuda-10.0/include/cusparse.h" "$(@D)/cuda/include/cusparse.h" && cp "/usr/local/cuda-10.0/include/cusparse_v2.h" "$(@D)/cuda/include/cusparse_v2.h" && cp "/usr/local/cuda-10.0/include/device_atomic_functions.h" "$(@D)/cuda/include/device_atomic_functions.h" && cp "/usr/local/cuda-10.0/include/device_atomic_functions.hpp" "$(@D)/cuda/include/device_atomic_functions.hpp" && cp "/usr/local/cuda-10.0/include/device_double_functions.h" "$(@D)/cuda/include/device_double_functions.h" && cp "/usr/local/cuda-10.0/include/device_double_functions.hpp" "$(@D)/cuda/include/device_double_functions.hpp" && cp "/usr/local/cuda-10.0/include/device_functions.h" "$(@D)/cuda/include/device_functions.h" && cp "/usr/local/cuda-10.0/include/device_functions.hpp" "$(@D)/cuda/include/device_functions.hpp" && cp "/usr/local/cuda-10.0/include/device_functions_decls.h" "$(@D)/cuda/include/device_functions_decls.h" && cp "/usr/local/cuda-10.0/include/device_launch_parameters.h" "$(@D)/cuda/include/device_launch_parameters.h" && cp "/usr/local/cuda-10.0/include/device_types.h" "$(@D)/cuda/include/device_types.h" && cp "/usr/local/cuda-10.0/include/driver_functions.h" "$(@D)/cuda/include/driver_functions.h" && cp "/usr/local/cuda-10.0/include/driver_types.h" "$(@D)/cuda/include/driver_types.h" && cp "/usr/local/cuda-10.0/include/dynlink_cuda.h" "$(@D)/cuda/include/dynlink_cuda.h" && cp "/usr/local/cuda-10.0/include/dynlink_cuda_cuda.h" "$(@D)/cuda/include/dynlink_cuda_cuda.h" && cp "/usr/local/cuda-10.0/include/dynlink_cuviddec.h" "$(@D)/cuda/include/dynlink_cuviddec.h" && cp "/usr/local/cuda-10.0/include/dynlink_nvcuvid.h" "$(@D)/cuda/include/dynlink_nvcuvid.h" && cp "/usr/local/cuda-10.0/include/fatBinaryCtl.h" "$(@D)/cuda/include/fatBinaryCtl.h" && cp "/usr/local/cuda-10.0/include/fatbinary.h" "$(@D)/cuda/include/fatbinary.h" && cp "/usr/local/cuda-10.0/include/host_config.h" "$(@D)/cuda/include/host_config.h" && cp "/usr/local/cuda-10.0/include/host_defines.h" "$(@D)/cuda/include/host_defines.h" && cp "/usr/local/cuda-10.0/include/library_types.h" "$(@D)/cuda/include/library_types.h" && cp "/usr/local/cuda-10.0/include/math_constants.h" "$(@D)/cuda/include/math_constants.h" && cp "/usr/local/cuda-10.0/include/math_functions.h" "$(@D)/cuda/include/math_functions.h" && cp "/usr/local/cuda-10.0/include/math_functions.hpp" "$(@D)/cuda/include/math_functions.hpp" && cp "/usr/local/cuda-10.0/include/math_functions_dbl_ptx3.h" "$(@D)/cuda/include/math_functions_dbl_ptx3.h" && cp "/usr/local/cuda-10.0/include/math_functions_dbl_ptx3.hpp" "$(@D)/cuda/include/math_functions_dbl_ptx3.hpp" && cp "/usr/local/cuda-10.0/include/mma.h" "$(@D)/cuda/include/mma.h" && cp "/usr/local/cuda-10.0/include/npp.h" "$(@D)/cuda/include/npp.h" && cp "/usr/local/cuda-10.0/include/nppcore.h" "$(@D)/cuda/include/nppcore.h" && cp "/usr/local/cuda-10.0/include/nppdefs.h" "$(@D)/cuda/include/nppdefs.h" && cp "/usr/local/cuda-10.0/include/nppi.h" "$(@D)/cuda/include/nppi.h" && cp "/usr/local/cuda-10.0/include/nppi_arithmetic_and_logical_operations.h" "$(@D)/cuda/include/nppi_arithmetic_and_logical_operations.h" && cp "/usr/local/cuda-10.0/include/nppi_color_conversion.h" "$(@D)/cuda/include/nppi_color_conversion.h" && cp "/usr/local/cuda-10.0/include/nppi_compression_functions.h" "$(@D)/cuda/include/nppi_compression_functions.h" && cp "/usr/local/cuda-10.0/include/nppi_computer_vision.h" "$(@D)/cuda/include/nppi_computer_vision.h" && cp "/usr/local/cuda-10.0/include/nppi_data_exchange_and_initialization.h" "$(@D)/cuda/include/nppi_data_exchange_and_initialization.h" && cp "/usr/local/cuda-10.0/include/nppi_filtering_functions.h" "$(@D)/cuda/include/nppi_filtering_functions.h" && cp "/usr/local/cuda-10.0/include/nppi_geometry_transforms.h" "$(@D)/cuda/include/nppi_geometry_transforms.h" && cp "/usr/local/cuda-10.0/include/nppi_linear_transforms.h" "$(@D)/cuda/include/nppi_linear_transforms.h" && cp "/usr/local/cuda-10.0/include/nppi_morphological_operations.h" "$(@D)/cuda/include/nppi_morphological_operations.h" && cp "/usr/local/cuda-10.0/include/nppi_statistics_functions.h" "$(@D)/cuda/include/nppi_statistics_functions.h" && cp "/usr/local/cuda-10.0/include/nppi_support_functions.h" "$(@D)/cuda/include/nppi_support_functions.h" && cp "/usr/local/cuda-10.0/include/nppi_threshold_and_compare_operations.h" "$(@D)/cuda/include/nppi_threshold_and_compare_operations.h" && cp "/usr/local/cuda-10.0/include/npps.h" "$(@D)/cuda/include/npps.h" && cp "/usr/local/cuda-10.0/include/npps_arithmetic_and_logical_operations.h" "$(@D)/cuda/include/npps_arithmetic_and_logical_operations.h" && cp "/usr/local/cuda-10.0/include/npps_conversion_functions.h" "$(@D)/cuda/include/npps_conversion_functions.h" && cp "/usr/local/cuda-10.0/include/npps_filtering_functions.h" "$(@D)/cuda/include/npps_filtering_functions.h" && cp "/usr/local/cuda-10.0/include/npps_initialization.h" "$(@D)/cuda/include/npps_initialization.h" && cp "/usr/local/cuda-10.0/include/npps_statistics_functions.h" "$(@D)/cuda/include/npps_statistics_functions.h" && cp "/usr/local/cuda-10.0/include/npps_support_functions.h" "$(@D)/cuda/include/npps_support_functions.h" && cp "/usr/local/cuda-10.0/include/nppversion.h" "$(@D)/cuda/include/nppversion.h" && cp "/usr/local/cuda-10.0/include/nvToolsExt.h" "$(@D)/cuda/include/nvToolsExt.h" && cp "/usr/local/cuda-10.0/include/nvToolsExtCuda.h" "$(@D)/cuda/include/nvToolsExtCuda.h" && cp "/usr/local/cuda-10.0/include/nvToolsExtCudaRt.h" "$(@D)/cuda/include/nvToolsExtCudaRt.h" && cp "/usr/local/cuda-10.0/include/nvToolsExtMeta.h" "$(@D)/cuda/include/nvToolsExtMeta.h" && cp "/usr/local/cuda-10.0/include/nvToolsExtSync.h" "$(@D)/cuda/include/nvToolsExtSync.h" && cp "/usr/local/cuda-10.0/include/nvblas.h" "$(@D)/cuda/include/nvblas.h" && cp "/usr/local/cuda-10.0/include/nvfunctional" "$(@D)/cuda/include/nvfunctional" && cp "/usr/local/cuda-10.0/include/nvgraph.h" "$(@D)/cuda/include/nvgraph.h" && cp "/usr/local/cuda-10.0/include/nvml.h" "$(@D)/cuda/include/nvml.h" && cp "/usr/local/cuda-10.0/include/nvrtc.h" "$(@D)/cuda/include/nvrtc.h" && cp "/usr/local/cuda-10.0/include/sm_20_atomic_functions.h" "$(@D)/cuda/include/sm_20_atomic_functions.h" && cp "/usr/local/cuda-10.0/include/sm_20_atomic_functions.hpp" "$(@D)/cuda/include/sm_20_atomic_functions.hpp" && cp "/usr/local/cuda-10.0/include/sm_20_intrinsics.h" "$(@D)/cuda/include/sm_20_intrinsics.h" && cp "/usr/local/cuda-10.0/include/sm_20_intrinsics.hpp" "$(@D)/cuda/include/sm_20_intrinsics.hpp" && cp "/usr/local/cuda-10.0/include/sm_30_intrinsics.h" "$(@D)/cuda/include/sm_30_intrinsics.h" && cp "/usr/local/cuda-10.0/include/sm_30_intrinsics.hpp" "$(@D)/cuda/include/sm_30_intrinsics.hpp" && cp "/usr/local/cuda-10.0/include/sm_32_atomic_functions.h" "$(@D)/cuda/include/sm_32_atomic_functions.h" && cp "/usr/local/cuda-10.0/include/sm_32_atomic_functions.hpp" "$(@D)/cuda/include/sm_32_atomic_functions.hpp" && cp "/usr/local/cuda-10.0/include/sm_32_intrinsics.h" "$(@D)/cuda/include/sm_32_intrinsics.h" && cp "/usr/local/cuda-10.0/include/sm_32_intrinsics.hpp" "$(@D)/cuda/include/sm_32_intrinsics.hpp" && cp "/usr/local/cuda-10.0/include/sm_35_atomic_functions.h" "$(@D)/cuda/include/sm_35_atomic_functions.h" && cp "/usr/local/cuda-10.0/include/sm_35_intrinsics.h" "$(@D)/cuda/include/sm_35_intrinsics.h" && cp "/usr/local/cuda-10.0/include/sm_60_atomic_functions.h" "$(@D)/cuda/include/sm_60_atomic_functions.h" && cp "/usr/local/cuda-10.0/include/sm_60_atomic_functions.hpp" "$(@D)/cuda/include/sm_60_atomic_functions.hpp" && cp "/usr/local/cuda-10.0/include/sm_61_intrinsics.h" "$(@D)/cuda/include/sm_61_intrinsics.h" && cp "/usr/local/cuda-10.0/include/sm_61_intrinsics.hpp" "$(@D)/cuda/include/sm_61_intrinsics.hpp" && cp "/usr/local/cuda-10.0/include/sobol_direction_vectors.h" "$(@D)/cuda/include/sobol_direction_vectors.h" && cp "/usr/local/cuda-10.0/include/surface_functions.h" "$(@D)/cuda/include/surface_functions.h" && cp "/usr/local/cuda-10.0/include/surface_functions.hpp" "$(@D)/cuda/include/surface_functions.hpp" && cp "/usr/local/cuda-10.0/include/surface_indirect_functions.h" "$(@D)/cuda/include/surface_indirect_functions.h" && cp "/usr/local/cuda-10.0/include/surface_indirect_functions.hpp" "$(@D)/cuda/include/surface_indirect_functions.hpp" && cp "/usr/local/cuda-10.0/include/surface_types.h" "$(@D)/cuda/include/surface_types.h" && cp "/usr/local/cuda-10.0/include/texture_fetch_functions.h" "$(@D)/cuda/include/texture_fetch_functions.h" && cp "/usr/local/cuda-10.0/include/texture_fetch_functions.hpp" "$(@D)/cuda/include/texture_fetch_functions.hpp" && cp "/usr/local/cuda-10.0/include/texture_indirect_functions.h" "$(@D)/cuda/include/texture_indirect_functions.h" && cp "/usr/local/cuda-10.0/include/texture_indirect_functions.hpp" "$(@D)/cuda/include/texture_indirect_functions.hpp" && cp "/usr/local/cuda-10.0/include/texture_types.h" "$(@D)/cuda/include/texture_types.h" && cp "/usr/local/cuda-10.0/include/thrust/adjacent_difference.h" "$(@D)/cuda/include/thrust/adjacent_difference.h" && cp "/usr/local/cuda-10.0/include/thrust/advance.h" "$(@D)/cuda/include/thrust/advance.h" && cp "/usr/local/cuda-10.0/include/thrust/binary_search.h" "$(@D)/cuda/include/thrust/binary_search.h" && cp "/usr/local/cuda-10.0/include/thrust/complex.h" "$(@D)/cuda/include/thrust/complex.h" && cp "/usr/local/cuda-10.0/include/thrust/copy.h" "$(@D)/cuda/include/thrust/copy.h" && cp "/usr/local/cuda-10.0/include/thrust/count.h" "$(@D)/cuda/include/thrust/count.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/adjacent_difference.inl" "$(@D)/cuda/include/thrust/detail/adjacent_difference.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/advance.inl" "$(@D)/cuda/include/thrust/detail/advance.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/allocator/allocator_traits.h" "$(@D)/cuda/include/thrust/detail/allocator/allocator_traits.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/allocator/allocator_traits.inl" "$(@D)/cuda/include/thrust/detail/allocator/allocator_traits.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/allocator/copy_construct_range.h" "$(@D)/cuda/include/thrust/detail/allocator/copy_construct_range.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/allocator/copy_construct_range.inl" "$(@D)/cuda/include/thrust/detail/allocator/copy_construct_range.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/allocator/default_construct_range.h" "$(@D)/cuda/include/thrust/detail/allocator/default_construct_range.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/allocator/default_construct_range.inl" "$(@D)/cuda/include/thrust/detail/allocator/default_construct_range.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/allocator/destroy_range.h" "$(@D)/cuda/include/thrust/detail/allocator/destroy_range.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/allocator/destroy_range.inl" "$(@D)/cuda/include/thrust/detail/allocator/destroy_range.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/allocator/fill_construct_range.h" "$(@D)/cuda/include/thrust/detail/allocator/fill_construct_range.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/allocator/fill_construct_range.inl" "$(@D)/cuda/include/thrust/detail/allocator/fill_construct_range.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/allocator/malloc_allocator.h" "$(@D)/cuda/include/thrust/detail/allocator/malloc_allocator.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/allocator/malloc_allocator.inl" "$(@D)/cuda/include/thrust/detail/allocator/malloc_allocator.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/allocator/no_throw_allocator.h" "$(@D)/cuda/include/thrust/detail/allocator/no_throw_allocator.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/allocator/tagged_allocator.h" "$(@D)/cuda/include/thrust/detail/allocator/tagged_allocator.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/allocator/tagged_allocator.inl" "$(@D)/cuda/include/thrust/detail/allocator/tagged_allocator.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/allocator/temporary_allocator.h" "$(@D)/cuda/include/thrust/detail/allocator/temporary_allocator.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/allocator/temporary_allocator.inl" "$(@D)/cuda/include/thrust/detail/allocator/temporary_allocator.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/binary_search.inl" "$(@D)/cuda/include/thrust/detail/binary_search.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/complex/arithmetic.h" "$(@D)/cuda/include/thrust/detail/complex/arithmetic.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/complex/c99math.h" "$(@D)/cuda/include/thrust/detail/complex/c99math.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/complex/catrig.h" "$(@D)/cuda/include/thrust/detail/complex/catrig.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/complex/catrigf.h" "$(@D)/cuda/include/thrust/detail/complex/catrigf.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/complex/ccosh.h" "$(@D)/cuda/include/thrust/detail/complex/ccosh.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/complex/ccoshf.h" "$(@D)/cuda/include/thrust/detail/complex/ccoshf.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/complex/cexp.h" "$(@D)/cuda/include/thrust/detail/complex/cexp.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/complex/cexpf.h" "$(@D)/cuda/include/thrust/detail/complex/cexpf.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/complex/clog.h" "$(@D)/cuda/include/thrust/detail/complex/clog.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/complex/clogf.h" "$(@D)/cuda/include/thrust/detail/complex/clogf.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/complex/complex.inl" "$(@D)/cuda/include/thrust/detail/complex/complex.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/complex/cpow.h" "$(@D)/cuda/include/thrust/detail/complex/cpow.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/complex/cpowf.h" "$(@D)/cuda/include/thrust/detail/complex/cpowf.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/complex/cproj.h" "$(@D)/cuda/include/thrust/detail/complex/cproj.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/complex/csinh.h" "$(@D)/cuda/include/thrust/detail/complex/csinh.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/complex/csinhf.h" "$(@D)/cuda/include/thrust/detail/complex/csinhf.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/complex/csqrt.h" "$(@D)/cuda/include/thrust/detail/complex/csqrt.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/complex/csqrtf.h" "$(@D)/cuda/include/thrust/detail/complex/csqrtf.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/complex/ctanh.h" "$(@D)/cuda/include/thrust/detail/complex/ctanh.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/complex/ctanhf.h" "$(@D)/cuda/include/thrust/detail/complex/ctanhf.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/complex/math_private.h" "$(@D)/cuda/include/thrust/detail/complex/math_private.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/complex/stream.h" "$(@D)/cuda/include/thrust/detail/complex/stream.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/config.h" "$(@D)/cuda/include/thrust/detail/config.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/config/compiler.h" "$(@D)/cuda/include/thrust/detail/config/compiler.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/config/compiler_fence.h" "$(@D)/cuda/include/thrust/detail/config/compiler_fence.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/config/config.h" "$(@D)/cuda/include/thrust/detail/config/config.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/config/debug.h" "$(@D)/cuda/include/thrust/detail/config/debug.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/config/device_system.h" "$(@D)/cuda/include/thrust/detail/config/device_system.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/config/exec_check_disable.h" "$(@D)/cuda/include/thrust/detail/config/exec_check_disable.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/config/forceinline.h" "$(@D)/cuda/include/thrust/detail/config/forceinline.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/config/global_workarounds.h" "$(@D)/cuda/include/thrust/detail/config/global_workarounds.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/config/host_device.h" "$(@D)/cuda/include/thrust/detail/config/host_device.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/config/host_system.h" "$(@D)/cuda/include/thrust/detail/config/host_system.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/config/simple_defines.h" "$(@D)/cuda/include/thrust/detail/config/simple_defines.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/contiguous_storage.h" "$(@D)/cuda/include/thrust/detail/contiguous_storage.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/contiguous_storage.inl" "$(@D)/cuda/include/thrust/detail/contiguous_storage.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/copy.h" "$(@D)/cuda/include/thrust/detail/copy.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/copy.inl" "$(@D)/cuda/include/thrust/detail/copy.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/copy_if.h" "$(@D)/cuda/include/thrust/detail/copy_if.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/copy_if.inl" "$(@D)/cuda/include/thrust/detail/copy_if.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/count.inl" "$(@D)/cuda/include/thrust/detail/count.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/cstdint.h" "$(@D)/cuda/include/thrust/detail/cstdint.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/device_delete.inl" "$(@D)/cuda/include/thrust/detail/device_delete.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/device_free.inl" "$(@D)/cuda/include/thrust/detail/device_free.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/device_malloc.inl" "$(@D)/cuda/include/thrust/detail/device_malloc.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/device_new.inl" "$(@D)/cuda/include/thrust/detail/device_new.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/device_ptr.inl" "$(@D)/cuda/include/thrust/detail/device_ptr.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/device_reference.inl" "$(@D)/cuda/include/thrust/detail/device_reference.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/device_vector.inl" "$(@D)/cuda/include/thrust/detail/device_vector.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/dispatch/is_trivial_copy.h" "$(@D)/cuda/include/thrust/detail/dispatch/is_trivial_copy.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/distance.inl" "$(@D)/cuda/include/thrust/detail/distance.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/equal.inl" "$(@D)/cuda/include/thrust/detail/equal.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/execute_with_allocator.h" "$(@D)/cuda/include/thrust/detail/execute_with_allocator.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/execution_policy.h" "$(@D)/cuda/include/thrust/detail/execution_policy.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/extrema.inl" "$(@D)/cuda/include/thrust/detail/extrema.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/fill.inl" "$(@D)/cuda/include/thrust/detail/fill.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/find.inl" "$(@D)/cuda/include/thrust/detail/find.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/for_each.inl" "$(@D)/cuda/include/thrust/detail/for_each.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/function.h" "$(@D)/cuda/include/thrust/detail/function.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/functional.inl" "$(@D)/cuda/include/thrust/detail/functional.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/functional/actor.h" "$(@D)/cuda/include/thrust/detail/functional/actor.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/functional/actor.inl" "$(@D)/cuda/include/thrust/detail/functional/actor.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/functional/argument.h" "$(@D)/cuda/include/thrust/detail/functional/argument.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/functional/composite.h" "$(@D)/cuda/include/thrust/detail/functional/composite.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/functional/operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/functional/operators/arithmetic_operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators/arithmetic_operators.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/functional/operators/assignment_operator.h" "$(@D)/cuda/include/thrust/detail/functional/operators/assignment_operator.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/functional/operators/bitwise_operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators/bitwise_operators.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/functional/operators/compound_assignment_operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators/compound_assignment_operators.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/functional/operators/logical_operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators/logical_operators.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/functional/operators/operator_adaptors.h" "$(@D)/cuda/include/thrust/detail/functional/operators/operator_adaptors.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/functional/operators/relational_operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators/relational_operators.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/functional/placeholder.h" "$(@D)/cuda/include/thrust/detail/functional/placeholder.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/functional/value.h" "$(@D)/cuda/include/thrust/detail/functional/value.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/gather.inl" "$(@D)/cuda/include/thrust/detail/gather.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/generate.inl" "$(@D)/cuda/include/thrust/detail/generate.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/get_iterator_value.h" "$(@D)/cuda/include/thrust/detail/get_iterator_value.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/host_vector.inl" "$(@D)/cuda/include/thrust/detail/host_vector.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/inner_product.inl" "$(@D)/cuda/include/thrust/detail/inner_product.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/integer_math.h" "$(@D)/cuda/include/thrust/detail/integer_math.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/integer_traits.h" "$(@D)/cuda/include/thrust/detail/integer_traits.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/internal_functional.h" "$(@D)/cuda/include/thrust/detail/internal_functional.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/logical.inl" "$(@D)/cuda/include/thrust/detail/logical.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/malloc_and_free.h" "$(@D)/cuda/include/thrust/detail/malloc_and_free.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/merge.inl" "$(@D)/cuda/include/thrust/detail/merge.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/minmax.h" "$(@D)/cuda/include/thrust/detail/minmax.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/mismatch.inl" "$(@D)/cuda/include/thrust/detail/mismatch.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/mpl/math.h" "$(@D)/cuda/include/thrust/detail/mpl/math.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/numeric_traits.h" "$(@D)/cuda/include/thrust/detail/numeric_traits.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/overlapped_copy.h" "$(@D)/cuda/include/thrust/detail/overlapped_copy.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/pair.inl" "$(@D)/cuda/include/thrust/detail/pair.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/partition.inl" "$(@D)/cuda/include/thrust/detail/partition.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/pointer.h" "$(@D)/cuda/include/thrust/detail/pointer.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/pointer.inl" "$(@D)/cuda/include/thrust/detail/pointer.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/range/head_flags.h" "$(@D)/cuda/include/thrust/detail/range/head_flags.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/range/tail_flags.h" "$(@D)/cuda/include/thrust/detail/range/tail_flags.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/raw_pointer_cast.h" "$(@D)/cuda/include/thrust/detail/raw_pointer_cast.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/raw_reference_cast.h" "$(@D)/cuda/include/thrust/detail/raw_reference_cast.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/reduce.inl" "$(@D)/cuda/include/thrust/detail/reduce.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/reference.h" "$(@D)/cuda/include/thrust/detail/reference.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/reference.inl" "$(@D)/cuda/include/thrust/detail/reference.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/reference_forward_declaration.h" "$(@D)/cuda/include/thrust/detail/reference_forward_declaration.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/remove.inl" "$(@D)/cuda/include/thrust/detail/remove.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/replace.inl" "$(@D)/cuda/include/thrust/detail/replace.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/reverse.inl" "$(@D)/cuda/include/thrust/detail/reverse.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/scan.inl" "$(@D)/cuda/include/thrust/detail/scan.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/scatter.inl" "$(@D)/cuda/include/thrust/detail/scatter.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/seq.h" "$(@D)/cuda/include/thrust/detail/seq.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/sequence.inl" "$(@D)/cuda/include/thrust/detail/sequence.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/set_operations.inl" "$(@D)/cuda/include/thrust/detail/set_operations.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/sort.inl" "$(@D)/cuda/include/thrust/detail/sort.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/static_assert.h" "$(@D)/cuda/include/thrust/detail/static_assert.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/static_map.h" "$(@D)/cuda/include/thrust/detail/static_map.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/swap.h" "$(@D)/cuda/include/thrust/detail/swap.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/swap.inl" "$(@D)/cuda/include/thrust/detail/swap.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/swap_ranges.inl" "$(@D)/cuda/include/thrust/detail/swap_ranges.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/tabulate.inl" "$(@D)/cuda/include/thrust/detail/tabulate.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/temporary_array.h" "$(@D)/cuda/include/thrust/detail/temporary_array.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/temporary_array.inl" "$(@D)/cuda/include/thrust/detail/temporary_array.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/temporary_buffer.h" "$(@D)/cuda/include/thrust/detail/temporary_buffer.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/transform.inl" "$(@D)/cuda/include/thrust/detail/transform.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/transform_reduce.inl" "$(@D)/cuda/include/thrust/detail/transform_reduce.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/transform_scan.inl" "$(@D)/cuda/include/thrust/detail/transform_scan.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/trivial_sequence.h" "$(@D)/cuda/include/thrust/detail/trivial_sequence.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/tuple.inl" "$(@D)/cuda/include/thrust/detail/tuple.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/tuple_meta_transform.h" "$(@D)/cuda/include/thrust/detail/tuple_meta_transform.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/tuple_transform.h" "$(@D)/cuda/include/thrust/detail/tuple_transform.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/type_traits.h" "$(@D)/cuda/include/thrust/detail/type_traits.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/type_traits/algorithm/intermediate_type_from_function_and_iterators.h" "$(@D)/cuda/include/thrust/detail/type_traits/algorithm/intermediate_type_from_function_and_iterators.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/type_traits/function_traits.h" "$(@D)/cuda/include/thrust/detail/type_traits/function_traits.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/type_traits/has_member_function.h" "$(@D)/cuda/include/thrust/detail/type_traits/has_member_function.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/type_traits/has_nested_type.h" "$(@D)/cuda/include/thrust/detail/type_traits/has_nested_type.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/type_traits/has_trivial_assign.h" "$(@D)/cuda/include/thrust/detail/type_traits/has_trivial_assign.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/type_traits/is_call_possible.h" "$(@D)/cuda/include/thrust/detail/type_traits/is_call_possible.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/type_traits/is_metafunction_defined.h" "$(@D)/cuda/include/thrust/detail/type_traits/is_metafunction_defined.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/type_traits/iterator/is_discard_iterator.h" "$(@D)/cuda/include/thrust/detail/type_traits/iterator/is_discard_iterator.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/type_traits/iterator/is_output_iterator.h" "$(@D)/cuda/include/thrust/detail/type_traits/iterator/is_output_iterator.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/type_traits/minimum_type.h" "$(@D)/cuda/include/thrust/detail/type_traits/minimum_type.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/type_traits/pointer_traits.h" "$(@D)/cuda/include/thrust/detail/type_traits/pointer_traits.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/type_traits/result_of_adaptable_function.h" "$(@D)/cuda/include/thrust/detail/type_traits/result_of_adaptable_function.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/uninitialized_copy.inl" "$(@D)/cuda/include/thrust/detail/uninitialized_copy.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/uninitialized_fill.inl" "$(@D)/cuda/include/thrust/detail/uninitialized_fill.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/unique.inl" "$(@D)/cuda/include/thrust/detail/unique.inl" && cp "/usr/local/cuda-10.0/include/thrust/detail/use_default.h" "$(@D)/cuda/include/thrust/detail/use_default.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/util/align.h" "$(@D)/cuda/include/thrust/detail/util/align.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/util/blocking.h" "$(@D)/cuda/include/thrust/detail/util/blocking.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/vector_base.h" "$(@D)/cuda/include/thrust/detail/vector_base.h" && cp "/usr/local/cuda-10.0/include/thrust/detail/vector_base.inl" "$(@D)/cuda/include/thrust/detail/vector_base.inl" && cp "/usr/local/cuda-10.0/include/thrust/device_allocator.h" "$(@D)/cuda/include/thrust/device_allocator.h" && cp "/usr/local/cuda-10.0/include/thrust/device_delete.h" "$(@D)/cuda/include/thrust/device_delete.h" && cp "/usr/local/cuda-10.0/include/thrust/device_free.h" "$(@D)/cuda/include/thrust/device_free.h" && cp "/usr/local/cuda-10.0/include/thrust/device_malloc.h" "$(@D)/cuda/include/thrust/device_malloc.h" && cp "/usr/local/cuda-10.0/include/thrust/device_malloc_allocator.h" "$(@D)/cuda/include/thrust/device_malloc_allocator.h" && cp "/usr/local/cuda-10.0/include/thrust/device_new.h" "$(@D)/cuda/include/thrust/device_new.h" && cp "/usr/local/cuda-10.0/include/thrust/device_new_allocator.h" "$(@D)/cuda/include/thrust/device_new_allocator.h" && cp "/usr/local/cuda-10.0/include/thrust/device_ptr.h" "$(@D)/cuda/include/thrust/device_ptr.h" && cp "/usr/local/cuda-10.0/include/thrust/device_reference.h" "$(@D)/cuda/include/thrust/device_reference.h" && cp "/usr/local/cuda-10.0/include/thrust/device_vector.h" "$(@D)/cuda/include/thrust/device_vector.h" && cp "/usr/local/cuda-10.0/include/thrust/distance.h" "$(@D)/cuda/include/thrust/distance.h" && cp "/usr/local/cuda-10.0/include/thrust/equal.h" "$(@D)/cuda/include/thrust/equal.h" && cp "/usr/local/cuda-10.0/include/thrust/execution_policy.h" "$(@D)/cuda/include/thrust/execution_policy.h" && cp "/usr/local/cuda-10.0/include/thrust/extrema.h" "$(@D)/cuda/include/thrust/extrema.h" && cp "/usr/local/cuda-10.0/include/thrust/fill.h" "$(@D)/cuda/include/thrust/fill.h" && cp "/usr/local/cuda-10.0/include/thrust/find.h" "$(@D)/cuda/include/thrust/find.h" && cp "/usr/local/cuda-10.0/include/thrust/for_each.h" "$(@D)/cuda/include/thrust/for_each.h" && cp "/usr/local/cuda-10.0/include/thrust/functional.h" "$(@D)/cuda/include/thrust/functional.h" && cp "/usr/local/cuda-10.0/include/thrust/gather.h" "$(@D)/cuda/include/thrust/gather.h" && cp "/usr/local/cuda-10.0/include/thrust/generate.h" "$(@D)/cuda/include/thrust/generate.h" && cp "/usr/local/cuda-10.0/include/thrust/host_vector.h" "$(@D)/cuda/include/thrust/host_vector.h" && cp "/usr/local/cuda-10.0/include/thrust/inner_product.h" "$(@D)/cuda/include/thrust/inner_product.h" && cp "/usr/local/cuda-10.0/include/thrust/iterator/constant_iterator.h" "$(@D)/cuda/include/thrust/iterator/constant_iterator.h" && cp "/usr/local/cuda-10.0/include/thrust/iterator/counting_iterator.h" "$(@D)/cuda/include/thrust/iterator/counting_iterator.h" && cp "/usr/local/cuda-10.0/include/thrust/iterator/detail/any_assign.h" "$(@D)/cuda/include/thrust/iterator/detail/any_assign.h" && cp "/usr/local/cuda-10.0/include/thrust/iterator/detail/any_system_tag.h" "$(@D)/cuda/include/thrust/iterator/detail/any_system_tag.h" && cp "/usr/local/cuda-10.0/include/thrust/iterator/detail/constant_iterator_base.h" "$(@D)/cuda/include/thrust/iterator/detail/constant_iterator_base.h" && cp "/usr/local/cuda-10.0/include/thrust/iterator/detail/counting_iterator.inl" "$(@D)/cuda/include/thrust/iterator/detail/counting_iterator.inl" && cp "/usr/local/cuda-10.0/include/thrust/iterator/detail/device_system_tag.h" "$(@D)/cuda/include/thrust/iterator/detail/device_system_tag.h" && cp "/usr/local/cuda-10.0/include/thrust/iterator/detail/discard_iterator_base.h" "$(@D)/cuda/include/thrust/iterator/detail/discard_iterator_base.h" && cp "/usr/local/cuda-10.0/include/thrust/iterator/detail/distance_from_result.h" "$(@D)/cuda/include/thrust/iterator/detail/distance_from_result.h" && cp "/usr/local/cuda-10.0/include/thrust/iterator/detail/host_system_tag.h" "$(@D)/cuda/include/thrust/iterator/detail/host_system_tag.h" && cp "/usr/local/cuda-10.0/include/thrust/iterator/detail/is_iterator_category.h" "$(@D)/cuda/include/thrust/iterator/detail/is_iterator_category.h" && cp "/usr/local/cuda-10.0/include/thrust/iterator/detail/is_trivial_iterator.h" "$(@D)/cuda/include/thrust/iterator/detail/is_trivial_iterator.h" && cp "/usr/local/cuda-10.0/include/thrust/iterator/detail/iterator_adaptor_base.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_adaptor_base.h" && cp "/usr/local/cuda-10.0/include/thrust/iterator/detail/iterator_category_to_system.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_category_to_system.h" && cp "/usr/local/cuda-10.0/include/thrust/iterator/detail/iterator_category_to_traversal.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_category_to_traversal.h" && cp "/usr/local/cuda-10.0/include/thrust/iterator/detail/iterator_category_with_system_and_traversal.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_category_with_system_and_traversal.h" && cp "/usr/local/cuda-10.0/include/thrust/iterator/detail/iterator_facade_category.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_facade_category.h" && cp "/usr/local/cuda-10.0/include/thrust/iterator/detail/iterator_traits.inl" "$(@D)/cuda/include/thrust/iterator/detail/iterator_traits.inl" && cp "/usr/local/cuda-10.0/include/thrust/iterator/detail/iterator_traversal_tags.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_traversal_tags.h" && cp "/usr/local/cuda-10.0/include/thrust/iterator/detail/join_iterator.h" "$(@D)/cuda/include/thrust/iterator/detail/join_iterator.h" && cp "/usr/local/cuda-10.0/include/thrust/iterator/detail/minimum_category.h" "$(@D)/cuda/include/thrust/iterator/detail/minimum_category.h" && cp "/usr/local/cuda-10.0/include/thrust/iterator/detail/minimum_system.h" "$(@D)/cuda/include/thrust/iterator/detail/minimum_system.h" && cp "/usr/local/cuda-10.0/include/thrust/iterator/detail/normal_iterator.h" "$(@D)/cuda/include/thrust/iterator/detail/normal_iterator.h" && cp "/usr/local/cuda-10.0/include/thrust/iterator/detail/permutation_iterator_base.h" "$(@D)/cuda/include/thrust/iterator/detail/permutation_iterator_base.h" && cp "/usr/local/cuda-10.0/include/thrust/iterator/detail/retag.h" "$(@D)/cuda/include/thrust/iterator/detail/retag.h" && cp "/usr/local/cuda-10.0/include/thrust/iterator/detail/reverse_iterator.inl" "$(@D)/cuda/include/thrust/iterator/detail/reverse_iterator.inl" && cp "/usr/local/cuda-10.0/include/thrust/iterator/detail/reverse_iterator_base.h" "$(@D)/cuda/include/thrust/iterator/detail/reverse_iterator_base.h" && cp "/usr/local/cuda-10.0/include/thrust/iterator/detail/tagged_iterator.h" "$(@D)/cuda/include/thrust/iterator/detail/tagged_iterator.h" && cp "/usr/local/cuda-10.0/include/thrust/iterator/detail/transform_iterator.inl" "$(@D)/cuda/include/thrust/iterator/detail/transform_iterator.inl" && cp "/usr/local/cuda-10.0/include/thrust/iterator/detail/transform_output_iterator.inl" "$(@D)/cuda/include/thrust/iterator/detail/transform_output_iterator.inl" && cp "/usr/local/cuda-10.0/include/thrust/iterator/detail/tuple_of_iterator_references.h" "$(@D)/cuda/include/thrust/iterator/detail/tuple_of_iterator_references.h" && cp "/usr/local/cuda-10.0/include/thrust/iterator/detail/universal_categories.h" "$(@D)/cuda/include/thrust/iterator/detail/universal_categories.h" && cp "/usr/local/cuda-10.0/include/thrust/iterator/detail/zip_iterator.inl" "$(@D)/cuda/include/thrust/iterator/detail/zip_iterator.inl" && cp "/usr/local/cuda-10.0/include/thrust/iterator/detail/zip_iterator_base.h" "$(@D)/cuda/include/thrust/iterator/detail/zip_iterator_base.h" && cp "/usr/local/cuda-10.0/include/thrust/iterator/discard_iterator.h" "$(@D)/cuda/include/thrust/iterator/discard_iterator.h" && cp "/usr/local/cuda-10.0/include/thrust/iterator/iterator_adaptor.h" "$(@D)/cuda/include/thrust/iterator/iterator_adaptor.h" && cp "/usr/local/cuda-10.0/include/thrust/iterator/iterator_categories.h" "$(@D)/cuda/include/thrust/iterator/iterator_categories.h" && cp "/usr/local/cuda-10.0/include/thrust/iterator/iterator_facade.h" "$(@D)/cuda/include/thrust/iterator/iterator_facade.h" && cp "/usr/local/cuda-10.0/include/thrust/iterator/iterator_traits.h" "$(@D)/cuda/include/thrust/iterator/iterator_traits.h" && cp "/usr/local/cuda-10.0/include/thrust/iterator/permutation_iterator.h" "$(@D)/cuda/include/thrust/iterator/permutation_iterator.h" && cp "/usr/local/cuda-10.0/include/thrust/iterator/retag.h" "$(@D)/cuda/include/thrust/iterator/retag.h" && cp "/usr/local/cuda-10.0/include/thrust/iterator/reverse_iterator.h" "$(@D)/cuda/include/thrust/iterator/reverse_iterator.h" && cp "/usr/local/cuda-10.0/include/thrust/iterator/transform_iterator.h" "$(@D)/cuda/include/thrust/iterator/transform_iterator.h" && cp "/usr/local/cuda-10.0/include/thrust/iterator/transform_output_iterator.h" "$(@D)/cuda/include/thrust/iterator/transform_output_iterator.h" && cp "/usr/local/cuda-10.0/include/thrust/iterator/zip_iterator.h" "$(@D)/cuda/include/thrust/iterator/zip_iterator.h" && cp "/usr/local/cuda-10.0/include/thrust/logical.h" "$(@D)/cuda/include/thrust/logical.h" && cp "/usr/local/cuda-10.0/include/thrust/memory.h" "$(@D)/cuda/include/thrust/memory.h" && cp "/usr/local/cuda-10.0/include/thrust/merge.h" "$(@D)/cuda/include/thrust/merge.h" && cp "/usr/local/cuda-10.0/include/thrust/mismatch.h" "$(@D)/cuda/include/thrust/mismatch.h" && cp "/usr/local/cuda-10.0/include/thrust/pair.h" "$(@D)/cuda/include/thrust/pair.h" && cp "/usr/local/cuda-10.0/include/thrust/partition.h" "$(@D)/cuda/include/thrust/partition.h" && cp "/usr/local/cuda-10.0/include/thrust/random.h" "$(@D)/cuda/include/thrust/random.h" && cp "/usr/local/cuda-10.0/include/thrust/random/detail/discard_block_engine.inl" "$(@D)/cuda/include/thrust/random/detail/discard_block_engine.inl" && cp "/usr/local/cuda-10.0/include/thrust/random/detail/linear_congruential_engine.inl" "$(@D)/cuda/include/thrust/random/detail/linear_congruential_engine.inl" && cp "/usr/local/cuda-10.0/include/thrust/random/detail/linear_congruential_engine_discard.h" "$(@D)/cuda/include/thrust/random/detail/linear_congruential_engine_discard.h" && cp "/usr/local/cuda-10.0/include/thrust/random/detail/linear_feedback_shift_engine.inl" "$(@D)/cuda/include/thrust/random/detail/linear_feedback_shift_engine.inl" && cp "/usr/local/cuda-10.0/include/thrust/random/detail/linear_feedback_shift_engine_wordmask.h" "$(@D)/cuda/include/thrust/random/detail/linear_feedback_shift_engine_wordmask.h" && cp "/usr/local/cuda-10.0/include/thrust/random/detail/mod.h" "$(@D)/cuda/include/thrust/random/detail/mod.h" && cp "/usr/local/cuda-10.0/include/thrust/random/detail/normal_distribution.inl" "$(@D)/cuda/include/thrust/random/detail/normal_distribution.inl" && cp "/usr/local/cuda-10.0/include/thrust/random/detail/normal_distribution_base.h" "$(@D)/cuda/include/thrust/random/detail/normal_distribution_base.h" && cp "/usr/local/cuda-10.0/include/thrust/random/detail/random_core_access.h" "$(@D)/cuda/include/thrust/random/detail/random_core_access.h" && cp "/usr/local/cuda-10.0/include/thrust/random/detail/subtract_with_carry_engine.inl" "$(@D)/cuda/include/thrust/random/detail/subtract_with_carry_engine.inl" && cp "/usr/local/cuda-10.0/include/thrust/random/detail/uniform_int_distribution.inl" "$(@D)/cuda/include/thrust/random/detail/uniform_int_distribution.inl" && cp "/usr/local/cuda-10.0/include/thrust/random/detail/uniform_real_distribution.inl" "$(@D)/cuda/include/thrust/random/detail/uniform_real_distribution.inl" && cp "/usr/local/cuda-10.0/include/thrust/random/detail/xor_combine_engine.inl" "$(@D)/cuda/include/thrust/random/detail/xor_combine_engine.inl" && cp "/usr/local/cuda-10.0/include/thrust/random/detail/xor_combine_engine_max.h" "$(@D)/cuda/include/thrust/random/detail/xor_combine_engine_max.h" && cp "/usr/local/cuda-10.0/include/thrust/random/discard_block_engine.h" "$(@D)/cuda/include/thrust/random/discard_block_engine.h" && cp "/usr/local/cuda-10.0/include/thrust/random/linear_congruential_engine.h" "$(@D)/cuda/include/thrust/random/linear_congruential_engine.h" && cp "/usr/local/cuda-10.0/include/thrust/random/linear_feedback_shift_engine.h" "$(@D)/cuda/include/thrust/random/linear_feedback_shift_engine.h" && cp "/usr/local/cuda-10.0/include/thrust/random/normal_distribution.h" "$(@D)/cuda/include/thrust/random/normal_distribution.h" && cp "/usr/local/cuda-10.0/include/thrust/random/subtract_with_carry_engine.h" "$(@D)/cuda/include/thrust/random/subtract_with_carry_engine.h" && cp "/usr/local/cuda-10.0/include/thrust/random/uniform_int_distribution.h" "$(@D)/cuda/include/thrust/random/uniform_int_distribution.h" && cp "/usr/local/cuda-10.0/include/thrust/random/uniform_real_distribution.h" "$(@D)/cuda/include/thrust/random/uniform_real_distribution.h" && cp "/usr/local/cuda-10.0/include/thrust/random/xor_combine_engine.h" "$(@D)/cuda/include/thrust/random/xor_combine_engine.h" && cp "/usr/local/cuda-10.0/include/thrust/reduce.h" "$(@D)/cuda/include/thrust/reduce.h" && cp "/usr/local/cuda-10.0/include/thrust/remove.h" "$(@D)/cuda/include/thrust/remove.h" && cp "/usr/local/cuda-10.0/include/thrust/replace.h" "$(@D)/cuda/include/thrust/replace.h" && cp "/usr/local/cuda-10.0/include/thrust/reverse.h" "$(@D)/cuda/include/thrust/reverse.h" && cp "/usr/local/cuda-10.0/include/thrust/scan.h" "$(@D)/cuda/include/thrust/scan.h" && cp "/usr/local/cuda-10.0/include/thrust/scatter.h" "$(@D)/cuda/include/thrust/scatter.h" && cp "/usr/local/cuda-10.0/include/thrust/sequence.h" "$(@D)/cuda/include/thrust/sequence.h" && cp "/usr/local/cuda-10.0/include/thrust/set_operations.h" "$(@D)/cuda/include/thrust/set_operations.h" && cp "/usr/local/cuda-10.0/include/thrust/sort.h" "$(@D)/cuda/include/thrust/sort.h" && cp "/usr/local/cuda-10.0/include/thrust/swap.h" "$(@D)/cuda/include/thrust/swap.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/cpp/detail/adjacent_difference.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/assign_value.h" "$(@D)/cuda/include/thrust/system/cpp/detail/assign_value.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/binary_search.h" "$(@D)/cuda/include/thrust/system/cpp/detail/binary_search.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/copy.h" "$(@D)/cuda/include/thrust/system/cpp/detail/copy.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/copy_if.h" "$(@D)/cuda/include/thrust/system/cpp/detail/copy_if.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/count.h" "$(@D)/cuda/include/thrust/system/cpp/detail/count.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/equal.h" "$(@D)/cuda/include/thrust/system/cpp/detail/equal.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/execution_policy.h" "$(@D)/cuda/include/thrust/system/cpp/detail/execution_policy.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/extrema.h" "$(@D)/cuda/include/thrust/system/cpp/detail/extrema.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/fill.h" "$(@D)/cuda/include/thrust/system/cpp/detail/fill.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/find.h" "$(@D)/cuda/include/thrust/system/cpp/detail/find.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/for_each.h" "$(@D)/cuda/include/thrust/system/cpp/detail/for_each.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/gather.h" "$(@D)/cuda/include/thrust/system/cpp/detail/gather.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/generate.h" "$(@D)/cuda/include/thrust/system/cpp/detail/generate.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/get_value.h" "$(@D)/cuda/include/thrust/system/cpp/detail/get_value.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/inner_product.h" "$(@D)/cuda/include/thrust/system/cpp/detail/inner_product.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/iter_swap.h" "$(@D)/cuda/include/thrust/system/cpp/detail/iter_swap.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/logical.h" "$(@D)/cuda/include/thrust/system/cpp/detail/logical.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/cpp/detail/malloc_and_free.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/memory.inl" "$(@D)/cuda/include/thrust/system/cpp/detail/memory.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/merge.h" "$(@D)/cuda/include/thrust/system/cpp/detail/merge.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/mismatch.h" "$(@D)/cuda/include/thrust/system/cpp/detail/mismatch.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/par.h" "$(@D)/cuda/include/thrust/system/cpp/detail/par.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/partition.h" "$(@D)/cuda/include/thrust/system/cpp/detail/partition.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/reduce.h" "$(@D)/cuda/include/thrust/system/cpp/detail/reduce.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/cpp/detail/reduce_by_key.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/remove.h" "$(@D)/cuda/include/thrust/system/cpp/detail/remove.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/replace.h" "$(@D)/cuda/include/thrust/system/cpp/detail/replace.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/reverse.h" "$(@D)/cuda/include/thrust/system/cpp/detail/reverse.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/scan.h" "$(@D)/cuda/include/thrust/system/cpp/detail/scan.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/scan_by_key.h" "$(@D)/cuda/include/thrust/system/cpp/detail/scan_by_key.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/scatter.h" "$(@D)/cuda/include/thrust/system/cpp/detail/scatter.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/sequence.h" "$(@D)/cuda/include/thrust/system/cpp/detail/sequence.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/set_operations.h" "$(@D)/cuda/include/thrust/system/cpp/detail/set_operations.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/sort.h" "$(@D)/cuda/include/thrust/system/cpp/detail/sort.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/swap_ranges.h" "$(@D)/cuda/include/thrust/system/cpp/detail/swap_ranges.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/tabulate.h" "$(@D)/cuda/include/thrust/system/cpp/detail/tabulate.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/cpp/detail/temporary_buffer.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/transform.h" "$(@D)/cuda/include/thrust/system/cpp/detail/transform.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/transform_reduce.h" "$(@D)/cuda/include/thrust/system/cpp/detail/transform_reduce.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/transform_scan.h" "$(@D)/cuda/include/thrust/system/cpp/detail/transform_scan.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/cpp/detail/uninitialized_copy.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/cpp/detail/uninitialized_fill.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/unique.h" "$(@D)/cuda/include/thrust/system/cpp/detail/unique.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/unique_by_key.h" "$(@D)/cuda/include/thrust/system/cpp/detail/unique_by_key.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/vector.inl" "$(@D)/cuda/include/thrust/system/cpp/detail/vector.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/execution_policy.h" "$(@D)/cuda/include/thrust/system/cpp/execution_policy.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/memory.h" "$(@D)/cuda/include/thrust/system/cpp/memory.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cpp/vector.h" "$(@D)/cuda/include/thrust/system/cpp/vector.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/config.h" "$(@D)/cuda/include/thrust/system/cuda/config.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/cuda/detail/adjacent_difference.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/assign_value.h" "$(@D)/cuda/include/thrust/system/cuda/detail/assign_value.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/binary_search.h" "$(@D)/cuda/include/thrust/system/cuda/detail/binary_search.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/copy.h" "$(@D)/cuda/include/thrust/system/cuda/detail/copy.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/copy_if.h" "$(@D)/cuda/include/thrust/system/cuda/detail/copy_if.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/core/agent_launcher.h" "$(@D)/cuda/include/thrust/system/cuda/detail/core/agent_launcher.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/core/alignment.h" "$(@D)/cuda/include/thrust/system/cuda/detail/core/alignment.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/core/triple_chevron_launch.h" "$(@D)/cuda/include/thrust/system/cuda/detail/core/triple_chevron_launch.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/core/util.h" "$(@D)/cuda/include/thrust/system/cuda/detail/core/util.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/count.h" "$(@D)/cuda/include/thrust/system/cuda/detail/count.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cross_system.h" "$(@D)/cuda/include/thrust/system/cuda/detail/cross_system.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/agent_histogram.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_histogram.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/agent_radix_sort_downsweep.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_radix_sort_downsweep.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/agent_radix_sort_upsweep.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_radix_sort_upsweep.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/agent_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_reduce.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/agent_reduce_by_key.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_reduce_by_key.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/agent_rle.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_rle.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/agent_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_scan.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/agent_segment_fixup.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_segment_fixup.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/agent_select_if.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_select_if.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/agent_spmv_csrt.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_spmv_csrt.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/agent_spmv_orig.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_spmv_orig.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/agent_spmv_row_based.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_spmv_row_based.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/single_pass_scan_operators.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/single_pass_scan_operators.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_adjacent_difference.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_adjacent_difference.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_discontinuity.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_discontinuity.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_exchange.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_exchange.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_histogram.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_histogram.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_load.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_load.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_radix_rank.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_radix_rank.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_radix_sort.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_radix_sort.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_raking_layout.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_raking_layout.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_reduce.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_scan.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_shuffle.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_shuffle.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_store.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_store.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/specializations/block_histogram_atomic.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_histogram_atomic.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/specializations/block_histogram_sort.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_histogram_sort.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_raking.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_raking.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_raking_commutative_only.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_raking_commutative_only.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_warp_reductions.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_warp_reductions.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_raking.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_raking.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans2.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans2.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans3.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans3.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/cub.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/cub.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/device_histogram.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_histogram.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/device_partition.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_partition.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/device_radix_sort.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_radix_sort.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/device_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_reduce.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/device_run_length_encode.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_run_length_encode.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/device_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_scan.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/device_segmented_radix_sort.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_segmented_radix_sort.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/device_segmented_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_segmented_reduce.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/device_select.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_select.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/device_spmv.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_spmv.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_histogram.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_histogram.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_radix_sort.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_radix_sort.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_reduce.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_reduce_by_key.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_reduce_by_key.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_rle.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_rle.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_scan.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_select_if.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_select_if.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_spmv_csrt.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_spmv_csrt.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_spmv_orig.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_spmv_orig.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_spmv_row_based.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_spmv_row_based.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/grid/grid_barrier.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/grid/grid_barrier.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/grid/grid_even_share.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/grid/grid_even_share.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/grid/grid_mapping.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/grid/grid_mapping.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/grid/grid_queue.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/grid/grid_queue.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/host/mutex.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/host/mutex.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/iterator/arg_index_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/arg_index_input_iterator.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/iterator/cache_modified_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/cache_modified_input_iterator.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/iterator/cache_modified_output_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/cache_modified_output_iterator.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/iterator/constant_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/constant_input_iterator.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/iterator/counting_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/counting_input_iterator.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/iterator/discard_output_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/discard_output_iterator.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/iterator/tex_obj_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/tex_obj_input_iterator.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/iterator/tex_ref_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/tex_ref_input_iterator.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/iterator/transform_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/transform_input_iterator.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/thread/thread_load.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_load.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/thread/thread_operators.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_operators.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/thread/thread_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_reduce.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/thread/thread_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_scan.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/thread/thread_search.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_search.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/thread/thread_store.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_store.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/util_allocator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_allocator.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/util_arch.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_arch.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/util_debug.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_debug.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/util_device.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_device.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/util_macro.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_macro.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/util_namespace.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_namespace.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/util_ptx.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_ptx.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/util_type.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_type.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/warp/specializations/warp_reduce_shfl.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/specializations/warp_reduce_shfl.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/warp/specializations/warp_reduce_smem.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/specializations/warp_reduce_smem.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/warp/specializations/warp_scan_shfl.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/specializations/warp_scan_shfl.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/warp/specializations/warp_scan_smem.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/specializations/warp_scan_smem.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/warp/warp_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/warp_reduce.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/warp/warp_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/warp_scan.cuh" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/equal.h" "$(@D)/cuda/include/thrust/system/cuda/detail/equal.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/error.inl" "$(@D)/cuda/include/thrust/system/cuda/detail/error.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/execution_policy.h" "$(@D)/cuda/include/thrust/system/cuda/detail/execution_policy.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/extrema.h" "$(@D)/cuda/include/thrust/system/cuda/detail/extrema.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/fill.h" "$(@D)/cuda/include/thrust/system/cuda/detail/fill.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/find.h" "$(@D)/cuda/include/thrust/system/cuda/detail/find.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/for_each.h" "$(@D)/cuda/include/thrust/system/cuda/detail/for_each.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/gather.h" "$(@D)/cuda/include/thrust/system/cuda/detail/gather.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/generate.h" "$(@D)/cuda/include/thrust/system/cuda/detail/generate.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/get_value.h" "$(@D)/cuda/include/thrust/system/cuda/detail/get_value.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/guarded_cuda_runtime_api.h" "$(@D)/cuda/include/thrust/system/cuda/detail/guarded_cuda_runtime_api.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/guarded_driver_types.h" "$(@D)/cuda/include/thrust/system/cuda/detail/guarded_driver_types.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/inner_product.h" "$(@D)/cuda/include/thrust/system/cuda/detail/inner_product.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/internal/copy_cross_system.h" "$(@D)/cuda/include/thrust/system/cuda/detail/internal/copy_cross_system.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/internal/copy_device_to_device.h" "$(@D)/cuda/include/thrust/system/cuda/detail/internal/copy_device_to_device.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/iter_swap.h" "$(@D)/cuda/include/thrust/system/cuda/detail/iter_swap.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/logical.h" "$(@D)/cuda/include/thrust/system/cuda/detail/logical.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/cuda/detail/malloc_and_free.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/memory.inl" "$(@D)/cuda/include/thrust/system/cuda/detail/memory.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/memory_buffer.h" "$(@D)/cuda/include/thrust/system/cuda/detail/memory_buffer.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/merge.h" "$(@D)/cuda/include/thrust/system/cuda/detail/merge.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/mismatch.h" "$(@D)/cuda/include/thrust/system/cuda/detail/mismatch.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/par.h" "$(@D)/cuda/include/thrust/system/cuda/detail/par.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/par_to_seq.h" "$(@D)/cuda/include/thrust/system/cuda/detail/par_to_seq.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/parallel_for.h" "$(@D)/cuda/include/thrust/system/cuda/detail/parallel_for.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/partition.h" "$(@D)/cuda/include/thrust/system/cuda/detail/partition.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/reduce.h" "$(@D)/cuda/include/thrust/system/cuda/detail/reduce.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/cuda/detail/reduce_by_key.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/remove.h" "$(@D)/cuda/include/thrust/system/cuda/detail/remove.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/replace.h" "$(@D)/cuda/include/thrust/system/cuda/detail/replace.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/reverse.h" "$(@D)/cuda/include/thrust/system/cuda/detail/reverse.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/scan.h" "$(@D)/cuda/include/thrust/system/cuda/detail/scan.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/scan_by_key.h" "$(@D)/cuda/include/thrust/system/cuda/detail/scan_by_key.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/scatter.h" "$(@D)/cuda/include/thrust/system/cuda/detail/scatter.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/sequence.h" "$(@D)/cuda/include/thrust/system/cuda/detail/sequence.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/set_operations.h" "$(@D)/cuda/include/thrust/system/cuda/detail/set_operations.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/sort.h" "$(@D)/cuda/include/thrust/system/cuda/detail/sort.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/swap_ranges.h" "$(@D)/cuda/include/thrust/system/cuda/detail/swap_ranges.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/tabulate.h" "$(@D)/cuda/include/thrust/system/cuda/detail/tabulate.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/cuda/detail/temporary_buffer.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/terminate.h" "$(@D)/cuda/include/thrust/system/cuda/detail/terminate.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/transform.h" "$(@D)/cuda/include/thrust/system/cuda/detail/transform.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/transform_reduce.h" "$(@D)/cuda/include/thrust/system/cuda/detail/transform_reduce.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/transform_scan.h" "$(@D)/cuda/include/thrust/system/cuda/detail/transform_scan.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/cuda/detail/uninitialized_copy.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/cuda/detail/uninitialized_fill.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/unique.h" "$(@D)/cuda/include/thrust/system/cuda/detail/unique.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/unique_by_key.h" "$(@D)/cuda/include/thrust/system/cuda/detail/unique_by_key.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/util.h" "$(@D)/cuda/include/thrust/system/cuda/detail/util.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/vector.inl" "$(@D)/cuda/include/thrust/system/cuda/detail/vector.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/error.h" "$(@D)/cuda/include/thrust/system/cuda/error.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/execution_policy.h" "$(@D)/cuda/include/thrust/system/cuda/execution_policy.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/experimental/pinned_allocator.h" "$(@D)/cuda/include/thrust/system/cuda/experimental/pinned_allocator.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/memory.h" "$(@D)/cuda/include/thrust/system/cuda/memory.h" && cp "/usr/local/cuda-10.0/include/thrust/system/cuda/vector.h" "$(@D)/cuda/include/thrust/system/cuda/vector.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/adl/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/detail/adl/adjacent_difference.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/adl/assign_value.h" "$(@D)/cuda/include/thrust/system/detail/adl/assign_value.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/adl/binary_search.h" "$(@D)/cuda/include/thrust/system/detail/adl/binary_search.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/adl/copy.h" "$(@D)/cuda/include/thrust/system/detail/adl/copy.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/adl/copy_if.h" "$(@D)/cuda/include/thrust/system/detail/adl/copy_if.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/adl/count.h" "$(@D)/cuda/include/thrust/system/detail/adl/count.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/adl/equal.h" "$(@D)/cuda/include/thrust/system/detail/adl/equal.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/adl/extrema.h" "$(@D)/cuda/include/thrust/system/detail/adl/extrema.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/adl/fill.h" "$(@D)/cuda/include/thrust/system/detail/adl/fill.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/adl/find.h" "$(@D)/cuda/include/thrust/system/detail/adl/find.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/adl/for_each.h" "$(@D)/cuda/include/thrust/system/detail/adl/for_each.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/adl/gather.h" "$(@D)/cuda/include/thrust/system/detail/adl/gather.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/adl/generate.h" "$(@D)/cuda/include/thrust/system/detail/adl/generate.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/adl/get_value.h" "$(@D)/cuda/include/thrust/system/detail/adl/get_value.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/adl/inner_product.h" "$(@D)/cuda/include/thrust/system/detail/adl/inner_product.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/adl/iter_swap.h" "$(@D)/cuda/include/thrust/system/detail/adl/iter_swap.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/adl/logical.h" "$(@D)/cuda/include/thrust/system/detail/adl/logical.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/adl/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/detail/adl/malloc_and_free.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/adl/merge.h" "$(@D)/cuda/include/thrust/system/detail/adl/merge.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/adl/mismatch.h" "$(@D)/cuda/include/thrust/system/detail/adl/mismatch.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/adl/partition.h" "$(@D)/cuda/include/thrust/system/detail/adl/partition.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/adl/reduce.h" "$(@D)/cuda/include/thrust/system/detail/adl/reduce.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/adl/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/detail/adl/reduce_by_key.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/adl/remove.h" "$(@D)/cuda/include/thrust/system/detail/adl/remove.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/adl/replace.h" "$(@D)/cuda/include/thrust/system/detail/adl/replace.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/adl/reverse.h" "$(@D)/cuda/include/thrust/system/detail/adl/reverse.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/adl/scan.h" "$(@D)/cuda/include/thrust/system/detail/adl/scan.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/adl/scan_by_key.h" "$(@D)/cuda/include/thrust/system/detail/adl/scan_by_key.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/adl/scatter.h" "$(@D)/cuda/include/thrust/system/detail/adl/scatter.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/adl/sequence.h" "$(@D)/cuda/include/thrust/system/detail/adl/sequence.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/adl/set_operations.h" "$(@D)/cuda/include/thrust/system/detail/adl/set_operations.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/adl/sort.h" "$(@D)/cuda/include/thrust/system/detail/adl/sort.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/adl/swap_ranges.h" "$(@D)/cuda/include/thrust/system/detail/adl/swap_ranges.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/adl/tabulate.h" "$(@D)/cuda/include/thrust/system/detail/adl/tabulate.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/adl/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/detail/adl/temporary_buffer.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/adl/transform.h" "$(@D)/cuda/include/thrust/system/detail/adl/transform.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/adl/transform_reduce.h" "$(@D)/cuda/include/thrust/system/detail/adl/transform_reduce.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/adl/transform_scan.h" "$(@D)/cuda/include/thrust/system/detail/adl/transform_scan.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/adl/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/detail/adl/uninitialized_copy.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/adl/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/detail/adl/uninitialized_fill.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/adl/unique.h" "$(@D)/cuda/include/thrust/system/detail/adl/unique.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/adl/unique_by_key.h" "$(@D)/cuda/include/thrust/system/detail/adl/unique_by_key.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/bad_alloc.h" "$(@D)/cuda/include/thrust/system/detail/bad_alloc.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/errno.h" "$(@D)/cuda/include/thrust/system/detail/errno.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/error_category.inl" "$(@D)/cuda/include/thrust/system/detail/error_category.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/error_code.inl" "$(@D)/cuda/include/thrust/system/detail/error_code.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/error_condition.inl" "$(@D)/cuda/include/thrust/system/detail/error_condition.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/detail/generic/adjacent_difference.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/adjacent_difference.inl" "$(@D)/cuda/include/thrust/system/detail/generic/adjacent_difference.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/advance.h" "$(@D)/cuda/include/thrust/system/detail/generic/advance.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/advance.inl" "$(@D)/cuda/include/thrust/system/detail/generic/advance.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/binary_search.h" "$(@D)/cuda/include/thrust/system/detail/generic/binary_search.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/binary_search.inl" "$(@D)/cuda/include/thrust/system/detail/generic/binary_search.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/copy.h" "$(@D)/cuda/include/thrust/system/detail/generic/copy.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/copy.inl" "$(@D)/cuda/include/thrust/system/detail/generic/copy.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/copy_if.h" "$(@D)/cuda/include/thrust/system/detail/generic/copy_if.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/copy_if.inl" "$(@D)/cuda/include/thrust/system/detail/generic/copy_if.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/count.h" "$(@D)/cuda/include/thrust/system/detail/generic/count.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/count.inl" "$(@D)/cuda/include/thrust/system/detail/generic/count.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/distance.h" "$(@D)/cuda/include/thrust/system/detail/generic/distance.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/distance.inl" "$(@D)/cuda/include/thrust/system/detail/generic/distance.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/equal.h" "$(@D)/cuda/include/thrust/system/detail/generic/equal.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/equal.inl" "$(@D)/cuda/include/thrust/system/detail/generic/equal.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/extrema.h" "$(@D)/cuda/include/thrust/system/detail/generic/extrema.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/extrema.inl" "$(@D)/cuda/include/thrust/system/detail/generic/extrema.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/fill.h" "$(@D)/cuda/include/thrust/system/detail/generic/fill.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/find.h" "$(@D)/cuda/include/thrust/system/detail/generic/find.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/find.inl" "$(@D)/cuda/include/thrust/system/detail/generic/find.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/for_each.h" "$(@D)/cuda/include/thrust/system/detail/generic/for_each.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/gather.h" "$(@D)/cuda/include/thrust/system/detail/generic/gather.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/gather.inl" "$(@D)/cuda/include/thrust/system/detail/generic/gather.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/generate.h" "$(@D)/cuda/include/thrust/system/detail/generic/generate.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/generate.inl" "$(@D)/cuda/include/thrust/system/detail/generic/generate.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/inner_product.h" "$(@D)/cuda/include/thrust/system/detail/generic/inner_product.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/inner_product.inl" "$(@D)/cuda/include/thrust/system/detail/generic/inner_product.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/logical.h" "$(@D)/cuda/include/thrust/system/detail/generic/logical.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/memory.h" "$(@D)/cuda/include/thrust/system/detail/generic/memory.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/memory.inl" "$(@D)/cuda/include/thrust/system/detail/generic/memory.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/merge.h" "$(@D)/cuda/include/thrust/system/detail/generic/merge.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/merge.inl" "$(@D)/cuda/include/thrust/system/detail/generic/merge.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/mismatch.h" "$(@D)/cuda/include/thrust/system/detail/generic/mismatch.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/mismatch.inl" "$(@D)/cuda/include/thrust/system/detail/generic/mismatch.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/partition.h" "$(@D)/cuda/include/thrust/system/detail/generic/partition.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/partition.inl" "$(@D)/cuda/include/thrust/system/detail/generic/partition.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/reduce.h" "$(@D)/cuda/include/thrust/system/detail/generic/reduce.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/reduce.inl" "$(@D)/cuda/include/thrust/system/detail/generic/reduce.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/detail/generic/reduce_by_key.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/reduce_by_key.inl" "$(@D)/cuda/include/thrust/system/detail/generic/reduce_by_key.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/remove.h" "$(@D)/cuda/include/thrust/system/detail/generic/remove.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/remove.inl" "$(@D)/cuda/include/thrust/system/detail/generic/remove.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/replace.h" "$(@D)/cuda/include/thrust/system/detail/generic/replace.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/replace.inl" "$(@D)/cuda/include/thrust/system/detail/generic/replace.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/reverse.h" "$(@D)/cuda/include/thrust/system/detail/generic/reverse.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/reverse.inl" "$(@D)/cuda/include/thrust/system/detail/generic/reverse.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/scalar/binary_search.h" "$(@D)/cuda/include/thrust/system/detail/generic/scalar/binary_search.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/scalar/binary_search.inl" "$(@D)/cuda/include/thrust/system/detail/generic/scalar/binary_search.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/scan.h" "$(@D)/cuda/include/thrust/system/detail/generic/scan.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/scan.inl" "$(@D)/cuda/include/thrust/system/detail/generic/scan.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/scan_by_key.h" "$(@D)/cuda/include/thrust/system/detail/generic/scan_by_key.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/scan_by_key.inl" "$(@D)/cuda/include/thrust/system/detail/generic/scan_by_key.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/scatter.h" "$(@D)/cuda/include/thrust/system/detail/generic/scatter.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/scatter.inl" "$(@D)/cuda/include/thrust/system/detail/generic/scatter.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/select_system.h" "$(@D)/cuda/include/thrust/system/detail/generic/select_system.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/sequence.h" "$(@D)/cuda/include/thrust/system/detail/generic/sequence.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/sequence.inl" "$(@D)/cuda/include/thrust/system/detail/generic/sequence.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/set_operations.h" "$(@D)/cuda/include/thrust/system/detail/generic/set_operations.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/set_operations.inl" "$(@D)/cuda/include/thrust/system/detail/generic/set_operations.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/sort.h" "$(@D)/cuda/include/thrust/system/detail/generic/sort.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/sort.inl" "$(@D)/cuda/include/thrust/system/detail/generic/sort.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/swap_ranges.h" "$(@D)/cuda/include/thrust/system/detail/generic/swap_ranges.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/swap_ranges.inl" "$(@D)/cuda/include/thrust/system/detail/generic/swap_ranges.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/tabulate.h" "$(@D)/cuda/include/thrust/system/detail/generic/tabulate.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/tabulate.inl" "$(@D)/cuda/include/thrust/system/detail/generic/tabulate.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/tag.h" "$(@D)/cuda/include/thrust/system/detail/generic/tag.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/detail/generic/temporary_buffer.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/temporary_buffer.inl" "$(@D)/cuda/include/thrust/system/detail/generic/temporary_buffer.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/transform.h" "$(@D)/cuda/include/thrust/system/detail/generic/transform.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/transform.inl" "$(@D)/cuda/include/thrust/system/detail/generic/transform.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/transform_reduce.h" "$(@D)/cuda/include/thrust/system/detail/generic/transform_reduce.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/transform_reduce.inl" "$(@D)/cuda/include/thrust/system/detail/generic/transform_reduce.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/transform_scan.h" "$(@D)/cuda/include/thrust/system/detail/generic/transform_scan.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/transform_scan.inl" "$(@D)/cuda/include/thrust/system/detail/generic/transform_scan.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/type_traits.h" "$(@D)/cuda/include/thrust/system/detail/generic/type_traits.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/detail/generic/uninitialized_copy.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/uninitialized_copy.inl" "$(@D)/cuda/include/thrust/system/detail/generic/uninitialized_copy.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/detail/generic/uninitialized_fill.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/uninitialized_fill.inl" "$(@D)/cuda/include/thrust/system/detail/generic/uninitialized_fill.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/unique.h" "$(@D)/cuda/include/thrust/system/detail/generic/unique.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/unique.inl" "$(@D)/cuda/include/thrust/system/detail/generic/unique.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/unique_by_key.h" "$(@D)/cuda/include/thrust/system/detail/generic/unique_by_key.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/generic/unique_by_key.inl" "$(@D)/cuda/include/thrust/system/detail/generic/unique_by_key.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/internal/decompose.h" "$(@D)/cuda/include/thrust/system/detail/internal/decompose.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/detail/sequential/adjacent_difference.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/assign_value.h" "$(@D)/cuda/include/thrust/system/detail/sequential/assign_value.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/binary_search.h" "$(@D)/cuda/include/thrust/system/detail/sequential/binary_search.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/copy.h" "$(@D)/cuda/include/thrust/system/detail/sequential/copy.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/copy.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/copy.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/copy_backward.h" "$(@D)/cuda/include/thrust/system/detail/sequential/copy_backward.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/copy_if.h" "$(@D)/cuda/include/thrust/system/detail/sequential/copy_if.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/count.h" "$(@D)/cuda/include/thrust/system/detail/sequential/count.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/equal.h" "$(@D)/cuda/include/thrust/system/detail/sequential/equal.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/execution_policy.h" "$(@D)/cuda/include/thrust/system/detail/sequential/execution_policy.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/extrema.h" "$(@D)/cuda/include/thrust/system/detail/sequential/extrema.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/fill.h" "$(@D)/cuda/include/thrust/system/detail/sequential/fill.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/find.h" "$(@D)/cuda/include/thrust/system/detail/sequential/find.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/for_each.h" "$(@D)/cuda/include/thrust/system/detail/sequential/for_each.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/gather.h" "$(@D)/cuda/include/thrust/system/detail/sequential/gather.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/general_copy.h" "$(@D)/cuda/include/thrust/system/detail/sequential/general_copy.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/generate.h" "$(@D)/cuda/include/thrust/system/detail/sequential/generate.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/get_value.h" "$(@D)/cuda/include/thrust/system/detail/sequential/get_value.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/inner_product.h" "$(@D)/cuda/include/thrust/system/detail/sequential/inner_product.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/insertion_sort.h" "$(@D)/cuda/include/thrust/system/detail/sequential/insertion_sort.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/iter_swap.h" "$(@D)/cuda/include/thrust/system/detail/sequential/iter_swap.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/logical.h" "$(@D)/cuda/include/thrust/system/detail/sequential/logical.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/detail/sequential/malloc_and_free.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/merge.h" "$(@D)/cuda/include/thrust/system/detail/sequential/merge.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/merge.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/merge.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/mismatch.h" "$(@D)/cuda/include/thrust/system/detail/sequential/mismatch.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/partition.h" "$(@D)/cuda/include/thrust/system/detail/sequential/partition.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/reduce.h" "$(@D)/cuda/include/thrust/system/detail/sequential/reduce.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/detail/sequential/reduce_by_key.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/remove.h" "$(@D)/cuda/include/thrust/system/detail/sequential/remove.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/replace.h" "$(@D)/cuda/include/thrust/system/detail/sequential/replace.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/reverse.h" "$(@D)/cuda/include/thrust/system/detail/sequential/reverse.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/scan.h" "$(@D)/cuda/include/thrust/system/detail/sequential/scan.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/scan_by_key.h" "$(@D)/cuda/include/thrust/system/detail/sequential/scan_by_key.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/scatter.h" "$(@D)/cuda/include/thrust/system/detail/sequential/scatter.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/sequence.h" "$(@D)/cuda/include/thrust/system/detail/sequential/sequence.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/set_operations.h" "$(@D)/cuda/include/thrust/system/detail/sequential/set_operations.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/sort.h" "$(@D)/cuda/include/thrust/system/detail/sequential/sort.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/sort.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/sort.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/stable_merge_sort.h" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_merge_sort.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/stable_merge_sort.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_merge_sort.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/stable_primitive_sort.h" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_primitive_sort.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/stable_primitive_sort.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_primitive_sort.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/stable_radix_sort.h" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_radix_sort.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/stable_radix_sort.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_radix_sort.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/swap_ranges.h" "$(@D)/cuda/include/thrust/system/detail/sequential/swap_ranges.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/tabulate.h" "$(@D)/cuda/include/thrust/system/detail/sequential/tabulate.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/detail/sequential/temporary_buffer.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/transform.h" "$(@D)/cuda/include/thrust/system/detail/sequential/transform.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/transform_reduce.h" "$(@D)/cuda/include/thrust/system/detail/sequential/transform_reduce.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/transform_scan.h" "$(@D)/cuda/include/thrust/system/detail/sequential/transform_scan.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/trivial_copy.h" "$(@D)/cuda/include/thrust/system/detail/sequential/trivial_copy.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/detail/sequential/uninitialized_copy.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/detail/sequential/uninitialized_fill.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/unique.h" "$(@D)/cuda/include/thrust/system/detail/sequential/unique.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/unique_by_key.h" "$(@D)/cuda/include/thrust/system/detail/sequential/unique_by_key.h" && cp "/usr/local/cuda-10.0/include/thrust/system/detail/system_error.inl" "$(@D)/cuda/include/thrust/system/detail/system_error.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/error_code.h" "$(@D)/cuda/include/thrust/system/error_code.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/omp/detail/adjacent_difference.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/assign_value.h" "$(@D)/cuda/include/thrust/system/omp/detail/assign_value.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/binary_search.h" "$(@D)/cuda/include/thrust/system/omp/detail/binary_search.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/copy.h" "$(@D)/cuda/include/thrust/system/omp/detail/copy.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/copy.inl" "$(@D)/cuda/include/thrust/system/omp/detail/copy.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/copy_if.h" "$(@D)/cuda/include/thrust/system/omp/detail/copy_if.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/copy_if.inl" "$(@D)/cuda/include/thrust/system/omp/detail/copy_if.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/count.h" "$(@D)/cuda/include/thrust/system/omp/detail/count.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/default_decomposition.h" "$(@D)/cuda/include/thrust/system/omp/detail/default_decomposition.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/default_decomposition.inl" "$(@D)/cuda/include/thrust/system/omp/detail/default_decomposition.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/equal.h" "$(@D)/cuda/include/thrust/system/omp/detail/equal.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/execution_policy.h" "$(@D)/cuda/include/thrust/system/omp/detail/execution_policy.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/extrema.h" "$(@D)/cuda/include/thrust/system/omp/detail/extrema.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/fill.h" "$(@D)/cuda/include/thrust/system/omp/detail/fill.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/find.h" "$(@D)/cuda/include/thrust/system/omp/detail/find.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/for_each.h" "$(@D)/cuda/include/thrust/system/omp/detail/for_each.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/for_each.inl" "$(@D)/cuda/include/thrust/system/omp/detail/for_each.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/gather.h" "$(@D)/cuda/include/thrust/system/omp/detail/gather.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/generate.h" "$(@D)/cuda/include/thrust/system/omp/detail/generate.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/get_value.h" "$(@D)/cuda/include/thrust/system/omp/detail/get_value.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/inner_product.h" "$(@D)/cuda/include/thrust/system/omp/detail/inner_product.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/iter_swap.h" "$(@D)/cuda/include/thrust/system/omp/detail/iter_swap.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/logical.h" "$(@D)/cuda/include/thrust/system/omp/detail/logical.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/omp/detail/malloc_and_free.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/memory.inl" "$(@D)/cuda/include/thrust/system/omp/detail/memory.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/merge.h" "$(@D)/cuda/include/thrust/system/omp/detail/merge.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/mismatch.h" "$(@D)/cuda/include/thrust/system/omp/detail/mismatch.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/par.h" "$(@D)/cuda/include/thrust/system/omp/detail/par.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/partition.h" "$(@D)/cuda/include/thrust/system/omp/detail/partition.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/partition.inl" "$(@D)/cuda/include/thrust/system/omp/detail/partition.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/reduce.h" "$(@D)/cuda/include/thrust/system/omp/detail/reduce.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/reduce.inl" "$(@D)/cuda/include/thrust/system/omp/detail/reduce.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/omp/detail/reduce_by_key.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/reduce_by_key.inl" "$(@D)/cuda/include/thrust/system/omp/detail/reduce_by_key.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/reduce_intervals.h" "$(@D)/cuda/include/thrust/system/omp/detail/reduce_intervals.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/reduce_intervals.inl" "$(@D)/cuda/include/thrust/system/omp/detail/reduce_intervals.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/remove.h" "$(@D)/cuda/include/thrust/system/omp/detail/remove.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/remove.inl" "$(@D)/cuda/include/thrust/system/omp/detail/remove.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/replace.h" "$(@D)/cuda/include/thrust/system/omp/detail/replace.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/reverse.h" "$(@D)/cuda/include/thrust/system/omp/detail/reverse.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/scan.h" "$(@D)/cuda/include/thrust/system/omp/detail/scan.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/scan_by_key.h" "$(@D)/cuda/include/thrust/system/omp/detail/scan_by_key.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/scatter.h" "$(@D)/cuda/include/thrust/system/omp/detail/scatter.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/sequence.h" "$(@D)/cuda/include/thrust/system/omp/detail/sequence.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/set_operations.h" "$(@D)/cuda/include/thrust/system/omp/detail/set_operations.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/sort.h" "$(@D)/cuda/include/thrust/system/omp/detail/sort.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/sort.inl" "$(@D)/cuda/include/thrust/system/omp/detail/sort.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/swap_ranges.h" "$(@D)/cuda/include/thrust/system/omp/detail/swap_ranges.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/tabulate.h" "$(@D)/cuda/include/thrust/system/omp/detail/tabulate.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/omp/detail/temporary_buffer.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/transform.h" "$(@D)/cuda/include/thrust/system/omp/detail/transform.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/transform_reduce.h" "$(@D)/cuda/include/thrust/system/omp/detail/transform_reduce.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/transform_scan.h" "$(@D)/cuda/include/thrust/system/omp/detail/transform_scan.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/omp/detail/uninitialized_copy.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/omp/detail/uninitialized_fill.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/unique.h" "$(@D)/cuda/include/thrust/system/omp/detail/unique.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/unique.inl" "$(@D)/cuda/include/thrust/system/omp/detail/unique.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/unique_by_key.h" "$(@D)/cuda/include/thrust/system/omp/detail/unique_by_key.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/unique_by_key.inl" "$(@D)/cuda/include/thrust/system/omp/detail/unique_by_key.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/detail/vector.inl" "$(@D)/cuda/include/thrust/system/omp/detail/vector.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/execution_policy.h" "$(@D)/cuda/include/thrust/system/omp/execution_policy.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/memory.h" "$(@D)/cuda/include/thrust/system/omp/memory.h" && cp "/usr/local/cuda-10.0/include/thrust/system/omp/vector.h" "$(@D)/cuda/include/thrust/system/omp/vector.h" && cp "/usr/local/cuda-10.0/include/thrust/system/system_error.h" "$(@D)/cuda/include/thrust/system/system_error.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/tbb/detail/adjacent_difference.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/assign_value.h" "$(@D)/cuda/include/thrust/system/tbb/detail/assign_value.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/binary_search.h" "$(@D)/cuda/include/thrust/system/tbb/detail/binary_search.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/copy.h" "$(@D)/cuda/include/thrust/system/tbb/detail/copy.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/copy.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/copy.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/copy_if.h" "$(@D)/cuda/include/thrust/system/tbb/detail/copy_if.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/copy_if.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/copy_if.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/count.h" "$(@D)/cuda/include/thrust/system/tbb/detail/count.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/equal.h" "$(@D)/cuda/include/thrust/system/tbb/detail/equal.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/execution_policy.h" "$(@D)/cuda/include/thrust/system/tbb/detail/execution_policy.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/extrema.h" "$(@D)/cuda/include/thrust/system/tbb/detail/extrema.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/fill.h" "$(@D)/cuda/include/thrust/system/tbb/detail/fill.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/find.h" "$(@D)/cuda/include/thrust/system/tbb/detail/find.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/for_each.h" "$(@D)/cuda/include/thrust/system/tbb/detail/for_each.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/for_each.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/for_each.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/gather.h" "$(@D)/cuda/include/thrust/system/tbb/detail/gather.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/generate.h" "$(@D)/cuda/include/thrust/system/tbb/detail/generate.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/get_value.h" "$(@D)/cuda/include/thrust/system/tbb/detail/get_value.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/inner_product.h" "$(@D)/cuda/include/thrust/system/tbb/detail/inner_product.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/iter_swap.h" "$(@D)/cuda/include/thrust/system/tbb/detail/iter_swap.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/logical.h" "$(@D)/cuda/include/thrust/system/tbb/detail/logical.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/tbb/detail/malloc_and_free.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/memory.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/memory.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/merge.h" "$(@D)/cuda/include/thrust/system/tbb/detail/merge.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/merge.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/merge.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/mismatch.h" "$(@D)/cuda/include/thrust/system/tbb/detail/mismatch.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/par.h" "$(@D)/cuda/include/thrust/system/tbb/detail/par.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/partition.h" "$(@D)/cuda/include/thrust/system/tbb/detail/partition.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/partition.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/partition.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/reduce.h" "$(@D)/cuda/include/thrust/system/tbb/detail/reduce.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/reduce.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/reduce.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/tbb/detail/reduce_by_key.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/reduce_by_key.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/reduce_by_key.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/reduce_intervals.h" "$(@D)/cuda/include/thrust/system/tbb/detail/reduce_intervals.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/remove.h" "$(@D)/cuda/include/thrust/system/tbb/detail/remove.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/remove.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/remove.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/replace.h" "$(@D)/cuda/include/thrust/system/tbb/detail/replace.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/reverse.h" "$(@D)/cuda/include/thrust/system/tbb/detail/reverse.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/scan.h" "$(@D)/cuda/include/thrust/system/tbb/detail/scan.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/scan.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/scan.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/scan_by_key.h" "$(@D)/cuda/include/thrust/system/tbb/detail/scan_by_key.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/scatter.h" "$(@D)/cuda/include/thrust/system/tbb/detail/scatter.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/sequence.h" "$(@D)/cuda/include/thrust/system/tbb/detail/sequence.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/set_operations.h" "$(@D)/cuda/include/thrust/system/tbb/detail/set_operations.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/sort.h" "$(@D)/cuda/include/thrust/system/tbb/detail/sort.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/sort.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/sort.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/swap_ranges.h" "$(@D)/cuda/include/thrust/system/tbb/detail/swap_ranges.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/tabulate.h" "$(@D)/cuda/include/thrust/system/tbb/detail/tabulate.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/tbb/detail/temporary_buffer.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/transform.h" "$(@D)/cuda/include/thrust/system/tbb/detail/transform.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/transform_reduce.h" "$(@D)/cuda/include/thrust/system/tbb/detail/transform_reduce.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/transform_scan.h" "$(@D)/cuda/include/thrust/system/tbb/detail/transform_scan.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/tbb/detail/uninitialized_copy.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/tbb/detail/uninitialized_fill.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/unique.h" "$(@D)/cuda/include/thrust/system/tbb/detail/unique.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/unique.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/unique.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/unique_by_key.h" "$(@D)/cuda/include/thrust/system/tbb/detail/unique_by_key.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/unique_by_key.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/unique_by_key.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/vector.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/vector.inl" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/execution_policy.h" "$(@D)/cuda/include/thrust/system/tbb/execution_policy.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/memory.h" "$(@D)/cuda/include/thrust/system/tbb/memory.h" && cp "/usr/local/cuda-10.0/include/thrust/system/tbb/vector.h" "$(@D)/cuda/include/thrust/system/tbb/vector.h" && cp "/usr/local/cuda-10.0/include/thrust/system_error.h" "$(@D)/cuda/include/thrust/system_error.h" && cp "/usr/local/cuda-10.0/include/thrust/tabulate.h" "$(@D)/cuda/include/thrust/tabulate.h" && cp "/usr/local/cuda-10.0/include/thrust/transform.h" "$(@D)/cuda/include/thrust/transform.h" && cp "/usr/local/cuda-10.0/include/thrust/transform_reduce.h" "$(@D)/cuda/include/thrust/transform_reduce.h" && cp "/usr/local/cuda-10.0/include/thrust/transform_scan.h" "$(@D)/cuda/include/thrust/transform_scan.h" && cp "/usr/local/cuda-10.0/include/thrust/tuple.h" "$(@D)/cuda/include/thrust/tuple.h" && cp "/usr/local/cuda-10.0/include/thrust/uninitialized_copy.h" "$(@D)/cuda/include/thrust/uninitialized_copy.h" && cp "/usr/local/cuda-10.0/include/thrust/uninitialized_fill.h" "$(@D)/cuda/include/thrust/uninitialized_fill.h" && cp "/usr/local/cuda-10.0/include/thrust/unique.h" "$(@D)/cuda/include/thrust/unique.h" && cp "/usr/local/cuda-10.0/include/thrust/version.h" "$(@D)/cuda/include/thrust/version.h" && cp "/usr/local/cuda-10.0/include/vector_functions.h" "$(@D)/cuda/include/vector_functions.h" && cp "/usr/local/cuda-10.0/include/vector_functions.hpp" "$(@D)/cuda/include/vector_functions.hpp" && cp "/usr/local/cuda-10.0/include/vector_types.h" "$(@D)/cuda/include/vector_types.h"
    """,
 )
 
@@ -1203,7 +1203,7 @@ genrule(
         "cuda/nvvm/libdevice/libdevice.10.bc",
     ],
     cmd = """
-if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp "/usr/local/cuda-9.0/nvvm/libdevice/libdevice.10.bc" "$(@D)//libdevice.10.bc"
+if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp "/usr/local/cuda-10.0/nvvm/libdevice/libdevice.10.bc" "$(@D)//libdevice.10.bc"
    """,
 )
 
@@ -1240,7 +1240,7 @@ genrule(
         "cuda/extras/CUPTI/include/openacc/cupti_openacc.h",
     ],
     cmd = """
-if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp "/usr/local/cuda-9.0/extras/CUPTI/include/GL/gl.h" "$(@D)/cuda/extras/CUPTI/include/GL/gl.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/GL/glew.h" "$(@D)/cuda/extras/CUPTI/include/GL/glew.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/GL/glext.h" "$(@D)/cuda/extras/CUPTI/include/GL/glext.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/GL/glu.h" "$(@D)/cuda/extras/CUPTI/include/GL/glu.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/GL/glut.h" "$(@D)/cuda/extras/CUPTI/include/GL/glut.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/GL/glx.h" "$(@D)/cuda/extras/CUPTI/include/GL/glx.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/GL/glxext.h" "$(@D)/cuda/extras/CUPTI/include/GL/glxext.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/GL/wglew.h" "$(@D)/cuda/extras/CUPTI/include/GL/wglew.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/GL/wglext.h" "$(@D)/cuda/extras/CUPTI/include/GL/wglext.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/cuda_stdint.h" "$(@D)/cuda/extras/CUPTI/include/cuda_stdint.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/cupti.h" "$(@D)/cuda/extras/CUPTI/include/cupti.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/cupti_activity.h" "$(@D)/cuda/extras/CUPTI/include/cupti_activity.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/cupti_callbacks.h" "$(@D)/cuda/extras/CUPTI/include/cupti_callbacks.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/cupti_driver_cbid.h" "$(@D)/cuda/extras/CUPTI/include/cupti_driver_cbid.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/cupti_events.h" "$(@D)/cuda/extras/CUPTI/include/cupti_events.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/cupti_metrics.h" "$(@D)/cuda/extras/CUPTI/include/cupti_metrics.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/cupti_nvtx_cbid.h" "$(@D)/cuda/extras/CUPTI/include/cupti_nvtx_cbid.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/cupti_result.h" "$(@D)/cuda/extras/CUPTI/include/cupti_result.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/cupti_runtime_cbid.h" "$(@D)/cuda/extras/CUPTI/include/cupti_runtime_cbid.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/cupti_version.h" "$(@D)/cuda/extras/CUPTI/include/cupti_version.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/generated_cudaGL_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cudaGL_meta.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/generated_cudaVDPAU_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cudaVDPAU_meta.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/generated_cuda_gl_interop_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cuda_gl_interop_meta.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/generated_cuda_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cuda_meta.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/generated_cuda_runtime_api_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cuda_runtime_api_meta.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/generated_cuda_vdpau_interop_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cuda_vdpau_interop_meta.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/generated_nvtx_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_nvtx_meta.h" && cp "/usr/local/cuda-9.0/extras/CUPTI/include/openacc/cupti_openacc.h" "$(@D)/cuda/extras/CUPTI/include/openacc/cupti_openacc.h"
+if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp "/usr/local/cuda-10.0/extras/CUPTI/include/GL/gl.h" "$(@D)/cuda/extras/CUPTI/include/GL/gl.h" && cp "/usr/local/cuda-10.0/extras/CUPTI/include/GL/glew.h" "$(@D)/cuda/extras/CUPTI/include/GL/glew.h" && cp "/usr/local/cuda-10.0/extras/CUPTI/include/GL/glext.h" "$(@D)/cuda/extras/CUPTI/include/GL/glext.h" && cp "/usr/local/cuda-10.0/extras/CUPTI/include/GL/glu.h" "$(@D)/cuda/extras/CUPTI/include/GL/glu.h" && cp "/usr/local/cuda-10.0/extras/CUPTI/include/GL/glut.h" "$(@D)/cuda/extras/CUPTI/include/GL/glut.h" && cp "/usr/local/cuda-10.0/extras/CUPTI/include/GL/glx.h" "$(@D)/cuda/extras/CUPTI/include/GL/glx.h" && cp "/usr/local/cuda-10.0/extras/CUPTI/include/GL/glxext.h" "$(@D)/cuda/extras/CUPTI/include/GL/glxext.h" && cp "/usr/local/cuda-10.0/extras/CUPTI/include/GL/wglew.h" "$(@D)/cuda/extras/CUPTI/include/GL/wglew.h" && cp "/usr/local/cuda-10.0/extras/CUPTI/include/GL/wglext.h" "$(@D)/cuda/extras/CUPTI/include/GL/wglext.h" && cp "/usr/local/cuda-10.0/extras/CUPTI/include/cuda_stdint.h" "$(@D)/cuda/extras/CUPTI/include/cuda_stdint.h" && cp "/usr/local/cuda-10.0/extras/CUPTI/include/cupti.h" "$(@D)/cuda/extras/CUPTI/include/cupti.h" && cp "/usr/local/cuda-10.0/extras/CUPTI/include/cupti_activity.h" "$(@D)/cuda/extras/CUPTI/include/cupti_activity.h" && cp "/usr/local/cuda-10.0/extras/CUPTI/include/cupti_callbacks.h" "$(@D)/cuda/extras/CUPTI/include/cupti_callbacks.h" && cp "/usr/local/cuda-10.0/extras/CUPTI/include/cupti_driver_cbid.h" "$(@D)/cuda/extras/CUPTI/include/cupti_driver_cbid.h" && cp "/usr/local/cuda-10.0/extras/CUPTI/include/cupti_events.h" "$(@D)/cuda/extras/CUPTI/include/cupti_events.h" && cp "/usr/local/cuda-10.0/extras/CUPTI/include/cupti_metrics.h" "$(@D)/cuda/extras/CUPTI/include/cupti_metrics.h" && cp "/usr/local/cuda-10.0/extras/CUPTI/include/cupti_nvtx_cbid.h" "$(@D)/cuda/extras/CUPTI/include/cupti_nvtx_cbid.h" && cp "/usr/local/cuda-10.0/extras/CUPTI/include/cupti_result.h" "$(@D)/cuda/extras/CUPTI/include/cupti_result.h" && cp "/usr/local/cuda-10.0/extras/CUPTI/include/cupti_runtime_cbid.h" "$(@D)/cuda/extras/CUPTI/include/cupti_runtime_cbid.h" && cp "/usr/local/cuda-10.0/extras/CUPTI/include/cupti_version.h" "$(@D)/cuda/extras/CUPTI/include/cupti_version.h" && cp "/usr/local/cuda-10.0/extras/CUPTI/include/generated_cudaGL_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cudaGL_meta.h" && cp "/usr/local/cuda-10.0/extras/CUPTI/include/generated_cudaVDPAU_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cudaVDPAU_meta.h" && cp "/usr/local/cuda-10.0/extras/CUPTI/include/generated_cuda_gl_interop_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cuda_gl_interop_meta.h" && cp "/usr/local/cuda-10.0/extras/CUPTI/include/generated_cuda_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cuda_meta.h" && cp "/usr/local/cuda-10.0/extras/CUPTI/include/generated_cuda_runtime_api_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cuda_runtime_api_meta.h" && cp "/usr/local/cuda-10.0/extras/CUPTI/include/generated_cuda_vdpau_interop_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cuda_vdpau_interop_meta.h" && cp "/usr/local/cuda-10.0/extras/CUPTI/include/generated_nvtx_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_nvtx_meta.h" && cp "/usr/local/cuda-10.0/extras/CUPTI/include/openacc/cupti_openacc.h" "$(@D)/cuda/extras/CUPTI/include/openacc/cupti_openacc.h"
    """,
 )
 
@@ -1248,17 +1248,17 @@ genrule(
     name = "cuda-lib",
     outs = [
         "cuda/lib/libcuda.so",
-        "cuda/lib/libcudart.so.9.0",
+        "cuda/lib/libcudart.so.10.0",
         "cuda/lib/libcudart_static.a",
-        "cuda/lib/libcublas.so.9.0",
-        "cuda/lib/libcusolver.so.9.0",
-        "cuda/lib/libcurand.so.9.0",
-        "cuda/lib/libcufft.so.9.0",
+        "cuda/lib/libcublas.so.10.0",
+        "cuda/lib/libcusolver.so.10.0",
+        "cuda/lib/libcurand.so.10.0",
+        "cuda/lib/libcufft.so.10.0",
         "cuda/lib/libcudnn.so.7",
-        "cuda/lib/libcupti.so.9.0",
+        "cuda/lib/libcupti.so.10.0",
     ],
     cmd = """
-if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp "/usr/local/cuda-9.0/targets/x86_64-linux/lib/stubs/libcuda.so" "$(@D)/cuda/lib/libcuda.so" && cp "/usr/local/cuda-9.0/targets/x86_64-linux/lib/libcudart.so.9.0.176" "$(@D)/cuda/lib/libcudart.so.9.0" && cp "/usr/local/cuda-9.0/targets/x86_64-linux/lib/libcudart_static.a" "$(@D)/cuda/lib/libcudart_static.a" && cp "/usr/local/cuda-9.0/targets/x86_64-linux/lib/libcublas.so.9.0.480" "$(@D)/cuda/lib/libcublas.so.9.0" && cp "/usr/local/cuda-9.0/targets/x86_64-linux/lib/libcusolver.so.9.0.176" "$(@D)/cuda/lib/libcusolver.so.9.0" && cp "/usr/local/cuda-9.0/targets/x86_64-linux/lib/libcurand.so.9.0.176" "$(@D)/cuda/lib/libcurand.so.9.0" && cp "/usr/local/cuda-9.0/targets/x86_64-linux/lib/libcufft.so.9.0.176" "$(@D)/cuda/lib/libcufft.so.9.0" && cp "/usr/lib/x86_64-linux-gnu/libcudnn.so.7.2.1" "$(@D)/cuda/lib/libcudnn.so.7" && cp "/usr/local/cuda/extras/CUPTI/lib64/libcupti.so.9.0.176" "$(@D)/cuda/lib/libcupti.so.9.0"
+if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp "/usr/local/cuda-10.0/targets/x86_64-linux/lib/stubs/libcuda.so" "$(@D)/cuda/lib/libcuda.so" && cp "/usr/local/cuda-10.0/targets/x86_64-linux/lib/libcudart.so.10.0.176" "$(@D)/cuda/lib/libcudart.so.10.0" && cp "/usr/local/cuda-10.0/targets/x86_64-linux/lib/libcudart_static.a" "$(@D)/cuda/lib/libcudart_static.a" && cp "/usr/local/cuda-10.0/targets/x86_64-linux/lib/libcublas.so.10.0.480" "$(@D)/cuda/lib/libcublas.so.10.0" && cp "/usr/local/cuda-10.0/targets/x86_64-linux/lib/libcusolver.so.10.0.176" "$(@D)/cuda/lib/libcusolver.so.10.0" && cp "/usr/local/cuda-10.0/targets/x86_64-linux/lib/libcurand.so.10.0.176" "$(@D)/cuda/lib/libcurand.so.10.0" && cp "/usr/local/cuda-10.0/targets/x86_64-linux/lib/libcufft.so.10.0.176" "$(@D)/cuda/lib/libcufft.so.10.0" && cp "/usr/lib/x86_64-linux-gnu/libcudnn.so.7.2.1" "$(@D)/cuda/lib/libcudnn.so.7" && cp "/usr/local/cuda/extras/CUPTI/lib64/libcupti.so.10.0.176" "$(@D)/cuda/lib/libcupti.so.10.0"
    """,
 )
 
diff --git a/third_party/toolchains/gpus/cuda/cuda/cuda_config.h b/third_party/toolchains/gpus/cuda/cuda/cuda_config.h
index 7cdaf144ad..b05bfb7326 100644
--- a/third_party/toolchains/gpus/cuda/cuda/cuda_config.h
+++ b/third_party/toolchains/gpus/cuda/cuda/cuda_config.h
@@ -19,9 +19,9 @@ limitations under the License.
 
 #define TF_CUDA_CAPABILITIES CudaVersion("3.0")
 
-#define TF_CUDA_VERSION "9.0"
+#define TF_CUDA_VERSION "10.0"
 #define TF_CUDNN_VERSION "7"
 
-#define TF_CUDA_TOOLKIT_PATH "/usr/local/cuda-9.0"
+#define TF_CUDA_TOOLKIT_PATH "/usr/local/cuda-10.0"
 
 #endif  // CUDA_CUDA_CONFIG_H_
diff --git a/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc/clang/bin/crosstool_wrapper_driver_is_not_gcc b/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc/clang/bin/crosstool_wrapper_driver_is_not_gcc
index 63893d3722..192314137d 100755
--- a/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc/clang/bin/crosstool_wrapper_driver_is_not_gcc
+++ b/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc/clang/bin/crosstool_wrapper_driver_is_not_gcc
@@ -49,9 +49,9 @@ import pipes
 CPU_COMPILER = ('/usr/bin/gcc')
 GCC_HOST_COMPILER_PATH = ('/usr/bin/gcc')
 
-NVCC_PATH = '/usr/local/cuda-9.0/bin/nvcc'
+NVCC_PATH = '/usr/local/cuda/bin/nvcc'
 PREFIX_DIR = os.path.dirname(GCC_HOST_COMPILER_PATH)
-NVCC_VERSION = '9.0'
+NVCC_VERSION = '10.0'
 
 def Log(s):
   print('gpus/crosstool: {0}'.format(s))
-- 
GitLab


From e0aa9387258df040e296f18c6b6fdb560bce6fcc Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sat, 29 Dec 2018 01:02:34 -0800
Subject: [PATCH 0078/1095] compat: Update forward compatibility horizon to
 2018-12-29

PiperOrigin-RevId: 227206663
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index df10dc1b79..3a9b3b3838 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 12, 28)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 12, 29)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From d2dd369f9dadb5dd3220ababa299ad89cd8e8574 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sat, 29 Dec 2018 08:46:45 -0800
Subject: [PATCH 0079/1095] Split input-related classes out of
 python/distribute/values.py into new file .../input_lib.py.

PiperOrigin-RevId: 227227637
---
 tensorflow/contrib/distribute/python/BUILD    |  31 +-
 .../python/collective_all_reduce_strategy.py  |  11 +-
 .../distribute/python/input_lib_test.py       | 480 ++++++++++++
 .../distribute/python/mirrored_strategy.py    |   8 +-
 .../distribute/python/one_device_strategy.py  |  12 +-
 .../python/parameter_server_strategy.py       |  13 +-
 .../contrib/distribute/python/tpu_strategy.py |  12 +-
 .../contrib/distribute/python/values_test.py  | 446 -----------
 tensorflow/python/distribute/BUILD            |  20 +-
 tensorflow/python/distribute/input_lib.py     | 707 ++++++++++++++++++
 .../python/distribute/mirrored_strategy.py    |  16 +-
 tensorflow/python/distribute/values.py        | 678 -----------------
 12 files changed, 1272 insertions(+), 1162 deletions(-)
 create mode 100644 tensorflow/contrib/distribute/python/input_lib_test.py
 create mode 100644 tensorflow/python/distribute/input_lib.py

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index 2d6a08df9a..f27224e46e 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -23,17 +23,14 @@ cuda_py_test(
     additional_deps = [
         ":combinations",
         ":mirrored_strategy",
-        ":multi_worker_test_base",
         "@absl_py//absl/testing:parameterized",
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:constant_op",
-        "//tensorflow/python:errors",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:training",
         "//tensorflow/python:variable_scope",
-        "//tensorflow/python/data/ops:dataset_ops",
         "//tensorflow/python/distribute:device_util",
         "//tensorflow/python/distribute:values",
         "//tensorflow/python/eager:context",
@@ -45,14 +42,36 @@ cuda_py_test(
     ],
 )
 
+cuda_py_test(
+    name = "input_lib_test",
+    srcs = ["input_lib_test.py"],
+    additional_deps = [
+        ":combinations",
+        ":mirrored_strategy",
+        ":multi_worker_test_base",
+        "@absl_py//absl/testing:parameterized",
+        "//tensorflow/core:protos_all_py",
+        "//tensorflow/python:errors",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow/python/distribute:input_lib",
+        "//tensorflow/python/distribute:values",
+        "//tensorflow/python/eager:context",
+        "//tensorflow/python/eager:test",
+    ],
+    tags = [
+        "no_pip",
+    ],
+)
+
 py_library(
     name = "mirrored_strategy",
     srcs = ["mirrored_strategy.py"],
     visibility = ["//tensorflow:internal"],
     deps = [
         "//tensorflow/python/distribute:distribute_lib",
+        "//tensorflow/python/distribute:input_lib",
         "//tensorflow/python/distribute:mirrored_strategy",
-        "//tensorflow/python/distribute:values",
     ],
 )
 
@@ -69,6 +88,7 @@ py_library(
         "//tensorflow/python:training",
         "//tensorflow/python:util",
         "//tensorflow/python/distribute:cross_device_ops",
+        "//tensorflow/python/distribute:input_lib",
         "//tensorflow/python/distribute:multi_worker_util",
         "//tensorflow/python/distribute:reduce_util",
         "//tensorflow/python/distribute:values",
@@ -119,6 +139,7 @@ py_library(
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:math_ops",
         "//tensorflow/python/distribute:distribute_lib",
+        "//tensorflow/python/distribute:input_lib",
         "//tensorflow/python/distribute:reduce_util",
         "//tensorflow/python/distribute:values",
         "//tensorflow/python/eager:context",
@@ -139,6 +160,7 @@ py_library(
         "//tensorflow/python:training",
         "//tensorflow/python/distribute:cross_device_ops",
         "//tensorflow/python/distribute:cross_device_utils",
+        "//tensorflow/python/distribute:input_lib",
         "//tensorflow/python/distribute:multi_worker_util",
         "//tensorflow/python/distribute:values",
         "//tensorflow/python/eager:context",
@@ -289,6 +311,7 @@ py_library(
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:tensor_util",
         "//tensorflow/python:util",
+        "//tensorflow/python/distribute:input_lib",
         "//tensorflow/python/distribute:reduce_util",
         "//tensorflow/python/distribute:values",
     ],
diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
index 12197c3d0d..f6361cb6e8 100644
--- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
+++ b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
@@ -26,6 +26,7 @@ from tensorflow.python.distribute import cross_device_ops as cross_device_ops_li
 from tensorflow.python.distribute import cross_device_utils
 from tensorflow.python.distribute import device_util
 from tensorflow.python.distribute import distribute_lib
+from tensorflow.python.distribute import input_lib
 from tensorflow.python.distribute import multi_worker_util
 from tensorflow.python.distribute import values
 from tensorflow.python.eager import context
@@ -130,7 +131,7 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
 
     self._collective_keys = cross_device_utils.CollectiveKeys()
     self._initialize_local(local_devices)
-    self._input_workers = values.InputWorkers(
+    self._input_workers = input_lib.InputWorkers(
         self._device_map, [(self._worker_device, self.worker_devices)])
     self._cross_device_ops = cross_device_ops_lib.CollectiveAllReduce(
         num_workers=self._num_workers,
@@ -229,13 +230,13 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
     """Distributes the dataset to each local GPU."""
     # TODO(yuefengz): shard the dataset.
     worker_index = 0
-    return values.PerReplicaDataset(
+    return input_lib.PerReplicaDataset(
         self._call_dataset_fn(dataset_fn), self._input_workers, worker_index,
         prefetch_on_device=True)
 
   def _make_dataset_iterator(self, dataset):
-    return values.DatasetIterator(dataset, self._input_workers,
-                                  self._num_replicas_in_sync)
+    return input_lib.DatasetIterator(dataset, self._input_workers,
+                                     self._num_replicas_in_sync)
 
   def _make_input_fn_iterator(
       self,
@@ -252,7 +253,7 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
         input_pipeline_id=input_pipeline_id,
         num_replicas_in_sync=self._num_replicas_in_sync)
 
-    return values.InputFunctionIterator(
+    return input_lib.InputFunctionIterator(
         input_fn, self._input_workers, [input_context])
 
   def _configure(self,
diff --git a/tensorflow/contrib/distribute/python/input_lib_test.py b/tensorflow/contrib/distribute/python/input_lib_test.py
new file mode 100644
index 0000000000..f589cd6ad5
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/input_lib_test.py
@@ -0,0 +1,480 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for the input_lib library."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from absl.testing import parameterized
+
+from tensorflow.contrib.distribute.python import combinations
+from tensorflow.contrib.distribute.python import multi_worker_test_base
+from tensorflow.core.protobuf import config_pb2
+from tensorflow.python.data.experimental.ops import batching
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.distribute import distribute_lib
+from tensorflow.python.distribute import input_lib
+from tensorflow.python.distribute import values
+from tensorflow.python.eager import context
+from tensorflow.python.eager import test
+from tensorflow.python.framework import errors
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import random_ops
+from tensorflow.python.util import nest
+
+
+class PerReplicaDatasetTest(test.TestCase):
+
+  config = config_pb2.ConfigProto()
+  config.allow_soft_placement = True
+
+  def _test_iterator(self, devices, dataset, expected_values):
+    device_map = values.ReplicaDeviceMap(devices)
+    input_workers = input_lib.InputWorkers(device_map)
+    per_replica_dataset = input_lib.PerReplicaDataset(dataset, input_workers, 0)
+    if context.executing_eagerly():
+      iterator = per_replica_dataset.make_one_shot_iterator()
+    else:
+      iterator = per_replica_dataset.make_initializable_iterator()
+      self.evaluate([iterator.initializer])
+
+    for expected_value in expected_values:
+      next_element = iterator.get_next_as_list()
+      computed_value = self.evaluate(next_element)
+      self.assertEqual(expected_value, computed_value)
+
+    with self.assertRaises(errors.OutOfRangeError):
+      next_element = iterator.get_next_as_list()
+      self.evaluate(next_element)
+
+  @test_util.run_in_graph_and_eager_modes
+  def testOneDevice(self):
+    devices = ["/device:CPU:0"]
+    dataset = dataset_ops.Dataset.range(10)
+
+    expected_values = [[i] for i in range(10)]
+
+    self._test_iterator(devices, dataset, expected_values)
+
+  @test_util.run_in_graph_and_eager_modes(config=config)
+  def testMultipleDevices(self):
+    if context.num_gpus() < 1 and context.executing_eagerly():
+      self.skipTest("A GPU is not available for this test in eager mode.")
+
+    devices = ["/device:CPU:0", "/device:GPU:0"]
+    dataset = dataset_ops.Dataset.range(10)
+
+    expected_values = [[i, i+1] for i in range(0, 10, 2)]
+
+    self._test_iterator(devices, dataset, expected_values)
+
+  @test_util.run_in_graph_and_eager_modes(config=config)
+  def testTupleDataset(self):
+    if context.num_gpus() < 1 and context.executing_eagerly():
+      self.skipTest("A GPU is not available for this test in eager mode.")
+
+    devices = ["/device:CPU:0", "/device:GPU:0"]
+    dataset1 = dataset_ops.Dataset.range(10)
+    dataset2 = dataset_ops.Dataset.range(10).map(lambda x: x**2)
+    dataset = dataset_ops.Dataset.zip((dataset1, dataset2))
+
+    expected_values = [[(i, i**2), (i+1, (i+1)**2)] for i in range(0, 10, 2)]
+
+    self._test_iterator(devices, dataset, expected_values)
+
+  @test_util.run_in_graph_and_eager_modes(config=config)
+  def testUnevenDatasetBatches(self):
+    if context.num_gpus() < 1 and context.executing_eagerly():
+      self.skipTest("A GPU is not available for this test in eager mode.")
+
+    devices = ["/device:CPU:0", "/device:GPU:0"]
+    dataset = dataset_ops.Dataset.range(11)
+
+    expected_values = [[i, i+1] for i in range(0, 10, 2)]
+    self._test_iterator(devices, dataset, expected_values)
+
+  def testInitializableIterator(self):
+    with context.graph_mode():
+      devices = ["/device:CPU:0"]
+      # Using random input since that is only allowed with initializable
+      # iterator.
+      dataset = dataset_ops.Dataset.from_tensor_slices(
+          random_ops.random_uniform((10,)))
+
+      device_map = values.ReplicaDeviceMap(devices)
+      input_workers = input_lib.InputWorkers(device_map)
+      per_replica_dataset = input_lib.PerReplicaDataset(
+          dataset, input_workers, 0)
+      iterator = per_replica_dataset.make_initializable_iterator()
+
+      self.evaluate(iterator.initializer)
+      next_element = iterator.get_next_as_list()
+      for _ in range(10):
+        self.evaluate(next_element)
+
+      # Should fail after the input is finished.
+      with self.assertRaises(errors.OutOfRangeError):
+        self.evaluate(next_element)
+
+      # After re-initializing the iterator, should be able to iterate again.
+      self.evaluate(iterator.initializer)
+      for _ in range(10):
+        self.evaluate(next_element)
+
+
+class MultiWorkerDatasetTest(multi_worker_test_base.MultiWorkerTestBase):
+
+  def _test_iterator(self, sess, iterator, devices, expected_values):
+    next_element = iterator.get_next()
+    for r, device in enumerate(devices):
+      v = values.select_replica(r, next_element)
+      # The `v` here can be a tuple.
+      for element in nest.flatten(v):
+        self.assertTrue(element.device in device)
+
+    for expected_value in expected_values:
+      t = [values.select_replica(r, next_element) for r in range(len(devices))]
+      actual = sess.run(t)
+      self.assertEqual(expected_value, actual)
+
+    with self.assertRaises(errors.OutOfRangeError):
+      sess.run([values.select_replica(r, next_element)
+                for r in range(len(devices))])
+
+  def _test_dataset(self, dataset_fn, worker_devices, devices,
+                    expected_values):
+    device_map = values.ReplicaDeviceMap(devices)
+    input_workers = input_lib.InputWorkers(device_map, worker_devices)
+    multi_worker_dataset = input_lib.MultiWorkerDataset(
+        dataset_fn, input_workers)
+    multi_worker_iterator = multi_worker_dataset.make_initializable_iterator()
+    with self.cached_session() as sess:
+      sess.run(multi_worker_iterator.initializer)
+      self._test_iterator(sess, multi_worker_iterator, devices, expected_values)
+
+  def _cpu_devices(self):
+    worker_devices = (
+        ("/job:worker/replica:0/task:0",
+         ["/job:worker/replica:0/task:0/device:CPU:0"]),
+        ("/job:worker/replica:0/task:1",
+         ["/job:worker/replica:0/task:1/device:CPU:0"])
+    )
+    devices = [
+        "/job:worker/replica:0/task:0/device:CPU:0",
+        "/job:worker/replica:0/task:1/device:CPU:0"
+    ]
+    return worker_devices, devices
+
+  def _cpu_and_one_gpu_devices(self):
+    worker_devices = (
+        ("/job:worker/replica:0/task:0", (
+            "/job:worker/replica:0/task:0/device:GPU:0",
+            "/job:worker/replica:0/task:0/device:CPU:0"
+        )),
+        ("/job:worker/replica:0/task:1", (
+            "/job:worker/replica:0/task:1/device:GPU:0",
+            "/job:worker/replica:0/task:1/device:CPU:0"
+        ))
+    )
+    devices = [
+        "/job:worker/replica:0/task:0/device:GPU:0",
+        "/job:worker/replica:0/task:0/device:CPU:0",
+        "/job:worker/replica:0/task:1/device:GPU:0",
+        "/job:worker/replica:0/task:1/device:CPU:0"
+    ]
+    return worker_devices, devices
+
+  def testDataDistributionOneDevicePerWorker(self):
+    worker_devices, devices = self._cpu_devices()
+    with context.graph_mode():
+      dataset_fn = lambda: dataset_ops.Dataset.range(8)
+      self._test_dataset(
+          dataset_fn, worker_devices, devices,
+          [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4], [5, 5], [6, 6], [7, 7]])
+
+  def testDataDistributionTwoDevicePerWorker(self):
+    if context.num_gpus() < 1:
+      self.skipTest("A GPU is not available for this test.")
+    worker_devices, devices = self._cpu_and_one_gpu_devices()
+    with context.graph_mode():
+      dataset_fn = lambda: dataset_ops.Dataset.range(8)
+      self._test_dataset(
+          dataset_fn, worker_devices, devices,
+          [[0, 1, 0, 1], [2, 3, 2, 3], [4, 5, 4, 5], [6, 7, 6, 7]])
+
+  def testTupleDataset(self):
+    worker_devices, devices = self._cpu_devices()
+
+    with context.graph_mode():
+
+      def dataset_fn():
+        dataset1 = dataset_ops.Dataset.range(8)
+        dataset2 = dataset_ops.Dataset.range(8).map(lambda x: x**2)
+        return dataset_ops.Dataset.zip((dataset1, dataset2))
+
+      expected_values = [[(i, i**2), (i, i**2)] for i in range(8)]
+      self._test_dataset(dataset_fn, worker_devices, devices,
+                         expected_values)
+
+  def testInitializableIterator(self):
+    worker_devices, devices = self._cpu_devices()
+    with context.graph_mode(), self.cached_session() as sess:
+      dataset_fn = lambda: dataset_ops.Dataset.range(8)
+      device_map = values.ReplicaDeviceMap(devices)
+      input_workers = input_lib.InputWorkers(device_map, worker_devices)
+      multi_worker_dataset = input_lib.MultiWorkerDataset(
+          dataset_fn, input_workers)
+      multi_worker_iterator = multi_worker_dataset.make_initializable_iterator()
+
+      sess.run(multi_worker_iterator.initializer)
+      self._test_iterator(
+          sess, multi_worker_iterator, devices,
+          [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4], [5, 5], [6, 6], [7, 7]])
+
+      # After re-initializing the iterator, should be able to iterate again.
+      sess.run(multi_worker_iterator.initializer)
+      self._test_iterator(
+          sess, multi_worker_iterator, devices,
+          [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4], [5, 5], [6, 6], [7, 7]])
+
+  def testValueErrorForIterator(self):
+    # Incompatiable arguments.
+    d1 = "/device:GPU:0"
+    d2 = "/device:GPU:1"
+    device_map = values.ReplicaDeviceMap([d1, d2])
+    input_workers = input_lib.InputWorkers(
+        device_map, (("w1", (d1,)), ("w2", (d2,))))
+    with self.assertRaises(ValueError):
+      input_lib.MultiWorkerDataIterator([("w1", None)], input_workers)
+
+  def testDuplicateDevices(self):
+    _, devices = self._cpu_devices()
+    devices.append("/job:worker/replica:0/task:0/device:CPU:0")
+    with self.assertRaises(ValueError):
+      _ = values.ReplicaDeviceMap(devices)
+
+
+class InputIteratorTestBase(test.TestCase):
+
+  def _test_iterator(self, input_type, dataset_fn, worker_device_pairs,
+                     expected_values, sess=None, split_batch_by=None):
+    devices = nest.flatten([ds for _, ds in worker_device_pairs])
+    device_map = values.ReplicaDeviceMap(devices)
+    input_workers = input_lib.InputWorkers(device_map, worker_device_pairs)
+
+    if input_type == "input_fn":
+      input_contexts = [
+          distribute_lib.InputContext() for _ in worker_device_pairs]
+      input_fn = lambda _: dataset_fn()
+      iterator = input_lib.InputFunctionIterator(
+          input_fn, input_workers, input_contexts)
+    else:
+      iterator = input_lib.DatasetIterator(
+          dataset_fn(), input_workers, split_batch_by)
+
+    evaluate = lambda x: sess.run(x) if sess else self.evaluate(x)
+
+    evaluate(control_flow_ops.group(iterator.initialize()))
+
+    for expected_value in expected_values:
+      next_element = iterator.get_next()
+      computed_value = evaluate(
+          [values.select_replica(r, next_element) for r in range(len(devices))])
+      self.assertAllEqual(expected_value, computed_value)
+
+    with self.assertRaises(errors.OutOfRangeError):
+      next_element = iterator.get_next()
+      evaluate([values.select_replica(r, next_element)
+                for r in range(len(devices))])
+
+    # After re-initializing the iterator, should be able to iterate again.
+    evaluate(control_flow_ops.group(iterator.initialize()))
+
+    for expected_value in expected_values:
+      next_element = iterator.get_next()
+      computed_value = evaluate(
+          [values.select_replica(r, next_element) for r in range(len(devices))])
+      self.assertAllEqual(expected_value, computed_value)
+
+
+class InputIteratorSingleWorkerTest(InputIteratorTestBase,
+                                    parameterized.TestCase):
+
+  @combinations.generate(combinations.combine(
+      mode=["graph", "eager"],
+      input_type=["input_fn", "dataset"]))
+  def testOneDeviceCPU(self, input_type):
+    worker_device_pairs = [("", ["/device:CPU:0"])]
+    dataset_fn = lambda: dataset_ops.Dataset.range(10)
+
+    expected_values = [[i] for i in range(10)]
+
+    self._test_iterator(input_type, dataset_fn, worker_device_pairs,
+                        expected_values)
+
+  @combinations.generate(combinations.combine(
+      mode=["graph", "eager"],
+      input_type=["input_fn", "dataset"],
+      required_gpus=1))
+  def testTwoDevicesOneGPUOneCPU(self, input_type):
+    worker_device_pairs = [("", ["/device:GPU:0", "/device:CPU:0"])]
+    dataset_fn = lambda: dataset_ops.Dataset.range(10)
+
+    expected_values = [[i, i+1] for i in range(0, 10, 2)]
+
+    self._test_iterator(input_type, dataset_fn, worker_device_pairs,
+                        expected_values)
+
+  @combinations.generate(combinations.combine(
+      mode=["graph", "eager"],
+      input_type=["input_fn", "dataset"],
+      required_gpus=1))
+  def testTupleDataset(self, input_type):
+    worker_device_pairs = [("", ["/device:GPU:0", "/device:CPU:0"])]
+    def dataset_fn():
+      dataset1 = dataset_ops.Dataset.range(10)
+      dataset2 = dataset_ops.Dataset.range(10).map(lambda x: x**2)
+      return dataset_ops.Dataset.zip((dataset1, dataset2))
+
+    expected_values = [[(i, i**2), (i+1, (i+1)**2)] for i in range(0, 10, 2)]
+
+    self._test_iterator(input_type, dataset_fn, worker_device_pairs,
+                        expected_values)
+
+  @combinations.generate(combinations.combine(
+      mode=["graph", "eager"],
+      input_type=["input_fn", "dataset"],
+      required_gpus=1))
+  def testUnevenDatasetBatches(self, input_type):
+    worker_device_pairs = [("", ["/device:GPU:0", "/device:CPU:0"])]
+    dataset_fn = lambda: dataset_ops.Dataset.range(11)
+
+    expected_values = [[i, i+1] for i in range(0, 10, 2)]
+    self._test_iterator(input_type, dataset_fn, worker_device_pairs,
+                        expected_values)
+
+  @combinations.generate(combinations.combine(
+      mode=["graph", "eager"],
+      input_type=["dataset"],
+      split_batch_by=[None, 2],
+      required_gpus=1))
+  def testBatchSplitting(self, input_type, split_batch_by):
+    worker_device_pairs = [("", ["/device:GPU:0", "/device:CPU:0"])]
+    batch_size = 10
+    dataset_fn = lambda: dataset_ops.Dataset.range(100).batch(batch_size)
+
+    updated_batch_size = (
+        batch_size // split_batch_by if split_batch_by else batch_size)
+    expected_values = [[range(i, i+updated_batch_size),
+                        range(i+updated_batch_size, i+2*updated_batch_size)]
+                       for i in range(0, 100, updated_batch_size*2)]
+
+    self._test_iterator(input_type, dataset_fn, worker_device_pairs,
+                        expected_values, sess=None,
+                        split_batch_by=split_batch_by)
+
+
+class InputIteratorMultiWorkerTest(
+    multi_worker_test_base.MultiWorkerTestBase, InputIteratorTestBase,
+    parameterized.TestCase):
+
+  def _cpu_devices(self):
+    return [
+        ("/job:worker/replica:0/task:0",
+         ["/job:worker/replica:0/task:0/device:CPU:0"]),
+        ("/job:worker/replica:0/task:1",
+         ["/job:worker/replica:0/task:1/device:CPU:0"])]
+
+  def _cpu_and_one_gpu_devices(self):
+    return [
+        ("/job:worker/replica:0/task:0", [
+            "/job:worker/replica:0/task:0/device:GPU:0",
+            "/job:worker/replica:0/task:0/device:CPU:0"
+        ]),
+        ("/job:worker/replica:0/task:1", [
+            "/job:worker/replica:0/task:1/device:GPU:0",
+            "/job:worker/replica:0/task:1/device:CPU:0"
+        ])
+    ]
+
+  @combinations.generate(combinations.combine(
+      mode=["graph"],
+      input_type=["input_fn", "dataset"]))
+  def testOneDevicePerWorker(self, input_type):
+    worker_devices = self._cpu_devices()
+    with context.graph_mode(), self.cached_session() as sess:
+      dataset_fn = lambda: dataset_ops.Dataset.range(4)
+      self._test_iterator(input_type, dataset_fn, worker_devices,
+                          [[0, 0], [1, 1], [2, 2], [3, 3]], sess)
+
+  @combinations.generate(combinations.combine(
+      mode=["graph"],
+      input_type=["input_fn", "dataset"],
+      required_gpus=1))
+  def testTwoDevicesPerWorker(self, input_type):
+    worker_devices = self._cpu_and_one_gpu_devices()
+    with context.graph_mode(), self.cached_session() as sess:
+      dataset_fn = lambda: dataset_ops.Dataset.range(4)
+      self._test_iterator(input_type, dataset_fn, worker_devices,
+                          [[0, 1, 0, 1], [2, 3, 2, 3]], sess)
+
+  @combinations.generate(combinations.combine(
+      mode=["graph"],
+      input_type=["input_fn", "dataset"]))
+  def testTupleDataset(self, input_type):
+    worker_devices = self._cpu_devices()
+    with context.graph_mode(), self.cached_session() as sess:
+      def dataset_fn():
+        dataset1 = dataset_ops.Dataset.range(4)
+        dataset2 = dataset_ops.Dataset.range(4).map(lambda x: x**2)
+        return dataset_ops.Dataset.zip((dataset1, dataset2))
+
+      expected_values = [[(i, i**2), (i, i**2)] for i in range(0, 4)]
+      self._test_iterator(input_type, dataset_fn, worker_devices,
+                          expected_values, sess)
+
+
+class SplitDatasetBatchTest(test.TestCase):
+
+  def testBatchDataset(self):
+    dataset = dataset_ops.Dataset.range(100).batch(20)
+    split_batch_by = 2
+    result_dataset = input_lib._split_dataset_batch(dataset, split_batch_by)
+    expected_values = [range(i, i+10) for i in range(0, 100, 10)]
+    result = [self.evaluate(el) for el in result_dataset]
+    self.assertAllEqual(expected_values, result)
+
+  def testMapAndBatchDataset(self):
+    dataset = dataset_ops.Dataset.range(100)
+    dataset = dataset.apply(batching.map_and_batch(lambda x: x, 20))
+    split_batch_by = 2
+    result_dataset = input_lib._split_dataset_batch(dataset, split_batch_by)
+    expected_values = [range(i, i+10) for i in range(0, 100, 10)]
+    result = [self.evaluate(el) for el in result_dataset]
+    self.assertAllEqual(expected_values, result)
+
+  def testPrefetchDataset(self):
+    dataset = dataset_ops.Dataset.range(100).batch(20).prefetch(1)
+    split_batch_by = 2
+    result_dataset = input_lib._split_dataset_batch(dataset, split_batch_by)
+    expected_values = [range(i, i+10) for i in range(0, 100, 10)]
+    result = [self.evaluate(el) for el in result_dataset]
+    self.assertAllEqual(expected_values, result)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy.py b/tensorflow/contrib/distribute/python/mirrored_strategy.py
index 71e50b83b0..db8fd98307 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy.py
@@ -21,8 +21,8 @@ from __future__ import print_function
 import functools
 
 from tensorflow.python.distribute import distribute_lib
+from tensorflow.python.distribute import input_lib
 from tensorflow.python.distribute import mirrored_strategy
-from tensorflow.python.distribute import values
 
 
 # pylint: disable=protected-access,invalid-name
@@ -135,14 +135,14 @@ class MirroredExtended(CoreMirroredExtended):
     Returns:
       An `InputIterator` which returns inputs for each step of the computation.
     """
-    return values.DatasetIterator(dataset, self._input_workers)
+    return input_lib.DatasetIterator(dataset, self._input_workers)
 
   def _distribute_dataset(self, dataset_fn):
     if self._local_mode:
-      return values.PerReplicaDataset(
+      return input_lib.PerReplicaDataset(
           self._call_dataset_fn(dataset_fn), self._input_workers, 0)
     else:
-      return values.MultiWorkerDataset(
+      return input_lib.MultiWorkerDataset(
           functools.partial(self._call_dataset_fn, dataset_fn),
           self._input_workers,
           auto_shard=self._auto_shard_dataset)
diff --git a/tensorflow/contrib/distribute/python/one_device_strategy.py b/tensorflow/contrib/distribute/python/one_device_strategy.py
index 700751d68c..fb470f8546 100644
--- a/tensorflow/contrib/distribute/python/one_device_strategy.py
+++ b/tensorflow/contrib/distribute/python/one_device_strategy.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 
 from tensorflow.python.distribute import device_util
 from tensorflow.python.distribute import distribute_lib
+from tensorflow.python.distribute import input_lib
 from tensorflow.python.distribute import values
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -52,7 +53,8 @@ class OneDeviceExtended(distribute_lib.DistributionStrategyExtended):
     worker = device_util.canonicalize("/device:CPU:0")
     worker_device_pairs = [(worker, [self._device])]
     device_map = values.SingleDeviceMap(device)
-    self._input_workers = values.InputWorkers(device_map, worker_device_pairs)
+    self._input_workers = input_lib.InputWorkers(
+        device_map, worker_device_pairs)
 
   def _create_variable(self, next_creator, *args, **kwargs):
     colocate_with = kwargs.pop("colocate_with", None)
@@ -67,17 +69,17 @@ class OneDeviceExtended(distribute_lib.DistributionStrategyExtended):
 
   def _make_dataset_iterator(self, dataset):
     """Make iterator from dataset without splitting the batch."""
-    return values.DatasetIterator(dataset, self._input_workers)
+    return input_lib.DatasetIterator(dataset, self._input_workers)
 
   def _distribute_dataset(self, dataset_fn):
-    return values.PerReplicaDataset(
+    return input_lib.PerReplicaDataset(
         self._call_dataset_fn(dataset_fn), self._input_workers, 0)
 
   def _make_input_fn_iterator(
       self,
       input_fn,
       replication_mode=distribute_lib.InputReplicationMode.PER_WORKER):
-    return values.InputFunctionIterator(
+    return input_lib.InputFunctionIterator(
         input_fn, self._input_workers, [distribute_lib.InputContext()])
 
   def _broadcast_to(self, tensor, destinations):
@@ -91,7 +93,7 @@ class OneDeviceExtended(distribute_lib.DistributionStrategyExtended):
       initial_loop_values = {}
     initial_loop_values = nest.flatten(initial_loop_values)
 
-    ctx = values.MultiStepContext()
+    ctx = input_lib.MultiStepContext()
     def body(i, *args):
       """A wrapper around `fn` to create the while loop body."""
       del args
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy.py b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
index a6e924b509..461e1bca21 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
@@ -24,6 +24,7 @@ from tensorflow.contrib.distribute.python import mirrored_strategy
 from tensorflow.python.distribute import cross_device_ops as cross_device_ops_lib
 from tensorflow.python.distribute import device_util
 from tensorflow.python.distribute import distribute_lib
+from tensorflow.python.distribute import input_lib
 from tensorflow.python.distribute import multi_worker_util
 from tensorflow.python.distribute import values
 from tensorflow.python.eager import context
@@ -153,7 +154,7 @@ class ParameterServerExtended(distribute_lib.DistributionStrategyExtended):
       compute_devices = (worker_device,)
 
     self._device_map = values.ReplicaDeviceMap(compute_devices)
-    self._input_workers = values.InputWorkers(
+    self._input_workers = input_lib.InputWorkers(
         self._device_map, [(worker_device, compute_devices)])
 
     # In distributed mode, place variables on ps jobs in a round-robin fashion.
@@ -210,7 +211,7 @@ class ParameterServerExtended(distribute_lib.DistributionStrategyExtended):
       compute_devices = (_LOCAL_CPU,)
 
     self._device_map = values.ReplicaDeviceMap(compute_devices)
-    self._input_workers = values.InputWorkers(
+    self._input_workers = input_lib.InputWorkers(
         self._device_map, [(worker_device, compute_devices)])
 
     # If there is only one GPU, put everything on that GPU. Otherwise, place
@@ -237,13 +238,13 @@ class ParameterServerExtended(distribute_lib.DistributionStrategyExtended):
 
   def _distribute_dataset(self, dataset_fn):
     """Distributes the dataset to each local GPU."""
-    return values.PerReplicaDataset(
+    return input_lib.PerReplicaDataset(
         self._call_dataset_fn(dataset_fn), self._input_workers, 0,
         prefetch_on_device=True)
 
   def _make_dataset_iterator(self, dataset):
-    return values.DatasetIterator(dataset, self._input_workers,
-                                  self._num_replicas_in_sync)
+    return input_lib.DatasetIterator(dataset, self._input_workers,
+                                     self._num_replicas_in_sync)
 
   def _make_input_fn_iterator(
       self,
@@ -262,7 +263,7 @@ class ParameterServerExtended(distribute_lib.DistributionStrategyExtended):
         num_input_pipelines=num_input_pipelines,
         input_pipeline_id=input_pipeline_id,
         num_replicas_in_sync=self._num_replicas_in_sync)
-    return values.InputFunctionIterator(
+    return input_lib.InputFunctionIterator(
         input_fn, self._input_workers, [input_context])
 
   def _broadcast_to(self, tensor, destinations):
diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index 89b48d3f13..10b7ef0407 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -33,6 +33,7 @@ from tensorflow.python.client import session as session_lib
 from tensorflow.python.distribute import cross_device_ops as cross_device_ops_lib
 from tensorflow.python.distribute import device_util
 from tensorflow.python.distribute import distribute_lib
+from tensorflow.python.distribute import input_lib
 from tensorflow.python.distribute import reduce_util
 from tensorflow.python.distribute import values
 from tensorflow.python.distribute.cluster_resolver import tpu_cluster_resolver as resolver_lib
@@ -204,7 +205,8 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
         (self.get_host(hid), [self.get_host_cpu_device(hid)])
         for hid in range(self.num_hosts)
     ]
-    self._input_workers = values.InputWorkers(input_device_map, worker_devices)
+    self._input_workers = input_lib.InputWorkers(
+        input_device_map, worker_devices)
 
     # TODO(sourabhbajaj): Remove this once performance of running one step
     # at a time is comparable to multiple steps.
@@ -304,11 +306,11 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
   def _make_dataset_iterator(self, dataset):
     """Make iterators for each of the TPU hosts."""
 
-    return values.DatasetIterator(dataset, self._input_workers,
-                                  self._num_replicas_in_sync)
+    return input_lib.DatasetIterator(dataset, self._input_workers,
+                                     self._num_replicas_in_sync)
 
   def _distribute_dataset(self, dataset_fn):
-    return values.MultiWorkerDataset(
+    return input_lib.MultiWorkerDataset(
         functools.partial(self._call_dataset_fn, dataset_fn),
         self._input_workers)
 
@@ -339,7 +341,7 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
     if initial_loop_values is None:
       initial_loop_values = {}
     initial_loop_values = nest.flatten(initial_loop_values)
-    ctx = values.MultiStepContext()
+    ctx = input_lib.MultiStepContext()
 
     def run_fn(*args, **kwargs):
       """Single step on the TPU device."""
diff --git a/tensorflow/contrib/distribute/python/values_test.py b/tensorflow/contrib/distribute/python/values_test.py
index 73efb524b9..51c58b0b2f 100644
--- a/tensorflow/contrib/distribute/python/values_test.py
+++ b/tensorflow/contrib/distribute/python/values_test.py
@@ -22,28 +22,20 @@ import os
 from absl.testing import parameterized
 
 from tensorflow.contrib.distribute.python import combinations
-from tensorflow.contrib.distribute.python import multi_worker_test_base
 from tensorflow.core.protobuf import config_pb2
-from tensorflow.python.data.experimental.ops import batching
-from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.distribute import device_util
-from tensorflow.python.distribute import distribute_lib
 from tensorflow.python.distribute import values
 from tensorflow.python.eager import context
 from tensorflow.python.eager import test
 from tensorflow.python.estimator import model_fn as model_fn_lib
 from tensorflow.python.framework import constant_op
-from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import control_flow_ops
-from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables as variables_lib
 from tensorflow.python.training import saver as saver_lib
-from tensorflow.python.util import nest
 
 
 class DistributedValuesTest(test.TestCase):
@@ -354,444 +346,6 @@ class RegroupAndSelectDeviceTest(test.TestCase):
                                                merged_estimator_spec))
 
 
-class PerReplicaDatasetTest(test.TestCase):
-
-  config = config_pb2.ConfigProto()
-  config.allow_soft_placement = True
-
-  def _test_iterator(self, devices, dataset, expected_values):
-    device_map = values.ReplicaDeviceMap(devices)
-    input_workers = values.InputWorkers(device_map)
-    per_replica_dataset = values.PerReplicaDataset(dataset, input_workers, 0)
-    if context.executing_eagerly():
-      iterator = per_replica_dataset.make_one_shot_iterator()
-    else:
-      iterator = per_replica_dataset.make_initializable_iterator()
-      self.evaluate([iterator.initializer])
-
-    for expected_value in expected_values:
-      next_element = iterator.get_next_as_list()
-      computed_value = self.evaluate(next_element)
-      self.assertEqual(expected_value, computed_value)
-
-    with self.assertRaises(errors.OutOfRangeError):
-      next_element = iterator.get_next_as_list()
-      self.evaluate(next_element)
-
-  @test_util.run_in_graph_and_eager_modes
-  def testOneDevice(self):
-    devices = ["/device:CPU:0"]
-    dataset = dataset_ops.Dataset.range(10)
-
-    expected_values = [[i] for i in range(10)]
-
-    self._test_iterator(devices, dataset, expected_values)
-
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testMultipleDevices(self):
-    if context.num_gpus() < 1 and context.executing_eagerly():
-      self.skipTest("A GPU is not available for this test in eager mode.")
-
-    devices = ["/device:CPU:0", "/device:GPU:0"]
-    dataset = dataset_ops.Dataset.range(10)
-
-    expected_values = [[i, i+1] for i in range(0, 10, 2)]
-
-    self._test_iterator(devices, dataset, expected_values)
-
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testTupleDataset(self):
-    if context.num_gpus() < 1 and context.executing_eagerly():
-      self.skipTest("A GPU is not available for this test in eager mode.")
-
-    devices = ["/device:CPU:0", "/device:GPU:0"]
-    dataset1 = dataset_ops.Dataset.range(10)
-    dataset2 = dataset_ops.Dataset.range(10).map(lambda x: x**2)
-    dataset = dataset_ops.Dataset.zip((dataset1, dataset2))
-
-    expected_values = [[(i, i**2), (i+1, (i+1)**2)] for i in range(0, 10, 2)]
-
-    self._test_iterator(devices, dataset, expected_values)
-
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testUnevenDatasetBatches(self):
-    if context.num_gpus() < 1 and context.executing_eagerly():
-      self.skipTest("A GPU is not available for this test in eager mode.")
-
-    devices = ["/device:CPU:0", "/device:GPU:0"]
-    dataset = dataset_ops.Dataset.range(11)
-
-    expected_values = [[i, i+1] for i in range(0, 10, 2)]
-    self._test_iterator(devices, dataset, expected_values)
-
-  def testInitializableIterator(self):
-    with context.graph_mode():
-      devices = ["/device:CPU:0"]
-      # Using random input since that is only allowed with initializable
-      # iterator.
-      dataset = dataset_ops.Dataset.from_tensor_slices(
-          random_ops.random_uniform((10,)))
-
-      device_map = values.ReplicaDeviceMap(devices)
-      input_workers = values.InputWorkers(device_map)
-      per_replica_dataset = values.PerReplicaDataset(dataset, input_workers, 0)
-      iterator = per_replica_dataset.make_initializable_iterator()
-
-      self.evaluate(iterator.initializer)
-      next_element = iterator.get_next_as_list()
-      for _ in range(10):
-        self.evaluate(next_element)
-
-      # Should fail after the input is finished.
-      with self.assertRaises(errors.OutOfRangeError):
-        self.evaluate(next_element)
-
-      # After re-initializing the iterator, should be able to iterate again.
-      self.evaluate(iterator.initializer)
-      for _ in range(10):
-        self.evaluate(next_element)
-
-
-class MultiWorkerDatasetTest(multi_worker_test_base.MultiWorkerTestBase):
-
-  def _test_iterator(self, sess, iterator, devices, expected_values):
-    next_element = iterator.get_next()
-    for r, device in enumerate(devices):
-      v = values.select_replica(r, next_element)
-      # The `v` here can be a tuple.
-      for element in nest.flatten(v):
-        self.assertTrue(element.device in device)
-
-    for expected_value in expected_values:
-      t = [values.select_replica(r, next_element) for r in range(len(devices))]
-      actual = sess.run(t)
-      self.assertEqual(expected_value, actual)
-
-    with self.assertRaises(errors.OutOfRangeError):
-      sess.run([values.select_replica(r, next_element)
-                for r in range(len(devices))])
-
-  def _test_dataset(self, dataset_fn, worker_devices, devices,
-                    expected_values):
-    device_map = values.ReplicaDeviceMap(devices)
-    input_workers = values.InputWorkers(device_map, worker_devices)
-    multi_worker_dataset = values.MultiWorkerDataset(
-        dataset_fn, input_workers)
-    multi_worker_iterator = multi_worker_dataset.make_initializable_iterator()
-    with self.cached_session() as sess:
-      sess.run(multi_worker_iterator.initializer)
-      self._test_iterator(sess, multi_worker_iterator, devices, expected_values)
-
-  def _cpu_devices(self):
-    worker_devices = (
-        ("/job:worker/replica:0/task:0",
-         ["/job:worker/replica:0/task:0/device:CPU:0"]),
-        ("/job:worker/replica:0/task:1",
-         ["/job:worker/replica:0/task:1/device:CPU:0"])
-    )
-    devices = [
-        "/job:worker/replica:0/task:0/device:CPU:0",
-        "/job:worker/replica:0/task:1/device:CPU:0"
-    ]
-    return worker_devices, devices
-
-  def _cpu_and_one_gpu_devices(self):
-    worker_devices = (
-        ("/job:worker/replica:0/task:0", (
-            "/job:worker/replica:0/task:0/device:GPU:0",
-            "/job:worker/replica:0/task:0/device:CPU:0"
-        )),
-        ("/job:worker/replica:0/task:1", (
-            "/job:worker/replica:0/task:1/device:GPU:0",
-            "/job:worker/replica:0/task:1/device:CPU:0"
-        ))
-    )
-    devices = [
-        "/job:worker/replica:0/task:0/device:GPU:0",
-        "/job:worker/replica:0/task:0/device:CPU:0",
-        "/job:worker/replica:0/task:1/device:GPU:0",
-        "/job:worker/replica:0/task:1/device:CPU:0"
-    ]
-    return worker_devices, devices
-
-  def testDataDistributionOneDevicePerWorker(self):
-    worker_devices, devices = self._cpu_devices()
-    with context.graph_mode():
-      dataset_fn = lambda: dataset_ops.Dataset.range(8)
-      self._test_dataset(
-          dataset_fn, worker_devices, devices,
-          [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4], [5, 5], [6, 6], [7, 7]])
-
-  def testDataDistributionTwoDevicePerWorker(self):
-    if context.num_gpus() < 1:
-      self.skipTest("A GPU is not available for this test.")
-    worker_devices, devices = self._cpu_and_one_gpu_devices()
-    with context.graph_mode():
-      dataset_fn = lambda: dataset_ops.Dataset.range(8)
-      self._test_dataset(
-          dataset_fn, worker_devices, devices,
-          [[0, 1, 0, 1], [2, 3, 2, 3], [4, 5, 4, 5], [6, 7, 6, 7]])
-
-  def testTupleDataset(self):
-    worker_devices, devices = self._cpu_devices()
-
-    with context.graph_mode():
-
-      def dataset_fn():
-        dataset1 = dataset_ops.Dataset.range(8)
-        dataset2 = dataset_ops.Dataset.range(8).map(lambda x: x**2)
-        return dataset_ops.Dataset.zip((dataset1, dataset2))
-
-      expected_values = [[(i, i**2), (i, i**2)] for i in range(8)]
-      self._test_dataset(dataset_fn, worker_devices, devices,
-                         expected_values)
-
-  def testInitializableIterator(self):
-    worker_devices, devices = self._cpu_devices()
-    with context.graph_mode(), self.cached_session() as sess:
-      dataset_fn = lambda: dataset_ops.Dataset.range(8)
-      device_map = values.ReplicaDeviceMap(devices)
-      input_workers = values.InputWorkers(device_map, worker_devices)
-      multi_worker_dataset = values.MultiWorkerDataset(
-          dataset_fn, input_workers)
-      multi_worker_iterator = multi_worker_dataset.make_initializable_iterator()
-
-      sess.run(multi_worker_iterator.initializer)
-      self._test_iterator(
-          sess, multi_worker_iterator, devices,
-          [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4], [5, 5], [6, 6], [7, 7]])
-
-      # After re-initializing the iterator, should be able to iterate again.
-      sess.run(multi_worker_iterator.initializer)
-      self._test_iterator(
-          sess, multi_worker_iterator, devices,
-          [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4], [5, 5], [6, 6], [7, 7]])
-
-  def testValueErrorForIterator(self):
-    # Incompatiable arguments.
-    d1 = "/device:GPU:0"
-    d2 = "/device:GPU:1"
-    device_map = values.ReplicaDeviceMap([d1, d2])
-    input_workers = values.InputWorkers(
-        device_map, (("w1", (d1,)), ("w2", (d2,))))
-    with self.assertRaises(ValueError):
-      values.MultiWorkerDataIterator([("w1", None)], input_workers)
-
-  def testDuplicateDevices(self):
-    _, devices = self._cpu_devices()
-    devices.append("/job:worker/replica:0/task:0/device:CPU:0")
-    with self.assertRaises(ValueError):
-      _ = values.ReplicaDeviceMap(devices)
-
-
-class InputIteratorTestBase(test.TestCase):
-
-  def _test_iterator(self, input_type, dataset_fn, worker_device_pairs,
-                     expected_values, sess=None, split_batch_by=None):
-    devices = nest.flatten([ds for _, ds in worker_device_pairs])
-    device_map = values.ReplicaDeviceMap(devices)
-    input_workers = values.InputWorkers(device_map, worker_device_pairs)
-
-    if input_type == "input_fn":
-      input_contexts = [
-          distribute_lib.InputContext() for _ in worker_device_pairs]
-      input_fn = lambda _: dataset_fn()
-      iterator = values.InputFunctionIterator(
-          input_fn, input_workers, input_contexts)
-    else:
-      iterator = values.DatasetIterator(
-          dataset_fn(), input_workers, split_batch_by)
-
-    evaluate = lambda x: sess.run(x) if sess else self.evaluate(x)
-
-    evaluate(control_flow_ops.group(iterator.initialize()))
-
-    for expected_value in expected_values:
-      next_element = iterator.get_next()
-      computed_value = evaluate(
-          [values.select_replica(r, next_element) for r in range(len(devices))])
-      self.assertAllEqual(expected_value, computed_value)
-
-    with self.assertRaises(errors.OutOfRangeError):
-      next_element = iterator.get_next()
-      evaluate([values.select_replica(r, next_element)
-                for r in range(len(devices))])
-
-    # After re-initializing the iterator, should be able to iterate again.
-    evaluate(control_flow_ops.group(iterator.initialize()))
-
-    for expected_value in expected_values:
-      next_element = iterator.get_next()
-      computed_value = evaluate(
-          [values.select_replica(r, next_element) for r in range(len(devices))])
-      self.assertAllEqual(expected_value, computed_value)
-
-
-class InputIteratorSingleWorkerTest(InputIteratorTestBase,
-                                    parameterized.TestCase):
-
-  @combinations.generate(combinations.combine(
-      mode=["graph", "eager"],
-      input_type=["input_fn", "dataset"]))
-  def testOneDeviceCPU(self, input_type):
-    worker_device_pairs = [("", ["/device:CPU:0"])]
-    dataset_fn = lambda: dataset_ops.Dataset.range(10)
-
-    expected_values = [[i] for i in range(10)]
-
-    self._test_iterator(input_type, dataset_fn, worker_device_pairs,
-                        expected_values)
-
-  @combinations.generate(combinations.combine(
-      mode=["graph", "eager"],
-      input_type=["input_fn", "dataset"],
-      required_gpus=1))
-  def testTwoDevicesOneGPUOneCPU(self, input_type):
-    worker_device_pairs = [("", ["/device:GPU:0", "/device:CPU:0"])]
-    dataset_fn = lambda: dataset_ops.Dataset.range(10)
-
-    expected_values = [[i, i+1] for i in range(0, 10, 2)]
-
-    self._test_iterator(input_type, dataset_fn, worker_device_pairs,
-                        expected_values)
-
-  @combinations.generate(combinations.combine(
-      mode=["graph", "eager"],
-      input_type=["input_fn", "dataset"],
-      required_gpus=1))
-  def testTupleDataset(self, input_type):
-    worker_device_pairs = [("", ["/device:GPU:0", "/device:CPU:0"])]
-    def dataset_fn():
-      dataset1 = dataset_ops.Dataset.range(10)
-      dataset2 = dataset_ops.Dataset.range(10).map(lambda x: x**2)
-      return dataset_ops.Dataset.zip((dataset1, dataset2))
-
-    expected_values = [[(i, i**2), (i+1, (i+1)**2)] for i in range(0, 10, 2)]
-
-    self._test_iterator(input_type, dataset_fn, worker_device_pairs,
-                        expected_values)
-
-  @combinations.generate(combinations.combine(
-      mode=["graph", "eager"],
-      input_type=["input_fn", "dataset"],
-      required_gpus=1))
-  def testUnevenDatasetBatches(self, input_type):
-    worker_device_pairs = [("", ["/device:GPU:0", "/device:CPU:0"])]
-    dataset_fn = lambda: dataset_ops.Dataset.range(11)
-
-    expected_values = [[i, i+1] for i in range(0, 10, 2)]
-    self._test_iterator(input_type, dataset_fn, worker_device_pairs,
-                        expected_values)
-
-  @combinations.generate(combinations.combine(
-      mode=["graph", "eager"],
-      input_type=["dataset"],
-      split_batch_by=[None, 2],
-      required_gpus=1))
-  def testBatchSplitting(self, input_type, split_batch_by):
-    worker_device_pairs = [("", ["/device:GPU:0", "/device:CPU:0"])]
-    batch_size = 10
-    dataset_fn = lambda: dataset_ops.Dataset.range(100).batch(batch_size)
-
-    updated_batch_size = (
-        batch_size // split_batch_by if split_batch_by else batch_size)
-    expected_values = [[range(i, i+updated_batch_size),
-                        range(i+updated_batch_size, i+2*updated_batch_size)]
-                       for i in range(0, 100, updated_batch_size*2)]
-
-    self._test_iterator(input_type, dataset_fn, worker_device_pairs,
-                        expected_values, sess=None,
-                        split_batch_by=split_batch_by)
-
-
-class InputIteratorMultiWorkerTest(
-    multi_worker_test_base.MultiWorkerTestBase, InputIteratorTestBase,
-    parameterized.TestCase):
-
-  def _cpu_devices(self):
-    return [
-        ("/job:worker/replica:0/task:0",
-         ["/job:worker/replica:0/task:0/device:CPU:0"]),
-        ("/job:worker/replica:0/task:1",
-         ["/job:worker/replica:0/task:1/device:CPU:0"])]
-
-  def _cpu_and_one_gpu_devices(self):
-    return [
-        ("/job:worker/replica:0/task:0", [
-            "/job:worker/replica:0/task:0/device:GPU:0",
-            "/job:worker/replica:0/task:0/device:CPU:0"
-        ]),
-        ("/job:worker/replica:0/task:1", [
-            "/job:worker/replica:0/task:1/device:GPU:0",
-            "/job:worker/replica:0/task:1/device:CPU:0"
-        ])
-    ]
-
-  @combinations.generate(combinations.combine(
-      mode=["graph"],
-      input_type=["input_fn", "dataset"]))
-  def testOneDevicePerWorker(self, input_type):
-    worker_devices = self._cpu_devices()
-    with context.graph_mode(), self.cached_session() as sess:
-      dataset_fn = lambda: dataset_ops.Dataset.range(4)
-      self._test_iterator(input_type, dataset_fn, worker_devices,
-                          [[0, 0], [1, 1], [2, 2], [3, 3]], sess)
-
-  @combinations.generate(combinations.combine(
-      mode=["graph"],
-      input_type=["input_fn", "dataset"],
-      required_gpus=1))
-  def testTwoDevicesPerWorker(self, input_type):
-    worker_devices = self._cpu_and_one_gpu_devices()
-    with context.graph_mode(), self.cached_session() as sess:
-      dataset_fn = lambda: dataset_ops.Dataset.range(4)
-      self._test_iterator(input_type, dataset_fn, worker_devices,
-                          [[0, 1, 0, 1], [2, 3, 2, 3]], sess)
-
-  @combinations.generate(combinations.combine(
-      mode=["graph"],
-      input_type=["input_fn", "dataset"]))
-  def testTupleDataset(self, input_type):
-    worker_devices = self._cpu_devices()
-    with context.graph_mode(), self.cached_session() as sess:
-      def dataset_fn():
-        dataset1 = dataset_ops.Dataset.range(4)
-        dataset2 = dataset_ops.Dataset.range(4).map(lambda x: x**2)
-        return dataset_ops.Dataset.zip((dataset1, dataset2))
-
-      expected_values = [[(i, i**2), (i, i**2)] for i in range(0, 4)]
-      self._test_iterator(input_type, dataset_fn, worker_devices,
-                          expected_values, sess)
-
-
-class SplitDatasetBatchTest(test.TestCase):
-
-  def testBatchDataset(self):
-    dataset = dataset_ops.Dataset.range(100).batch(20)
-    split_batch_by = 2
-    result_dataset = values._split_dataset_batch(dataset, split_batch_by)
-    expected_values = [range(i, i+10) for i in range(0, 100, 10)]
-    result = [self.evaluate(el) for el in result_dataset]
-    self.assertAllEqual(expected_values, result)
-
-  def testMapAndBatchDataset(self):
-    dataset = dataset_ops.Dataset.range(100)
-    dataset = dataset.apply(batching.map_and_batch(lambda x: x, 20))
-    split_batch_by = 2
-    result_dataset = values._split_dataset_batch(dataset, split_batch_by)
-    expected_values = [range(i, i+10) for i in range(0, 100, 10)]
-    result = [self.evaluate(el) for el in result_dataset]
-    self.assertAllEqual(expected_values, result)
-
-  def testPrefetchDataset(self):
-    dataset = dataset_ops.Dataset.range(100).batch(20).prefetch(1)
-    split_batch_by = 2
-    result_dataset = values._split_dataset_batch(dataset, split_batch_by)
-    expected_values = [range(i, i+10) for i in range(0, 100, 10)]
-    result = [self.evaluate(el) for el in result_dataset]
-    self.assertAllEqual(expected_values, result)
-
-
 class MirroredVariableTest(test.TestCase, parameterized.TestCase):
 
   config = config_pb2.ConfigProto()
diff --git a/tensorflow/python/distribute/BUILD b/tensorflow/python/distribute/BUILD
index 02957b2fef..987fb00454 100644
--- a/tensorflow/python/distribute/BUILD
+++ b/tensorflow/python/distribute/BUILD
@@ -219,6 +219,7 @@ py_library(
         ":cross_device_ops",
         ":device_util",
         ":distribute_lib",
+        ":input_lib",
         ":multi_worker_util",
         ":reduce_util",
         ":shared_variable_creator",
@@ -253,6 +254,23 @@ py_library(
     ],
 )
 
+py_library(
+    name = "input_lib",
+    srcs = ["input_lib.py"],
+    deps = [
+        ":device_util",
+        ":distribute_lib",
+        ":input_ops",
+        ":values",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:control_flow_ops",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:util",
+        "//tensorflow/python/data/ops:multi_device_iterator_ops",
+        "//tensorflow/python/eager:context",
+    ],
+)
+
 py_library(
     name = "input_ops",
     srcs = ["input_ops.py"],
@@ -348,14 +366,12 @@ py_library(
     deps = [
         ":device_util",
         ":distribute_lib",
-        ":input_ops",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:control_flow_ops",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:resource_variable_ops",
         "//tensorflow/python:training",
         "//tensorflow/python:util",
-        "//tensorflow/python/data/ops:multi_device_iterator_ops",
         "//tensorflow/python/eager:context",
         "//tensorflow/python/training/checkpointable:base",
         "@six_archive//:six",
diff --git a/tensorflow/python/distribute/input_lib.py b/tensorflow/python/distribute/input_lib.py
new file mode 100644
index 0000000000..cbe6518e5c
--- /dev/null
+++ b/tensorflow/python/distribute/input_lib.py
@@ -0,0 +1,707 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Various classes representing distributed inputs."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.data.experimental.ops import batching
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.data.ops import multi_device_iterator_ops
+from tensorflow.python.distribute import device_util
+from tensorflow.python.distribute import distribution_strategy_context
+from tensorflow.python.distribute import input_ops
+from tensorflow.python.distribute import values
+from tensorflow.python.eager import context
+from tensorflow.python.framework import device as tf_device
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_util
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.util import nest
+
+
+class InputWorkers(object):
+  """A 1-to-many mapping from input worker devices to compute devices."""
+
+  def __init__(self, device_map, worker_device_pairs=None, logical_device=0):
+    """Initialize an `InputWorkers` object.
+
+    Args:
+      device_map: A `DeviceMap` with the computation devices fed by the
+        input workers.
+      worker_device_pairs: A sequence of pairs:
+        `(input device, a tuple of compute devices fed by that input device)`.
+      logical_device: The logical device of `device_map` to feed.
+    """
+    self._device_map = device_map
+    self._logical_device = logical_device
+    if worker_device_pairs is None:
+      worker_device_pairs = ((
+          device_util.canonicalize("/device:CPU:0"),
+          device_map.logical_to_actual_devices(logical_device)),)
+    self._input_worker_devices = tuple(d for d, _ in worker_device_pairs)
+    self._fed_devices = tuple(tuple(device_util.canonicalize(d) for d in f)
+                              for _, f in worker_device_pairs)
+    flattened = tuple(d for l in self._fed_devices for d in l)
+    assert (flattened ==
+            device_map.logical_to_actual_devices(logical_device)), (
+                "flattened: %s logical device %d: %s" %
+                (flattened, logical_device,
+                 device_map.logical_to_actual_devices(logical_device)))
+
+  @property
+  def device_map(self):
+    return self._device_map
+
+  @property
+  def logical_device(self):
+    return self._logical_device
+
+  @property
+  def num_workers(self):
+    return len(self._input_worker_devices)
+
+  @property
+  def worker_devices(self):
+    return self._input_worker_devices
+
+  def compute_devices_for_worker(self, worker_index):
+    return self._fed_devices[worker_index]
+
+  def __repr__(self):
+    devices = self.worker_devices
+    debug_repr = ",\n".join("  %d %s: %s" %
+                            (i, devices[i], self._fed_devices[i])
+                            for i in range(len(devices)))
+    return "%s:{\n%s\n  device_map: %s}" % (
+        self.__class__.__name__, debug_repr, self._device_map)
+
+
+class PerReplicaDataIterator(object):
+  """An iterator (like `tf.data.Iterator`) into a `PerReplicaDataset`."""
+
+  def __init__(self, iterator, input_workers, worker_index, prefetch_on_device):
+    assert isinstance(input_workers, InputWorkers)
+    self._iterator = iterator
+    self._input_workers = input_workers
+    self._worker_index = worker_index
+    self._prefetch_on_device = prefetch_on_device
+
+  @property
+  def initializer(self):
+    return self._iterator.initializer
+
+  def get_next_as_list(self, name=None):
+    """Scatter the input across devices."""
+    if self._prefetch_on_device:
+      data_list = self._iterator.get_next()
+    else:
+      batch = self._iterator.get_next(name=name)
+      data_list = []
+      def get_ith(i):
+        return lambda x: x[i]
+
+      devices = self._input_workers.compute_devices_for_worker(
+          self._worker_index)
+      for i, d in enumerate(devices):
+        v = nest.map_structure(get_ith(i), batch)
+        if context.executing_eagerly():
+          with ops.device(d):
+            v = nest.map_structure(array_ops.identity, v)
+        data_list.append(v)
+
+    return data_list
+
+  def get_next(self, name=None):
+    assert self._input_workers.num_workers == 1
+    data_list = self.get_next_as_list(name)
+    return values.regroup(self._input_workers.device_map, data_list)
+
+  @property
+  def output_classes(self):
+    return self._iterator.output_classes
+
+  @property
+  def output_shapes(self):
+    return self._iterator.output_shapes
+
+  @property
+  def output_types(self):
+    return self._iterator.output_types
+
+
+class PerReplicaDataset(object):
+  """Like `tf.data.Dataset` split devices, producing `PerReplica` data."""
+
+  def __init__(self, dataset, input_workers, worker_index,
+               prefetch_on_device=None):
+    assert isinstance(input_workers, InputWorkers)
+    assert worker_index is not None
+    assert worker_index is not True  # pylint: disable=g-bool-id-comparison
+    assert worker_index is not False  # pylint: disable=g-bool-id-comparison
+    self._input_workers = input_workers
+    self._worker_index = worker_index
+
+    # Default to using prefetching, unless specified.
+    self._prefetch_on_device = prefetch_on_device
+    if self._prefetch_on_device is None:
+      self._prefetch_on_device = True
+
+    self._dataset = dataset
+    if not self._prefetch_on_device:
+      # TODO(priyag): If dropping remainder is not appropriate, find another
+      # approach to distributing the dataset when not possible to divide evenly.
+      # Possibly not an issue when we start using PartitionedDataset.
+      num_replicas = len(
+          self._input_workers.compute_devices_for_worker(self._worker_index))
+      self._dataset = self._dataset.batch(num_replicas, drop_remainder=True)
+    else:
+      self._replica_devices = self._input_workers.compute_devices_for_worker(
+          self._worker_index)
+
+  def make_one_shot_iterator(self):
+    """Get a one time use iterator for the distributed PerReplicaDataset."""
+    # Graph mode with one shot iterator is disabled.
+    if not context.executing_eagerly():
+      raise ValueError("Cannot create a one shot iterator. Please use "
+                       "`make_initializable_iterator()` instead.")
+    if self._prefetch_on_device:
+      dataset_iterator = multi_device_iterator_ops.MultiDeviceIterator(
+          self._dataset, self._replica_devices)
+    else:
+      dataset_iterator = dataset_ops.make_one_shot_iterator(self._dataset)
+    return PerReplicaDataIterator(
+        dataset_iterator,
+        self._input_workers,
+        self._worker_index,
+        prefetch_on_device=self._prefetch_on_device)
+
+  def make_initializable_iterator(self):
+    """Get an initializable iterator for the distributed PerReplicaDataset."""
+    # Eager mode generates already initialized iterators. Hence we cannot create
+    # an initializable iterator.
+    if context.executing_eagerly():
+      raise ValueError("Cannot create initializable iterator in Eager mode. "
+                       "Please use `make_one_shot_iterator` instead.")
+    if self._prefetch_on_device:
+      dataset_iterator = multi_device_iterator_ops.MultiDeviceIterator(
+          self._dataset, self._replica_devices)
+    else:
+      dataset_iterator = dataset_ops.make_initializable_iterator(self._dataset)
+    return PerReplicaDataIterator(
+        dataset_iterator, self._input_workers, self._worker_index,
+        prefetch_on_device=self._prefetch_on_device)
+
+
+class MultiWorkerDataIterator(object):
+  """An iterator (like `tf.data.Iterator`) into a `MultiWorkerDataset`."""
+
+  def __init__(self, iterators, input_workers):
+    """Initialize the `MultiWorkerDataIterator` object.
+
+    Args:
+      iterators: a list of worker, iterator pairs.
+      input_workers: an `InputWorkers` object.
+
+    Raises:
+      ValueError: if iterators and input_workers are not compatible.
+    """
+    assert isinstance(input_workers, InputWorkers)
+    workers = tuple(d for d, _ in iterators)
+    if workers != input_workers.worker_devices:
+      raise ValueError("iterators and input_workers are not compatible. "
+                       "iterator workers: %r input_workers devices: %r" %
+                       (workers, input_workers.worker_devices))
+    self._iterators = tuple(i for _, i in iterators)
+    self._input_workers = input_workers
+
+  @property
+  def initializer(self):
+    return control_flow_ops.group(
+        tuple(iterator.initializer for iterator in self._iterators))
+
+  def get_iterator(self, worker):
+    for i, w in enumerate(self._input_workers.worker_devices):
+      if worker == w:
+        return self._iterators[i]
+    return None
+
+  @property
+  def output_shapes(self):
+    return self._iterators[0].output_shapes
+
+  @property
+  def output_types(self):
+    return self._iterators[0].output_types
+
+  def get_next(self, name=None):
+    """Scatter the input across hosts and devices."""
+    replicas = []
+    for worker, iterator in zip(self._input_workers.worker_devices,
+                                self._iterators):
+      if name is not None:
+        d = tf_device.DeviceSpec.from_string(worker)
+        new_name = "%s_%s_%d" % (name, d.job, d.task)
+      else:
+        new_name = None
+      with ops.device(worker):
+        data_per_worker = iterator.get_next_as_list(name=new_name)
+        # Append to replicas to get a flat list of values indexed by replica.
+        replicas.extend(data_per_worker)
+
+    return values.regroup(self._input_workers.device_map, replicas)
+
+
+class MultiWorkerDataset(object):
+  """Like a `tf.data.Dataset` that distributes data to different workers.
+
+  Each worker gets one shard of the input dataset. This currently does not work
+  in eager mode.
+  """
+
+  def __init__(self, dataset_fn, input_workers, prefetch_on_device=None,
+               auto_shard=False):
+    """Initialize the MultiWorkerDataset object.
+
+    Args:
+      dataset_fn: a function or a list of functions that returns a
+        `tf.data.Dataset`.
+      input_workers: an `InputWorkers` object.
+      prefetch_on_device: whether to prefetch to devices.
+      auto_shard: whether to auto-shard the dataset.
+    """
+    assert isinstance(input_workers, InputWorkers)
+    if isinstance(dataset_fn, (list, tuple)):
+      if len(dataset_fn) != input_workers.num_workers:
+        raise ValueError("If `dataset_fn` is a list, it must have one entry "
+                         "per worker")
+    # TODO(rohanj): b/120673685 to track re-enabling auto sharding.
+    if auto_shard:
+      raise ValueError("Currently autosharding is not supported.")
+    self._input_workers = input_workers
+    self._datasets = []
+    # TODO(yuefengz, priyag): support different set of jobs for input
+    # processing.
+    for i, worker in enumerate(input_workers.worker_devices):
+      with ops.device(worker):
+        if isinstance(dataset_fn, (list, tuple)):
+          worker_input = dataset_fn[i]()
+        else:
+          worker_input = dataset_fn()
+        dataset = PerReplicaDataset(worker_input, input_workers, i,
+                                    prefetch_on_device=prefetch_on_device)
+        self._datasets.append((worker, dataset))
+
+  def make_one_shot_iterator(self):
+    iterators = []
+    for worker, dataset in self._datasets:
+      with ops.device(worker):
+        iterators.append((worker, dataset_ops.make_one_shot_iterator(dataset)))
+    return MultiWorkerDataIterator(iterators, self._input_workers)
+
+  def make_initializable_iterator(self):
+    iterators = []
+    for worker, dataset in self._datasets:
+      with ops.device(worker):
+        iterators.append(
+            (worker, dataset_ops.make_initializable_iterator(dataset)))
+    return MultiWorkerDataIterator(iterators, self._input_workers)
+
+
+class InputIterator(object):
+  """An input iterator, intended to be passed to `DistributionStrategy.run`."""
+
+  def get_next(self):
+    """Returns the next inputs for all replicas."""
+    raise NotImplementedError("must be implemented in descendants")
+
+  def initialize(self):
+    """Initialize the underlying input dataset, when applicable.
+
+    In eager mode, this will create a new iterator and return it.
+    In graph mode, this will initialize the same underlying iterator(s).
+
+    Users are required to call this if
+    - This iterator was returned from a call to `make_input_fn_iterator` with an
+      input function that returns a dataset.
+    - Or this iterator was returned from a call to `make_dataset_iterator`.
+
+    Returns:
+      A list of initialization ops to be executed.
+    """
+    raise NotImplementedError("must be implemented in descendants")
+
+
+class InputIteratorImpl(InputIterator):
+  """Common implementation for all input iterators."""
+
+  def __init__(self, input_workers, iterators):
+    assert isinstance(input_workers, InputWorkers)
+    if not input_workers.worker_devices:
+      raise ValueError("Should have at least one worker for input iterator.")
+
+    self._iterators = iterators
+    self._input_workers = input_workers
+
+  def get_next(self, name=None):
+    """Returns the next input from the iterator for all replicas."""
+    replicas = []
+    for i, worker in enumerate(self._input_workers.worker_devices):
+      if name is not None:
+        d = tf_device.DeviceSpec.from_string(worker)
+        new_name = "%s_%s_%d" % (name, d.job, d.task)
+      else:
+        new_name = None
+      with ops.device(worker):
+        # Make `replicas` a flat list of values across all replicas.
+        replicas.extend(self._iterators[i].get_next_as_list(new_name))
+
+    return values.regroup(self._input_workers.device_map, replicas)
+
+  def initialize(self):
+    """Initialze underlying iterators.
+
+    Returns:
+      A list of any initializer ops that should be run.
+    """
+    init_ops = []
+    for it in self._iterators:
+      init_ops.extend(it.initialize())
+    return init_ops
+
+  # TODO(priyag): Remove when we switch to using `MultiDeviceIterator` for TPUs.
+  @property
+  def output_classes(self):
+    return self._iterators[0].output_classes
+
+  # TODO(priyag): Remove when we switch to using `MultiDeviceIterator` for TPUs.
+  @property
+  def output_shapes(self):
+    return self._iterators[0].output_shapes
+
+  # TODO(priyag): Remove when we switch to using `MultiDeviceIterator` for TPUs.
+  @property
+  def output_types(self):
+    return self._iterators[0].output_types
+
+  # TODO(priyag): Remove when we switch to using `MultiDeviceIterator` for TPUs.
+  def get_iterator(self, worker):
+    for i, w in enumerate(self._input_workers.worker_devices):
+      if worker == w:
+        return self._iterators[i]
+    return None
+
+
+class InputFunctionIterator(InputIteratorImpl):
+  """Iterator created from input function."""
+
+  def __init__(self, input_fn, input_workers, input_contexts):
+    """Make an iterator for input provided via an input function.
+
+    Currently implements PER_WORKER mode, in which the `input_fn` is called
+    once on each worker.
+
+    TODO(priyag): Add other replication modes.
+    TODO(priyag): Allow taking input function that returns a callable that
+    returns nest of tensors.
+
+    Args:
+      input_fn: Input function that returns a `tf.data.Dataset` object.
+      input_workers: an `InputWorkers` object.
+      input_contexts: A list of `InputContext` instances to be passed to call(s)
+        to `input_fn`. Length and order should match worker order in
+        `worker_device_pairs`.
+    """
+    assert isinstance(input_workers, InputWorkers)
+    if input_workers.num_workers != len(input_contexts):
+      raise ValueError(
+          "Number of input workers (%d) is not same as number of "
+          "input_contexts (%d)" %
+          (input_workers.num_workers, len(input_contexts)))
+
+    iterators = []
+    for i, ctx in enumerate(input_contexts):
+      worker = input_workers.worker_devices[i]
+      with ops.device(worker):
+        result = input_fn(ctx)
+        if not isinstance(result, dataset_ops.DatasetV2):
+          raise ValueError("input_fn must return a tf.data.Dataset.")
+        devices = input_workers.compute_devices_for_worker(i)
+        iterator = _SingleWorkerDatasetIterator(result, worker, devices)
+        iterators.append(iterator)
+
+    super(InputFunctionIterator, self).__init__(input_workers, iterators)
+
+
+class DatasetIterator(InputIteratorImpl):
+  """Iterator created from input dataset."""
+
+  def __init__(self, dataset, input_workers, split_batch_by=None):
+    """Make an iterator for the dataset on given devices.
+
+    If `split_batch_by` is not None, we "split" each batch of the
+    dataset by `split_batch_by` value. To achieve this, we first unbatch the
+    input dataset and then rebatch it with the per replica batch size that is
+    calculated using `global_batch_size // split_batch_by`.
+    The currently supported datasets are as follows:
+    `dataset.batch()` is the last operation on the dataset OR
+    `dataset.apply(map_and_batch)` is the last operation on the dataset OR
+    `dataset.batch().prefetch()` are the last 2 operations on the dataset OR
+    `dataset.apply(map_and_batch).prefetch()` are the last 2 operations.
+
+    TODO(priyag): Support multi worker / host cases properly by cloning
+    and sharding the dataset on each worker. Current setup will only work in
+    some cases, such as in-graph multi worker GPU case. If the input pipeline
+    has random shuffling (with a different seed on each worker), each worker
+    will see random input from the same overall dataset in each step. Otherwise,
+    each worker will see the same input in each step.
+
+    Args:
+      dataset: `tf.data.Dataset` that will be used as the input source.
+      input_workers: an `InputWorkers` object.
+      split_batch_by: Optional integer. If present, we "split" each batch of the
+        dataset by `split_batch_by` value.
+    """
+    assert isinstance(input_workers, InputWorkers)
+    if split_batch_by:
+      dataset = _split_dataset_batch(dataset, split_batch_by)
+
+    iterators = []
+    for i, worker in enumerate(input_workers.worker_devices):
+      with ops.device(worker):
+        worker_devices = input_workers.compute_devices_for_worker(i)
+        cloned_dataset = dataset
+        if not context.executing_eagerly():
+          cloned_dataset = input_ops._clone_dataset(dataset)  # pylint: disable=protected-access
+        iterator = _SingleWorkerDatasetIterator(cloned_dataset, worker,
+                                                worker_devices)
+        iterators.append(iterator)
+
+    super(DatasetIterator, self).__init__(input_workers, iterators)
+
+
+class _SingleWorkerDatasetIterator(object):
+  """Iterator for a single `tf.data.Dataset`."""
+
+  def __init__(self, dataset, worker, devices):
+    """Create iterator for the `dataset` to fetch data to worker's `devices` .
+
+    `MultiDeviceIterator` is used to prefetch input to the devices on the
+    given worker.
+
+    Args:
+      dataset: A `tf.data.Dataset` instance.
+      worker: Worker on which ops should be created.
+      devices: Distribute data from `dataset` to these devices.
+    """
+    self._dataset = dataset
+    self._worker = worker
+    self._devices = devices
+    self._make_iterator()
+
+  def _make_iterator(self):
+    """Make appropriate iterator on the dataset."""
+    with ops.device(self._worker):
+      self._iterator = multi_device_iterator_ops.MultiDeviceIterator(
+          self._dataset, self._devices)
+
+  def get_next_as_list(self, name=None):
+    """Get next element from the underlying iterator."""
+    del name
+    with ops.device(self._worker):
+      data_list = self._iterator.get_next()
+      return data_list
+
+  def initialize(self):
+    """Initialze underlying iterator.
+
+    In eager execution, this simply recreates the underlying iterator.
+    In graph execution, it returns the initializer ops for the underlying
+    iterator.
+
+    Returns:
+      A list of any initializer ops that should be run.
+    """
+    if context.executing_eagerly():
+      self._make_iterator()
+      return []
+    else:
+      return [self._iterator.initializer]
+
+  @property
+  def output_classes(self):
+    return self._iterator.output_classes
+
+  @property
+  def output_shapes(self):
+    return self._iterator.output_shapes
+
+  @property
+  def output_types(self):
+    return self._iterator.output_types
+
+
+def _split_dataset_batch(dataset, split_batch_by):
+  """Divide a batch-ed dataset's batches into smaller batches."""
+  # TODO(sourabhbajaj): Remove this in lieu of distributed datasets
+  # pylint: disable=protected-access
+  def _get_batch_dataset(d):
+    """Get the underlying batch dataset from the dataset object."""
+    if isinstance(d, dataset_ops.DatasetV1Adapter):
+      d = d._dataset
+
+    if isinstance(d, (dataset_ops.BatchDataset, batching._MapAndBatchDataset)):
+      return d
+    elif isinstance(d, dataset_ops.PrefetchDataset):
+      return _get_batch_dataset(d._input_dataset)
+    raise ValueError(
+        "Unable to get batched dataset from the input dataset. `batch` "
+        "`map_and_batch` need to be the last operations on the dataset. "
+        "The batch operations can be followed by a prefetch.")
+
+  batched_dataset = _get_batch_dataset(dataset)
+  if isinstance(batched_dataset, dataset_ops.BatchDataset):
+    batch_size = batched_dataset._batch_size
+    drop_remainder = batched_dataset._drop_remainder
+  elif isinstance(batched_dataset, batching._MapAndBatchDataset):
+    batch_size = batched_dataset._batch_size_t
+    drop_remainder = batched_dataset._drop_remainder_t
+
+  prefetch_buffer = None
+  if isinstance(dataset, dataset_ops.PrefetchDataset):
+    prefetch_buffer = dataset._buffer_size
+  elif (isinstance(dataset, dataset_ops.DatasetV1Adapter)
+        and isinstance(dataset._dataset, dataset_ops.PrefetchDataset)):
+    prefetch_buffer = dataset._dataset._buffer_size
+  # pylint: enable=protected-access
+
+  if tensor_util.is_tensor(batch_size):
+    batch_size = tensor_util.constant_value(batch_size)
+
+  if tensor_util.is_tensor(drop_remainder):
+    drop_remainder = tensor_util.constant_value(drop_remainder)
+
+  if batch_size % split_batch_by:
+    raise ValueError(
+        "Batch size %s cannot be sharded evenly across replicas %s" % (
+            batch_size, split_batch_by))
+  new_batch_size = batch_size // split_batch_by
+
+  dataset = dataset.apply(batching.unbatch())
+  dataset = dataset.batch(new_batch_size, drop_remainder=drop_remainder)
+  if prefetch_buffer is not None:
+    dataset = dataset.prefetch(prefetch_buffer)
+  return dataset
+
+
+class MultiStepContext(object):
+  """A context object that can be used to capture things when running steps.
+
+  This context object is useful when running multiple steps at a time using the
+  `experimental_run_steps_on_iterator` API. For e.g. it allows the user's step
+  function to specify which outputs to emit at what frequency. Currently it
+  supports capturing output from the last step, as well as capturing non tensor
+  outputs.  In the future it will be augmented to support other use cases such
+  as output each N steps.
+  """
+
+  def __init__(self):
+    """Initialize an output context.
+
+    Returns:
+      A context object.
+    """
+    self._last_step_outputs = {}
+    self._last_step_outputs_reduce_ops = {}
+    self._non_tensor_outputs = {}
+
+  @property
+  def last_step_outputs(self):
+    """A dictionary consisting of outputs to be captured on last step.
+
+    Keys in the dictionary are names of tensors to be captured, as specified
+    when `set_last_step_output` is called.
+    Values in the dictionary are the tensors themselves. If
+    `set_last_step_output` was called with a `reduce_op` for this output,
+    then the value is the reduced value.
+
+    Returns:
+      A dictionary with last step outputs.
+    """
+    return self._last_step_outputs
+
+  def _set_last_step_outputs(self, outputs):
+    """Replace the entire dictionary of last step outputs."""
+    if not isinstance(outputs, dict):
+      raise ValueError("Need a dictionary to set last_step_outputs.")
+    self._last_step_outputs = outputs
+
+  def set_last_step_output(self, name, output, reduce_op=None):
+    """Set `output` with `name` to be outputted from the last step.
+
+    Args:
+      name: String, name to identify the output. Doesn't need to match tensor
+        name.
+      output: The tensors that should be outputted with `name`. See below for
+        actual types supported.
+      reduce_op: Reduction method to use to reduce outputs from multiple
+        replicas. Required if `set_last_step_output` is called in a replica
+        context. Optional in cross_replica_context.
+        When present, the outputs from all the replicas are reduced using the
+        current distribution strategy's `reduce` method. Hence, the type of
+        `output` must be what's supported by the corresponding `reduce` method.
+        For e.g. if using MirroredStrategy and reduction is set, output
+        must be a `PerReplica` value.
+        The reduce method is also recorded in a dictionary
+        `_last_step_outputs_reduce_ops` for later interpreting of the
+        outputs as already reduced or not.
+    """
+    if distribution_strategy_context.in_cross_replica_context():
+      self._last_step_outputs_reduce_ops[name] = reduce_op
+      if reduce_op is None:
+        self._last_step_outputs[name] = output
+      else:
+        distribution = distribution_strategy_context.get_distribution_strategy()
+        self._last_step_outputs[name] = distribution.reduce(reduce_op, output)
+    else:
+      assert reduce_op is not None
+      def merge_fn(distribution, value):
+        self._last_step_outputs[name] = distribution.reduce(reduce_op, value)
+        # Setting this inside the `merge_fn` because all replicas share the same
+        # context object, so it's more robust to set it only once (even if all
+        # the replicas are trying to set the same value).
+        self._last_step_outputs_reduce_ops[name] = reduce_op
+
+      distribution_strategy_context.get_replica_context().merge_call(
+          merge_fn, args=(output,))
+
+  @property
+  def non_tensor_outputs(self):
+    """A dictionary consisting of any non tensor outputs to be captured."""
+    return self._non_tensor_outputs
+
+  def set_non_tensor_output(self, name, output):
+    """Set `output` with `name` to be captured as a non tensor output."""
+    if distribution_strategy_context.in_cross_replica_context():
+      self._non_tensor_outputs[name] = output
+    else:
+      def merge_fn(distribution, value):
+        # NOTE(priyag): For non tensor outputs, we simply return all the values
+        # in a list as reduction doesn't make sense on non tensors.
+        self._non_tensor_outputs[name] = distribution.unwrap(value)
+      distribution_strategy_context.get_replica_context().merge_call(
+          merge_fn, args=(output,))
diff --git a/tensorflow/python/distribute/mirrored_strategy.py b/tensorflow/python/distribute/mirrored_strategy.py
index 601eafbb5e..37b493d0f7 100644
--- a/tensorflow/python/distribute/mirrored_strategy.py
+++ b/tensorflow/python/distribute/mirrored_strategy.py
@@ -27,6 +27,7 @@ from tensorflow.python import pywrap_tensorflow
 from tensorflow.python.distribute import cross_device_ops as cross_device_ops_lib
 from tensorflow.python.distribute import device_util
 from tensorflow.python.distribute import distribute_lib
+from tensorflow.python.distribute import input_lib
 from tensorflow.python.distribute import multi_worker_util
 from tensorflow.python.distribute import reduce_util
 from tensorflow.python.distribute import shared_variable_creator
@@ -456,7 +457,7 @@ class MirroredExtended(distribute_lib.DistributionStrategyExtended):
         "No duplicates allowed in `devices` argument: %s" % devices)
     # TODO(josh11b): Require at least 2 devices?
     self._device_map = values.ReplicaDeviceMap(devices)
-    self._input_workers = values.InputWorkers(self._device_map)
+    self._input_workers = input_lib.InputWorkers(self._device_map)
     self._inferred_cross_device_ops = cross_device_ops_lib.choose_the_best(
         devices)
 
@@ -489,7 +490,8 @@ class MirroredExtended(distribute_lib.DistributionStrategyExtended):
     self._default_device = workers[0]
 
     self._device_map = values.ReplicaDeviceMap(devices)
-    self._input_workers = values.InputWorkers(self._device_map, worker_devices)
+    self._input_workers = input_lib.InputWorkers(
+        self._device_map, worker_devices)
     self._inferred_cross_device_ops = cross_device_ops_lib.MultiWorkerAllReduce(
         workers, _infer_num_gpus_per_worker(devices))
 
@@ -543,16 +545,16 @@ class MirroredExtended(distribute_lib.DistributionStrategyExtended):
   def _distribute_dataset(self, dataset_fn):
     if self._local_mode:
       worker_index = 0
-      return values.PerReplicaDataset(
+      return input_lib.PerReplicaDataset(
           self._call_dataset_fn(dataset_fn), self._input_workers, worker_index)
     else:
-      return values.MultiWorkerDataset(
+      return input_lib.MultiWorkerDataset(
           functools.partial(self._call_dataset_fn, dataset_fn),
           self._input_workers,
           auto_shard=False)
 
   def _make_dataset_iterator(self, dataset):
-    return values.DatasetIterator(
+    return input_lib.DatasetIterator(
         dataset, self._input_workers, self._num_replicas_in_sync)
 
   def _make_input_fn_iterator(
@@ -566,7 +568,7 @@ class MirroredExtended(distribute_lib.DistributionStrategyExtended):
           num_input_pipelines=num_workers,
           input_pipeline_id=i,
           num_replicas_in_sync=self._num_replicas_in_sync))
-    return values.InputFunctionIterator(
+    return input_lib.InputFunctionIterator(
         input_fn, self._input_workers, input_contexts)
 
   # TODO(priyag): Deal with OutOfRange errors once b/111349762 is fixed.
@@ -576,7 +578,7 @@ class MirroredExtended(distribute_lib.DistributionStrategyExtended):
       initial_loop_values = {}
     initial_loop_values = nest.flatten(initial_loop_values)
 
-    ctx = values.MultiStepContext()
+    ctx = input_lib.MultiStepContext()
     def body(i, *args):
       """A wrapper around `fn` to create the while loop body."""
       del args
diff --git a/tensorflow/python/distribute/values.py b/tensorflow/python/distribute/values.py
index 1f5077a75a..a9dcabdab6 100644
--- a/tensorflow/python/distribute/values.py
+++ b/tensorflow/python/distribute/values.py
@@ -23,17 +23,12 @@ import contextlib
 import weakref
 import six
 
-from tensorflow.python.data.experimental.ops import batching
-from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.data.ops import multi_device_iterator_ops
 from tensorflow.python.distribute import device_util
 from tensorflow.python.distribute import distribute_lib
 from tensorflow.python.distribute import distribution_strategy_context
-from tensorflow.python.distribute import input_ops
 from tensorflow.python.distribute import reduce_util
 from tensorflow.python.eager import context
 from tensorflow.python.eager import tape
-from tensorflow.python.framework import device as tf_device
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import array_ops
@@ -1409,679 +1404,6 @@ def update_regroup(extended, device_map, updates, group):
   return nest.pack_sequence_as(regrouped, grouped_flat)
 
 
-class InputWorkers(object):
-  """A 1-to-many mapping from input worker devices to compute devices."""
-
-  def __init__(self, device_map, worker_device_pairs=None, logical_device=0):
-    """Initialize an `InputWorkers` object.
-
-    Args:
-      device_map: A `DeviceMap` with the computation devices fed by the
-        input workers.
-      worker_device_pairs: A sequence of pairs:
-        `(input device, a tuple of compute devices fed by that input device)`.
-      logical_device: The logical device of `device_map` to feed.
-    """
-    self._device_map = device_map
-    self._logical_device = logical_device
-    if worker_device_pairs is None:
-      worker_device_pairs = ((
-          device_util.canonicalize("/device:CPU:0"),
-          device_map.logical_to_actual_devices(logical_device)),)
-    self._input_worker_devices = tuple(d for d, _ in worker_device_pairs)
-    self._fed_devices = tuple(tuple(device_util.canonicalize(d) for d in f)
-                              for _, f in worker_device_pairs)
-    flattened = tuple(d for l in self._fed_devices for d in l)
-    assert (flattened ==
-            device_map.logical_to_actual_devices(logical_device)), (
-                "flattened: %s logical device %d: %s" %
-                (flattened, logical_device,
-                 device_map.logical_to_actual_devices(logical_device)))
-
-  @property
-  def device_map(self):
-    return self._device_map
-
-  @property
-  def logical_device(self):
-    return self._logical_device
-
-  @property
-  def num_workers(self):
-    return len(self._input_worker_devices)
-
-  @property
-  def worker_devices(self):
-    return self._input_worker_devices
-
-  def compute_devices_for_worker(self, worker_index):
-    return self._fed_devices[worker_index]
-
-  def __repr__(self):
-    devices = self.worker_devices
-    debug_repr = ",\n".join("  %d %s: %s" %
-                            (i, devices[i], self._fed_devices[i])
-                            for i in range(len(devices)))
-    return "%s:{\n%s\n  device_map: %s}" % (
-        self.__class__.__name__, debug_repr, self._device_map)
-
-
-class PerReplicaDataIterator(object):
-  """An iterator (like `tf.data.Iterator`) into a `PerReplicaDataset`."""
-
-  def __init__(self, iterator, input_workers, worker_index, prefetch_on_device):
-    assert isinstance(input_workers, InputWorkers)
-    self._iterator = iterator
-    self._input_workers = input_workers
-    self._worker_index = worker_index
-    self._prefetch_on_device = prefetch_on_device
-
-  @property
-  def initializer(self):
-    return self._iterator.initializer
-
-  def get_next_as_list(self, name=None):
-    """Scatter the input across devices."""
-    if self._prefetch_on_device:
-      data_list = self._iterator.get_next()
-    else:
-      batch = self._iterator.get_next(name=name)
-      data_list = []
-      def get_ith(i):
-        return lambda x: x[i]
-
-      devices = self._input_workers.compute_devices_for_worker(
-          self._worker_index)
-      for i, d in enumerate(devices):
-        v = nest.map_structure(get_ith(i), batch)
-        if context.executing_eagerly():
-          with ops.device(d):
-            v = nest.map_structure(array_ops.identity, v)
-        data_list.append(v)
-
-    return data_list
-
-  def get_next(self, name=None):
-    assert self._input_workers.num_workers == 1
-    data_list = self.get_next_as_list(name)
-    return regroup(self._input_workers.device_map, data_list)
-
-  @property
-  def output_classes(self):
-    return self._iterator.output_classes
-
-  @property
-  def output_shapes(self):
-    return self._iterator.output_shapes
-
-  @property
-  def output_types(self):
-    return self._iterator.output_types
-
-
-class PerReplicaDataset(object):
-  """Like `tf.data.Dataset` split devices, producing `PerReplica` data."""
-
-  def __init__(self, dataset, input_workers, worker_index,
-               prefetch_on_device=None):
-    assert isinstance(input_workers, InputWorkers)
-    assert worker_index is not None
-    assert worker_index is not True
-    assert worker_index is not False
-    self._input_workers = input_workers
-    self._worker_index = worker_index
-
-    # Default to using prefetching, unless specified.
-    self._prefetch_on_device = prefetch_on_device
-    if self._prefetch_on_device is None:
-      self._prefetch_on_device = True
-
-    self._dataset = dataset
-    if not self._prefetch_on_device:
-      # TODO(priyag): If dropping remainder is not appropriate, find another
-      # approach to distributing the dataset when not possible to divide evenly.
-      # Possibly not an issue when we start using PartitionedDataset.
-      num_replicas = len(
-          self._input_workers.compute_devices_for_worker(self._worker_index))
-      self._dataset = self._dataset.batch(num_replicas, drop_remainder=True)
-    else:
-      self._replica_devices = self._input_workers.compute_devices_for_worker(
-          self._worker_index)
-
-  def make_one_shot_iterator(self):
-    """Get a one time use iterator for the distributed PerReplicaDataset."""
-    # Graph mode with one shot iterator is disabled.
-    if not context.executing_eagerly():
-      raise ValueError("Cannot create a one shot iterator. Please use "
-                       "`make_initializable_iterator()` instead.")
-    if self._prefetch_on_device:
-      dataset_iterator = multi_device_iterator_ops.MultiDeviceIterator(
-          self._dataset, self._replica_devices)
-    else:
-      dataset_iterator = dataset_ops.make_one_shot_iterator(self._dataset)
-    return PerReplicaDataIterator(
-        dataset_iterator,
-        self._input_workers,
-        self._worker_index,
-        prefetch_on_device=self._prefetch_on_device)
-
-  def make_initializable_iterator(self):
-    """Get an initializable iterator for the distributed PerReplicaDataset."""
-    # Eager mode generates already initialized iterators. Hence we cannot create
-    # an initializable iterator.
-    if context.executing_eagerly():
-      raise ValueError("Cannot create initializable iterator in Eager mode. "
-                       "Please use `make_one_shot_iterator` instead.")
-    if self._prefetch_on_device:
-      dataset_iterator = multi_device_iterator_ops.MultiDeviceIterator(
-          self._dataset, self._replica_devices)
-    else:
-      dataset_iterator = dataset_ops.make_initializable_iterator(self._dataset)
-    return PerReplicaDataIterator(
-        dataset_iterator, self._input_workers, self._worker_index,
-        prefetch_on_device=self._prefetch_on_device)
-
-
-class MultiWorkerDataIterator(object):
-  """An iterator (like `tf.data.Iterator`) into a `MultiWorkerDataset`."""
-
-  def __init__(self, iterators, input_workers):
-    """Initialize the `MultiWorkerDataIterator` object.
-
-    Args:
-      iterators: a list of worker, iterator pairs.
-      input_workers: an `InputWorkers` object.
-
-    Raises:
-      ValueError: if iterators and input_workers are not compatible.
-    """
-    assert isinstance(input_workers, InputWorkers)
-    workers = tuple(d for d, _ in iterators)
-    if workers != input_workers.worker_devices:
-      raise ValueError("iterators and input_workers are not compatible. "
-                       "iterator workers: %r input_workers devices: %r" %
-                       (workers, input_workers.worker_devices))
-    self._iterators = tuple(i for _, i in iterators)
-    self._input_workers = input_workers
-
-  @property
-  def initializer(self):
-    return control_flow_ops.group(
-        tuple(iterator.initializer for iterator in self._iterators))
-
-  def get_iterator(self, worker):
-    for i, w in enumerate(self._input_workers.worker_devices):
-      if worker == w:
-        return self._iterators[i]
-    return None
-
-  @property
-  def output_shapes(self):
-    return self._iterators[0].output_shapes
-
-  @property
-  def output_types(self):
-    return self._iterators[0].output_types
-
-  def get_next(self, name=None):
-    """Scatter the input across hosts and devices."""
-    replicas = []
-    for worker, iterator in zip(self._input_workers.worker_devices,
-                                self._iterators):
-      if name is not None:
-        d = tf_device.DeviceSpec.from_string(worker)
-        new_name = "%s_%s_%d" % (name, d.job, d.task)
-      else:
-        new_name = None
-      with ops.device(worker):
-        data_per_worker = iterator.get_next_as_list(name=new_name)
-        # Append to replicas to get a flat list of values indexed by replica.
-        replicas.extend(data_per_worker)
-
-    return regroup(self._input_workers.device_map, replicas)
-
-
-class MultiWorkerDataset(object):
-  """Like a `tf.data.Dataset` that distributes data to different workers.
-
-  Each worker gets one shard of the input dataset. This currently does not work
-  in eager mode.
-  """
-
-  def __init__(self, dataset_fn, input_workers, prefetch_on_device=None,
-               auto_shard=False):
-    """Initialize the MultiWorkerDataset object.
-
-    Args:
-      dataset_fn: a function or a list of functions that returns a
-        `tf.data.Dataset`.
-      input_workers: an `InputWorkers` object.
-      prefetch_on_device: whether to prefetch to devices.
-      auto_shard: whether to auto-shard the dataset.
-    """
-    assert isinstance(input_workers, InputWorkers)
-    if isinstance(dataset_fn, (list, tuple)):
-      if len(dataset_fn) != input_workers.num_workers:
-        raise ValueError("If `dataset_fn` is a list, it must have one entry "
-                         "per worker")
-    # TODO(rohanj): b/120673685 to track re-enabling auto sharding.
-    if auto_shard:
-      raise ValueError("Currently autosharding is not supported.")
-    self._input_workers = input_workers
-    self._datasets = []
-    # TODO(yuefengz, priyag): support different set of jobs for input
-    # processing.
-    for i, worker in enumerate(input_workers.worker_devices):
-      with ops.device(worker):
-        if isinstance(dataset_fn, (list, tuple)):
-          worker_input = dataset_fn[i]()
-        else:
-          worker_input = dataset_fn()
-        dataset = PerReplicaDataset(worker_input, input_workers, i,
-                                    prefetch_on_device=prefetch_on_device)
-        self._datasets.append((worker, dataset))
-
-  def make_one_shot_iterator(self):
-    iterators = []
-    for worker, dataset in self._datasets:
-      with ops.device(worker):
-        iterators.append((worker, dataset_ops.make_one_shot_iterator(dataset)))
-    return MultiWorkerDataIterator(iterators, self._input_workers)
-
-  def make_initializable_iterator(self):
-    iterators = []
-    for worker, dataset in self._datasets:
-      with ops.device(worker):
-        iterators.append(
-            (worker, dataset_ops.make_initializable_iterator(dataset)))
-    return MultiWorkerDataIterator(iterators, self._input_workers)
-
-
-class InputIterator(object):
-  """An input iterator, intended to be passed to `DistributionStrategy.run`."""
-
-  def get_next(self):
-    """Returns the next inputs for all replicas."""
-    raise NotImplementedError("must be implemented in descendants")
-
-  def initialize(self):
-    """Initialize the underlying input dataset, when applicable.
-
-    In eager mode, this will create a new iterator and return it.
-    In graph mode, this will initialize the same underlying iterator(s).
-
-    Users are required to call this if
-    - This iterator was returned from a call to `make_input_fn_iterator` with an
-      input function that returns a dataset.
-    - Or this iterator was returned from a call to `make_dataset_iterator`.
-
-    Returns:
-      A list of initialization ops to be executed.
-    """
-    raise NotImplementedError("must be implemented in descendants")
-
-
-class InputIteratorImpl(InputIterator):
-  """Common implementation for all input iterators."""
-
-  def __init__(self, input_workers, iterators):
-    assert isinstance(input_workers, InputWorkers)
-    if not input_workers.worker_devices:
-      raise ValueError("Should have at least one worker for input iterator.")
-
-    self._iterators = iterators
-    self._input_workers = input_workers
-
-  def get_next(self, name=None):
-    """Returns the next input from the iterator for all replicas."""
-    replicas = []
-    for i, worker in enumerate(self._input_workers.worker_devices):
-      if name is not None:
-        d = tf_device.DeviceSpec.from_string(worker)
-        new_name = "%s_%s_%d" % (name, d.job, d.task)
-      else:
-        new_name = None
-      with ops.device(worker):
-        # Make `replicas` a flat list of values across all replicas.
-        replicas.extend(self._iterators[i].get_next_as_list(new_name))
-
-    return regroup(self._input_workers.device_map, replicas)
-
-  def initialize(self):
-    """Initialze underlying iterators.
-
-    Returns:
-      A list of any initializer ops that should be run.
-    """
-    init_ops = []
-    for it in self._iterators:
-      init_ops.extend(it.initialize())
-    return init_ops
-
-  # TODO(priyag): Remove when we switch to using `MultiDeviceIterator` for TPUs.
-  @property
-  def output_classes(self):
-    return self._iterators[0].output_classes
-
-  # TODO(priyag): Remove when we switch to using `MultiDeviceIterator` for TPUs.
-  @property
-  def output_shapes(self):
-    return self._iterators[0].output_shapes
-
-  # TODO(priyag): Remove when we switch to using `MultiDeviceIterator` for TPUs.
-  @property
-  def output_types(self):
-    return self._iterators[0].output_types
-
-  # TODO(priyag): Remove when we switch to using `MultiDeviceIterator` for TPUs.
-  def get_iterator(self, worker):
-    for i, w in enumerate(self._input_workers.worker_devices):
-      if worker == w:
-        return self._iterators[i]
-    return None
-
-
-class InputFunctionIterator(InputIteratorImpl):
-  """Iterator created from input function."""
-
-  def __init__(self, input_fn, input_workers, input_contexts):
-    """Make an iterator for input provided via an input function.
-
-    Currently implements PER_WORKER mode, in which the `input_fn` is called
-    once on each worker.
-
-    TODO(priyag): Add other replication modes.
-    TODO(priyag): Allow taking input function that returns a callable that
-    returns nest of tensors.
-
-    Args:
-      input_fn: Input function that returns a `tf.data.Dataset` object.
-      input_workers: an `InputWorkers` object.
-      input_contexts: A list of `InputContext` instances to be passed to call(s)
-        to `input_fn`. Length and order should match worker order in
-        `worker_device_pairs`.
-    """
-    assert isinstance(input_workers, InputWorkers)
-    if input_workers.num_workers != len(input_contexts):
-      raise ValueError(
-          "Number of input workers (%d) is not same as number of "
-          "input_contexts (%d)" %
-          (input_workers.num_workers, len(input_contexts)))
-
-    iterators = []
-    for i, ctx in enumerate(input_contexts):
-      worker = input_workers.worker_devices[i]
-      with ops.device(worker):
-        result = input_fn(ctx)
-        if not isinstance(result, dataset_ops.DatasetV2):
-          raise ValueError("input_fn must return a tf.data.Dataset.")
-        devices = input_workers.compute_devices_for_worker(i)
-        iterator = _SingleWorkerDatasetIterator(result, worker, devices)
-        iterators.append(iterator)
-
-    super(InputFunctionIterator, self).__init__(input_workers, iterators)
-
-
-class DatasetIterator(InputIteratorImpl):
-  """Iterator created from input dataset."""
-
-  def __init__(self, dataset, input_workers, split_batch_by=None):
-    """Make an iterator for the dataset on given devices.
-
-    If `split_batch_by` is not None, we "split" each batch of the
-    dataset by `split_batch_by` value. To achieve this, we first unbatch the
-    input dataset and then rebatch it with the per replica batch size that is
-    calculated using `global_batch_size // split_batch_by`.
-    The currently supported datasets are as follows:
-    `dataset.batch()` is the last operation on the dataset OR
-    `dataset.apply(map_and_batch)` is the last operation on the dataset OR
-    `dataset.batch().prefetch()` are the last 2 operations on the dataset OR
-    `dataset.apply(map_and_batch).prefetch()` are the last 2 operations.
-
-    TODO(priyag): Support multi worker / host cases properly by cloning
-    and sharding the dataset on each worker. Current setup will only work in
-    some cases, such as in-graph multi worker GPU case. If the input pipeline
-    has random shuffling (with a different seed on each worker), each worker
-    will see random input from the same overall dataset in each step. Otherwise,
-    each worker will see the same input in each step.
-
-    Args:
-      dataset: `tf.data.Dataset` that will be used as the input source.
-      input_workers: an `InputWorkers` object.
-      split_batch_by: Optional integer. If present, we "split" each batch of the
-        dataset by `split_batch_by` value.
-    """
-    assert isinstance(input_workers, InputWorkers)
-    if split_batch_by:
-      dataset = _split_dataset_batch(dataset, split_batch_by)
-
-    iterators = []
-    for i, worker in enumerate(input_workers.worker_devices):
-      with ops.device(worker):
-        worker_devices = input_workers.compute_devices_for_worker(i)
-        cloned_dataset = dataset
-        if not context.executing_eagerly():
-          cloned_dataset = input_ops._clone_dataset(dataset)  # pylint: disable=protected-access
-        iterator = _SingleWorkerDatasetIterator(cloned_dataset, worker,
-                                                worker_devices)
-        iterators.append(iterator)
-
-    super(DatasetIterator, self).__init__(input_workers, iterators)
-
-
-class _SingleWorkerDatasetIterator(object):
-  """Iterator for a single `tf.data.Dataset`."""
-
-  def __init__(self, dataset, worker, devices):
-    """Create iterator for the `dataset` to fetch data to worker's `devices` .
-
-    `MultiDeviceIterator` is used to prefetch input to the devices on the
-    given worker.
-
-    Args:
-      dataset: A `tf.data.Dataset` instance.
-      worker: Worker on which ops should be created.
-      devices: Distribute data from `dataset` to these devices.
-    """
-    self._dataset = dataset
-    self._worker = worker
-    self._devices = devices
-    self._make_iterator()
-
-  def _make_iterator(self):
-    """Make appropriate iterator on the dataset."""
-    with ops.device(self._worker):
-      self._iterator = multi_device_iterator_ops.MultiDeviceIterator(
-          self._dataset, self._devices)
-
-  def get_next_as_list(self, name=None):
-    """Get next element from the underlying iterator."""
-    del name
-    with ops.device(self._worker):
-      data_list = self._iterator.get_next()
-      return data_list
-
-  def initialize(self):
-    """Initialze underlying iterator.
-
-    In eager execution, this simply recreates the underlying iterator.
-    In graph execution, it returns the initializer ops for the underlying
-    iterator.
-
-    Returns:
-      A list of any initializer ops that should be run.
-    """
-    if context.executing_eagerly():
-      self._make_iterator()
-      return []
-    else:
-      return [self._iterator.initializer]
-
-  @property
-  def output_classes(self):
-    return self._iterator.output_classes
-
-  @property
-  def output_shapes(self):
-    return self._iterator.output_shapes
-
-  @property
-  def output_types(self):
-    return self._iterator.output_types
-
-
-def _split_dataset_batch(dataset, split_batch_by):
-  """Divide a batch-ed dataset's batches into smaller batches."""
-  # TODO(sourabhbajaj): Remove this in lieu of distributed datasets
-  # pylint: disable=protected-access
-  def _get_batch_dataset(d):
-    """Get the underlying batch dataset from the dataset object."""
-    if isinstance(d, dataset_ops.DatasetV1Adapter):
-      d = d._dataset
-
-    if isinstance(d, (dataset_ops.BatchDataset, batching._MapAndBatchDataset)):
-      return d
-    elif isinstance(d, dataset_ops.PrefetchDataset):
-      return _get_batch_dataset(d._input_dataset)
-    raise ValueError(
-        "Unable to get batched dataset from the input dataset. `batch` "
-        "`map_and_batch` need to be the last operations on the dataset. "
-        "The batch operations can be followed by a prefetch.")
-
-  batched_dataset = _get_batch_dataset(dataset)
-  if isinstance(batched_dataset, dataset_ops.BatchDataset):
-    batch_size = batched_dataset._batch_size
-    drop_remainder = batched_dataset._drop_remainder
-  elif isinstance(batched_dataset, batching._MapAndBatchDataset):
-    batch_size = batched_dataset._batch_size_t
-    drop_remainder = batched_dataset._drop_remainder_t
-
-  prefetch_buffer = None
-  if isinstance(dataset, dataset_ops.PrefetchDataset):
-    prefetch_buffer = dataset._buffer_size
-  elif (isinstance(dataset, dataset_ops.DatasetV1Adapter)
-        and isinstance(dataset._dataset, dataset_ops.PrefetchDataset)):
-    prefetch_buffer = dataset._dataset._buffer_size
-  # pylint: enable=protected-access
-
-  if tensor_util.is_tensor(batch_size):
-    batch_size = tensor_util.constant_value(batch_size)
-
-  if tensor_util.is_tensor(drop_remainder):
-    drop_remainder = tensor_util.constant_value(drop_remainder)
-
-  if batch_size % split_batch_by:
-    raise ValueError(
-        "Batch size %s cannot be sharded evenly across replicas %s" % (
-            batch_size, split_batch_by))
-  new_batch_size = batch_size // split_batch_by
-
-  dataset = dataset.apply(batching.unbatch())
-  dataset = dataset.batch(new_batch_size, drop_remainder=drop_remainder)
-  if prefetch_buffer is not None:
-    dataset = dataset.prefetch(prefetch_buffer)
-  return dataset
-
-
-class MultiStepContext(object):
-  """A context object that can be used to capture things when running steps.
-
-  This context object is useful when running multiple steps at a time using the
-  `experimental_run_steps_on_iterator` API. For e.g. it allows the user's step
-  function to specify which outputs to emit at what frequency. Currently it
-  supports capturing output from the last step, as well as capturing non tensor
-  outputs.  In the future it will be augmented to support other use cases such
-  as output each N steps.
-  """
-
-  def __init__(self):
-    """Initialize an output context.
-
-    Returns:
-      A context object.
-    """
-    self._last_step_outputs = {}
-    self._last_step_outputs_reduce_ops = {}
-    self._non_tensor_outputs = {}
-
-  @property
-  def last_step_outputs(self):
-    """A dictionary consisting of outputs to be captured on last step.
-
-    Keys in the dictionary are names of tensors to be captured, as specified
-    when `set_last_step_output` is called.
-    Values in the dictionary are the tensors themselves. If
-    `set_last_step_output` was called with a `reduce_op` for this output,
-    then the value is the reduced value.
-
-    Returns:
-      A dictionary with last step outputs.
-    """
-    return self._last_step_outputs
-
-  def _set_last_step_outputs(self, outputs):
-    """Replace the entire dictionary of last step outputs."""
-    if not isinstance(outputs, dict):
-      raise ValueError("Need a dictionary to set last_step_outputs.")
-    self._last_step_outputs = outputs
-
-  def set_last_step_output(self, name, output, reduce_op=None):
-    """Set `output` with `name` to be outputted from the last step.
-
-    Args:
-      name: String, name to identify the output. Doesn't need to match tensor
-        name.
-      output: The tensors that should be outputted with `name`. See below for
-        actual types supported.
-      reduce_op: Reduction method to use to reduce outputs from multiple
-        replicas. Required if `set_last_step_output` is called in a replica
-        context. Optional in cross_replica_context.
-        When present, the outputs from all the replicas are reduced using the
-        current distribution strategy's `reduce` method. Hence, the type of
-        `output` must be what's supported by the corresponding `reduce` method.
-        For e.g. if using MirroredStrategy and reduction is set, output
-        must be a `PerReplica` value.
-        The reduce method is also recorded in a dictionary
-        `_last_step_outputs_reduce_ops` for later interpreting of the
-        outputs as already reduced or not.
-    """
-    if distribution_strategy_context.in_cross_replica_context():
-      self._last_step_outputs_reduce_ops[name] = reduce_op
-      if reduce_op is None:
-        self._last_step_outputs[name] = output
-      else:
-        distribution = distribution_strategy_context.get_distribution_strategy()
-        self._last_step_outputs[name] = distribution.reduce(reduce_op, output)
-    else:
-      assert reduce_op is not None
-      def merge_fn(distribution, value):
-        self._last_step_outputs[name] = distribution.reduce(reduce_op, value)
-        # Setting this inside the `merge_fn` because all replicas share the same
-        # context object, so it's more robust to set it only once (even if all
-        # the replicas are trying to set the same value).
-        self._last_step_outputs_reduce_ops[name] = reduce_op
-
-      distribution_strategy_context.get_replica_context().merge_call(
-          merge_fn, args=(output,))
-
-  @property
-  def non_tensor_outputs(self):
-    """A dictionary consisting of any non tensor outputs to be captured."""
-    return self._non_tensor_outputs
-
-  def set_non_tensor_output(self, name, output):
-    """Set `output` with `name` to be captured as a non tensor output."""
-    if distribution_strategy_context.in_cross_replica_context():
-      self._non_tensor_outputs[name] = output
-    else:
-      def merge_fn(distribution, value):
-        # NOTE(priyag): For non tensor outputs, we simply return all the values
-        # in a list as reduction doesn't make sense on non tensors.
-        self._non_tensor_outputs[name] = distribution.unwrap(value)
-      distribution_strategy_context.get_replica_context().merge_call(
-          merge_fn, args=(output,))
-
-
 def value_container(val):
   """Returns the container that this per-replica `value` belongs to.
 
-- 
GitLab


From 9fb415c6eac3cd245de736e7ebc54094416c1013 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sun, 30 Dec 2018 01:07:37 +0000
Subject: [PATCH 0080/1095] Fix TypeError when using tf.keras.utils.plot_model

This fix fixes the issue raised in 24622
where a TypeError was raised when using tf.keras.utils.plot_model.

This fix fixes 24622.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/keras/utils/vis_utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/keras/utils/vis_utils.py b/tensorflow/python/keras/utils/vis_utils.py
index 82bc2755bd..fa848ae8a7 100644
--- a/tensorflow/python/keras/utils/vis_utils.py
+++ b/tensorflow/python/keras/utils/vis_utils.py
@@ -120,7 +120,7 @@ def model_to_dot(model, show_shapes=False, show_layer_names=True, rankdir='TB'):
     for i, node in enumerate(layer._inbound_nodes):
       node_key = layer.name + '_ib-' + str(i)
       if node_key in model._network_nodes:  # pylint: disable=protected-access
-        for inbound_layer in node.inbound_layers:
+        for inbound_layer in nest.flatten(node.inbound_layers):
           inbound_layer_id = str(id(inbound_layer))
           layer_id = str(id(layer))
           dot.add_edge(pydot.Edge(inbound_layer_id, layer_id))
-- 
GitLab


From 129bf6c79a6793951d0110d31b1b2c0198479fff Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sun, 30 Dec 2018 01:09:11 +0000
Subject: [PATCH 0081/1095] Add missing python import

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/keras/utils/vis_utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/python/keras/utils/vis_utils.py b/tensorflow/python/keras/utils/vis_utils.py
index fa848ae8a7..c7c45f381e 100644
--- a/tensorflow/python/keras/utils/vis_utils.py
+++ b/tensorflow/python/keras/utils/vis_utils.py
@@ -67,6 +67,7 @@ def model_to_dot(model, show_shapes=False, show_layer_names=True, rankdir='TB'):
   """
   from tensorflow.python.keras.layers.wrappers import Wrapper
   from tensorflow.python.keras.models import Sequential
+  from tensorflow.python.util import nest
 
   _check_pydot()
   dot = pydot.Dot()
-- 
GitLab


From 3849a1b682717226ce011718ea382be9a24b4b0c Mon Sep 17 00:00:00 2001
From: manhyuk <manhyuk@kw.ac.kr>
Date: Sun, 30 Dec 2018 15:33:29 +0900
Subject: [PATCH 0082/1095] fix typo

---
 tensorflow/core/framework/model.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/framework/model.h b/tensorflow/core/framework/model.h
index bb1d7b6bff..d9f7185362 100644
--- a/tensorflow/core/framework/model.h
+++ b/tensorflow/core/framework/model.h
@@ -310,7 +310,7 @@ class Node {
   std::map<string, std::shared_ptr<Parameter>> parameters_ GUARDED_BY(mu_);
   std::list<std::shared_ptr<Node>> inputs_ GUARDED_BY(mu_);
 
-  // The reference to the output node is not owned so that that deletion of a
+  // The reference to the output node is not owned so that deletion of a
   // node results in recursive deletion of the subtree rooted in the node.
   Node* const output_;
 };
-- 
GitLab


From a97dbfcf9b727a9eeec81e4a19ecf56a36e84a0c Mon Sep 17 00:00:00 2001
From: manhyuk <manhyuk@kw.ac.kr>
Date: Sun, 30 Dec 2018 15:33:39 +0900
Subject: [PATCH 0083/1095] fix typo

---
 tensorflow/core/kernels/cudnn_rnn_ops.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/kernels/cudnn_rnn_ops.cc b/tensorflow/core/kernels/cudnn_rnn_ops.cc
index d37f5fb9da..196494cbcf 100644
--- a/tensorflow/core/kernels/cudnn_rnn_ops.cc
+++ b/tensorflow/core/kernels/cudnn_rnn_ops.cc
@@ -743,7 +743,7 @@ Status DoBackward(
     /* forward inputs */
     const Tensor* input, const Tensor* input_h, const Tensor* input_c,
     const Tensor* params,
-    /* forward outptus */
+    /* forward outputs */
     const Tensor* output, const Tensor* output_h, const Tensor* output_c,
     /* backprop inputs */
     const Tensor* output_backprop, const Tensor* output_h_backprop,
-- 
GitLab


From bbc8c15b85358b9df029f16e31bc663b8f9d1fef Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 30 Dec 2019 01:02:39 -0800
Subject: [PATCH 0084/1095] compat: Update forward compatibility horizon to
 2018-12-30

PiperOrigin-RevId: 227267901
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 3a9b3b3838..c41e58a823 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 12, 29)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 12, 30)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 53e677de105c2edbfdc2d86ceb31d06765b08512 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sun, 30 Dec 2018 10:07:06 +0000
Subject: [PATCH 0085/1095] Fix incorrect display of keras model summary

This fix fixes the issue raised in 24627 where the
display of keras model summary is incorrect (regression
from 1.12). The reason was that `layer.name`
(vs. layer object itself) should be used,

This fix fixes 24627.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/keras/utils/layer_utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/keras/utils/layer_utils.py b/tensorflow/python/keras/utils/layer_utils.py
index ead5afd1ae..1d85e8a25f 100644
--- a/tensorflow/python/keras/utils/layer_utils.py
+++ b/tensorflow/python/keras/utils/layer_utils.py
@@ -196,7 +196,7 @@ def print_summary(model, line_length=None, positions=None, print_fn=None):
         continue
 
       for inbound_layer, node_index, tensor_index, _ in node.iterate_inbound():
-        connections.append('{}[{}][{}]'.format(inbound_layer, node_index,
+        connections.append('{}[{}][{}]'.format(inbound_layer.name, node_index,
                                                tensor_index))
 
     name = layer.name
-- 
GitLab


From c985bd0dce0f8a7ccf334c9782d051c81ad00f1d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 30 Dec 2019 02:54:20 -0800
Subject: [PATCH 0086/1095] Standardize some names from `TFGAN` -> `TF-GAN`.

A noop.

PiperOrigin-RevId: 227273663
---
 tensorflow/contrib/gan/BUILD                  |  2 +-
 .../estimator/python/gan_estimator_impl.py    | 16 +++++++-------
 .../gan/python/losses/python/losses_impl.py   |  4 ++--
 .../python/losses/python/tuple_losses_impl.py |  2 +-
 tensorflow/contrib/gan/python/namedtuples.py  |  6 ++---
 tensorflow/contrib/gan/python/train.py        | 22 ++++++++++---------
 6 files changed, 27 insertions(+), 25 deletions(-)

diff --git a/tensorflow/contrib/gan/BUILD b/tensorflow/contrib/gan/BUILD
index ae8320cfb2..e1d198fe7b 100644
--- a/tensorflow/contrib/gan/BUILD
+++ b/tensorflow/contrib/gan/BUILD
@@ -1,4 +1,4 @@
-# Files for using TFGAN framework.
+# Files for using TF-GAN framework.
 load("//tensorflow:tensorflow.bzl", "py_test")
 
 package(default_visibility = [
diff --git a/tensorflow/contrib/gan/python/estimator/python/gan_estimator_impl.py b/tensorflow/contrib/gan/python/estimator/python/gan_estimator_impl.py
index adb7222821..dd904611d1 100644
--- a/tensorflow/contrib/gan/python/estimator/python/gan_estimator_impl.py
+++ b/tensorflow/contrib/gan/python/estimator/python/gan_estimator_impl.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""A TFGAN-backed GAN Estimator."""
+"""A TF-GAN-backed GAN Estimator."""
 
 from __future__ import absolute_import
 from __future__ import division
@@ -56,10 +56,10 @@ _summary_type_map = {
 class GANEstimator(estimator.Estimator):
   """An estimator for Generative Adversarial Networks (GANs).
 
-  This Estimator is backed by TFGAN. The network functions follow the TFGAN API
-  except for one exception: if either `generator_fn` or `discriminator_fn` have
-  an argument called `mode`, then the tf.Estimator mode is passed in for that
-  argument. This helps with operations like batch normalization, which have
+  This Estimator is backed by TF-GAN. The network functions follow the TF-GAN
+  API except for one exception: if either `generator_fn` or `discriminator_fn`
+  have an argument called `mode`, then the tf.Estimator mode is passed in for
+  that argument. This helps with operations like batch normalization, which have
   different train and evaluation behavior.
 
   Example:
@@ -68,7 +68,7 @@ class GANEstimator(estimator.Estimator):
       import tensorflow as tf
       tfgan = tf.contrib.gan
 
-      # See TFGAN's `train.py` for a description of the generator and
+      # See TF-GAN's `train.py` for a description of the generator and
       # discriminator API.
       def generator_fn(generator_inputs):
         ...
@@ -123,13 +123,13 @@ class GANEstimator(estimator.Estimator):
         to continue training a previously saved model.
       generator_fn: A python function that takes a Tensor, Tensor list, or
         Tensor dictionary as inputs and returns the outputs of the GAN
-        generator. See `TFGAN` for more details and examples. Additionally, if
+        generator. See `TF-GAN` for more details and examples. Additionally, if
         it has an argument called `mode`, the Estimator's `mode` will be passed
         in (ex TRAIN, EVAL, PREDICT). This is useful for things like batch
         normalization.
       discriminator_fn: A python function that takes the output of
         `generator_fn` or real data in the GAN setup, and `generator_inputs`.
-        Outputs a Tensor in the range [-inf, inf]. See `TFGAN` for more details
+        Outputs a Tensor in the range [-inf, inf]. See `TF-GAN` for more details
         and examples.
       generator_loss_fn: The loss function on the generator. Takes a `GANModel`
         tuple.
diff --git a/tensorflow/contrib/gan/python/losses/python/losses_impl.py b/tensorflow/contrib/gan/python/losses/python/losses_impl.py
index a0a86c6337..930c91786f 100644
--- a/tensorflow/contrib/gan/python/losses/python/losses_impl.py
+++ b/tensorflow/contrib/gan/python/losses/python/losses_impl.py
@@ -28,7 +28,7 @@ wasserstein_gradient_penalty
 All losses must be able to accept 1D or 2D Tensors, so as to be compatible with
 patchGAN style losses (https://arxiv.org/abs/1611.07004).
 
-To make these losses usable in the TFGAN framework, please create a tuple
+To make these losses usable in the TF-GAN framework, please create a tuple
 version of the losses with `losses_utils.py`.
 """
 
@@ -320,7 +320,7 @@ def wasserstein_gradient_penalty(
     generated_data: Output of the generator.
     generator_inputs: Exact argument to pass to the generator, which is used
       as optional conditioning to the discriminator.
-    discriminator_fn: A discriminator function that conforms to TFGAN API.
+    discriminator_fn: A discriminator function that conforms to TF-GAN API.
     discriminator_scope: If not `None`, reuse discriminators from this scope.
     epsilon: A small positive number added for numerical stability when
       computing the gradient norm.
diff --git a/tensorflow/contrib/gan/python/losses/python/tuple_losses_impl.py b/tensorflow/contrib/gan/python/losses/python/tuple_losses_impl.py
index 221c70c38b..76e57df7f6 100644
--- a/tensorflow/contrib/gan/python/losses/python/tuple_losses_impl.py
+++ b/tensorflow/contrib/gan/python/losses/python/tuple_losses_impl.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""TFGAN utilities for loss functions that accept GANModel namedtuples.
+"""TF-GAN utilities for loss functions that accept GANModel namedtuples.
 
 The losses and penalties in this file all correspond to losses in
 `losses_impl.py`. Losses in that file take individual arguments, whereas in this
diff --git a/tensorflow/contrib/gan/python/namedtuples.py b/tensorflow/contrib/gan/python/namedtuples.py
index 969b68449d..73dfee4fde 100644
--- a/tensorflow/contrib/gan/python/namedtuples.py
+++ b/tensorflow/contrib/gan/python/namedtuples.py
@@ -12,10 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Named tuples for TFGAN.
+"""Named tuples for TF-GAN.
 
-TFGAN training occurs in four steps, and each step communicates with the next
-step via one of these named tuples. At each step, you can either use a TFGAN
+TF-GAN training occurs in four steps, and each step communicates with the next
+step via one of these named tuples. At each step, you can either use a TF-GAN
 helper function in `train.py`, or you can manually construct a tuple.
 """
 
diff --git a/tensorflow/contrib/gan/python/train.py b/tensorflow/contrib/gan/python/train.py
index 4c7bee41b3..f36a5d346e 100644
--- a/tensorflow/contrib/gan/python/train.py
+++ b/tensorflow/contrib/gan/python/train.py
@@ -12,12 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""The TFGAN project provides a lightweight GAN training/testing framework.
+"""The TF-GAN project provides a lightweight GAN training/testing framework.
 
 This file contains the core helper functions to create and train a GAN model.
 See the README or examples in `tensorflow_models` for details on how to use.
 
-TFGAN training occurs in four steps:
+TF-GAN training occurs in four steps:
 1) Create a model
 2) Add a loss
 3) Create train ops
@@ -645,9 +645,10 @@ def gan_loss(
         type(model))
 
   # Optionally create pooled model.
-  pooled_model = (
-      _tensor_pool_adjusted_model(model, tensor_pool_fn)
-      if tensor_pool_fn else model)
+  if tensor_pool_fn:
+    pooled_model = _tensor_pool_adjusted_model(model, tensor_pool_fn)
+  else:
+    pooled_model = model
 
   # Create standard losses.
   gen_loss = generator_loss_fn(model, add_summaries=add_summaries)
@@ -665,10 +666,11 @@ def gan_loss(
   if _use_aux_loss(mutual_information_penalty_weight):
     gen_info_loss = tfgan_losses.mutual_information_penalty(
         model, add_summaries=add_summaries)
-    dis_info_loss = (
-        gen_info_loss
-        if tensor_pool_fn is None else tfgan_losses.mutual_information_penalty(
-            pooled_model, add_summaries=add_summaries))
+    if tensor_pool_fn is None:
+      dis_info_loss = gen_info_loss
+    else:
+      dis_info_loss = tfgan_losses.mutual_information_penalty(
+          pooled_model, add_summaries=add_summaries)
     gen_loss += mutual_information_penalty_weight * gen_info_loss
     dis_loss += mutual_information_penalty_weight * dis_info_loss
   if _use_aux_loss(aux_cond_generator_weight):
@@ -929,7 +931,7 @@ def gan_train_ops(
     **kwargs):
   """Returns GAN train ops.
 
-  The highest-level call in TFGAN. It is composed of functions that can also
+  The highest-level call in TF-GAN. It is composed of functions that can also
   be called, should a user require more control over some part of the GAN
   training process.
 
-- 
GitLab


From cb86c83b06bdb0ad04d8c383515fbab5999dcfdc Mon Sep 17 00:00:00 2001
From: David Norman <davidn@graphcore.ai>
Date: Sun, 30 Dec 2018 12:04:13 +0000
Subject: [PATCH 0087/1095] Allow the reducewindow -> reduce optimization to be
 disabled

---
 .../xla/service/algebraic_simplifier.cc       | 86 ++++++++++---------
 .../xla/service/algebraic_simplifier.h        | 13 +++
 2 files changed, 57 insertions(+), 42 deletions(-)

diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
index cad70a8d10..137e86216f 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
@@ -2595,51 +2595,53 @@ Status AlgebraicSimplifierVisitor::HandleReduceWindow(
                                   function));
   }
 
-  // A reduce window can be expressed as a reduce and a reshape if all
-  // dimensions either have a window size of one or the entire dimension. If
-  // there is no stride, dilation, or padding, this is as easy as checking the
-  // size of the output shape and window dimension.
-  //
-  // The reshape is a bitcast since it adds one-sized dimensions. Often these
-  // ones are immediately removed as well with another reshape. The
-  // implementation of reduce tends to be slightly more efficient at reducing
-  // entire dimensions compared to reduce window.
-  auto effective_reduce_dims = [&] {
-    if (window_util::HasStride(window) || window_util::HasDilation(window) ||
-        window_util::HasPadding(window)) {
-      return absl::InlinedVector<int64, 8>{};
-    }
-    absl::InlinedVector<int64, 8> reduce_dims;
-    for (int64 i = 0; i < window.dimensions_size(); ++i) {
-      if (window.dimensions(i).size() == 1) {
-        continue;
-      } else if (reduce_window->shape().dimensions(i) == 1) {
-        reduce_dims.push_back(i);
-      } else {
+  if (options_.enable_window_reduce_to_reduce_replacement()) {
+    // A reduce window can be expressed as a reduce and a reshape if all
+    // dimensions either have a window size of one or the entire dimension. If
+    // there is no stride, dilation, or padding, this is as easy as checking the
+    // size of the output shape and window dimension.
+    //
+    // The reshape is a bitcast since it adds one-sized dimensions. Often these
+    // ones are immediately removed as well with another reshape. The
+    // implementation of reduce tends to be slightly more efficient at reducing
+    // entire dimensions compared to reduce window.
+    auto effective_reduce_dims = [&] {
+      if (window_util::HasStride(window) || window_util::HasDilation(window) ||
+          window_util::HasPadding(window)) {
         return absl::InlinedVector<int64, 8>{};
       }
-    }
-    return reduce_dims;
-  }();
+      absl::InlinedVector<int64, 8> reduce_dims;
+      for (int64 i = 0; i < window.dimensions_size(); ++i) {
+        if (window.dimensions(i).size() == 1) {
+          continue;
+        } else if (reduce_window->shape().dimensions(i) == 1) {
+          reduce_dims.push_back(i);
+        } else {
+          return absl::InlinedVector<int64, 8>{};
+        }
+      }
+      return reduce_dims;
+    }();
 
-  // If a reduce window can be expressed as a reduce, do so and reshape the
-  // output.
-  if (!effective_reduce_dims.empty()) {
-    Shape reduce_shape = ShapeUtil::FilterDimensions(
-        [&](int64 dim) {
-          return !absl::c_linear_search(effective_reduce_dims, dim);
-        },
-        reduce_window->shape());
-    HloInstruction* reduce =
-        computation_->AddInstruction(HloInstruction::CreateReduce(
-            /*shape=*/reduce_shape,
-            /*operand=*/operand,
-            /*init_value=*/reduce_window->mutable_operand(1),
-            /*dimensions_to_reduce=*/effective_reduce_dims,
-            /*reduce_computation=*/function));
-    return ReplaceWithNewInstruction(
-        reduce_window,
-        HloInstruction::CreateReshape(reduce_window->shape(), reduce));
+    // If a reduce window can be expressed as a reduce, do so and reshape the
+    // output.
+    if (!effective_reduce_dims.empty()) {
+      Shape reduce_shape = ShapeUtil::FilterDimensions(
+          [&](int64 dim) {
+            return !absl::c_linear_search(effective_reduce_dims, dim);
+          },
+          reduce_window->shape());
+      HloInstruction* reduce =
+          computation_->AddInstruction(HloInstruction::CreateReduce(
+              /*shape=*/reduce_shape,
+              /*operand=*/operand,
+              /*init_value=*/reduce_window->mutable_operand(1),
+              /*dimensions_to_reduce=*/effective_reduce_dims,
+              /*reduce_computation=*/function));
+      return ReplaceWithNewInstruction(
+          reduce_window,
+          HloInstruction::CreateReshape(reduce_window->shape(), reduce));
+    }
   }
 
   // This optimization folds a pad op into reduce_window.
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.h b/tensorflow/compiler/xla/service/algebraic_simplifier.h
index d2775b9faf..695fc3af3e 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.h
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.h
@@ -75,12 +75,25 @@ class AlgebraicSimplifierOptions {
     return enable_permutation_sort_replacement_;
   }
 
+  // If enable_window_reduce_replacement is true, the kReduceWindow instruction
+  // can be optimized by replacement with simpler operations.
+  void set_enable_window_reduce_to_reduce_replacement(
+      bool enable_window_reduce_to_reduce_replacement) {
+    enable_window_reduce_to_reduce_replacement_ =
+        enable_window_reduce_to_reduce_replacement;
+  }
+  bool enable_window_reduce_to_reduce_replacement() const {
+    return enable_window_reduce_to_reduce_replacement_;
+  }
+
+
  private:
   ValidBitcastCallback valid_bitcast_callback_;
   bool is_layout_sensitive_{false};
   bool enable_dot_strength_reduction_{true};
   bool enable_conv_simplification_{true};
   bool enable_permutation_sort_replacement_{false};
+  bool enable_window_reduce_to_reduce_replacement_{true};
 };
 
 // A pass which performs algebraic simplifications.
-- 
GitLab


From 44bb7adf1d04f090817f008fc47e4d81a23b1049 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 30 Dec 2019 06:18:15 -0800
Subject: [PATCH 0088/1095] [tfgan/tpu] Add loss summaries during
 TPUGANEstimator training if not on TPU.

PiperOrigin-RevId: 227283183
---
 .../gan/python/estimator/python/tpu_gan_estimator_impl.py    | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/gan/python/estimator/python/tpu_gan_estimator_impl.py b/tensorflow/contrib/gan/python/estimator/python/tpu_gan_estimator_impl.py
index 295d1382e2..bf51b7fc45 100644
--- a/tensorflow/contrib/gan/python/estimator/python/tpu_gan_estimator_impl.py
+++ b/tensorflow/contrib/gan/python/estimator/python/tpu_gan_estimator_impl.py
@@ -294,9 +294,10 @@ def _get_estimator_spec(
         gan_model, gan_loss, gan_loss_no_reduction, get_eval_metric_ops_fn)
   else:  # model_fn_lib.ModeKeys.TRAIN:
     gan_loss = tfgan_tuples.GANLoss(
-        generator_loss=generator_loss_fn(gan_model, add_summaries=False),
+        generator_loss=generator_loss_fn(
+            gan_model, add_summaries=not is_on_tpu),
         discriminator_loss=discriminator_loss_fn(
-            gan_model, add_summaries=False))
+            gan_model, add_summaries=not is_on_tpu))
 
     # Construct optimizers if arguments were callable. For TPUs, they must be
     # `CrossShardOptimizer`.
-- 
GitLab


From 8f60a381d210478f21762a6cf14f547a05e98878 Mon Sep 17 00:00:00 2001
From: Michael Case <mikecase@google.com>
Date: Mon, 30 Dec 2019 09:31:02 -0800
Subject: [PATCH 0089/1095] Replace py_test with tf_py_test in more BUILD
 files.

This will allow some easier refactoring for python test dependencies. Also,
looking at impl, I feel like there is almost no case in which using
tensorflow.bzl:py_test is preferable to tensorflow.bzl:tf_py_test.

PiperOrigin-RevId: 227291326
---
 tensorflow/python/autograph/impl/BUILD        |  16 +-
 tensorflow/python/eager/BUILD                 |  29 +-
 tensorflow/python/feature_column/BUILD        |  38 +-
 tensorflow/python/keras/BUILD                 | 541 ++++++++----------
 tensorflow/python/saved_model/BUILD           |  53 +-
 .../python/training/checkpointable/BUILD      |  39 +-
 6 files changed, 320 insertions(+), 396 deletions(-)

diff --git a/tensorflow/python/autograph/impl/BUILD b/tensorflow/python/autograph/impl/BUILD
index 201a888754..66f7915696 100644
--- a/tensorflow/python/autograph/impl/BUILD
+++ b/tensorflow/python/autograph/impl/BUILD
@@ -1,6 +1,6 @@
 licenses(["notice"])  # Apache 2.0
 
-load("//tensorflow:tensorflow.bzl", "py_test")
+load("//tensorflow:tensorflow.bzl", "tf_py_test")
 
 filegroup(
     name = "all_files",
@@ -37,25 +37,23 @@ py_library(
     ],
 )
 
-py_test(
+tf_py_test(
     name = "api_test",
     srcs = ["api_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
+    additional_deps = [
         ":impl",
+        "//third_party/py/numpy",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python/autograph/utils",
-        "//third_party/py/numpy",
     ],
 )
 
-py_test(
+tf_py_test(
     name = "conversion_test",
     srcs = ["conversion_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
+    additional_deps = [
         ":impl",
-        "//tensorflow/python:client_testlib",
         "@gast_archive//:gast",
+        "//tensorflow/python:client_testlib",
     ],
 )
diff --git a/tensorflow/python/eager/BUILD b/tensorflow/python/eager/BUILD
index cd5c0be283..1f1cb22d58 100644
--- a/tensorflow/python/eager/BUILD
+++ b/tensorflow/python/eager/BUILD
@@ -1,6 +1,6 @@
 licenses(["notice"])  # Apache 2.0
 
-load("//tensorflow:tensorflow.bzl", "py_test", "tf_cc_binary")
+load("//tensorflow:tensorflow.bzl", "tf_py_test", "tf_cc_binary")
 load("//tensorflow:tensorflow.bzl", "cuda_py_test")
 load(
     "//tensorflow/tools/test:performance.bzl",
@@ -255,11 +255,10 @@ py_library(
     ],
 )
 
-py_test(
+tf_py_test(
     name = "execution_callbacks_test",
     srcs = ["execution_callbacks_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
+    additional_deps = [
         ":execution_callbacks",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_ops",
@@ -373,11 +372,10 @@ tf_py_logged_benchmark(
     target = "//tensorflow/python/eager:benchmarks_test",
 )
 
-py_test(
+tf_py_test(
     name = "tape_test",
     srcs = ["tape_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
+    additional_deps = [
         ":backprop",
         ":context",
         ":test",
@@ -414,20 +412,19 @@ cuda_py_test(
     ],
 )
 
-py_test(
+tf_py_test(
     name = "pywrap_tfe_test",
     srcs = ["pywrap_tfe_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
+    additional_deps = [
         ":backprop",
         ":context",
         ":core",
         ":test",
+        "//third_party/py/numpy",
         "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:math_ops",
         "//tensorflow/python:pywrap_tensorflow",
         "//tensorflow/python:random_ops",
-        "//third_party/py/numpy",
     ],
 )
 
@@ -491,11 +488,10 @@ py_library(
     ],
 )
 
-py_test(
+tf_py_test(
     name = "def_function_test",
     srcs = ["def_function_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
+    additional_deps = [
         ":def_function",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:constant_op",
@@ -519,11 +515,10 @@ py_library(
     ],
 )
 
-py_test(
+tf_py_test(
     name = "wrap_function_test",
     srcs = ["wrap_function_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
+    additional_deps = [
         ":wrap_function",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_ops",
diff --git a/tensorflow/python/feature_column/BUILD b/tensorflow/python/feature_column/BUILD
index d24a7ae80c..2b986348b7 100644
--- a/tensorflow/python/feature_column/BUILD
+++ b/tensorflow/python/feature_column/BUILD
@@ -4,7 +4,7 @@ package(
 
 licenses(["notice"])  # Apache 2.0
 
-load("//tensorflow:tensorflow.bzl", "py_test")
+load("//tensorflow:tensorflow.bzl", "tf_py_test")
 
 py_library(
     name = "feature_column_py",
@@ -94,19 +94,13 @@ filegroup(
     ],
 )
 
-py_test(
+tf_py_test(
     name = "feature_column_test",
     srcs = ["feature_column_test.py"],
-    data = [":vocabulary_testdata"],
-    srcs_version = "PY2AND3",
-    tags = [
-        "no_cuda_on_cpu_tap",
-        "no_pip",
-        "no_windows",
-    ],
-    deps = [
+    additional_deps = [
         ":feature_column",
         ":feature_column_py",
+        "//third_party/py/numpy",
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:client_testlib",
@@ -125,24 +119,22 @@ py_test(
         "//tensorflow/python:variables",
         "//tensorflow/python/eager:backprop",
         "//tensorflow/python/eager:context",
-        "//third_party/py/numpy",
     ],
-)
-
-py_test(
-    name = "feature_column_v2_test",
-    srcs = ["feature_column_v2_test.py"],
     data = [":vocabulary_testdata"],
-    shard_count = 5,
-    srcs_version = "PY2AND3",
     tags = [
         "no_cuda_on_cpu_tap",
         "no_pip",
         "no_windows",
     ],
-    deps = [
+)
+
+tf_py_test(
+    name = "feature_column_v2_test",
+    srcs = ["feature_column_v2_test.py"],
+    additional_deps = [
         ":feature_column_py",
         ":feature_column_v2",
+        "//third_party/py/numpy",
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:client_testlib",
@@ -162,6 +154,12 @@ py_test(
         "//tensorflow/python/eager:backprop",
         "//tensorflow/python/eager:context",
         "//tensorflow/python/estimator:estimator_py",
-        "//third_party/py/numpy",
+    ],
+    data = [":vocabulary_testdata"],
+    shard_count = 5,
+    tags = [
+        "no_cuda_on_cpu_tap",
+        "no_pip",
+        "no_windows",
     ],
 )
diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index d833ae0fa2..914568f652 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -7,7 +7,7 @@ package(default_visibility = ["//visibility:public"])
 
 exports_files(["LICENSE"])
 
-load("//tensorflow:tensorflow.bzl", "py_test")
+load("//tensorflow:tensorflow.bzl", "tf_py_test")
 load("//tensorflow:tensorflow.bzl", "cuda_py_test")
 
 config_setting(
@@ -304,180 +304,167 @@ py_library(
     ],
 )
 
-py_test(
+tf_py_test(
     name = "integration_test",
     size = "medium",
     srcs = ["integration_test.py"],
-    srcs_version = "PY2AND3",
-    tags = ["notsan"],
-    deps = [
+    additional_deps = [
         ":keras",
+        "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:layers",
         "//tensorflow/python:nn",
-        "//third_party/py/numpy",
-        "@absl_py//absl/testing:parameterized",
     ],
+    tags = ["notsan"],
 )
 
-py_test(
+tf_py_test(
     name = "activations_test",
     size = "small",
     srcs = ["activations_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
+    additional_deps = [
         ":keras",
-        "//tensorflow/python:client_testlib",
-        "//third_party/py/numpy",
         "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
     ],
 )
 
-py_test(
+tf_py_test(
     name = "constraints_test",
     size = "small",
     srcs = ["constraints_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
+    additional_deps = [
         ":keras",
-        "//tensorflow/python:client_testlib",
-        "//third_party/py/numpy",
         "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
     ],
 )
 
-py_test(
+tf_py_test(
     name = "initializers_test",
     size = "small",
     srcs = ["initializers_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
+    additional_deps = [
         ":keras",
+        "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:init_ops",
-        "//third_party/py/numpy",
-        "@absl_py//absl/testing:parameterized",
     ],
 )
 
-py_test(
+tf_py_test(
     name = "regularizers_test",
     size = "small",
     srcs = ["regularizers_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
+    additional_deps = [
         ":keras",
-        "//tensorflow/python:client_testlib",
         "@absl_py//absl/testing:parameterized",
+        "//tensorflow/python:client_testlib",
     ],
 )
 
-py_test(
+tf_py_test(
     name = "optimizers_test",
     size = "medium",
     srcs = ["optimizers_test.py"],
-    shard_count = 2,
-    srcs_version = "PY2AND3",
-    tags = ["notsan"],
-    deps = [
+    additional_deps = [
         ":keras",
+        "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:training",
-        "//third_party/py/numpy",
-        "@absl_py//absl/testing:parameterized",
     ],
+    shard_count = 2,
+    tags = ["notsan"],
 )
 
-py_test(
+tf_py_test(
     name = "losses_test",
     size = "small",
     srcs = ["losses_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
+    additional_deps = [
         ":keras",
-        "//tensorflow/python:client_testlib",
-        "//third_party/py/numpy",
         "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
     ],
 )
 
-py_test(
+tf_py_test(
     name = "metrics_functional_test",
     size = "medium",
     srcs = ["metrics_functional_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
+    additional_deps = [
         ":keras",
-        "//tensorflow/python:client_testlib",
         "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
     ],
 )
 
-py_test(
+tf_py_test(
     name = "metrics_test",
     size = "medium",
     srcs = ["metrics_test.py"],
-    shard_count = 4,
-    srcs_version = "PY2AND3",
-    deps = [
+    additional_deps = [
         ":keras",
-        "//tensorflow/python:client_testlib",
-        "//third_party/py/numpy",
         "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
     ],
+    shard_count = 4,
 )
 
-py_test(
+tf_py_test(
     name = "applications_test",
     size = "enormous",
     srcs = ["applications/applications_test.py"],
-    shard_count = 2,
-    srcs_version = "PY2AND3",
-    deps = [
+    additional_deps = [
         ":keras",
-        "//tensorflow/python:client_testlib",
         "@absl_py//absl/testing:parameterized",
+        "//tensorflow/python:client_testlib",
     ],
+    shard_count = 2,
 )
 
-py_test(
+tf_py_test(
     name = "advanced_activations_test",
     size = "medium",
     srcs = ["layers/advanced_activations_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
+    additional_deps = [
         ":keras",
-        "//tensorflow/python:client_testlib",
         "@absl_py//absl/testing:parameterized",
+        "//tensorflow/python:client_testlib",
     ],
 )
 
-py_test(
+tf_py_test(
     name = "convolutional_recurrent_test",
     size = "large",
     srcs = ["layers/convolutional_recurrent_test.py"],
-    shard_count = 2,
-    srcs_version = "PY2AND3",
-    deps = [
+    additional_deps = [
         ":keras",
-        "//tensorflow/python:client_testlib",
-        "//third_party/py/numpy",
         "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
     ],
+    shard_count = 2,
 )
 
-py_test(
+tf_py_test(
     name = "convolutional_test",
     size = "large",
     srcs = ["layers/convolutional_test.py"],
-    shard_count = 11,
-    srcs_version = "PY2AND3",
-    deps = [
+    additional_deps = [
         ":keras",
-        "//tensorflow/python:client_testlib",
-        "//third_party/py/numpy",
         "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
     ],
+    shard_count = 11,
 )
 
 cuda_py_test(
@@ -494,30 +481,28 @@ cuda_py_test(
     tags = ["no_windows_gpu"],
 )
 
-py_test(
+tf_py_test(
     name = "pooling_test",
     size = "large",
     srcs = ["layers/pooling_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
+    additional_deps = [
         ":keras",
-        "//tensorflow/python:client_testlib",
         "@absl_py//absl/testing:parameterized",
+        "//tensorflow/python:client_testlib",
     ],
 )
 
-py_test(
+tf_py_test(
     name = "core_test",
     size = "medium",
     srcs = ["layers/core_test.py"],
-    shard_count = 3,
-    srcs_version = "PY2AND3",
-    deps = [
+    additional_deps = [
         ":keras",
-        "//tensorflow/python:client_testlib",
-        "//third_party/py/numpy",
         "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
     ],
+    shard_count = 3,
 )
 
 cuda_py_test(
@@ -531,121 +516,113 @@ cuda_py_test(
     ],
 )
 
-py_test(
+tf_py_test(
     name = "local_test",
     size = "medium",
     srcs = ["layers/local_test.py"],
-    shard_count = 2,
-    srcs_version = "PY2AND3",
-    tags = ["no_windows"],
-    deps = [
+    additional_deps = [
         ":keras",
-        "//tensorflow/python:client_testlib",
-        "//third_party/py/numpy",
         "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
     ],
+    shard_count = 2,
+    tags = ["no_windows"],
 )
 
-py_test(
+tf_py_test(
     name = "merge_test",
     size = "small",
     srcs = ["layers/merge_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
+    additional_deps = [
         ":keras",
-        "//tensorflow/python:client_testlib",
-        "//third_party/py/numpy",
         "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
     ],
 )
 
-py_test(
+tf_py_test(
     name = "noise_test",
     size = "small",
     srcs = ["layers/noise_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
+    additional_deps = [
         ":keras",
-        "//tensorflow/python:client_testlib",
         "@absl_py//absl/testing:parameterized",
+        "//tensorflow/python:client_testlib",
     ],
 )
 
-py_test(
+tf_py_test(
     name = "normalization_test",
     size = "medium",
     srcs = ["layers/normalization_test.py"],
-    shard_count = 3,
-    srcs_version = "PY2AND3",
-    tags = ["notsan"],
-    deps = [
+    additional_deps = [
         ":keras",
-        "//tensorflow/python:client_testlib",
-        "//third_party/py/numpy",
         "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
     ],
+    shard_count = 3,
+    tags = ["notsan"],
 )
 
-py_test(
+tf_py_test(
     name = "simplernn_test",
     size = "medium",
     srcs = ["layers/simplernn_test.py"],
-    shard_count = 4,
-    srcs_version = "PY2AND3",
-    tags = ["notsan"],
-    deps = [
+    additional_deps = [
         ":keras",
-        "//tensorflow/python:client_testlib",
-        "//third_party/py/numpy",
         "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
     ],
+    shard_count = 4,
+    tags = ["notsan"],
 )
 
-py_test(
+tf_py_test(
     name = "gru_test",
     size = "large",
     srcs = ["layers/gru_test.py"],
-    shard_count = 2,
-    srcs_version = "PY2AND3",
-    tags = ["notsan"],  # http://b/62136390
-    deps = [
+    additional_deps = [
         ":keras",
-        "//tensorflow/python:client_testlib",
-        "//third_party/py/numpy",
         "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
     ],
+    shard_count = 2,
+    tags = ["notsan"],  # http://b/62136390
 )
 
-py_test(
+tf_py_test(
     name = "lstm_test",
     size = "medium",
     srcs = ["layers/lstm_test.py"],
+    additional_deps = [
+        ":keras",
+        "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
+    ],
     shard_count = 4,
-    srcs_version = "PY2AND3",
     tags = [
         "noasan",  # times out b/63678675
         "notsan",  # http://b/62189182
     ],
-    deps = [
-        ":keras",
-        "//tensorflow/python:client_testlib",
-        "//third_party/py/numpy",
-        "@absl_py//absl/testing:parameterized",
-    ],
 )
 
-py_test(
+tf_py_test(
     name = "recurrent_test",
     size = "medium",
     srcs = ["layers/recurrent_test.py"],
-    shard_count = 4,
-    srcs_version = "PY2AND3",
-    deps = [
+    additional_deps = [
         ":keras",
-        "//tensorflow/python:client_testlib",
-        "//third_party/py/numpy",
         "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
     ],
+    shard_count = 4,
 )
 
 cuda_py_test(
@@ -674,55 +651,57 @@ cuda_py_test(
     shard_count = 6,
 )
 
-py_test(
+tf_py_test(
     name = "serialization_test",
     size = "small",
     srcs = ["layers/serialization_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
+    additional_deps = [
         ":keras",
-        "//tensorflow/python:client_testlib",
         "@absl_py//absl/testing:parameterized",
+        "//tensorflow/python:client_testlib",
     ],
 )
 
-py_test(
+tf_py_test(
     name = "wrappers_test",
     size = "medium",
     srcs = ["layers/wrappers_test.py"],
+    additional_deps = [
+        ":keras",
+        "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
+    ],
     shard_count = 4,
-    srcs_version = "PY2AND3",
     tags = [
         "noasan",  # http://b/78599823
         "notsan",
     ],
-    deps = [
-        ":keras",
-        "//tensorflow/python:client_testlib",
-        "//third_party/py/numpy",
-        "@absl_py//absl/testing:parameterized",
-    ],
 )
 
-py_test(
+tf_py_test(
     name = "scikit_learn_test",
     size = "small",
     srcs = ["wrappers/scikit_learn_test.py"],
-    srcs_version = "PY2AND3",
-    tags = ["notsan"],
-    deps = [
+    additional_deps = [
         ":keras",
-        "//tensorflow/python:client_testlib",
-        "//third_party/py/numpy",
         "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
     ],
+    tags = ["notsan"],
 )
 
-py_test(
+tf_py_test(
     name = "data_utils_test",
     size = "large",
     srcs = ["utils/data_utils_test.py"],
-    srcs_version = "PY2AND3",
+    additional_deps = [
+        ":keras",
+        "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
+    ],
     tags = [
         "no_oss",
         "no_windows",
@@ -730,64 +709,54 @@ py_test(
         "notsan",
         "optonly",  # times out
     ],
-    deps = [
-        ":keras",
-        "//tensorflow/python:client_testlib",
-        "//third_party/py/numpy",
-        "@absl_py//absl/testing:parameterized",
-    ],
 )
 
-py_test(
+tf_py_test(
     name = "generic_utils_test",
     size = "small",
     srcs = ["utils/generic_utils_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
+    additional_deps = [
         ":keras",
-        "//tensorflow/python:client_testlib",
         "@absl_py//absl/testing:parameterized",
+        "//tensorflow/python:client_testlib",
     ],
 )
 
-py_test(
+tf_py_test(
     name = "tf_utils_test",
     size = "small",
     srcs = ["utils/tf_utils_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
+    additional_deps = [
         ":keras",
         "//tensorflow/python:client_testlib",
     ],
 )
 
-py_test(
+tf_py_test(
     name = "io_utils_test",
     size = "small",
     srcs = ["utils/io_utils_test.py"],
-    srcs_version = "PY2AND3",
+    additional_deps = [
+        ":keras",
+        "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
+    ],
     tags = [
         "no_windows",  # TODO: needs investigation on Windows
         "notsan",
     ],
-    deps = [
-        ":keras",
-        "//tensorflow/python:client_testlib",
-        "//third_party/py/numpy",
-        "@absl_py//absl/testing:parameterized",
-    ],
 )
 
-py_test(
+tf_py_test(
     name = "np_utils_test",
     size = "small",
     srcs = ["utils/np_utils_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
+    additional_deps = [
         ":keras",
-        "//tensorflow/python:client_testlib",
-        "//third_party/py/numpy",
         "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
     ],
 )
 
@@ -818,287 +787,267 @@ cuda_py_test(
     ],
 )
 
-py_test(
+tf_py_test(
     name = "conv_utils_test",
     size = "small",
     srcs = ["utils/conv_utils_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
+    additional_deps = [
         ":keras",
-        "//tensorflow/python:client_testlib",
-        "//third_party/py/numpy",
         "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
     ],
 )
 
-py_test(
+tf_py_test(
     name = "image_test",
     size = "medium",
     srcs = ["preprocessing/image_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
+    additional_deps = [
         ":keras",
-        "//tensorflow/python:client_testlib",
-        "//third_party/py/numpy",
         "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
     ],
 )
 
-py_test(
+tf_py_test(
     name = "sequence_test",
     size = "small",
     srcs = ["preprocessing/sequence_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
+    additional_deps = [
         ":keras",
-        "//tensorflow/python:client_testlib",
-        "//third_party/py/numpy",
         "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
     ],
 )
 
-py_test(
+tf_py_test(
     name = "text_test",
     size = "small",
     srcs = ["preprocessing/text_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
+    additional_deps = [
         ":keras",
-        "//tensorflow/python:client_testlib",
-        "//third_party/py/numpy",
         "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
     ],
 )
 
-py_test(
+tf_py_test(
     name = "callbacks_test",
     size = "medium",
     srcs = ["callbacks_test.py"],
-    srcs_version = "PY2AND3",
-    tags = ["notsan"],
-    deps = [
+    additional_deps = [
         ":keras",
-        "//tensorflow/python:client_testlib",
-        "//third_party/py/numpy",
         "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
     ],
+    tags = ["notsan"],
 )
 
-py_test(
+tf_py_test(
     name = "correctness_test",
     size = "medium",
     srcs = ["engine/correctness_test.py"],
-    shard_count = 2,
-    srcs_version = "PY2AND3",
-    tags = ["notsan"],
-    deps = [
+    additional_deps = [
         ":keras",
-        "//tensorflow/python:client_testlib",
-        "//third_party/py/numpy",
         "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
     ],
+    shard_count = 2,
+    tags = ["notsan"],
 )
 
-py_test(
+tf_py_test(
     name = "training_test",
     size = "medium",
     srcs = ["engine/training_test.py"],
-    shard_count = 16,
-    srcs_version = "PY2AND3",
-    tags = ["notsan"],
-    deps = [
+    additional_deps = [
         ":keras",
-        "//tensorflow/python:client_testlib",
-        "//third_party/py/numpy",
         "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
     ],
+    shard_count = 16,
+    tags = ["notsan"],
 )
 
-py_test(
+tf_py_test(
     name = "training_dataset_test",
     size = "medium",
     srcs = ["engine/training_dataset_test.py"],
-    shard_count = 4,
-    srcs_version = "PY2AND3",
-    deps = [
+    additional_deps = [
         ":keras",
-        "//tensorflow/python:client_testlib",
-        "//third_party/py/numpy",
         "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
     ],
+    shard_count = 4,
 )
 
-py_test(
+tf_py_test(
     name = "training_generator_test",
     size = "large",
     srcs = ["engine/training_generator_test.py"],
+    additional_deps = [
+        ":keras",
+        "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
+    ],
     shard_count = 3,
-    srcs_version = "PY2AND3",
     tags = [
         "no_oss",
         "notsan",
     ],
-    deps = [
-        ":keras",
-        "//tensorflow/python:client_testlib",
-        "//third_party/py/numpy",
-        "@absl_py//absl/testing:parameterized",
-    ],
 )
 
-py_test(
+tf_py_test(
     name = "feature_columns_integration_test",
     size = "small",
     srcs = ["engine/feature_columns_integration_test.py"],
-    srcs_version = "PY2AND3",
-    tags = ["notsan"],
-    deps = [
+    additional_deps = [
         ":keras",
+        "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python/feature_column:feature_column_py",
-        "//third_party/py/numpy",
-        "@absl_py//absl/testing:parameterized",
     ],
+    tags = ["notsan"],
 )
 
-py_test(
+tf_py_test(
     name = "training_eager_test",
     size = "medium",
     srcs = ["engine/training_eager_test.py"],
-    srcs_version = "PY2AND3",
-    tags = ["notsan"],
-    deps = [
+    additional_deps = [
         ":keras",
-        "//tensorflow/python:client_testlib",
-        "//third_party/py/numpy",
         "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
     ],
+    tags = ["notsan"],
 )
 
-py_test(
+tf_py_test(
     name = "training_utils_test",
     size = "medium",
     srcs = ["engine/training_utils_test.py"],
-    srcs_version = "PY2AND3",
-    tags = ["notsan"],
-    deps = [
+    additional_deps = [
         ":keras",
-        "//tensorflow/python:client_testlib",
-        "//third_party/py/numpy",
         "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
     ],
+    tags = ["notsan"],
 )
 
-py_test(
+tf_py_test(
     name = "model_subclassing_test",
     size = "medium",
     srcs = ["model_subclassing_test.py"],
-    shard_count = 4,
-    srcs_version = "PY2AND3",
-    tags = ["notsan"],
-    deps = [
+    additional_deps = [
         ":keras",
-        "//tensorflow/python:client_testlib",
-        "//third_party/py/numpy",
         "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
     ],
+    shard_count = 4,
+    tags = ["notsan"],
 )
 
-py_test(
+tf_py_test(
     name = "topology_test",
     size = "medium",
     srcs = ["engine/topology_test.py"],
-    srcs_version = "PY2AND3",
-    tags = [
-        "no-internal-py3",
-    ],
-    deps = [
+    additional_deps = [
         ":keras",
-        "//tensorflow/python:client_testlib",
-        "//third_party/py/numpy",
         "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
+    ],
+    tags = [
+        "no-internal-py3",
     ],
 )
 
-py_test(
+tf_py_test(
     name = "base_layer_test",
     size = "small",
     srcs = ["engine/base_layer_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
+    additional_deps = [
         ":keras",
-        "//tensorflow/python:client_testlib",
-        "//third_party/py/numpy",
         "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
     ],
 )
 
-py_test(
+tf_py_test(
     name = "saving_test",
     size = "medium",
     srcs = ["engine/saving_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
+    additional_deps = [
         ":keras",
-        "//tensorflow/python:client_testlib",
-        "//third_party/py/numpy",
         "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
     ],
 )
 
-py_test(
+tf_py_test(
     name = "sequential_test",
     size = "medium",
     srcs = ["engine/sequential_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
+    additional_deps = [
         ":keras",
-        "//tensorflow/python:client_testlib",
-        "//third_party/py/numpy",
         "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
     ],
 )
 
-py_test(
+tf_py_test(
     name = "models_test",
     size = "medium",
     srcs = ["models_test.py"],
-    shard_count = 2,
-    srcs_version = "PY2AND3",
-    tags = ["notsan"],  # b/67509773
-    deps = [
+    additional_deps = [
         ":keras",
+        "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:training",
-        "//third_party/py/numpy",
-        "@absl_py//absl/testing:parameterized",
     ],
+    shard_count = 2,
+    tags = ["notsan"],  # b/67509773
 )
 
-py_test(
+tf_py_test(
     name = "backend_test",
     size = "medium",
     srcs = ["backend_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
+    additional_deps = [
         ":keras",
+        "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:util",
-        "//third_party/py/numpy",
-        "@absl_py//absl/testing:parameterized",
     ],
 )
 
-py_test(
+tf_py_test(
     name = "keras_parameterized_test",
     size = "small",
     srcs = ["keras_parameterized_test.py"],
-    srcs_version = "PY2AND3",
-    tags = ["notsan"],
-    deps = [
+    additional_deps = [
         ":keras",
-        "//tensorflow/python:client_testlib",
-        "//third_party/py/numpy",
         "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
     ],
+    tags = ["notsan"],
 )
diff --git a/tensorflow/python/saved_model/BUILD b/tensorflow/python/saved_model/BUILD
index 40d7e2f25e..8da6ff5142 100644
--- a/tensorflow/python/saved_model/BUILD
+++ b/tensorflow/python/saved_model/BUILD
@@ -11,7 +11,7 @@ licenses(["notice"])  # Apache 2.0
 
 exports_files(["LICENSE"])
 
-load("//tensorflow:tensorflow.bzl", "py_test")
+load("//tensorflow:tensorflow.bzl", "tf_py_test")
 load("//tensorflow/core:platform/default/build_config.bzl", "tf_proto_library")
 load("//tensorflow/core:platform/default/build_config.bzl", "tf_additional_all_protos")
 
@@ -98,17 +98,16 @@ py_library(
     ],
 )
 
-py_test(
+tf_py_test(
     name = "loader_test",
     size = "small",
     srcs = ["loader_test.py"],
-    srcs_version = "PY2AND3",
-    visibility = ["//visibility:private"],
-    deps = [
+    additional_deps = [
         ":builder",
         ":loader",
         ":signature_def_utils",
         ":utils",
+        "@absl_py//absl/testing:parameterized",
         "//tensorflow/python:client",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:control_flow_ops",
@@ -118,7 +117,6 @@ py_test(
         "//tensorflow/python:state_ops",
         "//tensorflow/python:training",
         "//tensorflow/python:variables",
-        "@absl_py//absl/testing:parameterized",
     ],
 )
 
@@ -155,15 +153,11 @@ py_library(
     ],
 )
 
-py_test(
+tf_py_test(
     name = "saved_model_test",
     size = "small",
     srcs = ["saved_model_test.py"],
-    data = ["//tensorflow/cc/saved_model:saved_model_half_plus_two"],
-    srcs_version = "PY2AND3",
-    tags = ["no_windows"],
-    visibility = ["//visibility:private"],
-    deps = [
+    additional_deps = [
         ":builder",
         ":constants",
         ":loader",
@@ -186,6 +180,8 @@ py_test(
         "//tensorflow/python:util",
         "//tensorflow/python:variables",
     ],
+    data = ["//tensorflow/cc/saved_model:saved_model_half_plus_two"],
+    tags = ["no_windows"],
 )
 
 py_library(
@@ -205,13 +201,11 @@ py_library(
     ],
 )
 
-py_test(
+tf_py_test(
     name = "utils_test",
     size = "small",
     srcs = ["utils_test.py"],
-    srcs_version = "PY2AND3",
-    visibility = ["//visibility:private"],
-    deps = [
+    additional_deps = [
         ":utils",
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python:array_ops",
@@ -237,13 +231,11 @@ py_library(
     ],
 )
 
-py_test(
+tf_py_test(
     name = "signature_def_utils_test",
     size = "small",
     srcs = ["signature_def_utils_test.py"],
-    srcs_version = "PY2AND3",
-    visibility = ["//visibility:private"],
-    deps = [
+    additional_deps = [
         ":signature_constants",
         ":signature_def_utils",
         ":utils",
@@ -254,12 +246,11 @@ py_test(
     ],
 )
 
-py_test(
+tf_py_test(
     name = "simple_save_test",
     size = "small",
     srcs = ["simple_save_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
+    additional_deps = [
         ":loader",
         ":signature_constants",
         ":simple_save",
@@ -305,18 +296,17 @@ py_library(
     ],
 )
 
-py_test(
+tf_py_test(
     name = "save_test",
     srcs = ["save_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
+    additional_deps = [
         ":loader",
         ":save",
         ":signature_constants",
         ":tag_constants",
+        "@absl_py//absl/testing:parameterized",
         "//tensorflow/python/eager:def_function",
         "//tensorflow/python/eager:test",
-        "@absl_py//absl/testing:parameterized",
     ],
 )
 
@@ -339,11 +329,10 @@ py_library(
     ],
 )
 
-py_test(
+tf_py_test(
     name = "load_test",
     srcs = ["load_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
+    additional_deps = [
         ":load",
         ":save",
         "//tensorflow/python:constant_op",
@@ -408,10 +397,10 @@ py_library(
     ],
 )
 
-py_test(
+tf_py_test(
     name = "nested_structure_coder_test",
     srcs = ["nested_structure_coder_test.py"],
-    deps = [
+    additional_deps = [
         ":nested_structure_coder",
         ":struct_py",
         "//tensorflow/python:framework",
diff --git a/tensorflow/python/training/checkpointable/BUILD b/tensorflow/python/training/checkpointable/BUILD
index 595ce2a0da..855dc4fb68 100644
--- a/tensorflow/python/training/checkpointable/BUILD
+++ b/tensorflow/python/training/checkpointable/BUILD
@@ -11,7 +11,7 @@ licenses(["notice"])  # Apache 2.0
 
 exports_files(["LICENSE"])
 
-load("//tensorflow:tensorflow.bzl", "py_test")
+load("//tensorflow:tensorflow.bzl", "tf_py_test")
 load("//tensorflow/compiler/tests:build_defs.bzl", "tf_xla_py_test")
 
 py_library(
@@ -32,11 +32,10 @@ py_library(
     ],
 )
 
-py_test(
+tf_py_test(
     name = "base_test",
     srcs = ["base_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
+    additional_deps = [
         ":base",
         "//tensorflow/python:client_testlib",
     ],
@@ -52,11 +51,10 @@ py_library(
     ],
 )
 
-py_test(
+tf_py_test(
     name = "tracking_test",
     srcs = ["tracking_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
+    additional_deps = [
         ":base",
         ":tracking",
         "//tensorflow/python:client_testlib",
@@ -79,11 +77,10 @@ py_library(
     ],
 )
 
-py_test(
+tf_py_test(
     name = "data_structures_test",
     srcs = ["data_structures_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
+    additional_deps = [
         ":data_structures",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:framework_test_lib",
@@ -129,15 +126,15 @@ py_library(
     ],
 )
 
-py_test(
+tf_py_test(
     name = "util_test",
     srcs = ["util_test.py"],
-    srcs_version = "PY2AND3",
-    tags = ["notsan"],  # b/74395663
-    deps = [
+    additional_deps = [
         ":base",
         ":tracking",
         ":util",
+        "@absl_py//absl/testing:parameterized",
+        "@six_archive//:six",
         "//tensorflow/python:checkpoint_management",
         "//tensorflow/python:constant_op",
         "//tensorflow/python:control_flow_ops",
@@ -160,9 +157,8 @@ py_test(
         "//tensorflow/python/eager:test",
         "//tensorflow/python/keras:engine",
         "//tensorflow/python/keras:layers",
-        "@absl_py//absl/testing:parameterized",
-        "@six_archive//:six",
     ],
+    tags = ["notsan"],  # b/74395663
 )
 
 tf_xla_py_test(
@@ -188,15 +184,15 @@ tf_xla_py_test(
     ],
 )
 
-py_test(
+tf_py_test(
     name = "util_with_v1_optimizers_test",
     srcs = ["util_with_v1_optimizers_test.py"],
-    srcs_version = "PY2AND3",
-    tags = ["notsan"],  # b/74395663
-    deps = [
+    additional_deps = [
         ":base",
         ":tracking",
         ":util",
+        "@absl_py//absl/testing:parameterized",
+        "@six_archive//:six",
         "//tensorflow/python:checkpoint_management",
         "//tensorflow/python:constant_op",
         "//tensorflow/python:control_flow_ops",
@@ -220,7 +216,6 @@ py_test(
         "//tensorflow/python/eager:test",
         "//tensorflow/python/keras:engine",
         "//tensorflow/python/keras:layers",
-        "@absl_py//absl/testing:parameterized",
-        "@six_archive//:six",
     ],
+    tags = ["notsan"],  # b/74395663
 )
-- 
GitLab


From 4083cc3b3b9614e5ed013858d8727a2e08df1e25 Mon Sep 17 00:00:00 2001
From: Siju <siju.samuel@huawei.com>
Date: Mon, 31 Dec 2018 13:28:05 +0530
Subject: [PATCH 0090/1095] Update pattern_matcher.h

---
 tensorflow/compiler/xla/service/pattern_matcher.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/xla/service/pattern_matcher.h b/tensorflow/compiler/xla/service/pattern_matcher.h
index c362a60d94..83a73a90fb 100644
--- a/tensorflow/compiler/xla/service/pattern_matcher.h
+++ b/tensorflow/compiler/xla/service/pattern_matcher.h
@@ -1878,7 +1878,7 @@ class HloInstructionPattern {
   // Make this a templated function to work around gcc 4.9.4 template infinite
   // recursion bug.
   template <typename Dummy = void>
-  constexpr auto WithShapeEqualTo(const ::xla::Shape* shape)
+  constexpr auto WithShapeEqualTo(const ::xla::Shape* shape) const
       -> decltype(this->WithShape(Shape().EqualTo(shape))) {
     return WithShape(Shape().EqualTo(shape));
   }
@@ -1886,7 +1886,7 @@ class HloInstructionPattern {
   // Make this a templated function to work around gcc 4.9.4 template infinite
   // recursion bug.
   template <typename Dummy = void>
-  constexpr auto WithShapeCompatibleTo(const ::xla::Shape* shape)
+  constexpr auto WithShapeCompatibleTo(const ::xla::Shape* shape) const
       -> decltype(this->WithShape(Shape().CompatibleTo(shape))) {
     return WithShape(Shape().CompatibleTo(shape));
   }
-- 
GitLab


From 75557c1e21c89cddb23c66c661655075371ac28b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 31 Dec 2019 01:02:28 -0800
Subject: [PATCH 0091/1095] compat: Update forward compatibility horizon to
 2018-12-31

PiperOrigin-RevId: 227335513
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index c41e58a823..19efe84ca4 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 12, 30)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 12, 31)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From a36fc1c38b9cf0541cffc8ea053dfb8bf01c58e0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 31 Dec 2019 02:44:28 -0800
Subject: [PATCH 0092/1095] Remove NCCL remote config and dummy config.

We currently have two ways to disable NCCL support:
A) leave TF_NCCL_VERSION env variable undefined
B) bazel flag '--config=nonccl' or '--define=no_nccl_support=true'

After this change A) will build NCCL from source instead.
Add license to other binary targets, now that we ship NCCL with them.

PiperOrigin-RevId: 227342886
---
 tensorflow/opensource_only.files    | 13 ++++++-------
 tensorflow/tools/lib_package/BUILD  |  2 ++
 third_party/gpus/cuda_configure.bzl |  4 ++--
 third_party/nccl/archive.BUILD      |  2 +-
 third_party/nccl/nccl_configure.bzl | 23 +++++++++++------------
 third_party/nccl/remote.BUILD.tpl   |  6 ------
 6 files changed, 22 insertions(+), 28 deletions(-)
 delete mode 100644 third_party/nccl/remote.BUILD.tpl

diff --git a/tensorflow/opensource_only.files b/tensorflow/opensource_only.files
index 88800c2951..1054c285d6 100644
--- a/tensorflow/opensource_only.files
+++ b/tensorflow/opensource_only.files
@@ -69,13 +69,6 @@ tensorflow/third_party/toolchains/cpus/arm/BUILD
 tensorflow/third_party/toolchains/cpus/py3/BUILD
 tensorflow/third_party/toolchains/cpus/py/BUILD
 tensorflow/third_party/toolchains/BUILD
-tensorflow/third_party/nccl/remote.BUILD.tpl
-tensorflow/third_party/nccl/archive.BUILD
-tensorflow/third_party/nccl/LICENSE
-tensorflow/third_party/nccl/system.BUILD.tpl
-tensorflow/third_party/nccl/nccl_configure.bzl
-tensorflow/third_party/nccl/build_defs.bzl.tpl
-tensorflow/third_party/nccl/BUILD
 tensorflow/third_party/gpus/BUILD
 tensorflow/third_party/gpus/crosstool/clang/bin/crosstool_wrapper_driver_rocm.tpl
 tensorflow/third_party/gpus/crosstool/clang/bin/crosstool_wrapper_driver_is_not_gcc.tpl
@@ -175,6 +168,12 @@ tensorflow/third_party/llvm/expand_cmake_vars.py
 tensorflow/third_party/llvm/llvm.autogenerated.BUILD
 tensorflow/third_party/llvm/llvm.bzl
 tensorflow/third_party/icu/udata.patch
+tensorflow/third_party/nccl/archive.BUILD
+tensorflow/third_party/nccl/LICENSE
+tensorflow/third_party/nccl/system.BUILD.tpl
+tensorflow/third_party/nccl/nccl_configure.bzl
+tensorflow/third_party/nccl/build_defs.bzl.tpl
+tensorflow/third_party/nccl/BUILD
 tensorflow/third_party/fft2d/BUILD
 tensorflow/third_party/fft2d/fft.h
 tensorflow/third_party/fft2d/LICENSE
diff --git a/tensorflow/tools/lib_package/BUILD b/tensorflow/tools/lib_package/BUILD
index 1186189844..86bd510792 100644
--- a/tensorflow/tools/lib_package/BUILD
+++ b/tensorflow/tools/lib_package/BUILD
@@ -162,6 +162,7 @@ genrule(
         "//conditions:default": [],
     }) + if_cuda([
         "@cub_archive//:LICENSE.TXT",
+        "@local_config_nccl//:LICENSE",
     ]) + if_mkl([
         "//third_party/mkl:LICENSE",
         "//third_party/mkl_dnn:LICENSE",
@@ -232,6 +233,7 @@ genrule(
         "//conditions:default": [],
     }) + if_cuda([
         "@cub_archive//:LICENSE.TXT",
+        "@local_config_nccl//:LICENSE",
     ]) + if_mkl([
         "//third_party/mkl:LICENSE",
         "//third_party/mkl_dnn:LICENSE",
diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl
index 8aa5b89cdd..8de4fe58e5 100644
--- a/third_party/gpus/cuda_configure.bzl
+++ b/third_party/gpus/cuda_configure.bzl
@@ -400,7 +400,7 @@ def _cuda_include_path(repository_ctx, cuda_config):
   return "\n".join(inc_entries)
 
 
-def _enable_cuda(repository_ctx):
+def enable_cuda(repository_ctx):
   if "TF_NEED_CUDA" in repository_ctx.os.environ:
     enable_cuda = repository_ctx.os.environ["TF_NEED_CUDA"].strip()
     return enable_cuda == "1"
@@ -1545,7 +1545,7 @@ def _create_remote_cuda_repository(repository_ctx, remote_config_repo):
 
 def _cuda_autoconf_impl(repository_ctx):
   """Implementation of the cuda_autoconf repository rule."""
-  if not _enable_cuda(repository_ctx):
+  if not enable_cuda(repository_ctx):
     _create_dummy_repository(repository_ctx)
   elif _TF_CUDA_CONFIG_REPO in repository_ctx.os.environ:
     _create_remote_cuda_repository(
diff --git a/third_party/nccl/archive.BUILD b/third_party/nccl/archive.BUILD
index 211b794bb0..5901c6b296 100644
--- a/third_party/nccl/archive.BUILD
+++ b/third_party/nccl/archive.BUILD
@@ -96,7 +96,7 @@ cc_library(
     ],
     hdrs = ["nccl.h"],
     copts = cuda_default_copts() + ["-Wno-vla"],
-    include_prefix = "third_party/nccl/",
+    include_prefix = "third_party/nccl",
     visibility = ["//visibility:public"],
     deps = [
         ":device",
diff --git a/third_party/nccl/nccl_configure.bzl b/third_party/nccl/nccl_configure.bzl
index 1e6422b49e..f7d7d55383 100644
--- a/third_party/nccl/nccl_configure.bzl
+++ b/third_party/nccl/nccl_configure.bzl
@@ -13,7 +13,9 @@ load(
     "auto_configure_fail",
     "compute_capabilities",
     "cuda_toolkit_path",
+    "enable_cuda",
     "find_cuda_define",
+    "get_cpu_value",
     "matches_version",
 )
 
@@ -22,7 +24,7 @@ _NCCL_HDR_PATH = "NCCL_HDR_PATH"
 _NCCL_INSTALL_PATH = "NCCL_INSTALL_PATH"
 _TF_CUDA_COMPUTE_CAPABILITIES = "TF_CUDA_COMPUTE_CAPABILITIES"
 _TF_NCCL_VERSION = "TF_NCCL_VERSION"
-_TF_NCCL_CONFIG_REPO = "TF_NCCL_CONFIG_REPO"
+_TF_NEED_CUDA = "TF_NEED_CUDA"
 
 _DEFINE_NCCL_MAJOR = "#define NCCL_MAJOR"
 _DEFINE_NCCL_MINOR = "#define NCCL_MINOR"
@@ -116,26 +118,23 @@ def _check_nccl_version(repository_ctx, nccl_install_path, nccl_hdr_path, nccl_v
     header_version = "%s.%s.%s" % (major_version, minor_version, patch_version)
     if not matches_version(nccl_version, header_version):
         auto_configure_fail(
-            ("NCCL library version detected from %s/nccl.h (%s) does not match " +
-             "TF_NCCL_VERSION (%s). To fix this rerun configure again.") %
+            ("NCCL library version detected from %s/nccl.h (%s) does not " +
+             "match TF_NCCL_VERSION (%s). To fix this rerun configure again.") %
             (header_dir, header_version, nccl_version),
         )
 
 def _nccl_configure_impl(repository_ctx):
     """Implementation of the nccl_configure repository rule."""
-    if _TF_NCCL_VERSION not in repository_ctx.os.environ:
+    if not enable_cuda(repository_ctx) or \
+       get_cpu_value(repository_ctx) not in ("Linux", "FreeBSD"):
         # Add a dummy build file to make bazel query happy.
         repository_ctx.file("BUILD", _NCCL_DUMMY_BUILD_CONTENT)
         return
 
-    if _TF_NCCL_CONFIG_REPO in repository_ctx.os.environ:
-        # Forward to the pre-configured remote repository.
-        repository_ctx.template("BUILD", _label("remote.BUILD.tpl"), {
-            "%{target}": repository_ctx.os.environ[_TF_NCCL_CONFIG_REPO],
-        })
-        return
+    nccl_version = ""
+    if _TF_NCCL_VERSION in repository_ctx.os.environ:
+        nccl_version = repository_ctx.os.environ[_TF_NCCL_VERSION].strip()
 
-    nccl_version = repository_ctx.os.environ[_TF_NCCL_VERSION].strip()
     if nccl_version == "":
         # Alias to open source build from @nccl_archive.
         repository_ctx.file("BUILD", _NCCL_ARCHIVE_BUILD_CONTENT)
@@ -179,7 +178,7 @@ nccl_configure = repository_rule(
         _NCCL_INSTALL_PATH,
         _TF_NCCL_VERSION,
         _TF_CUDA_COMPUTE_CAPABILITIES,
-        _TF_NCCL_CONFIG_REPO,
+        _TF_NEED_CUDA,
     ],
 )
 """Detects and configures the NCCL configuration.
diff --git a/third_party/nccl/remote.BUILD.tpl b/third_party/nccl/remote.BUILD.tpl
deleted file mode 100644
index d66fc5563d..0000000000
--- a/third_party/nccl/remote.BUILD.tpl
+++ /dev/null
@@ -1,6 +0,0 @@
-licenses(["restricted"])
-
-package(default_visibility = ["//visibility:public"])
-
-alias(name="LICENSE", actual = "%{target}:LICENSE")
-alias(name = "nccl", actual = "%{target}:nccl")
-- 
GitLab


From 853d417a891ecdc893bc2c448d426833df74c7e0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 31 Dec 2019 03:21:38 -0800
Subject: [PATCH 0093/1095] Replace deprecated `to_float` with `tf.cast(_,
 dtypes.float32`).

A noop.

PiperOrigin-RevId: 227345610
---
 tensorflow/contrib/gan/BUILD                    |  5 ++---
 .../gan/python/losses/python/losses_impl.py     | 17 +++++++++++------
 2 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/tensorflow/contrib/gan/BUILD b/tensorflow/contrib/gan/BUILD
index e1d198fe7b..0fa229880b 100644
--- a/tensorflow/contrib/gan/BUILD
+++ b/tensorflow/contrib/gan/BUILD
@@ -145,16 +145,15 @@ py_library(
         "//tensorflow/contrib/framework:framework_py",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:clip_ops",
+        "//tensorflow/python:dtypes",
         "//tensorflow/python:framework_ops",
-        "//tensorflow/python:gradients",
+        "//tensorflow/python:gradients_impl",
         "//tensorflow/python:math_ops",
         "//tensorflow/python:random_ops",
         "//tensorflow/python:summary",
         "//tensorflow/python:tensor_util",
         "//tensorflow/python:variable_scope",
-        "//tensorflow/python/ops/distributions",
         "//tensorflow/python/ops/losses",
-        "//third_party/py/numpy",
     ],
 )
 
diff --git a/tensorflow/contrib/gan/python/losses/python/losses_impl.py b/tensorflow/contrib/gan/python/losses/python/losses_impl.py
index 930c91786f..1f1ae2df4d 100644
--- a/tensorflow/contrib/gan/python/losses/python/losses_impl.py
+++ b/tensorflow/contrib/gan/python/losses/python/losses_impl.py
@@ -38,6 +38,7 @@ from __future__ import print_function
 
 
 from tensorflow.contrib.framework.python.ops import variables as contrib_variables_lib
+from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import array_ops
@@ -69,6 +70,10 @@ __all__ = [
 ]
 
 
+def _to_float(tensor):
+  return math_ops.cast(tensor, dtypes.float32)
+
+
 # Wasserstein losses from `Wasserstein GAN` (https://arxiv.org/abs/1701.07875).
 def wasserstein_generator_loss(
     discriminator_gen_outputs,
@@ -98,7 +103,7 @@ def wasserstein_generator_loss(
   """
   with ops.name_scope(scope, 'generator_wasserstein_loss', (
       discriminator_gen_outputs, weights)) as scope:
-    discriminator_gen_outputs = math_ops.to_float(discriminator_gen_outputs)
+    discriminator_gen_outputs = _to_float(discriminator_gen_outputs)
 
     loss = - discriminator_gen_outputs
     loss = losses.compute_weighted_loss(
@@ -144,8 +149,8 @@ def wasserstein_discriminator_loss(
   with ops.name_scope(scope, 'discriminator_wasserstein_loss', (
       discriminator_real_outputs, discriminator_gen_outputs, real_weights,
       generated_weights)) as scope:
-    discriminator_real_outputs = math_ops.to_float(discriminator_real_outputs)
-    discriminator_gen_outputs = math_ops.to_float(discriminator_gen_outputs)
+    discriminator_real_outputs = _to_float(discriminator_real_outputs)
+    discriminator_gen_outputs = _to_float(discriminator_gen_outputs)
     discriminator_real_outputs.shape.assert_is_compatible_with(
         discriminator_gen_outputs.shape)
 
@@ -647,7 +652,7 @@ def least_squares_generator_loss(
   """
   with ops.name_scope(scope, 'lsq_generator_loss',
                       (discriminator_gen_outputs, real_label)) as scope:
-    discriminator_gen_outputs = math_ops.to_float(discriminator_gen_outputs)
+    discriminator_gen_outputs = _to_float(discriminator_gen_outputs)
     loss = math_ops.squared_difference(
         discriminator_gen_outputs, real_label) / 2.0
     loss = losses.compute_weighted_loss(
@@ -702,8 +707,8 @@ def least_squares_discriminator_loss(
   """
   with ops.name_scope(scope, 'lsq_discriminator_loss',
                       (discriminator_gen_outputs, real_label)) as scope:
-    discriminator_real_outputs = math_ops.to_float(discriminator_real_outputs)
-    discriminator_gen_outputs = math_ops.to_float(discriminator_gen_outputs)
+    discriminator_real_outputs = _to_float(discriminator_real_outputs)
+    discriminator_gen_outputs = _to_float(discriminator_gen_outputs)
     discriminator_real_outputs.shape.assert_is_compatible_with(
         discriminator_gen_outputs.shape)
 
-- 
GitLab


From 7914dbf7a2b9c2b555e3e55423905bdb22c6d759 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 31 Dec 2019 06:34:09 -0800
Subject: [PATCH 0094/1095] [tfgan] Fix documentation bug classifer_impl.

PiperOrigin-RevId: 227356865
---
 .../eval/python/classifier_metrics_impl.py    | 24 +++++++++----------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py
index ea55241b34..31f0d34ed6 100644
--- a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py
+++ b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Model evaluation tools for TFGAN.
+"""Model evaluation tools for TF-GAN.
 
 These methods come from https://arxiv.org/abs/1606.03498,
 https://arxiv.org/abs/1706.08500, and https://arxiv.org/abs/1801.01401.
@@ -795,9 +795,9 @@ def kernel_classifier_distance(real_images,
       on a classifier.
     num_classifier_batches: Number of batches to split images in to in order to
       efficiently run them through the classifier network.
-    max_estimator_block_size: integer, default 1024. The distance estimator
-      splits samples into blocks for computational efficiency. Larger values are
-      more computationally expensive but decrease the variance of the distance
+    max_block_size: integer, default 1024. The distance estimator splits samples
+      into blocks for computational efficiency. Larger values are more
+      computationally expensive but decrease the variance of the distance
       estimate.
     dtype: if not None, coerce activations to this dtype before computations.
 
@@ -872,9 +872,9 @@ def kernel_classifier_distance_and_std(real_images,
       on a classifier.
     num_classifier_batches: Number of batches to split images in to in order to
       efficiently run them through the classifier network.
-    max_estimator_block_size: integer, default 1024. The distance estimator
-      splits samples into blocks for computational efficiency. Larger values are
-      more computationally expensive but decrease the variance of the distance
+    max_block_size: integer, default 1024. The distance estimator splits samples
+      into blocks for computational efficiency. Larger values are more
+      computationally expensive but decrease the variance of the distance
       estimate. Having a smaller block size also gives a better estimate of the
       standard error.
     dtype: if not None, coerce activations to this dtype before computations.
@@ -911,7 +911,7 @@ def kernel_classifier_distance_and_std(real_images,
   gen_a = array_ops.concat(array_ops.unstack(gen_a), 0)
 
   return kernel_classifier_distance_and_std_from_activations(
-      real_a, gen_a, max_block_size=max_block_size)
+      real_a, gen_a, max_block_size, dtype)
 
 
 kernel_inception_distance_and_std = functools.partial(
@@ -968,14 +968,14 @@ def kernel_classifier_distance_from_activations(real_activations,
       into blocks for computational efficiency. Larger values are more
       computationally expensive but decrease the variance of the distance
       estimate.
-    dtype: if not None, coerce activations to this dtype before computations.
+    dtype: If not None, coerce activations to this dtype before computations.
 
   Returns:
    The Kernel Inception Distance. A floating-point scalar of the same type
    as the output of the activations.
   """
   return kernel_classifier_distance_and_std_from_activations(
-      real_activations, generated_activations, max_block_size=max_block_size)[0]
+      real_activations, generated_activations, max_block_size, dtype)[0]
 
 
 def kernel_classifier_distance_and_std_from_activations(real_activations,
@@ -1030,7 +1030,7 @@ def kernel_classifier_distance_and_std_from_activations(real_activations,
       computationally expensive but decrease the variance of the distance
       estimate. Having a smaller block size also gives a better estimate of the
       standard error.
-    dtype: if not None, coerce activations to this dtype before computations.
+    dtype: If not None, coerce activations to this dtype before computations.
 
   Returns:
    The Kernel Inception Distance. A floating-point scalar of the same type
@@ -1081,7 +1081,7 @@ def kernel_classifier_distance_and_std_from_activations(real_activations,
   dim = math_ops.cast(real_activations.shape[1], dtype)
 
   def compute_kid_block(i):
-    'Compute the ith block of the KID estimate.'
+    """Computes the ith block of the KID estimate."""
     r_s = inds_r[i]
     r_e = inds_r[i + 1]
     r = real_activations[r_s:r_e]
-- 
GitLab


From 3ae375aa92fbb6155f82393735d0b98d8fb9c1b2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 31 Dec 2019 09:45:15 -0800
Subject: [PATCH 0095/1095] Fix naming in README.

PiperOrigin-RevId: 227367542
---
 tensorflow/contrib/gan/README.md | 31 ++++++++++++++++---------------
 1 file changed, 16 insertions(+), 15 deletions(-)

diff --git a/tensorflow/contrib/gan/README.md b/tensorflow/contrib/gan/README.md
index 1762353a0d..db7dc51daa 100644
--- a/tensorflow/contrib/gan/README.md
+++ b/tensorflow/contrib/gan/README.md
@@ -1,7 +1,8 @@
 <!-- TODO(joelshor): Add images to the examples. -->
-# TensorFlow-GAN (TFGAN)
+<!-- TODO(joelshor): Add link to new location when b/122114187 is done. -->
+# TensorFlow-GAN (TF-GAN)
 
-TFGAN is a lightweight library for training and evaluating Generative
+TF-GAN is a lightweight library for training and evaluating Generative
 Adversarial Networks (GANs). This technique allows you to train a network
 (called the 'generator') to sample from a distribution, without having to
 explicitly model the distribution and without writing an explicit loss. For
@@ -17,27 +18,27 @@ import tensorflow as tf
 tfgan = tf.contrib.gan
 ```
 
-## Why TFGAN?
+## Why TF-GAN?
 
 * Easily train generator and discriminator networks with well-tested, flexible [library calls](https://www.tensorflow.org/code/tensorflow/contrib/gan/python/train.py). You can
-mix TFGAN, native TF, and other custom frameworks
+mix TF-GAN, native TF, and other custom frameworks
 * Use already implemented [GAN losses and penalties](https://www.tensorflow.org/code/tensorflow/contrib/gan/python/losses/python/losses_impl.py) (ex Wasserstein loss, gradient penalty, mutual information penalty, etc)
 * [Monitor and visualize](https://www.tensorflow.org/code/tensorflow/contrib/gan/python/eval/python/summaries_impl.py) GAN progress during training, and [evaluate](https://www.tensorflow.org/code/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py) them
 * Use already-implemented [tricks](https://www.tensorflow.org/code/tensorflow/contrib/gan/python/features/python/) to stabilize and improve training
 * Develop based on examples of [common GAN setups](https://github.com/tensorflow/models/tree/master/research/gan/)
-* Use the TFGAN-backed [GANEstimator](https://www.tensorflow.org/code/tensorflow/contrib/gan/python/estimator/python/gan_estimator_impl.py) to easily train a GAN model
-* Improvements in TFGAN infrastructure will automatically benefit your TFGAN project
+* Use the TF-GAN-backed [GANEstimator](https://www.tensorflow.org/code/tensorflow/contrib/gan/python/estimator/python/gan_estimator_impl.py) to easily train a GAN model
+* Improvements in TF-GAN infrastructure will automatically benefit your TF-GAN project
 * Stay up-to-date with research as we add more algorithms
 
-## What are the TFGAN components?
+## What are the TF-GAN components?
 
-TFGAN is composed of several parts which were design to exist independently.
+TF-GAN is composed of several parts which were design to exist independently.
 These include the following main pieces (explained in detail below).
 
 *   [core](https://www.tensorflow.org/code/tensorflow/contrib/gan/python/train.py):
     provides the main infrastructure needed to train a GAN. Training occurs in
     four phases, and each phase can be completed by custom-code or by using a
-    TFGAN library call.
+    TF-GAN library call.
 
 *   [features](https://www.tensorflow.org/code/tensorflow/contrib/gan/python/features/python/):
     Many common GAN operations and normalization techniques are implemented for
@@ -56,7 +57,7 @@ These include the following main pieces (explained in detail below).
     generative models.
 
 *   [examples](https://github.com/tensorflow/models/tree/master/research/gan/)
-    and [tutorial](http://https://github.com/tensorflow/models/tree/master/research/gan/tutorial.ipynb): See examples of how to use TFGAN
+    and [tutorial](http://https://github.com/tensorflow/models/tree/master/research/gan/tutorial.ipynb): See examples of how to use TF-GAN
     to make GAN training easier, or use the more complicated examples to
     jumpstart your own project. These include unconditional and conditional
     GANs, InfoGANs, adversarial losses on existing networks, and image-to-image
@@ -64,7 +65,7 @@ These include the following main pieces (explained in detail below).
 
 ## Training a GAN model
 
-Training in TFGAN typically consists of the following steps:
+Training in TF-GAN typically consists of the following steps:
 
 1. Specify the input to your networks.
 1. Set up your generator and discriminator using a `GANModel`.
@@ -72,12 +73,12 @@ Training in TFGAN typically consists of the following steps:
 1. Create your train ops using a `GANTrainOps`.
 1. Run your train ops.
 
-At each stage, you can either use TFGAN's convenience functions, or you can
+At each stage, you can either use TF-GAN's convenience functions, or you can
 perform the step manually for fine-grained control. We provide examples below.
 
 There are various types of GAN setups. For instance, you can train a generator
 to sample unconditionally from a learned distribution, or you can condition on
-extra information such as a class label. TFGAN is compatible with many setups,
+extra information such as a class label. TF-GAN is compatible with many setups,
 and we demonstrate a few below:
 
 ### Examples
@@ -255,9 +256,9 @@ with variable_scope.variable_scope(dis_scope, reuse=True):
   discriminator_real_outputs = discriminator_fn(images)
 generator_variables = variables_lib.get_trainable_variables(gen_scope)
 discriminator_variables = variables_lib.get_trainable_variables(dis_scope)
-# Depending on what TFGAN features you use, you don't always need to supply
+# Depending on what TF-GAN features you use, you don't always need to supply
 # every `GANModel` field. At a minimum, you need to include the discriminator
-# outputs and variables if you want to use TFGAN to construct losses.
+# outputs and variables if you want to use TF-GAN to construct losses.
 gan_model = tfgan.GANModel(
     generator_inputs,
     generated_data,
-- 
GitLab


From ceb64c15db61855183d3b88504b5469af0f423c7 Mon Sep 17 00:00:00 2001
From: Dheeraj Rajaram Reddy <dheeraj98reddy@gmail.com>
Date: Tue, 1 Jan 2019 01:25:55 +0530
Subject: [PATCH 0096/1095] Add take_while experimental dataset op (tests
 pending)

---
 .../core/kernels/data/experimental/BUILD      |  14 +
 .../experimental/take_while_dataset_op.cc     | 242 ++++++++++++++++++
 .../core/ops/experimental_dataset_ops.cc      |  11 +
 .../python/data/experimental/__init__.py      |   1 +
 tensorflow/python/data/experimental/ops/BUILD |  14 +
 .../data/experimental/ops/take_while_ops.py   |  79 ++++++
 6 files changed, 361 insertions(+)
 create mode 100644 tensorflow/core/kernels/data/experimental/take_while_dataset_op.cc
 create mode 100644 tensorflow/python/data/experimental/ops/take_while_ops.py

diff --git a/tensorflow/core/kernels/data/experimental/BUILD b/tensorflow/core/kernels/data/experimental/BUILD
index 7433303f77..4ce14b0140 100644
--- a/tensorflow/core/kernels/data/experimental/BUILD
+++ b/tensorflow/core/kernels/data/experimental/BUILD
@@ -293,6 +293,19 @@ tf_kernel_library(
     ],
 )
 
+tf_kernel_library(
+    name = "take_while_dataset_op",
+    srcs = ["take_while_dataset_op.cc"],
+    deps = [
+        "//tensorflow/core:core_cpu_internal",
+        "//tensorflow/core:dataset_ops_op_lib",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+        "//tensorflow/core/kernels/data:captured_function",
+    ],
+)
+
 tf_kernel_library(
     name = "to_tf_record_op",
     srcs = ["to_tf_record_op.cc"],
@@ -365,6 +378,7 @@ tf_kernel_library(
         ":sql_dataset_op",
         ":stats_aggregator_ops",
         ":stats_dataset_ops",
+        ":take_while_dataset_op",
         ":threadpool_dataset_op",
         ":to_tf_record_op",
         ":unbatch_dataset_op",
diff --git a/tensorflow/core/kernels/data/experimental/take_while_dataset_op.cc b/tensorflow/core/kernels/data/experimental/take_while_dataset_op.cc
new file mode 100644
index 0000000000..271b919bc9
--- /dev/null
+++ b/tensorflow/core/kernels/data/experimental/take_while_dataset_op.cc
@@ -0,0 +1,242 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <iterator>
+#include <vector>
+
+#include "tensorflow/core/common_runtime/function.h"
+#include "tensorflow/core/framework/dataset.h"
+#include "tensorflow/core/framework/partial_tensor_shape.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/kernels/data/captured_function.h"
+
+namespace tensorflow {
+namespace data {
+namespace {
+
+// See documentation in ../../ops/dataset_ops.cc for a high-level
+// description of the following op.
+
+class TakeWhileDatasetOp : public UnaryDatasetOpKernel {
+ public:
+  explicit TakeWhileDatasetOp(OpKernelConstruction* ctx)
+      : UnaryDatasetOpKernel(ctx) {
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("predicate", &func_));
+    OP_REQUIRES_OK(
+        ctx, ctx->GetAttr("preserve_cardinality", &preserve_cardinality_));
+  }
+
+  void MakeDataset(OpKernelContext* ctx, DatasetBase* input,
+                   DatasetBase** output) override {
+    std::unique_ptr<CapturedFunction> captured_func;
+    OP_REQUIRES_OK(ctx, CapturedFunction::Create(func_, ctx, "other_arguments",
+                                                 &captured_func));
+    
+    // TODO (squadrick): check short-circuit
+    *output = new Dataset(ctx, input, func_, std::move(captured_func), 
+                          preserve_cardinality_);
+  }
+
+ private:
+  class Dataset : public DatasetBase {
+   public:
+    Dataset(OpKernelContext* ctx, const DatasetBase* input,
+            const NameAttrList& func,
+            std::unique_ptr<CapturedFunction> captured_func,
+            bool preserve_cardinality)
+        : DatasetBase(DatasetContext(ctx)),
+          input_(input),
+          func_(func),
+          captured_func_(std::move(captured_func)),
+          preserve_cardinality_(preserve_cardinality) {
+      input_->Ref();
+    }
+
+    ~Dataset() override { input_->Unref(); }
+
+    std::unique_ptr<IteratorBase> MakeIteratorInternal(
+        const string& prefix) const override {
+      return std::unique_ptr<IteratorBase>(
+          new Iterator({this, strings::StrCat(prefix, "::TakeWhile")}));
+    }
+
+    const DataTypeVector& output_dtypes() const override {
+      return input_->output_dtypes();
+    }
+    const std::vector<PartialTensorShape>& output_shapes() const override {
+      return input_->output_shapes();
+    }
+
+    string DebugString() const override { return "TakeWhileDatasetOp::Dataset"; }
+
+    int64 Cardinality() const override { return input_->Cardinality(); }
+
+   protected:
+    Status AsGraphDefInternal(SerializationContext* ctx,
+                              DatasetGraphDefBuilder* b,
+                              Node** output) const override {
+      TF_RETURN_IF_ERROR(b->AddFunction(ctx, func_.name()));
+      Node* input_node;
+      TF_RETURN_IF_ERROR(b->AddInputDataset(ctx, input_, &input_node));
+
+      std::vector<Node*> other_arguments;
+      other_arguments.reserve(captured_func_->captured_inputs().size());
+      DataTypeVector other_arguments_types;
+      other_arguments_types.reserve(captured_func_->captured_inputs().size());
+      for (const Tensor& t : captured_func_->captured_inputs()) {
+        Node* node;
+        DatasetBase* input;
+        Status s = GetDatasetFromVariantTensor(t, &input);
+        if (s.ok()) {
+          TF_RETURN_IF_ERROR(b->AddInputDataset(ctx, input, &node));
+        } else {
+          TF_RETURN_IF_ERROR(b->AddTensor(t, &node));
+        }
+        other_arguments.emplace_back(node);
+        other_arguments_types.emplace_back(t.dtype());
+      }
+      AttrValue f_attr;
+      b->BuildAttrValue(func_, &f_attr);
+
+      AttrValue other_arguments_types_attr;
+      b->BuildAttrValue(other_arguments_types, &other_arguments_types_attr);
+
+      AttrValue preserve_cardinality_attr;
+      b->BuildAttrValue(preserve_cardinality_, &preserve_cardinality_attr);
+
+      TF_RETURN_IF_ERROR(b->AddDataset(
+            this, {std::make_pair(0, input_node)},
+            {std::make_pair(1, other_arguments)},
+            {std::make_pair("predicate", f_attr),
+             std::make_pair("Targuments", other_arguments_types_attr),
+             std::make_pair("preserve_cardinality", 
+                            preserve_cardinality_attr)},
+             output));
+      return Status::OK();
+    }
+
+   private:
+    class Iterator : public DatasetIterator<Dataset> {
+     public:
+      explicit Iterator(const Params& params)
+          : DatasetIterator<Dataset>(params) {}
+
+      Status Initialize(IteratorContext* ctx) override {
+        TF_RETURN_IF_ERROR(
+            dataset()->input_->MakeIterator(ctx, prefix(), &input_impl_));
+        return dataset()->captured_func_->Instantiate(
+            ctx, &instantiated_captured_func_);
+      }
+
+      Status GetNextInternal(IteratorContext* ctx,
+                             std::vector<Tensor>* out_tensors,
+                             bool* end_of_sequence) override {
+        {
+          tf_shared_lock l(mu_);
+          if(!input_impl_) {
+            *end_of_sequence = true;
+            return Status::OK();
+          }
+          TF_RETURN_IF_ERROR(
+              input_impl_->GetNext(ctx, out_tensors, end_of_sequence));
+        }
+        if (*end_of_sequence) {
+          mutex_lock l(mu_);
+          input_impl_.reset();
+          return Status::OK();
+        }
+
+        std::vector<Tensor> bool_output;
+
+        Status s = instantiated_captured_func_->RunWithBorrowedArgs(
+            ctx, *out_tensors, &bool_output);
+
+        if (s.ok()) {
+          if(bool_output.size() != 1 || bool_output[0].dtype() != DT_BOOL ||
+             bool_output[0].NumElements() != 1) {
+            return errors::InvalidArgument(
+                "`predicate` must returns a scalar bool tensor.");
+          }
+          auto cond = bool_output[0].scalar<bool>()();
+          if (!cond) { // predicate is false
+            *end_of_sequence = true;
+            return Status::OK();
+          }
+        } else if (errors::IsOutOfRange(s)) {
+          if (dataset()->preserve_cardinality_) {
+            // To guarantee that the transformation preserves the cardinality of
+            // the dataset, we convert `OutOfRange` to `InvalidArgument` as the
+            // former may be interpreted by a caller as the end of sequence.
+            return errors::InvalidArgument(
+                "Function invocation produced OutOfRangeError: ",
+                s.error_message());
+          } else {
+            // `f` may deliberately raise `errors::OutOfRange` to indicate
+            // that we should terminate the iteration early.
+            *end_of_sequence = true;
+            return Status::OK();
+          }
+        }
+        return s;
+      }
+
+     protected:
+      std::shared_ptr<model::Node> CreateNode(
+          IteratorContext* ctx, model::Node::Args args) const override {
+        return model::MakeKnownRatioNode(std::move(args),
+                                         /*ratio=*/1);
+      }
+
+      Status SaveInternal(IteratorStateWriter* writer) override {
+        mutex_lock l(mu_);
+        if (input_impl_)
+          TF_RETURN_IF_ERROR(SaveInput(writer, input_impl_));
+        else
+          TF_RETURN_IF_ERROR(
+              writer->WriteScalar(full_name("input_impls_empty"), ""));
+        return Status::OK();
+      }
+
+      Status RestoreInternal(IteratorContext* ctx,
+                             IteratorStateReader* reader) override {
+        mutex_lock l(mu_);
+        if (reader->Contains(full_name("input_impls_empty")))
+          input_impl_.reset();
+        else
+          TF_RETURN_IF_ERROR(RestoreInput(ctx, reader, input_impl_));
+        return Status::OK();
+      }
+
+     private:
+      mutex mu_;
+      std::unique_ptr<IteratorBase> input_impl_ GUARDED_BY(mu_);
+      std::unique_ptr<InstantiatedCapturedFunction> instantiated_captured_func_;
+    };
+
+    const DatasetBase* const input_;
+    const NameAttrList func_;
+    const std::unique_ptr<CapturedFunction> captured_func_;
+    const bool preserve_cardinality_;
+  };
+
+  NameAttrList func_;
+  bool preserve_cardinality_;
+};
+
+REGISTER_KERNEL_BUILDER(Name("ExperimentalTakeWhileDataset").Device(DEVICE_CPU),
+                        TakeWhileDatasetOp);
+
+}  // namespace
+}  // namespace data
+}  // namespace tensorflow
diff --git a/tensorflow/core/ops/experimental_dataset_ops.cc b/tensorflow/core/ops/experimental_dataset_ops.cc
index f904e2536d..d10a13f06f 100644
--- a/tensorflow/core/ops/experimental_dataset_ops.cc
+++ b/tensorflow/core/ops/experimental_dataset_ops.cc
@@ -352,6 +352,17 @@ REGISTER_OP("ExperimentalStatsAggregatorSummary")
     .Output("summary: string")
     .SetShapeFn(shape_inference::ScalarShape);
 
+REGISTER_OP("ExperimentalTakeWhileDataset")
+    .Input("input_dataset: variant")
+    .Input("other_arguments: Targuments")
+    .Output("handle: variant")
+    .Attr("predicate: func")
+    .Attr("Targuments: list(type) >= 0")
+    .Attr("output_types: list(type) >= 1")
+    .Attr("output_shapes: list(shape) >= 1")
+    .Attr("preserve_cardinality: bool = false")
+    .SetShapeFn(shape_inference::ScalarShape);
+
 REGISTER_OP("ExperimentalUnbatchDataset")
     .Input("input_dataset: variant")
     .Output("handle: variant")
diff --git a/tensorflow/python/data/experimental/__init__.py b/tensorflow/python/data/experimental/__init__.py
index ffc2e5ef5f..016acefa1f 100644
--- a/tensorflow/python/data/experimental/__init__.py
+++ b/tensorflow/python/data/experimental/__init__.py
@@ -115,6 +115,7 @@ from tensorflow.python.data.experimental.ops.shuffle_ops import shuffle_and_repe
 from tensorflow.python.data.experimental.ops.stats_aggregator import StatsAggregator
 from tensorflow.python.data.experimental.ops.stats_ops import latency_stats
 from tensorflow.python.data.experimental.ops.stats_options import StatsOptions
+from tensorflow.python.data.experimental.ops.take_while_ops import take_while
 from tensorflow.python.data.experimental.ops.threading_options import ThreadingOptions
 from tensorflow.python.data.experimental.ops.unique import unique
 from tensorflow.python.data.experimental.ops.writers import TFRecordWriter
diff --git a/tensorflow/python/data/experimental/ops/BUILD b/tensorflow/python/data/experimental/ops/BUILD
index 60c20e0bcf..95e9509678 100644
--- a/tensorflow/python/data/experimental/ops/BUILD
+++ b/tensorflow/python/data/experimental/ops/BUILD
@@ -354,6 +354,19 @@ py_library(
     ],
 )
 
+py_library(
+    name = "take_while_ops",
+    srcs = ["take_while_ops.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/python:experimental_dataset_ops_gen",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:function",
+        "//tensorflow/python/data/ops:dataset_ops",
+    ],
+)
+
 py_library(
     name = "threading_options",
     srcs = ["threading_options.py"],
@@ -454,6 +467,7 @@ py_library(
         ":shuffle_ops",
         ":sleep",
         ":stats_ops",
+        "take_while_ops",
         ":threadpool",
         ":unique",
         ":writers",
diff --git a/tensorflow/python/data/experimental/ops/take_while_ops.py b/tensorflow/python/data/experimental/ops/take_while_ops.py
new file mode 100644
index 0000000000..edbecd0666
--- /dev/null
+++ b/tensorflow/python/data/experimental/ops/take_while_ops.py
@@ -0,0 +1,79 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""take-while dataset transformation."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import collections
+
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.data.util import structure as structure_lib
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import gen_experimental_dataset_ops
+from tensorflow.python.util.tf_export import tf_export
+
+
+class _TakeWhileDataset(dataset_ops.UnaryUnchangedStructureDataset):
+  """A dataset that stops iteration when `predicate` returns false."""
+
+  def __init__(self, input_dataset, predicate):
+    """See `take_while()` for details."""
+    
+    self._input_dataset = input_dataset
+    wrapped_func = dataset_ops.StructuredFunctionWrapper(
+            predicate,
+            self._transformation_name(),
+            dataset=self._input_dataset)
+    
+    if not wrapped_func.output_structure.is_compatible_with(
+            structure_lib.TensorStructure(dtypes.bool, [])):
+        raise ValueError("`predicate` must return a scalar boolean tensor.")
+     
+    self._predicate = wrapped_func
+
+    variant_tensor = gen_experimental_dataset_ops.experimental_take_while_dataset(
+            self._input_dataset._variant_tensor,
+            other_arguments=self._predicate.function.captured_inputs,
+            predicate=self._predicate.function,
+            preserve_cardinality=True,
+            **dataset_ops.flat_structure(self))
+    super(_TakeWhileDataset, self).__init__(input_dataset, variant_tensor)
+
+  def _functions(self):
+      return [self._predicate]
+
+  def _transformation_name(self):
+      return "tf.data.experimental.take_while()"
+
+
+@tf_export("data.experimental.take_while")
+def take_while(predicate):
+  """A transformation that stops dataset iteration based on a `predicate` condition
+
+  Args:
+    predicate: A function that maps a nested structure of tensors 
+    (having shapes and types defined by `self.output_shapes` and 
+    `self.output_types`) to a scalar `tf.bool` tensor.
+
+  Returns:
+    A `Dataset` transformation function, which can be passed to
+    `tf.data.Dataset.apply`.
+  """
+  def _apply_fn(dataset):
+      return _TakeWhileDataset(dataset, predicate)
+
+  return _apply_fn
-- 
GitLab


From 15d49deb3c5d4048718af19a2e0e0279e8f204d5 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Tue, 1 Jan 2019 00:16:30 +0000
Subject: [PATCH 0097/1095] Update eigen library to 88fc23324517 to fix 24457

This fix updates eigen library to 88fc23324517 so that the issue
raised in 24457 could be fixed.

This fix fixes 24457.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 11ce55feda..d32caceb9e 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -136,11 +136,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "eigen_archive",
         build_file = clean_dep("//third_party:eigen.BUILD"),
-        sha256 = "753fbb58d0a49b6bcbcfb126ebfa2e21fc97f7471529ba835a096008ce588d8a",
-        strip_prefix = "eigen-eigen-9f48e814419e",
+        sha256 = "9de38f2d162c51599b802f7c36d9f3773980d19ac908c61638f8344d2c10e1ca",
+        strip_prefix = "eigen-eigen-88fc23324517",
         urls = [
-            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/9f48e814419e.tar.gz",
-            "https://bitbucket.org/eigen/eigen/get/9f48e814419e.tar.gz",
+            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/88fc23324517..tar.gz",
+            "https://bitbucket.org/eigen/eigen/get/88fc23324517.tar.gz",
         ],
     )
 
-- 
GitLab


From d60d04b68a91ac3086b93d8a83c148db8e84edc6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 1 Jan 2019 01:02:27 -0800
Subject: [PATCH 0098/1095] compat: Update forward compatibility horizon to
 2019-01-01

PiperOrigin-RevId: 227410769
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 19efe84ca4..2284d0021c 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 12, 31)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 1)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From c9de778aa12004428ae99186965655c97c8a1c5a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 1 Jan 2019 04:09:09 -0800
Subject: [PATCH 0099/1095] More `TFGAN` -> `TF-GAN`.

PiperOrigin-RevId: 227422079
---
 tensorflow/contrib/gan/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/gan/__init__.py b/tensorflow/contrib/gan/__init__.py
index f1946c7f92..ea51e8a569 100644
--- a/tensorflow/contrib/gan/__init__.py
+++ b/tensorflow/contrib/gan/__init__.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""TFGAN is a lightweight library for training and evaluating GANs.
+"""TF-GAN is a lightweight library for training and evaluating GANs.
 
 In addition to providing the infrastructure for easily training and evaluating
 GANS, this library contains modules for a TFGAN-backed Estimator,
-- 
GitLab


From 34b98c8f0ef3ee852e90dcebdde935ceae31dfe0 Mon Sep 17 00:00:00 2001
From: Siju <siju.samuel@huawei.com>
Date: Tue, 1 Jan 2019 18:55:56 +0530
Subject: [PATCH 0100/1095] Unused variable removed in softmax_op_functor.h

                                            ^
---
 tensorflow/core/kernels/softmax_op_functor.h | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tensorflow/core/kernels/softmax_op_functor.h b/tensorflow/core/kernels/softmax_op_functor.h
index c8bc1ad3bb..218698f3ff 100644
--- a/tensorflow/core/kernels/softmax_op_functor.h
+++ b/tensorflow/core/kernels/softmax_op_functor.h
@@ -57,7 +57,6 @@ struct SoftmaxEigenImpl {
     Eigen::DSizes<int, 2> one_by_class(1, num_classes);
 #else
     Eigen::IndexList<Eigen::type2index<kClassDim> > along_class;
-    Eigen::IndexList<Eigen::type2index<1> > depth_dim;
     Eigen::IndexList<int, Eigen::type2index<1> > batch_by_one;
     batch_by_one.set(0, batch_size);
     Eigen::IndexList<Eigen::type2index<1>, int> one_by_class;
-- 
GitLab


From fcffdde4d6a918a7971631b444e8fc0cb8b8244b Mon Sep 17 00:00:00 2001
From: Siju <siju.samuel@huawei.com>
Date: Tue, 1 Jan 2019 19:19:48 +0530
Subject: [PATCH 0101/1095] SynchronousMemcpy changed to SynchronousMemcpyD2H
 and SynchronousMemcpyH2D

---
 .../core/common_runtime/gpu/gpu_debug_allocator.cc | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/common_runtime/gpu/gpu_debug_allocator.cc b/tensorflow/core/common_runtime/gpu/gpu_debug_allocator.cc
index 989ddbe4af..7268d6cf6a 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_debug_allocator.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_debug_allocator.cc
@@ -44,7 +44,8 @@ bool CheckMask(se::StreamExecutor* exec, void* ptr, int64* mask) {
   se::DeviceMemory<int64> gpu_ptr{se::DeviceMemoryBase{ptr, MASK_BYTES}};
   int64 tmp[MASK_WORDS];
 
-  if (!exec->SynchronousMemcpy(&tmp, gpu_ptr, MASK_BYTES)) {
+  Status result = exec->SynchronousMemcpyD2H(gpu_ptr, MASK_BYTES, tmp);
+  if (!result.ok()) {
     LOG(FATAL) << "Could not copy debug mask";
   }
 
@@ -63,7 +64,8 @@ bool CheckMask(se::StreamExecutor* exec, void* ptr, int64* mask) {
 
 void InitMask(se::StreamExecutor* exec, void* ptr, int64* mask) {
   se::DeviceMemory<int64> gpu_ptr{se::DeviceMemoryBase{ptr, MASK_BYTES}};
-  if (!exec->SynchronousMemcpy(&gpu_ptr, mask, MASK_BYTES)) {
+  Status result = exec->SynchronousMemcpyH2D(mask, MASK_BYTES, &gpu_ptr);
+  if (!result.ok()) {
     LOG(FATAL) << "Could not copy debug mask";
   }
 }
@@ -171,7 +173,9 @@ void* GPUNanResetAllocator::AllocateRaw(size_t alignment, size_t num_bytes) {
   se::DeviceMemory<float> nan_ptr{
       se::DeviceMemoryBase{static_cast<float*>(allocated_ptr), req_size}};
 
-  if (!stream_exec_->SynchronousMemcpy(&nan_ptr, &nans[0], req_size)) {
+  Status result = stream_exec_->SynchronousMemcpyH2D(&nans[0], req_size, 
+                                                     &nan_ptr);
+  if (!result.ok()) {
     LOG(ERROR) << "Could not initialize to NaNs";
   }
 
@@ -185,7 +189,9 @@ void GPUNanResetAllocator::DeallocateRaw(void* ptr) {
                             std::nanf(""));
     se::DeviceMemory<float> nan_ptr{
         se::DeviceMemoryBase{static_cast<float*>(ptr), req_size}};
-    if (!stream_exec_->SynchronousMemcpy(&nan_ptr, &nans[0], req_size)) {
+    Status result = stream_exec_->SynchronousMemcpyH2D(&nans[0], req_size, 
+                                                       &nan_ptr);
+    if (!result.ok()) {
       LOG(ERROR) << "Could not initialize to NaNs";
     }
   }
-- 
GitLab


From 3c80e17e79c1920dbba1201d0c50a818772f7e1f Mon Sep 17 00:00:00 2001
From: Siju <siju.samuel@huawei.com>
Date: Tue, 1 Jan 2019 19:23:23 +0530
Subject: [PATCH 0102/1095] Removed unnecessary spaces

---
 tensorflow/core/common_runtime/gpu/gpu_debug_allocator.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/common_runtime/gpu/gpu_debug_allocator.cc b/tensorflow/core/common_runtime/gpu/gpu_debug_allocator.cc
index 7268d6cf6a..77737a76b6 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_debug_allocator.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_debug_allocator.cc
@@ -173,7 +173,7 @@ void* GPUNanResetAllocator::AllocateRaw(size_t alignment, size_t num_bytes) {
   se::DeviceMemory<float> nan_ptr{
       se::DeviceMemoryBase{static_cast<float*>(allocated_ptr), req_size}};
 
-  Status result = stream_exec_->SynchronousMemcpyH2D(&nans[0], req_size, 
+  Status result = stream_exec_->SynchronousMemcpyH2D(&nans[0], req_size,
                                                      &nan_ptr);
   if (!result.ok()) {
     LOG(ERROR) << "Could not initialize to NaNs";
@@ -189,7 +189,7 @@ void GPUNanResetAllocator::DeallocateRaw(void* ptr) {
                             std::nanf(""));
     se::DeviceMemory<float> nan_ptr{
         se::DeviceMemoryBase{static_cast<float*>(ptr), req_size}};
-    Status result = stream_exec_->SynchronousMemcpyH2D(&nans[0], req_size, 
+    Status result = stream_exec_->SynchronousMemcpyH2D(&nans[0], req_size,
                                                        &nan_ptr);
     if (!result.ok()) {
       LOG(ERROR) << "Could not initialize to NaNs";
-- 
GitLab


From dd06c3959737752fe84cd133e0b5137a8ee2d956 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 1 Jan 2019 05:56:40 -0800
Subject: [PATCH 0103/1095] More `TFGAN` -> `TF-GAN`.

PiperOrigin-RevId: 227427693
---
 tensorflow/contrib/gan/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/gan/__init__.py b/tensorflow/contrib/gan/__init__.py
index ea51e8a569..1e6000898f 100644
--- a/tensorflow/contrib/gan/__init__.py
+++ b/tensorflow/contrib/gan/__init__.py
@@ -24,7 +24,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-# Collapse TFGAN into a tiered namespace.
+# Collapse TF-GAN into a tiered namespace.
 from tensorflow.contrib.gan.python import estimator
 from tensorflow.contrib.gan.python import eval  # pylint:disable=redefined-builtin
 from tensorflow.contrib.gan.python import features
-- 
GitLab


From 5356b6898b512d3202088299ab422208fc5f8bb5 Mon Sep 17 00:00:00 2001
From: Dheeraj Rajaram Reddy <dheeraj98reddy@gmail.com>
Date: Tue, 1 Jan 2019 22:26:40 +0530
Subject: [PATCH 0104/1095] Make required changes

Remove preserve_cardinality
Update license year
Propagate predicate's error to the caller
Inline _transformation_name
---
 .../experimental/take_while_dataset_op.cc     | 46 ++++---------------
 .../core/ops/experimental_dataset_ops.cc      |  1 -
 .../data/experimental/ops/take_while_ops.py   |  8 +---
 3 files changed, 12 insertions(+), 43 deletions(-)

diff --git a/tensorflow/core/kernels/data/experimental/take_while_dataset_op.cc b/tensorflow/core/kernels/data/experimental/take_while_dataset_op.cc
index 271b919bc9..b18cc32051 100644
--- a/tensorflow/core/kernels/data/experimental/take_while_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/take_while_dataset_op.cc
@@ -1,4 +1,4 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -33,8 +33,6 @@ class TakeWhileDatasetOp : public UnaryDatasetOpKernel {
   explicit TakeWhileDatasetOp(OpKernelConstruction* ctx)
       : UnaryDatasetOpKernel(ctx) {
     OP_REQUIRES_OK(ctx, ctx->GetAttr("predicate", &func_));
-    OP_REQUIRES_OK(
-        ctx, ctx->GetAttr("preserve_cardinality", &preserve_cardinality_));
   }
 
   void MakeDataset(OpKernelContext* ctx, DatasetBase* input,
@@ -44,22 +42,19 @@ class TakeWhileDatasetOp : public UnaryDatasetOpKernel {
                                                  &captured_func));
     
     // TODO (squadrick): check short-circuit
-    *output = new Dataset(ctx, input, func_, std::move(captured_func), 
-                          preserve_cardinality_);
+    *output = new Dataset(ctx, input, func_, std::move(captured_func));
   }
 
  private:
   class Dataset : public DatasetBase {
    public:
     Dataset(OpKernelContext* ctx, const DatasetBase* input,
-            const NameAttrList& func,
-            std::unique_ptr<CapturedFunction> captured_func,
-            bool preserve_cardinality)
+            const NameAttrList& func, 
+            std::unique_ptr<CapturedFunction> captured_func)
         : DatasetBase(DatasetContext(ctx)),
           input_(input),
           func_(func),
-          captured_func_(std::move(captured_func)),
-          preserve_cardinality_(preserve_cardinality) {
+          captured_func_(std::move(captured_func)) {
       input_->Ref();
     }
 
@@ -74,13 +69,14 @@ class TakeWhileDatasetOp : public UnaryDatasetOpKernel {
     const DataTypeVector& output_dtypes() const override {
       return input_->output_dtypes();
     }
+
     const std::vector<PartialTensorShape>& output_shapes() const override {
       return input_->output_shapes();
     }
 
     string DebugString() const override { return "TakeWhileDatasetOp::Dataset"; }
 
-    int64 Cardinality() const override { return input_->Cardinality(); }
+    int64 Cardinality() const override { return kUnknownCardinality; }
 
    protected:
     Status AsGraphDefInternal(SerializationContext* ctx,
@@ -112,16 +108,11 @@ class TakeWhileDatasetOp : public UnaryDatasetOpKernel {
       AttrValue other_arguments_types_attr;
       b->BuildAttrValue(other_arguments_types, &other_arguments_types_attr);
 
-      AttrValue preserve_cardinality_attr;
-      b->BuildAttrValue(preserve_cardinality_, &preserve_cardinality_attr);
-
       TF_RETURN_IF_ERROR(b->AddDataset(
             this, {std::make_pair(0, input_node)},
             {std::make_pair(1, other_arguments)},
             {std::make_pair("predicate", f_attr),
-             std::make_pair("Targuments", other_arguments_types_attr),
-             std::make_pair("preserve_cardinality", 
-                            preserve_cardinality_attr)},
+             std::make_pair("Targuments", other_arguments_types_attr)},
              output));
       return Status::OK();
     }
@@ -168,27 +159,12 @@ class TakeWhileDatasetOp : public UnaryDatasetOpKernel {
             return errors::InvalidArgument(
                 "`predicate` must returns a scalar bool tensor.");
           }
-          auto cond = bool_output[0].scalar<bool>()();
-          if (!cond) { // predicate is false
-            *end_of_sequence = true;
-            return Status::OK();
-          }
-        } else if (errors::IsOutOfRange(s)) {
-          if (dataset()->preserve_cardinality_) {
-            // To guarantee that the transformation preserves the cardinality of
-            // the dataset, we convert `OutOfRange` to `InvalidArgument` as the
-            // former may be interpreted by a caller as the end of sequence.
-            return errors::InvalidArgument(
-                "Function invocation produced OutOfRangeError: ",
-                s.error_message());
-          } else {
-            // `f` may deliberately raise `errors::OutOfRange` to indicate
-            // that we should terminate the iteration early.
+          if (!bool_output[0].scalar<bool>()()) {
             *end_of_sequence = true;
             return Status::OK();
           }
         }
-        return s;
+        return s; // propagate error to caller
       }
 
      protected:
@@ -227,11 +203,9 @@ class TakeWhileDatasetOp : public UnaryDatasetOpKernel {
     const DatasetBase* const input_;
     const NameAttrList func_;
     const std::unique_ptr<CapturedFunction> captured_func_;
-    const bool preserve_cardinality_;
   };
 
   NameAttrList func_;
-  bool preserve_cardinality_;
 };
 
 REGISTER_KERNEL_BUILDER(Name("ExperimentalTakeWhileDataset").Device(DEVICE_CPU),
diff --git a/tensorflow/core/ops/experimental_dataset_ops.cc b/tensorflow/core/ops/experimental_dataset_ops.cc
index d10a13f06f..9ac50ffc72 100644
--- a/tensorflow/core/ops/experimental_dataset_ops.cc
+++ b/tensorflow/core/ops/experimental_dataset_ops.cc
@@ -360,7 +360,6 @@ REGISTER_OP("ExperimentalTakeWhileDataset")
     .Attr("Targuments: list(type) >= 0")
     .Attr("output_types: list(type) >= 1")
     .Attr("output_shapes: list(shape) >= 1")
-    .Attr("preserve_cardinality: bool = false")
     .SetShapeFn(shape_inference::ScalarShape);
 
 REGISTER_OP("ExperimentalUnbatchDataset")
diff --git a/tensorflow/python/data/experimental/ops/take_while_ops.py b/tensorflow/python/data/experimental/ops/take_while_ops.py
index edbecd0666..224eb8aa77 100644
--- a/tensorflow/python/data/experimental/ops/take_while_ops.py
+++ b/tensorflow/python/data/experimental/ops/take_while_ops.py
@@ -1,4 +1,4 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -36,7 +36,7 @@ class _TakeWhileDataset(dataset_ops.UnaryUnchangedStructureDataset):
     self._input_dataset = input_dataset
     wrapped_func = dataset_ops.StructuredFunctionWrapper(
             predicate,
-            self._transformation_name(),
+            "tf.data.experimental.take_while()",
             dataset=self._input_dataset)
     
     if not wrapped_func.output_structure.is_compatible_with(
@@ -49,16 +49,12 @@ class _TakeWhileDataset(dataset_ops.UnaryUnchangedStructureDataset):
             self._input_dataset._variant_tensor,
             other_arguments=self._predicate.function.captured_inputs,
             predicate=self._predicate.function,
-            preserve_cardinality=True,
             **dataset_ops.flat_structure(self))
     super(_TakeWhileDataset, self).__init__(input_dataset, variant_tensor)
 
   def _functions(self):
       return [self._predicate]
 
-  def _transformation_name(self):
-      return "tf.data.experimental.take_while()"
-
 
 @tf_export("data.experimental.take_while")
 def take_while(predicate):
-- 
GitLab


From b9abbc5424e6e883a1ec481344e3b080a29c49b8 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 1 Jan 2019 09:05:30 -0800
Subject: [PATCH 0105/1095] More `TFGAN` to `TF-GAN`.

PiperOrigin-RevId: 227437819
---
 tensorflow/contrib/gan/python/eval/__init__.py                  | 2 +-
 tensorflow/contrib/gan/python/eval/python/classifier_metrics.py | 2 +-
 .../contrib/gan/python/eval/python/classifier_metrics_test.py   | 2 +-
 tensorflow/contrib/gan/python/eval/python/sliced_wasserstein.py | 2 +-
 tensorflow/contrib/gan/python/eval/python/summaries.py          | 2 +-
 tensorflow/contrib/gan/python/eval/python/summaries_impl.py     | 2 +-
 tensorflow/contrib/gan/python/eval/python/summaries_test.py     | 2 +-
 7 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/tensorflow/contrib/gan/python/eval/__init__.py b/tensorflow/contrib/gan/python/eval/__init__.py
index f86b851305..92e9abf8a3 100644
--- a/tensorflow/contrib/gan/python/eval/__init__.py
+++ b/tensorflow/contrib/gan/python/eval/__init__.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""TFGAN evaluation module.
+"""TF-GAN evaluation module.
 
 This module supports techniques such as Inception Score, Frechet Inception
 distance, and Sliced Wasserstein distance.
diff --git a/tensorflow/contrib/gan/python/eval/python/classifier_metrics.py b/tensorflow/contrib/gan/python/eval/python/classifier_metrics.py
index 1c872626a9..a52e899114 100644
--- a/tensorflow/contrib/gan/python/eval/python/classifier_metrics.py
+++ b/tensorflow/contrib/gan/python/eval/python/classifier_metrics.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Model evaluation tools for TFGAN."""
+"""Model evaluation tools for TF-GAN."""
 
 from __future__ import absolute_import
 from __future__ import division
diff --git a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_test.py b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_test.py
index dbff1d2a36..ab2afb21fd 100644
--- a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_test.py
+++ b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_test.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for TFGAN classifier_metrics."""
+"""Tests for TF-GAN classifier_metrics."""
 
 from __future__ import absolute_import
 from __future__ import division
diff --git a/tensorflow/contrib/gan/python/eval/python/sliced_wasserstein.py b/tensorflow/contrib/gan/python/eval/python/sliced_wasserstein.py
index 523968bed9..326fcb3cdb 100644
--- a/tensorflow/contrib/gan/python/eval/python/sliced_wasserstein.py
+++ b/tensorflow/contrib/gan/python/eval/python/sliced_wasserstein.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Model evaluation tools for TFGAN."""
+"""Model evaluation tools for TF-GAN."""
 
 from __future__ import absolute_import
 from __future__ import division
diff --git a/tensorflow/contrib/gan/python/eval/python/summaries.py b/tensorflow/contrib/gan/python/eval/python/summaries.py
index ecfdb39499..1b202dfc97 100644
--- a/tensorflow/contrib/gan/python/eval/python/summaries.py
+++ b/tensorflow/contrib/gan/python/eval/python/summaries.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Common TFGAN summaries."""
+"""Common TF-GAN summaries."""
 
 from __future__ import absolute_import
 from __future__ import division
diff --git a/tensorflow/contrib/gan/python/eval/python/summaries_impl.py b/tensorflow/contrib/gan/python/eval/python/summaries_impl.py
index f9995bb19d..9f448d3a16 100644
--- a/tensorflow/contrib/gan/python/eval/python/summaries_impl.py
+++ b/tensorflow/contrib/gan/python/eval/python/summaries_impl.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Common TFGAN summaries."""
+"""Common TF-GAN summaries."""
 
 from __future__ import absolute_import
 from __future__ import division
diff --git a/tensorflow/contrib/gan/python/eval/python/summaries_test.py b/tensorflow/contrib/gan/python/eval/python/summaries_test.py
index 54a6f8d4d9..53fc7cb8ed 100644
--- a/tensorflow/contrib/gan/python/eval/python/summaries_test.py
+++ b/tensorflow/contrib/gan/python/eval/python/summaries_test.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for TFGAN summaries."""
+"""Tests for TF-GAN summaries."""
 
 from __future__ import absolute_import
 from __future__ import division
-- 
GitLab


From 07ab5fa7f9dab3fb644a0261382c95e5be072741 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 1 Jan 2019 09:36:41 -0800
Subject: [PATCH 0106/1095] assertTrue(isinstance(xxx)) -> assertIsInstance

PiperOrigin-RevId: 227439314
---
 .../python/eval/python/classifier_metrics_test.py  | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_test.py b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_test.py
index ab2afb21fd..bd17571a05 100644
--- a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_test.py
+++ b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_test.py
@@ -234,7 +234,7 @@ class ClassifierMetricsTest(test.TestCase, parameterized.TestCase):
     else:
       logits = classifier_metrics.run_inception(img, _get_dummy_graphdef())
 
-    self.assertTrue(isinstance(logits, ops.Tensor))
+    self.assertIsInstance(logits, ops.Tensor)
     logits.shape.assert_is_compatible_with([batch_size, 1001])
 
     # Check that none of the model variables are trainable.
@@ -258,7 +258,7 @@ class ClassifierMetricsTest(test.TestCase, parameterized.TestCase):
           img, _get_dummy_graphdef(),
           output_tensor=classifier_metrics.INCEPTION_FINAL_POOL)
 
-    self.assertTrue(isinstance(pool, ops.Tensor))
+    self.assertIsInstance(pool, ops.Tensor)
     pool.shape.assert_is_compatible_with([batch_size, 2048])
 
     # Check that none of the model variables are trainable.
@@ -276,8 +276,8 @@ class ClassifierMetricsTest(test.TestCase, parameterized.TestCase):
             classifier_metrics.INCEPTION_FINAL_POOL
         ])
 
-    self.assertTrue(isinstance(logits, ops.Tensor))
-    self.assertTrue(isinstance(pool, ops.Tensor))
+    self.assertIsInstance(logits, ops.Tensor)
+    self.assertIsInstance(pool, ops.Tensor)
     logits.shape.assert_is_compatible_with([batch_size, 1001])
     pool.shape.assert_is_compatible_with([batch_size, 2048])
 
@@ -290,7 +290,7 @@ class ClassifierMetricsTest(test.TestCase, parameterized.TestCase):
         classifier_metrics.inception_score,
         array_ops.zeros([6, 299, 299, 3]),
         num_batches=3)
-    self.assertTrue(isinstance(score, ops.Tensor))
+    self.assertIsInstance(score, ops.Tensor)
     score.shape.assert_has_rank(0)
 
     # Check that none of the model variables are trainable.
@@ -302,7 +302,7 @@ class ClassifierMetricsTest(test.TestCase, parameterized.TestCase):
     distance = _run_with_mock(
         classifier_metrics.frechet_inception_distance, img, img)
 
-    self.assertTrue(isinstance(distance, ops.Tensor))
+    self.assertIsInstance(distance, ops.Tensor)
     distance.shape.assert_has_rank(0)
 
     # Check that none of the model variables are trainable.
@@ -314,7 +314,7 @@ class ClassifierMetricsTest(test.TestCase, parameterized.TestCase):
     distance = _run_with_mock(classifier_metrics.kernel_inception_distance, img,
                               img)
 
-    self.assertTrue(isinstance(distance, ops.Tensor))
+    self.assertIsInstance(distance, ops.Tensor)
     distance.shape.assert_has_rank(0)
 
     # Check that none of the model variables are trainable.
-- 
GitLab


From 4e07d4ae73de5632bb49514a5b486140bdd8516e Mon Sep 17 00:00:00 2001
From: Dheeraj Rajaram Reddy <dheeraj98reddy@gmail.com>
Date: Tue, 1 Jan 2019 23:47:28 +0530
Subject: [PATCH 0107/1095] Add take_while serialization test

---
 .../take_while_dataset_serialization_test.py  | 45 +++++++++++++++++++
 1 file changed, 45 insertions(+)
 create mode 100644 tensorflow/python/data/experimental/kernel_tests/serialization/take_while_dataset_serialization_test.py

diff --git a/tensorflow/python/data/experimental/kernel_tests/serialization/take_while_dataset_serialization_test.py b/tensorflow/python/data/experimental/kernel_tests/serialization/take_while_dataset_serialization_test.py
new file mode 100644
index 0000000000..9b5498cca5
--- /dev/null
+++ b/tensorflow/python/data/experimental/kernel_tests/serialization/take_while_dataset_serialization_test.py
@@ -0,0 +1,45 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for the TakeWhileDataset serialization."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.data.experimental.kernel_tests.serialization import dataset_serialization_test_base
+from tensorflow.python.data.experimental.ops import take_while_ops
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.platform import test
+
+
+class TakeWhileDatasetSerializationTest(
+    dataset_serialization_test_base.DatasetSerializationTestBase):
+
+  def _build_dataset(self, num_elements, upper_bound):
+    return dataset_ops.Dataset.range(num_elements).apply(
+              take_while_ops.take_while(lambda x: x < upper_bound))
+
+  def testCore(self):
+    def run_test(num_elem1, num_elem2, upper_bound):
+        self.run_core_tests(lambda: self._build_dataset(num_elem1, upper_bound),
+                            lambda: self._build_dataset(num_elem2, upper_bound), 
+                            upper_bound)
+
+    run_test(23, 10, 7)
+    run_test(10, 50, 0)
+    run_test(25, 30, 25)
+
+
+if __name__ == "__main__":
+  test.main()
-- 
GitLab


From d1959dddd107c79db8c8d33d0a37ebc9fe8b0a21 Mon Sep 17 00:00:00 2001
From: Michael Case <mikecase@google.com>
Date: Tue, 1 Jan 2019 13:19:42 -0800
Subject: [PATCH 0108/1095] Remove explicit dependencies tests have on
 keras-related targets.

Changing all of these deps to be on a single large target. This is
because when we refactor the build files we will want only a few public
targets that things outside of Keras can depend on. And, it will make
copybara transforms much easier (can swap out single target with dummy
in opensource).

PiperOrigin-RevId: 227449197
---
 tensorflow/compiler/tests/BUILD            |  3 +++
 tensorflow/contrib/checkpoint/python/BUILD | 22 +++++++--------
 tensorflow/contrib/feature_column/BUILD    | 31 ++++++++++------------
 tensorflow/python/BUILD                    |  7 +----
 4 files changed, 29 insertions(+), 34 deletions(-)

diff --git a/tensorflow/compiler/tests/BUILD b/tensorflow/compiler/tests/BUILD
index fa02cf9cbe..f80cb1812f 100644
--- a/tensorflow/compiler/tests/BUILD
+++ b/tensorflow/compiler/tests/BUILD
@@ -230,6 +230,7 @@ tf_xla_py_test(
         "//tensorflow/python:framework",
         "//tensorflow/python:platform_test",
         "//tensorflow/python:random_ops",
+        "//tensorflow/python:standard_ops",
     ],
 )
 
@@ -677,6 +678,7 @@ tf_xla_py_test(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:platform_test",
         "//tensorflow/python:random_ops",
+        "//tensorflow/python:standard_ops",
     ],
 )
 
@@ -826,6 +828,7 @@ tf_xla_py_test(
         ":xla_test",
         "//tensorflow/python:framework",
         "//tensorflow/python:platform_test",
+        "//tensorflow/python:standard_ops",
         "//tensorflow/python:stateless_random_ops",
     ],
 )
diff --git a/tensorflow/contrib/checkpoint/python/BUILD b/tensorflow/contrib/checkpoint/python/BUILD
index ada4168726..4e529322c7 100644
--- a/tensorflow/contrib/checkpoint/python/BUILD
+++ b/tensorflow/contrib/checkpoint/python/BUILD
@@ -2,7 +2,7 @@ licenses(["notice"])  # Apache 2.0
 
 package(default_visibility = ["//tensorflow:internal"])
 
-load("//tensorflow:tensorflow.bzl", "py_test")
+load("//tensorflow:tensorflow.bzl", "tf_py_test")
 
 py_library(
     name = "checkpoint",
@@ -27,17 +27,17 @@ py_library(
     ],
 )
 
-py_test(
+tf_py_test(
     name = "containers_test",
     srcs = ["containers_test.py"],
-    deps = [
+    additional_deps = [
         ":containers",
+        "@six_archive//:six",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:resource_variable_ops",
         "//tensorflow/python/training/checkpointable:base",
         "//tensorflow/python/training/checkpointable:util",
-        "@six_archive//:six",
     ],
 )
 
@@ -53,18 +53,18 @@ py_library(
     ],
 )
 
-py_test(
+tf_py_test(
     name = "python_state_test",
     srcs = ["python_state_test.py"],
-    deps = [
+    additional_deps = [
         ":python_state",
+        "//third_party/py/numpy",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:session",
         "//tensorflow/python:variables",
         "//tensorflow/python/eager:test",
         "//tensorflow/python/training/checkpointable:util",
-        "//third_party/py/numpy",
     ],
 )
 
@@ -80,10 +80,10 @@ py_library(
     ],
 )
 
-py_test(
+tf_py_test(
     name = "split_dependency_test",
     srcs = ["split_dependency_test.py"],
-    deps = [
+    additional_deps = [
         ":split_dependency",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:framework_test_lib",
@@ -106,10 +106,10 @@ py_library(
     ],
 )
 
-py_test(
+tf_py_test(
     name = "visualize_test",
     srcs = ["visualize_test.py"],
-    deps = [
+    additional_deps = [
         ":visualize",
         "//tensorflow/python:constant_op",
         "//tensorflow/python:resource_variable_ops",
diff --git a/tensorflow/contrib/feature_column/BUILD b/tensorflow/contrib/feature_column/BUILD
index 4c1d1a29f2..4e29e25599 100644
--- a/tensorflow/contrib/feature_column/BUILD
+++ b/tensorflow/contrib/feature_column/BUILD
@@ -6,7 +6,7 @@ package(
 
 licenses(["notice"])  # Apache 2.0
 
-load("//tensorflow:tensorflow.bzl", "py_test")
+load("//tensorflow:tensorflow.bzl", "tf_py_test")
 
 py_library(
     name = "feature_column_py",
@@ -37,13 +37,13 @@ py_library(
     ],
 )
 
-py_test(
+tf_py_test(
     name = "sequence_feature_column_test",
     srcs = ["python/feature_column/sequence_feature_column_test.py"],
-    srcs_version = "PY2AND3",
-    tags = ["no_pip"],
-    deps = [
+    additional_deps = [
         ":sequence_feature_column",
+        "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:errors",
@@ -53,17 +53,14 @@ py_test(
         "//tensorflow/python:sparse_tensor",
         "//tensorflow/python:training",
         "//tensorflow/python/feature_column:feature_column_py",
-        "//third_party/py/numpy",
-        "@absl_py//absl/testing:parameterized",
     ],
+    tags = ["no_pip"],
 )
 
-py_test(
+tf_py_test(
     name = "sequence_feature_column_integration_test",
     srcs = ["python/feature_column/sequence_feature_column_integration_test.py"],
-    srcs_version = "PY2AND3",
-    tags = ["no_pip"],
-    deps = [
+    additional_deps = [
         ":sequence_feature_column",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_ops",
@@ -73,6 +70,7 @@ py_test(
         "//tensorflow/python/feature_column:feature_column_py",
         "//tensorflow/python/keras:layers",
     ],
+    tags = ["no_pip"],
 )
 
 py_library(
@@ -94,14 +92,14 @@ py_library(
     ],
 )
 
-py_test(
+tf_py_test(
     name = "sequence_feature_column_v2_test",
     srcs = ["python/feature_column/sequence_feature_column_v2_test.py"],
-    srcs_version = "PY2AND3",
-    tags = ["no_pip"],
-    deps = [
+    additional_deps = [
         ":sequence_feature_column",
         ":sequence_feature_column_v2",
+        "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:errors",
@@ -112,7 +110,6 @@ py_test(
         "//tensorflow/python:training",
         "//tensorflow/python/feature_column:feature_column_py",
         "//tensorflow/python/feature_column:feature_column_v2_test",
-        "//third_party/py/numpy",
-        "@absl_py//absl/testing:parameterized",
     ],
+    tags = ["no_pip"],
 )
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 7503c899aa..6724177d17 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -1067,6 +1067,7 @@ py_library(
     name = "extra_py_tests_deps",
     srcs_version = "PY2AND3",
     deps = [
+        ":keras_lib",
         "//third_party/py/numpy",
         "@six_archive//:six",
     ],
@@ -1081,7 +1082,6 @@ py_library(
         ":client",
         ":errors",
         ":framework_for_generated_wrappers",
-        ":layers_base",
         ":platform",
         ":platform_test",
         ":pywrap_tensorflow",
@@ -1096,7 +1096,6 @@ py_library(
         "//tensorflow/python/eager:backprop",
         "//tensorflow/python/eager:context",
         "//tensorflow/python/eager:tape",
-        "//tensorflow/python/keras:layers",
         "//third_party/py/numpy",
         "@six_archive//:six",
     ],
@@ -3311,7 +3310,6 @@ cuda_py_test(
         ":framework_test_lib",
         ":functional_ops",
         ":gradients",
-        ":layers",
         ":list_ops",
         ":math_grad",
         ":math_ops",
@@ -3708,7 +3706,6 @@ tf_py_test(
         ":framework",
         ":framework_for_generated_wrappers",
         ":framework_test_lib",
-        ":layers",
         ":math_ops",
         ":metrics",
         ":platform",
@@ -4982,7 +4979,6 @@ tf_py_test(
         ":variable_scope",
         ":variables",
         "//third_party/py/numpy",
-        "//tensorflow/python/feature_column:feature_column_py",
     ],
 )
 
@@ -5753,7 +5749,6 @@ cuda_py_test(
         ":constant_op",
         ":dtypes",
         ":functional_ops",
-        ":layers",
         ":math_ops",
         ":nn",
         ":ops",
-- 
GitLab


From efb14346f9344db2f07c94428ee86f86100720b0 Mon Sep 17 00:00:00 2001
From: Ruoxin Sang <rxsang@google.com>
Date: Tue, 1 Jan 2019 17:58:48 -0800
Subject: [PATCH 0109/1095] Set `_is_compiled` attribute for Keras custom
 compile for predict.

PiperOrigin-RevId: 227461814
---
 .../python/keras/engine/distributed_training_utils.py       | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tensorflow/python/keras/engine/distributed_training_utils.py b/tensorflow/python/keras/engine/distributed_training_utils.py
index a4fc2cf196..231c59ba93 100644
--- a/tensorflow/python/keras/engine/distributed_training_utils.py
+++ b/tensorflow/python/keras/engine/distributed_training_utils.py
@@ -671,6 +671,12 @@ def _prepare_feed_values(model, inputs, targets, sample_weights, mode):
 
 def _custom_compile_for_predict(model):
   """Custom compile for TPU predict mode."""
+  if not model.built:
+    # Model is not compilable because it does not know its number of inputs
+    # and outputs, nor their shapes and names. We will compile after the first
+    # time the model gets called on training data.
+    return
+  model._is_compiled = True
   model.total_loss = None
   model._fit_function = None
   model._eval_function = None
-- 
GitLab


From b277e326766ac3a0eff3e72e9e53dadcc31674e2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 1 Jan 2019 23:48:05 -0800
Subject: [PATCH 0110/1095] TFGAN -> TF-GAN

PiperOrigin-RevId: 227481763
---
 tensorflow/contrib/gan/python/estimator/__init__.py             | 2 +-
 .../contrib/gan/python/estimator/python/gan_estimator_test.py   | 2 +-
 tensorflow/contrib/gan/python/estimator/python/head_impl.py     | 2 +-
 tensorflow/contrib/gan/python/estimator/python/head_test.py     | 2 +-
 .../gan/python/estimator/python/stargan_estimator_impl.py       | 2 +-
 .../gan/python/estimator/python/stargan_estimator_test.py       | 2 +-
 .../gan/python/estimator/python/tpu_gan_estimator_impl.py       | 2 +-
 .../gan/python/estimator/python/tpu_gan_estimator_test.py       | 2 +-
 8 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/tensorflow/contrib/gan/python/estimator/__init__.py b/tensorflow/contrib/gan/python/estimator/__init__.py
index 75cccb5ea0..c1433749c1 100644
--- a/tensorflow/contrib/gan/python/estimator/__init__.py
+++ b/tensorflow/contrib/gan/python/estimator/__init__.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""TFGAN estimator module.
+"""TF-GAN estimator module.
 
 GANEstimator provides all the infrastructure support of a TensorFlow Estimator
 with the feature support of TFGAN.
diff --git a/tensorflow/contrib/gan/python/estimator/python/gan_estimator_test.py b/tensorflow/contrib/gan/python/estimator/python/gan_estimator_test.py
index 5a3d29cf0b..5b9c54e43a 100644
--- a/tensorflow/contrib/gan/python/estimator/python/gan_estimator_test.py
+++ b/tensorflow/contrib/gan/python/estimator/python/gan_estimator_test.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for TFGAN's estimator.py."""
+"""Tests for TF-GAN's estimator.py."""
 
 from __future__ import absolute_import
 from __future__ import division
diff --git a/tensorflow/contrib/gan/python/estimator/python/head_impl.py b/tensorflow/contrib/gan/python/estimator/python/head_impl.py
index 1a0ee6dfc4..cbe990b476 100644
--- a/tensorflow/contrib/gan/python/estimator/python/head_impl.py
+++ b/tensorflow/contrib/gan/python/estimator/python/head_impl.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""A TFGAN-backed GAN Estimator."""
+"""A TF-GAN-backed GAN Estimator."""
 
 from __future__ import absolute_import
 from __future__ import division
diff --git a/tensorflow/contrib/gan/python/estimator/python/head_test.py b/tensorflow/contrib/gan/python/estimator/python/head_test.py
index 8205bc889d..5b50234a0e 100644
--- a/tensorflow/contrib/gan/python/estimator/python/head_test.py
+++ b/tensorflow/contrib/gan/python/estimator/python/head_test.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for TFGAN's head.py."""
+"""Tests for TF-GAN's head.py."""
 
 from __future__ import absolute_import
 from __future__ import division
diff --git a/tensorflow/contrib/gan/python/estimator/python/stargan_estimator_impl.py b/tensorflow/contrib/gan/python/estimator/python/stargan_estimator_impl.py
index f60e16bc04..2a485e7d47 100644
--- a/tensorflow/contrib/gan/python/estimator/python/stargan_estimator_impl.py
+++ b/tensorflow/contrib/gan/python/estimator/python/stargan_estimator_impl.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""A TFGAN-backed StarGAN Estimator."""
+"""A TF-GAN-backed StarGAN Estimator."""
 
 from __future__ import absolute_import
 from __future__ import division
diff --git a/tensorflow/contrib/gan/python/estimator/python/stargan_estimator_test.py b/tensorflow/contrib/gan/python/estimator/python/stargan_estimator_test.py
index 2ec7938c7c..00ca393c51 100644
--- a/tensorflow/contrib/gan/python/estimator/python/stargan_estimator_test.py
+++ b/tensorflow/contrib/gan/python/estimator/python/stargan_estimator_test.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for TFGAN's stargan_estimator.py."""
+"""Tests for TF-GAN's stargan_estimator.py."""
 
 from __future__ import absolute_import
 from __future__ import division
diff --git a/tensorflow/contrib/gan/python/estimator/python/tpu_gan_estimator_impl.py b/tensorflow/contrib/gan/python/estimator/python/tpu_gan_estimator_impl.py
index bf51b7fc45..8f2a22c78a 100644
--- a/tensorflow/contrib/gan/python/estimator/python/tpu_gan_estimator_impl.py
+++ b/tensorflow/contrib/gan/python/estimator/python/tpu_gan_estimator_impl.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""A TFGAN-backed GAN Estimator that works on TPU."""
+"""A TF-GAN-backed GAN Estimator that works on TPU."""
 
 from __future__ import absolute_import
 from __future__ import division
diff --git a/tensorflow/contrib/gan/python/estimator/python/tpu_gan_estimator_test.py b/tensorflow/contrib/gan/python/estimator/python/tpu_gan_estimator_test.py
index 0a08b4386f..9fdcc08334 100644
--- a/tensorflow/contrib/gan/python/estimator/python/tpu_gan_estimator_test.py
+++ b/tensorflow/contrib/gan/python/estimator/python/tpu_gan_estimator_test.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for TFGAN's TPU Estimator."""
+"""Tests for TF-GAN's TPU Estimator."""
 
 from __future__ import absolute_import
 from __future__ import division
-- 
GitLab


From 5ed6676c75192429132bbc0da179ad3e3367089c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 2 Jan 2019 01:02:25 -0800
Subject: [PATCH 0111/1095] compat: Update forward compatibility horizon to
 2019-01-02

PiperOrigin-RevId: 227487773
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 2284d0021c..949bb0dc0c 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 1)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 2)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 26d81c48681c17fbfe1c586b54512d8a89062183 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 2 Jan 2019 02:58:21 -0800
Subject: [PATCH 0112/1095] Improve code generated by scatter expander

After this change if the indexes dimension is 1 then we remove it and
treat the scatter as a scatter with scalar indices.

PiperOrigin-RevId: 227497329
---
 tensorflow/compiler/xla/service/scatter_expander.cc | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/tensorflow/compiler/xla/service/scatter_expander.cc b/tensorflow/compiler/xla/service/scatter_expander.cc
index e8496dbd72..8b9955faf8 100644
--- a/tensorflow/compiler/xla/service/scatter_expander.cc
+++ b/tensorflow/compiler/xla/service/scatter_expander.cc
@@ -26,7 +26,6 @@ limitations under the License.
 
 namespace xla {
 
-
 // Transposes the given scatter_indices such that the index_vector_dim becomes
 // the most-minor dimension.
 static StatusOr<HloInstruction*> TransposeIndexVectorDimToLast(
@@ -60,6 +59,13 @@ static StatusOr<HloInstruction*> CanonicalizeScatterIndices(
   TF_ASSIGN_OR_RETURN(
       HloInstruction * transposed_scatter_indices,
       TransposeIndexVectorDimToLast(scatter_indices, index_vector_dim));
+  if (ShapeUtil::Rank(scatter_indices->shape()) == index_vector_dim + 1 &&
+      scatter_indices->shape().dimensions(index_vector_dim) == 1) {
+    auto new_shape =
+        ShapeUtil::DeleteDimension(index_vector_dim, scatter_indices->shape());
+    TF_ASSIGN_OR_RETURN(scatter_indices,
+                        MakeReshapeHlo(new_shape, scatter_indices));
+  }
   bool indices_are_scalar =
       index_vector_dim == scatter_indices->shape().dimensions_size();
 
@@ -214,9 +220,6 @@ static StatusOr<std::vector<HloInstruction*>> ScatterLoopBody(
   HloInstruction* updates = loop_state[2];
 
   bool has_scalar_indices = scatter_indices->shape().dimensions_size() == 1;
-  CHECK_EQ(has_scalar_indices,
-           dim_numbers.index_vector_dim() ==
-               scatter->operand(1)->shape().dimensions_size());
 
   // Build a vector form of the induction variable of the while loop.
   TF_ASSIGN_OR_RETURN(
-- 
GitLab


From 2b5cbcdd8d84d9673c9f17b4400ec6d70e2c8d73 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 2 Jan 2019 04:33:33 -0800
Subject: [PATCH 0113/1095] More `TFGAN` -> `TF-GAN`.

PiperOrigin-RevId: 227504393
---
 tensorflow/contrib/gan/python/estimator/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/gan/python/estimator/__init__.py b/tensorflow/contrib/gan/python/estimator/__init__.py
index c1433749c1..03a639165b 100644
--- a/tensorflow/contrib/gan/python/estimator/__init__.py
+++ b/tensorflow/contrib/gan/python/estimator/__init__.py
@@ -15,7 +15,7 @@
 """TF-GAN estimator module.
 
 GANEstimator provides all the infrastructure support of a TensorFlow Estimator
-with the feature support of TFGAN.
+with the feature support of TF-GAN.
 """
 
 from __future__ import absolute_import
-- 
GitLab


From 8ba62643f9e33f6684252f68ffd8c282d96a7262 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 2 Jan 2019 05:14:40 -0800
Subject: [PATCH 0114/1095] Ran build_cleaner.

PiperOrigin-RevId: 227507517
---
 tensorflow/contrib/gan/BUILD | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/tensorflow/contrib/gan/BUILD b/tensorflow/contrib/gan/BUILD
index 0fa229880b..97184dabb0 100644
--- a/tensorflow/contrib/gan/BUILD
+++ b/tensorflow/contrib/gan/BUILD
@@ -569,22 +569,18 @@ py_test(
     deps = [
         ":namedtuples",
         ":stargan_estimator",
-        ":tuple_losses",
         "//tensorflow/contrib/layers:layers_py",
-        "//tensorflow/contrib/learn",
-        "//tensorflow/core:protos_all_py",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:client_testlib",
-        "//tensorflow/python:dtypes",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:math_ops",
         "//tensorflow/python:metrics",
-        "//tensorflow/python:parsing_ops",
         "//tensorflow/python:summary",
         "//tensorflow/python:training",
         "//tensorflow/python:training_util",
         "//tensorflow/python:variable_scope",
-        "//tensorflow/python/estimator:estimator_py",
+        "//tensorflow/python/estimator:model_fn",
+        "//tensorflow/python/estimator:numpy_io",
         "//third_party/py/numpy",
         "@absl_py//absl/testing:parameterized",
         "@six_archive//:six",
-- 
GitLab


From 7700b1ebeb981b025eccc6e115b2209107d8b8fa Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 2 Jan 2019 05:22:10 -0800
Subject: [PATCH 0115/1095] Fix lint warnings.

PiperOrigin-RevId: 227508006
---
 .../gan/python/estimator/python/stargan_estimator_test.py    | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/gan/python/estimator/python/stargan_estimator_test.py b/tensorflow/contrib/gan/python/estimator/python/stargan_estimator_test.py
index 00ca393c51..c00ff43997 100644
--- a/tensorflow/contrib/gan/python/estimator/python/stargan_estimator_test.py
+++ b/tensorflow/contrib/gan/python/estimator/python/stargan_estimator_test.py
@@ -80,7 +80,7 @@ class StarGetGANModelTest(test.TestCase, parameterized.TestCase):
     self.assertEqual(input_data, gan_model.input_data)
     self.assertIsNotNone(gan_model.generated_data)
     self.assertIsNotNone(gan_model.generated_data_domain_target)
-    self.assertEqual(1, len(gan_model.generator_variables))
+    self.assertLen(gan_model.generator_variables, 1)
     self.assertIsNotNone(gan_model.generator_scope)
     self.assertIsNotNone(gan_model.generator_fn)
     if mode == model_fn_lib.ModeKeys.PREDICT:
@@ -109,7 +109,7 @@ class StarGetGANModelTest(test.TestCase, parameterized.TestCase):
           gan_model.discriminator_input_data_domain_predication)
       self.assertIsNotNone(
           gan_model.discriminator_generated_data_domain_predication)
-      self.assertEqual(2, len(gan_model.discriminator_variables))  # 1 FC layer
+      self.assertLen(gan_model.discriminator_variables, 2)  # 1 FC layer
       self.assertIsNotNone(gan_model.discriminator_scope)
       self.assertIsNotNone(gan_model.discriminator_fn)
 
@@ -163,6 +163,7 @@ class GetEstimatorSpecTest(test.TestCase, parameterized.TestCase):
 
   @classmethod
   def setUpClass(cls):
+    super(GetEstimatorSpecTest, cls).setUpClass()
     cls._generator_optimizer = training.GradientDescentOptimizer(1.0)
     cls._discriminator_optimizer = training.GradientDescentOptimizer(1.0)
 
-- 
GitLab


From 44442969c0596728dbdb98035bffc2baf1790f13 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 2 Jan 2019 05:25:21 -0800
Subject: [PATCH 0116/1095] Clang breaks with the libstdc++ version on ubuntu
 14.04, so install a newer libstdc++, like we do for the CUDA 9 image already.

PiperOrigin-RevId: 227508237
---
 .../Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04      | 13 +++++++++++--
 third_party/toolchains/BUILD                        |  2 +-
 .../toolchains/preconfig/generate/containers.bzl    |  2 +-
 .../toolchains/preconfig/generate/workspace.bzl     |  2 +-
 4 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/tensorflow/tools/ci_build/Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04 b/tensorflow/tools/ci_build/Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04
index 4fe86066c9..553fbb2860 100644
--- a/tensorflow/tools/ci_build/Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04
+++ b/tensorflow/tools/ci_build/Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04
@@ -1,7 +1,7 @@
 # To push a new version, run:
 # $ docker build -f Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04 \
-#       --tag "gcr.io/asci-toolchain/nosla-cuda10.0-cudnn7-ubuntu14.04" .
-# $ docker push gcr.io/asci-toolchain/nosla-cuda10.0-cudnn7-ubuntu14.04
+#       --tag "gcr.io/tensorflow-testing/nosla-cuda10.0-cudnn7-ubuntu14.04" .
+# $ docker push gcr.io/tensorflow-testing/nosla-cuda10.0-cudnn7-ubuntu14.04
 
 FROM ubuntu:14.04
 LABEL maintainer="Manuel Klimek <klimek@google.com>"
@@ -68,6 +68,15 @@ RUN ln -s libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1
 RUN ln -s /usr/lib/x86_64-linux-gnu/libnccl.so /usr/lib/libnccl.so \
  && ln -s /usr/lib/x86_64-linux-gnu/libnccl.so /usr/lib/libnccl.so.2
 
+# Install a newer version of libstdc++, as new clang versions do not work
+# with the stock ubuntu 14.04 libstdc++.
+RUN apt-get update && \
+    apt-get install -y software-properties-common && \
+    add-apt-repository ppa:ubuntu-toolchain-r/test -y && \
+    apt-get update && \
+    apt-get install -y libstdc++-7-dev && \
+    rm -rf /var/lib/apt/lists/*
+
 # Copy and run the install scripts.
 COPY install/*.sh /install/
 ARG DEBIAN_FRONTEND=noninteractive
diff --git a/third_party/toolchains/BUILD b/third_party/toolchains/BUILD
index 9da417fd5f..28f1e6979c 100644
--- a/third_party/toolchains/BUILD
+++ b/third_party/toolchains/BUILD
@@ -45,6 +45,6 @@ platform(
     remote_execution_properties = """
         properties: {
             name: "container-image"
-            value:"docker://gcr.io/asci-toolchain/nosla-cuda10.0-cudnn7-ubuntu14.04@%s"
+            value:"docker://gcr.io/tensorflow-testing/nosla-cuda10.0-cudnn7-ubuntu14.04@%s"
         }""" % container_digests["cuda10.0-cudnn7-ubuntu14.04"],
 )
diff --git a/third_party/toolchains/preconfig/generate/containers.bzl b/third_party/toolchains/preconfig/generate/containers.bzl
index c56c6f3346..32d5f29b52 100644
--- a/third_party/toolchains/preconfig/generate/containers.bzl
+++ b/third_party/toolchains/preconfig/generate/containers.bzl
@@ -1,4 +1,4 @@
 container_digests = {
     "cuda9.0-cudnn7-ubuntu14.04": "sha256:c43ed5341dd765042e0bbd1bf50fadeedd649d1e0c34d81999cb6ce30916cb95",
-    "cuda10.0-cudnn7-ubuntu14.04": "sha256:919e75247743ae1244d5d72ee9f18090379d4a9035e5853010f6d59d87cd2e8b",
+    "cuda10.0-cudnn7-ubuntu14.04": "sha256:e36f05f1ff39e39ddf07122e37f2b1895948bb6f7acc3db37a3c496be5e66228",
 }
diff --git a/third_party/toolchains/preconfig/generate/workspace.bzl b/third_party/toolchains/preconfig/generate/workspace.bzl
index f30c2f1ae6..a3268585e1 100644
--- a/third_party/toolchains/preconfig/generate/workspace.bzl
+++ b/third_party/toolchains/preconfig/generate/workspace.bzl
@@ -18,7 +18,7 @@ def _remote_config_workspace():
     container_pull(
         name = "cuda10.0-cudnn7-ubuntu14.04",
         registry = "gcr.io",
-        repository = "asci-toolchain/nosla-cuda10.0-cudnn7-ubuntu14.04",
+        repository = "tensorflow-testing/nosla-cuda10.0-cudnn7-ubuntu14.04",
         digest = container_digests["cuda10.0-cudnn7-ubuntu14.04"],
     )
 
-- 
GitLab


From ab7465e957d2f5b6f8ada982cf91266629092677 Mon Sep 17 00:00:00 2001
From: Dheeraj Rajaram Reddy <dheeraj98reddy@gmail.com>
Date: Wed, 2 Jan 2019 19:23:57 +0530
Subject: [PATCH 0117/1095] Add take_while tests

---
 .../data/experimental/kernel_tests/BUILD      | 22 +++++
 .../kernel_tests/serialization/BUILD          | 18 ++++
 .../kernel_tests/take_while_test.py           | 94 +++++++++++++++++++
 3 files changed, 134 insertions(+)
 create mode 100644 tensorflow/python/data/experimental/kernel_tests/take_while_test.py

diff --git a/tensorflow/python/data/experimental/kernel_tests/BUILD b/tensorflow/python/data/experimental/kernel_tests/BUILD
index 9362a3e8eb..b4196685e2 100644
--- a/tensorflow/python/data/experimental/kernel_tests/BUILD
+++ b/tensorflow/python/data/experimental/kernel_tests/BUILD
@@ -638,6 +638,28 @@ py_library(
     ],
 )
 
+py_test(
+    name = "take_while_test",
+    size = "small",
+    srcs = ["take_while_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:errors",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:script_ops",
+        "//tensorflow/python:sparse_tensor",
+        "//tensorflow/python/data/experimental/ops:take_while_ops",
+        "//tensorflow/python/data/kernel_tests:test_base",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow/python/eager:context",
+        "//third_party/py/numpy",
+    ],
+)
+
 py_test(
     name = "tf_record_writer_test",
     size = "small",
diff --git a/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD b/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD
index 4a2e28f496..00b05e4d30 100644
--- a/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD
+++ b/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD
@@ -666,6 +666,24 @@ py_test(
     ],
 )
 
+py_test(
+    name = "take_while_dataset_serialization_test",
+    size = "small",
+    srcs = ["take_while_dataset_serialization_test.py"],
+    srcs_version = "PY2AND3",
+    tags = [
+        "no_oss",
+        "no_pip",
+        "no_windows",
+    ],
+    deps = [
+        ":dataset_serialization_test_base",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python/data/experimental/ops:take_while_ops",
+        "//tensorflow/python/data/ops:dataset_ops",
+    ],
+)
+
 py_test(
     name = "textline_dataset_serialization_test",
     size = "medium",
diff --git a/tensorflow/python/data/experimental/kernel_tests/take_while_test.py b/tensorflow/python/data/experimental/kernel_tests/take_while_test.py
new file mode 100644
index 0000000000..d561a3cd35
--- /dev/null
+++ b/tensorflow/python/data/experimental/kernel_tests/take_while_test.py
@@ -0,0 +1,94 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for `tf.data.experimental.Dataset.take_while()`."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.python.data.kernel_tests import test_base
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.data.experimental.ops import take_while_ops
+from tensorflow.python.framework import errors
+from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.framework import test_util
+from tensorflow.python.framework import constant_op
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import functional_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.platform import test
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class TakeWhileTest(test_base.DatasetTestBase):
+
+  def testTakeWhileDataset(self):
+    def do_test(num_elements, window_size):
+      def predicate_func(elem):
+        return array_ops.shape(elem)[0] > (window_size - 1)
+
+      flatten_func = lambda x: dataset_ops.Dataset.from_tensor_slices(x)
+      take_while = take_while_ops.take_while(predicate_func)
+
+      dataset = dataset_ops.Dataset.range(num_elements).batch(window_size)
+      dataset = dataset.apply(take_while).flat_map(flatten_func)
+      
+      self.assertDatasetProduces(dataset, 
+              np.arange(int(num_elements / window_size) * window_size))
+
+    do_test(14, 2)
+    do_test(15, 2)
+    do_test(100, 3)
+
+  def testTakeWhileDatasetRange(self):
+    def get_dataset(num_elemets, upper_bound):
+      return dataset_ops.Dataset.range(num_elemets).apply(
+              take_while_ops.take_while(lambda x: x < upper_bound))
+        
+    def do_test(num_elemets, upper_bound):  
+      self.assertDatasetProduces(get_dataset(num_elemets, upper_bound), 
+              np.arange(upper_bound))
+    
+    def out_of_bounds(num_elemets, upper_bound):
+      with self.assertRaises(errors.OutOfRangeError):
+        self.assertDatasetProduces(get_dataset(num_elemets, upper_bound), 
+                np.arange(upper_bound))
+
+    do_test(10, 2)
+    do_test(16, 7)
+    do_test(100, 99)
+    out_of_bounds(100, 101)
+    out_of_bounds(0, 1)
+
+  def testTakeWhileDatasetString(self):
+    def stringNotEquals(string):
+        return lambda x: math_ops.not_equal(x, constant_op.constant(string))
+
+    string = ["this", "is", "the", "test", "for", "strings"]
+    dataset = dataset_ops.Dataset.from_tensor_slices(string).apply(
+            take_while_ops.take_while(stringNotEquals("test")))
+    
+    next_element = self.getNext(dataset)
+    self.assertEqual(b"this", self.evaluate(next_element()))
+    self.assertEqual(b"is", self.evaluate(next_element()))
+    self.assertEqual(b"the", self.evaluate(next_element()))
+
+    with self.assertRaises(errors.OutOfRangeError):
+        self.assertEqual(b"test", self.evaluate(next_element()))
+
+
+if __name__ == "__main__":
+  test.main()
-- 
GitLab


From 7c9c3b1f28fbf8666e47fa51dec4d52b898c32e5 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 2 Jan 2019 08:44:47 -0800
Subject: [PATCH 0118/1095] Remove unused values from key-value sort HLOs

If a value has no user then there is no good reason to keep it in the
sort. Removing it will reduce memory traffic, memory usage and
potentially improve performance.

PiperOrigin-RevId: 227526906
---
 tensorflow/compiler/xla/service/BUILD         |  1 +
 .../xla/service/algebraic_simplifier.cc       | 77 +++++++++++++++++++
 .../xla/service/algebraic_simplifier_test.cc  | 68 ++++++++++++++++
 .../compiler/xla/service/pattern_matcher.h    |  2 +-
 4 files changed, 147 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index 201646e70d..0db1d4a78d 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -1579,6 +1579,7 @@ cc_library(
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/core:lib",
         "@com_google_absl//absl/algorithm:container",
+        "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/container:inlined_vector",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
index cad70a8d10..42a4291ce9 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
@@ -26,6 +26,7 @@ limitations under the License.
 #include <vector>
 
 #include "absl/algorithm/container.h"
+#include "absl/container/flat_hash_map.h"
 #include "absl/container/inlined_vector.h"
 #include "absl/memory/memory.h"
 #include "absl/strings/str_cat.h"
@@ -369,6 +370,11 @@ class AlgebraicSimplifierVisitor : public DfsHloVisitorWithDefault {
   // Tries to convert slice(reshape(X)) into reshape(slice(X))
   StatusOr<bool> TryToReorderSliceAndReshape(HloInstruction* slice);
 
+  // If the sort instruction has a tuple shape then looks for unused output
+  // values and removes them from the sort instruction. Returns true if the
+  // graph have been modified.
+  StatusOr<bool> RemoveUnusedOperandFromSort(HloInstruction* sort);
+
   // Current HloComputation instance the AlgebraicSimplifierVisitor is
   // traversing.
   HloComputation* computation_;
@@ -2814,6 +2820,69 @@ Status AlgebraicSimplifierVisitor::HandleSelect(HloInstruction* select) {
   return Status::OK();
 }
 
+StatusOr<bool> AlgebraicSimplifierVisitor::RemoveUnusedOperandFromSort(
+    HloInstruction* sort) {
+  if (!sort->shape().IsTuple()) {
+    return false;
+  }
+
+  if (sort->parent()->root_instruction() == sort) {
+    // Can't analyse users of the root instruction.
+    return false;
+  }
+
+  // Index 0 is the sorting key used by the sort HLO itself.
+  absl::flat_hash_set<int64> used_indices{0};
+  for (const HloInstruction* user : sort->users()) {
+    if (user->opcode() != HloOpcode::kGetTupleElement) {
+      // Can't analyse users other then get-tuple-element.
+      return false;
+    }
+    used_indices.insert(user->tuple_index());
+  }
+
+  if (used_indices.size() == sort->operand_count()) {
+    // All operands are used.
+    return false;
+  }
+
+  std::vector<HloInstruction*> operands{sort->mutable_operand(0)};
+  std::vector<Shape> new_shapes{sort->operand(0)->shape()};
+  for (int64 i = 1; i < sort->operand_count(); ++i) {
+    if (used_indices.count(i)) {
+      operands.push_back(sort->mutable_operand(i));
+      new_shapes.push_back(sort->operand(i)->shape());
+    }
+  }
+  Shape new_sort_shape = new_shapes.size() == 1
+                             ? new_shapes[0]
+                             : ShapeUtil::MakeTupleShape(new_shapes);
+  HloInstruction* new_sort = computation_->AddInstruction(
+      sort->CloneWithNewOperands(new_sort_shape, operands));
+
+  // Map from original get-tuple-element tuple index to new HLO instruction
+  absl::flat_hash_map<int64, HloInstruction*> result_map;
+  if (new_sort->shape().IsTuple()) {
+    // Old sort key maps to new sort key.
+    int64 new_index = 0;
+    for (int64 i = 0; i < sort->operand_count(); ++i) {
+      if (used_indices.count(i)) {
+        result_map[i] =
+            computation_->AddInstruction(HloInstruction::CreateGetTupleElement(
+                new_shapes[new_index], new_sort, new_index));
+        ++new_index;
+      }
+    }
+  } else {
+    result_map[0] = new_sort;
+  }
+  for (HloInstruction* user : sort->users()) {
+    TF_RETURN_IF_ERROR(
+        user->ReplaceAllUsesWith(result_map.at(user->tuple_index())));
+  }
+  return true;
+}
+
 Status AlgebraicSimplifierVisitor::HandleSort(HloInstruction* sort) {
   auto operand = sort->mutable_operand(0);
   int64 dimension_to_sort = sort->dimensions(0);
@@ -2826,6 +2895,14 @@ Status AlgebraicSimplifierVisitor::HandleSort(HloInstruction* sort) {
     return ReplaceWithNewInstruction(
         sort, HloInstruction::CreateTuple(sort->operands()));
   }
+
+  // Remove the unused values from a key-value sort.
+  TF_ASSIGN_OR_RETURN(bool removed_operand, RemoveUnusedOperandFromSort(sort));
+  if (removed_operand) {
+    changed_ = true;
+    return Status::OK();
+  }
+
   if (!options_.enable_permutation_sort_replacement()) {
     return Status::OK();
   }
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
index 51ad748ff8..211c5bf05a 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
@@ -2709,6 +2709,74 @@ TEST_F(AlgebraicSimplifierTest, DontReplacePermutationSortWrongDimensions) {
   EXPECT_FALSE(simplifier.Run(module.get()).ValueOrDie());
 }
 
+TEST_F(AlgebraicSimplifierTest, RemoveUnusedSortOperandArrayResult) {
+  const char* hlo_string = R"(
+   HloModule permutation_sort
+
+    ENTRY sort_computation {
+      keys = f32[64,8732]{1,0} parameter(0)
+      values = s32[64,8732]{1,0} parameter(1)
+      sort = (f32[64,8732]{1,0}, s32[64,8732]{1,0}) sort(keys, values),
+        dimensions={1}
+      ROOT gte = f32[64,8732]{1,0} get-tuple-element(sort), index=0
+    })";
+  TF_ASSERT_OK_AND_ASSIGN(auto module,
+                          ParseAndReturnVerifiedModule(hlo_string));
+
+  AlgebraicSimplifierOptions options(bitcasting_callback());
+  AlgebraicSimplifier simplifier(options);
+  EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
+  auto root = module->entry_computation()->root_instruction();
+  EXPECT_THAT(root, GmockMatch(m::Sort(m::Parameter(0))));
+}
+
+TEST_F(AlgebraicSimplifierTest, RemoveUnusedSortOperandTuple) {
+  const char* hlo_string = R"(
+   HloModule permutation_sort
+
+    ENTRY sort_computation {
+      keys = f32[64,87] parameter(0)
+      values.0 = s32[64,87] parameter(1)
+      values.1 = u32[64,87] parameter(2)
+      sort = (f32[64,87], s32[64,87], u32[64,87]) sort(
+          keys, values.0, values.1),
+        dimensions={1}
+      gte.0 = f32[64,87] get-tuple-element(sort), index=0
+      gte.1 = u32[64,87] get-tuple-element(sort), index=2
+      ROOT tuple = (f32[64,87], u32[64,87]) tuple(gte.0, gte.1)
+    })";
+  TF_ASSERT_OK_AND_ASSIGN(auto module,
+                          ParseAndReturnVerifiedModule(hlo_string));
+
+  AlgebraicSimplifierOptions options(bitcasting_callback());
+  AlgebraicSimplifier simplifier(options);
+  EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
+  auto root = module->entry_computation()->root_instruction();
+  EXPECT_THAT(
+      root,
+      GmockMatch(m::Tuple(
+          m::GetTupleElement(m::Sort(m::Parameter(0), m::Parameter(2)), 0),
+          m::GetTupleElement(m::Sort(m::Parameter(0), m::Parameter(2)), 1))));
+}
+
+TEST_F(AlgebraicSimplifierTest, DontRemoveUnusedSortKey) {
+  const char* hlo_string = R"(
+   HloModule permutation_sort
+
+    ENTRY sort_computation {
+      keys = f32[64,8732]{1,0} parameter(0)
+      values = s32[64,8732]{1,0} parameter(1)
+      sort = (f32[64,8732]{1,0}, s32[64,8732]{1,0}) sort(keys, values), dimensions={1}
+      ROOT gte = s32[64,8732]{1,0} get-tuple-element(sort), index=1
+    })";
+  TF_ASSERT_OK_AND_ASSIGN(auto module,
+                          ParseAndReturnVerifiedModule(hlo_string));
+
+  AlgebraicSimplifierOptions options(bitcasting_callback());
+  AlgebraicSimplifier simplifier(options);
+  EXPECT_FALSE(simplifier.Run(module.get()).ValueOrDie());
+}
+
 TEST_F(AlgebraicSimplifierTest, ReplaceEffectiveScalarKeyValueSortWithTuple) {
   auto builder = HloComputation::Builder(TestName());
 
diff --git a/tensorflow/compiler/xla/service/pattern_matcher.h b/tensorflow/compiler/xla/service/pattern_matcher.h
index c362a60d94..198c6e1069 100644
--- a/tensorflow/compiler/xla/service/pattern_matcher.h
+++ b/tensorflow/compiler/xla/service/pattern_matcher.h
@@ -2057,7 +2057,6 @@ XLA_UNOP_PATTERN(SendDone)
 XLA_UNOP_PATTERN(Sign)
 XLA_UNOP_PATTERN(Sin)
 XLA_UNOP_PATTERN(Slice)
-XLA_UNOP_PATTERN(Sort)
 XLA_UNOP_PATTERN(Tanh)
 XLA_UNOP_PATTERN(Transpose)
 #undef XLA_UNOP_PATTERN
@@ -2238,6 +2237,7 @@ XLA_VARIADIC_OP_PATTERN(Concatenate);
 XLA_VARIADIC_OP_PATTERN(CustomCall);
 XLA_VARIADIC_OP_PATTERN(Map)
 XLA_VARIADIC_OP_PATTERN(Reduce);
+XLA_VARIADIC_OP_PATTERN(Sort);
 XLA_VARIADIC_OP_PATTERN(Tuple);
 
 // Helpers for matching non-constant instructions.
-- 
GitLab


From 8fabc8091802e4edcd3051b8fba8fdf045388040 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 2 Jan 2019 08:57:10 -0800
Subject: [PATCH 0119/1095] minor refactor

PiperOrigin-RevId: 227528417
---
 tensorflow/python/grappler/cluster.py | 29 +++++++++------------------
 1 file changed, 10 insertions(+), 19 deletions(-)

diff --git a/tensorflow/python/grappler/cluster.py b/tensorflow/python/grappler/cluster.py
index 079d07115b..428b52402c 100644
--- a/tensorflow/python/grappler/cluster.py
+++ b/tensorflow/python/grappler/cluster.py
@@ -71,26 +71,21 @@ class Cluster(object):
     return self._tf_cluster
 
   def ListDevices(self):
-    """Returns the list of available hardware devices."""
-    devices = []
-    if self._tf_cluster is not None:
-      ret_from_swig = tf_cluster.TF_ListDevices(self._tf_cluster)
-      devices = []
-      for raw_dev in ret_from_swig:
-        devices.append(device_properties_pb2.NamedDevice.FromString(raw_dev))
-    return devices
+    """Returns a list of available hardware devices."""
+    if self._tf_cluster is None:
+      return []
+    return [device_properties_pb2.NamedDevice.FromString(device)
+            for device in tf_cluster.TF_ListDevices(self._tf_cluster)]
 
   def ListAvailableOps(self):
-    """Returns a list of all the available operations (sorted alphatically)."""
+    """Returns a list of all available operations (sorted alphabetically)."""
     return tf_cluster.TF_ListAvailableOps()
 
   def GetSupportedDevices(self, item):
     return tf_cluster.TF_GetSupportedDevices(self._tf_cluster, item.tf_item)
 
   def EstimatePerformance(self, device):
-    """Estimate the performance of the specified device."""
-    serialized = device.SerializeToString()
-    return tf_cluster.TF_EstimatePerformance(serialized)
+    return tf_cluster.TF_EstimatePerformance(device.SerializeToString())
 
   def MeasureCosts(self, item):
     """Returns the cost of running the specified item.
@@ -107,10 +102,8 @@ class Cluster(object):
       return None
 
     op_perf_bytes_list, run_time, step_stats_bytes = ret_from_swig
-    op_perfs = []
-    for op_perf_bytes in op_perf_bytes_list:
-      op_perfs.append(
-          op_performance_data_pb2.OpPerformance.FromString(op_perf_bytes))
+    op_perfs = [op_performance_data_pb2.OpPerformance.FromString(op_perf_bytes)
+                for op_perf_bytes in op_perf_bytes_list]
     return (op_perfs, run_time,
             step_stats_pb2.StepStats.FromString(step_stats_bytes))
 
@@ -122,11 +115,9 @@ class Cluster(object):
     Returns: A hashtable indexed by device name.
     """
     with errors.raise_exception_on_not_ok_status() as status:
-      ret_from_swig = tf_cluster.TF_DeterminePeakMemoryUsage(
+      return tf_cluster.TF_DeterminePeakMemoryUsage(
           item.tf_item, self._tf_cluster, status)
 
-    return ret_from_swig
-
 
 @contextlib.contextmanager
 def Provision(allow_soft_placement=True,
-- 
GitLab


From b978ad9b3c0327d406301f1b32f6d6d11688ec6e Mon Sep 17 00:00:00 2001
From: Alexey Radul <axch@google.com>
Date: Wed, 2 Jan 2019 08:59:37 -0800
Subject: [PATCH 0120/1095] Opportunistically teach test decorators in
 framework/test_util.py to return the decorated function's result, if any. 
 This makes the most sense for decorators that run the decorated function no
 more than once, since that makes it obvious which result to return.

This helps generalize them to sub-test or non-test scenarios where the
decorated function may return something valuable.

The change should be a no-op for tests, because they return None
anyway, and None was what the decorators were unconditionally
returning before this change.

PiperOrigin-RevId: 227528698
---
 tensorflow/python/framework/test_util.py | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py
index 7a6ecaf5a8..21d21cc7f4 100644
--- a/tensorflow/python/framework/test_util.py
+++ b/tensorflow/python/framework/test_util.py
@@ -373,7 +373,7 @@ def skip_if(condition):
       else:
         skip = condition
       if not skip:
-        fn(*args, **kwargs)
+        return fn(*args, **kwargs)
 
     return wrapper
 
@@ -410,7 +410,7 @@ def enable_control_flow_v2(fn):
     enable_control_flow_v2_old = control_flow_util.ENABLE_CONTROL_FLOW_V2
     control_flow_util.ENABLE_CONTROL_FLOW_V2 = True
     try:
-      fn(*args, **kwargs)
+      return fn(*args, **kwargs)
     finally:
       control_flow_util.ENABLE_CONTROL_FLOW_V2 = enable_control_flow_v2_old
 
@@ -594,9 +594,9 @@ def assert_no_new_tensors(f):
       ops.get_default_graph()._graph_key = outside_graph_key
       if outside_executed_eagerly:
         with context.eager_mode():
-          f(self, **kwargs)
+          result = f(self, **kwargs)
       else:
-        f(self, **kwargs)
+        result = f(self, **kwargs)
     # Make an effort to clear caches, which would otherwise look like leaked
     # Tensors.
     context.context()._clear_caches()  # pylint: disable=protected-access
@@ -610,6 +610,7 @@ def assert_no_new_tensors(f):
           len(tensors_after),
           str(tensors_after),
       )))
+    return result
 
   return decorator
 
@@ -741,7 +742,7 @@ def assert_no_garbage_created(f):
     gc.set_debug(gc.DEBUG_SAVEALL)
     gc.collect()
     previous_garbage = len(gc.garbage)
-    f(self, **kwargs)
+    result = f(self, **kwargs)
     gc.collect()
     new_garbage = len(gc.garbage)
     if new_garbage > previous_garbage:
@@ -786,6 +787,7 @@ def assert_no_garbage_created(f):
     # not hold on to every object in other tests.
     gc.set_debug(previous_debug_flags)
     gc.enable()
+    return result
 
   return decorator
 
@@ -1074,9 +1076,9 @@ def deprecated_graph_mode_only(func=None):
     def decorated(self, *args, **kwargs):
       if tf2.enabled():
         with context.graph_mode():
-          f(self, *args, **kwargs)
+          return f(self, *args, **kwargs)
       else:
-        f(self, *args, **kwargs)
+        return f(self, *args, **kwargs)
 
     return decorated
 
@@ -1126,7 +1128,7 @@ def run_v1_only(reason, func=None):
       if tf2.enabled():
         self.skipTest(reason)
 
-      f(self, *args, **kwargs)
+      return f(self, *args, **kwargs)
 
     return decorated
 
@@ -1163,7 +1165,7 @@ def run_v2_only(func=None):
       if not tf2.enabled():
         self.skipTest("Test is only comptaible in v2")
 
-      f(self, *args, **kwargs)
+      return f(self, *args, **kwargs)
 
     return decorated
 
@@ -1196,7 +1198,7 @@ def run_gpu_only(func=None):
       if not is_gpu_available():
         self.skipTest("Test requires GPU")
 
-      f(self, *args, **kwargs)
+      return f(self, *args, **kwargs)
 
     return decorated
 
@@ -1229,7 +1231,7 @@ def run_cuda_only(func=None):
       if not is_gpu_available(cuda_only=True):
         self.skipTest("Test requires CUDA GPU")
 
-      f(self, *args, **kwargs)
+      return f(self, *args, **kwargs)
 
     return decorated
 
-- 
GitLab


From e0068499474cfaa6940db5f80a1f82565520c048 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 2 Jan 2019 10:19:03 -0800
Subject: [PATCH 0121/1095] Standardize some names from `TFGAN` -> `TF-GAN`.

A noop.

PiperOrigin-RevId: 227540293
---
 RELEASE.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/RELEASE.md b/RELEASE.md
index 282430d123..0a56e69098 100644
--- a/RELEASE.md
+++ b/RELEASE.md
@@ -849,7 +849,7 @@ answered questions, and were part of inspiring discussions.
 * Remove `tf.contrib.data.Iterator.from_dataset()` method. Use
   `Dataset.make_initializable_iterator()` instead.
 * Remove seldom used and unnecessary `tf.contrib.data.Iterator.dispose_op()`.
-* Reorder some TFGAN loss functions in a non-backwards compatible way.
+* Reorder some TF-GAN loss functions in a non-backwards compatible way.
 
 ## Known Issues
 * In Python 3, `Dataset.from_generator()` does not support Unicode strings.
-- 
GitLab


From 55a21ee6b1ff9ef3ad8f00507650364897093a90 Mon Sep 17 00:00:00 2001
From: Michael Case <mikecase@google.com>
Date: Wed, 2 Jan 2019 10:20:37 -0800
Subject: [PATCH 0122/1095] Internal Change.

PiperOrigin-RevId: 227540535
---
 tensorflow/python/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 6724177d17..933ccf1c7b 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -79,6 +79,7 @@ py_library(
     srcs_version = "PY2AND3",
     visibility = [
         "//tensorflow:__pkg__",
+        "//tensorflow:internal",
         "//tensorflow/python/estimator:__subpackages__",
         "//tensorflow/python/keras:__subpackages__",
         "//tensorflow/python/tools:__pkg__",
-- 
GitLab


From 6cb28e4889aab7124ea792d1d013bc6ae90f28b5 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Wed, 2 Jan 2019 10:27:12 -0800
Subject: [PATCH 0123/1095] Move TPU Initialization out of the TPU Strategy
 constructor.

PiperOrigin-RevId: 227541646
---
 tensorflow/contrib/distribute/__init__.py     |  2 +
 tensorflow/contrib/distribute/python/BUILD    |  2 +-
 .../contrib/distribute/python/combinations.py | 18 ++++--
 .../contrib/distribute/python/tpu_strategy.py | 64 +++++++++----------
 tensorflow/python/training/session_manager.py |  8 ++-
 5 files changed, 55 insertions(+), 39 deletions(-)

diff --git a/tensorflow/contrib/distribute/__init__.py b/tensorflow/contrib/distribute/__init__.py
index 8ec73654e3..4c3f9b8f02 100644
--- a/tensorflow/contrib/distribute/__init__.py
+++ b/tensorflow/contrib/distribute/__init__.py
@@ -30,6 +30,7 @@ from tensorflow.contrib.distribute.python.monitor import Monitor
 from tensorflow.contrib.distribute.python.one_device_strategy import OneDeviceStrategy
 from tensorflow.contrib.distribute.python.parameter_server_strategy import ParameterServerStrategy
 from tensorflow.contrib.distribute.python.step_fn import *
+from tensorflow.contrib.distribute.python.tpu_strategy import initialize_tpu_system
 from tensorflow.contrib.distribute.python.tpu_strategy import TPUStrategy
 from tensorflow.python.distribute.cross_device_ops import *
 from tensorflow.python.distribute.distribute_config import DistributeConfig
@@ -58,6 +59,7 @@ _allowed_symbols = [
     'StandardSingleLossStep',
     'ReplicaContext',
     'TPUStrategy',
+    'initialize_tpu_system',
     'get_cross_replica_context',
     'get_distribution_strategy',
     'get_loss_reduction',
diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index f27224e46e..78ad3214fb 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -1,8 +1,8 @@
 # Implementation of a prototype TF distributed computation library.
 
+load("//tensorflow/compiler/tests:build_defs.bzl", "tf_xla_py_test")
 load("//tensorflow:tensorflow.bzl", "py_test")
 load("//tensorflow:tensorflow.bzl", "cuda_py_test")
-load("//tensorflow/compiler/tests:build_defs.bzl", "tf_xla_py_test")
 
 package(
     default_visibility = [
diff --git a/tensorflow/contrib/distribute/python/combinations.py b/tensorflow/contrib/distribute/python/combinations.py
index 4a934953ad..f6c4291659 100644
--- a/tensorflow/contrib/distribute/python/combinations.py
+++ b/tensorflow/contrib/distribute/python/combinations.py
@@ -46,7 +46,7 @@ import unittest
 from absl.testing import parameterized
 import six
 
-from tensorflow.contrib.cluster_resolver import TPUClusterResolver
+from tensorflow.contrib import cluster_resolver
 from tensorflow.contrib.distribute.python import mirrored_strategy as mirrored_lib
 from tensorflow.contrib.distribute.python import one_device_strategy as one_device_lib
 from tensorflow.contrib.distribute.python import tpu_strategy as tpu_lib
@@ -321,6 +321,15 @@ class NamedDistribution(object):
     return self._required_tpu
 
 
+def _get_tpu_strategy_creator(steps_per_run):
+  def _create_tpu_strategy():
+    resolver = cluster_resolver.TPUClusterResolver("")
+    tpu_lib.initialize_tpu_system(resolver)
+    strategy = tpu_lib.TPUStrategy(resolver, steps_per_run=steps_per_run)
+    return strategy
+  return _create_tpu_strategy
+
+
 # pylint: disable=g-long-lambda
 default_strategy = NamedDistribution(
     "Default",
@@ -330,13 +339,12 @@ one_device_strategy = NamedDistribution(
     "OneDeviceCPU", lambda: one_device_lib.OneDeviceStrategy("/cpu:0"),
     required_gpus=None)
 tpu_strategy = NamedDistribution(
-    "TPU", lambda: tpu_lib.TPUStrategy(
-        TPUClusterResolver(""), steps_per_run=2),
+    "TPU", _get_tpu_strategy_creator(steps_per_run=2),
     required_tpu=True)
 tpu_strategy_one_step = NamedDistribution(
-    "TPUOneStep", lambda: tpu_lib.TPUStrategy(
-        TPUClusterResolver(""), steps_per_run=1),
+    "TPUOneStep", _get_tpu_strategy_creator(steps_per_run=1),
     required_tpu=True)
+
 mirrored_strategy_with_one_cpu = NamedDistribution(
     "Mirrored1CPU",
     lambda: mirrored_lib.MirroredStrategy(["/cpu:0"]))
diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index 10b7ef0407..9cb2e461ab 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -51,6 +51,29 @@ from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.util import nest
 
 
+_TPU_INITIALIZE_SYSTEM_COLLECTION = "TPU_STRATEGY_INITIALIZE"
+
+
+def initialize_tpu_system(cluster_resolver=None):
+  """Initialize the TPU devices in a separate session and graph.
+
+  Args:
+    cluster_resolver: A tf.contrib.cluster_resolver.TPUClusterResolver,
+        which provides information about the TPU cluster.
+  """
+  if cluster_resolver is None:
+    cluster_resolver = resolver_lib.TPUClusterResolver("")
+  master = cluster_resolver.master()
+
+  logging.info("Initializing the TPU system.")
+  session_config = config_pb2.ConfigProto(allow_soft_placement=True)
+
+  with ops.Graph().as_default():
+    with session_lib.Session(config=session_config, target=master) as sess:
+      sess.run([tpu.initialize_system()])
+  logging.info("Finished initializing TPU system.")
+
+
 def get_tpu_system_metadata(tpu_cluster_resolver):
   """Retrieves TPU system metadata given a TPUClusterResolver."""
   master = tpu_cluster_resolver.master()
@@ -164,11 +187,6 @@ class TPUStrategy(distribute_lib.DistributionStrategy):
 class TPUExtended(distribute_lib.DistributionStrategyExtended):
   """Implementation of TPUStrategy."""
 
-  # Track what TPU devices have been initialized. This is *intentionally*
-  # shared across all instances of TPUExtended as we want to keep track of which
-  # devices are initialized globally.
-  _initialized_devices = []
-
   def __init__(self,
                container_strategy,
                tpu_cluster_resolver=None,
@@ -213,32 +231,6 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
     self.steps_per_run = steps_per_run
     self._require_static_shapes = True
 
-    # Initialize the TPU devices.
-    self._initialize_tpu()
-
-  def _initialize_tpu(self):
-    """Initialize the TPU devices in a separate session and graph.
-
-    We keep track of all the TPU devices that we're initialized as we should
-    only be running TPU initialize once for the entire process.
-    """
-    master = self._tpu_cluster_resolver.master()
-    # Verify TPU has not already been initialized in this process.
-    if master in TPUExtended._initialized_devices:
-      logging.info("TPU master %s has already been initialized." % master)
-      return
-
-    logging.info("Initializing the TPU system.")
-    session_config = config_pb2.ConfigProto(allow_soft_placement=True)
-    self._configure(session_config)
-    with ops.Graph().as_default():
-      with session_lib.Session(config=session_config, target=master) as sess:
-        sess.run([tpu.initialize_system()])
-    logging.info("Finized initializing TPU system.")
-
-    # Update Strategy state to make sure we can track device initialization.
-    TPUExtended._initialized_devices.append(master)
-
   def _validate_colocate_with_variable(self, colocate_with_variable):
     values.validate_colocate_tpu_variable(colocate_with_variable, self)
 
@@ -468,7 +460,15 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
       # TODO(priyag): Add appopriate call here when eager is supported for TPUs.
       raise NotImplementedError("Eager mode not supported in TPUStrategy.")
     else:
-      return []
+      # TODO(jhseu): We need this hack because DistributionStrategies must be
+      # pickleable for copy.deepcopy(). Remove when initialize_system goes away.
+      graph = ops.get_default_graph()
+      tpu_init = graph.get_collection(_TPU_INITIALIZE_SYSTEM_COLLECTION)
+      if tpu_init:
+        return tpu_init
+      graph.add_to_collection(_TPU_INITIALIZE_SYSTEM_COLLECTION,
+                              tpu.initialize_system())
+      return graph.get_collection(_TPU_INITIALIZE_SYSTEM_COLLECTION)
 
   def _finalize(self):
     if context.executing_eagerly():
diff --git a/tensorflow/python/training/session_manager.py b/tensorflow/python/training/session_manager.py
index 0f68fcfe8b..1ec9639177 100644
--- a/tensorflow/python/training/session_manager.py
+++ b/tensorflow/python/training/session_manager.py
@@ -25,6 +25,7 @@ from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.training import checkpoint_management
+from tensorflow.python.training import distribution_strategy_context
 from tensorflow.python.util.tf_export import tf_export
 
 
@@ -182,7 +183,12 @@ class SessionManager(object):
     """
     self._target = master
     sess = session.Session(self._target, graph=self._graph, config=config)
-
+    # TODO(jhseu): Delete once tpu.initialize_system() goes away.
+    initialize_ops = (
+        distribution_strategy_context.get_distribution_strategy().initialize()
+    )
+    if initialize_ops:
+      sess.run(initialize_ops)
     if checkpoint_dir and checkpoint_filename_with_path:
       raise ValueError("Can not provide both checkpoint_dir and "
                        "checkpoint_filename_with_path.")
-- 
GitLab


From 25ee17bafb33f0f72bea84c1166e20d75955f273 Mon Sep 17 00:00:00 2001
From: Jian Li <jianlijianli@google.com>
Date: Wed, 2 Jan 2019 10:43:05 -0800
Subject: [PATCH 0124/1095] Fix a mismatch in fixed point softmax.

PiperOrigin-RevId: 227544341
---
 tensorflow/lite/kernels/activations.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/lite/kernels/activations.cc b/tensorflow/lite/kernels/activations.cc
index 13ef94c016..4463a6c5a6 100644
--- a/tensorflow/lite/kernels/activations.cc
+++ b/tensorflow/lite/kernels/activations.cc
@@ -57,9 +57,9 @@ struct PreluOpData : public OpData {
 };
 
 namespace {
-TfLiteStatus CheckInputQuantParams(TfLiteContext* context,
-                                   const TfLiteTensor* input,
-                                   const TfLiteTensor* output) {
+TfLiteStatus CheckOutputQuantParams(TfLiteContext* context,
+                                    const TfLiteTensor* input,
+                                    const TfLiteTensor* output) {
   if (input->type == kTfLiteUInt8) {
     TF_LITE_ENSURE_EQ(context, output->params.zero_point, 0);
     TF_LITE_ENSURE(context, output->params.scale == 1. / 256);
@@ -236,7 +236,7 @@ TfLiteStatus SoftmaxPrepare(TfLiteContext* context, TfLiteNode* node) {
   TF_LITE_ENSURE(context, num_dims >= 1 && num_dims <= 4);
 
   if (input->type == kTfLiteUInt8 || input->type == kTfLiteInt8) {
-    if (CheckInputQuantParams(context, input, output) == kTfLiteError) {
+    if (CheckOutputQuantParams(context, input, output) == kTfLiteError) {
       return kTfLiteError;
     }
 
-- 
GitLab


From d7e56c9b27d0e712bd268e1c06d57beb481083e0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 2 Jan 2019 10:54:39 -0800
Subject: [PATCH 0125/1095] Fixes bug in tf.print that prevents it from
 printing tensors of strings.

PiperOrigin-RevId: 227546311
---
 tensorflow/core/framework/tensor.cc           | 24 ++++++++++++-------
 tensorflow/core/framework/tensor_test.cc      | 15 ++++++++----
 .../python/kernel_tests/logging_ops_test.py   | 13 ++++++++++
 tensorflow/python/ops/logging_ops.py          |  2 +-
 4 files changed, 40 insertions(+), 14 deletions(-)

diff --git a/tensorflow/core/framework/tensor.cc b/tensorflow/core/framework/tensor.cc
index 7e841489eb..0c96ec8168 100644
--- a/tensorflow/core/framework/tensor.cc
+++ b/tensorflow/core/framework/tensor.cc
@@ -932,10 +932,18 @@ namespace {
 // logic is so simple we can just replicate it here, where it is close to its
 // usage and easy to change later. And there's the extra benefit of not
 // accessing an 'internal' namespace.
-inline const strings::AlphaNum& PrintOneElement(const strings::AlphaNum& a) {
+inline const strings::AlphaNum& PrintOneElement(const strings::AlphaNum& a,
+                                                bool print_v2) {
   return a;
 }
-inline float PrintOneElement(const Eigen::half& h) {
+inline string PrintOneElement(const string& a, bool print_v2) {
+  if (print_v2) {
+    return "\"" + str_util::CEscape(a) + "\"";
+  } else {
+    return str_util::CEscape(a);
+  }
+}
+inline float PrintOneElement(const Eigen::half& h, bool print_v2) {
   return static_cast<float>(h);
 }
 
@@ -957,7 +965,7 @@ void PrintOneDim(int dim_index, const gtl::InlinedVector<int64, 4>& shape,
         return;
       }
       if (i > 0) strings::StrAppend(result, " ");
-      strings::StrAppend(result, PrintOneElement(data[(*data_index)++]));
+      strings::StrAppend(result, PrintOneElement(data[(*data_index)++], false));
     }
     return;
   }
@@ -1000,7 +1008,7 @@ void PrintOneDimV2(int dim_index, const gtl::InlinedVector<int64, 4>& shape,
   // We have recursed beyond all the dimensions into a single element
   // of the tensor.
   if (dim_index == num_dims) {
-    strings::StrAppend(result, PrintOneElement(data[data_index]));
+    strings::StrAppend(result, PrintOneElement(data[data_index], true));
     return;
   }
 
@@ -1048,7 +1056,7 @@ string SummarizeArray(int64 limit, int64 num_elts,
   if (shape.empty()) {
     for (int64 i = 0; i < limit; ++i) {
       if (i > 0) strings::StrAppend(&ret, " ");
-      strings::StrAppend(&ret, PrintOneElement(array[i]));
+      strings::StrAppend(&ret, PrintOneElement(array[i], print_v2));
     }
     if (num_elts > limit) strings::StrAppend(&ret, "...");
     return ret;
@@ -1123,6 +1131,9 @@ string Tensor::SummarizeValue(int64 max_entries, bool print_v2) const {
       // will emit "1 0..." which is more compact.
       return SummarizeArray<bool>(limit, num_elts, shape_, data, print_v2);
       break;
+    case DT_STRING:
+      return SummarizeArray<string>(limit, num_elts, shape_, data, print_v2);
+      break;
     default: {
       // All irregular cases
       string ret;
@@ -1134,9 +1145,6 @@ string Tensor::SummarizeValue(int64 max_entries, bool print_v2) const {
       for (size_t i = 0; i < limit; ++i) {
         if (i > 0) strings::StrAppend(&ret, " ");
         switch (dtype()) {
-          case DT_STRING:
-            strings::StrAppend(&ret, str_util::CEscape(flat<string>()(i)));
-            break;
           case DT_VARIANT: {
             const Variant& v = flat<Variant>()(i);
             strings::StrAppend(&ret, v.DebugString());
diff --git a/tensorflow/core/framework/tensor_test.cc b/tensorflow/core/framework/tensor_test.cc
index 713f91fe04..d4aed38761 100644
--- a/tensorflow/core/framework/tensor_test.cc
+++ b/tensorflow/core/framework/tensor_test.cc
@@ -1370,7 +1370,7 @@ TEST(SummarizeValue, STRING) {
   EXPECT_EQ("one two three four five", x.SummarizeValue(16));
   x = MkTensor<string>(DT_STRING, TensorShape({5, 1, 5}),
                        {"one", "two", "three", "four", "five"});
-  EXPECT_EQ("one two three four five one...", x.SummarizeValue(6));
+  EXPECT_EQ("[[one two three four five]][[one...]]...", x.SummarizeValue(6));
 }
 
 TEST(SummarizeValue, INT32_PRINT_V2) {
@@ -1423,11 +1423,16 @@ TEST(SummarizeValue, BOOL_PRINT_V2) {
 TEST(SummarizeValue, STRING_PRINT_V2) {
   Tensor x = MkTensor<string>(DT_STRING, TensorShape({5}),
                               {"one", "two", "three", "four", "five"});
-  EXPECT_EQ("[one two three four five]", x.SummarizeValue(16, true));
-  EXPECT_EQ("[one two three four five]", x.SummarizeValue(-1, true));
-  x = MkTensor<string>(DT_STRING, TensorShape({5, 1, 5}),
+  EXPECT_EQ("[\"one\" \"two\" \"three\" \"four\" \"five\"]",
+            x.SummarizeValue(16, true));
+  EXPECT_EQ("[\"one\" \"two\" \"three\" \"four\" \"five\"]",
+            x.SummarizeValue(-1, true));
+  EXPECT_EQ("[\"one\" \"two\" ... \"four\" \"five\"]",
+            x.SummarizeValue(2, true));
+  x = MkTensor<string>(DT_STRING, TensorShape({2, 2}),
                        {"one", "two", "three", "four", "five"});
-  EXPECT_EQ("[one two three four five one...]", x.SummarizeValue(6, true));
+  EXPECT_EQ("[[\"one\" \"two\"]\n [\"three\" \"four\"]]",
+            x.SummarizeValue(16, true));
 }
 
 void BM_CreateAndDestroy(int iters) {
diff --git a/tensorflow/python/kernel_tests/logging_ops_test.py b/tensorflow/python/kernel_tests/logging_ops_test.py
index 85035e5f7d..3896b138c9 100644
--- a/tensorflow/python/kernel_tests/logging_ops_test.py
+++ b/tensorflow/python/kernel_tests/logging_ops_test.py
@@ -19,6 +19,7 @@ from __future__ import division
 from __future__ import print_function
 
 import os
+import string
 import sys
 import tempfile
 
@@ -37,6 +38,7 @@ from tensorflow.python.ops import string_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
 
+
 class LoggingOpsTest(test.TestCase):
 
   @test_util.run_deprecated_v1
@@ -80,6 +82,17 @@ class PrintV2Test(test.TestCase):
       expected = "[0 1 2 ... 7 8 9]"
       self.assertTrue((expected + "\n") in printed.contents())
 
+  @test_util.run_in_graph_and_eager_modes()
+  def testPrintOneStringTensor(self):
+    with self.cached_session():
+      tensor = ops.convert_to_tensor([char for char in string.ascii_lowercase])
+      with self.captureWritesToStream(sys.stderr) as printed:
+        print_op = logging_ops.print_v2(tensor)
+        self.evaluate(print_op)
+
+      expected = "[\"a\" \"b\" \"c\" ... \"x\" \"y\" \"z\"]"
+      self.assertIn((expected + "\n"), printed.contents())
+
   @test_util.run_in_graph_and_eager_modes()
   def testPrintOneTensorVarySummarize(self):
     with self.cached_session():
diff --git a/tensorflow/python/ops/logging_ops.py b/tensorflow/python/ops/logging_ops.py
index 5a948a2194..3cb16eb81e 100644
--- a/tensorflow/python/ops/logging_ops.py
+++ b/tensorflow/python/ops/logging_ops.py
@@ -263,7 +263,7 @@ def print_v2(*inputs, **kwargs):
   # If we are only printing a single string scalar, there is no need to format
   if (len(inputs) == 1 and tensor_util.is_tensor(inputs[0])
       and (not isinstance(inputs[0], sparse_tensor.SparseTensor))
-      and inputs[0].shape and (inputs[0].dtype == dtypes.string)):
+      and (inputs[0].shape.ndims == 0)and (inputs[0].dtype == dtypes.string)):
     formatted_string = inputs[0]
   # Otherwise, we construct an appropriate template for the tensors we are
   # printing, and format the template using those tensors.
-- 
GitLab


From e859bba35844942f1c60e3ea9f3d82bd574ed7e4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 2 Jan 2019 11:10:49 -0800
Subject: [PATCH 0126/1095] Fix example in documentation for sparse_split.

PiperOrigin-RevId: 227549459
---
 tensorflow/python/ops/sparse_ops.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/sparse_ops.py b/tensorflow/python/ops/sparse_ops.py
index f6265f5e6a..a149d98730 100644
--- a/tensorflow/python/ops/sparse_ops.py
+++ b/tensorflow/python/ops/sparse_ops.py
@@ -806,8 +806,8 @@ def sparse_split(keyword_required=KeywordRequired(),
   Graphically the output tensors are:
 
       output_tensor[0] =
-      [    a ]
-      [b c   ]
+      [    a   ]
+      [b c     ]
 
       output_tensor[1] =
       [ d e  ]
-- 
GitLab


From af42cb228793f9a1d3f28f91bdb5334924b4a504 Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Wed, 2 Jan 2019 11:10:57 -0800
Subject: [PATCH 0127/1095] Enable Eigen custom contraction kernels support.

PiperOrigin-RevId: 227549475
---
 tensorflow/contrib/rnn/BUILD               | 11 +++++++++--
 tensorflow/contrib/rnn/kernels/blas_gemm.h |  4 ++++
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/rnn/BUILD b/tensorflow/contrib/rnn/BUILD
index 44b232e0f2..39b6885968 100644
--- a/tensorflow/contrib/rnn/BUILD
+++ b/tensorflow/contrib/rnn/BUILD
@@ -227,7 +227,10 @@ tf_custom_op_library(
         "kernels/lstm_ops_gpu.cu.cc",
         "kernels/lstm_ops.h",
     ],
-    deps = ["//tensorflow/core/kernels:eigen_helpers"],
+    deps = [
+        "//tensorflow/core/kernels:eigen_contraction_kernel",
+        "//tensorflow/core/kernels:eigen_helpers",
+    ],
 )
 
 tf_gen_op_wrapper_py(
@@ -249,7 +252,10 @@ tf_custom_op_library(
         "kernels/gru_ops_gpu.cu.cc",
         "kernels/gru_ops.h",
     ],
-    deps = ["//tensorflow/core/kernels:eigen_helpers"],
+    deps = [
+        "//tensorflow/core/kernels:eigen_contraction_kernel",
+        "//tensorflow/core/kernels:eigen_helpers",
+    ],
 )
 
 tf_gen_op_wrapper_py(
@@ -346,6 +352,7 @@ tf_kernel_library(
     deps = [
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
+        "//tensorflow/core/kernels:eigen_contraction_kernel",
         "//tensorflow/core/kernels:eigen_helpers",
         "//third_party/eigen3",
     ],
diff --git a/tensorflow/contrib/rnn/kernels/blas_gemm.h b/tensorflow/contrib/rnn/kernels/blas_gemm.h
index d37210d4b8..12f3182a6a 100644
--- a/tensorflow/contrib/rnn/kernels/blas_gemm.h
+++ b/tensorflow/contrib/rnn/kernels/blas_gemm.h
@@ -21,6 +21,10 @@ limitations under the License.
 #include "tensorflow/core/kernels/eigen_activations.h"
 #include "tensorflow/core/platform/types.h"
 
+#if defined(TENSORFLOW_USE_CUSTOM_CONTRACTION_KERNEL)
+#include "tensorflow/core/kernels/eigen_contraction_kernel.h"
+#endif
+
 namespace tensorflow {
 class OpKernelContext;
 namespace functor {
-- 
GitLab


From 3f4249ef9c9b0003c6314e6a3449337c7446b110 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 2 Jan 2019 11:24:55 -0800
Subject: [PATCH 0128/1095] Internal change.

PiperOrigin-RevId: 227551788
---
 tensorflow/compiler/xla/client/lib/BUILD | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/client/lib/BUILD b/tensorflow/compiler/xla/client/lib/BUILD
index 97fb4b9e0e..966a581981 100644
--- a/tensorflow/compiler/xla/client/lib/BUILD
+++ b/tensorflow/compiler/xla/client/lib/BUILD
@@ -352,7 +352,11 @@ cc_library(
 xla_test(
     name = "quantize_test",
     srcs = ["quantize_test.cc"],
-    tags = ["enable_for_xla_interpreter"],
+    # TODO(b/122119490): re-enable TAP after fixing.
+    tags = [
+        "enable_for_xla_interpreter",
+        "notap",
+    ],
     deps = [
         ":quantize",
         "//tensorflow/compiler/xla:test",
-- 
GitLab


From eadd684c5b9752c43a26a5ed1663f195ac126077 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Wed, 2 Jan 2019 11:27:06 -0800
Subject: [PATCH 0129/1095] Add root mean squared error v2 metric.

PiperOrigin-RevId: 227552168
---
 tensorflow/python/keras/metrics.py      | 47 +++++++++++++++++++++++++
 tensorflow/python/keras/metrics_test.py | 35 ++++++++++++++++++
 2 files changed, 82 insertions(+)

diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index 1e0359d9e5..d9842f7810 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -1491,6 +1491,53 @@ class CategoricalHinge(MeanMetricWrapper):
     return super(CategoricalHinge, cls).from_config(config)
 
 
+class RootMeanSquaredError(Mean):
+  """Computes root mean squared error metric between `y_true` and `y_pred`.
+
+  Usage:
+
+  ```python
+  m = tf.keras.metrics.RootMeanSquaredError()
+  m.update_state([2., 4., 6.], [1., 3., 2.])
+  print('Final result: ', m.result().numpy())  # Final result: 2.449
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile('sgd', metrics=[tf.keras.metrics.RootMeanSquaredError()])
+  ```
+  """
+
+  def __init__(self, name='root_mean_squared_error', dtype=None):
+    super(RootMeanSquaredError, self).__init__(name, dtype=dtype)
+
+  def update_state(self, y_true, y_pred, sample_weight=None):
+    """Accumulates root mean squared error statistics.
+
+    Args:
+      y_true: The ground truth values.
+      y_pred: The predicted values.
+      sample_weight: Optional weighting of each example. Defaults to 1. Can be a
+        `Tensor` whose rank is either 0, or the same rank as `y_true`, and must
+        be broadcastable to `y_true`.
+
+    Returns:
+      Update op.
+    """
+    y_true = math_ops.cast(y_true, self._dtype)
+    y_pred = math_ops.cast(y_pred, self._dtype)
+    y_pred, y_true, sample_weight = squeeze_or_expand_dimensions(
+        y_pred, y_true, sample_weight)
+    error_sq = math_ops.square(y_pred - y_true)
+    return super(RootMeanSquaredError, self).update_state(
+        error_sq, sample_weight=sample_weight)
+
+  def result(self):
+    return math_ops.sqrt(math_ops.div_no_nan(self.total, self.count))
+
+
 def accuracy(y_true, y_pred):
   y_pred.get_shape().assert_is_compatible_with(y_true.get_shape())
   if y_true.dtype != y_pred.dtype:
diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py
index b4eb314e40..2bce89fd1a 100644
--- a/tensorflow/python/keras/metrics_test.py
+++ b/tensorflow/python/keras/metrics_test.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import math
 import os
 from absl.testing import parameterized
 import numpy as np
@@ -1249,6 +1250,40 @@ class CategoricalHingeTest(test.TestCase):
     self.assertAllClose(0.5, self.evaluate(result), atol=1e-5)
 
 
+@test_util.run_all_in_graph_and_eager_modes
+class RootMeanSquaredErrorTest(test.TestCase):
+
+  def test_config(self):
+    rmse_obj = metrics.RootMeanSquaredError(name='rmse', dtype=dtypes.int32)
+    self.assertEqual(rmse_obj.name, 'rmse')
+    self.assertEqual(rmse_obj._dtype, dtypes.int32)
+
+    rmse_obj2 = metrics.RootMeanSquaredError.from_config(rmse_obj.get_config())
+    self.assertEqual(rmse_obj2.name, 'rmse')
+    self.assertEqual(rmse_obj2._dtype, dtypes.int32)
+
+  def test_unweighted(self):
+    rmse_obj = metrics.RootMeanSquaredError()
+    self.evaluate(variables.variables_initializer(rmse_obj.variables))
+    y_true = constant_op.constant((2, 4, 6))
+    y_pred = constant_op.constant((1, 3, 2))
+
+    update_op = rmse_obj.update_state(y_true, y_pred)
+    self.evaluate(update_op)
+    result = rmse_obj.result()
+    # error = [-1, -1, -4], square(error) = [1, 1, 16], mean = 18/3 = 6
+    self.assertAllClose(math.sqrt(6), result, atol=1e-3)
+
+  def test_weighted(self):
+    rmse_obj = metrics.RootMeanSquaredError()
+    self.evaluate(variables.variables_initializer(rmse_obj.variables))
+    y_true = constant_op.constant((2, 4, 6, 8))
+    y_pred = constant_op.constant((1, 3, 2, 3))
+    sample_weight = constant_op.constant((0, 1, 0, 1))
+    result = rmse_obj(y_true, y_pred, sample_weight=sample_weight)
+    self.assertAllClose(math.sqrt(13), self.evaluate(result), atol=1e-3)
+
+
 def _get_model(compile_metrics):
   model_layers = [
       layers.Dense(3, activation='relu', kernel_initializer='ones'),
-- 
GitLab


From 3da41f3a4d0bcbbd96295b3afe16bdda2043a767 Mon Sep 17 00:00:00 2001
From: Michael Kuperstein <mkuper@google.com>
Date: Wed, 2 Jan 2019 11:27:09 -0800
Subject: [PATCH 0130/1095] [XLA] Implement scalar index form of DS/DUS on GPU
 and CPU

This form is not yet documented, unsupported by HLO passes, and only
passes verification under a flag.

PiperOrigin-RevId: 227552176
---
 .../xla/service/elemental_ir_emitter.cc       | 31 +++++++++++---
 .../llvm_ir/dynamic_update_slice_util.cc      | 41 +++++++++++++++----
 2 files changed, 58 insertions(+), 14 deletions(-)

diff --git a/tensorflow/compiler/xla/service/elemental_ir_emitter.cc b/tensorflow/compiler/xla/service/elemental_ir_emitter.cc
index 6f928fcbaa..f84c115e0a 100644
--- a/tensorflow/compiler/xla/service/elemental_ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/elemental_ir_emitter.cc
@@ -1758,9 +1758,18 @@ StatusOr<llvm::Value*> ElementalIrEmitter::EmitElementalDynamicSlice(
     auto index_typed_const = [&](uint64 c) -> llvm::Constant* {
       return llvm::ConstantInt::get(index_type, c);
     };
-    llvm_ir::IrArray::Index dim_index(1, index_typed_const(i));
-    TF_ASSIGN_OR_RETURN(llvm::Value * start_index_value,
-                        operand_to_generator.at(hlo->operand(1))(dim_index));
+    // TODO(b/118437727): Remove the R1 path.
+    llvm::Value* start_index_value;
+    if (hlo->operand(1)->shape().rank() == 1) {
+      llvm_ir::IrArray::Index dim_index(1, index_typed_const(i));
+      TF_ASSIGN_OR_RETURN(start_index_value,
+                          operand_to_generator.at(hlo->operand(1))(dim_index));
+    } else {
+      llvm_ir::IrArray::Index zero_index(index_type);
+      TF_ASSIGN_OR_RETURN(
+          start_index_value,
+          operand_to_generator.at(hlo->operand(1 + i))(zero_index));
+    }
 
     // Clamp the start index so that the sliced portion fits in the operand:
     // start_index = clamp(start_index, 0, operand_dim_size - output_dim_size)
@@ -1905,9 +1914,19 @@ StatusOr<llvm::Value*> ElementalIrEmitter::EmitElementalDynamicUpdateSlice(
     auto index_typed_const = [&](uint64 c) -> llvm::Constant* {
       return llvm::ConstantInt::get(index_type, c);
     };
-    llvm_ir::IrArray::Index dim_index(1, index_typed_const(i));
-    TF_ASSIGN_OR_RETURN(llvm::Value * start_index_value,
-                        operand_to_generator.at(start_hlo)(dim_index));
+
+    llvm::Value* start_index_value;
+    // TODO(b/118437727): Remove the R1 path.
+    if (hlo->operand(2)->shape().rank() == 1) {
+      llvm_ir::IrArray::Index dim_index(1, index_typed_const(i));
+      TF_ASSIGN_OR_RETURN(start_index_value,
+                          operand_to_generator.at(hlo->operand(2))(dim_index));
+    } else {
+      llvm_ir::IrArray::Index zero_index(index_type);
+      TF_ASSIGN_OR_RETURN(
+          start_index_value,
+          operand_to_generator.at(hlo->operand(2 + i))(zero_index));
+    }
 
     // Clamp the start index so that the update region fits in the operand.
     // start_index = clamp(start_index, 0, input_dim_size - update_dim_size)
diff --git a/tensorflow/compiler/xla/service/llvm_ir/dynamic_update_slice_util.cc b/tensorflow/compiler/xla/service/llvm_ir/dynamic_update_slice_util.cc
index 1da7794532..c66eaec8fb 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/dynamic_update_slice_util.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/dynamic_update_slice_util.cc
@@ -36,8 +36,10 @@ bool CanUpdateDynamicSliceInPlace(HloInstruction* dynamic_update_slice,
 // EmitFusedDynamicUpdateSliceInPlace.
 //
 // Emits a sequential loop if launch_dimensions is null.
+using IndexGenerator = std::function<StatusOr<llvm::Value*>(int64)>;
+
 static Status EmitDynamicUpdateSliceInPlaceImpl(
-    const Shape& update_shape, const ElementGenerator& start_indices_generator,
+    const Shape& update_shape, const IndexGenerator& start_indices_generator,
     bool is_signed, ElementGenerator update_array_generator,
     const IrArray& output_array, const gpu::LaunchDimensions* launch_dimensions,
     absl::string_view name, llvm::IRBuilder<>* b) {
@@ -47,8 +49,7 @@ static Status EmitDynamicUpdateSliceInPlaceImpl(
   const int64 rank = output_shape.rank();
   IrArray::Index start_index(b->getInt64Ty(), rank);
   for (int64 i = 0; i < rank; ++i) {
-    IrArray::Index dim_index({b->getInt64(i)});
-    TF_ASSIGN_OR_RETURN(start_index[i], start_indices_generator(dim_index));
+    TF_ASSIGN_OR_RETURN(start_index[i], start_indices_generator(i));
     llvm::Value* output_dim_size = llvm::ConstantInt::get(
         start_index[i]->getType(), output_shape.dimensions(i));
     llvm::Value* update_dim_size = llvm::ConstantInt::get(
@@ -112,9 +113,20 @@ Status EmitDynamicUpdateSliceInPlace(absl::Span<const IrArray> operand_arrays,
   Shape output_shape = output_array.GetShape();
   Shape update_shape = update_array.GetShape();
 
-  ElementGenerator start_indices_generator = [&](const IrArray::Index& index) {
-    return start_indices_array.EmitReadArrayElement(index, b);
-  };
+  IndexGenerator start_indices_generator;
+  // TODO(b/118437727): Remove the R1 path, and rename the variables.
+  if (start_indices_array.GetShape().rank() == 1) {
+    start_indices_generator = [&](int64 index) {
+      return start_indices_array.EmitReadArrayElement(
+          IrArray::Index({b->getInt64(index)}), b);
+    };
+  } else {
+    start_indices_generator = [&](int64 index) {
+      return operand_arrays[2 + index].EmitReadArrayElement(
+          IrArray::Index(b->getInt64Ty()), b);
+    };
+  }
+
   ElementGenerator update_array_generator = [&](const IrArray::Index& index) {
     return update_array.EmitReadArrayElement(index, b);
   };
@@ -165,8 +177,21 @@ static Status EmitFusedDynamicUpdateSliceInPlaceImpl(
                                elemental_emitter);
   TF_RETURN_IF_ERROR(dynamic_update_slice->Accept(&fused_emitter));
   ElementGenerator update_array_generator = fused_emitter.GetGenerator(update);
-  ElementGenerator start_indices_generator =
-      fused_emitter.GetGenerator(start_indices);
+
+  // TODO(b/118437727): Remove the R1 path, and rename the variables.
+  IndexGenerator start_indices_generator;
+  if (start_indices->shape().rank() == 1) {
+    start_indices_generator = [&](int64 index) {
+      return fused_emitter.GetGenerator(start_indices)(
+          IrArray::Index({b->getInt64(index)}));
+    };
+  } else {
+    start_indices_generator = [&](int64 index) {
+      ElementGenerator element_generator =
+          fused_emitter.GetGenerator(dynamic_update_slice->operand(2 + index));
+      return element_generator(IrArray::Index(b->getInt64Ty()));
+    };
+  }
 
   bool is_signed = ShapeUtil::ElementIsSigned(start_indices->shape());
   return EmitDynamicUpdateSliceInPlaceImpl(
-- 
GitLab


From a552dc59596fa7d17bfcacc1cb4efc60d7db7d43 Mon Sep 17 00:00:00 2001
From: Saurabh Saxena <srbs@google.com>
Date: Wed, 2 Jan 2019 11:32:05 -0800
Subject: [PATCH 0131/1095] Implements dynamic_size in TensorArray v2. Adds a
 TensorListResize op and changes implementation of set_item to resize the list
 if index is out of bounds before calling the SetItem kernel.

PiperOrigin-RevId: 227553120
---
 .../base_api/api_def_TensorListResize.pbtxt   | 10 +++
 .../python_api/api_def_TensorListResize.pbtxt |  4 ++
 tensorflow/core/kernels/list_kernels.cc       | 59 +++++++++++++++
 tensorflow/core/ops/list_ops.cc               | 18 +++++
 .../python/kernel_tests/list_ops_test.py      | 55 ++++++++++++++
 .../kernel_tests/tensor_array_ops_test.py     | 25 +++----
 tensorflow/python/ops/list_ops.py             | 34 ++++++++-
 tensorflow/python/ops/tensor_array_ops.py     | 71 ++++++++-----------
 tensorflow/python/ops/while_v2.py             | 15 +---
 9 files changed, 224 insertions(+), 67 deletions(-)
 create mode 100644 tensorflow/core/api_def/base_api/api_def_TensorListResize.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_TensorListResize.pbtxt

diff --git a/tensorflow/core/api_def/base_api/api_def_TensorListResize.pbtxt b/tensorflow/core/api_def/base_api/api_def_TensorListResize.pbtxt
new file mode 100644
index 0000000000..5b34f8cec7
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_TensorListResize.pbtxt
@@ -0,0 +1,10 @@
+op {
+  graph_op_name: "TensorListResize"
+  summary: "Resizes the list."
+  description: <<END
+
+input_handle: the input list
+size: size of the output list
+
+END
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_TensorListResize.pbtxt b/tensorflow/core/api_def/python_api/api_def_TensorListResize.pbtxt
new file mode 100644
index 0000000000..0d689d4f2b
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_TensorListResize.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "TensorListResize"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/kernels/list_kernels.cc b/tensorflow/core/kernels/list_kernels.cc
index 2e230595d4..5859c20d89 100644
--- a/tensorflow/core/kernels/list_kernels.cc
+++ b/tensorflow/core/kernels/list_kernels.cc
@@ -513,6 +513,65 @@ REGISTER_TENSOR_LIST_GET_ITEM_GPU(bool)
 
 #endif  // GOOGLE_CUDA
 
+class TensorListResize : public OpKernel {
+ public:
+  explicit TensorListResize(OpKernelConstruction* c) : OpKernel(c) {}
+
+  void Compute(OpKernelContext* c) override {
+    const TensorList* input_list =
+        c->input(0).scalar<Variant>()().get<TensorList>();
+    OP_REQUIRES(c, input_list != nullptr,
+                errors::InvalidArgument(
+                    "Input handle is not a list. Saw: '",
+                    c->input(0).scalar<Variant>()().DebugString(), "'"));
+    int32 size = c->input(1).scalar<int32>()();
+    OP_REQUIRES(
+        c, size >= 0,
+        errors::InvalidArgument(
+            "TensorListSlice expects size to be non-negative. Got: ", size));
+
+    AllocatorAttributes attr;
+    attr.set_on_host(true);
+    std::unique_ptr<Tensor> maybe_result = c->forward_input(
+        0, 0, DT_VARIANT, TensorShape{}, c->input_memory_type(0), attr);
+    if (maybe_result != nullptr) {
+      maybe_result->scalar<Variant>()().get<TensorList>()->tensors.resize(
+          size, Tensor(DT_INVALID));
+    } else {
+      Tensor* result;
+      OP_REQUIRES_OK(c, c->allocate_output(0, TensorShape{}, &result, attr));
+      TensorList output_list;
+      output_list.element_shape = input_list->element_shape;
+      output_list.element_dtype = input_list->element_dtype;
+      output_list.max_num_elements = input_list->max_num_elements;
+      if (size > input_list->tensors.size()) {
+        output_list.tensors.insert(output_list.tensors.begin(),
+                                   input_list->tensors.begin(),
+                                   input_list->tensors.end());
+        // Add DT_INVALID tensors to the end of the list if the requested size
+        // is larger than the list length.
+        output_list.tensors.resize(size, Tensor(DT_INVALID));
+      } else {
+        output_list.tensors.insert(output_list.tensors.begin(),
+                                   input_list->tensors.begin(),
+                                   input_list->tensors.begin() + size);
+      }
+      result->scalar<Variant>()() = std::move(output_list);
+    }
+  }
+};
+
+REGISTER_KERNEL_BUILDER(Name("TensorListResize").Device(DEVICE_CPU),
+                        TensorListResize);
+
+#if GOOGLE_CUDA
+
+REGISTER_KERNEL_BUILDER(
+    Name("TensorListResize").Device(DEVICE_GPU).HostMemory("size"),
+    TensorListResize);
+
+#endif  // GOOGLE_CUDA
+
 class TensorListSetItem : public OpKernel {
  public:
   explicit TensorListSetItem(OpKernelConstruction* c) : OpKernel(c) {
diff --git a/tensorflow/core/ops/list_ops.cc b/tensorflow/core/ops/list_ops.cc
index 01ebcd1543..5722170071 100644
--- a/tensorflow/core/ops/list_ops.cc
+++ b/tensorflow/core/ops/list_ops.cc
@@ -367,6 +367,24 @@ REGISTER_OP("TensorListGetItem")
       return Status::OK();
     });
 
+REGISTER_OP("TensorListResize")
+    .Input("input_handle: variant")
+    .Input("size: int32")
+    .Output("output_handle: variant")
+    .SetIsStateful()
+    .SetShapeFn([](shape_inference::InferenceContext* c) {
+      // Check that `size` has scalar shape.
+      shape_inference::ShapeHandle size_shape = c->input(1);
+      shape_inference::ShapeHandle unused;
+      TF_RETURN_IF_ERROR(c->WithRank(size_shape, 0, &unused));
+      c->set_output(0, c->Scalar());
+      auto* handle_data = c->input_handle_shapes_and_types(0);
+      if (handle_data != nullptr) {
+        c->set_output_handle_shapes_and_types(0, *handle_data);
+      }
+      return Status::OK();
+    });
+
 REGISTER_OP("TensorListSetItem")
     .Input("input_handle: variant")
     .Input("index: int32")
diff --git a/tensorflow/python/kernel_tests/list_ops_test.py b/tensorflow/python/kernel_tests/list_ops_test.py
index 489f6c9b00..054ac36755 100644
--- a/tensorflow/python/kernel_tests/list_ops_test.py
+++ b/tensorflow/python/kernel_tests/list_ops_test.py
@@ -345,6 +345,20 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       self.assertAllEqual(self.evaluate(e), 10.0)
     self.assertAllEqual(self.evaluate(tape.gradient(e, t)), 2.0)
 
+  @test_util.run_deprecated_v1
+  @test_util.enable_control_flow_v2
+  def testSkipEagerSetItemIndexOutOfBounds(self):
+    l = list_ops.empty_tensor_list(
+        element_dtype=dtypes.float32, element_shape=[])
+    e0 = constant_op.constant(5.)
+    l = list_ops.tensor_list_set_item(
+        l, 0, 2. * e0, resize_if_index_out_of_bounds=True)
+    l = list_ops.tensor_list_set_item(
+        l, 1, 1., resize_if_index_out_of_bounds=True)
+    t = list_ops.tensor_list_stack(l, element_dtype=dtypes.float32)
+    grad = gradients_impl.gradients(t, e0)[0]
+    self.assertAllEqual(self.evaluate(grad), 2.)
+
   @test_util.run_deprecated_v1
   def testSetOnEmptyListWithMaxNumElementsFails(self):
     l = list_ops.empty_tensor_list(
@@ -1096,6 +1110,47 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
                                    element_shape=[1],
                                    lengths=[1, 1])
 
+  def testResizeGrow(self):
+    l = list_ops.tensor_list_from_tensor([1., 2.], element_shape=[])
+    l = list_ops.tensor_list_resize(l, 4)
+    self.assertEqual(self.evaluate(list_ops.tensor_list_length(l)), 4)
+    self.assertEqual(
+        self.evaluate(
+            list_ops.tensor_list_get_item(l, 0, element_dtype=dtypes.float32)),
+        1.)
+    self.assertEqual(
+        self.evaluate(
+            list_ops.tensor_list_get_item(l, 1, element_dtype=dtypes.float32)),
+        2.)
+
+  def testResizeShrink(self):
+    l = list_ops.tensor_list_from_tensor([1., 2., 3.], element_shape=[])
+    l = list_ops.tensor_list_resize(l, 2)
+    self.assertEqual(self.evaluate(list_ops.tensor_list_length(l)), 2)
+    self.assertAllEqual(
+        self.evaluate(
+            list_ops.tensor_list_stack(l, element_dtype=dtypes.float32)),
+        [1., 2.])
+
+  def testResizeWithInvalidSizeFails(self):
+    with self.assertRaisesRegexp(
+        errors.InvalidArgumentError,
+        "TensorListSlice expects size to be non-negative"):
+      l = list_ops.tensor_list_from_tensor([1., 2., 3.], element_shape=[])
+      l = list_ops.tensor_list_resize(l, -1)
+      self.evaluate(l)
+
+  @test_util.run_deprecated_v1
+  @test_util.enable_control_flow_v2
+  def testSkipEagerResizeGrad(self):
+    t = constant_op.constant([1., 2., 3.])
+    l = list_ops.tensor_list_from_tensor(t, element_shape=[])
+    l = list_ops.tensor_list_set_item(
+        l, 3, 4., resize_if_index_out_of_bounds=True)
+    t1 = list_ops.tensor_list_stack(l, element_dtype=dtypes.float32)
+    grad = gradients_impl.gradients(t1, t)[0]
+    self.assertAllEqual(self.evaluate(grad), [1., 1., 1.])
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/kernel_tests/tensor_array_ops_test.py b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
index 147e7fde57..9fb5791048 100644
--- a/tensorflow/python/kernel_tests/tensor_array_ops_test.py
+++ b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
@@ -1000,7 +1000,6 @@ class TensorArrayTest(test.TestCase):
     # self._testWhileLoopWritePackGradients(
     #     dynamic_size=False, dtype=tf.int64)
 
-  @test_util.disable_control_flow_v2("b/117943489 (dynamic_size)")
   @test_util.run_v1_only("b/117943489")
   def testSkipEagerWhileLoopDynamicWritePackGradients(self):
     self._testWhileLoopWritePackGradients(
@@ -1207,11 +1206,14 @@ class TensorArrayTest(test.TestCase):
       c1 = constant_op.constant([4.0, 5.0])
       w1 = w0.write(3, c1)
 
-      with self.assertRaisesOpError(
-          r"Could not read index 0 twice because it was cleared after a "
-          r"previous read \(perhaps try setting clear_after_read = false\?\)"):
-        with ops.control_dependencies([r0]):
-          self.evaluate(w1.read(0))
+      if not control_flow_util.ENABLE_CONTROL_FLOW_V2:
+        # TensorArray v2 does not support clear_after_read.
+        with self.assertRaisesOpError(
+            r"Could not read index 0 twice because it was cleared after a "
+            r"previous read \(perhaps try setting clear_after_read = false\?\)"
+        ):
+          with ops.control_dependencies([r0]):
+            self.evaluate(w1.read(0))
 
       r1 = w1.read(1)
       self.assertAllEqual(c1.get_shape(), r1.shape)
@@ -1220,7 +1222,6 @@ class TensorArrayTest(test.TestCase):
       with self.assertRaises(ValueError):
         w1.write(4, c2)
 
-  @test_util.disable_control_flow_v2("b/117943489 (dynamic_size)")
   @test_util.run_v1_only("b/117943489")
   def testUnpackShape(self):
     self._testUnpackShape()
@@ -1300,12 +1301,10 @@ class TensorArrayTest(test.TestCase):
       grad = gradients_impl.gradients(ys=[r], xs=[x])
       self.assertAllEqual(np.array([1.0, 1.0, 1.0]), self.evaluate(grad)[0])
 
-  @test_util.disable_control_flow_v2("b/117943489")
   @test_util.run_v1_only("b/117943489")
   def testSkipEagerTensorArrayUnpackDynamic(self):
     self._testTensorArrayUnpackDynamic()
 
-  @test_util.disable_control_flow_v2("b/117943489")
   @test_util.run_v1_only("b/117943489")
   def testSkipEagerTensorArraySplitDynamic(self):
     with self.session(use_gpu=True) as sess:
@@ -1345,7 +1344,10 @@ class TensorArrayTest(test.TestCase):
           dtype=dtypes.float32, size=0, dynamic_size=False, infer_shape=True)
       self.assertEqual(0, ta.size().eval())
       # Don't actually perform the pack.  This stores the static shape.
-      ta.unstack(array_ops.zeros([0, 3, 5])).mark_used()
+      if control_flow_util.ENABLE_CONTROL_FLOW_V2:
+        ta = ta.unstack(array_ops.zeros([0, 3, 5]))
+      else:
+        ta.unstack(array_ops.zeros([0, 3, 5])).mark_used()
       packed = ta.stack()
       concatenated = ta.concat()
       self.assertAllEqual([0, 3, 5], self.evaluate(packed).shape)
@@ -1353,12 +1355,11 @@ class TensorArrayTest(test.TestCase):
       # first dimension of zero
       self.assertAllEqual([0, 5], self.evaluate(concatenated).shape)
 
-  @test_util.disable_control_flow_v2("b/117943489")
   @test_util.run_v1_only("b/117943489")
   def testSkipEagerTensorArrayEvalEmptyWithDefault(self):
     self._testTensorArrayEvalEmptyWithDefault()
 
-  @test_util.disable_control_flow_v2("b/117943489")
+  @test_util.disable_control_flow_v2("b/117943286")
   @test_util.run_v1_only("b/117943489")
   def testSkipEagerTensorArrayScatterReadAndGradients(self):
     with self.session(use_gpu=True) as session:
diff --git a/tensorflow/python/ops/list_ops.py b/tensorflow/python/ops/list_ops.py
index dbaae886d4..3eac97655a 100644
--- a/tensorflow/python/ops/list_ops.py
+++ b/tensorflow/python/ops/list_ops.py
@@ -28,6 +28,12 @@ from tensorflow.python.ops import gen_list_ops
 # pylint: disable=wildcard-import
 from tensorflow.python.ops.gen_list_ops import *
 # pylint: enable=wildcard-import
+from tensorflow.python.util.lazy_loader import LazyLoader
+
+# list_ops -> control_flow_ops -> tensor_array_ops -> list_ops
+control_flow_ops = LazyLoader(
+    "control_flow_ops", globals(),
+    "tensorflow.python.ops.control_flow_ops")
 
 
 ops.NotDifferentiable("TensorListConcatLists")
@@ -80,6 +86,25 @@ def tensor_list_split(tensor, element_shape, lengths, name=None):
       name=name)
 
 
+def tensor_list_set_item(input_handle,
+                         index,
+                         item,
+                         resize_if_index_out_of_bounds=False,
+                         name=None):
+  """Sets `item` at `index` in input list."""
+  if resize_if_index_out_of_bounds:
+    input_list_size = gen_list_ops.tensor_list_length(input_handle)
+    # TODO(srbs): This could cause some slowdown. Consider fusing resize
+    # functionality in the SetItem op.
+    input_handle = control_flow_ops.cond(
+        index >= input_list_size,
+        lambda: gen_list_ops.tensor_list_resize(  # pylint: disable=g-long-lambda
+            input_handle, index + 1),
+        lambda: input_handle)
+  return gen_list_ops.tensor_list_set_item(
+      input_handle=input_handle, index=index, item=item, name=name)
+
+
 @ops.RegisterGradient("TensorListPushBack")
 def _PushBackGrad(op, dresult):
   return gen_list_ops.tensor_list_pop_back(
@@ -164,6 +189,13 @@ def _TensorListSetItemGrad(op, dlist):
   return list_grad, index_grad, element_grad
 
 
+@ops.RegisterGradient("TensorListResize")
+def _TensorListResizeGrad(op, dlist):
+  input_list, _ = op.inputs
+  input_list_size = gen_list_ops.tensor_list_length(input_list)
+  return gen_list_ops.tensor_list_resize(dlist, input_list_size), None
+
+
 @ops.RegisterGradient("TensorListGather")
 def _TensorListGatherGrad(op, dtensor):
   _, indices = op.inputs
@@ -176,7 +208,7 @@ def _TensorListGatherGrad(op, dtensor):
 def _TensorListScatterGrad(op, dlist):
   t, indices, _ = op.inputs
   return gen_list_ops.tensor_list_gather(
-      dlist, indices, element_dtype=t.dtype), None
+      dlist, indices, element_dtype=t.dtype), None, None
 
 
 def _build_element_shape(shape):
diff --git a/tensorflow/python/ops/tensor_array_ops.py b/tensorflow/python/ops/tensor_array_ops.py
index aece7cfe25..9b8bd2c8d3 100644
--- a/tensorflow/python/ops/tensor_array_ops.py
+++ b/tensorflow/python/ops/tensor_array_ops.py
@@ -114,6 +114,7 @@ class _GraphTensorArray(object):
 
     if clear_after_read is None:
       clear_after_read = True
+    self._dynamic_size = None
     dynamic_size = dynamic_size or False
 
     self._dtype = dtype
@@ -221,7 +222,9 @@ class _GraphTensorArray(object):
     """See TensorArray."""
     flow = array_ops.identity(self._flow)
     ta = TensorArray(
-        dtype=self._dtype, handle=self._handle, flow=flow,
+        dtype=self._dtype,
+        handle=self._handle,
+        flow=flow,
         infer_shape=self._infer_shape,
         colocate_with_first_write_call=self._colocate_with_first_write_call)
     ta._element_shape = self._element_shape
@@ -278,7 +281,9 @@ class _GraphTensorArray(object):
             flow_in=self._flow,
             name=name)
       ta = TensorArray(
-          dtype=self._dtype, handle=self._handle, flow=flow_out,
+          dtype=self._dtype,
+          handle=self._handle,
+          flow=flow_out,
           colocate_with_first_write_call=self._colocate_with_first_write_call)
       ta._infer_shape = self._infer_shape
       ta._element_shape = self._element_shape
@@ -349,7 +354,9 @@ class _GraphTensorArray(object):
             flow_in=self._flow,
             name=name)
       ta = TensorArray(
-          dtype=self._dtype, handle=self._handle, flow=flow_out,
+          dtype=self._dtype,
+          handle=self._handle,
+          flow=flow_out,
           colocate_with_first_write_call=self._colocate_with_first_write_call)
       ta._infer_shape = self._infer_shape
       ta._element_shape = self._element_shape
@@ -378,7 +385,9 @@ class _GraphTensorArray(object):
             flow_in=self._flow,
             name=name)
       ta = TensorArray(
-          dtype=self._dtype, handle=self._handle, flow=flow_out,
+          dtype=self._dtype,
+          handle=self._handle,
+          flow=flow_out,
           colocate_with_first_write_call=self._colocate_with_first_write_call)
       ta._infer_shape = self._infer_shape
       ta._element_shape = self._element_shape
@@ -448,7 +457,7 @@ class _GraphTensorArrayV2(object):
     del tensor_array_name
     del colocate_with_first_write_call
 
-    del dynamic_size  # TODO(b/117943489): Unused for now.
+    self._dynamic_size = dynamic_size
 
     if (flow is not None and
         (not isinstance(flow, ops.Tensor) or flow.dtype != dtypes.variant)):
@@ -525,10 +534,7 @@ class _GraphTensorArrayV2(object):
   def identity(self):
     """See TensorArray."""
     flow = array_ops.identity(self._flow)
-    ta = TensorArray(
-        dtype=self._dtype, flow=flow, infer_shape=self._infer_shape)
-    ta._element_shape = self._element_shape
-    return ta
+    return build_ta_with_new_flow(self, flow)
 
   def grad(self, source, flow=None, name=None):
     """Not supported."""
@@ -553,11 +559,12 @@ class _GraphTensorArrayV2(object):
       if self._infer_shape:
         self._merge_element_shape(value.shape)
       flow_out = list_ops.tensor_list_set_item(
-          input_handle=self._flow, index=index, item=value, name=name)
-      ta = TensorArray(dtype=self._dtype, handle=None, flow=flow_out)
-      ta._infer_shape = self._infer_shape
-      ta._element_shape = self._element_shape
-      return ta
+          input_handle=self._flow,
+          index=index,
+          item=value,
+          resize_if_index_out_of_bounds=self._dynamic_size,
+          name=name)
+      return build_ta_with_new_flow(self, flow_out)
 
   def stack(self, name=None):
     """See TensorArray."""
@@ -596,15 +603,7 @@ class _GraphTensorArrayV2(object):
         self._merge_element_shape(value.shape[1:])
       flow_out = list_ops.tensor_list_from_tensor(
           tensor=value, element_shape=value.shape[1:])
-      ta = TensorArray(
-          dtype=self._dtype,
-          handle=self.handle,
-          flow=flow_out,
-          colocate_with_first_write_call=self._colocate_with_first_write_call)
-      ta._infer_shape = self._infer_shape
-      ta._element_shape = self._element_shape
-      ta._colocate_with = self._colocate_with
-      return ta
+      return build_ta_with_new_flow(self, flow_out)
 
   @tf_should_use.should_use_result
   def scatter(self, indices, value, name=None):
@@ -616,15 +615,7 @@ class _GraphTensorArrayV2(object):
         self._merge_element_shape(value.shape[1:])
       flow_out = list_ops.tensor_list_scatter(
           tensor=value, indices=indices, element_shape=-1)
-      ta = TensorArray(
-          dtype=self._dtype,
-          handle=self.handle,
-          flow=flow_out,
-          colocate_with_first_write_call=self._colocate_with_first_write_call)
-      ta._infer_shape = self._infer_shape
-      ta._element_shape = self._element_shape
-      ta._colocate_with = self._colocate_with
-      return ta
+      return build_ta_with_new_flow(self, flow_out)
 
   @tf_should_use.should_use_result
   def split(self, value, lengths, name=None):
@@ -644,15 +635,7 @@ class _GraphTensorArrayV2(object):
           lengths=lengths_64,
           element_shape=self._element_shape[0] if self._element_shape else None,
           name=name)
-      ta = TensorArray(
-          dtype=self._dtype,
-          handle=self.handle,
-          flow=flow_out,
-          colocate_with_first_write_call=self._colocate_with_first_write_call)
-      ta._infer_shape = self._infer_shape
-      ta._element_shape = self._element_shape
-      ta._colocate_with = self._colocate_with
-      return ta
+      return build_ta_with_new_flow(self, flow_out)
 
   def size(self, name=None):
     """See TensorArray."""
@@ -1043,6 +1026,10 @@ class TensorArray(object):
     """The reference to the TensorArray."""
     return self._implementation.handle
 
+  @property
+  def _dynamic_size(self):
+    return self._implementation._dynamic_size
+
   @property
   def _infer_shape(self):
     return self._implementation._infer_shape
@@ -1228,8 +1215,10 @@ class TensorArray(object):
 
 
 def build_ta_with_new_flow(old_ta, flow):
+  """Builds a TensorArray with a new `flow` tensor."""
   ta = TensorArray(
       dtype=old_ta.dtype,
+      dynamic_size=old_ta._dynamic_size,
       handle=old_ta.handle,
       flow=flow,
       infer_shape=old_ta._infer_shape,
diff --git a/tensorflow/python/ops/while_v2.py b/tensorflow/python/ops/while_v2.py
index 8547cb94a0..aaed86c1ca 100644
--- a/tensorflow/python/ops/while_v2.py
+++ b/tensorflow/python/ops/while_v2.py
@@ -849,19 +849,8 @@ def _pack_sequence_as(structure_with_tas, loop_vars):
   """Like `nest.pack_sequence_as` but also replaces flows with TensorArrays."""
 
   def flow_to_tensor_array(flow, ta):  # pylint: disable=missing-docstring
-    if isinstance(ta, tensor_array_ops.TensorArray):
-      # pylint: disable=protected-access
-      new_ta = tensor_array_ops.TensorArray(
-          dtype=ta.dtype,
-          handle=ta.handle,
-          flow=flow,
-          infer_shape=ta._infer_shape,
-          colocate_with_first_write_call=ta._colocate_with_first_write_call)
-      new_ta._colocate_with = ta._colocate_with
-      new_ta._element_shape = ta._element_shape
-      # pylint: enable=protected-access
-      return new_ta
-    return flow
+    return (tensor_array_ops.build_ta_with_new_flow(ta, flow) if isinstance(  # pylint: disable=g-long-ternary
+        ta, tensor_array_ops.TensorArray) else flow)
 
   flattened_loop_vars = [
       flow_to_tensor_array(*z)
-- 
GitLab


From 6966757f563887e6577b3914cf07cfc50a3f7469 Mon Sep 17 00:00:00 2001
From: Michael Kuperstein <mkuper@google.com>
Date: Wed, 2 Jan 2019 11:33:31 -0800
Subject: [PATCH 0132/1095] [XLA] Add pass to turn R1 indices to DS/DUS into
 scalar indices.

This introduces a pass that transforms the rank-1 form into the scalar form.
It is currently unused, but will be used in the beginning of every backend
pipeline to help transition from R1 to scalar indices.

PiperOrigin-RevId: 227553411
---
 tensorflow/compiler/xla/service/BUILD         |  32 +++++
 .../xla/service/dynamic_index_splitter.cc     |  97 +++++++++++++
 .../xla/service/dynamic_index_splitter.h      |  37 +++++
 .../service/dynamic_index_splitter_test.cc    | 134 ++++++++++++++++++
 4 files changed, 300 insertions(+)
 create mode 100644 tensorflow/compiler/xla/service/dynamic_index_splitter.cc
 create mode 100644 tensorflow/compiler/xla/service/dynamic_index_splitter.h
 create mode 100644 tensorflow/compiler/xla/service/dynamic_index_splitter_test.cc

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index 0db1d4a78d..eed958dc6b 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -3688,6 +3688,38 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "dynamic_index_splitter",
+    srcs = ["dynamic_index_splitter.cc"],
+    hdrs = ["dynamic_index_splitter.h"],
+    deps = [
+        ":hlo_casting_utils",
+        "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla:statusor",
+        "//tensorflow/compiler/xla/service:hlo",
+        "//tensorflow/compiler/xla/service:hlo_pass",
+        "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/container:flat_hash_set",
+        "@com_google_absl//absl/container:inlined_vector",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+tf_cc_test(
+    name = "dynamic_index_splitter_test",
+    srcs = ["dynamic_index_splitter_test.cc"],
+    deps = [
+        ":dynamic_index_splitter",
+        ":hlo",
+        ":hlo_matchers",
+        "//tensorflow/compiler/xla:statusor",
+        "//tensorflow/compiler/xla:test",
+        "//tensorflow/compiler/xla:test_helpers",
+        "//tensorflow/compiler/xla/tests:hlo_test_base",
+        "//tensorflow/compiler/xla/tests:xla_internal_test_main",
+    ],
+)
+
 tf_cc_test(
     name = "ar_crs_combiner_test",
     srcs = ["ar_crs_combiner_test.cc"],
diff --git a/tensorflow/compiler/xla/service/dynamic_index_splitter.cc b/tensorflow/compiler/xla/service/dynamic_index_splitter.cc
new file mode 100644
index 0000000000..ede4a5afdb
--- /dev/null
+++ b/tensorflow/compiler/xla/service/dynamic_index_splitter.cc
@@ -0,0 +1,97 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/dynamic_index_splitter.h"
+
+#include <map>
+
+#include "absl/container/flat_hash_map.h"
+#include "absl/container/flat_hash_set.h"
+#include "absl/container/inlined_vector.h"
+#include "tensorflow/compiler/xla/service/hlo_casting_utils.h"
+#include "tensorflow/compiler/xla/service/hlo_computation.h"
+#include "tensorflow/compiler/xla/service/hlo_instruction.h"
+#include "tensorflow/compiler/xla/service/hlo_instructions.h"
+#include "tensorflow/compiler/xla/service/hlo_opcode.h"
+#include "tensorflow/compiler/xla/shape_util.h"
+
+namespace xla {
+
+StatusOr<bool> DynamicIndexSplitter::Run(HloModule* module) {
+  bool changed = false;
+
+  std::vector<HloComputation*> computations =
+      module->MakeNonfusionComputations();
+  for (HloComputation* computation : computations) {
+    for (HloInstruction* dynamic_op : computation->MakeInstructionPostOrder()) {
+      switch (dynamic_op->opcode()) {
+        case HloOpcode::kDynamicSlice:
+        case HloOpcode::kDynamicUpdateSlice:
+          break;
+        default:
+          continue;
+      }
+      auto parent = dynamic_op->parent();
+      bool is_update = dynamic_op->opcode() == HloOpcode::kDynamicUpdateSlice;
+      int64 index_operand_number = Cast<HloDynamicIndexInstruction>(dynamic_op)
+                                       ->first_index_operand_number();
+      auto index_operand = dynamic_op->mutable_operand(index_operand_number);
+      if (ShapeUtil::IsScalar(index_operand->shape())) {
+        // This DS/DUS already uses scalar indices.
+        continue;
+      }
+      TF_RET_CHECK(index_operand->shape().rank() == 1);
+      int64 num_indices = index_operand->shape().dimensions(0);
+      if (num_indices == 0) {
+        // If the operand dimension is 0, directly replace R0 DS/DUS with the
+        // operand (for DS) or update (for DUS).
+        if (is_update) {
+          TF_CHECK_OK(parent->ReplaceInstruction(
+              dynamic_op, dynamic_op->mutable_operand(1)));
+        } else {
+          TF_CHECK_OK(parent->ReplaceInstruction(
+              dynamic_op, dynamic_op->mutable_operand(0)));
+        }
+        changed = true;
+        continue;
+      }
+      auto index_element_type = index_operand->shape().element_type();
+      std::vector<HloInstruction*> index_array;
+      for (int64 dim = 0; dim < num_indices; ++dim) {
+        auto slice = parent->AddInstruction(HloInstruction::CreateSlice(
+            ShapeUtil::MakeShape(index_element_type, {1}), index_operand, {dim},
+            {dim + 1}, {1}));
+        auto bitcast = parent->AddInstruction(HloInstruction::CreateReshape(
+            ShapeUtil::MakeShape(index_element_type, {}), slice));
+        index_array.push_back(bitcast);
+      }
+      auto new_dynamic_op =
+          is_update
+              ? HloInstruction::CreateDynamicUpdateSlice(
+                    dynamic_op->shape(), dynamic_op->mutable_operand(0),
+                    dynamic_op->mutable_operand(1), absl::MakeSpan(index_array))
+              : HloInstruction::CreateDynamicSlice(
+                    dynamic_op->shape(), dynamic_op->mutable_operand(0),
+                    absl::MakeSpan(index_array),
+                    dynamic_op->dynamic_slice_sizes());
+      TF_CHECK_OK(parent->ReplaceWithNewInstruction(dynamic_op,
+                                                    std::move(new_dynamic_op)));
+      changed = true;
+    }
+  }
+  return changed;
+}
+
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/dynamic_index_splitter.h b/tensorflow/compiler/xla/service/dynamic_index_splitter.h
new file mode 100644
index 0000000000..3c12e3a4af
--- /dev/null
+++ b/tensorflow/compiler/xla/service/dynamic_index_splitter.h
@@ -0,0 +1,37 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_DYNAMIC_INDEX_SPLITTER_H_
+#define TENSORFLOW_COMPILER_XLA_SERVICE_DYNAMIC_INDEX_SPLITTER_H_
+
+#include "absl/strings/string_view.h"
+#include "tensorflow/compiler/xla/service/hlo_module.h"
+#include "tensorflow/compiler/xla/service/hlo_pass_interface.h"
+#include "tensorflow/compiler/xla/statusor.h"
+
+namespace xla {
+
+// Convert R1 index operands to DynamicSlice and DynamicUpdateSlice ops into
+// separate scalars.
+class DynamicIndexSplitter : public HloModulePass {
+ public:
+  DynamicIndexSplitter() = default;
+  absl::string_view name() const override { return "dynamic-index-splitter"; }
+  StatusOr<bool> Run(HloModule* module) override;
+};
+
+}  // namespace xla
+
+#endif  // TENSORFLOW_COMPILER_XLA_SERVICE_DYNAMIC_INDEX_SPLITTER_H_
diff --git a/tensorflow/compiler/xla/service/dynamic_index_splitter_test.cc b/tensorflow/compiler/xla/service/dynamic_index_splitter_test.cc
new file mode 100644
index 0000000000..98029d1faf
--- /dev/null
+++ b/tensorflow/compiler/xla/service/dynamic_index_splitter_test.cc
@@ -0,0 +1,134 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/dynamic_index_splitter.h"
+
+#include "tensorflow/compiler/xla/service/hlo_computation.h"
+#include "tensorflow/compiler/xla/service/hlo_instruction.h"
+#include "tensorflow/compiler/xla/service/hlo_matchers.h"
+#include "tensorflow/compiler/xla/service/hlo_opcode.h"
+#include "tensorflow/compiler/xla/test.h"
+#include "tensorflow/compiler/xla/test_helpers.h"
+#include "tensorflow/compiler/xla/tests/hlo_test_base.h"
+
+namespace xla {
+namespace {
+
+namespace op = xla::testing::opcode_matchers;
+class DynamicIndexSplitterTest : public HloTestBase {};
+
+TEST_F(DynamicIndexSplitterTest, DynamicSlice) {
+  const char* const kDynamicSlice = R"(
+    HloModule DynamicSlice_module
+
+    ENTRY entry (operand: s32[4,5,6], indices: s32[3]) -> s32[1,1,1] {
+      operand = s32[4,5,6] parameter(0)
+      indices = s32[3] parameter(1)
+      ROOT dynamic-slice = s32[1,1,1] dynamic-slice(operand, indices), dynamic_slice_sizes={1,1,1}
+    }
+  )";
+
+  HloModuleConfig config;
+  DebugOptions debug_options = config.debug_options();
+  debug_options.set_xla_allow_scalar_index_dynamic_ops(true);
+  config.set_debug_options(debug_options);
+
+  TF_ASSERT_OK_AND_ASSIGN(auto module, ParseHloString(kDynamicSlice, config));
+  TF_ASSERT_OK_AND_ASSIGN(bool changed,
+                          DynamicIndexSplitter().Run(module.get()));
+  EXPECT_TRUE(changed);
+  ASSERT_THAT(module->entry_computation()->root_instruction(),
+              op::DynamicSlice(op::Parameter(0),
+                               op::Reshape(op::Slice(op::Parameter(1))),
+                               op::Reshape(op::Slice(op::Parameter(1))),
+                               op::Reshape(op::Slice(op::Parameter(1)))));
+
+  for (int i = 0; i < 3; ++i) {
+    const HloInstruction* slice = module->entry_computation()
+                                      ->root_instruction()
+                                      ->operand(i + 1)
+                                      ->operand(0);
+    EXPECT_EQ(slice->slice_starts(0), i);
+    EXPECT_EQ(slice->slice_limits(0), i + 1);
+  }
+}
+
+TEST_F(DynamicIndexSplitterTest, DynamicUpdateSlice) {
+  const char* const kDynamicUpdateSlice = R"(
+    HloModule DynamicUpdatedSlice_module
+
+    ENTRY entry (operand: s32[4,5,6], indices: s32[3], update: s32[1,1,1]) -> s32[4,5,6] {
+      operand = s32[4,5,6] parameter(0)
+      indices = s32[3] parameter(1)
+      update = s32[1,1,1] parameter(2)
+      ROOT dynamic-update-slice = s32[4,5,6] dynamic-update-slice(operand, update, indices)
+    }
+  )";
+
+  HloModuleConfig config;
+  DebugOptions debug_options = config.debug_options();
+  debug_options.set_xla_allow_scalar_index_dynamic_ops(true);
+  config.set_debug_options(debug_options);
+
+  TF_ASSERT_OK_AND_ASSIGN(auto module,
+                          ParseHloString(kDynamicUpdateSlice, config));
+  TF_ASSERT_OK_AND_ASSIGN(bool changed,
+                          DynamicIndexSplitter().Run(module.get()));
+  EXPECT_TRUE(changed);
+  ASSERT_THAT(module->entry_computation()->root_instruction(),
+              op::DynamicUpdateSlice(op::Parameter(0), op::Parameter(2),
+                                     op::Reshape(op::Slice(op::Parameter(1))),
+                                     op::Reshape(op::Slice(op::Parameter(1))),
+                                     op::Reshape(op::Slice(op::Parameter(1)))));
+
+  for (int i = 0; i < 3; ++i) {
+    const HloInstruction* slice = module->entry_computation()
+                                      ->root_instruction()
+                                      ->operand(i + 2)
+                                      ->operand(0);
+    EXPECT_EQ(slice->slice_starts(0), i);
+    EXPECT_EQ(slice->slice_limits(0), i + 1);
+  }
+}
+
+TEST_F(DynamicIndexSplitterTest, AlreadyScalar) {
+  const char* const kDynamicSlice = R"(
+    HloModule DynamicSlice_module
+
+    ENTRY entry (operand: s32[4,5,6], index.0: s32[], index.1: s32[], index.2: s32[]) -> s32[1,1,1] {
+      operand = s32[4,5,6] parameter(0)
+      index.0 = s32[] parameter(1)
+      index.1 = s32[] parameter(2)
+      index.2 = s32[] parameter(3)
+      ROOT dynamic-slice = s32[1,1,1] dynamic-slice(operand, index.0, index.1, index.2), dynamic_slice_sizes={1,1,1}
+    }
+  )";
+
+  HloModuleConfig config;
+  DebugOptions debug_options = config.debug_options();
+  debug_options.set_xla_allow_scalar_index_dynamic_ops(true);
+  config.set_debug_options(debug_options);
+
+  TF_ASSERT_OK_AND_ASSIGN(auto module, ParseHloString(kDynamicSlice, config));
+  TF_ASSERT_OK_AND_ASSIGN(bool changed,
+                          DynamicIndexSplitter().Run(module.get()));
+  EXPECT_FALSE(changed);
+  EXPECT_THAT(module->entry_computation()->root_instruction(),
+              op::DynamicSlice(op::Parameter(0), op::Parameter(1),
+                               op::Parameter(2), op::Parameter(3)));
+}
+
+}  // namespace
+}  // namespace xla
-- 
GitLab


From 39ce126739b88ed6ecfe65bf4b765f32fb0e1e8a Mon Sep 17 00:00:00 2001
From: Andiry Xu <andiryxu@google.com>
Date: Wed, 2 Jan 2019 11:39:17 -0800
Subject: [PATCH 0133/1095] Add step_stats_pyclif to BUILD

PiperOrigin-RevId: 227554561
---
 tensorflow/core/BUILD | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 4862c90aca..1afac057d9 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -2009,6 +2009,13 @@ tf_pyclif_proto_library(
     visibility = ["//visibility:public"],
 )
 
+tf_pyclif_proto_library(
+    name = "framework/step_stats_pyclif",
+    proto_lib = ":protos_all_cc",
+    proto_srcfile = "framework/step_stats.proto",
+    visibility = ["//visibility:public"],
+)
+
 tf_pyclif_proto_library(
     name = "framework/types_pyclif",
     proto_lib = ":protos_all_cc",
-- 
GitLab


From 89466c1e2bacf68366cf25121c27521c5f6e025f Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Wed, 2 Jan 2019 12:17:44 -0800
Subject: [PATCH 0134/1095] Adding get_config fns and restore config tests for
 metrics.

PiperOrigin-RevId: 227560755
---
 tensorflow/python/keras/metrics.py      |  38 ++++++++
 tensorflow/python/keras/metrics_test.py | 115 +++++++++++++++++++++++-
 2 files changed, 149 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index d9842f7810..6b81151d5c 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -612,6 +612,7 @@ class _ConfusionMatrixConditionCount(Metric):
     """
     super(_ConfusionMatrixConditionCount, self).__init__(name=name, dtype=dtype)
     self._confusion_matrix_cond = confusion_matrix_cond
+    self.init_thresholds = thresholds
     self.thresholds = metrics_utils.parse_init_thresholds(
         thresholds, default_threshold=0.5)
     self.accumulator = self.add_weight(
@@ -648,6 +649,11 @@ class _ConfusionMatrixConditionCount(Metric):
     for v in self.variables:
       K.set_value(v, np.zeros((num_thresholds,)))
 
+  def get_config(self):
+    config = {'thresholds': self.init_thresholds}
+    base_config = super(_ConfusionMatrixConditionCount, self).get_config()
+    return dict(list(base_config.items()) + list(config.items()))
+
 
 @keras_export('keras.metrics.FalsePositives')
 class FalsePositives(_ConfusionMatrixConditionCount):
@@ -894,6 +900,7 @@ class Precision(Metric):
       dtype: (Optional) data type of the metric result.
     """
     super(Precision, self).__init__(name=name, dtype=dtype)
+    self.init_thresholds = thresholds
     self.thresholds = metrics_utils.parse_init_thresholds(
         thresholds, default_threshold=0.5)
     self.tp = self.add_weight(
@@ -932,6 +939,11 @@ class Precision(Metric):
     for v in self.variables:
       K.set_value(v, np.zeros((num_thresholds,)))
 
+  def get_config(self):
+    config = {'thresholds': self.init_thresholds}
+    base_config = super(Precision, self).get_config()
+    return dict(list(base_config.items()) + list(config.items()))
+
 
 @keras_export('keras.metrics.Recall')
 class Recall(Metric):
@@ -978,6 +990,7 @@ class Recall(Metric):
       dtype: (Optional) data type of the metric result.
     """
     super(Recall, self).__init__(name=name, dtype=dtype)
+    self.init_thresholds = thresholds
     self.thresholds = metrics_utils.parse_init_thresholds(
         thresholds, default_threshold=0.5)
     self.tp = self.add_weight(
@@ -1016,6 +1029,11 @@ class Recall(Metric):
     for v in self.variables:
       K.set_value(v, np.zeros((num_thresholds,)))
 
+  def get_config(self):
+    config = {'thresholds': self.init_thresholds}
+    base_config = super(Recall, self).get_config()
+    return dict(list(base_config.items()) + list(config.items()))
+
 
 @six.add_metaclass(abc.ABCMeta)
 class SensitivitySpecificityBase(Metric):
@@ -1132,6 +1150,8 @@ class SensitivityAtSpecificity(SensitivitySpecificityBase):
     """
     if specificity < 0 or specificity > 1:
       raise ValueError('`specificity` must be in the range [0, 1].')
+    self.specificity = specificity
+    self.num_thresholds = num_thresholds
     super(SensitivityAtSpecificity, self).__init__(
         specificity, num_thresholds=num_thresholds, name=name, dtype=dtype)
 
@@ -1149,6 +1169,14 @@ class SensitivityAtSpecificity(SensitivitySpecificityBase):
     return math_ops.div_no_nan(self.tp[min_index],
                                self.tp[min_index] + self.fn[min_index])
 
+  def get_config(self):
+    config = {
+        'num_thresholds': self.num_thresholds,
+        'specificity': self.specificity
+    }
+    base_config = super(SensitivityAtSpecificity, self).get_config()
+    return dict(list(base_config.items()) + list(config.items()))
+
 
 @keras_export('keras.metrics.SpecificityAtSensitivity')
 class SpecificityAtSensitivity(SensitivitySpecificityBase):
@@ -1201,6 +1229,8 @@ class SpecificityAtSensitivity(SensitivitySpecificityBase):
     """
     if sensitivity < 0 or sensitivity > 1:
       raise ValueError('`sensitivity` must be in the range [0, 1].')
+    self.sensitivity = sensitivity
+    self.num_thresholds = num_thresholds
     super(SpecificityAtSensitivity, self).__init__(
         sensitivity, num_thresholds=num_thresholds, name=name, dtype=dtype)
 
@@ -1218,6 +1248,14 @@ class SpecificityAtSensitivity(SensitivitySpecificityBase):
     return math_ops.div_no_nan(self.tn[min_index],
                                self.tn[min_index] + self.fp[min_index])
 
+  def get_config(self):
+    config = {
+        'num_thresholds': self.num_thresholds,
+        'sensitivity': self.sensitivity
+    }
+    base_config = super(SpecificityAtSensitivity, self).get_config()
+    return dict(list(base_config.items()) + list(config.items()))
+
 
 @keras_export('keras.metrics.CosineProximity')
 class CosineProximity(MeanMetricWrapper):
diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py
index 2bce89fd1a..8f8befd413 100644
--- a/tensorflow/python/keras/metrics_test.py
+++ b/tensorflow/python/keras/metrics_test.py
@@ -77,6 +77,13 @@ class KerasMeanTest(test.TestCase):
     self.assertEqual(self.evaluate(m.total), 0)
     self.assertEqual(self.evaluate(m.count), 0)
 
+    # Check save and restore config
+    m2 = metrics.Mean.from_config(m.get_config())
+    self.assertEqual(m2.name, 'my_mean')
+    self.assertTrue(m2.stateful)
+    self.assertEqual(m2.dtype, dtypes.float32)
+    self.assertEqual(len(m2.variables), 2)
+
   def test_mean_with_sample_weight(self):
     m = metrics.Mean(dtype=dtypes.float64)
     self.assertEqual(m.dtype, dtypes.float64)
@@ -190,6 +197,13 @@ class KerasAccuracyTest(test.TestCase):
     result = self.evaluate(acc_obj.result())
     self.assertEqual(result, 1)  # 2/2
 
+    # Check save and restore config
+    a2 = metrics.Accuracy.from_config(acc_obj.get_config())
+    self.assertEqual(a2.name, 'my acc')
+    self.assertTrue(a2.stateful)
+    self.assertEqual(len(a2.variables), 2)
+    self.assertEqual(a2.dtype, dtypes.float32)
+
     # check with sample_weight
     result_t = acc_obj([[2], [1]], [[2], [0]], sample_weight=[[0.5], [0.2]])
     result = self.evaluate(result_t)
@@ -331,6 +345,12 @@ class FalsePositivesTest(test.TestCase):
     self.assertEqual(len(fp_obj.variables), 1)
     self.assertEqual(fp_obj.thresholds, [0.4, 0.9])
 
+    # Check save and restore config
+    fp_obj2 = metrics.FalsePositives.from_config(fp_obj.get_config())
+    self.assertEqual(fp_obj2.name, 'my_fp')
+    self.assertEqual(len(fp_obj2.variables), 1)
+    self.assertEqual(fp_obj2.thresholds, [0.4, 0.9])
+
   def test_unweighted(self):
     fp_obj = metrics.FalsePositives()
     self.evaluate(variables.variables_initializer(fp_obj.variables))
@@ -405,6 +425,12 @@ class FalseNegativesTest(test.TestCase):
     self.assertEqual(len(fn_obj.variables), 1)
     self.assertEqual(fn_obj.thresholds, [0.4, 0.9])
 
+    # Check save and restore config
+    fn_obj2 = metrics.FalseNegatives.from_config(fn_obj.get_config())
+    self.assertEqual(fn_obj2.name, 'my_fn')
+    self.assertEqual(len(fn_obj2.variables), 1)
+    self.assertEqual(fn_obj2.thresholds, [0.4, 0.9])
+
   def test_unweighted(self):
     fn_obj = metrics.FalseNegatives()
     self.evaluate(variables.variables_initializer(fn_obj.variables))
@@ -467,6 +493,12 @@ class TrueNegativesTest(test.TestCase):
     self.assertEqual(len(tn_obj.variables), 1)
     self.assertEqual(tn_obj.thresholds, [0.4, 0.9])
 
+    # Check save and restore config
+    tn_obj2 = metrics.TrueNegatives.from_config(tn_obj.get_config())
+    self.assertEqual(tn_obj2.name, 'my_tn')
+    self.assertEqual(len(tn_obj2.variables), 1)
+    self.assertEqual(tn_obj2.thresholds, [0.4, 0.9])
+
   def test_unweighted(self):
     tn_obj = metrics.TrueNegatives()
     self.evaluate(variables.variables_initializer(tn_obj.variables))
@@ -529,6 +561,12 @@ class TruePositivesTest(test.TestCase):
     self.assertEqual(len(tp_obj.variables), 1)
     self.assertEqual(tp_obj.thresholds, [0.4, 0.9])
 
+    # Check save and restore config
+    tp_obj2 = metrics.TruePositives.from_config(tp_obj.get_config())
+    self.assertEqual(tp_obj2.name, 'my_tp')
+    self.assertEqual(len(tp_obj2.variables), 1)
+    self.assertEqual(tp_obj2.thresholds, [0.4, 0.9])
+
   def test_unweighted(self):
     tp_obj = metrics.TruePositives()
     self.evaluate(variables.variables_initializer(tp_obj.variables))
@@ -592,6 +630,12 @@ class PrecisionTest(test.TestCase):
                      ['true_positives:0', 'false_positives:0'])
     self.assertEqual(p_obj.thresholds, [0.4, 0.9])
 
+    # Check save and restore config
+    p_obj2 = metrics.Precision.from_config(p_obj.get_config())
+    self.assertEqual(p_obj2.name, 'my_precision')
+    self.assertEqual(len(p_obj2.variables), 2)
+    self.assertEqual(p_obj2.thresholds, [0.4, 0.9])
+
   def test_value_is_idempotent(self):
     p_obj = metrics.Precision(thresholds=[0.3, 0.72])
     y_pred = random_ops.random_uniform(shape=(10, 3))
@@ -704,6 +748,12 @@ class RecallTest(test.TestCase):
                      ['true_positives:0', 'false_negatives:0'])
     self.assertEqual(r_obj.thresholds, [0.4, 0.9])
 
+    # Check save and restore config
+    r_obj2 = metrics.Recall.from_config(r_obj.get_config())
+    self.assertEqual(r_obj2.name, 'my_recall')
+    self.assertEqual(len(r_obj2.variables), 2)
+    self.assertEqual(r_obj2.thresholds, [0.4, 0.9])
+
   def test_value_is_idempotent(self):
     r_obj = metrics.Recall(thresholds=[0.3, 0.72])
     y_pred = random_ops.random_uniform(shape=(10, 3))
@@ -812,8 +862,15 @@ class SensitivityAtSpecificityTest(test.TestCase, parameterized.TestCase):
         0.4, num_thresholds=100, name='sensitivity_at_specificity_1')
     self.assertEqual(s_obj.name, 'sensitivity_at_specificity_1')
     self.assertLen(s_obj.variables, 4)
-    self.assertEqual(s_obj.value, 0.4)
-    self.assertLen(s_obj.thresholds, 100)
+    self.assertEqual(s_obj.specificity, 0.4)
+    self.assertEqual(s_obj.num_thresholds, 100)
+
+    # Check save and restore config
+    s_obj2 = metrics.SensitivityAtSpecificity.from_config(s_obj.get_config())
+    self.assertEqual(s_obj2.name, 'sensitivity_at_specificity_1')
+    self.assertLen(s_obj2.variables, 4)
+    self.assertEqual(s_obj2.specificity, 0.4)
+    self.assertEqual(s_obj2.num_thresholds, 100)
 
   def test_value_is_idempotent(self):
     s_obj = metrics.SensitivityAtSpecificity(0.7)
@@ -901,8 +958,15 @@ class SpecificityAtSensitivityTest(test.TestCase, parameterized.TestCase):
         0.4, num_thresholds=100, name='specificity_at_sensitivity_1')
     self.assertEqual(s_obj.name, 'specificity_at_sensitivity_1')
     self.assertLen(s_obj.variables, 4)
-    self.assertEqual(s_obj.value, 0.4)
-    self.assertLen(s_obj.thresholds, 100)
+    self.assertEqual(s_obj.sensitivity, 0.4)
+    self.assertEqual(s_obj.num_thresholds, 100)
+
+    # Check save and restore config
+    s_obj2 = metrics.SpecificityAtSensitivity.from_config(s_obj.get_config())
+    self.assertEqual(s_obj2.name, 'specificity_at_sensitivity_1')
+    self.assertLen(s_obj2.variables, 4)
+    self.assertEqual(s_obj2.sensitivity, 0.4)
+    self.assertEqual(s_obj2.num_thresholds, 100)
 
   def test_value_is_idempotent(self):
     s_obj = metrics.SpecificityAtSensitivity(0.7)
@@ -990,6 +1054,11 @@ class CosineProximityTest(test.TestCase):
     self.assertEqual(cosine_obj.name, 'my_cos')
     self.assertEqual(cosine_obj._dtype, dtypes.int32)
 
+    # Check save and restore config
+    cosine_obj2 = metrics.CosineProximity.from_config(cosine_obj.get_config())
+    self.assertEqual(cosine_obj2.name, 'my_cos')
+    self.assertEqual(cosine_obj2._dtype, dtypes.int32)
+
   def test_unweighted(self):
     cosine_obj = metrics.CosineProximity()
     self.evaluate(variables.variables_initializer(cosine_obj.variables))
@@ -1024,6 +1093,11 @@ class MeanAbsoluteErrorTest(test.TestCase):
     self.assertEqual(mae_obj.name, 'my_mae')
     self.assertEqual(mae_obj._dtype, dtypes.int32)
 
+    # Check save and restore config
+    mae_obj2 = metrics.MeanAbsoluteError.from_config(mae_obj.get_config())
+    self.assertEqual(mae_obj2.name, 'my_mae')
+    self.assertEqual(mae_obj2._dtype, dtypes.int32)
+
   def test_unweighted(self):
     mae_obj = metrics.MeanAbsoluteError()
     self.evaluate(variables.variables_initializer(mae_obj.variables))
@@ -1058,6 +1132,12 @@ class MeanAbsolutePercentageErrorTest(test.TestCase):
     self.assertEqual(mape_obj.name, 'my_mape')
     self.assertEqual(mape_obj._dtype, dtypes.int32)
 
+    # Check save and restore config
+    mape_obj2 = metrics.MeanAbsolutePercentageError.from_config(
+        mape_obj.get_config())
+    self.assertEqual(mape_obj2.name, 'my_mape')
+    self.assertEqual(mape_obj2._dtype, dtypes.int32)
+
   def test_unweighted(self):
     mape_obj = metrics.MeanAbsolutePercentageError()
     self.evaluate(variables.variables_initializer(mape_obj.variables))
@@ -1091,6 +1171,11 @@ class MeanSquaredErrorTest(test.TestCase):
     self.assertEqual(mse_obj.name, 'my_mse')
     self.assertEqual(mse_obj._dtype, dtypes.int32)
 
+    # Check save and restore config
+    mse_obj2 = metrics.MeanSquaredError.from_config(mse_obj.get_config())
+    self.assertEqual(mse_obj2.name, 'my_mse')
+    self.assertEqual(mse_obj2._dtype, dtypes.int32)
+
   def test_unweighted(self):
     mse_obj = metrics.MeanSquaredError()
     self.evaluate(variables.variables_initializer(mse_obj.variables))
@@ -1125,6 +1210,12 @@ class MeanSquaredLogarithmicErrorTest(test.TestCase):
     self.assertEqual(msle_obj.name, 'my_msle')
     self.assertEqual(msle_obj._dtype, dtypes.int32)
 
+    # Check save and restore config
+    msle_obj2 = metrics.MeanSquaredLogarithmicError.from_config(
+        msle_obj.get_config())
+    self.assertEqual(msle_obj2.name, 'my_msle')
+    self.assertEqual(msle_obj2._dtype, dtypes.int32)
+
   def test_unweighted(self):
     msle_obj = metrics.MeanSquaredLogarithmicError()
     self.evaluate(variables.variables_initializer(msle_obj.variables))
@@ -1158,6 +1249,11 @@ class HingeTest(test.TestCase):
     self.assertEqual(hinge_obj.name, 'hinge')
     self.assertEqual(hinge_obj._dtype, dtypes.int32)
 
+    # Check save and restore config
+    hinge_obj2 = metrics.Hinge.from_config(hinge_obj.get_config())
+    self.assertEqual(hinge_obj2.name, 'hinge')
+    self.assertEqual(hinge_obj2._dtype, dtypes.int32)
+
   def test_unweighted(self):
     hinge_obj = metrics.Hinge()
     self.evaluate(variables.variables_initializer(hinge_obj.variables))
@@ -1191,6 +1287,11 @@ class SquaredHingeTest(test.TestCase):
     self.assertEqual(sq_hinge_obj.name, 'sq_hinge')
     self.assertEqual(sq_hinge_obj._dtype, dtypes.int32)
 
+    # Check save and restore config
+    sq_hinge_obj2 = metrics.SquaredHinge.from_config(sq_hinge_obj.get_config())
+    self.assertEqual(sq_hinge_obj2.name, 'sq_hinge')
+    self.assertEqual(sq_hinge_obj2._dtype, dtypes.int32)
+
   def test_unweighted(self):
     sq_hinge_obj = metrics.SquaredHinge()
     self.evaluate(variables.variables_initializer(sq_hinge_obj.variables))
@@ -1225,6 +1326,12 @@ class CategoricalHingeTest(test.TestCase):
     self.assertEqual(cat_hinge_obj.name, 'cat_hinge')
     self.assertEqual(cat_hinge_obj._dtype, dtypes.int32)
 
+    # Check save and restore config
+    cat_hinge_obj2 = metrics.CategoricalHinge.from_config(
+        cat_hinge_obj.get_config())
+    self.assertEqual(cat_hinge_obj2.name, 'cat_hinge')
+    self.assertEqual(cat_hinge_obj2._dtype, dtypes.int32)
+
   def test_unweighted(self):
     cat_hinge_obj = metrics.CategoricalHinge()
     self.evaluate(variables.variables_initializer(cat_hinge_obj.variables))
-- 
GitLab


From 96774689252c24259fc9ba894ef5530a80fb043e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 2 Jan 2019 12:18:33 -0800
Subject: [PATCH 0135/1095] Update ops-related pbtxt files.

PiperOrigin-RevId: 227560892
---
 tensorflow/core/ops/compat/ops_history.v1.pbtxt | 16 ++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                   | 16 ++++++++++++++++
 2 files changed, 32 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 8ec03d4fe9..d27f7a5eac 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -76829,6 +76829,22 @@ op {
     }
   }
 }
+op {
+  name: "TensorListResize"
+  input_arg {
+    name: "input_handle"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "size"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "output_handle"
+    type: DT_VARIANT
+  }
+  is_stateful: true
+}
 op {
   name: "TensorListScatter"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 97e68ee9a6..412b4160d9 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -36961,6 +36961,22 @@ op {
     }
   }
 }
+op {
+  name: "TensorListResize"
+  input_arg {
+    name: "input_handle"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "size"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "output_handle"
+    type: DT_VARIANT
+  }
+  is_stateful: true
+}
 op {
   name: "TensorListScatter"
   input_arg {
-- 
GitLab


From f0e5909787ff17f5b22b3efda88c744a7f116207 Mon Sep 17 00:00:00 2001
From: Jian Li <jianlijianli@google.com>
Date: Wed, 2 Jan 2019 12:32:19 -0800
Subject: [PATCH 0136/1095] Mark custom layer norm as deprecated.

PiperOrigin-RevId: 227562915
---
 tensorflow/lite/kernels/layer_norm_lstm.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tensorflow/lite/kernels/layer_norm_lstm.cc b/tensorflow/lite/kernels/layer_norm_lstm.cc
index 49e8a53c82..ce0c21dfcb 100644
--- a/tensorflow/lite/kernels/layer_norm_lstm.cc
+++ b/tensorflow/lite/kernels/layer_norm_lstm.cc
@@ -13,6 +13,10 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+// DEPRECATED: Tensorflow Lite has implemented layer norm lstm as builtin Op and
+// the implementation of layer norm lstm as custom Op in this file is
+// deprecated. It is only kept for backward compatibility.
+//
 // Layer Normalization LSTM op that applies normalization by mean and standard
 // deviation to the activation of the LSTM layers. Please see
 // https://arxiv.org/abs/1607.06450 for details.
-- 
GitLab


From e379cbf262feefc13ebc2d1c978457545510841a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 2 Jan 2019 12:46:32 -0800
Subject: [PATCH 0137/1095] Go: Update generated wrapper functions for
 TensorFlow ops. PiperOrigin-RevId: 227565205

---
 tensorflow/go/op/wrappers.go | 168 +++++++++++++++++------------------
 1 file changed, 84 insertions(+), 84 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index 4c47f12d52..6fe61f512f 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -16991,6 +16991,90 @@ func ResourceApplyRMSProp(scope *Scope, var_ tf.Output, ms tf.Output, mom tf.Out
 	return scope.AddOperation(opspec)
 }
 
+// Store the input tensor in the state of the current session.
+//
+// Arguments:
+//	value: The tensor to be stored.
+//
+// Returns The handle for the tensor stored in the session state, represented
+// as a ResourceHandle object.
+func GetSessionHandleV2(scope *Scope, value tf.Output) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "GetSessionHandleV2",
+		Input: []tf.Input{
+			value,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// ResourceApplyAdamAttr is an optional argument to ResourceApplyAdam.
+type ResourceApplyAdamAttr func(optionalAttr)
+
+// ResourceApplyAdamUseLocking sets the optional use_locking attribute to value.
+//
+// value: If `True`, updating of the var, m, and v tensors will be protected
+// by a lock; otherwise the behavior is undefined, but may exhibit less
+// contention.
+// If not specified, defaults to false
+func ResourceApplyAdamUseLocking(value bool) ResourceApplyAdamAttr {
+	return func(m optionalAttr) {
+		m["use_locking"] = value
+	}
+}
+
+// ResourceApplyAdamUseNesterov sets the optional use_nesterov attribute to value.
+//
+// value: If `True`, uses the nesterov update.
+// If not specified, defaults to false
+func ResourceApplyAdamUseNesterov(value bool) ResourceApplyAdamAttr {
+	return func(m optionalAttr) {
+		m["use_nesterov"] = value
+	}
+}
+
+// Update '*var' according to the Adam algorithm.
+//
+// $$lr_t := \text{learning\_rate} * \sqrt{1 - beta_2^t} / (1 - beta_1^t)$$
+// $$m_t := beta_1 * m_{t-1} + (1 - beta_1) * g$$
+// $$v_t := beta_2 * v_{t-1} + (1 - beta_2) * g * g$$
+// $$variable := variable - lr_t * m_t / (\sqrt{v_t} + \epsilon)$$
+//
+// Arguments:
+//	var_: Should be from a Variable().
+//	m: Should be from a Variable().
+//	v: Should be from a Variable().
+//	beta1_power: Must be a scalar.
+//	beta2_power: Must be a scalar.
+//	lr: Scaling factor. Must be a scalar.
+//	beta1: Momentum factor. Must be a scalar.
+//	beta2: Momentum factor. Must be a scalar.
+//	epsilon: Ridge term. Must be a scalar.
+//	grad: The gradient.
+//
+// Returns the created operation.
+func ResourceApplyAdam(scope *Scope, var_ tf.Output, m tf.Output, v tf.Output, beta1_power tf.Output, beta2_power tf.Output, lr tf.Output, beta1 tf.Output, beta2 tf.Output, epsilon tf.Output, grad tf.Output, optional ...ResourceApplyAdamAttr) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ResourceApplyAdam",
+		Input: []tf.Input{
+			var_, m, v, beta1_power, beta2_power, lr, beta1, beta2, epsilon, grad,
+		},
+		Attrs: attrs,
+	}
+	return scope.AddOperation(opspec)
+}
+
 // SerializeManySparseAttr is an optional argument to SerializeManySparse.
 type SerializeManySparseAttr func(optionalAttr)
 
@@ -28987,90 +29071,6 @@ func AdjustHue(scope *Scope, images tf.Output, delta tf.Output) (output tf.Outpu
 	return op.Output(0)
 }
 
-// ResourceApplyAdamAttr is an optional argument to ResourceApplyAdam.
-type ResourceApplyAdamAttr func(optionalAttr)
-
-// ResourceApplyAdamUseLocking sets the optional use_locking attribute to value.
-//
-// value: If `True`, updating of the var, m, and v tensors will be protected
-// by a lock; otherwise the behavior is undefined, but may exhibit less
-// contention.
-// If not specified, defaults to false
-func ResourceApplyAdamUseLocking(value bool) ResourceApplyAdamAttr {
-	return func(m optionalAttr) {
-		m["use_locking"] = value
-	}
-}
-
-// ResourceApplyAdamUseNesterov sets the optional use_nesterov attribute to value.
-//
-// value: If `True`, uses the nesterov update.
-// If not specified, defaults to false
-func ResourceApplyAdamUseNesterov(value bool) ResourceApplyAdamAttr {
-	return func(m optionalAttr) {
-		m["use_nesterov"] = value
-	}
-}
-
-// Update '*var' according to the Adam algorithm.
-//
-// $$lr_t := \text{learning\_rate} * \sqrt{1 - beta_2^t} / (1 - beta_1^t)$$
-// $$m_t := beta_1 * m_{t-1} + (1 - beta_1) * g$$
-// $$v_t := beta_2 * v_{t-1} + (1 - beta_2) * g * g$$
-// $$variable := variable - lr_t * m_t / (\sqrt{v_t} + \epsilon)$$
-//
-// Arguments:
-//	var_: Should be from a Variable().
-//	m: Should be from a Variable().
-//	v: Should be from a Variable().
-//	beta1_power: Must be a scalar.
-//	beta2_power: Must be a scalar.
-//	lr: Scaling factor. Must be a scalar.
-//	beta1: Momentum factor. Must be a scalar.
-//	beta2: Momentum factor. Must be a scalar.
-//	epsilon: Ridge term. Must be a scalar.
-//	grad: The gradient.
-//
-// Returns the created operation.
-func ResourceApplyAdam(scope *Scope, var_ tf.Output, m tf.Output, v tf.Output, beta1_power tf.Output, beta2_power tf.Output, lr tf.Output, beta1 tf.Output, beta2 tf.Output, epsilon tf.Output, grad tf.Output, optional ...ResourceApplyAdamAttr) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "ResourceApplyAdam",
-		Input: []tf.Input{
-			var_, m, v, beta1_power, beta2_power, lr, beta1, beta2, epsilon, grad,
-		},
-		Attrs: attrs,
-	}
-	return scope.AddOperation(opspec)
-}
-
-// Store the input tensor in the state of the current session.
-//
-// Arguments:
-//	value: The tensor to be stored.
-//
-// Returns The handle for the tensor stored in the session state, represented
-// as a ResourceHandle object.
-func GetSessionHandleV2(scope *Scope, value tf.Output) (handle tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "GetSessionHandleV2",
-		Input: []tf.Input{
-			value,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // ResizeBicubicGradAttr is an optional argument to ResizeBicubicGrad.
 type ResizeBicubicGradAttr func(optionalAttr)
 
-- 
GitLab


From 276a456b3551a329881e0104edf8ea848d446033 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Wed, 2 Jan 2019 13:25:11 -0800
Subject: [PATCH 0138/1095] Remove unused imports

---
 tensorflow/contrib/tensorrt/test/identity_output_test.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/test/identity_output_test.py b/tensorflow/contrib/tensorrt/test/identity_output_test.py
index da7d70876d..b376e30476 100644
--- a/tensorflow/contrib/tensorrt/test/identity_output_test.py
+++ b/tensorflow/contrib/tensorrt/test/identity_output_test.py
@@ -29,9 +29,7 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import gen_array_ops
 from tensorflow.python.ops import math_ops
-from tensorflow.python.ops import nn
 from tensorflow.python.platform import test
 
 
-- 
GitLab


From 7d2be9eef5f4b74da5b5c6f473ea2e6ad57398dc Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 2 Jan 2019 13:03:28 -0800
Subject: [PATCH 0139/1095] Modify TFLite NNAPI interface to use a struct
 instead of using directly function calls. Add an empty implementation for
 NNAPI outside Android.

PiperOrigin-RevId: 227567811
---
 .../lite/delegates/nnapi/nnapi_delegate.cc    |  161 +--
 tensorflow/lite/nnapi/BUILD                   |   12 +
 tensorflow/lite/nnapi/NeuralNetworksShim.cc   |  137 ++
 tensorflow/lite/nnapi/NeuralNetworksShim.h    | 1205 ++++++++---------
 tensorflow/lite/nnapi/nnapi_lib_test.cc       |   90 ++
 tensorflow/lite/nnapi_delegate.cc             |  135 +-
 6 files changed, 931 insertions(+), 809 deletions(-)
 create mode 100644 tensorflow/lite/nnapi/NeuralNetworksShim.cc
 create mode 100644 tensorflow/lite/nnapi/nnapi_lib_test.cc

diff --git a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
index 8af159e6fb..9487d3331f 100644
--- a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
+++ b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
@@ -48,57 +48,39 @@ namespace {
   } while (0)
 
 namespace {
-int32_t GetAndroidSdkVersion() {
-#ifdef __ANDROID__
-  const char* sdkProp = "ro.build.version.sdk";
-  char sdkVersion[PROP_VALUE_MAX];
-  int length = __system_property_get(sdkProp, sdkVersion);
-  if (length != 0) {
-    for (int i = 0; i < length; ++i) {
-      int digit = sdkVersion[i] - '0';
-      if (digit < 0 || digit > 9) {
-        // Non-numeric SDK version, assume it's higher then expected;
-        return std::numeric_limits<int32_t>::max();
-      }
-    }
-    return atoi(sdkVersion);
-  }
-#endif  // __ANDROID__
-  return 0;
-}
-
 constexpr int32_t kMinSdkVersionForNNAPI = 27;
 constexpr int32_t kMinSdkVersionForNNAPI11 = 28;
-static const int32_t kAndroidSdkVersion = GetAndroidSdkVersion();
-
 }  // namespace
 
 // RAII NN API Model Destructor for use with std::unique_ptr
 struct NNFreeModel {
   void operator()(ANeuralNetworksModel* model) {
-    ANeuralNetworksModel_free(model);
+    NnApiImplementation()->ANeuralNetworksModel_free(model);
   }
 };
 // RAII NN API Compilation Destructor for use with std::unique_ptr
 struct NNFreeCompilation {
   void operator()(ANeuralNetworksCompilation* model) {
-    ANeuralNetworksCompilation_free(model);
+    NnApiImplementation()->ANeuralNetworksCompilation_free(model);
   }
 };
 
 // Manage NNAPI shared memory handle
 class NNMemory {
  public:
-  NNMemory(const char* name, size_t size) {
 #ifdef __ANDROID__
+  NNMemory(const NnApi* nnapi, const char* name, size_t size) {
+    nnapi_ = nnapi;
     byte_size_ = size;
-    fd_ = ASharedMemory_create(name, size);
+    fd_ = nnapi_->ASharedMemory_create(name, size);
     data_ptr_ = reinterpret_cast<uint8_t*>(
         mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd_, 0));
-    ANeuralNetworksMemory_createFromFd(size, PROT_READ | PROT_WRITE, fd_, 0,
-                                       &nn_memory_handle_);
-#endif
+    nnapi_->ANeuralNetworksMemory_createFromFd(size, PROT_READ | PROT_WRITE,
+                                               fd_, 0, &nn_memory_handle_);
   }
+#else
+  NNMemory(const NnApi* /*nnapi*/, const char* /*name*/, size_t /*size*/) {}
+#endif
 
   ~NNMemory() {
 #ifdef __ANDROID__
@@ -106,7 +88,7 @@ class NNMemory {
       munmap(data_ptr_, byte_size_);
     }
     if (nn_memory_handle_) {
-      ANeuralNetworksMemory_free(nn_memory_handle_);
+      nnapi_->ANeuralNetworksMemory_free(nn_memory_handle_);
     }
     if (fd_ > 0) close(fd_);
 #endif
@@ -117,6 +99,7 @@ class NNMemory {
 
  private:
 #ifdef __ANDROID__
+  const NnApi* nnapi_;
   int fd_ = 0;
   size_t byte_size_ = 0;
 #endif
@@ -166,9 +149,10 @@ class OperandMapping {
 // operands for both tensors and parameters, and TFLite separates the two.
 class NNAPIOpBuilder {
  public:
-  NNAPIOpBuilder(TfLiteContext* context, OperandMapping* tensor_mapping,
-                 ANeuralNetworksModel* nn_model)
-      : context_(context),
+  NNAPIOpBuilder(const NnApi* nnapi, TfLiteContext* context,
+                 OperandMapping* tensor_mapping, ANeuralNetworksModel* nn_model)
+      : nnapi_(nnapi),
+        context_(context),
         operand_mapping_(tensor_mapping),
         nn_model_(nn_model) {}
 
@@ -224,7 +208,8 @@ class NNAPIOpBuilder {
         .dimensionCount = dimension_count,
         .dimensions = dims.data()};
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context_, ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
+        context_,
+        nnapi_->ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
     int ann_operand = operand_mapping_->add_new_non_tensor_operand();
     augmented_outputs_.push_back(ann_operand);
     return kTfLiteOk;
@@ -241,7 +226,8 @@ class NNAPIOpBuilder {
         reinterpret_cast<uint32_t*>(tensor->dims->data), tensor->params.scale,
         tensor->params.zero_point};
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context_, ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
+        context_,
+        nnapi_->ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
     augmented_outputs_.push_back(ann_index);
 
     *ann_tensor_index_out = ann_index;
@@ -298,13 +284,14 @@ class NNAPIOpBuilder {
         nn_type, static_cast<uint32_t>(tensor->dims->size),
         reinterpret_cast<uint32_t*>(tensor->dims->data), scale, zeroPoint};
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context_, ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
+        context_,
+        nnapi_->ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
 
     if (tensor->allocation_type == kTfLiteMmapRo) {
       // TODO(b/80630405): Use NNAPIAllocation.
       RETURN_TFLITE_ERROR_IF_NN_ERROR(
           context_,
-          ANeuralNetworksModel_setOperandValue(
+          nnapi_->ANeuralNetworksModel_setOperandValue(
               nn_model_, ann_tensor_index, tensor->data.raw, tensor->bytes));
     }
 
@@ -317,7 +304,7 @@ class NNAPIOpBuilder {
     // Actually add a NN API operation
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
         context_,
-        ANeuralNetworksModel_addOperation(
+        nnapi_->ANeuralNetworksModel_addOperation(
             nn_model_, type, static_cast<uint32_t>(augmented_inputs_.size()),
             augmented_inputs_.data(),
             static_cast<uint32_t>(augmented_outputs_.size()),
@@ -332,11 +319,12 @@ class NNAPIOpBuilder {
   TfLiteStatus AddScalarOperand(T value, int32_t nn_type) {
     ANeuralNetworksOperandType operand_type{.type = nn_type};
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context_, ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
+        context_,
+        nnapi_->ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
     int ann_operand = operand_mapping_->add_new_non_tensor_operand();
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context_, ANeuralNetworksModel_setOperandValue(nn_model_, ann_operand,
-                                                       &value, sizeof(T)));
+        context_, nnapi_->ANeuralNetworksModel_setOperandValue(
+                      nn_model_, ann_operand, &value, sizeof(T)));
     augmented_inputs_.push_back(ann_operand);
     return kTfLiteOk;
   }
@@ -347,15 +335,19 @@ class NNAPIOpBuilder {
     ANeuralNetworksOperandType operand_type{
         .type = nn_type, .dimensionCount = 1, .dimensions = &num_values};
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context_, ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
+        context_,
+        nnapi_->ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
     int ann_operand = operand_mapping_->add_new_non_tensor_operand();
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context_, ANeuralNetworksModel_setOperandValue(
+        context_, nnapi_->ANeuralNetworksModel_setOperandValue(
                       nn_model_, ann_operand, values, sizeof(T) * num_values));
     augmented_inputs_.push_back(ann_operand);
     return kTfLiteOk;
   }
 
+  // Access to NNAPI.
+  const NnApi* const nnapi_;
+
   // TfLiteContext for error handling.
   TfLiteContext* const context_;
 
@@ -391,7 +383,7 @@ ANeuralNetworksOperationType BasicMappingFn(
 // The kernel that represents the node sub set of TF Lite being run on NN API.
 class NNAPIDelegateKernel {
  public:
-  NNAPIDelegateKernel() = default;
+  NNAPIDelegateKernel() { nnapi_ = NnApiImplementation(); }
 
   typedef ANeuralNetworksOperationType (*MappingFn)(
       const NNAPIOpMappingArgs& mapping_args);
@@ -400,7 +392,7 @@ class NNAPIDelegateKernel {
   // when called. You can use this function to see if a node is supported
   // (i.e. that MappingFn is not nullptr).
   static MappingFn Map(TfLiteContext* context, int builtin_code, int version,
-                       TfLiteNode* node) {
+                       int android_sdk_version, TfLiteNode* node) {
     switch (builtin_code) {
       case kTfLiteBuiltinAdd:
         if (version == 1) {
@@ -519,7 +511,7 @@ class NNAPIDelegateKernel {
         }
         break;
       case kTfLiteBuiltinSqueeze:
-        if (version == 1 && kAndroidSdkVersion >= kMinSdkVersionForNNAPI11) {
+        if (version == 1 && android_sdk_version >= kMinSdkVersionForNNAPI11) {
           return [](const NNAPIOpMappingArgs& mapping_args)
                      -> ANeuralNetworksOperationType {
             auto builtin = reinterpret_cast<TfLiteSqueezeParams*>(
@@ -635,7 +627,7 @@ class NNAPIDelegateKernel {
         }
         break;
       case kTfLiteBuiltinSub:
-        if (version == 1 && kAndroidSdkVersion >= kMinSdkVersionForNNAPI11 &&
+        if (version == 1 && android_sdk_version >= kMinSdkVersionForNNAPI11 &&
             context->tensors[node->inputs->data[0]].type == kTfLiteFloat32) {
           // NNAPI only support float sub.
           return [](const NNAPIOpMappingArgs& mapping_args)
@@ -648,7 +640,7 @@ class NNAPIDelegateKernel {
         }
         break;
       case kTfLiteBuiltinDiv:
-        if (version == 1 && kAndroidSdkVersion >= kMinSdkVersionForNNAPI11 &&
+        if (version == 1 && android_sdk_version >= kMinSdkVersionForNNAPI11 &&
             context->tensors[node->inputs->data[0]].type == kTfLiteFloat32) {
           // NNAPI only support float div.
           return [](const NNAPIOpMappingArgs& mapping_args)
@@ -661,7 +653,7 @@ class NNAPIDelegateKernel {
         }
         break;
       case kTfLiteBuiltinPad:
-        if (version == 1 && kAndroidSdkVersion >= kMinSdkVersionForNNAPI11 &&
+        if (version == 1 && android_sdk_version >= kMinSdkVersionForNNAPI11 &&
             node->inputs->size == 2 &&
             context->tensors[node->inputs->data[0]].type == kTfLiteFloat32) {
           // NNAPI does not support specifying the padding value.
@@ -671,12 +663,12 @@ class NNAPIDelegateKernel {
         }
         break;
       case kTfLiteBuiltinSpaceToBatchNd:
-        if (version == 1 && kAndroidSdkVersion >= kMinSdkVersionForNNAPI11) {
+        if (version == 1 && android_sdk_version >= kMinSdkVersionForNNAPI11) {
           return BasicMappingFn<ANEURALNETWORKS_SPACE_TO_BATCH_ND>;
         }
         break;
       case kTfLiteBuiltinStridedSlice:
-        if (version == 1 && kAndroidSdkVersion >= kMinSdkVersionForNNAPI11) {
+        if (version == 1 && android_sdk_version >= kMinSdkVersionForNNAPI11) {
           return [](const NNAPIOpMappingArgs& mapping_args)
                      -> ANeuralNetworksOperationType {
             auto builtin = reinterpret_cast<TfLiteStridedSliceParams*>(
@@ -694,7 +686,7 @@ class NNAPIDelegateKernel {
         // dimensions.
         // TODO(b/110888333): Support dynamically-sized tensors in delegates.
         if ((version == 1) &&
-            (kAndroidSdkVersion >= kMinSdkVersionForNNAPI11) &&
+            (android_sdk_version >= kMinSdkVersionForNNAPI11) &&
             (node->inputs->size > 1) &&
             (context->tensors[node->inputs->data[1]].allocation_type ==
              kTfLiteMmapRo)) {
@@ -792,7 +784,7 @@ class NNAPIDelegateKernel {
         break;
       case kTfLiteBuiltinMean:
         // NNAPI does not support generating a scalar as output for MEAN.
-        if (version == 1 && kAndroidSdkVersion >= kMinSdkVersionForNNAPI11 &&
+        if (version == 1 && android_sdk_version >= kMinSdkVersionForNNAPI11 &&
             context->tensors[node->inputs->data[0]].type == kTfLiteFloat32 &&
             context->tensors[node->outputs->data[0]].dims->size > 0) {
           return [](const NNAPIOpMappingArgs& mapping_args)
@@ -836,8 +828,8 @@ class NNAPIDelegateKernel {
 
     if (!nn_model_) {
       ANeuralNetworksModel* model;
-      RETURN_TFLITE_ERROR_IF_NN_ERROR(context,
-                                      ANeuralNetworksModel_create(&model));
+      RETURN_TFLITE_ERROR_IF_NN_ERROR(
+          context, nnapi_->ANeuralNetworksModel_create(&model));
       nn_model_.reset(model);
 
       TF_LITE_ENSURE_STATUS(
@@ -847,10 +839,10 @@ class NNAPIDelegateKernel {
     if (!nn_compilation_) {
       ANeuralNetworksCompilation* compilation;
       RETURN_TFLITE_ERROR_IF_NN_ERROR(
-          context,
-          ANeuralNetworksCompilation_create(nn_model_.get(), &compilation));
+          context, nnapi_->ANeuralNetworksCompilation_create(nn_model_.get(),
+                                                             &compilation));
       RETURN_TFLITE_ERROR_IF_NN_ERROR(
-          context, ANeuralNetworksCompilation_finish(compilation));
+          context, nnapi_->ANeuralNetworksCompilation_finish(compilation));
       nn_compilation_.reset(compilation);
     }
     return kTfLiteOk;
@@ -859,8 +851,8 @@ class NNAPIDelegateKernel {
   TfLiteStatus Invoke(TfLiteContext* context, TfLiteNode* node) {
     ANeuralNetworksExecution* execution = nullptr;
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context,
-        ANeuralNetworksExecution_create(nn_compilation_.get(), &execution));
+        context, nnapi_->ANeuralNetworksExecution_create(nn_compilation_.get(),
+                                                         &execution));
 
     // Set the input tensor buffers. Note: we access tflite tensors using
     // absolute indices but NN api indices inputs by relative indices.
@@ -880,7 +872,7 @@ class NNAPIDelegateKernel {
                tensor->data.raw, tensor->bytes);
         RETURN_TFLITE_ERROR_IF_NN_ERROR(
             context,
-            ANeuralNetworksExecution_setInputFromMemory(
+            nnapi_->ANeuralNetworksExecution_setInputFromMemory(
                 execution, relative_input_index, nullptr,
                 nn_input_memory_->get_handle(), input_offset, tensor->bytes));
         input_offset += tensor->bytes;
@@ -895,7 +887,7 @@ class NNAPIDelegateKernel {
       TfLiteTensor* tensor = &context->tensors[output_index];
       RETURN_TFLITE_ERROR_IF_NN_ERROR(
           context,
-          ANeuralNetworksExecution_setOutputFromMemory(
+          nnapi_->ANeuralNetworksExecution_setOutputFromMemory(
               execution, relative_output_index, nullptr,
               nn_output_memory_->get_handle(), output_offset, tensor->bytes));
       output_offset += tensor->bytes;
@@ -911,7 +903,7 @@ class NNAPIDelegateKernel {
       // reading and writing into the same buffer during a invocation.
       // TODO(110369471): using double shared buffer to minimize the copies.
       RETURN_TFLITE_ERROR_IF_NN_ERROR(
-          context, ANeuralNetworksExecution_setOutput(
+          context, nnapi_->ANeuralNetworksExecution_setOutput(
                        execution, relative_output_index, nullptr,
                        tensor->data.raw, tensor->bytes));
       relative_output_index++;
@@ -919,10 +911,12 @@ class NNAPIDelegateKernel {
     // Invoke ANN in blocking fashion.
     ANeuralNetworksEvent* event = nullptr;
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context, ANeuralNetworksExecution_startCompute(execution, &event));
-    RETURN_TFLITE_ERROR_IF_NN_ERROR(context, ANeuralNetworksEvent_wait(event));
-    ANeuralNetworksEvent_free(event);
-    ANeuralNetworksExecution_free(execution);
+        context,
+        nnapi_->ANeuralNetworksExecution_startCompute(execution, &event));
+    RETURN_TFLITE_ERROR_IF_NN_ERROR(context,
+                                    nnapi_->ANeuralNetworksEvent_wait(event));
+    nnapi_->ANeuralNetworksEvent_free(event);
+    nnapi_->ANeuralNetworksExecution_free(execution);
 
     // copy results from shared memory to the destination.
     output_offset = 0;
@@ -937,6 +931,8 @@ class NNAPIDelegateKernel {
   }
 
  private:
+  // Access to NNApi.
+  const NnApi* nnapi_;
   // ANN API state.
   std::unique_ptr<ANeuralNetworksModel, NNFreeModel> nn_model_;
   std::unique_ptr<ANeuralNetworksCompilation, NNFreeCompilation>
@@ -957,7 +953,7 @@ class NNAPIDelegateKernel {
     // The operand builder allows creating a single op. We create it at this
     // reduced power position rather than in the for loop to avoid reallocating
     // the vectors.
-    NNAPIOpBuilder builder(context, &operand_mapping_, nn_model_.get());
+    NNAPIOpBuilder builder(nnapi_, context, &operand_mapping_, nn_model_.get());
     // Add Tensors
     // allocate outside to avoid realloc
     for (auto node_index : nodes_) {
@@ -980,9 +976,10 @@ class NNAPIDelegateKernel {
         }
       }
       // Get op type and operands
-      int nn_op_type = Map(context, reg->builtin_code, reg->version, node)(
-          {context, &builder, node, &model_state_outputs_,
-           &model_state_tfl_inputs_});
+      int nn_op_type = Map(
+          context, reg->builtin_code, reg->version, nnapi_->android_sdk_version,
+          node)({context, &builder, node, &model_state_outputs_,
+                 &model_state_tfl_inputs_});
       // Map outputs to NN API tensor indices.
       for (auto output_index : TfLiteIntArrayView(node->outputs)) {
         TF_LITE_ENSURE_STATUS(builder.AddTensorOutput(output_index));
@@ -1028,25 +1025,27 @@ class NNAPIDelegateKernel {
 
     // Tell ANN to declare inputs/outputs
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context, ANeuralNetworksModel_identifyInputsAndOutputs(
+        context, nnapi_->ANeuralNetworksModel_identifyInputsAndOutputs(
                      nn_model_.get(), inputs.size(), inputs.data(),
                      outputs.size(), outputs.data()));
 
     // Set relaxed computation mode for fp32 if possible.
-    if (kAndroidSdkVersion >= kMinSdkVersionForNNAPI11) {
+    if (nnapi_->android_sdk_version >= kMinSdkVersionForNNAPI11) {
       RETURN_TFLITE_ERROR_IF_NN_ERROR(
-          context, ANeuralNetworksModel_relaxComputationFloat32toFloat16(
-                       nn_model_.get(), context->allow_fp32_relax_to_fp16));
+          context,
+          nnapi_->ANeuralNetworksModel_relaxComputationFloat32toFloat16(
+              nn_model_.get(), context->allow_fp32_relax_to_fp16));
     }
 
     // Finalize the model
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context, ANeuralNetworksModel_finish(nn_model_.get()));
+        context, nnapi_->ANeuralNetworksModel_finish(nn_model_.get()));
 
     // Create shared memory pool for inputs and outputs.
-    nn_input_memory_.reset(new NNMemory("input_pool", total_input_byte_size));
+    nn_input_memory_.reset(
+        new NNMemory(nnapi_, "input_pool", total_input_byte_size));
     nn_output_memory_.reset(
-        new NNMemory("output_pool", total_output_byte_size));
+        new NNMemory(nnapi_, "output_pool", total_output_byte_size));
 
     return kTfLiteOk;
   }
@@ -1062,7 +1061,9 @@ TfLiteDelegate* NnApiDelegate() {
       .Prepare = [](TfLiteContext* context,
                     TfLiteDelegate* delegate) -> TfLiteStatus {
         // Do not check nodes_ if NN API is unavailable.
-        if (kAndroidSdkVersion < kMinSdkVersionForNNAPI || !NNAPIExists()) {
+        const NnApi* nnapi = NnApiImplementation();
+        if (nnapi->android_sdk_version < kMinSdkVersionForNNAPI ||
+            !nnapi->nnapi_exists) {
           return kTfLiteOk;
         }
 
@@ -1075,6 +1076,7 @@ TfLiteDelegate* NnApiDelegate() {
         TfLiteIntArray* plan;
         TF_LITE_ENSURE_STATUS(context->GetExecutionPlan(context, &plan));
 
+        int android_sdk_version = NnApiImplementation()->android_sdk_version;
         // Check for every node if it is supported
         // TODO(b/80625235): Fix this to do more careful checking of versioning.
         for (int node_index : TfLiteIntArrayView(plan)) {
@@ -1083,7 +1085,8 @@ TfLiteDelegate* NnApiDelegate() {
           TF_LITE_ENSURE_STATUS(context->GetNodeAndRegistration(
               context, node_index, &node, &registration));
           if (NNAPIDelegateKernel::Map(context, registration->builtin_code,
-                                       registration->version, node)) {
+                                       registration->version,
+                                       android_sdk_version, node)) {
             supported_nodes.push_back(node_index);
           }
         }
diff --git a/tensorflow/lite/nnapi/BUILD b/tensorflow/lite/nnapi/BUILD
index 467a2b7a7b..390c3730cb 100644
--- a/tensorflow/lite/nnapi/BUILD
+++ b/tensorflow/lite/nnapi/BUILD
@@ -6,8 +6,20 @@ package(default_visibility = [
 
 cc_library(
     name = "nnapi_lib",
+    srcs = [
+        "NeuralNetworksShim.cc",
+    ],
     hdrs = [
         "NeuralNetworksShim.h",
     ],
     linkopts = ["-ldl"],
 )
+
+cc_test(
+    name = "nnapi_lib_test",
+    srcs = ["nnapi_lib_test.cc"],
+    deps = [
+        "//tensorflow/lite/nnapi:nnapi_lib",
+        "@com_google_googletest//:gtest_main",
+    ],
+)
diff --git a/tensorflow/lite/nnapi/NeuralNetworksShim.cc b/tensorflow/lite/nnapi/NeuralNetworksShim.cc
new file mode 100644
index 0000000000..c063525cae
--- /dev/null
+++ b/tensorflow/lite/nnapi/NeuralNetworksShim.cc
@@ -0,0 +1,137 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/nnapi/NeuralNetworksShim.h"
+
+#include <cstdlib>
+
+#ifdef __ANDROID__
+#include <dlfcn.h>
+#include <sys/mman.h>
+#include <sys/system_properties.h>
+#include <unistd.h>
+#endif
+
+#define NNAPI_LOG(format, ...) fprintf(stderr, format "\n", __VA_ARGS__);
+
+namespace {
+
+#ifdef __ANDROID__
+int32_t GetAndroidSdkVersion() {
+  const char* sdkProp = "ro.build.version.sdk";
+  char sdkVersion[PROP_VALUE_MAX];
+  int length = __system_property_get(sdkProp, sdkVersion);
+  if (length != 0) {
+    int32_t result = 0;
+    for (int i = 0; i < length; ++i) {
+      int digit = sdkVersion[i] - '0';
+      if (digit < 0 || digit > 9) {
+        // Non-numeric SDK version, assume it's higher than expected;
+        return 0xffff;
+      }
+      result = result * 10 + digit;
+    }
+    return result;
+  }
+  return 0;
+}
+
+void* LoadFunction(void* handle, const char* name) {
+  if (handle == nullptr) {
+    return nullptr;
+  }
+  void* fn = dlsym(handle, name);
+  if (fn == nullptr) {
+    NNAPI_LOG("nnapi error: unable to open function %s", name);
+  }
+  return fn;
+}
+
+#define LOAD_FUNCTION(handle, name) \
+  nnapi.name = reinterpret_cast<name##_fn>(LoadFunction(handle, #name));
+
+#else
+
+#define LOAD_FUNCTION(handle, name) nnapi.name = nullptr;
+
+#endif
+
+const NnApi LoadNnApi() {
+  NnApi nnapi = {};
+
+#ifdef __ANDROID__
+  void* libneuralnetworks = nullptr;
+  void* libandroid = nullptr;
+  nnapi.android_sdk_version = GetAndroidSdkVersion();
+  if (nnapi.android_sdk_version < 27) {
+    NNAPI_LOG("nnapi error: requires android sdk version to be at least %d",
+              27);
+  } else {
+    // TODO: change RTLD_LOCAL? Assumes there can be multiple instances of nn
+    // api RT
+    libneuralnetworks = dlopen("libneuralnetworks.so", RTLD_LAZY | RTLD_LOCAL);
+    if (libneuralnetworks == nullptr) {
+      NNAPI_LOG("nnapi error: unable to open library %s",
+                "libneuralnetworks.so");
+    }
+    libandroid = dlopen("libandroid.so", RTLD_LAZY | RTLD_LOCAL);
+    if (libneuralnetworks == nullptr) {
+      NNAPI_LOG("nnapi error: unable to open library %s", "libandroid.so");
+    }
+  }
+  nnapi.nnapi_exists = libneuralnetworks != nullptr;
+#else
+  nnapi.nnapi_exists = false;
+  nnapi.android_sdk_version = 0;
+#endif
+
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksMemory_createFromFd);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksMemory_free);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksModel_create);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksModel_free);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksModel_finish);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksModel_addOperand);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksModel_setOperandValue);
+  LOAD_FUNCTION(libneuralnetworks,
+                ANeuralNetworksModel_setOperandValueFromMemory);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksModel_addOperation);
+  LOAD_FUNCTION(libneuralnetworks,
+                ANeuralNetworksModel_identifyInputsAndOutputs);
+  LOAD_FUNCTION(libneuralnetworks,
+                ANeuralNetworksModel_relaxComputationFloat32toFloat16);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksCompilation_create);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksCompilation_free);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksCompilation_setPreference);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksCompilation_finish);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksExecution_create);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksExecution_free);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksExecution_setInput);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksExecution_setInputFromMemory);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksExecution_setOutput);
+  LOAD_FUNCTION(libneuralnetworks,
+                ANeuralNetworksExecution_setOutputFromMemory);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksExecution_startCompute);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksEvent_wait);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksEvent_free);
+  LOAD_FUNCTION(libandroid, ASharedMemory_create);
+
+  return nnapi;
+}
+
+}  // namespace
+
+const NnApi* NnApiImplementation() {
+  static const NnApi nnapi = LoadNnApi();
+  return &nnapi;
+}
diff --git a/tensorflow/lite/nnapi/NeuralNetworksShim.h b/tensorflow/lite/nnapi/NeuralNetworksShim.h
index c39502f4ac..3fd869e4af 100644
--- a/tensorflow/lite/nnapi/NeuralNetworksShim.h
+++ b/tensorflow/lite/nnapi/NeuralNetworksShim.h
@@ -15,69 +15,10 @@ limitations under the License.
 #ifndef TENSORFLOW_LITE_NNAPI_NEURALNETWORKSSHIM_H_
 #define TENSORFLOW_LITE_NNAPI_NEURALNETWORKSSHIM_H_
 
-#include <dlfcn.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 
-// helpers
-
-#define NNAPI_LOG(format, ...) fprintf(stderr, format "\n", __VA_ARGS__);
-#define LOAD_FUNCTION(name) \
-  static name##_fn fn = reinterpret_cast<name##_fn>(loadFunction(#name));
-#define EXECUTE_FUNCTION(...) \
-  if (fn != nullptr) {        \
-    fn(__VA_ARGS__);          \
-  }
-#define EXECUTE_FUNCTION_RETURN(...) return fn != nullptr ? fn(__VA_ARGS__) : 0;
-
-inline void* loadLibrary(const char* name) {
-  // TODO: change RTLD_LOCAL? Assumes there can be multiple instances of nn
-  // api RT
-  void* handle = nullptr;
-#ifdef __ANDROID__
-  handle = dlopen(name, RTLD_LAZY | RTLD_LOCAL);
-  if (handle == nullptr) {
-    NNAPI_LOG("nnapi error: unable to open library %s", name);
-  }
-#endif
-  return handle;
-}
-
-typedef int (*ASharedMemory_create_fn)(const char* name, size_t size);
-
-// ASharedMemory_create was added in Android 8.0, so safe to use with NNAPI
-// which was added in 8.1.
-inline int ASharedMemory_create(const char* name, size_t size) {
-  static void* handle = loadLibrary("libandroid.so");
-  static ASharedMemory_create_fn fn =
-      handle != nullptr ? reinterpret_cast<ASharedMemory_create_fn>(
-                              dlsym(handle, "ASharedMemory_create"))
-                        : nullptr;
-  return fn(name, size);
-}
-
-inline void* getLibraryHandle() {
-  static void* handle = loadLibrary("libneuralnetworks.so");
-  return handle;
-}
-
-inline void* loadFunction(const char* name) {
-  void* fn = nullptr;
-  if (getLibraryHandle() != nullptr) {
-    fn = dlsym(getLibraryHandle(), name);
-  }
-  if (fn == nullptr) {
-    NNAPI_LOG("nnapi error: unable to open function %s", name);
-  }
-  return fn;
-}
-
-inline bool NNAPIExists() {
-  static bool nnapi_is_available = getLibraryHandle();
-  return nnapi_is_available;
-}
-
 // NN api types based on NNAPI header file
 // https://developer.android.com/ndk/reference/group/neural-networks
 
@@ -407,606 +348,564 @@ typedef int (*ANeuralNetworksEvent_wait_fn)(ANeuralNetworksEvent* event);
 
 typedef void (*ANeuralNetworksEvent_free_fn)(ANeuralNetworksEvent* event);
 
-/**
- * Creates a shared memory object from a file descriptor.
- *
- * The shared memory is backed by a file descriptor via mmap.
- * See {@link ANeuralNetworksMemory} for a description on how to use
- * this shared memory.
- *
- * @param size The requested size in bytes.
- *             Must not be larger than the file size.
- * @param prot The desired memory protection for the mapping.
- *             It is either PROT_NONE or the bitwise OR of one or
- *             more of the following flags: PROT_READ, PROT_WRITE.
- * @param fd The requested file descriptor.
- *           The file descriptor has to be mmap-able. The file
- *           descriptor will be duplicated.
- * @param offset The offset to the beginning of the file of the area to map.
- *               The offset has to be aligned to a page size.
- * @param memory The memory object to be created.
- *               Set to NULL if unsuccessful.
- *
- * @return ANEURALNETWORKS_NO_ERROR if the request completed normally.
- */
-inline int ANeuralNetworksMemory_createFromFd(size_t size, int protect, int fd,
-                                              size_t offset,
-                                              ANeuralNetworksMemory** memory) {
-  LOAD_FUNCTION(ANeuralNetworksMemory_createFromFd);
-  EXECUTE_FUNCTION_RETURN(size, protect, fd, offset, memory);
-}
-
-/**
- * Delete a memory object.
- *
- * Destroys the object used by the run time to keep track of the memory.
- * This will free the underlying actual memory if no other code has open
- * handles to this memory.
- *
- * @param memory The memory object to be freed.
- */
-inline void ANeuralNetworksMemory_free(ANeuralNetworksMemory* memory) {
-  LOAD_FUNCTION(ANeuralNetworksMemory_free);
-  EXECUTE_FUNCTION(memory);
-}
-
-/**
- * Create an empty {@link ANeuralNetworksModel}.
- *
- * <p>This only creates the object. Computation is performed once
- * {@link ANeuralNetworksExecution_startCompute} is invoked.
- *
- * The model should be constructed with calls to
- * {@link ANeuralNetworksModel_addOperation} and
- * {@link ANeuralNetworksModel_addOperand}
- *
- * <p>{@link ANeuralNetworksModel_finish} should be called once the model
- * has been fully constructed.</p>
- *
- * <p>{@link ANeuralNetworksModel_free} should be called once the model
- * is no longer needed.</p>
- *
- * @param model The {@link ANeuralNetworksModel} to be created.
- *              Set to NULL if unsuccessful.
- *
- * @return ANEURALNETWORKS_NO_ERROR if successful.
- */
-inline int ANeuralNetworksModel_create(ANeuralNetworksModel** model) {
-  LOAD_FUNCTION(ANeuralNetworksModel_create);
-  EXECUTE_FUNCTION_RETURN(model);
-}
-
-/**
- * Destroy a model.
- *
- * The model need not have been finished by a call to
- * {@link ANeuralNetworksModel_finish}.
- *
- * See {@link ANeuralNetworksModel} for information on multithreaded usage.
- *
- * @param model The model to be destroyed. Passing NULL is acceptable and
- *              results in no operation.
- */
-inline void ANeuralNetworksModel_free(ANeuralNetworksModel* model) {
-  LOAD_FUNCTION(ANeuralNetworksModel_free);
-  EXECUTE_FUNCTION(model);
-}
-
-/**
- * Indicate that we have finished modifying a model. Required before
- * calling {@link ANeuralNetworksCompilation_compile}.
- *
- * An application is responsible to make sure that no other thread uses
- * the model at the same time.
- *
- * See {@link ANeuralNetworksModel} for information on multithreaded usage.
- *
- * @param model The model to be finished.
- *
- * @return ANEURALNETWORKS_NO_ERROR if successful.
- */
-inline int ANeuralNetworksModel_finish(ANeuralNetworksModel* model) {
-  LOAD_FUNCTION(ANeuralNetworksModel_finish);
-  EXECUTE_FUNCTION_RETURN(model);
-}
-
-/**
- * Add an operand to a model.
- *
- * The order in which the operands are added is important. The first one added
- * to a model will have the index value 0, the second 1, etc. These indexes are
- * used as operand identifiers in {@link ANeuralNetworksModel_addOperation},
- * {@link ANeuralNetworksExecution_setInput},
- * {@link ANeuralNetworksExecution_setInputFromMemory},
- * {@link ANeuralNetworksExecution_setOutput},
- * {@link ANeuralNetworksExecution_setOutputFromMemory} and
- * {@link ANeuralNetworksExecution_setOperandValue}.
- *
- * To build a model that can accommodate inputs of various sizes, as you may
- * want to do for a CNN, set the size of the dimensions that will vary at run
- * time to 0. If you do so, provide the full dimensions when calling
- * {@link ANeuralNetworksExecution_setInput} or {@link
- * ANeuralNetworksExecution_setInputFromMemory}.
- *
- * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
- * been called will return an error.
- *
- * See {@link ANeuralNetworksModel} for information on multithreaded usage.
- *
- * @param model The model to be modified.
- * @param type The {@link ANeuralNetworksOperandType} that describes the shape
- * of the operand.
- *
- * @return ANEURALNETWORKS_NO_ERROR if successful.
- */
-inline int ANeuralNetworksModel_addOperand(
-    ANeuralNetworksModel* model, const ANeuralNetworksOperandType* type) {
-  LOAD_FUNCTION(ANeuralNetworksModel_addOperand);
-  EXECUTE_FUNCTION_RETURN(model, type);
-}
-
-/**
- * Sets an operand to a constant value.
- *
- * For scalar values, the content of buffer is copied into the model.
- *
- * For tensor values, a pointer to the buffer is stored within the model.
- * The application is responsible for not changing the content of this region
- * until all executions using this model have completed. As the data may
- * be copied during processing, modifying the data after this call yields
- * undefined results.
- *
- * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
- * been called will return an error.
- *
- * See {@link ANeuralNetworksModel} for information on multithreaded usage.
- *
- * @param model The model to be modified.
- * @param index The index of the model operand we're setting.
- * @param buffer A pointer to the data to use.
- * @param length The size in bytes of the data value.
- *
- * @return ANEURALNETWORKS_NO_ERROR if successful.
- */
-inline int ANeuralNetworksModel_setOperandValue(ANeuralNetworksModel* model,
-                                                int32_t index,
-                                                const void* buffer,
-                                                size_t length) {
-  LOAD_FUNCTION(ANeuralNetworksModel_setOperandValue);
-  EXECUTE_FUNCTION_RETURN(model, index, buffer, length);
-}
-
-/**
- * Sets an operand to a value stored in a memory object.
- *
- * The content of the memory is not copied. A reference to that memory is stored
- * inside the model. The application is responsible for not changing the content
- * of the memory region until all executions using this model have completed.
- * As the data may be copied during processing, modifying the data after this
- * call yields undefined results.
- *
- * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
- * been called will return an error.
- *
- * See {@link ANeuralNetworksModel} for information on multithreaded usage.
- *
- * @param model The model to be modified.
- * @param index The index of the model operand we're setting.
- * @param buffer A pointer to the data to use.
- * @param memory The memory containing the data.
- * @param offset This specifies the location of the data within the memory.
- *               The offset is in bytes from the start of memory.
- * @param length The size in bytes of the data value.
- *
- * @return ANEURALNETWORKS_NO_ERROR if successful.
- */
-inline int ANeuralNetworksModel_setOperandValueFromMemory(
-    ANeuralNetworksModel* model, int32_t index,
-    const ANeuralNetworksMemory* memory, size_t offset, size_t length) {
-  LOAD_FUNCTION(ANeuralNetworksModel_setOperandValueFromMemory);
-  EXECUTE_FUNCTION_RETURN(model, index, memory, offset, length);
-}
-
-/**
- * Add an operation to a model.
- *
- * @param model The model to be modified.
- * @param type The type of the operation.
- * @param inputCount The number of entries in the inputs array.
- * @param inputs An array of indexes identifying each operand.
- * @param outputCount The number of entries in the outputs array.
- * @param outputs An array of indexes identifying each operand.
- *
- * The operands specified by inputs and outputs must have been
- * previously added by calls to {@link ANeuralNetworksModel_addOperand}.
- *
- * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
- * been called will return an error.
- *
- * See {@link ANeuralNetworksModel} for information on multithreaded usage.
- *
- * @return ANEURALNETWORKS_NO_ERROR if successful.
- */
-inline int ANeuralNetworksModel_addOperation(ANeuralNetworksModel* model,
-                                             ANeuralNetworksOperationType type,
-                                             uint32_t inputCount,
-                                             const uint32_t* inputs,
-                                             uint32_t outputCount,
-                                             const uint32_t* outputs) {
-  LOAD_FUNCTION(ANeuralNetworksModel_addOperation);
-  EXECUTE_FUNCTION_RETURN(model, type, inputCount, inputs, outputCount,
-                          outputs);
-}
-
-/**
- * Specifies which operands will be the model's inputs and outputs.
- *
- * An operand cannot be used for both input and output. Doing so will
- * return an error.
- *
- * @param model The model to be modified.
- * @param inputCount The number of entries in the inputs array.
- * @param inputs An array of indexes identifying the input operands.
- * @param outputCount The number of entries in the outputs array.
- * @param outputs An array of indexes identifying the output operands.
- *
- * The operands specified by inputs and outputs must have been
- * previously added by calls to {@link ANeuralNetworksModel_addOperand}.
- *
- * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
- * been called will return an error.
- *
- * See {@link ANeuralNetworksModel} for information on multithreaded usage.
- *
- */
-inline int ANeuralNetworksModel_identifyInputsAndOutputs(
-    ANeuralNetworksModel* model, uint32_t inputCount, const uint32_t* inputs,
-    uint32_t outputCount, const uint32_t* outputs) {
-  LOAD_FUNCTION(ANeuralNetworksModel_identifyInputsAndOutputs);
-  EXECUTE_FUNCTION_RETURN(model, inputCount, inputs, outputCount, outputs);
-}
-
-/**
- * Specifies whether {@link ANEURALNETWORKS_TENSOR_FLOAT32} is allowed to be
- * calculated with range and/or precision as low as that of the IEEE 754 16-bit
- * floating-point format. By default, {@link ANEURALNETWORKS_TENSOR_FLOAT32}
- * must be calculated using at least the range and precision of the IEEE 754
- * 32-bit floating-point format.
- *
- * @param model The model to be modified.
- * @param allow 'true' indicates {@link ANEURALNETWORKS_TENSOR_FLOAT32} may be
- *              calculated with range and/or precision as low as that of the
- *              IEEE 754 16-bit floating point format. 'false' indicates
- *              {@link ANEURALNETWORKS_TENSOR_FLOAT32} must be calculated using
- *              at least the range and precision of the IEEE 754 32-bit floating
- *              point format.
- *
- * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
- * been called will return an error.
- *
- * Available since API level 28.
- *
- * See {@link ANeuralNetworksModel} for information on multithreaded usage.
- */
-inline int ANeuralNetworksModel_relaxComputationFloat32toFloat16(
-    ANeuralNetworksModel* model, bool allow) {
-  LOAD_FUNCTION(ANeuralNetworksModel_relaxComputationFloat32toFloat16);
-  EXECUTE_FUNCTION_RETURN(model, allow);
-}
-
-/**
- * Create a {@link ANeuralNetworksCompilation} to compile the given model.
- * This only creates the object. Compilation is only performed once
- * {@link ANeuralNetworksCompilation_start} is invoked.
- *
- * <p>The provided model must outlive the compilation.</p>
- *
- * The model must already have been finished by a call to
- * {@link ANeuralNetworksModel_finish}.
- *
- * See {@link ANeuralNetworksCompilation} for information on multithreaded
- * usage.
- *
- * @param model The {@link ANeuralNetworksModel} to be compiled.
- * @param compilation The newly created object or NULL if unsuccessful.
- *
- * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA
- *         if the model is invalid.
- */
-inline int ANeuralNetworksCompilation_create(
-    ANeuralNetworksModel* model, ANeuralNetworksCompilation** compilation) {
-  LOAD_FUNCTION(ANeuralNetworksCompilation_create);
-  EXECUTE_FUNCTION_RETURN(model, compilation);
-}
-
-/**
- * Destroy a compilation.
- *
- * <p>If called on a compilation for which
- * {@link ANeuralNetworksCompilation_start} has been called, the
- * function will return immediately but will mark the compilation to be deleted
- * once the compilation completes. The {@link ANeuralNetworksCompilation_wait}
- * will return ERROR_DELETED.
- *
- * See {@link ANeuralNetworksCompilation} for information on multithreaded
- * usage.
- *
- * @param compilation The compilation to be destroyed. Passing NULL is
- * acceptable and results in no operation.
- */
-inline void ANeuralNetworksCompilation_free(
-    ANeuralNetworksCompilation* compilation) {
-  LOAD_FUNCTION(ANeuralNetworksCompilation_free);
-  EXECUTE_FUNCTION(compilation);
-}
-
-/**
- * Sets the execution preference.
- *
- * <p>Provides guidance to the runtime when trade-offs are possible.</p>
- *
- * See {@link ANeuralNetworksCompilation} for information on multithreaded
- * usage.
- *
- * @param compilation The compilation to be modified.
- * @param preference Either {@link PREFER_LOW_POWER},
- *                  {@link PREFER_SINGLE_FAST_ANSWER}, or
- *                  {@link PREFER_SUSTAINED_SPEED}.
- *
- * @return ANEURALNETWORKS_NO_ERROR if successful.
- */
-inline int ANeuralNetworksCompilation_setPreference(
-    ANeuralNetworksCompilation* compilation, int32_t preference) {
-  LOAD_FUNCTION(ANeuralNetworksCompilation_setPreference);
-  EXECUTE_FUNCTION_RETURN(compilation, preference);
-}
-
-/**
- * Waits until the compilation completes.
- *
- * More than one thread can wait on a compilation. When the compilation
- * completes, all threads will be released.
- *
- * See {@link ANeuralNetworksCompilation} for information on multithreaded
- * usage.
- *
- * @return ANEURALNETWORKS_NO_ERROR if the compilation completed normally.
- */
-inline int ANeuralNetworksCompilation_finish(
-    ANeuralNetworksCompilation* compilation) {
-  LOAD_FUNCTION(ANeuralNetworksCompilation_finish);
-  EXECUTE_FUNCTION_RETURN(compilation);
-}
-/**
- * Create a {@link ANeuralNetworksExecution} to apply the given compilation.
- * This only creates the object. Computation is only performed once
- * {@link ANeuralNetworksExecution_startCompute} is invoked.
- *
- * <p>The provided compilation must outlive the execution.</p>
- *
- * See {@link ANeuralNetworksExecution} for information on multithreaded usage.
- *
- * @param compilation The {@link ANeuralNetworksCompilation} to be evaluated.
- * @param execution The newly created object or NULL if unsuccessful.
- *
- * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA
- *         if the compilation is invalid.
- */
-inline int ANeuralNetworksExecution_create(
-    ANeuralNetworksCompilation* compilation,
-    ANeuralNetworksExecution** execution) {
-  LOAD_FUNCTION(ANeuralNetworksExecution_create);
-  EXECUTE_FUNCTION_RETURN(compilation, execution);
-}
-
-/**
- * Destroy an execution.
- *
- * <p>If called on an execution for which
- * {@link ANeuralNetworksExecution_startCompute} has been called, the
- * function will return immediately but will mark the execution to be deleted
- * once the computation completes.   The {link ANeuralNetworksExecution_wait}
- * will return ANEURALNETWORKS_ERROR_DELETED.
- *
- * See {@link ANeuralNetworksExecution} for information on multithreaded usage.
- *
- * @param execution The execution to be destroyed. Passing NULL is acceptable
- * and results in no operation.
- */
-inline void ANeuralNetworksExecution_free(ANeuralNetworksExecution* execution) {
-  LOAD_FUNCTION(ANeuralNetworksExecution_free);
-  EXECUTE_FUNCTION(execution);
-}
-
-/**
- * Associate a user buffer with an input of the model of the
- * {@link ANeuralNetworksExecution}.
- *
- * <p>The provided buffer must outlive the execution.</p>
- *
- * See {@link ANeuralNetworksExecution} for information on multithreaded usage.
- *
- * @param execution The execution to be modified.
- * @param index The index of the input argument we are setting. It is
- *              an index into the lists passed to
- *              {@link ANeuralNetworksModel_identifyInputsAndOutputs}. It is not
- *              the index associated with {@link
- * ANeuralNetworksModel_addOperand}.
- * @param type The type of the operand. This should be used to specify the
- *             dimensions that were set to 0 when the operand was added to the
- *             model. All other properties of the type must be the same as
- *             specified in the model. If the type is the same as specified
- *             when the model was built, NULL can be passed.
- * @param buffer The buffer containing the data.
- * @param length The length in bytes of the buffer.
- *
- * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA if
- * the name is not recognized or the buffer is too small for the input.
- */
-inline int ANeuralNetworksExecution_setInput(
-    ANeuralNetworksExecution* execution, int32_t index,
-    const ANeuralNetworksOperandType* type, const void* buffer, size_t length) {
-  LOAD_FUNCTION(ANeuralNetworksExecution_setInput);
-  EXECUTE_FUNCTION_RETURN(execution, index, type, buffer, length);
-}
-
-/**
- * Associate part of a memory object with an input of the model of the
- * {@link ANeuralNetworksExecution}.
- *
- * <p>The provided memory must outlive the execution.</p>
- *
- * See {@link ANeuralNetworksExecution} for information on multithreaded usage.
- *
- * @param execution The execution to be modified.
- * @param index The index of the input argument we are setting. It is
- *              an index into the lists passed to
- *              {@link ANeuralNetworksModel_identifyInputsAndOutputs}. It is not
- *              the index associated with {@link
- * ANeuralNetworksModel_addOperand}.
- * @param type The type of the operand. This can be used to specify the
- *             dimensions that were set to 0 when the operand was added to the
- *             model. All other values must be the same as specified in the
- *             model. If the type is the same as specified when the model
- *             was built, NULL can be passed.
- * @param memory The memory containing the data.
- * @param offset This specifies the location of the data within the memory.
- *               The offset is in bytes from the start of memory.
- * @param length The size in bytes of the data value.
- *
- * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA if
- * the name is not recognized or the buffer is too small for the input.
- */
-inline int ANeuralNetworksExecution_setInputFromMemory(
-    ANeuralNetworksExecution* execution, int32_t index,
-    const ANeuralNetworksOperandType* type, const ANeuralNetworksMemory* memory,
-    size_t offset, size_t length) {
-  LOAD_FUNCTION(ANeuralNetworksExecution_setInputFromMemory);
-  EXECUTE_FUNCTION_RETURN(execution, index, type, memory, offset, length);
-}
+typedef int (*ASharedMemory_create_fn)(const char* name, size_t size);
 
-/**
- * Associate a user buffer with an output of the model of the
- * {@link ANeuralNetworksExecution}.
- *
- * <p>The provided buffer must outlive the execution.</p>
- *
- * See {@link ANeuralNetworksExecution} for information on multithreaded usage.
- *
- * @param execution The execution to be modified.
- * @param index The index of the output argument we are setting. It is
- *              an index into the lists passed to
- *              {@link ANeuralNetworksModel_identifyInputsAndOutputs}. It is not
- *              the index associated with {@link
- * ANeuralNetworksModel_addOperand}.
- * @param type The type of the operand. This can be used to specify the
- *             dimensions that were set to 0 when the operand was added to the
- *             model. All other values must be the same as specified in the
- *             model. If the type is the same as specified when the model
- *             was built, NULL can be passed.
- * @param buffer The buffer where the data is to be written.
- * @param length The length in bytes of the buffer.
- *
- * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA if
- * the name is not recognized or the buffer is too small for the output.
- */
-inline int ANeuralNetworksExecution_setOutput(
-    ANeuralNetworksExecution* execution, int32_t index,
-    const ANeuralNetworksOperandType* type, void* buffer, size_t length) {
-  LOAD_FUNCTION(ANeuralNetworksExecution_setOutput);
-  EXECUTE_FUNCTION_RETURN(execution, index, type, buffer, length);
-}
+struct NnApi {
+  bool nnapi_exists;
+  int32_t android_sdk_version;
+
+  /**
+   * Creates a shared memory object from a file descriptor.
+   *
+   * The shared memory is backed by a file descriptor via mmap.
+   * See {@link ANeuralNetworksMemory} for a description on how to use
+   * this shared memory.
+   *
+   * @param size The requested size in bytes.
+   *             Must not be larger than the file size.
+   * @param prot The desired memory protection for the mapping.
+   *             It is either PROT_NONE or the bitwise OR of one or
+   *             more of the following flags: PROT_READ, PROT_WRITE.
+   * @param fd The requested file descriptor.
+   *           The file descriptor has to be mmap-able. The file
+   *           descriptor will be duplicated.
+   * @param offset The offset to the beginning of the file of the area to map.
+   *               The offset has to be aligned to a page size.
+   * @param memory The memory object to be created.
+   *               Set to NULL if unsuccessful.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if the request completed normally.
+   */
+  int (*ANeuralNetworksMemory_createFromFd)(size_t size, int protect, int fd,
+                                            size_t offset,
+                                            ANeuralNetworksMemory** memory);
+
+  /**
+   * Delete a memory object.
+   *
+   * Destroys the object used by the run time to keep track of the memory.
+   * This will free the underlying actual memory if no other code has open
+   * handles to this memory.
+   *
+   * @param memory The memory object to be freed.
+   */
+  void (*ANeuralNetworksMemory_free)(ANeuralNetworksMemory* memory);
+
+  /**
+   * Create an empty {@link ANeuralNetworksModel}.
+   *
+   * <p>This only creates the object. Computation is performed once
+   * {@link ANeuralNetworksExecution_startCompute} is invoked.
+   *
+   * The model should be constructed with calls to
+   * {@link ANeuralNetworksModel_addOperation} and
+   * {@link ANeuralNetworksModel_addOperand}
+   *
+   * <p>{@link ANeuralNetworksModel_finish} should be called once the model
+   * has been fully constructed.</p>
+   *
+   * <p>{@link ANeuralNetworksModel_free} should be called once the model
+   * is no longer needed.</p>
+   *
+   * @param model The {@link ANeuralNetworksModel} to be created.
+   *              Set to NULL if unsuccessful.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful.
+   */
+  int (*ANeuralNetworksModel_create)(ANeuralNetworksModel** model);
+
+  /**
+   * Destroy a model.
+   *
+   * The model need not have been finished by a call to
+   * {@link ANeuralNetworksModel_finish}.
+   *
+   * See {@link ANeuralNetworksModel} for information on multithreaded usage.
+   *
+   * @param model The model to be destroyed. Passing NULL is acceptable and
+   *              results in no operation.
+   */
+  void (*ANeuralNetworksModel_free)(ANeuralNetworksModel* model);
+
+  /**
+   * Indicate that we have finished modifying a model. Required before
+   * calling {@link ANeuralNetworksCompilation_compile}.
+   *
+   * An application is responsible to make sure that no other thread uses
+   * the model at the same time.
+   *
+   * See {@link ANeuralNetworksModel} for information on multithreaded usage.
+   *
+   * @param model The model to be finished.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful.
+   */
+  int (*ANeuralNetworksModel_finish)(ANeuralNetworksModel* model);
+
+  /**
+   * Add an operand to a model.
+   *
+   * The order in which the operands are added is important. The first one added
+   * to a model will have the index value 0, the second 1, etc. These indexes
+   * are used as operand identifiers in
+   * {@link ANeuralNetworksModel_addOperation},
+   * {@link ANeuralNetworksExecution_setInput},
+   * {@link ANeuralNetworksExecution_setInputFromMemory},
+   * {@link ANeuralNetworksExecution_setOutput},
+   * {@link ANeuralNetworksExecution_setOutputFromMemory} and
+   * {@link ANeuralNetworksExecution_setOperandValue}.
+   *
+   * To build a model that can accommodate inputs of various sizes, as you may
+   * want to do for a CNN, set the size of the dimensions that will vary at run
+   * time to 0. If you do so, provide the full dimensions when calling
+   * {@link ANeuralNetworksExecution_setInput} or {@link
+   * ANeuralNetworksExecution_setInputFromMemory}.
+   *
+   * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
+   * been called will return an error.
+   *
+   * See {@link ANeuralNetworksModel} for information on multithreaded usage.
+   *
+   * @param model The model to be modified.
+   * @param type The {@link ANeuralNetworksOperandType} that describes the shape
+   * of the operand.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful.
+   */
+  int (*ANeuralNetworksModel_addOperand)(
+      ANeuralNetworksModel* model, const ANeuralNetworksOperandType* type);
+
+  /**
+   * Sets an operand to a constant value.
+   *
+   * For scalar values, the content of buffer is copied into the model.
+   *
+   * For tensor values, a pointer to the buffer is stored within the model.
+   * The application is responsible for not changing the content of this region
+   * until all executions using this model have completed. As the data may
+   * be copied during processing, modifying the data after this call yields
+   * undefined results.
+   *
+   * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
+   * been called will return an error.
+   *
+   * See {@link ANeuralNetworksModel} for information on multithreaded usage.
+   *
+   * @param model The model to be modified.
+   * @param index The index of the model operand we're setting.
+   * @param buffer A pointer to the data to use.
+   * @param length The size in bytes of the data value.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful.
+   */
+  int (*ANeuralNetworksModel_setOperandValue)(ANeuralNetworksModel* model,
+                                              int32_t index, const void* buffer,
+                                              size_t length);
+
+  /**
+   * Sets an operand to a value stored in a memory object.
+   *
+   * The content of the memory is not copied. A reference to that memory is
+   * stored inside the model. The application is responsible for not changing
+   * the content of the memory region until all executions using this model have
+   * completed.
+   * As the data may be copied during processing, modifying the data after this
+   * call yields undefined results.
+   *
+   * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
+   * been called will return an error.
+   *
+   * See {@link ANeuralNetworksModel} for information on multithreaded usage.
+   *
+   * @param model The model to be modified.
+   * @param index The index of the model operand we're setting.
+   * @param buffer A pointer to the data to use.
+   * @param memory The memory containing the data.
+   * @param offset This specifies the location of the data within the memory.
+   *               The offset is in bytes from the start of memory.
+   * @param length The size in bytes of the data value.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful.
+   */
+  int (*ANeuralNetworksModel_setOperandValueFromMemory)(
+      ANeuralNetworksModel* model, int32_t index,
+      const ANeuralNetworksMemory* memory, size_t offset, size_t length);
+
+  /**
+   * Add an operation to a model.
+   *
+   * @param model The model to be modified.
+   * @param type The type of the operation.
+   * @param inputCount The number of entries in the inputs array.
+   * @param inputs An array of indexes identifying each operand.
+   * @param outputCount The number of entries in the outputs array.
+   * @param outputs An array of indexes identifying each operand.
+   *
+   * The operands specified by inputs and outputs must have been
+   * previously added by calls to {@link ANeuralNetworksModel_addOperand}.
+   *
+   * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
+   * been called will return an error.
+   *
+   * See {@link ANeuralNetworksModel} for information on multithreaded usage.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful.
+   */
+  int (*ANeuralNetworksModel_addOperation)(ANeuralNetworksModel* model,
+                                           ANeuralNetworksOperationType type,
+                                           uint32_t inputCount,
+                                           const uint32_t* inputs,
+                                           uint32_t outputCount,
+                                           const uint32_t* outputs);
+
+  /**
+   * Specifies which operands will be the model's inputs and outputs.
+   *
+   * An operand cannot be used for both input and output. Doing so will
+   * return an error.
+   *
+   * @param model The model to be modified.
+   * @param inputCount The number of entries in the inputs array.
+   * @param inputs An array of indexes identifying the input operands.
+   * @param outputCount The number of entries in the outputs array.
+   * @param outputs An array of indexes identifying the output operands.
+   *
+   * The operands specified by inputs and outputs must have been
+   * previously added by calls to {@link ANeuralNetworksModel_addOperand}.
+   *
+   * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
+   * been called will return an error.
+   *
+   * See {@link ANeuralNetworksModel} for information on multithreaded usage.
+   *
+   */
+  int (*ANeuralNetworksModel_identifyInputsAndOutputs)(
+      ANeuralNetworksModel* model, uint32_t inputCount, const uint32_t* inputs,
+      uint32_t outputCount, const uint32_t* outputs);
+
+  /**
+   * Specifies whether {@link ANEURALNETWORKS_TENSOR_FLOAT32} is allowed to be
+   * calculated with range and/or precision as low as that of the
+   * IEEE 754 16-bit floating-point format. By default,
+   * {@link ANEURALNETWORKS_TENSOR_FLOAT32} must be calculated using at least
+   * the range and precision of the IEEE 754 32-bit floating-point format.
+   *
+   * @param model The model to be modified.
+   * @param allow 'true' indicates {@link ANEURALNETWORKS_TENSOR_FLOAT32} may be
+   *              calculated with range and/or precision as low as that of the
+   *              IEEE 754 16-bit floating point format. 'false' indicates
+   *              {@link ANEURALNETWORKS_TENSOR_FLOAT32} must be calculated
+   *              using at least the range and precision of the IEEE 754 32-bit
+   *              floating point format.
+   *
+   * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
+   * been called will return an error.
+   *
+   * Available since API level 28.
+   *
+   * See {@link ANeuralNetworksModel} for information on multithreaded usage.
+   */
+  int (*ANeuralNetworksModel_relaxComputationFloat32toFloat16)(
+      ANeuralNetworksModel* model, bool allow);
+
+  /**
+   * Create a {@link ANeuralNetworksCompilation} to compile the given model.
+   * This only creates the object. Compilation is only performed once
+   * {@link ANeuralNetworksCompilation_start} is invoked.
+   *
+   * <p>The provided model must outlive the compilation.</p>
+   *
+   * The model must already have been finished by a call to
+   * {@link ANeuralNetworksModel_finish}.
+   *
+   * See {@link ANeuralNetworksCompilation} for information on multithreaded
+   * usage.
+   *
+   * @param model The {@link ANeuralNetworksModel} to be compiled.
+   * @param compilation The newly created object or NULL if unsuccessful.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA
+   *         if the model is invalid.
+   */
+  int (*ANeuralNetworksCompilation_create)(
+      ANeuralNetworksModel* model, ANeuralNetworksCompilation** compilation);
+
+  /**
+   * Destroy a compilation.
+   *
+   * <p>If called on a compilation for which
+   * {@link ANeuralNetworksCompilation_start} has been called, the
+   * function will return immediately but will mark the compilation to be
+   * deleted once the compilation completes. The
+   * {@link ANeuralNetworksCompilation_wait} will return ERROR_DELETED.
+   *
+   * See {@link ANeuralNetworksCompilation} for information on multithreaded
+   * usage.
+   *
+   * @param compilation The compilation to be destroyed. Passing NULL is
+   * acceptable and results in no operation.
+   */
+  void (*ANeuralNetworksCompilation_free)(
+      ANeuralNetworksCompilation* compilation);
+
+  /**
+   * Sets the execution preference.
+   *
+   * <p>Provides guidance to the runtime when trade-offs are possible.</p>
+   *
+   * See {@link ANeuralNetworksCompilation} for information on multithreaded
+   * usage.
+   *
+   * @param compilation The compilation to be modified.
+   * @param preference Either {@link PREFER_LOW_POWER},
+   *                  {@link PREFER_SINGLE_FAST_ANSWER}, or
+   *                  {@link PREFER_SUSTAINED_SPEED}.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful.
+   */
+  int (*ANeuralNetworksCompilation_setPreference)(
+      ANeuralNetworksCompilation* compilation, int32_t preference);
+
+  /**
+   * Waits until the compilation completes.
+   *
+   * More than one thread can wait on a compilation. When the compilation
+   * completes, all threads will be released.
+   *
+   * See {@link ANeuralNetworksCompilation} for information on multithreaded
+   * usage.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if the compilation completed normally.
+   */
+  int (*ANeuralNetworksCompilation_finish)(
+      ANeuralNetworksCompilation* compilation);
+
+  /**
+   * Create a {@link ANeuralNetworksExecution} to apply the given compilation.
+   * This only creates the object. Computation is only performed once
+   * {@link ANeuralNetworksExecution_startCompute} is invoked.
+   *
+   * <p>The provided compilation must outlive the execution.</p>
+   *
+   * See {@link ANeuralNetworksExecution} for information on multithreaded
+   * usage.
+   *
+   * @param compilation The {@link ANeuralNetworksCompilation} to be evaluated.
+   * @param execution The newly created object or NULL if unsuccessful.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA
+   *         if the compilation is invalid.
+   */
+  int (*ANeuralNetworksExecution_create)(
+      ANeuralNetworksCompilation* compilation,
+      ANeuralNetworksExecution** execution);
+
+  /**
+   * Destroy an execution.
+   *
+   * <p>If called on an execution for which
+   * {@link ANeuralNetworksExecution_startCompute} has been called, the
+   * function will return immediately but will mark the execution to be deleted
+   * once the computation completes.   The {link ANeuralNetworksExecution_wait}
+   * will return ANEURALNETWORKS_ERROR_DELETED.
+   *
+   * See {@link ANeuralNetworksExecution} for information on multithreaded
+   * usage.
+   *
+   * @param execution The execution to be destroyed. Passing NULL is acceptable
+   * and results in no operation.
+   */
+  void (*ANeuralNetworksExecution_free)(ANeuralNetworksExecution* execution);
+
+  /**
+   * Associate a user buffer with an input of the model of the
+   * {@link ANeuralNetworksExecution}.
+   *
+   * <p>The provided buffer must outlive the execution.</p>
+   *
+   * See {@link ANeuralNetworksExecution} for information on multithreaded
+   * usage.
+   *
+   * @param execution The execution to be modified.
+   * @param index The index of the input argument we are setting. It is
+   *              an index into the lists passed to
+   *              {@link ANeuralNetworksModel_identifyInputsAndOutputs}. It is
+   *              not the index associated with {@link
+   * ANeuralNetworksModel_addOperand}.
+   * @param type The type of the operand. This should be used to specify the
+   *             dimensions that were set to 0 when the operand was added to the
+   *             model. All other properties of the type must be the same as
+   *             specified in the model. If the type is the same as specified
+   *             when the model was built, NULL can be passed.
+   * @param buffer The buffer containing the data.
+   * @param length The length in bytes of the buffer.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA if
+   * the name is not recognized or the buffer is too small for the input.
+   */
+  int (*ANeuralNetworksExecution_setInput)(
+      ANeuralNetworksExecution* execution, int32_t index,
+      const ANeuralNetworksOperandType* type, const void* buffer,
+      size_t length);
+
+  /**
+   * Associate part of a memory object with an input of the model of the
+   * {@link ANeuralNetworksExecution}.
+   *
+   * <p>The provided memory must outlive the execution.</p>
+   *
+   * See {@link ANeuralNetworksExecution} for information on multithreaded
+   * usage.
+   *
+   * @param execution The execution to be modified.
+   * @param index The index of the input argument we are setting. It is
+   *              an index into the lists passed to
+   *              {@link ANeuralNetworksModel_identifyInputsAndOutputs}. It is
+   *              not the index associated with {@link
+   * ANeuralNetworksModel_addOperand}.
+   * @param type The type of the operand. This can be used to specify the
+   *             dimensions that were set to 0 when the operand was added to the
+   *             model. All other values must be the same as specified in the
+   *             model. If the type is the same as specified when the model
+   *             was built, NULL can be passed.
+   * @param memory The memory containing the data.
+   * @param offset This specifies the location of the data within the memory.
+   *               The offset is in bytes from the start of memory.
+   * @param length The size in bytes of the data value.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA if
+   * the name is not recognized or the buffer is too small for the input.
+   */
+  int (*ANeuralNetworksExecution_setInputFromMemory)(
+      ANeuralNetworksExecution* execution, int32_t index,
+      const ANeuralNetworksOperandType* type,
+      const ANeuralNetworksMemory* memory, size_t offset, size_t length);
+
+  /**
+   * Associate a user buffer with an output of the model of the
+   * {@link ANeuralNetworksExecution}.
+   *
+   * <p>The provided buffer must outlive the execution.</p>
+   *
+   * See {@link ANeuralNetworksExecution} for information on multithreaded
+   * usage.
+   *
+   * @param execution The execution to be modified.
+   * @param index The index of the output argument we are setting. It is
+   *              an index into the lists passed to
+   *              {@link ANeuralNetworksModel_identifyInputsAndOutputs}. It is
+   *              not the index associated with {@link
+   * ANeuralNetworksModel_addOperand}.
+   * @param type The type of the operand. This can be used to specify the
+   *             dimensions that were set to 0 when the operand was added to the
+   *             model. All other values must be the same as specified in the
+   *             model. If the type is the same as specified when the model
+   *             was built, NULL can be passed.
+   * @param buffer The buffer where the data is to be written.
+   * @param length The length in bytes of the buffer.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA if
+   * the name is not recognized or the buffer is too small for the output.
+   */
+  int (*ANeuralNetworksExecution_setOutput)(
+      ANeuralNetworksExecution* execution, int32_t index,
+      const ANeuralNetworksOperandType* type, void* buffer, size_t length);
+
+  /**
+   * Associate part of a memory object with an output of the model of the
+   * {@link ANeuralNetworksExecution}.
+   *
+   * <p>The provided memory must outlive the execution.</p>
+   *
+   * See {@link ANeuralNetworksExecution} for information on multithreaded
+   * usage.
+   *
+   * @param execution The execution to be modified.
+   * @param index The index of the output argument we are setting. It is
+   *              an index into the lists passed to
+   *              {@link ANeuralNetworksModel_identifyInputsAndOutputs}. It is
+   *              not the index associated with {@link
+   * ANeuralNetworksModel_addOperand}.
+   * @param type The type of the operand. This can be used to specify the
+   *             dimensions that were set to 0 when the operand was added to the
+   *             model. All other values must be the same as specified in the
+   *             model. If the type is the same as specified when the model
+   *             was built, NULL can be passed.
+   * @param memory The memory where the data is to be stored.
+   * @param offset This specifies the location of the data within the memory.
+   *               The offset is in bytes from the start of memory.
+   * @param length The length in bytes of the data value.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA if
+   * the name is not recognized or the buffer is too small for the output.
+   */
+  int (*ANeuralNetworksExecution_setOutputFromMemory)(
+      ANeuralNetworksExecution* execution, int32_t index,
+      const ANeuralNetworksOperandType* type,
+      const ANeuralNetworksMemory* memory, size_t offset, size_t length);
+
+  /**
+   * Schedule evaluation of the execution.
+   *
+   * <p>Schedules evaluation of the execution. Once the model has been
+   * applied and the outputs are ready to be consumed, the execution will be
+   * signaled. Use {@link ANeuralNetworksExecution_wait} to wait for that
+   * signal.
+   * </p>
+   *
+   * Multiple executions can be scheduled and evaluated concurrently, and
+   * compilations can be performed concurrently with executions. The runtime
+   * makes no guarantee on the ordering of the completion of compilations and
+   * executions. If it's important to the application, the application should
+   * enforce the ordering by using {@link ANeuralNetworksCompilation_wait} and
+   * {@link ANeuralNetworksExecution_wait}.
+   *
+   * ANeuralNetworksExecution_wait must be called to recuperate the resources
+   * used by the execution.
+   *
+   * See {@link ANeuralNetworksExecution} for information on multithreaded
+   * usage.
+   *
+   * @param execution The execution to be scheduled and executed.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful.
+   */
+  int (*ANeuralNetworksExecution_startCompute)(
+      ANeuralNetworksExecution* execution, ANeuralNetworksEvent** event);
+
+  /**
+   * Waits until the execution completes.
+   *
+   * More than one thread can wait on an event. When the execution completes,
+   * all threads will be released.
+   *
+   * See {@link ANeuralNetworksExecution} for information on multithreaded
+   * usage.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if the execution completed normally.
+   */
+  int (*ANeuralNetworksEvent_wait)(ANeuralNetworksEvent* event);
 
-/**
- * Associate part of a memory object with an output of the model of the
- * {@link ANeuralNetworksExecution}.
- *
- * <p>The provided memory must outlive the execution.</p>
- *
- * See {@link ANeuralNetworksExecution} for information on multithreaded usage.
- *
- * @param execution The execution to be modified.
- * @param index The index of the output argument we are setting. It is
- *              an index into the lists passed to
- *              {@link ANeuralNetworksModel_identifyInputsAndOutputs}. It is not
- *              the index associated with {@link
- * ANeuralNetworksModel_addOperand}.
- * @param type The type of the operand. This can be used to specify the
- *             dimensions that were set to 0 when the operand was added to the
- *             model. All other values must be the same as specified in the
- *             model. If the type is the same as specified when the model
- *             was built, NULL can be passed.
- * @param memory The memory where the data is to be stored.
- * @param offset This specifies the location of the data within the memory.
- *               The offset is in bytes from the start of memory.
- * @param length The length in bytes of the data value.
- *
- * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA if
- * the name is not recognized or the buffer is too small for the output.
- */
-inline int ANeuralNetworksExecution_setOutputFromMemory(
-    ANeuralNetworksExecution* execution, int32_t index,
-    const ANeuralNetworksOperandType* type, const ANeuralNetworksMemory* memory,
-    size_t offset, size_t length) {
-  LOAD_FUNCTION(ANeuralNetworksExecution_setOutputFromMemory);
-  EXECUTE_FUNCTION_RETURN(execution, index, type, memory, offset, length);
-}
+  /**
+   * Destroys the event.
+   *
+   * See {@link ANeuralNetworksExecution} for information on multithreaded
+   * usage.
+   */
+  void (*ANeuralNetworksEvent_free)(ANeuralNetworksEvent* event);
 
-/**
- * Schedule evaluation of the execution.
- *
- * <p>Schedules evaluation of the execution. Once the model has been
- * applied and the outputs are ready to be consumed, the execution will be
- * signaled. Use {@link ANeuralNetworksExecution_wait} to wait for that signal.
- * </p>
- *
- * Multiple executions can be scheduled and evaluated concurrently, and
- * compilations can be performed concurrently with executions. The runtime makes
- * no guarantee on the ordering of the completion of compilations and
- * executions. If it's important to the application, the application should
- * enforce the ordering by using {@link ANeuralNetworksCompilation_wait} and
- * {@link ANeuralNetworksExecution_wait}.
- *
- * ANeuralNetworksExecution_wait must be called to recuperate the resources used
- * by the execution.
- *
- * See {@link ANeuralNetworksExecution} for information on multithreaded usage.
- *
- * @param execution The execution to be scheduled and executed.
- *
- * @return ANEURALNETWORKS_NO_ERROR if successful.
- */
-inline int ANeuralNetworksExecution_startCompute(
-    ANeuralNetworksExecution* execution, ANeuralNetworksEvent** event) {
-  LOAD_FUNCTION(ANeuralNetworksExecution_startCompute);
-  EXECUTE_FUNCTION_RETURN(execution, event);
-}
+  // ASharedMemory_create was added in Android 8.0, so safe to use with NNAPI
+  // which was added in 8.1.
+  int (*ASharedMemory_create)(const char* name, size_t size);
 
-/**
- * Waits until the execution completes.
- *
- * More than one thread can wait on an event. When the execution completes,
- * all threads will be released.
- *
- * See {@link ANeuralNetworksExecution} for information on multithreaded usage.
- *
- * @return ANEURALNETWORKS_NO_ERROR if the execution completed normally.
- */
-inline int ANeuralNetworksEvent_wait(ANeuralNetworksEvent* event) {
-  LOAD_FUNCTION(ANeuralNetworksEvent_wait);
-  EXECUTE_FUNCTION_RETURN(event);
-}
+  /**/
+};
 
 /**
- * Destroys the event.
- *
- * See {@link ANeuralNetworksExecution} for information on multithreaded usage.
+ * Load the NNAPI implementation from the shared libraries.
+ * The NnApi structure is filled with all the pointers. If one function doesn't
+ * exist, a null pointer is stored.
  */
-inline void ANeuralNetworksEvent_free(ANeuralNetworksEvent* event) {
-  LOAD_FUNCTION(ANeuralNetworksEvent_free);
-  EXECUTE_FUNCTION(event);
-}
-
-/**/
+const NnApi* NnApiImplementation();
 
 #endif  // TENSORFLOW_LITE_NNAPI_NEURALNETWORKSSHIM_H_
diff --git a/tensorflow/lite/nnapi/nnapi_lib_test.cc b/tensorflow/lite/nnapi/nnapi_lib_test.cc
new file mode 100644
index 0000000000..aaf0ffaa8e
--- /dev/null
+++ b/tensorflow/lite/nnapi/nnapi_lib_test.cc
@@ -0,0 +1,90 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <gtest/gtest.h>
+#include "tensorflow/lite/nnapi/NeuralNetworksShim.h"
+
+namespace {
+
+TEST(NnapiLibTest, NnApiImplementation) {
+  const NnApi* nnapi = NnApiImplementation();
+  EXPECT_NE(nnapi, nullptr);
+#ifdef __ANDROID__
+  EXPECT_TRUE(nnapi->nnapi_exists);
+  EXPECT_GT(nnapi->android_sdk_version, 0);
+  EXPECT_NE(nnapi->ANeuralNetworksMemory_createFromFd, nullptr);
+  EXPECT_NE(nnapi->ANeuralNetworksMemory_free, nullptr);
+  EXPECT_NE(nnapi->ANeuralNetworksModel_create, nullptr);
+  EXPECT_NE(nnapi->ANeuralNetworksModel_free, nullptr);
+  EXPECT_NE(nnapi->ANeuralNetworksModel_finish, nullptr);
+  EXPECT_NE(nnapi->ANeuralNetworksModel_addOperand, nullptr);
+  EXPECT_NE(nnapi->ANeuralNetworksModel_setOperandValue, nullptr);
+  EXPECT_NE(nnapi->ANeuralNetworksModel_setOperandValueFromMemory, nullptr);
+  EXPECT_NE(nnapi->ANeuralNetworksModel_addOperation, nullptr);
+  EXPECT_NE(nnapi->ANeuralNetworksModel_identifyInputsAndOutputs, nullptr);
+  if (nnapi->android_sdk_version >= 28) {
+    // relaxComputationFloat32toFloat16 only available with Android 9.0 (P).
+    EXPECT_NE(nnapi->ANeuralNetworksModel_relaxComputationFloat32toFloat16,
+              nullptr);
+  } else {
+    EXPECT_EQ(nnapi->ANeuralNetworksModel_relaxComputationFloat32toFloat16,
+              nullptr);
+  }
+  EXPECT_NE(nnapi->ANeuralNetworksCompilation_create, nullptr);
+  EXPECT_NE(nnapi->ANeuralNetworksCompilation_free, nullptr);
+  EXPECT_NE(nnapi->ANeuralNetworksCompilation_setPreference, nullptr);
+  EXPECT_NE(nnapi->ANeuralNetworksCompilation_finish, nullptr);
+  EXPECT_NE(nnapi->ANeuralNetworksExecution_create, nullptr);
+  EXPECT_NE(nnapi->ANeuralNetworksExecution_free, nullptr);
+  EXPECT_NE(nnapi->ANeuralNetworksExecution_setInput, nullptr);
+  EXPECT_NE(nnapi->ANeuralNetworksExecution_setInputFromMemory, nullptr);
+  EXPECT_NE(nnapi->ANeuralNetworksExecution_setOutput, nullptr);
+  EXPECT_NE(nnapi->ANeuralNetworksExecution_setOutputFromMemory, nullptr);
+  EXPECT_NE(nnapi->ANeuralNetworksExecution_startCompute, nullptr);
+  EXPECT_NE(nnapi->ANeuralNetworksEvent_wait, nullptr);
+  EXPECT_NE(nnapi->ANeuralNetworksEvent_free, nullptr);
+  EXPECT_NE(nnapi->ASharedMemory_create, nullptr);
+#else
+  EXPECT_FALSE(nnapi->nnapi_exists);
+  EXPECT_EQ(nnapi->android_sdk_version, 0);
+  EXPECT_EQ(nnapi->ANeuralNetworksMemory_createFromFd, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksMemory_free, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksModel_create, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksModel_free, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksModel_finish, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksModel_addOperand, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksModel_setOperandValue, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksModel_setOperandValueFromMemory, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksModel_addOperation, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksModel_identifyInputsAndOutputs, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksModel_relaxComputationFloat32toFloat16,
+            nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksCompilation_create, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksCompilation_free, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksCompilation_setPreference, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksCompilation_finish, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksExecution_create, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksExecution_free, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksExecution_setInput, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksExecution_setInputFromMemory, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksExecution_setOutput, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksExecution_setOutputFromMemory, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksExecution_startCompute, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksEvent_wait, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksEvent_free, nullptr);
+  EXPECT_EQ(nnapi->ASharedMemory_create, nullptr);
+#endif
+}
+
+}  // namespace
diff --git a/tensorflow/lite/nnapi_delegate.cc b/tensorflow/lite/nnapi_delegate.cc
index dc8e81cde7..44b9930acc 100644
--- a/tensorflow/lite/nnapi_delegate.cc
+++ b/tensorflow/lite/nnapi_delegate.cc
@@ -84,56 +84,27 @@ void logError(const char* format, ...) {
 static const int64_t kOperandIdNotSet = -1;
 static const int64_t kOperandNotNeeded = -2;
 
-namespace {
-
-int32_t GetAndroidSdkVersion() {
-#ifdef __ANDROID__
-  const char* sdkProp = "ro.build.version.sdk";
-  char sdkVersion[PROP_VALUE_MAX];
-  int length = __system_property_get(sdkProp, sdkVersion);
-  if (length != 0) {
-    for (int i = 0; i < length; ++i) {
-      int digit = sdkVersion[i] - '0';
-      if (digit < 0 || digit > 9) {
-        // Non-numeric SDK version, assume it's higher then expected;
-        return 0xFFFF;
-      }
-    }
-    return atoi(sdkVersion);
-  }
-  FATAL("No %s prop", sdkProp);
-#endif  // __ANDROID__
-  return 0;
-}
-
-int32_t GetAndroidSdkVersionCached() {
-  static int32_t androidSdkVersion = GetAndroidSdkVersion();
-  return androidSdkVersion;
-}
-
-}  // namespace
-
 NNAPIAllocation::NNAPIAllocation(const char* filename,
                                  ErrorReporter* error_reporter)
     : MMAPAllocation(filename, error_reporter) {
   if (mmapped_buffer_ != MAP_FAILED)
-    CHECK_NN(ANeuralNetworksMemory_createFromFd(buffer_size_bytes_, PROT_READ,
-                                                mmap_fd_, 0, &handle_));
+    CHECK_NN(NnApiImplementation()->ANeuralNetworksMemory_createFromFd(
+        buffer_size_bytes_, PROT_READ, mmap_fd_, 0, &handle_));
 }
 
 NNAPIAllocation::~NNAPIAllocation() {
   if (handle_) {
-    ANeuralNetworksMemory_free(handle_);
+    NnApiImplementation()->ANeuralNetworksMemory_free(handle_);
   }
 }
 
 NNAPIDelegate::~NNAPIDelegate() {
   if (nn_compiled_model_) {
-    ANeuralNetworksCompilation_free(nn_compiled_model_);
+    NnApiImplementation()->ANeuralNetworksCompilation_free(nn_compiled_model_);
     nn_compiled_model_ = nullptr;
   }
   if (nn_model_) {
-    ANeuralNetworksModel_free(nn_model_);
+    NnApiImplementation()->ANeuralNetworksModel_free(nn_model_);
     nn_model_ = nullptr;
     // TODO(aselle): Is this thread-safe and callable multiple times?
   }
@@ -145,6 +116,7 @@ TfLiteStatus addTensorOperands(tflite::Subgraph* subgraph,
                                ANeuralNetworksModel* nn_model,
                                uint32_t* no_of_operands_added,
                                std::vector<int64_t>* nnapi_ids) {
+  const NnApi* nnapi = NnApiImplementation();
   uint32_t next_id = 0;
   for (size_t i = 0; i < subgraph->tensors_size(); i++) {
     // Skip temporaries and RNN back-edges.
@@ -198,24 +170,24 @@ TfLiteStatus addTensorOperands(tflite::Subgraph* subgraph,
         nn_type, static_cast<uint32_t>(tensor->dims->size),
         reinterpret_cast<uint32_t*>(tensor->dims->data), scale, zeroPoint};
     RETURN_ERROR_IF_NN_FAILED(
-        ANeuralNetworksModel_addOperand(nn_model, &operand_type));
+        nnapi->ANeuralNetworksModel_addOperand(nn_model, &operand_type));
     // TODO(aselle): Based on Michael's suggestion, limiting this to read
     // only memory
     if (tensor->allocation_type == kTfLiteMmapRo) {
       if (const NNAPIAllocation* alloc = dynamic_cast<const NNAPIAllocation*>(
               static_cast<const Allocation*>(tensor->allocation))) {
         RETURN_ERROR_IF_NN_FAILED(
-            ANeuralNetworksModel_setOperandValueFromMemory(
+            nnapi->ANeuralNetworksModel_setOperandValueFromMemory(
                 nn_model, next_id, alloc->memory(),
                 alloc->offset(tensor->data.raw), tensor->bytes));
       } else {
-        RETURN_ERROR_IF_NN_FAILED(ANeuralNetworksModel_setOperandValue(
+        RETURN_ERROR_IF_NN_FAILED(nnapi->ANeuralNetworksModel_setOperandValue(
             nn_model, next_id, tensor->data.raw, tensor->bytes));
       }
     } else if (tensor->bytes == 0) {
       // These size 0 tensors are optional tensors reserved.
-      RETURN_ERROR_IF_NN_FAILED(
-          ANeuralNetworksModel_setOperandValue(nn_model, next_id, nullptr, 0));
+      RETURN_ERROR_IF_NN_FAILED(nnapi->ANeuralNetworksModel_setOperandValue(
+          nn_model, next_id, nullptr, 0));
     }
 
     ++next_id;
@@ -244,6 +216,7 @@ TfLiteStatus AddOpsAndParams(
     uint32_t next_id, std::vector<int>* model_state_inputs,
     std::vector<int>* model_state_outputs,
     const std::vector<int64_t>& tensor_id_to_nnapi_id) {
+  const NnApi* nnapi = NnApiImplementation();
   for (size_t i = 0; i < subgraph->nodes_size(); i++) {
     const auto* node_and_registration = subgraph->node_and_registration(i);
     const TfLiteNode& node = node_and_registration->first;
@@ -258,21 +231,21 @@ TfLiteStatus AddOpsAndParams(
     MapAndAddTensorIds(node.outputs->data, node.outputs->size,
                        &augmented_outputs, tensor_id_to_nnapi_id);
 
-    auto add_scalar_int32 = [&nn_model, &augmented_inputs,
+    auto add_scalar_int32 = [nnapi, &nn_model, &augmented_inputs,
                              &next_id](int value) {
       ANeuralNetworksOperandType operand_type{.type = ANEURALNETWORKS_INT32};
-      CHECK_NN(ANeuralNetworksModel_addOperand(nn_model, &operand_type))
-      CHECK_NN(ANeuralNetworksModel_setOperandValue(nn_model, next_id, &value,
-                                                    sizeof(int32_t)))
+      CHECK_NN(nnapi->ANeuralNetworksModel_addOperand(nn_model, &operand_type))
+      CHECK_NN(nnapi->ANeuralNetworksModel_setOperandValue(
+          nn_model, next_id, &value, sizeof(int32_t)))
       augmented_inputs.push_back(next_id++);
     };
 
-    auto add_scalar_float32 = [&nn_model, &augmented_inputs,
+    auto add_scalar_float32 = [nnapi, &nn_model, &augmented_inputs,
                                &next_id](float value) {
       ANeuralNetworksOperandType operand_type{.type = ANEURALNETWORKS_FLOAT32};
-      CHECK_NN(ANeuralNetworksModel_addOperand(nn_model, &operand_type))
-      CHECK_NN(ANeuralNetworksModel_setOperandValue(nn_model, next_id, &value,
-                                                    sizeof(float)))
+      CHECK_NN(nnapi->ANeuralNetworksModel_addOperand(nn_model, &operand_type))
+      CHECK_NN(nnapi->ANeuralNetworksModel_setOperandValue(
+          nn_model, next_id, &value, sizeof(float)))
       augmented_inputs.push_back(next_id++);
     };
 
@@ -281,8 +254,8 @@ TfLiteStatus AddOpsAndParams(
           .type = ANEURALNETWORKS_TENSOR_INT32,
           .dimensionCount = 1,
           .dimensions = &num_values};
-      CHECK_NN(ANeuralNetworksModel_addOperand(nn_model, &operand_type))
-      CHECK_NN(ANeuralNetworksModel_setOperandValue(
+      CHECK_NN(nnapi->ANeuralNetworksModel_addOperand(nn_model, &operand_type))
+      CHECK_NN(nnapi->ANeuralNetworksModel_setOperandValue(
           nn_model, next_id, values, sizeof(int32_t) * num_values));
       augmented_inputs.push_back(next_id++);
     };
@@ -291,15 +264,16 @@ TfLiteStatus AddOpsAndParams(
     // For each state_out tensor, a corresponding state_in operand needs to be
     // created for NNAPI.
     auto duplicate_state_tensor_float32 =
-        [subgraph, &nn_model, &next_id, &augmented_inputs, &model_state_inputs,
-         &model_state_outputs](int tensor_id) {
+        [nnapi, subgraph, &nn_model, &next_id, &augmented_inputs,
+         &model_state_inputs, &model_state_outputs](int tensor_id) {
           const TfLiteTensor* tensor = subgraph->tensor(tensor_id);
           ANeuralNetworksOperandType operand_type{
               ANEURALNETWORKS_TENSOR_FLOAT32,
               static_cast<uint32_t>(tensor->dims->size),
               reinterpret_cast<uint32_t*>(tensor->dims->data),
               tensor->params.scale, tensor->params.zero_point};
-          CHECK_NN(ANeuralNetworksModel_addOperand(nn_model, &operand_type));
+          CHECK_NN(
+              nnapi->ANeuralNetworksModel_addOperand(nn_model, &operand_type));
           augmented_inputs.push_back(next_id);
           model_state_inputs->push_back(next_id);
           model_state_outputs->push_back(tensor_id);
@@ -388,7 +362,7 @@ TfLiteStatus AddOpsAndParams(
     };
 
     // LSTM in NNAPI requires scratch tensor as an output operand.
-    auto add_lstm_scratch_tensor_float32 = [subgraph, &node, &nn_model,
+    auto add_lstm_scratch_tensor_float32 = [nnapi, subgraph, &node, &nn_model,
                                             &next_id, &augmented_outputs]() {
       if (node.temporaries->size == 0) return;
       int scratch_buffer_index = node.temporaries->data[0];
@@ -398,7 +372,7 @@ TfLiteStatus AddOpsAndParams(
           static_cast<uint32_t>(tensor->dims->size),
           reinterpret_cast<uint32_t*>(tensor->dims->data), tensor->params.scale,
           tensor->params.zero_point};
-      CHECK_NN(ANeuralNetworksModel_addOperand(nn_model, &operand_type));
+      CHECK_NN(nnapi->ANeuralNetworksModel_addOperand(nn_model, &operand_type));
       augmented_outputs.insert(augmented_outputs.begin(), next_id++);
     };
 
@@ -427,15 +401,16 @@ TfLiteStatus AddOpsAndParams(
     };
 
     // Handle optional input tensors.
-    auto add_optional_tensors = [&nn_model, &augmented_inputs,
+    auto add_optional_tensors = [nnapi, &nn_model, &augmented_inputs,
                                  &next_id](int nn_type) {
       for (size_t idx = 0; idx < augmented_inputs.size(); idx++) {
         if (augmented_inputs[idx] == kOptionalTensor) {
           const std::vector<uint32_t> dim = {0, 0};
           ANeuralNetworksOperandType operand_type{nn_type, 2, dim.data(), 0, 0};
-          CHECK_NN(ANeuralNetworksModel_addOperand(nn_model, &operand_type))
-          CHECK_NN(ANeuralNetworksModel_setOperandValue(nn_model, next_id,
-                                                        nullptr, 0))
+          CHECK_NN(
+              nnapi->ANeuralNetworksModel_addOperand(nn_model, &operand_type))
+          CHECK_NN(nnapi->ANeuralNetworksModel_setOperandValue(
+              nn_model, next_id, nullptr, 0))
           augmented_inputs[idx] = next_id++;
         }
       }
@@ -696,13 +671,13 @@ TfLiteStatus AddOpsAndParams(
         break;
     }
 
-    if (nnapi_version == 11 && GetAndroidSdkVersionCached() < 28) {
+    if (nnapi_version == 11 && nnapi->android_sdk_version < 28) {
       logError("Op %d needs NNAPI1.1", builtin);
       return kTfLiteError;
     }
 
     // Add the operation.
-    RETURN_ERROR_IF_NN_FAILED(ANeuralNetworksModel_addOperation(
+    RETURN_ERROR_IF_NN_FAILED(nnapi->ANeuralNetworksModel_addOperation(
         nn_model, nn_op_type, static_cast<uint32_t>(augmented_inputs.size()),
         augmented_inputs.data(),
         static_cast<uint32_t>(augmented_outputs.size()),
@@ -714,9 +689,10 @@ TfLiteStatus AddOpsAndParams(
 TfLiteStatus NNAPIDelegate::BuildGraph(Subgraph* subgraph) {
   if (nn_model_ && nn_compiled_model_) return model_status_;
 
+  const NnApi* nnapi = NnApiImplementation();
   // TODO(aselle): This is not correct. need to handle resize invalidation.
   if (!nn_model_) {
-    CHECK_NN(ANeuralNetworksModel_create(&nn_model_));
+    CHECK_NN(nnapi->ANeuralNetworksModel_create(&nn_model_));
 
     // Find which tensors should be added to NNAPI. TFLite has temporaries
     // and RNN back-edges which are are not valid for NNAPI. We look through all
@@ -763,21 +739,22 @@ TfLiteStatus NNAPIDelegate::BuildGraph(Subgraph* subgraph) {
                        model_states_outputs_.size(), &augmented_outputs,
                        tensor_id_to_nnapi_id);
 
-    CHECK_NN(ANeuralNetworksModel_identifyInputsAndOutputs(
+    CHECK_NN(nnapi->ANeuralNetworksModel_identifyInputsAndOutputs(
         nn_model_, static_cast<uint32_t>(augmented_inputs.size()),
         reinterpret_cast<const uint32_t*>(augmented_inputs.data()),
         static_cast<uint32_t>(augmented_outputs.size()),
         reinterpret_cast<const uint32_t*>(augmented_outputs.data())));
 
-    if (GetAndroidSdkVersionCached() >= 28) {
-      CHECK_NN(ANeuralNetworksModel_relaxComputationFloat32toFloat16(
+    if (nnapi->android_sdk_version >= 28) {
+      CHECK_NN(nnapi->ANeuralNetworksModel_relaxComputationFloat32toFloat16(
           nn_model_, subgraph->GetAllowFp16PrecisionForFp32()));
     }
-    CHECK_NN(ANeuralNetworksModel_finish(nn_model_));
+    CHECK_NN(nnapi->ANeuralNetworksModel_finish(nn_model_));
   }
   if (!nn_compiled_model_) {
-    CHECK_NN(ANeuralNetworksCompilation_create(nn_model_, &nn_compiled_model_));
-    CHECK_NN(ANeuralNetworksCompilation_finish(nn_compiled_model_));
+    CHECK_NN(nnapi->ANeuralNetworksCompilation_create(nn_model_,
+                                                      &nn_compiled_model_));
+    CHECK_NN(nnapi->ANeuralNetworksCompilation_finish(nn_compiled_model_));
   }
   return kTfLiteOk;
 }
@@ -793,8 +770,10 @@ TfLiteStatus NNAPIDelegate::Invoke(Subgraph* subgraph) {
     return model_status_;
   }
 
+  const NnApi* nnapi = NnApiImplementation();
   ANeuralNetworksExecution* execution = nullptr;
-  CHECK_NN(ANeuralNetworksExecution_create(nn_compiled_model_, &execution));
+  CHECK_NN(
+      nnapi->ANeuralNetworksExecution_create(nn_compiled_model_, &execution));
 
   // Currently perform deep copy of input buffer
   for (size_t i = 0; i < subgraph->inputs().size(); i++) {
@@ -802,7 +781,7 @@ TfLiteStatus NNAPIDelegate::Invoke(Subgraph* subgraph) {
     // TODO(aselle): Is this what we want or do we want input instead?
     // TODO(aselle): This should be called setInputValue maybe to be cons.
     TfLiteTensor* tensor = subgraph->tensor(input);
-    CHECK_NN(ANeuralNetworksExecution_setInput(
+    CHECK_NN(nnapi->ANeuralNetworksExecution_setInput(
         execution, i, nullptr, tensor->data.raw, tensor->bytes));
   }
 
@@ -810,7 +789,7 @@ TfLiteStatus NNAPIDelegate::Invoke(Subgraph* subgraph) {
   for (size_t i = 0; i < subgraph->outputs().size(); i++) {
     int output = subgraph->outputs()[i];
     TfLiteTensor* tensor = subgraph->tensor(output);
-    CHECK_NN(ANeuralNetworksExecution_setOutput(
+    CHECK_NN(nnapi->ANeuralNetworksExecution_setOutput(
         execution, i, nullptr, tensor->data.raw, tensor->bytes));
   }
 
@@ -822,21 +801,21 @@ TfLiteStatus NNAPIDelegate::Invoke(Subgraph* subgraph) {
     // Here we are using a deep copy for state_in tensors so that we are not
     // reading and writing into the same buffer during a invocation.
     // TODO(miaowang): using double shared buffer to minimize the copies.
-    CHECK_NN(ANeuralNetworksExecution_setInput(
+    CHECK_NN(nnapi->ANeuralNetworksExecution_setInput(
         execution, i + subgraph->inputs().size(), nullptr, tensor->data.raw,
         tensor->bytes));
     // Tell NNAPI where to output the state_out.
-    CHECK_NN(ANeuralNetworksExecution_setOutput(
+    CHECK_NN(nnapi->ANeuralNetworksExecution_setOutput(
         execution, i + subgraph->outputs().size(), nullptr, tensor->data.raw,
         tensor->bytes));
   }
 
   // Currently use blocking compute.
   ANeuralNetworksEvent* event = nullptr;
-  CHECK_NN(ANeuralNetworksExecution_startCompute(execution, &event));
-  CHECK_NN(ANeuralNetworksEvent_wait(event));
-  ANeuralNetworksEvent_free(event);
-  ANeuralNetworksExecution_free(execution);
+  CHECK_NN(nnapi->ANeuralNetworksExecution_startCompute(execution, &event));
+  CHECK_NN(nnapi->ANeuralNetworksEvent_wait(event));
+  nnapi->ANeuralNetworksEvent_free(event);
+  nnapi->ANeuralNetworksExecution_free(execution);
 
 #if 0
   printf("From the NN API:\n");
@@ -854,6 +833,8 @@ TfLiteStatus NNAPIDelegate::Invoke(Subgraph* subgraph) {
   return kTfLiteOk;
 }
 
-bool NNAPIDelegate::IsSupported() { return NNAPIExists(); }
+bool NNAPIDelegate::IsSupported() {
+  return NnApiImplementation()->nnapi_exists;
+}
 
 }  // namespace tflite
-- 
GitLab


From 185c0233e5533788fd5c4679acd0a4b64484dc03 Mon Sep 17 00:00:00 2001
From: Jian Li <jianlijianli@google.com>
Date: Wed, 2 Jan 2019 13:42:04 -0800
Subject: [PATCH 0140/1095] Fix an error in neon tensor util. vget_high_s8 gets
 the high bits.

PiperOrigin-RevId: 227574080
---
 tensorflow/lite/kernels/internal/optimized/neon_tensor_utils.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/kernels/internal/optimized/neon_tensor_utils.cc b/tensorflow/lite/kernels/internal/optimized/neon_tensor_utils.cc
index cf40ebb241..cf1bda2661 100644
--- a/tensorflow/lite/kernels/internal/optimized/neon_tensor_utils.cc
+++ b/tensorflow/lite/kernels/internal/optimized/neon_tensor_utils.cc
@@ -144,7 +144,7 @@ void NeonMatrixBatchVectorMultiplyAccumulate(
         // registers).
         int16x8_t prod_16x8 =
             vmull_s8(vget_low_s8(s1_8x16), vget_low_s8(s2_8x16));
-        // Multiply the high bits (i.e. the lower 8 8bit numbers in the
+        // Multiply the high bits (i.e. the higher 8 8bit numbers in the
         // registers), and accumulate with the result of the low bits product.
         // The assumption here is that overflow will not happen as we quantize
         // our values to be in the range [-127, 127]. As such the sum of the 2
-- 
GitLab


From 9cb5ebb7dfbe262204cf153a84fb17cd4fc70f00 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 2 Jan 2019 13:58:06 -0800
Subject: [PATCH 0141/1095] Add a ragged rank op and register it to the op
 dispatcher

PiperOrigin-RevId: 227576633
---
 tensorflow/python/ops/array_ops.py            |  1 +
 tensorflow/python/ops/ragged/BUILD            | 13 +++
 .../python/ops/ragged/ragged_array_ops.py     | 30 +++++++
 .../python/ops/ragged/ragged_dispatch.py      |  1 +
 .../python/ops/ragged/ragged_dispatch_test.py |  4 +
 .../python/ops/ragged/ragged_rank_op_test.py  | 89 +++++++++++++++++++
 6 files changed, 138 insertions(+)
 create mode 100644 tensorflow/python/ops/ragged/ragged_rank_op_test.py

diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py
index 81ea473466..07a7e180fb 100644
--- a/tensorflow/python/ops/array_ops.py
+++ b/tensorflow/python/ops/array_ops.py
@@ -430,6 +430,7 @@ def size_internal(input, name=None, optimize=True, out_type=dtypes.int32):
 
 
 @tf_export("rank")
+@dispatch.add_dispatch_support
 def rank(input, name=None):
   # pylint: disable=redefined-builtin
   """Returns the rank of a tensor.
diff --git a/tensorflow/python/ops/ragged/BUILD b/tensorflow/python/ops/ragged/BUILD
index 89b8c4a2b3..46f7fa62a3 100644
--- a/tensorflow/python/ops/ragged/BUILD
+++ b/tensorflow/python/ops/ragged/BUILD
@@ -720,6 +720,19 @@ py_test(
     ],
 )
 
+py_test(
+    name = "ragged_rank_op_test",
+    srcs = ["ragged_rank_op_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":ragged_array_ops",
+        ":ragged_factory_ops",
+        ":ragged_test_util",
+        "//tensorflow/python:framework_test_lib",
+        "@absl_py//absl/testing:parameterized",
+    ],
+)
+
 py_test(
     name = "ragged_tile_op_test",
     srcs = ["ragged_tile_op_test.py"],
diff --git a/tensorflow/python/ops/ragged/ragged_array_ops.py b/tensorflow/python/ops/ragged/ragged_array_ops.py
index 3c5e697cdc..a6b2442f09 100644
--- a/tensorflow/python/ops/ragged/ragged_array_ops.py
+++ b/tensorflow/python/ops/ragged/ragged_array_ops.py
@@ -1226,3 +1226,33 @@ def _nrows(rt_input, out_type=dtypes.int64, name=None):
   else:
     with ops.name_scope(name, 'RaggedNRows', [rt_input]):
       return array_ops.shape(rt_input, out_type=out_type)[0]
+
+
+#===============================================================================
+# ragged.rank
+#===============================================================================
+def rank(input, name=None):  # pylint: disable=redefined-builtin
+  """Returns the rank of a RaggedTensor.
+
+  Returns a 0-D `int32` `Tensor` representing the rank of `input`.
+
+  For example:
+
+  ```python
+  # shape of tensor 't' is [2, None, None]
+  t = tf.ragged.constant([[[1], [2, 2]], [[3, 3, 3], [4, 4, 4, 4]]])
+  tf.rank(t)  # 3
+  ```
+
+  Args:
+    input: A `RaggedTensor`
+    name: A name for the operation (optional).
+
+  Returns:
+    A `Tensor` of type `int32`.
+  """
+  with ops.name_scope(name, 'RaggedRank', [input]) as name:
+    if not ragged_tensor.is_ragged(input):
+      return array_ops.rank(input, name)
+
+    return input.ragged_rank + array_ops.rank(input.flat_values)
diff --git a/tensorflow/python/ops/ragged/ragged_dispatch.py b/tensorflow/python/ops/ragged/ragged_dispatch.py
index b024bd9f07..52c4142346 100644
--- a/tensorflow/python/ops/ragged/ragged_dispatch.py
+++ b/tensorflow/python/ops/ragged/ragged_dispatch.py
@@ -418,6 +418,7 @@ _RAGGED_DISPATCH_OPS = [
     (array_ops.gather, _ragged_gather_v1, ['params', 'indices']),
     (array_ops.gather_v2, ragged_array_ops.gather, ['params', 'indices']),
     (array_ops.gather_nd, ragged_array_ops.gather_nd, ['params', 'indices']),
+    (array_ops.rank, ragged_array_ops.rank, ['input']),
     (array_ops.stack, ragged_array_ops.stack, ['[values]']),
     (array_ops.tile, ragged_array_ops.tile, ['input']),
     (array_ops.where, ragged_array_ops.where, ['condition', 'x', 'y']),
diff --git a/tensorflow/python/ops/ragged/ragged_dispatch_test.py b/tensorflow/python/ops/ragged/ragged_dispatch_test.py
index 9d70470f05..0c546accc3 100644
--- a/tensorflow/python/ops/ragged/ragged_dispatch_test.py
+++ b/tensorflow/python/ops/ragged/ragged_dispatch_test.py
@@ -676,6 +676,10 @@ class RaggedElementwiseOpsTest(ragged_test_util.RaggedTensorTestCase,
                   1
           },
           expected=[False, True]),
+      dict(
+          op=array_ops.rank,
+          kwargs={'input': ragged_factory_ops.constant_value([[8, 3], [5]])},
+          expected=2),
   ])
   def testRaggedDispatch(self, op, expected, args=(), kwargs=None):
     if kwargs is None: kwargs = {}
diff --git a/tensorflow/python/ops/ragged/ragged_rank_op_test.py b/tensorflow/python/ops/ragged/ragged_rank_op_test.py
new file mode 100644
index 0000000000..54eee3bc04
--- /dev/null
+++ b/tensorflow/python/ops/ragged/ragged_rank_op_test.py
@@ -0,0 +1,89 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for ragged.rank op."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from absl.testing import parameterized
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops.ragged import ragged_array_ops
+from tensorflow.python.ops.ragged import ragged_factory_ops
+from tensorflow.python.ops.ragged import ragged_test_util
+from tensorflow.python.platform import googletest
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class RaggedRankOpTest(ragged_test_util.RaggedTensorTestCase,
+                       parameterized.TestCase):
+
+  @parameterized.parameters([
+      # Rank 0
+      dict(
+          test_input=1,
+          expected_rank=0,
+      ),
+      # Rank 1
+      dict(
+          test_input=[1],
+          expected_rank=1,
+      ),
+      dict(
+          test_input=[1, 2, 3, 4],
+          expected_rank=1,
+      ),
+      # Rank 2
+      dict(
+          test_input=[[1], [2], [3]],
+          expected_rank=2,
+      ),
+      # Rank 3
+      dict(
+          test_input=[[[1], [2, 3]], [[4], [5, 6, 7]]],
+          expected_rank=3,
+      ),
+      # Rank 3, ragged_rank=2
+      dict(
+          test_input=[[[1], [2, 3], [10, 20]],
+                      [[4], [5, 6, 7]]],
+          expected_rank=3,
+          ragged_rank=2,
+      ),
+      # Rank 4, ragged_rank=3 with dimensions: {2, (1, 2), (2), (1, 2)}
+      dict(
+          test_input=[[[[1], [2]]],
+                      [[[3, 4], [5, 6]], [[7, 8], [9, 10]]]],
+          expected_rank=4,
+      ),
+      # Rank 4, ragged_rank=2 with dimensions: {2, (1, 2), (1, 2), 2}
+      dict(
+          test_input=[
+              [[[1, 2]]],
+              [[[5, 6], [7, 8]],
+               [[9, 10], [11, 12]]]],
+          expected_rank=4,
+          ragged_rank=2,
+      ),
+
+  ])
+  def testRaggedRank(self, test_input, expected_rank, ragged_rank=None):
+    test_input = ragged_factory_ops.constant(
+        test_input, ragged_rank=ragged_rank)
+    self.assertAllEqual(ragged_array_ops.rank(
+        test_input), expected_rank)
+
+
+if __name__ == '__main__':
+  googletest.main()
-- 
GitLab


From 0c2565de1108fc2063ec04b335ef7356298b7ad6 Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Wed, 2 Jan 2019 14:13:27 -0800
Subject: [PATCH 0142/1095] Don't lower nested control flow if we're compiling
 to XLA.

PiperOrigin-RevId: 227579512
---
 .../python/kernel_tests/cond_v2_test.py       | 46 ++++++++++++++++++-
 tensorflow/python/ops/control_flow_util.py    |  9 ++++
 tensorflow/python/ops/control_flow_util_v2.py |  2 +-
 3 files changed, 55 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/kernel_tests/cond_v2_test.py b/tensorflow/python/kernel_tests/cond_v2_test.py
index 2c99026105..050da5ff6c 100644
--- a/tensorflow/python/kernel_tests/cond_v2_test.py
+++ b/tensorflow/python/kernel_tests/cond_v2_test.py
@@ -145,6 +145,22 @@ class CondV2Test(test.TestCase):
     self.assertEqual(cond_op.type, "If")
     return output, cond_op
 
+  def _createNestedCond(self, name):
+    """Like _createCond but creates a nested cond_v2 call as well."""
+    pred = constant_op.constant(True, name="pred")
+    x = constant_op.constant(1.0, name="x")
+
+    def true_fn():
+      return cond_v2.cond_v2(pred, lambda: x, lambda: x + 1)
+
+    def false_fn():
+      return x + 2
+
+    output = cond_v2.cond_v2(pred, true_fn, false_fn, name=name)
+    cond_op = output.op.inputs[0].op
+    self.assertEqual(cond_op.type, "If")
+    return output, cond_op
+
   def testDefaultName(self):
     with ops.Graph().as_default():
       _, cond_op = self._createCond(None)
@@ -645,9 +661,14 @@ class CondV2Test(test.TestCase):
       # Build the cond_v2 in an XLA context
       xla_context = control_flow_ops.XLAControlFlowContext()
       xla_context.Enter()
-      cond_output, _ = self._createCond("cond")
+      cond_output, cond_op = self._createCond("cond")
       xla_context.Exit()
 
+      # Check lowering attr is not set.
+      with self.assertRaises(ValueError):
+        cond_op.get_attr("_lower_using_switch_merge")
+
+      # Check the actual graph that is run.
       run_options = config_pb2.RunOptions(output_partition_graphs=True)
       run_metadata = config_pb2.RunMetadata()
       sess.run(cond_output, options=run_options, run_metadata=run_metadata)
@@ -672,6 +693,29 @@ class CondV2Test(test.TestCase):
           if_found,
           "An `If` op was not found, but the graph should not be lowered.")
 
+  @test_util.run_deprecated_v1
+  def testNestedLoweringDisabledInXLA(self):
+    # Build the cond_v2 in an XLA context
+    xla_context = control_flow_ops.XLAControlFlowContext()
+    xla_context.Enter()
+    _, cond_op = self._createNestedCond("cond")
+    xla_context.Exit()
+
+    # Check lowering attr is not set for either If node.
+    with self.assertRaises(ValueError):
+      cond_op.get_attr("_lower_using_switch_merge")
+
+    nested_if_ops = []
+    for func in ops.get_default_graph()._functions.values():
+      nested_if_ops.extend(op for op in func._graph.get_operations()
+                           if op.type == "If")
+    self.assertEqual(len(nested_if_ops), 1)
+    with self.assertRaises(ValueError):
+      nested_if_ops[0].get_attr("_lower_using_switch_merge")
+
+    # TODO(skyewm): check the actual graphs that are run once we have a way to
+    # programmatically access those graphs.
+
   @test_util.run_deprecated_v1
   def testLoweringDisabledWithSingleThreadedExecutorContext(self):
     with self.session(graph=ops.Graph()) as sess:
diff --git a/tensorflow/python/ops/control_flow_util.py b/tensorflow/python/ops/control_flow_util.py
index 8f5442da5e..ff0dff0042 100644
--- a/tensorflow/python/ops/control_flow_util.py
+++ b/tensorflow/python/ops/control_flow_util.py
@@ -57,6 +57,15 @@ def InXlaContext(graph):
   return GetContainingXLAContext(ctxt) is not None
 
 
+def GraphOrParentsInXlaContext(graph):
+  while True:
+    if InXlaContext(graph): return True
+    try:
+      graph = graph.outer_graph
+    except AttributeError:
+      return False
+
+
 def IsInWhileLoop(op):
   ctxt = op._get_control_flow_context()  # pylint: disable=protected-access
   return GetContainingWhileContext(ctxt) is not None
diff --git a/tensorflow/python/ops/control_flow_util_v2.py b/tensorflow/python/ops/control_flow_util_v2.py
index 5f56850884..58917ad264 100644
--- a/tensorflow/python/ops/control_flow_util_v2.py
+++ b/tensorflow/python/ops/control_flow_util_v2.py
@@ -114,7 +114,7 @@ def maybe_set_lowering_attr(op):
   Args:
     op: An `If` or `While` Operation.
   """
-  if (not control_flow_util.IsInXLAContext(op) and
+  if (not control_flow_util.GraphOrParentsInXlaContext(op.graph) and
       context.context().get_function_call_options().executor_type
       != "SINGLE_THREADED_EXECUTOR"):
     # pylint: disable=protected-access
-- 
GitLab


From eef699b9f8c158d51a00f127b1a2854d29c6751b Mon Sep 17 00:00:00 2001
From: Russell Power <power@google.com>
Date: Wed, 2 Jan 2019 14:22:57 -0800
Subject: [PATCH 0143/1095] Rename shutdown mode field to better fit behavior.

PiperOrigin-RevId: 227581084
---
 .../contrib/tpu/python/tpu/session_support.py    | 16 ++++++++++------
 tensorflow/core/util/event.proto                 |  2 +-
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/session_support.py b/tensorflow/contrib/tpu/python/tpu/session_support.py
index 3e463823c8..f5735cecc3 100644
--- a/tensorflow/contrib/tpu/python/tpu/session_support.py
+++ b/tensorflow/contrib/tpu/python/tpu/session_support.py
@@ -185,7 +185,8 @@ def all_worker_devices(session):
   """Return a list of devices for each worker in the system."""
   devices = session.list_devices()
   return [
-      device.name for device in devices
+      device.name
+      for device in devices
       if ':CPU:' in device.name and 'coordinator' not in device.name
   ]
 
@@ -255,12 +256,14 @@ class WatchdogManager(threading.Thread):
     self._worker_manager.configure(
         event_pb2.WorkerHeartbeatRequest(
             watchdog_config=event_pb2.WatchdogConfig(
-                timeout_ms=self.shutdown_timeout * 1000,)))
+                timeout_ms=self.shutdown_timeout * 1000,),
+            shutdown_mode=event_pb2.WAIT_FOR_COORDINATOR))
 
   def configure_and_run(self):
-    logging.info('Enabling watchdog timer with %d second timeout '
-                 'and %d second ping interval.',
-                 self.shutdown_timeout, self.ping_interval)
+    logging.info(
+        'Enabling watchdog timer with %d second timeout '
+        'and %d second ping interval.', self.shutdown_timeout,
+        self.ping_interval)
     self._reset_manager()
     self._running = True
     self.start()
@@ -269,7 +272,8 @@ class WatchdogManager(threading.Thread):
     logging.info('Stopping worker watchdog.')
     self._worker_manager.configure(
         event_pb2.WorkerHeartbeatRequest(
-            watchdog_config=event_pb2.WatchdogConfig(timeout_ms=-1,)))
+            watchdog_config=event_pb2.WatchdogConfig(timeout_ms=-1,),
+            shutdown_mode=event_pb2.NOT_CONFIGURED))
     self._running = False
     self.join()
 
diff --git a/tensorflow/core/util/event.proto b/tensorflow/core/util/event.proto
index 9ce85be551..2d3ae62777 100644
--- a/tensorflow/core/util/event.proto
+++ b/tensorflow/core/util/event.proto
@@ -95,7 +95,7 @@ enum WorkerHealth {
 // signal is received.
 enum WorkerShutdownMode {
   DEFAULT = 0;
-  SHUTDOWN_IMMEDIATELY = 1;
+  NOT_CONFIGURED = 1;
   WAIT_FOR_COORDINATOR = 2;
 }
 
-- 
GitLab


From 85ca0e9b815fc8438f1c10379c18393783021e15 Mon Sep 17 00:00:00 2001
From: "Joshua V. Dillon" <jvdillon@google.com>
Date: Wed, 2 Jan 2019 14:31:31 -0800
Subject: [PATCH 0144/1095] Remove tf.contrib.timeseries dependency on TF
 distributions.

PiperOrigin-RevId: 227582617
---
 .../timeseries/python/timeseries/BUILD        |  3 +-
 .../timeseries/python/timeseries/ar_model.py  | 19 ++++---------
 .../python/timeseries/math_utils.py           | 28 +++++++++++++++++++
 .../timeseries/state_space_models/BUILD       |  2 --
 .../filtering_postprocessor.py                |  8 ++----
 .../state_space_models/kalman_filter.py       |  9 +++---
 6 files changed, 43 insertions(+), 26 deletions(-)

diff --git a/tensorflow/contrib/timeseries/python/timeseries/BUILD b/tensorflow/contrib/timeseries/python/timeseries/BUILD
index 4b90b596b2..a0c3204d41 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/BUILD
+++ b/tensorflow/contrib/timeseries/python/timeseries/BUILD
@@ -361,9 +361,10 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":feature_keys",
+        ":math_utils",
         ":model",
         ":model_utils",
-        "//tensorflow/contrib/distributions:distributions_py",
+        "//tensorflow/contrib/rnn:rnn_py",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:check_ops",
         "//tensorflow/python:constant_op",
diff --git a/tensorflow/contrib/timeseries/python/timeseries/ar_model.py b/tensorflow/contrib/timeseries/python/timeseries/ar_model.py
index bcadf4094e..a8d5e1a49d 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/ar_model.py
+++ b/tensorflow/contrib/timeseries/python/timeseries/ar_model.py
@@ -18,9 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib import distributions
-
 from tensorflow.contrib.rnn.python.ops import lstm_ops
+from tensorflow.contrib.timeseries.python.timeseries import math_utils
 from tensorflow.contrib.timeseries.python.timeseries import model
 from tensorflow.contrib.timeseries.python.timeseries import model_utils
 from tensorflow.contrib.timeseries.python.timeseries.feature_keys import PredictionFeatures
@@ -462,8 +461,8 @@ class ARModel(model.TimeSeriesModel):
     if self.loss == ARModel.NORMAL_LIKELIHOOD_LOSS:
       covariance = prediction_ops["covariance"]
       sigma = math_ops.sqrt(gen_math_ops.maximum(covariance, 1e-5))
-      normal = distributions.Normal(loc=targets, scale=sigma)
-      loss_op = -math_ops.reduce_sum(normal.log_prob(prediction))
+      loss_op = -math_ops.reduce_sum(
+          math_utils.normal_log_prob(targets, sigma, prediction))
     else:
       assert self.loss == ARModel.SQUARED_LOSS, self.loss
       loss_op = math_ops.reduce_sum(math_ops.square(prediction - targets))
@@ -965,16 +964,11 @@ class AnomalyMixtureARModel(ARModel):
       anomaly_variance = prediction_ops["anomaly_params"]
       anomaly_sigma = math_ops.sqrt(
           gen_math_ops.maximum(anomaly_variance, 1e-5))
-      normal = distributions.Normal(loc=targets, scale=anomaly_sigma)
-      log_prob = normal.log_prob(prediction)
+      log_prob = math_utils.normal_log_prob(targets, anomaly_sigma, prediction)
     else:
       assert self._anomaly_distribution == AnomalyMixtureARModel.CAUCHY_ANOMALY
       anomaly_scale = prediction_ops["anomaly_params"]
-      cauchy = distributions.StudentT(
-          df=array_ops.ones([], dtype=anomaly_scale.dtype),
-          loc=targets,
-          scale=anomaly_scale)
-      log_prob = cauchy.log_prob(prediction)
+      log_prob = math_utils.cauchy_log_prob(targets, anomaly_scale, prediction)
     return log_prob
 
   def loss_op(self, targets, prediction_ops):
@@ -983,8 +977,7 @@ class AnomalyMixtureARModel(ARModel):
     covariance = prediction_ops["covariance"]
     # Normal data log probability.
     sigma = math_ops.sqrt(gen_math_ops.maximum(covariance, 1e-5))
-    normal1 = distributions.Normal(loc=targets, scale=sigma)
-    log_prob1 = normal1.log_prob(prediction)
+    log_prob1 = math_utils.normal_log_prob(targets, sigma, prediction)
     log_prob1 += math_ops.log(1 - self._anomaly_prior_probability)
     # Anomaly log probability.
     log_prob2 = self._anomaly_log_prob(targets, prediction_ops)
diff --git a/tensorflow/contrib/timeseries/python/timeseries/math_utils.py b/tensorflow/contrib/timeseries/python/timeseries/math_utils.py
index aab3306438..b7375e5055 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/math_utils.py
+++ b/tensorflow/contrib/timeseries/python/timeseries/math_utils.py
@@ -21,6 +21,8 @@ from __future__ import print_function
 import collections
 import math
 
+import numpy as np
+
 from tensorflow.contrib import lookup
 from tensorflow.contrib.layers.python.layers import layers
 
@@ -43,6 +45,32 @@ from tensorflow.python.ops import variable_scope
 from tensorflow.python.util import nest
 
 
+def normal_log_prob(loc, scale, x):
+  """Computes the Normal log pdf."""
+  z = (x - loc) / scale
+  return -0.5 * (math_ops.square(z)
+                 + np.log(2. * np.pi) + math_ops.log(scale))
+
+
+def cauchy_log_prob(loc, scale, x):
+  """Computes the Cauchy log pdf."""
+  z = (x - loc) / scale
+  return (-np.log(np.pi) - math_ops.log(scale) -
+          math_ops.log1p(math_ops.square(z)))
+
+
+def mvn_tril_log_prob(loc, scale_tril, x):
+  """Computes the MVN log pdf under tril scale. Doesn't handle batches."""
+  x0 = x - loc
+  z = linalg_ops.matrix_triangular_solve(
+      scale_tril, x0[..., array_ops.newaxis])[..., 0]
+  log_det_cov = 2. * math_ops.reduce_sum(math_ops.log(
+      array_ops.matrix_diag_part(scale_tril)), axis=-1)
+  d = math_ops.cast(array_ops.shape(scale_tril)[-1], log_det_cov.dtype)
+  return -0.5 * (math_ops.reduce_sum(math_ops.square(z), axis=-1)
+                 + d * np.log(2. * np.pi) + log_det_cov)
+
+
 def clip_covariance(
     covariance_matrix, maximum_variance_ratio, minimum_variance):
   """Enforce constraints on a covariance matrix to improve numerical stability.
diff --git a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/BUILD b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/BUILD
index 125750e763..cf5e749042 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/BUILD
+++ b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/BUILD
@@ -78,7 +78,6 @@ py_library(
     srcs = ["kalman_filter.py"],
     srcs_version = "PY2AND3",
     deps = [
-        "//tensorflow/contrib/distributions:distributions_py",
         "//tensorflow/contrib/timeseries/python/timeseries:math_utils",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:control_flow_ops",
@@ -235,7 +234,6 @@ py_library(
     srcs = ["filtering_postprocessor.py"],
     srcs_version = "PY2AND3",
     deps = [
-        "//tensorflow/contrib/distributions:distributions_py",
         "//tensorflow/contrib/timeseries/python/timeseries:math_utils",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:check_ops",
diff --git a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/filtering_postprocessor.py b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/filtering_postprocessor.py
index e9e2ac0aaf..3fa2fbd9f7 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/filtering_postprocessor.py
+++ b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/filtering_postprocessor.py
@@ -22,8 +22,6 @@ import abc
 
 import six
 
-from tensorflow.contrib import distributions
-
 from tensorflow.contrib.timeseries.python.timeseries import math_utils
 
 from tensorflow.python.framework import dtypes
@@ -91,10 +89,10 @@ def cauchy_alternative_to_gaussian(current_times, current_values, outputs):
   """
   del current_times  # unused
   cauchy_scale = math_utils.entropy_matched_cauchy_scale(outputs["covariance"])
-  individual_log_pdfs = distributions.StudentT(
-      df=array_ops.ones([], dtype=current_values.dtype),
+  individual_log_pdfs = math_utils.cauchy_log_prob(
       loc=outputs["mean"],
-      scale=cauchy_scale).log_prob(current_values)
+      scale=cauchy_scale,
+      x=current_values)
   return math_ops.reduce_sum(individual_log_pdfs, axis=1)
 
 
diff --git a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/kalman_filter.py b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/kalman_filter.py
index a614386121..c0ec797bc5 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/kalman_filter.py
+++ b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/kalman_filter.py
@@ -18,8 +18,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib import distributions
-
 from tensorflow.contrib.timeseries.python.timeseries import math_utils
 
 from tensorflow.python.framework import dtypes
@@ -137,9 +135,10 @@ class KalmanFilter(object):
     with ops.control_dependencies([non_negative_assert]):
       observation_covariance_cholesky = linalg_ops.cholesky(
           symmetrized_observation_covariance)
-    log_prediction_prob = distributions.MultivariateNormalTriL(
-        predicted_observation, observation_covariance_cholesky).log_prob(
-            observation)
+    log_prediction_prob = math_utils.mvn_tril_log_prob(
+        loc=predicted_observation,
+        scale_tril=observation_covariance_cholesky,
+        x=observation)
     (posterior_state,
      posterior_state_var) = self.posterior_from_prior_state(
          prior_state=estimated_state,
-- 
GitLab


From 790a635a044c9b33103f1e02f57faf08c86eee36 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Wed, 2 Jan 2019 14:53:55 -0800
Subject: [PATCH 0145/1095] Stop Model.save_weights from printing a deprication
 warning (TF format)

Also has it save relative paths in the checkpoint proto, which removes a behavior difference between it and CheckpointManager.

PiperOrigin-RevId: 227586345
---
 tensorflow/python/keras/engine/network.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/keras/engine/network.py b/tensorflow/python/keras/engine/network.py
index 0837ce6780..8e130aef40 100644
--- a/tensorflow/python/keras/engine/network.py
+++ b/tensorflow/python/keras/engine/network.py
@@ -1382,9 +1382,10 @@ class Network(base_layer.Layer):
             % (optimizer,))
       self._checkpointable_saver.save(filepath, session=session)
       # Record this checkpoint so it's visible from tf.train.latest_checkpoint.
-      checkpoint_management.update_checkpoint_state(
+      checkpoint_management.update_checkpoint_state_internal(
           save_dir=os.path.dirname(filepath),
           model_checkpoint_path=filepath,
+          save_relative_paths=True,
           all_model_checkpoint_paths=[filepath])
 
   def load_weights(self, filepath, by_name=False):
-- 
GitLab


From a7a3bbfcbf53381631dded0e293031e93d056ebc Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Wed, 2 Jan 2019 15:00:14 -0800
Subject: [PATCH 0146/1095] [Grappler] Cleanup function/function_optimizer

1. Migrate to absl containers
2. Compute function signature hash without allocating temporary sorted containers

PiperOrigin-RevId: 227587343
---
 tensorflow/core/grappler/optimizers/BUILD     |   2 +
 .../grappler/optimizers/function_optimizer.cc | 152 +++++++++---------
 .../grappler/optimizers/meta_optimizer.cc     |  19 +--
 tensorflow/core/grappler/utils/BUILD          |   4 +
 tensorflow/core/grappler/utils/functions.cc   |  80 ++++-----
 tensorflow/core/grappler/utils/functions.h    |  52 +++---
 .../core/grappler/utils/functions_test.cc     |   6 +-
 7 files changed, 155 insertions(+), 160 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index 3a1ab28dae..512c3b07b4 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -151,6 +151,8 @@ cc_library(
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/utils:functions",
         "@com_google_absl//absl/algorithm:container",
+        "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
     ],
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.cc b/tensorflow/core/grappler/optimizers/function_optimizer.cc
index 73c950b3fc..d074676c3d 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer.cc
@@ -15,10 +15,11 @@ limitations under the License.
 
 #include "tensorflow/core/grappler/optimizers/function_optimizer.h"
 
-#include <unordered_map>
 #include <vector>
 
 #include "absl/algorithm/container.h"
+#include "absl/container/flat_hash_map.h"
+#include "absl/container/flat_hash_set.h"
 #include "absl/memory/memory.h"
 #include "absl/strings/str_replace.h"
 #include "absl/strings/substitute.h"
@@ -163,10 +164,10 @@ struct FunctionSpecializationSignature {
 
   string func_name;
   bool is_in_fetch_set;
-  gtl::FlatSet<OutputPort> active_outputs;
-  std::unordered_map<string, DataType> type_parameters;
-  std::unordered_map<string, AttrValue> body_parameters;
-  std::unordered_map<InputPort, string> const_inputs;
+  absl::flat_hash_set<OutputPort> active_outputs;
+  absl::flat_hash_map<string, DataType> type_parameters;
+  absl::flat_hash_map<string, AttrValue> body_parameters;
+  absl::flat_hash_map<InputPort, string> const_inputs;
 
   bool operator==(const FunctionSpecializationSignature& other) const {
     bool equals = func_name == other.func_name &&
@@ -189,48 +190,45 @@ struct FunctionSpecializationSignature {
     return true;
   }
 
-  // TODO(ezhulenev): Migrate to AbslHashValue.
-  // TODO(ezhulenev): Optimize performance by computing hashes of unordered
-  // values first, and then compute a hash of sorted hashes.
-  struct Hash {
-    uint64 operator()(FunctionSpecializationSignature const& s) const {
-      uint64 h = Hash64(s.func_name);
-      h = Hash64Combine(std::hash<bool>()(s.is_in_fetch_set), h);
-
-      // Use std::set/std::map for deterministic iteration order.
-
-      std::set<OutputPort> active_outputs(s.active_outputs.begin(),
-                                          s.active_outputs.end());
-      for (const auto& active_output : active_outputs) {
-        h = Hash64Combine(std::hash<int>()(active_output), h);
-      }
-
-      std::map<string, DataType> types(s.type_parameters.begin(),
-                                       s.type_parameters.end());
-      for (const auto& pair : types) {
-        AttrValue attr_value;
-        attr_value.set_type(pair.second);
-        h = Hash64Combine(Hash64(pair.first), h);
-        h = Hash64Combine(AttrValueHash(attr_value), h);
-      }
-
-      std::map<string, AttrValue> body(s.body_parameters.begin(),
-                                       s.body_parameters.end());
-      for (const auto& pair : body) {
-        h = Hash64Combine(Hash64(pair.first), h);
-        h = Hash64Combine(FastAttrValueHash(pair.second), h);
-      }
-
-      std::map<InputPort, string> inputs(s.const_inputs.begin(),
-                                         s.const_inputs.end());
-      for (const auto& pair : inputs) {
-        h = Hash64Combine(std::hash<int>()(pair.first), h);
-        h = Hash64Combine(Hash64(pair.second), h);
-      }
-
-      return h;
-    }
-  };
+  template <typename H>
+  friend H AbslHashValue(H h, const FunctionSpecializationSignature& s) {
+    H base = H::combine(std::move(h), s.func_name, s.is_in_fetch_set);
+
+    // First pre-compute hashes for all values in collections with
+    // non-deterministic iteration order.
+    std::vector<uint64> hashes;
+    hashes.reserve(s.active_outputs.size()         //
+                   + s.type_parameters.size() * 2  //
+                   + s.body_parameters.size() * 2  //
+                   + s.const_inputs.size() * 2);
+
+    absl::c_transform(s.active_outputs, std::back_inserter(hashes),
+                      hash<OutputPort>());
+
+    using TypeParam = std::pair<const string, DataType>;
+    absl::c_for_each(s.type_parameters, [&hashes](const TypeParam& type_param) {
+      AttrValue attr_value;
+      attr_value.set_type(type_param.second);
+      hashes.push_back(Hash64(type_param.first));
+      hashes.push_back(AttrValueHash(attr_value));
+    });
+
+    using BodyParam = std::pair<const string, AttrValue>;
+    absl::c_for_each(s.body_parameters, [&hashes](const BodyParam& body_param) {
+      hashes.push_back(Hash64(body_param.first));
+      hashes.push_back(FastAttrValueHash(body_param.second));
+    });
+
+    using ConstInput = std::pair<const InputPort, string>;
+    absl::c_for_each(s.const_inputs, [&hashes](const ConstInput& const_input) {
+      hashes.push_back(hash<InputPort>()(const_input.first));
+      hashes.push_back(Hash64(const_input.second));
+    });
+
+    // Combine all pre-computed hashes in a deterministic order.
+    absl::c_sort(hashes);
+    return H::combine_contiguous(std::move(base), hashes.data(), hashes.size());
+  }
 };
 
 struct FunctionSpecialization {
@@ -238,13 +236,13 @@ struct FunctionSpecialization {
   // True if the function caller node is in GrapplerItem fetch set.
   bool is_in_fetch_set;
   // Names of the tensors that were pushed down into the function body.
-  gtl::FlatSet<string> const_inputs;
+  absl::flat_hash_set<string> const_inputs;
   // Control dependencies of pushed down const inputs have to be attached to
   // function caller node.
-  gtl::FlatSet<string> control_deps;
+  absl::flat_hash_set<string> control_deps;
   // Output tensors (ports) that consumed by other nodes in the graph or in a
   // GrapplerItem fetch set.
-  gtl::FlatSet<int> active_outputs;
+  absl::flat_hash_set<int> active_outputs;
   // Mapping from original function output port to the output port of
   // specialized function. If function specialization changes the number of
   // function outputs it's required to update all node consumers.
@@ -285,12 +283,13 @@ class FunctionOptimizerContext {
     return flr_;
   }
 
-  const gtl::FlatMap<SafeTensorId, SafeTensorId, SafeTensorId::Hasher>&
+  const absl::flat_hash_map<SafeTensorId, SafeTensorId, SafeTensorId::Hasher>&
   tensor_mapping() const {
     return tensor_mapping_;
   }
 
-  const gtl::FlatMap<string, std::vector<string>>& control_overrides() const {
+  const absl::flat_hash_map<string, std::vector<string>>& control_overrides()
+      const {
     return control_overrides_;
   }
 
@@ -298,7 +297,9 @@ class FunctionOptimizerContext {
 
   const string& grappler_item_id() const { return grappler_item_id_; }
 
-  const gtl::FlatSet<string>& fetch_tensors() const { return fetch_tensors_; }
+  const absl::flat_hash_set<string>& fetch_tensors() const {
+    return fetch_tensors_;
+  }
 
   const DeviceSet* devices() const {
     // Create fake devices lazily only if we need a DeviceSet.
@@ -365,7 +366,7 @@ class FunctionOptimizerContext {
 
  private:
   void InitializeTrulyConstNodes(const GrapplerItem& item) {
-    gtl::FlatSet<string> feed_nodes;
+    absl::flat_hash_set<string> feed_nodes;
     for (const auto& feed : item.feed) {
       feed_nodes.insert(NodeName(feed.first));
     }
@@ -411,7 +412,7 @@ class FunctionOptimizerContext {
   FunctionLibraryRuntime* flr_ = nullptr;
 
   // Fully defined names of the devices available to the GrapplerItem.
-  const gtl::FlatSet<string> available_device_names_;
+  const absl::flat_hash_set<string> available_device_names_;
 
   // List of available `FakedDevices` (lazily initialized, see devices()).
   mutable std::vector<std::unique_ptr<Device>> available_devices_;
@@ -421,16 +422,15 @@ class FunctionOptimizerContext {
   mutable DeviceSet available_device_set_;
 
   // Nodes that are Const and not in feed.
-  std::unordered_map<string, const NodeDef*> truly_const_nodes_;
+  absl::flat_hash_map<string, const NodeDef*> truly_const_nodes_;
   // Specialized functions.
-  std::unordered_map<FunctionSpecializationSignature,
-                     const FunctionSpecialization,
-                     FunctionSpecializationSignature::Hash>
+  absl::flat_hash_map<FunctionSpecializationSignature,
+                      const FunctionSpecialization>
       specialized_functions_;
 
   // GrapplerItem.fetch is a vector of tensors.
-  gtl::FlatSet<string> fetch_tensors_;  // format: node_name:port
-  gtl::FlatSet<string> fetch_nodes_;    // format: node_name
+  absl::flat_hash_set<string> fetch_tensors_;  // format: node_name:port
+  absl::flat_hash_set<string> fetch_nodes_;    // format: node_name
 
   // After function inlining and specialization, the optimized graph might be in
   // invalid state, nodes can read from non-existing function call nodes that
@@ -439,7 +439,7 @@ class FunctionOptimizerContext {
   //
   // Tensor mapping that has to be applied to the graph after all functions
   // optimizations (invalidated tensor id -> optimized graph tensor id).
-  gtl::FlatMap<SafeTensorId, SafeTensorId, SafeTensorId::Hasher>
+  absl::flat_hash_map<SafeTensorId, SafeTensorId, SafeTensorId::Hasher>
       tensor_mapping_;
 
   // When we inline a function into the optimized graph, we no longer have the
@@ -448,7 +448,7 @@ class FunctionOptimizerContext {
   // to all side-effectful ops inside the function body.
   //
   // Invalidated function call node name -> Inlined side-effectful nodes
-  gtl::FlatMap<string, std::vector<string>> control_overrides_;
+  absl::flat_hash_map<string, std::vector<string>> control_overrides_;
 
   // Use graph view to find active outputs of the function caller nodes.
   GraphView graph_view_;
@@ -472,10 +472,10 @@ const FunctionDef* FindFunctionCall(const FunctionOptimizerContext& ctx,
   return ctx.function_library().Find(node.op());
 }
 
-gtl::FlatSet<int> GetActiveOutputs(const NodeDef& node,
-                                   const FunctionOptimizerContext& ctx,
-                                   int size_hint = 0) {
-  gtl::FlatSet<int> active_outputs;
+absl::flat_hash_set<int> GetActiveOutputs(const NodeDef& node,
+                                          const FunctionOptimizerContext& ctx,
+                                          int size_hint = 0) {
+  absl::flat_hash_set<int> active_outputs;
   active_outputs.reserve(static_cast<size_t>(size_hint));
 
   // 1. Output can be consumed by the other graph node.
@@ -508,7 +508,7 @@ bool HasUnusedOutputs(const NodeDef& func_node, const FunctionDef& func,
   // number of output args is the same as number of possible function caller
   // node outputs.
   int num_outputs = func.signature().output_arg_size();
-  const gtl::FlatSet<int> active_outputs =
+  const absl::flat_hash_set<int> active_outputs =
       GetActiveOutputs(func_node, ctx, /*size_hind*/ num_outputs);
 
   return active_outputs.size() != num_outputs;
@@ -519,7 +519,7 @@ bool HasUnusedOutputs(const NodeDef& func_node, const FunctionDef& func,
 FunctionDefLibrary PruneFunctionLibrary(const FunctionLibraryDefinition& flib,
                                         const GraphDef& optimized_graph) {
   FunctionLibraryDefinition pruned_flib =
-      ReachableFunctionLibraryDefinition(flib, optimized_graph);
+      flib.ReachableDefinitions(optimized_graph);
 
   int pruned_functions = static_cast<int>(pruned_flib.num_functions()) -
                          static_cast<int>(flib.num_functions());
@@ -534,8 +534,8 @@ FunctionDefLibrary PruneFunctionLibrary(const FunctionLibraryDefinition& flib,
 Status PushDownConstInputs(const NodeDef& func_node,
                            const FunctionOptimizerContext& ctx,
                            GrapplerFunctionItem* item,
-                           gtl::FlatSet<string>* const_inputs,
-                           gtl::FlatSet<string>* control_deps) {
+                           absl::flat_hash_set<string>* const_inputs,
+                           absl::flat_hash_set<string>* control_deps) {
   // Record node control dependencies in the control_deps set.
   const auto record_control_deps = [&](const NodeDef* const_input) {
     for (int i = const_input->input_size() - 1; i >= 0; --i) {
@@ -585,7 +585,7 @@ void RemovePushedDownConstInputs(const FunctionSpecialization& specialization,
 
   // Attach control dependencies of pushed down const input to the caller node.
   if (!specialization.control_deps.empty()) {
-    gtl::FlatSet<string> existing_control_deps;
+    absl::flat_hash_set<string> existing_control_deps;
 
     for (const string& input : keep_inputs) {
       existing_control_deps.insert(AsControlDependency(NodeName(input)));
@@ -797,8 +797,8 @@ Status SpecializeFunction(const NodeDef& func_node, const FunctionDef& func,
 
   // Push const inputs into the function body, and keep track of their control
   // dependencies.
-  gtl::FlatSet<string> const_inputs;
-  gtl::FlatSet<string> control_deps;
+  absl::flat_hash_set<string> const_inputs;
+  absl::flat_hash_set<string> control_deps;
   TF_RETURN_IF_ERROR(PushDownConstInputs(func_node, *ctx, &item, &const_inputs,
                                          &control_deps));
 
@@ -1005,7 +1005,7 @@ Status InlineDirectFunctionCall(const NodeDef& func_node,
 
   // Mapping from input placeholder name to function input position.
   int idx = 0;
-  std::unordered_map<string, int> input_placeholders_idx;
+  absl::flat_hash_map<string, int> input_placeholders_idx;
   for (const InputArgExpansion& input_arg : item.inputs()) {
     for (const string& placeholder : input_arg.placeholders) {
       input_placeholders_idx[placeholder] = idx++;
@@ -1699,7 +1699,7 @@ Status FunctionOptimizer::Optimize(Cluster*, const GrapplerItem& item,
   if (!ctx.control_overrides().empty()) {
     for (NodeDef& node : *optimized_graph->mutable_node()) {
       // Keep track of new control inputs to the node.
-      gtl::FlatSet<string> add_ctrl_inputs;
+      absl::flat_hash_set<string> add_ctrl_inputs;
 
       // Remove all invalidated control inputs.
       for (int idx = 0; idx < node.input_size(); /* see below */) {
diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer.cc b/tensorflow/core/grappler/optimizers/meta_optimizer.cc
index 67699b093d..a84bb1d62f 100644
--- a/tensorflow/core/grappler/optimizers/meta_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/meta_optimizer.cc
@@ -427,6 +427,14 @@ Status MetaOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
   VLOG(1) << "Starting optimization for grappler item: " << item.id;
   optimization_results_.clear();
 
+  // Constructs a FunctionLibraryDefinition with functions that are reachable
+  // from the nodes of the graph.
+  const auto minimized_flib =
+      [](const GraphDef& graph) -> FunctionLibraryDefinition {
+    return FunctionLibraryDefinition(OpRegistry::Global(), graph.library())
+        .ReachableDefinitions(graph);
+  };
+
   // 0. Original graph might contain a huge function library, that is mostly
   // unused. This library copied over by each individual Grappler optimizer,
   // which adds a huge overhead. Before starting optimization passes we just
@@ -436,11 +444,7 @@ Status MetaOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
   GraphDef trimmed_graph;  // do not copy graph with a potentially huge library
   *trimmed_graph.mutable_node() = item.graph.node();
   *trimmed_graph.mutable_versions() = item.graph.versions();
-  *trimmed_graph.mutable_library() =
-      grappler::ReachableFunctionLibraryDefinition(
-          FunctionLibraryDefinition(OpRegistry::Global(), item.graph.library()),
-          item.graph)
-          .ToProto();
+  *trimmed_graph.mutable_library() = minimized_flib(item.graph).ToProto();
 
   GrapplerItem trimmed_item = item.WithGraph(std::move(trimmed_graph));
 
@@ -472,10 +476,7 @@ Status MetaOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
   }
 
   // 2. Optimize functions reachable from the optimized graph.
-  FunctionLibraryDefinition flib = ReachableFunctionLibraryDefinition(
-      FunctionLibraryDefinition(OpRegistry::Global(),
-                                optimized_graph->library()),
-      *optimized_graph);
+  FunctionLibraryDefinition flib = minimized_flib(*optimized_graph);
 
   // Find functions for which we might need to compute a gradient at runtime.
   absl::flat_hash_set<string> differentiable_functions;
diff --git a/tensorflow/core/grappler/utils/BUILD b/tensorflow/core/grappler/utils/BUILD
index cd69cf895c..89417f85c2 100644
--- a/tensorflow/core/grappler/utils/BUILD
+++ b/tensorflow/core/grappler/utils/BUILD
@@ -178,6 +178,9 @@ cc_library(
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
+        "@com_google_absl//absl/algorithm:container",
+        "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/strings",
     ],
 )
@@ -196,6 +199,7 @@ tf_cc_test(
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
+        "@com_google_absl//absl/container:flat_hash_map",
     ],
 )
 
diff --git a/tensorflow/core/grappler/utils/functions.cc b/tensorflow/core/grappler/utils/functions.cc
index f2894a942b..7c2180ae40 100644
--- a/tensorflow/core/grappler/utils/functions.cc
+++ b/tensorflow/core/grappler/utils/functions.cc
@@ -14,8 +14,9 @@ limitations under the License.
 ==============================================================================*/
 #include "tensorflow/core/grappler/utils/functions.h"
 
-#include <unordered_map>
-
+#include "absl/container/flat_hash_map.h"
+#include "absl/container/flat_hash_set.h"
+#include "absl/strings/str_cat.h"
 #include "absl/strings/substitute.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/function.h"
@@ -28,7 +29,6 @@ limitations under the License.
 #include "tensorflow/core/framework/versions.pb.h"
 #include "tensorflow/core/grappler/op_types.h"
 #include "tensorflow/core/grappler/utils.h"
-#include "tensorflow/core/lib/gtl/map_util.h"
 #include "tensorflow/core/lib/strings/scanner.h"
 
 namespace tensorflow {
@@ -76,16 +76,6 @@ Status ResolveFunctionBodyNodeAttrPlaceholders(
 
 }  // namespace
 
-FunctionLibraryDefinition ReachableFunctionLibraryDefinition(
-    const FunctionLibraryDefinition& flib, const GraphDef& graph) {
-  return flib.ReachableDefinitions(graph);
-}
-
-FunctionLibraryDefinition ReachableFunctionLibraryDefinition(
-    const FunctionLibraryDefinition& flib, const FunctionDef& func) {
-  return flib.ReachableDefinitions(func);
-}
-
 void GrapplerFunctionConnectivity::RegisterInputArgExpansion(
     InputArgExpansion input_arg_expansion) {
   string input_name = input_arg_expansion.input_name;
@@ -94,7 +84,7 @@ void GrapplerFunctionConnectivity::RegisterInputArgExpansion(
   for (int i = 0; i < placeholders.size(); ++i) {
     const string& placeholder = input_arg_expansion.placeholders[i];
     input_arg_placeholders_.insert(
-        {placeholder, InputArgPlaceholder{input_name, /*input_position=*/i}});
+        {placeholder, InputArgPlaceholder{input_name, /*input_index=*/i}});
   }
   input_arg_expansions_.insert(
       {std::move(input_name), std::move(input_arg_expansion)});
@@ -193,7 +183,7 @@ Status GrapplerFunctionConnectivity::ExpandFunctionDefInput(
           // If position is not defined expand node output range
           for (int i = output_range.first; i < output_range.second; ++i) {
             graph_def_inputs->push_back(
-                i == 0 ? node_name : strings::StrCat(node_name, ":", i));
+                i == 0 ? node_name : absl::StrCat(node_name, ":", i));
           }
         } else {
           if (position > (output_range.second - output_range.first)) {
@@ -203,7 +193,7 @@ Status GrapplerFunctionConnectivity::ExpandFunctionDefInput(
           }
           int pos = output_range.first + position;
           graph_def_inputs->push_back(
-              pos == 0 ? node_name : strings::StrCat(node_name, ":", pos));
+              pos == 0 ? node_name : absl::StrCat(node_name, ":", pos));
         }
 
         return Status::OK();
@@ -232,39 +222,39 @@ Status GrapplerFunctionConnectivity::ExpandNodeInputs(
 
 Status GrapplerFunctionConnectivity::AsFunctionDefInput(
     const string& graph_def_input, string* func_def_input) const {
-  using gtl::FindOrNull;
-
   if (IsControlInput(graph_def_input)) {
     *func_def_input = graph_def_input;
     return Status::OK();
   }
 
-  int position;
-  string node_name = ParseNodeName(graph_def_input, &position);
-  CHECK_GE(position, 0);
+  const TensorId tensor = ParseTensorName(graph_def_input);
+  DCHECK_GE(tensor.index(), 0);
+
+  const absl::string_view node_name = tensor.node();
+  const int index = tensor.index();
 
   // Check if it's an input arg placeholder
-  if (position == 0) {
-    const InputArgPlaceholder* placeholder =
-        FindOrNull(input_arg_placeholders_, node_name);
-    if (placeholder != nullptr) {
-      *func_def_input = strings::StrCat(placeholder->input_name, ":",
-                                        placeholder->input_position);
+  if (tensor.index() == 0) {
+    const auto is_input_placeholder = input_arg_placeholders_.find(node_name);
+    if (is_input_placeholder != input_arg_placeholders_.end()) {
+      const InputArgPlaceholder& placeholder = is_input_placeholder->second;
+      *func_def_input =
+          absl::StrCat(placeholder.input_name, ":", placeholder.input_index);
       return Status::OK();
     }
   }
 
   // It must be output from one of the function body nodes
-  const tensorflow::NameRangeMap* outputs_range_map =
-      FindOrNull(function_body_outputs_, node_name);
-  if (outputs_range_map != nullptr) {
-    for (const auto& el : *outputs_range_map) {
+  const auto is_body_output = function_body_outputs_.find(tensor.node());
+  if (is_body_output != function_body_outputs_.end()) {
+    const tensorflow::NameRangeMap& outputs_range_map = is_body_output->second;
+
+    for (const auto& el : outputs_range_map) {
       const auto& output_name = el.first;
       const auto& output_range = el.second;
-      if (position >= output_range.first && position < output_range.second) {
-        int pos = position - output_range.first;
-        *func_def_input =
-            strings::StrCat(node_name, ":", output_name, ":", pos);
+      if (index >= output_range.first && index < output_range.second) {
+        int pos = index - output_range.first;
+        *func_def_input = absl::StrCat(node_name, ":", output_name, ":", pos);
         return Status::OK();
       }
     }
@@ -426,7 +416,7 @@ bool IsParametrized(const FunctionDef& func) {
 
 Status InstantiationTypeParameters(
     const FunctionDef& func, const AttrSlice& func_instantiation_attr,
-    std::unordered_map<string, DataType>* type_parameters) {
+    absl::flat_hash_map<string, DataType>* type_parameters) {
   if (!type_parameters->empty()) {
     return errors::InvalidArgument("Type parameters output map must be empty");
   }
@@ -454,7 +444,7 @@ Status InstantiationTypeParameters(
 
 Status InstantiationBodyParameters(
     const FunctionDef& func, const AttrSlice& func_instantiation_attr,
-    std::unordered_map<string, AttrValue>* body_parameters) {
+    absl::flat_hash_map<string, AttrValue>* body_parameters) {
   if (!body_parameters->empty()) {
     return errors::InvalidArgument("Body parameters output map must be empty");
   }
@@ -514,8 +504,7 @@ Status MakeGrapplerFunctionItem(const FunctionDef& func,
 
   // Function body shares the library with the graph that instantiated it. We do
   // not need a full copy of the function library, just the reachable subset.
-  *function_body.mutable_library() =
-      ReachableFunctionLibraryDefinition(flib, func).ToProto();
+  *function_body.mutable_library() = flib.ReachableDefinitions(func).ToProto();
 
   VLOG(3) << absl::Substitute(
       "Deleted $0 unreachable functions from the Grappler function item "
@@ -645,7 +634,7 @@ Status RegisterGrapplerFunctionConnectivity(
   return Status::OK();
 }
 
-Status ReplaceInputWithConst(const NodeDef& input_const, int input_position,
+Status ReplaceInputWithConst(const NodeDef& input_const, int input_index,
                              GrapplerFunctionItem* item) {
   if (!IsConstant(input_const)) {
     return errors::InvalidArgument("Input node ", input_const.name(),
@@ -657,7 +646,7 @@ Status ReplaceInputWithConst(const NodeDef& input_const, int input_position,
   // Find input arg expansion and input placeholder position in it for the
   // given function input position.
   InputArgExpansion* input_arg_expansion = nullptr;
-  int placeholder_idx = input_position;
+  int placeholder_idx = input_index;
 
   for (InputArgExpansion& input : inputs) {
     if (placeholder_idx < input.placeholders.size()) {
@@ -668,9 +657,8 @@ Status ReplaceInputWithConst(const NodeDef& input_const, int input_position,
   }
 
   if (input_arg_expansion == nullptr) {
-    return errors::InvalidArgument(
-        "Input placeholder not found: input_position=", input_position,
-        " function=", item->id);
+    return errors::InvalidArgument("Input placeholder not found: input_index=",
+                                   input_index, " function=", item->id);
   }
 
   // Delete placeholder from input expansion.
@@ -699,7 +687,7 @@ Status ReplaceInputWithConst(const NodeDef& input_const, int input_position,
   return Status::OK();
 }
 
-Status RemoveUnusedOutputs(const gtl::FlatSet<int>& active_outputs,
+Status RemoveUnusedOutputs(const absl::flat_hash_set<int>& active_outputs,
                            GrapplerFunctionItem* item,
                            std::vector<std::pair<int, int>>* output_mapping) {
   DCHECK(output_mapping->empty());
@@ -713,7 +701,7 @@ Status RemoveUnusedOutputs(const gtl::FlatSet<int>& active_outputs,
     }
   }
 
-  gtl::FlatSet<const OutputArgExpansion*> unused_output_args;
+  absl::flat_hash_set<const OutputArgExpansion*> unused_output_args;
 
   const auto is_unused_output_arg = [&](const OutputArgExpansion& output) {
     return unused_output_args.find(&output) != unused_output_args.end();
diff --git a/tensorflow/core/grappler/utils/functions.h b/tensorflow/core/grappler/utils/functions.h
index 038cf5f527..ce8a3e5ac7 100644
--- a/tensorflow/core/grappler/utils/functions.h
+++ b/tensorflow/core/grappler/utils/functions.h
@@ -18,7 +18,9 @@ limitations under the License.
 
 #include <memory>
 #include <string>
-#include <unordered_map>
+
+#include "absl/container/flat_hash_map.h"
+#include "absl/container/flat_hash_set.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/function.h"
 #include "tensorflow/core/framework/function.pb.h"
@@ -30,13 +32,6 @@ limitations under the License.
 namespace tensorflow {
 namespace grappler {
 
-// Returns a copy of FunctionLibraryDefinition with subset of functions that are
-// reachable from the nodes of the graph.
-FunctionLibraryDefinition ReachableFunctionLibraryDefinition(
-    const FunctionLibraryDefinition& flib, const GraphDef& graph);
-FunctionLibraryDefinition ReachableFunctionLibraryDefinition(
-    const FunctionLibraryDefinition& flib, const FunctionDef& func);
-
 // Depending on the function instantiation attributes, input argument to the
 // function might be a single tensor, list of tensors of the same type, or a
 // list of tensors of different types.
@@ -81,12 +76,12 @@ class GrapplerFunctionConnectivity {
   void RegisterFunctionBodyOutputs(const string& node_name,
                                    tensorflow::NameRangeMap&& outputs);
 
-  // Expand input encoded in FunctionDef format (name[:output][:position]) into
+  // Expands input encoded in FunctionDef format (name[:output][:position]) into
   // multiple inputs in GraphDef format (name[:position]).
   Status ExpandFunctionDefInput(const string& func_def_input,
                                 std::vector<string>* graph_def_inputs) const;
 
-  // Update Node inputs from FunctionDef to GraphDef format.
+  // Updates Node inputs from FunctionDef to GraphDef format.
   Status ExpandNodeInputs(NodeDef* function_body_node) const;
 
   // When expanding inputs in function def format, single input might be
@@ -96,29 +91,31 @@ class GrapplerFunctionConnectivity {
   // instantiation attributes and length of input args (and node def outputs) is
   // known.
 
-  // Map from GraphDef input format to FunctionDef input format using registered
-  // input arg expansion and function body outputs.
+  // Converts input name from GraphDef format (name[:position]) to the
+  // FunctionDef input format (name[:output][:position]) using registered input
+  // arg expansion and function body outputs.
   Status AsFunctionDefInput(const string& graph_def_input,
                             string* func_def_input) const;
 
-  // Update Node inputs from GraphDef to FunctionDef format.
+  // Updates Node inputs from GraphDef to FunctionDef format.
   Status AsFunctionDefNode(NodeDef* function_body_node) const;
 
  private:
   // Mapping from input name to input arg expansion.
-  std::unordered_map<string, InputArgExpansion> input_arg_expansions_;
+  absl::flat_hash_map<string, InputArgExpansion> input_arg_expansions_;
   // Mapping from function body node name to output names range map.
-  std::unordered_map<string, tensorflow::NameRangeMap> function_body_outputs_;
+  absl::flat_hash_map<string, tensorflow::NameRangeMap> function_body_outputs_;
 
+  // For each placeholder added to the function instantiation graph, we keep a
+  // mapping back to the function input argument name and index.
   struct InputArgPlaceholder {
-    string input_name;   // Name of the function input argument.
-    int input_position;  // Index of a tensor in the function input argument
-                         // expansion, it can be greater than `0` if input
-                         // argument is a list of tensors (aka list(type)).
+    string input_name;  // Name of the function input argument.
+    int input_index;    // Index of a tensor in the function input argument
+                        // expansion, it can be greater than `0` if input
+                        // argument is a list of tensors (aka list(type)).
   };
-
   // Mapping from input arg placeholder to the function input tensor.
-  std::unordered_map<string, InputArgPlaceholder> input_arg_placeholders_;
+  absl::flat_hash_map<string, InputArgPlaceholder> input_arg_placeholders_;
 };
 
 // Get Function type attributes using attributes of a node that instantiated
@@ -172,7 +169,8 @@ class GrapplerFunctionItem : public GrapplerItem {
   friend Status ReplaceInputWithConst(const NodeDef&, int,
                                       GrapplerFunctionItem*);
   friend Status RemoveUnusedOutputs(
-      const gtl::FlatSet<int>& active_outputs, GrapplerFunctionItem* item,
+      const absl::flat_hash_set<int>& active_outputs,
+      GrapplerFunctionItem* item,
       std::vector<std::pair<int, int>>* output_mapping);
 
   GrapplerFunctionItem(string func_name, string description,
@@ -191,7 +189,7 @@ class GrapplerFunctionItem : public GrapplerItem {
 
   std::set<string> input_arg_placeholders_;
 
-  bool is_stateful_;
+  bool is_stateful_ = false;
 };
 
 // Check if function input/output types are fully defined only at instantiation
@@ -210,14 +208,14 @@ bool IsParametrized(const FunctionDef& func);
 // caller node. Return error if type can't be resolved.
 Status InstantiationTypeParameters(
     const FunctionDef& func, const AttrSlice& func_instantiation_attr,
-    std::unordered_map<string, DataType>* type_parameters);
+    absl::flat_hash_map<string, DataType>* type_parameters);
 
 // Resolve function instantiation body parameters (values for the function body
 // attr placeholders) from the attributes of the caller node. Return error if
 // type can't be resolved.
 Status InstantiationBodyParameters(
     const FunctionDef& func, const AttrSlice& func_instantiation_attr,
-    std::unordered_map<string, AttrValue>* body_parameters);
+    absl::flat_hash_map<string, AttrValue>* body_parameters);
 
 // Register GrapplerFunctionItem input arg expansion and function body outputs
 // in the GrapplerFunctionConnectivity. Use function library definition to
@@ -227,7 +225,7 @@ Status RegisterGrapplerFunctionConnectivity(
     GrapplerFunctionConnectivity* connectivity);
 
 // Replace one of the function inputs with a constant.
-Status ReplaceInputWithConst(const NodeDef& input_const, int input_position,
+Status ReplaceInputWithConst(const NodeDef& input_const, int input_index,
                              GrapplerFunctionItem* item);
 
 // Remove function output arguments that do not have any active outputs (output
@@ -236,7 +234,7 @@ Status ReplaceInputWithConst(const NodeDef& input_const, int input_position,
 // potentially be connected to the same output argument (in case of tensor list
 // outputs). Add output mapping for all active outputs that changed it's output
 // position (std::pair<old position, new position>).
-Status RemoveUnusedOutputs(const gtl::FlatSet<int>& active_outputs,
+Status RemoveUnusedOutputs(const absl::flat_hash_set<int>& active_outputs,
                            GrapplerFunctionItem* item,
                            std::vector<std::pair<int, int>>* output_mapping);
 
diff --git a/tensorflow/core/grappler/utils/functions_test.cc b/tensorflow/core/grappler/utils/functions_test.cc
index 5923850eca..29d6100d23 100644
--- a/tensorflow/core/grappler/utils/functions_test.cc
+++ b/tensorflow/core/grappler/utils/functions_test.cc
@@ -14,6 +14,8 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/core/grappler/utils/functions.h"
+
+#include "absl/container/flat_hash_map.h"
 #include "tensorflow/cc/ops/standard_ops.h"
 #include "tensorflow/core/framework/function_testlib.h"
 #include "tensorflow/core/framework/node_def.pb.h"
@@ -77,7 +79,7 @@ TEST_F(FunctionsTest, InstantiationParameters) {
   func_instantiation_attr["B"].set_type(DT_INT32);
   func_instantiation_attr["C"].set_type(DT_DOUBLE);
 
-  std::unordered_map<string, DataType> type_parameters;
+  absl::flat_hash_map<string, DataType> type_parameters;
   TF_EXPECT_OK(InstantiationTypeParameters(
       func, AttrSlice(&func_instantiation_attr), &type_parameters));
 
@@ -86,7 +88,7 @@ TEST_F(FunctionsTest, InstantiationParameters) {
   EXPECT_EQ(DT_INT32, type_parameters["B"]);
   EXPECT_EQ(DT_DOUBLE, type_parameters["C"]);
 
-  std::unordered_map<string, AttrValue> body_parameters;
+  absl::flat_hash_map<string, AttrValue> body_parameters;
   TF_EXPECT_OK(InstantiationBodyParameters(
       func, AttrSlice(&func_instantiation_attr), &body_parameters));
 
-- 
GitLab


From 26d8486baa67d2c545aa4384f9d666dc7b9ad530 Mon Sep 17 00:00:00 2001
From: Rachel Lim <rachelim@google.com>
Date: Wed, 2 Jan 2019 15:11:44 -0800
Subject: [PATCH 0147/1095] documentation fix

PiperOrigin-RevId: 227589407
---
 tensorflow/python/data/experimental/ops/optimization_options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/data/experimental/ops/optimization_options.py b/tensorflow/python/data/experimental/ops/optimization_options.py
index 41a819d94b..eb95484ac5 100644
--- a/tensorflow/python/data/experimental/ops/optimization_options.py
+++ b/tensorflow/python/data/experimental/ops/optimization_options.py
@@ -33,7 +33,7 @@ class OptimizationOptions(options.OptionsBase):
   ```python
   options = tf.data.Options()
   options.experimental_optimization.map_vectorization = True
-  options.apply_default_optimizations = False
+  options.experimental_optimization.apply_default_optimizations = False
   dataset = dataset.with_options(options)
   ```
   """
-- 
GitLab


From 29bc2e98eaf1dd730574886ffe1b317e982323d4 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Wed, 2 Jan 2019 15:47:20 -0800
Subject: [PATCH 0148/1095] Validate that all variables under the model were
 created in the distribution strategy scope.

PiperOrigin-RevId: 227595107
---
 .../contrib/distribute/python/keras_test.py   | 44 +++++++++++++++++++
 tensorflow/python/keras/engine/training.py    | 14 ++++++
 2 files changed, 58 insertions(+)

diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index b99d11b923..ba5a5e6400 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -245,6 +245,18 @@ def all_strategy_combinations():
   return strategy_minus_tpu_combinations() + tpu_strategy_combinations()
 
 
+def all_strategy_combinations_minus_default():
+  strategy_minus_default_combinations = combinations.combine(
+      distribution=[
+          combinations.one_device_strategy,
+          combinations.mirrored_strategy_with_gpu_and_cpu,
+          combinations.mirrored_strategy_with_two_gpus,
+          combinations.core_mirrored_strategy_with_gpu_and_cpu,
+          combinations.core_mirrored_strategy_with_two_gpus],
+      mode=['graph', 'eager'])
+  return strategy_minus_default_combinations + tpu_strategy_combinations()
+
+
 # TODO(priyag): Add v2 optimizers here.
 def strategy_and_optimizer_combinations():
   return combinations.times(
@@ -1149,5 +1161,37 @@ class TestDistributionStrategyWithNormalizationLayer(
       np.testing.assert_allclose(out.std(), 1.0, atol=1e-1)
 
 
+class TestDistributionStrategyVariableValidation(test.TestCase,
+                                                 parameterized.TestCase):
+
+  @combinations.generate(all_strategy_combinations_minus_default())
+  def test_layer_outside_scope(self, distribution):
+    with self.cached_session():
+      with self.assertRaisesRegexp(
+          ValueError, 'was not created in the distribution strategy'):
+        x = keras.layers.Input(shape=(3,), name='input')
+        y = keras.layers.Dense(4, name='dense')(x)
+        with distribution.scope():
+          model = keras.Model(x, y)
+          optimizer = gradient_descent.GradientDescentOptimizer(0.001)
+          loss = 'mse'
+          metrics = ['mae', keras.metrics.CategoricalAccuracy()]
+          model.compile(optimizer, loss, metrics=metrics)
+
+  @combinations.generate(all_strategy_combinations_minus_default())
+  def test_model_outside_scope(self, distribution):
+    with self.cached_session():
+      with self.assertRaisesRegexp(
+          ValueError, 'was not created in the distribution strategy'):
+        x = keras.layers.Input(shape=(3,), name='input')
+        y = keras.layers.Dense(4, name='dense')(x)
+        model = keras.Model(x, y)
+        with distribution.scope():
+          optimizer = gradient_descent.GradientDescentOptimizer(0.001)
+          loss = 'mse'
+          metrics = ['mae', keras.metrics.CategoricalAccuracy()]
+          model.compile(optimizer, loss, metrics=metrics)
+
+
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index be3afafd3e..1dc3b81cfc 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -548,6 +548,20 @@ class Model(Network):
       trainable_weights = self.trainable_weights
       self._collected_trainable_weights = trainable_weights
 
+      # Validate all variables were correctly created in distribution scope.
+      if self._distribution_strategy and not self._compile_distribution:
+        for v in self.variables:
+          if v.distribute_strategy is not self._distribution_strategy:
+            raise ValueError(
+                'Variable (%s) was not created in the distribution strategy '
+                'scope of (%s). It is most likely due to not all layers or '
+                'the model or optimizer being created outside the distribution '
+                'strategy scope. Try to make sure your code looks similar '
+                'to the following.\n'
+                'with strategy.scope():\n'
+                '  model=_create_model()\n'
+                '  model.compile(...)'% (v, self._distribution_strategy))
+
   @property
   def metrics(self):
     """Returns the model's metrics added using `compile`, `add_metric` APIs."""
-- 
GitLab


From 6af0243ba6842330d7bcce6499287f6e9a640c9f Mon Sep 17 00:00:00 2001
From: Penporn Koanantakool <penporn@google.com>
Date: Wed, 2 Jan 2019 15:52:17 -0800
Subject: [PATCH 0149/1095] Fix a typo in Near. Before this change, when the
 values are not 'Near', the program reports the address of b instead of the
 value of b[i].

PiperOrigin-RevId: 227595818
---
 tensorflow/core/framework/tensor_testutil.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/framework/tensor_testutil.h b/tensorflow/core/framework/tensor_testutil.h
index 3163002851..b58292b3b0 100644
--- a/tensorflow/core/framework/tensor_testutil.h
+++ b/tensorflow/core/framework/tensor_testutil.h
@@ -206,7 +206,7 @@ struct Expector<T, true> {
     const T* b = y.flat<T>().data();
     for (int i = 0; i < size; ++i) {
       EXPECT_TRUE(Near(a[i], b[i], abs_err))
-          << "a = " << a[i] << " b = " << b << " index = " << i;
+          << "a = " << a[i] << " b = " << b[i] << " index = " << i;
     }
   }
 };
-- 
GitLab


From f0dde544f263013867e8c66c82db85e66c42d530 Mon Sep 17 00:00:00 2001
From: Michael Kuperstein <mkuper@google.com>
Date: Wed, 2 Jan 2019 16:00:27 -0800
Subject: [PATCH 0150/1095] [XLA] Don't go out of bounds in
 Shape::dynamic_dimensions_.

PiperOrigin-RevId: 227596997
---
 tensorflow/compiler/xla/shape.cc | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/shape.cc b/tensorflow/compiler/xla/shape.cc
index c04a916154..f05d816268 100644
--- a/tensorflow/compiler/xla/shape.cc
+++ b/tensorflow/compiler/xla/shape.cc
@@ -27,7 +27,19 @@ Shape::Shape(const ShapeProto& shape_proto) {
   for (const int64 dimension : shape_proto.dimensions()) {
     add_dimensions(dimension);
   }
-  for (int i = 0; i < shape_proto.is_dynamic_dimension_size(); i++) {
+  // A malformed proto may have different is_dynamic_dimension_size and
+  // dimensions_size. Since C++ is evil, and we have no good way of bailing out
+  // in a constructor, conservatively trim the is_dynamic_dimension size.
+  // TODO(b/120111794): Make this a hard error when we have a factory method
+  // instead of a constructor.
+  if (shape_proto.dimensions_size() !=
+      shape_proto.is_dynamic_dimension_size()) {
+    LOG(ERROR) << "Malformed shape proto: number of is_dynamic_dimension "
+                  "fields does not match number of dimension fields";
+  }
+  int64 num_dynamic_dimension_fields = std::min(
+      shape_proto.dimensions_size(), shape_proto.is_dynamic_dimension_size());
+  for (int i = 0; i < num_dynamic_dimension_fields; i++) {
     dynamic_dimensions_[i] = shape_proto.is_dynamic_dimension(i);
   }
   tuple_shapes_.reserve(shape_proto.tuple_shapes_size());
-- 
GitLab


From 0b6177c2fa509d951d4086eab88fc506792cfe43 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 2 Jan 2019 16:08:42 -0800
Subject: [PATCH 0151/1095] Fix make_initializable_iterator comment

PiperOrigin-RevId: 227598505
---
 tensorflow/python/data/ops/dataset_ops.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py
index 1ba00a8e69..61210f3a7f 100644
--- a/tensorflow/python/data/ops/dataset_ops.py
+++ b/tensorflow/python/data/ops/dataset_ops.py
@@ -1756,8 +1756,8 @@ def make_initializable_iterator(dataset):
     RuntimeError: If eager execution is enabled.
   """
   try:
-    # Call the defined `make_one_shot_iterator()` if there is one, because some
-    # datasets (e.g. for prefetching) override its behavior.
+    # Call the defined `make_initializable_iterator()` if there is one, because
+    # some datasets (e.g. for prefetching) override its behavior.
     return dataset.make_initializable_iterator()
   except AttributeError:
     return DatasetV1Adapter(dataset).make_initializable_iterator()
-- 
GitLab


From dd3adf935a800e25132b187d65aa62e97504f50c Mon Sep 17 00:00:00 2001
From: Kay Zhu <kayzhu@google.com>
Date: Wed, 2 Jan 2019 16:11:52 -0800
Subject: [PATCH 0152/1095] [XLA] Enable interpreter backend test by default
 for xla/tests without having to using additional backend tags
 "enable_for_xla_interpreter".

PiperOrigin-RevId: 227598976
---
 tensorflow/compiler/xla/client/lib/BUILD      |   6 -
 .../compiler/xla/service/hlo_evaluator.cc     |   5 +
 .../compiler/xla/service/hlo_evaluator.h      |  23 +++
 .../xla/service/hlo_evaluator_typed_visitor.h |  32 ++--
 .../compiler/xla/service/interpreter/BUILD    |   3 +
 .../xla/service/interpreter/compiler.cc       |   6 +
 tensorflow/compiler/xla/tests/BUILD           | 144 ++++--------------
 .../xla/tests/array_elementwise_ops_test.cc   |  13 ++
 .../compiler/xla/tests/bfloat16_test.cc       |   8 +-
 tensorflow/compiler/xla/tests/client_test.cc  |   5 +-
 .../xla/tests/gather_operation_test.cc        |   4 +-
 .../xla/tests/local_client_execute_test.cc    |   6 +-
 12 files changed, 115 insertions(+), 140 deletions(-)

diff --git a/tensorflow/compiler/xla/client/lib/BUILD b/tensorflow/compiler/xla/client/lib/BUILD
index 966a581981..d863ebfea1 100644
--- a/tensorflow/compiler/xla/client/lib/BUILD
+++ b/tensorflow/compiler/xla/client/lib/BUILD
@@ -93,7 +93,6 @@ cc_library(
 xla_test(
     name = "constants_test",
     srcs = ["constants_test.cc"],
-    tags = ["enable_for_xla_interpreter"],
     deps = [
         ":constants",
         "//tensorflow/compiler/xla:test",
@@ -147,7 +146,6 @@ cc_library(
 xla_test(
     name = "math_test",
     srcs = ["math_test.cc"],
-    tags = ["enable_for_xla_interpreter"],
     deps = [
         ":math",
         "//tensorflow/compiler/xla:literal_util",
@@ -181,7 +179,6 @@ cc_library(
 xla_test(
     name = "matrix_test",
     srcs = ["matrix_test.cc"],
-    tags = ["enable_for_xla_interpreter"],
     deps = [
         ":matrix",
         ":slicing",
@@ -295,7 +292,6 @@ cc_library(
 xla_test(
     name = "slicing_test",
     srcs = ["slicing_test.cc"],
-    tags = ["enable_for_xla_interpreter"],
     deps = [
         ":slicing",
         "//tensorflow/compiler/xla:literal_util",
@@ -324,7 +320,6 @@ cc_library(
 xla_test(
     name = "sorting_test",
     srcs = ["sorting_test.cc"],
-    tags = ["enable_for_xla_interpreter"],
     deps = [
         ":sorting",
         "//tensorflow/compiler/xla:test",
@@ -354,7 +349,6 @@ xla_test(
     srcs = ["quantize_test.cc"],
     # TODO(b/122119490): re-enable TAP after fixing.
     tags = [
-        "enable_for_xla_interpreter",
         "notap",
     ],
     deps = [
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc
index 794b97a7e0..4f5484293c 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc
@@ -138,6 +138,11 @@ StatusOr<Literal> Compare<complex64>(const Shape& shape, HloOpcode opcode,
 
 }  // namespace
 
+// Note that unsupported types by the typed visitor does not necessarily imply
+// the non-typed HloEvaluator (parent evaluator) would not support them either
+// in the type-agnostic handler. For e.g., HandleGetTupleElement in the parent
+// type-agnostic evaluator will be able to accept Tuple primitive type, whereas
+// HloEvaluatorTypedVisitor cannot.
 HloEvaluator::HloEvaluator(int64 max_loop_iterations)
     : max_loop_iterations_(max_loop_iterations) {
   typed_visitors_[PRED] =
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.h b/tensorflow/compiler/xla/service/hlo_evaluator.h
index 44d5e5c030..d8e3019576 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.h
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.h
@@ -211,6 +211,29 @@ class HloEvaluator : public DfsHloVisitorWithDefault {
 
   Status HandleReduce(HloInstruction* reduce) override;
 
+  // Unsupported HLOs, note some of them (such as BatchNorm*) are typically
+  // expanded in a semantic-preserving way into other HLOs by adding exanpsion
+  // HLO pass to the HLO optimization pass during compilation, which can then be
+  // handled by the evaluator.
+  Status HandleBatchNormGrad(HloInstruction* batch_norm_grad) override {
+    return Unimplemented("BatchNormGrad HLO is unsupported by the evaluator.");
+  };
+  Status HandleBatchNormInference(
+      HloInstruction* batch_norm_inference) override {
+    return Unimplemented(
+        "BatchNormInference HLO is unsupported by the evaluator.");
+  };
+  Status HandleBatchNormTraining(HloInstruction* batch_norm_training) override {
+    return Unimplemented(
+        "BatchNormTraining HLO is unsupported by the evaluator.");
+  };
+  Status HandleInfeed(HloInstruction* infeed) override {
+    return Unimplemented("Infeed HLO is unsupported by the evaluator.");
+  };
+  Status HandleOutfeed(HloInstruction* outfeed) override {
+    return Unimplemented("Outfeed HLO is unsupported by the evaluator.");
+  };
+
   // Returns the already-evaluated literal result for the instruction.
   // A Constant instruction is considered evaluated and its literal will be
   // returned directly without looking up the cache.
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
index 6f3208be13..b1bbf49d4a 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
@@ -917,7 +917,11 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
   Status HandleClamp(HloInstruction* clamp) {
     std::function<ElementwiseT(ElementwiseT, ElementwiseT, ElementwiseT)>
         clamp_op = [](ElementwiseT low, ElementwiseT value, ElementwiseT high) {
-          return std::fmin(high, std::fmax(value, low));
+          if (std::isnan(low) || std::isnan(high)) {
+            return static_cast<ElementwiseT>(NAN);
+          }
+          return static_cast<ElementwiseT>(
+              std::fmin(high, std::fmax(value, low)));
         };
     TF_ASSIGN_OR_RETURN(
         parent_->evaluated_[clamp],
@@ -2664,12 +2668,13 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
     return HandleReducePrecision<ElementwiseT>(reduce_precision);
   }
 
-  template <typename NativeT,
-            typename std::enable_if<
-                std::is_same<NativeT, bfloat16>::value ||
-                std::is_same<NativeT, Eigen::half>::value ||
-                std::is_integral<NativeT>::value ||
-                std::is_floating_point<NativeT>::value>::type* = nullptr>
+  template <
+      typename NativeT,
+      typename std::enable_if<
+          std::is_same<NativeT, bfloat16>::value ||
+          std::is_same<NativeT, Eigen::half>::value ||
+          std::is_integral<NativeT>::value || is_complex_t<NativeT>::value ||
+          std::is_floating_point<NativeT>::value>::type* = nullptr>
   Status HandleIota(HloInstruction* instruction) {
     auto* iota = Cast<HloIotaInstruction>(instruction);
     const int64 iota_size = iota->shape().dimensions(iota->iota_dimension());
@@ -2700,12 +2705,13 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
 
     return Status::OK();
   }
-  template <typename NativeT,
-            typename std::enable_if<
-                !(std::is_same<NativeT, bfloat16>::value ||
-                  std::is_same<NativeT, Eigen::half>::value ||
-                  std::is_integral<NativeT>::value ||
-                  std::is_floating_point<NativeT>::value)>::type* = nullptr>
+  template <
+      typename NativeT,
+      typename std::enable_if<
+          !(std::is_same<NativeT, bfloat16>::value ||
+            std::is_same<NativeT, Eigen::half>::value ||
+            std::is_integral<NativeT>::value || is_complex_t<NativeT>::value ||
+            std::is_floating_point<NativeT>::value)>::type* = nullptr>
   Status HandleIota(HloInstruction* iota) {
     return UnsupportedTypeError(iota);
   }
diff --git a/tensorflow/compiler/xla/service/interpreter/BUILD b/tensorflow/compiler/xla/service/interpreter/BUILD
index a981d94a99..8999f38ccd 100644
--- a/tensorflow/compiler/xla/service/interpreter/BUILD
+++ b/tensorflow/compiler/xla/service/interpreter/BUILD
@@ -32,6 +32,7 @@ cc_library(
         "//tensorflow/compiler/xla:status_macros",
         "//tensorflow/compiler/xla:statusor",
         "//tensorflow/compiler/xla/service:algebraic_simplifier",
+        "//tensorflow/compiler/xla/service:batchnorm_expander",
         "//tensorflow/compiler/xla/service:compiler",
         "//tensorflow/compiler/xla/service:computation_placer",
         "//tensorflow/compiler/xla/service:executable",
@@ -41,12 +42,14 @@ cc_library(
         "//tensorflow/compiler/xla/service:hlo_cost_analysis",
         "//tensorflow/compiler/xla/service:hlo_cse",
         "//tensorflow/compiler/xla/service:hlo_dce",
+        "//tensorflow/compiler/xla/service:hlo_get_dimension_size_rewriter",
         "//tensorflow/compiler/xla/service:hlo_module_config",
         "//tensorflow/compiler/xla/service:hlo_pass",
         "//tensorflow/compiler/xla/service:hlo_pass_pipeline",
         "//tensorflow/compiler/xla/service:hlo_subcomputation_unification",
         "//tensorflow/compiler/xla/service:layout_assignment",
         "//tensorflow/compiler/xla/service:map_inliner",
+        "//tensorflow/compiler/xla/service:reduce_precision_insertion",
         "//tensorflow/compiler/xla/service:reshape_mover",
         "//tensorflow/compiler/xla/service:while_loop_simplifier",
         "//tensorflow/core:lib",
diff --git a/tensorflow/compiler/xla/service/interpreter/compiler.cc b/tensorflow/compiler/xla/service/interpreter/compiler.cc
index d37ae94bf6..69f611a094 100644
--- a/tensorflow/compiler/xla/service/interpreter/compiler.cc
+++ b/tensorflow/compiler/xla/service/interpreter/compiler.cc
@@ -31,6 +31,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/interpreter/executable.h"
 #include "tensorflow/compiler/xla/service/layout_assignment.h"
 #include "tensorflow/compiler/xla/service/map_inliner.h"
+#include "tensorflow/compiler/xla/service/reduce_precision_insertion.h"
 #include "tensorflow/compiler/xla/service/reshape_mover.h"
 #include "tensorflow/compiler/xla/service/while_loop_simplifier.h"
 #include "tensorflow/compiler/xla/status_macros.h"
@@ -46,6 +47,11 @@ Status InterpreterCompiler::RunHloOptimization(HloModule* hlo_module) {
   pipeline.AddPass<LayoutAssignment>(
       hlo_module->mutable_entry_computation_layout(),
       LayoutAssignment::InstructionCanChangeLayout);
+
+  ReducePrecisionInsertion::AddPasses(
+      &pipeline, hlo_module->config().debug_options(),
+      ReducePrecisionInsertion::PassTiming::BEFORE_OPTIMIZATION);
+
   return pipeline.Run(hlo_module).status();
 }
 
diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index ee24d4d99c..578e716b59 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -276,9 +276,6 @@ cc_library(
 xla_test(
     name = "bad_rng_shape_validation_test",
     srcs = ["bad_rng_shape_validation_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:statusor",
         "//tensorflow/compiler/xla:test",
@@ -344,9 +341,6 @@ xla_test(
 xla_test(
     name = "check_execution_arity_test",
     srcs = ["check_execution_arity_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:literal",
         "//tensorflow/compiler/xla:shape_util",
@@ -367,9 +361,6 @@ xla_test(
 xla_test(
     name = "query_inferred_shape_test",
     srcs = ["query_inferred_shape_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:statusor",
@@ -387,9 +378,6 @@ xla_test(
 xla_test(
     name = "while_test",
     srcs = ["while_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:literal",
         "//tensorflow/compiler/xla:shape_util",
@@ -413,6 +401,10 @@ xla_test(
 xla_test(
     name = "xla_hlo_profile_test",
     srcs = ["xla_hlo_profile_test.cc"],
+    blacklisted_backends = [
+        # Hlo profiles are not supported on the interpreter backend.
+        "interpreter",
+    ],
     deps = [
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:shape_util",
@@ -436,9 +428,6 @@ xla_test(
 xla_test(
     name = "axpy_simple_test",
     srcs = ["axpy_simple_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla/client:local_client",
         "//tensorflow/compiler/xla/client:xla_builder",
@@ -453,7 +442,6 @@ xla_test(
 xla_test(
     name = "map_test",
     srcs = ["map_test.cc"],
-    tags = ["enable_for_xla_interpreter"],
     deps = [
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:literal",
@@ -506,9 +494,6 @@ xla_test(
 xla_test(
     name = "pred_test",
     srcs = ["pred_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla/client:local_client",
@@ -524,9 +509,6 @@ xla_test(
 xla_test(
     name = "select_test",
     srcs = ["select_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/client:global_data",
@@ -544,7 +526,6 @@ xla_test(
 xla_test(
     name = "conditional_test",
     srcs = ["conditional_test.cc"],
-    tags = ["enable_for_xla_interpreter"],
     deps = [
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/client:global_data",
@@ -562,7 +543,6 @@ xla_test(
 xla_test(
     name = "unary_op_test",
     srcs = ["unary_op_test.cc"],
-    tags = ["enable_for_xla_interpreter"],
     deps = [
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/client:global_data",
@@ -623,9 +603,6 @@ xla_test(
 xla_test(
     name = "deconstruct_tuple_test",
     srcs = ["deconstruct_tuple_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:literal",
         "//tensorflow/compiler/xla:shape_util",
@@ -648,7 +625,6 @@ xla_test(
     name = "array_elementwise_ops_test",
     srcs = ["array_elementwise_ops_test.cc"],
     shard_count = 25,
-    tags = ["enable_for_xla_interpreter"],
     deps = [
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:array3d",
@@ -698,7 +674,6 @@ xla_test(
 xla_test(
     name = "reduce_precision_test",
     srcs = ["reduce_precision_test.cc"],
-    tags = ["enable_for_xla_interpreter"],
     deps = [
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:literal",
@@ -725,7 +700,6 @@ xla_test(
     srcs = ["dot_operation_test.cc"],
     shard_count = 20,
     tags = [
-        "enable_for_xla_interpreter",
         "optonly",
     ],
     deps = [
@@ -806,9 +780,6 @@ xla_test(
 xla_test(
     name = "transpose_test",
     srcs = ["transpose_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:reference_util",
@@ -828,9 +799,6 @@ xla_test(
 xla_test(
     name = "constants_test",
     srcs = ["constants_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:array3d",
@@ -951,6 +919,11 @@ xla_test(
 xla_test(
     name = "batch_normalization_test",
     srcs = ["batch_normalization_test.cc"],
+    blacklisted_backends = [
+        # BatchNorm HLOs are not handled by the interpreter backend, and the
+        # BatchNorm expander is not run on the interpreter.
+        "interpreter",
+    ],
     shard_count = 40,
     deps = [
         ":test_utils",
@@ -1042,9 +1015,6 @@ xla_test(
     name = "slice_test",
     srcs = ["slice_test.cc"],
     shard_count = 40,
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:reference_util",
@@ -1065,9 +1035,6 @@ xla_test(
 xla_test(
     name = "multidimensional_slice_test",
     srcs = ["multidimensional_slice_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:array3d",
@@ -1085,9 +1052,6 @@ xla_test(
     name = "dynamic_ops_test",
     timeout = "moderate",
     srcs = ["dynamic_ops_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:reference_util",
@@ -1113,9 +1077,6 @@ xla_test(
 xla_test(
     name = "tuple_test",
     srcs = ["tuple_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:literal",
@@ -1139,9 +1100,6 @@ xla_test(
 xla_test(
     name = "vector_ops_reduce_test",
     srcs = ["vector_ops_reduce_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:array3d",
@@ -1162,7 +1120,6 @@ xla_test(
     srcs = ["reduce_test.cc"],
     shard_count = 40,
     tags = [
-        "enable_for_xla_interpreter",
         "optonly",
     ],
     deps = [
@@ -1229,7 +1186,6 @@ xla_test(
     srcs = [],
     shard_count = 20,
     tags = [
-        "enable_for_xla_interpreter",
         "optonly",
     ],
     xla_test_library_deps = [":reduce_window_test_library"],
@@ -1241,7 +1197,6 @@ xla_test(
     timeout = "long",
     srcs = ["select_and_scatter_test.cc"],
     tags = [
-        "enable_for_xla_interpreter",
         "optonly",
     ],
     deps = [
@@ -1267,9 +1222,6 @@ xla_test(
 xla_test(
     name = "copy_test",
     srcs = ["copy_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         ":client_library_test_base",
         "//tensorflow/compiler/xla:array2d",
@@ -1290,9 +1242,6 @@ xla_test(
 xla_test(
     name = "reduce_hlo_test",
     srcs = ["reduce_hlo_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla/service:hlo_parser",
         "//tensorflow/compiler/xla/tests:hlo_test_base",
@@ -1306,9 +1255,6 @@ xla_test(
 xla_test(
     name = "token_hlo_test",
     srcs = ["token_hlo_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla/service:hlo_parser",
         "//tensorflow/compiler/xla/service:hlo_verifier",
@@ -1323,9 +1269,6 @@ xla_test(
 xla_test(
     name = "call_test",
     srcs = ["call_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:literal",
         "//tensorflow/compiler/xla:literal_util",
@@ -1368,9 +1311,6 @@ xla_test(
 xla_test(
     name = "binop_scaling_test",
     srcs = ["binop_scaling_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:array4d",
@@ -1388,9 +1328,6 @@ xla_test(
 xla_test(
     name = "broadcast_simple_test",
     srcs = ["broadcast_simple_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:array4d",
@@ -1410,9 +1347,6 @@ xla_test(
 xla_test(
     name = "pad_test",
     srcs = ["pad_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:array4d",
@@ -1434,9 +1368,6 @@ xla_test(
 xla_test(
     name = "fmax_test",
     srcs = ["fmax_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla/client:local_client",
         "//tensorflow/compiler/xla/client:xla_builder",
@@ -1451,9 +1382,6 @@ xla_test(
 xla_test(
     name = "log_test",
     srcs = ["log_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla/client:local_client",
         "//tensorflow/compiler/xla/client:xla_builder",
@@ -1468,9 +1396,6 @@ xla_test(
 xla_test(
     name = "matrix_ops_simple_test",
     srcs = ["matrix_ops_simple_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:literal",
@@ -1497,6 +1422,10 @@ xla_test(
 xla_test(
     name = "prng_test",
     srcs = ["prng_test.cc"],
+    blacklisted_backends = [
+        # TODO(b/122047800) support RNGs on the interpreter backend.
+        "interpreter",
+    ],
     deps = [
         "//tensorflow/compiler/xla:literal",
         "//tensorflow/compiler/xla:shape_util",
@@ -1517,9 +1446,6 @@ xla_test(
     name = "reshape_test",
     srcs = ["reshape_test.cc"],
     shard_count = 30,
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:array4d",
@@ -1545,9 +1471,6 @@ xla_test(
 xla_test(
     name = "reverse_test",
     srcs = ["reverse_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:array4d",
@@ -1566,9 +1489,6 @@ xla_test(
 xla_test(
     name = "vector_ops_simple_test",
     srcs = ["vector_ops_simple_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:array4d",
         "//tensorflow/compiler/xla:shape_util",
@@ -1592,9 +1512,6 @@ xla_test(
 xla_test(
     name = "concat_test",
     srcs = ["concat_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:array3d",
@@ -1615,9 +1532,6 @@ xla_test(
 xla_test(
     name = "convert_test",
     srcs = ["convert_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:xla_data_proto",
@@ -1637,6 +1551,10 @@ xla_test(
 xla_test(
     name = "all_reduce_test",
     srcs = ["all_reduce_test.cc"],
+    blacklisted_backends = [
+        # All reduce is not supported on the interpreter backend.
+        "interpreter",
+    ],
     deps = [
         "//tensorflow/compiler/xla:literal",
         "//tensorflow/compiler/xla:shape_util",
@@ -1661,9 +1579,6 @@ xla_test(
 xla_test(
     name = "bitcast_convert_test",
     srcs = ["bitcast_convert_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:xla_data_proto",
@@ -1703,9 +1618,6 @@ xla_test(
 xla_test(
     name = "floor_ceil_test",
     srcs = ["floor_ceil_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla/client:local_client",
         "//tensorflow/compiler/xla/client:xla_builder",
@@ -1767,6 +1679,10 @@ xla_test(
 xla_test(
     name = "execution_profile_test",
     srcs = ["execution_profile_test.cc"],
+    blacklisted_backends = [
+        # Execution profiles are not supported on the interpreter backend.
+        "interpreter",
+    ],
     deps = [
         ":client_library_test_base",
         "//tensorflow/compiler/xla/client:global_data",
@@ -1781,6 +1697,10 @@ xla_test(
     name = "execution_profile_test_with_xla_hlo_profile",
     srcs = ["execution_profile_test.cc"],
     args = ["--xla_hlo_profile"],
+    blacklisted_backends = [
+        # Hlo profiles are not supported on the interpreter backend.
+        "interpreter",
+    ],
     deps = [
         ":client_library_test_base",
         "//tensorflow/compiler/xla/client:global_data",
@@ -1794,9 +1714,6 @@ xla_test(
 xla_test(
     name = "replay_test",
     srcs = ["replay_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:literal",
         "//tensorflow/compiler/xla:protobuf_util",
@@ -1819,9 +1736,6 @@ xla_test(
 xla_test(
     name = "broadcast_test",
     srcs = ["broadcast_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:literal",
         "//tensorflow/compiler/xla:shape_util",
@@ -1883,9 +1797,6 @@ xla_test(
 xla_test(
     name = "fusion_test",
     srcs = ["fusion_test.cc"],
-    tags = [
-        "enable_for_xla_interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:literal",
@@ -2003,6 +1914,10 @@ xla_test(
 xla_test(
     name = "outfeed_in_nested_computation_test",
     srcs = ["outfeed_in_nested_computation_test.cc"],
+    blacklisted_backends = [
+        # Outfeed ops are not supported on the interpreter backend.
+        "interpreter",
+    ],
     deps = [
         "//tensorflow/compiler/xla/tests:local_client_test_base",
         "//tensorflow/compiler/xla/tests:xla_internal_test_main",
@@ -2179,7 +2094,6 @@ xla_test(
     srcs = ["iota_test.cc"],
     shard_count = 30,
     tags = [
-        "enable_for_xla_interpreter",
         # Require optimized builds, iota_test_cpu is very slow in fastbuild.
         "optonly",
     ],
diff --git a/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc b/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc
index 915b456b52..e4cc8b4199 100644
--- a/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc
+++ b/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc
@@ -2047,6 +2047,19 @@ XLA_TEST_F(ArrayElementwiseOpTest, NonNanClampF32) {
                              error_spec_);
 }
 
+XLA_TEST_F(ArrayElementwiseOpTest, ClampF32) {
+  SetFastMathDisabled(true);
+  XlaBuilder builder(TestName());
+  auto minimum = ConstantR1<float>(&builder, {1.0f, -6.5f, 1.0f, 2.25f, NAN});
+  auto argument =
+      ConstantR1<float>(&builder, {2.0f, 10.0f, -5.0f, 1.0f, 10.0f});
+  auto maximum = ConstantR1<float>(&builder, {3.0f, 0.5f, 25.5f, NAN, 123.0f});
+  Clamp(minimum, argument, maximum);
+
+  ComputeAndCompareR1<float>(&builder, {2.0f, 0.5f, 1.0f, NAN, NAN}, {},
+                             error_spec_);
+}
+
 XLA_TEST_F(ArrayElementwiseOpTest, ClampF32Scalar) {
   XlaBuilder builder(TestName());
   auto minimum = ConstantR0<float>(&builder, 0.0f);
diff --git a/tensorflow/compiler/xla/tests/bfloat16_test.cc b/tensorflow/compiler/xla/tests/bfloat16_test.cc
index e9728e636f..63e4811705 100644
--- a/tensorflow/compiler/xla/tests/bfloat16_test.cc
+++ b/tensorflow/compiler/xla/tests/bfloat16_test.cc
@@ -76,7 +76,9 @@ XLA_TEST_F(Bfloat16Test, NegateScalarF16) {
                                 error_spec_);
 }
 
-XLA_TEST_F(Bfloat16Test, BatchNormTraining) {
+// Disabled on interpreter since BatchNormExanper is not run by default on the
+// intepreter backend.
+XLA_TEST_F(Bfloat16Test, DISABLED_ON_INTERPRETER(BatchNormTraining)) {
   const int kFeatureIndex = 2;
   XlaBuilder builder(TestName());
 
@@ -110,7 +112,9 @@ XLA_TEST_F(Bfloat16Test, BatchNormTraining) {
   ComputeAndCompareTuple(&builder, expected, {}, ErrorSpec(0.01, 0.02));
 }
 
-XLA_TEST_F(Bfloat16Test, BatchNormGrad) {
+// Disabled on interpreter since BatchNormExanper is not run by default on the
+// intepreter backend.
+XLA_TEST_F(Bfloat16Test, DISABLED_ON_INTERPRETER(BatchNormGrad)) {
   const int kFeatureIndex = 2;
   XlaBuilder builder(TestName());
 
diff --git a/tensorflow/compiler/xla/tests/client_test.cc b/tensorflow/compiler/xla/tests/client_test.cc
index d5b3a4d14f..247328b730 100644
--- a/tensorflow/compiler/xla/tests/client_test.cc
+++ b/tensorflow/compiler/xla/tests/client_test.cc
@@ -109,7 +109,10 @@ XLA_TEST_F(ClientTest, ExecuteWithTupleLayout) {
                                      /*minor_to_major=*/{1, 0})));
 }
 
-XLA_TEST_F(ClientTest, DISABLED_ON_GPU(ExecuteParallel)) {
+// Disabled for interpreter since ExecuteAsyncOnStream is not implemented on
+// interpreter backend.
+XLA_TEST_F(ClientTest,
+           DISABLED_ON_INTERPRETER(DISABLED_ON_GPU(ExecuteParallel))) {
   XlaComputation add_with_one_arg, mul_with_two_args, dot_with_one_arg;
   Shape shape = ShapeUtil::MakeShape(S32, {2, 2});
 
diff --git a/tensorflow/compiler/xla/tests/gather_operation_test.cc b/tensorflow/compiler/xla/tests/gather_operation_test.cc
index daa89398a6..d65b67a535 100644
--- a/tensorflow/compiler/xla/tests/gather_operation_test.cc
+++ b/tensorflow/compiler/xla/tests/gather_operation_test.cc
@@ -600,7 +600,9 @@ ENTRY main {
 
 class GatherClientLibraryTest : public ClientLibraryTestBase {};
 
-XLA_TEST_F(GatherClientLibraryTest, DISABLED_ON_GPU(Basic)) {
+// Disabled on interpreter since ExectuteAsyncOnStream is not supported.
+XLA_TEST_F(GatherClientLibraryTest,
+           DISABLED_ON_INTERPRETER(DISABLED_ON_GPU(Basic))) {
   // We create this HLO, but using the XlaBuilder API.
   //
   // ENTRY main {
diff --git a/tensorflow/compiler/xla/tests/local_client_execute_test.cc b/tensorflow/compiler/xla/tests/local_client_execute_test.cc
index 6522c56325..96527886b7 100644
--- a/tensorflow/compiler/xla/tests/local_client_execute_test.cc
+++ b/tensorflow/compiler/xla/tests/local_client_execute_test.cc
@@ -842,7 +842,8 @@ XLA_TEST_F(LocalClientExecuteTest, ShapeBufferToLiteralConversion64bit) {
        LiteralUtil::CreateR0<int64>(123456789000LL)}));
 }
 
-XLA_TEST_F(LocalClientExecuteTest, InfeedTest) {
+// Disabled on interpreter backend since infeed HLO is unsupported.
+XLA_TEST_F(LocalClientExecuteTest, DISABLED_ON_INTERPRETER(InfeedTest)) {
   XlaBuilder builder(TestName());
   const Shape shape = ShapeUtil::MakeShape(F32, {3});
   auto in = Infeed(&builder, shape);
@@ -867,7 +868,8 @@ XLA_TEST_F(LocalClientExecuteTest, InfeedTest) {
   LiteralTestUtil::ExpectR1Equal<float>({-4.0, 125.0, 45.0}, result);
 }
 
-XLA_TEST_F(LocalClientExecuteTest, InfeedOutfeedTest) {
+// Disabled on interpreter backend since infeed/outfeed HLOs are unsupported.
+XLA_TEST_F(LocalClientExecuteTest, DISABLED_ON_INTERPRETER(InfeedOutfeedTest)) {
   XlaBuilder builder(TestName());
   const Shape shape = ShapeUtil::MakeShape(F32, {3});
   auto in = Infeed(&builder, shape);
-- 
GitLab


From 4435d81822bef10437b3ec7a73300db62a5a53ad Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Wed, 2 Jan 2019 16:16:35 -0800
Subject: [PATCH 0153/1095] Throw a more helpful error message when trying to
 export a subclassed Model

Previously it said to call _set_inputs. More directly people should just specify signatures=...

PiperOrigin-RevId: 227599645
---
 tensorflow/python/saved_model/save_test.py | 25 ++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/tensorflow/python/saved_model/save_test.py b/tensorflow/python/saved_model/save_test.py
index 4573495998..e643ff3dbf 100644
--- a/tensorflow/python/saved_model/save_test.py
+++ b/tensorflow/python/saved_model/save_test.py
@@ -31,6 +31,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_spec
 from tensorflow.python.framework import test_util
+from tensorflow.python.keras.engine import training
 from tensorflow.python.keras.layers import core
 from tensorflow.python.lib.io import file_io
 from tensorflow.python.ops import lookup_ops
@@ -306,6 +307,30 @@ class SaveTest(test.TestCase):
       self.assertNotIn("T", complex_node.attr)
       self.assertNotIn("Tout", complex_node.attr)
 
+  def test_subclassed_no_signature(self):
+
+    class Subclassed(training.Model):
+
+      def call(self, inputs):
+        return inputs * 2.
+
+    save_dir = os.path.join(self.get_temp_dir(), "saved_model")
+    model = Subclassed()
+    with self.assertRaisesRegexp(
+        ValueError, "no @tf.function-decorated methods"):
+      save.save(model, save_dir)
+
+    traced_call = def_function.function(
+        model.call,
+        input_signature=(tensor_spec.TensorSpec(
+            (None, None),
+            dtype=dtypes.float32),))
+    save.save(model, save_dir, traced_call)
+    self.assertAllClose({"output_0": [[8., 10.], [10., 12.]]},
+                        _import_and_infer(
+                            save_dir,
+                            {"inputs": [[4., 5.], [5., 6.]]}))
+
 
 class AssetTests(test.TestCase):
 
-- 
GitLab


From da7b71f67147ff4795c5c0168d1f225ba2b4b522 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 2 Jan 2019 16:32:13 -0800
Subject: [PATCH 0154/1095] Automated rollback of commit
 7d2be9eef5f4b74da5b5c6f473ea2e6ad57398dc

PiperOrigin-RevId: 227601883
---
 .../lite/delegates/nnapi/nnapi_delegate.cc    |  161 ++-
 tensorflow/lite/nnapi/BUILD                   |   12 -
 tensorflow/lite/nnapi/NeuralNetworksShim.cc   |  137 --
 tensorflow/lite/nnapi/NeuralNetworksShim.h    | 1205 +++++++++--------
 tensorflow/lite/nnapi/nnapi_lib_test.cc       |   90 --
 tensorflow/lite/nnapi_delegate.cc             |  135 +-
 6 files changed, 809 insertions(+), 931 deletions(-)
 delete mode 100644 tensorflow/lite/nnapi/NeuralNetworksShim.cc
 delete mode 100644 tensorflow/lite/nnapi/nnapi_lib_test.cc

diff --git a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
index 9487d3331f..8af159e6fb 100644
--- a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
+++ b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
@@ -48,39 +48,57 @@ namespace {
   } while (0)
 
 namespace {
+int32_t GetAndroidSdkVersion() {
+#ifdef __ANDROID__
+  const char* sdkProp = "ro.build.version.sdk";
+  char sdkVersion[PROP_VALUE_MAX];
+  int length = __system_property_get(sdkProp, sdkVersion);
+  if (length != 0) {
+    for (int i = 0; i < length; ++i) {
+      int digit = sdkVersion[i] - '0';
+      if (digit < 0 || digit > 9) {
+        // Non-numeric SDK version, assume it's higher then expected;
+        return std::numeric_limits<int32_t>::max();
+      }
+    }
+    return atoi(sdkVersion);
+  }
+#endif  // __ANDROID__
+  return 0;
+}
+
 constexpr int32_t kMinSdkVersionForNNAPI = 27;
 constexpr int32_t kMinSdkVersionForNNAPI11 = 28;
+static const int32_t kAndroidSdkVersion = GetAndroidSdkVersion();
+
 }  // namespace
 
 // RAII NN API Model Destructor for use with std::unique_ptr
 struct NNFreeModel {
   void operator()(ANeuralNetworksModel* model) {
-    NnApiImplementation()->ANeuralNetworksModel_free(model);
+    ANeuralNetworksModel_free(model);
   }
 };
 // RAII NN API Compilation Destructor for use with std::unique_ptr
 struct NNFreeCompilation {
   void operator()(ANeuralNetworksCompilation* model) {
-    NnApiImplementation()->ANeuralNetworksCompilation_free(model);
+    ANeuralNetworksCompilation_free(model);
   }
 };
 
 // Manage NNAPI shared memory handle
 class NNMemory {
  public:
+  NNMemory(const char* name, size_t size) {
 #ifdef __ANDROID__
-  NNMemory(const NnApi* nnapi, const char* name, size_t size) {
-    nnapi_ = nnapi;
     byte_size_ = size;
-    fd_ = nnapi_->ASharedMemory_create(name, size);
+    fd_ = ASharedMemory_create(name, size);
     data_ptr_ = reinterpret_cast<uint8_t*>(
         mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd_, 0));
-    nnapi_->ANeuralNetworksMemory_createFromFd(size, PROT_READ | PROT_WRITE,
-                                               fd_, 0, &nn_memory_handle_);
-  }
-#else
-  NNMemory(const NnApi* /*nnapi*/, const char* /*name*/, size_t /*size*/) {}
+    ANeuralNetworksMemory_createFromFd(size, PROT_READ | PROT_WRITE, fd_, 0,
+                                       &nn_memory_handle_);
 #endif
+  }
 
   ~NNMemory() {
 #ifdef __ANDROID__
@@ -88,7 +106,7 @@ class NNMemory {
       munmap(data_ptr_, byte_size_);
     }
     if (nn_memory_handle_) {
-      nnapi_->ANeuralNetworksMemory_free(nn_memory_handle_);
+      ANeuralNetworksMemory_free(nn_memory_handle_);
     }
     if (fd_ > 0) close(fd_);
 #endif
@@ -99,7 +117,6 @@ class NNMemory {
 
  private:
 #ifdef __ANDROID__
-  const NnApi* nnapi_;
   int fd_ = 0;
   size_t byte_size_ = 0;
 #endif
@@ -149,10 +166,9 @@ class OperandMapping {
 // operands for both tensors and parameters, and TFLite separates the two.
 class NNAPIOpBuilder {
  public:
-  NNAPIOpBuilder(const NnApi* nnapi, TfLiteContext* context,
-                 OperandMapping* tensor_mapping, ANeuralNetworksModel* nn_model)
-      : nnapi_(nnapi),
-        context_(context),
+  NNAPIOpBuilder(TfLiteContext* context, OperandMapping* tensor_mapping,
+                 ANeuralNetworksModel* nn_model)
+      : context_(context),
         operand_mapping_(tensor_mapping),
         nn_model_(nn_model) {}
 
@@ -208,8 +224,7 @@ class NNAPIOpBuilder {
         .dimensionCount = dimension_count,
         .dimensions = dims.data()};
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context_,
-        nnapi_->ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
+        context_, ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
     int ann_operand = operand_mapping_->add_new_non_tensor_operand();
     augmented_outputs_.push_back(ann_operand);
     return kTfLiteOk;
@@ -226,8 +241,7 @@ class NNAPIOpBuilder {
         reinterpret_cast<uint32_t*>(tensor->dims->data), tensor->params.scale,
         tensor->params.zero_point};
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context_,
-        nnapi_->ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
+        context_, ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
     augmented_outputs_.push_back(ann_index);
 
     *ann_tensor_index_out = ann_index;
@@ -284,14 +298,13 @@ class NNAPIOpBuilder {
         nn_type, static_cast<uint32_t>(tensor->dims->size),
         reinterpret_cast<uint32_t*>(tensor->dims->data), scale, zeroPoint};
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context_,
-        nnapi_->ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
+        context_, ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
 
     if (tensor->allocation_type == kTfLiteMmapRo) {
       // TODO(b/80630405): Use NNAPIAllocation.
       RETURN_TFLITE_ERROR_IF_NN_ERROR(
           context_,
-          nnapi_->ANeuralNetworksModel_setOperandValue(
+          ANeuralNetworksModel_setOperandValue(
               nn_model_, ann_tensor_index, tensor->data.raw, tensor->bytes));
     }
 
@@ -304,7 +317,7 @@ class NNAPIOpBuilder {
     // Actually add a NN API operation
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
         context_,
-        nnapi_->ANeuralNetworksModel_addOperation(
+        ANeuralNetworksModel_addOperation(
             nn_model_, type, static_cast<uint32_t>(augmented_inputs_.size()),
             augmented_inputs_.data(),
             static_cast<uint32_t>(augmented_outputs_.size()),
@@ -319,12 +332,11 @@ class NNAPIOpBuilder {
   TfLiteStatus AddScalarOperand(T value, int32_t nn_type) {
     ANeuralNetworksOperandType operand_type{.type = nn_type};
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context_,
-        nnapi_->ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
+        context_, ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
     int ann_operand = operand_mapping_->add_new_non_tensor_operand();
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context_, nnapi_->ANeuralNetworksModel_setOperandValue(
-                      nn_model_, ann_operand, &value, sizeof(T)));
+        context_, ANeuralNetworksModel_setOperandValue(nn_model_, ann_operand,
+                                                       &value, sizeof(T)));
     augmented_inputs_.push_back(ann_operand);
     return kTfLiteOk;
   }
@@ -335,19 +347,15 @@ class NNAPIOpBuilder {
     ANeuralNetworksOperandType operand_type{
         .type = nn_type, .dimensionCount = 1, .dimensions = &num_values};
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context_,
-        nnapi_->ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
+        context_, ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
     int ann_operand = operand_mapping_->add_new_non_tensor_operand();
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context_, nnapi_->ANeuralNetworksModel_setOperandValue(
+        context_, ANeuralNetworksModel_setOperandValue(
                       nn_model_, ann_operand, values, sizeof(T) * num_values));
     augmented_inputs_.push_back(ann_operand);
     return kTfLiteOk;
   }
 
-  // Access to NNAPI.
-  const NnApi* const nnapi_;
-
   // TfLiteContext for error handling.
   TfLiteContext* const context_;
 
@@ -383,7 +391,7 @@ ANeuralNetworksOperationType BasicMappingFn(
 // The kernel that represents the node sub set of TF Lite being run on NN API.
 class NNAPIDelegateKernel {
  public:
-  NNAPIDelegateKernel() { nnapi_ = NnApiImplementation(); }
+  NNAPIDelegateKernel() = default;
 
   typedef ANeuralNetworksOperationType (*MappingFn)(
       const NNAPIOpMappingArgs& mapping_args);
@@ -392,7 +400,7 @@ class NNAPIDelegateKernel {
   // when called. You can use this function to see if a node is supported
   // (i.e. that MappingFn is not nullptr).
   static MappingFn Map(TfLiteContext* context, int builtin_code, int version,
-                       int android_sdk_version, TfLiteNode* node) {
+                       TfLiteNode* node) {
     switch (builtin_code) {
       case kTfLiteBuiltinAdd:
         if (version == 1) {
@@ -511,7 +519,7 @@ class NNAPIDelegateKernel {
         }
         break;
       case kTfLiteBuiltinSqueeze:
-        if (version == 1 && android_sdk_version >= kMinSdkVersionForNNAPI11) {
+        if (version == 1 && kAndroidSdkVersion >= kMinSdkVersionForNNAPI11) {
           return [](const NNAPIOpMappingArgs& mapping_args)
                      -> ANeuralNetworksOperationType {
             auto builtin = reinterpret_cast<TfLiteSqueezeParams*>(
@@ -627,7 +635,7 @@ class NNAPIDelegateKernel {
         }
         break;
       case kTfLiteBuiltinSub:
-        if (version == 1 && android_sdk_version >= kMinSdkVersionForNNAPI11 &&
+        if (version == 1 && kAndroidSdkVersion >= kMinSdkVersionForNNAPI11 &&
             context->tensors[node->inputs->data[0]].type == kTfLiteFloat32) {
           // NNAPI only support float sub.
           return [](const NNAPIOpMappingArgs& mapping_args)
@@ -640,7 +648,7 @@ class NNAPIDelegateKernel {
         }
         break;
       case kTfLiteBuiltinDiv:
-        if (version == 1 && android_sdk_version >= kMinSdkVersionForNNAPI11 &&
+        if (version == 1 && kAndroidSdkVersion >= kMinSdkVersionForNNAPI11 &&
             context->tensors[node->inputs->data[0]].type == kTfLiteFloat32) {
           // NNAPI only support float div.
           return [](const NNAPIOpMappingArgs& mapping_args)
@@ -653,7 +661,7 @@ class NNAPIDelegateKernel {
         }
         break;
       case kTfLiteBuiltinPad:
-        if (version == 1 && android_sdk_version >= kMinSdkVersionForNNAPI11 &&
+        if (version == 1 && kAndroidSdkVersion >= kMinSdkVersionForNNAPI11 &&
             node->inputs->size == 2 &&
             context->tensors[node->inputs->data[0]].type == kTfLiteFloat32) {
           // NNAPI does not support specifying the padding value.
@@ -663,12 +671,12 @@ class NNAPIDelegateKernel {
         }
         break;
       case kTfLiteBuiltinSpaceToBatchNd:
-        if (version == 1 && android_sdk_version >= kMinSdkVersionForNNAPI11) {
+        if (version == 1 && kAndroidSdkVersion >= kMinSdkVersionForNNAPI11) {
           return BasicMappingFn<ANEURALNETWORKS_SPACE_TO_BATCH_ND>;
         }
         break;
       case kTfLiteBuiltinStridedSlice:
-        if (version == 1 && android_sdk_version >= kMinSdkVersionForNNAPI11) {
+        if (version == 1 && kAndroidSdkVersion >= kMinSdkVersionForNNAPI11) {
           return [](const NNAPIOpMappingArgs& mapping_args)
                      -> ANeuralNetworksOperationType {
             auto builtin = reinterpret_cast<TfLiteStridedSliceParams*>(
@@ -686,7 +694,7 @@ class NNAPIDelegateKernel {
         // dimensions.
         // TODO(b/110888333): Support dynamically-sized tensors in delegates.
         if ((version == 1) &&
-            (android_sdk_version >= kMinSdkVersionForNNAPI11) &&
+            (kAndroidSdkVersion >= kMinSdkVersionForNNAPI11) &&
             (node->inputs->size > 1) &&
             (context->tensors[node->inputs->data[1]].allocation_type ==
              kTfLiteMmapRo)) {
@@ -784,7 +792,7 @@ class NNAPIDelegateKernel {
         break;
       case kTfLiteBuiltinMean:
         // NNAPI does not support generating a scalar as output for MEAN.
-        if (version == 1 && android_sdk_version >= kMinSdkVersionForNNAPI11 &&
+        if (version == 1 && kAndroidSdkVersion >= kMinSdkVersionForNNAPI11 &&
             context->tensors[node->inputs->data[0]].type == kTfLiteFloat32 &&
             context->tensors[node->outputs->data[0]].dims->size > 0) {
           return [](const NNAPIOpMappingArgs& mapping_args)
@@ -828,8 +836,8 @@ class NNAPIDelegateKernel {
 
     if (!nn_model_) {
       ANeuralNetworksModel* model;
-      RETURN_TFLITE_ERROR_IF_NN_ERROR(
-          context, nnapi_->ANeuralNetworksModel_create(&model));
+      RETURN_TFLITE_ERROR_IF_NN_ERROR(context,
+                                      ANeuralNetworksModel_create(&model));
       nn_model_.reset(model);
 
       TF_LITE_ENSURE_STATUS(
@@ -839,10 +847,10 @@ class NNAPIDelegateKernel {
     if (!nn_compilation_) {
       ANeuralNetworksCompilation* compilation;
       RETURN_TFLITE_ERROR_IF_NN_ERROR(
-          context, nnapi_->ANeuralNetworksCompilation_create(nn_model_.get(),
-                                                             &compilation));
+          context,
+          ANeuralNetworksCompilation_create(nn_model_.get(), &compilation));
       RETURN_TFLITE_ERROR_IF_NN_ERROR(
-          context, nnapi_->ANeuralNetworksCompilation_finish(compilation));
+          context, ANeuralNetworksCompilation_finish(compilation));
       nn_compilation_.reset(compilation);
     }
     return kTfLiteOk;
@@ -851,8 +859,8 @@ class NNAPIDelegateKernel {
   TfLiteStatus Invoke(TfLiteContext* context, TfLiteNode* node) {
     ANeuralNetworksExecution* execution = nullptr;
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context, nnapi_->ANeuralNetworksExecution_create(nn_compilation_.get(),
-                                                         &execution));
+        context,
+        ANeuralNetworksExecution_create(nn_compilation_.get(), &execution));
 
     // Set the input tensor buffers. Note: we access tflite tensors using
     // absolute indices but NN api indices inputs by relative indices.
@@ -872,7 +880,7 @@ class NNAPIDelegateKernel {
                tensor->data.raw, tensor->bytes);
         RETURN_TFLITE_ERROR_IF_NN_ERROR(
             context,
-            nnapi_->ANeuralNetworksExecution_setInputFromMemory(
+            ANeuralNetworksExecution_setInputFromMemory(
                 execution, relative_input_index, nullptr,
                 nn_input_memory_->get_handle(), input_offset, tensor->bytes));
         input_offset += tensor->bytes;
@@ -887,7 +895,7 @@ class NNAPIDelegateKernel {
       TfLiteTensor* tensor = &context->tensors[output_index];
       RETURN_TFLITE_ERROR_IF_NN_ERROR(
           context,
-          nnapi_->ANeuralNetworksExecution_setOutputFromMemory(
+          ANeuralNetworksExecution_setOutputFromMemory(
               execution, relative_output_index, nullptr,
               nn_output_memory_->get_handle(), output_offset, tensor->bytes));
       output_offset += tensor->bytes;
@@ -903,7 +911,7 @@ class NNAPIDelegateKernel {
       // reading and writing into the same buffer during a invocation.
       // TODO(110369471): using double shared buffer to minimize the copies.
       RETURN_TFLITE_ERROR_IF_NN_ERROR(
-          context, nnapi_->ANeuralNetworksExecution_setOutput(
+          context, ANeuralNetworksExecution_setOutput(
                        execution, relative_output_index, nullptr,
                        tensor->data.raw, tensor->bytes));
       relative_output_index++;
@@ -911,12 +919,10 @@ class NNAPIDelegateKernel {
     // Invoke ANN in blocking fashion.
     ANeuralNetworksEvent* event = nullptr;
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context,
-        nnapi_->ANeuralNetworksExecution_startCompute(execution, &event));
-    RETURN_TFLITE_ERROR_IF_NN_ERROR(context,
-                                    nnapi_->ANeuralNetworksEvent_wait(event));
-    nnapi_->ANeuralNetworksEvent_free(event);
-    nnapi_->ANeuralNetworksExecution_free(execution);
+        context, ANeuralNetworksExecution_startCompute(execution, &event));
+    RETURN_TFLITE_ERROR_IF_NN_ERROR(context, ANeuralNetworksEvent_wait(event));
+    ANeuralNetworksEvent_free(event);
+    ANeuralNetworksExecution_free(execution);
 
     // copy results from shared memory to the destination.
     output_offset = 0;
@@ -931,8 +937,6 @@ class NNAPIDelegateKernel {
   }
 
  private:
-  // Access to NNApi.
-  const NnApi* nnapi_;
   // ANN API state.
   std::unique_ptr<ANeuralNetworksModel, NNFreeModel> nn_model_;
   std::unique_ptr<ANeuralNetworksCompilation, NNFreeCompilation>
@@ -953,7 +957,7 @@ class NNAPIDelegateKernel {
     // The operand builder allows creating a single op. We create it at this
     // reduced power position rather than in the for loop to avoid reallocating
     // the vectors.
-    NNAPIOpBuilder builder(nnapi_, context, &operand_mapping_, nn_model_.get());
+    NNAPIOpBuilder builder(context, &operand_mapping_, nn_model_.get());
     // Add Tensors
     // allocate outside to avoid realloc
     for (auto node_index : nodes_) {
@@ -976,10 +980,9 @@ class NNAPIDelegateKernel {
         }
       }
       // Get op type and operands
-      int nn_op_type = Map(
-          context, reg->builtin_code, reg->version, nnapi_->android_sdk_version,
-          node)({context, &builder, node, &model_state_outputs_,
-                 &model_state_tfl_inputs_});
+      int nn_op_type = Map(context, reg->builtin_code, reg->version, node)(
+          {context, &builder, node, &model_state_outputs_,
+           &model_state_tfl_inputs_});
       // Map outputs to NN API tensor indices.
       for (auto output_index : TfLiteIntArrayView(node->outputs)) {
         TF_LITE_ENSURE_STATUS(builder.AddTensorOutput(output_index));
@@ -1025,27 +1028,25 @@ class NNAPIDelegateKernel {
 
     // Tell ANN to declare inputs/outputs
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context, nnapi_->ANeuralNetworksModel_identifyInputsAndOutputs(
+        context, ANeuralNetworksModel_identifyInputsAndOutputs(
                      nn_model_.get(), inputs.size(), inputs.data(),
                      outputs.size(), outputs.data()));
 
     // Set relaxed computation mode for fp32 if possible.
-    if (nnapi_->android_sdk_version >= kMinSdkVersionForNNAPI11) {
+    if (kAndroidSdkVersion >= kMinSdkVersionForNNAPI11) {
       RETURN_TFLITE_ERROR_IF_NN_ERROR(
-          context,
-          nnapi_->ANeuralNetworksModel_relaxComputationFloat32toFloat16(
-              nn_model_.get(), context->allow_fp32_relax_to_fp16));
+          context, ANeuralNetworksModel_relaxComputationFloat32toFloat16(
+                       nn_model_.get(), context->allow_fp32_relax_to_fp16));
     }
 
     // Finalize the model
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context, nnapi_->ANeuralNetworksModel_finish(nn_model_.get()));
+        context, ANeuralNetworksModel_finish(nn_model_.get()));
 
     // Create shared memory pool for inputs and outputs.
-    nn_input_memory_.reset(
-        new NNMemory(nnapi_, "input_pool", total_input_byte_size));
+    nn_input_memory_.reset(new NNMemory("input_pool", total_input_byte_size));
     nn_output_memory_.reset(
-        new NNMemory(nnapi_, "output_pool", total_output_byte_size));
+        new NNMemory("output_pool", total_output_byte_size));
 
     return kTfLiteOk;
   }
@@ -1061,9 +1062,7 @@ TfLiteDelegate* NnApiDelegate() {
       .Prepare = [](TfLiteContext* context,
                     TfLiteDelegate* delegate) -> TfLiteStatus {
         // Do not check nodes_ if NN API is unavailable.
-        const NnApi* nnapi = NnApiImplementation();
-        if (nnapi->android_sdk_version < kMinSdkVersionForNNAPI ||
-            !nnapi->nnapi_exists) {
+        if (kAndroidSdkVersion < kMinSdkVersionForNNAPI || !NNAPIExists()) {
           return kTfLiteOk;
         }
 
@@ -1076,7 +1075,6 @@ TfLiteDelegate* NnApiDelegate() {
         TfLiteIntArray* plan;
         TF_LITE_ENSURE_STATUS(context->GetExecutionPlan(context, &plan));
 
-        int android_sdk_version = NnApiImplementation()->android_sdk_version;
         // Check for every node if it is supported
         // TODO(b/80625235): Fix this to do more careful checking of versioning.
         for (int node_index : TfLiteIntArrayView(plan)) {
@@ -1085,8 +1083,7 @@ TfLiteDelegate* NnApiDelegate() {
           TF_LITE_ENSURE_STATUS(context->GetNodeAndRegistration(
               context, node_index, &node, &registration));
           if (NNAPIDelegateKernel::Map(context, registration->builtin_code,
-                                       registration->version,
-                                       android_sdk_version, node)) {
+                                       registration->version, node)) {
             supported_nodes.push_back(node_index);
           }
         }
diff --git a/tensorflow/lite/nnapi/BUILD b/tensorflow/lite/nnapi/BUILD
index 390c3730cb..467a2b7a7b 100644
--- a/tensorflow/lite/nnapi/BUILD
+++ b/tensorflow/lite/nnapi/BUILD
@@ -6,20 +6,8 @@ package(default_visibility = [
 
 cc_library(
     name = "nnapi_lib",
-    srcs = [
-        "NeuralNetworksShim.cc",
-    ],
     hdrs = [
         "NeuralNetworksShim.h",
     ],
     linkopts = ["-ldl"],
 )
-
-cc_test(
-    name = "nnapi_lib_test",
-    srcs = ["nnapi_lib_test.cc"],
-    deps = [
-        "//tensorflow/lite/nnapi:nnapi_lib",
-        "@com_google_googletest//:gtest_main",
-    ],
-)
diff --git a/tensorflow/lite/nnapi/NeuralNetworksShim.cc b/tensorflow/lite/nnapi/NeuralNetworksShim.cc
deleted file mode 100644
index c063525cae..0000000000
--- a/tensorflow/lite/nnapi/NeuralNetworksShim.cc
+++ /dev/null
@@ -1,137 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#include "tensorflow/lite/nnapi/NeuralNetworksShim.h"
-
-#include <cstdlib>
-
-#ifdef __ANDROID__
-#include <dlfcn.h>
-#include <sys/mman.h>
-#include <sys/system_properties.h>
-#include <unistd.h>
-#endif
-
-#define NNAPI_LOG(format, ...) fprintf(stderr, format "\n", __VA_ARGS__);
-
-namespace {
-
-#ifdef __ANDROID__
-int32_t GetAndroidSdkVersion() {
-  const char* sdkProp = "ro.build.version.sdk";
-  char sdkVersion[PROP_VALUE_MAX];
-  int length = __system_property_get(sdkProp, sdkVersion);
-  if (length != 0) {
-    int32_t result = 0;
-    for (int i = 0; i < length; ++i) {
-      int digit = sdkVersion[i] - '0';
-      if (digit < 0 || digit > 9) {
-        // Non-numeric SDK version, assume it's higher than expected;
-        return 0xffff;
-      }
-      result = result * 10 + digit;
-    }
-    return result;
-  }
-  return 0;
-}
-
-void* LoadFunction(void* handle, const char* name) {
-  if (handle == nullptr) {
-    return nullptr;
-  }
-  void* fn = dlsym(handle, name);
-  if (fn == nullptr) {
-    NNAPI_LOG("nnapi error: unable to open function %s", name);
-  }
-  return fn;
-}
-
-#define LOAD_FUNCTION(handle, name) \
-  nnapi.name = reinterpret_cast<name##_fn>(LoadFunction(handle, #name));
-
-#else
-
-#define LOAD_FUNCTION(handle, name) nnapi.name = nullptr;
-
-#endif
-
-const NnApi LoadNnApi() {
-  NnApi nnapi = {};
-
-#ifdef __ANDROID__
-  void* libneuralnetworks = nullptr;
-  void* libandroid = nullptr;
-  nnapi.android_sdk_version = GetAndroidSdkVersion();
-  if (nnapi.android_sdk_version < 27) {
-    NNAPI_LOG("nnapi error: requires android sdk version to be at least %d",
-              27);
-  } else {
-    // TODO: change RTLD_LOCAL? Assumes there can be multiple instances of nn
-    // api RT
-    libneuralnetworks = dlopen("libneuralnetworks.so", RTLD_LAZY | RTLD_LOCAL);
-    if (libneuralnetworks == nullptr) {
-      NNAPI_LOG("nnapi error: unable to open library %s",
-                "libneuralnetworks.so");
-    }
-    libandroid = dlopen("libandroid.so", RTLD_LAZY | RTLD_LOCAL);
-    if (libneuralnetworks == nullptr) {
-      NNAPI_LOG("nnapi error: unable to open library %s", "libandroid.so");
-    }
-  }
-  nnapi.nnapi_exists = libneuralnetworks != nullptr;
-#else
-  nnapi.nnapi_exists = false;
-  nnapi.android_sdk_version = 0;
-#endif
-
-  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksMemory_createFromFd);
-  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksMemory_free);
-  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksModel_create);
-  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksModel_free);
-  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksModel_finish);
-  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksModel_addOperand);
-  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksModel_setOperandValue);
-  LOAD_FUNCTION(libneuralnetworks,
-                ANeuralNetworksModel_setOperandValueFromMemory);
-  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksModel_addOperation);
-  LOAD_FUNCTION(libneuralnetworks,
-                ANeuralNetworksModel_identifyInputsAndOutputs);
-  LOAD_FUNCTION(libneuralnetworks,
-                ANeuralNetworksModel_relaxComputationFloat32toFloat16);
-  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksCompilation_create);
-  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksCompilation_free);
-  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksCompilation_setPreference);
-  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksCompilation_finish);
-  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksExecution_create);
-  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksExecution_free);
-  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksExecution_setInput);
-  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksExecution_setInputFromMemory);
-  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksExecution_setOutput);
-  LOAD_FUNCTION(libneuralnetworks,
-                ANeuralNetworksExecution_setOutputFromMemory);
-  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksExecution_startCompute);
-  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksEvent_wait);
-  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksEvent_free);
-  LOAD_FUNCTION(libandroid, ASharedMemory_create);
-
-  return nnapi;
-}
-
-}  // namespace
-
-const NnApi* NnApiImplementation() {
-  static const NnApi nnapi = LoadNnApi();
-  return &nnapi;
-}
diff --git a/tensorflow/lite/nnapi/NeuralNetworksShim.h b/tensorflow/lite/nnapi/NeuralNetworksShim.h
index 3fd869e4af..c39502f4ac 100644
--- a/tensorflow/lite/nnapi/NeuralNetworksShim.h
+++ b/tensorflow/lite/nnapi/NeuralNetworksShim.h
@@ -15,10 +15,69 @@ limitations under the License.
 #ifndef TENSORFLOW_LITE_NNAPI_NEURALNETWORKSSHIM_H_
 #define TENSORFLOW_LITE_NNAPI_NEURALNETWORKSSHIM_H_
 
+#include <dlfcn.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 
+// helpers
+
+#define NNAPI_LOG(format, ...) fprintf(stderr, format "\n", __VA_ARGS__);
+#define LOAD_FUNCTION(name) \
+  static name##_fn fn = reinterpret_cast<name##_fn>(loadFunction(#name));
+#define EXECUTE_FUNCTION(...) \
+  if (fn != nullptr) {        \
+    fn(__VA_ARGS__);          \
+  }
+#define EXECUTE_FUNCTION_RETURN(...) return fn != nullptr ? fn(__VA_ARGS__) : 0;
+
+inline void* loadLibrary(const char* name) {
+  // TODO: change RTLD_LOCAL? Assumes there can be multiple instances of nn
+  // api RT
+  void* handle = nullptr;
+#ifdef __ANDROID__
+  handle = dlopen(name, RTLD_LAZY | RTLD_LOCAL);
+  if (handle == nullptr) {
+    NNAPI_LOG("nnapi error: unable to open library %s", name);
+  }
+#endif
+  return handle;
+}
+
+typedef int (*ASharedMemory_create_fn)(const char* name, size_t size);
+
+// ASharedMemory_create was added in Android 8.0, so safe to use with NNAPI
+// which was added in 8.1.
+inline int ASharedMemory_create(const char* name, size_t size) {
+  static void* handle = loadLibrary("libandroid.so");
+  static ASharedMemory_create_fn fn =
+      handle != nullptr ? reinterpret_cast<ASharedMemory_create_fn>(
+                              dlsym(handle, "ASharedMemory_create"))
+                        : nullptr;
+  return fn(name, size);
+}
+
+inline void* getLibraryHandle() {
+  static void* handle = loadLibrary("libneuralnetworks.so");
+  return handle;
+}
+
+inline void* loadFunction(const char* name) {
+  void* fn = nullptr;
+  if (getLibraryHandle() != nullptr) {
+    fn = dlsym(getLibraryHandle(), name);
+  }
+  if (fn == nullptr) {
+    NNAPI_LOG("nnapi error: unable to open function %s", name);
+  }
+  return fn;
+}
+
+inline bool NNAPIExists() {
+  static bool nnapi_is_available = getLibraryHandle();
+  return nnapi_is_available;
+}
+
 // NN api types based on NNAPI header file
 // https://developer.android.com/ndk/reference/group/neural-networks
 
@@ -348,564 +407,606 @@ typedef int (*ANeuralNetworksEvent_wait_fn)(ANeuralNetworksEvent* event);
 
 typedef void (*ANeuralNetworksEvent_free_fn)(ANeuralNetworksEvent* event);
 
-typedef int (*ASharedMemory_create_fn)(const char* name, size_t size);
+/**
+ * Creates a shared memory object from a file descriptor.
+ *
+ * The shared memory is backed by a file descriptor via mmap.
+ * See {@link ANeuralNetworksMemory} for a description on how to use
+ * this shared memory.
+ *
+ * @param size The requested size in bytes.
+ *             Must not be larger than the file size.
+ * @param prot The desired memory protection for the mapping.
+ *             It is either PROT_NONE or the bitwise OR of one or
+ *             more of the following flags: PROT_READ, PROT_WRITE.
+ * @param fd The requested file descriptor.
+ *           The file descriptor has to be mmap-able. The file
+ *           descriptor will be duplicated.
+ * @param offset The offset to the beginning of the file of the area to map.
+ *               The offset has to be aligned to a page size.
+ * @param memory The memory object to be created.
+ *               Set to NULL if unsuccessful.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if the request completed normally.
+ */
+inline int ANeuralNetworksMemory_createFromFd(size_t size, int protect, int fd,
+                                              size_t offset,
+                                              ANeuralNetworksMemory** memory) {
+  LOAD_FUNCTION(ANeuralNetworksMemory_createFromFd);
+  EXECUTE_FUNCTION_RETURN(size, protect, fd, offset, memory);
+}
 
-struct NnApi {
-  bool nnapi_exists;
-  int32_t android_sdk_version;
-
-  /**
-   * Creates a shared memory object from a file descriptor.
-   *
-   * The shared memory is backed by a file descriptor via mmap.
-   * See {@link ANeuralNetworksMemory} for a description on how to use
-   * this shared memory.
-   *
-   * @param size The requested size in bytes.
-   *             Must not be larger than the file size.
-   * @param prot The desired memory protection for the mapping.
-   *             It is either PROT_NONE or the bitwise OR of one or
-   *             more of the following flags: PROT_READ, PROT_WRITE.
-   * @param fd The requested file descriptor.
-   *           The file descriptor has to be mmap-able. The file
-   *           descriptor will be duplicated.
-   * @param offset The offset to the beginning of the file of the area to map.
-   *               The offset has to be aligned to a page size.
-   * @param memory The memory object to be created.
-   *               Set to NULL if unsuccessful.
-   *
-   * @return ANEURALNETWORKS_NO_ERROR if the request completed normally.
-   */
-  int (*ANeuralNetworksMemory_createFromFd)(size_t size, int protect, int fd,
-                                            size_t offset,
-                                            ANeuralNetworksMemory** memory);
-
-  /**
-   * Delete a memory object.
-   *
-   * Destroys the object used by the run time to keep track of the memory.
-   * This will free the underlying actual memory if no other code has open
-   * handles to this memory.
-   *
-   * @param memory The memory object to be freed.
-   */
-  void (*ANeuralNetworksMemory_free)(ANeuralNetworksMemory* memory);
-
-  /**
-   * Create an empty {@link ANeuralNetworksModel}.
-   *
-   * <p>This only creates the object. Computation is performed once
-   * {@link ANeuralNetworksExecution_startCompute} is invoked.
-   *
-   * The model should be constructed with calls to
-   * {@link ANeuralNetworksModel_addOperation} and
-   * {@link ANeuralNetworksModel_addOperand}
-   *
-   * <p>{@link ANeuralNetworksModel_finish} should be called once the model
-   * has been fully constructed.</p>
-   *
-   * <p>{@link ANeuralNetworksModel_free} should be called once the model
-   * is no longer needed.</p>
-   *
-   * @param model The {@link ANeuralNetworksModel} to be created.
-   *              Set to NULL if unsuccessful.
-   *
-   * @return ANEURALNETWORKS_NO_ERROR if successful.
-   */
-  int (*ANeuralNetworksModel_create)(ANeuralNetworksModel** model);
-
-  /**
-   * Destroy a model.
-   *
-   * The model need not have been finished by a call to
-   * {@link ANeuralNetworksModel_finish}.
-   *
-   * See {@link ANeuralNetworksModel} for information on multithreaded usage.
-   *
-   * @param model The model to be destroyed. Passing NULL is acceptable and
-   *              results in no operation.
-   */
-  void (*ANeuralNetworksModel_free)(ANeuralNetworksModel* model);
-
-  /**
-   * Indicate that we have finished modifying a model. Required before
-   * calling {@link ANeuralNetworksCompilation_compile}.
-   *
-   * An application is responsible to make sure that no other thread uses
-   * the model at the same time.
-   *
-   * See {@link ANeuralNetworksModel} for information on multithreaded usage.
-   *
-   * @param model The model to be finished.
-   *
-   * @return ANEURALNETWORKS_NO_ERROR if successful.
-   */
-  int (*ANeuralNetworksModel_finish)(ANeuralNetworksModel* model);
-
-  /**
-   * Add an operand to a model.
-   *
-   * The order in which the operands are added is important. The first one added
-   * to a model will have the index value 0, the second 1, etc. These indexes
-   * are used as operand identifiers in
-   * {@link ANeuralNetworksModel_addOperation},
-   * {@link ANeuralNetworksExecution_setInput},
-   * {@link ANeuralNetworksExecution_setInputFromMemory},
-   * {@link ANeuralNetworksExecution_setOutput},
-   * {@link ANeuralNetworksExecution_setOutputFromMemory} and
-   * {@link ANeuralNetworksExecution_setOperandValue}.
-   *
-   * To build a model that can accommodate inputs of various sizes, as you may
-   * want to do for a CNN, set the size of the dimensions that will vary at run
-   * time to 0. If you do so, provide the full dimensions when calling
-   * {@link ANeuralNetworksExecution_setInput} or {@link
-   * ANeuralNetworksExecution_setInputFromMemory}.
-   *
-   * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
-   * been called will return an error.
-   *
-   * See {@link ANeuralNetworksModel} for information on multithreaded usage.
-   *
-   * @param model The model to be modified.
-   * @param type The {@link ANeuralNetworksOperandType} that describes the shape
-   * of the operand.
-   *
-   * @return ANEURALNETWORKS_NO_ERROR if successful.
-   */
-  int (*ANeuralNetworksModel_addOperand)(
-      ANeuralNetworksModel* model, const ANeuralNetworksOperandType* type);
-
-  /**
-   * Sets an operand to a constant value.
-   *
-   * For scalar values, the content of buffer is copied into the model.
-   *
-   * For tensor values, a pointer to the buffer is stored within the model.
-   * The application is responsible for not changing the content of this region
-   * until all executions using this model have completed. As the data may
-   * be copied during processing, modifying the data after this call yields
-   * undefined results.
-   *
-   * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
-   * been called will return an error.
-   *
-   * See {@link ANeuralNetworksModel} for information on multithreaded usage.
-   *
-   * @param model The model to be modified.
-   * @param index The index of the model operand we're setting.
-   * @param buffer A pointer to the data to use.
-   * @param length The size in bytes of the data value.
-   *
-   * @return ANEURALNETWORKS_NO_ERROR if successful.
-   */
-  int (*ANeuralNetworksModel_setOperandValue)(ANeuralNetworksModel* model,
-                                              int32_t index, const void* buffer,
-                                              size_t length);
-
-  /**
-   * Sets an operand to a value stored in a memory object.
-   *
-   * The content of the memory is not copied. A reference to that memory is
-   * stored inside the model. The application is responsible for not changing
-   * the content of the memory region until all executions using this model have
-   * completed.
-   * As the data may be copied during processing, modifying the data after this
-   * call yields undefined results.
-   *
-   * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
-   * been called will return an error.
-   *
-   * See {@link ANeuralNetworksModel} for information on multithreaded usage.
-   *
-   * @param model The model to be modified.
-   * @param index The index of the model operand we're setting.
-   * @param buffer A pointer to the data to use.
-   * @param memory The memory containing the data.
-   * @param offset This specifies the location of the data within the memory.
-   *               The offset is in bytes from the start of memory.
-   * @param length The size in bytes of the data value.
-   *
-   * @return ANEURALNETWORKS_NO_ERROR if successful.
-   */
-  int (*ANeuralNetworksModel_setOperandValueFromMemory)(
-      ANeuralNetworksModel* model, int32_t index,
-      const ANeuralNetworksMemory* memory, size_t offset, size_t length);
-
-  /**
-   * Add an operation to a model.
-   *
-   * @param model The model to be modified.
-   * @param type The type of the operation.
-   * @param inputCount The number of entries in the inputs array.
-   * @param inputs An array of indexes identifying each operand.
-   * @param outputCount The number of entries in the outputs array.
-   * @param outputs An array of indexes identifying each operand.
-   *
-   * The operands specified by inputs and outputs must have been
-   * previously added by calls to {@link ANeuralNetworksModel_addOperand}.
-   *
-   * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
-   * been called will return an error.
-   *
-   * See {@link ANeuralNetworksModel} for information on multithreaded usage.
-   *
-   * @return ANEURALNETWORKS_NO_ERROR if successful.
-   */
-  int (*ANeuralNetworksModel_addOperation)(ANeuralNetworksModel* model,
-                                           ANeuralNetworksOperationType type,
-                                           uint32_t inputCount,
-                                           const uint32_t* inputs,
-                                           uint32_t outputCount,
-                                           const uint32_t* outputs);
-
-  /**
-   * Specifies which operands will be the model's inputs and outputs.
-   *
-   * An operand cannot be used for both input and output. Doing so will
-   * return an error.
-   *
-   * @param model The model to be modified.
-   * @param inputCount The number of entries in the inputs array.
-   * @param inputs An array of indexes identifying the input operands.
-   * @param outputCount The number of entries in the outputs array.
-   * @param outputs An array of indexes identifying the output operands.
-   *
-   * The operands specified by inputs and outputs must have been
-   * previously added by calls to {@link ANeuralNetworksModel_addOperand}.
-   *
-   * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
-   * been called will return an error.
-   *
-   * See {@link ANeuralNetworksModel} for information on multithreaded usage.
-   *
-   */
-  int (*ANeuralNetworksModel_identifyInputsAndOutputs)(
-      ANeuralNetworksModel* model, uint32_t inputCount, const uint32_t* inputs,
-      uint32_t outputCount, const uint32_t* outputs);
-
-  /**
-   * Specifies whether {@link ANEURALNETWORKS_TENSOR_FLOAT32} is allowed to be
-   * calculated with range and/or precision as low as that of the
-   * IEEE 754 16-bit floating-point format. By default,
-   * {@link ANEURALNETWORKS_TENSOR_FLOAT32} must be calculated using at least
-   * the range and precision of the IEEE 754 32-bit floating-point format.
-   *
-   * @param model The model to be modified.
-   * @param allow 'true' indicates {@link ANEURALNETWORKS_TENSOR_FLOAT32} may be
-   *              calculated with range and/or precision as low as that of the
-   *              IEEE 754 16-bit floating point format. 'false' indicates
-   *              {@link ANEURALNETWORKS_TENSOR_FLOAT32} must be calculated
-   *              using at least the range and precision of the IEEE 754 32-bit
-   *              floating point format.
-   *
-   * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
-   * been called will return an error.
-   *
-   * Available since API level 28.
-   *
-   * See {@link ANeuralNetworksModel} for information on multithreaded usage.
-   */
-  int (*ANeuralNetworksModel_relaxComputationFloat32toFloat16)(
-      ANeuralNetworksModel* model, bool allow);
-
-  /**
-   * Create a {@link ANeuralNetworksCompilation} to compile the given model.
-   * This only creates the object. Compilation is only performed once
-   * {@link ANeuralNetworksCompilation_start} is invoked.
-   *
-   * <p>The provided model must outlive the compilation.</p>
-   *
-   * The model must already have been finished by a call to
-   * {@link ANeuralNetworksModel_finish}.
-   *
-   * See {@link ANeuralNetworksCompilation} for information on multithreaded
-   * usage.
-   *
-   * @param model The {@link ANeuralNetworksModel} to be compiled.
-   * @param compilation The newly created object or NULL if unsuccessful.
-   *
-   * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA
-   *         if the model is invalid.
-   */
-  int (*ANeuralNetworksCompilation_create)(
-      ANeuralNetworksModel* model, ANeuralNetworksCompilation** compilation);
-
-  /**
-   * Destroy a compilation.
-   *
-   * <p>If called on a compilation for which
-   * {@link ANeuralNetworksCompilation_start} has been called, the
-   * function will return immediately but will mark the compilation to be
-   * deleted once the compilation completes. The
-   * {@link ANeuralNetworksCompilation_wait} will return ERROR_DELETED.
-   *
-   * See {@link ANeuralNetworksCompilation} for information on multithreaded
-   * usage.
-   *
-   * @param compilation The compilation to be destroyed. Passing NULL is
-   * acceptable and results in no operation.
-   */
-  void (*ANeuralNetworksCompilation_free)(
-      ANeuralNetworksCompilation* compilation);
-
-  /**
-   * Sets the execution preference.
-   *
-   * <p>Provides guidance to the runtime when trade-offs are possible.</p>
-   *
-   * See {@link ANeuralNetworksCompilation} for information on multithreaded
-   * usage.
-   *
-   * @param compilation The compilation to be modified.
-   * @param preference Either {@link PREFER_LOW_POWER},
-   *                  {@link PREFER_SINGLE_FAST_ANSWER}, or
-   *                  {@link PREFER_SUSTAINED_SPEED}.
-   *
-   * @return ANEURALNETWORKS_NO_ERROR if successful.
-   */
-  int (*ANeuralNetworksCompilation_setPreference)(
-      ANeuralNetworksCompilation* compilation, int32_t preference);
-
-  /**
-   * Waits until the compilation completes.
-   *
-   * More than one thread can wait on a compilation. When the compilation
-   * completes, all threads will be released.
-   *
-   * See {@link ANeuralNetworksCompilation} for information on multithreaded
-   * usage.
-   *
-   * @return ANEURALNETWORKS_NO_ERROR if the compilation completed normally.
-   */
-  int (*ANeuralNetworksCompilation_finish)(
-      ANeuralNetworksCompilation* compilation);
-
-  /**
-   * Create a {@link ANeuralNetworksExecution} to apply the given compilation.
-   * This only creates the object. Computation is only performed once
-   * {@link ANeuralNetworksExecution_startCompute} is invoked.
-   *
-   * <p>The provided compilation must outlive the execution.</p>
-   *
-   * See {@link ANeuralNetworksExecution} for information on multithreaded
-   * usage.
-   *
-   * @param compilation The {@link ANeuralNetworksCompilation} to be evaluated.
-   * @param execution The newly created object or NULL if unsuccessful.
-   *
-   * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA
-   *         if the compilation is invalid.
-   */
-  int (*ANeuralNetworksExecution_create)(
-      ANeuralNetworksCompilation* compilation,
-      ANeuralNetworksExecution** execution);
-
-  /**
-   * Destroy an execution.
-   *
-   * <p>If called on an execution for which
-   * {@link ANeuralNetworksExecution_startCompute} has been called, the
-   * function will return immediately but will mark the execution to be deleted
-   * once the computation completes.   The {link ANeuralNetworksExecution_wait}
-   * will return ANEURALNETWORKS_ERROR_DELETED.
-   *
-   * See {@link ANeuralNetworksExecution} for information on multithreaded
-   * usage.
-   *
-   * @param execution The execution to be destroyed. Passing NULL is acceptable
-   * and results in no operation.
-   */
-  void (*ANeuralNetworksExecution_free)(ANeuralNetworksExecution* execution);
-
-  /**
-   * Associate a user buffer with an input of the model of the
-   * {@link ANeuralNetworksExecution}.
-   *
-   * <p>The provided buffer must outlive the execution.</p>
-   *
-   * See {@link ANeuralNetworksExecution} for information on multithreaded
-   * usage.
-   *
-   * @param execution The execution to be modified.
-   * @param index The index of the input argument we are setting. It is
-   *              an index into the lists passed to
-   *              {@link ANeuralNetworksModel_identifyInputsAndOutputs}. It is
-   *              not the index associated with {@link
-   * ANeuralNetworksModel_addOperand}.
-   * @param type The type of the operand. This should be used to specify the
-   *             dimensions that were set to 0 when the operand was added to the
-   *             model. All other properties of the type must be the same as
-   *             specified in the model. If the type is the same as specified
-   *             when the model was built, NULL can be passed.
-   * @param buffer The buffer containing the data.
-   * @param length The length in bytes of the buffer.
-   *
-   * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA if
-   * the name is not recognized or the buffer is too small for the input.
-   */
-  int (*ANeuralNetworksExecution_setInput)(
-      ANeuralNetworksExecution* execution, int32_t index,
-      const ANeuralNetworksOperandType* type, const void* buffer,
-      size_t length);
-
-  /**
-   * Associate part of a memory object with an input of the model of the
-   * {@link ANeuralNetworksExecution}.
-   *
-   * <p>The provided memory must outlive the execution.</p>
-   *
-   * See {@link ANeuralNetworksExecution} for information on multithreaded
-   * usage.
-   *
-   * @param execution The execution to be modified.
-   * @param index The index of the input argument we are setting. It is
-   *              an index into the lists passed to
-   *              {@link ANeuralNetworksModel_identifyInputsAndOutputs}. It is
-   *              not the index associated with {@link
-   * ANeuralNetworksModel_addOperand}.
-   * @param type The type of the operand. This can be used to specify the
-   *             dimensions that were set to 0 when the operand was added to the
-   *             model. All other values must be the same as specified in the
-   *             model. If the type is the same as specified when the model
-   *             was built, NULL can be passed.
-   * @param memory The memory containing the data.
-   * @param offset This specifies the location of the data within the memory.
-   *               The offset is in bytes from the start of memory.
-   * @param length The size in bytes of the data value.
-   *
-   * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA if
-   * the name is not recognized or the buffer is too small for the input.
-   */
-  int (*ANeuralNetworksExecution_setInputFromMemory)(
-      ANeuralNetworksExecution* execution, int32_t index,
-      const ANeuralNetworksOperandType* type,
-      const ANeuralNetworksMemory* memory, size_t offset, size_t length);
-
-  /**
-   * Associate a user buffer with an output of the model of the
-   * {@link ANeuralNetworksExecution}.
-   *
-   * <p>The provided buffer must outlive the execution.</p>
-   *
-   * See {@link ANeuralNetworksExecution} for information on multithreaded
-   * usage.
-   *
-   * @param execution The execution to be modified.
-   * @param index The index of the output argument we are setting. It is
-   *              an index into the lists passed to
-   *              {@link ANeuralNetworksModel_identifyInputsAndOutputs}. It is
-   *              not the index associated with {@link
-   * ANeuralNetworksModel_addOperand}.
-   * @param type The type of the operand. This can be used to specify the
-   *             dimensions that were set to 0 when the operand was added to the
-   *             model. All other values must be the same as specified in the
-   *             model. If the type is the same as specified when the model
-   *             was built, NULL can be passed.
-   * @param buffer The buffer where the data is to be written.
-   * @param length The length in bytes of the buffer.
-   *
-   * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA if
-   * the name is not recognized or the buffer is too small for the output.
-   */
-  int (*ANeuralNetworksExecution_setOutput)(
-      ANeuralNetworksExecution* execution, int32_t index,
-      const ANeuralNetworksOperandType* type, void* buffer, size_t length);
-
-  /**
-   * Associate part of a memory object with an output of the model of the
-   * {@link ANeuralNetworksExecution}.
-   *
-   * <p>The provided memory must outlive the execution.</p>
-   *
-   * See {@link ANeuralNetworksExecution} for information on multithreaded
-   * usage.
-   *
-   * @param execution The execution to be modified.
-   * @param index The index of the output argument we are setting. It is
-   *              an index into the lists passed to
-   *              {@link ANeuralNetworksModel_identifyInputsAndOutputs}. It is
-   *              not the index associated with {@link
-   * ANeuralNetworksModel_addOperand}.
-   * @param type The type of the operand. This can be used to specify the
-   *             dimensions that were set to 0 when the operand was added to the
-   *             model. All other values must be the same as specified in the
-   *             model. If the type is the same as specified when the model
-   *             was built, NULL can be passed.
-   * @param memory The memory where the data is to be stored.
-   * @param offset This specifies the location of the data within the memory.
-   *               The offset is in bytes from the start of memory.
-   * @param length The length in bytes of the data value.
-   *
-   * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA if
-   * the name is not recognized or the buffer is too small for the output.
-   */
-  int (*ANeuralNetworksExecution_setOutputFromMemory)(
-      ANeuralNetworksExecution* execution, int32_t index,
-      const ANeuralNetworksOperandType* type,
-      const ANeuralNetworksMemory* memory, size_t offset, size_t length);
-
-  /**
-   * Schedule evaluation of the execution.
-   *
-   * <p>Schedules evaluation of the execution. Once the model has been
-   * applied and the outputs are ready to be consumed, the execution will be
-   * signaled. Use {@link ANeuralNetworksExecution_wait} to wait for that
-   * signal.
-   * </p>
-   *
-   * Multiple executions can be scheduled and evaluated concurrently, and
-   * compilations can be performed concurrently with executions. The runtime
-   * makes no guarantee on the ordering of the completion of compilations and
-   * executions. If it's important to the application, the application should
-   * enforce the ordering by using {@link ANeuralNetworksCompilation_wait} and
-   * {@link ANeuralNetworksExecution_wait}.
-   *
-   * ANeuralNetworksExecution_wait must be called to recuperate the resources
-   * used by the execution.
-   *
-   * See {@link ANeuralNetworksExecution} for information on multithreaded
-   * usage.
-   *
-   * @param execution The execution to be scheduled and executed.
-   *
-   * @return ANEURALNETWORKS_NO_ERROR if successful.
-   */
-  int (*ANeuralNetworksExecution_startCompute)(
-      ANeuralNetworksExecution* execution, ANeuralNetworksEvent** event);
-
-  /**
-   * Waits until the execution completes.
-   *
-   * More than one thread can wait on an event. When the execution completes,
-   * all threads will be released.
-   *
-   * See {@link ANeuralNetworksExecution} for information on multithreaded
-   * usage.
-   *
-   * @return ANEURALNETWORKS_NO_ERROR if the execution completed normally.
-   */
-  int (*ANeuralNetworksEvent_wait)(ANeuralNetworksEvent* event);
+/**
+ * Delete a memory object.
+ *
+ * Destroys the object used by the run time to keep track of the memory.
+ * This will free the underlying actual memory if no other code has open
+ * handles to this memory.
+ *
+ * @param memory The memory object to be freed.
+ */
+inline void ANeuralNetworksMemory_free(ANeuralNetworksMemory* memory) {
+  LOAD_FUNCTION(ANeuralNetworksMemory_free);
+  EXECUTE_FUNCTION(memory);
+}
 
-  /**
-   * Destroys the event.
-   *
-   * See {@link ANeuralNetworksExecution} for information on multithreaded
-   * usage.
-   */
-  void (*ANeuralNetworksEvent_free)(ANeuralNetworksEvent* event);
+/**
+ * Create an empty {@link ANeuralNetworksModel}.
+ *
+ * <p>This only creates the object. Computation is performed once
+ * {@link ANeuralNetworksExecution_startCompute} is invoked.
+ *
+ * The model should be constructed with calls to
+ * {@link ANeuralNetworksModel_addOperation} and
+ * {@link ANeuralNetworksModel_addOperand}
+ *
+ * <p>{@link ANeuralNetworksModel_finish} should be called once the model
+ * has been fully constructed.</p>
+ *
+ * <p>{@link ANeuralNetworksModel_free} should be called once the model
+ * is no longer needed.</p>
+ *
+ * @param model The {@link ANeuralNetworksModel} to be created.
+ *              Set to NULL if unsuccessful.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if successful.
+ */
+inline int ANeuralNetworksModel_create(ANeuralNetworksModel** model) {
+  LOAD_FUNCTION(ANeuralNetworksModel_create);
+  EXECUTE_FUNCTION_RETURN(model);
+}
 
-  // ASharedMemory_create was added in Android 8.0, so safe to use with NNAPI
-  // which was added in 8.1.
-  int (*ASharedMemory_create)(const char* name, size_t size);
+/**
+ * Destroy a model.
+ *
+ * The model need not have been finished by a call to
+ * {@link ANeuralNetworksModel_finish}.
+ *
+ * See {@link ANeuralNetworksModel} for information on multithreaded usage.
+ *
+ * @param model The model to be destroyed. Passing NULL is acceptable and
+ *              results in no operation.
+ */
+inline void ANeuralNetworksModel_free(ANeuralNetworksModel* model) {
+  LOAD_FUNCTION(ANeuralNetworksModel_free);
+  EXECUTE_FUNCTION(model);
+}
 
-  /**/
-};
+/**
+ * Indicate that we have finished modifying a model. Required before
+ * calling {@link ANeuralNetworksCompilation_compile}.
+ *
+ * An application is responsible to make sure that no other thread uses
+ * the model at the same time.
+ *
+ * See {@link ANeuralNetworksModel} for information on multithreaded usage.
+ *
+ * @param model The model to be finished.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if successful.
+ */
+inline int ANeuralNetworksModel_finish(ANeuralNetworksModel* model) {
+  LOAD_FUNCTION(ANeuralNetworksModel_finish);
+  EXECUTE_FUNCTION_RETURN(model);
+}
+
+/**
+ * Add an operand to a model.
+ *
+ * The order in which the operands are added is important. The first one added
+ * to a model will have the index value 0, the second 1, etc. These indexes are
+ * used as operand identifiers in {@link ANeuralNetworksModel_addOperation},
+ * {@link ANeuralNetworksExecution_setInput},
+ * {@link ANeuralNetworksExecution_setInputFromMemory},
+ * {@link ANeuralNetworksExecution_setOutput},
+ * {@link ANeuralNetworksExecution_setOutputFromMemory} and
+ * {@link ANeuralNetworksExecution_setOperandValue}.
+ *
+ * To build a model that can accommodate inputs of various sizes, as you may
+ * want to do for a CNN, set the size of the dimensions that will vary at run
+ * time to 0. If you do so, provide the full dimensions when calling
+ * {@link ANeuralNetworksExecution_setInput} or {@link
+ * ANeuralNetworksExecution_setInputFromMemory}.
+ *
+ * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
+ * been called will return an error.
+ *
+ * See {@link ANeuralNetworksModel} for information on multithreaded usage.
+ *
+ * @param model The model to be modified.
+ * @param type The {@link ANeuralNetworksOperandType} that describes the shape
+ * of the operand.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if successful.
+ */
+inline int ANeuralNetworksModel_addOperand(
+    ANeuralNetworksModel* model, const ANeuralNetworksOperandType* type) {
+  LOAD_FUNCTION(ANeuralNetworksModel_addOperand);
+  EXECUTE_FUNCTION_RETURN(model, type);
+}
+
+/**
+ * Sets an operand to a constant value.
+ *
+ * For scalar values, the content of buffer is copied into the model.
+ *
+ * For tensor values, a pointer to the buffer is stored within the model.
+ * The application is responsible for not changing the content of this region
+ * until all executions using this model have completed. As the data may
+ * be copied during processing, modifying the data after this call yields
+ * undefined results.
+ *
+ * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
+ * been called will return an error.
+ *
+ * See {@link ANeuralNetworksModel} for information on multithreaded usage.
+ *
+ * @param model The model to be modified.
+ * @param index The index of the model operand we're setting.
+ * @param buffer A pointer to the data to use.
+ * @param length The size in bytes of the data value.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if successful.
+ */
+inline int ANeuralNetworksModel_setOperandValue(ANeuralNetworksModel* model,
+                                                int32_t index,
+                                                const void* buffer,
+                                                size_t length) {
+  LOAD_FUNCTION(ANeuralNetworksModel_setOperandValue);
+  EXECUTE_FUNCTION_RETURN(model, index, buffer, length);
+}
 
 /**
- * Load the NNAPI implementation from the shared libraries.
- * The NnApi structure is filled with all the pointers. If one function doesn't
- * exist, a null pointer is stored.
+ * Sets an operand to a value stored in a memory object.
+ *
+ * The content of the memory is not copied. A reference to that memory is stored
+ * inside the model. The application is responsible for not changing the content
+ * of the memory region until all executions using this model have completed.
+ * As the data may be copied during processing, modifying the data after this
+ * call yields undefined results.
+ *
+ * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
+ * been called will return an error.
+ *
+ * See {@link ANeuralNetworksModel} for information on multithreaded usage.
+ *
+ * @param model The model to be modified.
+ * @param index The index of the model operand we're setting.
+ * @param buffer A pointer to the data to use.
+ * @param memory The memory containing the data.
+ * @param offset This specifies the location of the data within the memory.
+ *               The offset is in bytes from the start of memory.
+ * @param length The size in bytes of the data value.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if successful.
  */
-const NnApi* NnApiImplementation();
+inline int ANeuralNetworksModel_setOperandValueFromMemory(
+    ANeuralNetworksModel* model, int32_t index,
+    const ANeuralNetworksMemory* memory, size_t offset, size_t length) {
+  LOAD_FUNCTION(ANeuralNetworksModel_setOperandValueFromMemory);
+  EXECUTE_FUNCTION_RETURN(model, index, memory, offset, length);
+}
+
+/**
+ * Add an operation to a model.
+ *
+ * @param model The model to be modified.
+ * @param type The type of the operation.
+ * @param inputCount The number of entries in the inputs array.
+ * @param inputs An array of indexes identifying each operand.
+ * @param outputCount The number of entries in the outputs array.
+ * @param outputs An array of indexes identifying each operand.
+ *
+ * The operands specified by inputs and outputs must have been
+ * previously added by calls to {@link ANeuralNetworksModel_addOperand}.
+ *
+ * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
+ * been called will return an error.
+ *
+ * See {@link ANeuralNetworksModel} for information on multithreaded usage.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if successful.
+ */
+inline int ANeuralNetworksModel_addOperation(ANeuralNetworksModel* model,
+                                             ANeuralNetworksOperationType type,
+                                             uint32_t inputCount,
+                                             const uint32_t* inputs,
+                                             uint32_t outputCount,
+                                             const uint32_t* outputs) {
+  LOAD_FUNCTION(ANeuralNetworksModel_addOperation);
+  EXECUTE_FUNCTION_RETURN(model, type, inputCount, inputs, outputCount,
+                          outputs);
+}
+
+/**
+ * Specifies which operands will be the model's inputs and outputs.
+ *
+ * An operand cannot be used for both input and output. Doing so will
+ * return an error.
+ *
+ * @param model The model to be modified.
+ * @param inputCount The number of entries in the inputs array.
+ * @param inputs An array of indexes identifying the input operands.
+ * @param outputCount The number of entries in the outputs array.
+ * @param outputs An array of indexes identifying the output operands.
+ *
+ * The operands specified by inputs and outputs must have been
+ * previously added by calls to {@link ANeuralNetworksModel_addOperand}.
+ *
+ * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
+ * been called will return an error.
+ *
+ * See {@link ANeuralNetworksModel} for information on multithreaded usage.
+ *
+ */
+inline int ANeuralNetworksModel_identifyInputsAndOutputs(
+    ANeuralNetworksModel* model, uint32_t inputCount, const uint32_t* inputs,
+    uint32_t outputCount, const uint32_t* outputs) {
+  LOAD_FUNCTION(ANeuralNetworksModel_identifyInputsAndOutputs);
+  EXECUTE_FUNCTION_RETURN(model, inputCount, inputs, outputCount, outputs);
+}
+
+/**
+ * Specifies whether {@link ANEURALNETWORKS_TENSOR_FLOAT32} is allowed to be
+ * calculated with range and/or precision as low as that of the IEEE 754 16-bit
+ * floating-point format. By default, {@link ANEURALNETWORKS_TENSOR_FLOAT32}
+ * must be calculated using at least the range and precision of the IEEE 754
+ * 32-bit floating-point format.
+ *
+ * @param model The model to be modified.
+ * @param allow 'true' indicates {@link ANEURALNETWORKS_TENSOR_FLOAT32} may be
+ *              calculated with range and/or precision as low as that of the
+ *              IEEE 754 16-bit floating point format. 'false' indicates
+ *              {@link ANEURALNETWORKS_TENSOR_FLOAT32} must be calculated using
+ *              at least the range and precision of the IEEE 754 32-bit floating
+ *              point format.
+ *
+ * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
+ * been called will return an error.
+ *
+ * Available since API level 28.
+ *
+ * See {@link ANeuralNetworksModel} for information on multithreaded usage.
+ */
+inline int ANeuralNetworksModel_relaxComputationFloat32toFloat16(
+    ANeuralNetworksModel* model, bool allow) {
+  LOAD_FUNCTION(ANeuralNetworksModel_relaxComputationFloat32toFloat16);
+  EXECUTE_FUNCTION_RETURN(model, allow);
+}
+
+/**
+ * Create a {@link ANeuralNetworksCompilation} to compile the given model.
+ * This only creates the object. Compilation is only performed once
+ * {@link ANeuralNetworksCompilation_start} is invoked.
+ *
+ * <p>The provided model must outlive the compilation.</p>
+ *
+ * The model must already have been finished by a call to
+ * {@link ANeuralNetworksModel_finish}.
+ *
+ * See {@link ANeuralNetworksCompilation} for information on multithreaded
+ * usage.
+ *
+ * @param model The {@link ANeuralNetworksModel} to be compiled.
+ * @param compilation The newly created object or NULL if unsuccessful.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA
+ *         if the model is invalid.
+ */
+inline int ANeuralNetworksCompilation_create(
+    ANeuralNetworksModel* model, ANeuralNetworksCompilation** compilation) {
+  LOAD_FUNCTION(ANeuralNetworksCompilation_create);
+  EXECUTE_FUNCTION_RETURN(model, compilation);
+}
+
+/**
+ * Destroy a compilation.
+ *
+ * <p>If called on a compilation for which
+ * {@link ANeuralNetworksCompilation_start} has been called, the
+ * function will return immediately but will mark the compilation to be deleted
+ * once the compilation completes. The {@link ANeuralNetworksCompilation_wait}
+ * will return ERROR_DELETED.
+ *
+ * See {@link ANeuralNetworksCompilation} for information on multithreaded
+ * usage.
+ *
+ * @param compilation The compilation to be destroyed. Passing NULL is
+ * acceptable and results in no operation.
+ */
+inline void ANeuralNetworksCompilation_free(
+    ANeuralNetworksCompilation* compilation) {
+  LOAD_FUNCTION(ANeuralNetworksCompilation_free);
+  EXECUTE_FUNCTION(compilation);
+}
+
+/**
+ * Sets the execution preference.
+ *
+ * <p>Provides guidance to the runtime when trade-offs are possible.</p>
+ *
+ * See {@link ANeuralNetworksCompilation} for information on multithreaded
+ * usage.
+ *
+ * @param compilation The compilation to be modified.
+ * @param preference Either {@link PREFER_LOW_POWER},
+ *                  {@link PREFER_SINGLE_FAST_ANSWER}, or
+ *                  {@link PREFER_SUSTAINED_SPEED}.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if successful.
+ */
+inline int ANeuralNetworksCompilation_setPreference(
+    ANeuralNetworksCompilation* compilation, int32_t preference) {
+  LOAD_FUNCTION(ANeuralNetworksCompilation_setPreference);
+  EXECUTE_FUNCTION_RETURN(compilation, preference);
+}
+
+/**
+ * Waits until the compilation completes.
+ *
+ * More than one thread can wait on a compilation. When the compilation
+ * completes, all threads will be released.
+ *
+ * See {@link ANeuralNetworksCompilation} for information on multithreaded
+ * usage.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if the compilation completed normally.
+ */
+inline int ANeuralNetworksCompilation_finish(
+    ANeuralNetworksCompilation* compilation) {
+  LOAD_FUNCTION(ANeuralNetworksCompilation_finish);
+  EXECUTE_FUNCTION_RETURN(compilation);
+}
+/**
+ * Create a {@link ANeuralNetworksExecution} to apply the given compilation.
+ * This only creates the object. Computation is only performed once
+ * {@link ANeuralNetworksExecution_startCompute} is invoked.
+ *
+ * <p>The provided compilation must outlive the execution.</p>
+ *
+ * See {@link ANeuralNetworksExecution} for information on multithreaded usage.
+ *
+ * @param compilation The {@link ANeuralNetworksCompilation} to be evaluated.
+ * @param execution The newly created object or NULL if unsuccessful.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA
+ *         if the compilation is invalid.
+ */
+inline int ANeuralNetworksExecution_create(
+    ANeuralNetworksCompilation* compilation,
+    ANeuralNetworksExecution** execution) {
+  LOAD_FUNCTION(ANeuralNetworksExecution_create);
+  EXECUTE_FUNCTION_RETURN(compilation, execution);
+}
+
+/**
+ * Destroy an execution.
+ *
+ * <p>If called on an execution for which
+ * {@link ANeuralNetworksExecution_startCompute} has been called, the
+ * function will return immediately but will mark the execution to be deleted
+ * once the computation completes.   The {link ANeuralNetworksExecution_wait}
+ * will return ANEURALNETWORKS_ERROR_DELETED.
+ *
+ * See {@link ANeuralNetworksExecution} for information on multithreaded usage.
+ *
+ * @param execution The execution to be destroyed. Passing NULL is acceptable
+ * and results in no operation.
+ */
+inline void ANeuralNetworksExecution_free(ANeuralNetworksExecution* execution) {
+  LOAD_FUNCTION(ANeuralNetworksExecution_free);
+  EXECUTE_FUNCTION(execution);
+}
+
+/**
+ * Associate a user buffer with an input of the model of the
+ * {@link ANeuralNetworksExecution}.
+ *
+ * <p>The provided buffer must outlive the execution.</p>
+ *
+ * See {@link ANeuralNetworksExecution} for information on multithreaded usage.
+ *
+ * @param execution The execution to be modified.
+ * @param index The index of the input argument we are setting. It is
+ *              an index into the lists passed to
+ *              {@link ANeuralNetworksModel_identifyInputsAndOutputs}. It is not
+ *              the index associated with {@link
+ * ANeuralNetworksModel_addOperand}.
+ * @param type The type of the operand. This should be used to specify the
+ *             dimensions that were set to 0 when the operand was added to the
+ *             model. All other properties of the type must be the same as
+ *             specified in the model. If the type is the same as specified
+ *             when the model was built, NULL can be passed.
+ * @param buffer The buffer containing the data.
+ * @param length The length in bytes of the buffer.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA if
+ * the name is not recognized or the buffer is too small for the input.
+ */
+inline int ANeuralNetworksExecution_setInput(
+    ANeuralNetworksExecution* execution, int32_t index,
+    const ANeuralNetworksOperandType* type, const void* buffer, size_t length) {
+  LOAD_FUNCTION(ANeuralNetworksExecution_setInput);
+  EXECUTE_FUNCTION_RETURN(execution, index, type, buffer, length);
+}
+
+/**
+ * Associate part of a memory object with an input of the model of the
+ * {@link ANeuralNetworksExecution}.
+ *
+ * <p>The provided memory must outlive the execution.</p>
+ *
+ * See {@link ANeuralNetworksExecution} for information on multithreaded usage.
+ *
+ * @param execution The execution to be modified.
+ * @param index The index of the input argument we are setting. It is
+ *              an index into the lists passed to
+ *              {@link ANeuralNetworksModel_identifyInputsAndOutputs}. It is not
+ *              the index associated with {@link
+ * ANeuralNetworksModel_addOperand}.
+ * @param type The type of the operand. This can be used to specify the
+ *             dimensions that were set to 0 when the operand was added to the
+ *             model. All other values must be the same as specified in the
+ *             model. If the type is the same as specified when the model
+ *             was built, NULL can be passed.
+ * @param memory The memory containing the data.
+ * @param offset This specifies the location of the data within the memory.
+ *               The offset is in bytes from the start of memory.
+ * @param length The size in bytes of the data value.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA if
+ * the name is not recognized or the buffer is too small for the input.
+ */
+inline int ANeuralNetworksExecution_setInputFromMemory(
+    ANeuralNetworksExecution* execution, int32_t index,
+    const ANeuralNetworksOperandType* type, const ANeuralNetworksMemory* memory,
+    size_t offset, size_t length) {
+  LOAD_FUNCTION(ANeuralNetworksExecution_setInputFromMemory);
+  EXECUTE_FUNCTION_RETURN(execution, index, type, memory, offset, length);
+}
+
+/**
+ * Associate a user buffer with an output of the model of the
+ * {@link ANeuralNetworksExecution}.
+ *
+ * <p>The provided buffer must outlive the execution.</p>
+ *
+ * See {@link ANeuralNetworksExecution} for information on multithreaded usage.
+ *
+ * @param execution The execution to be modified.
+ * @param index The index of the output argument we are setting. It is
+ *              an index into the lists passed to
+ *              {@link ANeuralNetworksModel_identifyInputsAndOutputs}. It is not
+ *              the index associated with {@link
+ * ANeuralNetworksModel_addOperand}.
+ * @param type The type of the operand. This can be used to specify the
+ *             dimensions that were set to 0 when the operand was added to the
+ *             model. All other values must be the same as specified in the
+ *             model. If the type is the same as specified when the model
+ *             was built, NULL can be passed.
+ * @param buffer The buffer where the data is to be written.
+ * @param length The length in bytes of the buffer.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA if
+ * the name is not recognized or the buffer is too small for the output.
+ */
+inline int ANeuralNetworksExecution_setOutput(
+    ANeuralNetworksExecution* execution, int32_t index,
+    const ANeuralNetworksOperandType* type, void* buffer, size_t length) {
+  LOAD_FUNCTION(ANeuralNetworksExecution_setOutput);
+  EXECUTE_FUNCTION_RETURN(execution, index, type, buffer, length);
+}
+
+/**
+ * Associate part of a memory object with an output of the model of the
+ * {@link ANeuralNetworksExecution}.
+ *
+ * <p>The provided memory must outlive the execution.</p>
+ *
+ * See {@link ANeuralNetworksExecution} for information on multithreaded usage.
+ *
+ * @param execution The execution to be modified.
+ * @param index The index of the output argument we are setting. It is
+ *              an index into the lists passed to
+ *              {@link ANeuralNetworksModel_identifyInputsAndOutputs}. It is not
+ *              the index associated with {@link
+ * ANeuralNetworksModel_addOperand}.
+ * @param type The type of the operand. This can be used to specify the
+ *             dimensions that were set to 0 when the operand was added to the
+ *             model. All other values must be the same as specified in the
+ *             model. If the type is the same as specified when the model
+ *             was built, NULL can be passed.
+ * @param memory The memory where the data is to be stored.
+ * @param offset This specifies the location of the data within the memory.
+ *               The offset is in bytes from the start of memory.
+ * @param length The length in bytes of the data value.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA if
+ * the name is not recognized or the buffer is too small for the output.
+ */
+inline int ANeuralNetworksExecution_setOutputFromMemory(
+    ANeuralNetworksExecution* execution, int32_t index,
+    const ANeuralNetworksOperandType* type, const ANeuralNetworksMemory* memory,
+    size_t offset, size_t length) {
+  LOAD_FUNCTION(ANeuralNetworksExecution_setOutputFromMemory);
+  EXECUTE_FUNCTION_RETURN(execution, index, type, memory, offset, length);
+}
+
+/**
+ * Schedule evaluation of the execution.
+ *
+ * <p>Schedules evaluation of the execution. Once the model has been
+ * applied and the outputs are ready to be consumed, the execution will be
+ * signaled. Use {@link ANeuralNetworksExecution_wait} to wait for that signal.
+ * </p>
+ *
+ * Multiple executions can be scheduled and evaluated concurrently, and
+ * compilations can be performed concurrently with executions. The runtime makes
+ * no guarantee on the ordering of the completion of compilations and
+ * executions. If it's important to the application, the application should
+ * enforce the ordering by using {@link ANeuralNetworksCompilation_wait} and
+ * {@link ANeuralNetworksExecution_wait}.
+ *
+ * ANeuralNetworksExecution_wait must be called to recuperate the resources used
+ * by the execution.
+ *
+ * See {@link ANeuralNetworksExecution} for information on multithreaded usage.
+ *
+ * @param execution The execution to be scheduled and executed.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if successful.
+ */
+inline int ANeuralNetworksExecution_startCompute(
+    ANeuralNetworksExecution* execution, ANeuralNetworksEvent** event) {
+  LOAD_FUNCTION(ANeuralNetworksExecution_startCompute);
+  EXECUTE_FUNCTION_RETURN(execution, event);
+}
+
+/**
+ * Waits until the execution completes.
+ *
+ * More than one thread can wait on an event. When the execution completes,
+ * all threads will be released.
+ *
+ * See {@link ANeuralNetworksExecution} for information on multithreaded usage.
+ *
+ * @return ANEURALNETWORKS_NO_ERROR if the execution completed normally.
+ */
+inline int ANeuralNetworksEvent_wait(ANeuralNetworksEvent* event) {
+  LOAD_FUNCTION(ANeuralNetworksEvent_wait);
+  EXECUTE_FUNCTION_RETURN(event);
+}
+
+/**
+ * Destroys the event.
+ *
+ * See {@link ANeuralNetworksExecution} for information on multithreaded usage.
+ */
+inline void ANeuralNetworksEvent_free(ANeuralNetworksEvent* event) {
+  LOAD_FUNCTION(ANeuralNetworksEvent_free);
+  EXECUTE_FUNCTION(event);
+}
+
+/**/
 
 #endif  // TENSORFLOW_LITE_NNAPI_NEURALNETWORKSSHIM_H_
diff --git a/tensorflow/lite/nnapi/nnapi_lib_test.cc b/tensorflow/lite/nnapi/nnapi_lib_test.cc
deleted file mode 100644
index aaf0ffaa8e..0000000000
--- a/tensorflow/lite/nnapi/nnapi_lib_test.cc
+++ /dev/null
@@ -1,90 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#include <gtest/gtest.h>
-#include "tensorflow/lite/nnapi/NeuralNetworksShim.h"
-
-namespace {
-
-TEST(NnapiLibTest, NnApiImplementation) {
-  const NnApi* nnapi = NnApiImplementation();
-  EXPECT_NE(nnapi, nullptr);
-#ifdef __ANDROID__
-  EXPECT_TRUE(nnapi->nnapi_exists);
-  EXPECT_GT(nnapi->android_sdk_version, 0);
-  EXPECT_NE(nnapi->ANeuralNetworksMemory_createFromFd, nullptr);
-  EXPECT_NE(nnapi->ANeuralNetworksMemory_free, nullptr);
-  EXPECT_NE(nnapi->ANeuralNetworksModel_create, nullptr);
-  EXPECT_NE(nnapi->ANeuralNetworksModel_free, nullptr);
-  EXPECT_NE(nnapi->ANeuralNetworksModel_finish, nullptr);
-  EXPECT_NE(nnapi->ANeuralNetworksModel_addOperand, nullptr);
-  EXPECT_NE(nnapi->ANeuralNetworksModel_setOperandValue, nullptr);
-  EXPECT_NE(nnapi->ANeuralNetworksModel_setOperandValueFromMemory, nullptr);
-  EXPECT_NE(nnapi->ANeuralNetworksModel_addOperation, nullptr);
-  EXPECT_NE(nnapi->ANeuralNetworksModel_identifyInputsAndOutputs, nullptr);
-  if (nnapi->android_sdk_version >= 28) {
-    // relaxComputationFloat32toFloat16 only available with Android 9.0 (P).
-    EXPECT_NE(nnapi->ANeuralNetworksModel_relaxComputationFloat32toFloat16,
-              nullptr);
-  } else {
-    EXPECT_EQ(nnapi->ANeuralNetworksModel_relaxComputationFloat32toFloat16,
-              nullptr);
-  }
-  EXPECT_NE(nnapi->ANeuralNetworksCompilation_create, nullptr);
-  EXPECT_NE(nnapi->ANeuralNetworksCompilation_free, nullptr);
-  EXPECT_NE(nnapi->ANeuralNetworksCompilation_setPreference, nullptr);
-  EXPECT_NE(nnapi->ANeuralNetworksCompilation_finish, nullptr);
-  EXPECT_NE(nnapi->ANeuralNetworksExecution_create, nullptr);
-  EXPECT_NE(nnapi->ANeuralNetworksExecution_free, nullptr);
-  EXPECT_NE(nnapi->ANeuralNetworksExecution_setInput, nullptr);
-  EXPECT_NE(nnapi->ANeuralNetworksExecution_setInputFromMemory, nullptr);
-  EXPECT_NE(nnapi->ANeuralNetworksExecution_setOutput, nullptr);
-  EXPECT_NE(nnapi->ANeuralNetworksExecution_setOutputFromMemory, nullptr);
-  EXPECT_NE(nnapi->ANeuralNetworksExecution_startCompute, nullptr);
-  EXPECT_NE(nnapi->ANeuralNetworksEvent_wait, nullptr);
-  EXPECT_NE(nnapi->ANeuralNetworksEvent_free, nullptr);
-  EXPECT_NE(nnapi->ASharedMemory_create, nullptr);
-#else
-  EXPECT_FALSE(nnapi->nnapi_exists);
-  EXPECT_EQ(nnapi->android_sdk_version, 0);
-  EXPECT_EQ(nnapi->ANeuralNetworksMemory_createFromFd, nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksMemory_free, nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksModel_create, nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksModel_free, nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksModel_finish, nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksModel_addOperand, nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksModel_setOperandValue, nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksModel_setOperandValueFromMemory, nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksModel_addOperation, nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksModel_identifyInputsAndOutputs, nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksModel_relaxComputationFloat32toFloat16,
-            nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksCompilation_create, nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksCompilation_free, nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksCompilation_setPreference, nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksCompilation_finish, nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksExecution_create, nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksExecution_free, nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksExecution_setInput, nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksExecution_setInputFromMemory, nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksExecution_setOutput, nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksExecution_setOutputFromMemory, nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksExecution_startCompute, nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksEvent_wait, nullptr);
-  EXPECT_EQ(nnapi->ANeuralNetworksEvent_free, nullptr);
-  EXPECT_EQ(nnapi->ASharedMemory_create, nullptr);
-#endif
-}
-
-}  // namespace
diff --git a/tensorflow/lite/nnapi_delegate.cc b/tensorflow/lite/nnapi_delegate.cc
index 44b9930acc..dc8e81cde7 100644
--- a/tensorflow/lite/nnapi_delegate.cc
+++ b/tensorflow/lite/nnapi_delegate.cc
@@ -84,27 +84,56 @@ void logError(const char* format, ...) {
 static const int64_t kOperandIdNotSet = -1;
 static const int64_t kOperandNotNeeded = -2;
 
+namespace {
+
+int32_t GetAndroidSdkVersion() {
+#ifdef __ANDROID__
+  const char* sdkProp = "ro.build.version.sdk";
+  char sdkVersion[PROP_VALUE_MAX];
+  int length = __system_property_get(sdkProp, sdkVersion);
+  if (length != 0) {
+    for (int i = 0; i < length; ++i) {
+      int digit = sdkVersion[i] - '0';
+      if (digit < 0 || digit > 9) {
+        // Non-numeric SDK version, assume it's higher then expected;
+        return 0xFFFF;
+      }
+    }
+    return atoi(sdkVersion);
+  }
+  FATAL("No %s prop", sdkProp);
+#endif  // __ANDROID__
+  return 0;
+}
+
+int32_t GetAndroidSdkVersionCached() {
+  static int32_t androidSdkVersion = GetAndroidSdkVersion();
+  return androidSdkVersion;
+}
+
+}  // namespace
+
 NNAPIAllocation::NNAPIAllocation(const char* filename,
                                  ErrorReporter* error_reporter)
     : MMAPAllocation(filename, error_reporter) {
   if (mmapped_buffer_ != MAP_FAILED)
-    CHECK_NN(NnApiImplementation()->ANeuralNetworksMemory_createFromFd(
-        buffer_size_bytes_, PROT_READ, mmap_fd_, 0, &handle_));
+    CHECK_NN(ANeuralNetworksMemory_createFromFd(buffer_size_bytes_, PROT_READ,
+                                                mmap_fd_, 0, &handle_));
 }
 
 NNAPIAllocation::~NNAPIAllocation() {
   if (handle_) {
-    NnApiImplementation()->ANeuralNetworksMemory_free(handle_);
+    ANeuralNetworksMemory_free(handle_);
   }
 }
 
 NNAPIDelegate::~NNAPIDelegate() {
   if (nn_compiled_model_) {
-    NnApiImplementation()->ANeuralNetworksCompilation_free(nn_compiled_model_);
+    ANeuralNetworksCompilation_free(nn_compiled_model_);
     nn_compiled_model_ = nullptr;
   }
   if (nn_model_) {
-    NnApiImplementation()->ANeuralNetworksModel_free(nn_model_);
+    ANeuralNetworksModel_free(nn_model_);
     nn_model_ = nullptr;
     // TODO(aselle): Is this thread-safe and callable multiple times?
   }
@@ -116,7 +145,6 @@ TfLiteStatus addTensorOperands(tflite::Subgraph* subgraph,
                                ANeuralNetworksModel* nn_model,
                                uint32_t* no_of_operands_added,
                                std::vector<int64_t>* nnapi_ids) {
-  const NnApi* nnapi = NnApiImplementation();
   uint32_t next_id = 0;
   for (size_t i = 0; i < subgraph->tensors_size(); i++) {
     // Skip temporaries and RNN back-edges.
@@ -170,24 +198,24 @@ TfLiteStatus addTensorOperands(tflite::Subgraph* subgraph,
         nn_type, static_cast<uint32_t>(tensor->dims->size),
         reinterpret_cast<uint32_t*>(tensor->dims->data), scale, zeroPoint};
     RETURN_ERROR_IF_NN_FAILED(
-        nnapi->ANeuralNetworksModel_addOperand(nn_model, &operand_type));
+        ANeuralNetworksModel_addOperand(nn_model, &operand_type));
     // TODO(aselle): Based on Michael's suggestion, limiting this to read
     // only memory
     if (tensor->allocation_type == kTfLiteMmapRo) {
       if (const NNAPIAllocation* alloc = dynamic_cast<const NNAPIAllocation*>(
               static_cast<const Allocation*>(tensor->allocation))) {
         RETURN_ERROR_IF_NN_FAILED(
-            nnapi->ANeuralNetworksModel_setOperandValueFromMemory(
+            ANeuralNetworksModel_setOperandValueFromMemory(
                 nn_model, next_id, alloc->memory(),
                 alloc->offset(tensor->data.raw), tensor->bytes));
       } else {
-        RETURN_ERROR_IF_NN_FAILED(nnapi->ANeuralNetworksModel_setOperandValue(
+        RETURN_ERROR_IF_NN_FAILED(ANeuralNetworksModel_setOperandValue(
             nn_model, next_id, tensor->data.raw, tensor->bytes));
       }
     } else if (tensor->bytes == 0) {
       // These size 0 tensors are optional tensors reserved.
-      RETURN_ERROR_IF_NN_FAILED(nnapi->ANeuralNetworksModel_setOperandValue(
-          nn_model, next_id, nullptr, 0));
+      RETURN_ERROR_IF_NN_FAILED(
+          ANeuralNetworksModel_setOperandValue(nn_model, next_id, nullptr, 0));
     }
 
     ++next_id;
@@ -216,7 +244,6 @@ TfLiteStatus AddOpsAndParams(
     uint32_t next_id, std::vector<int>* model_state_inputs,
     std::vector<int>* model_state_outputs,
     const std::vector<int64_t>& tensor_id_to_nnapi_id) {
-  const NnApi* nnapi = NnApiImplementation();
   for (size_t i = 0; i < subgraph->nodes_size(); i++) {
     const auto* node_and_registration = subgraph->node_and_registration(i);
     const TfLiteNode& node = node_and_registration->first;
@@ -231,21 +258,21 @@ TfLiteStatus AddOpsAndParams(
     MapAndAddTensorIds(node.outputs->data, node.outputs->size,
                        &augmented_outputs, tensor_id_to_nnapi_id);
 
-    auto add_scalar_int32 = [nnapi, &nn_model, &augmented_inputs,
+    auto add_scalar_int32 = [&nn_model, &augmented_inputs,
                              &next_id](int value) {
       ANeuralNetworksOperandType operand_type{.type = ANEURALNETWORKS_INT32};
-      CHECK_NN(nnapi->ANeuralNetworksModel_addOperand(nn_model, &operand_type))
-      CHECK_NN(nnapi->ANeuralNetworksModel_setOperandValue(
-          nn_model, next_id, &value, sizeof(int32_t)))
+      CHECK_NN(ANeuralNetworksModel_addOperand(nn_model, &operand_type))
+      CHECK_NN(ANeuralNetworksModel_setOperandValue(nn_model, next_id, &value,
+                                                    sizeof(int32_t)))
       augmented_inputs.push_back(next_id++);
     };
 
-    auto add_scalar_float32 = [nnapi, &nn_model, &augmented_inputs,
+    auto add_scalar_float32 = [&nn_model, &augmented_inputs,
                                &next_id](float value) {
       ANeuralNetworksOperandType operand_type{.type = ANEURALNETWORKS_FLOAT32};
-      CHECK_NN(nnapi->ANeuralNetworksModel_addOperand(nn_model, &operand_type))
-      CHECK_NN(nnapi->ANeuralNetworksModel_setOperandValue(
-          nn_model, next_id, &value, sizeof(float)))
+      CHECK_NN(ANeuralNetworksModel_addOperand(nn_model, &operand_type))
+      CHECK_NN(ANeuralNetworksModel_setOperandValue(nn_model, next_id, &value,
+                                                    sizeof(float)))
       augmented_inputs.push_back(next_id++);
     };
 
@@ -254,8 +281,8 @@ TfLiteStatus AddOpsAndParams(
           .type = ANEURALNETWORKS_TENSOR_INT32,
           .dimensionCount = 1,
           .dimensions = &num_values};
-      CHECK_NN(nnapi->ANeuralNetworksModel_addOperand(nn_model, &operand_type))
-      CHECK_NN(nnapi->ANeuralNetworksModel_setOperandValue(
+      CHECK_NN(ANeuralNetworksModel_addOperand(nn_model, &operand_type))
+      CHECK_NN(ANeuralNetworksModel_setOperandValue(
           nn_model, next_id, values, sizeof(int32_t) * num_values));
       augmented_inputs.push_back(next_id++);
     };
@@ -264,16 +291,15 @@ TfLiteStatus AddOpsAndParams(
     // For each state_out tensor, a corresponding state_in operand needs to be
     // created for NNAPI.
     auto duplicate_state_tensor_float32 =
-        [nnapi, subgraph, &nn_model, &next_id, &augmented_inputs,
-         &model_state_inputs, &model_state_outputs](int tensor_id) {
+        [subgraph, &nn_model, &next_id, &augmented_inputs, &model_state_inputs,
+         &model_state_outputs](int tensor_id) {
           const TfLiteTensor* tensor = subgraph->tensor(tensor_id);
           ANeuralNetworksOperandType operand_type{
               ANEURALNETWORKS_TENSOR_FLOAT32,
               static_cast<uint32_t>(tensor->dims->size),
               reinterpret_cast<uint32_t*>(tensor->dims->data),
               tensor->params.scale, tensor->params.zero_point};
-          CHECK_NN(
-              nnapi->ANeuralNetworksModel_addOperand(nn_model, &operand_type));
+          CHECK_NN(ANeuralNetworksModel_addOperand(nn_model, &operand_type));
           augmented_inputs.push_back(next_id);
           model_state_inputs->push_back(next_id);
           model_state_outputs->push_back(tensor_id);
@@ -362,7 +388,7 @@ TfLiteStatus AddOpsAndParams(
     };
 
     // LSTM in NNAPI requires scratch tensor as an output operand.
-    auto add_lstm_scratch_tensor_float32 = [nnapi, subgraph, &node, &nn_model,
+    auto add_lstm_scratch_tensor_float32 = [subgraph, &node, &nn_model,
                                             &next_id, &augmented_outputs]() {
       if (node.temporaries->size == 0) return;
       int scratch_buffer_index = node.temporaries->data[0];
@@ -372,7 +398,7 @@ TfLiteStatus AddOpsAndParams(
           static_cast<uint32_t>(tensor->dims->size),
           reinterpret_cast<uint32_t*>(tensor->dims->data), tensor->params.scale,
           tensor->params.zero_point};
-      CHECK_NN(nnapi->ANeuralNetworksModel_addOperand(nn_model, &operand_type));
+      CHECK_NN(ANeuralNetworksModel_addOperand(nn_model, &operand_type));
       augmented_outputs.insert(augmented_outputs.begin(), next_id++);
     };
 
@@ -401,16 +427,15 @@ TfLiteStatus AddOpsAndParams(
     };
 
     // Handle optional input tensors.
-    auto add_optional_tensors = [nnapi, &nn_model, &augmented_inputs,
+    auto add_optional_tensors = [&nn_model, &augmented_inputs,
                                  &next_id](int nn_type) {
       for (size_t idx = 0; idx < augmented_inputs.size(); idx++) {
         if (augmented_inputs[idx] == kOptionalTensor) {
           const std::vector<uint32_t> dim = {0, 0};
           ANeuralNetworksOperandType operand_type{nn_type, 2, dim.data(), 0, 0};
-          CHECK_NN(
-              nnapi->ANeuralNetworksModel_addOperand(nn_model, &operand_type))
-          CHECK_NN(nnapi->ANeuralNetworksModel_setOperandValue(
-              nn_model, next_id, nullptr, 0))
+          CHECK_NN(ANeuralNetworksModel_addOperand(nn_model, &operand_type))
+          CHECK_NN(ANeuralNetworksModel_setOperandValue(nn_model, next_id,
+                                                        nullptr, 0))
           augmented_inputs[idx] = next_id++;
         }
       }
@@ -671,13 +696,13 @@ TfLiteStatus AddOpsAndParams(
         break;
     }
 
-    if (nnapi_version == 11 && nnapi->android_sdk_version < 28) {
+    if (nnapi_version == 11 && GetAndroidSdkVersionCached() < 28) {
       logError("Op %d needs NNAPI1.1", builtin);
       return kTfLiteError;
     }
 
     // Add the operation.
-    RETURN_ERROR_IF_NN_FAILED(nnapi->ANeuralNetworksModel_addOperation(
+    RETURN_ERROR_IF_NN_FAILED(ANeuralNetworksModel_addOperation(
         nn_model, nn_op_type, static_cast<uint32_t>(augmented_inputs.size()),
         augmented_inputs.data(),
         static_cast<uint32_t>(augmented_outputs.size()),
@@ -689,10 +714,9 @@ TfLiteStatus AddOpsAndParams(
 TfLiteStatus NNAPIDelegate::BuildGraph(Subgraph* subgraph) {
   if (nn_model_ && nn_compiled_model_) return model_status_;
 
-  const NnApi* nnapi = NnApiImplementation();
   // TODO(aselle): This is not correct. need to handle resize invalidation.
   if (!nn_model_) {
-    CHECK_NN(nnapi->ANeuralNetworksModel_create(&nn_model_));
+    CHECK_NN(ANeuralNetworksModel_create(&nn_model_));
 
     // Find which tensors should be added to NNAPI. TFLite has temporaries
     // and RNN back-edges which are are not valid for NNAPI. We look through all
@@ -739,22 +763,21 @@ TfLiteStatus NNAPIDelegate::BuildGraph(Subgraph* subgraph) {
                        model_states_outputs_.size(), &augmented_outputs,
                        tensor_id_to_nnapi_id);
 
-    CHECK_NN(nnapi->ANeuralNetworksModel_identifyInputsAndOutputs(
+    CHECK_NN(ANeuralNetworksModel_identifyInputsAndOutputs(
         nn_model_, static_cast<uint32_t>(augmented_inputs.size()),
         reinterpret_cast<const uint32_t*>(augmented_inputs.data()),
         static_cast<uint32_t>(augmented_outputs.size()),
         reinterpret_cast<const uint32_t*>(augmented_outputs.data())));
 
-    if (nnapi->android_sdk_version >= 28) {
-      CHECK_NN(nnapi->ANeuralNetworksModel_relaxComputationFloat32toFloat16(
+    if (GetAndroidSdkVersionCached() >= 28) {
+      CHECK_NN(ANeuralNetworksModel_relaxComputationFloat32toFloat16(
           nn_model_, subgraph->GetAllowFp16PrecisionForFp32()));
     }
-    CHECK_NN(nnapi->ANeuralNetworksModel_finish(nn_model_));
+    CHECK_NN(ANeuralNetworksModel_finish(nn_model_));
   }
   if (!nn_compiled_model_) {
-    CHECK_NN(nnapi->ANeuralNetworksCompilation_create(nn_model_,
-                                                      &nn_compiled_model_));
-    CHECK_NN(nnapi->ANeuralNetworksCompilation_finish(nn_compiled_model_));
+    CHECK_NN(ANeuralNetworksCompilation_create(nn_model_, &nn_compiled_model_));
+    CHECK_NN(ANeuralNetworksCompilation_finish(nn_compiled_model_));
   }
   return kTfLiteOk;
 }
@@ -770,10 +793,8 @@ TfLiteStatus NNAPIDelegate::Invoke(Subgraph* subgraph) {
     return model_status_;
   }
 
-  const NnApi* nnapi = NnApiImplementation();
   ANeuralNetworksExecution* execution = nullptr;
-  CHECK_NN(
-      nnapi->ANeuralNetworksExecution_create(nn_compiled_model_, &execution));
+  CHECK_NN(ANeuralNetworksExecution_create(nn_compiled_model_, &execution));
 
   // Currently perform deep copy of input buffer
   for (size_t i = 0; i < subgraph->inputs().size(); i++) {
@@ -781,7 +802,7 @@ TfLiteStatus NNAPIDelegate::Invoke(Subgraph* subgraph) {
     // TODO(aselle): Is this what we want or do we want input instead?
     // TODO(aselle): This should be called setInputValue maybe to be cons.
     TfLiteTensor* tensor = subgraph->tensor(input);
-    CHECK_NN(nnapi->ANeuralNetworksExecution_setInput(
+    CHECK_NN(ANeuralNetworksExecution_setInput(
         execution, i, nullptr, tensor->data.raw, tensor->bytes));
   }
 
@@ -789,7 +810,7 @@ TfLiteStatus NNAPIDelegate::Invoke(Subgraph* subgraph) {
   for (size_t i = 0; i < subgraph->outputs().size(); i++) {
     int output = subgraph->outputs()[i];
     TfLiteTensor* tensor = subgraph->tensor(output);
-    CHECK_NN(nnapi->ANeuralNetworksExecution_setOutput(
+    CHECK_NN(ANeuralNetworksExecution_setOutput(
         execution, i, nullptr, tensor->data.raw, tensor->bytes));
   }
 
@@ -801,21 +822,21 @@ TfLiteStatus NNAPIDelegate::Invoke(Subgraph* subgraph) {
     // Here we are using a deep copy for state_in tensors so that we are not
     // reading and writing into the same buffer during a invocation.
     // TODO(miaowang): using double shared buffer to minimize the copies.
-    CHECK_NN(nnapi->ANeuralNetworksExecution_setInput(
+    CHECK_NN(ANeuralNetworksExecution_setInput(
         execution, i + subgraph->inputs().size(), nullptr, tensor->data.raw,
         tensor->bytes));
     // Tell NNAPI where to output the state_out.
-    CHECK_NN(nnapi->ANeuralNetworksExecution_setOutput(
+    CHECK_NN(ANeuralNetworksExecution_setOutput(
         execution, i + subgraph->outputs().size(), nullptr, tensor->data.raw,
         tensor->bytes));
   }
 
   // Currently use blocking compute.
   ANeuralNetworksEvent* event = nullptr;
-  CHECK_NN(nnapi->ANeuralNetworksExecution_startCompute(execution, &event));
-  CHECK_NN(nnapi->ANeuralNetworksEvent_wait(event));
-  nnapi->ANeuralNetworksEvent_free(event);
-  nnapi->ANeuralNetworksExecution_free(execution);
+  CHECK_NN(ANeuralNetworksExecution_startCompute(execution, &event));
+  CHECK_NN(ANeuralNetworksEvent_wait(event));
+  ANeuralNetworksEvent_free(event);
+  ANeuralNetworksExecution_free(execution);
 
 #if 0
   printf("From the NN API:\n");
@@ -833,8 +854,6 @@ TfLiteStatus NNAPIDelegate::Invoke(Subgraph* subgraph) {
   return kTfLiteOk;
 }
 
-bool NNAPIDelegate::IsSupported() {
-  return NnApiImplementation()->nnapi_exists;
-}
+bool NNAPIDelegate::IsSupported() { return NNAPIExists(); }
 
 }  // namespace tflite
-- 
GitLab


From a460c9525d416e442c245c87ccb6da9032cc5fd6 Mon Sep 17 00:00:00 2001
From: Scott Zhu <scottzhu@google.com>
Date: Wed, 2 Jan 2019 16:58:33 -0800
Subject: [PATCH 0155/1095] Fix masking generate logic in sequence model.

sequence model was incorrectly using the layer output instead of input to generate mask, and compute_mask of the layer usually does not have shape check.

PiperOrigin-RevId: 227605298
---
 tensorflow/python/keras/engine/sequential.py  |   9 +-
 tensorflow/python/keras/layers/gru_test.py    | 112 ++++++++--------
 tensorflow/python/keras/layers/lstm_test.py   |   8 +-
 .../python/keras/layers/simplernn_test.py     |   8 +-
 .../python/keras/layers/unified_gru_test.py   |  64 +++++----
 .../python/keras/layers/unified_lstm_test.py  | 125 +++++++++---------
 6 files changed, 153 insertions(+), 173 deletions(-)

diff --git a/tensorflow/python/keras/engine/sequential.py b/tensorflow/python/keras/engine/sequential.py
index dffd3c8a69..3627091198 100644
--- a/tensorflow/python/keras/engine/sequential.py
+++ b/tensorflow/python/keras/engine/sequential.py
@@ -262,16 +262,17 @@ class Sequential(Model):
           with ops.name_scope(layer._name_scope()):
             layer._maybe_build(x)
           layer.built = True
+        if layer.supports_masking:
+          mask = layer.compute_mask(x, mask)
+        else:
+          mask = None
+
         if context.executing_eagerly():
           x = layer(x, **kwargs)
         elif layer.dynamic:
           x = layer._symbolic_call(x)
         else:
           x = layer.call(x, **kwargs)
-        if layer.supports_masking:
-          mask = layer.compute_mask(x, mask)
-        else:
-          mask = None
       if not context.executing_eagerly():
         x._keras_mask = mask
     return x, mask
diff --git a/tensorflow/python/keras/layers/gru_test.py b/tensorflow/python/keras/layers/gru_test.py
index 52a7944d60..e2b65de661 100644
--- a/tensorflow/python/keras/layers/gru_test.py
+++ b/tensorflow/python/keras/layers/gru_test.py
@@ -117,63 +117,6 @@ class GRULayerTest(keras_parameterized.TestCase):
                   run_eagerly=testing_utils.should_run_eagerly())
     model.fit(inputs, targets, epochs=1, batch_size=2, verbose=1)
 
-
-@tf_test_util.run_all_in_graph_and_eager_modes
-class GRULayerGenericTest(test.TestCase):
-
-  def test_constraints_GRU(self):
-    embedding_dim = 4
-    layer_class = keras.layers.GRU
-    k_constraint = keras.constraints.max_norm(0.01)
-    r_constraint = keras.constraints.max_norm(0.01)
-    b_constraint = keras.constraints.max_norm(0.01)
-    layer = layer_class(
-        5,
-        return_sequences=False,
-        weights=None,
-        input_shape=(None, embedding_dim),
-        kernel_constraint=k_constraint,
-        recurrent_constraint=r_constraint,
-        bias_constraint=b_constraint)
-    layer.build((None, None, embedding_dim))
-    self.assertEqual(layer.cell.kernel.constraint, k_constraint)
-    self.assertEqual(layer.cell.recurrent_kernel.constraint, r_constraint)
-    self.assertEqual(layer.cell.bias.constraint, b_constraint)
-
-  def test_from_config_GRU(self):
-    layer_class = keras.layers.GRU
-    for stateful in (False, True):
-      l1 = layer_class(units=1, stateful=stateful)
-      l2 = layer_class.from_config(l1.get_config())
-      assert l1.get_config() == l2.get_config()
-
-  def test_regularizers_GRU(self):
-    embedding_dim = 4
-    layer_class = keras.layers.GRU
-    layer = layer_class(
-        5,
-        return_sequences=False,
-        weights=None,
-        input_shape=(None, embedding_dim),
-        kernel_regularizer=keras.regularizers.l1(0.01),
-        recurrent_regularizer=keras.regularizers.l1(0.01),
-        bias_regularizer='l2',
-        activity_regularizer='l1')
-    layer.build((None, None, 2))
-    self.assertEqual(len(layer.losses), 3)
-
-    x = keras.backend.variable(np.ones((2, 3, 2)))
-    layer(x)
-    if context.executing_eagerly():
-      self.assertEqual(len(layer.losses), 4)
-    else:
-      self.assertEqual(len(layer.get_losses_for(x)), 1)
-
-
-class GRULayerV1OnlyTest(test.TestCase):
-
-  @tf_test_util.run_v1_only('b/120941292')
-  @tf_test_util.run_in_graph_and_eager_modes
   def test_statefulness_GRU(self):
     num_samples = 2
     timesteps = 3
@@ -192,7 +135,8 @@ class GRULayerV1OnlyTest(test.TestCase):
     layer = layer_class(
         units, return_sequences=False, stateful=True, weights=None)
     model.add(layer)
-    model.compile(optimizer='sgd', loss='mse')
+    model.compile(optimizer='sgd', loss='mse',
+                  run_eagerly=testing_utils.should_run_eagerly())
     out1 = model.predict(np.ones((num_samples, timesteps)))
     self.assertEqual(out1.shape, (num_samples, units))
 
@@ -237,5 +181,57 @@ class GRULayerV1OnlyTest(test.TestCase):
     np.testing.assert_allclose(out7, out6, atol=1e-5)
 
 
+@tf_test_util.run_all_in_graph_and_eager_modes
+class GRULayerGenericTest(test.TestCase):
+
+  def test_constraints_GRU(self):
+    embedding_dim = 4
+    layer_class = keras.layers.GRU
+    k_constraint = keras.constraints.max_norm(0.01)
+    r_constraint = keras.constraints.max_norm(0.01)
+    b_constraint = keras.constraints.max_norm(0.01)
+    layer = layer_class(
+        5,
+        return_sequences=False,
+        weights=None,
+        input_shape=(None, embedding_dim),
+        kernel_constraint=k_constraint,
+        recurrent_constraint=r_constraint,
+        bias_constraint=b_constraint)
+    layer.build((None, None, embedding_dim))
+    self.assertEqual(layer.cell.kernel.constraint, k_constraint)
+    self.assertEqual(layer.cell.recurrent_kernel.constraint, r_constraint)
+    self.assertEqual(layer.cell.bias.constraint, b_constraint)
+
+  def test_from_config_GRU(self):
+    layer_class = keras.layers.GRU
+    for stateful in (False, True):
+      l1 = layer_class(units=1, stateful=stateful)
+      l2 = layer_class.from_config(l1.get_config())
+      assert l1.get_config() == l2.get_config()
+
+  def test_regularizers_GRU(self):
+    embedding_dim = 4
+    layer_class = keras.layers.GRU
+    layer = layer_class(
+        5,
+        return_sequences=False,
+        weights=None,
+        input_shape=(None, embedding_dim),
+        kernel_regularizer=keras.regularizers.l1(0.01),
+        recurrent_regularizer=keras.regularizers.l1(0.01),
+        bias_regularizer='l2',
+        activity_regularizer='l1')
+    layer.build((None, None, 2))
+    self.assertEqual(len(layer.losses), 3)
+
+    x = keras.backend.variable(np.ones((2, 3, 2)))
+    layer(x)
+    if context.executing_eagerly():
+      self.assertEqual(len(layer.losses), 4)
+    else:
+      self.assertEqual(len(layer.get_losses_for(x)), 1)
+
+
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/keras/layers/lstm_test.py b/tensorflow/python/keras/layers/lstm_test.py
index e8c55438c9..38c0177e39 100644
--- a/tensorflow/python/keras/layers/lstm_test.py
+++ b/tensorflow/python/keras/layers/lstm_test.py
@@ -23,7 +23,6 @@ import numpy as np
 
 from tensorflow.python import keras
 from tensorflow.python.eager import context
-from tensorflow.python.framework import test_util
 from tensorflow.python.keras import keras_parameterized
 from tensorflow.python.keras import testing_utils
 from tensorflow.python.platform import test
@@ -340,11 +339,6 @@ class LSTMLayerTest(keras_parameterized.TestCase):
     else:
       self.assertEqual(len(layer.get_losses_for(x)), 1)
 
-
-class LSTMLayerV1OnlyTest(test.TestCase):
-
-  @test_util.run_v1_only('b/120941292')
-  @test_util.run_in_graph_and_eager_modes
   def test_statefulness_LSTM(self):
     num_samples = 2
     timesteps = 3
@@ -363,7 +357,7 @@ class LSTMLayerV1OnlyTest(test.TestCase):
         units, return_sequences=False, stateful=True, weights=None)
     model.add(layer)
     model.compile(optimizer=gradient_descent.GradientDescentOptimizer(0.01),
-                  loss='mse')
+                  loss='mse', run_eagerly=testing_utils.should_run_eagerly())
     out1 = model.predict(np.ones((num_samples, timesteps)))
     self.assertEqual(out1.shape, (num_samples, units))
 
diff --git a/tensorflow/python/keras/layers/simplernn_test.py b/tensorflow/python/keras/layers/simplernn_test.py
index 390ae789e1..0e599dade9 100644
--- a/tensorflow/python/keras/layers/simplernn_test.py
+++ b/tensorflow/python/keras/layers/simplernn_test.py
@@ -22,7 +22,6 @@ import numpy as np
 
 from tensorflow.python import keras
 from tensorflow.python.eager import context
-from tensorflow.python.framework import test_util
 from tensorflow.python.keras import keras_parameterized
 from tensorflow.python.keras import testing_utils
 from tensorflow.python.platform import test
@@ -141,11 +140,6 @@ class SimpleRNNLayerTest(keras_parameterized.TestCase):
     else:
       self.assertEqual(len(layer.get_losses_for(x)), 1)
 
-
-class SimpleRNNLayerV1OnlyTest(test.TestCase):
-
-  @test_util.run_v1_only('b/120941292')
-  @test_util.run_in_graph_and_eager_modes
   def test_statefulness_SimpleRNN(self):
     num_samples = 2
     timesteps = 3
@@ -164,7 +158,7 @@ class SimpleRNNLayerV1OnlyTest(test.TestCase):
         units, return_sequences=False, stateful=True, weights=None)
     model.add(layer)
     model.compile(optimizer=gradient_descent.GradientDescentOptimizer(0.01),
-                  loss='mse')
+                  loss='mse', run_eagerly=testing_utils.should_run_eagerly())
     out1 = model.predict(np.ones((num_samples, timesteps)))
     self.assertEqual(out1.shape, (num_samples, units))
 
diff --git a/tensorflow/python/keras/layers/unified_gru_test.py b/tensorflow/python/keras/layers/unified_gru_test.py
index 244ffdb8b6..11322764ac 100644
--- a/tensorflow/python/keras/layers/unified_gru_test.py
+++ b/tensorflow/python/keras/layers/unified_gru_test.py
@@ -402,39 +402,6 @@ class UnifiedGRUTest(keras_parameterized.TestCase):
     else:
       self.assertEqual(len(layer.get_losses_for(x)), 1)
 
-
-class GRULayerGradientTapeTest(test.TestCase):
-
-  @test_util.run_in_graph_and_eager_modes(config=_config)
-  def test_in_tape(self):
-    if not context.executing_eagerly():
-      self.skipTest('bloo')
-    time_steps = 10
-    embedding_size = 11
-    gru_unit_size = 12
-
-    gru = keras.layers.UnifiedGRU(gru_unit_size,
-                                  return_sequences=True,
-                                  return_state=True,
-                                  recurrent_activation='sigmoid',
-                                  recurrent_initializer='glorot_uniform')
-
-    x = random_ops.random_uniform([1, time_steps, embedding_size])
-    y = random_ops.random_uniform([1, gru_unit_size])
-
-    with backprop.GradientTape() as tape:
-      hidden_state = array_ops.zeros([1, gru_unit_size], dtype=dtypes.float32)
-      _, state = gru(x, initial_state=hidden_state)
-
-      loss = math_ops.reduce_mean(math_ops.square(state - y))
-
-    tape.gradient(loss, gru.variables)
-
-
-class GRULayerV1OnlyTest(test.TestCase, parameterized.TestCase):
-
-  @test_util.run_v1_only('b/120941292')
-  @test_util.run_in_graph_and_eager_modes(config=_config)
   def test_statefulness_GRU(self):
     num_samples = 2
     timesteps = 3
@@ -452,7 +419,8 @@ class GRULayerV1OnlyTest(test.TestCase, parameterized.TestCase):
     layer = layer_class(
         units, return_sequences=False, stateful=True, weights=None)
     model.add(layer)
-    model.compile(optimizer='sgd', loss='mse')
+    model.compile(optimizer=gradient_descent.GradientDescentOptimizer(0.01),
+                  loss='mse', run_eagerly=testing_utils.should_run_eagerly())
     out1 = model.predict(np.ones((num_samples, timesteps)))
     self.assertEqual(out1.shape, (num_samples, units))
 
@@ -497,6 +465,34 @@ class GRULayerV1OnlyTest(test.TestCase, parameterized.TestCase):
     np.testing.assert_allclose(out7, out6, atol=1e-5)
 
 
+class GRULayerGradientTapeTest(test.TestCase):
+
+  @test_util.run_in_graph_and_eager_modes(config=_config)
+  def test_in_tape(self):
+    if not context.executing_eagerly():
+      self.skipTest('bloo')
+    time_steps = 10
+    embedding_size = 11
+    gru_unit_size = 12
+
+    gru = keras.layers.UnifiedGRU(gru_unit_size,
+                                  return_sequences=True,
+                                  return_state=True,
+                                  recurrent_activation='sigmoid',
+                                  recurrent_initializer='glorot_uniform')
+
+    x = random_ops.random_uniform([1, time_steps, embedding_size])
+    y = random_ops.random_uniform([1, gru_unit_size])
+
+    with backprop.GradientTape() as tape:
+      hidden_state = array_ops.zeros([1, gru_unit_size], dtype=dtypes.float32)
+      _, state = gru(x, initial_state=hidden_state)
+
+      loss = math_ops.reduce_mean(math_ops.square(state - y))
+
+    tape.gradient(loss, gru.variables)
+
+
 class GRULayerGraphOnlyTest(test.TestCase):
 
   # Need session for test
diff --git a/tensorflow/python/keras/layers/unified_lstm_test.py b/tensorflow/python/keras/layers/unified_lstm_test.py
index 811b8d128e..375894b166 100644
--- a/tensorflow/python/keras/layers/unified_lstm_test.py
+++ b/tensorflow/python/keras/layers/unified_lstm_test.py
@@ -571,6 +571,68 @@ class UnifiedLSTMTest(keras_parameterized.TestCase):
     else:
       self.assertEqual(len(layer.get_losses_for(x)), 1)
 
+  def test_statefulness_LSTM(self):
+    num_samples = 2
+    timesteps = 3
+    embedding_dim = 4
+    units = 2
+    layer_class = keras.layers.UnifiedLSTM
+    model = keras.models.Sequential()
+    model.add(
+        keras.layers.Embedding(
+            4,
+            embedding_dim,
+            mask_zero=True,
+            input_length=timesteps,
+            batch_input_shape=(num_samples, timesteps)))
+    layer = layer_class(
+        units, return_sequences=False, stateful=True, weights=None)
+    model.add(layer)
+    model.compile(optimizer=gradient_descent.GradientDescentOptimizer(0.01),
+                  loss='mse', run_eagerly=testing_utils.should_run_eagerly())
+    out1 = model.predict(np.ones((num_samples, timesteps)))
+    self.assertEqual(out1.shape, (num_samples, units))
+
+    # train once so that the states change
+    model.train_on_batch(
+        np.ones((num_samples, timesteps)), np.ones((num_samples, units)))
+    out2 = model.predict(np.ones((num_samples, timesteps)))
+
+    # if the state is not reset, output should be different
+    self.assertNotEqual(out1.max(), out2.max())
+
+    # check that output changes after states are reset
+    # (even though the model itself didn't change)
+    layer.reset_states()
+    out3 = model.predict(np.ones((num_samples, timesteps)))
+    self.assertNotEqual(out2.max(), out3.max())
+
+    # check that container-level reset_states() works
+    model.reset_states()
+    out4 = model.predict(np.ones((num_samples, timesteps)))
+    self.assertAllClose(out3, out4, atol=1e-5)
+
+    # check that the call to `predict` updated the states
+    out5 = model.predict(np.ones((num_samples, timesteps)))
+    self.assertNotEqual(out4.max(), out5.max())
+
+    # Check masking
+    layer.reset_states()
+
+    left_padded_input = np.ones((num_samples, timesteps))
+    left_padded_input[0, :1] = 0
+    left_padded_input[1, :2] = 0
+    out6 = model.predict(left_padded_input)
+
+    layer.reset_states()
+
+    right_padded_input = np.ones((num_samples, timesteps))
+    right_padded_input[0, -1:] = 0
+    right_padded_input[1, -2:] = 0
+    out7 = model.predict(right_padded_input)
+
+    self.assertAllClose(out7, out6, atol=1e-5)
+
 
 class LSTMLayerGraphOnlyTest(test.TestCase):
 
@@ -697,69 +759,6 @@ class LSTMLayerV1OnlyTest(test.TestCase, parameterized.TestCase):
         },
         input_shape=(num_samples, timesteps, embedding_dim))
 
-  @test_util.run_in_graph_and_eager_modes(config=_config)
-  def test_statefulness_LSTM(self):
-    num_samples = 2
-    timesteps = 3
-    embedding_dim = 4
-    units = 2
-    layer_class = keras.layers.UnifiedLSTM
-    model = keras.models.Sequential()
-    model.add(
-        keras.layers.Embedding(
-            4,
-            embedding_dim,
-            mask_zero=True,
-            input_length=timesteps,
-            batch_input_shape=(num_samples, timesteps)))
-    layer = layer_class(
-        units, return_sequences=False, stateful=True, weights=None)
-    model.add(layer)
-    model.compile(
-        optimizer=gradient_descent.GradientDescentOptimizer(0.01), loss='mse')
-    out1 = model.predict(np.ones((num_samples, timesteps)))
-    self.assertEqual(out1.shape, (num_samples, units))
-
-    # train once so that the states change
-    model.train_on_batch(
-        np.ones((num_samples, timesteps)), np.ones((num_samples, units)))
-    out2 = model.predict(np.ones((num_samples, timesteps)))
-
-    # if the state is not reset, output should be different
-    self.assertNotEqual(out1.max(), out2.max())
-
-    # check that output changes after states are reset
-    # (even though the model itself didn't change)
-    layer.reset_states()
-    out3 = model.predict(np.ones((num_samples, timesteps)))
-    self.assertNotEqual(out2.max(), out3.max())
-
-    # check that container-level reset_states() works
-    model.reset_states()
-    out4 = model.predict(np.ones((num_samples, timesteps)))
-    self.assertAllClose(out3, out4, atol=1e-5)
-
-    # check that the call to `predict` updated the states
-    out5 = model.predict(np.ones((num_samples, timesteps)))
-    self.assertNotEqual(out4.max(), out5.max())
-
-    # Check masking
-    layer.reset_states()
-
-    left_padded_input = np.ones((num_samples, timesteps))
-    left_padded_input[0, :1] = 0
-    left_padded_input[1, :2] = 0
-    out6 = model.predict(left_padded_input)
-
-    layer.reset_states()
-
-    right_padded_input = np.ones((num_samples, timesteps))
-    right_padded_input[0, -1:] = 0
-    right_padded_input[1, -2:] = 0
-    out7 = model.predict(right_padded_input)
-
-    self.assertAllClose(out7, out6, atol=1e-5)
-
 
 class UnifiedLSTMPerformanceTest(test.Benchmark):
 
-- 
GitLab


From 563ff22e7f6c9431072d2d9af6e876b3b4843920 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 2 Jan 2019 17:06:04 -0800
Subject: [PATCH 0156/1095] Fix bug where sample weights were being ignored in
 the tf.distribute.Strategy / numpy input case.

PiperOrigin-RevId: 227606473
---
 tensorflow/python/keras/engine/training.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 1dc3b81cfc..8263467c37 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -2191,7 +2191,6 @@ class Model(Network):
         else:
           x = dataset_ops.Dataset.from_tensor_slices((var_x, var_y))
 
-        x = dataset_ops.Dataset.from_tensor_slices((var_x, var_y))
         if shuffle:
           # 1024 is a good buffer size since it is much larger than the average
           # batch size provided by the user and provides sufficient randomness.
-- 
GitLab


From b38e065943271461bd9afce7ae8f1bd00c3d6bf9 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 2 Jan 2019 17:34:44 -0800
Subject: [PATCH 0157/1095] use packages@tensorflow.org as author email for
 python package

PiperOrigin-RevId: 227609414
---
 tensorflow/lite/tools/pip_package/setup.py | 2 +-
 tensorflow/tools/pip_package/setup.py      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/tools/pip_package/setup.py b/tensorflow/lite/tools/pip_package/setup.py
index 64d62ee1f2..82f9ac5b8f 100644
--- a/tensorflow/lite/tools/pip_package/setup.py
+++ b/tensorflow/lite/tools/pip_package/setup.py
@@ -136,7 +136,7 @@ setup(
     long_description='\n'.join(DOCLINES[2:]),
     url='https://www.tensorflow.org/lite/',
     author='Google Inc.',
-    author_email='opensource@google.com',
+    author_email='packages@tensorflow.org',
     license='Apache 2.0',
     include_package_data=True,
     keywords='tflite tensorflow tensor machine learning',
diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py
index 3927540cc7..9567e0aff7 100644
--- a/tensorflow/tools/pip_package/setup.py
+++ b/tensorflow/tools/pip_package/setup.py
@@ -248,7 +248,7 @@ setup(
     url='https://www.tensorflow.org/',
     download_url='https://github.com/tensorflow/tensorflow/tags',
     author='Google Inc.',
-    author_email='opensource@google.com',
+    author_email='packages@tensorflow.org',
     # Contained modules and scripts.
     packages=find_packages(),
     entry_points={
-- 
GitLab


From 894278658bf6da29831371670862688f46153be0 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Wed, 2 Jan 2019 17:56:16 -0800
Subject: [PATCH 0158/1095] Add top k categorical accuracy and sparse top k
 categorical accuracy v2 metrics.

PiperOrigin-RevId: 227611439
---
 tensorflow/python/keras/metrics.py      | 78 +++++++++++++++++++++++++
 tensorflow/python/keras/metrics_test.py | 76 ++++++++++++++++++++++++
 2 files changed, 154 insertions(+)

diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index 6b81151d5c..73c116a745 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -590,6 +590,84 @@ class SparseCategoricalAccuracy(MeanMetricWrapper):
     return super(SparseCategoricalAccuracy, cls).from_config(config)
 
 
+class TopKCategoricalAccuracy(MeanMetricWrapper):
+  """Computes how often targets are in the top `K` predictions.
+
+  Usage:
+
+  ```python
+  m = tf.keras.metrics.TopKCategoricalAccuracy()
+  m.update_state([[0, 0, 1], [0, 1, 0]], [[0.1, 0.9, 0.8], [0.05, 0.95, 0]])
+  print('Final result: ', m.result().numpy())  # Final result: 1.0
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile('sgd', metrics=[tf.keras.metrics.TopKCategoricalAccuracy()])
+  ```
+  """
+
+  def __init__(self, k=5, name='top_k_categorical_accuracy', dtype=None):
+    """Creates a `TopKCategoricalAccuracy` instance.
+
+    Args:
+      k: (Optional) Number of top elements to look at for computing accuracy.
+        Defaults to 5.
+      name: (Optional) string name of the metric instance.
+      dtype: (Optional) data type of the metric result.
+    """
+    super(TopKCategoricalAccuracy, self).__init__(
+        top_k_categorical_accuracy, name, dtype=dtype, k=k)
+
+  @classmethod
+  def from_config(cls, config):
+    if 'fn' in config:
+      config.pop('fn')
+    return super(TopKCategoricalAccuracy, cls).from_config(config)
+
+
+class SparseTopKCategoricalAccuracy(MeanMetricWrapper):
+  """Computes how often integer targets are in the top `K` predictions.
+
+  Usage:
+
+  ```python
+  m = tf.keras.metrics.SparseTopKCategoricalAccuracy()
+  m.update_state([2, 1], [[0.1, 0.9, 0.8], [0.05, 0.95, 0]])
+  print('Final result: ', m.result().numpy())  # Final result: 1.0
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile(
+    'sgd',
+    metrics=[tf.keras.metrics.SparseTopKCategoricalAccuracy()])
+  ```
+  """
+
+  def __init__(self, k=5, name='sparse_top_k_categorical_accuracy', dtype=None):
+    """Creates a `SparseTopKCategoricalAccuracy` instance.
+
+    Args:
+      k: (Optional) Number of top elements to look at for computing accuracy.
+        Defaults to 5.
+      name: (Optional) string name of the metric instance.
+      dtype: (Optional) data type of the metric result.
+    """
+    super(SparseTopKCategoricalAccuracy, self).__init__(
+        sparse_top_k_categorical_accuracy, name, dtype=dtype, k=k)
+
+  @classmethod
+  def from_config(cls, config):
+    if 'fn' in config:
+      config.pop('fn')
+    return super(SparseTopKCategoricalAccuracy, cls).from_config(config)
+
+
 class _ConfusionMatrixConditionCount(Metric):
   """Calculates the number of the given confusion matrix condition."""
 
diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py
index 8f8befd413..fb635e7055 100644
--- a/tensorflow/python/keras/metrics_test.py
+++ b/tensorflow/python/keras/metrics_test.py
@@ -1391,6 +1391,82 @@ class RootMeanSquaredErrorTest(test.TestCase):
     self.assertAllClose(math.sqrt(13), self.evaluate(result), atol=1e-3)
 
 
+@test_util.run_all_in_graph_and_eager_modes
+class TopKCategoricalAccuracyTest(test.TestCase):
+
+  def test_config(self):
+    a_obj = metrics.TopKCategoricalAccuracy(name='topkca', dtype=dtypes.int32)
+    self.assertEqual(a_obj.name, 'topkca')
+    self.assertEqual(a_obj._dtype, dtypes.int32)
+
+    a_obj2 = metrics.TopKCategoricalAccuracy.from_config(a_obj.get_config())
+    self.assertEqual(a_obj2.name, 'topkca')
+    self.assertEqual(a_obj2._dtype, dtypes.int32)
+
+  def test_correctness(self):
+    a_obj = metrics.TopKCategoricalAccuracy()
+    self.evaluate(variables.variables_initializer(a_obj.variables))
+    y_true = constant_op.constant([[0, 0, 1], [0, 1, 0]])
+    y_pred = constant_op.constant([[0.1, 0.9, 0.8], [0.05, 0.95, 0]])
+
+    result = a_obj(y_true, y_pred)
+    self.assertEqual(1, self.evaluate(result))  # both the samples match
+
+    # With `k` < 5.
+    a_obj = metrics.TopKCategoricalAccuracy(k=1)
+    self.evaluate(variables.variables_initializer(a_obj.variables))
+    result = a_obj(y_true, y_pred)
+    self.assertEqual(0.5, self.evaluate(result))  # only sample #2 matches
+
+    # With `k` > 5.
+    y_true = constant_op.constant([[0, 0, 1, 0, 0, 0, 0],
+                                   [0, 1, 0, 0, 0, 0, 0]])
+    y_pred = constant_op.constant([[0.5, 0.9, 0.1, 0.7, 0.6, 0.5, 0.4],
+                                   [0.05, 0.95, 0, 0, 0, 0, 0]])
+    a_obj = metrics.TopKCategoricalAccuracy(k=6)
+    self.evaluate(variables.variables_initializer(a_obj.variables))
+    result = a_obj(y_true, y_pred)
+    self.assertEqual(0.5, self.evaluate(result))  # only 1 sample matches.
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class SparseTopKCategoricalAccuracyTest(test.TestCase):
+
+  def test_config(self):
+    a_obj = metrics.SparseTopKCategoricalAccuracy(
+        name='stopkca', dtype=dtypes.int32)
+    self.assertEqual(a_obj.name, 'stopkca')
+    self.assertEqual(a_obj._dtype, dtypes.int32)
+
+    a_obj2 = metrics.SparseTopKCategoricalAccuracy.from_config(
+        a_obj.get_config())
+    self.assertEqual(a_obj2.name, 'stopkca')
+    self.assertEqual(a_obj2._dtype, dtypes.int32)
+
+  def test_correctness(self):
+    a_obj = metrics.SparseTopKCategoricalAccuracy()
+    self.evaluate(variables.variables_initializer(a_obj.variables))
+    y_true = constant_op.constant([2, 1])
+    y_pred = constant_op.constant([[0.1, 0.9, 0.8], [0.05, 0.95, 0]])
+
+    result = a_obj(y_true, y_pred)
+    self.assertEqual(1, self.evaluate(result))  # both the samples match
+
+    # With `k` < 5.
+    a_obj = metrics.SparseTopKCategoricalAccuracy(k=1)
+    self.evaluate(variables.variables_initializer(a_obj.variables))
+    result = a_obj(y_true, y_pred)
+    self.assertEqual(0.5, self.evaluate(result))  # only sample #2 matches
+
+    # With `k` > 5.
+    y_pred = constant_op.constant([[0.5, 0.9, 0.1, 0.7, 0.6, 0.5, 0.4],
+                                   [0.05, 0.95, 0, 0, 0, 0, 0]])
+    a_obj = metrics.SparseTopKCategoricalAccuracy(k=6)
+    self.evaluate(variables.variables_initializer(a_obj.variables))
+    result = a_obj(y_true, y_pred)
+    self.assertEqual(0.5, self.evaluate(result))  # only 1 sample matches.
+
+
 def _get_model(compile_metrics):
   model_layers = [
       layers.Dense(3, activation='relu', kernel_initializer='ones'),
-- 
GitLab


From a8bf983c60f3684220996cd518d2b86e31dd6e1e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 2 Jan 2019 21:49:05 -0800
Subject: [PATCH 0159/1095] use packages@tensorflow.org as author email for
 python package

PiperOrigin-RevId: 227630041
---
 tensorflow/contrib/tpu/profiler/pip_package/setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/tpu/profiler/pip_package/setup.py b/tensorflow/contrib/tpu/profiler/pip_package/setup.py
index f27ae38e04..807cf26fe9 100644
--- a/tensorflow/contrib/tpu/profiler/pip_package/setup.py
+++ b/tensorflow/contrib/tpu/profiler/pip_package/setup.py
@@ -33,7 +33,7 @@ setup(
     long_description='Tools for capture TPU profile',
     url='https://www.tensorflow.org/tfrc/',
     author='Google Inc.',
-    author_email='opensource@google.com',
+    author_email='packages@tensorflow.org',
     packages=['cloud_tpu_profiler'],
     package_data={
         'cloud_tpu_profiler': ['data/*'],
-- 
GitLab


From 7e71784d707cee4e14689b55a59bb1e23ed1a3f5 Mon Sep 17 00:00:00 2001
From: Heungsub Lee <sub@subl.ee>
Date: Thu, 3 Jan 2019 17:12:21 +0900
Subject: [PATCH 0160/1095] Fix quotation typo around "task"

---
 tensorflow/contrib/distribute/python/mirrored_strategy.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy.py b/tensorflow/contrib/distribute/python/mirrored_strategy.py
index db8fd98307..e635190b50 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy.py
@@ -48,7 +48,7 @@ class MirroredStrategy(distribute_lib.DistributionStrategy):
   distributed environment.
 
   There are several important concepts for distributed TensorFlow, e.g.
-  `client`, `job`, 'task', `cluster`, `in-graph replication` and
+  `client`, `job`, `task`, `cluster`, `in-graph replication` and
   'synchronous training' and they have already been defined in the
   [TensorFlow's documentation](https://www.tensorflow.org/deploy/distributed).
   The distribution strategy inherits these concepts as well and in addition to
-- 
GitLab


From df0bd2904e2ace75c53e0e7663a2f9cff23f261d Mon Sep 17 00:00:00 2001
From: Kay Zhu <kayzhu@google.com>
Date: Thu, 3 Jan 2019 00:30:28 -0800
Subject: [PATCH 0161/1095] [XLA] Support RNG in HloEvaluator.

PiperOrigin-RevId: 227642508
---
 .../xla/service/hlo_constant_folding.cc       |  5 +-
 .../compiler/xla/service/hlo_evaluator.cc     |  8 ++
 .../compiler/xla/service/hlo_evaluator.h      |  5 +
 .../xla/service/hlo_evaluator_typed_visitor.h | 99 ++++++++++++++++++-
 tensorflow/compiler/xla/tests/BUILD           |  4 -
 tensorflow/compiler/xla/tests/prng_test.cc    | 41 +++++++-
 6 files changed, 153 insertions(+), 9 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_constant_folding.cc b/tensorflow/compiler/xla/service/hlo_constant_folding.cc
index c58d00ff50..799cc9fd91 100644
--- a/tensorflow/compiler/xla/service/hlo_constant_folding.cc
+++ b/tensorflow/compiler/xla/service/hlo_constant_folding.cc
@@ -52,7 +52,7 @@ StatusOr<bool> HloConstantFolding::Run(HloModule* module) {
           computation->root_instruction() != instruction) {
         continue;
       }
-      // Skip Constant, Parameter, Tuple, AfterAll operation.
+      // Skip Constant, Parameter, Tuple, AfterAll, Rng operations.
       // Tuple constants are not directly supported by any backends, hence
       // folding Tuple is not useful and would in fact be expanded back into
       // kTuple by Algebraic Simplifier.
@@ -62,7 +62,8 @@ StatusOr<bool> HloConstantFolding::Run(HloModule* module) {
       if (instruction->opcode() == HloOpcode::kParameter ||
           instruction->opcode() == HloOpcode::kConstant ||
           instruction->opcode() == HloOpcode::kTuple ||
-          instruction->opcode() == HloOpcode::kAfterAll) {
+          instruction->opcode() == HloOpcode::kAfterAll ||
+          instruction->opcode() == HloOpcode::kRng) {
         continue;
       }
 
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc
index 4f5484293c..e897d09e1b 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc
@@ -233,6 +233,14 @@ StatusOr<Literal> HloEvaluator::Evaluate(
   for (const auto& literal_ptr : arg_literals) {
     arg_literals_.push_back(&*literal_ptr);
   }
+  if (computation.parent()->config().seed()) {
+    seed_ = computation.parent()->config().seed();
+  } else {
+    std::random_device rd;
+    seed_ = rd();
+  }
+
+  engine_ = std::minstd_rand0(seed_);
 
   TF_RETURN_IF_ERROR(computation.Accept(this));
   return GetEvaluatedLiteralFor(computation.root_instruction()).Clone();
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.h b/tensorflow/compiler/xla/service/hlo_evaluator.h
index d8e3019576..4fc5d10aed 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.h
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.h
@@ -290,6 +290,11 @@ class HloEvaluator : public DfsHloVisitorWithDefault {
   // Max loop iterations to execute with no maximum if negative.
   int64 max_loop_iterations_;
 
+  // Module-level seed handle.
+  uint64 seed_;
+  // RNG engine.
+  std::minstd_rand0 engine_;
+
   TF_DISALLOW_COPY_AND_ASSIGN(HloEvaluator);
 };
 
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
index b1bbf49d4a..cf3509d0aa 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
@@ -2653,7 +2653,7 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
   template <typename NativeT, typename std::enable_if<std::is_same<
                                   double, NativeT>::value>::type* = nullptr>
   Status HandleReducePrecision(HloInstruction* reduce_precision) {
-    return InvalidArgument("Double not supported for reduce precision");
+    return InvalidArgument("Double is not supported for reduce precision");
   }
 
   template <
@@ -2719,6 +2719,103 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
     return HandleIota<ReturnT>(iota);
   }
 
+  template <typename NativeT,
+            typename std::enable_if<
+                !(std::is_integral<NativeT>::value ||
+                  std::is_floating_point<NativeT>::value)>::type* = nullptr>
+  Status HandleRng(HloInstruction* random) {
+    return UnsupportedTypeError(random);
+  }
+  template <typename NativeT,
+            typename std::enable_if<
+                (std::is_floating_point<NativeT>::value)>::type* = nullptr>
+  Status HandleRng(HloInstruction* random) {
+    RandomDistribution distribution = random->random_distribution();
+    const auto result_shape = random->shape();
+    Literal result(result_shape);
+
+    switch (distribution) {
+      case RNG_UNIFORM: {
+        const Literal& low =
+            parent_->GetEvaluatedLiteralFor(random->operand(0));
+        const Literal& high =
+            parent_->GetEvaluatedLiteralFor(random->operand(1));
+
+        std::uniform_real_distribution<NativeT> generator(
+            low.Get<NativeT>({}), high.Get<NativeT>({}));
+
+        TF_RETURN_IF_ERROR(
+            result.Populate<NativeT>([&](absl::Span<const int64> /*indexes*/) {
+              return generator(parent_->engine_);
+            }));
+        break;
+      }
+      case RNG_NORMAL: {
+        const Literal& mean =
+            parent_->GetEvaluatedLiteralFor(random->operand(0));
+        const Literal& stddev =
+            parent_->GetEvaluatedLiteralFor(random->operand(1));
+
+        std::normal_distribution<NativeT> generator(mean.Get<NativeT>({}),
+                                                    stddev.Get<NativeT>({}));
+
+        TF_RETURN_IF_ERROR(
+            result.Populate<NativeT>([&](absl::Span<const int64> /*indexes*/) {
+              return generator(parent_->engine_);
+            }));
+        break;
+      }
+      default:
+        return UnimplementedStrCat("The distribution ",
+                                   RandomDistribution_Name(distribution),
+                                   " is not implemented.");
+    }
+    parent_->evaluated_[random] = std::move(result);
+    return Status::OK();
+  }
+  template <typename NativeT,
+            typename std::enable_if<(std::is_integral<NativeT>::value)>::type* =
+                nullptr>
+  Status HandleRng(HloInstruction* random) {
+    RandomDistribution distribution = random->random_distribution();
+    const auto result_shape = random->shape();
+    Literal result(result_shape);
+
+    switch (distribution) {
+      case RNG_UNIFORM: {
+        const Literal& low =
+            parent_->GetEvaluatedLiteralFor(random->operand(0));
+        const Literal& high =
+            parent_->GetEvaluatedLiteralFor(random->operand(1));
+
+        // Note std::uniform_int_distribution assumes interval is closed, i.e.,
+        // [low, high], but we want [low, high) instead. Hence high-1 is used as
+        // the upper range.
+        std::uniform_int_distribution<int64> generator(
+            low.Get<NativeT>({}), high.Get<NativeT>({}) - 1);
+
+        TF_RETURN_IF_ERROR(
+            result.Populate<NativeT>([&](absl::Span<const int64> /*indexes*/) {
+              return static_cast<NativeT>(generator(parent_->engine_));
+            }));
+        break;
+      }
+      case RNG_NORMAL: {
+        return Unimplemented(
+            "Normal distribution is not supported for integral types.");
+      }
+      default:
+        return UnimplementedStrCat("The distribution ",
+                                   RandomDistribution_Name(distribution),
+                                   " is not implemented.");
+    }
+    parent_->evaluated_[random] = std::move(result);
+    return Status::OK();
+  }
+  Status HandleRng(HloInstruction* random) override {
+    return HandleRng<ReturnT>(random);
+  }
+
  private:
   // Creates a vector of multipliers which can be used to create a linear index
   // into shape.
diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index 578e716b59..2c11ba4283 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -1422,10 +1422,6 @@ xla_test(
 xla_test(
     name = "prng_test",
     srcs = ["prng_test.cc"],
-    blacklisted_backends = [
-        # TODO(b/122047800) support RNGs on the interpreter backend.
-        "interpreter",
-    ],
     deps = [
         "//tensorflow/compiler/xla:literal",
         "//tensorflow/compiler/xla:shape_util",
diff --git a/tensorflow/compiler/xla/tests/prng_test.cc b/tensorflow/compiler/xla/tests/prng_test.cc
index 8f2c26f0ee..e49bcf26bd 100644
--- a/tensorflow/compiler/xla/tests/prng_test.cc
+++ b/tensorflow/compiler/xla/tests/prng_test.cc
@@ -80,7 +80,9 @@ XLA_TEST_F(PrngTest, LargeU01) { UniformTest<float>(0, 1, {0x100, 0x100}); }
 XLA_TEST_F(PrngTest, TwelveValuesU524) { UniformTest<int32>(5, 24, {12}); }
 
 // TODO(b/71543667): Fix Rng ops on LLVM backends.
-XLA_TEST_F(PrngTest, DISABLED_ON_GPU(DISABLED_ON_CPU(ScalarBF16Tests))) {
+// TODO(b/122047800): Interpreter does not support BF16 for RNG ops.
+XLA_TEST_F(PrngTest, DISABLED_ON_INTERPRETER(
+                         DISABLED_ON_GPU(DISABLED_ON_CPU(ScalarBF16Tests)))) {
   for (int64 seed = 0; seed < 100; ++seed) {
     // The largest negative number smaller than zero in bf16 that's not
     // denormalized.
@@ -103,7 +105,9 @@ XLA_TEST_F(PrngTest, DISABLED_ON_GPU(DISABLED_ON_CPU(ScalarBF16Tests))) {
 }
 
 // TODO(b/71543667): Fix Rng ops on LLVM backends.
-XLA_TEST_F(PrngTest, DISABLED_ON_GPU(DISABLED_ON_CPU(ScalarBF16CountTests))) {
+// TODO(b/122047800): Interpreter does not support BF16 for RNG ops.
+XLA_TEST_F(PrngTest, DISABLED_ON_INTERPRETER(DISABLED_ON_GPU(
+                         DISABLED_ON_CPU(ScalarBF16CountTests)))) {
   // There are 3 BF16 values in the range of [32.25, 33): 32.25, 32.5, 32.75,
   // they should get similar counts.
   bfloat16 low = static_cast<bfloat16>(32.25);
@@ -276,6 +280,39 @@ XLA_TEST_F(PrngTest, PassInGlobalRngSeed) {
   EXPECT_FALSE(LiteralTestUtil::Equal(result5, result6));
 }
 
+// This test verifies that the two RNG instructions with the same parameters in
+// the same HloComputation produces different values.
+XLA_TEST_F(PrngTest, DifferentValuesForIdenticalRngNodesInSameComputation) {
+  // Build a U[0,1) computation.
+  auto build_computation = [this]() {
+    XlaBuilder builder(TestName());
+    auto a = RngUniform(ConstantR0<int32>(&builder, 0),
+                        ConstantR0<int32>(&builder, 100),
+                        ShapeUtil::MakeShape(S32, {10}));
+    auto b = RngUniform(ConstantR0<int32>(&builder, 0),
+                        ConstantR0<int32>(&builder, 100),
+                        ShapeUtil::MakeShape(S32, {10}));
+    Tuple(&builder, {a, b});
+    return builder.Build();
+  };
+
+  ExecutionOptions execution_options = execution_options_;
+  execution_options.set_seed(42);
+
+  Literal result_tuple;
+  {
+    TF_ASSERT_OK_AND_ASSIGN(auto computation, build_computation());
+    TF_ASSERT_OK_AND_ASSIGN(
+        result_tuple, client_->ExecuteAndTransfer(computation, /*arguments=*/{},
+                                                  &execution_options));
+  }
+
+  auto results = result_tuple.DecomposeTuple();
+  ASSERT_EQ(results.size(), 2);
+
+  EXPECT_FALSE(LiteralTestUtil::Equal(results[0], results[1]));
+}
+
 XLA_TEST_F(PrngTest, TenValuesN01) {
   XlaBuilder builder(TestName());
   RngNormal(ConstantR0<float>(&builder, 0), ConstantR0<float>(&builder, 1),
-- 
GitLab


From 37326d1497de6de3ea32a1b121f99b2e50ac2dbc Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 3 Jan 2019 01:02:58 -0800
Subject: [PATCH 0162/1095] compat: Update forward compatibility horizon to
 2019-01-03

PiperOrigin-RevId: 227646086
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 949bb0dc0c..c60c42ee65 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 2)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 3)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 61fde9e1e3c5d995aa20f7bf2781ba60db5bf246 Mon Sep 17 00:00:00 2001
From: Heungsub Lee <sub@subl.ee>
Date: Thu, 3 Jan 2019 18:26:13 +0900
Subject: [PATCH 0163/1095] Single quote to backtick for consistency

---
 tensorflow/contrib/distribute/python/mirrored_strategy.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy.py b/tensorflow/contrib/distribute/python/mirrored_strategy.py
index e635190b50..e3ab2bf19e 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy.py
@@ -49,7 +49,7 @@ class MirroredStrategy(distribute_lib.DistributionStrategy):
 
   There are several important concepts for distributed TensorFlow, e.g.
   `client`, `job`, `task`, `cluster`, `in-graph replication` and
-  'synchronous training' and they have already been defined in the
+  `synchronous training` and they have already been defined in the
   [TensorFlow's documentation](https://www.tensorflow.org/deploy/distributed).
   The distribution strategy inherits these concepts as well and in addition to
   that we also clarify several more concepts:
-- 
GitLab


From eb54349cb4274ab797917a9e0699decec0b9794c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 3 Jan 2019 03:11:44 -0800
Subject: [PATCH 0164/1095] Remove NCCL from RBE dockers. NCCL is built from
 source now.

PiperOrigin-RevId: 227661008
---
 .../Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04         |  9 ---------
 .../ci_build/Dockerfile.rbe.cuda9.0-cudnn7-ubuntu14.04 |  9 ---------
 tensorflow/tools/docker/Dockerfile.devel-gpu           | 10 ----------
 3 files changed, 28 deletions(-)

diff --git a/tensorflow/tools/ci_build/Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04 b/tensorflow/tools/ci_build/Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04
index 553fbb2860..d08d31d913 100644
--- a/tensorflow/tools/ci_build/Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04
+++ b/tensorflow/tools/ci_build/Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04
@@ -19,7 +19,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates
 ENV CUDA_VERSION 10.0.130
 ENV CUDA_PKG_VERSION 10-0=$CUDA_VERSION-1
 ENV CUDNN_VERSION 7.3.1.20
-ENV NCCL_VERSION 2.3.5
 ENV TENSORRT_VERSION 5.0.2
 ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
 ENV NVIDIA_REQUIRE_CUDA "cuda>=10.0,driver>=410"
@@ -48,26 +47,18 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         libcudnn7=$CUDNN_VERSION-1+cuda10.0 \
         libcudnn7=$CUDNN_VERSION-1+cuda10.0 \
         libcudnn7-dev=$CUDNN_VERSION-1+cuda10.0 \
-        libnccl2=$NCCL_VERSION-2+cuda10.0 \
-        libnccl-dev=$NCCL_VERSION-2+cuda10.0 \
         nvinfer-runtime-trt-repo-ubuntu1604-$TENSORRT_VERSION-ga-cuda10.0 && \
     apt-get update && apt-get install -y --no-install-recommends \
         libnvinfer5=$TENSORRT_VERSION-1+cuda10.0 \
         libnvinfer-dev=$TENSORRT_VERSION-1+cuda10.0 && \
     ln -s cuda-10.0 /usr/local/cuda && \
     apt-mark hold libcudnn7 && \
-    apt-mark hold libnccl2 && \
     rm -rf /var/lib/apt/lists/*
 
 # TODO(b/110903506): Provide a link to the SONAME of libcuda.so.
 # https://github.com/NVIDIA/nvidia-docker/issues/775
 RUN ln -s libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1
 
-# TODO(klimek): Once the TODO in tensorflow's configure.py to correctly find
-# libnccl is resolved, delete this block.
-RUN ln -s /usr/lib/x86_64-linux-gnu/libnccl.so /usr/lib/libnccl.so \
- && ln -s /usr/lib/x86_64-linux-gnu/libnccl.so /usr/lib/libnccl.so.2
-
 # Install a newer version of libstdc++, as new clang versions do not work
 # with the stock ubuntu 14.04 libstdc++.
 RUN apt-get update && \
diff --git a/tensorflow/tools/ci_build/Dockerfile.rbe.cuda9.0-cudnn7-ubuntu14.04 b/tensorflow/tools/ci_build/Dockerfile.rbe.cuda9.0-cudnn7-ubuntu14.04
index 60a23e1edb..9d5d51447c 100644
--- a/tensorflow/tools/ci_build/Dockerfile.rbe.cuda9.0-cudnn7-ubuntu14.04
+++ b/tensorflow/tools/ci_build/Dockerfile.rbe.cuda9.0-cudnn7-ubuntu14.04
@@ -25,7 +25,6 @@ ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH}
 ENV NVIDIA_VISIBLE_DEVICES all
 ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
 ENV NVIDIA_REQUIRE_CUDA "cuda>=9.0"
-ENV NCCL_VERSION 2.2.13
 ENV TENSORRT_VERSION 5.0.2
 ENV CUDNN_VERSION 7.1.4.18
 
@@ -45,14 +44,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         cuda-cudart-$CUDA_PKG_VERSION \
         cuda-libraries-$CUDA_PKG_VERSION \
         cuda-cublas-9-0=9.0.176.4-1 \
-        libnccl2=$NCCL_VERSION-1+cuda9.0 \
         cuda-libraries-dev-$CUDA_PKG_VERSION \
         cuda-nvml-dev-$CUDA_PKG_VERSION \
         cuda-minimal-build-$CUDA_PKG_VERSION \
         cuda-command-line-tools-$CUDA_PKG_VERSION \
         cuda-core-9-0=9.0.176.3-1 \
         cuda-cublas-dev-9-0=9.0.176.4-1 \
-        libnccl-dev=$NCCL_VERSION-1+cuda9.0 \
         libcudnn7-dev=$CUDNN_VERSION-1+cuda9.0 \
         libcudnn7=$CUDNN_VERSION-1+cuda9.0 \
         nvinfer-runtime-trt-repo-ubuntu1604-$TENSORRT_VERSION-ga-cuda9.0 && \
@@ -60,7 +57,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         libnvinfer5=$TENSORRT_VERSION-1+cuda9.0 \
         libnvinfer-dev=$TENSORRT_VERSION-1+cuda9.0 && \
     ln -s cuda-9.0 /usr/local/cuda && \
-    apt-mark hold libnccl2 && \
     apt-mark hold libcudnn7 libcudnn7-dev && \
     rm -rf /var/lib/apt/lists/*
 
@@ -71,11 +67,6 @@ RUN echo "/usr/local/nvidia/lib" >> /etc/ld.so.conf.d/nvidia.conf && \
 # https://github.com/NVIDIA/nvidia-docker/issues/775
 RUN ln -s libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1
 
-# TODO(klimek): Once the TODO in tensorflow's configure.py to correctly find
-# libnccl is resolved, delete this block.
-RUN ln -s /usr/lib/x86_64-linux-gnu/libnccl.so /usr/lib/libnccl.so \
- && ln -s /usr/lib/x86_64-linux-gnu/libnccl.so /usr/lib/libnccl.so.2
-
 # Install a newer version of libstdc++, as new clang versions do not work
 # with the stock ubuntu 14.04 libstdc++.
 RUN apt-get update && \
diff --git a/tensorflow/tools/docker/Dockerfile.devel-gpu b/tensorflow/tools/docker/Dockerfile.devel-gpu
index 1ad359ddcc..e085ee7170 100644
--- a/tensorflow/tools/docker/Dockerfile.devel-gpu
+++ b/tensorflow/tools/docker/Dockerfile.devel-gpu
@@ -15,8 +15,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         git \
         libcudnn7=7.2.1.38-1+cuda9.0 \
         libcudnn7-dev=7.2.1.38-1+cuda9.0 \
-        libnccl2=2.2.13-1+cuda9.0 \
-        libnccl-dev=2.2.13-1+cuda9.0 \
         libcurl3-dev \
         libfreetype6-dev \
         libhdf5-serial-dev \
@@ -41,11 +39,6 @@ RUN apt-get update && \
         apt-get install libnvinfer4=4.1.2-1+cuda9.0 && \
         apt-get install libnvinfer-dev=4.1.2-1+cuda9.0
 
-# Link NCCL libray and header where the build script expects them.
-RUN mkdir /usr/local/cuda-9.0/lib &&  \
-    ln -s /usr/lib/x86_64-linux-gnu/libnccl.so.2 /usr/local/cuda/lib/libnccl.so.2 && \
-    ln -s /usr/include/nccl.h /usr/local/cuda/include/nccl.h
-
 RUN curl -fSsL -O https://bootstrap.pypa.io/get-pip.py && \
     python get-pip.py && \
     rm get-pip.py
@@ -111,9 +104,6 @@ ENV TF_CUDA_COMPUTE_CAPABILITIES=3.5,5.2,6.0,6.1,7.0
 ENV TF_CUDA_VERSION=9.0
 ENV TF_CUDNN_VERSION=7
 
-# NCCL 2.x
-ENV TF_NCCL_VERSION=2
-
 RUN ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 && \
     LD_LIBRARY_PATH=/usr/local/cuda/lib64/stubs:${LD_LIBRARY_PATH} \
     tensorflow/tools/ci_build/builds/configured GPU \
-- 
GitLab


From 31cf2b97f0f3b1a90d26a56016c4b984f62dea3e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 3 Jan 2019 06:51:42 -0800
Subject: [PATCH 0165/1095] Add all unary operations to list of
 autograph-supported ops.

PiperOrigin-RevId: 227681912
---
 .../converters/logical_expressions.py         | 44 ++++++-------
 .../converters/logical_expressions_test.py    |  7 ++
 .../python/autograph/operators/__init__.py    |  2 +
 .../python/autograph/operators/logical.py     | 65 +++++--------------
 4 files changed, 46 insertions(+), 72 deletions(-)

diff --git a/tensorflow/python/autograph/converters/logical_expressions.py b/tensorflow/python/autograph/converters/logical_expressions.py
index dfcaafdc9e..ea9740a22e 100644
--- a/tensorflow/python/autograph/converters/logical_expressions.py
+++ b/tensorflow/python/autograph/converters/logical_expressions.py
@@ -38,29 +38,29 @@ from tensorflow.python.autograph.pyct import templates
 SAFE_BOOLEAN_OPERAND = 'SAFE_BOOLEAN_OPERAND'
 
 
+OP_MAPPING = {
+    gast.And: 'ag__.and_',
+    gast.Eq: 'ag__.eq',
+    gast.NotEq: 'ag__.not_eq',
+    gast.Lt: 'ag__.lt',
+    gast.LtE: 'ag__.lt_e',
+    gast.Gt: 'ag__.gt',
+    gast.GtE: 'ag__.gt_e',
+    gast.Is: 'ag__.is_',
+    gast.IsNot: 'ag__.is_not',
+    gast.In: 'ag__.in_',
+    gast.Not: 'ag__.not_',
+    gast.NotIn: 'ag__.not_in',
+    gast.Or: 'ag__.or_',
+    gast.UAdd: 'ag__.u_add',
+    gast.USub: 'ag__.u_sub',
+    gast.Invert: 'ag__.invert',
+}
+
+
 class LogicalExpressionTransformer(converter.Base):
   """Converts logical expressions to corresponding TF calls."""
 
-  def __init__(self, ctx):
-    super(LogicalExpressionTransformer, self).__init__(ctx)
-    # TODO(mdan): For completeness and consistency, overload everything.
-    self.op_mapping = {
-        gast.And: 'ag__.and_',
-        gast.Eq: 'ag__.eq',
-        gast.NotEq: 'ag__.not_eq',
-        gast.Lt: 'ag__.lt',
-        gast.LtE: 'ag__.lt_e',
-        gast.Gt: 'ag__.gt',
-        gast.GtE: 'ag__.gt_e',
-        gast.Is: 'ag__.is_',
-        gast.IsNot: 'ag__.is_not',
-        gast.In: 'ag__.in_',
-        gast.Not: 'ag__.not_',
-        gast.NotIn: 'ag__.not_in',
-        gast.Or: 'ag__.or_',
-        gast.USub: 'ag__.u_sub',
-    }
-
   def _expect_simple_symbol(self, operand):
     if isinstance(operand, gast.Name):
       return
@@ -74,11 +74,11 @@ class LogicalExpressionTransformer(converter.Base):
 
   def _has_matching_func(self, operator):
     op_type = type(operator)
-    return op_type in self.op_mapping
+    return op_type in OP_MAPPING
 
   def _matching_func(self, operator):
     op_type = type(operator)
-    return self.op_mapping[op_type]
+    return OP_MAPPING[op_type]
 
   def _as_function(self, func_name, args, args_as_lambda=False):
     if args_as_lambda:
diff --git a/tensorflow/python/autograph/converters/logical_expressions_test.py b/tensorflow/python/autograph/converters/logical_expressions_test.py
index 687412750e..67ccd1fb47 100644
--- a/tensorflow/python/autograph/converters/logical_expressions_test.py
+++ b/tensorflow/python/autograph/converters/logical_expressions_test.py
@@ -77,6 +77,13 @@ class LogicalExpressionTest(converter_testing.TestCase):
     with self.converted(test_fn, logical_expressions, {}) as result:
       self.assertTrue(result.test_fn('a', ('a',)))
 
+  def test_unary_ops(self):
+    def test_fn(a):
+      return ~a, -a, +a
+
+    with self.converted(test_fn, logical_expressions, {}) as result:
+      self.assertEqual(result.test_fn(1), (-2, -1, 1))
+
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/autograph/operators/__init__.py b/tensorflow/python/autograph/operators/__init__.py
index 7a580fe324..35f8028c29 100644
--- a/tensorflow/python/autograph/operators/__init__.py
+++ b/tensorflow/python/autograph/operators/__init__.py
@@ -52,6 +52,7 @@ from tensorflow.python.autograph.operators.logical import eq
 from tensorflow.python.autograph.operators.logical import gt
 from tensorflow.python.autograph.operators.logical import gt_e
 from tensorflow.python.autograph.operators.logical import in_
+from tensorflow.python.autograph.operators.logical import invert
 from tensorflow.python.autograph.operators.logical import is_
 from tensorflow.python.autograph.operators.logical import is_not
 from tensorflow.python.autograph.operators.logical import lt
@@ -60,6 +61,7 @@ from tensorflow.python.autograph.operators.logical import not_
 from tensorflow.python.autograph.operators.logical import not_eq
 from tensorflow.python.autograph.operators.logical import not_in
 from tensorflow.python.autograph.operators.logical import or_
+from tensorflow.python.autograph.operators.logical import u_add
 from tensorflow.python.autograph.operators.logical import u_sub
 from tensorflow.python.autograph.operators.py_builtins import float_
 from tensorflow.python.autograph.operators.py_builtins import int_
diff --git a/tensorflow/python/autograph/operators/logical.py b/tensorflow/python/autograph/operators/logical.py
index 569db5b91b..dadb0daf1a 100644
--- a/tensorflow/python/autograph/operators/logical.py
+++ b/tensorflow/python/autograph/operators/logical.py
@@ -18,6 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import operator
+
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import gen_math_ops
@@ -35,7 +37,7 @@ def and_(a, b):
   a_val = a()
   if tensor_util.is_tensor(a_val):
     return _tf_lazy_and(a_val, b)
-  return _py_lazy_and(a_val, b)
+  return a_val and b()
 
 
 def _tf_lazy_and(cond, b):
@@ -44,17 +46,12 @@ def _tf_lazy_and(cond, b):
   return control_flow_ops.cond(cond, b, lambda: cond)
 
 
-def _py_lazy_and(cond, b):
-  """Lazy-eval equivalent of "and" in Python."""
-  return cond and b()
-
-
 def or_(a, b):
   """Functional form of "or". Uses lazy evaluation semantics."""
   a_val = a()
   if tensor_util.is_tensor(a_val):
     return _tf_lazy_or(a_val, b)
-  return _py_lazy_or(a_val, b)
+  return a_val or b()
 
 
 def _tf_lazy_or(cond, b):
@@ -63,16 +60,11 @@ def _tf_lazy_or(cond, b):
   return control_flow_ops.cond(cond, lambda: cond, b)
 
 
-def _py_lazy_or(cond, b):
-  """Lazy-eval equivalent of "or" in Python."""
-  return cond or b()
-
-
 def eq(a, b):
   """Functional form of "equal"."""
   if tensor_util.is_tensor(a) or tensor_util.is_tensor(b):
     return _tf_equal(a, b)
-  return _py_equal(a, b)
+  return a == b
 
 
 def _tf_equal(a, b):
@@ -80,11 +72,6 @@ def _tf_equal(a, b):
   return gen_math_ops.equal(a, b)
 
 
-def _py_equal(a, b):
-  """Overload of "equal" that falls back to Python's default implementation."""
-  return a == b
-
-
 def not_eq(a, b):
   """Functional form of "not-equal"."""
   return not_(eq(a, b))
@@ -92,25 +79,8 @@ def not_eq(a, b):
 
 # Default implementation for the remainings.
 
-
-def gt(a, b):
-  """Functional form of "less-than"."""
-  return a > b
-
-
-def gt_e(a, b):
-  """Functional form of "less-than"."""
-  return a >= b
-
-
-def is_(a, b):
-  """Functional form of "less-than"."""
-  return a is b
-
-
-def is_not(a, b):
-  """Functional form of "less-than"."""
-  return a is not b
+is_ = operator.is_
+is_not = operator.is_not
 
 
 def in_(a, b):
@@ -119,21 +89,16 @@ def in_(a, b):
   return a in b
 
 
-def lt(a, b):
-  """Functional form of "less-than"."""
-  return a < b
-
-
-def lt_e(a, b):
-  """Functional form of "less-than"."""
-  return a <= b
-
-
 def not_in(a, b):
   """Functional form of "less-than"."""
   return a not in b
 
+gt = operator.gt
+gt_e = operator.ge
+lt = operator.lt
+lt_e = operator.le
+
 
-def u_sub(a):
-  """Functional form of "unary-sub"."""
-  return -a
+u_add = operator.pos
+u_sub = operator.neg
+invert = operator.invert
-- 
GitLab


From dfad4e97b2d00df768e9e8df16179a6f7ceda43c Mon Sep 17 00:00:00 2001
From: James Keeling <jtkeeling@google.com>
Date: Thu, 3 Jan 2019 09:08:14 -0800
Subject: [PATCH 0166/1095] Fix odd number of backticks in docstring.

This was causing the documentation to render incorrectly.

PiperOrigin-RevId: 227698008
---
 tensorflow/python/ops/linalg/linear_operator_kronecker.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/linalg/linear_operator_kronecker.py b/tensorflow/python/ops/linalg/linear_operator_kronecker.py
index f7e785caa5..005b9b429b 100644
--- a/tensorflow/python/ops/linalg/linear_operator_kronecker.py
+++ b/tensorflow/python/ops/linalg/linear_operator_kronecker.py
@@ -71,7 +71,7 @@ class LinearOperatorKronecker(linear_operator.LinearOperator):
   `op1 x op2 x .. opJ` (we omit parentheses as the Kronecker product is
   associative).
 
-  If `opj` has shape `batch_shape_j` + [M_j, N_j`, then the composed operator
+  If `opj` has shape `batch_shape_j + [M_j, N_j]`, then the composed operator
   will have shape equal to `broadcast_batch_shape + [prod M_j, prod N_j]`,
   where the product is over all operators.
 
-- 
GitLab


From d04b09d65653bb78ad3b14448bf9ef867d9041d4 Mon Sep 17 00:00:00 2001
From: Blake Hechtman <blakehechtman@google.com>
Date: Thu, 3 Jan 2019 09:19:36 -0800
Subject: [PATCH 0167/1095] [XLA] Propagate the layout of transposes and
 reshapes that add one sized dimensions in a depth first order.

PiperOrigin-RevId: 227699358
---
 .../compiler/xla/service/layout_assignment.cc | 23 +++++++++++++++----
 1 file changed, 19 insertions(+), 4 deletions(-)

diff --git a/tensorflow/compiler/xla/service/layout_assignment.cc b/tensorflow/compiler/xla/service/layout_assignment.cc
index 1c4ac178f0..ebf7a88fae 100644
--- a/tensorflow/compiler/xla/service/layout_assignment.cc
+++ b/tensorflow/compiler/xla/service/layout_assignment.cc
@@ -1236,6 +1236,23 @@ Status LayoutAssignment::PropagateUseConstraintToDefs(
       });
 }
 
+namespace {
+// A transpose or a reshape that only changes trivial dimensions have meaningful
+// layouts that are valuable to propagate in a depthfirst manner to avoid
+// unassinged layouts in the graph.
+bool InstructionShouldPropagateDepthFirst(const HloInstruction& hlo) {
+  switch (hlo.opcode()) {
+    case HloOpcode::kReshape:
+      return std::get<0>(hlo.ReshapeMerelyInsertsOrDeletes1SizedDimensions());
+    case HloOpcode::kTranspose:
+      return true;
+    default:
+      return false;
+  }
+}
+
+}  // namespace
+
 Status LayoutAssignment::PropagateOperandConstraint(
     const OperandLayoutConstraint& operand_constraint,
     LayoutConstraints* constraints) {
@@ -1370,7 +1387,7 @@ Status LayoutAssignment::PropagateOperandConstraint(
             TF_RETURN_IF_ERROR(constraints->SetBufferLayout(
                 *layout, *buffer,
                 /*mandatory=*/user->opcode() == HloOpcode::kReduce,
-                /*dfs=*/false));
+                /*dfs=*/InstructionShouldPropagateDepthFirst(*user)));
           }
         }
         return Status::OK();
@@ -1420,11 +1437,9 @@ Status LayoutAssignment::PropagateBufferConstraintToOperands(
             ChooseOperandLayoutFromOutputLayout(buffer_constraint.layout(),
                                                 instruction, operand_no);
         if (operand_layout != nullptr) {
-          // Do not propagate operand constraints of transposes and reshapes, it
-          // tends to create really bad layouts.
           TF_RETURN_IF_ERROR(constraints->SetArrayOperandLayout(
               *operand_layout, instruction, operand_no, /*mandatory=*/false,
-              /*dfs=*/false));
+              /*dfs=*/InstructionShouldPropagateDepthFirst(*instruction)));
         }
       } else {
         VLOG(6) << "Operand already has a constraint "
-- 
GitLab


From a5b70b7cb46e18846ba8875a13da08e7444c5136 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Thu, 3 Jan 2019 09:20:12 -0800
Subject: [PATCH 0168/1095] Fix for an optimizer.get_config() exception when
 exporting a SavedModel

Tests tf.saved_model.save() with Optimizerv2 instead of v1

PiperOrigin-RevId: 227699428
---
 tensorflow/python/keras/backend.py         | 5 +++++
 tensorflow/python/saved_model/save_test.py | 6 +++---
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/keras/backend.py b/tensorflow/python/keras/backend.py
index 42d94e77a0..6693243a68 100644
--- a/tensorflow/python/keras/backend.py
+++ b/tensorflow/python/keras/backend.py
@@ -2794,6 +2794,11 @@ def get_value(x):
   """
   if context.executing_eagerly():
     return x.numpy()
+  elif not getattr(x, '_in_graph_mode', True):
+    # This is a variable which was created in an eager context, but is being
+    # evaluated from a Graph.
+    with context.eager_mode():
+      return x.numpy()
   elif ops.inside_function():
     raise RuntimeError('Cannot get value inside Tensorflow graph function.')
   return x.eval(session=get_session())
diff --git a/tensorflow/python/saved_model/save_test.py b/tensorflow/python/saved_model/save_test.py
index e643ff3dbf..533b954190 100644
--- a/tensorflow/python/saved_model/save_test.py
+++ b/tensorflow/python/saved_model/save_test.py
@@ -41,7 +41,7 @@ from tensorflow.python.saved_model import loader
 from tensorflow.python.saved_model import save
 from tensorflow.python.saved_model import signature_constants
 from tensorflow.python.saved_model import tag_constants
-from tensorflow.python.training import adam
+from tensorflow.python.keras.optimizer_v2 import adam
 from tensorflow.python.training.checkpointable import tracking
 from tensorflow.python.training.checkpointable import util
 
@@ -50,7 +50,7 @@ class _ModelWithOptimizer(util.Checkpoint):
 
   def __init__(self):
     self.dense = core.Dense(1)
-    self.optimizer = adam.AdamOptimizer(0.01)
+    self.optimizer = adam.Adam(0.01)
 
   @def_function.function(
       input_signature=(tensor_spec.TensorSpec([None, 2], dtypes.float32),
@@ -401,7 +401,7 @@ class _ModelWithOptimizerUsingDefun(util.Checkpoint):
 
   def __init__(self):
     self.dense = core.Dense(1)
-    self.optimizer = adam.AdamOptimizer(0.01)
+    self.optimizer = adam.Adam(0.01)
 
   # Using defun due to control flow v2 cycles, b/121159261. def_function uses
   # conds to gate variable initialization and so triggers cond reference cycles,
-- 
GitLab


From 78acbc6ee7fa557504599d7847bcd96a9a977d40 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Thu, 3 Jan 2019 09:21:25 -0800
Subject: [PATCH 0169/1095] TFTS: Remove flaky assertions

PiperOrigin-RevId: 227699591
---
 tensorflow/contrib/timeseries/examples/predict_test.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/tensorflow/contrib/timeseries/examples/predict_test.py b/tensorflow/contrib/timeseries/examples/predict_test.py
index 678fd71cd8..b353f85cb5 100644
--- a/tensorflow/contrib/timeseries/examples/predict_test.py
+++ b/tensorflow/contrib/timeseries/examples/predict_test.py
@@ -43,10 +43,6 @@ class PeriodTrendExampleTest(test.TestCase):
     self.assertAllEqual([700], mean.shape)
     self.assertAllEqual([700], upper_limit.shape)
     self.assertAllEqual([700], lower_limit.shape)
-    # Check that variance hasn't blown up too much. This is a relatively good
-    # indication that training was successful.
-    self.assertLess(upper_limit[-1] - lower_limit[-1],
-                    1.5 * (upper_limit[0] - lower_limit[0]))
 
   def test_ar(self):
     (times, observed, all_times, mean,
@@ -55,7 +51,6 @@ class PeriodTrendExampleTest(test.TestCase):
     self.assertAllEqual(all_times.shape, mean.shape)
     self.assertAllEqual(all_times.shape, upper_limit.shape)
     self.assertAllEqual(all_times.shape, lower_limit.shape)
-    self.assertLess((upper_limit - lower_limit).mean(), 4.)
 
 
 if __name__ == "__main__":
-- 
GitLab


From c2ec7217a8b122d6a561da5d7d96b94ede9b7684 Mon Sep 17 00:00:00 2001
From: Jianwei Xie <xiejw@google.com>
Date: Thu, 3 Jan 2019 09:53:37 -0800
Subject: [PATCH 0170/1095] Add model saving in Resnet50 code.

In addition,
1. Optimized the weights saving logic in Keras for TPU.
2. Fixed the bug in optimizer for TPU mirrored variable.

PiperOrigin-RevId: 227704010
---
 tensorflow/python/keras/engine/saving.py             | 11 ++++++++++-
 tensorflow/python/keras/optimizer_v2/BUILD           |  1 +
 tensorflow/python/keras/optimizer_v2/optimizer_v2.py |  4 +++-
 3 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/keras/engine/saving.py b/tensorflow/python/keras/engine/saving.py
index 91eba0acab..0604a389a9 100644
--- a/tensorflow/python/keras/engine/saving.py
+++ b/tensorflow/python/keras/engine/saving.py
@@ -25,6 +25,7 @@ import os
 import numpy as np
 from six.moves import zip  # pylint: disable=redefined-builtin
 
+from tensorflow.python.framework import ops
 from tensorflow.python.keras import backend as K
 from tensorflow.python.keras import optimizers
 from tensorflow.python.keras.utils import conv_utils
@@ -736,9 +737,17 @@ def save_weights_to_hdf5_group(f, layers):
   f.attrs['backend'] = K.backend().encode('utf8')
   f.attrs['keras_version'] = str(keras_version).encode('utf8')
 
+  # On TPUs, modifying the graph between session.runs() triggers some expensive
+  # recompilation overhead. To avoid this, we build up the full set of tensors
+  # to save before fetching weights, thus only modifying the graph once.
+  layer_weights_dict = {}
+  for layer in layers:
+    layer_weights_dict[layer.name] = [ops.convert_to_tensor(w)
+                                      for w in layer.weights]
+
   for layer in layers:
     g = f.create_group(layer.name)
-    symbolic_weights = layer.weights
+    symbolic_weights = layer_weights_dict[layer.name]
     weight_values = K.batch_get_value(symbolic_weights)
     weight_names = []
     for i, (w, val) in enumerate(zip(symbolic_weights, weight_values)):
diff --git a/tensorflow/python/keras/optimizer_v2/BUILD b/tensorflow/python/keras/optimizer_v2/BUILD
index 05f6e5ac48..964cf5fcba 100644
--- a/tensorflow/python/keras/optimizer_v2/BUILD
+++ b/tensorflow/python/keras/optimizer_v2/BUILD
@@ -34,6 +34,7 @@ py_library(
         "//tensorflow/python:variable_scope",
         "//tensorflow/python:variables",
         "//tensorflow/python/distribute:reduce_util",
+        "//tensorflow/python/distribute:values",
     ],
 )
 
diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
index b740b2f05b..98f87a41ae 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
@@ -28,6 +28,7 @@ import six
 from tensorflow.python.distribute import distribute_lib
 from tensorflow.python.distribute import distribution_strategy_context as distribute_ctx
 from tensorflow.python.distribute import reduce_util as ds_reduce_util
+from tensorflow.python.distribute import values as distributed_values
 from tensorflow.python.eager import backprop
 from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes
@@ -576,7 +577,8 @@ class OptimizerV2(checkpointable.CheckpointableBase):
     value = self._get_hyper(hyperparameter_name)
     if callable(value):
       return value()
-    if isinstance(value, (ops.Tensor, tf_variables.Variable)):
+    if isinstance(value, (ops.Tensor, tf_variables.Variable,
+                          distributed_values.TPUMirroredVariable)):
       return backend.get_value(value)
     return value
 
-- 
GitLab


From 3e2408a52822439b04de0a0a070a200c8f15509b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 3 Jan 2019 10:34:07 -0800
Subject: [PATCH 0171/1095] Modify NNAPI delegate SVDF test to use 0 bias.

PiperOrigin-RevId: 227710985
---
 tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc b/tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc
index 2da14cc1de..5da052eb42 100644
--- a/tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc
+++ b/tensorflow/lite/delegates/nnapi/nnapi_delegate_test.cc
@@ -2003,7 +2003,9 @@ class BaseSVDFOpModel : public SingleOpModelWithNNAPI {
     input_ = AddInput(TensorType_FLOAT32);
     weights_feature_ = AddInput(weights_feature_type);
     weights_time_ = AddInput(weights_time_type);
-    bias_ = AddNullInput();
+    // TODO(b/121383394) : figure out why optional bias causes TFLite segfault
+    // when using NNAPI delegate.
+    bias_ = AddInput(TensorType_FLOAT32);
     const int num_filters = units * rank;
     activation_state_ = AddInput(
         TensorData{TensorType_FLOAT32, {batches, memory_size * num_filters}},
@@ -2019,6 +2021,8 @@ class BaseSVDFOpModel : public SingleOpModelWithNNAPI {
         {units_},                             // bias tensor
         {batches, memory_size * num_filters}  // activation_state tensor
     });
+    // TODO(b/121383394) : remove once the optional bias bug is fixed.
+    PopulateTensor(bias_, std::vector<float>(units_));
   }
 
   // Populates the weights_feature tensor.
-- 
GitLab


From 140f0a3cde3696ca5128322cffda178563c8d04f Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Thu, 3 Jan 2019 10:35:02 -0800
Subject: [PATCH 0172/1095] Finish migration to the new scope based API for
 using distribution strategy.

PiperOrigin-RevId: 227711127
---
 tensorflow/contrib/distribute/README.md       | 30 ++++++++-----------
 .../distribute/python/examples/keras_mnist.py | 19 ++++++------
 2 files changed, 21 insertions(+), 28 deletions(-)

diff --git a/tensorflow/contrib/distribute/README.md b/tensorflow/contrib/distribute/README.md
index 8a8dc159ad..dbcaf8185f 100644
--- a/tensorflow/contrib/distribute/README.md
+++ b/tensorflow/contrib/distribute/README.md
@@ -43,28 +43,19 @@ the workers.
 
 Let's see how to scale to multiple GPUs on one machine using `MirroredStrategy` with [tf.keras] (https://www.tensorflow.org/guide/keras).
 
-Take a very simple model consisting of a single layer:
+Let's define a simple input dataset for training this model. Note that currently we require using
+[`tf.data.Dataset`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset)
+with `DistributionStrategy`.
 
 ```python
 import tensorflow as tf
 from tensorflow import keras
 
-inputs = tf.keras.layers.Input(shape=(1,))
-predictions = tf.keras.layers.Dense(1)(inputs)
-model = tf.keras.models.Model(inputs=inputs, outputs=predictions)
-```
-
-Let's also define a simple input dataset for training this model. Note that currently we require using
-[`tf.data.Dataset`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset)
-with `DistributionStrategy`.
-
-```python
 features = tf.data.Dataset.from_tensors([1.]).repeat(10000).batch(10)
 labels = tf.data.Dataset.from_tensors([1.]).repeat(10000).batch(10)
 train_dataset = tf.data.Dataset.zip((features, labels))
 ```
 
-
 To distribute this Keras model on multiple GPUs using `MirroredStrategy` we
 first instantiate a `MirroredStrategy` object.
 
@@ -72,14 +63,17 @@ first instantiate a `MirroredStrategy` object.
 distribution = tf.contrib.distribute.MirroredStrategy()
 ```
 
-We then compile the Keras model and pass the `MirroredStrategy` object in the
-`distribute` argument (apart from other usual arguments like `loss` and
-`optimizer`).
+Take a very simple model consisting of a single layer. We need to create and compile
+the model under the distribution strategy scope.
 
 ```python
-model.compile(loss='mean_squared_error',
-              optimizer=tf.train.GradientDescentOptimizer(learning_rate=0.2),
-              distribute=distribution)
+with distribution.scope():
+  inputs = tf.keras.layers.Input(shape=(1,))
+  predictions = tf.keras.layers.Dense(1)(inputs)
+  model = tf.keras.models.Model(inputs=inputs, outputs=predictions)
+
+  model.compile(loss='mean_squared_error',
+                optimizer=tf.train.GradientDescentOptimizer(learning_rate=0.2))
 ```
 
 To train the model we call Keras `fit` API using the input dataset that we
diff --git a/tensorflow/contrib/distribute/python/examples/keras_mnist.py b/tensorflow/contrib/distribute/python/examples/keras_mnist.py
index 60fda99664..1ce91ecaf2 100644
--- a/tensorflow/contrib/distribute/python/examples/keras_mnist.py
+++ b/tensorflow/contrib/distribute/python/examples/keras_mnist.py
@@ -109,22 +109,21 @@ def main(_):
   tf.enable_eager_execution()
 
   train_ds, eval_ds, input_shape = get_input_datasets()
-  model = get_model(input_shape)
 
   # Instantiate the MirroredStrategy object. If we don't specify `num_gpus` or
   # the `devices` argument then all the GPUs available on the machine are used.
   # TODO(priyag): Use `tf.distribute.MirroredStrategy` once available.
   strategy = mirrored_strategy.MirroredStrategy(['/gpu:0', '/cpu:0'])
 
-  optimizer = rmsprop.RMSProp(learning_rate=0.001)
-
-  # Compile the model by passing the distribution strategy object to the
-  # `distribute` argument. `fit`, `evaluate` and `predict` will be distributed
-  # based on the strategy instantiated.
-  model.compile(loss=tf.keras.losses.categorical_crossentropy,
-                optimizer=optimizer,
-                metrics=['accuracy'],
-                distribute=strategy)
+  # Create and compile the model under Distribution strategy scope.
+  # `fit`, `evaluate` and `predict` will be distributed based on the strategy
+  # model was compiled with.
+  with strategy.scope():
+    model = get_model(input_shape)
+    optimizer = rmsprop.RMSProp(learning_rate=0.001)
+    model.compile(loss=tf.keras.losses.categorical_crossentropy,
+                  optimizer=optimizer,
+                  metrics=['accuracy'])
 
   # Train the model with the train dataset.
   model.fit(x=train_ds, epochs=20, steps_per_epoch=468)
-- 
GitLab


From 64596a68235709e7f3246d0b120a4146521a998b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 3 Jan 2019 10:37:35 -0800
Subject: [PATCH 0173/1095] No public changes.

PiperOrigin-RevId: 227711548
---
 tensorflow/lite/experimental/swift/BUILD      | 100 +++++
 tensorflow/lite/experimental/swift/LICENSE    | 202 ++++++++++
 tensorflow/lite/experimental/swift/README.md  |  54 +++
 .../swift/Sources/Interpreter.swift           | 265 ++++++++++++++
 .../swift/Sources/InterpreterError.swift      |  99 +++++
 .../swift/Sources/InterpreterOptions.swift    |  29 ++
 .../experimental/swift/Sources/Model.swift    |  40 ++
 .../Sources/QuantizationParameters.swift      |  38 ++
 .../experimental/swift/Sources/Tensor.swift   | 138 +++++++
 .../Configs/TensorFlowLite.tulsigen           |  57 +++
 .../project.tulsiconf                         |  14 +
 .../project.pbxproj                           | 345 ++++++++++++++++++
 .../TensorFlowLiteApp/AppDelegate.swift       |  24 ++
 .../Array+TensorFlowLite.swift                |  22 ++
 .../AppIcon.appiconset/Contents.json          |  98 +++++
 .../Assets.xcassets/Contents.json             |   6 +
 .../Base.lproj/LaunchScreen.storyboard        |  44 +++
 .../Base.lproj/Main.storyboard                |  95 +++++
 .../Data+TensorFlowLite.swift                 |  13 +
 .../TensorFlowLiteApp/Info.plist              |  46 +++
 .../TensorFlowLiteApp/ViewController.swift    | 299 +++++++++++++++
 .../swift/Tests/InterpreterOptionsTests.swift |  54 +++
 .../swift/Tests/InterpreterTests.swift        | 315 ++++++++++++++++
 .../experimental/swift/Tests/ModelTests.swift |  59 +++
 .../Tests/QuantizationParametersTests.swift   |  43 +++
 .../swift/Tests/TensorTests.swift             |  83 +++++
 .../tools/pip_package/pip_smoke_test.py       |  36 +-
 27 files changed, 2603 insertions(+), 15 deletions(-)
 create mode 100644 tensorflow/lite/experimental/swift/BUILD
 create mode 100644 tensorflow/lite/experimental/swift/LICENSE
 create mode 100644 tensorflow/lite/experimental/swift/README.md
 create mode 100644 tensorflow/lite/experimental/swift/Sources/Interpreter.swift
 create mode 100644 tensorflow/lite/experimental/swift/Sources/InterpreterError.swift
 create mode 100644 tensorflow/lite/experimental/swift/Sources/InterpreterOptions.swift
 create mode 100644 tensorflow/lite/experimental/swift/Sources/Model.swift
 create mode 100644 tensorflow/lite/experimental/swift/Sources/QuantizationParameters.swift
 create mode 100644 tensorflow/lite/experimental/swift/Sources/Tensor.swift
 create mode 100644 tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/Configs/TensorFlowLite.tulsigen
 create mode 100644 tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/project.tulsiconf
 create mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp.xcodeproj/project.pbxproj
 create mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/AppDelegate.swift
 create mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Array+TensorFlowLite.swift
 create mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/AppIcon.appiconset/Contents.json
 create mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/Contents.json
 create mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/LaunchScreen.storyboard
 create mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/Main.storyboard
 create mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Data+TensorFlowLite.swift
 create mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Info.plist
 create mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/ViewController.swift
 create mode 100644 tensorflow/lite/experimental/swift/Tests/InterpreterOptionsTests.swift
 create mode 100644 tensorflow/lite/experimental/swift/Tests/InterpreterTests.swift
 create mode 100644 tensorflow/lite/experimental/swift/Tests/ModelTests.swift
 create mode 100644 tensorflow/lite/experimental/swift/Tests/QuantizationParametersTests.swift
 create mode 100644 tensorflow/lite/experimental/swift/Tests/TensorTests.swift

diff --git a/tensorflow/lite/experimental/swift/BUILD b/tensorflow/lite/experimental/swift/BUILD
new file mode 100644
index 0000000000..3bd288a1e1
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/BUILD
@@ -0,0 +1,100 @@
+# TensorFlow Lite for Swift.
+
+package(default_visibility = ["//visibility:private"])
+
+licenses(["notice"])  # Apache 2.0
+
+exports_files(["LICENSE"])
+
+load("@build_bazel_rules_apple//apple:ios.bzl", "ios_application", "ios_unit_test")
+load("@build_bazel_rules_swift//swift:swift.bzl", "swift_library")
+
+MINIMUM_OS_VERSION = "9.0"
+
+SWIFT_COPTS = [
+    "-wmo",
+]
+
+swift_library(
+    name = "TensorFlowLite",
+    srcs = glob(["Sources/*.swift"]),
+    copts = SWIFT_COPTS,
+    module_name = "TensorFlowLite",
+    tags = ["manual"],
+    deps = [
+        "//tensorflow/lite/experimental/c:c_api",
+    ],
+)
+
+ios_unit_test(
+    name = "TensorFlowLiteTests",
+    size = "small",
+    minimum_os_version = MINIMUM_OS_VERSION,
+    tags = [
+        "manual",
+        # DISABLED: Following sanitizer tests are not supported by iOS test targets.
+        "noasan",
+        "nomsan",
+        "notsan",
+    ],
+    deps = [":TensorFlowLiteTestsLib"],
+)
+
+swift_library(
+    name = "TensorFlowLiteTestsLib",
+    testonly = 1,
+    srcs = glob(["Tests/*.swift"]),
+    copts = SWIFT_COPTS,
+    tags = ["manual"],
+    deps = [
+        ":TensorFlowLite",
+        ":TestResources",
+    ],
+)
+
+objc_library(
+    name = "TestResources",
+    resources = [
+        "//tensorflow/lite:testdata/add.bin",
+        "//tensorflow/lite:testdata/add_quantized.bin",
+        "//tensorflow/lite:testdata/multi_add.bin",
+    ],
+    tags = ["manual"],
+)
+
+ios_application(
+    name = "TensorFlowLiteApp",
+    app_icons = glob(["TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/AppIcon.appiconset/**"]),
+    bundle_id = "com.tensorflow.lite.swift.TensorFlowLite",
+    families = [
+        "ipad",
+        "iphone",
+    ],
+    infoplists = ["TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Info.plist"],
+    launch_storyboard = "TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/LaunchScreen.storyboard",
+    minimum_os_version = MINIMUM_OS_VERSION,
+    sdk_frameworks = [
+        "CoreGraphics",
+    ],
+    tags = ["manual"],
+    deps = [":TensorFlowLiteAppLib"],
+)
+
+swift_library(
+    name = "TensorFlowLiteAppLib",
+    srcs = glob(["TestApps/TensorFlowLiteApp/TensorFlowLiteApp/*.swift"]),
+    tags = ["manual"],
+    deps = [
+        ":TensorFlowLite",
+        ":TensorFlowLiteAppResources",
+    ],
+)
+
+objc_library(
+    name = "TensorFlowLiteAppResources",
+    storyboards = glob([
+        "TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/*.storyboard",
+    ]),
+    tags = ["manual"],
+    deps = [":TestResources"],
+)
diff --git a/tensorflow/lite/experimental/swift/LICENSE b/tensorflow/lite/experimental/swift/LICENSE
new file mode 100644
index 0000000000..d645695673
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/LICENSE
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/tensorflow/lite/experimental/swift/README.md b/tensorflow/lite/experimental/swift/README.md
new file mode 100644
index 0000000000..deb16c1e91
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/README.md
@@ -0,0 +1,54 @@
+# TensorFlow Lite for Swift
+
+[TensorFlow Lite](https://www.tensorflow.org/lite/) is TensorFlow's lightweight
+solution for Swift developers. It enables low-latency inference of on-device
+machine learning models with a small binary size and fast performance supporting
+hardware acceleration.
+
+## Getting Started
+
+### Bazel
+
+In your `BUILD` file, add the `TensorFlowLite` dependency:
+
+```python
+swift_library(
+  # ...
+  deps = [
+      "//tensorflow/lite/swift:TensorFlowLite",
+  ],
+)
+```
+
+In your Swift files, import the module:
+
+```swift
+import TensorFlowLite
+```
+
+### Tulsi
+
+Open the `TensorFlowLite.tulsiproj` using the [TulsiApp](https://github.com/bazelbuild/tulsi) or by
+running the [`generate_xcodeproj.sh`](https://github.com/bazelbuild/tulsi/blob/master/src/tools/generate_xcodeproj.sh)
+script:
+
+```shell
+generate_xcodeproj.sh --genconfig tensorflow/lite/swift/TensorFlowLite.tulsiproj:TensorFlowLite --outputfolder ~/path/to/generated/TensorFlowLite.xcodeproj
+```
+
+### CocoaPods
+
+Add the following to your `Podfile`:
+
+```ruby
+use_frameworks!
+pod 'TensorFlowLiteSwift'
+```
+
+Then, run `pod install`.
+
+In your Swift files, import the module:
+
+```swift
+import TensorFlowLite
+```
diff --git a/tensorflow/lite/experimental/swift/Sources/Interpreter.swift b/tensorflow/lite/experimental/swift/Sources/Interpreter.swift
new file mode 100644
index 0000000000..a14b5966b1
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/Sources/Interpreter.swift
@@ -0,0 +1,265 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Foundation
+import TensorFlowLiteCAPI
+
+/// A TensorFlow Lite interpreter that performs inference from a given model.
+public final class Interpreter {
+
+  /// The `TFL_Interpreter` C pointer type represented as an `UnsafePointer<TFL_Interpreter>`.
+  private typealias CInterpreter = OpaquePointer
+
+  /// Total number of input tensors associated with the model.
+  public var inputTensorCount: Int {
+    return Int(TFL_InterpreterGetInputTensorCount(cInterpreter))
+  }
+
+  /// Total number of output tensors associated with the model.
+  public var outputTensorCount: Int {
+    return Int(TFL_InterpreterGetOutputTensorCount(cInterpreter))
+  }
+
+  /// The underlying `TFL_Interpreter` C pointer.
+  private var cInterpreter: CInterpreter?
+
+  /// Creates a new model interpreter instance.
+  ///
+  /// - Parameters:
+  ///   - modelPath: Local file path to a TensorFlow Lite model.
+  ///   - options: Custom configurations for the interpreter. The default is `nil` indicating that
+  ///       interpreter will determine the configuration options.
+  /// - Throws: An error if the model could not be loaded or the interpreter could not be created.
+  public init(modelPath: String, options: InterpreterOptions? = nil) throws {
+    guard let model = Model(filePath: modelPath) else { throw InterpreterError.failedToLoadModel }
+
+    let cInterpreterOptions: OpaquePointer? = try options.map { options in
+      guard let cOptions = TFL_NewInterpreterOptions() else {
+        throw InterpreterError.failedToCreateInterpreter
+      }
+      if let threadCount = options.threadCount, threadCount > 0 {
+        TFL_InterpreterOptionsSetNumThreads(cOptions, Int32(threadCount))
+      }
+      if options.isErrorLoggingEnabled {
+        TFL_InterpreterOptionsSetErrorReporter(
+          cOptions,
+          { (_, format, arguments) in
+            guard let cFormat = format,
+                  let message = String(cFormat: cFormat, arguments: arguments)
+            else {
+              return
+            }
+            print(String(describing: InterpreterError.tensorFlowLiteError(message)))
+          },
+          nil
+        )
+      }
+      return cOptions
+    }
+    defer { TFL_DeleteInterpreterOptions(cInterpreterOptions) }
+
+    guard let cInterpreter = TFL_NewInterpreter(model.cModel, cInterpreterOptions) else {
+      throw InterpreterError.failedToCreateInterpreter
+    }
+    self.cInterpreter = cInterpreter
+  }
+
+  deinit {
+    TFL_DeleteInterpreter(cInterpreter)
+  }
+
+  /// Invokes the interpreter to perform inference from the loaded graph.
+  ///
+  /// - Throws: An error if the model was not ready because tensors were not allocated.
+  public func invoke() throws {
+    guard TFL_InterpreterInvoke(cInterpreter) == kTfLiteOk else {
+      // TODO(b/117510052): Determine which error to throw.
+      throw InterpreterError.allocateTensorsRequired
+    }
+  }
+
+  /// Returns the input tensor at the given index.
+  ///
+  /// - Parameters:
+  ///   - index: The index for the input tensor.
+  /// - Throws: An error if the index is invalid or the tensors have not been allocated.
+  /// - Returns: The input tensor at the given index.
+  public func input(at index: Int) throws -> Tensor {
+    let maxIndex = inputTensorCount - 1
+    guard case 0...maxIndex = index else {
+      throw InterpreterError.invalidTensorIndex(index: index, maxIndex: maxIndex)
+    }
+    guard let cTensor = TFL_InterpreterGetInputTensor(cInterpreter, Int32(index)),
+          let bytes = TFL_TensorData(cTensor),
+          let nameCString = TFL_TensorName(cTensor)
+    else {
+      throw InterpreterError.allocateTensorsRequired
+    }
+    guard let dataType = TensorDataType(type: TFL_TensorType(cTensor)) else {
+      throw InterpreterError.invalidTensorDataType
+    }
+
+    let name = String(cString: nameCString)
+    let rank = TFL_TensorNumDims(cTensor)
+    let dimensions = (0..<rank).map { Int(TFL_TensorDim(cTensor, $0)) }
+    let shape = TensorShape(dimensions)
+    let byteCount = TFL_TensorByteSize(cTensor)
+    let data = Data(bytes: bytes, count: byteCount)
+    let cQuantizationParams = TFL_TensorQuantizationParams(cTensor)
+    let scale = cQuantizationParams.scale
+    let zeroPoint = Int(cQuantizationParams.zero_point)
+    var quantizationParameters: QuantizationParameters? = nil
+    if scale != 0.0 {
+      // TODO(b/117510052): Update this check once the TfLiteQuantizationParams struct has a mode.
+      quantizationParameters = QuantizationParameters(scale: scale, zeroPoint: zeroPoint)
+    }
+    let tensor = Tensor(
+      name: name,
+      dataType: dataType,
+      shape: shape,
+      data: data,
+      quantizationParameters: quantizationParameters
+    )
+    return tensor
+  }
+
+  /// Returns the output tensor at the given index.
+  ///
+  /// - Parameters:
+  ///   - index: The index for the output tensor.
+  /// - Throws: An error if the index is invalid, tensors haven't been allocated, or interpreter
+  ///     hasn't been invoked for models that dynamically compute output tensors based on the values
+  ///     of its input tensors.
+  /// - Returns: The output tensor at the given index.
+  public func output(at index: Int) throws -> Tensor {
+    let maxIndex = outputTensorCount - 1
+    guard case 0...maxIndex = index else {
+      throw InterpreterError.invalidTensorIndex(index: index, maxIndex: maxIndex)
+    }
+    guard let cTensor = TFL_InterpreterGetOutputTensor(cInterpreter, Int32(index)),
+          let bytes = TFL_TensorData(cTensor),
+          let nameCString = TFL_TensorName(cTensor)
+    else {
+      // TODO(b/117510052): Determine which error to throw.
+      throw InterpreterError.invokeInterpreterRequired
+    }
+    guard let dataType = TensorDataType(type: TFL_TensorType(cTensor)) else {
+      throw InterpreterError.invalidTensorDataType
+    }
+
+    let name = String(cString: nameCString)
+    let rank = TFL_TensorNumDims(cTensor)
+    let dimensions = (0..<rank).map { Int(TFL_TensorDim(cTensor, $0)) }
+    let shape = TensorShape(dimensions)
+    let byteCount = TFL_TensorByteSize(cTensor)
+    let data = Data(bytes: bytes, count: byteCount)
+    let cQuantizationParams = TFL_TensorQuantizationParams(cTensor)
+    let scale = cQuantizationParams.scale
+    let zeroPoint = Int(cQuantizationParams.zero_point)
+    var quantizationParameters: QuantizationParameters? = nil
+    if scale != 0.0 {
+      // TODO(b/117510052): Update this check once the TfLiteQuantizationParams struct has a mode.
+      quantizationParameters = QuantizationParameters(scale: scale, zeroPoint: zeroPoint)
+    }
+    let tensor = Tensor(
+      name: name,
+      dataType: dataType,
+      shape: shape,
+      data: data,
+      quantizationParameters: quantizationParameters
+    )
+    return tensor
+  }
+
+  /// Resizes the input tensor at the given index to the specified tensor shape.
+  ///
+  /// - Note: After resizing an input tensor, the client **must** explicitly call
+  ///     `allocateTensors()` before attempting to access the resized tensor data or invoking the
+  ///     interpreter to perform inference.
+  /// - Parameters:
+  ///   - index: The index for the input tensor.
+  ///   - shape: The shape that the input tensor should be resized to.
+  /// - Throws: An error if the input tensor at the given index could not be resized.
+  public func resizeInput(at index: Int, to shape: TensorShape) throws {
+    let maxIndex = inputTensorCount - 1
+    guard case 0...maxIndex = index else {
+      throw InterpreterError.invalidTensorIndex(index: index, maxIndex: maxIndex)
+    }
+    guard TFL_InterpreterResizeInputTensor(
+            cInterpreter,
+            Int32(index),
+            shape.int32Dimensions,
+            Int32(shape.rank)
+          ) == kTfLiteOk
+    else {
+      throw InterpreterError.failedToResizeInputTensor(index: index)
+    }
+  }
+
+  /// Copies the given data to the input tensor at the given index.
+  ///
+  /// - Parameters:
+  ///   - data: The data to be copied to the input tensor's data buffer.
+  ///   - index: The index for the input tensor.
+  /// - Throws: An error if the `data.count` does not match the input tensor's `data.count` or if
+  ///     the given index is invalid.
+  /// - Returns: The input tensor with the copied data.
+  @discardableResult
+  public func copy(_ data: Data, toInputAt index: Int) throws -> Tensor {
+    let maxIndex = inputTensorCount - 1
+    guard case 0...maxIndex = index else {
+      throw InterpreterError.invalidTensorIndex(index: index, maxIndex: maxIndex)
+    }
+    guard let cTensor = TFL_InterpreterGetInputTensor(cInterpreter, Int32(index)) else {
+      throw InterpreterError.allocateTensorsRequired
+    }
+
+    let byteCount = TFL_TensorByteSize(cTensor)
+    guard data.count == byteCount else {
+      throw InterpreterError.invalidTensorDataCount(provided: data.count, required: byteCount)
+    }
+
+    let status = data.withUnsafeBytes { TFL_TensorCopyFromBuffer(cTensor, $0, data.count) }
+    guard status == kTfLiteOk else { throw InterpreterError.failedToCopyDataToInputTensor }
+    return try input(at: index)
+  }
+
+  /// Allocates memory for all input tensors based on their `TensorShape`s.
+  ///
+  /// - Note: This is a relatively expensive operation and should only be called after creating the
+  ///     interpreter and/or resizing any input tensors.
+  /// - Throws: An error if memory could not be allocated for the input tensors.
+  public func allocateTensors() throws {
+    guard TFL_InterpreterAllocateTensors(cInterpreter) == kTfLiteOk else {
+      throw InterpreterError.failedToAllocateTensors
+    }
+  }
+}
+
+// MARK: - Extensions
+
+extension String {
+  /// Returns a new `String` initialized by using the given format C array as a template into which
+  /// the remaining argument values are substituted according to the user’s default locale.
+  ///
+  /// - Note: Returns `nil` if a new `String` could not be constructed from the given values.
+  /// - Parameters:
+  ///   - cFormat: The format C array as a template for substituting values.
+  ///   - arguments: A C pointer to a `va_list` of arguments to substitute into `cFormat`.
+  init?(cFormat: UnsafePointer<CChar>, arguments: CVaListPointer) {
+    var buffer: UnsafeMutablePointer<CChar>?
+    guard vasprintf(&buffer, cFormat, arguments) != 0, let cString = buffer else { return nil }
+    self.init(validatingUTF8: cString)
+  }
+}
diff --git a/tensorflow/lite/experimental/swift/Sources/InterpreterError.swift b/tensorflow/lite/experimental/swift/Sources/InterpreterError.swift
new file mode 100644
index 0000000000..5de58b997a
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/Sources/InterpreterError.swift
@@ -0,0 +1,99 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Foundation
+
+/// TensorFlow Lite interpreter errors.
+public enum InterpreterError: Error {
+  case invalidTensorIndex(index: Int, maxIndex: Int)
+  case invalidTensorDataCount(provided: Int, required: Int)
+  case invalidTensorDataType
+  case failedToLoadModel
+  case failedToCreateInterpreter
+  case failedToResizeInputTensor(index: Int)
+  case failedToCopyDataToInputTensor
+  case failedToAllocateTensors
+  case allocateTensorsRequired
+  case invokeInterpreterRequired
+  case tensorFlowLiteError(String)
+}
+
+// MARK: - Extensions
+
+extension InterpreterError: LocalizedError {
+  /// Localized description of the interpreter error.
+  public var errorDescription: String? {
+    switch self {
+    case .invalidTensorIndex(let index, let maxIndex):
+      return "Invalid tensor index \(index), max index is \(maxIndex)."
+    case .invalidTensorDataCount(let providedCount, let requiredCount):
+      return "Provided data count \(providedCount) must match the required count \(requiredCount)."
+    case .invalidTensorDataType:
+      return "Tensor data type is unsupported or could not be determined because of a model error."
+    case .failedToLoadModel:
+      return "Failed to load the given model."
+    case .failedToCreateInterpreter:
+      return "Failed to create the interpreter."
+    case .failedToResizeInputTensor(let index):
+      return "Failed to resize input tesnor at index \(index)."
+    case .failedToCopyDataToInputTensor:
+      return "Failed to copy data to input tensor."
+    case .failedToAllocateTensors:
+      return "Failed to allocate memory for input tensors."
+    case .allocateTensorsRequired:
+      return "Must call allocateTensors()."
+    case .invokeInterpreterRequired:
+      return "Must call invoke()."
+    case .tensorFlowLiteError(let message):
+      return "TensorFlow Lite Error: \(message)"
+    }
+  }
+}
+
+extension InterpreterError: CustomStringConvertible {
+  /// Textual representation of the TensorFlow Lite interpreter error.
+  public var description: String {
+    return errorDescription ?? "Unknown error."
+  }
+}
+
+#if swift(>=4.2)
+extension InterpreterError: Equatable {}
+#else
+extension InterpreterError: Equatable {
+  public static func == (lhs: InterpreterError, rhs: InterpreterError) -> Bool {
+    switch (lhs, rhs) {
+    case (.invalidTensorDataType, .invalidTensorDataType),
+         (.failedToLoadModel, .failedToLoadModel),
+         (.failedToCreateInterpreter, .failedToCreateInterpreter),
+         (.failedToAllocateTensors, .failedToAllocateTensors),
+         (.allocateTensorsRequired, .allocateTensorsRequired),
+         (.invokeInterpreterRequired, .invokeInterpreterRequired):
+      return true
+    case (.invalidTensorIndex(let lhsIndex, let lhsMaxIndex),
+          .invalidTensorIndex(let rhsIndex, let rhsMaxIndex)):
+      return lhsIndex == rhsIndex && lhsMaxIndex == rhsMaxIndex
+    case (.invalidTensorDataCount(let lhsProvidedCount, let lhsRequiredCount),
+          .invalidTensorDataCount(let rhsProvidedCount, let rhsRequiredCount)):
+      return lhsProvidedCount == rhsProvidedCount && lhsRequiredCount == rhsRequiredCount
+    case (.failedToResizeInputTensor(let lhsIndex), .failedToResizeInputTensor(let rhsIndex)):
+      return lhsIndex == rhsIndex
+    case (.tensorFlowLiteError(let lhsMessage), .tensorFlowLiteError(let rhsMessage)):
+      return lhsMessage == rhsMessage
+    default:
+      return false
+    }
+  }
+}
+#endif  // swift(>=4.2)
diff --git a/tensorflow/lite/experimental/swift/Sources/InterpreterOptions.swift b/tensorflow/lite/experimental/swift/Sources/InterpreterOptions.swift
new file mode 100644
index 0000000000..2365fd7ade
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/Sources/InterpreterOptions.swift
@@ -0,0 +1,29 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Foundation
+
+/// Custom configuration options for a TensorFlow Lite interpreter.
+public struct InterpreterOptions: Equatable {
+
+  /// Maximum number of CPU threads that the interpreter should run on. Default is `nil` which
+  /// indicates that the `Interpreter` will decide the number of threads to use.
+  public var threadCount: Int? = nil
+
+  /// Whether error logging to the console is enabled. The default is `false`.
+  public var isErrorLoggingEnabled = false
+
+  /// Creates a new instance of interpreter options.
+  public init() {}
+}
diff --git a/tensorflow/lite/experimental/swift/Sources/Model.swift b/tensorflow/lite/experimental/swift/Sources/Model.swift
new file mode 100644
index 0000000000..e8c49ff1ae
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/Sources/Model.swift
@@ -0,0 +1,40 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Foundation
+import TensorFlowLiteCAPI
+
+/// A TensorFlow Lite model used by the 'Interpreter` to perform inference.
+final class Model {
+
+  /// The `TFL_Model` C pointer type represented as an `UnsafePointer<TFL_Model>`.
+  typealias CModel = OpaquePointer
+
+  /// The underlying `TFL_Model` C pointer.
+  let cModel: CModel?
+
+  /// Creates a new model instance.
+  ///
+  /// - Precondition: Initialization can fail if the given `filePath` is invalid.
+  /// - Parameters:
+  ///   - filePath: Local file path to a TensorFlow Lite model.
+  init?(filePath: String) {
+    guard !filePath.isEmpty, let cModel = TFL_NewModelFromFile(filePath) else { return nil }
+    self.cModel = cModel
+  }
+
+  deinit {
+    TFL_DeleteModel(cModel)
+  }
+}
diff --git a/tensorflow/lite/experimental/swift/Sources/QuantizationParameters.swift b/tensorflow/lite/experimental/swift/Sources/QuantizationParameters.swift
new file mode 100644
index 0000000000..f367875644
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/Sources/QuantizationParameters.swift
@@ -0,0 +1,38 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Foundation
+
+/// Parameters that determine the mapping of quantized values to real values. Quantized values can
+/// be mapped to float values using the following conversion:
+/// `realValue = scale * (quantizedValue - zeroPoint)`.
+public struct QuantizationParameters {
+
+  /// Difference between real values corresponding to consecutive quantized values differing by 1.
+  /// For example, the range of quantized values for `UInt8` data type is [0, 255].
+  public let scale: Float
+
+  /// Quantized value that corresponds to the real 0 value.
+  public let zeroPoint: Int
+
+  /// Creates a new quantization parameters instance.
+  ///
+  /// - Parameters:
+  ///   - scale: Scale value for asymmetric quantization.
+  ///   - zeroPoint: Zero point for asymmetric quantization.
+  init(scale: Float, zeroPoint: Int) {
+    self.scale = scale
+    self.zeroPoint = zeroPoint
+  }
+}
diff --git a/tensorflow/lite/experimental/swift/Sources/Tensor.swift b/tensorflow/lite/experimental/swift/Sources/Tensor.swift
new file mode 100644
index 0000000000..b738d87549
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/Sources/Tensor.swift
@@ -0,0 +1,138 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Foundation
+import TensorFlowLiteCAPI
+
+/// An input or output tensor in a TensorFlow Lite graph.
+public struct Tensor {
+
+  /// Name of the tensor.
+  public let name: String
+
+  /// Data type of the tensor.
+  public let dataType: TensorDataType
+
+  /// Shape of the tensor.
+  public let shape: TensorShape
+
+  /// Data in the input or output tensor.
+  public let data: Data
+
+  /// Quantization parameters for the tensor if using a quantized model.
+  public let quantizationParameters: QuantizationParameters?
+
+  /// Creates a new input or output tensor instance.
+  ///
+  /// - Parameters:
+  ///   - name: Name of the tensor.
+  ///   - dataType: Data type of the tensor.
+  ///   - data: Data in the input tensor.
+  ///   - quantizationParameters Quantization parameters for the tensor if using a quantized model.
+  ///       The default is `nil`.
+  init(
+    name: String,
+    dataType: TensorDataType,
+    shape: TensorShape,
+    data: Data,
+    quantizationParameters: QuantizationParameters? = nil
+  ) {
+    self.name = name
+    self.dataType = dataType
+    self.shape = shape
+    self.data = data
+    self.quantizationParameters = quantizationParameters
+  }
+}
+
+/// Supported TensorFlow Lite tensor data types.
+public enum TensorDataType: Equatable {
+  /// 32-bit single precision floating point tensor data type.
+  case float32
+  /// 8-bit unsigned integer tensor data type.
+  case uInt8
+  /// 16-bit signed integer tensor data type.
+  case int16
+  /// 32-bit signed integer tensor data type.
+  case int32
+  /// 64-bit signed integer tensor data type.
+  case int64
+  /// Boolean tensor data type.
+  case bool
+
+  /// Creates a new tensor data type from the given `TFL_Type` or `nil` if the data type is
+  /// unsupported or could not be determined because there was an error.
+  ///
+  /// - Parameter type: A data type supported by a tensor.
+  init?(type: TFL_Type) {
+    switch type {
+    case kTfLiteFloat32:
+      self = .float32
+    case kTfLiteUInt8:
+      self = .uInt8
+    case kTfLiteInt16:
+      self = .int16
+    case kTfLiteInt32:
+      self = .int32
+    case kTfLiteInt64:
+      self = .int64
+    case kTfLiteBool:
+      self = .bool
+    case kTfLiteNoType:
+      fallthrough
+    default:
+      return nil
+    }
+  }
+}
+
+/// The shape of a TensorFlow Lite tensor.
+public struct TensorShape {
+
+  /// The number of dimensions of the tensor.
+  public let rank: Int
+
+  /// Array of dimensions for the tensor.
+  public let dimensions: [Int]
+
+  /// Array of `Int32` dimensions for the tensor.
+  var int32Dimensions: [Int32] { return dimensions.map(Int32.init) }
+
+  /// Creates a new tensor shape instance with the given array of dimensions.
+  ///
+  /// - Parameters:
+  ///   - dimensions: Dimensions for the tensor.
+  public init(_ dimensions: [Int]) {
+    self.rank = dimensions.count
+    self.dimensions = dimensions
+  }
+
+  /// Creates a new tensor shape instance with the given elements representing the dimensions.
+  ///
+  /// - Parameters:
+  ///   - elements: Dimensions for the tensor.
+  public init(_ elements: Int...) {
+    self.init(elements)
+  }
+}
+
+extension TensorShape: ExpressibleByArrayLiteral {
+  /// Creates a new tensor shape instance with the given array literal representing the dimensions.
+  ///
+  /// - Parameters:
+  ///   - arrayLiteral: Dimensions for the tensor.
+  public init(arrayLiteral: Int...) {
+    self.init(arrayLiteral)
+  }
+}
diff --git a/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/Configs/TensorFlowLite.tulsigen b/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/Configs/TensorFlowLite.tulsigen
new file mode 100644
index 0000000000..4010fab49e
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/Configs/TensorFlowLite.tulsigen
@@ -0,0 +1,57 @@
+{
+  "sourceFilters" : [
+    "third_party/tensorflow/lite/experimental/c",
+    "third_party/tensorflow/lite/experimental/swift",
+    "third_party/tensorflow/lite/experimental/swift/Sources",
+    "third_party/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp",
+    "third_party/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj",
+    "third_party/tensorflow/lite/experimental/swift/Tests",
+  ],
+  "buildTargets" : [
+    "//third_party/tensorflow/lite/experimental/swift:TensorFlowLite",
+    "//third_party/tensorflow/lite/experimental/swift:TensorFlowLiteApp",
+    "//third_party/tensorflow/lite/experimental/swift:TensorFlowLiteTests",
+  ],
+  "projectName" : "TensorFlowLite",
+  "optionSet" : {
+    "LaunchActionPreActionScript" : {
+      "p" : "$(inherited)"
+    },
+    "BazelBuildStartupOptionsRelease" : {
+      "p" : "$(inherited)"
+    },
+    "BazelBuildOptionsRelease" : {
+      "p" : "$(inherited)"
+    },
+    "BazelBuildOptionsDebug" : {
+      "p" : "$(inherited)"
+    },
+    "EnvironmentVariables" : {
+      "p" : "$(inherited)"
+    },
+    "BuildActionPreActionScript" : {
+      "p" : "$(inherited)"
+    },
+    "CommandlineArguments" : {
+      "p" : "$(inherited)"
+    },
+    "TestActionPreActionScript" : {
+      "p" : "$(inherited)"
+    },
+    "BazelBuildStartupOptionsDebug" : {
+      "p" : "$(inherited)"
+    },
+    "BuildActionPostActionScript" : {
+      "p" : "$(inherited)"
+    },
+    "TestActionPostActionScript" : {
+      "p" : "$(inherited)"
+    },
+    "LaunchActionPostActionScript" : {
+      "p" : "$(inherited)"
+    }
+  },
+  "additionalFilePaths" : [
+    "third_party/tensorflow/lite/experimental/swift/BUILD"
+  ]
+}
diff --git a/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/project.tulsiconf b/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/project.tulsiconf
new file mode 100644
index 0000000000..14cff94453
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/project.tulsiconf
@@ -0,0 +1,14 @@
+{
+  "configDefaults" : {
+    "optionSet" : {
+      "ProjectPrioritizesSwift" : {
+        "p" : "YES"
+      }
+    }
+  },
+  "projectName" : "TensorFlowLite",
+  "packages" : [
+    "third_party/tensorflow/lite/experimental/swift"
+  ],
+  "workspaceRoot" : "../../../../../.."
+}
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp.xcodeproj/project.pbxproj b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp.xcodeproj/project.pbxproj
new file mode 100644
index 0000000000..fbbf9a1de2
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp.xcodeproj/project.pbxproj
@@ -0,0 +1,345 @@
+// !$*UTF8*$!
+{
+	archiveVersion = 1;
+	classes = {
+	};
+	objectVersion = 50;
+	objects = {
+
+/* Begin PBXBuildFile section */
+		4A7304B421500B8400C90B21 /* Data+TensorFlowLite.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4A7304B321500B8300C90B21 /* Data+TensorFlowLite.swift */; };
+		4AA72B732146ED64006C3AEF /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4AA72B722146ED64006C3AEF /* AppDelegate.swift */; };
+		4AA72B752146ED64006C3AEF /* ViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4AA72B742146ED64006C3AEF /* ViewController.swift */; };
+		4AA72B782146ED64006C3AEF /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 4AA72B762146ED64006C3AEF /* Main.storyboard */; };
+		4AA72B7A2146ED66006C3AEF /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 4AA72B792146ED66006C3AEF /* Assets.xcassets */; };
+		4AA72B7D2146ED66006C3AEF /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 4AA72B7B2146ED66006C3AEF /* LaunchScreen.storyboard */; };
+		4ADDE0CE2176600E00FF07A2 /* Array+TensorFlowLite.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4ADDE0CD2176600900FF07A2 /* Array+TensorFlowLite.swift */; };
+/* End PBXBuildFile section */
+
+/* Begin PBXFileReference section */
+		4A7304B321500B8300C90B21 /* Data+TensorFlowLite.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "Data+TensorFlowLite.swift"; sourceTree = "<group>"; };
+		4AA72B6F2146ED64006C3AEF /* TensorFlowLiteApp.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = TensorFlowLiteApp.app; sourceTree = BUILT_PRODUCTS_DIR; };
+		4AA72B722146ED64006C3AEF /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = "<group>"; };
+		4AA72B742146ED64006C3AEF /* ViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ViewController.swift; sourceTree = "<group>"; };
+		4AA72B772146ED64006C3AEF /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/Main.storyboard; sourceTree = "<group>"; };
+		4AA72B792146ED66006C3AEF /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = "<group>"; };
+		4AA72B7C2146ED66006C3AEF /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = "<group>"; };
+		4AA72B7E2146ED66006C3AEF /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
+		4ADDE0CD2176600900FF07A2 /* Array+TensorFlowLite.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "Array+TensorFlowLite.swift"; sourceTree = "<group>"; };
+/* End PBXFileReference section */
+
+/* Begin PBXFrameworksBuildPhase section */
+		4AA72B6C2146ED64006C3AEF /* Frameworks */ = {
+			isa = PBXFrameworksBuildPhase;
+			buildActionMask = 2147483647;
+			files = (
+			);
+			runOnlyForDeploymentPostprocessing = 0;
+		};
+/* End PBXFrameworksBuildPhase section */
+
+/* Begin PBXGroup section */
+		4AA72B662146ED64006C3AEF = {
+			isa = PBXGroup;
+			children = (
+				4AA72B712146ED64006C3AEF /* TensorFlowLiteApp */,
+				4AA72B702146ED64006C3AEF /* Products */,
+			);
+			sourceTree = "<group>";
+		};
+		4AA72B702146ED64006C3AEF /* Products */ = {
+			isa = PBXGroup;
+			children = (
+				4AA72B6F2146ED64006C3AEF /* TensorFlowLiteApp.app */,
+			);
+			name = Products;
+			sourceTree = "<group>";
+		};
+		4AA72B712146ED64006C3AEF /* TensorFlowLiteApp */ = {
+			isa = PBXGroup;
+			children = (
+				4AA72B722146ED64006C3AEF /* AppDelegate.swift */,
+				4ADDE0CD2176600900FF07A2 /* Array+TensorFlowLite.swift */,
+				4A7304B321500B8300C90B21 /* Data+TensorFlowLite.swift */,
+				4AA72B742146ED64006C3AEF /* ViewController.swift */,
+				4AA72B762146ED64006C3AEF /* Main.storyboard */,
+				4AA72B792146ED66006C3AEF /* Assets.xcassets */,
+				4AA72B7B2146ED66006C3AEF /* LaunchScreen.storyboard */,
+				4AA72B7E2146ED66006C3AEF /* Info.plist */,
+			);
+			path = TensorFlowLiteApp;
+			sourceTree = "<group>";
+		};
+/* End PBXGroup section */
+
+/* Begin PBXNativeTarget section */
+		4AA72B6E2146ED64006C3AEF /* TensorFlowLiteApp */ = {
+			isa = PBXNativeTarget;
+			buildConfigurationList = 4AA72B812146ED66006C3AEF /* Build configuration list for PBXNativeTarget "TensorFlowLiteApp" */;
+			buildPhases = (
+				4AA72B6B2146ED64006C3AEF /* Sources */,
+				4AA72B6C2146ED64006C3AEF /* Frameworks */,
+				4AA72B6D2146ED64006C3AEF /* Resources */,
+			);
+			buildRules = (
+			);
+			dependencies = (
+			);
+			name = TensorFlowLiteApp;
+			productName = TensorFlowLiteApp;
+			productReference = 4AA72B6F2146ED64006C3AEF /* TensorFlowLiteApp.app */;
+			productType = "com.apple.product-type.application";
+		};
+/* End PBXNativeTarget section */
+
+/* Begin PBXProject section */
+		4AA72B672146ED64006C3AEF /* Project object */ = {
+			isa = PBXProject;
+			attributes = {
+				LastSwiftUpdateCheck = 0940;
+				LastUpgradeCheck = 0940;
+				ORGANIZATIONNAME = Google;
+				TargetAttributes = {
+					4AA72B6E2146ED64006C3AEF = {
+						CreatedOnToolsVersion = 9.4.1;
+					};
+				};
+			};
+			buildConfigurationList = 4AA72B6A2146ED64006C3AEF /* Build configuration list for PBXProject "TensorFlowLiteApp" */;
+			compatibilityVersion = "Xcode 9.3";
+			developmentRegion = en;
+			hasScannedForEncodings = 0;
+			knownRegions = (
+				en,
+				Base,
+			);
+			mainGroup = 4AA72B662146ED64006C3AEF;
+			productRefGroup = 4AA72B702146ED64006C3AEF /* Products */;
+			projectDirPath = "";
+			projectRoot = "";
+			targets = (
+				4AA72B6E2146ED64006C3AEF /* TensorFlowLiteApp */,
+			);
+		};
+/* End PBXProject section */
+
+/* Begin PBXResourcesBuildPhase section */
+		4AA72B6D2146ED64006C3AEF /* Resources */ = {
+			isa = PBXResourcesBuildPhase;
+			buildActionMask = 2147483647;
+			files = (
+				4AA72B7D2146ED66006C3AEF /* LaunchScreen.storyboard in Resources */,
+				4AA72B7A2146ED66006C3AEF /* Assets.xcassets in Resources */,
+				4AA72B782146ED64006C3AEF /* Main.storyboard in Resources */,
+			);
+			runOnlyForDeploymentPostprocessing = 0;
+		};
+/* End PBXResourcesBuildPhase section */
+
+/* Begin PBXSourcesBuildPhase section */
+		4AA72B6B2146ED64006C3AEF /* Sources */ = {
+			isa = PBXSourcesBuildPhase;
+			buildActionMask = 2147483647;
+			files = (
+				4AA72B732146ED64006C3AEF /* AppDelegate.swift in Sources */,
+				4ADDE0CE2176600E00FF07A2 /* Array+TensorFlowLite.swift in Sources */,
+				4A7304B421500B8400C90B21 /* Data+TensorFlowLite.swift in Sources */,
+				4AA72B752146ED64006C3AEF /* ViewController.swift in Sources */,
+			);
+			runOnlyForDeploymentPostprocessing = 0;
+		};
+/* End PBXSourcesBuildPhase section */
+
+/* Begin PBXVariantGroup section */
+		4AA72B762146ED64006C3AEF /* Main.storyboard */ = {
+			isa = PBXVariantGroup;
+			children = (
+				4AA72B772146ED64006C3AEF /* Base */,
+			);
+			name = Main.storyboard;
+			sourceTree = "<group>";
+		};
+		4AA72B7B2146ED66006C3AEF /* LaunchScreen.storyboard */ = {
+			isa = PBXVariantGroup;
+			children = (
+				4AA72B7C2146ED66006C3AEF /* Base */,
+			);
+			name = LaunchScreen.storyboard;
+			sourceTree = "<group>";
+		};
+/* End PBXVariantGroup section */
+
+/* Begin XCBuildConfiguration section */
+		4AA72B7F2146ED66006C3AEF /* Debug */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				ALWAYS_SEARCH_USER_PATHS = NO;
+				CLANG_ANALYZER_NONNULL = YES;
+				CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
+				CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
+				CLANG_CXX_LIBRARY = "libc++";
+				CLANG_ENABLE_MODULES = YES;
+				CLANG_ENABLE_OBJC_ARC = YES;
+				CLANG_ENABLE_OBJC_WEAK = YES;
+				CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
+				CLANG_WARN_BOOL_CONVERSION = YES;
+				CLANG_WARN_COMMA = YES;
+				CLANG_WARN_CONSTANT_CONVERSION = YES;
+				CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
+				CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
+				CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
+				CLANG_WARN_EMPTY_BODY = YES;
+				CLANG_WARN_ENUM_CONVERSION = YES;
+				CLANG_WARN_INFINITE_RECURSION = YES;
+				CLANG_WARN_INT_CONVERSION = YES;
+				CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
+				CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
+				CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
+				CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
+				CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
+				CLANG_WARN_STRICT_PROTOTYPES = YES;
+				CLANG_WARN_SUSPICIOUS_MOVE = YES;
+				CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
+				CLANG_WARN_UNREACHABLE_CODE = YES;
+				CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
+				CODE_SIGN_IDENTITY = "iPhone Developer";
+				COPY_PHASE_STRIP = NO;
+				DEBUG_INFORMATION_FORMAT = dwarf;
+				ENABLE_STRICT_OBJC_MSGSEND = YES;
+				ENABLE_TESTABILITY = YES;
+				GCC_C_LANGUAGE_STANDARD = gnu11;
+				GCC_DYNAMIC_NO_PIC = NO;
+				GCC_NO_COMMON_BLOCKS = YES;
+				GCC_OPTIMIZATION_LEVEL = 0;
+				GCC_PREPROCESSOR_DEFINITIONS = (
+					"DEBUG=1",
+					"$(inherited)",
+				);
+				GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
+				GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
+				GCC_WARN_UNDECLARED_SELECTOR = YES;
+				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
+				GCC_WARN_UNUSED_FUNCTION = YES;
+				GCC_WARN_UNUSED_VARIABLE = YES;
+				IPHONEOS_DEPLOYMENT_TARGET = 11.4;
+				MTL_ENABLE_DEBUG_INFO = YES;
+				ONLY_ACTIVE_ARCH = YES;
+				SDKROOT = iphoneos;
+				SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG;
+				SWIFT_OPTIMIZATION_LEVEL = "-Onone";
+			};
+			name = Debug;
+		};
+		4AA72B802146ED66006C3AEF /* Release */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				ALWAYS_SEARCH_USER_PATHS = NO;
+				CLANG_ANALYZER_NONNULL = YES;
+				CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
+				CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
+				CLANG_CXX_LIBRARY = "libc++";
+				CLANG_ENABLE_MODULES = YES;
+				CLANG_ENABLE_OBJC_ARC = YES;
+				CLANG_ENABLE_OBJC_WEAK = YES;
+				CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
+				CLANG_WARN_BOOL_CONVERSION = YES;
+				CLANG_WARN_COMMA = YES;
+				CLANG_WARN_CONSTANT_CONVERSION = YES;
+				CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
+				CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
+				CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
+				CLANG_WARN_EMPTY_BODY = YES;
+				CLANG_WARN_ENUM_CONVERSION = YES;
+				CLANG_WARN_INFINITE_RECURSION = YES;
+				CLANG_WARN_INT_CONVERSION = YES;
+				CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
+				CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
+				CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
+				CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
+				CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
+				CLANG_WARN_STRICT_PROTOTYPES = YES;
+				CLANG_WARN_SUSPICIOUS_MOVE = YES;
+				CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
+				CLANG_WARN_UNREACHABLE_CODE = YES;
+				CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
+				CODE_SIGN_IDENTITY = "iPhone Developer";
+				COPY_PHASE_STRIP = NO;
+				DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
+				ENABLE_NS_ASSERTIONS = NO;
+				ENABLE_STRICT_OBJC_MSGSEND = YES;
+				GCC_C_LANGUAGE_STANDARD = gnu11;
+				GCC_NO_COMMON_BLOCKS = YES;
+				GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
+				GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
+				GCC_WARN_UNDECLARED_SELECTOR = YES;
+				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
+				GCC_WARN_UNUSED_FUNCTION = YES;
+				GCC_WARN_UNUSED_VARIABLE = YES;
+				IPHONEOS_DEPLOYMENT_TARGET = 11.4;
+				MTL_ENABLE_DEBUG_INFO = NO;
+				SDKROOT = iphoneos;
+				SWIFT_COMPILATION_MODE = wholemodule;
+				SWIFT_OPTIMIZATION_LEVEL = "-O";
+				VALIDATE_PRODUCT = YES;
+			};
+			name = Release;
+		};
+		4AA72B822146ED66006C3AEF /* Debug */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
+				CODE_SIGN_STYLE = Automatic;
+				INFOPLIST_FILE = TensorFlowLiteApp/Info.plist;
+				LD_RUNPATH_SEARCH_PATHS = (
+					"$(inherited)",
+					"@executable_path/Frameworks",
+				);
+				PRODUCT_BUNDLE_IDENTIFIER = com.tensorflow.lite.swift.TensorFlowLite;
+				PRODUCT_NAME = "$(TARGET_NAME)";
+				SWIFT_VERSION = 4.0;
+				TARGETED_DEVICE_FAMILY = "1,2";
+			};
+			name = Debug;
+		};
+		4AA72B832146ED66006C3AEF /* Release */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
+				CODE_SIGN_STYLE = Automatic;
+				INFOPLIST_FILE = TensorFlowLiteApp/Info.plist;
+				LD_RUNPATH_SEARCH_PATHS = (
+					"$(inherited)",
+					"@executable_path/Frameworks",
+				);
+				PRODUCT_BUNDLE_IDENTIFIER = com.tensorflow.lite.swift.TensorFlowLite;
+				PRODUCT_NAME = "$(TARGET_NAME)";
+				SWIFT_VERSION = 4.0;
+				TARGETED_DEVICE_FAMILY = "1,2";
+			};
+			name = Release;
+		};
+/* End XCBuildConfiguration section */
+
+/* Begin XCConfigurationList section */
+		4AA72B6A2146ED64006C3AEF /* Build configuration list for PBXProject "TensorFlowLiteApp" */ = {
+			isa = XCConfigurationList;
+			buildConfigurations = (
+				4AA72B7F2146ED66006C3AEF /* Debug */,
+				4AA72B802146ED66006C3AEF /* Release */,
+			);
+			defaultConfigurationIsVisible = 0;
+			defaultConfigurationName = Release;
+		};
+		4AA72B812146ED66006C3AEF /* Build configuration list for PBXNativeTarget "TensorFlowLiteApp" */ = {
+			isa = XCConfigurationList;
+			buildConfigurations = (
+				4AA72B822146ED66006C3AEF /* Debug */,
+				4AA72B832146ED66006C3AEF /* Release */,
+			);
+			defaultConfigurationIsVisible = 0;
+			defaultConfigurationName = Release;
+		};
+/* End XCConfigurationList section */
+	};
+	rootObject = 4AA72B672146ED64006C3AEF /* Project object */;
+}
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/AppDelegate.swift b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/AppDelegate.swift
new file mode 100644
index 0000000000..ffa90a06ad
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/AppDelegate.swift
@@ -0,0 +1,24 @@
+import UIKit
+
+@UIApplicationMain
+
+final class AppDelegate: UIResponder, UIApplicationDelegate {
+
+  /// The main window of the app.
+  var window: UIWindow?
+
+  func application(
+    _ application: UIApplication,
+    didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]? = nil
+  ) -> Bool {
+    return true
+  }
+}
+
+// MARK: - Extensions
+
+#if !swift(>=4.2)
+extension UIApplication {
+  typealias LaunchOptionsKey = UIApplicationLaunchOptionsKey
+}
+#endif  // !swift(>=4.2)
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Array+TensorFlowLite.swift b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Array+TensorFlowLite.swift
new file mode 100644
index 0000000000..56df1ce659
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Array+TensorFlowLite.swift
@@ -0,0 +1,22 @@
+import Foundation
+
+extension Array {
+  /// Creates a new array from the bytes of the given unsafe data.
+  ///
+  /// - Warning: The array's `Element` type must be trivial in that it can be copied bit for bit
+  ///     with no indirection or reference-counting operations; otherwise, copying the raw bytes in
+  ///     the `unsafeData`'s buffer to a new array returns an unsafe copy.
+  /// - Note: Returns `nil` if `unsafeData.count` is not a multiple of
+  ///     `MemoryLayout<Element>.stride`.
+  /// - Parameter unsafeData: The data containing the bytes to turn into an array.
+  init?(unsafeData: Data) {
+    guard unsafeData.count % MemoryLayout<Element>.stride == 0 else { return nil }
+    let elements = unsafeData.withUnsafeBytes {
+      UnsafeBufferPointer<Element>(
+        start: $0,
+        count: unsafeData.count / MemoryLayout<Element>.stride
+      )
+    }
+    self.init(elements)
+  }
+}
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/AppIcon.appiconset/Contents.json b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/AppIcon.appiconset/Contents.json
new file mode 100644
index 0000000000..d8db8d65fd
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/AppIcon.appiconset/Contents.json
@@ -0,0 +1,98 @@
+{
+  "images" : [
+    {
+      "idiom" : "iphone",
+      "size" : "20x20",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "20x20",
+      "scale" : "3x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "29x29",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "29x29",
+      "scale" : "3x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "40x40",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "40x40",
+      "scale" : "3x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "60x60",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "60x60",
+      "scale" : "3x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "20x20",
+      "scale" : "1x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "20x20",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "29x29",
+      "scale" : "1x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "29x29",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "40x40",
+      "scale" : "1x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "40x40",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "76x76",
+      "scale" : "1x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "76x76",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "83.5x83.5",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "ios-marketing",
+      "size" : "1024x1024",
+      "scale" : "1x"
+    }
+  ],
+  "info" : {
+    "version" : 1,
+    "author" : "xcode"
+  }
+}
\ No newline at end of file
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/Contents.json b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/Contents.json
new file mode 100644
index 0000000000..da4a164c91
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/Contents.json
@@ -0,0 +1,6 @@
+{
+  "info" : {
+    "version" : 1,
+    "author" : "xcode"
+  }
+}
\ No newline at end of file
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/LaunchScreen.storyboard b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/LaunchScreen.storyboard
new file mode 100644
index 0000000000..a07a1321be
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/LaunchScreen.storyboard
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="14109" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" launchScreen="YES" useTraitCollections="YES" colorMatched="YES" initialViewController="01J-lp-oVM">
+    <device id="retina4_7" orientation="portrait">
+        <adaptation id="fullscreen"/>
+    </device>
+    <dependencies>
+        <deployment identifier="iOS"/>
+        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="14088"/>
+        <capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
+    </dependencies>
+    <scenes>
+        <!--View Controller-->
+        <scene sceneID="EHf-IW-A2E">
+            <objects>
+                <viewController id="01J-lp-oVM" sceneMemberID="viewController">
+                    <layoutGuides>
+                        <viewControllerLayoutGuide type="top" id="Llm-lL-Icb"/>
+                        <viewControllerLayoutGuide type="bottom" id="xb3-aO-Qok"/>
+                    </layoutGuides>
+                    <view key="view" contentMode="scaleToFill" id="Ze5-6b-2t3">
+                        <rect key="frame" x="0.0" y="0.0" width="375" height="667"/>
+                        <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
+                        <subviews>
+                            <label opaque="NO" userInteractionEnabled="NO" contentMode="left" horizontalHuggingPriority="251" verticalHuggingPriority="251" text="TensorFlowLite" textAlignment="center" lineBreakMode="tailTruncation" baselineAdjustment="alignBaselines" adjustsFontSizeToFit="NO" translatesAutoresizingMaskIntoConstraints="NO" id="3Gq-PV-hia">
+                                <rect key="frame" x="16" y="315" width="343" height="38.5"/>
+                                <fontDescription key="fontDescription" type="boldSystem" pointSize="32"/>
+                                <nil key="textColor"/>
+                                <nil key="highlightedColor"/>
+                            </label>
+                        </subviews>
+                        <color key="backgroundColor" red="1" green="1" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
+                        <constraints>
+                            <constraint firstItem="3Gq-PV-hia" firstAttribute="leading" secondItem="Ze5-6b-2t3" secondAttribute="leading" constant="16" id="aXL-9T-5Pf"/>
+                            <constraint firstItem="3Gq-PV-hia" firstAttribute="centerY" secondItem="Ze5-6b-2t3" secondAttribute="centerY" id="cDf-Go-1FR"/>
+                            <constraint firstAttribute="trailing" secondItem="3Gq-PV-hia" secondAttribute="trailing" constant="16" id="fB9-BX-A3B"/>
+                        </constraints>
+                    </view>
+                </viewController>
+                <placeholder placeholderIdentifier="IBFirstResponder" id="iYj-Kq-Ea1" userLabel="First Responder" sceneMemberID="firstResponder"/>
+            </objects>
+            <point key="canvasLocation" x="52" y="374.66266866566718"/>
+        </scene>
+    </scenes>
+</document>
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/Main.storyboard b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/Main.storyboard
new file mode 100644
index 0000000000..d0e91d63c4
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/Main.storyboard
@@ -0,0 +1,95 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="14313.18" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" useTraitCollections="YES" colorMatched="YES" initialViewController="BYZ-38-t0r">
+    <device id="retina4_7" orientation="portrait">
+        <adaptation id="fullscreen"/>
+    </device>
+    <dependencies>
+        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="14283.14"/>
+        <capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
+    </dependencies>
+    <scenes>
+        <!--View Controller-->
+        <scene sceneID="tne-QT-ifu">
+            <objects>
+                <viewController storyboardIdentifier="viewController" useStoryboardIdentifierAsRestorationIdentifier="YES" id="BYZ-38-t0r" customClass="ViewController" customModule="third_party_tensorflow_lite_experimental_swift_TensorFlowLiteAppLib" sceneMemberID="viewController">
+                    <layoutGuides>
+                        <viewControllerLayoutGuide type="top" id="y3c-jy-aDJ"/>
+                        <viewControllerLayoutGuide type="bottom" id="wfy-db-euE"/>
+                    </layoutGuides>
+                    <view key="view" contentMode="scaleToFill" id="8bC-Xf-vdC">
+                        <rect key="frame" x="0.0" y="0.0" width="375" height="667"/>
+                        <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
+                        <subviews>
+                            <textView clipsSubviews="YES" multipleTouchEnabled="YES" contentMode="scaleToFill" editable="NO" selectable="NO" translatesAutoresizingMaskIntoConstraints="NO" id="7Mj-sL-hrd">
+                                <rect key="frame" x="0.0" y="367" width="375" height="300"/>
+                                <color key="backgroundColor" red="0.0" green="0.47843137250000001" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
+                                <constraints>
+                                    <constraint firstAttribute="height" constant="300" id="YUb-MC-D5w"/>
+                                </constraints>
+                                <color key="textColor" cocoaTouchSystemColor="tableCellGroupedBackgroundColor"/>
+                                <fontDescription key="fontDescription" type="system" pointSize="14"/>
+                                <textInputTraits key="textInputTraits" autocapitalizationType="sentences"/>
+                            </textView>
+                            <toolbar opaque="NO" clearsContextBeforeDrawing="NO" contentMode="scaleToFill" translatesAutoresizingMaskIntoConstraints="NO" id="Qwg-EP-bd6" userLabel="Bottom Toolbar">
+                                <rect key="frame" x="0.0" y="323" width="375" height="44"/>
+                                <constraints>
+                                    <constraint firstAttribute="height" constant="44" id="jhT-Q0-E9N"/>
+                                </constraints>
+                                <items>
+                                    <barButtonItem style="plain" systemItem="flexibleSpace" id="P3q-uA-YUa"/>
+                                    <barButtonItem title="Invoke Interpreter" id="A4J-Mg-nmd" userLabel="Invoke Button">
+                                        <connections>
+                                            <action selector="invokeInterpreter:" destination="BYZ-38-t0r" id="lZU-x7-PsJ"/>
+                                        </connections>
+                                    </barButtonItem>
+                                    <barButtonItem style="plain" systemItem="flexibleSpace" id="Qad-Pa-ySg"/>
+                                </items>
+                            </toolbar>
+                            <toolbar opaque="NO" clearsContextBeforeDrawing="NO" contentMode="scaleToFill" translatesAutoresizingMaskIntoConstraints="NO" id="Gkb-TR-PCB" userLabel="Top Toolbar">
+                                <rect key="frame" x="0.0" y="28" width="375" height="44"/>
+                                <constraints>
+                                    <constraint firstAttribute="height" constant="44" id="hSD-2q-fUE"/>
+                                </constraints>
+                                <items>
+                                    <barButtonItem style="plain" id="LKw-TX-bbH">
+                                        <segmentedControl key="customView" opaque="NO" contentMode="scaleToFill" contentHorizontalAlignment="left" contentVerticalAlignment="top" segmentControlStyle="bar" selectedSegmentIndex="0" id="rhA-nW-xzT">
+                                            <rect key="frame" x="16" y="7" width="343" height="30"/>
+                                            <autoresizingMask key="autoresizingMask" flexibleMaxX="YES" flexibleMaxY="YES"/>
+                                            <segments>
+                                                <segment title="Add"/>
+                                                <segment title="AddQuantized"/>
+                                                <segment title="MultiAdd"/>
+                                            </segments>
+                                            <connections>
+                                                <action selector="modelChanged:" destination="BYZ-38-t0r" eventType="valueChanged" id="YnG-Ov-B5D"/>
+                                            </connections>
+                                        </segmentedControl>
+                                    </barButtonItem>
+                                </items>
+                            </toolbar>
+                        </subviews>
+                        <color key="backgroundColor" red="1" green="1" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
+                        <constraints>
+                            <constraint firstAttribute="trailing" secondItem="Gkb-TR-PCB" secondAttribute="trailing" id="4Cr-Sf-I7n"/>
+                            <constraint firstItem="7Mj-sL-hrd" firstAttribute="bottom" secondItem="wfy-db-euE" secondAttribute="top" id="6ot-zD-sze"/>
+                            <constraint firstItem="7Mj-sL-hrd" firstAttribute="top" secondItem="Qwg-EP-bd6" secondAttribute="bottom" id="ELA-C6-NiG"/>
+                            <constraint firstAttribute="trailing" secondItem="7Mj-sL-hrd" secondAttribute="trailing" id="HDO-xr-mBl"/>
+                            <constraint firstItem="Gkb-TR-PCB" firstAttribute="leading" secondItem="8bC-Xf-vdC" secondAttribute="leading" id="Kmo-6K-gS4"/>
+                            <constraint firstItem="Qwg-EP-bd6" firstAttribute="leading" secondItem="8bC-Xf-vdC" secondAttribute="leading" id="hGu-lm-fMG"/>
+                            <constraint firstAttribute="trailing" secondItem="Qwg-EP-bd6" secondAttribute="trailing" id="iXR-LK-nTO"/>
+                            <constraint firstItem="7Mj-sL-hrd" firstAttribute="leading" secondItem="8bC-Xf-vdC" secondAttribute="leading" id="nr7-jW-ZYf"/>
+                            <constraint firstItem="Gkb-TR-PCB" firstAttribute="top" secondItem="y3c-jy-aDJ" secondAttribute="bottom" constant="8" id="uCF-VW-rR0"/>
+                        </constraints>
+                    </view>
+                    <connections>
+                        <outlet property="invokeButton" destination="A4J-Mg-nmd" id="UxZ-Ft-E45"/>
+                        <outlet property="modelControl" destination="rhA-nW-xzT" id="KKf-TT-BQ2"/>
+                        <outlet property="resultsTextView" destination="7Mj-sL-hrd" id="T4I-z4-tYA"/>
+                    </connections>
+                </viewController>
+                <placeholder placeholderIdentifier="IBFirstResponder" id="dkx-z0-nzr" sceneMemberID="firstResponder"/>
+            </objects>
+            <point key="canvasLocation" x="125.59999999999999" y="133.5832083958021"/>
+        </scene>
+    </scenes>
+</document>
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Data+TensorFlowLite.swift b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Data+TensorFlowLite.swift
new file mode 100644
index 0000000000..bc8a70c848
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Data+TensorFlowLite.swift
@@ -0,0 +1,13 @@
+import Foundation
+
+extension Data {
+  /// Creates a new buffer by copying the buffer pointer of the given array.
+  ///
+  /// - Warning: The given array's element type `T` must be trivial in that it can be copied bit
+  ///     for bit with no indirection or reference-counting operations; otherwise, reinterpreting
+  ///     data from the resulting buffer has undefined behavior.
+  /// - Parameter array: An array with elements of type `T`.
+  init<T>(copyingBufferOf array: [T]) {
+    self = array.withUnsafeBufferPointer(Data.init)
+  }
+}
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Info.plist b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Info.plist
new file mode 100644
index 0000000000..3ca3875f04
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Info.plist
@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>CFBundleDevelopmentRegion</key>
+	<string>en</string>
+	<key>CFBundleExecutable</key>
+	<string>$(EXECUTABLE_NAME)</string>
+	<key>CFBundleIdentifier</key>
+	<string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
+	<key>CFBundleInfoDictionaryVersion</key>
+	<string>6.0</string>
+	<key>CFBundleName</key>
+	<string>$(PRODUCT_NAME)</string>
+	<key>CFBundlePackageType</key>
+	<string>APPL</string>
+	<key>CFBundleShortVersionString</key>
+	<string>1.0</string>
+	<key>CFBundleVersion</key>
+	<string>0.0.1</string>
+	<key>LSRequiresIPhoneOS</key>
+	<true/>
+	<key>NSCameraUsageDescription</key>
+	<string>NSCameraUsageDescription</string>
+	<key>NSPhotoLibraryUsageDescription</key>
+	<string>Select a photo to detect objects in.</string>
+	<key>UILaunchStoryboardName</key>
+	<string>LaunchScreen</string>
+	<key>UIMainStoryboardFile</key>
+	<string>Main</string>
+	<key>UIRequiredDeviceCapabilities</key>
+	<array>
+		<string>armv7</string>
+	</array>
+	<key>UISupportedInterfaceOrientations</key>
+	<array>
+		<string>UIInterfaceOrientationPortrait</string>
+		<string>UIInterfaceOrientationPortraitUpsideDown</string>
+	</array>
+	<key>UISupportedInterfaceOrientations~ipad</key>
+	<array>
+		<string>UIInterfaceOrientationPortrait</string>
+		<string>UIInterfaceOrientationPortraitUpsideDown</string>
+	</array>
+</dict>
+</plist>
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/ViewController.swift b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/ViewController.swift
new file mode 100644
index 0000000000..73c74fd19c
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/ViewController.swift
@@ -0,0 +1,299 @@
+import TensorFlowLite
+import UIKit
+
+class ViewController: UIViewController {
+
+  // MARK: - Properties
+
+  /// TensorFlowLite interpreter object for performing inference from a given model.
+  private var interpreter: Interpreter?
+
+  /// Serial dispatch queue for managing `Interpreter` calls.
+  private let interpreterQueue = DispatchQueue(
+    label: Constant.dispatchQueueLabel,
+    qos: .userInitiated
+  )
+
+  /// The currently selected model.
+  private var currentModel: Model {
+    guard let currentModel = Model(rawValue: modelControl.selectedSegmentIndex) else {
+      preconditionFailure("Invalid model for selected segment index.")
+    }
+    return currentModel
+  }
+
+  /// A description of the current model.
+  private var modelDescription: String {
+    guard let interpreter = interpreter else { return "" }
+    let inputCount = interpreter.inputTensorCount
+    let outputCount = interpreter.outputTensorCount
+    let inputTensors = (0..<inputCount).map { index in
+      var tensorInfo = "  Input \(index + 1): "
+      do {
+        let tensor = try interpreter.input(at: index)
+        tensorInfo += "\(tensor)"
+      } catch let error {
+        tensorInfo += "\(error.localizedDescription)"
+      }
+      return tensorInfo
+    }.joined(separator: "\n")
+    let outputTensors = (0..<outputCount).map { index in
+      var tensorInfo = "  Output \(index + 1): "
+      do {
+        let tensor = try interpreter.output(at: index)
+        tensorInfo += "\(tensor)"
+      } catch let error {
+        tensorInfo += "\(error.localizedDescription)"
+      }
+      return tensorInfo
+    }.joined(separator: "\n")
+    return "Model Description:\n" +
+             "  Input Tensor Count = \(inputCount)\n\(inputTensors)\n\n" +
+             "  Output Tensor Count = \(outputCount)\n\(outputTensors)"
+  }
+
+  // MARK: - IBOutlets
+
+  /// A segmented control for changing models. See the `Model` enum for available models.
+  @IBOutlet private var modelControl: UISegmentedControl!
+
+  @IBOutlet private var resultsTextView: UITextView!
+  @IBOutlet private var invokeButton: UIBarButtonItem!
+
+  // MARK: - UIViewController
+
+  override func viewDidLoad() {
+    super.viewDidLoad()
+
+    invokeButton.isEnabled = false
+    loadModel()
+  }
+
+  // MARK: - IBActions
+
+  @IBAction func modelChanged(_ sender: Any) {
+    invokeButton.isEnabled = false
+    updateResultsText("Switched to the \(currentModel.description) model.")
+    loadModel()
+  }
+
+  @IBAction func invokeInterpreter(_ sender: Any) {
+    switch currentModel {
+    case .add:
+      invokeAdd()
+    case .addQuantized:
+      invokeAddQuantized()
+    case .multiAdd:
+      invokeMultiAdd()
+    }
+  }
+
+  // MARK: - Private
+
+  private func loadModel() {
+    let fileInfo = currentModel.fileInfo
+    guard let modelPath = Bundle.main.path(forResource: fileInfo.name, ofType: fileInfo.extension)
+    else {
+      updateResultsText("Failed to load the \(currentModel.description) model.")
+      return
+    }
+    setUpInterpreter(withModelPath: modelPath)
+  }
+
+  private func setUpInterpreter(withModelPath modelPath: String) {
+    interpreterQueue.async {
+      do {
+        var options = InterpreterOptions()
+        options.isErrorLoggingEnabled = true
+        self.interpreter = try Interpreter(modelPath: modelPath, options: options)
+      } catch let error {
+        self.updateResultsText(
+          "Failed to create the interpreter with error: \(error.localizedDescription)"
+        )
+        return
+      }
+      safeDispatchOnMain { self.invokeButton.isEnabled = true }
+    }
+  }
+
+  private func invokeAdd() {
+    interpreterQueue.async {
+      guard let interpreter = self.interpreter else {
+        self.updateResultsText(Constant.nilInterpreterErrorMessage)
+        return
+      }
+      do {
+        try interpreter.resizeInput(at: 0, to: [2])
+        try interpreter.allocateTensors()
+        let input: [Float32] = [1, 3]
+        let resultsText = self.modelDescription + "\n\n" +
+          "Performing 2 add operations on input \(input.description) equals: "
+        self.updateResultsText(resultsText)
+        let data = Data(copyingBufferOf: input)
+        try interpreter.copy(data, toInputAt: 0)
+        try interpreter.invoke()
+        let outputTensor = try interpreter.output(at: 0)
+        let results: () -> String = {
+          guard let results = [Float32](unsafeData: outputTensor.data) else { return "No results." }
+          return resultsText + results.description
+        }
+        self.updateResultsText(results())
+      } catch let error {
+        self.updateResultsText(
+          "Failed to invoke the interpreter with error: \(error.localizedDescription)"
+        )
+        return
+      }
+    }
+  }
+
+  private func invokeAddQuantized() {
+    interpreterQueue.async {
+      guard let interpreter = self.interpreter else {
+        self.updateResultsText(Constant.nilInterpreterErrorMessage)
+        return
+      }
+      do {
+        try interpreter.resizeInput(at: 0, to: [2])
+        try interpreter.allocateTensors()
+        let input: [UInt8] = [1, 3]
+        let resultsText = self.modelDescription + "\n\n" +
+          "Performing 2 add operations on quantized input \(input.description) equals: "
+        self.updateResultsText(resultsText)
+        let data = Data(input)
+        try interpreter.copy(data, toInputAt: 0)
+        try interpreter.invoke()
+        let outputTensor = try interpreter.output(at: 0)
+        let results: () -> String = {
+          guard let quantizationParameters = outputTensor.quantizationParameters else {
+            return "No results."
+          }
+          let quantizedResults = [UInt8](outputTensor.data)
+          let dequantizedResults = quantizedResults.map {
+            quantizationParameters.scale * Float(Int($0) - quantizationParameters.zeroPoint)
+          }
+          return resultsText + quantizedResults.description +
+                   ", dequantized results: " + dequantizedResults.description
+        }
+        self.updateResultsText(results())
+      } catch let error {
+        self.updateResultsText(
+          "Failed to invoke the interpreter with error: \(error.localizedDescription)"
+        )
+        return
+      }
+    }
+  }
+
+  private func invokeMultiAdd() {
+    interpreterQueue.async {
+      guard let interpreter = self.interpreter else {
+        self.updateResultsText(Constant.nilInterpreterErrorMessage)
+        return
+      }
+      do {
+        let shape = TensorShape(2)
+        try (0..<interpreter.inputTensorCount).forEach { index in
+          try interpreter.resizeInput(at: index, to: shape)
+        }
+        try interpreter.allocateTensors()
+        let inputs = try (0..<interpreter.inputTensorCount).map { index -> [Float32] in
+          let input = [Float32(index + 1), Float32(index + 2)]
+          let data = Data(copyingBufferOf: input)
+          try interpreter.copy(data, toInputAt: index)
+          return input
+        }
+        let resultsText = self.modelDescription + "\n\n" +
+          "Performing 3 add operations on inputs \(inputs.description) equals: "
+        self.updateResultsText(resultsText)
+        try interpreter.invoke()
+        let results = try (0..<interpreter.outputTensorCount).map { index -> [Float32] in
+          let tensor = try interpreter.output(at: index)
+          return [Float32](unsafeData: tensor.data) ?? []
+        }
+        self.updateResultsText(resultsText + results.description)
+      } catch let error {
+        self.updateResultsText(
+          "Failed to invoke the interpreter with error: \(error.localizedDescription)"
+        )
+        return
+      }
+    }
+  }
+
+  private func updateResultsText(_ text: String? = nil) {
+    safeDispatchOnMain { self.resultsTextView.text = text }
+  }
+}
+
+// MARK: - Constants
+
+private enum Constant {
+  static let dispatchQueueLabel = "TensorFlowLiteInterpreterQueue"
+  static let nilInterpreterErrorMessage =
+    "Failed to invoke the interpreter because the interpreter was nil."
+}
+
+/// Models that can be loaded by the TensorFlow Lite `Interpreter`.
+private enum Model: Int, CustomStringConvertible {
+  /// A float model that performs two add operations on one input tensor and returns the result in
+  /// one output tensor.
+  case add = 0
+  /// A quantized model that performs two add operations on one input tensor and returns the result
+  /// in one output tensor.
+  case addQuantized = 1
+  /// A float model that performs three add operations on four input tensors and returns the results
+  /// in 2 output tensors.
+  case multiAdd = 2
+
+  var fileInfo: (name: String, extension: String) {
+    switch self {
+    case .add:
+      return Add.fileInfo
+    case .addQuantized:
+      return AddQuantized.fileInfo
+    case .multiAdd:
+      return MultiAdd.fileInfo
+    }
+  }
+
+  // MARK: - CustomStringConvertible
+
+  var description: String {
+    switch self {
+    case .add:
+      return Add.name
+    case .addQuantized:
+      return AddQuantized.name
+    case .multiAdd:
+      return MultiAdd.name
+    }
+  }
+}
+
+/// Values for the `Add` model.
+private enum Add {
+  static let name = "Add"
+  static let fileInfo = (name: "add", extension: "bin")
+}
+
+/// Values for the `AddQuantized` model.
+private enum AddQuantized {
+  static let name = "AddQuantized"
+  static let fileInfo = (name: "add_quantized", extension: "bin")
+}
+
+/// Values for the `MultiAdd` model.
+private enum MultiAdd {
+  static let name = "MultiAdd"
+  static let fileInfo = (name: "multi_add", extension: "bin")
+}
+
+// MARK: - Fileprivate
+
+/// Safely dispatches the given block on the main queue. If the current thread is `main`, the block
+/// is executed synchronously; otherwise, the block is executed asynchronously on the main thread.
+fileprivate func safeDispatchOnMain(_ block: @escaping () -> Void) {
+  if Thread.isMainThread { block(); return }
+  DispatchQueue.main.async { block() }
+}
diff --git a/tensorflow/lite/experimental/swift/Tests/InterpreterOptionsTests.swift b/tensorflow/lite/experimental/swift/Tests/InterpreterOptionsTests.swift
new file mode 100644
index 0000000000..54b4f59b28
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/Tests/InterpreterOptionsTests.swift
@@ -0,0 +1,54 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+@testable import TensorFlowLite
+import XCTest
+
+class InterpreterOptionsTests: XCTestCase {
+
+  func testInterpreterOptions_InitWithDefaultValues() {
+    let options = InterpreterOptions()
+    XCTAssertNil(options.threadCount)
+    XCTAssertFalse(options.isErrorLoggingEnabled)
+  }
+
+  func testInterpreterOptions_InitWithCustomValues() {
+    var options = InterpreterOptions()
+    options.threadCount = 2
+    XCTAssertEqual(options.threadCount, 2)
+    options.isErrorLoggingEnabled = true
+    XCTAssertTrue(options.isErrorLoggingEnabled)
+  }
+
+  func testInterpreterOptions_Equatable() {
+    var options1 = InterpreterOptions()
+    var options2 = InterpreterOptions()
+    XCTAssertEqual(options1, options2)
+
+    options1.threadCount = 2
+    options2.threadCount = 2
+    XCTAssertEqual(options1, options2)
+
+    options2.threadCount = 3
+    XCTAssertNotEqual(options1, options2)
+    options2.threadCount = 2
+
+    options1.isErrorLoggingEnabled = true
+    options2.isErrorLoggingEnabled = true
+    XCTAssertEqual(options1, options2)
+
+    options2.isErrorLoggingEnabled = false
+    XCTAssertNotEqual(options1, options2)
+  }
+}
diff --git a/tensorflow/lite/experimental/swift/Tests/InterpreterTests.swift b/tensorflow/lite/experimental/swift/Tests/InterpreterTests.swift
new file mode 100644
index 0000000000..e98da5f951
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/Tests/InterpreterTests.swift
@@ -0,0 +1,315 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+@testable import TensorFlowLite
+import XCTest
+
+class InterpreterTests: XCTestCase {
+
+  var interpreter: Interpreter!
+
+  override func setUp() {
+    super.setUp()
+
+    interpreter = try! Interpreter(modelPath: AddModel.path)
+  }
+
+  override func tearDown() {
+    interpreter = nil
+
+    super.tearDown()
+  }
+
+  func testInterpreter_InitWithModelPath() {
+    XCTAssertNoThrow(try Interpreter(modelPath: AddModel.path))
+  }
+
+  func testInterpreter_Init_ThrowsFailedToLoadModel() {
+    XCTAssertThrowsError(try Interpreter(modelPath: "/invalid/path")) { error in
+      self.assertEqualErrors(actual: error, expected: .failedToLoadModel)
+    }
+  }
+
+  func testInterpreter_InitWithModelPathAndOptions() {
+    var options = InterpreterOptions()
+    options.threadCount = 2
+    XCTAssertNoThrow(try Interpreter(modelPath: AddModel.path, options: options))
+  }
+
+  func testInterpreter_InputTensorCount() {
+    XCTAssertEqual(interpreter.inputTensorCount, AddModel.inputTensorCount)
+  }
+
+  func testInterpreter_OutputTensorCount() {
+    XCTAssertEqual(interpreter.outputTensorCount, AddModel.outputTensorCount)
+  }
+
+  func testInterpreter_Invoke() throws {
+    try interpreter.allocateTensors()
+    XCTAssertNoThrow(try interpreter.invoke())
+  }
+
+  func testInterpreter_Invoke_ThrowsAllocateTensorsRequired_ModelNotReady() {
+    XCTAssertThrowsError(try interpreter.invoke()) { error in
+      self.assertEqualErrors(actual: error, expected: .allocateTensorsRequired)
+    }
+  }
+
+  func testInterpreter_InputTensorAtIndex() throws {
+    try setUpAddModelInputTensor()
+    let inputTensor = try interpreter.input(at: AddModel.validIndex)
+    XCTAssertEqual(inputTensor, AddModel.inputTensor)
+  }
+
+  func testInterpreter_InputTensorAtIndex_QuantizedModel() throws {
+    interpreter = try Interpreter(modelPath: AddQuantizedModel.path)
+    try setUpAddQuantizedModelInputTensor()
+    let inputTensor = try interpreter.input(at: AddQuantizedModel.inputOutputIndex)
+    XCTAssertEqual(inputTensor, AddQuantizedModel.inputTensor)
+  }
+
+  func testInterpreter_InputTensorAtIndex_ThrowsInvalidIndex() throws {
+    try interpreter.allocateTensors()
+    XCTAssertThrowsError(try interpreter.input(at: AddModel.invalidIndex)) { error in
+      let maxIndex = AddModel.inputTensorCount - 1
+      self.assertEqualErrors(
+        actual: error,
+        expected: .invalidTensorIndex(index: AddModel.invalidIndex, maxIndex: maxIndex)
+      )
+    }
+  }
+
+  func testInterpreter_InputTensorAtIndex_ThrowsAllocateTensorsRequired() {
+    XCTAssertThrowsError(try interpreter.input(at: AddModel.validIndex)) { error in
+      self.assertEqualErrors(actual: error, expected: .allocateTensorsRequired)
+    }
+  }
+
+  func testInterpreter_OutputTensorAtIndex() throws {
+    try setUpAddModelInputTensor()
+    try interpreter.invoke()
+    let outputTensor = try interpreter.output(at: AddModel.validIndex)
+    XCTAssertEqual(outputTensor, AddModel.outputTensor)
+    let expectedResults = [Float32](unsafeData: outputTensor.data)
+    XCTAssertEqual(expectedResults, AddModel.results)
+  }
+
+  func testInterpreter_OutputTensorAtIndex_QuantizedModel() throws {
+    interpreter = try Interpreter(modelPath: AddQuantizedModel.path)
+    try setUpAddQuantizedModelInputTensor()
+    try interpreter.invoke()
+    let outputTensor = try interpreter.output(at: AddQuantizedModel.inputOutputIndex)
+    XCTAssertEqual(outputTensor, AddQuantizedModel.outputTensor)
+    let expectedResults = [UInt8](outputTensor.data)
+    XCTAssertEqual(expectedResults, AddQuantizedModel.results)
+  }
+
+  func testInterpreter_OutputTensorAtIndex_ThrowsInvalidIndex() throws {
+    try interpreter.allocateTensors()
+    try interpreter.invoke()
+    XCTAssertThrowsError(try interpreter.output(at: AddModel.invalidIndex)) { error in
+      let maxIndex = AddModel.outputTensorCount - 1
+      self.assertEqualErrors(
+        actual: error,
+        expected: .invalidTensorIndex(index: AddModel.invalidIndex, maxIndex: maxIndex)
+      )
+    }
+  }
+
+  func testInterpreter_OutputTensorAtIndex_ThrowsInvokeInterpreterRequired() {
+    XCTAssertThrowsError(try interpreter.output(at: AddModel.validIndex)) { error in
+      self.assertEqualErrors(actual: error, expected: .invokeInterpreterRequired)
+    }
+  }
+
+  func testInterpreter_ResizeInputTensorAtIndexToShape() {
+    XCTAssertNoThrow(try interpreter.resizeInput(at: AddModel.validIndex, to: [2, 2, 3]))
+    XCTAssertNoThrow(try interpreter.allocateTensors())
+  }
+
+  func testInterpreter_ResizeInputTensorAtIndexToShape_ThrowsInvalidIndex() {
+    XCTAssertThrowsError(try interpreter.resizeInput(
+      at: AddModel.invalidIndex,
+      to: [2, 2, 3]
+    )) { error in
+      let maxIndex = AddModel.inputTensorCount - 1
+      self.assertEqualErrors(
+        actual: error,
+        expected: .invalidTensorIndex(index: AddModel.invalidIndex, maxIndex: maxIndex)
+      )
+    }
+  }
+
+  func testInterpreter_CopyDataToInputTensorAtIndex() throws {
+    try interpreter.resizeInput(at: AddModel.validIndex, to: AddModel.shape)
+    try interpreter.allocateTensors()
+    let inputTensor = try interpreter.copy(AddModel.inputData, toInputAt: AddModel.validIndex)
+    XCTAssertEqual(inputTensor.data, AddModel.inputData)
+  }
+
+  func testInterpreter_CopyDataToInputTensorAtIndex_ThrowsInvalidIndex() {
+    XCTAssertThrowsError(try interpreter.copy(
+      AddModel.inputData,
+      toInputAt: AddModel.invalidIndex
+    )) { error in
+      let maxIndex = AddModel.inputTensorCount - 1
+      self.assertEqualErrors(
+        actual: error,
+        expected: .invalidTensorIndex(index: AddModel.invalidIndex, maxIndex: maxIndex)
+      )
+    }
+  }
+
+  func testInterpreter_CopyDataToInputTensorAtIndex_ThrowsInvalidDataCount() throws {
+    try interpreter.resizeInput(at: AddModel.validIndex, to: AddModel.shape)
+    try interpreter.allocateTensors()
+    let invalidData = Data(count: AddModel.dataCount - 1)
+    XCTAssertThrowsError(try interpreter.copy(
+      invalidData,
+      toInputAt: AddModel.validIndex
+    )) { error in
+      self.assertEqualErrors(
+        actual: error,
+        expected: .invalidTensorDataCount(provided: invalidData.count, required: AddModel.dataCount)
+      )
+    }
+  }
+
+  func testInterpreter_AllocateTensors() {
+    XCTAssertNoThrow(try interpreter.allocateTensors())
+  }
+
+  // MARK: - Private
+
+  private func setUpAddModelInputTensor() throws {
+    precondition(interpreter != nil)
+    try interpreter.resizeInput(at: AddModel.validIndex, to: AddModel.shape)
+    try interpreter.allocateTensors()
+    try interpreter.copy(AddModel.inputData, toInputAt: AddModel.validIndex)
+  }
+
+  private func setUpAddQuantizedModelInputTensor() throws {
+    precondition(interpreter != nil)
+    try interpreter.resizeInput(at: AddQuantizedModel.inputOutputIndex, to: AddQuantizedModel.shape)
+    try interpreter.allocateTensors()
+    try interpreter.copy(AddQuantizedModel.inputData, toInputAt: AddQuantizedModel.inputOutputIndex)
+  }
+
+  private func assertEqualErrors(actual: Error, expected: InterpreterError) {
+    guard let actual = actual as? InterpreterError else {
+      XCTFail("Actual error should be of type InterpreterError.")
+      return
+    }
+    XCTAssertEqual(actual, expected)
+  }
+}
+
+// MARK: - Constants
+
+/// Values for the `add.bin` model.
+private enum AddModel {
+  static let info = (name: "add", extension: "bin")
+  static let inputTensorCount = 1
+  static let outputTensorCount = 1
+  static let invalidIndex = 1
+  static let validIndex = 0
+  static let shape: TensorShape = [2]
+  static let dataCount = inputData.count
+  static let inputData = Data(copyingBufferOf: [Float32(1.0), Float32(3.0)])
+  static let outputData = Data(copyingBufferOf: [Float32(3.0), Float32(9.0)])
+  static let results = [Float32(3.0), Float32(9.0)]
+
+  static let inputTensor = Tensor(
+    name: "input",
+    dataType: .float32,
+    shape: shape,
+    data: inputData
+  )
+  static let outputTensor = Tensor(
+    name: "output",
+    dataType: .float32,
+    shape: shape,
+    data: outputData
+  )
+
+  static var path: String = {
+    let bundle = Bundle(for: InterpreterTests.self)
+    guard let path = bundle.path(forResource: info.name, ofType: info.extension) else { return "" }
+    return path
+  }()
+}
+
+/// Values for the `add_quantized.bin` model.
+private enum AddQuantizedModel {
+  static let info = (name: "add_quantized", extension: "bin")
+  static let inputOutputIndex = 0
+  static let shape: TensorShape = [2]
+  static let inputData = Data([1, 3])
+  static let outputData = Data([3, 9])
+  static let quantizationParameters = QuantizationParameters(scale: 0.003922, zeroPoint: 0)
+  static let results: [UInt8] = [3, 9]
+
+  static let inputTensor = Tensor(
+    name: "input",
+    dataType: .uInt8,
+    shape: shape,
+    data: inputData,
+    quantizationParameters: quantizationParameters
+  )
+  static let outputTensor = Tensor(
+    name: "output",
+    dataType: .uInt8,
+    shape: shape,
+    data: outputData,
+    quantizationParameters: quantizationParameters
+  )
+
+  static var path: String = {
+    let bundle = Bundle(for: InterpreterTests.self)
+    guard let path = bundle.path(forResource: info.name, ofType: info.extension) else { return "" }
+    return path
+  }()
+}
+
+// MARK: - Extensions
+
+extension Array {
+  /// Creates a new array from the bytes of the given unsafe data.
+  ///
+  /// - Note: Returns `nil` if `unsafeData.count` is not a multiple of
+  ///     `MemoryLayout<Element>.stride`.
+  /// - Parameter unsafeData: The data containing the bytes to turn into an array.
+  init?(unsafeData: Data) {
+    guard unsafeData.count % MemoryLayout<Element>.stride == 0 else { return nil }
+    let elements = unsafeData.withUnsafeBytes {
+      UnsafeBufferPointer<Element>(
+        start: $0,
+        count: unsafeData.count / MemoryLayout<Element>.stride
+      )
+    }
+    self.init(elements)
+  }
+}
+
+extension Data {
+  /// Creates a new buffer by copying the buffer pointer of the given array.
+  ///
+  /// - Warning: The given array's element type `T` must be trivial in that it can be copied bit
+  ///     for bit with no indirection or reference-counting operations; otherwise, reinterpreting
+  ///     data from the resulting buffer has undefined behavior.
+  /// - Parameter array: An array with elements of type `T`.
+  init<T>(copyingBufferOf array: [T]) {
+    self = array.withUnsafeBufferPointer(Data.init)
+  }
+}
diff --git a/tensorflow/lite/experimental/swift/Tests/ModelTests.swift b/tensorflow/lite/experimental/swift/Tests/ModelTests.swift
new file mode 100644
index 0000000000..025db18906
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/Tests/ModelTests.swift
@@ -0,0 +1,59 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+@testable import TensorFlowLite
+import XCTest
+
+class ModelTests: XCTestCase {
+
+  var modelPath: String!
+
+  override func setUp() {
+    super.setUp()
+
+    let bundle = Bundle(for: type(of: self))
+    guard let modelPath = bundle.path(
+            forResource: Constant.modelInfo.name,
+            ofType: Constant.modelInfo.extension)
+    else {
+      XCTFail("Failed to get the model file path.")
+      return
+    }
+    self.modelPath = modelPath
+  }
+
+  override func tearDown() {
+    modelPath = nil
+
+    super.tearDown()
+  }
+
+  func testModel_InitWithFilePath() {
+    XCTAssertNotNil(Model(filePath: modelPath))
+  }
+
+  func testModel_InitWithEmptyFilePath_FailsInitialization() {
+    XCTAssertNil(Model(filePath: ""))
+  }
+
+  func testModel_InitWithInvalidFilePath_FailsInitialization() {
+    XCTAssertNil(Model(filePath: "invalid/path"))
+  }
+}
+
+// MARK: - Constants
+
+private enum Constant {
+  static let modelInfo = (name: "add", extension: "bin")
+}
diff --git a/tensorflow/lite/experimental/swift/Tests/QuantizationParametersTests.swift b/tensorflow/lite/experimental/swift/Tests/QuantizationParametersTests.swift
new file mode 100644
index 0000000000..65648c2698
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/Tests/QuantizationParametersTests.swift
@@ -0,0 +1,43 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+@testable import TensorFlowLite
+import XCTest
+
+class QuantizationParametersTests: XCTestCase {
+
+  func testQuantizationParameters_InitWithCustomValues() {
+    let parameters = QuantizationParameters(scale: 0.5, zeroPoint: 1)
+    XCTAssertEqual(parameters.scale, 0.5)
+    XCTAssertEqual(parameters.zeroPoint, 1)
+  }
+
+  func testQuantizationParameters_Equatable() {
+    let parameters1 = QuantizationParameters(scale: 0.5, zeroPoint: 1)
+    let parameters2 = QuantizationParameters(scale: 0.5, zeroPoint: 1)
+    XCTAssertEqual(parameters1, parameters2)
+
+    let parameters3 = QuantizationParameters(scale: 0.4, zeroPoint: 1)
+    XCTAssertNotEqual(parameters1, parameters3)
+    XCTAssertNotEqual(parameters2, parameters3)
+  }
+}
+
+// MARK: - Extensions
+
+extension QuantizationParameters: Equatable {
+  public static func == (lhs: QuantizationParameters, rhs: QuantizationParameters) -> Bool {
+    return lhs.scale == rhs.scale && lhs.zeroPoint == rhs.zeroPoint
+  }
+}
diff --git a/tensorflow/lite/experimental/swift/Tests/TensorTests.swift b/tensorflow/lite/experimental/swift/Tests/TensorTests.swift
new file mode 100644
index 0000000000..4540043a16
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/Tests/TensorTests.swift
@@ -0,0 +1,83 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+@testable import TensorFlowLite
+import XCTest
+
+class TensorTests: XCTestCase {
+
+  // MARK: - Tensor
+
+  func testTensor_Init() {
+    let name = "InputTensor"
+    let dataType: TensorDataType = .uInt8
+    let shape = TensorShape(Constant.dimensions)
+    guard let data = name.data(using: .utf8) else { XCTFail("Data should not be nil."); return }
+    let quantizationParameters = QuantizationParameters(scale: 0.5, zeroPoint: 1)
+    let inputTensor = Tensor(
+      name: name,
+      dataType: dataType,
+      shape: shape,
+      data: data,
+      quantizationParameters: quantizationParameters
+    )
+    XCTAssertEqual(inputTensor.name, name)
+    XCTAssertEqual(inputTensor.dataType, dataType)
+    XCTAssertEqual(inputTensor.shape, shape)
+    XCTAssertEqual(inputTensor.data, data)
+    XCTAssertEqual(inputTensor.quantizationParameters, quantizationParameters)
+  }
+
+  // MARK: - TensorShape
+
+  func testTensorShape_InitWithArray() {
+    let shape = TensorShape(Constant.dimensions)
+    XCTAssertEqual(shape.rank, Constant.dimensions.count)
+    XCTAssertEqual(shape.dimensions, Constant.dimensions)
+  }
+
+  func testTensorShape_InitWithElements() {
+    let shape = TensorShape(2, 2, 3)
+    XCTAssertEqual(shape.rank, Constant.dimensions.count)
+    XCTAssertEqual(shape.dimensions, Constant.dimensions)
+  }
+
+  func testTensorShape_InitWithArrayLiteral() {
+    let shape: TensorShape = [2, 2, 3]
+    XCTAssertEqual(shape.rank, Constant.dimensions.count)
+    XCTAssertEqual(shape.dimensions, Constant.dimensions)
+  }
+}
+
+// MARK: - Constants
+
+private enum Constant {
+  /// Array of 2 arrays of 2 arrays of 3 numbers: [[[1, 1, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]]].
+  static let dimensions = [2, 2, 3]
+}
+
+// MARK: - Extensions
+
+extension TensorShape: Equatable {
+  public static func == (lhs: TensorShape, rhs: TensorShape) -> Bool {
+    return lhs.rank == rhs.rank && lhs.dimensions == rhs.dimensions
+  }
+}
+
+extension Tensor: Equatable {
+  public static func == (lhs: Tensor, rhs: Tensor) -> Bool {
+    return lhs.name == rhs.name && lhs.dataType == rhs.dataType && lhs.shape == rhs.shape &&
+           lhs.data == rhs.data && lhs.quantizationParameters == rhs.quantizationParameters
+  }
+}
diff --git a/tensorflow/tools/pip_package/pip_smoke_test.py b/tensorflow/tools/pip_package/pip_smoke_test.py
index 51d010c9e1..952c71c615 100644
--- a/tensorflow/tools/pip_package/pip_smoke_test.py
+++ b/tensorflow/tools/pip_package/pip_smoke_test.py
@@ -30,14 +30,19 @@ os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../..")))
 PIP_PACKAGE_QUERY_EXPRESSION = (
     "deps(//tensorflow/tools/pip_package:build_pip_package)")
 
+# List of file paths containing BUILD files that should not be included for the
+# pip smoke test.
+BUILD_BLACKLIST = [
+    "tensorflow/lite/examples/android",
+    "tensorflow/lite/experimental/swift",
+]
 
 def GetBuild(dir_base):
   """Get the list of BUILD file all targets recursively startind at dir_base."""
   items = []
   for root, _, files in os.walk(dir_base):
     for name in files:
-      if (name == "BUILD" and
-          root.find("tensorflow/lite/examples/android") == -1):
+      if (name == "BUILD" and root not in BUILD_BLACKLIST):
         items.append("//" + root + ":all")
   return items
 
@@ -67,9 +72,9 @@ def BuildPyTestDependencies():
 
 PYTHON_TARGETS, PY_TEST_QUERY_EXPRESSION = BuildPyTestDependencies()
 
-# Hard-coded blacklist of files if not included in pip package
 # TODO(amitpatankar): Clean up blacklist.
-BLACKLIST = [
+# List of dependencies that should not included in the pip package.
+DEPENDENCY_BLACKLIST = [
     "//tensorflow/python:extra_py_tests_deps",
     "//tensorflow/cc/saved_model:saved_model_half_plus_two",
     "//tensorflow:no_tensorflow_py_deps",
@@ -82,9 +87,7 @@ BLACKLIST = [
     "//tensorflow/core/kernels/cloud:bigquery_reader_ops",
     "//tensorflow/python/feature_column:vocabulary_testdata",
     "//tensorflow/python:framework/test_file_system.so",
-    # contrib
-    "//tensorflow/contrib/session_bundle:session_bundle_half_plus_two",
-    "//tensorflow/contrib/keras:testing_utils",
+    # lite
     "//tensorflow/lite/experimental/examples/lstm:tflite_lstm",
     "//tensorflow/lite/experimental/examples/lstm:tflite_lstm.py",
     "//tensorflow/lite/experimental/examples/lstm:unidirectional_sequence_lstm_test",  # pylint:disable=line-too-long
@@ -93,6 +96,9 @@ BLACKLIST = [
     "//tensorflow/lite/python:interpreter_test",
     "//tensorflow/lite/python:interpreter.py",
     "//tensorflow/lite/python:interpreter_test.py",
+    # contrib
+    "//tensorflow/contrib/session_bundle:session_bundle_half_plus_two",
+    "//tensorflow/contrib/keras:testing_utils",
     "//tensorflow/contrib/ffmpeg:test_data",
     "//tensorflow/contrib/fused_conv:fused_conv2d_bias_activation_op_test_base",
     "//tensorflow/contrib/hadoop:test_data",
@@ -149,8 +155,8 @@ def main():
   # File extensions and endings to ignore
   ignore_extensions = ["_test", "_test.py", "_test_gpu", "_test_gpu.py"]
 
-  ignored_files = 0
-  blacklisted_files = len(BLACKLIST)
+  ignored_files_count = 0
+  blacklisted_dependencies_count = len(DEPENDENCY_BLACKLIST)
   # Compare dependencies
   for dependency in tf_py_test_dependencies_list:
     if dependency and dependency.startswith("//tensorflow"):
@@ -158,16 +164,16 @@ def main():
       # Ignore extensions
       if any(dependency.endswith(ext) for ext in ignore_extensions):
         ignore = True
-        ignored_files += 1
+        ignored_files_count += 1
 
-      # Check if the dependency is in the pip package, the blacklist, or
-      # should be ignored because of its file extension
+      # Check if the dependency is in the pip package, the dependency blacklist,
+      # or should be ignored because of its file extension.
       if not (ignore or dependency in pip_package_dependencies_list or
-              dependency in BLACKLIST):
+              dependency in DEPENDENCY_BLACKLIST):
         missing_dependencies.append(dependency)
 
-  print("Ignored files: %d" % ignored_files)
-  print("Blacklisted files: %d" % blacklisted_files)
+  print("Ignored files count: %d" % ignored_files_count)
+  print("Blacklisted dependencies count: %d" % blacklisted_dependencies_count)
   if missing_dependencies:
     print("Missing the following dependencies from pip_packages:")
     for missing_dependency in missing_dependencies:
-- 
GitLab


From e5b4b8daf1db2f49173a0eace291172cc9b1e0d3 Mon Sep 17 00:00:00 2001
From: Ben Zinberg <bzinberg@google.com>
Date: Thu, 3 Jan 2019 10:37:47 -0800
Subject: [PATCH 0174/1095] Minor correction to the docs: `tf.Dimension(None)`
 does not `==` itself.

PiperOrigin-RevId: 227711581
---
 tensorflow/python/framework/tensor_shape.py | 49 +++++++++++----------
 1 file changed, 25 insertions(+), 24 deletions(-)

diff --git a/tensorflow/python/framework/tensor_shape.py b/tensorflow/python/framework/tensor_shape.py
index a4c626c64c..a7537bb5f1 100644
--- a/tensorflow/python/framework/tensor_shape.py
+++ b/tensorflow/python/framework/tensor_shape.py
@@ -271,10 +271,11 @@ class Dimension(object):
     Dimensions are combined as follows:
 
     ```python
-    tf.Dimension(n)   .merge_with(tf.Dimension(n))    == tf.Dimension(n)
-    tf.Dimension(n)   .merge_with(tf.Dimension(None)) == tf.Dimension(n)
-    tf.Dimension(None).merge_with(tf.Dimension(n))    == tf.Dimension(n)
-    tf.Dimension(None).merge_with(tf.Dimension(None)) == tf.Dimension(None)
+    tf.Dimension(n)   .merge_with(tf.Dimension(n))     == tf.Dimension(n)
+    tf.Dimension(n)   .merge_with(tf.Dimension(None))  == tf.Dimension(n)
+    tf.Dimension(None).merge_with(tf.Dimension(n))     == tf.Dimension(n)
+    # equivalent to tf.Dimension(None)
+    tf.Dimension(None).merge_with(tf.Dimension(None))
 
     # raises ValueError for n != m
     tf.Dimension(n)   .merge_with(tf.Dimension(m))
@@ -304,10 +305,10 @@ class Dimension(object):
     Dimensions are summed as follows:
 
     ```python
-    tf.Dimension(m)    + tf.Dimension(n)    == tf.Dimension(m + n)
-    tf.Dimension(m)    + tf.Dimension(None) == tf.Dimension(None)
-    tf.Dimension(None) + tf.Dimension(n)    == tf.Dimension(None)
-    tf.Dimension(None) + tf.Dimension(None) == tf.Dimension(None)
+    tf.Dimension(m)    + tf.Dimension(n)     == tf.Dimension(m + n)
+    tf.Dimension(m)    + tf.Dimension(None)  # equiv. to tf.Dimension(None)
+    tf.Dimension(None) + tf.Dimension(n)     # equiv. to tf.Dimension(None)
+    tf.Dimension(None) + tf.Dimension(None)  # equiv. to tf.Dimension(None)
     ```
 
     Args:
@@ -339,10 +340,10 @@ class Dimension(object):
     Dimensions are subtracted as follows:
 
     ```python
-    tf.Dimension(m)    - tf.Dimension(n)    == tf.Dimension(m - n)
-    tf.Dimension(m)    - tf.Dimension(None) == tf.Dimension(None)
-    tf.Dimension(None) - tf.Dimension(n)    == tf.Dimension(None)
-    tf.Dimension(None) - tf.Dimension(None) == tf.Dimension(None)
+    tf.Dimension(m)    - tf.Dimension(n)     == tf.Dimension(m - n)
+    tf.Dimension(m)    - tf.Dimension(None)  # equiv. to tf.Dimension(None)
+    tf.Dimension(None) - tf.Dimension(n)     # equiv. to tf.Dimension(None)
+    tf.Dimension(None) - tf.Dimension(None)  # equiv. to tf.Dimension(None)
     ```
 
     Args:
@@ -378,10 +379,10 @@ class Dimension(object):
     Dimensions are summed as follows:
 
     ```python
-    tf.Dimension(m)    * tf.Dimension(n)    == tf.Dimension(m * n)
-    tf.Dimension(m)    * tf.Dimension(None) == tf.Dimension(None)
-    tf.Dimension(None) * tf.Dimension(n)    == tf.Dimension(None)
-    tf.Dimension(None) * tf.Dimension(None) == tf.Dimension(None)
+    tf.Dimension(m)    * tf.Dimension(n)     == tf.Dimension(m * n)
+    tf.Dimension(m)    * tf.Dimension(None)  # equiv. to tf.Dimension(None)
+    tf.Dimension(None) * tf.Dimension(n)     # equiv. to tf.Dimension(None)
+    tf.Dimension(None) * tf.Dimension(None)  # equiv. to tf.Dimension(None)
     ```
 
     Args:
@@ -417,10 +418,10 @@ class Dimension(object):
     Dimensions are divided as follows:
 
     ```python
-    tf.Dimension(m)    // tf.Dimension(n)    == tf.Dimension(m // n)
-    tf.Dimension(m)    // tf.Dimension(None) == tf.Dimension(None)
-    tf.Dimension(None) // tf.Dimension(n)    == tf.Dimension(None)
-    tf.Dimension(None) // tf.Dimension(None) == tf.Dimension(None)
+    tf.Dimension(m)    // tf.Dimension(n)     == tf.Dimension(m // n)
+    tf.Dimension(m)    // tf.Dimension(None)  # equiv. to tf.Dimension(None)
+    tf.Dimension(None) // tf.Dimension(n)     # equiv. to tf.Dimension(None)
+    tf.Dimension(None) // tf.Dimension(None)  # equiv. to tf.Dimension(None)
     ```
 
     Args:
@@ -475,10 +476,10 @@ class Dimension(object):
     Dimension moduli are computed as follows:
 
     ```python
-    tf.Dimension(m)    % tf.Dimension(n)    == tf.Dimension(m % n)
-    tf.Dimension(m)    % tf.Dimension(None) == tf.Dimension(None)
-    tf.Dimension(None) % tf.Dimension(n)    == tf.Dimension(None)
-    tf.Dimension(None) % tf.Dimension(None) == tf.Dimension(None)
+    tf.Dimension(m)    % tf.Dimension(n)     == tf.Dimension(m % n)
+    tf.Dimension(m)    % tf.Dimension(None)  # equiv. to tf.Dimension(None)
+    tf.Dimension(None) % tf.Dimension(n)     # equiv. to tf.Dimension(None)
+    tf.Dimension(None) % tf.Dimension(None)  # equiv. to tf.Dimension(None)
     ```
 
     Args:
-- 
GitLab


From 0d969989e0e4fbddb8c57eadf8f4c72d975fbd6b Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Thu, 3 Jan 2019 10:41:14 -0800
Subject: [PATCH 0175/1095] Automated rollback of commit
 4d9173668fd3ba410532efd901467312b7418752

PiperOrigin-RevId: 227712180
---
 tensorflow/cc/BUILD              |  1 +
 tensorflow/cc/saved_model/BUILD  |  2 +
 tensorflow/compiler/tf2xla/BUILD |  1 +
 tensorflow/core/BUILD            |  3 +-
 tensorflow/core/kernels/BUILD    | 83 +-------------------------------
 tensorflow/tensorflow.bzl        |  2 +
 6 files changed, 10 insertions(+), 82 deletions(-)

diff --git a/tensorflow/cc/BUILD b/tensorflow/cc/BUILD
index a09becc49b..cf6d6050fa 100644
--- a/tensorflow/cc/BUILD
+++ b/tensorflow/cc/BUILD
@@ -150,6 +150,7 @@ cc_library_with_android_deps(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:ops",
         "//tensorflow/core:protos_all_cc",
     ],
 )
diff --git a/tensorflow/cc/saved_model/BUILD b/tensorflow/cc/saved_model/BUILD
index 52345a376c..836678d364 100644
--- a/tensorflow/cc/saved_model/BUILD
+++ b/tensorflow/cc/saved_model/BUILD
@@ -81,6 +81,7 @@ cc_library(
     ] + if_not_mobile([
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:lib",
+        "//tensorflow/core:ops",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:tensorflow",
     ]) + if_android([
@@ -100,6 +101,7 @@ cc_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:ops",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core/util/tensor_bundle:naming",
         # mobile not supported yet
diff --git a/tensorflow/compiler/tf2xla/BUILD b/tensorflow/compiler/tf2xla/BUILD
index d8123e956f..dfb6b57161 100644
--- a/tensorflow/compiler/tf2xla/BUILD
+++ b/tensorflow/compiler/tf2xla/BUILD
@@ -281,6 +281,7 @@ tf_cc_test(
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
     ],
 )
 
diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 1afac057d9..f9552d5965 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -1526,6 +1526,7 @@ cc_library(
         ":framework_internal",
         ":lib",
         ":lib_internal",
+        ":ops",
         ":protos_all_cc",
         ":shape_inference_testutil",
         ":tensor_testutil",
@@ -3072,6 +3073,7 @@ tf_cuda_library(
         ":lib",
         ":lib_internal",
         ":metrics",
+        ":ops",
         ":proto_text",
         ":protos_all_cc",
         "//tensorflow/core/debug:debug_graph_utils",
@@ -3854,7 +3856,6 @@ tf_cc_test(
         "ops/cudnn_rnn_ops_test.cc",
     ],
     deps = [
-        ":cudnn_rnn_ops",
         "//tensorflow/core",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index c72d04d3af..fd1df63c95 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -157,7 +157,6 @@ tf_kernel_library(
     name = "collective_ops",
     prefix = "collective_ops",
     deps = [
-        "//tensorflow/core:collective_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:protos_all_cc",
@@ -222,7 +221,6 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/core:nn_ops_op_lib",
         "//third_party/eigen3",
     ],
     alwayslink = 1,
@@ -313,7 +311,6 @@ tf_kernel_library(
         "//tensorflow/core/nccl:nccl_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:gpu_headers_lib",
-        "//tensorflow/core:nccl_ops_op_lib",
     ]),
 )
 
@@ -459,7 +456,6 @@ cc_library(
     name = "batch_kernels",
     srcs = ["batch_kernels.cc"],
     deps = [
-        "//tensorflow/core:batch_ops_op_lib",
         "//tensorflow/core:framework_headers_lib",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core/kernels:concat_lib_hdrs",
@@ -686,7 +682,6 @@ ARRAY_DEPS = [
     ":ops_util",
     ":transpose_functor",
     "//tensorflow/core:array_grad",
-    "//tensorflow/core:array_ops_op_lib",
     "//tensorflow/core:core_cpu",
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
@@ -715,7 +710,6 @@ tf_kernel_library(
     deps = [
         "//tensorflow/core:framework_headers_lib",
         "//tensorflow/core:lib",
-        "//tensorflow/core:set_ops_op_lib",
         "//third_party/eigen3",
     ],
 )
@@ -1079,7 +1073,6 @@ tf_kernel_library(
     srcs = ["ragged_gather_op.cc"],
     deps = [
         "//tensorflow/core:framework",
-        "//tensorflow/core:ragged_array_ops_op_lib",
     ],
 )
 
@@ -1090,7 +1083,6 @@ tf_cc_test(
     deps = [
         ":ragged_gather_op",
         "//tensorflow/core:framework",
-        "//tensorflow/core:ragged_array_ops_op_lib",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
@@ -1103,7 +1095,6 @@ tf_kernel_library(
     srcs = ["ragged_range_op.cc"],
     deps = [
         "//tensorflow/core:framework",
-        "//tensorflow/core:ragged_math_ops_op_lib",
     ],
 )
 
@@ -1113,7 +1104,6 @@ tf_cc_test(
     deps = [
         ":ragged_range_op",
         "//tensorflow/core:framework",
-        "//tensorflow/core:ragged_math_ops_op_lib",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
@@ -1126,7 +1116,6 @@ tf_kernel_library(
     srcs = ["ragged_tensor_to_sparse_kernel.cc"],
     deps = [
         "//tensorflow/core:framework",
-        "//tensorflow/core:ragged_conversion_ops_op_lib",
     ],
 )
 
@@ -1138,7 +1127,6 @@ tf_cc_test(
         ":ragged_tensor_to_sparse_kernel",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
-        "//tensorflow/core:ragged_conversion_ops_op_lib",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
@@ -1152,7 +1140,6 @@ tf_kernel_library(
     visibility = ["//visibility:public"],
     deps = [
         ":gpu_util_hdrs",
-        "//tensorflow/core:cudnn_rnn_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
@@ -1229,7 +1216,6 @@ tf_cuda_cc_test(
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
-        "//tensorflow/core:math_ops_op_lib",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
@@ -1754,7 +1740,6 @@ tf_kernel_library(
     prefix = "candidate_sampler_ops",
     deps = [
         ":range_sampler",
-        "//tensorflow/core:candidate_sampling_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
     ],
@@ -1788,7 +1773,6 @@ tf_kernel_library(
     name = "control_flow_ops",
     prefix = "control_flow_ops",
     deps = [
-        "//tensorflow/core:control_flow_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
     ],
@@ -1800,7 +1784,6 @@ tf_kernel_library(
     deps = [
         ":bounds_check",
         ":ops_util",
-        "//tensorflow/core:ctc_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core/util/ctc:ctc_beam_search_lib",
@@ -1881,7 +1864,6 @@ DATA_FLOW_DEPS = [
     ":typed_queue",
     "//third_party/eigen3",
     "//tensorflow/core:core_cpu",
-    "//tensorflow/core:data_flow_ops_op_lib",
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
@@ -1946,7 +1928,6 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/core:scoped_allocator_ops_op_lib",
     ],
 )
 
@@ -1965,7 +1946,6 @@ tf_cuda_cc_test(
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
-        "//tensorflow/core:math_ops_op_lib",
         "//tensorflow/core:proto_text",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:test",
@@ -2013,7 +1993,6 @@ tf_kernel_library(
 DYNAMIC_DEPS = [
     ":bounds_check",
     "//tensorflow/core:core_cpu",
-    "//tensorflow/core:data_flow_ops_op_lib",
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
@@ -2046,7 +2025,6 @@ LOOKUP_DEPS = [
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
-    "//tensorflow/core:lookup_ops_op_lib",
 ]
 
 tf_kernel_library(
@@ -2075,7 +2053,6 @@ tf_kernel_library(
     deps = [
         ":lookup_table_init_op",
         ":lookup_table_op",
-        "//tensorflow/core:checkpoint_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//third_party/eigen3",
@@ -2086,7 +2063,6 @@ tf_kernel_library(
     name = "load_and_remap_matrix_op",
     srcs = ["load_and_remap_matrix_op.cc"],
     deps = [
-        "//tensorflow/core:checkpoint_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
@@ -2233,7 +2209,6 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/core:resource_variable_ops_op_lib",
         "@com_google_absl//absl/strings",
     ],
 )
@@ -2250,7 +2225,6 @@ tf_kernel_library(
         ":concat_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
-        "//tensorflow/core:list_ops_op_lib",
         "//third_party/eigen3",
     ],
 )
@@ -2261,7 +2235,6 @@ tf_kernel_library(
     deps = [
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
-        "//tensorflow/core:user_ops_op_lib",
     ],
 )
 
@@ -2284,7 +2257,6 @@ tf_kernel_library(
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:framework",
-        "//tensorflow/core:functional_ops_op_lib",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//third_party/eigen3",
@@ -2297,7 +2269,6 @@ tf_kernel_library(
     deps = [
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:framework",
-        "//tensorflow/core:functional_ops_op_lib",
         "//tensorflow/core:lib",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core/grappler:grappler_item",
@@ -2340,7 +2311,6 @@ IMAGE_DEPS = [
     "//third_party/eigen3",
     "//tensorflow/core:framework",
     "//tensorflow/core:gif_internal",
-    "//tensorflow/core:image_ops_op_lib",
     "//tensorflow/core:jpeg_internal",
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
@@ -2680,7 +2650,6 @@ cc_library(
 IO_DEPS = [
     ":ops_util",
     "//tensorflow/core:framework",
-    "//tensorflow/core:io_ops_op_lib",
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
     "//tensorflow/core:protos_all_cc",
@@ -2724,7 +2693,6 @@ SAVE_RESTORE_DEPS = [
     ":bounds_check_lib",
     ":save_restore_tensor",
     "//tensorflow/core:framework",
-    "//tensorflow/core:io_ops_op_lib",
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
     "//tensorflow/core:protos_all_cc",
@@ -2843,7 +2811,6 @@ LINALG_DEPS = [
     "//third_party/eigen3",
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
-    "//tensorflow/core:linalg_ops_op_lib",
 ] + if_cuda([
     ":cuda_solvers",
     ":transpose_functor",
@@ -2987,7 +2954,6 @@ LOGGING_DEPS = [
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
-    "//tensorflow/core:logging_ops_op_lib",
     "//tensorflow/core:protos_all_cc",
 ]
 
@@ -3059,7 +3025,6 @@ tf_kernel_library(
         ":bounds_check",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
-        "//tensorflow/core:manip_ops_op_lib",
         "//third_party/eigen3",
     ],
 )
@@ -3091,7 +3056,6 @@ MATH_DEPS = [
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
     "//tensorflow/core:math_grad",
-    "//tensorflow/core:math_ops_op_lib",
     "//third_party/eigen3",
 ]
 
@@ -3202,7 +3166,7 @@ tf_kernel_library(
 tf_kernel_library(
     name = "cwise_op",
     prefix = "cwise_op",
-    deps = MATH_DEPS + ["//tensorflow/core:bitwise_ops_op_lib"],
+    deps = MATH_DEPS,
 )
 
 tf_kernel_library(
@@ -3221,7 +3185,6 @@ tf_kernel_library(
     name = "fft_ops",
     prefix = "fft_ops",
     deps = MATH_DEPS + [
-        "//tensorflow/core:spectral_ops_op_lib",
     ] + if_cuda([
         "//tensorflow/core/platform/default/build_config:cufft_plugin",
     ]),
@@ -3420,10 +3383,7 @@ tf_cuda_cc_test(
         ":quantized_ops",
         "//tensorflow/cc:cc_ops",
         "//tensorflow/cc:client_session",
-        "//tensorflow/core:array_ops_op_lib",
         "//tensorflow/core:framework",
-        "//tensorflow/core:math_ops_op_lib",
-        "//tensorflow/core:nn_ops_op_lib",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
@@ -3681,7 +3641,6 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/core:nn_ops_op_lib",
     ] + select({
         ":xsmm_convolutions": [
             "@libxsmm_archive//:xsmm_avx",
@@ -3703,7 +3662,6 @@ tf_kernel_library(
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
-        "//tensorflow/core:nn_ops_op_lib",
     ] + if_cuda([
         "@cub_archive//:cub",
         "@local_config_cuda//cuda:cudnn_header",
@@ -3723,7 +3681,6 @@ tf_kernel_library(
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
-        "//tensorflow/core:nn_ops_op_lib",
     ] + if_cuda([
         "@local_config_cuda//cuda:cudnn_header",
     ]),
@@ -3771,9 +3728,8 @@ NN_DEPS = [
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
     "//tensorflow/core:nn_grad",
-    "//tensorflow/core:nn_ops_op_lib",
     "//third_party/eigen3",
-] + if_mkl(["//tensorflow/core:mkl_nn_ops_op_lib"])
+]
 
 tf_kernel_library(
     name = "batch_norm_op",
@@ -3893,7 +3849,6 @@ tf_kernel_library(
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:nn_grad",
-        "//tensorflow/core:nn_ops_op_lib",
     ] + if_cuda(["@cub_archive//:cub"]),
 )
 
@@ -4001,7 +3956,6 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/core:nn_ops_op_lib",
         "//tensorflow/core:stream_executor",
         "//third_party/eigen3",
     ],
@@ -4045,7 +3999,6 @@ tf_kernel_library(
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
-        "//tensorflow/core:nn_ops_op_lib",
         "//third_party/eigen3",
     ],
 )
@@ -4127,7 +4080,6 @@ cc_library(
 PARSING_DEPS = [
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
-    "//tensorflow/core:parsing_ops_op_lib",
     "//tensorflow/core:proto_text",
     "//tensorflow/core:protos_all_cc",
 ]
@@ -4196,7 +4148,6 @@ RANDOM_OPS_DEPS = [
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
-    "//tensorflow/core:random_ops_op_lib",
 ]
 
 tf_kernel_library(
@@ -4235,7 +4186,6 @@ tf_kernel_library(
         ":random_op",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
-        "//tensorflow/core:stateless_random_ops_op_lib",
     ],
 )
 
@@ -4250,8 +4200,6 @@ cc_library(
 REQUIRED_DEPS = [
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
-    "//tensorflow/core:no_op_op_lib",
-    "//tensorflow/core:sendrecv_ops_op_lib",
 ]
 
 tf_kernel_library(
@@ -4312,7 +4260,6 @@ cc_library(
 SPARSE_DEPS = [
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
-    "//tensorflow/core:sparse_ops_op_lib",
 ]
 
 tf_kernel_library(
@@ -4557,7 +4504,6 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/core:sdca_ops_op_lib",
         "//third_party/eigen3",
         "@farmhash_archive//:farmhash",
     ],
@@ -4597,7 +4543,6 @@ STATE_DEPS = [
     "//third_party/eigen3",
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
-    "//tensorflow/core:state_ops_op_lib",
 ] + if_sycl(["//tensorflow/core:sycl_runtime"])
 
 tf_kernel_library(
@@ -4735,7 +4680,6 @@ STRING_DEPS = [
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
-    "//tensorflow/core:string_ops_op_lib",
 ]
 
 tf_kernel_library(
@@ -4886,7 +4830,6 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/core:string_ops_op_lib",
         "//third_party/eigen3",
         "//third_party/icu/data:conversion_data",
         "@icu//:common",
@@ -4908,7 +4851,6 @@ tf_kernel_library(
         ":variable_ops",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
-        "//tensorflow/core:training_ops_op_lib",
         "//third_party/eigen3",
     ],
 )
@@ -4967,7 +4909,6 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/core:random_ops_op_lib",
     ],
 )
 
@@ -4995,7 +4936,6 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/core:random_ops_op_lib",
     ],
 )
 
@@ -5904,12 +5844,9 @@ tf_kernel_library(
         ":ops_util",
         ":pooling_ops",
         ":quantization_utils",
-        "//tensorflow/core:array_ops_op_lib",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
-        "//tensorflow/core:math_ops_op_lib",
-        "//tensorflow/core:nn_ops_op_lib",
         "//third_party/eigen3",
         "@gemmlowp",
     ],
@@ -6479,7 +6416,6 @@ tf_kernel_library(
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:protos_all_cc",
-        "//tensorflow/core:remote_fused_graph_ops_op_lib",
     ],
 )
 
@@ -6634,8 +6570,6 @@ tf_mkl_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/core:mkl_nn_ops_op_lib",
-        "//tensorflow/core:nn_ops_op_lib",
     ] + mkl_deps(),
 )
 
@@ -6685,8 +6619,6 @@ tf_mkl_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/core:mkl_nn_ops_op_lib",
-        "//tensorflow/core:nn_ops_op_lib",
     ] + mkl_deps(),
 )
 
@@ -6705,8 +6637,6 @@ tf_mkl_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/core:mkl_nn_ops_op_lib",
-        "//tensorflow/core:nn_ops_op_lib",
     ] + mkl_deps(),
 )
 
@@ -6720,8 +6650,6 @@ tf_mkl_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/core:mkl_nn_ops_op_lib",
-        "//tensorflow/core:nn_ops_op_lib",
         "//third_party/eigen3",
     ] + mkl_deps(),
 )
@@ -6736,8 +6664,6 @@ tf_mkl_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/core:mkl_nn_ops_op_lib",
-        "//tensorflow/core:nn_ops_op_lib",
         "//third_party/eigen3",
     ] + mkl_deps(),
 )
@@ -6878,7 +6804,6 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:protos_all_cc",
-        "//tensorflow/core:summary_ops_op_lib",
         "//tensorflow/core/lib/db:sqlite",
     ] + if_not_v2([
         "//tensorflow/contrib/tensorboard/db:schema",
@@ -6893,7 +6818,6 @@ tf_kernel_library(
         "decode_proto_op.cc",
     ],
     deps = [
-        "//tensorflow/core:decode_proto_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core/util/proto:decode",
@@ -6907,7 +6831,6 @@ tf_kernel_library(
     name = "encode_proto_op",
     srcs = ["encode_proto_op.cc"],
     deps = [
-        "//tensorflow/core:encode_proto_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core/util/proto:descriptors",
@@ -6925,7 +6848,6 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/core:rpc_ops_op_lib",
         "//tensorflow/core/util/rpc:call_container",
         "//tensorflow/core/util/rpc:rpc_factory",
         "//tensorflow/core/util/rpc:rpc_factory_registry",
@@ -6938,7 +6860,6 @@ tf_kernel_library(
     srcs = ["unicode_script_op.cc"],
     deps = [
         "//tensorflow/core:framework",
-        "//tensorflow/core:string_ops_op_lib",
         "@icu//:common",
     ],
 )
diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl
index 1024b686eb..b9069768e8 100644
--- a/tensorflow/tensorflow.bzl
+++ b/tensorflow/tensorflow.bzl
@@ -590,6 +590,7 @@ def tf_gen_op_wrappers_cc(
             clean_dep("//tensorflow/core:core_cpu"),
             clean_dep("//tensorflow/core:framework"),
             clean_dep("//tensorflow/core:lib"),
+            clean_dep("//tensorflow/core:ops"),
             clean_dep("//tensorflow/core:protos_all_cc"),
         ]) + if_android([
             clean_dep("//tensorflow/core:android_tensorflow_lib"),
@@ -606,6 +607,7 @@ def tf_gen_op_wrappers_cc(
             clean_dep("//tensorflow/core:core_cpu"),
             clean_dep("//tensorflow/core:framework"),
             clean_dep("//tensorflow/core:lib"),
+            clean_dep("//tensorflow/core:ops"),
             clean_dep("//tensorflow/core:protos_all_cc"),
         ]) + if_android([
             clean_dep("//tensorflow/core:android_tensorflow_lib"),
-- 
GitLab


From f6fa3515db311b666dd4bd529f9b8b87575ce978 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Thu, 3 Jan 2019 10:44:43 -0800
Subject: [PATCH 0176/1095] Silence a noisy __del__

Popped up sometimes, looking something like:

Exception ignored in: <bound method BaseSession.__del__ of <tensorflow.python.client.session.Session object at 0x7ff8f47827f0>>
Traceback (most recent call last):
  File "[...]tensorflow/python/client/session.py", line 738, in __del__
TypeError: 'NoneType' object is not callable

Which I'm guessing is TF_DeleteSession being None.

PiperOrigin-RevId: 227712751
---
 tensorflow/python/client/session.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/client/session.py b/tensorflow/python/client/session.py
index 87a200ed33..b97eb884b3 100644
--- a/tensorflow/python/client/session.py
+++ b/tensorflow/python/client/session.py
@@ -736,10 +736,11 @@ class BaseSession(SessionInterface):
     if self._session is not None:
       try:
         tf_session.TF_DeleteSession(self._session)
-      except AttributeError:
-        # At shutdown, `c_api_util` or `tf_session` may have been garbage
-        # collected, causing the above method calls to fail. In this case,
-        # silently leak since the program is about to terminate anyway.
+      except (AttributeError, TypeError):
+        # At shutdown, `c_api_util`, `tf_session`, or
+        # `tf_session.TF_DeleteSession` may have been garbage collected, causing
+        # the above method calls to fail. In this case, silently leak since the
+        # program is about to terminate anyway.
         pass
       self._session = None
 
-- 
GitLab


From e518527f10048b10956c7192de1411c36a3a1938 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 3 Jan 2019 11:59:51 -0800
Subject: [PATCH 0177/1095] Spectral Normalization implementation in TFGAN

PiperOrigin-RevId: 227725854
---
 tensorflow/contrib/gan/BUILD                  |  43 +++
 .../contrib/gan/python/features/__init__.py   |   3 +
 .../features/python/spectral_normalization.py |  32 ++
 .../python/spectral_normalization_impl.py     | 315 ++++++++++++++++
 .../python/spectral_normalization_test.py     | 354 ++++++++++++++++++
 5 files changed, 747 insertions(+)
 create mode 100644 tensorflow/contrib/gan/python/features/python/spectral_normalization.py
 create mode 100644 tensorflow/contrib/gan/python/features/python/spectral_normalization_impl.py
 create mode 100644 tensorflow/contrib/gan/python/features/python/spectral_normalization_test.py

diff --git a/tensorflow/contrib/gan/BUILD b/tensorflow/contrib/gan/BUILD
index 97184dabb0..0626875b76 100644
--- a/tensorflow/contrib/gan/BUILD
+++ b/tensorflow/contrib/gan/BUILD
@@ -132,6 +132,7 @@ py_library(
         ":clip_weights",
         ":conditioning_utils",
         ":random_tensor_pool",
+        ":spectral_normalization",
         ":virtual_batchnorm",
         "//tensorflow/python:util",
     ],
@@ -676,3 +677,45 @@ py_test(
         "//third_party/py/numpy",
     ],
 )
+
+py_library(
+    name = "spectral_normalization",
+    srcs = [
+        "python/features/python/spectral_normalization.py",
+        "python/features/python/spectral_normalization_impl.py",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:init_ops",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:standard_ops",
+        "//tensorflow/python:variable_scope",
+        "//tensorflow/python/keras:engine",
+    ],
+)
+
+py_test(
+    name = "spectral_normalization_test",
+    srcs = ["python/features/python/spectral_normalization_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":spectral_normalization",
+        "//tensorflow/contrib/layers:layers_py",
+        "//tensorflow/contrib/slim",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:init_ops",
+        "//tensorflow/python:layers",
+        "//tensorflow/python:linalg_ops",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:variable_scope",
+        "//tensorflow/python:variables",
+        "//tensorflow/python/keras:layers",
+        "//third_party/py/numpy",
+    ],
+)
diff --git a/tensorflow/contrib/gan/python/features/__init__.py b/tensorflow/contrib/gan/python/features/__init__.py
index 4816daf760..410c3a0205 100644
--- a/tensorflow/contrib/gan/python/features/__init__.py
+++ b/tensorflow/contrib/gan/python/features/__init__.py
@@ -27,11 +27,13 @@ from __future__ import print_function
 from tensorflow.contrib.gan.python.features.python import clip_weights
 from tensorflow.contrib.gan.python.features.python import conditioning_utils
 from tensorflow.contrib.gan.python.features.python import random_tensor_pool
+from tensorflow.contrib.gan.python.features.python import spectral_normalization
 from tensorflow.contrib.gan.python.features.python import virtual_batchnorm
 
 from tensorflow.contrib.gan.python.features.python.clip_weights import *
 from tensorflow.contrib.gan.python.features.python.conditioning_utils import *
 from tensorflow.contrib.gan.python.features.python.random_tensor_pool import *
+from tensorflow.contrib.gan.python.features.python.spectral_normalization import *
 from tensorflow.contrib.gan.python.features.python.virtual_batchnorm import *
 # pylint: enable=unused-import,wildcard-import
 
@@ -40,5 +42,6 @@ from tensorflow.python.util.all_util import remove_undocumented
 _allowed_symbols = clip_weights.__all__
 _allowed_symbols += conditioning_utils.__all__
 _allowed_symbols += random_tensor_pool.__all__
+_allowed_symbols += spectral_normalization.__all__
 _allowed_symbols += virtual_batchnorm.__all__
 remove_undocumented(__name__, _allowed_symbols)
diff --git a/tensorflow/contrib/gan/python/features/python/spectral_normalization.py b/tensorflow/contrib/gan/python/features/python/spectral_normalization.py
new file mode 100644
index 0000000000..54d3d0a218
--- /dev/null
+++ b/tensorflow/contrib/gan/python/features/python/spectral_normalization.py
@@ -0,0 +1,32 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Keras-like layers and utilities that implement Spectral Normalization.
+
+Based on "Spectral Normalization for Generative Adversarial Networks" by Miyato,
+et al in ICLR 2018. https://openreview.net/pdf?id=B1QRgziT-
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.contrib.gan.python.features.python import spectral_normalization_impl
+# pylint: disable=wildcard-import
+from tensorflow.contrib.gan.python.features.python.spectral_normalization_impl import *
+# pylint: enable=wildcard-import
+from tensorflow.python.util.all_util import remove_undocumented
+
+__all__ = spectral_normalization_impl.__all__
+remove_undocumented(__name__, __all__)
diff --git a/tensorflow/contrib/gan/python/features/python/spectral_normalization_impl.py b/tensorflow/contrib/gan/python/features/python/spectral_normalization_impl.py
new file mode 100644
index 0000000000..0cc653f0a7
--- /dev/null
+++ b/tensorflow/contrib/gan/python/features/python/spectral_normalization_impl.py
@@ -0,0 +1,315 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Keras-like layers and utilities that implement Spectral Normalization.
+
+Based on "Spectral Normalization for Generative Adversarial Networks" by Miyato,
+et al in ICLR 2018. https://openreview.net/pdf?id=B1QRgziT-
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import contextlib
+import numbers
+import re
+
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.keras.engine import base_layer_utils as keras_base_layer_utils
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import init_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import nn
+from tensorflow.python.ops import variable_scope
+from tensorflow.python.platform import tf_logging as logging
+
+__all__ = [
+    'compute_spectral_norm', 'spectral_normalize', 'spectral_norm_regularizer',
+    'spectral_normalization_custom_getter', 'keras_spectral_normalization'
+]
+
+# tf.bfloat16 should work, but tf.matmul converts those to tf.float32 which then
+# can't directly be assigned back to the tf.bfloat16 variable.
+_OK_DTYPES_FOR_SPECTRAL_NORM = (dtypes.float16, dtypes.float32, dtypes.float64)
+_PERSISTED_U_VARIABLE_SUFFIX = 'spectral_norm_u'
+
+
+def compute_spectral_norm(w_tensor, power_iteration_rounds=1, name=None):
+  """Estimates the largest singular value in the weight tensor.
+
+  Args:
+    w_tensor: The weight matrix whose spectral norm should be computed.
+    power_iteration_rounds: The number of iterations of the power method to
+      perform. A higher number yeilds a better approximation.
+    name: An optional scope name.
+
+  Returns:
+    The largest singular value (the spectral norm) of w.
+  """
+  with variable_scope.variable_scope(name, 'spectral_norm'):
+    # The paper says to flatten convnet kernel weights from
+    # (C_out, C_in, KH, KW) to (C_out, C_in * KH * KW). But TensorFlow's Conv2D
+    # kernel weight shape is (KH, KW, C_in, C_out), so it should be reshaped to
+    # (KH * KW * C_in, C_out), and similarly for other layers that put output
+    # channels as last dimension.
+    # n.b. this means that w here is equivalent to w.T in the paper.
+    w = array_ops.reshape(w_tensor, (-1, w_tensor.get_shape()[-1]))
+
+    # Persisted approximation of first left singular vector of matrix `w`.
+    u_var = variable_scope.get_variable(
+        _PERSISTED_U_VARIABLE_SUFFIX,
+        shape=(w.shape[0], 1),
+        dtype=w.dtype,
+        initializer=init_ops.random_normal_initializer(),
+        trainable=False)
+    u = u_var
+
+    # Use power iteration method to approximate spectral norm.
+    for _ in range(power_iteration_rounds):
+      # `v` approximates the first right singular vector of matrix `w`.
+      v = nn.l2_normalize(math_ops.matmul(array_ops.transpose(w), u))
+      u = nn.l2_normalize(math_ops.matmul(w, v))
+
+    # Update persisted approximation.
+    with ops.control_dependencies([u_var.assign(u, name='update_u')]):
+      u = array_ops.identity(u)
+
+    u = array_ops.stop_gradient(u)
+    v = array_ops.stop_gradient(v)
+
+    # Largest singular value of `w`.
+    spectral_norm = math_ops.matmul(
+        math_ops.matmul(array_ops.transpose(u), w), v)
+    spectral_norm.shape.assert_is_fully_defined()
+    spectral_norm.shape.assert_is_compatible_with([1, 1])
+
+    return spectral_norm[0][0]
+
+
+def spectral_normalize(w, power_iteration_rounds=1, name=None):
+  """Normalizes a weight matrix by its spectral norm.
+
+  Args:
+    w: The weight matrix to be normalized.
+    power_iteration_rounds: The number of iterations of the power method to
+      perform. A higher number yeilds a better approximation.
+    name: An optional scope name.
+
+  Returns:
+    A normalized weight matrix tensor.
+  """
+  with variable_scope.variable_scope(name, 'spectral_normalize'):
+    w_normalized = w / compute_spectral_norm(
+        w, power_iteration_rounds=power_iteration_rounds)
+    return array_ops.reshape(w_normalized, w.get_shape())
+
+
+def spectral_norm_regularizer(scale, power_iteration_rounds=1, scope=None):
+  """Returns a functions that can be used to apply spectral norm regularization.
+
+  Small spectral norms enforce a small Lipschitz constant, which is necessary
+  for Wasserstein GANs.
+
+  Args:
+    scale: A scalar multiplier. 0.0 disables the regularizer.
+    power_iteration_rounds: The number of iterations of the power method to
+      perform. A higher number yeilds a better approximation.
+    scope: An optional scope name.
+
+  Returns:
+    A function with the signature `sn(weights)` that applies spectral norm
+    regularization.
+
+  Raises:
+    ValueError: If scale is negative or if scale is not a float.
+  """
+  if isinstance(scale, numbers.Integral):
+    raise ValueError('scale cannot be an integer: %s' % scale)
+  if isinstance(scale, numbers.Real):
+    if scale < 0.0:
+      raise ValueError(
+          'Setting a scale less than 0 on a regularizer: %g' % scale)
+    if scale == 0.0:
+      logging.info('Scale of 0 disables regularizer.')
+      return lambda _: None
+
+  def sn(weights, name=None):
+    """Applies spectral norm regularization to weights."""
+    with ops.name_scope(scope, 'SpectralNormRegularizer', [weights]) as name:
+      scale_t = ops.convert_to_tensor(
+          scale, dtype=weights.dtype.base_dtype, name='scale')
+      return math_ops.multiply(
+          scale_t,
+          compute_spectral_norm(
+              weights, power_iteration_rounds=power_iteration_rounds),
+          name=name)
+
+  return sn
+
+
+def _default_name_filter(name):
+  """A filter function to identify common names of weight variables.
+
+  Args:
+    name: The variable name.
+
+  Returns:
+    Whether `name` is a standard name for a weight/kernel variables used in the
+    Keras, tf.layers, tf.contrib.layers or tf.contrib.slim libraries.
+  """
+  match = re.match(r'(.*\/)?(depthwise_|pointwise_)?(weights|kernel)$', name)
+  return match is not None
+
+
+def spectral_normalization_custom_getter(name_filter=_default_name_filter,
+                                         power_iteration_rounds=1):
+  """Custom getter that performs Spectral Normalization on a weight tensor.
+
+  Specifically it divides the weight tensor by its largest singular value. This
+  is intended to stabilize GAN training, by making the discriminator satisfy a
+  local 1-Lipschitz constraint.
+
+  Based on [Spectral Normalization for Generative Adversarial Networks][sn-gan].
+
+  [sn-gan]: https://openreview.net/forum?id=B1QRgziT-
+
+  To reproduce an SN-GAN, apply this custom_getter to every weight tensor of
+  your discriminator. The last dimension of the weight tensor must be the number
+  of output channels.
+
+  Apply this to layers by supplying this as the `custom_getter` of a
+  `tf.variable_scope`. For example:
+
+    with tf.variable_scope('discriminator',
+                           custom_getter=spectral_norm_getter()):
+      net = discriminator_fn(net)
+
+  IMPORTANT: Keras does not respect the custom_getter supplied by the
+  VariableScope, so Keras users should use `keras_spectral_normalization`
+  instead of (or in addition to) this approach.
+
+  It is important to carefully select to which weights you want to apply
+  Spectral Normalization. In general you want to normalize the kernels of
+  convolution and dense layers, but you do not want to normalize biases. You
+  also want to avoid normalizing batch normalization (and similar) variables,
+  but in general such layers play poorly with Spectral Normalization, since the
+  gamma can cancel out the normalization in other layers. By default we supply a
+  filter that matches the kernel variable names of the dense and convolution
+  layers of the tf.layers, tf.contrib.layers, tf.keras and tf.contrib.slim
+  libraries. If you are using anything else you'll need a custom `name_filter`.
+
+  This custom getter internally creates a variable used to compute the spectral
+  norm by power iteration. It will update every time the variable is accessed,
+  which means the normalized discriminator weights may change slightly whilst
+  training the generator. Whilst unusual, this matches how the paper's authors
+  implement it, and in general additional rounds of power iteration can't hurt.
+
+  Args:
+    name_filter: Optionally, a method that takes a Variable name as input and
+      returns whether this Variable should be normalized.
+    power_iteration_rounds: The number of iterations of the power method to
+      perform per step. A higher number yeilds a better approximation of the
+      true spectral norm.
+
+  Returns:
+    A custom getter function that applies Spectral Normalization to all
+    Variables whose names match `name_filter`.
+
+  Raises:
+    ValueError: If name_filter is not callable.
+  """
+  if not callable(name_filter):
+    raise ValueError('name_filter must be callable')
+
+  def _internal_getter(getter, name, *args, **kwargs):
+    """A custom getter function that applies Spectral Normalization.
+
+    Args:
+      getter: The true getter to call.
+      name: Name of new/existing variable, in the same format as
+        tf.get_variable.
+      *args: Other positional arguments, in the same format as tf.get_variable.
+      **kwargs: Keyword arguments, in the same format as tf.get_variable.
+
+    Returns:
+      The return value of `getter(name, *args, **kwargs)`, spectrally
+      normalized.
+
+    Raises:
+      ValueError: If used incorrectly, or if `dtype` is not supported.
+    """
+    if not name_filter(name):
+      return getter(name, *args, **kwargs)
+
+    if name.endswith(_PERSISTED_U_VARIABLE_SUFFIX):
+      raise ValueError(
+          'Cannot apply Spectral Normalization to internal variables created '
+          'for Spectral Normalization. Tried to normalized variable [%s]' %
+          name)
+
+    if kwargs['dtype'] not in _OK_DTYPES_FOR_SPECTRAL_NORM:
+      raise ValueError('Disallowed data type {}'.format(kwargs['dtype']))
+
+    # This layer's weight Variable/PartitionedVariable.
+    w_tensor = getter(name, *args, **kwargs)
+
+    if len(w_tensor.get_shape()) < 2:
+      raise ValueError(
+          'Spectral norm can only be applied to multi-dimensional tensors')
+
+    return spectral_normalize(
+        w_tensor,
+        power_iteration_rounds=power_iteration_rounds,
+        name=(name + '/spectral_normalize'))
+
+  return _internal_getter
+
+
+@contextlib.contextmanager
+def keras_spectral_normalization(name_filter=_default_name_filter,
+                                 power_iteration_rounds=1):
+  """A context manager that enables Spectral Normalization for Keras.
+
+  Keras doesn't respect the `custom_getter` in the VariableScope, so this is a
+  bit of a hack to make things work.
+
+  Usage:
+    with keras_spectral_normalization():
+      net = discriminator_fn(net)
+
+  Args:
+    name_filter: Optionally, a method that takes a Variable name as input and
+      returns whether this Variable should be normalized.
+    power_iteration_rounds: The number of iterations of the power method to
+      perform per step. A higher number yeilds a better approximation of the
+      true spectral norm.
+
+  Yields:
+    A context manager that wraps the standard Keras variable creation method
+    with the `spectral_normalization_custom_getter`.
+  """
+  original_make_variable = keras_base_layer_utils.make_variable
+  sn_getter = spectral_normalization_custom_getter(
+      name_filter=name_filter, power_iteration_rounds=power_iteration_rounds)
+
+  def make_variable_wrapper(name, *args, **kwargs):
+    return sn_getter(original_make_variable, name, *args, **kwargs)
+
+  keras_base_layer_utils.make_variable = make_variable_wrapper
+
+  yield
+
+  keras_base_layer_utils.make_variable = original_make_variable
diff --git a/tensorflow/contrib/gan/python/features/python/spectral_normalization_test.py b/tensorflow/contrib/gan/python/features/python/spectral_normalization_test.py
new file mode 100644
index 0000000000..4ea21f70ec
--- /dev/null
+++ b/tensorflow/contrib/gan/python/features/python/spectral_normalization_test.py
@@ -0,0 +1,354 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for features.spectral_normalization."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib import slim
+from tensorflow.contrib.gan.python.features.python import spectral_normalization_impl as spectral_normalization
+from tensorflow.contrib.layers.python.layers import layers as contrib_layers
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.keras.layers import convolutional as keras_convolutional
+from tensorflow.python.keras.layers import core as keras_core
+from tensorflow.python.layers import convolutional as layers_convolutional
+from tensorflow.python.layers import core as layers_core
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import init_ops
+from tensorflow.python.ops import linalg_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import variable_scope
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import test
+
+
+class SpectralNormalizationTest(test.TestCase):
+
+  def testComputeSpectralNorm(self):
+    weights = variable_scope.get_variable(
+        'w', dtype=dtypes.float32, shape=[2, 3, 50, 100])
+    weights = math_ops.multiply(weights, 10.0)
+    s = linalg_ops.svd(
+        array_ops.reshape(weights, [-1, weights.shape[-1]]), compute_uv=False)
+    true_sn = s[..., 0]
+    estimated_sn = spectral_normalization.compute_spectral_norm(weights)
+
+    with self.cached_session() as sess:
+      sess.run(variables.global_variables_initializer())
+      np_true_sn = sess.run(true_sn)
+      for i in range(50):
+        est = sess.run(estimated_sn)
+        if i < 1:
+          np_est_1 = est
+        if i < 4:
+          np_est_5 = est
+        if i < 9:
+          np_est_10 = est
+        np_est_50 = est
+
+      # Check that the estimate improves with more iterations.
+      self.assertAlmostEqual(np_true_sn, np_est_50, 0)
+      self.assertGreater(
+          abs(np_true_sn - np_est_10), abs(np_true_sn - np_est_50))
+      self.assertGreater(
+          abs(np_true_sn - np_est_5), abs(np_true_sn - np_est_10))
+      self.assertGreater(abs(np_true_sn - np_est_1), abs(np_true_sn - np_est_5))
+
+  def testSpectralNormalize(self):
+    weights = variable_scope.get_variable(
+        'w', dtype=dtypes.float32, shape=[2, 3, 50, 100])
+    weights = math_ops.multiply(weights, 10.0)
+    normalized_weights = spectral_normalization.spectral_normalize(
+        weights, power_iteration_rounds=1)
+
+    unnormalized_sigma = linalg_ops.svd(
+        array_ops.reshape(weights, [-1, weights.shape[-1]]),
+        compute_uv=False)[..., 0]
+    normalized_sigma = linalg_ops.svd(
+        array_ops.reshape(normalized_weights, [-1, weights.shape[-1]]),
+        compute_uv=False)[..., 0]
+
+    with self.cached_session() as sess:
+      sess.run(variables.global_variables_initializer())
+      s0 = sess.run(unnormalized_sigma)
+
+      for i in range(50):
+        sigma = sess.run(normalized_sigma)
+        if i < 1:
+          s1 = sigma
+        if i < 5:
+          s5 = sigma
+        if i < 10:
+          s10 = sigma
+        s50 = sigma
+
+      self.assertAlmostEqual(1., s50, 0)
+      self.assertGreater(abs(s10 - 1.), abs(s50 - 1.))
+      self.assertGreater(abs(s5 - 1.), abs(s10 - 1.))
+      self.assertGreater(abs(s1 - 1.), abs(s5 - 1.))
+      self.assertGreater(abs(s0 - 1.), abs(s1 - 1.))
+
+  def _testLayerHelper(self, build_layer_fn, w_shape, b_shape, is_keras=False):
+    x = array_ops.placeholder(dtypes.float32, shape=[2, 10, 10, 3])
+
+    w_initial = np.random.randn(*w_shape) * 10
+    w_initializer = init_ops.constant_initializer(w_initial)
+    b_initial = np.random.randn(*b_shape)
+    b_initializer = init_ops.constant_initializer(b_initial)
+
+    if is_keras:
+      context_manager = spectral_normalization.keras_spectral_normalization()
+    else:
+      getter = spectral_normalization.spectral_normalization_custom_getter()
+      context_manager = variable_scope.variable_scope('', custom_getter=getter)
+
+    with context_manager:
+      (net,
+       expected_normalized_vars, expected_not_normalized_vars) = build_layer_fn(
+           x, w_initializer, b_initializer)
+
+    x_data = np.random.rand(*x.shape)
+
+    with self.cached_session() as sess:
+      sess.run(variables.global_variables_initializer())
+
+      # Before running a forward pass we still expect the variables values to
+      # differ from the initial value because of the normalizer.
+      w_befores = []
+      for name, var in expected_normalized_vars.items():
+        w_before = sess.run(var)
+        w_befores.append(w_before)
+        self.assertFalse(
+            np.allclose(w_initial, w_before),
+            msg=('%s appears not to be normalized. Before: %s After: %s' %
+                 (name, w_initial, w_before)))
+
+      # Not true for the unnormalized variables.
+      for name, var in expected_not_normalized_vars.items():
+        b_before = sess.run(var)
+        self.assertTrue(
+            np.allclose(b_initial, b_before),
+            msg=('%s appears to be unexpectedly normalized. '
+                 'Before: %s After: %s' % (name, b_initial, b_before)))
+
+      # Run a bunch of forward passes.
+      for _ in range(1000):
+        _ = sess.run(net, feed_dict={x: x_data})
+
+      # We expect this to have improved the estimate of the spectral norm,
+      # which should have changed the variable values and brought them close
+      # to the true Spectral Normalized values.
+      _, s, _ = np.linalg.svd(w_initial.reshape([-1, 3]))
+      exactly_normalized = w_initial / s[0]
+      for w_before, (name, var) in zip(w_befores,
+                                       expected_normalized_vars.items()):
+        w_after = sess.run(var)
+        self.assertFalse(
+            np.allclose(w_before, w_after, rtol=1e-8, atol=1e-8),
+            msg=('%s did not improve over many iterations. '
+                 'Before: %s After: %s' % (name, w_before, w_after)))
+        self.assertAllClose(
+            exactly_normalized,
+            w_after,
+            rtol=1e-4,
+            atol=1e-4,
+            msg=('Estimate of spectral norm for %s was innacurate. '
+                 'Normalized matrices do not match.'
+                 'Estimate: %s Actual: %s' % (name, w_after,
+                                              exactly_normalized)))
+
+  def testConv2D_Layers(self):
+
+    def build_layer_fn(x, w_initializer, b_initializer):
+      layer = layers_convolutional.Conv2D(
+          filters=3,
+          kernel_size=3,
+          padding='same',
+          kernel_initializer=w_initializer,
+          bias_initializer=b_initializer)
+      net = layer.apply(x)
+      expected_normalized_vars = {'tf.layers.Conv2d.kernel': layer.kernel}
+      expected_not_normalized_vars = {'tf.layers.Conv2d.bias': layer.bias}
+
+      return net, expected_normalized_vars, expected_not_normalized_vars
+
+    self._testLayerHelper(build_layer_fn, (3, 3, 3, 3), (3,))
+
+  def testConv2D_ContribLayers(self):
+
+    def build_layer_fn(x, w_initializer, b_initializer):
+      var_collection = {
+          'weights': ['CONTRIB_LAYERS_CONV2D_WEIGHTS'],
+          'biases': ['CONTRIB_LAYERS_CONV2D_BIASES']
+      }
+      net = contrib_layers.conv2d(
+          x,
+          3,
+          3,
+          weights_initializer=w_initializer,
+          biases_initializer=b_initializer,
+          variables_collections=var_collection)
+      weight_vars = ops.get_collection('CONTRIB_LAYERS_CONV2D_WEIGHTS')
+      self.assertEquals(1, len(weight_vars))
+      bias_vars = ops.get_collection('CONTRIB_LAYERS_CONV2D_BIASES')
+      self.assertEquals(1, len(bias_vars))
+      expected_normalized_vars = {
+          'contrib.layers.conv2d.weights': weight_vars[0]
+      }
+      expected_not_normalized_vars = {
+          'contrib.layers.conv2d.bias': bias_vars[0]
+      }
+
+      return net, expected_normalized_vars, expected_not_normalized_vars
+
+    self._testLayerHelper(build_layer_fn, (3, 3, 3, 3), (3,))
+
+  def testConv2D_Slim(self):
+
+    def build_layer_fn(x, w_initializer, b_initializer):
+      var_collection = {
+          'weights': ['SLIM_CONV2D_WEIGHTS'],
+          'biases': ['SLIM_CONV2D_BIASES']
+      }
+      net = slim.conv2d(
+          x,
+          3,
+          3,
+          weights_initializer=w_initializer,
+          biases_initializer=b_initializer,
+          variables_collections=var_collection)
+      weight_vars = ops.get_collection('SLIM_CONV2D_WEIGHTS')
+      self.assertEquals(1, len(weight_vars))
+      bias_vars = ops.get_collection('SLIM_CONV2D_BIASES')
+      self.assertEquals(1, len(bias_vars))
+      expected_normalized_vars = {'slim.conv2d.weights': weight_vars[0]}
+      expected_not_normalized_vars = {'slim.conv2d.bias': bias_vars[0]}
+
+      return net, expected_normalized_vars, expected_not_normalized_vars
+
+    self._testLayerHelper(build_layer_fn, (3, 3, 3, 3), (3,))
+
+  def testConv2D_Keras(self):
+
+    def build_layer_fn(x, w_initializer, b_initializer):
+      layer = keras_convolutional.Conv2D(
+          filters=3,
+          kernel_size=3,
+          padding='same',
+          kernel_initializer=w_initializer,
+          bias_initializer=b_initializer)
+      net = layer.apply(x)
+      expected_normalized_vars = {'keras.layers.Conv2d.kernel': layer.kernel}
+      expected_not_normalized_vars = {'keras.layers.Conv2d.bias': layer.bias}
+
+      return net, expected_normalized_vars, expected_not_normalized_vars
+
+    self._testLayerHelper(build_layer_fn, (3, 3, 3, 3), (3,), is_keras=True)
+
+  def testFC_Layers(self):
+
+    def build_layer_fn(x, w_initializer, b_initializer):
+      x = layers_core.Flatten()(x)
+      layer = layers_core.Dense(
+          units=3,
+          kernel_initializer=w_initializer,
+          bias_initializer=b_initializer)
+      net = layer.apply(x)
+      expected_normalized_vars = {'tf.layers.Dense.kernel': layer.kernel}
+      expected_not_normalized_vars = {'tf.layers.Dense.bias': layer.bias}
+
+      return net, expected_normalized_vars, expected_not_normalized_vars
+
+    self._testLayerHelper(build_layer_fn, (300, 3), (3,))
+
+  def testFC_ContribLayers(self):
+
+    def build_layer_fn(x, w_initializer, b_initializer):
+      var_collection = {
+          'weights': ['CONTRIB_LAYERS_FC_WEIGHTS'],
+          'biases': ['CONTRIB_LAYERS_FC_BIASES']
+      }
+      x = contrib_layers.flatten(x)
+      net = contrib_layers.fully_connected(
+          x,
+          3,
+          weights_initializer=w_initializer,
+          biases_initializer=b_initializer,
+          variables_collections=var_collection)
+      weight_vars = ops.get_collection('CONTRIB_LAYERS_FC_WEIGHTS')
+      self.assertEquals(1, len(weight_vars))
+      bias_vars = ops.get_collection('CONTRIB_LAYERS_FC_BIASES')
+      self.assertEquals(1, len(bias_vars))
+      expected_normalized_vars = {
+          'contrib.layers.fully_connected.weights': weight_vars[0]
+      }
+      expected_not_normalized_vars = {
+          'contrib.layers.fully_connected.bias': bias_vars[0]
+      }
+
+      return net, expected_normalized_vars, expected_not_normalized_vars
+
+    self._testLayerHelper(build_layer_fn, (300, 3), (3,))
+
+  def testFC_Slim(self):
+
+    def build_layer_fn(x, w_initializer, b_initializer):
+      var_collection = {
+          'weights': ['SLIM_FC_WEIGHTS'],
+          'biases': ['SLIM_FC_BIASES']
+      }
+      x = slim.flatten(x)
+      net = slim.fully_connected(
+          x,
+          3,
+          weights_initializer=w_initializer,
+          biases_initializer=b_initializer,
+          variables_collections=var_collection)
+      weight_vars = ops.get_collection('SLIM_FC_WEIGHTS')
+      self.assertEquals(1, len(weight_vars))
+      bias_vars = ops.get_collection('SLIM_FC_BIASES')
+      self.assertEquals(1, len(bias_vars))
+      expected_normalized_vars = {
+          'slim.fully_connected.weights': weight_vars[0]
+      }
+      expected_not_normalized_vars = {'slim.fully_connected.bias': bias_vars[0]}
+
+      return net, expected_normalized_vars, expected_not_normalized_vars
+
+    self._testLayerHelper(build_layer_fn, (300, 3), (3,))
+
+  def testFC_Keras(self):
+
+    def build_layer_fn(x, w_initializer, b_initializer):
+      x = keras_core.Flatten()(x)
+      layer = keras_core.Dense(
+          units=3,
+          kernel_initializer=w_initializer,
+          bias_initializer=b_initializer)
+      net = layer.apply(x)
+      expected_normalized_vars = {'keras.layers.Dense.kernel': layer.kernel}
+      expected_not_normalized_vars = {'keras.layers.Dense.bias': layer.bias}
+
+      return net, expected_normalized_vars, expected_not_normalized_vars
+
+    self._testLayerHelper(build_layer_fn, (300, 3), (3,), is_keras=True)
+
+
+if __name__ == '__main__':
+  test.main()
-- 
GitLab


From 41c30afbfb6a752f09f9b26db807b1a1ac494e8a Mon Sep 17 00:00:00 2001
From: Karim Nosir <karimnosseir@google.com>
Date: Thu, 3 Jan 2019 12:07:50 -0800
Subject: [PATCH 0178/1095] Enable kernel tests for ios.

PiperOrigin-RevId: 227727388
---
 tensorflow/core/BUILD                         |   9 +-
 tensorflow/lite/kernels/BUILD                 | 163 ------------------
 .../lite/kernels/batch_to_space_nd_test.cc    |   2 +
 tensorflow/lite/kernels/pad_test.cc           |   8 +
 tensorflow/lite/kernels/reshape_test.cc       |  10 ++
 .../lite/kernels/space_to_batch_nd_test.cc    |   4 +
 .../lite/kernels/space_to_depth_test.cc       |   2 +
 tensorflow/lite/kernels/strided_slice_test.cc |   2 +
 tensorflow/lite/kernels/transpose_test.cc     |   4 +
 9 files changed, 39 insertions(+), 165 deletions(-)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index f9552d5965..6b32e1fead 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -2348,7 +2348,12 @@ cc_library(
 
 cc_library(
     name = "tflite_portable_logging",
-    srcs = [],
+    srcs = [
+    ] + if_ios([
+        "platform/default/logging.cc",
+        "platform/env_time.cc",
+        "platform/posix/env_time.cc",
+    ]),
     hdrs = [
         "lib/bfloat16/bfloat16.h",
         "platform/default/integral_types.h",
@@ -2357,7 +2362,7 @@ cc_library(
         "platform/macros.h",
         "platform/platform.h",
         "platform/types.h",
-    ] + if_windows(["platform/windows/integral_types.h"]),
+    ] + if_windows(["platform/windows/integral_types.h"]) + if_ios(["platform/env_time.h"]),
     copts = tf_copts(),
     linkopts = ["-ldl"],
     deps = [
diff --git a/tensorflow/lite/kernels/BUILD b/tensorflow/lite/kernels/BUILD
index 5cc06c7a63..9abc33d297 100644
--- a/tensorflow/lite/kernels/BUILD
+++ b/tensorflow/lite/kernels/BUILD
@@ -25,9 +25,6 @@ tf_cc_test(
     name = "optional_tensor_test",
     size = "small",
     srcs = ["optional_tensor_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -122,9 +119,6 @@ tf_cc_test(
     name = "kernel_util_test",
     size = "small",
     srcs = ["kernel_util_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":kernel_util",
         "//tensorflow/lite/testing:util",
@@ -136,9 +130,6 @@ tf_cc_test(
     name = "test_util_test",
     size = "small",
     srcs = ["test_util_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",  # TODO(b/117786830)
-    ],
     deps = [
         ":test_util",
         "//tensorflow/lite/testing:util",
@@ -304,9 +295,6 @@ tf_cc_test(
     name = "audio_spectrogram_test",
     size = "small",
     srcs = ["audio_spectrogram_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -320,9 +308,6 @@ tf_cc_test(
     name = "mfcc_test",
     size = "small",
     srcs = ["mfcc_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -336,9 +321,6 @@ tf_cc_test(
     name = "detection_postprocess_test",
     size = "small",
     srcs = ["detection_postprocess_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -352,9 +334,6 @@ tf_cc_test(
     name = "relu1_test",
     size = "small",
     srcs = ["relu1_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -368,9 +347,6 @@ tf_cc_test(
     name = "sparse_output_fully_connected_test",
     size = "small",
     srcs = ["sparse_output_fully_connected_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -384,7 +360,6 @@ tf_cc_test(
     name = "activations_test",
     size = "small",
     srcs = ["activations_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -397,7 +372,6 @@ tf_cc_test(
     name = "add_test",
     size = "small",
     srcs = ["add_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -410,9 +384,6 @@ tf_cc_test(
     name = "arg_min_max_test",
     size = "small",
     srcs = ["arg_min_max_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -425,9 +396,6 @@ tf_cc_test(
     name = "div_test",
     size = "small",
     srcs = ["div_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -440,9 +408,6 @@ tf_cc_test(
     name = "sub_test",
     size = "small",
     srcs = ["sub_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -455,9 +420,6 @@ tf_cc_test(
     name = "transpose_test",
     size = "small",
     srcs = ["transpose_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -472,9 +434,6 @@ tf_cc_test(
     name = "space_to_batch_nd_test",
     size = "small",
     srcs = ["space_to_batch_nd_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -487,9 +446,6 @@ tf_cc_test(
     name = "batch_to_space_nd_test",
     size = "small",
     srcs = ["batch_to_space_nd_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -502,9 +458,6 @@ tf_cc_test(
     name = "cast_test",
     size = "small",
     srcs = ["cast_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -517,7 +470,6 @@ tf_cc_test(
     name = "concatenation_test",
     size = "small",
     srcs = ["concatenation_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -530,7 +482,6 @@ tf_cc_test(
     name = "conv_test",
     size = "small",
     srcs = ["conv_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -544,7 +495,6 @@ tf_cc_test(
     name = "depthwise_conv_test",
     size = "small",
     srcs = ["depthwise_conv_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -558,9 +508,6 @@ tf_cc_test(
     name = "dequantize_test",
     size = "small",
     srcs = ["dequantize_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -574,7 +521,6 @@ tf_cc_test(
     name = "basic_rnn_test",
     size = "small",
     srcs = ["basic_rnn_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -587,9 +533,6 @@ tf_cc_test(
     name = "bidirectional_sequence_lstm_test",
     size = "small",
     srcs = ["bidirectional_sequence_lstm_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -603,9 +546,6 @@ tf_cc_test(
     name = "floor_test",
     size = "small",
     srcs = ["floor_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -618,9 +558,6 @@ tf_cc_test(
     name = "elementwise_test",
     size = "small",
     srcs = ["elementwise_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -633,9 +570,6 @@ tf_cc_test(
     name = "unidirectional_sequence_lstm_test",
     size = "small",
     srcs = ["unidirectional_sequence_lstm_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -663,9 +597,6 @@ tf_cc_test(
     name = "unidirectional_sequence_rnn_test",
     size = "small",
     srcs = ["unidirectional_sequence_rnn_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -678,7 +609,6 @@ tf_cc_test(
     name = "l2norm_test",
     size = "small",
     srcs = ["l2norm_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -691,9 +621,6 @@ tf_cc_test(
     name = "exp_test",
     size = "small",
     srcs = ["exp_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -706,9 +633,6 @@ tf_cc_test(
     name = "fake_quant_test",
     size = "small",
     srcs = ["fake_quant_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -721,9 +645,6 @@ tf_cc_test(
     name = "maximum_minimum_test",
     size = "small",
     srcs = ["maximum_minimum_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -736,9 +657,6 @@ tf_cc_test(
     name = "reduce_test",
     size = "small",
     srcs = ["reduce_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -751,7 +669,6 @@ tf_cc_test(
     name = "mul_test",
     size = "small",
     srcs = ["mul_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -764,9 +681,6 @@ tf_cc_test(
     name = "pad_test",
     size = "small",
     srcs = ["pad_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -779,7 +693,6 @@ tf_cc_test(
     name = "reshape_test",
     size = "small",
     srcs = ["reshape_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -792,9 +705,6 @@ tf_cc_test(
     name = "gather_test",
     size = "small",
     srcs = ["gather_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -808,9 +718,6 @@ tf_cc_test(
     name = "topk_v2_test",
     size = "small",
     srcs = ["topk_v2_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -824,7 +731,6 @@ tf_cc_test(
     name = "resize_bilinear_test",
     size = "small",
     srcs = ["resize_bilinear_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -837,7 +743,6 @@ tf_cc_test(
     name = "resize_nearest_neighbor_test",
     size = "small",
     srcs = ["resize_nearest_neighbor_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -850,7 +755,6 @@ tf_cc_test(
     name = "svdf_test",
     size = "small",
     srcs = ["svdf_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -863,7 +767,6 @@ tf_cc_test(
     name = "embedding_lookup_test",
     size = "small",
     srcs = ["embedding_lookup_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -876,7 +779,6 @@ tf_cc_test(
     name = "embedding_lookup_sparse_test",
     size = "small",
     srcs = ["embedding_lookup_sparse_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -889,7 +791,6 @@ tf_cc_test(
     name = "fully_connected_test",
     size = "small",
     srcs = ["fully_connected_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -904,7 +805,6 @@ tf_cc_test(
     name = "local_response_norm_test",
     size = "small",
     srcs = ["local_response_norm_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -917,7 +817,6 @@ tf_cc_test(
     name = "pooling_test",
     size = "small",
     srcs = ["pooling_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -930,7 +829,6 @@ tf_cc_test(
     name = "softmax_test",
     size = "small",
     srcs = ["softmax_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -944,9 +842,6 @@ tf_cc_test(
     name = "log_softmax_test",
     size = "small",
     srcs = ["log_softmax_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -960,7 +855,6 @@ tf_cc_test(
     name = "lsh_projection_test",
     size = "small",
     srcs = ["lsh_projection_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -973,7 +867,6 @@ tf_cc_test(
     name = "hashtable_lookup_test",
     size = "small",
     srcs = ["hashtable_lookup_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -987,7 +880,6 @@ tf_cc_test(
     name = "layer_norm_lstm_test",
     size = "small",
     srcs = ["layer_norm_lstm_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -1001,7 +893,6 @@ tf_cc_test(
     name = "lstm_test",
     size = "small",
     srcs = ["lstm_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -1014,7 +905,6 @@ tf_cc_test(
     name = "skip_gram_test",
     size = "small",
     srcs = ["skip_gram_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -1028,7 +918,6 @@ tf_cc_test(
     name = "space_to_depth_test",
     size = "small",
     srcs = ["space_to_depth_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -1041,9 +930,6 @@ tf_cc_test(
     name = "split_test",
     size = "small",
     srcs = ["split_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -1056,9 +942,6 @@ tf_cc_test(
     name = "split_v_test",
     size = "small",
     srcs = ["split_v_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -1071,9 +954,6 @@ tf_cc_test(
     name = "squeeze_test",
     size = "small",
     srcs = ["squeeze_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -1086,9 +966,6 @@ tf_cc_test(
     name = "strided_slice_test",
     size = "small",
     srcs = ["strided_slice_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -1101,9 +978,6 @@ tf_cc_test(
     name = "tile_test",
     size = "small",
     srcs = ["tile_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -1119,9 +993,6 @@ tf_cc_test(
     srcs = [
         "comparisons_test.cc",
     ],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -1134,9 +1005,6 @@ tf_cc_test(
     name = "neg_test",
     size = "small",
     srcs = ["neg_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -1151,9 +1019,6 @@ tf_cc_test(
     srcs = [
         "select_test.cc",
     ],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -1168,9 +1033,6 @@ tf_cc_test(
     srcs = [
         "slice_test.cc",
     ],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -1183,9 +1045,6 @@ tf_cc_test(
     name = "transpose_conv_test",
     size = "small",
     srcs = ["transpose_conv_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -1199,9 +1058,6 @@ tf_cc_test(
     name = "expand_dims_test",
     size = "small",
     srcs = ["expand_dims_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -1215,9 +1071,6 @@ tf_cc_test(
     name = "sparse_to_dense_test",
     size = "small",
     srcs = ["sparse_to_dense_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -1231,9 +1084,6 @@ tf_cc_test(
     name = "shape_test",
     size = "small",
     srcs = ["shape_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -1247,9 +1097,6 @@ tf_cc_test(
     name = "pow_test",
     size = "small",
     srcs = ["pow_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -1263,7 +1110,6 @@ tf_cc_test(
     name = "pack_test",
     size = "small",
     srcs = ["pack_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -1277,7 +1123,6 @@ tf_cc_test(
     name = "one_hot_test",
     size = "small",
     srcs = ["one_hot_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -1290,7 +1135,6 @@ tf_cc_test(
     name = "logical_test",
     size = "small",
     srcs = ["logical_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -1304,7 +1148,6 @@ tf_cc_test(
     name = "unpack_test",
     size = "small",
     srcs = ["unpack_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:builtin_op_data",
@@ -1318,7 +1161,6 @@ tf_cc_test(
     name = "floor_div_test",
     size = "small",
     srcs = ["floor_div_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:builtin_op_data",
@@ -1332,7 +1174,6 @@ tf_cc_test(
     name = "zeros_like_test",
     size = "small",
     srcs = ["zeros_like_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:builtin_op_data",
@@ -1346,7 +1187,6 @@ tf_cc_test(
     name = "floor_mod_test",
     size = "small",
     srcs = ["floor_mod_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:builtin_op_data",
@@ -1360,7 +1200,6 @@ tf_cc_test(
     name = "range_test",
     size = "small",
     srcs = ["range_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:builtin_op_data",
@@ -1374,7 +1213,6 @@ tf_cc_test(
     name = "squared_difference_test",
     size = "small",
     srcs = ["squared_difference_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
@@ -1387,7 +1225,6 @@ tf_cc_test(
     name = "fill_test",
     size = "small",
     srcs = ["fill_test.cc"],
-    tags = ["tflite_not_portable_ios"],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
diff --git a/tensorflow/lite/kernels/batch_to_space_nd_test.cc b/tensorflow/lite/kernels/batch_to_space_nd_test.cc
index a3e06d4c89..f330895599 100644
--- a/tensorflow/lite/kernels/batch_to_space_nd_test.cc
+++ b/tensorflow/lite/kernels/batch_to_space_nd_test.cc
@@ -114,6 +114,7 @@ TEST(BatchToSpaceNDOpTest, SimpleDynamicTest) {
                                                4, 8, 11, 15, 12, 16}));
 }
 
+#ifdef GTEST_HAS_DEATH_TEST
 TEST(BatchToSpaceNDOpTest, InvalidShapeTest) {
   EXPECT_DEATH(BatchToSpaceNDOpConstModel({3, 2, 2, 1}, {2, 2}, {0, 0, 0, 0}),
                "Cannot allocate tensors");
@@ -131,6 +132,7 @@ TEST(BatchToSpaceNDOpTest, InvalidCropsDynamicTest) {
   m.SetCrops({0, 0, -1, 0});
   EXPECT_DEATH(m.Invoke(), "crops.2. >= 0 was not true.");
 }
+#endif
 
 }  // namespace
 }  // namespace tflite
diff --git a/tensorflow/lite/kernels/pad_test.cc b/tensorflow/lite/kernels/pad_test.cc
index 415a285c70..3caa4065dc 100644
--- a/tensorflow/lite/kernels/pad_test.cc
+++ b/tensorflow/lite/kernels/pad_test.cc
@@ -175,6 +175,7 @@ class PadOpDynamicModel : public PadOpModel<float> {
   }
 };
 
+#ifdef GTEST_HAS_DEATH_TEST
 TEST(PadOpTest, TooManyDimensions) {
   EXPECT_DEATH(
       PadOpConstModel({TensorType_FLOAT32, {1, 2, 3, 4, 5, 6, 7, 8, 9}}, {9, 2},
@@ -195,6 +196,7 @@ TEST(PadOpTest, InvalidPadValue) {
                       {0, 0, 1, -1, 2, -1, 0, 0}, {TensorType_FLOAT32}),
       "Pad value has to be greater than equal to 0.");
 }
+#endif
 
 TEST(PadOpTest, SimpleConstTest) {
   // Padding is represented as four 2-D lists representing above padding and
@@ -306,6 +308,7 @@ class QuantizedPadOpTest : public ::testing::Test {
   }
 };
 
+#ifdef GTEST_HAS_DEATH_TEST
 TEST_F(QuantizedPadOpTest, ZeroNotInQuantizationRange) {
   // The test_util and actual quantization code currently ensure that the range
   // must include zero, but if that ever changes, this test will catch it.
@@ -314,6 +317,7 @@ TEST_F(QuantizedPadOpTest, ZeroNotInQuantizationRange) {
                                  {TensorType_UINT8, {}, 1.0, 2.0}),
                ".*Check failed: f_min <= 0.*");
 }
+#endif
 
 TEST_F(QuantizedPadOpTest, SimpleConstTest) {
   // Padding is represented as four 2-D lists representing above padding and
@@ -371,6 +375,7 @@ TEST_F(QuantizedPadOpTest, AdvancedDynamicTest) {
   EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 4, 7, 1}));
 }
 
+#ifdef GTEST_HAS_DEATH_TEST
 TEST(PadV2OpTest, TooManyDimensions) {
   EXPECT_DEATH(PadV2OpConstModel<float>(
                    {TensorType_FLOAT32, {1, 2, 3, 4, 5, 6, 7, 8, 9}}, {9, 2},
@@ -392,6 +397,7 @@ TEST(PadV2OpTest, InvalidPadValue) {
                                         {TensorType_FLOAT32}),
                "Pad value has to be greater than equal to 0.");
 }
+#endif
 
 TEST(PadV2OpTest, SimpleConstTest) {
   // Padding is represented as four 2-D lists representing above padding and
@@ -495,6 +501,7 @@ class QuantizedPadV2OpTest : public ::testing::Test {
   }
 };
 
+#ifdef GTEST_HAS_DEATH_TEST
 TEST_F(QuantizedPadV2OpTest, ZeroNotInQuantizationRange) {
   // The test_util and actual quantization code currently ensure that the range
   // must include zero, but if that ever changes, this test will catch it.
@@ -504,6 +511,7 @@ TEST_F(QuantizedPadV2OpTest, ZeroNotInQuantizationRange) {
                                  {TensorType_UINT8, {}, 1.0, 2.0}),
       ".*Check failed: f_min <= 0.*");
 }
+#endif
 
 TEST_F(QuantizedPadV2OpTest, SimpleConstTest) {
   // Padding is represented as four 2-D lists representing above padding and
diff --git a/tensorflow/lite/kernels/reshape_test.cc b/tensorflow/lite/kernels/reshape_test.cc
index 00bbbef57e..f98f3eb9ae 100644
--- a/tensorflow/lite/kernels/reshape_test.cc
+++ b/tensorflow/lite/kernels/reshape_test.cc
@@ -123,6 +123,7 @@ class ReshapeOpModel : public SingleOpModel {
   int output_;
 };
 
+#ifdef GTEST_HAS_DEATH_TEST
 TEST_P(ReshapeOpTest, MismatchedDimensions) {
   if (GetParam() == kAsTensor) {
     ReshapeOpModel<float> m({1, 2, 4, 1}, {2}, {2, 1}, GetParam());
@@ -133,12 +134,15 @@ TEST_P(ReshapeOpTest, MismatchedDimensions) {
                  "num_input_elements != num_output_elements");
   }
 }
+#endif
 
 TEST_P(ReshapeOpTest, TooManyDimensions) {
   if (GetParam() == kAsReshapeOption) {
+#ifdef GTEST_HAS_DEATH_TEST
     EXPECT_DEATH(ReshapeOpModel<float>({1, 1, 2, 1, 1, 1, 1, 1, 1}, {9},
                                        {1, 1, 1, 1, 1, 1, 1, 1, 2}, GetParam()),
                  "Found too many dimensions");
+#endif
   } else {
     ReshapeOpModel<float> m({1, 1, 2, 1, 1, 1, 1, 1, 1}, {9},
                             {1, 1, 1, 1, 1, 1, 1, 1, 2}, GetParam());
@@ -150,6 +154,7 @@ TEST_P(ReshapeOpTest, TooManyDimensions) {
   }
 }
 
+#ifdef GTEST_HAS_DEATH_TEST
 TEST_P(ReshapeOpTest, TooManySpecialDimensions) {
   if (GetParam() != kAsTensor) {
     EXPECT_DEATH(
@@ -160,6 +165,7 @@ TEST_P(ReshapeOpTest, TooManySpecialDimensions) {
     EXPECT_DEATH(m.Invoke(), "stretch_dim != -1");
   }
 }
+#endif
 
 // Create the model with a 2x2 shape. Processing still works because the new
 // shape ends up being hardcoded as a flat vector.
@@ -202,12 +208,16 @@ TEST_P(ReshapeOpTest, ScalarOutput) {
 // and output are scalars.
 TEST_P(ReshapeOpTest, LegacyScalarOutput) {
   if (GetParam() == kAsConstantTensor) {
+#ifdef GTEST_HAS_DEATH_TEST
     EXPECT_DEATH(ReshapeOpModel<float>({1}, {1}, {0}, GetParam()),
                  "num_input_elements != num_output_elements");
+#endif
   } else if (GetParam() == kAsTensor) {
+#ifdef GTEST_HAS_DEATH_TEST
     ReshapeOpModel<float> m({1}, {1}, {0}, GetParam());
     m.SetInput({3});
     EXPECT_DEATH(m.Invoke(), "num_input_elements != num_output_elements");
+#endif
   } else {
     ReshapeOpModel<float> m({1}, {1}, {0}, GetParam());
     m.SetInput({3});
diff --git a/tensorflow/lite/kernels/space_to_batch_nd_test.cc b/tensorflow/lite/kernels/space_to_batch_nd_test.cc
index 4d55ba56b7..c5d6e9a530 100644
--- a/tensorflow/lite/kernels/space_to_batch_nd_test.cc
+++ b/tensorflow/lite/kernels/space_to_batch_nd_test.cc
@@ -106,12 +106,14 @@ class SpaceToBatchNDOpDynamicModel : public SpaceToBatchNDOpModel {
   }
 };
 
+#ifdef GTEST_HAS_DEATH_TEST
 TEST(SpaceToBatchNDOpTest, InvalidShapeTest) {
   EXPECT_DEATH(
       SpaceToBatchNDOpConstModel({TensorType_FLOAT32, {1, 3, 3, 1}}, {2, 2},
                                  {0, 0, 0, 0}, {TensorType_FLOAT32}),
       "Cannot allocate tensors");
 }
+#endif
 
 TEST(SpaceToBatchNDOpTest, SimpleConstTest) {
   SpaceToBatchNDOpConstModel m({TensorType_FLOAT32, {1, 4, 4, 1}}, {2, 2},
@@ -220,6 +222,7 @@ class QuantizedSpaceToBatchNDOpTest : public ::testing::Test {
   }
 };
 
+#ifdef GTEST_HAS_DEATH_TEST
 TEST_F(QuantizedSpaceToBatchNDOpTest, ZeroNotInQuantizationRange) {
   // The test_util and actual quantization code currently ensure that the range
   // must include zero, but if that ever changes, this test will catch it.
@@ -228,6 +231,7 @@ TEST_F(QuantizedSpaceToBatchNDOpTest, ZeroNotInQuantizationRange) {
                    {0, 0, 1, 1, 1, 1, 0, 0}, {TensorType_UINT8, {}, 1.0, 2.0}),
                ".*Check failed: f_min <= 0.*");
 }
+#endif
 
 TEST_F(QuantizedSpaceToBatchNDOpTest, SimplePaddingConstTest) {
   SpaceToBatchNDOpConstModel m({TensorType_UINT8, {1, 5, 2, 1}, -1.0, 1.0},
diff --git a/tensorflow/lite/kernels/space_to_depth_test.cc b/tensorflow/lite/kernels/space_to_depth_test.cc
index 5744669b6d..3fa8d86348 100644
--- a/tensorflow/lite/kernels/space_to_depth_test.cc
+++ b/tensorflow/lite/kernels/space_to_depth_test.cc
@@ -50,10 +50,12 @@ class SpaceToDepthOpModel : public SingleOpModel {
   int output_;
 };
 
+#ifdef GTEST_HAS_DEATH_TEST
 TEST(SpaceToDepthOpModel, BadBlockSize) {
   EXPECT_DEATH(SpaceToDepthOpModel({TensorType_FLOAT32, {1, 2, 2, 1}}, 3),
                "Cannot allocate tensors");
 }
+#endif
 
 TEST(SpaceToDepthOpModel, Float32) {
   SpaceToDepthOpModel m({TensorType_FLOAT32, {1, 2, 2, 2}}, 2);
diff --git a/tensorflow/lite/kernels/strided_slice_test.cc b/tensorflow/lite/kernels/strided_slice_test.cc
index 122e01b99e..34875bf049 100644
--- a/tensorflow/lite/kernels/strided_slice_test.cc
+++ b/tensorflow/lite/kernels/strided_slice_test.cc
@@ -72,6 +72,7 @@ class StridedSliceOpModel : public SingleOpModel {
   int output_;
 };
 
+#ifdef GTEST_HAS_DEATH_TEST
 TEST(StridedSliceOpTest, UnsupportedInputSize) {
   EXPECT_DEATH(
       StridedSliceOpModel<>({2, 2, 2, 2, 2}, {5}, {5}, {5}, 0, 0, 0, 0, 0),
@@ -84,6 +85,7 @@ TEST(StridedSliceOpTest, UnssupportedArgs) {
   EXPECT_DEATH(StridedSliceOpModel<>({3, 2}, {2}, {2}, {2}, 0, 0, 0, 1, 0),
                "new_axis_mask is not implemented yet.");
 }
+#endif
 
 TEST(StridedSliceOpTest, In1D) {
   StridedSliceOpModel<> m({4}, {1}, {1}, {1}, 0, 0, 0, 0, 0);
diff --git a/tensorflow/lite/kernels/transpose_test.cc b/tensorflow/lite/kernels/transpose_test.cc
index 3ebaf3ca27..93df2c81db 100644
--- a/tensorflow/lite/kernels/transpose_test.cc
+++ b/tensorflow/lite/kernels/transpose_test.cc
@@ -184,6 +184,7 @@ class TransposeOpDynamicModel : public TransposeOpModel {
   }
 };
 
+#ifdef GTEST_HAS_DEATH_TEST
 TEST(TransposeTest, TestUnequalPermSize) {
   EXPECT_DEATH(TransposeOpConstModel({1, 3, 3, 1}, {2}, {2, 2}), "2 != 4");
 }
@@ -194,6 +195,7 @@ TEST(TransposeTest, TestPermOutOfBounds) {
   EXPECT_DEATH(TransposeOpConstModel({1, 3, 3, 1}, {4}, {0, 1, 2, 4}),
                "Transpose op permutations array is out of bounds.");
 }
+#endif
 
 TEST(TransposeTest, Test1DInputConstTensor) {
   TransposeOpConstModel m({3}, {1}, {0});
@@ -252,10 +254,12 @@ TEST(TransposeTest, Test3DInputDynamicTensor) {
                                 2, 6, 10, 14, 18, 22, 3, 7, 11, 15, 19, 23}));
 }
 
+#ifdef GTEST_HAS_DEATH_TEST
 TEST(TransposeTest, Test5DInputTensor) {
   EXPECT_DEATH(TransposeOpConstModel({1, 2, 3, 4, 5}, {5}, {0, 1, 2, 3, 4}),
                "Transpose op only supports 1D-4D input arrays.");
 }
+#endif
 
 TEST(TransposeTest, SimpleTestNoReorderConstTensor) {
   TransposeOpConstModel m({1, 2, 3, 1}, {4}, {0, 1, 2, 3});
-- 
GitLab


From aa36aa0a912a4addfb387e5fc0f759db225d03cc Mon Sep 17 00:00:00 2001
From: Davide Libenzi <dlibenzi@google.com>
Date: Thu, 3 Jan 2019 12:26:05 -0800
Subject: [PATCH 0179/1095] Allow XRT release operations to support vectors of
 keys.

PiperOrigin-RevId: 227730040
---
 .../compiler/xrt/kernels/xrt_compile_ops.cc   | 15 ++-
 .../compiler/xrt/kernels/xrt_state_ops.h      | 16 +--
 .../compiler/xrt/ops/xrt_compile_ops.cc       |  6 +-
 tensorflow/compiler/xrt/ops/xrt_state_ops.cc  |  5 +-
 tensorflow/compiler/xrt/tests/raw_api_test.cc | 98 ++++++++++++++++++-
 5 files changed, 117 insertions(+), 23 deletions(-)

diff --git a/tensorflow/compiler/xrt/kernels/xrt_compile_ops.cc b/tensorflow/compiler/xrt/kernels/xrt_compile_ops.cc
index 2ccdf0f02d..2ee1a6cd1a 100644
--- a/tensorflow/compiler/xrt/kernels/xrt_compile_ops.cc
+++ b/tensorflow/compiler/xrt/kernels/xrt_compile_ops.cc
@@ -215,11 +215,6 @@ XRTReleaseCompilationRefOp::~XRTReleaseCompilationRefOp() = default;
 void XRTReleaseCompilationRefOp::Compute(OpKernelContext* ctx) {
   VLOG(1) << "XRTReleaseCompilationRefOp::Compute";
 
-  const Tensor& key_tensor = ctx->input(0);
-  OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(key_tensor.shape()),
-              errors::Internal("computation key should be a string scalar"));
-  int64 uid = key_tensor.scalar<int64>()();
-
   ResourceMgr* rm;
   OP_REQUIRES_OK(ctx, XRTGenericDeviceAccessor::GetResourceManager(ctx, &rm));
 
@@ -230,9 +225,13 @@ void XRTReleaseCompilationRefOp::Compute(OpKernelContext* ctx) {
                           kXRTCompilationCacheResourceName, &cache));
   core::ScopedUnref cache_unref(cache);
 
-  OP_REQUIRES_OK(ctx, cache->Release(uid));
-
-  VLOG(2) << "Released computation handle " << uid;
+  const Tensor& keys_tensor = ctx->input(0);
+  auto flat_keys = keys_tensor.flat<int64>();
+  for (int64 i = 0; i < flat_keys.size(); ++i) {
+    int64 key = flat_keys(i);
+    OP_REQUIRES_OK(ctx, cache->Release(key));
+    VLOG(2) << "Released computation handle " << key;
+  }
 }
 
 }  // namespace
diff --git a/tensorflow/compiler/xrt/kernels/xrt_state_ops.h b/tensorflow/compiler/xrt/kernels/xrt_state_ops.h
index 2e2f3ff116..c8def16bbc 100644
--- a/tensorflow/compiler/xrt/kernels/xrt_state_ops.h
+++ b/tensorflow/compiler/xrt/kernels/xrt_state_ops.h
@@ -453,17 +453,17 @@ class XRTReleaseAllocationOp : public OpKernel {
   void Compute(OpKernelContext* ctx) override {
     VLOG(1) << "XRTReleaseAllocationOp::Compute";
 
-    const Tensor& allocation_handle = ctx->input(0);
-    OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(allocation_handle.shape()),
-                errors::Internal("handle input should be an int64 scalar"));
-    int64 key = allocation_handle.scalar<int64>()();
-
     ResourceMgr* rm;
     OP_REQUIRES_OK(ctx, DeviceAccessor::GetResourceManager(ctx, &rm));
 
-    OP_REQUIRES_OK(ctx, XRTTupleAllocation::DeleteFromResourceManager(rm, key));
-
-    VLOG(2) << "Released allocation handle " << key;
+    const Tensor& allocation_handle = ctx->input(0);
+    auto flat_keys = allocation_handle.flat<int64>();
+    for (int64 i = 0; i < flat_keys.size(); ++i) {
+      int64 key = flat_keys(i);
+      OP_REQUIRES_OK(ctx,
+                     XRTTupleAllocation::DeleteFromResourceManager(rm, key));
+      VLOG(2) << "Released allocation handle " << key;
+    }
   }
 };
 
diff --git a/tensorflow/compiler/xrt/ops/xrt_compile_ops.cc b/tensorflow/compiler/xrt/ops/xrt_compile_ops.cc
index 7b3b50c695..9dd964e546 100644
--- a/tensorflow/compiler/xrt/ops/xrt_compile_ops.cc
+++ b/tensorflow/compiler/xrt/ops/xrt_compile_ops.cc
@@ -44,10 +44,10 @@ REGISTER_OP("XRTReleaseCompilationHandle")
     .SetShapeFn(tensorflow::shape_inference::NoOutputs)
     .Doc(
         R"(
-Discards a computation from the compilation cache. The handle cannot be
-subsequently used.
+Discards one or more computation handles from the compilation cache.
+The handle(s) cannot be subsequently used.
 
-'handle' is an id returned from a XRTCompile Op.
+'handle' is an ID (or vector of IDs) returned from a XRTCompile Op.
 )");
 
 }  // namespace tensorflow
diff --git a/tensorflow/compiler/xrt/ops/xrt_state_ops.cc b/tensorflow/compiler/xrt/ops/xrt_state_ops.cc
index fe6bee0dac..db58f0797d 100644
--- a/tensorflow/compiler/xrt/ops/xrt_state_ops.cc
+++ b/tensorflow/compiler/xrt/ops/xrt_state_ops.cc
@@ -127,10 +127,11 @@ REGISTER_OP("XRTReleaseAllocationHandle")
     .SetShapeFn(tensorflow::shape_inference::NoOutputs)
     .Doc(
         R"(
-Discards an allocation from device memory. The handle cannot be subsequently
+Discards one or more device memory handles. The handle(s) cannot be subsequently
 used.
 
-'handle' is the id returned from the Op that produced the on-device allocation.
+'handle' is the ID (or a vector of IDs) returned from the Op that produced the
+on-device allocation.
 )");
 
 REGISTER_OP("XRTReleaseAllAllocations")
diff --git a/tensorflow/compiler/xrt/tests/raw_api_test.cc b/tensorflow/compiler/xrt/tests/raw_api_test.cc
index 5f8121703e..c8479cb778 100644
--- a/tensorflow/compiler/xrt/tests/raw_api_test.cc
+++ b/tensorflow/compiler/xrt/tests/raw_api_test.cc
@@ -258,8 +258,102 @@ TEST(RawApiTest, AllocAndRewrite) {
   EXPECT_TRUE(new_response.ParseFromString(outputs[0].scalar<string>()()));
   EXPECT_TRUE(CompareLiteralProtos(new_literal, new_response));
 
-  auto release =
-      ops::XRTReleaseAllocationHandle(root, Input(allocation_handle));
+  Tensor release_tensor(DT_INT64, TensorShape({1}));
+  release_tensor.flat<int64>()(0) = allocation_handle;
+
+  auto release = ops::XRTReleaseAllocationHandle(root, release_tensor);
+  TF_EXPECT_OK(session.Run(tensorflow::ClientSession::FeedType(), {}, {release},
+                           &outputs));
+}
+
+TEST(RawApiTest, AllocReleaseMany) {
+  xrt::XLAAllocation alloc1;
+  *alloc1.mutable_value() =
+      xla::LiteralUtil::CreateR2({{4, 5}, {6, 7}}).ToProto();
+  xrt::XLAAllocation alloc2;
+  *alloc2.mutable_value() =
+      xla::LiteralUtil::CreateR2({{6, 7}, {4, 5}}).ToProto();
+
+  Scope root = Scope::NewRootScope().WithDevice(DeviceFromFlag());
+  auto value1 =
+      ops::Const(root.WithDevice("/device:CPU:0"), alloc1.SerializeAsString());
+  auto value2 =
+      ops::Const(root.WithDevice("/device:CPU:0"), alloc2.SerializeAsString());
+  auto handle1 = ops::XRTAllocate(root, value1);
+  auto handle2 = ops::XRTAllocate(root, value2);
+  TF_ASSERT_OK(root.status());
+
+  tensorflow::ClientSession session(root);
+  std::vector<tensorflow::Tensor> outputs;
+  TF_EXPECT_OK(session.Run({handle1, handle2}, &outputs));
+  EXPECT_EQ(outputs.size(), 2);
+
+  int64 allocation_handle1 = outputs[0].scalar<int64>()();
+  int64 allocation_handle2 = outputs[1].scalar<int64>()();
+
+  Tensor release_tensor(DT_INT64, TensorShape({2}));
+  release_tensor.flat<int64>()(0) = allocation_handle1;
+  release_tensor.flat<int64>()(1) = allocation_handle2;
+
+  auto release = ops::XRTReleaseAllocationHandle(root, release_tensor);
+  outputs.clear();
+  TF_EXPECT_OK(session.Run(tensorflow::ClientSession::FeedType(), {}, {release},
+                           &outputs));
+}
+
+TEST(RawApiTest, CompileAndReleaseMany) {
+  xrt::XLAComputation c1;
+  auto config1 = c1.mutable_config();
+  auto shapes1 = config1->mutable_program_shape();
+  *shapes1->add_parameters() =
+      xla::ShapeUtil::MakeShape(xla::F32, {2}).ToProto();
+  *shapes1->add_parameters() =
+      xla::ShapeUtil::MakeShape(xla::F32, {2}).ToProto();
+  *shapes1->mutable_result() =
+      xla::ShapeUtil::MakeShape(xla::F32, {2}).ToProto();
+  StoreComputationSnapshot(AddAndScale(), c1.mutable_hlo_snapshot());
+
+  xrt::XLAComputation c2;
+  auto config2 = c2.mutable_config();
+  auto shapes2 = config2->mutable_program_shape();
+  *shapes2->add_parameters() =
+      xla::ShapeUtil::MakeShape(xla::F32, {2}).ToProto();
+  *shapes2->add_parameters() =
+      xla::ShapeUtil::MakeShape(xla::F32, {2}).ToProto();
+  *shapes2->mutable_result() =
+      xla::ShapeUtil::MakeTupleShape({xla::ShapeUtil::MakeShape(xla::F32, {2})})
+          .ToProto();
+  StoreComputationSnapshot(AddAndTuple(), c2.mutable_hlo_snapshot());
+
+  xrt::XRTExecutionConfig e;
+  e.set_release_input_handles(true);
+  e.set_release_compilation_handle(false);
+
+  Scope root = Scope::NewRootScope().WithDevice(DeviceFromFlag());
+  auto e_config =
+      ops::Const(root.WithDevice("/device:CPU:0"), e.SerializeAsString());
+  auto computation1 =
+      ops::Const(root.WithDevice("/device:CPU:0"), c1.SerializeAsString());
+  auto c_handle1 = ops::XRTCompile(root, computation1);
+  auto computation2 =
+      ops::Const(root.WithDevice("/device:CPU:0"), c2.SerializeAsString());
+  auto c_handle2 = ops::XRTCompile(root, computation2);
+  TF_ASSERT_OK(root.status());
+
+  ClientSession session(root);
+  std::vector<Tensor> outputs;
+  TF_EXPECT_OK(session.Run({c_handle1.handle, c_handle2.handle}, &outputs));
+  EXPECT_EQ(outputs.size(), 2);
+
+  int64 compilation_handle1 = outputs[0].scalar<int64>()();
+  int64 compilation_handle2 = outputs[1].scalar<int64>()();
+
+  Tensor release_tensor(DT_INT64, TensorShape({2}));
+  release_tensor.flat<int64>()(0) = compilation_handle1;
+  release_tensor.flat<int64>()(1) = compilation_handle2;
+
+  auto release = ops::XRTReleaseCompilationHandle(root, release_tensor);
+  outputs.clear();
   TF_EXPECT_OK(session.Run(tensorflow::ClientSession::FeedType(), {}, {release},
                            &outputs));
 }
-- 
GitLab


From cb4f2d41f22b6df49c80555a911d7dc2f08b7b90 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 3 Jan 2019 12:27:49 -0800
Subject: [PATCH 0180/1095] Fix autograph detection of tensors that are wrapped
 in python structures.

A dict with tensor values did not register as containing a Tensor, causing the python branch to be taken instead of the tf branch.

PiperOrigin-RevId: 227730234
---
 tensorflow/python/autograph/operators/BUILD   |  1 +
 .../autograph/operators/control_flow.py       |  4 ++-
 .../autograph/operators/control_flow_test.py  | 30 +++++++++++++------
 3 files changed, 25 insertions(+), 10 deletions(-)

diff --git a/tensorflow/python/autograph/operators/BUILD b/tensorflow/python/autograph/operators/BUILD
index aedb901845..21a66c86b7 100644
--- a/tensorflow/python/autograph/operators/BUILD
+++ b/tensorflow/python/autograph/operators/BUILD
@@ -38,6 +38,7 @@ py_library(
         "//tensorflow/python:list_ops",
         "//tensorflow/python:tensor_array_ops",
         "//tensorflow/python:tensor_util",
+        "//tensorflow/python:util",
         "//tensorflow/python:variables",
         "//tensorflow/python/autograph/utils",
         "//tensorflow/python/data/ops:dataset_ops",
diff --git a/tensorflow/python/autograph/operators/control_flow.py b/tensorflow/python/autograph/operators/control_flow.py
index afa3787d42..035ea1bd92 100644
--- a/tensorflow/python/autograph/operators/control_flow.py
+++ b/tensorflow/python/autograph/operators/control_flow.py
@@ -23,6 +23,7 @@ from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import gen_math_ops
+from tensorflow.python.util import nest
 
 
 def for_stmt(iter_, extra_test, body, init_state):
@@ -160,7 +161,8 @@ def while_stmt(test, body, init_state, extra_deps, opts=None):
   # TODO(mdan): Consider adding a generic mechanism for dynamic dispatch.
   # That could be something as simple as a collection of dispatch rules, with
   # some prioritization.
-  if any(tensor_util.is_tensor(v) for v in init_state + extra_deps):
+  if any(tensor_util.is_tensor(v)
+         for v in nest.flatten(init_state + extra_deps)):
     return _tf_while_stmt(test, body, init_state, opts)
   else:
     return _py_while_stmt(test, body, init_state, opts)
diff --git a/tensorflow/python/autograph/operators/control_flow_test.py b/tensorflow/python/autograph/operators/control_flow_test.py
index 0a7d4b6402..f9e006f7ad 100644
--- a/tensorflow/python/autograph/operators/control_flow_test.py
+++ b/tensorflow/python/autograph/operators/control_flow_test.py
@@ -36,7 +36,7 @@ class ForLoopTest(test.TestCase):
         extra_test=lambda s: True,
         body=lambda i, s: (s + i,),
         init_state=(0,))
-    with self.cached_session() as sess:
+    with self.cached_session():
       self.assertEqual((10,), self.evaluate(s))
 
   def test_python(self):
@@ -55,7 +55,7 @@ class ForLoopTest(test.TestCase):
         extra_test=lambda s: True,
         body=lambda i, s: (s + i,),
         init_state=(0,))
-    with self.cached_session() as sess:
+    with self.cached_session():
       self.assertEqual((10,), self.evaluate(s))
 
 
@@ -65,18 +65,30 @@ class WhileLoopTest(test.TestCase):
   def test_tensor(self):
     n = constant_op.constant(5)
     results = control_flow.while_stmt(
-        test=lambda i, s: i < n,
-        body=lambda i, s: (i + 1, s + i,),
+        test=lambda i, sum: i < n,
+        body=lambda i, sum: (i + 1, sum + i,),
         init_state=(0, 0),
         extra_deps=(n,))
-    with self.cached_session() as sess:
+    with self.cached_session():
       self.assertEqual((5, 10), self.evaluate(results))
 
+  @test_util.run_deprecated_v1
+  def test_tensor_dict_state(self):
+    n = 5
+    init_state = {'i': constant_op.constant(0), 'sum': constant_op.constant(0)}
+    results = control_flow.while_stmt(
+        test=lambda s: s['i'] < n,
+        body=lambda s: ({'i': s['i'] + 1, 'sum': s['sum'] + s['i']},),
+        init_state=(init_state,),
+        extra_deps=())
+    with self.cached_session():
+      self.assertEqual(({'i': 5, 'sum': 10},), self.evaluate(results))
+
   def test_python(self):
     n = 5
     results = control_flow.while_stmt(
-        test=lambda i, s: i < n,
-        body=lambda i, s: (i + 1, s + i),
+        test=lambda i, sum: i < n,
+        body=lambda i, sum: (i + 1, sum + i),
         init_state=(0, 0),
         extra_deps=(n,))
     self.assertEqual((5, 10), results)
@@ -93,7 +105,7 @@ class IfStmtTest(test.TestCase):
 
   @test_util.run_deprecated_v1
   def test_tensor(self):
-    with self.cached_session() as sess:
+    with self.cached_session():
       t = self.single_return_if_stmt(constant_op.constant(True))
       self.assertEqual(1, self.evaluate(t))
       t = self.single_return_if_stmt(constant_op.constant(False))
@@ -105,7 +117,7 @@ class IfStmtTest(test.TestCase):
 
   @test_util.run_deprecated_v1
   def test_tensor_multiple_returns(self):
-    with self.cached_session() as sess:
+    with self.cached_session():
       t = self.multi_return_if_stmt(constant_op.constant(True))
       self.assertAllEqual([1, 2], self.evaluate(t))
       t = self.multi_return_if_stmt(constant_op.constant(False))
-- 
GitLab


From 7e2d53c1c371f38c7f0ef13c1c06336b22a195c0 Mon Sep 17 00:00:00 2001
From: Shivani Agrawal <shivaniagrawal@google.com>
Date: Thu, 3 Jan 2019 12:46:47 -0800
Subject: [PATCH 0181/1095] [tf.data] Adds the expected check for better
 debugging.

PiperOrigin-RevId: 227733165
---
 tensorflow/core/kernels/data/experimental/scan_dataset_op.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/core/kernels/data/experimental/scan_dataset_op.cc b/tensorflow/core/kernels/data/experimental/scan_dataset_op.cc
index 76ab33fe98..87263b5606 100644
--- a/tensorflow/core/kernels/data/experimental/scan_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/scan_dataset_op.cc
@@ -186,6 +186,8 @@ class ScanDatasetOp : public UnaryDatasetOpKernel {
 
         Status s = instantiated_captured_func_->Run(ctx, std::move(args),
                                                     &state_and_output);
+        DCHECK(state_and_output.size() <=
+               dataset()->state_types_.size() + output_dtypes().size());
         if (s.ok()) {
           state_.clear();
           size_t i = 0;
-- 
GitLab


From f9d4786119cda668d9b05d4ae49f2a4fe8d5738f Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Thu, 3 Jan 2019 13:01:24 -0800
Subject: [PATCH 0182/1095] Make tests all have their own main so
 initialization works.

PiperOrigin-RevId: 227735303
---
 tensorflow/lite/toco/BUILD                     | 12 ++++++++----
 tensorflow/lite/toco/import_tensorflow_test.cc |  7 +++++++
 tensorflow/lite/toco/toco_convert_test.cc      |  7 +++++++
 tensorflow/lite/toco/toco_port_test.cc         |  7 +++++++
 tensorflow/lite/toco/tooling_util_test.cc      |  9 ++++++++-
 5 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/tensorflow/lite/toco/BUILD b/tensorflow/lite/toco/BUILD
index 93d41fcae1..d77234c80a 100644
--- a/tensorflow/lite/toco/BUILD
+++ b/tensorflow/lite/toco/BUILD
@@ -348,7 +348,8 @@ tf_cc_test(
         "//tensorflow/core:lib",
         "//tensorflow/core:ops",
         "//tensorflow/core:protos_all_cc",
-        "@com_google_googletest//:gtest_main",
+        "//tensorflow/lite/testing:util",
+        "@com_google_googletest//:gtest",
     ],
 )
 
@@ -387,7 +388,8 @@ tf_cc_test(
         ":model",
         ":tooling_util",
         "//tensorflow/core:lib",
-        "@com_google_googletest//:gtest_main",
+        "//tensorflow/lite/testing:util",
+        "@com_google_googletest//:gtest",
     ],
 )
 
@@ -451,12 +453,13 @@ tf_cc_test(
         ":toco_port",
         ":toco_tooling",
         ":types_proto_cc",
-        "@com_google_googletest//:gtest_main",
+        "@com_google_googletest//:gtest",
         "@com_google_absl//absl/strings",
         "//tensorflow/core:lib",
         # We cannot embed the core:ops dependency directly into :toco_tooling as
         # it can conflict with downstream deps when toco is used as a library.
         "//tensorflow/core:ops",
+        "//tensorflow/lite/testing:util",
     ],
 )
 
@@ -468,6 +471,7 @@ tf_cc_test(
     ],
     deps = [
         ":toco_port",
-        "@com_google_googletest//:gtest_main",
+        "//tensorflow/lite/testing:util",
+        "@com_google_googletest//:gtest",
     ],
 )
diff --git a/tensorflow/lite/toco/import_tensorflow_test.cc b/tensorflow/lite/toco/import_tensorflow_test.cc
index 260704fd2a..70382f546b 100644
--- a/tensorflow/lite/toco/import_tensorflow_test.cc
+++ b/tensorflow/lite/toco/import_tensorflow_test.cc
@@ -23,6 +23,7 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor.pb.h"
 #include "tensorflow/core/framework/tensor_shape.pb.h"
 #include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/lite/testing/util.h"
 
 namespace toco {
 
@@ -564,3 +565,9 @@ TEST(ImportTest, UnsupportedOpWithMultipleOutputs) {
 
 }  // namespace
 }  // namespace toco
+
+int main(int argc, char** argv) {
+  ::tflite::LogToStderr();
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/tensorflow/lite/toco/toco_convert_test.cc b/tensorflow/lite/toco/toco_convert_test.cc
index c3c440db94..3730c53ae1 100644
--- a/tensorflow/lite/toco/toco_convert_test.cc
+++ b/tensorflow/lite/toco/toco_convert_test.cc
@@ -15,6 +15,7 @@ limitations under the License.
 #include "tensorflow/lite/toco/toco_convert.h"
 #include <gmock/gmock.h>
 #include <gtest/gtest.h>
+#include "tensorflow/lite/testing/util.h"
 
 namespace toco {
 namespace {
@@ -171,3 +172,9 @@ TEST(TocoTest, TransientStringTensors) {
 
 }  // namespace
 }  // namespace toco
+
+int main(int argc, char** argv) {
+  ::tflite::LogToStderr();
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/tensorflow/lite/toco/toco_port_test.cc b/tensorflow/lite/toco/toco_port_test.cc
index f5fbb4caeb..d80d423ed7 100644
--- a/tensorflow/lite/toco/toco_port_test.cc
+++ b/tensorflow/lite/toco/toco_port_test.cc
@@ -13,6 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 #include "tensorflow/lite/toco/toco_port.h"
+#include "tensorflow/lite/testing/util.h"
 #include "tensorflow/lite/toco/toco_types.h"
 
 #include <gmock/gmock.h>
@@ -56,3 +57,9 @@ TEST(TocoPortTest, JoinPath) {
 }  // namespace
 }  // namespace port
 }  // namespace toco
+
+int main(int argc, char** argv) {
+  ::tflite::LogToStderr();
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/tensorflow/lite/toco/tooling_util_test.cc b/tensorflow/lite/toco/tooling_util_test.cc
index 6f1c9c563a..e44b94b712 100644
--- a/tensorflow/lite/toco/tooling_util_test.cc
+++ b/tensorflow/lite/toco/tooling_util_test.cc
@@ -16,9 +16,10 @@ limitations under the License.
 #include <vector>
 
 #include <gtest/gtest.h>
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/lite/testing/util.h"
 #include "tensorflow/lite/toco/model.h"
 #include "tensorflow/lite/toco/tooling_util.h"
-#include "tensorflow/core/lib/core/status.h"
 
 namespace toco {
 
@@ -203,3 +204,9 @@ TEST(FusedActivationTest, DefaultsToUnfused) {
 }
 
 }  // namespace toco
+
+int main(int argc, char** argv) {
+  ::tflite::LogToStderr();
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
-- 
GitLab


From 689db4ac81b13279677ceefd84700de3d0ec9925 Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Thu, 3 Jan 2019 13:29:58 -0800
Subject: [PATCH 0183/1095] Replace SimpleGraphView with GraphTopologyView.

+ added separate enter/advance predicates for DFS traversal

PiperOrigin-RevId: 227740260
---
 tensorflow/core/grappler/optimizers/BUILD     |   7 +
 .../optimizers/arithmetic_optimizer.cc        |  47 ++---
 .../grappler/optimizers/loop_optimizer.cc     |  40 +++--
 .../grappler/optimizers/memory_optimizer.cc   |  67 ++++---
 tensorflow/core/grappler/utils.cc             | 169 ------------------
 tensorflow/core/grappler/utils.h              |  73 --------
 tensorflow/core/grappler/utils/traversal.cc   |  19 +-
 tensorflow/core/grappler/utils/traversal.h    |  39 +++-
 .../core/grappler/utils/traversal_test.cc     |  46 ++++-
 9 files changed, 184 insertions(+), 323 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index 512c3b07b4..e2b9de7cc5 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -254,12 +254,16 @@ cc_library(
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core/grappler:graph_topology_view",
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/costs:graph_properties",
         "//tensorflow/core/grappler/utils:symbolic_shapes",
         "//tensorflow/core/grappler/utils:topological_sort",
+        "//tensorflow/core/grappler/utils:traversal",
+        "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/container:flat_hash_set",
     ],
 )
 
@@ -411,6 +415,7 @@ cc_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core/grappler:graph_topology_view",
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:op_types",
@@ -613,11 +618,13 @@ cc_library(
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core/grappler:graph_topology_view",
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler/costs:graph_properties",
         "//tensorflow/core/grappler/utils:frame",
+        "//tensorflow/core/grappler/utils:traversal",
         "@com_google_absl//absl/container:flat_hash_set",
     ],
 )
diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
index 94a87c3ff2..e85e0473b3 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
@@ -22,6 +22,8 @@ limitations under the License.
 #include <unordered_set>
 #include <vector>
 
+#include "absl/container/flat_hash_map.h"
+#include "absl/container/flat_hash_set.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/attr_value_util.h"
 #include "tensorflow/core/framework/node_def.pb.h"
@@ -31,6 +33,7 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor_shape.pb.h"
 #include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/grappler/costs/graph_properties.h"
+#include "tensorflow/core/grappler/graph_topology_view.h"
 #include "tensorflow/core/grappler/grappler_item.h"
 #include "tensorflow/core/grappler/op_types.h"
 #include "tensorflow/core/grappler/optimizers/constant_folding.h"
@@ -38,6 +41,7 @@ limitations under the License.
 #include "tensorflow/core/grappler/utils.h"
 #include "tensorflow/core/grappler/utils/symbolic_shapes.h"
 #include "tensorflow/core/grappler/utils/topological_sort.h"
+#include "tensorflow/core/grappler/utils/traversal.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/stringpiece.h"
 #include "tensorflow/core/lib/hash/hash.h"
@@ -3334,36 +3338,37 @@ bool ArithmeticOptimizer::CanDedup(const NodeDef& node) const {
 }
 
 void ArithmeticOptimizer::DedupComputations() {
-  bool stop = true;
-  SimpleGraphView graph_view;
-  if (!graph_view.Initialize(*optimized_graph_).ok()) {
-    LOG(WARNING) << "Failed to build SimpleGraphView.";
+  GraphTopologyView graph_view;
+  if (!graph_view.InitializeFromGraph(*optimized_graph_).ok()) {
+    LOG(WARNING) << "Failed to initialize GraphTopologyView.";
     return;
   }
-  std::set<int> duplicates;
+
+  const absl::flat_hash_set<string> ops_to_traverse = {
+      "Identity", "IdentityN", "Reshape", "ExpandDims",
+      "Enter",    "Switch",    "Merge"};
+
   // Populate feed_inplace_op;
-  std::unordered_set<const NodeDef*> feeds_inplace_op;
-  for (int i = 0; i < optimized_graph_->node_size(); ++i) {
-    const NodeDef& root = optimized_graph_->node(i);
-    if (feeds_inplace_op.find(&root) != feeds_inplace_op.end()) {
-      continue;
-    }
+  absl::flat_hash_set<const NodeDef*> feeds_inplace_op;
+
+  for (const NodeDef& root : optimized_graph_->node()) {
+    if (feeds_inplace_op.find(&root) != feeds_inplace_op.end()) continue;
 
     if (ModifiesInputsInPlace(root)) {
-      const std::unordered_set<string> op_types_to_traverse = {
-          root.op(),    "Identity", "IdentityN", "Reshape",
-          "ExpandDims", "Enter",    "Switch",    "Merge"};
+      const auto is_continue_traversal = [&](const NodeDef* node) -> bool {
+        return node->op() == root.op() || ops_to_traverse.count(node->op()) > 0;
+      };
 
-      graph_view.DepthFirstSearchWithCallback(
-          op_types_to_traverse, i,
-          [&](const NodeDef& node) {
-            feeds_inplace_op.insert(&node);
-            return false;
-          },
-          SimpleGraphView::kFollowInputs /* search through inputs */);
+      DfsTraversal(graph_view, {&root}, TraversalDirection::kFollowInputs,
+                   DfsPredicates::Advance(is_continue_traversal),
+                   DfsCallbacks::PreOrder([&](const NodeDef* node) {
+                     feeds_inplace_op.insert(node);
+                   }));
     }
   }
 
+  bool stop = true;
+  std::set<int> duplicates;
   do {
     stop = true;
     UniqueNodes nodes;
diff --git a/tensorflow/core/grappler/optimizers/loop_optimizer.cc b/tensorflow/core/grappler/optimizers/loop_optimizer.cc
index 3606473840..cf5e4db29f 100644
--- a/tensorflow/core/grappler/optimizers/loop_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/loop_optimizer.cc
@@ -30,12 +30,14 @@ limitations under the License.
 #include "tensorflow/core/framework/op.h"
 #include "tensorflow/core/framework/tensor.pb.h"
 #include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/grappler/graph_topology_view.h"
 #include "tensorflow/core/grappler/grappler_item.h"
 #include "tensorflow/core/grappler/mutable_graph_view.h"
 #include "tensorflow/core/grappler/op_types.h"
 #include "tensorflow/core/grappler/optimizers/constant_folding.h"
 #include "tensorflow/core/grappler/optimizers/evaluation_utils.h"
 #include "tensorflow/core/grappler/utils/frame.h"
+#include "tensorflow/core/grappler/utils/traversal.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/stringpiece.h"
 #include "tensorflow/core/lib/gtl/inlined_vector.h"
@@ -451,16 +453,29 @@ Status LoopInvariantNodeMotionOptimizer::Optimize() {
 }
 
 std::vector<int> GetStackPushNodesToConvert(
-    const SimpleGraphView& graph_view,
+    const GraphTopologyView& graph_view,
     const std::unordered_set<string>& nodes_to_preserve, int stack_node_idx) {
   VLOG(1) << "Stack node: " << graph_view.graph()->node(stack_node_idx).name();
+
   const std::unordered_set<string> op_types_to_traverse(
       {"Stack", "StackV2", "Enter", "RefEnter", "Switch", "RefSwitch",
        "Identity", "RefIdentity"});
+  const auto is_op_to_traverse = [&](const NodeDef* node) -> bool {
+    return op_types_to_traverse.find(node->op()) != op_types_to_traverse.end();
+  };
+
   std::vector<int> nodes_to_convert;
-  std::set<int> fanout;
-  graph_view.DepthFirstSearch(op_types_to_traverse, stack_node_idx, &fanout);
-  for (int fanout_idx : fanout) {
+  std::vector<int> fanouts;
+
+  DfsTraversal(graph_view, {graph_view.GetNode(stack_node_idx)},
+               TraversalDirection::kFollowOutputs,
+               DfsPredicates::Advance(is_op_to_traverse),
+               DfsCallbacks::PreOrder([&](const NodeDef* node) {
+                 const absl::optional<int> idx = graph_view.GetNodeIndex(*node);
+                 fanouts.push_back(idx.value());
+               }));
+
+  for (int fanout_idx : fanouts) {
     const NodeDef& fanout_node = graph_view.graph()->node(fanout_idx);
     VLOG(1) << "Fanout " << fanout_idx << " : " << fanout_node.name();
     if (IsStackPushOp(fanout_node)) {
@@ -468,13 +483,12 @@ std::vector<int> GetStackPushNodesToConvert(
       // happen when the graph we have contains only the forward pass for a loop
       // (as when the forward and backward passes are split across different
       // functions).
-      if (graph_view.has_node(fanout_node.input(0))) {
-        const NodeDef* stack_node =
-            &graph_view.node(graph_view.index(fanout_node.input(0)));
+      if (graph_view.HasNode(fanout_node.input(0))) {
+        const NodeDef* stack_node = graph_view.GetNode(fanout_node.input(0));
         while (stack_node->op() != "Stack" && stack_node->op() != "StackV2" &&
                stack_node->input_size() > 0 &&
-               graph_view.has_node(stack_node->input(0))) {
-          stack_node = &graph_view.node(graph_view.index(stack_node->input(0)));
+               graph_view.HasNode(stack_node->input(0))) {
+          stack_node = graph_view.GetNode(stack_node->input(0));
         }
         if (nodes_to_preserve.find(stack_node->name()) ==
             nodes_to_preserve.end()) {
@@ -488,7 +502,7 @@ std::vector<int> GetStackPushNodesToConvert(
                    op_types_to_traverse.end()) {
       continue;
     } else if (!IsStackPopOp(fanout_node) ||
-               (!graph_view.outputs(fanout_idx).empty() ||
+               (!graph_view.GetFanout(fanout_idx).empty() ||
                 nodes_to_preserve.find(fanout_node.name()) !=
                     nodes_to_preserve.end())) {
       // The node is either a stack pop with consumers or something unexpected
@@ -497,14 +511,16 @@ std::vector<int> GetStackPushNodesToConvert(
       break;
     }
   }
+
   return nodes_to_convert;
 }
 
 Status RemoveStackOps(const std::unordered_set<string>& nodes_to_preserve,
                       GraphDef* optimized_graph) {
   NodeMap node_map(optimized_graph);
-  SimpleGraphView graph_view;
-  TF_RETURN_IF_ERROR(graph_view.Initialize(*optimized_graph));
+  GraphTopologyView graph_view;
+  TF_RETURN_IF_ERROR(graph_view.InitializeFromGraph(*optimized_graph));
+
   for (int node_idx = 0; node_idx < optimized_graph->node_size(); ++node_idx) {
     if (IsStackOp(optimized_graph->node(node_idx))) {
       for (int push_node_idx : GetStackPushNodesToConvert(
diff --git a/tensorflow/core/grappler/optimizers/memory_optimizer.cc b/tensorflow/core/grappler/optimizers/memory_optimizer.cc
index 6e33645537..b50d50f842 100644
--- a/tensorflow/core/grappler/optimizers/memory_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/memory_optimizer.cc
@@ -30,6 +30,7 @@ limitations under the License.
 #include "tensorflow/core/grappler/costs/graph_memory.h"
 #include "tensorflow/core/grappler/costs/graph_properties.h"
 #include "tensorflow/core/grappler/costs/utils.h"
+#include "tensorflow/core/grappler/graph_topology_view.h"
 #include "tensorflow/core/grappler/grappler_item.h"
 #include "tensorflow/core/grappler/mutable_graph_view.h"
 #include "tensorflow/core/grappler/op_types.h"
@@ -188,13 +189,14 @@ std::vector<RecomputedSubGraph> GetOpGroupsToRecompute(
       }
     }
     // Recompute only nodes which eventually feed into a target node.
-    connected_subgraph(node_map,
-                       true,   // Collect inputs
-                       false,  // Collect outputs
-                       [&unpruned_recompute_nodes](const NodeDef& node) {
-                         return unpruned_recompute_nodes.count(&node) != 0;
-                       },
-                       &current_recomputation.recomputed_source_nodes);
+    connected_subgraph(
+        node_map,
+        true,   // Collect inputs
+        false,  // Collect outputs
+        [&unpruned_recompute_nodes](const NodeDef& node) {
+          return unpruned_recompute_nodes.count(&node) != 0;
+        },
+        &current_recomputation.recomputed_source_nodes);
     if (current_recomputation.target_nodes.empty()) {
       continue;
     }
@@ -1268,46 +1270,55 @@ Status RelaxAllocatorConstraints(GraphDef* optimized_graph) {
     return Status::OK();
   }
 
-  std::unordered_set<int> optimized_nodes;
-  SimpleGraphView graph_view;
-  TF_RETURN_IF_ERROR(graph_view.Initialize(*optimized_graph));
+  GraphTopologyView graph_view;
+  TF_RETURN_IF_ERROR(graph_view.InitializeFromGraph(*optimized_graph));
+  std::unordered_set<const NodeDef*> optimized_nodes;
+
   for (int i : assign_nodes) {
-    if (optimized_nodes.find(i) == optimized_nodes.end()) {
-      const NodeDef& assign_node = optimized_graph->node(i);
-      optimized_nodes.insert(i);
-      std::vector<int> assign_nodes_in_fanout;
-      assign_nodes_in_fanout.push_back(i);
-      std::set<int> transitive_fanout;
-      graph_view.DepthFirstSearch(std::unordered_set<string>{}, i,
-                                  &transitive_fanout);
+    const NodeDef& assign_node = optimized_graph->node(i);
+
+    if (optimized_nodes.find(&assign_node) == optimized_nodes.end()) {
+      std::vector<const NodeDef*> assign_nodes_in_fanout;
+      optimized_nodes.insert(&assign_node);
+      assign_nodes_in_fanout.push_back(&assign_node);
+
+      std::vector<const NodeDef*> transitive_fanout;
+      DfsTraversal(graph_view, {graph_view.GetNode(i)},
+                   TraversalDirection::kFollowOutputs,
+                   DfsCallbacks::PreOrder([&](const NodeDef* node) {
+                     transitive_fanout.push_back(node);
+                   }));
+
       bool relax_constraint = true;
       // If all nodes in the transitive fanout are on the same device as the
       // assign node, there is no need to allocate the output in pinned memory.
-      for (int fanout : transitive_fanout) {
-        const NodeDef& fanout_node = optimized_graph->node(fanout);
+      for (const NodeDef* fanout_node : transitive_fanout) {
+        // const NodeDef& fanout_node = optimized_graph->node(fanout);
         if (relax_constraint &&
-            (IsSend(fanout_node) ||
-             CrossesTaskOrCpuGpuBoundary(fanout_node, assign_node))) {
+            (IsSend(*fanout_node) ||
+             CrossesTaskOrCpuGpuBoundary(*fanout_node, assign_node))) {
           relax_constraint = false;
           break;
         }
-        if (optimized_nodes.find(fanout) == optimized_nodes.end() &&
-            IsAssign(fanout_node)) {
-          assign_nodes_in_fanout.push_back(fanout);
+        if (optimized_nodes.find(fanout_node) == optimized_nodes.end() &&
+            IsAssign(*fanout_node)) {
+          assign_nodes_in_fanout.push_back(fanout_node);
         }
       }
 
       if (relax_constraint) {
-        for (int assign_idx : assign_nodes_in_fanout) {
+        for (const NodeDef* assign_node_in_fanout : assign_nodes_in_fanout) {
           // If all devices match in fanout of node(i) then, by transitivity,
           // they must also match in the fanout of other assign nodes
           // in the fanout of node(i), so we can process them here,
           // and save computing their transitive fanout later.
-          optimized_nodes.insert(assign_idx);
+          optimized_nodes.insert(assign_node_in_fanout);
 
           // Set an attribute telling AssignOp to ignore allocator constraints.
+          const absl::optional<int> assign_node_idx =
+              graph_view.GetNodeIndex(*assign_node_in_fanout);
           NodeDef* assign_node_to_relax =
-              optimized_graph->mutable_node(assign_idx);
+              optimized_graph->mutable_node(assign_node_idx.value());
           (*assign_node_to_relax
                 ->mutable_attr())["_grappler_relax_allocator_constraints"]
               .set_b(true);
diff --git a/tensorflow/core/grappler/utils.cc b/tensorflow/core/grappler/utils.cc
index d8fb88d95f..cc9a38b2d2 100644
--- a/tensorflow/core/grappler/utils.cc
+++ b/tensorflow/core/grappler/utils.cc
@@ -402,175 +402,6 @@ void EraseNodesFromGraph(const std::set<string>& nodes_to_delete,
   EraseNodesFromGraphImpl(nodes_idx_to_delete, graph);
 }
 
-Status SimpleGraphView::Initialize(
-    const GraphDef& graph,
-    const std::vector<std::pair<const NodeDef*, const NodeDef*>>*
-        extra_dependencies,
-    bool dedup_inputs, bool dedup_outputs) {
-  graph_ = &graph;
-  const int num_nodes = graph.node_size();
-  inputs_.clear();
-  inputs_.resize(num_nodes);
-  outputs_.clear();
-  outputs_.resize(num_nodes);
-  name_to_index_.clear();
-  name_to_index_.reserve(num_nodes);
-  index_to_name_.clear();
-  index_to_name_.reserve(num_nodes);
-
-  // Build map from name to index and vice versa.
-  for (int node_idx = 0; node_idx < num_nodes; ++node_idx) {
-    const NodeDef& node = graph.node(node_idx);
-    name_to_index_.emplace(node.name(), node_idx);
-    index_to_name_.push_back(node.name());
-  }
-
-  if (extra_dependencies) {
-    for (const auto& dep : *extra_dependencies) {
-      auto itr_src = name_to_index_.find(dep.first->name());
-      if (itr_src == name_to_index_.end()) {
-        return errors::InvalidArgument("Non-existent src ", dep.first->name());
-      }
-      auto itr_tgt = name_to_index_.find(dep.second->name());
-      if (itr_tgt == name_to_index_.end()) {
-        return errors::InvalidArgument("Non-existent tgt ", dep.second->name());
-      }
-      const int src_idx = itr_src->second;
-      const int tgt_idx = itr_tgt->second;
-      inputs_[tgt_idx].push_back(src_idx);
-      outputs_[src_idx].push_back(tgt_idx);
-    }
-  }
-
-  // Build forward and reverse adjacency lists.
-  for (int node_idx = 0; node_idx < num_nodes; ++node_idx) {
-    const NodeDef& node = graph.node(node_idx);
-    inputs_[node_idx].reserve(node.input_size());
-    for (const string& input : node.input()) {
-      auto it = name_to_index_.find(NodeName(input));
-      if (it == name_to_index_.end()) {
-        return errors::InvalidArgument("Non-existent input ", input,
-                                       " for node ", node.name());
-      }
-      const int input_idx = it->second;
-      inputs_[node_idx].push_back(input_idx);
-      outputs_[input_idx].push_back(node_idx);
-    }
-    if (dedup_inputs) {
-      // Dedup the input list while it's still hot in cache.
-      STLSortAndRemoveDuplicates(&inputs_[node_idx]);
-    }
-  }
-
-  // Dedup outputs.
-  if (dedup_outputs) {
-    for (int node_idx = 0; node_idx < num_nodes; ++node_idx) {
-      STLSortAndRemoveDuplicates(&outputs_[node_idx]);
-    }
-  }
-  return Status::OK();
-}
-
-void SimpleGraphView::DepthFirstSearch(
-    const std::unordered_set<string>& op_types_to_traverse, int root_node,
-    std::set<int>* nodes_found,
-    SimpleGraphView::SearchDirection direction) const {
-  nodes_found->clear();
-  const string& op_type = graph_->node(root_node).op();
-  if (!op_types_to_traverse.empty() &&
-      op_types_to_traverse.find(op_type) == op_types_to_traverse.end()) {
-    return;
-  }
-  std::vector<int> stack;
-  stack.reserve(32);
-  stack.push_back(root_node);
-
-  auto push_neighbors = [&stack,
-                         &nodes_found](absl::Span<const int> neighbors) {
-    for (auto output_idx : neighbors) {
-      if (nodes_found->find(output_idx) == nodes_found->end()) {
-        stack.push_back(output_idx);
-      }
-    }
-  };
-
-  while (!stack.empty()) {
-    const int node_idx = stack.back();
-    stack.pop_back();
-    nodes_found->insert(node_idx);
-    const string& op_type = graph_->node(node_idx).op();
-    if (op_types_to_traverse.empty() ||
-        op_types_to_traverse.find(op_type) != op_types_to_traverse.end()) {
-      if (direction == kFollowOutputs) {
-        push_neighbors(this->outputs(node_idx));
-      } else {
-        push_neighbors(this->inputs(node_idx));
-      }
-    }
-  }
-}
-
-void SimpleGraphView::DepthFirstSearchWithCallback(
-    const std::unordered_set<string>& op_types_to_traverse, int node_idx,
-    SimpleGraphView::DFSCallback callback,
-    SimpleGraphView::SearchDirection direction) const {
-  std::set<int> nodes_found;
-  nodes_found.clear();
-  const string& op_type = graph_->node(node_idx).op();
-  if (!op_types_to_traverse.empty() &&
-      op_types_to_traverse.find(op_type) == op_types_to_traverse.end()) {
-    return;
-  }
-  std::vector<int> stack;
-  stack.reserve(32);
-  stack.push_back(node_idx);
-  auto push_neighbors = [&stack,
-                         &nodes_found](absl::Span<const int> neighbors) {
-    for (auto output_idx : neighbors) {
-      if (nodes_found.find(output_idx) == nodes_found.end()) {
-        stack.push_back(output_idx);
-      }
-    }
-  };
-  while (!stack.empty()) {
-    const int node_idx = stack.back();
-    stack.pop_back();
-    if (callback(graph_->node(node_idx))) {
-      return;
-    }
-    nodes_found.insert(node_idx);
-    const string& op_type = graph_->node(node_idx).op();
-    if (op_types_to_traverse.empty() ||
-        op_types_to_traverse.find(op_type) != op_types_to_traverse.end()) {
-      if (direction == kFollowOutputs) {
-        push_neighbors(this->outputs(node_idx));
-      } else {
-        push_neighbors(this->inputs(node_idx));
-      }
-    }
-  }
-}
-
-string SimpleGraphView::PrintToString() const {
-  string str;
-  for (int i = 0; i < num_nodes(); ++i) {
-    strings::StrAppend(&str, "Node ", i, "'", node_name(i), "'\n", "Inputs: [");
-    for (int input : inputs(i)) {
-      strings::StrAppend(&str, input, " '", node_name(input), "', ");
-    }
-    strings::StrAppend(&str, "]\n", "Outputs: [");
-    for (int j = 0; j < outputs(i).size(); ++j) {
-      const int output = outputs(i)[j];
-      if (j > 0) {
-        strings::StrAppend(&str, ", ");
-      }
-      strings::StrAppend(&str, output, " '", node_name(output), "'");
-    }
-    strings::StrAppend(&str, "]\n");
-  }
-  return str;
-}
-
 #define HANDLE_CASE(DTYPE)                                          \
   case DTYPE:                                                       \
     if (!SafeSetScalarTensorValue<EnumToDataType<DTYPE>::Type>(     \
diff --git a/tensorflow/core/grappler/utils.h b/tensorflow/core/grappler/utils.h
index a9d6abd6db..1e820977ae 100644
--- a/tensorflow/core/grappler/utils.h
+++ b/tensorflow/core/grappler/utils.h
@@ -302,79 +302,6 @@ void EraseNodesFromGraph(std::vector<int>&& nodes_to_delete, GraphDef* graph);
 void EraseNodesFromGraph(const std::set<string>& nodes_to_delete,
                          GraphDef* graph);
 
-class SimpleGraphView {
- public:
-  // Build a graph view for the specified graphdef.
-  Status Initialize(const GraphDef& graph) {
-    return Initialize(graph, nullptr, true, true);
-  }
-  // Build a graph view for the specified graphdef augmented with the additional
-  // edges specified in 'extra_dependencies' if any. Note that
-  // extra_dependencies can be null.
-  Status Initialize(
-      const GraphDef& graph,
-      const std::vector<std::pair<const NodeDef*, const NodeDef*>>*
-          extra_dependencies) {
-    return Initialize(graph, extra_dependencies, true, true);
-  }
-  Status Initialize(
-      const GraphDef& graph,
-      const std::vector<std::pair<const NodeDef*, const NodeDef*>>*
-          extra_dependencies,
-      bool dedup_inputs, bool dedup_outputs);
-
-  const GraphDef* graph() const { return graph_; }
-  inline int num_nodes() const { return index_to_name_.size(); }
-  inline bool has_node(const string& node_name) const {
-    return name_to_index_.find(node_name) != name_to_index_.end();
-  }
-  inline const int index(const string& node_name) const {
-    const auto& it = name_to_index_.find(node_name);
-    DCHECK(it != name_to_index_.end());
-    return it == name_to_index_.end() ? -1 : it->second;
-  }
-  inline const NodeDef& node(int node_idx) const {
-    return graph_->node(node_idx);
-  }
-  inline const string& node_name(int node_idx) const {
-    return index_to_name_[node_idx];
-  }
-  inline const gtl::InlinedVector<int, 4>& inputs(int node_idx) const {
-    return inputs_[node_idx];
-  }
-  inline const gtl::InlinedVector<int, 2>& outputs(int node_idx) const {
-    return outputs_[node_idx];
-  }
-
-  enum SearchDirection { kFollowOutputs = 1, kFollowInputs = 2 };
-
-  // Traverse the graph starting at `node_idx`, collecting indices of nodes
-  // visited in nodes_found. If a node has an op in `op_types_to_traverse`, the
-  // walk continues to its children. It is assumed that *graph_ was not modified
-  // after the call to Initialize().
-  // If `op_types_to_traverse` is empty the DFS will traverse any node type.
-  void DepthFirstSearch(const std::unordered_set<string>& op_types_to_traverse,
-                        int node_idx, std::set<int>* nodes_found,
-                        SearchDirection direction = kFollowOutputs) const;
-
-  typedef std::function<bool(const NodeDef&)> DFSCallback;
-
-  // Like DepthFirstSearch, but invoke `callback` as each node is discovered. If
-  // `callback` returns true, the search is terminated early.
-  void DepthFirstSearchWithCallback(
-      const std::unordered_set<string>& op_types_to_traverse, int node_idx,
-      DFSCallback callback, SearchDirection direction = kFollowOutputs) const;
-
-  string PrintToString() const;
-
- private:
-  const GraphDef* graph_;  // Not owned.
-  std::vector<string> index_to_name_;
-  gtl::FlatMap<string, int> name_to_index_;
-  std::vector<gtl::InlinedVector<int, 4>> inputs_;
-  std::vector<gtl::InlinedVector<int, 2>> outputs_;
-};
-
 }  // end namespace grappler
 }  // end namespace tensorflow
 
diff --git a/tensorflow/core/grappler/utils/traversal.cc b/tensorflow/core/grappler/utils/traversal.cc
index 3231094fd4..c602e8c0e4 100644
--- a/tensorflow/core/grappler/utils/traversal.cc
+++ b/tensorflow/core/grappler/utils/traversal.cc
@@ -44,8 +44,8 @@ enum class NodeState { kNotVisited, kVisiting, kDone };
 
 void DfsTraversal(const GraphTopologyView& graph_view,
                   const absl::Span<const NodeDef* const> from,
-                  TraversalDirection direction,
-                  const std::function<bool(const NodeDef*)>& should_visit,
+                  const TraversalDirection direction,
+                  const DfsPredicates& predicates,
                   const DfsCallbacks& callbacks) {
   std::vector<DfsStackElem> stack;
   stack.reserve(from.size());
@@ -66,8 +66,8 @@ void DfsTraversal(const GraphTopologyView& graph_view,
     NodeState& state = node_state[w.node];
     if (state == NodeState::kDone) continue;
 
-    // Skip nodes that do not match predicate.
-    if (should_visit && !should_visit(graph_view.GetNode(w.node))) {
+    // Skip nodes that we should not enter.
+    if (predicates.enter && !predicates.enter(graph_view.GetNode(w.node))) {
       state = NodeState::kDone;
       continue;
     }
@@ -98,6 +98,11 @@ void DfsTraversal(const GraphTopologyView& graph_view,
     // Enqueue the node again with the children_visited flag set to true.
     stack.emplace_back(w.node, true, w.src);
 
+    // Check if we can continue traversal from the current node.
+    if (predicates.advance && !predicates.advance(graph_view.GetNode(w.node))) {
+      continue;
+    }
+
     // Now enqueue the fanin/fanout nodes.
     if (direction == TraversalDirection::kFollowInputs) {
       for (const int fanin : graph_view.GetFanin(w.node)) {
@@ -111,14 +116,10 @@ void DfsTraversal(const GraphTopologyView& graph_view,
   }
 }
 
-// Traverse the graph in DFS order in the given direction, starting from the
-// list of nodes specified in the `from` argument. Call corresponding
-// callbacks for each visited node.
 void DfsTraversal(const GraphTopologyView& graph_view,
                   const absl::Span<const NodeDef* const> from,
                   TraversalDirection direction, const DfsCallbacks& callbacks) {
-  const auto visit_all = [](const NodeDef*) { return true; };
-  DfsTraversal(graph_view, from, direction, visit_all, callbacks);
+  DfsTraversal(graph_view, from, direction, {}, callbacks);
 }
 
 }  // namespace grappler
diff --git a/tensorflow/core/grappler/utils/traversal.h b/tensorflow/core/grappler/utils/traversal.h
index ec6009096d..5c9dada493 100644
--- a/tensorflow/core/grappler/utils/traversal.h
+++ b/tensorflow/core/grappler/utils/traversal.h
@@ -33,6 +33,7 @@ enum class TraversalDirection { kFollowInputs, kFollowOutputs };
 // corresponding back edges. Moreover, the pre and post order will assume that
 // these back edges will be cut.
 struct DfsCallbacks {
+  DfsCallbacks() = default;
   DfsCallbacks(std::function<void(const NodeDef*)> pre,
                std::function<void(const NodeDef*)> post,
                std::function<void(const NodeDef*, const NodeDef*)> back_edge)
@@ -53,16 +54,40 @@ struct DfsCallbacks {
   std::function<void(const NodeDef*, const NodeDef*)> on_back_edge;
 };
 
+// Encapsulate DFS predicates for traversing the graph.
+//
+// The `enter` predicate decides if traversal should enter the node, and the
+// `advance` predicate decides if the traversal should follow inputs/outputs
+// from the node.
+//
+// If predicates are empty (default initialized), it's assumed that we can enter
+// into any node and advance from any node respectively.
+struct DfsPredicates {
+  DfsPredicates() = default;
+  DfsPredicates(std::function<bool(const NodeDef*)> enter,
+                std::function<bool(const NodeDef*)> advance)
+      : enter(std::move(enter)), advance(std::move(advance)) {}
+
+  static DfsPredicates Enter(std::function<bool(const NodeDef*)> enter) {
+    return DfsPredicates(std::move(enter), nullptr);
+  }
+
+  static DfsPredicates Advance(std::function<bool(const NodeDef*)> advance) {
+    return DfsPredicates(nullptr, std::move(advance));
+  }
+
+  std::function<bool(const NodeDef*)> enter;
+  std::function<bool(const NodeDef*)> advance;
+};
+
 // Traverse the graph in DFS order in the given direction, starting from the
-// list of nodes specified in the `from` argument. Use `should_visit` to decide
-// if the node should be visited. This predicate also applied to the `from`
-// nodes. Call corresponding callbacks for each visited node. If `should_visit`
-// is empty (default initialized) it's assumed that traversal can visit all the
-// nodes.
+// list of nodes specified in the `from` argument. Use `predicates` to decide if
+// traversal should enter/advance to/from the graph node. These predicates also
+// applied to the `from` nodes. Call corresponding callbacks for each visited
+// node.
 void DfsTraversal(const GraphTopologyView& graph_view,
                   absl::Span<const NodeDef* const> from,
-                  TraversalDirection direction,
-                  const std::function<bool(const NodeDef*)>& should_visit,
+                  TraversalDirection direction, const DfsPredicates& predicates,
                   const DfsCallbacks& callbacks);
 
 // Traverse the graph in DFS order in the given direction, starting from the
diff --git a/tensorflow/core/grappler/utils/traversal_test.cc b/tensorflow/core/grappler/utils/traversal_test.cc
index 5a68fa201e..7b36d328e9 100644
--- a/tensorflow/core/grappler/utils/traversal_test.cc
+++ b/tensorflow/core/grappler/utils/traversal_test.cc
@@ -159,7 +159,7 @@ TEST(TraversalTest, OutputDfsWithLoop) {
   EXPECT_EQ(back_edges, expected_edges);
 }
 
-TEST(TraversalTest, DfsWithPredicate) {
+TEST(TraversalTest, DfsWithEnterPredicate) {
   const string op = "OpIsNotImportantInThisTest";
 
   GraphDef graph = ::tensorflow::test::function::GDef(  //
@@ -171,8 +171,8 @@ TEST(TraversalTest, DfsWithPredicate) {
        NDef("6", op, {"3", "5"}, {})},                  //
       /*funcs=*/{});
 
-  // Do not go through nodes '2' and '3'.
-  const auto predicate = [](const NodeDef* node) {
+  // Do not enter the nodes '2' and '3'.
+  const auto enter = [](const NodeDef* node) {
     return node->name() != "2" && node->name() != "3";
   };
 
@@ -185,7 +185,8 @@ TEST(TraversalTest, DfsWithPredicate) {
   GraphTopologyView graph_view;
   TF_CHECK_OK(graph_view.InitializeFromGraph(graph));
   DfsTraversal(graph_view, start_nodes, TraversalDirection::kFollowOutputs,
-               predicate, MkCallbacks(&pre_order, &post_order, &back_edges));
+               DfsPredicates::Enter(enter),
+               MkCallbacks(&pre_order, &post_order, &back_edges));
 
   const std::vector<string> expected_pre = {"1", "4", "5", "6"};
   const std::vector<string> expected_post = {"6", "5", "4", "1"};
@@ -195,6 +196,43 @@ TEST(TraversalTest, DfsWithPredicate) {
   EXPECT_TRUE(back_edges.empty());
 }
 
+TEST(TraversalTest, DfsWithAdvancePredicate) {
+  const string op = "OpIsNotImportantInThisTest";
+
+  GraphDef graph = ::tensorflow::test::function::GDef(  //
+      {NDef("1", op, {}, {}),                           //       2 -> 3
+       NDef("2", op, {"1"}, {}),                        // 1 -> /      \ -> 6
+       NDef("3", op, {"2"}, {}),                        //      \      /
+       NDef("4", op, {"1"}, {}),                        //       4 -> 5
+       NDef("5", op, {"4"}, {}),                        //
+       NDef("6", op, {"3", "5"}, {})},                  //
+      {} /* empty function library*/);
+
+  // Do not advance from the nodes '2' and '3'.
+  const auto advance = [](const NodeDef* node) {
+    return node->name() != "2" && node->name() != "3";
+  };
+
+  std::vector<const NodeDef*> start_nodes = {&graph.node(0)};
+
+  std::vector<string> pre_order;
+  std::vector<string> post_order;
+  std::vector<string> back_edges;
+
+  GraphTopologyView graph_view;
+  TF_CHECK_OK(graph_view.InitializeFromGraph(graph));
+  DfsTraversal(graph_view, start_nodes, TraversalDirection::kFollowOutputs,
+               DfsPredicates::Advance(advance),
+               MkCallbacks(&pre_order, &post_order, &back_edges));
+
+  const std::vector<string> expected_pre = {"1", "4", "5", "6", "2"};
+  const std::vector<string> expected_post = {"6", "5", "4", "2", "1"};
+
+  EXPECT_EQ(pre_order, expected_pre);
+  EXPECT_EQ(post_order, expected_post);
+  EXPECT_TRUE(back_edges.empty());
+}
+
 }  // namespace
 }  // namespace grappler
 }  // namespace tensorflow
-- 
GitLab


From 75f12a50203ca31370c5edc02d650ee4a47fdf03 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micka=C3=ABl=20Schoentgen?= <contact@tiger-222.fr>
Date: Thu, 3 Jan 2019 22:36:43 +0100
Subject: [PATCH 0184/1095] Fix several DeprecationWarning: invlid escape
 sequence
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Mickaël Schoentgen <contact@tiger-222.fr>
---
 .../contrib/kernel_methods/python/losses.py      |  2 +-
 .../python/learn/learn_io/generator_io_test.py   |  4 ++--
 .../opt/python/training/adam_gs_optimizer.py     |  2 +-
 tensorflow/contrib/optimizer_v2/adam.py          |  2 +-
 tensorflow/python/client/session_test.py         |  2 +-
 .../python/feature_column/feature_column_test.py | 12 ++++++------
 .../feature_column/feature_column_v2_test.py     | 16 ++++++++--------
 .../python/kernel_tests/confusion_matrix_test.py |  4 ++--
 tensorflow/python/training/adam.py               |  2 +-
 tensorflow/tools/ci_build/copy_binary.py         |  2 +-
 .../scripts_allreduce/k8s_generate_yaml_lib.py   |  2 +-
 .../windows/msvc_wrapper_for_nvcc.py             |  2 +-
 .../windows/msvc_wrapper_for_nvcc.py             |  2 +-
 .../gcc-nvcc/windows/msvc_wrapper_for_nvcc.py    |  2 +-
 14 files changed, 28 insertions(+), 28 deletions(-)

diff --git a/tensorflow/contrib/kernel_methods/python/losses.py b/tensorflow/contrib/kernel_methods/python/losses.py
index 4ef0a66a52..294a7d69a7 100644
--- a/tensorflow/contrib/kernel_methods/python/losses.py
+++ b/tensorflow/contrib/kernel_methods/python/losses.py
@@ -34,7 +34,7 @@ def sparse_multiclass_hinge_loss(
     scope=None,
     loss_collection=ops.GraphKeys.LOSSES,
     reduction=losses.Reduction.SUM_BY_NONZERO_WEIGHTS):
-  """Adds Ops for computing the multiclass hinge loss.
+  r"""Adds Ops for computing the multiclass hinge loss.
 
   The implementation is based on the following paper:
   On the Algorithmic Implementation of Multiclass Kernel-based Vector Machines
diff --git a/tensorflow/contrib/learn/python/learn/learn_io/generator_io_test.py b/tensorflow/contrib/learn/python/learn/learn_io/generator_io_test.py
index 5e90d1fa20..318046733b 100644
--- a/tensorflow/contrib/learn/python/learn/learn_io/generator_io_test.py
+++ b/tensorflow/contrib/learn/python/learn/learn_io/generator_io_test.py
@@ -174,7 +174,7 @@ class GeneratorIoTest(test.TestCase):
       return np.arange(32, 36)
 
     with self.cached_session():
-      with self.assertRaisesRegexp(TypeError, 'x\(\) must be generator'):
+      with self.assertRaisesRegexp(TypeError, r'x\(\) must be generator'):
         failing_input_fn = generator_io.generator_input_fn(
             generator, batch_size=2, shuffle=False, num_epochs=1)
         failing_input_fn()
@@ -185,7 +185,7 @@ class GeneratorIoTest(test.TestCase):
       yield np.arange(32, 36)
 
     with self.cached_session():
-      with self.assertRaisesRegexp(TypeError, 'x\(\) must yield dict'):
+      with self.assertRaisesRegexp(TypeError, r'x\(\) must yield dict'):
         failing_input_fn = generator_io.generator_input_fn(
             generator, batch_size=2, shuffle=False, num_epochs=1)
         failing_input_fn()
diff --git a/tensorflow/contrib/opt/python/training/adam_gs_optimizer.py b/tensorflow/contrib/opt/python/training/adam_gs_optimizer.py
index 3fb649ea82..855fbf58bf 100644
--- a/tensorflow/contrib/opt/python/training/adam_gs_optimizer.py
+++ b/tensorflow/contrib/opt/python/training/adam_gs_optimizer.py
@@ -41,7 +41,7 @@ class AdamGSOptimizer(optimizer.Optimizer):
   def __init__(self, global_step=0, learning_rate=0.001,
                beta1=0.9, beta2=0.999, epsilon=1e-8,
                use_locking=False, name="Adam"):
-    """Construct a new Adam optimizer.
+    r"""Construct a new Adam optimizer.
 
     Branched from tf.train.AdamOptimizer. The only difference is to pass
     global step for computing beta1 and beta2 accumulators, instead of having
diff --git a/tensorflow/contrib/optimizer_v2/adam.py b/tensorflow/contrib/optimizer_v2/adam.py
index 248ffb1f7e..1b7800f324 100644
--- a/tensorflow/contrib/optimizer_v2/adam.py
+++ b/tensorflow/contrib/optimizer_v2/adam.py
@@ -36,7 +36,7 @@ class AdamOptimizer(optimizer_v2.OptimizerV2):
 
   def __init__(self, learning_rate=0.001, beta1=0.9, beta2=0.999, epsilon=1e-8,
                use_locking=False, name="Adam"):
-    """Construct a new Adam optimizer.
+    r"""Construct a new Adam optimizer.
 
     Initialization:
 
diff --git a/tensorflow/python/client/session_test.py b/tensorflow/python/client/session_test.py
index c4a118a414..da6218663d 100644
--- a/tensorflow/python/client/session_test.py
+++ b/tensorflow/python/client/session_test.py
@@ -2036,7 +2036,7 @@ class SessionTest(test_util.TensorFlowTestCase):
     with self.cached_session() as sess:
       a = array_ops.placeholder(dtype=dtypes.string)
       with self.assertRaisesRegexp(
-          TypeError, 'Type of feed value 1 with type <(\w+) \'int\'> is not'):
+          TypeError, r'Type of feed value 1 with type <(\w+) \'int\'> is not'):
         sess.run(a, feed_dict={a: 1})
 
 
diff --git a/tensorflow/python/feature_column/feature_column_test.py b/tensorflow/python/feature_column/feature_column_test.py
index daa0a3b3a4..0ded2bf8c9 100644
--- a/tensorflow/python/feature_column/feature_column_test.py
+++ b/tensorflow/python/feature_column/feature_column_test.py
@@ -1832,7 +1832,7 @@ class LinearModelTest(test.TestCase):
       }
     with self.assertRaisesRegexp(
         ValueError,
-        'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
+        r'Batch size \(first dimension\) of each feature must be same.'):
       fc.linear_model(features, [price1, price2])
 
   def test_subset_of_static_batch_size_mismatch(self):
@@ -1847,7 +1847,7 @@ class LinearModelTest(test.TestCase):
       }
       with self.assertRaisesRegexp(
           ValueError,
-          'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
+          r'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
         fc.linear_model(features, [price1, price2, price3])
 
   def test_runtime_batch_size_mismatch(self):
@@ -2467,7 +2467,7 @@ class _LinearModelTest(test.TestCase):
       }
     with self.assertRaisesRegexp(
         ValueError,
-        'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
+        r'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
       get_keras_linear_model_predictions(features, [price1, price2])
 
   def test_subset_of_static_batch_size_mismatch(self):
@@ -2482,7 +2482,7 @@ class _LinearModelTest(test.TestCase):
       }
       with self.assertRaisesRegexp(
           ValueError,
-          'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
+          r'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
         get_keras_linear_model_predictions(features, [price1, price2, price3])
 
   def test_runtime_batch_size_mismatch(self):
@@ -2974,7 +2974,7 @@ class FunctionalInputLayerTest(test.TestCase):
       }
       with self.assertRaisesRegexp(
           ValueError,
-          'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
+          r'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
         fc.input_layer(features, [price1, price2])
 
   def test_subset_of_static_batch_size_mismatch(self):
@@ -2989,7 +2989,7 @@ class FunctionalInputLayerTest(test.TestCase):
       }
       with self.assertRaisesRegexp(
           ValueError,
-          'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
+          r'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
         fc.input_layer(features, [price1, price2, price3])
 
   def test_runtime_batch_size_mismatch(self):
diff --git a/tensorflow/python/feature_column/feature_column_v2_test.py b/tensorflow/python/feature_column/feature_column_v2_test.py
index a247425369..2a864a0573 100644
--- a/tensorflow/python/feature_column/feature_column_v2_test.py
+++ b/tensorflow/python/feature_column/feature_column_v2_test.py
@@ -2052,7 +2052,7 @@ class LinearModelTest(test.TestCase):
       }
     with self.assertRaisesRegexp(
         ValueError,
-        'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
+        r'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
       model = fc.LinearModel([price1, price2])
       model(features)
 
@@ -2068,7 +2068,7 @@ class LinearModelTest(test.TestCase):
       }
       with self.assertRaisesRegexp(
           ValueError,
-          'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
+          r'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
         model = fc.LinearModel([price1, price2, price3])
         model(features)
 
@@ -2818,7 +2818,7 @@ class OldLinearModelTest(test.TestCase):
       }
     with self.assertRaisesRegexp(
         ValueError,
-        'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
+        r'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
       fc_old.linear_model(features, [price1, price2])
 
   def test_subset_of_static_batch_size_mismatch(self):
@@ -2833,7 +2833,7 @@ class OldLinearModelTest(test.TestCase):
       }
       with self.assertRaisesRegexp(
           ValueError,
-          'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
+          r'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
         fc_old.linear_model(features, [price1, price2, price3])
 
   def test_runtime_batch_size_mismatch(self):
@@ -3435,7 +3435,7 @@ class DenseFeaturesTest(test.TestCase):
       }
       with self.assertRaisesRegexp(
           ValueError,
-          'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
+          r'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
         fc.DenseFeatures([price1, price2])(features)
 
   def test_subset_of_static_batch_size_mismatch(self):
@@ -3450,7 +3450,7 @@ class DenseFeaturesTest(test.TestCase):
       }
       with self.assertRaisesRegexp(
           ValueError,
-          'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
+          r'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
         fc.DenseFeatures([price1, price2, price3])(features)
 
   def test_runtime_batch_size_mismatch(self):
@@ -4141,7 +4141,7 @@ class FunctionalInputLayerTest(test.TestCase):
       }
       with self.assertRaisesRegexp(
           ValueError,
-          'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
+          r'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
         fc_old.input_layer(features, [price1, price2])
 
   def test_subset_of_static_batch_size_mismatch(self):
@@ -4156,7 +4156,7 @@ class FunctionalInputLayerTest(test.TestCase):
       }
       with self.assertRaisesRegexp(
           ValueError,
-          'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
+          r'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
         fc_old.input_layer(features, [price1, price2, price3])
 
   def test_runtime_batch_size_mismatch(self):
diff --git a/tensorflow/python/kernel_tests/confusion_matrix_test.py b/tensorflow/python/kernel_tests/confusion_matrix_test.py
index ae13c8e32e..6670f0326c 100644
--- a/tensorflow/python/kernel_tests/confusion_matrix_test.py
+++ b/tensorflow/python/kernel_tests/confusion_matrix_test.py
@@ -472,7 +472,7 @@ class RemoveSqueezableDimensionsTest(test.TestCase):
       }
       with self.assertRaisesRegexp(
           errors_impl.InvalidArgumentError,
-          "Can not squeeze dim\[2\]"):
+          r"Can not squeeze dim\[2\]"):
         dynamic_labels.eval(feed_dict=feed_dict)
       self.assertAllEqual(
           prediction_values, dynamic_predictions.eval(feed_dict=feed_dict))
@@ -500,7 +500,7 @@ class RemoveSqueezableDimensionsTest(test.TestCase):
           label_values, dynamic_labels.eval(feed_dict=feed_dict))
       with self.assertRaisesRegexp(
           errors_impl.InvalidArgumentError,
-          "Can not squeeze dim\[2\]"):
+          r"Can not squeeze dim\[2\]"):
         dynamic_predictions.eval(feed_dict=feed_dict)
 
 
diff --git a/tensorflow/python/training/adam.py b/tensorflow/python/training/adam.py
index 0c701f4712..c204a5c1ee 100644
--- a/tensorflow/python/training/adam.py
+++ b/tensorflow/python/training/adam.py
@@ -39,7 +39,7 @@ class AdamOptimizer(optimizer.Optimizer):
 
   def __init__(self, learning_rate=0.001, beta1=0.9, beta2=0.999, epsilon=1e-8,
                use_locking=False, name="Adam"):
-    """Construct a new Adam optimizer.
+    r"""Construct a new Adam optimizer.
 
     Initialization:
 
diff --git a/tensorflow/tools/ci_build/copy_binary.py b/tensorflow/tools/ci_build/copy_binary.py
index 148526492d..40a7443745 100755
--- a/tensorflow/tools/ci_build/copy_binary.py
+++ b/tensorflow/tools/ci_build/copy_binary.py
@@ -33,7 +33,7 @@ import tempfile
 import zipfile
 
 TF_NIGHTLY_REGEX = (r"(.+)tf_nightly(|_gpu)-(\d\.[\d]{1,2}"
-                    "\.\d.dev[\d]{0,8})-(.+)\.whl")
+                    r"\.\d.dev[\d]{0,8})-(.+)\.whl")
 BINARY_STRING_TEMPLATE = "%s-%s-%s.whl"
 
 
diff --git a/tensorflow/tools/dist_test/scripts_allreduce/k8s_generate_yaml_lib.py b/tensorflow/tools/dist_test/scripts_allreduce/k8s_generate_yaml_lib.py
index c570d1a9f8..038a712d53 100644
--- a/tensorflow/tools/dist_test/scripts_allreduce/k8s_generate_yaml_lib.py
+++ b/tensorflow/tools/dist_test/scripts_allreduce/k8s_generate_yaml_lib.py
@@ -195,7 +195,7 @@ def generate_RSA(bits=2048, exponent=65537):
 
 def get_change_ssh_port(use_hostnet, port):
   if use_hostnet == 1:
-    return "sed -i '/Port 22/c\Port {}' /etc/ssh/sshd_config".format(port)
+    return r"sed -i '/Port 22/c\Port {}' /etc/ssh/sshd_config".format(port)
 
   return ''
 
diff --git a/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda10.0/windows/msvc_wrapper_for_nvcc.py b/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda10.0/windows/msvc_wrapper_for_nvcc.py
index 00483951af..2ea20b8b53 100755
--- a/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda10.0/windows/msvc_wrapper_for_nvcc.py
+++ b/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda10.0/windows/msvc_wrapper_for_nvcc.py
@@ -104,7 +104,7 @@ def InvokeNvcc(argv, log=False):
   """
 
   src_files = [f for f in argv if
-               re.search('\.cpp$|\.cc$|\.c$|\.cxx$|\.C$', f)]
+               re.search(r'\.cpp$|\.cc$|\.c$|\.cxx$|\.C$', f)]
   if len(src_files) == 0:
     raise Error('No source files found for cuda compilation.')
 
diff --git a/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/windows/msvc_wrapper_for_nvcc.py b/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/windows/msvc_wrapper_for_nvcc.py
index 859b3196d5..2d0898e9cb 100755
--- a/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/windows/msvc_wrapper_for_nvcc.py
+++ b/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/windows/msvc_wrapper_for_nvcc.py
@@ -104,7 +104,7 @@ def InvokeNvcc(argv, log=False):
   """
 
   src_files = [f for f in argv if
-               re.search('\.cpp$|\.cc$|\.c$|\.cxx$|\.C$', f)]
+               re.search(r'\.cpp$|\.cc$|\.c$|\.cxx$|\.C$', f)]
   if len(src_files) == 0:
     raise Error('No source files found for cuda compilation.')
 
diff --git a/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc/windows/msvc_wrapper_for_nvcc.py b/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc/windows/msvc_wrapper_for_nvcc.py
index 859b3196d5..2d0898e9cb 100755
--- a/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc/windows/msvc_wrapper_for_nvcc.py
+++ b/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc/windows/msvc_wrapper_for_nvcc.py
@@ -104,7 +104,7 @@ def InvokeNvcc(argv, log=False):
   """
 
   src_files = [f for f in argv if
-               re.search('\.cpp$|\.cc$|\.c$|\.cxx$|\.C$', f)]
+               re.search(r'\.cpp$|\.cc$|\.c$|\.cxx$|\.C$', f)]
   if len(src_files) == 0:
     raise Error('No source files found for cuda compilation.')
 
-- 
GitLab


From 4ee52f8b0751004e76164464938cafed9f1f3623 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Thu, 3 Jan 2019 13:33:29 -0800
Subject: [PATCH 0185/1095] [TF:XLA] Bump open source llvm revision to r350327

PiperOrigin-RevId: 227740939
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 11ce55feda..7792824dbf 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -498,11 +498,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "llvm",
         build_file = clean_dep("//third_party/llvm:llvm.autogenerated.BUILD"),
-        sha256 = "65a1aeb29e5940f9f480a41e904659d944e738458afd139caa7bde14bd6aab8a",
-        strip_prefix = "llvm-331ffd31b3dd49b3f02a27556938b836b679f564",
+        sha256 = "3a1c8a817d2d9a85b85e1ad776a90faa51f475bb924fbf70383892bb1faceac9",
+        strip_prefix = "llvm-6da47bee848304d215e3ca08b2a4a4dc9c954310",
         urls = [
-            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/331ffd31b3dd49b3f02a27556938b836b679f564.tar.gz",
-            "https://github.com/llvm-mirror/llvm/archive/331ffd31b3dd49b3f02a27556938b836b679f564.tar.gz",
+            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/6da47bee848304d215e3ca08b2a4a4dc9c954310.tar.gz",
+            "https://github.com/llvm-mirror/llvm/archive/6da47bee848304d215e3ca08b2a4a4dc9c954310.tar.gz",
         ],
     )
 
-- 
GitLab


From 6f4410f580ff48078f33c343b72d742095535fae Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Thu, 3 Jan 2019 14:05:20 -0800
Subject: [PATCH 0186/1095] Disable on mac a flaky test.

PiperOrigin-RevId: 227746737
---
 tensorflow/lite/kernels/internal/BUILD | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/lite/kernels/internal/BUILD b/tensorflow/lite/kernels/internal/BUILD
index 5eb8a353b7..92b4705908 100644
--- a/tensorflow/lite/kernels/internal/BUILD
+++ b/tensorflow/lite/kernels/internal/BUILD
@@ -654,6 +654,8 @@ cc_test(
         "logsoftmax_quantized_test.cc",
     ],
     tags = [
+        # TODO(b/122242739): Reenable after fixing the flakiness?
+        "nomac",
         "tflite_not_portable",
     ],
     deps = [
-- 
GitLab


From 4dc0ae2baf454ba3d3ee531d889079b0ff05bc95 Mon Sep 17 00:00:00 2001
From: Blake Hechtman <blakehechtman@google.com>
Date: Thu, 3 Jan 2019 14:13:01 -0800
Subject: [PATCH 0187/1095] Fix typo.

PiperOrigin-RevId: 227748088
---
 tensorflow/compiler/xla/service/layout_assignment.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/layout_assignment.cc b/tensorflow/compiler/xla/service/layout_assignment.cc
index ebf7a88fae..09246db68e 100644
--- a/tensorflow/compiler/xla/service/layout_assignment.cc
+++ b/tensorflow/compiler/xla/service/layout_assignment.cc
@@ -1239,7 +1239,7 @@ Status LayoutAssignment::PropagateUseConstraintToDefs(
 namespace {
 // A transpose or a reshape that only changes trivial dimensions have meaningful
 // layouts that are valuable to propagate in a depthfirst manner to avoid
-// unassinged layouts in the graph.
+// unassigned layouts in the graph.
 bool InstructionShouldPropagateDepthFirst(const HloInstruction& hlo) {
   switch (hlo.opcode()) {
     case HloOpcode::kReshape:
-- 
GitLab


From 7c9323bedc48c98be3c07b72ec1d6f4dccdefb35 Mon Sep 17 00:00:00 2001
From: Penporn Koanantakool <penporn@google.com>
Date: Thu, 3 Jan 2019 14:22:17 -0800
Subject: [PATCH 0188/1095] Turn on MKL-DNN contraction kernels by default. To
 disable them, build with --define=tensorflow_mkldnn_contraction_kernel=0.
 Make TestFoldFusedBatchNormsWithConcat use ExpectClose which is more suitable
 for floating-point comparison than ExpectNear.

PiperOrigin-RevId: 227749705
---
 .bazelrc                                                       | 3 ---
 tensorflow/tools/graph_transforms/fold_old_batch_norms_test.cc | 2 +-
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/.bazelrc b/.bazelrc
index ceba7bfdba..cd7e13ddfc 100644
--- a/.bazelrc
+++ b/.bazelrc
@@ -93,9 +93,6 @@ build --define=PREFIX=/usr
 build --define=LIBDIR=$(PREFIX)/lib
 build --define=INCLUDEDIR=$(PREFIX)/include
 
-# Disable MKL-DNN contraction kernels by default.
-build --define=tensorflow_mkldnn_contraction_kernel=0
-
 # Default options should come above this line
 
 # Options from ./configure
diff --git a/tensorflow/tools/graph_transforms/fold_old_batch_norms_test.cc b/tensorflow/tools/graph_transforms/fold_old_batch_norms_test.cc
index 435f46c107..6c7174926d 100644
--- a/tensorflow/tools/graph_transforms/fold_old_batch_norms_test.cc
+++ b/tensorflow/tools/graph_transforms/fold_old_batch_norms_test.cc
@@ -291,7 +291,7 @@ class FoldOldBatchNormsTest : public ::testing::Test {
     std::vector<Tensor> fused_outputs;
     TF_ASSERT_OK(fused_session->Run({}, {"output"}, {}, &fused_outputs));
 
-    test::ExpectTensorNear<float>(original_outputs[0], fused_outputs[0], 1e-5);
+    test::ExpectClose(original_outputs[0], fused_outputs[0]);
 
     for (const NodeDef& node : fused_graph_def.node()) {
       EXPECT_NE("FusedBatchNorm", node.op());
-- 
GitLab


From 9d78259894c0b533ac6454c9eee27f8eb8bef638 Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Thu, 3 Jan 2019 14:29:35 -0800
Subject: [PATCH 0189/1095] Add "element_shape" attr to TensorListConcat and
 use it in _GraphTensorArrayV2.

When using while_v2, this change prevents errors like this:
  All except the first dimension must be fully defined when concating an empty tensor list. element_shape: <unknown>

Previously, TensorListConcat op could only determine the element size
from the TensorList object at runtime. In the case of a while loop
that executes zero times, it wouldn't be able to determine the size
since no element was ever seen by the runtime. However, we may have
determined the element size during graph construction, so we use the
attribute to tell the runtime what the element size is.

Note that the original TensorArray implementation already does this.

PiperOrigin-RevId: 227750868
---
 tensorflow/core/kernels/list_kernels.h        | 44 +++++++++++++------
 tensorflow/core/ops/list_ops.cc               | 14 ++++--
 tensorflow/python/ops/list_ops.py             |  6 ++-
 .../ops/parallel_for/control_flow_ops_test.py |  5 ---
 tensorflow/python/ops/tensor_array_ops.py     | 12 +++--
 5 files changed, 53 insertions(+), 28 deletions(-)

diff --git a/tensorflow/core/kernels/list_kernels.h b/tensorflow/core/kernels/list_kernels.h
index 686679474c..99b4f42084 100644
--- a/tensorflow/core/kernels/list_kernels.h
+++ b/tensorflow/core/kernels/list_kernels.h
@@ -158,6 +158,17 @@ class TensorListConcat : public OpKernel {
       std::vector<std::unique_ptr<typename TTypes<T, 2>::ConstMatrix>>;
   explicit TensorListConcat(OpKernelConstruction* c) : OpKernel(c) {
     OP_REQUIRES_OK(c, c->GetAttr("element_dtype", &element_dtype_));
+    // TODO(skyewm): the HasAttr check can be removed once the
+    // element_shape_except_first_dim attr has been checked in for 2 weeks
+    // (around 1/14/2019).
+    if (c->HasAttr("element_shape")) {
+      PartialTensorShape element_shape;
+      OP_REQUIRES_OK(c, c->GetAttr("element_shape", &element_shape));
+      if (!element_shape.unknown_rank()) {
+        element_shape_except_first_dim_ = PartialTensorShape(
+            gtl::ArraySlice<int64>(element_shape.dim_sizes()).subspan(1));
+      }
+    }
   }
 
   ~TensorListConcat() {}
@@ -178,29 +189,33 @@ class TensorListConcat : public OpKernel {
             " but list elements ", DataTypeString(tensor_list->element_dtype)));
     // If the TensorList is empty, its element_shape must be fully defined
     // except for the first dimension.
-    PartialTensorShape shape_except_first_dim;
-    if (!tensor_list->element_shape.unknown_rank()) {
-      OP_REQUIRES(c, tensor_list->element_shape.dims() >= 1,
-                  errors::InvalidArgument(
-                      "Concat requires elements to be at least vectors, ",
-                      "found scalars instead."));
-      shape_except_first_dim = PartialTensorShape(
-          gtl::ArraySlice<int64>(tensor_list->element_shape.dim_sizes())
-              .subspan(1));
+    if (!element_shape_except_first_dim_.IsFullyDefined()) {
+      if (!tensor_list->element_shape.unknown_rank()) {
+        OP_REQUIRES(c, tensor_list->element_shape.dims() >= 1,
+                    errors::InvalidArgument(
+                        "Concat requires elements to be at least vectors, ",
+                        "found scalars instead."));
+        PartialTensorShape shape_except_first_dim(
+            gtl::ArraySlice<int64>(tensor_list->element_shape.dim_sizes())
+                .subspan(1));
+        PartialTensorShape tmp = element_shape_except_first_dim_;
+        OP_REQUIRES_OK(c, tmp.MergeWith(shape_except_first_dim,
+                                        &element_shape_except_first_dim_));
+      }
     }
     OP_REQUIRES(c,
                 !tensor_list->tensors.empty() ||
-                    shape_except_first_dim.IsFullyDefined(),
+                    element_shape_except_first_dim_.IsFullyDefined(),
                 errors::InvalidArgument(
                     "All except the first dimension must be fully defined ",
                     "when concating an empty tensor list. element_shape: ",
                     tensor_list->element_shape.DebugString()));
     // 1. Compute the shape of the output tensor.
-    // If `shape_except_first_dim` is fully-defined we just prepend the leading
-    // dim to it. Otherwise we use the shape of the first element tensor and
-    // check to make sure shapes of all tensors are compatible.
+    // If `element_shape_except_first_dim_` is fully-defined we just prepend the
+    // leading dim to it. Otherwise we use the shape of the first element tensor
+    // and check to make sure shapes of all tensors are compatible.
     TensorShape output_shape;
-    if (!shape_except_first_dim.AsTensorShape(&output_shape)) {
+    if (!element_shape_except_first_dim_.AsTensorShape(&output_shape)) {
       const Tensor& element_tensor = tensor_list->tensors[0];
       OP_REQUIRES(
           c, TensorShapeUtils::IsVectorOrHigher(element_tensor.shape()),
@@ -268,6 +283,7 @@ class TensorListConcat : public OpKernel {
 
  private:
   DataType element_dtype_;
+  PartialTensorShape element_shape_except_first_dim_;
 };
 
 template <typename Device, typename T>
diff --git a/tensorflow/core/ops/list_ops.cc b/tensorflow/core/ops/list_ops.cc
index 5722170071..cbc9c7a2f4 100644
--- a/tensorflow/core/ops/list_ops.cc
+++ b/tensorflow/core/ops/list_ops.cc
@@ -212,10 +212,16 @@ REGISTER_OP("TensorListConcat")
     .Output("tensor: element_dtype")
     .Output("lengths: int64")
     .Attr("element_dtype: type")
+    .Attr("element_shape: shape = { unknown_rank: true }")
     .SetShapeFn([](shape_inference::InferenceContext* c) {
       DataType element_dtype;
       TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype));
-      shape_inference::ShapeHandle element_shape = c->UnknownShape();
+      PartialTensorShape raw_element_shape;
+      TF_RETURN_IF_ERROR(c->GetAttr("element_shape", &raw_element_shape));
+      shape_inference::ShapeHandle element_shape;
+      TF_RETURN_IF_ERROR(c->MakeShapeFromPartialTensorShape(raw_element_shape,
+                                                            &element_shape));
+
       auto* handle_data = c->input_handle_shapes_and_types(0);
       if (handle_data != nullptr && handle_data->size() != 1) {
         return errors::InvalidArgument(
@@ -231,10 +237,10 @@ REGISTER_OP("TensorListConcat")
               DataTypeString(list_shape_type.dtype), " but expected type ",
               DataTypeString(element_dtype));
         }
-        shape_inference::ShapeHandle ignored;
+        shape_inference::ShapeHandle merged;
         TF_RETURN_IF_ERROR(
-            c->Merge(element_shape, list_shape_type.shape, &ignored));
-        element_shape = list_shape_type.shape;
+            c->Merge(element_shape, list_shape_type.shape, &merged));
+        element_shape = merged;
       }
       if (c->RankKnown(element_shape)) {
         shape_inference::ShapeHandle result;
diff --git a/tensorflow/python/ops/list_ops.py b/tensorflow/python/ops/list_ops.py
index 3eac97655a..7bb9ca89e0 100644
--- a/tensorflow/python/ops/list_ops.py
+++ b/tensorflow/python/ops/list_ops.py
@@ -71,11 +71,13 @@ def tensor_list_from_tensor(tensor, element_shape, name=None):
       name=name)
 
 
-def tensor_list_concat(input_handle, element_dtype, name=None):
+def tensor_list_concat(input_handle, element_dtype, element_shape=None,
+                       name=None):
   # Ignore the lengths output of TensorListConcat. It is only used during
   # gradient computation.
   return gen_list_ops.tensor_list_concat(
-      input_handle=input_handle, element_dtype=element_dtype, name=name)[0]
+      input_handle=input_handle, element_dtype=element_dtype,
+      element_shape=element_shape, name=name)[0]
 
 
 def tensor_list_split(tensor, element_shape, lengths, name=None):
diff --git a/tensorflow/python/ops/parallel_for/control_flow_ops_test.py b/tensorflow/python/ops/parallel_for/control_flow_ops_test.py
index 8acb0d839c..933bddd8cc 100644
--- a/tensorflow/python/ops/parallel_for/control_flow_ops_test.py
+++ b/tensorflow/python/ops/parallel_for/control_flow_ops_test.py
@@ -27,7 +27,6 @@ from tensorflow.core.example import example_pb2
 from tensorflow.core.example import feature_pb2
 from tensorflow.python.client import session
 from tensorflow.python.eager import backprop
-from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -105,10 +104,6 @@ class PForTest(test.TestCase):
     flags.FLAGS.op_conversion_fallback_to_while_loop = False
 
   def test_parallel_iterations(self):
-    # TODO(b/121334512): Remove this check once this passes in Eager mode.
-    if context.executing_eagerly():
-      return
-
     for parallel_iterations in [2, 3, 8, 10]:
       x = random_ops.random_uniform([8, 3])
 
diff --git a/tensorflow/python/ops/tensor_array_ops.py b/tensorflow/python/ops/tensor_array_ops.py
index 9b8bd2c8d3..90a8b0af46 100644
--- a/tensorflow/python/ops/tensor_array_ops.py
+++ b/tensorflow/python/ops/tensor_array_ops.py
@@ -588,10 +588,16 @@ class _GraphTensorArrayV2(object):
 
   def concat(self, name=None):
     """See TensorArray."""
-    value = list_ops.tensor_list_concat(
-        input_handle=self._flow, element_dtype=self._dtype, name=name)
     if self._element_shape and self._element_shape[0].dims is not None:
-      value.set_shape([None] + self._element_shape[0].dims[1:])
+      element_shape = [None] + self._element_shape[0].dims[1:]
+    else:
+      element_shape = None
+
+    value = list_ops.tensor_list_concat(
+        input_handle=self._flow,
+        element_dtype=self._dtype,
+        element_shape=element_shape,
+        name=name)
     return value
 
   @tf_should_use.should_use_result
-- 
GitLab


From 000dc6e0face09fd2a2886b611e6d1d3d7776e5d Mon Sep 17 00:00:00 2001
From: Jian Li <jianlijianli@google.com>
Date: Thu, 3 Jan 2019 14:36:59 -0800
Subject: [PATCH 0190/1095] Update svdf test case.

PiperOrigin-RevId: 227752127
---
 tensorflow/lite/kernels/svdf_test.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/kernels/svdf_test.cc b/tensorflow/lite/kernels/svdf_test.cc
index fc2bb49223..c420260bf5 100644
--- a/tensorflow/lite/kernels/svdf_test.cc
+++ b/tensorflow/lite/kernels/svdf_test.cc
@@ -209,7 +209,7 @@ class HybridSVDFOpModel : public BaseSVDFOpModel {
     tensor_type_ = tensor_type;
   }
 
-  void SetWeights(int weights_idx, std::vector<float> f) {
+  void SetWeights(int weights_idx, const std::vector<float>& f) {
     if (tensor_type_ == TensorType_UINT8) {
       SymmetricQuantizeAndPopulate(weights_idx, f);
     } else {
-- 
GitLab


From 7d57d32e440bc4f1654fb217fd701531a82c3587 Mon Sep 17 00:00:00 2001
From: Tim Shen <timshen@google.com>
Date: Thu, 3 Jan 2019 14:45:53 -0800
Subject: [PATCH 0191/1095] Fix dso_loader.cc's includes.

PiperOrigin-RevId: 227753675
---
 tensorflow/contrib/mpi_collectives/BUILD                  | 2 +-
 tensorflow/stream_executor/platform/default/dso_loader.cc | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/mpi_collectives/BUILD b/tensorflow/contrib/mpi_collectives/BUILD
index ecac06354d..d943ae6880 100644
--- a/tensorflow/contrib/mpi_collectives/BUILD
+++ b/tensorflow/contrib/mpi_collectives/BUILD
@@ -52,7 +52,7 @@ tf_custom_op_library(
     deps = [
         ":mpi_defines",
         ":mpi_message_proto_cc",
-        "//tensorflow/stream_executor:stream_executor_headers_lib",
+        "//tensorflow/core:stream_executor_headers_lib",
         "//third_party/mpi",
     ],
 )
diff --git a/tensorflow/stream_executor/platform/default/dso_loader.cc b/tensorflow/stream_executor/platform/default/dso_loader.cc
index 0f0bce3253..668eeee3f3 100644
--- a/tensorflow/stream_executor/platform/default/dso_loader.cc
+++ b/tensorflow/stream_executor/platform/default/dso_loader.cc
@@ -28,7 +28,7 @@ limitations under the License.
 #include "tensorflow/stream_executor/lib/path.h"
 #include "tensorflow/stream_executor/lib/str_util.h"
 #include "tensorflow/stream_executor/lib/stringprintf.h"
-#include "tensorflow/stream_executor/platform/dso_loader.h"
+#include "tensorflow/stream_executor/platform/default/dso_loader.h"
 #include "tensorflow/stream_executor/platform/logging.h"
 #include "tensorflow/stream_executor/platform/port.h"
 
-- 
GitLab


From 1051c377051b2ee24a495318737358d9ccf7280f Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Wed, 2 Jan 2019 14:34:42 -0800
Subject: [PATCH 0192/1095] Copy ben's changes

---
 .../contrib/tensorrt/convert/convert_nodes.cc |  25 ++-
 .../contrib/tensorrt/test/conv2d_test.py      | 172 ++++++++++++++++++
 2 files changed, 194 insertions(+), 3 deletions(-)
 create mode 100644 tensorflow/contrib/tensorrt/test/conv2d_test.py

diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 2a402fe699..84d7f1aee5 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -62,14 +62,14 @@ limitations under the License.
 
 #define TFTRT_RETURN_ERROR_IF_FALSE(status, node) \
   do {                                            \
-    if (status == false) {                        \
+    if ((status) == false) {                        \
       TFTRT_INTERNAL_ERROR_AT_NODE(node);         \
     }                                             \
   } while (0)
 
 #define TFTRT_RETURN_ERROR_IF_NULLPTR(ptr, node) \
   do {                                           \
-    if (ptr == nullptr) {                        \
+    if ((ptr) == nullptr) {                        \
       TFTRT_INTERNAL_ERROR_AT_NODE(node);        \
     }                                            \
   } while (0)
@@ -1577,12 +1577,14 @@ tensorflow::Status ConvertConv2DHelper(OpConverterParams* params, int group) {
   const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
   TFAttrs attrs(node_def);
 
+  int c_index = 1;
   int h_index = 2;
   int w_index = 3;
   auto data_format = attrs.get<string>("data_format");
   if (data_format == "NHWC") {
     TF_RETURN_IF_ERROR(params->converter->TransposeTensor(
         const_cast<nvinfer1::ITensor*>(tensor), {0, 3, 1, 2}, &tensor));
+    c_index = 3;
     h_index = 1;
     w_index = 2;
     // TODO(jie): transpose it
@@ -1618,14 +1620,30 @@ tensorflow::Status ConvertConv2DHelper(OpConverterParams* params, int group) {
           << tf_stride[3];
   const nvinfer1::DimsHW stride(tf_stride[h_index], tf_stride[w_index]);
 
+  auto tf_dilations = attrs.get<std::vector<int>>("dilations");
+  if ((int)tf_dilations.size() != 4) {
+    return tensorflow::errors::InvalidArgument(
+        "Convolution dilations field must specify 4 dimensions " +
+        node_def.name());
+  }
+  if (tf_dilations[0] != 1 || tf_dilations[c_index] != 1) {
+    return tensorflow::errors::Unimplemented(
+        "Dilation rate must be 1 for batch and channel dimensions, at ",
+        node_def.name());
+  }
+  nvinfer1::DimsHW dilation(tf_dilations[h_index], tf_dilations[w_index]);
+
   std::vector<std::pair<int, int>> padding;
   // TODO(jie): padding.
   if (attrs.get<string>("padding") == "SAME") {
     // This is NCHW tensor with no batch dimension.
     //  1 -> h
     //  2 -> w
+    nvinfer1::DimsHW effective_kernel_size = kernel_size;
+    effective_kernel_size.h() += (kernel_size.h() - 1) * (dilation.h() - 1);
+    effective_kernel_size.w() += (kernel_size.w() - 1) * (dilation.w() - 1);
     padding = CreateSamePadding(
-        stride, kernel_size,
+        stride, effective_kernel_size,
         {static_cast<int>(tensor_dim.d[1]), static_cast<int>(tensor_dim.d[2])});
   } else {
     padding = {{0, 0}, {0, 0}};
@@ -1659,6 +1677,7 @@ tensorflow::Status ConvertConv2DHelper(OpConverterParams* params, int group) {
   layer->setPadding({padding[0].first, padding[1].first});
   layer->setName(node_def.name().c_str());
   layer->setNbGroups(num_groups);
+  layer->setDilation(dilation);
   const nvinfer1::ITensor* output_tensor = layer->getOutput(0);
   VLOG(2) << "TENSOR out: " << DebugString(output_tensor->getDimensions());
   VLOG(2) << "data_format: " << data_format;
diff --git a/tensorflow/contrib/tensorrt/test/conv2d_test.py b/tensorflow/contrib/tensorrt/test/conv2d_test.py
new file mode 100644
index 0000000000..bcf20fc504
--- /dev/null
+++ b/tensorflow/contrib/tensorrt/test/conv2d_test.py
@@ -0,0 +1,172 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Model script to test TF-TensorRT integration."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import gen_array_ops
+from tensorflow.python.ops import gen_nn_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import nn_ops
+from tensorflow.python.platform import test
+
+
+def conv2d_layer(inputs, filters, kernel_size, strides=(1, 1), padding='valid',
+                 data_format='channels_last', dilation_rate=(1, 1), name=None):
+  dtype = inputs.dtype
+  c_axis = -1 if data_format == 'channels_last' else 1
+  nchan = inputs.shape[c_axis]
+  weights_shape = (kernel_size[0], kernel_size[1], nchan, filters)
+  weights = constant_op.constant(np.random.randn(*weights_shape), dtype=dtype)
+  padding = padding.upper()
+  if data_format == 'channels_last':
+    strides = [1] + list(strides) + [1]
+    dilations = [1] + list(dilation_rate) + [1]
+    data_format = 'NHWC'
+  else:
+    strides = [1, 1] + list(strides)
+    dilations = [1, 1] + list(dilation_rate)
+    data_format = 'NCHW'
+  return gen_nn_ops.conv2d(inputs, weights, strides=strides, padding=padding,
+                           dilations=dilations, data_format=data_format)
+
+def div_round_up(n, d):
+  return (n - 1) // d + 1
+
+class Conv2DNCHWTest(trt_test.TfTrtIntegrationTestBase):
+
+  def GetParams(self):
+    """Testing conversion of Conv2D (data_format=NCHW) in TF-TRT conversion."""
+    np.random.seed(1234)
+    dtype = dtypes.float32
+    input_name = "input"
+    n, c, h, w = 13, 3, 7, 11
+    num_filters = 5
+    input_dims = [n, c, h, w]
+    output_name = "output"
+    g = ops.Graph()
+    with g.as_default():
+      inp = array_ops.placeholder(
+          dtype=dtype, shape=[None] + input_dims[1:], name=input_name)
+      with g.device("/GPU:0"):
+        results = []
+        for kernel_size in [(3, 3), (3, 2)]:
+          for dilation_rate in [(1, 1), (2, 3)]:
+            result = conv2d_layer(inp, num_filters, kernel_size,
+                                  dilation_rate=dilation_rate, padding='same',
+                                  data_format='channels_first')
+            results.append(result)
+        output = sum(results)
+        output = array_ops.identity(output, name=output_name)
+    return trt_test.TfTrtIntegrationTestParams(
+        gdef=g.as_graph_def(),
+        input_names=[input_name],
+        input_dims=[input_dims],
+        output_names=[output_name],
+        expected_output_dims=[(n, num_filters, h, w)])
+
+  def ExpectedEnginesToBuild(self, run_params):
+    """Return the expected engines to build."""
+    return ["my_trt_op_0"]
+
+
+class Conv2DStridedNCHWTest(trt_test.TfTrtIntegrationTestBase):
+
+  def GetParams(self):
+    """Testing conversion of strided Conv2D (data_format=NCHW) in TF-TRT
+    conversion."""
+    np.random.seed(1234)
+    dtype = dtypes.float32
+    input_name = "input"
+    n, c, h, w = 13, 3, 7, 11
+    num_filters = 5
+    input_dims = [n, c, h, w]
+    output_name = "output"
+    g = ops.Graph()
+    with g.as_default():
+      inp = array_ops.placeholder(
+          dtype=dtype, shape=[None] + input_dims[1:], name=input_name)
+      with g.device("/GPU:0"):
+        output = inp
+        output = conv2d_layer(output, num_filters, (3, 2), strides=(2, 2),
+                              padding='same', data_format='channels_first')
+        h = div_round_up(h, 2)
+        w = div_round_up(w, 2)
+        output = conv2d_layer(output, num_filters, (3, 3), strides=(2, 2),
+                              dilation_rate=(2, 3), padding='same',
+                              data_format='channels_first')
+        h = div_round_up(h, 2)
+        w = div_round_up(w, 2)
+        output = array_ops.identity(output, name=output_name)
+    return trt_test.TfTrtIntegrationTestParams(
+        gdef=g.as_graph_def(),
+        input_names=[input_name],
+        input_dims=[input_dims],
+        output_names=[output_name],
+        expected_output_dims=[(n, num_filters, h, w)])
+
+  def ExpectedEnginesToBuild(self, run_params):
+    """Return the expected engines to build."""
+    return ["my_trt_op_0"]
+
+
+class Conv2DNHWCTest(trt_test.TfTrtIntegrationTestBase):
+
+  def GetParams(self):
+    """Testing conversion of Conv2D (data_format=NHWC) in TF-TRT conversion."""
+    np.random.seed(1234)
+    dtype = dtypes.float32
+    input_name = "input"
+    n, h, w, c = 13, 7, 11, 3
+    num_filters = 5
+    input_dims = [n, h, w, c]
+    output_name = "output"
+    g = ops.Graph()
+    with g.as_default():
+      inp = array_ops.placeholder(
+          dtype=dtype, shape=[None] + input_dims[1:], name=input_name)
+      with g.device("/GPU:0"):
+        results = []
+        for kernel_size in [(3, 3), (3, 2)]:
+          for dilation_rate in [(1, 1), (2, 3)]:
+            result = conv2d_layer(inp, num_filters, kernel_size,
+                                  dilation_rate=dilation_rate, padding='same',
+                                  data_format='channels_last')
+            results.append(result)
+        output = sum(results)
+        output = array_ops.identity(output, name=output_name)
+    return trt_test.TfTrtIntegrationTestParams(
+        gdef=g.as_graph_def(),
+        input_names=[input_name],
+        input_dims=[input_dims],
+        output_names=[output_name],
+        expected_output_dims=[(n, h, w, num_filters)])
+
+  def ExpectedEnginesToBuild(self, run_params):
+    """Return the expected engines to build."""
+    return ["my_trt_op_0"]
+
+
+if __name__ == "__main__":
+  test.main()
-- 
GitLab


From db95d28671c790de369a84c891f9bc971663e0e4 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Wed, 2 Jan 2019 15:12:40 -0800
Subject: [PATCH 0193/1095] Tidy up Conv2D converter

---
 tensorflow/contrib/tensorrt/BUILD             |  1 +
 .../contrib/tensorrt/convert/convert_nodes.cc | 94 ++++++++-----------
 .../contrib/tensorrt/test/conv2d_test.py      |  6 +-
 3 files changed, 43 insertions(+), 58 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index 089c9ad97c..c17dea02f1 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -491,6 +491,7 @@ cuda_py_tests(
         "test/binary_tensor_weight_broadcast_test.py",
         "test/concatenation_test.py",
         "test/const_broadcast_test.py",
+        "test/conv2d_test.py",
         "test/identity_output_test.py",
         "test/manual_test.py",
         "test/memory_alignment_test.py",
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 84d7f1aee5..614cc47130 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -62,14 +62,14 @@ limitations under the License.
 
 #define TFTRT_RETURN_ERROR_IF_FALSE(status, node) \
   do {                                            \
-    if ((status) == false) {                        \
+    if ((status) == false) {                      \
       TFTRT_INTERNAL_ERROR_AT_NODE(node);         \
     }                                             \
   } while (0)
 
 #define TFTRT_RETURN_ERROR_IF_NULLPTR(ptr, node) \
   do {                                           \
-    if ((ptr) == nullptr) {                        \
+    if ((ptr) == nullptr) {                      \
       TFTRT_INTERNAL_ERROR_AT_NODE(node);        \
     }                                            \
   } while (0)
@@ -1567,41 +1567,56 @@ tensorflow::Status ConvertConv2DHelper(OpConverterParams* params, int group) {
                                              node_def.name());
   }
   TRT_ShapedWeights weights_rsck = inputs.at(1).weights();
-  VLOG(2) << "weight shape: " << weights_rsck.DebugString();
   if (weights_rsck.shape_.nbDims != 4) {
     return tensorflow::errors::Internal(
         "Conv2D expects kernel of dimension 4, at: " + node_def.name());
   }
+  TFAttrs attrs(node_def);
+  auto data_format = attrs.get<string>("data_format");
+  int c_index = (data_format == "NHWC") ? 3 : 1;
+  int h_index = (data_format == "NHWC") ? 1 : 2;
+  int w_index = (data_format == "NHWC") ? 2 : 3;
+  auto tf_dilations = attrs.get<std::vector<int>>("dilations");
+  if (tf_dilations.size() != 4) {
+    return tensorflow::errors::InvalidArgument(
+        "Convolution dilations field must specify 4 dimensions, at ",
+        node_def.name());
+  }
+  if (tf_dilations[0] != 1 || tf_dilations[c_index] != 1) {
+    return tensorflow::errors::Unimplemented(
+        "Dilation rate must be 1 for batch and channel dimensions, at ",
+        node_def.name());
+  }
+  nvinfer1::DimsHW dilation(tf_dilations[h_index], tf_dilations[w_index]);
+  const auto tf_stride = attrs.get<std::vector<int>>("strides");
+  if (tf_stride[0] != 1 || tf_stride[c_index] != 1) {
+    return tensorflow::errors::Unimplemented(
+        "Stride must be 1 for batch and channel dimensions, at ",
+        node_def.name());
+  }
+  const nvinfer1::DimsHW stride(tf_stride[h_index], tf_stride[w_index]);
   if (params->validation_only) return tensorflow::Status::OK();
 
   const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
-  TFAttrs attrs(node_def);
 
-  int c_index = 1;
-  int h_index = 2;
-  int w_index = 3;
-  auto data_format = attrs.get<string>("data_format");
-  if (data_format == "NHWC") {
+  // Transpose to NCHW (NCHW is required for IConvLayer).
+  const bool need_transpose = (data_format == "NHWC");
+  if (need_transpose) {
     TF_RETURN_IF_ERROR(params->converter->TransposeTensor(
         const_cast<nvinfer1::ITensor*>(tensor), {0, 3, 1, 2}, &tensor));
-    c_index = 3;
-    h_index = 1;
-    w_index = 2;
-    // TODO(jie): transpose it
   }
-
-  // tensor after transpose (NCHW)
+  // Dimensions of transposed tensor.
   const auto tensor_dim = tensor->getDimensions();
 
+  // This is a depthwise convolution when num_groups is 0. Otherwise, num_groups
+  // will be 1.
   int num_groups = group;
-  if (num_groups == 0) num_groups = tensor_dim.d[0];  // depthwise convolution
-  VLOG(2) << "groups count: " << num_groups;
+  if (num_groups == 0) num_groups = tensor_dim.d[0];
 
   if (params->converter->precision_mode() == FP16MODE) {
     weights_rsck =
         ConvertFP32ToFP16(params->weight_store, inputs.at(1).weights());
   }
-
   TRT_ShapedWeights weights =
       params->weight_store->GetTempWeights(weights_rsck);
   ReorderRSCKToKCRS(weights_rsck, &weights, num_groups);
@@ -1610,35 +1625,10 @@ tensorflow::Status ConvertConv2DHelper(OpConverterParams* params, int group) {
   nvinfer1::DimsHW kernel_size;
   kernel_size.h() = weights.shape_.d[2];
   kernel_size.w() = weights.shape_.d[3];
-  VLOG(2) << "RSCK: " << weights.DebugString();
-  VLOG(2) << "kernel size: " << kernel_size.h() << ", " << kernel_size.w();
-
-  // TODO(jie): stride. (NHWC/NCHW)
-  const auto tf_stride = attrs.get<std::vector<int>>("strides");
-  VLOG(2) << "h_INDEX" << h_index << ", w_index " << w_index;
-  VLOG(2) << "stride: " << tf_stride[0] << tf_stride[1] << tf_stride[2]
-          << tf_stride[3];
-  const nvinfer1::DimsHW stride(tf_stride[h_index], tf_stride[w_index]);
-
-  auto tf_dilations = attrs.get<std::vector<int>>("dilations");
-  if ((int)tf_dilations.size() != 4) {
-    return tensorflow::errors::InvalidArgument(
-        "Convolution dilations field must specify 4 dimensions " +
-        node_def.name());
-  }
-  if (tf_dilations[0] != 1 || tf_dilations[c_index] != 1) {
-    return tensorflow::errors::Unimplemented(
-        "Dilation rate must be 1 for batch and channel dimensions, at ",
-        node_def.name());
-  }
-  nvinfer1::DimsHW dilation(tf_dilations[h_index], tf_dilations[w_index]);
 
+  // Add padding.
   std::vector<std::pair<int, int>> padding;
-  // TODO(jie): padding.
   if (attrs.get<string>("padding") == "SAME") {
-    // This is NCHW tensor with no batch dimension.
-    //  1 -> h
-    //  2 -> w
     nvinfer1::DimsHW effective_kernel_size = kernel_size;
     effective_kernel_size.h() += (kernel_size.h() - 1) * (dilation.h() - 1);
     effective_kernel_size.w() += (kernel_size.w() - 1) * (dilation.w() - 1);
@@ -1648,13 +1638,9 @@ tensorflow::Status ConvertConv2DHelper(OpConverterParams* params, int group) {
   } else {
     padding = {{0, 0}, {0, 0}};
   }
-
   if (padding[0].first != padding[0].second ||
       padding[1].first != padding[1].second) {
-    // TODO(jie): handle asymmetric padding
-    VLOG(2) << "Padding!!!: " << padding[0].first << padding[0].second
-            << padding[1].first << padding[1].second;
-    VLOG(2) << "TENSOR before: " << DebugString(tensor->getDimensions());
+    // Handle asymmetric padding.
     auto pad_layer = params->converter->network()->addPadding(
         *const_cast<nvinfer1::ITensor*>(tensor),
         nvinfer1::DimsHW(padding[0].first, padding[1].first),
@@ -1664,25 +1650,23 @@ tensorflow::Status ConvertConv2DHelper(OpConverterParams* params, int group) {
         const_cast<nvinfer1::ITensor*>(tensor), pad_layer->getOutput(0));
     padding = {{0, 0}, {0, 0}};
     tensor = pad_layer->getOutput(0);
-    VLOG(2) << "TENSOR after: " << DebugString(tensor->getDimensions());
   }
 
+  // Add convolution.
   nvinfer1::IConvolutionLayer* layer =
       params->converter->network()->addConvolution(
           *const_cast<nvinfer1::ITensor*>(tensor), noutput, kernel_size,
           weights.GetTrtWeights(), biases.GetTrtWeights());
   TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name());
-
   layer->setStride(stride);
   layer->setPadding({padding[0].first, padding[1].first});
   layer->setName(node_def.name().c_str());
   layer->setNbGroups(num_groups);
   layer->setDilation(dilation);
   const nvinfer1::ITensor* output_tensor = layer->getOutput(0);
-  VLOG(2) << "TENSOR out: " << DebugString(output_tensor->getDimensions());
-  VLOG(2) << "data_format: " << data_format;
-  if (data_format == "NHWC") {
-    // TODO(jie): transpose it back!
+
+  // Restore transpose.
+  if (need_transpose) {
     TF_RETURN_IF_ERROR(params->converter->TransposeTensor(
         const_cast<nvinfer1::ITensor*>(output_tensor), {0, 2, 3, 1},
         &output_tensor));
diff --git a/tensorflow/contrib/tensorrt/test/conv2d_test.py b/tensorflow/contrib/tensorrt/test/conv2d_test.py
index bcf20fc504..c5228dea33 100644
--- a/tensorflow/contrib/tensorrt/test/conv2d_test.py
+++ b/tensorflow/contrib/tensorrt/test/conv2d_test.py
@@ -88,7 +88,7 @@ class Conv2DNCHWTest(trt_test.TfTrtIntegrationTestBase):
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
-    return ["my_trt_op_0"]
+    return ["TRTEngineOp_0"]
 
 
 class Conv2DStridedNCHWTest(trt_test.TfTrtIntegrationTestBase):
@@ -128,7 +128,7 @@ class Conv2DStridedNCHWTest(trt_test.TfTrtIntegrationTestBase):
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
-    return ["my_trt_op_0"]
+    return ["TRTEngineOp_0"]
 
 
 class Conv2DNHWCTest(trt_test.TfTrtIntegrationTestBase):
@@ -165,7 +165,7 @@ class Conv2DNHWCTest(trt_test.TfTrtIntegrationTestBase):
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
-    return ["my_trt_op_0"]
+    return ["TRTEngineOp_0"]
 
 
 if __name__ == "__main__":
-- 
GitLab


From a42746772faea0ab58d421dc68220ebc98c053bc Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Wed, 2 Jan 2019 16:13:58 -0800
Subject: [PATCH 0194/1095] Add unit test for Conv2D validator

---
 .../contrib/tensorrt/convert/convert_nodes.cc |  23 ++--
 .../tensorrt/convert/convert_nodes_test.cc    | 102 ++++++++++++++++++
 2 files changed, 118 insertions(+), 7 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 614cc47130..34ba2aeaa4 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -1556,6 +1556,11 @@ enum class ConvolutionType { DEFAULT, DEPTHWISE_CONV };
 tensorflow::Status ConvertConv2DHelper(OpConverterParams* params, int group) {
   const auto& inputs = params->inputs;
   const auto& node_def = params->node_def;
+  if (inputs.size() != 2) {
+    return tensorflow::errors::InvalidArgument("Two inputs are expected for ",
+                                               node_def.op(), ", at ",
+                                               node_def.name());
+  }
   if (inputs.at(0).is_weights()) {
     return tensorflow::errors::Unimplemented(
         node_def.op(), " is only implemented for tensors, not weights, at ",
@@ -1568,8 +1573,8 @@ tensorflow::Status ConvertConv2DHelper(OpConverterParams* params, int group) {
   }
   TRT_ShapedWeights weights_rsck = inputs.at(1).weights();
   if (weights_rsck.shape_.nbDims != 4) {
-    return tensorflow::errors::Internal(
-        "Conv2D expects kernel of dimension 4, at: " + node_def.name());
+    return tensorflow::errors::InvalidArgument(
+        "Conv2D expects kernel of dimension 4, at " + node_def.name());
   }
   TFAttrs attrs(node_def);
   auto data_format = attrs.get<string>("data_format");
@@ -1587,8 +1592,13 @@ tensorflow::Status ConvertConv2DHelper(OpConverterParams* params, int group) {
         "Dilation rate must be 1 for batch and channel dimensions, at ",
         node_def.name());
   }
-  nvinfer1::DimsHW dilation(tf_dilations[h_index], tf_dilations[w_index]);
+  const nvinfer1::DimsHW dilation(tf_dilations[h_index], tf_dilations[w_index]);
   const auto tf_stride = attrs.get<std::vector<int>>("strides");
+  if (tf_stride.size() != 4) {
+    return tensorflow::errors::InvalidArgument(
+        "Convolution strides field must specify 4 dimensions, at ",
+        node_def.name());
+  }
   if (tf_stride[0] != 1 || tf_stride[c_index] != 1) {
     return tensorflow::errors::Unimplemented(
         "Stride must be 1 for batch and channel dimensions, at ",
@@ -1608,10 +1618,9 @@ tensorflow::Status ConvertConv2DHelper(OpConverterParams* params, int group) {
   // Dimensions of transposed tensor.
   const auto tensor_dim = tensor->getDimensions();
 
-  // This is a depthwise convolution when num_groups is 0. Otherwise, num_groups
-  // will be 1.
-  int num_groups = group;
-  if (num_groups == 0) num_groups = tensor_dim.d[0];
+  // For depthwise convolution, group will be 0 so set num_groups to size of
+  // input's channel dim. For a non-depthwise conv, num_groups will be 1.
+  const int num_groups = (group == 0) ? tensor_dim.d[0] : group;
 
   if (params->converter->precision_mode() == FP16MODE) {
     weights_rsck =
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
index a2ddfbffa5..cb73d2e07b 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
@@ -2378,6 +2378,8 @@ TEST_F(OpConverterTest, ConvertStridedSlice) {
   };
 
   {
+    // Input is weights, should fail.
+    Reset();
     NodeDef node_def = get_strided_slice_nodedef();
     AddTestWeights<int32>("input", {1, 2, 3}, {1, 2, 3, 4, 5, 6});
     AddTestWeights<int32>("begin", {4}, {0, 0, 0, 0});
@@ -2619,6 +2621,106 @@ TEST_F(OpConverterTest, ConvertStridedSlice) {
   }
 }
 
+TEST_F(OpConverterTest, ConvertConv2D) {
+  {
+    // Input list is empty, should fail.
+    NodeDef node_def = MakeNodeDef("my_conv2d", "Conv2D", {});
+    RunValidationAndConversion(
+        node_def, error::INVALID_ARGUMENT,
+        "Two inputs are expected for Conv2D, at my_conv2d");
+  }
+
+  // Get nodedef for Conv2D layer.
+  auto get_conv2d_nodedef = [](
+      std::vector<int> strides = {1, 1, 1, 1}, string padding = "SAME",
+      string data_format = "NCHW",
+      std::vector<int> dilations = {1, 1, 1, 1}) -> NodeDef {
+    Scope s = Scope::NewRootScope();
+    auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT);
+    auto filter = ops::Placeholder(s.WithOpName("weights"), DT_FLOAT);
+    ops::Conv2D::Attrs attrs =
+        ops::Conv2D::Attrs().DataFormat(data_format).Dilations(dilations);
+    auto conv2d = ops::Conv2D(s.WithOpName("my_conv2d"), input, filter, strides,
+                              padding, attrs);
+    return conv2d.operation.node()->def();
+  };
+
+  {
+    // Input is weights, should fail.
+    Reset();
+    NodeDef node_def = get_conv2d_nodedef();
+    AddTestWeights<float>("input", {1, 2, 3}, {1, 2, 3, 4, 5, 6});
+    AddTestWeights<float>("weights", {3, 3, 1, 1}, {1, 2, 3, 4, 5, 6, 7, 8, 9});
+    RunValidationAndConversion(
+        node_def, error::UNIMPLEMENTED,
+        "Conv2D is only implemented for tensors, not weights, at my_conv2d");
+  }
+  {
+    // Filter is tensor, should fail.
+    Reset();
+    NodeDef node_def = get_conv2d_nodedef();
+    AddTestTensor("input", {1, 2, 3});
+    AddTestTensor("weights", {3, 3, 1, 1});
+    RunValidationAndConversion(
+        node_def, error::UNIMPLEMENTED,
+        "Kernel for Conv2D must be constant weights, at my_conv2d");
+  }
+  {
+    // Filter is not 4D, should fail.
+    Reset();
+    NodeDef node_def = get_conv2d_nodedef();
+    AddTestTensor("input", {1, 2, 3});
+    AddTestWeights<float>("weights", {3, 3, 1}, {1, 2, 3, 4, 5, 6, 7, 8, 9});
+    RunValidationAndConversion(
+        node_def, error::INVALID_ARGUMENT,
+        "Conv2D expects kernel of dimension 4, at my_conv2d");
+  }
+  {
+    // Dilations is not 4D, should fail.
+    Reset();
+    NodeDef node_def =
+        get_conv2d_nodedef({1, 1, 1, 1}, "SAME", "NCHW", {1, 1, 1});
+    AddTestTensor("input", {1, 2, 3});
+    AddTestWeights<float>("weights", {3, 3, 1, 1}, {1, 2, 3, 4, 5, 6, 7, 8, 9});
+    RunValidationAndConversion(
+        node_def, error::INVALID_ARGUMENT,
+        "Convolution dilations field must specify 4 dimensions, at my_conv2d");
+  }
+  {
+    // Dilation value is not 1 for channel, should fail.
+    Reset();
+    NodeDef node_def =
+        get_conv2d_nodedef({1, 1, 1, 1}, "SAME", "NCHW", {1, 2, 1, 1});
+    AddTestTensor("input", {1, 2, 3});
+    AddTestWeights<float>("weights", {3, 3, 1, 1}, {1, 2, 3, 4, 5, 6, 7, 8, 9});
+    RunValidationAndConversion(node_def, error::UNIMPLEMENTED,
+                               "Dilation rate must be 1 for batch and channel "
+                               "dimensions, at my_conv2d");
+  }
+  {
+    // Strides is not 4D, should fail.
+    Reset();
+    NodeDef node_def =
+        get_conv2d_nodedef({1, 1, 1}, "SAME", "NCHW", {1, 1, 1, 1});
+    AddTestTensor("input", {1, 2, 3});
+    AddTestWeights<float>("weights", {3, 3, 1, 1}, {1, 2, 3, 4, 5, 6, 7, 8, 9});
+    RunValidationAndConversion(
+        node_def, error::INVALID_ARGUMENT,
+        "Convolution strides field must specify 4 dimensions, at my_conv2d");
+  }
+  {
+    // Stride value is not 1 for channel, should fail.
+    Reset();
+    NodeDef node_def =
+        get_conv2d_nodedef({1, 2, 1, 1}, "SAME", "NCHW", {1, 1, 1, 1});
+    AddTestTensor("input", {1, 2, 3});
+    AddTestWeights<float>("weights", {3, 3, 1, 1}, {1, 2, 3, 4, 5, 6, 7, 8, 9});
+    RunValidationAndConversion(
+        node_def, error::UNIMPLEMENTED,
+        "Stride must be 1 for batch and channel dimensions, at my_conv2d");
+  }
+}
+
 }  // namespace convert
 }  // namespace tensorrt
 }  // namespace tensorflow
-- 
GitLab


From 560e4dd67fa8f9b0b99b940e35d039cf6c40194a Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Wed, 2 Jan 2019 16:20:44 -0800
Subject: [PATCH 0195/1095] Remove unused imports

---
 tensorflow/contrib/tensorrt/test/conv2d_test.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/test/conv2d_test.py b/tensorflow/contrib/tensorrt/test/conv2d_test.py
index c5228dea33..f7d9ca6826 100644
--- a/tensorflow/contrib/tensorrt/test/conv2d_test.py
+++ b/tensorflow/contrib/tensorrt/test/conv2d_test.py
@@ -25,10 +25,7 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import gen_array_ops
 from tensorflow.python.ops import gen_nn_ops
-from tensorflow.python.ops import math_ops
-from tensorflow.python.ops import nn_ops
 from tensorflow.python.platform import test
 
 
-- 
GitLab


From cf471f0c15b0c9c78c44a238fe60fe1e31289503 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Thu, 3 Jan 2019 14:56:44 -0800
Subject: [PATCH 0196/1095] Add positive test cases for Conv2D. Reuse
 build_graph in conv2d_test.

---
 .../contrib/tensorrt/convert/convert_nodes.cc |   5 +-
 .../tensorrt/convert/convert_nodes_test.cc    | 134 ++++++++++++++++++
 .../contrib/tensorrt/test/conv2d_test.py      | 106 ++++++--------
 3 files changed, 183 insertions(+), 62 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 34ba2aeaa4..179d631a3e 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -62,14 +62,14 @@ limitations under the License.
 
 #define TFTRT_RETURN_ERROR_IF_FALSE(status, node) \
   do {                                            \
-    if ((status) == false) {                      \
+    if (status == false) {                        \
       TFTRT_INTERNAL_ERROR_AT_NODE(node);         \
     }                                             \
   } while (0)
 
 #define TFTRT_RETURN_ERROR_IF_NULLPTR(ptr, node) \
   do {                                           \
-    if ((ptr) == nullptr) {                      \
+    if (ptr == nullptr) {                        \
       TFTRT_INTERNAL_ERROR_AT_NODE(node);        \
     }                                            \
   } while (0)
@@ -1593,6 +1593,7 @@ tensorflow::Status ConvertConv2DHelper(OpConverterParams* params, int group) {
         node_def.name());
   }
   const nvinfer1::DimsHW dilation(tf_dilations[h_index], tf_dilations[w_index]);
+
   const auto tf_stride = attrs.get<std::vector<int>>("strides");
   if (tf_stride.size() != 4) {
     return tensorflow::errors::InvalidArgument(
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
index cb73d2e07b..1e49026a34 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
@@ -2697,6 +2697,17 @@ TEST_F(OpConverterTest, ConvertConv2D) {
                                "Dilation rate must be 1 for batch and channel "
                                "dimensions, at my_conv2d");
   }
+  {
+    // Dilation value is not 1 for channel (NHWC), should fail.
+    Reset();
+    NodeDef node_def =
+        get_conv2d_nodedef({1, 1, 1, 1}, "SAME", "NHWC", {1, 1, 1, 2});
+    AddTestTensor("input", {2, 3, 1});
+    AddTestWeights<float>("weights", {3, 3, 1, 1}, {1, 2, 3, 4, 5, 6, 7, 8, 9});
+    RunValidationAndConversion(node_def, error::UNIMPLEMENTED,
+                               "Dilation rate must be 1 for batch and channel "
+                               "dimensions, at my_conv2d");
+  }
   {
     // Strides is not 4D, should fail.
     Reset();
@@ -2719,6 +2730,129 @@ TEST_F(OpConverterTest, ConvertConv2D) {
         node_def, error::UNIMPLEMENTED,
         "Stride must be 1 for batch and channel dimensions, at my_conv2d");
   }
+
+  struct TestParams {
+    TestParams(const std::vector<int>& input_dims,
+               const std::vector<float>& input,
+               const std::vector<int>& filter_dims,
+               const std::vector<float>& filter,
+               const std::vector<int>& strides, const string& padding,
+               const string& data_format, const std::vector<int>& dilations,
+               const std::vector<int>& expected_output_dims,
+               const std::vector<float>& expected_output)
+        : input_dims(input_dims),
+          input(input),
+          filter_dims(filter_dims),
+          filter(filter),
+          strides(strides),
+          padding(padding),
+          data_format(data_format),
+          dilations(dilations),
+          expected_output_dims(expected_output_dims),
+          expected_output(expected_output) {}
+
+    std::vector<int> input_dims;
+    std::vector<float> input;
+    std::vector<int> filter_dims;
+    std::vector<float> filter;
+    std::vector<int> strides;
+    string padding;
+    string data_format;
+    std::vector<int> dilations;
+    std::vector<int> expected_output_dims;
+    std::vector<float> expected_output;
+  };
+
+  // Ok.
+  const int kConv2DOKCases = 6;
+  TestParams ok_params[kConv2DOKCases] = {
+      // Basic
+      TestParams{/*input_dims=*/{1, 2, 3},
+                 /*input=*/{0, 1, 2, 3, 3, 4},
+                 /*filter_dims=*/{1, 2, 1, 1},
+                 /*filter=*/{-1, 1},
+                 /*strides=*/{1, 1, 1, 1},
+                 /*padding=*/"VALID",
+                 /*data_format=*/"NCHW",
+                 /*dilations=*/{1, 1, 1, 1},
+                 /*expected_output_dims=*/{1, 2, 2},
+                 /*expected_output=*/{1, 1, 0, 1}},
+      // SAME padding (Asymmetric)
+      TestParams{/*input_dims=*/{1, 2, 3},
+                 /*input=*/{0, 1, 2, 3, 3, 4},
+                 /*filter_dims=*/{1, 2, 1, 1},
+                 /*filter=*/{-1, 1},
+                 /*strides=*/{1, 1, 1, 1},
+                 /*padding=*/"SAME",
+                 /*data_format=*/"NCHW",
+                 /*dilations=*/{1, 1, 1, 1},
+                 /*expected_output_dims=*/{1, 2, 3},
+                 /*expected_output=*/{1, 1, -2, 0, 1, -4}},
+      // SAME padding (Symmetric)
+      TestParams{/*input_dims=*/{1, 2, 3},
+                 /*input=*/{0, 1, 2, 3, 3, 4},
+                 /*filter_dims=*/{1, 3, 1, 1},
+                 /*filter=*/{-1, 0, 1},
+                 /*strides=*/{1, 1, 1, 1},
+                 /*padding=*/"SAME",
+                 /*data_format=*/"NCHW",
+                 /*dilations=*/{1, 1, 1, 1},
+                 /*expected_output_dims=*/{1, 2, 3},
+                 /*expected_output=*/{1, 2, -1, 3, 1, -3}},
+      // NHWC
+      TestParams{/*input_dims=*/{2, 3, 1},
+                 /*input=*/{0, 1, 2, 3, 3, 4},
+                 /*filter_dims=*/{1, 2, 1, 1},
+                 /*filter=*/{-1, 1},
+                 /*strides=*/{1, 1, 1, 1},
+                 /*padding=*/"VALID",
+                 /*data_format=*/"NHWC",
+                 /*dilations=*/{1, 1, 1, 1},
+                 /*expected_output_dims=*/{2, 2, 1},
+                 /*expected_output=*/{1, 1, 0, 1}},
+      // Dilated
+      TestParams{/*input_dims=*/{1, 2, 3},
+                 /*input=*/{0, 1, 2, 3, 3, 4},
+                 /*filter_dims=*/{1, 2, 1, 1},
+                 /*filter=*/{-1, 1},
+                 /*strides=*/{1, 1, 1, 1},
+                 /*padding=*/"VALID",
+                 /*data_format=*/"NCHW",
+                 /*dilations=*/{1, 1, 1, 2},
+                 /*expected_output_dims=*/{1, 2, 1},
+                 /*expected_output=*/{2, 1}},
+      // Strided
+      TestParams{/*input_dims=*/{1, 2, 4},
+                 /*input=*/{0, 1, 2, 2, 3, 4, 4, 7},
+                 /*filter_dims=*/{1, 2, 1, 1},
+                 /*filter=*/{-1, 1},
+                 /*strides=*/{1, 1, 1, 2},
+                 /*padding=*/"VALID",
+                 /*data_format=*/"NCHW",
+                 /*dilations=*/{1, 1, 1, 1},
+                 /*expected_output_dims=*/{1, 2, 2},
+                 /*expected_output=*/{1, 0, 1, 3}},
+  };
+
+  for (int i = 0; i < kConv2DOKCases; i++) {
+    Reset();
+    NodeDef node_def =
+        get_conv2d_nodedef(ok_params[i].strides, ok_params[i].padding,
+                           ok_params[i].data_format, ok_params[i].dilations);
+    AddTestTensor("input", ok_params[i].input_dims);
+    AddTestWeights<float>("weights", ok_params[i].filter_dims,
+                          ok_params[i].filter);
+    RunValidationAndConversion(node_def);
+    TRT_TensorOrWeights output;
+    TF_EXPECT_OK(GetTensorOrWeights("my_conv2d", &output));
+    EXPECT_TRUE(output.is_tensor());
+    ExpectTrtDimsEqualsArray(ok_params[i].expected_output_dims,
+                             output.tensor()->getDimensions());
+    std::vector<float> output_data(ok_params[i].expected_output.size());
+    BuildAndRun<float>({{"input", ok_params[i].input}}, "my_conv2d",
+                       &output_data);
+    EXPECT_THAT(output_data, ElementsAreArray(ok_params[i].expected_output));
+  }
 }
 
 }  // namespace convert
diff --git a/tensorflow/contrib/tensorrt/test/conv2d_test.py b/tensorflow/contrib/tensorrt/test/conv2d_test.py
index f7d9ca6826..68b6534823 100644
--- a/tensorflow/contrib/tensorrt/test/conv2d_test.py
+++ b/tensorflow/contrib/tensorrt/test/conv2d_test.py
@@ -51,37 +51,60 @@ def conv2d_layer(inputs, filters, kernel_size, strides=(1, 1), padding='valid',
 def div_round_up(n, d):
   return (n - 1) // d + 1
 
+def build_graph(input_dims, dtype, num_filters, data_format, kernel_sizes,
+                dilation_rates, padding='same'):
+  g = ops.Graph()
+  with g.as_default():
+    inp = array_ops.placeholder(
+        dtype=dtype, shape=[None] + input_dims[1:], name="input")
+    with g.device("/GPU:0"):
+      results = []
+      for kernel_size in kernel_sizes:
+        for dilation_rate in dilation_rates:
+          result = conv2d_layer(inp, num_filters, kernel_size, (1, 1),
+                                padding, data_format, dilation_rate)
+          results.append(result)
+      output = sum(results)
+      output = array_ops.identity(output, name="output")
+  return g
+
+
 class Conv2DNCHWTest(trt_test.TfTrtIntegrationTestBase):
 
   def GetParams(self):
     """Testing conversion of Conv2D (data_format=NCHW) in TF-TRT conversion."""
     np.random.seed(1234)
-    dtype = dtypes.float32
-    input_name = "input"
-    n, c, h, w = 13, 3, 7, 11
-    num_filters = 5
-    input_dims = [n, c, h, w]
-    output_name = "output"
-    g = ops.Graph()
-    with g.as_default():
-      inp = array_ops.placeholder(
-          dtype=dtype, shape=[None] + input_dims[1:], name=input_name)
-      with g.device("/GPU:0"):
-        results = []
-        for kernel_size in [(3, 3), (3, 2)]:
-          for dilation_rate in [(1, 1), (2, 3)]:
-            result = conv2d_layer(inp, num_filters, kernel_size,
-                                  dilation_rate=dilation_rate, padding='same',
-                                  data_format='channels_first')
-            results.append(result)
-        output = sum(results)
-        output = array_ops.identity(output, name=output_name)
+    input_dims = [13, 3, 7, 11]
+    g = build_graph(input_dims=input_dims, dtype=dtypes.float32, num_filters=5,
+                    data_format="channels_first", kernel_sizes=[(3, 3), (3, 2)],
+                    dilation_rates=[(1, 1), (2, 3)])
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
-        input_names=[input_name],
+        input_names=["input"],
         input_dims=[input_dims],
-        output_names=[output_name],
-        expected_output_dims=[(n, num_filters, h, w)])
+        output_names=["output"],
+        expected_output_dims=[(13, 5, 7, 11)])
+
+  def ExpectedEnginesToBuild(self, run_params):
+    """Return the expected engines to build."""
+    return ["TRTEngineOp_0"]
+
+
+class Conv2DNHWCTest(trt_test.TfTrtIntegrationTestBase):
+
+  def GetParams(self):
+    """Testing conversion of Conv2D (data_format=NCHW) in TF-TRT conversion."""
+    np.random.seed(1234)
+    input_dims = [13, 7, 11, 3]
+    g = build_graph(input_dims=input_dims, dtype=dtypes.float32, num_filters=5,
+                    data_format="channels_last", kernel_sizes=[(3, 3), (3, 2)],
+                    dilation_rates=[(1, 1), (2, 3)])
+    return trt_test.TfTrtIntegrationTestParams(
+        gdef=g.as_graph_def(),
+        input_names=["input"],
+        input_dims=[input_dims],
+        output_names=["output"],
+        expected_output_dims=[(13, 7, 11, 5)])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
@@ -128,42 +151,5 @@ class Conv2DStridedNCHWTest(trt_test.TfTrtIntegrationTestBase):
     return ["TRTEngineOp_0"]
 
 
-class Conv2DNHWCTest(trt_test.TfTrtIntegrationTestBase):
-
-  def GetParams(self):
-    """Testing conversion of Conv2D (data_format=NHWC) in TF-TRT conversion."""
-    np.random.seed(1234)
-    dtype = dtypes.float32
-    input_name = "input"
-    n, h, w, c = 13, 7, 11, 3
-    num_filters = 5
-    input_dims = [n, h, w, c]
-    output_name = "output"
-    g = ops.Graph()
-    with g.as_default():
-      inp = array_ops.placeholder(
-          dtype=dtype, shape=[None] + input_dims[1:], name=input_name)
-      with g.device("/GPU:0"):
-        results = []
-        for kernel_size in [(3, 3), (3, 2)]:
-          for dilation_rate in [(1, 1), (2, 3)]:
-            result = conv2d_layer(inp, num_filters, kernel_size,
-                                  dilation_rate=dilation_rate, padding='same',
-                                  data_format='channels_last')
-            results.append(result)
-        output = sum(results)
-        output = array_ops.identity(output, name=output_name)
-    return trt_test.TfTrtIntegrationTestParams(
-        gdef=g.as_graph_def(),
-        input_names=[input_name],
-        input_dims=[input_dims],
-        output_names=[output_name],
-        expected_output_dims=[(n, h, w, num_filters)])
-
-  def ExpectedEnginesToBuild(self, run_params):
-    """Return the expected engines to build."""
-    return ["TRTEngineOp_0"]
-
-
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From 2ae28cb35f8060a1e7168b216adb13ce665cb349 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Thu, 3 Jan 2019 14:57:47 -0800
Subject: [PATCH 0197/1095] Add v2 log loss.

PiperOrigin-RevId: 227755606
---
 tensorflow/python/keras/losses.py      | 46 +++++++++++++
 tensorflow/python/keras/losses_test.py | 91 ++++++++++++++++++++++++++
 2 files changed, 137 insertions(+)

diff --git a/tensorflow/python/keras/losses.py b/tensorflow/python/keras/losses.py
index d05dfb03f1..e6db795bb8 100644
--- a/tensorflow/python/keras/losses.py
+++ b/tensorflow/python/keras/losses.py
@@ -500,6 +500,46 @@ class CategoricalHinge(Loss):
     return categorical_hinge(y_true, y_pred)
 
 
+class LogLoss(Loss):
+  """Computes the log loss between `y_true` and `y_pred`.
+
+  logloss = -y log(p) - (1-y) log(1-p)
+
+  Usage:
+
+  ```python
+  l = tf.losses.LogLoss()
+  loss = l([0., 1., 1.], [1., 0., 1.])
+  print('Loss: ', loss.numpy())  # Loss: 10.745
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile('sgd', loss=tf.losses.LogLoss())
+  ```
+
+  Args:
+    epsilon: A small increment to add to avoid taking a log of zero.
+    reduction: Type of `tf.losses.Reduction` to apply to loss. Default value is
+      `SUM_OVER_BATCH_SIZE`.
+    name: Optional name for the op.
+  """
+
+  def __init__(self,
+               epsilon=1e-7,
+               reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
+               name=None):
+    super(LogLoss, self).__init__(reduction=reduction, name=name)
+    self.epsilon = epsilon
+
+  def call(self, y_true, y_pred):
+    y_pred = ops.convert_to_tensor(y_pred)
+    y_true = math_ops.cast(y_true, y_pred.dtype)
+    return logloss(y_true, y_pred, epsilon=self.epsilon)
+
+
 @keras_export('keras.metrics.mean_squared_error',
               'keras.metrics.mse',
               'keras.metrics.MSE',
@@ -562,6 +602,12 @@ def categorical_hinge(y_true, y_pred):
   return math_ops.maximum(0., neg - pos + 1.)
 
 
+def logloss(y_true, y_pred, epsilon=1e-7):
+  losses = math_ops.multiply(y_true, math_ops.log(y_pred + epsilon))
+  losses += math_ops.multiply((1 - y_true), math_ops.log(1 - y_pred + epsilon))
+  return K.mean(-losses, axis=-1)
+
+
 @keras_export('keras.losses.logcosh')
 def logcosh(y_true, y_pred):
   """Logarithm of the hyperbolic cosine of the prediction error.
diff --git a/tensorflow/python/keras/losses_test.py b/tensorflow/python/keras/losses_test.py
index 19ed7c8ed9..76fadc4a24 100644
--- a/tensorflow/python/keras/losses_test.py
+++ b/tensorflow/python/keras/losses_test.py
@@ -1003,5 +1003,96 @@ class CategoricalHingeTest(test.TestCase):
     self.assertAlmostEqual(self.evaluate(loss), 0., 3)
 
 
+@test_util.run_all_in_graph_and_eager_modes
+class LogLossTest(test.TestCase):
+
+  def setup(self):
+    # TODO(psv): Change to setUp() after b/122319309 is fixed.
+    y_pred = np.asarray([.9, .2, .2, .8, .4, .6]).reshape((2, 3))
+    y_true = np.asarray([1., 0., 1., 1., 0., 0.]).reshape((2, 3))
+    epsilon = 1e-7  # to avoid log 0
+
+    self.batch_size = 6
+    self.expected_losses = np.multiply(y_true, np.log(y_pred + epsilon))
+    self.expected_losses += np.multiply(1 - y_true,
+                                        np.log(1 - y_pred + epsilon))
+    self.expected_losses = -self.expected_losses
+
+    self.y_pred = constant_op.constant(y_pred)
+    self.y_true = constant_op.constant(y_true)
+
+  def test_config(self):
+    log_loss_obj = keras.losses.LogLoss(
+        reduction=losses_impl.ReductionV2.SUM, name='log')
+    self.assertEqual(log_loss_obj.name, 'log')
+    self.assertEqual(log_loss_obj.reduction, losses_impl.ReductionV2.SUM)
+
+  def test_all_correct(self):
+    self.setup()
+    log_loss_obj = keras.losses.LogLoss()
+    loss = log_loss_obj(self.y_true, self.y_true)
+    self.assertAlmostEqual(self.evaluate(loss), 0.0, 3)
+
+  def test_unweighted(self):
+    self.setup()
+    log_loss_obj = keras.losses.LogLoss()
+    loss = log_loss_obj(self.y_true, self.y_pred)
+    actual_loss = np.sum(self.expected_losses) / self.batch_size
+    self.assertAlmostEqual(self.evaluate(loss), actual_loss, 3)
+
+  def test_scalar_weighted(self):
+    self.setup()
+    log_loss_obj = keras.losses.LogLoss()
+    sample_weight = 2.3
+    loss = log_loss_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+    actual_loss = sample_weight * np.sum(self.expected_losses) / self.batch_size
+    self.assertAlmostEqual(self.evaluate(loss), actual_loss, 3)
+
+    # Verify we get the same output when the same input is given
+    loss_2 = log_loss_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+    self.assertAlmostEqual(self.evaluate(loss), self.evaluate(loss_2), 3)
+
+  def test_sample_weighted(self):
+    self.setup()
+    log_loss_obj = keras.losses.LogLoss()
+    sample_weight = constant_op.constant((1.2, 3.4), shape=(2, 1))
+
+    loss = log_loss_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+    actual_loss = np.multiply(
+        self.expected_losses,
+        np.asarray([1.2, 1.2, 1.2, 3.4, 3.4, 3.4]).reshape((2, 3)))
+    actual_loss = np.sum(actual_loss) / self.batch_size
+    self.assertAlmostEqual(self.evaluate(loss), actual_loss, 3)
+
+  def test_timestep_weighted(self):
+    log_loss_obj = keras.losses.LogLoss()
+
+    y_pred = np.asarray([.9, .2, .2, .8, .4, .6]).reshape((2, 3, 1))
+    y_true = np.asarray([1., 0., 1., 1., 0., 0.]).reshape((2, 3, 1))
+    epsilon = 1e-7  # to avoid log 0
+    batch_size = 6
+
+    expected_losses = np.multiply(y_true, np.log(y_pred + epsilon))
+    expected_losses += np.multiply(1 - y_true, np.log(1 - y_pred + epsilon))
+
+    y_pred = constant_op.constant(y_pred)
+    y_true = constant_op.constant(y_true)
+    sample_weight = np.array([3, 6, 5, 0, 4, 2]).reshape((2, 3, 1))
+    loss = log_loss_obj(
+        y_true,
+        y_pred,
+        sample_weight=constant_op.constant(sample_weight, shape=(2, 3)))
+    actual_loss = np.multiply(-expected_losses, sample_weight)
+    actual_loss = np.sum(actual_loss) / batch_size
+    self.assertAlmostEqual(self.evaluate(loss), actual_loss, 3)
+
+  def test_zero_weighted(self):
+    self.setup()
+    log_loss_obj = keras.losses.LogLoss()
+    sample_weight = 0
+    loss = log_loss_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+    self.assertAlmostEqual(self.evaluate(loss), 0., 3)
+
+
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From 76df46cc71916d7027334b26cc2b390e4c432251 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 3 Jan 2019 15:05:44 -0800
Subject: [PATCH 0198/1095] Automated rollback of commit
 4d9173668fd3ba410532efd901467312b7418752

PiperOrigin-RevId: 227757156
---
 tensorflow/cc/BUILD              |  1 -
 tensorflow/cc/saved_model/BUILD  |  2 -
 tensorflow/compiler/tf2xla/BUILD |  1 -
 tensorflow/core/BUILD            |  3 +-
 tensorflow/core/kernels/BUILD    | 83 +++++++++++++++++++++++++++++++-
 tensorflow/tensorflow.bzl        |  2 -
 6 files changed, 82 insertions(+), 10 deletions(-)

diff --git a/tensorflow/cc/BUILD b/tensorflow/cc/BUILD
index cf6d6050fa..a09becc49b 100644
--- a/tensorflow/cc/BUILD
+++ b/tensorflow/cc/BUILD
@@ -150,7 +150,6 @@ cc_library_with_android_deps(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/core:ops",
         "//tensorflow/core:protos_all_cc",
     ],
 )
diff --git a/tensorflow/cc/saved_model/BUILD b/tensorflow/cc/saved_model/BUILD
index 836678d364..52345a376c 100644
--- a/tensorflow/cc/saved_model/BUILD
+++ b/tensorflow/cc/saved_model/BUILD
@@ -81,7 +81,6 @@ cc_library(
     ] + if_not_mobile([
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:lib",
-        "//tensorflow/core:ops",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:tensorflow",
     ]) + if_android([
@@ -101,7 +100,6 @@ cc_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/core:ops",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core/util/tensor_bundle:naming",
         # mobile not supported yet
diff --git a/tensorflow/compiler/tf2xla/BUILD b/tensorflow/compiler/tf2xla/BUILD
index dfb6b57161..d8123e956f 100644
--- a/tensorflow/compiler/tf2xla/BUILD
+++ b/tensorflow/compiler/tf2xla/BUILD
@@ -281,7 +281,6 @@ tf_cc_test(
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
-        "//tensorflow/core:testlib",
     ],
 )
 
diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 6b32e1fead..7c9decbd09 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -1526,7 +1526,6 @@ cc_library(
         ":framework_internal",
         ":lib",
         ":lib_internal",
-        ":ops",
         ":protos_all_cc",
         ":shape_inference_testutil",
         ":tensor_testutil",
@@ -3078,7 +3077,6 @@ tf_cuda_library(
         ":lib",
         ":lib_internal",
         ":metrics",
-        ":ops",
         ":proto_text",
         ":protos_all_cc",
         "//tensorflow/core/debug:debug_graph_utils",
@@ -3861,6 +3859,7 @@ tf_cc_test(
         "ops/cudnn_rnn_ops_test.cc",
     ],
     deps = [
+        ":cudnn_rnn_ops",
         "//tensorflow/core",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index fd1df63c95..c72d04d3af 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -157,6 +157,7 @@ tf_kernel_library(
     name = "collective_ops",
     prefix = "collective_ops",
     deps = [
+        "//tensorflow/core:collective_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:protos_all_cc",
@@ -221,6 +222,7 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:nn_ops_op_lib",
         "//third_party/eigen3",
     ],
     alwayslink = 1,
@@ -311,6 +313,7 @@ tf_kernel_library(
         "//tensorflow/core/nccl:nccl_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:gpu_headers_lib",
+        "//tensorflow/core:nccl_ops_op_lib",
     ]),
 )
 
@@ -456,6 +459,7 @@ cc_library(
     name = "batch_kernels",
     srcs = ["batch_kernels.cc"],
     deps = [
+        "//tensorflow/core:batch_ops_op_lib",
         "//tensorflow/core:framework_headers_lib",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core/kernels:concat_lib_hdrs",
@@ -682,6 +686,7 @@ ARRAY_DEPS = [
     ":ops_util",
     ":transpose_functor",
     "//tensorflow/core:array_grad",
+    "//tensorflow/core:array_ops_op_lib",
     "//tensorflow/core:core_cpu",
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
@@ -710,6 +715,7 @@ tf_kernel_library(
     deps = [
         "//tensorflow/core:framework_headers_lib",
         "//tensorflow/core:lib",
+        "//tensorflow/core:set_ops_op_lib",
         "//third_party/eigen3",
     ],
 )
@@ -1073,6 +1079,7 @@ tf_kernel_library(
     srcs = ["ragged_gather_op.cc"],
     deps = [
         "//tensorflow/core:framework",
+        "//tensorflow/core:ragged_array_ops_op_lib",
     ],
 )
 
@@ -1083,6 +1090,7 @@ tf_cc_test(
     deps = [
         ":ragged_gather_op",
         "//tensorflow/core:framework",
+        "//tensorflow/core:ragged_array_ops_op_lib",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
@@ -1095,6 +1103,7 @@ tf_kernel_library(
     srcs = ["ragged_range_op.cc"],
     deps = [
         "//tensorflow/core:framework",
+        "//tensorflow/core:ragged_math_ops_op_lib",
     ],
 )
 
@@ -1104,6 +1113,7 @@ tf_cc_test(
     deps = [
         ":ragged_range_op",
         "//tensorflow/core:framework",
+        "//tensorflow/core:ragged_math_ops_op_lib",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
@@ -1116,6 +1126,7 @@ tf_kernel_library(
     srcs = ["ragged_tensor_to_sparse_kernel.cc"],
     deps = [
         "//tensorflow/core:framework",
+        "//tensorflow/core:ragged_conversion_ops_op_lib",
     ],
 )
 
@@ -1127,6 +1138,7 @@ tf_cc_test(
         ":ragged_tensor_to_sparse_kernel",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
+        "//tensorflow/core:ragged_conversion_ops_op_lib",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
@@ -1140,6 +1152,7 @@ tf_kernel_library(
     visibility = ["//visibility:public"],
     deps = [
         ":gpu_util_hdrs",
+        "//tensorflow/core:cudnn_rnn_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
@@ -1216,6 +1229,7 @@ tf_cuda_cc_test(
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
+        "//tensorflow/core:math_ops_op_lib",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
@@ -1740,6 +1754,7 @@ tf_kernel_library(
     prefix = "candidate_sampler_ops",
     deps = [
         ":range_sampler",
+        "//tensorflow/core:candidate_sampling_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
     ],
@@ -1773,6 +1788,7 @@ tf_kernel_library(
     name = "control_flow_ops",
     prefix = "control_flow_ops",
     deps = [
+        "//tensorflow/core:control_flow_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
     ],
@@ -1784,6 +1800,7 @@ tf_kernel_library(
     deps = [
         ":bounds_check",
         ":ops_util",
+        "//tensorflow/core:ctc_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core/util/ctc:ctc_beam_search_lib",
@@ -1864,6 +1881,7 @@ DATA_FLOW_DEPS = [
     ":typed_queue",
     "//third_party/eigen3",
     "//tensorflow/core:core_cpu",
+    "//tensorflow/core:data_flow_ops_op_lib",
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
@@ -1928,6 +1946,7 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:scoped_allocator_ops_op_lib",
     ],
 )
 
@@ -1946,6 +1965,7 @@ tf_cuda_cc_test(
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
+        "//tensorflow/core:math_ops_op_lib",
         "//tensorflow/core:proto_text",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:test",
@@ -1993,6 +2013,7 @@ tf_kernel_library(
 DYNAMIC_DEPS = [
     ":bounds_check",
     "//tensorflow/core:core_cpu",
+    "//tensorflow/core:data_flow_ops_op_lib",
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
@@ -2025,6 +2046,7 @@ LOOKUP_DEPS = [
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
+    "//tensorflow/core:lookup_ops_op_lib",
 ]
 
 tf_kernel_library(
@@ -2053,6 +2075,7 @@ tf_kernel_library(
     deps = [
         ":lookup_table_init_op",
         ":lookup_table_op",
+        "//tensorflow/core:checkpoint_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//third_party/eigen3",
@@ -2063,6 +2086,7 @@ tf_kernel_library(
     name = "load_and_remap_matrix_op",
     srcs = ["load_and_remap_matrix_op.cc"],
     deps = [
+        "//tensorflow/core:checkpoint_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
@@ -2209,6 +2233,7 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:resource_variable_ops_op_lib",
         "@com_google_absl//absl/strings",
     ],
 )
@@ -2225,6 +2250,7 @@ tf_kernel_library(
         ":concat_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
+        "//tensorflow/core:list_ops_op_lib",
         "//third_party/eigen3",
     ],
 )
@@ -2235,6 +2261,7 @@ tf_kernel_library(
     deps = [
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
+        "//tensorflow/core:user_ops_op_lib",
     ],
 )
 
@@ -2257,6 +2284,7 @@ tf_kernel_library(
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:framework",
+        "//tensorflow/core:functional_ops_op_lib",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//third_party/eigen3",
@@ -2269,6 +2297,7 @@ tf_kernel_library(
     deps = [
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:framework",
+        "//tensorflow/core:functional_ops_op_lib",
         "//tensorflow/core:lib",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core/grappler:grappler_item",
@@ -2311,6 +2340,7 @@ IMAGE_DEPS = [
     "//third_party/eigen3",
     "//tensorflow/core:framework",
     "//tensorflow/core:gif_internal",
+    "//tensorflow/core:image_ops_op_lib",
     "//tensorflow/core:jpeg_internal",
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
@@ -2650,6 +2680,7 @@ cc_library(
 IO_DEPS = [
     ":ops_util",
     "//tensorflow/core:framework",
+    "//tensorflow/core:io_ops_op_lib",
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
     "//tensorflow/core:protos_all_cc",
@@ -2693,6 +2724,7 @@ SAVE_RESTORE_DEPS = [
     ":bounds_check_lib",
     ":save_restore_tensor",
     "//tensorflow/core:framework",
+    "//tensorflow/core:io_ops_op_lib",
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
     "//tensorflow/core:protos_all_cc",
@@ -2811,6 +2843,7 @@ LINALG_DEPS = [
     "//third_party/eigen3",
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
+    "//tensorflow/core:linalg_ops_op_lib",
 ] + if_cuda([
     ":cuda_solvers",
     ":transpose_functor",
@@ -2954,6 +2987,7 @@ LOGGING_DEPS = [
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
+    "//tensorflow/core:logging_ops_op_lib",
     "//tensorflow/core:protos_all_cc",
 ]
 
@@ -3025,6 +3059,7 @@ tf_kernel_library(
         ":bounds_check",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
+        "//tensorflow/core:manip_ops_op_lib",
         "//third_party/eigen3",
     ],
 )
@@ -3056,6 +3091,7 @@ MATH_DEPS = [
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
     "//tensorflow/core:math_grad",
+    "//tensorflow/core:math_ops_op_lib",
     "//third_party/eigen3",
 ]
 
@@ -3166,7 +3202,7 @@ tf_kernel_library(
 tf_kernel_library(
     name = "cwise_op",
     prefix = "cwise_op",
-    deps = MATH_DEPS,
+    deps = MATH_DEPS + ["//tensorflow/core:bitwise_ops_op_lib"],
 )
 
 tf_kernel_library(
@@ -3185,6 +3221,7 @@ tf_kernel_library(
     name = "fft_ops",
     prefix = "fft_ops",
     deps = MATH_DEPS + [
+        "//tensorflow/core:spectral_ops_op_lib",
     ] + if_cuda([
         "//tensorflow/core/platform/default/build_config:cufft_plugin",
     ]),
@@ -3383,7 +3420,10 @@ tf_cuda_cc_test(
         ":quantized_ops",
         "//tensorflow/cc:cc_ops",
         "//tensorflow/cc:client_session",
+        "//tensorflow/core:array_ops_op_lib",
         "//tensorflow/core:framework",
+        "//tensorflow/core:math_ops_op_lib",
+        "//tensorflow/core:nn_ops_op_lib",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
@@ -3641,6 +3681,7 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:nn_ops_op_lib",
     ] + select({
         ":xsmm_convolutions": [
             "@libxsmm_archive//:xsmm_avx",
@@ -3662,6 +3703,7 @@ tf_kernel_library(
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
+        "//tensorflow/core:nn_ops_op_lib",
     ] + if_cuda([
         "@cub_archive//:cub",
         "@local_config_cuda//cuda:cudnn_header",
@@ -3681,6 +3723,7 @@ tf_kernel_library(
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
+        "//tensorflow/core:nn_ops_op_lib",
     ] + if_cuda([
         "@local_config_cuda//cuda:cudnn_header",
     ]),
@@ -3728,8 +3771,9 @@ NN_DEPS = [
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
     "//tensorflow/core:nn_grad",
+    "//tensorflow/core:nn_ops_op_lib",
     "//third_party/eigen3",
-]
+] + if_mkl(["//tensorflow/core:mkl_nn_ops_op_lib"])
 
 tf_kernel_library(
     name = "batch_norm_op",
@@ -3849,6 +3893,7 @@ tf_kernel_library(
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:nn_grad",
+        "//tensorflow/core:nn_ops_op_lib",
     ] + if_cuda(["@cub_archive//:cub"]),
 )
 
@@ -3956,6 +4001,7 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:nn_ops_op_lib",
         "//tensorflow/core:stream_executor",
         "//third_party/eigen3",
     ],
@@ -3999,6 +4045,7 @@ tf_kernel_library(
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
+        "//tensorflow/core:nn_ops_op_lib",
         "//third_party/eigen3",
     ],
 )
@@ -4080,6 +4127,7 @@ cc_library(
 PARSING_DEPS = [
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
+    "//tensorflow/core:parsing_ops_op_lib",
     "//tensorflow/core:proto_text",
     "//tensorflow/core:protos_all_cc",
 ]
@@ -4148,6 +4196,7 @@ RANDOM_OPS_DEPS = [
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
+    "//tensorflow/core:random_ops_op_lib",
 ]
 
 tf_kernel_library(
@@ -4186,6 +4235,7 @@ tf_kernel_library(
         ":random_op",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
+        "//tensorflow/core:stateless_random_ops_op_lib",
     ],
 )
 
@@ -4200,6 +4250,8 @@ cc_library(
 REQUIRED_DEPS = [
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
+    "//tensorflow/core:no_op_op_lib",
+    "//tensorflow/core:sendrecv_ops_op_lib",
 ]
 
 tf_kernel_library(
@@ -4260,6 +4312,7 @@ cc_library(
 SPARSE_DEPS = [
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
+    "//tensorflow/core:sparse_ops_op_lib",
 ]
 
 tf_kernel_library(
@@ -4504,6 +4557,7 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:sdca_ops_op_lib",
         "//third_party/eigen3",
         "@farmhash_archive//:farmhash",
     ],
@@ -4543,6 +4597,7 @@ STATE_DEPS = [
     "//third_party/eigen3",
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
+    "//tensorflow/core:state_ops_op_lib",
 ] + if_sycl(["//tensorflow/core:sycl_runtime"])
 
 tf_kernel_library(
@@ -4680,6 +4735,7 @@ STRING_DEPS = [
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
+    "//tensorflow/core:string_ops_op_lib",
 ]
 
 tf_kernel_library(
@@ -4830,6 +4886,7 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:string_ops_op_lib",
         "//third_party/eigen3",
         "//third_party/icu/data:conversion_data",
         "@icu//:common",
@@ -4851,6 +4908,7 @@ tf_kernel_library(
         ":variable_ops",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
+        "//tensorflow/core:training_ops_op_lib",
         "//third_party/eigen3",
     ],
 )
@@ -4909,6 +4967,7 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:random_ops_op_lib",
     ],
 )
 
@@ -4936,6 +4995,7 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:random_ops_op_lib",
     ],
 )
 
@@ -5844,9 +5904,12 @@ tf_kernel_library(
         ":ops_util",
         ":pooling_ops",
         ":quantization_utils",
+        "//tensorflow/core:array_ops_op_lib",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
+        "//tensorflow/core:math_ops_op_lib",
+        "//tensorflow/core:nn_ops_op_lib",
         "//third_party/eigen3",
         "@gemmlowp",
     ],
@@ -6416,6 +6479,7 @@ tf_kernel_library(
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:remote_fused_graph_ops_op_lib",
     ],
 )
 
@@ -6570,6 +6634,8 @@ tf_mkl_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:mkl_nn_ops_op_lib",
+        "//tensorflow/core:nn_ops_op_lib",
     ] + mkl_deps(),
 )
 
@@ -6619,6 +6685,8 @@ tf_mkl_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:mkl_nn_ops_op_lib",
+        "//tensorflow/core:nn_ops_op_lib",
     ] + mkl_deps(),
 )
 
@@ -6637,6 +6705,8 @@ tf_mkl_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:mkl_nn_ops_op_lib",
+        "//tensorflow/core:nn_ops_op_lib",
     ] + mkl_deps(),
 )
 
@@ -6650,6 +6720,8 @@ tf_mkl_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:mkl_nn_ops_op_lib",
+        "//tensorflow/core:nn_ops_op_lib",
         "//third_party/eigen3",
     ] + mkl_deps(),
 )
@@ -6664,6 +6736,8 @@ tf_mkl_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:mkl_nn_ops_op_lib",
+        "//tensorflow/core:nn_ops_op_lib",
         "//third_party/eigen3",
     ] + mkl_deps(),
 )
@@ -6804,6 +6878,7 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:summary_ops_op_lib",
         "//tensorflow/core/lib/db:sqlite",
     ] + if_not_v2([
         "//tensorflow/contrib/tensorboard/db:schema",
@@ -6818,6 +6893,7 @@ tf_kernel_library(
         "decode_proto_op.cc",
     ],
     deps = [
+        "//tensorflow/core:decode_proto_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core/util/proto:decode",
@@ -6831,6 +6907,7 @@ tf_kernel_library(
     name = "encode_proto_op",
     srcs = ["encode_proto_op.cc"],
     deps = [
+        "//tensorflow/core:encode_proto_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core/util/proto:descriptors",
@@ -6848,6 +6925,7 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:rpc_ops_op_lib",
         "//tensorflow/core/util/rpc:call_container",
         "//tensorflow/core/util/rpc:rpc_factory",
         "//tensorflow/core/util/rpc:rpc_factory_registry",
@@ -6860,6 +6938,7 @@ tf_kernel_library(
     srcs = ["unicode_script_op.cc"],
     deps = [
         "//tensorflow/core:framework",
+        "//tensorflow/core:string_ops_op_lib",
         "@icu//:common",
     ],
 )
diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl
index b9069768e8..1024b686eb 100644
--- a/tensorflow/tensorflow.bzl
+++ b/tensorflow/tensorflow.bzl
@@ -590,7 +590,6 @@ def tf_gen_op_wrappers_cc(
             clean_dep("//tensorflow/core:core_cpu"),
             clean_dep("//tensorflow/core:framework"),
             clean_dep("//tensorflow/core:lib"),
-            clean_dep("//tensorflow/core:ops"),
             clean_dep("//tensorflow/core:protos_all_cc"),
         ]) + if_android([
             clean_dep("//tensorflow/core:android_tensorflow_lib"),
@@ -607,7 +606,6 @@ def tf_gen_op_wrappers_cc(
             clean_dep("//tensorflow/core:core_cpu"),
             clean_dep("//tensorflow/core:framework"),
             clean_dep("//tensorflow/core:lib"),
-            clean_dep("//tensorflow/core:ops"),
             clean_dep("//tensorflow/core:protos_all_cc"),
         ]) + if_android([
             clean_dep("//tensorflow/core:android_tensorflow_lib"),
-- 
GitLab


From 8f1e280350d1a0a6b4134327ecd029195438e767 Mon Sep 17 00:00:00 2001
From: Tim Shen <timshen@google.com>
Date: Thu, 3 Jan 2019 15:08:10 -0800
Subject: [PATCH 0199/1095] Add a clang-specific warning -Wmismatched-tags.

PiperOrigin-RevId: 227757550
---
 tensorflow/stream_executor/cuda/cuda_dnn.cc | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.cc b/tensorflow/stream_executor/cuda/cuda_dnn.cc
index 7c05701895..01ff248150 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.cc
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.cc
@@ -51,6 +51,12 @@ limitations under the License.
 #include "absl/strings/string_view.h"
 // clang-format on
 
+#pragma clang diagnostic push
+
+// Make sure that Eigen::half forward declaration in dnn.h matches the
+// declaration in Eigen.
+#pragma clang diagnostic warning "-Wmismatched-tags"
+
 namespace stream_executor {
 namespace cuda {
 
@@ -4531,5 +4537,7 @@ void initialize_cudnn() {
 
 }  // namespace stream_executor
 
+#pragma clang diagnostic pop
+
 REGISTER_MODULE_INITIALIZER(register_cudnn,
                             { stream_executor::initialize_cudnn(); });
-- 
GitLab


From 2c65e8b01301d3d001952b3fc92d4a06c1f08bf5 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Thu, 3 Jan 2019 15:23:43 -0800
Subject: [PATCH 0200/1095] Throw an error when running loop inside scope

PiperOrigin-RevId: 227760023
---
 .../contrib/distribute/python/keras_test.py   | 19 +++++++++++++++++--
 .../engine/distributed_training_utils.py      | 11 +++++++++++
 .../keras/engine/training_distributed.py      |  6 ++++++
 3 files changed, 34 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index ba5a5e6400..4cfcc0375f 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -1161,8 +1161,8 @@ class TestDistributionStrategyWithNormalizationLayer(
       np.testing.assert_allclose(out.std(), 1.0, atol=1e-1)
 
 
-class TestDistributionStrategyVariableValidation(test.TestCase,
-                                                 parameterized.TestCase):
+class TestDistributionStrategyValidation(test.TestCase,
+                                         parameterized.TestCase):
 
   @combinations.generate(all_strategy_combinations_minus_default())
   def test_layer_outside_scope(self, distribution):
@@ -1192,6 +1192,21 @@ class TestDistributionStrategyVariableValidation(test.TestCase,
           metrics = ['mae', keras.metrics.CategoricalAccuracy()]
           model.compile(optimizer, loss, metrics=metrics)
 
+  @combinations.generate(all_strategy_combinations_minus_default())
+  def test_loop_in_scope(self, distribution):
+    with self.cached_session():
+      with self.assertRaisesRegexp(
+          RuntimeError, 'should not be run inside the distribution strategy'):
+        with distribution.scope():
+          x = keras.layers.Input(shape=(3,), name='input')
+          y = keras.layers.Dense(4, name='dense')(x)
+          model = keras.Model(x, y)
+          optimizer = gradient_descent.GradientDescentOptimizer(0.001)
+          loss = 'mse'
+          model.compile(optimizer, loss)
+          input_array = np.zeros((3, 3), dtype=np.float32)
+          model.predict(input_array)
+
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/keras/engine/distributed_training_utils.py b/tensorflow/python/keras/engine/distributed_training_utils.py
index 231c59ba93..9e3bdd26cb 100644
--- a/tensorflow/python/keras/engine/distributed_training_utils.py
+++ b/tensorflow/python/keras/engine/distributed_training_utils.py
@@ -38,10 +38,21 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.training import distribution_strategy_context
 from tensorflow.python.training.mode_keys import ModeKeys
 from tensorflow.python.util import nest
 
 
+def validate_not_in_strategy_scope():
+  """Validate fit/eval/predict are not running in DS scope."""
+  if distribution_strategy_context.has_distribution_strategy():
+    if distribution_strategy_context.in_cross_replica_context():
+      raise RuntimeError(
+          'Fit/Eval/Predict should not be run inside the distribution strategy '
+          'scope. Only model creation and compilation should be in '
+          'distribution strategy scope.')
+
+
 def set_weights(distribution_strategy, dist_model, weights):
   """Sets the weights of the replicated models.
 
diff --git a/tensorflow/python/keras/engine/training_distributed.py b/tensorflow/python/keras/engine/training_distributed.py
index 0bd79f2b47..9f63f7bada 100644
--- a/tensorflow/python/keras/engine/training_distributed.py
+++ b/tensorflow/python/keras/engine/training_distributed.py
@@ -53,6 +53,8 @@ def fit_distributed(model,
                     steps_per_epoch=None,
                     validation_steps=None):
   """Fit loop for Distribution Strategies."""
+  # TODO(b/122314600): Remove the scope validate.
+  distributed_training_utils.validate_not_in_strategy_scope()
   distributed_training_utils.validate_callbacks(callbacks, model.optimizer)
   distributed_training_utils.validate_inputs(
       x, y, model._distribution_strategy)
@@ -136,6 +138,8 @@ def evaluate_distributed(model,
                          steps=None,
                          callbacks=None):
   """Evaluate loop for Distribution Strategies."""
+  # TODO(b/122314600): Remove the scope validate.
+  distributed_training_utils.validate_not_in_strategy_scope()
   distributed_training_utils.validate_inputs(x, y, model._distribution_strategy)
   first_x_value = nest.flatten(x)[0]
   if isinstance(first_x_value, np.ndarray):
@@ -171,6 +175,8 @@ def predict_distributed(model,
                         steps=None,
                         callbacks=None):
   """Predict loop for Distribution Strategies."""
+  # TODO(b/122314600): Remove the scope validate.
+  distributed_training_utils.validate_not_in_strategy_scope()
   distributed_training_utils.validate_inputs(
       x, None, model._distribution_strategy)
   first_x_value = nest.flatten(x)[0]
-- 
GitLab


From 6932e795621a76a7e520bee529d915265b40e9bd Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 3 Jan 2019 15:26:41 -0800
Subject: [PATCH 0201/1095] In tf.name_scope, throw an exception when the
 `default_name` arg is passed in but not a string and warn the user that they
 likely meant to pass it in as the `values` kwarg.

PiperOrigin-RevId: 227760445
---
 tensorflow/python/framework/ops.py      | 8 ++++++++
 tensorflow/python/framework/ops_test.py | 3 +++
 2 files changed, 11 insertions(+)

diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index 5dc8e418a2..079fad4210 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -6029,7 +6029,15 @@ class name_scope(object):  # pylint: disable=invalid-name
       name: The name argument that is passed to the op function.
       default_name: The default name to use if the `name` argument is `None`.
       values: The list of `Tensor` arguments that are passed to the op function.
+
+    Raises:
+      TypeError: if `default_name` is passed in but not a string.
     """
+    if not (default_name is None or isinstance(default_name, six.string_types)):
+      raise TypeError(
+          "`default_name` type (%s) is not a string type. You likely meant to "
+          "pass this into the `values` kwarg."
+          % type(default_name))
     self._name = default_name if name is None else name
     self._default_name = default_name
     self._values = values
diff --git a/tensorflow/python/framework/ops_test.py b/tensorflow/python/framework/ops_test.py
index 2d7ee1a99e..58d311fe4e 100644
--- a/tensorflow/python/framework/ops_test.py
+++ b/tensorflow/python/framework/ops_test.py
@@ -2052,6 +2052,9 @@ class OpScopeTest(test_util.TensorFlowTestCase):
     with ops.name_scope(None, default_scope_name, [a, b]) as scope:
       self.assertEqual("%s/" % default_scope_name, scope)
       self.assertEqual(g0, ops.get_default_graph())
+    with self.assertRaises(TypeError):
+      with ops.name_scope(scope_name, [a, b]):
+        pass
 
   def _testGraphElements(self, graph_elements):
     scope_name = "my_scope"
-- 
GitLab


From 88c658a1bc737385e2edcbb7068406321151d875 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 3 Jan 2019 16:18:43 -0800
Subject: [PATCH 0202/1095] Update ops-related pbtxt files.

PiperOrigin-RevId: 227768824
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 28 +++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 |  9 ++++++
 2 files changed, 37 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index d27f7a5eac..28d085b2d2 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -76625,6 +76625,34 @@ op {
     type: "type"
   }
 }
+op {
+  name: "TensorListConcat"
+  input_arg {
+    name: "input_handle"
+    type: DT_VARIANT
+  }
+  output_arg {
+    name: "tensor"
+    type_attr: "element_dtype"
+  }
+  output_arg {
+    name: "lengths"
+    type: DT_INT64
+  }
+  attr {
+    name: "element_dtype"
+    type: "type"
+  }
+  attr {
+    name: "element_shape"
+    type: "shape"
+    default_value {
+      shape {
+        unknown_rank: true
+      }
+    }
+  }
+}
 op {
   name: "TensorListConcatLists"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 412b4160d9..b4767d35d2 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -36756,6 +36756,15 @@ op {
     name: "element_dtype"
     type: "type"
   }
+  attr {
+    name: "element_shape"
+    type: "shape"
+    default_value {
+      shape {
+        unknown_rank: true
+      }
+    }
+  }
 }
 op {
   name: "TensorListConcatLists"
-- 
GitLab


From be57fb3a05ccfe01089fc2926e885c35ad5bda00 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 3 Jan 2019 16:34:45 -0800
Subject: [PATCH 0203/1095] Small refactoring in preparation for a larger
 change.

PiperOrigin-RevId: 227771218
---
 tensorflow/lite/delegates/flex/kernel.cc      | 116 +++++++++++-------
 tensorflow/lite/delegates/flex/kernel_test.cc |   5 +-
 2 files changed, 70 insertions(+), 51 deletions(-)

diff --git a/tensorflow/lite/delegates/flex/kernel.cc b/tensorflow/lite/delegates/flex/kernel.cc
index d6e12ef650..250fc31043 100644
--- a/tensorflow/lite/delegates/flex/kernel.cc
+++ b/tensorflow/lite/delegates/flex/kernel.cc
@@ -52,11 +52,11 @@ namespace flex {
 namespace kernel {
 
 // Controls the lifetime of tensor handles in a vector.
-class VectorOfHandles {
+class OpOutputs {
  public:
-  explicit VectorOfHandles(int num_elements) : vector_(num_elements, nullptr) {}
+  explicit OpOutputs(int num_elements) : vector_(num_elements, nullptr) {}
 
-  ~VectorOfHandles() {
+  ~OpOutputs() {
     for (auto* handle : vector_) {
       if (handle) handle->Unref();
     }
@@ -72,6 +72,21 @@ class VectorOfHandles {
   tensorflow::gtl::InlinedVector<tensorflow::TensorHandle*, 2> vector_;
 };
 
+// A single node within the larger 'op'. Note that this kernel executes many
+// TensorFlow ops within a single TF Lite op.
+struct OpNode {
+  // The name of the TensorFlow op to execute.
+  string name;
+  // Index of this node into TF Lite's operator list.
+  int index;
+  // The corresponding NodeDef, containing the attributes for the op.
+  tensorflow::NodeDef nodedef;
+  // List of inputs, as TF Lite tensor indices.
+  std::vector<int> inputs;
+  // List of outputs, as TF Lite tensor indices.
+  std::vector<int> outputs;
+};
+
 // Executes the TensorFlow op given by 'op_name', with the attributes specified
 // in 'nodedef'. Inputs and outputs are given as indices into the 'buffer_map'.
 tensorflow::Status ExecuteFlexOp(tensorflow::EagerContext* eager_context,
@@ -114,7 +129,7 @@ tensorflow::Status ExecuteFlexOp(tensorflow::EagerContext* eager_context,
   }
 
   int num_retvals = outputs.size();
-  VectorOfHandles retvals(num_retvals);
+  OpOutputs retvals(num_retvals);
   TF_RETURN_WITH_CONTEXT_IF_ERROR(
       EagerExecute(&op, retvals.GetVector(), &num_retvals),
       " (while executing '", op_name, "' via Eager)");
@@ -133,22 +148,38 @@ tensorflow::Status ExecuteFlexOp(tensorflow::EagerContext* eager_context,
   return tensorflow::Status::OK();
 }
 
-// A single node within the larger 'op'. Note that this kernel executes many
-// TensorFlow ops within a single TF Lite op.
-struct OpNode {
-  // The name of the TensorFlow op to execute.
-  string name;
-  // Index of this node into TF Lite's operator list.
-  int index;
-  // The corresponding NodeDef, containing the attributes for the op.
-  tensorflow::NodeDef nodedef;
-  // List of inputs, as TF Lite tensor indices.
-  std::vector<int> inputs;
-  // List of outputs, as TF Lite tensor indices.
-  std::vector<int> outputs;
-};
+tensorflow::Status InitializeNodeDef(const void* custom_initial_data,
+                                     int custom_initial_data_size,
+                                     OpNode* node_data) {
+  if (!custom_initial_data) {
+    return tensorflow::errors::Internal(
+        "Cannot convert empty data into a valid NodeDef");
+  }
+  // The flexbuffer contains a vector where the first elements is the
+  // op name and the second is a serialized NodeDef.
+  const flexbuffers::Vector& v =
+      flexbuffers::GetRoot(
+          reinterpret_cast<const uint8_t*>(custom_initial_data),
+          custom_initial_data_size)
+          .AsVector();
+
+  node_data->name = v[0].AsString().str();
+  if (!node_data->nodedef.ParseFromString(v[1].AsString().str())) {
+    node_data->nodedef.Clear();
+    return tensorflow::errors::Internal(
+        "Failed to parse data into a valid NodeDef");
+  }
+
+  // Fill NodeDef with defaults if it's a valid op.
+  const tensorflow::OpRegistrationData* op_reg_data;
+  TF_RETURN_IF_ERROR(tensorflow::OpRegistry::Global()->LookUp(
+      node_data->nodedef.op(), &op_reg_data));
+  AddDefaultsToNodeDef(op_reg_data->op_def, &node_data->nodedef);
 
-// The Larger 'op', which contains all the nodes in a supported subgraph.
+  return tensorflow::Status::OK();
+}
+
+// The larger 'op', which contains all the nodes in a supported subgraph.
 struct OpData {
   tensorflow::EagerContext* eager_context;
   BufferMap* buffer_map;
@@ -182,6 +213,7 @@ void* Init(TfLiteContext* context, const char* buffer, size_t length) {
   }
 
   CHECK(params->nodes_to_replace);
+  tensorflow::Status status;
   for (auto node_index : TfLiteIntArrayView(params->nodes_to_replace)) {
     TfLiteNode* node;
     TfLiteRegistration* reg;
@@ -192,29 +224,10 @@ void* Init(TfLiteContext* context, const char* buffer, size_t length) {
 
     node_data.index = node_index;
     node_data.name = "";
-    if (node->custom_initial_data) {
-      // The flexbuffer contains a vector where the first elements is the
-      // op name and the second is a serialized NodeDef.
-      const flexbuffers::Vector& v =
-          flexbuffers::GetRoot(
-              reinterpret_cast<const uint8_t*>(node->custom_initial_data),
-              node->custom_initial_data_size)
-              .AsVector();
-
-      node_data.name = v[0].AsString().str();
-      if (!node_data.nodedef.ParseFromString(v[1].AsString().str())) {
-        // We will just leave the nodedef empty and error out in Eval().
-        node_data.nodedef.Clear();
-      }
-    }
 
-    // Fill NodeDef with defaults if it's a valid op.
-    const tensorflow::OpRegistrationData* op_reg_data;
-    auto tf_status = tensorflow::OpRegistry::Global()->LookUp(
-        node_data.nodedef.op(), &op_reg_data);
-    if (tf_status.ok()) {
-      AddDefaultsToNodeDef(op_reg_data->op_def, &node_data.nodedef);
-    }
+    status = InitializeNodeDef(node->custom_initial_data,
+                               node->custom_initial_data_size, &node_data);
+    if (!status.ok()) break;
 
     for (auto input_index : TfLiteIntArrayView(node->inputs)) {
       node_data.inputs.push_back(input_index);
@@ -224,6 +237,13 @@ void* Init(TfLiteContext* context, const char* buffer, size_t length) {
     }
   }
 
+  if (ConvertStatus(context, status) != kTfLiteOk) {
+    // We can't return an error from this function but ConvertStatus will
+    // report them and we will stop processing in Prepare() if anything went
+    // wrong.
+    return op_data;
+  }
+
   return op_data;
 }
 
@@ -263,6 +283,12 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
   }
 
   for (const auto& node_data : op_data->nodes) {
+    if (node_data.nodedef.op().empty()) {
+      context->ReportError(context, "Invalid NodeDef in Flex op '%s'",
+                           node_data.name.c_str());
+      return kTfLiteError;
+    }
+
     for (int tensor_index : node_data.inputs) {
       ++tensor_ref_count[tensor_index];
     }
@@ -303,12 +329,8 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
   for (const auto& node_data : op_data->nodes) {
     SCOPED_TAGGED_OPERATOR_PROFILE(
         reinterpret_cast<profiling::Profiler*>(context->profiler),
-        node_data.name.c_str(), node_data.index);
-    if (node_data.nodedef.op().empty()) {
-      context->ReportError(context, "Invalid NodeDef in Flex op '%s'",
-                           node_data.name.c_str());
-      return kTfLiteError;
-    }
+        node_data->name.c_str(), node_data->index);
+
     auto status =
         ExecuteFlexOp(eager_context, buffer_map, node_data.name,
                       node_data.nodedef, node_data.inputs, node_data.outputs);
diff --git a/tensorflow/lite/delegates/flex/kernel_test.cc b/tensorflow/lite/delegates/flex/kernel_test.cc
index cc5c8b32a0..647c199d4f 100644
--- a/tensorflow/lite/delegates/flex/kernel_test.cc
+++ b/tensorflow/lite/delegates/flex/kernel_test.cc
@@ -166,10 +166,7 @@ TEST_F(KernelTest, WrongSetOfNodes) {
     return GenericPrepare(context, delegate, {0, 1});
   });
 
-  SetShape(0, {2, 2, 1});
-  SetValues(0, {1.1f, 2.2f, 3.3f, 4.4f});
-
-  ASSERT_FALSE(Invoke());
+  ASSERT_NE(interpreter_->AllocateTensors(), kTfLiteOk);
   ASSERT_THAT(error_reporter().error_messages(),
               ContainsRegex("Invalid NodeDef in Flex op"));
 }
-- 
GitLab


From 03258acbf396a1241dee813e86848dc768394afd Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Thu, 3 Jan 2019 17:15:41 -0800
Subject: [PATCH 0204/1095] [tf.data] Replacing active / total parallel calls
 metric with thread utilization metric.

PiperOrigin-RevId: 227776910
---
 .../experimental/map_and_batch_dataset_op.cc  | 22 ++++++++-----------
 .../data/parallel_interleave_dataset_op.cc    | 19 +++++++---------
 .../kernels/data/parallel_map_iterator.cc     | 19 +++++++---------
 .../kernel_tests/stats_dataset_test_base.py   |  4 +---
 4 files changed, 26 insertions(+), 38 deletions(-)

diff --git a/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc b/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc
index ef75c84456..a6348e4647 100644
--- a/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc
@@ -259,7 +259,7 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
                                             params.dataset->batch_size_)) {
         std::vector<string> components =
             str_util::Split(params.prefix, "::", str_util::SkipEmpty());
-        prefix_end_ = components.back();
+        key_prefix_ = components.back();
       }
 
       ~Iterator() override {
@@ -397,8 +397,9 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
         const auto& stats_aggregator = ctx->stats_aggregator();
         if (stats_aggregator) {
           stats_aggregator->AddScalar(
-              strings::StrCat(prefix_end_, "::active_parallel_calls"),
-              static_cast<float>(num_calls_));
+              strings::StrCat(key_prefix_, "::thread_utilization"),
+              static_cast<float>(num_calls_) /
+                  static_cast<float>(num_parallel_calls_->value));
         }
         cond_var_->notify_all();
       }
@@ -637,18 +638,13 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
               num_calls_++;
             }
           }
-          const std::shared_ptr<StatsAggregator>& stats_aggregator =
-              ctx->stats_aggregator();
+          const auto& stats_aggregator = ctx->stats_aggregator();
           if (stats_aggregator) {
             mutex_lock l(*mu_);
-            // TODO(shivaniagrawal): add `parallel_calls_utilization` in the
-            // monitoring code or as histogram at fixed time intervals.
             stats_aggregator->AddScalar(
-                strings::StrCat(prefix_end_, "::active_parallel_calls"),
-                static_cast<float>(num_calls_));
-            stats_aggregator->AddScalar(
-                strings::StrCat(prefix_end_, "::num_parallel_calls"),
-                static_cast<float>(num_parallel_calls_->value));
+                strings::StrCat(key_prefix_, "::thread_utilization"),
+                static_cast<float>(num_calls_) /
+                    static_cast<float>(num_parallel_calls_->value));
           }
           for (const auto& call : new_calls) {
             CallFunction(ctx, call.first, call.second);
@@ -803,7 +799,7 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
       int64 waiting_ GUARDED_BY(*mu_) = 0;
       // Identifies the maximum number of batch results to store.
       int64 max_batch_results_ GUARDED_BY(*mu_);
-      string prefix_end_;
+      string key_prefix_;
       std::unique_ptr<InstantiatedCapturedFunction> instantiated_captured_func_;
     };
 
diff --git a/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc b/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc
index 74791669ce..315f96d4b5 100644
--- a/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc
+++ b/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc
@@ -209,7 +209,7 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
                 false /* low_latency_hint */)) {
         std::vector<string> components =
             str_util::Split(params.prefix, "::", str_util::SkipEmpty());
-        prefix_end_ = components.back();
+        key_prefix_ = components.back();
       }
 
       ~ParallelInterleaveIterator() override {
@@ -430,8 +430,9 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
         const auto& stats_aggregator = ctx->stats_aggregator();
         if (stats_aggregator) {
           stats_aggregator->AddScalar(
-              strings::StrCat(prefix_end_, "::active_parallel_calls"),
-              static_cast<float>(num_calls_));
+              strings::StrCat(key_prefix_, "::thread_utilization"),
+              static_cast<float>(num_calls_) /
+                  static_cast<float>(num_parallel_calls_->value));
         }
         cond_var_->notify_all();
       }
@@ -515,14 +516,10 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
           }
           const auto& stats_aggregator = ctx->stats_aggregator();
           if (stats_aggregator) {
-            // TODO(shivaniagrawal): add `parallel_calls_utilization` in the
-            // monitoring code or as histogram at fixed time intervals.
             stats_aggregator->AddScalar(
-                strings::StrCat(prefix_end_, "::active_parallel_calls"),
-                static_cast<float>(num_calls_));
-            stats_aggregator->AddScalar(
-                strings::StrCat(prefix_end_, "::num_parallel_calls"),
-                static_cast<float>(num_parallel_calls_->value));
+                strings::StrCat(key_prefix_, "::thread_utilization"),
+                static_cast<float>(num_calls_) /
+                    static_cast<float>(num_parallel_calls_->value));
           }
           cond_var_->notify_all();
         }
@@ -693,7 +690,7 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
 
       // Identifies whether background activity should be cancelled.
       bool cancelled_ GUARDED_BY(*mu_) = false;
-      string prefix_end_;
+      string key_prefix_;
       std::unique_ptr<InstantiatedCapturedFunction> instantiated_captured_func_;
     };
 
diff --git a/tensorflow/core/kernels/data/parallel_map_iterator.cc b/tensorflow/core/kernels/data/parallel_map_iterator.cc
index b62e7059ba..9a4ce981d0 100644
--- a/tensorflow/core/kernels/data/parallel_map_iterator.cc
+++ b/tensorflow/core/kernels/data/parallel_map_iterator.cc
@@ -60,7 +60,7 @@ class ParallelMapIterator : public DatasetBaseIterator {
         preserve_cardinality_(params.preserve_cardinality) {
     std::vector<string> components =
         str_util::Split(base_params.prefix, "::", str_util::SkipEmpty());
-    prefix_end_ = components.back();
+    key_prefix_ = components.back();
   }
 
   ~ParallelMapIterator() override {
@@ -207,8 +207,9 @@ class ParallelMapIterator : public DatasetBaseIterator {
     const auto& stats_aggregator = ctx->stats_aggregator();
     if (stats_aggregator) {
       stats_aggregator->AddScalar(
-          strings::StrCat(prefix_end_, "::active_parallel_calls"),
-          static_cast<float>(num_calls_));
+          strings::StrCat(key_prefix_, "::thread_utilization"),
+          static_cast<float>(num_calls_) /
+              static_cast<float>(num_parallel_calls_->value));
     }
     RecordBufferEnqueue(ctx.get(), result->return_values);
     result->notification.Notify();
@@ -300,14 +301,10 @@ class ParallelMapIterator : public DatasetBaseIterator {
         }
         const auto& stats_aggregator = ctx->stats_aggregator();
         if (stats_aggregator) {
-          // TODO(shivaniagrawal): add `parallel_calls_utilization` in the
-          // monitoring code or as histogram at fixed time intervals.
-          stats_aggregator->AddScalar(
-              strings::StrCat(prefix_end_, "::active_parallel_calls"),
-              static_cast<float>(num_calls_));
           stats_aggregator->AddScalar(
-              strings::StrCat(prefix_end_, "::num_parallel_calls"),
-              static_cast<float>(num_parallel_calls_->value));
+              strings::StrCat(key_prefix_, "::thread_utilization"),
+              static_cast<float>(num_calls_) /
+                  static_cast<float>(num_parallel_calls_->value));
         }
         cond_var_->notify_all();
       }
@@ -403,7 +400,7 @@ class ParallelMapIterator : public DatasetBaseIterator {
       GUARDED_BY(*mu_);
   std::unique_ptr<Thread> runner_thread_ GUARDED_BY(*mu_);
   bool cancelled_ GUARDED_BY(*mu_) = false;
-  string prefix_end_;
+  string key_prefix_;
 };
 
 }  // namespace
diff --git a/tensorflow/python/data/experimental/kernel_tests/stats_dataset_test_base.py b/tensorflow/python/data/experimental/kernel_tests/stats_dataset_test_base.py
index b80aab994e..f5a15f4c84 100644
--- a/tensorflow/python/data/experimental/kernel_tests/stats_dataset_test_base.py
+++ b/tensorflow/python/data/experimental/kernel_tests/stats_dataset_test_base.py
@@ -104,9 +104,7 @@ class StatsDatasetTestBase(test_base.DatasetTestBase):
         self._assertSummaryHasCountMoreOrEqualGeneralisedTag(
             summary_str, "::execution_time", float(i + 1))
       self._assertSummaryContains(summary_str,
-                                  dataset_name + "::num_parallel_calls")
-      self._assertSummaryContains(summary_str,
-                                  dataset_name + "::active_parallel_calls")
+                                  dataset_name + "::thread_utilization")
     with self.assertRaises(errors.OutOfRangeError):
       self.evaluate(next_element())
     if function_processing_time:
-- 
GitLab


From 8cfb8d2c11c93e36d7db2635518427260fcc4d41 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 3 Jan 2019 17:16:06 -0800
Subject: [PATCH 0205/1095] Turn OpNode into a proper class, in preparation for
 a larger change.

PiperOrigin-RevId: 227776959
---
 tensorflow/lite/delegates/flex/kernel.cc | 118 +++++++++++++----------
 1 file changed, 68 insertions(+), 50 deletions(-)

diff --git a/tensorflow/lite/delegates/flex/kernel.cc b/tensorflow/lite/delegates/flex/kernel.cc
index 250fc31043..7d18c39098 100644
--- a/tensorflow/lite/delegates/flex/kernel.cc
+++ b/tensorflow/lite/delegates/flex/kernel.cc
@@ -74,17 +74,66 @@ class OpOutputs {
 
 // A single node within the larger 'op'. Note that this kernel executes many
 // TensorFlow ops within a single TF Lite op.
-struct OpNode {
+class OpNode {
+ public:
+  OpNode() {}
+  ~OpNode() {}
+
+  const string& name() const { return name_; }
+  void set_name(const string& name) { name_ = name; }
+
+  int index() const { return index_; }
+  void set_index(int index) { index_ = index; }
+
+  const tensorflow::NodeDef& nodedef() const { return nodedef_; }
+
+  const std::vector<int>& inputs() const { return inputs_; }
+  std::vector<int>* mutable_inputs() { return &inputs_; }
+
+  const std::vector<int>& outputs() const { return outputs_; }
+  std::vector<int>* mutable_outputs() { return &outputs_; }
+
+  tensorflow::Status InitializeNodeDef(const void* custom_initial_data,
+                                       int custom_initial_data_size) {
+    if (!custom_initial_data) {
+      return tensorflow::errors::Internal(
+          "Cannot convert empty data into a valid NodeDef");
+    }
+    // The flexbuffer contains a vector where the first elements is the
+    // op name and the second is a serialized NodeDef.
+    const flexbuffers::Vector& v =
+        flexbuffers::GetRoot(
+            reinterpret_cast<const uint8_t*>(custom_initial_data),
+            custom_initial_data_size)
+            .AsVector();
+
+    name_ = v[0].AsString().str();
+    if (!nodedef_.ParseFromString(v[1].AsString().str())) {
+      nodedef_.Clear();
+      return tensorflow::errors::Internal(
+          "Failed to parse data into a valid NodeDef");
+    }
+
+    // Fill NodeDef with defaults if it's a valid op.
+    const tensorflow::OpRegistrationData* op_reg_data;
+    TF_RETURN_IF_ERROR(
+        tensorflow::OpRegistry::Global()->LookUp(nodedef_.op(), &op_reg_data));
+    AddDefaultsToNodeDef(op_reg_data->op_def, &nodedef_);
+
+    return tensorflow::Status::OK();
+  }
+
+ private:
   // The name of the TensorFlow op to execute.
-  string name;
+  string name_;
   // Index of this node into TF Lite's operator list.
-  int index;
+  int index_;
   // The corresponding NodeDef, containing the attributes for the op.
-  tensorflow::NodeDef nodedef;
+  tensorflow::NodeDef nodedef_;
   // List of inputs, as TF Lite tensor indices.
-  std::vector<int> inputs;
+  std::vector<int> inputs_;
   // List of outputs, as TF Lite tensor indices.
-  std::vector<int> outputs;
+  std::vector<int> outputs_;
 };
 
 // Executes the TensorFlow op given by 'op_name', with the attributes specified
@@ -148,37 +197,6 @@ tensorflow::Status ExecuteFlexOp(tensorflow::EagerContext* eager_context,
   return tensorflow::Status::OK();
 }
 
-tensorflow::Status InitializeNodeDef(const void* custom_initial_data,
-                                     int custom_initial_data_size,
-                                     OpNode* node_data) {
-  if (!custom_initial_data) {
-    return tensorflow::errors::Internal(
-        "Cannot convert empty data into a valid NodeDef");
-  }
-  // The flexbuffer contains a vector where the first elements is the
-  // op name and the second is a serialized NodeDef.
-  const flexbuffers::Vector& v =
-      flexbuffers::GetRoot(
-          reinterpret_cast<const uint8_t*>(custom_initial_data),
-          custom_initial_data_size)
-          .AsVector();
-
-  node_data->name = v[0].AsString().str();
-  if (!node_data->nodedef.ParseFromString(v[1].AsString().str())) {
-    node_data->nodedef.Clear();
-    return tensorflow::errors::Internal(
-        "Failed to parse data into a valid NodeDef");
-  }
-
-  // Fill NodeDef with defaults if it's a valid op.
-  const tensorflow::OpRegistrationData* op_reg_data;
-  TF_RETURN_IF_ERROR(tensorflow::OpRegistry::Global()->LookUp(
-      node_data->nodedef.op(), &op_reg_data));
-  AddDefaultsToNodeDef(op_reg_data->op_def, &node_data->nodedef);
-
-  return tensorflow::Status::OK();
-}
-
 // The larger 'op', which contains all the nodes in a supported subgraph.
 struct OpData {
   tensorflow::EagerContext* eager_context;
@@ -222,18 +240,18 @@ void* Init(TfLiteContext* context, const char* buffer, size_t length) {
     op_data->nodes.push_back(OpNode());
     OpNode& node_data = op_data->nodes.back();
 
-    node_data.index = node_index;
-    node_data.name = "";
+    node_data.set_index(node_index);
+    node_data.set_name("");
 
-    status = InitializeNodeDef(node->custom_initial_data,
-                               node->custom_initial_data_size, &node_data);
+    status = node_data.InitializeNodeDef(node->custom_initial_data,
+                                         node->custom_initial_data_size);
     if (!status.ok()) break;
 
     for (auto input_index : TfLiteIntArrayView(node->inputs)) {
-      node_data.inputs.push_back(input_index);
+      node_data.mutable_inputs()->push_back(input_index);
     }
     for (auto output_index : TfLiteIntArrayView(node->outputs)) {
-      node_data.outputs.push_back(output_index);
+      node_data.mutable_outputs()->push_back(output_index);
     }
   }
 
@@ -283,13 +301,13 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
   }
 
   for (const auto& node_data : op_data->nodes) {
-    if (node_data.nodedef.op().empty()) {
+    if (node_data.nodedef().op().empty()) {
       context->ReportError(context, "Invalid NodeDef in Flex op '%s'",
-                           node_data.name.c_str());
+                           node_data.name().c_str());
       return kTfLiteError;
     }
 
-    for (int tensor_index : node_data.inputs) {
+    for (int tensor_index : node_data.inputs()) {
       ++tensor_ref_count[tensor_index];
     }
   }
@@ -329,11 +347,11 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
   for (const auto& node_data : op_data->nodes) {
     SCOPED_TAGGED_OPERATOR_PROFILE(
         reinterpret_cast<profiling::Profiler*>(context->profiler),
-        node_data->name.c_str(), node_data->index);
+        node_data->name().c_str(), node_data->index());
 
-    auto status =
-        ExecuteFlexOp(eager_context, buffer_map, node_data.name,
-                      node_data.nodedef, node_data.inputs, node_data.outputs);
+    auto status = ExecuteFlexOp(eager_context, buffer_map, node_data.name(),
+                                node_data.nodedef(), node_data.inputs(),
+                                node_data.outputs());
     TF_LITE_ENSURE_OK(context, ConvertStatus(context, status));
   }
 
-- 
GitLab


From 1bad2986b24e520b6b1558d5970b4b25abff0ab3 Mon Sep 17 00:00:00 2001
From: Tong Shen <endlessroad@google.com>
Date: Thu, 3 Jan 2019 17:22:28 -0800
Subject: [PATCH 0206/1095] Support outside compilation in TPUPartitionedCall.

PiperOrigin-RevId: 227777744
---
 tensorflow/compiler/tf2xla/BUILD              |  1 +
 .../compiler/tf2xla/side_effect_util.cc       | 39 +++++++++++++++++++
 tensorflow/compiler/tf2xla/side_effect_util.h |  4 ++
 3 files changed, 44 insertions(+)

diff --git a/tensorflow/compiler/tf2xla/BUILD b/tensorflow/compiler/tf2xla/BUILD
index d8123e956f..0366ec45fb 100644
--- a/tensorflow/compiler/tf2xla/BUILD
+++ b/tensorflow/compiler/tf2xla/BUILD
@@ -669,6 +669,7 @@ cc_library(
     name = "side_effect_util",
     srcs = ["side_effect_util.cc"],
     hdrs = ["side_effect_util.h"],
+    visibility = [":friends"],
     deps = [
         "//tensorflow/core:core_cpu",
         "@com_google_absl//absl/strings",
diff --git a/tensorflow/compiler/tf2xla/side_effect_util.cc b/tensorflow/compiler/tf2xla/side_effect_util.cc
index b62f8e9115..a9144cfc4f 100644
--- a/tensorflow/compiler/tf2xla/side_effect_util.cc
+++ b/tensorflow/compiler/tf2xla/side_effect_util.cc
@@ -26,6 +26,45 @@ const char kXlaTokenArgNodeName[] = "_xla_token_arg_node";
 
 const char kXlaHasHostTransferAttrName[] = "_xla_has_host_transfer";
 
+Status SetDeviceOrdinalAttributeForNode(Node* node, int device_ordinal) {
+  if (!HasNodeAttr(node->def(), kXlaHasHostTransferAttrName)) {
+    return errors::InvalidArgument("Node ", node->DebugString(),
+                                   " does not have attribute ",
+                                   kXlaHasHostTransferAttrName);
+  }
+
+  if (node->type_string() == "_XlaRecvAtHost" ||
+      node->type_string() == "_XlaSendFromHost") {
+    node->ClearAttr("device_ordinal");
+    node->AddAttr("device_ordinal", device_ordinal);
+  } else if (node->type_string() == "If") {
+    AttrValue device_ordinal_value;
+    device_ordinal_value.set_i(device_ordinal);
+    for (const string& attr_name :
+         std::vector<string>{"then_branch", "else_branch"}) {
+      NameAttrList branch_func;
+      TF_RETURN_IF_ERROR(GetNodeAttr(node->attrs(), attr_name, &branch_func));
+      (*branch_func.mutable_attr())["device_ordinal"] = device_ordinal_value;
+      node->ClearAttr(attr_name);
+      node->AddAttr(attr_name, branch_func);
+    }
+  } else if (node->type_string() == "While") {
+    AttrValue device_ordinal_value;
+    device_ordinal_value.set_i(device_ordinal);
+    for (const string& attr_name : std::vector<string>{"cond", "body"}) {
+      NameAttrList branch_func;
+      TF_RETURN_IF_ERROR(GetNodeAttr(node->attrs(), attr_name, &branch_func));
+      (*branch_func.mutable_attr())["device_ordinal"] = device_ordinal_value;
+      node->ClearAttr(attr_name);
+      node->AddAttr(attr_name, branch_func);
+    }
+  } else {
+    return errors::Internal("Unknown node type to set 'device_ordinal': ",
+                            node->DebugString());
+  }
+  return Status::OK();
+}
+
 std::set<std::string> CalculateTokenInputsForOutputToken(const Graph& g) {
   std::set<std::string> results;
   Node* first_side_effecting_node_on_path = nullptr;
diff --git a/tensorflow/compiler/tf2xla/side_effect_util.h b/tensorflow/compiler/tf2xla/side_effect_util.h
index 7081b362c3..75e1f253fb 100644
--- a/tensorflow/compiler/tf2xla/side_effect_util.h
+++ b/tensorflow/compiler/tf2xla/side_effect_util.h
@@ -38,6 +38,10 @@ extern const char kXlaTokenArgNodeName[];
 // This node have XlaRecvAtHost/XlaSendFromHost in its associated functions.
 extern const char kXlaHasHostTransferAttrName[];
 
+// Sets device ordinal attribute for nodes with attribute
+// `kXlaHasHostTransferAttrName`.
+Status SetDeviceOrdinalAttributeForNode(Node* node, int device_ordinal);
+
 // Calculates side-effect dependencies for the graph's token output.
 // Returns a set of node names representing these dependencies.
 std::set<std::string> CalculateTokenInputsForOutputToken(const Graph& g);
-- 
GitLab


From 7f9d4f0b93f256440662fa12e5508b552537803a Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Thu, 3 Jan 2019 17:31:51 -0800
Subject: [PATCH 0207/1095] Enable Tensorflow custom contraction kernel support
 in XLA single threaded matmul.

PiperOrigin-RevId: 227778830
---
 tensorflow/compiler/xla/service/cpu/BUILD                     | 1 +
 .../xla/service/cpu/runtime_single_threaded_matmul.cc         | 4 ++++
 2 files changed, 5 insertions(+)

diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD
index f49b5110be..a4ca772c7c 100644
--- a/tensorflow/compiler/xla/service/cpu/BUILD
+++ b/tensorflow/compiler/xla/service/cpu/BUILD
@@ -631,6 +631,7 @@ cc_library(
     deps = [
         ":runtime_matvec",
         "//tensorflow/core:framework_lite",
+        "//tensorflow/core/kernels:eigen_contraction_kernel",
         "//third_party/eigen3",
     ],
 )
diff --git a/tensorflow/compiler/xla/service/cpu/runtime_single_threaded_matmul.cc b/tensorflow/compiler/xla/service/cpu/runtime_single_threaded_matmul.cc
index 1ed743afc3..1f7204e67a 100644
--- a/tensorflow/compiler/xla/service/cpu/runtime_single_threaded_matmul.cc
+++ b/tensorflow/compiler/xla/service/cpu/runtime_single_threaded_matmul.cc
@@ -20,6 +20,10 @@ limitations under the License.
 #include "tensorflow/core/platform/dynamic_annotations.h"
 #include "tensorflow/core/platform/types.h"
 
+#if defined(TENSORFLOW_USE_CUSTOM_CONTRACTION_KERNEL)
+#include "tensorflow/core/kernels/eigen_contraction_kernel.h"
+#endif
+
 using tensorflow::int32;
 using tensorflow::int64;
 
-- 
GitLab


From 846dba8e81fde8a47196c9b04d31f847e5cc336f Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Thu, 3 Jan 2019 18:05:00 -0800
Subject: [PATCH 0208/1095] Automated rollback of commit
 64596a68235709e7f3246d0b120a4146521a998b

PiperOrigin-RevId: 227782472
---
 tensorflow/lite/experimental/swift/BUILD      | 100 -----
 tensorflow/lite/experimental/swift/LICENSE    | 202 ----------
 tensorflow/lite/experimental/swift/README.md  |  54 ---
 .../swift/Sources/Interpreter.swift           | 265 --------------
 .../swift/Sources/InterpreterError.swift      |  99 -----
 .../swift/Sources/InterpreterOptions.swift    |  29 --
 .../experimental/swift/Sources/Model.swift    |  40 --
 .../Sources/QuantizationParameters.swift      |  38 --
 .../experimental/swift/Sources/Tensor.swift   | 138 -------
 .../Configs/TensorFlowLite.tulsigen           |  57 ---
 .../project.tulsiconf                         |  14 -
 .../project.pbxproj                           | 345 ------------------
 .../TensorFlowLiteApp/AppDelegate.swift       |  24 --
 .../Array+TensorFlowLite.swift                |  22 --
 .../AppIcon.appiconset/Contents.json          |  98 -----
 .../Assets.xcassets/Contents.json             |   6 -
 .../Base.lproj/LaunchScreen.storyboard        |  44 ---
 .../Base.lproj/Main.storyboard                |  95 -----
 .../Data+TensorFlowLite.swift                 |  13 -
 .../TensorFlowLiteApp/Info.plist              |  46 ---
 .../TensorFlowLiteApp/ViewController.swift    | 299 ---------------
 .../swift/Tests/InterpreterOptionsTests.swift |  54 ---
 .../swift/Tests/InterpreterTests.swift        | 315 ----------------
 .../experimental/swift/Tests/ModelTests.swift |  59 ---
 .../Tests/QuantizationParametersTests.swift   |  43 ---
 .../swift/Tests/TensorTests.swift             |  83 -----
 .../tools/pip_package/pip_smoke_test.py       |  36 +-
 27 files changed, 15 insertions(+), 2603 deletions(-)
 delete mode 100644 tensorflow/lite/experimental/swift/BUILD
 delete mode 100644 tensorflow/lite/experimental/swift/LICENSE
 delete mode 100644 tensorflow/lite/experimental/swift/README.md
 delete mode 100644 tensorflow/lite/experimental/swift/Sources/Interpreter.swift
 delete mode 100644 tensorflow/lite/experimental/swift/Sources/InterpreterError.swift
 delete mode 100644 tensorflow/lite/experimental/swift/Sources/InterpreterOptions.swift
 delete mode 100644 tensorflow/lite/experimental/swift/Sources/Model.swift
 delete mode 100644 tensorflow/lite/experimental/swift/Sources/QuantizationParameters.swift
 delete mode 100644 tensorflow/lite/experimental/swift/Sources/Tensor.swift
 delete mode 100644 tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/Configs/TensorFlowLite.tulsigen
 delete mode 100644 tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/project.tulsiconf
 delete mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp.xcodeproj/project.pbxproj
 delete mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/AppDelegate.swift
 delete mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Array+TensorFlowLite.swift
 delete mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/AppIcon.appiconset/Contents.json
 delete mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/Contents.json
 delete mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/LaunchScreen.storyboard
 delete mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/Main.storyboard
 delete mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Data+TensorFlowLite.swift
 delete mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Info.plist
 delete mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/ViewController.swift
 delete mode 100644 tensorflow/lite/experimental/swift/Tests/InterpreterOptionsTests.swift
 delete mode 100644 tensorflow/lite/experimental/swift/Tests/InterpreterTests.swift
 delete mode 100644 tensorflow/lite/experimental/swift/Tests/ModelTests.swift
 delete mode 100644 tensorflow/lite/experimental/swift/Tests/QuantizationParametersTests.swift
 delete mode 100644 tensorflow/lite/experimental/swift/Tests/TensorTests.swift

diff --git a/tensorflow/lite/experimental/swift/BUILD b/tensorflow/lite/experimental/swift/BUILD
deleted file mode 100644
index 3bd288a1e1..0000000000
--- a/tensorflow/lite/experimental/swift/BUILD
+++ /dev/null
@@ -1,100 +0,0 @@
-# TensorFlow Lite for Swift.
-
-package(default_visibility = ["//visibility:private"])
-
-licenses(["notice"])  # Apache 2.0
-
-exports_files(["LICENSE"])
-
-load("@build_bazel_rules_apple//apple:ios.bzl", "ios_application", "ios_unit_test")
-load("@build_bazel_rules_swift//swift:swift.bzl", "swift_library")
-
-MINIMUM_OS_VERSION = "9.0"
-
-SWIFT_COPTS = [
-    "-wmo",
-]
-
-swift_library(
-    name = "TensorFlowLite",
-    srcs = glob(["Sources/*.swift"]),
-    copts = SWIFT_COPTS,
-    module_name = "TensorFlowLite",
-    tags = ["manual"],
-    deps = [
-        "//tensorflow/lite/experimental/c:c_api",
-    ],
-)
-
-ios_unit_test(
-    name = "TensorFlowLiteTests",
-    size = "small",
-    minimum_os_version = MINIMUM_OS_VERSION,
-    tags = [
-        "manual",
-        # DISABLED: Following sanitizer tests are not supported by iOS test targets.
-        "noasan",
-        "nomsan",
-        "notsan",
-    ],
-    deps = [":TensorFlowLiteTestsLib"],
-)
-
-swift_library(
-    name = "TensorFlowLiteTestsLib",
-    testonly = 1,
-    srcs = glob(["Tests/*.swift"]),
-    copts = SWIFT_COPTS,
-    tags = ["manual"],
-    deps = [
-        ":TensorFlowLite",
-        ":TestResources",
-    ],
-)
-
-objc_library(
-    name = "TestResources",
-    resources = [
-        "//tensorflow/lite:testdata/add.bin",
-        "//tensorflow/lite:testdata/add_quantized.bin",
-        "//tensorflow/lite:testdata/multi_add.bin",
-    ],
-    tags = ["manual"],
-)
-
-ios_application(
-    name = "TensorFlowLiteApp",
-    app_icons = glob(["TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/AppIcon.appiconset/**"]),
-    bundle_id = "com.tensorflow.lite.swift.TensorFlowLite",
-    families = [
-        "ipad",
-        "iphone",
-    ],
-    infoplists = ["TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Info.plist"],
-    launch_storyboard = "TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/LaunchScreen.storyboard",
-    minimum_os_version = MINIMUM_OS_VERSION,
-    sdk_frameworks = [
-        "CoreGraphics",
-    ],
-    tags = ["manual"],
-    deps = [":TensorFlowLiteAppLib"],
-)
-
-swift_library(
-    name = "TensorFlowLiteAppLib",
-    srcs = glob(["TestApps/TensorFlowLiteApp/TensorFlowLiteApp/*.swift"]),
-    tags = ["manual"],
-    deps = [
-        ":TensorFlowLite",
-        ":TensorFlowLiteAppResources",
-    ],
-)
-
-objc_library(
-    name = "TensorFlowLiteAppResources",
-    storyboards = glob([
-        "TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/*.storyboard",
-    ]),
-    tags = ["manual"],
-    deps = [":TestResources"],
-)
diff --git a/tensorflow/lite/experimental/swift/LICENSE b/tensorflow/lite/experimental/swift/LICENSE
deleted file mode 100644
index d645695673..0000000000
--- a/tensorflow/lite/experimental/swift/LICENSE
+++ /dev/null
@@ -1,202 +0,0 @@
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
diff --git a/tensorflow/lite/experimental/swift/README.md b/tensorflow/lite/experimental/swift/README.md
deleted file mode 100644
index deb16c1e91..0000000000
--- a/tensorflow/lite/experimental/swift/README.md
+++ /dev/null
@@ -1,54 +0,0 @@
-# TensorFlow Lite for Swift
-
-[TensorFlow Lite](https://www.tensorflow.org/lite/) is TensorFlow's lightweight
-solution for Swift developers. It enables low-latency inference of on-device
-machine learning models with a small binary size and fast performance supporting
-hardware acceleration.
-
-## Getting Started
-
-### Bazel
-
-In your `BUILD` file, add the `TensorFlowLite` dependency:
-
-```python
-swift_library(
-  # ...
-  deps = [
-      "//tensorflow/lite/swift:TensorFlowLite",
-  ],
-)
-```
-
-In your Swift files, import the module:
-
-```swift
-import TensorFlowLite
-```
-
-### Tulsi
-
-Open the `TensorFlowLite.tulsiproj` using the [TulsiApp](https://github.com/bazelbuild/tulsi) or by
-running the [`generate_xcodeproj.sh`](https://github.com/bazelbuild/tulsi/blob/master/src/tools/generate_xcodeproj.sh)
-script:
-
-```shell
-generate_xcodeproj.sh --genconfig tensorflow/lite/swift/TensorFlowLite.tulsiproj:TensorFlowLite --outputfolder ~/path/to/generated/TensorFlowLite.xcodeproj
-```
-
-### CocoaPods
-
-Add the following to your `Podfile`:
-
-```ruby
-use_frameworks!
-pod 'TensorFlowLiteSwift'
-```
-
-Then, run `pod install`.
-
-In your Swift files, import the module:
-
-```swift
-import TensorFlowLite
-```
diff --git a/tensorflow/lite/experimental/swift/Sources/Interpreter.swift b/tensorflow/lite/experimental/swift/Sources/Interpreter.swift
deleted file mode 100644
index a14b5966b1..0000000000
--- a/tensorflow/lite/experimental/swift/Sources/Interpreter.swift
+++ /dev/null
@@ -1,265 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import Foundation
-import TensorFlowLiteCAPI
-
-/// A TensorFlow Lite interpreter that performs inference from a given model.
-public final class Interpreter {
-
-  /// The `TFL_Interpreter` C pointer type represented as an `UnsafePointer<TFL_Interpreter>`.
-  private typealias CInterpreter = OpaquePointer
-
-  /// Total number of input tensors associated with the model.
-  public var inputTensorCount: Int {
-    return Int(TFL_InterpreterGetInputTensorCount(cInterpreter))
-  }
-
-  /// Total number of output tensors associated with the model.
-  public var outputTensorCount: Int {
-    return Int(TFL_InterpreterGetOutputTensorCount(cInterpreter))
-  }
-
-  /// The underlying `TFL_Interpreter` C pointer.
-  private var cInterpreter: CInterpreter?
-
-  /// Creates a new model interpreter instance.
-  ///
-  /// - Parameters:
-  ///   - modelPath: Local file path to a TensorFlow Lite model.
-  ///   - options: Custom configurations for the interpreter. The default is `nil` indicating that
-  ///       interpreter will determine the configuration options.
-  /// - Throws: An error if the model could not be loaded or the interpreter could not be created.
-  public init(modelPath: String, options: InterpreterOptions? = nil) throws {
-    guard let model = Model(filePath: modelPath) else { throw InterpreterError.failedToLoadModel }
-
-    let cInterpreterOptions: OpaquePointer? = try options.map { options in
-      guard let cOptions = TFL_NewInterpreterOptions() else {
-        throw InterpreterError.failedToCreateInterpreter
-      }
-      if let threadCount = options.threadCount, threadCount > 0 {
-        TFL_InterpreterOptionsSetNumThreads(cOptions, Int32(threadCount))
-      }
-      if options.isErrorLoggingEnabled {
-        TFL_InterpreterOptionsSetErrorReporter(
-          cOptions,
-          { (_, format, arguments) in
-            guard let cFormat = format,
-                  let message = String(cFormat: cFormat, arguments: arguments)
-            else {
-              return
-            }
-            print(String(describing: InterpreterError.tensorFlowLiteError(message)))
-          },
-          nil
-        )
-      }
-      return cOptions
-    }
-    defer { TFL_DeleteInterpreterOptions(cInterpreterOptions) }
-
-    guard let cInterpreter = TFL_NewInterpreter(model.cModel, cInterpreterOptions) else {
-      throw InterpreterError.failedToCreateInterpreter
-    }
-    self.cInterpreter = cInterpreter
-  }
-
-  deinit {
-    TFL_DeleteInterpreter(cInterpreter)
-  }
-
-  /// Invokes the interpreter to perform inference from the loaded graph.
-  ///
-  /// - Throws: An error if the model was not ready because tensors were not allocated.
-  public func invoke() throws {
-    guard TFL_InterpreterInvoke(cInterpreter) == kTfLiteOk else {
-      // TODO(b/117510052): Determine which error to throw.
-      throw InterpreterError.allocateTensorsRequired
-    }
-  }
-
-  /// Returns the input tensor at the given index.
-  ///
-  /// - Parameters:
-  ///   - index: The index for the input tensor.
-  /// - Throws: An error if the index is invalid or the tensors have not been allocated.
-  /// - Returns: The input tensor at the given index.
-  public func input(at index: Int) throws -> Tensor {
-    let maxIndex = inputTensorCount - 1
-    guard case 0...maxIndex = index else {
-      throw InterpreterError.invalidTensorIndex(index: index, maxIndex: maxIndex)
-    }
-    guard let cTensor = TFL_InterpreterGetInputTensor(cInterpreter, Int32(index)),
-          let bytes = TFL_TensorData(cTensor),
-          let nameCString = TFL_TensorName(cTensor)
-    else {
-      throw InterpreterError.allocateTensorsRequired
-    }
-    guard let dataType = TensorDataType(type: TFL_TensorType(cTensor)) else {
-      throw InterpreterError.invalidTensorDataType
-    }
-
-    let name = String(cString: nameCString)
-    let rank = TFL_TensorNumDims(cTensor)
-    let dimensions = (0..<rank).map { Int(TFL_TensorDim(cTensor, $0)) }
-    let shape = TensorShape(dimensions)
-    let byteCount = TFL_TensorByteSize(cTensor)
-    let data = Data(bytes: bytes, count: byteCount)
-    let cQuantizationParams = TFL_TensorQuantizationParams(cTensor)
-    let scale = cQuantizationParams.scale
-    let zeroPoint = Int(cQuantizationParams.zero_point)
-    var quantizationParameters: QuantizationParameters? = nil
-    if scale != 0.0 {
-      // TODO(b/117510052): Update this check once the TfLiteQuantizationParams struct has a mode.
-      quantizationParameters = QuantizationParameters(scale: scale, zeroPoint: zeroPoint)
-    }
-    let tensor = Tensor(
-      name: name,
-      dataType: dataType,
-      shape: shape,
-      data: data,
-      quantizationParameters: quantizationParameters
-    )
-    return tensor
-  }
-
-  /// Returns the output tensor at the given index.
-  ///
-  /// - Parameters:
-  ///   - index: The index for the output tensor.
-  /// - Throws: An error if the index is invalid, tensors haven't been allocated, or interpreter
-  ///     hasn't been invoked for models that dynamically compute output tensors based on the values
-  ///     of its input tensors.
-  /// - Returns: The output tensor at the given index.
-  public func output(at index: Int) throws -> Tensor {
-    let maxIndex = outputTensorCount - 1
-    guard case 0...maxIndex = index else {
-      throw InterpreterError.invalidTensorIndex(index: index, maxIndex: maxIndex)
-    }
-    guard let cTensor = TFL_InterpreterGetOutputTensor(cInterpreter, Int32(index)),
-          let bytes = TFL_TensorData(cTensor),
-          let nameCString = TFL_TensorName(cTensor)
-    else {
-      // TODO(b/117510052): Determine which error to throw.
-      throw InterpreterError.invokeInterpreterRequired
-    }
-    guard let dataType = TensorDataType(type: TFL_TensorType(cTensor)) else {
-      throw InterpreterError.invalidTensorDataType
-    }
-
-    let name = String(cString: nameCString)
-    let rank = TFL_TensorNumDims(cTensor)
-    let dimensions = (0..<rank).map { Int(TFL_TensorDim(cTensor, $0)) }
-    let shape = TensorShape(dimensions)
-    let byteCount = TFL_TensorByteSize(cTensor)
-    let data = Data(bytes: bytes, count: byteCount)
-    let cQuantizationParams = TFL_TensorQuantizationParams(cTensor)
-    let scale = cQuantizationParams.scale
-    let zeroPoint = Int(cQuantizationParams.zero_point)
-    var quantizationParameters: QuantizationParameters? = nil
-    if scale != 0.0 {
-      // TODO(b/117510052): Update this check once the TfLiteQuantizationParams struct has a mode.
-      quantizationParameters = QuantizationParameters(scale: scale, zeroPoint: zeroPoint)
-    }
-    let tensor = Tensor(
-      name: name,
-      dataType: dataType,
-      shape: shape,
-      data: data,
-      quantizationParameters: quantizationParameters
-    )
-    return tensor
-  }
-
-  /// Resizes the input tensor at the given index to the specified tensor shape.
-  ///
-  /// - Note: After resizing an input tensor, the client **must** explicitly call
-  ///     `allocateTensors()` before attempting to access the resized tensor data or invoking the
-  ///     interpreter to perform inference.
-  /// - Parameters:
-  ///   - index: The index for the input tensor.
-  ///   - shape: The shape that the input tensor should be resized to.
-  /// - Throws: An error if the input tensor at the given index could not be resized.
-  public func resizeInput(at index: Int, to shape: TensorShape) throws {
-    let maxIndex = inputTensorCount - 1
-    guard case 0...maxIndex = index else {
-      throw InterpreterError.invalidTensorIndex(index: index, maxIndex: maxIndex)
-    }
-    guard TFL_InterpreterResizeInputTensor(
-            cInterpreter,
-            Int32(index),
-            shape.int32Dimensions,
-            Int32(shape.rank)
-          ) == kTfLiteOk
-    else {
-      throw InterpreterError.failedToResizeInputTensor(index: index)
-    }
-  }
-
-  /// Copies the given data to the input tensor at the given index.
-  ///
-  /// - Parameters:
-  ///   - data: The data to be copied to the input tensor's data buffer.
-  ///   - index: The index for the input tensor.
-  /// - Throws: An error if the `data.count` does not match the input tensor's `data.count` or if
-  ///     the given index is invalid.
-  /// - Returns: The input tensor with the copied data.
-  @discardableResult
-  public func copy(_ data: Data, toInputAt index: Int) throws -> Tensor {
-    let maxIndex = inputTensorCount - 1
-    guard case 0...maxIndex = index else {
-      throw InterpreterError.invalidTensorIndex(index: index, maxIndex: maxIndex)
-    }
-    guard let cTensor = TFL_InterpreterGetInputTensor(cInterpreter, Int32(index)) else {
-      throw InterpreterError.allocateTensorsRequired
-    }
-
-    let byteCount = TFL_TensorByteSize(cTensor)
-    guard data.count == byteCount else {
-      throw InterpreterError.invalidTensorDataCount(provided: data.count, required: byteCount)
-    }
-
-    let status = data.withUnsafeBytes { TFL_TensorCopyFromBuffer(cTensor, $0, data.count) }
-    guard status == kTfLiteOk else { throw InterpreterError.failedToCopyDataToInputTensor }
-    return try input(at: index)
-  }
-
-  /// Allocates memory for all input tensors based on their `TensorShape`s.
-  ///
-  /// - Note: This is a relatively expensive operation and should only be called after creating the
-  ///     interpreter and/or resizing any input tensors.
-  /// - Throws: An error if memory could not be allocated for the input tensors.
-  public func allocateTensors() throws {
-    guard TFL_InterpreterAllocateTensors(cInterpreter) == kTfLiteOk else {
-      throw InterpreterError.failedToAllocateTensors
-    }
-  }
-}
-
-// MARK: - Extensions
-
-extension String {
-  /// Returns a new `String` initialized by using the given format C array as a template into which
-  /// the remaining argument values are substituted according to the user’s default locale.
-  ///
-  /// - Note: Returns `nil` if a new `String` could not be constructed from the given values.
-  /// - Parameters:
-  ///   - cFormat: The format C array as a template for substituting values.
-  ///   - arguments: A C pointer to a `va_list` of arguments to substitute into `cFormat`.
-  init?(cFormat: UnsafePointer<CChar>, arguments: CVaListPointer) {
-    var buffer: UnsafeMutablePointer<CChar>?
-    guard vasprintf(&buffer, cFormat, arguments) != 0, let cString = buffer else { return nil }
-    self.init(validatingUTF8: cString)
-  }
-}
diff --git a/tensorflow/lite/experimental/swift/Sources/InterpreterError.swift b/tensorflow/lite/experimental/swift/Sources/InterpreterError.swift
deleted file mode 100644
index 5de58b997a..0000000000
--- a/tensorflow/lite/experimental/swift/Sources/InterpreterError.swift
+++ /dev/null
@@ -1,99 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import Foundation
-
-/// TensorFlow Lite interpreter errors.
-public enum InterpreterError: Error {
-  case invalidTensorIndex(index: Int, maxIndex: Int)
-  case invalidTensorDataCount(provided: Int, required: Int)
-  case invalidTensorDataType
-  case failedToLoadModel
-  case failedToCreateInterpreter
-  case failedToResizeInputTensor(index: Int)
-  case failedToCopyDataToInputTensor
-  case failedToAllocateTensors
-  case allocateTensorsRequired
-  case invokeInterpreterRequired
-  case tensorFlowLiteError(String)
-}
-
-// MARK: - Extensions
-
-extension InterpreterError: LocalizedError {
-  /// Localized description of the interpreter error.
-  public var errorDescription: String? {
-    switch self {
-    case .invalidTensorIndex(let index, let maxIndex):
-      return "Invalid tensor index \(index), max index is \(maxIndex)."
-    case .invalidTensorDataCount(let providedCount, let requiredCount):
-      return "Provided data count \(providedCount) must match the required count \(requiredCount)."
-    case .invalidTensorDataType:
-      return "Tensor data type is unsupported or could not be determined because of a model error."
-    case .failedToLoadModel:
-      return "Failed to load the given model."
-    case .failedToCreateInterpreter:
-      return "Failed to create the interpreter."
-    case .failedToResizeInputTensor(let index):
-      return "Failed to resize input tesnor at index \(index)."
-    case .failedToCopyDataToInputTensor:
-      return "Failed to copy data to input tensor."
-    case .failedToAllocateTensors:
-      return "Failed to allocate memory for input tensors."
-    case .allocateTensorsRequired:
-      return "Must call allocateTensors()."
-    case .invokeInterpreterRequired:
-      return "Must call invoke()."
-    case .tensorFlowLiteError(let message):
-      return "TensorFlow Lite Error: \(message)"
-    }
-  }
-}
-
-extension InterpreterError: CustomStringConvertible {
-  /// Textual representation of the TensorFlow Lite interpreter error.
-  public var description: String {
-    return errorDescription ?? "Unknown error."
-  }
-}
-
-#if swift(>=4.2)
-extension InterpreterError: Equatable {}
-#else
-extension InterpreterError: Equatable {
-  public static func == (lhs: InterpreterError, rhs: InterpreterError) -> Bool {
-    switch (lhs, rhs) {
-    case (.invalidTensorDataType, .invalidTensorDataType),
-         (.failedToLoadModel, .failedToLoadModel),
-         (.failedToCreateInterpreter, .failedToCreateInterpreter),
-         (.failedToAllocateTensors, .failedToAllocateTensors),
-         (.allocateTensorsRequired, .allocateTensorsRequired),
-         (.invokeInterpreterRequired, .invokeInterpreterRequired):
-      return true
-    case (.invalidTensorIndex(let lhsIndex, let lhsMaxIndex),
-          .invalidTensorIndex(let rhsIndex, let rhsMaxIndex)):
-      return lhsIndex == rhsIndex && lhsMaxIndex == rhsMaxIndex
-    case (.invalidTensorDataCount(let lhsProvidedCount, let lhsRequiredCount),
-          .invalidTensorDataCount(let rhsProvidedCount, let rhsRequiredCount)):
-      return lhsProvidedCount == rhsProvidedCount && lhsRequiredCount == rhsRequiredCount
-    case (.failedToResizeInputTensor(let lhsIndex), .failedToResizeInputTensor(let rhsIndex)):
-      return lhsIndex == rhsIndex
-    case (.tensorFlowLiteError(let lhsMessage), .tensorFlowLiteError(let rhsMessage)):
-      return lhsMessage == rhsMessage
-    default:
-      return false
-    }
-  }
-}
-#endif  // swift(>=4.2)
diff --git a/tensorflow/lite/experimental/swift/Sources/InterpreterOptions.swift b/tensorflow/lite/experimental/swift/Sources/InterpreterOptions.swift
deleted file mode 100644
index 2365fd7ade..0000000000
--- a/tensorflow/lite/experimental/swift/Sources/InterpreterOptions.swift
+++ /dev/null
@@ -1,29 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import Foundation
-
-/// Custom configuration options for a TensorFlow Lite interpreter.
-public struct InterpreterOptions: Equatable {
-
-  /// Maximum number of CPU threads that the interpreter should run on. Default is `nil` which
-  /// indicates that the `Interpreter` will decide the number of threads to use.
-  public var threadCount: Int? = nil
-
-  /// Whether error logging to the console is enabled. The default is `false`.
-  public var isErrorLoggingEnabled = false
-
-  /// Creates a new instance of interpreter options.
-  public init() {}
-}
diff --git a/tensorflow/lite/experimental/swift/Sources/Model.swift b/tensorflow/lite/experimental/swift/Sources/Model.swift
deleted file mode 100644
index e8c49ff1ae..0000000000
--- a/tensorflow/lite/experimental/swift/Sources/Model.swift
+++ /dev/null
@@ -1,40 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import Foundation
-import TensorFlowLiteCAPI
-
-/// A TensorFlow Lite model used by the 'Interpreter` to perform inference.
-final class Model {
-
-  /// The `TFL_Model` C pointer type represented as an `UnsafePointer<TFL_Model>`.
-  typealias CModel = OpaquePointer
-
-  /// The underlying `TFL_Model` C pointer.
-  let cModel: CModel?
-
-  /// Creates a new model instance.
-  ///
-  /// - Precondition: Initialization can fail if the given `filePath` is invalid.
-  /// - Parameters:
-  ///   - filePath: Local file path to a TensorFlow Lite model.
-  init?(filePath: String) {
-    guard !filePath.isEmpty, let cModel = TFL_NewModelFromFile(filePath) else { return nil }
-    self.cModel = cModel
-  }
-
-  deinit {
-    TFL_DeleteModel(cModel)
-  }
-}
diff --git a/tensorflow/lite/experimental/swift/Sources/QuantizationParameters.swift b/tensorflow/lite/experimental/swift/Sources/QuantizationParameters.swift
deleted file mode 100644
index f367875644..0000000000
--- a/tensorflow/lite/experimental/swift/Sources/QuantizationParameters.swift
+++ /dev/null
@@ -1,38 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import Foundation
-
-/// Parameters that determine the mapping of quantized values to real values. Quantized values can
-/// be mapped to float values using the following conversion:
-/// `realValue = scale * (quantizedValue - zeroPoint)`.
-public struct QuantizationParameters {
-
-  /// Difference between real values corresponding to consecutive quantized values differing by 1.
-  /// For example, the range of quantized values for `UInt8` data type is [0, 255].
-  public let scale: Float
-
-  /// Quantized value that corresponds to the real 0 value.
-  public let zeroPoint: Int
-
-  /// Creates a new quantization parameters instance.
-  ///
-  /// - Parameters:
-  ///   - scale: Scale value for asymmetric quantization.
-  ///   - zeroPoint: Zero point for asymmetric quantization.
-  init(scale: Float, zeroPoint: Int) {
-    self.scale = scale
-    self.zeroPoint = zeroPoint
-  }
-}
diff --git a/tensorflow/lite/experimental/swift/Sources/Tensor.swift b/tensorflow/lite/experimental/swift/Sources/Tensor.swift
deleted file mode 100644
index b738d87549..0000000000
--- a/tensorflow/lite/experimental/swift/Sources/Tensor.swift
+++ /dev/null
@@ -1,138 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import Foundation
-import TensorFlowLiteCAPI
-
-/// An input or output tensor in a TensorFlow Lite graph.
-public struct Tensor {
-
-  /// Name of the tensor.
-  public let name: String
-
-  /// Data type of the tensor.
-  public let dataType: TensorDataType
-
-  /// Shape of the tensor.
-  public let shape: TensorShape
-
-  /// Data in the input or output tensor.
-  public let data: Data
-
-  /// Quantization parameters for the tensor if using a quantized model.
-  public let quantizationParameters: QuantizationParameters?
-
-  /// Creates a new input or output tensor instance.
-  ///
-  /// - Parameters:
-  ///   - name: Name of the tensor.
-  ///   - dataType: Data type of the tensor.
-  ///   - data: Data in the input tensor.
-  ///   - quantizationParameters Quantization parameters for the tensor if using a quantized model.
-  ///       The default is `nil`.
-  init(
-    name: String,
-    dataType: TensorDataType,
-    shape: TensorShape,
-    data: Data,
-    quantizationParameters: QuantizationParameters? = nil
-  ) {
-    self.name = name
-    self.dataType = dataType
-    self.shape = shape
-    self.data = data
-    self.quantizationParameters = quantizationParameters
-  }
-}
-
-/// Supported TensorFlow Lite tensor data types.
-public enum TensorDataType: Equatable {
-  /// 32-bit single precision floating point tensor data type.
-  case float32
-  /// 8-bit unsigned integer tensor data type.
-  case uInt8
-  /// 16-bit signed integer tensor data type.
-  case int16
-  /// 32-bit signed integer tensor data type.
-  case int32
-  /// 64-bit signed integer tensor data type.
-  case int64
-  /// Boolean tensor data type.
-  case bool
-
-  /// Creates a new tensor data type from the given `TFL_Type` or `nil` if the data type is
-  /// unsupported or could not be determined because there was an error.
-  ///
-  /// - Parameter type: A data type supported by a tensor.
-  init?(type: TFL_Type) {
-    switch type {
-    case kTfLiteFloat32:
-      self = .float32
-    case kTfLiteUInt8:
-      self = .uInt8
-    case kTfLiteInt16:
-      self = .int16
-    case kTfLiteInt32:
-      self = .int32
-    case kTfLiteInt64:
-      self = .int64
-    case kTfLiteBool:
-      self = .bool
-    case kTfLiteNoType:
-      fallthrough
-    default:
-      return nil
-    }
-  }
-}
-
-/// The shape of a TensorFlow Lite tensor.
-public struct TensorShape {
-
-  /// The number of dimensions of the tensor.
-  public let rank: Int
-
-  /// Array of dimensions for the tensor.
-  public let dimensions: [Int]
-
-  /// Array of `Int32` dimensions for the tensor.
-  var int32Dimensions: [Int32] { return dimensions.map(Int32.init) }
-
-  /// Creates a new tensor shape instance with the given array of dimensions.
-  ///
-  /// - Parameters:
-  ///   - dimensions: Dimensions for the tensor.
-  public init(_ dimensions: [Int]) {
-    self.rank = dimensions.count
-    self.dimensions = dimensions
-  }
-
-  /// Creates a new tensor shape instance with the given elements representing the dimensions.
-  ///
-  /// - Parameters:
-  ///   - elements: Dimensions for the tensor.
-  public init(_ elements: Int...) {
-    self.init(elements)
-  }
-}
-
-extension TensorShape: ExpressibleByArrayLiteral {
-  /// Creates a new tensor shape instance with the given array literal representing the dimensions.
-  ///
-  /// - Parameters:
-  ///   - arrayLiteral: Dimensions for the tensor.
-  public init(arrayLiteral: Int...) {
-    self.init(arrayLiteral)
-  }
-}
diff --git a/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/Configs/TensorFlowLite.tulsigen b/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/Configs/TensorFlowLite.tulsigen
deleted file mode 100644
index 4010fab49e..0000000000
--- a/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/Configs/TensorFlowLite.tulsigen
+++ /dev/null
@@ -1,57 +0,0 @@
-{
-  "sourceFilters" : [
-    "third_party/tensorflow/lite/experimental/c",
-    "third_party/tensorflow/lite/experimental/swift",
-    "third_party/tensorflow/lite/experimental/swift/Sources",
-    "third_party/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp",
-    "third_party/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj",
-    "third_party/tensorflow/lite/experimental/swift/Tests",
-  ],
-  "buildTargets" : [
-    "//third_party/tensorflow/lite/experimental/swift:TensorFlowLite",
-    "//third_party/tensorflow/lite/experimental/swift:TensorFlowLiteApp",
-    "//third_party/tensorflow/lite/experimental/swift:TensorFlowLiteTests",
-  ],
-  "projectName" : "TensorFlowLite",
-  "optionSet" : {
-    "LaunchActionPreActionScript" : {
-      "p" : "$(inherited)"
-    },
-    "BazelBuildStartupOptionsRelease" : {
-      "p" : "$(inherited)"
-    },
-    "BazelBuildOptionsRelease" : {
-      "p" : "$(inherited)"
-    },
-    "BazelBuildOptionsDebug" : {
-      "p" : "$(inherited)"
-    },
-    "EnvironmentVariables" : {
-      "p" : "$(inherited)"
-    },
-    "BuildActionPreActionScript" : {
-      "p" : "$(inherited)"
-    },
-    "CommandlineArguments" : {
-      "p" : "$(inherited)"
-    },
-    "TestActionPreActionScript" : {
-      "p" : "$(inherited)"
-    },
-    "BazelBuildStartupOptionsDebug" : {
-      "p" : "$(inherited)"
-    },
-    "BuildActionPostActionScript" : {
-      "p" : "$(inherited)"
-    },
-    "TestActionPostActionScript" : {
-      "p" : "$(inherited)"
-    },
-    "LaunchActionPostActionScript" : {
-      "p" : "$(inherited)"
-    }
-  },
-  "additionalFilePaths" : [
-    "third_party/tensorflow/lite/experimental/swift/BUILD"
-  ]
-}
diff --git a/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/project.tulsiconf b/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/project.tulsiconf
deleted file mode 100644
index 14cff94453..0000000000
--- a/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/project.tulsiconf
+++ /dev/null
@@ -1,14 +0,0 @@
-{
-  "configDefaults" : {
-    "optionSet" : {
-      "ProjectPrioritizesSwift" : {
-        "p" : "YES"
-      }
-    }
-  },
-  "projectName" : "TensorFlowLite",
-  "packages" : [
-    "third_party/tensorflow/lite/experimental/swift"
-  ],
-  "workspaceRoot" : "../../../../../.."
-}
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp.xcodeproj/project.pbxproj b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp.xcodeproj/project.pbxproj
deleted file mode 100644
index fbbf9a1de2..0000000000
--- a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp.xcodeproj/project.pbxproj
+++ /dev/null
@@ -1,345 +0,0 @@
-// !$*UTF8*$!
-{
-	archiveVersion = 1;
-	classes = {
-	};
-	objectVersion = 50;
-	objects = {
-
-/* Begin PBXBuildFile section */
-		4A7304B421500B8400C90B21 /* Data+TensorFlowLite.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4A7304B321500B8300C90B21 /* Data+TensorFlowLite.swift */; };
-		4AA72B732146ED64006C3AEF /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4AA72B722146ED64006C3AEF /* AppDelegate.swift */; };
-		4AA72B752146ED64006C3AEF /* ViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4AA72B742146ED64006C3AEF /* ViewController.swift */; };
-		4AA72B782146ED64006C3AEF /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 4AA72B762146ED64006C3AEF /* Main.storyboard */; };
-		4AA72B7A2146ED66006C3AEF /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 4AA72B792146ED66006C3AEF /* Assets.xcassets */; };
-		4AA72B7D2146ED66006C3AEF /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 4AA72B7B2146ED66006C3AEF /* LaunchScreen.storyboard */; };
-		4ADDE0CE2176600E00FF07A2 /* Array+TensorFlowLite.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4ADDE0CD2176600900FF07A2 /* Array+TensorFlowLite.swift */; };
-/* End PBXBuildFile section */
-
-/* Begin PBXFileReference section */
-		4A7304B321500B8300C90B21 /* Data+TensorFlowLite.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "Data+TensorFlowLite.swift"; sourceTree = "<group>"; };
-		4AA72B6F2146ED64006C3AEF /* TensorFlowLiteApp.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = TensorFlowLiteApp.app; sourceTree = BUILT_PRODUCTS_DIR; };
-		4AA72B722146ED64006C3AEF /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = "<group>"; };
-		4AA72B742146ED64006C3AEF /* ViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ViewController.swift; sourceTree = "<group>"; };
-		4AA72B772146ED64006C3AEF /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/Main.storyboard; sourceTree = "<group>"; };
-		4AA72B792146ED66006C3AEF /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = "<group>"; };
-		4AA72B7C2146ED66006C3AEF /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = "<group>"; };
-		4AA72B7E2146ED66006C3AEF /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
-		4ADDE0CD2176600900FF07A2 /* Array+TensorFlowLite.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "Array+TensorFlowLite.swift"; sourceTree = "<group>"; };
-/* End PBXFileReference section */
-
-/* Begin PBXFrameworksBuildPhase section */
-		4AA72B6C2146ED64006C3AEF /* Frameworks */ = {
-			isa = PBXFrameworksBuildPhase;
-			buildActionMask = 2147483647;
-			files = (
-			);
-			runOnlyForDeploymentPostprocessing = 0;
-		};
-/* End PBXFrameworksBuildPhase section */
-
-/* Begin PBXGroup section */
-		4AA72B662146ED64006C3AEF = {
-			isa = PBXGroup;
-			children = (
-				4AA72B712146ED64006C3AEF /* TensorFlowLiteApp */,
-				4AA72B702146ED64006C3AEF /* Products */,
-			);
-			sourceTree = "<group>";
-		};
-		4AA72B702146ED64006C3AEF /* Products */ = {
-			isa = PBXGroup;
-			children = (
-				4AA72B6F2146ED64006C3AEF /* TensorFlowLiteApp.app */,
-			);
-			name = Products;
-			sourceTree = "<group>";
-		};
-		4AA72B712146ED64006C3AEF /* TensorFlowLiteApp */ = {
-			isa = PBXGroup;
-			children = (
-				4AA72B722146ED64006C3AEF /* AppDelegate.swift */,
-				4ADDE0CD2176600900FF07A2 /* Array+TensorFlowLite.swift */,
-				4A7304B321500B8300C90B21 /* Data+TensorFlowLite.swift */,
-				4AA72B742146ED64006C3AEF /* ViewController.swift */,
-				4AA72B762146ED64006C3AEF /* Main.storyboard */,
-				4AA72B792146ED66006C3AEF /* Assets.xcassets */,
-				4AA72B7B2146ED66006C3AEF /* LaunchScreen.storyboard */,
-				4AA72B7E2146ED66006C3AEF /* Info.plist */,
-			);
-			path = TensorFlowLiteApp;
-			sourceTree = "<group>";
-		};
-/* End PBXGroup section */
-
-/* Begin PBXNativeTarget section */
-		4AA72B6E2146ED64006C3AEF /* TensorFlowLiteApp */ = {
-			isa = PBXNativeTarget;
-			buildConfigurationList = 4AA72B812146ED66006C3AEF /* Build configuration list for PBXNativeTarget "TensorFlowLiteApp" */;
-			buildPhases = (
-				4AA72B6B2146ED64006C3AEF /* Sources */,
-				4AA72B6C2146ED64006C3AEF /* Frameworks */,
-				4AA72B6D2146ED64006C3AEF /* Resources */,
-			);
-			buildRules = (
-			);
-			dependencies = (
-			);
-			name = TensorFlowLiteApp;
-			productName = TensorFlowLiteApp;
-			productReference = 4AA72B6F2146ED64006C3AEF /* TensorFlowLiteApp.app */;
-			productType = "com.apple.product-type.application";
-		};
-/* End PBXNativeTarget section */
-
-/* Begin PBXProject section */
-		4AA72B672146ED64006C3AEF /* Project object */ = {
-			isa = PBXProject;
-			attributes = {
-				LastSwiftUpdateCheck = 0940;
-				LastUpgradeCheck = 0940;
-				ORGANIZATIONNAME = Google;
-				TargetAttributes = {
-					4AA72B6E2146ED64006C3AEF = {
-						CreatedOnToolsVersion = 9.4.1;
-					};
-				};
-			};
-			buildConfigurationList = 4AA72B6A2146ED64006C3AEF /* Build configuration list for PBXProject "TensorFlowLiteApp" */;
-			compatibilityVersion = "Xcode 9.3";
-			developmentRegion = en;
-			hasScannedForEncodings = 0;
-			knownRegions = (
-				en,
-				Base,
-			);
-			mainGroup = 4AA72B662146ED64006C3AEF;
-			productRefGroup = 4AA72B702146ED64006C3AEF /* Products */;
-			projectDirPath = "";
-			projectRoot = "";
-			targets = (
-				4AA72B6E2146ED64006C3AEF /* TensorFlowLiteApp */,
-			);
-		};
-/* End PBXProject section */
-
-/* Begin PBXResourcesBuildPhase section */
-		4AA72B6D2146ED64006C3AEF /* Resources */ = {
-			isa = PBXResourcesBuildPhase;
-			buildActionMask = 2147483647;
-			files = (
-				4AA72B7D2146ED66006C3AEF /* LaunchScreen.storyboard in Resources */,
-				4AA72B7A2146ED66006C3AEF /* Assets.xcassets in Resources */,
-				4AA72B782146ED64006C3AEF /* Main.storyboard in Resources */,
-			);
-			runOnlyForDeploymentPostprocessing = 0;
-		};
-/* End PBXResourcesBuildPhase section */
-
-/* Begin PBXSourcesBuildPhase section */
-		4AA72B6B2146ED64006C3AEF /* Sources */ = {
-			isa = PBXSourcesBuildPhase;
-			buildActionMask = 2147483647;
-			files = (
-				4AA72B732146ED64006C3AEF /* AppDelegate.swift in Sources */,
-				4ADDE0CE2176600E00FF07A2 /* Array+TensorFlowLite.swift in Sources */,
-				4A7304B421500B8400C90B21 /* Data+TensorFlowLite.swift in Sources */,
-				4AA72B752146ED64006C3AEF /* ViewController.swift in Sources */,
-			);
-			runOnlyForDeploymentPostprocessing = 0;
-		};
-/* End PBXSourcesBuildPhase section */
-
-/* Begin PBXVariantGroup section */
-		4AA72B762146ED64006C3AEF /* Main.storyboard */ = {
-			isa = PBXVariantGroup;
-			children = (
-				4AA72B772146ED64006C3AEF /* Base */,
-			);
-			name = Main.storyboard;
-			sourceTree = "<group>";
-		};
-		4AA72B7B2146ED66006C3AEF /* LaunchScreen.storyboard */ = {
-			isa = PBXVariantGroup;
-			children = (
-				4AA72B7C2146ED66006C3AEF /* Base */,
-			);
-			name = LaunchScreen.storyboard;
-			sourceTree = "<group>";
-		};
-/* End PBXVariantGroup section */
-
-/* Begin XCBuildConfiguration section */
-		4AA72B7F2146ED66006C3AEF /* Debug */ = {
-			isa = XCBuildConfiguration;
-			buildSettings = {
-				ALWAYS_SEARCH_USER_PATHS = NO;
-				CLANG_ANALYZER_NONNULL = YES;
-				CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
-				CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
-				CLANG_CXX_LIBRARY = "libc++";
-				CLANG_ENABLE_MODULES = YES;
-				CLANG_ENABLE_OBJC_ARC = YES;
-				CLANG_ENABLE_OBJC_WEAK = YES;
-				CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
-				CLANG_WARN_BOOL_CONVERSION = YES;
-				CLANG_WARN_COMMA = YES;
-				CLANG_WARN_CONSTANT_CONVERSION = YES;
-				CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
-				CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
-				CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
-				CLANG_WARN_EMPTY_BODY = YES;
-				CLANG_WARN_ENUM_CONVERSION = YES;
-				CLANG_WARN_INFINITE_RECURSION = YES;
-				CLANG_WARN_INT_CONVERSION = YES;
-				CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
-				CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
-				CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
-				CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
-				CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
-				CLANG_WARN_STRICT_PROTOTYPES = YES;
-				CLANG_WARN_SUSPICIOUS_MOVE = YES;
-				CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
-				CLANG_WARN_UNREACHABLE_CODE = YES;
-				CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
-				CODE_SIGN_IDENTITY = "iPhone Developer";
-				COPY_PHASE_STRIP = NO;
-				DEBUG_INFORMATION_FORMAT = dwarf;
-				ENABLE_STRICT_OBJC_MSGSEND = YES;
-				ENABLE_TESTABILITY = YES;
-				GCC_C_LANGUAGE_STANDARD = gnu11;
-				GCC_DYNAMIC_NO_PIC = NO;
-				GCC_NO_COMMON_BLOCKS = YES;
-				GCC_OPTIMIZATION_LEVEL = 0;
-				GCC_PREPROCESSOR_DEFINITIONS = (
-					"DEBUG=1",
-					"$(inherited)",
-				);
-				GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
-				GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
-				GCC_WARN_UNDECLARED_SELECTOR = YES;
-				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
-				GCC_WARN_UNUSED_FUNCTION = YES;
-				GCC_WARN_UNUSED_VARIABLE = YES;
-				IPHONEOS_DEPLOYMENT_TARGET = 11.4;
-				MTL_ENABLE_DEBUG_INFO = YES;
-				ONLY_ACTIVE_ARCH = YES;
-				SDKROOT = iphoneos;
-				SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG;
-				SWIFT_OPTIMIZATION_LEVEL = "-Onone";
-			};
-			name = Debug;
-		};
-		4AA72B802146ED66006C3AEF /* Release */ = {
-			isa = XCBuildConfiguration;
-			buildSettings = {
-				ALWAYS_SEARCH_USER_PATHS = NO;
-				CLANG_ANALYZER_NONNULL = YES;
-				CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
-				CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
-				CLANG_CXX_LIBRARY = "libc++";
-				CLANG_ENABLE_MODULES = YES;
-				CLANG_ENABLE_OBJC_ARC = YES;
-				CLANG_ENABLE_OBJC_WEAK = YES;
-				CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
-				CLANG_WARN_BOOL_CONVERSION = YES;
-				CLANG_WARN_COMMA = YES;
-				CLANG_WARN_CONSTANT_CONVERSION = YES;
-				CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
-				CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
-				CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
-				CLANG_WARN_EMPTY_BODY = YES;
-				CLANG_WARN_ENUM_CONVERSION = YES;
-				CLANG_WARN_INFINITE_RECURSION = YES;
-				CLANG_WARN_INT_CONVERSION = YES;
-				CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
-				CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
-				CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
-				CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
-				CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
-				CLANG_WARN_STRICT_PROTOTYPES = YES;
-				CLANG_WARN_SUSPICIOUS_MOVE = YES;
-				CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
-				CLANG_WARN_UNREACHABLE_CODE = YES;
-				CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
-				CODE_SIGN_IDENTITY = "iPhone Developer";
-				COPY_PHASE_STRIP = NO;
-				DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
-				ENABLE_NS_ASSERTIONS = NO;
-				ENABLE_STRICT_OBJC_MSGSEND = YES;
-				GCC_C_LANGUAGE_STANDARD = gnu11;
-				GCC_NO_COMMON_BLOCKS = YES;
-				GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
-				GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
-				GCC_WARN_UNDECLARED_SELECTOR = YES;
-				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
-				GCC_WARN_UNUSED_FUNCTION = YES;
-				GCC_WARN_UNUSED_VARIABLE = YES;
-				IPHONEOS_DEPLOYMENT_TARGET = 11.4;
-				MTL_ENABLE_DEBUG_INFO = NO;
-				SDKROOT = iphoneos;
-				SWIFT_COMPILATION_MODE = wholemodule;
-				SWIFT_OPTIMIZATION_LEVEL = "-O";
-				VALIDATE_PRODUCT = YES;
-			};
-			name = Release;
-		};
-		4AA72B822146ED66006C3AEF /* Debug */ = {
-			isa = XCBuildConfiguration;
-			buildSettings = {
-				ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
-				CODE_SIGN_STYLE = Automatic;
-				INFOPLIST_FILE = TensorFlowLiteApp/Info.plist;
-				LD_RUNPATH_SEARCH_PATHS = (
-					"$(inherited)",
-					"@executable_path/Frameworks",
-				);
-				PRODUCT_BUNDLE_IDENTIFIER = com.tensorflow.lite.swift.TensorFlowLite;
-				PRODUCT_NAME = "$(TARGET_NAME)";
-				SWIFT_VERSION = 4.0;
-				TARGETED_DEVICE_FAMILY = "1,2";
-			};
-			name = Debug;
-		};
-		4AA72B832146ED66006C3AEF /* Release */ = {
-			isa = XCBuildConfiguration;
-			buildSettings = {
-				ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
-				CODE_SIGN_STYLE = Automatic;
-				INFOPLIST_FILE = TensorFlowLiteApp/Info.plist;
-				LD_RUNPATH_SEARCH_PATHS = (
-					"$(inherited)",
-					"@executable_path/Frameworks",
-				);
-				PRODUCT_BUNDLE_IDENTIFIER = com.tensorflow.lite.swift.TensorFlowLite;
-				PRODUCT_NAME = "$(TARGET_NAME)";
-				SWIFT_VERSION = 4.0;
-				TARGETED_DEVICE_FAMILY = "1,2";
-			};
-			name = Release;
-		};
-/* End XCBuildConfiguration section */
-
-/* Begin XCConfigurationList section */
-		4AA72B6A2146ED64006C3AEF /* Build configuration list for PBXProject "TensorFlowLiteApp" */ = {
-			isa = XCConfigurationList;
-			buildConfigurations = (
-				4AA72B7F2146ED66006C3AEF /* Debug */,
-				4AA72B802146ED66006C3AEF /* Release */,
-			);
-			defaultConfigurationIsVisible = 0;
-			defaultConfigurationName = Release;
-		};
-		4AA72B812146ED66006C3AEF /* Build configuration list for PBXNativeTarget "TensorFlowLiteApp" */ = {
-			isa = XCConfigurationList;
-			buildConfigurations = (
-				4AA72B822146ED66006C3AEF /* Debug */,
-				4AA72B832146ED66006C3AEF /* Release */,
-			);
-			defaultConfigurationIsVisible = 0;
-			defaultConfigurationName = Release;
-		};
-/* End XCConfigurationList section */
-	};
-	rootObject = 4AA72B672146ED64006C3AEF /* Project object */;
-}
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/AppDelegate.swift b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/AppDelegate.swift
deleted file mode 100644
index ffa90a06ad..0000000000
--- a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/AppDelegate.swift
+++ /dev/null
@@ -1,24 +0,0 @@
-import UIKit
-
-@UIApplicationMain
-
-final class AppDelegate: UIResponder, UIApplicationDelegate {
-
-  /// The main window of the app.
-  var window: UIWindow?
-
-  func application(
-    _ application: UIApplication,
-    didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]? = nil
-  ) -> Bool {
-    return true
-  }
-}
-
-// MARK: - Extensions
-
-#if !swift(>=4.2)
-extension UIApplication {
-  typealias LaunchOptionsKey = UIApplicationLaunchOptionsKey
-}
-#endif  // !swift(>=4.2)
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Array+TensorFlowLite.swift b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Array+TensorFlowLite.swift
deleted file mode 100644
index 56df1ce659..0000000000
--- a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Array+TensorFlowLite.swift
+++ /dev/null
@@ -1,22 +0,0 @@
-import Foundation
-
-extension Array {
-  /// Creates a new array from the bytes of the given unsafe data.
-  ///
-  /// - Warning: The array's `Element` type must be trivial in that it can be copied bit for bit
-  ///     with no indirection or reference-counting operations; otherwise, copying the raw bytes in
-  ///     the `unsafeData`'s buffer to a new array returns an unsafe copy.
-  /// - Note: Returns `nil` if `unsafeData.count` is not a multiple of
-  ///     `MemoryLayout<Element>.stride`.
-  /// - Parameter unsafeData: The data containing the bytes to turn into an array.
-  init?(unsafeData: Data) {
-    guard unsafeData.count % MemoryLayout<Element>.stride == 0 else { return nil }
-    let elements = unsafeData.withUnsafeBytes {
-      UnsafeBufferPointer<Element>(
-        start: $0,
-        count: unsafeData.count / MemoryLayout<Element>.stride
-      )
-    }
-    self.init(elements)
-  }
-}
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/AppIcon.appiconset/Contents.json b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/AppIcon.appiconset/Contents.json
deleted file mode 100644
index d8db8d65fd..0000000000
--- a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/AppIcon.appiconset/Contents.json
+++ /dev/null
@@ -1,98 +0,0 @@
-{
-  "images" : [
-    {
-      "idiom" : "iphone",
-      "size" : "20x20",
-      "scale" : "2x"
-    },
-    {
-      "idiom" : "iphone",
-      "size" : "20x20",
-      "scale" : "3x"
-    },
-    {
-      "idiom" : "iphone",
-      "size" : "29x29",
-      "scale" : "2x"
-    },
-    {
-      "idiom" : "iphone",
-      "size" : "29x29",
-      "scale" : "3x"
-    },
-    {
-      "idiom" : "iphone",
-      "size" : "40x40",
-      "scale" : "2x"
-    },
-    {
-      "idiom" : "iphone",
-      "size" : "40x40",
-      "scale" : "3x"
-    },
-    {
-      "idiom" : "iphone",
-      "size" : "60x60",
-      "scale" : "2x"
-    },
-    {
-      "idiom" : "iphone",
-      "size" : "60x60",
-      "scale" : "3x"
-    },
-    {
-      "idiom" : "ipad",
-      "size" : "20x20",
-      "scale" : "1x"
-    },
-    {
-      "idiom" : "ipad",
-      "size" : "20x20",
-      "scale" : "2x"
-    },
-    {
-      "idiom" : "ipad",
-      "size" : "29x29",
-      "scale" : "1x"
-    },
-    {
-      "idiom" : "ipad",
-      "size" : "29x29",
-      "scale" : "2x"
-    },
-    {
-      "idiom" : "ipad",
-      "size" : "40x40",
-      "scale" : "1x"
-    },
-    {
-      "idiom" : "ipad",
-      "size" : "40x40",
-      "scale" : "2x"
-    },
-    {
-      "idiom" : "ipad",
-      "size" : "76x76",
-      "scale" : "1x"
-    },
-    {
-      "idiom" : "ipad",
-      "size" : "76x76",
-      "scale" : "2x"
-    },
-    {
-      "idiom" : "ipad",
-      "size" : "83.5x83.5",
-      "scale" : "2x"
-    },
-    {
-      "idiom" : "ios-marketing",
-      "size" : "1024x1024",
-      "scale" : "1x"
-    }
-  ],
-  "info" : {
-    "version" : 1,
-    "author" : "xcode"
-  }
-}
\ No newline at end of file
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/Contents.json b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/Contents.json
deleted file mode 100644
index da4a164c91..0000000000
--- a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/Contents.json
+++ /dev/null
@@ -1,6 +0,0 @@
-{
-  "info" : {
-    "version" : 1,
-    "author" : "xcode"
-  }
-}
\ No newline at end of file
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/LaunchScreen.storyboard b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/LaunchScreen.storyboard
deleted file mode 100644
index a07a1321be..0000000000
--- a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/LaunchScreen.storyboard
+++ /dev/null
@@ -1,44 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="14109" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" launchScreen="YES" useTraitCollections="YES" colorMatched="YES" initialViewController="01J-lp-oVM">
-    <device id="retina4_7" orientation="portrait">
-        <adaptation id="fullscreen"/>
-    </device>
-    <dependencies>
-        <deployment identifier="iOS"/>
-        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="14088"/>
-        <capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
-    </dependencies>
-    <scenes>
-        <!--View Controller-->
-        <scene sceneID="EHf-IW-A2E">
-            <objects>
-                <viewController id="01J-lp-oVM" sceneMemberID="viewController">
-                    <layoutGuides>
-                        <viewControllerLayoutGuide type="top" id="Llm-lL-Icb"/>
-                        <viewControllerLayoutGuide type="bottom" id="xb3-aO-Qok"/>
-                    </layoutGuides>
-                    <view key="view" contentMode="scaleToFill" id="Ze5-6b-2t3">
-                        <rect key="frame" x="0.0" y="0.0" width="375" height="667"/>
-                        <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
-                        <subviews>
-                            <label opaque="NO" userInteractionEnabled="NO" contentMode="left" horizontalHuggingPriority="251" verticalHuggingPriority="251" text="TensorFlowLite" textAlignment="center" lineBreakMode="tailTruncation" baselineAdjustment="alignBaselines" adjustsFontSizeToFit="NO" translatesAutoresizingMaskIntoConstraints="NO" id="3Gq-PV-hia">
-                                <rect key="frame" x="16" y="315" width="343" height="38.5"/>
-                                <fontDescription key="fontDescription" type="boldSystem" pointSize="32"/>
-                                <nil key="textColor"/>
-                                <nil key="highlightedColor"/>
-                            </label>
-                        </subviews>
-                        <color key="backgroundColor" red="1" green="1" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
-                        <constraints>
-                            <constraint firstItem="3Gq-PV-hia" firstAttribute="leading" secondItem="Ze5-6b-2t3" secondAttribute="leading" constant="16" id="aXL-9T-5Pf"/>
-                            <constraint firstItem="3Gq-PV-hia" firstAttribute="centerY" secondItem="Ze5-6b-2t3" secondAttribute="centerY" id="cDf-Go-1FR"/>
-                            <constraint firstAttribute="trailing" secondItem="3Gq-PV-hia" secondAttribute="trailing" constant="16" id="fB9-BX-A3B"/>
-                        </constraints>
-                    </view>
-                </viewController>
-                <placeholder placeholderIdentifier="IBFirstResponder" id="iYj-Kq-Ea1" userLabel="First Responder" sceneMemberID="firstResponder"/>
-            </objects>
-            <point key="canvasLocation" x="52" y="374.66266866566718"/>
-        </scene>
-    </scenes>
-</document>
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/Main.storyboard b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/Main.storyboard
deleted file mode 100644
index d0e91d63c4..0000000000
--- a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/Main.storyboard
+++ /dev/null
@@ -1,95 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="14313.18" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" useTraitCollections="YES" colorMatched="YES" initialViewController="BYZ-38-t0r">
-    <device id="retina4_7" orientation="portrait">
-        <adaptation id="fullscreen"/>
-    </device>
-    <dependencies>
-        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="14283.14"/>
-        <capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
-    </dependencies>
-    <scenes>
-        <!--View Controller-->
-        <scene sceneID="tne-QT-ifu">
-            <objects>
-                <viewController storyboardIdentifier="viewController" useStoryboardIdentifierAsRestorationIdentifier="YES" id="BYZ-38-t0r" customClass="ViewController" customModule="third_party_tensorflow_lite_experimental_swift_TensorFlowLiteAppLib" sceneMemberID="viewController">
-                    <layoutGuides>
-                        <viewControllerLayoutGuide type="top" id="y3c-jy-aDJ"/>
-                        <viewControllerLayoutGuide type="bottom" id="wfy-db-euE"/>
-                    </layoutGuides>
-                    <view key="view" contentMode="scaleToFill" id="8bC-Xf-vdC">
-                        <rect key="frame" x="0.0" y="0.0" width="375" height="667"/>
-                        <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
-                        <subviews>
-                            <textView clipsSubviews="YES" multipleTouchEnabled="YES" contentMode="scaleToFill" editable="NO" selectable="NO" translatesAutoresizingMaskIntoConstraints="NO" id="7Mj-sL-hrd">
-                                <rect key="frame" x="0.0" y="367" width="375" height="300"/>
-                                <color key="backgroundColor" red="0.0" green="0.47843137250000001" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
-                                <constraints>
-                                    <constraint firstAttribute="height" constant="300" id="YUb-MC-D5w"/>
-                                </constraints>
-                                <color key="textColor" cocoaTouchSystemColor="tableCellGroupedBackgroundColor"/>
-                                <fontDescription key="fontDescription" type="system" pointSize="14"/>
-                                <textInputTraits key="textInputTraits" autocapitalizationType="sentences"/>
-                            </textView>
-                            <toolbar opaque="NO" clearsContextBeforeDrawing="NO" contentMode="scaleToFill" translatesAutoresizingMaskIntoConstraints="NO" id="Qwg-EP-bd6" userLabel="Bottom Toolbar">
-                                <rect key="frame" x="0.0" y="323" width="375" height="44"/>
-                                <constraints>
-                                    <constraint firstAttribute="height" constant="44" id="jhT-Q0-E9N"/>
-                                </constraints>
-                                <items>
-                                    <barButtonItem style="plain" systemItem="flexibleSpace" id="P3q-uA-YUa"/>
-                                    <barButtonItem title="Invoke Interpreter" id="A4J-Mg-nmd" userLabel="Invoke Button">
-                                        <connections>
-                                            <action selector="invokeInterpreter:" destination="BYZ-38-t0r" id="lZU-x7-PsJ"/>
-                                        </connections>
-                                    </barButtonItem>
-                                    <barButtonItem style="plain" systemItem="flexibleSpace" id="Qad-Pa-ySg"/>
-                                </items>
-                            </toolbar>
-                            <toolbar opaque="NO" clearsContextBeforeDrawing="NO" contentMode="scaleToFill" translatesAutoresizingMaskIntoConstraints="NO" id="Gkb-TR-PCB" userLabel="Top Toolbar">
-                                <rect key="frame" x="0.0" y="28" width="375" height="44"/>
-                                <constraints>
-                                    <constraint firstAttribute="height" constant="44" id="hSD-2q-fUE"/>
-                                </constraints>
-                                <items>
-                                    <barButtonItem style="plain" id="LKw-TX-bbH">
-                                        <segmentedControl key="customView" opaque="NO" contentMode="scaleToFill" contentHorizontalAlignment="left" contentVerticalAlignment="top" segmentControlStyle="bar" selectedSegmentIndex="0" id="rhA-nW-xzT">
-                                            <rect key="frame" x="16" y="7" width="343" height="30"/>
-                                            <autoresizingMask key="autoresizingMask" flexibleMaxX="YES" flexibleMaxY="YES"/>
-                                            <segments>
-                                                <segment title="Add"/>
-                                                <segment title="AddQuantized"/>
-                                                <segment title="MultiAdd"/>
-                                            </segments>
-                                            <connections>
-                                                <action selector="modelChanged:" destination="BYZ-38-t0r" eventType="valueChanged" id="YnG-Ov-B5D"/>
-                                            </connections>
-                                        </segmentedControl>
-                                    </barButtonItem>
-                                </items>
-                            </toolbar>
-                        </subviews>
-                        <color key="backgroundColor" red="1" green="1" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
-                        <constraints>
-                            <constraint firstAttribute="trailing" secondItem="Gkb-TR-PCB" secondAttribute="trailing" id="4Cr-Sf-I7n"/>
-                            <constraint firstItem="7Mj-sL-hrd" firstAttribute="bottom" secondItem="wfy-db-euE" secondAttribute="top" id="6ot-zD-sze"/>
-                            <constraint firstItem="7Mj-sL-hrd" firstAttribute="top" secondItem="Qwg-EP-bd6" secondAttribute="bottom" id="ELA-C6-NiG"/>
-                            <constraint firstAttribute="trailing" secondItem="7Mj-sL-hrd" secondAttribute="trailing" id="HDO-xr-mBl"/>
-                            <constraint firstItem="Gkb-TR-PCB" firstAttribute="leading" secondItem="8bC-Xf-vdC" secondAttribute="leading" id="Kmo-6K-gS4"/>
-                            <constraint firstItem="Qwg-EP-bd6" firstAttribute="leading" secondItem="8bC-Xf-vdC" secondAttribute="leading" id="hGu-lm-fMG"/>
-                            <constraint firstAttribute="trailing" secondItem="Qwg-EP-bd6" secondAttribute="trailing" id="iXR-LK-nTO"/>
-                            <constraint firstItem="7Mj-sL-hrd" firstAttribute="leading" secondItem="8bC-Xf-vdC" secondAttribute="leading" id="nr7-jW-ZYf"/>
-                            <constraint firstItem="Gkb-TR-PCB" firstAttribute="top" secondItem="y3c-jy-aDJ" secondAttribute="bottom" constant="8" id="uCF-VW-rR0"/>
-                        </constraints>
-                    </view>
-                    <connections>
-                        <outlet property="invokeButton" destination="A4J-Mg-nmd" id="UxZ-Ft-E45"/>
-                        <outlet property="modelControl" destination="rhA-nW-xzT" id="KKf-TT-BQ2"/>
-                        <outlet property="resultsTextView" destination="7Mj-sL-hrd" id="T4I-z4-tYA"/>
-                    </connections>
-                </viewController>
-                <placeholder placeholderIdentifier="IBFirstResponder" id="dkx-z0-nzr" sceneMemberID="firstResponder"/>
-            </objects>
-            <point key="canvasLocation" x="125.59999999999999" y="133.5832083958021"/>
-        </scene>
-    </scenes>
-</document>
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Data+TensorFlowLite.swift b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Data+TensorFlowLite.swift
deleted file mode 100644
index bc8a70c848..0000000000
--- a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Data+TensorFlowLite.swift
+++ /dev/null
@@ -1,13 +0,0 @@
-import Foundation
-
-extension Data {
-  /// Creates a new buffer by copying the buffer pointer of the given array.
-  ///
-  /// - Warning: The given array's element type `T` must be trivial in that it can be copied bit
-  ///     for bit with no indirection or reference-counting operations; otherwise, reinterpreting
-  ///     data from the resulting buffer has undefined behavior.
-  /// - Parameter array: An array with elements of type `T`.
-  init<T>(copyingBufferOf array: [T]) {
-    self = array.withUnsafeBufferPointer(Data.init)
-  }
-}
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Info.plist b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Info.plist
deleted file mode 100644
index 3ca3875f04..0000000000
--- a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Info.plist
+++ /dev/null
@@ -1,46 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
-<plist version="1.0">
-<dict>
-	<key>CFBundleDevelopmentRegion</key>
-	<string>en</string>
-	<key>CFBundleExecutable</key>
-	<string>$(EXECUTABLE_NAME)</string>
-	<key>CFBundleIdentifier</key>
-	<string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
-	<key>CFBundleInfoDictionaryVersion</key>
-	<string>6.0</string>
-	<key>CFBundleName</key>
-	<string>$(PRODUCT_NAME)</string>
-	<key>CFBundlePackageType</key>
-	<string>APPL</string>
-	<key>CFBundleShortVersionString</key>
-	<string>1.0</string>
-	<key>CFBundleVersion</key>
-	<string>0.0.1</string>
-	<key>LSRequiresIPhoneOS</key>
-	<true/>
-	<key>NSCameraUsageDescription</key>
-	<string>NSCameraUsageDescription</string>
-	<key>NSPhotoLibraryUsageDescription</key>
-	<string>Select a photo to detect objects in.</string>
-	<key>UILaunchStoryboardName</key>
-	<string>LaunchScreen</string>
-	<key>UIMainStoryboardFile</key>
-	<string>Main</string>
-	<key>UIRequiredDeviceCapabilities</key>
-	<array>
-		<string>armv7</string>
-	</array>
-	<key>UISupportedInterfaceOrientations</key>
-	<array>
-		<string>UIInterfaceOrientationPortrait</string>
-		<string>UIInterfaceOrientationPortraitUpsideDown</string>
-	</array>
-	<key>UISupportedInterfaceOrientations~ipad</key>
-	<array>
-		<string>UIInterfaceOrientationPortrait</string>
-		<string>UIInterfaceOrientationPortraitUpsideDown</string>
-	</array>
-</dict>
-</plist>
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/ViewController.swift b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/ViewController.swift
deleted file mode 100644
index 73c74fd19c..0000000000
--- a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/ViewController.swift
+++ /dev/null
@@ -1,299 +0,0 @@
-import TensorFlowLite
-import UIKit
-
-class ViewController: UIViewController {
-
-  // MARK: - Properties
-
-  /// TensorFlowLite interpreter object for performing inference from a given model.
-  private var interpreter: Interpreter?
-
-  /// Serial dispatch queue for managing `Interpreter` calls.
-  private let interpreterQueue = DispatchQueue(
-    label: Constant.dispatchQueueLabel,
-    qos: .userInitiated
-  )
-
-  /// The currently selected model.
-  private var currentModel: Model {
-    guard let currentModel = Model(rawValue: modelControl.selectedSegmentIndex) else {
-      preconditionFailure("Invalid model for selected segment index.")
-    }
-    return currentModel
-  }
-
-  /// A description of the current model.
-  private var modelDescription: String {
-    guard let interpreter = interpreter else { return "" }
-    let inputCount = interpreter.inputTensorCount
-    let outputCount = interpreter.outputTensorCount
-    let inputTensors = (0..<inputCount).map { index in
-      var tensorInfo = "  Input \(index + 1): "
-      do {
-        let tensor = try interpreter.input(at: index)
-        tensorInfo += "\(tensor)"
-      } catch let error {
-        tensorInfo += "\(error.localizedDescription)"
-      }
-      return tensorInfo
-    }.joined(separator: "\n")
-    let outputTensors = (0..<outputCount).map { index in
-      var tensorInfo = "  Output \(index + 1): "
-      do {
-        let tensor = try interpreter.output(at: index)
-        tensorInfo += "\(tensor)"
-      } catch let error {
-        tensorInfo += "\(error.localizedDescription)"
-      }
-      return tensorInfo
-    }.joined(separator: "\n")
-    return "Model Description:\n" +
-             "  Input Tensor Count = \(inputCount)\n\(inputTensors)\n\n" +
-             "  Output Tensor Count = \(outputCount)\n\(outputTensors)"
-  }
-
-  // MARK: - IBOutlets
-
-  /// A segmented control for changing models. See the `Model` enum for available models.
-  @IBOutlet private var modelControl: UISegmentedControl!
-
-  @IBOutlet private var resultsTextView: UITextView!
-  @IBOutlet private var invokeButton: UIBarButtonItem!
-
-  // MARK: - UIViewController
-
-  override func viewDidLoad() {
-    super.viewDidLoad()
-
-    invokeButton.isEnabled = false
-    loadModel()
-  }
-
-  // MARK: - IBActions
-
-  @IBAction func modelChanged(_ sender: Any) {
-    invokeButton.isEnabled = false
-    updateResultsText("Switched to the \(currentModel.description) model.")
-    loadModel()
-  }
-
-  @IBAction func invokeInterpreter(_ sender: Any) {
-    switch currentModel {
-    case .add:
-      invokeAdd()
-    case .addQuantized:
-      invokeAddQuantized()
-    case .multiAdd:
-      invokeMultiAdd()
-    }
-  }
-
-  // MARK: - Private
-
-  private func loadModel() {
-    let fileInfo = currentModel.fileInfo
-    guard let modelPath = Bundle.main.path(forResource: fileInfo.name, ofType: fileInfo.extension)
-    else {
-      updateResultsText("Failed to load the \(currentModel.description) model.")
-      return
-    }
-    setUpInterpreter(withModelPath: modelPath)
-  }
-
-  private func setUpInterpreter(withModelPath modelPath: String) {
-    interpreterQueue.async {
-      do {
-        var options = InterpreterOptions()
-        options.isErrorLoggingEnabled = true
-        self.interpreter = try Interpreter(modelPath: modelPath, options: options)
-      } catch let error {
-        self.updateResultsText(
-          "Failed to create the interpreter with error: \(error.localizedDescription)"
-        )
-        return
-      }
-      safeDispatchOnMain { self.invokeButton.isEnabled = true }
-    }
-  }
-
-  private func invokeAdd() {
-    interpreterQueue.async {
-      guard let interpreter = self.interpreter else {
-        self.updateResultsText(Constant.nilInterpreterErrorMessage)
-        return
-      }
-      do {
-        try interpreter.resizeInput(at: 0, to: [2])
-        try interpreter.allocateTensors()
-        let input: [Float32] = [1, 3]
-        let resultsText = self.modelDescription + "\n\n" +
-          "Performing 2 add operations on input \(input.description) equals: "
-        self.updateResultsText(resultsText)
-        let data = Data(copyingBufferOf: input)
-        try interpreter.copy(data, toInputAt: 0)
-        try interpreter.invoke()
-        let outputTensor = try interpreter.output(at: 0)
-        let results: () -> String = {
-          guard let results = [Float32](unsafeData: outputTensor.data) else { return "No results." }
-          return resultsText + results.description
-        }
-        self.updateResultsText(results())
-      } catch let error {
-        self.updateResultsText(
-          "Failed to invoke the interpreter with error: \(error.localizedDescription)"
-        )
-        return
-      }
-    }
-  }
-
-  private func invokeAddQuantized() {
-    interpreterQueue.async {
-      guard let interpreter = self.interpreter else {
-        self.updateResultsText(Constant.nilInterpreterErrorMessage)
-        return
-      }
-      do {
-        try interpreter.resizeInput(at: 0, to: [2])
-        try interpreter.allocateTensors()
-        let input: [UInt8] = [1, 3]
-        let resultsText = self.modelDescription + "\n\n" +
-          "Performing 2 add operations on quantized input \(input.description) equals: "
-        self.updateResultsText(resultsText)
-        let data = Data(input)
-        try interpreter.copy(data, toInputAt: 0)
-        try interpreter.invoke()
-        let outputTensor = try interpreter.output(at: 0)
-        let results: () -> String = {
-          guard let quantizationParameters = outputTensor.quantizationParameters else {
-            return "No results."
-          }
-          let quantizedResults = [UInt8](outputTensor.data)
-          let dequantizedResults = quantizedResults.map {
-            quantizationParameters.scale * Float(Int($0) - quantizationParameters.zeroPoint)
-          }
-          return resultsText + quantizedResults.description +
-                   ", dequantized results: " + dequantizedResults.description
-        }
-        self.updateResultsText(results())
-      } catch let error {
-        self.updateResultsText(
-          "Failed to invoke the interpreter with error: \(error.localizedDescription)"
-        )
-        return
-      }
-    }
-  }
-
-  private func invokeMultiAdd() {
-    interpreterQueue.async {
-      guard let interpreter = self.interpreter else {
-        self.updateResultsText(Constant.nilInterpreterErrorMessage)
-        return
-      }
-      do {
-        let shape = TensorShape(2)
-        try (0..<interpreter.inputTensorCount).forEach { index in
-          try interpreter.resizeInput(at: index, to: shape)
-        }
-        try interpreter.allocateTensors()
-        let inputs = try (0..<interpreter.inputTensorCount).map { index -> [Float32] in
-          let input = [Float32(index + 1), Float32(index + 2)]
-          let data = Data(copyingBufferOf: input)
-          try interpreter.copy(data, toInputAt: index)
-          return input
-        }
-        let resultsText = self.modelDescription + "\n\n" +
-          "Performing 3 add operations on inputs \(inputs.description) equals: "
-        self.updateResultsText(resultsText)
-        try interpreter.invoke()
-        let results = try (0..<interpreter.outputTensorCount).map { index -> [Float32] in
-          let tensor = try interpreter.output(at: index)
-          return [Float32](unsafeData: tensor.data) ?? []
-        }
-        self.updateResultsText(resultsText + results.description)
-      } catch let error {
-        self.updateResultsText(
-          "Failed to invoke the interpreter with error: \(error.localizedDescription)"
-        )
-        return
-      }
-    }
-  }
-
-  private func updateResultsText(_ text: String? = nil) {
-    safeDispatchOnMain { self.resultsTextView.text = text }
-  }
-}
-
-// MARK: - Constants
-
-private enum Constant {
-  static let dispatchQueueLabel = "TensorFlowLiteInterpreterQueue"
-  static let nilInterpreterErrorMessage =
-    "Failed to invoke the interpreter because the interpreter was nil."
-}
-
-/// Models that can be loaded by the TensorFlow Lite `Interpreter`.
-private enum Model: Int, CustomStringConvertible {
-  /// A float model that performs two add operations on one input tensor and returns the result in
-  /// one output tensor.
-  case add = 0
-  /// A quantized model that performs two add operations on one input tensor and returns the result
-  /// in one output tensor.
-  case addQuantized = 1
-  /// A float model that performs three add operations on four input tensors and returns the results
-  /// in 2 output tensors.
-  case multiAdd = 2
-
-  var fileInfo: (name: String, extension: String) {
-    switch self {
-    case .add:
-      return Add.fileInfo
-    case .addQuantized:
-      return AddQuantized.fileInfo
-    case .multiAdd:
-      return MultiAdd.fileInfo
-    }
-  }
-
-  // MARK: - CustomStringConvertible
-
-  var description: String {
-    switch self {
-    case .add:
-      return Add.name
-    case .addQuantized:
-      return AddQuantized.name
-    case .multiAdd:
-      return MultiAdd.name
-    }
-  }
-}
-
-/// Values for the `Add` model.
-private enum Add {
-  static let name = "Add"
-  static let fileInfo = (name: "add", extension: "bin")
-}
-
-/// Values for the `AddQuantized` model.
-private enum AddQuantized {
-  static let name = "AddQuantized"
-  static let fileInfo = (name: "add_quantized", extension: "bin")
-}
-
-/// Values for the `MultiAdd` model.
-private enum MultiAdd {
-  static let name = "MultiAdd"
-  static let fileInfo = (name: "multi_add", extension: "bin")
-}
-
-// MARK: - Fileprivate
-
-/// Safely dispatches the given block on the main queue. If the current thread is `main`, the block
-/// is executed synchronously; otherwise, the block is executed asynchronously on the main thread.
-fileprivate func safeDispatchOnMain(_ block: @escaping () -> Void) {
-  if Thread.isMainThread { block(); return }
-  DispatchQueue.main.async { block() }
-}
diff --git a/tensorflow/lite/experimental/swift/Tests/InterpreterOptionsTests.swift b/tensorflow/lite/experimental/swift/Tests/InterpreterOptionsTests.swift
deleted file mode 100644
index 54b4f59b28..0000000000
--- a/tensorflow/lite/experimental/swift/Tests/InterpreterOptionsTests.swift
+++ /dev/null
@@ -1,54 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-@testable import TensorFlowLite
-import XCTest
-
-class InterpreterOptionsTests: XCTestCase {
-
-  func testInterpreterOptions_InitWithDefaultValues() {
-    let options = InterpreterOptions()
-    XCTAssertNil(options.threadCount)
-    XCTAssertFalse(options.isErrorLoggingEnabled)
-  }
-
-  func testInterpreterOptions_InitWithCustomValues() {
-    var options = InterpreterOptions()
-    options.threadCount = 2
-    XCTAssertEqual(options.threadCount, 2)
-    options.isErrorLoggingEnabled = true
-    XCTAssertTrue(options.isErrorLoggingEnabled)
-  }
-
-  func testInterpreterOptions_Equatable() {
-    var options1 = InterpreterOptions()
-    var options2 = InterpreterOptions()
-    XCTAssertEqual(options1, options2)
-
-    options1.threadCount = 2
-    options2.threadCount = 2
-    XCTAssertEqual(options1, options2)
-
-    options2.threadCount = 3
-    XCTAssertNotEqual(options1, options2)
-    options2.threadCount = 2
-
-    options1.isErrorLoggingEnabled = true
-    options2.isErrorLoggingEnabled = true
-    XCTAssertEqual(options1, options2)
-
-    options2.isErrorLoggingEnabled = false
-    XCTAssertNotEqual(options1, options2)
-  }
-}
diff --git a/tensorflow/lite/experimental/swift/Tests/InterpreterTests.swift b/tensorflow/lite/experimental/swift/Tests/InterpreterTests.swift
deleted file mode 100644
index e98da5f951..0000000000
--- a/tensorflow/lite/experimental/swift/Tests/InterpreterTests.swift
+++ /dev/null
@@ -1,315 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-@testable import TensorFlowLite
-import XCTest
-
-class InterpreterTests: XCTestCase {
-
-  var interpreter: Interpreter!
-
-  override func setUp() {
-    super.setUp()
-
-    interpreter = try! Interpreter(modelPath: AddModel.path)
-  }
-
-  override func tearDown() {
-    interpreter = nil
-
-    super.tearDown()
-  }
-
-  func testInterpreter_InitWithModelPath() {
-    XCTAssertNoThrow(try Interpreter(modelPath: AddModel.path))
-  }
-
-  func testInterpreter_Init_ThrowsFailedToLoadModel() {
-    XCTAssertThrowsError(try Interpreter(modelPath: "/invalid/path")) { error in
-      self.assertEqualErrors(actual: error, expected: .failedToLoadModel)
-    }
-  }
-
-  func testInterpreter_InitWithModelPathAndOptions() {
-    var options = InterpreterOptions()
-    options.threadCount = 2
-    XCTAssertNoThrow(try Interpreter(modelPath: AddModel.path, options: options))
-  }
-
-  func testInterpreter_InputTensorCount() {
-    XCTAssertEqual(interpreter.inputTensorCount, AddModel.inputTensorCount)
-  }
-
-  func testInterpreter_OutputTensorCount() {
-    XCTAssertEqual(interpreter.outputTensorCount, AddModel.outputTensorCount)
-  }
-
-  func testInterpreter_Invoke() throws {
-    try interpreter.allocateTensors()
-    XCTAssertNoThrow(try interpreter.invoke())
-  }
-
-  func testInterpreter_Invoke_ThrowsAllocateTensorsRequired_ModelNotReady() {
-    XCTAssertThrowsError(try interpreter.invoke()) { error in
-      self.assertEqualErrors(actual: error, expected: .allocateTensorsRequired)
-    }
-  }
-
-  func testInterpreter_InputTensorAtIndex() throws {
-    try setUpAddModelInputTensor()
-    let inputTensor = try interpreter.input(at: AddModel.validIndex)
-    XCTAssertEqual(inputTensor, AddModel.inputTensor)
-  }
-
-  func testInterpreter_InputTensorAtIndex_QuantizedModel() throws {
-    interpreter = try Interpreter(modelPath: AddQuantizedModel.path)
-    try setUpAddQuantizedModelInputTensor()
-    let inputTensor = try interpreter.input(at: AddQuantizedModel.inputOutputIndex)
-    XCTAssertEqual(inputTensor, AddQuantizedModel.inputTensor)
-  }
-
-  func testInterpreter_InputTensorAtIndex_ThrowsInvalidIndex() throws {
-    try interpreter.allocateTensors()
-    XCTAssertThrowsError(try interpreter.input(at: AddModel.invalidIndex)) { error in
-      let maxIndex = AddModel.inputTensorCount - 1
-      self.assertEqualErrors(
-        actual: error,
-        expected: .invalidTensorIndex(index: AddModel.invalidIndex, maxIndex: maxIndex)
-      )
-    }
-  }
-
-  func testInterpreter_InputTensorAtIndex_ThrowsAllocateTensorsRequired() {
-    XCTAssertThrowsError(try interpreter.input(at: AddModel.validIndex)) { error in
-      self.assertEqualErrors(actual: error, expected: .allocateTensorsRequired)
-    }
-  }
-
-  func testInterpreter_OutputTensorAtIndex() throws {
-    try setUpAddModelInputTensor()
-    try interpreter.invoke()
-    let outputTensor = try interpreter.output(at: AddModel.validIndex)
-    XCTAssertEqual(outputTensor, AddModel.outputTensor)
-    let expectedResults = [Float32](unsafeData: outputTensor.data)
-    XCTAssertEqual(expectedResults, AddModel.results)
-  }
-
-  func testInterpreter_OutputTensorAtIndex_QuantizedModel() throws {
-    interpreter = try Interpreter(modelPath: AddQuantizedModel.path)
-    try setUpAddQuantizedModelInputTensor()
-    try interpreter.invoke()
-    let outputTensor = try interpreter.output(at: AddQuantizedModel.inputOutputIndex)
-    XCTAssertEqual(outputTensor, AddQuantizedModel.outputTensor)
-    let expectedResults = [UInt8](outputTensor.data)
-    XCTAssertEqual(expectedResults, AddQuantizedModel.results)
-  }
-
-  func testInterpreter_OutputTensorAtIndex_ThrowsInvalidIndex() throws {
-    try interpreter.allocateTensors()
-    try interpreter.invoke()
-    XCTAssertThrowsError(try interpreter.output(at: AddModel.invalidIndex)) { error in
-      let maxIndex = AddModel.outputTensorCount - 1
-      self.assertEqualErrors(
-        actual: error,
-        expected: .invalidTensorIndex(index: AddModel.invalidIndex, maxIndex: maxIndex)
-      )
-    }
-  }
-
-  func testInterpreter_OutputTensorAtIndex_ThrowsInvokeInterpreterRequired() {
-    XCTAssertThrowsError(try interpreter.output(at: AddModel.validIndex)) { error in
-      self.assertEqualErrors(actual: error, expected: .invokeInterpreterRequired)
-    }
-  }
-
-  func testInterpreter_ResizeInputTensorAtIndexToShape() {
-    XCTAssertNoThrow(try interpreter.resizeInput(at: AddModel.validIndex, to: [2, 2, 3]))
-    XCTAssertNoThrow(try interpreter.allocateTensors())
-  }
-
-  func testInterpreter_ResizeInputTensorAtIndexToShape_ThrowsInvalidIndex() {
-    XCTAssertThrowsError(try interpreter.resizeInput(
-      at: AddModel.invalidIndex,
-      to: [2, 2, 3]
-    )) { error in
-      let maxIndex = AddModel.inputTensorCount - 1
-      self.assertEqualErrors(
-        actual: error,
-        expected: .invalidTensorIndex(index: AddModel.invalidIndex, maxIndex: maxIndex)
-      )
-    }
-  }
-
-  func testInterpreter_CopyDataToInputTensorAtIndex() throws {
-    try interpreter.resizeInput(at: AddModel.validIndex, to: AddModel.shape)
-    try interpreter.allocateTensors()
-    let inputTensor = try interpreter.copy(AddModel.inputData, toInputAt: AddModel.validIndex)
-    XCTAssertEqual(inputTensor.data, AddModel.inputData)
-  }
-
-  func testInterpreter_CopyDataToInputTensorAtIndex_ThrowsInvalidIndex() {
-    XCTAssertThrowsError(try interpreter.copy(
-      AddModel.inputData,
-      toInputAt: AddModel.invalidIndex
-    )) { error in
-      let maxIndex = AddModel.inputTensorCount - 1
-      self.assertEqualErrors(
-        actual: error,
-        expected: .invalidTensorIndex(index: AddModel.invalidIndex, maxIndex: maxIndex)
-      )
-    }
-  }
-
-  func testInterpreter_CopyDataToInputTensorAtIndex_ThrowsInvalidDataCount() throws {
-    try interpreter.resizeInput(at: AddModel.validIndex, to: AddModel.shape)
-    try interpreter.allocateTensors()
-    let invalidData = Data(count: AddModel.dataCount - 1)
-    XCTAssertThrowsError(try interpreter.copy(
-      invalidData,
-      toInputAt: AddModel.validIndex
-    )) { error in
-      self.assertEqualErrors(
-        actual: error,
-        expected: .invalidTensorDataCount(provided: invalidData.count, required: AddModel.dataCount)
-      )
-    }
-  }
-
-  func testInterpreter_AllocateTensors() {
-    XCTAssertNoThrow(try interpreter.allocateTensors())
-  }
-
-  // MARK: - Private
-
-  private func setUpAddModelInputTensor() throws {
-    precondition(interpreter != nil)
-    try interpreter.resizeInput(at: AddModel.validIndex, to: AddModel.shape)
-    try interpreter.allocateTensors()
-    try interpreter.copy(AddModel.inputData, toInputAt: AddModel.validIndex)
-  }
-
-  private func setUpAddQuantizedModelInputTensor() throws {
-    precondition(interpreter != nil)
-    try interpreter.resizeInput(at: AddQuantizedModel.inputOutputIndex, to: AddQuantizedModel.shape)
-    try interpreter.allocateTensors()
-    try interpreter.copy(AddQuantizedModel.inputData, toInputAt: AddQuantizedModel.inputOutputIndex)
-  }
-
-  private func assertEqualErrors(actual: Error, expected: InterpreterError) {
-    guard let actual = actual as? InterpreterError else {
-      XCTFail("Actual error should be of type InterpreterError.")
-      return
-    }
-    XCTAssertEqual(actual, expected)
-  }
-}
-
-// MARK: - Constants
-
-/// Values for the `add.bin` model.
-private enum AddModel {
-  static let info = (name: "add", extension: "bin")
-  static let inputTensorCount = 1
-  static let outputTensorCount = 1
-  static let invalidIndex = 1
-  static let validIndex = 0
-  static let shape: TensorShape = [2]
-  static let dataCount = inputData.count
-  static let inputData = Data(copyingBufferOf: [Float32(1.0), Float32(3.0)])
-  static let outputData = Data(copyingBufferOf: [Float32(3.0), Float32(9.0)])
-  static let results = [Float32(3.0), Float32(9.0)]
-
-  static let inputTensor = Tensor(
-    name: "input",
-    dataType: .float32,
-    shape: shape,
-    data: inputData
-  )
-  static let outputTensor = Tensor(
-    name: "output",
-    dataType: .float32,
-    shape: shape,
-    data: outputData
-  )
-
-  static var path: String = {
-    let bundle = Bundle(for: InterpreterTests.self)
-    guard let path = bundle.path(forResource: info.name, ofType: info.extension) else { return "" }
-    return path
-  }()
-}
-
-/// Values for the `add_quantized.bin` model.
-private enum AddQuantizedModel {
-  static let info = (name: "add_quantized", extension: "bin")
-  static let inputOutputIndex = 0
-  static let shape: TensorShape = [2]
-  static let inputData = Data([1, 3])
-  static let outputData = Data([3, 9])
-  static let quantizationParameters = QuantizationParameters(scale: 0.003922, zeroPoint: 0)
-  static let results: [UInt8] = [3, 9]
-
-  static let inputTensor = Tensor(
-    name: "input",
-    dataType: .uInt8,
-    shape: shape,
-    data: inputData,
-    quantizationParameters: quantizationParameters
-  )
-  static let outputTensor = Tensor(
-    name: "output",
-    dataType: .uInt8,
-    shape: shape,
-    data: outputData,
-    quantizationParameters: quantizationParameters
-  )
-
-  static var path: String = {
-    let bundle = Bundle(for: InterpreterTests.self)
-    guard let path = bundle.path(forResource: info.name, ofType: info.extension) else { return "" }
-    return path
-  }()
-}
-
-// MARK: - Extensions
-
-extension Array {
-  /// Creates a new array from the bytes of the given unsafe data.
-  ///
-  /// - Note: Returns `nil` if `unsafeData.count` is not a multiple of
-  ///     `MemoryLayout<Element>.stride`.
-  /// - Parameter unsafeData: The data containing the bytes to turn into an array.
-  init?(unsafeData: Data) {
-    guard unsafeData.count % MemoryLayout<Element>.stride == 0 else { return nil }
-    let elements = unsafeData.withUnsafeBytes {
-      UnsafeBufferPointer<Element>(
-        start: $0,
-        count: unsafeData.count / MemoryLayout<Element>.stride
-      )
-    }
-    self.init(elements)
-  }
-}
-
-extension Data {
-  /// Creates a new buffer by copying the buffer pointer of the given array.
-  ///
-  /// - Warning: The given array's element type `T` must be trivial in that it can be copied bit
-  ///     for bit with no indirection or reference-counting operations; otherwise, reinterpreting
-  ///     data from the resulting buffer has undefined behavior.
-  /// - Parameter array: An array with elements of type `T`.
-  init<T>(copyingBufferOf array: [T]) {
-    self = array.withUnsafeBufferPointer(Data.init)
-  }
-}
diff --git a/tensorflow/lite/experimental/swift/Tests/ModelTests.swift b/tensorflow/lite/experimental/swift/Tests/ModelTests.swift
deleted file mode 100644
index 025db18906..0000000000
--- a/tensorflow/lite/experimental/swift/Tests/ModelTests.swift
+++ /dev/null
@@ -1,59 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-@testable import TensorFlowLite
-import XCTest
-
-class ModelTests: XCTestCase {
-
-  var modelPath: String!
-
-  override func setUp() {
-    super.setUp()
-
-    let bundle = Bundle(for: type(of: self))
-    guard let modelPath = bundle.path(
-            forResource: Constant.modelInfo.name,
-            ofType: Constant.modelInfo.extension)
-    else {
-      XCTFail("Failed to get the model file path.")
-      return
-    }
-    self.modelPath = modelPath
-  }
-
-  override func tearDown() {
-    modelPath = nil
-
-    super.tearDown()
-  }
-
-  func testModel_InitWithFilePath() {
-    XCTAssertNotNil(Model(filePath: modelPath))
-  }
-
-  func testModel_InitWithEmptyFilePath_FailsInitialization() {
-    XCTAssertNil(Model(filePath: ""))
-  }
-
-  func testModel_InitWithInvalidFilePath_FailsInitialization() {
-    XCTAssertNil(Model(filePath: "invalid/path"))
-  }
-}
-
-// MARK: - Constants
-
-private enum Constant {
-  static let modelInfo = (name: "add", extension: "bin")
-}
diff --git a/tensorflow/lite/experimental/swift/Tests/QuantizationParametersTests.swift b/tensorflow/lite/experimental/swift/Tests/QuantizationParametersTests.swift
deleted file mode 100644
index 65648c2698..0000000000
--- a/tensorflow/lite/experimental/swift/Tests/QuantizationParametersTests.swift
+++ /dev/null
@@ -1,43 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-@testable import TensorFlowLite
-import XCTest
-
-class QuantizationParametersTests: XCTestCase {
-
-  func testQuantizationParameters_InitWithCustomValues() {
-    let parameters = QuantizationParameters(scale: 0.5, zeroPoint: 1)
-    XCTAssertEqual(parameters.scale, 0.5)
-    XCTAssertEqual(parameters.zeroPoint, 1)
-  }
-
-  func testQuantizationParameters_Equatable() {
-    let parameters1 = QuantizationParameters(scale: 0.5, zeroPoint: 1)
-    let parameters2 = QuantizationParameters(scale: 0.5, zeroPoint: 1)
-    XCTAssertEqual(parameters1, parameters2)
-
-    let parameters3 = QuantizationParameters(scale: 0.4, zeroPoint: 1)
-    XCTAssertNotEqual(parameters1, parameters3)
-    XCTAssertNotEqual(parameters2, parameters3)
-  }
-}
-
-// MARK: - Extensions
-
-extension QuantizationParameters: Equatable {
-  public static func == (lhs: QuantizationParameters, rhs: QuantizationParameters) -> Bool {
-    return lhs.scale == rhs.scale && lhs.zeroPoint == rhs.zeroPoint
-  }
-}
diff --git a/tensorflow/lite/experimental/swift/Tests/TensorTests.swift b/tensorflow/lite/experimental/swift/Tests/TensorTests.swift
deleted file mode 100644
index 4540043a16..0000000000
--- a/tensorflow/lite/experimental/swift/Tests/TensorTests.swift
+++ /dev/null
@@ -1,83 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-@testable import TensorFlowLite
-import XCTest
-
-class TensorTests: XCTestCase {
-
-  // MARK: - Tensor
-
-  func testTensor_Init() {
-    let name = "InputTensor"
-    let dataType: TensorDataType = .uInt8
-    let shape = TensorShape(Constant.dimensions)
-    guard let data = name.data(using: .utf8) else { XCTFail("Data should not be nil."); return }
-    let quantizationParameters = QuantizationParameters(scale: 0.5, zeroPoint: 1)
-    let inputTensor = Tensor(
-      name: name,
-      dataType: dataType,
-      shape: shape,
-      data: data,
-      quantizationParameters: quantizationParameters
-    )
-    XCTAssertEqual(inputTensor.name, name)
-    XCTAssertEqual(inputTensor.dataType, dataType)
-    XCTAssertEqual(inputTensor.shape, shape)
-    XCTAssertEqual(inputTensor.data, data)
-    XCTAssertEqual(inputTensor.quantizationParameters, quantizationParameters)
-  }
-
-  // MARK: - TensorShape
-
-  func testTensorShape_InitWithArray() {
-    let shape = TensorShape(Constant.dimensions)
-    XCTAssertEqual(shape.rank, Constant.dimensions.count)
-    XCTAssertEqual(shape.dimensions, Constant.dimensions)
-  }
-
-  func testTensorShape_InitWithElements() {
-    let shape = TensorShape(2, 2, 3)
-    XCTAssertEqual(shape.rank, Constant.dimensions.count)
-    XCTAssertEqual(shape.dimensions, Constant.dimensions)
-  }
-
-  func testTensorShape_InitWithArrayLiteral() {
-    let shape: TensorShape = [2, 2, 3]
-    XCTAssertEqual(shape.rank, Constant.dimensions.count)
-    XCTAssertEqual(shape.dimensions, Constant.dimensions)
-  }
-}
-
-// MARK: - Constants
-
-private enum Constant {
-  /// Array of 2 arrays of 2 arrays of 3 numbers: [[[1, 1, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]]].
-  static let dimensions = [2, 2, 3]
-}
-
-// MARK: - Extensions
-
-extension TensorShape: Equatable {
-  public static func == (lhs: TensorShape, rhs: TensorShape) -> Bool {
-    return lhs.rank == rhs.rank && lhs.dimensions == rhs.dimensions
-  }
-}
-
-extension Tensor: Equatable {
-  public static func == (lhs: Tensor, rhs: Tensor) -> Bool {
-    return lhs.name == rhs.name && lhs.dataType == rhs.dataType && lhs.shape == rhs.shape &&
-           lhs.data == rhs.data && lhs.quantizationParameters == rhs.quantizationParameters
-  }
-}
diff --git a/tensorflow/tools/pip_package/pip_smoke_test.py b/tensorflow/tools/pip_package/pip_smoke_test.py
index 952c71c615..51d010c9e1 100644
--- a/tensorflow/tools/pip_package/pip_smoke_test.py
+++ b/tensorflow/tools/pip_package/pip_smoke_test.py
@@ -30,19 +30,14 @@ os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../..")))
 PIP_PACKAGE_QUERY_EXPRESSION = (
     "deps(//tensorflow/tools/pip_package:build_pip_package)")
 
-# List of file paths containing BUILD files that should not be included for the
-# pip smoke test.
-BUILD_BLACKLIST = [
-    "tensorflow/lite/examples/android",
-    "tensorflow/lite/experimental/swift",
-]
 
 def GetBuild(dir_base):
   """Get the list of BUILD file all targets recursively startind at dir_base."""
   items = []
   for root, _, files in os.walk(dir_base):
     for name in files:
-      if (name == "BUILD" and root not in BUILD_BLACKLIST):
+      if (name == "BUILD" and
+          root.find("tensorflow/lite/examples/android") == -1):
         items.append("//" + root + ":all")
   return items
 
@@ -72,9 +67,9 @@ def BuildPyTestDependencies():
 
 PYTHON_TARGETS, PY_TEST_QUERY_EXPRESSION = BuildPyTestDependencies()
 
+# Hard-coded blacklist of files if not included in pip package
 # TODO(amitpatankar): Clean up blacklist.
-# List of dependencies that should not included in the pip package.
-DEPENDENCY_BLACKLIST = [
+BLACKLIST = [
     "//tensorflow/python:extra_py_tests_deps",
     "//tensorflow/cc/saved_model:saved_model_half_plus_two",
     "//tensorflow:no_tensorflow_py_deps",
@@ -87,7 +82,9 @@ DEPENDENCY_BLACKLIST = [
     "//tensorflow/core/kernels/cloud:bigquery_reader_ops",
     "//tensorflow/python/feature_column:vocabulary_testdata",
     "//tensorflow/python:framework/test_file_system.so",
-    # lite
+    # contrib
+    "//tensorflow/contrib/session_bundle:session_bundle_half_plus_two",
+    "//tensorflow/contrib/keras:testing_utils",
     "//tensorflow/lite/experimental/examples/lstm:tflite_lstm",
     "//tensorflow/lite/experimental/examples/lstm:tflite_lstm.py",
     "//tensorflow/lite/experimental/examples/lstm:unidirectional_sequence_lstm_test",  # pylint:disable=line-too-long
@@ -96,9 +93,6 @@ DEPENDENCY_BLACKLIST = [
     "//tensorflow/lite/python:interpreter_test",
     "//tensorflow/lite/python:interpreter.py",
     "//tensorflow/lite/python:interpreter_test.py",
-    # contrib
-    "//tensorflow/contrib/session_bundle:session_bundle_half_plus_two",
-    "//tensorflow/contrib/keras:testing_utils",
     "//tensorflow/contrib/ffmpeg:test_data",
     "//tensorflow/contrib/fused_conv:fused_conv2d_bias_activation_op_test_base",
     "//tensorflow/contrib/hadoop:test_data",
@@ -155,8 +149,8 @@ def main():
   # File extensions and endings to ignore
   ignore_extensions = ["_test", "_test.py", "_test_gpu", "_test_gpu.py"]
 
-  ignored_files_count = 0
-  blacklisted_dependencies_count = len(DEPENDENCY_BLACKLIST)
+  ignored_files = 0
+  blacklisted_files = len(BLACKLIST)
   # Compare dependencies
   for dependency in tf_py_test_dependencies_list:
     if dependency and dependency.startswith("//tensorflow"):
@@ -164,16 +158,16 @@ def main():
       # Ignore extensions
       if any(dependency.endswith(ext) for ext in ignore_extensions):
         ignore = True
-        ignored_files_count += 1
+        ignored_files += 1
 
-      # Check if the dependency is in the pip package, the dependency blacklist,
-      # or should be ignored because of its file extension.
+      # Check if the dependency is in the pip package, the blacklist, or
+      # should be ignored because of its file extension
       if not (ignore or dependency in pip_package_dependencies_list or
-              dependency in DEPENDENCY_BLACKLIST):
+              dependency in BLACKLIST):
         missing_dependencies.append(dependency)
 
-  print("Ignored files count: %d" % ignored_files_count)
-  print("Blacklisted dependencies count: %d" % blacklisted_dependencies_count)
+  print("Ignored files: %d" % ignored_files)
+  print("Blacklisted files: %d" % blacklisted_files)
   if missing_dependencies:
     print("Missing the following dependencies from pip_packages:")
     for missing_dependency in missing_dependencies:
-- 
GitLab


From 2e55da567448374f9f7e2498c1e338d07baf812f Mon Sep 17 00:00:00 2001
From: Andy Ly <lyandy@google.com>
Date: Thu, 3 Jan 2019 18:15:14 -0800
Subject: [PATCH 0209/1095] [Grappler] MutableGraphView dedup control
 dependencies on init and on mutations. Update max regular output port on
 mutations.

PiperOrigin-RevId: 227783394
---
 tensorflow/core/grappler/graph_view.h         |   5 +-
 .../core/grappler/mutable_graph_view.cc       | 275 ++++++++----
 tensorflow/core/grappler/mutable_graph_view.h |  72 ++--
 .../core/grappler/mutable_graph_view_test.cc  | 395 ++++++++++++++----
 .../optimizers/data/graph_test_utils.cc       |   4 +-
 .../optimizers/data/latency_all_edges_test.cc |   4 +-
 6 files changed, 552 insertions(+), 203 deletions(-)

diff --git a/tensorflow/core/grappler/graph_view.h b/tensorflow/core/grappler/graph_view.h
index 16156d0f20..a17d17524a 100644
--- a/tensorflow/core/grappler/graph_view.h
+++ b/tensorflow/core/grappler/graph_view.h
@@ -172,9 +172,8 @@ class GraphViewInternal {
     if (fanin.index() < -1) {
       return false;
     }
-    string fanin_string = TensorIdToString(fanin);
-    for (int i = 0; i < node.input_size(); ++i) {
-      if (node.input(i) == fanin_string) {
+    for (const string& input : node.input()) {
+      if (ParseTensorName(input) == fanin) {
         return true;
       }
     }
diff --git a/tensorflow/core/grappler/mutable_graph_view.cc b/tensorflow/core/grappler/mutable_graph_view.cc
index ca4d5255c0..5df8bd8113 100644
--- a/tensorflow/core/grappler/mutable_graph_view.cc
+++ b/tensorflow/core/grappler/mutable_graph_view.cc
@@ -39,8 +39,103 @@ bool IsTensorIdPortValid(const TensorId& tensor_id) {
   return tensor_id.index() >= Graph::kControlSlot;
 }
 
+// Determines if node is an Identity where it's first regular input is a Switch
+// node.
+bool IsIdentityConsumingSwitch(const MutableGraphView& graph,
+                               const NodeDef& node) {
+  if ((IsIdentity(node) || IsIdentityNSingleInput(node)) &&
+      node.input_size() > 0) {
+    TensorId tensor_id = ParseTensorName(node.input(0));
+    if (tensor_id.index() == Graph::kControlSlot) return false;
+    NodeDef* input_node = graph.GetNode(tensor_id.node());
+    return IsSwitch(*input_node);
+  }
+  return false;
+}
+
+// Determines if node input can be deduped by regular inputs when used as a
+// control dependency. Specifically, if a node is an Identity that leads to a
+// Switch node, when used as a control dependency, that control dependency
+// should not be deduped even though the same node is used as a regular input.
+bool CanDedupControlWithRegularInput(const MutableGraphView& graph,
+                                     const NodeDef& control_node) {
+  return !IsIdentityConsumingSwitch(graph, control_node);
+}
+
+// Determines if node input can be deduped by regular inputs when used as a
+// control dependency. Specifically, if a node is an Identity that leads to a
+// Switch node, when used as a control dependency, that control dependency
+// should not be deduped even though the same node is used as a regular input.
+bool CanDedupControlWithRegularInput(const MutableGraphView& graph,
+                                     absl::string_view control_node_name) {
+  NodeDef* control_node = graph.GetNode(control_node_name);
+  return CanDedupControlWithRegularInput(graph, *control_node);
+}
+
 }  // namespace
 
+void MutableGraphView::AddAndDedupFanouts(NodeDef* node) {
+  absl::flat_hash_set<absl::string_view> fanins;
+  absl::flat_hash_set<absl::string_view> controlling_fanins;
+  int pos = 0;
+  const int last_idx = node->input_size() - 1;
+  int last_pos = last_idx;
+  while (pos <= last_pos) {
+    TensorId tensor_id = ParseTensorName(node->input(pos));
+    absl::string_view input_node_name = tensor_id.node();
+    bool is_control_input = IsControlInput(tensor_id);
+    bool can_dedup_control_with_regular_input =
+        CanDedupControlWithRegularInput(*this, input_node_name);
+    bool can_dedup_control =
+        is_control_input && (can_dedup_control_with_regular_input ||
+                             (!can_dedup_control_with_regular_input &&
+                              controlling_fanins.contains(input_node_name)));
+    if (!gtl::InsertIfNotPresent(&fanins, input_node_name) &&
+        can_dedup_control) {
+      node->mutable_input()->SwapElements(pos, last_pos--);
+    } else {
+      OutputPort output(nodes()[input_node_name], tensor_id.index());
+
+      if (is_control_input) {
+        fanouts()[output].emplace(node, Graph::kControlSlot);
+      } else {
+        max_regular_output_port()[output.node] =
+            std::max(max_regular_output_port()[output.node], output.port_id);
+        fanouts()[output].emplace(node, pos);
+      }
+      ++pos;
+    }
+    if (is_control_input) {
+      controlling_fanins.insert(input_node_name);
+    }
+  }
+
+  if (last_pos < last_idx) {
+    node->mutable_input()->DeleteSubrange(last_pos + 1, last_idx - last_pos);
+  }
+}
+
+void MutableGraphView::UpdateMaxRegularOutputPortForRemovedFanin(
+    const OutputPort& fanin,
+    const absl::flat_hash_set<InputPort>& fanin_fanouts) {
+  int max_port = max_regular_output_port()[fanin.node];
+  if (!fanin_fanouts.empty() || max_port != fanin.port_id) {
+    return;
+  }
+  bool updated_max_port = false;
+  for (int i = fanin.port_id - 1; i >= 0; --i) {
+    OutputPort fanin_port(fanin.node, i);
+    if (!fanouts()[fanin_port].empty()) {
+      max_regular_output_port()[fanin.node] = i;
+      updated_max_port = true;
+      break;
+    }
+  }
+  if (!updated_max_port) {
+    max_regular_output_port().erase(fanin.node);
+  }
+}
+
 const absl::flat_hash_set<MutableGraphView::InputPort>&
 MutableGraphView::GetFanout(const GraphView::OutputPort& port) const {
   return GetFanout(MutableGraphView::OutputPort(const_cast<NodeDef*>(port.node),
@@ -65,16 +160,16 @@ NodeDef* MutableGraphView::AddNode(NodeDef&& node) {
 
   AddUniqueNodeOrDie(node_in_graph);
 
-  AddFanouts(node_in_graph);
+  AddAndDedupFanouts(node_in_graph);
   return node_in_graph;
 }
 
-void MutableGraphView::UpdateFanouts(absl::string_view from_node,
+bool MutableGraphView::UpdateFanouts(absl::string_view from_node,
                                      absl::string_view to_node) {
   NodeDef* from_node_ptr = GetNode(from_node);
   NodeDef* to_node_ptr = GetNode(to_node);
   if (from_node_ptr && to_node_ptr) {
-    UpdateFanouts(from_node_ptr, to_node_ptr);
+    return UpdateFanoutsInternal(from_node_ptr, to_node_ptr);
   } else if (!from_node_ptr) {
     LOG(WARNING) << absl::Substitute(
         "Can't update fanouts from '$0' to '$1', from node was not found.",
@@ -84,9 +179,11 @@ void MutableGraphView::UpdateFanouts(absl::string_view from_node,
         "Can't update fanouts from '$0' to '$1', to node was not found.",
         from_node, to_node);
   }
+  return false;
 }
 
-void MutableGraphView::UpdateFanouts(NodeDef* from_node, NodeDef* to_node) {
+bool MutableGraphView::UpdateFanoutsInternal(NodeDef* from_node,
+                                             NodeDef* to_node) {
   VLOG(2) << absl::Substitute("Update fanouts from '$0' to '$1'.",
                               from_node->name(), to_node->name());
 
@@ -112,6 +209,7 @@ void MutableGraphView::UpdateFanouts(NodeDef* from_node, NodeDef* to_node) {
   // input to some other node.
   int keep_max_regular_output_port = -1;
 
+  bool modified = false;
   for (const Edge& edge : regular_edges) {
     const OutputPort output_port = edge.src;
     const InputPort input_port = edge.dst;
@@ -120,7 +218,7 @@ void MutableGraphView::UpdateFanouts(NodeDef* from_node, NodeDef* to_node) {
     // AddAndUpdateFanoutsWithoutSelfLoops test for an example).
     if (input_port.node == to_node) {
       keep_max_regular_output_port =
-          std::max(keep_max_regular_output_port, input_port.port_id);
+          std::max(keep_max_regular_output_port, output_port.port_id);
       continue;
     }
 
@@ -135,6 +233,11 @@ void MutableGraphView::UpdateFanouts(NodeDef* from_node, NodeDef* to_node) {
     remove_edge(output_port, input_port);
     // Add an edge between the `to_node` and new fanout node.
     add_edge(OutputPort(to_node, output_port.port_id), input_port);
+    // Dedup control dependency.
+    if (CanDedupControlWithRegularInput(*this, *to_node)) {
+      RemoveControllingFaninInternal(input_port.node, to_node);
+    }
+    modified = true;
   }
 
   // For the control fanouts we do not know the input index in a NodeDef,
@@ -142,7 +245,6 @@ void MutableGraphView::UpdateFanouts(NodeDef* from_node, NodeDef* to_node) {
 
   auto control_fanouts =
       GetFanout(GraphView::OutputPort(from_node, Graph::kControlSlot));
-  if (control_fanouts.empty()) return;
 
   const string from_control_input = absl::StrCat("^", from_node->name());
   const string to_control_input = absl::StrCat("^", to_node->name());
@@ -151,20 +253,9 @@ void MutableGraphView::UpdateFanouts(NodeDef* from_node, NodeDef* to_node) {
     // Node can't be control dependency of itself.
     if (control_port.node == to_node) continue;
 
-    // Find and update input corresponding to control dependency.
     NodeDef* node = control_port.node;
-    for (int i = node->input_size() - 1; i >= 0; --i) {
-      const string& input = node->input(i);
-      if (!IsControlInput(input)) break;  // we reached regular inputs
-      if (input == from_control_input) {
-        node->set_input(i, to_control_input);
-      }
-    }
-
-    // Remove old edge between the `from_node` and the fanout node.
-    remove_edge(OutputPort(from_node, Graph::kControlSlot), control_port);
-    // Add an edge between the `to_node` and new fanout node.
-    add_edge(OutputPort(to_node, Graph::kControlSlot), control_port);
+    modified |= RemoveControllingFaninInternal(node, from_node);
+    modified |= AddFaninInternal(node, {to_node, Graph::kControlSlot});
   }
 
   // Because we update all regular fanouts of `from_node`, we can just copy
@@ -177,21 +268,33 @@ void MutableGraphView::UpdateFanouts(NodeDef* from_node, NodeDef* to_node) {
   } else {
     max_regular_output_port().erase(from_node);
   }
+
+  return modified;
 }
 
 bool MutableGraphView::AddFaninInternal(NodeDef* node,
                                         const OutputPort& fanin) {
   int num_non_controlling_fanins =
       NumFanins(*node, /*include_controlling_nodes=*/false);
+  bool input_is_control = fanin.port_id == Graph::kControlSlot;
+  bool can_dedup_control_with_regular_input =
+      CanDedupControlWithRegularInput(*this, *fanin.node);
+  // Don't add duplicate control dependencies.
+  if (input_is_control) {
+    const int start =
+        can_dedup_control_with_regular_input ? 0 : num_non_controlling_fanins;
+    for (int i = start; i < node->input_size(); ++i) {
+      if (ParseTensorName(node->input(i)).node() == fanin.node->name()) {
+        return false;
+      }
+    }
+  }
+
   InputPort input;
   input.node = node;
-  input.port_id = fanin.port_id == Graph::kControlSlot
-                      ? Graph::kControlSlot
-                      : num_non_controlling_fanins;
+  input.port_id =
+      input_is_control ? Graph::kControlSlot : num_non_controlling_fanins;
 
-  if (!gtl::InsertIfNotPresent(&fanouts()[fanin], input)) {
-    return false;
-  }
   node->add_input(TensorIdToString({fanin.node->name(), fanin.port_id}));
   if (fanin.port_id > Graph::kControlSlot) {
     int node_input_size = node->input_size() - 1;
@@ -202,6 +305,17 @@ bool MutableGraphView::AddFaninInternal(NodeDef* node,
                                           num_non_controlling_fanins);
     }
   }
+
+  fanouts()[fanin].insert(input);
+  if (max_regular_output_port()[fanin.node] < fanin.port_id) {
+    max_regular_output_port()[fanin.node] = fanin.port_id;
+  }
+
+  // Dedup control dependencies.
+  if (!input_is_control && can_dedup_control_with_regular_input) {
+    RemoveControllingFaninInternal(node, fanin.node);
+  }
+
   return true;
 }
 
@@ -244,16 +358,18 @@ bool MutableGraphView::RemoveFanins(NodeDef* node,
       input.port_id =
           tensor_id.index() == Graph::kControlSlot ? Graph::kControlSlot : i;
 
+      auto& fanouts_set = fanouts()[fanin];
       if (remove_fanin) {
-        fanouts()[fanin].erase(input);
+        fanouts_set.erase(input);
       } else {
         // Shift inputs to be retained.
         if (tensor_id.index() > Graph::kControlSlot) {
-          fanouts()[fanin].erase(input);
-          fanouts()[fanin].insert(InputPort(node, i));
+          fanouts_set.erase(input);
+          fanouts_set.insert(InputPort(node, i));
         }
         mutable_inputs->SwapElements(i, curr_pos++);
       }
+      UpdateMaxRegularOutputPortForRemovedFanin(fanin, fanouts_set);
 
       modified = true;
     } else {
@@ -314,15 +430,12 @@ bool MutableGraphView::UpdateFanin(absl::string_view node_name,
     return false;
   }
 
-  bool is_from_fanin_control = from_fanin.index() == Graph::kControlSlot;
-  bool is_to_fanin_control = to_fanin.index() == Graph::kControlSlot;
   // When replacing a non control dependency fanin with a control dependency, or
   // vice versa, remove and add, so ports can be updated properly in fanout(s).
-  if (is_from_fanin_control || is_to_fanin_control) {
+  if (from_fanin.index() == Graph::kControlSlot ||
+      to_fanin.index() == Graph::kControlSlot) {
     bool modified = RemoveFanins(node, {from_fanin});
-    if (!HasFanin(*node, to_fanin)) {
-      modified |= AddFaninInternal(node, to_fanin);
-    }
+    modified |= AddFaninInternal(node, to_fanin);
     return modified;
   }
 
@@ -336,79 +449,47 @@ bool MutableGraphView::UpdateFanin(absl::string_view node_name,
   string to_fanin_string = TensorIdToString(to_fanin);
   int num_inputs = node->input_size();
   bool modified = false;
+  absl::flat_hash_set<InputPort>* from_fanin_port_fanouts = nullptr;
+  absl::flat_hash_set<InputPort>* to_fanin_port_fanouts = nullptr;
   for (int i = 0; i < num_inputs; ++i) {
     if (ParseTensorName(node->input(i)) == from_fanin) {
-      OutputPort from_fanin_port(from_fanin_node, from_fanin.index());
       InputPort old_input;
       old_input.node = node;
       old_input.port_id =
           from_fanin.index() == Graph::kControlSlot ? Graph::kControlSlot : i;
-      fanouts()[from_fanin_port].erase(old_input);
+      if (from_fanin_port_fanouts == nullptr) {
+        OutputPort from_fanin_port(from_fanin_node, from_fanin.index());
+        from_fanin_port_fanouts = &fanouts()[from_fanin_port];
+      }
+      from_fanin_port_fanouts->erase(old_input);
 
-      OutputPort to_fanin_port(to_fanin_node, to_fanin.index());
       InputPort new_input;
       new_input.node = node;
       new_input.port_id =
           to_fanin.index() == Graph::kControlSlot ? Graph::kControlSlot : i;
-      fanouts()[to_fanin_port].insert(new_input);
+      if (to_fanin_port_fanouts == nullptr) {
+        OutputPort to_fanin_port(to_fanin_node, to_fanin.index());
+        to_fanin_port_fanouts = &fanouts()[to_fanin_port];
+      }
+      to_fanin_port_fanouts->insert(new_input);
 
       node->set_input(i, to_fanin_string);
       modified = true;
     }
   }
 
-  return modified;
-}
-
-bool MutableGraphView::DedupControllingFanins(NodeDef* node) {
-  absl::flat_hash_set<absl::string_view> fanins;
-  absl::flat_hash_set<string> removed_fanins;
-  int pos = 0;
-  const int last_idx = node->input_size() - 1;
-  int last_pos = last_idx;
-  while (pos <= last_pos) {
-    const string& input = node->input(pos);
-    TensorId tensor_id = ParseTensorName(input);
-    if (!gtl::InsertIfNotPresent(&fanins, tensor_id.node()) &&
-        IsControlInput(tensor_id)) {
-      node->mutable_input()->SwapElements(pos, last_pos--);
-      removed_fanins.insert(input);
-    } else {
-      ++pos;
+  // Dedup control dependencies and update max regular output ports.
+  if (modified) {
+    UpdateMaxRegularOutputPortForRemovedFanin(
+        {from_fanin_node, from_fanin.index()}, *from_fanin_port_fanouts);
+    if (max_regular_output_port()[to_fanin_node] < to_fanin.index()) {
+      max_regular_output_port()[to_fanin_node] = to_fanin.index();
     }
-  }
-
-  if (last_pos < last_idx) {
-    absl::flat_hash_set<string> retained_fanins(
-        node->input().begin(), node->input().begin() + last_pos + 1);
-    for (const auto& removed : removed_fanins) {
-      if (!retained_fanins.contains(removed)) {
-        OutputPort fanin(nodes()[ParseTensorName(removed).node()],
-                         Graph::kControlSlot);
-        fanouts()[fanin].erase({node, Graph::kControlSlot});
-      }
+    if (CanDedupControlWithRegularInput(*this, *to_fanin_node)) {
+      RemoveControllingFaninInternal(node, to_fanin_node);
     }
-    node->mutable_input()->DeleteSubrange(last_pos + 1, last_idx - last_pos);
-    return true;
   }
 
-  return false;
-}
-
-bool MutableGraphView::DedupControllingFanins(absl::string_view node_name) {
-  NodeDef* node = GetNode(node_name);
-  if (node == nullptr) {
-    return false;
-  }
-  return DedupControllingFanins(node);
-}
-
-bool MutableGraphView::DedupControllingFanins() {
-  const int num_nodes = graph()->node_size();
-  bool modified = false;
-  for (int i = 0; i < num_nodes; ++i) {
-    modified |= DedupControllingFanins(graph()->mutable_node(i));
-  }
   return modified;
 }
 
@@ -463,6 +544,24 @@ bool MutableGraphView::AddControllingFanin(absl::string_view node_name,
   }
 }
 
+bool MutableGraphView::RemoveControllingFaninInternal(NodeDef* node,
+                                                      NodeDef* fanin_node) {
+  for (int i = node->input_size() - 1; i >= 0; --i) {
+    TensorId tensor_id = ParseTensorName(node->input(i));
+    if (tensor_id.index() > Graph::kControlSlot) {
+      break;
+    }
+    if (tensor_id.node() == fanin_node->name()) {
+      fanouts()[{fanin_node, Graph::kControlSlot}].erase(
+          {node, Graph::kControlSlot});
+      node->mutable_input()->SwapElements(i, node->input_size() - 1);
+      node->mutable_input()->RemoveLast();
+      return true;
+    }
+  }
+  return false;
+}
+
 void MutableGraphView::DeleteNodes(const std::set<string>& nodes_to_delete) {
   for (const string& node_name_to_delete : nodes_to_delete)
     RemoveFaninsInternal(nodes().at(node_name_to_delete),
@@ -488,7 +587,9 @@ void MutableGraphView::RemoveFaninsInternal(NodeDef* deleted_node,
     else
       input.port_id = i;
 
-    fanouts()[fanin].erase(input);
+    auto& fanouts_set = fanouts()[fanin];
+    fanouts_set.erase(input);
+    UpdateMaxRegularOutputPortForRemovedFanin(fanin, fanouts_set);
   }
 }
 
diff --git a/tensorflow/core/grappler/mutable_graph_view.h b/tensorflow/core/grappler/mutable_graph_view.h
index f7c2a1118e..93f0958d5e 100644
--- a/tensorflow/core/grappler/mutable_graph_view.h
+++ b/tensorflow/core/grappler/mutable_graph_view.h
@@ -24,8 +24,10 @@ limitations under the License.
 #include "absl/types/span.h"
 #include "tensorflow/core/framework/graph.pb.h"
 #include "tensorflow/core/framework/node_def.pb.h"
+#include "tensorflow/core/graph/graph.h"
 #include "tensorflow/core/graph/tensor_id.h"
 #include "tensorflow/core/grappler/graph_view.h"
+#include "tensorflow/core/grappler/op_types.h"
 #include "tensorflow/core/platform/types.h"
 
 namespace tensorflow {
@@ -41,7 +43,7 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
  public:
   explicit MutableGraphView(GraphDef* graph) : GraphViewInternal(graph) {
     for (NodeDef& node : *graph->mutable_node()) AddUniqueNodeOrDie(&node);
-    for (NodeDef& node : *graph->mutable_node()) AddFanouts(&node);
+    for (NodeDef& node : *graph->mutable_node()) AddAndDedupFanouts(&node);
   }
 
   // Lookup fanouts/fanins using immutable ports.
@@ -63,16 +65,20 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   // Updates all fanouts (input ports fetching output tensors) from `from_node`
   // to the `to_node`, including control dependencies.
   //
-  // Example: We have 2 nodes that use `bar` node output tensors as inputs:
-  //   1. foo1(bar:0, bar:1, other:0, ^bar)
+  // Example: We have 3 nodes that use `bar` node output tensors as inputs:
+  //   1. foo1(bar:0, bar:1, other:0)
   //   2. foo2(bar:1, other:1)
+  //   3. foo3(other:2, ^bar)
   //
   // After calling ForwardOutputs(bar, new_bar):
-  //   1. foo1(new_bar:0, new_bar:1, other:0, ^new_bar)
+  //   1. foo1(new_bar:0, new_bar:1, other:0)
   //   2. foo2(new_bar:1, other:1)
-  void UpdateFanouts(absl::string_view from_node, absl::string_view to_node);
+  //   3. foo3(other:2, ^new_bar)
+  //
+  // This will return true iff the nodes are modified.
+  bool UpdateFanouts(absl::string_view from_node, absl::string_view to_node);
 
-  // Add fanin to node `node_name`. If the node or fanin do not exist in the
+  // Adds fanin to node `node_name`. If the node or fanin do not exist in the
   // graph, nothing will be modified in the graph. If fanin is a control
   // dependency, existing control dependencies will be checked first before
   // adding. Otherwise fanin will be added after existing non control dependency
@@ -82,7 +88,7 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   // already exists, the node will not be modified.
   bool AddFanin(absl::string_view node_name, const TensorId& fanin);
 
-  // Remove fanin from node `node_name`. If the node or fanin do not exist in
+  // Removes fanin from node `node_name`. If the node or fanin do not exist in
   // the graph, nothing will be modified in the graph. If there are multiple
   // inputs that match the fanin, all of them will be removed.
   //
@@ -90,32 +96,20 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   // fanin, the node will not be modified.
   bool RemoveFanin(absl::string_view node_name, const TensorId& fanin);
 
-  // Remove all fanins from node `node_name`. Control dependencies will be
+  // Removes all fanins from node `node_name`. Control dependencies will be
   // retained if keep_controlling_fanins is true.
   //
   // This will return true iff the node is modified.
   bool RemoveAllFanins(absl::string_view node_name,
                        bool keep_controlling_fanins);
 
-  // Replace all fanins `from_fanin` with `to_fanin` in node `node_name`. If
+  // Replaces all fanins `from_fanin` with `to_fanin` in node `node_name`. If
   // the fanins or node do not exist, nothing will be modified in the graph.
   //
   // This will return true iff the node is modified.
   bool UpdateFanin(absl::string_view node_name, const TensorId& from_fanin,
                    const TensorId& to_fanin);
 
-  // Removes redundant control fanins from node `node_name`.
-  //
-  // This will return true iff the node is modified.
-  // TODO(lyandy): Measure performance of deduping on every AddFanin compared to
-  // deduping once at the end.
-  bool DedupControllingFanins(absl::string_view node_name);
-
-  // Removes redundant control fanins from all nodes in the graph.
-  //
-  // This will return true iff the node is modified.
-  bool DedupControllingFanins();
-
   // Adds a control dependency to the target node named `node_name`.
   //
   // Case 1: If the fanin is not a Switch node, the control dependency is simply
@@ -140,27 +134,42 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   void DeleteNodes(const std::set<string>& nodes_to_delete);
 
  private:
+  // Adds fanouts for fanins of node to graph, while deduping control
+  // dependencies from existing control dependencies and regular fanins. Note,
+  // node inputs will be mutated if control dependencies can be deduped.
+  void AddAndDedupFanouts(NodeDef* node);
+
+  // Finds next output port smaller than fanin.port_id and update. The
+  // max_regular_output_port is only updated if fanin.port_id is the same as the
+  // current max_regular_output_port and if the fanouts set is empty. If there
+  // are no regular outputs, max_regular_output_port will be erased.
+  void UpdateMaxRegularOutputPortForRemovedFanin(
+      const OutputPort& fanin,
+      const absl::flat_hash_set<InputPort>& fanin_fanouts);
+
   // Updates all fanouts (input ports fetching output tensors) from `from_node`
   // to the `to_node`, including control dependencies.
   //
-  // Example: We have 2 nodes that use `bar` node output tensors as inputs:
-  //   1. foo1(bar:0, bar:1, other:0, ^bar)
+  // Example: We have 3 nodes that use `bar` node output tensors as inputs:
+  //   1. foo1(bar:0, bar:1, other:0)
   //   2. foo2(bar:1, other:1)
+  //   3. foo3(other:2, ^bar)
   //
   // After calling ForwardOutputs(bar, new_bar):
-  //   1. foo1(new_bar:0, new_bar:1, other:0, ^new_bar)
+  //   1. foo1(new_bar:0, new_bar:1, other:0)
   //   2. foo2(new_bar:1, other:1)
+  //   3. foo3(other:2, ^new_bar)
   //
   // IMPORTANT: If `from_node` or `to_node` is not in the underlying graph, the
   // behavior is undefined.
-  void UpdateFanouts(NodeDef* from_node, NodeDef* to_node);
+  bool UpdateFanoutsInternal(NodeDef* from_node, NodeDef* to_node);
 
   // Removes fanins of the deleted node from internal state. Control
   // dependencies are retained iff keep_controlling_fanins is true.
   void RemoveFaninsInternal(NodeDef* deleted_node,
                             bool keep_controlling_fanins);
 
-  // Add fanin to node. If fanin is a control dependency, existing control
+  // Adds fanin to node. If fanin is a control dependency, existing control
   // dependencies will be checked first before adding. Otherwise fanin will be
   // added after existing non control dependency inputs.
   //
@@ -168,7 +177,7 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   // already exists, the node will not be modified.
   bool AddFaninInternal(NodeDef* node, const OutputPort& fanin);
 
-  // Add fanin to node. If the node or fanin do not exist in the graph, nothing
+  // Adds fanin to node. If the node or fanin do not exist in the graph, nothing
   // will be modified in the graph. If fanin is a control dependency, existing
   // control dependencies will be checked first before adding. Otherwise fanin
   // will be added after existing non control dependency inputs.
@@ -178,10 +187,15 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   bool AddFaninInternal(NodeDef* node, const TensorId& fanin);
 
   // Removes any fanin in node that matches to a fanin in fanins.
+  //
+  // This will return true iff the node is modified.
   bool RemoveFanins(NodeDef* node, absl::Span<const TensorId> fanins);
 
-  // Removes redundant control fanins from node.
-  bool DedupControllingFanins(NodeDef* node);
+  // Removes controlling fanin `fanin_node` from node if such controlling fanin
+  // exists.
+  //
+  // This will return true iff the node is modified.
+  bool RemoveControllingFaninInternal(NodeDef* node, NodeDef* fanin_node);
 };
 
 }  // end namespace grappler
diff --git a/tensorflow/core/grappler/mutable_graph_view_test.cc b/tensorflow/core/grappler/mutable_graph_view_test.cc
index cdc212f6f9..526ca85ec0 100644
--- a/tensorflow/core/grappler/mutable_graph_view_test.cc
+++ b/tensorflow/core/grappler/mutable_graph_view_test.cc
@@ -35,7 +35,8 @@ TEST(MutableGraphViewTest, AddAndUpdateFanouts) {
       {NDef("bar", "NotImportant", {}, {}),
        NDef("other", "NotImportant", {}, {}),
        NDef("foo_1", "NotImportant", {"bar", "other", "bar:1", "^bar"}),
-       NDef("foo_2", "NotImportant", {"other:1", "bar:2", "^bar"})},
+       NDef("foo_2", "NotImportant", {"other:1", "bar:2", "^bar"}),
+       NDef("foo_3", "NotImportant", {"other:2", "^bar"})},
       /*funcs=*/{});
 
   MutableGraphView graph(&graph_def);
@@ -43,7 +44,56 @@ TEST(MutableGraphViewTest, AddAndUpdateFanouts) {
   NodeDef* new_bar = graph.AddNode(NDef("new_bar", "NotImportant", {}, {}));
   NodeDef* bar = graph.GetNode("bar");
 
-  graph.UpdateFanouts(bar->name(), new_bar->name());
+  EXPECT_TRUE(graph.UpdateFanouts(bar->name(), new_bar->name()));
+
+  // Fanout nodes must have their inputs updated.
+  NodeDef* foo_1 = graph.GetNode("foo_1");
+  ASSERT_NE(foo_1, nullptr);
+  ASSERT_EQ(foo_1->input_size(), 3);
+  EXPECT_EQ(foo_1->input(0), "new_bar");
+  EXPECT_EQ(foo_1->input(1), "other");
+  EXPECT_EQ(foo_1->input(2), "new_bar:1");
+
+  NodeDef* foo_2 = graph.GetNode("foo_2");
+  ASSERT_NE(foo_2, nullptr);
+  ASSERT_EQ(foo_2->input_size(), 2);
+  EXPECT_EQ(foo_2->input(0), "other:1");
+  EXPECT_EQ(foo_2->input(1), "new_bar:2");
+
+  NodeDef* foo_3 = graph.GetNode("foo_3");
+  ASSERT_NE(foo_3, nullptr);
+  ASSERT_EQ(foo_3->input_size(), 2);
+  EXPECT_EQ(foo_3->input(0), "other:2");
+  EXPECT_EQ(foo_3->input(1), "^new_bar");
+
+  // And fanouts mapping must be also updated for both nodes.
+  bool include_control_fanouts = true;
+  auto old_node_fanouts = graph.GetFanouts(*bar, include_control_fanouts);
+  auto new_node_fanouts = graph.GetFanouts(*new_bar, include_control_fanouts);
+
+  EXPECT_TRUE(old_node_fanouts.empty());
+
+  EXPECT_EQ(new_node_fanouts.size(), 4);
+  EXPECT_EQ(new_node_fanouts.count(MutableGraphView::InputPort(foo_1, 0)), 1);
+  EXPECT_EQ(new_node_fanouts.count(MutableGraphView::InputPort(foo_1, 2)), 1);
+  EXPECT_EQ(new_node_fanouts.count(MutableGraphView::InputPort(foo_2, 1)), 1);
+  EXPECT_EQ(new_node_fanouts.count(MutableGraphView::InputPort(foo_3, -1)), 1);
+}
+
+TEST(MutableGraphViewTest, AddAndUpdateFanoutsKeepControls) {
+  GraphDef graph_def = test::function::GDef(
+      {NDef("bar_1", "Switch", {}, {}), NDef("bar_2", "Identity", {"bar_1:1"}),
+       NDef("other", "NotImportant", {}, {}),
+       NDef("foo_1", "NotImportant", {"bar_2", "other", "bar_2:1", "^bar_2"}),
+       NDef("foo_2", "NotImportant", {"other:1", "bar_2:2", "^bar_2"})},
+      /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  NodeDef* new_bar = graph.AddNode(NDef("new_bar", "Identity", {"bar_1:2"}));
+  NodeDef* bar_2 = graph.GetNode("bar_2");
+
+  EXPECT_TRUE(graph.UpdateFanouts(bar_2->name(), new_bar->name()));
 
   // Fanout nodes must have their inputs updated.
   NodeDef* foo_1 = graph.GetNode("foo_1");
@@ -63,7 +113,7 @@ TEST(MutableGraphViewTest, AddAndUpdateFanouts) {
 
   // And fanouts mapping must be also updated for both nodes.
   bool include_control_fanouts = true;
-  auto old_node_fanouts = graph.GetFanouts(*bar, include_control_fanouts);
+  auto old_node_fanouts = graph.GetFanouts(*bar_2, include_control_fanouts);
   auto new_node_fanouts = graph.GetFanouts(*new_bar, include_control_fanouts);
 
   EXPECT_TRUE(old_node_fanouts.empty());
@@ -78,7 +128,8 @@ TEST(MutableGraphViewTest, AddAndUpdateFanoutsWithoutSelfLoops) {
   // Actual node.op() is not important in this test.
   GraphDef graph_def =
       test::function::GDef({NDef("bar", "NotImportant", {}, {}),
-                            NDef("foo", "NotImportant", {"bar", "^bar"})},
+                            NDef("foo_1", "NotImportant", {"bar", "^bar"}),
+                            NDef("foo_2", "NotImportant", {"^bar"})},
                            /*funcs=*/{});
 
   MutableGraphView graph(&graph_def);
@@ -87,14 +138,18 @@ TEST(MutableGraphViewTest, AddAndUpdateFanoutsWithoutSelfLoops) {
   NodeDef* new_bar = graph.AddNode(NDef("new_bar", "NewBar", {"bar"}, {}));
   NodeDef* bar = graph.GetNode("bar");
 
-  graph.UpdateFanouts("bar", new_bar->name());
+  EXPECT_TRUE(graph.UpdateFanouts("bar", new_bar->name()));
 
   // Foo node must read from `new_bar`.
-  NodeDef* foo = graph.GetNode("foo");
-  ASSERT_NE(foo, nullptr);
-  ASSERT_EQ(foo->input_size(), 2);
-  EXPECT_EQ(foo->input(0), "new_bar");
-  EXPECT_EQ(foo->input(1), "^new_bar");
+  NodeDef* foo_1 = graph.GetNode("foo_1");
+  ASSERT_NE(foo_1, nullptr);
+  ASSERT_EQ(foo_1->input_size(), 1);
+  EXPECT_EQ(foo_1->input(0), "new_bar");
+
+  NodeDef* foo_2 = graph.GetNode("foo_2");
+  ASSERT_NE(foo_2, nullptr);
+  ASSERT_EQ(foo_2->input_size(), 1);
+  EXPECT_EQ(foo_2->input(0), "^new_bar");
 
   // And the `new_bar` should read from the original `bar`.
   ASSERT_EQ(new_bar->input_size(), 1);
@@ -109,8 +164,8 @@ TEST(MutableGraphViewTest, AddAndUpdateFanoutsWithoutSelfLoops) {
   EXPECT_EQ(bar_fanouts.count(MutableGraphView::InputPort(new_bar, 0)), 1);
 
   EXPECT_EQ(new_bar_fanouts.size(), 2);
-  EXPECT_EQ(new_bar_fanouts.count(MutableGraphView::InputPort(foo, 0)), 1);
-  EXPECT_EQ(new_bar_fanouts.count(MutableGraphView::InputPort(foo, -1)), 1);
+  EXPECT_EQ(new_bar_fanouts.count(MutableGraphView::InputPort(foo_1, 0)), 1);
+  EXPECT_EQ(new_bar_fanouts.count(MutableGraphView::InputPort(foo_2, -1)), 1);
 }
 
 GraphDef SimpleMutateFaninGraph() {
@@ -176,7 +231,7 @@ TEST(MutableGraphViewTest, AddFanin) {
   expected_node = NDef("", "", {"b", "a:1", "a:1", "b:2"});
   TestAddFanin("foo_3", {"b", 2}, /*modified=*/true, &expected_node);
   // Add input to node with 1 input multiple controls.
-  expected_node = NDef("", "", {"b", "a", "^c", "^a"});
+  expected_node = NDef("", "", {"b", "a", "^c"});
   TestAddFanin("foo_2", {"a", 0}, /*modified=*/true, &expected_node);
   // Add input to node with multiple inputs and controls.
   expected_node = NDef("", "", {"a", "b:2", "b:2", "a:1", "^d", "^c"});
@@ -201,8 +256,8 @@ TEST(MutableGraphViewTest, AddFanin) {
   TestAddFanin("foo_2", {"d", Graph::kControlSlot}, /*modified=*/true,
                &expected_node);
   // Add control to node with multiple input multiple controls.
-  expected_node = NDef("", "", {"a", "b:2", "b:2", "^c", "^d", "^a"});
-  TestAddFanin("foo_4", {"a", Graph::kControlSlot}, /*modified=*/true,
+  expected_node = NDef("", "", {"a", "b:2", "b:2", "^c", "^d"});
+  TestAddFanin("foo_4", {"a", Graph::kControlSlot}, /*modified=*/false,
                &expected_node);
   // Add control to node with 0 inputs 0 controls.
   expected_node = NDef("", "", {"^a"});
@@ -431,7 +486,7 @@ TEST(MutableGraphViewTest, UpdateFanin) {
   TestUpdateFanin("foo_4", {"d", Graph::kControlSlot}, {"d", 1},
                   /*modified=*/true, &expected_node);
   // Update fanin from control to control.
-  expected_node = NDef("", "", {"a", "b:2", "b:2", "^d", "^b"});
+  expected_node = NDef("", "", {"a", "b:2", "b:2", "^d"});
   TestUpdateFanin("foo_4", {"c", Graph::kControlSlot},
                   {"b", Graph::kControlSlot}, /*modified=*/true,
                   &expected_node);
@@ -463,85 +518,266 @@ TEST(MutableGraphViewTest, UpdateFanin) {
                   /*modified=*/false, /*expected_node=*/nullptr);
 }
 
-GraphDef SimpleDuplicateControllingFaninsGraph() {
+TEST(MutableGraphViewTest, DedupControllingFaninsOnGraphInit) {
+  // Actual node.op() is not important in this test.
+  GraphDef graph_def = test::function::GDef(
+      {
+          NDef("a", "NotImportant", {}, {}),
+          NDef("b", "NotImportant", {}, {}),
+          NDef("c", "Switch", {}, {}),
+          NDef("d", "Identity", {"c:1"}),
+          NDef("foo_1", "IdentityN", {"a", "b:1", "^b"}),
+          NDef("foo_2", "IdentityN", {"a", "^b", "^b"}),
+          NDef("foo_3", "IdentityN", {"a", "b:1", "^b", "^b"}),
+          NDef("foo_4", "IdentityN", {"a:2", "b:1", "^b", "^b", "^a", "^a"}),
+          NDef("foo_5", "NotImportant", {"a:2", "b:1", "^b", "^b", "^a", "^a"}),
+          NDef("foo_6", "Identity", {"d", "^d"}),
+          NDef("foo_7", "NotImportant",
+               {"a:3", "b:2", "d", "^d", "^d", "^a", "^b", "^a", "^b"}),
+      },
+      /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  EXPECT_EQ(graph.graph()->node_size(), 11);
+  NodeDef* a = graph.GetNode("a");
+  ASSERT_NE(a, nullptr);
+  ASSERT_EQ(a->input_size(), 0);
+  NodeDef* b = graph.GetNode("b");
+  ASSERT_NE(b, nullptr);
+  ASSERT_EQ(b->input_size(), 0);
+  NodeDef* c = graph.GetNode("c");
+  ASSERT_NE(c, nullptr);
+  ASSERT_EQ(c->input_size(), 0);
+  NodeDef* d = graph.GetNode("d");
+  ASSERT_NE(d, nullptr);
+  ASSERT_EQ(d->input_size(), 1);
+  EXPECT_EQ(d->input(0), "c:1");
+  NodeDef* foo_1 = graph.GetNode("foo_1");
+  ASSERT_NE(foo_1, nullptr);
+  ASSERT_EQ(foo_1->input_size(), 2);
+  EXPECT_EQ(foo_1->input(0), "a");
+  EXPECT_EQ(foo_1->input(1), "b:1");
+  NodeDef* foo_2 = graph.GetNode("foo_2");
+  ASSERT_NE(foo_2, nullptr);
+  ASSERT_EQ(foo_2->input_size(), 2);
+  EXPECT_EQ(foo_2->input(0), "a");
+  EXPECT_EQ(foo_2->input(1), "^b");
+  NodeDef* foo_3 = graph.GetNode("foo_3");
+  ASSERT_NE(foo_3, nullptr);
+  ASSERT_EQ(foo_3->input_size(), 2);
+  EXPECT_EQ(foo_3->input(0), "a");
+  EXPECT_EQ(foo_3->input(1), "b:1");
+  NodeDef* foo_4 = graph.GetNode("foo_4");
+  ASSERT_NE(foo_4, nullptr);
+  ASSERT_EQ(foo_4->input_size(), 2);
+  EXPECT_EQ(foo_4->input(0), "a:2");
+  EXPECT_EQ(foo_4->input(1), "b:1");
+  NodeDef* foo_5 = graph.GetNode("foo_5");
+  ASSERT_NE(foo_5, nullptr);
+  ASSERT_EQ(foo_5->input_size(), 2);
+  EXPECT_EQ(foo_5->input(0), "a:2");
+  EXPECT_EQ(foo_5->input(1), "b:1");
+  NodeDef* foo_6 = graph.GetNode("foo_6");
+  ASSERT_NE(foo_6, nullptr);
+  ASSERT_EQ(foo_6->input_size(), 2);
+  EXPECT_EQ(foo_6->input(0), "d");
+  EXPECT_EQ(foo_6->input(1), "^d");
+  NodeDef* foo_7 = graph.GetNode("foo_7");
+  ASSERT_NE(foo_7, nullptr);
+  ASSERT_EQ(foo_7->input_size(), 4);
+  EXPECT_EQ(foo_7->input(0), "a:3");
+  EXPECT_EQ(foo_7->input(1), "b:2");
+  EXPECT_EQ(foo_7->input(2), "d");
+  EXPECT_EQ(foo_7->input(3), "^d");
+}
+
+TEST(MutableGraphViewTest, DedupControllingFaninsOnAddFanin) {
+  // Actual node.op() is not important in this test.
+  GraphDef graph_def = test::function::GDef(
+      {NDef("a", "NotImportant", {}, {}), NDef("b", "NotImportant", {"^a"}),
+       NDef("c", "NotImportant", {"a:1"})},
+      /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  EXPECT_TRUE(graph.AddFanin("b", {"a", 2}));
+  NodeDef* b = graph.GetNode("b");
+  ASSERT_NE(b, nullptr);
+  ASSERT_EQ(b->input_size(), 1);
+  EXPECT_EQ(b->input(0), "a:2");
+
+  EXPECT_FALSE(graph.AddFanin("c", {"a", Graph::kControlSlot}));
+  NodeDef* c = graph.GetNode("c");
+  ASSERT_NE(c, nullptr);
+  ASSERT_EQ(c->input_size(), 1);
+  EXPECT_EQ(c->input(0), "a:1");
+}
+
+TEST(MutableGraphViewTest, NoDedupControlFlowControllingFaninsOnAddFanin) {
+  GraphDef graph_def = test::function::GDef(
+      {NDef("a", "Switch", {}, {}), NDef("b", "Identity", {"a:1"}),
+       NDef("c", "", {}, {}), NDef("d", "", {}, {})},
+      /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  EXPECT_TRUE(graph.AddFanin("c", {"b", 2}));
+  NodeDef* c = graph.GetNode("c");
+  ASSERT_NE(c, nullptr);
+  ASSERT_EQ(c->input_size(), 1);
+  EXPECT_EQ(c->input(0), "b:2");
+  EXPECT_TRUE(graph.AddFanin("c", {"b", Graph::kControlSlot}));
+  ASSERT_EQ(c->input_size(), 2);
+  EXPECT_EQ(c->input(0), "b:2");
+  EXPECT_EQ(c->input(1), "^b");
+  EXPECT_FALSE(graph.AddFanin("c", {"b", Graph::kControlSlot}));
+  ASSERT_EQ(c->input_size(), 2);
+  EXPECT_EQ(c->input(0), "b:2");
+  EXPECT_EQ(c->input(1), "^b");
+
+  EXPECT_TRUE(graph.AddFanin("d", {"b", Graph::kControlSlot}));
+  NodeDef* d = graph.GetNode("d");
+  ASSERT_NE(d, nullptr);
+  ASSERT_EQ(d->input_size(), 1);
+  EXPECT_EQ(d->input(0), "^b");
+  EXPECT_FALSE(graph.AddFanin("d", {"b", Graph::kControlSlot}));
+  ASSERT_EQ(d->input_size(), 1);
+  EXPECT_EQ(d->input(0), "^b");
+  EXPECT_TRUE(graph.AddFanin("d", {"b", 3}));
+  ASSERT_EQ(d->input_size(), 2);
+  EXPECT_EQ(d->input(0), "b:3");
+  EXPECT_EQ(d->input(1), "^b");
+}
+
+TEST(MutableGraphViewTest, DedupControllingFaninsOnUpdateFanin) {
   // Actual node.op() is not important in this test.
   GraphDef graph_def = test::function::GDef(
       {NDef("a", "NotImportant", {}, {}), NDef("b", "NotImportant", {}, {}),
-       NDef("foo_1", "NotImportant", {"a", "b:1", "^b"}),
-       NDef("foo_2", "NotImportant", {"a", "^b", "^b"}),
-       NDef("foo_3", "NotImportant", {"a", "b:1", "^b", "^b"}),
-       NDef("foo_4", "NotImportant", {"a:2", "b:1", "^b", "^b", "^a", "^a"})},
+       NDef("c", "NotImportant", {"a:1", "^b"})},
       /*funcs=*/{});
-  return graph_def;
+
+  MutableGraphView graph(&graph_def);
+
+  EXPECT_TRUE(graph.UpdateFanin("c", {"a", 1}, {"b", 2}));
+  NodeDef* c = graph.GetNode("c");
+  ASSERT_NE(c, nullptr);
+  ASSERT_EQ(c->input_size(), 1);
+  EXPECT_EQ(c->input(0), "b:2");
 }
 
-void CheckDedupControllingFaninsForNode(MutableGraphView* graph,
-                                        absl::string_view node_name,
-                                        const NodeDef* expected_node) {
-  // Deduping again should result in no change.
-  EXPECT_FALSE(graph->DedupControllingFanins(node_name));
-  NodeDef* node = graph->GetNode(node_name);
-  ASSERT_NE(node, nullptr);
-  ASSERT_EQ(node->input_size(), expected_node->input_size());
-  CompareNodeInputs(*graph, expected_node, node);
-  for (int i = 0; i < node->input_size(); ++i) {
-    TensorId tensor_id = ParseTensorName(node->input(i));
-    if (tensor_id.index() > Graph::kControlSlot) {
-      CheckFanout(*graph, {tensor_id.node(), Graph::kControlSlot}, node_name);
-    }
-  }
+TEST(MutableGraphViewTest, NoDedupControlFlowControllingFaninsOnUpdateFanin) {
+  GraphDef graph_def = test::function::GDef(
+      {NDef("a", "Switch", {}, {}), NDef("b", "Identity", {"a:1"}),
+       NDef("c", "Identity", {"a:2"}), NDef("d", "NotImportant", {"c", "^b"}),
+       NDef("e", "NotImportant", {"b", "^c"})},
+      /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  EXPECT_TRUE(graph.UpdateFanin("d", {"b", Graph::kControlSlot},
+                                {"c", Graph::kControlSlot}));
+  NodeDef* d = graph.GetNode("d");
+  ASSERT_NE(d, nullptr);
+  ASSERT_EQ(d->input_size(), 2);
+  EXPECT_EQ(d->input(0), "c");
+  EXPECT_EQ(d->input(1), "^c");
+
+  EXPECT_TRUE(graph.UpdateFanin("e", {"b", 0}, {"c", 3}));
+  NodeDef* e = graph.GetNode("e");
+  ASSERT_NE(e, nullptr);
+  ASSERT_EQ(e->input_size(), 2);
+  EXPECT_EQ(e->input(0), "c:3");
+  EXPECT_EQ(e->input(1), "^c");
+
+  EXPECT_TRUE(graph.UpdateFanin("e", {"c", 3}, {"c", Graph::kControlSlot}));
+  ASSERT_NE(e, nullptr);
+  ASSERT_EQ(e->input_size(), 1);
+  EXPECT_EQ(e->input(0), "^c");
+}
+
+TEST(MutableGraphViewTest, UpdateMaxRegularOutputPortOnAddFanin) {
+  // Actual node.op() is not important in this test.
+  GraphDef graph_def = test::function::GDef(
+      {NDef("a", "NotImportant", {}, {}), NDef("b", "NotImportant", {"a:1"}),
+       NDef("c", "NotImportant", {"^b"})},
+      /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  EXPECT_TRUE(graph.AddFanin("c", {"a", 3}));
+  NodeDef* a = graph.GetNode("a");
+  ASSERT_NE(a, nullptr);
+  auto fanouts = graph.GetFanouts(*a, /*include_controlled_nodes=*/true);
+  EXPECT_EQ(fanouts.size(), 2);
+  NodeDef* b = graph.GetNode("b");
+  ASSERT_NE(b, nullptr);
+  EXPECT_EQ(fanouts.count(MutableGraphView::InputPort(b, 0)), 1);
+  NodeDef* c = graph.GetNode("c");
+  ASSERT_NE(c, nullptr);
+  EXPECT_EQ(fanouts.count(MutableGraphView::InputPort(c, 0)), 1);
 }
 
-void TestDedupControllingFaninsForNode(MutableGraphView* graph,
-                                       absl::string_view node_name,
-                                       const NodeDef* expected_node) {
-  EXPECT_TRUE(graph->DedupControllingFanins(node_name));
-  CheckDedupControllingFaninsForNode(graph, node_name, expected_node);
+TEST(MutableGraphViewTest, UpdateMaxRegularOutputPortOnRemoveFanin) {
+  // Actual node.op() is not important in this test.
+  GraphDef graph_def = test::function::GDef(
+      {NDef("a", "NotImportant", {}, {}), NDef("b", "NotImportant", {"a:1"}),
+       NDef("c", "NotImportant", {"a:2"})},
+      /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  EXPECT_TRUE(graph.RemoveFanin("c", {"a", 2}));
+  NodeDef* a = graph.GetNode("a");
+  ASSERT_NE(a, nullptr);
+  auto fanouts = graph.GetFanouts(*a, /*include_controlled_nodes=*/true);
+  EXPECT_EQ(fanouts.size(), 1);
+  NodeDef* b = graph.GetNode("b");
+  ASSERT_NE(b, nullptr);
+  EXPECT_EQ(fanouts.count(MutableGraphView::InputPort(b, 0)), 1);
 }
 
-TEST(MutableGraphViewTest, DedupControllingFaninsForNode) {
-  GraphDef graph_def = SimpleDuplicateControllingFaninsGraph();
+TEST(MutableGraphViewTest, KeepMaxRegularOutputPortOnRemoveFanin) {
+  // Actual node.op() is not important in this test.
+  GraphDef graph_def = test::function::GDef(
+      {NDef("a", "NotImportant", {}, {}), NDef("b", "NotImportant", {"a:1"}),
+       NDef("c", "NotImportant", {"a:2"})},
+      /*funcs=*/{});
 
   MutableGraphView graph(&graph_def);
 
-  NodeDef expected_node;
-  // Remove redundant control dependency '^b'.
-  expected_node = NDef("", "", {"a", "b:1"});
-  TestDedupControllingFaninsForNode(&graph, "foo_1", &expected_node);
-  // Remove extra control dependency '^b'.
-  expected_node = NDef("", "", {"a", "^b"});
-  TestDedupControllingFaninsForNode(&graph, "foo_2", &expected_node);
-  // Remove redundant and extra control dependencies '^b'.
-  expected_node = NDef("", "", {"a", "b:1"});
-  TestDedupControllingFaninsForNode(&graph, "foo_3", &expected_node);
-  // Remove multiple redundant control dependencies.
-  expected_node = NDef("", "", {"a:2", "b:1"});
-  TestDedupControllingFaninsForNode(&graph, "foo_4", &expected_node);
-  // Missing node.
-  EXPECT_FALSE(graph.DedupControllingFanins("missing"));
+  EXPECT_TRUE(graph.RemoveFanin("b", {"a", 1}));
+  NodeDef* a = graph.GetNode("a");
+  ASSERT_NE(a, nullptr);
+  auto fanouts = graph.GetFanouts(*a, /*include_controlled_nodes=*/true);
+  EXPECT_EQ(fanouts.size(), 1);
+  NodeDef* c = graph.GetNode("c");
+  ASSERT_NE(c, nullptr);
+  EXPECT_EQ(fanouts.count(MutableGraphView::InputPort(c, 0)), 1);
 }
 
-TEST(MutableGraphViewTest, DedupControllingFaninsForGraph) {
-  GraphDef graph_def = SimpleDuplicateControllingFaninsGraph();
+TEST(MutableGraphViewTest, UpdateMaxRegularOutputPortOnUpdateFanin) {
+  // Actual node.op() is not important in this test.
+  GraphDef graph_def = test::function::GDef(
+      {NDef("a", "NotImportant", {}, {}), NDef("b", "NotImportant", {"a:1"}),
+       NDef("c", "NotImportant", {"a:2"})},
+      /*funcs=*/{});
 
   MutableGraphView graph(&graph_def);
-  EXPECT_TRUE(graph.DedupControllingFanins());
-  // Deduping again should result in no change.
-  EXPECT_FALSE(graph.DedupControllingFanins());
 
-  NodeDef expected_node;
-  // Remove redundant control dependency '^b'.
-  expected_node = NDef("", "", {"a", "b:1"});
-  CheckDedupControllingFaninsForNode(&graph, "foo_1", &expected_node);
-  // Remove extra control dependency '^b'.
-  expected_node = NDef("", "", {"a", "^b"});
-  CheckDedupControllingFaninsForNode(&graph, "foo_2", &expected_node);
-  // Remove redundant and extra control dependencies '^b'.
-  expected_node = NDef("", "", {"a", "b:1"});
-  CheckDedupControllingFaninsForNode(&graph, "foo_3", &expected_node);
-  // Remove multiple redundant control dependencies.
-  expected_node = NDef("", "", {"a:2", "b:1"});
-  CheckDedupControllingFaninsForNode(&graph, "foo_4", &expected_node);
+  EXPECT_TRUE(graph.UpdateFanin("c", {"a", 2}, {"b", 3}));
+  NodeDef* a = graph.GetNode("a");
+  ASSERT_NE(a, nullptr);
+  auto a_fanouts = graph.GetFanouts(*a, /*include_controlled_nodes=*/true);
+  EXPECT_EQ(a_fanouts.size(), 1);
+  NodeDef* b = graph.GetNode("b");
+  ASSERT_NE(b, nullptr);
+  EXPECT_EQ(a_fanouts.count(MutableGraphView::InputPort(b, 0)), 1);
+  auto b_fanouts = graph.GetFanouts(*b, /*include_controlled_nodes=*/true);
+  EXPECT_EQ(b_fanouts.size(), 1);
+  NodeDef* c = graph.GetNode("c");
+  ASSERT_NE(c, nullptr);
+  EXPECT_EQ(b_fanouts.count(MutableGraphView::InputPort(c, 0)), 1);
 }
 
 TEST(MutableGraphViewTest, AddControllingFaninMissing) {
@@ -655,7 +891,7 @@ TEST(MutableGraphViewTest, AddControllingFaninSwitchWithNoExistingIdentity) {
 TEST(MutableGraphViewTest, AddControllingFaninSwitchWithExistingAddedIdentity) {
   GraphDef graph_def = test::function::GDef(
       {NDef("a", "NotImportant", {}, {}), NDef("switch", "Switch", {}, {}),
-       NDef("ConstantFoldingCtrl/switch_0", "Identity", {}, {})},
+       NDef("ConstantFoldingCtrl/switch_0", "Identity", {"switch"})},
       /*funcs=*/{});
 
   MutableGraphView graph(&graph_def);
@@ -694,9 +930,8 @@ TEST(MutableGraphViewTest, DeleteNodes) {
   auto bar_fanouts = graph.GetFanouts(*bar, include_control_fanouts);
   auto other_fanouts = graph.GetFanouts(*other, include_control_fanouts);
 
-  EXPECT_EQ(bar_fanouts.size(), 2);
+  EXPECT_EQ(bar_fanouts.size(), 1);
   EXPECT_EQ(bar_fanouts.count(MutableGraphView::InputPort(foo_2, 1)), 1);
-  EXPECT_EQ(bar_fanouts.count(MutableGraphView::InputPort(foo_2, -1)), 1);
 
   EXPECT_EQ(other_fanouts.size(), 1);
   EXPECT_EQ(other_fanouts.count(MutableGraphView::InputPort(foo_2, 0)), 1);
diff --git a/tensorflow/core/grappler/optimizers/data/graph_test_utils.cc b/tensorflow/core/grappler/optimizers/data/graph_test_utils.cc
index 9d8b388a3a..202dfb5ac8 100644
--- a/tensorflow/core/grappler/optimizers/data/graph_test_utils.cc
+++ b/tensorflow/core/grappler/optimizers/data/graph_test_utils.cc
@@ -42,7 +42,7 @@ NodeDef MakeMapAndBatchNode(StringPiece name, StringPiece input_node_name,
                             StringPiece function_name) {
   return test::function::NDef(
       name, "ExperimentalMapAndBatchDataset",
-      {string(input_node_name), "", string(batch_size_node_name),
+      {string(input_node_name), string(batch_size_node_name),
        string(num_parallel_calls_node_name), string(drop_remainder_node_name)},
       {{"f", FunctionDefHelper::FunctionRef(string(function_name))},
        {"Targuments", {}},
@@ -68,7 +68,7 @@ NodeDef MakeParallelInterleaveNode(StringPiece name,
                                    StringPiece function_name, bool sloppy) {
   return test::function::NDef(
       name, "ParallelInterleaveDatasetV2",
-      {string(input_node_name), "", string(cycle_length_node_name),
+      {string(input_node_name), string(cycle_length_node_name),
        string(block_length_node_name), string(num_parallel_calls_node_name)},
       {
           {"f", FunctionDefHelper::FunctionRef(string(function_name))},
diff --git a/tensorflow/core/grappler/optimizers/data/latency_all_edges_test.cc b/tensorflow/core/grappler/optimizers/data/latency_all_edges_test.cc
index d428d04a66..426c1dca5b 100644
--- a/tensorflow/core/grappler/optimizers/data/latency_all_edges_test.cc
+++ b/tensorflow/core/grappler/optimizers/data/latency_all_edges_test.cc
@@ -30,9 +30,9 @@ TEST(LatencyAllEdgesTest, AddLatenciesAfterTensorMapPrefetch) {
   using test::function::NDef;
   GrapplerItem item;
   NodeDef component_node =
-      NDef("component_nodes", "Const", {}, {{"value", 1}, {"dtype", DT_INT32}});
+      NDef("component_node", "Const", {}, {{"value", 1}, {"dtype", DT_INT32}});
   NodeDef from_tensor_node =
-      NDef("from_tensor_nodes", "TensorDataset", {"component_nodes"},
+      NDef("from_tensor_node", "TensorDataset", {"component_node"},
            {{"Toutput_types", {}}, {"output_shapes", {}}});
 
   NodeDef captured_input_node = NDef("captured_input_node", "Const", {},
-- 
GitLab


From db48d17b671623a24dcee7e90f8b18d87f51d009 Mon Sep 17 00:00:00 2001
From: Michael Kuperstein <mkuper@google.com>
Date: Thu, 3 Jan 2019 18:17:13 -0800
Subject: [PATCH 0210/1095] [XLA] Make the verifier check bitcasts don't change
 the element type

This is what BitcastConvert is for.

PiperOrigin-RevId: 227783600
---
 .../compiler/xla/service/hlo_verifier.cc      |  8 ++++++
 .../compiler/xla/service/hlo_verifier_test.cc | 18 +++++++++++++
 .../while_loop_invariant_code_motion_test.cc  | 27 ++++++++++---------
 3 files changed, 41 insertions(+), 12 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_verifier.cc b/tensorflow/compiler/xla/service/hlo_verifier.cc
index 14e29533c2..9274d42280 100644
--- a/tensorflow/compiler/xla/service/hlo_verifier.cc
+++ b/tensorflow/compiler/xla/service/hlo_verifier.cc
@@ -387,6 +387,14 @@ Status ShapeVerifier::HandleReduce(HloInstruction* reduce) {
 
 Status ShapeVerifier::HandleBitcast(HloInstruction* bitcast) {
   TF_RETURN_IF_ERROR(CheckOperandCount(bitcast, 1));
+  // Bitcasts are not allowed to change the element type.
+  if (bitcast->operand(0)->shape().element_type() !=
+      bitcast->shape().element_type()) {
+    return InternalError(
+        "Bitcast can not change the element type from %s to %s",
+        PrimitiveType_Name(bitcast->operand(0)->shape().element_type()),
+        PrimitiveType_Name(bitcast->shape().element_type()));
+  }
   return Status::OK();
 }
 
diff --git a/tensorflow/compiler/xla/service/hlo_verifier_test.cc b/tensorflow/compiler/xla/service/hlo_verifier_test.cc
index 91f247a9bb..9b2d884f79 100644
--- a/tensorflow/compiler/xla/service/hlo_verifier_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_verifier_test.cc
@@ -480,5 +480,23 @@ TEST_F(HloVerifierTestLayoutSensitive, ConcatWithLayoutChangeNotAllowed) {
   EXPECT_THAT(status.error_message(),
               HasSubstr("Instruction shouldn't change layouts"));
 }
+
+TEST_F(HloVerifierTest, BitcastCanNotChangeElementType) {
+  const char* const hlo_string = R"(
+  HloModule Module
+
+  ENTRY BitcastCanNotChangeElementType {
+   constant.0 = f32[2] constant({0.0, 0.0})
+   ROOT bitcast = s32[2] bitcast(constant.0)
+  }
+  )";
+  TF_ASSERT_OK_AND_ASSIGN(auto module, ParseHloString(hlo_string));
+
+  auto status = verifier().Run(module.get()).status();
+  ASSERT_FALSE(status.ok());
+  EXPECT_THAT(status.error_message(),
+              HasSubstr("Bitcast can not change the element type"));
+}
+
 }  // namespace
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/while_loop_invariant_code_motion_test.cc b/tensorflow/compiler/xla/service/while_loop_invariant_code_motion_test.cc
index 8e7c4bc882..3587c016b4 100644
--- a/tensorflow/compiler/xla/service/while_loop_invariant_code_motion_test.cc
+++ b/tensorflow/compiler/xla/service/while_loop_invariant_code_motion_test.cc
@@ -299,7 +299,7 @@ TEST_F(WhileLoopInvariantCodeMotionTest, DontHoistBitcastAlone) {
   // bitcast either.
   auto m = CreateNewVerifiedModule();
   auto scalar_s32 = ShapeUtil::MakeShape(S32, {});
-  auto scalar_f32 = ShapeUtil::MakeShape(F32, {});
+  auto effective_scalar_s32 = ShapeUtil::MakeShape(S32, {1});
   auto token_shape = ShapeUtil::MakeTokenShape();
   Shape while_shape =
       ShapeUtil::MakeTupleShape({scalar_s32, scalar_s32, token_shape});
@@ -314,10 +314,12 @@ TEST_F(WhileLoopInvariantCodeMotionTest, DontHoistBitcastAlone) {
         HloInstruction::CreateGetTupleElement(scalar_s32, param, 1));
     HloInstruction* in_token = builder.AddInstruction(
         HloInstruction::CreateGetTupleElement(token_shape, param, 2));
-    HloInstruction* bitcast_inst = builder.AddInstruction(
-        HloInstruction::CreateUnary(scalar_f32, HloOpcode::kBitcast, gte_0));
-    HloInstruction* out_token = builder.AddInstruction(
-        HloInstruction::CreateOutfeed(scalar_f32, bitcast_inst, in_token, ""));
+    HloInstruction* bitcast_inst =
+        builder.AddInstruction(HloInstruction::CreateUnary(
+            effective_scalar_s32, HloOpcode::kBitcast, gte_0));
+    HloInstruction* out_token =
+        builder.AddInstruction(HloInstruction::CreateOutfeed(
+            effective_scalar_s32, bitcast_inst, in_token, ""));
     builder.AddInstruction(
         HloInstruction::CreateTuple({gte_0, gte_1, out_token}));
 
@@ -352,9 +354,9 @@ TEST_F(WhileLoopInvariantCodeMotionTest, HoistBitcastIfNeeded) {
   // The bitcast's user can be hoisted, so hoist the bitcast too.
   auto m = CreateNewVerifiedModule();
   auto scalar_s32 = ShapeUtil::MakeShape(S32, {});
-  auto scalar_f32 = ShapeUtil::MakeShape(F32, {});
-  Shape while_shape =
-      ShapeUtil::MakeTupleShape({scalar_s32, scalar_f32, scalar_f32});
+  auto effective_scalar_s32 = ShapeUtil::MakeShape(S32, {1});
+  Shape while_shape = ShapeUtil::MakeTupleShape(
+      {scalar_s32, effective_scalar_s32, effective_scalar_s32});
 
   HloComputation* while_body = [&]() {
     HloComputation::Builder builder(TestName() + ".while_body");
@@ -363,12 +365,13 @@ TEST_F(WhileLoopInvariantCodeMotionTest, HoistBitcastIfNeeded) {
     HloInstruction* gte_0 = builder.AddInstruction(
         HloInstruction::CreateGetTupleElement(scalar_s32, param, 0));
     HloInstruction* gte_1 = builder.AddInstruction(
-        HloInstruction::CreateGetTupleElement(scalar_f32, param, 1));
-    HloInstruction* bitcast_inst = builder.AddInstruction(
-        HloInstruction::CreateUnary(scalar_f32, HloOpcode::kBitcast, gte_0));
+        HloInstruction::CreateGetTupleElement(effective_scalar_s32, param, 1));
+    HloInstruction* bitcast_inst =
+        builder.AddInstruction(HloInstruction::CreateUnary(
+            effective_scalar_s32, HloOpcode::kBitcast, gte_0));
     HloInstruction* add_inst =
         builder.AddInstruction(HloInstruction::CreateBinary(
-            scalar_f32, HloOpcode::kAdd, bitcast_inst, gte_1));
+            effective_scalar_s32, HloOpcode::kAdd, bitcast_inst, gte_1));
     builder.AddInstruction(
         HloInstruction::CreateTuple({gte_0, gte_1, add_inst}));
 
-- 
GitLab


From 328ca94210ba39e13b51228200c8d2da27246f7f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 3 Jan 2019 18:19:05 -0800
Subject: [PATCH 0211/1095] Refactor ExecuteFlexOp into smaller methods of
 OpNode.

PiperOrigin-RevId: 227783768
---
 tensorflow/lite/delegates/flex/kernel.cc | 145 +++++++++++++----------
 1 file changed, 85 insertions(+), 60 deletions(-)

diff --git a/tensorflow/lite/delegates/flex/kernel.cc b/tensorflow/lite/delegates/flex/kernel.cc
index 7d18c39098..3012241f4f 100644
--- a/tensorflow/lite/delegates/flex/kernel.cc
+++ b/tensorflow/lite/delegates/flex/kernel.cc
@@ -88,10 +88,18 @@ class OpNode {
   const tensorflow::NodeDef& nodedef() const { return nodedef_; }
 
   const std::vector<int>& inputs() const { return inputs_; }
-  std::vector<int>* mutable_inputs() { return &inputs_; }
+  void InitializeInputs(const TfLiteIntArray* inputs) {
+    for (int index : TfLiteIntArrayView(inputs)) {
+      inputs_.push_back(index);
+    }
+  }
 
   const std::vector<int>& outputs() const { return outputs_; }
-  std::vector<int>* mutable_outputs() { return &outputs_; }
+  void InitializeOutputs(const TfLiteIntArray* outputs) {
+    for (int index : TfLiteIntArrayView(outputs)) {
+      outputs_.push_back(index);
+    }
+  }
 
   tensorflow::Status InitializeNodeDef(const void* custom_initial_data,
                                        int custom_initial_data_size) {
@@ -123,6 +131,66 @@ class OpNode {
     return tensorflow::Status::OK();
   }
 
+  // Build thew new EagerOperation. In case of error, the returned 'op' is
+  // guaranteed to be 'nullptr'.
+  tensorflow::Status BuildEagerOp(
+      tensorflow::EagerContext* eager_context,
+      std::unique_ptr<tensorflow::EagerOperation>* op) {
+    op->reset();
+
+    const tensorflow::AttrTypeMap* attr_types;
+    bool is_function = false;
+    TF_RETURN_WITH_CONTEXT_IF_ERROR(
+        tensorflow::AttrTypeMapForOp(name_.c_str(), &attr_types, &is_function),
+        " (while processing attributes of '", name_, "')");
+    if (is_function) {
+      return tensorflow::errors::NotFound(
+          "Operation '", name_,
+          "' is not registered.  (while processing attributes of '", name_,
+          "')");
+    }
+
+    op->reset(new tensorflow::EagerOperation(eager_context, name_.c_str(),
+                                             /*is_function=*/false,
+                                             attr_types));
+    for (const auto& attr : nodedef_.attr()) {
+      (*op)->MutableAttrs()->Set(attr.first, attr.second);
+    }
+
+    return tensorflow::Status::OK();
+  }
+
+  tensorflow::Status BuildEagerInputs(BufferMap* buffer_map,
+                                      tensorflow::EagerOperation* op) {
+    for (int input_index : inputs_) {
+      if (!buffer_map->HasTensor(input_index)) {
+        return tensorflow::errors::Internal(
+            "Cannot read from invalid tensor index ", input_index);
+      }
+      auto* handle = new tensorflow::TensorHandle(
+          buffer_map->GetTensor(input_index), nullptr, nullptr, nullptr);
+      op->AddInput(handle);
+      handle->Unref();
+
+      if (buffer_map->IsForwardable(input_index)) {
+        // Take it out of the map, so Eager/TF can reuse the buffer for an
+        // output tensor of the op.
+        buffer_map->RemoveTensor(input_index);
+      }
+    }
+    return tensorflow::Status::OK();
+  }
+
+  tensorflow::Status PersistEagerOutputs(BufferMap* buffer_map,
+                                         OpOutputs* retvals) {
+    for (int i = 0; i < outputs_.size(); ++i) {
+      const tensorflow::Tensor* tensor = nullptr;
+      TF_RETURN_IF_ERROR(retvals->GetHandle(i)->Tensor(&tensor));
+      buffer_map->SetFromTensorFlow(outputs_[i], *tensor);
+    }
+    return tensorflow::Status::OK();
+  }
+
  private:
   // The name of the TensorFlow op to execute.
   string name_;
@@ -139,60 +207,23 @@ class OpNode {
 // Executes the TensorFlow op given by 'op_name', with the attributes specified
 // in 'nodedef'. Inputs and outputs are given as indices into the 'buffer_map'.
 tensorflow::Status ExecuteFlexOp(tensorflow::EagerContext* eager_context,
-                                 BufferMap* buffer_map, const string& op_name,
-                                 const tensorflow::NodeDef& nodedef,
-                                 const std::vector<int>& inputs,
-                                 const std::vector<int>& outputs) {
-  const tensorflow::AttrTypeMap* attr_types;
-  bool is_function = false;
-  TF_RETURN_WITH_CONTEXT_IF_ERROR(
-      tensorflow::AttrTypeMapForOp(op_name.c_str(), &attr_types, &is_function),
-      " (while processing attributes of '", op_name, "')");
-  if (is_function) {
-    return tensorflow::errors::NotFound(
-        "Operation '", op_name,
-        "' is not registered.  (while processing attributes of '", op_name,
-        "')");
-  }
-  tensorflow::EagerOperation op(eager_context, op_name.c_str(),
-                                /*is_function=*/false, attr_types);
-  for (const auto& attr : nodedef.attr()) {
-    op.MutableAttrs()->Set(attr.first, attr.second);
-  }
+                                 BufferMap* buffer_map, OpNode* node_data) {
+  std::unique_ptr<tensorflow::EagerOperation> op;
+  TF_RETURN_IF_ERROR(node_data->BuildEagerOp(eager_context, &op));
+  TF_RETURN_IF_ERROR(node_data->BuildEagerInputs(buffer_map, op.get()));
 
-  for (int input_index : inputs) {
-    if (!buffer_map->HasTensor(input_index)) {
-      return tensorflow::errors::Internal(
-          "Cannot read from invalid tensor index ", input_index);
-    }
-    auto* handle = new tensorflow::TensorHandle(
-        buffer_map->GetTensor(input_index), nullptr, nullptr, nullptr);
-    op.AddInput(handle);
-    handle->Unref();
-
-    if (buffer_map->IsForwardable(input_index)) {
-      // Take it out of the map, so Eager/TF can reuse the buffer for an output
-      // tensor of the op.
-      buffer_map->RemoveTensor(input_index);
-    }
-  }
-
-  int num_retvals = outputs.size();
+  int num_retvals = node_data->outputs().size();
   OpOutputs retvals(num_retvals);
   TF_RETURN_WITH_CONTEXT_IF_ERROR(
-      EagerExecute(&op, retvals.GetVector(), &num_retvals),
-      " (while executing '", op_name, "' via Eager)");
+      EagerExecute(op.get(), retvals.GetVector(), &num_retvals),
+      " (while executing '", node_data->name(), "' via Eager)");
 
-  if (num_retvals != outputs.size()) {
+  if (num_retvals != node_data->outputs().size()) {
     return tensorflow::errors::Internal(
         "Unexpected number of outputs from EagerExecute");
   }
 
-  for (int i = 0; i < num_retvals; ++i) {
-    const tensorflow::Tensor* tensor = nullptr;
-    TF_RETURN_IF_ERROR(retvals.GetHandle(i)->Tensor(&tensor));
-    buffer_map->SetFromTensorFlow(outputs[i], *tensor);
-  }
+  TF_RETURN_IF_ERROR(node_data->PersistEagerOutputs(buffer_map, &retvals));
 
   return tensorflow::Status::OK();
 }
@@ -247,12 +278,8 @@ void* Init(TfLiteContext* context, const char* buffer, size_t length) {
                                          node->custom_initial_data_size);
     if (!status.ok()) break;
 
-    for (auto input_index : TfLiteIntArrayView(node->inputs)) {
-      node_data.mutable_inputs()->push_back(input_index);
-    }
-    for (auto output_index : TfLiteIntArrayView(node->outputs)) {
-      node_data.mutable_outputs()->push_back(output_index);
-    }
+    node_data.InitializeInputs(node->inputs);
+    node_data.InitializeOutputs(node->outputs);
   }
 
   if (ConvertStatus(context, status) != kTfLiteOk) {
@@ -325,7 +352,7 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
 }
 
 TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
-  const auto* op_data = reinterpret_cast<OpData*>(node->user_data);
+  auto* op_data = reinterpret_cast<OpData*>(node->user_data);
   BufferMap* buffer_map = op_data->buffer_map;
   tensorflow::EagerContext* eager_context = op_data->eager_context;
 
@@ -344,14 +371,12 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
   }
 
   // Execute the TensorFlow Ops sequentially.
-  for (const auto& node_data : op_data->nodes) {
+  for (OpNode& node_data : op_data->nodes) {
     SCOPED_TAGGED_OPERATOR_PROFILE(
         reinterpret_cast<profiling::Profiler*>(context->profiler),
-        node_data->name().c_str(), node_data->index());
+        node_data.name().c_str(), node_data.index());
 
-    auto status = ExecuteFlexOp(eager_context, buffer_map, node_data.name(),
-                                node_data.nodedef(), node_data.inputs(),
-                                node_data.outputs());
+    auto status = ExecuteFlexOp(eager_context, buffer_map, &node_data);
     TF_LITE_ENSURE_OK(context, ConvertStatus(context, status));
   }
 
-- 
GitLab


From d0ce85e772fa17afcc7914c63bcab1b90df6664e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 3 Jan 2019 18:26:08 -0800
Subject: [PATCH 0212/1095] Internal change.

PiperOrigin-RevId: 227784390
---
 tensorflow/compiler/xla/service/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index eed958dc6b..b84792cfc3 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -3576,6 +3576,7 @@ cc_library(
 tf_cc_test(
     name = "indexed_array_analysis_test",
     srcs = ["indexed_array_analysis_test.cc"],
+    extra_copts = ["-Wno-string-plus-int"],
     deps = [
         ":hlo_matchers",
         ":indexed_array_analysis",
-- 
GitLab


From a2f7f39d982682fa8de050e001522581570c510f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 3 Jan 2019 19:50:14 -0800
Subject: [PATCH 0213/1095] Add support non-stacking(cross-links) but connected
 to other bidi-lstm ops case, tensorflow equivalent:
 tf.nn.static_bidirectional_rnn

PiperOrigin-RevId: 227791656
---
 .../kernels/bidirectional_sequence_lstm.cc    |  54 +++--
 .../bidirectional_sequence_lstm_test.cc       | 202 +++++++++++++++++-
 2 files changed, 233 insertions(+), 23 deletions(-)

diff --git a/tensorflow/lite/kernels/bidirectional_sequence_lstm.cc b/tensorflow/lite/kernels/bidirectional_sequence_lstm.cc
index 1ddfe7201e..31c6e3f44c 100644
--- a/tensorflow/lite/kernels/bidirectional_sequence_lstm.cc
+++ b/tensorflow/lite/kernels/bidirectional_sequence_lstm.cc
@@ -105,7 +105,10 @@ constexpr int kBwInputActivationStateTensor = 37;
 // Cell state tensors of size {n_batch, n_cell}
 constexpr int kBwInputCellStateTensor = 38;
 
-// Auxiliary input and weights when stacking.
+// Used as auxiliary input and weights when stacking for
+// tf.contrib.rnn.stack_bidirectional_rnn case (with cross links); Used as input
+// to the backward cell when stacking for tf.nn.static_bidirectional_rnn case
+// (without cross links).
 constexpr int kAuxInputTensor = 39;  // Optional
 // Forward weights.
 constexpr int kFwAuxInputToInputWeightsTensor = 40;   // Optional
@@ -459,8 +462,8 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
   const TfLiteTensor* bw_aux_input_to_output_weights =
       GetOptionalInputTensor(context, node, kBwAuxInputToOutputWeightsTensor);
 
-  const bool aux_inputs_all_or_none =
-      ((aux_input != nullptr) && (fw_aux_input_to_cell_weights != nullptr) &&
+  const bool aux_inputs_weights_all_or_none =
+      ((fw_aux_input_to_cell_weights != nullptr) &&
        (fw_aux_input_to_forget_weights != nullptr) &&
        (fw_aux_input_to_output_weights != nullptr) &&
        (bw_aux_input_to_cell_weights != nullptr) &&
@@ -472,8 +475,9 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
        (bw_aux_input_to_cell_weights == nullptr) &&
        (bw_aux_input_to_forget_weights == nullptr) &&
        (bw_aux_input_to_output_weights == nullptr));
-  TF_LITE_ENSURE(context, aux_inputs_all_or_none);
-  const bool has_aux_input = (aux_input != nullptr);
+  TF_LITE_ENSURE(context, aux_inputs_weights_all_or_none);
+
+  const bool has_aux_input = (fw_aux_input_to_forget_weights != nullptr);
 
   if (has_aux_input) {
     // Check that aux_input has the same dimensions (except last) as the input.
@@ -870,6 +874,9 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
   const TfLiteTensor* bw_aux_input_to_output_weights =
       GetOptionalInputTensor(context, node, kBwAuxInputToOutputWeightsTensor);
 
+  const bool has_previous_bw_output = (aux_input != nullptr);
+  const bool use_aux_input = (fw_aux_input_to_forget_weights != nullptr);
+
   // Populate a TfLiteLSTMParams struct for the evaluation functions.
   TfLiteLSTMParams lstm_params = {params->activation, params->cell_clip,
                                   params->proj_clip, kTfLiteLSTMFullKernel};
@@ -879,6 +886,26 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
   const auto actual_bw_output = params->merge_outputs ? fw_output : bw_output;
 
   const bool time_major = params->time_major;
+
+  // We want to cover the following cases:
+  //
+  // If not stacking (not connected after other bidi lstms):
+  //   both fw & bw will just use `input`; aux_input will be null.
+  //
+  // If stacking with cross_links, TensorFlow equivalent
+  // (tf.contrib.rnn.stack_bidirectional_rnn):
+  //   both fw & bw will use `input`, but aux_input will be none null.
+  //   Note, this time, whether connected after other bidi lstms both works.
+  //
+  // If stacking without cross_links, but connected after other bidi lstms,
+  // TensorFlow equivalent (tf.nn.static_bidirectional_rnn):
+  //   fw will use `input`, bw will use aux_input, and the `real aux_input`
+  //   will be null.
+
+  const bool non_stacking_mode = !use_aux_input && has_previous_bw_output;
+  const TfLiteTensor* bw_input = non_stacking_mode ? aux_input : input;
+  const TfLiteTensor* real_aux_input = non_stacking_mode ? nullptr : aux_input;
+
   switch (fw_input_to_output_weights->type) {
     case kTfLiteFloat32: {
       TfLiteStatus fw_pass_status = lstm_eval::EvalFloat(
@@ -891,7 +918,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
           /*input_layer_norm_coefficients=*/nullptr,
           /*forget_layer_norm_coefficients=*/nullptr,
           /*cell_layer_norm_coefficients=*/nullptr,
-          /*output_layer_norm_coefficients=*/nullptr, aux_input,
+          /*output_layer_norm_coefficients=*/nullptr, real_aux_input,
           fw_aux_input_to_input_weights, fw_aux_input_to_forget_weights,
           fw_aux_input_to_cell_weights, fw_aux_input_to_output_weights,
           fw_input_gate_bias, fw_forget_gate_bias, fw_cell_bias,
@@ -902,7 +929,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
       TF_LITE_ENSURE_OK(context, fw_pass_status);
 
       TfLiteStatus bw_pass_status = lstm_eval::EvalFloat(
-          input, bw_input_to_input_weights, bw_input_to_forget_weights,
+          bw_input, bw_input_to_input_weights, bw_input_to_forget_weights,
           bw_input_to_cell_weights, bw_input_to_output_weights,
           bw_recurrent_to_input_weights, bw_recurrent_to_forget_weights,
           bw_recurrent_to_cell_weights, bw_recurrent_to_output_weights,
@@ -911,7 +938,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
           /*input_layer_norm_coefficients=*/nullptr,
           /*forget_layer_norm_coefficients=*/nullptr,
           /*cell_layer_norm_coefficients=*/nullptr,
-          /*output_layer_norm_coefficients=*/nullptr, aux_input,
+          /*output_layer_norm_coefficients=*/nullptr, real_aux_input,
           bw_aux_input_to_input_weights, bw_aux_input_to_forget_weights,
           bw_aux_input_to_cell_weights, bw_aux_input_to_output_weights,
           bw_input_gate_bias, bw_forget_gate_bias, bw_cell_bias,
@@ -942,9 +969,8 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
       TfLiteTensor* recovered_cell_weights =
           GetTemporary(context, node, kRecoveredCellWeights);
       TfLiteTensor* aux_input_quantized =
-          (aux_input == nullptr)
-              ? nullptr
-              : GetTemporary(context, node, kAuxInputQuantized);
+          use_aux_input ? GetTemporary(context, node, kAuxInputQuantized)
+                        : nullptr;
 
       TfLiteStatus fw_pass_status = lstm_eval::EvalHybrid(
           input, fw_input_to_input_weights, fw_input_to_forget_weights,
@@ -956,7 +982,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
           /*input_layer_norm_coefficients=*/nullptr,
           /*forget_layer_norm_coefficients=*/nullptr,
           /*cell_layer_norm_coefficients=*/nullptr,
-          /*output_layer_norm_coefficients=*/nullptr, aux_input,
+          /*output_layer_norm_coefficients=*/nullptr, real_aux_input,
           fw_aux_input_to_input_weights, fw_aux_input_to_forget_weights,
           fw_aux_input_to_cell_weights, fw_aux_input_to_output_weights,
           fw_input_gate_bias, fw_forget_gate_bias, fw_cell_bias,
@@ -970,7 +996,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
       TF_LITE_ENSURE_OK(context, fw_pass_status);
 
       TfLiteStatus bw_pass_status = lstm_eval::EvalHybrid(
-          input, bw_input_to_input_weights, bw_input_to_forget_weights,
+          bw_input, bw_input_to_input_weights, bw_input_to_forget_weights,
           bw_input_to_cell_weights, bw_input_to_output_weights,
           bw_recurrent_to_input_weights, bw_recurrent_to_forget_weights,
           bw_recurrent_to_cell_weights, bw_recurrent_to_output_weights,
@@ -979,7 +1005,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
           /*input_layer_norm_coefficients=*/nullptr,
           /*forget_layer_norm_coefficients=*/nullptr,
           /*cell_layer_norm_coefficients=*/nullptr,
-          /*output_layer_norm_coefficients=*/nullptr, aux_input,
+          /*output_layer_norm_coefficients=*/nullptr, real_aux_input,
           bw_aux_input_to_input_weights, bw_aux_input_to_forget_weights,
           bw_aux_input_to_cell_weights, bw_aux_input_to_output_weights,
           bw_input_gate_bias, bw_forget_gate_bias, bw_cell_bias,
diff --git a/tensorflow/lite/kernels/bidirectional_sequence_lstm_test.cc b/tensorflow/lite/kernels/bidirectional_sequence_lstm_test.cc
index f5df6d15af..59ea47a2a2 100644
--- a/tensorflow/lite/kernels/bidirectional_sequence_lstm_test.cc
+++ b/tensorflow/lite/kernels/bidirectional_sequence_lstm_test.cc
@@ -38,7 +38,7 @@ class BidirectionalLSTMOpModel : public SingleOpModel {
                            int sequence_length, bool use_cifg,
                            bool use_peephole, bool use_projection_weights,
                            bool use_projection_bias, bool merge_outputs,
-                           float cell_clip, float proj_clip,
+                           bool use_aux_input, float cell_clip, float proj_clip,
                            bool quantize_weights, bool time_major,
                            const std::vector<std::vector<int>>& input_shapes)
       : n_batch_(n_batch),
@@ -185,7 +185,11 @@ class BidirectionalLSTMOpModel : public SingleOpModel {
       bw_output_ = AddOutput(TensorType_FLOAT32);
     }
 
-    aux_input_ = AddNullInput();
+    if (use_aux_input) {
+      aux_input_ = AddInput(TensorType_FLOAT32);
+    } else {
+      aux_input_ = AddNullInput();
+    }
     fw_aux_input_to_input_weights_ = AddNullInput();
     fw_aux_input_to_forget_weights_ = AddNullInput();
     fw_aux_input_to_cell_weights_ = AddNullInput();
@@ -302,6 +306,10 @@ class BidirectionalLSTMOpModel : public SingleOpModel {
     PopulateTensor(input_, offset, begin, end);
   }
 
+  void SetAuxInput(int offset, float* begin, float* end) {
+    PopulateTensor(aux_input_, offset, begin, end);
+  }
+
   std::vector<float> GetFwOutput() { return ExtractVector<float>(fw_output_); }
   std::vector<float> GetBwOutput() { return ExtractVector<float>(bw_output_); }
 
@@ -406,7 +414,8 @@ TEST_P(LSTMOpTest, BlackBoxTestNoCifgNoPeepholeNoProjectionNoClipping) {
   BidirectionalLSTMOpModel lstm(
       n_batch, n_input, n_cell, n_output, sequence_length, /*use_cifg=*/false,
       /*use_peephole=*/false, /*use_projection_weights=*/false,
-      /*use_projection_bias=*/false, /*merge_outputs=*/false, /*cell_clip=*/0.0,
+      /*use_projection_bias=*/false, /*merge_outputs=*/false,
+      /*use_aux_input=*/false, /*cell_clip=*/0.0,
       /*proj_clip=*/0.0, quantize_weights, /*time_major=*/true,
       {
           {sequence_length, n_batch, n_input},  // input tensor
@@ -570,7 +579,8 @@ TEST_P(LSTMOpTest, BlackBoxTestMergedOutput) {
   BidirectionalLSTMOpModel lstm(
       n_batch, n_input, n_cell, n_output, sequence_length, /*use_cifg=*/false,
       /*use_peephole=*/false, /*use_projection_weights=*/false,
-      /*use_projection_bias=*/false, /*merge_outputs=*/true, /*cell_clip=*/0.0,
+      /*use_projection_bias=*/false, /*merge_outputs=*/true,
+      /*use_aux_input=*/false, /*cell_clip=*/0.0,
       /*proj_clip=*/0.0, quantize_weights, /*time_major=*/true,
       {
           {sequence_length, n_batch, n_input},  // input tensor
@@ -733,7 +743,8 @@ TEST(LSTMOpTest, BlackBoxTestNoCifgNoPeepholeNoProjectionNoClippingReverse) {
   BidirectionalLSTMOpModel lstm(
       n_batch, n_input, n_cell, n_output, sequence_length, /*use_cifg=*/false,
       /*use_peephole=*/false, /*use_projection_weights=*/false,
-      /*use_projection_bias=*/false, /*merge_outputs=*/false, /*cell_clip=*/0.0,
+      /*use_projection_bias=*/false, /*merge_outputs=*/false,
+      /*use_aux_input=*/false, /*cell_clip=*/0.0,
       /*proj_clip=*/0.0, /*quantize_weights=*/false, /*time_major=*/true,
       {
           {sequence_length, n_batch, n_input},  // input tensor
@@ -895,7 +906,8 @@ TEST(LSTMOpTest, BlackBoxTestWithCifgWithPeepholeNoProjectionNoClipping) {
   BidirectionalLSTMOpModel lstm(
       n_batch, n_input, n_cell, n_output, sequence_length, /*use_cifg=*/true,
       /*use_peephole=*/true, /*use_projection_weights=*/false,
-      /*use_projection_bias=*/false, /*merge_outputs=*/false, /*cell_clip=*/0.0,
+      /*use_projection_bias=*/false, /*merge_outputs=*/false,
+      /*use_aux_input=*/false, /*cell_clip=*/0.0,
       /*proj_clip=*/0.0, /*quantize_weights=*/false, /*time_major=*/true,
       {
           {sequence_length, n_batch, n_input},  // input tensor
@@ -1047,7 +1059,8 @@ TEST(LSTMOpTest,
   BidirectionalLSTMOpModel lstm(
       n_batch, n_input, n_cell, n_output, sequence_length, /*use_cifg=*/true,
       /*use_peephole=*/true, /*use_projection_weights=*/false,
-      /*use_projection_bias=*/false, /*merge_outputs=*/false, /*cell_clip=*/0.0,
+      /*use_projection_bias=*/false, /*merge_outputs=*/false,
+      /*use_aux_input=*/false, /*cell_clip=*/0.0,
       /*proj_clip=*/0.0, /*quantize_weights=*/false, /*time_major=*/true,
       {
           {sequence_length, n_batch, n_input},  // input tensor
@@ -1199,7 +1212,8 @@ TEST(LSTMOpTest, BlackBoxTestWithPeepholeWithProjectionNoClipping) {
   BidirectionalLSTMOpModel lstm(
       n_batch, n_input, n_cell, n_output, sequence_length, /*use_cifg=*/false,
       /*use_peephole=*/true, /*use_projection_weights=*/true,
-      /*use_projection_bias=*/false, /*merge_outputs=*/false, /*cell_clip=*/0.0,
+      /*use_projection_bias=*/false, /*merge_outputs=*/false,
+      /*use_aux_input=*/false, /*cell_clip=*/0.0,
       /*proj_clip=*/0.0, /*quantize_weights=*/false, /*time_major=*/true,
       {
           {sequence_length, n_batch, n_input},  // input tensor
@@ -1903,7 +1917,8 @@ TEST(LSTMOpTest, BlackBoxTestWithPeepholeWithProjectionNoClippingBatchMajor) {
   BidirectionalLSTMOpModel lstm(
       n_batch, n_input, n_cell, n_output, sequence_length, /*use_cifg=*/false,
       /*use_peephole=*/true, /*use_projection_weights=*/true,
-      /*use_projection_bias=*/false, /*merge_outputs=*/false, /*cell_clip=*/0.0,
+      /*use_projection_bias=*/false, /*merge_outputs=*/false,
+      /*use_aux_input=*/false, /*cell_clip=*/0.0,
       /*proj_clip=*/0.0, /*quantize_weights=*/false, /*time_major=*/false,
       {
           {n_batch, sequence_length, n_input},  // input tensor
@@ -2590,6 +2605,175 @@ TEST(LSTMOpTest, BlackBoxTestWithPeepholeWithProjectionNoClippingBatchMajor) {
   EXPECT_THAT(combined, ElementsAreArray(ArrayFloatNear(expected)));
 }
 
+// Same as the no cifg no peephole no projection no clipping test, but have an
+// aux input (without aux input weights), this is the case when stacking but no
+// cross-links.
+TEST_P(LSTMOpTest, BlackBoxTestWithAuxInput) {
+  const int n_batch = 1;
+  const int n_input = 2;
+  // n_cell and n_output have the same size when there is no projection.
+  const int n_cell = 4;
+  const int n_output = 4;
+  const int sequence_length = 3;
+  const bool quantize_weights = GetParam();
+
+  BidirectionalLSTMOpModel lstm(
+      n_batch, n_input, n_cell, n_output, sequence_length, /*use_cifg=*/false,
+      /*use_peephole=*/false, /*use_projection_weights=*/false,
+      /*use_projection_bias=*/false, /*merge_outputs=*/false,
+      /*use_aux_input=*/true, /*cell_clip=*/0.0,
+      /*proj_clip=*/0.0, quantize_weights, /*time_major=*/true,
+      {
+          {sequence_length, n_batch, n_input},  // input tensor
+
+          // Forward cell
+          {n_cell, n_input},  // input_to_input_weight tensor
+          {n_cell, n_input},  // input_to_forget_weight tensor
+          {n_cell, n_input},  // input_to_cell_weight tensor
+          {n_cell, n_input},  // input_to_output_weight tensor
+
+          {n_cell, n_output},  // recurrent_to_input_weight tensor
+          {n_cell, n_output},  // recurrent_to_forget_weight tensor
+          {n_cell, n_output},  // recurrent_to_cell_weight tensor
+          {n_cell, n_output},  // recurrent_to_output_weight tensor
+
+          {0},  // cell_to_input_weight tensor
+          {0},  // cell_to_forget_weight tensor
+          {0},  // cell_to_output_weight tensor
+
+          {n_cell},  // input_gate_bias tensor
+          {n_cell},  // forget_gate_bias tensor
+          {n_cell},  // cell_bias tensor
+          {n_cell},  // output_gate_bias tensor
+
+          {0, 0},  // projection_weight tensor
+          {0},     // projection_bias tensor
+
+          // Backward cell
+          {n_cell, n_input},  // input_to_input_weight tensor
+          {n_cell, n_input},  // input_to_forget_weight tensor
+          {n_cell, n_input},  // input_to_cell_weight tensor
+          {n_cell, n_input},  // input_to_output_weight tensor
+
+          {n_cell, n_output},  // recurrent_to_input_weight tensor
+          {n_cell, n_output},  // recurrent_to_forget_weight tensor
+          {n_cell, n_output},  // recurrent_to_cell_weight tensor
+          {n_cell, n_output},  // recurrent_to_output_weight tensor
+
+          {0},  // cell_to_input_weight tensor
+          {0},  // cell_to_forget_weight tensor
+          {0},  // cell_to_output_weight tensor
+
+          {n_cell},  // input_gate_bias tensor
+          {n_cell},  // forget_gate_bias tensor
+          {n_cell},  // cell_bias tensor
+          {n_cell},  // output_gate_bias tensor
+
+          {0, 0},  // projection_weight tensor
+          {0},     // projection_bias tensor
+
+          {n_batch, n_output},  // activation_state tensor
+          {n_batch, n_cell},    // cell_state tensor
+
+          {n_batch, n_output},  // activation_state tensor
+          {n_batch, n_cell},    // cell_state tensor
+
+          // TODO(b/121134029): Update tests so tensor shapes after state tensor
+          // are used. They are currently ignored by test_util.
+          {sequence_length, n_batch, n_input},  // aux_input tensor
+          {n_cell, 0},                          // aux_fw_input_to_input tensor
+          {n_cell, 0},                          // aux_fw_input_to_forget tensor
+          {n_cell, 0},                          // aux_fw_input_to_cell tensor
+          {n_cell, 0},                          // aux_fw_input_to_output tensor
+          {n_cell, 0},                          // aux_bw_input_to_input tensor
+          {n_cell, 0},                          // aux_bw_input_to_forget tensor
+          {n_cell, 0},                          // aux_bw_input_to_cell tensor
+          {n_cell, 0},                          // aux_bw_input_to_output tensor
+      });
+
+  lstm.SetInputToInputWeights({-0.45018822, -0.02338299, -0.0870589,
+                               -0.34550029, 0.04266912, -0.15680569,
+                               -0.34856534, 0.43890524});
+
+  lstm.SetInputToCellWeights({-0.50013041, 0.1370284, 0.11810488, 0.2013163,
+                              -0.20583314, 0.44344562, 0.22077113,
+                              -0.29909778});
+
+  lstm.SetInputToForgetWeights({0.09701663, 0.20334584, -0.50592935,
+                                -0.31343272, -0.40032279, 0.44781327,
+                                0.01387155, -0.35593212});
+
+  lstm.SetInputToOutputWeights({-0.25065863, -0.28290087, 0.04613829,
+                                0.40525138, 0.44272184, 0.03897077, -0.1556896,
+                                0.19487578});
+
+  lstm.SetInputGateBias({0., 0., 0., 0.});
+
+  lstm.SetCellBias({0., 0., 0., 0.});
+
+  lstm.SetForgetGateBias({1., 1., 1., 1.});
+
+  lstm.SetOutputGateBias({0., 0., 0., 0.});
+
+  lstm.SetRecurrentToInputWeights(
+      {-0.0063535, -0.2042388, 0.31454784, -0.35746509, 0.28902304, 0.08183324,
+       -0.16555229, 0.02286911, -0.13566875, 0.03034258, 0.48091322,
+       -0.12528998, 0.24077177, -0.51332325, -0.33502164, 0.10629296});
+
+  lstm.SetRecurrentToCellWeights(
+      {-0.3407414, 0.24443203, -0.2078532, 0.26320225, 0.05695659, -0.00123841,
+       -0.4744786, -0.35869038, -0.06418842, -0.13502428, -0.501764, 0.22830659,
+       -0.46367589, 0.26016325, -0.03894562, -0.16368064});
+
+  lstm.SetRecurrentToForgetWeights(
+      {-0.48684245, -0.06655136, 0.42224967, 0.2112639, 0.27654213, 0.20864892,
+       -0.07646349, 0.45877004, 0.00141793, -0.14609534, 0.36447752, 0.09196436,
+       0.28053468, 0.01560611, -0.20127171, -0.01140004});
+
+  lstm.SetRecurrentToOutputWeights(
+      {0.43385774, -0.17194885, 0.2718237, 0.09215671, 0.24107647, -0.39835793,
+       0.18212086, 0.01301402, 0.48572797, -0.50656658, 0.20047462, -0.20607421,
+       -0.51818722, -0.15390486, 0.0468148, 0.39922136});
+
+  // Input should have n_input * sequence_length many values.
+  static float lstm_input[] = {2., 3., 3., 4., 1., 1.};
+  static float lstm_fw_golden_output[] = {
+      -0.02973187, 0.1229473,  0.20885126, -0.15358765,
+      -0.03716109, 0.12507336, 0.41193449, -0.20860538,
+      -0.15053082, 0.09120187, 0.24278517, -0.12222792};
+  static float lstm_bw_golden_output[] = {
+      -0.0806187, 0.139077, 0.400476,   -0.197842, -0.0332076, 0.123838,
+      0.309777,   -0.17621, -0.0490733, 0.0739237, 0.067706,   -0.0208124};
+
+  float* batch0_start = lstm_input;
+  float* batch0_end = batch0_start + lstm.num_inputs() * lstm.sequence_length();
+
+  lstm.SetInput(0, batch0_start, batch0_end);
+  // Aux input and input are the same, so we should observe the same outputs
+  // as there's no aux input.
+  lstm.SetAuxInput(0, batch0_start, batch0_end);
+
+  lstm.Invoke();
+
+  float* fw_golden_start = lstm_fw_golden_output;
+  float* fw_golden_end =
+      fw_golden_start + lstm.num_fw_outputs() * lstm.sequence_length();
+  std::vector<float> fw_expected;
+  fw_expected.insert(fw_expected.end(), fw_golden_start, fw_golden_end);
+  EXPECT_THAT(lstm.GetFwOutput(),
+              ElementsAreArray(
+                  ArrayFloatNear(fw_expected, quantize_weights ? 1e-2 : 1e-5)));
+
+  float* bw_golden_start = lstm_bw_golden_output;
+  float* bw_golden_end =
+      bw_golden_start + lstm.num_bw_outputs() * lstm.sequence_length();
+  std::vector<float> bw_expected;
+  bw_expected.insert(bw_expected.end(), bw_golden_start, bw_golden_end);
+  EXPECT_THAT(lstm.GetBwOutput(),
+              ElementsAreArray(
+                  ArrayFloatNear(bw_expected, quantize_weights ? 1e-2 : 1e-5)));
+}
+
 }  // namespace
 }  // namespace tflite
 
-- 
GitLab


From cbb4b0bcd794d40a29196f0670396415c24b2881 Mon Sep 17 00:00:00 2001
From: manhyuk <manhyuk@kw.ac.kr>
Date: Fri, 4 Jan 2019 13:27:45 +0900
Subject: [PATCH 0214/1095] fix typo

---
 tensorflow/contrib/cmake/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/cmake/README.md b/tensorflow/contrib/cmake/README.md
index b2badc5785..b4f4b028f6 100644
--- a/tensorflow/contrib/cmake/README.md
+++ b/tensorflow/contrib/cmake/README.md
@@ -147,7 +147,7 @@ suitable interface for project configuration and dependency setting.
     *   Go (required if you need ssl support, optional)
     *   NASM/YASM (required by grpc for ssl support, optional)
 2.  Start CMake GUI
-3.  Click on `Browse Source` and direct to the the folder
+3.  Click on `Browse Source` and direct to the folder
     `<tensorflow-source>/tensorflow/contrib/cmake`
 4.  Click on `Browse Build` and spectify a location that you want tensorflow to
     be build
-- 
GitLab


From e913092464cc5376343063d7905d491d29808c09 Mon Sep 17 00:00:00 2001
From: manhyuk <manhyuk@kw.ac.kr>
Date: Fri, 4 Jan 2019 13:28:05 +0900
Subject: [PATCH 0215/1095] fix typo

---
 tensorflow/contrib/ignite/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/ignite/README.md b/tensorflow/contrib/ignite/README.md
index 5a8c650fb9..7493ea6759 100644
--- a/tensorflow/contrib/ignite/README.md
+++ b/tensorflow/contrib/ignite/README.md
@@ -30,7 +30,7 @@ system based on Apache Ignite.
 
 ## Features
 
-Ignite Dataset provides features that that you can use in a wide range of cases. The most important and interesting features are described below.
+Ignite Dataset provides features that you can use in a wide range of cases. The most important and interesting features are described below.
 
 ### Distributed In-Memory Datasource
 [Apache Ignite](https://ignite.apache.org/) is a distributed in-memory database, caching, and processing platform that provides fast data access. It allows you to avoid limitations of hard drive and store and operate with as much data as you need in distributed cluster. You can utilize
-- 
GitLab


From f332be115267be60ed7e16fed661555c15f7d990 Mon Sep 17 00:00:00 2001
From: Pariksheet Pinjari <pariksheet.pinjari@huawei.com>
Date: Fri, 4 Jan 2019 12:35:09 +0530
Subject: [PATCH 0216/1095] Update api_def_RegexReplace.pbtxt

Documentation issue fixed
---
 tensorflow/core/api_def/base_api/api_def_RegexReplace.pbtxt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/api_def/base_api/api_def_RegexReplace.pbtxt b/tensorflow/core/api_def/base_api/api_def_RegexReplace.pbtxt
index 70ad521926..2cc1a55676 100644
--- a/tensorflow/core/api_def/base_api/api_def_RegexReplace.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_RegexReplace.pbtxt
@@ -10,7 +10,7 @@ op {
   }
   in_arg {
     name: "rewrite"
-    description: "The rewrite to be applied to the matched expresion."
+    description: "The rewrite to be applied to the matched expression."
   }
   out_arg {
     name: "output"
-- 
GitLab


From 72f0dadfffd8c2f635ec58ded5c38154644c6f3f Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Thu, 3 Jan 2019 23:49:44 -0800
Subject: [PATCH 0217/1095] Automated rollback of commit
 7d57d32e440bc4f1654fb217fd701531a82c3587

PiperOrigin-RevId: 227809861
---
 tensorflow/contrib/mpi_collectives/BUILD                  | 2 +-
 tensorflow/stream_executor/platform/default/dso_loader.cc | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/mpi_collectives/BUILD b/tensorflow/contrib/mpi_collectives/BUILD
index d943ae6880..ecac06354d 100644
--- a/tensorflow/contrib/mpi_collectives/BUILD
+++ b/tensorflow/contrib/mpi_collectives/BUILD
@@ -52,7 +52,7 @@ tf_custom_op_library(
     deps = [
         ":mpi_defines",
         ":mpi_message_proto_cc",
-        "//tensorflow/core:stream_executor_headers_lib",
+        "//tensorflow/stream_executor:stream_executor_headers_lib",
         "//third_party/mpi",
     ],
 )
diff --git a/tensorflow/stream_executor/platform/default/dso_loader.cc b/tensorflow/stream_executor/platform/default/dso_loader.cc
index 668eeee3f3..0f0bce3253 100644
--- a/tensorflow/stream_executor/platform/default/dso_loader.cc
+++ b/tensorflow/stream_executor/platform/default/dso_loader.cc
@@ -28,7 +28,7 @@ limitations under the License.
 #include "tensorflow/stream_executor/lib/path.h"
 #include "tensorflow/stream_executor/lib/str_util.h"
 #include "tensorflow/stream_executor/lib/stringprintf.h"
-#include "tensorflow/stream_executor/platform/default/dso_loader.h"
+#include "tensorflow/stream_executor/platform/dso_loader.h"
 #include "tensorflow/stream_executor/platform/logging.h"
 #include "tensorflow/stream_executor/platform/port.h"
 
-- 
GitLab


From 6965d80c135a7c0008d2e17ce74529ab5798a5e4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 4 Jan 2019 01:02:44 -0800
Subject: [PATCH 0218/1095] compat: Update forward compatibility horizon to
 2019-01-04

PiperOrigin-RevId: 227817111
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index c60c42ee65..66c4fa4893 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 3)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 4)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From f76dc80f344440637b0b8ae84754c50bb4868b97 Mon Sep 17 00:00:00 2001
From: Dheeraj Rajaram Reddy <dheeraj98reddy@gmail.com>
Date: Fri, 4 Jan 2019 15:19:33 +0530
Subject: [PATCH 0219/1095] Add api_def for ExperimentalTakeWhileDataset

---
 ...api_def_ExperimentalTakeWhileDataset.pbtxt | 25 +++++++++++++++++++
 1 file changed, 25 insertions(+)
 create mode 100644 tensorflow/core/api_def/base_api/api_def_ExperimentalTakeWhileDataset.pbtxt

diff --git a/tensorflow/core/api_def/base_api/api_def_ExperimentalTakeWhileDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_ExperimentalTakeWhileDataset.pbtxt
new file mode 100644
index 0000000000..699e0c2e39
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_ExperimentalTakeWhileDataset.pbtxt
@@ -0,0 +1,25 @@
+op {
+  graph_op_name: "ExperimentalTakeWhileDataset"
+  visibility: HIDDEN
+  in_arg {
+    name: "other_arguments"
+    description: <<END
+A list of tensors, typically values that were captured when
+building a closure for `predicate`.
+END
+  }
+  attr {
+    name: "predicate"
+    description: <<END
+A function returning a scalar boolean.
+END
+  }
+  summary: "Creates a dataset that stops iteration when predicate` is false."
+  description: <<END
+The `predicate` function must return a scalar boolean and accept the
+following arguments:
+
+* One tensor for each component of an element of `input_dataset`.
+* One tensor for each value in `other_arguments`.
+END
+}
-- 
GitLab


From 2456898958279c1c75ac24da473665f418f2ec94 Mon Sep 17 00:00:00 2001
From: Dheeraj Rajaram Reddy <dheeraj98reddy@gmail.com>
Date: Fri, 4 Jan 2019 15:25:21 +0530
Subject: [PATCH 0220/1095] Format files according to TF style guide and minor
 fixes

---
 .../experimental/take_while_dataset_op.cc     | 32 ++++----
 .../kernel_tests/take_while_test.py           | 82 +++++++++----------
 .../data/experimental/ops/take_while_ops.py   | 40 ++++-----
 3 files changed, 72 insertions(+), 82 deletions(-)

diff --git a/tensorflow/core/kernels/data/experimental/take_while_dataset_op.cc b/tensorflow/core/kernels/data/experimental/take_while_dataset_op.cc
index b18cc32051..cf9c5f0cb6 100644
--- a/tensorflow/core/kernels/data/experimental/take_while_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/take_while_dataset_op.cc
@@ -40,7 +40,7 @@ class TakeWhileDatasetOp : public UnaryDatasetOpKernel {
     std::unique_ptr<CapturedFunction> captured_func;
     OP_REQUIRES_OK(ctx, CapturedFunction::Create(func_, ctx, "other_arguments",
                                                  &captured_func));
-    
+
     // TODO (squadrick): check short-circuit
     *output = new Dataset(ctx, input, func_, std::move(captured_func));
   }
@@ -49,7 +49,7 @@ class TakeWhileDatasetOp : public UnaryDatasetOpKernel {
   class Dataset : public DatasetBase {
    public:
     Dataset(OpKernelContext* ctx, const DatasetBase* input,
-            const NameAttrList& func, 
+            const NameAttrList& func,
             std::unique_ptr<CapturedFunction> captured_func)
         : DatasetBase(DatasetContext(ctx)),
           input_(input),
@@ -74,7 +74,9 @@ class TakeWhileDatasetOp : public UnaryDatasetOpKernel {
       return input_->output_shapes();
     }
 
-    string DebugString() const override { return "TakeWhileDatasetOp::Dataset"; }
+    string DebugString() const override {
+      return "TakeWhileDatasetOp::Dataset";
+    }
 
     int64 Cardinality() const override { return kUnknownCardinality; }
 
@@ -109,11 +111,11 @@ class TakeWhileDatasetOp : public UnaryDatasetOpKernel {
       b->BuildAttrValue(other_arguments_types, &other_arguments_types_attr);
 
       TF_RETURN_IF_ERROR(b->AddDataset(
-            this, {std::make_pair(0, input_node)},
-            {std::make_pair(1, other_arguments)},
-            {std::make_pair("predicate", f_attr),
-             std::make_pair("Targuments", other_arguments_types_attr)},
-             output));
+          this, {std::make_pair(0, input_node)},
+          {std::make_pair(1, other_arguments)},
+          {std::make_pair("predicate", f_attr),
+           std::make_pair("Targuments", other_arguments_types_attr)},
+          output));
       return Status::OK();
     }
 
@@ -135,7 +137,7 @@ class TakeWhileDatasetOp : public UnaryDatasetOpKernel {
                              bool* end_of_sequence) override {
         {
           tf_shared_lock l(mu_);
-          if(!input_impl_) {
+          if (!input_impl_) {
             *end_of_sequence = true;
             return Status::OK();
           }
@@ -154,17 +156,15 @@ class TakeWhileDatasetOp : public UnaryDatasetOpKernel {
             ctx, *out_tensors, &bool_output);
 
         if (s.ok()) {
-          if(bool_output.size() != 1 || bool_output[0].dtype() != DT_BOOL ||
-             bool_output[0].NumElements() != 1) {
+          if (bool_output.size() != 1 || bool_output[0].dtype() != DT_BOOL ||
+              bool_output[0].NumElements() != 1) {
             return errors::InvalidArgument(
                 "`predicate` must returns a scalar bool tensor.");
           }
-          if (!bool_output[0].scalar<bool>()()) {
-            *end_of_sequence = true;
-            return Status::OK();
-          }
+          *end_of_sequence = !bool_output[0].scalar<bool>()();
+          return Status::OK();
         }
-        return s; // propagate error to caller
+        return s;  // propagate error to caller
       }
 
      protected:
diff --git a/tensorflow/python/data/experimental/kernel_tests/take_while_test.py b/tensorflow/python/data/experimental/kernel_tests/take_while_test.py
index d561a3cd35..b1e7974471 100644
--- a/tensorflow/python/data/experimental/kernel_tests/take_while_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/take_while_test.py
@@ -18,76 +18,70 @@ from __future__ import division
 from __future__ import print_function
 
 import numpy as np
+from absl.testing import parameterized
 
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.experimental.ops import take_while_ops
 from tensorflow.python.framework import errors
-from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import test_util
 from tensorflow.python.framework import constant_op
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import functional_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
 
 
 @test_util.run_all_in_graph_and_eager_modes
-class TakeWhileTest(test_base.DatasetTestBase):
-
-  def testTakeWhileDataset(self):
-    def do_test(num_elements, window_size):
-      def predicate_func(elem):
-        return array_ops.shape(elem)[0] > (window_size - 1)
-
-      flatten_func = lambda x: dataset_ops.Dataset.from_tensor_slices(x)
-      take_while = take_while_ops.take_while(predicate_func)
-
-      dataset = dataset_ops.Dataset.range(num_elements).batch(window_size)
-      dataset = dataset.apply(take_while).flat_map(flatten_func)
-      
-      self.assertDatasetProduces(dataset, 
-              np.arange(int(num_elements / window_size) * window_size))
-
-    do_test(14, 2)
-    do_test(15, 2)
-    do_test(100, 3)
-
-  def testTakeWhileDatasetRange(self):
-    def get_dataset(num_elemets, upper_bound):
-      return dataset_ops.Dataset.range(num_elemets).apply(
-              take_while_ops.take_while(lambda x: x < upper_bound))
-        
-    def do_test(num_elemets, upper_bound):  
-      self.assertDatasetProduces(get_dataset(num_elemets, upper_bound), 
-              np.arange(upper_bound))
-    
-    def out_of_bounds(num_elemets, upper_bound):
+class TakeWhileTest(test_base.DatasetTestBase, parameterized.TestCase):
+  @parameterized.parameters(
+      (14, 2),
+      (15, 2),
+      (100, 3))
+  def testTakeWhileDataset(self, num_elements, window_size):
+    def _predicate_func(elem):
+      return array_ops.shape(elem)[0] > (window_size - 1)
+
+    take_while = take_while_ops.take_while(_predicate_func)
+
+    dataset = dataset_ops.Dataset.range(num_elements).batch(window_size)
+    dataset = dataset.apply(take_while).flat_map(
+        lambda x: dataset_ops.Dataset.from_tensor_slices(x))
+
+    expected_num_elements = int(num_elements / window_size) * window_size
+    self.assertDatasetProduces(dataset, np.arange(expected_num_elements))
+
+  @parameterized.parameters(
+      (10, 2, False),
+      (16, 7, False),
+      (100, 99, False),
+      (100, 101, True),
+      (0, 1, True))
+  def testTakeWhileDatasetRange(self, num_elements, upper_bound, out_of_bounds):
+    dataset = dataset_ops.Dataset.range(num_elements).apply(
+        take_while_ops.take_while(lambda x: x < upper_bound))
+
+    if out_of_bounds:
       with self.assertRaises(errors.OutOfRangeError):
-        self.assertDatasetProduces(get_dataset(num_elemets, upper_bound), 
-                np.arange(upper_bound))
+        self.assertDatasetProduces(dataset, np.arange(upper_bound))
 
-    do_test(10, 2)
-    do_test(16, 7)
-    do_test(100, 99)
-    out_of_bounds(100, 101)
-    out_of_bounds(0, 1)
+    else:
+      self.assertDatasetProduces(dataset, np.arange(upper_bound))
 
   def testTakeWhileDatasetString(self):
-    def stringNotEquals(string):
-        return lambda x: math_ops.not_equal(x, constant_op.constant(string))
+    def not_equal(string):
+      return lambda x: math_ops.not_equal(x, constant_op.constant(string))
 
     string = ["this", "is", "the", "test", "for", "strings"]
     dataset = dataset_ops.Dataset.from_tensor_slices(string).apply(
-            take_while_ops.take_while(stringNotEquals("test")))
-    
+        take_while_ops.take_while(not_equal("test")))
+
     next_element = self.getNext(dataset)
     self.assertEqual(b"this", self.evaluate(next_element()))
     self.assertEqual(b"is", self.evaluate(next_element()))
     self.assertEqual(b"the", self.evaluate(next_element()))
 
     with self.assertRaises(errors.OutOfRangeError):
-        self.assertEqual(b"test", self.evaluate(next_element()))
+      self.assertEqual(b"test", self.evaluate(next_element()))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/experimental/ops/take_while_ops.py b/tensorflow/python/data/experimental/ops/take_while_ops.py
index 224eb8aa77..c9518cc9ad 100644
--- a/tensorflow/python/data/experimental/ops/take_while_ops.py
+++ b/tensorflow/python/data/experimental/ops/take_while_ops.py
@@ -17,12 +17,9 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import collections
-
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.util import structure as structure_lib
 from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import ops
 from tensorflow.python.ops import gen_experimental_dataset_ops
 from tensorflow.python.util.tf_export import tf_export
 
@@ -32,28 +29,27 @@ class _TakeWhileDataset(dataset_ops.UnaryUnchangedStructureDataset):
 
   def __init__(self, input_dataset, predicate):
     """See `take_while()` for details."""
-    
+
     self._input_dataset = input_dataset
     wrapped_func = dataset_ops.StructuredFunctionWrapper(
-            predicate,
-            "tf.data.experimental.take_while()",
-            dataset=self._input_dataset)
-    
+        predicate,
+        "tf.data.experimental.take_while()",
+        dataset=self._input_dataset)
+
     if not wrapped_func.output_structure.is_compatible_with(
-            structure_lib.TensorStructure(dtypes.bool, [])):
-        raise ValueError("`predicate` must return a scalar boolean tensor.")
-     
+        structure_lib.TensorStructure(dtypes.bool, [])):
+      raise ValueError("`predicate` must return a scalar boolean tensor.")
+    
     self._predicate = wrapped_func
-
-    variant_tensor = gen_experimental_dataset_ops.experimental_take_while_dataset(
-            self._input_dataset._variant_tensor,
-            other_arguments=self._predicate.function.captured_inputs,
-            predicate=self._predicate.function,
-            **dataset_ops.flat_structure(self))
-    super(_TakeWhileDataset, self).__init__(input_dataset, variant_tensor)
+    var_tensor = gen_experimental_dataset_ops.experimental_take_while_dataset(
+        self._input_dataset._variant_tensor, # pylint: disable=protected-access
+        other_arguments=self._predicate.function.captured_inputs,
+        predicate=self._predicate.function,
+        **dataset_ops.flat_structure(self))
+    super(_TakeWhileDataset, self).__init__(input_dataset, var_tensor)
 
   def _functions(self):
-      return [self._predicate]
+    return [self._predicate]
 
 
 @tf_export("data.experimental.take_while")
@@ -61,8 +57,8 @@ def take_while(predicate):
   """A transformation that stops dataset iteration based on a `predicate` condition
 
   Args:
-    predicate: A function that maps a nested structure of tensors 
-    (having shapes and types defined by `self.output_shapes` and 
+    predicate: A function that maps a nested structure of tensors
+    (having shapes and types defined by `self.output_shapes` and
     `self.output_types`) to a scalar `tf.bool` tensor.
 
   Returns:
@@ -70,6 +66,6 @@ def take_while(predicate):
     `tf.data.Dataset.apply`.
   """
   def _apply_fn(dataset):
-      return _TakeWhileDataset(dataset, predicate)
+    return _TakeWhileDataset(dataset, predicate)
 
   return _apply_fn
-- 
GitLab


From 28edbb813a3a94892d57ce29354e69c0a960a23e Mon Sep 17 00:00:00 2001
From: Dheeraj Rajaram Reddy <dheeraj98reddy@gmail.com>
Date: Fri, 4 Jan 2019 15:32:40 +0530
Subject: [PATCH 0221/1095] Use parameterized test case

---
 .../take_while_dataset_serialization_test.py  | 26 ++++++++++---------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/tensorflow/python/data/experimental/kernel_tests/serialization/take_while_dataset_serialization_test.py b/tensorflow/python/data/experimental/kernel_tests/serialization/take_while_dataset_serialization_test.py
index 9b5498cca5..a96e37f0b4 100644
--- a/tensorflow/python/data/experimental/kernel_tests/serialization/take_while_dataset_serialization_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/serialization/take_while_dataset_serialization_test.py
@@ -17,6 +17,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from absl.testing import parameterized
+
 from tensorflow.python.data.experimental.kernel_tests.serialization import dataset_serialization_test_base
 from tensorflow.python.data.experimental.ops import take_while_ops
 from tensorflow.python.data.ops import dataset_ops
@@ -24,21 +26,21 @@ from tensorflow.python.platform import test
 
 
 class TakeWhileDatasetSerializationTest(
-    dataset_serialization_test_base.DatasetSerializationTestBase):
+    dataset_serialization_test_base.DatasetSerializationTestBase,
+    parameterized.TestCase):
 
   def _build_dataset(self, num_elements, upper_bound):
     return dataset_ops.Dataset.range(num_elements).apply(
-              take_while_ops.take_while(lambda x: x < upper_bound))
-
-  def testCore(self):
-    def run_test(num_elem1, num_elem2, upper_bound):
-        self.run_core_tests(lambda: self._build_dataset(num_elem1, upper_bound),
-                            lambda: self._build_dataset(num_elem2, upper_bound), 
-                            upper_bound)
-
-    run_test(23, 10, 7)
-    run_test(10, 50, 0)
-    run_test(25, 30, 25)
+        take_while_ops.take_while(lambda x: x < upper_bound))
+
+  @parameterized.parameters(
+      (23, 10, 7),
+      (10, 50, 0),
+      (25, 30, 25))
+  def testCore(self, num_elem1, num_elem2, upper_bound):
+    self.run_core_tests(lambda: self._build_dataset(num_elem1, upper_bound),
+                        lambda: self._build_dataset(num_elem2, upper_bound),
+                        upper_bound)
 
 
 if __name__ == "__main__":
-- 
GitLab


From 8e4ca33c8b5861526bcdc43eb1af6d73e1670e92 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 4 Jan 2019 04:10:23 -0800
Subject: [PATCH 0222/1095] Correctly handle S32 and U32 in the AR-CRS combiner

Some of the transforms we are doing are only valid on floating point types
so we have to condition them on the element type of the operations.

PiperOrigin-RevId: 227836047
---
 tensorflow/compiler/xla/service/BUILD         |  1 +
 .../compiler/xla/service/ar_crs_combiner.cc   | 24 +++++++++++++++----
 2 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index b84792cfc3..755c477a12 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -3678,6 +3678,7 @@ cc_library(
         ":pattern_matcher",
         "//tensorflow/compiler/xla:literal",
         "//tensorflow/compiler/xla:literal_util",
+        "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:status_macros",
         "//tensorflow/compiler/xla:statusor",
         "//tensorflow/compiler/xla:types",
diff --git a/tensorflow/compiler/xla/service/ar_crs_combiner.cc b/tensorflow/compiler/xla/service/ar_crs_combiner.cc
index 47d2c7e357..4a227d3b5c 100644
--- a/tensorflow/compiler/xla/service/ar_crs_combiner.cc
+++ b/tensorflow/compiler/xla/service/ar_crs_combiner.cc
@@ -26,6 +26,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/hlo_instruction.h"
 #include "tensorflow/compiler/xla/service/hlo_opcode.h"
 #include "tensorflow/compiler/xla/service/pattern_matcher.h"
+#include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/status_macros.h"
 #include "tensorflow/compiler/xla/types.h"
 #include "tensorflow/compiler/xla/xla_data.pb.h"
@@ -44,11 +45,24 @@ bool MatchesArCrsPattern(HloInstruction* instruction) {
     if (instruction->user_count() != 1) {
       return false;
     }
-    auto opcode = instruction->opcode();
-    return opcode == HloOpcode::kBitcast || opcode == HloOpcode::kTranspose ||
-           opcode == HloOpcode::kReshape || opcode == HloOpcode::kConvert ||
-           opcode == HloOpcode::kAdd || opcode == HloOpcode::kSubtract ||
-           opcode == HloOpcode::kMultiply;
+    switch (instruction->opcode()) {
+      case HloOpcode::kBitcast:
+      case HloOpcode::kTranspose:
+      case HloOpcode::kReshape:
+        return true;
+      case HloOpcode::kConvert:
+        // Can be moved across if both input and output is either float or
+        // integer (e.g. S32<->U32 or F32<->BF16)
+        return ShapeUtil::ElementIsFloating(instruction->shape()) ==
+               ShapeUtil::ElementIsFloating(instruction->operand(0)->shape());
+      case HloOpcode::kAdd:
+      case HloOpcode::kSubtract:
+      case HloOpcode::kMultiply:
+        // Only supported for floating point operands.
+        return ShapeUtil::ElementIsFloating(instruction->shape());
+      default:
+        return false;
+    }
   };
 
   auto computation_is_addition = [](HloComputation* c) {
-- 
GitLab


From 0bc20afeec2b06437a7196b0d191d61f181cd81a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 4 Jan 2019 07:27:10 -0800
Subject: [PATCH 0223/1095] Update RBE CPU Dockerfile with instructions on how
 to build/push.

PiperOrigin-RevId: 227853285
---
 tensorflow/tools/ci_build/Dockerfile.rbe.cpu | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tensorflow/tools/ci_build/Dockerfile.rbe.cpu b/tensorflow/tools/ci_build/Dockerfile.rbe.cpu
index 7e5860aeec..500fb6e0b3 100644
--- a/tensorflow/tools/ci_build/Dockerfile.rbe.cpu
+++ b/tensorflow/tools/ci_build/Dockerfile.rbe.cpu
@@ -1,3 +1,8 @@
+# To push a new version, run:
+# $ docker build -f Dockerfile.rbe.cpu \
+#       --tag "gcr.io/tensorflow-testing/nosla-ubuntu16.04" .
+# $ docker push gcr.io/tensorflow-testing/nosla-ubuntu16.04
+
 FROM launcher.gcr.io/google/rbe-ubuntu16-04:r327695
 LABEL maintainer="Yu Yi <yiyu@google.com>"
 
-- 
GitLab


From bd2f78fd205faa739c657d26156bd27e02223df0 Mon Sep 17 00:00:00 2001
From: Scott Zhu <scottzhu@google.com>
Date: Fri, 4 Jan 2019 08:31:47 -0800
Subject: [PATCH 0224/1095] Make contrib.seq2seq more compatible with tf 2.0.
 (1/6)

1. Add a new loss object which wrap around the existing loss function, which
use can use it keras model.
2. Slightly update the code for readability and reduce duplication.
3. Update the test case to be eager compatible.

PiperOrigin-RevId: 227860442
---
 .../seq2seq/python/kernel_tests/loss_test.py  | 290 ++++++++++++++----
 tensorflow/contrib/seq2seq/python/ops/loss.py | 109 +++++--
 2 files changed, 315 insertions(+), 84 deletions(-)

diff --git a/tensorflow/contrib/seq2seq/python/kernel_tests/loss_test.py b/tensorflow/contrib/seq2seq/python/kernel_tests/loss_test.py
index 5aa32b532f..41b2a53ca5 100644
--- a/tensorflow/contrib/seq2seq/python/kernel_tests/loss_test.py
+++ b/tensorflow/contrib/seq2seq/python/kernel_tests/loss_test.py
@@ -14,80 +14,254 @@
 # ==============================================================================
 
 """Tests for contrib.seq2seq.python.seq2seq.loss_ops."""
-# pylint: disable=unused-import,g-bad-import-order
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
-# pylint: enable=unused-import
 
 import numpy as np
 
 from tensorflow.contrib.seq2seq.python.ops import loss
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import init_ops
-from tensorflow.python.ops import variable_scope
 from tensorflow.python.platform import test
 
 
+@test_util.run_all_in_graph_and_eager_modes
 class LossTest(test.TestCase):
 
+  def setUp(self):
+    self.batch_size = 2
+    self.sequence_length = 3
+    self.number_of_classes = 5
+    logits = [
+        constant_op.constant(i + 0.5, shape=[self.batch_size,
+                                             self.number_of_classes])
+        for i in range(self.sequence_length)
+    ]
+    self.logits = array_ops.stack(logits, axis=1)
+    targets = [
+        constant_op.constant(i, dtypes.int32, shape=[self.batch_size])
+        for i in range(self.sequence_length)
+    ]
+    self.targets = array_ops.stack(targets, axis=1)
+    weights = [
+        constant_op.constant(1.0, shape=[self.batch_size])
+        for _ in range(self.sequence_length)
+    ]
+    self.weights = array_ops.stack(weights, axis=1)
+    # expected_loss = sparse_softmax_cross_entropy_with_logits(targets, logits)
+    # where targets = [0, 1, 2], and logits = [[0.5] * 5, [1.5] * 5, [2.5] * 5]
+    self.expected_loss = 1.60944
+
   def testSequenceLoss(self):
-    with self.session(use_gpu=True) as sess:
-      with variable_scope.variable_scope(
-          'root', initializer=init_ops.constant_initializer(0.5)):
-        batch_size = 2
-        sequence_length = 3
-        number_of_classes = 5
-        logits = [
-            constant_op.constant(
-                i + 0.5, shape=[batch_size, number_of_classes])
-            for i in range(sequence_length)
-        ]
-        logits = array_ops.stack(logits, axis=1)
-        targets = [
-            constant_op.constant(
-                i, dtypes.int32, shape=[batch_size])
-            for i in range(sequence_length)
-        ]
-        targets = array_ops.stack(targets, axis=1)
-        weights = [
-            constant_op.constant(
-                1.0, shape=[batch_size]) for i in range(sequence_length)
-        ]
-        weights = array_ops.stack(weights, axis=1)
-
-        average_loss_per_example = loss.sequence_loss(
-            logits, targets, weights,
-            average_across_timesteps=True,
-            average_across_batch=True)
-        res = sess.run(average_loss_per_example)
-        self.assertAllClose(1.60944, res)
-
-        average_loss_per_sequence = loss.sequence_loss(
-            logits, targets, weights,
-            average_across_timesteps=False,
-            average_across_batch=True)
-        res = sess.run(average_loss_per_sequence)
-        compare_per_sequence = np.ones((sequence_length)) * 1.60944
-        self.assertAllClose(compare_per_sequence, res)
-
-        average_loss_per_batch = loss.sequence_loss(
-            logits, targets, weights,
-            average_across_timesteps=True,
-            average_across_batch=False)
-        res = sess.run(average_loss_per_batch)
-        compare_per_batch = np.ones((batch_size)) * 1.60944
-        self.assertAllClose(compare_per_batch, res)
-
-        total_loss = loss.sequence_loss(
-            logits, targets, weights,
-            average_across_timesteps=False,
-            average_across_batch=False)
-        res = sess.run(total_loss)
-        compare_total = np.ones((batch_size, sequence_length)) * 1.60944
-        self.assertAllClose(compare_total, res)
+    with self.test_session(use_gpu=True):
+      average_loss_per_example = loss.sequence_loss(
+          self.logits, self.targets, self.weights,
+          average_across_timesteps=True,
+          average_across_batch=True)
+      res = self.evaluate(average_loss_per_example)
+      self.assertAllClose(self.expected_loss, res)
+
+      average_loss_per_sequence = loss.sequence_loss(
+          self.logits, self.targets, self.weights,
+          average_across_timesteps=False,
+          average_across_batch=True)
+      res = self.evaluate(average_loss_per_sequence)
+      compare_per_sequence = np.full((self.sequence_length), self.expected_loss)
+      self.assertAllClose(compare_per_sequence, res)
+
+      average_loss_per_batch = loss.sequence_loss(
+          self.logits, self.targets, self.weights,
+          average_across_timesteps=True,
+          average_across_batch=False)
+      res = self.evaluate(average_loss_per_batch)
+      compare_per_batch = np.full((self.batch_size), self.expected_loss)
+      self.assertAllClose(compare_per_batch, res)
+
+      total_loss = loss.sequence_loss(
+          self.logits, self.targets, self.weights,
+          average_across_timesteps=False,
+          average_across_batch=False)
+      res = self.evaluate(total_loss)
+      compare_total = np.full((self.batch_size, self.sequence_length),
+                              self.expected_loss)
+      self.assertAllClose(compare_total, res)
+
+  def testSequenceLossClass(self):
+    with self.test_session(use_gpu=True):
+      seq_loss = loss.SequenceLoss(average_across_timesteps=True,
+                                   average_across_batch=True,
+                                   sum_over_timesteps=False,
+                                   sum_over_batch=False)
+      average_loss_per_example = seq_loss(
+          self.targets, self.logits, self.weights)
+      res = self.evaluate(average_loss_per_example)
+      self.assertAllClose(self.expected_loss, res)
+
+      seq_loss = loss.SequenceLoss(average_across_timesteps=False,
+                                   average_across_batch=True,
+                                   sum_over_timesteps=False,
+                                   sum_over_batch=False)
+      average_loss_per_sequence = seq_loss(
+          self.targets, self.logits, self.weights)
+      res = self.evaluate(average_loss_per_sequence)
+      compare_per_sequence = np.full((self.sequence_length), self.expected_loss)
+      self.assertAllClose(compare_per_sequence, res)
+
+      seq_loss = loss.SequenceLoss(average_across_timesteps=True,
+                                   average_across_batch=False,
+                                   sum_over_timesteps=False,
+                                   sum_over_batch=False)
+      average_loss_per_batch = seq_loss(
+          self.targets, self.logits, self.weights)
+      res = self.evaluate(average_loss_per_batch)
+      compare_per_batch = np.full((self.batch_size), self.expected_loss)
+      self.assertAllClose(compare_per_batch, res)
+
+      seq_loss = loss.SequenceLoss(average_across_timesteps=False,
+                                   average_across_batch=False,
+                                   sum_over_timesteps=False,
+                                   sum_over_batch=False)
+      total_loss = seq_loss(
+          self.targets, self.logits, self.weights)
+      res = self.evaluate(total_loss)
+      compare_total = np.full((self.batch_size, self.sequence_length),
+                              self.expected_loss)
+      self.assertAllClose(compare_total, res)
+
+  def testSumReduction(self):
+    with self.test_session(use_gpu=True):
+      seq_loss = loss.SequenceLoss(average_across_timesteps=False,
+                                   average_across_batch=False,
+                                   sum_over_timesteps=True,
+                                   sum_over_batch=True)
+      average_loss_per_example = seq_loss(
+          self.targets, self.logits, self.weights)
+      res = self.evaluate(average_loss_per_example)
+      self.assertAllClose(self.expected_loss, res)
+
+      seq_loss = loss.SequenceLoss(average_across_timesteps=False,
+                                   average_across_batch=False,
+                                   sum_over_timesteps=False,
+                                   sum_over_batch=True)
+      average_loss_per_sequence = seq_loss(
+          self.targets, self.logits, self.weights)
+      res = self.evaluate(average_loss_per_sequence)
+      compare_per_sequence = np.full((self.sequence_length), self.expected_loss)
+      self.assertAllClose(compare_per_sequence, res)
+
+      seq_loss = loss.SequenceLoss(average_across_timesteps=False,
+                                   average_across_batch=False,
+                                   sum_over_timesteps=True,
+                                   sum_over_batch=False)
+      average_loss_per_batch = seq_loss(
+          self.targets, self.logits, self.weights)
+      res = self.evaluate(average_loss_per_batch)
+      compare_per_batch = np.full((self.batch_size), self.expected_loss)
+      self.assertAllClose(compare_per_batch, res)
+
+      seq_loss = loss.SequenceLoss(average_across_timesteps=False,
+                                   average_across_batch=False,
+                                   sum_over_timesteps=False,
+                                   sum_over_batch=False)
+      total_loss = seq_loss(
+          self.targets, self.logits, self.weights)
+      res = self.evaluate(total_loss)
+      compare_total = np.full((self.batch_size, self.sequence_length),
+                              self.expected_loss)
+      self.assertAllClose(compare_total, res)
+
+  def testWeightedSumReduction(self):
+    weights = [
+        constant_op.constant(1.0, shape=[self.batch_size])
+        for _ in range(self.sequence_length)
+    ]
+    # Make the last element in the sequence to have zero weights.
+    weights[-1] = constant_op.constant(0.0, shape=[self.batch_size])
+    self.weights = array_ops.stack(weights, axis=1)
+    with self.test_session(use_gpu=True):
+      seq_loss = loss.SequenceLoss(average_across_timesteps=False,
+                                   average_across_batch=False,
+                                   sum_over_timesteps=True,
+                                   sum_over_batch=True)
+      average_loss_per_example = seq_loss(
+          self.targets, self.logits, self.weights)
+      res = self.evaluate(average_loss_per_example)
+      self.assertAllClose(self.expected_loss, res)
+
+      seq_loss = loss.SequenceLoss(average_across_timesteps=False,
+                                   average_across_batch=False,
+                                   sum_over_timesteps=False,
+                                   sum_over_batch=True)
+      average_loss_per_sequence = seq_loss(
+          self.targets, self.logits, self.weights)
+      res = self.evaluate(average_loss_per_sequence)
+      compare_per_sequence = np.full((self.sequence_length), self.expected_loss)
+      # The last element in every sequence are zeros, which will be filtered.
+      compare_per_sequence[-1] = 0.
+      self.assertAllClose(compare_per_sequence, res)
+
+      seq_loss = loss.SequenceLoss(average_across_timesteps=False,
+                                   average_across_batch=False,
+                                   sum_over_timesteps=True,
+                                   sum_over_batch=False)
+      average_loss_per_batch = seq_loss(self.targets, self.logits, self.weights)
+      res = self.evaluate(average_loss_per_batch)
+      compare_per_batch = np.full((self.batch_size), self.expected_loss)
+      self.assertAllClose(compare_per_batch, res)
+
+      seq_loss = loss.SequenceLoss(average_across_timesteps=False,
+                                   average_across_batch=False,
+                                   sum_over_timesteps=False,
+                                   sum_over_batch=False)
+      total_loss = seq_loss(self.targets, self.logits, self.weights)
+      res = self.evaluate(total_loss)
+      compare_total = np.full((self.batch_size, self.sequence_length),
+                              self.expected_loss)
+      # The last element in every sequence are zeros, which will be filtered.
+      compare_total[:, -1] = 0
+      self.assertAllClose(compare_total, res)
+
+  def testZeroWeights(self):
+    weights = [
+        constant_op.constant(0.0, shape=[self.batch_size])
+        for _ in range(self.sequence_length)
+    ]
+    weights = array_ops.stack(weights, axis=1)
+    with self.test_session(use_gpu=True):
+      average_loss_per_example = loss.sequence_loss(
+          self.logits, self.targets, weights,
+          average_across_timesteps=True,
+          average_across_batch=True)
+      res = self.evaluate(average_loss_per_example)
+      self.assertAllClose(0.0, res)
+
+      average_loss_per_sequence = loss.sequence_loss(
+          self.logits, self.targets, weights,
+          average_across_timesteps=False,
+          average_across_batch=True)
+      res = self.evaluate(average_loss_per_sequence)
+      compare_per_sequence = np.zeros((self.sequence_length))
+      self.assertAllClose(compare_per_sequence, res)
+
+      average_loss_per_batch = loss.sequence_loss(
+          self.logits, self.targets, weights,
+          average_across_timesteps=True,
+          average_across_batch=False)
+      res = self.evaluate(average_loss_per_batch)
+      compare_per_batch = np.zeros((self.batch_size))
+      self.assertAllClose(compare_per_batch, res)
+
+      total_loss = loss.sequence_loss(
+          self.logits, self.targets, weights,
+          average_across_timesteps=False,
+          average_across_batch=False)
+      res = self.evaluate(total_loss)
+      compare_total = np.zeros((self.batch_size, self.sequence_length))
+      self.assertAllClose(compare_total, res)
+
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/contrib/seq2seq/python/ops/loss.py b/tensorflow/contrib/seq2seq/python/ops/loss.py
index 39a6d2f58b..0fbfd61870 100644
--- a/tensorflow/contrib/seq2seq/python/ops/loss.py
+++ b/tensorflow/contrib/seq2seq/python/ops/loss.py
@@ -20,11 +20,12 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.framework import ops
+from tensorflow.python.keras.losses import Loss
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn_ops
 
-__all__ = ["sequence_loss"]
+__all__ = ["sequence_loss", "SequenceLoss"]
 
 
 def sequence_loss(logits,
@@ -32,16 +33,26 @@ def sequence_loss(logits,
                   weights,
                   average_across_timesteps=True,
                   average_across_batch=True,
+                  sum_over_timesteps=False,
+                  sum_over_batch=False,
                   softmax_loss_function=None,
                   name=None):
   """Weighted cross-entropy loss for a sequence of logits.
 
-  Depending on the values of `average_across_timesteps` and
-  `average_across_batch`, the return Tensor will have rank 0, 1, or 2 as these
-  arguments reduce the cross-entropy at each target, which has shape
-  `[batch_size, sequence_length]`, over their respective dimensions. For
-  example, if `average_across_timesteps` is `True` and `average_across_batch`
-  is `False`, then the return Tensor will have shape `[batch_size]`.
+  Depending on the values of `average_across_timesteps` / `sum_over_timesteps`
+  and `average_across_batch` / `sum_over_batch`, the return Tensor will have
+  rank 0, 1, or 2 as these arguments reduce the cross-entropy at each target,
+  which has shape `[batch_size, sequence_length]`, over their respective
+  dimensions. For example, if `average_across_timesteps` is `True` and
+  `average_across_batch` is `False`, then the return Tensor will have shape
+  `[batch_size]`.
+
+  Note that `average_across_timesteps` and `sum_over_timesteps` cannot be True
+  at same time. Same for `average_across_batch` and `sum_over_batch`.
+
+  The recommended loss reduction in tf 2.0 has been changed to sum_over, instead
+  of weighted average. User are recommend to use `sum_over_timesteps` and
+  `sum_over_batch` for reduction.
 
   Args:
     logits: A Tensor of shape
@@ -58,6 +69,12 @@ def sequence_loss(logits,
       dimension and divide the cost by the total label weight across timesteps.
     average_across_batch: If set, sum the cost across the batch dimension and
       divide the returned cost by the batch size.
+    sum_over_timesteps: If set, sum the cost across the sequence dimension and
+      divide the size of the sequence. Note that any element with 0 weights will
+      be excluded from size calculation.
+    sum_over_batch: if set, sum the cost across the batch dimension and divide
+      the total cost by the batch size. Not that any element with 0 weights will
+      be excluded from size calculation.
     softmax_loss_function: Function (labels, logits) -> loss-batch
       to be used instead of the standard softmax (the default if this is None).
       **Note that to avoid confusion, it is required for the function to accept
@@ -78,11 +95,15 @@ def sequence_loss(logits,
     raise ValueError("Logits must be a "
                      "[batch_size x sequence_length x logits] tensor")
   if len(targets.get_shape()) != 2:
-    raise ValueError("Targets must be a [batch_size x sequence_length] "
-                     "tensor")
+    raise ValueError("Targets must be a [batch_size x sequence_length] tensor")
   if len(weights.get_shape()) != 2:
-    raise ValueError("Weights must be a [batch_size x sequence_length] "
-                     "tensor")
+    raise ValueError("Weights must be a [batch_size x sequence_length] tensor")
+  if average_across_timesteps and sum_over_timesteps:
+    raise ValueError("average_across_timesteps and sum_over_timesteps cannot "
+                     "be set to True at same time.")
+  if average_across_batch and sum_over_batch:
+    raise ValueError("average_across_batch and sum_over_batch cannot be set "
+                     "to True at same time.")
   with ops.name_scope(name, "sequence_loss", [logits, targets, weights]):
     num_classes = array_ops.shape(logits)[2]
     logits_flat = array_ops.reshape(logits, [-1, num_classes])
@@ -96,20 +117,56 @@ def sequence_loss(logits,
     if average_across_timesteps and average_across_batch:
       crossent = math_ops.reduce_sum(crossent)
       total_size = math_ops.reduce_sum(weights)
-      total_size += 1e-12  # to avoid division by 0 for all-0 weights
-      crossent /= total_size
+      crossent = math_ops.div_no_nan(crossent, total_size)
+    elif sum_over_timesteps and sum_over_batch:
+      crossent = math_ops.reduce_sum(crossent)
+      total_count = math_ops.cast(math_ops.count_nonzero(weights),
+                                  crossent.dtype)
+      crossent = math_ops.div_no_nan(crossent, total_count)
     else:
-      batch_size = array_ops.shape(logits)[0]
-      sequence_length = array_ops.shape(logits)[1]
-      crossent = array_ops.reshape(crossent, [batch_size, sequence_length])
-    if average_across_timesteps and not average_across_batch:
-      crossent = math_ops.reduce_sum(crossent, axis=[1])
-      total_size = math_ops.reduce_sum(weights, axis=[1])
-      total_size += 1e-12  # to avoid division by 0 for all-0 weights
-      crossent /= total_size
-    if not average_across_timesteps and average_across_batch:
-      crossent = math_ops.reduce_sum(crossent, axis=[0])
-      total_size = math_ops.reduce_sum(weights, axis=[0])
-      total_size += 1e-12  # to avoid division by 0 for all-0 weights
-      crossent /= total_size
+      crossent = array_ops.reshape(crossent, array_ops.shape(logits)[0:2])
+      if average_across_timesteps or average_across_batch:
+        reduce_axis = [0] if average_across_batch else [1]
+        crossent = math_ops.reduce_sum(crossent, axis=reduce_axis)
+        total_size = math_ops.reduce_sum(weights, axis=reduce_axis)
+        crossent = math_ops.div_no_nan(crossent, total_size)
+      elif sum_over_timesteps or sum_over_batch:
+        reduce_axis = [0] if sum_over_batch else [1]
+        crossent = math_ops.reduce_sum(crossent, axis=reduce_axis)
+        total_count = math_ops.cast(
+            math_ops.count_nonzero(weights, axis=reduce_axis),
+            dtype=crossent.dtype)
+        crossent = math_ops.div_no_nan(crossent, total_count)
     return crossent
+
+
+class SequenceLoss(Loss):
+  """Weighted cross-entropy loss for a sequence of logits."""
+
+  def __init__(self,
+               average_across_timesteps=False,
+               average_across_batch=False,
+               sum_over_timesteps=True,
+               sum_over_batch=True,
+               softmax_loss_function=None,
+               name=None):
+    super(SequenceLoss, self).__init__(name=name)
+    self.average_across_timesteps = average_across_timesteps
+    self.average_across_batch = average_across_batch
+    self.sum_over_timesteps = sum_over_timesteps
+    self.sum_over_batch = sum_over_batch
+    self.softmax_loss_function = softmax_loss_function
+
+  def __call__(self, y_true, y_pred, sample_weight=None):
+    """Override the parent __call__ to have a customized reduce behavior."""
+    return sequence_loss(y_pred, y_true, sample_weight,
+                         average_across_timesteps=self.average_across_timesteps,
+                         average_across_batch=self.average_across_batch,
+                         sum_over_timesteps=self.sum_over_timesteps,
+                         sum_over_batch=self.sum_over_batch,
+                         softmax_loss_function=self.softmax_loss_function,
+                         name=self.name)
+
+  def call(self, y_true, y_pred):
+    # Skip this method since the __call__ contains real implementation.
+    pass
-- 
GitLab


From 7a830dab71dc530226936306b109fecead4366d6 Mon Sep 17 00:00:00 2001
From: Dheeraj Rajaram Reddy <dheeraj98reddy@gmail.com>
Date: Fri, 4 Jan 2019 22:42:38 +0530
Subject: [PATCH 0225/1095] Add ShortCircuit optimization for take_while and
 corresponding test

---
 .../core/kernels/data/experimental/BUILD      |  1 +
 .../experimental/take_while_dataset_op.cc     | 80 +++++++++++++------
 .../kernel_tests/take_while_test.py           | 16 ++++
 3 files changed, 73 insertions(+), 24 deletions(-)

diff --git a/tensorflow/core/kernels/data/experimental/BUILD b/tensorflow/core/kernels/data/experimental/BUILD
index 4ce14b0140..53d1f02a3a 100644
--- a/tensorflow/core/kernels/data/experimental/BUILD
+++ b/tensorflow/core/kernels/data/experimental/BUILD
@@ -303,6 +303,7 @@ tf_kernel_library(
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core/kernels/data:captured_function",
+        "//tensorflow/core/kernels/data:dataset_utils"
     ],
 )
 
diff --git a/tensorflow/core/kernels/data/experimental/take_while_dataset_op.cc b/tensorflow/core/kernels/data/experimental/take_while_dataset_op.cc
index cf9c5f0cb6..b670e4efaa 100644
--- a/tensorflow/core/kernels/data/experimental/take_while_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/take_while_dataset_op.cc
@@ -20,6 +20,7 @@ limitations under the License.
 #include "tensorflow/core/framework/partial_tensor_shape.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/kernels/data/captured_function.h"
+#include "tensorflow/core/kernels/data/dataset_utils.h"
 
 namespace tensorflow {
 namespace data {
@@ -30,6 +31,10 @@ namespace {
 
 class TakeWhileDatasetOp : public UnaryDatasetOpKernel {
  public:
+  using LoopIteratorPredicate =
+      std::function<Status(IteratorContext*, InstantiatedCapturedFunction*,
+                           std::vector<Tensor>, bool*)>;
+
   explicit TakeWhileDatasetOp(OpKernelConstruction* ctx)
       : UnaryDatasetOpKernel(ctx) {
     OP_REQUIRES_OK(ctx, ctx->GetAttr("predicate", &func_));
@@ -41,8 +46,45 @@ class TakeWhileDatasetOp : public UnaryDatasetOpKernel {
     OP_REQUIRES_OK(ctx, CapturedFunction::Create(func_, ctx, "other_arguments",
                                                  &captured_func));
 
-    // TODO (squadrick): check short-circuit
-    *output = new Dataset(ctx, input, func_, std::move(captured_func));
+    std::vector<int> indices;
+    OP_REQUIRES_OK(ctx, ComputeShortCircuitIndices(ctx, func_, &indices));
+    OP_REQUIRES(
+        ctx, indices.size() <= 1,
+        errors::InvalidArgument("`predicate` has more than one return value."));
+
+    LoopIteratorPredicate loop_pred;
+    if (indices.empty()) {
+      loop_pred = [](IteratorContext* ctx,
+                     InstantiatedCapturedFunction* inst_captured_func,
+                     const std::vector<Tensor>& args, bool* end_of_sequence) {
+        std::vector<Tensor> result;
+        TF_RETURN_IF_ERROR(
+            inst_captured_func->RunWithBorrowedArgs(ctx, args, &result));
+
+        if (result.size() != 1 || result[0].dtype() != DT_BOOL ||
+            result[0].NumElements() != 1) {
+          return errors::InvalidArgument(
+              "`predicate` must returns a scalar bool tensor.");
+        }
+        *end_of_sequence = !result[0].scalar<bool>()();
+        return Status::OK();
+      };
+    } else {
+      loop_pred = [indices](IteratorContext* ctx,
+                            InstantiatedCapturedFunction* inst_captured_func,
+                            const std::vector<Tensor>& args,
+                            bool* end_of_sequence) {
+        const Tensor& predicate = args[indices[0]];
+        if (predicate.dtype() != DT_BOOL || predicate.NumElements() != 1) {
+          return errors::InvalidArgument(
+              "`predicate` must returns a scalar bool tensor.");
+        }
+        *end_of_sequence = !predicate.scalar<bool>()();
+        return Status::OK();
+      };
+    }
+    *output = new Dataset(ctx, input, func_, std::move(captured_func),
+                          std::move(loop_pred));
   }
 
  private:
@@ -50,11 +92,13 @@ class TakeWhileDatasetOp : public UnaryDatasetOpKernel {
    public:
     Dataset(OpKernelContext* ctx, const DatasetBase* input,
             const NameAttrList& func,
-            std::unique_ptr<CapturedFunction> captured_func)
+            std::unique_ptr<CapturedFunction> captured_func,
+            LoopIteratorPredicate loop_pred)
         : DatasetBase(DatasetContext(ctx)),
           input_(input),
           func_(func),
-          captured_func_(std::move(captured_func)) {
+          captured_func_(std::move(captured_func)),
+          loop_pred_(std::move(loop_pred)) {
       input_->Ref();
     }
 
@@ -62,8 +106,8 @@ class TakeWhileDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::TakeWhile")}));
+      return std::unique_ptr<IteratorBase>(new Iterator(
+          {this, strings::StrCat(prefix, "::TakeWhile")}, loop_pred_));
     }
 
     const DataTypeVector& output_dtypes() const override {
@@ -122,8 +166,8 @@ class TakeWhileDatasetOp : public UnaryDatasetOpKernel {
    private:
     class Iterator : public DatasetIterator<Dataset> {
      public:
-      explicit Iterator(const Params& params)
-          : DatasetIterator<Dataset>(params) {}
+      explicit Iterator(const Params& params, LoopIteratorPredicate loop_pred)
+          : DatasetIterator<Dataset>(params), loop_pred_(loop_pred) {}
 
       Status Initialize(IteratorContext* ctx) override {
         TF_RETURN_IF_ERROR(
@@ -149,22 +193,8 @@ class TakeWhileDatasetOp : public UnaryDatasetOpKernel {
           input_impl_.reset();
           return Status::OK();
         }
-
-        std::vector<Tensor> bool_output;
-
-        Status s = instantiated_captured_func_->RunWithBorrowedArgs(
-            ctx, *out_tensors, &bool_output);
-
-        if (s.ok()) {
-          if (bool_output.size() != 1 || bool_output[0].dtype() != DT_BOOL ||
-              bool_output[0].NumElements() != 1) {
-            return errors::InvalidArgument(
-                "`predicate` must returns a scalar bool tensor.");
-          }
-          *end_of_sequence = !bool_output[0].scalar<bool>()();
-          return Status::OK();
-        }
-        return s;  // propagate error to caller
+        return loop_pred_(ctx, instantiated_captured_func_.get(), *out_tensors,
+                          end_of_sequence);
       }
 
      protected:
@@ -198,11 +228,13 @@ class TakeWhileDatasetOp : public UnaryDatasetOpKernel {
       mutex mu_;
       std::unique_ptr<IteratorBase> input_impl_ GUARDED_BY(mu_);
       std::unique_ptr<InstantiatedCapturedFunction> instantiated_captured_func_;
+      const LoopIteratorPredicate loop_pred_;
     };
 
     const DatasetBase* const input_;
     const NameAttrList func_;
     const std::unique_ptr<CapturedFunction> captured_func_;
+    const LoopIteratorPredicate loop_pred_;
   };
 
   NameAttrList func_;
diff --git a/tensorflow/python/data/experimental/kernel_tests/take_while_test.py b/tensorflow/python/data/experimental/kernel_tests/take_while_test.py
index b1e7974471..3faee0639f 100644
--- a/tensorflow/python/data/experimental/kernel_tests/take_while_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/take_while_test.py
@@ -83,6 +83,22 @@ class TakeWhileTest(test_base.DatasetTestBase, parameterized.TestCase):
     with self.assertRaises(errors.OutOfRangeError):
       self.assertEqual(b"test", self.evaluate(next_element()))
 
+  def testTakewhileDatasetShortCircuit(self):
+    def _predicate_func(data_elem):
+      return data_elem
+
+    boolean_array = [True, True, True, True, False]
+    dataset = dataset_ops.Dataset.from_tensor_slices(boolean_array).apply(
+        take_while_ops.take_while(_predicate_func))
+
+    next_element = self.getNext(dataset)
+    self.assertEqual(True, self.evaluate(next_element()))
+    self.assertEqual(True, self.evaluate(next_element()))
+    self.assertEqual(True, self.evaluate(next_element()))
+    self.assertEqual(True, self.evaluate(next_element()))
+
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(next_element())
 
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From 533d43705aa018d3bd07dc448ad63625776be63c Mon Sep 17 00:00:00 2001
From: "Joshua V. Dillon" <jvdillon@google.com>
Date: Fri, 4 Jan 2019 09:13:05 -0800
Subject: [PATCH 0226/1095] Remove tf.contrib.distributions seq2seq dependecy.

PiperOrigin-RevId: 227865533
---
 tensorflow/contrib/seq2seq/BUILD              |  2 -
 .../python/kernel_tests/basic_decoder_test.py |  6 +-
 .../contrib/seq2seq/python/ops/helper.py      | 86 ++++++++++++++++---
 3 files changed, 76 insertions(+), 18 deletions(-)

diff --git a/tensorflow/contrib/seq2seq/BUILD b/tensorflow/contrib/seq2seq/BUILD
index 18b56cd219..89176180ae 100644
--- a/tensorflow/contrib/seq2seq/BUILD
+++ b/tensorflow/contrib/seq2seq/BUILD
@@ -33,7 +33,6 @@ tf_custom_op_py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":beam_search_ops",
-        "//tensorflow/contrib/distributions:distributions_py",
         "//tensorflow/contrib/layers:layers_py",
         "//tensorflow/contrib/rnn:rnn_py",
         "//tensorflow/contrib/util:util_py",
@@ -59,7 +58,6 @@ tf_custom_op_py_library(
         "//tensorflow/python:tensor_util",
         "//tensorflow/python:util",
         "//tensorflow/python:variable_scope",
-        "//tensorflow/python/ops/distributions",
         "//third_party/py/numpy",
         "@six_archive//:six",
     ],
diff --git a/tensorflow/contrib/seq2seq/python/kernel_tests/basic_decoder_test.py b/tensorflow/contrib/seq2seq/python/kernel_tests/basic_decoder_test.py
index b7f9f3fb09..abcf71c61b 100644
--- a/tensorflow/contrib/seq2seq/python/kernel_tests/basic_decoder_test.py
+++ b/tensorflow/contrib/seq2seq/python/kernel_tests/basic_decoder_test.py
@@ -34,8 +34,6 @@ from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import rnn_cell
 from tensorflow.python.ops import variables
 from tensorflow.python.ops import variable_scope
-from tensorflow.python.ops.distributions import bernoulli
-from tensorflow.python.ops.distributions import categorical
 from tensorflow.python.platform import test
 # pylint: enable=g-import-not-at-top
 
@@ -517,7 +515,7 @@ class BasicDecoderTest(test.TestCase):
         vocabulary_size)
 
     # The sample function samples categorically from the logits.
-    sample_fn = lambda x: categorical.Categorical(logits=x).sample()
+    sample_fn = lambda x: helper_py.categorical_sample(logits=x)
     # The next inputs are a one-hot encoding of the sampled labels.
     next_inputs_fn = (
         lambda x: array_ops.one_hot(x, vocabulary_size, dtype=dtypes.float32))
@@ -599,7 +597,7 @@ class BasicDecoderTest(test.TestCase):
 
     # The sample function samples independent bernoullis from the logits.
     sample_fn = (
-        lambda x: bernoulli.Bernoulli(logits=x, dtype=dtypes.bool).sample())
+        lambda x: helper_py.bernoulli_sample(logits=x, dtype=dtypes.bool))
     # The next inputs are a one-hot encoding of the sampled labels.
     next_inputs_fn = math_ops.to_float
     end_fn = lambda sample_ids: sample_ids[:, end_token]
diff --git a/tensorflow/contrib/seq2seq/python/ops/helper.py b/tensorflow/contrib/seq2seq/python/ops/helper.py
index 3245cc5e72..033c2eb080 100644
--- a/tensorflow/contrib/seq2seq/python/ops/helper.py
+++ b/tensorflow/contrib/seq2seq/python/ops/helper.py
@@ -32,9 +32,8 @@ from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import embedding_ops
 from tensorflow.python.ops import gen_array_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import tensor_array_ops
-from tensorflow.python.ops.distributions import bernoulli
-from tensorflow.python.ops.distributions import categorical
 from tensorflow.python.util import nest
 
 __all__ = [
@@ -51,6 +50,68 @@ __all__ = [
 _transpose_batch_time = decoder._transpose_batch_time  # pylint: disable=protected-access
 
 
+# The following sample functions (_call_sampler, bernoulli_sample,
+# categorical_sample) mimic TensorFlow Probability distribution semantics.
+
+
+def _call_sampler(sample_n_fn, sample_shape, name=None):
+  """Reshapes vector of samples."""
+  with ops.name_scope(name, "call_sampler", values=[sample_shape]):
+    sample_shape = ops.convert_to_tensor(
+        sample_shape, dtype=dtypes.int32, name="sample_shape")
+    # Ensure sample_shape is a vector (vs just a scalar).
+    pad = math_ops.cast(math_ops.equal(array_ops.rank(sample_shape), 0),
+                        dtypes.int32)
+    sample_shape = array_ops.reshape(
+        sample_shape,
+        array_ops.pad(array_ops.shape(sample_shape),
+                      paddings=[[pad, 0]],
+                      constant_values=1))
+    samples = sample_n_fn(math_ops.reduce_prod(sample_shape))
+    batch_event_shape = array_ops.shape(samples)[1:]
+    final_shape = array_ops.concat([sample_shape, batch_event_shape], 0)
+    return array_ops.reshape(samples, final_shape)
+
+
+def bernoulli_sample(probs=None, logits=None, dtype=dtypes.int32,
+                     sample_shape=(), seed=None):
+  """Samples from Bernoulli distribution."""
+  if probs is None:
+    probs = math_ops.sigmoid(logits, name="probs")
+  else:
+    probs = ops.convert_to_tensor(probs, name="probs")
+  batch_shape_tensor = array_ops.shape(probs)
+  def _sample_n(n):
+    """Sample vector of Bernoullis."""
+    new_shape = array_ops.concat([[n], batch_shape_tensor], 0)
+    uniform = random_ops.random_uniform(
+        new_shape, seed=seed, dtype=probs.dtype)
+    return math_ops.cast(math_ops.less(uniform, probs), dtype)
+  return _call_sampler(_sample_n, sample_shape)
+
+
+def categorical_sample(logits, dtype=dtypes.int32,
+                       sample_shape=(), seed=None):
+  """Samples from categorical distribution."""
+  logits = ops.convert_to_tensor(logits, name="logits")
+  event_size = array_ops.shape(logits)[-1]
+  batch_shape_tensor = array_ops.shape(logits)[:-1]
+  def _sample_n(n):
+    """Sample vector of categoricals."""
+    if logits.shape.ndims == 2:
+      logits_2d = logits
+    else:
+      logits_2d = array_ops.reshape(logits, [-1, event_size])
+    sample_dtype = dtypes.int64 if logits.dtype.size > 4 else dtypes.int32
+    draws = random_ops.multinomial(
+        logits_2d, n, seed=seed, output_dtype=sample_dtype)
+    draws = array_ops.reshape(
+        array_ops.transpose(draws),
+        array_ops.concat([[n], batch_shape_tensor], 0))
+    return math_ops.cast(draws, dtype)
+  return _call_sampler(_sample_n, sample_shape)
+
+
 def _unstack_ta(inp):
   return tensor_array_ops.TensorArray(
       dtype=inp.dtype, size=array_ops.shape(inp)[0],
@@ -307,14 +368,14 @@ class ScheduledEmbeddingTrainingHelper(TrainingHelper):
     with ops.name_scope(name, "ScheduledEmbeddingTrainingHelperSample",
                         [time, outputs, state]):
       # Return -1s where we did not sample, and sample_ids elsewhere
-      select_sampler = bernoulli.Bernoulli(
-          probs=self._sampling_probability, dtype=dtypes.bool)
-      select_sample = select_sampler.sample(
-          sample_shape=self.batch_size, seed=self._scheduling_seed)
-      sample_id_sampler = categorical.Categorical(logits=outputs)
+      select_sample = bernoulli_sample(
+          probs=self._sampling_probability,
+          dtype=dtypes.bool,
+          sample_shape=self.batch_size,
+          seed=self._scheduling_seed)
       return array_ops.where(
           select_sample,
-          sample_id_sampler.sample(seed=self._seed),
+          categorical_sample(logits=outputs, seed=self._seed),
           gen_array_ops.fill([self.batch_size], -1))
 
   def next_inputs(self, time, outputs, state, sample_ids, name=None):
@@ -425,8 +486,10 @@ class ScheduledOutputTrainingHelper(TrainingHelper):
   def sample(self, time, outputs, state, name=None):
     with ops.name_scope(name, "ScheduledOutputTrainingHelperSample",
                         [time, outputs, state]):
-      sampler = bernoulli.Bernoulli(probs=self._sampling_probability)
-      return sampler.sample(sample_shape=self.batch_size, seed=self._seed)
+      return bernoulli_sample(
+          probs=self._sampling_probability,
+          sample_shape=self.batch_size,
+          seed=self._seed)
 
   def next_inputs(self, time, outputs, state, sample_ids, name=None):
     with ops.name_scope(name, "ScheduledOutputTrainingHelperNextInputs",
@@ -610,8 +673,7 @@ class SampleEmbeddingHelper(GreedyEmbeddingHelper):
     else:
       logits = outputs / self._softmax_temperature
 
-    sample_id_sampler = categorical.Categorical(logits=logits)
-    sample_ids = sample_id_sampler.sample(seed=self._seed)
+    sample_ids = categorical_sample(logits=logits, seed=self._seed)
 
     return sample_ids
 
-- 
GitLab


From 586c507c879c816eb001bfb32a14deb7053640a6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 4 Jan 2019 09:58:03 -0800
Subject: [PATCH 0227/1095] [tfgan] Adds GANEstimator wrapper that allows
 backpropagation to input latent space.

PiperOrigin-RevId: 227871387
---
 tensorflow/contrib/gan/BUILD                  |  36 +++
 .../contrib/gan/python/estimator/__init__.py  |   5 +-
 .../estimator/python/latent_gan_estimator.py  |  28 +++
 .../python/latent_gan_estimator_impl.py       | 205 ++++++++++++++++++
 .../python/latent_gan_estimator_test.py       | 119 ++++++++++
 5 files changed, 392 insertions(+), 1 deletion(-)
 create mode 100644 tensorflow/contrib/gan/python/estimator/python/latent_gan_estimator.py
 create mode 100644 tensorflow/contrib/gan/python/estimator/python/latent_gan_estimator_impl.py
 create mode 100644 tensorflow/contrib/gan/python/estimator/python/latent_gan_estimator_test.py

diff --git a/tensorflow/contrib/gan/BUILD b/tensorflow/contrib/gan/BUILD
index 0626875b76..08df38ad27 100644
--- a/tensorflow/contrib/gan/BUILD
+++ b/tensorflow/contrib/gan/BUILD
@@ -107,6 +107,7 @@ py_library(
     deps = [
         ":gan_estimator",
         ":head",
+        ":latent_gan_estimator",
         ":stargan_estimator",
         ":tpu_gan_estimator",
         "//tensorflow/python:util",
@@ -643,6 +644,41 @@ py_test(
     ],
 )
 
+py_library(
+    name = "latent_gan_estimator",
+    srcs = [
+        "python/estimator/python/latent_gan_estimator.py",
+        "python/estimator/python/latent_gan_estimator_impl.py",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":train",
+        "//tensorflow/python:clip_ops",
+        "//tensorflow/python:gradients",
+        "//tensorflow/python:random_ops",
+        "//tensorflow/python:summary",
+        "//tensorflow/python:training_util",
+        "//tensorflow/python:variable_scope",
+        "//tensorflow/python/estimator:estimator_py",
+    ],
+)
+
+py_test(
+    name = "latent_gan_estimator_test",
+    srcs = [
+        "python/estimator/python/latent_gan_estimator_test.py",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":latent_gan_estimator",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:training",
+        "//tensorflow/python:variable_scope",
+        "//tensorflow/python/estimator:run_config",
+        "//tensorflow/python/ops/losses",
+    ],
+)
+
 py_library(
     name = "sliced_wasserstein",
     srcs = [
diff --git a/tensorflow/contrib/gan/python/estimator/__init__.py b/tensorflow/contrib/gan/python/estimator/__init__.py
index 03a639165b..430266555b 100644
--- a/tensorflow/contrib/gan/python/estimator/__init__.py
+++ b/tensorflow/contrib/gan/python/estimator/__init__.py
@@ -26,11 +26,13 @@ from __future__ import print_function
 # pylint: disable=unused-import,wildcard-import
 from tensorflow.contrib.gan.python.estimator.python import gan_estimator
 from tensorflow.contrib.gan.python.estimator.python import head
+from tensorflow.contrib.gan.python.estimator.python import latent_gan_estimator
 from tensorflow.contrib.gan.python.estimator.python import stargan_estimator
 from tensorflow.contrib.gan.python.estimator.python import tpu_gan_estimator
 
 from tensorflow.contrib.gan.python.estimator.python.gan_estimator import *
 from tensorflow.contrib.gan.python.estimator.python.head import *
+from tensorflow.contrib.gan.python.estimator.python.latent_gan_estimator import *
 from tensorflow.contrib.gan.python.estimator.python.stargan_estimator import *
 from tensorflow.contrib.gan.python.estimator.python.tpu_gan_estimator import *
 # pylint: enable=unused-import,wildcard-import
@@ -41,7 +43,8 @@ _allowed_symbols = ([
     'gan_estimator',
     'stargan_estimator',
     'tpu_gan_estimator',
+    'latent_gan_estimator',
     'head',
 ] + gan_estimator.__all__ + stargan_estimator.__all__ + head.__all__ +
-                    tpu_gan_estimator.__all__)
+                    tpu_gan_estimator.__all__ + latent_gan_estimator.__all__)
 remove_undocumented(__name__, _allowed_symbols)
diff --git a/tensorflow/contrib/gan/python/estimator/python/latent_gan_estimator.py b/tensorflow/contrib/gan/python/estimator/python/latent_gan_estimator.py
new file mode 100644
index 0000000000..4e164e2416
--- /dev/null
+++ b/tensorflow/contrib/gan/python/estimator/python/latent_gan_estimator.py
@@ -0,0 +1,28 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""`tf.Learn` components for `Train Input Estimator`."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.contrib.gan.python.estimator.python import latent_gan_estimator_impl
+# pylint: disable=wildcard-import
+from tensorflow.contrib.gan.python.estimator.python.latent_gan_estimator_impl import *
+# pylint: enable=wildcard-import
+from tensorflow.python.util.all_util import remove_undocumented
+
+__all__ = latent_gan_estimator_impl.__all__
+remove_undocumented(__name__, __all__)
diff --git a/tensorflow/contrib/gan/python/estimator/python/latent_gan_estimator_impl.py b/tensorflow/contrib/gan/python/estimator/python/latent_gan_estimator_impl.py
new file mode 100644
index 0000000000..f5afc77319
--- /dev/null
+++ b/tensorflow/contrib/gan/python/estimator/python/latent_gan_estimator_impl.py
@@ -0,0 +1,205 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Implements an estimator wrapper that allows training the input latent space.
+
+This file implements a latent gan estimator that wraps around a previously
+trained GAN. The latent gan estimator trains a single variable z, representing
+the hidden latent distribution that is the 'noise' input to the GAN. By training
+z, the inpainting estimator can move around the latent z space towards
+minimizing a specific loss function.
+
+The latent gan estimator has a few key differences from a normal estimator.
+
+First: the variables in the estimator should not be saved, as we are not
+updating the original GAN and are only adding a new z variable that is meant
+to be different for each run. In order to do distributed training using
+train_and_evaluate, the Tensorflow RunConfig is expected to save checkpoints
+by having either save_checkpoints_steps or save_checkpoints_secs saved.
+To avoid this conflict, we purposely set the save_checkpoints_steps value in
+the RunConfig to be one step more than the total number of steps that the
+inpainter estimator will run.
+
+Second: we need to specify warm start settings, as we are reloading the
+GAN model into a different graph (specifically, one with a new z variable).
+The warm start settings defined below reload all GAN variables and ignore the
+new z variable (and the optimizer).
+
+Usage:
+
+  def _generator(net, mode):
+    ...
+
+  def _discriminator(net, condition, mode):
+    ...
+
+  def _loss(gan_model, features, labels, add_summaries):
+    ...
+
+  def optimizer():
+    ...
+
+  params = {<required params>}
+  config = tf.estimator.RunConfig()
+  tmp_dir = path/to/output/storage
+
+  estimator = latent_gan_estimator.get_latent_gan_estimator(
+      _generator, _discriminator, _loss, optimizer, params, config, tmp_dir)
+
+  def input_fn():
+    ...
+
+  estimator.train(input_fn=input_fn)
+
+See latent_gan_estimator_test.py or tensorflow_models/gan/face_inpainting for
+further examples.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import functools
+from tensorflow.contrib.gan.python import train as tfgan_train
+from tensorflow.python.estimator import estimator
+from tensorflow.python.estimator import model_fn as model_fn_lib
+from tensorflow.python.ops import clip_ops
+from tensorflow.python.ops import gradients_impl
+from tensorflow.python.ops import random_ops
+from tensorflow.python.ops import variable_scope
+from tensorflow.python.summary import summary
+from tensorflow.python.training import training_util
+
+
+INPUT_NAME = 'new_var_z_input'  # The name for the new z space input variable.
+OPTIMIZER_NAME = 'latent_gan_optimizer'  # The name for the new optimizer vars.
+
+__all__ = [
+    'get_latent_gan_estimator',
+]
+
+
+def _get_latent_gan_model_fn(generator_fn, discriminator_fn, loss_fn,
+                             optimizer):
+  """Sets up a model function that wraps around a given GAN."""
+  def model_fn(features, labels, mode, params):
+    """Model function defining an inpainting estimator."""
+    batch_size = params['batch_size']
+    z_shape = [batch_size] + params['z_shape']
+    add_summaries = params['add_summaries']
+    input_clip = params['input_clip']
+
+    z = variable_scope.get_variable(
+        name=INPUT_NAME, initializer=random_ops.truncated_normal(z_shape),
+        constraint=lambda x: clip_ops.clip_by_value(x, -input_clip, input_clip))
+
+    generator = functools.partial(generator_fn, mode=mode)
+    discriminator = functools.partial(discriminator_fn, mode=mode)
+    gan_model = tfgan_train.gan_model(generator_fn=generator,
+                                      discriminator_fn=discriminator,
+                                      real_data=labels,
+                                      generator_inputs=z,
+                                      check_shapes=False)
+
+    loss = loss_fn(gan_model, features, labels, add_summaries)
+
+    # Use a variable scope to make sure that estimator variables dont cause
+    # save/load problems when restoring from ckpts.
+    with variable_scope.variable_scope(OPTIMIZER_NAME):
+      opt = optimizer(learning_rate=params['learning_rate'],
+                      **params['opt_kwargs'])
+      train_op = opt.minimize(
+          loss=loss, global_step=training_util.get_or_create_global_step(),
+          var_list=[z])
+
+    if add_summaries:
+      z_grads = gradients_impl.gradients(loss, z)
+      summary.scalar('z_loss/z_grads', clip_ops.global_norm(z_grads))
+      summary.scalar('z_loss/loss', loss)
+
+    return model_fn_lib.EstimatorSpec(mode=mode,
+                                      predictions=gan_model.generated_data,
+                                      loss=loss,
+                                      train_op=train_op)
+  return model_fn
+
+
+def get_latent_gan_estimator(generator_fn, discriminator_fn, loss_fn,
+                             optimizer, params, config, ckpt_dir,
+                             warmstart_options=True):
+  """Gets an estimator that passes gradients to the input.
+
+  This function takes in a generator and adds a trainable z variable that is
+  used as input to this generator_fn. The generator itself is treated as a black
+  box through which gradients can pass through without updating any weights. The
+  result is a trainable way to traverse the GAN latent space. The loss_fn is
+  used to actually train the z variable. The generator_fn and discriminator_fn
+  should be previously trained by the tfgan library (on reload, the variables
+  are expected to follow the tfgan format. It may be possible to use the
+  latent gan estimator with entirely custom GANs that do not use the tfgan
+  library as long as the appropriate variables are wired properly).
+
+  Args:
+    generator_fn: a function defining a Tensorflow graph for a GAN generator.
+      The weights defined in this graph should already be defined in the given
+      checkpoint location. Should have 'mode' as an argument.
+    discriminator_fn: a function defining a Tensorflow graph for a GAN
+      discriminator. Should have 'mode' as an argument.
+    loss_fn: a function defining a Tensorflow graph for a GAN loss. Takes in a
+      GANModel tuple, features, labels, and add_summaries as inputs.
+    optimizer: a tf.Optimizer or a function that returns a tf.Optimizer with no
+      inputs.
+   params: An object containing the following parameters:
+      - batch_size: an int indicating the size of the training batch.
+      - z_shape: the desired shape of the input z values (not counting batch).
+      - learning_rate: a scalar or function defining a learning rate applied to
+        optimizer.
+      - input_clip: the amount to clip the x training variable by.
+      - add_summaries: whether or not to add summaries.
+      - opt_kwargs: optimizer kwargs.
+    config: tf.RunConfig. Should point model to output dir and should indicate
+     whether to save checkpoints (to avoid saving checkpoints, set
+     save_checkpoints_steps to a number larger than the number of train steps).
+     The model_dir field in the RunConfig should point to a directory WITHOUT
+     any saved checkpoints.
+    ckpt_dir: the directory where the model checkpoints live. The checkpoint is
+     used to warm start the underlying GAN. This should NOT be the same as
+     config.model_dir.
+    warmstart_options: boolean, None, or a WarmStartSettings object. If set to
+      True, uses a default WarmStartSettings object. If set to False or None,
+      does not use warm start. If using a custom WarmStartSettings object, make
+      sure that new variables are properly accounted for when reloading the
+      underlying GAN. Defaults to True.
+  Returns:
+    An estimator spec defining a GAN input training estimator.
+  """
+  model_fn = _get_latent_gan_model_fn(generator_fn, discriminator_fn,
+                                      loss_fn, optimizer)
+
+  if isinstance(warmstart_options, estimator.WarmStartSettings):
+    ws = warmstart_options
+  elif warmstart_options:
+    # Default WarmStart loads all variable names except INPUT_NAME and
+    # OPTIMIZER_NAME.
+    var_regex = '^(?!.*(%s|%s).*)' % (INPUT_NAME, OPTIMIZER_NAME)
+    ws = estimator.WarmStartSettings(ckpt_to_initialize_from=ckpt_dir,
+                                     vars_to_warm_start=var_regex)
+  else:
+    ws = None
+
+  if 'opt_kwargs' not in params:
+    params['opt_kwargs'] = {}
+
+  return estimator.Estimator(model_fn=model_fn, config=config, params=params,
+                             warm_start_from=ws)
diff --git a/tensorflow/contrib/gan/python/estimator/python/latent_gan_estimator_test.py b/tensorflow/contrib/gan/python/estimator/python/latent_gan_estimator_test.py
new file mode 100644
index 0000000000..ac139e532e
--- /dev/null
+++ b/tensorflow/contrib/gan/python/estimator/python/latent_gan_estimator_test.py
@@ -0,0 +1,119 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for latent_gan_estimator.
+
+See g3.tp.tensorflow.contrib.gan.python.estimator.python.latent_gan_estimator.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import tempfile
+import numpy as np
+from tensorflow.contrib.gan.python.estimator.python import latent_gan_estimator
+from tensorflow.python.estimator import run_config as run_config
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import variable_scope
+from tensorflow.python.ops.losses import losses
+from tensorflow.python.platform import test
+from tensorflow.python.training import training
+
+
+class TrainInputEstimatorTest(test.TestCase):
+
+  def test_get_input_training_estimator(self):
+    """Integration test to make sure the input_training_estimator works."""
+
+    # Create dummy test input tensors.
+    true_features = np.reshape(np.random.uniform(size=100), (10, 10))
+    true_labels = np.reshape(np.random.uniform(size=100), (5, 20))
+    expected_z_output = [[1, -1], [-1, 1]]
+
+    # Fill out required parameters randomly, includes optimizer kwargs.
+    params = {
+        'batch_size': 2,
+        'z_shape': [2],
+        'learning_rate': 1.0,
+        'input_clip': 1.0,
+        'add_summaries': False,
+        'opt_kwargs': {
+            'beta1': 0.1
+        }
+    }
+
+    input_z_shape = [params['batch_size']] + params['z_shape']
+
+    # Create dummy model functions that represent an underlying GANEstimator and
+    # the input training wrapper. Make sure that everything is wired up
+    # correctly in the internals of each dummy function.
+    def _generator(net, mode):
+      """The generator function will get the newly created z variable."""
+      del mode
+      self.assertSequenceEqual(net.shape, input_z_shape)
+      gen_dummy_var = variable_scope.get_variable(
+          name='generator_dummy_variable',
+          initializer=array_ops.ones(input_z_shape))
+      return net * gen_dummy_var
+
+    def _discriminator(net, condition, mode):
+      """The discriminator function will get either the z variable or labels."""
+      del condition, mode
+      try:
+        self.assertSequenceEqual(net.shape, true_labels.shape)
+      except AssertionError:
+        self.assertSequenceEqual(net.shape, input_z_shape)
+      return net
+
+    def _loss(gan_model, features, labels, _):
+      """Make sure that features and labels are passed in from input."""
+      self.assertTrue(np.array_equal(features, true_features))
+      self.assertTrue(np.array_equal(labels, true_labels))
+      return losses.absolute_difference(expected_z_output,
+                                        gan_model.generated_data)
+
+    optimizer = training.AdamOptimizer
+
+    # We are not loading checkpoints, so set the corresponding directory to a
+    # dummy directories.
+    tmp_dir = tempfile.mkdtemp()
+    config = run_config.RunConfig(model_dir=tmp_dir,
+                                  save_summary_steps=None,
+                                  save_checkpoints_steps=1,
+                                  save_checkpoints_secs=None)
+
+    # Get the estimator. Disable warm start so that there is no attempted
+    # checkpoint reloading.
+    estimator = latent_gan_estimator.get_latent_gan_estimator(
+        _generator, _discriminator, _loss, optimizer, params, config, tmp_dir,
+        warmstart_options=None)
+
+    # Train for a few steps.
+    def dummy_input():
+      return true_features, true_labels
+    estimator.train(input_fn=dummy_input, steps=10)
+
+    # Make sure the generator variables did not change, but the z variables did
+    # change.
+    self.assertTrue(np.array_equal(
+        estimator.get_variable_value('Generator/generator_dummy_variable'),
+        np.ones(input_z_shape)))
+    self.assertTrue(np.array_equal(
+        estimator.get_variable_value('new_var_z_input'),
+        expected_z_output))
+
+
+if __name__ == '__main__':
+  test.main()
-- 
GitLab


From df01a441b477b20346c5cbf8248018798b75ff0f Mon Sep 17 00:00:00 2001
From: Sergei Lebedev <slebedev@google.com>
Date: Fri, 4 Jan 2019 10:04:36 -0800
Subject: [PATCH 0228/1095] Decoupled ResourceVariable from RefVariable

* RefVariable._try_guard_against_uninitialized_dependencies and related
  methods are now functions used by both variables subclasses
* Removed implicit chaining of ResourceVariable ->Tensor convertor with
  that of RefVariable. Previously, ResourceVariable._dense_var_to_tensor
  would return NotImplemented if called with a different dtype. This
  effectively delegated the conversion to RefVariable._TensorConversionFunction
  which allowed compatible but not exactly matching dtypes. The change
  makes this behavior explicit in ResourceVariable._dense_var_to_tensor.

PiperOrigin-RevId: 227872550
---
 .../feature_column/feature_column_v2_test.py  |   8 +-
 .../python/ops/resource_variable_ops.py       | 109 ++++--
 tensorflow/python/ops/variables.py            | 316 +++++++++---------
 3 files changed, 251 insertions(+), 182 deletions(-)

diff --git a/tensorflow/python/feature_column/feature_column_v2_test.py b/tensorflow/python/feature_column/feature_column_v2_test.py
index a247425369..253e4c322b 100644
--- a/tensorflow/python/feature_column/feature_column_v2_test.py
+++ b/tensorflow/python/feature_column/feature_column_v2_test.py
@@ -2015,7 +2015,7 @@ class LinearModelTest(test.TestCase):
       }
       model(features)
       for var in model.variables:
-        self.assertTrue(isinstance(var, variables_lib.RefVariable))
+        self.assertIsInstance(var, variables_lib.VariableV1)
       variable_names = [var.name for var in model.variables]
       self.assertItemsEqual([
           'linear_model/dense_feature_bucketized/weights:0',
@@ -4010,7 +4010,7 @@ class FunctionalInputLayerTest(test.TestCase):
       self.assertEqual(0, len(cols_to_vars[dense_feature_bucketized]))
       self.assertEqual(1, len(cols_to_vars[some_embedding_column]))
       self.assertIsInstance(cols_to_vars[some_embedding_column][0],
-                            variables_lib.Variable)
+                            variables_lib.VariableV1)
       self.assertAllEqual(cols_to_vars[some_embedding_column][0].shape, [5, 10])
 
   @test_util.run_deprecated_v1
@@ -6839,7 +6839,7 @@ class EmbeddingColumnTest(test.TestCase):
     self.assertItemsEqual(('dense_features/aaa_embedding/embedding_weights:0',),
                           tuple([v.name for v in global_vars]))
     for v in global_vars:
-      self.assertTrue(isinstance(v, variables_lib.RefVariable))
+      self.assertIsInstance(v, variables_lib.Variable)
     trainable_vars = ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES)
     self.assertItemsEqual(('dense_features/aaa_embedding/embedding_weights:0',),
                           tuple([v.name for v in trainable_vars]))
@@ -7732,7 +7732,7 @@ class SharedEmbeddingColumnTest(test.TestCase):
         ['aaa_bbb_shared_embedding:0', 'ccc_ddd_shared_embedding:0'],
         tuple([v.name for v in global_vars]))
     for v in global_vars:
-      self.assertTrue(isinstance(v, variables_lib.RefVariable))
+      self.assertIsInstance(v, variables_lib.Variable)
     trainable_vars = ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES)
     if trainable:
       self.assertItemsEqual(
diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index 3e26463ddd..ede4bc0078 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -36,6 +36,7 @@ from tensorflow.python.ops import gen_array_ops
 from tensorflow.python.ops import gen_resource_variable_ops
 from tensorflow.python.ops import gen_state_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variables
 # go/tf-wildcard-import
 # pylint: disable=wildcard-import
@@ -160,8 +161,7 @@ def shape_safe_assign_variable_handle(handle, shape, value, name=None):
                                                       name=name)
 
 
-# TODO(apassos) make this be variables.Variable
-class ResourceVariable(variables.RefVariable):
+class ResourceVariable(variables.VariableV1):
   """Variable based on resource handles.
 
   See the [Variables How To](https://tensorflow.org/guide/variables)
@@ -297,6 +297,15 @@ class ResourceVariable(variables.RefVariable):
           dtype=dtype,
           constraint=constraint)
 
+  def __repr__(self):
+    if context.executing_eagerly() and not self._in_graph_mode:
+      return "<tf.Variable '%s' shape=%s dtype=%s, numpy=%s>" % (
+          self.name, self.get_shape(), self.dtype.name,
+          ops.numpy_text(self.read_value(), is_repr=True))
+    else:
+      return "<tf.Variable '%s' shape=%s dtype=%s>" % (
+          self.name, self.get_shape(), self.dtype.name)
+
   # pylint: disable=unused-argument
   def _init_from_args(self,
                       initial_value=None,
@@ -437,12 +446,15 @@ class ResourceVariable(variables.RefVariable):
                 gen_resource_variable_ops.var_is_initialized_op(self._handle))
           if initial_value is not None:
             with ops.name_scope("Assign") as n, ops.colocate_with(self._handle):
+              # pylint: disable=protected-access
               self._initializer_op = (
                   gen_resource_variable_ops.assign_variable_op(
                       self._handle,
-                      self._try_guard_against_uninitialized_dependencies(
+                      variables._try_guard_against_uninitialized_dependencies(
+                          name,
                           initial_value),
                       name=n))
+              # pylint: enable=protected-access
           with ops.name_scope("Read"), ops.colocate_with(self._handle):
             # Manually assign reads to the handle's device to avoid log
             # messages.
@@ -683,6 +695,10 @@ class ResourceVariable(variables.RefVariable):
     """The op for this variable."""
     return self._handle.op
 
+  @property
+  def trainable(self):
+    return self._trainable
+
   def eval(self, session=None):
     """Evaluates and returns the value of this variable."""
     if context.executing_eagerly():
@@ -818,10 +834,6 @@ class ResourceVariable(variables.RefVariable):
     return ResourceVariable(
         variable_def=variable_def, import_scope=import_scope)
 
-  def _ref(self):
-    """Unsupported."""
-    raise NotImplementedError("ResourceVariable does not implement _ref()")
-
   def set_shape(self, shape):
     """Unsupported."""
     raise NotImplementedError("ResourceVariable does not implement set_shape()")
@@ -994,6 +1006,55 @@ class ResourceVariable(variables.RefVariable):
         self.handle, sparse_delta.indices,
         ops.convert_to_tensor(sparse_delta.values, self.dtype), name=name))
 
+  def batch_scatter_update(self, sparse_delta, use_locking=False, name=None):
+    """Assigns `IndexedSlices` to this variable batch-wise.
+
+    Analogous to `batch_gather`. This assumes that this variable and the
+    sparse_delta IndexedSlices have a series of leading dimensions that are the
+    same for all of them, and the updates are performed on the last dimension of
+    indices. In other words, the dimensions should be the following:
+
+    `num_prefix_dims = sparse_delta.indices.ndims - 1`
+    `batch_dim = num_prefix_dims + 1`
+    `sparse_delta.updates.shape = sparse_delta.indices.shape + var.shape[
+         batch_dim:]`
+
+    where
+
+    `sparse_delta.updates.shape[:num_prefix_dims]`
+    `== sparse_delta.indices.shape[:num_prefix_dims]`
+    `== var.shape[:num_prefix_dims]`
+
+    And the operation performed can be expressed as:
+
+    `var[i_1, ..., i_n,
+         sparse_delta.indices[i_1, ..., i_n, j]] = sparse_delta.updates[
+            i_1, ..., i_n, j]`
+
+    When sparse_delta.indices is a 1D tensor, this operation is equivalent to
+    `scatter_update`.
+
+    To avoid this operation one can looping over the first `ndims` of the
+    variable and using `scatter_update` on the subtensors that result of slicing
+    the first dimension. This is a valid option for `ndims = 1`, but less
+    efficient than this implementation.
+
+    Args:
+      sparse_delta: `IndexedSlices` to be assigned to this variable.
+      use_locking: If `True`, use locking during the operation.
+      name: the name of the operation.
+
+    Returns:
+      A `Tensor` that will hold the new value of this variable after
+      the scattered subtraction has completed.
+
+    Raises:
+      ValueError: if `sparse_delta` is not an `IndexedSlices`.
+    """
+    return self._lazy_read(state_ops.batch_scatter_update(
+        self, sparse_delta.indices, sparse_delta.values,
+        use_locking=use_locking, name=name))
+
   def scatter_nd_sub(self, indices, updates, name=None):
     """Applies sparse subtraction to individual values or slices in a Variable.
 
@@ -1178,8 +1239,10 @@ class ResourceVariable(variables.RefVariable):
 
   def _dense_var_to_tensor(self, dtype=None, name=None, as_ref=False):
     del name
-    if dtype is not None and dtype != self.dtype:
-      return NotImplemented
+    if dtype is not None and not dtype.is_compatible_with(self.dtype):
+      raise ValueError(
+          "Incompatible type conversion requested to type {!r} for variable "
+          "of type {!r}".format(dtype.name, self.dtype.name))
     if as_ref:
       return self.read_value().op.inputs[0]
     else:
@@ -1231,6 +1294,12 @@ def _dense_var_to_tensor(var, dtype=None, name=None, as_ref=False):
   return var._dense_var_to_tensor(dtype=dtype, name=name, as_ref=as_ref)  # pylint: disable=protected-access
 
 
+# Register a conversion function which reads the value of the variable,
+# allowing instances of the class to be used as tensors.
+ops.register_tensor_conversion_function(ResourceVariable, _dense_var_to_tensor)
+ops.register_dense_tensor_like_type(ResourceVariable)
+
+
 class _UnreadVariable(ResourceVariable):
   """Represents a future for a read of a variable.
 
@@ -1291,7 +1360,7 @@ class _UnreadVariable(ResourceVariable):
     """The op for this variable."""
     return self._parent_op
 
-ops.register_tensor_conversion_function(_UnreadVariable, _dense_var_to_tensor)
+
 ops.register_dense_tensor_like_type(_UnreadVariable)
 
 
@@ -1390,29 +1459,15 @@ class _MixedPrecisionVariable(ResourceVariable):
 
   def _dense_var_to_tensor(self, dtype=None, name=None, as_ref=False):
     del name
-    dtype = dtype or self.read_dtype
-    if dtype != self.read_dtype or as_ref:
+    if (dtype is not None and
+        not dtype.is_compatible_with(self.read_dtype) or as_ref):
       return NotImplemented
-    else:
-      res = self.value()
-    return res
+    return self.value()
 
   def _should_act_as_resource_variable(self):
     """To pass resource_variable_ops.is_resource_variable check."""
     pass
 
-# Register a conversion function which reads the value of the variable,
-# allowing instances of the class to be used as tensors.
-
-# Note: registering for Variable after ResourceVariable because inheritance will
-# otherwise lead to the wrong behavior.
-ops.register_tensor_conversion_function(ResourceVariable, _dense_var_to_tensor)
-ops.register_tensor_conversion_function(
-    variables.Variable, variables.Variable._TensorConversionFunction)  # pylint: disable=protected-access
-
-# pylint: disable=protected-access
-ops.register_dense_tensor_like_type(ResourceVariable)
-
 
 @ops.RegisterGradient("ReadVariableOp")
 def _ReadGrad(_, grad):
diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py
index 657f64deaa..a4967e1c0f 100644
--- a/tensorflow/python/ops/variables.py
+++ b/tensorflow/python/ops/variables.py
@@ -59,21 +59,6 @@ def _make_getter(captured_getter, captured_previous):
   return getter
 
 
-def _has_cycle(op, path):
-  """Detect cycles in the dependencies of `initial_value`."""
-  if op.name in path:
-    return True
-  path.add(op.name)
-  for op_input in op.inputs:
-    if _has_cycle(op_input.op, path):
-      return True
-  for op_control_input in op.control_inputs:
-    if _has_cycle(op_control_input, path):
-      return True
-  path.remove(op.name)
-  return False
-
-
 @tf_export("VariableSynchronization")
 class VariableSynchronization(enum.Enum):
   """Indicates when a distributed variable will be synced.
@@ -1041,6 +1026,10 @@ class Variable(six.with_metaclass(VariableMetaclass,
     """Alias of `Variable.shape`."""
     return self.shape
 
+  def _gather_saveables_for_checkpoint(self):
+    """For implementing `Checkpointable`. This object is saveable on its own."""
+    return {checkpointable.VARIABLE_VALUE_KEY: self}
+
   def to_proto(self, export_scope=None):
     """Converts a `Variable` to a `VariableDef` protocol buffer.
 
@@ -1144,6 +1133,9 @@ class Variable(six.with_metaclass(VariableMetaclass,
         return None
 
 
+Variable._OverloadAllOperators()  # pylint: disable=protected-access
+
+
 @tf_export(v1=["Variable"])
 class VariableV1(Variable):
   """See the [Variables Guide](https://tensorflow.org/guide/variables).
@@ -1582,7 +1574,8 @@ class RefVariable(VariableV1):
         # using their initialized_value() method.
         self._initializer_op = state_ops.assign(
             self._variable,
-            self._try_guard_against_uninitialized_dependencies(
+            _try_guard_against_uninitialized_dependencies(
+                name,
                 self._initial_value),
             validate_shape=validate_shape).op
 
@@ -2161,134 +2154,6 @@ class RefVariable(VariableV1):
     else:
       return v.value()
 
-  def _gather_saveables_for_checkpoint(self):
-    """For implementing `Checkpointable`. This object is saveable on its own."""
-    return {checkpointable.VARIABLE_VALUE_KEY: self}
-
-  def _try_guard_against_uninitialized_dependencies(self, initial_value):
-    """Attempt to guard against dependencies on uninitialized variables.
-
-    Replace references to variables in `initial_value` with references to the
-    variable's initialized values. The initialized values are essentially
-    conditional TensorFlow graphs that return a variable's value if it is
-    initialized or its `initial_value` if it hasn't been initialized. This
-    replacement is done on a best effort basis:
-
-    - If the `initial_value` graph contains cycles, we don't do any
-      replacements for that graph.
-    - If the variables that `initial_value` depends on are not present in the
-      `GLOBAL_VARIABLES` or `LOCAL_VARIABLES` we don't replace them.
-
-    In these cases, it is up to the caller to ensure that the `initial_value`
-    graph uses initialized variables or that they guard access to variables
-    using their `initialized_value` method.
-
-    Args:
-      initial_value: `Tensor`. The initial value.
-    Returns:
-      A `Tensor` suitable to initialize a variable.
-    Raises:
-      TypeError: If `initial_value` is not a `Tensor`.
-    """
-    if not isinstance(initial_value, ops.Tensor):
-      raise TypeError("initial_value needs to be a Tensor: %s" % initial_value)
-
-    # Don't modify initial_value if it contains any cyclic dependencies.
-    if _has_cycle(initial_value.op, path=set()):
-      return initial_value
-
-    return self._safe_initial_value_from_tensor(initial_value, op_cache={})
-
-  def _safe_initial_value_from_tensor(self, tensor, op_cache):
-    """Replace dependencies on variables with their initialized values.
-
-    Args:
-      tensor: A `Tensor`. The tensor to replace.
-      op_cache: A dict mapping operation names to `Operation`s. Used to memoize
-        the results so as to avoid creating redundant operations.
-    Returns:
-      A `Tensor` compatible with `tensor`. Any inputs that lead to variable
-      values will be replaced with a corresponding graph that uses the
-      variable's initialized values. This is done on a best-effort basis. If no
-      modifications need to be made then `tensor` will be returned unchanged.
-    """
-    op = tensor.op
-    new_op = op_cache.get(op.name)
-    if new_op is None:
-      new_op = self._safe_initial_value_from_op(op, op_cache)
-      op_cache[op.name] = new_op
-    return new_op.outputs[tensor.value_index]
-
-  def _safe_initial_value_from_op(self, op, op_cache):
-    """Replace dependencies on variables with their initialized values.
-
-    Args:
-      op: An `Operation`. The operation to replace.
-      op_cache: A dict mapping operation names to `Operation`s. Used to memoize
-        the results so as to avoid creating redundant operations.
-    Returns:
-      An `Operation` compatible with `op`. Any inputs that lead to variable
-      values will be replaced with a corresponding graph that uses the
-      variable's initialized values. This is done on a best-effort basis. If no
-      modifications need to be made then `op` will be returned unchanged.
-    """
-    op_type = op.node_def.op
-    if op_type in ("IsVariableInitialized", "VarIsInitializedOp",
-                   "ReadVariableOp"):
-      return op
-
-    # Attempt to find the initialized_value of any variable reference / handles.
-    # TODO(b/70206927): Fix handling of ResourceVariables.
-    if op_type in ("Variable", "VariableV2", "VarHandleOp"):
-      initialized_value = self._find_initialized_value_for_variable(op)
-      return op if initialized_value is None else initialized_value.op
-
-    # Recursively build initializer expressions for inputs.
-    modified = False
-    new_op_inputs = []
-    for op_input in op.inputs:
-      new_op_input = self._safe_initial_value_from_tensor(op_input, op_cache)
-      new_op_inputs.append(new_op_input)
-      modified = modified or (new_op_input != op_input)
-
-    # If at least one input was modified, replace the op.
-    if modified:
-      new_op_type = op_type
-      if new_op_type == "RefSwitch":
-        new_op_type = "Switch"
-      new_op_name = op.node_def.name + "_" + self.name
-      new_op_name = new_op_name.replace(":", "_")
-      return self.graph.create_op(
-          new_op_type, new_op_inputs,
-          op._output_types,  # pylint: disable=protected-access
-          name=new_op_name, attrs=op.node_def.attr)
-
-    return op
-
-  def _find_initialized_value_for_variable(self, variable_op):
-    """Find the initialized value for a variable op.
-
-    To do so, lookup the variable op in the variables collection.
-
-    Args:
-      variable_op: A variable `Operation`.
-    Returns:
-      A `Tensor` representing the initialized value for the variable or `None`
-      if the initialized value could not be found.
-    """
-    try:
-      var_names = [variable_op.node_def.name, variable_op.node_def.name + ":0"]
-      for collection_name in (ops.GraphKeys.GLOBAL_VARIABLES,
-                              ops.GraphKeys.LOCAL_VARIABLES):
-        for var in self.graph.get_collection(collection_name):
-          if var.name in var_names:
-            return var.initialized_value()
-    except AttributeError:
-      # Return None when an incomplete user-defined variable type was put in
-      # the collection.
-      return None
-    return None
-
   # NOTE(mrry): This enables the Variable's overloaded "right" binary
   # operators to run when the left operand is an ndarray, because it
   # accords the Variable class higher priority than an ndarray, or a
@@ -2441,6 +2306,151 @@ class RefVariable(VariableV1):
     return self._save_slice_info
 
 
+def _try_guard_against_uninitialized_dependencies(name, initial_value):
+  """Attempt to guard against dependencies on uninitialized variables.
+
+  Replace references to variables in `initial_value` with references to the
+  variable's initialized values. The initialized values are essentially
+  conditional TensorFlow graphs that return a variable's value if it is
+  initialized or its `initial_value` if it hasn't been initialized. This
+  replacement is done on a best effort basis:
+
+  - If the `initial_value` graph contains cycles, we don't do any
+    replacements for that graph.
+  - If the variables that `initial_value` depends on are not present in the
+    `GLOBAL_VARIABLES` or `LOCAL_VARIABLES` we don't replace them.
+
+  In these cases, it is up to the caller to ensure that the `initial_value`
+  graph uses initialized variables or that they guard access to variables
+  using their `initialized_value` method.
+
+  Args:
+    name: Variable name.
+    initial_value: `Tensor`. The initial value.
+  Returns:
+    A `Tensor` suitable to initialize a variable.
+  Raises:
+    TypeError: If `initial_value` is not a `Tensor`.
+  """
+  if not isinstance(initial_value, ops.Tensor):
+    raise TypeError("initial_value needs to be a Tensor: %s" % initial_value)
+
+  # Don't modify initial_value if it contains any cyclic dependencies.
+  if _has_cycle(initial_value.op, path=set()):
+    return initial_value
+  return _safe_initial_value_from_tensor(name, initial_value, op_cache={})
+
+
+def _has_cycle(op, path):
+  """Detect cycles in the dependencies of `initial_value`."""
+  if op.name in path:
+    return True
+  path.add(op.name)
+  for op_input in op.inputs:
+    if _has_cycle(op_input.op, path):
+      return True
+  for op_control_input in op.control_inputs:
+    if _has_cycle(op_control_input, path):
+      return True
+  path.remove(op.name)
+  return False
+
+
+def _safe_initial_value_from_tensor(name, tensor, op_cache):
+  """Replace dependencies on variables with their initialized values.
+
+  Args:
+    name: Variable name.
+    tensor: A `Tensor`. The tensor to replace.
+    op_cache: A dict mapping operation names to `Operation`s. Used to memoize
+      the results so as to avoid creating redundant operations.
+  Returns:
+    A `Tensor` compatible with `tensor`. Any inputs that lead to variable
+    values will be replaced with a corresponding graph that uses the
+    variable's initialized values. This is done on a best-effort basis. If no
+    modifications need to be made then `tensor` will be returned unchanged.
+  """
+  op = tensor.op
+  new_op = op_cache.get(op.name)
+  if new_op is None:
+    new_op = _safe_initial_value_from_op(name, op, op_cache)
+    op_cache[op.name] = new_op
+  return new_op.outputs[tensor.value_index]
+
+
+def _safe_initial_value_from_op(name, op, op_cache):
+  """Replace dependencies on variables with their initialized values.
+
+  Args:
+    name: Variable name.
+    op: An `Operation`. The operation to replace.
+    op_cache: A dict mapping operation names to `Operation`s. Used to memoize
+      the results so as to avoid creating redundant operations.
+  Returns:
+    An `Operation` compatible with `op`. Any inputs that lead to variable
+    values will be replaced with a corresponding graph that uses the
+    variable's initialized values. This is done on a best-effort basis. If no
+    modifications need to be made then `op` will be returned unchanged.
+  """
+  op_type = op.node_def.op
+  if op_type in ("IsVariableInitialized", "VarIsInitializedOp",
+                 "ReadVariableOp"):
+    return op
+
+  # Attempt to find the initialized_value of any variable reference / handles.
+  # TODO(b/70206927): Fix handling of ResourceVariables.
+  if op_type in ("Variable", "VariableV2", "VarHandleOp"):
+    initialized_value = _find_initialized_value_for_variable(op)
+    return op if initialized_value is None else initialized_value.op
+
+  # Recursively build initializer expressions for inputs.
+  modified = False
+  new_op_inputs = []
+  for op_input in op.inputs:
+    new_op_input = _safe_initial_value_from_tensor(name, op_input, op_cache)
+    new_op_inputs.append(new_op_input)
+    modified = modified or (new_op_input != op_input)
+
+  # If at least one input was modified, replace the op.
+  if modified:
+    new_op_type = op_type
+    if new_op_type == "RefSwitch":
+      new_op_type = "Switch"
+    new_op_name = op.node_def.name + "_" + name
+    new_op_name = new_op_name.replace(":", "_")
+    return op.graph.create_op(
+        new_op_type, new_op_inputs,
+        op._output_types,  # pylint: disable=protected-access
+        name=new_op_name, attrs=op.node_def.attr)
+
+  return op
+
+
+def _find_initialized_value_for_variable(variable_op):
+  """Find the initialized value for a variable op.
+
+  To do so, lookup the variable op in the variables collection.
+
+  Args:
+    variable_op: A variable `Operation`.
+  Returns:
+    A `Tensor` representing the initialized value for the variable or `None`
+    if the initialized value could not be found.
+  """
+  try:
+    var_names = [variable_op.node_def.name, variable_op.node_def.name + ":0"]
+    for collection_name in (ops.GraphKeys.GLOBAL_VARIABLES,
+                            ops.GraphKeys.LOCAL_VARIABLES):
+      for var in variable_op.graph.get_collection(collection_name):
+        if var.name in var_names:
+          return var.initialized_value()
+  except AttributeError:
+    # Return None when an incomplete user-defined variable type was put in
+    # the collection.
+    return None
+  return None
+
+
 class PartitionedVariable(object):
   """A container for partitioned `Variable` objects.
 
@@ -2659,6 +2669,15 @@ class PartitionedVariable(object):
       return assign_list
     return [assign.op for assign in assign_list]
 
+
+# Register a conversion function which reads the value of the variable,
+# allowing instances of the class to be used as tensors.
+ops.register_tensor_conversion_function(
+    RefVariable,
+    RefVariable._TensorConversionFunction)  # pylint: disable=protected-access
+ops.register_dense_tensor_like_type(RefVariable)
+
+
 @tf_export(v1=["global_variables"])
 def global_variables(scope=None):
   """Returns global variables.
@@ -2982,12 +3001,7 @@ def report_uninitialized_variables(var_list=None,
         # uninitialized variables.
         return array_ops.boolean_mask(variable_names_tensor, variables_mask)
 
-# pylint: disable=protected-access
-Variable._OverloadAllOperators()
 
 ops.register_tensor_conversion_function(
-    PartitionedVariable, PartitionedVariable._TensorConversionFunction)
-# pylint: enable=protected-access
-
-
-ops.register_dense_tensor_like_type(Variable)
+    PartitionedVariable,
+    PartitionedVariable._TensorConversionFunction)  # pylint: disable=protected-access
-- 
GitLab


From 2a3d6401a3f21e4c9d043d90da48444d6e24e057 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 4 Jan 2019 10:09:12 -0800
Subject: [PATCH 0229/1095] Remove NHWC format restriction while using
 batch_group_count for convolutions of depthwise backprop filters

PiperOrigin-RevId: 227873314
---
 tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc b/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc
index b0bc764030..6b049f653c 100644
--- a/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc
+++ b/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc
@@ -428,11 +428,8 @@ xla::StatusOr<xla::XlaOp> MakeXlaBackpropFilterConvOp(
   int n_dim = GetTensorBatchDimIndex(num_dims, attrs.data_format);
   int c_dim = GetTensorFeatureDimIndex(num_dims, attrs.data_format);
 
-  // The conversion logic below assumes that the data format is NHWC, so we also
-  // check that here.
   bool use_batch_group_count =
-      filter_tensor_shape.dim_size(num_dims - 1) == 1 && attrs.depthwise &&
-      attrs.data_format == FORMAT_NHWC;
+      filter_tensor_shape.dim_size(num_dims - 1) == 1 && attrs.depthwise;
 
   std::vector<std::pair<int64, int64>> padding(attrs.num_spatial_dims);
   std::vector<int64> rhs_dilation(attrs.num_spatial_dims);
-- 
GitLab


From 07816cc1e13c02d7ad579d1279a5fca883126c35 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 4 Jan 2019 10:09:49 -0800
Subject: [PATCH 0230/1095] Fix markdown rendering.

PiperOrigin-RevId: 227873426
---
 tensorflow/python/ops/variables.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py
index a4967e1c0f..f9fc72a6da 100644
--- a/tensorflow/python/ops/variables.py
+++ b/tensorflow/python/ops/variables.py
@@ -306,6 +306,7 @@ class Variable(six.with_metaclass(VariableMetaclass,
 
   Here replacing adding `use_resource=True` when constructing the variable will
   fix any nondeterminism issues:
+
   ```
   v = tf.Variable(True, use_resource=True)
   tf.cond(v, lambda: v.assign(False), my_false_fn)
-- 
GitLab


From daa75aff18fd42598d1fb68f13da13042e886c07 Mon Sep 17 00:00:00 2001
From: Saurabh Saxena <srbs@google.com>
Date: Fri, 4 Jan 2019 10:17:31 -0800
Subject: [PATCH 0231/1095] Allows TensorListScatter to scatter at
 non-contiguous indices to make it consistent with TensorArray.scatter.

PiperOrigin-RevId: 227874653
---
 tensorflow/core/kernels/list_kernels.h        | 31 ++++++++++----
 .../python/kernel_tests/list_ops_test.py      | 41 +++++++++++++++++++
 .../kernel_tests/tensor_array_ops_test.py     |  5 +--
 tensorflow/python/ops/list_ops.py             | 14 +++++--
 4 files changed, 77 insertions(+), 14 deletions(-)

diff --git a/tensorflow/core/kernels/list_kernels.h b/tensorflow/core/kernels/list_kernels.h
index 99b4f42084..559e14b03c 100644
--- a/tensorflow/core/kernels/list_kernels.h
+++ b/tensorflow/core/kernels/list_kernels.h
@@ -521,14 +521,31 @@ class TensorListScatter : public OpKernel {
                     "Specified a list with shape ", element_shape.DebugString(),
                     " from a tensor with shape ", output_shape.DebugString()));
     output_list.element_shape = element_shape;
-    output_list.tensors.reserve(indices.NumElements());
+
+    OP_REQUIRES(c, indices.NumElements() == input_tensor.shape().dim_size(0),
+                errors::InvalidArgument(
+                    "Invalid number of rows in input tensor. Expected: ",
+                    indices.NumElements(),
+                    " Actual: ", input_tensor.shape().dim_size(0)));
+
+    // Validate indices and resize output_list.tensors to fit the highest index.
+    {
+      size_t list_size = 0;
+      for (int index = 0; index < indices.NumElements(); ++index) {
+        const int i = indices.flat<int32>()(index);
+        OP_REQUIRES(c, i >= 0,
+                    errors::InvalidArgument(
+                        "Indices in TensorListScatter must all be positive."));
+        if (i >= list_size) {
+          list_size = i + 1;
+        }
+      }
+      output_list.tensors.resize(list_size, Tensor(DT_INVALID));
+    }
+
     for (int index = 0; index < indices.NumElements(); ++index) {
       const int i = indices.flat<int32>()(index);
-      OP_REQUIRES(c, i < input_tensor.shape().dim_size(0),
-                  errors::InvalidArgument(
-                      "Trying to scatter index ", i, " from tensor with ",
-                      input_tensor.shape().dim_size(0), " rows."));
-      Tensor tmp = input_tensor.Slice(i, i + 1);
+      Tensor tmp = input_tensor.Slice(index, index + 1);
       TensorShape tmp_shape = tmp.shape();
       tmp_shape.RemoveDim(0);
       OP_REQUIRES(c, tmp.CopyFrom(tmp, tmp_shape),
@@ -541,7 +558,7 @@ class TensorListScatter : public OpKernel {
       // many small ondes.
       aligned.flat<T>().device(c->eigen_device<Device>()) =
           tmp.unaligned_flat<T>();
-      output_list.tensors.push_back(aligned);
+      std::swap(output_list.tensors[i], aligned);
     }
     output_tensor->scalar<Variant>()() = std::move(output_list);
   }
diff --git a/tensorflow/python/kernel_tests/list_ops_test.py b/tensorflow/python/kernel_tests/list_ops_test.py
index 054ac36755..fc0e270667 100644
--- a/tensorflow/python/kernel_tests/list_ops_test.py
+++ b/tensorflow/python/kernel_tests/list_ops_test.py
@@ -290,6 +290,47 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       t = list_ops.tensor_list_gather(l, [], element_dtype=dtypes.float32)
       self.evaluate(t)
 
+  def testGatherGradWithNonContiguousIndices(self):
+    with backprop.GradientTape(persistent=True) as tape:
+      t = constant_op.constant([1.0, 2.0, 3.0])
+      l = list_ops.tensor_list_from_tensor(t, element_shape=[])
+      c = constant_op.constant(5.0)
+      tape.watch(c)
+      l = list_ops.tensor_list_set_item(l, 1, c)
+      t = list_ops.tensor_list_gather(l, [1], element_dtype=dtypes.float32)
+      self.assertAllEqual(self.evaluate(t), [5.0])
+      s = t[0] * t[0]
+    dt = tape.gradient(s, c)
+    self.assertAllEqual(self.evaluate(dt), 10.0)
+    dl = tape.gradient(t, l)
+    dl_length = list_ops.tensor_list_length(dl)
+    self.assertAllEqual(self.evaluate(dl_length), 3)
+
+  def testScatterOutputListSize(self):
+    c0 = constant_op.constant([1.0, 2.0])
+    l = list_ops.tensor_list_scatter(
+        c0, [1, 3], ops.convert_to_tensor([], dtype=dtypes.int32))
+    # TensorListScatter should return a list with size largest index + 1.
+    self.assertEqual(self.evaluate(list_ops.tensor_list_length(l)), 4)
+
+  def testScatterWithInvalidRowsInInputTensorFails(self):
+    c0 = constant_op.constant([1.0, 2.0])
+    with self.assertRaisesRegexp(
+        errors.InvalidArgumentError,
+        "Invalid number of rows in input tensor. Expected: 3 Actual: 2"):
+      l = list_ops.tensor_list_scatter(
+          c0, [1, 0, 2], ops.convert_to_tensor([], dtype=dtypes.int32))
+      self.evaluate(l)
+
+  def testScatterWithNegativeIndicesFails(self):
+    c0 = constant_op.constant([1.0, 2.0])
+    with self.assertRaisesRegexp(
+        errors.InvalidArgumentError,
+        "Indices in TensorListScatter must all be positive."):
+      l = list_ops.tensor_list_scatter(
+          c0, [-1, -2], ops.convert_to_tensor([], dtype=dtypes.int32))
+      self.evaluate(l)
+
   def testScatterGrad(self):
     with backprop.GradientTape() as tape:
       c0 = constant_op.constant([1.0, 2.0])
diff --git a/tensorflow/python/kernel_tests/tensor_array_ops_test.py b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
index 9fb5791048..303e9a006a 100644
--- a/tensorflow/python/kernel_tests/tensor_array_ops_test.py
+++ b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
@@ -1359,7 +1359,6 @@ class TensorArrayTest(test.TestCase):
   def testSkipEagerTensorArrayEvalEmptyWithDefault(self):
     self._testTensorArrayEvalEmptyWithDefault()
 
-  @test_util.disable_control_flow_v2("b/117943286")
   @test_util.run_v1_only("b/117943489")
   def testSkipEagerTensorArrayScatterReadAndGradients(self):
     with self.session(use_gpu=True) as session:
@@ -1387,8 +1386,8 @@ class TensorArrayTest(test.TestCase):
       self.assertAllEqual([10.0, -10.0], read_vals[1])
       self.assertAllEqual([[2.0, 3.0], [4.0, 5.0]], grad_vals[0])
 
-  @test_util.disable_control_flow_v2("b/117943286")
-  @test_util.run_v1_only("b/117943286")
+  @test_util.disable_control_flow_v2("b/118890905")
+  @test_util.run_v1_only("b/118890905")
   def testTensorArrayWriteGatherAndGradients(self):
     with self.session(use_gpu=True) as session:
       ta = tensor_array_ops.TensorArray(
diff --git a/tensorflow/python/ops/list_ops.py b/tensorflow/python/ops/list_ops.py
index 7bb9ca89e0..df928ea85d 100644
--- a/tensorflow/python/ops/list_ops.py
+++ b/tensorflow/python/ops/list_ops.py
@@ -200,10 +200,16 @@ def _TensorListResizeGrad(op, dlist):
 
 @ops.RegisterGradient("TensorListGather")
 def _TensorListGatherGrad(op, dtensor):
-  _, indices = op.inputs
-  return gen_list_ops.tensor_list_scatter(
-      tensor=dtensor, indices=indices,
-      element_shape=ops.convert_to_tensor(-1, dtype=dtypes.int32)), None
+  input_list, indices = op.inputs
+  dlist = gen_list_ops.tensor_list_scatter(
+      tensor=dtensor,
+      indices=indices,
+      element_shape=ops.convert_to_tensor(-1, dtype=dtypes.int32))
+  # TensorListScatter returns a list with size `max(indices) + 1`
+  # so we manually resize it to match the size of the input list.
+  input_list_size = gen_list_ops.tensor_list_length(input_list)
+  dlist = gen_list_ops.tensor_list_resize(dlist, input_list_size)
+  return dlist, None
 
 
 @ops.RegisterGradient("TensorListScatter")
-- 
GitLab


From d183818d4d5e85a8f469402d273f8f03fa21f4b6 Mon Sep 17 00:00:00 2001
From: Scott Zhu <scottzhu@google.com>
Date: Fri, 4 Jan 2019 10:24:02 -0800
Subject: [PATCH 0232/1095] Update the math formula for LogLoss to match the
 parameters.

PiperOrigin-RevId: 227875597
---
 tensorflow/python/keras/losses.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/keras/losses.py b/tensorflow/python/keras/losses.py
index e6db795bb8..6a428b28a9 100644
--- a/tensorflow/python/keras/losses.py
+++ b/tensorflow/python/keras/losses.py
@@ -503,7 +503,7 @@ class CategoricalHinge(Loss):
 class LogLoss(Loss):
   """Computes the log loss between `y_true` and `y_pred`.
 
-  logloss = -y log(p) - (1-y) log(1-p)
+  logloss = - y_true * log(y_pred) - (1 - y_true) * log(1 - y_pred)
 
   Usage:
 
-- 
GitLab


From 59d5fbcc1a5e677387aac2bbf919faddf0e43377 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Fri, 4 Jan 2019 10:36:25 -0800
Subject: [PATCH 0233/1095] Automated rollback of commit
 64596a68235709e7f3246d0b120a4146521a998b

PiperOrigin-RevId: 227877471
---
 tensorflow/lite/experimental/swift/BUILD      | 100 +++++
 tensorflow/lite/experimental/swift/LICENSE    | 202 ++++++++++
 tensorflow/lite/experimental/swift/README.md  |  54 +++
 .../swift/Sources/Interpreter.swift           | 265 ++++++++++++++
 .../swift/Sources/InterpreterError.swift      |  99 +++++
 .../swift/Sources/InterpreterOptions.swift    |  29 ++
 .../experimental/swift/Sources/Model.swift    |  40 ++
 .../Sources/QuantizationParameters.swift      |  38 ++
 .../experimental/swift/Sources/Tensor.swift   | 138 +++++++
 .../Configs/TensorFlowLite.tulsigen           |  57 +++
 .../project.tulsiconf                         |  14 +
 .../project.pbxproj                           | 345 ++++++++++++++++++
 .../TensorFlowLiteApp/AppDelegate.swift       |  24 ++
 .../Array+TensorFlowLite.swift                |  22 ++
 .../AppIcon.appiconset/Contents.json          |  98 +++++
 .../Assets.xcassets/Contents.json             |   6 +
 .../Base.lproj/LaunchScreen.storyboard        |  44 +++
 .../Base.lproj/Main.storyboard                |  95 +++++
 .../Data+TensorFlowLite.swift                 |  13 +
 .../TensorFlowLiteApp/Info.plist              |  46 +++
 .../TensorFlowLiteApp/ViewController.swift    | 299 +++++++++++++++
 .../swift/Tests/InterpreterOptionsTests.swift |  54 +++
 .../swift/Tests/InterpreterTests.swift        | 315 ++++++++++++++++
 .../experimental/swift/Tests/ModelTests.swift |  59 +++
 .../Tests/QuantizationParametersTests.swift   |  43 +++
 .../swift/Tests/TensorTests.swift             |  83 +++++
 .../tools/pip_package/pip_smoke_test.py       |  36 +-
 27 files changed, 2603 insertions(+), 15 deletions(-)
 create mode 100644 tensorflow/lite/experimental/swift/BUILD
 create mode 100644 tensorflow/lite/experimental/swift/LICENSE
 create mode 100644 tensorflow/lite/experimental/swift/README.md
 create mode 100644 tensorflow/lite/experimental/swift/Sources/Interpreter.swift
 create mode 100644 tensorflow/lite/experimental/swift/Sources/InterpreterError.swift
 create mode 100644 tensorflow/lite/experimental/swift/Sources/InterpreterOptions.swift
 create mode 100644 tensorflow/lite/experimental/swift/Sources/Model.swift
 create mode 100644 tensorflow/lite/experimental/swift/Sources/QuantizationParameters.swift
 create mode 100644 tensorflow/lite/experimental/swift/Sources/Tensor.swift
 create mode 100644 tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/Configs/TensorFlowLite.tulsigen
 create mode 100644 tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/project.tulsiconf
 create mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp.xcodeproj/project.pbxproj
 create mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/AppDelegate.swift
 create mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Array+TensorFlowLite.swift
 create mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/AppIcon.appiconset/Contents.json
 create mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/Contents.json
 create mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/LaunchScreen.storyboard
 create mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/Main.storyboard
 create mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Data+TensorFlowLite.swift
 create mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Info.plist
 create mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/ViewController.swift
 create mode 100644 tensorflow/lite/experimental/swift/Tests/InterpreterOptionsTests.swift
 create mode 100644 tensorflow/lite/experimental/swift/Tests/InterpreterTests.swift
 create mode 100644 tensorflow/lite/experimental/swift/Tests/ModelTests.swift
 create mode 100644 tensorflow/lite/experimental/swift/Tests/QuantizationParametersTests.swift
 create mode 100644 tensorflow/lite/experimental/swift/Tests/TensorTests.swift

diff --git a/tensorflow/lite/experimental/swift/BUILD b/tensorflow/lite/experimental/swift/BUILD
new file mode 100644
index 0000000000..3bd288a1e1
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/BUILD
@@ -0,0 +1,100 @@
+# TensorFlow Lite for Swift.
+
+package(default_visibility = ["//visibility:private"])
+
+licenses(["notice"])  # Apache 2.0
+
+exports_files(["LICENSE"])
+
+load("@build_bazel_rules_apple//apple:ios.bzl", "ios_application", "ios_unit_test")
+load("@build_bazel_rules_swift//swift:swift.bzl", "swift_library")
+
+MINIMUM_OS_VERSION = "9.0"
+
+SWIFT_COPTS = [
+    "-wmo",
+]
+
+swift_library(
+    name = "TensorFlowLite",
+    srcs = glob(["Sources/*.swift"]),
+    copts = SWIFT_COPTS,
+    module_name = "TensorFlowLite",
+    tags = ["manual"],
+    deps = [
+        "//tensorflow/lite/experimental/c:c_api",
+    ],
+)
+
+ios_unit_test(
+    name = "TensorFlowLiteTests",
+    size = "small",
+    minimum_os_version = MINIMUM_OS_VERSION,
+    tags = [
+        "manual",
+        # DISABLED: Following sanitizer tests are not supported by iOS test targets.
+        "noasan",
+        "nomsan",
+        "notsan",
+    ],
+    deps = [":TensorFlowLiteTestsLib"],
+)
+
+swift_library(
+    name = "TensorFlowLiteTestsLib",
+    testonly = 1,
+    srcs = glob(["Tests/*.swift"]),
+    copts = SWIFT_COPTS,
+    tags = ["manual"],
+    deps = [
+        ":TensorFlowLite",
+        ":TestResources",
+    ],
+)
+
+objc_library(
+    name = "TestResources",
+    resources = [
+        "//tensorflow/lite:testdata/add.bin",
+        "//tensorflow/lite:testdata/add_quantized.bin",
+        "//tensorflow/lite:testdata/multi_add.bin",
+    ],
+    tags = ["manual"],
+)
+
+ios_application(
+    name = "TensorFlowLiteApp",
+    app_icons = glob(["TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/AppIcon.appiconset/**"]),
+    bundle_id = "com.tensorflow.lite.swift.TensorFlowLite",
+    families = [
+        "ipad",
+        "iphone",
+    ],
+    infoplists = ["TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Info.plist"],
+    launch_storyboard = "TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/LaunchScreen.storyboard",
+    minimum_os_version = MINIMUM_OS_VERSION,
+    sdk_frameworks = [
+        "CoreGraphics",
+    ],
+    tags = ["manual"],
+    deps = [":TensorFlowLiteAppLib"],
+)
+
+swift_library(
+    name = "TensorFlowLiteAppLib",
+    srcs = glob(["TestApps/TensorFlowLiteApp/TensorFlowLiteApp/*.swift"]),
+    tags = ["manual"],
+    deps = [
+        ":TensorFlowLite",
+        ":TensorFlowLiteAppResources",
+    ],
+)
+
+objc_library(
+    name = "TensorFlowLiteAppResources",
+    storyboards = glob([
+        "TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/*.storyboard",
+    ]),
+    tags = ["manual"],
+    deps = [":TestResources"],
+)
diff --git a/tensorflow/lite/experimental/swift/LICENSE b/tensorflow/lite/experimental/swift/LICENSE
new file mode 100644
index 0000000000..d645695673
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/LICENSE
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/tensorflow/lite/experimental/swift/README.md b/tensorflow/lite/experimental/swift/README.md
new file mode 100644
index 0000000000..deb16c1e91
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/README.md
@@ -0,0 +1,54 @@
+# TensorFlow Lite for Swift
+
+[TensorFlow Lite](https://www.tensorflow.org/lite/) is TensorFlow's lightweight
+solution for Swift developers. It enables low-latency inference of on-device
+machine learning models with a small binary size and fast performance supporting
+hardware acceleration.
+
+## Getting Started
+
+### Bazel
+
+In your `BUILD` file, add the `TensorFlowLite` dependency:
+
+```python
+swift_library(
+  # ...
+  deps = [
+      "//tensorflow/lite/swift:TensorFlowLite",
+  ],
+)
+```
+
+In your Swift files, import the module:
+
+```swift
+import TensorFlowLite
+```
+
+### Tulsi
+
+Open the `TensorFlowLite.tulsiproj` using the [TulsiApp](https://github.com/bazelbuild/tulsi) or by
+running the [`generate_xcodeproj.sh`](https://github.com/bazelbuild/tulsi/blob/master/src/tools/generate_xcodeproj.sh)
+script:
+
+```shell
+generate_xcodeproj.sh --genconfig tensorflow/lite/swift/TensorFlowLite.tulsiproj:TensorFlowLite --outputfolder ~/path/to/generated/TensorFlowLite.xcodeproj
+```
+
+### CocoaPods
+
+Add the following to your `Podfile`:
+
+```ruby
+use_frameworks!
+pod 'TensorFlowLiteSwift'
+```
+
+Then, run `pod install`.
+
+In your Swift files, import the module:
+
+```swift
+import TensorFlowLite
+```
diff --git a/tensorflow/lite/experimental/swift/Sources/Interpreter.swift b/tensorflow/lite/experimental/swift/Sources/Interpreter.swift
new file mode 100644
index 0000000000..a14b5966b1
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/Sources/Interpreter.swift
@@ -0,0 +1,265 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Foundation
+import TensorFlowLiteCAPI
+
+/// A TensorFlow Lite interpreter that performs inference from a given model.
+public final class Interpreter {
+
+  /// The `TFL_Interpreter` C pointer type represented as an `UnsafePointer<TFL_Interpreter>`.
+  private typealias CInterpreter = OpaquePointer
+
+  /// Total number of input tensors associated with the model.
+  public var inputTensorCount: Int {
+    return Int(TFL_InterpreterGetInputTensorCount(cInterpreter))
+  }
+
+  /// Total number of output tensors associated with the model.
+  public var outputTensorCount: Int {
+    return Int(TFL_InterpreterGetOutputTensorCount(cInterpreter))
+  }
+
+  /// The underlying `TFL_Interpreter` C pointer.
+  private var cInterpreter: CInterpreter?
+
+  /// Creates a new model interpreter instance.
+  ///
+  /// - Parameters:
+  ///   - modelPath: Local file path to a TensorFlow Lite model.
+  ///   - options: Custom configurations for the interpreter. The default is `nil` indicating that
+  ///       interpreter will determine the configuration options.
+  /// - Throws: An error if the model could not be loaded or the interpreter could not be created.
+  public init(modelPath: String, options: InterpreterOptions? = nil) throws {
+    guard let model = Model(filePath: modelPath) else { throw InterpreterError.failedToLoadModel }
+
+    let cInterpreterOptions: OpaquePointer? = try options.map { options in
+      guard let cOptions = TFL_NewInterpreterOptions() else {
+        throw InterpreterError.failedToCreateInterpreter
+      }
+      if let threadCount = options.threadCount, threadCount > 0 {
+        TFL_InterpreterOptionsSetNumThreads(cOptions, Int32(threadCount))
+      }
+      if options.isErrorLoggingEnabled {
+        TFL_InterpreterOptionsSetErrorReporter(
+          cOptions,
+          { (_, format, arguments) in
+            guard let cFormat = format,
+                  let message = String(cFormat: cFormat, arguments: arguments)
+            else {
+              return
+            }
+            print(String(describing: InterpreterError.tensorFlowLiteError(message)))
+          },
+          nil
+        )
+      }
+      return cOptions
+    }
+    defer { TFL_DeleteInterpreterOptions(cInterpreterOptions) }
+
+    guard let cInterpreter = TFL_NewInterpreter(model.cModel, cInterpreterOptions) else {
+      throw InterpreterError.failedToCreateInterpreter
+    }
+    self.cInterpreter = cInterpreter
+  }
+
+  deinit {
+    TFL_DeleteInterpreter(cInterpreter)
+  }
+
+  /// Invokes the interpreter to perform inference from the loaded graph.
+  ///
+  /// - Throws: An error if the model was not ready because tensors were not allocated.
+  public func invoke() throws {
+    guard TFL_InterpreterInvoke(cInterpreter) == kTfLiteOk else {
+      // TODO(b/117510052): Determine which error to throw.
+      throw InterpreterError.allocateTensorsRequired
+    }
+  }
+
+  /// Returns the input tensor at the given index.
+  ///
+  /// - Parameters:
+  ///   - index: The index for the input tensor.
+  /// - Throws: An error if the index is invalid or the tensors have not been allocated.
+  /// - Returns: The input tensor at the given index.
+  public func input(at index: Int) throws -> Tensor {
+    let maxIndex = inputTensorCount - 1
+    guard case 0...maxIndex = index else {
+      throw InterpreterError.invalidTensorIndex(index: index, maxIndex: maxIndex)
+    }
+    guard let cTensor = TFL_InterpreterGetInputTensor(cInterpreter, Int32(index)),
+          let bytes = TFL_TensorData(cTensor),
+          let nameCString = TFL_TensorName(cTensor)
+    else {
+      throw InterpreterError.allocateTensorsRequired
+    }
+    guard let dataType = TensorDataType(type: TFL_TensorType(cTensor)) else {
+      throw InterpreterError.invalidTensorDataType
+    }
+
+    let name = String(cString: nameCString)
+    let rank = TFL_TensorNumDims(cTensor)
+    let dimensions = (0..<rank).map { Int(TFL_TensorDim(cTensor, $0)) }
+    let shape = TensorShape(dimensions)
+    let byteCount = TFL_TensorByteSize(cTensor)
+    let data = Data(bytes: bytes, count: byteCount)
+    let cQuantizationParams = TFL_TensorQuantizationParams(cTensor)
+    let scale = cQuantizationParams.scale
+    let zeroPoint = Int(cQuantizationParams.zero_point)
+    var quantizationParameters: QuantizationParameters? = nil
+    if scale != 0.0 {
+      // TODO(b/117510052): Update this check once the TfLiteQuantizationParams struct has a mode.
+      quantizationParameters = QuantizationParameters(scale: scale, zeroPoint: zeroPoint)
+    }
+    let tensor = Tensor(
+      name: name,
+      dataType: dataType,
+      shape: shape,
+      data: data,
+      quantizationParameters: quantizationParameters
+    )
+    return tensor
+  }
+
+  /// Returns the output tensor at the given index.
+  ///
+  /// - Parameters:
+  ///   - index: The index for the output tensor.
+  /// - Throws: An error if the index is invalid, tensors haven't been allocated, or interpreter
+  ///     hasn't been invoked for models that dynamically compute output tensors based on the values
+  ///     of its input tensors.
+  /// - Returns: The output tensor at the given index.
+  public func output(at index: Int) throws -> Tensor {
+    let maxIndex = outputTensorCount - 1
+    guard case 0...maxIndex = index else {
+      throw InterpreterError.invalidTensorIndex(index: index, maxIndex: maxIndex)
+    }
+    guard let cTensor = TFL_InterpreterGetOutputTensor(cInterpreter, Int32(index)),
+          let bytes = TFL_TensorData(cTensor),
+          let nameCString = TFL_TensorName(cTensor)
+    else {
+      // TODO(b/117510052): Determine which error to throw.
+      throw InterpreterError.invokeInterpreterRequired
+    }
+    guard let dataType = TensorDataType(type: TFL_TensorType(cTensor)) else {
+      throw InterpreterError.invalidTensorDataType
+    }
+
+    let name = String(cString: nameCString)
+    let rank = TFL_TensorNumDims(cTensor)
+    let dimensions = (0..<rank).map { Int(TFL_TensorDim(cTensor, $0)) }
+    let shape = TensorShape(dimensions)
+    let byteCount = TFL_TensorByteSize(cTensor)
+    let data = Data(bytes: bytes, count: byteCount)
+    let cQuantizationParams = TFL_TensorQuantizationParams(cTensor)
+    let scale = cQuantizationParams.scale
+    let zeroPoint = Int(cQuantizationParams.zero_point)
+    var quantizationParameters: QuantizationParameters? = nil
+    if scale != 0.0 {
+      // TODO(b/117510052): Update this check once the TfLiteQuantizationParams struct has a mode.
+      quantizationParameters = QuantizationParameters(scale: scale, zeroPoint: zeroPoint)
+    }
+    let tensor = Tensor(
+      name: name,
+      dataType: dataType,
+      shape: shape,
+      data: data,
+      quantizationParameters: quantizationParameters
+    )
+    return tensor
+  }
+
+  /// Resizes the input tensor at the given index to the specified tensor shape.
+  ///
+  /// - Note: After resizing an input tensor, the client **must** explicitly call
+  ///     `allocateTensors()` before attempting to access the resized tensor data or invoking the
+  ///     interpreter to perform inference.
+  /// - Parameters:
+  ///   - index: The index for the input tensor.
+  ///   - shape: The shape that the input tensor should be resized to.
+  /// - Throws: An error if the input tensor at the given index could not be resized.
+  public func resizeInput(at index: Int, to shape: TensorShape) throws {
+    let maxIndex = inputTensorCount - 1
+    guard case 0...maxIndex = index else {
+      throw InterpreterError.invalidTensorIndex(index: index, maxIndex: maxIndex)
+    }
+    guard TFL_InterpreterResizeInputTensor(
+            cInterpreter,
+            Int32(index),
+            shape.int32Dimensions,
+            Int32(shape.rank)
+          ) == kTfLiteOk
+    else {
+      throw InterpreterError.failedToResizeInputTensor(index: index)
+    }
+  }
+
+  /// Copies the given data to the input tensor at the given index.
+  ///
+  /// - Parameters:
+  ///   - data: The data to be copied to the input tensor's data buffer.
+  ///   - index: The index for the input tensor.
+  /// - Throws: An error if the `data.count` does not match the input tensor's `data.count` or if
+  ///     the given index is invalid.
+  /// - Returns: The input tensor with the copied data.
+  @discardableResult
+  public func copy(_ data: Data, toInputAt index: Int) throws -> Tensor {
+    let maxIndex = inputTensorCount - 1
+    guard case 0...maxIndex = index else {
+      throw InterpreterError.invalidTensorIndex(index: index, maxIndex: maxIndex)
+    }
+    guard let cTensor = TFL_InterpreterGetInputTensor(cInterpreter, Int32(index)) else {
+      throw InterpreterError.allocateTensorsRequired
+    }
+
+    let byteCount = TFL_TensorByteSize(cTensor)
+    guard data.count == byteCount else {
+      throw InterpreterError.invalidTensorDataCount(provided: data.count, required: byteCount)
+    }
+
+    let status = data.withUnsafeBytes { TFL_TensorCopyFromBuffer(cTensor, $0, data.count) }
+    guard status == kTfLiteOk else { throw InterpreterError.failedToCopyDataToInputTensor }
+    return try input(at: index)
+  }
+
+  /// Allocates memory for all input tensors based on their `TensorShape`s.
+  ///
+  /// - Note: This is a relatively expensive operation and should only be called after creating the
+  ///     interpreter and/or resizing any input tensors.
+  /// - Throws: An error if memory could not be allocated for the input tensors.
+  public func allocateTensors() throws {
+    guard TFL_InterpreterAllocateTensors(cInterpreter) == kTfLiteOk else {
+      throw InterpreterError.failedToAllocateTensors
+    }
+  }
+}
+
+// MARK: - Extensions
+
+extension String {
+  /// Returns a new `String` initialized by using the given format C array as a template into which
+  /// the remaining argument values are substituted according to the user’s default locale.
+  ///
+  /// - Note: Returns `nil` if a new `String` could not be constructed from the given values.
+  /// - Parameters:
+  ///   - cFormat: The format C array as a template for substituting values.
+  ///   - arguments: A C pointer to a `va_list` of arguments to substitute into `cFormat`.
+  init?(cFormat: UnsafePointer<CChar>, arguments: CVaListPointer) {
+    var buffer: UnsafeMutablePointer<CChar>?
+    guard vasprintf(&buffer, cFormat, arguments) != 0, let cString = buffer else { return nil }
+    self.init(validatingUTF8: cString)
+  }
+}
diff --git a/tensorflow/lite/experimental/swift/Sources/InterpreterError.swift b/tensorflow/lite/experimental/swift/Sources/InterpreterError.swift
new file mode 100644
index 0000000000..5de58b997a
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/Sources/InterpreterError.swift
@@ -0,0 +1,99 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Foundation
+
+/// TensorFlow Lite interpreter errors.
+public enum InterpreterError: Error {
+  case invalidTensorIndex(index: Int, maxIndex: Int)
+  case invalidTensorDataCount(provided: Int, required: Int)
+  case invalidTensorDataType
+  case failedToLoadModel
+  case failedToCreateInterpreter
+  case failedToResizeInputTensor(index: Int)
+  case failedToCopyDataToInputTensor
+  case failedToAllocateTensors
+  case allocateTensorsRequired
+  case invokeInterpreterRequired
+  case tensorFlowLiteError(String)
+}
+
+// MARK: - Extensions
+
+extension InterpreterError: LocalizedError {
+  /// Localized description of the interpreter error.
+  public var errorDescription: String? {
+    switch self {
+    case .invalidTensorIndex(let index, let maxIndex):
+      return "Invalid tensor index \(index), max index is \(maxIndex)."
+    case .invalidTensorDataCount(let providedCount, let requiredCount):
+      return "Provided data count \(providedCount) must match the required count \(requiredCount)."
+    case .invalidTensorDataType:
+      return "Tensor data type is unsupported or could not be determined because of a model error."
+    case .failedToLoadModel:
+      return "Failed to load the given model."
+    case .failedToCreateInterpreter:
+      return "Failed to create the interpreter."
+    case .failedToResizeInputTensor(let index):
+      return "Failed to resize input tesnor at index \(index)."
+    case .failedToCopyDataToInputTensor:
+      return "Failed to copy data to input tensor."
+    case .failedToAllocateTensors:
+      return "Failed to allocate memory for input tensors."
+    case .allocateTensorsRequired:
+      return "Must call allocateTensors()."
+    case .invokeInterpreterRequired:
+      return "Must call invoke()."
+    case .tensorFlowLiteError(let message):
+      return "TensorFlow Lite Error: \(message)"
+    }
+  }
+}
+
+extension InterpreterError: CustomStringConvertible {
+  /// Textual representation of the TensorFlow Lite interpreter error.
+  public var description: String {
+    return errorDescription ?? "Unknown error."
+  }
+}
+
+#if swift(>=4.2)
+extension InterpreterError: Equatable {}
+#else
+extension InterpreterError: Equatable {
+  public static func == (lhs: InterpreterError, rhs: InterpreterError) -> Bool {
+    switch (lhs, rhs) {
+    case (.invalidTensorDataType, .invalidTensorDataType),
+         (.failedToLoadModel, .failedToLoadModel),
+         (.failedToCreateInterpreter, .failedToCreateInterpreter),
+         (.failedToAllocateTensors, .failedToAllocateTensors),
+         (.allocateTensorsRequired, .allocateTensorsRequired),
+         (.invokeInterpreterRequired, .invokeInterpreterRequired):
+      return true
+    case (.invalidTensorIndex(let lhsIndex, let lhsMaxIndex),
+          .invalidTensorIndex(let rhsIndex, let rhsMaxIndex)):
+      return lhsIndex == rhsIndex && lhsMaxIndex == rhsMaxIndex
+    case (.invalidTensorDataCount(let lhsProvidedCount, let lhsRequiredCount),
+          .invalidTensorDataCount(let rhsProvidedCount, let rhsRequiredCount)):
+      return lhsProvidedCount == rhsProvidedCount && lhsRequiredCount == rhsRequiredCount
+    case (.failedToResizeInputTensor(let lhsIndex), .failedToResizeInputTensor(let rhsIndex)):
+      return lhsIndex == rhsIndex
+    case (.tensorFlowLiteError(let lhsMessage), .tensorFlowLiteError(let rhsMessage)):
+      return lhsMessage == rhsMessage
+    default:
+      return false
+    }
+  }
+}
+#endif  // swift(>=4.2)
diff --git a/tensorflow/lite/experimental/swift/Sources/InterpreterOptions.swift b/tensorflow/lite/experimental/swift/Sources/InterpreterOptions.swift
new file mode 100644
index 0000000000..2365fd7ade
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/Sources/InterpreterOptions.swift
@@ -0,0 +1,29 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Foundation
+
+/// Custom configuration options for a TensorFlow Lite interpreter.
+public struct InterpreterOptions: Equatable {
+
+  /// Maximum number of CPU threads that the interpreter should run on. Default is `nil` which
+  /// indicates that the `Interpreter` will decide the number of threads to use.
+  public var threadCount: Int? = nil
+
+  /// Whether error logging to the console is enabled. The default is `false`.
+  public var isErrorLoggingEnabled = false
+
+  /// Creates a new instance of interpreter options.
+  public init() {}
+}
diff --git a/tensorflow/lite/experimental/swift/Sources/Model.swift b/tensorflow/lite/experimental/swift/Sources/Model.swift
new file mode 100644
index 0000000000..e8c49ff1ae
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/Sources/Model.swift
@@ -0,0 +1,40 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Foundation
+import TensorFlowLiteCAPI
+
+/// A TensorFlow Lite model used by the 'Interpreter` to perform inference.
+final class Model {
+
+  /// The `TFL_Model` C pointer type represented as an `UnsafePointer<TFL_Model>`.
+  typealias CModel = OpaquePointer
+
+  /// The underlying `TFL_Model` C pointer.
+  let cModel: CModel?
+
+  /// Creates a new model instance.
+  ///
+  /// - Precondition: Initialization can fail if the given `filePath` is invalid.
+  /// - Parameters:
+  ///   - filePath: Local file path to a TensorFlow Lite model.
+  init?(filePath: String) {
+    guard !filePath.isEmpty, let cModel = TFL_NewModelFromFile(filePath) else { return nil }
+    self.cModel = cModel
+  }
+
+  deinit {
+    TFL_DeleteModel(cModel)
+  }
+}
diff --git a/tensorflow/lite/experimental/swift/Sources/QuantizationParameters.swift b/tensorflow/lite/experimental/swift/Sources/QuantizationParameters.swift
new file mode 100644
index 0000000000..f367875644
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/Sources/QuantizationParameters.swift
@@ -0,0 +1,38 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Foundation
+
+/// Parameters that determine the mapping of quantized values to real values. Quantized values can
+/// be mapped to float values using the following conversion:
+/// `realValue = scale * (quantizedValue - zeroPoint)`.
+public struct QuantizationParameters {
+
+  /// Difference between real values corresponding to consecutive quantized values differing by 1.
+  /// For example, the range of quantized values for `UInt8` data type is [0, 255].
+  public let scale: Float
+
+  /// Quantized value that corresponds to the real 0 value.
+  public let zeroPoint: Int
+
+  /// Creates a new quantization parameters instance.
+  ///
+  /// - Parameters:
+  ///   - scale: Scale value for asymmetric quantization.
+  ///   - zeroPoint: Zero point for asymmetric quantization.
+  init(scale: Float, zeroPoint: Int) {
+    self.scale = scale
+    self.zeroPoint = zeroPoint
+  }
+}
diff --git a/tensorflow/lite/experimental/swift/Sources/Tensor.swift b/tensorflow/lite/experimental/swift/Sources/Tensor.swift
new file mode 100644
index 0000000000..b738d87549
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/Sources/Tensor.swift
@@ -0,0 +1,138 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Foundation
+import TensorFlowLiteCAPI
+
+/// An input or output tensor in a TensorFlow Lite graph.
+public struct Tensor {
+
+  /// Name of the tensor.
+  public let name: String
+
+  /// Data type of the tensor.
+  public let dataType: TensorDataType
+
+  /// Shape of the tensor.
+  public let shape: TensorShape
+
+  /// Data in the input or output tensor.
+  public let data: Data
+
+  /// Quantization parameters for the tensor if using a quantized model.
+  public let quantizationParameters: QuantizationParameters?
+
+  /// Creates a new input or output tensor instance.
+  ///
+  /// - Parameters:
+  ///   - name: Name of the tensor.
+  ///   - dataType: Data type of the tensor.
+  ///   - data: Data in the input tensor.
+  ///   - quantizationParameters Quantization parameters for the tensor if using a quantized model.
+  ///       The default is `nil`.
+  init(
+    name: String,
+    dataType: TensorDataType,
+    shape: TensorShape,
+    data: Data,
+    quantizationParameters: QuantizationParameters? = nil
+  ) {
+    self.name = name
+    self.dataType = dataType
+    self.shape = shape
+    self.data = data
+    self.quantizationParameters = quantizationParameters
+  }
+}
+
+/// Supported TensorFlow Lite tensor data types.
+public enum TensorDataType: Equatable {
+  /// 32-bit single precision floating point tensor data type.
+  case float32
+  /// 8-bit unsigned integer tensor data type.
+  case uInt8
+  /// 16-bit signed integer tensor data type.
+  case int16
+  /// 32-bit signed integer tensor data type.
+  case int32
+  /// 64-bit signed integer tensor data type.
+  case int64
+  /// Boolean tensor data type.
+  case bool
+
+  /// Creates a new tensor data type from the given `TFL_Type` or `nil` if the data type is
+  /// unsupported or could not be determined because there was an error.
+  ///
+  /// - Parameter type: A data type supported by a tensor.
+  init?(type: TFL_Type) {
+    switch type {
+    case kTfLiteFloat32:
+      self = .float32
+    case kTfLiteUInt8:
+      self = .uInt8
+    case kTfLiteInt16:
+      self = .int16
+    case kTfLiteInt32:
+      self = .int32
+    case kTfLiteInt64:
+      self = .int64
+    case kTfLiteBool:
+      self = .bool
+    case kTfLiteNoType:
+      fallthrough
+    default:
+      return nil
+    }
+  }
+}
+
+/// The shape of a TensorFlow Lite tensor.
+public struct TensorShape {
+
+  /// The number of dimensions of the tensor.
+  public let rank: Int
+
+  /// Array of dimensions for the tensor.
+  public let dimensions: [Int]
+
+  /// Array of `Int32` dimensions for the tensor.
+  var int32Dimensions: [Int32] { return dimensions.map(Int32.init) }
+
+  /// Creates a new tensor shape instance with the given array of dimensions.
+  ///
+  /// - Parameters:
+  ///   - dimensions: Dimensions for the tensor.
+  public init(_ dimensions: [Int]) {
+    self.rank = dimensions.count
+    self.dimensions = dimensions
+  }
+
+  /// Creates a new tensor shape instance with the given elements representing the dimensions.
+  ///
+  /// - Parameters:
+  ///   - elements: Dimensions for the tensor.
+  public init(_ elements: Int...) {
+    self.init(elements)
+  }
+}
+
+extension TensorShape: ExpressibleByArrayLiteral {
+  /// Creates a new tensor shape instance with the given array literal representing the dimensions.
+  ///
+  /// - Parameters:
+  ///   - arrayLiteral: Dimensions for the tensor.
+  public init(arrayLiteral: Int...) {
+    self.init(arrayLiteral)
+  }
+}
diff --git a/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/Configs/TensorFlowLite.tulsigen b/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/Configs/TensorFlowLite.tulsigen
new file mode 100644
index 0000000000..4010fab49e
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/Configs/TensorFlowLite.tulsigen
@@ -0,0 +1,57 @@
+{
+  "sourceFilters" : [
+    "third_party/tensorflow/lite/experimental/c",
+    "third_party/tensorflow/lite/experimental/swift",
+    "third_party/tensorflow/lite/experimental/swift/Sources",
+    "third_party/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp",
+    "third_party/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj",
+    "third_party/tensorflow/lite/experimental/swift/Tests",
+  ],
+  "buildTargets" : [
+    "//third_party/tensorflow/lite/experimental/swift:TensorFlowLite",
+    "//third_party/tensorflow/lite/experimental/swift:TensorFlowLiteApp",
+    "//third_party/tensorflow/lite/experimental/swift:TensorFlowLiteTests",
+  ],
+  "projectName" : "TensorFlowLite",
+  "optionSet" : {
+    "LaunchActionPreActionScript" : {
+      "p" : "$(inherited)"
+    },
+    "BazelBuildStartupOptionsRelease" : {
+      "p" : "$(inherited)"
+    },
+    "BazelBuildOptionsRelease" : {
+      "p" : "$(inherited)"
+    },
+    "BazelBuildOptionsDebug" : {
+      "p" : "$(inherited)"
+    },
+    "EnvironmentVariables" : {
+      "p" : "$(inherited)"
+    },
+    "BuildActionPreActionScript" : {
+      "p" : "$(inherited)"
+    },
+    "CommandlineArguments" : {
+      "p" : "$(inherited)"
+    },
+    "TestActionPreActionScript" : {
+      "p" : "$(inherited)"
+    },
+    "BazelBuildStartupOptionsDebug" : {
+      "p" : "$(inherited)"
+    },
+    "BuildActionPostActionScript" : {
+      "p" : "$(inherited)"
+    },
+    "TestActionPostActionScript" : {
+      "p" : "$(inherited)"
+    },
+    "LaunchActionPostActionScript" : {
+      "p" : "$(inherited)"
+    }
+  },
+  "additionalFilePaths" : [
+    "third_party/tensorflow/lite/experimental/swift/BUILD"
+  ]
+}
diff --git a/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/project.tulsiconf b/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/project.tulsiconf
new file mode 100644
index 0000000000..14cff94453
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/project.tulsiconf
@@ -0,0 +1,14 @@
+{
+  "configDefaults" : {
+    "optionSet" : {
+      "ProjectPrioritizesSwift" : {
+        "p" : "YES"
+      }
+    }
+  },
+  "projectName" : "TensorFlowLite",
+  "packages" : [
+    "third_party/tensorflow/lite/experimental/swift"
+  ],
+  "workspaceRoot" : "../../../../../.."
+}
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp.xcodeproj/project.pbxproj b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp.xcodeproj/project.pbxproj
new file mode 100644
index 0000000000..fbbf9a1de2
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp.xcodeproj/project.pbxproj
@@ -0,0 +1,345 @@
+// !$*UTF8*$!
+{
+	archiveVersion = 1;
+	classes = {
+	};
+	objectVersion = 50;
+	objects = {
+
+/* Begin PBXBuildFile section */
+		4A7304B421500B8400C90B21 /* Data+TensorFlowLite.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4A7304B321500B8300C90B21 /* Data+TensorFlowLite.swift */; };
+		4AA72B732146ED64006C3AEF /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4AA72B722146ED64006C3AEF /* AppDelegate.swift */; };
+		4AA72B752146ED64006C3AEF /* ViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4AA72B742146ED64006C3AEF /* ViewController.swift */; };
+		4AA72B782146ED64006C3AEF /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 4AA72B762146ED64006C3AEF /* Main.storyboard */; };
+		4AA72B7A2146ED66006C3AEF /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 4AA72B792146ED66006C3AEF /* Assets.xcassets */; };
+		4AA72B7D2146ED66006C3AEF /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 4AA72B7B2146ED66006C3AEF /* LaunchScreen.storyboard */; };
+		4ADDE0CE2176600E00FF07A2 /* Array+TensorFlowLite.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4ADDE0CD2176600900FF07A2 /* Array+TensorFlowLite.swift */; };
+/* End PBXBuildFile section */
+
+/* Begin PBXFileReference section */
+		4A7304B321500B8300C90B21 /* Data+TensorFlowLite.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "Data+TensorFlowLite.swift"; sourceTree = "<group>"; };
+		4AA72B6F2146ED64006C3AEF /* TensorFlowLiteApp.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = TensorFlowLiteApp.app; sourceTree = BUILT_PRODUCTS_DIR; };
+		4AA72B722146ED64006C3AEF /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = "<group>"; };
+		4AA72B742146ED64006C3AEF /* ViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ViewController.swift; sourceTree = "<group>"; };
+		4AA72B772146ED64006C3AEF /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/Main.storyboard; sourceTree = "<group>"; };
+		4AA72B792146ED66006C3AEF /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = "<group>"; };
+		4AA72B7C2146ED66006C3AEF /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = "<group>"; };
+		4AA72B7E2146ED66006C3AEF /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
+		4ADDE0CD2176600900FF07A2 /* Array+TensorFlowLite.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "Array+TensorFlowLite.swift"; sourceTree = "<group>"; };
+/* End PBXFileReference section */
+
+/* Begin PBXFrameworksBuildPhase section */
+		4AA72B6C2146ED64006C3AEF /* Frameworks */ = {
+			isa = PBXFrameworksBuildPhase;
+			buildActionMask = 2147483647;
+			files = (
+			);
+			runOnlyForDeploymentPostprocessing = 0;
+		};
+/* End PBXFrameworksBuildPhase section */
+
+/* Begin PBXGroup section */
+		4AA72B662146ED64006C3AEF = {
+			isa = PBXGroup;
+			children = (
+				4AA72B712146ED64006C3AEF /* TensorFlowLiteApp */,
+				4AA72B702146ED64006C3AEF /* Products */,
+			);
+			sourceTree = "<group>";
+		};
+		4AA72B702146ED64006C3AEF /* Products */ = {
+			isa = PBXGroup;
+			children = (
+				4AA72B6F2146ED64006C3AEF /* TensorFlowLiteApp.app */,
+			);
+			name = Products;
+			sourceTree = "<group>";
+		};
+		4AA72B712146ED64006C3AEF /* TensorFlowLiteApp */ = {
+			isa = PBXGroup;
+			children = (
+				4AA72B722146ED64006C3AEF /* AppDelegate.swift */,
+				4ADDE0CD2176600900FF07A2 /* Array+TensorFlowLite.swift */,
+				4A7304B321500B8300C90B21 /* Data+TensorFlowLite.swift */,
+				4AA72B742146ED64006C3AEF /* ViewController.swift */,
+				4AA72B762146ED64006C3AEF /* Main.storyboard */,
+				4AA72B792146ED66006C3AEF /* Assets.xcassets */,
+				4AA72B7B2146ED66006C3AEF /* LaunchScreen.storyboard */,
+				4AA72B7E2146ED66006C3AEF /* Info.plist */,
+			);
+			path = TensorFlowLiteApp;
+			sourceTree = "<group>";
+		};
+/* End PBXGroup section */
+
+/* Begin PBXNativeTarget section */
+		4AA72B6E2146ED64006C3AEF /* TensorFlowLiteApp */ = {
+			isa = PBXNativeTarget;
+			buildConfigurationList = 4AA72B812146ED66006C3AEF /* Build configuration list for PBXNativeTarget "TensorFlowLiteApp" */;
+			buildPhases = (
+				4AA72B6B2146ED64006C3AEF /* Sources */,
+				4AA72B6C2146ED64006C3AEF /* Frameworks */,
+				4AA72B6D2146ED64006C3AEF /* Resources */,
+			);
+			buildRules = (
+			);
+			dependencies = (
+			);
+			name = TensorFlowLiteApp;
+			productName = TensorFlowLiteApp;
+			productReference = 4AA72B6F2146ED64006C3AEF /* TensorFlowLiteApp.app */;
+			productType = "com.apple.product-type.application";
+		};
+/* End PBXNativeTarget section */
+
+/* Begin PBXProject section */
+		4AA72B672146ED64006C3AEF /* Project object */ = {
+			isa = PBXProject;
+			attributes = {
+				LastSwiftUpdateCheck = 0940;
+				LastUpgradeCheck = 0940;
+				ORGANIZATIONNAME = Google;
+				TargetAttributes = {
+					4AA72B6E2146ED64006C3AEF = {
+						CreatedOnToolsVersion = 9.4.1;
+					};
+				};
+			};
+			buildConfigurationList = 4AA72B6A2146ED64006C3AEF /* Build configuration list for PBXProject "TensorFlowLiteApp" */;
+			compatibilityVersion = "Xcode 9.3";
+			developmentRegion = en;
+			hasScannedForEncodings = 0;
+			knownRegions = (
+				en,
+				Base,
+			);
+			mainGroup = 4AA72B662146ED64006C3AEF;
+			productRefGroup = 4AA72B702146ED64006C3AEF /* Products */;
+			projectDirPath = "";
+			projectRoot = "";
+			targets = (
+				4AA72B6E2146ED64006C3AEF /* TensorFlowLiteApp */,
+			);
+		};
+/* End PBXProject section */
+
+/* Begin PBXResourcesBuildPhase section */
+		4AA72B6D2146ED64006C3AEF /* Resources */ = {
+			isa = PBXResourcesBuildPhase;
+			buildActionMask = 2147483647;
+			files = (
+				4AA72B7D2146ED66006C3AEF /* LaunchScreen.storyboard in Resources */,
+				4AA72B7A2146ED66006C3AEF /* Assets.xcassets in Resources */,
+				4AA72B782146ED64006C3AEF /* Main.storyboard in Resources */,
+			);
+			runOnlyForDeploymentPostprocessing = 0;
+		};
+/* End PBXResourcesBuildPhase section */
+
+/* Begin PBXSourcesBuildPhase section */
+		4AA72B6B2146ED64006C3AEF /* Sources */ = {
+			isa = PBXSourcesBuildPhase;
+			buildActionMask = 2147483647;
+			files = (
+				4AA72B732146ED64006C3AEF /* AppDelegate.swift in Sources */,
+				4ADDE0CE2176600E00FF07A2 /* Array+TensorFlowLite.swift in Sources */,
+				4A7304B421500B8400C90B21 /* Data+TensorFlowLite.swift in Sources */,
+				4AA72B752146ED64006C3AEF /* ViewController.swift in Sources */,
+			);
+			runOnlyForDeploymentPostprocessing = 0;
+		};
+/* End PBXSourcesBuildPhase section */
+
+/* Begin PBXVariantGroup section */
+		4AA72B762146ED64006C3AEF /* Main.storyboard */ = {
+			isa = PBXVariantGroup;
+			children = (
+				4AA72B772146ED64006C3AEF /* Base */,
+			);
+			name = Main.storyboard;
+			sourceTree = "<group>";
+		};
+		4AA72B7B2146ED66006C3AEF /* LaunchScreen.storyboard */ = {
+			isa = PBXVariantGroup;
+			children = (
+				4AA72B7C2146ED66006C3AEF /* Base */,
+			);
+			name = LaunchScreen.storyboard;
+			sourceTree = "<group>";
+		};
+/* End PBXVariantGroup section */
+
+/* Begin XCBuildConfiguration section */
+		4AA72B7F2146ED66006C3AEF /* Debug */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				ALWAYS_SEARCH_USER_PATHS = NO;
+				CLANG_ANALYZER_NONNULL = YES;
+				CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
+				CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
+				CLANG_CXX_LIBRARY = "libc++";
+				CLANG_ENABLE_MODULES = YES;
+				CLANG_ENABLE_OBJC_ARC = YES;
+				CLANG_ENABLE_OBJC_WEAK = YES;
+				CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
+				CLANG_WARN_BOOL_CONVERSION = YES;
+				CLANG_WARN_COMMA = YES;
+				CLANG_WARN_CONSTANT_CONVERSION = YES;
+				CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
+				CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
+				CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
+				CLANG_WARN_EMPTY_BODY = YES;
+				CLANG_WARN_ENUM_CONVERSION = YES;
+				CLANG_WARN_INFINITE_RECURSION = YES;
+				CLANG_WARN_INT_CONVERSION = YES;
+				CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
+				CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
+				CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
+				CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
+				CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
+				CLANG_WARN_STRICT_PROTOTYPES = YES;
+				CLANG_WARN_SUSPICIOUS_MOVE = YES;
+				CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
+				CLANG_WARN_UNREACHABLE_CODE = YES;
+				CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
+				CODE_SIGN_IDENTITY = "iPhone Developer";
+				COPY_PHASE_STRIP = NO;
+				DEBUG_INFORMATION_FORMAT = dwarf;
+				ENABLE_STRICT_OBJC_MSGSEND = YES;
+				ENABLE_TESTABILITY = YES;
+				GCC_C_LANGUAGE_STANDARD = gnu11;
+				GCC_DYNAMIC_NO_PIC = NO;
+				GCC_NO_COMMON_BLOCKS = YES;
+				GCC_OPTIMIZATION_LEVEL = 0;
+				GCC_PREPROCESSOR_DEFINITIONS = (
+					"DEBUG=1",
+					"$(inherited)",
+				);
+				GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
+				GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
+				GCC_WARN_UNDECLARED_SELECTOR = YES;
+				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
+				GCC_WARN_UNUSED_FUNCTION = YES;
+				GCC_WARN_UNUSED_VARIABLE = YES;
+				IPHONEOS_DEPLOYMENT_TARGET = 11.4;
+				MTL_ENABLE_DEBUG_INFO = YES;
+				ONLY_ACTIVE_ARCH = YES;
+				SDKROOT = iphoneos;
+				SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG;
+				SWIFT_OPTIMIZATION_LEVEL = "-Onone";
+			};
+			name = Debug;
+		};
+		4AA72B802146ED66006C3AEF /* Release */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				ALWAYS_SEARCH_USER_PATHS = NO;
+				CLANG_ANALYZER_NONNULL = YES;
+				CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
+				CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
+				CLANG_CXX_LIBRARY = "libc++";
+				CLANG_ENABLE_MODULES = YES;
+				CLANG_ENABLE_OBJC_ARC = YES;
+				CLANG_ENABLE_OBJC_WEAK = YES;
+				CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
+				CLANG_WARN_BOOL_CONVERSION = YES;
+				CLANG_WARN_COMMA = YES;
+				CLANG_WARN_CONSTANT_CONVERSION = YES;
+				CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
+				CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
+				CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
+				CLANG_WARN_EMPTY_BODY = YES;
+				CLANG_WARN_ENUM_CONVERSION = YES;
+				CLANG_WARN_INFINITE_RECURSION = YES;
+				CLANG_WARN_INT_CONVERSION = YES;
+				CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
+				CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
+				CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
+				CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
+				CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
+				CLANG_WARN_STRICT_PROTOTYPES = YES;
+				CLANG_WARN_SUSPICIOUS_MOVE = YES;
+				CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
+				CLANG_WARN_UNREACHABLE_CODE = YES;
+				CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
+				CODE_SIGN_IDENTITY = "iPhone Developer";
+				COPY_PHASE_STRIP = NO;
+				DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
+				ENABLE_NS_ASSERTIONS = NO;
+				ENABLE_STRICT_OBJC_MSGSEND = YES;
+				GCC_C_LANGUAGE_STANDARD = gnu11;
+				GCC_NO_COMMON_BLOCKS = YES;
+				GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
+				GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
+				GCC_WARN_UNDECLARED_SELECTOR = YES;
+				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
+				GCC_WARN_UNUSED_FUNCTION = YES;
+				GCC_WARN_UNUSED_VARIABLE = YES;
+				IPHONEOS_DEPLOYMENT_TARGET = 11.4;
+				MTL_ENABLE_DEBUG_INFO = NO;
+				SDKROOT = iphoneos;
+				SWIFT_COMPILATION_MODE = wholemodule;
+				SWIFT_OPTIMIZATION_LEVEL = "-O";
+				VALIDATE_PRODUCT = YES;
+			};
+			name = Release;
+		};
+		4AA72B822146ED66006C3AEF /* Debug */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
+				CODE_SIGN_STYLE = Automatic;
+				INFOPLIST_FILE = TensorFlowLiteApp/Info.plist;
+				LD_RUNPATH_SEARCH_PATHS = (
+					"$(inherited)",
+					"@executable_path/Frameworks",
+				);
+				PRODUCT_BUNDLE_IDENTIFIER = com.tensorflow.lite.swift.TensorFlowLite;
+				PRODUCT_NAME = "$(TARGET_NAME)";
+				SWIFT_VERSION = 4.0;
+				TARGETED_DEVICE_FAMILY = "1,2";
+			};
+			name = Debug;
+		};
+		4AA72B832146ED66006C3AEF /* Release */ = {
+			isa = XCBuildConfiguration;
+			buildSettings = {
+				ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
+				CODE_SIGN_STYLE = Automatic;
+				INFOPLIST_FILE = TensorFlowLiteApp/Info.plist;
+				LD_RUNPATH_SEARCH_PATHS = (
+					"$(inherited)",
+					"@executable_path/Frameworks",
+				);
+				PRODUCT_BUNDLE_IDENTIFIER = com.tensorflow.lite.swift.TensorFlowLite;
+				PRODUCT_NAME = "$(TARGET_NAME)";
+				SWIFT_VERSION = 4.0;
+				TARGETED_DEVICE_FAMILY = "1,2";
+			};
+			name = Release;
+		};
+/* End XCBuildConfiguration section */
+
+/* Begin XCConfigurationList section */
+		4AA72B6A2146ED64006C3AEF /* Build configuration list for PBXProject "TensorFlowLiteApp" */ = {
+			isa = XCConfigurationList;
+			buildConfigurations = (
+				4AA72B7F2146ED66006C3AEF /* Debug */,
+				4AA72B802146ED66006C3AEF /* Release */,
+			);
+			defaultConfigurationIsVisible = 0;
+			defaultConfigurationName = Release;
+		};
+		4AA72B812146ED66006C3AEF /* Build configuration list for PBXNativeTarget "TensorFlowLiteApp" */ = {
+			isa = XCConfigurationList;
+			buildConfigurations = (
+				4AA72B822146ED66006C3AEF /* Debug */,
+				4AA72B832146ED66006C3AEF /* Release */,
+			);
+			defaultConfigurationIsVisible = 0;
+			defaultConfigurationName = Release;
+		};
+/* End XCConfigurationList section */
+	};
+	rootObject = 4AA72B672146ED64006C3AEF /* Project object */;
+}
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/AppDelegate.swift b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/AppDelegate.swift
new file mode 100644
index 0000000000..ffa90a06ad
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/AppDelegate.swift
@@ -0,0 +1,24 @@
+import UIKit
+
+@UIApplicationMain
+
+final class AppDelegate: UIResponder, UIApplicationDelegate {
+
+  /// The main window of the app.
+  var window: UIWindow?
+
+  func application(
+    _ application: UIApplication,
+    didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]? = nil
+  ) -> Bool {
+    return true
+  }
+}
+
+// MARK: - Extensions
+
+#if !swift(>=4.2)
+extension UIApplication {
+  typealias LaunchOptionsKey = UIApplicationLaunchOptionsKey
+}
+#endif  // !swift(>=4.2)
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Array+TensorFlowLite.swift b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Array+TensorFlowLite.swift
new file mode 100644
index 0000000000..56df1ce659
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Array+TensorFlowLite.swift
@@ -0,0 +1,22 @@
+import Foundation
+
+extension Array {
+  /// Creates a new array from the bytes of the given unsafe data.
+  ///
+  /// - Warning: The array's `Element` type must be trivial in that it can be copied bit for bit
+  ///     with no indirection or reference-counting operations; otherwise, copying the raw bytes in
+  ///     the `unsafeData`'s buffer to a new array returns an unsafe copy.
+  /// - Note: Returns `nil` if `unsafeData.count` is not a multiple of
+  ///     `MemoryLayout<Element>.stride`.
+  /// - Parameter unsafeData: The data containing the bytes to turn into an array.
+  init?(unsafeData: Data) {
+    guard unsafeData.count % MemoryLayout<Element>.stride == 0 else { return nil }
+    let elements = unsafeData.withUnsafeBytes {
+      UnsafeBufferPointer<Element>(
+        start: $0,
+        count: unsafeData.count / MemoryLayout<Element>.stride
+      )
+    }
+    self.init(elements)
+  }
+}
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/AppIcon.appiconset/Contents.json b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/AppIcon.appiconset/Contents.json
new file mode 100644
index 0000000000..d8db8d65fd
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/AppIcon.appiconset/Contents.json
@@ -0,0 +1,98 @@
+{
+  "images" : [
+    {
+      "idiom" : "iphone",
+      "size" : "20x20",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "20x20",
+      "scale" : "3x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "29x29",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "29x29",
+      "scale" : "3x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "40x40",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "40x40",
+      "scale" : "3x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "60x60",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "60x60",
+      "scale" : "3x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "20x20",
+      "scale" : "1x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "20x20",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "29x29",
+      "scale" : "1x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "29x29",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "40x40",
+      "scale" : "1x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "40x40",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "76x76",
+      "scale" : "1x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "76x76",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "83.5x83.5",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "ios-marketing",
+      "size" : "1024x1024",
+      "scale" : "1x"
+    }
+  ],
+  "info" : {
+    "version" : 1,
+    "author" : "xcode"
+  }
+}
\ No newline at end of file
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/Contents.json b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/Contents.json
new file mode 100644
index 0000000000..da4a164c91
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/Contents.json
@@ -0,0 +1,6 @@
+{
+  "info" : {
+    "version" : 1,
+    "author" : "xcode"
+  }
+}
\ No newline at end of file
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/LaunchScreen.storyboard b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/LaunchScreen.storyboard
new file mode 100644
index 0000000000..a07a1321be
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/LaunchScreen.storyboard
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="14109" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" launchScreen="YES" useTraitCollections="YES" colorMatched="YES" initialViewController="01J-lp-oVM">
+    <device id="retina4_7" orientation="portrait">
+        <adaptation id="fullscreen"/>
+    </device>
+    <dependencies>
+        <deployment identifier="iOS"/>
+        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="14088"/>
+        <capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
+    </dependencies>
+    <scenes>
+        <!--View Controller-->
+        <scene sceneID="EHf-IW-A2E">
+            <objects>
+                <viewController id="01J-lp-oVM" sceneMemberID="viewController">
+                    <layoutGuides>
+                        <viewControllerLayoutGuide type="top" id="Llm-lL-Icb"/>
+                        <viewControllerLayoutGuide type="bottom" id="xb3-aO-Qok"/>
+                    </layoutGuides>
+                    <view key="view" contentMode="scaleToFill" id="Ze5-6b-2t3">
+                        <rect key="frame" x="0.0" y="0.0" width="375" height="667"/>
+                        <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
+                        <subviews>
+                            <label opaque="NO" userInteractionEnabled="NO" contentMode="left" horizontalHuggingPriority="251" verticalHuggingPriority="251" text="TensorFlowLite" textAlignment="center" lineBreakMode="tailTruncation" baselineAdjustment="alignBaselines" adjustsFontSizeToFit="NO" translatesAutoresizingMaskIntoConstraints="NO" id="3Gq-PV-hia">
+                                <rect key="frame" x="16" y="315" width="343" height="38.5"/>
+                                <fontDescription key="fontDescription" type="boldSystem" pointSize="32"/>
+                                <nil key="textColor"/>
+                                <nil key="highlightedColor"/>
+                            </label>
+                        </subviews>
+                        <color key="backgroundColor" red="1" green="1" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
+                        <constraints>
+                            <constraint firstItem="3Gq-PV-hia" firstAttribute="leading" secondItem="Ze5-6b-2t3" secondAttribute="leading" constant="16" id="aXL-9T-5Pf"/>
+                            <constraint firstItem="3Gq-PV-hia" firstAttribute="centerY" secondItem="Ze5-6b-2t3" secondAttribute="centerY" id="cDf-Go-1FR"/>
+                            <constraint firstAttribute="trailing" secondItem="3Gq-PV-hia" secondAttribute="trailing" constant="16" id="fB9-BX-A3B"/>
+                        </constraints>
+                    </view>
+                </viewController>
+                <placeholder placeholderIdentifier="IBFirstResponder" id="iYj-Kq-Ea1" userLabel="First Responder" sceneMemberID="firstResponder"/>
+            </objects>
+            <point key="canvasLocation" x="52" y="374.66266866566718"/>
+        </scene>
+    </scenes>
+</document>
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/Main.storyboard b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/Main.storyboard
new file mode 100644
index 0000000000..d0e91d63c4
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/Main.storyboard
@@ -0,0 +1,95 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="14313.18" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" useTraitCollections="YES" colorMatched="YES" initialViewController="BYZ-38-t0r">
+    <device id="retina4_7" orientation="portrait">
+        <adaptation id="fullscreen"/>
+    </device>
+    <dependencies>
+        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="14283.14"/>
+        <capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
+    </dependencies>
+    <scenes>
+        <!--View Controller-->
+        <scene sceneID="tne-QT-ifu">
+            <objects>
+                <viewController storyboardIdentifier="viewController" useStoryboardIdentifierAsRestorationIdentifier="YES" id="BYZ-38-t0r" customClass="ViewController" customModule="third_party_tensorflow_lite_experimental_swift_TensorFlowLiteAppLib" sceneMemberID="viewController">
+                    <layoutGuides>
+                        <viewControllerLayoutGuide type="top" id="y3c-jy-aDJ"/>
+                        <viewControllerLayoutGuide type="bottom" id="wfy-db-euE"/>
+                    </layoutGuides>
+                    <view key="view" contentMode="scaleToFill" id="8bC-Xf-vdC">
+                        <rect key="frame" x="0.0" y="0.0" width="375" height="667"/>
+                        <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
+                        <subviews>
+                            <textView clipsSubviews="YES" multipleTouchEnabled="YES" contentMode="scaleToFill" editable="NO" selectable="NO" translatesAutoresizingMaskIntoConstraints="NO" id="7Mj-sL-hrd">
+                                <rect key="frame" x="0.0" y="367" width="375" height="300"/>
+                                <color key="backgroundColor" red="0.0" green="0.47843137250000001" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
+                                <constraints>
+                                    <constraint firstAttribute="height" constant="300" id="YUb-MC-D5w"/>
+                                </constraints>
+                                <color key="textColor" cocoaTouchSystemColor="tableCellGroupedBackgroundColor"/>
+                                <fontDescription key="fontDescription" type="system" pointSize="14"/>
+                                <textInputTraits key="textInputTraits" autocapitalizationType="sentences"/>
+                            </textView>
+                            <toolbar opaque="NO" clearsContextBeforeDrawing="NO" contentMode="scaleToFill" translatesAutoresizingMaskIntoConstraints="NO" id="Qwg-EP-bd6" userLabel="Bottom Toolbar">
+                                <rect key="frame" x="0.0" y="323" width="375" height="44"/>
+                                <constraints>
+                                    <constraint firstAttribute="height" constant="44" id="jhT-Q0-E9N"/>
+                                </constraints>
+                                <items>
+                                    <barButtonItem style="plain" systemItem="flexibleSpace" id="P3q-uA-YUa"/>
+                                    <barButtonItem title="Invoke Interpreter" id="A4J-Mg-nmd" userLabel="Invoke Button">
+                                        <connections>
+                                            <action selector="invokeInterpreter:" destination="BYZ-38-t0r" id="lZU-x7-PsJ"/>
+                                        </connections>
+                                    </barButtonItem>
+                                    <barButtonItem style="plain" systemItem="flexibleSpace" id="Qad-Pa-ySg"/>
+                                </items>
+                            </toolbar>
+                            <toolbar opaque="NO" clearsContextBeforeDrawing="NO" contentMode="scaleToFill" translatesAutoresizingMaskIntoConstraints="NO" id="Gkb-TR-PCB" userLabel="Top Toolbar">
+                                <rect key="frame" x="0.0" y="28" width="375" height="44"/>
+                                <constraints>
+                                    <constraint firstAttribute="height" constant="44" id="hSD-2q-fUE"/>
+                                </constraints>
+                                <items>
+                                    <barButtonItem style="plain" id="LKw-TX-bbH">
+                                        <segmentedControl key="customView" opaque="NO" contentMode="scaleToFill" contentHorizontalAlignment="left" contentVerticalAlignment="top" segmentControlStyle="bar" selectedSegmentIndex="0" id="rhA-nW-xzT">
+                                            <rect key="frame" x="16" y="7" width="343" height="30"/>
+                                            <autoresizingMask key="autoresizingMask" flexibleMaxX="YES" flexibleMaxY="YES"/>
+                                            <segments>
+                                                <segment title="Add"/>
+                                                <segment title="AddQuantized"/>
+                                                <segment title="MultiAdd"/>
+                                            </segments>
+                                            <connections>
+                                                <action selector="modelChanged:" destination="BYZ-38-t0r" eventType="valueChanged" id="YnG-Ov-B5D"/>
+                                            </connections>
+                                        </segmentedControl>
+                                    </barButtonItem>
+                                </items>
+                            </toolbar>
+                        </subviews>
+                        <color key="backgroundColor" red="1" green="1" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
+                        <constraints>
+                            <constraint firstAttribute="trailing" secondItem="Gkb-TR-PCB" secondAttribute="trailing" id="4Cr-Sf-I7n"/>
+                            <constraint firstItem="7Mj-sL-hrd" firstAttribute="bottom" secondItem="wfy-db-euE" secondAttribute="top" id="6ot-zD-sze"/>
+                            <constraint firstItem="7Mj-sL-hrd" firstAttribute="top" secondItem="Qwg-EP-bd6" secondAttribute="bottom" id="ELA-C6-NiG"/>
+                            <constraint firstAttribute="trailing" secondItem="7Mj-sL-hrd" secondAttribute="trailing" id="HDO-xr-mBl"/>
+                            <constraint firstItem="Gkb-TR-PCB" firstAttribute="leading" secondItem="8bC-Xf-vdC" secondAttribute="leading" id="Kmo-6K-gS4"/>
+                            <constraint firstItem="Qwg-EP-bd6" firstAttribute="leading" secondItem="8bC-Xf-vdC" secondAttribute="leading" id="hGu-lm-fMG"/>
+                            <constraint firstAttribute="trailing" secondItem="Qwg-EP-bd6" secondAttribute="trailing" id="iXR-LK-nTO"/>
+                            <constraint firstItem="7Mj-sL-hrd" firstAttribute="leading" secondItem="8bC-Xf-vdC" secondAttribute="leading" id="nr7-jW-ZYf"/>
+                            <constraint firstItem="Gkb-TR-PCB" firstAttribute="top" secondItem="y3c-jy-aDJ" secondAttribute="bottom" constant="8" id="uCF-VW-rR0"/>
+                        </constraints>
+                    </view>
+                    <connections>
+                        <outlet property="invokeButton" destination="A4J-Mg-nmd" id="UxZ-Ft-E45"/>
+                        <outlet property="modelControl" destination="rhA-nW-xzT" id="KKf-TT-BQ2"/>
+                        <outlet property="resultsTextView" destination="7Mj-sL-hrd" id="T4I-z4-tYA"/>
+                    </connections>
+                </viewController>
+                <placeholder placeholderIdentifier="IBFirstResponder" id="dkx-z0-nzr" sceneMemberID="firstResponder"/>
+            </objects>
+            <point key="canvasLocation" x="125.59999999999999" y="133.5832083958021"/>
+        </scene>
+    </scenes>
+</document>
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Data+TensorFlowLite.swift b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Data+TensorFlowLite.swift
new file mode 100644
index 0000000000..bc8a70c848
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Data+TensorFlowLite.swift
@@ -0,0 +1,13 @@
+import Foundation
+
+extension Data {
+  /// Creates a new buffer by copying the buffer pointer of the given array.
+  ///
+  /// - Warning: The given array's element type `T` must be trivial in that it can be copied bit
+  ///     for bit with no indirection or reference-counting operations; otherwise, reinterpreting
+  ///     data from the resulting buffer has undefined behavior.
+  /// - Parameter array: An array with elements of type `T`.
+  init<T>(copyingBufferOf array: [T]) {
+    self = array.withUnsafeBufferPointer(Data.init)
+  }
+}
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Info.plist b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Info.plist
new file mode 100644
index 0000000000..3ca3875f04
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Info.plist
@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>CFBundleDevelopmentRegion</key>
+	<string>en</string>
+	<key>CFBundleExecutable</key>
+	<string>$(EXECUTABLE_NAME)</string>
+	<key>CFBundleIdentifier</key>
+	<string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
+	<key>CFBundleInfoDictionaryVersion</key>
+	<string>6.0</string>
+	<key>CFBundleName</key>
+	<string>$(PRODUCT_NAME)</string>
+	<key>CFBundlePackageType</key>
+	<string>APPL</string>
+	<key>CFBundleShortVersionString</key>
+	<string>1.0</string>
+	<key>CFBundleVersion</key>
+	<string>0.0.1</string>
+	<key>LSRequiresIPhoneOS</key>
+	<true/>
+	<key>NSCameraUsageDescription</key>
+	<string>NSCameraUsageDescription</string>
+	<key>NSPhotoLibraryUsageDescription</key>
+	<string>Select a photo to detect objects in.</string>
+	<key>UILaunchStoryboardName</key>
+	<string>LaunchScreen</string>
+	<key>UIMainStoryboardFile</key>
+	<string>Main</string>
+	<key>UIRequiredDeviceCapabilities</key>
+	<array>
+		<string>armv7</string>
+	</array>
+	<key>UISupportedInterfaceOrientations</key>
+	<array>
+		<string>UIInterfaceOrientationPortrait</string>
+		<string>UIInterfaceOrientationPortraitUpsideDown</string>
+	</array>
+	<key>UISupportedInterfaceOrientations~ipad</key>
+	<array>
+		<string>UIInterfaceOrientationPortrait</string>
+		<string>UIInterfaceOrientationPortraitUpsideDown</string>
+	</array>
+</dict>
+</plist>
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/ViewController.swift b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/ViewController.swift
new file mode 100644
index 0000000000..73c74fd19c
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/ViewController.swift
@@ -0,0 +1,299 @@
+import TensorFlowLite
+import UIKit
+
+class ViewController: UIViewController {
+
+  // MARK: - Properties
+
+  /// TensorFlowLite interpreter object for performing inference from a given model.
+  private var interpreter: Interpreter?
+
+  /// Serial dispatch queue for managing `Interpreter` calls.
+  private let interpreterQueue = DispatchQueue(
+    label: Constant.dispatchQueueLabel,
+    qos: .userInitiated
+  )
+
+  /// The currently selected model.
+  private var currentModel: Model {
+    guard let currentModel = Model(rawValue: modelControl.selectedSegmentIndex) else {
+      preconditionFailure("Invalid model for selected segment index.")
+    }
+    return currentModel
+  }
+
+  /// A description of the current model.
+  private var modelDescription: String {
+    guard let interpreter = interpreter else { return "" }
+    let inputCount = interpreter.inputTensorCount
+    let outputCount = interpreter.outputTensorCount
+    let inputTensors = (0..<inputCount).map { index in
+      var tensorInfo = "  Input \(index + 1): "
+      do {
+        let tensor = try interpreter.input(at: index)
+        tensorInfo += "\(tensor)"
+      } catch let error {
+        tensorInfo += "\(error.localizedDescription)"
+      }
+      return tensorInfo
+    }.joined(separator: "\n")
+    let outputTensors = (0..<outputCount).map { index in
+      var tensorInfo = "  Output \(index + 1): "
+      do {
+        let tensor = try interpreter.output(at: index)
+        tensorInfo += "\(tensor)"
+      } catch let error {
+        tensorInfo += "\(error.localizedDescription)"
+      }
+      return tensorInfo
+    }.joined(separator: "\n")
+    return "Model Description:\n" +
+             "  Input Tensor Count = \(inputCount)\n\(inputTensors)\n\n" +
+             "  Output Tensor Count = \(outputCount)\n\(outputTensors)"
+  }
+
+  // MARK: - IBOutlets
+
+  /// A segmented control for changing models. See the `Model` enum for available models.
+  @IBOutlet private var modelControl: UISegmentedControl!
+
+  @IBOutlet private var resultsTextView: UITextView!
+  @IBOutlet private var invokeButton: UIBarButtonItem!
+
+  // MARK: - UIViewController
+
+  override func viewDidLoad() {
+    super.viewDidLoad()
+
+    invokeButton.isEnabled = false
+    loadModel()
+  }
+
+  // MARK: - IBActions
+
+  @IBAction func modelChanged(_ sender: Any) {
+    invokeButton.isEnabled = false
+    updateResultsText("Switched to the \(currentModel.description) model.")
+    loadModel()
+  }
+
+  @IBAction func invokeInterpreter(_ sender: Any) {
+    switch currentModel {
+    case .add:
+      invokeAdd()
+    case .addQuantized:
+      invokeAddQuantized()
+    case .multiAdd:
+      invokeMultiAdd()
+    }
+  }
+
+  // MARK: - Private
+
+  private func loadModel() {
+    let fileInfo = currentModel.fileInfo
+    guard let modelPath = Bundle.main.path(forResource: fileInfo.name, ofType: fileInfo.extension)
+    else {
+      updateResultsText("Failed to load the \(currentModel.description) model.")
+      return
+    }
+    setUpInterpreter(withModelPath: modelPath)
+  }
+
+  private func setUpInterpreter(withModelPath modelPath: String) {
+    interpreterQueue.async {
+      do {
+        var options = InterpreterOptions()
+        options.isErrorLoggingEnabled = true
+        self.interpreter = try Interpreter(modelPath: modelPath, options: options)
+      } catch let error {
+        self.updateResultsText(
+          "Failed to create the interpreter with error: \(error.localizedDescription)"
+        )
+        return
+      }
+      safeDispatchOnMain { self.invokeButton.isEnabled = true }
+    }
+  }
+
+  private func invokeAdd() {
+    interpreterQueue.async {
+      guard let interpreter = self.interpreter else {
+        self.updateResultsText(Constant.nilInterpreterErrorMessage)
+        return
+      }
+      do {
+        try interpreter.resizeInput(at: 0, to: [2])
+        try interpreter.allocateTensors()
+        let input: [Float32] = [1, 3]
+        let resultsText = self.modelDescription + "\n\n" +
+          "Performing 2 add operations on input \(input.description) equals: "
+        self.updateResultsText(resultsText)
+        let data = Data(copyingBufferOf: input)
+        try interpreter.copy(data, toInputAt: 0)
+        try interpreter.invoke()
+        let outputTensor = try interpreter.output(at: 0)
+        let results: () -> String = {
+          guard let results = [Float32](unsafeData: outputTensor.data) else { return "No results." }
+          return resultsText + results.description
+        }
+        self.updateResultsText(results())
+      } catch let error {
+        self.updateResultsText(
+          "Failed to invoke the interpreter with error: \(error.localizedDescription)"
+        )
+        return
+      }
+    }
+  }
+
+  private func invokeAddQuantized() {
+    interpreterQueue.async {
+      guard let interpreter = self.interpreter else {
+        self.updateResultsText(Constant.nilInterpreterErrorMessage)
+        return
+      }
+      do {
+        try interpreter.resizeInput(at: 0, to: [2])
+        try interpreter.allocateTensors()
+        let input: [UInt8] = [1, 3]
+        let resultsText = self.modelDescription + "\n\n" +
+          "Performing 2 add operations on quantized input \(input.description) equals: "
+        self.updateResultsText(resultsText)
+        let data = Data(input)
+        try interpreter.copy(data, toInputAt: 0)
+        try interpreter.invoke()
+        let outputTensor = try interpreter.output(at: 0)
+        let results: () -> String = {
+          guard let quantizationParameters = outputTensor.quantizationParameters else {
+            return "No results."
+          }
+          let quantizedResults = [UInt8](outputTensor.data)
+          let dequantizedResults = quantizedResults.map {
+            quantizationParameters.scale * Float(Int($0) - quantizationParameters.zeroPoint)
+          }
+          return resultsText + quantizedResults.description +
+                   ", dequantized results: " + dequantizedResults.description
+        }
+        self.updateResultsText(results())
+      } catch let error {
+        self.updateResultsText(
+          "Failed to invoke the interpreter with error: \(error.localizedDescription)"
+        )
+        return
+      }
+    }
+  }
+
+  private func invokeMultiAdd() {
+    interpreterQueue.async {
+      guard let interpreter = self.interpreter else {
+        self.updateResultsText(Constant.nilInterpreterErrorMessage)
+        return
+      }
+      do {
+        let shape = TensorShape(2)
+        try (0..<interpreter.inputTensorCount).forEach { index in
+          try interpreter.resizeInput(at: index, to: shape)
+        }
+        try interpreter.allocateTensors()
+        let inputs = try (0..<interpreter.inputTensorCount).map { index -> [Float32] in
+          let input = [Float32(index + 1), Float32(index + 2)]
+          let data = Data(copyingBufferOf: input)
+          try interpreter.copy(data, toInputAt: index)
+          return input
+        }
+        let resultsText = self.modelDescription + "\n\n" +
+          "Performing 3 add operations on inputs \(inputs.description) equals: "
+        self.updateResultsText(resultsText)
+        try interpreter.invoke()
+        let results = try (0..<interpreter.outputTensorCount).map { index -> [Float32] in
+          let tensor = try interpreter.output(at: index)
+          return [Float32](unsafeData: tensor.data) ?? []
+        }
+        self.updateResultsText(resultsText + results.description)
+      } catch let error {
+        self.updateResultsText(
+          "Failed to invoke the interpreter with error: \(error.localizedDescription)"
+        )
+        return
+      }
+    }
+  }
+
+  private func updateResultsText(_ text: String? = nil) {
+    safeDispatchOnMain { self.resultsTextView.text = text }
+  }
+}
+
+// MARK: - Constants
+
+private enum Constant {
+  static let dispatchQueueLabel = "TensorFlowLiteInterpreterQueue"
+  static let nilInterpreterErrorMessage =
+    "Failed to invoke the interpreter because the interpreter was nil."
+}
+
+/// Models that can be loaded by the TensorFlow Lite `Interpreter`.
+private enum Model: Int, CustomStringConvertible {
+  /// A float model that performs two add operations on one input tensor and returns the result in
+  /// one output tensor.
+  case add = 0
+  /// A quantized model that performs two add operations on one input tensor and returns the result
+  /// in one output tensor.
+  case addQuantized = 1
+  /// A float model that performs three add operations on four input tensors and returns the results
+  /// in 2 output tensors.
+  case multiAdd = 2
+
+  var fileInfo: (name: String, extension: String) {
+    switch self {
+    case .add:
+      return Add.fileInfo
+    case .addQuantized:
+      return AddQuantized.fileInfo
+    case .multiAdd:
+      return MultiAdd.fileInfo
+    }
+  }
+
+  // MARK: - CustomStringConvertible
+
+  var description: String {
+    switch self {
+    case .add:
+      return Add.name
+    case .addQuantized:
+      return AddQuantized.name
+    case .multiAdd:
+      return MultiAdd.name
+    }
+  }
+}
+
+/// Values for the `Add` model.
+private enum Add {
+  static let name = "Add"
+  static let fileInfo = (name: "add", extension: "bin")
+}
+
+/// Values for the `AddQuantized` model.
+private enum AddQuantized {
+  static let name = "AddQuantized"
+  static let fileInfo = (name: "add_quantized", extension: "bin")
+}
+
+/// Values for the `MultiAdd` model.
+private enum MultiAdd {
+  static let name = "MultiAdd"
+  static let fileInfo = (name: "multi_add", extension: "bin")
+}
+
+// MARK: - Fileprivate
+
+/// Safely dispatches the given block on the main queue. If the current thread is `main`, the block
+/// is executed synchronously; otherwise, the block is executed asynchronously on the main thread.
+fileprivate func safeDispatchOnMain(_ block: @escaping () -> Void) {
+  if Thread.isMainThread { block(); return }
+  DispatchQueue.main.async { block() }
+}
diff --git a/tensorflow/lite/experimental/swift/Tests/InterpreterOptionsTests.swift b/tensorflow/lite/experimental/swift/Tests/InterpreterOptionsTests.swift
new file mode 100644
index 0000000000..54b4f59b28
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/Tests/InterpreterOptionsTests.swift
@@ -0,0 +1,54 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+@testable import TensorFlowLite
+import XCTest
+
+class InterpreterOptionsTests: XCTestCase {
+
+  func testInterpreterOptions_InitWithDefaultValues() {
+    let options = InterpreterOptions()
+    XCTAssertNil(options.threadCount)
+    XCTAssertFalse(options.isErrorLoggingEnabled)
+  }
+
+  func testInterpreterOptions_InitWithCustomValues() {
+    var options = InterpreterOptions()
+    options.threadCount = 2
+    XCTAssertEqual(options.threadCount, 2)
+    options.isErrorLoggingEnabled = true
+    XCTAssertTrue(options.isErrorLoggingEnabled)
+  }
+
+  func testInterpreterOptions_Equatable() {
+    var options1 = InterpreterOptions()
+    var options2 = InterpreterOptions()
+    XCTAssertEqual(options1, options2)
+
+    options1.threadCount = 2
+    options2.threadCount = 2
+    XCTAssertEqual(options1, options2)
+
+    options2.threadCount = 3
+    XCTAssertNotEqual(options1, options2)
+    options2.threadCount = 2
+
+    options1.isErrorLoggingEnabled = true
+    options2.isErrorLoggingEnabled = true
+    XCTAssertEqual(options1, options2)
+
+    options2.isErrorLoggingEnabled = false
+    XCTAssertNotEqual(options1, options2)
+  }
+}
diff --git a/tensorflow/lite/experimental/swift/Tests/InterpreterTests.swift b/tensorflow/lite/experimental/swift/Tests/InterpreterTests.swift
new file mode 100644
index 0000000000..e98da5f951
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/Tests/InterpreterTests.swift
@@ -0,0 +1,315 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+@testable import TensorFlowLite
+import XCTest
+
+class InterpreterTests: XCTestCase {
+
+  var interpreter: Interpreter!
+
+  override func setUp() {
+    super.setUp()
+
+    interpreter = try! Interpreter(modelPath: AddModel.path)
+  }
+
+  override func tearDown() {
+    interpreter = nil
+
+    super.tearDown()
+  }
+
+  func testInterpreter_InitWithModelPath() {
+    XCTAssertNoThrow(try Interpreter(modelPath: AddModel.path))
+  }
+
+  func testInterpreter_Init_ThrowsFailedToLoadModel() {
+    XCTAssertThrowsError(try Interpreter(modelPath: "/invalid/path")) { error in
+      self.assertEqualErrors(actual: error, expected: .failedToLoadModel)
+    }
+  }
+
+  func testInterpreter_InitWithModelPathAndOptions() {
+    var options = InterpreterOptions()
+    options.threadCount = 2
+    XCTAssertNoThrow(try Interpreter(modelPath: AddModel.path, options: options))
+  }
+
+  func testInterpreter_InputTensorCount() {
+    XCTAssertEqual(interpreter.inputTensorCount, AddModel.inputTensorCount)
+  }
+
+  func testInterpreter_OutputTensorCount() {
+    XCTAssertEqual(interpreter.outputTensorCount, AddModel.outputTensorCount)
+  }
+
+  func testInterpreter_Invoke() throws {
+    try interpreter.allocateTensors()
+    XCTAssertNoThrow(try interpreter.invoke())
+  }
+
+  func testInterpreter_Invoke_ThrowsAllocateTensorsRequired_ModelNotReady() {
+    XCTAssertThrowsError(try interpreter.invoke()) { error in
+      self.assertEqualErrors(actual: error, expected: .allocateTensorsRequired)
+    }
+  }
+
+  func testInterpreter_InputTensorAtIndex() throws {
+    try setUpAddModelInputTensor()
+    let inputTensor = try interpreter.input(at: AddModel.validIndex)
+    XCTAssertEqual(inputTensor, AddModel.inputTensor)
+  }
+
+  func testInterpreter_InputTensorAtIndex_QuantizedModel() throws {
+    interpreter = try Interpreter(modelPath: AddQuantizedModel.path)
+    try setUpAddQuantizedModelInputTensor()
+    let inputTensor = try interpreter.input(at: AddQuantizedModel.inputOutputIndex)
+    XCTAssertEqual(inputTensor, AddQuantizedModel.inputTensor)
+  }
+
+  func testInterpreter_InputTensorAtIndex_ThrowsInvalidIndex() throws {
+    try interpreter.allocateTensors()
+    XCTAssertThrowsError(try interpreter.input(at: AddModel.invalidIndex)) { error in
+      let maxIndex = AddModel.inputTensorCount - 1
+      self.assertEqualErrors(
+        actual: error,
+        expected: .invalidTensorIndex(index: AddModel.invalidIndex, maxIndex: maxIndex)
+      )
+    }
+  }
+
+  func testInterpreter_InputTensorAtIndex_ThrowsAllocateTensorsRequired() {
+    XCTAssertThrowsError(try interpreter.input(at: AddModel.validIndex)) { error in
+      self.assertEqualErrors(actual: error, expected: .allocateTensorsRequired)
+    }
+  }
+
+  func testInterpreter_OutputTensorAtIndex() throws {
+    try setUpAddModelInputTensor()
+    try interpreter.invoke()
+    let outputTensor = try interpreter.output(at: AddModel.validIndex)
+    XCTAssertEqual(outputTensor, AddModel.outputTensor)
+    let expectedResults = [Float32](unsafeData: outputTensor.data)
+    XCTAssertEqual(expectedResults, AddModel.results)
+  }
+
+  func testInterpreter_OutputTensorAtIndex_QuantizedModel() throws {
+    interpreter = try Interpreter(modelPath: AddQuantizedModel.path)
+    try setUpAddQuantizedModelInputTensor()
+    try interpreter.invoke()
+    let outputTensor = try interpreter.output(at: AddQuantizedModel.inputOutputIndex)
+    XCTAssertEqual(outputTensor, AddQuantizedModel.outputTensor)
+    let expectedResults = [UInt8](outputTensor.data)
+    XCTAssertEqual(expectedResults, AddQuantizedModel.results)
+  }
+
+  func testInterpreter_OutputTensorAtIndex_ThrowsInvalidIndex() throws {
+    try interpreter.allocateTensors()
+    try interpreter.invoke()
+    XCTAssertThrowsError(try interpreter.output(at: AddModel.invalidIndex)) { error in
+      let maxIndex = AddModel.outputTensorCount - 1
+      self.assertEqualErrors(
+        actual: error,
+        expected: .invalidTensorIndex(index: AddModel.invalidIndex, maxIndex: maxIndex)
+      )
+    }
+  }
+
+  func testInterpreter_OutputTensorAtIndex_ThrowsInvokeInterpreterRequired() {
+    XCTAssertThrowsError(try interpreter.output(at: AddModel.validIndex)) { error in
+      self.assertEqualErrors(actual: error, expected: .invokeInterpreterRequired)
+    }
+  }
+
+  func testInterpreter_ResizeInputTensorAtIndexToShape() {
+    XCTAssertNoThrow(try interpreter.resizeInput(at: AddModel.validIndex, to: [2, 2, 3]))
+    XCTAssertNoThrow(try interpreter.allocateTensors())
+  }
+
+  func testInterpreter_ResizeInputTensorAtIndexToShape_ThrowsInvalidIndex() {
+    XCTAssertThrowsError(try interpreter.resizeInput(
+      at: AddModel.invalidIndex,
+      to: [2, 2, 3]
+    )) { error in
+      let maxIndex = AddModel.inputTensorCount - 1
+      self.assertEqualErrors(
+        actual: error,
+        expected: .invalidTensorIndex(index: AddModel.invalidIndex, maxIndex: maxIndex)
+      )
+    }
+  }
+
+  func testInterpreter_CopyDataToInputTensorAtIndex() throws {
+    try interpreter.resizeInput(at: AddModel.validIndex, to: AddModel.shape)
+    try interpreter.allocateTensors()
+    let inputTensor = try interpreter.copy(AddModel.inputData, toInputAt: AddModel.validIndex)
+    XCTAssertEqual(inputTensor.data, AddModel.inputData)
+  }
+
+  func testInterpreter_CopyDataToInputTensorAtIndex_ThrowsInvalidIndex() {
+    XCTAssertThrowsError(try interpreter.copy(
+      AddModel.inputData,
+      toInputAt: AddModel.invalidIndex
+    )) { error in
+      let maxIndex = AddModel.inputTensorCount - 1
+      self.assertEqualErrors(
+        actual: error,
+        expected: .invalidTensorIndex(index: AddModel.invalidIndex, maxIndex: maxIndex)
+      )
+    }
+  }
+
+  func testInterpreter_CopyDataToInputTensorAtIndex_ThrowsInvalidDataCount() throws {
+    try interpreter.resizeInput(at: AddModel.validIndex, to: AddModel.shape)
+    try interpreter.allocateTensors()
+    let invalidData = Data(count: AddModel.dataCount - 1)
+    XCTAssertThrowsError(try interpreter.copy(
+      invalidData,
+      toInputAt: AddModel.validIndex
+    )) { error in
+      self.assertEqualErrors(
+        actual: error,
+        expected: .invalidTensorDataCount(provided: invalidData.count, required: AddModel.dataCount)
+      )
+    }
+  }
+
+  func testInterpreter_AllocateTensors() {
+    XCTAssertNoThrow(try interpreter.allocateTensors())
+  }
+
+  // MARK: - Private
+
+  private func setUpAddModelInputTensor() throws {
+    precondition(interpreter != nil)
+    try interpreter.resizeInput(at: AddModel.validIndex, to: AddModel.shape)
+    try interpreter.allocateTensors()
+    try interpreter.copy(AddModel.inputData, toInputAt: AddModel.validIndex)
+  }
+
+  private func setUpAddQuantizedModelInputTensor() throws {
+    precondition(interpreter != nil)
+    try interpreter.resizeInput(at: AddQuantizedModel.inputOutputIndex, to: AddQuantizedModel.shape)
+    try interpreter.allocateTensors()
+    try interpreter.copy(AddQuantizedModel.inputData, toInputAt: AddQuantizedModel.inputOutputIndex)
+  }
+
+  private func assertEqualErrors(actual: Error, expected: InterpreterError) {
+    guard let actual = actual as? InterpreterError else {
+      XCTFail("Actual error should be of type InterpreterError.")
+      return
+    }
+    XCTAssertEqual(actual, expected)
+  }
+}
+
+// MARK: - Constants
+
+/// Values for the `add.bin` model.
+private enum AddModel {
+  static let info = (name: "add", extension: "bin")
+  static let inputTensorCount = 1
+  static let outputTensorCount = 1
+  static let invalidIndex = 1
+  static let validIndex = 0
+  static let shape: TensorShape = [2]
+  static let dataCount = inputData.count
+  static let inputData = Data(copyingBufferOf: [Float32(1.0), Float32(3.0)])
+  static let outputData = Data(copyingBufferOf: [Float32(3.0), Float32(9.0)])
+  static let results = [Float32(3.0), Float32(9.0)]
+
+  static let inputTensor = Tensor(
+    name: "input",
+    dataType: .float32,
+    shape: shape,
+    data: inputData
+  )
+  static let outputTensor = Tensor(
+    name: "output",
+    dataType: .float32,
+    shape: shape,
+    data: outputData
+  )
+
+  static var path: String = {
+    let bundle = Bundle(for: InterpreterTests.self)
+    guard let path = bundle.path(forResource: info.name, ofType: info.extension) else { return "" }
+    return path
+  }()
+}
+
+/// Values for the `add_quantized.bin` model.
+private enum AddQuantizedModel {
+  static let info = (name: "add_quantized", extension: "bin")
+  static let inputOutputIndex = 0
+  static let shape: TensorShape = [2]
+  static let inputData = Data([1, 3])
+  static let outputData = Data([3, 9])
+  static let quantizationParameters = QuantizationParameters(scale: 0.003922, zeroPoint: 0)
+  static let results: [UInt8] = [3, 9]
+
+  static let inputTensor = Tensor(
+    name: "input",
+    dataType: .uInt8,
+    shape: shape,
+    data: inputData,
+    quantizationParameters: quantizationParameters
+  )
+  static let outputTensor = Tensor(
+    name: "output",
+    dataType: .uInt8,
+    shape: shape,
+    data: outputData,
+    quantizationParameters: quantizationParameters
+  )
+
+  static var path: String = {
+    let bundle = Bundle(for: InterpreterTests.self)
+    guard let path = bundle.path(forResource: info.name, ofType: info.extension) else { return "" }
+    return path
+  }()
+}
+
+// MARK: - Extensions
+
+extension Array {
+  /// Creates a new array from the bytes of the given unsafe data.
+  ///
+  /// - Note: Returns `nil` if `unsafeData.count` is not a multiple of
+  ///     `MemoryLayout<Element>.stride`.
+  /// - Parameter unsafeData: The data containing the bytes to turn into an array.
+  init?(unsafeData: Data) {
+    guard unsafeData.count % MemoryLayout<Element>.stride == 0 else { return nil }
+    let elements = unsafeData.withUnsafeBytes {
+      UnsafeBufferPointer<Element>(
+        start: $0,
+        count: unsafeData.count / MemoryLayout<Element>.stride
+      )
+    }
+    self.init(elements)
+  }
+}
+
+extension Data {
+  /// Creates a new buffer by copying the buffer pointer of the given array.
+  ///
+  /// - Warning: The given array's element type `T` must be trivial in that it can be copied bit
+  ///     for bit with no indirection or reference-counting operations; otherwise, reinterpreting
+  ///     data from the resulting buffer has undefined behavior.
+  /// - Parameter array: An array with elements of type `T`.
+  init<T>(copyingBufferOf array: [T]) {
+    self = array.withUnsafeBufferPointer(Data.init)
+  }
+}
diff --git a/tensorflow/lite/experimental/swift/Tests/ModelTests.swift b/tensorflow/lite/experimental/swift/Tests/ModelTests.swift
new file mode 100644
index 0000000000..025db18906
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/Tests/ModelTests.swift
@@ -0,0 +1,59 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+@testable import TensorFlowLite
+import XCTest
+
+class ModelTests: XCTestCase {
+
+  var modelPath: String!
+
+  override func setUp() {
+    super.setUp()
+
+    let bundle = Bundle(for: type(of: self))
+    guard let modelPath = bundle.path(
+            forResource: Constant.modelInfo.name,
+            ofType: Constant.modelInfo.extension)
+    else {
+      XCTFail("Failed to get the model file path.")
+      return
+    }
+    self.modelPath = modelPath
+  }
+
+  override func tearDown() {
+    modelPath = nil
+
+    super.tearDown()
+  }
+
+  func testModel_InitWithFilePath() {
+    XCTAssertNotNil(Model(filePath: modelPath))
+  }
+
+  func testModel_InitWithEmptyFilePath_FailsInitialization() {
+    XCTAssertNil(Model(filePath: ""))
+  }
+
+  func testModel_InitWithInvalidFilePath_FailsInitialization() {
+    XCTAssertNil(Model(filePath: "invalid/path"))
+  }
+}
+
+// MARK: - Constants
+
+private enum Constant {
+  static let modelInfo = (name: "add", extension: "bin")
+}
diff --git a/tensorflow/lite/experimental/swift/Tests/QuantizationParametersTests.swift b/tensorflow/lite/experimental/swift/Tests/QuantizationParametersTests.swift
new file mode 100644
index 0000000000..65648c2698
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/Tests/QuantizationParametersTests.swift
@@ -0,0 +1,43 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+@testable import TensorFlowLite
+import XCTest
+
+class QuantizationParametersTests: XCTestCase {
+
+  func testQuantizationParameters_InitWithCustomValues() {
+    let parameters = QuantizationParameters(scale: 0.5, zeroPoint: 1)
+    XCTAssertEqual(parameters.scale, 0.5)
+    XCTAssertEqual(parameters.zeroPoint, 1)
+  }
+
+  func testQuantizationParameters_Equatable() {
+    let parameters1 = QuantizationParameters(scale: 0.5, zeroPoint: 1)
+    let parameters2 = QuantizationParameters(scale: 0.5, zeroPoint: 1)
+    XCTAssertEqual(parameters1, parameters2)
+
+    let parameters3 = QuantizationParameters(scale: 0.4, zeroPoint: 1)
+    XCTAssertNotEqual(parameters1, parameters3)
+    XCTAssertNotEqual(parameters2, parameters3)
+  }
+}
+
+// MARK: - Extensions
+
+extension QuantizationParameters: Equatable {
+  public static func == (lhs: QuantizationParameters, rhs: QuantizationParameters) -> Bool {
+    return lhs.scale == rhs.scale && lhs.zeroPoint == rhs.zeroPoint
+  }
+}
diff --git a/tensorflow/lite/experimental/swift/Tests/TensorTests.swift b/tensorflow/lite/experimental/swift/Tests/TensorTests.swift
new file mode 100644
index 0000000000..4540043a16
--- /dev/null
+++ b/tensorflow/lite/experimental/swift/Tests/TensorTests.swift
@@ -0,0 +1,83 @@
+// Copyright 2018 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+@testable import TensorFlowLite
+import XCTest
+
+class TensorTests: XCTestCase {
+
+  // MARK: - Tensor
+
+  func testTensor_Init() {
+    let name = "InputTensor"
+    let dataType: TensorDataType = .uInt8
+    let shape = TensorShape(Constant.dimensions)
+    guard let data = name.data(using: .utf8) else { XCTFail("Data should not be nil."); return }
+    let quantizationParameters = QuantizationParameters(scale: 0.5, zeroPoint: 1)
+    let inputTensor = Tensor(
+      name: name,
+      dataType: dataType,
+      shape: shape,
+      data: data,
+      quantizationParameters: quantizationParameters
+    )
+    XCTAssertEqual(inputTensor.name, name)
+    XCTAssertEqual(inputTensor.dataType, dataType)
+    XCTAssertEqual(inputTensor.shape, shape)
+    XCTAssertEqual(inputTensor.data, data)
+    XCTAssertEqual(inputTensor.quantizationParameters, quantizationParameters)
+  }
+
+  // MARK: - TensorShape
+
+  func testTensorShape_InitWithArray() {
+    let shape = TensorShape(Constant.dimensions)
+    XCTAssertEqual(shape.rank, Constant.dimensions.count)
+    XCTAssertEqual(shape.dimensions, Constant.dimensions)
+  }
+
+  func testTensorShape_InitWithElements() {
+    let shape = TensorShape(2, 2, 3)
+    XCTAssertEqual(shape.rank, Constant.dimensions.count)
+    XCTAssertEqual(shape.dimensions, Constant.dimensions)
+  }
+
+  func testTensorShape_InitWithArrayLiteral() {
+    let shape: TensorShape = [2, 2, 3]
+    XCTAssertEqual(shape.rank, Constant.dimensions.count)
+    XCTAssertEqual(shape.dimensions, Constant.dimensions)
+  }
+}
+
+// MARK: - Constants
+
+private enum Constant {
+  /// Array of 2 arrays of 2 arrays of 3 numbers: [[[1, 1, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]]].
+  static let dimensions = [2, 2, 3]
+}
+
+// MARK: - Extensions
+
+extension TensorShape: Equatable {
+  public static func == (lhs: TensorShape, rhs: TensorShape) -> Bool {
+    return lhs.rank == rhs.rank && lhs.dimensions == rhs.dimensions
+  }
+}
+
+extension Tensor: Equatable {
+  public static func == (lhs: Tensor, rhs: Tensor) -> Bool {
+    return lhs.name == rhs.name && lhs.dataType == rhs.dataType && lhs.shape == rhs.shape &&
+           lhs.data == rhs.data && lhs.quantizationParameters == rhs.quantizationParameters
+  }
+}
diff --git a/tensorflow/tools/pip_package/pip_smoke_test.py b/tensorflow/tools/pip_package/pip_smoke_test.py
index 51d010c9e1..952c71c615 100644
--- a/tensorflow/tools/pip_package/pip_smoke_test.py
+++ b/tensorflow/tools/pip_package/pip_smoke_test.py
@@ -30,14 +30,19 @@ os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../..")))
 PIP_PACKAGE_QUERY_EXPRESSION = (
     "deps(//tensorflow/tools/pip_package:build_pip_package)")
 
+# List of file paths containing BUILD files that should not be included for the
+# pip smoke test.
+BUILD_BLACKLIST = [
+    "tensorflow/lite/examples/android",
+    "tensorflow/lite/experimental/swift",
+]
 
 def GetBuild(dir_base):
   """Get the list of BUILD file all targets recursively startind at dir_base."""
   items = []
   for root, _, files in os.walk(dir_base):
     for name in files:
-      if (name == "BUILD" and
-          root.find("tensorflow/lite/examples/android") == -1):
+      if (name == "BUILD" and root not in BUILD_BLACKLIST):
         items.append("//" + root + ":all")
   return items
 
@@ -67,9 +72,9 @@ def BuildPyTestDependencies():
 
 PYTHON_TARGETS, PY_TEST_QUERY_EXPRESSION = BuildPyTestDependencies()
 
-# Hard-coded blacklist of files if not included in pip package
 # TODO(amitpatankar): Clean up blacklist.
-BLACKLIST = [
+# List of dependencies that should not included in the pip package.
+DEPENDENCY_BLACKLIST = [
     "//tensorflow/python:extra_py_tests_deps",
     "//tensorflow/cc/saved_model:saved_model_half_plus_two",
     "//tensorflow:no_tensorflow_py_deps",
@@ -82,9 +87,7 @@ BLACKLIST = [
     "//tensorflow/core/kernels/cloud:bigquery_reader_ops",
     "//tensorflow/python/feature_column:vocabulary_testdata",
     "//tensorflow/python:framework/test_file_system.so",
-    # contrib
-    "//tensorflow/contrib/session_bundle:session_bundle_half_plus_two",
-    "//tensorflow/contrib/keras:testing_utils",
+    # lite
     "//tensorflow/lite/experimental/examples/lstm:tflite_lstm",
     "//tensorflow/lite/experimental/examples/lstm:tflite_lstm.py",
     "//tensorflow/lite/experimental/examples/lstm:unidirectional_sequence_lstm_test",  # pylint:disable=line-too-long
@@ -93,6 +96,9 @@ BLACKLIST = [
     "//tensorflow/lite/python:interpreter_test",
     "//tensorflow/lite/python:interpreter.py",
     "//tensorflow/lite/python:interpreter_test.py",
+    # contrib
+    "//tensorflow/contrib/session_bundle:session_bundle_half_plus_two",
+    "//tensorflow/contrib/keras:testing_utils",
     "//tensorflow/contrib/ffmpeg:test_data",
     "//tensorflow/contrib/fused_conv:fused_conv2d_bias_activation_op_test_base",
     "//tensorflow/contrib/hadoop:test_data",
@@ -149,8 +155,8 @@ def main():
   # File extensions and endings to ignore
   ignore_extensions = ["_test", "_test.py", "_test_gpu", "_test_gpu.py"]
 
-  ignored_files = 0
-  blacklisted_files = len(BLACKLIST)
+  ignored_files_count = 0
+  blacklisted_dependencies_count = len(DEPENDENCY_BLACKLIST)
   # Compare dependencies
   for dependency in tf_py_test_dependencies_list:
     if dependency and dependency.startswith("//tensorflow"):
@@ -158,16 +164,16 @@ def main():
       # Ignore extensions
       if any(dependency.endswith(ext) for ext in ignore_extensions):
         ignore = True
-        ignored_files += 1
+        ignored_files_count += 1
 
-      # Check if the dependency is in the pip package, the blacklist, or
-      # should be ignored because of its file extension
+      # Check if the dependency is in the pip package, the dependency blacklist,
+      # or should be ignored because of its file extension.
       if not (ignore or dependency in pip_package_dependencies_list or
-              dependency in BLACKLIST):
+              dependency in DEPENDENCY_BLACKLIST):
         missing_dependencies.append(dependency)
 
-  print("Ignored files: %d" % ignored_files)
-  print("Blacklisted files: %d" % blacklisted_files)
+  print("Ignored files count: %d" % ignored_files_count)
+  print("Blacklisted dependencies count: %d" % blacklisted_dependencies_count)
   if missing_dependencies:
     print("Missing the following dependencies from pip_packages:")
     for missing_dependency in missing_dependencies:
-- 
GitLab


From d28fddf859f0d9093a1279ceda52ebfb2d915788 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Fri, 4 Jan 2019 10:57:17 -0800
Subject: [PATCH 0234/1095] Fix some metric doc strings.

PiperOrigin-RevId: 227880840
---
 tensorflow/python/keras/metrics.py | 42 +++++++++++++++---------------
 1 file changed, 21 insertions(+), 21 deletions(-)

diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index 73c116a745..7f13cc46e3 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -1382,8 +1382,8 @@ class MeanAbsoluteError(MeanMetricWrapper):
 
   Usage:
   ```python
-  mae = tf.metrics.MeanAbsoluteError()
-  mae.update_state([0., 0., 1., 1.], [1., 1., 1., 0.])
+  m = tf.metrics.MeanAbsoluteError()
+  m.update_state([0., 0., 1., 1.], [1., 1., 1., 0.])
   print('Final result: ', m.result().numpy())  # Final result: 0.75
   ```
 
@@ -1391,7 +1391,7 @@ class MeanAbsoluteError(MeanMetricWrapper):
 
   ```python
   model = keras.models.Model(inputs, outputs)
-  model.compile('sgd', loss=tf.keras.losses.MeanAbsoluteError())
+  model.compile('sgd', metrics=[tf.keras.metrics.MeanAbsoluteError()])
   ```
   """
 
@@ -1416,8 +1416,8 @@ class MeanAbsolutePercentageError(MeanMetricWrapper):
   Usage:
 
   ```python
-  mape = tf.keras.losses.MeanAbsolutePercentageError()
-  mape.update_state([0., 0., 1., 1.], [1., 1., 1., 0.])
+  m = tf.keras.metrics.MeanAbsolutePercentageError()
+  m.update_state([0., 0., 1., 1.], [1., 1., 1., 0.])
   print('Final result: ', m.result().numpy())  # Final result: 5e+08
   ```
 
@@ -1425,7 +1425,7 @@ class MeanAbsolutePercentageError(MeanMetricWrapper):
 
   ```python
   model = keras.models.Model(inputs, outputs)
-  model.compile('sgd', loss=tf.keras.losses.MeanAbsolutePercentageError())
+  model.compile('sgd', metrics=[tf.keras.metrics.MeanAbsolutePercentageError()])
   ```
   """
 
@@ -1450,8 +1450,8 @@ class MeanSquaredError(MeanMetricWrapper):
   Usage:
 
   ```python
-  mape = tf.keras.losses.MeanSquaredError()
-  mape.update_state([0., 0., 1., 1.], [1., 1., 1., 0.])
+  m = tf.keras.metrics.MeanSquaredError()
+  m.update_state([0., 0., 1., 1.], [1., 1., 1., 0.])
   print('Final result: ', m.result().numpy())  # Final result: 0.75
   ```
 
@@ -1459,7 +1459,7 @@ class MeanSquaredError(MeanMetricWrapper):
 
   ```python
   model = keras.models.Model(inputs, outputs)
-  model.compile('sgd', loss=tf.keras.losses.MeanSquaredError())
+  model.compile('sgd', metrics=[tf.keras.metrics.MeanSquaredError()])
   ```
   """
 
@@ -1484,8 +1484,8 @@ class MeanSquaredLogarithmicError(MeanMetricWrapper):
   Usage:
 
   ```python
-  msle = tf.keras.losses.MeanSquaredLogarithmicError()
-  msle.update_state([0., 0., 1., 1.], [1., 1., 1., 0.])
+  m = tf.keras.metrics.MeanSquaredLogarithmicError()
+  m.update_state([0., 0., 1., 1.], [1., 1., 1., 0.])
   print('Final result: ', m.result().numpy())  # Final result: 0.36034
   ```
 
@@ -1493,7 +1493,7 @@ class MeanSquaredLogarithmicError(MeanMetricWrapper):
 
   ```python
   model = keras.models.Model(inputs, outputs)
-  model.compile('sgd', loss=tf.keras.losses.MeanSquaredLogarithmicError())
+  model.compile('sgd', metrics=[tf.keras.metrics.MeanSquaredLogarithmicError()])
   ```
   """
 
@@ -1518,8 +1518,8 @@ class Hinge(MeanMetricWrapper):
   Usage:
 
   ```python
-  h = tf.keras.metrics.Hinge()
-  h.update_state([0., 1., 1.], [1., 0., 1.])
+  m = tf.keras.metrics.Hinge()
+  m.update_state([0., 1., 1.], [1., 0., 1.])
   print('Final result: ', m.result().numpy())  # Final result: 0.66
   ```
 
@@ -1527,7 +1527,7 @@ class Hinge(MeanMetricWrapper):
 
   ```python
   model = keras.models.Model(inputs, outputs)
-  model.compile('sgd', loss=tf.keras.metrics.Hinge())
+  model.compile('sgd', metrics=[tf.keras.metrics.Hinge()])
   ```
   """
 
@@ -1551,8 +1551,8 @@ class SquaredHinge(MeanMetricWrapper):
   Usage:
 
   ```python
-  h = tf.keras.metrics.SquaredHinge()
-  h.update_state([0., 1., 1.], [1., 0., 1.])
+  m = tf.keras.metrics.SquaredHinge()
+  m.update_state([0., 1., 1.], [1., 0., 1.])
   print('Final result: ', m.result().numpy())  # Final result: 0.66
   ```
 
@@ -1560,7 +1560,7 @@ class SquaredHinge(MeanMetricWrapper):
 
   ```python
   model = keras.models.Model(inputs, outputs)
-  model.compile('sgd', loss=tf.keras.metrics.SquaredHinge())
+  model.compile('sgd', metrics=[tf.keras.metrics.SquaredHinge()])
   ```
   """
 
@@ -1584,8 +1584,8 @@ class CategoricalHinge(MeanMetricWrapper):
   Usage:
 
   ```python
-  h = tf.keras.metrics.CategoricalHinge()
-  h.update_state([0., 1., 1.], [1., 0., 1.])
+  m = tf.keras.metrics.CategoricalHinge()
+  m.update_state([0., 1., 1.], [1., 0., 1.])
   print('Final result: ', m.result().numpy())  # Final result: 1.0
   ```
 
@@ -1593,7 +1593,7 @@ class CategoricalHinge(MeanMetricWrapper):
 
   ```python
   model = keras.models.Model(inputs, outputs)
-  model.compile('sgd', loss=tf.keras.metrics.CategoricalHinge())
+  model.compile('sgd', metrics=[tf.keras.metrics.CategoricalHinge()])
   ```
   """
 
-- 
GitLab


From 49d70dff868661bc7aa999227dad64bde2538899 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 4 Jan 2019 10:58:35 -0800
Subject: [PATCH 0235/1095] Make tf.where documentation clearer.

PiperOrigin-RevId: 227881065
---
 tensorflow/python/ops/array_ops.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py
index 07a7e180fb..81558db04f 100644
--- a/tensorflow/python/ops/array_ops.py
+++ b/tensorflow/python/ops/array_ops.py
@@ -3191,7 +3191,7 @@ def where(condition, x=None, y=None, name=None):
 
   Returns:
     A `Tensor` with the same type and shape as `x`, `y` if they are non-None.
-    A `Tensor` with shape `(num_true, dim_size(condition))`.
+    Otherwise, a `Tensor` with shape `(num_true, rank(condition))`.
 
   Raises:
     ValueError: When exactly one of `x` or `y` is non-None.
-- 
GitLab


From 9d72bb94c054b9d3a00f39e6a99c02995b207ae6 Mon Sep 17 00:00:00 2001
From: Bixia Zheng <bixia@google.com>
Date: Fri, 4 Jan 2019 11:04:06 -0800
Subject: [PATCH 0236/1095] [XLA] The HLO evaluator should preserve the layouts
 for instructions inside a fusion.

When evaluating a fusion, the HLO evaluator makes a copy of the fusion and
resets the layout of each instruction in the copy to default layout before
evaluating the copy of the fusion. In doing that, it changes the layout of the
fusion intputs and outputs and causes incorrect results. This change fixes the
evaluator to not overwrite the layouts for the instructions inside a fusion.

Add test cases for fusion inputs, single output fusion and multiple output
fusion.

PiperOrigin-RevId: 227882142
---
 .../compiler/xla/service/hlo_evaluator.cc     |  4 +-
 .../xla/service/hlo_evaluator_test.cc         | 66 +++++++++++++++++++
 2 files changed, 69 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc
index e897d09e1b..9754b8906b 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc
@@ -1097,7 +1097,9 @@ Status HloEvaluator::HandleFusion(HloInstruction* fusion) {
       fusion->fused_instructions_computation()->Clone(
           /*suffix=*/"clone_with_layout", &context);
   for (auto* instruction : cloned_fused_computation->instructions()) {
-    LayoutUtil::SetToDefaultLayout(instruction->mutable_shape());
+    if (!LayoutUtil::HasLayout(instruction->shape())) {
+      LayoutUtil::SetToDefaultLayout(instruction->mutable_shape());
+    }
   }
   auto readded_computation =
       empty_hlo_module.AddEntryComputation(std::move(cloned_fused_computation));
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_test.cc b/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
index 0298da2174..8056193d99 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
@@ -2924,5 +2924,71 @@ ENTRY main {
             "Expected 1 argument, but got 2.");
 }
 
+TEST_F(HloEvaluatorTest, PreserveFusionInputLayout) {
+  constexpr absl::string_view hlo_text = R"(
+    HloModule FusionInputLayout
+
+    fused_computation {
+      param_0 = f32[20,20]{0,1} parameter(0)
+      ROOT bitcast = f32[20,20]{1,0} bitcast(param_0)
+    }
+
+    ENTRY kernel_entry {
+      parameter.0 = f32[20,20]{0,1} parameter(0)
+      ROOT fusion = f32[20,20]{1,0} fusion(parameter.0),
+        kind=kLoop, calls=fused_computation
+    })";
+
+  TF_ASSERT_OK_AND_ASSIGN(m_, ParseAndReturnVerifiedModule(hlo_text));
+  auto args = MakeFakeArguments(m_.get()).ConsumeValueOrDie();
+  Literal actual = Evaluate({&args[0]});
+  EXPECT_TRUE(absl::c_equal(args[0].data<float>(), actual.data<float>()));
+}
+
+TEST_F(HloEvaluatorTest, PreserveFusionOutputLayout) {
+  constexpr absl::string_view hlo_text = R"(
+    HloModule FusionOutputLayout
+
+    fused_computation {
+      param_0 = f32[20,20]{1,0} parameter(0)
+      ROOT bitcast = f32[20,20]{0,1} bitcast(param_0)
+    }
+
+    ENTRY kernel_entry {
+      parameter.0 = f32[20,20]{1,0} parameter(0)
+      ROOT fusion = f32[20,20]{0,1} fusion(parameter.0),
+        kind=kLoop, calls=fused_computation
+    })";
+
+  TF_ASSERT_OK_AND_ASSIGN(m_, ParseAndReturnVerifiedModule(hlo_text));
+  auto args = MakeFakeArguments(m_.get()).ConsumeValueOrDie();
+  Literal actual = Evaluate({&args[0]});
+  EXPECT_TRUE(absl::c_equal(args[0].data<float>(), actual.data<float>()));
+}
+
+TEST_F(HloEvaluatorTest, PreserveMOFusionOutputLayout) {
+  constexpr absl::string_view hlo_text = R"(
+    HloModule MOFusionOutputLayout
+
+    fused_computation {
+      param_0 = f32[20,20]{1,0} parameter(0)
+      bitcast = f32[20,20]{0,1} bitcast(param_0)
+      ROOT tuple = (f32[20,20]{0,1}) tuple(bitcast)
+    }
+
+    ENTRY kernel_entry {
+      parameter.0 = f32[20,20]{1,0} parameter(0)
+      ROOT fusion = (f32[20,20]{0,1}) fusion(parameter.0),
+        kind=kLoop, calls=fused_computation
+    })";
+
+  TF_ASSERT_OK_AND_ASSIGN(m_, ParseAndReturnVerifiedModule(hlo_text));
+  auto args = MakeFakeArguments(m_.get()).ConsumeValueOrDie();
+  Literal actual_tuple = Evaluate({&args[0]});
+  std::vector<Literal> actual_literals = actual_tuple.DecomposeTuple();
+  EXPECT_TRUE(
+      absl::c_equal(args[0].data<float>(), actual_literals[0].data<float>()));
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From d76c2d0a570495854617c318a2708636f5530f27 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Fri, 4 Jan 2019 11:10:12 -0800
Subject: [PATCH 0237/1095] Improve the error message for model
 fit/eval/predict in scope

PiperOrigin-RevId: 227883249
---
 tensorflow/contrib/distribute/python/keras_test.py          | 2 +-
 .../python/keras/engine/distributed_training_utils.py       | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index 4cfcc0375f..1cb5fa30a3 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -1196,7 +1196,7 @@ class TestDistributionStrategyValidation(test.TestCase,
   def test_loop_in_scope(self, distribution):
     with self.cached_session():
       with self.assertRaisesRegexp(
-          RuntimeError, 'should not be run inside the distribution strategy'):
+          RuntimeError, 'should not be run inside the tf.distribute.Strategy'):
         with distribution.scope():
           x = keras.layers.Input(shape=(3,), name='input')
           y = keras.layers.Dense(4, name='dense')(x)
diff --git a/tensorflow/python/keras/engine/distributed_training_utils.py b/tensorflow/python/keras/engine/distributed_training_utils.py
index 9e3bdd26cb..1678d84307 100644
--- a/tensorflow/python/keras/engine/distributed_training_utils.py
+++ b/tensorflow/python/keras/engine/distributed_training_utils.py
@@ -48,9 +48,9 @@ def validate_not_in_strategy_scope():
   if distribution_strategy_context.has_distribution_strategy():
     if distribution_strategy_context.in_cross_replica_context():
       raise RuntimeError(
-          'Fit/Eval/Predict should not be run inside the distribution strategy '
-          'scope. Only model creation and compilation should be in '
-          'distribution strategy scope.')
+          'Fit/Eval/Predict should not be run inside the tf.distribute.Strategy'
+          ' scope. Only model creation and compilation should be in '
+          'tf.distribute.Strategy scope.')
 
 
 def set_weights(distribution_strategy, dist_model, weights):
-- 
GitLab


From 735d26a821d9f69f58ad12ea841da9bc8c5a65ee Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 4 Jan 2019 11:18:24 -0800
Subject: [PATCH 0238/1095] Make session_handle of DirectSession an unique
 identifier and accessible from OptimizationPassRegistry::PRE_PLACEMENT pass.

PiperOrigin-RevId: 227884596
---
 tensorflow/core/common_runtime/direct_session.cc      | 11 +++++++----
 tensorflow/core/common_runtime/direct_session.h       |  1 +
 tensorflow/core/common_runtime/executor.cc            |  3 +++
 tensorflow/core/common_runtime/executor.h             |  2 ++
 .../core/common_runtime/graph_execution_state.cc      |  3 +++
 .../core/common_runtime/graph_execution_state.h       |  4 ++++
 .../core/common_runtime/optimization_registry.h       |  1 +
 tensorflow/core/framework/op_kernel.h                 |  6 ++++++
 8 files changed, 27 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/common_runtime/direct_session.cc b/tensorflow/core/common_runtime/direct_session.cc
index 51b2c68c76..36f1a92aab 100644
--- a/tensorflow/core/common_runtime/direct_session.cc
+++ b/tensorflow/core/common_runtime/direct_session.cc
@@ -59,6 +59,7 @@ limitations under the License.
 #include "tensorflow/core/lib/gtl/array_slice.h"
 #include "tensorflow/core/lib/gtl/stl_util.h"
 #include "tensorflow/core/lib/monitoring/counter.h"
+#include "tensorflow/core/lib/random/random.h"
 #include "tensorflow/core/lib/strings/numbers.h"
 #include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
@@ -303,10 +304,8 @@ DirectSession::DirectSession(const SessionOptions& options,
   if (!status.ok()) {
     LOG(ERROR) << status.error_message();
   }
-  // NOTE(mrry): We do not need to use a unique string for the session
-  // handle, because DirectSession owns its devices. This may change
-  // in future versions.
-  session_handle_ = "direct";
+  session_handle_ =
+      strings::StrCat("direct", strings::FpToString(random::New64()));
   int devices_added = 0;
   if (options.config.log_device_placement()) {
     const string mapping_str = device_mgr_->DeviceMappingString();
@@ -371,6 +370,7 @@ Status DirectSession::MaybeInitializeExecutionState(
   GraphExecutionStateOptions options;
   options.device_set = &device_set_;
   options.session_options = &options_;
+  options.session_handle = session_handle_;
   // TODO(mrry,suharshs): We explicitly copy `graph` so that
   // `MakeForBaseGraph()` can take ownership of its
   // contents. Previously this happened implicitly in calls to the
@@ -533,6 +533,7 @@ Status DirectSession::RunInternal(int64 step_id, const RunOptions& run_options,
   CancellationManager step_cancellation_manager;
   args.cancellation_manager = &step_cancellation_manager;
   args.session_state = &session_state_;
+  args.session_handle = session_handle_;
   args.tensor_store = &run_state.tensor_store;
   args.step_container = &run_state.step_container;
   args.sync_on_finish = sync_on_finish_;
@@ -888,6 +889,7 @@ Status DirectSession::PRunSetup(const std::vector<string>& input_names,
     SchedClosure(pool, std::move(c));
   };
   args.session_state = &session_state_;
+  args.session_handle = session_handle_;
   args.tensor_store = &run_state->tensor_store;
   args.step_container = &run_state->step_container;
   if (LogMemory::IsEnabled()) {
@@ -1465,6 +1467,7 @@ Status DirectSession::CreateGraphs(
     prune_options.device_set = &device_set_;
     prune_options.session_options = &options_;
     prune_options.stateful_placements = stateful_placements_;
+    prune_options.session_handle = session_handle_;
     TF_RETURN_IF_ERROR(GraphExecutionState::MakeForPrunedGraph(
         execution_state_->original_graph_def().library(), prune_options,
         execution_state_->original_graph_def(), subgraph_options,
diff --git a/tensorflow/core/common_runtime/direct_session.h b/tensorflow/core/common_runtime/direct_session.h
index 6754e9cfb7..bcac341544 100644
--- a/tensorflow/core/common_runtime/direct_session.h
+++ b/tensorflow/core/common_runtime/direct_session.h
@@ -317,6 +317,7 @@ class DirectSession : public Session {
   std::vector<Device*> devices_;  // not owned
   DeviceSet device_set_;
 
+  // Unique session identifier.
   string session_handle_;
   mutex graph_state_lock_;
   bool graph_created_ GUARDED_BY(graph_state_lock_) = false;
diff --git a/tensorflow/core/common_runtime/executor.cc b/tensorflow/core/common_runtime/executor.cc
index df2ee6c618..07c8c4a5d4 100644
--- a/tensorflow/core/common_runtime/executor.cc
+++ b/tensorflow/core/common_runtime/executor.cc
@@ -1244,6 +1244,7 @@ class ExecutorState {
   Rendezvous* rendezvous_;
   CollectiveExecutor* collective_executor_ = nullptr;
   SessionState* session_state_;
+  string session_handle_;
   TensorStore* tensor_store_;
   // Step-local container.
   ScopedStepContainer* step_container_;
@@ -1371,6 +1372,7 @@ ExecutorState::ExecutorState(const Executor::Args& args, ExecutorImpl* impl)
       rendezvous_(args.rendezvous),
       collective_executor_(args.collective_executor),
       session_state_(args.session_state),
+      session_handle_(args.session_handle),
       tensor_store_(args.tensor_store),
       step_container_(args.step_container),
       stats_collector_(args.stats_collector),
@@ -1616,6 +1618,7 @@ void ExecutorState::Process(TaggedNode tagged_node, int64 scheduled_nsec) {
   params.rendezvous = rendezvous_;
   params.collective_executor = collective_executor_;
   params.session_state = session_state_;
+  params.session_handle = session_handle_;
   params.tensor_store = tensor_store_;
   params.cancellation_manager = cancellation_manager_;
   params.call_frame = call_frame_;
diff --git a/tensorflow/core/common_runtime/executor.h b/tensorflow/core/common_runtime/executor.h
index 02930168a4..4be60c6771 100644
--- a/tensorflow/core/common_runtime/executor.h
+++ b/tensorflow/core/common_runtime/executor.h
@@ -88,6 +88,8 @@ class Executor {
     CallFrameInterface* call_frame = nullptr;
     CancellationManager* cancellation_manager = nullptr;
     SessionState* session_state = nullptr;
+    // Unique session identifier. Can be empty.
+    string session_handle;
     TensorStore* tensor_store = nullptr;
     ScopedStepContainer* step_container = nullptr;
     CollectiveExecutor* collective_executor = nullptr;
diff --git a/tensorflow/core/common_runtime/graph_execution_state.cc b/tensorflow/core/common_runtime/graph_execution_state.cc
index 04d658f047..9ecbc34f5f 100644
--- a/tensorflow/core/common_runtime/graph_execution_state.cc
+++ b/tensorflow/core/common_runtime/graph_execution_state.cc
@@ -59,6 +59,7 @@ GraphExecutionState::GraphExecutionState(
     : stateful_placements_(options.stateful_placements),
       device_set_(options.device_set),
       session_options_(options.session_options),
+      session_handle_(options.session_handle),
       flib_def_(new FunctionLibraryDefinition(OpRegistry::Global(),
                                               graph_def->library())),
       graph_(nullptr) {
@@ -198,6 +199,7 @@ Status GraphExecutionState::Extend(
   GraphExecutionStateOptions combined_options;
   combined_options.device_set = device_set_;
   combined_options.session_options = session_options_;
+  combined_options.session_handle = session_handle_;
   combined_options.stateful_placements = stateful_placements_;
 
   // NOTE(mrry): `gdef` is no longer valid after the constructor
@@ -558,6 +560,7 @@ Status GraphExecutionState::InitBaseGraph(const BuildGraphOptions& options) {
   RestoreStatefulNodes(new_graph.get());
 
   GraphOptimizationPassOptions optimization_options;
+  optimization_options.session_handle = session_handle_;
   optimization_options.session_options = session_options_;
   optimization_options.graph = &new_graph;
   optimization_options.flib_def = flib_def_.get();
diff --git a/tensorflow/core/common_runtime/graph_execution_state.h b/tensorflow/core/common_runtime/graph_execution_state.h
index 9cabe478a6..56315bb1ef 100644
--- a/tensorflow/core/common_runtime/graph_execution_state.h
+++ b/tensorflow/core/common_runtime/graph_execution_state.h
@@ -41,6 +41,8 @@ struct RewriteGraphMetadata;
 struct GraphExecutionStateOptions {
   const DeviceSet* device_set = nullptr;
   const SessionOptions* session_options = nullptr;
+  // Unique session identifier. Can be empty.
+  string session_handle;
   // A map from node name to device name, representing the unchangeable
   // placement of stateful nodes.
   std::unordered_map<string, string> stateful_placements;
@@ -192,6 +194,8 @@ class GraphExecutionState {
   GraphDef original_graph_def_;            // Immutable after ctor.
   const DeviceSet* device_set_;            // Not owned
   const SessionOptions* session_options_;  // Not owned
+  // Unique session identifier. Can be empty.
+  string session_handle_;
 
   // Map from name to Node for the full graph in placed_.
   NodeNameToCostIdMap node_name_to_cost_id_map_;
diff --git a/tensorflow/core/common_runtime/optimization_registry.h b/tensorflow/core/common_runtime/optimization_registry.h
index 6fcd2afd27..19a76529bb 100644
--- a/tensorflow/core/common_runtime/optimization_registry.h
+++ b/tensorflow/core/common_runtime/optimization_registry.h
@@ -35,6 +35,7 @@ struct SessionOptions;
 // as a key into a state dictionary if it wants to keep state across
 // calls.
 struct GraphOptimizationPassOptions {
+  // Filled in by DirectSession for PRE_PLACEMENT optimizations. Can be empty.
   string session_handle;
   const SessionOptions* session_options = nullptr;
   const CostModel* cost_model = nullptr;
diff --git a/tensorflow/core/framework/op_kernel.h b/tensorflow/core/framework/op_kernel.h
index bccb2bf3c7..aa07cbd380 100644
--- a/tensorflow/core/framework/op_kernel.h
+++ b/tensorflow/core/framework/op_kernel.h
@@ -606,6 +606,9 @@ class OpKernelContext {
     // The session state for this op.
     SessionState* session_state = nullptr;
 
+    // Unique session identifier. Can be empty.
+    string session_handle;
+
     // The tensor store for this op.
     TensorStore* tensor_store = nullptr;
 
@@ -1034,6 +1037,9 @@ class OpKernelContext {
   // An op kernel can access the session state it belongs to.
   SessionState* session_state() const { return params_->session_state; }
 
+  // Unique identifier of the session it belongs to. Can be empty.
+  string session_handle() const { return params_->session_handle; }
+
   // An op kernel can access the tensor store of the run it belongs to.
   TensorStore* tensor_store() const { return params_->tensor_store; }
 
-- 
GitLab


From 8c22259497e9914c37a7adcd33aebaf754473a02 Mon Sep 17 00:00:00 2001
From: Laurent Le Brun <laurentlb@gmail.com>
Date: Fri, 4 Jan 2019 20:37:31 +0100
Subject: [PATCH 0239/1095] Update dependency on rules_closure

This makes this command succeed:
  bazel build //tensorflow/tools/pip_package:build_pip_package --incompatible_disallow_data_transition

This is needed for a future change in Bazel.
---
 WORKSPACE | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/WORKSPACE b/WORKSPACE
index 2277e83a3f..1c59686f16 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -4,11 +4,11 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file"
 
 http_archive(
     name = "io_bazel_rules_closure",
-    sha256 = "a38539c5b5c358548e75b44141b4ab637bba7c4dc02b46b1f62a96d6433f56ae",
-    strip_prefix = "rules_closure-dbb96841cc0a5fb2664c37822803b06dab20c7d1",
+    sha256 = "43c9b882fa921923bcba764453f4058d102bece35a37c9f6383c713004aacff1",
+    strip_prefix = "rules_closure-9889e2348259a5aad7e805547c1a0cf311cfcd91",
     urls = [
-        "https://mirror.bazel.build/github.com/bazelbuild/rules_closure/archive/dbb96841cc0a5fb2664c37822803b06dab20c7d1.tar.gz",
-        "https://github.com/bazelbuild/rules_closure/archive/dbb96841cc0a5fb2664c37822803b06dab20c7d1.tar.gz",  # 2018-04-13
+        "https://mirror.bazel.build/github.com/bazelbuild/rules_closure/archive/9889e2348259a5aad7e805547c1a0cf311cfcd91.tar.gz",
+        "https://github.com/bazelbuild/rules_closure/archive/9889e2348259a5aad7e805547c1a0cf311cfcd91.tar.gz",  # 2018-12-21
     ],
 )
 
-- 
GitLab


From 3e37a58b6615582cca0006be22d95edf4945f078 Mon Sep 17 00:00:00 2001
From: Pete Warden <petewarden@google.com>
Date: Fri, 4 Jan 2019 11:44:16 -0800
Subject: [PATCH 0240/1095] Update documentation for Apollo3 board

PiperOrigin-RevId: 227889248
---
 tensorflow/lite/experimental/micro/README.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/experimental/micro/README.md b/tensorflow/lite/experimental/micro/README.md
index dab7ae80f3..464c7b6ad7 100644
--- a/tensorflow/lite/experimental/micro/README.md
+++ b/tensorflow/lite/experimental/micro/README.md
@@ -187,7 +187,7 @@ You should see the following log with the magic string `~~~ALL TEST PASSED~~~`:
 02:25:22.4253 [DEBUG] uart0: [+0.16ms host +0s virt 0.28s virt from start]   Progam has exited with code:0x00000000
 ```
 
-## Building for Apollo3
+## Building for Ambiq Micro Apollo3Blue EVB
 
 Follow these steps to get the pushbutton yes/no example working on Apollo 3:
 
@@ -201,7 +201,8 @@ Follow these steps to get the pushbutton yes/no example working on Apollo 3:
     tensorflow/lite/experimental/micro/tools/make/Makefile TARGET=apollo3evb
     pushbutton_cmsis_speech_test_bin
 4.  Install [Segger JLink tools](https://www.segger.com/downloads/jlink/)
-5.  Connect the Apollo3 EVB (with mic shield) to the computer and power it on
+5.  Connect the Apollo3 EVB (with mic shield in slot 3 of Microbus Shield board)
+    to the computer and power it on.
 6.  Start the GDB server in a new terminal with the following command:
     JLinkGDBServer -select USB -device AMA3B1KK-KBR -endian little -if SWD
     -speed 1000 -noir -noLocalhostOnly
-- 
GitLab


From 37afc40b6f33073193114c081bcda939289deca0 Mon Sep 17 00:00:00 2001
From: Tim Shen <timshen@google.com>
Date: Fri, 4 Jan 2019 11:57:01 -0800
Subject: [PATCH 0241/1095] Roll-forward with fix: Fix dso_loader.cc's
 includes.

tf_custom_op_library already adds core:stream_executor_headers_lib as a dep, so we must not add a duplicated dep here.

PiperOrigin-RevId: 227891141
---
 tensorflow/contrib/mpi_collectives/BUILD                  | 1 -
 tensorflow/stream_executor/platform/default/dso_loader.cc | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/tensorflow/contrib/mpi_collectives/BUILD b/tensorflow/contrib/mpi_collectives/BUILD
index ecac06354d..a7be92a35e 100644
--- a/tensorflow/contrib/mpi_collectives/BUILD
+++ b/tensorflow/contrib/mpi_collectives/BUILD
@@ -52,7 +52,6 @@ tf_custom_op_library(
     deps = [
         ":mpi_defines",
         ":mpi_message_proto_cc",
-        "//tensorflow/stream_executor:stream_executor_headers_lib",
         "//third_party/mpi",
     ],
 )
diff --git a/tensorflow/stream_executor/platform/default/dso_loader.cc b/tensorflow/stream_executor/platform/default/dso_loader.cc
index 0f0bce3253..668eeee3f3 100644
--- a/tensorflow/stream_executor/platform/default/dso_loader.cc
+++ b/tensorflow/stream_executor/platform/default/dso_loader.cc
@@ -28,7 +28,7 @@ limitations under the License.
 #include "tensorflow/stream_executor/lib/path.h"
 #include "tensorflow/stream_executor/lib/str_util.h"
 #include "tensorflow/stream_executor/lib/stringprintf.h"
-#include "tensorflow/stream_executor/platform/dso_loader.h"
+#include "tensorflow/stream_executor/platform/default/dso_loader.h"
 #include "tensorflow/stream_executor/platform/logging.h"
 #include "tensorflow/stream_executor/platform/port.h"
 
-- 
GitLab


From 45b2b27f6f30ec981300fb6987736f14caa45f29 Mon Sep 17 00:00:00 2001
From: Yuxin Wu <ppwwyyxx@users.noreply.github.com>
Date: Fri, 4 Jan 2019 12:17:10 -0800
Subject: [PATCH 0242/1095] improve docs of depthwise conv

---
 tensorflow/python/ops/nn_impl.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/ops/nn_impl.py b/tensorflow/python/ops/nn_impl.py
index ec5ea44526..7abfde5149 100644
--- a/tensorflow/python/ops/nn_impl.py
+++ b/tensorflow/python/ops/nn_impl.py
@@ -467,7 +467,7 @@ def depthwise_conv2d(input,
   to `channel_multiplier` channels for each), then concatenates the results
   together.  The output has `in_channels * channel_multiplier` channels.
 
-  In detail,
+  In detail, with the default NHWC format,
 
       output[b, i, j, k * channel_multiplier + q] = sum_{di, dj}
            filter[di, dj, k, q] * input[b, strides[1] * i + rate[0] * di,
@@ -540,7 +540,7 @@ def depthwise_conv2d_v2(input,
   to `channel_multiplier` channels for each), then concatenates the results
   together.  The output has `in_channels * channel_multiplier` channels.
 
-  In detail,
+  In detail, with the default NHWC format,
 
       output[b, i, j, k * channel_multiplier + q] = sum_{di, dj}
            filter[di, dj, k, q] * input[b, strides[1] * i + rate[0] * di,
@@ -599,7 +599,7 @@ def separable_conv2d(input,
   between dimensions `[1, 2]` and `3`, not spatial separability between
   dimensions `1` and `2`.
 
-  In detail,
+  In detail, with the default NHWC format,
 
       output[b, i, j, k] = sum_{di, dj, q, r}
           input[b, strides[1] * i + di, strides[2] * j + dj, q] *
@@ -699,7 +699,7 @@ def separable_conv2d_v2(
   between dimensions `[1, 2]` and `3`, not spatial separability between
   dimensions `1` and `2`.
 
-  In detail,
+  In detail, with the default NHWC format,
 
       output[b, i, j, k] = sum_{di, dj, q, r}
           input[b, strides[1] * i + di, strides[2] * j + dj, q] *
-- 
GitLab


From f1d4d18a622a3bb030ebc92b60763fe35a755bd5 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Fri, 4 Jan 2019 12:17:54 -0800
Subject: [PATCH 0243/1095] Add logcosh v2 loss and metric.

PiperOrigin-RevId: 227894520
---
 tensorflow/python/keras/losses.py       | 27 +++++++++
 tensorflow/python/keras/losses_test.py  | 81 +++++++++++++++++++++++++
 tensorflow/python/keras/metrics.py      | 31 ++++++++++
 tensorflow/python/keras/metrics_test.py | 43 +++++++++++++
 4 files changed, 182 insertions(+)

diff --git a/tensorflow/python/keras/losses.py b/tensorflow/python/keras/losses.py
index 6a428b28a9..1ff564054b 100644
--- a/tensorflow/python/keras/losses.py
+++ b/tensorflow/python/keras/losses.py
@@ -540,6 +540,33 @@ class LogLoss(Loss):
     return logloss(y_true, y_pred, epsilon=self.epsilon)
 
 
+class Logcosh(Loss):
+  """Computes the logarithm of the hyperbolic cosine of the prediction error.
+
+  logcosh = log((exp(x) + exp(-x))/2) where x is the error `y_pred` - `y_true`.
+
+  Usage:
+
+  ```python
+  l = tf.losses.Logcosh()
+  loss = l([0., 1., 1.], [1., 0., 1.])
+  print('Loss: ', loss.numpy())  # Loss: 0.289
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile('sgd', loss=tf.losses.Logcosh())
+  ```
+  """
+
+  def call(self, y_true, y_pred):
+    y_pred = ops.convert_to_tensor(y_pred)
+    y_true = math_ops.cast(y_true, y_pred.dtype)
+    return logcosh(y_true, y_pred)
+
+
 @keras_export('keras.metrics.mean_squared_error',
               'keras.metrics.mse',
               'keras.metrics.MSE',
diff --git a/tensorflow/python/keras/losses_test.py b/tensorflow/python/keras/losses_test.py
index 76fadc4a24..aa00ba5a91 100644
--- a/tensorflow/python/keras/losses_test.py
+++ b/tensorflow/python/keras/losses_test.py
@@ -1094,5 +1094,86 @@ class LogLossTest(test.TestCase):
     self.assertAlmostEqual(self.evaluate(loss), 0., 3)
 
 
+@test_util.run_all_in_graph_and_eager_modes
+class LogcoshTest(test.TestCase):
+
+  def setup(self):
+    y_pred = np.asarray([1, 9, 2, -5, -2, 6]).reshape((2, 3))
+    y_true = np.asarray([4, 8, 12, 8, 1, 3]).reshape((2, 3))
+
+    self.batch_size = 6
+    error = y_pred - y_true
+    self.expected_losses = np.log((np.exp(error) + np.exp(-error)) / 2)
+
+    self.y_pred = constant_op.constant(y_pred, dtype=dtypes.float32)
+    self.y_true = constant_op.constant(y_true)
+
+  def test_config(self):
+    logcosh_obj = keras.losses.Logcosh(
+        reduction=losses_impl.ReductionV2.SUM, name='logcosh_loss')
+    self.assertEqual(logcosh_obj.name, 'logcosh_loss')
+    self.assertEqual(logcosh_obj.reduction, losses_impl.ReductionV2.SUM)
+
+  def test_unweighted(self):
+    self.setup()
+    logcosh_obj = keras.losses.Logcosh()
+
+    loss = logcosh_obj(self.y_true, self.y_pred)
+    expected_loss = np.sum(self.expected_losses) / self.batch_size
+    self.assertAlmostEqual(self.evaluate(loss), expected_loss, 3)
+
+  def test_scalar_weighted(self):
+    self.setup()
+    logcosh_obj = keras.losses.Logcosh()
+    sample_weight = 2.3
+
+    loss = logcosh_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+    expected_loss = sample_weight * np.sum(
+        self.expected_losses) / self.batch_size
+    self.assertAlmostEqual(self.evaluate(loss), expected_loss, 3)
+
+    # Verify we get the same output when the same input is given
+    loss_2 = logcosh_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+    self.assertAlmostEqual(self.evaluate(loss), self.evaluate(loss_2), 3)
+
+  def test_sample_weighted(self):
+    self.setup()
+    logcosh_obj = keras.losses.Logcosh()
+
+    sample_weight = constant_op.constant([1.2, 3.4], shape=(2, 1))
+    loss = logcosh_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+
+    expected_loss = np.multiply(
+        self.expected_losses,
+        np.asarray([1.2, 1.2, 1.2, 3.4, 3.4, 3.4]).reshape((2, 3)))
+    expected_loss = np.sum(expected_loss) / self.batch_size
+    self.assertAlmostEqual(self.evaluate(loss), expected_loss, 3)
+
+  def test_timestep_weighted(self):
+    self.setup()
+    logcosh_obj = keras.losses.Logcosh()
+    y_true = np.asarray([1, 9, 2, -5, -2, 6]).reshape(2, 3, 1)
+    y_pred = np.asarray([4, 8, 12, 8, 1, 3]).reshape(2, 3, 1)
+    error = y_pred - y_true
+    expected_losses = np.log((np.exp(error) + np.exp(-error)) / 2)
+    sample_weight = np.array([3, 6, 5, 0, 4, 2]).reshape((2, 3, 1))
+
+    y_pred = constant_op.constant(y_pred, dtype=dtypes.float32)
+    y_true = constant_op.constant(y_true)
+    loss = logcosh_obj(
+        y_true,
+        y_pred,
+        sample_weight=constant_op.constant(sample_weight, shape=(2, 3)))
+    expected_loss = np.sum(expected_losses * sample_weight) / self.batch_size
+    self.assertAlmostEqual(self.evaluate(loss), expected_loss, 3)
+
+  def test_zero_weighted(self):
+    self.setup()
+    logcosh_obj = keras.losses.Logcosh()
+    sample_weight = 0
+    loss = logcosh_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+    self.assertAlmostEqual(self.evaluate(loss), 0., 3)
+
+
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index 7f13cc46e3..2de0bc474c 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -1654,6 +1654,37 @@ class RootMeanSquaredError(Mean):
     return math_ops.sqrt(math_ops.div_no_nan(self.total, self.count))
 
 
+class Logcosh(MeanMetricWrapper):
+  """Computes the logarithm of the hyperbolic cosine of the prediction error.
+
+  logcosh = log((exp(x) + exp(-x))/2) where x is the error `y_pred` - `y_true`.
+
+  Usage:
+
+  ```python
+  m = tf.keras.metrics.Logcosh()
+  m.update_state([0., 1., 1.], [1., 0., 1.])
+  print('Final result: ', m.result().numpy())  # Final result: 0.289
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile('sgd', metrics=[tf.keras.metrics.Logcosh()])
+  ```
+  """
+
+  def __init__(self, name='logcosh', dtype=None):
+    super(Logcosh, self).__init__(logcosh, name, dtype=dtype)
+
+  @classmethod
+  def from_config(cls, config):
+    if 'fn' in config:
+      config.pop('fn')
+    return super(Logcosh, cls).from_config(config)
+
+
 def accuracy(y_true, y_pred):
   y_pred.get_shape().assert_is_compatible_with(y_true.get_shape())
   if y_true.dtype != y_pred.dtype:
diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py
index fb635e7055..a9d88789ed 100644
--- a/tensorflow/python/keras/metrics_test.py
+++ b/tensorflow/python/keras/metrics_test.py
@@ -1467,6 +1467,49 @@ class SparseTopKCategoricalAccuracyTest(test.TestCase):
     self.assertEqual(0.5, self.evaluate(result))  # only 1 sample matches.
 
 
+@test_util.run_all_in_graph_and_eager_modes
+class LogcoshTest(test.TestCase):
+
+  def setup(self):
+    y_pred = np.asarray([1, 9, 2, -5, -2, 6]).reshape((2, 3))
+    y_true = np.asarray([4, 8, 12, 8, 1, 3]).reshape((2, 3))
+
+    self.batch_size = 6
+    error = y_pred - y_true
+    self.expected_results = np.log((np.exp(error) + np.exp(-error)) / 2)
+
+    self.y_pred = constant_op.constant(y_pred, dtype=dtypes.float32)
+    self.y_true = constant_op.constant(y_true)
+
+  def test_config(self):
+    logcosh_obj = metrics.Logcosh(name='logcosh', dtype=dtypes.int32)
+    self.assertEqual(logcosh_obj.name, 'logcosh')
+    self.assertEqual(logcosh_obj._dtype, dtypes.int32)
+
+  def test_unweighted(self):
+    self.setup()
+    logcosh_obj = metrics.Logcosh()
+    self.evaluate(variables.variables_initializer(logcosh_obj.variables))
+
+    update_op = logcosh_obj.update_state(self.y_true, self.y_pred)
+    self.evaluate(update_op)
+    result = logcosh_obj.result()
+    expected_result = np.sum(self.expected_results) / self.batch_size
+    self.assertAllClose(result, expected_result, atol=1e-3)
+
+  def test_weighted(self):
+    self.setup()
+    logcosh_obj = metrics.Logcosh()
+    self.evaluate(variables.variables_initializer(logcosh_obj.variables))
+    sample_weight = constant_op.constant([1.2, 3.4], shape=(2, 1))
+    result = logcosh_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+
+    sample_weight = np.asarray([1.2, 1.2, 1.2, 3.4, 3.4, 3.4]).reshape((2, 3))
+    expected_result = np.multiply(self.expected_results, sample_weight)
+    expected_result = np.sum(expected_result) / np.sum(sample_weight)
+    self.assertAllClose(self.evaluate(result), expected_result, atol=1e-3)
+
+
 def _get_model(compile_metrics):
   model_layers = [
       layers.Dense(3, activation='relu', kernel_initializer='ones'),
-- 
GitLab


From f9bd1568aa6f0e0b8c40c10d73caeca80c3c19ab Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Fri, 4 Jan 2019 12:18:26 -0800
Subject: [PATCH 0244/1095] [XLA] Use absl::flat_hash_{map,set}::contains()
 instead of count().

contains() is a much better description of what we're trying to do!

Also change a few std containers over to absl containers so we can advantage of
this.

PiperOrigin-RevId: 227894609
---
 tensorflow/compiler/xla/BUILD                 |  1 +
 tensorflow/compiler/xla/client/xla_builder.cc |  8 +++--
 tensorflow/compiler/xla/client/xla_builder.h  |  3 +-
 tensorflow/compiler/xla/reference_util.cc     | 25 ++++++++------
 tensorflow/compiler/xla/service/BUILD         |  9 +++++
 .../xla/service/algebraic_simplifier.cc       |  5 +--
 tensorflow/compiler/xla/service/backend.cc    |  8 ++---
 tensorflow/compiler/xla/service/backend.h     |  4 ++-
 .../compiler/xla/service/buffer_assignment.cc | 34 +++++++------------
 .../compiler/xla/service/buffer_assignment.h  |  7 ++--
 .../xla/service/buffer_assignment_test.cc     |  7 ++--
 .../compiler/xla/service/channel_tracker.cc   |  4 +--
 .../compiler/xla/service/channel_tracker.h    |  4 ++-
 .../xla/service/cpu/cpu_layout_assignment.cc  |  3 +-
 .../compiler/xla/service/cpu/ir_emitter.cc    | 12 +++----
 .../compiler/xla/service/cpu/ir_emitter.h     |  2 +-
 tensorflow/compiler/xla/service/gpu/BUILD     |  7 +++-
 .../xla/service/gpu/buffer_allocations.cc     | 11 +++---
 .../xla/service/gpu/buffer_allocations.h      |  4 ++-
 .../xla/service/gpu/hlo_to_ir_bindings.cc     |  7 ++--
 .../xla/service/gpu/hlo_to_ir_bindings.h      |  6 ++--
 .../xla/service/gpu/multi_output_fusion.cc    |  2 +-
 .../xla/service/gpu/stream_assignment.cc      |  9 ++---
 .../xla/service/gpu/thunk_schedule.cc         | 16 +++++----
 .../compiler/xla/service/gpu/thunk_schedule.h | 12 ++++---
 .../compiler/xla/service/heap_simulator.cc    | 22 ++++++------
 .../compiler/xla/service/hlo_computation.cc   | 15 ++++----
 .../xla/service/hlo_computation_test.cc       |  5 +--
 .../xla/service/hlo_dataflow_analysis.cc      |  2 +-
 tensorflow/compiler/xla/service/hlo_dce.cc    |  5 +--
 .../compiler/xla/service/hlo_graph_dumper.cc  | 16 ++++-----
 .../xla/service/hlo_instruction_test.cc       | 33 +++++++++---------
 .../xla/service/hlo_liveness_analysis.cc      |  5 +--
 tensorflow/compiler/xla/service/hlo_module.cc | 29 +++++++---------
 .../xla/service/hlo_module_group_metadata.cc  |  4 +--
 .../xla/service/hlo_module_group_metadata.h   |  2 +-
 .../compiler/xla/service/hlo_ordering.cc      |  2 +-
 .../compiler/xla/service/hlo_pass_pipeline.cc |  2 +-
 .../compiler/xla/service/hlo_schedule.cc      | 10 +++---
 .../compiler/xla/service/hlo_schedule.h       |  2 +-
 .../compiler/xla/service/hlo_sharding.cc      |  5 +--
 .../xla/service/hlo_sharding_metadata.cc      |  4 +--
 .../xla/service/indexed_array_analysis.cc     |  6 ++--
 .../compiler/xla/service/layout_assignment.cc |  2 +-
 .../compiler/xla/service/layout_assignment.h  |  4 +--
 tensorflow/compiler/xla/service/llvm_ir/BUILD |  1 +
 .../xla/service/llvm_ir/alias_analysis.h      |  7 ++--
 .../xla/service/llvm_ir/fused_ir_emitter.cc   |  2 +-
 .../xla/service/llvm_ir/fused_ir_emitter.h    |  6 ++--
 .../xla/service/multi_output_fusion.cc        |  2 +-
 .../xla/service/tuple_points_to_analysis.cc   |  8 ++---
 .../while_loop_invariant_code_motion.cc       |  4 +--
 .../xla/service/while_loop_simplifier.cc      |  2 +-
 53 files changed, 225 insertions(+), 192 deletions(-)

diff --git a/tensorflow/compiler/xla/BUILD b/tensorflow/compiler/xla/BUILD
index 722d137668..8a0e2db2b5 100644
--- a/tensorflow/compiler/xla/BUILD
+++ b/tensorflow/compiler/xla/BUILD
@@ -741,6 +741,7 @@ cc_library(
         "//tensorflow/compiler/xla/service:hlo_evaluator",
         "//tensorflow/compiler/xla/service:shape_inference",
         "//tensorflow/core:lib",
+        "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/types:span",
     ],
diff --git a/tensorflow/compiler/xla/client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_builder.cc
index 4b752def1a..433033cae6 100644
--- a/tensorflow/compiler/xla/client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_builder.cc
@@ -22,6 +22,8 @@ limitations under the License.
 #include <utility>
 
 #include "absl/algorithm/container.h"
+#include "absl/container/flat_hash_map.h"
+#include "absl/container/flat_hash_set.h"
 #include "absl/memory/memory.h"
 #include "absl/strings/match.h"
 #include "absl/strings/str_cat.h"
@@ -193,9 +195,9 @@ StatusOr<ProgramShape> XlaBuilder::GetProgramShape(XlaOp root) const {
 }
 
 void XlaBuilder::IsConstantVisitor(const int64 op_handle,
-                                   std::set<int64>* visited,
+                                   absl::flat_hash_set<int64>* visited,
                                    bool* is_constant) const {
-  if (visited->count(op_handle) != 0 || !*is_constant) {
+  if (visited->contains(op_handle) || !*is_constant) {
     return;
   }
 
@@ -2415,7 +2417,7 @@ StatusOr<bool> XlaBuilder::IsConstant(const XlaOp& operand) const {
   TF_RETURN_IF_ERROR(LookUpInstruction(operand).status());
 
   bool is_constant = true;
-  std::set<int64> visited;
+  absl::flat_hash_set<int64> visited;
   IsConstantVisitor(operand.handle(), &visited, &is_constant);
   return is_constant;
 }
diff --git a/tensorflow/compiler/xla/client/xla_builder.h b/tensorflow/compiler/xla/client/xla_builder.h
index 68ddf2cdd8..ebef8e0879 100644
--- a/tensorflow/compiler/xla/client/xla_builder.h
+++ b/tensorflow/compiler/xla/client/xla_builder.h
@@ -727,7 +727,8 @@ class XlaBuilder {
   // operation such as `RngNormal` or `Infeed`. The visitor walks the
   // computation starting at a given operation and sets is_constant to false iff
   // a parameter or stateful operation is encountered.
-  void IsConstantVisitor(const int64 op_handle, std::set<int64>* visited,
+  void IsConstantVisitor(const int64 op_handle,
+                         absl::flat_hash_set<int64>* visited,
                          bool* is_constant) const;
 
   // Checks bounds for convolution parameters.
diff --git a/tensorflow/compiler/xla/reference_util.cc b/tensorflow/compiler/xla/reference_util.cc
index 033f6c9b0f..08b78ee244 100644
--- a/tensorflow/compiler/xla/reference_util.cc
+++ b/tensorflow/compiler/xla/reference_util.cc
@@ -18,6 +18,7 @@ limitations under the License.
 #include <array>
 #include <utility>
 
+#include "absl/container/flat_hash_set.h"
 #include "absl/memory/memory.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/literal_util.h"
@@ -605,24 +606,26 @@ ReferenceUtil::ReduceToRowArray2D(
     const std::function<float(float, float)>& reduce_function) {
   std::vector<float> result;
   CHECK_EQ(dims.size(), 3);
-  const std::set<int64> dim_set(dims.begin(), dims.end());
+  const absl::flat_hash_set<int64> dim_set(dims.begin(), dims.end());
   CHECK_EQ(dim_set.size(), 3);
-  for (int64 a0 = 0; a0 == 0 || (!dim_set.count(0) && a0 < array.n1()); ++a0) {
-    for (int64 a1 = 0; a1 == 0 || (!dim_set.count(1) && a1 < array.n2());
+  for (int64 a0 = 0; a0 == 0 || (!dim_set.contains(0) && a0 < array.n1());
+       ++a0) {
+    for (int64 a1 = 0; a1 == 0 || (!dim_set.contains(1) && a1 < array.n2());
          ++a1) {
-      for (int64 a2 = 0; a2 == 0 || (!dim_set.count(2) && a2 < array.n3());
+      for (int64 a2 = 0; a2 == 0 || (!dim_set.contains(2) && a2 < array.n3());
            ++a2) {
-        for (int64 a3 = 0; a3 == 0 || (!dim_set.count(3) && a3 < array.n4());
+        for (int64 a3 = 0; a3 == 0 || (!dim_set.contains(3) && a3 < array.n4());
              ++a3) {
           float accumulator = init;
-          for (int64 i0 = 0; i0 == 0 || (dim_set.count(0) && i0 < array.n1());
-               ++i0) {
-            for (int64 i1 = 0; i1 == 0 || (dim_set.count(1) && i1 < array.n2());
-                 ++i1) {
+          for (int64 i0 = 0;
+               i0 == 0 || (dim_set.contains(0) && i0 < array.n1()); ++i0) {
+            for (int64 i1 = 0;
+                 i1 == 0 || (dim_set.contains(1) && i1 < array.n2()); ++i1) {
               for (int64 i2 = 0;
-                   i2 == 0 || (dim_set.count(2) && i2 < array.n3()); ++i2) {
+                   i2 == 0 || (dim_set.contains(2) && i2 < array.n3()); ++i2) {
                 for (int64 i3 = 0;
-                     i3 == 0 || (dim_set.count(3) && i3 < array.n4()); ++i3) {
+                     i3 == 0 || (dim_set.contains(3) && i3 < array.n4());
+                     ++i3) {
                   // Handle zero-sized arrays.
                   if (array.n1() > 0 && array.n2() > 0 && array.n3() > 0 &&
                       array.n4() > 0) {
diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index 755c477a12..85a21a8518 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -515,6 +515,7 @@ tf_cc_test(
         "//tensorflow/compiler/xla:window_util",
         "//tensorflow/compiler/xla/tests:hlo_test_base",
         "//tensorflow/compiler/xla/tests:xla_internal_test_main",
+        "@com_google_absl//absl/container:flat_hash_map",
     ],
 )
 
@@ -677,6 +678,7 @@ cc_library(
         "//tensorflow/core:lib",
         "//tensorflow/core:stream_executor_no_cuda",
         "//third_party/eigen3",
+        "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
         "@com_google_absl//absl/types:span",
@@ -1002,6 +1004,7 @@ cc_library(
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
         "@com_google_absl//absl/types:span",
@@ -1136,6 +1139,7 @@ tf_cc_test(
         "//tensorflow/compiler/xla/tests:xla_internal_test_main",
         "//tensorflow/core:lib",
         "//tensorflow/core:test",
+        "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/memory",
     ],
 )
@@ -1580,6 +1584,7 @@ cc_library(
         "//tensorflow/core:lib",
         "@com_google_absl//absl/algorithm:container",
         "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/container:inlined_vector",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
@@ -2116,6 +2121,7 @@ tf_cc_test(
         "//tensorflow/compiler/xla:test_helpers",
         "//tensorflow/compiler/xla/tests:hlo_test_base",
         "//tensorflow/compiler/xla/tests:xla_internal_test_main",
+        "@com_google_absl//absl/container:flat_hash_set",
     ],
 )
 
@@ -2288,6 +2294,7 @@ cc_library(
         "//tensorflow/compiler/xla:types",
         "//tensorflow/compiler/xla:util",
         "//tensorflow/core:lib",
+        "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
     ],
@@ -2548,6 +2555,7 @@ cc_library(
         "//tensorflow/compiler/xla:types",
         "//tensorflow/compiler/xla:util",
         "//tensorflow/core:lib",
+        "@com_google_absl//absl/container:flat_hash_set",
     ],
 )
 
@@ -3188,6 +3196,7 @@ cc_library(
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:regexp_internal",
+        "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/strings",
         "@com_google_absl//absl/strings:str_format",
         "@com_google_absl//absl/types:optional",
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
index c573b60161..3d269dc04f 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
@@ -27,6 +27,7 @@ limitations under the License.
 
 #include "absl/algorithm/container.h"
 #include "absl/container/flat_hash_map.h"
+#include "absl/container/flat_hash_set.h"
 #include "absl/container/inlined_vector.h"
 #include "absl/memory/memory.h"
 #include "absl/strings/str_cat.h"
@@ -2539,7 +2540,7 @@ Status AlgebraicSimplifierVisitor::HandleReduce(HloInstruction* reduce) {
     }
     if (can_move_reshape_into_reduce) {
       changed_ = true;
-      std::unordered_set<int64> dimensions_not_to_reduce;
+      absl::flat_hash_set<int64> dimensions_not_to_reduce;
       for (auto dim_pair : unmodified_dims) {
         if (arg_dim_in_output[dim_pair.second]) {
           dimensions_not_to_reduce.insert(dim_pair.first);
@@ -2547,7 +2548,7 @@ Status AlgebraicSimplifierVisitor::HandleReduce(HloInstruction* reduce) {
       }
       std::vector<int64> new_reduce_dimensions;
       for (int64 i = 0; i < arg->operand(0)->shape().rank(); ++i) {
-        if (dimensions_not_to_reduce.count(i) == 0) {
+        if (!dimensions_not_to_reduce.contains(i)) {
           new_reduce_dimensions.push_back(i);
         }
       }
diff --git a/tensorflow/compiler/xla/service/backend.cc b/tensorflow/compiler/xla/service/backend.cc
index 2cf24a9dd5..215e8ced4b 100644
--- a/tensorflow/compiler/xla/service/backend.cc
+++ b/tensorflow/compiler/xla/service/backend.cc
@@ -115,12 +115,10 @@ StatusOr<StreamPool::Ptr> Backend::BorrowStream(int device_ordinal) {
 
 StatusOr<StreamPool::Ptr> Backend::BorrowStream(se::StreamExecutor* executor) {
   tensorflow::mutex_lock l(mu_);
-  if (0 == stream_pools_.count(executor)) {
-    stream_pools_.emplace(std::piecewise_construct,
-                          std::forward_as_tuple(executor),
-                          std::forward_as_tuple());
+  if (!stream_pools_.contains(executor)) {
+    stream_pools_.emplace(executor, absl::make_unique<StreamPool>());
   }
-  return stream_pools_.at(executor).BorrowStream(executor);
+  return stream_pools_.at(executor)->BorrowStream(executor);
 }
 
 Backend::Backend(se::Platform* platform, Compiler* compiler,
diff --git a/tensorflow/compiler/xla/service/backend.h b/tensorflow/compiler/xla/service/backend.h
index 7ca993fb26..c35f033dc0 100644
--- a/tensorflow/compiler/xla/service/backend.h
+++ b/tensorflow/compiler/xla/service/backend.h
@@ -22,6 +22,7 @@ limitations under the License.
 #include <string>
 #include <vector>
 
+#include "absl/container/flat_hash_map.h"
 #include "absl/strings/str_cat.h"
 #include "absl/types/span.h"
 #include "tensorflow/compiler/xla/service/compiler.h"
@@ -175,7 +176,8 @@ class Backend {
   tensorflow::mutex mu_;
 
   // Mapping from stream executor to stream pools, used by `BorrowStream` above.
-  std::map<se::StreamExecutor*, StreamPool> stream_pools_ GUARDED_BY(mu_);
+  absl::flat_hash_map<se::StreamExecutor*, std::unique_ptr<StreamPool>>
+      stream_pools_ GUARDED_BY(mu_);
 
   // The default memory allocator to use.
   std::unique_ptr<StreamExecutorMemoryAllocator> memory_allocator_;
diff --git a/tensorflow/compiler/xla/service/buffer_assignment.cc b/tensorflow/compiler/xla/service/buffer_assignment.cc
index 6f4c1104f3..c997d594c4 100644
--- a/tensorflow/compiler/xla/service/buffer_assignment.cc
+++ b/tensorflow/compiler/xla/service/buffer_assignment.cc
@@ -138,8 +138,8 @@ Status GatherComputationsByAllocationType(
     worklist.pop_front();
     const HloComputation* computation = worklist_front.first;
     bool is_thread_local = worklist_front.second;
-    bool in_thread_local_set = thread_local_set.count(computation) > 0;
-    bool in_global_set = global_set.count(computation) > 0;
+    bool in_thread_local_set = thread_local_set.contains(computation);
+    bool in_global_set = global_set.contains(computation);
 
     // If the computation has already been added to the respective set, then
     // nothing to do.
@@ -207,9 +207,9 @@ Status GatherComputationsByAllocationType(
 
   // Add the computations to the vectors in post order.
   for (auto* computation : module->MakeComputationPostOrder()) {
-    if (thread_local_set.count(computation) > 0) {
+    if (thread_local_set.contains(computation)) {
       thread_local_computations->push_back(computation);
-    } else if (global_set.count(computation) > 0) {
+    } else if (global_set.contains(computation)) {
       global_computations->push_back(computation);
     }
     // If the computation is not reachable from the entry computation, then it
@@ -219,13 +219,6 @@ Status GatherComputationsByAllocationType(
   return Status::OK();
 }
 
-size_t BufferAllocation::Slice::Hasher::operator()(Slice s) const {
-  uint64 h = std::hash<int64>()(s.index());
-  h = tensorflow::Hash64Combine(h, std::hash<int64>()(s.offset()));
-  h = tensorflow::Hash64Combine(h, std::hash<int64>()(s.size()));
-  return h;
-}
-
 string BufferAllocation::Slice::ToString() const {
   return absl::StrCat("{index:", index(), ", offset:", offset_,
                       ", size:", size_, "}");
@@ -240,7 +233,7 @@ BufferAllocation::Slice BufferAllocation::GetSlice(
 void BufferAllocation::AddAssignment(const LogicalBuffer& buffer, int64 offset,
                                      int64 size) {
   VLOG(4) << "Trying to add " << buffer << " to allocation #" << index();
-  CHECK(assigned_buffers_.count(&buffer) == 0)
+  CHECK(!assigned_buffers_.contains(&buffer))
       << "LogicalBuffer " << buffer << " already assigned to allocation "
       << index_;
   CHECK_LE(offset, size_) << "LogicalBuffer " << buffer
@@ -346,7 +339,7 @@ const PointsToSet& BufferAssignment::GetPointsToSet(
 
 bool BufferAssignment::HasAllocation(const LogicalBuffer& buffer) const {
   TF_CHECK_OK(points_to_analysis().VerifyBuffer(buffer));
-  return allocation_index_for_buffer_.count(&buffer) > 0;
+  return allocation_index_for_buffer_.contains(&buffer);
 }
 
 const BufferAllocation& BufferAssignment::GetAssignedAllocation(
@@ -401,7 +394,7 @@ bool BufferAssignment::HasAllocationAt(const HloInstruction* instruction,
                                        const ShapeIndex& index) const {
   for (const LogicalBuffer* buffer :
        GetPointsToSet(instruction).element(index)) {
-    if (allocation_index_for_buffer_.count(buffer) > 0) {
+    if (allocation_index_for_buffer_.contains(buffer)) {
       return true;
     }
   }
@@ -459,8 +452,7 @@ bool BufferAssignment::SharesSliceAtIndex(
 
 bool BufferAssignment::HaveDisjointSlices(const HloInstruction* hlo_a,
                                           const HloInstruction* hlo_b) const {
-  using SliceSet =
-      flat_hash_set<BufferAllocation::Slice, BufferAllocation::Slice::Hasher>;
+  using SliceSet = flat_hash_set<BufferAllocation::Slice>;
   // Gets the slices all of instr's subshapes.  If any subshape doesn't have an
   // assigned slice, returns the empty set.
   auto collect_slices = [&](const HloInstruction* instr) -> SliceSet {
@@ -519,7 +511,7 @@ BufferAllocation* BufferAssignment::NewAllocation(const LogicalBuffer& buffer,
 void BufferAssignment::AddAssignment(BufferAllocation* allocation,
                                      const LogicalBuffer& buffer, int64 offset,
                                      int64 size) {
-  CHECK_EQ(0, allocation_index_for_buffer_.count(&buffer))
+  CHECK(!allocation_index_for_buffer_.contains(&buffer))
       << "LogicalBuffer " << buffer << " already has an allocation.";
   CHECK(allocation->is_reusable() || allocation->assigned_buffers().empty())
       << "Non-reusable allocation already assigned a buffer: "
@@ -988,7 +980,7 @@ Status BufferAssigner::AssignBuffersForComputation(
   std::vector<BufferAllocation::Index> allocation_indices;
   for (const LogicalBuffer* buffer : sorted_buffers) {
     VLOG(3) << "Assigning allocation to: " << *buffer;
-    if (colocated_buffers.count(buffer) > 0) {
+    if (colocated_buffers.contains(buffer)) {
       // Colocated buffers are currently assigned in an earlier pass.
       VLOG(3) << "Skipping colocated buffer: " << *buffer;
       continue;
@@ -1056,7 +1048,7 @@ Status BufferAssigner::AssignBuffersForComputation(
              assignment->GetAllSlices(operand, /*index=*/{})) {
           BufferAllocation* allocation =
               assignment->GetMutableAllocation(operand_slice.index());
-          if (colocated_allocations.count(allocation->index()) == 0) {
+          if (!colocated_allocations.contains(allocation->index())) {
             // TODO(b/32491382) Colocated buffers are currently assigned in an
             // earlier pass, and so can break the "increasing allocation size"
             // invariant in this function (causing this CHECK to fail). However,
@@ -1087,7 +1079,7 @@ Status BufferAssigner::AssignBuffersForComputation(
         // Instructions are iterated in increasing buffer size, so any
         // previously create allocation must be large enough to hold this
         // instruction's output (with the exception of colocated buffers).
-        if (colocated_allocations.count(allocation->index()) == 0) {
+        if (!colocated_allocations.contains(allocation->index())) {
           // TODO(b/32491382) Colocated buffers are currently assigned in an
           // earlier pass, and so can break the "increasing allocation size"
           // invariant in this function (causing this CHECK to fail). However,
@@ -1376,7 +1368,7 @@ void BufferAssigner::AddSetToColocatedBufferSets(
   std::vector<size_t> overlap_set_indices;
   for (size_t index = 0; index < colocated_buffer_sets->size(); ++index) {
     for (const LogicalBuffer* buffer : colocated_set) {
-      if ((*colocated_buffer_sets)[index].count(buffer) > 0) {
+      if ((*colocated_buffer_sets)[index].contains(buffer)) {
         VLOG(5) << "Found overlap with existing set on buffer "
                 << buffer->ToString() << "\n"
                 << ColocatedBufferSetsToString((*colocated_buffer_sets)[index],
diff --git a/tensorflow/compiler/xla/service/buffer_assignment.h b/tensorflow/compiler/xla/service/buffer_assignment.h
index 0a9fdede80..4baab9b6ad 100644
--- a/tensorflow/compiler/xla/service/buffer_assignment.h
+++ b/tensorflow/compiler/xla/service/buffer_assignment.h
@@ -186,9 +186,10 @@ class BufferAllocation {
              end > other.offset_;
     }
 
-    struct Hasher {
-      size_t operator()(Slice s) const;
-    };
+    template <typename H>
+    friend H AbslHashValue(H h, const Slice& s) {
+      return H::combine(std::move(h), s.index(), s.offset(), s.size());
+    }
 
     string ToString() const;
 
diff --git a/tensorflow/compiler/xla/service/buffer_assignment_test.cc b/tensorflow/compiler/xla/service/buffer_assignment_test.cc
index 8f482e6ba8..370fad7ffb 100644
--- a/tensorflow/compiler/xla/service/buffer_assignment_test.cc
+++ b/tensorflow/compiler/xla/service/buffer_assignment_test.cc
@@ -21,6 +21,7 @@ limitations under the License.
 #include <utility>
 #include <vector>
 
+#include "absl/container/flat_hash_set.h"
 #include "absl/memory/memory.h"
 #include "tensorflow/compiler/xla/literal.h"
 #include "tensorflow/compiler/xla/service/buffer_value.h"
@@ -309,7 +310,7 @@ class BufferAssignmentTest : public HloTestBase {
 static bool BuffersDistinct(const std::vector<const HloInstruction*>& a,
                             const std::vector<const HloInstruction*>& b,
                             const BufferAssignment& assignment) {
-  std::set<BufferAllocation::Slice> a_slices;
+  absl::flat_hash_set<BufferAllocation::Slice> a_slices;
   for (const HloInstruction* instruction : a) {
     if (assignment.HasTopLevelAllocation(instruction)) {
       a_slices.insert(
@@ -319,8 +320,8 @@ static bool BuffersDistinct(const std::vector<const HloInstruction*>& a,
 
   for (const HloInstruction* instruction : b) {
     if (assignment.HasTopLevelAllocation(instruction)) {
-      if (a_slices.count(assignment.GetUniqueTopLevelSlice(instruction)
-                             .ConsumeValueOrDie())) {
+      if (a_slices.contains(assignment.GetUniqueTopLevelSlice(instruction)
+                                .ConsumeValueOrDie())) {
         return false;
       }
     }
diff --git a/tensorflow/compiler/xla/service/channel_tracker.cc b/tensorflow/compiler/xla/service/channel_tracker.cc
index 3c2d1ae6d8..b517495f2e 100644
--- a/tensorflow/compiler/xla/service/channel_tracker.cc
+++ b/tensorflow/compiler/xla/service/channel_tracker.cc
@@ -72,7 +72,7 @@ ChannelHandle ChannelTracker::AllocateHandle(ChannelHandle::ChannelType type) {
 }
 
 Status ChannelTracker::RegisterSendInternal(const ChannelHandle& handle) {
-  if (opaque_to_channel_.count(handle.handle()) == 0) {
+  if (!opaque_to_channel_.contains(handle.handle())) {
     return NotFound("channel handle not found: %d", handle.handle());
   }
   Channel& channel = opaque_to_channel_[handle.handle()];
@@ -94,7 +94,7 @@ Status ChannelTracker::RegisterSendInternal(const ChannelHandle& handle) {
 }
 
 Status ChannelTracker::RegisterRecvInternal(const ChannelHandle& handle) {
-  if (opaque_to_channel_.count(handle.handle()) == 0) {
+  if (!opaque_to_channel_.contains(handle.handle())) {
     return NotFound("channel handle not found: %d", handle.handle());
   }
   Channel& channel = opaque_to_channel_[handle.handle()];
diff --git a/tensorflow/compiler/xla/service/channel_tracker.h b/tensorflow/compiler/xla/service/channel_tracker.h
index 52037bf9b5..89e17eba36 100644
--- a/tensorflow/compiler/xla/service/channel_tracker.h
+++ b/tensorflow/compiler/xla/service/channel_tracker.h
@@ -18,6 +18,7 @@ limitations under the License.
 
 #include <map>
 
+#include "absl/container/flat_hash_map.h"
 #include "absl/types/span.h"
 #include "tensorflow/compiler/xla/service/hlo_module.h"
 #include "tensorflow/compiler/xla/status.h"
@@ -83,7 +84,8 @@ class ChannelTracker {
 
   // Mapping from ChannelHandle value to the corresponding registered
   // Channel object.
-  std::map<int64, Channel> opaque_to_channel_ GUARDED_BY(channel_mutex_);
+  absl::flat_hash_map<int64, Channel> opaque_to_channel_
+      GUARDED_BY(channel_mutex_);
 
   TF_DISALLOW_COPY_AND_ASSIGN(ChannelTracker);
 };
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_layout_assignment.cc b/tensorflow/compiler/xla/service/cpu/cpu_layout_assignment.cc
index 1cb154e9ca..ff11409482 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_layout_assignment.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_layout_assignment.cc
@@ -46,8 +46,7 @@ static bool ShouldMakeAllUsersColMajor(const HloInstruction* instruction) {
   for (auto* user : instruction->users()) {
     optional<int64> operand_idx = ProfitableToMakeDotOperandColumnMajor(*user);
     if (!operand_idx || user->operand(*operand_idx) != instruction ||
-        std::count(user->operands().begin(), user->operands().end(),
-                   instruction) != 1) {
+        absl::c_count(user->operands(), instruction) != 1) {
       return false;
     }
   }
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
index 169d628923..e4760aedf1 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
@@ -24,11 +24,9 @@ limitations under the License.
 #include <utility>
 #include <vector>
 
+// IWYU pragma: no_include "llvm/IR/Intrinsics.gen.inc"
 #include "absl/container/flat_hash_map.h"
 #include "absl/container/flat_hash_set.h"
-#include "tensorflow/core/lib/math/math_util.h"
-#include "tensorflow/core/platform/logging.h"
-// IWYU pragma: no_include "llvm/IR/Intrinsics.gen.inc"
 #include "absl/strings/str_cat.h"
 #include "absl/strings/str_format.h"
 #include "absl/types/span.h"
@@ -70,6 +68,8 @@ limitations under the License.
 #include "tensorflow/compiler/xla/window_util.h"
 #include "tensorflow/core/lib/core/bits.h"
 #include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/math/math_util.h"
+#include "tensorflow/core/platform/logging.h"
 
 namespace xla {
 
@@ -1399,7 +1399,7 @@ static bool ReductionPreservesLayout(const HloInstruction& reduce) {
 
   int64 delta = 0;
   for (int64 i = 0; i < operand_shape.dimensions_size(); i++) {
-    if (reduced_dims.count(i)) {
+    if (reduced_dims.contains(i)) {
       delta++;
     } else {
       InsertOrDie(&unreduced_dim_map, i, i - delta);
@@ -1412,7 +1412,7 @@ static bool ReductionPreservesLayout(const HloInstruction& reduce) {
   for (int64 operand_dim_idx = 0;
        operand_dim_idx < operand_shape.dimensions_size(); operand_dim_idx++) {
     int64 operand_dim = operand_shape.layout().minor_to_major(operand_dim_idx);
-    if (!reduced_dims.count(operand_dim)) {
+    if (!reduced_dims.contains(operand_dim)) {
       if (FindOrDie(unreduced_dim_map, operand_dim) !=
           result_shape.layout().minor_to_major(result_dim_idx++)) {
         return false;
@@ -1990,7 +1990,7 @@ Status IrEmitter::HandleSlice(HloInstruction* slice) {
   // The memcpy will copy elements that are logically this shape (allowed to be
   // scalar).
   const Shape logical_element_shape = ShapeUtil::FilterDimensions(
-      [&inner_dims](int64 dim) -> bool { return inner_dims.count(dim); },
+      [&inner_dims](int64 dim) { return inner_dims.contains(dim); },
       operand->shape());
 
   const int64 primitive_elements_per_logical_element =
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.h b/tensorflow/compiler/xla/service/cpu/ir_emitter.h
index db76de4bb2..a6fb11dcbf 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.h
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.h
@@ -448,7 +448,7 @@ class IrEmitter : public DfsHloVisitorWithDefault,
       computation_to_profile_idx_;
 
   // Maps HLOs to Values emitted for them.
-  std::unordered_map<const HloInstruction*, llvm::Value*> emitted_value_;
+  absl::flat_hash_map<const HloInstruction*, llvm::Value*> emitted_value_;
 
   llvm_ir::AliasAnalysis alias_analysis_;
 
diff --git a/tensorflow/compiler/xla/service/gpu/BUILD b/tensorflow/compiler/xla/service/gpu/BUILD
index 6c23f921f4..d10a11d096 100644
--- a/tensorflow/compiler/xla/service/gpu/BUILD
+++ b/tensorflow/compiler/xla/service/gpu/BUILD
@@ -94,8 +94,8 @@ cc_library(
         "//tensorflow/compiler/xla:util",
         "//tensorflow/compiler/xla/service:hlo",
         "//tensorflow/compiler/xla/service:hlo_reachability",
-        "//tensorflow/core:lib",
         "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/memory",
     ],
 )
@@ -135,6 +135,8 @@ cc_library(
         "//tensorflow/compiler/xla/service/llvm_ir:llvm_util",
         "//tensorflow/compiler/xla/service/llvm_ir:tuple_ops",
         "//tensorflow/core:lib",
+        "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/strings",
         "@com_google_absl//absl/types:span",
         "@llvm//:core",
@@ -263,7 +265,9 @@ cc_library(
         "//tensorflow/compiler/xla/service:buffer_assignment",
         "//tensorflow/compiler/xla/service:device_memory_allocator",
         "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
         "//tensorflow/core:stream_executor_no_cuda",
+        "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/types:span",
     ],
@@ -362,6 +366,7 @@ cc_library(
         "//tensorflow/core/platform/default/build_config:stream_executor_cuda",  # build_cleaner: keep
         "//tensorflow/stream_executor",
         "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
         "@com_google_absl//absl/strings:str_format",
diff --git a/tensorflow/compiler/xla/service/gpu/buffer_allocations.cc b/tensorflow/compiler/xla/service/gpu/buffer_allocations.cc
index 528209abc7..eb59ee5a1d 100644
--- a/tensorflow/compiler/xla/service/gpu/buffer_allocations.cc
+++ b/tensorflow/compiler/xla/service/gpu/buffer_allocations.cc
@@ -24,6 +24,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/types.h"
 #include "tensorflow/compiler/xla/util.h"
 #include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/gtl/map_util.h"
 #include "tensorflow/core/lib/strings/numbers.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/types.h"
@@ -57,16 +58,16 @@ StatusOr<std::unique_ptr<BufferAllocations>> BufferAllocations::Builder::Build(
 
     // If buffer #i's address is already registered (e.g. external arguments or
     // result buffers), use that registered buffer.
-    if (registered_buffers_.count(i)) {
-      se::DeviceMemoryBase address = FindOrDie(registered_buffers_, i);
-      if (reinterpret_cast<uintptr_t>(address.opaque()) % expected_alignment !=
+    if (se::DeviceMemoryBase* address =
+            tensorflow::gtl::FindOrNull(registered_buffers_, i)) {
+      if (reinterpret_cast<uintptr_t>(address->opaque()) % expected_alignment !=
           0) {
         return InternalError(
             "Address of registered buffer %d must be a multiple of %x, but "
             "was %p",
-            i, kEntryParameterAlignBytes, address.opaque());
+            i, kEntryParameterAlignBytes, address->opaque());
       }
-      buffer_allocations->SetBuffer(i, FindOrDie(registered_buffers_, i));
+      buffer_allocations->SetBuffer(i, *address);
       continue;
     }
 
diff --git a/tensorflow/compiler/xla/service/gpu/buffer_allocations.h b/tensorflow/compiler/xla/service/gpu/buffer_allocations.h
index 14186b8faa..9413ac2cff 100644
--- a/tensorflow/compiler/xla/service/gpu/buffer_allocations.h
+++ b/tensorflow/compiler/xla/service/gpu/buffer_allocations.h
@@ -20,6 +20,7 @@ limitations under the License.
 #include <set>
 #include <vector>
 
+#include "absl/container/flat_hash_map.h"
 #include "absl/types/span.h"
 #include "tensorflow/compiler/xla/service/buffer_assignment.h"
 #include "tensorflow/compiler/xla/service/device_memory_allocator.h"
@@ -52,7 +53,8 @@ class BufferAllocations {
         DeviceMemoryAllocator* memory_allocator);
 
    private:
-    std::map<BufferAllocation::Index, se::DeviceMemoryBase> registered_buffers_;
+    absl::flat_hash_map<BufferAllocation::Index, se::DeviceMemoryBase>
+        registered_buffers_;
   };
 
   ~BufferAllocations();
diff --git a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc
index 9634c92786..69aaaceca1 100644
--- a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc
+++ b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.h"
 
+#include "absl/container/flat_hash_set.h"
 #include "absl/strings/str_cat.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/Function.h"
@@ -45,10 +46,10 @@ void HloToIrBindings::EmitBasePointersForHlos(
 
   // An HLO can have duplicated operands. This data structure remembers which
   // operand HLOs are already bound to avoid rebinding the same HLO.
-  std::set<const HloInstruction*> already_bound_for_this_function;
+  absl::flat_hash_set<const HloInstruction*> already_bound_for_this_function;
   auto arg_iter = function->arg_begin();
   for (const HloInstruction* io_hlo : io_hlos) {
-    if (!already_bound_for_this_function.count(io_hlo)) {
+    if (!already_bound_for_this_function.contains(io_hlo)) {
       if (!is_nested_ && io_hlo->opcode() == HloOpcode::kGetTupleElement) {
         BindHloToIrValue(*io_hlo, EmitGetTupleElement(io_hlo, &*arg_iter));
       } else {
@@ -63,7 +64,7 @@ void HloToIrBindings::EmitBasePointersForHlos(
   temp_buffer_base_->setName("temp_buffer");
 
   for (const HloInstruction* non_io_hlo : non_io_hlos) {
-    if (already_bound_for_this_function.count(non_io_hlo)) {
+    if (already_bound_for_this_function.contains(non_io_hlo)) {
       continue;
     }
     already_bound_for_this_function.insert(non_io_hlo);
diff --git a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.h b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.h
index c0edae530c..f57b594e9c 100644
--- a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.h
+++ b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.h
@@ -18,6 +18,7 @@ limitations under the License.
 
 #include <unordered_map>
 
+#include "absl/container/flat_hash_map.h"
 #include "absl/types/span.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Value.h"
@@ -61,7 +62,7 @@ class HloToIrBindings {
 
   // Returns whether `hlo` is bound to an LLVM IR value.
   bool BoundToIrValue(const HloInstruction& hlo) const {
-    return base_ptrs_.count(&hlo);
+    return base_ptrs_.contains(&hlo);
   }
 
   llvm::Value* GetTempBufferBase() const { return temp_buffer_base_; }
@@ -110,7 +111,8 @@ class HloToIrBindings {
   // For an instruction that generates multiple outputs, the root will be a
   // tuple shape. The IrArray for each element output is stored in the subnode
   // in the ShapeTree.
-  std::unordered_map<const HloInstruction*, ShapeTree<llvm::Value*>> base_ptrs_;
+  absl::flat_hash_map<const HloInstruction*, ShapeTree<llvm::Value*>>
+      base_ptrs_;
 
   // The address of the memory block that contains all temporary buffers.
   llvm::Value* temp_buffer_base_ = nullptr;
diff --git a/tensorflow/compiler/xla/service/gpu/multi_output_fusion.cc b/tensorflow/compiler/xla/service/gpu/multi_output_fusion.cc
index 01fddcede6..02e1207f37 100644
--- a/tensorflow/compiler/xla/service/gpu/multi_output_fusion.cc
+++ b/tensorflow/compiler/xla/service/gpu/multi_output_fusion.cc
@@ -67,7 +67,7 @@ int64 GpuMultiOutputFusion::GetProfit(HloInstruction* instr1,
   }
   int64 profit = 0;
   for (auto instr : instr2->operands()) {
-    if (!IsProfitableOperand(instr) || in_list.count(instr) == 0) {
+    if (!IsProfitableOperand(instr) || !in_list.contains(instr)) {
       continue;
     }
     profit += ShapeUtil::ByteSizeOf(instr->shape());
diff --git a/tensorflow/compiler/xla/service/gpu/stream_assignment.cc b/tensorflow/compiler/xla/service/gpu/stream_assignment.cc
index 4775baf44a..1dedbd3bef 100644
--- a/tensorflow/compiler/xla/service/gpu/stream_assignment.cc
+++ b/tensorflow/compiler/xla/service/gpu/stream_assignment.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/service/gpu/stream_assignment.h"
 
+#include "absl/container/flat_hash_set.h"
 #include "absl/memory/memory.h"
 #include "tensorflow/compiler/xla/map_util.h"
 #include "tensorflow/compiler/xla/service/gpu/ir_emission_utils.h"
@@ -25,7 +26,7 @@ namespace xla {
 namespace gpu {
 
 bool StreamAssignment::HasStreamAssigned(const HloInstruction& hlo) const {
-  return hlo_to_stream_number_.count(&hlo);
+  return hlo_to_stream_number_.contains(&hlo);
 }
 
 int StreamAssignment::StreamNumberForHlo(const HloInstruction& hlo) const {
@@ -98,10 +99,10 @@ int ComputeStreamToAssign(
   // greedy approach. First, we compute as forbidden_stream_numbers the
   // streams assigned to GEMMs that are concurrent with `hlo`. Then, we assign
   // `hlo` a different stream.
-  std::set<int> forbidden_stream_numbers;
+  absl::flat_hash_set<int> forbidden_stream_numbers;
   for (const auto* seen_gemm : seen_gemms) {
     int stream_num = stream_assignment.StreamNumberForHlo(*seen_gemm);
-    if (!forbidden_stream_numbers.count(stream_num) &&
+    if (!forbidden_stream_numbers.contains(stream_num) &&
         CanRunConcurrently(*seen_gemm, hlo, reachability)) {
       forbidden_stream_numbers.insert(stream_num);
     }
@@ -109,7 +110,7 @@ int ComputeStreamToAssign(
 
   for (int stream_num = 0; stream_num < stream_assignment.StreamCount();
        ++stream_num) {
-    if (!forbidden_stream_numbers.count(stream_num)) {
+    if (!forbidden_stream_numbers.contains(stream_num)) {
       return stream_num;
     }
   }
diff --git a/tensorflow/compiler/xla/service/gpu/thunk_schedule.cc b/tensorflow/compiler/xla/service/gpu/thunk_schedule.cc
index 6b2d76764a..25bad67bab 100644
--- a/tensorflow/compiler/xla/service/gpu/thunk_schedule.cc
+++ b/tensorflow/compiler/xla/service/gpu/thunk_schedule.cc
@@ -14,17 +14,19 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/compiler/xla/service/gpu/thunk_schedule.h"
+#include "absl/container/flat_hash_map.h"
 #include "tensorflow/compiler/xla/array2d.h"
 #include "tensorflow/compiler/xla/map_util.h"
 #include "tensorflow/compiler/xla/types.h"
+#include "tensorflow/core/lib/gtl/map_util.h"
 
 namespace xla {
 namespace gpu {
 
 void ThunkSchedule::AddDependenciesOnTransitiveOperands(
     const Thunk& thunk, const HloInstruction& operand,
-    const std::unordered_map<const HloInstruction*, Thunk*>& hlo_to_thunk) {
-  if (hlo_to_thunk.count(&operand)) {
+    const absl::flat_hash_map<const HloInstruction*, Thunk*>& hlo_to_thunk) {
+  if (hlo_to_thunk.contains(&operand)) {
     // If `operand` is mapped to a thunk, adds `operand` to `thunk`'s dependency
     // list if `operand` is assigned to a different stream. As an optimization,
     // we skip `operand`'s operands because `operand` depends on them already.
@@ -48,14 +50,14 @@ ThunkSchedule::ThunkSchedule(
     const std::vector<HloInstruction*>& hlo_total_order)
     : thunks_(std::move(thunks)),
       stream_assignment_(std::move(stream_assignment)) {
-  std::unordered_map<const HloInstruction*, Thunk*> hlo_to_thunk;
+  absl::flat_hash_map<const HloInstruction*, Thunk*> hlo_to_thunk;
   for (const auto& thunk : *thunks_) {
     InsertOrDie(&hlo_to_thunk, thunk->hlo_instruction(), thunk.get());
   }
 
   for (HloInstruction* hlo : hlo_total_order) {
-    if (hlo_to_thunk.count(hlo)) {
-      thunk_total_order_.push_back(FindOrDie(hlo_to_thunk, hlo));
+    if (Thunk** thunk = tensorflow::gtl::FindOrNull(hlo_to_thunk, hlo)) {
+      thunk_total_order_.push_back(*thunk);
     }
   }
 
@@ -106,7 +108,7 @@ void ThunkSchedule::RemoveRedundantDependencyEdges() {
   // redundant dependency edge.
   Array2D<int> last_dependency(stream_count, stream_count, -1);
   for (const Thunk* dst : thunk_total_order_) {
-    if (!depends_on_.count(dst)) {
+    if (!depends_on_.contains(dst)) {
       continue;
     }
 
@@ -134,7 +136,7 @@ void ThunkSchedule::RemoveRedundantDependencyEdges() {
 
 const std::list<const Thunk*>& ThunkSchedule::DependsOn(
     const Thunk* thunk) const {
-  if (depends_on_.count(thunk)) {
+  if (depends_on_.contains(thunk)) {
     return FindOrDie(depends_on_, thunk);
   } else {
     return empty_thunk_list_;
diff --git a/tensorflow/compiler/xla/service/gpu/thunk_schedule.h b/tensorflow/compiler/xla/service/gpu/thunk_schedule.h
index 43b628a1ba..549378debd 100644
--- a/tensorflow/compiler/xla/service/gpu/thunk_schedule.h
+++ b/tensorflow/compiler/xla/service/gpu/thunk_schedule.h
@@ -21,6 +21,8 @@ limitations under the License.
 #include <unordered_map>
 #include <vector>
 
+#include "absl/container/flat_hash_map.h"
+#include "absl/container/flat_hash_set.h"
 #include "tensorflow/compiler/xla/service/gpu/stream_assignment.h"
 #include "tensorflow/compiler/xla/service/gpu/thunk.h"
 #include "tensorflow/compiler/xla/service/hlo_instruction.h"
@@ -54,7 +56,9 @@ class ThunkSchedule {
   // Thunks that `thunk` depends on.
   const std::list<const Thunk*>& DependsOn(const Thunk* thunk) const;
   // Whether `thunk` is depended by another thunk.
-  bool Depended(const Thunk* thunk) const { return depended_by_.count(thunk); }
+  bool Depended(const Thunk* thunk) const {
+    return depended_by_.contains(thunk);
+  }
 
   // Delegates to StreamAssignment.
   int StreamCount() const { return stream_assignment_->StreamCount(); }
@@ -75,13 +79,13 @@ class ThunkSchedule {
   // thunk.hlo_instruction().
   void AddDependenciesOnTransitiveOperands(
       const Thunk& thunk, const HloInstruction& operand,
-      const std::unordered_map<const HloInstruction*, Thunk*>& hlo_to_thunk);
+      const absl::flat_hash_map<const HloInstruction*, Thunk*>& hlo_to_thunk);
 
   std::unique_ptr<ThunkSequence> thunks_;
   std::vector<Thunk*> thunk_total_order_;
 
-  std::unordered_map<const Thunk*, std::list<const Thunk*>> depends_on_;
-  std::set<const Thunk*> depended_by_;
+  absl::flat_hash_map<const Thunk*, std::list<const Thunk*>> depends_on_;
+  absl::flat_hash_set<const Thunk*> depended_by_;
   std::list<const Thunk*> empty_thunk_list_;
 
   std::unique_ptr<StreamAssignment> stream_assignment_;
diff --git a/tensorflow/compiler/xla/service/heap_simulator.cc b/tensorflow/compiler/xla/service/heap_simulator.cc
index 9220865867..70ae7078ae 100644
--- a/tensorflow/compiler/xla/service/heap_simulator.cc
+++ b/tensorflow/compiler/xla/service/heap_simulator.cc
@@ -199,7 +199,7 @@ Status HeapSimulator::RunComputation(
 
       // If the buffer has no users and isn't an entry parameter or output, it
       // must be a dead value.
-      if (live_buffers.count(buffer) == 0) {
+      if (!live_buffers.contains(buffer)) {
         dead_buffers_to_free.push_back(buffer);
       }
     }
@@ -253,7 +253,7 @@ Status HeapSimulator::RunComputation(
       bool shared = false;
       if (options_.may_reuse_operand_buffers) {
         for (const BufferValue* operand_buffer : operand_buffers_to_free) {
-          if (reused_buffers.count(operand_buffer) != 0) {
+          if (reused_buffers.contains(operand_buffer)) {
             continue;
           }
           if (buffer->instruction()->IsUserOf(operand_buffer->instruction()) &&
@@ -374,15 +374,15 @@ bool HeapSimulator::IgnoreBuffer(const BufferValue* buffer) const {
     return true;
   }
   return options_.buffers_to_assign != nullptr &&
-         options_.buffers_to_assign->count(buffer) == 0;
+         !options_.buffers_to_assign->contains(buffer);
 }
 
 // Alloc always calls the underlying heap algorithm.
 void HeapSimulator::Alloc(const BufferValue* buffer,
                           const HloInstruction* instruction) {
-  CHECK(allocated_buffers_.count(buffer) == 0)
+  CHECK(!allocated_buffers_.contains(buffer))
       << "Alloc called on allocated buffer: " << *buffer;
-  CHECK(freed_buffers_.count(buffer) == 0)
+  CHECK(!freed_buffers_.contains(buffer))
       << "Alloc called on freed buffer: " << *buffer;
 
   allocated_buffers_.insert(buffer);
@@ -411,9 +411,9 @@ void HeapSimulator::Free(const BufferValue* buffer,
     buffer = group->canonical;
   }
 
-  CHECK(allocated_buffers_.count(buffer) > 0)
+  CHECK(allocated_buffers_.contains(buffer))
       << "Free called on non-allocated buffer: " << *buffer;
-  CHECK(freed_buffers_.count(buffer) == 0)
+  CHECK(!freed_buffers_.contains(buffer))
       << "Free called on freed buffer: " << *buffer;
 
   freed_buffers_.insert(buffer);
@@ -433,11 +433,11 @@ void HeapSimulator::ShareBuffer(const BufferValue* buffer,
                                 const HloInstruction* instruction) {
   CHECK_LE(size_fn_(*buffer), size_fn_(*shared))
       << "ShareBuffer oversized buffer" << *buffer << " shared: " << *shared;
-  CHECK(allocated_buffers_.count(buffer) == 0)
+  CHECK(!allocated_buffers_.contains(buffer))
       << "ShareBuffer called on allocated buffer: " << *buffer;
-  CHECK(freed_buffers_.count(buffer) == 0)
+  CHECK(!freed_buffers_.contains(buffer))
       << "ShareBuffer called on freed buffer: " << *buffer;
-  CHECK(freed_buffers_.count(shared) == 0)
+  CHECK(!freed_buffers_.contains(shared))
       << "ShareBuffer called on freed shared buffer: " << *shared;
 
   const BufferValue* canonical = nullptr;
@@ -452,7 +452,7 @@ void HeapSimulator::ShareBuffer(const BufferValue* buffer,
   } else {
     // The 'shared' buffer doesn't have a group; it must be the canonical.  Add
     // both 'buffer' and 'shared' to a new group.
-    CHECK(allocated_buffers_.count(shared) > 0)
+    CHECK(allocated_buffers_.contains(shared))
         << "ShareBuffer called on non-allocated shared buffer: " << *shared;
     auto group = std::make_shared<SharedGroup>();
     canonical = shared;
diff --git a/tensorflow/compiler/xla/service/hlo_computation.cc b/tensorflow/compiler/xla/service/hlo_computation.cc
index 52ca67afb8..10f0cf3a34 100644
--- a/tensorflow/compiler/xla/service/hlo_computation.cc
+++ b/tensorflow/compiler/xla/service/hlo_computation.cc
@@ -207,14 +207,14 @@ Status HloComputation::RemoveInstructionAndUnusedOperands(
   TF_RET_CHECK(instruction->user_count() == 0);
   TF_RET_CHECK(IsRemovable(instruction))
       << "Cannot remove instruction: " << instruction->ToString();
-  std::unordered_set<HloInstruction*> removed;
+  absl::flat_hash_set<HloInstruction*> removed;
   std::queue<HloInstruction*> worklist;
   worklist.push(instruction);
   while (!worklist.empty()) {
     HloInstruction* item = worklist.front();
     worklist.pop();
 
-    if (removed.count(item) != 0 || item->user_count() != 0 ||
+    if (removed.contains(item) || item->user_count() != 0 ||
         item == root_instruction() || !IsRemovable(item) ||
         (item->HasSideEffect() && item != instruction)) {
       continue;
@@ -694,13 +694,14 @@ bool HloComputation::operator==(const HloComputation& other) const {
   if (this == &other) {
     return true;
   }
-  std::set<std::pair<const HloInstruction*, const HloInstruction*>> visited;
+  absl::flat_hash_set<std::pair<const HloInstruction*, const HloInstruction*>>
+      visited;
   std::function<bool(const HloInstruction*, const HloInstruction*)> eq =
       [&visited, &eq](const HloInstruction* a, const HloInstruction* b) {
         // If <a,b> are visited but not identical, the recursion should have
         // been aborted. So, if <a,b> are visited at this point, they must be
         // identical.
-        if (visited.count(std::make_pair(a, b)) > 0) {
+        if (visited.contains(std::make_pair(a, b))) {
           return true;
         }
         visited.emplace(a, b);
@@ -803,13 +804,13 @@ Status HloComputation::AcceptOrdered(
         << root->ToString();
   }
   TF_RET_CHECK(order.size() == instruction_count());
-  std::unordered_set<const HloInstruction*> visited;
+  absl::flat_hash_set<const HloInstruction*> visited;
   for (const HloInstruction* instruction : order) {
     VLOG(3) << "Visiting ordered: " << instruction->ToString();
-    TF_RET_CHECK(instruction_iterators_.count(instruction) == 1)
+    TF_RET_CHECK(instruction_iterators_.contains(instruction))
         << "Instruction " << instruction->name() << " is not in computation "
         << name();
-    TF_RET_CHECK(visited.count(instruction) == 0)
+    TF_RET_CHECK(!visited.contains(instruction))
         << "Instruction " << instruction->name()
         << " appears more than once in order";
     HloInstruction* mutable_instruction =
diff --git a/tensorflow/compiler/xla/service/hlo_computation_test.cc b/tensorflow/compiler/xla/service/hlo_computation_test.cc
index 0361c87428..216d56b868 100644
--- a/tensorflow/compiler/xla/service/hlo_computation_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_computation_test.cc
@@ -17,6 +17,7 @@ limitations under the License.
 
 #include <set>
 
+#include "absl/container/flat_hash_set.h"
 #include "tensorflow/compiler/xla/literal.h"
 #include "tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h"
 #include "tensorflow/compiler/xla/service/hlo_instruction.h"
@@ -226,7 +227,7 @@ TEST_F(HloComputationTest, VisitWithMultipleRoots) {
         : computation_(computation) {}
 
     Status DefaultAction(HloInstruction* hlo_instruction) override {
-      EXPECT_EQ(0, visited_set_.count(hlo_instruction));
+      EXPECT_FALSE(visited_set_.contains(hlo_instruction));
       visited_set_.insert(hlo_instruction);
       last_visited_ = hlo_instruction;
       return Status::OK();
@@ -239,7 +240,7 @@ TEST_F(HloComputationTest, VisitWithMultipleRoots) {
     }
 
     HloComputation* computation_;
-    std::set<HloInstruction*> visited_set_;
+    absl::flat_hash_set<HloInstruction*> visited_set_;
     int64 finish_visit_calls_ = 0;
     HloInstruction* last_visited_ = nullptr;
   };
diff --git a/tensorflow/compiler/xla/service/hlo_dataflow_analysis.cc b/tensorflow/compiler/xla/service/hlo_dataflow_analysis.cc
index 1924204df0..0db452b85f 100644
--- a/tensorflow/compiler/xla/service/hlo_dataflow_analysis.cc
+++ b/tensorflow/compiler/xla/service/hlo_dataflow_analysis.cc
@@ -107,7 +107,7 @@ bool HloDataflowAnalysis::AreTransitiveUsesElementwiseOrTuple(
           return false;
         }
       }
-      if (!visited.count(user)) {
+      if (!visited.contains(user)) {
         stack.push_back(user);
       }
     }
diff --git a/tensorflow/compiler/xla/service/hlo_dce.cc b/tensorflow/compiler/xla/service/hlo_dce.cc
index 7d35e251ca..a5a11f09cf 100644
--- a/tensorflow/compiler/xla/service/hlo_dce.cc
+++ b/tensorflow/compiler/xla/service/hlo_dce.cc
@@ -20,6 +20,7 @@ limitations under the License.
 #include <utility>
 #include <vector>
 
+#include "absl/container/flat_hash_set.h"
 #include "tensorflow/compiler/xla/service/hlo_computation.h"
 #include "tensorflow/compiler/xla/service/hlo_instruction.h"
 #include "tensorflow/compiler/xla/service/hlo_module.h"
@@ -65,7 +66,7 @@ StatusOr<bool> HloDCE::Run(HloModule* module) {
 
   // Now DCE HloComputations.  First, collect the computations that are
   // referenced by some remaining instruction.
-  std::unordered_set<HloComputation*> live_computations;
+  absl::flat_hash_set<HloComputation*> live_computations;
   if (HloComputation* entry_computation = module->entry_computation()) {
     live_computations.insert(entry_computation);
   }
@@ -79,7 +80,7 @@ StatusOr<bool> HloDCE::Run(HloModule* module) {
 
   // Remove dead computations.
   for (auto* computation : module->MakeComputationPostOrder()) {
-    if (live_computations.count(computation) == 0) {
+    if (!live_computations.contains(computation)) {
       TF_RETURN_IF_ERROR(module->RemoveEmbeddedComputation(computation));
       changed = true;
     }
diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
index c6eaead8dd..22bcb2030b 100644
--- a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
+++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
@@ -24,9 +24,9 @@ limitations under the License.
 #include <queue>
 #include <string>
 #include <tuple>
-#include <unordered_map>
 #include <vector>
 
+#include "absl/container/flat_hash_map.h"
 #include "absl/strings/match.h"
 #include "absl/strings/str_cat.h"
 #include "absl/strings/str_format.h"
@@ -380,7 +380,7 @@ class HloDotDumper {
   // Each HloInstruction dumped gets a monotically-increasing node ID.  This
   // must start at 1, because that's where graphviz's accounting starts.
   int64 next_node_id_ = 1;
-  std::unordered_map<const HloInstruction*, int64> node_ids_;
+  absl::flat_hash_map<const HloInstruction*, int64> node_ids_;
 
   // The "root" tag doesn't have an associated HloInstruction pointer, so we
   // need to store it outside the map.
@@ -397,7 +397,7 @@ class HloDotDumper {
 
   // Each HloComputation that's emitted gets a monotonically-increasing ID.
   int64 next_cluster_id_ = 1;
-  std::unordered_map<const HloComputation*, int64> cluster_ids_;
+  absl::flat_hash_map<const HloComputation*, int64> cluster_ids_;
 
   // Edges to print from Footer().  Edges come at the end because graphviz is
   // unhappy if an edge from a subcomputation to a node in the outer computation
@@ -407,7 +407,7 @@ class HloDotDumper {
 
   // When coloring by sharding information, we track the sharding string
   // representation to color association, by round-robin the color schemes.
-  std::unordered_map<HloSharding, ColorScheme, HloSharding::Hasher>
+  absl::flat_hash_map<HloSharding, ColorScheme, HloSharding::Hasher>
       sharding_colors_;
   int64 next_shard_color_ = 0;
 };
@@ -1286,7 +1286,7 @@ NodeFilter MakeNodeRadiusAroundFilter(const HloInstruction* root,
                                       int64 radius) {
   // First, find the neighborhood of nodes with distance from root <= radius.
   // These nodes are our initial set of "normal" nodes.
-  std::unordered_map<const HloInstruction*, NodeFilterResult> nodes;
+  absl::flat_hash_map<const HloInstruction*, NodeFilterResult> nodes;
   std::deque<std::pair<const HloInstruction*, /*depth*/ int64>> worklist;
   worklist.push_back({root, 0});
   while (!worklist.empty()) {
@@ -1307,7 +1307,7 @@ NodeFilter MakeNodeRadiusAroundFilter(const HloInstruction* root,
     // are not interesting to the graph at hand.
     if (instr == root || instr->opcode() != HloOpcode::kTuple) {
       for (const HloInstruction* operand : instr->operands()) {
-        if (!nodes.count(operand)) {
+        if (!nodes.contains(operand)) {
           worklist.push_back({operand, depth + 1});
         }
       }
@@ -1335,7 +1335,7 @@ NodeFilter MakeNodeRadiusAroundFilter(const HloInstruction* root,
       continue;
     }
     for (const HloInstruction* user : instr->users()) {
-      if (!nodes.count(user)) {
+      if (!nodes.contains(user)) {
         worklist.push_back({user, depth + 1});
       }
     }
@@ -1344,7 +1344,7 @@ NodeFilter MakeNodeRadiusAroundFilter(const HloInstruction* root,
   auto is_displayed = [&](const HloInstruction* instr) {
     // Constants are displayed inline with their users; they're never omitted.
     // Nodes in subcomputations are always shown.
-    return nodes.count(instr) > 0 || instr->opcode() == HloOpcode::kConstant ||
+    return nodes.contains(instr) || instr->opcode() == HloOpcode::kConstant ||
            instr->parent() != root->parent();
   };
 
diff --git a/tensorflow/compiler/xla/service/hlo_instruction_test.cc b/tensorflow/compiler/xla/service/hlo_instruction_test.cc
index 8048e332cb..35f031f29a 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_instruction_test.cc
@@ -20,6 +20,7 @@ limitations under the License.
 #include <utility>
 #include <vector>
 
+#include "absl/container/flat_hash_map.h"
 #include "tensorflow/compiler/xla/literal.h"
 #include "tensorflow/compiler/xla/protobuf_util.h"
 #include "tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h"
@@ -55,13 +56,13 @@ class OpAndUserCollectingVisitor : public DfsHloVisitorWithDefault {
   }
 
   Status HandleParameter(HloInstruction* parameter) override {
-    EXPECT_EQ(0, count_.count(parameter));
+    EXPECT_FALSE(count_.contains(parameter));
     count_[parameter] = GetCountsForNode(parameter);
     return Status::OK();
   }
 
   Status HandleConstant(HloInstruction* constant) override {
-    EXPECT_EQ(0, count_.count(constant));
+    EXPECT_FALSE(count_.contains(constant));
     count_[constant] = GetCountsForNode(constant);
     return Status::OK();
   }
@@ -69,25 +70,25 @@ class OpAndUserCollectingVisitor : public DfsHloVisitorWithDefault {
   Status HandleAdd(HloInstruction* add) override {
     auto lhs = add->operand(0);
     auto rhs = add->operand(1);
-    EXPECT_EQ(0, count_.count(add));
-    EXPECT_GT(count_.count(lhs), 0);
-    EXPECT_GT(count_.count(rhs), 0);
+    EXPECT_FALSE(count_.contains(add));
+    EXPECT_TRUE(count_.contains(lhs));
+    EXPECT_TRUE(count_.contains(rhs));
     count_[add] = GetCountsForNode(add);
     return Status::OK();
   }
 
   Status HandleNegate(HloInstruction* negate) override {
     auto operand = negate->operand(0);
-    EXPECT_EQ(0, count_.count(negate));
-    EXPECT_GT(count_.count(operand), 0);
+    EXPECT_FALSE(count_.contains(negate));
+    EXPECT_TRUE(count_.contains(operand));
     count_[negate] = GetCountsForNode(negate);
     return Status::OK();
   }
 
   Status HandleMap(HloInstruction* map) override {
-    EXPECT_EQ(0, count_.count(map));
+    EXPECT_FALSE(count_.contains(map));
     for (HloInstruction* arg : map->operands()) {
-      EXPECT_GT(count_.count(arg), 0);
+      EXPECT_TRUE(count_.contains(arg));
     }
     count_[map] = GetCountsForNode(map);
     return Status::OK();
@@ -96,9 +97,9 @@ class OpAndUserCollectingVisitor : public DfsHloVisitorWithDefault {
   Status HandleReduce(HloInstruction* reduce) override {
     auto arg = reduce->operand(0);
     auto init_value = reduce->operand(1);
-    EXPECT_EQ(0, count_.count(reduce));
-    EXPECT_GT(count_.count(arg), 0);
-    EXPECT_GT(count_.count(init_value), 0);
+    EXPECT_FALSE(count_.contains(reduce));
+    EXPECT_TRUE(count_.contains(arg));
+    EXPECT_TRUE(count_.contains(init_value));
     count_[reduce] = GetCountsForNode(reduce);
     return Status::OK();
   }
@@ -128,7 +129,7 @@ class OpAndUserCollectingVisitor : public DfsHloVisitorWithDefault {
   }
 
   // Counters for HLOs. Maps HLO to a NumOpsAndUsers.
-  std::unordered_map<const HloInstruction*, NumOpsAndUsers> count_;
+  absl::flat_hash_map<const HloInstruction*, NumOpsAndUsers> count_;
 };
 
 TEST_F(HloInstructionTest, BasicProperties) {
@@ -137,7 +138,7 @@ TEST_F(HloInstructionTest, BasicProperties) {
   EXPECT_EQ(HloOpcode::kParameter, parameter->opcode());
   EXPECT_TRUE(ShapeUtil::IsScalarWithElementType(parameter->shape(), F32));
   EXPECT_FALSE(ShapeUtil::IsScalarWithElementType(parameter->shape(), S32));
-  EXPECT_EQ(0, parameter->operand_count());
+  EXPECT_FALSE(parameter->operand_count());
 }
 
 TEST_F(HloInstructionTest, UserWithTwoOperands) {
@@ -981,9 +982,9 @@ TEST_F(HloInstructionTest, FunctionVisitor) {
   module->AddEntryComputation(builder.Build());
 
   int visit_num = 0;
-  std::unordered_map<HloInstruction*, int> visit_order;
+  absl::flat_hash_map<HloInstruction*, int> visit_order;
   EXPECT_IS_OK(add->Accept([&visit_num, &visit_order](HloInstruction* inst) {
-    EXPECT_EQ(0, visit_order.count(inst));
+    EXPECT_FALSE(visit_order.contains(inst));
     visit_order[inst] = visit_num;
     visit_num++;
     return Status::OK();
diff --git a/tensorflow/compiler/xla/service/hlo_liveness_analysis.cc b/tensorflow/compiler/xla/service/hlo_liveness_analysis.cc
index 5bf055f3c0..e14bcfa7f6 100644
--- a/tensorflow/compiler/xla/service/hlo_liveness_analysis.cc
+++ b/tensorflow/compiler/xla/service/hlo_liveness_analysis.cc
@@ -17,6 +17,7 @@ limitations under the License.
 
 #include <deque>
 
+#include "absl/container/flat_hash_set.h"
 #include "absl/memory/memory.h"
 #include "absl/strings/str_cat.h"
 #include "tensorflow/compiler/xla/map_util.h"
@@ -36,11 +37,11 @@ namespace xla {
 namespace {
 
 using Worklist = std::deque<const HloInstruction*>;
-using Workset = std::unordered_set<const HloInstruction*>;
+using Workset = absl::flat_hash_set<const HloInstruction*>;
 
 void AddToWorklist(const HloInstruction* instruction, Worklist* worklist,
                    Workset* workset) {
-  if (workset->count(instruction) == 0) {
+  if (!workset->contains(instruction)) {
     worklist->push_back(instruction);
     workset->insert(instruction);
     VLOG(3) << "ADD instruction: " << instruction->name();
diff --git a/tensorflow/compiler/xla/service/hlo_module.cc b/tensorflow/compiler/xla/service/hlo_module.cc
index fe8371384c..ed78189fed 100644
--- a/tensorflow/compiler/xla/service/hlo_module.cc
+++ b/tensorflow/compiler/xla/service/hlo_module.cc
@@ -392,15 +392,12 @@ namespace {
 // Returns whether `hlo` is used outside the given subcomputation.
 // `instructions_in_subcomputation` is the instruction set of the given
 // subcomputation.
-bool IsUsedOutsideSubcomputation(
-    const HloInstruction& hlo,
-    const std::unordered_set<HloInstruction*>& instructions_in_subcomputation) {
-  for (HloInstruction* user : hlo.users()) {
-    if (!instructions_in_subcomputation.count(user)) {
-      return true;
-    }
-  }
-  return false;
+bool IsUsedOutsideSubcomputation(const HloInstruction& hlo,
+                                 const absl::flat_hash_set<HloInstruction*>&
+                                     instructions_in_subcomputation) {
+  return absl::c_any_of(hlo.users(), [&](HloInstruction* user) {
+    return !instructions_in_subcomputation.contains(user);
+  });
 }
 }  // anonymous namespace
 
@@ -411,9 +408,9 @@ HloInstruction* HloModule::OutlineExpressionFromComputation(
 
   // A map from original instructions to their counterparts in the new outlined
   // function.
-  std::unordered_map<HloInstruction*, HloInstruction*> outlined_instructions;
+  absl::flat_hash_map<HloInstruction*, HloInstruction*> outlined_instructions;
   // A set that contains all instructions to be outlined.
-  std::unordered_set<HloInstruction*> instruction_set_to_outline(
+  absl::flat_hash_set<HloInstruction*> instruction_set_to_outline(
       instructions_to_outline.begin(), instructions_to_outline.end());
   std::vector<HloInstruction*> arguments;
   std::vector<HloInstruction*> outputs;
@@ -502,7 +499,7 @@ std::vector<HloComputation*> HloModule::MakeComputationPostOrder() const {
   // First determine all root computations by building a set of nonroot
   // computations (computations which are called by an instruction in the
   // module).
-  std::set<HloComputation*> nonroot_computations;
+  absl::flat_hash_set<HloComputation*> nonroot_computations;
   for (auto& computation : computations_) {
     for (auto* instruction : computation->instructions()) {
       for (HloComputation* called_computation :
@@ -515,19 +512,19 @@ std::vector<HloComputation*> HloModule::MakeComputationPostOrder() const {
   // Keep track of computations which have already been added to the post
   // order. This prevents duplication as an embedded computation may be called
   // from two different root computations.
-  std::set<HloComputation*> added_computations;
+  absl::flat_hash_set<HloComputation*> added_computations;
   std::vector<HloComputation*> post_order;
   for (auto& computation : computations_) {
-    if (nonroot_computations.count(computation.get()) == 0) {
+    if (!nonroot_computations.contains(computation.get())) {
       for (HloComputation* embedded_computation :
            computation->MakeEmbeddedComputationsList()) {
-        if (added_computations.count(embedded_computation) == 0) {
+        if (!added_computations.contains(embedded_computation)) {
           post_order.push_back(embedded_computation);
           added_computations.insert(embedded_computation);
         }
       }
       // Root computations should only be encountered once.
-      CHECK_EQ(0, added_computations.count(computation.get()));
+      CHECK(!added_computations.contains(computation.get()));
       post_order.push_back(computation.get());
       added_computations.insert(computation.get());
     }
diff --git a/tensorflow/compiler/xla/service/hlo_module_group_metadata.cc b/tensorflow/compiler/xla/service/hlo_module_group_metadata.cc
index 80f8ca2226..47734bc55c 100644
--- a/tensorflow/compiler/xla/service/hlo_module_group_metadata.cc
+++ b/tensorflow/compiler/xla/service/hlo_module_group_metadata.cc
@@ -199,7 +199,7 @@ bool HloModuleGroupMetadata::IsChannelInstruction(
 }
 
 bool HloModuleGroupMetadata::IsCompanionInstruction(HloInstruction* hlo) const {
-  return companion_set_index_.count(hlo) > 0;
+  return companion_set_index_.contains(hlo);
 }
 
 bool HloModuleGroupMetadata::InstructionCommunicates(
@@ -510,7 +510,7 @@ Status HloModuleGroupMetadata::CheckCommunicatingInstruction(
   HloComputation* computation = instruction->parent();
   const HloModule* module = computation->parent();
   if (module->entry_computation() == computation ||
-      tracked_instructions_.count(computation) > 0) {
+      tracked_instructions_.contains(computation)) {
     return Status::OK();
   }
   return FailedPrecondition("channel is used in disallowed computation");
diff --git a/tensorflow/compiler/xla/service/hlo_module_group_metadata.h b/tensorflow/compiler/xla/service/hlo_module_group_metadata.h
index 5cd0e38f38..3ed95c1050 100644
--- a/tensorflow/compiler/xla/service/hlo_module_group_metadata.h
+++ b/tensorflow/compiler/xla/service/hlo_module_group_metadata.h
@@ -178,7 +178,7 @@ class HloModuleGroupMetadata {
   // Precondition: IsCompanionWhile(instruction) is true.
   const std::vector<HloInstruction*>& Companions(
       const HloInstruction* instruction) const {
-    CHECK_EQ(companion_set_index_.count(instruction), 1);
+    CHECK(companion_set_index_.contains(instruction));
     return companion_set(companion_set_index_.at(instruction));
   }
 
diff --git a/tensorflow/compiler/xla/service/hlo_ordering.cc b/tensorflow/compiler/xla/service/hlo_ordering.cc
index ca6a154809..0cec61c257 100644
--- a/tensorflow/compiler/xla/service/hlo_ordering.cc
+++ b/tensorflow/compiler/xla/service/hlo_ordering.cc
@@ -367,7 +367,7 @@ bool SequentialHloOrdering::ExecutesBeforeInSameComputation(
     const HloInstruction* a, const HloInstruction* b) const {
   CHECK_EQ(a->parent(), b->parent());
   // If either instruction is not in the order, then 'a' and 'b' are unordered.
-  if (order_position_.count(a) == 0 || order_position_.count(b) == 0) {
+  if (!order_position_.contains(a) || !order_position_.contains(b)) {
     return false;
   }
   return order_position_.at(a) < order_position_.at(b);
diff --git a/tensorflow/compiler/xla/service/hlo_pass_pipeline.cc b/tensorflow/compiler/xla/service/hlo_pass_pipeline.cc
index 33ce7e23a8..ae8c08cf1d 100644
--- a/tensorflow/compiler/xla/service/hlo_pass_pipeline.cc
+++ b/tensorflow/compiler/xla/service/hlo_pass_pipeline.cc
@@ -89,7 +89,7 @@ std::vector<HloPassInterface*> HloPassPipeline::GetEnabledPasses(
 
   std::vector<HloPassInterface*> enabled_passes;
   for (auto& pass : passes_) {
-    if (disabled_pass_names.count(string(pass->name())) == 0) {
+    if (!disabled_pass_names.contains(pass->name())) {
       enabled_passes.push_back(pass.get());
     }
   }
diff --git a/tensorflow/compiler/xla/service/hlo_schedule.cc b/tensorflow/compiler/xla/service/hlo_schedule.cc
index 8f6eb974c5..e75373501c 100644
--- a/tensorflow/compiler/xla/service/hlo_schedule.cc
+++ b/tensorflow/compiler/xla/service/hlo_schedule.cc
@@ -140,7 +140,7 @@ Status HloSchedule::UpdateComputationSchedule(
   std::queue<HloInstruction*> worklist;
 
   for (HloInstruction* instruction : computation->instructions()) {
-    if (ids_in_schedule.count(instruction->unique_id()) == 0) {
+    if (!ids_in_schedule.contains(instruction->unique_id())) {
       // This is a newly added instruction which is not in the schedule.
       if (instruction->operands().empty()) {
         worklist.push(instruction);
@@ -204,7 +204,7 @@ Status HloSchedule::Update() {
   std::vector<HloComputation*> nonfusion_computations =
       module_->MakeNonfusionComputations();
   for (const HloComputation* computation : nonfusion_computations) {
-    TF_RET_CHECK(sequences_.count(computation->unique_id()) == 1)
+    TF_RET_CHECK(sequences_.contains(computation->unique_id()))
         << "Computation " << computation->name() << " not in HloSchedule.";
   }
   if (sequences_.size() > nonfusion_computations.size()) {
@@ -215,7 +215,7 @@ Status HloSchedule::Update() {
       nonfusion_computations_ids.insert(computation->unique_id());
     }
     for (auto it = sequences_.begin(); it != sequences_.end();) {
-      if (nonfusion_computations_ids.count(it->first) == 0) {
+      if (!nonfusion_computations_ids.contains(it->first)) {
         sequences_.erase(it++);
       } else {
         ++it;
@@ -244,7 +244,7 @@ Status HloSchedule::Verify() const {
       << "Schedule has " << sequences_.size() << " sequences, but module has "
       << nonfusion_computations.size() << " non-fusion computations";
   for (const HloComputation* computation : nonfusion_computations) {
-    TF_RET_CHECK(sequences_.count(computation->unique_id()) == 1)
+    TF_RET_CHECK(sequences_.contains(computation->unique_id()))
         << "Computation " << computation->name()
         << " missing from HLO schedule.";
   }
@@ -268,7 +268,7 @@ Status HloSchedule::Verify() const {
         << instruction_position.size() << " instructions, expected "
         << computation->instruction_count();
     for (const HloInstruction* instruction : computation->instructions()) {
-      TF_RET_CHECK(instruction_position.count(instruction) == 1)
+      TF_RET_CHECK(instruction_position.contains(instruction))
           << "Instruction " << instruction->name() << " is not in schedule";
     }
 
diff --git a/tensorflow/compiler/xla/service/hlo_schedule.h b/tensorflow/compiler/xla/service/hlo_schedule.h
index 486ddbf499..a5f54ae2c3 100644
--- a/tensorflow/compiler/xla/service/hlo_schedule.h
+++ b/tensorflow/compiler/xla/service/hlo_schedule.h
@@ -110,7 +110,7 @@ class HloSchedule {
 
   // Returns true if the schedule has a sequence for the given computation.
   bool is_computation_scheduled(const HloComputation* computation) const {
-    return sequences_.count(computation->unique_id()) == 1;
+    return sequences_.contains(computation->unique_id());
   }
 
   // Updates the schedule such that it is (again) a valid schedule for the
diff --git a/tensorflow/compiler/xla/service/hlo_sharding.cc b/tensorflow/compiler/xla/service/hlo_sharding.cc
index bdd6ec1169..6c9c32f179 100644
--- a/tensorflow/compiler/xla/service/hlo_sharding.cc
+++ b/tensorflow/compiler/xla/service/hlo_sharding.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/service/hlo_sharding.h"
 
+#include "absl/container/flat_hash_set.h"
 #include "absl/strings/str_cat.h"
 #include "absl/strings/str_join.h"
 #include "tensorflow/compiler/xla/overflow_util.h"
@@ -316,7 +317,7 @@ Status HloSharding::ValidateNonTuple(const Shape& shape,
   // All tile assignments must be less than the number of available cores and
   // unique.
   Status status = Status::OK();
-  std::set<int64> seen_cores;
+  absl::flat_hash_set<int64> seen_cores;
   tile_assignment_.Each(
       [&](absl::Span<const int64> indices, int32 core) {
         // Don't overwrite a bad status, so we report the first error.
@@ -324,7 +325,7 @@ Status HloSharding::ValidateNonTuple(const Shape& shape,
           if (core >= num_devices) {
             status = tensorflow::errors::InvalidArgument(StrCat(
                 "core ", core, " > ", num_devices, " in tile assignment"));
-          } else if (seen_cores.count(core) != 0) {
+          } else if (seen_cores.contains(core)) {
             status = tensorflow::errors::InvalidArgument(
                 StrCat("core ", core, " is not unique in tile assignment"));
           }
diff --git a/tensorflow/compiler/xla/service/hlo_sharding_metadata.cc b/tensorflow/compiler/xla/service/hlo_sharding_metadata.cc
index b414d2a663..094d98bc6e 100644
--- a/tensorflow/compiler/xla/service/hlo_sharding_metadata.cc
+++ b/tensorflow/compiler/xla/service/hlo_sharding_metadata.cc
@@ -99,7 +99,7 @@ std::vector<PassThrough> LocatePassThroughDomainLinks(
         << "Instruction is not a kDomain: " << instruction->ToString();
     for (HloInstruction* user : instruction->users()) {
       if (user->opcode() == HloOpcode::kDomain &&
-          domain.exit_domains.count(user) != 0) {
+          domain.exit_domains.contains(user)) {
         pass_through.emplace_back(user, instruction);
         VLOG(2) << "Found passthrough domain link:";
         VLOG(2) << "  " << user->ToString();
@@ -253,7 +253,7 @@ StatusOr<bool> ApplyShardingFromUsers(HloInstruction* instruction,
       instruction->shape(), HloSharding::AssignDevice(kUnassignedDevice));
   for (HloInstruction* user : instruction->users()) {
     if (user->opcode() == HloOpcode::kDomain &&
-        domain.exit_domains.count(user) > 0) {
+        domain.exit_domains.contains(user)) {
       // If a user is a domain and it is registered in the domain exits, then
       // the instruction sharding is taken directly from the domain, and no
       // further users need to be visited.
diff --git a/tensorflow/compiler/xla/service/indexed_array_analysis.cc b/tensorflow/compiler/xla/service/indexed_array_analysis.cc
index a41cf714c5..76bf48870d 100644
--- a/tensorflow/compiler/xla/service/indexed_array_analysis.cc
+++ b/tensorflow/compiler/xla/service/indexed_array_analysis.cc
@@ -103,7 +103,7 @@ Status IndexedArrayAnalysis::TraverseAndPopulateCache(
 
   do {
     const HloInstruction* instr = stack.back();
-    if (cache_.count(instr)) {
+    if (cache_.contains(instr)) {
       stack.pop_back();
       continue;
     }
@@ -111,9 +111,9 @@ Status IndexedArrayAnalysis::TraverseAndPopulateCache(
     switch (FindOrDie(dfs_state_map, instr)) {
       case kDiscovered: {
         for (const HloInstruction* operand : instr->operands()) {
-          if (!cache_.count(operand)) {
+          if (!cache_.contains(operand)) {
             stack.push_back(operand);
-            CHECK(!dfs_state_map.count(operand) ||
+            CHECK(!dfs_state_map.contains(operand) ||
                   dfs_state_map[operand] == kDiscovered);
             dfs_state_map[operand] = kDiscovered;
           }
diff --git a/tensorflow/compiler/xla/service/layout_assignment.cc b/tensorflow/compiler/xla/service/layout_assignment.cc
index 09246db68e..406cd0a219 100644
--- a/tensorflow/compiler/xla/service/layout_assignment.cc
+++ b/tensorflow/compiler/xla/service/layout_assignment.cc
@@ -2135,7 +2135,7 @@ Status LayoutAssignment::ClearPreviousPassSideEffects(HloModule* module) {
     for (HloInstruction* instruction :
          computation->MakeInstructionPostOrder()) {
       if (instruction->opcode() == HloOpcode::kCopy &&
-          added_copies_.count(instruction) > 0) {
+          added_copies_.contains(instruction)) {
         VLOG(5) << "Removing added copy: " << instruction->ToString();
         TF_RETURN_IF_ERROR(
             instruction->ReplaceAllUsesWith(instruction->mutable_operand(0)));
diff --git a/tensorflow/compiler/xla/service/layout_assignment.h b/tensorflow/compiler/xla/service/layout_assignment.h
index 3b081de3c7..5701cb5b02 100644
--- a/tensorflow/compiler/xla/service/layout_assignment.h
+++ b/tensorflow/compiler/xla/service/layout_assignment.h
@@ -243,7 +243,7 @@ class ChannelLayoutConstraints {
 
   // Returns true if channel_id has a layout constraint.
   bool IsChannelConstrained(int64 channel_id) const {
-    return constraints_.count(channel_id) > 0;
+    return constraints_.contains(channel_id);
   }
 
   // Given `shape`, apply the layout for `channel_id`. `channel_id` must already
@@ -276,7 +276,7 @@ class ChannelLayoutConstraints {
   }
 
  private:
-  std::unordered_map<int64, Layout> constraints_;
+  absl::flat_hash_map<int64, Layout> constraints_;
 };
 
 // HLO pass which assigns layouts to all instructions in the HLO module while
diff --git a/tensorflow/compiler/xla/service/llvm_ir/BUILD b/tensorflow/compiler/xla/service/llvm_ir/BUILD
index 728a66b388..54e8fe1947 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/BUILD
+++ b/tensorflow/compiler/xla/service/llvm_ir/BUILD
@@ -169,6 +169,7 @@ cc_library(
         "//tensorflow/compiler/xla/service:elemental_ir_emitter",
         "//tensorflow/compiler/xla/service:hlo",
         "//tensorflow/core:lib",
+        "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/types:optional",
         "@com_google_absl//absl/types:span",
         "@llvm//:core",
diff --git a/tensorflow/compiler/xla/service/llvm_ir/alias_analysis.h b/tensorflow/compiler/xla/service/llvm_ir/alias_analysis.h
index 2b46b3c396..12e2f449e2 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/alias_analysis.h
+++ b/tensorflow/compiler/xla/service/llvm_ir/alias_analysis.h
@@ -76,15 +76,12 @@ class AliasAnalysis {
   // A map from a buffer slice to metadata corresponding to its alias.scope
   // metadata.  The index kParameterAliasSet is used to hold aliasing
   // information for parameters.
-  absl::flat_hash_map<BufferAllocation::Slice, llvm::MDNode*,
-                      BufferAllocation::Slice::Hasher>
+  absl::flat_hash_map<BufferAllocation::Slice, llvm::MDNode*>
       alias_scope_metadata_;
 
   // A map from a buffer slice to metadata corresponding to its noalias
   // metadata.
-  absl::flat_hash_map<BufferAllocation::Slice, llvm::MDNode*,
-                      BufferAllocation::Slice::Hasher>
-      noalias_metadata_;
+  absl::flat_hash_map<BufferAllocation::Slice, llvm::MDNode*> noalias_metadata_;
 };
 
 }  // namespace llvm_ir
diff --git a/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.cc b/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.cc
index 03a475b40b..e440f05e2b 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.cc
@@ -35,7 +35,7 @@ using llvm_ir::IrArray;
 Status FusedIrEmitter::DefaultAction(HloInstruction* hlo) {
   indexed_generators_[hlo] =
       [=](const IrArray::Index& index) -> StatusOr<llvm::Value*> {
-    if (generated_value_cache_[hlo].count(index.multidim()) > 0) {
+    if (generated_value_cache_[hlo].contains(index.multidim())) {
       llvm::Value* generated_value =
           generated_value_cache_[hlo][index.multidim()];
       llvm::BasicBlock* generated_value_bb = nullptr;
diff --git a/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.h b/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.h
index 1b9c61f670..e6d52a580c 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.h
+++ b/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.h
@@ -19,6 +19,7 @@ limitations under the License.
 #include <map>
 #include <unordered_map>
 
+#include "absl/container/flat_hash_map.h"
 #include "absl/types/optional.h"
 #include "absl/types/span.h"
 #include "llvm/IR/IRBuilder.h"
@@ -134,8 +135,9 @@ class FusedIrEmitter : public DfsHloVisitorWithDefault {
 
   // Cache of generated values, lest we regenerate an element of a node with
   // multiple outgoing edges
-  std::unordered_map<const HloInstruction*,
-                     std::map<std::vector<llvm::Value*>, llvm::Value*>>
+  absl::flat_hash_map<
+      const HloInstruction*,
+      absl::flat_hash_map<std::vector<llvm::Value*>, llvm::Value*>>
       generated_value_cache_;
 };
 
diff --git a/tensorflow/compiler/xla/service/multi_output_fusion.cc b/tensorflow/compiler/xla/service/multi_output_fusion.cc
index 9ccdd7d8d8..53d52d9a3d 100644
--- a/tensorflow/compiler/xla/service/multi_output_fusion.cc
+++ b/tensorflow/compiler/xla/service/multi_output_fusion.cc
@@ -198,7 +198,7 @@ void MultiOutputFusion::Update(HloInstruction* instr1, HloInstruction* instr2) {
     if (instr == fusion || is_fused(instr) || is_connected(fusion, instr)) {
       continue;
     }
-    if (in_list.count(instr) > 0) {
+    if (in_list.contains(instr)) {
       continue;
     }
     int64 profit = GetProfit(instr, fusion);
diff --git a/tensorflow/compiler/xla/service/tuple_points_to_analysis.cc b/tensorflow/compiler/xla/service/tuple_points_to_analysis.cc
index b1f0672c60..dec719b04a 100644
--- a/tensorflow/compiler/xla/service/tuple_points_to_analysis.cc
+++ b/tensorflow/compiler/xla/service/tuple_points_to_analysis.cc
@@ -19,6 +19,7 @@ limitations under the License.
 #include <utility>
 #include <vector>
 
+#include "absl/container/flat_hash_set.h"
 #include "absl/memory/memory.h"
 #include "absl/strings/str_cat.h"
 #include "absl/strings/str_format.h"
@@ -55,11 +56,10 @@ bool PointsToSet::IsAmbiguous() const {
 
 bool PointsToSet::IsDistinct() const {
   bool distinct = true;
-  std::set<const LogicalBuffer*> all_points_to;
-  ForEachElement([&distinct, &all_points_to](const ShapeIndex& /*index*/,
-                                             const BufferList& points_to) {
+  absl::flat_hash_set<const LogicalBuffer*> all_points_to;
+  ForEachElement([&](const ShapeIndex& /*index*/, const BufferList& points_to) {
     for (auto& buffer : points_to) {
-      if (all_points_to.count(buffer) != 0) {
+      if (all_points_to.contains(buffer)) {
         distinct = false;
       }
       all_points_to.insert(buffer);
diff --git a/tensorflow/compiler/xla/service/while_loop_invariant_code_motion.cc b/tensorflow/compiler/xla/service/while_loop_invariant_code_motion.cc
index a1c627a319..69cc8feb3f 100644
--- a/tensorflow/compiler/xla/service/while_loop_invariant_code_motion.cc
+++ b/tensorflow/compiler/xla/service/while_loop_invariant_code_motion.cc
@@ -89,7 +89,7 @@ static void CreateLoopInvariantCopy(
 
     HloInstruction* next_operand =
         frame->instruction->mutable_operand(frame->operand_index++);
-    if (hoisted_instructions->count(next_operand) ||
+    if (hoisted_instructions->contains(next_operand) ||
         next_operand == while_body_param) {
       continue;
     }
@@ -241,7 +241,7 @@ WhileLoopInvariantCodeMotion::TryHoistingInvariantInstructionsFromWhileBody(
 
     auto is_invariant = [&](HloInstruction* op) {
       return hoisted_instructions.find(op) != hoisted_instructions.end() ||
-             unhoisted_invariant_instructions.count(op) ||
+             unhoisted_invariant_instructions.contains(op) ||
              op->opcode() == HloOpcode::kConstant;
     };
 
diff --git a/tensorflow/compiler/xla/service/while_loop_simplifier.cc b/tensorflow/compiler/xla/service/while_loop_simplifier.cc
index 772faa25e2..de884b037d 100644
--- a/tensorflow/compiler/xla/service/while_loop_simplifier.cc
+++ b/tensorflow/compiler/xla/service/while_loop_simplifier.cc
@@ -127,7 +127,7 @@ static StatusOr<bool> TryRemoveDeadWhileParams(HloInstruction* while_op) {
   // through to the while body's root, count that element as "used", since
   // removing that element would be observable.
   for (int64 i = 0; i < while_body_root->operand_count(); ++i) {
-    if (used_tuple_indices.count(i)) {
+    if (used_tuple_indices.contains(i)) {
       continue;
     }
 
-- 
GitLab


From b4813a0cff9f7ad1278aee376a5302f31c4558a2 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Fri, 4 Jan 2019 12:29:13 -0800
Subject: [PATCH 0245/1095] [XLA] Use absl::c_foo rather than std::foo.

No functional change.

PiperOrigin-RevId: 227896034
---
 tensorflow/compiler/xla/BUILD                 |  1 +
 .../compiler/xla/client/lib/sorting_test.cc   |  2 +-
 tensorflow/compiler/xla/layout_util.cc        |  6 +-
 .../compiler/xla/metric_table_report.cc       |  4 +-
 tensorflow/compiler/xla/service/BUILD         |  1 +
 .../xla/service/algebraic_simplifier.cc       |  7 +-
 .../compiler/xla/service/buffer_assignment.cc | 85 +++++++++----------
 .../compiler/xla/service/call_graph_test.cc   |  2 +-
 .../xla/service/computation_layout.cc         |  4 +-
 .../compiler/xla/service/copy_insertion.cc    | 18 ++--
 .../compiler/xla/service/cpu/disassembler.cc  | 21 +++--
 .../compiler/xla/service/cpu/ir_emitter.cc    | 12 +--
 .../xla/service/gpu/ir_emission_utils.cc      | 15 ++--
 .../xla/service/gpu/ir_emitter_unnested.cc    |  8 +-
 .../compiler/xla/service/heap_simulator.cc    | 41 +++++----
 .../xla/service/hlo_alias_analysis.cc         | 30 +++----
 tensorflow/compiler/xla/service/hlo_buffer.cc |  2 +-
 .../compiler/xla/service/hlo_computation.cc   | 12 ++-
 .../xla/service/hlo_dataflow_analysis.cc      | 36 +++-----
 .../compiler/xla/service/hlo_dce_test.cc      |  4 +-
 .../compiler/xla/service/hlo_domain_map.cc    |  8 +-
 .../compiler/xla/service/hlo_evaluator.cc     |  3 +-
 .../xla/service/hlo_evaluator_typed_visitor.h |  3 +-
 .../compiler/xla/service/hlo_graph_dumper.cc  | 42 +++++----
 .../compiler/xla/service/hlo_instruction.cc   | 37 ++++----
 .../compiler/xla/service/hlo_instructions.cc  | 18 ++--
 tensorflow/compiler/xla/service/hlo_module.cc | 18 ++--
 .../xla/service/hlo_module_dce_test.cc        |  4 +-
 tensorflow/compiler/xla/service/hlo_parser.cc |  2 +-
 .../xla/service/hlo_profile_printer.cc        | 10 +--
 .../compiler/xla/service/hlo_reachability.cc  |  2 +-
 .../xla/service/hlo_rematerialization.cc      | 25 +++---
 .../compiler/xla/service/hlo_sharding.cc      |  9 +-
 .../compiler/xla/service/hlo_sharding.h       |  9 +-
 .../xla/service/hlo_tfgraph_builder.cc        |  3 +-
 tensorflow/compiler/xla/service/hlo_value.cc  |  2 +-
 .../service/human_readable_profile_builder.cc |  6 +-
 .../xla/service/instruction_fusion.cc         | 30 +++----
 .../compiler/xla/service/layout_assignment.cc |  9 +-
 .../xla/service/llvm_ir/alias_analysis.cc     |  4 +-
 .../compiler/xla/service/name_uniquer.cc      |  2 +-
 .../compiler/xla/service/platform_util.cc     |  4 +-
 .../compiler/xla/service/shape_inference.cc   | 22 +++--
 .../compiler/xla/service/transpose_folding.cc |  6 +-
 .../xla/service/tuple_points_to_analysis.cc   | 26 +++---
 .../service/tuple_points_to_analysis_test.cc  |  5 +-
 .../xla/service/while_loop_simplifier.cc      |  5 +-
 .../xla/service/while_loop_simplifier_test.cc | 11 ++-
 tensorflow/compiler/xla/shape.cc              |  3 +-
 tensorflow/compiler/xla/shape_util.cc         |  3 +-
 tensorflow/compiler/xla/sparse_index_array.h  |  2 +-
 .../compiler/xla/tests/convolution_test.cc    |  4 +-
 tensorflow/compiler/xla/util.cc               |  2 +-
 tensorflow/compiler/xla/util.h                |  3 +-
 tensorflow/compiler/xla/window_util.cc        | 32 ++++---
 55 files changed, 305 insertions(+), 380 deletions(-)

diff --git a/tensorflow/compiler/xla/BUILD b/tensorflow/compiler/xla/BUILD
index 8a0e2db2b5..45623c2718 100644
--- a/tensorflow/compiler/xla/BUILD
+++ b/tensorflow/compiler/xla/BUILD
@@ -717,6 +717,7 @@ cc_library(
         ":types",
         ":xla_data_proto",
         "//tensorflow/core:lib",
+        "@com_google_absl//absl/algorithm:container",
         "@com_google_absl//absl/strings",
         "@com_google_absl//absl/types:span",
     ],
diff --git a/tensorflow/compiler/xla/client/lib/sorting_test.cc b/tensorflow/compiler/xla/client/lib/sorting_test.cc
index 27ff36c749..0fbd138aca 100644
--- a/tensorflow/compiler/xla/client/lib/sorting_test.cc
+++ b/tensorflow/compiler/xla/client/lib/sorting_test.cc
@@ -77,7 +77,7 @@ XLA_TEST_F(SortingTest, TopKFullSort) {
   auto x = ConstantR1<float>(&builder, inputs);
   xla::GetTupleElement(xla::TopK(x, kSize), 0);
 
-  std::sort(inputs.begin(), inputs.end(), std::greater<float>());
+  absl::c_sort(inputs, std::greater<float>());
   ComputeAndCompareR1<float>(&builder, inputs, {});
 }
 
diff --git a/tensorflow/compiler/xla/layout_util.cc b/tensorflow/compiler/xla/layout_util.cc
index 42649d6692..2fe9b56c6b 100644
--- a/tensorflow/compiler/xla/layout_util.cc
+++ b/tensorflow/compiler/xla/layout_util.cc
@@ -290,8 +290,8 @@ Layout CreateDefaultLayoutForRank(int64 rank) {
 /* static */ bool LayoutUtil::HasLayout(const Shape& shape) {
   if (shape.IsTuple()) {
     // Tuple shape: all subshapes must have a layout.
-    return std::all_of(shape.tuple_shapes().begin(), shape.tuple_shapes().end(),
-                       [](const Shape& s) { return HasLayout(s); });
+    return absl::c_all_of(shape.tuple_shapes(),
+                          [](const Shape& s) { return HasLayout(s); });
   } else if (!shape.IsArray()) {
     // Opaque, token types etc. ignore layout.
     return true;
@@ -424,7 +424,7 @@ Status LayoutUtil::CopyLayoutBetweenShapes(const Shape& src, Shape* dst) {
     positions_in_layout.push_back(
         PositionInContainer(layout.minor_to_major(), dim));
   }
-  std::sort(positions_in_layout.begin(), positions_in_layout.end());
+  absl::c_sort(positions_in_layout);
   for (size_t i = 1; i < positions_in_layout.size(); ++i) {
     if (1 != positions_in_layout[i] - positions_in_layout[i - 1]) {
       return false;
diff --git a/tensorflow/compiler/xla/metric_table_report.cc b/tensorflow/compiler/xla/metric_table_report.cc
index 4eab4fa429..ad1699a1ae 100644
--- a/tensorflow/compiler/xla/metric_table_report.cc
+++ b/tensorflow/compiler/xla/metric_table_report.cc
@@ -55,7 +55,7 @@ string MetricTableReport::MakeReport(double expected_metric_sum) {
   const auto metric_greater = [](const Entry& a, const Entry& b) {
     return a.metric > b.metric;
   };
-  std::sort(entries_.begin(), entries_.end(), metric_greater);
+  absl::c_sort(entries_, metric_greater);
 
   // Create the report
   AppendLine();
@@ -117,7 +117,7 @@ std::vector<MetricTableReport::Category> MetricTableReport::MakeCategories(
   auto metric_sum_greater = [](const Category& a, const Category& b) {
     return a.metric_sum > b.metric_sum;
   };
-  std::sort(categories.begin(), categories.end(), metric_sum_greater);
+  absl::c_sort(categories, metric_sum_greater);
 
   return categories;
 }
diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index 85a21a8518..3c18c450e6 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -3414,6 +3414,7 @@ cc_library(
         ":hlo_profile_printer_data",
         ":human_readable_profile_builder",
         "//tensorflow/compiler/xla:types",
+        "@com_google_absl//absl/algorithm:container",
         "@com_google_absl//absl/strings",
     ],
 )
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
index 3d269dc04f..003cb7e791 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
@@ -2216,8 +2216,7 @@ Status AlgebraicSimplifierVisitor::HandleReverse(HloInstruction* reverse) {
   auto dim_is_one = [&](int64 i) -> bool {
     return reverse->shape().dimensions(i) == 1;
   };
-  if (std::all_of(reverse->dimensions().begin(), reverse->dimensions().end(),
-                  dim_is_one)) {
+  if (absl::c_all_of(reverse->dimensions(), dim_is_one)) {
     return ReplaceInstruction(reverse, reverse->mutable_operand(0));
   }
   return Status::OK();
@@ -2492,9 +2491,9 @@ Status AlgebraicSimplifierVisitor::HandleReduce(HloInstruction* reduce) {
     // Create a new reduce with the combined reduction dimensions of both
     // reduces.
     std::vector<int64> arg_dims = arg->dimensions();
-    std::sort(arg_dims.begin(), arg_dims.end());
+    absl::c_sort(arg_dims);
     std::vector<int64> reduce_dims = reduce->dimensions();
-    std::sort(reduce_dims.begin(), reduce_dims.end());
+    absl::c_sort(reduce_dims);
     // Transform reduce_dims to the same rank as the operand of the operand.
     for (int64 arg_dim : arg_dims) {
       for (int64& dim : reduce_dims) {
diff --git a/tensorflow/compiler/xla/service/buffer_assignment.cc b/tensorflow/compiler/xla/service/buffer_assignment.cc
index c997d594c4..c3285f516b 100644
--- a/tensorflow/compiler/xla/service/buffer_assignment.cc
+++ b/tensorflow/compiler/xla/service/buffer_assignment.cc
@@ -86,10 +86,9 @@ std::vector<int64> ColorInterferenceGraph(
   // first, but it would be good to investigate other ordering heuristics too.
   std::vector<int64> nodes(node_count);
   std::iota(nodes.begin(), nodes.end(), 0);
-  std::sort(nodes.begin(), nodes.end(),
-            [&interference_map](const int64 i, const int64 j) {
-              return interference_map[i].size() > interference_map[j].size();
-            });
+  absl::c_sort(nodes, [&interference_map](const int64 i, const int64 j) {
+    return interference_map[i].size() > interference_map[j].size();
+  });
 
   const int64 kColorUnassigned = -1;
   std::vector<int64> assigned_colors(node_count, kColorUnassigned);
@@ -272,11 +271,12 @@ BufferAllocationProto BufferAllocation::ToProto() const {
     proto_assigned->set_offset(buffer_offset_size.second.offset);
     proto_assigned->set_size(buffer_offset_size.second.size);
   }
-  std::sort(proto.mutable_assigned()->begin(), proto.mutable_assigned()->end(),
-            [](const BufferAllocationProto::Assigned& assign1,
-               const BufferAllocationProto::Assigned& assign2) {
-              return assign1.logical_buffer_id() < assign2.logical_buffer_id();
-            });
+  absl::c_sort(*proto.mutable_assigned(),
+               [](const BufferAllocationProto::Assigned& assign1,
+                  const BufferAllocationProto::Assigned& assign2) {
+                 return assign1.logical_buffer_id() <
+                        assign2.logical_buffer_id();
+               });
   return proto;
 }
 
@@ -308,10 +308,10 @@ string BufferAllocation::ToString() const {
   for (const auto& buffer_offset_size : assigned_buffers_) {
     sorted_buffers.push_back(buffer_offset_size.first);
   }
-  std::sort(sorted_buffers.begin(), sorted_buffers.end(),
-            [](const LogicalBuffer* a, const LogicalBuffer* b) {
-              return a->id() < b->id();
-            });
+  absl::c_sort(sorted_buffers,
+               [](const LogicalBuffer* a, const LogicalBuffer* b) {
+                 return a->id() < b->id();
+               });
   for (const LogicalBuffer* buffer : sorted_buffers) {
     const OffsetSize& offset_size = FindOrDie(assigned_buffers_, buffer);
     StrAppend(&output, absl::StrFormat(
@@ -479,10 +479,9 @@ bool BufferAssignment::HaveDisjointSlices(const HloInstruction* hlo_a,
   // didn't return the empty set) for both HLOs, and the two resulting sets of
   // slices are disjoint.
   return !slices_a.empty() && !slices_b.empty() &&
-         std::none_of(slices_a.begin(), slices_a.end(),
-                      [&](const BufferAllocation::Slice& slice) {
-                        return slices_b.count(slice) > 0;
-                      });
+         absl::c_none_of(slices_a, [&](const BufferAllocation::Slice& slice) {
+           return slices_b.contains(slice);
+         });
 }
 
 StatusOr<BufferAllocation::Slice>
@@ -952,28 +951,28 @@ Status BufferAssigner::AssignBuffersForComputation(
   // operands (assuming operands are the same/larger size) enabling the
   // important reuse case where an elementwise instruction reuses one of its
   // operand's buffer. This improves locality.
-  std::sort(sorted_buffers.begin(), sorted_buffers.end(),
-            [has_sequential_order, &liveness, &post_order_position, assignment](
-                const LogicalBuffer* a, const LogicalBuffer* b) {
-              // Primary sort is by decreasing buffer size.
-              const int64 a_size = assignment->buffer_size_(*a);
-              const int64 b_size = assignment->buffer_size_(*b);
-              if (a_size != b_size) {
-                return a_size > b_size;  // use ">" for decreasing size.
-              }
-              // Otherwise live out buffers come before others, if the
-              // instructions are sequentially ordered.
-              if (has_sequential_order) {
-                const bool a_live_out = liveness.MaybeLiveOut(*a);
-                const bool b_live_out = liveness.MaybeLiveOut(*b);
-                if (a_live_out != b_live_out) {
-                  return a_live_out;
-                }
-              }
-              // Final tiebreaker is in instruction post order.
-              return post_order_position.at(a->instruction()) <
-                     post_order_position.at(b->instruction());
-            });
+  absl::c_sort(sorted_buffers,
+               [has_sequential_order, &liveness, &post_order_position,
+                assignment](const LogicalBuffer* a, const LogicalBuffer* b) {
+                 // Primary sort is by decreasing buffer size.
+                 const int64 a_size = assignment->buffer_size_(*a);
+                 const int64 b_size = assignment->buffer_size_(*b);
+                 if (a_size != b_size) {
+                   return a_size > b_size;  // use ">" for decreasing size.
+                 }
+                 // Otherwise live out buffers come before others, if the
+                 // instructions are sequentially ordered.
+                 if (has_sequential_order) {
+                   const bool a_live_out = liveness.MaybeLiveOut(*a);
+                   const bool b_live_out = liveness.MaybeLiveOut(*b);
+                   if (a_live_out != b_live_out) {
+                     return a_live_out;
+                   }
+                 }
+                 // Final tiebreaker is in instruction post order.
+                 return post_order_position.at(a->instruction()) <
+                        post_order_position.at(b->instruction());
+               });
 
   // BufferAllocations are necessarily created in decreasing size order. Keep
   // indices of previously created BufferAllocations in allocation_indices.
@@ -1305,10 +1304,10 @@ std::vector<const LogicalBuffer*> ComputePeakMemoryLogicalBuffers(
                              live_buffers.end());
 
   // Stabily sort the live buffers.
-  std::sort(live_buffers_vector.begin(), live_buffers_vector.end(),
-            [](const LogicalBuffer* a, const LogicalBuffer* b) {
-              return a->id() < b->id();
-            });
+  absl::c_sort(live_buffers_vector,
+               [](const LogicalBuffer* a, const LogicalBuffer* b) {
+                 return a->id() < b->id();
+               });
   return live_buffers_vector;
 }
 
diff --git a/tensorflow/compiler/xla/service/call_graph_test.cc b/tensorflow/compiler/xla/service/call_graph_test.cc
index 7924134200..5de724f892 100644
--- a/tensorflow/compiler/xla/service/call_graph_test.cc
+++ b/tensorflow/compiler/xla/service/call_graph_test.cc
@@ -384,7 +384,7 @@ TEST_F(CallGraphTest, ComplexGraph) {
 
   // Verify visitation order of some computations in the graph.
   auto index_of = [&visited](const HloComputation* comp) {
-    auto it = std::find(visited.begin(), visited.end(), comp);
+    auto it = absl::c_find(visited, comp);
     EXPECT_NE(it, visited.end());
     return std::distance(visited.begin(), it);
   };
diff --git a/tensorflow/compiler/xla/service/computation_layout.cc b/tensorflow/compiler/xla/service/computation_layout.cc
index efc893818d..92d1ca4ba5 100644
--- a/tensorflow/compiler/xla/service/computation_layout.cc
+++ b/tensorflow/compiler/xla/service/computation_layout.cc
@@ -42,8 +42,8 @@ void ComputationLayout::SetToDefaultLayout() {
 }
 
 bool ComputationLayout::LayoutIsSet() const {
-  return std::all_of(parameter_layouts_.begin(), parameter_layouts_.end(),
-                     [](const ShapeLayout& s) { return s.LayoutIsSet(); }) &&
+  return absl::c_all_of(parameter_layouts_,
+                        [](const ShapeLayout& s) { return s.LayoutIsSet(); }) &&
          result_layout_.LayoutIsSet();
 }
 
diff --git a/tensorflow/compiler/xla/service/copy_insertion.cc b/tensorflow/compiler/xla/service/copy_insertion.cc
index e0165d557d..2b837901f0 100644
--- a/tensorflow/compiler/xla/service/copy_insertion.cc
+++ b/tensorflow/compiler/xla/service/copy_insertion.cc
@@ -539,10 +539,9 @@ class CopyRemover {
         }
 
         std::vector<const HloValue*> values = buffer.values();
-        std::sort(values.begin(), values.end(),
-                  [this](const HloValue* a, const HloValue* b) {
-                    return ordering_.IsDefinedBefore(*a, *b);
-                  });
+        absl::c_sort(values, [this](const HloValue* a, const HloValue* b) {
+          return ordering_.IsDefinedBefore(*a, *b);
+        });
 
         // Create a list containing all of the values in the buffer.
         AddValueList(values, &value_to_node);
@@ -842,12 +841,11 @@ class CopyRemover {
       copy_value_node->next->prev = operand_node;
 
       // Patch up uses. Remove use of copy from operand_node uses.
-      auto it =
-          std::find_if(operand_node->uses.begin(), operand_node->uses.end(),
-                       [copy_value_node](const HloUse* use) {
-                         return use->instruction ==
-                                copy_value_node->value->defining_instruction();
-                       });
+      auto it = absl::c_find_if(
+          operand_node->uses, [copy_value_node](const HloUse* use) {
+            return use->instruction ==
+                   copy_value_node->value->defining_instruction();
+          });
       CHECK(it != operand_node->uses.end());
       operand_node->uses.erase(it);
 
diff --git a/tensorflow/compiler/xla/service/cpu/disassembler.cc b/tensorflow/compiler/xla/service/cpu/disassembler.cc
index 3ae64142cd..c3c6847b7b 100644
--- a/tensorflow/compiler/xla/service/cpu/disassembler.cc
+++ b/tensorflow/compiler/xla/service/cpu/disassembler.cc
@@ -77,17 +77,16 @@ StatusOr<DisassemblerResult> Disassembler::DisassembleObjectFile(
     }
 
     // Sort the symbols in increasing address order.
-    std::sort(
-        symbols.begin(), symbols.end(),
-        [](const llvm::object::SymbolRef& a, const llvm::object::SymbolRef& b) {
-          // getAddress returns a Expected object. Assert there is no error
-          // before extracting the address.
-          llvm::Expected<uint64_t> a_address_or_error = a.getAddress();
-          CHECK(a_address_or_error);
-          llvm::Expected<uint64_t> b_address_or_error = b.getAddress();
-          CHECK(b_address_or_error);
-          return a_address_or_error.get() < b_address_or_error.get();
-        });
+    absl::c_sort(symbols, [](const llvm::object::SymbolRef& a,
+                             const llvm::object::SymbolRef& b) {
+      // getAddress returns a Expected object. Assert there is no error
+      // before extracting the address.
+      llvm::Expected<uint64_t> a_address_or_error = a.getAddress();
+      CHECK(a_address_or_error);
+      llvm::Expected<uint64_t> b_address_or_error = b.getAddress();
+      CHECK(b_address_or_error);
+      return a_address_or_error.get() < b_address_or_error.get();
+    });
 
     // Construct ArrayRef pointing to section contents.
     llvm::StringRef section_content_string;
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
index e4760aedf1..b352848824 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
@@ -1709,10 +1709,8 @@ StatusOr<bool> IrEmitter::EmitVectorizedReduce(
       vectorization_factor_in_bytes /
       ShapeUtil::ByteSizeOfPrimitiveType(reduce->shape().element_type());
 
-  bool is_reduction_over_minor_dimension =
-      std::find(dimensions.begin(), dimensions.end(),
-                LayoutUtil::Minor(arg->shape().layout(), 0)) !=
-      dimensions.end();
+  bool is_reduction_over_minor_dimension = absl::c_linear_search(
+      dimensions, LayoutUtil::Minor(arg->shape().layout(), 0));
 
   unsigned element_alignment = tensorflow::MathUtil::GCD<unsigned>(
       ShapeUtil::ByteSizeOfPrimitiveType(reduce->shape().element_type()),
@@ -2401,8 +2399,7 @@ StatusOr<bool> IrEmitter::EmitFastConcatenate(
   int64 concat_dim = concatenate->dimensions(0);
   const Layout& output_layout = output_shape.layout();
   auto output_min2maj = LayoutUtil::MinorToMajor(output_layout);
-  auto concat_dim_layout_itr =
-      std::find(output_min2maj.begin(), output_min2maj.end(), concat_dim);
+  auto concat_dim_layout_itr = absl::c_find(output_min2maj, concat_dim);
 
   std::vector<int64> inner_dims(output_min2maj.begin(), concat_dim_layout_itr);
   std::vector<int64> outer_dims(std::next(concat_dim_layout_itr),
@@ -2956,8 +2953,7 @@ Status IrEmitter::ElementTypesSameAndSupported(
 
   TF_RET_CHECK(!operands.empty());
   PrimitiveType primitive_type = operands[0]->shape().element_type();
-  if (std::find(supported_types.begin(), supported_types.end(),
-                primitive_type) == supported_types.end()) {
+  if (!absl::c_linear_search(supported_types, primitive_type)) {
     return Unimplemented("unsupported operand type %s in op %s",
                          PrimitiveType_Name(primitive_type),
                          HloOpcodeString(instruction.opcode()));
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emission_utils.cc b/tensorflow/compiler/xla/service/gpu/ir_emission_utils.cc
index 5d25a032a9..caccb18899 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emission_utils.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emission_utils.cc
@@ -154,20 +154,17 @@ bool IsReductionToVector(const HloInstruction& reduce) {
   const HloInstruction* input = reduce.operand(0);
   std::vector<int64> dims_to_keep;
   for (int64 dim = 0; dim < input->shape().dimensions().size(); ++dim) {
-    if (!std::count(reduce.dimensions().begin(), reduce.dimensions().end(),
-                    dim)) {
+    if (!absl::c_linear_search(reduce.dimensions(), dim)) {
       dims_to_keep.push_back(dim);
     }
   }
   return LayoutUtil::AreDimensionsConsecutive(input->shape().layout(),
                                               dims_to_keep) &&
-         ShapeUtil::Equal(reduce.shape(), ShapeUtil::FilterDimensions(
-                                              [&dims_to_keep](int64 dim) {
-                                                return std::count(
-                                                    dims_to_keep.begin(),
-                                                    dims_to_keep.end(), dim);
-                                              },
-                                              input->shape()));
+         ShapeUtil::Equal(
+             reduce.shape(),
+             ShapeUtil::FilterDimensions(
+                 [&](int64 dim) { return absl::c_count(dims_to_keep, dim); },
+                 input->shape()));
 }
 
 // This emits a device-side call to
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
index 063d493d90..923b3d5945 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
@@ -1506,10 +1506,10 @@ std::unique_ptr<KernelThunk> IrEmitterUnnested::BuildKernelThunk(
                     return !allocation->is_constant();
                   });
 
-  std::sort(non_constant_buffers.begin(), non_constant_buffers.end(),
-            [](const BufferAllocation* a, const BufferAllocation* b) {
-              return a->index() < b->index();
-            });
+  absl::c_sort(non_constant_buffers,
+               [](const BufferAllocation* a, const BufferAllocation* b) {
+                 return a->index() < b->index();
+               });
 
   llvm::Function* kernel = BuildKernelPrototype(*inst, non_constant_buffers);
 
diff --git a/tensorflow/compiler/xla/service/heap_simulator.cc b/tensorflow/compiler/xla/service/heap_simulator.cc
index 70ae7078ae..4fca981c6a 100644
--- a/tensorflow/compiler/xla/service/heap_simulator.cc
+++ b/tensorflow/compiler/xla/service/heap_simulator.cc
@@ -225,10 +225,10 @@ Status HeapSimulator::RunComputation(
       }
     }
     // Sort to get a deterministic iteration order.
-    std::sort(operand_buffers_to_free.begin(), operand_buffers_to_free.end(),
-              [](const BufferValue* x, const BufferValue* y) {
-                return x->id() < y->id();
-              });
+    absl::c_sort(operand_buffers_to_free,
+                 [](const BufferValue* x, const BufferValue* y) {
+                   return x->id() < y->id();
+                 });
 
     // Allocate buffers defined by this instruction.  This is the latest point
     // that we can allocate; right before the buffer is first used.  This must
@@ -335,10 +335,9 @@ Status HeapSimulator::RunComputation(
     to_free.push_back(buffer);
   }
 
-  std::sort(to_free.begin(), to_free.end(),
-            [](const BufferValue* x, const BufferValue* y) {
-              return x->id() < y->id();
-            });
+  absl::c_sort(to_free, [](const BufferValue* x, const BufferValue* y) {
+    return x->id() < y->id();
+  });
   for (const BufferValue* buffer : to_free) {
     VLOG(3) << "Freeing pending: " << buffer->ToString();
     Free(buffer, root);
@@ -596,7 +595,7 @@ void DecreasingSizeRunsHeap::CallAndDrainRun() {
   }
 
   // Call ops in the run sorted by decreasing size, breaking ties by buffer id.
-  std::sort(run_.begin(), run_.end(), [](const Op& a, const Op& b) {
+  absl::c_sort(run_, [](const Op& a, const Op& b) {
     if (a.size != b.size) {
       return a.size > b.size;
     }
@@ -866,23 +865,23 @@ HeapSimulator::Result GlobalDecreasingSizeBestFitHeap::Finish() {
   for (auto& entry : buffer_intervals_) {
     sorted_buffer_intervals.push_back(entry.second);
   }
-  std::sort(sorted_buffer_intervals.begin(), sorted_buffer_intervals.end(),
-            [](const BufferInterval& x, const BufferInterval& y) {
-              if (x.size != y.size) {
-                return x.size > y.size;
-              }
-              if (x.end - x.start != y.end - y.start) {
-                return x.end - x.start > y.end - y.start;
-              }
-              return x.buffer->id() < y.buffer->id();
-            });
+  absl::c_sort(sorted_buffer_intervals,
+               [](const BufferInterval& x, const BufferInterval& y) {
+                 if (x.size != y.size) {
+                   return x.size > y.size;
+                 }
+                 if (x.end - x.start != y.end - y.start) {
+                   return x.end - x.start > y.end - y.start;
+                 }
+                 return x.buffer->id() < y.buffer->id();
+               });
 
   BufferIntervalTree interval_tree(sorted_buffer_intervals.size());
   for (auto& buffer_interval : sorted_buffer_intervals) {
     auto chunks_overlapping_in_time = interval_tree.ChunksOverlappingInTime(
         buffer_interval.start, buffer_interval.end);
-    std::sort(
-        chunks_overlapping_in_time.begin(), chunks_overlapping_in_time.end(),
+    absl::c_sort(
+        chunks_overlapping_in_time,
         [](const Chunk& x, const Chunk& y) { return x.offset < y.offset; });
 
     // Find the minimum free chunk that can hold this buffer.
diff --git a/tensorflow/compiler/xla/service/hlo_alias_analysis.cc b/tensorflow/compiler/xla/service/hlo_alias_analysis.cc
index 68094d8907..4c46ea595e 100644
--- a/tensorflow/compiler/xla/service/hlo_alias_analysis.cc
+++ b/tensorflow/compiler/xla/service/hlo_alias_analysis.cc
@@ -117,7 +117,7 @@ class BufferValueMap {
     for (const auto& pair : buffers_) {
       buffer_numbers.push_back(pair.first);
     }
-    std::sort(buffer_numbers.begin(), buffer_numbers.end());
+    absl::c_sort(buffer_numbers);
     return buffer_numbers;
   }
 
@@ -319,7 +319,7 @@ class BufferValueMap {
     ComputeWhileAliasedBuffers(value, &aliased_buffers);
     ComputeConditionalAliasedBuffers(value, &aliased_buffers);
     // Uniquify aliased buffers.
-    std::sort(aliased_buffers.begin(), aliased_buffers.end());
+    absl::c_sort(aliased_buffers);
     aliased_buffers.erase(
         std::unique(aliased_buffers.begin(), aliased_buffers.end()),
         aliased_buffers.end());
@@ -367,7 +367,7 @@ std::vector<const HloBuffer*> HloAliasAnalysis::ComputeBuffersAt(
   }
 
   // Sort and uniquify vector before returning.
-  std::sort(buffers.begin(), buffers.end(), HloBuffer::IdLessThan);
+  absl::c_sort(buffers, HloBuffer::IdLessThan);
   buffers.erase(std::unique(buffers.begin(), buffers.end()), buffers.end());
 
   return buffers;
@@ -430,8 +430,7 @@ Status HloAliasAnalysis::Verify() const {
   for (const auto& pair : value_to_buffer_) {
     const HloValue* value = pair.first;
     const HloBuffer& buffer = *pair.second;
-    TF_RET_CHECK(std::find(buffer.values().begin(), buffer.values().end(),
-                           value) != buffer.values().end());
+    TF_RET_CHECK(absl::c_linear_search(buffer.values(), value));
   }
 
   for (HloBuffer::Id id = 0; id < buffers_.size(); ++id) {
@@ -515,7 +514,7 @@ StatusOr<std::unique_ptr<HloAliasAnalysis>> HloAliasAnalysis::Run(
     auto& value_set = buffer_map.GetValuesInBuffer(buffer_number);
     std::vector<const HloValue*> sorted_values(value_set.begin(),
                                                value_set.end());
-    std::sort(sorted_values.begin(), sorted_values.end(), HloValue::IdLessThan);
+    absl::c_sort(sorted_values, HloValue::IdLessThan);
     alias_analysis->buffers_.emplace_back(next_id++, sorted_values);
     for (const HloValue* value : sorted_values) {
       alias_analysis->value_to_buffer_[value] =
@@ -547,16 +546,15 @@ bool HloAliasAnalysis::HasLiveRangeInterference(
     // tie-break using value ID. The tie-break is necessary because we need a
     // strict weak order for std::sort.
     std::vector<const HloValue*> values = buffer.values();
-    std::sort(values.begin(), values.end(),
-              [&ordering](const HloValue* a, const HloValue* b) {
-                if (ordering.IsDefinedBefore(*a, *b)) {
-                  return true;
-                } else if (ordering.IsDefinedBefore(*b, *a)) {
-                  return false;
-                } else {
-                  return a->id() < b->id();
-                }
-              });
+    absl::c_sort(values, [&ordering](const HloValue* a, const HloValue* b) {
+      if (ordering.IsDefinedBefore(*a, *b)) {
+        return true;
+      } else if (ordering.IsDefinedBefore(*b, *a)) {
+        return false;
+      } else {
+        return a->id() < b->id();
+      }
+    });
 
     // Walk through the ordered vector of values. First verify that the values
     // are totally ordered with respect to 'ordering', then check that no
diff --git a/tensorflow/compiler/xla/service/hlo_buffer.cc b/tensorflow/compiler/xla/service/hlo_buffer.cc
index 9c3aa0e64d..32e48651b3 100644
--- a/tensorflow/compiler/xla/service/hlo_buffer.cc
+++ b/tensorflow/compiler/xla/service/hlo_buffer.cc
@@ -49,7 +49,7 @@ std::vector<HloPosition> HloBuffer::ComputePositions() const {
                      value->positions().end());
   }
   // Remove duplicates and sort positions.
-  std::sort(positions.begin(), positions.end());
+  absl::c_sort(positions);
   positions.erase(std::unique(positions.begin(), positions.end()),
                   positions.end());
   return positions;
diff --git a/tensorflow/compiler/xla/service/hlo_computation.cc b/tensorflow/compiler/xla/service/hlo_computation.cc
index 10f0cf3a34..9e15722633 100644
--- a/tensorflow/compiler/xla/service/hlo_computation.cc
+++ b/tensorflow/compiler/xla/service/hlo_computation.cc
@@ -531,11 +531,10 @@ HloComputation::CreateFromProto(
   HloInstruction* root = instruction_map.at(proto.root_id());
 
   // Sort the instructions in the proto id's order.
-  std::sort(instructions.begin(), instructions.end(),
-            [&](const std::unique_ptr<HloInstruction>& a,
-                const std::unique_ptr<HloInstruction>& b) {
-              return to_proto_id[a.get()] < to_proto_id[b.get()];
-            });
+  absl::c_sort(instructions, [&](const std::unique_ptr<HloInstruction>& a,
+                                 const std::unique_ptr<HloInstruction>& b) {
+    return to_proto_id[a.get()] < to_proto_id[b.get()];
+  });
 
   TF_RETURN_IF_ERROR([&]() -> Status {
     std::vector<bool> parameters_seen(parameter_count);
@@ -800,8 +799,7 @@ Status HloComputation::AcceptOrdered(
     absl::Span<HloInstruction* const> order) const {
   VLOG(3) << "Accepting visitor with order.";
   for (HloInstruction* root : CollectUnreachableRoots()) {
-    TF_RET_CHECK(std::find(order.begin(), order.end(), root) != order.end())
-        << root->ToString();
+    TF_RET_CHECK(absl::c_linear_search(order, root)) << root->ToString();
   }
   TF_RET_CHECK(order.size() == instruction_count());
   absl::flat_hash_set<const HloInstruction*> visited;
diff --git a/tensorflow/compiler/xla/service/hlo_dataflow_analysis.cc b/tensorflow/compiler/xla/service/hlo_dataflow_analysis.cc
index 0db452b85f..3144a84805 100644
--- a/tensorflow/compiler/xla/service/hlo_dataflow_analysis.cc
+++ b/tensorflow/compiler/xla/service/hlo_dataflow_analysis.cc
@@ -256,7 +256,7 @@ bool HloDataflowAnalysis::Phi(
         input_value_ids.push_back(value->id());
       }
     }
-    std::sort(input_value_ids.begin(), input_value_ids.end());
+    absl::c_sort(input_value_ids);
     input_value_ids.erase(
         std::unique(input_value_ids.begin(), input_value_ids.end()),
         input_value_ids.end());
@@ -271,8 +271,7 @@ bool HloDataflowAnalysis::Phi(
     if (current_value_defined_here) {
       VLOG(5) << "current_value_defined_here: " << current_value->ToString();
       CHECK(current_value->is_phi());
-      auto it = std::find(input_value_ids.begin(), input_value_ids.end(),
-                          current_value->id());
+      auto it = absl::c_find(input_value_ids, current_value->id());
       if (it != input_value_ids.end()) {
         input_value_ids.erase(it);
       }
@@ -921,8 +920,7 @@ StatusOr<std::unique_ptr<HloDataflowAnalysis>> HloDataflowAnalysis::Run(
   for (auto& pair : dataflow_analysis->values_) {
     dataflow_analysis->values_vector_.push_back(&pair.second);
   }
-  std::sort(dataflow_analysis->values_vector_.begin(),
-            dataflow_analysis->values_vector_.end(), HloValue::IdLessThan);
+  absl::c_sort(dataflow_analysis->values_vector_, HloValue::IdLessThan);
 
   TF_DCHECK_OK(dataflow_analysis->Verify());
 
@@ -937,9 +935,7 @@ Status HloDataflowAnalysis::Verify() const {
   for (const HloValue* value : values()) {
     for (const HloPosition& position : value->positions()) {
       const HloValueSet& value_set = GetValueSet(position);
-      TF_RET_CHECK(std::find(value_set.values().begin(),
-                             value_set.values().end(),
-                             value) != value_set.values().end())
+      TF_RET_CHECK(absl::c_linear_search(value_set.values(), value))
           << "Value set at position " << position << " does not contain value "
           << value->ToShortString();
     }
@@ -954,9 +950,7 @@ Status HloDataflowAnalysis::Verify() const {
         const HloValueSet& value_set = pair.second;
         const HloPosition position{instruction, index};
         for (const HloValue* value : value_set.values()) {
-          TF_RET_CHECK(std::find(value->positions().begin(),
-                                 value->positions().end(),
-                                 position) != value->positions().end())
+          TF_RET_CHECK(absl::c_linear_search(value->positions(), position))
               << "Value set at position " << position
               << " unexpectedly contains value " << value->ToShortString();
         }
@@ -1041,11 +1035,10 @@ bool HloDataflowAnalysis::CanShareOperandBufferWithUser(
       // Check if one operand of kAdd fused root is kDot or kConvolution.
       auto* add = user->fused_expression_root();
       auto add_operand_it =
-          std::find_if(add->operands().begin(), add->operands().end(),
-                       [&](HloInstruction* operand) {
-                         return operand->opcode() == HloOpcode::kConvolution ||
-                                operand->opcode() == HloOpcode::kDot;
-                       });
+          absl::c_find_if(add->operands(), [&](HloInstruction* operand) {
+            return operand->opcode() == HloOpcode::kConvolution ||
+                   operand->opcode() == HloOpcode::kDot;
+          });
       if (add_operand_it == add->operands().end()) {
         return false;
       }
@@ -1100,16 +1093,15 @@ bool HloDataflowAnalysis::CanShareOperandBufferWithUser(
     // *) The root instruction of the called computation is element-wise on
     //    'operand'.
     const bool found_caller_use =
-        std::find_if(uses.begin(), uses.end(), [user](const HloUse& use) {
+        absl::c_find_if(uses, [user](const HloUse& use) {
           return use.instruction == user;
         }) != uses.end();
     auto* callee_root = user->to_apply()->root_instruction();
     const bool found_elementwise_callee_use =
-        std::find_if(
-            uses.begin(), uses.end(), [callee_root](const HloUse& use) {
-              return use.instruction == callee_root &&
-                     callee_root->IsElementwiseOnOperand(use.operand_number);
-            }) != uses.end();
+        absl::c_find_if(uses, [callee_root](const HloUse& use) {
+          return use.instruction == callee_root &&
+                 callee_root->IsElementwiseOnOperand(use.operand_number);
+        }) != uses.end();
     return uses.size() == 2 && found_caller_use && found_elementwise_callee_use;
   }
 
diff --git a/tensorflow/compiler/xla/service/hlo_dce_test.cc b/tensorflow/compiler/xla/service/hlo_dce_test.cc
index 1fa4259a3e..b5d72b386f 100644
--- a/tensorflow/compiler/xla/service/hlo_dce_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_dce_test.cc
@@ -43,9 +43,7 @@ class HloDceTest : public HloTestBase {
   // Returns whether the given instruction exists in the given computation.
   bool HasInstruction(const HloComputation& computation,
                       const HloInstruction* instruction) {
-    return std::find(computation.instructions().begin(),
-                     computation.instructions().end(),
-                     instruction) != computation.instructions().end();
+    return absl::c_linear_search(computation.instructions(), instruction);
   }
 };
 
diff --git a/tensorflow/compiler/xla/service/hlo_domain_map.cc b/tensorflow/compiler/xla/service/hlo_domain_map.cc
index c6d02f9f67..7cdb7f6bdf 100644
--- a/tensorflow/compiler/xla/service/hlo_domain_map.cc
+++ b/tensorflow/compiler/xla/service/hlo_domain_map.cc
@@ -230,10 +230,10 @@ HloDomainMap::MakeNonDomainInstructions(
     }
   }
   // sort instructions according to instructions_order
-  std::sort(instructions.begin(), instructions.end(),
-            [&instructions_order](HloInstruction* a, HloInstruction* b) {
-              return instructions_order.at(a) < instructions_order.at(b);
-            });
+  absl::c_sort(instructions,
+               [&instructions_order](HloInstruction* a, HloInstruction* b) {
+                 return instructions_order.at(a) < instructions_order.at(b);
+               });
   return instructions;
 }
 
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc
index 9754b8906b..3a97b56c66 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc
@@ -1248,8 +1248,7 @@ StatusOr<Literal> EvaluateSortInternal(HloInstruction* sort,
         // Extract a slice from the keys and values literals that correspond to
         // exactly the row in dimension 'sort_dim'.
         std::vector<int64> limit_indices(indices.begin(), indices.end());
-        std::for_each(limit_indices.begin(), limit_indices.end(),
-                      [](int64& index) { ++index; });
+        absl::c_for_each(limit_indices, [](int64& index) { ++index; });
         limit_indices[sort_dim] = sort_dim_elements;
         TF_ASSIGN_OR_RETURN(auto keys_to_sort,
                             keys_literal.Slice(indices, limit_indices)
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
index cf3509d0aa..cc5f6cc682 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
@@ -1673,8 +1673,7 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
           // Extract a slice from the literal that corresponds to exactly the
           // row in dimension 'sort_dim'.
           std::vector<int64> limit_indices(indices.begin(), indices.end());
-          std::for_each(limit_indices.begin(), limit_indices.end(),
-                        [](int64& index) { ++index; });
+          absl::c_for_each(limit_indices, [](int64& index) { ++index; });
           limit_indices[sort_dim] = sort_dim_elements;
           TF_ASSIGN_OR_RETURN(auto row_to_sort,
                               keys_literal.Slice(indices, limit_indices)
diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
index 22bcb2030b..4c7f5e9e7d 100644
--- a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
+++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
@@ -561,8 +561,8 @@ bool HloDotDumper::ShouldShowSubcomputation(const HloComputation* subcomp) {
   }
 
   // Show the subcomputation if we're showing any of its members.
-  return std::any_of(
-      subcomp->instructions().begin(), subcomp->instructions().end(),
+  return absl::c_any_of(
+      subcomp->instructions(),
       [&](const HloInstruction* instr) { return filter_.Show(instr); });
 }
 
@@ -735,15 +735,14 @@ bool HloDotDumper::ShouldMergeIntoUsers(const HloInstruction* instr) const {
   const int kMinUsersToOmit = 3;
   return instr->opcode() == HloOpcode::kParameter && instr->shape().IsTuple() &&
          !instr->IsFused() &&
-         std::count_if(instr->users().begin(), instr->users().end(),
-                       [&](const HloInstruction* user) {
-                         return filter_.Show(user);
-                       }) > kMinUsersToOmit &&
-         std::all_of(instr->users().begin(), instr->users().end(),
-                     [&](const HloInstruction* user) {
-                       return !filter_.Show(user) ||
-                              user->opcode() == HloOpcode::kGetTupleElement;
-                     });
+         absl::c_count_if(instr->users(),
+                          [&](const HloInstruction* user) {
+                            return filter_.Show(user);
+                          }) > kMinUsersToOmit &&
+         absl::c_all_of(instr->users(), [&](const HloInstruction* user) {
+           return !filter_.Show(user) ||
+                  user->opcode() == HloOpcode::kGetTupleElement;
+         });
 }
 
 string HloDotDumper::DumpInstruction(const HloInstruction* instr) {
@@ -900,12 +899,11 @@ ColorScheme HloDotDumper::GetInstructionColor(const HloInstruction* instr) {
   // the same color as a parameter.  Unless the merged-in parameter is a
   // parameter to a fusion node that is bound to a constant -- these aren't
   // "real" parameters from the user's perspective.
-  if (std::any_of(instr->operands().begin(), instr->operands().end(),
-                  [&](const HloInstruction* operand) {
-                    return operand->opcode() == HloOpcode::kParameter &&
-                           ShouldMergeIntoUsers(operand) &&
-                           TryGetFusionParameterConstant(operand) == nullptr;
-                  })) {
+  if (absl::c_any_of(instr->operands(), [&](const HloInstruction* operand) {
+        return operand->opcode() == HloOpcode::kParameter &&
+               ShouldMergeIntoUsers(operand) &&
+               TryGetFusionParameterConstant(operand) == nullptr;
+      })) {
     return parameter_color;
   }
 
@@ -1355,12 +1353,11 @@ NodeFilter MakeNodeRadiusAroundFilter(const HloInstruction* root,
     NodeFilterResult& filter_result = kv.second;
     const auto& operands = instr->operands();
 
-    if (std::any_of(operands.begin(), operands.end(), is_displayed) &&
-        !std::all_of(operands.begin(), operands.end(), is_displayed)) {
+    if (absl::c_any_of(operands, is_displayed) &&
+        !absl::c_all_of(operands, is_displayed)) {
       // Mark nodes with some operands omitted appropriately.
       filter_result = kSomeOperandsOmitted;
-    } else if (!operands.empty() &&
-               std::none_of(operands.begin(), operands.end(), is_displayed)) {
+    } else if (!operands.empty() && absl::c_none_of(operands, is_displayed)) {
       // Mark nodes with *all* operands omitted appropriately.
       filter_result = kOmitNodeOperands;
     }
@@ -1368,8 +1365,7 @@ NodeFilter MakeNodeRadiusAroundFilter(const HloInstruction* root,
     // Promote nodes with type kSomeUsersOmitted to kNormalNode if all of their
     // users made it into the graph.
     if (filter_result == kSomeUsersOmitted &&
-        std::all_of(instr->users().begin(), instr->users().end(),
-                    is_displayed)) {
+        absl::c_all_of(instr->users(), is_displayed)) {
       filter_result = kNormalNode;
     }
   }
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc
index 66bc73740f..029d170317 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.cc
+++ b/tensorflow/compiler/xla/service/hlo_instruction.cc
@@ -83,15 +83,14 @@ StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
     return computation_map.at(proto.called_computation_ids(index));
   };
 
-  TF_RET_CHECK(std::all_of(
-      proto.operand_ids().begin(), proto.operand_ids().end(),
-      [&instruction_map](int64 id) { return instruction_map.contains(id); }))
+  TF_RET_CHECK(
+      absl::c_all_of(proto.operand_ids(),
+                     [&](int64 id) { return instruction_map.contains(id); }))
       << proto.name() << " instruction contains invalid operand id(s)";
 
-  TF_RET_CHECK(std::all_of(
-      proto.called_computation_ids().begin(),
-      proto.called_computation_ids().end(),
-      [&computation_map](int64 id) { return computation_map.contains(id); }))
+  TF_RET_CHECK(
+      absl::c_all_of(proto.called_computation_ids(),
+                     [&](int64 id) { return computation_map.contains(id); }))
       << proto.name() << " instruction references invalid computation id(s)";
 
   Shape shape(proto.shape());
@@ -1599,12 +1598,10 @@ HloInstruction::InstructionVector HloInstruction::unique_operands() const {
 
 Status HloInstruction::AddControlDependencyTo(HloInstruction* instruction) {
   TF_RET_CHECK(instruction->parent() == parent());
-  if (std::find(control_successors_.begin(), control_successors_.end(),
-                instruction) == control_successors_.end()) {
+  if (!absl::c_linear_search(control_successors_, instruction)) {
     control_successors_.push_back(instruction);
-    TF_RET_CHECK(std::find(instruction->control_predecessors_.begin(),
-                           instruction->control_predecessors_.end(),
-                           this) == instruction->control_predecessors_.end());
+    TF_RET_CHECK(
+        !absl::c_linear_search(instruction->control_predecessors_, this));
     instruction->control_predecessors_.push_back(this);
   }
   return Status::OK();
@@ -1853,7 +1850,7 @@ void HloInstruction::RemoveUser(HloInstruction* user) {
   user_set_.erase(set_it);
   // This is linear in the number of the users, but a vector provides a stable
   // iteration order and much faster traversal.
-  auto vec_it = std::find(users_.begin(), users_.end(), user);
+  auto vec_it = absl::c_find(users_, user);
   CHECK(vec_it != users_.end());
   users_.erase(vec_it);
 }
@@ -1871,8 +1868,7 @@ Status HloInstruction::ReplaceUseWith(HloInstruction* user,
 
   RemoveUser(user);
 
-  TF_RET_CHECK(
-      std::count(user->operands_.begin(), user->operands_.end(), this) >= 0);
+  TF_RET_CHECK(absl::c_count(user->operands_, this) >= 0);
   std::replace(user->operands_.begin(), user->operands_.end(), this,
                new_producer);
   new_producer->AddUser(user);
@@ -1907,8 +1903,7 @@ Status HloInstruction::ReplaceOperandWithDifferentShape(
   VLOG(3) << "Replacing operand " << operand_num << " of " << name() << " with "
           << new_operand->name() << ", was " << old_operand->name();
 
-  if (std::find(operands_.begin(), operands_.end(), old_operand) ==
-      operands_.end()) {
+  if (!absl::c_linear_search(operands_, old_operand)) {
     old_operand->RemoveUser(this);
   }
   new_operand->AddUser(this);
@@ -2945,10 +2940,10 @@ StatusOr<HloInstruction::FusionKind> StringToFusionKind(
 
 string PaddingConfigToString(const PaddingConfig& padding) {
   bool has_interior_padding =
-      std::any_of(padding.dimensions().begin(), padding.dimensions().end(),
-                  [](const PaddingConfig::PaddingConfigDimension& dim) {
-                    return dim.interior_padding() != 0;
-                  });
+      absl::c_any_of(padding.dimensions(),
+                     [](const PaddingConfig::PaddingConfigDimension& dim) {
+                       return dim.interior_padding() != 0;
+                     });
   return StrJoin(
       padding.dimensions(), "x",
       [&](string* out, const PaddingConfig::PaddingConfigDimension& dim) {
diff --git a/tensorflow/compiler/xla/service/hlo_instructions.cc b/tensorflow/compiler/xla/service/hlo_instructions.cc
index 7170dd7d81..c24bf41ff8 100644
--- a/tensorflow/compiler/xla/service/hlo_instructions.cc
+++ b/tensorflow/compiler/xla/service/hlo_instructions.cc
@@ -42,11 +42,9 @@ using absl::StrJoin;
 bool IsInstructionElementwiseOnOperand(const HloInstruction* instruction,
                                        const HloInstruction* operand) {
   std::vector<int64> operand_indices = instruction->OperandIndices(operand);
-  return std::all_of(
-      operand_indices.begin(), operand_indices.end(),
-      [instruction](int64 operand_index) {
-        return instruction->IsElementwiseOnOperand(operand_index);
-      });
+  return absl::c_all_of(operand_indices, [instruction](int64 operand_index) {
+    return instruction->IsElementwiseOnOperand(operand_index);
+  });
 }
 
 string PrecisionConfigToString(const PrecisionConfig& precision_config) {
@@ -814,8 +812,7 @@ std::vector<string> HloSliceInstruction::ExtraAttributesToStringImpl(
   std::vector<string> bounds;
   bounds.reserve(slice_starts_.size());
   const bool omit_stride =
-      std::all_of(slice_strides_.begin(), slice_strides_.end(),
-                  [](int64 stride) { return stride == 1; });
+      absl::c_all_of(slice_strides_, [](int64 stride) { return stride == 1; });
   for (int i = 0; i < slice_starts_.size(); ++i) {
     string stride_str = omit_stride ? "" : StrCat(":", slice_strides_[i]);
     bounds.push_back(
@@ -1051,8 +1048,7 @@ HloInstruction* HloFusionInstruction::AddFusionOperand(
 
 void HloFusionInstruction::MergeFusionInstruction(
     HloFusionInstruction* instruction_to_merge) {
-  CHECK(std::find(operands().begin(), operands().end(), instruction_to_merge) !=
-        operands().end());
+  CHECK(absl::c_linear_search(operands(), instruction_to_merge));
   // Clone the instruction from which to merge fused instructions.
   std::unique_ptr<HloInstruction> cloned = instruction_to_merge->Clone();
   HloFusionInstruction* cloned_fusion =
@@ -1219,8 +1215,8 @@ HloInstruction* HloFusionInstruction::CloneAndFuseInternal(
     // corresponding fused parameter instruction. Renumber parameters as
     // necessary to make parameter numbers consistent with their index in the
     // fused_parameter_ vector.
-    bool in_operand_list = std::find(operands().begin(), operands().end(),
-                                     instruction_to_fuse) != operands().end();
+    bool in_operand_list =
+        absl::c_linear_search(operands(), instruction_to_fuse);
     CHECK(add_output || in_operand_list);
     if (instruction_to_fuse->opcode() == HloOpcode::kTuple) {
       // We assume all uses of a kTuple operation are GTE ops, not another
diff --git a/tensorflow/compiler/xla/service/hlo_module.cc b/tensorflow/compiler/xla/service/hlo_module.cc
index ed78189fed..258f918f47 100644
--- a/tensorflow/compiler/xla/service/hlo_module.cc
+++ b/tensorflow/compiler/xla/service/hlo_module.cc
@@ -107,11 +107,10 @@ HloComputation* HloModule::AddEntryComputation(
 }
 
 Status HloModule::RemoveEmbeddedComputation(HloComputation* to_remove) {
-  auto it =
-      std::find_if(computations_.begin(), computations_.end(),
-                   [&to_remove](const std::unique_ptr<HloComputation>& comp) {
-                     return comp.get() == to_remove;
-                   });
+  auto it = absl::c_find_if(
+      computations_, [&to_remove](const std::unique_ptr<HloComputation>& comp) {
+        return comp.get() == to_remove;
+      });
   TF_RET_CHECK(it->get() == to_remove);
   computations_.erase(it);
   return Status::OK();
@@ -304,11 +303,10 @@ StatusOr<std::unique_ptr<HloModule>> HloModule::CreateFromProto(
   auto module = absl::make_unique<HloModule>(proto.name(), module_config);
 
   // Sort the computations in the proto id's order.
-  std::sort(computations.begin(), computations.end(),
-            [&](const std::unique_ptr<HloComputation>& a,
-                const std::unique_ptr<HloComputation>& b) {
-              return to_proto_id[a.get()] < to_proto_id[b.get()];
-            });
+  absl::c_sort(computations, [&](const std::unique_ptr<HloComputation>& a,
+                                 const std::unique_ptr<HloComputation>& b) {
+    return to_proto_id[a.get()] < to_proto_id[b.get()];
+  });
 
   // Add sorted computations to the module.
   for (auto& computation : computations) {
diff --git a/tensorflow/compiler/xla/service/hlo_module_dce_test.cc b/tensorflow/compiler/xla/service/hlo_module_dce_test.cc
index e535b7d749..f6e2866204 100644
--- a/tensorflow/compiler/xla/service/hlo_module_dce_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_module_dce_test.cc
@@ -38,9 +38,7 @@ class HloModuleDceTest : public HloTestBase {
   // Returns whether the given instruction exists in the given computation.
   bool HasInstruction(const HloComputation& computation,
                       const HloInstruction* instruction) {
-    return std::find(computation.instructions().begin(),
-                     computation.instructions().end(),
-                     instruction) != computation.instructions().end();
+    return absl::c_linear_search(computation.instructions(), instruction);
   }
 
   // Returns whether the while instruction with name 'while_name' in
diff --git a/tensorflow/compiler/xla/service/hlo_parser.cc b/tensorflow/compiler/xla/service/hlo_parser.cc
index 730d39500b..638396308c 100644
--- a/tensorflow/compiler/xla/service/hlo_parser.cc
+++ b/tensorflow/compiler/xla/service/hlo_parser.cc
@@ -2746,7 +2746,7 @@ bool HloParser::ParseConvolutionDimensionNumbers(
   }
 
   auto is_unique = [](string str) -> bool {
-    std::sort(str.begin(), str.end());
+    absl::c_sort(str);
     return std::unique(str.begin(), str.end()) == str.end();
   };
 
diff --git a/tensorflow/compiler/xla/service/hlo_profile_printer.cc b/tensorflow/compiler/xla/service/hlo_profile_printer.cc
index 5eb707a957..9cc202aa9f 100644
--- a/tensorflow/compiler/xla/service/hlo_profile_printer.cc
+++ b/tensorflow/compiler/xla/service/hlo_profile_printer.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/service/hlo_profile_printer.h"
 
+#include "absl/algorithm/container.h"
 #include "absl/strings/str_cat.h"
 #include "tensorflow/compiler/xla/service/human_readable_profile_builder.h"
 
@@ -34,11 +35,10 @@ string PrintHloProfile(const HloProfilePrinterData& hlo_profile_printer_data,
   for (const HloComputationInfo& computation_info :
        hlo_profile_printer_data.computation_infos()) {
     const auto& instruction_infos = computation_info.instruction_infos();
-    bool any_instruction_profiled =
-        std::any_of(instruction_infos.begin(), instruction_infos.end(),
-                    [&](const HloInstructionInfo& instruction_info) {
-                      return counters[instruction_info.profile_index()] != 0;
-                    });
+    bool any_instruction_profiled = absl::c_any_of(
+        instruction_infos, [&](const HloInstructionInfo& instruction_info) {
+          return counters[instruction_info.profile_index()] != 0;
+        });
 
     if (!any_instruction_profiled) {
       continue;
diff --git a/tensorflow/compiler/xla/service/hlo_reachability.cc b/tensorflow/compiler/xla/service/hlo_reachability.cc
index edaa4c59e2..0fced7f15b 100644
--- a/tensorflow/compiler/xla/service/hlo_reachability.cc
+++ b/tensorflow/compiler/xla/service/hlo_reachability.cc
@@ -49,7 +49,7 @@ void HloReachabilityMap::SetReachabilityToUnionHelper(
     absl::Span<const HloInstruction* const> inputs,
     const HloInstruction* instruction, BitVector* bit_vector) {
   // If instruction is part of inputs, don't reset the bit_vector.
-  if (std::find(inputs.begin(), inputs.end(), instruction) == inputs.end()) {
+  if (!absl::c_linear_search(inputs, instruction)) {
     bit_vector->SetToZero();
   }
   bit_vector->Set(GetIndex(instruction));
diff --git a/tensorflow/compiler/xla/service/hlo_rematerialization.cc b/tensorflow/compiler/xla/service/hlo_rematerialization.cc
index ac74e2432f..9ca14ca18a 100644
--- a/tensorflow/compiler/xla/service/hlo_rematerialization.cc
+++ b/tensorflow/compiler/xla/service/hlo_rematerialization.cc
@@ -235,8 +235,7 @@ class InstructionList {
     }
 
     // Now scan forwards until we find one of the before_instructions.
-    while (std::find(before_instructions.begin(), before_instructions.end(),
-                     min_position_item) == before_instructions.end()) {
+    while (!absl::c_linear_search(before_instructions, min_position_item)) {
       min_position_item = min_position_item->next;
     }
     return InsertBefore(to_insert, min_position_item);
@@ -302,7 +301,7 @@ ItemList GetUsers(const InstructionList& instruction_list,
       // A buffer may be used by the instruction via more than one alias. For
       // example, a buffer which appears in more than one element of a tuple.
       Item* user_item = instruction_list.GetItem(user);
-      if (std::find(users.begin(), users.end(), user_item) == users.end()) {
+      if (!absl::c_linear_search(users, user_item)) {
         users.push_back(user_item);
       }
     }
@@ -456,8 +455,7 @@ class MemoryUsageTracker {
       return false;
     }
     const BufferIdList& in_progress_uses = in_progress_item_->buffers_used;
-    return std::find(in_progress_uses.begin(), in_progress_uses.end(),
-                     buffer_id) != in_progress_uses.end();
+    return absl::c_linear_search(in_progress_uses, buffer_id);
   }
 
   // Returns whether the given instruction is live at the current program
@@ -535,8 +533,7 @@ MemoryUsageTracker::MemoryUsageTracker(
         bool unused;
         for (Item* user_item : GetUsers(instruction_list_, logical_buffer,
                                         points_to_analysis, &unused)) {
-          if (std::find(buffer->users.begin(), buffer->users.end(),
-                        user_item) == buffer->users.end()) {
+          if (!absl::c_linear_search(buffer->users, user_item)) {
             buffer->users.push_back(user_item);
             buffer->unfinished_user_count++;
             user_item->buffers_used.push_back(buffer->id);
@@ -784,8 +781,7 @@ bool MemoryUsageTracker::Check() const {
 
     for (const Buffer& buffer : buffers_) {
       if (buffer.defining_instruction->instruction == instruction) {
-        CHECK(std::find(defined_buffers.begin(), defined_buffers.end(),
-                        buffer.id) != defined_buffers.end())
+        CHECK(absl::c_linear_search(defined_buffers, buffer.id))
             << "Instruction " << instruction->name()
             << " defined buffers is missing: " << buffer.ToString();
       }
@@ -808,8 +804,7 @@ bool MemoryUsageTracker::Check() const {
     int64 unfinished_uses = 0;
     for (Item* user : buffer.users) {
       const BufferIdList& used_buffers = user->buffers_used;
-      CHECK(std::find(used_buffers.begin(), used_buffers.end(), buffer.id) !=
-            used_buffers.end())
+      CHECK(absl::c_linear_search(used_buffers, buffer.id))
           << "Instruction " << user->instruction->name()
           << " used buffers is missing " << buffer.ToString();
       if (!IsFinished(user)) {
@@ -836,10 +831,10 @@ int64 RematerializationCost(const HloInstruction* instruction,
   // If none of the users of 'instruction' have been placed in the sequence (as
   // tracked by memory_tracker), then rematerialization of 'instruction' is a
   // zero-cost move of 'instruction' in the sequence.
-  if (!std::any_of(instruction->users().begin(), instruction->users().end(),
-                   [&memory_tracker](const HloInstruction* inst) {
-                     return memory_tracker.IsPlaced(inst);
-                   })) {
+  if (!absl::c_any_of(instruction->users(),
+                      [&memory_tracker](const HloInstruction* inst) {
+                        return memory_tracker.IsPlaced(inst);
+                      })) {
     return 0;
   }
 
diff --git a/tensorflow/compiler/xla/service/hlo_sharding.cc b/tensorflow/compiler/xla/service/hlo_sharding.cc
index 6c9c32f179..37cc146bd7 100644
--- a/tensorflow/compiler/xla/service/hlo_sharding.cc
+++ b/tensorflow/compiler/xla/service/hlo_sharding.cc
@@ -107,13 +107,12 @@ string HloSharding::ToString() const {
 
 bool HloSharding::UsesDevice(int64 device) const {
   if (IsTuple()) {
-    return std::any_of(
-        tuple_elements_.begin(), tuple_elements_.end(),
-        [&](const HloSharding& s) { return s.UsesDevice(device); });
+    return absl::c_any_of(tuple_elements_, [&](const HloSharding& s) {
+      return s.UsesDevice(device);
+    });
   }
   const auto& devices = tile_assignment_;
-  return replicated_ ||
-         std::find(devices.begin(), devices.end(), device) != devices.end();
+  return replicated_ || absl::c_linear_search(devices, device);
 }
 
 std::map<int64, int64> HloSharding::UsedDevices(int64* count) const {
diff --git a/tensorflow/compiler/xla/service/hlo_sharding.h b/tensorflow/compiler/xla/service/hlo_sharding.h
index 9775505f86..5789ae0998 100644
--- a/tensorflow/compiler/xla/service/hlo_sharding.h
+++ b/tensorflow/compiler/xla/service/hlo_sharding.h
@@ -101,8 +101,8 @@ class HloSharding {
     if (!IsTuple()) {
       return replicated_;
     }
-    return std::all_of(tuple_elements_.begin(), tuple_elements_.end(),
-                       [](const HloSharding& s) { return s.IsReplicated(); });
+    return absl::c_all_of(
+        tuple_elements_, [](const HloSharding& s) { return s.IsReplicated(); });
   }
 
   // Returns true if the tile size is the same as the input size.
@@ -110,8 +110,9 @@ class HloSharding {
     if (!IsTuple()) {
       return maximal_;
     }
-    return std::all_of(tuple_elements_.begin(), tuple_elements_.end(),
-                       [](const HloSharding& s) { return s.IsTileMaximal(); });
+    return absl::c_all_of(tuple_elements_, [](const HloSharding& s) {
+      return s.IsTileMaximal();
+    });
   }
 
   // Returns true if the sharding defines an operation on the given device.
diff --git a/tensorflow/compiler/xla/service/hlo_tfgraph_builder.cc b/tensorflow/compiler/xla/service/hlo_tfgraph_builder.cc
index 4ea39a7628..c1f69db74e 100644
--- a/tensorflow/compiler/xla/service/hlo_tfgraph_builder.cc
+++ b/tensorflow/compiler/xla/service/hlo_tfgraph_builder.cc
@@ -61,8 +61,7 @@ void CleanNodeName(string* name) {
   name->erase(std::remove(name->begin(), name->end(), '%'), name->end());
   const string chars_to_replace = "<>[]";
   auto pred = [&](char c) {
-    return std::find(chars_to_replace.begin(), chars_to_replace.end(), c) !=
-           chars_to_replace.end();
+    return absl::c_linear_search(chars_to_replace, c);
   };
   std::replace_if(name->begin(), name->end(), pred, '_');
 }
diff --git a/tensorflow/compiler/xla/service/hlo_value.cc b/tensorflow/compiler/xla/service/hlo_value.cc
index d409df06be..218b33b2ac 100644
--- a/tensorflow/compiler/xla/service/hlo_value.cc
+++ b/tensorflow/compiler/xla/service/hlo_value.cc
@@ -209,7 +209,7 @@ std::ostream& operator<<(std::ostream& out, const HloValue& value) {
 }
 
 void HloValueSet::SortAndUniquifyValues() {
-  std::sort(values_.begin(), values_.end(), HloValue::IdLessThan);
+  absl::c_sort(values_, HloValue::IdLessThan);
   values_.erase(std::unique(values_.begin(), values_.end(), HloValue::IdEqual),
                 values_.end());
 }
diff --git a/tensorflow/compiler/xla/service/human_readable_profile_builder.cc b/tensorflow/compiler/xla/service/human_readable_profile_builder.cc
index 90904ac001..88fc62bd1e 100644
--- a/tensorflow/compiler/xla/service/human_readable_profile_builder.cc
+++ b/tensorflow/compiler/xla/service/human_readable_profile_builder.cc
@@ -128,9 +128,9 @@ string HumanReadableProfileBuilder::ToString() const {
 
   // Sort ops in decreasing order of cycles, and print them.
   std::vector<OpInfo> sorted_ops(op_infos_);
-  std::sort(
-      sorted_ops.begin(), sorted_ops.end(),
-      [](const OpInfo& a, const OpInfo& b) { return a.cycles > b.cycles; });
+  absl::c_sort(sorted_ops, [](const OpInfo& a, const OpInfo& b) {
+    return a.cycles > b.cycles;
+  });
   for (const auto& op : sorted_ops) {
     print_op(op);
   }
diff --git a/tensorflow/compiler/xla/service/instruction_fusion.cc b/tensorflow/compiler/xla/service/instruction_fusion.cc
index f6f25c4413..b97060535d 100644
--- a/tensorflow/compiler/xla/service/instruction_fusion.cc
+++ b/tensorflow/compiler/xla/service/instruction_fusion.cc
@@ -178,19 +178,18 @@ bool InstructionFusion::EffectivelyAtMostUnary(HloInstruction* hlo) {
           output_rank = std::max(output_rank, ShapeUtil::TrueRank(subshape));
         }
       });
-  return std::count_if(hlo->operands().begin(), hlo->operands().end(),
-                       [output_rank](HloInstruction* operand) {
-                         if (operand->opcode() == HloOpcode::kBroadcast ||
-                             operand->opcode() == HloOpcode::kIota) {
-                           return false;
-                         }
-                         if (operand->opcode() == HloOpcode::kConstant &&
-                             ShapeUtil::IsEffectiveScalar(operand->shape())) {
-                           return false;
-                         }
-                         return ShapeUtil::TrueRank(operand->shape()) >=
-                                output_rank;
-                       }) <= 1;
+  return absl::c_count_if(
+             hlo->operands(), [output_rank](HloInstruction* operand) {
+               if (operand->opcode() == HloOpcode::kBroadcast ||
+                   operand->opcode() == HloOpcode::kIota) {
+                 return false;
+               }
+               if (operand->opcode() == HloOpcode::kConstant &&
+                   ShapeUtil::IsEffectiveScalar(operand->shape())) {
+                 return false;
+               }
+               return ShapeUtil::TrueRank(operand->shape()) >= output_rank;
+             }) <= 1;
 }
 
 bool InstructionFusion::CanFuseOnAllPaths(
@@ -409,9 +408,8 @@ class ReversePostOrderFusionQueue : public FusionQueue {
       }
       sorted_operand_numbers.push_back(i);
     }
-    std::sort(
-        sorted_operand_numbers.begin(), sorted_operand_numbers.end(),
-        [&](int64 i, int64 j) {
+    absl::c_sort(
+        sorted_operand_numbers, [&](int64 i, int64 j) {
           // Instructions with higher priority in the queue come first.
           return (
               FindOrDie(post_order_index_, instruction->mutable_operand(i)) >
diff --git a/tensorflow/compiler/xla/service/layout_assignment.cc b/tensorflow/compiler/xla/service/layout_assignment.cc
index 406cd0a219..10ff7bb6d4 100644
--- a/tensorflow/compiler/xla/service/layout_assignment.cc
+++ b/tensorflow/compiler/xla/service/layout_assignment.cc
@@ -147,12 +147,9 @@ bool LayoutConstraints::OperandBufferForwarded(
   PointsToSet::BufferSet* output_buffers = GetBufferSet(instruction);
   PointsToSet::BufferSet* operand_buffers =
       GetBufferSet(instruction->operand(operand_no));
-  for (const LogicalBuffer* output_buffer : *output_buffers) {
-    if (operand_buffers->count(output_buffer) > 0) {
-      return true;
-    }
-  }
-  return false;
+  return absl::c_any_of(*output_buffers, [&](const LogicalBuffer* b) {
+    return operand_buffers->count(b) > 0;
+  });
 }
 
 Status LayoutConstraints::SetBufferLayout(const Layout& layout,
diff --git a/tensorflow/compiler/xla/service/llvm_ir/alias_analysis.cc b/tensorflow/compiler/xla/service/llvm_ir/alias_analysis.cc
index 643ecd0fba..ce3d922ca7 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/alias_analysis.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/alias_analysis.cc
@@ -81,9 +81,7 @@ void AliasAnalysis::AddAliasingInformationToIrArray(const HloInstruction& hlo,
     if (hlo.opcode() == HloOpcode::kParameter) {
       const std::vector<HloInstruction*>& parameter_instructions =
           module_.entry_computation()->parameter_instructions();
-      if (std::find(parameter_instructions.begin(),
-                    parameter_instructions.end(),
-                    &hlo) != parameter_instructions.end()) {
+      if (absl::c_linear_search(parameter_instructions, &hlo)) {
         array->MarkInvariantOverWholeProgram(context_);
       }
     }
diff --git a/tensorflow/compiler/xla/service/name_uniquer.cc b/tensorflow/compiler/xla/service/name_uniquer.cc
index daa718879d..f6feed2993 100644
--- a/tensorflow/compiler/xla/service/name_uniquer.cc
+++ b/tensorflow/compiler/xla/service/name_uniquer.cc
@@ -34,7 +34,7 @@ bool IsAllowed(char character) {
 }  // namespace
 
 NameUniquer::NameUniquer(const string& separator) {
-  CHECK(std::all_of(separator.begin(), separator.end(), IsAllowed))
+  CHECK(absl::c_all_of(separator, IsAllowed))
       << "separator should comprises allowed characters only";
   separator_ = separator;
 }
diff --git a/tensorflow/compiler/xla/service/platform_util.cc b/tensorflow/compiler/xla/service/platform_util.cc
index 896b73cda4..0491f641fc 100644
--- a/tensorflow/compiler/xla/service/platform_util.cc
+++ b/tensorflow/compiler/xla/service/platform_util.cc
@@ -260,8 +260,8 @@ PlatformUtil::GetStreamExecutors(
     // Block here in thread_pool destructor until all devices are initialized.
   }
   VLOG(1) << "Device initialization complete";
-  if (std::all_of(stream_executors.begin(), stream_executors.end(),
-                  [](se::StreamExecutor* s) { return s == nullptr; })) {
+  if (absl::c_all_of(stream_executors,
+                     [](se::StreamExecutor* s) { return s == nullptr; })) {
     return InternalError("no supported devices found for platform %s",
                          platform->Name());
   }
diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc
index 4a2dd09742..d711c8ef0d 100644
--- a/tensorflow/compiler/xla/service/shape_inference.cc
+++ b/tensorflow/compiler/xla/service/shape_inference.cc
@@ -534,9 +534,8 @@ Status ValidateDotDimensionNumbers(
                           absl::Span<const int64> contracting_dims,
                           absl::Span<const int64> batch_dims) -> bool {
     auto in_range = [&rank](int64 i) -> bool { return 0 <= i && i < rank; };
-    return std::all_of(contracting_dims.begin(), contracting_dims.end(),
-                       in_range) &&
-           std::all_of(batch_dims.begin(), batch_dims.end(), in_range);
+    return absl::c_all_of(contracting_dims, in_range) &&
+           absl::c_all_of(batch_dims, in_range);
   };
 
   absl::Span<const int64> lhs_contracting_dimensions =
@@ -563,9 +562,8 @@ Status ValidateDotDimensionNumbers(
     auto is_unique = [&dim_set](int64 i) -> bool {
       return dim_set.insert(i).second;
     };
-    return std::all_of(contracting_dims.begin(), contracting_dims.end(),
-                       is_unique) &&
-           std::all_of(batch_dims.begin(), batch_dims.end(), is_unique);
+    return absl::c_all_of(contracting_dims, is_unique) &&
+           absl::c_all_of(batch_dims, is_unique);
   };
 
   if (!dims_unique(lhs_contracting_dimensions, lhs_batch_dimensions) ||
@@ -1589,29 +1587,29 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
   input_dnums[1] = dnums.input_feature_dimension();
   std::copy(dnums.input_spatial_dimensions().begin(),
             dnums.input_spatial_dimensions().end(), input_dnums.begin() + 2);
-  std::sort(input_dnums.begin(), input_dnums.end());
+  absl::c_sort(input_dnums);
 
   std::vector<int64> window_dnums(num_dims);
   window_dnums[0] = dnums.kernel_input_feature_dimension();
   window_dnums[1] = dnums.kernel_output_feature_dimension();
   std::copy(dnums.kernel_spatial_dimensions().begin(),
             dnums.kernel_spatial_dimensions().end(), window_dnums.begin() + 2);
-  std::sort(window_dnums.begin(), window_dnums.end());
+  absl::c_sort(window_dnums);
 
   std::vector<int64> output_dnums(num_dims);
   output_dnums[0] = dnums.output_batch_dimension();
   output_dnums[1] = dnums.output_feature_dimension();
   std::copy(dnums.output_spatial_dimensions().begin(),
             dnums.output_spatial_dimensions().end(), output_dnums.begin() + 2);
-  std::sort(output_dnums.begin(), output_dnums.end());
+  absl::c_sort(output_dnums);
 
   std::vector<int64> expected_dnums(num_dims);
   std::iota(expected_dnums.begin(), expected_dnums.end(), 0);
 
   const auto in_range = [num_dims](int64 i) { return 0 <= i && i < num_dims; };
-  if (!std::all_of(input_dnums.begin(), input_dnums.end(), in_range) ||
-      !std::all_of(window_dnums.begin(), window_dnums.end(), in_range) ||
-      !std::all_of(output_dnums.begin(), output_dnums.end(), in_range)) {
+  if (!absl::c_all_of(input_dnums, in_range) ||
+      !absl::c_all_of(window_dnums, in_range) ||
+      !absl::c_all_of(output_dnums, in_range)) {
     return InvalidArgument(
         "A dimension number is out of range in convolution: %s.",
         dnums.DebugString());
diff --git a/tensorflow/compiler/xla/service/transpose_folding.cc b/tensorflow/compiler/xla/service/transpose_folding.cc
index 15eb46bac0..a95ca2bf2a 100644
--- a/tensorflow/compiler/xla/service/transpose_folding.cc
+++ b/tensorflow/compiler/xla/service/transpose_folding.cc
@@ -130,8 +130,7 @@ bool FoldTransposeIntoConvolution(InstructionOperandsPair pair) {
 
   HloInstruction* new_lhs;
   const int64 kLhsIdx = 0;
-  if (std::find(operand_indices.begin(), operand_indices.end(), kLhsIdx) !=
-      operand_indices.end()) {
+  if (absl::c_linear_search(operand_indices, kLhsIdx)) {
     HloInstruction& transpose = *convolution.mutable_operand(kLhsIdx);
     const auto& transpose_dimensions = transpose.dimensions();
     HloInstruction& transpose_operand = *transpose.mutable_operand(0);
@@ -154,8 +153,7 @@ bool FoldTransposeIntoConvolution(InstructionOperandsPair pair) {
 
   HloInstruction* new_rhs;
   const int64 kRhsIdx = 1;
-  if (std::find(operand_indices.begin(), operand_indices.end(), kRhsIdx) !=
-      operand_indices.end()) {
+  if (absl::c_linear_search(operand_indices, kRhsIdx)) {
     HloInstruction& transpose = *convolution.mutable_operand(kRhsIdx);
     const auto& transpose_dimensions = transpose.dimensions();
     HloInstruction& transpose_operand = *transpose.mutable_operand(0);
diff --git a/tensorflow/compiler/xla/service/tuple_points_to_analysis.cc b/tensorflow/compiler/xla/service/tuple_points_to_analysis.cc
index dec719b04a..5e505aaf02 100644
--- a/tensorflow/compiler/xla/service/tuple_points_to_analysis.cc
+++ b/tensorflow/compiler/xla/service/tuple_points_to_analysis.cc
@@ -87,9 +87,7 @@ bool PointsToSet::ContainsBuffer(const LogicalBuffer& buffer) const {
   bool found = false;
   ForEachElement([&found, &buffer](const ShapeIndex& /*index*/,
                                    const BufferList& pointed_to_buffers) {
-    if (!found &&
-        std::find(pointed_to_buffers.begin(), pointed_to_buffers.end(),
-                  &buffer) != pointed_to_buffers.end()) {
+    if (!found && absl::c_linear_search(pointed_to_buffers, &buffer)) {
       found = true;
     }
   });
@@ -99,8 +97,7 @@ bool PointsToSet::ContainsBuffer(const LogicalBuffer& buffer) const {
 bool PointsToSet::ContainsBufferAtIndex(const LogicalBuffer& buffer,
                                         const ShapeIndex& index) const {
   const auto& pointed_to_buffers = element(index);
-  return std::find(pointed_to_buffers.begin(), pointed_to_buffers.end(),
-                   &buffer) != pointed_to_buffers.end();
+  return absl::c_linear_search(pointed_to_buffers, &buffer);
 }
 
 void PointsToSet::AddPointedToBuffer(const LogicalBuffer& buffer,
@@ -604,9 +601,8 @@ bool TuplePointsToAnalysis::DoesNotUseOperandBuffer(
   } else if (user->opcode() == HloOpcode::kFusion &&
              user->fusion_kind() == HloInstruction::FusionKind::kLoop) {
     // Find fusion parameter associated with 'operand'.
-    auto it = std::find_if(
-        user->fused_parameters().begin(), user->fused_parameters().end(),
-        [=](HloInstruction* fused_param) {
+    auto it = absl::c_find_if(
+        user->fused_parameters(), [&](HloInstruction* fused_param) {
           return user->operand(fused_param->parameter_number()) == operand;
         });
     CHECK(it != user->fused_parameters().end());
@@ -672,9 +668,8 @@ bool TuplePointsToAnalysis::HasUniqueFusedUseOfOperandAt(
   }
   // Find fusion parameter associated with 'operand'.
   const auto& fused_params = fusion->fused_parameters();
-  auto fused_param_it = std::find_if(
-      fused_params.begin(), fused_params.end(),
-      [&](HloInstruction* fused_param) {
+  auto fused_param_it =
+      absl::c_find_if(fused_params, [&](HloInstruction* fused_param) {
         return fusion->operand(fused_param->parameter_number()) == operand;
       });
   if (fused_param_it == fused_params.end()) {
@@ -743,11 +738,10 @@ bool TuplePointsToAnalysis::CanShareOperandBufferWithUser(
       // Check if one operand of kAdd fused root is kDot or kConvolution.
       auto* add = user->fused_expression_root();
       auto add_operand_it =
-          std::find_if(add->operands().begin(), add->operands().end(),
-                       [&](HloInstruction* operand) {
-                         return operand->opcode() == HloOpcode::kConvolution ||
-                                operand->opcode() == HloOpcode::kDot;
-                       });
+          absl::c_find_if(add->operands(), [&](HloInstruction* operand) {
+            return operand->opcode() == HloOpcode::kConvolution ||
+                   operand->opcode() == HloOpcode::kDot;
+          });
       if (add_operand_it == add->operands().end()) {
         return false;
       }
diff --git a/tensorflow/compiler/xla/service/tuple_points_to_analysis_test.cc b/tensorflow/compiler/xla/service/tuple_points_to_analysis_test.cc
index d8875ca747..7599e1e6ad 100644
--- a/tensorflow/compiler/xla/service/tuple_points_to_analysis_test.cc
+++ b/tensorflow/compiler/xla/service/tuple_points_to_analysis_test.cc
@@ -721,9 +721,8 @@ class FusionPointsToAnalysisTest : public TuplePointsToAnalysisTest {
   // to fusion 'operand'.
   HloInstruction* GetFusionParameterForOperand(HloInstruction* fusion,
                                                HloInstruction* operand) {
-    auto it = std::find_if(
-        fusion->fused_instructions().begin(),
-        fusion->fused_instructions().end(), [=](const HloInstruction* fused) {
+    auto it = absl::c_find_if(
+        fusion->fused_instructions(), [&](const HloInstruction* fused) {
           return fused->opcode() == HloOpcode::kParameter &&
                  fusion->operand(fused->parameter_number()) == operand;
         });
diff --git a/tensorflow/compiler/xla/service/while_loop_simplifier.cc b/tensorflow/compiler/xla/service/while_loop_simplifier.cc
index de884b037d..09d5409571 100644
--- a/tensorflow/compiler/xla/service/while_loop_simplifier.cc
+++ b/tensorflow/compiler/xla/service/while_loop_simplifier.cc
@@ -109,8 +109,7 @@ static StatusOr<bool> TryRemoveDeadWhileParams(HloInstruction* while_op) {
       // operand appears in, but it may appear more than once!
       if (user->user_count() == 1 && user->users().front() == while_body_root &&
           while_body_root->operand_index(user) == user->tuple_index() &&
-          std::count(while_body_root->operands().begin(),
-                     while_body_root->operands().end(), user) == 1) {
+          absl::c_count(while_body_root->operands(), user) == 1) {
         continue;
       }
 
@@ -158,7 +157,7 @@ static StatusOr<bool> TryRemoveDeadWhileParams(HloInstruction* while_op) {
   // Build up maps from the old/new to the new/old tuple indices.
   std::vector<int64> new_to_old_tuple_idx(used_tuple_indices.begin(),
                                           used_tuple_indices.end());
-  std::sort(new_to_old_tuple_idx.begin(), new_to_old_tuple_idx.end());
+  absl::c_sort(new_to_old_tuple_idx);
 
   absl::flat_hash_map<int64, int64> old_to_new_tuple_idx;
   for (int64 new_idx = 0; new_idx < new_to_old_tuple_idx.size(); ++new_idx) {
diff --git a/tensorflow/compiler/xla/service/while_loop_simplifier_test.cc b/tensorflow/compiler/xla/service/while_loop_simplifier_test.cc
index 3713989ca2..ecca76b1e8 100644
--- a/tensorflow/compiler/xla/service/while_loop_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/while_loop_simplifier_test.cc
@@ -407,13 +407,12 @@ TEST_F(WhileLoopSimplifierTest, RemoveUnusedLoopOperands) {
   // The original while instruction is still left in the module as a dead
   // instruction, find a while instruction with a different name as the new
   // while instruction.
+  const auto& instrs = m->entry_computation()->instructions();
   HloInstruction* new_while_op =
-      *std::find_if(m->entry_computation()->instructions().begin(),
-                    m->entry_computation()->instructions().end(),
-                    [&](const HloInstruction* instr) {
-                      return (instr->opcode() == HloOpcode::kWhile &&
-                              instr->name() != "while");
-                    });
+      *absl::c_find_if(instrs, [&](const HloInstruction* instr) {
+        return (instr->opcode() == HloOpcode::kWhile &&
+                instr->name() != "while");
+      });
 
   auto scalar_s32 = ShapeUtil::MakeShape(S32, {});
   EXPECT_TRUE(
diff --git a/tensorflow/compiler/xla/shape.cc b/tensorflow/compiler/xla/shape.cc
index f05d816268..ea4f722ef4 100644
--- a/tensorflow/compiler/xla/shape.cc
+++ b/tensorflow/compiler/xla/shape.cc
@@ -87,8 +87,7 @@ bool Shape::is_static() const {
       }
     }
   }
-  return !std::any_of(dynamic_dimensions_.begin(), dynamic_dimensions_.end(),
-                      [](bool b) { return b; });
+  return !absl::c_any_of(dynamic_dimensions_, [](bool b) { return b; });
 }
 
 void Shape::DeleteDimension(int64 dim_to_delete) {
diff --git a/tensorflow/compiler/xla/shape_util.cc b/tensorflow/compiler/xla/shape_util.cc
index 7f68c0ae1f..4726ee6a5b 100644
--- a/tensorflow/compiler/xla/shape_util.cc
+++ b/tensorflow/compiler/xla/shape_util.cc
@@ -405,8 +405,7 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
 }
 
 /* static */ bool ShapeUtil::IsNestedTuple(const Shape& shape) {
-  return IsTuple(shape) && std::any_of(shape.tuple_shapes().begin(),
-                                       shape.tuple_shapes().end(), IsTuple);
+  return IsTuple(shape) && absl::c_any_of(shape.tuple_shapes(), IsTuple);
 }
 
 /* static */ bool ShapeUtil::IsEmptyTuple(const Shape& shape) {
diff --git a/tensorflow/compiler/xla/sparse_index_array.h b/tensorflow/compiler/xla/sparse_index_array.h
index a96d483462..0c25355467 100644
--- a/tensorflow/compiler/xla/sparse_index_array.h
+++ b/tensorflow/compiler/xla/sparse_index_array.h
@@ -135,7 +135,7 @@ void SparseIndexArray::SortWithValues(absl::Span<NativeT> values) {
   auto sort_order_less = [this](int64 lhs, int64 rhs) {
     return IndexUtil::CompareIndices(At(lhs), At(rhs)) < 0;
   };
-  std::sort(sort_order.begin(), sort_order.end(), sort_order_less);
+  absl::c_sort(sort_order, sort_order_less);
 
   // Reorder the array elements according to sort_order.  Work through the array
   // and follow cycles so we can do the reorder in-place.
diff --git a/tensorflow/compiler/xla/tests/convolution_test.cc b/tensorflow/compiler/xla/tests/convolution_test.cc
index 2496938912..9db9f2563b 100644
--- a/tensorflow/compiler/xla/tests/convolution_test.cc
+++ b/tensorflow/compiler/xla/tests/convolution_test.cc
@@ -467,8 +467,8 @@ XLA_TEST_F(ConvolutionTest, Convolve3D_1x4x2x3x3_2x2x2x3x3_Valid) {
 // servers. The error message is missing the operator ++.
 template <typename T>
 void iota_int_init_value(std::vector<T>& values, int init_value) {
-  std::for_each(values.begin(), values.end(),
-                [&](T& value) { value = static_cast<T>(init_value++); });
+  absl::c_for_each(values,
+                   [&](T& value) { value = static_cast<T>(init_value++); });
 }
 
 template <typename T>
diff --git a/tensorflow/compiler/xla/util.cc b/tensorflow/compiler/xla/util.cc
index 68cab7387c..34b73b5206 100644
--- a/tensorflow/compiler/xla/util.cc
+++ b/tensorflow/compiler/xla/util.cc
@@ -86,7 +86,7 @@ bool IsPermutation(absl::Span<const int64> permutation, int64 rank) {
     CHECK_LT(index, rank);
     output[index] = 0;
   }
-  return std::find(output.begin(), output.end(), -1) == output.end();
+  return !absl::c_linear_search(output, -1);
 }
 
 std::vector<int64> InversePermutation(
diff --git a/tensorflow/compiler/xla/util.h b/tensorflow/compiler/xla/util.h
index 6722641e9d..f2fd17dc99 100644
--- a/tensorflow/compiler/xla/util.h
+++ b/tensorflow/compiler/xla/util.h
@@ -324,8 +324,7 @@ bool IsIdentityPermutation(absl::Span<const int64> permutation);
 
 template <typename Container>
 int64 PositionInContainer(const Container& container, int64 value) {
-  return std::distance(container.begin(),
-                       std::find(container.begin(), container.end(), value));
+  return std::distance(container.begin(), absl::c_find(container, value));
 }
 
 // Formats the container as a comma-separated string. StrAppend must support
diff --git a/tensorflow/compiler/xla/window_util.cc b/tensorflow/compiler/xla/window_util.cc
index 51c73b3d17..e001cc35f9 100644
--- a/tensorflow/compiler/xla/window_util.cc
+++ b/tensorflow/compiler/xla/window_util.cc
@@ -17,6 +17,7 @@ limitations under the License.
 
 #include <vector>
 
+#include "absl/algorithm/container.h"
 #include "absl/strings/str_cat.h"
 #include "tensorflow/compiler/xla/types.h"
 #include "tensorflow/compiler/xla/xla_data.pb.h"
@@ -137,25 +138,23 @@ bool HasPadding(const Window& window) {
 }
 
 bool HasSymmetricPadding(const Window& window) {
-  return std::all_of(window.dimensions().begin(), window.dimensions().end(),
-                     [](const WindowDimension& dim) {
-                       return dim.padding_low() == dim.padding_high();
-                     });
+  return absl::c_all_of(window.dimensions(), [](const WindowDimension& dim) {
+    return dim.padding_low() == dim.padding_high();
+  });
 }
 
 bool HasSymmetricPadding(const PaddingConfig& padding_config) {
-  return std::all_of(padding_config.dimensions().begin(),
-                     padding_config.dimensions().end(),
-                     [](const PaddingConfig::PaddingConfigDimension& dim) {
-                       return dim.edge_padding_low() == dim.edge_padding_high();
-                     });
+  return absl::c_all_of(padding_config.dimensions(),
+                        [](const PaddingConfig::PaddingConfigDimension& dim) {
+                          return dim.edge_padding_low() ==
+                                 dim.edge_padding_high();
+                        });
 }
 
 bool HasNegativePadding(const Window& window) {
-  return std::any_of(window.dimensions().begin(), window.dimensions().end(),
-                     [](const WindowDimension& dim) {
-                       return dim.padding_low() < 0 || dim.padding_high() < 0;
-                     });
+  return absl::c_any_of(window.dimensions(), [](const WindowDimension& dim) {
+    return dim.padding_low() < 0 || dim.padding_high() < 0;
+  });
 }
 
 bool HasBaseDilation(const Window& window) {
@@ -190,10 +189,9 @@ bool AllOrNoneReversed(const Window& window) {
     return true;
   }
   bool reversed = window.dimensions()[0].window_reversal();
-  return std::all_of(window.dimensions().begin(), window.dimensions().end(),
-                     [&](const WindowDimension& dim) {
-                       return dim.window_reversal() == reversed;
-                     });
+  return absl::c_all_of(window.dimensions(), [&](const WindowDimension& dim) {
+    return dim.window_reversal() == reversed;
+  });
 }
 
 bool HasDilation(const Window& window) {
-- 
GitLab


From e2071f6a0502a5bc81abd0cb533f3b52ba0e7855 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Fri, 4 Jan 2019 12:59:45 -0800
Subject: [PATCH 0246/1095] Add poisson v2 loss and metric.

PiperOrigin-RevId: 227900433
---
 tensorflow/python/keras/losses.py       | 47 +++++++++-----
 tensorflow/python/keras/losses_test.py  | 81 +++++++++++++++++++++++++
 tensorflow/python/keras/metrics.py      | 31 ++++++++++
 tensorflow/python/keras/metrics_test.py | 46 ++++++++++++++
 4 files changed, 189 insertions(+), 16 deletions(-)

diff --git a/tensorflow/python/keras/losses.py b/tensorflow/python/keras/losses.py
index 1ff564054b..46e729c197 100644
--- a/tensorflow/python/keras/losses.py
+++ b/tensorflow/python/keras/losses.py
@@ -519,25 +519,39 @@ class LogLoss(Loss):
   model = keras.models.Model(inputs, outputs)
   model.compile('sgd', loss=tf.losses.LogLoss())
   ```
-
-  Args:
-    epsilon: A small increment to add to avoid taking a log of zero.
-    reduction: Type of `tf.losses.Reduction` to apply to loss. Default value is
-      `SUM_OVER_BATCH_SIZE`.
-    name: Optional name for the op.
   """
 
-  def __init__(self,
-               epsilon=1e-7,
-               reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
-               name=None):
-    super(LogLoss, self).__init__(reduction=reduction, name=name)
-    self.epsilon = epsilon
+  def call(self, y_true, y_pred):
+    y_pred = ops.convert_to_tensor(y_pred)
+    y_true = math_ops.cast(y_true, y_pred.dtype)
+    return logloss(y_true, y_pred)
+
+
+class Poisson(Loss):
+  """Computes the poisson loss between `y_true` and `y_pred`.
+
+  loss = y_pred - y_true * log(y_pred)
+
+  Usage:
+
+  ```python
+  p = tf.losses.Poisson()
+  loss = p([1, 9, 2], [4, 8, 12])
+  print('Loss: ', loss.numpy())  # Loss: -4.63
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile('sgd', loss=tf.losses.Poisson())
+  ```
+  """
 
   def call(self, y_true, y_pred):
     y_pred = ops.convert_to_tensor(y_pred)
     y_true = math_ops.cast(y_true, y_pred.dtype)
-    return logloss(y_true, y_pred, epsilon=self.epsilon)
+    return poisson(y_true, y_pred)
 
 
 class Logcosh(Loss):
@@ -629,9 +643,10 @@ def categorical_hinge(y_true, y_pred):
   return math_ops.maximum(0., neg - pos + 1.)
 
 
-def logloss(y_true, y_pred, epsilon=1e-7):
-  losses = math_ops.multiply(y_true, math_ops.log(y_pred + epsilon))
-  losses += math_ops.multiply((1 - y_true), math_ops.log(1 - y_pred + epsilon))
+def logloss(y_true, y_pred):
+  losses = math_ops.multiply(y_true, math_ops.log(y_pred + K.epsilon()))
+  losses += math_ops.multiply((1 - y_true),
+                              math_ops.log(1 - y_pred + K.epsilon()))
   return K.mean(-losses, axis=-1)
 
 
diff --git a/tensorflow/python/keras/losses_test.py b/tensorflow/python/keras/losses_test.py
index aa00ba5a91..f34a6caa18 100644
--- a/tensorflow/python/keras/losses_test.py
+++ b/tensorflow/python/keras/losses_test.py
@@ -1175,5 +1175,86 @@ class LogcoshTest(test.TestCase):
     self.assertAlmostEqual(self.evaluate(loss), 0., 3)
 
 
+@test_util.run_all_in_graph_and_eager_modes
+class PoissonTest(test.TestCase):
+
+  def setup(self):
+    self.np_y_pred = np.asarray([1, 9, 2, 5, 2, 6]).reshape((2, 3))
+    self.np_y_true = np.asarray([4, 8, 12, 8, 1, 3]).reshape((2, 3))
+
+    self.batch_size = 6
+    self.expected_losses = self.np_y_pred - np.multiply(self.np_y_true,
+                                                        np.log(self.np_y_pred))
+
+    self.y_pred = constant_op.constant(self.np_y_pred, dtype=dtypes.float32)
+    self.y_true = constant_op.constant(self.np_y_true)
+
+  def test_config(self):
+    poisson_obj = keras.losses.Poisson(
+        reduction=losses_impl.ReductionV2.SUM, name='poisson')
+    self.assertEqual(poisson_obj.name, 'poisson')
+    self.assertEqual(poisson_obj.reduction, losses_impl.ReductionV2.SUM)
+
+  def test_unweighted(self):
+    self.setup()
+    poisson_obj = keras.losses.Poisson()
+
+    loss = poisson_obj(self.y_true, self.y_pred)
+    expected_loss = np.sum(self.expected_losses) / self.batch_size
+    self.assertAlmostEqual(self.evaluate(loss), expected_loss, 3)
+
+  def test_scalar_weighted(self):
+    self.setup()
+    poisson_obj = keras.losses.Poisson()
+    sample_weight = 2.3
+    loss = poisson_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+
+    expected_loss = sample_weight * np.sum(
+        self.expected_losses) / self.batch_size
+    self.assertAlmostEqual(self.evaluate(loss), expected_loss, 3)
+    self.assertAlmostEqual(self.evaluate(loss), expected_loss, 3)
+
+    # Verify we get the same output when the same input is given
+    loss_2 = poisson_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+    self.assertAlmostEqual(self.evaluate(loss), self.evaluate(loss_2), 3)
+
+  def test_sample_weighted(self):
+    self.setup()
+    poisson_obj = keras.losses.Poisson()
+
+    sample_weight = constant_op.constant([1.2, 3.4], shape=(2, 1))
+    loss = poisson_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+
+    expected_loss = np.multiply(
+        self.expected_losses,
+        np.asarray([1.2, 1.2, 1.2, 3.4, 3.4, 3.4]).reshape((2, 3)))
+    expected_loss = np.sum(expected_loss) / self.batch_size
+    self.assertAlmostEqual(self.evaluate(loss), expected_loss, 3)
+
+  def test_timestep_weighted(self):
+    self.setup()
+    poisson_obj = keras.losses.Poisson()
+    y_true = self.np_y_true.reshape(2, 3, 1)
+    y_pred = self.np_y_pred.reshape(2, 3, 1)
+    sample_weight = np.asarray([3, 6, 5, 0, 4, 2]).reshape(2, 3, 1)
+    expected_losses = y_pred - np.multiply(y_true, np.log(y_pred))
+
+    y_pred = constant_op.constant(y_pred, dtype=dtypes.float32)
+    y_true = constant_op.constant(y_true)
+
+    loss = poisson_obj(
+        y_true,
+        y_pred,
+        sample_weight=constant_op.constant(sample_weight, shape=(2, 3)))
+    expected_loss = np.sum(expected_losses * sample_weight) / self.batch_size
+    self.assertAlmostEqual(self.evaluate(loss), expected_loss, 3)
+
+  def test_zero_weighted(self):
+    self.setup()
+    poisson_obj = keras.losses.Poisson()
+    loss = poisson_obj(self.y_true, self.y_pred, sample_weight=0)
+    self.assertAlmostEqual(self.evaluate(loss), 0., 3)
+
+
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index 2de0bc474c..71568e3fc5 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -1685,6 +1685,37 @@ class Logcosh(MeanMetricWrapper):
     return super(Logcosh, cls).from_config(config)
 
 
+class Poisson(MeanMetricWrapper):
+  """Computes the poisson metric between `y_true` and `y_pred`.
+
+  metric = y_pred - y_true * log(y_pred)
+
+  Usage:
+
+  ```python
+  m = tf.keras.metrics.Poisson()
+  m.update_state([1, 9, 2], [4, 8, 12])
+  print('Final result: ', m.result().numpy())  # Final result: -4.63
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile('sgd', metrics=[tf.keras.metrics.Poisson()])
+  ```
+  """
+
+  def __init__(self, name='poisson', dtype=None):
+    super(Poisson, self).__init__(poisson, name, dtype=dtype)
+
+  @classmethod
+  def from_config(cls, config):
+    if 'fn' in config:
+      config.pop('fn')
+    return super(Poisson, cls).from_config(config)
+
+
 def accuracy(y_true, y_pred):
   y_pred.get_shape().assert_is_compatible_with(y_true.get_shape())
   if y_true.dtype != y_pred.dtype:
diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py
index a9d88789ed..88b763d569 100644
--- a/tensorflow/python/keras/metrics_test.py
+++ b/tensorflow/python/keras/metrics_test.py
@@ -1510,6 +1510,52 @@ class LogcoshTest(test.TestCase):
     self.assertAllClose(self.evaluate(result), expected_result, atol=1e-3)
 
 
+@test_util.run_all_in_graph_and_eager_modes
+class PoissonTest(test.TestCase):
+
+  def setup(self):
+    y_pred = np.asarray([1, 9, 2, 5, 2, 6]).reshape((2, 3))
+    y_true = np.asarray([4, 8, 12, 8, 1, 3]).reshape((2, 3))
+
+    self.batch_size = 6
+    self.expected_results = y_pred - np.multiply(y_true, np.log(y_pred))
+
+    self.y_pred = constant_op.constant(y_pred, dtype=dtypes.float32)
+    self.y_true = constant_op.constant(y_true)
+
+  def test_config(self):
+    poisson_obj = metrics.Poisson(name='poisson', dtype=dtypes.int32)
+    self.assertEqual(poisson_obj.name, 'poisson')
+    self.assertEqual(poisson_obj._dtype, dtypes.int32)
+
+    poisson_obj2 = metrics.Poisson.from_config(poisson_obj.get_config())
+    self.assertEqual(poisson_obj2.name, 'poisson')
+    self.assertEqual(poisson_obj2._dtype, dtypes.int32)
+
+  def test_unweighted(self):
+    self.setup()
+    poisson_obj = metrics.Poisson()
+    self.evaluate(variables.variables_initializer(poisson_obj.variables))
+
+    update_op = poisson_obj.update_state(self.y_true, self.y_pred)
+    self.evaluate(update_op)
+    result = poisson_obj.result()
+    expected_result = np.sum(self.expected_results) / self.batch_size
+    self.assertAllClose(result, expected_result, atol=1e-3)
+
+  def test_weighted(self):
+    self.setup()
+    poisson_obj = metrics.Poisson()
+    self.evaluate(variables.variables_initializer(poisson_obj.variables))
+    sample_weight = constant_op.constant([1.2, 3.4], shape=(2, 1))
+
+    result = poisson_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+    sample_weight = np.asarray([1.2, 1.2, 1.2, 3.4, 3.4, 3.4]).reshape((2, 3))
+    expected_result = np.multiply(self.expected_results, sample_weight)
+    expected_result = np.sum(expected_result) / np.sum(sample_weight)
+    self.assertAllClose(self.evaluate(result), expected_result, atol=1e-3)
+
+
 def _get_model(compile_metrics):
   model_layers = [
       layers.Dense(3, activation='relu', kernel_initializer='ones'),
-- 
GitLab


From d198ec5f99258f443af8e16825ae9c77e26c5db6 Mon Sep 17 00:00:00 2001
From: Yuefeng Zhou <yuefengz@google.com>
Date: Fri, 4 Jan 2019 13:11:46 -0800
Subject: [PATCH 0247/1095] Move ParameterServerStrategy to core.

PiperOrigin-RevId: 227902368
---
 tensorflow/contrib/distribute/python/BUILD    |  15 +-
 .../python/parameter_server_strategy.py       | 467 +---------------
 .../python/parameter_server_strategy_test.py  | 339 +++++++----
 tensorflow/python/distribute/BUILD            |  22 +
 .../distribute/parameter_server_strategy.py   | 528 ++++++++++++++++++
 5 files changed, 813 insertions(+), 558 deletions(-)
 create mode 100644 tensorflow/python/distribute/parameter_server_strategy.py

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index 78ad3214fb..de6b6f1f84 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -80,19 +80,10 @@ py_library(
     srcs = ["parameter_server_strategy.py"],
     visibility = ["//tensorflow:internal"],
     deps = [
-        ":mirrored_strategy",
-        "//tensorflow/core:protos_all_py",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:resource_variable_ops",
-        "//tensorflow/python:training",
-        "//tensorflow/python:util",
-        "//tensorflow/python/distribute:cross_device_ops",
-        "//tensorflow/python/distribute:input_lib",
-        "//tensorflow/python/distribute:multi_worker_util",
-        "//tensorflow/python/distribute:reduce_util",
+        "//tensorflow/python/distribute:distribute_lib",
+        "//tensorflow/python/distribute:parameter_server_strategy",
         "//tensorflow/python/distribute:values",
-        "//tensorflow/python/eager:context",
+        "//tensorflow/python/distribute/cluster_resolver:cluster_resolver_lib",
     ],
 )
 
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy.py b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
index 461e1bca21..bb0b8eb992 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
@@ -18,35 +18,24 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import copy
-
-from tensorflow.contrib.distribute.python import mirrored_strategy
-from tensorflow.python.distribute import cross_device_ops as cross_device_ops_lib
-from tensorflow.python.distribute import device_util
 from tensorflow.python.distribute import distribute_lib
 from tensorflow.python.distribute import input_lib
-from tensorflow.python.distribute import multi_worker_util
-from tensorflow.python.distribute import values
-from tensorflow.python.eager import context
-from tensorflow.python.framework import device as tf_device
-from tensorflow.python.framework import ops
-from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import resource_variable_ops
-from tensorflow.python.ops import variable_scope as vs
-from tensorflow.python.platform import tf_logging as logging
-from tensorflow.python.training import device_setter
-from tensorflow.python.util import nest
+from tensorflow.python.distribute import parameter_server_strategy
+from tensorflow.python.distribute.cluster_resolver import SimpleClusterResolver
+from tensorflow.python.distribute.cluster_resolver import TFConfigClusterResolver
+
+# pylint: disable=protected-access,invalid-name,line-too-long
+CoreParameterServerStrategy = parameter_server_strategy.ParameterServerStrategy
+CoreParameterServerExtended = parameter_server_strategy.ParameterServerStrategyExtended
 
-_LOCAL_CPU = "/device:CPU:0"
-_LOCAL_GPU_0 = "/device:GPU:0"
+# pylint: enable=protected-access,invalid-name,line-too-long
 
 
-# TODO(yuefengz): maybe cache variables on local CPU.
-# TODO(yuefengz): we may want to set session options to disallow communication
-# between workers.
 class ParameterServerStrategy(distribute_lib.DistributionStrategy):
   """A parameter server DistributionStrategy.
 
+  *** contrib version ***
+
   This strategy class works for both local training and between-graph replicated
   training for multiple workers. If `cluster_spec` is specified, either passed
   in to __init__() method or parsed from the
@@ -101,434 +90,24 @@ class ParameterServerStrategy(distribute_lib.DistributionStrategy):
         ParameterServerExtended(self, num_gpus_per_worker))
 
 
-class ParameterServerExtended(distribute_lib.DistributionStrategyExtended):
+class ParameterServerExtended(CoreParameterServerExtended):
   """Implementation of ParameterServerStrategy."""
 
   def __init__(self, container_strategy, num_gpus_per_worker):
-    super(ParameterServerExtended, self).__init__(container_strategy)
-    self._num_gpus_per_worker = num_gpus_per_worker
-    self._initialize_local(num_gpus_per_worker)
-
-    # We typically don't need to do all-reduce in this strategy.
-    self._cross_device_ops = (
-        cross_device_ops_lib.ReductionToOneDeviceCrossDeviceOps(
-            reduce_to_device=_LOCAL_CPU))
-
-  def _initialize_multi_worker(self, num_gpus_per_worker, cluster_spec,
-                               task_type, task_id):
-    """Initialize devices for multiple workers.
-
-    It creates variable devices and compute devices. Variables and operations
-    will be assigned to them respectively. We have one compute device per
-    replica. The variable device is a device function or device string. The
-    default variable device assigns variables to parameter servers in a
-    round-robin fashion.
-
-    Args:
-      num_gpus_per_worker: number of local GPUs or GPUs per worker.
-      cluster_spec: a dict, ClusterDef or ClusterSpec object specifying the
-        cluster configurations.
-      task_type: the current task type.
-      task_id: the current task id.
-
-    Raises:
-      ValueError: if the cluster_spec doesn't have ps jobs.
-    """
-    assert cluster_spec
-    if not task_type or task_id is None:
-      raise ValueError("When `cluster_spec` is given, you must also specify "
-                       "`task_type` and `task_id`")
-    cluster_spec = multi_worker_util.normalize_cluster_spec(cluster_spec)
-
-    worker_device = "/job:%s/task:%d" % (self._task_type, self._task_id)
-
-    # Define compute devices which is a list of device strings and one for each
-    # replica. When there are GPUs, replicate operations on these GPUs.
-    # Otherwise, place operations on CPU.
-    if num_gpus_per_worker > 0:
-      compute_devices = tuple(
-          "%s/device:GPU:%d" % (worker_device, i)
-          for i in range(num_gpus_per_worker)
-      )
-    else:
-      compute_devices = (worker_device,)
-
-    self._device_map = values.ReplicaDeviceMap(compute_devices)
-    self._input_workers = input_lib.InputWorkers(
-        self._device_map, [(worker_device, compute_devices)])
-
-    # In distributed mode, place variables on ps jobs in a round-robin fashion.
-    # Note that devices returned from `replica_device_setter` are not
-    # canonical and therefore we don't canonicalize all variable devices to
-    # make them consistent.
-    # TODO(yuefengz): support passing a strategy object to control variable
-    # assignment.
-    # TODO(yuefengz): merge the logic of replica_device_setter into this
-    # class.
-    num_ps_replicas = len(cluster_spec.as_dict().get("ps", []))
-    if num_ps_replicas == 0:
-      raise ValueError("The cluster spec needs to have `ps` jobs.")
-    self._variable_device = device_setter.replica_device_setter(
-        ps_tasks=num_ps_replicas,
-        worker_device=worker_device,
-        merge_devices=True,
-        cluster=cluster_spec)
-
-    # The `_parameter_devices` is needed for the `parameter_devices` property
-    # and is a list of all variable devices. Here parameter devices are all
-    # tasks of the "ps" job.
-    self._parameter_devices = tuple(map("/job:ps/task:{}".format,
-                                        range(num_ps_replicas)))
-
-    # Add a default device so that ops without specified devices will not end up
-    # on other workers.
-    self._default_device = worker_device
-
-    self._is_chief = multi_worker_util.is_chief(cluster_spec, task_type,
-                                                task_id)
-    self._cluster_spec = cluster_spec
-    self._task_type = task_type
-    self._task_id = task_id
-
-    logging.info(
-        "Multi-worker ParameterServerStrategy with "
-        "cluster_spec = %r, task_type = %r, task_id = %r, "
-        "num_ps_replicas = %r, is_chief = %r, device_map = %r, "
-        "variable_device = %r", cluster_spec.as_dict(), task_type, task_id,
-        num_ps_replicas, self._is_chief, self._device_map,
-        self._variable_device)
-
-  def _initialize_local(self, num_gpus_per_worker):
-    """Initialize internal devices for local training."""
-    worker_device = device_util.canonicalize("/device:CPU:0")
-    # Define compute devices which is a list of device strings and one for each
-    # replica. When there are GPUs, replicate operations on these GPUs.
-    # Otherwise, place operations on CPU.
-    if num_gpus_per_worker > 0:
-      compute_devices = tuple(
-          map("/device:GPU:{}".format, range(num_gpus_per_worker)))
-    else:
-      compute_devices = (_LOCAL_CPU,)
-
-    self._device_map = values.ReplicaDeviceMap(compute_devices)
-    self._input_workers = input_lib.InputWorkers(
-        self._device_map, [(worker_device, compute_devices)])
-
-    # If there is only one GPU, put everything on that GPU. Otherwise, place
-    # variables on CPU.
-    if num_gpus_per_worker == 1:
-      assert len(compute_devices) == 1
-      self._variable_device = _LOCAL_GPU_0
-      self._parameter_devices = (_LOCAL_GPU_0,)
-    else:
-      self._variable_device = _LOCAL_CPU
-      self._parameter_devices = (_LOCAL_CPU,)
-
-    self._is_chief = True
-    self._cluster_spec = None
-    self._task_type = None
-    self._task_id = None
-
-    logging.info(
-        "ParameterServerStrategy with compute_devices = %r, "
-        "variable_device = %r", compute_devices, self._variable_device)
-
-  def _validate_colocate_with_variable(self, colocate_with_variable):
-    values.validate_colocate(colocate_with_variable, self)
-
-  def _distribute_dataset(self, dataset_fn):
-    """Distributes the dataset to each local GPU."""
-    return input_lib.PerReplicaDataset(
-        self._call_dataset_fn(dataset_fn), self._input_workers, 0,
-        prefetch_on_device=True)
+    # Use TFConfigClusterResolver to parse TF_CONFIG. We don't want to change
+    # the constructor's interface to allow customized cluster resolver. Use
+    # SimpleClusterResolver to override num_accelerators.
+    tfconfig = TFConfigClusterResolver()
+    cluster_resolver = SimpleClusterResolver(
+        cluster_spec=tfconfig.cluster_spec(),
+        task_type=tfconfig.task_type,
+        task_index=tfconfig.task_index,
+        num_accelerators=num_gpus_per_worker)
+    super(ParameterServerExtended, self).__init__(
+        container_strategy, cluster_resolver=cluster_resolver)
 
   def _make_dataset_iterator(self, dataset):
-    return input_lib.DatasetIterator(dataset, self._input_workers,
-                                     self._num_replicas_in_sync)
-
-  def _make_input_fn_iterator(
-      self,
-      input_fn,
-      replication_mode=distribute_lib.InputReplicationMode.PER_WORKER):
-    """Distributes the dataset to each local GPU."""
-    if self._cluster_spec:
-      input_pipeline_id = multi_worker_util.id_in_cluster(
-          self._cluster_spec, self._task_type, self._task_id)
-      num_input_pipelines = multi_worker_util.worker_count(
-          self._cluster_spec, self._task_type)
-    else:
-      input_pipeline_id = 0
-      num_input_pipelines = 1
-    input_context = distribute_lib.InputContext(
-        num_input_pipelines=num_input_pipelines,
-        input_pipeline_id=input_pipeline_id,
-        num_replicas_in_sync=self._num_replicas_in_sync)
-    return input_lib.InputFunctionIterator(
-        input_fn, self._input_workers, [input_context])
-
-  def _broadcast_to(self, tensor, destinations):
-    # This is both a fast path for Python constants, and a way to delay
-    # converting Python values to a tensor until we know what type it
-    # should be converted to. Otherwise we have trouble with:
-    #   global_step.assign_add(1)
-    # since the `1` gets broadcast as an int32 but global_step is int64.
-    if isinstance(tensor, (float, int)):
-      return tensor
-    if not cross_device_ops_lib.check_destinations(destinations):
-      # TODO(josh11b): Use current logical device instead of 0 here.
-      destinations = values.LogicalDeviceSpec(
-          device_map=self._device_map, logical_device=0)
-    return self._cross_device_ops.broadcast(tensor, destinations)
-
-  def _allow_variable_partition(self):
-    return not context.executing_eagerly()
-
-  # TODO(yuefengz): not all ops in device_setter.STANDARD_PS_OPS will go through
-  # this creator, such as "MutableHashTable".
-  def _create_variable(self, next_creator, *args, **kwargs):
-    if self._num_replicas_in_sync > 1:
-      aggregation = kwargs.pop("aggregation", vs.VariableAggregation.NONE)
-      if aggregation not in (
-          vs.VariableAggregation.NONE,
-          vs.VariableAggregation.SUM,
-          vs.VariableAggregation.MEAN,
-          vs.VariableAggregation.ONLY_FIRST_REPLICA
-      ):
-        raise ValueError("Invalid variable aggregation mode: " + aggregation +
-                         " for variable: " + kwargs["name"])
-
-      def var_creator(*args, **kwargs):
-        """Create an AggregatingVariable and fix up collections."""
-        # Record what collections this variable should be added to.
-        collections = kwargs.pop("collections", None)
-        if collections is None:
-          collections = [ops.GraphKeys.GLOBAL_VARIABLES]
-        kwargs["collections"] = []
-
-        # Create and wrap the variable.
-        v = next_creator(*args, **kwargs)
-        wrapped = values.AggregatingVariable(
-            self._container_strategy(), v, aggregation)
-
-        # Add the wrapped variable to the requested collections.
-        # The handling of eager mode and the global step matches
-        # ResourceVariable._init_from_args().
-        if not context.executing_eagerly():
-          g = ops.get_default_graph()
-          # If "trainable" is True, next_creator() will add the contained
-          # variable to the TRAINABLE_VARIABLES collection, so we manually
-          # remove it and replace with the wrapper. We can't set "trainable"
-          # to False for next_creator() since that causes functions like
-          # implicit_gradients to skip those variables.
-          if kwargs.get("trainable", True):
-            collections.append(ops.GraphKeys.TRAINABLE_VARIABLES)
-            l = g.get_collection_ref(ops.GraphKeys.TRAINABLE_VARIABLES)
-            l.remove(v)
-          g.add_to_collections(collections, wrapped)
-        elif ops.GraphKeys.GLOBAL_STEP in collections:
-          ops.add_to_collections(ops.GraphKeys.GLOBAL_STEP, wrapped)
-
-        return wrapped
-    else:
-      var_creator = next_creator
-
-    if "colocate_with" in kwargs:
-      with ops.device(None):
-        with ops.colocate_with(kwargs["colocate_with"]):
-          return var_creator(*args, **kwargs)
-
-    with ops.colocate_with(None, ignore_existing=True):
-      with ops.device(self._variable_device):
-        return var_creator(*args, **kwargs)
-
-  def _call_for_each_replica(self, fn, args, kwargs):
-    # pylint: disable=protected-access
-    return mirrored_strategy._call_for_each_replica(
-        self._container_strategy(), self._device_map, fn, args, kwargs)
-
-  def _verify_destinations_not_different_worker(self, destinations):
-    if not self._cluster_spec:
-      return
-    if destinations is None:
-      return
-    for d in cross_device_ops_lib.get_devices_from(destinations):
-      d_spec = tf_device.DeviceSpec.from_string(d)
-      if d_spec.job == self._task_type and d_spec.task != self._task_id:
-        raise ValueError(
-            "Cannot reduce to another worker: %r, current worker is %r" %
-            (d, self._input_workers.worker_devices[0]))
-
-  def _reduce_to(self, reduce_op, value, destinations):
-    self._verify_destinations_not_different_worker(destinations)
-    if not isinstance(value, values.DistributedValues):
-      # pylint: disable=protected-access
-      return cross_device_ops_lib.reduce_non_distributed_value(
-          reduce_op, self._device_map, value, destinations)
-    return self._cross_device_ops.reduce(
-        reduce_op, value, destinations=destinations)
-
-  def _batch_reduce_to(self, reduce_op, value_destination_pairs):
-    for _, destinations in value_destination_pairs:
-      self._verify_destinations_not_different_worker(destinations)
-    return self._cross_device_ops.batch_reduce(reduce_op,
-                                               value_destination_pairs)
-
-  def _select_single_value(self, structured):
-    """Select any single values in `structured`."""
-
-    def _select_fn(x):  # pylint: disable=g-missing-docstring
-      if isinstance(x, values.Mirrored):
-        if len(x.devices) == 1:
-          return x.primary
-        else:
-          raise ValueError(
-              "You cannot update variable with a Mirrored object with multiple "
-              "components %r when using ParameterServerStrategy. You must "
-              "specify a single value or a Mirrored with a single value." % x)
-      elif isinstance(x, values.PerReplica):
-        raise ValueError(
-            "You cannot update variable with a PerReplica object %r when using "
-            "ParameterServerStrategy. You must specify a single value or a "
-            "Mirrored with a single value" % x)
-      else:
-        return x
-
-    return nest.map_structure(_select_fn, structured)
-
-  def _update(self, var, fn, args, kwargs, group):
-    if isinstance(var, values.AggregatingVariable):
-      var = var.get()
-    if not isinstance(var, resource_variable_ops.ResourceVariable):
-      raise ValueError(
-          "You can not update `var` %r. It must be a Variable." % var)
-    with ops.colocate_with(var), distribute_lib.UpdateContext(var.device):
-      result = fn(var, *self._select_single_value(args),
-                  **self._select_single_value(kwargs))
-      if group:
-        return result
-      else:
-        return nest.map_structure(self._unwrap, result)
-
-  # TODO(yuefengz): does it need to call _select_single_value?
-  def _update_non_slot(self, colocate_with, fn, args, kwargs, group):
-    with ops.device(
-        colocate_with.device), distribute_lib.UpdateContext(colocate_with):
-      result = fn(*args, **kwargs)
-      if group:
-        return result
-      else:
-        return nest.map_structure(self._unwrap, result)
-
-  def _unwrap(self, val):
-    if isinstance(val, values.DistributedValues):
-      return val.values
-    return (val,)
-
-  def value_container(self, val):
-    if (hasattr(val, "_aggregating_container") and
-        not isinstance(val, values.AggregatingVariable)):
-      wrapper = val._aggregating_container()  # pylint: disable=protected-access
-      if wrapper is not None:
-        return wrapper
-    return val
-
-  def read_var(self, var):
-    # No need to distinguish between normal variables and replica-local
-    # variables.
-    return array_ops.identity(var)
-
-  def _configure(self,
-                 session_config=None,
-                 cluster_spec=None,
-                 task_type=None,
-                 task_id=None):
-    """Configures the strategy class.
-
-    The strategy object will be re-initialized if `cluster_spec` is given but
-    was not passed in the constructor.
-
-    Args:
-      session_config: not used currently.
-      cluster_spec: a dict, ClusterDef or ClusterSpec object specifying the
-        cluster configurations.
-      task_type: the current task type.
-      task_id: the current task id.
-
-    Raises:
-      ValueError: if `cluster_spec` is given but `task_type` or `task_id` is
-        not.
-    """
-    if not self._cluster_spec and cluster_spec:
-      # If a `cluster_spec` is already passed in, do nothing here.
-      # TODO(yuefengz): check `cluster_spec` is the same if this object has
-      # already been initialized with a `cluster_spec`.
-      if task_type is None or task_id is None:
-        raise ValueError("When `cluster_spec` is given, must also specify "
-                         "`task_type` and `task_id`.")
-      self._cluster_spec = multi_worker_util.normalize_cluster_spec(
-          cluster_spec)
-      self._task_type = task_type
-      self._task_id = task_id
-      self._initialize_multi_worker(self._num_gpus_per_worker,
-                                    self._cluster_spec, task_type, task_id)
-
-    if session_config:
-      session_config.CopyFrom(self._update_config_proto(session_config))
-
-  def _update_config_proto(self, config_proto):
-    updated_config = copy.deepcopy(config_proto)
-    if not self._cluster_spec:
-      updated_config.isolate_session_state = True
-      return updated_config
-
-    updated_config.isolate_session_state = False
-
-    assert self._task_type
-    assert self._task_id is not None
-
-    # The device filters prevent communication between workers.
-    if self._task_type not in ["chief", "worker"]:
-      return updated_config
-    del updated_config.device_filters[:]
-    updated_config.device_filters.extend(
-        ["/job:%s/task:%d" % (self._task_type, self._task_id), "/job:ps"])
-    return updated_config
-
-  @property
-  def _num_replicas_in_sync(self):
-    return self._device_map.num_replicas_in_graph
-
-  @property
-  def worker_devices(self):
-    return self._device_map.all_devices
-
-  @property
-  def worker_devices_by_replica(self):
-    return self._device_map.devices_by_replica
-
-  @property
-  def parameter_devices(self):
-    return self._parameter_devices
-
-  def non_slot_devices(self, var_list):
-    return min(var_list, key=lambda x: x.name)
-
-  @property
-  def experimental_between_graph(self):
-    # TODO(yuefengz): Should this return False in the local case?
-    return True
-
-  @property
-  def experimental_should_init(self):
-    return self._is_chief
-
-  @property
-  def should_checkpoint(self):
-    return self._is_chief
-
-  @property
-  def should_save_summary(self):
-    return self._is_chief
+    return input_lib.DatasetIterator(dataset, self._input_workers)
 
   # TODO(priyag): Delete this once all strategies use global batch size.
   @property
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
index ce7065f220..d8c6c50db2 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
@@ -29,10 +29,13 @@ from tensorflow.contrib.distribute.python import strategy_test_lib
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.distribute import device_util
+from tensorflow.python.distribute import distribute_lib
 from tensorflow.python.distribute import distribution_strategy_context as ds_context
 from tensorflow.python.distribute import multi_worker_util
+from tensorflow.python.distribute import parameter_server_strategy as core_parameter_server_strategy
 from tensorflow.python.distribute import reduce_util
 from tensorflow.python.distribute import values
+from tensorflow.python.distribute.cluster_resolver import SimpleClusterResolver
 from tensorflow.python.eager import backprop
 from tensorflow.python.eager import context
 from tensorflow.python.estimator import run_config
@@ -50,6 +53,7 @@ from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
 from tensorflow.python.training import training_util
+from tensorflow.python.training.server_lib import ClusterSpec
 
 CHIEF = run_config.TaskType.CHIEF
 WORKER = run_config.TaskType.WORKER
@@ -63,6 +67,57 @@ def _get_replica_id_integer():
   return replica_id
 
 
+class MockCoreParameterServerStrategy(distribute_lib.DistributionStrategy):
+  """Mock the strategy to allow cluster resolver as an argument."""
+
+  def __init__(self, cluster_resolver):
+    super(MockCoreParameterServerStrategy, self).__init__(
+        core_parameter_server_strategy.ParameterServerStrategyExtended(
+            self, cluster_resolver=cluster_resolver))
+
+
+def create_test_objects(cluster_spec=None,
+                        task_type=None,
+                        task_id=None,
+                        num_gpus=None,
+                        sess_config=None,
+                        use_core_strategy=False):
+  sess_config = sess_config or config_pb2.ConfigProto()
+  if num_gpus is None:
+    num_gpus = context.num_gpus()
+  if use_core_strategy:
+    if cluster_spec and task_type and task_id is not None:
+      cluster_resolver = SimpleClusterResolver(
+          cluster_spec=multi_worker_util.normalize_cluster_spec(cluster_spec),
+          task_type=task_type,
+          task_index=task_id,
+          num_accelerators=num_gpus)
+      target = 'grpc://' + cluster_spec[WORKER][task_id]
+    else:
+      cluster_resolver = SimpleClusterResolver(
+          ClusterSpec({}), num_accelerators=num_gpus)
+      target = ''
+
+    distribution = MockCoreParameterServerStrategy(cluster_resolver)
+    sess_config = copy.deepcopy(sess_config)
+    sess_config = distribution.update_config_proto(sess_config)
+  else:
+    distribution = parameter_server_strategy.ParameterServerStrategy(
+        num_gpus_per_worker=num_gpus)
+    if task_type:
+      sess_config = copy.deepcopy(sess_config)
+      distribution.configure(
+          session_config=sess_config,
+          cluster_spec=cluster_spec,
+          task_type=task_type,
+          task_id=task_id)
+      target = 'grpc://' + cluster_spec[WORKER][task_id]
+    else:
+      target = ''
+
+  return distribution, target, sess_config
+
+
 class ParameterServerStrategyTestBase(
     multi_worker_test_base.MultiWorkerTestBase):
 
@@ -76,24 +131,27 @@ class ParameterServerStrategyTestBase(
     self._sess_config = config_pb2.ConfigProto(allow_soft_placement=True)
     super(ParameterServerStrategyTestBase, self).setUp()
 
-  def _get_test_objects(self, task_type, task_id, num_gpus):
-    distribution = parameter_server_strategy.ParameterServerStrategy(
-        num_gpus_per_worker=num_gpus)
-    if not task_type:
-      return distribution, '', self._sess_config
-
-    sess_config = copy.deepcopy(self._sess_config)
-    distribution.configure(
-        session_config=sess_config,
+  def _get_test_objects(self,
+                        task_type,
+                        task_id,
+                        num_gpus,
+                        use_core_strategy=False):
+    return create_test_objects(
         cluster_spec=self._cluster_spec,
         task_type=task_type,
-        task_id=task_id)
-    return (distribution, 'grpc://' + self._cluster_spec[WORKER][task_id],
-            sess_config)
-
-  def _test_device_assignment_distributed(self, task_type, task_id, num_gpus):
+        task_id=task_id,
+        num_gpus=num_gpus,
+        sess_config=self._sess_config,
+        use_core_strategy=use_core_strategy)
+
+  def _test_device_assignment_distributed(self,
+                                          task_type,
+                                          task_id,
+                                          num_gpus,
+                                          use_core_strategy=False):
     worker_device = '/job:%s/replica:0/task:%d' % (task_type, task_id)
-    d, _, sess_config = self._get_test_objects(task_type, task_id, num_gpus)
+    d, _, sess_config = self._get_test_objects(
+        task_type, task_id, num_gpus, use_core_strategy=use_core_strategy)
     with ops.Graph().as_default(), \
          self.cached_session(target=self._default_target,
                              config=sess_config) as sess, \
@@ -191,8 +249,9 @@ class ParameterServerStrategyTestBase(
         self.assertEqual(f_val, 46.0)
 
   def _test_device_assignment_distributed_enable_partitioner(
-      self, task_type, task_id, num_gpus):
-    d, _, sess_config = self._get_test_objects(task_type, task_id, num_gpus)
+      self, task_type, task_id, num_gpus, use_core_strategy=False):
+    d, _, sess_config = self._get_test_objects(
+        task_type, task_id, num_gpus, use_core_strategy=use_core_strategy)
     num_shards = len(d.parameter_devices)
     partitioner = partitioned_variables.fixed_size_partitioner(num_shards)
     with ops.Graph().as_default(), \
@@ -340,9 +399,13 @@ class ParameterServerStrategyTestBase(
         self.assertEqual(z_val, 43.0)
         self.assertEqual(f_val, 46.0)
 
-  def _test_simple_increment(self, task_type, task_id, num_gpus):
+  def _test_simple_increment(self,
+                             task_type,
+                             task_id,
+                             num_gpus,
+                             use_core_strategy=False):
     d, master_target, sess_config = self._get_test_objects(
-        task_type, task_id, num_gpus)
+        task_type, task_id, num_gpus, use_core_strategy=use_core_strategy)
     if d.extended._cluster_spec:
       num_workers = len(d.extended._cluster_spec.as_dict().get(WORKER))
       if 'chief' in d.extended._cluster_spec.as_dict():
@@ -410,9 +473,13 @@ class ParameterServerStrategyTestBase(
               y_val == 20.0 + 1.0 * num_workers * d.num_replicas_in_sync and
               z_val == 30.0 + 1.0 * num_workers)
 
-  def _test_minimize_loss_graph(self, task_type, task_id, num_gpus):
+  def _test_minimize_loss_graph(self,
+                                task_type,
+                                task_id,
+                                num_gpus,
+                                use_core_strategy=False):
     d, master_target, sess_config = self._get_test_objects(
-        task_type, task_id, num_gpus)
+        task_type, task_id, num_gpus, use_core_strategy=use_core_strategy)
     if task_type:
       # Multi-worker
       assert hasattr(d.extended, '_cluster_spec') and d.extended._cluster_spec
@@ -498,10 +565,15 @@ class ParameterServerStrategyTestBase(
       self.assertLess(error_after, error_before)
       return error_after < error_before
 
-  def _test_input_fn_iterator(self, task_type, task_id, num_gpus, input_fn,
-                              expected_values):
+  def _test_input_fn_iterator(self,
+                              task_type,
+                              task_id,
+                              num_gpus,
+                              input_fn,
+                              expected_values,
+                              use_core_strategy=False):
     distribution, master_target, config = self._get_test_objects(
-        task_type, task_id, num_gpus)
+        task_type, task_id, num_gpus, use_core_strategy=use_core_strategy)
     devices = distribution.extended.worker_devices
 
     with ops.Graph().as_default(), \
@@ -541,66 +613,93 @@ class ParameterServerStrategyTest(ParameterServerStrategyTestBase,
         num_workers=3, num_ps=2)
     cls._default_target = 'grpc://' + cls._cluster_spec[WORKER][0]
 
-  def test_num_replicas_in_sync(self):
-    distribution = parameter_server_strategy.ParameterServerStrategy(
-        num_gpus_per_worker=2)
+  @combinations.generate(
+      combinations.combine(mode=['graph'], use_core_strategy=[True, False]))
+  def test_num_replicas_in_sync(self, use_core_strategy):
+    strategy, _, _ = create_test_objects(
+        num_gpus=2, use_core_strategy=use_core_strategy)
     # All the devices on a given worker are in sync which in this case is the
     # number of gpus on each worker.
-    self.assertEqual(2, distribution.num_replicas_in_sync)
+    self.assertEqual(2, strategy.num_replicas_in_sync)
 
-  def testDeviceAssignmentLocalCPU(self):
-    distribution = parameter_server_strategy.ParameterServerStrategy(
-        num_gpus_per_worker=0)
+  @combinations.generate(
+      combinations.combine(mode=['graph'], use_core_strategy=[True, False]))
+  def testDeviceAssignmentLocalCPU(self, use_core_strategy):
+    strategy, _, _ = create_test_objects(
+        num_gpus=0, use_core_strategy=use_core_strategy)
     self._test_device_assignment_local(
-        distribution, compute_device='CPU', variable_device='CPU', num_gpus=0)
+        strategy, compute_device='CPU', variable_device='CPU', num_gpus=0)
 
-  def testDeviceAssignmentLocalOneGPU(self):
-    distribution = parameter_server_strategy.ParameterServerStrategy(
-        num_gpus_per_worker=1)
+  @combinations.generate(
+      combinations.combine(mode=['graph'], use_core_strategy=[True, False]))
+  def testDeviceAssignmentLocalOneGPU(self, use_core_strategy):
+    strategy, _, _ = create_test_objects(
+        num_gpus=1, use_core_strategy=use_core_strategy)
     self._test_device_assignment_local(
-        distribution, compute_device='GPU', variable_device='GPU', num_gpus=1)
+        strategy, compute_device='GPU', variable_device='GPU', num_gpus=1)
 
-  def testDeviceAssignmentLocalTwoGPUs(self):
-    distribution = parameter_server_strategy.ParameterServerStrategy(
-        num_gpus_per_worker=2)
+  @combinations.generate(
+      combinations.combine(mode=['graph'], use_core_strategy=[True, False]))
+  def testDeviceAssignmentLocalTwoGPUs(self, use_core_strategy):
+    strategy, _, _ = create_test_objects(
+        num_gpus=2, use_core_strategy=use_core_strategy)
     self._test_device_assignment_local(
-        distribution, compute_device='GPU', variable_device='CPU', num_gpus=2)
+        strategy, compute_device='GPU', variable_device='CPU', num_gpus=2)
 
   @combinations.generate(
-      combinations.combine(mode=['graph'], num_gpus=[0, 1, 2]))
-  def testDeviceAssignmentDistributed(self, num_gpus):
-    self._test_device_assignment_distributed('worker', 1, num_gpus)
+      combinations.combine(
+          mode=['graph'], num_gpus=[0, 1, 2], use_core_strategy=[True, False]))
+  def testDeviceAssignmentDistributed(self, num_gpus, use_core_strategy):
+    self._test_device_assignment_distributed(
+        'worker', 1, num_gpus, use_core_strategy=use_core_strategy)
 
   @combinations.generate(
-      combinations.combine(mode=['graph'], num_gpus=[0, 1, 2]))
-  def testDeviceAssignmentDistributedEnablePartitioner(self, num_gpus):
+      combinations.combine(
+          mode=['graph'], num_gpus=[0, 1, 2], use_core_strategy=[True, False]))
+  def testDeviceAssignmentDistributedEnablePartitioner(self, num_gpus,
+                                                       use_core_strategy):
     self._test_device_assignment_distributed_enable_partitioner(
-        'worker', 1, num_gpus)
+        'worker', 1, num_gpus, use_core_strategy=use_core_strategy)
 
-  def testSimpleBetweenGraph(self):
-    self._run_between_graph_clients(self._test_simple_increment,
-                                    self._cluster_spec, context.num_gpus())
+  @combinations.generate(
+      combinations.combine(mode=['graph'], use_core_strategy=[True, False]))
+  def testSimpleBetweenGraph(self, use_core_strategy):
+    self._run_between_graph_clients(
+        self._test_simple_increment,
+        self._cluster_spec,
+        context.num_gpus(),
+        use_core_strategy=use_core_strategy)
 
   @combinations.generate(
-      combinations.combine(mode=['graph'], num_gpus=[0, 1, 2]))
-  def testLocalSimpleIncrement(self, num_gpus):
-    self._test_simple_increment(None, 0, num_gpus)
+      combinations.combine(
+          mode=['graph'], num_gpus=[0, 1, 2], use_core_strategy=[True, False]))
+  def testLocalSimpleIncrement(self, num_gpus, use_core_strategy):
+    self._test_simple_increment(None, 0, num_gpus, use_core_strategy)
 
   @combinations.generate(
-      combinations.combine(mode=['graph'], num_gpus=[0, 1, 2]))
-  def testMinimizeLossGraphDistributed(self, num_gpus):
-    self._run_between_graph_clients(self._test_minimize_loss_graph,
-                                    self._cluster_spec, num_gpus)
+      combinations.combine(
+          mode=['graph'], num_gpus=[0, 1, 2], use_core_strategy=[True, False]))
+  def testMinimizeLossGraphDistributed(self, num_gpus, use_core_strategy):
+    self._run_between_graph_clients(
+        self._test_minimize_loss_graph,
+        self._cluster_spec,
+        num_gpus,
+        use_core_strategy=use_core_strategy)
 
   @combinations.generate(
-      combinations.combine(mode=['graph'], num_gpus=[0, 1, 2]))
-  def testMinimizeLossGraphLocal(self, num_gpus):
-    self._test_minimize_loss_graph(None, None, num_gpus)
+      combinations.combine(
+          mode=['graph'], num_gpus=[0, 1, 2], use_core_strategy=[True, False]))
+  def testMinimizeLossGraphLocal(self, num_gpus, use_core_strategy):
+    self._test_minimize_loss_graph(None, None, num_gpus, use_core_strategy)
 
   # TODO(priyag): Refactor this and other multi worker tests.
   @combinations.generate(
-      combinations.combine(mode=['graph'], num_gpus=[1, 2], required_gpus=1))
-  def testMakeInputFnIteratorDistributed(self, num_gpus):
+      combinations.combine(
+          mode=['graph'],
+          num_gpus=[1, 2],
+          required_gpus=1,
+          use_core_strategy=[True, False]))
+  def testMakeInputFnIteratorDistributed(self, num_gpus, use_core_strategy):
     if context.num_gpus() < num_gpus:
       self.skipTest('Not enough GPUs')
     dataset_fn = lambda: dataset_ops.Dataset.range(100)
@@ -612,12 +711,21 @@ class ParameterServerStrategyTest(ParameterServerStrategyTestBase,
         expected_num_replicas_in_sync=num_gpus,
         expected_num_input_pipelines=3,
         expected_input_pipeline_id=1)  # because task_id = 1
-    self._test_input_fn_iterator('worker', 1, num_gpus,
-                                 input_fn, expected_values)
+    self._test_input_fn_iterator(
+        'worker',
+        1,
+        num_gpus,
+        input_fn,
+        expected_values,
+        use_core_strategy=use_core_strategy)
 
   @combinations.generate(
-      combinations.combine(mode=['graph'], num_gpus=[1, 2], required_gpus=1))
-  def testMakeInputFnIteratorLocal(self, num_gpus):
+      combinations.combine(
+          mode=['graph'],
+          num_gpus=[1, 2],
+          required_gpus=1,
+          use_core_strategy=[True, False]))
+  def testMakeInputFnIteratorLocal(self, num_gpus, use_core_strategy):
     if context.num_gpus() < num_gpus:
       self.skipTest('Not enough GPUs')
     dataset_fn = lambda: dataset_ops.Dataset.range(100)
@@ -629,23 +737,31 @@ class ParameterServerStrategyTest(ParameterServerStrategyTestBase,
         expected_num_replicas_in_sync=num_gpus,
         expected_num_input_pipelines=1,
         expected_input_pipeline_id=0)  # only one worker and pipeline for local.
-    self._test_input_fn_iterator(None, None, num_gpus,
-                                 input_fn, expected_values)
+    self._test_input_fn_iterator(
+        None,
+        None,
+        num_gpus,
+        input_fn,
+        expected_values,
+        use_core_strategy=use_core_strategy)
 
-  def testGlobalStepUpdate(self):
-    strategy = parameter_server_strategy.ParameterServerStrategy(
-        num_gpus_per_worker=context.num_gpus())
+  @combinations.generate(
+      combinations.combine(mode=['graph'], use_core_strategy=[True, False]))
+  def testGlobalStepUpdate(self, use_core_strategy):
+    strategy, _, _ = create_test_objects(use_core_strategy=use_core_strategy)
     self._test_global_step_update(strategy)
 
-  def testUpdateConfigProtoMultiWorker(self):
-    distribution = parameter_server_strategy.ParameterServerStrategy(
-        num_gpus_per_worker=2)
-    distribution.configure(
+  @combinations.generate(
+      combinations.combine(mode=['graph'], use_core_strategy=[True, False]))
+  def testUpdateConfigProtoMultiWorker(self, use_core_strategy):
+    strategy, _, _ = create_test_objects(
+        num_gpus=2, use_core_strategy=use_core_strategy)
+    strategy.configure(
         cluster_spec=self._cluster_spec, task_type='worker', task_id=1)
 
     config_proto = config_pb2.ConfigProto(device_filters=['to_be_overridden'])
 
-    new_config = distribution.update_config_proto(config_proto)
+    new_config = strategy.update_config_proto(config_proto)
 
     # Verify device filters.
     self.assertEqual(['/job:worker/task:1', '/job:ps'],
@@ -654,12 +770,14 @@ class ParameterServerStrategyTest(ParameterServerStrategyTestBase,
     # Verify isolate_session_state
     self.assertFalse(new_config.isolate_session_state)
 
-  def testUpdateConfigProtoLocal(self):
-    distribution = parameter_server_strategy.ParameterServerStrategy(
-        num_gpus_per_worker=2)
+  @combinations.generate(
+      combinations.combine(mode=['graph'], use_core_strategy=[True, False]))
+  def testUpdateConfigProtoLocal(self, use_core_strategy):
+    strategy, _, _ = create_test_objects(
+        num_gpus=2, use_core_strategy=use_core_strategy)
 
     config_proto = config_pb2.ConfigProto()
-    new_config = distribution.update_config_proto(config_proto)
+    new_config = strategy.update_config_proto(config_proto)
 
     # Verify isolate_session_state
     self.assertTrue(new_config.isolate_session_state)
@@ -674,20 +792,31 @@ class ParameterServerStrategyWithChiefTest(ParameterServerStrategyTestBase,
         num_workers=3, num_ps=2, has_chief=True)
     cls._default_target = 'grpc://' + cls._cluster_spec[CHIEF][0]
 
-  def testSimpleBetweenGraph(self):
-    self._run_between_graph_clients(self._test_simple_increment,
-                                    self._cluster_spec, context.num_gpus())
+  @combinations.generate(
+      combinations.combine(mode=['graph'], use_core_strategy=[True, False]))
+  def testSimpleBetweenGraph(self, use_core_strategy):
+    self._run_between_graph_clients(
+        self._test_simple_increment,
+        self._cluster_spec,
+        context.num_gpus(),
+        use_core_strategy=use_core_strategy)
 
   @combinations.generate(
-      combinations.combine(mode=['graph'], num_gpus=[0, 1, 2]))
-  def testMinimizeLossGraph(self, num_gpus):
-    self._run_between_graph_clients(self._test_minimize_loss_graph,
-                                    self._cluster_spec, num_gpus)
+      combinations.combine(
+          mode=['graph'], num_gpus=[0, 1, 2], use_core_strategy=[True, False]))
+  def testMinimizeLossGraph(self, num_gpus, use_core_strategy):
+    self._run_between_graph_clients(
+        self._test_minimize_loss_graph,
+        self._cluster_spec,
+        num_gpus,
+        use_core_strategy=use_core_strategy)
 
-  def testGlobalStepIsWrappedOnTwoGPUs(self):
-    distribution = parameter_server_strategy.ParameterServerStrategy(
-        num_gpus_per_worker=2)
-    with ops.Graph().as_default(), distribution.scope():
+  @combinations.generate(
+      combinations.combine(mode=['graph'], use_core_strategy=[True, False]))
+  def testGlobalStepIsWrappedOnTwoGPUs(self, use_core_strategy):
+    strategy, _, _ = create_test_objects(
+        num_gpus=2, use_core_strategy=use_core_strategy)
+    with ops.Graph().as_default(), strategy.scope():
       created_step = training_util.create_global_step()
       get_step = training_util.get_global_step()
       self.assertEqual(created_step, get_step,
@@ -696,12 +825,14 @@ class ParameterServerStrategyWithChiefTest(ParameterServerStrategyTestBase,
                              id(get_step), get_step.__class__.__name__)))
       self.assertIs(values.AggregatingVariable, type(created_step))
       self.assertIs(values.AggregatingVariable, type(get_step))
-      self.assertIs(distribution, created_step.distribute_strategy)
+      self.assertIs(strategy, created_step.distribute_strategy)
 
-  def testGlobalStepIsNotWrappedOnOneGPU(self):
-    distribution = parameter_server_strategy.ParameterServerStrategy(
-        num_gpus_per_worker=1)
-    with ops.Graph().as_default(), distribution.scope():
+  @combinations.generate(
+      combinations.combine(mode=['graph'], use_core_strategy=[True, False]))
+  def testGlobalStepIsNotWrappedOnOneGPU(self, use_core_strategy):
+    strategy, _, _ = create_test_objects(
+        num_gpus=1, use_core_strategy=use_core_strategy)
+    with ops.Graph().as_default(), strategy.scope():
       created_step = training_util.create_global_step()
       get_step = training_util.get_global_step()
       self.assertEqual(created_step, get_step,
@@ -710,20 +841,24 @@ class ParameterServerStrategyWithChiefTest(ParameterServerStrategyTestBase,
                              id(get_step), get_step.__class__.__name__)))
       self.assertIs(resource_variable_ops.ResourceVariable, type(created_step))
       self.assertIs(resource_variable_ops.ResourceVariable, type(get_step))
-      self.assertIs(distribution, created_step.distribute_strategy)
+      self.assertIs(strategy, created_step.distribute_strategy)
+
+  @combinations.generate(
+      combinations.combine(mode=['graph'], use_core_strategy=[True, False]))
+  def testValueContainer(self, use_core_strategy):
+    strategy, _, _ = create_test_objects(
+        num_gpus=2, use_core_strategy=use_core_strategy)
+    with ops.Graph().as_default(), strategy.scope():
 
-  def testValueContainer(self):
-    distribution = parameter_server_strategy.ParameterServerStrategy(
-        num_gpus_per_worker=2)
-    with ops.Graph().as_default(), distribution.scope():
       def f():
         with backprop.GradientTape() as tape:
           v = variable_scope.get_variable('v', initializer=10.0)
           _ = v * v
         v, = tape.watched_variables()
-        w = distribution.extended.value_container(v)
+        w = strategy.extended.value_container(v)
         self.assertIs(values.AggregatingVariable, type(w))
-      distribution.extended.call_for_each_replica(f)
+
+      strategy.extended.call_for_each_replica(f)
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/distribute/BUILD b/tensorflow/python/distribute/BUILD
index 987fb00454..faaa61934a 100644
--- a/tensorflow/python/distribute/BUILD
+++ b/tensorflow/python/distribute/BUILD
@@ -242,6 +242,28 @@ py_library(
     ],
 )
 
+py_library(
+    name = "parameter_server_strategy",
+    srcs = ["parameter_server_strategy.py"],
+    visibility = ["//tensorflow:internal"],
+    deps = [
+        ":input_lib",
+        ":mirrored_strategy",
+        "//tensorflow/core:protos_all_py",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:resource_variable_ops",
+        "//tensorflow/python:training",
+        "//tensorflow/python:util",
+        "//tensorflow/python/distribute:cross_device_ops",
+        "//tensorflow/python/distribute:multi_worker_util",
+        "//tensorflow/python/distribute:reduce_util",
+        "//tensorflow/python/distribute:values",
+        "//tensorflow/python/distribute/cluster_resolver:cluster_resolver_lib",
+        "//tensorflow/python/eager:context",
+    ],
+)
+
 py_library(
     name = "multi_worker_util",
     srcs = [
diff --git a/tensorflow/python/distribute/parameter_server_strategy.py b/tensorflow/python/distribute/parameter_server_strategy.py
new file mode 100644
index 0000000000..71fbffdc0d
--- /dev/null
+++ b/tensorflow/python/distribute/parameter_server_strategy.py
@@ -0,0 +1,528 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Classes implementing a multi-worker ps DistributionStrategy."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import copy
+
+from tensorflow.contrib.distribute.python import mirrored_strategy
+from tensorflow.python.distribute import cross_device_ops as cross_device_ops_lib
+from tensorflow.python.distribute import device_util
+from tensorflow.python.distribute import distribute_lib
+from tensorflow.python.distribute import input_lib
+from tensorflow.python.distribute import multi_worker_util
+from tensorflow.python.distribute import values
+from tensorflow.python.distribute.cluster_resolver import SimpleClusterResolver
+from tensorflow.python.distribute.cluster_resolver import TFConfigClusterResolver
+from tensorflow.python.eager import context
+from tensorflow.python.framework import device as tf_device
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.ops import variable_scope as vs
+from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.training import device_setter
+from tensorflow.python.util import nest
+
+_LOCAL_CPU = "/device:CPU:0"
+_LOCAL_GPU_0 = "/device:GPU:0"
+
+
+# TODO(yuefengz): maybe cache variables on local CPU.
+class ParameterServerStrategy(distribute_lib.DistributionStrategy):
+  """A parameter server DistributionStrategy.
+
+  This strategy class works for both local training and between-graph replicated
+  training for multiple workers. It uses `TFConfigClusterResolver` to detect
+  configurations for multi-worker training. In multi-worker training mode, i.e.
+  `TFConfigClusterResolver` has detected 'TF_CONFIG' environment variable and
+  'TF_CONFIG' has a cluster spec, variables and updates to those variables are
+  assigned to parameter servers and other operations are assigned to workers.
+  In local training mode, variables are assigned to local CPU or the only GPU.
+  When each worker has more than one GPU, operations will be replicated on these
+  GPUs. In both cases, operations are replicated but variables are not and these
+  workers share a common view for which paramater server a variable is assigned
+  to.
+
+  This class assumes between-graph replication will be used and works on a graph
+  for a particular worker. Note that each graph and worker is independent.
+  This means that while each worker will synchronously compute a single gradient
+  update across all GPUs, updates between workers proceed asynchronously.
+  Operations that occur only on the first replica (such as incrementing the
+  global step), will occur on the first replica *of every worker*.
+
+  It is expected to call `call_for_each_replica(fn, ...)` for any
+  operations which potentially can be replicated across replicas (i.e. multiple
+  GPUs) even if there is only CPU or one GPU. When defining the `fn`, extra
+  caution needs to be taken:
+
+  1) It is generally not recommended to open a device scope under the strategy's
+  scope. A device scope (i.e. calling `tf.device`) will be merged with or
+  override the device for operations but will not change the device for
+  variables.
+
+  2) It is also not recommended to open a colocation scope (i.e. calling
+  `tf.colocate_with`) under the strategy's scope. For colocating variables,
+  use `distribution.colocate_vars_with` instead. Colocation of ops will possibly
+  create conflicts of device assignment.
+  """
+
+  def __init__(self):
+    """Initializes this strategy with default TFConfigClusterResolver."""
+    super(ParameterServerStrategy, self).__init__(
+        ParameterServerStrategyExtended(self))
+
+
+class ParameterServerStrategyExtended(
+    distribute_lib.DistributionStrategyExtended):
+  """Implementation of ParameterServerStrategy."""
+
+  def __init__(self,
+               container_strategy,
+               cluster_resolver=TFConfigClusterResolver()):
+    super(ParameterServerStrategyExtended, self).__init__(container_strategy)
+    self._initialize_strategy(cluster_resolver)
+
+    # We typically don't need to do all-reduce in this strategy.
+    self._cross_device_ops = (
+        cross_device_ops_lib.ReductionToOneDeviceCrossDeviceOps(
+            reduce_to_device=_LOCAL_CPU))
+
+  def _initialize_strategy(self, cluster_resolver):
+    if cluster_resolver.cluster_spec().as_dict():
+      self._initialize_multi_worker(cluster_resolver)
+    else:
+      self._initialize_local(cluster_resolver)
+    # Save the num_gpus_per_worker for configure method.
+    self._num_gpus_per_worker = cluster_resolver.num_accelerators()
+
+  def _initialize_multi_worker(self, cluster_resolver):
+    """Initialize devices for multiple workers.
+
+    It creates variable devices and compute devices. Variables and operations
+    will be assigned to them respectively. We have one compute device per
+    replica. The variable device is a device function or device string. The
+    default variable device assigns variables to parameter servers in a
+    round-robin fashion.
+
+    Args:
+      cluster_resolver: a descendant of `ClusterResolver` object.
+
+    Raises:
+      ValueError: if the cluster doesn't have ps jobs.
+    """
+    num_gpus = cluster_resolver.num_accelerators()
+    cluster_spec = cluster_resolver.cluster_spec()
+    task_type = cluster_resolver.task_type
+    task_id = cluster_resolver.task_index
+    if not task_type or task_id is None:
+      raise ValueError("When `cluster_spec` is given, you must also specify "
+                       "`task_type` and `task_id`")
+    cluster_spec = multi_worker_util.normalize_cluster_spec(cluster_spec)
+    assert cluster_spec.as_dict()
+
+    worker_device = "/job:%s/task:%d" % (task_type, task_id)
+
+    # Define compute devices which is a list of device strings and one for each
+    # replica. When there are GPUs, replicate operations on these GPUs.
+    # Otherwise, place operations on CPU.
+    if num_gpus > 0:
+      compute_devices = tuple(
+          "%s/device:GPU:%d" % (worker_device, i) for i in range(num_gpus))
+    else:
+      compute_devices = (worker_device,)
+
+    self._device_map = values.ReplicaDeviceMap(compute_devices)
+    self._input_workers = input_lib.InputWorkers(
+        self._device_map, [(worker_device, compute_devices)])
+
+    # In distributed mode, place variables on ps jobs in a round-robin fashion.
+    # Note that devices returned from `replica_device_setter` are not
+    # canonical and therefore we don't canonicalize all variable devices to
+    # make them consistent.
+    # TODO(yuefengz): support passing a strategy object to control variable
+    # assignment.
+    # TODO(yuefengz): merge the logic of replica_device_setter into this
+    # class.
+    num_ps_replicas = len(cluster_spec.as_dict().get("ps", []))
+    if num_ps_replicas == 0:
+      raise ValueError("The cluster spec needs to have `ps` jobs.")
+    self._variable_device = device_setter.replica_device_setter(
+        ps_tasks=num_ps_replicas,
+        worker_device=worker_device,
+        merge_devices=True,
+        cluster=cluster_spec)
+
+    # The `_parameter_devices` is needed for the `parameter_devices` property
+    # and is a list of all variable devices. Here parameter devices are all
+    # tasks of the "ps" job.
+    self._parameter_devices = tuple(map("/job:ps/task:{}".format,
+                                        range(num_ps_replicas)))
+
+    # Add a default device so that ops without specified devices will not end up
+    # on other workers.
+    self._default_device = worker_device
+
+    self._is_chief = multi_worker_util.is_chief(cluster_spec, task_type,
+                                                task_id)
+    self._cluster_spec = cluster_spec
+    self._task_type = task_type
+    self._task_id = task_id
+
+    logging.info(
+        "Multi-worker ParameterServerStrategy with "
+        "cluster_spec = %r, task_type = %r, task_id = %r, "
+        "num_ps_replicas = %r, is_chief = %r, device_map = %r, "
+        "variable_device = %r", cluster_spec.as_dict(), task_type, task_id,
+        num_ps_replicas, self._is_chief, self._device_map,
+        self._variable_device)
+
+  def _initialize_local(self, cluster_resolver):
+    """Initialize internal devices for local training."""
+    worker_device = device_util.canonicalize("/device:CPU:0")
+    num_gpus = cluster_resolver.num_accelerators()
+    # Define compute devices which is a list of device strings and one for each
+    # replica. When there are GPUs, replicate operations on these GPUs.
+    # Otherwise, place operations on CPU.
+    if num_gpus > 0:
+      compute_devices = tuple(map("/device:GPU:{}".format, range(num_gpus)))
+    else:
+      compute_devices = (_LOCAL_CPU,)
+
+    self._device_map = values.ReplicaDeviceMap(compute_devices)
+    self._input_workers = input_lib.InputWorkers(
+        self._device_map, [(worker_device, compute_devices)])
+
+    # If there is only one GPU, put everything on that GPU. Otherwise, place
+    # variables on CPU.
+    if num_gpus == 1:
+      assert len(compute_devices) == 1
+      self._variable_device = _LOCAL_GPU_0
+      self._parameter_devices = (_LOCAL_GPU_0,)
+    else:
+      self._variable_device = _LOCAL_CPU
+      self._parameter_devices = (_LOCAL_CPU,)
+
+    self._is_chief = True
+    self._cluster_spec = None
+    self._task_type = None
+    self._task_id = None
+
+    logging.info(
+        "ParameterServerStrategy with compute_devices = %r, "
+        "variable_device = %r", compute_devices, self._variable_device)
+
+  def _validate_colocate_with_variable(self, colocate_with_variable):
+    values.validate_colocate(colocate_with_variable, self)
+
+  def _distribute_dataset(self, dataset_fn):
+    """Distributes the dataset to each local GPU."""
+    return input_lib.PerReplicaDataset(
+        self._call_dataset_fn(dataset_fn),
+        self._input_workers,
+        0,
+        prefetch_on_device=True)
+
+  def _make_dataset_iterator(self, dataset):
+    return input_lib.DatasetIterator(dataset, self._input_workers,
+                                     self._num_replicas_in_sync)
+
+  def _make_input_fn_iterator(
+      self,
+      input_fn,
+      replication_mode=distribute_lib.InputReplicationMode.PER_WORKER):
+    """Distributes the dataset to each local GPU."""
+    if self._cluster_spec:
+      input_pipeline_id = multi_worker_util.id_in_cluster(
+          self._cluster_spec, self._task_type, self._task_id)
+      num_input_pipelines = multi_worker_util.worker_count(
+          self._cluster_spec, self._task_type)
+    else:
+      input_pipeline_id = 0
+      num_input_pipelines = 1
+    input_context = distribute_lib.InputContext(
+        num_input_pipelines=num_input_pipelines,
+        input_pipeline_id=input_pipeline_id,
+        num_replicas_in_sync=self._num_replicas_in_sync)
+    return input_lib.InputFunctionIterator(input_fn, self._input_workers,
+                                           [input_context])
+
+  def _broadcast_to(self, tensor, destinations):
+    # This is both a fast path for Python constants, and a way to delay
+    # converting Python values to a tensor until we know what type it
+    # should be converted to. Otherwise we have trouble with:
+    #   global_step.assign_add(1)
+    # since the `1` gets broadcast as an int32 but global_step is int64.
+    if isinstance(tensor, (float, int)):
+      return tensor
+    if not cross_device_ops_lib.check_destinations(destinations):
+      # TODO(josh11b): Use current logical device instead of 0 here.
+      destinations = values.LogicalDeviceSpec(
+          device_map=self._device_map, logical_device=0)
+    return self._cross_device_ops.broadcast(tensor, destinations)
+
+  def _allow_variable_partition(self):
+    return not context.executing_eagerly()
+
+  # TODO(yuefengz): not all ops in device_setter.STANDARD_PS_OPS will go through
+  # this creator, such as "MutableHashTable".
+  def _create_variable(self, next_creator, *args, **kwargs):
+    if self._num_replicas_in_sync > 1:
+      aggregation = kwargs.pop("aggregation", vs.VariableAggregation.NONE)
+      if aggregation not in (
+          vs.VariableAggregation.NONE,
+          vs.VariableAggregation.SUM,
+          vs.VariableAggregation.MEAN,
+          vs.VariableAggregation.ONLY_FIRST_REPLICA
+      ):
+        raise ValueError("Invalid variable aggregation mode: " + aggregation +
+                         " for variable: " + kwargs["name"])
+
+      def var_creator(*args, **kwargs):
+        """Create an AggregatingVariable and fix up collections."""
+        # Record what collections this variable should be added to.
+        collections = kwargs.pop("collections", None)
+        if collections is None:
+          collections = [ops.GraphKeys.GLOBAL_VARIABLES]
+        kwargs["collections"] = []
+
+        # Create and wrap the variable.
+        v = next_creator(*args, **kwargs)
+        wrapped = values.AggregatingVariable(
+            self._container_strategy(), v, aggregation)
+
+        # Add the wrapped variable to the requested collections.
+        # The handling of eager mode and the global step matches
+        # ResourceVariable._init_from_args().
+        if not context.executing_eagerly():
+          g = ops.get_default_graph()
+          # If "trainable" is True, next_creator() will add the contained
+          # variable to the TRAINABLE_VARIABLES collection, so we manually
+          # remove it and replace with the wrapper. We can't set "trainable"
+          # to False for next_creator() since that causes functions like
+          # implicit_gradients to skip those variables.
+          if kwargs.get("trainable", True):
+            collections.append(ops.GraphKeys.TRAINABLE_VARIABLES)
+            l = g.get_collection_ref(ops.GraphKeys.TRAINABLE_VARIABLES)
+            l.remove(v)
+          g.add_to_collections(collections, wrapped)
+        elif ops.GraphKeys.GLOBAL_STEP in collections:
+          ops.add_to_collections(ops.GraphKeys.GLOBAL_STEP, wrapped)
+
+        return wrapped
+    else:
+      var_creator = next_creator
+
+    if "colocate_with" in kwargs:
+      with ops.device(None):
+        with ops.colocate_with(kwargs["colocate_with"]):
+          return var_creator(*args, **kwargs)
+
+    with ops.colocate_with(None, ignore_existing=True):
+      with ops.device(self._variable_device):
+        return var_creator(*args, **kwargs)
+
+  def _call_for_each_replica(self, fn, args, kwargs):
+    # pylint: disable=protected-access
+    return mirrored_strategy._call_for_each_replica(
+        self._container_strategy(), self._device_map, fn, args, kwargs)
+
+  def _verify_destinations_not_different_worker(self, destinations):
+    if not self._cluster_spec:
+      return
+    if destinations is None:
+      return
+    for d in cross_device_ops_lib.get_devices_from(destinations):
+      d_spec = tf_device.DeviceSpec.from_string(d)
+      if d_spec.job == self._task_type and d_spec.task != self._task_id:
+        raise ValueError(
+            "Cannot reduce to another worker: %r, current worker is %r" %
+            (d, self._input_workers.worker_devices[0]))
+
+  def _reduce_to(self, reduce_op, value, destinations):
+    self._verify_destinations_not_different_worker(destinations)
+    if not isinstance(value, values.DistributedValues):
+      # pylint: disable=protected-access
+      return cross_device_ops_lib.reduce_non_distributed_value(
+          reduce_op, self._device_map, value, destinations)
+    return self._cross_device_ops.reduce(
+        reduce_op, value, destinations=destinations)
+
+  def _batch_reduce_to(self, reduce_op, value_destination_pairs):
+    for _, destinations in value_destination_pairs:
+      self._verify_destinations_not_different_worker(destinations)
+    return self._cross_device_ops.batch_reduce(reduce_op,
+                                               value_destination_pairs)
+
+  def _select_single_value(self, structured):
+    """Select any single values in `structured`."""
+
+    def _select_fn(x):  # pylint: disable=g-missing-docstring
+      if isinstance(x, values.Mirrored):
+        if len(x.devices) == 1:
+          return x.primary
+        else:
+          raise ValueError(
+              "You cannot update variable with a Mirrored object with multiple "
+              "components %r when using ParameterServerStrategy. You must "
+              "specify a single value or a Mirrored with a single value." % x)
+      elif isinstance(x, values.PerReplica):
+        raise ValueError(
+            "You cannot update variable with a PerReplica object %r when using "
+            "ParameterServerStrategy. You must specify a single value or a "
+            "Mirrored with a single value" % x)
+      else:
+        return x
+
+    return nest.map_structure(_select_fn, structured)
+
+  def _update(self, var, fn, args, kwargs, group):
+    if isinstance(var, values.AggregatingVariable):
+      var = var.get()
+    if not isinstance(var, resource_variable_ops.ResourceVariable):
+      raise ValueError(
+          "You can not update `var` %r. It must be a Variable." % var)
+    with ops.colocate_with(var), distribute_lib.UpdateContext(var.device):
+      result = fn(var, *self._select_single_value(args),
+                  **self._select_single_value(kwargs))
+      if group:
+        return result
+      else:
+        return nest.map_structure(self._unwrap, result)
+
+  # TODO(yuefengz): does it need to call _select_single_value?
+  def _update_non_slot(self, colocate_with, fn, args, kwargs, group):
+    with ops.device(
+        colocate_with.device), distribute_lib.UpdateContext(colocate_with):
+      result = fn(*args, **kwargs)
+      if group:
+        return result
+      else:
+        return nest.map_structure(self._unwrap, result)
+
+  def _unwrap(self, val):
+    if isinstance(val, values.DistributedValues):
+      return val.values
+    return (val,)
+
+  def value_container(self, val):
+    if (hasattr(val, "_aggregating_container") and
+        not isinstance(val, values.AggregatingVariable)):
+      wrapper = val._aggregating_container()  # pylint: disable=protected-access
+      if wrapper is not None:
+        return wrapper
+    return val
+
+  def read_var(self, var):
+    # No need to distinguish between normal variables and replica-local
+    # variables.
+    return array_ops.identity(var)
+
+  def _configure(self,
+                 session_config=None,
+                 cluster_spec=None,
+                 task_type=None,
+                 task_id=None):
+    """Configures the strategy class.
+
+    The strategy object will be re-initialized if `cluster_spec` is given but
+    was not passed in the constructor.
+
+    Args:
+      session_config: not used currently.
+      cluster_spec: a dict, ClusterDef or ClusterSpec object specifying the
+        cluster configurations.
+      task_type: the current task type.
+      task_id: the current task id.
+
+    Raises:
+      ValueError: if `cluster_spec` is given but `task_type` or `task_id` is
+        not.
+    """
+    if cluster_spec:
+      # Use the num_gpus_per_worker recorded in constructor since _configure
+      # doesn't take num_gpus.
+      cluster_resolver = SimpleClusterResolver(
+          cluster_spec=multi_worker_util.normalize_cluster_spec(cluster_spec),
+          task_type=task_type,
+          task_index=task_id,
+          num_accelerators=self._num_gpus_per_worker)
+      self._initialize_multi_worker(cluster_resolver)
+
+    if session_config:
+      session_config.CopyFrom(self._update_config_proto(session_config))
+
+  def _update_config_proto(self, config_proto):
+    updated_config = copy.deepcopy(config_proto)
+    if not self._cluster_spec:
+      updated_config.isolate_session_state = True
+      return updated_config
+
+    updated_config.isolate_session_state = False
+
+    assert self._task_type
+    assert self._task_id is not None
+
+    # The device filters prevent communication between workers.
+    if self._task_type not in ["chief", "worker"]:
+      return updated_config
+    del updated_config.device_filters[:]
+    updated_config.device_filters.extend(
+        ["/job:%s/task:%d" % (self._task_type, self._task_id), "/job:ps"])
+    return updated_config
+
+  @property
+  def _num_replicas_in_sync(self):
+    return self._device_map.num_replicas_in_graph
+
+  @property
+  def worker_devices(self):
+    return self._device_map.all_devices
+
+  @property
+  def worker_devices_by_replica(self):
+    return self._device_map.devices_by_replica
+
+  @property
+  def parameter_devices(self):
+    return self._parameter_devices
+
+  def non_slot_devices(self, var_list):
+    return min(var_list, key=lambda x: x.name)
+
+  @property
+  def experimental_between_graph(self):
+    # TODO(yuefengz): Should this return False in the local case?
+    return True
+
+  @property
+  def experimental_should_init(self):
+    return self._is_chief
+
+  @property
+  def should_checkpoint(self):
+    return self._is_chief
+
+  @property
+  def should_save_summary(self):
+    return self._is_chief
+
+  # TODO(priyag): Delete this once all strategies use global batch size.
+  @property
+  def _global_batch_size(self):
+    return True
-- 
GitLab


From cea2e9c4fab95890873114bf60a5f19facca0c2f Mon Sep 17 00:00:00 2001
From: Sergei Lebedev <slebedev@google.com>
Date: Fri, 4 Jan 2019 13:17:37 -0800
Subject: [PATCH 0248/1095] Removed set_shape overrides from ResourceVariable
 subclasses.

These are essentially no-ops and allow for reads returning tensors inconsistent
with variable shape:

    >>> import tensorflow as tf
    >>> tf.enable_eager_execution()
    >>> v = tf.Variable(0)
    >>> t = v.assign(42)
    >>> t.set_shape((100500, ))
    >>> t.shape
    (100500, )
    >>> tf.convert_to_tensor(t)
    <tf.Tensor: id=9, shape=(), dtype=int32, numpy=42>

PiperOrigin-RevId: 227903155
---
 tensorflow/python/ops/resource_variable_ops.py | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index ede4bc0078..d2e5dd9cfe 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -1351,10 +1351,6 @@ class _UnreadVariable(ResourceVariable):
       return gen_resource_variable_ops.read_variable_op(self._handle,
                                                         self._dtype)
 
-  def set_shape(self, shape):
-    self._shape = shape
-    self._cached_shape_as_list = None
-
   @property
   def op(self):
     """The op for this variable."""
@@ -1443,10 +1439,6 @@ class _MixedPrecisionVariable(ResourceVariable):
       else:
         return res
 
-  def set_shape(self, shape):
-    self._shape = shape
-    self._cached_shape_as_list = None
-
   @property
   def op(self):
     """The op for this variable."""
-- 
GitLab


From 4614b04cc076c543501124670da8b8da417cbf9f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 4 Jan 2019 13:49:49 -0800
Subject: [PATCH 0249/1095] One more refactoring of kernel.cc. This introduces
 OpInputs to match OpOutputs, which is slightly enhanced.

PiperOrigin-RevId: 227908106
---
 tensorflow/lite/delegates/flex/kernel.cc | 127 ++++++++++++++---------
 1 file changed, 80 insertions(+), 47 deletions(-)

diff --git a/tensorflow/lite/delegates/flex/kernel.cc b/tensorflow/lite/delegates/flex/kernel.cc
index 3012241f4f..0005b6e5b0 100644
--- a/tensorflow/lite/delegates/flex/kernel.cc
+++ b/tensorflow/lite/delegates/flex/kernel.cc
@@ -51,24 +51,57 @@ namespace tflite {
 namespace flex {
 namespace kernel {
 
-// Controls the lifetime of tensor handles in a vector.
+// A list of inputs of a given node of the TensorFlow/Eager graph.
+class OpInputs {
+ public:
+  explicit OpInputs(const TfLiteIntArray* indexes) {
+    for (int index : TfLiteIntArrayView(indexes)) {
+      inputs_.push_back(index);
+    }
+  }
+  ~OpInputs() {}
+
+  int Size() const { return inputs_.size(); }
+
+  int TfLiteIndex(int i) const { return inputs_[i]; }
+
+ private:
+  std::vector<int> inputs_;
+};
+
+// A list of outputs of a given node of the TensorFlow/Eager graph, along with
+// the actual outputs of the EagerOperation.
 class OpOutputs {
  public:
-  explicit OpOutputs(int num_elements) : vector_(num_elements, nullptr) {}
+  explicit OpOutputs(const TfLiteIntArray* indexes) {
+    for (int index : TfLiteIntArrayView(indexes)) {
+      outputs_.push_back(index);
+    }
+    vector_.resize(outputs_.size());
+  }
+  ~OpOutputs() { ResetTensorHandles(); }
+
+  int Size() const { return outputs_.size(); }
+
+  int TfLiteIndex(int i) const { return outputs_[i]; }
 
-  ~OpOutputs() {
-    for (auto* handle : vector_) {
-      if (handle) handle->Unref();
+  // Carefully unreference all the handles in the eager output vector.
+  void ResetTensorHandles() {
+    for (int i = 0; i < vector_.size(); ++i) {
+      if (vector_[i]) {
+        vector_[i]->Unref();
+        vector_[i] = nullptr;
+      }
     }
   }
 
-  tensorflow::gtl::InlinedVector<tensorflow::TensorHandle*, 2>* GetVector() {
+  tensorflow::gtl::InlinedVector<tensorflow::TensorHandle*, 2>*
+  GetTensorHandles() {
     return &vector_;
   }
 
-  tensorflow::TensorHandle* GetHandle(int index) { return vector_[index]; }
-
  private:
+  std::vector<int> outputs_;
   tensorflow::gtl::InlinedVector<tensorflow::TensorHandle*, 2> vector_;
 };
 
@@ -76,7 +109,8 @@ class OpOutputs {
 // TensorFlow ops within a single TF Lite op.
 class OpNode {
  public:
-  OpNode() {}
+  OpNode(const TfLiteIntArray* inputs, const TfLiteIntArray* outputs)
+      : inputs_(inputs), outputs_(outputs) {}
   ~OpNode() {}
 
   const string& name() const { return name_; }
@@ -87,19 +121,14 @@ class OpNode {
 
   const tensorflow::NodeDef& nodedef() const { return nodedef_; }
 
-  const std::vector<int>& inputs() const { return inputs_; }
-  void InitializeInputs(const TfLiteIntArray* inputs) {
-    for (int index : TfLiteIntArrayView(inputs)) {
-      inputs_.push_back(index);
-    }
-  }
+  const OpInputs& inputs() const { return inputs_; }
+  OpInputs* mutable_inputs() { return &inputs_; }
 
-  const std::vector<int>& outputs() const { return outputs_; }
-  void InitializeOutputs(const TfLiteIntArray* outputs) {
-    for (int index : TfLiteIntArrayView(outputs)) {
-      outputs_.push_back(index);
-    }
-  }
+  const OpOutputs& outputs() const { return outputs_; }
+  OpOutputs* mutable_outputs() { return &outputs_; }
+
+  int NumInputs() const { return inputs_.Size(); }
+  int NumOutputs() const { return outputs_.Size(); }
 
   tensorflow::Status InitializeNodeDef(const void* custom_initial_data,
                                        int custom_initial_data_size) {
@@ -162,7 +191,8 @@ class OpNode {
 
   tensorflow::Status BuildEagerInputs(BufferMap* buffer_map,
                                       tensorflow::EagerOperation* op) {
-    for (int input_index : inputs_) {
+    for (int i = 0; i < inputs_.Size(); ++i) {
+      int input_index = inputs_.TfLiteIndex(i);
       if (!buffer_map->HasTensor(input_index)) {
         return tensorflow::errors::Internal(
             "Cannot read from invalid tensor index ", input_index);
@@ -181,17 +211,21 @@ class OpNode {
     return tensorflow::Status::OK();
   }
 
-  tensorflow::Status PersistEagerOutputs(BufferMap* buffer_map,
-                                         OpOutputs* retvals) {
-    for (int i = 0; i < outputs_.size(); ++i) {
+  tensorflow::Status PersistEagerOutputs(BufferMap* buffer_map) {
+    auto* handles = outputs_.GetTensorHandles();
+    for (int i = 0; i < outputs_.Size(); ++i) {
       const tensorflow::Tensor* tensor = nullptr;
-      TF_RETURN_IF_ERROR(retvals->GetHandle(i)->Tensor(&tensor));
-      buffer_map->SetFromTensorFlow(outputs_[i], *tensor);
+      TF_RETURN_IF_ERROR(handles->at(i)->Tensor(&tensor));
+      buffer_map->SetFromTensorFlow(outputs_.TfLiteIndex(i), *tensor);
     }
+    outputs_.ResetTensorHandles();
     return tensorflow::Status::OK();
   }
 
  private:
+  OpNode(const OpNode&) = delete;
+  OpNode& operator=(const OpNode&) = delete;
+
   // The name of the TensorFlow op to execute.
   string name_;
   // Index of this node into TF Lite's operator list.
@@ -199,9 +233,9 @@ class OpNode {
   // The corresponding NodeDef, containing the attributes for the op.
   tensorflow::NodeDef nodedef_;
   // List of inputs, as TF Lite tensor indices.
-  std::vector<int> inputs_;
+  OpInputs inputs_;
   // List of outputs, as TF Lite tensor indices.
-  std::vector<int> outputs_;
+  OpOutputs outputs_;
 };
 
 // Executes the TensorFlow op given by 'op_name', with the attributes specified
@@ -212,18 +246,18 @@ tensorflow::Status ExecuteFlexOp(tensorflow::EagerContext* eager_context,
   TF_RETURN_IF_ERROR(node_data->BuildEagerOp(eager_context, &op));
   TF_RETURN_IF_ERROR(node_data->BuildEagerInputs(buffer_map, op.get()));
 
-  int num_retvals = node_data->outputs().size();
-  OpOutputs retvals(num_retvals);
+  int num_retvals = node_data->NumOutputs();
   TF_RETURN_WITH_CONTEXT_IF_ERROR(
-      EagerExecute(op.get(), retvals.GetVector(), &num_retvals),
+      EagerExecute(op.get(), node_data->mutable_outputs()->GetTensorHandles(),
+                   &num_retvals),
       " (while executing '", node_data->name(), "' via Eager)");
 
-  if (num_retvals != node_data->outputs().size()) {
+  if (num_retvals != node_data->NumOutputs()) {
     return tensorflow::errors::Internal(
         "Unexpected number of outputs from EagerExecute");
   }
 
-  TF_RETURN_IF_ERROR(node_data->PersistEagerOutputs(buffer_map, &retvals));
+  TF_RETURN_IF_ERROR(node_data->PersistEagerOutputs(buffer_map));
 
   return tensorflow::Status::OK();
 }
@@ -232,7 +266,7 @@ tensorflow::Status ExecuteFlexOp(tensorflow::EagerContext* eager_context,
 struct OpData {
   tensorflow::EagerContext* eager_context;
   BufferMap* buffer_map;
-  std::vector<OpNode> nodes;
+  std::vector<std::unique_ptr<OpNode>> nodes;
   std::vector<int> subgraph_inputs;
   std::vector<int> subgraph_outputs;
 };
@@ -261,6 +295,8 @@ void* Init(TfLiteContext* context, const char* buffer, size_t length) {
     op_data->subgraph_inputs.push_back(tensor_index);
   }
 
+  op_data->nodes.reserve(params->nodes_to_replace->size);
+
   CHECK(params->nodes_to_replace);
   tensorflow::Status status;
   for (auto node_index : TfLiteIntArrayView(params->nodes_to_replace)) {
@@ -268,8 +304,8 @@ void* Init(TfLiteContext* context, const char* buffer, size_t length) {
     TfLiteRegistration* reg;
     context->GetNodeAndRegistration(context, node_index, &node, &reg);
 
-    op_data->nodes.push_back(OpNode());
-    OpNode& node_data = op_data->nodes.back();
+    op_data->nodes.emplace_back(new OpNode(node->inputs, node->outputs));
+    OpNode& node_data = *op_data->nodes.back();
 
     node_data.set_index(node_index);
     node_data.set_name("");
@@ -277,9 +313,6 @@ void* Init(TfLiteContext* context, const char* buffer, size_t length) {
     status = node_data.InitializeNodeDef(node->custom_initial_data,
                                          node->custom_initial_data_size);
     if (!status.ok()) break;
-
-    node_data.InitializeInputs(node->inputs);
-    node_data.InitializeOutputs(node->outputs);
   }
 
   if (ConvertStatus(context, status) != kTfLiteOk) {
@@ -328,14 +361,14 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
   }
 
   for (const auto& node_data : op_data->nodes) {
-    if (node_data.nodedef().op().empty()) {
+    if (node_data->nodedef().op().empty()) {
       context->ReportError(context, "Invalid NodeDef in Flex op '%s'",
-                           node_data.name().c_str());
+                           node_data->name().c_str());
       return kTfLiteError;
     }
 
-    for (int tensor_index : node_data.inputs()) {
-      ++tensor_ref_count[tensor_index];
+    for (int i = 0; i < node_data->inputs().Size(); ++i) {
+      ++tensor_ref_count[node_data->inputs().TfLiteIndex(i)];
     }
   }
 
@@ -371,12 +404,12 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
   }
 
   // Execute the TensorFlow Ops sequentially.
-  for (OpNode& node_data : op_data->nodes) {
+  for (auto& node_data : op_data->nodes) {
     SCOPED_TAGGED_OPERATOR_PROFILE(
         reinterpret_cast<profiling::Profiler*>(context->profiler),
-        node_data.name().c_str(), node_data.index());
+        node_data->name().c_str(), node_data->index());
 
-    auto status = ExecuteFlexOp(eager_context, buffer_map, &node_data);
+    auto status = ExecuteFlexOp(eager_context, buffer_map, node_data.get());
     TF_LITE_ENSURE_OK(context, ConvertStatus(context, status));
   }
 
-- 
GitLab


From 962d8821ebb23e918bae3680bab5a7f6df32cd7f Mon Sep 17 00:00:00 2001
From: Michael Kuperstein <mkuper@google.com>
Date: Fri, 4 Jan 2019 13:52:30 -0800
Subject: [PATCH 0250/1095] [XLA] Adapt HLO pipeline to scalar-index DS and DUS

This and adapts HLO to work with the scalar DynamicSlice / DynamicUpdateSlice
form. This means it also effectively switches the backends to use the new form.

Also a bunch of test churn - tests that don't run optimizations (and, hence,
don't trigger the splitter pass) needed to be converted.

PiperOrigin-RevId: 227908542
---
 .../xla/service/algebraic_simplifier.cc       | 27 ++++-----
 .../xla/service/algebraic_simplifier_test.cc  | 26 +++++----
 .../xla/service/buffer_assignment_test.cc     |  6 +-
 tensorflow/compiler/xla/service/cpu/BUILD     | 18 +++---
 .../compiler/xla/service/cpu/cpu_compiler.cc  |  2 +
 tensorflow/compiler/xla/service/gpu/BUILD     | 12 ++--
 .../service/gpu/multi_output_fusion_test.cc   |  5 +-
 .../xla/service/gpu/nvptx_compiler.cc         |  2 +
 .../xla/service/hlo_creation_utils.cc         | 37 ++++++++++--
 .../xla/service/hlo_evaluator_typed_visitor.h | 22 +-------
 .../compiler/xla/service/hlo_verifier.cc      | 19 +------
 .../compiler/xla/service/hlo_verifier_test.cc |  5 +-
 .../compiler/xla/service/interpreter/BUILD    |  9 +--
 .../xla/service/interpreter/compiler.cc       |  2 +
 .../xla/service/layout_assignment_test.cc     |  7 ++-
 .../compiler/xla/service/pattern_matcher.h    |  2 +-
 .../compiler/xla/service/shape_inference.cc   | 10 ++--
 .../compiler/xla/service/shape_inference.h    |  4 +-
 tensorflow/compiler/xla/shape_util.cc         |  1 +
 tensorflow/compiler/xla/tests/BUILD           |  1 +
 tensorflow/compiler/xla/tests/test_utils.cc   | 52 +++++++----------
 .../compiler/xla/tests/test_utils_test.cc     | 56 ++++++++++---------
 22 files changed, 162 insertions(+), 163 deletions(-)

diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
index 003cb7e791..5ac746c9f3 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
@@ -1380,6 +1380,9 @@ StatusOr<HloInstruction*> AlgebraicSimplifierVisitor::OptimizeDotOfGather(
   // => output dimensions: DS ({M x N}, {0, start}, {M, 1}) => {M x 1}.
 
   bool lhs_is_dynamic_slice = lhs->opcode() == HloOpcode::kDynamicSlice;
+  HloDynamicSliceInstruction* dynamic_slice =
+      lhs_is_dynamic_slice ? Cast<HloDynamicSliceInstruction>(lhs)
+                           : Cast<HloDynamicSliceInstruction>(rhs);
 
   // ctA:
   HloInstruction* left_operand =
@@ -1397,8 +1400,6 @@ StatusOr<HloInstruction*> AlgebraicSimplifierVisitor::OptimizeDotOfGather(
       HloInstruction::CreateDot(memoized_shape, left_operand, right_operand,
                                 dnums, dot->precision_config()));
   // Get pair {start, 0} or {0, start}.
-  HloInstruction* original_start_indices =
-      lhs_is_dynamic_slice ? lhs->mutable_operand(1) : rhs->mutable_operand(1);
   // Position of start:
   int index_of_non_zero_start = lhs_is_dynamic_slice
                                     ? 1 - lhs_contracting_dimension
@@ -1407,23 +1408,19 @@ StatusOr<HloInstruction*> AlgebraicSimplifierVisitor::OptimizeDotOfGather(
   int index_of_zero_start = 1 - index_of_non_zero_start;
 
   // Slice out start and 0 components and reorder if necessary.
-  auto indices_type = original_start_indices->shape().element_type();
+  auto indices_type = dynamic_slice->operand(1)->shape().element_type();
   Shape s_shape = ShapeUtil::MakeShape(indices_type, {1});
   Shape d_shape = ShapeUtil::MakeShape(indices_type, {2});
   HloInstruction* non_zero_start =
-      computation_->AddInstruction(HloInstruction::CreateSlice(
-          s_shape, original_start_indices, {index_of_non_zero_start},
-          {index_of_non_zero_start + 1}, {1}));
+      dynamic_slice->mutable_operand(1 + index_of_non_zero_start);
   HloInstruction* zero_start =
-      computation_->AddInstruction(HloInstruction::CreateSlice(
-          s_shape, original_start_indices, {index_of_zero_start},
-          {index_of_zero_start + 1}, {1}));
-  HloInstruction* new_start_indices =
-      lhs_is_dynamic_slice
-          ? computation_->AddInstruction(HloInstruction::CreateConcatenate(
-                d_shape, {non_zero_start, zero_start}, 0))
-          : computation_->AddInstruction(HloInstruction::CreateConcatenate(
-                d_shape, {zero_start, non_zero_start}, 0));
+      dynamic_slice->mutable_operand(1 + index_of_zero_start);
+  std::vector<HloInstruction*> new_start_indices;
+  if (lhs_is_dynamic_slice) {
+    new_start_indices = {non_zero_start, zero_start};
+  } else {
+    new_start_indices = {zero_start, non_zero_start};
+  }
 
   // Build DynamicSlice(ctA x ctB).
   const int new_slice_m = lhs_is_dynamic_slice ? 1 : m;
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
index 211c5bf05a..e39321f1f3 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
@@ -4535,14 +4535,17 @@ TEST_P(DotOfGatherSimplificationTest, ConstantRHS) {
 
   int32 start_row = (spec.lcd == 0) ? 0 : spec.s;
   int32 start_col = (spec.lcd == 0) ? spec.s : 0;
-  const auto start_indices =
+  std::vector<HloInstruction*> start_indices = {
       builder.AddInstruction(HloInstruction::CreateConstant(
-          LiteralUtil::CreateR1<int32>({start_row, start_col})));
+          LiteralUtil::CreateR0<int32>(start_row))),
+      builder.AddInstruction(HloInstruction::CreateConstant(
+          LiteralUtil::CreateR0<int32>(start_col)))};
   int64 slice_row_size = (spec.lcd == 0) ? spec.k : 1;
   int64 slice_col_size = (spec.lcd == 0) ? 1 : spec.k;
-  Shape ds_shape = ShapeUtil::MakeShape(F32, {slice_row_size, slice_col_size});
+  std::vector<int64> slice_sizes = {slice_row_size, slice_col_size};
+  Shape ds_shape = ShapeUtil::MakeShape(F32, slice_sizes);
   auto* ds = builder.AddInstruction(HloInstruction::CreateDynamicSlice(
-      ds_shape, lhs, start_indices, {slice_row_size, slice_col_size}));
+      ds_shape, lhs, start_indices, slice_sizes));
 
   int64 rhs_rows = (spec.rcd == 0) ? spec.k : spec.n;
   int64 rhs_cols = (spec.rcd == 0) ? spec.n : spec.k;
@@ -4575,7 +4578,7 @@ TEST_P(DotOfGatherSimplificationTest, ConstantRHS) {
   } else {
     EXPECT_THAT(computation->root_instruction(),
                 GmockMatch(m::DynamicSlice(m::Dot(m::Constant(), m::Constant()),
-                                           m::Concatenate())));
+                                           m::Constant(), m::Constant())));
   }
 }
 
@@ -4613,14 +4616,17 @@ TEST_P(DotOfGatherSimplificationTest, ConstantLHS) {
 
   int32 start_row = (spec.rcd == 0) ? 0 : spec.s;
   int32 start_col = (spec.rcd == 0) ? spec.s : 0;
-  const auto start_indices =
+  std::vector<HloInstruction*> start_indices = {
+      builder.AddInstruction(HloInstruction::CreateConstant(
+          LiteralUtil::CreateR0<int32>(start_row))),
       builder.AddInstruction(HloInstruction::CreateConstant(
-          LiteralUtil::CreateR1<int32>({start_row, start_col})));
+          LiteralUtil::CreateR0<int32>(start_col)))};
   int64 slice_row_size = (spec.rcd == 0) ? spec.k : 1;
   int64 slice_col_size = (spec.rcd == 0) ? 1 : spec.k;
-  Shape ds_shape = ShapeUtil::MakeShape(F32, {slice_row_size, slice_col_size});
+  std::vector<int64> slice_sizes = {slice_row_size, slice_col_size};
+  Shape ds_shape = ShapeUtil::MakeShape(F32, slice_sizes);
   auto* ds = builder.AddInstruction(HloInstruction::CreateDynamicSlice(
-      ds_shape, rhs, start_indices, {slice_row_size, slice_col_size}));
+      ds_shape, rhs, start_indices, slice_sizes));
 
   DotDimensionNumbers dot_dnums;
   dot_dnums.add_lhs_contracting_dimensions(spec.lcd);
@@ -4645,7 +4651,7 @@ TEST_P(DotOfGatherSimplificationTest, ConstantLHS) {
   } else {
     EXPECT_THAT(computation->root_instruction(),
                 GmockMatch(m::DynamicSlice(m::Dot(m::Constant(), m::Constant()),
-                                           m::Concatenate())));
+                                           m::Constant(), m::Constant())));
   }
 }
 
diff --git a/tensorflow/compiler/xla/service/buffer_assignment_test.cc b/tensorflow/compiler/xla/service/buffer_assignment_test.cc
index 370fad7ffb..1b4e93a2f3 100644
--- a/tensorflow/compiler/xla/service/buffer_assignment_test.cc
+++ b/tensorflow/compiler/xla/service/buffer_assignment_test.cc
@@ -2486,9 +2486,9 @@ while_body {
   get-tuple-element.3 = s32[] get-tuple-element(state), index=0
   constant.2 = s32[] constant(128)
   add.5 = s32[] add(get-tuple-element.3, constant.2)
-  constant.3 = s32[3]{0} constant({0, 0, 0})
-  dynamic-update-slice.5 = f32[1280,1,128]{2,1,0} dynamic-update-slice(get-tuple-element.4, broadcast.6, constant.3)
-  dynamic-update-slice.9 = f32[1280,1,128]{2,1,0} dynamic-update-slice(dynamic-update-slice.5, broadcast.6, constant.3)
+  constant.3 = s32[] constant(0)
+  dynamic-update-slice.5 = f32[1280,1,128]{2,1,0} dynamic-update-slice(get-tuple-element.4, broadcast.6, constant.3, constant.3, constant.3)
+  dynamic-update-slice.9 = f32[1280,1,128]{2,1,0} dynamic-update-slice(dynamic-update-slice.5, broadcast.6, constant.3, constant.3, constant.3)
   ROOT tuple.85 = (s32[], s32[], s32[2]{0}, f32[1280,1,128]{2,1,0}) tuple(add.5, dynamic-update-slice.9)
 }
 
diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD
index a4ca772c7c..5c72e1ff71 100644
--- a/tensorflow/compiler/xla/service/cpu/BUILD
+++ b/tensorflow/compiler/xla/service/cpu/BUILD
@@ -1,6 +1,14 @@
 # Description:
 #    LLVM-based CPU backend for XLA.
 
+load("//tensorflow/compiler/xla:xla.bzl", "ORC_JIT_MEMORY_MAPPER_TARGETS")
+load(
+    "//third_party/mkl:build_defs.bzl",
+    "mkl_deps",
+)
+load("//tensorflow:tensorflow.bzl", "tf_cc_binary", "tf_cc_test")
+load(":build_defs.bzl", "runtime_copts")
+
 licenses(["notice"])  # Apache 2.0
 
 package(
@@ -14,15 +22,6 @@ package_group(
     ],
 )
 
-load(":build_defs.bzl", "runtime_copts")
-load("//tensorflow:tensorflow.bzl", "tf_cc_test")
-load("//tensorflow:tensorflow.bzl", "tf_cc_binary")
-load("//tensorflow/compiler/xla:xla.bzl", "ORC_JIT_MEMORY_MAPPER_TARGETS")
-load(
-    "//third_party/mkl:build_defs.bzl",
-    "mkl_deps",
-)
-
 # Filegroup used to collect source files for dependency checking.
 filegroup(
     name = "c_srcs",
@@ -114,6 +113,7 @@ cc_library(
         "//tensorflow/compiler/xla/service:conditional_simplifier",
         "//tensorflow/compiler/xla/service:convolution_group_converter",
         "//tensorflow/compiler/xla/service:dot_decomposer",
+        "//tensorflow/compiler/xla/service:dynamic_index_splitter",
         "//tensorflow/compiler/xla/service:executable",
         "//tensorflow/compiler/xla/service:flatten_call_graph",
         "//tensorflow/compiler/xla/service:hlo",
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
index dbab839aee..91c64e45b9 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
@@ -69,6 +69,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/cpu/simple_orc_jit.h"
 #include "tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h"
 #include "tensorflow/compiler/xla/service/dot_decomposer.h"
+#include "tensorflow/compiler/xla/service/dynamic_index_splitter.h"
 #include "tensorflow/compiler/xla/service/flatten_call_graph.h"
 #include "tensorflow/compiler/xla/service/hlo.pb.h"
 #include "tensorflow/compiler/xla/service/hlo_computation.h"
@@ -244,6 +245,7 @@ Status CpuCompiler::RunHloPassesThroughLayoutAssn(
   HloPassPipeline pipeline("HLO passes through layout assignment");
   pipeline.AddInvariantChecker<HloVerifier>(/*layout_sensitive=*/false,
                                             /*allow_mixed_precision=*/false);
+  pipeline.AddPass<DynamicIndexSplitter>();
   pipeline.AddPass<CpuHloSupportChecker>();
 
   ReducePrecisionInsertion::AddPasses(
diff --git a/tensorflow/compiler/xla/service/gpu/BUILD b/tensorflow/compiler/xla/service/gpu/BUILD
index d10a11d096..5e81281bae 100644
--- a/tensorflow/compiler/xla/service/gpu/BUILD
+++ b/tensorflow/compiler/xla/service/gpu/BUILD
@@ -3,6 +3,11 @@
 
 load("//tensorflow/compiler/xla/tests:build_defs.bzl", "xla_test")
 load("//tensorflow/compiler/xla:xla.bzl", "xla_proto_library")
+load(
+    "//tensorflow/core:platform/default/build_config_root.bzl",
+    "tf_cuda_tests_tags",
+)
+load("//tensorflow:tensorflow.bzl", "tf_cc_test")
 
 licenses(["notice"])  # Apache 2.0
 
@@ -24,12 +29,6 @@ filegroup(
     ]),
 )
 
-load("//tensorflow:tensorflow.bzl", "tf_cc_test")
-load(
-    "//tensorflow/core:platform/default/build_config_root.bzl",
-    "tf_cuda_tests_tags",
-)
-
 xla_proto_library(
     name = "backend_configs",
     srcs = ["backend_configs.proto"],
@@ -700,6 +699,7 @@ cc_library(
         "//tensorflow/compiler/xla/service:call_inliner",
         "//tensorflow/compiler/xla/service:conditional_simplifier",
         "//tensorflow/compiler/xla/service:convolution_group_converter",
+        "//tensorflow/compiler/xla/service:dynamic_index_splitter",
         "//tensorflow/compiler/xla/service:executable",
         "//tensorflow/compiler/xla/service:flatten_call_graph",
         "//tensorflow/compiler/xla/service:hlo",
diff --git a/tensorflow/compiler/xla/service/gpu/multi_output_fusion_test.cc b/tensorflow/compiler/xla/service/gpu/multi_output_fusion_test.cc
index d16c87ba5c..40b87b16a1 100644
--- a/tensorflow/compiler/xla/service/gpu/multi_output_fusion_test.cc
+++ b/tensorflow/compiler/xla/service/gpu/multi_output_fusion_test.cc
@@ -628,8 +628,7 @@ TEST_F(MultiOutputFusionTest, MultiOutputFusionDUS) {
       p.1 = s32[1]{0} parameter(1)
       p.2 = f16[1,96,1024]{2,1,0} parameter(2)
       c.0 = s32[] constant(0)
-      pad = s32[3]{0} pad(p.1, c.0), padding=0_2
-      ROOT %dynamic-update-slice = f16[50,96,1024]{2,1,0} dynamic-update-slice(p.0, p.2, pad)
+      ROOT %dynamic-update-slice = f16[50,96,1024]{2,1,0} dynamic-update-slice(p.0, p.2, p.1, c.0, c.0)
     }
 
     fusion.2 {
@@ -638,7 +637,7 @@ TEST_F(MultiOutputFusionTest, MultiOutputFusionDUS) {
       p.2 = f16[1,96,1024]{2,1,0} parameter(2)
       c.0 = s32[] constant(0)
       pad = s32[3]{0} pad(p.1, c.0), padding=0_2
-      ROOT %dynamic-update-slice = f16[50,96,1024]{2,1,0} dynamic-update-slice(p.0, p.2, pad)
+      ROOT %dynamic-update-slice = f16[50,96,1024]{2,1,0} dynamic-update-slice(p.0, p.2, p.1, c.0, c.0)
     }
 
     ENTRY entry {
diff --git a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
index cd369d5598..9be7609508 100644
--- a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
+++ b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
@@ -37,6 +37,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/call_inliner.h"
 #include "tensorflow/compiler/xla/service/conditional_simplifier.h"
 #include "tensorflow/compiler/xla/service/convolution_group_converter.h"
+#include "tensorflow/compiler/xla/service/dynamic_index_splitter.h"
 #include "tensorflow/compiler/xla/service/flatten_call_graph.h"
 #include "tensorflow/compiler/xla/service/gpu/cudnn_batchnorm_rewriter.h"
 #include "tensorflow/compiler/xla/service/gpu/cudnn_conv_algorithm_picker.h"
@@ -152,6 +153,7 @@ Status OptimizeHloModule(HloModule* hlo_module, se::StreamExecutor* stream_exec,
     HloPassPipeline pipeline("optimization");
     pipeline.AddInvariantChecker<HloVerifier>(/*layout_sensitive=*/false,
                                               /*allow_mixed_precision=*/false);
+    pipeline.AddPass<DynamicIndexSplitter>();
     pipeline.AddPass<GpuHloSupportChecker>();
     ReducePrecisionInsertion::AddPasses(
         &pipeline, hlo_module->config().debug_options(),
diff --git a/tensorflow/compiler/xla/service/hlo_creation_utils.cc b/tensorflow/compiler/xla/service/hlo_creation_utils.cc
index 8cea95a73e..bb5d21c654 100644
--- a/tensorflow/compiler/xla/service/hlo_creation_utils.cc
+++ b/tensorflow/compiler/xla/service/hlo_creation_utils.cc
@@ -19,6 +19,7 @@ limitations under the License.
 #include "absl/strings/str_cat.h"
 #include "tensorflow/compiler/xla/literal.h"
 #include "tensorflow/compiler/xla/literal_util.h"
+#include "tensorflow/compiler/xla/service/hlo_instruction.h"
 #include "tensorflow/compiler/xla/service/hlo_module.h"
 #include "tensorflow/compiler/xla/service/shape_inference.h"
 #include "tensorflow/compiler/xla/util.h"
@@ -105,12 +106,26 @@ StatusOr<HloInstruction*> MakeDynamicSliceHlo(
     absl::Span<const int64> slice_sizes) {
   HloComputation* computation = operand->parent();
   CHECK_EQ(computation, start_indices->parent());
+  int64 rank = start_indices->shape().dimensions(0);
+  std::vector<HloInstruction*> scalar_start_indices;
+  for (int i = 0; i < rank; ++i) {
+    // TODO(b/118437727): Update callers to provide scalars directly.
+    auto slice = computation->AddInstruction(HloInstruction::CreateSlice(
+        ShapeUtil::MakeShape(start_indices->shape().element_type(), {1}),
+        start_indices, {i}, {i + 1}, {1}));
+    scalar_start_indices.push_back(
+        computation->AddInstruction(HloInstruction::CreateReshape(
+            ShapeUtil::MakeShape(start_indices->shape().element_type(), {}),
+            slice)));
+  }
+  std::vector<Shape> scalar_start_indices_shapes(
+      rank, ShapeUtil::MakeShape(start_indices->shape().element_type(), {}));
   TF_ASSIGN_OR_RETURN(
       Shape dynamic_slice_shape,
       ShapeInference::InferDynamicSliceShape(
-          operand->shape(), {start_indices->shape()}, slice_sizes));
+          operand->shape(), scalar_start_indices_shapes, slice_sizes));
   return computation->AddInstruction(HloInstruction::CreateDynamicSlice(
-      dynamic_slice_shape, operand, start_indices, slice_sizes));
+      dynamic_slice_shape, operand, scalar_start_indices, slice_sizes));
 }
 
 StatusOr<HloInstruction*> MakeDynamicUpdateSliceHlo(
@@ -119,12 +134,26 @@ StatusOr<HloInstruction*> MakeDynamicUpdateSliceHlo(
   HloComputation* computation = operand->parent();
   CHECK_EQ(computation, update->parent());
   CHECK_EQ(computation, start_indices->parent());
+  int64 rank = start_indices->shape().dimensions(0);
+  std::vector<HloInstruction*> scalar_start_indices;
+  for (int i = 0; i < rank; ++i) {
+    // TODO(b/118437727): Update callers to provide scalars directly.
+    auto slice = computation->AddInstruction(HloInstruction::CreateSlice(
+        ShapeUtil::MakeShape(start_indices->shape().element_type(), {1}),
+        start_indices, {i}, {i + 1}, {1}));
+    scalar_start_indices.push_back(
+        computation->AddInstruction(HloInstruction::CreateReshape(
+            ShapeUtil::MakeShape(start_indices->shape().element_type(), {}),
+            slice)));
+  }
+  std::vector<Shape> scalar_start_indices_shapes(
+      rank, ShapeUtil::MakeShape(start_indices->shape().element_type(), {}));
   TF_ASSIGN_OR_RETURN(
       Shape dynamic_update_slice_shape,
       ShapeInference::InferDynamicUpdateSliceShape(
-          operand->shape(), update->shape(), {start_indices->shape()}));
+          operand->shape(), update->shape(), scalar_start_indices_shapes));
   return computation->AddInstruction(HloInstruction::CreateDynamicUpdateSlice(
-      dynamic_update_slice_shape, operand, update, start_indices));
+      dynamic_update_slice_shape, operand, update, scalar_start_indices));
 }
 
 StatusOr<HloInstruction*> MakeBroadcastHlo(
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
index cc5f6cc682..698b177310 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
@@ -1410,22 +1410,12 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
     auto operand = dynamic_slice->operand(0);
     auto start_indices = dynamic_slice->operand(1);
     auto result_shape = dynamic_slice->shape();
-    // TODO(b/118437727): Remove all of this nonsense.
-    // We may get an instruction without a parent module. In this case, assume
-    // scalar indices are not allowed.
-    bool allow_scalar_index = false;
-    if (dynamic_slice->GetModule() != nullptr) {
-      allow_scalar_index = dynamic_slice->GetModule()
-                               ->config()
-                               .debug_options()
-                               .xla_allow_scalar_index_dynamic_ops();
-    }
     TF_ASSIGN_OR_RETURN(
         auto inferred_return_shape,
         ShapeInference::InferDynamicSliceShape(
             operand->shape(),
             Cast<HloDynamicSliceInstruction>(dynamic_slice)->index_shapes(),
-            dynamic_slice->dynamic_slice_sizes(), allow_scalar_index));
+            dynamic_slice->dynamic_slice_sizes()));
     TF_RET_CHECK(ShapeUtil::Compatible(result_shape, inferred_return_shape))
         << "return shape is set to: " << ShapeUtil::HumanString(result_shape)
         << " but is inferred to be: "
@@ -1483,20 +1473,12 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
     auto update = dynamic_update_slice->operand(1);
     auto start_indices = dynamic_update_slice->operand(2);
     auto result_shape = dynamic_update_slice->shape();
-    bool allow_scalar_index = false;
-    if (dynamic_update_slice->GetModule() != nullptr) {
-      allow_scalar_index = dynamic_update_slice->GetModule()
-                               ->config()
-                               .debug_options()
-                               .xla_allow_scalar_index_dynamic_ops();
-    }
     TF_ASSIGN_OR_RETURN(
         auto inferred_return_shape,
         ShapeInference::InferDynamicUpdateSliceShape(
             operand->shape(), update->shape(),
             Cast<HloDynamicUpdateSliceInstruction>(dynamic_update_slice)
-                ->index_shapes(),
-            allow_scalar_index));
+                ->index_shapes()));
     TF_RET_CHECK(ShapeUtil::Compatible(result_shape, inferred_return_shape))
         << "return shape is set to: " << ShapeUtil::HumanString(result_shape)
         << " but is inferred to be: "
diff --git a/tensorflow/compiler/xla/service/hlo_verifier.cc b/tensorflow/compiler/xla/service/hlo_verifier.cc
index 9274d42280..2c69c27ce5 100644
--- a/tensorflow/compiler/xla/service/hlo_verifier.cc
+++ b/tensorflow/compiler/xla/service/hlo_verifier.cc
@@ -504,38 +504,23 @@ Status ShapeVerifier::HandleSlice(HloInstruction* slice) {
 }
 
 Status ShapeVerifier::HandleDynamicSlice(HloInstruction* dynamic_slice) {
-  const DebugOptions& debug_options =
-      dynamic_slice->GetModule()->config().debug_options();
-  const bool allow_scalar_indices =
-      debug_options.xla_allow_scalar_index_dynamic_ops();
-  if (!allow_scalar_indices) {
-    TF_RETURN_IF_ERROR(CheckOperandCount(dynamic_slice, 2));
-  }
   return CheckShape(
       dynamic_slice,
       ShapeInference::InferDynamicSliceShape(
           dynamic_slice->operand(0)->shape(),
           Cast<HloDynamicSliceInstruction>(dynamic_slice)->index_shapes(),
-          dynamic_slice->dynamic_slice_sizes(), allow_scalar_indices));
+          dynamic_slice->dynamic_slice_sizes()));
 }
 
 Status ShapeVerifier::HandleDynamicUpdateSlice(
     HloInstruction* dynamic_update_slice) {
-  const DebugOptions& debug_options =
-      dynamic_update_slice->GetModule()->config().debug_options();
-  const bool allow_scalar_indices =
-      debug_options.xla_allow_scalar_index_dynamic_ops();
-  if (!allow_scalar_indices) {
-    TF_RETURN_IF_ERROR(CheckOperandCount(dynamic_update_slice, 3));
-  }
   return CheckShape(
       dynamic_update_slice,
       ShapeInference::InferDynamicUpdateSliceShape(
           dynamic_update_slice->operand(0)->shape(),
           dynamic_update_slice->operand(1)->shape(),
           Cast<HloDynamicUpdateSliceInstruction>(dynamic_update_slice)
-              ->index_shapes(),
-          allow_scalar_indices));
+              ->index_shapes()));
 }
 
 Status ShapeVerifier::HandleTuple(HloInstruction* tuple) {
diff --git a/tensorflow/compiler/xla/service/hlo_verifier_test.cc b/tensorflow/compiler/xla/service/hlo_verifier_test.cc
index 9b2d884f79..d27c62a879 100644
--- a/tensorflow/compiler/xla/service/hlo_verifier_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_verifier_test.cc
@@ -450,8 +450,9 @@ TEST_F(HloVerifierTestLayoutSensitive, SliceWithLayoutChangeNotAllowed) {
    HloModule SliceWithLayoutChange
     ENTRY SliceWithLayoutChange {
       par0 = f32[4,5]{0,1} parameter(0)
-      par1 = s32[2] parameter(1)
-      ROOT dslice0 = f32[3,4]{1,0} dynamic-slice(par0, par1),
+      par1 = s32[] parameter(1)
+      par2 = s32[] parameter(2)
+      ROOT dslice0 = f32[3,4]{1,0} dynamic-slice(par0, par1, par2),
         dynamic_slice_sizes={3,4}
     }
   )";
diff --git a/tensorflow/compiler/xla/service/interpreter/BUILD b/tensorflow/compiler/xla/service/interpreter/BUILD
index 8999f38ccd..2e1d2cfbda 100644
--- a/tensorflow/compiler/xla/service/interpreter/BUILD
+++ b/tensorflow/compiler/xla/service/interpreter/BUILD
@@ -1,12 +1,12 @@
-licenses(["notice"])  # Apache 2.0
-
-package(default_visibility = ["//visibility:public"])
-
 load(
     "//tensorflow/core:platform/default/build_config_root.bzl",
     "if_static",
 )
 
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//visibility:public"])
+
 cc_library(
     name = "interpreter_transfer_manager",
     srcs = ["interpreter_transfer_manager.cc"],
@@ -35,6 +35,7 @@ cc_library(
         "//tensorflow/compiler/xla/service:batchnorm_expander",
         "//tensorflow/compiler/xla/service:compiler",
         "//tensorflow/compiler/xla/service:computation_placer",
+        "//tensorflow/compiler/xla/service:dynamic_index_splitter",
         "//tensorflow/compiler/xla/service:executable",
         "//tensorflow/compiler/xla/service:flatten_call_graph",
         "//tensorflow/compiler/xla/service:hlo",
diff --git a/tensorflow/compiler/xla/service/interpreter/compiler.cc b/tensorflow/compiler/xla/service/interpreter/compiler.cc
index 69f611a094..4818b2dae0 100644
--- a/tensorflow/compiler/xla/service/interpreter/compiler.cc
+++ b/tensorflow/compiler/xla/service/interpreter/compiler.cc
@@ -21,6 +21,7 @@ limitations under the License.
 #include "absl/memory/memory.h"
 #include "tensorflow/compiler/xla/service/algebraic_simplifier.h"
 #include "tensorflow/compiler/xla/service/computation_placer.h"
+#include "tensorflow/compiler/xla/service/dynamic_index_splitter.h"
 #include "tensorflow/compiler/xla/service/flatten_call_graph.h"
 #include "tensorflow/compiler/xla/service/hlo_constant_folding.h"
 #include "tensorflow/compiler/xla/service/hlo_cse.h"
@@ -44,6 +45,7 @@ namespace interpreter {
 Status InterpreterCompiler::RunHloOptimization(HloModule* hlo_module) {
   HloPassPipeline pipeline("Interpreter");
 
+  pipeline.AddPass<DynamicIndexSplitter>();
   pipeline.AddPass<LayoutAssignment>(
       hlo_module->mutable_entry_computation_layout(),
       LayoutAssignment::InstructionCanChangeLayout);
diff --git a/tensorflow/compiler/xla/service/layout_assignment_test.cc b/tensorflow/compiler/xla/service/layout_assignment_test.cc
index 387b385157..c8cf3c47d3 100644
--- a/tensorflow/compiler/xla/service/layout_assignment_test.cc
+++ b/tensorflow/compiler/xla/service/layout_assignment_test.cc
@@ -960,8 +960,9 @@ TEST_F(LayoutAssignmentTest, CopyDSliceOperandToAvoidImplicitLayoutChange) {
     ENTRY CopyDSliceOperandToAvoidImplicitLayoutChange {
       par0 = f32[3,4]{1,0} parameter(0)
       par1 = f32[4,5]{0,1} parameter(1)
-      par2 = s32[2] parameter(2)
-      dslice0 = f32[3,4] dynamic-slice(par1, par2), dynamic_slice_sizes={3,4}
+      par2 = s32[] parameter(2)
+      par3 = s32[] parameter(3)
+      dslice0 = f32[3,4] dynamic-slice(par1, par2, par3), dynamic_slice_sizes={3,4}
       ROOT add0 = f32[3,4]{1,0} add(par0,dslice0)
     }
   )";
@@ -982,7 +983,7 @@ TEST_F(LayoutAssignmentTest, CopyDSliceOperandToAvoidImplicitLayoutChange) {
                   m::Parameter(),
                   m::DynamicSlice(
                       m::Copy(m::Parameter(1)).WithShapeEqualTo(&shape_copy),
-                      m::Parameter(2)))));
+                      m::Parameter(2), m::Parameter(3)))));
 }
 
 TEST_F(LayoutAssignmentTest, CopyConcatOperandToAvoidImplicitLayoutChange) {
diff --git a/tensorflow/compiler/xla/service/pattern_matcher.h b/tensorflow/compiler/xla/service/pattern_matcher.h
index 802b2c0c85..9e3d106021 100644
--- a/tensorflow/compiler/xla/service/pattern_matcher.h
+++ b/tensorflow/compiler/xla/service/pattern_matcher.h
@@ -2118,7 +2118,6 @@ XLA_BINOP_PATTERN(Divide)
 XLA_BINOP_PATTERN(Complex)
 XLA_BINOP_PATTERN(Convolution)
 XLA_BINOP_PATTERN(Dot)
-XLA_BINOP_PATTERN(DynamicSlice)
 XLA_COMMUTATIVE_BINOP_PATTERN(Eq)
 XLA_BINOP_PATTERN(Gather)
 XLA_BINOP_PATTERN(Ge)
@@ -2235,6 +2234,7 @@ inline auto WithOperands(Matcher&& m, int64 operand_num, FirstArg&& first_arg,
 XLA_VARIADIC_OP_PATTERN(AfterAll);
 XLA_VARIADIC_OP_PATTERN(Concatenate);
 XLA_VARIADIC_OP_PATTERN(CustomCall);
+XLA_VARIADIC_OP_PATTERN(DynamicSlice)
 XLA_VARIADIC_OP_PATTERN(Map)
 XLA_VARIADIC_OP_PATTERN(Reduce);
 XLA_VARIADIC_OP_PATTERN(Sort);
diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc
index d711c8ef0d..b729dd7660 100644
--- a/tensorflow/compiler/xla/service/shape_inference.cc
+++ b/tensorflow/compiler/xla/service/shape_inference.cc
@@ -2099,15 +2099,15 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
     }
 
     const Shape& start_indices_shape = start_index_shapes[0];
-    TF_RETURN_IF_ERROR(
-        ExpectArray(start_indices_shape, "start indices of dynamic slice"));
-
     VLOG(2) << StrFormat(
         "slicing shape %s at dynamic start_indices %s with slice_sizes={%s}",
         ShapeUtil::HumanString(operand_shape),
         ShapeUtil::HumanString(start_indices_shape),
         StrJoin(slice_sizes, ", "));
 
+    TF_RETURN_IF_ERROR(
+        ExpectArray(start_indices_shape, "start indices of dynamic slice"));
+
     if (start_indices_shape.rank() != 1) {
       return InvalidArgument(
           "Dynamic slice start indices of rank %d must be rank1.",
@@ -2151,7 +2151,7 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
             "Dynamic slice start indices must be of integral type.");
       }
       for (const Shape& index_shape : start_index_shapes) {
-        if (!ShapeUtil::Equal(first_index_shape, index_shape)) {
+        if (!ShapeUtil::Compatible(first_index_shape, index_shape)) {
           return InvalidArgument(
               "Dynamic slice start indices must all have the same shape, got "
               "mismatching indices with shapes %s and %s.",
@@ -2258,7 +2258,7 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
             "Dynamic update slice start indices must be of integral type.");
       }
       for (const Shape& index_shape : start_index_shapes) {
-        if (!ShapeUtil::Equal(first_index_shape, index_shape)) {
+        if (!ShapeUtil::Compatible(first_index_shape, index_shape)) {
           return InvalidArgument(
               "Dynamic update slice start indices must all have the same "
               "shape, got mismatching indices with shapes %s and %s.",
diff --git a/tensorflow/compiler/xla/service/shape_inference.h b/tensorflow/compiler/xla/service/shape_inference.h
index e440e364c8..7d39ef38e0 100644
--- a/tensorflow/compiler/xla/service/shape_inference.h
+++ b/tensorflow/compiler/xla/service/shape_inference.h
@@ -177,14 +177,14 @@ class ShapeInference {
   // in 'slice_sizes', with dynamic start indices shape 'start_indices_shape'.
   static StatusOr<Shape> InferDynamicSliceShape(
       const Shape& operand_shape, absl::Span<const Shape> start_index_shapes,
-      absl::Span<const int64> slice_sizes, bool allow_scalar_indices = false);
+      absl::Span<const int64> slice_sizes, bool allow_scalar_indices = true);
 
   // Infers the shape produced by a dynamic update slice operation based
   // on the shape of operand and update.
   static StatusOr<Shape> InferDynamicUpdateSliceShape(
       const Shape& operand_shape, const Shape& update_shape,
       absl::Span<const Shape> start_index_shapes,
-      bool allow_scalar_indices = false);
+      bool allow_scalar_indices = true);
 
   // Infers the shape produced by doing a compile-time-constant indexing into
   // the given input shape. This is essential for operations on tuples, because
diff --git a/tensorflow/compiler/xla/shape_util.cc b/tensorflow/compiler/xla/shape_util.cc
index 4726ee6a5b..7f359311c3 100644
--- a/tensorflow/compiler/xla/shape_util.cc
+++ b/tensorflow/compiler/xla/shape_util.cc
@@ -112,6 +112,7 @@ bool CompareShapes(const Shape& lhs, const Shape& rhs, bool compare_layouts,
 
   if (compare_layouts) {
     if (lhs.layout().format() != rhs.layout().format()) {
+      VLOG(3) << "CompareShapes: lhs layout format != rhs layout format";
       return false;
     }
     if (LayoutUtil::IsDenseArray(lhs)) {
diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index 2c11ba4283..9433264917 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -71,6 +71,7 @@ cc_library(
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/service:hlo",
+        "//tensorflow/compiler/xla/service:hlo_casting_utils",
         "//tensorflow/compiler/xla/service:hlo_dataflow_analysis",
         "//tensorflow/compiler/xla/service:hlo_verifier",
         "//tensorflow/compiler/xla/service:transfer_manager",
diff --git a/tensorflow/compiler/xla/tests/test_utils.cc b/tensorflow/compiler/xla/tests/test_utils.cc
index 18df41c117..95c89b0ba6 100644
--- a/tensorflow/compiler/xla/tests/test_utils.cc
+++ b/tensorflow/compiler/xla/tests/test_utils.cc
@@ -19,6 +19,7 @@ limitations under the License.
 #include "absl/memory/memory.h"
 #include "tensorflow/compiler/xla/literal_util.h"
 #include "tensorflow/compiler/xla/primitive_util.h"
+#include "tensorflow/compiler/xla/service/hlo_casting_utils.h"
 #include "tensorflow/compiler/xla/service/hlo_dataflow_analysis.h"
 #include "tensorflow/compiler/xla/service/hlo_verifier.h"
 #include "tensorflow/compiler/xla/service/transfer_manager.h"
@@ -274,16 +275,9 @@ bool NeedsInitValue(const HloUse& use) {
 
 // Generate random values that are constrained to the input_shape minus the
 // output_shape so as not to produce wrapping slices, for instance.
-Literal MakeRandomIndex(absl::Span<const int64> index_space,
-                        std::minstd_rand0* engine) {
-  std::vector<int32> start_indices(index_space.size());
-  if (engine != nullptr) {
-    for (int i = 0; i < index_space.size(); ++i) {
-      std::uniform_int_distribution<int32> generator(0, index_space[i]);
-      start_indices[i] = generator(*engine);
-    }
-  }
-  return LiteralUtil::CreateR1<int32>(start_indices);
+Literal MakeRandomIndex(int64 index_bound, std::minstd_rand0* engine) {
+  std::uniform_int_distribution<int32> generator(0, index_bound);
+  return LiteralUtil::CreateR0<int32>(generator(*engine));
 }
 
 // Use dataflow analysis on each parameter to see if there are uses that would
@@ -300,8 +294,8 @@ std::vector<HloInstruction*> FindConstrainedUses(
       HloInstruction* instruction = use.instruction;
       const HloOpcode opcode = instruction->opcode();
       const int64 op_num = use.operand_number;
-      if ((opcode == HloOpcode::kDynamicSlice && op_num == 1) ||
-          (opcode == HloOpcode::kDynamicUpdateSlice && op_num == 2)) {
+      if ((opcode == HloOpcode::kDynamicSlice && op_num >= 1) ||
+          (opcode == HloOpcode::kDynamicUpdateSlice && op_num >= 2)) {
         constrained_uses.push_back(instruction);
       } else if (opcode == HloOpcode::kFusion) {
         const HloInstruction* const to_analyze =
@@ -336,7 +330,7 @@ std::vector<HloInstruction*> FindConstrainedUses(
 StatusOr<Literal> CreateLiteralForConstrainedUses(
     const absl::Span<HloInstruction* const> constrained_uses,
     const HloInstruction& param, std::minstd_rand0* engine) {
-  std::vector<int64> index_space;
+  int64 index_bound = INT64_MAX;
   bool no_duplicates = false;
   bool needs_constant = false;
   ConstantType constant_type = ConstantType::kUnknown;
@@ -348,19 +342,16 @@ StatusOr<Literal> CreateLiteralForConstrainedUses(
         const Shape& slice_shape = use->opcode() == HloOpcode::kDynamicSlice
                                        ? use->shape()
                                        : use->operand(1)->shape();
-        const int64 rank = indexed_shape.rank();
-        if (!index_space.empty()) {
-          TF_RET_CHECK(rank == index_space.size());
-          for (int64 i = 0; i < rank; ++i) {
-            index_space[i] = std::min(
-                index_space[i], ShapeUtil::GetDimension(indexed_shape, i) -
-                                    ShapeUtil::GetDimension(slice_shape, i));
-          }
-        } else {
-          index_space.resize(rank);
-          for (int64 i = 0; i < rank; ++i) {
-            index_space[i] = ShapeUtil::GetDimension(indexed_shape, i) -
-                             ShapeUtil::GetDimension(slice_shape, i);
+        const int64 first_index =
+            Cast<HloDynamicIndexInstruction>(use)->first_index_operand_number();
+        for (int64 operand = first_index; operand < use->operand_count();
+             ++operand) {
+          if (use->operand(operand) == &param) {
+            index_bound = std::min(
+                index_bound,
+                ShapeUtil::GetDimension(indexed_shape, operand - first_index) -
+                    ShapeUtil::GetDimension(slice_shape,
+                                            operand - first_index));
           }
         }
         break;
@@ -388,16 +379,13 @@ StatusOr<Literal> CreateLiteralForConstrainedUses(
   }
   int constraint_count = 0;
   constraint_count += no_duplicates ? 1 : 0;
-  constraint_count += !index_space.empty() ? 1 : 0;
+  constraint_count += (index_bound != INT64_MAX) ? 1 : 0;
   constraint_count += needs_constant ? 1 : 0;
   if (constraint_count > 1) {
     return Unimplemented("Conflicting operand generation constraints.");
   }
-  if (!index_space.empty()) {
-    // constrained_uses looks through bitcasts, so param and indexed_space may
-    // not have the same shape.  (For example, param might be an R0 while
-    // indexed_space might have size 1.)
-    return MakeRandomIndex(index_space, engine)
+  if (index_bound != INT64_MAX) {
+    return MakeRandomIndex(index_bound, engine)
         .Reshape(param.shape().dimensions());
   } else if (needs_constant) {
     switch (constant_type) {
diff --git a/tensorflow/compiler/xla/tests/test_utils_test.cc b/tensorflow/compiler/xla/tests/test_utils_test.cc
index 4b4f164d1b..591d6c1922 100644
--- a/tensorflow/compiler/xla/tests/test_utils_test.cc
+++ b/tensorflow/compiler/xla/tests/test_utils_test.cc
@@ -79,25 +79,26 @@ XLA_TEST_F(TestUtilsTest, MultipleIndexSpacesForDynamicSlices) {
                     R"(HloModule index_space_module
 
     ENTRY IndexSpace {
-      index_param = s32[3]{0} parameter(0)
-      array_param.1 = f32[123,4,789]{0,1,2} parameter(1)
-      array_param.2 = f32[3,3000,5]{0,1,2} parameter(2)
-      dynamic-slice.1 = f32[1,2,3] dynamic-slice(array_param.1, index_param), dynamic_slice_sizes={1,2,3}
-      ROOT dynamic-slice.2 = f32[3,2,2] dynamic-slice(array_param.2, index_param), dynamic_slice_sizes={3,2,2}
+      index_param.0 = s32[] parameter(0)
+      index_param.1 = s32[] parameter(1)
+      index_param.2 = s32[] parameter(2)
+      array_param.1 = f32[123,4,789]{0,1,2} parameter(3)
+      array_param.2 = f32[3,3000,5]{0,1,2} parameter(4)
+      dynamic-slice.1 = f32[1,2,3] dynamic-slice(array_param.1, index_param.0, index_param.1, index_param.2), dynamic_slice_sizes={1,2,3}
+      ROOT dynamic-slice.2 = f32[3,2,2] dynamic-slice(array_param.2, index_param.0, index_param.1, index_param.2), dynamic_slice_sizes={3,2,2}
     })")
                     .ValueOrDie();
   TF_ASSERT_OK_AND_ASSIGN(std::vector<Literal> args,
                           MakeFakeArguments(module.get()));
-  ASSERT_EQ(args.size(), 3);
-  const Literal& index_arg = args[0];
+  ASSERT_EQ(args.size(), 5);
 
-  EXPECT_EQ(index_arg.Get<int32>({0}), 0);
+  EXPECT_EQ(args[0].Get<int32>({}), 0);
 
-  EXPECT_GE(index_arg.Get<int32>({1}), 0);
-  EXPECT_LE(index_arg.Get<int32>({1}), 2);
+  EXPECT_GE(args[1].Get<int32>({}), 0);
+  EXPECT_LE(args[0].Get<int32>({}), 2);
 
-  EXPECT_GE(index_arg.Get<int32>({2}), 0);
-  EXPECT_LE(index_arg.Get<int32>({2}), 3);
+  EXPECT_GE(args[2].Get<int32>({}), 0);
+  EXPECT_LE(args[2].Get<int32>({}), 3);
 }
 
 XLA_TEST_F(TestUtilsTest, MultipleIndexSpacesForDynamicUpdateSlices) {
@@ -105,28 +106,29 @@ XLA_TEST_F(TestUtilsTest, MultipleIndexSpacesForDynamicUpdateSlices) {
                     R"(HloModule index_space_module
 
     ENTRY IndexSpace {
-      index_param = s32[3]{0} parameter(0)
-      array_param.1 = f32[123,4,789]{0,1,2} parameter(1)
-      array_param.2 = f32[3,3000,5]{0,1,2} parameter(2)
-      update_param.1 = f32[1,2,3]{0,1,2} parameter(3)
-      update_param.2 = f32[3,2,2]{0,1,2} parameter(4)
-
-      dynamic-update-slice.1 = f32[123,4,789] dynamic-update-slice(array_param.1, update_param.1, index_param)
-      ROOT dynamic-update-slice.2 = f32[3,3000,5] dynamic-update-slice(array_param.2, update_param.2, index_param)
+      index_param.0 = s32[] parameter(0)
+      index_param.1 = s32[] parameter(1)
+      index_param.2 = s32[] parameter(2)
+      array_param.1 = f32[123,4,789]{0,1,2} parameter(3)
+      array_param.2 = f32[3,3000,5]{0,1,2} parameter(4)
+      update_param.1 = f32[1,2,3]{0,1,2} parameter(5)
+      update_param.2 = f32[3,2,2]{0,1,2} parameter(6)
+
+      dynamic-update-slice.1 = f32[123,4,789] dynamic-update-slice(array_param.1, update_param.1, index_param.0, index_param.1, index_param.2)
+      ROOT dynamic-update-slice.2 = f32[3,3000,5] dynamic-update-slice(array_param.2, update_param.2, index_param.0, index_param.1, index_param.2)
     })")
                     .ValueOrDie();
   TF_ASSERT_OK_AND_ASSIGN(std::vector<Literal> args,
                           MakeFakeArguments(module.get()));
-  ASSERT_EQ(args.size(), 5);
-  const Literal& index_arg = args[0];
+  ASSERT_EQ(args.size(), 7);
 
-  EXPECT_EQ(index_arg.Get<int32>({0}), 0);
+  EXPECT_EQ(args[0].Get<int32>({}), 0);
 
-  EXPECT_GE(index_arg.Get<int32>({1}), 0);
-  EXPECT_LE(index_arg.Get<int32>({1}), 2);
+  EXPECT_GE(args[1].Get<int32>({}), 0);
+  EXPECT_LE(args[0].Get<int32>({}), 2);
 
-  EXPECT_GE(index_arg.Get<int32>({2}), 0);
-  EXPECT_LE(index_arg.Get<int32>({2}), 3);
+  EXPECT_GE(args[2].Get<int32>({}), 0);
+  EXPECT_LE(args[2].Get<int32>({}), 3);
 }
 
 XLA_TEST_F(TestUtilsTest, NoDuplicatesFloats) {
-- 
GitLab


From efe565bc0981e80a52a97f3961cfba3e87023b42 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Fri, 4 Jan 2019 14:08:26 -0800
Subject: [PATCH 0251/1095] Make the initial tf.train.Checkpoint.restore() read
 Tensors in a batch

Should be faster when reading from distributed file systems. Does not affect cases where restore-on-create is necessary, but as long as variable objects have been created and tracked before restore() their reads should be batched together.

PiperOrigin-RevId: 227911381
---
 .../python/training/checkpointable/util.py    | 22 +++------
 .../training/saving/functional_saver.py       | 45 +++++++++++--------
 2 files changed, 31 insertions(+), 36 deletions(-)

diff --git a/tensorflow/python/training/checkpointable/util.py b/tensorflow/python/training/checkpointable/util.py
index c890a7f440..a45263f5c6 100644
--- a/tensorflow/python/training/checkpointable/util.py
+++ b/tensorflow/python/training/checkpointable/util.py
@@ -176,25 +176,13 @@ class _CheckpointRestoreCoordinator(object):
         raise AssertionError(
             ("Saveable keys changed when validating. Got back %s, was "
              "expecting %s") % (tensor_saveables.keys(), validated_names))
-      for saveable in validated_saveables:
-        if saveable.device:
-          device = saveable_object_util.set_cpu0(saveable.device)
-        else:
-          device = None
-        with ops.device(device):
-          tensors = []
-          for spec in saveable.specs:
-            tensors.append(
-                io_ops.restore_v2(
-                    self.save_path_tensor,
-                    [spec.name],
-                    [spec.slice_spec],
-                    [spec.dtype])[0])
-          restore_op = saveable.restore(tensors, restored_shapes=None)
-        if not context.executing_eagerly():
+      new_restore_ops = functional_saver.restore_from_saveable_objects(
+          self.save_path_tensor, validated_saveables)
+      if not context.executing_eagerly():
+        restore_ops.extend(new_restore_ops)
+        for saveable, restore_op in zip(validated_saveables, new_restore_ops):
           assert saveable.name not in self.restore_ops_by_name
           self.restore_ops_by_name[saveable.name] = restore_op
-          restore_ops.append(restore_op)
     return restore_ops
 
 
diff --git a/tensorflow/python/training/saving/functional_saver.py b/tensorflow/python/training/saving/functional_saver.py
index 51f618ddd3..4ff2742c2f 100644
--- a/tensorflow/python/training/saving/functional_saver.py
+++ b/tensorflow/python/training/saving/functional_saver.py
@@ -107,25 +107,32 @@ class Saver(object):
       A scalar string Tensor containing `file_prefix` with control dependencies
       on the restore ops.
     """
-    restore_specs = []
-    tensor_structure = []
-    for saveable in self._saveable_objects:
-      saveable_tensor_structure = []
-      tensor_structure.append(saveable_tensor_structure)
-      for spec in saveable.specs:
-        saveable_tensor_structure.append(spec.name)
-        restore_specs.append((spec.name, spec.slice_spec, spec.dtype))
-    tensor_names, tensor_slices, tensor_dtypes = zip(*restore_specs)
-    with ops.device("cpu:0"):
-      restored_tensors = io_ops.restore_v2(
-          file_prefix, tensor_names, tensor_slices, tensor_dtypes)
-    structured_restored_tensors = nest.pack_sequence_as(
-        tensor_structure, restored_tensors)
-    restore_ops = []
-    for saveable, restored_tensors in zip(self._saveable_objects,
-                                          structured_restored_tensors):
-      restore_ops.append(saveable.restore(restored_tensors,
-                                          restored_shapes=None))
+    restore_ops = restore_from_saveable_objects(
+        file_prefix, self._saveable_objects)
     with ops.device("cpu:0"):
       with ops.control_dependencies(restore_ops):
         return array_ops.identity(file_prefix)
+
+
+def restore_from_saveable_objects(file_prefix, saveable_objects):
+  """Reads from a checkpoint and returns restore ops for `saveable_objects`s."""
+  restore_specs = []
+  tensor_structure = []
+  for saveable in saveable_objects:
+    saveable_tensor_structure = []
+    tensor_structure.append(saveable_tensor_structure)
+    for spec in saveable.specs:
+      saveable_tensor_structure.append(spec.name)
+      restore_specs.append((spec.name, spec.slice_spec, spec.dtype))
+  tensor_names, tensor_slices, tensor_dtypes = zip(*restore_specs)
+  with ops.device("cpu:0"):
+    restored_tensors = io_ops.restore_v2(
+        file_prefix, tensor_names, tensor_slices, tensor_dtypes)
+  structured_restored_tensors = nest.pack_sequence_as(
+      tensor_structure, restored_tensors)
+  restore_ops = []
+  for saveable, restored_tensors in zip(saveable_objects,
+                                        structured_restored_tensors):
+    restore_ops.append(saveable.restore(restored_tensors,
+                                        restored_shapes=None))
+  return restore_ops
-- 
GitLab


From 41e333f0193f89bc889a31f2a32b4119261aa222 Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Fri, 4 Jan 2019 14:10:40 -0800
Subject: [PATCH 0252/1095] [tf.data] Adding prefetching of input iterators for
 parallel interleave.

PiperOrigin-RevId: 227911701
---
 .../data/parallel_interleave_dataset_op.cc    | 752 +++++++++++-------
 .../data/kernel_tests/interleave_test.py      | 123 +--
 2 files changed, 511 insertions(+), 364 deletions(-)

diff --git a/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc b/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc
index 315f96d4b5..dc2663d1e0 100644
--- a/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc
+++ b/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc
@@ -14,6 +14,7 @@ limitations under the License.
 ==============================================================================*/
 #include <atomic>
 #include <deque>
+#include <memory>
 #include <utility>
 
 #include "tensorflow/core/common_runtime/function.h"
@@ -21,6 +22,7 @@ limitations under the License.
 #include "tensorflow/core/framework/partial_tensor_shape.h"
 #include "tensorflow/core/framework/stats_aggregator.h"
 #include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/kernels/data/captured_function.h"
 #include "tensorflow/core/kernels/data/dataset_utils.h"
 #include "tensorflow/core/lib/core/threadpool.h"
@@ -43,12 +45,7 @@ namespace {
 //
 // Furthermore, this class favors modularity over extended functionality. In
 // particular, it refrains from implementing configurable buffering of output
-// elements and prefetching of input iterators, relying on other parts of
-// tf.data to provide this functionality if necessary.
-//
-// The above design choices were made with automated optimizations in mind,
-// isolating the degree of parallelism as the single tunable knob of this
-// implementation.
+// elements and prefetching of input iterators.
 class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
  public:
   explicit ParallelInterleaveDatasetOp(OpKernelConstruction* ctx)
@@ -237,27 +234,20 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
       Status GetNextInternal(IteratorContext* ctx,
                              std::vector<Tensor>* out_tensors,
                              bool* end_of_sequence) override {
-        std::shared_ptr<InvocationResult> result;
-        do {
-          result.reset();
-          {
-            mutex_lock l(*mu_);
-            EnsureRunnerThreadStarted(ctx);
-            while (ShouldWait(&result)) {
-              RecordStop(ctx);
-              cond_var_->wait(l);
-              RecordStart(ctx);
-            }
-            if (!result) {
-              *end_of_sequence = true;
-              return Status::OK();
-            }
+        std::shared_ptr<Result> result;
+        {
+          mutex_lock l(*mu_);
+          EnsureThreadsStarted(ctx);
+          while (!Consume(&result)) {
+            RecordStop(ctx);
+            cond_var_->wait(l);
+            RecordStart(ctx);
           }
-          RecordStop(ctx);
-          result->notification.WaitForNotification();
-          RecordStart(ctx);
-        } while (result->skip);
-
+        }
+        if (!result) {
+          *end_of_sequence = true;
+          return Status::OK();
+        }
         if (result->status.ok()) {
           *out_tensors = std::move(result->return_values);
           RecordBufferDequeue(ctx, *out_tensors);
@@ -281,37 +271,22 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
         while (num_calls_ > 0) {
           cond_var_->wait(l);
         }
-        CHECK_EQ(num_calls_, 0);
+        DCHECK_EQ(num_calls_, 0);
         TF_RETURN_IF_ERROR(SaveInput(writer, input_impl_));
-        TF_RETURN_IF_ERROR(writer->WriteScalar(
-            full_name("invocation_results.size"), invocation_results_.size()));
-        for (size_t i = 0; i < invocation_results_.size(); i++) {
-          std::shared_ptr<InvocationResult> result = invocation_results_[i];
-          TF_RETURN_IF_ERROR(WriteStatusLocked(writer, i, result->status));
-          TF_RETURN_IF_ERROR(writer->WriteScalar(
-              full_name(strings::StrCat("invocation_results[", i, "].size")),
-              result->return_values.size()));
-          for (size_t j = 0; j < result->return_values.size(); j++) {
-            TF_RETURN_IF_ERROR(writer->WriteTensor(
-                full_name(
-                    strings::StrCat("invocation_results[", i, "][", j, "]")),
-                result->return_values[j]));
-          }
-          if (result->skip) {
-            TF_RETURN_IF_ERROR(writer->WriteScalar(
-                full_name(strings::StrCat("invocation_results[", i, "].skip")),
-                ""));
-          }
-        }
+        TF_RETURN_IF_ERROR(
+            writer->WriteScalar(full_name("block_index"), block_index_));
         TF_RETURN_IF_ERROR(
             writer->WriteScalar(full_name("cycle_index"), cycle_index_));
         if (end_of_input_) {
           TF_RETURN_IF_ERROR(
               writer->WriteScalar(full_name("end_of_input"), ""));
         }
+        TF_RETURN_IF_ERROR(writer->WriteScalar(full_name("element_id_counter"),
+                                               element_id_counter_));
         TF_RETURN_IF_ERROR(
             writer->WriteScalar(full_name("num_open"), num_open_));
         TF_RETURN_IF_ERROR(WriteCurrentElements(writer));
+        TF_RETURN_IF_ERROR(WriteFutureElements(writer));
         return Status::OK();
       }
 
@@ -319,114 +294,268 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
                              IteratorStateReader* reader) override {
         mutex_lock l(*mu_);
         TF_RETURN_IF_ERROR(RestoreInput(ctx, reader, input_impl_));
-        int64 invocation_results_size;
-        TF_RETURN_IF_ERROR(reader->ReadScalar(
-            full_name("invocation_results.size"), &invocation_results_size));
-        for (size_t i = 0; i < invocation_results_size; i++) {
-          std::shared_ptr<InvocationResult> result(new InvocationResult());
-          invocation_results_.push_back(result);
-          TF_RETURN_IF_ERROR(ReadStatusLocked(reader, i, &result->status));
-          size_t num_return_values;
-          {
-            int64 size;
-            TF_RETURN_IF_ERROR(reader->ReadScalar(
-                full_name(strings::StrCat("invocation_results[", i, "].size")),
-                &size));
-            num_return_values = static_cast<size_t>(size);
-            if (num_return_values != size) {
-              return errors::InvalidArgument(strings::StrCat(
-                  full_name(
-                      strings::StrCat("invocation_results[", i, "].size")),
-                  ": ", size, " is not a valid value of type size_t."));
-            }
-          }
-          result->return_values.reserve(num_return_values);
-          for (size_t j = 0; j < num_return_values; j++) {
-            result->return_values.emplace_back();
-            TF_RETURN_IF_ERROR(
-                reader->ReadTensor(full_name(strings::StrCat(
-                                       "invocation_results[", i, "][", j, "]")),
-                                   &result->return_values.back()));
-          }
-          result->skip = reader->Contains(
-              full_name(strings::StrCat("invocation_results[", i, "].skip")));
-          result->notification.Notify();
-        }
+        TF_RETURN_IF_ERROR(
+            reader->ReadScalar(full_name("block_index"), &block_index_));
         TF_RETURN_IF_ERROR(
             reader->ReadScalar(full_name("cycle_index"), &cycle_index_));
+        TF_RETURN_IF_ERROR(reader->ReadScalar(full_name("element_id_counter"),
+                                              &element_id_counter_));
         if (reader->Contains(full_name("end_of_input"))) end_of_input_ = true;
         TF_RETURN_IF_ERROR(
             reader->ReadScalar(full_name("num_open"), &num_open_));
         TF_RETURN_IF_ERROR(ReadCurrentElements(ctx, reader));
+        TF_RETURN_IF_ERROR(ReadFutureElements(ctx, reader));
         return Status::OK();
       }
 
      private:
+      // Represents the result of fetching an element from a dataset.
+      struct Result {
+        Status status;
+        std::vector<Tensor> return_values;
+        // Indicates whether the result is ready to be consumed.
+        bool is_ready = false;
+      };
+
+      // The interleave transformation repeatedly inputs elements, applies the
+      // user-provided function to transform the input elements to datasets, and
+      // interleaves the elements of these datasets as its output.
+      //
+      // This structure represents an input element and derived state.
       struct Element {
+        // Unique identifier, needed to support checkpointing.
+        int64 id;
+        // The actual input element.
+        std::vector<Tensor> inputs;
+        // Iterator created from the input element.
         std::unique_ptr<IteratorBase> iterator;
-        std::vector<Tensor> inputs;  // inputs for creating the iterator
-        bool in_use;
+        mutex mu;
+        // Buffer for storing the outputs of `iterator`.
+        std::deque<std::shared_ptr<Result>> results GUARDED_BY(mu);
+        // Indicates whether the element is used by a worker thread.
+        bool in_use = false;
       };
 
-      struct InvocationResult {
-        Notification notification;  // used for coordination with the consumer
-        Status status;              // the invocation status
-        std::vector<Tensor> return_values;  // the invocation result values
-        bool skip;  // if set the result should be skipped
-      };
+      // Advances the position in the interleave cycle to the next cycle
+      // element.
+      void AdvanceToNextInCycle() EXCLUSIVE_LOCKS_REQUIRED(*mu_) {
+        block_index_ = 0;
+        cycle_index_ = (cycle_index_ + 1) % dataset()->cycle_length_;
+      }
 
-      void EnsureRunnerThreadStarted(IteratorContext* ctx)
+      // Advances the position in the interleave cycle by one.
+      void AdvancePosition() EXCLUSIVE_LOCKS_REQUIRED(*mu_) {
+        ++block_index_;
+        if (block_index_ == dataset()->block_length_) {
+          AdvanceToNextInCycle();
+        }
+      }
+
+      // Consumes a result (if available), returning an indication of whether
+      // a result is available. If `true` is returned, `result` either
+      // points to a valid result or is null if end of input has been reached.
+      bool Consume(std::shared_ptr<Result>* result)
           EXCLUSIVE_LOCKS_REQUIRED(*mu_) {
-        if (!runner_thread_) {
-          std::shared_ptr<IteratorContext> new_ctx(new IteratorContext(*ctx));
-          runner_thread_.reset(ctx->env()->StartThread(
-              {}, "tf_data_parallel_interleave_runner",
-              [this, new_ctx]() { RunnerThread(new_ctx); }));
+        if (!sloppy_) {
+          return ConsumeHelper(result);
+        }
+        // If we are allowed to be sloppy (i.e. return results out of order),
+        // try to find an element in the cycle that has a result available.
+        for (int i = 0; i < dataset()->cycle_length_; ++i) {
+          if (ConsumeHelper(result)) {
+            return true;
+          }
+          AdvanceToNextInCycle();
+        }
+        return false;
+      }
+
+      bool ConsumeHelper(std::shared_ptr<Result>* result)
+          EXCLUSIVE_LOCKS_REQUIRED(*mu_) {
+        while (true) {
+          std::shared_ptr<Element> element = current_elements_[cycle_index_];
+          if (element) {
+            mutex_lock l(element->mu);
+            if (!element->results.empty()) {
+              if (element->results.front()->is_ready) {
+                // We found a result.
+                std::swap(*result, element->results.front());
+                element->results.pop_front();
+                AdvancePosition();
+                cond_var_->notify_all();
+                return true;
+              } else {
+                // Wait for the result to become ready.
+                return false;
+              }
+            } else if (!element->iterator) {
+              // We reached the end of input for this element. Reset
+              // it and move on to the next cycle element.
+              current_elements_[cycle_index_].reset();
+              AdvanceToNextInCycle();
+              cond_var_->notify_all();
+              continue;
+            } else {
+              // Wait for the iterator to produce a result.
+              return false;
+            }
+          } else {
+            if (!future_elements_.empty() || !end_of_input_) {
+              // Wait for an element to be created.
+              return false;
+            }
+            // No new elements will be created; try to find a
+            // non-empty element in the cycle.
+            for (int i = 0; i < dataset()->cycle_length_; ++i) {
+              AdvanceToNextInCycle();
+              if (current_elements_[cycle_index_]) {
+                break;
+              }
+            }
+            if (current_elements_[cycle_index_]) {
+              continue;
+            }
+            // End of input has been reached.
+            return true;
+          }
         }
       }
 
-      // Fetches up to `results.size()` outputs from the cycle element at
-      // position `cycle_index`.
+      // Manages current cycle elements, creating new iterators as needed and
+      // asynchronously fetching results from existing iterators.
       //
-      // If end of input is encountered, the `skip` field of the invocation
-      // result is used to identify results that should be skipped.
-      void FetchOutputs(
-          const std::shared_ptr<IteratorContext>& ctx, IteratorBase* iterator,
-          int64 cycle_index,
-          const std::vector<std::shared_ptr<InvocationResult>>& results)
-          LOCKS_EXCLUDED(*mu_) {
+      // This method runs in the `current_elements_manager_` background thread.
+      void CurrentElementsManager(const std::shared_ptr<IteratorContext>& ctx) {
         RecordStart(ctx.get());
         auto cleanup = gtl::MakeCleanup([this, ctx] { RecordStop(ctx.get()); });
-        bool end_of_input = false;
-        for (auto& result : results) {
-          if (!end_of_input) {
-            result->status = iterator->GetNext(
-                ctx.get(), &result->return_values, &end_of_input);
+        auto busy = [this]() EXCLUSIVE_LOCKS_REQUIRED(*mu_) -> bool {
+          const bool has_more_elements =
+              !future_elements_.empty() || !end_of_input_;
+          const int block_length = dataset()->block_length_;
+          bool all_elements_busy = true;
+          for (auto& element : current_elements_) {
+            if (!element) {
+              if (has_more_elements) {
+                all_elements_busy = false;
+                break;
+              }
+            } else {
+              mutex_lock l(element->mu);
+              if (!element->in_use && element->iterator &&
+                  element->results.size() < block_length) {
+                all_elements_busy = false;
+                break;
+              }
+            }
           }
-          if (end_of_input) {
-            result->skip = true;
+          return all_elements_busy || num_calls_ >= num_parallel_calls_->value;
+        };
+        while (true) {
+          mutex_lock l(*mu_);
+
+          // Wait until this thread is cancelled, the end of input has been
+          // reached.
+          while (!cancelled_ && (!end_of_input_ || num_open_ > 0) && busy()) {
+            RecordStop(ctx.get());
+            cond_var_->wait(l);
+            RecordStart(ctx.get());
           }
-          RecordBufferEnqueue(ctx.get(), result->return_values);
-          {
-            mutex_lock l(*mu_);
-            result->notification.Notify();
-            cond_var_->notify_all();
+
+          if (cancelled_ ||
+              (future_elements_.empty() && end_of_input_ && num_open_ == 0)) {
+            return;
+          }
+
+          for (int i = 0; i < dataset()->cycle_length_; ++i) {
+            int idx = (cycle_index_ + i) % dataset()->cycle_length_;
+            if (!current_elements_[idx]) {
+              if (!future_elements_.empty()) {
+                current_elements_[idx] = std::move(future_elements_.back());
+                future_elements_.pop_back();
+              } else {
+                current_elements_[idx] = MakeElement(ctx);
+                if (!current_elements_[idx]) {
+                  continue;
+                }
+              }
+            }
+            std::shared_ptr<Element> element = current_elements_[idx];
+            if (!element->in_use && element->iterator) {
+              int64 num_results;
+              {
+                mutex_lock l(element->mu);
+                num_results =
+                    dataset()->block_length_ - element->results.size();
+              }
+              if (num_results > 0) {
+                num_calls_++;
+                element->in_use = true;
+                thread_pool_->Schedule(
+                    std::bind(&ParallelInterleaveIterator::FetchResults, this,
+                              ctx, std::move(element), num_results));
+              }
+            }
+          }
+          const auto& stats_aggregator = ctx->stats_aggregator();
+          if (stats_aggregator) {
+            stats_aggregator->AddScalar(
+                strings::StrCat(key_prefix_, "::thread_utilization"),
+                static_cast<float>(num_calls_) /
+                    static_cast<float>(num_parallel_calls_->value));
           }
-          if (!result->status.ok()) {
+          cond_var_->notify_all();
+        }
+      }
+
+      void EnsureThreadsStarted(IteratorContext* ctx)
+          EXCLUSIVE_LOCKS_REQUIRED(*mu_) {
+        if (!current_elements_manager_) {
+          auto new_ctx = std::make_shared<IteratorContext>(*ctx);
+          current_elements_manager_ =
+              WrapUnique<Thread>(ctx->env()->StartThread(
+                  {}, "tf_data_parallel_interleave_current",
+                  [this, new_ctx]() { CurrentElementsManager(new_ctx); }));
+        }
+        if (!future_elements_manager_) {
+          auto new_ctx = std::make_shared<IteratorContext>(*ctx);
+          future_elements_manager_ = WrapUnique<Thread>(ctx->env()->StartThread(
+              {}, "tf_data_parallel_interleave_future",
+              [this, new_ctx]() { FutureElementsManager(new_ctx); }));
+        }
+      }
+
+      // Fetches up to `dataset()->block_length_` results from `element`.
+      void FetchResults(const std::shared_ptr<IteratorContext>& ctx,
+                        const std::shared_ptr<Element>& element,
+                        int64 num_results) LOCKS_EXCLUDED(*mu_) {
+        RecordStart(ctx.get());
+        auto cleanup = gtl::MakeCleanup([this, ctx] { RecordStop(ctx.get()); });
+        bool end_of_input = false;
+        for (int64 i = 0; i < num_results; ++i) {
+          auto result = std::make_shared<Result>();
+          result->status = element->iterator->GetNext(
+              ctx.get(), &result->return_values, &end_of_input);
+          if (end_of_input) {
             break;
           }
+          RecordBufferEnqueue(ctx.get(), result->return_values);
+          mutex_lock l(*mu_);
+          mutex_lock l2(element->mu);
+          element->results.push_back(result);
+          result->is_ready = true;
+          cond_var_->notify_all();
         }
 
         mutex_lock l(*mu_);
-        current_elements_[cycle_index].in_use = false;
+        // Release the ownership of the cycle element iterator.
+        element->in_use = false;
         if (end_of_input) {
-          // Release the ownership of the cycle element iterator, closing the
-          // iterator if end of input was encountered.
-          current_elements_[cycle_index].iterator.reset();
-          current_elements_[cycle_index].inputs.clear();
-          num_open_--;
+          // Close the iterator if end of input was encountered.
+          element->iterator.reset();
+          element->inputs.clear();
+          --num_open_;
         }
-        num_calls_--;
+        --num_calls_;
         const auto& stats_aggregator = ctx->stats_aggregator();
         if (stats_aggregator) {
           stats_aggregator->AddScalar(
@@ -437,82 +566,47 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
         cond_var_->notify_all();
       }
 
-      // Method responsible for 1) creating iterators out of input elements, 2)
-      // determining the order in which elements are fetched from the iterators,
-      // and 3) scheduling the fetching of the elements to a threadpool.
+      // Manages futures cycle elements, creating new iterators as needed and
+      // asynchronously fetching results from existing iterators.
       //
-      // This method runs in the `runner_thread` background thread.
-      void RunnerThread(const std::shared_ptr<IteratorContext>& ctx) {
+      // This method runs in the `future_elements_manager_` background thread.
+      void FutureElementsManager(const std::shared_ptr<IteratorContext>& ctx) {
         RecordStart(ctx.get());
         auto cleanup = gtl::MakeCleanup([this, ctx] { RecordStop(ctx.get()); });
         auto busy = [this]() EXCLUSIVE_LOCKS_REQUIRED(*mu_) -> bool {
-          return current_elements_[cycle_index_].in_use ||
-                 num_calls_ >= num_parallel_calls_->value ||
-                 invocation_results_.size() >=
-                     dataset()->cycle_length_ * dataset()->block_length_;
+          return num_calls_ >= num_parallel_calls_->value ||
+                 future_elements_.size() >= dataset()->cycle_length_;
         };
         while (true) {
           mutex_lock l(*mu_);
+
           // Wait until this thread is cancelled, the end of input has been
           // reached, or the cycle element at the `cycle_index_` position is
-          // not in use and there is space in the `invocation_results_` queue.
-          while (!cancelled_ && (!end_of_input_ || num_open_ > 0) && busy()) {
+          // not in use.
+          while (!cancelled_ && !end_of_input_ && busy()) {
             RecordStop(ctx.get());
             cond_var_->wait(l);
             RecordStart(ctx.get());
           }
 
-          if (cancelled_ || (end_of_input_ && num_open_ == 0)) {
+          if (cancelled_ || end_of_input_) {
             return;
           }
 
-          while ((!end_of_input_ || num_open_ > 0) && !busy()) {
-            if (!current_elements_[cycle_index_].iterator) {
-              // Try to create a new iterator from the next input element.
-              Status status = input_impl_->GetNext(
-                  ctx.get(), &current_elements_[cycle_index_].inputs,
-                  &end_of_input_);
-              if (!status.ok()) {
-                invocation_results_.emplace_back(new InvocationResult());
-                std::shared_ptr<InvocationResult>& result =
-                    invocation_results_.back();
-                result->status.Update(status);
-                result->notification.Notify();
-                break;
-              }
-              if (!end_of_input_) {
-                Status status = MakeIteratorFromInputElement(
-                    ctx.get(), current_elements_[cycle_index_].inputs,
-                    cycle_index_, *instantiated_captured_func_, prefix(),
-                    &current_elements_[cycle_index_].iterator);
-                if (!status.ok()) {
-                  invocation_results_.emplace_back(new InvocationResult());
-                  std::shared_ptr<InvocationResult>& result =
-                      invocation_results_.back();
-                  result->status.Update(status);
-                  result->notification.Notify();
-                  break;
-                }
-                ++num_open_;
-              }
+          while (!end_of_input_ && !busy()) {
+            std::shared_ptr<Element> element = MakeElement(ctx);
+            if (!element) {
+              break;
             }
-            if (current_elements_[cycle_index_].iterator) {
-              // Pre-allocate invocation results for outputs to be fetched
-              // and then fetch the outputs asynchronously.
-              std::vector<std::shared_ptr<InvocationResult>> results;
-              results.reserve(dataset()->block_length_);
-              for (int i = 0; i < dataset()->block_length_; ++i) {
-                invocation_results_.emplace_back(new InvocationResult());
-                results.push_back(invocation_results_.back());
-              }
-              num_calls_++;
-              current_elements_[cycle_index_].in_use = true;
-              thread_pool_->Schedule(
-                  std::bind(&ParallelInterleaveIterator::FetchOutputs, this,
-                            ctx, current_elements_[cycle_index_].iterator.get(),
-                            cycle_index_, std::move(results)));
+            future_elements_.push_front(element);
+            if (!element->iterator) {
+              continue;
             }
-            cycle_index_ = (cycle_index_ + 1) % dataset()->cycle_length_;
+            ++num_calls_;
+            element->in_use = true;
+            thread_pool_->Schedule(
+                std::bind(&ParallelInterleaveIterator::FetchResults, this, ctx,
+                          std::move(element), dataset()->block_length_));
           }
           const auto& stats_aggregator = ctx->stats_aggregator();
           if (stats_aggregator) {
@@ -525,56 +619,66 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
         }
       }
 
-      // Determines whether the caller needs to wait for a result. Upon
-      // returning false, `result` will either be NULL if end of input has been
-      // reached or point to the result.
-      bool ShouldWait(std::shared_ptr<InvocationResult>* result)
+      // Creates a new element.
+      std::shared_ptr<Element> MakeElement(
+          const std::shared_ptr<IteratorContext>& ctx)
           EXCLUSIVE_LOCKS_REQUIRED(*mu_) {
-        if (sloppy_) {
-          for (auto it = invocation_results_.begin();
-               it != invocation_results_.end(); ++it) {
-            if ((*it)->notification.HasBeenNotified()) {
-              std::swap(*result, *it);
-              invocation_results_.erase(it);
-              cond_var_->notify_all();
-              return false;
-            }
+        auto element = std::make_shared<Element>();
+        element->id = element_id_counter_++;
+        Status status =
+            input_impl_->GetNext(ctx.get(), &element->inputs, &end_of_input_);
+        if (!status.ok()) {
+          auto result = std::make_shared<Result>();
+          result->is_ready = true;
+          result->status = status;
+          mutex_lock l(element->mu);
+          element->results.push_back(std::move(result));
+          return element;
+        }
+        if (!end_of_input_) {
+          Status status = MakeIteratorFromInputElement(
+              ctx.get(), element->inputs, element->id,
+              *instantiated_captured_func_, prefix(), &element->iterator);
+          if (!status.ok()) {
+            auto result = std::make_shared<Result>();
+            result->is_ready = true;
+            result->status = status;
+            mutex_lock l(element->mu);
+            element->results.push_back(std::move(result));
+            return element;
           }
-          return !invocation_results_.empty() ||
-                 (!end_of_input_ || num_open_ > 0);
+          ++num_open_;
         } else {
-          if (!invocation_results_.empty()) {
-            std::swap(*result, invocation_results_.front());
-            invocation_results_.pop_front();
-            cond_var_->notify_all();
-            return false;
-          }
-          return (!end_of_input_ || num_open_ > 0);
+          element.reset();
         }
+        return element;
       }
 
-      Status WriteStatusLocked(IteratorStateWriter* writer, size_t index,
+      Status WriteStatusLocked(IteratorStateWriter* writer,
+                               const string& key_prefix, size_t idx,
                                const Status& status)
           EXCLUSIVE_LOCKS_REQUIRED(*mu_) {
         TF_RETURN_IF_ERROR(writer->WriteScalar(
-            CodeKey(index), static_cast<int64>(status.code())));
+            CodeKey(key_prefix, idx), static_cast<int64>(status.code())));
         if (!status.ok()) {
-          TF_RETURN_IF_ERROR(writer->WriteScalar(ErrorMessageKey(index),
-                                                 status.error_message()));
+          TF_RETURN_IF_ERROR(writer->WriteScalar(
+              ErrorMessageKey(key_prefix, idx), status.error_message()));
         }
         return Status::OK();
       }
 
-      Status ReadStatusLocked(IteratorStateReader* reader, size_t index,
+      Status ReadStatusLocked(IteratorStateReader* reader,
+                              const string& key_prefix, size_t idx,
                               Status* status) EXCLUSIVE_LOCKS_REQUIRED(*mu_) {
         int64 code_int;
-        TF_RETURN_IF_ERROR(reader->ReadScalar(CodeKey(index), &code_int));
+        TF_RETURN_IF_ERROR(
+            reader->ReadScalar(CodeKey(key_prefix, idx), &code_int));
         error::Code code = static_cast<error::Code>(code_int);
 
         if (code != error::Code::OK) {
           string error_message;
-          TF_RETURN_IF_ERROR(
-              reader->ReadScalar(ErrorMessageKey(index), &error_message));
+          TF_RETURN_IF_ERROR(reader->ReadScalar(
+              ErrorMessageKey(key_prefix, idx), &error_message));
           *status = Status(code, error_message);
         } else {
           *status = Status::OK();
@@ -582,64 +686,178 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
         return Status::OK();
       }
 
-      string CodeKey(size_t index) {
+      string CodeKey(const string& key_prefix, size_t idx) {
         return full_name(
-            strings::StrCat("invocation_results[", index, "].code"));
+            strings::StrCat(key_prefix, ".results[", idx, "].code"));
       }
 
-      string ErrorMessageKey(size_t index) {
+      string ErrorMessageKey(const string& key_prefix, size_t idx) {
         return full_name(
-            strings::StrCat("invocation_results[", index, "].error_message"));
+            strings::StrCat(key_prefix, ".results[", idx, "].error_message"));
+      }
+
+      Status WriteElement(std::shared_ptr<Element> element, int idx,
+                          const string& key_prefix, IteratorStateWriter* writer)
+          EXCLUSIVE_LOCKS_REQUIRED(*mu_) {
+        if (element->iterator) {
+          TF_RETURN_IF_ERROR(SaveInput(writer, element->iterator));
+          TF_RETURN_IF_ERROR(writer->WriteScalar(
+              full_name(strings::StrCat(key_prefix, "[", idx, "].id")),
+              element->id));
+          TF_RETURN_IF_ERROR(writer->WriteScalar(
+              full_name(strings::StrCat(key_prefix, "[", idx, "].inputs.size")),
+              element->inputs.size()));
+          for (int i = 0; i < element->inputs.size(); i++) {
+            TF_RETURN_IF_ERROR(writer->WriteTensor(
+                full_name(
+                    strings::StrCat(key_prefix, "[", idx, "].inputs[", i, "]")),
+                element->inputs[i]));
+          }
+        }
+        mutex_lock l(element->mu);
+        TF_RETURN_IF_ERROR(writer->WriteScalar(
+            full_name(strings::StrCat(key_prefix, "[", idx, "].results.size")),
+            element->results.size()));
+        for (size_t i = 0; i < element->results.size(); i++) {
+          std::shared_ptr<Result> result = element->results[i];
+          TF_RETURN_IF_ERROR(WriteStatusLocked(
+              writer, strings::StrCat(key_prefix, "[", idx, "]"), i,
+              result->status));
+          TF_RETURN_IF_ERROR(writer->WriteScalar(
+              full_name(strings::StrCat(key_prefix, "[", idx, "].results[", i,
+                                        "].size")),
+              result->return_values.size()));
+          for (size_t j = 0; j < result->return_values.size(); j++) {
+            TF_RETURN_IF_ERROR(writer->WriteTensor(
+                full_name(strings::StrCat(key_prefix, "[", idx, "].results[", i,
+                                          "][", j, "]")),
+                result->return_values[j]));
+          }
+          if (result->is_ready) {
+            TF_RETURN_IF_ERROR(writer->WriteScalar(
+                full_name(strings::StrCat(key_prefix, "[", idx, "].results[", i,
+                                          "].is_ready")),
+                ""));
+          }
+        }
+        return Status::OK();
       }
 
       Status WriteCurrentElements(IteratorStateWriter* writer)
           EXCLUSIVE_LOCKS_REQUIRED(*mu_) {
+        TF_RETURN_IF_ERROR(writer->WriteScalar(
+            full_name("current_elements.size"), current_elements_.size()));
         for (int idx = 0; idx < current_elements_.size(); idx++) {
-          if (current_elements_[idx].iterator) {
-            TF_RETURN_IF_ERROR(
-                SaveInput(writer, current_elements_[idx].iterator));
-            TF_RETURN_IF_ERROR(writer->WriteScalar(
-                full_name(
-                    strings::StrCat("current_elements[", idx, "].inputs.size")),
-                current_elements_[idx].inputs.size()));
-            for (int i = 0; i < current_elements_[idx].inputs.size(); i++) {
-              TF_RETURN_IF_ERROR(writer->WriteTensor(
-                  full_name(strings::StrCat("current_elements[", idx,
-                                            "].inputs[", i, "]")),
-                  current_elements_[idx].inputs[i]));
-            }
+          if (current_elements_[idx]) {
+            TF_RETURN_IF_ERROR(WriteElement(current_elements_[idx], idx,
+                                            "current_elements", writer));
           }
         }
         return Status::OK();
       }
 
+      Status WriteFutureElements(IteratorStateWriter* writer)
+          EXCLUSIVE_LOCKS_REQUIRED(*mu_) {
+        TF_RETURN_IF_ERROR(writer->WriteScalar(
+            full_name("future_elements.size"), future_elements_.size()));
+        for (int idx = 0; idx < future_elements_.size(); idx++) {
+          if (future_elements_[idx]) {
+            TF_RETURN_IF_ERROR(WriteElement(future_elements_[idx], idx,
+                                            "future_elements", writer));
+          }
+        }
+        return Status::OK();
+      }
+
+      Status ReadElement(IteratorContext* ctx, IteratorStateReader* reader,
+                         int idx, const string& key_prefix,
+                         std::shared_ptr<Element>* out)
+          EXCLUSIVE_LOCKS_REQUIRED(*mu_) {
+        if (!reader->Contains(full_name(
+                strings::StrCat(key_prefix, "[", idx, "].results.size")))) {
+          return Status::OK();
+        }
+        auto element = std::make_shared<Element>();
+        mutex_lock l(element->mu);
+        int64 results_size;
+        TF_RETURN_IF_ERROR(reader->ReadScalar(
+            full_name(strings::StrCat(key_prefix, "[", idx, "].results.size")),
+            &results_size));
+        element->results.resize(results_size);
+        for (size_t i = 0; i < results_size; i++) {
+          auto result = std::make_shared<Result>();
+          TF_RETURN_IF_ERROR(ReadStatusLocked(
+              reader, strings::StrCat(key_prefix, "[", idx, "]"), i,
+              &result->status));
+          int64 num_return_values;
+          TF_RETURN_IF_ERROR(reader->ReadScalar(
+              full_name(strings::StrCat(key_prefix, "[", idx, "].results[", i,
+                                        "].size")),
+              &num_return_values));
+          result->return_values.reserve(num_return_values);
+          for (size_t j = 0; j < num_return_values; j++) {
+            result->return_values.emplace_back();
+            TF_RETURN_IF_ERROR(reader->ReadTensor(
+                full_name(strings::StrCat(key_prefix, "[", idx, "].results[", i,
+                                          "][", j, "]")),
+                &result->return_values.back()));
+          }
+          result->is_ready = reader->Contains(full_name(strings::StrCat(
+              key_prefix, "[", idx, "].results[", i, "].is_ready")));
+          element->results[i] = std::move(result);
+        }
+        if (!reader->Contains(full_name(
+                strings::StrCat(key_prefix, "[", idx, "].inputs.size")))) {
+          element->iterator.reset();
+          *out = std::move(element);
+          return Status::OK();
+        }
+        int64 inputs_size;
+        TF_RETURN_IF_ERROR(reader->ReadScalar(
+            full_name(strings::StrCat(key_prefix, "[", idx, "].inputs.size")),
+            &inputs_size));
+        element->inputs.resize(inputs_size);
+        for (int i = 0; i < inputs_size; i++) {
+          TF_RETURN_IF_ERROR(reader->ReadTensor(
+              full_name(
+                  strings::StrCat(key_prefix, "[", idx, "].inputs[", i, "]")),
+              &element->inputs[i]));
+        }
+        TF_RETURN_IF_ERROR(reader->ReadScalar(
+            full_name(strings::StrCat(key_prefix, "[", idx, "].id")),
+            &element->id));
+        TF_RETURN_IF_ERROR(MakeIteratorFromInputElement(
+            ctx, element->inputs, element->id,
+            *instantiated_captured_func_.get(), prefix(), &element->iterator));
+        TF_RETURN_IF_ERROR(RestoreInput(ctx, reader, element->iterator));
+        *out = std::move(element);
+        return Status::OK();
+      }
+
       Status ReadCurrentElements(IteratorContext* ctx,
                                  IteratorStateReader* reader)
           EXCLUSIVE_LOCKS_REQUIRED(*mu_) {
+        int64 size;
+        TF_RETURN_IF_ERROR(
+            reader->ReadScalar(full_name("current_elements.size"), &size));
+        DCHECK_EQ(current_elements_.size(), size);
         for (int idx = 0; idx < current_elements_.size(); idx++) {
-          if (reader->Contains(full_name(strings::StrCat(
-                  "current_elements[", idx, "].inputs.size")))) {
-            int64 inputs_size;
-            TF_RETURN_IF_ERROR(reader->ReadScalar(
-                full_name(
-                    strings::StrCat("current_elements[", idx, "].inputs.size")),
-                &inputs_size));
-            current_elements_[idx].inputs.resize(inputs_size);
-            for (int i = 0; i < inputs_size; i++) {
-              TF_RETURN_IF_ERROR(reader->ReadTensor(
-                  full_name(strings::StrCat("current_elements[", idx,
-                                            "].inputs[", i, "]")),
-                  &current_elements_[idx].inputs[i]));
-            }
-            TF_RETURN_IF_ERROR(MakeIteratorFromInputElement(
-                ctx, current_elements_[idx].inputs, idx,
-                *instantiated_captured_func_.get(), prefix(),
-                &current_elements_[idx].iterator));
-            TF_RETURN_IF_ERROR(
-                RestoreInput(ctx, reader, current_elements_[idx].iterator));
-          } else {
-            current_elements_[idx].iterator.reset();
-          }
+          TF_RETURN_IF_ERROR(ReadElement(ctx, reader, idx, "current_elements",
+                                         &current_elements_[idx]));
+        }
+        return Status::OK();
+      }
+
+      Status ReadFutureElements(IteratorContext* ctx,
+                                IteratorStateReader* reader)
+          EXCLUSIVE_LOCKS_REQUIRED(*mu_) {
+        int64 size;
+        TF_RETURN_IF_ERROR(
+            reader->ReadScalar(full_name("future_elements.size"), &size));
+        future_elements_.resize(size);
+        for (int idx = 0; idx < future_elements_.size(); idx++) {
+          TF_RETURN_IF_ERROR(ReadElement(ctx, reader, idx, "future_elements",
+                                         &future_elements_[idx]));
         }
         return Status::OK();
       }
@@ -648,12 +866,11 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
       // the worker threads.
       const std::shared_ptr<mutex> mu_;
 
-      // Used for coordination between the main thread, the runner thread, and
-      // the worker threads. In particular, the runner thread should only
-      // schedule new calls when the number of in-flight calls is less than the
-      // user specified level of parallelism, there are slots available in the
-      // `invocation_results_` buffer, the current cycle element is not in use,
-      // and there are elements left to be fetched.
+      // Used for coordination between the main thread, the manager threads, and
+      // the threadpool threads. In particular, the managers thread should only
+      // schedule new calls into the threadpool when the number of in-flight
+      // calls is less than the user specified level of parallelism and there
+      // are slots available in the element `results` buffer.
       const std::shared_ptr<condition_variable> cond_var_;
 
       // Identifies the maximum number of parallel calls.
@@ -665,18 +882,17 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
       // Iterator for input elements.
       std::unique_ptr<IteratorBase> input_impl_ GUARDED_BY(*mu_);
 
-      // Identifies current cycle element.
-      int64 cycle_index_ = 0;
+      // Identifies position in the interleave cycle.
+      int64 block_index_ GUARDED_BY(*mu_) = 0;
+      int64 cycle_index_ GUARDED_BY(*mu_) = 0;
 
-      // Iterators for the current cycle elements. Concurrent access is
-      // protected by `element_in_use_`.
-      std::vector<Element> current_elements_ GUARDED_BY(*mu_);
+      // Elements of the current interleave cycle.
+      std::vector<std::shared_ptr<Element>> current_elements_ GUARDED_BY(*mu_);
 
-      // Buffer for storing the invocation results.
-      std::deque<std::shared_ptr<InvocationResult>> invocation_results_
-          GUARDED_BY(*mu_);
+      // Elements to be used in the interleave cycle in the future.
+      std::deque<std::shared_ptr<Element>> future_elements_ GUARDED_BY(*mu_);
 
-      // Identifies whether end of input has been reached.
+      // Identifies whether the global end of input has been reached.
       bool end_of_input_ GUARDED_BY(*mu_) = false;
 
       // Identifies the number of open iterators.
@@ -686,9 +902,11 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
       int64 num_calls_ GUARDED_BY(*mu_) = 0;
 
       std::unique_ptr<thread::ThreadPool> thread_pool_;
-      std::unique_ptr<Thread> runner_thread_ GUARDED_BY(*mu_);
+      std::unique_ptr<Thread> current_elements_manager_ GUARDED_BY(*mu_);
+      std::unique_ptr<Thread> future_elements_manager_ GUARDED_BY(*mu_);
+      int64 element_id_counter_ GUARDED_BY(*mu_) = 0;
 
-      // Identifies whether background activity should be cancelled.
+      // Identifies whether background threads should be cancelled.
       bool cancelled_ GUARDED_BY(*mu_) = false;
       string key_prefix_;
       std::unique_ptr<InstantiatedCapturedFunction> instantiated_captured_func_;
diff --git a/tensorflow/python/data/kernel_tests/interleave_test.py b/tensorflow/python/data/kernel_tests/interleave_test.py
index 4fb61b2daf..4b427ff5a4 100644
--- a/tensorflow/python/data/kernel_tests/interleave_test.py
+++ b/tensorflow/python/data/kernel_tests/interleave_test.py
@@ -17,19 +17,15 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import threading
-
 from absl.testing import parameterized
 import numpy as np
 
-from tensorflow.python.data.experimental.ops import threading_options
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import script_ops
 from tensorflow.python.ops import sparse_ops
 from tensorflow.python.platform import test
 
@@ -78,47 +74,6 @@ def _interleave(lists, cycle_length, block_length):
           break
 
 
-def _make_coordinated_sloppy_dataset(input_values, cycle_length, block_length,
-                                     num_parallel_calls):
-  """Produces a dataset iterator and events to control the order of elements.
-
-  Args:
-    input_values: the values to generate lists to interleave from
-    cycle_length: the length of the interleave cycle
-    block_length: the length of the interleave block
-    num_parallel_calls: the degree of interleave parallelism
-
-  Returns:
-    A dataset iterator (represented as `get_next` op) and events that can be
-    used to control the order of output elements.
-  """
-
-  # Set up threading events used to sequence when items are produced that
-  # are subsequently interleaved. These events allow us to deterministically
-  # simulate slowdowns and force sloppiness.
-  coordination_events = {i: threading.Event() for i in input_values}
-
-  def map_py_fn(x):
-    coordination_events[x].wait()
-    coordination_events[x].clear()
-    return x * x
-
-  def map_fn(x):
-    return script_ops.py_func(map_py_fn, [x], x.dtype)
-
-  def interleave_fn(x):
-    dataset = dataset_ops.Dataset.from_tensors(x)
-    dataset = dataset.repeat(x)
-    return dataset.map(map_fn)
-
-  options = dataset_ops.Options()
-  options.experimental_deterministic = False
-  dataset = dataset_ops.Dataset.from_tensor_slices(input_values).repeat(
-      2).interleave(interleave_fn, cycle_length, block_length,
-                    num_parallel_calls).with_options(options)
-  return dataset, coordination_events
-
-
 def _repeat(values, count):
   """Produces a list of lists suitable for testing interleave.
 
@@ -252,63 +207,37 @@ class InterleaveTest(test_base.DatasetTestBase, parameterized.TestCase):
       self.evaluate(get_next())
 
   @parameterized.named_parameters(
-      ("1", np.int64([4, 5, 6]), 2, 1, 1),
-      ("2", np.int64([4, 5, 6]), 2, 1, 2),
-      ("3", np.int64([4, 5, 6]), 2, 3, 1),
-      ("4", np.int64([4, 5, 6]), 2, 3, 2),
-      ("5", np.int64([4, 5, 6]), 3, 2, 1),
-      ("6", np.int64([4, 5, 6]), 3, 2, 2),
-      ("7", np.int64([4, 5, 6]), 3, 2, 3),
-      ("8", np.int64([4, 0, 6]), 2, 3, 1),
-      ("9", np.int64([4, 0, 6]), 2, 3, 2),
+      ("1", np.int64([4, 5, 6]), 1, 3, 1),
+      ("2", np.int64([4, 5, 6]), 2, 1, 1),
+      ("3", np.int64([4, 5, 6]), 2, 1, 2),
+      ("4", np.int64([4, 5, 6]), 2, 3, 1),
+      ("5", np.int64([4, 5, 6]), 2, 3, 2),
+      ("6", np.int64([4, 5, 6]), 7, 2, 1),
+      ("7", np.int64([4, 5, 6]), 7, 2, 3),
+      ("8", np.int64([4, 5, 6]), 7, 2, 5),
+      ("9", np.int64([4, 5, 6]), 7, 2, 7),
+      ("10", np.int64([4, 0, 6]), 2, 3, 1),
+      ("11", np.int64([4, 0, 6]), 2, 3, 2),
   )
-  def testSloppyInterleaveInOrder(self, input_values, cycle_length,
+  def testSloppyInterleaveDataset(self, input_values, cycle_length,
                                   block_length, num_parallel_calls):
-    dataset, coordination_events = _make_coordinated_sloppy_dataset(
-        input_values, cycle_length, block_length, num_parallel_calls)
-    options = dataset_ops.Options()
-    options.experimental_threading = threading_options.ThreadingOptions()
-    options.experimental_threading.private_threadpool_size = (
-        num_parallel_calls + 1)
-    dataset = dataset.with_options(options)
-
-    get_next = self.getNext(dataset, requires_initialization=True)
-    for expected_element in _interleave(
-        _repeat(input_values, 2), cycle_length, block_length):
-      coordination_events[expected_element].set()
-      self.assertEqual(expected_element * expected_element,
-                       self.evaluate(get_next()))
-    with self.assertRaises(errors.OutOfRangeError):
-      self.evaluate(get_next())
-
-  @parameterized.named_parameters(
-      ("1", np.int64([4, 5, 6]), 2, 1, 2),
-      ("2", np.int64([4, 5, 6]), 2, 3, 2),
-      ("3", np.int64([4, 5, 6]), 3, 2, 3),
-      ("4", np.int64([4, 0, 6]), 2, 3, 2),
-  )
-  def testSloppyInterleaveOutOfOrder(self, input_values, cycle_length,
-                                     block_length, num_parallel_calls):
-    dataset, coordination_events = _make_coordinated_sloppy_dataset(
-        input_values, cycle_length, block_length, num_parallel_calls)
+    count = 2
+    dataset = dataset_ops.Dataset.from_tensor_slices(input_values).repeat(
+        count).interleave(
+            lambda x: dataset_ops.Dataset.from_tensors(x).repeat(x),
+            cycle_length, block_length, num_parallel_calls)
     options = dataset_ops.Options()
-    options.experimental_threading = threading_options.ThreadingOptions()
-    options.experimental_threading.private_threadpool_size = (
-        num_parallel_calls + 1)
+    options.experimental_deterministic = False
     dataset = dataset.with_options(options)
-    get_next = self.getNext(dataset, requires_initialization=True)
-    elements = [
-        x for x in _interleave(
-            _repeat(input_values, 2), cycle_length, block_length)
+    expected_output = [
+        element for element in _interleave(
+            _repeat(input_values, count), cycle_length, block_length)
     ]
-    for i in [1, 4, 7]:
-      elements[i], elements[i + 1] = elements[i + 1], elements[i]
-
-    for element in elements:
-      coordination_events[element].set()
-      self.assertEqual(element * element, self.evaluate(get_next()))
-    with self.assertRaises(errors.OutOfRangeError):
-      self.evaluate(get_next())
+    get_next = self.getNext(dataset)
+    actual_output = []
+    for _ in range(len(expected_output)):
+      actual_output.append(self.evaluate(get_next()))
+    self.assertAllEqual(expected_output.sort(), actual_output.sort())
 
 
 if __name__ == "__main__":
-- 
GitLab


From dba2134638812e6375a26012a485b55a24c45a19 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 4 Jan 2019 14:26:00 -0800
Subject: [PATCH 0253/1095] Switch RBE image location for CUDA 9 to
 tensorflow-testing.

PiperOrigin-RevId: 227914044
---
 .../tools/ci_build/Dockerfile.rbe.cuda9.0-cudnn7-ubuntu14.04  | 4 ++--
 third_party/toolchains/BUILD                                  | 2 +-
 third_party/toolchains/preconfig/generate/containers.bzl      | 2 +-
 third_party/toolchains/preconfig/generate/workspace.bzl       | 2 +-
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/tensorflow/tools/ci_build/Dockerfile.rbe.cuda9.0-cudnn7-ubuntu14.04 b/tensorflow/tools/ci_build/Dockerfile.rbe.cuda9.0-cudnn7-ubuntu14.04
index 9d5d51447c..4ce4214065 100644
--- a/tensorflow/tools/ci_build/Dockerfile.rbe.cuda9.0-cudnn7-ubuntu14.04
+++ b/tensorflow/tools/ci_build/Dockerfile.rbe.cuda9.0-cudnn7-ubuntu14.04
@@ -1,7 +1,7 @@
 # To push a new version, run:
 # $ docker build -f Dockerfile.rbe.cuda9.0-cudnn7-ubuntu14.04 \
-#       --tag "gcr.io/asci-toolchain/nosla-cuda9.0-cudnn7-ubuntu14.04" .
-# $ docker push gcr.io/asci-toolchain/nosla-cuda9.0-cudnn7-ubuntu14.04
+#       --tag "gcr.io/tensorflow-testing/nosla-cuda9.0-cudnn7-ubuntu14.04" .
+# $ docker push gcr.io/tensorflow-testing/nosla-cuda9.0-cudnn7-ubuntu14.04
 #
 # TODO(klimek): Include clang in this image so we can also target clang
 # builds.
diff --git a/third_party/toolchains/BUILD b/third_party/toolchains/BUILD
index 28f1e6979c..f2248341c4 100644
--- a/third_party/toolchains/BUILD
+++ b/third_party/toolchains/BUILD
@@ -32,7 +32,7 @@ platform(
     remote_execution_properties = """
         properties: {
             name: "container-image"
-            value:"docker://gcr.io/asci-toolchain/nosla-cuda9.0-cudnn7-ubuntu14.04@%s"
+            value:"docker://gcr.io/tensorflow-testing/nosla-cuda9.0-cudnn7-ubuntu14.04@%s"
         }""" % container_digests["cuda9.0-cudnn7-ubuntu14.04"],
 )
 
diff --git a/third_party/toolchains/preconfig/generate/containers.bzl b/third_party/toolchains/preconfig/generate/containers.bzl
index 32d5f29b52..67e43485ed 100644
--- a/third_party/toolchains/preconfig/generate/containers.bzl
+++ b/third_party/toolchains/preconfig/generate/containers.bzl
@@ -1,4 +1,4 @@
 container_digests = {
-    "cuda9.0-cudnn7-ubuntu14.04": "sha256:c43ed5341dd765042e0bbd1bf50fadeedd649d1e0c34d81999cb6ce30916cb95",
+    "cuda9.0-cudnn7-ubuntu14.04": "sha256:006a76ee1838122ff7f21ebac85f24c1ef350d4dd79b3ceff0e4fe649ed90d33",
     "cuda10.0-cudnn7-ubuntu14.04": "sha256:e36f05f1ff39e39ddf07122e37f2b1895948bb6f7acc3db37a3c496be5e66228",
 }
diff --git a/third_party/toolchains/preconfig/generate/workspace.bzl b/third_party/toolchains/preconfig/generate/workspace.bzl
index a3268585e1..eb74022c24 100644
--- a/third_party/toolchains/preconfig/generate/workspace.bzl
+++ b/third_party/toolchains/preconfig/generate/workspace.bzl
@@ -11,7 +11,7 @@ def _remote_config_workspace():
     container_pull(
         name = "cuda9.0-cudnn7-ubuntu14.04",
         registry = "gcr.io",
-        repository = "asci-toolchain/nosla-cuda9.0-cudnn7-ubuntu14.04",
+        repository = "tensorflow-testing/nosla-cuda9.0-cudnn7-ubuntu14.04",
         digest = container_digests["cuda9.0-cudnn7-ubuntu14.04"],
     )
 
-- 
GitLab


From 2f246ef177ad3cb8aba3b4de0a628822cbeab71d Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Fri, 4 Jan 2019 14:27:22 -0800
Subject: [PATCH 0254/1095] Upgrade ci docker images to cuda 10.

These really should be gone already...

PiperOrigin-RevId: 227914258
---
 tensorflow/tools/ci_build/Dockerfile.gpu     | 4 ++--
 tensorflow/tools/ci_build/Dockerfile.rbe.gpu | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/tools/ci_build/Dockerfile.gpu b/tensorflow/tools/ci_build/Dockerfile.gpu
index a4cad4b6c6..ff1fefe892 100644
--- a/tensorflow/tools/ci_build/Dockerfile.gpu
+++ b/tensorflow/tools/ci_build/Dockerfile.gpu
@@ -1,4 +1,4 @@
-FROM nvidia/cuda:9.0-cudnn7-devel-ubuntu16.04
+FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu16.04
 
 LABEL maintainer="Jan Prach <jendap@google.com>"
 
@@ -24,7 +24,7 @@ COPY install/.bazelrc /etc/bazel.bazelrc
 ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH
 
 # Link NCCL libray and header where the build script expects them.
-RUN mkdir /usr/local/cuda-9.0/lib &&  \
+RUN mkdir /usr/local/cuda/lib &&  \
     ln -s /usr/lib/x86_64-linux-gnu/libnccl.so.2 /usr/local/cuda/lib/libnccl.so.2 && \
     ln -s /usr/include/nccl.h /usr/local/cuda/include/nccl.h
 
diff --git a/tensorflow/tools/ci_build/Dockerfile.rbe.gpu b/tensorflow/tools/ci_build/Dockerfile.rbe.gpu
index b656205836..c4912a65b6 100644
--- a/tensorflow/tools/ci_build/Dockerfile.rbe.gpu
+++ b/tensorflow/tools/ci_build/Dockerfile.rbe.gpu
@@ -1,4 +1,4 @@
-FROM nvidia/cuda:9.0-cudnn7-devel-ubuntu16.04
+FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu16.04
 
 LABEL maintainer="Nick Lopez <ngiraldo@google.com>"
 
-- 
GitLab


From 7cc1ff9072ce4c2c7f36b7011e24486071ad13ee Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Fri, 4 Jan 2019 14:35:17 -0800
Subject: [PATCH 0255/1095] Better exception for tf.constant(tf.constant(x))

PiperOrigin-RevId: 227915529
---
 tensorflow/python/framework/constant_op.py         |  4 ++++
 tensorflow/python/kernel_tests/constant_op_test.py | 11 +++++++++++
 2 files changed, 15 insertions(+)

diff --git a/tensorflow/python/framework/constant_op.py b/tensorflow/python/framework/constant_op.py
index ade0797dcd..62a25f3fea 100644
--- a/tensorflow/python/framework/constant_op.py
+++ b/tensorflow/python/framework/constant_op.py
@@ -277,6 +277,10 @@ def _constant_impl(
                     (num_t, shape, shape.num_elements()))
   g = ops.get_default_graph()
   tensor_value = attr_value_pb2.AttrValue()
+  if tensor_util.is_tensor(value):
+    raise ValueError(
+        ("Cannot create a tf.constant from symbolic Tensor %s. Did you mean "
+         "tf.convert_to_tensor?") % (value,))
   tensor_value.tensor.CopyFrom(
       tensor_util.make_tensor_proto(
           value, dtype=dtype, shape=shape, verify_shape=verify_shape,
diff --git a/tensorflow/python/kernel_tests/constant_op_test.py b/tensorflow/python/kernel_tests/constant_op_test.py
index 583082c2aa..2b743ed4c4 100644
--- a/tensorflow/python/kernel_tests/constant_op_test.py
+++ b/tensorflow/python/kernel_tests/constant_op_test.py
@@ -24,6 +24,7 @@ from google.protobuf import text_format
 
 from tensorflow.core.framework import graph_pb2
 from tensorflow.core.framework import tensor_pb2
+from tensorflow.python.eager import def_function
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes as dtypes_lib
 from tensorflow.python.framework import errors_impl
@@ -630,6 +631,16 @@ class OnesTest(test.TestCase):
         self.assertEqual([2, 3], z.get_shape())
         self.assertAllEqual(z.eval(), np.ones([2, 3]))
 
+  @test_util.run_in_graph_and_eager_modes
+  def testIteratedConstantSensibleException(self):
+
+    @def_function.function
+    def constant_on_constant(x):
+      return constant_op.constant(x)
+
+    with self.assertRaisesRegexp(ValueError, "convert_to_tensor"):
+      constant_on_constant(constant_op.constant(2))
+
 
 class OnesLikeTest(test.TestCase):
 
-- 
GitLab


From 73b6e92b4b3faf0c20687af3ba63989fc5234f13 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Fri, 4 Jan 2019 14:35:23 -0800
Subject: [PATCH 0256/1095] Remove usage of the initialize and finalize api

PiperOrigin-RevId: 227915547
---
 .../distribute/python/metrics_v1_test.py      |  3 ---
 .../distribute/python/minimize_loss_test.py   | 19 +------------------
 .../contrib/distribute/python/step_fn_test.py |  4 ----
 3 files changed, 1 insertion(+), 25 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/metrics_v1_test.py b/tensorflow/contrib/distribute/python/metrics_v1_test.py
index 32a0d19943..7472f6dde4 100644
--- a/tensorflow/contrib/distribute/python/metrics_v1_test.py
+++ b/tensorflow/contrib/distribute/python/metrics_v1_test.py
@@ -122,7 +122,6 @@ class MetricsV1Test(test.TestCase, parameterized.TestCase):
         batches_per_update = distribution.num_replicas_in_sync
 
       self.evaluate(iterator.initializer)
-      self.evaluate(distribution.initialize())
       self.evaluate(variables.local_variables_initializer())
 
       batches_consumed = 0
@@ -136,8 +135,6 @@ class MetricsV1Test(test.TestCase, parameterized.TestCase):
         if batches_consumed >= 4:  # Consume 4 input batches in total.
           break
 
-      self.evaluate(distribution.finalize())
-
   @combinations.generate(all_combinations() + tpu_combinations())
   def testMean(self, distribution):
     def _dataset_fn():
diff --git a/tensorflow/contrib/distribute/python/minimize_loss_test.py b/tensorflow/contrib/distribute/python/minimize_loss_test.py
index 824c4b0937..b0e24a53f6 100644
--- a/tensorflow/contrib/distribute/python/minimize_loss_test.py
+++ b/tensorflow/contrib/distribute/python/minimize_loss_test.py
@@ -75,7 +75,6 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
         return distribution.run_steps_on_dataset(
             step_fn, iterator, iterations=2).run_op
 
-      self.evaluate(distribution.initialize())
       if not context.executing_eagerly():
         with self.cached_session() as sess:
           run_step = sess.make_callable(run_step())
@@ -84,12 +83,9 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
       weights, biases = [], []
       for _ in range(5):
         run_step()
-
         weights.append(self.evaluate(layer.kernel))
         biases.append(self.evaluate(layer.bias))
 
-      self.evaluate(distribution.finalize())
-
       error = abs(numpy.add(numpy.squeeze(weights), numpy.squeeze(biases)) - 1)
       is_not_increasing = all(y <= x for x, y in zip(error, error[1:]))
       self.assertTrue(is_not_increasing)
@@ -152,7 +148,7 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
     # `distribution.scope`.
     with variable_scope.variable_creator_scope(
         appending_creator), distribution.scope():
-      model_fn, dataset_fn, layer = minimize_loss_example(
+      model_fn, dataset_fn, _ = minimize_loss_example(
           optimizer_fn,
           use_bias=True,
           use_callable_loss=True,
@@ -169,16 +165,12 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
         return distribution.run_steps_on_dataset(
             step_fn, iterator, iterations=1).run_op
 
-      self.evaluate(distribution.initialize())
       if not context.executing_eagerly():
         with self.cached_session() as sess:
           run_step = sess.make_callable(run_step())
       self.evaluate(variables_lib.global_variables_initializer())
-
       run_step()
 
-      self.evaluate(distribution.finalize())
-
       def get_expected_variables(optimizer_fn, num_parameter_devices):
         variables_map = {
             "GradientDescent": ["dense/kernel", "dense/bias"],
@@ -241,7 +233,6 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
         return distribution.run_steps_on_dataset(
             step_fn, iterator, iterations=1).run_op
 
-      self.evaluate(distribution.initialize())
       if not context.executing_eagerly():
         with self.cached_session() as sess:
           run_step = sess.make_callable(run_step())
@@ -267,8 +258,6 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
               expected_moving_mean - averaged_batch_mean(i)) * (1.0 - momentum))
           self.assertNear(expected_moving_means[i], moving_means[i], 0.0001)
 
-      self.evaluate(distribution.finalize())
-
   @combinations.generate(
       combinations.times(
           combinations.combine(
@@ -335,7 +324,6 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
         return distribution.run_steps_on_dataset(
             step_fn, iterator, iterations=1).run_op
 
-      self.evaluate(distribution.initialize())
       if not context.executing_eagerly():
         with self.cached_session() as sess:
           run_step = sess.make_callable(run_step())
@@ -370,8 +358,6 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
         # One of the mean loss reductions.
         self.assertNear(weight, 2 + 10.6, 0.0001)
 
-      self.evaluate(distribution.finalize())
-
   @combinations.generate(
       combinations.times(
           combinations.distributions_and_v1_optimizers(),
@@ -458,7 +444,6 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
             reduced=False, distribution=distribution)
         return (ctx.run_op, ctx.last_step_outputs["replica_loss_reduced"])
 
-      self.evaluate(distribution.initialize())
       if not context.executing_eagerly():
         with self.cached_session() as sess:
           run_step = sess.make_callable(run_step())
@@ -471,8 +456,6 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
         weights.append(self.evaluate(layer.kernel))
         biases.append(self.evaluate(layer.bias))
 
-      self.evaluate(distribution.finalize())
-
       loss_is_not_increasing = all(y <= x for x, y in zip(losses, losses[1:]))
       self.assertTrue(loss_is_not_increasing)
 
diff --git a/tensorflow/contrib/distribute/python/step_fn_test.py b/tensorflow/contrib/distribute/python/step_fn_test.py
index 1ff9b9ceec..a77d6d0bec 100644
--- a/tensorflow/contrib/distribute/python/step_fn_test.py
+++ b/tensorflow/contrib/distribute/python/step_fn_test.py
@@ -45,7 +45,6 @@ class SingleLossStepTest(test.TestCase, parameterized.TestCase):
       single_loss_step, layer = single_loss_example(
           optimizer_fn, distribution, use_bias=True, iterations_per_step=2)
 
-      self.evaluate(distribution.initialize())
       if context.executing_eagerly():
         run_step = single_loss_step
       else:
@@ -57,12 +56,9 @@ class SingleLossStepTest(test.TestCase, parameterized.TestCase):
       weights, biases = [], []
       for _ in range(5):
         run_step()
-
         weights.append(self.evaluate(layer.kernel))
         biases.append(self.evaluate(layer.bias))
 
-      self.evaluate(distribution.finalize())
-
       error = abs(numpy.add(numpy.squeeze(weights), numpy.squeeze(biases)) - 1)
       is_not_increasing = all(y <= x for x, y in zip(error, error[1:]))
       self.assertTrue(is_not_increasing)
-- 
GitLab


From 1dee496b6addb67c269151d4a96b81608b1eb463 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 4 Jan 2019 14:43:02 -0800
Subject: [PATCH 0257/1095] TOCO's default_range parameters are floats, not
 integers.

PiperOrigin-RevId: 227916837
---
 tensorflow/lite/python/tflite_convert.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/python/tflite_convert.py b/tensorflow/lite/python/tflite_convert.py
index 341b539bea..401a592273 100644
--- a/tensorflow/lite/python/tflite_convert.py
+++ b/tensorflow/lite/python/tflite_convert.py
@@ -343,13 +343,13 @@ def run_main(_):
             "floats. Used for quantized input tensors. (default None)"))
   parser.add_argument(
       "--default_ranges_min",
-      type=int,
+      type=float,
       help=("Default value for min bound of min/max range values used for all "
             "arrays without a specified range, Intended for experimenting with "
             "quantization via \"dummy quantization\". (default None)"))
   parser.add_argument(
       "--default_ranges_max",
-      type=int,
+      type=float,
       help=("Default value for max bound of min/max range values used for all "
             "arrays without a specified range, Intended for experimenting with "
             "quantization via \"dummy quantization\". (default None)"))
-- 
GitLab


From 58edbcb97b90596ee5f4cd907dfa7dd748c1843a Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Fri, 4 Jan 2019 15:03:28 -0800
Subject: [PATCH 0258/1095] Internal change.

PiperOrigin-RevId: 227920137
---
 tensorflow/tools/ci_build/builds/run_pip_tests.sh    | 3 ++-
 tensorflow/tools/ci_build/ci_parameterized_build.sh  | 9 ++++++---
 tensorflow/tools/ci_build/osx/cpu/run_contrib.sh     | 1 +
 tensorflow/tools/ci_build/osx/cpu/run_py2_cc_core.sh | 1 +
 4 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/tensorflow/tools/ci_build/builds/run_pip_tests.sh b/tensorflow/tools/ci_build/builds/run_pip_tests.sh
index 7d5cf3f843..a095633a22 100755
--- a/tensorflow/tools/ci_build/builds/run_pip_tests.sh
+++ b/tensorflow/tools/ci_build/builds/run_pip_tests.sh
@@ -88,7 +88,8 @@ if [[ ${IS_GPU} == "1" ]]; then
   PIP_TEST_FILTER_TAG="-no_gpu,-no_pip_gpu,${PIP_TEST_FILTER_TAG}"
 fi
 if [[ ${IS_MAC} == "1" ]]; then
-  PIP_TEST_FILTER_TAG="-nomac,${PIP_TEST_FILTER_TAG}"
+  # TODO(b/122370901): Fix nomac, no_mac inconsistency.
+  PIP_TEST_FILTER_TAG="-nomac,-no_mac,${PIP_TEST_FILTER_TAG}"
 fi
 
 # Bazel flags we need for all tests:
diff --git a/tensorflow/tools/ci_build/ci_parameterized_build.sh b/tensorflow/tools/ci_build/ci_parameterized_build.sh
index 435ec7ca68..41b9f241d5 100755
--- a/tensorflow/tools/ci_build/ci_parameterized_build.sh
+++ b/tensorflow/tools/ci_build/ci_parameterized_build.sh
@@ -398,7 +398,8 @@ if [[ "${TF_BUILD_APPEND_ARGUMENTS}" == *"--test_tag_filters="* ]]; then
         NEW_ITEM="${NEW_ITEM},-benchmark-test"
       fi
       if [[ ${IS_MAC} == "1" ]] && [[ ${NEW_ITEM} != *"nomac"* ]]; then
-        NEW_ITEM="${NEW_ITEM},-nomac"
+        # TODO(b/122370901): Fix nomac, no_mac inconsistency.
+        NEW_ITEM="${NEW_ITEM},-nomac,-no_mac"
       fi
       EXTRA_ARGS="${EXTRA_ARGS} ${NEW_ITEM}"
     else
@@ -408,11 +409,13 @@ if [[ "${TF_BUILD_APPEND_ARGUMENTS}" == *"--test_tag_filters="* ]]; then
 else
   EXTRA_ARGS="${EXTRA_ARGS} ${TF_BUILD_APPEND_ARGUMENTS} --test_tag_filters=-no_oss,-oss_serial,-benchmark-test"
   if [[ ${IS_MAC} == "1" ]]; then
-    EXTRA_ARGS="${EXTRA_ARGS},-nomac"
+    # TODO(b/122370901): Fix nomac, no_mac inconsistency.
+    EXTRA_ARGS="${EXTRA_ARGS},-nomac,-no_mac"
   fi
   EXTRA_ARGS="${EXTRA_ARGS} --build_tag_filters=-no_oss,-oss_serial,-benchmark-test"
   if [[ ${IS_MAC} == "1" ]]; then
-    EXTRA_ARGS="${EXTRA_ARGS},-nomac"
+    # TODO(b/122370901): Fix nomac, no_mac inconsistency.
+    EXTRA_ARGS="${EXTRA_ARGS},-nomac,-no_mac"
   fi
 fi
 
diff --git a/tensorflow/tools/ci_build/osx/cpu/run_contrib.sh b/tensorflow/tools/ci_build/osx/cpu/run_contrib.sh
index 3efd994d78..1184d4acec 100755
--- a/tensorflow/tools/ci_build/osx/cpu/run_contrib.sh
+++ b/tensorflow/tools/ci_build/osx/cpu/run_contrib.sh
@@ -31,6 +31,7 @@ export CC_OPT_FLAGS='-mavx'
 export PYTHON_BIN_PATH=$(which python2)
 yes "" | $PYTHON_BIN_PATH configure.py
 which bazel
+# TODO(b/122370901): Fix nomac, no_mac inconsistency.
 bazel test --test_tag_filters=-no_oss,-gpu,-benchmark-test,-nomac,-no_mac \
     --test_timeout 300,450,1200,3600 \
     --test_size_filters=small,medium --config=opt \
diff --git a/tensorflow/tools/ci_build/osx/cpu/run_py2_cc_core.sh b/tensorflow/tools/ci_build/osx/cpu/run_py2_cc_core.sh
index adee0d3171..d39340b1d8 100755
--- a/tensorflow/tools/ci_build/osx/cpu/run_py2_cc_core.sh
+++ b/tensorflow/tools/ci_build/osx/cpu/run_py2_cc_core.sh
@@ -32,6 +32,7 @@ export CC_OPT_FLAGS='-mavx'
 export PYTHON_BIN_PATH=$(which python2)
 yes "" | $PYTHON_BIN_PATH configure.py
 which bazel
+# TODO(b/122370901): Fix nomac, no_mac inconsistency.
 bazel test --test_tag_filters=-no_oss,-gpu,-benchmark-test,-nomac,-no_mac \
     --test_timeout 300,450,1200,3600 --config=opt \
     --announce_rc \
-- 
GitLab


From 07777005d9b3feff4c8d74348739ad17de510131 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 4 Jan 2019 15:07:06 -0800
Subject: [PATCH 0259/1095] Implement memory prefetching for PresizedCuckooMap

PiperOrigin-RevId: 227920727
---
 tensorflow/core/BUILD                         |  1 +
 tensorflow/core/util/presized_cuckoo_map.h    | 10 ++++
 .../core/util/presized_cuckoo_map_test.cc     | 49 ++++++++++++++++++-
 3 files changed, 59 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 7c9decbd09..460353df41 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -3813,6 +3813,7 @@ tf_cc_tests(
         "//tensorflow/core/kernels:ops_util",
         "//third_party/eigen3",
         "@com_google_absl//absl/base",
+        "@com_google_absl//absl/time",
     ],
 )
 
diff --git a/tensorflow/core/util/presized_cuckoo_map.h b/tensorflow/core/util/presized_cuckoo_map.h
index f88ad2faaf..1cdde34562 100644
--- a/tensorflow/core/util/presized_cuckoo_map.h
+++ b/tensorflow/core/util/presized_cuckoo_map.h
@@ -20,6 +20,7 @@ limitations under the License.
 #include <vector>
 #include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/platform/macros.h"
+#include "tensorflow/core/platform/prefetch.h"
 
 namespace tensorflow {
 
@@ -132,6 +133,15 @@ class PresizedCuckooMap {
            FindInBucket(k, fast_map_to_buckets(h2(tk)), out);
   }
 
+  // Prefetch memory associated with the key k into cache levels specified by
+  // hint.
+  template <port::PrefetchHint hint = port::PREFETCH_HINT_T0>
+  void PrefetchKey(const key_type k) const {
+    const uint64 tk = key_transform(k);
+    port::prefetch<hint>(&buckets_[fast_map_to_buckets(tk)].keys);
+    port::prefetch<hint>(&buckets_[fast_map_to_buckets(h2(tk))].keys);
+  }
+
   int64 MemoryUsed() const {
     return sizeof(PresizedCuckooMap<value>) + sizeof(CuckooPathQueue);
   }
diff --git a/tensorflow/core/util/presized_cuckoo_map_test.cc b/tensorflow/core/util/presized_cuckoo_map_test.cc
index f2be1e8a2f..8dbb2ec065 100644
--- a/tensorflow/core/util/presized_cuckoo_map_test.cc
+++ b/tensorflow/core/util/presized_cuckoo_map_test.cc
@@ -13,12 +13,14 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/core/util/presized_cuckoo_map.h"
 #include <array>
+
+#include "absl/time/clock.h"
 #include "tensorflow/core/platform/env.h"
 #include "tensorflow/core/platform/fingerprint.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/core/platform/test_benchmark.h"
+#include "tensorflow/core/util/presized_cuckoo_map.h"
 
 namespace tensorflow {
 namespace {
@@ -50,6 +52,51 @@ TEST(PresizedCuckooMapTest, Basic) {
   EXPECT_EQ(out, 2);
 }
 
+TEST(PresizedCuckooMapTest, Prefetch) {
+  {
+    PresizedCuckooMap<int64> pscm(2);
+    EXPECT_TRUE(pscm.InsertUnique(1, 2));
+    // Works for both present and absent keys.
+    pscm.PrefetchKey(1);
+    pscm.PrefetchKey(2);
+  }
+
+  // Do not run in debug mode, when prefetch is not implemented, or when
+  // sanitizers are enabled.
+#if defined(NDEBUG) && defined(__GNUC__) && !defined(ADDRESS_SANITIZER) && \
+    !defined(MEMORY_SANITIZER) && !defined(THREAD_SANITIZER) &&            \
+    !defined(UNDEFINED_BEHAVIOR_SANITIZER)
+  const auto now = [] { return absl::Now(); };
+
+  // Make size enough to not fit in L2 cache (16.7 Mb)
+  static constexpr int size = 1 << 22;
+  PresizedCuckooMap<int64> pscm(size);
+  for (int i = 0; i < size; ++i) {
+    pscm.InsertUnique(i, i);
+  }
+
+  absl::Duration no_prefetch, prefetch;
+  int64 out;
+  for (int iter = 0; iter < 10; ++iter) {
+    auto time = now();
+    for (int i = 0; i < size; ++i) {
+      testing::DoNotOptimize(pscm.Find(i, &out));
+    }
+    no_prefetch += now() - time;
+
+    time = now();
+    for (int i = 0; i < size; ++i) {
+      pscm.PrefetchKey(i + 20);
+      testing::DoNotOptimize(pscm.Find(i, &out));
+    }
+    prefetch += now() - time;
+  }
+
+  // no_prefetch is at least 30% slower.
+  EXPECT_GE(1.0 * no_prefetch / prefetch, 1.3);
+#endif
+}
+
 TEST(PresizedCuckooMapTest, TooManyItems) {
   static constexpr int kTableSize = 1000;
   PresizedCuckooMap<int> pscm(kTableSize);
-- 
GitLab


From 620fa70346ef6cd2c7f13c9986625850d523cf0b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 4 Jan 2019 15:23:23 -0800
Subject: [PATCH 0260/1095] Go: Update generated wrapper functions for
 TensorFlow ops. PiperOrigin-RevId: 227923029

---
 tensorflow/go/op/wrappers.go | 1844 ++++++++++++++++++++++++++++++++--
 1 file changed, 1782 insertions(+), 62 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index 6fe61f512f..208c15d75a 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -327,6 +327,192 @@ func FakeQuantWithMinMaxArgs(scope *Scope, inputs tf.Output, optional ...FakeQua
 	return op.Output(0)
 }
 
+// Subtracts sparse `updates` from an existing tensor according to `indices`.
+//
+// This operation creates a new tensor by subtracting sparse `updates` from the
+// passed in `tensor`.
+// This operation is very similar to `tf.scatter_nd_sub`, except that the updates
+// are subtracted from an existing tensor (as opposed to a variable). If the memory
+// for the existing tensor cannot be re-used, a copy is made and updated.
+//
+// `indices` is an integer tensor containing indices into a new tensor of shape
+// `shape`.  The last dimension of `indices` can be at most the rank of `shape`:
+//
+//     indices.shape[-1] <= shape.rank
+//
+// The last dimension of `indices` corresponds to indices into elements
+// (if `indices.shape[-1] = shape.rank`) or slices
+// (if `indices.shape[-1] < shape.rank`) along dimension `indices.shape[-1]` of
+// `shape`.  `updates` is a tensor with shape
+//
+//     indices.shape[:-1] + shape[indices.shape[-1]:]
+//
+// The simplest form of tensor_scatter_sub is to subtract individual elements
+// from a tensor by index. For example, say we want to insert 4 scattered elements
+// in a rank-1 tensor with 8 elements.
+//
+// In Python, this scatter subtract operation would look like this:
+//
+// ```python
+//     indices = tf.constant([[4], [3], [1], [7]])
+//     updates = tf.constant([9, 10, 11, 12])
+//     tensor = tf.ones([8], dtype=tf.int32)
+//     updated = tf.tensor_scatter_sub(tensor, indices, updates)
+//     with tf.Session() as sess:
+//       print(sess.run(scatter))
+// ```
+//
+// The resulting tensor would look like this:
+//
+//     [1, -10, 1, -9, -8, 1, 1, -11]
+//
+// We can also, insert entire slices of a higher rank tensor all at once. For
+// example, if we wanted to insert two slices in the first dimension of a
+// rank-3 tensor with two matrices of new values.
+//
+// In Python, this scatter add operation would look like this:
+//
+// ```python
+//     indices = tf.constant([[0], [2]])
+//     updates = tf.constant([[[5, 5, 5, 5], [6, 6, 6, 6],
+//                             [7, 7, 7, 7], [8, 8, 8, 8]],
+//                            [[5, 5, 5, 5], [6, 6, 6, 6],
+//                             [7, 7, 7, 7], [8, 8, 8, 8]]])
+//     tensor = tf.ones([4, 4, 4])
+//     updated = tf.tensor_scatter_sub(tensor, indices, updates)
+//     with tf.Session() as sess:
+//       print(sess.run(scatter))
+// ```
+//
+// The resulting tensor would look like this:
+//
+//     [[[-4, -4, -4, -4], [-5, -5, -5, -5], [-6, -6, -6, -6], [-7, -7, -7, -7]],
+//      [[1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1]],
+//      [[-4, -4, -4, -4], [-5, -5, -5, -5], [-6, -6, -6, -6], [-7, -7, -7, -7]],
+//      [[1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1]]]
+//
+// Note that on CPU, if an out of bound index is found, an error is returned.
+// On GPU, if an out of bound index is found, the index is ignored.
+//
+// Arguments:
+//	tensor: Tensor to copy/update.
+//	indices: Index tensor.
+//	updates: Updates to scatter into output.
+//
+// Returns A new tensor copied from tensor and updates subtracted according to the indices.
+func TensorScatterSub(scope *Scope, tensor tf.Output, indices tf.Output, updates tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "TensorScatterSub",
+		Input: []tf.Input{
+			tensor, indices, updates,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Scatter `updates` into an existing tensor according to `indices`.
+//
+// This operation creates a new tensor by applying sparse `updates` to the passed
+// in `tensor`.
+// This operation is very similar to `tf.scatter_nd`, except that the updates are
+// scattered onto an existing tensor (as opposed to a zero-tensor). If the memory
+// for the existing tensor cannot be re-used, a copy is made and updated.
+//
+// If `indices` contains duplicates, then their updates are accumulated (summed).
+//
+// **WARNING**: The order in which updates are applied is nondeterministic, so the
+// output will be nondeterministic if `indices` contains duplicates -- because
+// of some numerical approximation issues, numbers summed in different order
+// may yield different results.
+//
+// `indices` is an integer tensor containing indices into a new tensor of shape
+// `shape`.  The last dimension of `indices` can be at most the rank of `shape`:
+//
+//     indices.shape[-1] <= shape.rank
+//
+// The last dimension of `indices` corresponds to indices into elements
+// (if `indices.shape[-1] = shape.rank`) or slices
+// (if `indices.shape[-1] < shape.rank`) along dimension `indices.shape[-1]` of
+// `shape`.  `updates` is a tensor with shape
+//
+//     indices.shape[:-1] + shape[indices.shape[-1]:]
+//
+// The simplest form of scatter is to insert individual elements in a tensor by
+// index. For example, say we want to insert 4 scattered elements in a rank-1
+// tensor with 8 elements.
+//
+// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+// <img style="width:100%" src="https://www.tensorflow.org/images/ScatterNd1.png" alt>
+// </div>
+//
+// In Python, this scatter operation would look like this:
+//
+// ```python
+//     indices = tf.constant([[4], [3], [1], [7]])
+//     updates = tf.constant([9, 10, 11, 12])
+//     tensor = tf.ones([8], dtype=tf.int32)
+//     updated = tf.tensor_scatter_update(tensor, indices, updates)
+//     with tf.Session() as sess:
+//       print(sess.run(scatter))
+// ```
+//
+// The resulting tensor would look like this:
+//
+//     [1, 11, 1, 10, 9, 1, 1, 12]
+//
+// We can also, insert entire slices of a higher rank tensor all at once. For
+// example, if we wanted to insert two slices in the first dimension of a
+// rank-3 tensor with two matrices of new values.
+//
+// In Python, this scatter operation would look like this:
+//
+// ```python
+//     indices = tf.constant([[0], [2]])
+//     updates = tf.constant([[[5, 5, 5, 5], [6, 6, 6, 6],
+//                             [7, 7, 7, 7], [8, 8, 8, 8]],
+//                            [[5, 5, 5, 5], [6, 6, 6, 6],
+//                             [7, 7, 7, 7], [8, 8, 8, 8]]])
+//     tensor = tf.ones([4, 4, 4])
+//     updated = tf.tensor_scatter_update(tensor, indices, updates)
+//     with tf.Session() as sess:
+//       print(sess.run(scatter))
+// ```
+//
+// The resulting tensor would look like this:
+//
+//     [[[5, 5, 5, 5], [6, 6, 6, 6], [7, 7, 7, 7], [8, 8, 8, 8]],
+//      [[1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1]],
+//      [[5, 5, 5, 5], [6, 6, 6, 6], [7, 7, 7, 7], [8, 8, 8, 8]],
+//      [[1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1]]]
+//
+// Note that on CPU, if an out of bound index is found, an error is returned.
+// On GPU, if an out of bound index is found, the index is ignored.
+//
+// Arguments:
+//	tensor: Tensor to copy/update.
+//	indices: Index tensor.
+//	updates: Updates to scatter into output.
+//
+// Returns A new tensor with the given shape and updates applied according
+// to the indices.
+func TensorScatterUpdate(scope *Scope, tensor tf.Output, indices tf.Output, updates tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "TensorScatterUpdate",
+		Input: []tf.Input{
+			tensor, indices, updates,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Scatter `updates` into a new tensor according to `indices`.
 //
 // Creates a new tensor by applying sparse `updates` to individual values or
@@ -334,6 +520,10 @@ func FakeQuantWithMinMaxArgs(scope *Scope, inputs tf.Output, optional ...FakeQua
 // the given `shape` according to indices.  This operator is the inverse of the
 // `tf.gather_nd` operator which extracts values or slices from a given tensor.
 //
+// This operation is similar to tensor_scatter_add, except that the tensor is
+// zero-initialized. Calling `tf.scatter_nd(indices, values, shape)` is identical
+// to `tensor_scatter_add(tf.zeros(shape, values.dtype), indices, values)`
+//
 // If `indices` contains duplicates, then their updates are accumulated (summed).
 //
 // **WARNING**: The order in which updates are applied is nondeterministic, so the
@@ -464,6 +654,15 @@ func QuantizeAndDequantizeV2RangeGiven(value bool) QuantizeAndDequantizeV2Attr {
 }
 
 // QuantizeAndDequantizeV2RoundMode sets the optional round_mode attribute to value.
+//
+// value: The 'round_mode' attribute controls which rounding tie-breaking algorithm is
+// used when rounding float values to their quantized equivalents. The following
+// rounding modes are currently supported:
+//
+// *   HALF_TO_EVEN: this is the default round_mode.
+// *   HALF_UP: round towards positive. In this mode 7.5 rounds up to 8 and -7.5
+//     rounds up to -7.
+//
 // If not specified, defaults to "HALF_TO_EVEN"
 func QuantizeAndDequantizeV2RoundMode(value string) QuantizeAndDequantizeV2Attr {
 	return func(m optionalAttr) {
@@ -523,7 +722,7 @@ func QuantizeAndDequantizeV2RoundMode(value string) QuantizeAndDequantizeV2Attr
 //
 // output = round(clamp(value, input_min, input_max) * scale_factor) / scale_factor.
 //
-// The above round function uses half to even rounding.
+// The above round function rounds the value based on the given round_mode.
 //
 //
 // Arguments:
@@ -3422,11 +3621,11 @@ func PopulationCount(scope *Scope, x tf.Output) (y tf.Output) {
 // bucketized values for a single feature.
 //
 // Arguments:
-//	float_values: float; List of Rank 2 Tensor each containing float values for a single feature.
+//	float_values: float; List of Rank 1 Tensor each containing float values for a single feature.
 //	bucket_boundaries: float; List of Rank 1 Tensors each containing the bucket boundaries for a single
 // feature.
 //
-// Returns int; List of Rank 2 Tensors each containing the bucketized values for a single feature.
+// Returns int; List of Rank 1 Tensors each containing the bucketized values for a single feature.
 func BoostedTreesBucketize(scope *Scope, float_values []tf.Output, bucket_boundaries []tf.Output) (buckets []tf.Output) {
 	if scope.Err() != nil {
 		return
@@ -3497,15 +3696,16 @@ func BoostedTreesQuantileStreamResourceFlush(scope *Scope, quantile_stream_resou
 
 // Makes the summary of quantiles for the batch.
 //
-// An op that takes a list of tensors and outputs the quantile summaries for each tensor.
+// An op that takes a list of tensors (one tensor per feature) and outputs the
+// quantile summaries for each tensor.
 //
 // Arguments:
-//	float_values: float; List of Rank 2 Tensors each containing values for a single feature.
+//	float_values: float; List of Rank 1 Tensors each containing values for a single feature.
 //	example_weights: float; Rank 1 Tensor with weights per instance.
 //	epsilon: float; The required maximum approximation error.
 //
-// Returns float; List of Rank 2 Tensors each containing the quantile summary (value, weight,
-// min_rank, max_rank) of a single feature.
+// Returns float; List of Rank 2 Tensors each containing the quantile summary
+// (value, weight, min_rank, max_rank) of a single feature.
 func BoostedTreesMakeQuantileSummaries(scope *Scope, float_values []tf.Output, example_weights tf.Output, epsilon tf.Output) (summaries []tf.Output) {
 	if scope.Err() != nil {
 		return
@@ -3806,6 +4006,70 @@ func BoostedTreesEnsembleResourceHandleOp(scope *Scope, optional ...BoostedTrees
 	return op.Output(0)
 }
 
+// Output the logits for the given input data
+//
+// Arguments:
+//	tree_handle: Handle to the tree resource.
+//	dense_features: Rank 2 dense features tensor.
+//	logits_dimension: Scalar, dimension of the logits.
+//
+// Returns The logits predictions from the tree for each instance in the batch.
+func TensorForestTreePredict(scope *Scope, tree_handle tf.Output, dense_features tf.Output, logits_dimension int64) (logits tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"logits_dimension": logits_dimension}
+	opspec := tf.OpSpec{
+		Type: "TensorForestTreePredict",
+		Input: []tf.Input{
+			tree_handle, dense_features,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Get the number of nodes in a tree
+//
+// Arguments:
+//	tree_handle: Handle to the tree resource.
+//
+// Returns The size of the tree.
+func TensorForestTreeSize(scope *Scope, tree_handle tf.Output) (tree_size tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "TensorForestTreeSize",
+		Input: []tf.Input{
+			tree_handle,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Creates a tree resource and returns a handle to it.
+//
+// Arguments:
+//	tree_handle: Handle to the tree resource to be created.
+//	tree_config: Serialized proto string of the boosted_trees.Tree.
+//
+// Returns the created operation.
+func TensorForestCreateTreeVariable(scope *Scope, tree_handle tf.Output, tree_config tf.Output) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "TensorForestCreateTreeVariable",
+		Input: []tf.Input{
+			tree_handle, tree_config,
+		},
+	}
+	return scope.AddOperation(opspec)
+}
+
 // ComputeAccidentalHitsAttr is an optional argument to ComputeAccidentalHits.
 type ComputeAccidentalHitsAttr func(optionalAttr)
 
@@ -4829,6 +5093,119 @@ func CudnnRNNParamsToCanonical(scope *Scope, num_layers tf.Output, num_units tf.
 	return weights, biases
 }
 
+// CudnnRNNBackpropV3Attr is an optional argument to CudnnRNNBackpropV3.
+type CudnnRNNBackpropV3Attr func(optionalAttr)
+
+// CudnnRNNBackpropV3RnnMode sets the optional rnn_mode attribute to value.
+// If not specified, defaults to "lstm"
+func CudnnRNNBackpropV3RnnMode(value string) CudnnRNNBackpropV3Attr {
+	return func(m optionalAttr) {
+		m["rnn_mode"] = value
+	}
+}
+
+// CudnnRNNBackpropV3InputMode sets the optional input_mode attribute to value.
+// If not specified, defaults to "linear_input"
+func CudnnRNNBackpropV3InputMode(value string) CudnnRNNBackpropV3Attr {
+	return func(m optionalAttr) {
+		m["input_mode"] = value
+	}
+}
+
+// CudnnRNNBackpropV3Direction sets the optional direction attribute to value.
+// If not specified, defaults to "unidirectional"
+func CudnnRNNBackpropV3Direction(value string) CudnnRNNBackpropV3Attr {
+	return func(m optionalAttr) {
+		m["direction"] = value
+	}
+}
+
+// CudnnRNNBackpropV3Dropout sets the optional dropout attribute to value.
+// If not specified, defaults to 0
+func CudnnRNNBackpropV3Dropout(value float32) CudnnRNNBackpropV3Attr {
+	return func(m optionalAttr) {
+		m["dropout"] = value
+	}
+}
+
+// CudnnRNNBackpropV3Seed sets the optional seed attribute to value.
+// If not specified, defaults to 0
+func CudnnRNNBackpropV3Seed(value int64) CudnnRNNBackpropV3Attr {
+	return func(m optionalAttr) {
+		m["seed"] = value
+	}
+}
+
+// CudnnRNNBackpropV3Seed2 sets the optional seed2 attribute to value.
+// If not specified, defaults to 0
+func CudnnRNNBackpropV3Seed2(value int64) CudnnRNNBackpropV3Attr {
+	return func(m optionalAttr) {
+		m["seed2"] = value
+	}
+}
+
+// Backprop step of CudnnRNNV3.
+//
+// Compute the backprop of both data and weights in a RNN. Takes an extra
+//     "sequence_lengths" input than CudnnRNNBackprop.
+//
+// rnn_mode: Indicates the type of the RNN model.
+// input_mode: Indicates whether there is a linear projection between the input and
+//     the actual computation before the first layer. 'skip_input' is only allowed
+//     when input_size == num_units; 'auto_select' implies 'skip_input' when
+//     input_size == num_units; otherwise, it implies 'linear_input'.
+// direction: Indicates whether a bidirectional model will be used. Should be
+//   "unidirectional" or "bidirectional".
+// dropout: Dropout probability. When set to 0., dropout is disabled.
+// seed: The 1st part of a seed to initialize dropout.
+// seed2: The 2nd part of a seed to initialize dropout.
+// input: A 3-D tensor with the shape of [seq_length, batch_size, input_size].
+// input_h: A 3-D tensor with the shape of [num_layer * dir, batch_size,
+//     num_units].
+// input_c: For LSTM, a 3-D tensor with the shape of
+//     [num_layer * dir, batch, num_units]. For other models, it is ignored.
+// params: A 1-D tensor that contains the weights and biases in an opaque layout.
+//     The size must be created through CudnnRNNParamsSize, and initialized
+//     separately. Note that they might not be compatible across different
+//     generations. So it is a good idea to save and restore
+// sequence_lengths: a vector of lengths of each input sequence.
+// output: A 3-D tensor with the shape of [seq_length, batch_size,
+//     dir * num_units].
+// output_h: The same shape has input_h.
+// output_c: The same shape as input_c for LSTM. An empty tensor for other models.
+// output_backprop: A 3-D tensor with the same shape as output in the forward pass.
+// output_h_backprop: A 3-D tensor with the same shape as output_h in the forward
+//     pass.
+// output_c_backprop: A 3-D tensor with the same shape as output_c in the forward
+//     pass.
+// reserve_space: The same reserve_space produced in the forward operation.
+// input_backprop: The backprop to input in the forward pass. Has the same shape
+//     as input.
+// input_h_backprop: The backprop to input_h in the forward pass. Has the same
+//     shape as input_h.
+// input_c_backprop: The backprop to input_c in the forward pass. Has the same
+//     shape as input_c.
+// params_backprop: The backprop to the params buffer in the forward pass. Has the
+//     same shape as params.
+func CudnnRNNBackpropV3(scope *Scope, input tf.Output, input_h tf.Output, input_c tf.Output, params tf.Output, sequence_lengths tf.Output, output tf.Output, output_h tf.Output, output_c tf.Output, output_backprop tf.Output, output_h_backprop tf.Output, output_c_backprop tf.Output, reserve_space tf.Output, host_reserved tf.Output, optional ...CudnnRNNBackpropV3Attr) (input_backprop tf.Output, input_h_backprop tf.Output, input_c_backprop tf.Output, params_backprop tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "CudnnRNNBackpropV3",
+		Input: []tf.Input{
+			input, input_h, input_c, params, sequence_lengths, output, output_h, output_c, output_backprop, output_h_backprop, output_c_backprop, reserve_space, host_reserved,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1), op.Output(2), op.Output(3)
+}
+
 // CudnnRNNBackpropV2Attr is an optional argument to CudnnRNNBackpropV2.
 type CudnnRNNBackpropV2Attr func(optionalAttr)
 
@@ -6697,6 +7074,34 @@ func Sign(scope *Scope, x tf.Output) (y tf.Output) {
 	return op.Output(0)
 }
 
+// Creates a dataset that passes a sliding window over `input_dataset`.
+//
+// Arguments:
+//
+//	window_size: A scalar representing the number of elements in the
+// sliding window.
+//	window_shift: A scalar representing the steps moving the sliding window
+// forward in one iteration. It must be positive.
+//	window_stride: A scalar representing the stride of the input elements of the sliding window.
+// It must be positive.
+//
+//
+func ExperimentalSlidingWindowDataset(scope *Scope, input_dataset tf.Output, window_size tf.Output, window_shift tf.Output, window_stride tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	opspec := tf.OpSpec{
+		Type: "ExperimentalSlidingWindowDataset",
+		Input: []tf.Input{
+			input_dataset, window_size, window_shift, window_stride,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Returns which elements of x are finite.
 //
 // @compatibility(numpy)
@@ -7608,6 +8013,33 @@ func Inv(scope *Scope, x tf.Output) (y tf.Output) {
 	return op.Output(0)
 }
 
+// Creates a dataset that batches input elements into a SparseTensor.
+//
+// Arguments:
+//	input_dataset: A handle to an input dataset. Must have a single component.
+//	batch_size: A scalar representing the number of elements to accumulate in a
+// batch.
+//	row_shape: A vector representing the dense shape of each row in the produced
+// SparseTensor. The shape may be partially specified, using `-1` to indicate
+// that a particular dimension should use the maximum size of all batch elements.
+//
+//
+func ExperimentalDenseToSparseBatchDataset(scope *Scope, input_dataset tf.Output, batch_size tf.Output, row_shape tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	opspec := tf.OpSpec{
+		Type: "ExperimentalDenseToSparseBatchDataset",
+		Input: []tf.Input{
+			input_dataset, batch_size, row_shape,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // ComplexAbsAttr is an optional argument to ComplexAbs.
 type ComplexAbsAttr func(optionalAttr)
 
@@ -8442,6 +8874,26 @@ func ResourceSparseApplyAdadelta(scope *Scope, var_ tf.Output, accum tf.Output,
 	return scope.AddOperation(opspec)
 }
 
+// Checks whether a tree has been initialized.
+//
+// Arguments:
+//	tree_handle: Handle to the tree.
+//
+// Returns Whether the tree is initialized.
+func TensorForestTreeIsInitializedOp(scope *Scope, tree_handle tf.Output) (is_initialized tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "TensorForestTreeIsInitializedOp",
+		Input: []tf.Input{
+			tree_handle,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Gets next element for the provided shard number.
 //
 // Arguments:
@@ -10977,7 +11429,6 @@ func OneHotAxis(value int64) OneHotAttr {
 // =========
 //
 // Suppose that
-//
 // ```
 //   indices = [0, 2, -1, 1]
 //   depth = 3
@@ -10987,16 +11438,15 @@ func OneHotAxis(value int64) OneHotAttr {
 // ```
 //
 // Then output is `[4 x 3]`:
-//
-//     ```output =
-//       [5.0 0.0 0.0]  // one_hot(0)
-//       [0.0 0.0 5.0]  // one_hot(2)
-//       [0.0 0.0 0.0]  // one_hot(-1)
-//       [0.0 5.0 0.0]  // one_hot(1)
-//     ```
+// ```
+// output =
+//   [5.0 0.0 0.0]  // one_hot(0)
+//   [0.0 0.0 5.0]  // one_hot(2)
+//   [0.0 0.0 0.0]  // one_hot(-1)
+//   [0.0 5.0 0.0]  // one_hot(1)
+// ```
 //
 // Suppose that
-//
 // ```
 //   indices = [0, 2, -1, 1]
 //   depth = 3
@@ -11006,19 +11456,19 @@ func OneHotAxis(value int64) OneHotAttr {
 // ```
 //
 // Then output is `[3 x 4]`:
+// ```
+// output =
+//   [0.0 3.0 3.0 3.0]
+//   [3.0 3.0 3.0 0.0]
+//   [3.0 3.0 3.0 3.0]
+//   [3.0 0.0 3.0 3.0]
+// //  ^                one_hot(0)
+// //      ^            one_hot(2)
+// //          ^        one_hot(-1)
+// //              ^    one_hot(1)
+// ```
 //
-//     ```output =
-//       [0.0 3.0 3.0 3.0]
-//       [3.0 3.0 3.0 0.0]
-//       [3.0 3.0 3.0 3.0]
-//       [3.0 0.0 3.0 3.0]
-//     //  ^                one_hot(0)
-//     //      ^            one_hot(2)
-//     //          ^        one_hot(-1)
-//     //              ^    one_hot(1)
-//     ```
 // Suppose that
-//
 // ```
 //   indices = [[0, 2], [1, -1]]
 //   depth = 3
@@ -11028,15 +11478,16 @@ func OneHotAxis(value int64) OneHotAttr {
 // ```
 //
 // Then output is `[2 x 2 x 3]`:
-//
-//     ```output =
-//       [
-//         [1.0, 0.0, 0.0]  // one_hot(0)
-//         [0.0, 0.0, 1.0]  // one_hot(2)
-//       ][
-//         [0.0, 1.0, 0.0]  // one_hot(1)
-//         [0.0, 0.0, 0.0]  // one_hot(-1)
-//       ]```
+// ```
+// output =
+//   [
+//     [1.0, 0.0, 0.0]  // one_hot(0)
+//     [0.0, 0.0, 1.0]  // one_hot(2)
+//   ][
+//     [0.0, 1.0, 0.0]  // one_hot(1)
+//     [0.0, 0.0, 0.0]  // one_hot(-1)
+//   ]
+// ```
 //
 // Arguments:
 //	indices: A tensor of indices.
@@ -11594,8 +12045,8 @@ func ResizeBicubic(scope *Scope, images tf.Output, size tf.Output, optional ...R
 // Arguments:
 //	params_nested_splits: The `nested_row_splits` tensors that define the row-partitioning for the
 // `params` RaggedTensor input.
-//	params_dense_values: The `inner_values` for the `params` RaggedTensor. There was a terminology change
-// at the python level from dense_values to inner_values, so dense_values is the
+//	params_dense_values: The `flat_values` for the `params` RaggedTensor. There was a terminology change
+// at the python level from dense_values to flat_values, so dense_values is the
 // deprecated name.
 //	indices: Indices in the outermost dimension of `params` of the values that should be
 // gathered.
@@ -11604,7 +12055,7 @@ func ResizeBicubic(scope *Scope, images tf.Output, size tf.Output, optional ...R
 // `indices.shape.ndims + params.ragged_rank - 1`.
 //
 // Returns The `nested_row_splits` tensors that define the row-partitioning for the
-// returned RaggedTensor.The `inner_values` for the returned RaggedTensor.
+// returned RaggedTensor.The `flat_values` for the returned RaggedTensor.
 func RaggedGather(scope *Scope, params_nested_splits []tf.Output, params_dense_values tf.Output, indices tf.Output, OUTPUT_RAGGED_RANK int64) (output_nested_splits []tf.Output, output_dense_values tf.Output) {
 	if scope.Err() != nil {
 		return
@@ -11685,7 +12136,7 @@ func NonMaxSuppressionV2(scope *Scope, boxes tf.Output, scores tf.Output, max_ou
 //
 // Arguments:
 //	rt_nested_splits: The `row_splits` for the `RaggedTensor`.
-//	rt_dense_values: The `inner_values` for the `RaggedTensor`.
+//	rt_dense_values: The `flat_values` for the `RaggedTensor`.
 //
 // Returns The indices for the `SparseTensor`.The values of the `SparseTensor`.`sparse_dense_shape` is a tight bounding box of the input `RaggedTensor`.
 func RaggedTensorToSparse(scope *Scope, rt_nested_splits []tf.Output, rt_dense_values tf.Output) (sparse_indices tf.Output, sparse_values tf.Output, sparse_dense_shape tf.Output) {
@@ -12936,9 +13387,7 @@ func ResourceScatterNdAddUseLocking(value bool) ResourceScatterNdAddAttr {
 	}
 }
 
-// Adds sparse `updates` to individual values or slices within a given
-//
-// variable according to `indices`.
+// Applies sparse addition to individual values or slices in a Variable.
 //
 // `ref` is a `Tensor` with rank `P` and `indices` is a `Tensor` of rank `Q`.
 //
@@ -12952,24 +13401,24 @@ func ResourceScatterNdAddUseLocking(value bool) ResourceScatterNdAddAttr {
 // `updates` is `Tensor` of rank `Q-1+P-K` with shape:
 //
 // ```
-// [d_0, ..., d_{Q-2}, ref.shape[K], ..., ref.shape[P-1]].
+// [d_0, ..., d_{Q-2}, ref.shape[K], ..., ref.shape[P-1]]
 // ```
 //
-// For example, say we want to update 4 scattered elements to a rank-1 tensor to
-// 8 elements. In Python, that update would look like this:
+// For example, say we want to add 4 scattered elements to a rank-1 tensor to
+// 8 elements. In Python, that addition would look like this:
 //
 // ```python
-//     ref = tf.Variable([1, 2, 3, 4, 5, 6, 7, 8], use_resource=True)
-//     indices = tf.constant([[4], [3], [1] ,[7]])
-//     updates = tf.constant([9, 10, 11, 12])
-//     update = tf.scatter_nd_add(ref, indices, updates)
-//     with tf.Session() as sess:
-//       print sess.run(update)
+// ref = tf.Variable([1, 2, 3, 4, 5, 6, 7, 8], use_resource=True)
+// indices = tf.constant([[4], [3], [1], [7]])
+// updates = tf.constant([9, 10, 11, 12])
+// add = tf.scatter_nd_add(ref, indices, updates)
+// with tf.Session() as sess:
+//   print sess.run(add)
 // ```
 //
 // The resulting update to ref would look like this:
 //
-//     [1, 12, 3, 14, 14, 6, 7, 20]
+//     [1, 13, 3, 14, 14, 6, 7, 20]
 //
 // See `tf.scatter_nd` for more details about how to make updates to
 // slices.
@@ -13274,6 +13723,91 @@ func StatelessRandomNormal(scope *Scope, shape tf.Output, seed tf.Output, option
 	return op.Output(0)
 }
 
+// UnicodeDecodeAttr is an optional argument to UnicodeDecode.
+type UnicodeDecodeAttr func(optionalAttr)
+
+// UnicodeDecodeErrors sets the optional errors attribute to value.
+//
+// value: Error handling policy when there is invalid formatting found in the input.
+// The value of 'strict' will cause the operation to produce a InvalidArgument
+// error on any invalid input formatting. A value of 'replace' (the default) will
+// cause the operation to replace any invalid formatting in the input with the
+// `replacement_char` codepoint. A value of 'ignore' will cause the operation to
+// skip any invalid formatting in the input and produce no corresponding output
+// character.
+// If not specified, defaults to "replace"
+func UnicodeDecodeErrors(value string) UnicodeDecodeAttr {
+	return func(m optionalAttr) {
+		m["errors"] = value
+	}
+}
+
+// UnicodeDecodeReplacementChar sets the optional replacement_char attribute to value.
+//
+// value: The replacement character codepoint to be used in place of any invalid
+// formatting in the input when `errors='replace'`. Any valid unicode codepoint may
+// be used. The default value is the default unicode replacement character is
+// 0xFFFD or U+65533.)
+// If not specified, defaults to 65533
+func UnicodeDecodeReplacementChar(value int64) UnicodeDecodeAttr {
+	return func(m optionalAttr) {
+		m["replacement_char"] = value
+	}
+}
+
+// UnicodeDecodeReplaceControlCharacters sets the optional replace_control_characters attribute to value.
+//
+// value: Whether to replace the C0 control characters (00-1F) with the
+// `replacement_char`. Default is false.
+// If not specified, defaults to false
+func UnicodeDecodeReplaceControlCharacters(value bool) UnicodeDecodeAttr {
+	return func(m optionalAttr) {
+		m["replace_control_characters"] = value
+	}
+}
+
+// Decodes each string in `input` into a sequence of Unicode code points.
+//
+// The character codepoints for all strings are returned using a single vector
+// `char_values`, with strings expanded to characters in row-major order.
+//
+// The `row_splits` tensor indicates where the codepoints for
+// each input string begin and end within the `char_values` tensor.
+// In particular, the values for the `i`th
+// string (in row-major order) are stored in the slice
+// `[row_splits[i]:row_splits[i+1]]`. Thus:
+//
+// * `char_values[row_splits[i]+j]` is the Unicode codepoint for the `j`th
+//   character in the `i`th string (in row-major order).
+// * `row_splits[i+1] - row_splits[i]` is the number of characters in the `i`th
+//   string (in row-major order).
+//
+// Arguments:
+//	input: The text to be decoded. Can have any shape. Note that the output is flattened
+// to a vector of char values.
+//	input_encoding: Text encoding of the input strings. This is any of the encodings supported
+// by ICU ucnv algorithmic converters. Examples: `"UTF-16", "US ASCII", "UTF-8"`.
+//
+// Returns A 1D int32 tensor containing the row splits.A 1D int32 Tensor containing the decoded codepoints.
+func UnicodeDecode(scope *Scope, input tf.Output, input_encoding string, optional ...UnicodeDecodeAttr) (row_splits tf.Output, char_values tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"input_encoding": input_encoding}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "UnicodeDecode",
+		Input: []tf.Input{
+			input,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1)
+}
+
 // Adds up a SparseTensor and a dense Tensor, using these special rules:
 //
 // (1) Broadcasts the dense side to have the same shape as the sparse side, if
@@ -13322,6 +13856,84 @@ func Erfc(scope *Scope, x tf.Output) (y tf.Output) {
 	return op.Output(0)
 }
 
+// UnicodeEncodeAttr is an optional argument to UnicodeEncode.
+type UnicodeEncodeAttr func(optionalAttr)
+
+// UnicodeEncodeErrors sets the optional errors attribute to value.
+//
+// value: Error handling policy when there is invalid formatting found in the input.
+// The value of 'strict' will cause the operation to produce a InvalidArgument
+// error on any invalid input formatting. A value of 'replace' (the default) will
+// cause the operation to replace any invalid formatting in the input with the
+// `replacement_char` codepoint. A value of 'ignore' will cause the operation to
+// skip any invalid formatting in the input and produce no corresponding output
+// character.
+// If not specified, defaults to "replace"
+func UnicodeEncodeErrors(value string) UnicodeEncodeAttr {
+	return func(m optionalAttr) {
+		m["errors"] = value
+	}
+}
+
+// UnicodeEncodeReplacementChar sets the optional replacement_char attribute to value.
+//
+// value: The replacement character codepoint to be used in place of any invalid
+// formatting in the input when `errors='replace'`. Any valid unicode codepoint may
+// be used. The default value is the default unicode replacement character is
+// 0xFFFD (U+65533).
+// If not specified, defaults to 65533
+func UnicodeEncodeReplacementChar(value int64) UnicodeEncodeAttr {
+	return func(m optionalAttr) {
+		m["replacement_char"] = value
+	}
+}
+
+// Encode a tensor of ints into unicode strings.
+//
+// Returns a vector of strings, where `output[i]` is constructed by encoding the
+// Unicode codepoints in `input_values[input_splits[i]:input_splits[i+1]]`
+// using `output_encoding`.
+//
+// ---
+//
+// Example:
+//
+// ```
+// input_values = [72, 101, 108, 108, 111, 87, 111, 114, 108, 100]
+// input_splits = [0, 5, 10]
+// output_encoding = 'UTF-8'
+//
+// output = ['Hello', 'World']
+// ```
+//
+// Arguments:
+//	input_values: A 1D tensor containing the unicode codepoints that should be encoded.
+//	input_splits: A 1D tensor specifying how the unicode codepoints should be split into strings.
+// In particular, `output[i]` is constructed by encoding the codepoints in the
+// slice `input_values[input_splits[i]:input_splits[i+1]]`.
+//	output_encoding: Unicode encoding of the output strings. Valid encodings are: `"UTF-8",
+// "UTF-16-BE", and "UTF-32-BE"`.
+//
+// Returns The 1-D Tensor of strings encoded from the provided unicode codepoints.
+func UnicodeEncode(scope *Scope, input_values tf.Output, input_splits tf.Output, output_encoding string, optional ...UnicodeEncodeAttr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"output_encoding": output_encoding}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "UnicodeEncode",
+		Input: []tf.Input{
+			input_values, input_splits,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Returns the number of tensors in the input tensor list.
 //
 // input_handle: the input list
@@ -14644,6 +15256,117 @@ func MultiDeviceIteratorToStringHandle(scope *Scope, multi_device_iterator tf.Ou
 	return op.Output(0)
 }
 
+// CudnnRNNV3Attr is an optional argument to CudnnRNNV3.
+type CudnnRNNV3Attr func(optionalAttr)
+
+// CudnnRNNV3RnnMode sets the optional rnn_mode attribute to value.
+// If not specified, defaults to "lstm"
+func CudnnRNNV3RnnMode(value string) CudnnRNNV3Attr {
+	return func(m optionalAttr) {
+		m["rnn_mode"] = value
+	}
+}
+
+// CudnnRNNV3InputMode sets the optional input_mode attribute to value.
+// If not specified, defaults to "linear_input"
+func CudnnRNNV3InputMode(value string) CudnnRNNV3Attr {
+	return func(m optionalAttr) {
+		m["input_mode"] = value
+	}
+}
+
+// CudnnRNNV3Direction sets the optional direction attribute to value.
+// If not specified, defaults to "unidirectional"
+func CudnnRNNV3Direction(value string) CudnnRNNV3Attr {
+	return func(m optionalAttr) {
+		m["direction"] = value
+	}
+}
+
+// CudnnRNNV3Dropout sets the optional dropout attribute to value.
+// If not specified, defaults to 0
+func CudnnRNNV3Dropout(value float32) CudnnRNNV3Attr {
+	return func(m optionalAttr) {
+		m["dropout"] = value
+	}
+}
+
+// CudnnRNNV3Seed sets the optional seed attribute to value.
+// If not specified, defaults to 0
+func CudnnRNNV3Seed(value int64) CudnnRNNV3Attr {
+	return func(m optionalAttr) {
+		m["seed"] = value
+	}
+}
+
+// CudnnRNNV3Seed2 sets the optional seed2 attribute to value.
+// If not specified, defaults to 0
+func CudnnRNNV3Seed2(value int64) CudnnRNNV3Attr {
+	return func(m optionalAttr) {
+		m["seed2"] = value
+	}
+}
+
+// CudnnRNNV3IsTraining sets the optional is_training attribute to value.
+// If not specified, defaults to true
+func CudnnRNNV3IsTraining(value bool) CudnnRNNV3Attr {
+	return func(m optionalAttr) {
+		m["is_training"] = value
+	}
+}
+
+// A RNN backed by cuDNN.
+//
+// Computes the RNN from the input and initial states, with respect to the params
+// buffer. Accepts one extra input "sequence_lengths" than CudnnRNN.
+//
+// rnn_mode: Indicates the type of the RNN model.
+// input_mode: Indicates whether there is a linear projection between the input and
+//   the actual computation before the first layer. 'skip_input' is only allowed
+//   when input_size == num_units; 'auto_select' implies 'skip_input' when
+//   input_size == num_units; otherwise, it implies 'linear_input'.
+// direction: Indicates whether a bidirectional model will be used. Should be
+//   "unidirectional" or "bidirectional".
+// dropout: Dropout probability. When set to 0., dropout is disabled.
+// seed: The 1st part of a seed to initialize dropout.
+// seed2: The 2nd part of a seed to initialize dropout.
+// input: A 3-D tensor with the shape of [seq_length, batch_size, input_size].
+// input_h: A 3-D tensor with the shape of [num_layer * dir, batch_size,
+//     num_units].
+// input_c: For LSTM, a 3-D tensor with the shape of
+//     [num_layer * dir, batch, num_units]. For other models, it is ignored.
+// params: A 1-D tensor that contains the weights and biases in an opaque layout.
+//     The size must be created through CudnnRNNParamsSize, and initialized
+//     separately. Note that they might not be compatible across different
+//     generations. So it is a good idea to save and restore
+// sequence_lengths: a vector of lengths of each input sequence.
+// output: A 3-D tensor with the shape of [seq_length, batch_size,
+//     dir * num_units].
+// output_h: The same shape has input_h.
+// output_c: The same shape as input_c for LSTM. An empty tensor for other models.
+// is_training: Indicates whether this operation is used for inferenece or
+//   training.
+// reserve_space: An opaque tensor that can be used in backprop calculation. It
+//   is only produced if is_training is true.
+func CudnnRNNV3(scope *Scope, input tf.Output, input_h tf.Output, input_c tf.Output, params tf.Output, sequence_lengths tf.Output, optional ...CudnnRNNV3Attr) (output tf.Output, output_h tf.Output, output_c tf.Output, reserve_space tf.Output, host_reserved tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "CudnnRNNV3",
+		Input: []tf.Input{
+			input, input_h, input_c, params, sequence_lengths,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1), op.Output(2), op.Output(3), op.Output(4)
+}
+
 // Applies softmax to a batched N-D `SparseTensor`.
 //
 // The inputs represent an N-D SparseTensor  with logical shape `[..., B, C]`
@@ -16873,6 +17596,23 @@ func SparseReduceSum(scope *Scope, input_indices tf.Output, input_values tf.Outp
 	return op.Output(0)
 }
 
+// Records the latency of producing `input_dataset` elements in a StatsAggregator.
+func ExperimentalLatencyStatsDataset(scope *Scope, input_dataset tf.Output, tag tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	opspec := tf.OpSpec{
+		Type: "ExperimentalLatencyStatsDataset",
+		Input: []tf.Input{
+			input_dataset, tag,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // SparseTensorDenseMatMulAttr is an optional argument to SparseTensorDenseMatMul.
 type SparseTensorDenseMatMulAttr func(optionalAttr)
 
@@ -17555,6 +18295,69 @@ func Timestamp(scope *Scope) (ts tf.Output) {
 	return op.Output(0)
 }
 
+// ResourceSparseApplyKerasMomentumAttr is an optional argument to ResourceSparseApplyKerasMomentum.
+type ResourceSparseApplyKerasMomentumAttr func(optionalAttr)
+
+// ResourceSparseApplyKerasMomentumUseLocking sets the optional use_locking attribute to value.
+//
+// value: If `True`, updating of the var and accum tensors will be protected
+// by a lock; otherwise the behavior is undefined, but may exhibit less
+// contention.
+// If not specified, defaults to false
+func ResourceSparseApplyKerasMomentumUseLocking(value bool) ResourceSparseApplyKerasMomentumAttr {
+	return func(m optionalAttr) {
+		m["use_locking"] = value
+	}
+}
+
+// ResourceSparseApplyKerasMomentumUseNesterov sets the optional use_nesterov attribute to value.
+//
+// value: If `True`, the tensor passed to compute grad will be
+// var + momentum * accum, so in the end, the var you get is actually
+// var + momentum * accum.
+// If not specified, defaults to false
+func ResourceSparseApplyKerasMomentumUseNesterov(value bool) ResourceSparseApplyKerasMomentumAttr {
+	return func(m optionalAttr) {
+		m["use_nesterov"] = value
+	}
+}
+
+// Update relevant entries in '*var' and '*accum' according to the momentum scheme.
+//
+// Set use_nesterov = True if you want to use Nesterov momentum.
+//
+// That is for rows we have grad for, we update var and accum as follows:
+//
+// accum = accum * momentum - lr * grad
+// var += accum
+//
+// Arguments:
+//	var_: Should be from a Variable().
+//	accum: Should be from a Variable().
+//	lr: Learning rate. Must be a scalar.
+//	grad: The gradient.
+//	indices: A vector of indices into the first dimension of var and accum.
+//	momentum: Momentum. Must be a scalar.
+//
+// Returns the created operation.
+func ResourceSparseApplyKerasMomentum(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.Output, grad tf.Output, indices tf.Output, momentum tf.Output, optional ...ResourceSparseApplyKerasMomentumAttr) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ResourceSparseApplyKerasMomentum",
+		Input: []tf.Input{
+			var_, accum, lr, grad, indices, momentum,
+		},
+		Attrs: attrs,
+	}
+	return scope.AddOperation(opspec)
+}
+
 // VariableShapeAttr is an optional argument to VariableShape.
 type VariableShapeAttr func(optionalAttr)
 
@@ -18293,6 +19096,96 @@ func DivNoNan(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	return op.Output(0)
 }
 
+// UnicodeDecodeWithOffsetsAttr is an optional argument to UnicodeDecodeWithOffsets.
+type UnicodeDecodeWithOffsetsAttr func(optionalAttr)
+
+// UnicodeDecodeWithOffsetsErrors sets the optional errors attribute to value.
+//
+// value: Error handling policy when there is invalid formatting found in the input.
+// The value of 'strict' will cause the operation to produce a InvalidArgument
+// error on any invalid input formatting. A value of 'replace' (the default) will
+// cause the operation to replace any invalid formatting in the input with the
+// `replacement_char` codepoint. A value of 'ignore' will cause the operation to
+// skip any invalid formatting in the input and produce no corresponding output
+// character.
+// If not specified, defaults to "replace"
+func UnicodeDecodeWithOffsetsErrors(value string) UnicodeDecodeWithOffsetsAttr {
+	return func(m optionalAttr) {
+		m["errors"] = value
+	}
+}
+
+// UnicodeDecodeWithOffsetsReplacementChar sets the optional replacement_char attribute to value.
+//
+// value: The replacement character codepoint to be used in place of any invalid
+// formatting in the input when `errors='replace'`. Any valid unicode codepoint may
+// be used. The default value is the default unicode replacement character is
+// 0xFFFD or U+65533.)
+// If not specified, defaults to 65533
+func UnicodeDecodeWithOffsetsReplacementChar(value int64) UnicodeDecodeWithOffsetsAttr {
+	return func(m optionalAttr) {
+		m["replacement_char"] = value
+	}
+}
+
+// UnicodeDecodeWithOffsetsReplaceControlCharacters sets the optional replace_control_characters attribute to value.
+//
+// value: Whether to replace the C0 control characters (00-1F) with the
+// `replacement_char`. Default is false.
+// If not specified, defaults to false
+func UnicodeDecodeWithOffsetsReplaceControlCharacters(value bool) UnicodeDecodeWithOffsetsAttr {
+	return func(m optionalAttr) {
+		m["replace_control_characters"] = value
+	}
+}
+
+// Decodes each string in `input` into a sequence of Unicode code points.
+//
+// The character codepoints for all strings are returned using a single vector
+// `char_values`, with strings expanded to characters in row-major order.
+// Similarly, the character start byte offsets are returned using a single vector
+// `char_to_byte_starts`, with strings expanded in row-major order.
+//
+// The `row_splits` tensor indicates where the codepoints and start offsets for
+// each input string begin and end within the `char_values` and
+// `char_to_byte_starts` tensors.  In particular, the values for the `i`th
+// string (in row-major order) are stored in the slice
+// `[row_splits[i]:row_splits[i+1]]`. Thus:
+//
+// * `char_values[row_splits[i]+j]` is the Unicode codepoint for the `j`th
+//   character in the `i`th string (in row-major order).
+// * `char_to_bytes_starts[row_splits[i]+j]` is the start byte offset for the `j`th
+//   character in the `i`th string (in row-major order).
+// * `row_splits[i+1] - row_splits[i]` is the number of characters in the `i`th
+//   string (in row-major order).
+//
+// Arguments:
+//	input: The text to be decoded. Can have any shape. Note that the output is flattened
+// to a vector of char values.
+//	input_encoding: Text encoding of the input strings. This is any of the encodings supported
+// by ICU ucnv algorithmic converters. Examples: `"UTF-16", "US ASCII", "UTF-8"`.
+//
+// Returns A 1D int32 tensor containing the row splits.A 1D int32 Tensor containing the decoded codepoints.A 1D int32 Tensor containing the byte index in the input string where each
+// character in `char_values` starts.
+func UnicodeDecodeWithOffsets(scope *Scope, input tf.Output, input_encoding string, optional ...UnicodeDecodeWithOffsetsAttr) (row_splits tf.Output, char_values tf.Output, char_to_byte_starts tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"input_encoding": input_encoding}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "UnicodeDecodeWithOffsets",
+		Input: []tf.Input{
+			input,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1), op.Output(2)
+}
+
 // Returns x - y element-wise.
 //
 // *NOTE*: `Subtract` supports broadcasting. More about broadcasting
@@ -20606,15 +21499,62 @@ func Zeta(scope *Scope, x tf.Output, q tf.Output) (z tf.Output) {
 	return op.Output(0)
 }
 
+// Returns the cardinality of `input_dataset`.
+//
+// Returns the cardinality of `input_dataset`.
+//
+// Arguments:
+//	input_dataset: A variant tensor representing the dataset to return cardinality for.
+//
+// Returns The cardinality of `input_dataset`. Named constants are used to represent
+// infinite and unknown cardinality.
+func ExperimentalDatasetCardinality(scope *Scope, input_dataset tf.Output) (cardinality tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "ExperimentalDatasetCardinality",
+		Input: []tf.Input{
+			input_dataset,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Creates a dataset that executes a SQL query and emits rows of the result set.
+//
+// Arguments:
+//	driver_name: The database type. Currently, the only supported type is 'sqlite'.
+//	data_source_name: A connection string to connect to the database.
+//	query: A SQL query to execute.
+//
+//
+func ExperimentalSqlDataset(scope *Scope, driver_name tf.Output, data_source_name tf.Output, query tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	opspec := tf.OpSpec{
+		Type: "ExperimentalSqlDataset",
+		Input: []tf.Input{
+			driver_name, data_source_name, query,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Inverse fast Fourier transform.
 //
 // Computes the inverse 1-dimensional discrete Fourier transform over the
 // inner-most dimension of `input`.
 //
 // Arguments:
-//	input: A complex64 tensor.
+//	input: A complex tensor.
 //
-// Returns A complex64 tensor of the same shape as `input`. The inner-most
+// Returns A complex tensor of the same shape as `input`. The inner-most
 //   dimension of `input` is replaced with its inverse 1D Fourier transform.
 //
 // @compatibility(numpy)
@@ -20640,9 +21580,9 @@ func IFFT(scope *Scope, input tf.Output) (output tf.Output) {
 // 2 dimensions of `input`.
 //
 // Arguments:
-//	input: A complex64 tensor.
+//	input: A complex tensor.
 //
-// Returns A complex64 tensor of the same shape as `input`. The inner-most 2
+// Returns A complex tensor of the same shape as `input`. The inner-most 2
 //   dimensions of `input` are replaced with their 2D Fourier transform.
 //
 // @compatibility(numpy)
@@ -20668,9 +21608,9 @@ func FFT2D(scope *Scope, input tf.Output) (output tf.Output) {
 // inner-most 2 dimensions of `input`.
 //
 // Arguments:
-//	input: A complex64 tensor.
+//	input: A complex tensor.
 //
-// Returns A complex64 tensor of the same shape as `input`. The inner-most 2
+// Returns A complex tensor of the same shape as `input`. The inner-most 2
 //   dimensions of `input` are replaced with their inverse 2D Fourier transform.
 //
 // @compatibility(numpy)
@@ -21010,6 +21950,44 @@ func ReaderNumRecordsProducedV2(scope *Scope, reader_handle tf.Output) (records_
 	return op.Output(0)
 }
 
+// TensorListConcatAttr is an optional argument to TensorListConcat.
+type TensorListConcatAttr func(optionalAttr)
+
+// TensorListConcatElementShape sets the optional element_shape attribute to value.
+// If not specified, defaults to <unknown_rank:true >
+func TensorListConcatElementShape(value tf.Shape) TensorListConcatAttr {
+	return func(m optionalAttr) {
+		m["element_shape"] = value
+	}
+}
+
+// Concats all tensors in the list along the 0th dimension.
+//
+// Requires that all tensors have the same shape except the first dimension.
+//
+// input_handle: The input list.
+// tensor: The concated result.
+// lengths: Output tensor containing sizes of the 0th dimension of tensors in the list, used for computing the gradient.
+//
+func TensorListConcat(scope *Scope, input_handle tf.Output, element_dtype tf.DataType, optional ...TensorListConcatAttr) (tensor tf.Output, lengths tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"element_dtype": element_dtype}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "TensorListConcat",
+		Input: []tf.Input{
+			input_handle,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1)
+}
+
 // Returns the set of files matching one or more glob patterns.
 //
 // Note that this routine only supports wildcard characters in the
@@ -22051,6 +23029,61 @@ func SparseSparseMinimum(scope *Scope, a_indices tf.Output, a_values tf.Output,
 	return op.Output(0), op.Output(1)
 }
 
+// ResourceApplyAdamWithAmsgradAttr is an optional argument to ResourceApplyAdamWithAmsgrad.
+type ResourceApplyAdamWithAmsgradAttr func(optionalAttr)
+
+// ResourceApplyAdamWithAmsgradUseLocking sets the optional use_locking attribute to value.
+//
+// value: If `True`, updating of the var, m, and v tensors will be protected
+// by a lock; otherwise the behavior is undefined, but may exhibit less
+// contention.
+// If not specified, defaults to false
+func ResourceApplyAdamWithAmsgradUseLocking(value bool) ResourceApplyAdamWithAmsgradAttr {
+	return func(m optionalAttr) {
+		m["use_locking"] = value
+	}
+}
+
+// Update '*var' according to the Adam algorithm.
+//
+// $$lr_t := \text{learning\_rate} * \sqrt{1 - beta_2^t} / (1 - beta_1^t)$$
+// $$m_t := beta_1 * m_{t-1} + (1 - beta_1) * g$$
+// $$v_t := beta_2 * v_{t-1} + (1 - beta_2) * g * g$$
+// $$vhat_t := max{vhat_{t-1}, v_t}$$
+// $$variable := variable - lr_t * m_t / (\sqrt{vhat_t} + \epsilon)$$
+//
+// Arguments:
+//	var_: Should be from a Variable().
+//	m: Should be from a Variable().
+//	v: Should be from a Variable().
+//	vhat: Should be from a Variable().
+//	beta1_power: Must be a scalar.
+//	beta2_power: Must be a scalar.
+//	lr: Scaling factor. Must be a scalar.
+//	beta1: Momentum factor. Must be a scalar.
+//	beta2: Momentum factor. Must be a scalar.
+//	epsilon: Ridge term. Must be a scalar.
+//	grad: The gradient.
+//
+// Returns the created operation.
+func ResourceApplyAdamWithAmsgrad(scope *Scope, var_ tf.Output, m tf.Output, v tf.Output, vhat tf.Output, beta1_power tf.Output, beta2_power tf.Output, lr tf.Output, beta1 tf.Output, beta2 tf.Output, epsilon tf.Output, grad tf.Output, optional ...ResourceApplyAdamWithAmsgradAttr) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ResourceApplyAdamWithAmsgrad",
+		Input: []tf.Input{
+			var_, m, v, vhat, beta1_power, beta2_power, lr, beta1, beta2, epsilon, grad,
+		},
+		Attrs: attrs,
+	}
+	return scope.AddOperation(opspec)
+}
+
 // MapUnstageNoKeyAttr is an optional argument to MapUnstageNoKey.
 type MapUnstageNoKeyAttr func(optionalAttr)
 
@@ -22452,6 +23485,93 @@ func SetSize(scope *Scope, set_indices tf.Output, set_values tf.Output, set_shap
 	return op.Output(0)
 }
 
+// Adds sparse `updates` to an existing tensor according to `indices`.
+//
+// This operation creates a new tensor by adding sparse `updates` to the passed
+// in `tensor`.
+// This operation is very similar to `tf.scatter_nd_add`, except that the updates
+// are added onto an existing tensor (as opposed to a variable). If the memory
+// for the existing tensor cannot be re-used, a copy is made and updated.
+//
+// `indices` is an integer tensor containing indices into a new tensor of shape
+// `shape`.  The last dimension of `indices` can be at most the rank of `shape`:
+//
+//     indices.shape[-1] <= shape.rank
+//
+// The last dimension of `indices` corresponds to indices into elements
+// (if `indices.shape[-1] = shape.rank`) or slices
+// (if `indices.shape[-1] < shape.rank`) along dimension `indices.shape[-1]` of
+// `shape`.  `updates` is a tensor with shape
+//
+//     indices.shape[:-1] + shape[indices.shape[-1]:]
+//
+// The simplest form of tensor_scatter_add is to add individual elements to a
+// tensor by index. For example, say we want to add 4 elements in a rank-1
+// tensor with 8 elements.
+//
+// In Python, this scatter add operation would look like this:
+//
+// ```python
+//     indices = tf.constant([[4], [3], [1], [7]])
+//     updates = tf.constant([9, 10, 11, 12])
+//     tensor = tf.ones([8], dtype=tf.int32)
+//     updated = tf.tensor_scatter_add(tensor, indices, updates)
+//     with tf.Session() as sess:
+//       print(sess.run(scatter))
+// ```
+//
+// The resulting tensor would look like this:
+//
+//     [1, 12, 1, 11, 10, 1, 1, 13]
+//
+// We can also, insert entire slices of a higher rank tensor all at once. For
+// example, if we wanted to insert two slices in the first dimension of a
+// rank-3 tensor with two matrices of new values.
+//
+// In Python, this scatter add operation would look like this:
+//
+// ```python
+//     indices = tf.constant([[0], [2]])
+//     updates = tf.constant([[[5, 5, 5, 5], [6, 6, 6, 6],
+//                             [7, 7, 7, 7], [8, 8, 8, 8]],
+//                            [[5, 5, 5, 5], [6, 6, 6, 6],
+//                             [7, 7, 7, 7], [8, 8, 8, 8]]])
+//     tensor = tf.ones([4, 4, 4])
+//     updated = tf.tensor_scatter_add(tensor, indices, updates)
+//     with tf.Session() as sess:
+//       print(sess.run(scatter))
+// ```
+//
+// The resulting tensor would look like this:
+//
+//     [[[6, 6, 6, 6], [7, 7, 7, 7], [8, 8, 8, 8], [9, 9, 9, 9]],
+//      [[1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1]],
+//      [[6, 6, 6, 6], [7, 7, 7, 7], [8, 8, 8, 8], [9, 9, 9, 9]],
+//      [[1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1]]]
+//
+// Note that on CPU, if an out of bound index is found, an error is returned.
+// On GPU, if an out of bound index is found, the index is ignored.
+//
+// Arguments:
+//	tensor: Tensor to copy/update.
+//	indices: Index tensor.
+//	updates: Updates to scatter into output.
+//
+// Returns A new tensor copied from tensor and updates added according to the indices.
+func TensorScatterAdd(scope *Scope, tensor tf.Output, indices tf.Output, updates tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "TensorScatterAdd",
+		Input: []tf.Input{
+			tensor, indices, updates,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Computes the sign and the log of the absolute value of the determinant of
 //
 // one or more square matrices.
@@ -22676,6 +23796,84 @@ func QuantizedResizeBilinear(scope *Scope, images tf.Output, size tf.Output, min
 	return op.Output(0), op.Output(1), op.Output(2)
 }
 
+// Creates a dataset that uses a custom thread pool to compute `input_dataset`.
+//
+// Arguments:
+//
+//	num_threads: Identifies the number of threads to use for the private threadpool.
+//
+//
+func ExperimentalPrivateThreadPoolDataset(scope *Scope, input_dataset tf.Output, num_threads tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	opspec := tf.OpSpec{
+		Type: "ExperimentalPrivateThreadPoolDataset",
+		Input: []tf.Input{
+			input_dataset, num_threads,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// ExperimentalParseExampleDatasetAttr is an optional argument to ExperimentalParseExampleDataset.
+type ExperimentalParseExampleDatasetAttr func(optionalAttr)
+
+// ExperimentalParseExampleDatasetSloppy sets the optional sloppy attribute to value.
+// If not specified, defaults to false
+func ExperimentalParseExampleDatasetSloppy(value bool) ExperimentalParseExampleDatasetAttr {
+	return func(m optionalAttr) {
+		m["sloppy"] = value
+	}
+}
+
+// Transforms `input_dataset` containing `Example` protos as vectors of DT_STRING into a dataset of `Tensor` or `SparseTensor` objects representing the parsed features.
+//
+// Arguments:
+//
+//
+//	dense_defaults: A dict mapping string keys to `Tensor`s.
+// The keys of the dict must match the dense_keys of the feature.
+//	sparse_keys: A list of string keys in the examples features.
+// The results for these keys will be returned as `SparseTensor` objects.
+//	dense_keys: A list of Ndense string Tensors (scalars).
+// The keys expected in the Examples features associated with dense values.
+//	sparse_types: A list of `DTypes` of the same length as `sparse_keys`.
+// Only `tf.float32` (`FloatList`), `tf.int64` (`Int64List`),
+// and `tf.string` (`BytesList`) are supported.
+//	dense_shapes: List of tuples with the same length as `dense_keys`.
+// The shape of the data for each dense feature referenced by `dense_keys`.
+// Required for any input tensors identified by `dense_keys`.  Must be
+// either fully defined, or may contain an unknown first dimension.
+// An unknown first dimension means the feature is treated as having
+// a variable number of blocks, and the output shape along this dimension
+// is considered unknown at graph build time.  Padding is applied for
+// minibatch elements smaller than the maximum number of blocks for the
+// given feature along this dimension.
+//	output_types: The type list for the return values.
+//	output_shapes: The list of shapes being produced.
+func ExperimentalParseExampleDataset(scope *Scope, input_dataset tf.Output, num_parallel_calls tf.Output, dense_defaults []tf.Output, sparse_keys []string, dense_keys []string, sparse_types []tf.DataType, dense_shapes []tf.Shape, output_types []tf.DataType, output_shapes []tf.Shape, optional ...ExperimentalParseExampleDatasetAttr) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"sparse_keys": sparse_keys, "dense_keys": dense_keys, "sparse_types": sparse_types, "dense_shapes": dense_shapes, "output_types": output_types, "output_shapes": output_shapes}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ExperimentalParseExampleDataset",
+		Input: []tf.Input{
+			input_dataset, num_parallel_calls, tf.OutputList(dense_defaults),
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // SdcaOptimizerAttr is an optional argument to SdcaOptimizer.
 type SdcaOptimizerAttr func(optionalAttr)
 
@@ -23420,6 +24618,26 @@ func MatMul(scope *Scope, a tf.Output, b tf.Output, optional ...MatMulAttr) (pro
 	return op.Output(0)
 }
 
+// Serializes the tree handle to a proto
+//
+// Arguments:
+//	tree_handle: Handle to the tree resource to be serialized.
+//
+// Returns Serialied proto string of the tree resource.
+func TensorForestTreeSerialize(scope *Scope, tree_handle tf.Output) (tree_config tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "TensorForestTreeSerialize",
+		Input: []tf.Input{
+			tree_handle,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // SparseMatMulAttr is an optional argument to SparseMatMul.
 type SparseMatMulAttr func(optionalAttr)
 
@@ -26575,6 +27793,28 @@ func Invert(scope *Scope, x tf.Output) (y tf.Output) {
 	return op.Output(0)
 }
 
+// Deserialize bucket boundaries and ready flag into current QuantileAccumulator.
+//
+// An op that deserializes bucket boundaries and are boundaries ready flag into current QuantileAccumulator.
+//
+// Arguments:
+//	quantile_stream_resource_handle: resource handle referring to a QuantileStreamResource.
+//	bucket_boundaries: float; List of Rank 1 Tensors each containing the bucket boundaries for a feature.
+//
+// Returns the created operation.
+func BoostedTreesQuantileStreamResourceDeserialize(scope *Scope, quantile_stream_resource_handle tf.Output, bucket_boundaries []tf.Output) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "BoostedTreesQuantileStreamResourceDeserialize",
+		Input: []tf.Input{
+			quantile_stream_resource_handle, tf.OutputList(bucket_boundaries),
+		},
+	}
+	return scope.AddOperation(opspec)
+}
+
 // Inverse 3D fast Fourier transform.
 //
 // Computes the inverse 3-dimensional discrete Fourier transform over the
@@ -27102,6 +28342,29 @@ func AudioSummaryV2(scope *Scope, tag tf.Output, tensor tf.Output, sample_rate t
 	return op.Output(0)
 }
 
+// Splits a tensor into a list.
+//
+// list[i] corresponds to lengths[i] tensors from the input tensor.
+// The tensor must have rank at least 1 and contain exactly sum(lengths) elements.
+//
+// tensor: The input tensor.
+// element_shape: A shape compatible with that of elements in the tensor.
+// lengths: Vector of sizes of the 0th dimension of tensors in the list.
+// output_handle: The list.
+func TensorListSplit(scope *Scope, tensor tf.Output, element_shape tf.Output, lengths tf.Output) (output_handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "TensorListSplit",
+		Input: []tf.Input{
+			tensor, element_shape, lengths,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // AvgPoolAttr is an optional argument to AvgPool.
 type AvgPoolAttr func(optionalAttr)
 
@@ -27222,6 +28485,26 @@ func TensorListGetItem(scope *Scope, input_handle tf.Output, index tf.Output, el
 	return op.Output(0)
 }
 
+// Resizes the list.
+//
+//
+// input_handle: the input list
+// size: size of the output list
+//
+func TensorListResize(scope *Scope, input_handle tf.Output, size tf.Output) (output_handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "TensorListResize",
+		Input: []tf.Input{
+			input_handle, size,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Returns a diagonal tensor with a given diagonal values.
 //
 // Given a `diagonal`, this operation returns a tensor with the `diagonal` and
@@ -27412,7 +28695,7 @@ func TensorListScatter(scope *Scope, tensor tf.Output, indices tf.Output, elemen
 //	limits: The limits of each range.
 //	deltas: The deltas of each range.
 //
-// Returns The `row_splits` for the returned `RaggedTensor`.The `inner_values` for the returned `RaggedTensor`.
+// Returns The `row_splits` for the returned `RaggedTensor`.The `flat_values` for the returned `RaggedTensor`.
 func RaggedRange(scope *Scope, starts tf.Output, limits tf.Output, deltas tf.Output) (rt_nested_splits tf.Output, rt_dense_values tf.Output) {
 	if scope.Err() != nil {
 		return
@@ -27744,6 +29027,66 @@ func AdjustSaturation(scope *Scope, images tf.Output, scale tf.Output) (output t
 	return op.Output(0)
 }
 
+// ResourceApplyKerasMomentumAttr is an optional argument to ResourceApplyKerasMomentum.
+type ResourceApplyKerasMomentumAttr func(optionalAttr)
+
+// ResourceApplyKerasMomentumUseLocking sets the optional use_locking attribute to value.
+//
+// value: If `True`, updating of the var and accum tensors will be protected
+// by a lock; otherwise the behavior is undefined, but may exhibit less
+// contention.
+// If not specified, defaults to false
+func ResourceApplyKerasMomentumUseLocking(value bool) ResourceApplyKerasMomentumAttr {
+	return func(m optionalAttr) {
+		m["use_locking"] = value
+	}
+}
+
+// ResourceApplyKerasMomentumUseNesterov sets the optional use_nesterov attribute to value.
+//
+// value: If `True`, the tensor passed to compute grad will be
+// var + momentum * accum, so in the end, the var you get is actually
+// var + momentum * accum.
+// If not specified, defaults to false
+func ResourceApplyKerasMomentumUseNesterov(value bool) ResourceApplyKerasMomentumAttr {
+	return func(m optionalAttr) {
+		m["use_nesterov"] = value
+	}
+}
+
+// Update '*var' according to the momentum scheme. Set use_nesterov = True if you
+//
+// want to use Nesterov momentum.
+//
+// accum = accum * momentum - lr * grad
+// var += accum
+//
+// Arguments:
+//	var_: Should be from a Variable().
+//	accum: Should be from a Variable().
+//	lr: Scaling factor. Must be a scalar.
+//	grad: The gradient.
+//	momentum: Momentum. Must be a scalar.
+//
+// Returns the created operation.
+func ResourceApplyKerasMomentum(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.Output, grad tf.Output, momentum tf.Output, optional ...ResourceApplyKerasMomentumAttr) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ResourceApplyKerasMomentum",
+		Input: []tf.Input{
+			var_, accum, lr, grad, momentum,
+		},
+		Attrs: attrs,
+	}
+	return scope.AddOperation(opspec)
+}
+
 // MatrixSolveAttr is an optional argument to MatrixSolve.
 type MatrixSolveAttr func(optionalAttr)
 
@@ -27813,6 +29156,70 @@ func DatasetToGraph(scope *Scope, input_dataset tf.Output) (graph tf.Output) {
 	return op.Output(0)
 }
 
+// LuAttr is an optional argument to Lu.
+type LuAttr func(optionalAttr)
+
+// LuOutputIdxType sets the optional output_idx_type attribute to value.
+// If not specified, defaults to DT_INT32
+func LuOutputIdxType(value tf.DataType) LuAttr {
+	return func(m optionalAttr) {
+		m["output_idx_type"] = value
+	}
+}
+
+// Computes the LU decomposition of one or more square matrices.
+//
+// The input is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions
+// form square matrices.
+//
+// The input has to be invertible.
+//
+// The output consists of two tensors LU and P containing the LU decomposition
+// of all input submatrices `[..., :, :]`. LU encodes the lower triangular and
+// upper triangular factors.
+//
+// For each input submatrix of shape `[M, M]`, L is a lower triangular matrix of
+// shape `[M, M]` with unit diagonal whose entries correspond to the strictly lower
+// triangular part of LU. U is a upper triangular matrix of shape `[M, M]` whose
+// entries correspond to the upper triangular part, including the diagonal, of LU.
+//
+// P represents a permutation matrix encoded as a list of indices each between `0`
+// and `M-1`, inclusive. If P_mat denotes the permutation matrix corresponding to
+// P, then the L, U and P satisfies P_mat * input = L * U.
+//
+// Arguments:
+//	input: A tensor of shape `[..., M, M]` whose inner-most 2 dimensions form matrices of
+// size `[M, M]`.
+//
+// Returns A tensor of shape `[..., M, M]` whose strictly lower triangular part denotes the
+// lower triangular factor `L` with unit diagonal, and whose upper triangular part
+// denotes the upper triangular factor `U`.Permutation of the rows encoded as a list of indices in `0..M-1`. Shape is
+// `[..., M]`.
+// @compatibility(scipy)
+// Similar to `scipy.linalg.lu`, except the triangular factors `L` and `U` are
+// packed into a single tensor, the permutation is applied to `input` instead of
+// the right hand side and the permutation `P` is returned as a list of indices
+// instead of a permutation matrix.
+// @end_compatibility
+func Lu(scope *Scope, input tf.Output, optional ...LuAttr) (lu tf.Output, p tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "Lu",
+		Input: []tf.Input{
+			input,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1)
+}
+
 // Computes the matrix square root of one or more square matrices:
 //
 // matmul(sqrtm(A), sqrtm(A)) = A
@@ -29948,6 +31355,43 @@ func Iterator(scope *Scope, shared_name string, container string, output_types [
 	return op.Output(0)
 }
 
+// TensorForestTreeResourceHandleOpAttr is an optional argument to TensorForestTreeResourceHandleOp.
+type TensorForestTreeResourceHandleOpAttr func(optionalAttr)
+
+// TensorForestTreeResourceHandleOpContainer sets the optional container attribute to value.
+// If not specified, defaults to ""
+func TensorForestTreeResourceHandleOpContainer(value string) TensorForestTreeResourceHandleOpAttr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// TensorForestTreeResourceHandleOpSharedName sets the optional shared_name attribute to value.
+// If not specified, defaults to ""
+func TensorForestTreeResourceHandleOpSharedName(value string) TensorForestTreeResourceHandleOpAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// Creates a handle to a TensorForestTreeResource
+func TensorForestTreeResourceHandleOp(scope *Scope, optional ...TensorForestTreeResourceHandleOpAttr) (resource tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "TensorForestTreeResourceHandleOp",
+
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // CropAndResizeGradImageAttr is an optional argument to CropAndResizeGradImage.
 type CropAndResizeGradImageAttr func(optionalAttr)
 
@@ -30312,6 +31756,29 @@ func FakeParam(scope *Scope, dtype tf.DataType, shape tf.Shape) (output tf.Outpu
 	return op.Output(0)
 }
 
+// Returns the next representable value of `x1` in the direction of `x2`, element-wise.
+//
+// This operation returns the same result as the C++ std::nextafter function.
+//
+// It can also return a subnormal number.
+//
+// @compatibility(cpp)
+// Equivalent to C++ std::nextafter function.
+// @end_compatibility
+func NextAfter(scope *Scope, x1 tf.Output, x2 tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "NextAfter",
+		Input: []tf.Input{
+			x1, x2,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Computes the gradient for the inverse of `x` wrt its input.
 //
 // Specifically, `grad = -dy * y*y`, where `y = 1/x`, and `dy`
@@ -30461,6 +31928,71 @@ func BoostedTreesQuantileStreamResourceAddSummaries(scope *Scope, quantile_strea
 	return scope.AddOperation(opspec)
 }
 
+// Creates a Dataset that returns pseudorandom numbers.
+//
+// Arguments:
+//	seed: A scalar seed for the random number generator. If either seed or
+// seed2 is set to be non-zero, the random number generator is seeded
+// by the given seed.  Otherwise, a random seed is used.
+//	seed2: A second scalar seed to avoid seed collision.
+//
+//
+func ExperimentalRandomDataset(scope *Scope, seed tf.Output, seed2 tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	opspec := tf.OpSpec{
+		Type: "ExperimentalRandomDataset",
+		Input: []tf.Input{
+			seed, seed2,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// A dataset that splits the elements of its input into multiple elements.
+func ExperimentalUnbatchDataset(scope *Scope, input_dataset tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	opspec := tf.OpSpec{
+		Type: "ExperimentalUnbatchDataset",
+		Input: []tf.Input{
+			input_dataset,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Creates a dataset that overrides the maximum intra-op parallelism.
+//
+// Arguments:
+//
+//	max_intra_op_parallelism: Identifies the maximum intra-op parallelism to use.
+//
+//
+func ExperimentalMaxIntraOpParallelismDataset(scope *Scope, input_dataset tf.Output, max_intra_op_parallelism tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	opspec := tf.OpSpec{
+		Type: "ExperimentalMaxIntraOpParallelismDataset",
+		Input: []tf.Input{
+			input_dataset, max_intra_op_parallelism,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // StringSplitV2Attr is an optional argument to StringSplitV2.
 type StringSplitV2Attr func(optionalAttr)
 
@@ -30823,6 +32355,83 @@ func DeserializeIterator(scope *Scope, resource_handle tf.Output, serialized tf.
 	return scope.AddOperation(opspec)
 }
 
+// ResourceScatterNdSubAttr is an optional argument to ResourceScatterNdSub.
+type ResourceScatterNdSubAttr func(optionalAttr)
+
+// ResourceScatterNdSubUseLocking sets the optional use_locking attribute to value.
+//
+// value: An optional bool. Defaults to True. If True, the assignment will
+// be protected by a lock; otherwise the behavior is undefined,
+// but may exhibit less contention.
+// If not specified, defaults to true
+func ResourceScatterNdSubUseLocking(value bool) ResourceScatterNdSubAttr {
+	return func(m optionalAttr) {
+		m["use_locking"] = value
+	}
+}
+
+// Applies sparse subtraction to individual values or slices in a Variable.
+//
+// `ref` is a `Tensor` with rank `P` and `indices` is a `Tensor` of rank `Q`.
+//
+// `indices` must be integer tensor, containing indices into `ref`.
+// It must be shape `[d_0, ..., d_{Q-2}, K]` where `0 < K <= P`.
+//
+// The innermost dimension of `indices` (with length `K`) corresponds to
+// indices into elements (if `K = P`) or slices (if `K < P`) along the `K`th
+// dimension of `ref`.
+//
+// `updates` is `Tensor` of rank `Q-1+P-K` with shape:
+//
+// ```
+// [d_0, ..., d_{Q-2}, ref.shape[K], ..., ref.shape[P-1]]
+// ```
+//
+// For example, say we want to subtract 4 scattered elements from a rank-1 tensor
+// with 8 elements. In Python, that subtraction would look like this:
+//
+// ```python
+// ref = tf.Variable([1, 2, 3, 4, 5, 6, 7, 8], use_resource=True)
+// indices = tf.constant([[4], [3], [1], [7]])
+// updates = tf.constant([9, 10, 11, 12])
+// sub = tf.scatter_nd_sub(ref, indices, updates)
+// with tf.Session() as sess:
+//   print sess.run(sub)
+// ```
+//
+// The resulting update to ref would look like this:
+//
+//     [1, -9, 3, -6, -4, 6, 7, -4]
+//
+// See `tf.scatter_nd` for more details about how to make updates to
+// slices.
+//
+// Arguments:
+//	ref: A resource handle. Must be from a VarHandleOp.
+//	indices: A Tensor. Must be one of the following types: int32, int64.
+// A tensor of indices into ref.
+//	updates: A Tensor. Must have the same type as ref. A tensor of
+// values to add to ref.
+//
+// Returns the created operation.
+func ResourceScatterNdSub(scope *Scope, ref tf.Output, indices tf.Output, updates tf.Output, optional ...ResourceScatterNdSubAttr) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ResourceScatterNdSub",
+		Input: []tf.Input{
+			ref, indices, updates,
+		},
+		Attrs: attrs,
+	}
+	return scope.AddOperation(opspec)
+}
+
 // TensorArrayConcatV2Attr is an optional argument to TensorArrayConcatV2.
 type TensorArrayConcatV2Attr func(optionalAttr)
 
@@ -31032,6 +32641,43 @@ func TFRecordDataset(scope *Scope, filenames tf.Output, compression_type tf.Outp
 	return op.Output(0)
 }
 
+// ExperimentalStatsAggregatorHandleAttr is an optional argument to ExperimentalStatsAggregatorHandle.
+type ExperimentalStatsAggregatorHandleAttr func(optionalAttr)
+
+// ExperimentalStatsAggregatorHandleContainer sets the optional container attribute to value.
+// If not specified, defaults to ""
+func ExperimentalStatsAggregatorHandleContainer(value string) ExperimentalStatsAggregatorHandleAttr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// ExperimentalStatsAggregatorHandleSharedName sets the optional shared_name attribute to value.
+// If not specified, defaults to ""
+func ExperimentalStatsAggregatorHandleSharedName(value string) ExperimentalStatsAggregatorHandleAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// Creates a statistics manager resource.
+func ExperimentalStatsAggregatorHandle(scope *Scope, optional ...ExperimentalStatsAggregatorHandleAttr) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ExperimentalStatsAggregatorHandle",
+
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // A container for an iterator resource.
 //
 // Returns A handle to the iterator that can be passed to a "MakeIterator" or
@@ -31157,6 +32803,21 @@ func BatchToSpace(scope *Scope, input tf.Output, crops tf.Output, block_size int
 	return op.Output(0)
 }
 
+// Produces a summary of any statistics recorded by the given statistics manager.
+func ExperimentalStatsAggregatorSummary(scope *Scope, iterator tf.Output) (summary tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "ExperimentalStatsAggregatorSummary",
+		Input: []tf.Input{
+			iterator,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Makes a new iterator from the given `dataset` and stores it in `iterator`.
 //
 // This operation may be executed multiple times. Each execution will reset the
@@ -31488,6 +33149,26 @@ func FIFOQueueV2(scope *Scope, component_types []tf.DataType, optional ...FIFOQu
 	return op.Output(0)
 }
 
+// Deserializes a proto into the tree handle
+//
+// Arguments:
+//	tree_handle: Handle to the tree resource to be restored.
+//	tree_config: Serialied proto string of the boosted_trees.Tree proto.
+//
+// Returns the created operation.
+func TensorForestTreeDeserialize(scope *Scope, tree_handle tf.Output, tree_config tf.Output) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "TensorForestTreeDeserialize",
+		Input: []tf.Input{
+			tree_handle, tree_config,
+		},
+	}
+	return scope.AddOperation(opspec)
+}
+
 // Constructs an Optional variant from a tuple of tensors.
 func OptionalFromValue(scope *Scope, components []tf.Output) (optional tf.Output) {
 	if scope.Err() != nil {
@@ -31705,9 +33386,9 @@ func IteratorGetNextAsOptional(scope *Scope, iterator tf.Output, output_types []
 // dimension of `input`.
 //
 // Arguments:
-//	input: A complex64 tensor.
+//	input: A complex tensor.
 //
-// Returns A complex64 tensor of the same shape as `input`. The inner-most
+// Returns A complex tensor of the same shape as `input`. The inner-most
 //   dimension of `input` is replaced with its 1D Fourier transform.
 //
 // @compatibility(numpy)
@@ -32418,6 +34099,28 @@ func Merge(scope *Scope, inputs []tf.Output) (output tf.Output, value_index tf.O
 	return op.Output(0), op.Output(1)
 }
 
+// Writes the given dataset to the given file using the TFRecord format.
+//
+// Arguments:
+//	input_dataset: A variant tensor representing the dataset to write.
+//	filename: A scalar string tensor representing the filename to use.
+//	compression_type: A scalar string tensor containing either (i) the empty string (no
+// compression), (ii) "ZLIB", or (iii) "GZIP".
+//
+// Returns the created operation.
+func ExperimentalDatasetToTFRecord(scope *Scope, input_dataset tf.Output, filename tf.Output, compression_type tf.Output) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "ExperimentalDatasetToTFRecord",
+		Input: []tf.Input{
+			input_dataset, filename, compression_type,
+		},
+	}
+	return scope.AddOperation(opspec)
+}
+
 // QueueCloseV2Attr is an optional argument to QueueCloseV2.
 type QueueCloseV2Attr func(optionalAttr)
 
@@ -32754,6 +34457,23 @@ func Rpc(scope *Scope, address tf.Output, method tf.Output, request tf.Output, o
 	return op.Output(0)
 }
 
+// Records the bytes size of each element of `input_dataset` in a StatsAggregator.
+func ExperimentalBytesProducedStatsDataset(scope *Scope, input_dataset tf.Output, tag tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	opspec := tf.OpSpec{
+		Type: "ExperimentalBytesProducedStatsDataset",
+		Input: []tf.Input{
+			input_dataset, tag,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // StackPushV2Attr is an optional argument to StackPushV2.
 type StackPushV2Attr func(optionalAttr)
 
-- 
GitLab


From 55fda7fb66f1774ac5271e523259faa7d2a44d0c Mon Sep 17 00:00:00 2001
From: Davide Libenzi <dlibenzi@google.com>
Date: Fri, 4 Jan 2019 15:31:48 -0800
Subject: [PATCH 0261/1095] Allow XLA input/output buffer aliasing to work via
 XRT.

PiperOrigin-RevId: 227924255
---
 .../xla/service/cpu/cpu_executable.cc         |  29 +++-
 .../xla/service/gpu/gpu_executable.cc         |  26 +++-
 tensorflow/compiler/xrt/kernels/BUILD         |   1 +
 .../compiler/xrt/kernels/xrt_execute_op.cc    |  17 +++
 tensorflow/compiler/xrt/tests/raw_api_test.cc | 127 +++++++++++++++++-
 tensorflow/compiler/xrt/xrt_state.cc          |  43 +++++-
 tensorflow/compiler/xrt/xrt_state.h           |  23 +++-
 7 files changed, 249 insertions(+), 17 deletions(-)

diff --git a/tensorflow/compiler/xla/service/cpu/cpu_executable.cc b/tensorflow/compiler/xla/service/cpu/cpu_executable.cc
index 412c2715b9..23d0af3423 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_executable.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_executable.cc
@@ -213,6 +213,8 @@ StatusOr<ScopedShapedBuffer> CpuExecutable::CreateResultShapedBuffer(
       /*on_host_shape=*/result_shape(),
       /*on_device_shape=*/result_shape(), run_options->allocator(),
       stream->parent()->device_ordinal());
+  const HloInputOutputAliasConfig& input_output_alias =
+      module().input_output_alias_config();
 
   // Move OwningDeviceMemory values which contain the array(s) of the result
   // into the respective location in ScopedShapedBuffer which is returned to the
@@ -232,12 +234,31 @@ StatusOr<ScopedShapedBuffer> CpuExecutable::CreateResultShapedBuffer(
         TF_ASSIGN_OR_RETURN(
             const BufferAllocation::Slice slice,
             this->assignment_->GetUniqueSlice(src, buffer_source->index()));
-        CHECK(!slice.allocation()->is_entry_computation_parameter());
-
         const BufferAllocation::Index buffer_index = slice.index();
         OwningDeviceMemory& buffer = buffers[buffer_index];
-        CHECK(!buffer.is_null() || buffer.size() == 0);
-        *device_memory = buffer.Forget();
+        if (!slice.allocation()->is_entry_computation_parameter()) {
+          // If the buffer coming out of the result is from a parameter, the
+          // owning buffer will be null, and that means the caller aliased some
+          // parameter buffer to an output one (via the
+          // HloInputOutputAliasConfig API). If that is the case, the caller
+          // will receive a partially complete scoped shaped buffer, which they
+          // will have to fill up on return. Unfortunately the interface to the
+          // execute APIs are ShapedBuffer pointer based, which assumes caller
+          // ownership, and hence a buffer coming from there cannot be part of
+          // the new ScopedShapedBuffer we create for the result (which assumes
+          // ownership).
+          *device_memory = buffer.Forget();
+        } else {
+          auto output_alias = input_output_alias.GetAliasedOutput(
+              slice.allocation()->parameter_number(),
+              slice.allocation()->param_shape_index());
+          CHECK(output_alias)
+              << "Ouput buffer is coming from parameter "
+              << slice.allocation()->parameter_number() << " at index "
+              << slice.allocation()->param_shape_index()
+              << ", but no alias exists";
+          CHECK_EQ(*output_alias, index);
+        }
         return Status::OK();
       }));
   return std::move(result_buffer);
diff --git a/tensorflow/compiler/xla/service/gpu/gpu_executable.cc b/tensorflow/compiler/xla/service/gpu/gpu_executable.cc
index 128cecd765..434060ad89 100644
--- a/tensorflow/compiler/xla/service/gpu/gpu_executable.cc
+++ b/tensorflow/compiler/xla/service/gpu/gpu_executable.cc
@@ -310,12 +310,34 @@ StatusOr<ScopedShapedBuffer> GpuExecutable::ExecuteOnStream(
         TF_ASSIGN_OR_RETURN(
             const BufferAllocation::Slice slice,
             this->assignment_->GetUniqueSlice(src_hlo, sources[0]->index()));
-        CHECK(!slice.allocation()->is_entry_computation_parameter());
 
         se::DeviceMemoryBase src_base =
             buffer_allocations->GetDeviceAddress(slice.index());
         CHECK(!src_base.is_null() || src_base.size() == 0);
-        *device_memory = src_base;
+        if (!slice.allocation()->is_entry_computation_parameter()) {
+          // If the buffer coming out of the result is from a parameter, it
+          // means the caller aliased some parameter buffer to an output one
+          // (via the HloInputOutputAliasConfig API). If that is the case, the
+          // caller will receive a partially complete scoped shaped buffer,
+          // which they will have to fill up on return.
+          // Unfortunately the interface to the execute APIs are ShapedBuffer
+          // pointer based, which assumes caller ownership, and hence a buffer
+          // coming from there cannot be part of the new ScopedShapedBuffer we
+          // create for the result (which assumes ownership).
+          *device_memory = src_base;
+        } else {
+          const HloInputOutputAliasConfig& input_output_alias =
+              module().input_output_alias_config();
+          auto output_alias = input_output_alias.GetAliasedOutput(
+              slice.allocation()->parameter_number(),
+              slice.allocation()->param_shape_index());
+          CHECK(output_alias)
+              << "Ouput buffer is coming from parameter "
+              << slice.allocation()->parameter_number() << " at index "
+              << slice.allocation()->param_shape_index()
+              << ", but no alias exists";
+          CHECK_EQ(*output_alias, index);
+        }
         buffers_in_result.insert(src_base);
         return Status::OK();
       }));
diff --git a/tensorflow/compiler/xrt/kernels/BUILD b/tensorflow/compiler/xrt/kernels/BUILD
index 67f475846e..c44769dfe3 100644
--- a/tensorflow/compiler/xrt/kernels/BUILD
+++ b/tensorflow/compiler/xrt/kernels/BUILD
@@ -55,6 +55,7 @@ cc_library(
         "//tensorflow/compiler/xla/client:xla_computation",
         "//tensorflow/compiler/xla/service:compiler",
         "//tensorflow/compiler/xla/service:computation_placer",
+        "//tensorflow/compiler/xla/service:hlo",
         "//tensorflow/compiler/xrt:xrt_proto",
         "//tensorflow/compiler/xrt:xrt_utils",
         "//tensorflow/core:core_cpu_internal",
diff --git a/tensorflow/compiler/xrt/kernels/xrt_execute_op.cc b/tensorflow/compiler/xrt/kernels/xrt_execute_op.cc
index 7f0ac123a5..7544541ca4 100644
--- a/tensorflow/compiler/xrt/kernels/xrt_execute_op.cc
+++ b/tensorflow/compiler/xrt/kernels/xrt_execute_op.cc
@@ -19,6 +19,7 @@ limitations under the License.
 
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/service/computation_placer.h"
+#include "tensorflow/compiler/xla/service/hlo_input_output_alias_config.h"
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/status_macros.h"
 #include "tensorflow/compiler/xla/statusor.h"
@@ -228,6 +229,22 @@ Status XRTExecuteOp::DoWork(OpKernelContext* context) {
   TF_RETURN_IF_ERROR(XRTTupleAllocation::CreateFromBuffer(
       shaped_buffer, device_ref.backend(), device_ref.device_ordinal(),
       &output_tuple));
+
+  // The ScopedShapedBuffer returned by the executable Run() API, in case of
+  // input/output buffer aliasing, might have holes in it, which need to be
+  // filled using the proper input tuples buffers which are the source of
+  // aliasing.
+  const xla::HloInputOutputAliasConfig& input_output_alias =
+      executable->executable()->module().input_output_alias_config();
+  auto alias_function = [&](const xla::ShapeIndex& output_index,
+                            int64 param_number,
+                            const xla::ShapeIndex& param_index) -> Status {
+    TF_RET_CHECK(param_number < input_tuples.size());
+    return output_tuple->AliasBufferFrom(*input_tuples[param_number],
+                                         param_index, output_index);
+  };
+  TF_RETURN_IF_ERROR(input_output_alias.ForEachAliasWithStatus(alias_function));
+
   if (config_proto.return_exploded_tuple() &&
       output_tuple->on_device_shape().IsTuple()) {
     int64 tuple_element_count =
diff --git a/tensorflow/compiler/xrt/tests/raw_api_test.cc b/tensorflow/compiler/xrt/tests/raw_api_test.cc
index c8479cb778..f81faf0e61 100644
--- a/tensorflow/compiler/xrt/tests/raw_api_test.cc
+++ b/tensorflow/compiler/xrt/tests/raw_api_test.cc
@@ -96,14 +96,21 @@ xla::LiteralProto FloatMatrix(
   return array.ToProto();
 }
 
+xla::Literal ReadOutputLiteral(const std::vector<Tensor>& outputs, size_t idx) {
+  xla::LiteralProto response;
+  CHECK(response.ParseFromString(outputs[idx].scalar<string>()()));
+  return xla::Literal::CreateFromProto(response).ValueOrDie();
+}
+
 bool CompareLiteralProtos(const xla::LiteralProto& a,
                           const xla::LiteralProto& b) {
   auto l_a = xla::Literal::CreateFromProto(a).ValueOrDie();
   auto l_b = xla::Literal::CreateFromProto(b).ValueOrDie();
   bool equal = l_a == l_b;
   if (!equal) {
-    LOG(INFO) << "LiteralProtos don't match: " << a.DebugString()
-              << " != " << b.DebugString();
+    LOG(INFO) << "LiteralProtos don't match:\n"
+              << a.DebugString() << "\n!=\n"
+              << b.DebugString();
   }
   return equal;
 }
@@ -113,8 +120,19 @@ bool CompareLiteralToLiteralProto(const xla::Literal& a,
   auto l_b = xla::Literal::CreateFromProto(b).ValueOrDie();
   bool equal = a == l_b;
   if (!equal) {
-    LOG(INFO) << "Literal and LiteralProto don't match "
-              << a.ToProto().DebugString() << " != " << b.DebugString();
+    LOG(INFO) << "Literal and LiteralProto don't match:\n"
+              << a.ToProto().DebugString() << "\n!=\n"
+              << b.DebugString();
+  }
+  return equal;
+}
+
+bool CompareLiterals(const xla::Literal& a, const xla::Literal& b) {
+  bool equal = a == b;
+  if (!equal) {
+    LOG(INFO) << "Literals don't match:\n"
+              << a.ToProto().DebugString() << "\n!=\n"
+              << b.ToProto().DebugString();
   }
   return equal;
 }
@@ -939,6 +957,107 @@ TEST(RawApiTest, LeakCompilationReference) {
   TF_EXPECT_OK(session.Run({c_handle.handle}, &outputs));
 }
 
+TEST(RawApiTest, CompileAndExecuteWithReusedBuffers) {
+  xla::Shape element_shape = xla::ShapeUtil::MakeShape(xla::F32, {2});
+  xla::Shape shape =
+      xla::ShapeUtil::MakeTupleShape({element_shape, element_shape});
+  xla::Shape return_shape = xla::ShapeUtil::MakeTupleShape(
+      {element_shape, element_shape, element_shape, element_shape});
+  xla::XlaBuilder builder("ReuseBuffer");
+  auto param = xla::Parameter(&builder, 0, shape, "param");
+  auto p0 = xla::GetTupleElement(param, 0);
+  auto p1 = xla::GetTupleElement(param, 1);
+  auto add = xla::Add(p0, p1);
+  auto sub = xla::Sub(p0, p1);
+  xla::Tuple(&builder, {add, sub, p0, p1});
+
+  // Flip the tuple literals in the input handle.
+  builder.SetUpAlias({1}, 0, {0});
+  builder.SetUpAlias({0}, 0, {1});
+
+  auto computation = builder.Build().ValueOrDie();
+
+  auto literal0 = xla::LiteralUtil::CreateR1<float>({1.0f, 2.0f});
+  auto literal1 = xla::LiteralUtil::CreateR1<float>({5.0f, 9.0f});
+  auto literal = xla::LiteralUtil::MakeTuple({&literal0, &literal1});
+
+  xrt::XLAAllocation param_alloc;
+  *param_alloc.mutable_value() = literal.ToProto();
+
+  xrt::XLAComputation c;
+  auto config = c.mutable_config();
+  auto shapes = config->mutable_program_shape();
+  *shapes->add_parameters() = shape.ToProto();
+  *shapes->mutable_result() = return_shape.ToProto();
+  StoreComputationSnapshot(computation, c.mutable_hlo_snapshot());
+
+  xrt::XRTExecutionConfig e;
+  e.set_release_input_handles(false);
+  e.set_release_compilation_handle(true);
+
+  Scope root = Scope::NewRootScope().WithDevice(DeviceFromFlag());
+  ClientSession session(root);
+  auto e_config =
+      ops::Const(root.WithDevice("/device:CPU:0"), e.SerializeAsString());
+  auto c_data =
+      ops::Const(root.WithDevice("/device:CPU:0"), c.SerializeAsString());
+  auto c_handle = ops::XRTCompile(root, c_data);
+  auto param_value = ops::Const(root.WithDevice("/device:CPU:0"),
+                                param_alloc.SerializeAsString());
+  auto param_handle = ops::XRTAllocate(root, param_value);
+  TF_ASSERT_OK(root.status());
+
+  std::vector<Tensor> outputs;
+  TF_EXPECT_OK(session.Run({param_handle}, &outputs));
+
+  int64 alloc_handle = outputs[0].scalar<int64>()();
+
+  // Note that we release the result handle immediately, but since we aliased
+  // the output buffers onto the input allocation ones (held in alloc_handle),
+  // we can fetch the result from there.
+  auto result =
+      ops::XRTExecute(root, c_handle.handle, e_config, {Input(alloc_handle)});
+  auto read_back = ops::XRTReadLiteral(root, result);
+  auto release = ops::XRTReleaseAllocationHandle(
+      root.WithControlDependencies(read_back), result);
+  TF_ASSERT_OK(root.status());
+
+  outputs.clear();
+  TF_EXPECT_OK(session.Run(tensorflow::ClientSession::FeedType(), {read_back},
+                           {release}, &outputs));
+
+  xla::Literal exec_literal = ReadOutputLiteral(outputs, 0);
+  auto exec_literal_parts = exec_literal.DecomposeTuple();
+  ASSERT_EQ(exec_literal_parts.size(), 4);
+
+  EXPECT_TRUE(CompareLiterals(exec_literal_parts[2], literal0));
+  EXPECT_TRUE(CompareLiterals(exec_literal_parts[3], literal1));
+
+  // Now we read back the original input handle values, which at this point
+  // should contain the result of the XLA computation.
+  auto read_handle = ops::XRTReadLiteral(root, Input(alloc_handle));
+  TF_ASSERT_OK(root.status());
+  auto release_handle = ops::XRTReleaseAllocationHandle(
+      root.WithControlDependencies(read_handle), Input(alloc_handle));
+  TF_ASSERT_OK(root.status());
+
+  outputs.clear();
+  TF_EXPECT_OK(session.Run(tensorflow::ClientSession::FeedType(), {read_handle},
+                           {release_handle}, &outputs));
+
+  xla::Literal return_literal = ReadOutputLiteral(outputs, 0);
+
+  auto expected_literal0 = xla::LiteralUtil::CreateR1<float>({6.0f, 11.0f});
+  auto expected_literal1 = xla::LiteralUtil::CreateR1<float>({-4.0f, -7.0f});
+  // The first element of the computation returned tuple would be the add
+  // (expected_literal0), but since we flipped the buffers, the sub
+  // (expected_literal1) should come first.
+  auto expected_literal =
+      xla::LiteralUtil::MakeTuple({&expected_literal1, &expected_literal0});
+
+  EXPECT_TRUE(CompareLiterals(return_literal, expected_literal));
+}
+
 TEST(RawApiTest, CompileAndExecuteWithS64Argument) {
   xrt::XLAAllocation p0;
   *p0.mutable_value() = xla::LiteralUtil::CreateR0<int64>(11031965).ToProto();
diff --git a/tensorflow/compiler/xrt/xrt_state.cc b/tensorflow/compiler/xrt/xrt_state.cc
index 343460ff10..13c275aaca 100644
--- a/tensorflow/compiler/xrt/xrt_state.cc
+++ b/tensorflow/compiler/xrt/xrt_state.cc
@@ -133,7 +133,8 @@ Status AllocateScopedShapedBuffer(
 XRTBufferAllocation::XRTBufferAllocation(const se::DeviceMemoryBase& allocation,
                                          int device_ordinal,
                                          xla::DeviceMemoryAllocator* allocator)
-    : allocation_(allocation),
+    : size_(allocation.size()),
+      allocation_(allocation),
       device_ordinal_(device_ordinal),
       allocator_(allocator) {
   if (VLOG_IS_ON(2)) {
@@ -223,8 +224,19 @@ Status XRTTupleAllocation::ToLiteral(xla::Backend* backend, int device_ordinal,
                                      xla::Literal* literal) {
   auto transfer_manager = backend->transfer_manager();
   TF_ASSIGN_OR_RETURN(auto stream, backend->BorrowStream(device_ordinal));
+
+  // Validate the allocation buffers as if nulls gets to
+  // TransferLiteralFromDevice() a CHECK is issued.
+  xla::ShapedBuffer shaped_buffer = ToShapedBuffer();
+  for (auto& index_buffer : shaped_buffer.buffers()) {
+    if (index_buffer.second.is_null()) {
+      return errors::InvalidArgument("Literal buffer at index ",
+                                     index_buffer.first.ToString(),
+                                     " has been released");
+    }
+  }
   TF_ASSIGN_OR_RETURN(*literal, transfer_manager->TransferLiteralFromDevice(
-                                    stream.get(), ToShapedBuffer()));
+                                    stream.get(), shaped_buffer));
   return Status::OK();
 }
 
@@ -505,11 +517,34 @@ xla::ShapedBuffer XRTTupleAllocation::ToShapedBuffer() {
   return shaped_buffer;
 }
 
+Status XRTTupleAllocation::AliasBufferFrom(const XRTTupleAllocation& source,
+                                           const xla::ShapeIndex& source_index,
+                                           const xla::ShapeIndex& dest_index) {
+  XRTBufferAllocation* source_buffer = source.buffers_.element(source_index);
+  XRTBufferAllocation* dest_buffer = buffers_.element(dest_index);
+  // We allow the destination size being zero, because there are cases where we
+  // are coming in later filling in null/uninitialized device buffers.
+  // In all other cases, the size of the new buffer must match.
+  if (source_buffer->size() != dest_buffer->size() &&
+      dest_buffer->size() != 0) {
+    return errors::InvalidArgument(
+        "Source buffer at index ", source_index.ToString(),
+        " does not match the size of destination buffer at index ",
+        dest_index.ToString(), ": ", source_buffer->size(), " vs ",
+        dest_buffer->size());
+  }
+  *buffers_.mutable_element(dest_index) = source_buffer;
+  source_buffer->Ref();
+  dest_buffer->Unref();
+  return Status::OK();
+}
+
 xla::ShapeTree<xla::MaybeOwningDeviceMemory>
-XRTTupleAllocation::ToDeviceMemoryTree(bool release) {
+XRTTupleAllocation::ToDeviceMemoryTree(
+    const std::function<bool(const xla::ShapeIndex&)>& release_checker) {
   xla::ShapeTree<xla::MaybeOwningDeviceMemory> shaped_tree(on_device_shape());
   for (const auto& buffer : buffers_) {
-    if (!release) {
+    if (!release_checker(buffer.first)) {
       *shaped_tree.mutable_element(buffer.first) = buffer.second->allocation();
     } else {
       *shaped_tree.mutable_element(buffer.first) = xla::OwningDeviceMemory(
diff --git a/tensorflow/compiler/xrt/xrt_state.h b/tensorflow/compiler/xrt/xrt_state.h
index 3e3d502412..ac4be3a064 100644
--- a/tensorflow/compiler/xrt/xrt_state.h
+++ b/tensorflow/compiler/xrt/xrt_state.h
@@ -18,6 +18,7 @@ limitations under the License.
 #ifndef TENSORFLOW_COMPILER_XRT_XRT_STATE_H_
 #define TENSORFLOW_COMPILER_XRT_XRT_STATE_H_
 
+#include <functional>
 #include <memory>
 #include <string>
 #include <vector>
@@ -58,7 +59,14 @@ class XRTBufferAllocation : public core::RefCounted {
   // freed when the reference count drops to zero.
   void DiscardAllocation();
 
+  // Returns the expected size of the allocation. Since DiscardAllocation() will
+  // set allocation_ to {null,0}, and since later we might want to replace the
+  // discarded buffer with a new one, we need to be able to verify the size
+  // compatibility.
+  uint64 size() const { return size_; }
+
  private:
+  uint64 size_ = 0;
   se::DeviceMemoryBase allocation_;
   int device_ordinal_;
   xla::DeviceMemoryAllocator* allocator_;
@@ -168,9 +176,18 @@ class XRTTupleAllocation : public ResourceBase {
   // the same shape as on_host_shape.
   xla::ShapedBuffer ToShapedBuffer();
 
-  // Returns the device memory tree of this allocation. If 'release' is set, the
-  // ownership of the device memory is transferred to the result.
-  xla::ShapeTree<xla::MaybeOwningDeviceMemory> ToDeviceMemoryTree(bool release);
+  // Aliases the source buffer at source_index into the current tuple allocation
+  // dest_index.
+  Status AliasBufferFrom(const XRTTupleAllocation& source,
+                         const xla::ShapeIndex& source_index,
+                         const xla::ShapeIndex& dest_index);
+
+  // Returns the device memory tree of this allocation. If the release_checker
+  // function returns true for a given index, the ownership of the device memory
+  // at that index is transferred to the result. Every attempt to read the value
+  // at that index will fail.
+  xla::ShapeTree<xla::MaybeOwningDeviceMemory> ToDeviceMemoryTree(
+      const std::function<bool(const xla::ShapeIndex&)>& release_checker);
 
   string DebugString() override { return "XLA allocation handle"; }
 
-- 
GitLab


From 02bfac424857fa45621efab5908a31ffd4ce799b Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Fri, 4 Jan 2019 15:32:34 -0800
Subject: [PATCH 0262/1095] Add kullback leibler divergence v2 loss and metric.

PiperOrigin-RevId: 227924402
---
 tensorflow/python/keras/losses.py       | 27 +++++++++
 tensorflow/python/keras/losses_test.py  | 80 +++++++++++++++++++++++++
 tensorflow/python/keras/metrics.py      | 32 ++++++++++
 tensorflow/python/keras/metrics_test.py | 47 +++++++++++++++
 4 files changed, 186 insertions(+)

diff --git a/tensorflow/python/keras/losses.py b/tensorflow/python/keras/losses.py
index 46e729c197..de1605a09c 100644
--- a/tensorflow/python/keras/losses.py
+++ b/tensorflow/python/keras/losses.py
@@ -581,6 +581,33 @@ class Logcosh(Loss):
     return logcosh(y_true, y_pred)
 
 
+class KullbackLeiblerDivergence(Loss):
+  """Computes kullback leibler divergence loss between `y_true` and `y_pred`.
+
+  loss = y_true * log(y_true / y_pred)
+
+  Usage:
+
+  ```python
+  k = tf.losses.KullbackLeiblerDivergence()
+  loss = k([.4, .9, .2], [.5, .8, .12])
+  print('Loss: ', loss.numpy())  # Loss: -0.043
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile('sgd', loss=tf.losses.KullbackLeiblerDivergence())
+  ```
+  """
+
+  def call(self, y_true, y_pred):
+    y_pred = ops.convert_to_tensor(y_pred)
+    y_true = math_ops.cast(y_true, y_pred.dtype)
+    return kullback_leibler_divergence(y_true, y_pred)
+
+
 @keras_export('keras.metrics.mean_squared_error',
               'keras.metrics.mse',
               'keras.metrics.MSE',
diff --git a/tensorflow/python/keras/losses_test.py b/tensorflow/python/keras/losses_test.py
index f34a6caa18..caa4ff4c2e 100644
--- a/tensorflow/python/keras/losses_test.py
+++ b/tensorflow/python/keras/losses_test.py
@@ -1256,5 +1256,85 @@ class PoissonTest(test.TestCase):
     self.assertAlmostEqual(self.evaluate(loss), 0., 3)
 
 
+@test_util.run_all_in_graph_and_eager_modes
+class KullbackLeiblerDivergenceTest(test.TestCase):
+
+  def setup(self):
+    self.np_y_pred = np.asarray([.4, .9, .12, .36, .3, .4]).reshape((2, 3))
+    self.np_y_true = np.asarray([.5, .8, .12, .7, .43, .8]).reshape((2, 3))
+
+    self.batch_size = 2
+    self.expected_losses = np.multiply(self.np_y_true,
+                                       np.log(self.np_y_true / self.np_y_pred))
+
+    self.y_pred = constant_op.constant(self.np_y_pred, dtype=dtypes.float32)
+    self.y_true = constant_op.constant(self.np_y_true)
+
+  def test_config(self):
+    k_obj = keras.losses.KullbackLeiblerDivergence(
+        reduction=losses_impl.ReductionV2.SUM, name='kld')
+    self.assertEqual(k_obj.name, 'kld')
+    self.assertEqual(k_obj.reduction, losses_impl.ReductionV2.SUM)
+
+  def test_unweighted(self):
+    self.setup()
+    k_obj = keras.losses.KullbackLeiblerDivergence()
+
+    loss = k_obj(self.y_true, self.y_pred)
+    expected_loss = np.sum(self.expected_losses) / self.batch_size
+    self.assertAlmostEqual(self.evaluate(loss), expected_loss, 3)
+
+  def test_scalar_weighted(self):
+    self.setup()
+    k_obj = keras.losses.KullbackLeiblerDivergence()
+    sample_weight = 2.3
+
+    loss = k_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+    expected_loss = sample_weight * np.sum(
+        self.expected_losses) / self.batch_size
+    self.assertAlmostEqual(self.evaluate(loss), expected_loss, 3)
+
+    # Verify we get the same output when the same input is given
+    loss_2 = k_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+    self.assertAlmostEqual(self.evaluate(loss), self.evaluate(loss_2), 3)
+
+  def test_sample_weighted(self):
+    self.setup()
+    k_obj = keras.losses.KullbackLeiblerDivergence()
+    sample_weight = constant_op.constant([1.2, 3.4], shape=(2, 1))
+    loss = k_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+
+    expected_loss = np.multiply(
+        self.expected_losses,
+        np.asarray([1.2, 1.2, 1.2, 3.4, 3.4, 3.4]).reshape(2, 3))
+    expected_loss = np.sum(expected_loss) / self.batch_size
+    self.assertAlmostEqual(self.evaluate(loss), expected_loss, 3)
+
+  def test_timestep_weighted(self):
+    self.setup()
+    k_obj = keras.losses.KullbackLeiblerDivergence()
+    y_true = self.np_y_true.reshape(2, 3, 1)
+    y_pred = self.np_y_pred.reshape(2, 3, 1)
+    sample_weight = np.asarray([3, 6, 5, 0, 4, 2]).reshape(2, 3)
+    expected_losses = np.sum(
+        np.multiply(y_true, np.log(y_true / y_pred)), axis=-1)
+
+    y_pred = constant_op.constant(y_pred, dtype=dtypes.float32)
+    y_true = constant_op.constant(y_true)
+    loss = k_obj(
+        y_true, y_pred, sample_weight=constant_op.constant(sample_weight))
+
+    num_timesteps = 3
+    expected_loss = np.sum(expected_losses * sample_weight) / (
+        self.batch_size * num_timesteps)
+    self.assertAlmostEqual(self.evaluate(loss), expected_loss, 3)
+
+  def test_zero_weighted(self):
+    self.setup()
+    k_obj = keras.losses.KullbackLeiblerDivergence()
+    loss = k_obj(self.y_true, self.y_pred, sample_weight=0)
+    self.assertAlmostEqual(self.evaluate(loss), 0., 3)
+
+
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index 71568e3fc5..9a2e057780 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -1716,6 +1716,38 @@ class Poisson(MeanMetricWrapper):
     return super(Poisson, cls).from_config(config)
 
 
+class KullbackLeiblerDivergence(MeanMetricWrapper):
+  """Computes kullback leibler divergence metric between `y_true` and `y_pred`.
+
+  metric = y_true * log(y_true / y_pred)
+
+  Usage:
+
+  ```python
+  m = tf.keras.metrics.KullbackLeiblerDivergence()
+  m.update_state([.4, .9, .2], [.5, .8, .12])
+  print('Final result: ', m.result().numpy())  # Final result: -0.043
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile('sgd', metrics=[tf.keras.metrics.KullbackLeiblerDivergence()])
+  ```
+  """
+
+  def __init__(self, name='kullback_leibler_divergence', dtype=None):
+    super(KullbackLeiblerDivergence, self).__init__(
+        kullback_leibler_divergence, name, dtype=dtype)
+
+  @classmethod
+  def from_config(cls, config):
+    if 'fn' in config:
+      config.pop('fn')
+    return super(KullbackLeiblerDivergence, cls).from_config(config)
+
+
 def accuracy(y_true, y_pred):
   y_pred.get_shape().assert_is_compatible_with(y_true.get_shape())
   if y_true.dtype != y_pred.dtype:
diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py
index 88b763d569..a02b95bb46 100644
--- a/tensorflow/python/keras/metrics_test.py
+++ b/tensorflow/python/keras/metrics_test.py
@@ -1556,6 +1556,53 @@ class PoissonTest(test.TestCase):
     self.assertAllClose(self.evaluate(result), expected_result, atol=1e-3)
 
 
+@test_util.run_all_in_graph_and_eager_modes
+class KullbackLeiblerDivergenceTest(test.TestCase):
+
+  def setup(self):
+    y_pred = np.asarray([.4, .9, .12, .36, .3, .4]).reshape((2, 3))
+    y_true = np.asarray([.5, .8, .12, .7, .43, .8]).reshape((2, 3))
+
+    self.batch_size = 2
+    self.expected_results = np.multiply(y_true, np.log(y_true / y_pred))
+
+    self.y_pred = constant_op.constant(y_pred, dtype=dtypes.float32)
+    self.y_true = constant_op.constant(y_true)
+
+  def test_config(self):
+    k_obj = metrics.KullbackLeiblerDivergence(name='kld', dtype=dtypes.int32)
+    self.assertEqual(k_obj.name, 'kld')
+    self.assertEqual(k_obj._dtype, dtypes.int32)
+
+    k_obj2 = metrics.KullbackLeiblerDivergence.from_config(k_obj.get_config())
+    self.assertEqual(k_obj2.name, 'kld')
+    self.assertEqual(k_obj2._dtype, dtypes.int32)
+
+  def test_unweighted(self):
+    self.setup()
+    k_obj = metrics.KullbackLeiblerDivergence()
+    self.evaluate(variables.variables_initializer(k_obj.variables))
+
+    update_op = k_obj.update_state(self.y_true, self.y_pred)
+    self.evaluate(update_op)
+    result = k_obj.result()
+    expected_result = np.sum(self.expected_results) / self.batch_size
+    self.assertAllClose(result, expected_result, atol=1e-3)
+
+  def test_weighted(self):
+    self.setup()
+    k_obj = metrics.KullbackLeiblerDivergence()
+    self.evaluate(variables.variables_initializer(k_obj.variables))
+
+    sample_weight = constant_op.constant([1.2, 3.4], shape=(2, 1))
+    result = k_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+
+    sample_weight = np.asarray([1.2, 1.2, 1.2, 3.4, 3.4, 3.4]).reshape((2, 3))
+    expected_result = np.multiply(self.expected_results, sample_weight)
+    expected_result = np.sum(expected_result) / (1.2 + 3.4)
+    self.assertAllClose(self.evaluate(result), expected_result, atol=1e-3)
+
+
 def _get_model(compile_metrics):
   model_layers = [
       layers.Dense(3, activation='relu', kernel_initializer='ones'),
-- 
GitLab


From 984ace61cdef46145a63081151c7eeb8609c4133 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 4 Jan 2019 15:45:38 -0800
Subject: [PATCH 0263/1095] Report the CUcontext in the cuda_driver log spew,
 not the CudaContext pointer. CUDA tools (like nvprof, cuda-memcheck) report
 the CUcontext, so this makes it possible to match them up during debugging.

PiperOrigin-RevId: 227926402
---
 .../stream_executor/cuda/cuda_driver.cc       | 28 ++++++++++---------
 1 file changed, 15 insertions(+), 13 deletions(-)

diff --git a/tensorflow/stream_executor/cuda/cuda_driver.cc b/tensorflow/stream_executor/cuda/cuda_driver.cc
index b34d1f722e..ca7a717bdb 100644
--- a/tensorflow/stream_executor/cuda/cuda_driver.cc
+++ b/tensorflow/stream_executor/cuda/cuda_driver.cc
@@ -431,7 +431,8 @@ bool DeviceOptionsToContextFlags(const DeviceOptions &device_options,
     *context = CreatedContexts::Add(new_context);
     CHECK(*context != nullptr)
         << "success in this call must entail non-null result";
-    VLOG(2) << "created or reused context " << context << " for this thread";
+    VLOG(2) << "created or reused context " << new_context
+            << " for this thread";
     return port::Status::OK();
   }
 
@@ -769,13 +770,13 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   ScopedActivateContext activated{context};
   CUresult res = cuStreamCreate(out, 0);
   if (res != CUDA_SUCCESS) {
-    LOG(ERROR) << "could not allocate CUDA stream for context " << context
-               << ": " << ToString(res);
+    LOG(ERROR) << "could not allocate CUDA stream for context "
+               << context->context() << ": " << ToString(res);
     return false;
   }
 
   VLOG(2) << "successfully created stream " << *out << " for context "
-          << context << " on thread";
+          << context->context() << " on thread";
   return true;
 }
 
@@ -788,11 +789,11 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   ScopedActivateContext activated{context};
   CUresult res = cuStreamDestroy(*stream);
   if (res != CUDA_SUCCESS) {
-    LOG(ERROR) << "failed to destroy CUDA stream for context " << context
-               << ": " << ToString(res);
+    LOG(ERROR) << "failed to destroy CUDA stream for context "
+               << context->context() << ": " << ToString(res);
   } else {
     VLOG(2) << "successfully destroyed stream " << *stream << " for context "
-            << context;
+            << context->context();
     *stream = nullptr;
   }
 }
@@ -809,8 +810,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
     return nullptr;
   }
   void *ptr = reinterpret_cast<void *>(result);
-  VLOG(2) << "allocated " << ptr << " for context " << context << " of "
-          << bytes << " bytes";
+  VLOG(2) << "allocated " << ptr << " for context " << context->context()
+          << " of " << bytes << " bytes";
   return ptr;
 }
 
@@ -823,7 +824,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
     LOG(ERROR) << "failed to free device memory at " << location
                << "; result: " << ToString(res);
   } else {
-    VLOG(2) << "deallocated " << location << " for context " << context;
+    VLOG(2) << "deallocated " << location << " for context "
+            << context->context();
   }
 }
 
@@ -839,8 +841,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
     return nullptr;
   }
   void *ptr = reinterpret_cast<void *>(result);
-  VLOG(2) << "allocated " << ptr << " for context " << context << " of "
-          << bytes << " bytes in unified memory";
+  VLOG(2) << "allocated " << ptr << " for context " << context->context()
+          << " of " << bytes << " bytes in unified memory";
   return ptr;
 }
 
@@ -854,7 +856,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                << "; result: " << ToString(res);
   } else {
     VLOG(2) << "deallocated unified memory at " << location << " for context "
-            << context;
+            << context->context();
   }
 }
 
-- 
GitLab


From 57d501422c16dd73b391d19bcc1e203942cc6b24 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 4 Jan 2019 15:55:26 -0800
Subject: [PATCH 0264/1095] Add Keras LSTM model to correctness test.

PiperOrigin-RevId: 227927675
---
 tensorflow/contrib/distribute/python/BUILD    |   3 +-
 .../contrib/distribute/python/combinations.py |  15 ++-
 .../python/keras_correctness_test.py          | 100 +++++++++++++++---
 3 files changed, 99 insertions(+), 19 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index de6b6f1f84..c620a0448e 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -670,11 +670,12 @@ py_library(
 
 cuda_py_test(
     name = "keras_correctness_test",
+    size = "medium",
     srcs = ["keras_correctness_test.py"],
     additional_deps = [
         ":keras_correctness_test_lib",
     ],
-    shard_count = 16,
+    shard_count = 20,
     tags = [
         "multi_and_single_gpu",
         "no_oss",  # TODO(b/117919883): Fix python error.
diff --git a/tensorflow/contrib/distribute/python/combinations.py b/tensorflow/contrib/distribute/python/combinations.py
index f6c4291659..8db0e9c316 100644
--- a/tensorflow/contrib/distribute/python/combinations.py
+++ b/tensorflow/contrib/distribute/python/combinations.py
@@ -321,11 +321,12 @@ class NamedDistribution(object):
     return self._required_tpu
 
 
-def _get_tpu_strategy_creator(steps_per_run):
+def _get_tpu_strategy_creator(steps_per_run, **kwargs):
   def _create_tpu_strategy():
     resolver = cluster_resolver.TPUClusterResolver("")
     tpu_lib.initialize_tpu_system(resolver)
-    strategy = tpu_lib.TPUStrategy(resolver, steps_per_run=steps_per_run)
+    strategy = tpu_lib.TPUStrategy(resolver,
+                                   steps_per_run=steps_per_run, **kwargs)
     return strategy
   return _create_tpu_strategy
 
@@ -344,7 +345,15 @@ tpu_strategy = NamedDistribution(
 tpu_strategy_one_step = NamedDistribution(
     "TPUOneStep", _get_tpu_strategy_creator(steps_per_run=1),
     required_tpu=True)
-
+# TODO(b/122327153): Remove below two NamedDistributions.
+tpu_strategy_loop_on_device = NamedDistribution(
+    "TPULoopOnDevice", _get_tpu_strategy_creator(
+        steps_per_run=2, _disable_training_loop_on_host=True),
+    required_tpu=True)
+tpu_strategy_one_step_loop_on_device = NamedDistribution(
+    "TPUOneStepLoopOnDevice", _get_tpu_strategy_creator(
+        steps_per_run=1, _disable_training_loop_on_host=True),
+    required_tpu=True)
 mirrored_strategy_with_one_cpu = NamedDistribution(
     "Mirrored1CPU",
     lambda: mirrored_lib.MirroredStrategy(["/cpu:0"]))
diff --git a/tensorflow/contrib/distribute/python/keras_correctness_test.py b/tensorflow/contrib/distribute/python/keras_correctness_test.py
index 3abdee2c0e..1c4519ef71 100644
--- a/tensorflow/contrib/distribute/python/keras_correctness_test.py
+++ b/tensorflow/contrib/distribute/python/keras_correctness_test.py
@@ -62,24 +62,41 @@ def all_strategy_combinations_with_graph_mode():
   return combinations.combine(distribution=all_strategies, mode=['graph'])
 
 
+def tpu_strategies_with_host_training_loop_disabled():
+  strategies = [s for s in all_strategies if not s.required_tpu]
+  strategies.append(combinations.tpu_strategy_loop_on_device)
+  strategies.append(combinations.tpu_strategy_one_step_loop_on_device)
+  return strategies
+
+
 def strategy_and_input_combinations():
   def cnn_model_with_batch_norm(**kwargs):
     return _create_cnn_model(with_batch_norm=True, **kwargs)
 
-  return (
-      combinations.times(
-          combinations.combine(distribution=all_strategies),
-          combinations.combine(mode=['graph', 'eager'],
-                               use_numpy=[True, False],
-                               use_validation_data=[True, False]),
-          combinations.combine(model_with_data=[
-              ModelWithData('dnn', _create_dnn_model, _dnn_training_data),
-              ModelWithData('cnn', _create_cnn_model, _cnn_training_data),
-              ModelWithData('cnn_batch_norm',
-                            cnn_model_with_batch_norm,
-                            _cnn_training_data,
-                            with_batch_norm=True),
-          ])))
+  combinations_without_embedding_model = combinations.times(
+      combinations.combine(
+          distribution=tpu_strategies_with_host_training_loop_disabled()),
+      combinations.combine(mode=['graph', 'eager'],
+                           use_numpy=[True, False],
+                           use_validation_data=[True, False]),
+      combinations.combine(model_with_data=
+                           [ModelWithData('lstm',
+                                          _create_lstm_model,
+                                          _lstm_training_data)]))
+
+  combinations_with_embedding_model = combinations.times(
+      combinations.combine(distribution=all_strategies),
+      combinations.combine(mode=['graph', 'eager'],
+                           use_numpy=[True, False],
+                           use_validation_data=[True, False]),
+      combinations.combine(model_with_data=[
+          ModelWithData('dnn', _create_dnn_model, _dnn_training_data),
+          ModelWithData('cnn', _create_cnn_model, _cnn_training_data),
+          ModelWithData('cnn_batch_norm', cnn_model_with_batch_norm,
+                        _cnn_training_data, with_batch_norm=True),]))
+
+  return (combinations_with_embedding_model +
+          combinations_without_embedding_model)
 
 
 class MaybeDistributionScope(object):
@@ -191,6 +208,59 @@ def _create_cnn_model(initial_weights=None, distribution=None,
   return model
 
 
+def _lstm_training_data(count=_GLOBAL_BATCH_SIZE * _EVAL_STEPS,
+                        min_words=10,
+                        max_words=20,
+                        max_word_id=99,
+                        num_classes=2):
+  distribution = []
+  for _ in range(num_classes):
+    dist = np.abs(np.random.randn(max_word_id))
+    dist /= np.sum(dist)
+    distribution.append(dist)
+
+  features = []
+  labels = []
+  for _ in range(count):
+    label = np.random.randint(0, num_classes, size=1)[0]
+    num_words = np.random.randint(min_words, max_words, size=1)[0]
+    word_ids = np.random.choice(
+        max_word_id, size=num_words, replace=True, p=distribution[label])
+    word_ids = word_ids
+    labels.append(label)
+    features.append(word_ids)
+
+  features = keras.preprocessing.sequence.pad_sequences(
+      features, maxlen=max_words)
+  x_train = np.asarray(features, dtype=np.float32)
+  y_train = np.asarray(labels, dtype=np.int32).reshape((count, 1))
+  x_predict = x_train
+  return x_train, y_train, x_predict
+
+
+def _create_lstm_model(max_words=20,
+                       initial_weights=None,
+                       distribution=None):
+  with MaybeDistributionScope(distribution):
+    word_ids = keras.layers.Input(
+        shape=(max_words,), dtype=np.int32, name='words')
+    word_embed = keras.layers.Embedding(input_dim=100, output_dim=10)(word_ids)
+    lstm_embed = keras.layers.LSTM(
+        units=8, return_sequences=False)(word_embed)
+
+    preds = keras.layers.Dense(2, activation='softmax')(lstm_embed)
+    model = keras.Model(inputs=[word_ids], outputs=[preds])
+
+    if initial_weights:
+      model.set_weights(initial_weights)
+
+    model.compile(
+        optimizer=gradient_descent.GradientDescentOptimizer(learning_rate=0.1),
+        loss='sparse_categorical_crossentropy',
+        metrics=['sparse_categorical_accuracy'])
+  return model
+
+
 def batch_wrapper(dataset, batch_size, distribution, repeat=None):
   if repeat:
     dataset = dataset.repeat(repeat)
@@ -243,7 +313,7 @@ def get_correctness_test_inputs(use_numpy, use_validation_data,
     }
   else:
     if len(x_train) < _GLOBAL_BATCH_SIZE * _EVAL_STEPS:
-      # Currently, we cannot detech the size of a dataset. So, the eval steps is
+      # Currently, we cannot detect the size of a dataset. So, the eval steps is
       # hard coded.
       raise ValueError('x_train must have at least '
                        '_GLOBAL_BATCH_SIZE * _EVAL_STEPS samples')
-- 
GitLab


From 571d0114eda553e2d1b5c9c71f77c2211b5914e3 Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Fri, 4 Jan 2019 16:10:25 -0800
Subject: [PATCH 0265/1095] Internal Cleanup.

PiperOrigin-RevId: 227929845
---
 .../keras/optimizer_v2/optimizer_v2_test.py    | 18 ------------------
 tensorflow/python/keras/optimizers.py          |  5 +++--
 2 files changed, 3 insertions(+), 20 deletions(-)

diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
index 290bdcad14..58c8b1f343 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
@@ -27,7 +27,6 @@ import numpy as np
 from tensorflow.python import keras
 from tensorflow.python.eager import context
 from tensorflow.python.eager import def_function
-from tensorflow.python.eager import function
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -262,23 +261,6 @@ class OptimizerTest(test.TestCase):
       self.evaluate(sgd.iterations.initializer)
       self.assertEqual(0, self.evaluate(sgd.iterations))
 
-  @test_util.run_in_graph_and_eager_modes
-  def testSerializationWithinDefun(self):
-    with self.cached_session():
-      sgd = gradient_descent.SGD(3.0)
-      var0 = resource_variable_ops.ResourceVariable([1.0, 2.0],
-                                                    dtype=dtypes.float32)
-      loss = lambda: 3 * var0
-      sgd.minimize(loss, [var0])
-
-      def serialize():
-        config = sgd.get_config()
-        gradient_descent.SGD.from_config(config)
-
-      compiled_serialize = function.defun(serialize)
-      with self.assertRaisesRegexp(RuntimeError, 'inside Tensorflow graph'):
-        compiled_serialize()
-
   @test_util.run_in_graph_and_eager_modes
   def testConfig(self):
     with self.cached_session():
diff --git a/tensorflow/python/keras/optimizers.py b/tensorflow/python/keras/optimizers.py
index 0ab52fd0a0..12168e5574 100644
--- a/tensorflow/python/keras/optimizers.py
+++ b/tensorflow/python/keras/optimizers.py
@@ -575,7 +575,7 @@ class Adamax(Optimizer):
 
   def get_updates(self, loss, params):
     grads = self.get_gradients(loss, params)
-    self.updates = [state_ops.assign_add(self.iterations, 1)]
+    self.updates = []
 
     lr = self.lr
     if self.initial_decay > 0:
@@ -583,7 +583,8 @@ class Adamax(Optimizer):
           1. / (1. + self.decay * math_ops.cast(self.iterations,
                                                 K.dtype(self.decay))))
 
-    t = math_ops.cast(self.iterations, K.floatx()) + 1
+    with ops.control_dependencies([state_ops.assign_add(self.iterations, 1)]):
+      t = math_ops.cast(self.iterations, K.floatx())
     lr_t = lr / (1. - math_ops.pow(self.beta_1, t))
 
     shapes = [K.int_shape(p) for p in params]
-- 
GitLab


From abdbe6ee3d81d22d70e0181cea2a3bb261f7c09f Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Fri, 4 Jan 2019 16:31:04 -0800
Subject: [PATCH 0266/1095] Add anonymous shared name GUID to avoid kernel
 cache

Fixes #19671
We generate unique shared names for variables in eager mode to avoid
unintended sharing. This results in a memory leak due to the kernel
cache expanding with each new shared_name encountered. To avoid this
we reserve a shared name to send in with the op. When the handle
creation code sees this shared name, it generates a unique name
internally and returns a fresh handle each time.

PiperOrigin-RevId: 227932460
---
 tensorflow/core/framework/resource_handle.cc  |  3 ++
 tensorflow/core/framework/resource_handle.h   |  4 +++
 tensorflow/core/framework/resource_mgr.cc     | 12 ++++++-
 tensorflow/core/framework/resource_mgr.h      | 33 ++++++++++++-------
 tensorflow/python/eager/context.py            | 21 ++++++++++++
 tensorflow/python/eager/def_function.py       |  9 ++---
 tensorflow/python/eager/memory_test.py        | 11 +++++++
 .../python/ops/resource_variable_ops.py       |  6 ++--
 8 files changed, 81 insertions(+), 18 deletions(-)

diff --git a/tensorflow/core/framework/resource_handle.cc b/tensorflow/core/framework/resource_handle.cc
index fc3a329b3b..a3c48d63d4 100644
--- a/tensorflow/core/framework/resource_handle.cc
+++ b/tensorflow/core/framework/resource_handle.cc
@@ -19,6 +19,9 @@ limitations under the License.
 
 namespace tensorflow {
 
+const char ResourceHandle::ANONYMOUS_NAME[] =
+    "cd2c89b7-88b7-44c8-ad83-06c2a9158347";
+
 ResourceHandle::ResourceHandle() {}
 
 ResourceHandle::ResourceHandle(const ResourceHandleProto& proto) {
diff --git a/tensorflow/core/framework/resource_handle.h b/tensorflow/core/framework/resource_handle.h
index db213669a3..c3beffb8ac 100644
--- a/tensorflow/core/framework/resource_handle.h
+++ b/tensorflow/core/framework/resource_handle.h
@@ -67,6 +67,10 @@ class ResourceHandle {
 
   string DebugString() const;
 
+  // GUID for anonymous resources. Resources with this shared_name will have
+  // their shared_name replaced with a GUID at creation time
+  static const char ANONYMOUS_NAME[];
+
  public:
   string device_;
   string container_;
diff --git a/tensorflow/core/framework/resource_mgr.cc b/tensorflow/core/framework/resource_mgr.cc
index 9f3204ab96..6a94ff6642 100644
--- a/tensorflow/core/framework/resource_mgr.cc
+++ b/tensorflow/core/framework/resource_mgr.cc
@@ -13,6 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#include <atomic>
+
 #include "tensorflow/core/framework/resource_mgr.h"
 
 #include "tensorflow/core/framework/device_attributes.pb.h"
@@ -26,6 +28,10 @@ limitations under the License.
 #include "tensorflow/core/platform/demangle.h"
 
 namespace tensorflow {
+
+// Used to generate unique names for anonymous variables
+static std::atomic<int64> current_id_;
+
 ResourceHandle MakeResourceHandle(OpKernelContext* ctx, const string& container,
                                   const string& name,
                                   const TypeIndex& type_index) {
@@ -38,7 +44,11 @@ ResourceHandle MakeResourceHandle(OpKernelContext* ctx, const string& container,
     actual_container = ctx->resource_manager()->default_container();
   }
   result.set_container(actual_container);
-  result.set_name(name);
+  if (name == ResourceHandle::ANONYMOUS_NAME) {
+    result.set_name(strings::StrCat("_AnonymousVar", current_id_.fetch_add(1)));
+  } else {
+    result.set_name(name);
+  }
   result.set_hash_code(type_index.hash_code());
   result.set_maybe_type_name(type_index.name());
   return result;
diff --git a/tensorflow/core/framework/resource_mgr.h b/tensorflow/core/framework/resource_mgr.h
index 3195cd2e9d..8a7c25da92 100644
--- a/tensorflow/core/framework/resource_mgr.h
+++ b/tensorflow/core/framework/resource_mgr.h
@@ -619,20 +619,31 @@ ResourceHandleOp<T>::ResourceHandleOp(OpKernelConstruction* context)
 
 template <typename T>
 void ResourceHandleOp<T>::Compute(OpKernelContext* ctx) {
-  if (!initialized_.load()) {
-    mutex_lock ml(mutex_);
-    // Checking again to see if another thread has initialized the resource.
+  if (name_ == ResourceHandle::ANONYMOUS_NAME) {
+    AllocatorAttributes attr;
+    attr.set_on_host(true);
+    Tensor handle;
+    OP_REQUIRES_OK(
+        ctx, ctx->allocate_temp(DT_RESOURCE, TensorShape({}), &handle, attr));
+    handle.scalar<ResourceHandle>()() =
+        MakeResourceHandle<T>(ctx, container_, name_);
+    ctx->set_output(0, handle);
+  } else {
     if (!initialized_.load()) {
-      AllocatorAttributes attr;
-      attr.set_on_host(true);
-      OP_REQUIRES_OK(ctx, ctx->allocate_temp(DT_RESOURCE, TensorShape({}),
-                                             &resource_, attr));
-      resource_.scalar<ResourceHandle>()() =
-          MakeResourceHandle<T>(ctx, container_, name_);
-      initialized_.store(true);
+      mutex_lock ml(mutex_);
+      // Checking again to see if another thread has initialized the resource.
+      if (!initialized_.load()) {
+        AllocatorAttributes attr;
+        attr.set_on_host(true);
+        OP_REQUIRES_OK(ctx, ctx->allocate_temp(DT_RESOURCE, TensorShape({}),
+                                               &resource_, attr));
+        resource_.scalar<ResourceHandle>()() =
+            MakeResourceHandle<T>(ctx, container_, name_);
+        initialized_.store(true);
+      }
     }
+    ctx->set_output(0, resource_);
   }
-  ctx->set_output(0, resource_);
 }
 
 template <typename T>
diff --git a/tensorflow/python/eager/context.py b/tensorflow/python/eager/context.py
index cd43dc7ab2..faaf742c26 100644
--- a/tensorflow/python/eager/context.py
+++ b/tensorflow/python/eager/context.py
@@ -788,6 +788,27 @@ def in_eager_mode():
   return executing_eagerly()
 
 
+def shared_name(name=None):
+  """Returns the anonymous shared name GUID if no shared name is specified.
+
+  In eager mode we need to use a unique shared name to avoid spurious sharing
+  issues. The runtime generates a unique name on our behalf when the reserved
+  GUID is used as a shared name.
+
+  Args:
+    name: Optional shared name
+
+  Returns:
+    Eager compatible shared name.
+  """
+  if name or not executing_eagerly():
+    return name
+
+  # Ensure a unique name when eager execution is enabled to avoid spurious
+  # sharing issues.
+  return "cd2c89b7-88b7-44c8-ad83-06c2a9158347"
+
+
 def graph_mode():
   """Context-manager to disable eager execution for the current thread."""
   return context()._mode(GRAPH_MODE)  # pylint: disable=protected-access
diff --git a/tensorflow/python/eager/def_function.py b/tensorflow/python/eager/def_function.py
index ebc47d1566..aa4f20df49 100644
--- a/tensorflow/python/eager/def_function.py
+++ b/tensorflow/python/eager/def_function.py
@@ -130,8 +130,9 @@ class UnliftedInitializerVariable(resource_variable_ops.ResourceVariable):
                         if init_from_fn else [initial_value]) as name:
       # pylint: disable=protected-access
       with ops.init_scope():
-        shared_name = ops._name_from_scope_name(name)
-        shared_name = "%s_%d" % (shared_name, ops.uid())
+        handle_name = ops._name_from_scope_name(name)
+        unique_id = "%s_%d" % (handle_name, ops.uid())
+        shared_name = context.shared_name(unique_id)
       with ops.name_scope("Initializer"), ops.device(None):
         initial_value = ops.convert_to_tensor(
             initial_value() if init_from_fn else initial_value,
@@ -144,8 +145,8 @@ class UnliftedInitializerVariable(resource_variable_ops.ResourceVariable):
             name=name,
             graph_mode=self._in_graph_mode)
       self._shape = initial_value.shape
-      self._unique_id = shared_name
-      self._handle_name = shared_name + ":0"
+      self._unique_id = unique_id
+      self._handle_name = handle_name + ":0"
       self._dtype = initial_value.dtype.base_dtype
       self._constraint = constraint
       assert initial_value is not None
diff --git a/tensorflow/python/eager/memory_test.py b/tensorflow/python/eager/memory_test.py
index 5e4516239c..9d29180379 100644
--- a/tensorflow/python/eager/memory_test.py
+++ b/tensorflow/python/eager/memory_test.py
@@ -33,6 +33,7 @@ from tensorflow.python.eager import context
 from tensorflow.python.eager import test
 from tensorflow.python.framework import dtypes
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops.variables import Variable
 
 # memory_profiler might not be available in the OSS version of TensorFlow.
 try:
@@ -81,6 +82,16 @@ class MemoryTest(test.TestCase):
           "Maximum allowed increase: %f") % (initial, increase,
                                              increase_threshold_absolute_mb)
 
+  def testMemoryLeakAnonymousVariable(self):
+    if memory_profiler is None:
+      self.skipTest("memory_profiler required to run this test")
+
+    def f():
+      inputs = Variable(array_ops.zeros([32, 100], dtypes.float32))
+      del inputs
+
+    self.assertNotIncreasingMemory(f, num_iters=10000)
+
   def testMemoryLeakInSimpleModelForwardOnly(self):
     if memory_profiler is None:
       self.skipTest("memory_profiler required to run this test")
diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index d2e5dd9cfe..1d98fb2c89 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -403,10 +403,12 @@ class ResourceVariable(variables.VariableV1):
         handle_name = ops._name_from_scope_name(name)
         if self._in_graph_mode:
           shared_name = handle_name
+          unique_id = shared_name
         else:
           # When in eager mode use a uid for the shared_name, to prevent
           # accidental sharing.
-          shared_name = "%s_%d" % (handle_name, ops.uid())
+          unique_id = "%s_%d" % (handle_name, ops.uid())
+          shared_name = context.shared_name()
         # Use attr_scope and device(None) to simulate the behavior of
         # colocate_with when the variable we want to colocate with doesn't
         # yet exist.
@@ -434,7 +436,7 @@ class ResourceVariable(variables.VariableV1):
               "variable inside a loop or conditional, use a lambda as the "
               "initializer." % name)
         # pylint: enable=protected-access
-        self._unique_id = shared_name
+        self._unique_id = unique_id
         self._initial_value = initial_value if self._in_graph_mode else None
         self._handle_name = handle_name + ":0"
         self._dtype = initial_value.dtype.base_dtype
-- 
GitLab


From 84bddea0109598e47fbe76ba1d443fdcb34c9bf4 Mon Sep 17 00:00:00 2001
From: Paul Donnelly <pauldonnelly@google.com>
Date: Fri, 4 Jan 2019 16:50:13 -0800
Subject: [PATCH 0267/1095] Improve Maxpool op to use cuDNN 7.3 improvements to
 cudnnPoolingForward()

PiperOrigin-RevId: 227934668
---
 tensorflow/core/kernels/maxpooling_op.cc      |  8 ++
 .../core/kernels/maxpooling_op_gpu.cu.cc      |  1 -
 tensorflow/core/kernels/pooling_ops_common.cc | 83 +++++++++++++++----
 tensorflow/stream_executor/cuda/cuda_dnn.cc   | 25 ++++++
 tensorflow/stream_executor/cuda/cuda_dnn.h    |  8 ++
 tensorflow/stream_executor/dnn.h              | 11 +++
 tensorflow/stream_executor/stream.cc          | 22 +++++
 tensorflow/stream_executor/stream.h           |  7 ++
 8 files changed, 149 insertions(+), 16 deletions(-)

diff --git a/tensorflow/core/kernels/maxpooling_op.cc b/tensorflow/core/kernels/maxpooling_op.cc
index 507fc99837..ab235843f7 100644
--- a/tensorflow/core/kernels/maxpooling_op.cc
+++ b/tensorflow/core/kernels/maxpooling_op.cc
@@ -41,6 +41,7 @@ limitations under the License.
 #include "tensorflow/core/util/use_cudnn.h"
 
 #if GOOGLE_CUDA
+#include "cuda/include/cudnn.h"
 #include "tensorflow/core/kernels/maxpooling_op_gpu.h"
 #include "tensorflow/core/kernels/pooling_ops_common_gpu.h"
 #include "tensorflow/core/platform/stream_executor.h"
@@ -1134,11 +1135,18 @@ class MaxPoolingNoMaskOp<GPUDevice, T> : public OpKernel {
                 errors::InvalidArgument(
                     "qint8 should be used with data_format NCHW_VECT_C."));
 
+#if CUDNN_VERSION >= 7300
+    if (use_dnn_) {
+      DnnPoolingOp<T>::Compute(context, se::dnn::PoolingMode::kMaximum, ksize_,
+                               stride_, padding_, data_format_, tensor_in,
+                               out_shape, propagate_nans_);
+#else
     // These is_int8x4 checks avoid linker errors for missing qint8 kernels.
     if (!is_int8x4 && use_dnn_ && data_format_ == FORMAT_NCHW) {
       DnnPoolingOp<T>::Compute(context, se::dnn::PoolingMode::kMaximum, ksize_,
                                stride_, padding_, data_format_, tensor_in,
                                out_shape, propagate_nans_);
+#endif
     } else {
       Tensor* output = nullptr;
       OP_REQUIRES_OK(context, context->allocate_output(0, out_shape, &output));
diff --git a/tensorflow/core/kernels/maxpooling_op_gpu.cu.cc b/tensorflow/core/kernels/maxpooling_op_gpu.cu.cc
index 56d0340547..f28811ffa4 100644
--- a/tensorflow/core/kernels/maxpooling_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/maxpooling_op_gpu.cu.cc
@@ -390,7 +390,6 @@ bool MaxPoolForwardNoMask_NCHW_VECT_C::operator()(
       0, d.stream()>>>(output_size, bottom_data, height, width, channels,
                        pooled_height, pooled_width, kernel_h, kernel_w,
                        stride_h, stride_w, pad_t, pad_l, top_data);
-  d.synchronize();
   return d.ok();
 }
 
diff --git a/tensorflow/core/kernels/pooling_ops_common.cc b/tensorflow/core/kernels/pooling_ops_common.cc
index e583f7feb4..69122f467c 100644
--- a/tensorflow/core/kernels/pooling_ops_common.cc
+++ b/tensorflow/core/kernels/pooling_ops_common.cc
@@ -21,6 +21,7 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor.h"
 
 #if GOOGLE_CUDA
+#include "cuda/include/cudnn.h"
 #include "tensorflow/core/kernels/conv_2d.h"
 #include "tensorflow/core/kernels/pooling_ops_common_gpu.h"
 #include "tensorflow/core/platform/stream_executor.h"
@@ -28,6 +29,20 @@ limitations under the License.
 
 namespace tensorflow {
 
+namespace {
+
+template <typename T>
+struct RawType {
+  using type = T;
+};
+
+template <>
+struct RawType<qint8> {
+  using type = int8;
+};
+
+}  // namespace
+
 PoolParameters::PoolParameters(OpKernelContext* context,
                                const std::vector<int32>& ksize,
                                const std::vector<int32>& stride,
@@ -156,7 +171,10 @@ void DnnPoolingOp<T>::Compute(OpKernelContext* context,
     return;
   }
 
-  /// For now, cudnn does not support NHWC format, so we need to convert it
+  int batch_size = params.tensor_in_batch;
+  int depth = params.depth;
+#if CUDNN_VERSION < 7300
+  /// Earlier versions do not support NHWC format, so we need to convert it
   /// to NCHW before calling cudnn. We need to get rid of this once it is done
   Tensor transformed_input;
   if (data_format == FORMAT_NHWC) {
@@ -181,7 +199,31 @@ void DnnPoolingOp<T>::Compute(OpKernelContext* context,
   } else {
     transformed_output = *tensor_out;
   }
-
+  se::dnn::DataLayout data_layout = se::dnn::DataLayout::kBatchDepthYX;
+#else
+  auto& transformed_input = tensor_in;
+  auto& transformed_output = *tensor_out;
+  se::dnn::DataLayout data_layout;
+  switch (data_format) {
+    case FORMAT_NHWC:
+      data_layout = se::dnn::DataLayout::kBatchYXDepth;
+      break;
+    case FORMAT_NCHW:
+      data_layout = se::dnn::DataLayout::kBatchDepthYX;
+      break;
+    case FORMAT_NCHW_VECT_C:
+      // NCHW_VECT_C is not supported by cudnnPoolingForward(), but can be
+      // emulated via NHWC.
+      data_layout = se::dnn::DataLayout::kBatchYXDepth;
+      batch_size *= depth;
+      depth = 4;
+      break;
+    default:
+      OP_REQUIRES(context, false,
+                  errors::InvalidArgument("Unsupported format: ",
+                                          ToString(data_format)));
+  }
+#endif
   /// Get ready to call cudnn
   se::dnn::PoolingDescriptor pooling_desc;
   pooling_desc.set_pooling_mode(pooling_mode)
@@ -194,23 +236,27 @@ void DnnPoolingOp<T>::Compute(OpKernelContext* context,
       .set_propagate_nans(propagate_nans);
 
   se::dnn::BatchDescriptor input_desc;
-  input_desc.set_count(params.tensor_in_batch)
+  input_desc.set_count(batch_size)
       .set_height(params.tensor_in_rows)
       .set_width(params.tensor_in_cols)
-      .set_feature_map_count(params.depth)
-      .set_layout(se::dnn::DataLayout::kBatchDepthYX);
+      .set_feature_map_count(depth)
+      .set_layout(data_layout);
 
   se::dnn::BatchDescriptor output_desc;
-  output_desc.set_count(params.tensor_in_batch)
+  output_desc.set_count(batch_size)
       .set_height(params.out_height)
       .set_width(params.out_width)
-      .set_feature_map_count(params.depth)
-      .set_layout(se::dnn::DataLayout::kBatchDepthYX);
+      .set_feature_map_count(depth)
+      .set_layout(data_layout);
+
+  auto input_data =
+      AsDeviceMemory(reinterpret_cast<const typename RawType<T>::type*>(
+                         transformed_input.template flat<T>().data()),
+                     transformed_input.template flat<T>().size());
 
-  auto input_data = AsDeviceMemory(transformed_input.template flat<T>().data(),
-                                   transformed_input.template flat<T>().size());
   auto output_data =
-      AsDeviceMemory(transformed_output.template flat<T>().data(),
+      AsDeviceMemory(reinterpret_cast<const typename RawType<T>::type*>(
+                         transformed_output.template flat<T>().data()),
                      transformed_output.template flat<T>().size());
 
   auto* stream = context->op_device_context()->stream();
@@ -222,15 +268,17 @@ void DnnPoolingOp<T>::Compute(OpKernelContext* context,
                     .ok();
   OP_REQUIRES(context, status,
               errors::Internal("cudnn PoolForward launch failed"));
-
+#if CUDNN_VERSION < 7300
   if (data_format == FORMAT_NHWC) {
     /// Transform the output data from NCHW back to NHWC
     auto toConstTensor = [](const Tensor& x) -> const Tensor { return x; };
-    functor::NCHWToNHWC<GPUDevice, T, 4>()(
+    using RT = typename RawType<T>::type;
+    functor::NCHWToNHWC<GPUDevice, RT, 4>()(
         context->eigen_device<Device>(),
-        toConstTensor(transformed_output).template tensor<T, 4>(),
-        tensor_out->tensor<T, 4>());
+        toConstTensor(transformed_output).template tensor<RT, 4>(),
+        tensor_out->tensor<RT, 4>());
   }
+#endif
 }
 
 template <typename T>
@@ -388,6 +436,11 @@ void DnnPoolingGradOp<T>::Compute(
   template class DnnPoolingOp<T>; \
   template class DnnPoolingGradOp<T>;
 TF_CALL_GPU_NUMBER_TYPES(DEFINE_DNN_OPS)
+
+#if CUDNN_VERSION >= 7300
+template class DnnPoolingOp<qint8>;
+#endif
+
 #undef DEFINE_DNN_OPS
 
 #endif  // GOOGLE_CUDA
diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.cc b/tensorflow/stream_executor/cuda/cuda_dnn.cc
index 01ff248150..a34aa9354d 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.cc
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.cc
@@ -4207,6 +4207,31 @@ bool CudnnSupport::DoPoolForward(
   return IsStatusOk(status, /*report_error=*/true);
 }
 
+bool CudnnSupport::DoPoolForward(
+    Stream* stream, const dnn::PoolingDescriptor& pooling_dimensions,
+    const dnn::BatchDescriptor& input_dimensions,
+    const DeviceMemory<int8>& input_data,
+    const dnn::BatchDescriptor& output_dimensions,
+    DeviceMemory<int8>* output_data, ScratchAllocator* workspace_allocator) {
+  // Alpha is the scaling factor for input.
+  float alpha = 1.0;
+  // Beta is the scaling factor for output.
+  float beta = 0.0;
+
+  CudnnTensorDescriptor src_desc(input_dimensions, CUDNN_DATA_INT8);
+  CudnnTensorDescriptor dest_desc(output_dimensions, CUDNN_DATA_INT8);
+  CudnnPoolingDescriptor pooling_desc(pooling_dimensions);
+
+  auto cudnn = cudnn_->GetHandle(parent_, stream);
+  auto status = [&] {
+    RETURN_IF_CUDNN_ERROR(cudnnPoolingForward(
+        cudnn.handle(), pooling_desc.handle(), &alpha, src_desc.handle(),
+        input_data.opaque(), &beta, dest_desc.handle(), output_data->opaque()));
+    return port::Status::OK();
+  }();
+  return IsStatusOk(status, /*report_error=*/true);
+}
+
 bool CudnnSupport::DoPoolBackward(
     Stream* stream, const dnn::PoolingDescriptor& pooling_dimensions,
     const dnn::BatchDescriptor& input_dimensions,
diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.h b/tensorflow/stream_executor/cuda/cuda_dnn.h
index 044ed54514..4cce3c5626 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.h
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.h
@@ -540,6 +540,14 @@ class CudnnSupport : public dnn::DnnSupport {
                      DeviceMemory<Eigen::half>* output_data,
                      ScratchAllocator* workspace_allocator) override;
 
+  bool DoPoolForward(Stream* stream,
+                     const dnn::PoolingDescriptor& pooling_dimensions,
+                     const dnn::BatchDescriptor& input_dimensions,
+                     const DeviceMemory<int8>& input_data,
+                     const dnn::BatchDescriptor& output_dimensions,
+                     DeviceMemory<int8>* output_data,
+                     ScratchAllocator* workspace_allocator) override;
+
   bool DoPoolBackward(Stream* stream,
                       const dnn::PoolingDescriptor& pooling_dimensions,
                       const dnn::BatchDescriptor& input_dimensions,
diff --git a/tensorflow/stream_executor/dnn.h b/tensorflow/stream_executor/dnn.h
index 1001824ed5..ff7b02cf6b 100644
--- a/tensorflow/stream_executor/dnn.h
+++ b/tensorflow/stream_executor/dnn.h
@@ -1617,6 +1617,17 @@ class DnnSupport {
     return false;
   }
 
+  virtual bool DoPoolForward(Stream* stream,
+                             const dnn::PoolingDescriptor& pooling_dimensions,
+                             const dnn::BatchDescriptor& input_dimensions,
+                             const DeviceMemory<int8>& input_data,
+                             const dnn::BatchDescriptor& output_dimensions,
+                             DeviceMemory<int8>* output_data,
+                             ScratchAllocator* workspace_allocator) {
+    LOG(FATAL) << "DoPoolForward not implemented for int8.";
+    return false;
+  }
+
   // Performs differentiation of the pooling operation.
   virtual bool DoPoolBackward(Stream* stream,
                               const dnn::PoolingDescriptor& pooling_dimensions,
diff --git a/tensorflow/stream_executor/stream.cc b/tensorflow/stream_executor/stream.cc
index 3edc66cde8..1b14e4c339 100644
--- a/tensorflow/stream_executor/stream.cc
+++ b/tensorflow/stream_executor/stream.cc
@@ -1490,6 +1490,28 @@ Stream &Stream::ThenPoolForward(
   return *this;
 }
 
+Stream &Stream::ThenPoolForward(
+    const dnn::PoolingDescriptor &pooling_dimensions,
+    const dnn::BatchDescriptor &input_dimensions,
+    const DeviceMemory<int8> &input_data,
+    const dnn::BatchDescriptor &output_dimensions,
+    DeviceMemory<int8> *output_data, ScratchAllocator *workspace_allocator) {
+  VLOG_CALL(PARAM(pooling_dimensions), PARAM(input_dimensions),
+            PARAM(input_data), PARAM(output_dimensions), PARAM(output_data),
+            PARAM(workspace_allocator));
+
+  if (ok()) {
+    if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
+      CheckError(dnn->DoPoolForward(this, pooling_dimensions, input_dimensions,
+                                    input_data, output_dimensions, output_data,
+                                    workspace_allocator));
+    } else {
+      SetErrorAndLogNoDnnSupport();
+    }
+  }
+  return *this;
+}
+
 Stream &Stream::ThenPoolBackward(
     const dnn::PoolingDescriptor &pooling_dimensions,
     const dnn::BatchDescriptor &input_dimensions,
diff --git a/tensorflow/stream_executor/stream.h b/tensorflow/stream_executor/stream.h
index 0fc90cf83d..d5e2fdf58d 100644
--- a/tensorflow/stream_executor/stream.h
+++ b/tensorflow/stream_executor/stream.h
@@ -650,6 +650,13 @@ class Stream {
                           DeviceMemory<Eigen::half> *output_data,
                           ScratchAllocator *workspace_allocator = nullptr);
 
+  Stream &ThenPoolForward(const dnn::PoolingDescriptor &pooling_dimensions,
+                          const dnn::BatchDescriptor &input_dimensions,
+                          const DeviceMemory<int8> &input_data,
+                          const dnn::BatchDescriptor &output_dimensions,
+                          DeviceMemory<int8> *output_data,
+                          ScratchAllocator *workspace_allocator = nullptr);
+
   Stream &ThenPoolBackward(const dnn::PoolingDescriptor &pooling_dimensions,
                            const dnn::BatchDescriptor &input_dimensions,
                            const DeviceMemory<double> &input_data,
-- 
GitLab


From 75f2e9c266ec68d202a11ea5514898ccd1735d86 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Fri, 4 Jan 2019 16:51:01 -0800
Subject: [PATCH 0268/1095] Automated rollback of commit
 7cc1ff9072ce4c2c7f36b7011e24486071ad13ee

PiperOrigin-RevId: 227934767
---
 tensorflow/python/framework/constant_op.py         |  4 ----
 tensorflow/python/kernel_tests/constant_op_test.py | 11 -----------
 2 files changed, 15 deletions(-)

diff --git a/tensorflow/python/framework/constant_op.py b/tensorflow/python/framework/constant_op.py
index 62a25f3fea..ade0797dcd 100644
--- a/tensorflow/python/framework/constant_op.py
+++ b/tensorflow/python/framework/constant_op.py
@@ -277,10 +277,6 @@ def _constant_impl(
                     (num_t, shape, shape.num_elements()))
   g = ops.get_default_graph()
   tensor_value = attr_value_pb2.AttrValue()
-  if tensor_util.is_tensor(value):
-    raise ValueError(
-        ("Cannot create a tf.constant from symbolic Tensor %s. Did you mean "
-         "tf.convert_to_tensor?") % (value,))
   tensor_value.tensor.CopyFrom(
       tensor_util.make_tensor_proto(
           value, dtype=dtype, shape=shape, verify_shape=verify_shape,
diff --git a/tensorflow/python/kernel_tests/constant_op_test.py b/tensorflow/python/kernel_tests/constant_op_test.py
index 2b743ed4c4..583082c2aa 100644
--- a/tensorflow/python/kernel_tests/constant_op_test.py
+++ b/tensorflow/python/kernel_tests/constant_op_test.py
@@ -24,7 +24,6 @@ from google.protobuf import text_format
 
 from tensorflow.core.framework import graph_pb2
 from tensorflow.core.framework import tensor_pb2
-from tensorflow.python.eager import def_function
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes as dtypes_lib
 from tensorflow.python.framework import errors_impl
@@ -631,16 +630,6 @@ class OnesTest(test.TestCase):
         self.assertEqual([2, 3], z.get_shape())
         self.assertAllEqual(z.eval(), np.ones([2, 3]))
 
-  @test_util.run_in_graph_and_eager_modes
-  def testIteratedConstantSensibleException(self):
-
-    @def_function.function
-    def constant_on_constant(x):
-      return constant_op.constant(x)
-
-    with self.assertRaisesRegexp(ValueError, "convert_to_tensor"):
-      constant_on_constant(constant_op.constant(2))
-
 
 class OnesLikeTest(test.TestCase):
 
-- 
GitLab


From fe66882827806db63761b593c70c43340f40750e Mon Sep 17 00:00:00 2001
From: Martin Wicke <wicke@google.com>
Date: Fri, 4 Jan 2019 16:51:20 -0800
Subject: [PATCH 0269/1095] Refactor compatibility upgrade tool to use pasta
 (an AST based refactoring tool) instead of line based edits.

This allows for some simplification (e.g., we can now remove arguments with a dict entry) and for much more powerful transformations.

This passes all tests with no destructive test changes (though there are cosmetic test changes).

Also adds transformations for tf.to_dtype -> tf.cast(..., dtype=...).

PiperOrigin-RevId: 227934801
---
 tensorflow/tools/compatibility/BUILD          |  12 +-
 tensorflow/tools/compatibility/ast_edits.py   | 668 +++++++++---------
 .../tools/compatibility/ast_edits_test.py     |  27 +-
 tensorflow/tools/compatibility/tf_upgrade.py  |  26 +-
 .../tools/compatibility/tf_upgrade_test.py    |   2 +-
 .../tools/compatibility/tf_upgrade_v2.py      | 288 +++++---
 .../tools/compatibility/tf_upgrade_v2_test.py | 105 +--
 tensorflow/tools/pip_package/BUILD            |   1 +
 tensorflow/workspace.bzl                      |   2 +
 third_party/pasta/BUILD                       |   1 +
 third_party/pasta/BUILD.bazel                 |  29 +
 third_party/pasta/BUILD.system                |  13 +
 third_party/pasta/workspace.bzl               |  16 +
 13 files changed, 687 insertions(+), 503 deletions(-)
 create mode 100644 third_party/pasta/BUILD
 create mode 100644 third_party/pasta/BUILD.bazel
 create mode 100644 third_party/pasta/BUILD.system
 create mode 100644 third_party/pasta/workspace.bzl

diff --git a/tensorflow/tools/compatibility/BUILD b/tensorflow/tools/compatibility/BUILD
index a9902d77f5..452e6db941 100644
--- a/tensorflow/tools/compatibility/BUILD
+++ b/tensorflow/tools/compatibility/BUILD
@@ -1,17 +1,21 @@
-licenses(["notice"])  # Apache 2.0
-
-package(default_visibility = ["//tensorflow:internal"])
-
 load(
     "//tensorflow:tensorflow.bzl",
     "tf_copts",  # @unused
     "tf_cc_test",  # @unused
 )
 
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//tensorflow:internal"])
+
 py_library(
     name = "ast_edits",
     srcs = ["ast_edits.py"],
     srcs_version = "PY2AND3",
+    deps = [
+        "@pasta",
+        "@six_archive//:six",
+    ],
 )
 
 py_test(
diff --git a/tensorflow/tools/compatibility/ast_edits.py b/tensorflow/tools/compatibility/ast_edits.py
index 9106ec97c8..0ec6c2ac8c 100644
--- a/tensorflow/tools/compatibility/ast_edits.py
+++ b/tensorflow/tools/compatibility/ast_edits.py
@@ -19,7 +19,6 @@ from __future__ import division
 from __future__ import print_function
 
 import ast
-import collections
 import os
 import re
 import shutil
@@ -27,6 +26,9 @@ import sys
 import tempfile
 import traceback
 
+import pasta
+import six
+
 # Some regular expressions we will need for parsing
 FIND_OPEN = re.compile(r"^\s*(\[).*$")
 FIND_STRING_CHARS = re.compile(r"['\"]")
@@ -44,264 +46,294 @@ class APIChangeSpec(object):
     notifications)
   * `function_reorders`: maps functions whose argument order has changed to the
     list of arguments in the new order
-  * `function_handle`: maps function names to custom handlers for the function
   * `function_warnings`: maps full names of functions to warnings that will be
     printed out if the function is used. (e.g. tf.nn.convolution())
-  * `unrestricted_function_warnings`: maps names of functions to warnings that
-    will be printed out when the function is used (e.g. foo.convolution()).
-  * `function_keyword_additions`: maps function names to a map of arg->value
-    names that should be passed to the function.
+  * `function_transformers`: maps function names to custom handlers
 
   For an example, see `TFAPIChangeSpec`.
   """
 
 
-class _FileEditTuple(
-    collections.namedtuple("_FileEditTuple",
-                           ["comment", "line", "start", "old", "new"])):
-  """Each edit that is recorded by a _FileEditRecorder.
+class _PastaEditVisitor(ast.NodeVisitor):
+  """AST Visitor that processes function calls.
 
-  Fields:
-    comment: A description of the edit and why it was made.
-    line: The line number in the file where the edit occurs (1-indexed).
-    start: The column number in the file where the edit occurs (0-indexed).
-    old: text string to remove (this must match what was in file).
-    new: text string to add in place of `old`.
+  Updates function calls from old API version to new API version using a given
+  change spec.
   """
 
-  __slots__ = ()
+  def __init__(self, api_change_spec):
+    self._api_change_spec = api_change_spec
+    self._log = []   # Holds 3-tuples: line, col, msg.
+    self._errors = []  # Same structure as _log.
+    self._stack = []  # Allow easy access to parents.
 
+  # Overridden to maintain a stack of nodes to allow for parent access
+  def visit(self, node):
+    self._stack.append(node)
+    super(_PastaEditVisitor, self).visit(node)
+    self._stack.pop()
 
-class _FileEditRecorder(object):
-  """Record changes that need to be done to the file."""
+  @property
+  def errors(self):
+    return self._errors
 
-  def __init__(self, filename):
-    # all edits are lists of chars
-    self._filename = filename
+  @property
+  def log(self):
+    return self._log
 
-    self._line_to_edit = collections.defaultdict(list)
-    self._errors = []
+  def _format_log(self, log):
+    text = ""
+    for log_entry in log:
+      text += "Line %d:%d: %s\n" % log_entry
+    return text
 
-  def process(self, text):
-    """Process a list of strings, each corresponding to the recorded changes.
+  def log_text(self):
+    return self._format_log(self.log)
 
-    Args:
-      text: A list of lines of text (assumed to contain newlines)
-    Returns:
-      A tuple of the modified text and a textual description of what is done.
-    Raises:
-      ValueError: if substitution source location does not have expected text.
-    """
+  def add_log(self, lineno, col, msg):
+    self._log.append((lineno, col, msg))
+    print("Line %d:%d: %s" % (lineno, col, msg))
 
-    change_report = ""
-
-    # Iterate of each line
-    for line, edits in self._line_to_edit.items():
-      offset = 0
-      # sort by column so that edits are processed in order in order to make
-      # indexing adjustments cumulative for changes that change the string
-      # length
-      edits.sort(key=lambda x: x.start)
-
-      # Extract each line to a list of characters, because mutable lists
-      # are editable, unlike immutable strings.
-      char_array = list(text[line - 1])
-
-      # Record a description of the change
-      change_report += "%r Line %d\n" % (self._filename, line)
-      change_report += "-" * 80 + "\n\n"
-      for e in edits:
-        change_report += "%s\n" % e.comment
-      change_report += "\n    Old: %s" % (text[line - 1])
-
-      # Make underscore buffers for underlining where in the line the edit was
-      change_list = [" "] * len(text[line - 1])
-      change_list_new = [" "] * len(text[line - 1])
-
-      # Iterate for each edit
-      for e in edits:
-        # Create effective start, end by accounting for change in length due
-        # to previous edits
-        start_eff = e.start + offset
-        end_eff = start_eff + len(e.old)
-
-        # Make sure the edit is changing what it should be changing
-        old_actual = "".join(char_array[start_eff:end_eff])
-        if old_actual != e.old:
-          raise ValueError("Expected text %r but got %r" %
-                           ("".join(e.old), "".join(old_actual)))
-        # Make the edit
-        char_array[start_eff:end_eff] = list(e.new)
-
-        # Create the underline highlighting of the before and after
-        change_list[e.start:e.start + len(e.old)] = "~" * len(e.old)
-        change_list_new[start_eff:end_eff] = "~" * len(e.new)
-
-        # Keep track of how to generate effective ranges
-        offset += len(e.new) - len(e.old)
-
-      # Finish the report comment
-      change_report += "         %s\n" % "".join(change_list)
-      text[line - 1] = "".join(char_array)
-      change_report += "    New: %s" % (text[line - 1])
-      change_report += "         %s\n\n" % "".join(change_list_new)
-    return "".join(text), change_report, self._errors
-
-  def add(self, comment, line, start, old, new, error=None):
-    """Add a new change that is needed.
+  def add_error(self, lineno, col, msg):
+    # All errors are also added to the regular log.
+    self.add_log(lineno, col, msg)
+    self._errors.append((lineno, col, msg))
 
-    Args:
-      comment: A description of what was changed
-      line: Line number (1 indexed)
-      start: Column offset (0 indexed)
-      old: old text
-      new: new text
-      error: this "edit" is something that cannot be fixed automatically
-    Returns:
-      None
-    """
-
-    self._line_to_edit[line].append(
-        _FileEditTuple(comment, line, start, old, new))
-    if error:
-      self._errors.append("%s:%d: %s" % (self._filename, line, error))
-
-
-class _ASTCallVisitor(ast.NodeVisitor):
-  """AST Visitor that processes function calls.
+  def add_logs(self, logs):
+    """Record a log and print it.
 
-  Updates function calls from old API version to new API version using a given
-  change spec.
-  """
-
-  def __init__(self, filename, lines, api_change_spec):
-    self._filename = filename
-    self._file_edit = _FileEditRecorder(filename)
-    self._lines = lines
-    self._api_change_spec = api_change_spec
+    The log should be a tuple (lineno, col_offset, msg), which will be printed
+    and then recorded. It is part of the log available in the self.log property.
 
-  def process(self, lines):
-    return self._file_edit.process(lines)
-
-  def generic_visit(self, node):
-    ast.NodeVisitor.generic_visit(self, node)
-
-  def _rename_functions(self, node, full_name):
-    symbol_renames = self._api_change_spec.symbol_renames
-    try:
-      new_name = symbol_renames[full_name]
-      self._file_edit.add("Renamed function %r to %r" % (full_name, new_name),
-                          node.lineno, node.col_offset, full_name, new_name)
-    except KeyError:
-      pass
-
-  def _print_warning_for_function(self, node, full_name):
-    function_warnings = self._api_change_spec.function_warnings
-    try:
-      warning_message = function_warnings[full_name]
-      warning_message = warning_message.replace("<function name>", full_name)
-      self._file_edit.add(warning_message,
-                          node.lineno, node.col_offset, full_name, full_name,
-                          error="%s requires manual check." % full_name)
-    except KeyError:
-      pass
+    Args:
+      logs: The log to add. Must be a tuple (lineno, col_offset, msg).
+    """
+    self._log.extend(logs)
+    for log in logs:
+      print("Line %d:%d: %s" % log)
 
-  def _print_warning_for_function_unrestricted(self, node):
-    """Print a warning when specific functions are called.
+  def add_errors(self, errors):
+    """Record an error and print it.
 
-    The function _print_warning_for_function matches the full name of the called
-    function, e.g., tf.foo.bar(). This function matches the function name that
-    is called, as long as the function is an attribute. For example,
-    `tf.foo.bar()` and `foo.bar()` are matched, but not `bar()`.
+    The error must be a tuple (lineno, col_offset, msg), which will be printed
+    and then recorded as both a log and an error. It is therefore part of the
+    log available in the self.log as well as the self.errors property.
 
     Args:
-      node: ast.Call object
+      errors: The log to add. Must be a tuple (lineno, col_offset, msg).
     """
-    function_warnings = getattr(
-        self._api_change_spec, "unrestricted_function_warnings", {})
-    if isinstance(node.func, ast.Attribute):
-      function_name = node.func.attr
-      try:
-        warning_message = function_warnings[function_name]
-        self._file_edit.add(warning_message,
-                            node.lineno, node.col_offset, "", "",
-                            error="%s requires manual check." % function_name)
-      except KeyError:
-        pass
-
-  def _get_attribute_full_path(self, node):
-    """Traverse an attribute to generate a full name e.g. tf.foo.bar.
+    self.add_logs(errors)
+    self._errors.extend(errors)
+
+  def _get_applicable_entries(self, transformer_field, full_name, name):
+    """Get all list entries indexed by name that apply to full_name or name."""
+    # Transformers are indexed to full name, name, or no name
+    # as a performance optimization.
+    function_transformers = getattr(self._api_change_spec,
+                                    transformer_field, {})
+
+    glob_name = "*." + name if name else None
+    transformers = []
+    if full_name in function_transformers:
+      transformers.append(function_transformers[full_name])
+    if glob_name in function_transformers:
+      transformers.append(function_transformers[glob_name])
+    if "*" in function_transformers:
+      transformers.append(function_transformers["*"])
+    return transformers
+
+  def _get_applicable_dict(self, transformer_field, full_name, name):
+    """Get all dict entries indexed by name that apply to full_name or name."""
+    # Transformers are indexed to full name, name, or no name
+    # as a performance optimization.
+    function_transformers = getattr(self._api_change_spec,
+                                    transformer_field, {})
+
+    glob_name = "*." + name if name else None
+    transformers = function_transformers.get("*", {}).copy()
+    transformers.update(function_transformers.get(glob_name, {}))
+    transformers.update(function_transformers.get(full_name, {}))
+    return transformers
+
+  def _get_full_name(self, node):
+    """Traverse an Attribute node to generate a full name, e.g., "tf.foo.bar".
+
+    This is the inverse of _full_name_node.
 
     Args:
       node: A Node of type Attribute.
 
     Returns:
-      a '.'-delimited full-name or None if the tree was not a simple form.
+      a '.'-delimited full-name or None if node was not Attribute or Name.
       i.e. `foo()+b).bar` returns None, while `a.b.c` would return "a.b.c".
     """
     curr = node
     items = []
     while not isinstance(curr, ast.Name):
       if not isinstance(curr, ast.Attribute):
-        return None, None
+        return None
       items.append(curr.attr)
       curr = curr.value
     items.append(curr.id)
-    return ".".join(reversed(items)), items[0]
+    return ".".join(reversed(items))
 
-  def _find_true_position(self, node):
-    """Return correct line number and column offset for a given node.
+  def _full_name_node(self, name, ctx=ast.Load()):
+    """Make an Attribute or Name node for name.
 
-    This is necessary mainly because ListComp's location reporting reports
-    the next token after the list comprehension list opening.
+    Translate a qualified name into nested Attribute nodes (and a Name node).
+
+    Args:
+      name: The name to translate to a node.
+      ctx: What context this name is used in. Defaults to Load()
 
     Returns:
-      lineno, offset for the given node
+      A Name or Attribute node.
+    """
+    names = name.split(".")
+    names.reverse()
+    node = ast.Name(id=names.pop(), ctx=ast.Load())
+    while names:
+      node = ast.Attribute(value=node, attr=names.pop(), ctx=ast.Load())
+
+    # Change outermost ctx to the one given to us (inner ones should be Load).
+    node.ctx = ctx
+    return node
+
+  def _maybe_add_warning(self, node, full_name):
+    """Adds an error to be printed about full_name at node."""
+    function_warnings = self._api_change_spec.function_warnings
+    if full_name in function_warnings:
+      warning_message = function_warnings[full_name]
+      warning_message = warning_message.replace("<function name>", full_name)
+      self.add_error(node.lineno, node.col_offset,
+                     "%s requires manual check: %s." % (full_name,
+                                                        warning_message))
+      return True
+    else:
+      return False
+
+  def _maybe_add_call_warning(self, node, full_name, name):
+    """Print a warning when specific functions are called.
+
+    The function _print_warning_for_function matches the full name of the called
+    function, e.g., tf.foo.bar(). This function matches the function name that
+    is called, as long as the function is an attribute. For example,
+    `tf.foo.bar()` and `foo.bar()` are matched, but not `bar()`.
 
     Args:
-      node: Node for which we wish to know the lineno and col_offset
+      node: ast.Call object
+      full_name: The precomputed full name of the callable, if one exists, None
+        otherwise.
+      name: The precomputed name of the callable, if one exists, None otherwise.
+
+    Returns:
+      Whether an error was recorded.
     """
-    if isinstance(node, ast.ListComp):
-      # Strangely, ast.ListComp returns the col_offset of the first token
-      # after the '[' token which appears to be a bug. Workaround by
-      # explicitly finding the real start of the list comprehension.
-      line = node.lineno
-      col = node.col_offset
-      # loop over lines
-      while 1:
-        # Reverse the text to and regular expression search for whitespace
-        text = self._lines[line - 1]
-        reversed_preceding_text = text[:col][::-1]
-        # First find if a [ can be found with only whitespace between it and
-        # col.
-        m = FIND_OPEN.match(reversed_preceding_text)
-        if m:
-          new_col_offset = col - m.start(1) - 1
-          return line, new_col_offset
+    # Only look for *.-warnings here, the other will be handled by the Attribute
+    # visitor. Also, do not warn for bare functions, only if the call func is
+    # an attribute.
+    warned = False
+    if isinstance(node.func, ast.Attribute):
+      warned = self._maybe_add_warning(node, "*." + name)
+
+    # All arg warnings are handled here, since only we have the args
+    arg_warnings = self._get_applicable_dict("function_arg_warnings",
+                                             full_name, name)
+
+    used_args = [kw.arg for kw in node.keywords]
+    for arg, warning in arg_warnings.items():
+      if arg in used_args:
+        warned = True
+        warning_message = warning.replace("<function name>", full_name or name)
+        self.add_error(node.lineno, node.col_offset,
+                       "%s called with %s argument requires manual check: %s." %
+                       (full_name or name, arg, warning_message))
+
+    return warned
+
+  def _maybe_rename(self, parent, node, full_name):
+    """Replace node (Attribute or Name) with a node representing full_name."""
+    new_name = self._api_change_spec.symbol_renames.get(full_name, None)
+    if new_name:
+      self.add_log(node.lineno, node.col_offset,
+                   "Renamed %r to %r" % (full_name, new_name))
+      new_node = self._full_name_node(new_name, node.ctx)
+      ast.copy_location(new_node, node)
+      pasta.ast_utils.replace_child(parent, node, new_node)
+      return True
+    else:
+      return False
+
+  def _maybe_change_to_function_call(self, parent, node, full_name):
+    """Wraps node (typically, an Attribute or Expr) in a Call."""
+    if full_name in self._api_change_spec.change_to_function:
+      if not isinstance(parent, ast.Call):
+        # ast.Call's constructor is really picky about how many arguments it
+        # wants, and also, it changed between Py2 and Py3.
+        if six.PY2:
+          new_node = ast.Call(node, [], [], None, None)
         else:
-          if (reversed_preceding_text == "" or
-              reversed_preceding_text.isspace()):
-            line = line - 1
-            prev_line = self._lines[line - 1]
-            # TODO(aselle):
-            # this is poor comment detection, but it is good enough for
-            # cases where the comment does not contain string literal starting/
-            # ending characters. If ast gave us start and end locations of the
-            # ast nodes rather than just start, we could use string literal
-            # node ranges to filter out spurious #'s that appear in string
-            # literals.
-            comment_start = prev_line.find("#")
-            if comment_start == -1:
-              col = len(prev_line) - 1
-            elif FIND_STRING_CHARS.search(prev_line[comment_start:]) is None:
-              col = comment_start
-            else:
-              return None, None
-          else:
-            return None, None
-    # Most other nodes return proper locations (with notably does not), but
-    # it is not possible to use that in an argument.
-    return node.lineno, node.col_offset
+          new_node = ast.Call(node, [], [])
+        pasta.ast_utils.replace_child(parent, node, new_node)
+        ast.copy_location(new_node, node)
+        self.add_log(node.lineno, node.col_offset,
+                     "Changed %r to a function call" % full_name)
+        return True
+    return False
+
+  def _maybe_add_arg_names(self, node, full_name):
+    """Make args into keyword args if function called full_name requires it."""
+    function_reorders = self._api_change_spec.function_reorders
+
+    if full_name in function_reorders:
+      reordered = function_reorders[full_name]
+      new_keywords = []
+      for idx, arg in enumerate(node.args):
+        keyword_arg = reordered[idx]
+        new_keywords.append(ast.keyword(arg=keyword_arg, value=arg))
+
+      if new_keywords:
+        self.add_log(node.lineno, node.col_offset,
+                     "Added keywords to args of function %r" % full_name)
+        node.args = []
+        node.keywords = new_keywords + (node.keywords or [])
+        return True
+    return False
+
+  def _maybe_modify_args(self, node, full_name, name):
+    """Rename keyword args if the function called full_name requires it."""
+    renamed_keywords = self._get_applicable_dict("function_keyword_renames",
+                                                 full_name, name)
+
+    if not renamed_keywords:
+      return False
+
+    modified = False
+    new_keywords = []
+    for keyword in node.keywords:
+      argkey = keyword.arg
+      if argkey in renamed_keywords:
+        modified = True
+        if renamed_keywords[argkey] is None:
+          lineno = getattr(keyword, "lineno", node.lineno)
+          col_offset = getattr(keyword, "col_offset", node.col_offset)
+          self.add_log(lineno, col_offset,
+                       "Removed argument %s for function %s" % (
+                           argkey, full_name or name))
+        else:
+          keyword.arg = renamed_keywords[argkey]
+          lineno = getattr(keyword, "lineno", node.lineno)
+          col_offset = getattr(keyword, "col_offset", node.col_offset)
+          self.add_log(lineno, col_offset,
+                       "Renamed keyword argument for %s from %s to %s" % (
+                           full_name, argkey, renamed_keywords[argkey]))
+          new_keywords.append(keyword)
+      else:
+        new_keywords.append(keyword)
+
+    if modified:
+      node.keywords = new_keywords
+    return modified
 
   def visit_Call(self, node):  # pylint: disable=invalid-name
     """Handle visiting a call node in the AST.
@@ -309,104 +341,74 @@ class _ASTCallVisitor(ast.NodeVisitor):
     Args:
       node: Current Node
     """
-    self._print_warning_for_function_unrestricted(node)
-
-    # Find a simple attribute name path e.g. "tf.foo.bar"
-    full_name, name = self._get_attribute_full_path(node.func)
-
-    # Make sure the func is marked as being part of a call
-    node.func.is_function_for_call = True
+    assert self._stack[-1] is node
 
+    # Get the name for this call, so we can index stuff with it.
+    full_name = self._get_full_name(node.func)
     if full_name:
-      # Call special handlers
-      function_handles = self._api_change_spec.function_handle
-      glob_name = "*.{}".format(name)
-      if glob_name in function_handles:
-        function_handles[glob_name](self._file_edit, node, self._lines)
-      if full_name in function_handles:
-        function_handles[full_name](self._file_edit, node, self._lines)
-
-      # Examine any non-keyword argument and make it into a keyword argument
-      # if reordering required.
-      function_reorders = self._api_change_spec.function_reorders
-      function_keyword_renames = (
-          self._api_change_spec.function_keyword_renames)
-
-      if full_name in function_reorders:
-        reordered = function_reorders[full_name]
-        for idx, arg in enumerate(node.args):
-          lineno, col_offset = self._find_true_position(arg)
-          if lineno is None or col_offset is None:
-            self._file_edit.add(
-                "Failed to add keyword %r to reordered function %r" %
-                (reordered[idx], full_name),
-                arg.lineno,
-                arg.col_offset,
-                "",
-                "",
-                error="A necessary keyword argument failed to be inserted.")
-          else:
-            keyword_arg = reordered[idx]
-            if (full_name in function_keyword_renames and
-                keyword_arg in function_keyword_renames[full_name]):
-              keyword_arg = function_keyword_renames[full_name][keyword_arg]
-            self._file_edit.add("Added keyword %r to reordered function %r" %
-                                (reordered[idx], full_name), lineno, col_offset,
-                                "", keyword_arg + "=")
-
-      # Examine each keyword argument and convert it to the final renamed form
-      renamed_keywords = ({} if full_name not in function_keyword_renames else
-                          function_keyword_renames[full_name])
-      for keyword in node.keywords:
-        argkey = keyword.arg
-        argval = keyword.value
-
-        if argkey in renamed_keywords:
-          argval_lineno, argval_col_offset = self._find_true_position(argval)
-          if argval_lineno is not None and argval_col_offset is not None:
-            # TODO(aselle): We should scan backward to find the start of the
-            # keyword key. Unfortunately ast does not give you the location of
-            # keyword keys, so we are forced to infer it from the keyword arg
-            # value.
-            key_start = argval_col_offset - len(argkey) - 1
-            key_end = key_start + len(argkey) + 1
-            if (self._lines[argval_lineno - 1][key_start:key_end] == argkey +
-                "="):
-              self._file_edit.add("Renamed keyword argument from %r to %r" %
-                                  (argkey,
-                                   renamed_keywords[argkey]), argval_lineno,
-                                  argval_col_offset - len(argkey) - 1,
-                                  argkey + "=", renamed_keywords[argkey] + "=")
-              continue
-          self._file_edit.add(
-              "Failed to rename keyword argument from %r to %r" %
-              (argkey, renamed_keywords[argkey]),
-              argval.lineno,
-              argval.col_offset - len(argkey) - 1,
-              "",
-              "",
-              error="Failed to find keyword lexographically. Fix manually.")
-
-    ast.NodeVisitor.generic_visit(self, node)
+      name = full_name.split(".")[-1]
+    elif isinstance(node.func, ast.Name):
+      name = node.func.id
+    elif isinstance(node.func, ast.Attribute):
+      name = node.func.attr
+    else:
+      name = None
+
+    # Call standard transformers for this node.
+    # Make sure warnings come first, since args or names triggering warnings
+    # may be removed by the other transformations.
+    self._maybe_add_call_warning(node, full_name, name)
+    # Make all args into kwargs
+    self._maybe_add_arg_names(node, full_name)
+    # Argument name changes or deletions
+    self._maybe_modify_args(node, full_name, name)
+
+    # Call transformers. These have the ability to modify the node, and if they
+    # do, will return the new node they created (or the same node if they just
+    # changed it). The are given the parent, but we will take care of
+    # integrating their changes into the parent if they return a new node.
+    #
+    # These are matched on the old name, since renaming is performed by the
+    # Attribute visitor, which happens later.
+    transformers = self._get_applicable_entries("function_transformers",
+                                                full_name, name)
+
+    parent = self._stack[-2]
+
+    for transformer in transformers:
+      logs = []
+      errors = []
+      new_node = transformer(parent, node, full_name, name, logs, errors)
+      self.add_logs(logs)
+      self.add_errors(errors)
+      if new_node:
+        if new_node is not node:
+          pasta.ast_utils.replace_child(parent, node, new_node)
+          node = new_node
+          self._stack[-1] = node
+
+    self.generic_visit(node)
 
   def visit_Attribute(self, node):  # pylint: disable=invalid-name
-    """Handle bare Attributes i.e. [tf.foo, tf.bar].
+    """Handle bare Attributes i.e. [tf.foo, tf.bar]."""
+    assert self._stack[-1] is node
 
-    Args:
-      node: Node that is of type ast.Attribute
-    """
-    full_name, _ = self._get_attribute_full_path(node)
+    full_name = self._get_full_name(node)
     if full_name:
+      parent = self._stack[-2]
+
       # Make sure the warning comes first, otherwise the name may have changed
-      self._print_warning_for_function(node, full_name)
-      self._rename_functions(node, full_name)
-    if full_name in self._api_change_spec.change_to_function:
-      if not hasattr(node, "is_function_for_call"):
-        new_text = full_name + "()"
-        self._file_edit.add("Changed %r to %r" % (full_name, new_text),
-                            node.lineno, node.col_offset, full_name, new_text)
+      self._maybe_add_warning(node, full_name)
 
-    ast.NodeVisitor.generic_visit(self, node)
+      # Once we did a modification, node is invalid and not worth inspecting
+      # further. Also, we only perform modifications for simple nodes, so
+      # There'd be no point in descending further.
+      if self._maybe_rename(parent, node, full_name):
+        return
+      if self._maybe_change_to_function_call(parent, node, full_name):
+        return
+
+    self.generic_visit(node)
 
 
 class ASTCodeUpgrader(object):
@@ -429,16 +431,42 @@ class ASTCodeUpgrader(object):
     """
 
     # Write to a temporary file, just in case we are doing an implace modify.
+    # pylint: disable=g-backslash-continuation
     with open(in_filename, "r") as in_file, \
         tempfile.NamedTemporaryFile("w", delete=False) as temp_file:
       ret = self.process_opened_file(in_filename, in_file, out_filename,
                                      temp_file)
+    # pylint: enable=g-backslash-continuation
 
     shutil.move(temp_file.name, out_filename)
     return ret
 
-  # Broad exceptions are required here because ast throws whatever it wants.
-  # pylint: disable=broad-except
+  def _format_errors(self, errors, in_filename):
+    return ["%s:%d:%d: %s" % ((in_filename,) + error) for error in errors]
+
+  def update_string_pasta(self, text, in_filename):
+    """Updates a file using pasta."""
+    try:
+      t = pasta.parse(text)
+    except (SyntaxError, ValueError, TypeError):
+      log = "Failed to parse.\n\n" + traceback.format_exc()
+      return 0, "", log, []
+
+    visitor = _PastaEditVisitor(self._api_change_spec)
+    visitor.visit(t)
+
+    errors = self._format_errors(visitor.errors, in_filename)
+    return 1, pasta.dump(t), visitor.log_text(), errors
+
+  def _format_log(self, log, in_filename, out_filename):
+    text = "-" * 80 + "\n"
+    text += "Processing file %r\n outputting to %r\n" % (in_filename,
+                                                         out_filename)
+    text += "-" * 80 + "\n\n"
+    text += log
+    text += "-" * 80 + "\n\n"
+    return text
+
   def process_opened_file(self, in_filename, in_file, out_filename, out_file):
     """Process the given python file for incompatible changes.
 
@@ -453,30 +481,16 @@ class ASTCodeUpgrader(object):
     Returns:
       A tuple representing number of files processed, log of actions, errors
     """
-    process_errors = []
-    text = "-" * 80 + "\n"
-    text += "Processing file %r\n outputting to %r\n" % (in_filename,
-                                                         out_filename)
-    text += "-" * 80 + "\n\n"
-
-    parsed_ast = None
     lines = in_file.readlines()
-    try:
-      parsed_ast = ast.parse("".join(lines))
-    except Exception:
-      text += "Failed to parse %r\n\n" % in_filename
-      text += traceback.format_exc()
-    if parsed_ast:
-      visitor = _ASTCallVisitor(in_filename, lines, self._api_change_spec)
-      visitor.visit(parsed_ast)
-      out_text, new_text, process_errors = visitor.process(lines)
-      text += new_text
-      if out_file:
-        out_file.write(out_text)
-    text += "\n"
-    return 1, text, process_errors
-
-  # pylint: enable=broad-except
+    processed_file, new_file_content, log, process_errors = (
+        self.update_string_pasta("".join(lines), in_filename))
+
+    if out_file and processed_file:
+      out_file.write(new_file_content)
+
+    return (processed_file,
+            self._format_log(log, in_filename, out_filename),
+            process_errors)
 
   def process_tree(self, root_directory, output_root_directory,
                    copy_other_files):
diff --git a/tensorflow/tools/compatibility/ast_edits_test.py b/tensorflow/tools/compatibility/ast_edits_test.py
index 99f20a026f..b2e9f67638 100644
--- a/tensorflow/tools/compatibility/ast_edits_test.py
+++ b/tensorflow/tools/compatibility/ast_edits_test.py
@@ -39,6 +39,8 @@ following new APIs:
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
+import ast
+import pasta
 import six
 from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test as test_lib
@@ -54,7 +56,6 @@ class NoUpdateSpec(ast_edits.APIChangeSpec):
     self.function_keyword_renames = {}
     self.symbol_renames = {}
     self.function_warnings = {}
-    self.unrestricted_function_warnings = {}
     self.change_to_function = {}
 
 
@@ -401,7 +402,8 @@ class TestAstEdits(test_util.TensorFlowTestCase):
 
       def __init__(self):
         NoUpdateSpec.__init__(self)
-        self.unrestricted_function_warnings = {"foo": "not good"}
+        self.function_warnings = {"*.foo": "not good"}
+
     texts = ["object.foo()", "get_object().foo()",
              "get_object().foo()", "object.foo().bar()"]
     for text in texts:
@@ -416,5 +418,26 @@ class TestAstEdits(test_util.TensorFlowTestCase):
       self.assertNotIn("not good", report)
 
 
+class ManualEditsTest(test_util.TensorFlowTestCase):
+
+  def disabled_test_arg_order(self):
+    """Tests that generated arg order is sane."""
+    text = "f(a)"
+    t = pasta.parse(text)
+    node = pasta.ast_utils.find_nodes_by_type(t, (ast.Call,))[0]
+    arg = ast.keyword(arg="b", value=ast.Num(n=0))
+    node.keywords.append(arg)
+
+    # This is only needed in Python3, and I think it's a bug (but maybe in ast).
+    arg.value.lineno = 0
+    arg.value.col_offset = 0
+
+    # pasta.dump should never put kwargs before args, even if the col_offset is
+    # messed up.
+    # This fails if run with python3, but works find for python2.
+    # In python3, the dump yields "f(b=0, a)".
+    self.assertEqual(pasta.dump(t), "f(a, b=0)")
+
+
 if __name__ == "__main__":
   test_lib.main()
diff --git a/tensorflow/tools/compatibility/tf_upgrade.py b/tensorflow/tools/compatibility/tf_upgrade.py
index 68d2c02570..241b08510f 100644
--- a/tensorflow/tools/compatibility/tf_upgrade.py
+++ b/tensorflow/tools/compatibility/tf_upgrade.py
@@ -175,27 +175,13 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.op_scope": ["values", "name", "default_name"],
     }
 
-    # Specially handled functions.
-    self.function_handle = {"tf.reverse": self._reverse_handler}
-
     # Warnings that should be printed if corresponding functions are used.
-    self.function_warnings = {}
-
-  @staticmethod
-  def _reverse_handler(file_edit_recorder, node, lines):
-    del lines
-    # TODO(aselle): Could check for a literal list of bools and try to convert
-    # them to indices.
-    comment = ("ERROR: tf.reverse has had its argument semantics changed "
-               "significantly the converter cannot detect this reliably, so "
-               "you need to inspect this usage manually.\n")
-    file_edit_recorder.add(
-        comment,
-        node.lineno,
-        node.col_offset,
-        "tf.reverse",
-        "tf.reverse",
-        error="tf.reverse requires manual check.")
+    self.function_warnings = {
+        "tf.reverse":
+            "ERROR: tf.reverse has had its argument semantics changed "
+            "significantly. The converter cannot detect this reliably, so "
+            "you need to inspect this usage manually.\n",
+    }
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/tools/compatibility/tf_upgrade_test.py b/tensorflow/tools/compatibility/tf_upgrade_test.py
index 66325ea2ad..cf05575a9d 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_test.py
@@ -112,7 +112,7 @@ class TestUpgrade(test_util.TensorFlowTestCase):
     text = "tf.reverse(a, b)\n"
     _, unused_report, errors, new_text = self._upgrade(text)
     self.assertEqual(new_text, new_text)
-    self.assertEqual(errors, ["test.py:1: tf.reverse requires manual check."])
+    self.assertIn("tf.reverse requires manual check", errors[0])
 
   def testListComprehension(self):
     def _test(input, output):  # pylint: disable=redefined-builtin
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 82839afc3d..9d9c5878f7 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -18,7 +18,9 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import re
+import ast
+
+import pasta
 
 from tensorflow.tools.compatibility import ast_edits
 from tensorflow.tools.compatibility import renames_v2
@@ -31,7 +33,22 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
   def __init__(self):
     # Maps from a function name to a dictionary that describes how to
     # map from an old argument keyword to the new argument keyword.
+    # If the new argument is None, it will be removed.
+    # Only keyword args are handled, so make sure to also put any function in
+    # function_reorders to ensure that all args are made into keywords first.
     self.function_keyword_renames = {
+        "tf.gradients": {
+            "colocate_gradients_with_ops": None,
+        },
+        "tf.hessians": {
+            "colocate_gradients_with_ops": None,
+        },
+        "*.minimize": {
+            "colocate_gradients_with_ops": None,
+        },
+        "*.compute_gradients": {
+            "colocate_gradients_with_ops": None,
+        },
         "tf.argmin": {
             "dimension": "axis",
         },
@@ -672,14 +689,31 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     # positional arguments yourself, this could do the wrong thing.
     self.function_reorders = reorders_v2.reorders
 
-    # Specially handled functions.
-    self.function_handle = {
-        "tf.batch_gather": self._batch_gather_handler,
-        "tf.nn.dropout": self._dropout_handler,
-        "tf.gradients": self._colocate_handler("tf.gradients"),
-        "*.minimize": self._colocate_handler("Optimizer.minimize"),
-        "*.compute_gradients":
-            self._colocate_handler("Optimizer.compute_gradients"),
+    # Specially handled functions (pasta version)
+    # Each transformer is a callable which will be called with the arguments
+    #   transformer(parent, node, full_name, name, logs, errors)
+    # Where logs and errors are lists to which (line, col, msg) tuples can be
+    # appended, full_name is the FQN of the function called (or None if that is
+    # unknown), name is the name of the function called (or None is that is
+    # unknown). node is an ast.Call node representing this function call, and
+    # parent is its parent in the AST.
+    # The function may modify node (but not parent), and must return
+    # - none, if nothing was modified
+    # - node, if node was modified in place (make sure to use
+    #   pasta.ast_utils.replace_child to swap out children, otherwise formatting
+    #   may get messy)
+    # - a replacement for node, if the whole call node was replaced. The caller
+    #   will take care of changing parent.
+    self.function_transformers = {
+        "tf.nn.dropout": self._dropout_transformer,
+        "tf.batch_gather": self._batch_gather_transformer,
+        "tf.to_bfloat16": self._cast_transformer,
+        "tf.to_complex128": self._cast_transformer,
+        "tf.to_complex64": self._cast_transformer,
+        "tf.to_double": self._cast_transformer,
+        "tf.to_float": self._cast_transformer,
+        "tf.to_int32": self._cast_transformer,
+        "tf.to_int64": self._cast_transformer,
     }
 
     decay_function_comment = (
@@ -748,9 +782,45 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "compat.v1 for backward compatibility. Please update these calls to "
         "the TF 2.0 versions.")
 
+    export_saved_model_renamed = (
+        "(Manual edit required) Please rename the method export_savedmodel() "
+        "to export_saved_model(). Two things to note:\n\t(1) The argument "
+        "strip_default_attributes has been removed. The function will always "
+        "strip the default attributes from ops. If this breaks your code, "
+        "please switch to tf.compat.v1.estimator.Estimator.\n\t(2) This change "
+        "only effects core estimator. If you are using "
+        "tf.contrib.learn.Estimator, please switch to using core estimator.")
+
+    make_initializable_iterator_deprecation = (
+        "(Manual edit required) The "
+        "`tf.data.Dataset.make_initializable_iterator()` method has been "
+        "removed. If you are using the Estimator API, you can return a dataset "
+        "directly from your input functions without creating an iterator. "
+        "As a last resort, please replace calls to that method on `dataset` "
+        "with a call to "
+        "`tf.compat.v1.data.make_initializable_iterator(dataset)`.")
+
+    make_one_shot_iterator_deprecation = (
+        "(Manual edit required) The "
+        "`tf.data.Dataset.make_one_shot_iterator()` method has been "
+        "removed. If you are using eager execution, you can iterate over "
+        "`dataset` using a Python `for` loop. If you are using the Estimator "
+        "API, you can return a dataset directly from your input functions "
+        "without creating an iterator. As a last resort, please replace calls "
+        "to that method on `dataset` with a call to "
+        "`tf.compat.v1.data.make_one_shot_iterator(dataset)`.")
+
     # Function warnings. <function name> placeholder inside warnings will be
     # replaced by function name.
+    # You can use *. to add items which do not check the FQN, and apply to e.g.,
+    # methods.
     self.function_warnings = {
+        "*.export_savedmodel":
+            export_saved_model_renamed,
+        "*.make_initializable_iterator":
+            make_initializable_iterator_deprecation,
+        "*.make_one_shot_iterator":
+            make_one_shot_iterator_deprecation,
         "tf.assert_greater":
             assert_return_type_comment,
         "tf.assert_equal":
@@ -834,11 +904,6 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             default_loss_reduction_changed,
         "tf.estimator.BaselineRegressor":
             default_loss_reduction_changed,
-        "tf.hessians":
-            "tf.hessians no longer takes "
-            "'colocate_gradients_with_ops' argument. Also, "
-            "arguments have been reordered so that 'name' is the "
-            "last argument.",
         "tf.nn.conv1d":
             "WARNING: use_cudnn_on_gpu argument has been removed and \"value\""
             " was renamed to \"input\"",
@@ -1064,111 +1129,118 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             metrics_comment,
     }
 
+    # Warnings that are emitted only if a specific arg is found.
+    self.function_arg_warnings = {
+        "tf.gradients": {
+            "colocate_gradients_with_ops":
+                "tf.gradients no longer takes "
+                "'colocate_gradients_with_ops' argument, it behaves as if it "
+                "was set to True.",
+        },
+        "*.minimize": {
+            "colocate_gradients_with_ops":
+                "Optimizer.minimize no longer takes "
+                "'colocate_gradients_with_ops' argument, it behaves as if it "
+                "was set to True.",
+        },
+        "*.compute_gradients": {
+            "colocate_gradients_with_ops":
+                "Optimizer.compute_gradients no "
+                "longer takes 'colocate_gradients_with_ops' argument, it "
+                "behaves as if it was set to True.",
+        },
+    }
+
     self.symbol_renames = {
         name: new_name
         for name, new_name in self.symbol_renames.items()
     }
 
-    export_saved_model_renamed = (
-        "(Manual edit required) Please rename the method export_savedmodel() "
-        "to export_saved_model(). Two things to note:\n\t(1) The argument "
-        "strip_default_attributes has been removed. The function will always "
-        "strip the default attributes from ops. If this breaks your code, "
-        "please switch to tf.compat.v1.estimator.Estimator.\n\t(2) This change "
-        "only effects core estimator. If you are using "
-        "tf.contrib.learn.Estimator, please switch to using core estimator.")
-
-    make_initializable_iterator_deprecation = (
-        "(Manual edit required) The "
-        "`tf.data.Dataset.make_initializable_iterator()` method has been "
-        "removed. If you are using the Estimator API, you can return a dataset "
-        "directly from your input functions without creating an iterator. "
-        "As a last resort, please replace calls to that method on `dataset` "
-        "with a call to "
-        "`tf.compat.v1.data.make_initializable_iterator(dataset)`.")
-
-    make_one_shot_iterator_deprecation = (
-        "(Manual edit required) The "
-        "`tf.data.Dataset.make_one_shot_iterator()` method has been "
-        "removed. If you are using eager execution, you can iterate over "
-        "`dataset` using a Python `for` loop. If you are using the Estimator "
-        "API, you can return a dataset directly from your input functions "
-        "without creating an iterator. As a last resort, please replace calls "
-        "to that method on `dataset` with a call to "
-        "`tf.compat.v1.data.make_one_shot_iterator(dataset)`.")
+  @staticmethod
+  def _dropout_transformer(parent, node, full_name, name, logs, errors):
+    def _replace_keep_prob_node(parent, old_value):
+      """Replaces old_value with 1-(old_value)."""
+      one = ast.Num(n=1)
+      one.lineno = 0
+      one.col_offset = 0
+      new_value = ast.BinOp(left=one, op=ast.Sub(),
+                            right=old_value)
+      # This copies the prefix and suffix on old_value to new_value.
+      pasta.ast_utils.replace_child(parent, old_value, new_value)
+      ast.copy_location(new_value, old_value)
+      # Put parentheses around keep_prob.value (and remove the old prefix/
+      # suffix, they should only be around new_value).
+      pasta.base.formatting.set(old_value, "prefix", "(")
+      pasta.base.formatting.set(old_value, "suffix", ")")
 
-    # Specify warnings for functions that aren't restricted to the tf.x.y.z
-    # format. This should only be used for methods with unique names, e.g.
-    # export_savedmodel, which is only defined in Estimator objects.
-    self.unrestricted_function_warnings = {
-        "export_savedmodel": export_saved_model_renamed,
-        "make_initializable_iterator": make_initializable_iterator_deprecation,
-        "make_one_shot_iterator": make_one_shot_iterator_deprecation,
-    }
+    # Check if we have a keep_prob keyword arg
+    for keep_prob in node.keywords:
+      if keep_prob.arg == "keep_prob":
+        logs.append((node.lineno, node.col_offset,
+                     "Changing keep_prob arg of tf.nn.dropout to rate, and "
+                     "recomputing value. Please check this transformation.\n"))
+        keep_prob.arg = "rate"
+        _replace_keep_prob_node(keep_prob, keep_prob.value)
+        return node
 
-  @staticmethod
-  def _dropout_handler(file_edit_recorder, node, lines):
-    del lines
+    # Maybe it was a positional arg
     if len(node.args) < 2:
-      comment = ("ERROR: tf.nn.dropout did not take arguments, so automatic "
-                 "transformation was disabled. tf.nn.dropout has changed "
-                 "the semantics of the second argument.")
-      file_edit_recorder.add(
-          comment,
-          node.lineno,
-          node.col_offset,
-          "tf.nn.dropout",
-          "tf.nn.dropout",
-          error="tf.nn.dropout requires manual check.")
+      errors.append((node.lineno, node.col_offset,
+                     "ERROR: tf.nn.dropout called without arguments, so "
+                     "automatic fix was disabled. tf.nn.dropout has changed "
+                     "the semantics of the second argument."))
     else:
-      comment = ("WARNING: tf.nn.dropout has changed the semantics of the "
-                 "second argument. Please check the transformation.\n")
-      file_edit_recorder.add(
-          comment,
-          node.args[1].lineno,
-          node.args[1].col_offset,
-          "",
-          "1 - ")
+      _replace_keep_prob_node(node, node.args[1])
+      logs.append((node.lineno, node.col_offset,
+                   "Changing keep_prob arg of tf.nn.dropout to rate, and "
+                   "recomputing value.\n"))
+      errors.append((node.lineno, node.col_offset,
+                     "WARNING: tf.nn.dropout has changed the semantics of the "
+                     "second argument. Please check the applied transformation."
+                    ))
+      return node
 
   @staticmethod
-  def _colocate_handler(name):
-    def _helper(file_edit_recorder, node, lines):
-      """Handler for updating colocate arguments."""
-      del lines
-      for keyword in node.keywords:
-        if keyword.arg == "colocate_gradients_with_ops":
-          # TODO(jhseu): Since ast_edit.py does string replacement, there's no
-          # straightforward way to remove the argument. Try to fix before 2.0 is
-          # final.
-          comment = ("For tf.gradients and tf.Optimizer.minimize, "
-                     "colocate_gradients_with_op has been removed and now "
-                     "defaults to True.")
-          file_edit_recorder.add(
-              comment,
-              node.lineno,
-              node.col_offset,
-              "",
-              "",
-              error="{} requires manual check.".format(name))
-    return _helper
+  def _cast_transformer(parent, node, full_name, name, logs, errors):
+    """Transforms to_int and to_float to cast(..., dtype=...)."""
 
-  @staticmethod
-  def _batch_gather_handler(file_edit_recorder, node, lines):
-    lineno = node.lineno
-    column = node.col_offset
+    # Find out the dtype to cast to from the function name
+    dtype_str = name[3:]
+    new_arg = ast.keyword(arg="dtype",
+                          value=ast.Attribute(value=ast.Name(id="tf",
+                                                             ctx=ast.Load()),
+                                              attr=dtype_str, ctx=ast.Load()))
+
+    # Python3 ast requires the args for the Attribute, but codegen will mess up
+    # the arg order if we just set them to 0.
+    new_arg.value.lineno = node.lineno
+    new_arg.value.col_offset = node.col_offset+100
 
-    # Find the position to add the batch_dims argument.  We add it as the
-    # first argument, since that's easiest.  This is safe because we included
-    # batch_gather in self.reordered_function_names, so it will have all
-    # of its arguments changed to keyword arguments.
-    m = re.match(r"tf\s*\.\s*batch_gather\s*\(", lines[lineno - 1][column:])
-    if m is not None:
-      file_edit_recorder.add(
-          "Added keyword argument 'batch_dims=-1' to 'tf.batch_gather'",
-          lineno, column + m.end(), "", "batch_dims=-1, ")
+    node.keywords.append(new_arg)
+    if isinstance(node.func, ast.Attribute):
+      node.func.attr = "cast"
     else:
-      file_edit_recorder.add(
-          "Unable to add keyword argument 'batch_dims=-1' to 'tf.batch_gather'",
-          lineno, column, "", "",
-          error="Unable to add keyword argument batch_dims=-1 to "
-          "tf.batch_gather; please add it manually.")
+      assert isinstance(node.func, ast.Name)
+      node.func.id = "cast"
+
+    logs.append((node.lineno, node.col_offset,
+                 "Changed %s call to tf.cast(..., dtype=tf.%s)." % (full_name,
+                                                                    dtype_str)))
+    return node
+
+  @staticmethod
+  def _batch_gather_transformer(parent, node, full_name, name, logs, errors):
+    # Check if the call already has a batch_dims argument
+    if any([kw.arg == "batch_dims" for kw in node.keywords]):
+      logs.append((node.lineno, node.col_offset, "tf.batch_gather already has "
+                   "batch_dims argument. Neat."))
+      return None
+
+    minus_one = ast.Num(n=-1)
+    minus_one.lineno = 0
+    minus_one.col_offset = 0
+    new_arg = ast.keyword("batch_dims", minus_one)
+    node.keywords.append(new_arg)
+    logs.append((node.lineno, node.col_offset,
+                 "Added keyword argument batch_dims=-1 to tf.batch_gather."))
+    return node
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index 54120cfdd7..d8bc0ba75a 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -239,8 +239,8 @@ class TestUpgrade(test_util.TensorFlowTestCase):
     }
     function_warnings = (
         tf_upgrade_v2.TFAPIChangeSpec().function_warnings)
-    function_handles = (
-        tf_upgrade_v2.TFAPIChangeSpec().function_handle)
+    function_transformers = (
+        tf_upgrade_v2.TFAPIChangeSpec().function_transformers)
     keyword_renames = (
         tf_upgrade_v2.TFAPIChangeSpec().function_keyword_renames)
 
@@ -255,7 +255,7 @@ class TestUpgrade(test_util.TensorFlowTestCase):
 
         for name in names_v1:
           tf_name = "tf.%s" % name
-          if tf_name in function_warnings or tf_name in function_handles:
+          if tf_name in function_warnings or tf_name in function_transformers:
             continue  # These require manual change
           if tf_name in v1_name_exceptions:
             continue
@@ -362,15 +362,14 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
 
       text = "%s(a, b)\n" % decay
       _, report, errors, _ = self._upgrade(text)
-      self.assertEqual(errors, ["test.py:1: %s requires manual check." % decay])
+      self.assertIn("%s requires manual check" % decay, errors[0])
       self.assertIn("%s has been changed" % decay, report)
 
   def testPiecewiseDecay(self):
     text = "tf.train.piecewise_constant_decay(a, b)\n"
     _, report, errors, _ = self._upgrade(text)
-    self.assertEqual(
-        errors,
-        ["test.py:1: tf.train.piecewise_constant_decay requires manual check."])
+    self.assertIn("tf.train.piecewise_constant_decay requires manual check",
+                  errors[0])
     self.assertIn("tf.train.piecewise_constant_decay has been changed", report)
 
   def testMetrics(self):
@@ -414,7 +413,7 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
       text = ns + "(a, b)"
       _, report, errors, new_text = self._upgrade(text)
       self.assertEqual("tf.compat.v1.metrics." + m + "(a, b)", new_text)
-      self.assertEqual(errors, ["test.py:1: %s requires manual check." % ns])
+      self.assertIn("test.py:1:0: %s requires manual check" % ns, errors[0])
       self.assertIn(
           "WARNING: tf.metrics have been converted to object oriented"
           " versions in TF 2.0 and after. The metric function calls have been "
@@ -445,7 +444,7 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
       text = ns + "(a, b)"
       _, report, errors, new_text = self._upgrade(text)
       self.assertEqual("tf.compat.v1.losses." + l + "(a, b)", new_text)
-      self.assertEqual(errors, ["test.py:1: %s requires manual check." % ns])
+      self.assertIn("test.py:1:0: %s requires manual check" % ns, errors[0])
       self.assertIn(
           "WARNING: tf.losses have been converted to object oriented"
           " versions in TF 2.0 and after. The loss function calls have been "
@@ -463,7 +462,7 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
       text = ns + "(a, b)"
       _, report, errors, new_text = self._upgrade(text)
       self.assertEqual(text, new_text)
-      self.assertEqual(errors, ["test.py:1: %s requires manual check." % ns])
+      self.assertIn("%s requires manual check" % ns, errors[0])
       self.assertIn("loss_reduction has been changed", report)
 
   def testDropout(self):
@@ -471,15 +470,40 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
     _, unused_report, unused_errors, new_text = self._upgrade(text)
     self.assertEqual(
         new_text,
-        "tf.nn.dropout(x, 1 - keep_prob, name=\"foo\")\n",
+        "tf.nn.dropout(x, 1 - (keep_prob), name=\"foo\")\n",
+    )
+
+    text = "tf.nn.dropout(x, keep_prob=.4, name=\"foo\")\n"
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(
+        new_text,
+        "tf.nn.dropout(x, rate=1 - (.4), name=\"foo\")\n",
+    )
+
+    text = (
+        "tf.nn.dropout(x,  # Stuff before\n"
+        "              keep_prob=.4,  # Stuff after\n"
+        "              name=\"foo\")\n"
+    )
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(
+        new_text,
+        "tf.nn.dropout(x,  # Stuff before\n"
+        "              rate=1 - (.4),  # Stuff after\n"
+        "              name=\"foo\")\n",
     )
 
     text = "tf.nn.dropout(x)\n"
     _, unused_report, errors, new_text = self._upgrade(text)
     self.assertEqual(new_text, text)
+    self.assertIn("tf.nn.dropout called without arguments", errors[0])
+
+  def testDropoutExpr(self):
+    text = "tf.nn.dropout(x, 1 - func(3 + 4.), name=\"foo\")\n"
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
     self.assertEqual(
-        errors,
-        ["test.py:1: tf.nn.dropout requires manual check."]
+        new_text,
+        "tf.nn.dropout(x, 1 - (1 - func(3 + 4.)), name=\"foo\")\n",
     )
 
   def testCountNonZeroChanges(self):
@@ -543,9 +567,11 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
 
     text = "tf.gradients(a, colocate_gradients_with_ops=False)\n"
     _, unused_report, errors, new_text = self._upgrade(text)
-    self.assertEqual(text, new_text)
-    self.assertEqual(errors, ["test.py:1: tf.gradients requires manual check."])
+    self.assertEqual("tf.gradients(a)\n", new_text)
+    self.assertIn("tf.gradients", errors[0])
+    self.assertIn("requires manual check", errors[0])
 
+  def testColocateGradientsWithOpsMinimize(self):
     text = "optimizer.minimize(a, foo=False)\n"
     _, unused_report, errors, new_text = self._upgrade(text)
     self.assertEqual(text, new_text)
@@ -553,10 +579,11 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
 
     text = "optimizer.minimize(a, colocate_gradients_with_ops=False)\n"
     _, unused_report, errors, new_text = self._upgrade(text)
-    self.assertEqual(text, new_text)
-    self.assertEqual(errors,
-                     ["test.py:1: Optimizer.minimize requires manual check."])
+    self.assertEqual("optimizer.minimize(a)\n", new_text)
+    self.assertIn("requires manual check", errors[0])
+    self.assertIn("minimize", errors[0])
 
+  def testColocateGradientsWithOpsComputeGradients(self):
     text = "optimizer.compute_gradients(a, foo=False)\n"
     _, unused_report, errors, new_text = self._upgrade(text)
     self.assertEqual(text, new_text)
@@ -564,10 +591,9 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
 
     text = "optimizer.compute_gradients(a, colocate_gradients_with_ops=False)\n"
     _, unused_report, errors, new_text = self._upgrade(text)
-    self.assertEqual(text, new_text)
-    self.assertEqual(errors,
-                     ["test.py:1: Optimizer.compute_gradients "
-                      "requires manual check."])
+    self.assertEqual("optimizer.compute_gradients(a)\n", new_text)
+    self.assertIn("requires manual check", errors[0])
+    self.assertIn("compute_gradients", errors[0])
 
   def testExportSavedModelRename(self):
     text = "self.est.export_savedmodel(path)"
@@ -673,7 +699,7 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
     _, report, errors, new_text = self._upgrade(text)
     self.assertEqual(new_text, expected_text)
     self.assertIn(
-        "tf.nn.softmax_cross_entropy_with_logits requires manual check.",
+        "tf.nn.softmax_cross_entropy_with_logits requires manual check",
         errors[0])
     self.assertIn(
         "tf.nn.softmax_cross_entropy_with_logits behavior has changed. ",
@@ -817,27 +843,24 @@ tf.print('abc')
 
   def testBatchGather(self):
     text = "tf.batch_gather(foo, bar)"
-    expected_text = "tf.gather(batch_dims=-1, params=foo, indices=bar)"
+    expected_text1 = "tf.gather(params=foo, indices=bar, batch_dims=-1)"
+    expected_text2 = "tf.gather(batch_dims=-1, params=foo, indices=bar)"
     _, unused_report, unused_errors, new_text = self._upgrade(text)
-    self.assertEqual(new_text, expected_text)
+    self.assertIn(new_text, [expected_text1, expected_text2])
 
     text = "tf.batch_gather(params=foo, indices=bar)"
-    expected_text = "tf.gather(batch_dims=-1, params=foo, indices=bar)"
-    _, unused_report, unused_errors, new_text = self._upgrade(text)
-    self.assertEqual(new_text, expected_text)
-
-    text = "tf.batch_gather  (  foo, bar)"
-    expected_text = "tf.gather  (batch_dims=-1,   params=foo, indices=bar)"
-    _, unused_report, unused_errors, new_text = self._upgrade(text)
-    self.assertEqual(new_text, expected_text)
-
-    text = "(tf.batch_gather\n(foo, bar))"
-    expected_text = "(tf.gather\n(params=foo, indices=bar))"
-    expected_errors = ["test.py:1: Unable to add keyword argument batch_dims=-1"
-                       " to tf.batch_gather; please add it manually."]
-    _, unused_report, errors, new_text = self._upgrade(text)
-    self.assertEqual(errors, expected_errors)
-    self.assertEqual(new_text, expected_text)
+    expected_text1 = "tf.gather(params=foo, indices=bar, batch_dims=-1)"
+    expected_text2 = "tf.gather(batch_dims=-1, params=foo, indices=bar)"
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertIn(new_text, [expected_text1, expected_text2])
+
+  def testCast(self):
+    for dtype in ["int32", "int64", "float", "double",
+                  "complex64", "complex128", "bfloat16"]:
+      text = "tf.to_%s(x, name='test')" % dtype
+      expected_text = "tf.cast(x, name='test', dtype=tf.%s)" % dtype
+      _, unused_report, unused_errors, new_text = self._upgrade(text)
+      self.assertEqual(expected_text, new_text)
 
 
 class TestUpgradeFiles(test_util.TensorFlowTestCase):
diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD
index 93a85763f5..c51b45a49c 100644
--- a/tensorflow/tools/pip_package/BUILD
+++ b/tensorflow/tools/pip_package/BUILD
@@ -169,6 +169,7 @@ filegroup(
         "@local_config_sycl//sycl:LICENSE.text",
         "@nasm//:LICENSE",
         "@nsync//:LICENSE",
+        "@pasta//:LICENSE",
         "@pcre//:LICENCE",
         "@png_archive//:LICENSE",
         "@protobuf_archive//:LICENSE",
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 729eb8069f..b43312abb2 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -29,6 +29,7 @@ load("//third_party/jpeg:workspace.bzl", jpeg = "repo")
 load("//third_party/nasm:workspace.bzl", nasm = "repo")
 load("//third_party/kissfft:workspace.bzl", kissfft = "repo")
 load("//third_party/keras_applications_archive:workspace.bzl", keras_applications = "repo")
+load("//third_party/pasta:workspace.bzl", pasta = "repo")
 
 def initialize_third_party():
     """ Load third party repositories.  See above load() statements. """
@@ -41,6 +42,7 @@ def initialize_third_party():
     kissfft()
     jpeg()
     nasm()
+    pasta()
 
 # Sanitize a dependency so that it works correctly from code that includes
 # TensorFlow as a submodule.
diff --git a/third_party/pasta/BUILD b/third_party/pasta/BUILD
new file mode 100644
index 0000000000..9bd256a579
--- /dev/null
+++ b/third_party/pasta/BUILD
@@ -0,0 +1 @@
+# Empty BUILD file to force build system to see this directory at all.
diff --git a/third_party/pasta/BUILD.bazel b/third_party/pasta/BUILD.bazel
new file mode 100644
index 0000000000..6be33511e4
--- /dev/null
+++ b/third_party/pasta/BUILD.bazel
@@ -0,0 +1,29 @@
+# Description:
+#   AST-based python refactoring.
+
+licenses(["notice"])  # Apache2
+
+exports_files(["LICENSE"])
+
+py_library(
+    name = "pasta",
+    srcs = [
+        "__init__.py",
+        "augment/__init__.py",
+        "augment/errors.py",
+        "augment/import_utils.py",
+        "augment/inline.py",
+        "augment/rename.py",
+        "base/__init__.py",
+        "base/annotate.py",
+        "base/ast_constants.py",
+        "base/ast_utils.py",
+        "base/codegen.py",
+        "base/formatting.py",
+        "base/scope.py",
+        "base/test_utils.py",
+        "base/token_generator.py",
+    ],
+    srcs_version = "PY2AND3",
+    visibility = ["//visibility:public"],
+)
diff --git a/third_party/pasta/BUILD.system b/third_party/pasta/BUILD.system
new file mode 100644
index 0000000000..6adc953c5a
--- /dev/null
+++ b/third_party/pasta/BUILD.system
@@ -0,0 +1,13 @@
+# Description: Pasta, AST based python refactoring.
+
+licenses(["notice"])  # Apache2
+
+filegroup(
+    name = "LICENSE",
+    visibility = ["//visibility:public"],
+)
+
+py_library(
+    name = "pasta",
+    visibility = ["//visibility:public"],
+)
diff --git a/third_party/pasta/workspace.bzl b/third_party/pasta/workspace.bzl
new file mode 100644
index 0000000000..063bdb7b98
--- /dev/null
+++ b/third_party/pasta/workspace.bzl
@@ -0,0 +1,16 @@
+"""Loads pasta python package."""
+
+load("//third_party:repo.bzl", "third_party_http_archive")
+
+def repo():
+    third_party_http_archive(
+        name = "pasta",
+        urls = [
+            "https://mirror.bazel.build/github.com/google/pasta/archive/c3d72cdee6fc806251949e912510444d58d7413c.tar.gz",
+            "https://github.com/google/pasta/archive/c3d72cdee6fc806251949e912510444d58d7413c.tar.gz",
+        ],
+        strip_prefix = "pasta-c3d72cdee6fc806251949e912510444d58d7413c/pasta",
+        sha256 = "b5905f9cecc4b28363c563f3c4cb0545288bd35f7cc72c55066e97e53befc084",
+        build_file = "//third_party/pasta:BUILD.bazel",
+        system_build_file = "//third_party/pasta:BUILD.system",
+    )
-- 
GitLab


From 4b90409c6b9e11d0bf0517a8fc49f9826167cb0a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 4 Jan 2019 17:04:27 -0800
Subject: [PATCH 0270/1095] Several fixes/improvements to wrap_function: 1.
 Makes gradient tape work on captures post-pruning 2. Allows users to prune
 with wrapped_func.inputs even when the func includes captures 3. Makes sure
 to write variables to collections for better legacy compatibility in
 compat.v1 (currently if the outer init_scope is eager, variables won't be
 written to the global_variables / trainable_variables collections)

PiperOrigin-RevId: 227936352
---
 tensorflow/python/eager/wrap_function.py      | 23 +++++++++-
 tensorflow/python/eager/wrap_function_test.py | 44 +++++++++++++++++++
 2 files changed, 66 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/eager/wrap_function.py b/tensorflow/python/eager/wrap_function.py
index 0930b6116d..2c65e97871 100644
--- a/tensorflow/python/eager/wrap_function.py
+++ b/tensorflow/python/eager/wrap_function.py
@@ -38,8 +38,20 @@ class VariableHolder(object):
     self._variables = []
 
   def variable_creator_scope(self, next_creator, **kwargs):
+    """Creates variables & adds them to collections to match legacy code."""
     v = next_creator(**kwargs)
     self._variables.append(v)
+
+    collections = kwargs.get("collections")
+    trainable = v.trainable
+
+    if collections is None:
+      collections = [ops.GraphKeys.GLOBAL_VARIABLES]
+    if trainable and ops.GraphKeys.TRAINABLE_VARIABLES not in collections:
+      collections = list(collections) + [ops.GraphKeys.TRAINABLE_VARIABLES]
+
+    ops.add_to_collections(collections, v)
+
     return v
 
   def __call__(self, *args, **kwargs):
@@ -61,6 +73,13 @@ class WrappedFunction(function.Function):
     for f in flat_feeds:
       if not isinstance(f, ops.Tensor):
         raise ValueError("Feeds must be tensors.")
+
+    # Ignoring all feeds that are captures allows prune to be called
+    # using wrapped_func.inputs even when it uses variables
+    internal_captures = self.graph.internal_captures
+    flat_feeds = [f for f in flat_feeds
+                  if f not in internal_captures]
+
     tensor_fetches = []
     operation_fetches = []
     for f in flat_fetches:
@@ -87,7 +106,7 @@ class WrappedFunction(function.Function):
           sink_tensor = control_flow_ops.no_op()
     lift_map = lift_to_graph.lift_to_graph(
         sink_tensor, pruned_graph,
-        sources=flat_feeds + self.graph.internal_captures)
+        sources=flat_feeds + internal_captures)
     for original_fetch, identity_fetch in zip(
         tensor_fetches, identity_fetches):
       lift_map[original_fetch] = lift_map[identity_fetch]
@@ -98,6 +117,8 @@ class WrappedFunction(function.Function):
     pruned_graph.inputs.extend(lift_map[x] for x in flat_feeds)
     pruned_graph.inputs.extend(pruned_graph.captures.values())
 
+    pruned_graph.variables = self.graph.variables
+
     def _structured_output_mapping(fetched):
       lifted = lift_map[fetched]
       if isinstance(lifted, ops.Operation):
diff --git a/tensorflow/python/eager/wrap_function_test.py b/tensorflow/python/eager/wrap_function_test.py
index 65dd73aafc..17e204b4d8 100644
--- a/tensorflow/python/eager/wrap_function_test.py
+++ b/tensorflow/python/eager/wrap_function_test.py
@@ -18,6 +18,7 @@ from __future__ import division
 from __future__ import print_function
 
 
+from tensorflow.python.eager import backprop
 from tensorflow.python.eager import wrap_function
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -90,11 +91,54 @@ class WrapFunctionTest(test.TestCase):
 
     f_wrapped = wrap_function.wrap_function(f, [])
     self.assertAllEqual(6.0, f_wrapped())
+
+    # Test pruning directly on the inputs
+    pruned = f_wrapped.prune(
+        feeds=f_wrapped.inputs,
+        fetches=f_wrapped.graph.get_tensor_by_name('fetch:0'))
+    self.assertAllEqual(6.0, pruned())
+
+    # Test pruning with no inputs
     pruned = f_wrapped.prune(
         feeds=(),
         fetches=f_wrapped.graph.get_tensor_by_name('fetch:0'))
     self.assertAllEqual(6.0, pruned())
 
+  def testGradientsOfPrune(self):
+
+    v1 = variables.Variable(2.)
+    v2_holder = []
+
+    def f(z):
+      v2 = variables.Variable(3.)
+      v2_holder.append(v2)
+      return array_ops.identity(v1 * v2 * z, 'fetch')
+
+    f_wrapped = wrap_function.wrap_function(
+        f, [tensor_spec.TensorSpec((), dtype=dtypes.float32)])
+
+    x = constant_op.constant(1.)
+    with backprop.GradientTape() as tape:
+      tape.watch(x)
+      out = f_wrapped(x)
+    grads = tape.gradient(out, [x, v1, v2_holder[0]])
+
+    self.assertAllEqual(6.0, out)
+    self.assertAllEqual([6.0, 3.0, 2.0], grads)
+
+    pruned = f_wrapped.prune(
+        feeds=f_wrapped.inputs,
+        fetches=f_wrapped.graph.get_tensor_by_name('fetch:0'))
+
+    x = constant_op.constant(1.)
+    with backprop.GradientTape() as tape:
+      tape.watch(x)
+      out = pruned(x)
+    grads = tape.gradient(out, [x, v1, v2_holder[0]])
+
+    self.assertAllEqual(6.0, out)
+    self.assertAllEqual([6.0, 3.0, 2.0], grads)
+
   def testPruneOperations(self):
 
     v = variables.Variable(0)
-- 
GitLab


From 542e30bab143034047ee18a0b060648be78df58c Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Fri, 4 Jan 2019 17:11:01 -0800
Subject: [PATCH 0271/1095] Enable nnapi_delegate_test in oss builds

PiperOrigin-RevId: 227937061
---
 tensorflow/lite/delegates/nnapi/BUILD | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/lite/delegates/nnapi/BUILD b/tensorflow/lite/delegates/nnapi/BUILD
index fd954ba222..dda3c02567 100644
--- a/tensorflow/lite/delegates/nnapi/BUILD
+++ b/tensorflow/lite/delegates/nnapi/BUILD
@@ -3,6 +3,7 @@ package(default_visibility = [
 ])
 
 load("//tensorflow:tensorflow.bzl", "tf_cc_test")
+load("//tensorflow/lite:special_rules.bzl", "tflite_portable_test_suite")
 
 licenses(["notice"])  # Apache 2.0
 
@@ -23,7 +24,7 @@ tf_cc_test(
     name = "nnapi_delegate_test",
     size = "small",
     srcs = ["nnapi_delegate_test.cc"],
-    tags = ["no_oss"],
+    tags = ["tflite_not_portable_ios"],
     deps = [
         ":nnapi_delegate",
         "//tensorflow/lite:framework",
@@ -32,3 +33,5 @@ tf_cc_test(
         "@com_google_googletest//:gtest",
     ],
 )
+
+tflite_portable_test_suite()
-- 
GitLab


From 774ede5790c6cf11da05ca1082e9870db7b843f9 Mon Sep 17 00:00:00 2001
From: Karmel Allison <karmel@google.com>
Date: Fri, 4 Jan 2019 17:23:06 -0800
Subject: [PATCH 0272/1095] Consolidated duplicate tests for model cloning and
 added Keras test decorators where appropriate.

Updated models_tests to run in v1/v2. This required several other changes:
* Added a testing utility to get v2 optimizers
* Ensured that subclassed models cleared key compilation attrs upon clone_reset (_is_compiled; optimizer)
* Modified train/eval loops to check self._is_compiled rather than the existence of other attrs created during compiling, as these other attrs would not be reset during cloning.

PiperOrigin-RevId: 227938218
---
 tensorflow/python/keras/BUILD              |   7 +-
 tensorflow/python/keras/engine/training.py |   4 +-
 tensorflow/python/keras/models.py          |  44 +-
 tensorflow/python/keras/models_test.py     | 734 ++++++++-------------
 tensorflow/python/keras/testing_utils.py   |  56 +-
 5 files changed, 375 insertions(+), 470 deletions(-)

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 914568f652..3ffff61c2b 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -1,5 +1,7 @@
 # Description:
 #   Contains the Keras API (internal TensorFlow version).
+load("//tensorflow:tensorflow.bzl", "tf_py_test")
+load("//tensorflow:tensorflow.bzl", "cuda_py_test")
 
 licenses(["notice"])  # Apache 2.0
 
@@ -7,9 +9,6 @@ package(default_visibility = ["//visibility:public"])
 
 exports_files(["LICENSE"])
 
-load("//tensorflow:tensorflow.bzl", "tf_py_test")
-load("//tensorflow:tensorflow.bzl", "cuda_py_test")
-
 config_setting(
     name = "empty_condition",
     values = {"define": "UNUSED=unused"},
@@ -1022,7 +1021,7 @@ tf_py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:training",
     ],
-    shard_count = 2,
+    shard_count = 8,
     tags = ["notsan"],  # b/67509773
 )
 
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 8263467c37..bd06751428 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -1991,7 +1991,7 @@ class Model(Network):
           ' without calling `model.compile` after ?', 1)
 
   def _make_train_function_helper(self, fn_name, outputs, metric_updates=None):
-    if not hasattr(self, fn_name):
+    if not self._is_compiled:
       raise RuntimeError('You must compile your model before using it.')
     self._check_trainable_weights_consistency()
     if getattr(self, fn_name) is None:
@@ -2040,7 +2040,7 @@ class Model(Network):
         '_fit_function', [self.total_loss] + metrics_tensors)
 
   def _make_test_function_helper(self, fn_name, outputs, metric_updates=None):
-    if not hasattr(self, fn_name):
+    if not self._is_compiled:
       raise RuntimeError('You must compile your model before using it.')
     if getattr(self, fn_name) is None:
       inputs = (self._feed_inputs +
diff --git a/tensorflow/python/keras/models.py b/tensorflow/python/keras/models.py
index f36c89041c..851de6e71f 100644
--- a/tensorflow/python/keras/models.py
+++ b/tensorflow/python/keras/models.py
@@ -281,8 +281,6 @@ def clone_model(model, input_tensors=None):
 
 
 # "Clone" a subclassed model by reseting all of the attributes.
-
-
 def _in_place_subclassed_model_reset(model):
   """Substitute for model cloning that works for subclassed models.
 
@@ -382,11 +380,30 @@ def _in_place_subclassed_model_reset(model):
       for name in attributes_to_cache:
         attributes_cache[name] = getattr(model, name)
   model._original_attributes_cache = attributes_cache
-  # Reset built state
+  _reset_build_compile_trackers(model)
+  model._setattr_tracking = setattr_tracking
+
+
+def _reset_build_compile_trackers(model):
+  """Reset state trackers for model.
+
+  Note that we do not actually zero out attributes such as optimizer,
+  but instead rely on the expectation that all of the attrs will be
+  over-written on calling build/compile/etc. This is somewhat fragile,
+  insofar as we check elsewhere for the presence of these attributes as
+  evidence of having been built/compiled/etc. Pending a better way to do this,
+  we reset key attributes here to allow building and compiling.
+
+  Args:
+    model: the model that is being reset
+  """
+  # Reset build state
   model.built = False
   model.inputs = None
   model.outputs = None
-  model._setattr_tracking = setattr_tracking
+  # Reset compile state
+  model._is_compiled = False  # pylint:disable=protected-access
+  model.optimizer = None
 
 
 def in_place_subclassed_model_state_restoration(model):
@@ -418,9 +435,7 @@ def in_place_subclassed_model_state_restoration(model):
     model._setattr_tracking = setattr_tracking
   else:
     # Restore to the state of a never-called model.
-    model.built = False
-    model.inputs = None
-    model.outputs = None
+    _reset_build_compile_trackers(model)
 
 
 def clone_and_build_model(
@@ -462,7 +477,10 @@ def clone_and_build_model(
       - cloning a subclassed model with `in_place_reset` set to False.
       - compiling the clone when the original model has not been compiled.
   """
-  if compile_clone and not model.optimizer:
+  # Grab optimizer now, as we reset-in-place for subclassed models, but
+  # want to maintain access to the original optimizer.
+  orig_optimizer = model.optimizer
+  if compile_clone and not orig_optimizer:
     raise ValueError(
         'Error when cloning model: compile_clone was set to True, but the '
         'original model has not been compiled.')
@@ -498,14 +516,14 @@ def clone_and_build_model(
         input_tensors = input_tensors[0]
       clone._set_inputs(input_tensors)
 
-  if compile_clone and model.optimizer:
-    if isinstance(model.optimizer, optimizers.TFOptimizer):
+  if compile_clone:
+    if isinstance(orig_optimizer, optimizers.TFOptimizer):
       optimizer = optimizers.TFOptimizer(
-          model.optimizer.optimizer, optimizer_iterations)
+          orig_optimizer.optimizer, optimizer_iterations)
       K.track_tf_optimizer(optimizer)
     else:
-      optimizer_config = model.optimizer.get_config()
-      optimizer = model.optimizer.__class__.from_config(optimizer_config)
+      optimizer_config = orig_optimizer.get_config()
+      optimizer = orig_optimizer.__class__.from_config(optimizer_config)
       if optimizer_iterations is not None:
         optimizer.iterations = optimizer_iterations
 
diff --git a/tensorflow/python/keras/models_test.py b/tensorflow/python/keras/models_test.py
index 0a5f9a7bea..3eab10f624 100644
--- a/tensorflow/python/keras/models_test.py
+++ b/tensorflow/python/keras/models_test.py
@@ -19,18 +19,21 @@ from __future__ import division
 from __future__ import print_function
 
 import copy
+import functools
 import os
 
+from absl.testing import parameterized
 import numpy as np
 
 from tensorflow.python import keras
 from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
-from tensorflow.python.framework import test_util
 from tensorflow.python.keras import backend as K
+from tensorflow.python.keras import keras_parameterized
 from tensorflow.python.keras import metrics
 from tensorflow.python.keras import models
+from tensorflow.python.keras import testing_utils
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import resource_variable_ops
@@ -52,158 +55,181 @@ class TestModel(keras.Model):
     return self.layer1(x)
 
 
-def sequential_model(add_input_layer, include_input_shape=True):
-  model = keras.models.Sequential()
+def _get_layers(input_shape=(4,), add_input_layer=False):
   if add_input_layer:
-    model.add(keras.layers.InputLayer(input_shape=(4,)))
-    model.add(keras.layers.Dense(4))
-  elif include_input_shape:
-    model.add(keras.layers.Dense(4, input_shape=(4,)))
+    model_layers = [keras.layers.InputLayer(input_shape=input_shape),
+                    keras.layers.Dense(4)]
+  elif input_shape:
+    model_layers = [keras.layers.Dense(4, input_shape=input_shape)]
   else:
-    model.add(keras.layers.Dense(4))
-  model.add(keras.layers.BatchNormalization())
-  model.add(keras.layers.Dropout(0.5))
-  model.add(keras.layers.Dense(4))
-  return model
-
-
-class TestModelCloning(test.TestCase):
-
-  @test_util.run_v1_only('b/120545219')
-  def test_clone_sequential_model(self):
-    with self.cached_session():
-      val_a = np.random.random((10, 4))
-      val_out = np.random.random((10, 4))
-
-      model = sequential_model(False)
-
-    # Everything should work in a new session.
-    keras.backend.clear_session()
-
-    with self.cached_session():
-      # With placeholder creation
-      new_model = keras.models.clone_model(model)
+    model_layers = [keras.layers.Dense(4)]
+
+  model_layers += [
+      keras.layers.BatchNormalization(),
+      keras.layers.Dropout(0.5),
+      keras.layers.Dense(4)]
+
+  return model_layers
+
+
+def _get_model(input_shape=(4,)):
+  model_layers = _get_layers(input_shape=None, add_input_layer=False)
+  return testing_utils.get_model_from_layers(
+      model_layers, input_shape=input_shape)
+
+
+class TestModelCloning(keras_parameterized.TestCase):
+
+  @keras_parameterized.run_all_keras_modes
+  @parameterized.named_parameters([
+      {'testcase_name': 'has_input_layer',
+       'input_shape': (4,),
+       'add_input_layer': True,
+       'share_weights': False},
+      {'testcase_name': 'no_input_layer',
+       'input_shape': None,
+       'add_input_layer': False,
+       'share_weights': False},
+      {'testcase_name': 'has_input_layer_share_weights',
+       'input_shape': (4,),
+       'add_input_layer': True,
+       'share_weights': True},
+      {'testcase_name': 'no_input_layer_share_weights',
+       'input_shape': None,
+       'add_input_layer': False,
+       'share_weights': True},
+  ])
+  def test_clone_sequential_model(
+      self, input_shape, add_input_layer, share_weights):
+
+    if share_weights:
+      clone_fn = functools.partial(
+          keras.models._clone_sequential_model, share_weights=True)
+    else:
+      clone_fn = keras.models.clone_model
+
+    val_a = np.random.random((10, 4))
+    model = models.Sequential(_get_layers(input_shape, add_input_layer))
+    # Sanity check
+    self.assertEqual(
+        isinstance(model._layers[0], keras.layers.InputLayer),
+        add_input_layer)
+    self.assertEqual(model._is_graph_network, add_input_layer)
+
+    # With placeholder creation -- clone model should have an InputLayer
+    # if the original model has one.
+    new_model = clone_fn(model)
+    self.assertEqual(
+        isinstance(new_model._layers[0], keras.layers.InputLayer),
+        add_input_layer)
+    self.assertEqual(new_model._is_graph_network, model._is_graph_network)
+    if input_shape:
       # update ops from batch norm needs to be included
       self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2)
-      new_model.compile('rmsprop', 'mse')
-      new_model.train_on_batch(val_a, val_out)
-
-      # On top of new tensor
-      input_a = keras.Input(shape=(4,))
-      new_model = keras.models.clone_model(model, input_tensors=input_a)
-      self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2)
-      new_model.compile('rmsprop', 'mse')
-      new_model.train_on_batch(val_a, val_out)
 
-      # On top of new, non-Keras tensor
+    # On top of new tensor  -- clone model should always have an InputLayer.
+    input_a = keras.Input(shape=(4,))
+    new_model = clone_fn(model, input_tensors=input_a)
+    self.assertIsInstance(new_model._layers[0], keras.layers.InputLayer)
+    self.assertTrue(new_model._is_graph_network)
+
+    # On top of new, non-Keras tensor  -- clone model should always have an
+    # InputLayer.
+    if not context.executing_eagerly():
+      # TODO(b/121277734):Skip Eager contexts, as Input() layers raise an error
+      # saying they should not be used with EagerTensors
       input_a = keras.backend.variable(val_a)
-      new_model = keras.models.clone_model(model, input_tensors=input_a)
-      self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2)
-      new_model.compile('rmsprop', 'mse')
-      new_model.train_on_batch(None, val_out)
-
-  @test_util.run_v1_only('b/120545219')
-  def test_clone_sequential_model_input_layer(self):
-
-    def test_input_layer(include_inputs):
-      with self.cached_session():
-        val_a = np.random.random((10, 4))
-        model = sequential_model(include_inputs, include_inputs)
-        # Sanity check
-        self.assertEqual(
-            isinstance(model._layers[0], keras.layers.InputLayer),
-            include_inputs)
-        self.assertEqual(model._is_graph_network, include_inputs)
-
-      keras.backend.clear_session()
-      with self.cached_session():
-        # With placeholder creation -- clone model should have an InputLayer
-        # if the original model has one.
-        new_model = keras.models.clone_model(model)
-        self.assertEqual(
-            isinstance(new_model._layers[0], keras.layers.InputLayer),
-            include_inputs)
-        self.assertEqual(new_model._is_graph_network, model._is_graph_network)
-
-        # On top of new tensor  -- clone model should always have an InputLayer.
-        input_a = keras.Input(shape=(4,))
-        new_model = keras.models.clone_model(model, input_tensors=input_a)
-        self.assertIsInstance(new_model._layers[0], keras.layers.InputLayer)
-        self.assertTrue(new_model._is_graph_network)
-
-        # On top of new, non-Keras tensor  -- clone model should always have an
-        # InputLayer.
-        input_a = keras.backend.variable(val_a)
-        new_model = keras.models.clone_model(model, input_tensors=input_a)
-        self.assertIsInstance(new_model._layers[0], keras.layers.InputLayer)
-        self.assertTrue(new_model._is_graph_network)
-
-    test_input_layer(True)
-    test_input_layer(False)
-
-  @test_util.run_v1_only('b/120545219')
-  def test_clone_functional_model(self):
-    with self.cached_session():
-      val_a = np.random.random((10, 4))
-      val_b = np.random.random((10, 4))
-      val_out = np.random.random((10, 4))
-
-      input_a = keras.Input(shape=(4,))
-      input_b = keras.Input(shape=(4,))
-      dense_1 = keras.layers.Dense(4,)
-      dense_2 = keras.layers.Dense(4,)
-
-      x_a = dense_1(input_a)
-      x_a = keras.layers.Dropout(0.5)(x_a)
-      x_a = keras.layers.BatchNormalization()(x_a)
-      x_b = dense_1(input_b)
-      x_a = dense_2(x_a)
-      outputs = keras.layers.add([x_a, x_b])
-      model = keras.models.Model([input_a, input_b], outputs)
-
-    # Everything should work in a new session.
-    keras.backend.clear_session()
-
-    with self.cached_session():
-      # With placeholder creation
-      new_model = keras.models.clone_model(model)
-      self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2)
-      new_model.compile('rmsprop', 'mse')
-      new_model.train_on_batch([val_a, val_b], val_out)
-
-      # On top of new tensors
-      input_a = keras.Input(shape=(4,), name='a')
-      input_b = keras.Input(shape=(4,), name='b')
-      new_model = keras.models.clone_model(
-          model, input_tensors=[input_a, input_b])
-      self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2)
-      new_model.compile('rmsprop', 'mse')
-      new_model.train_on_batch([val_a, val_b], val_out)
-
-      # On top of new, non-Keras tensors
+      new_model = clone_fn(model, input_tensors=input_a)
+      self.assertIsInstance(new_model._layers[0], keras.layers.InputLayer)
+      self.assertTrue(new_model._is_graph_network)
+
+  @keras_parameterized.run_all_keras_modes
+  @parameterized.named_parameters([
+      {'testcase_name': 'clone_weights', 'share_weights': False},
+      {'testcase_name': 'share_weights', 'share_weights': True},
+  ])
+  def test_clone_functional_model(self, share_weights):
+    if share_weights:
+      clone_fn = functools.partial(
+          keras.models._clone_functional_model, share_weights=True)
+    else:
+      clone_fn = keras.models.clone_model
+
+    val_a = np.random.random((10, 4))
+    val_b = np.random.random((10, 4))
+    val_out = np.random.random((10, 4))
+
+    input_a = keras.Input(shape=(4,))
+    input_b = keras.Input(shape=(4,))
+    dense_1 = keras.layers.Dense(4,)
+    dense_2 = keras.layers.Dense(4,)
+
+    x_a = dense_1(input_a)
+    x_a = keras.layers.Dropout(0.5)(x_a)
+    x_a = keras.layers.BatchNormalization()(x_a)
+    x_b = dense_1(input_b)
+    x_a = dense_2(x_a)
+    outputs = keras.layers.add([x_a, x_b])
+    model = keras.models.Model([input_a, input_b], outputs)
+
+    # With placeholder creation
+    new_model = clone_fn(model)
+    self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2)
+    new_model.compile(
+        testing_utils.get_v2_optimizer('rmsprop'), 'mse',
+        run_eagerly=testing_utils.should_run_eagerly())
+    new_model.train_on_batch([val_a, val_b], val_out)
+
+    # On top of new tensors
+    input_a = keras.Input(shape=(4,), name='a')
+    input_b = keras.Input(shape=(4,), name='b')
+    new_model = keras.models.clone_model(
+        model, input_tensors=[input_a, input_b])
+    self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2)
+    new_model.compile(
+        testing_utils.get_v2_optimizer('rmsprop'), 'mse',
+        run_eagerly=testing_utils.should_run_eagerly())
+    new_model.train_on_batch([val_a, val_b], val_out)
+
+    # On top of new, non-Keras tensors
+    if not context.executing_eagerly():
+      # TODO(b/121277734):Skip Eager contexts, as Input() layers raise an error
+      # saying they should not be used with EagerTensors
       input_a = keras.backend.variable(val_a)
       input_b = keras.backend.variable(val_b)
-      new_model = keras.models.clone_model(
-          model, input_tensors=[input_a, input_b])
+      new_model = clone_fn(model, input_tensors=[input_a, input_b])
       self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2)
-      new_model.compile('rmsprop', 'mse')
+      new_model.compile(
+          testing_utils.get_v2_optimizer('rmsprop'), 'mse',
+          run_eagerly=testing_utils.should_run_eagerly())
       new_model.train_on_batch(None, val_out)
 
-  @test_util.run_in_graph_and_eager_modes
-  def test_clone_functional_model_with_masking(self):
-    with self.cached_session():
-      x = np.array([[[1], [1]], [[0], [0]]])
-      inputs = keras.Input((2, 1))
-      outputs = keras.layers.Masking(mask_value=0)(inputs)
-      outputs = keras.layers.TimeDistributed(
-          keras.layers.Dense(1, kernel_initializer='one'))(outputs)
-      model = keras.Model(inputs, outputs)
-
-      model = keras.models.clone_model(model)
-      model.compile(loss='mse', optimizer=adam.AdamOptimizer(0.01))
-      y = np.array([[[1], [1]], [[1], [1]]])
-      loss = model.train_on_batch(x, y)
-      self.assertEqual(float(loss), 0.)
+  @keras_parameterized.run_all_keras_modes
+  @parameterized.named_parameters([
+      {'testcase_name': 'clone_weights', 'share_weights': False},
+      {'testcase_name': 'share_weights', 'share_weights': True},
+  ])
+  def test_clone_functional_with_masking(self, share_weights):
+    if share_weights:
+      clone_fn = functools.partial(
+          keras.models._clone_functional_model, share_weights=True)
+    else:
+      clone_fn = keras.models.clone_model
+
+    x = np.array([[[1.], [1.]], [[0.], [0.]]])
+    inputs = keras.Input((2, 1))
+    outputs = keras.layers.Masking(mask_value=0)(inputs)
+    outputs = keras.layers.TimeDistributed(
+        keras.layers.Dense(1, kernel_initializer='one'))(outputs)
+    model = keras.Model(inputs, outputs)
+
+    model = clone_fn(model)
+    model.compile(
+        loss='mse', optimizer=testing_utils.get_v2_optimizer('adam'),
+        run_eagerly=testing_utils.should_run_eagerly())
+    y = np.array([[[1], [1]], [[1], [1]]])
+    loss = model.train_on_batch(x, y)
+    self.assertEqual(float(loss), 0.)
 
   def test_model_cloning_invalid_use_cases(self):
     seq_model = keras.models.Sequential()
@@ -249,168 +275,23 @@ class TestModelCloning(test.TestCase):
       self.assertFalse(has_placeholder)
 
 
-class TestModelCloningLayerPreserveWeights(test.TestCase):
-
-  @test_util.run_deprecated_v1
-  def test_clone_sequential_model(self):
-    with self.cached_session():
-      val_a = np.random.random((10, 4))
-      val_out = np.random.random((10, 4))
-
-      model = sequential_model(False)
-
-    # Everything should work in a new session.
-    keras.backend.clear_session()
-
-    with self.cached_session():
-      # With placeholder creation
-      new_model = keras.models._clone_sequential_model(
-          model, share_weights=True)
-      # update ops from batch norm needs to be included
-      self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2)
-      new_model.compile('rmsprop', 'mse')
-      new_model.train_on_batch(val_a, val_out)
-
-      # On top of new tensor
-      input_a = keras.Input(shape=(4,))
-      new_model = keras.models._clone_sequential_model(
-          model, input_tensors=input_a, share_weights=True)
-      self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2)
-      new_model.compile('rmsprop', 'mse')
-      new_model.train_on_batch(val_a, val_out)
-
-      # On top of new, non-Keras tensor
-      input_a = keras.backend.variable(val_a)
-      new_model = keras.models._clone_sequential_model(
-          model, input_tensors=input_a, share_weights=True)
-      self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2)
-      new_model.compile('rmsprop', 'mse')
-      new_model.train_on_batch(None, val_out)
-
-  @test_util.run_deprecated_v1
-  def test_clone_sequential_model_input_layer(self):
-
-    @test_util.run_deprecated_v1
-    def test_input_layer(include_inputs):
-      with self.cached_session():
-        val_a = np.random.random((10, 4))
-        model = sequential_model(include_inputs, include_inputs)
-        # Sanity check
-        self.assertEqual(
-            isinstance(model._layers[0], keras.layers.InputLayer),
-            include_inputs)
-        self.assertEqual(model._is_graph_network, include_inputs)
-
-      keras.backend.clear_session()
-      with self.cached_session():
-        # With placeholder creation -- clone model should have an InputLayer
-        # if the original model has one.
-        new_model = keras.models._clone_sequential_model(
-            model, share_weights=True)
-        self.assertEqual(
-            isinstance(new_model._layers[0], keras.layers.InputLayer),
-            include_inputs)
-        self.assertEqual(new_model._is_graph_network, model._is_graph_network)
-
-        # On top of new tensor  -- clone model should always have an InputLayer.
-        input_a = keras.Input(shape=(4,))
-        new_model = keras.models._clone_sequential_model(
-            model, input_tensors=input_a, share_weights=True)
-        self.assertIsInstance(new_model._layers[0], keras.layers.InputLayer)
-        self.assertTrue(new_model._is_graph_network)
-
-        # On top of new, non-Keras tensor  -- clone model should always have an
-        # InputLayer.
-        input_a = keras.backend.variable(val_a)
-        new_model = keras.models._clone_sequential_model(
-            model, input_tensors=input_a, share_weights=True)
-        self.assertIsInstance(new_model._layers[0], keras.layers.InputLayer)
-        self.assertTrue(new_model._is_graph_network)
-
-    test_input_layer(True)
-    test_input_layer(False)
-
-  @test_util.run_deprecated_v1
-  def test_clone_functional_model(self):
-    with self.cached_session():
-      val_a = np.random.random((10, 4))
-      val_b = np.random.random((10, 4))
-      val_out = np.random.random((10, 4))
-
-      input_a = keras.Input(shape=(4,))
-      input_b = keras.Input(shape=(4,))
-      dense_1 = keras.layers.Dense(4,)
-      dense_2 = keras.layers.Dense(4,)
-
-      x_a = dense_1(input_a)
-      x_a = keras.layers.Dropout(0.5)(x_a)
-      x_a = keras.layers.BatchNormalization()(x_a)
-      x_b = dense_1(input_b)
-      x_a = dense_2(x_a)
-      outputs = keras.layers.add([x_a, x_b])
-      model = keras.models.Model([input_a, input_b], outputs)
-
-    # Everything should work in a new session.
-    keras.backend.clear_session()
-
-    with self.cached_session():
-      # With placeholder creation
-      new_model = keras.models._clone_functional_model(
-          model, share_weights=True)
-      self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2)
-      new_model.compile('rmsprop', 'mse')
-      new_model.train_on_batch([val_a, val_b], val_out)
-
-      # On top of new tensors
-      input_a = keras.Input(shape=(4,), name='a')
-      input_b = keras.Input(shape=(4,), name='b')
-      new_model = keras.models._clone_functional_model(
-          model, input_tensors=[input_a, input_b], share_weights=True)
-      self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2)
-      new_model.compile('rmsprop', 'mse')
-      new_model.train_on_batch([val_a, val_b], val_out)
-
-      # On top of new, non-Keras tensors
-      input_a = keras.backend.variable(val_a)
-      input_b = keras.backend.variable(val_b)
-      new_model = keras.models._clone_functional_model(
-          model, input_tensors=[input_a, input_b], share_weights=True)
-      self.assertEqual(len(new_model.get_updates_for(new_model.inputs)), 2)
-      new_model.compile('rmsprop', 'mse')
-      new_model.train_on_batch(None, val_out)
-
-  @test_util.run_in_graph_and_eager_modes
-  def test_clone_functional_model_with_masking(self):
-    with self.cached_session():
-      x = np.array([[[1], [1]], [[0], [0]]])
-      inputs = keras.Input((2, 1))
-      outputs = keras.layers.Masking(mask_value=0)(inputs)
-      outputs = keras.layers.TimeDistributed(
-          keras.layers.Dense(1, kernel_initializer='one'))(outputs)
-      model = keras.Model(inputs, outputs)
-
-      model = keras.models._clone_functional_model(
-          model, share_weights=True)
-      model.compile(loss='mse', optimizer=adam.AdamOptimizer(0.01))
-      y = np.array([[[1], [1]], [[1], [1]]])
-      loss = model.train_on_batch(x, y)
-      self.assertEqual(float(loss), 0.)
-
-
 def _has_placeholder(graph):
   ops_types = [op.type for op in graph.get_operations()]
   return any('Placeholder' in s for s in ops_types)
 
 
-class CheckpointingTests(test.TestCase):
+@keras_parameterized.run_with_all_model_types
+@keras_parameterized.run_all_keras_modes
+class CheckpointingTests(keras_parameterized.TestCase):
 
-  @test_util.run_in_graph_and_eager_modes
   def test_optimizer_dependency(self):
-    model = keras.models.Sequential()
-    model.add(keras.layers.Dense(1, input_shape=(4,)))
-    opt = adam.AdamOptimizer(0.01)
-    model.compile(optimizer=opt, loss='mse')
-    model.fit(x=np.array([[1., 2., 3., 4.]]), y=[1.], epochs=2)
+    model = _get_model()
+    opt = adam.AdamOptimizer(.01)
+    model.compile(
+        optimizer=opt, loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
+
+    model.fit(x=np.array([[1., 2., 3., 4.]]), y=np.array([1.]), epochs=2)
     save_prefix = os.path.join(self.get_temp_dir(), 'ckpt')
     beta1_power, _ = opt._get_beta_accumulators()
     self.evaluate(beta1_power.assign(12.))
@@ -420,7 +301,8 @@ class CheckpointingTests(test.TestCase):
     self.assertEqual(12., self.evaluate(beta1_power))
 
 
-class TestModelBackend(test.TestCase):
+@keras_parameterized.run_all_keras_modes
+class TestModelBackend(keras_parameterized.TestCase):
 
   def test_model_backend_float64_use_cases(self):
     # Test case for GitHub issue 19318
@@ -430,7 +312,9 @@ class TestModelBackend(test.TestCase):
     x = keras.Input((5,))
     y = keras.layers.Dense(1)(x)
     model = keras.models.Model(x, y)
-    model.compile('rmsprop', 'mse')
+    model.compile(
+        testing_utils.get_v2_optimizer('rmsprop'), 'mse',
+        run_eagerly=testing_utils.should_run_eagerly())
 
     keras.backend.set_floatx(floatx)
 
@@ -465,48 +349,46 @@ class TestModelDeepCopy(test.TestCase):
                       model_copy.get_weights()[0]))
 
 
-@test_util.run_v1_only('b/120545219')
-class TestCloneAndBuildModel(test.TestCase):
+@keras_parameterized.run_all_keras_modes
+class TestCloneAndBuildModel(keras_parameterized.TestCase):
 
+  @keras_parameterized.run_with_all_model_types
   def test_clone_and_build_non_compiled_model(self):
-    with self.cached_session():
-      inp = np.random.random((10, 4))
-      out = np.random.random((10, 4))
+    inp = np.random.random((10, 4))
+    out = np.random.random((10, 4))
 
-      model = keras.models.Sequential()
-      model.add(keras.layers.Dense(4, input_shape=(4,)))
-      model.add(keras.layers.BatchNormalization())
-      model.add(keras.layers.Dropout(0.5))
-      model.add(keras.layers.Dense(4))
-
-    # Everything should work in a new session.
-    keras.backend.clear_session()
-
-    with self.cached_session():
-      with self.assertRaisesRegexp(ValueError, 'has not been compiled'):
-        models.clone_and_build_model(model, compile_clone=True)
-
-      # With placeholder creation
-      new_model = models.clone_and_build_model(model, compile_clone=False)
-      with self.assertRaisesRegexp(RuntimeError, 'must compile'):
-        new_model.evaluate(inp, out)
-      with self.assertRaisesRegexp(RuntimeError, 'must compile'):
-        new_model.train_on_batch(inp, out)
-      new_model.compile('rmsprop', 'mse')
-      new_model.train_on_batch(inp, out)
+    model = _get_model()
+
+    with self.assertRaisesRegexp(ValueError, 'has not been compiled'):
+      models.clone_and_build_model(model, compile_clone=True)
 
-      # Create new tensors for inputs and targets
-      input_a = keras.Input(shape=(4,))
-      target_a = keras.Input(shape=(4,))
-      new_model = models.clone_and_build_model(model, input_tensors=input_a,
-                                               target_tensors=[target_a],
-                                               compile_clone=False)
-      with self.assertRaisesRegexp(RuntimeError, 'must compile'):
-        new_model.evaluate(inp, out)
-      with self.assertRaisesRegexp(RuntimeError, 'must compile'):
-        new_model.train_on_batch(inp, out)
-      new_model.compile('rmsprop', 'mse')
+    is_subclassed = (testing_utils.get_model_type() == 'subclass')
+    # With placeholder creation
+    new_model = models.clone_and_build_model(
+        model, compile_clone=False, in_place_reset=is_subclassed)
+    with self.assertRaisesRegexp(RuntimeError, 'must compile'):
+      new_model.evaluate(inp, out)
+    with self.assertRaisesRegexp(RuntimeError, 'must compile'):
+      new_model.train_on_batch(inp, out)
+    new_model.compile(
+        testing_utils.get_v2_optimizer('rmsprop'), 'mse',
+        run_eagerly=testing_utils.should_run_eagerly())
+    new_model.train_on_batch(inp, out)
+
+    # Create new tensors for inputs and targets
+    input_a = keras.Input(shape=(4,))
+    target_a = keras.Input(shape=(4,))
+    new_model = models.clone_and_build_model(
+        model, input_tensors=input_a, target_tensors=[target_a],
+        compile_clone=False, in_place_reset=is_subclassed)
+    with self.assertRaisesRegexp(RuntimeError, 'must compile'):
+      new_model.evaluate(inp, out)
+    with self.assertRaisesRegexp(RuntimeError, 'must compile'):
       new_model.train_on_batch(inp, out)
+    new_model.compile(
+        testing_utils.get_v2_optimizer('rmsprop'), 'mse',
+        run_eagerly=testing_utils.should_run_eagerly())
+    new_model.train_on_batch(inp, out)
 
   def _assert_same_compile_params(self, model):
     """Assert that two models have the same compile parameters."""
@@ -519,134 +401,88 @@ class TestCloneAndBuildModel(test.TestCase):
     self.assertEqual(['acc', metrics.categorical_accuracy],
                      model._compile_metrics)
 
-  def _clone_and_build_test_helper(self, model, is_subclassed=False):
+  def _clone_and_build_test_helper(self, model, model_type):
     inp = np.random.random((10, 4))
     out = np.random.random((10, 4))
 
-    # Everything should work in a new session.
-    keras.backend.clear_session()
-
-    with self.cached_session():
-      # With placeholder creation
-      new_model = models.clone_and_build_model(
-          model, compile_clone=True, in_place_reset=is_subclassed)
+    is_subclassed = (model_type == 'subclass')
+
+    # With placeholder creation
+    new_model = models.clone_and_build_model(
+        model, compile_clone=True, in_place_reset=is_subclassed)
+
+    self._assert_same_compile_params(new_model)
+    new_model.train_on_batch(inp, out)
+    new_model.evaluate(inp, out)
+
+    # Create new tensors for inputs and targets
+    input_a = keras.Input(shape=(4,), name='a')
+    new_model = models.clone_and_build_model(
+        model, input_tensors=input_a, compile_clone=True,
+        in_place_reset=is_subclassed)
+    self._assert_same_compile_params(new_model)
+    new_model.train_on_batch(inp, out)
+    new_model.evaluate(inp, out)
+
+    target_a = keras.Input(shape=(4,), name='b')
+    new_model = models.clone_and_build_model(
+        model, input_tensors=input_a, target_tensors=[target_a],
+        compile_clone=True, in_place_reset=is_subclassed)
+    self._assert_same_compile_params(new_model)
+    new_model.train_on_batch(inp, out)
+    new_model.evaluate(inp, out)
+
+  @keras_parameterized.run_with_all_model_types
+  def test_clone_and_build_compiled(self):
+    model = _get_model()
+    model.compile(
+        testing_utils.get_v2_optimizer('rmsprop'), 'mse',
+        metrics=['acc', metrics.categorical_accuracy],
+        run_eagerly=testing_utils.should_run_eagerly())
+
+    self._clone_and_build_test_helper(model, testing_utils.get_model_type())
+
+  def test_clone_and_build_sequential_without_inputs_defined(self):
+    model = models.Sequential(_get_layers(input_shape=None))
+    model.compile(
+        testing_utils.get_v2_optimizer('rmsprop'),
+        'mse', metrics=['acc', metrics.categorical_accuracy],
+        run_eagerly=testing_utils.should_run_eagerly())
+    self._clone_and_build_test_helper(model, 'sequential')
 
-      self._assert_same_compile_params(new_model)
-      new_model.train_on_batch(inp, out)
-      new_model.evaluate(inp, out)
-
-      # Create new tensors for inputs and targets
-      input_a = keras.Input(shape=(4,), name='a')
-      new_model = models.clone_and_build_model(
-          model, input_tensors=input_a, compile_clone=True,
-          in_place_reset=is_subclassed)
-      self._assert_same_compile_params(new_model)
-      new_model.train_on_batch(inp, out)
-      new_model.evaluate(inp, out)
-
-      target_a = keras.Input(shape=(4,), name='b')
-      new_model = models.clone_and_build_model(
-          model, input_tensors=input_a, target_tensors=[target_a],
-          compile_clone=True, in_place_reset=is_subclassed)
-      self._assert_same_compile_params(new_model)
-      new_model.train_on_batch(inp, out)
-      new_model.evaluate(inp, out)
-
-  def test_clone_and_build_compiled_sequential_model(self):
-    with self.cached_session():
-      model = keras.models.Sequential()
-      model.add(keras.layers.Dense(4, input_shape=(4,)))
-      model.add(keras.layers.BatchNormalization())
-      model.add(keras.layers.Dropout(0.5))
-      model.add(keras.layers.Dense(4))
-      model.compile('rmsprop', 'mse',
-                    metrics=['acc', metrics.categorical_accuracy])
-
-    self._clone_and_build_test_helper(model)
-
-  def test_clone_and_build_functional_model(self):
-    with self.cached_session():
-      input_a = keras.Input(shape=(4,))
-      dense_1 = keras.layers.Dense(4,)
-      dense_2 = keras.layers.Dense(4,)
-
-      x_a = dense_1(input_a)
-      x_a = keras.layers.Dropout(0.5)(x_a)
-      x_a = keras.layers.BatchNormalization()(x_a)
-      x_a = dense_2(x_a)
-      model = keras.models.Model(input_a, x_a)
-      model.compile('rmsprop', 'mse',
-                    metrics=['acc', metrics.categorical_accuracy])
-
-    self._clone_and_build_test_helper(model)
-
-  def test_clone_and_build_subclassed_model(self):
-    class SubclassedModel(keras.Model):
-
-      def __init__(self):
-        super(SubclassedModel, self).__init__()
-        self.layer1 = keras.layers.Dense(4)
-        self.layer2 = keras.layers.Dense(4)
-
-      def call(self, inp):
-        out = self.layer1(inp)
-        out = keras.layers.BatchNormalization()(out)
-        out = keras.layers.Dropout(0.5)(out)
-        out = self.layer2(out)
-        return out
-
-    with self.cached_session():
-      model = SubclassedModel()
-      model.compile('rmsprop', 'mse',
-                    metrics=['acc', metrics.categorical_accuracy])
-    self._clone_and_build_test_helper(model, True)
+    inp = np.random.random((10, 4))
+    out = np.random.random((10, 4))
+    model.train_on_batch(inp, out)
+    self._clone_and_build_test_helper(model, 'sequential')
 
   def assert_optimizer_iterations_increases(self, optimizer):
-    with self.cached_session():
-      input_a = keras.Input(shape=(4,))
-      dense_1 = keras.layers.Dense(4,)
-      dense_2 = keras.layers.Dense(4,)
-
-      x_a = dense_1(input_a)
-      x_a = keras.layers.Dropout(0.5)(x_a)
-      x_a = keras.layers.BatchNormalization()(x_a)
-      x_a = dense_2(x_a)
-      model = keras.models.Model(input_a, x_a)
-      model.compile(optimizer, 'mse',
-                    metrics=['acc', metrics.categorical_accuracy])
+    model = _get_model()
+    model.compile(
+        optimizer, 'mse', metrics=['acc', metrics.categorical_accuracy],
+        run_eagerly=testing_utils.should_run_eagerly())
 
-      global_step = keras.backend.variable(123, dtype=dtypes.int64)
-      clone_model = models.clone_and_build_model(
-          model, compile_clone=True, optimizer_iterations=global_step)
+    global_step = keras.backend.variable(123, dtype=dtypes.int64)
+    clone_model = models.clone_and_build_model(
+        model, compile_clone=True, optimizer_iterations=global_step,
+        in_place_reset=(testing_utils.get_model_type() == 'subclass'))
 
-      inp = np.random.random((10, 4))
-      out = np.random.random((10, 4))
-      clone_model.train_on_batch(inp, out)
+    inp = np.random.random((10, 4))
+    out = np.random.random((10, 4))
+    clone_model.train_on_batch(inp, out)
 
-      self.assertEqual(K.eval(global_step), 124)
+    self.assertEqual(K.eval(global_step), 124)
 
+  @keras_parameterized.run_with_all_model_types
   def test_replace_tf_optimizer_iterations_variable(self):
     self.assert_optimizer_iterations_increases(adam.AdamOptimizer(0.01))
 
+  @keras_parameterized.run_with_all_model_types
   def test_replace_keras_optimizer_iterations_variable(self):
-    self.assert_optimizer_iterations_increases('adam')
+    if testing_utils.should_run_eagerly():
+      # This needs to be updated to run with v2 optimizers.
+      self.skipTest('b/120991591')
 
-  def test_replace_keras_optimizer_v2_iterations_variable(self):
-    self.assert_optimizer_iterations_increases(
-        keras.optimizer_v2.adam.Adam(0.01))
-
-  def test_clone_and_build_sequential_model_without_inputs_defined(self):
-    with self.cached_session():
-      model = sequential_model(False, False)
-      model.compile('rmsprop', 'mse',
-                    metrics=['acc', metrics.categorical_accuracy])
-    self._clone_and_build_test_helper(model, False)
-
-    with self.cached_session():
-      inp = np.random.random((10, 4))
-      out = np.random.random((10, 4))
-      model.train_on_batch(inp, out)
-    self._clone_and_build_test_helper(model, False)
+    self.assert_optimizer_iterations_increases('adam')
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/keras/testing_utils.py b/tensorflow/python/keras/testing_utils.py
index fd062b0ab3..6448f87d25 100644
--- a/tensorflow/python/keras/testing_utils.py
+++ b/tensorflow/python/keras/testing_utils.py
@@ -25,6 +25,13 @@ import numpy as np
 from tensorflow.python import keras
 from tensorflow.python.eager import context
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.keras.optimizer_v2 import adadelta as adadelta_v2
+from tensorflow.python.keras.optimizer_v2 import adagrad as adagrad_v2
+from tensorflow.python.keras.optimizer_v2 import adam as adam_v2
+from tensorflow.python.keras.optimizer_v2 import adamax as adamax_v2
+from tensorflow.python.keras.optimizer_v2 import gradient_descent as gradient_descent_v2
+from tensorflow.python.keras.optimizer_v2 import nadam as nadam_v2
+from tensorflow.python.keras.optimizer_v2 import rmsprop as rmsprop_v2
 from tensorflow.python.training.rmsprop import RMSPropOptimizer
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util import tf_inspect
@@ -355,11 +362,20 @@ class _SubclassModel(keras.Model):
 
   def __init__(self, layers):
     super(_SubclassModel, self).__init__()
-    self.all_layers = layers
+    # Note that clone and build doesn't support lists of layers in subclassed
+    # models. Adding each layer directly here.
+    for i, layer in enumerate(layers):
+      setattr(self, self._layer_name_for_i(i), layer)
+
+    self.num_layers = len(layers)
+
+  def _layer_name_for_i(self, i):
+    return 'layer{}'.format(i)
 
   def call(self, inputs, **kwargs):
     x = inputs
-    for layer in self.all_layers:
+    for i in range(self.num_layers):
+      layer = getattr(self, self._layer_name_for_i(i))
       x = layer(x)
     return x
 
@@ -626,3 +642,39 @@ def get_multi_io_model(
     return keras.Model(inputs, outputs)
 
   raise ValueError('Unknown model type {}'.format(model_type))
+
+
+_V2_OPTIMIZER_MAP = {
+    'adadelta': adadelta_v2.Adadelta,
+    'adagrad': adagrad_v2.Adagrad,
+    'adam': adam_v2.Adam,
+    'adamax': adamax_v2.Adamax,
+    'nadam': nadam_v2.Nadam,
+    'rmsprop': rmsprop_v2.RMSprop,
+    'sgd': gradient_descent_v2.SGD
+}
+
+
+def get_v2_optimizer(name, **kwargs):
+  """Get the v2 optimizer requested.
+
+  This is only necessary until v2 are the default, as we are testing in Eager,
+  and Eager + v1 optimizers fail tests. When we are in v2, the strings alone
+  should be sufficient, and this mapping can theoretically be removed.
+
+  Args:
+    name: string name of Keras v2 optimizer.
+    **kwargs: any kwargs to pass to the optimizer constructor.
+
+  Returns:
+    Initialized Keras v2 optimizer.
+
+  Raises:
+    ValueError: if an unknown name was passed.
+  """
+  try:
+    return _V2_OPTIMIZER_MAP[name](**kwargs)
+  except KeyError:
+    raise ValueError(
+        'Could not find requested v2 optimizer: {}\nValid choices: {}'.format(
+            name, list(_V2_OPTIMIZER_MAP.keys())))
-- 
GitLab


From 17bf2b33a90017c8536ef4a242bb60636bfc5e8a Mon Sep 17 00:00:00 2001
From: Russell Power <power@google.com>
Date: Fri, 4 Jan 2019 17:45:29 -0800
Subject: [PATCH 0273/1095] Graph construction: skip testing for duplicate
 edges during source/sink setup.

When fixing up source/sink edges, we are only adding missing edges.  Skip the
test for duplicate edges in this case, which avoids a linear traversal over
sink edges.

PiperOrigin-RevId: 227940303
---
 tensorflow/core/graph/algorithm.cc | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/graph/algorithm.cc b/tensorflow/core/graph/algorithm.cc
index 9b4200e0b4..25bd3516a1 100644
--- a/tensorflow/core/graph/algorithm.cc
+++ b/tensorflow/core/graph/algorithm.cc
@@ -222,11 +222,12 @@ bool FixupSourceAndSinkEdges(Graph* g) {
   bool changed = false;
   for (Node* n : g->nodes()) {
     if (!n->IsSource() && n->in_edges().empty()) {
-      g->AddControlEdge(g->source_node(), n);
+      g->AddControlEdge(g->source_node(), n,
+                        true /* skip test for duplicates */);
       changed = true;
     }
     if (!n->IsSink() && n->out_edges().empty()) {
-      g->AddControlEdge(n, g->sink_node());
+      g->AddControlEdge(n, g->sink_node(), true /* skip test for duplicates */);
       changed = true;
     }
   }
-- 
GitLab


From 1814e22d3cf78fd6e5b172c6de1a44cd6abc25f3 Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Fri, 4 Jan 2019 17:45:29 -0800
Subject: [PATCH 0274/1095] Fix some empty struct build warnings in TFLite

PiperOrigin-RevId: 227940304
---
 tensorflow/lite/c/builtin_op_data.h | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/tensorflow/lite/c/builtin_op_data.h b/tensorflow/lite/c/builtin_op_data.h
index 46ac656ecf..332c2db145 100644
--- a/tensorflow/lite/c/builtin_op_data.h
+++ b/tensorflow/lite/c/builtin_op_data.h
@@ -25,6 +25,11 @@ extern "C" {
 
 // TODO(aselle): Consider using "if this then that" for testing.
 
+// Useful placeholder to put in otherwise empty structs to avoid size warnings.
+typedef struct {
+  char dummy;
+} EmptyStructPlaceholder;
+
 // IMPORTANT: All new members of structs must be added at the end to ensure
 // backwards compatibility.
 
@@ -152,9 +157,11 @@ typedef struct {
 } TfLiteAddParams;
 
 typedef struct {
+  EmptyStructPlaceholder placeholder;
 } TfLiteSpaceToBatchNDParams;
 
 typedef struct {
+  EmptyStructPlaceholder placeholder;
 } TfLiteBatchToSpaceNDParams;
 
 typedef struct {
@@ -230,9 +237,11 @@ typedef struct {
 } TfLiteResizeNearestNeighborParams;
 
 typedef struct {
+  EmptyStructPlaceholder placeholder;
 } TfLitePadParams;
 
 typedef struct {
+  EmptyStructPlaceholder placeholder;
 } TfLitePadV2Params;
 
 typedef struct {
@@ -272,6 +281,7 @@ typedef struct {
 } TfLiteGatherParams;
 
 typedef struct {
+  EmptyStructPlaceholder placeholder;
 } TfLiteTransposeParams;
 
 typedef struct {
-- 
GitLab


From 447189935cbc1ab3aeb2ac67f81f03d1a188e548 Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Fri, 4 Jan 2019 18:40:27 -0800
Subject: [PATCH 0275/1095] Make TRT tests respect RewriterConfig set by
 individual test case everywhere.

PiperOrigin-RevId: 227944638
---
 .../test/tf_trt_integration_test_base.py       | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
index 495a9391a1..671abba6a6 100644
--- a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
+++ b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
@@ -239,8 +239,8 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
 
   def _GetConfigProto(self, run_params, graph_state):
     """Get config proto based on specific settings."""
+    conversion_params = self.GetConversionParams(run_params)
     if graph_state != GraphState.ORIGINAL and run_params.use_optimizer:
-      conversion_params = self.GetConversionParams(run_params)
       rewriter_cfg = trt_convert.get_tensorrt_rewriter_config(
           conversion_params.rewriter_config, conversion_params.max_batch_size,
           conversion_params.max_workspace_size_bytes,
@@ -254,6 +254,9 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
       graph_options = config_pb2.GraphOptions(rewrite_options=rewriter_cfg)
     else:
       graph_options = config_pb2.GraphOptions()
+      if conversion_params.rewriter_config is not None:
+        graph_options.rewrite_options.CopyFrom(
+            conversion_params.rewriter_config)
 
     config = config_pb2.ConfigProto(
         gpu_options=self._GetGPUOptions(), graph_options=graph_options)
@@ -321,16 +324,13 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
     return self._RunGraph(
         run_params, gdef, input_data, config, GraphState.CALIBRATE, num_runs=5)
 
-  def _GetTrtGraphDef(self, run_params, gdef):
+  def _GetTrtGraphDef(self, run_params, graph_state, gdef):
     """Return trt converted graphdef."""
     params = self._GetParamsCached()
     conversion_params = self.GetConversionParams(run_params)
     logging.info(conversion_params)
 
-    config_for_trt = config_pb2.ConfigProto(gpu_options=self._GetGPUOptions())
-    if conversion_params.rewriter_config is not None:
-      config_for_trt.graph_options.rewrite_options.CopyFrom(
-          conversion_params.rewriter_config)
+    config_for_trt = self._GetConfigProto(run_params, graph_state)
     return trt_convert.create_inference_graph(
         input_graph_def=gdef,
         outputs=params.input_names + params.output_names,
@@ -506,7 +506,8 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
         result = self._RunCalibration(run_params, input_gdef, input_data,
                                       calib_config)
       else:
-        calib_gdef = self._GetTrtGraphDef(run_params, input_gdef)
+        calib_gdef = self._GetTrtGraphDef(run_params, GraphState.CALIBRATE,
+                                          input_gdef)
         self._VerifyGraphDef(run_params, calib_gdef, GraphState.CALIBRATE)
         result = self._RunCalibration(run_params, calib_gdef, input_data,
                                       calib_config)
@@ -527,7 +528,8 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
     logging.info("Running final inference graph, config:\n%s",
                  str(infer_config))
     if not run_params.use_optimizer:
-      infer_gdef = self._GetTrtGraphDef(run_params, infer_gdef)
+      infer_gdef = self._GetTrtGraphDef(run_params, GraphState.INFERENCE,
+                                        infer_gdef)
       self._VerifyGraphDef(run_params, infer_gdef, GraphState.INFERENCE)
 
     result = self._RunGraph(run_params, infer_gdef, input_data, infer_config,
-- 
GitLab


From d93409b9dfc3bfaacfa452fab603a089931b3335 Mon Sep 17 00:00:00 2001
From: Davide Libenzi <dlibenzi@google.com>
Date: Fri, 4 Jan 2019 19:00:39 -0800
Subject: [PATCH 0276/1095] Automated rollback of commit
 55fda7fb66f1774ac5271e523259faa7d2a44d0c

PiperOrigin-RevId: 227945898
---
 .../xla/service/cpu/cpu_executable.cc         |  29 +---
 .../xla/service/gpu/gpu_executable.cc         |  26 +---
 tensorflow/compiler/xrt/kernels/BUILD         |   1 -
 .../compiler/xrt/kernels/xrt_execute_op.cc    |  17 ---
 tensorflow/compiler/xrt/tests/raw_api_test.cc | 127 +-----------------
 tensorflow/compiler/xrt/xrt_state.cc          |  43 +-----
 tensorflow/compiler/xrt/xrt_state.h           |  23 +---
 7 files changed, 17 insertions(+), 249 deletions(-)

diff --git a/tensorflow/compiler/xla/service/cpu/cpu_executable.cc b/tensorflow/compiler/xla/service/cpu/cpu_executable.cc
index 23d0af3423..412c2715b9 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_executable.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_executable.cc
@@ -213,8 +213,6 @@ StatusOr<ScopedShapedBuffer> CpuExecutable::CreateResultShapedBuffer(
       /*on_host_shape=*/result_shape(),
       /*on_device_shape=*/result_shape(), run_options->allocator(),
       stream->parent()->device_ordinal());
-  const HloInputOutputAliasConfig& input_output_alias =
-      module().input_output_alias_config();
 
   // Move OwningDeviceMemory values which contain the array(s) of the result
   // into the respective location in ScopedShapedBuffer which is returned to the
@@ -234,31 +232,12 @@ StatusOr<ScopedShapedBuffer> CpuExecutable::CreateResultShapedBuffer(
         TF_ASSIGN_OR_RETURN(
             const BufferAllocation::Slice slice,
             this->assignment_->GetUniqueSlice(src, buffer_source->index()));
+        CHECK(!slice.allocation()->is_entry_computation_parameter());
+
         const BufferAllocation::Index buffer_index = slice.index();
         OwningDeviceMemory& buffer = buffers[buffer_index];
-        if (!slice.allocation()->is_entry_computation_parameter()) {
-          // If the buffer coming out of the result is from a parameter, the
-          // owning buffer will be null, and that means the caller aliased some
-          // parameter buffer to an output one (via the
-          // HloInputOutputAliasConfig API). If that is the case, the caller
-          // will receive a partially complete scoped shaped buffer, which they
-          // will have to fill up on return. Unfortunately the interface to the
-          // execute APIs are ShapedBuffer pointer based, which assumes caller
-          // ownership, and hence a buffer coming from there cannot be part of
-          // the new ScopedShapedBuffer we create for the result (which assumes
-          // ownership).
-          *device_memory = buffer.Forget();
-        } else {
-          auto output_alias = input_output_alias.GetAliasedOutput(
-              slice.allocation()->parameter_number(),
-              slice.allocation()->param_shape_index());
-          CHECK(output_alias)
-              << "Ouput buffer is coming from parameter "
-              << slice.allocation()->parameter_number() << " at index "
-              << slice.allocation()->param_shape_index()
-              << ", but no alias exists";
-          CHECK_EQ(*output_alias, index);
-        }
+        CHECK(!buffer.is_null() || buffer.size() == 0);
+        *device_memory = buffer.Forget();
         return Status::OK();
       }));
   return std::move(result_buffer);
diff --git a/tensorflow/compiler/xla/service/gpu/gpu_executable.cc b/tensorflow/compiler/xla/service/gpu/gpu_executable.cc
index 434060ad89..128cecd765 100644
--- a/tensorflow/compiler/xla/service/gpu/gpu_executable.cc
+++ b/tensorflow/compiler/xla/service/gpu/gpu_executable.cc
@@ -310,34 +310,12 @@ StatusOr<ScopedShapedBuffer> GpuExecutable::ExecuteOnStream(
         TF_ASSIGN_OR_RETURN(
             const BufferAllocation::Slice slice,
             this->assignment_->GetUniqueSlice(src_hlo, sources[0]->index()));
+        CHECK(!slice.allocation()->is_entry_computation_parameter());
 
         se::DeviceMemoryBase src_base =
             buffer_allocations->GetDeviceAddress(slice.index());
         CHECK(!src_base.is_null() || src_base.size() == 0);
-        if (!slice.allocation()->is_entry_computation_parameter()) {
-          // If the buffer coming out of the result is from a parameter, it
-          // means the caller aliased some parameter buffer to an output one
-          // (via the HloInputOutputAliasConfig API). If that is the case, the
-          // caller will receive a partially complete scoped shaped buffer,
-          // which they will have to fill up on return.
-          // Unfortunately the interface to the execute APIs are ShapedBuffer
-          // pointer based, which assumes caller ownership, and hence a buffer
-          // coming from there cannot be part of the new ScopedShapedBuffer we
-          // create for the result (which assumes ownership).
-          *device_memory = src_base;
-        } else {
-          const HloInputOutputAliasConfig& input_output_alias =
-              module().input_output_alias_config();
-          auto output_alias = input_output_alias.GetAliasedOutput(
-              slice.allocation()->parameter_number(),
-              slice.allocation()->param_shape_index());
-          CHECK(output_alias)
-              << "Ouput buffer is coming from parameter "
-              << slice.allocation()->parameter_number() << " at index "
-              << slice.allocation()->param_shape_index()
-              << ", but no alias exists";
-          CHECK_EQ(*output_alias, index);
-        }
+        *device_memory = src_base;
         buffers_in_result.insert(src_base);
         return Status::OK();
       }));
diff --git a/tensorflow/compiler/xrt/kernels/BUILD b/tensorflow/compiler/xrt/kernels/BUILD
index c44769dfe3..67f475846e 100644
--- a/tensorflow/compiler/xrt/kernels/BUILD
+++ b/tensorflow/compiler/xrt/kernels/BUILD
@@ -55,7 +55,6 @@ cc_library(
         "//tensorflow/compiler/xla/client:xla_computation",
         "//tensorflow/compiler/xla/service:compiler",
         "//tensorflow/compiler/xla/service:computation_placer",
-        "//tensorflow/compiler/xla/service:hlo",
         "//tensorflow/compiler/xrt:xrt_proto",
         "//tensorflow/compiler/xrt:xrt_utils",
         "//tensorflow/core:core_cpu_internal",
diff --git a/tensorflow/compiler/xrt/kernels/xrt_execute_op.cc b/tensorflow/compiler/xrt/kernels/xrt_execute_op.cc
index 7544541ca4..7f0ac123a5 100644
--- a/tensorflow/compiler/xrt/kernels/xrt_execute_op.cc
+++ b/tensorflow/compiler/xrt/kernels/xrt_execute_op.cc
@@ -19,7 +19,6 @@ limitations under the License.
 
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/service/computation_placer.h"
-#include "tensorflow/compiler/xla/service/hlo_input_output_alias_config.h"
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/status_macros.h"
 #include "tensorflow/compiler/xla/statusor.h"
@@ -229,22 +228,6 @@ Status XRTExecuteOp::DoWork(OpKernelContext* context) {
   TF_RETURN_IF_ERROR(XRTTupleAllocation::CreateFromBuffer(
       shaped_buffer, device_ref.backend(), device_ref.device_ordinal(),
       &output_tuple));
-
-  // The ScopedShapedBuffer returned by the executable Run() API, in case of
-  // input/output buffer aliasing, might have holes in it, which need to be
-  // filled using the proper input tuples buffers which are the source of
-  // aliasing.
-  const xla::HloInputOutputAliasConfig& input_output_alias =
-      executable->executable()->module().input_output_alias_config();
-  auto alias_function = [&](const xla::ShapeIndex& output_index,
-                            int64 param_number,
-                            const xla::ShapeIndex& param_index) -> Status {
-    TF_RET_CHECK(param_number < input_tuples.size());
-    return output_tuple->AliasBufferFrom(*input_tuples[param_number],
-                                         param_index, output_index);
-  };
-  TF_RETURN_IF_ERROR(input_output_alias.ForEachAliasWithStatus(alias_function));
-
   if (config_proto.return_exploded_tuple() &&
       output_tuple->on_device_shape().IsTuple()) {
     int64 tuple_element_count =
diff --git a/tensorflow/compiler/xrt/tests/raw_api_test.cc b/tensorflow/compiler/xrt/tests/raw_api_test.cc
index f81faf0e61..c8479cb778 100644
--- a/tensorflow/compiler/xrt/tests/raw_api_test.cc
+++ b/tensorflow/compiler/xrt/tests/raw_api_test.cc
@@ -96,21 +96,14 @@ xla::LiteralProto FloatMatrix(
   return array.ToProto();
 }
 
-xla::Literal ReadOutputLiteral(const std::vector<Tensor>& outputs, size_t idx) {
-  xla::LiteralProto response;
-  CHECK(response.ParseFromString(outputs[idx].scalar<string>()()));
-  return xla::Literal::CreateFromProto(response).ValueOrDie();
-}
-
 bool CompareLiteralProtos(const xla::LiteralProto& a,
                           const xla::LiteralProto& b) {
   auto l_a = xla::Literal::CreateFromProto(a).ValueOrDie();
   auto l_b = xla::Literal::CreateFromProto(b).ValueOrDie();
   bool equal = l_a == l_b;
   if (!equal) {
-    LOG(INFO) << "LiteralProtos don't match:\n"
-              << a.DebugString() << "\n!=\n"
-              << b.DebugString();
+    LOG(INFO) << "LiteralProtos don't match: " << a.DebugString()
+              << " != " << b.DebugString();
   }
   return equal;
 }
@@ -120,19 +113,8 @@ bool CompareLiteralToLiteralProto(const xla::Literal& a,
   auto l_b = xla::Literal::CreateFromProto(b).ValueOrDie();
   bool equal = a == l_b;
   if (!equal) {
-    LOG(INFO) << "Literal and LiteralProto don't match:\n"
-              << a.ToProto().DebugString() << "\n!=\n"
-              << b.DebugString();
-  }
-  return equal;
-}
-
-bool CompareLiterals(const xla::Literal& a, const xla::Literal& b) {
-  bool equal = a == b;
-  if (!equal) {
-    LOG(INFO) << "Literals don't match:\n"
-              << a.ToProto().DebugString() << "\n!=\n"
-              << b.ToProto().DebugString();
+    LOG(INFO) << "Literal and LiteralProto don't match "
+              << a.ToProto().DebugString() << " != " << b.DebugString();
   }
   return equal;
 }
@@ -957,107 +939,6 @@ TEST(RawApiTest, LeakCompilationReference) {
   TF_EXPECT_OK(session.Run({c_handle.handle}, &outputs));
 }
 
-TEST(RawApiTest, CompileAndExecuteWithReusedBuffers) {
-  xla::Shape element_shape = xla::ShapeUtil::MakeShape(xla::F32, {2});
-  xla::Shape shape =
-      xla::ShapeUtil::MakeTupleShape({element_shape, element_shape});
-  xla::Shape return_shape = xla::ShapeUtil::MakeTupleShape(
-      {element_shape, element_shape, element_shape, element_shape});
-  xla::XlaBuilder builder("ReuseBuffer");
-  auto param = xla::Parameter(&builder, 0, shape, "param");
-  auto p0 = xla::GetTupleElement(param, 0);
-  auto p1 = xla::GetTupleElement(param, 1);
-  auto add = xla::Add(p0, p1);
-  auto sub = xla::Sub(p0, p1);
-  xla::Tuple(&builder, {add, sub, p0, p1});
-
-  // Flip the tuple literals in the input handle.
-  builder.SetUpAlias({1}, 0, {0});
-  builder.SetUpAlias({0}, 0, {1});
-
-  auto computation = builder.Build().ValueOrDie();
-
-  auto literal0 = xla::LiteralUtil::CreateR1<float>({1.0f, 2.0f});
-  auto literal1 = xla::LiteralUtil::CreateR1<float>({5.0f, 9.0f});
-  auto literal = xla::LiteralUtil::MakeTuple({&literal0, &literal1});
-
-  xrt::XLAAllocation param_alloc;
-  *param_alloc.mutable_value() = literal.ToProto();
-
-  xrt::XLAComputation c;
-  auto config = c.mutable_config();
-  auto shapes = config->mutable_program_shape();
-  *shapes->add_parameters() = shape.ToProto();
-  *shapes->mutable_result() = return_shape.ToProto();
-  StoreComputationSnapshot(computation, c.mutable_hlo_snapshot());
-
-  xrt::XRTExecutionConfig e;
-  e.set_release_input_handles(false);
-  e.set_release_compilation_handle(true);
-
-  Scope root = Scope::NewRootScope().WithDevice(DeviceFromFlag());
-  ClientSession session(root);
-  auto e_config =
-      ops::Const(root.WithDevice("/device:CPU:0"), e.SerializeAsString());
-  auto c_data =
-      ops::Const(root.WithDevice("/device:CPU:0"), c.SerializeAsString());
-  auto c_handle = ops::XRTCompile(root, c_data);
-  auto param_value = ops::Const(root.WithDevice("/device:CPU:0"),
-                                param_alloc.SerializeAsString());
-  auto param_handle = ops::XRTAllocate(root, param_value);
-  TF_ASSERT_OK(root.status());
-
-  std::vector<Tensor> outputs;
-  TF_EXPECT_OK(session.Run({param_handle}, &outputs));
-
-  int64 alloc_handle = outputs[0].scalar<int64>()();
-
-  // Note that we release the result handle immediately, but since we aliased
-  // the output buffers onto the input allocation ones (held in alloc_handle),
-  // we can fetch the result from there.
-  auto result =
-      ops::XRTExecute(root, c_handle.handle, e_config, {Input(alloc_handle)});
-  auto read_back = ops::XRTReadLiteral(root, result);
-  auto release = ops::XRTReleaseAllocationHandle(
-      root.WithControlDependencies(read_back), result);
-  TF_ASSERT_OK(root.status());
-
-  outputs.clear();
-  TF_EXPECT_OK(session.Run(tensorflow::ClientSession::FeedType(), {read_back},
-                           {release}, &outputs));
-
-  xla::Literal exec_literal = ReadOutputLiteral(outputs, 0);
-  auto exec_literal_parts = exec_literal.DecomposeTuple();
-  ASSERT_EQ(exec_literal_parts.size(), 4);
-
-  EXPECT_TRUE(CompareLiterals(exec_literal_parts[2], literal0));
-  EXPECT_TRUE(CompareLiterals(exec_literal_parts[3], literal1));
-
-  // Now we read back the original input handle values, which at this point
-  // should contain the result of the XLA computation.
-  auto read_handle = ops::XRTReadLiteral(root, Input(alloc_handle));
-  TF_ASSERT_OK(root.status());
-  auto release_handle = ops::XRTReleaseAllocationHandle(
-      root.WithControlDependencies(read_handle), Input(alloc_handle));
-  TF_ASSERT_OK(root.status());
-
-  outputs.clear();
-  TF_EXPECT_OK(session.Run(tensorflow::ClientSession::FeedType(), {read_handle},
-                           {release_handle}, &outputs));
-
-  xla::Literal return_literal = ReadOutputLiteral(outputs, 0);
-
-  auto expected_literal0 = xla::LiteralUtil::CreateR1<float>({6.0f, 11.0f});
-  auto expected_literal1 = xla::LiteralUtil::CreateR1<float>({-4.0f, -7.0f});
-  // The first element of the computation returned tuple would be the add
-  // (expected_literal0), but since we flipped the buffers, the sub
-  // (expected_literal1) should come first.
-  auto expected_literal =
-      xla::LiteralUtil::MakeTuple({&expected_literal1, &expected_literal0});
-
-  EXPECT_TRUE(CompareLiterals(return_literal, expected_literal));
-}
-
 TEST(RawApiTest, CompileAndExecuteWithS64Argument) {
   xrt::XLAAllocation p0;
   *p0.mutable_value() = xla::LiteralUtil::CreateR0<int64>(11031965).ToProto();
diff --git a/tensorflow/compiler/xrt/xrt_state.cc b/tensorflow/compiler/xrt/xrt_state.cc
index 13c275aaca..343460ff10 100644
--- a/tensorflow/compiler/xrt/xrt_state.cc
+++ b/tensorflow/compiler/xrt/xrt_state.cc
@@ -133,8 +133,7 @@ Status AllocateScopedShapedBuffer(
 XRTBufferAllocation::XRTBufferAllocation(const se::DeviceMemoryBase& allocation,
                                          int device_ordinal,
                                          xla::DeviceMemoryAllocator* allocator)
-    : size_(allocation.size()),
-      allocation_(allocation),
+    : allocation_(allocation),
       device_ordinal_(device_ordinal),
       allocator_(allocator) {
   if (VLOG_IS_ON(2)) {
@@ -224,19 +223,8 @@ Status XRTTupleAllocation::ToLiteral(xla::Backend* backend, int device_ordinal,
                                      xla::Literal* literal) {
   auto transfer_manager = backend->transfer_manager();
   TF_ASSIGN_OR_RETURN(auto stream, backend->BorrowStream(device_ordinal));
-
-  // Validate the allocation buffers as if nulls gets to
-  // TransferLiteralFromDevice() a CHECK is issued.
-  xla::ShapedBuffer shaped_buffer = ToShapedBuffer();
-  for (auto& index_buffer : shaped_buffer.buffers()) {
-    if (index_buffer.second.is_null()) {
-      return errors::InvalidArgument("Literal buffer at index ",
-                                     index_buffer.first.ToString(),
-                                     " has been released");
-    }
-  }
   TF_ASSIGN_OR_RETURN(*literal, transfer_manager->TransferLiteralFromDevice(
-                                    stream.get(), shaped_buffer));
+                                    stream.get(), ToShapedBuffer()));
   return Status::OK();
 }
 
@@ -517,34 +505,11 @@ xla::ShapedBuffer XRTTupleAllocation::ToShapedBuffer() {
   return shaped_buffer;
 }
 
-Status XRTTupleAllocation::AliasBufferFrom(const XRTTupleAllocation& source,
-                                           const xla::ShapeIndex& source_index,
-                                           const xla::ShapeIndex& dest_index) {
-  XRTBufferAllocation* source_buffer = source.buffers_.element(source_index);
-  XRTBufferAllocation* dest_buffer = buffers_.element(dest_index);
-  // We allow the destination size being zero, because there are cases where we
-  // are coming in later filling in null/uninitialized device buffers.
-  // In all other cases, the size of the new buffer must match.
-  if (source_buffer->size() != dest_buffer->size() &&
-      dest_buffer->size() != 0) {
-    return errors::InvalidArgument(
-        "Source buffer at index ", source_index.ToString(),
-        " does not match the size of destination buffer at index ",
-        dest_index.ToString(), ": ", source_buffer->size(), " vs ",
-        dest_buffer->size());
-  }
-  *buffers_.mutable_element(dest_index) = source_buffer;
-  source_buffer->Ref();
-  dest_buffer->Unref();
-  return Status::OK();
-}
-
 xla::ShapeTree<xla::MaybeOwningDeviceMemory>
-XRTTupleAllocation::ToDeviceMemoryTree(
-    const std::function<bool(const xla::ShapeIndex&)>& release_checker) {
+XRTTupleAllocation::ToDeviceMemoryTree(bool release) {
   xla::ShapeTree<xla::MaybeOwningDeviceMemory> shaped_tree(on_device_shape());
   for (const auto& buffer : buffers_) {
-    if (!release_checker(buffer.first)) {
+    if (!release) {
       *shaped_tree.mutable_element(buffer.first) = buffer.second->allocation();
     } else {
       *shaped_tree.mutable_element(buffer.first) = xla::OwningDeviceMemory(
diff --git a/tensorflow/compiler/xrt/xrt_state.h b/tensorflow/compiler/xrt/xrt_state.h
index ac4be3a064..3e3d502412 100644
--- a/tensorflow/compiler/xrt/xrt_state.h
+++ b/tensorflow/compiler/xrt/xrt_state.h
@@ -18,7 +18,6 @@ limitations under the License.
 #ifndef TENSORFLOW_COMPILER_XRT_XRT_STATE_H_
 #define TENSORFLOW_COMPILER_XRT_XRT_STATE_H_
 
-#include <functional>
 #include <memory>
 #include <string>
 #include <vector>
@@ -59,14 +58,7 @@ class XRTBufferAllocation : public core::RefCounted {
   // freed when the reference count drops to zero.
   void DiscardAllocation();
 
-  // Returns the expected size of the allocation. Since DiscardAllocation() will
-  // set allocation_ to {null,0}, and since later we might want to replace the
-  // discarded buffer with a new one, we need to be able to verify the size
-  // compatibility.
-  uint64 size() const { return size_; }
-
  private:
-  uint64 size_ = 0;
   se::DeviceMemoryBase allocation_;
   int device_ordinal_;
   xla::DeviceMemoryAllocator* allocator_;
@@ -176,18 +168,9 @@ class XRTTupleAllocation : public ResourceBase {
   // the same shape as on_host_shape.
   xla::ShapedBuffer ToShapedBuffer();
 
-  // Aliases the source buffer at source_index into the current tuple allocation
-  // dest_index.
-  Status AliasBufferFrom(const XRTTupleAllocation& source,
-                         const xla::ShapeIndex& source_index,
-                         const xla::ShapeIndex& dest_index);
-
-  // Returns the device memory tree of this allocation. If the release_checker
-  // function returns true for a given index, the ownership of the device memory
-  // at that index is transferred to the result. Every attempt to read the value
-  // at that index will fail.
-  xla::ShapeTree<xla::MaybeOwningDeviceMemory> ToDeviceMemoryTree(
-      const std::function<bool(const xla::ShapeIndex&)>& release_checker);
+  // Returns the device memory tree of this allocation. If 'release' is set, the
+  // ownership of the device memory is transferred to the result.
+  xla::ShapeTree<xla::MaybeOwningDeviceMemory> ToDeviceMemoryTree(bool release);
 
   string DebugString() override { return "XLA allocation handle"; }
 
-- 
GitLab


From 47c68e37dee827d9c3fe95f39e5fb49d07a573a5 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 4 Jan 2019 19:47:17 -0800
Subject: [PATCH 0277/1095] Automated rollback of commit
 57d501422c16dd73b391d19bcc1e203942cc6b24

PiperOrigin-RevId: 227949022
---
 tensorflow/contrib/distribute/python/BUILD    |   3 +-
 .../contrib/distribute/python/combinations.py |  15 +--
 .../python/keras_correctness_test.py          | 100 +++---------------
 3 files changed, 19 insertions(+), 99 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index c620a0448e..de6b6f1f84 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -670,12 +670,11 @@ py_library(
 
 cuda_py_test(
     name = "keras_correctness_test",
-    size = "medium",
     srcs = ["keras_correctness_test.py"],
     additional_deps = [
         ":keras_correctness_test_lib",
     ],
-    shard_count = 20,
+    shard_count = 16,
     tags = [
         "multi_and_single_gpu",
         "no_oss",  # TODO(b/117919883): Fix python error.
diff --git a/tensorflow/contrib/distribute/python/combinations.py b/tensorflow/contrib/distribute/python/combinations.py
index 8db0e9c316..f6c4291659 100644
--- a/tensorflow/contrib/distribute/python/combinations.py
+++ b/tensorflow/contrib/distribute/python/combinations.py
@@ -321,12 +321,11 @@ class NamedDistribution(object):
     return self._required_tpu
 
 
-def _get_tpu_strategy_creator(steps_per_run, **kwargs):
+def _get_tpu_strategy_creator(steps_per_run):
   def _create_tpu_strategy():
     resolver = cluster_resolver.TPUClusterResolver("")
     tpu_lib.initialize_tpu_system(resolver)
-    strategy = tpu_lib.TPUStrategy(resolver,
-                                   steps_per_run=steps_per_run, **kwargs)
+    strategy = tpu_lib.TPUStrategy(resolver, steps_per_run=steps_per_run)
     return strategy
   return _create_tpu_strategy
 
@@ -345,15 +344,7 @@ tpu_strategy = NamedDistribution(
 tpu_strategy_one_step = NamedDistribution(
     "TPUOneStep", _get_tpu_strategy_creator(steps_per_run=1),
     required_tpu=True)
-# TODO(b/122327153): Remove below two NamedDistributions.
-tpu_strategy_loop_on_device = NamedDistribution(
-    "TPULoopOnDevice", _get_tpu_strategy_creator(
-        steps_per_run=2, _disable_training_loop_on_host=True),
-    required_tpu=True)
-tpu_strategy_one_step_loop_on_device = NamedDistribution(
-    "TPUOneStepLoopOnDevice", _get_tpu_strategy_creator(
-        steps_per_run=1, _disable_training_loop_on_host=True),
-    required_tpu=True)
+
 mirrored_strategy_with_one_cpu = NamedDistribution(
     "Mirrored1CPU",
     lambda: mirrored_lib.MirroredStrategy(["/cpu:0"]))
diff --git a/tensorflow/contrib/distribute/python/keras_correctness_test.py b/tensorflow/contrib/distribute/python/keras_correctness_test.py
index 1c4519ef71..3abdee2c0e 100644
--- a/tensorflow/contrib/distribute/python/keras_correctness_test.py
+++ b/tensorflow/contrib/distribute/python/keras_correctness_test.py
@@ -62,41 +62,24 @@ def all_strategy_combinations_with_graph_mode():
   return combinations.combine(distribution=all_strategies, mode=['graph'])
 
 
-def tpu_strategies_with_host_training_loop_disabled():
-  strategies = [s for s in all_strategies if not s.required_tpu]
-  strategies.append(combinations.tpu_strategy_loop_on_device)
-  strategies.append(combinations.tpu_strategy_one_step_loop_on_device)
-  return strategies
-
-
 def strategy_and_input_combinations():
   def cnn_model_with_batch_norm(**kwargs):
     return _create_cnn_model(with_batch_norm=True, **kwargs)
 
-  combinations_without_embedding_model = combinations.times(
-      combinations.combine(
-          distribution=tpu_strategies_with_host_training_loop_disabled()),
-      combinations.combine(mode=['graph', 'eager'],
-                           use_numpy=[True, False],
-                           use_validation_data=[True, False]),
-      combinations.combine(model_with_data=
-                           [ModelWithData('lstm',
-                                          _create_lstm_model,
-                                          _lstm_training_data)]))
-
-  combinations_with_embedding_model = combinations.times(
-      combinations.combine(distribution=all_strategies),
-      combinations.combine(mode=['graph', 'eager'],
-                           use_numpy=[True, False],
-                           use_validation_data=[True, False]),
-      combinations.combine(model_with_data=[
-          ModelWithData('dnn', _create_dnn_model, _dnn_training_data),
-          ModelWithData('cnn', _create_cnn_model, _cnn_training_data),
-          ModelWithData('cnn_batch_norm', cnn_model_with_batch_norm,
-                        _cnn_training_data, with_batch_norm=True),]))
-
-  return (combinations_with_embedding_model +
-          combinations_without_embedding_model)
+  return (
+      combinations.times(
+          combinations.combine(distribution=all_strategies),
+          combinations.combine(mode=['graph', 'eager'],
+                               use_numpy=[True, False],
+                               use_validation_data=[True, False]),
+          combinations.combine(model_with_data=[
+              ModelWithData('dnn', _create_dnn_model, _dnn_training_data),
+              ModelWithData('cnn', _create_cnn_model, _cnn_training_data),
+              ModelWithData('cnn_batch_norm',
+                            cnn_model_with_batch_norm,
+                            _cnn_training_data,
+                            with_batch_norm=True),
+          ])))
 
 
 class MaybeDistributionScope(object):
@@ -208,59 +191,6 @@ def _create_cnn_model(initial_weights=None, distribution=None,
   return model
 
 
-def _lstm_training_data(count=_GLOBAL_BATCH_SIZE * _EVAL_STEPS,
-                        min_words=10,
-                        max_words=20,
-                        max_word_id=99,
-                        num_classes=2):
-  distribution = []
-  for _ in range(num_classes):
-    dist = np.abs(np.random.randn(max_word_id))
-    dist /= np.sum(dist)
-    distribution.append(dist)
-
-  features = []
-  labels = []
-  for _ in range(count):
-    label = np.random.randint(0, num_classes, size=1)[0]
-    num_words = np.random.randint(min_words, max_words, size=1)[0]
-    word_ids = np.random.choice(
-        max_word_id, size=num_words, replace=True, p=distribution[label])
-    word_ids = word_ids
-    labels.append(label)
-    features.append(word_ids)
-
-  features = keras.preprocessing.sequence.pad_sequences(
-      features, maxlen=max_words)
-  x_train = np.asarray(features, dtype=np.float32)
-  y_train = np.asarray(labels, dtype=np.int32).reshape((count, 1))
-  x_predict = x_train
-  return x_train, y_train, x_predict
-
-
-def _create_lstm_model(max_words=20,
-                       initial_weights=None,
-                       distribution=None):
-  with MaybeDistributionScope(distribution):
-    word_ids = keras.layers.Input(
-        shape=(max_words,), dtype=np.int32, name='words')
-    word_embed = keras.layers.Embedding(input_dim=100, output_dim=10)(word_ids)
-    lstm_embed = keras.layers.LSTM(
-        units=8, return_sequences=False)(word_embed)
-
-    preds = keras.layers.Dense(2, activation='softmax')(lstm_embed)
-    model = keras.Model(inputs=[word_ids], outputs=[preds])
-
-    if initial_weights:
-      model.set_weights(initial_weights)
-
-    model.compile(
-        optimizer=gradient_descent.GradientDescentOptimizer(learning_rate=0.1),
-        loss='sparse_categorical_crossentropy',
-        metrics=['sparse_categorical_accuracy'])
-  return model
-
-
 def batch_wrapper(dataset, batch_size, distribution, repeat=None):
   if repeat:
     dataset = dataset.repeat(repeat)
@@ -313,7 +243,7 @@ def get_correctness_test_inputs(use_numpy, use_validation_data,
     }
   else:
     if len(x_train) < _GLOBAL_BATCH_SIZE * _EVAL_STEPS:
-      # Currently, we cannot detect the size of a dataset. So, the eval steps is
+      # Currently, we cannot detech the size of a dataset. So, the eval steps is
       # hard coded.
       raise ValueError('x_train must have at least '
                        '_GLOBAL_BATCH_SIZE * _EVAL_STEPS samples')
-- 
GitLab


From 5633ba3fc829a31c297a412929f5953e8c7a1f2f Mon Sep 17 00:00:00 2001
From: Jason Zaman <jason@perfinion.com>
Date: Wed, 2 Jan 2019 18:29:06 +0800
Subject: [PATCH 0278/1095] cloud oauth_client: update for OpenSSL 1.1.0
 compatibility

EVP_MD_CTX_cleanup was removed in OpenSSL 1.1.0. There is a call to
EVP_MD_CTX_destroy right after and _destroy will call _cleanup if
required. EVP_MD_CTX_destroy exists in OpenSSL 1.0, 1.1 and BoringSSL so
removing the call to _cleanup works everywhere.

Signed-off-by: Jason Zaman <jason@perfinion.com>
---
 tensorflow/core/platform/cloud/oauth_client.cc      | 6 +++++-
 tensorflow/core/platform/cloud/oauth_client_test.cc | 1 -
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/platform/cloud/oauth_client.cc b/tensorflow/core/platform/cloud/oauth_client.cc
index 9b85cae9b9..a8657359a3 100644
--- a/tensorflow/core/platform/cloud/oauth_client.cc
+++ b/tensorflow/core/platform/cloud/oauth_client.cc
@@ -95,6 +95,11 @@ Status CreateSignature(RSA* private_key, StringPiece to_sign,
   if (!md) {
     return errors::Internal("Could not get a sha256 encryptor.");
   }
+
+  // EVP_MD_CTX_destroy is renamed to EVP_MD_CTX_free in OpenSSL 1.1.0 but
+  // the old name is still retained as a compatibility macro.
+  // Keep this around until support is dropped for OpenSSL 1.0
+  // https://www.openssl.org/news/cl110.txt
   std::unique_ptr<EVP_MD_CTX, std::function<void(EVP_MD_CTX*)>> md_ctx(
       EVP_MD_CTX_create(), [](EVP_MD_CTX* ptr) { EVP_MD_CTX_destroy(ptr); });
   if (!md_ctx) {
@@ -119,7 +124,6 @@ Status CreateSignature(RSA* private_key, StringPiece to_sign,
   if (EVP_DigestSignFinal(md_ctx.get(), sig.get(), &sig_len) != 1) {
     return errors::Internal("DigestFinal (signature compute) failed.");
   }
-  EVP_MD_CTX_cleanup(md_ctx.get());
   return Base64Encode(StringPiece(reinterpret_cast<char*>(sig.get()), sig_len),
                       signature);
 }
diff --git a/tensorflow/core/platform/cloud/oauth_client_test.cc b/tensorflow/core/platform/cloud/oauth_client_test.cc
index 1cd0641cd3..ce3b9d79c8 100644
--- a/tensorflow/core/platform/cloud/oauth_client_test.cc
+++ b/tensorflow/core/platform/cloud/oauth_client_test.cc
@@ -166,7 +166,6 @@ TEST(OAuthClientTest, GetTokenFromServiceAccountJson) {
                 const_cast<unsigned char*>(
                     reinterpret_cast<const unsigned char*>(signature.data())),
                 signature.size()));
-  EVP_MD_CTX_cleanup(md_ctx);
 
   // Free all the crypto-related resources.
   EVP_PKEY_free(key);
-- 
GitLab


From 13fc6cfefb8204d5144f4c94b4b8cb50f69e8e8c Mon Sep 17 00:00:00 2001
From: Russell Power <power@google.com>
Date: Fri, 4 Jan 2019 21:25:17 -0800
Subject: [PATCH 0279/1095] Grappler micro-optimizations for virtual scheduler.

* Only collect additional memory information when VLOG(1) is active.
* Use const reference to avoid copy.

PiperOrigin-RevId: 227954880
---
 .../core/grappler/costs/virtual_scheduler.cc  | 26 ++++++++++++-------
 .../core/grappler/costs/virtual_scheduler.h   |  3 +++
 .../grappler/costs/virtual_scheduler_test.cc  |  4 ++-
 3 files changed, 23 insertions(+), 10 deletions(-)

diff --git a/tensorflow/core/grappler/costs/virtual_scheduler.cc b/tensorflow/core/grappler/costs/virtual_scheduler.cc
index ae5200b359..7071c679dc 100644
--- a/tensorflow/core/grappler/costs/virtual_scheduler.cc
+++ b/tensorflow/core/grappler/costs/virtual_scheduler.cc
@@ -319,6 +319,7 @@ VirtualScheduler::VirtualScheduler(const GrapplerItem* grappler_item,
       placer_(cluster) {
   graph_costs_.num_ops_total = 0;
   initialized_ = false;
+  track_mem_usage_snapshot_ = VLOG_IS_ON(1);
 }
 
 VirtualScheduler::VirtualScheduler(const bool use_static_shapes,
@@ -331,6 +332,7 @@ VirtualScheduler::VirtualScheduler(const bool use_static_shapes,
       placer_(cluster) {
   graph_costs_.num_ops_total = 0;
   initialized_ = false;
+  track_mem_usage_snapshot_ = VLOG_IS_ON(1);
 }
 
 Status VirtualScheduler::Init(const GrapplerItem* item) {
@@ -562,7 +564,7 @@ void VirtualScheduler::MaybeUpdateInputOutput(const NodeDef* node) {
       inputs.push_back(control_message);
       outputs.push_back(control_message);
     } else {
-      auto output_properties =
+      const auto& output_properties =
           graph_properties_->GetOutputProperties(NodeName(input_source_name));
       // Like with HasInputProperties, if a node does not have output
       // properties, it's likely it was pruned during the shape inference run.
@@ -778,13 +780,16 @@ bool VirtualScheduler::MarkCurrNodeExecuted(const Costs& node_costs) {
   auto& op_cost = FindOrCreateZero(op_name, &op_to_cost_);
   op_cost = CombineCosts(op_cost, node_costs);
 
-  // Also keep track of op counts and costs per op (with their shapes).
-  OpContext op_context = GetCurrNode();
-  string node_description = GetOpDescription(op_context.op_info);
-  op_counts_[node_description] += 1;
-  op_costs_[node_description] =
-      std::make_pair(node_costs.execution_time.asMicroSeconds().count(),
-                     !node_costs.inaccurate);
+  if (VLOG_IS_ON(2)) {
+    // Also keep track of op counts and costs per op (with their shapes).
+    OpContext op_context = GetCurrNode();
+
+    string node_description = GetOpDescription(op_context.op_info);
+    op_counts_[node_description] += 1;
+    op_costs_[node_description] =
+        std::make_pair(node_costs.execution_time.asMicroSeconds().count(),
+                       !node_costs.inaccurate);
+  }
 
   // Update node and device states.
   auto& node_state = node_map_[node];
@@ -868,7 +873,10 @@ bool VirtualScheduler::MarkCurrNodeExecuted(const Costs& node_costs) {
     // check max memory usage.
     if (device.memory_usage > device.max_memory_usage) {
       device.max_memory_usage = device.memory_usage;
-      device.mem_usage_snapshot_at_peak = device.nodes_in_memory;
+
+      if (track_mem_usage_snapshot_) {
+        device.mem_usage_snapshot_at_peak = device.nodes_in_memory;
+      }
     }
   }
 
diff --git a/tensorflow/core/grappler/costs/virtual_scheduler.h b/tensorflow/core/grappler/costs/virtual_scheduler.h
index 6a835f32d1..f500e4ab51 100644
--- a/tensorflow/core/grappler/costs/virtual_scheduler.h
+++ b/tensorflow/core/grappler/costs/virtual_scheduler.h
@@ -305,6 +305,8 @@ class VirtualScheduler {
     return &node_map_;
   }
 
+  void enable_mem_usage_tracking() { track_mem_usage_snapshot_ = true; }
+
  private:
   // Constants.
   const string kAttrInputSrc = "input_source_";
@@ -356,6 +358,7 @@ class VirtualScheduler {
   const GrapplerItem* grappler_item_;  // Not owned.
   bool use_static_shapes_;
   bool initialized_;
+  bool track_mem_usage_snapshot_;
 
   VirtualPlacer placer_;  // owned.
 };
diff --git a/tensorflow/core/grappler/costs/virtual_scheduler_test.cc b/tensorflow/core/grappler/costs/virtual_scheduler_test.cc
index 0a695458e1..c5651715f4 100644
--- a/tensorflow/core/grappler/costs/virtual_scheduler_test.cc
+++ b/tensorflow/core/grappler/costs/virtual_scheduler_test.cc
@@ -31,7 +31,9 @@ namespace grappler {
 class TestVirtualScheduler : public VirtualScheduler {
  public:
   TestVirtualScheduler(const bool use_static_shapes, Cluster* cluster)
-      : VirtualScheduler(use_static_shapes, cluster, &ready_node_manager_) {}
+      : VirtualScheduler(use_static_shapes, cluster, &ready_node_manager_) {
+    enable_mem_usage_tracking();
+  }
 
   FRIEND_TEST(VirtualSchedulerTest, MemoryUsage);
   FRIEND_TEST(VirtualSchedulerTest, ControlDependency);
-- 
GitLab


From 20dc97f0f11534adca40d66e5268eb74e012e254 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 4 Jan 2019 21:37:11 -0800
Subject: [PATCH 0280/1095] export find_all_hinted_output_nodes for Ophint.

PiperOrigin-RevId: 227955652
---
 tensorflow/lite/python/convert_test.py | 23 ++++++++++++++++++
 tensorflow/lite/python/op_hint.py      | 32 +++++++++++++++++++++++++-
 2 files changed, 54 insertions(+), 1 deletion(-)

diff --git a/tensorflow/lite/python/convert_test.py b/tensorflow/lite/python/convert_test.py
index cf49ee2b47..a29d431322 100644
--- a/tensorflow/lite/python/convert_test.py
+++ b/tensorflow/lite/python/convert_test.py
@@ -29,6 +29,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import test_util
 from tensorflow.python.framework.graph_util_impl import _bfs_for_reachable_nodes
 from tensorflow.python.framework.graph_util_impl import _extract_graph_summary
+from tensorflow.python.framework.graph_util_impl import _node_name
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
@@ -389,6 +390,28 @@ class ConvertTestOpHint(test_util.TensorFlowTestCase):
     with self.assertRaises(ValueError):
       convert.convert_dtype_to_tflite_type(dtypes.bool)
 
+  def testFindHintedOutputNodes(self):
+    """Test if all hinted output nodes are correctly found."""
+
+    def _build_ophinted_op(name, input1, input2):
+      custom_op = op_hint.OpHint(name)
+      input1 = custom_op.add_input(input1)
+      input2 = custom_op.add_input(input2)
+      output = math_ops.mul(input1, input2)
+      return custom_op.add_output(output)
+
+    output_1 = _build_ophinted_op("custom_op_1", array_ops.constant([1.]),
+                                  array_ops.constant([2.]))
+    output_2 = _build_ophinted_op("custom_op_2", array_ops.constant([3.]),
+                                  array_ops.constant([4.]))
+    with self.cached_session() as sess:
+      hinted_outputs_nodes = op_hint.find_all_hinted_output_nodes(sess)
+      expected_hinted_output_nodes = [
+          _node_name(output_1.name),
+          _node_name(output_2.name)
+      ]
+      self.assertCountEqual(hinted_outputs_nodes, expected_hinted_output_nodes)
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/lite/python/op_hint.py b/tensorflow/lite/python/op_hint.py
index 8d7f9316bf..6ec050171f 100644
--- a/tensorflow/lite/python/op_hint.py
+++ b/tensorflow/lite/python/op_hint.py
@@ -964,6 +964,35 @@ def _convert_op_hints_to_stubs_helper(
   return curr_graph_def
 
 
+def find_all_hinted_output_nodes(session=None, graph_def=None):
+  """Find all Ophints output nodes in the graph.
+
+  This is used to get all the output nodes those are ophinted, it is important
+  for operation like convert_variables_to_constants keep all ophints structure.
+  Note: only one of session or graph_def should be used, not both.
+
+  Args:
+    session: A TensorFlow session that contains the graph to convert.
+    graph_def: A graph def that we should convert.
+
+  Returns:
+    A list of OpHints output nodes.
+  Raises:
+    ValueError: If both session and graph_def are provided.
+  """
+  if session is not None and graph_def is not None:
+    raise ValueError("Provide only one of session and graph_def.")
+  hinted_outputs_nodes = []
+  if session is not None:
+    hints = _find_all_hints_in_graph_def(session.graph_def)
+  elif graph_def is not None:
+    hints = _find_all_hints_in_graph_def(graph_def)
+  for hint in _six.itervalues(hints):
+    _, ouput_nodes = hint.flattened_inputs_and_outputs()
+    hinted_outputs_nodes.extend(ouput_nodes)
+  return hinted_outputs_nodes
+
+
 def convert_op_hints_to_stubs(session=None,
                               graph_def=None,
                               write_callback=lambda graph_def, comments: None):
@@ -996,6 +1025,7 @@ def convert_op_hints_to_stubs(session=None,
 
 
 _allowed_symbols = [
-    "OpHint", "convert_op_hints_to_stubs", "convert_op_hints_to_stubs_new"
+    "OpHint", "convert_op_hints_to_stubs", "convert_op_hints_to_stubs_new",
+    "find_all_hinted_output_nodes"
 ]
 remove_undocumented(__name__, _allowed_symbols)
-- 
GitLab


From 058bb7c3512455d2d83d2094a6f6e60b0ca6ee05 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 4 Jan 2019 22:49:14 -0800
Subject: [PATCH 0281/1095] Rollback flaky test

PiperOrigin-RevId: 227959516
---
 tensorflow/core/BUILD                         |  1 -
 .../core/util/presized_cuckoo_map_test.cc     | 48 ++-----------------
 2 files changed, 5 insertions(+), 44 deletions(-)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 460353df41..7c9decbd09 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -3813,7 +3813,6 @@ tf_cc_tests(
         "//tensorflow/core/kernels:ops_util",
         "//third_party/eigen3",
         "@com_google_absl//absl/base",
-        "@com_google_absl//absl/time",
     ],
 )
 
diff --git a/tensorflow/core/util/presized_cuckoo_map_test.cc b/tensorflow/core/util/presized_cuckoo_map_test.cc
index 8dbb2ec065..f2c7904b00 100644
--- a/tensorflow/core/util/presized_cuckoo_map_test.cc
+++ b/tensorflow/core/util/presized_cuckoo_map_test.cc
@@ -15,7 +15,6 @@ limitations under the License.
 
 #include <array>
 
-#include "absl/time/clock.h"
 #include "tensorflow/core/platform/env.h"
 #include "tensorflow/core/platform/fingerprint.h"
 #include "tensorflow/core/platform/test.h"
@@ -53,48 +52,11 @@ TEST(PresizedCuckooMapTest, Basic) {
 }
 
 TEST(PresizedCuckooMapTest, Prefetch) {
-  {
-    PresizedCuckooMap<int64> pscm(2);
-    EXPECT_TRUE(pscm.InsertUnique(1, 2));
-    // Works for both present and absent keys.
-    pscm.PrefetchKey(1);
-    pscm.PrefetchKey(2);
-  }
-
-  // Do not run in debug mode, when prefetch is not implemented, or when
-  // sanitizers are enabled.
-#if defined(NDEBUG) && defined(__GNUC__) && !defined(ADDRESS_SANITIZER) && \
-    !defined(MEMORY_SANITIZER) && !defined(THREAD_SANITIZER) &&            \
-    !defined(UNDEFINED_BEHAVIOR_SANITIZER)
-  const auto now = [] { return absl::Now(); };
-
-  // Make size enough to not fit in L2 cache (16.7 Mb)
-  static constexpr int size = 1 << 22;
-  PresizedCuckooMap<int64> pscm(size);
-  for (int i = 0; i < size; ++i) {
-    pscm.InsertUnique(i, i);
-  }
-
-  absl::Duration no_prefetch, prefetch;
-  int64 out;
-  for (int iter = 0; iter < 10; ++iter) {
-    auto time = now();
-    for (int i = 0; i < size; ++i) {
-      testing::DoNotOptimize(pscm.Find(i, &out));
-    }
-    no_prefetch += now() - time;
-
-    time = now();
-    for (int i = 0; i < size; ++i) {
-      pscm.PrefetchKey(i + 20);
-      testing::DoNotOptimize(pscm.Find(i, &out));
-    }
-    prefetch += now() - time;
-  }
-
-  // no_prefetch is at least 30% slower.
-  EXPECT_GE(1.0 * no_prefetch / prefetch, 1.3);
-#endif
+  PresizedCuckooMap<int64> pscm(2);
+  EXPECT_TRUE(pscm.InsertUnique(1, 2));
+  // Works for both present and absent keys.
+  pscm.PrefetchKey(1);
+  pscm.PrefetchKey(2);
 }
 
 TEST(PresizedCuckooMapTest, TooManyItems) {
-- 
GitLab


From 9e8962b78b3c514cb1163c01fa7ab767592f952f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sat, 5 Jan 2019 01:03:08 -0800
Subject: [PATCH 0282/1095] compat: Update forward compatibility horizon to
 2019-01-05

PiperOrigin-RevId: 227969226
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 66c4fa4893..3b808d71a5 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 4)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 5)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 186f573d7067b1df32e3bf2ffb7d34cffcdb4c64 Mon Sep 17 00:00:00 2001
From: Dheeraj Rajaram Reddy <dheeraj98reddy@gmail.com>
Date: Sat, 5 Jan 2019 17:31:21 +0530
Subject: [PATCH 0283/1095] Minor bug fixes

`take_while` ShortCircuit tests are parameterized
Fix errors in `BUILD` files using `buildifier`
LoopIteratorPredicate takes `vector<Tensor>&`
---
 tensorflow/core/kernels/data/experimental/BUILD |  2 +-
 .../data/experimental/take_while_dataset_op.cc  |  8 +++++---
 .../python/data/experimental/kernel_tests/BUILD |  4 ++--
 .../kernel_tests/take_while_test.py             | 17 +++++++++++------
 tensorflow/python/data/experimental/ops/BUILD   |  4 ++--
 5 files changed, 21 insertions(+), 14 deletions(-)

diff --git a/tensorflow/core/kernels/data/experimental/BUILD b/tensorflow/core/kernels/data/experimental/BUILD
index 53d1f02a3a..d71b473b19 100644
--- a/tensorflow/core/kernels/data/experimental/BUILD
+++ b/tensorflow/core/kernels/data/experimental/BUILD
@@ -303,7 +303,7 @@ tf_kernel_library(
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core/kernels/data:captured_function",
-        "//tensorflow/core/kernels/data:dataset_utils"
+        "//tensorflow/core/kernels/data:dataset_utils",
     ],
 )
 
diff --git a/tensorflow/core/kernels/data/experimental/take_while_dataset_op.cc b/tensorflow/core/kernels/data/experimental/take_while_dataset_op.cc
index b670e4efaa..3a6f70e504 100644
--- a/tensorflow/core/kernels/data/experimental/take_while_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/take_while_dataset_op.cc
@@ -21,6 +21,7 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/kernels/data/captured_function.h"
 #include "tensorflow/core/kernels/data/dataset_utils.h"
+#include "tensorflow/core/util/ptr_util.h"
 
 namespace tensorflow {
 namespace data {
@@ -33,7 +34,7 @@ class TakeWhileDatasetOp : public UnaryDatasetOpKernel {
  public:
   using LoopIteratorPredicate =
       std::function<Status(IteratorContext*, InstantiatedCapturedFunction*,
-                           std::vector<Tensor>, bool*)>;
+                           std::vector<Tensor>&, bool*)>;
 
   explicit TakeWhileDatasetOp(OpKernelConstruction* ctx)
       : UnaryDatasetOpKernel(ctx) {
@@ -106,8 +107,9 @@ class TakeWhileDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(new Iterator(
-          {this, strings::StrCat(prefix, "::TakeWhile")}, loop_pred_));
+      return MakeUnique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::TakeWhile")},
+          loop_pred_);
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/python/data/experimental/kernel_tests/BUILD b/tensorflow/python/data/experimental/kernel_tests/BUILD
index b4196685e2..e832231912 100644
--- a/tensorflow/python/data/experimental/kernel_tests/BUILD
+++ b/tensorflow/python/data/experimental/kernel_tests/BUILD
@@ -650,13 +650,13 @@ py_test(
         "//tensorflow/python:dtypes",
         "//tensorflow/python:errors",
         "//tensorflow/python:framework_test_lib",
-        "//tensorflow/python:script_ops",
-        "//tensorflow/python:sparse_tensor",
+        "//tensorflow/python:math_ops",
         "//tensorflow/python/data/experimental/ops:take_while_ops",
         "//tensorflow/python/data/kernel_tests:test_base",
         "//tensorflow/python/data/ops:dataset_ops",
         "//tensorflow/python/eager:context",
         "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/take_while_test.py b/tensorflow/python/data/experimental/kernel_tests/take_while_test.py
index 3faee0639f..7392f5420a 100644
--- a/tensorflow/python/data/experimental/kernel_tests/take_while_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/take_while_test.py
@@ -83,19 +83,24 @@ class TakeWhileTest(test_base.DatasetTestBase, parameterized.TestCase):
     with self.assertRaises(errors.OutOfRangeError):
       self.assertEqual(b"test", self.evaluate(next_element()))
 
-  def testTakewhileDatasetShortCircuit(self):
+  @parameterized.parameters(
+      (5, 3),
+      (10, 0),
+      (100, 5),
+      (8, 7))
+  def testTakewhileDatasetShortCircuit(self, size, index):
     def _predicate_func(data_elem):
       return data_elem
 
-    boolean_array = [True, True, True, True, False]
+    boolean_array = [True]*size
+    boolean_array[index] = False
     dataset = dataset_ops.Dataset.from_tensor_slices(boolean_array).apply(
         take_while_ops.take_while(_predicate_func))
 
     next_element = self.getNext(dataset)
-    self.assertEqual(True, self.evaluate(next_element()))
-    self.assertEqual(True, self.evaluate(next_element()))
-    self.assertEqual(True, self.evaluate(next_element()))
-    self.assertEqual(True, self.evaluate(next_element()))
+
+    for _ in range(index):
+      self.assertTrue(self.evaluate(next_element()))
 
     with self.assertRaises(errors.OutOfRangeError):
       self.evaluate(next_element())
diff --git a/tensorflow/python/data/experimental/ops/BUILD b/tensorflow/python/data/experimental/ops/BUILD
index 95e9509678..56bf59344f 100644
--- a/tensorflow/python/data/experimental/ops/BUILD
+++ b/tensorflow/python/data/experimental/ops/BUILD
@@ -359,9 +359,9 @@ py_library(
     srcs = ["take_while_ops.py"],
     srcs_version = "PY2AND3",
     deps = [
+        "//tensorflow/python:dtypes",
         "//tensorflow/python:experimental_dataset_ops_gen",
         "//tensorflow/python:framework_ops",
-        "//tensorflow/python:dtypes",
         "//tensorflow/python:function",
         "//tensorflow/python/data/ops:dataset_ops",
     ],
@@ -467,7 +467,7 @@ py_library(
         ":shuffle_ops",
         ":sleep",
         ":stats_ops",
-        "take_while_ops",
+        ":take_while_ops",
         ":threadpool",
         ":unique",
         ":writers",
-- 
GitLab


From f69eca8cef2e8536364bb5a6e5f966a5e071812f Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Sat, 5 Jan 2019 13:29:23 -0800
Subject: [PATCH 0284/1095] Enable test that works with control flow v2 now.

PiperOrigin-RevId: 228006449
---
 tensorflow/python/kernel_tests/tensor_array_ops_test.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tensorflow/python/kernel_tests/tensor_array_ops_test.py b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
index 303e9a006a..3f1bd60d1a 100644
--- a/tensorflow/python/kernel_tests/tensor_array_ops_test.py
+++ b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
@@ -1005,7 +1005,6 @@ class TensorArrayTest(test.TestCase):
     self._testWhileLoopWritePackGradients(
         dynamic_size=True, dtype=dtypes.float32)
 
-  @test_util.disable_control_flow_v2("b/119323158")
   def testGradSerialTwoLoops(self):
     with self.session(use_gpu=True):
       def loop(x):
-- 
GitLab


From 933109e06a36d1e85daa59214ee37f0e3d170cc8 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sun, 6 Jan 2019 01:03:37 -0800
Subject: [PATCH 0285/1095] compat: Update forward compatibility horizon to
 2019-01-06

PiperOrigin-RevId: 228038143
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 3b808d71a5..b779c80aa7 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 5)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 6)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From a0804de99207eb89b38a315b7baa0eb2976931c7 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sun, 6 Jan 2019 03:06:52 -0800
Subject: [PATCH 0286/1095] Internal change

PiperOrigin-RevId: 228045421
---
 tensorflow/contrib/BUILD       | 1 -
 tensorflow/contrib/__init__.py | 1 -
 2 files changed, 2 deletions(-)

diff --git a/tensorflow/contrib/BUILD b/tensorflow/contrib/BUILD
index 832db0f4ab..307bb6eca3 100644
--- a/tensorflow/contrib/BUILD
+++ b/tensorflow/contrib/BUILD
@@ -63,7 +63,6 @@ py_library(
         "//tensorflow/contrib/libsvm",
         "//tensorflow/contrib/linear_optimizer:sdca_estimator_py",
         "//tensorflow/contrib/linear_optimizer:sdca_ops_py",
-        "//tensorflow/contrib/lite/python:lite",
         "//tensorflow/contrib/lookup:lookup_py",
         "//tensorflow/contrib/losses:losses_py",
         "//tensorflow/contrib/losses:metric_learning_py",
diff --git a/tensorflow/contrib/__init__.py b/tensorflow/contrib/__init__.py
index 4f1a2a5693..af59686120 100644
--- a/tensorflow/contrib/__init__.py
+++ b/tensorflow/contrib/__init__.py
@@ -91,7 +91,6 @@ from tensorflow.contrib import tpu
 from tensorflow.contrib import training
 from tensorflow.contrib import util
 from tensorflow.contrib.eager.python import tfe as eager
-from tensorflow.contrib.lite.python import lite
 from tensorflow.contrib.optimizer_v2 import optimizer_v2_symbols as optimizer_v2
 from tensorflow.contrib.receptive_field import receptive_field_api as receptive_field
 from tensorflow.contrib.recurrent.python import recurrent_api as recurrent
-- 
GitLab


From c67e5c39503c4ca2f1b383acfe55cd4fd8fba37d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sun, 6 Jan 2019 03:15:34 -0800
Subject: [PATCH 0287/1095] Extend config generation to include a default CPU
 configuration and provide a platform definition for this configuration.

We create a new platform instead of changing the old one so we can transition
without the need to change all users atomically.

PiperOrigin-RevId: 228045771
---
 third_party/toolchains/BUILD                  | 28 ++++++++--
 .../toolchains/preconfig/generate/BUILD       |  6 ++
 .../preconfig/generate/containers.bzl         |  1 +
 .../preconfig/generate/generate.bzl           | 55 ++++++++++++-------
 .../toolchains/preconfig/generate/generate.sh | 31 ++++++++---
 .../preconfig/generate/workspace.bzl          |  7 +++
 6 files changed, 96 insertions(+), 32 deletions(-)

diff --git a/third_party/toolchains/BUILD b/third_party/toolchains/BUILD
index f2248341c4..6ed6e5c367 100644
--- a/third_party/toolchains/BUILD
+++ b/third_party/toolchains/BUILD
@@ -4,10 +4,9 @@ package(default_visibility = ["//visibility:public"])
 
 load("//third_party/toolchains/preconfig/generate:containers.bzl", "container_digests")
 
-# Platform for use with remote execution with
-# custom container based off RBE Ubuntu16_04
-# http://gcr.io/cloud-marketplace/google/rbe-ubuntu16-04
-# Built with //tensorflow/tools/ci_build/Dockerfile.rbe.cpu
+# TODO(b/122347293): This is the RBE config based on the CPU configuration / image provided
+# in the asci-toolchain setup. Delete this once we switched CPU remote builds to the
+# new platform below.
 platform(
     name = "rbe_ubuntu16_04-tf",
     constraint_values = [
@@ -23,6 +22,26 @@ platform(
         }""",
 )
 
+# Remote build platforms.
+# Each of the platform rules here provide a platform definition that is bound to a docker image.
+# The result of the skylark configuration is checked into
+# //tensorflow/third_party/toolchains/preconfig.
+
+# Built with //tensorflow/tools/ci_build/Dockerfile.rbe.cpu.
+platform(
+    name = "rbe_ubuntu16.04",
+    constraint_values = [
+        "@bazel_tools//platforms:x86_64",
+        "@bazel_tools//platforms:linux",
+    ],
+    remote_execution_properties = """
+        properties: {
+            name: "container-image"
+            value:"docker://gcr.io/tensorflow-testing/nosla-ubuntu16.04@%s"
+        }""" % container_digests["ubuntu16.04"],
+)
+
+# Built with //tensorflow/tools/ci_build/Dockerfile.rbe.cuda9.0-cudnn7-ubuntu14.04.
 platform(
     name = "rbe_cuda9.0-cudnn7-ubuntu14.04",
     constraint_values = [
@@ -36,6 +55,7 @@ platform(
         }""" % container_digests["cuda9.0-cudnn7-ubuntu14.04"],
 )
 
+# Built with //tensorflow/tools/ci_build/Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04.
 platform(
     name = "rbe_cuda10.0-cudnn7-ubuntu14.04",
     constraint_values = [
diff --git a/third_party/toolchains/preconfig/generate/BUILD b/third_party/toolchains/preconfig/generate/BUILD
index b4c98dc94d..ad79255251 100644
--- a/third_party/toolchains/preconfig/generate/BUILD
+++ b/third_party/toolchains/preconfig/generate/BUILD
@@ -2,6 +2,12 @@ licenses(["restricted"])
 
 load(":generate.bzl", "tensorflow_rbe_config")
 
+tensorflow_rbe_config(
+    name = "ubuntu16.04-py3-clang",
+    compiler = "clang",
+    python_version = "3",
+)
+
 tensorflow_rbe_config(
     name = "ubuntu14.04-py3-gcc-cuda9.0-cudnn7-tensorrt5",
     compiler = "gcc",
diff --git a/third_party/toolchains/preconfig/generate/containers.bzl b/third_party/toolchains/preconfig/generate/containers.bzl
index 67e43485ed..428208523b 100644
--- a/third_party/toolchains/preconfig/generate/containers.bzl
+++ b/third_party/toolchains/preconfig/generate/containers.bzl
@@ -1,4 +1,5 @@
 container_digests = {
+    "ubuntu16.04": "sha256:d0d98c53111c3ec071aa81632a2b0d6f210e5c2411c5172e31f99002125ec4de",
     "cuda9.0-cudnn7-ubuntu14.04": "sha256:006a76ee1838122ff7f21ebac85f24c1ef350d4dd79b3ceff0e4fe649ed90d33",
     "cuda10.0-cudnn7-ubuntu14.04": "sha256:e36f05f1ff39e39ddf07122e37f2b1895948bb6f7acc3db37a3c496be5e66228",
 }
diff --git a/third_party/toolchains/preconfig/generate/generate.bzl b/third_party/toolchains/preconfig/generate/generate.bzl
index 75deea41b8..fc485d43d2 100644
--- a/third_party/toolchains/preconfig/generate/generate.bzl
+++ b/third_party/toolchains/preconfig/generate/generate.bzl
@@ -3,30 +3,38 @@ load(
     "docker_toolchain_autoconfig",
 )
 
-def _tensorflow_rbe_config(name, cuda_version, cudnn_version, python_version, compiler, tensorrt_version):
-    docker_toolchain_autoconfig(
-        name = name,
-        base = "@cuda%s-cudnn%s-ubuntu14.04//image" % (cuda_version, cudnn_version),
-        bazel_version = "0.19.2",
+def _tensorflow_rbe_config(name, compiler, python_version, cuda_version = None, cudnn_version = None, tensorrt_version = None):
+    base = "@ubuntu16.04//image"
+    config_repos = [
+        "local_config_python",
+        "local_config_cc",
+    ]
+    env = {
+        "ABI_VERSION": "gcc",
+        "ABI_LIBC_VERSION": "glibc_2.19",
+        "BAZEL_COMPILER": compiler,
+        "BAZEL_HOST_SYSTEM": "i686-unknown-linux-gnu",
+        "BAZEL_TARGET_LIBC": "glibc_2.19",
+        "BAZEL_TARGET_CPU": "k8",
+        "BAZEL_TARGET_SYSTEM": "x86_64-unknown-linux-gnu",
+        "CC_TOOLCHAIN_NAME": "linux_gnu_x86",
+        "CC": compiler,
+        "PYTHON_BIN_PATH": "/usr/bin/python%s" % python_version,
+        "CLEAR_CACHE": "1",
+    }
+
+    if cuda_version != None:
+        base = "@cuda%s-cudnn%s-ubuntu14.04//image" % (cuda_version, cudnn_version)
+        # The cuda toolchain currently contains its own C++ toolchain definition,
+        # so we do not fetch local_config_cc.
         config_repos = [
-            "local_config_cuda",
             "local_config_python",
+            "local_config_cuda",
             "local_config_tensorrt",
-        ],
-        env = {
-            "ABI_VERSION": "gcc",
-            "ABI_LIBC_VERSION": "glibc_2.19",
-            "BAZEL_COMPILER": compiler,
-            "BAZEL_HOST_SYSTEM": "i686-unknown-linux-gnu",
-            "BAZEL_TARGET_LIBC": "glibc_2.19",
-            "BAZEL_TARGET_CPU": "k8",
-            "BAZEL_TARGET_SYSTEM": "x86_64-unknown-linux-gnu",
-            "CC_TOOLCHAIN_NAME": "linux_gnu_x86",
-            "CC": compiler,
-            "PYTHON_BIN_PATH": "/usr/bin/python%s" % python_version,
+        ]
+        env.update({
             "TF_NEED_CUDA": "1",
             "TF_CUDA_CLANG": "1" if compiler == "clang" else "0",
-            "CLEAR_CACHE": "1",
             "TF_CUDA_COMPUTE_CAPABILITIES": "3.0",
             "TF_ENABLE_XLA": "1",
             "TF_CUDNN_VERSION": cudnn_version,
@@ -35,7 +43,14 @@ def _tensorflow_rbe_config(name, cuda_version, cudnn_version, python_version, co
             "TF_NEED_TENSORRT" : "1",
             "TF_TENSORRT_VERSION": tensorrt_version,
             "TENSORRT_INSTALL_PATH": "/usr/lib/x86_64-linux-gnu",
-        },
+        })
+
+    docker_toolchain_autoconfig(
+        name = name,
+        base = base,
+        bazel_version = "0.21.0",
+        config_repos = config_repos,
+        env = env,
         mount_project = "$(mount_project)",
         tags = ["manual"],
         incompatible_changes_off = True,
diff --git a/third_party/toolchains/preconfig/generate/generate.sh b/third_party/toolchains/preconfig/generate/generate.sh
index 8e3a1e6ada..c05a4de6fb 100755
--- a/third_party/toolchains/preconfig/generate/generate.sh
+++ b/third_party/toolchains/preconfig/generate/generate.sh
@@ -37,8 +37,16 @@ TENSORRT_VERSION="${PLATFORM[5]}"
 
 # TODO(klimek): Put this into the name.
 
-if [[ "${COMPILER}" == "gcc" ]]; then
-  COMPILER="gcc-nvcc-${CUDA_VERSION}"
+if [[ -n "${CUDA_VERSION}" ]]; then
+  if [[ "${COMPILER}" == "gcc" ]]; then
+    COMPILER="gcc-nvcc-${CUDA_VERSION}"
+  fi
+  # Currently we create a special toolchain for clang when compiling with
+  # cuda enabled. We can get rid of this once the default toolchain bazel
+  # provides supports cuda.
+  if [[ "${COMPILER}" == "clang" ]]; then
+    COMPILER="cuda-clang"
+  fi
 fi
 
 echo "OS: ${OS}"
@@ -52,6 +60,8 @@ bazel build --define=mount_project="${PWD}" "${PKG}/generate:${TARGET}"
 cd "${TEMPDIR}"
 tar xvf "${ROOT}/bazel-bin/${PKG}/generate/${TARGET}_outputs.tar"
 
+# TODO(klimek): The skylark config rules should copy the files instead of
+# creating aliases.
 # Other than in @local_config_tensorrt, the header files in the remote config
 # repo are not relative to the repository root. Add a dummy include_prefix to
 # make them available as virtual includes.
@@ -74,14 +84,19 @@ mkdir "${OS}"
 # Python:
 mv local_config_python "${OS}/${PY_VERSION}"
 
-# Compiler:
-mv local_config_cuda/crosstool "${OS}/${COMPILER}"
+if [[ -n "${CUDA_VERSION}" ]]; then
+  # Compiler:
+  mv local_config_cuda/crosstool "${OS}/${COMPILER}"
 
-# CUDA:
-mv local_config_cuda "${OS}/${CUDA_VERSION}-${CUDNN_VERSION}"
+  # CUDA:
+  mv local_config_cuda "${OS}/${CUDA_VERSION}-${CUDNN_VERSION}"
 
-# TensorRT:
-mv local_config_tensorrt "${OS}/${TENSORRT_VERSION}"
+  # TensorRT:
+  mv local_config_tensorrt "${OS}/${TENSORRT_VERSION}"
+else
+  # Compiler:
+  mv local_config_cc "${OS}/${COMPILER}"
+fi
 
 # Cleanup for copybara.
 find "${OS}" -name 'BUILD' -o -name '*.bzl' |xargs buildifier
diff --git a/third_party/toolchains/preconfig/generate/workspace.bzl b/third_party/toolchains/preconfig/generate/workspace.bzl
index eb74022c24..0495173786 100644
--- a/third_party/toolchains/preconfig/generate/workspace.bzl
+++ b/third_party/toolchains/preconfig/generate/workspace.bzl
@@ -8,6 +8,13 @@ load(":containers.bzl", "container_digests")
 def _remote_config_workspace():
     container_repositories()
 
+    container_pull(
+        name = "ubuntu16.04",
+        registry = "gcr.io",
+        repository = "tensorflow-testing/nosla-ubuntu16.04",
+        digest = container_digests["ubuntu16.04"],
+    )
+
     container_pull(
         name = "cuda9.0-cudnn7-ubuntu14.04",
         registry = "gcr.io",
-- 
GitLab


From 92985e67564c119203f0aaca94b5a334d87c35e4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sun, 6 Jan 2019 04:03:21 -0800
Subject: [PATCH 0288/1095] Add clang and python3 configurations for ubuntu
 16.04.

PiperOrigin-RevId: 228048061
---
 tensorflow/opensource_only.files              |    3 +
 .../preconfig/ubuntu16.04/clang/BUILD         |  111 ++
 .../preconfig/ubuntu16.04/clang/CROSSTOOL     | 1209 +++++++++++++++++
 .../preconfig/ubuntu16.04/clang/WORKSPACE     |    2 +
 .../preconfig/ubuntu16.04/clang/cc_wrapper.sh |   25 +
 .../ubuntu16.04/clang/dummy_toolchain.bzl     |   23 +
 .../ubuntu16.04/clang/tools/cpp/empty.cc      |    1 +
 .../preconfig/ubuntu16.04/py3/BUILD           |  205 +++
 .../preconfig/ubuntu16.04/py3/WORKSPACE       |    2 +
 9 files changed, 1581 insertions(+)
 create mode 100755 third_party/toolchains/preconfig/ubuntu16.04/clang/BUILD
 create mode 100755 third_party/toolchains/preconfig/ubuntu16.04/clang/CROSSTOOL
 create mode 100644 third_party/toolchains/preconfig/ubuntu16.04/clang/WORKSPACE
 create mode 100755 third_party/toolchains/preconfig/ubuntu16.04/clang/cc_wrapper.sh
 create mode 100755 third_party/toolchains/preconfig/ubuntu16.04/clang/dummy_toolchain.bzl
 create mode 100755 third_party/toolchains/preconfig/ubuntu16.04/clang/tools/cpp/empty.cc
 create mode 100755 third_party/toolchains/preconfig/ubuntu16.04/py3/BUILD
 create mode 100644 third_party/toolchains/preconfig/ubuntu16.04/py3/WORKSPACE

diff --git a/tensorflow/opensource_only.files b/tensorflow/opensource_only.files
index 1054c285d6..e00d063cfe 100644
--- a/tensorflow/opensource_only.files
+++ b/tensorflow/opensource_only.files
@@ -53,6 +53,9 @@ tensorflow/third_party/toolchains/preconfig/generate/containers.bzl
 tensorflow/third_party/toolchains/preconfig/generate/generate.bzl
 tensorflow/third_party/toolchains/preconfig/generate/archives.bzl
 tensorflow/third_party/toolchains/preconfig/generate/BUILD
+tensorflow/third_party/toolchains/preconfig/ubuntu16.04/clang/BUILD
+tensorflow/third_party/toolchains/preconfig/ubuntu16.04/clang/dummy_toolchain.bzl
+tensorflow/third_party/toolchains/preconfig/ubuntu16.04/py3/BUILD
 tensorflow/third_party/toolchains/preconfig/win_1803/bazel_018/BUILD
 tensorflow/third_party/toolchains/preconfig/win_1803/bazel_018/dummy_toolchain.bzl
 tensorflow/third_party/toolchains/preconfig/win_1803/py36/BUILD
diff --git a/third_party/toolchains/preconfig/ubuntu16.04/clang/BUILD b/third_party/toolchains/preconfig/ubuntu16.04/clang/BUILD
new file mode 100755
index 0000000000..5a0c52f66a
--- /dev/null
+++ b/third_party/toolchains/preconfig/ubuntu16.04/clang/BUILD
@@ -0,0 +1,111 @@
+# Copyright 2016 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This becomes the BUILD file for @local_config_cc// under non-FreeBSD unixes.
+
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])  # Apache 2.0
+
+cc_library(
+    name = "malloc",
+)
+
+cc_library(
+    name = "stl",
+)
+
+filegroup(
+    name = "empty",
+    srcs = [],
+)
+
+filegroup(
+    name = "cc_wrapper",
+    srcs = ["cc_wrapper.sh"],
+)
+
+filegroup(
+    name = "compiler_deps",
+    srcs = glob(["extra_tools/**"]) + [":empty"],
+)
+
+# This is the entry point for --crosstool_top.  Toolchains are found
+# by lopping off the name of --crosstool_top and searching for
+# the "${CPU}" entry in the toolchains attribute.
+cc_toolchain_suite(
+    name = "toolchain",
+    toolchains = {
+        "k8|clang": ":cc-compiler-k8",
+        "k8": ":cc-compiler-k8",
+        "armeabi-v7a|compiler": ":cc-compiler-armeabi-v7a",
+        "armeabi-v7a": ":cc-compiler-armeabi-v7a",
+    },
+)
+
+cc_toolchain(
+    name = "cc-compiler-k8",
+    all_files = ":compiler_deps",
+    compiler_files = ":compiler_deps",
+    cpu = "k8",
+    dwp_files = ":empty",
+    dynamic_runtime_libs = [":empty"],
+    linker_files = ":compiler_deps",
+    objcopy_files = ":empty",
+    static_runtime_libs = [":empty"],
+    strip_files = ":empty",
+    supports_param_files = 1,
+    toolchain_identifier = "linux_gnu_x86",
+)
+
+toolchain(
+    name = "cc-toolchain-k8",
+    exec_compatible_with = [
+        # TODO(katre): add autodiscovered constraints for host CPU and OS.
+    ],
+    target_compatible_with = [
+        # TODO(katre): add autodiscovered constraints for host CPU and OS.
+    ],
+    toolchain = ":cc-compiler-k8",
+    toolchain_type = "@bazel_tools//tools/cpp:toolchain_type",
+)
+
+# Android tooling requires a default toolchain for the armeabi-v7a cpu.
+cc_toolchain(
+    name = "cc-compiler-armeabi-v7a",
+    all_files = ":empty",
+    compiler_files = ":empty",
+    cpu = "local",
+    dwp_files = ":empty",
+    dynamic_runtime_libs = [":empty"],
+    linker_files = ":empty",
+    objcopy_files = ":empty",
+    static_runtime_libs = [":empty"],
+    strip_files = ":empty",
+    supports_param_files = 1,
+    toolchain_identifier = "stub_armeabi-v7a",
+)
+
+toolchain(
+    name = "cc-toolchain-armeabi-v7a",
+    exec_compatible_with = [
+        # TODO(katre): add autodiscovered constraints for host CPU and OS.
+    ],
+    target_compatible_with = [
+        "@bazel_tools//platforms:arm",
+        "@bazel_tools//platforms:android",
+    ],
+    toolchain = ":cc-compiler-armabi-v7a",
+    toolchain_type = "@bazel_tools//tools/cpp:toolchain_type",
+)
diff --git a/third_party/toolchains/preconfig/ubuntu16.04/clang/CROSSTOOL b/third_party/toolchains/preconfig/ubuntu16.04/clang/CROSSTOOL
new file mode 100755
index 0000000000..48f82eb35d
--- /dev/null
+++ b/third_party/toolchains/preconfig/ubuntu16.04/clang/CROSSTOOL
@@ -0,0 +1,1209 @@
+# Copyright 2016 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+major_version: "local"
+minor_version: ""
+
+# Android tooling requires a default toolchain for the armeabi-v7a cpu.
+toolchain {
+  abi_version: "armeabi-v7a"
+  abi_libc_version: "armeabi-v7a"
+  builtin_sysroot: ""
+  compiler: "compiler"
+  host_system_name: "armeabi-v7a"
+  needsPic: true
+  supports_gold_linker: false
+  supports_incremental_linker: false
+  supports_fission: false
+  supports_interface_shared_objects: false
+  supports_normalizing_ar: false
+  supports_start_end_lib: false
+  target_libc: "armeabi-v7a"
+  target_cpu: "armeabi-v7a"
+  target_system_name: "armeabi-v7a"
+  toolchain_identifier: "stub_armeabi-v7a"
+
+  tool_path { name: "ar" path: "/bin/false" }
+  tool_path { name: "compat-ld" path: "/bin/false" }
+  tool_path { name: "cpp" path: "/bin/false" }
+  tool_path { name: "dwp" path: "/bin/false" }
+  tool_path { name: "gcc" path: "/bin/false" }
+  tool_path { name: "gcov" path: "/bin/false" }
+  tool_path { name: "ld" path: "/bin/false" }
+
+  tool_path { name: "nm" path: "/bin/false" }
+  tool_path { name: "objcopy" path: "/bin/false" }
+  tool_path { name: "objdump" path: "/bin/false" }
+  tool_path { name: "strip" path: "/bin/false" }
+  linking_mode_flags { mode: DYNAMIC }
+}
+
+toolchain {
+  toolchain_identifier: "linux_gnu_x86"
+  abi_version: "gcc"
+  abi_libc_version: "glibc_2.19"
+  builtin_sysroot: ""
+  compiler: "clang"
+  host_system_name: "i686-unknown-linux-gnu"
+  needsPic: true
+  supports_gold_linker: true
+  supports_incremental_linker: false
+  supports_fission: false
+  supports_interface_shared_objects: false
+  supports_normalizing_ar: false
+  supports_start_end_lib: true
+  target_libc: "glibc_2.19"
+  target_cpu: "k8"
+  target_system_name: "x86_64-unknown-linux-gnu"
+  cxx_flag: "-std=c++0x"
+  linker_flag: "-fuse-ld=gold"
+  linker_flag: "-Wl,-no-as-needed"
+  linker_flag: "-Wl,-z,relro,-z,now"
+  linker_flag: "-B/usr/local/bin"
+  linker_flag: "-lstdc++"
+  linker_flag: "-lm"
+  cxx_builtin_include_directory: "/usr/local/include"
+  cxx_builtin_include_directory: "/usr/local/lib/clang/7.0.0/include"
+  cxx_builtin_include_directory: "/usr/include/x86_64-linux-gnu"
+  cxx_builtin_include_directory: "/usr/include"
+  cxx_builtin_include_directory: "/usr/include/c++/4.9"
+  cxx_builtin_include_directory: "/usr/include/x86_64-linux-gnu/c++/4.9"
+  cxx_builtin_include_directory: "/usr/include/c++/4.9/backward"
+  objcopy_embed_flag: "-I"
+  objcopy_embed_flag: "binary"
+  unfiltered_cxx_flag: "-no-canonical-prefixes"
+  unfiltered_cxx_flag: "-Wno-builtin-macro-redefined"
+  unfiltered_cxx_flag: "-D__DATE__=\"redacted\""
+  unfiltered_cxx_flag: "-D__TIMESTAMP__=\"redacted\""
+  unfiltered_cxx_flag: "-D__TIME__=\"redacted\""
+  compiler_flag: "-U_FORTIFY_SOURCE"
+  compiler_flag: "-fstack-protector"
+  compiler_flag: "-Wall"
+  compiler_flag: "-Wthread-safety"
+  compiler_flag: "-Wself-assign"
+  compiler_flag: "-fcolor-diagnostics"
+  compiler_flag: "-fno-omit-frame-pointer"
+  tool_path {name: "ar" path: "/usr/bin/ar" }
+  tool_path {name: "ld" path: "/usr/bin/ld" }
+  tool_path {name: "cpp" path: "/usr/bin/cpp" }
+  tool_path {name: "gcc" path: "/usr/local/bin/clang" }
+  tool_path {name: "dwp" path: "/usr/bin/dwp" }
+  tool_path {name: "gcov" path: "None" }
+  tool_path {name: "nm" path: "/usr/bin/nm" }
+  tool_path {name: "objcopy" path: "/usr/bin/objcopy" }
+  tool_path {name: "objdump" path: "/usr/bin/objdump" }
+  tool_path {name: "strip" path: "/usr/bin/strip" }
+
+  compilation_mode_flags {
+    mode: DBG
+    compiler_flag: "-g"
+  }
+  compilation_mode_flags {
+    mode: OPT
+    compiler_flag: "-g0"
+    compiler_flag: "-O2"
+    compiler_flag: "-D_FORTIFY_SOURCE=1"
+    compiler_flag: "-DNDEBUG"
+    compiler_flag: "-ffunction-sections"
+    compiler_flag: "-fdata-sections"
+    linker_flag: "-Wl,--gc-sections"
+  }
+  linking_mode_flags { mode: DYNAMIC }
+
+
+    feature {
+      name: 'coverage'
+      provides: 'profile'
+      flag_set {
+        action: 'preprocess-assemble'
+        action: 'c-compile'
+        action: 'c++-compile'
+        action: 'c++-header-parsing'
+        action: 'c++-module-compile'
+        flag_group {
+        flag: '--coverage'
+      }
+      }
+      flag_set {
+        action: 'c++-link-dynamic-library'
+        action: 'c++-link-nodeps-dynamic-library'
+        action: 'c++-link-executable'
+        flag_group {
+        flag: '--coverage'
+      }
+      }
+    }
+  
+
+  feature {
+    name: 'fdo_optimize'
+    provides: 'profile'
+    flag_set {
+      action: 'c-compile'
+      action: 'c++-compile'
+      expand_if_all_available: 'fdo_profile_path'
+      flag_group {
+        flag: '-fprofile-use=%{fdo_profile_path}'
+        flag: '-fprofile-correction',
+      }
+    }
+  }
+}
+
+toolchain {
+  toolchain_identifier: "msys_x64_mingw"
+  abi_version: "local"
+  abi_libc_version: "local"
+  builtin_sysroot: ""
+  compiler: "mingw-gcc"
+  host_system_name: "local"
+  needsPic: false
+  target_libc: "mingw"
+  target_cpu: "x64_windows"
+  target_system_name: "local"
+
+  artifact_name_pattern {
+     category_name: 'executable'
+     prefix: ''
+     extension: '.exe'
+  }
+
+
+
+  linking_mode_flags { mode: DYNAMIC }
+}
+
+toolchain {
+  toolchain_identifier: "msvc_x64"
+  host_system_name: "local"
+  target_system_name: "local"
+
+  abi_version: "local"
+  abi_libc_version: "local"
+  target_cpu: "x64_windows"
+  compiler: "msvc-cl"
+  target_libc: "msvcrt"
+  default_python_version: "python2.7"
+
+
+
+  tool_path {
+    name: "ar"
+    path: ""
+  }
+  tool_path {
+    name: "ml"
+    path: ""
+  }
+  tool_path {
+    name: "cpp"
+    path: ""
+  }
+  tool_path {
+    name: "gcc"
+    path: ""
+  }
+  tool_path {
+    name: "gcov"
+    path: "wrapper/bin/msvc_nop.bat"
+  }
+  tool_path {
+    name: "ld"
+    path: ""
+  }
+  tool_path {
+    name: "nm"
+    path: "wrapper/bin/msvc_nop.bat"
+  }
+  tool_path {
+    name: "objcopy"
+    path: "wrapper/bin/msvc_nop.bat"
+  }
+  tool_path {
+    name: "objdump"
+    path: "wrapper/bin/msvc_nop.bat"
+  }
+  tool_path {
+    name: "strip"
+    path: "wrapper/bin/msvc_nop.bat"
+  }
+  supports_gold_linker: false
+  supports_start_end_lib: false
+  supports_interface_shared_objects: true
+  supports_incremental_linker: false
+  supports_normalizing_ar: true
+  needsPic: false
+
+  # TODO(pcloudy): Review those flags below, they should be defined by cl.exe
+  compiler_flag: "/DCOMPILER_MSVC"
+
+  # Don't define min/max macros in windows.h.
+  compiler_flag: "/DNOMINMAX"
+
+  # Platform defines.
+  compiler_flag: "/D_WIN32_WINNT=0x0601"
+  # Turn off warning messages.
+  compiler_flag: "/D_CRT_SECURE_NO_DEPRECATE"
+  compiler_flag: "/D_CRT_SECURE_NO_WARNINGS"
+
+  # Useful options to have on for compilation.
+  # Increase the capacity of object files to 2^32 sections.
+  compiler_flag: "/bigobj"
+  # Allocate 500MB for precomputed headers.
+  compiler_flag: "/Zm500"
+  # Catch C++ exceptions only and tell the compiler to assume that functions declared
+  # as extern "C" never throw a C++ exception.
+  compiler_flag: "/EHsc"
+
+  # Globally disabled warnings.
+  # Don't warn about elements of array being be default initialized.
+  compiler_flag: "/wd4351"
+  # Don't warn about no matching delete found.
+  compiler_flag: "/wd4291"
+  # Don't warn about diamond inheritance patterns.
+  compiler_flag: "/wd4250"
+  # Don't warn about insecure functions (e.g. non _s functions).
+  compiler_flag: "/wd4996"
+
+  linker_flag: "/MACHINE:X64"
+
+  feature {
+    name: "no_legacy_features"
+  }
+
+  artifact_name_pattern {
+     category_name: 'object_file'
+     prefix: ''
+     extension: '.obj'
+  }
+
+  artifact_name_pattern {
+     category_name: 'static_library'
+     prefix: ''
+     extension: '.lib'
+  }
+
+  artifact_name_pattern {
+     category_name: 'alwayslink_static_library'
+     prefix: ''
+     extension: '.lo.lib'
+  }
+
+  artifact_name_pattern {
+     category_name: 'executable'
+     prefix: ''
+     extension: '.exe'
+  }
+
+  artifact_name_pattern {
+     category_name: 'dynamic_library'
+     prefix: ''
+     extension: '.dll'
+  }
+
+  artifact_name_pattern {
+     category_name: 'interface_library'
+     prefix: ''
+     extension: '.if.lib'
+  }
+
+  # Suppress startup banner.
+  feature {
+    name: "nologo"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      action: "c++-module-compile"
+      action: "c++-module-codegen"
+      action: "c++-header-parsing"
+      action: "assemble"
+      action: "preprocess-assemble"
+      action: "c++-link-executable"
+      action: "c++-link-dynamic-library"
+      action: "c++-link-nodeps-dynamic-library"
+      action: "c++-link-static-library"
+      flag_group {
+        flag: "/nologo"
+      }
+    }
+  }
+
+  feature {
+    name: 'has_configured_linker_path'
+  }
+
+  # This feature indicates strip is not supported, building stripped binary will just result a copy of orignial binary
+  feature {
+    name: 'no_stripping'
+  }
+
+  # This feature indicates this is a toolchain targeting Windows.
+  feature {
+    name: 'targets_windows'
+    implies: 'copy_dynamic_libraries_to_binary'
+    enabled: true
+  }
+
+  feature {
+    name: 'copy_dynamic_libraries_to_binary'
+  }
+
+  action_config {
+    config_name: 'assemble'
+    action_name: 'assemble'
+    tool {
+      tool_path: ''
+    }
+    implies: 'compiler_input_flags'
+    implies: 'compiler_output_flags'
+    implies: 'nologo'
+    implies: 'msvc_env'
+    implies: 'sysroot'
+  }
+
+  action_config {
+    config_name: 'preprocess-assemble'
+    action_name: 'preprocess-assemble'
+    tool {
+      tool_path: ''
+    }
+    implies: 'compiler_input_flags'
+    implies: 'compiler_output_flags'
+    implies: 'nologo'
+    implies: 'msvc_env'
+    implies: 'sysroot'
+  }
+
+  action_config {
+    config_name: 'c-compile'
+    action_name: 'c-compile'
+    tool {
+      tool_path: ''
+    }
+    implies: 'compiler_input_flags'
+    implies: 'compiler_output_flags'
+    implies: 'legacy_compile_flags'
+    implies: 'nologo'
+    implies: 'msvc_env'
+    implies: 'parse_showincludes'
+    implies: 'user_compile_flags'
+    implies: 'sysroot'
+    implies: 'unfiltered_compile_flags'
+  }
+
+  action_config {
+    config_name: 'c++-compile'
+    action_name: 'c++-compile'
+    tool {
+      tool_path: ''
+    }
+    implies: 'compiler_input_flags'
+    implies: 'compiler_output_flags'
+    implies: 'legacy_compile_flags'
+    implies: 'nologo'
+    implies: 'msvc_env'
+    implies: 'parse_showincludes'
+    implies: 'user_compile_flags'
+    implies: 'sysroot'
+    implies: 'unfiltered_compile_flags'
+  }
+
+  action_config {
+    config_name: 'c++-link-executable'
+    action_name: 'c++-link-executable'
+    tool {
+      tool_path: ''
+    }
+    implies: 'nologo'
+    implies: 'linkstamps'
+    implies: 'output_execpath_flags'
+    implies: 'input_param_flags'
+    implies: 'user_link_flags'
+    implies: 'legacy_link_flags'
+    implies: 'linker_subsystem_flag'
+    implies: 'linker_param_file'
+    implies: 'msvc_env'
+    implies: 'no_stripping'
+  }
+
+  action_config {
+    config_name: 'c++-link-dynamic-library'
+    action_name: 'c++-link-dynamic-library'
+    tool {
+      tool_path: ''
+    }
+    implies: 'nologo'
+    implies: 'shared_flag'
+    implies: 'linkstamps'
+    implies: 'output_execpath_flags'
+    implies: 'input_param_flags'
+    implies: 'user_link_flags'
+    implies: 'legacy_link_flags'
+    implies: 'linker_subsystem_flag'
+    implies: 'linker_param_file'
+    implies: 'msvc_env'
+    implies: 'no_stripping'
+    implies: 'has_configured_linker_path'
+    implies: 'def_file'
+  }
+
+  action_config {
+      config_name: 'c++-link-nodeps-dynamic-library'
+      action_name: 'c++-link-nodeps-dynamic-library'
+      tool {
+        tool_path: ''
+      }
+      implies: 'nologo'
+      implies: 'shared_flag'
+      implies: 'linkstamps'
+      implies: 'output_execpath_flags'
+      implies: 'input_param_flags'
+      implies: 'user_link_flags'
+      implies: 'legacy_link_flags'
+      implies: 'linker_subsystem_flag'
+      implies: 'linker_param_file'
+      implies: 'msvc_env'
+      implies: 'no_stripping'
+      implies: 'has_configured_linker_path'
+      implies: 'def_file'
+    }
+
+  action_config {
+    config_name: 'c++-link-static-library'
+    action_name: 'c++-link-static-library'
+    tool {
+      tool_path: ''
+    }
+    implies: 'nologo'
+    implies: 'archiver_flags'
+    implies: 'input_param_flags'
+    implies: 'linker_param_file'
+    implies: 'msvc_env'
+  }
+
+  # TODO(b/65151735): Remove legacy_compile_flags feature when legacy fields are
+  # not used in this crosstool
+  feature {
+    name: 'legacy_compile_flags'
+    flag_set {
+      expand_if_all_available: 'legacy_compile_flags'
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-header-parsing'
+      action: 'c++-module-compile'
+      action: 'c++-module-codegen'
+      flag_group {
+        iterate_over: 'legacy_compile_flags'
+        flag: '%{legacy_compile_flags}'
+      }
+    }
+  }
+
+  feature {
+    name: "msvc_env"
+    env_set {
+      action: "c-compile"
+      action: "c++-compile"
+      action: "c++-module-compile"
+      action: "c++-module-codegen"
+      action: "c++-header-parsing"
+      action: "assemble"
+      action: "preprocess-assemble"
+      action: "c++-link-executable"
+      action: "c++-link-dynamic-library"
+      action: "c++-link-nodeps-dynamic-library"
+      action: "c++-link-static-library"
+      env_entry {
+        key: "PATH"
+        value: ""
+      }
+      env_entry {
+        key: "TMP"
+        value: ""
+      }
+      env_entry {
+        key: "TEMP"
+        value: ""
+      }
+    }
+    implies: 'msvc_compile_env'
+    implies: 'msvc_link_env'
+  }
+
+  feature {
+    name: "msvc_compile_env"
+    env_set {
+      action: "c-compile"
+      action: "c++-compile"
+      action: "c++-module-compile"
+      action: "c++-module-codegen"
+      action: "c++-header-parsing"
+      action: "assemble"
+      action: "preprocess-assemble"
+      env_entry {
+        key: "INCLUDE"
+        value: ""
+      }
+    }
+  }
+
+  feature {
+    name: "msvc_link_env"
+    env_set {
+      action: "c++-link-executable"
+      action: "c++-link-dynamic-library"
+      action: "c++-link-nodeps-dynamic-library"
+      action: "c++-link-static-library"
+      env_entry {
+        key: "LIB"
+        value: ""
+      }
+    }
+  }
+
+  feature {
+    name: 'include_paths'
+    flag_set {
+      action: "assemble"
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-header-parsing'
+      action: 'c++-module-compile'
+      flag_group {
+        iterate_over: 'quote_include_paths'
+        flag: '/I%{quote_include_paths}'
+      }
+      flag_group {
+        iterate_over: 'include_paths'
+        flag: '/I%{include_paths}'
+      }
+      flag_group {
+        iterate_over: 'system_include_paths'
+        flag: '/I%{system_include_paths}'
+      }
+    }
+  }
+
+  feature {
+    name: "preprocessor_defines"
+    flag_set {
+      action: "assemble"
+      action: "preprocess-assemble"
+      action: "c-compile"
+      action: "c++-compile"
+      action: "c++-header-parsing"
+      action: "c++-module-compile"
+      flag_group {
+        flag: "/D%{preprocessor_defines}"
+        iterate_over: "preprocessor_defines"
+      }
+    }
+  }
+
+  # Tell Bazel to parse the output of /showIncludes
+  feature {
+    name: 'parse_showincludes'
+    flag_set {
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-module-compile'
+      action: 'c++-header-parsing'
+      flag_group {
+        flag: "/showIncludes"
+      }
+    }
+  }
+
+
+  feature {
+    name: 'generate_pdb_file'
+    requires: {
+      feature: 'dbg'
+    }
+    requires: {
+      feature: 'fastbuild'
+    }
+  }
+
+  feature {
+    name: 'shared_flag'
+    flag_set {
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: '/DLL'
+      }
+    }
+  }
+
+  feature {
+    name: 'linkstamps'
+    flag_set {
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      expand_if_all_available: 'linkstamp_paths'
+      flag_group {
+        iterate_over: 'linkstamp_paths'
+        flag: '%{linkstamp_paths}'
+      }
+    }
+  }
+
+  feature {
+    name: 'output_execpath_flags'
+    flag_set {
+      expand_if_all_available: 'output_execpath'
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: '/OUT:%{output_execpath}'
+      }
+    }
+  }
+
+  feature {
+    name: 'archiver_flags'
+    flag_set {
+      expand_if_all_available: 'output_execpath'
+      action: 'c++-link-static-library'
+      flag_group {
+        flag: '/OUT:%{output_execpath}'
+      }
+    }
+  }
+
+  feature {
+    name: 'input_param_flags'
+    flag_set {
+      expand_if_all_available: 'interface_library_output_path'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "/IMPLIB:%{interface_library_output_path}"
+      }
+    }
+    flag_set {
+      expand_if_all_available: 'libopts'
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        iterate_over: 'libopts'
+        flag: '%{libopts}'
+      }
+    }
+    flag_set {
+      expand_if_all_available: 'libraries_to_link'
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      action: 'c++-link-static-library'
+      flag_group {
+        iterate_over: 'libraries_to_link'
+        flag_group {
+          expand_if_equal: {
+            variable: 'libraries_to_link.type'
+            value: 'object_file_group'
+          }
+          iterate_over: 'libraries_to_link.object_files'
+          flag_group {
+            flag: '%{libraries_to_link.object_files}'
+          }
+        }
+        flag_group {
+          expand_if_equal: {
+            variable: 'libraries_to_link.type'
+            value: 'object_file'
+          }
+          flag_group {
+            flag: '%{libraries_to_link.name}'
+          }
+        }
+        flag_group {
+          expand_if_equal: {
+            variable: 'libraries_to_link.type'
+            value: 'interface_library'
+          }
+          flag_group {
+            flag: '%{libraries_to_link.name}'
+          }
+        }
+        flag_group {
+          expand_if_equal: {
+            variable: 'libraries_to_link.type'
+            value: 'static_library'
+          }
+          flag_group {
+            expand_if_false: 'libraries_to_link.is_whole_archive'
+            flag: '%{libraries_to_link.name}'
+          }
+          flag_group {
+            expand_if_true: 'libraries_to_link.is_whole_archive'
+            flag: '/WHOLEARCHIVE:%{libraries_to_link.name}'
+          }
+        }
+      }
+    }
+  }
+
+  # Since this feature is declared earlier in the CROSSTOOL than
+  # "user_link_flags", this feature will be applied prior to it anwyhere they
+  # are both implied. And since "user_link_flags" contains the linkopts from
+  # the build rule, this allows the user to override the /SUBSYSTEM in the BUILD
+  # file.
+  feature {
+    name: 'linker_subsystem_flag'
+    flag_set {
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: '/SUBSYSTEM:CONSOLE'
+      }
+    }
+  }
+
+  # The "user_link_flags" contains user-defined linkopts (from build rules)
+  # so it should be defined after features that declare user-overridable flags.
+  # For example the "linker_subsystem_flag" defines a default "/SUBSYSTEM" flag
+  # but we want to let the user override it, therefore "link_flag_subsystem" is
+  # defined earlier in the CROSSTOOL file than "user_link_flags".
+  feature {
+    name: 'user_link_flags'
+    flag_set {
+      expand_if_all_available: 'user_link_flags'
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        iterate_over: 'user_link_flags'
+        flag: '%{user_link_flags}'
+      }
+    }
+  }
+  feature {
+    name: 'legacy_link_flags'
+    flag_set {
+      expand_if_all_available: 'legacy_link_flags'
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        iterate_over: 'legacy_link_flags'
+        flag: '%{legacy_link_flags}'
+      }
+    }
+  }
+
+  feature {
+    name: 'linker_param_file'
+    flag_set {
+      expand_if_all_available: 'linker_param_file'
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      action: 'c++-link-static-library'
+      flag_group {
+        flag: '@%{linker_param_file}'
+      }
+    }
+  }
+
+  feature {
+    name: 'static_link_msvcrt'
+  }
+
+  feature {
+    name: 'static_link_msvcrt_no_debug'
+    flag_set {
+      action: 'c-compile'
+      action: 'c++-compile'
+      flag_group {
+        flag: "/MT"
+      }
+    }
+    flag_set {
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "/DEFAULTLIB:libcmt.lib"
+      }
+    }
+    requires: { feature: 'fastbuild'}
+    requires: { feature: 'opt'}
+  }
+
+  feature {
+    name: 'dynamic_link_msvcrt_no_debug'
+    flag_set {
+      action: 'c-compile'
+      action: 'c++-compile'
+      flag_group {
+        flag: "/MD"
+      }
+    }
+    flag_set {
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "/DEFAULTLIB:msvcrt.lib"
+      }
+    }
+    requires: { feature: 'fastbuild'}
+    requires: { feature: 'opt'}
+  }
+
+  feature {
+    name: 'static_link_msvcrt_debug'
+    flag_set {
+      action: 'c-compile'
+      action: 'c++-compile'
+      flag_group {
+        flag: "/MTd"
+      }
+    }
+    flag_set {
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "/DEFAULTLIB:libcmtd.lib"
+      }
+    }
+    requires: { feature: 'dbg'}
+  }
+
+  feature {
+    name: 'dynamic_link_msvcrt_debug'
+    flag_set {
+      action: 'c-compile'
+      action: 'c++-compile'
+      flag_group {
+        flag: "/MDd"
+      }
+    }
+    flag_set {
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "/DEFAULTLIB:msvcrtd.lib"
+      }
+    }
+    requires: { feature: 'dbg'}
+  }
+
+  feature {
+    name: 'dbg'
+    flag_set {
+      action: 'c-compile'
+      action: 'c++-compile'
+      flag_group {
+        flag: "/Od"
+        flag: "/Z7"
+      }
+    }
+    flag_set {
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: ""
+        flag: "/INCREMENTAL:NO"
+      }
+    }
+    implies: 'generate_pdb_file'
+  }
+
+  feature {
+    name: 'fastbuild'
+    flag_set {
+      action: 'c-compile'
+      action: 'c++-compile'
+      flag_group {
+        flag: "/Od"
+        flag: "/Z7"
+      }
+    }
+    flag_set {
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: ""
+        flag: "/INCREMENTAL:NO"
+      }
+    }
+    implies: 'generate_pdb_file'
+  }
+
+  feature {
+    name: 'opt'
+    flag_set {
+      action: 'c-compile'
+      action: 'c++-compile'
+      flag_group {
+        flag: "/O2" # Implies /Og /Oi /Ot /Oy /Ob2 /Gs /GF /Gy
+      }
+    }
+    implies: 'frame_pointer'
+  }
+
+  # Keep stack frames for debugging, even in opt mode.
+  # Must come after /O1, /O2 and /Ox.
+  feature {
+    name: "frame_pointer"
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        flag: "/Oy-"
+      }
+    }
+  }
+
+  # Remove assert/DCHECKs in opt mode.
+  # You can have them back with --features=-disable_assertions.
+  feature {
+    name: 'disable_assertions'
+    enabled: true
+    flag_set {
+      action: 'c-compile'
+      action: 'c++-compile'
+      with_feature: {
+        feature: 'opt'
+      }
+      flag_group {
+        flag: "/DNDEBUG"
+      }
+    }
+  }
+
+  feature {
+    name: "determinism"
+    enabled: true
+    flag_set {
+      action: "c-compile"
+      action: "c++-compile"
+      flag_group {
+        # Make C++ compilation deterministic. Use linkstamping instead of these
+        # compiler symbols.
+        # TODO: detect clang on Windows and use "-Wno-builtin-macro-redefined"
+        flag: "/wd4117" # Trying to define or undefine a predefined macro
+        flag: "-D__DATE__=\"redacted\""
+        flag: "-D__TIMESTAMP__=\"redacted\""
+        flag: "-D__TIME__=\"redacted\""
+      }
+    }
+  }
+
+  feature {
+    name: 'treat_warnings_as_errors'
+    flag_set {
+      action: 'c-compile'
+      action: 'c++-compile'
+      flag_group {
+        flag: "/WX"
+      }
+    }
+  }
+
+  # Trade slower build time for smaller binary
+  feature {
+    name: 'smaller_binary'
+    enabled: true
+    flag_set {
+      action: 'c-compile'
+      action: 'c++-compile'
+      with_feature: {
+        feature: 'opt'
+      }
+      flag_group {
+        flag: "/Gy" # Enable function-level linking (-ffunction-sections)
+        flag: "/Gw" # Optimize global data (-fdata-sections)
+      }
+    }
+    flag_set {
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library',
+      action: 'c++-link-nodeps-dynamic-library'
+      with_feature: {
+        feature: 'opt'
+      }
+      flag_group {
+        flag: '/OPT:ICF' # Fold identical functions
+        flag: '/OPT:REF' # Eliminate unreferenced functions and data
+      }
+    }
+  }
+
+  # Suppress warnings that most users do not care
+  feature {
+    name: 'ignore_noisy_warnings'
+    enabled: true
+    flag_set {
+      action: 'c++-link-static-library'
+      flag_group {
+        # Suppress 'object file does not define any public symbols' warning
+        flag: '/ignore:4221'
+      }
+    }
+  }
+
+  feature {
+    name: 'user_compile_flags'
+    flag_set {
+      expand_if_all_available: 'user_compile_flags'
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-header-parsing'
+      action: 'c++-module-compile'
+      action: 'c++-module-codegen'
+      flag_group {
+        iterate_over: 'user_compile_flags'
+        flag: '%{user_compile_flags}'
+      }
+    }
+  }
+
+  feature {
+    name: 'sysroot'
+    flag_set {
+      expand_if_all_available: 'sysroot'
+      action: 'assemble'
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-header-parsing'
+      action: 'c++-module-compile'
+      action: 'c++-module-codegen'
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        iterate_over: 'sysroot'
+        flag: '--sysroot=%{sysroot}'
+      }
+    }
+  }
+
+  feature {
+    name: 'unfiltered_compile_flags'
+    flag_set {
+      expand_if_all_available: 'unfiltered_compile_flags'
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-header-parsing'
+      action: 'c++-module-compile'
+      action: 'c++-module-codegen'
+      flag_group {
+        iterate_over: 'unfiltered_compile_flags'
+        flag: '%{unfiltered_compile_flags}'
+      }
+    }
+  }
+
+  feature {
+    name: 'compiler_output_flags'
+    flag_set {
+      action: 'assemble'
+      flag_group {
+        expand_if_all_available: 'output_file'
+        expand_if_none_available: 'output_assembly_file'
+        expand_if_none_available: 'output_preprocess_file'
+        flag: '/Fo%{output_file}'
+        flag: '/Zi'
+      }
+    }
+    flag_set {
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-header-parsing'
+      action: 'c++-module-compile'
+      action: 'c++-module-codegen'
+      flag_group {
+        expand_if_all_available: 'output_file'
+        expand_if_none_available: 'output_assembly_file'
+        expand_if_none_available: 'output_preprocess_file'
+        flag: '/Fo%{output_file}'
+      }
+      flag_group {
+        expand_if_all_available: 'output_file'
+        expand_if_all_available: 'output_assembly_file'
+        flag: '/Fa%{output_file}'
+      }
+      flag_group {
+        expand_if_all_available: 'output_file'
+        expand_if_all_available: 'output_preprocess_file'
+        flag: '/P'
+        flag: '/Fi%{output_file}'
+      }
+    }
+  }
+
+  feature {
+    name: 'compiler_input_flags'
+    flag_set {
+      action: 'assemble'
+      action: 'preprocess-assemble'
+      action: 'c-compile'
+      action: 'c++-compile'
+      action: 'c++-header-parsing'
+      action: 'c++-module-compile'
+      action: 'c++-module-codegen'
+      flag_group {
+        expand_if_all_available: 'source_file'
+        flag: '/c'
+        flag: '%{source_file}'
+      }
+    }
+  }
+
+  feature {
+    name : 'def_file',
+    flag_set {
+      expand_if_all_available: 'def_file_path'
+      action: 'c++-link-executable'
+      action: 'c++-link-dynamic-library'
+      action: "c++-link-nodeps-dynamic-library"
+      flag_group {
+        flag: "/DEF:%{def_file_path}"
+        # We can specify a different DLL name in DEF file, /ignore:4070 suppresses
+        # the warning message about DLL name doesn't match the default one.
+        # See https://msdn.microsoft.com/en-us/library/sfkk2fz7.aspx
+        flag: "/ignore:4070"
+      }
+    }
+  }
+
+  feature {
+    name: 'windows_export_all_symbols'
+  }
+
+  feature {
+    name: 'no_windows_export_all_symbols'
+  }
+
+  linking_mode_flags { mode: DYNAMIC }
+}
diff --git a/third_party/toolchains/preconfig/ubuntu16.04/clang/WORKSPACE b/third_party/toolchains/preconfig/ubuntu16.04/clang/WORKSPACE
new file mode 100644
index 0000000000..bc05b4c36f
--- /dev/null
+++ b/third_party/toolchains/preconfig/ubuntu16.04/clang/WORKSPACE
@@ -0,0 +1,2 @@
+# DO NOT EDIT: automatically generated WORKSPACE file for cc_autoconf rule
+workspace(name = "local_config_cc")
diff --git a/third_party/toolchains/preconfig/ubuntu16.04/clang/cc_wrapper.sh b/third_party/toolchains/preconfig/ubuntu16.04/clang/cc_wrapper.sh
new file mode 100755
index 0000000000..42a751dccf
--- /dev/null
+++ b/third_party/toolchains/preconfig/ubuntu16.04/clang/cc_wrapper.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# Copyright 2015 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Ship the environment to the C++ action
+#
+set -eu
+
+# Set-up the environment
+
+
+# Call the C++ compiler
+/usr/local/bin/clang "$@"
diff --git a/third_party/toolchains/preconfig/ubuntu16.04/clang/dummy_toolchain.bzl b/third_party/toolchains/preconfig/ubuntu16.04/clang/dummy_toolchain.bzl
new file mode 100755
index 0000000000..45c0285d23
--- /dev/null
+++ b/third_party/toolchains/preconfig/ubuntu16.04/clang/dummy_toolchain.bzl
@@ -0,0 +1,23 @@
+# pylint: disable=g-bad-file-header
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Skylark rule that stubs a toolchain."""
+
+def _dummy_toolchain_impl(ctx):
+    ctx = ctx  # unused argument
+    toolchain = platform_common.ToolchainInfo()
+    return [toolchain]
+
+dummy_toolchain = rule(_dummy_toolchain_impl, attrs = {})
diff --git a/third_party/toolchains/preconfig/ubuntu16.04/clang/tools/cpp/empty.cc b/third_party/toolchains/preconfig/ubuntu16.04/clang/tools/cpp/empty.cc
new file mode 100755
index 0000000000..c272dabaeb
--- /dev/null
+++ b/third_party/toolchains/preconfig/ubuntu16.04/clang/tools/cpp/empty.cc
@@ -0,0 +1 @@
+int main() {}
\ No newline at end of file
diff --git a/third_party/toolchains/preconfig/ubuntu16.04/py3/BUILD b/third_party/toolchains/preconfig/ubuntu16.04/py3/BUILD
new file mode 100755
index 0000000000..77eaa4d512
--- /dev/null
+++ b/third_party/toolchains/preconfig/ubuntu16.04/py3/BUILD
@@ -0,0 +1,205 @@
+licenses(["restricted"])
+
+package(default_visibility = ["//visibility:public"])
+
+# To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib
+# See https://docs.python.org/3/extending/windows.html
+cc_import(
+    name = "python_lib",
+    interface_library = select({
+        ":windows": ":python_import_lib",
+        # A placeholder for Unix platforms which makes --no_build happy.
+        "//conditions:default": "not-existing.lib",
+    }),
+    system_provided = 1,
+)
+
+cc_library(
+    name = "python_headers",
+    hdrs = [":python_include"],
+    includes = ["python_include"],
+    deps = select({
+        ":windows": [":python_lib"],
+        "//conditions:default": [],
+    }),
+)
+
+cc_library(
+    name = "numpy_headers",
+    hdrs = [":numpy_include"],
+    includes = ["numpy_include"],
+)
+
+config_setting(
+    name = "windows",
+    values = {"cpu": "x64_windows"},
+    visibility = ["//visibility:public"],
+)
+
+genrule(
+    name = "python_include",
+    outs = [
+        "python_include/Python-ast.h",
+        "python_include/Python.h",
+        "python_include/abstract.h",
+        "python_include/accu.h",
+        "python_include/asdl.h",
+        "python_include/ast.h",
+        "python_include/bitset.h",
+        "python_include/bltinmodule.h",
+        "python_include/boolobject.h",
+        "python_include/bytearrayobject.h",
+        "python_include/bytes_methods.h",
+        "python_include/bytesobject.h",
+        "python_include/cellobject.h",
+        "python_include/ceval.h",
+        "python_include/classobject.h",
+        "python_include/code.h",
+        "python_include/codecs.h",
+        "python_include/compile.h",
+        "python_include/complexobject.h",
+        "python_include/datetime.h",
+        "python_include/descrobject.h",
+        "python_include/dictobject.h",
+        "python_include/dtoa.h",
+        "python_include/dynamic_annotations.h",
+        "python_include/enumobject.h",
+        "python_include/errcode.h",
+        "python_include/eval.h",
+        "python_include/fileobject.h",
+        "python_include/fileutils.h",
+        "python_include/floatobject.h",
+        "python_include/frameobject.h",
+        "python_include/funcobject.h",
+        "python_include/genobject.h",
+        "python_include/graminit.h",
+        "python_include/grammar.h",
+        "python_include/import.h",
+        "python_include/intrcheck.h",
+        "python_include/iterobject.h",
+        "python_include/listobject.h",
+        "python_include/longintrepr.h",
+        "python_include/longobject.h",
+        "python_include/marshal.h",
+        "python_include/memoryobject.h",
+        "python_include/metagrammar.h",
+        "python_include/methodobject.h",
+        "python_include/modsupport.h",
+        "python_include/moduleobject.h",
+        "python_include/namespaceobject.h",
+        "python_include/node.h",
+        "python_include/numpy/__multiarray_api.h",
+        "python_include/numpy/__ufunc_api.h",
+        "python_include/numpy/_neighborhood_iterator_imp.h",
+        "python_include/numpy/_numpyconfig.h",
+        "python_include/numpy/arrayobject.h",
+        "python_include/numpy/arrayscalars.h",
+        "python_include/numpy/halffloat.h",
+        "python_include/numpy/multiarray_api.txt",
+        "python_include/numpy/ndarrayobject.h",
+        "python_include/numpy/ndarraytypes.h",
+        "python_include/numpy/noprefix.h",
+        "python_include/numpy/npy_1_7_deprecated_api.h",
+        "python_include/numpy/npy_3kcompat.h",
+        "python_include/numpy/npy_common.h",
+        "python_include/numpy/npy_cpu.h",
+        "python_include/numpy/npy_endian.h",
+        "python_include/numpy/npy_interrupt.h",
+        "python_include/numpy/npy_math.h",
+        "python_include/numpy/npy_no_deprecated_api.h",
+        "python_include/numpy/npy_os.h",
+        "python_include/numpy/numpyconfig.h",
+        "python_include/numpy/old_defines.h",
+        "python_include/numpy/oldnumeric.h",
+        "python_include/numpy/ufunc_api.txt",
+        "python_include/numpy/ufuncobject.h",
+        "python_include/numpy/utils.h",
+        "python_include/object.h",
+        "python_include/objimpl.h",
+        "python_include/odictobject.h",
+        "python_include/opcode.h",
+        "python_include/osdefs.h",
+        "python_include/parsetok.h",
+        "python_include/patchlevel.h",
+        "python_include/pgen.h",
+        "python_include/pgenheaders.h",
+        "python_include/py_curses.h",
+        "python_include/pyarena.h",
+        "python_include/pyatomic.h",
+        "python_include/pycapsule.h",
+        "python_include/pyconfig.h",
+        "python_include/pyctype.h",
+        "python_include/pydebug.h",
+        "python_include/pyerrors.h",
+        "python_include/pyexpat.h",
+        "python_include/pyfpe.h",
+        "python_include/pygetopt.h",
+        "python_include/pyhash.h",
+        "python_include/pylifecycle.h",
+        "python_include/pymacconfig.h",
+        "python_include/pymacro.h",
+        "python_include/pymath.h",
+        "python_include/pymem.h",
+        "python_include/pyport.h",
+        "python_include/pystate.h",
+        "python_include/pystrcmp.h",
+        "python_include/pystrhex.h",
+        "python_include/pystrtod.h",
+        "python_include/pythonrun.h",
+        "python_include/pythread.h",
+        "python_include/pytime.h",
+        "python_include/rangeobject.h",
+        "python_include/setobject.h",
+        "python_include/sliceobject.h",
+        "python_include/structmember.h",
+        "python_include/structseq.h",
+        "python_include/symtable.h",
+        "python_include/sysmodule.h",
+        "python_include/token.h",
+        "python_include/traceback.h",
+        "python_include/tupleobject.h",
+        "python_include/typeslots.h",
+        "python_include/ucnhash.h",
+        "python_include/unicodeobject.h",
+        "python_include/warnings.h",
+        "python_include/weakrefobject.h",
+    ],
+    cmd = """
+cp -f "/usr/include/python3.5m/Python-ast.h" "$(@D)/python_include/Python-ast.h" && cp -f "/usr/include/python3.5m/Python.h" "$(@D)/python_include/Python.h" && cp -f "/usr/include/python3.5m/abstract.h" "$(@D)/python_include/abstract.h" && cp -f "/usr/include/python3.5m/accu.h" "$(@D)/python_include/accu.h" && cp -f "/usr/include/python3.5m/asdl.h" "$(@D)/python_include/asdl.h" && cp -f "/usr/include/python3.5m/ast.h" "$(@D)/python_include/ast.h" && cp -f "/usr/include/python3.5m/bitset.h" "$(@D)/python_include/bitset.h" && cp -f "/usr/include/python3.5m/bltinmodule.h" "$(@D)/python_include/bltinmodule.h" && cp -f "/usr/include/python3.5m/boolobject.h" "$(@D)/python_include/boolobject.h" && cp -f "/usr/include/python3.5m/bytearrayobject.h" "$(@D)/python_include/bytearrayobject.h" && cp -f "/usr/include/python3.5m/bytes_methods.h" "$(@D)/python_include/bytes_methods.h" && cp -f "/usr/include/python3.5m/bytesobject.h" "$(@D)/python_include/bytesobject.h" && cp -f "/usr/include/python3.5m/cellobject.h" "$(@D)/python_include/cellobject.h" && cp -f "/usr/include/python3.5m/ceval.h" "$(@D)/python_include/ceval.h" && cp -f "/usr/include/python3.5m/classobject.h" "$(@D)/python_include/classobject.h" && cp -f "/usr/include/python3.5m/code.h" "$(@D)/python_include/code.h" && cp -f "/usr/include/python3.5m/codecs.h" "$(@D)/python_include/codecs.h" && cp -f "/usr/include/python3.5m/compile.h" "$(@D)/python_include/compile.h" && cp -f "/usr/include/python3.5m/complexobject.h" "$(@D)/python_include/complexobject.h" && cp -f "/usr/include/python3.5m/datetime.h" "$(@D)/python_include/datetime.h" && cp -f "/usr/include/python3.5m/descrobject.h" "$(@D)/python_include/descrobject.h" && cp -f "/usr/include/python3.5m/dictobject.h" "$(@D)/python_include/dictobject.h" && cp -f "/usr/include/python3.5m/dtoa.h" "$(@D)/python_include/dtoa.h" && cp -f "/usr/include/python3.5m/dynamic_annotations.h" "$(@D)/python_include/dynamic_annotations.h" && cp -f "/usr/include/python3.5m/enumobject.h" "$(@D)/python_include/enumobject.h" && cp -f "/usr/include/python3.5m/errcode.h" "$(@D)/python_include/errcode.h" && cp -f "/usr/include/python3.5m/eval.h" "$(@D)/python_include/eval.h" && cp -f "/usr/include/python3.5m/fileobject.h" "$(@D)/python_include/fileobject.h" && cp -f "/usr/include/python3.5m/fileutils.h" "$(@D)/python_include/fileutils.h" && cp -f "/usr/include/python3.5m/floatobject.h" "$(@D)/python_include/floatobject.h" && cp -f "/usr/include/python3.5m/frameobject.h" "$(@D)/python_include/frameobject.h" && cp -f "/usr/include/python3.5m/funcobject.h" "$(@D)/python_include/funcobject.h" && cp -f "/usr/include/python3.5m/genobject.h" "$(@D)/python_include/genobject.h" && cp -f "/usr/include/python3.5m/graminit.h" "$(@D)/python_include/graminit.h" && cp -f "/usr/include/python3.5m/grammar.h" "$(@D)/python_include/grammar.h" && cp -f "/usr/include/python3.5m/import.h" "$(@D)/python_include/import.h" && cp -f "/usr/include/python3.5m/intrcheck.h" "$(@D)/python_include/intrcheck.h" && cp -f "/usr/include/python3.5m/iterobject.h" "$(@D)/python_include/iterobject.h" && cp -f "/usr/include/python3.5m/listobject.h" "$(@D)/python_include/listobject.h" && cp -f "/usr/include/python3.5m/longintrepr.h" "$(@D)/python_include/longintrepr.h" && cp -f "/usr/include/python3.5m/longobject.h" "$(@D)/python_include/longobject.h" && cp -f "/usr/include/python3.5m/marshal.h" "$(@D)/python_include/marshal.h" && cp -f "/usr/include/python3.5m/memoryobject.h" "$(@D)/python_include/memoryobject.h" && cp -f "/usr/include/python3.5m/metagrammar.h" "$(@D)/python_include/metagrammar.h" && cp -f "/usr/include/python3.5m/methodobject.h" "$(@D)/python_include/methodobject.h" && cp -f "/usr/include/python3.5m/modsupport.h" "$(@D)/python_include/modsupport.h" && cp -f "/usr/include/python3.5m/moduleobject.h" "$(@D)/python_include/moduleobject.h" && cp -f "/usr/include/python3.5m/namespaceobject.h" "$(@D)/python_include/namespaceobject.h" && cp -f "/usr/include/python3.5m/node.h" "$(@D)/python_include/node.h" && cp -f "/usr/include/python3.5m/numpy/__multiarray_api.h" "$(@D)/python_include/numpy/__multiarray_api.h" && cp -f "/usr/include/python3.5m/numpy/__ufunc_api.h" "$(@D)/python_include/numpy/__ufunc_api.h" && cp -f "/usr/include/python3.5m/numpy/_neighborhood_iterator_imp.h" "$(@D)/python_include/numpy/_neighborhood_iterator_imp.h" && cp -f "/usr/include/python3.5m/numpy/_numpyconfig.h" "$(@D)/python_include/numpy/_numpyconfig.h" && cp -f "/usr/include/python3.5m/numpy/arrayobject.h" "$(@D)/python_include/numpy/arrayobject.h" && cp -f "/usr/include/python3.5m/numpy/arrayscalars.h" "$(@D)/python_include/numpy/arrayscalars.h" && cp -f "/usr/include/python3.5m/numpy/halffloat.h" "$(@D)/python_include/numpy/halffloat.h" && cp -f "/usr/include/python3.5m/numpy/multiarray_api.txt" "$(@D)/python_include/numpy/multiarray_api.txt" && cp -f "/usr/include/python3.5m/numpy/ndarrayobject.h" "$(@D)/python_include/numpy/ndarrayobject.h" && cp -f "/usr/include/python3.5m/numpy/ndarraytypes.h" "$(@D)/python_include/numpy/ndarraytypes.h" && cp -f "/usr/include/python3.5m/numpy/noprefix.h" "$(@D)/python_include/numpy/noprefix.h" && cp -f "/usr/include/python3.5m/numpy/npy_1_7_deprecated_api.h" "$(@D)/python_include/numpy/npy_1_7_deprecated_api.h" && cp -f "/usr/include/python3.5m/numpy/npy_3kcompat.h" "$(@D)/python_include/numpy/npy_3kcompat.h" && cp -f "/usr/include/python3.5m/numpy/npy_common.h" "$(@D)/python_include/numpy/npy_common.h" && cp -f "/usr/include/python3.5m/numpy/npy_cpu.h" "$(@D)/python_include/numpy/npy_cpu.h" && cp -f "/usr/include/python3.5m/numpy/npy_endian.h" "$(@D)/python_include/numpy/npy_endian.h" && cp -f "/usr/include/python3.5m/numpy/npy_interrupt.h" "$(@D)/python_include/numpy/npy_interrupt.h" && cp -f "/usr/include/python3.5m/numpy/npy_math.h" "$(@D)/python_include/numpy/npy_math.h" && cp -f "/usr/include/python3.5m/numpy/npy_no_deprecated_api.h" "$(@D)/python_include/numpy/npy_no_deprecated_api.h" && cp -f "/usr/include/python3.5m/numpy/npy_os.h" "$(@D)/python_include/numpy/npy_os.h" && cp -f "/usr/include/python3.5m/numpy/numpyconfig.h" "$(@D)/python_include/numpy/numpyconfig.h" && cp -f "/usr/include/python3.5m/numpy/old_defines.h" "$(@D)/python_include/numpy/old_defines.h" && cp -f "/usr/include/python3.5m/numpy/oldnumeric.h" "$(@D)/python_include/numpy/oldnumeric.h" && cp -f "/usr/include/python3.5m/numpy/ufunc_api.txt" "$(@D)/python_include/numpy/ufunc_api.txt" && cp -f "/usr/include/python3.5m/numpy/ufuncobject.h" "$(@D)/python_include/numpy/ufuncobject.h" && cp -f "/usr/include/python3.5m/numpy/utils.h" "$(@D)/python_include/numpy/utils.h" && cp -f "/usr/include/python3.5m/object.h" "$(@D)/python_include/object.h" && cp -f "/usr/include/python3.5m/objimpl.h" "$(@D)/python_include/objimpl.h" && cp -f "/usr/include/python3.5m/odictobject.h" "$(@D)/python_include/odictobject.h" && cp -f "/usr/include/python3.5m/opcode.h" "$(@D)/python_include/opcode.h" && cp -f "/usr/include/python3.5m/osdefs.h" "$(@D)/python_include/osdefs.h" && cp -f "/usr/include/python3.5m/parsetok.h" "$(@D)/python_include/parsetok.h" && cp -f "/usr/include/python3.5m/patchlevel.h" "$(@D)/python_include/patchlevel.h" && cp -f "/usr/include/python3.5m/pgen.h" "$(@D)/python_include/pgen.h" && cp -f "/usr/include/python3.5m/pgenheaders.h" "$(@D)/python_include/pgenheaders.h" && cp -f "/usr/include/python3.5m/py_curses.h" "$(@D)/python_include/py_curses.h" && cp -f "/usr/include/python3.5m/pyarena.h" "$(@D)/python_include/pyarena.h" && cp -f "/usr/include/python3.5m/pyatomic.h" "$(@D)/python_include/pyatomic.h" && cp -f "/usr/include/python3.5m/pycapsule.h" "$(@D)/python_include/pycapsule.h" && cp -f "/usr/include/python3.5m/pyconfig.h" "$(@D)/python_include/pyconfig.h" && cp -f "/usr/include/python3.5m/pyctype.h" "$(@D)/python_include/pyctype.h" && cp -f "/usr/include/python3.5m/pydebug.h" "$(@D)/python_include/pydebug.h" && cp -f "/usr/include/python3.5m/pyerrors.h" "$(@D)/python_include/pyerrors.h" && cp -f "/usr/include/python3.5m/pyexpat.h" "$(@D)/python_include/pyexpat.h" && cp -f "/usr/include/python3.5m/pyfpe.h" "$(@D)/python_include/pyfpe.h" && cp -f "/usr/include/python3.5m/pygetopt.h" "$(@D)/python_include/pygetopt.h" && cp -f "/usr/include/python3.5m/pyhash.h" "$(@D)/python_include/pyhash.h" && cp -f "/usr/include/python3.5m/pylifecycle.h" "$(@D)/python_include/pylifecycle.h" && cp -f "/usr/include/python3.5m/pymacconfig.h" "$(@D)/python_include/pymacconfig.h" && cp -f "/usr/include/python3.5m/pymacro.h" "$(@D)/python_include/pymacro.h" && cp -f "/usr/include/python3.5m/pymath.h" "$(@D)/python_include/pymath.h" && cp -f "/usr/include/python3.5m/pymem.h" "$(@D)/python_include/pymem.h" && cp -f "/usr/include/python3.5m/pyport.h" "$(@D)/python_include/pyport.h" && cp -f "/usr/include/python3.5m/pystate.h" "$(@D)/python_include/pystate.h" && cp -f "/usr/include/python3.5m/pystrcmp.h" "$(@D)/python_include/pystrcmp.h" && cp -f "/usr/include/python3.5m/pystrhex.h" "$(@D)/python_include/pystrhex.h" && cp -f "/usr/include/python3.5m/pystrtod.h" "$(@D)/python_include/pystrtod.h" && cp -f "/usr/include/python3.5m/pythonrun.h" "$(@D)/python_include/pythonrun.h" && cp -f "/usr/include/python3.5m/pythread.h" "$(@D)/python_include/pythread.h" && cp -f "/usr/include/python3.5m/pytime.h" "$(@D)/python_include/pytime.h" && cp -f "/usr/include/python3.5m/rangeobject.h" "$(@D)/python_include/rangeobject.h" && cp -f "/usr/include/python3.5m/setobject.h" "$(@D)/python_include/setobject.h" && cp -f "/usr/include/python3.5m/sliceobject.h" "$(@D)/python_include/sliceobject.h" && cp -f "/usr/include/python3.5m/structmember.h" "$(@D)/python_include/structmember.h" && cp -f "/usr/include/python3.5m/structseq.h" "$(@D)/python_include/structseq.h" && cp -f "/usr/include/python3.5m/symtable.h" "$(@D)/python_include/symtable.h" && cp -f "/usr/include/python3.5m/sysmodule.h" "$(@D)/python_include/sysmodule.h" && cp -f "/usr/include/python3.5m/token.h" "$(@D)/python_include/token.h" && cp -f "/usr/include/python3.5m/traceback.h" "$(@D)/python_include/traceback.h" && cp -f "/usr/include/python3.5m/tupleobject.h" "$(@D)/python_include/tupleobject.h" && cp -f "/usr/include/python3.5m/typeslots.h" "$(@D)/python_include/typeslots.h" && cp -f "/usr/include/python3.5m/ucnhash.h" "$(@D)/python_include/ucnhash.h" && cp -f "/usr/include/python3.5m/unicodeobject.h" "$(@D)/python_include/unicodeobject.h" && cp -f "/usr/include/python3.5m/warnings.h" "$(@D)/python_include/warnings.h" && cp -f "/usr/include/python3.5m/weakrefobject.h" "$(@D)/python_include/weakrefobject.h"
+   """,
+)
+
+genrule(
+    name = "numpy_include",
+    outs = [
+        "numpy_include/numpy/__multiarray_api.h",
+        "numpy_include/numpy/__ufunc_api.h",
+        "numpy_include/numpy/_neighborhood_iterator_imp.h",
+        "numpy_include/numpy/_numpyconfig.h",
+        "numpy_include/numpy/arrayobject.h",
+        "numpy_include/numpy/arrayscalars.h",
+        "numpy_include/numpy/halffloat.h",
+        "numpy_include/numpy/multiarray_api.txt",
+        "numpy_include/numpy/ndarrayobject.h",
+        "numpy_include/numpy/ndarraytypes.h",
+        "numpy_include/numpy/noprefix.h",
+        "numpy_include/numpy/npy_1_7_deprecated_api.h",
+        "numpy_include/numpy/npy_3kcompat.h",
+        "numpy_include/numpy/npy_common.h",
+        "numpy_include/numpy/npy_cpu.h",
+        "numpy_include/numpy/npy_endian.h",
+        "numpy_include/numpy/npy_interrupt.h",
+        "numpy_include/numpy/npy_math.h",
+        "numpy_include/numpy/npy_no_deprecated_api.h",
+        "numpy_include/numpy/npy_os.h",
+        "numpy_include/numpy/numpyconfig.h",
+        "numpy_include/numpy/old_defines.h",
+        "numpy_include/numpy/oldnumeric.h",
+        "numpy_include/numpy/ufunc_api.txt",
+        "numpy_include/numpy/ufuncobject.h",
+        "numpy_include/numpy/utils.h",
+    ],
+    cmd = """
+cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/__multiarray_api.h" "$(@D)/numpy_include/numpy/__multiarray_api.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/__ufunc_api.h" "$(@D)/numpy_include/numpy/__ufunc_api.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/_neighborhood_iterator_imp.h" "$(@D)/numpy_include/numpy/_neighborhood_iterator_imp.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/_numpyconfig.h" "$(@D)/numpy_include/numpy/_numpyconfig.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/arrayobject.h" "$(@D)/numpy_include/numpy/arrayobject.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/arrayscalars.h" "$(@D)/numpy_include/numpy/arrayscalars.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/halffloat.h" "$(@D)/numpy_include/numpy/halffloat.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/multiarray_api.txt" "$(@D)/numpy_include/numpy/multiarray_api.txt" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/ndarrayobject.h" "$(@D)/numpy_include/numpy/ndarrayobject.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/ndarraytypes.h" "$(@D)/numpy_include/numpy/ndarraytypes.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/noprefix.h" "$(@D)/numpy_include/numpy/noprefix.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/npy_1_7_deprecated_api.h" "$(@D)/numpy_include/numpy/npy_1_7_deprecated_api.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/npy_3kcompat.h" "$(@D)/numpy_include/numpy/npy_3kcompat.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/npy_common.h" "$(@D)/numpy_include/numpy/npy_common.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/npy_cpu.h" "$(@D)/numpy_include/numpy/npy_cpu.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/npy_endian.h" "$(@D)/numpy_include/numpy/npy_endian.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/npy_interrupt.h" "$(@D)/numpy_include/numpy/npy_interrupt.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/npy_math.h" "$(@D)/numpy_include/numpy/npy_math.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/npy_no_deprecated_api.h" "$(@D)/numpy_include/numpy/npy_no_deprecated_api.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/npy_os.h" "$(@D)/numpy_include/numpy/npy_os.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/numpyconfig.h" "$(@D)/numpy_include/numpy/numpyconfig.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/old_defines.h" "$(@D)/numpy_include/numpy/old_defines.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/oldnumeric.h" "$(@D)/numpy_include/numpy/oldnumeric.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/ufunc_api.txt" "$(@D)/numpy_include/numpy/ufunc_api.txt" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/ufuncobject.h" "$(@D)/numpy_include/numpy/ufuncobject.h" && cp -f "/usr/lib/python3/dist-packages/numpy/core/include/numpy/utils.h" "$(@D)/numpy_include/numpy/utils.h"
+   """,
+)
diff --git a/third_party/toolchains/preconfig/ubuntu16.04/py3/WORKSPACE b/third_party/toolchains/preconfig/ubuntu16.04/py3/WORKSPACE
new file mode 100644
index 0000000000..1d298fefa3
--- /dev/null
+++ b/third_party/toolchains/preconfig/ubuntu16.04/py3/WORKSPACE
@@ -0,0 +1,2 @@
+# DO NOT EDIT: automatically generated WORKSPACE file for python_configure rule
+workspace(name = "local_config_python")
-- 
GitLab


From 1c7de3473c6b9eb6c5b3bd63ee75c203f7a5296c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sun, 6 Jan 2019 13:52:15 -0800
Subject: [PATCH 0289/1095] Fix typo; remove extra space.

PiperOrigin-RevId: 228076497
---
 tensorflow/contrib/training/python/training/training.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/training/python/training/training.py b/tensorflow/contrib/training/python/training/training.py
index c272a2ac14..fc6e38ab4a 100644
--- a/tensorflow/contrib/training/python/training/training.py
+++ b/tensorflow/contrib/training/python/training/training.py
@@ -419,7 +419,7 @@ def create_train_op(total_loss,
     update_ops = set(update_ops)
   if not global_update_ops.issubset(update_ops):
     logging.warning('update_ops in create_train_op does not contain all the '
-                    ' update_ops in GraphKeys.UPDATE_OPS')
+                    'update_ops in GraphKeys.UPDATE_OPS')
 
   # Make sure update_ops are computed before total_loss.
   if update_ops:
-- 
GitLab


From ebd5477f13c8d9585788b9721f821095fca7352e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sun, 6 Jan 2019 16:21:00 -0800
Subject: [PATCH 0290/1095] Make TF_DeleteServer check for __ANDROID__, too.

This should avoid a Wdelete-incomplete warning.

PiperOrigin-RevId: 228084360
---
 tensorflow/c/c_api.cc | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/tensorflow/c/c_api.cc b/tensorflow/c/c_api.cc
index 9580215a31..9f2f83920c 100644
--- a/tensorflow/c/c_api.cc
+++ b/tensorflow/c/c_api.cc
@@ -2881,6 +2881,9 @@ const char* TF_ServerTarget(TF_Server* server) {
 #endif
 }
 
-void TF_DeleteServer(TF_Server* server) { delete server; }
-
+void TF_DeleteServer(TF_Server* server) {
+#ifndef __ANDROID__
+  delete server;
+#endif
+}
 }  // end extern "C"
-- 
GitLab


From 38c91321421694432117b077294df43aa31d1193 Mon Sep 17 00:00:00 2001
From: Bixia Zheng <bixia@google.com>
Date: Sun, 6 Jan 2019 16:30:18 -0800
Subject: [PATCH 0291/1095] [XLA:GPU] Selectively exclude 0-2-1 transposed
 input parameters from the shared memory transpose implementation.

Previously, we either use the shared memory tranpose implementation for all the
0-2-1 tranposed input parameters in a fusion or not to use the shared memory
tranpose implement for any input parameters, if the kernel contains instructions
such as kReverse and kGather which can make the shared memory tranpose
implemetation unsafe. There are two problems in such an approach. First, the
set of instructions that can make the shared memory tranpose implementation
unsafe is more than the aforementioned two instructions. Second, even though a
fusion contains such instructions, it may still be safe to use shared memory
tranpose to implement some, if not all, 0-2-1 tranposed input parameters that
are not involved in the computation of such instructions. This change adds an
analysis to inspect the transitive users of each 0-2-1 tranposed input
parameter to decide whether it is safe to use the shared memory tranpose
implementation for the parameter.

Add two test cases.

PiperOrigin-RevId: 228084822
---
 .../xla/service/gpu/ir_emitter_unnested.cc    | 118 +++++++++++++++---
 .../gpu/tests/gpu_kernel_tiling_test.cc       |  78 ++++++++++--
 2 files changed, 170 insertions(+), 26 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
index 923b3d5945..f190af921e 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
@@ -2810,7 +2810,8 @@ void IrEmitterUnnested::EmitBlock(const TileGenerator& emit_one_tile,
 // unnested_hlo: The unnested hlo instruction for which the kernel is generated.
 //   Currently, these hlo instructions are supported: kLoop fusion, kCopy.
 // tiled_param_ids: The IDs for the parameters that are 0-2-1 transpose of
-//   other tensors with the same dimensions and need to be tiled and tranposed.
+//   other tensors with the same dimensions and are safe to be tranposed via
+//   the shared memory tranpose implementation.
 // mapping_scheme: The tiling scheme to use.
 // kernel_generator: Contains function objects for code generation, such as
 //   element generator, block prologue and epilogue generators.
@@ -2989,7 +2990,10 @@ LaunchDimensions IrEmitterUnnested::EmitKernel(
 
 // Emits a kernel for the given hlo instruction using a tiled 0-2-1 transpose
 // algorithm to improve the memory access patterns for the input parameters
-// with a shape that is a 0-2-1 transpose of the output tensor shape.
+// with a shape that is a 0-2-1 transpose of the output tensor shape. The caller
+// is responsible for making sure that it is safe to apply the shared memory
+// tranpose on the input parameters.
+//
 //
 // For the purpose of tiling, the output tensors have a logical shape of three
 // components 0-2-1 while the relevant input parameters have a logical shape
@@ -3040,26 +3044,99 @@ LaunchDimensions IrEmitterUnnested::EmitHlo021Tile(
 }
 
 namespace {
-// Returns true to indicate it is safe to use the tile based shared memory
-// transpose implementation to implement the kernel for the instruction.
+// A recursive function to inspect the users of a parameter to determine
+// whether it's safe for a parameter to participate in a shared-memory
+// transpose.
 //
-// An instruction is not safe for such an implementation if it can change the
-// element order of a tensor without changing the dimension of the tensor, and
-// the instruction has a corresponding elemental_ir_emitter.
-bool IsInstructionSafeForTileBasedTranspose(const HloInstruction* hlo) {
-  auto is_safe_for_tile_based_transpose = [&](const HloInstruction* instr) {
-    HloOpcode opcode = instr->opcode();
-    CHECK_NE(opcode, HloOpcode::kFusion);
-    return (opcode != HloOpcode::kReverse && opcode != HloOpcode::kGather);
-  };
+// Consider a fusion parameter P for which we might want to use a shmem
+// transpose.  If we do, we use a GPU thread block to preload a tile of P with
+// indices [z, y..y+31, x..x+31] to compute an output tile with the same indices
+// cooperatively, where z, y, x are the indices for the normalized input/output
+// tensor (see the document for FindTranspose021 for the definition of
+// normalized tensor for 0-2-1 transpose). This shmem transpose implementation
+// requires that the computation of the output tile only read elements within
+// the preload tile. If this is not true, we can't use a shmem transpose for P.
+//
+// If the computation of output element [z, y, x] only requires the element of
+// P with the same indices, the shmem tranpose implementation can be applied
+// to P safely. This is a sufficient but not necessary condition. We check all
+// the transitive users of P to see if we can find a user that may cause an
+// exception to the situation. If such a user is not found, we conclude that P
+// is safe for shmem transpose.
+//
+// This is trivially true for elementwise operations and some "data-movement"
+// ops like kTuple. However, it's not true for operations that can change the
+// dimensions of the inputs (e.g. pad, slice) and bitcast operation.
+// For example:
+//
+// fused_computation {
+//   param_0 = f32[64,64]{1,0} parameter(0)
+//   ROOT bitcast = f32[64,64]{0,1} bitcast(param_0)
+// }
+// The output element at logical address [0, 63] depends on the input element
+// at logical address [63, 0], which would not be within the shared-memory
+// block.
+//
+// TODO(bixia): In order to extend this for kInput fusion, that is reduction
+// with tranpose, we only need to end the use-chain checking with the input of
+// a reduce operations. In this case, the above description on "output" apply
+// to the result of such a use-chain, which provides the input to the reduce
+// operation.
+bool IsInstructionSafeForShmemTranspose(const HloInstruction* hlo) {
+  if (hlo->IsElementwise()) {
+    return absl::c_all_of(hlo->users(), [&](const HloInstruction* user) {
+      return IsInstructionSafeForShmemTranspose(user);
+    });
+  }
 
-  if (hlo->opcode() == HloOpcode::kFusion) {
-    return absl::c_all_of(hlo->fused_instructions_computation()->instructions(),
-                          is_safe_for_tile_based_transpose);
+  switch (hlo->opcode()) {
+    // Non-elementwise instructions that don't cause the shmem transpose
+    // to be unsafe, including the instructions that don't currently fuse.
+    case HloOpcode::kGetDimensionSize:
+      // The result of the operation doesn't rely on the content of the
+      // tensor. As such, there is no need to further inspect its users.
+      return true;
+    case HloOpcode::kGetTupleElement:
+    case HloOpcode::kMap:
+    case HloOpcode::kParameter:
+    case HloOpcode::kTuple:
+    case HloOpcode::kTupleSelect:
+      return absl::c_all_of(hlo->users(), [&](const HloInstruction* user) {
+        return IsInstructionSafeForShmemTranspose(user);
+      });
+
+    default:
+      return false;
   }
+}
 
-  return is_safe_for_tile_based_transpose(hlo);
+// Given a group of input parameters that are 0-2-1 tranpose of the outputs of
+// a fusion kernel, returns the input parameters that are safe for the shared
+// memory tranpose implementation.
+//
+// When a tile based shared memory transpose is used to implement an input with
+// 0-2-1 transpose, we preload a tile of the input elements
+// [z, y..y+31, x..x+31] to compute the output tile elements of the same
+// indices. Preloading the input tile this way is only safe when the computation
+// of the output tile elements do not need any input element outside the
+// preloaded tile. We inspect all the transitive users of the input parameter
+// up to the fusion root instruction to see if we can find any instruction
+// that can make preloading the input tile unsafe.
+std::vector<int64> FilterInputsForShmemTranspose(const HloInstruction* fusion,
+                                                 std::vector<int64> input_ids) {
+  std::vector<int64> filtered_input_ids;
+  for (int64 i = 0; i < input_ids.size(); ++i) {
+    const HloInstruction* input = fusion->fused_parameter(input_ids[i]);
+    if (IsInstructionSafeForShmemTranspose(input)) {
+      filtered_input_ids.push_back(input_ids[i]);
+    } else {
+      VLOG(10) << "Input not safe for shmem transpose " << input->ToString()
+               << "\n";
+    }
+  }
+  return filtered_input_ids;
 }
+
 }  // namespace
 
 bool IrEmitterUnnested::CheckAndEmitHloWithTile021(HloInstruction* hlo) {
@@ -3106,8 +3183,11 @@ bool IrEmitterUnnested::CheckAndEmitHloWithTile021(HloInstruction* hlo) {
     return false;
   }
 
-  if (!IsInstructionSafeForTileBasedTranspose(hlo)) {
-    return false;
+  if (opcode == HloOpcode::kFusion) {
+    params_012 = FilterInputsForShmemTranspose(hlo, params_012);
+    if (params_012.empty()) {
+      return false;
+    }
   }
 
   // Each of our shared memory tiles has 32*33 elements (so ~4kb, if the
diff --git a/tensorflow/compiler/xla/service/gpu/tests/gpu_kernel_tiling_test.cc b/tensorflow/compiler/xla/service/gpu/tests/gpu_kernel_tiling_test.cc
index a302b582ed..8004ebdc8e 100644
--- a/tensorflow/compiler/xla/service/gpu/tests/gpu_kernel_tiling_test.cc
+++ b/tensorflow/compiler/xla/service/gpu/tests/gpu_kernel_tiling_test.cc
@@ -65,7 +65,7 @@ TEST_F(GpuKernelTilingTest, UnnestedTransposeWithProperDimensionsTiled) {
   CompileAndVerifyIr(std::move(hlo_module),
                      R"(
 ; CHECK-LABEL: define void @copy
-; CHECK: tail call void @llvm.nvvm.barrier0()
+; CHECK: call void @llvm.nvvm.barrier0()
 ; CHECK: }
 )",
                      /*match_optimized_ir=*/true);
@@ -91,7 +91,7 @@ TEST_F(GpuKernelTilingTest, UnnestedTransposeWithSmallDimensionsNotTiled) {
   CompileAndVerifyIr(std::move(hlo_module),
                      R"(
 ; CHECK-LABEL: define void @copy
-; CHECK-NOT: tail call void @llvm.nvvm.barrier0()
+; CHECK-NOT: call void @llvm.nvvm.barrier0()
 ; CHECK: }
 )",
                      /*match_optimized_ir=*/true);
@@ -118,7 +118,7 @@ TEST_F(GpuKernelTilingTest, SimpleFusionWithTransposeTiled) {
   CompileAndVerifyIr(std::move(hlo_module),
                      R"(
 ; CHECK-LABEL: define void @fusion
-; CHECK: tail call void @llvm.nvvm.barrier0()
+; CHECK: call void @llvm.nvvm.barrier0()
 ; CHECK: }
 )",
                      /*match_optimized_ir=*/true);
@@ -152,7 +152,7 @@ TEST_F(GpuKernelTilingTest, MultipleOutputFusionWithOnePossibleTransposeTiled) {
   CompileAndVerifyIr(std::move(hlo_module),
                      R"(
 ; CHECK-LABEL: define void @fusion
-; CHECK: tail call void @llvm.nvvm.barrier0()
+; CHECK: call void @llvm.nvvm.barrier0()
 ; CHECK: }
 )",
                      /*match_optimized_ir=*/true);
@@ -187,13 +187,13 @@ TEST_F(GpuKernelTilingTest,
   CompileAndVerifyIr(std::move(hlo_module),
                      R"(
 ; CHECK-LABEL: define void @fusion
-; CHECK-NOT: tail call void @llvm.nvvm.barrier0()
+; CHECK-NOT: call void @llvm.nvvm.barrier0()
 ; CHECK: }
 )",
                      /*match_optimized_ir=*/true);
 }
 
-TEST_F(GpuKernelTilingTest, FusionTransposeWithReverseNotTiled) {
+TEST_F(GpuKernelTilingTest, TransposedInputWithUserReverseNotTiled) {
   const char *const kHloString = R"(
     HloModule FusionTransposeWithReverseNotTiled
     fused_computation.1 {
@@ -214,12 +214,76 @@ TEST_F(GpuKernelTilingTest, FusionTransposeWithReverseNotTiled) {
   CompileAndVerifyIr(std::move(hlo_module),
                      R"(
 ; CHECK-LABEL: define void @fusion
-; CHECK-NOT: tail call void @llvm.nvvm.barrier0()
+; CHECK-NOT: call void @llvm.nvvm.barrier0()
 ; CHECK: }
 )",
                      /*match_optimized_ir=*/true);
 }
 
+TEST_F(GpuKernelTilingTest, TransposedInputWithUserBitcastNotTiled) {
+  const char *const kHloString = R"(
+    HloModule TransposedInputWithUserBitcast
+
+    fused_computation {
+      param_0 = f32[20,20]{1,0} parameter(0)
+      ROOT bitcast = f32[20,20]{0,1} bitcast(param_0)
+    }
+
+    ENTRY kernel_entry {
+      parameter.0 = f32[20,20]{1,0} parameter(0)
+      ROOT fusion = f32[20,20]{0,1} fusion(parameter.0),
+        kind=kLoop, calls=fused_computation
+    })";
+
+  // Check that a call to llvm.nvvm.barrier0 is not generated.
+  auto hlo_module =
+      ParseHloString(kHloString, ConfigWithoutLayoutAssignment()).ValueOrDie();
+  CompileAndVerifyIr(std::move(hlo_module),
+                     R"(
+; CHECK-LABEL: define void @fusion
+; CHECK-NOT: call void @llvm.nvvm.barrier0()
+; CHECK: }
+)",
+                     /*match_optimized_ir=*/true);
+
+  // Check that the kernel runs correctly.
+  EXPECT_TRUE(RunAndCompareNoHloPasses(kHloString, ErrorSpec{0.0}));
+}
+
+TEST_F(GpuKernelTilingTest, TransposedInputWithoutUnsafeUseTiled) {
+  const char *const kHloString = R"(
+    HloModule TwoTransposedInputs
+
+    fused_computation {
+      param_0 = f32[64,64]{1,0} parameter(0)
+      param_1 = f32[64,64]{1,0} parameter(1)
+      bitcast = f32[64,64]{0,1} bitcast(param_0)
+      copy = f32[64,64]{0,1} copy(param_1)
+      ROOT tuple = (f32[64,64]{0,1}, f32[64,64]{0,1}) tuple(bitcast, copy)
+    }
+
+    ENTRY kernel_entry {
+      parameter.0 = f32[64,64]{1,0} parameter(0)
+      parameter.1 = f32[64,64]{1,0} parameter(1)
+      ROOT fusion = (f32[64,64]{0,1}, f32[64,64]{0,1})
+        fusion(parameter.0, parameter.1),
+        kind=kLoop, calls=fused_computation
+    })";
+
+  // Check that a call to llvm.nvvm.barrier0 is not generated.
+  auto hlo_module =
+      ParseHloString(kHloString, ConfigWithoutLayoutAssignment()).ValueOrDie();
+  CompileAndVerifyIr(std::move(hlo_module),
+                     R"(
+; CHECK-LABEL: define void @fusion
+; CHECK: call void @llvm.nvvm.barrier0()
+; CHECK: }
+)",
+                     /*match_optimized_ir=*/true);
+  // Check that the kernel runs correctly.
+  EXPECT_TRUE(RunAndCompareNoHloPasses(kHloString, ErrorSpec{0.0}));
+}
+
 }  // namespace
 }  // namespace gpu
 }  // namespace xla
-- 
GitLab


From 187a8cba72aae56ed8dfbe5c2d5779664b7683f2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sun, 6 Jan 2019 19:22:26 -0800
Subject: [PATCH 0292/1095] Change TensorFlow's resize_uninitialized wrapper to
 call libc++'s __resize_default_init extension when available.

PiperOrigin-RevId: 228095086
---
 tensorflow/core/BUILD              |  1 +
 tensorflow/core/lib/gtl/stl_util.h | 37 +++++++++++++++++++++++++-----
 2 files changed, 32 insertions(+), 6 deletions(-)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 7c9decbd09..5e6c8d4dd8 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -2193,6 +2193,7 @@ cc_library(
         ],
     }),
     deps = tf_additional_lib_deps() + [
+        "@com_google_absl//absl/meta:type_traits",
         "@com_google_absl//absl/strings",
         "//third_party/eigen3",
         "@com_google_absl//absl/base:core_headers",
diff --git a/tensorflow/core/lib/gtl/stl_util.h b/tensorflow/core/lib/gtl/stl_util.h
index ffeca4e88a..853a290bf6 100644
--- a/tensorflow/core/lib/gtl/stl_util.h
+++ b/tensorflow/core/lib/gtl/stl_util.h
@@ -23,9 +23,12 @@ limitations under the License.
 #include <iterator>
 #include <memory>
 #include <string>
+#include <type_traits>
 #include <utility>
 #include <vector>
 
+#include "absl/meta/type_traits.h"
+
 namespace tensorflow {
 namespace gtl {
 
@@ -48,16 +51,38 @@ inline const T* vector_as_array(const std::vector<T, Allocator>* v) {
   return v->data();
 }
 
+namespace gtl_internal {
+
+// HasMember is true_type or false_type, depending on whether or not
+// T has a __resize_default_init member. Resize will call the
+// __resize_default_init member if it exists, and will call the resize
+// member otherwise.
+template <typename string_type, typename = void>
+struct ResizeUninitializedTraits {
+  using HasMember = std::false_type;
+  static void Resize(string_type* s, size_t new_size) { s->resize(new_size); }
+};
+
+// __resize_default_init is provided by libc++ >= 8.0 and by Google's internal
+// ::string implementation.
+template <typename string_type>
+struct ResizeUninitializedTraits<
+    string_type, absl::void_t<decltype(std::declval<string_type&>()
+                                           .__resize_default_init(237))> > {
+  using HasMember = std::true_type;
+  static void Resize(string_type* s, size_t new_size) {
+    s->__resize_default_init(new_size);
+  }
+};
+
+}  // namespace gtl_internal
+
 // Like str->resize(new_size), except any new characters added to "*str" as a
 // result of resizing may be left uninitialized, rather than being filled with
 // '0' bytes. Typically used when code is then going to overwrite the backing
-// store of the string with known data. Uses a Google extension to ::string.
+// store of the string with known data.
 inline void STLStringResizeUninitialized(string* s, size_t new_size) {
-#if __google_stl_resize_uninitialized_string
-  s->resize_uninitialized(new_size);
-#else
-  s->resize(new_size);
-#endif
+  gtl_internal::ResizeUninitializedTraits<string>::Resize(s, new_size);
 }
 
 // Calls delete (non-array version) on the SECOND item (pointer) in each pair in
-- 
GitLab


From b8f2207ead5c6ce6e7e98479a88a08e4977aa83d Mon Sep 17 00:00:00 2001
From: Tom Hennigan <tomhennigan@google.com>
Date: Sun, 6 Jan 2019 23:46:22 -0800
Subject: [PATCH 0293/1095] Increase length of stack trace for original
 variable definition.

PiperOrigin-RevId: 228112246
---
 tensorflow/python/ops/variable_scope.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/variable_scope.py b/tensorflow/python/ops/variable_scope.py
index 64cdde3d85..35c00778ae 100644
--- a/tensorflow/python/ops/variable_scope.py
+++ b/tensorflow/python/ops/variable_scope.py
@@ -842,8 +842,11 @@ class _VariableStore(object):
         if isinstance(var, resource_variable_ops.ResourceVariable):
           raise ValueError(err_msg)
         tb = var.op.traceback[::-1]
-        # Throw away internal tf entries and only take a few lines.
-        tb = [x for x in tb if "tensorflow/python" not in x[0]][:3]
+        # Throw away internal tf entries and only take a few lines. In some
+        # cases the traceback can be longer (e.g. if someone uses factory
+        # functions to create variables) so we take more than needed in the
+        # default case.
+        tb = [x for x in tb if "tensorflow/python" not in x[0]][:5]
         raise ValueError("%s Originally defined at:\n\n%s" % (err_msg, "".join(
             traceback.format_list(tb))))
       found_var = self._vars[name]
-- 
GitLab


From 57d450732007e33db361ad8f16a1e8655a55dc39 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 7 Jan 2019 00:42:55 -0800
Subject: [PATCH 0294/1095] Internal change.

PiperOrigin-RevId: 228117862
---
 tensorflow/contrib/gan/BUILD | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tensorflow/contrib/gan/BUILD b/tensorflow/contrib/gan/BUILD
index 08df38ad27..db0868fb2c 100644
--- a/tensorflow/contrib/gan/BUILD
+++ b/tensorflow/contrib/gan/BUILD
@@ -2,7 +2,6 @@
 load("//tensorflow:tensorflow.bzl", "py_test")
 
 package(default_visibility = [
-    "//learning/brain/contrib/tfgan:__subpackages__",
     "//tensorflow:__subpackages__",
 ])
 
-- 
GitLab


From 134b9e580f639005f8e5986c53884183d6d9bde0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 7 Jan 2019 01:03:09 -0800
Subject: [PATCH 0295/1095] compat: Update forward compatibility horizon to
 2019-01-07

PiperOrigin-RevId: 228120298
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index b779c80aa7..dbad494e40 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 6)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 7)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 56ab6a2a7b224cb4a30e545035639ab84d82c58f Mon Sep 17 00:00:00 2001
From: Tom Hennigan <tomhennigan@google.com>
Date: Mon, 7 Jan 2019 01:31:17 -0800
Subject: [PATCH 0296/1095] Fix Template.variables for templates with empty
 name.

PiperOrigin-RevId: 228123554
---
 .../python/kernel_tests/template_test.py      | 15 ++++++++++
 tensorflow/python/ops/template.py             | 30 ++++---------------
 2 files changed, 20 insertions(+), 25 deletions(-)

diff --git a/tensorflow/python/kernel_tests/template_test.py b/tensorflow/python/kernel_tests/template_test.py
index 3b2a56bd1f..f587a7ec43 100644
--- a/tensorflow/python/kernel_tests/template_test.py
+++ b/tensorflow/python/kernel_tests/template_test.py
@@ -160,6 +160,21 @@ class TemplateTest(test.TestCase):
     self.assertEqual(1, len(result))
     self.assertNotEqual(len(first), len(result))
 
+  @test_util.run_in_graph_and_eager_modes
+  def test_template_with_empty_name(self):
+    tpl = template.make_template("", variable_scoped_function)
+    with variable_scope.variable_scope("outer"):
+      x = variable_scope.get_variable("x", [])
+      v = tpl()
+    self.assertEqual("outer/", tpl.variable_scope_name)
+    self.assertEqual("outer//dummy:0", v.name)
+    if context.executing_eagerly():
+      # In eager mode `x` is not visible to the template since the template does
+      # not rely on global collections.
+      self.assertEqual([v], tpl.variables)
+    else:
+      self.assertEqual([x, v], tpl.variables)
+
   @test_util.run_in_graph_and_eager_modes
   def test_template_with_name(self):
     tmpl1 = template.make_template("s1", variable_scoped_function)
diff --git a/tensorflow/python/ops/template.py b/tensorflow/python/ops/template.py
index 7c2d3be338..4eaa16a22c 100644
--- a/tensorflow/python/ops/template.py
+++ b/tensorflow/python/ops/template.py
@@ -387,8 +387,11 @@ class Template(checkpointable.CheckpointableBase):
     """Returns the variable scope name created by this Template."""
     if self._variable_scope:
       name = self._variable_scope.name
-      # To prevent partial matches on the scope_name, we add '/' at the end.
-      return name if name[-1] == "/" else name + "/"
+      if not name or name[-1] == "/":
+        return name
+      else:
+        # To prevent partial matches on the scope_name, we add '/' at the end.
+        return name + "/"
 
   @property
   def variables(self):
@@ -646,29 +649,6 @@ class EagerTemplate(Template):
         with self._template_store.as_default():
           return self._call_func(args, kwargs)
 
-  @property
-  def name(self):
-    """Returns the name given to this Template."""
-    return self._name
-
-  @property
-  def func(self):
-    """Returns the func given to this Template."""
-    return self._func
-
-  @property
-  def variable_scope(self):
-    """Returns the variable scope object created by this Template."""
-    return self._variable_scope
-
-  @property
-  def variable_scope_name(self):
-    """Returns the variable scope name created by this Template."""
-    if self._variable_scope:
-      name = self._variable_scope.name
-      # To prevent partial matches on the scope_name, we add '/' at the end.
-      return name if name[-1] == "/" else name + "/"
-
   @property
   def variables(self):
     """Returns the list of variables created by the Template."""
-- 
GitLab


From 48d0e4d813b914fc480c51de5c48903b9bcc4cc7 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 7 Jan 2019 01:46:24 -0800
Subject: [PATCH 0297/1095] Fixing pasta by not stripping the subdirectory.

PiperOrigin-RevId: 228125275
---
 third_party/pasta/BUILD.bazel    |  5 +++--
 third_party/pasta/build_defs.bzl | 12 ++++++++++++
 third_party/pasta/workspace.bzl  |  2 +-
 3 files changed, 16 insertions(+), 3 deletions(-)
 create mode 100644 third_party/pasta/build_defs.bzl

diff --git a/third_party/pasta/BUILD.bazel b/third_party/pasta/BUILD.bazel
index 6be33511e4..ade681b606 100644
--- a/third_party/pasta/BUILD.bazel
+++ b/third_party/pasta/BUILD.bazel
@@ -1,5 +1,6 @@
 # Description:
 #   AST-based python refactoring.
+load("@//third_party/pasta:build_defs.bzl", "copy_srcs")
 
 licenses(["notice"])  # Apache2
 
@@ -7,7 +8,7 @@ exports_files(["LICENSE"])
 
 py_library(
     name = "pasta",
-    srcs = [
+    srcs = copy_srcs([
         "__init__.py",
         "augment/__init__.py",
         "augment/errors.py",
@@ -23,7 +24,7 @@ py_library(
         "base/scope.py",
         "base/test_utils.py",
         "base/token_generator.py",
-    ],
+    ]),
     srcs_version = "PY2AND3",
     visibility = ["//visibility:public"],
 )
diff --git a/third_party/pasta/build_defs.bzl b/third_party/pasta/build_defs.bzl
new file mode 100644
index 0000000000..0a5316de40
--- /dev/null
+++ b/third_party/pasta/build_defs.bzl
@@ -0,0 +1,12 @@
+"""Skylark makros for building pasta."""
+
+def copy_srcs(srcs):
+    """Copies srcs from 'pasta' to parent directory."""
+    for src in srcs:
+        native.genrule(
+            name = src.replace(".", "_"),
+            srcs = ["pasta/" + src],
+            outs = [src],
+            cmd = "mkdir -p $$(dirname $@); cp $< $@",
+        )
+    return srcs
diff --git a/third_party/pasta/workspace.bzl b/third_party/pasta/workspace.bzl
index 063bdb7b98..e46cc4a45e 100644
--- a/third_party/pasta/workspace.bzl
+++ b/third_party/pasta/workspace.bzl
@@ -9,7 +9,7 @@ def repo():
             "https://mirror.bazel.build/github.com/google/pasta/archive/c3d72cdee6fc806251949e912510444d58d7413c.tar.gz",
             "https://github.com/google/pasta/archive/c3d72cdee6fc806251949e912510444d58d7413c.tar.gz",
         ],
-        strip_prefix = "pasta-c3d72cdee6fc806251949e912510444d58d7413c/pasta",
+        strip_prefix = "pasta-c3d72cdee6fc806251949e912510444d58d7413c",
         sha256 = "b5905f9cecc4b28363c563f3c4cb0545288bd35f7cc72c55066e97e53befc084",
         build_file = "//third_party/pasta:BUILD.bazel",
         system_build_file = "//third_party/pasta:BUILD.system",
-- 
GitLab


From f64f7f787d3596cb7c9228f131f06c159a0ec188 Mon Sep 17 00:00:00 2001
From: Tamara Norman <tamaranorman@google.com>
Date: Mon, 7 Jan 2019 02:38:36 -0800
Subject: [PATCH 0298/1095] Added option in conversion script to convert an
 entire directory in place when --inplace=True given as an argument

PiperOrigin-RevId: 228130782
---
 tensorflow/tools/compatibility/ast_edits.py   | 34 ++++++++++++++++++-
 .../tools/compatibility/tf_upgrade_v2_main.py | 10 +++++-
 2 files changed, 42 insertions(+), 2 deletions(-)

diff --git a/tensorflow/tools/compatibility/ast_edits.py b/tensorflow/tools/compatibility/ast_edits.py
index 0ec6c2ac8c..859fd95314 100644
--- a/tensorflow/tools/compatibility/ast_edits.py
+++ b/tensorflow/tools/compatibility/ast_edits.py
@@ -493,7 +493,7 @@ class ASTCodeUpgrader(object):
             process_errors)
 
   def process_tree(self, root_directory, output_root_directory,
-                   copy_other_files):
+                   copy_other_files, in_place):
     """Processes upgrades on an entire tree of python files in place.
 
     Note that only Python files. If you have custom code in other languages,
@@ -503,11 +503,20 @@ class ASTCodeUpgrader(object):
       root_directory: Directory to walk and process.
       output_root_directory: Directory to use as base.
       copy_other_files: Copy files that are not touched by this converter.
+      in_place: Allow the conversion of an entire directory in place.
 
     Returns:
       A tuple of files processed, the report string ofr all files, and errors
     """
 
+    if output_root_directory == root_directory:
+      if in_place:
+        return self.process_tree_inplace(root_directory)
+      else:
+        print("In order to copy a directory in place the `--inplace` input "
+              "arg must be set to `True`.")
+        sys.exit(1)
+
     # make sure output directory doesn't exist
     if output_root_directory and os.path.exists(output_root_directory):
       print("Output directory %r must not already exist." %
@@ -564,3 +573,26 @@ class ASTCodeUpgrader(object):
         os.makedirs(output_directory)
       shutil.copy(input_path, output_path)
     return file_count, report, tree_errors
+
+  def process_tree_inplace(self, root_directory):
+    """Process a directory of python files in place."""
+    files_to_process = []
+    for dir_name, _, file_list in os.walk(root_directory):
+      py_files = [os.path.join(dir_name,
+                               f) for f in file_list if f.endswith(".py")]
+      files_to_process += py_files
+
+    file_count = 0
+    tree_errors = []
+    report = ""
+    report += ("=" * 80) + "\n"
+    report += "Input tree: %r\n" % root_directory
+    report += ("=" * 80) + "\n"
+
+    for path in files_to_process:
+      file_count += 1
+      _, l_report, l_errors = self.process_file(path, path)
+      tree_errors += l_errors
+      report += l_report
+
+    return file_count, report, tree_errors
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_main.py b/tensorflow/tools/compatibility/tf_upgrade_v2_main.py
index 543d078642..870bc6f216 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_main.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_main.py
@@ -59,6 +59,14 @@ Simple usage:
             "copy the other files."),
       type=bool,
       default=True)
+  parser.add_argument(
+      "--inplace",
+      dest="in_place",
+      help=("If converting a whole tree of files, whether to "
+            "allow the conversion to be performed on the "
+            "files in the input tree."),
+      type=bool,
+      default=False)
   parser.add_argument(
       "--reportfile",
       dest="report_filename",
@@ -86,7 +94,7 @@ Simple usage:
           "--outtree=<output directory> argument is required when converting a "
           "file tree.")
     files_processed, report_text, errors = upgrade.process_tree(
-        args.input_tree, args.output_tree, args.copy_other_files)
+        args.input_tree, args.output_tree, args.copy_other_files, args.in_place)
   else:
     parser.print_help()
   if report_text:
-- 
GitLab


From 2703d8e84aced78a05ec7ad1ea0e5f4d0fe70f9f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 7 Jan 2019 07:03:10 -0800
Subject: [PATCH 0299/1095] Change //tensorflow/compiler/tf2xla/python:xla to a
 tf_custom_op_py_library

Previously it was a simple py_library what didn't linked in the op registration
code causing errors when no other dependency pulled in the actual
implementations.

PiperOrigin-RevId: 228158673
---
 tensorflow/compiler/tf2xla/ops/BUILD    | 17 ++++++++++++++++-
 tensorflow/compiler/tf2xla/python/BUILD |  7 ++++++-
 2 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/ops/BUILD b/tensorflow/compiler/tf2xla/ops/BUILD
index 4dce0a2102..7140b6a122 100644
--- a/tensorflow/compiler/tf2xla/ops/BUILD
+++ b/tensorflow/compiler/tf2xla/ops/BUILD
@@ -4,7 +4,11 @@ package(
 
 licenses(["notice"])  # Apache 2.0
 
-load("//tensorflow:tensorflow.bzl", "tf_gen_op_wrapper_py")
+load(
+    "//tensorflow:tensorflow.bzl",
+    "tf_custom_op_library",
+    "tf_gen_op_wrapper_py",
+)
 
 cc_library(
     name = "xla_ops",
@@ -24,3 +28,14 @@ tf_gen_op_wrapper_py(
         ":xla_ops",
     ],
 )
+
+tf_custom_op_library(
+    name = "_xla_ops.so",
+    srcs = [
+        "xla_ops.cc",
+    ],
+    deps = [
+        "//tensorflow/core:lib",
+        "@com_google_absl//absl/algorithm:container",
+    ],
+)
diff --git a/tensorflow/compiler/tf2xla/python/BUILD b/tensorflow/compiler/tf2xla/python/BUILD
index fef97b98c3..9abdb04d77 100644
--- a/tensorflow/compiler/tf2xla/python/BUILD
+++ b/tensorflow/compiler/tf2xla/python/BUILD
@@ -15,6 +15,7 @@ load(
     "//tensorflow/core:platform/default/build_config.bzl",
     "tf_py_clif_cc",
 )
+load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library")
 
 tf_py_clif_cc(
     name = "xla_op_registry",
@@ -27,9 +28,13 @@ tf_py_clif_cc(
     ],
 )
 
-py_library(
+tf_custom_op_py_library(
     name = "xla",
     srcs = ["xla.py"],
+    dso = ["//tensorflow/compiler/tf2xla/ops:_xla_ops.so"],
+    kernels = [
+        "//tensorflow/compiler/tf2xla/ops:xla_ops",
+    ],
     deps = [
         "//tensorflow/compiler/tf2xla/ops:gen_xla_ops",
         "//tensorflow/compiler/xla:xla_data_proto_py",
-- 
GitLab


From e55ec99c9ef627d84750853f9ae83196723852d5 Mon Sep 17 00:00:00 2001
From: Adam Weiss <adam@signal11.com>
Date: Thu, 13 Dec 2018 15:22:36 -0500
Subject: [PATCH 0300/1095] [XLA] Wrap
 LocalComputationBuilder::GetOrCreateLocalClient with StatusOr<T>

Prevents crashes when GPUs present, but below minimum capability level
---
 .../xla/python/local_computation_builder.cc   | 21 ++++++++++---------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/tensorflow/compiler/xla/python/local_computation_builder.cc b/tensorflow/compiler/xla/python/local_computation_builder.cc
index 5d191f5a18..6611c87673 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.cc
+++ b/tensorflow/compiler/xla/python/local_computation_builder.cc
@@ -101,7 +101,7 @@ int GetReplicaCount() {
   return g_replica_count;
 }
 
-LocalClient* GetOrCreateLocalClient() {
+StatusOr<LocalClient*> GetOrCreateLocalClient() {
   string* platform_name = GetPlatformNameString();
   tensorflow::mutex_lock lock(g_local_client_mutex);
   if (g_local_client != nullptr) {
@@ -110,7 +110,8 @@ LocalClient* GetOrCreateLocalClient() {
   LocalClientOptions options;
   options.set_platform(PlatformUtil::GetPlatform(*platform_name).ValueOrDie());
   options.set_number_of_replicas(g_replica_count);
-  g_local_client = ClientLibrary::GetOrCreateLocalClient(options).ValueOrDie();
+  TF_ASSIGN_OR_RETURN(g_local_client,
+                      ClientLibrary::GetOrCreateLocalClient(options));
   CHECK(g_local_client != nullptr);
   return g_local_client;
 }
@@ -118,7 +119,7 @@ LocalClient* GetOrCreateLocalClient() {
 Status TransferToInfeedLocal(const Literal& literal) {
   VLOG(1) << "Infeeding literal without replica number; shape: "
           << literal.shape();
-  LocalClient* client = GetOrCreateLocalClient();
+  TF_ASSIGN_OR_RETURN(LocalClient * client, GetOrCreateLocalClient());
   return client->TransferToInfeedLocal(literal, /*device_ordinal=*/0);
 }
 
@@ -126,7 +127,7 @@ Status TransferToInfeedLocalReplica(const Literal& literal,
                                     int replica_number) {
   VLOG(1) << "Infeeding shape " << literal.shape()
           << " to replica number: " << replica_number;
-  LocalClient* client = GetOrCreateLocalClient();
+  TF_ASSIGN_OR_RETURN(LocalClient * client, GetOrCreateLocalClient());
   TF_ASSIGN_OR_RETURN(int device_ordinal,
                       client->ReplicaNumberToDeviceOrdinal(replica_number));
   return client->TransferToInfeedLocal(literal, device_ordinal);
@@ -136,7 +137,7 @@ StatusOr<Literal> TransferFromOutfeedLocalReplica(const Shape& shape,
                                                   int replica_number) {
   VLOG(1) << "Outfeeding literal from replica number: " << replica_number
           << " shape: " << shape;
-  LocalClient* client = GetOrCreateLocalClient();
+  TF_ASSIGN_OR_RETURN(LocalClient * client, GetOrCreateLocalClient());
   TF_ASSIGN_OR_RETURN(int device_ordinal,
                       client->ReplicaNumberToDeviceOrdinal(replica_number));
   return client->TransferFromOutfeedLocal(shape, device_ordinal);
@@ -153,7 +154,7 @@ static StatusOr<ScopedShapedBuffer> ToBuffer(LocalClient* client,
 StatusOr<LocalShapedBuffer*> LocalShapedBuffer::FromLiteral(
     const Literal& argument, const absl::optional<Shape>& shape_with_layout,
     int replica_number) {
-  LocalClient* client = GetOrCreateLocalClient();
+  TF_ASSIGN_OR_RETURN(LocalClient * client, GetOrCreateLocalClient());
   TF_ASSIGN_OR_RETURN(int device_ordinal,
                       client->ReplicaNumberToDeviceOrdinal(replica_number));
   VLOG(1) << "Creating shaped buffer from literal on replica/ordinal: "
@@ -183,7 +184,7 @@ const Shape& LocalShapedBuffer::shape() const {
 }
 
 StatusOr<Literal> LocalShapedBuffer::ToLiteral() const {
-  LocalClient* client = GetOrCreateLocalClient();
+  TF_ASSIGN_OR_RETURN(LocalClient * client, GetOrCreateLocalClient());
   return client->ShapedBufferToLiteral(*shaped_buffer());
 }
 
@@ -319,7 +320,7 @@ CompiledLocalComputation::CompiledLocalComputation(
 
 StatusOr<LocalShapedBuffer*> CompiledLocalComputation::Execute(
     absl::Span<LocalShapedBuffer* const> argument_handles) {
-  LocalClient* client = GetOrCreateLocalClient();
+  TF_ASSIGN_OR_RETURN(LocalClient * client, GetOrCreateLocalClient());
   StatusOr<int> device_ordinal_status = client->ReplicaNumberToDeviceOrdinal(0);
   StatusOr<ScopedShapedBuffer> result_buffer_status;
   if (!device_ordinal_status.ok()) {
@@ -362,7 +363,7 @@ StatusOr<LocalShapedBuffer*> CompiledLocalComputation::Execute(
 
 StatusOr<LocalShapedBufferTuple*> CompiledLocalComputation::ExecutePerReplica(
     absl::Span<const std::vector<LocalShapedBuffer*>> argument_handles) {
-  LocalClient* client = GetOrCreateLocalClient();
+  TF_ASSIGN_OR_RETURN(LocalClient * client, GetOrCreateLocalClient());
   const int num_replicas = GetReplicaCount();
 
   if (argument_handles.size() != num_replicas) {
@@ -535,7 +536,7 @@ StatusOr<CompiledLocalComputation*> LocalComputation::Compile(
     argument_shape_pointers.push_back(&argument_shape);
   }
 
-  LocalClient* client = GetOrCreateLocalClient();
+  TF_ASSIGN_OR_RETURN(LocalClient * client, GetOrCreateLocalClient());
   ExecutableBuildOptions options;
   if (build_options != nullptr) {
     options = *build_options;
-- 
GitLab


From 36c3466356546f09198ce01982e6049dbf1a1afe Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 7 Jan 2019 10:31:04 -0800
Subject: [PATCH 0301/1095] Merged commit includes the following changes:
 228190834  by A. Unique TensorFlower<gardener@tensorflow.org>:

    Internal change

--
228187607  by A. Unique TensorFlower<gardener@tensorflow.org>:

    Split depthwise conv GPU code into multiple files

    This file was a bottleneck during compilation, often taking many minutes to compile. In local testing this change reduces the wall-clock build time for the depthwise conv GPU kernels from 163s to 105s.

--
228181904  by A. Unique TensorFlower<gardener@tensorflow.org>:

    Fix py2 and py3 of gen_git_version.py to fix 1.13 release.

    Byteify function to handle py2 and py3.

--
228161227  by A. Unique TensorFlower<gardener@tensorflow.org>:

    Create Tensorflow version of LLVM's opt.

    This is useful for iterating on individual passes in the OptimizationPassRegistry.
    This is run with a command of the form optimization_pass_runner --input_file_path=/tmp/input.pbtxt --output_file_path=/tmp/output.pbtxt --optimization_pass=NameOfGraphOptimizationPass

--

PiperOrigin-RevId: 228190834
---
 .../common_runtime/optimization_registry.h    |   4 +
 tensorflow/core/kernels/BUILD                 |  10 +-
 ...v_op_gpu.cu.cc => depthwise_conv_op_gpu.h} |  25 +-
 .../depthwise_conv_op_gpu_double.cu.cc        |  30 ++
 .../kernels/depthwise_conv_op_gpu_float.cu.cc |  30 ++
 .../kernels/depthwise_conv_op_gpu_half.cu.cc  |  30 ++
 tensorflow/opensource_only.files              | 416 +++++++++---------
 tensorflow/tools/git/gen_git_source.py        |   4 +-
 tensorflow/tools/optimization/BUILD           |  52 +++
 .../gpu_optimization_pass_runner_main.cc      |  60 +++
 .../optimization/optimization_pass_runner.cc  | 167 +++++++
 .../optimization/optimization_pass_runner.h   |  61 +++
 12 files changed, 661 insertions(+), 228 deletions(-)
 rename tensorflow/core/kernels/{depthwise_conv_op_gpu.cu.cc => depthwise_conv_op_gpu.h} (98%)
 create mode 100644 tensorflow/core/kernels/depthwise_conv_op_gpu_double.cu.cc
 create mode 100644 tensorflow/core/kernels/depthwise_conv_op_gpu_float.cu.cc
 create mode 100644 tensorflow/core/kernels/depthwise_conv_op_gpu_half.cu.cc
 create mode 100644 tensorflow/tools/optimization/BUILD
 create mode 100644 tensorflow/tools/optimization/gpu_optimization_pass_runner_main.cc
 create mode 100644 tensorflow/tools/optimization/optimization_pass_runner.cc
 create mode 100644 tensorflow/tools/optimization/optimization_pass_runner.h

diff --git a/tensorflow/core/common_runtime/optimization_registry.h b/tensorflow/core/common_runtime/optimization_registry.h
index 19a76529bb..0e31f389aa 100644
--- a/tensorflow/core/common_runtime/optimization_registry.h
+++ b/tensorflow/core/common_runtime/optimization_registry.h
@@ -95,6 +95,10 @@ class OptimizationPassRegistry {
   void Register(Grouping grouping, int phase,
                 std::unique_ptr<GraphOptimizationPass> pass);
 
+  const std::map<Grouping, GraphOptimizationPasses>& groups() {
+    return groups_;
+  }
+
   // Run all passes in grouping, ordered by phase, with the same
   // options.
   Status RunGrouping(Grouping grouping,
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index c72d04d3af..3891caf5d3 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -3695,7 +3695,15 @@ tf_kernel_library(
 
 tf_kernel_library(
     name = "depthwise_conv_op",
-    prefix = "depthwise_conv_op",
+    srcs = ["depthwise_conv_op.cc"],
+    hdrs = ["depthwise_conv_op.h"],
+    gpu_srcs = [
+        "depthwise_conv_op.h",
+        "depthwise_conv_op_gpu.h",
+        "depthwise_conv_op_gpu_double.cu.cc",
+        "depthwise_conv_op_gpu_float.cu.cc",
+        "depthwise_conv_op_gpu_half.cu.cc",
+    ],
     deps = [
         ":bounds_check",
         ":conv_ops",
diff --git a/tensorflow/core/kernels/depthwise_conv_op_gpu.cu.cc b/tensorflow/core/kernels/depthwise_conv_op_gpu.h
similarity index 98%
rename from tensorflow/core/kernels/depthwise_conv_op_gpu.cu.cc
rename to tensorflow/core/kernels/depthwise_conv_op_gpu.h
index e811968d27..098853e684 100644
--- a/tensorflow/core/kernels/depthwise_conv_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/depthwise_conv_op_gpu.h
@@ -13,6 +13,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#ifndef TENSORFLOW_CORE_KERNELS_DEPTHWISE_CONV_OP_GPU_H_
+#define TENSORFLOW_CORE_KERNELS_DEPTHWISE_CONV_OP_GPU_H_
+
 #if GOOGLE_CUDA
 #define EIGEN_USE_GPU
 
@@ -38,7 +41,7 @@ using Eigen::GpuDevice;
 
 // Returns whether depthwise convolution forward or backward input pass can be
 // performed using the faster ('Small') variant of the kernel.
-EIGEN_DEVICE_FUNC bool CanLaunchDepthwiseConv2dGPUSmall(
+inline EIGEN_DEVICE_FUNC bool CanLaunchDepthwiseConv2dGPUSmall(
     const DepthwiseArgs& args) {
   return args.depth_multiplier == 1 && args.stride == 1 && args.in_rows <= 32 &&
          args.in_cols <= 32 && args.in_rows == args.out_rows &&
@@ -51,7 +54,7 @@ EIGEN_DEVICE_FUNC bool CanLaunchDepthwiseConv2dGPUSmall(
 
 // Returns whether depthwise convolution backward filter pass can be performed
 // using the faster ('Small') variant of the kernel.
-EIGEN_DEVICE_FUNC bool CanLaunchDepthwiseConv2dBackpropFilterGPUSmall(
+inline EIGEN_DEVICE_FUNC bool CanLaunchDepthwiseConv2dBackpropFilterGPUSmall(
     const DepthwiseArgs& args, const int block_height) {
   return args.depth_multiplier == 1 && args.stride == 1 && args.in_rows <= 32 &&
          args.in_cols <= 32 && args.in_rows == args.out_rows &&
@@ -652,13 +655,12 @@ struct PseudoHalfType<Eigen::half> {
 };
 }  // namespace detail
 
-namespace {
 // Maps to float if T is __half, and to T otherwise.
 template <typename T>
 using PseudoHalfType = typename detail::PseudoHalfType<T>::Type;
 
 // Returns whether the context's GPU supports efficient fp16 math.
-bool HasFastHalfMath(OpKernelContext* ctx) {
+inline bool HasFastHalfMath(OpKernelContext* ctx) {
   int major, minor;
   ctx->op_device_context()
       ->stream()
@@ -669,7 +671,6 @@ bool HasFastHalfMath(OpKernelContext* ctx) {
   // GPUs before sm_53 don't support fp16 math, and sm_61's fp16 math is slow.
   return cuda_arch >= 530 && cuda_arch != 610;
 }
-}  // namespace
 
 template <typename T, DepthwiseConv2dDirection kDirection,
           int kKnownFilterWidth, int kKnownFilterHeight, int kBlockDepth,
@@ -808,10 +809,6 @@ void LaunchDepthwiseConvOp<GpuDevice, T>::operator()(OpKernelContext* ctx,
   }
 }
 
-template struct LaunchDepthwiseConvOp<GpuDevice, Eigen::half>;
-template struct LaunchDepthwiseConvOp<GpuDevice, float>;
-template struct LaunchDepthwiseConvOp<GpuDevice, double>;
-
 // A Cuda kernel to compute the depthwise convolution backprop w.r.t. input.
 template <typename T, int kKnownFilterWidth, int kKnownFilterHeight,
           int kKnownDepthMultiplier>
@@ -1030,10 +1027,6 @@ void LaunchDepthwiseConvBackpropInputOp<GpuDevice, T>::operator()(
   }
 }
 
-template struct LaunchDepthwiseConvBackpropInputOp<GpuDevice, Eigen::half>;
-template struct LaunchDepthwiseConvBackpropInputOp<GpuDevice, float>;
-template struct LaunchDepthwiseConvBackpropInputOp<GpuDevice, double>;
-
 // A Cuda kernel to compute the depthwise convolution backprop w.r.t. filter.
 template <typename T, int kKnownFilterWidth, int kKnownFilterHeight,
           int kKnownDepthMultiplier>
@@ -1803,9 +1796,7 @@ void LaunchDepthwiseConvBackpropFilterOp<GpuDevice, T>::operator()(
                  ctx, args, out_backprop, input, filter_backprop, data_format));
   }
 }
-
-template struct LaunchDepthwiseConvBackpropFilterOp<GpuDevice, Eigen::half>;
-template struct LaunchDepthwiseConvBackpropFilterOp<GpuDevice, float>;
-template struct LaunchDepthwiseConvBackpropFilterOp<GpuDevice, double>;
 }  // namespace tensorflow
 #endif  // GOOGLE_CUDA
+
+#endif  // TENSORFLOW_CORE_KERNELS_DEPTHWISE_CONV_OP_GPU_H_
diff --git a/tensorflow/core/kernels/depthwise_conv_op_gpu_double.cu.cc b/tensorflow/core/kernels/depthwise_conv_op_gpu_double.cu.cc
new file mode 100644
index 0000000000..073e7cf269
--- /dev/null
+++ b/tensorflow/core/kernels/depthwise_conv_op_gpu_double.cu.cc
@@ -0,0 +1,30 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/depthwise_conv_op.h"
+#include "tensorflow/core/kernels/depthwise_conv_op_gpu.h"
+
+namespace tensorflow {
+using Eigen::GpuDevice;
+
+template struct LaunchDepthwiseConvOp<GpuDevice, double>;
+template struct LaunchDepthwiseConvBackpropInputOp<GpuDevice, double>;
+template struct LaunchDepthwiseConvBackpropFilterOp<GpuDevice, double>;
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/depthwise_conv_op_gpu_float.cu.cc b/tensorflow/core/kernels/depthwise_conv_op_gpu_float.cu.cc
new file mode 100644
index 0000000000..4b0e15e476
--- /dev/null
+++ b/tensorflow/core/kernels/depthwise_conv_op_gpu_float.cu.cc
@@ -0,0 +1,30 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/depthwise_conv_op.h"
+#include "tensorflow/core/kernels/depthwise_conv_op_gpu.h"
+
+namespace tensorflow {
+using Eigen::GpuDevice;
+
+template struct LaunchDepthwiseConvOp<GpuDevice, float>;
+template struct LaunchDepthwiseConvBackpropInputOp<GpuDevice, float>;
+template struct LaunchDepthwiseConvBackpropFilterOp<GpuDevice, float>;
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/depthwise_conv_op_gpu_half.cu.cc b/tensorflow/core/kernels/depthwise_conv_op_gpu_half.cu.cc
new file mode 100644
index 0000000000..2db9fa4dff
--- /dev/null
+++ b/tensorflow/core/kernels/depthwise_conv_op_gpu_half.cu.cc
@@ -0,0 +1,30 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/depthwise_conv_op.h"
+#include "tensorflow/core/kernels/depthwise_conv_op_gpu.h"
+
+namespace tensorflow {
+using Eigen::GpuDevice;
+
+template struct LaunchDepthwiseConvOp<GpuDevice, Eigen::half>;
+template struct LaunchDepthwiseConvBackpropInputOp<GpuDevice, Eigen::half>;
+template struct LaunchDepthwiseConvBackpropFilterOp<GpuDevice, Eigen::half>;
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/opensource_only.files b/tensorflow/opensource_only.files
index e00d063cfe..40f87d6c36 100644
--- a/tensorflow/opensource_only.files
+++ b/tensorflow/opensource_only.files
@@ -1,248 +1,248 @@
-tensorflow/contrib/tpu/profiler/pip_package/BUILD
-tensorflow/contrib/tpu/profiler/pip_package/setup.py
-tensorflow/contrib/tpu/profiler/pip_package/README
-tensorflow/contrib/tpu/profiler/pip_package/build_pip_package.sh
-tensorflow/contrib/tpu/profiler/pip_package/cloud_tpu_profiler/main.py
-tensorflow/contrib/tpu/profiler/pip_package/cloud_tpu_profiler/__init__.py
-tensorflow/contrib/mpi/BUILD
-tensorflow/tools/ci_build/remote/BUILD
-tensorflow/tools/pip_package/README
-tensorflow/tools/pip_package/MANIFEST.in
-tensorflow/tools/pip_package/simple_console.py
-tensorflow/tools/pip_package/build_pip_package.sh
-tensorflow/tools/pip_package/check_load_py_test.py
-tensorflow/tools/pip_package/pip_smoke_test.py
-tensorflow/tools/pip_package/simple_console_for_windows.py
-tensorflow/tools/pip_package/setup.py
-tensorflow/tools/pip_package/BUILD
-tensorflow/tools/lib_package/concat_licenses.sh
-tensorflow/tools/lib_package/libtensorflow_test.c
-tensorflow/tools/lib_package/LibTensorFlowTest.java
-tensorflow/tools/lib_package/BUILD
-tensorflow/tools/lib_package/libtensorflow_test.sh
-tensorflow/tools/lib_package/README.md
-tensorflow/tools/lib_package/libtensorflow_java_test.sh
-tensorflow/tools/def_file_filter/def_file_filter_configure.bzl
-tensorflow/tools/def_file_filter/BUILD
-tensorflow/tools/def_file_filter/BUILD.tpl
-tensorflow/tools/def_file_filter/def_file_filter.py.tpl
-tensorflow/third_party/mkl/MKL_LICENSE
-tensorflow/third_party/mkl/LICENSE
-tensorflow/third_party/mkl/BUILD
-tensorflow/third_party/mkl/mkl.BUILD
-tensorflow/third_party/mkl/build_defs.bzl
-tensorflow/third_party/backports_weakref.BUILD
-tensorflow/third_party/toolchains/clang6/BUILD
+tensorflow/api_template_v1.__init__.py
+tensorflow/compat_template_v1.__init__.py
+tensorflow/stream_executor/BUILD
+tensorflow/third_party/curl.BUILD
+tensorflow/third_party/snappy.BUILD
+tensorflow/third_party/tflite_mobilenet_float.BUILD
+tensorflow/third_party/libxsmm.BUILD
+tensorflow/third_party/cub.BUILD
+tensorflow/third_party/repo.bzl
+tensorflow/third_party/systemlibs/curl.BUILD
+tensorflow/third_party/systemlibs/snappy.BUILD
+tensorflow/third_party/systemlibs/protobuf.BUILD
+tensorflow/third_party/systemlibs/absl_py.BUILD
+tensorflow/third_party/systemlibs/google_cloud_cpp.google.cloud.bigtable.BUILD
+tensorflow/third_party/systemlibs/astor.BUILD
+tensorflow/third_party/systemlibs/absl_py.absl.flags.BUILD
+tensorflow/third_party/systemlibs/png.BUILD
+tensorflow/third_party/systemlibs/BUILD.tpl
+tensorflow/third_party/systemlibs/boringssl.BUILD
+tensorflow/third_party/systemlibs/absl_py.absl.testing.BUILD
+tensorflow/third_party/systemlibs/swig.BUILD
+tensorflow/third_party/systemlibs/double_conversion.BUILD
+tensorflow/third_party/systemlibs/gif.BUILD
+tensorflow/third_party/systemlibs/cython.BUILD
+tensorflow/third_party/systemlibs/google_cloud_cpp.BUILD
+tensorflow/third_party/systemlibs/zlib.BUILD
+tensorflow/third_party/systemlibs/grpc.BUILD
+tensorflow/third_party/systemlibs/re2.BUILD
+tensorflow/third_party/systemlibs/build_defs.bzl.tpl
+tensorflow/third_party/systemlibs/protobuf.bzl
+tensorflow/third_party/systemlibs/gast.BUILD
+tensorflow/third_party/systemlibs/syslibs_configure.bzl
+tensorflow/third_party/systemlibs/nsync.BUILD
+tensorflow/third_party/systemlibs/jsoncpp.BUILD
+tensorflow/third_party/systemlibs/six.BUILD
+tensorflow/third_party/systemlibs/lmdb.BUILD
+tensorflow/third_party/systemlibs/BUILD
+tensorflow/third_party/systemlibs/googleapis.BUILD
+tensorflow/third_party/systemlibs/sqlite.BUILD
+tensorflow/third_party/systemlibs/termcolor.BUILD
+tensorflow/third_party/systemlibs/pcre.BUILD
+tensorflow/third_party/tflite_mobilenet_quant.BUILD
+tensorflow/third_party/common.bzl
+tensorflow/third_party/boringssl/BUILD
+tensorflow/third_party/kafka/config.patch
+tensorflow/third_party/kafka/BUILD
+tensorflow/third_party/android/android_configure.bzl
+tensorflow/third_party/android/android_configure.BUILD.tpl
+tensorflow/third_party/android/android.bzl.tpl
+tensorflow/third_party/android/BUILD
+tensorflow/third_party/astor.BUILD
+tensorflow/third_party/icu/udata.patch
+tensorflow/third_party/tflite_mobilenet.BUILD
+tensorflow/third_party/png.BUILD
+tensorflow/third_party/mpi/.gitignore
+tensorflow/third_party/mpi/BUILD
+tensorflow/third_party/nanopb.BUILD
+tensorflow/third_party/arm_neon_2_x86_sse.BUILD
+tensorflow/third_party/linenoise.BUILD
+tensorflow/third_party/llvm/llvm.autogenerated.BUILD
+tensorflow/third_party/llvm/llvm.bzl
+tensorflow/third_party/llvm/BUILD
+tensorflow/third_party/llvm/expand_cmake_vars.py
+tensorflow/third_party/swig.BUILD
+tensorflow/third_party/png_fix_rpi.patch
+tensorflow/third_party/double_conversion.BUILD
+tensorflow/third_party/gif.BUILD
+tensorflow/third_party/grpc/BUILD
+tensorflow/third_party/cython.BUILD
+tensorflow/third_party/eigen.BUILD
+tensorflow/third_party/codegen.BUILD
+tensorflow/third_party/zlib.BUILD
+tensorflow/third_party/tensorrt/BUILD.tpl
+tensorflow/third_party/tensorrt/tensorrt_configure.bzl
+tensorflow/third_party/tensorrt/build_defs.bzl.tpl
+tensorflow/third_party/tensorrt/remote.BUILD.tpl
+tensorflow/third_party/tensorrt/LICENSE
+tensorflow/third_party/tensorrt/BUILD
+tensorflow/third_party/tflite_ovic_testdata.BUILD
+tensorflow/third_party/fft2d/fft.h
+tensorflow/third_party/fft2d/fft2d.BUILD
+tensorflow/third_party/fft2d/LICENSE
+tensorflow/third_party/fft2d/BUILD
 tensorflow/third_party/toolchains/clang6/README.md
 tensorflow/third_party/toolchains/clang6/repo.bzl
-tensorflow/third_party/toolchains/clang6/CROSSTOOL.tpl
 tensorflow/third_party/toolchains/clang6/clang.BUILD
-tensorflow/third_party/toolchains/preconfig/ubuntu14.04/tensorrt5/BUILD
+tensorflow/third_party/toolchains/clang6/CROSSTOOL.tpl
+tensorflow/third_party/toolchains/clang6/BUILD
+tensorflow/third_party/toolchains/preconfig/generate/archives.bzl
+tensorflow/third_party/toolchains/preconfig/generate/workspace.bzl
+tensorflow/third_party/toolchains/preconfig/generate/generate.bzl
+tensorflow/third_party/toolchains/preconfig/generate/containers.bzl
+tensorflow/third_party/toolchains/preconfig/generate/BUILD
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/tensorrt5/build_defs.bzl
+tensorflow/third_party/toolchains/preconfig/ubuntu14.04/tensorrt5/BUILD
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/py3/BUILD
-tensorflow/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/BUILD
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc/BUILD
+tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/build_defs.bzl
+tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/BUILD
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/cuda/build_defs.bzl
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/cuda/BUILD
+tensorflow/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/BUILD
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda10.0/BUILD
-tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/build_defs.bzl
-tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/BUILD
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/nccl2/BUILD
-tensorflow/third_party/toolchains/preconfig/generate/workspace.bzl
-tensorflow/third_party/toolchains/preconfig/generate/containers.bzl
-tensorflow/third_party/toolchains/preconfig/generate/generate.bzl
-tensorflow/third_party/toolchains/preconfig/generate/archives.bzl
-tensorflow/third_party/toolchains/preconfig/generate/BUILD
-tensorflow/third_party/toolchains/preconfig/ubuntu16.04/clang/BUILD
-tensorflow/third_party/toolchains/preconfig/ubuntu16.04/clang/dummy_toolchain.bzl
-tensorflow/third_party/toolchains/preconfig/ubuntu16.04/py3/BUILD
-tensorflow/third_party/toolchains/preconfig/win_1803/bazel_018/BUILD
-tensorflow/third_party/toolchains/preconfig/win_1803/bazel_018/dummy_toolchain.bzl
 tensorflow/third_party/toolchains/preconfig/win_1803/py36/BUILD
 tensorflow/third_party/toolchains/preconfig/win_1803/BUILD
-tensorflow/third_party/toolchains/gpus/cuda/build_defs.bzl
-tensorflow/third_party/toolchains/gpus/cuda/BUILD
-tensorflow/third_party/toolchains/gpus/cuda/cuda/cuda_config.h
-tensorflow/third_party/toolchains/gpus/crosstool/BUILD
-tensorflow/third_party/toolchains/gpus/crosstool/CROSSTOOL
-tensorflow/third_party/toolchains/gpus/py/BUILD
-tensorflow/third_party/toolchains/cpus/arm/arm_compiler_configure.bzl
+tensorflow/third_party/toolchains/preconfig/win_1803/bazel_018/dummy_toolchain.bzl
+tensorflow/third_party/toolchains/preconfig/win_1803/bazel_018/BUILD
+tensorflow/third_party/toolchains/preconfig/ubuntu16.04/py3/BUILD
+tensorflow/third_party/toolchains/preconfig/ubuntu16.04/clang/dummy_toolchain.bzl
+tensorflow/third_party/toolchains/preconfig/ubuntu16.04/clang/BUILD
+tensorflow/third_party/toolchains/cpus/py3/BUILD
 tensorflow/third_party/toolchains/cpus/arm/CROSSTOOL.tpl
 tensorflow/third_party/toolchains/cpus/arm/BUILD
-tensorflow/third_party/toolchains/cpus/py3/BUILD
+tensorflow/third_party/toolchains/cpus/arm/arm_compiler_configure.bzl
 tensorflow/third_party/toolchains/cpus/py/BUILD
+tensorflow/third_party/toolchains/gpus/crosstool/CROSSTOOL
+tensorflow/third_party/toolchains/gpus/crosstool/BUILD
+tensorflow/third_party/toolchains/gpus/cuda/build_defs.bzl
+tensorflow/third_party/toolchains/gpus/cuda/cuda/cuda_config.h
+tensorflow/third_party/toolchains/gpus/cuda/BUILD
+tensorflow/third_party/toolchains/gpus/py/BUILD
 tensorflow/third_party/toolchains/BUILD
-tensorflow/third_party/gpus/BUILD
+tensorflow/third_party/nccl/archive.BUILD
+tensorflow/third_party/nccl/nccl_configure.bzl
+tensorflow/third_party/nccl/system.BUILD.tpl
+tensorflow/third_party/nccl/build_defs.bzl.tpl
+tensorflow/third_party/nccl/LICENSE
+tensorflow/third_party/nccl/BUILD
+tensorflow/third_party/farmhash.BUILD
+tensorflow/third_party/sycl/crosstool/BUILD
+tensorflow/third_party/backports_weakref.BUILD
+tensorflow/third_party/gast.BUILD
+tensorflow/third_party/clang_toolchain/cc_configure_clang.bzl
+tensorflow/third_party/clang_toolchain/download_clang.bzl
+tensorflow/third_party/clang_toolchain/BUILD
+tensorflow/third_party/mpi_collectives/BUILD
+tensorflow/third_party/jsoncpp.BUILD
+tensorflow/third_party/mkl_dnn/mkldnn.BUILD
+tensorflow/third_party/mkl_dnn/LICENSE
+tensorflow/third_party/python_runtime/BUILD
+tensorflow/third_party/ngraph/ngraph.BUILD
+tensorflow/third_party/ngraph/nlohmann_json.BUILD
+tensorflow/third_party/ngraph/build_defs.bzl
+tensorflow/third_party/ngraph/ngraph_tf.BUILD
+tensorflow/third_party/ngraph/NGRAPH_LICENSE
+tensorflow/third_party/ngraph/tbb.BUILD
+tensorflow/third_party/ngraph/LICENSE
+tensorflow/third_party/ngraph/BUILD
+tensorflow/third_party/gpus/crosstool/windows/msvc_wrapper_for_nvcc.py.tpl
+tensorflow/third_party/gpus/crosstool/CROSSTOOL.tpl
+tensorflow/third_party/gpus/crosstool/BUILD.tpl
+tensorflow/third_party/gpus/crosstool/remote.BUILD.tpl
 tensorflow/third_party/gpus/crosstool/clang/bin/crosstool_wrapper_driver_rocm.tpl
 tensorflow/third_party/gpus/crosstool/clang/bin/crosstool_wrapper_driver_is_not_gcc.tpl
-tensorflow/third_party/gpus/crosstool/CROSSTOOL.tpl
 tensorflow/third_party/gpus/crosstool/CROSSTOOL_hipcc.tpl
 tensorflow/third_party/gpus/crosstool/LICENSE
-tensorflow/third_party/gpus/crosstool/remote.BUILD.tpl
-tensorflow/third_party/gpus/crosstool/windows/msvc_wrapper_for_nvcc.py.tpl
-tensorflow/third_party/gpus/crosstool/BUILD.tpl
 tensorflow/third_party/gpus/crosstool/BUILD
-tensorflow/third_party/gpus/cuda/LICENSE
+tensorflow/third_party/gpus/rocm/BUILD.tpl
+tensorflow/third_party/gpus/rocm/rocm_config.h.tpl
+tensorflow/third_party/gpus/rocm/build_defs.bzl.tpl
+tensorflow/third_party/gpus/rocm/BUILD
 tensorflow/third_party/gpus/cuda/BUILD.tpl
+tensorflow/third_party/gpus/cuda/build_defs.bzl.tpl
+tensorflow/third_party/gpus/cuda/remote.BUILD.tpl
 tensorflow/third_party/gpus/cuda/BUILD.windows.tpl
 tensorflow/third_party/gpus/cuda/cuda_config.h.tpl
-tensorflow/third_party/gpus/cuda/remote.BUILD.tpl
+tensorflow/third_party/gpus/cuda/LICENSE
 tensorflow/third_party/gpus/cuda/BUILD
-tensorflow/third_party/gpus/cuda/build_defs.bzl.tpl
-tensorflow/third_party/gpus/rocm/rocm_config.h.tpl
-tensorflow/third_party/gpus/rocm/BUILD
-tensorflow/third_party/gpus/rocm/BUILD.tpl
-tensorflow/third_party/gpus/rocm/build_defs.bzl.tpl
 tensorflow/third_party/gpus/cuda_configure.bzl
 tensorflow/third_party/gpus/rocm_configure.bzl
-tensorflow/third_party/snappy.BUILD
-tensorflow/third_party/cython.BUILD
-tensorflow/third_party/farmhash.BUILD
-tensorflow/third_party/eigen3/Eigen/Cholesky
-tensorflow/third_party/eigen3/Eigen/QR
-tensorflow/third_party/eigen3/Eigen/LU
+tensorflow/third_party/gpus/BUILD
+tensorflow/third_party/py/python_configure.bzl
+tensorflow/third_party/py/BUILD.tpl
+tensorflow/third_party/py/numpy/BUILD
+tensorflow/third_party/py/remote.BUILD.tpl
+tensorflow/third_party/py/BUILD
+tensorflow/third_party/mkl/build_defs.bzl
+tensorflow/third_party/mkl/mkl.BUILD
+tensorflow/third_party/mkl/MKL_LICENSE
+tensorflow/third_party/mkl/LICENSE
+tensorflow/third_party/mkl/BUILD
+tensorflow/third_party/com_google_absl.BUILD
+tensorflow/third_party/six.BUILD
+tensorflow/third_party/lmdb.BUILD
+tensorflow/third_party/BUILD
+tensorflow/third_party/googleapis.BUILD
+tensorflow/third_party/sqlite.BUILD
+tensorflow/third_party/termcolor.BUILD
+tensorflow/third_party/protobuf/BUILD
+tensorflow/third_party/pcre.BUILD
+tensorflow/third_party/git/BUILD.tpl
+tensorflow/third_party/git/git_configure.bzl
+tensorflow/third_party/git/BUILD
+tensorflow/third_party/pprof.BUILD
+tensorflow/third_party/tflite_smartreply.BUILD
 tensorflow/third_party/eigen3/Eigen/Core
-tensorflow/third_party/eigen3/Eigen/SVD
+tensorflow/third_party/eigen3/Eigen/Cholesky
 tensorflow/third_party/eigen3/Eigen/Eigenvalues
-tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/FixedPoint
+tensorflow/third_party/eigen3/Eigen/SVD
+tensorflow/third_party/eigen3/Eigen/LU
+tensorflow/third_party/eigen3/Eigen/QR
+tensorflow/third_party/eigen3/unsupported/Eigen/MatrixFunctions
 tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/Tensor
-tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/PacketMathAVX512.h
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/ThreadPool
 tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/TypeCastingAVX512.h
-tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/PacketMathAVX2.h
 tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/MatMatProduct.h
-tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/FixedPointTypes.h
-tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/MatVecProduct.h
 tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/MatMatProductNEON.h
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/PacketMathAVX512.h
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/MatVecProduct.h
 tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/TypeCastingAVX2.h
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/FixedPointTypes.h
 tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/MatMatProductAVX2.h
-tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/ThreadPool
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/PacketMathAVX2.h
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/FixedPoint
 tensorflow/third_party/eigen3/unsupported/Eigen/SpecialFunctions
-tensorflow/third_party/eigen3/unsupported/Eigen/MatrixFunctions
 tensorflow/third_party/eigen3/LICENSE
 tensorflow/third_party/eigen3/BUILD
-tensorflow/third_party/systemlibs/build_defs.bzl.tpl
-tensorflow/third_party/systemlibs/absl_py.BUILD
-tensorflow/third_party/systemlibs/curl.BUILD
-tensorflow/third_party/systemlibs/termcolor.BUILD
-tensorflow/third_party/systemlibs/absl_py.absl.flags.BUILD
-tensorflow/third_party/systemlibs/grpc.BUILD
-tensorflow/third_party/systemlibs/swig.BUILD
-tensorflow/third_party/systemlibs/protobuf.bzl
-tensorflow/third_party/systemlibs/protobuf.BUILD
-tensorflow/third_party/systemlibs/BUILD
-tensorflow/third_party/systemlibs/google_cloud_cpp.BUILD
-tensorflow/third_party/systemlibs/astor.BUILD
-tensorflow/third_party/systemlibs/six.BUILD
-tensorflow/third_party/systemlibs/absl_py.absl.testing.BUILD
-tensorflow/third_party/systemlibs/boringssl.BUILD
-tensorflow/third_party/systemlibs/nsync.BUILD
-tensorflow/third_party/systemlibs/google_cloud_cpp.google.cloud.bigtable.BUILD
-tensorflow/third_party/systemlibs/gif.BUILD
-tensorflow/third_party/systemlibs/pcre.BUILD
-tensorflow/third_party/systemlibs/BUILD.tpl
-tensorflow/third_party/systemlibs/snappy.BUILD
-tensorflow/third_party/systemlibs/gast.BUILD
-tensorflow/third_party/systemlibs/cython.BUILD
-tensorflow/third_party/systemlibs/double_conversion.BUILD
-tensorflow/third_party/systemlibs/zlib.BUILD
-tensorflow/third_party/systemlibs/jsoncpp.BUILD
-tensorflow/third_party/systemlibs/re2.BUILD
-tensorflow/third_party/systemlibs/lmdb.BUILD
-tensorflow/third_party/systemlibs/googleapis.BUILD
-tensorflow/third_party/systemlibs/png.BUILD
-tensorflow/third_party/systemlibs/syslibs_configure.bzl
-tensorflow/third_party/systemlibs/sqlite.BUILD
-tensorflow/third_party/python_runtime/BUILD
-tensorflow/third_party/sycl/crosstool/BUILD
-tensorflow/third_party/ngraph/LICENSE
-tensorflow/third_party/ngraph/tbb.BUILD
-tensorflow/third_party/ngraph/BUILD
-tensorflow/third_party/ngraph/ngraph.BUILD
-tensorflow/third_party/ngraph/build_defs.bzl
-tensorflow/third_party/ngraph/NGRAPH_LICENSE
-tensorflow/third_party/ngraph/ngraph_tf.BUILD
-tensorflow/third_party/ngraph/nlohmann_json.BUILD
-tensorflow/third_party/clang_toolchain/download_clang.bzl
-tensorflow/third_party/clang_toolchain/BUILD
-tensorflow/third_party/clang_toolchain/cc_configure_clang.bzl
-tensorflow/third_party/gast.BUILD
-tensorflow/third_party/llvm/BUILD
-tensorflow/third_party/llvm/expand_cmake_vars.py
-tensorflow/third_party/llvm/llvm.autogenerated.BUILD
-tensorflow/third_party/llvm/llvm.bzl
-tensorflow/third_party/icu/udata.patch
-tensorflow/third_party/nccl/archive.BUILD
-tensorflow/third_party/nccl/LICENSE
-tensorflow/third_party/nccl/system.BUILD.tpl
-tensorflow/third_party/nccl/nccl_configure.bzl
-tensorflow/third_party/nccl/build_defs.bzl.tpl
-tensorflow/third_party/nccl/BUILD
-tensorflow/third_party/fft2d/BUILD
-tensorflow/third_party/fft2d/fft.h
-tensorflow/third_party/fft2d/LICENSE
-tensorflow/third_party/fft2d/fft2d.BUILD
-tensorflow/third_party/boringssl/BUILD
-tensorflow/third_party/mpi/.gitignore
-tensorflow/third_party/mpi/BUILD
-tensorflow/third_party/tensorrt/LICENSE
-tensorflow/third_party/tensorrt/BUILD
-tensorflow/third_party/tensorrt/build_defs.bzl.tpl
-tensorflow/third_party/tensorrt/BUILD.tpl
-tensorflow/third_party/tensorrt/tensorrt_configure.bzl
-tensorflow/third_party/tensorrt/remote.BUILD.tpl
-tensorflow/third_party/kafka/config.patch
-tensorflow/third_party/kafka/BUILD
-tensorflow/third_party/android/BUILD
-tensorflow/third_party/android/android.bzl.tpl
-tensorflow/third_party/android/android_configure.bzl
-tensorflow/third_party/android/android_configure.BUILD.tpl
-tensorflow/third_party/tflite_smartreply.BUILD
-tensorflow/third_party/mkl_dnn/LICENSE
-tensorflow/third_party/mkl_dnn/mkldnn.BUILD
-tensorflow/third_party/pcre.BUILD
-tensorflow/third_party/linenoise.BUILD
-tensorflow/third_party/sqlite.BUILD
-tensorflow/third_party/common.bzl
-tensorflow/third_party/com_google_absl.BUILD
-tensorflow/third_party/pprof.BUILD
-tensorflow/third_party/BUILD
-tensorflow/third_party/tflite_mobilenet_quant.BUILD
-tensorflow/third_party/lmdb.BUILD
-tensorflow/third_party/git/BUILD.tpl
-tensorflow/third_party/git/BUILD
-tensorflow/third_party/git/git_configure.bzl
-tensorflow/third_party/protobuf/BUILD
-tensorflow/third_party/tflite_mobilenet.BUILD
-tensorflow/third_party/py/BUILD
-tensorflow/third_party/py/BUILD.tpl
-tensorflow/third_party/py/remote.BUILD.tpl
-tensorflow/third_party/py/numpy/BUILD
-tensorflow/third_party/py/python_configure.bzl
-tensorflow/third_party/termcolor.BUILD
-tensorflow/third_party/png_fix_rpi.patch
-tensorflow/third_party/swig.BUILD
-tensorflow/third_party/astor.BUILD
-tensorflow/third_party/grpc/BUILD
-tensorflow/third_party/curl.BUILD
-tensorflow/third_party/arm_neon_2_x86_sse.BUILD
-tensorflow/third_party/png.BUILD
-tensorflow/third_party/googleapis.BUILD
-tensorflow/third_party/mpi_collectives/BUILD
-tensorflow/third_party/nanopb.BUILD
-tensorflow/third_party/gif.BUILD
-tensorflow/third_party/double_conversion.BUILD
-tensorflow/third_party/six.BUILD
-tensorflow/third_party/tflite_mobilenet_float.BUILD
-tensorflow/third_party/repo.bzl
-tensorflow/third_party/codegen.BUILD
-tensorflow/third_party/cub.BUILD
-tensorflow/third_party/jsoncpp.BUILD
-tensorflow/third_party/tflite_ovic_testdata.BUILD
-tensorflow/third_party/libxsmm.BUILD
-tensorflow/third_party/zlib.BUILD
-tensorflow/third_party/eigen.BUILD
-tensorflow/stream_executor/BUILD
-tensorflow/api_template_v1.__init__.py
-tensorflow/compat_template_v1.__init__.py
+tensorflow/tools/pip_package/simple_console.py
+tensorflow/tools/pip_package/setup.py
+tensorflow/tools/pip_package/MANIFEST.in
+tensorflow/tools/pip_package/README
+tensorflow/tools/pip_package/check_load_py_test.py
+tensorflow/tools/pip_package/pip_smoke_test.py
+tensorflow/tools/pip_package/BUILD
+tensorflow/tools/pip_package/build_pip_package.sh
+tensorflow/tools/pip_package/simple_console_for_windows.py
+tensorflow/tools/lib_package/LibTensorFlowTest.java
+tensorflow/tools/lib_package/README.md
+tensorflow/tools/lib_package/libtensorflow_test.sh
+tensorflow/tools/lib_package/libtensorflow_java_test.sh
+tensorflow/tools/lib_package/libtensorflow_test.c
+tensorflow/tools/lib_package/concat_licenses.sh
+tensorflow/tools/lib_package/BUILD
+tensorflow/tools/ci_build/remote/BUILD
+tensorflow/tools/def_file_filter/BUILD.tpl
+tensorflow/tools/def_file_filter/def_file_filter_configure.bzl
+tensorflow/tools/def_file_filter/def_file_filter.py.tpl
+tensorflow/tools/def_file_filter/BUILD
+tensorflow/contrib/mpi/BUILD
+tensorflow/contrib/tpu/profiler/pip_package/cloud_tpu_profiler/main.py
+tensorflow/contrib/tpu/profiler/pip_package/cloud_tpu_profiler/__init__.py
+tensorflow/contrib/tpu/profiler/pip_package/setup.py
+tensorflow/contrib/tpu/profiler/pip_package/README
+tensorflow/contrib/tpu/profiler/pip_package/BUILD
+tensorflow/contrib/tpu/profiler/pip_package/build_pip_package.sh
 tensorflow/api_template.__init__.py
 tensorflow/__init__.py
\ No newline at end of file
diff --git a/tensorflow/tools/git/gen_git_source.py b/tensorflow/tools/git/gen_git_source.py
index 645d817d9f..0c37508650 100755
--- a/tensorflow/tools/git/gen_git_source.py
+++ b/tensorflow/tools/git/gen_git_source.py
@@ -174,8 +174,8 @@ def get_git_version(git_base_path, git_tag_override):
       # There might be "-" in the tag name. But we can be sure that the final
       # two "-" are those inserted by the git describe command.
       abbrev_commit = split_val[-1]
-      val = bytes(
-          version_separator.join([git_tag_override, "0", abbrev_commit]))
+      val = version_separator.join(
+          [bytes(git_tag_override), b"0", abbrev_commit])
     return val if val else unknown_label
   except (subprocess.CalledProcessError, OSError):
     return unknown_label
diff --git a/tensorflow/tools/optimization/BUILD b/tensorflow/tools/optimization/BUILD
new file mode 100644
index 0000000000..aa6c850b0b
--- /dev/null
+++ b/tensorflow/tools/optimization/BUILD
@@ -0,0 +1,52 @@
+# Description:
+#   Utilities that perform useful transformations on graphs
+
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])  # Apache 2.0
+
+load(
+    "//tensorflow:tensorflow.bzl",
+    "tf_cc_binary",
+    "tf_cuda_library",
+)
+
+exports_files(["LICENSE"])
+
+tf_cuda_library(
+    name = "optimization_pass_runner_lib",
+    srcs = ["optimization_pass_runner.cc"],
+    hdrs = ["optimization_pass_runner.h"],
+    deps = [
+        "//tensorflow/contrib:contrib_ops_op_lib",
+        "//tensorflow/core:core_cpu",
+        "//tensorflow/core:core_cpu_base",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:framework_internal",
+        "//tensorflow/core:framework_lite",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:tensorflow",
+    ],
+)
+
+tf_cc_binary(
+    name = "gpu_optimization_pass_runner",
+    srcs = ["gpu_optimization_pass_runner_main.cc"],
+    deps = [
+        ":optimization_pass_runner_lib",
+        "//tensorflow/compiler/jit:xla_cpu_jit",
+        "//tensorflow/compiler/jit:xla_gpu_jit",
+        "//tensorflow/compiler/tf2xla:xla_compiler",
+        "//tensorflow/contrib:contrib_ops_op_lib",
+        "//tensorflow/core:core_cpu",
+        "//tensorflow/core:core_cpu_base",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:framework_internal",
+        "//tensorflow/core:framework_lite",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:tensorflow",
+        "@com_google_absl//absl/strings",
+    ],
+)
diff --git a/tensorflow/tools/optimization/gpu_optimization_pass_runner_main.cc b/tensorflow/tools/optimization/gpu_optimization_pass_runner_main.cc
new file mode 100644
index 0000000000..0d9f26cd5a
--- /dev/null
+++ b/tensorflow/tools/optimization/gpu_optimization_pass_runner_main.cc
@@ -0,0 +1,60 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+// This file creates a binary that can run any registered optimization pass.
+// ./xla_gpu_opt  --input_file_path=/tmp/input.pbtxt
+// --output_file_path=/tmp/output.pbtxt
+// --optimization_pass=NameOfGraphOptimizationPass
+
+#include "absl/strings/str_cat.h"
+#include "tensorflow/compiler/tf2xla/xla_op_registry.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/protobuf/config.pb.h"
+#include "tensorflow/tools/optimization/optimization_pass_runner.h"
+
+int main(int argc, char** argv) {
+  tensorflow::OptimizationPassRunner runner;
+  // Add fake devices for CPU, GPU, and XLA to ensure we have all devices we
+  // need.
+  // Most machines in our servers currently use 8 gpus. There is nothing special
+  // about this number and it can be decreased or increased to test other
+  // configurations.
+  int num_gpus_per_machine = 8;
+  for (int i = 0; i < num_gpus_per_machine; i++) {
+    TF_CHECK_OK(runner.AddDevice(
+        absl::StrCat("/job:localhost/replica:0/task:0/device:CPU:", i),
+        tensorflow::DEVICE_CPU));
+    TF_CHECK_OK(runner.AddDevice(
+        absl::StrCat("/job:localhost/replica:0/task:0/device:GPU:", i),
+        tensorflow::DEVICE_GPU));
+    TF_CHECK_OK(runner.AddDevice(
+        absl::StrCat("/job:localhost/replica:0/task:0/device:XLA_CPU:", i),
+        tensorflow::DEVICE_XLA_CPU));
+    TF_CHECK_OK(runner.AddDevice(
+        absl::StrCat("/job:localhost/replica:0/task:0/device:XLA_GPU:", i),
+        tensorflow::DEVICE_XLA_GPU));
+    TF_CHECK_OK(runner.AddDevice(
+        absl::StrCat("/job:localhost/replica:0/task:0/device:CPU_XLA_JIT:", i),
+        tensorflow::DEVICE_CPU_XLA_JIT));
+    TF_CHECK_OK(runner.AddDevice(
+        absl::StrCat("/job:localhost/replica:0/task:0/device:GPU_XLA_JIT:", i),
+        tensorflow::DEVICE_GPU_XLA_JIT));
+  }
+  // This binary is used to test TF:XLA behavior, so turn on auto_jit.
+  TF_CHECK_OK(runner.SetJitLevel(tensorflow::OptimizerOptions::GlobalJitLevel::
+                                     OptimizerOptions_GlobalJitLevel_ON_2));
+  // Run the actual "main" function.
+  TF_CHECK_OK(runner.RunMain(argc, argv));
+}
diff --git a/tensorflow/tools/optimization/optimization_pass_runner.cc b/tensorflow/tools/optimization/optimization_pass_runner.cc
new file mode 100644
index 0000000000..231ff08381
--- /dev/null
+++ b/tensorflow/tools/optimization/optimization_pass_runner.cc
@@ -0,0 +1,167 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+// This file creates a library that can run any registered optimization pass.
+// The binary that uses this will be run in a form similar to:
+// ./optimization_pass_runner  --input_file_path=/tmp/input.pbtxt
+// --output_file_path=/tmp/output.pbtxt
+// --optimization_pass=NameOfGraphOptimizationPass
+#include "tensorflow/tools/optimization/optimization_pass_runner.h"
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "tensorflow/core/common_runtime/device.h"
+#include "tensorflow/core/common_runtime/device_set.h"
+#include "tensorflow/core/common_runtime/optimization_registry.h"
+#include "tensorflow/core/framework/allocator.h"
+#include "tensorflow/core/framework/device_attributes.pb.h"
+#include "tensorflow/core/framework/function.h"
+#include "tensorflow/core/framework/function.pb.h"
+#include "tensorflow/core/framework/graph.pb.h"
+#include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/graph/graph.h"
+#include "tensorflow/core/graph/graph_constructor.h"
+#include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/platform/env.h"
+#include "tensorflow/core/platform/init_main.h"
+#include "tensorflow/core/protobuf/config.pb.h"
+#include "tensorflow/core/public/session_options.h"
+#include "tensorflow/core/util/command_line_flags.h"
+
+namespace tensorflow {
+
+namespace {
+// A fake device used to populate a DeviceSet.
+class FakeDevice : public Device {
+ private:
+  explicit FakeDevice(const DeviceAttributes& device_attributes)
+      : Device(nullptr, device_attributes) {}
+
+ public:
+  Status Sync() override;
+  static std::unique_ptr<Device> Make(const string& name, const string& type);
+};
+
+Status FakeDevice::Sync() {
+  return errors::Unimplemented("FakeDevice::Sync()");
+}
+
+std::unique_ptr<Device> FakeDevice::Make(const string& name,
+                                         const string& type) {
+  DeviceAttributes device_attributes;
+  device_attributes.set_name(name);
+  device_attributes.set_device_type(DeviceType(type).type());
+  return std::unique_ptr<Device>(new FakeDevice(device_attributes));
+}
+}  // namespace
+
+Status OptimizationPassRunner::RunMain(int argc, char** argv) {
+  string input_file_path;
+  string output_file_path;
+  string optimization_pass;
+
+  const std::vector<Flag> flag_list = {
+      Flag("input_file_path", &input_file_path, "Location of the input graph."),
+      Flag("output_file_path", &output_file_path,
+           "Location to write the resulting graph."),
+      // For now only a single optimization pass can be run.
+      Flag("optimization_pass", &optimization_pass,
+           "Which optimization pass to run."),
+  };
+  if (!Flags::Parse(&argc, argv, flag_list)) {
+    return errors::FailedPrecondition("Invalid flags passed");
+  }
+  port::InitMain(argv[0], &argc, &argv);
+
+  if (input_file_path.empty()) {
+    return errors::FailedPrecondition("input_file_path is a required flag.");
+  }
+  if (output_file_path.empty()) {
+    return errors::FailedPrecondition("output_file_path is a required flag.");
+  }
+  if (optimization_pass.empty()) {
+    return errors::FailedPrecondition("optimization_pass is a required flag.");
+  }
+
+  // Turn on XLA Auto-Jit.
+  auto session_options = absl::make_unique<SessionOptions>();
+  session_options->config.mutable_graph_options()
+      ->mutable_optimizer_options()
+      ->set_global_jit_level(jit_level_);
+  FunctionDefLibrary flib;
+  std::unique_ptr<Graph> graph = absl::make_unique<Graph>(OpRegistry::Global());
+
+  GraphOptimizationPassOptions options;
+  options.session_options = session_options.release();
+  options.graph = &graph;
+  options.flib_def =
+      new FunctionLibraryDefinition((*options.graph)->op_registry(), flib);
+
+  // Grab the data
+  GraphDef graphdef;
+  GraphConstructorOptions graph_opts;
+  graph_opts.expect_device_spec = true;
+  graph_opts.allow_internal_ops = true;
+  TF_RETURN_IF_ERROR(ReadTextProto(Env::Default(), input_file_path, &graphdef));
+  TF_RETURN_IF_ERROR(
+      ConvertGraphDefToGraph(graph_opts, graphdef, options.graph->get()));
+
+  // Add all devices that were previously configured with AddDevice.
+  DeviceSet device_set;
+  for (auto& device : devices_) {
+    device_set.AddDevice(device.get());
+  }
+  options.device_set = &device_set;
+
+  Status result = errors::NotFound(
+      "An OptimizationPass was not found with the desired name.");
+
+  // Run the optimization pass specified by the command line flag.
+  for (const auto& groups_and_passes :
+       OptimizationPassRegistry::Global()->groups()) {
+    for (const auto& phase_and_passes : groups_and_passes.second) {
+      for (const auto& pass : phase_and_passes.second) {
+        if (pass->name() == optimization_pass) {
+          result = pass->Run(options);
+        }
+      }
+    }
+  }
+
+  TF_RETURN_IF_ERROR(result);
+
+  // Write out the result.
+  options.graph->get()->ToGraphDef(&graphdef);
+  TF_RETURN_IF_ERROR(
+      WriteTextProto(Env::Default(), output_file_path, graphdef));
+  return Status::OK();
+}
+
+Status OptimizationPassRunner::SetJitLevel(
+    OptimizerOptions::GlobalJitLevel jit_level) {
+  jit_level_ = jit_level;
+  return Status::OK();
+}
+
+Status OptimizationPassRunner::AddDevice(const string& name,
+                                         const string& type) {
+  devices_.push_back(FakeDevice::Make(name, type));
+  return Status::OK();
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/tools/optimization/optimization_pass_runner.h b/tensorflow/tools/optimization/optimization_pass_runner.h
new file mode 100644
index 0000000000..3b26f64bcf
--- /dev/null
+++ b/tensorflow/tools/optimization/optimization_pass_runner.h
@@ -0,0 +1,61 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_TOOLS_OPTIMIZATION_OPTIMIZATION_PASS_RUNNER_H_
+#define TENSORFLOW_TOOLS_OPTIMIZATION_OPTIMIZATION_PASS_RUNNER_H_
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "tensorflow/core/common_runtime/device.h"
+#include "tensorflow/core/common_runtime/optimization_registry.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/protobuf/config.pb.h"
+
+namespace tensorflow {
+
+// OptimizationPassRunner can be initialized, populated with devices, then run
+// to test individual Tensorflow Optimization passes.
+class OptimizationPassRunner {
+ public:
+  explicit OptimizationPassRunner()
+      : jit_level_(OptimizerOptions::GlobalJitLevel::
+                       OptimizerOptions_GlobalJitLevel_DEFAULT) {}
+
+  // Add a fake device to the (initially empty) DeviceSet used for optimization.
+  // Names are of the form: "/job:localhost/replica:0/task:0/device:CPU:0"
+  Status AddDevice(const string& name, const string& type);
+
+  // Increasing the Jit level will cause XLA to compile parts of the tensorflow
+  // graph that it is able to.
+  Status SetJitLevel(OptimizerOptions::GlobalJitLevel jit_level);
+
+  // This can be called after adding devices and setting the jit level to parse
+  // command line flags and run the specified job. All 3 flags are required:
+  // input_file_path, output_file_path, optimization_pass.
+  //
+  // If this library becomes heavily used, the caller should be responsible for
+  // parsing any command line flags desired rather than this Method handling the
+  // work of a main() function.
+  Status RunMain(int argc, char** argv);
+
+ private:
+  OptimizerOptions::GlobalJitLevel jit_level_;
+  std::vector<std::unique_ptr<Device>> devices_;
+};
+
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_TOOLS_OPTIMIZATION_OPTIMIZATION_PASS_RUNNER_H_
-- 
GitLab


From 044b34b2819ec45ffc34c7b59e921ad4d8aaec9a Mon Sep 17 00:00:00 2001
From: Penporn Koanantakool <penporn@google.com>
Date: Mon, 7 Jan 2019 11:13:45 -0800
Subject: [PATCH 0302/1095] Disable MKL-DNN contraction kernels in Eigen when
 building with config=mkl to avoid duplicate MKL-DNN symbols. (config=mkl
 links to OpenMP MKL-DNN while tensorflow_mkldnn_contraction_kernel links to
 serial MKL-DNN.)

PiperOrigin-RevId: 228199406
---
 .bazelrc                         |   2 +
 tensorflow/opensource_only.files | 416 +++++++++++++++----------------
 2 files changed, 210 insertions(+), 208 deletions(-)

diff --git a/.bazelrc b/.bazelrc
index cd7e13ddfc..d5432aeb17 100644
--- a/.bazelrc
+++ b/.bazelrc
@@ -25,12 +25,14 @@ build --define framework_shared_object=true
 # If you would like to use a local MKL instead of downloading, please set the
 # environment variable "TF_MKL_ROOT" every time before build.
 build:mkl --define=build_with_mkl=true --define=enable_mkl=true
+build:mkl --define=tensorflow_mkldnn_contraction_kernel=0
 build:mkl -c opt
 
 # This config option is used to enable MKL-DNN open source library only,
 # without depending on MKL binary version.
 build:mkl_open_source_only --define=build_with_mkl_dnn_only=true
 build:mkl_open_source_only --define=build_with_mkl=true --define=enable_mkl=true
+build:mkl_open_source_only --define=tensorflow_mkldnn_contraction_kernel=0
 
 build:download_clang --crosstool_top=@local_config_download_clang//:toolchain
 build:download_clang --define=using_clang=true
diff --git a/tensorflow/opensource_only.files b/tensorflow/opensource_only.files
index 40f87d6c36..e00d063cfe 100644
--- a/tensorflow/opensource_only.files
+++ b/tensorflow/opensource_only.files
@@ -1,248 +1,248 @@
-tensorflow/api_template_v1.__init__.py
-tensorflow/compat_template_v1.__init__.py
-tensorflow/stream_executor/BUILD
-tensorflow/third_party/curl.BUILD
-tensorflow/third_party/snappy.BUILD
-tensorflow/third_party/tflite_mobilenet_float.BUILD
-tensorflow/third_party/libxsmm.BUILD
-tensorflow/third_party/cub.BUILD
-tensorflow/third_party/repo.bzl
-tensorflow/third_party/systemlibs/curl.BUILD
-tensorflow/third_party/systemlibs/snappy.BUILD
-tensorflow/third_party/systemlibs/protobuf.BUILD
-tensorflow/third_party/systemlibs/absl_py.BUILD
-tensorflow/third_party/systemlibs/google_cloud_cpp.google.cloud.bigtable.BUILD
-tensorflow/third_party/systemlibs/astor.BUILD
-tensorflow/third_party/systemlibs/absl_py.absl.flags.BUILD
-tensorflow/third_party/systemlibs/png.BUILD
-tensorflow/third_party/systemlibs/BUILD.tpl
-tensorflow/third_party/systemlibs/boringssl.BUILD
-tensorflow/third_party/systemlibs/absl_py.absl.testing.BUILD
-tensorflow/third_party/systemlibs/swig.BUILD
-tensorflow/third_party/systemlibs/double_conversion.BUILD
-tensorflow/third_party/systemlibs/gif.BUILD
-tensorflow/third_party/systemlibs/cython.BUILD
-tensorflow/third_party/systemlibs/google_cloud_cpp.BUILD
-tensorflow/third_party/systemlibs/zlib.BUILD
-tensorflow/third_party/systemlibs/grpc.BUILD
-tensorflow/third_party/systemlibs/re2.BUILD
-tensorflow/third_party/systemlibs/build_defs.bzl.tpl
-tensorflow/third_party/systemlibs/protobuf.bzl
-tensorflow/third_party/systemlibs/gast.BUILD
-tensorflow/third_party/systemlibs/syslibs_configure.bzl
-tensorflow/third_party/systemlibs/nsync.BUILD
-tensorflow/third_party/systemlibs/jsoncpp.BUILD
-tensorflow/third_party/systemlibs/six.BUILD
-tensorflow/third_party/systemlibs/lmdb.BUILD
-tensorflow/third_party/systemlibs/BUILD
-tensorflow/third_party/systemlibs/googleapis.BUILD
-tensorflow/third_party/systemlibs/sqlite.BUILD
-tensorflow/third_party/systemlibs/termcolor.BUILD
-tensorflow/third_party/systemlibs/pcre.BUILD
-tensorflow/third_party/tflite_mobilenet_quant.BUILD
-tensorflow/third_party/common.bzl
-tensorflow/third_party/boringssl/BUILD
-tensorflow/third_party/kafka/config.patch
-tensorflow/third_party/kafka/BUILD
-tensorflow/third_party/android/android_configure.bzl
-tensorflow/third_party/android/android_configure.BUILD.tpl
-tensorflow/third_party/android/android.bzl.tpl
-tensorflow/third_party/android/BUILD
-tensorflow/third_party/astor.BUILD
-tensorflow/third_party/icu/udata.patch
-tensorflow/third_party/tflite_mobilenet.BUILD
-tensorflow/third_party/png.BUILD
-tensorflow/third_party/mpi/.gitignore
-tensorflow/third_party/mpi/BUILD
-tensorflow/third_party/nanopb.BUILD
-tensorflow/third_party/arm_neon_2_x86_sse.BUILD
-tensorflow/third_party/linenoise.BUILD
-tensorflow/third_party/llvm/llvm.autogenerated.BUILD
-tensorflow/third_party/llvm/llvm.bzl
-tensorflow/third_party/llvm/BUILD
-tensorflow/third_party/llvm/expand_cmake_vars.py
-tensorflow/third_party/swig.BUILD
-tensorflow/third_party/png_fix_rpi.patch
-tensorflow/third_party/double_conversion.BUILD
-tensorflow/third_party/gif.BUILD
-tensorflow/third_party/grpc/BUILD
-tensorflow/third_party/cython.BUILD
-tensorflow/third_party/eigen.BUILD
-tensorflow/third_party/codegen.BUILD
-tensorflow/third_party/zlib.BUILD
-tensorflow/third_party/tensorrt/BUILD.tpl
-tensorflow/third_party/tensorrt/tensorrt_configure.bzl
-tensorflow/third_party/tensorrt/build_defs.bzl.tpl
-tensorflow/third_party/tensorrt/remote.BUILD.tpl
-tensorflow/third_party/tensorrt/LICENSE
-tensorflow/third_party/tensorrt/BUILD
-tensorflow/third_party/tflite_ovic_testdata.BUILD
-tensorflow/third_party/fft2d/fft.h
-tensorflow/third_party/fft2d/fft2d.BUILD
-tensorflow/third_party/fft2d/LICENSE
-tensorflow/third_party/fft2d/BUILD
+tensorflow/contrib/tpu/profiler/pip_package/BUILD
+tensorflow/contrib/tpu/profiler/pip_package/setup.py
+tensorflow/contrib/tpu/profiler/pip_package/README
+tensorflow/contrib/tpu/profiler/pip_package/build_pip_package.sh
+tensorflow/contrib/tpu/profiler/pip_package/cloud_tpu_profiler/main.py
+tensorflow/contrib/tpu/profiler/pip_package/cloud_tpu_profiler/__init__.py
+tensorflow/contrib/mpi/BUILD
+tensorflow/tools/ci_build/remote/BUILD
+tensorflow/tools/pip_package/README
+tensorflow/tools/pip_package/MANIFEST.in
+tensorflow/tools/pip_package/simple_console.py
+tensorflow/tools/pip_package/build_pip_package.sh
+tensorflow/tools/pip_package/check_load_py_test.py
+tensorflow/tools/pip_package/pip_smoke_test.py
+tensorflow/tools/pip_package/simple_console_for_windows.py
+tensorflow/tools/pip_package/setup.py
+tensorflow/tools/pip_package/BUILD
+tensorflow/tools/lib_package/concat_licenses.sh
+tensorflow/tools/lib_package/libtensorflow_test.c
+tensorflow/tools/lib_package/LibTensorFlowTest.java
+tensorflow/tools/lib_package/BUILD
+tensorflow/tools/lib_package/libtensorflow_test.sh
+tensorflow/tools/lib_package/README.md
+tensorflow/tools/lib_package/libtensorflow_java_test.sh
+tensorflow/tools/def_file_filter/def_file_filter_configure.bzl
+tensorflow/tools/def_file_filter/BUILD
+tensorflow/tools/def_file_filter/BUILD.tpl
+tensorflow/tools/def_file_filter/def_file_filter.py.tpl
+tensorflow/third_party/mkl/MKL_LICENSE
+tensorflow/third_party/mkl/LICENSE
+tensorflow/third_party/mkl/BUILD
+tensorflow/third_party/mkl/mkl.BUILD
+tensorflow/third_party/mkl/build_defs.bzl
+tensorflow/third_party/backports_weakref.BUILD
+tensorflow/third_party/toolchains/clang6/BUILD
 tensorflow/third_party/toolchains/clang6/README.md
 tensorflow/third_party/toolchains/clang6/repo.bzl
-tensorflow/third_party/toolchains/clang6/clang.BUILD
 tensorflow/third_party/toolchains/clang6/CROSSTOOL.tpl
-tensorflow/third_party/toolchains/clang6/BUILD
-tensorflow/third_party/toolchains/preconfig/generate/archives.bzl
-tensorflow/third_party/toolchains/preconfig/generate/workspace.bzl
-tensorflow/third_party/toolchains/preconfig/generate/generate.bzl
-tensorflow/third_party/toolchains/preconfig/generate/containers.bzl
-tensorflow/third_party/toolchains/preconfig/generate/BUILD
-tensorflow/third_party/toolchains/preconfig/ubuntu14.04/tensorrt5/build_defs.bzl
+tensorflow/third_party/toolchains/clang6/clang.BUILD
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/tensorrt5/BUILD
+tensorflow/third_party/toolchains/preconfig/ubuntu14.04/tensorrt5/build_defs.bzl
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/py3/BUILD
+tensorflow/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/BUILD
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc/BUILD
-tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/build_defs.bzl
-tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/BUILD
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/cuda/build_defs.bzl
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/cuda/BUILD
-tensorflow/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda9.0/BUILD
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda10.0/BUILD
+tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/build_defs.bzl
+tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/BUILD
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/nccl2/BUILD
+tensorflow/third_party/toolchains/preconfig/generate/workspace.bzl
+tensorflow/third_party/toolchains/preconfig/generate/containers.bzl
+tensorflow/third_party/toolchains/preconfig/generate/generate.bzl
+tensorflow/third_party/toolchains/preconfig/generate/archives.bzl
+tensorflow/third_party/toolchains/preconfig/generate/BUILD
+tensorflow/third_party/toolchains/preconfig/ubuntu16.04/clang/BUILD
+tensorflow/third_party/toolchains/preconfig/ubuntu16.04/clang/dummy_toolchain.bzl
+tensorflow/third_party/toolchains/preconfig/ubuntu16.04/py3/BUILD
+tensorflow/third_party/toolchains/preconfig/win_1803/bazel_018/BUILD
+tensorflow/third_party/toolchains/preconfig/win_1803/bazel_018/dummy_toolchain.bzl
 tensorflow/third_party/toolchains/preconfig/win_1803/py36/BUILD
 tensorflow/third_party/toolchains/preconfig/win_1803/BUILD
-tensorflow/third_party/toolchains/preconfig/win_1803/bazel_018/dummy_toolchain.bzl
-tensorflow/third_party/toolchains/preconfig/win_1803/bazel_018/BUILD
-tensorflow/third_party/toolchains/preconfig/ubuntu16.04/py3/BUILD
-tensorflow/third_party/toolchains/preconfig/ubuntu16.04/clang/dummy_toolchain.bzl
-tensorflow/third_party/toolchains/preconfig/ubuntu16.04/clang/BUILD
-tensorflow/third_party/toolchains/cpus/py3/BUILD
-tensorflow/third_party/toolchains/cpus/arm/CROSSTOOL.tpl
-tensorflow/third_party/toolchains/cpus/arm/BUILD
-tensorflow/third_party/toolchains/cpus/arm/arm_compiler_configure.bzl
-tensorflow/third_party/toolchains/cpus/py/BUILD
-tensorflow/third_party/toolchains/gpus/crosstool/CROSSTOOL
-tensorflow/third_party/toolchains/gpus/crosstool/BUILD
 tensorflow/third_party/toolchains/gpus/cuda/build_defs.bzl
-tensorflow/third_party/toolchains/gpus/cuda/cuda/cuda_config.h
 tensorflow/third_party/toolchains/gpus/cuda/BUILD
+tensorflow/third_party/toolchains/gpus/cuda/cuda/cuda_config.h
+tensorflow/third_party/toolchains/gpus/crosstool/BUILD
+tensorflow/third_party/toolchains/gpus/crosstool/CROSSTOOL
 tensorflow/third_party/toolchains/gpus/py/BUILD
+tensorflow/third_party/toolchains/cpus/arm/arm_compiler_configure.bzl
+tensorflow/third_party/toolchains/cpus/arm/CROSSTOOL.tpl
+tensorflow/third_party/toolchains/cpus/arm/BUILD
+tensorflow/third_party/toolchains/cpus/py3/BUILD
+tensorflow/third_party/toolchains/cpus/py/BUILD
 tensorflow/third_party/toolchains/BUILD
-tensorflow/third_party/nccl/archive.BUILD
-tensorflow/third_party/nccl/nccl_configure.bzl
-tensorflow/third_party/nccl/system.BUILD.tpl
-tensorflow/third_party/nccl/build_defs.bzl.tpl
-tensorflow/third_party/nccl/LICENSE
-tensorflow/third_party/nccl/BUILD
-tensorflow/third_party/farmhash.BUILD
-tensorflow/third_party/sycl/crosstool/BUILD
-tensorflow/third_party/backports_weakref.BUILD
-tensorflow/third_party/gast.BUILD
-tensorflow/third_party/clang_toolchain/cc_configure_clang.bzl
-tensorflow/third_party/clang_toolchain/download_clang.bzl
-tensorflow/third_party/clang_toolchain/BUILD
-tensorflow/third_party/mpi_collectives/BUILD
-tensorflow/third_party/jsoncpp.BUILD
-tensorflow/third_party/mkl_dnn/mkldnn.BUILD
-tensorflow/third_party/mkl_dnn/LICENSE
-tensorflow/third_party/python_runtime/BUILD
-tensorflow/third_party/ngraph/ngraph.BUILD
-tensorflow/third_party/ngraph/nlohmann_json.BUILD
-tensorflow/third_party/ngraph/build_defs.bzl
-tensorflow/third_party/ngraph/ngraph_tf.BUILD
-tensorflow/third_party/ngraph/NGRAPH_LICENSE
-tensorflow/third_party/ngraph/tbb.BUILD
-tensorflow/third_party/ngraph/LICENSE
-tensorflow/third_party/ngraph/BUILD
-tensorflow/third_party/gpus/crosstool/windows/msvc_wrapper_for_nvcc.py.tpl
-tensorflow/third_party/gpus/crosstool/CROSSTOOL.tpl
-tensorflow/third_party/gpus/crosstool/BUILD.tpl
-tensorflow/third_party/gpus/crosstool/remote.BUILD.tpl
+tensorflow/third_party/gpus/BUILD
 tensorflow/third_party/gpus/crosstool/clang/bin/crosstool_wrapper_driver_rocm.tpl
 tensorflow/third_party/gpus/crosstool/clang/bin/crosstool_wrapper_driver_is_not_gcc.tpl
+tensorflow/third_party/gpus/crosstool/CROSSTOOL.tpl
 tensorflow/third_party/gpus/crosstool/CROSSTOOL_hipcc.tpl
 tensorflow/third_party/gpus/crosstool/LICENSE
+tensorflow/third_party/gpus/crosstool/remote.BUILD.tpl
+tensorflow/third_party/gpus/crosstool/windows/msvc_wrapper_for_nvcc.py.tpl
+tensorflow/third_party/gpus/crosstool/BUILD.tpl
 tensorflow/third_party/gpus/crosstool/BUILD
-tensorflow/third_party/gpus/rocm/BUILD.tpl
-tensorflow/third_party/gpus/rocm/rocm_config.h.tpl
-tensorflow/third_party/gpus/rocm/build_defs.bzl.tpl
-tensorflow/third_party/gpus/rocm/BUILD
+tensorflow/third_party/gpus/cuda/LICENSE
 tensorflow/third_party/gpus/cuda/BUILD.tpl
-tensorflow/third_party/gpus/cuda/build_defs.bzl.tpl
-tensorflow/third_party/gpus/cuda/remote.BUILD.tpl
 tensorflow/third_party/gpus/cuda/BUILD.windows.tpl
 tensorflow/third_party/gpus/cuda/cuda_config.h.tpl
-tensorflow/third_party/gpus/cuda/LICENSE
+tensorflow/third_party/gpus/cuda/remote.BUILD.tpl
 tensorflow/third_party/gpus/cuda/BUILD
+tensorflow/third_party/gpus/cuda/build_defs.bzl.tpl
+tensorflow/third_party/gpus/rocm/rocm_config.h.tpl
+tensorflow/third_party/gpus/rocm/BUILD
+tensorflow/third_party/gpus/rocm/BUILD.tpl
+tensorflow/third_party/gpus/rocm/build_defs.bzl.tpl
 tensorflow/third_party/gpus/cuda_configure.bzl
 tensorflow/third_party/gpus/rocm_configure.bzl
-tensorflow/third_party/gpus/BUILD
-tensorflow/third_party/py/python_configure.bzl
-tensorflow/third_party/py/BUILD.tpl
-tensorflow/third_party/py/numpy/BUILD
-tensorflow/third_party/py/remote.BUILD.tpl
-tensorflow/third_party/py/BUILD
-tensorflow/third_party/mkl/build_defs.bzl
-tensorflow/third_party/mkl/mkl.BUILD
-tensorflow/third_party/mkl/MKL_LICENSE
-tensorflow/third_party/mkl/LICENSE
-tensorflow/third_party/mkl/BUILD
-tensorflow/third_party/com_google_absl.BUILD
-tensorflow/third_party/six.BUILD
-tensorflow/third_party/lmdb.BUILD
-tensorflow/third_party/BUILD
-tensorflow/third_party/googleapis.BUILD
-tensorflow/third_party/sqlite.BUILD
-tensorflow/third_party/termcolor.BUILD
-tensorflow/third_party/protobuf/BUILD
-tensorflow/third_party/pcre.BUILD
-tensorflow/third_party/git/BUILD.tpl
-tensorflow/third_party/git/git_configure.bzl
-tensorflow/third_party/git/BUILD
-tensorflow/third_party/pprof.BUILD
-tensorflow/third_party/tflite_smartreply.BUILD
-tensorflow/third_party/eigen3/Eigen/Core
+tensorflow/third_party/snappy.BUILD
+tensorflow/third_party/cython.BUILD
+tensorflow/third_party/farmhash.BUILD
 tensorflow/third_party/eigen3/Eigen/Cholesky
-tensorflow/third_party/eigen3/Eigen/Eigenvalues
-tensorflow/third_party/eigen3/Eigen/SVD
-tensorflow/third_party/eigen3/Eigen/LU
 tensorflow/third_party/eigen3/Eigen/QR
-tensorflow/third_party/eigen3/unsupported/Eigen/MatrixFunctions
+tensorflow/third_party/eigen3/Eigen/LU
+tensorflow/third_party/eigen3/Eigen/Core
+tensorflow/third_party/eigen3/Eigen/SVD
+tensorflow/third_party/eigen3/Eigen/Eigenvalues
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/FixedPoint
 tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/Tensor
-tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/ThreadPool
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/PacketMathAVX512.h
 tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/TypeCastingAVX512.h
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/PacketMathAVX2.h
 tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/MatMatProduct.h
-tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/MatMatProductNEON.h
-tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/PacketMathAVX512.h
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/FixedPointTypes.h
 tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/MatVecProduct.h
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/MatMatProductNEON.h
 tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/TypeCastingAVX2.h
-tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/FixedPointTypes.h
 tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/MatMatProductAVX2.h
-tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/PacketMathAVX2.h
-tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/FixedPoint
+tensorflow/third_party/eigen3/unsupported/Eigen/CXX11/ThreadPool
 tensorflow/third_party/eigen3/unsupported/Eigen/SpecialFunctions
+tensorflow/third_party/eigen3/unsupported/Eigen/MatrixFunctions
 tensorflow/third_party/eigen3/LICENSE
 tensorflow/third_party/eigen3/BUILD
-tensorflow/tools/pip_package/simple_console.py
-tensorflow/tools/pip_package/setup.py
-tensorflow/tools/pip_package/MANIFEST.in
-tensorflow/tools/pip_package/README
-tensorflow/tools/pip_package/check_load_py_test.py
-tensorflow/tools/pip_package/pip_smoke_test.py
-tensorflow/tools/pip_package/BUILD
-tensorflow/tools/pip_package/build_pip_package.sh
-tensorflow/tools/pip_package/simple_console_for_windows.py
-tensorflow/tools/lib_package/LibTensorFlowTest.java
-tensorflow/tools/lib_package/README.md
-tensorflow/tools/lib_package/libtensorflow_test.sh
-tensorflow/tools/lib_package/libtensorflow_java_test.sh
-tensorflow/tools/lib_package/libtensorflow_test.c
-tensorflow/tools/lib_package/concat_licenses.sh
-tensorflow/tools/lib_package/BUILD
-tensorflow/tools/ci_build/remote/BUILD
-tensorflow/tools/def_file_filter/BUILD.tpl
-tensorflow/tools/def_file_filter/def_file_filter_configure.bzl
-tensorflow/tools/def_file_filter/def_file_filter.py.tpl
-tensorflow/tools/def_file_filter/BUILD
-tensorflow/contrib/mpi/BUILD
-tensorflow/contrib/tpu/profiler/pip_package/cloud_tpu_profiler/main.py
-tensorflow/contrib/tpu/profiler/pip_package/cloud_tpu_profiler/__init__.py
-tensorflow/contrib/tpu/profiler/pip_package/setup.py
-tensorflow/contrib/tpu/profiler/pip_package/README
-tensorflow/contrib/tpu/profiler/pip_package/BUILD
-tensorflow/contrib/tpu/profiler/pip_package/build_pip_package.sh
+tensorflow/third_party/systemlibs/build_defs.bzl.tpl
+tensorflow/third_party/systemlibs/absl_py.BUILD
+tensorflow/third_party/systemlibs/curl.BUILD
+tensorflow/third_party/systemlibs/termcolor.BUILD
+tensorflow/third_party/systemlibs/absl_py.absl.flags.BUILD
+tensorflow/third_party/systemlibs/grpc.BUILD
+tensorflow/third_party/systemlibs/swig.BUILD
+tensorflow/third_party/systemlibs/protobuf.bzl
+tensorflow/third_party/systemlibs/protobuf.BUILD
+tensorflow/third_party/systemlibs/BUILD
+tensorflow/third_party/systemlibs/google_cloud_cpp.BUILD
+tensorflow/third_party/systemlibs/astor.BUILD
+tensorflow/third_party/systemlibs/six.BUILD
+tensorflow/third_party/systemlibs/absl_py.absl.testing.BUILD
+tensorflow/third_party/systemlibs/boringssl.BUILD
+tensorflow/third_party/systemlibs/nsync.BUILD
+tensorflow/third_party/systemlibs/google_cloud_cpp.google.cloud.bigtable.BUILD
+tensorflow/third_party/systemlibs/gif.BUILD
+tensorflow/third_party/systemlibs/pcre.BUILD
+tensorflow/third_party/systemlibs/BUILD.tpl
+tensorflow/third_party/systemlibs/snappy.BUILD
+tensorflow/third_party/systemlibs/gast.BUILD
+tensorflow/third_party/systemlibs/cython.BUILD
+tensorflow/third_party/systemlibs/double_conversion.BUILD
+tensorflow/third_party/systemlibs/zlib.BUILD
+tensorflow/third_party/systemlibs/jsoncpp.BUILD
+tensorflow/third_party/systemlibs/re2.BUILD
+tensorflow/third_party/systemlibs/lmdb.BUILD
+tensorflow/third_party/systemlibs/googleapis.BUILD
+tensorflow/third_party/systemlibs/png.BUILD
+tensorflow/third_party/systemlibs/syslibs_configure.bzl
+tensorflow/third_party/systemlibs/sqlite.BUILD
+tensorflow/third_party/python_runtime/BUILD
+tensorflow/third_party/sycl/crosstool/BUILD
+tensorflow/third_party/ngraph/LICENSE
+tensorflow/third_party/ngraph/tbb.BUILD
+tensorflow/third_party/ngraph/BUILD
+tensorflow/third_party/ngraph/ngraph.BUILD
+tensorflow/third_party/ngraph/build_defs.bzl
+tensorflow/third_party/ngraph/NGRAPH_LICENSE
+tensorflow/third_party/ngraph/ngraph_tf.BUILD
+tensorflow/third_party/ngraph/nlohmann_json.BUILD
+tensorflow/third_party/clang_toolchain/download_clang.bzl
+tensorflow/third_party/clang_toolchain/BUILD
+tensorflow/third_party/clang_toolchain/cc_configure_clang.bzl
+tensorflow/third_party/gast.BUILD
+tensorflow/third_party/llvm/BUILD
+tensorflow/third_party/llvm/expand_cmake_vars.py
+tensorflow/third_party/llvm/llvm.autogenerated.BUILD
+tensorflow/third_party/llvm/llvm.bzl
+tensorflow/third_party/icu/udata.patch
+tensorflow/third_party/nccl/archive.BUILD
+tensorflow/third_party/nccl/LICENSE
+tensorflow/third_party/nccl/system.BUILD.tpl
+tensorflow/third_party/nccl/nccl_configure.bzl
+tensorflow/third_party/nccl/build_defs.bzl.tpl
+tensorflow/third_party/nccl/BUILD
+tensorflow/third_party/fft2d/BUILD
+tensorflow/third_party/fft2d/fft.h
+tensorflow/third_party/fft2d/LICENSE
+tensorflow/third_party/fft2d/fft2d.BUILD
+tensorflow/third_party/boringssl/BUILD
+tensorflow/third_party/mpi/.gitignore
+tensorflow/third_party/mpi/BUILD
+tensorflow/third_party/tensorrt/LICENSE
+tensorflow/third_party/tensorrt/BUILD
+tensorflow/third_party/tensorrt/build_defs.bzl.tpl
+tensorflow/third_party/tensorrt/BUILD.tpl
+tensorflow/third_party/tensorrt/tensorrt_configure.bzl
+tensorflow/third_party/tensorrt/remote.BUILD.tpl
+tensorflow/third_party/kafka/config.patch
+tensorflow/third_party/kafka/BUILD
+tensorflow/third_party/android/BUILD
+tensorflow/third_party/android/android.bzl.tpl
+tensorflow/third_party/android/android_configure.bzl
+tensorflow/third_party/android/android_configure.BUILD.tpl
+tensorflow/third_party/tflite_smartreply.BUILD
+tensorflow/third_party/mkl_dnn/LICENSE
+tensorflow/third_party/mkl_dnn/mkldnn.BUILD
+tensorflow/third_party/pcre.BUILD
+tensorflow/third_party/linenoise.BUILD
+tensorflow/third_party/sqlite.BUILD
+tensorflow/third_party/common.bzl
+tensorflow/third_party/com_google_absl.BUILD
+tensorflow/third_party/pprof.BUILD
+tensorflow/third_party/BUILD
+tensorflow/third_party/tflite_mobilenet_quant.BUILD
+tensorflow/third_party/lmdb.BUILD
+tensorflow/third_party/git/BUILD.tpl
+tensorflow/third_party/git/BUILD
+tensorflow/third_party/git/git_configure.bzl
+tensorflow/third_party/protobuf/BUILD
+tensorflow/third_party/tflite_mobilenet.BUILD
+tensorflow/third_party/py/BUILD
+tensorflow/third_party/py/BUILD.tpl
+tensorflow/third_party/py/remote.BUILD.tpl
+tensorflow/third_party/py/numpy/BUILD
+tensorflow/third_party/py/python_configure.bzl
+tensorflow/third_party/termcolor.BUILD
+tensorflow/third_party/png_fix_rpi.patch
+tensorflow/third_party/swig.BUILD
+tensorflow/third_party/astor.BUILD
+tensorflow/third_party/grpc/BUILD
+tensorflow/third_party/curl.BUILD
+tensorflow/third_party/arm_neon_2_x86_sse.BUILD
+tensorflow/third_party/png.BUILD
+tensorflow/third_party/googleapis.BUILD
+tensorflow/third_party/mpi_collectives/BUILD
+tensorflow/third_party/nanopb.BUILD
+tensorflow/third_party/gif.BUILD
+tensorflow/third_party/double_conversion.BUILD
+tensorflow/third_party/six.BUILD
+tensorflow/third_party/tflite_mobilenet_float.BUILD
+tensorflow/third_party/repo.bzl
+tensorflow/third_party/codegen.BUILD
+tensorflow/third_party/cub.BUILD
+tensorflow/third_party/jsoncpp.BUILD
+tensorflow/third_party/tflite_ovic_testdata.BUILD
+tensorflow/third_party/libxsmm.BUILD
+tensorflow/third_party/zlib.BUILD
+tensorflow/third_party/eigen.BUILD
+tensorflow/stream_executor/BUILD
+tensorflow/api_template_v1.__init__.py
+tensorflow/compat_template_v1.__init__.py
 tensorflow/api_template.__init__.py
 tensorflow/__init__.py
\ No newline at end of file
-- 
GitLab


From 68340a644560241ca7098bb2d358a3cb6fb42a4d Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Mon, 7 Jan 2019 11:20:59 -0800
Subject: [PATCH 0303/1095] Move the fit-scope check to only error our when
 using numpy

PiperOrigin-RevId: 228200832
---
 .../python/keras/engine/training_distributed.py      | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/keras/engine/training_distributed.py b/tensorflow/python/keras/engine/training_distributed.py
index 9f63f7bada..92a46e399a 100644
--- a/tensorflow/python/keras/engine/training_distributed.py
+++ b/tensorflow/python/keras/engine/training_distributed.py
@@ -53,14 +53,14 @@ def fit_distributed(model,
                     steps_per_epoch=None,
                     validation_steps=None):
   """Fit loop for Distribution Strategies."""
-  # TODO(b/122314600): Remove the scope validate.
-  distributed_training_utils.validate_not_in_strategy_scope()
   distributed_training_utils.validate_callbacks(callbacks, model.optimizer)
   distributed_training_utils.validate_inputs(
       x, y, model._distribution_strategy)
 
   first_x_value = nest.flatten(x)[0]
   if isinstance(first_x_value, np.ndarray):
+    # TODO(b/122314600): Remove the scope validate.
+    distributed_training_utils.validate_not_in_strategy_scope()
     steps_per_epoch, batch_size = (
         distributed_training_utils.get_input_params(
             model._distribution_strategy, first_x_value, steps_per_epoch,
@@ -138,11 +138,11 @@ def evaluate_distributed(model,
                          steps=None,
                          callbacks=None):
   """Evaluate loop for Distribution Strategies."""
-  # TODO(b/122314600): Remove the scope validate.
-  distributed_training_utils.validate_not_in_strategy_scope()
   distributed_training_utils.validate_inputs(x, y, model._distribution_strategy)
   first_x_value = nest.flatten(x)[0]
   if isinstance(first_x_value, np.ndarray):
+    # TODO(b/122314600): Remove the scope validate.
+    distributed_training_utils.validate_not_in_strategy_scope()
     steps, batch_size = distributed_training_utils.get_input_params(
         model._distribution_strategy, first_x_value, steps, batch_size)
   batch_size = model._validate_or_infer_batch_size(batch_size, steps, x)
@@ -175,12 +175,12 @@ def predict_distributed(model,
                         steps=None,
                         callbacks=None):
   """Predict loop for Distribution Strategies."""
-  # TODO(b/122314600): Remove the scope validate.
-  distributed_training_utils.validate_not_in_strategy_scope()
   distributed_training_utils.validate_inputs(
       x, None, model._distribution_strategy)
   first_x_value = nest.flatten(x)[0]
   if isinstance(first_x_value, np.ndarray):
+    # TODO(b/122314600): Remove the scope validate.
+    distributed_training_utils.validate_not_in_strategy_scope()
     steps, batch_size = distributed_training_utils.get_input_params(
         model._distribution_strategy, first_x_value, steps, batch_size)
   batch_size = model._validate_or_infer_batch_size(batch_size, steps, x)
-- 
GitLab


From 2c0a8c16477999cc791705112f501d4530649938 Mon Sep 17 00:00:00 2001
From: Karim Nosir <karimnosseir@google.com>
Date: Mon, 7 Jan 2019 11:29:07 -0800
Subject: [PATCH 0304/1095] Add Unique Op implementation

PiperOrigin-RevId: 228202673
---
 tensorflow/lite/build_def.bzl                 |   1 +
 tensorflow/lite/kernels/BUILD                 |  12 ++
 tensorflow/lite/kernels/register.cc           |   2 +
 tensorflow/lite/kernels/unique.cc             | 164 ++++++++++++++++++
 tensorflow/lite/kernels/unique_test.cc        | 103 +++++++++++
 tensorflow/lite/testing/generate_examples.py  |  49 ++++++
 .../propagate_array_data_types.cc             |   8 +
 .../propagate_fixed_sizes.cc                  |  17 ++
 tensorflow/lite/toco/import_tensorflow.cc     | 113 ++++++------
 tensorflow/lite/toco/model.h                  |  14 +-
 tensorflow/lite/toco/tflite/operator.cc       |  28 +++
 tensorflow/lite/toco/tflite/operator_test.cc  |   9 +
 tensorflow/lite/toco/tooling_util.cc          |   1 +
 13 files changed, 467 insertions(+), 54 deletions(-)
 create mode 100644 tensorflow/lite/kernels/unique.cc
 create mode 100644 tensorflow/lite/kernels/unique_test.cc

diff --git a/tensorflow/lite/build_def.bzl b/tensorflow/lite/build_def.bzl
index c17eddf47b..343ec60f2a 100644
--- a/tensorflow/lite/build_def.bzl
+++ b/tensorflow/lite/build_def.bzl
@@ -311,6 +311,7 @@ def generated_test_models():
         "topk",
         "transpose",
         "transpose_conv",
+        "unique",
         "unpack",
         "unroll_batch_matmul",
         "where",
diff --git a/tensorflow/lite/kernels/BUILD b/tensorflow/lite/kernels/BUILD
index 9abc33d297..6809a4a3d0 100644
--- a/tensorflow/lite/kernels/BUILD
+++ b/tensorflow/lite/kernels/BUILD
@@ -221,6 +221,7 @@ cc_library(
         "transpose_conv.cc",
         "unidirectional_sequence_lstm.cc",
         "unidirectional_sequence_rnn.cc",
+        "unique.cc",
         "unpack.cc",
         "zeros_like.cc",
     ],
@@ -1233,6 +1234,17 @@ tf_cc_test(
     ],
 )
 
+tf_cc_test(
+    name = "unique_test",
+    srcs = ["unique_test.cc"],
+    deps = [
+        ":builtin_ops",
+        ":test_util",
+        "//tensorflow/lite:framework",
+        "@com_google_googletest//:gtest_main",
+    ],
+)
+
 filegroup(
     name = "all_files",
     srcs = glob(
diff --git a/tensorflow/lite/kernels/register.cc b/tensorflow/lite/kernels/register.cc
index 47296fa105..f17f39fc2b 100644
--- a/tensorflow/lite/kernels/register.cc
+++ b/tensorflow/lite/kernels/register.cc
@@ -129,6 +129,7 @@ TfLiteRegistration* Register_LEAKY_RELU();
 TfLiteRegistration* Register_SQUARED_DIFFERENCE();
 TfLiteRegistration* Register_FILL();
 TfLiteRegistration* Register_MIRROR_PAD();
+TfLiteRegistration* Register_UNIQUE();
 
 TfLiteStatus UnsupportedTensorFlowOp(TfLiteContext* context, TfLiteNode* node) {
   context->ReportError(
@@ -284,6 +285,7 @@ BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_SQUARED_DIFFERENCE, Register_SQUARED_DIFFERENCE());
   AddBuiltin(BuiltinOperator_FILL, Register_FILL());
   AddBuiltin(BuiltinOperator_MIRROR_PAD, Register_MIRROR_PAD());
+  AddBuiltin(BuiltinOperator_UNIQUE, Register_UNIQUE());
 
   // TODO(andrewharp, ahentz): Move these somewhere more appropriate so that
   // custom ops aren't always included by default.
diff --git a/tensorflow/lite/kernels/unique.cc b/tensorflow/lite/kernels/unique.cc
new file mode 100644
index 0000000000..80c033aa5c
--- /dev/null
+++ b/tensorflow/lite/kernels/unique.cc
@@ -0,0 +1,164 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <map>
+
+#include "tensorflow/lite/c/builtin_op_data.h"
+#include "tensorflow/lite/c/c_api_internal.h"
+#include "tensorflow/lite/kernels/internal/reference/reference_ops.h"
+#include "tensorflow/lite/kernels/internal/tensor.h"
+#include "tensorflow/lite/kernels/kernel_util.h"
+
+namespace tflite {
+namespace ops {
+namespace builtin {
+namespace unique {
+
+void* Init(TfLiteContext* context, const char* buffer, size_t length) {
+  return nullptr;
+}
+
+void Free(TfLiteContext* context, void* buffer) {}
+
+TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
+  static const int kOutputUniqueTensor = 0;
+  static const int kOutputIndexTensor = 1;
+
+  TF_LITE_ENSURE_EQ(context, NumInputs(node), 1);
+  TF_LITE_ENSURE_EQ(context, NumOutputs(node), 2);
+  const TfLiteTensor* input = GetInput(context, node, 0);
+  TfLiteTensor* output_unique_tensor =
+      GetOutput(context, node, kOutputUniqueTensor);
+  TfLiteTensor* output_index_tensor =
+      GetOutput(context, node, kOutputIndexTensor);
+
+  // The op only supports 1D input.
+  TF_LITE_ENSURE_EQ(context, NumDimensions(input), 1);
+  TfLiteIntArray* output_index_shape = TfLiteIntArrayCopy(input->dims);
+  // The unique values are determined during evaluation, so we don't know yet
+  // the size of the output tensor.
+  SetTensorToDynamic(output_unique_tensor);
+  return context->ResizeTensor(context, output_index_tensor,
+                               output_index_shape);
+}
+
+namespace {
+
+// Actual evaluation for the unique op.
+template <typename T, typename I>
+TfLiteStatus EvalImpl(TfLiteContext* context, const TfLiteTensor* input,
+                      TfLiteNode* node) {
+  // Map from value, to index in the unique elements vector.
+  // Note that we prefer to use map than unordered_map as it showed less
+  // increase in the binary size.
+  std::map<T, int> unique_values;
+  TfLiteTensor* output_indexes = GetOutput(context, node, 1);
+  I* indexes = GetTensorData<I>(output_indexes);
+  const T* data = GetTensorData<T>(input);
+  const int num_elements = NumElements(input);
+
+  for (int i = 0; i < num_elements; ++i) {
+    const auto element_it = unique_values.find(data[i]);
+    if (element_it != unique_values.end()) {
+      indexes[i] = element_it->second;
+    } else {
+      const int unique_index = unique_values.size();
+      unique_values[data[i]] = unique_index;
+      indexes[i] = unique_index;
+    }
+  }
+  // Allocate output tensor.
+  TfLiteTensor* unique_output = GetOutput(context, node, 0);
+  std::unique_ptr<TfLiteIntArray, void (*)(TfLiteIntArray*)> shape(
+      TfLiteIntArrayCreate(NumDimensions(input)), TfLiteIntArrayFree);
+  shape->data[0] = unique_values.size();
+  TF_LITE_ENSURE_STATUS(
+      context->ResizeTensor(context, unique_output, shape.release()));
+  // Set the values in the output tensor.
+  T* output_unique_values = GetTensorData<T>(unique_output);
+  for (int i = 0; i < unique_values.size(); ++i) {
+    output_unique_values[i] = data[indexes[i]];
+  }
+  return kTfLiteOk;
+}
+
+template <typename T>
+TfLiteStatus EvalImpl(TfLiteContext* context, const TfLiteTensor* input,
+                      TfLiteNode* node) {
+  auto* params = reinterpret_cast<TfLiteUniqueParams*>(node->builtin_data);
+  if (params == nullptr) {
+    context->ReportError(context, "Null params passed");
+    return kTfLiteError;
+  }
+  switch (params->index_out_type) {
+    case kTfLiteInt32:
+      return EvalImpl<T, int32_t>(context, input, node);
+    case kTfLiteInt64:
+      return EvalImpl<T, int64_t>(context, input, node);
+    default:
+      context->ReportError(
+          context,
+          "Unique index output array can only be Int32 or In64, requested: ",
+          TfLiteTypeGetName(params->index_out_type));
+  }
+  return kTfLiteError;
+}
+
+}  // namespace
+
+TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
+  const TfLiteTensor* input = GetInput(context, node, 0);
+  TfLiteTensor* output_index_tensor = GetOutput(context, node, 1);
+  TF_LITE_ENSURE_EQ(context, NumElements(output_index_tensor),
+                    NumElements(input));
+
+  switch (input->type) {
+    case kTfLiteInt8:
+      TF_LITE_ENSURE_STATUS(EvalImpl<int8_t>(context, input, node));
+      break;
+    case kTfLiteInt16:
+      TF_LITE_ENSURE_STATUS(EvalImpl<int16_t>(context, input, node));
+      break;
+    case kTfLiteInt32:
+      TF_LITE_ENSURE_STATUS(EvalImpl<int32_t>(context, input, node));
+      break;
+    case kTfLiteInt64:
+      TF_LITE_ENSURE_STATUS(EvalImpl<int64_t>(context, input, node));
+      break;
+    case kTfLiteFloat32:
+      TF_LITE_ENSURE_STATUS(EvalImpl<float>(context, input, node));
+      break;
+    case kTfLiteUInt8:
+      TF_LITE_ENSURE_STATUS(EvalImpl<uint8_t>(context, input, node));
+      break;
+    default:
+      context->ReportError(context, "Currently Unique doesn't support type: %s",
+                           TfLiteTypeGetName(input->type));
+      return kTfLiteError;
+  }
+  return kTfLiteOk;
+}
+
+}  // namespace unique
+
+TfLiteRegistration* Register_UNIQUE() {
+  static TfLiteRegistration r = {unique::Init, unique::Free, unique::Prepare,
+                                 unique::Eval};
+  return &r;
+}
+
+}  // namespace builtin
+}  // namespace ops
+}  // namespace tflite
diff --git a/tensorflow/lite/kernels/unique_test.cc b/tensorflow/lite/kernels/unique_test.cc
new file mode 100644
index 0000000000..1df5e6b796
--- /dev/null
+++ b/tensorflow/lite/kernels/unique_test.cc
@@ -0,0 +1,103 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <gtest/gtest.h>
+#include "tensorflow/lite/interpreter.h"
+#include "tensorflow/lite/kernels/register.h"
+#include "tensorflow/lite/kernels/test_util.h"
+#include "tensorflow/lite/model.h"
+
+namespace tflite {
+namespace {
+
+using ::testing::ElementsAreArray;
+
+template <typename T, typename I>
+class UniqueOpModel : public SingleOpModel {
+ public:
+  UniqueOpModel(const TensorData& input, TensorType input_type,
+                TensorType index_out_type) {
+    input_id_ = AddInput(input);
+    output_id_ = AddOutput(input_type);
+    output_index_id_ = AddOutput(index_out_type);
+    SetBuiltinOp(BuiltinOperator_UNIQUE, BuiltinOptions_UniqueOptions,
+                 CreateUniqueOptions(builder_, index_out_type).Union());
+    BuildInterpreter({GetShape(input_id_)});
+  }
+
+  int input_tensor_id() { return input_id_; }
+
+  std::vector<T> GetOutput() { return ExtractVector<T>(output_id_); }
+  std::vector<I> GetIndexesOutput() {
+    return ExtractVector<I>(output_index_id_);
+  }
+
+ protected:
+  int input_id_;
+  int output_id_;
+  int output_index_id_;
+};
+
+TEST(UniqueOpModelTest, OneElement) {
+  UniqueOpModel<float, int32_t> model({TensorType_FLOAT32, {1}},
+                                      TensorType_FLOAT32, TensorType_INT32);
+  model.PopulateTensor<float>(model.input_tensor_id(), {5});
+  model.Invoke();
+  EXPECT_THAT(model.GetOutput(), ElementsAreArray({5}));
+  EXPECT_THAT(model.GetIndexesOutput(), ElementsAreArray({0}));
+}
+
+TEST(UniqueOpModelTest, MultipleElements_AllUnique) {
+  UniqueOpModel<float, int32_t> model({TensorType_FLOAT32, {8}},
+                                      TensorType_FLOAT32, TensorType_INT32);
+  model.PopulateTensor<float>(model.input_tensor_id(),
+                              {5, 2, 3, 51, 6, 72, 7, 8});
+  model.Invoke();
+  EXPECT_THAT(model.GetOutput(), ElementsAreArray({5, 2, 3, 51, 6, 72, 7, 8}));
+  EXPECT_THAT(model.GetIndexesOutput(),
+              ElementsAreArray({0, 1, 2, 3, 4, 5, 6, 7}));
+}
+
+TEST(UniqueOpModelTest, MultipleElements_AllDuplicates) {
+  UniqueOpModel<float, int32_t> model({TensorType_FLOAT32, {7}},
+                                      TensorType_FLOAT32, TensorType_INT32);
+  model.PopulateTensor<float>(model.input_tensor_id(), {5, 5, 5, 5, 5, 5, 5});
+  model.Invoke();
+  EXPECT_THAT(model.GetOutput(), ElementsAreArray({5}));
+  EXPECT_THAT(model.GetIndexesOutput(),
+              ElementsAreArray({0, 0, 0, 0, 0, 0, 0}));
+}
+
+TEST(UniqueOpModelTest, MultipleElements_SomeDuplicates) {
+  UniqueOpModel<float, int32_t> model({TensorType_FLOAT32, {7}},
+                                      TensorType_FLOAT32, TensorType_INT32);
+  model.PopulateTensor<float>(model.input_tensor_id(), {2, 3, 5, 7, 2, 7, 3});
+  model.Invoke();
+  EXPECT_THAT(model.GetOutput(), ElementsAreArray({2, 3, 5, 7}));
+  EXPECT_THAT(model.GetIndexesOutput(),
+              ElementsAreArray({0, 1, 2, 3, 0, 3, 1}));
+}
+
+TEST(UniqueOpModelTest, MultipleElements_SomeDuplicates_IndexInt64) {
+  UniqueOpModel<float, int64_t> model({TensorType_FLOAT32, {7}},
+                                      TensorType_FLOAT32, TensorType_INT64);
+  model.PopulateTensor<float>(model.input_tensor_id(), {2, 3, 5, 7, 2, 7, 3});
+  model.Invoke();
+  EXPECT_THAT(model.GetOutput(), ElementsAreArray({2, 3, 5, 7}));
+  EXPECT_THAT(model.GetIndexesOutput(),
+              ElementsAreArray({0, 1, 2, 3, 0, 3, 1}));
+}
+
+}  // namespace
+}  // namespace tflite
diff --git a/tensorflow/lite/testing/generate_examples.py b/tensorflow/lite/testing/generate_examples.py
index dd7b3d0745..87e7d7eb02 100644
--- a/tensorflow/lite/testing/generate_examples.py
+++ b/tensorflow/lite/testing/generate_examples.py
@@ -3749,6 +3749,55 @@ def make_placeholder_with_default_tests(zip_path):
                     expected_tf_success=3)
 
 
+def make_unique_tests(zip_path):
+  """Make a set of tests for Unique op."""
+
+  test_parameters = [
+      {
+          "input_shape": [[1]],
+          "index_type": [tf.int32, tf.int64, None],
+          "input_values": [3]
+      },
+      {
+          "input_shape": [[5]],
+          "index_type": [tf.int32, tf.int64],
+          "input_values": [[3, 2, 1, 2, 3]]
+      },
+      {
+          "input_shape": [[7]],
+          "index_type": [tf.int32, tf.int64],
+          "input_values": [[1, 1, 1, 1, 1, 1, 1]]
+      },
+      {
+          "input_shape": [[5]],
+          "index_type": [tf.int32, tf.int64],
+          "input_values": [[3, 2, 1, 0, -1]]
+      }]
+
+  def build_graph(parameters):
+    """Build the graph for the test case."""
+
+    input_tensor = tf.placeholder(
+        dtype=tf.int32, name="input", shape=parameters["input_shape"])
+    if parameters["index_type"] is None:
+      output = tf.unique(input_tensor)
+    else:
+      output = tf.unique(input_tensor, parameters["index_type"])
+
+    return [input_tensor], output
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_values = [create_tensor_data(tf.int32, parameters["input_shape"])]
+    return input_values, sess.run(
+        outputs, feed_dict=dict(zip(inputs, input_values)))
+
+  make_zip_of_tests(
+      zip_path,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_success=9)
+
 # Toco binary path provided by the generate rule.
 bin_path = None
 
diff --git a/tensorflow/lite/toco/graph_transformations/propagate_array_data_types.cc b/tensorflow/lite/toco/graph_transformations/propagate_array_data_types.cc
index cbae6610d7..6d9aad66b6 100644
--- a/tensorflow/lite/toco/graph_transformations/propagate_array_data_types.cc
+++ b/tensorflow/lite/toco/graph_transformations/propagate_array_data_types.cc
@@ -252,6 +252,14 @@ void SetDataTypeForAllOutputs(Model* model, Operator* op,
       SetDataTypeForAllOutputs(model, op, data_type);
       break;
     }
+    case OperatorType::kUnique: {
+      CHECK_EQ(op->outputs.size(), 2);
+      const UniqueOperator* unique_op = static_cast<UniqueOperator*>(op);
+      const ArrayDataType data_type = model->GetArray(op->inputs[0]).data_type;
+      model->GetArray(op->outputs[0]).data_type = data_type;
+      model->GetArray(op->outputs[1]).data_type = unique_op->idx_out_type;
+      break;
+    }
     default: {
       // These operators produce outputs with the same type as their 1st input
       CHECK_GT(op->inputs.size(), 0);
diff --git a/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc b/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
index 0e653f08a0..5185afd22e 100644
--- a/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
+++ b/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
@@ -1828,6 +1828,20 @@ void ProcessMirrorPadOperator(Model* model, MirrorPadOperator* op) {
   output_array.copy_shape(output_shape);
 }
 
+void ProcessUniqueOperator(Model* model, UniqueOperator* op) {
+  const auto& input_array = model->GetArray(op->inputs[0]);
+  // We have 2 outputs, the shape of the index tensor, is the same size
+  // as the input array. The unique values tensor, is unknown until runtime.
+  CHECK_EQ(op->outputs.size(), 2);
+  auto& idx_output_array = model->GetArray(op->outputs[1]);
+
+  // Yield until input dims have been resolved, or output already computed
+  if (!input_array.has_shape() || idx_output_array.has_shape()) {
+    return;
+  }
+  idx_output_array.copy_shape(input_array.shape());
+}
+
 }  // namespace
 
 ::tensorflow::Status PropagateFixedSizes::Run(Model* model,
@@ -2103,6 +2117,9 @@ void ProcessMirrorPadOperator(Model* model, MirrorPadOperator* op) {
     case OperatorType::kMirrorPad:
       ProcessMirrorPadOperator(model, static_cast<MirrorPadOperator*>(op));
       break;
+    case OperatorType::kUnique:
+      ProcessUniqueOperator(model, static_cast<UniqueOperator*>(op));
+      break;
     default:
       // Unimplemented, another graph transformation should drop it.
       LOG(FATAL) << "Unhandled operator type " << OperatorTypeName(op->type);
diff --git a/tensorflow/lite/toco/import_tensorflow.cc b/tensorflow/lite/toco/import_tensorflow.cc
index e1f7eb82ee..86e04b2393 100644
--- a/tensorflow/lite/toco/import_tensorflow.cc
+++ b/tensorflow/lite/toco/import_tensorflow.cc
@@ -1190,7 +1190,7 @@ enum FlexSupport { kFlexOk, kFlexNotOk };
 // taken from the given NodeDef, and its number must match NumInputs, unless
 // kAnyNumInputs is passed in. If kFlexOk is passed in the resulting operator
 // will be eligible for being exported as a flex op.
-template <typename Op, int NumInputs, FlexSupport flex>
+template <typename Op, int NumInputs, int NumOutputs, FlexSupport flex>
 tensorflow::Status ConvertSimpleOperatorGeneric(
     const NodeDef& node, const TensorFlowImportFlags& tf_import_flags,
     Model* model) {
@@ -1203,6 +1203,11 @@ tensorflow::Status ConvertSimpleOperatorGeneric(
     op->inputs.push_back(node.input(i));
   }
   op->outputs.push_back(node.name());
+  if (NumOutputs > 1) {
+    for (int i = 1; i < NumOutputs; ++i) {
+      op->outputs.push_back(node.name() + ":" + std::to_string(i));
+    }
+  }
 
   if (flex == kFlexOk) {
     RetainTensorFlowNodeDef(node, op);
@@ -1213,20 +1218,20 @@ tensorflow::Status ConvertSimpleOperatorGeneric(
 }
 
 // Convert a simple operator which is not valid as a flex op.
-template <typename Op, int NumInputs = kAnyNumInputs>
+template <typename Op, int NumInputs, int NumOutputs>
 tensorflow::Status ConvertSimpleOperator(
     const NodeDef& node, const TensorFlowImportFlags& tf_import_flags,
     Model* model) {
-  return ConvertSimpleOperatorGeneric<Op, NumInputs, kFlexNotOk>(
+  return ConvertSimpleOperatorGeneric<Op, NumInputs, NumOutputs, kFlexNotOk>(
       node, tf_import_flags, model);
 }
 
 // Convert a simple operator which is valid as a flex op.
-template <typename Op, int NumInputs = kAnyNumInputs>
+template <typename Op, int NumInputs, int NumOutputs>
 tensorflow::Status ConvertSimpleOperatorFlexOk(
     const NodeDef& node, const TensorFlowImportFlags& tf_import_flags,
     Model* model) {
-  return ConvertSimpleOperatorGeneric<Op, NumInputs, kFlexOk>(
+  return ConvertSimpleOperatorGeneric<Op, NumInputs, NumOutputs, kFlexOk>(
       node, tf_import_flags, model);
 }
 
@@ -2333,14 +2338,15 @@ ConverterMapType GetTensorFlowNodeConverterMapForFlex() {
 
 ConverterMapType GetTensorFlowNodeConverterMap() {
   return std::unordered_map<std::string, ConverterType>({
-      {"Abs", ConvertSimpleOperator<AbsOperator>},
-      {"Add", ConvertSimpleOperator<AddOperator, 2>},
-      {"AddN", ConvertSimpleOperatorFlexOk<AddNOperator>},
-      {"All", ConvertSimpleOperator<TensorFlowAllOperator>},
+      {"Abs", ConvertSimpleOperator<AbsOperator, kAnyNumInputs, 1>},
+      {"Add", ConvertSimpleOperator<AddOperator, 2, 1>},
+      {"AddN", ConvertSimpleOperatorFlexOk<AddNOperator, kAnyNumInputs, 1>},
+      {"All", ConvertSimpleOperator<TensorFlowAllOperator, kAnyNumInputs, 1>},
       {"Any", ConvertReduceOperator<TensorFlowAnyOperator>},
       {"ArgMax", ConvertArgMaxOperator},
       {"ArgMin", ConvertArgMinOperator},
-      {"Assert", ConvertSimpleOperator<TensorFlowAssertOperator>},
+      {"Assert",
+       ConvertSimpleOperator<TensorFlowAssertOperator, kAnyNumInputs, 1>},
       {"AvgPool", ConvertAvgPoolOperator},
       {"BatchMatMul", ConvertBatchMatMulOperator},
       {"BatchNormWithGlobalNormalization",
@@ -2357,98 +2363,99 @@ ConverterMapType GetTensorFlowNodeConverterMap() {
       {"CTCBeamSearchDecoder", ConvertCTCBeamSearchDecoderOperator},
       {"DepthToSpace", ConvertDepthToSpaceOperator},
       {"DepthwiseConv2dNative", ConvertDepthwiseConvOperator},
-      {"Div", ConvertSimpleOperator<DivOperator, 2>},
+      {"Div", ConvertSimpleOperator<DivOperator, 2, 1>},
       {"DynamicPartition", ConvertDynamicPartitionOperator},
       {"DynamicStitch", ConvertDynamicStitchOperator},
-      {"Equal", ConvertSimpleOperator<TensorFlowEqualOperator, 2>},
-      {"Exp", ConvertSimpleOperator<ExpOperator, 1>},
-      {"ExpandDims", ConvertSimpleOperator<ExpandDimsOperator, 2>},
+      {"Equal", ConvertSimpleOperator<TensorFlowEqualOperator, 2, 1>},
+      {"Exp", ConvertSimpleOperator<ExpOperator, 1, 1>},
+      {"ExpandDims", ConvertSimpleOperator<ExpandDimsOperator, 2, 1>},
       {"FakeQuantWithMinMaxArgs", ConvertFakeQuantWithMinMaxArgs},
       {"FakeQuantWithMinMaxVars", ConvertFakeQuantWithMinMaxVars},
-      {"Fill", ConvertSimpleOperator<FillOperator, 2>},
+      {"Fill", ConvertSimpleOperator<FillOperator, 2, 1>},
       {"Floor", ConvertFloorOperator},
-      {"FloorDiv", ConvertSimpleOperator<FloorDivOperator, 2>},
-      {"FloorMod", ConvertSimpleOperator<FloorModOperator, 2>},
+      {"FloorDiv", ConvertSimpleOperator<FloorDivOperator, 2, 1>},
+      {"FloorMod", ConvertSimpleOperator<FloorModOperator, 2, 1>},
       {"FusedBatchNorm", ConvertFusedBatchNormOperator},
       {"Gather", ConvertGatherOperator},
       {"GatherV2", ConvertGatherOperator},
-      {"Greater", ConvertSimpleOperator<TensorFlowGreaterOperator, 2>},
+      {"Greater", ConvertSimpleOperator<TensorFlowGreaterOperator, 2, 1>},
       {"GreaterEqual",
-       ConvertSimpleOperator<TensorFlowGreaterEqualOperator, 2>},
+       ConvertSimpleOperator<TensorFlowGreaterEqualOperator, 2, 1>},
       {"Identity", ConvertIdentityOperator},
       {"LRN", ConvertLRNOperator},
       {"LeakyRelu", ConvertLeakyReluOperator},
       {"LegacyFedInput", ConvertPlaceholderOperator},
-      {"Less", ConvertSimpleOperator<TensorFlowLessOperator, 2>},
-      {"LessEqual", ConvertSimpleOperator<TensorFlowLessEqualOperator, 2>},
-      {"Log", ConvertSimpleOperator<LogOperator, 1>},
-      {"LogicalAnd", ConvertSimpleOperator<LogicalAndOperator, 2>},
-      {"LogicalOr", ConvertSimpleOperator<LogicalOrOperator, 2>},
-      {"LogicalNot", ConvertSimpleOperator<LogicalNotOperator, 1>},
-      {"LogSoftmax", ConvertSimpleOperator<LogSoftmaxOperator, 1>},
+      {"Less", ConvertSimpleOperator<TensorFlowLessOperator, 2, 1>},
+      {"LessEqual", ConvertSimpleOperator<TensorFlowLessEqualOperator, 2, 1>},
+      {"Log", ConvertSimpleOperator<LogOperator, 1, 1>},
+      {"LogicalAnd", ConvertSimpleOperator<LogicalAndOperator, 2, 1>},
+      {"LogicalOr", ConvertSimpleOperator<LogicalOrOperator, 2, 1>},
+      {"LogicalNot", ConvertSimpleOperator<LogicalNotOperator, 1, 1>},
+      {"LogSoftmax", ConvertSimpleOperator<LogSoftmaxOperator, 1, 1>},
       {"MatMul", ConvertMatMulOperator},
       {"Max", ConvertReduceOperator<TensorFlowMaxOperator>},
       {"MaxPool", ConvertMaxPoolOperator},
-      {"Maximum", ConvertSimpleOperator<TensorFlowMaximumOperator, 2>},
+      {"Maximum", ConvertSimpleOperator<TensorFlowMaximumOperator, 2, 1>},
       {"Mean", ConvertReduceOperator<MeanOperator>},
-      {"Merge", ConvertSimpleOperator<TensorFlowMergeOperator, 2>},
+      {"Merge", ConvertSimpleOperator<TensorFlowMergeOperator, 2, 1>},
       {"Min", ConvertReduceOperator<TensorFlowMinOperator>},
-      {"Minimum", ConvertSimpleOperator<TensorFlowMinimumOperator, 2>},
-      {"Mul", ConvertSimpleOperator<MulOperator, 2>},
-      {"Neg", ConvertSimpleOperator<NegOperator, 1>},
+      {"Minimum", ConvertSimpleOperator<TensorFlowMinimumOperator, 2, 1>},
+      {"Mul", ConvertSimpleOperator<MulOperator, 2, 1>},
+      {"Neg", ConvertSimpleOperator<NegOperator, 1, 1>},
       {"NextIteration", ConvertOperatorSpecialCasedAsRNNBackEdge},
       {"NoOp", ConvertNoOpOperator},
-      {"NotEqual", ConvertSimpleOperator<TensorFlowNotEqualOperator, 2>},
+      {"NotEqual", ConvertSimpleOperator<TensorFlowNotEqualOperator, 2, 1>},
       {"OneHot", ConvertOneHotOperator},
       {"Pack", ConvertPackOperator},
-      {"Pad", ConvertSimpleOperator<PadOperator, 2>},
-      {"PadV2", ConvertSimpleOperator<PadV2Operator, 3>},
+      {"Pad", ConvertSimpleOperator<PadOperator, 2, 1>},
+      {"PadV2", ConvertSimpleOperator<PadV2Operator, 3, 1>},
       {"ParallelDynamicStitch", ConvertDynamicStitchOperator},
       {"Placeholder", ConvertPlaceholderOperator},
       {"PlaceholderWithDefault", ConvertIdentityOperator},
-      {"Pow", ConvertSimpleOperator<PowOperator, 2>},
+      {"Pow", ConvertSimpleOperator<PowOperator, 2, 1>},
       {"Prod", ConvertReduceOperator<TensorFlowProdOperator>},
       {"RandomUniform", ConvertRandomUniform},
       {"Range", ConvertRangeOperator},
-      {"Rank", ConvertSimpleOperator<RankOperator, 1>},
-      {"RealDiv", ConvertSimpleOperator<DivOperator, 2>},
-      {"Relu", ConvertSimpleOperator<ReluOperator, 1>},
-      {"Relu6", ConvertSimpleOperator<Relu6Operator, 1>},
-      {"Reshape", ConvertSimpleOperator<TensorFlowReshapeOperator, 2>},
+      {"Rank", ConvertSimpleOperator<RankOperator, 1, 1>},
+      {"RealDiv", ConvertSimpleOperator<DivOperator, 2, 1>},
+      {"Relu", ConvertSimpleOperator<ReluOperator, 1, 1>},
+      {"Relu6", ConvertSimpleOperator<Relu6Operator, 1, 1>},
+      {"Reshape", ConvertSimpleOperator<TensorFlowReshapeOperator, 2, 1>},
       {"ResizeBilinear", ConvertResizeBilinearOperator},
       {"ResizeNearestNeighbor", ConvertResizeNearestNeighborOperator},
-      {"Rsqrt", ConvertSimpleOperator<TensorFlowRsqrtOperator, 1>},
-      {"Select", ConvertSimpleOperator<SelectOperator, 3>},
+      {"Rsqrt", ConvertSimpleOperator<TensorFlowRsqrtOperator, 1, 1>},
+      {"Select", ConvertSimpleOperator<SelectOperator, 3, 1>},
       {"Shape", ConvertShapeOperator},
-      {"Sigmoid", ConvertSimpleOperator<LogisticOperator, 1>},
-      {"Sin", ConvertSimpleOperator<SinOperator, 1>},
-      {"Slice", ConvertSimpleOperator<SliceOperator, 3>},
+      {"Sigmoid", ConvertSimpleOperator<LogisticOperator, 1, 1>},
+      {"Sin", ConvertSimpleOperator<SinOperator, 1, 1>},
+      {"Slice", ConvertSimpleOperator<SliceOperator, 3, 1>},
       {"Softmax", ConvertSoftmaxOperator},
       {"SpaceToBatchND", ConvertSpaceToBatchNDOperator},
       {"SpaceToDepth", ConvertSpaceToDepthOperator},
       {"SparseToDense", ConvertSparseToDenseOperator},
       {"Split", ConvertSplitOperator},
       {"SplitV", ConvertSplitVOperator},
-      {"Sqrt", ConvertSimpleOperator<TensorFlowSqrtOperator, 1>},
-      {"Square", ConvertSimpleOperator<TensorFlowSquareOperator, 1>},
+      {"Sqrt", ConvertSimpleOperator<TensorFlowSqrtOperator, 1, 1>},
+      {"Square", ConvertSimpleOperator<TensorFlowSquareOperator, 1, 1>},
       {"SquaredDifference",
-       ConvertSimpleOperator<SquaredDifferenceOperator, 2>},
+       ConvertSimpleOperator<SquaredDifferenceOperator, 2, 1>},
       {"Squeeze", ConvertSqueezeOperator},
       {"StopGradient", ConvertIdentityOperator},
       {"StridedSlice", ConvertStridedSliceOperator},
-      {"Sub", ConvertSimpleOperator<SubOperator, 2>},
+      {"Sub", ConvertSimpleOperator<SubOperator, 2, 1>},
       {"Sum", ConvertReduceOperator<TensorFlowSumOperator>},
       {"Svdf", ConvertSvdfOperator},
       {"Switch", ConvertSwitchOperator},
-      {"Tanh", ConvertSimpleOperator<TanhOperator, 1>},
-      {"Tile", ConvertSimpleOperator<TensorFlowTileOperator, 2>},
+      {"Tanh", ConvertSimpleOperator<TanhOperator, 1, 1>},
+      {"Tile", ConvertSimpleOperator<TensorFlowTileOperator, 2, 1>},
       {"TopK", ConvertTopKV2Operator},
       {"TopKV2", ConvertTopKV2Operator},
-      {"Transpose", ConvertSimpleOperator<TransposeOperator, 2>},
+      {"Transpose", ConvertSimpleOperator<TransposeOperator, 2, 1>},
       {"Unpack", ConvertUnpackOperator},
-      {"ZerosLike", ConvertSimpleOperator<TensorFlowZerosLikeOperator, 1>},
+      {"ZerosLike", ConvertSimpleOperator<TensorFlowZerosLikeOperator, 1, 1>},
       {"UnidirectionalSequenceLstm", ConvertUnidirectionalSequenceLstm},
       {"MirrorPad", ConvertMirrorPadOperator},
+      {"Unique", ConvertSimpleOperator<UniqueOperator, 1, 2>},
   });
 }
 
diff --git a/tensorflow/lite/toco/model.h b/tensorflow/lite/toco/model.h
index e71d36583e..bfa86c8059 100644
--- a/tensorflow/lite/toco/model.h
+++ b/tensorflow/lite/toco/model.h
@@ -157,7 +157,8 @@ enum class OperatorType : uint8 {
   kResizeNearestNeighbor,
   kLeakyRelu,
   kAbs,
-  kMirrorPad
+  kMirrorPad,
+  kUnique
 };
 
 // Helper to deal with TensorFlow arrays using a different ordering of
@@ -1953,6 +1954,17 @@ struct MirrorPadOperator : Operator {
   MirrorPadMode mode;
 };
 
+// Unique Operator:
+//
+// Inputs:
+//   inputs[0]: required: the input array
+//
+// TensorFlow equivalent: Unique
+struct UniqueOperator : Operator {
+  UniqueOperator() : Operator(OperatorType::kUnique) {}
+  ArrayDataType idx_out_type = ArrayDataType::kInt32;
+};
+
 // Alloc's are used for transient arrays only. An Alloc specifies which interval
 // of the "transient_data" workspace buffer passed to inference functions, is to
 // be used for the transient array at hand. The 'start' and 'end' values are
diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index d3fce6893f..8eb4d321ef 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -21,6 +21,7 @@ limitations under the License.
 #include "tensorflow/core/util/ptr_util.h"
 // TODO(ycling): Consider refactoring to extract the LSTM definition out of
 // graph_transformation module.
+#include "tensorflow/lite/schema/schema_generated.h"
 #include "tensorflow/lite/toco/graph_transformations/lstm_utils.h"
 #include "tensorflow/lite/toco/model.h"
 #include "tensorflow/lite/toco/tflite/builtin_operator.h"
@@ -1478,6 +1479,31 @@ class MirrorPad
                    : MirrorPadMode::kSymmetric;
   }
 
+  int GetVersion(const OperatorSignature& op) const override { return 1; }
+};
+
+class Unique : public BuiltinOperator<UniqueOperator, ::tflite::UniqueOptions,
+                                      ::tflite::BuiltinOptions_UniqueOptions> {
+ public:
+  using BuiltinOperator::BuiltinOperator;
+  flatbuffers::Offset<TfLiteOptions> WriteOptions(
+      const TocoOperator& op,
+      flatbuffers::FlatBufferBuilder* builder) const override {
+    const UniqueOperator& unique_op = static_cast<const UniqueOperator&>(op);
+    return ::tflite::CreateUniqueOptions(
+        *builder, unique_op.idx_out_type == toco::ArrayDataType::kInt64
+                      ? ::tflite::TensorType::TensorType_INT64
+                      : ::tflite::TensorType_INT32);
+  }
+  void ReadOptions(const TfLiteOptions& options,
+                   TocoOperator* op) const override {
+    UniqueOperator* unique_op = static_cast<UniqueOperator*>(op);
+    unique_op->idx_out_type =
+        options.idx_out_type() == ::tflite::TensorType_INT64
+            ? toco::ArrayDataType::kInt64
+            : toco::ArrayDataType::kInt32;
+  }
+
   int GetVersion(const OperatorSignature& op_signature) const override {
     return 1;
   }
@@ -1819,6 +1845,8 @@ std::vector<std::unique_ptr<BaseOperator>> BuildOperatorList(
       OperatorType::kSquaredDifference));
   ops.push_back(MakeUnique<MirrorPad>(::tflite::BuiltinOperator_MIRROR_PAD,
                                       OperatorType::kMirrorPad));
+  ops.push_back(MakeUnique<Unique>(::tflite::BuiltinOperator_UNIQUE,
+                                   OperatorType::kUnique));
 
   // Custom Operators.
   ops.push_back(
diff --git a/tensorflow/lite/toco/tflite/operator_test.cc b/tensorflow/lite/toco/tflite/operator_test.cc
index 849eace8cc..215eda82f6 100644
--- a/tensorflow/lite/toco/tflite/operator_test.cc
+++ b/tensorflow/lite/toco/tflite/operator_test.cc
@@ -629,6 +629,15 @@ TEST_F(OperatorTest, BuiltinMirrorPad) {
   EXPECT_EQ(op.mode, output_toco_op->mode);
 }
 
+TEST_F(OperatorTest, BuiltinUnique) {
+  UniqueOperator op;
+  op.idx_out_type = ArrayDataType::kInt64;
+  auto output_toco_op =
+      SerializeAndDeserialize(GetOperator("UNIQUE", OperatorType::kUnique), op);
+  ASSERT_NE(nullptr, output_toco_op.get());
+  EXPECT_EQ(output_toco_op->idx_out_type, op.idx_out_type);
+}
+
 }  // namespace
 }  // namespace tflite
 
diff --git a/tensorflow/lite/toco/tooling_util.cc b/tensorflow/lite/toco/tooling_util.cc
index af4cd386a2..2396de1a3d 100644
--- a/tensorflow/lite/toco/tooling_util.cc
+++ b/tensorflow/lite/toco/tooling_util.cc
@@ -416,6 +416,7 @@ const char* OperatorTypeName(OperatorType type) {
     HANDLE_OPERATORTYPENAME_CASE(LeakyRelu)
     HANDLE_OPERATORTYPENAME_CASE(SquaredDifference)
     HANDLE_OPERATORTYPENAME_CASE(MirrorPad)
+    HANDLE_OPERATORTYPENAME_CASE(Unique)
     default:
       LOG(FATAL) << "Unhandled op type";
 #undef HANDLE_OPERATORTYPENAME_CASE
-- 
GitLab


From b2db5d4089f6cebfabd461401eeac4ed8a976fe1 Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Mon, 7 Jan 2019 11:30:46 -0800
Subject: [PATCH 0305/1095] Fix windows build

PiperOrigin-RevId: 228203086
---
 tensorflow/core/framework/resource_handle.cc | 3 ---
 tensorflow/core/framework/resource_handle.h  | 3 ++-
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/framework/resource_handle.cc b/tensorflow/core/framework/resource_handle.cc
index a3c48d63d4..fc3a329b3b 100644
--- a/tensorflow/core/framework/resource_handle.cc
+++ b/tensorflow/core/framework/resource_handle.cc
@@ -19,9 +19,6 @@ limitations under the License.
 
 namespace tensorflow {
 
-const char ResourceHandle::ANONYMOUS_NAME[] =
-    "cd2c89b7-88b7-44c8-ad83-06c2a9158347";
-
 ResourceHandle::ResourceHandle() {}
 
 ResourceHandle::ResourceHandle(const ResourceHandleProto& proto) {
diff --git a/tensorflow/core/framework/resource_handle.h b/tensorflow/core/framework/resource_handle.h
index c3beffb8ac..d1f6771bf3 100644
--- a/tensorflow/core/framework/resource_handle.h
+++ b/tensorflow/core/framework/resource_handle.h
@@ -69,7 +69,8 @@ class ResourceHandle {
 
   // GUID for anonymous resources. Resources with this shared_name will have
   // their shared_name replaced with a GUID at creation time
-  static const char ANONYMOUS_NAME[];
+  static constexpr const char* ANONYMOUS_NAME =
+      "cd2c89b7-88b7-44c8-ad83-06c2a9158347";
 
  public:
   string device_;
-- 
GitLab


From 2325650d59511ad1e19e0102cf920b48cca0bfc5 Mon Sep 17 00:00:00 2001
From: Jian Li <jianlijianli@google.com>
Date: Mon, 7 Jan 2019 11:31:30 -0800
Subject: [PATCH 0306/1095] Remove false comments in portable kernel.

PiperOrigin-RevId: 228203258
---
 .../lite/kernels/internal/reference/portable_tensor_utils.cc     | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tensorflow/lite/kernels/internal/reference/portable_tensor_utils.cc b/tensorflow/lite/kernels/internal/reference/portable_tensor_utils.cc
index d692063a96..1acf0caad0 100644
--- a/tensorflow/lite/kernels/internal/reference/portable_tensor_utils.cc
+++ b/tensorflow/lite/kernels/internal/reference/portable_tensor_utils.cc
@@ -101,7 +101,6 @@ void PortableMatrixBatchVectorMultiplyAccumulate(
       __builtin_prefetch(row_ptr, 0 /* prefetch for read */,
                          3 /* temporal locality */);
 #endif
-      // For every block of 16 8-bit elements (128-bit register) from each row.
       for (col = 0; col < m_cols; ++col, ++row_ptr) {
         dotprod += (*row_ptr) * (vectors[col]);
       }  // for col
-- 
GitLab


From e740529d3f09071ef0691a8af24dc566f96cbbec Mon Sep 17 00:00:00 2001
From: Nick Felt <nickfelt@google.com>
Date: Mon, 7 Jan 2019 11:36:12 -0800
Subject: [PATCH 0307/1095] Ensure tf.linspace(start, stop, num)[-1] == stop

The contract of tf.linspace says "A sequence of num evenly-spaced values are generated beginning at start. If num > 1, the values in the sequence increase by stop - start / num - 1, so that the last one is exactly stop."

But the existing logic computes the last value as `start + (num - 1) * ((start - stop) / (num - 1))`.  This exactly equals `stop` for real numbers, but not for floating point numbers.  In particular, for start = 0.0, stop = 1.0, num = 42, this reduces to (1.0/41.0)*41.0, and for IEEE 754 32-bit floats, that resolves to ~0.99999994, not 1.0 (binary values 0x3F7FFFFF vs 0x3F800000).

After this change, `tf.linspace(start, stop, num)[-1] == stop` holds for all values assuming num > 1, because we just set it directly to `stop`.  This matches what numpy does here: https://github.com/numpy/numpy/blob/v1.15.0/numpy/core/function_base.py#L144-L145

PiperOrigin-RevId: 228204335
---
 tensorflow/core/kernels/sequence_ops.cc       |  9 ++++----
 tensorflow/core/kernels/sequence_ops_test.cc  | 21 +++++++++++++++++++
 .../python/kernel_tests/init_ops_test.py      | 16 ++++++++++++++
 3 files changed, 42 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/kernels/sequence_ops.cc b/tensorflow/core/kernels/sequence_ops.cc
index 9db0bd4d98..21c3b89f54 100644
--- a/tensorflow/core/kernels/sequence_ops.cc
+++ b/tensorflow/core/kernels/sequence_ops.cc
@@ -143,11 +143,12 @@ class LinSpaceOp : public OpKernel {
     OP_REQUIRES_OK(context,
                    context->allocate_output(0, TensorShape({num}), &out));
     auto flat = out->flat<T>();
-    if (num == 1) {
-      flat(0) = start;
-    } else {
+    flat(0) = start;
+    if (num > 1) {
       const T step = (stop - start) / (num - 1);
-      for (Tnum i = 0; i < num; ++i) flat(i) = start + step * i;
+      for (Tnum i = 1; i < num - 1; ++i) flat(i) = start + step * i;
+      // Ensure final value == stop; float arithmetic won't guarantee this.
+      flat(num - 1) = stop;
     }
   }
 };
diff --git a/tensorflow/core/kernels/sequence_ops_test.cc b/tensorflow/core/kernels/sequence_ops_test.cc
index 5f0e0a69a8..2247c44750 100644
--- a/tensorflow/core/kernels/sequence_ops_test.cc
+++ b/tensorflow/core/kernels/sequence_ops_test.cc
@@ -114,6 +114,27 @@ TEST_F(LinSpaceOpTest, Simple_D32) {
   test::ExpectTensorEqual<float>(expected, *GetOutput(0));
 }
 
+TEST_F(LinSpaceOpTest, Exact_Endpoints) {
+  MakeOp(DT_FLOAT, DT_INT32);
+
+  // Feed and run. The particular values 0., 1., and 42 are chosen to test that
+  // the last value is not calculated via an intermediate delta as (1./41)*41,
+  // because for IEEE 32-bit floats that returns 0.99999994 != 1.0.
+  AddInputFromArray<float>(TensorShape({}), {0.0});
+  AddInputFromArray<float>(TensorShape({}), {1.0});
+  AddInputFromArray<int32>(TensorShape({}), {42});
+  TF_ASSERT_OK(RunOpKernel());
+
+  // Check the output
+  Tensor output = *GetOutput(0);
+  float expected_start = 0.0;
+  float start = output.flat<float>()(0);
+  EXPECT_EQ(expected_start, start) << expected_start << " vs. " << start;
+  float expected_stop = 1.0;
+  float stop = output.flat<float>()(output.NumElements() - 1);
+  EXPECT_EQ(expected_stop, stop) << expected_stop << " vs. " << stop;
+}
+
 TEST_F(LinSpaceOpTest, Single_D64) {
   MakeOp(DT_FLOAT, DT_INT64);
 
diff --git a/tensorflow/python/kernel_tests/init_ops_test.py b/tensorflow/python/kernel_tests/init_ops_test.py
index 09b9944baa..4b9681afd2 100644
--- a/tensorflow/python/kernel_tests/init_ops_test.py
+++ b/tensorflow/python/kernel_tests/init_ops_test.py
@@ -592,6 +592,22 @@ class LinSpaceTest(test.TestCase):
       self.assertArrayNear(self._LinSpace(5., 5., 3), np.array([5.] * 3), 1e-5)
       self.assertArrayNear(self._LinSpace(5., 5., 4), np.array([5.] * 4), 1e-5)
 
+  def testEndpointsAreExact(self):
+    for self.force_gpu in self._gpu_modes():
+      # Test some cases that produce last values not equal to "stop" when
+      # computed via start + (num - 1) * ((stop - start) / (num - 1)), since
+      # float arithmetic will introduce error through precision loss.
+      self.assertAllEqual(
+          self._LinSpace(0., 1., 42)[[0, -1]], np.array([0., 1.], np.float32))
+      self.assertAllEqual(
+          self._LinSpace(-1., 0., 42)[[0, -1]], np.array([-1., 0.], np.float32))
+      self.assertAllEqual(
+          self._LinSpace(.1, .2, 4)[[0, -1]], np.array([.1, .2], np.float32))
+      # Check a case for float64 error too.
+      self.assertAllEqual(
+          self._LinSpace(np.array(0., np.float64), .1, 12)[[0, -1]],
+          np.array([0., .1], np.float64))
+
 
 class DeviceTest(test.TestCase):
 
-- 
GitLab


From 5167dd3495de7e857db8a2b578a08db8139c7d29 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Mon, 7 Jan 2019 11:53:00 -0800
Subject: [PATCH 0308/1095] Remove a remove_checkpoint deprecation warning
 showing up in a 2.x API

Inlines the only useful bits

PiperOrigin-RevId: 228207585
---
 tensorflow/python/training/checkpoint_management.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/training/checkpoint_management.py b/tensorflow/python/training/checkpoint_management.py
index a7ad1f70e5..21fa6b3b5d 100644
--- a/tensorflow/python/training/checkpoint_management.py
+++ b/tensorflow/python/training/checkpoint_management.py
@@ -621,7 +621,8 @@ class CheckpointManager(object):
                >= self._last_preserved_timestamp)):
         self._last_preserved_timestamp = timestamp
         continue
-      remove_checkpoint(filename)
+      _delete_file_if_exists(filename + ".index")
+      _delete_file_if_exists(filename + ".data-?????-of-?????")
 
   def _record_state(self):
     """Saves the `CheckpointManager`'s state in `directory`."""
-- 
GitLab


From f75b3374770d75b2f5b412ba4b32b497bec8b6aa Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 7 Jan 2019 12:05:42 -0800
Subject: [PATCH 0309/1095] fix spacing

PiperOrigin-RevId: 228210397
---
 tensorflow/python/grappler/cluster.i | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/grappler/cluster.i b/tensorflow/python/grappler/cluster.i
index 87795ffcfb..b0c1f71a85 100644
--- a/tensorflow/python/grappler/cluster.i
+++ b/tensorflow/python/grappler/cluster.i
@@ -132,7 +132,7 @@ struct GCluster {
 
 static GCluster TF_NewCluster(bool allow_soft_placement,
                    bool disable_detailed_stats, TF_Status* out_status) {
-    int num_cpu_cores = tensorflow::grappler::GetNumAvailableLogicalCPUCores();
+  int num_cpu_cores = tensorflow::grappler::GetNumAvailableLogicalCPUCores();
   int num_gpus = tensorflow::grappler::GetNumAvailableGPUs();
   int timeout_s = 60 * 10;
   tensorflow::grappler::Cluster* cluster_ =
-- 
GitLab


From 27822e43c671c7705b2be24891cc40ad628d1c91 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 7 Jan 2019 12:12:45 -0800
Subject: [PATCH 0310/1095] fix typos

PiperOrigin-RevId: 228211568
---
 tensorflow/core/grappler/clusters/virtual_cluster.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/grappler/clusters/virtual_cluster.cc b/tensorflow/core/grappler/clusters/virtual_cluster.cc
index dbd8f26c28..e80d6b54d9 100644
--- a/tensorflow/core/grappler/clusters/virtual_cluster.cc
+++ b/tensorflow/core/grappler/clusters/virtual_cluster.cc
@@ -67,8 +67,8 @@ Status VirtualCluster::Run(const GraphDef& graph,
                            const std::vector<string>& fetch,
                            RunMetadata* metadata) {
   // Initialize a virtual scheduler to process the graph. Make sure to use
-  // static shape inference to prevent the schedulrer from calling the Run
-  // method on the cluster, and create an infinite loop.
+  // static shape inference to prevent the scheduler from calling the Run
+  // method on the cluster and creating an infinite loop.
   GrapplerItem item;
   item.graph = graph;
   item.feed = feed;
-- 
GitLab


From 7ba8db889667a757339714253bb70f77bca5b2f0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 7 Jan 2019 12:18:06 -0800
Subject: [PATCH 0311/1095] Add missing include for `std::fegetround()`

PiperOrigin-RevId: 228212438
---
 tensorflow/core/platform/setround.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/core/platform/setround.cc b/tensorflow/core/platform/setround.cc
index 592626bfa1..5573b2fc93 100644
--- a/tensorflow/core/platform/setround.cc
+++ b/tensorflow/core/platform/setround.cc
@@ -15,6 +15,8 @@ limitations under the License.
 
 #include "tensorflow/core/platform/setround.h"
 
+#include <cfenv>  // NOLINT
+
 namespace tensorflow {
 namespace port {
 
-- 
GitLab


From 7236c9bb7612fb1a0212dd81f934e0e68fddc0a2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 7 Jan 2019 12:18:25 -0800
Subject: [PATCH 0312/1095] Fix issue with passing EagerTensors to fit when
 run_eagerly=False

PiperOrigin-RevId: 228212493
---
 .../python/keras/engine/training_arrays.py    |  4 ++
 .../keras/engine/training_eager_test.py       | 39 +++++++++++++------
 .../python/keras/engine/training_utils.py     | 19 +++++++++
 3 files changed, 51 insertions(+), 11 deletions(-)

diff --git a/tensorflow/python/keras/engine/training_arrays.py b/tensorflow/python/keras/engine/training_arrays.py
index 97025a9e18..9887fe34a8 100644
--- a/tensorflow/python/keras/engine/training_arrays.py
+++ b/tensorflow/python/keras/engine/training_arrays.py
@@ -200,6 +200,10 @@ def model_iteration(model,
   use_steps = steps_per_epoch is not None
   do_validation = val_inputs is not None
 
+  # Convert Eager Tensors to NumPy arrays to support batching/shuffling.
+  inputs, targets, sample_weights = training_utils. \
+      convert_eager_tensors_to_numpy((inputs, targets, sample_weights))
+
   # Prepare input data.
   ins = _prepare_feed_values(model, inputs, targets, sample_weights, mode)
   num_samples_or_steps = _get_num_samples_or_steps(ins, batch_size,
diff --git a/tensorflow/python/keras/engine/training_eager_test.py b/tensorflow/python/keras/engine/training_eager_test.py
index 27eaea23ba..a6e2c24ec2 100644
--- a/tensorflow/python/keras/engine/training_eager_test.py
+++ b/tensorflow/python/keras/engine/training_eager_test.py
@@ -28,13 +28,20 @@ from tensorflow.python.keras import keras_parameterized
 from tensorflow.python.keras import metrics as metrics_module
 from tensorflow.python.keras import testing_utils
 from tensorflow.python.keras.optimizer_v2 import rmsprop
+from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
 
 
 class TrainingTest(keras_parameterized.TestCase):
 
   @keras_parameterized.run_with_all_model_types(exclude_models='sequential')
+  @keras_parameterized.run_all_keras_modes
   def test_model_methods_with_eager_tensors_multi_io(self):
+    if not context.executing_eagerly():
+      # Only test V2 Function and V2 Eager modes, as V1 Graph mode with
+      # symbolic tensors has different requirements.
+      return
+
     input_a = keras.layers.Input(shape=(3,), name='input_a')
     input_b = keras.layers.Input(shape=(3,), name='input_b')
 
@@ -53,13 +60,13 @@ class TrainingTest(keras_parameterized.TestCase):
         loss,
         metrics=metrics,
         loss_weights=loss_weights,
-        run_eagerly=True,
+        run_eagerly=testing_utils.should_run_eagerly(),
         sample_weight_mode=None)
 
-    input_a = keras.backend.zeros(shape=(10, 3))
-    input_b = keras.backend.zeros(shape=(10, 3))
-    target_a = keras.backend.zeros(shape=(10, 4))
-    target_b = keras.backend.zeros(shape=(10, 4))
+    input_a = array_ops.zeros(shape=(10, 3))
+    input_b = array_ops.zeros(shape=(10, 3))
+    target_a = array_ops.zeros(shape=(10, 4))
+    target_b = array_ops.zeros(shape=(10, 4))
 
     model.fit(
         [input_a, input_b], [target_a, target_b],
@@ -107,16 +114,26 @@ class TrainingTest(keras_parameterized.TestCase):
     model.test_on_batch([input_a, input_b], [target_a, target_b])
 
   @keras_parameterized.run_with_all_model_types
+  @keras_parameterized.run_all_keras_modes
   def test_model_methods_with_eager_tensors_single_io(self):
+    if not context.executing_eagerly():
+      # Only test V2 Function and V2 Eager modes, as V1 Graph mode with
+      # symbolic tensors has different requirements.
+      return
+
     model = testing_utils.get_small_mlp(10, 4, 3)
 
     optimizer = rmsprop.RMSprop(learning_rate=0.001)
     loss = 'mse'
     metrics = ['mae', metrics_module.CategoricalAccuracy()]
-    model.compile(optimizer, loss, metrics=metrics, run_eagerly=True)
+    model.compile(
+        optimizer,
+        loss,
+        metrics=metrics,
+        run_eagerly=testing_utils.should_run_eagerly())
 
-    inputs = keras.backend.zeros(shape=(10, 3))
-    targets = keras.backend.zeros(shape=(10, 4))
+    inputs = array_ops.zeros(shape=(10, 3))
+    targets = array_ops.zeros(shape=(10, 4))
 
     model.fit(inputs, targets, epochs=1, batch_size=2, verbose=0)
     model.fit(inputs, targets, epochs=1, batch_size=3, verbose=0, shuffle=False)
@@ -134,8 +151,8 @@ class TrainingTest(keras_parameterized.TestCase):
                   loss='mse',
                   run_eagerly=True)
 
-    x = keras.backend.zeros(shape=(10, 3))
-    y = keras.backend.zeros(shape=(10, 4))
+    x = array_ops.zeros(shape=(10, 3))
+    y = array_ops.zeros(shape=(10, 4))
     dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).repeat(10).batch(5)
     iterator = dataset_ops.make_one_shot_iterator(dataset)
     validation_dataset = dataset_ops.Dataset.from_tensor_slices(
@@ -146,7 +163,7 @@ class TrainingTest(keras_parameterized.TestCase):
         ValueError, r'specify .* `steps_per_epoch`'):
       model.fit(iterator, epochs=1, verbose=0)
     if not context.executing_eagerly():
-      # In eager execution, `keras.backend.zeros` returns value tensors
+      # In eager execution, `array_ops.zeros` returns value tensors
       # which can be used for validation without a `validation_steps` argument.
       with self.assertRaisesRegexp(
           ValueError, r'provide either `batch_size` or `validation_steps`'):
diff --git a/tensorflow/python/keras/engine/training_utils.py b/tensorflow/python/keras/engine/training_utils.py
index d07e3cc4f7..5edf1de1b0 100644
--- a/tensorflow/python/keras/engine/training_utils.py
+++ b/tensorflow/python/keras/engine/training_utils.py
@@ -1402,3 +1402,22 @@ def set_run_eagerly_for_dict_structure(model, x):
       if isinstance(item, dict):
         model.run_eagerly = True
         return
+
+
+def convert_eager_tensors_to_numpy(structure):
+  """Convert every EagerTensor in `structure` to NumPy.
+
+  Arguments:
+    structure: An arbitrary structure of elements to be converted to NumPy
+      arrays.
+
+  Returns:
+    An identical structure with EagerTensors converted to NumPy arrays.
+  """
+
+  def _convert(element):
+    if isinstance(element, ops.EagerTensor):
+      return element.numpy()
+    return element
+
+  return nest.map_structure(_convert, structure)
-- 
GitLab


From e1a553ba1b10a6f39f01b364d7107d1c77a97776 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Mon, 7 Jan 2019 12:18:27 -0800
Subject: [PATCH 0313/1095] Add v2 huber loss.

PiperOrigin-RevId: 228212499
---
 tensorflow/python/keras/losses.py      |  80 ++++++++++++++++++++
 tensorflow/python/keras/losses_test.py | 101 +++++++++++++++++++++++++
 2 files changed, 181 insertions(+)

diff --git a/tensorflow/python/keras/losses.py b/tensorflow/python/keras/losses.py
index de1605a09c..5f9278823c 100644
--- a/tensorflow/python/keras/losses.py
+++ b/tensorflow/python/keras/losses.py
@@ -608,6 +608,53 @@ class KullbackLeiblerDivergence(Loss):
     return kullback_leibler_divergence(y_true, y_pred)
 
 
+class HuberLoss(Loss):
+  """Computes the huber loss between `y_true` and `y_pred`.
+
+  For each value x in `error=y_true-y_pred`, the following is calculated:
+
+    ```
+    0.5 * x^2                  if |x| <= d
+    0.5 * d^2 + d * (|x| - d)  if |x| > d
+    ```
+  where d is `delta`. See: https://en.wikipedia.org/wiki/Huber_loss
+
+  Usage:
+
+  ```python
+  l = tf.losses.HuberLoss()
+  loss = l([0., 1., 1.], [1., 0., 1.])
+  print('Loss: ', loss.numpy())  # Loss: 0.333
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile('sgd', loss=tf.losses.HuberLoss())
+  ```
+
+  Args:
+    delta: A float, the point where the huber loss function changes from a
+      quadratic to linear.
+    reduction: Type of `tf.losses.Reduction` to apply to loss. Default value is
+      `SUM_OVER_BATCH_SIZE`.
+    name: Optional name for the op.
+  """
+
+  def __init__(self,
+               delta=1.0,
+               reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
+               name=None):
+    super(HuberLoss, self).__init__(reduction=reduction, name=name)
+    self.delta = delta
+
+  def call(self, y_true, y_pred):
+    y_pred = ops.convert_to_tensor(y_pred)
+    y_true = math_ops.cast(y_true, y_pred.dtype)
+    return huber_loss(y_true, y_pred, delta=self.delta)
+
+
 @keras_export('keras.metrics.mean_squared_error',
               'keras.metrics.mse',
               'keras.metrics.MSE',
@@ -677,6 +724,39 @@ def logloss(y_true, y_pred):
   return K.mean(-losses, axis=-1)
 
 
+def huber_loss(y_true, y_pred, delta=1.0):
+  """Computes huber loss value.
+
+  For each value x in `error=y_true-y_pred`, the following is calculated:
+
+    ```
+    0.5 * x^2                  if |x| <= d
+    0.5 * d^2 + d * (|x| - d)  if |x| > d
+    ```
+  where d is `delta`. See: https://en.wikipedia.org/wiki/Huber_loss
+
+  Args:
+    y_true: tensor of true targets.
+    y_pred: tensor of predicted targets.
+    delta: A float, the point where the huber loss function changes from a
+      quadratic to linear.
+
+  Returns:
+    Tensor with one scalar loss entry per sample.
+  """
+  y_pred = math_ops.cast(y_pred, dtype=K.floatx())
+  y_true = math_ops.cast(y_true, dtype=K.floatx())
+  error = math_ops.subtract(y_pred, y_true)
+  abs_error = math_ops.abs(error)
+  quadratic = math_ops.minimum(abs_error, delta)
+  linear = math_ops.subtract(abs_error, quadratic)
+  return math_ops.add(
+      math_ops.multiply(
+          ops.convert_to_tensor(0.5, dtype=quadratic.dtype),
+          math_ops.multiply(quadratic, quadratic)),
+      math_ops.multiply(delta, linear))
+
+
 @keras_export('keras.losses.logcosh')
 def logcosh(y_true, y_pred):
   """Logarithm of the hyperbolic cosine of the prediction error.
diff --git a/tensorflow/python/keras/losses_test.py b/tensorflow/python/keras/losses_test.py
index caa4ff4c2e..5d6cd1be55 100644
--- a/tensorflow/python/keras/losses_test.py
+++ b/tensorflow/python/keras/losses_test.py
@@ -1336,5 +1336,106 @@ class KullbackLeiblerDivergenceTest(test.TestCase):
     self.assertAlmostEqual(self.evaluate(loss), 0., 3)
 
 
+@test_util.run_all_in_graph_and_eager_modes
+class HuberLossTest(test.TestCase):
+
+  def huber_loss(self, y_true, y_pred, delta=1.0):
+    error = y_pred - y_true
+    abs_error = np.abs(error)
+
+    quadratic = np.minimum(abs_error, delta)
+    linear = np.subtract(abs_error, quadratic)
+    return np.add(
+        np.multiply(0.5, np.multiply(quadratic, quadratic)),
+        np.multiply(delta, linear))
+
+  def setup(self, delta=1.0):
+    self.np_y_pred = np.asarray([.9, .2, .2, .8, .4, .6]).reshape((2, 3))
+    self.np_y_true = np.asarray([1., 0., 1., 1., 0., 0.]).reshape((2, 3))
+
+    self.batch_size = 6
+    self.expected_losses = self.huber_loss(self.np_y_true, self.np_y_pred,
+                                           delta)
+
+    self.y_pred = constant_op.constant(self.np_y_pred)
+    self.y_true = constant_op.constant(self.np_y_true)
+
+  def test_config(self):
+    h_obj = keras.losses.HuberLoss(
+        reduction=losses_impl.ReductionV2.SUM, name='huber')
+    self.assertEqual(h_obj.name, 'huber')
+    self.assertEqual(h_obj.reduction, losses_impl.ReductionV2.SUM)
+
+  def test_all_correct(self):
+    self.setup()
+    h_obj = keras.losses.HuberLoss()
+    loss = h_obj(self.y_true, self.y_true)
+    self.assertAlmostEqual(self.evaluate(loss), 0.0, 3)
+
+  def test_unweighted(self):
+    self.setup()
+    h_obj = keras.losses.HuberLoss()
+    loss = h_obj(self.y_true, self.y_pred)
+    actual_loss = np.sum(self.expected_losses) / self.batch_size
+    self.assertAlmostEqual(self.evaluate(loss), actual_loss, 3)
+
+  def test_scalar_weighted(self):
+    self.setup()
+    h_obj = keras.losses.HuberLoss()
+    sample_weight = 2.3
+    loss = h_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+    actual_loss = sample_weight * np.sum(self.expected_losses) / self.batch_size
+    self.assertAlmostEqual(self.evaluate(loss), actual_loss, 3)
+
+    # Verify we get the same output when the same input is given
+    loss_2 = h_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+    self.assertAlmostEqual(self.evaluate(loss), self.evaluate(loss_2), 3)
+
+  def test_sample_weighted(self):
+    self.setup()
+    h_obj = keras.losses.HuberLoss()
+    sample_weight = constant_op.constant((1.2, 3.4), shape=(2, 1))
+
+    loss = h_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+    actual_loss = np.multiply(
+        self.expected_losses,
+        np.asarray([1.2, 1.2, 1.2, 3.4, 3.4, 3.4]).reshape((2, 3)))
+    actual_loss = np.sum(actual_loss) / self.batch_size
+    self.assertAlmostEqual(self.evaluate(loss), actual_loss, 3)
+
+  def test_timestep_weighted(self):
+    self.setup()
+    h_obj = keras.losses.HuberLoss()
+    y_pred = self.np_y_pred.reshape((2, 3, 1))
+    y_true = self.np_y_true.reshape((2, 3, 1))
+    expected_losses = self.huber_loss(y_true, y_pred)
+
+    y_pred = constant_op.constant(y_pred)
+    y_true = constant_op.constant(y_true)
+    sample_weight = np.array([3, 6, 5, 0, 4, 2]).reshape((2, 3, 1))
+    loss = h_obj(
+        y_true,
+        y_pred,
+        sample_weight=constant_op.constant(sample_weight, shape=(2, 3)))
+    actual_loss = np.multiply(expected_losses, sample_weight)
+    actual_loss = np.sum(actual_loss) / self.batch_size
+    self.assertAlmostEqual(self.evaluate(loss), actual_loss, 3)
+
+  def test_zero_weighted(self):
+    self.setup()
+    h_obj = keras.losses.HuberLoss()
+    sample_weight = 0
+    loss = h_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+    self.assertAlmostEqual(self.evaluate(loss), 0., 3)
+
+  def test_non_default_delta(self):
+    self.setup(delta=0.8)
+    h_obj = keras.losses.HuberLoss(delta=0.8)
+    sample_weight = 2.3
+    loss = h_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+    actual_loss = sample_weight * np.sum(self.expected_losses) / self.batch_size
+    self.assertAlmostEqual(self.evaluate(loss), actual_loss, 3)
+
+
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From 7ba04bba51ac7500afd5fe0d8e2f883082cdd5fb Mon Sep 17 00:00:00 2001
From: Bixia Zheng <bixia@google.com>
Date: Mon, 7 Jan 2019 12:22:34 -0800
Subject: [PATCH 0314/1095] [XLA:GPU] Enhance column reduction implementation.

With KernelMappingScheme, a thread processes n elements, where
n=tile_size_x/num_thread_x. Previously, we only support n>1 for transpose and
row reduction. This change extends the kernel mapping scheme and code
generation to allow each thread to compute n partial results for column
reduction as follows:
.Add dilated_x to KernelMappingScheme, to indicate whether the multiple
elements processed by the same thread are contiguous or dilated. Dilated_x=true
is what we currently use for transpose while dilated_x=false is used to
support the vectorization of column reduction.
.Extend the stack storages that store the partial result address and current
output linear index address for each output tensor from a scalar to an array of
n elements.
.Add curr_iter_index_x to kernelCodegenInfo, to indicate which output element
that the compiler is currently generating code for. This information is used to
locate the partial result address and output linear index address for the
element.
.Modify the code generation to use n=2 for column reduction.
PiperOrigin-RevId: 228213135
---
 .../xla/service/gpu/ir_emitter_unnested.cc    | 330 +++++++++++-------
 .../xla/service/gpu/ir_emitter_unnested.h     |  20 +-
 .../compiler/xla/service/llvm_ir/ir_array.h   |   2 +
 .../xla/service/llvm_ir/kernel_tiling.cc      |   3 +-
 .../xla/service/llvm_ir/kernel_tiling.h       |  22 +-
 5 files changed, 248 insertions(+), 129 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
index f190af921e..2bc4912155 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
@@ -88,6 +88,9 @@ namespace xla {
 namespace gpu {
 
 using llvm_ir::KernelMappingScheme;
+using EmitElementFunction =
+    std::function<void(const llvm_ir::IrArray::Index& index, llvm::Value* y_loc,
+                       llvm::Value* x_loc, int64 x_iter_num)>;
 
 namespace {
 
@@ -2133,53 +2136,86 @@ int IrEmitterUnnested::ConstructInputReducedShapeAndCastInputIrArrayToShape(
 
 namespace {
 
-void EmitFullElementalTile(
-    const KernelMappingScheme* mapping_scheme,
-    const IrArray::Index& tile_origin_index, const string& loop_name,
-    KernelSupportLibrary* ksl, llvm::IRBuilder<>* builder, llvm::Value* y,
-    llvm::Value* x, llvm::Type* index_ty,
-    const std::function<void(const IrArray::Index&, llvm::Value*,
-                             llvm::Value*)>& emit_elem_function) {
+std::tuple<llvm::Value*, int64> GetStartOffsetAndStepForX(
+    int64 tile_size_x, int64 num_threads_x,
+    const KernelMappingScheme* mapping_scheme, llvm::IRBuilder<>* builder,
+    llvm::Value* x, llvm::Type* index_ty) {
+  llvm::Value* start_offset_x;
+  int64 step_x;
+  if (mapping_scheme->DilatedX()) {
+    start_offset_x = x;
+    step_x = num_threads_x;
+  } else {
+    start_offset_x = builder->CreateMul(
+        x, llvm::ConstantInt::get(index_ty, tile_size_x / num_threads_x));
+    step_x = 1;
+  }
+  return std::make_tuple(start_offset_x, step_x);
+}
+
+void EmitFullElementalTile(const KernelMappingScheme* mapping_scheme,
+                           const IrArray::Index& tile_origin_index,
+                           const string& loop_name, KernelSupportLibrary* ksl,
+                           llvm::IRBuilder<>* builder, llvm::Value* y,
+                           llvm::Value* x, llvm::Type* index_ty,
+                           const EmitElementFunction& emit_elem_function) {
   int64 num_threads_x = mapping_scheme->GetNumberOfThreadsForDimensionX();
   int64 num_threads_y = mapping_scheme->GetNumberOfThreadsForDimensionY();
   int64 tile_size_x = mapping_scheme->GetTileSizeForDimensionX();
   int64 tile_size_y = mapping_scheme->GetTileSizeForDimensionY();
+
+  llvm::Value* start_offset_x;
+  int64 step_x;
+  std::tie(start_offset_x, step_x) = GetStartOffsetAndStepForX(
+      tile_size_x, num_threads_x, mapping_scheme, builder, x, index_ty);
+  IrArray::Index source_idx =
+      tile_origin_index.AddOffsetToDim(y, KernelMappingScheme::DimY, builder)
+          .AddOffsetToDim(start_offset_x, KernelMappingScheme::DimX, builder);
   ksl->For(loop_name + "_y", /*start=*/llvm::ConstantInt::get(index_ty, 0),
            /*end=*/llvm::ConstantInt::get(index_ty, tile_size_y),
            /*step=*/llvm::ConstantInt::get(index_ty, num_threads_y),
            [&](llvm::Value* y_indvar) {
-             IrArray::Index source_idx_y = tile_origin_index.AddOffsetToDim(
+             IrArray::Index source_idx_y = source_idx.AddOffsetToDim(
                  y_indvar, KernelMappingScheme::DimY, builder);
              llvm::Value* y_loc = builder->CreateAdd(y_indvar, y);
-             for (int64 j = 0; j < tile_size_x; j += num_threads_x) {
-               IrArray::Index source_idx = source_idx_y.AddOffsetToDim(
-                   llvm::ConstantInt::get(index_ty, j),
+
+             for (int64 j = 0; j < tile_size_x / num_threads_x; j++) {
+               IrArray::Index source_idx_y_x = source_idx_y.AddOffsetToDim(
+                   llvm::ConstantInt::get(index_ty, j * step_x),
                    KernelMappingScheme::DimX, builder);
-               llvm::Value* x_loc =
-                   builder->CreateAdd(llvm::ConstantInt::get(index_ty, j), x);
-               emit_elem_function(source_idx, y_loc, x_loc);
+               llvm::Value* x_loc = builder->CreateAdd(
+                   llvm::ConstantInt::get(index_ty, j * step_x),
+                   start_offset_x);
+               emit_elem_function(source_idx_y_x, y_loc, x_loc, j);
              }
            });
 }
 
-void EmitPartialElementalTile(
-    const KernelMappingScheme* mapping_scheme,
-    const IrArray::Index& tile_origin_index, const string& loop_name,
-    KernelSupportLibrary* ksl, llvm::IRBuilder<>* builder, llvm::Value* y,
-    llvm::Value* x, llvm::Value* tile_height, llvm::Value* tile_width,
-    llvm::Type* index_ty,
-    const std::function<void(const IrArray::Index&, llvm::Value*,
-                             llvm::Value*)>& emit_elem_function) {
+void EmitPartialElementalTile(const KernelMappingScheme* mapping_scheme,
+                              const IrArray::Index& tile_origin_index,
+                              const string& loop_name,
+                              KernelSupportLibrary* ksl,
+                              llvm::IRBuilder<>* builder, llvm::Value* y,
+                              llvm::Value* x, llvm::Value* tile_height,
+                              llvm::Value* tile_width, llvm::Type* index_ty,
+                              const EmitElementFunction& emit_elem_function) {
   int64 num_threads_x = mapping_scheme->GetNumberOfThreadsForDimensionX();
   int64 num_threads_y = mapping_scheme->GetNumberOfThreadsForDimensionY();
   int64 tile_size_x = mapping_scheme->GetTileSizeForDimensionX();
 
-  for (int64 j = 0; j < tile_size_x; j += num_threads_x) {
-    IrArray::Index source_idx =
-        tile_origin_index.AddOffsetToDim(llvm::ConstantInt::get(index_ty, j),
-                                         KernelMappingScheme::DimX, builder);
-    llvm::Value* x_loc =
-        builder->CreateAdd(llvm::ConstantInt::get(index_ty, j), x);
+  llvm::Value* start_offset_x;
+  int64 step_x;
+  std::tie(start_offset_x, step_x) = GetStartOffsetAndStepForX(
+      tile_size_x, num_threads_x, mapping_scheme, builder, x, index_ty);
+  IrArray::Index source_idx =
+      tile_origin_index.AddOffsetToDim(y, KernelMappingScheme::DimY, builder)
+          .AddOffsetToDim(start_offset_x, KernelMappingScheme::DimX, builder);
+  for (int64 j = 0; j < tile_size_x / num_threads_x; j++) {
+    IrArray::Index source_idx_x =
+        source_idx.AddOffsetToDim(llvm::ConstantInt::get(index_ty, j * step_x),
+                                  KernelMappingScheme::DimX, builder);
+    llvm::Value* x_loc = builder->CreateAdd(
+        llvm::ConstantInt::get(index_ty, j * step_x), start_offset_x);
 
     ksl->If(
         loop_name + "_x_in_tile", builder->CreateICmpULT(x_loc, tile_width),
@@ -2199,14 +2235,13 @@ void EmitPartialElementalTile(
               /*step=*/llvm::ConstantInt::get(index_ty, num_threads_y),
               [&](llvm::Value* y_indvar) {
                 llvm::Value* y_loc = builder->CreateAdd(y_indvar, y);
-                ksl->If(
-                    loop_name + "_y_in_tile",
-                    builder->CreateICmpULT(y_loc, tile_height), [&] {
-                      emit_elem_function(
-                          source_idx.AddOffsetToDim(
-                              y_indvar, KernelMappingScheme::DimY, builder),
-                          y_loc, x_loc);
-                    });
+                ksl->If(loop_name + "_y_in_tile",
+                        builder->CreateICmpULT(y_loc, tile_height), [&] {
+                          emit_elem_function(
+                              source_idx_x.AddOffsetToDim(
+                                  y_indvar, KernelMappingScheme::DimY, builder),
+                              y_loc, x_loc, j);
+                        });
               });
         });
   }
@@ -2225,8 +2260,7 @@ void EmitTiledElementalCodeWithBoundsCheck(
     const IrArray::Index& tile_origin_index, const string& loop_name,
     KernelSupportLibrary* ksl, llvm::IRBuilder<>* builder, llvm::Value* y,
     llvm::Value* x, llvm::Value* tile_height, llvm::Value* tile_width,
-    const std::function<void(const IrArray::Index&, llvm::Value*,
-                             llvm::Value*)>& emit_elem_function) {
+    const EmitElementFunction& emit_elem_function) {
   int64 tile_size_x = mapping_scheme->GetTileSizeForDimensionX();
   int64 tile_size_y = mapping_scheme->GetTileSizeForDimensionY();
   llvm::Type* index_ty = tile_width->getType();
@@ -2262,7 +2296,7 @@ void EmitTiledElementalCodeWithBoundsCheck(
 void IrEmitterUnnested::EmitTileElementForCopy(
     HloInstruction* hlo, const llvm_ir::IrArray::Index& index,
     const KernelCodegenInfo* kernel_info, llvm::Value* y_loc,
-    llvm::Value* x_loc) {
+    llvm::Value* x_loc, int64 /*x_iter_num*/) {
   llvm_ir::TiledParameterInfo* tiled_param_info =
       kernel_info->GetTiledParameterInfo();
   // TODO(jlebar): Add AA metadata to this load.
@@ -2292,7 +2326,7 @@ void IrEmitterUnnested::EmitTileElementForCopy(
 void IrEmitterUnnested::EmitTileElementForFusion(
     HloInstruction* hlo, const llvm_ir::IrArray::Index& index,
     const KernelCodegenInfo* kernel_info, llvm::Value* y_loc,
-    llvm::Value* x_loc) {
+    llvm::Value* x_loc, int64 /*x_iter_num*/) {
   llvm_ir::TiledParameterInfo* tiled_param_info =
       kernel_info->GetTiledParameterInfo();
   std::vector<IrArray> output_arrays = ConstructIrArrayForOutputs(*hlo);
@@ -2393,6 +2427,23 @@ class ReductionCodegenInfo : public IrEmitterUnnested::KernelCodegenInfo {
                             : llvm_ir::KernelMappingScheme::DimX;
   }
 
+  int GetNumberOfPartialResults() const {
+    if (IsRowReduction()) {
+      return 1;
+    }
+    int64 num_thread = mapping_scheme_->GetNumberOfThreadsForDimensionX();
+    int64 tile_size = mapping_scheme_->GetTileSizeForDimensionX();
+    CHECK_EQ(tile_size % num_thread, 0);
+    return tile_size / num_thread;
+  }
+
+  int GetPartialResultIndex(int64 x_iter_num) const {
+    if (IsRowReduction()) {
+      return 0;
+    }
+    return x_iter_num;
+  }
+
  private:
   AddressVector partial_result_addresses_;
   AddressVector reduction_input_addresses_;
@@ -2452,10 +2503,11 @@ void IrEmitterUnnested::EmitPrologueForOneReduction(
   llvm::AllocaInst* reduction_input_address = Alloca(element_type);
   reduction_input_addresses->push_back(reduction_input_address);
 
+  int num_partial_results = reduction_info->GetNumberOfPartialResults();
   AddressVector* partial_result_addresses =
       reduction_info->GetMutablePartialResultAddresses();
   llvm::AllocaInst* partial_result_address =
-      Alloca(element_type, /*ArraySize=*/nullptr,
+      Alloca(element_type, /*ArraySize=*/b_.getInt32(num_partial_results),
              "partial_reduction_result." + llvm::Twine(reduce_idx));
   partial_result_addresses->push_back(partial_result_address);
 
@@ -2478,7 +2530,9 @@ void IrEmitterUnnested::EmitPrologueForOneReduction(
             .EmitReadArrayElement(IrArray::Index(b_.getInt32Ty()), &b_);
   }
 
-  Store(init_ir_value, partial_result_address);
+  for (int i = 0; i < num_partial_results; ++i) {
+    Store(init_ir_value, InBoundsGEP(partial_result_address, {b_.getInt32(i)}));
+  }
 }
 
 void IrEmitterUnnested::EmitPrologueForReduction(
@@ -2516,10 +2570,14 @@ void IrEmitterUnnested::EmitPrologueForReduction(
                                 std::move(output_shape_index));
   }
 
-  // Allocate stack storage to store the current output linear index and record
-  // the address of the storage.
+  int num_partial_results = reduction_info->GetNumberOfPartialResults();
+
+  // Allocate stack storage to store the linear indices for the current output,
+  // and record the address of the storage.
   reduction_info->SetCurrentOutputLinearIndexAddress(
-      Alloca(reduction_info->GetIndexType()));
+      Alloca(reduction_info->GetIndexType(),
+             /*ArraySize=*/b_.getInt32(num_partial_results),
+             "current_output_linear_index_address"));
 
   if (!reduction_info->IsRowReduction()) {
     llvm::Type* bool_ty = b_.getInt1Ty();
@@ -2589,36 +2647,45 @@ void IrEmitterUnnested::EmitEpilogueForReduction(
     llvm_ir::SetToFirstInsertPoint(if_output_inbound_data.true_block, &b_);
   }
 
+  int num_partial_results = reduction_info->GetNumberOfPartialResults();
+
   // Emit an atomic operation that accumulates the partial reduction to the
   // output element. For row reduction, this is only for lane 0 due to the
   // if-statement emitted above.
   for (int i = 0; i != num_reduces; ++i) {
-    IrArray::Index element_index(
-        /*linear=*/Load(reduction_info->GetCurrentOutputLinearIndexAddress(),
-                        "output_linear_addr"),
-        ShapeUtil::GetSubshape(unnested_hlo->shape(),
-                               reduction_output_shape_indices[i]),
-        &b_);
-    llvm::Value* output_address =
-        GetIrArray(*unnested_hlo, *unnested_hlo,
-                   reduction_output_shape_indices[i])
-            .EmitArrayElementAddress(element_index, &b_,
-                                     "output_element_address");
-    // Do not emit atomic operations if each element in the reduction result is
-    // computed by one block, that is the dimension being reduced has only one
-    // block.
-    const llvm_ir::KernelMappingScheme* mapping_scheme =
-        reduction_info->GetKernelMappingScheme();
-    if (mapping_scheme->GetTileBlockSizeForDimension(
-            llvm_ir::KernelMappingScheme::DimZ) == 1 &&
-        mapping_scheme->GetTileBlockSizeForDimension(
-            reduction_info->GetReducedDimensionEnum()) == 1) {
-      TF_CHECK_OK(EmitCallToNestedComputation(
-          *reducers[i], {output_address, partial_result_addresses[i]},
-          output_address));
-    } else {
-      TF_CHECK_OK(EmitAtomicOperationForNestedComputation(
-          *reducers[i], output_address, partial_result_addresses[i]));
+    for (int j = 0; j < num_partial_results; ++j) {
+      IrArray::Index element_index(
+          /*linear=*/Load(
+              InBoundsGEP(reduction_info->GetCurrentOutputLinearIndexAddress(),
+                          {b_.getInt32(j)}),
+              "output_linear_addr"),
+          ShapeUtil::GetSubshape(unnested_hlo->shape(),
+                                 reduction_output_shape_indices[i]),
+          &b_);
+      llvm::Value* output_address =
+          GetIrArray(*unnested_hlo, *unnested_hlo,
+                     reduction_output_shape_indices[i])
+              .EmitArrayElementAddress(element_index, &b_,
+                                       "output_element_address");
+      // Do not emit atomic operations if each element in the reduction result
+      // is computed by one block, that is the dimension being reduced has only
+      // one block.
+      const llvm_ir::KernelMappingScheme* mapping_scheme =
+          reduction_info->GetKernelMappingScheme();
+      if (mapping_scheme->GetTileBlockSizeForDimension(
+              llvm_ir::KernelMappingScheme::DimZ) == 1 &&
+          mapping_scheme->GetTileBlockSizeForDimension(
+              reduction_info->GetReducedDimensionEnum()) == 1) {
+        TF_CHECK_OK(EmitCallToNestedComputation(
+            *reducers[i],
+            {output_address,
+             InBoundsGEP(partial_result_addresses[i], {b_.getInt32(j)})},
+            output_address));
+      } else {
+        TF_CHECK_OK(EmitAtomicOperationForNestedComputation(
+            *reducers[i], output_address,
+            InBoundsGEP(partial_result_addresses[i], {b_.getInt32(j)})));
+      }
     }
   }
 }
@@ -2626,7 +2693,7 @@ void IrEmitterUnnested::EmitEpilogueForReduction(
 void IrEmitterUnnested::EmitTileElementForReduction(
     HloInstruction* unnested_hlo, const llvm_ir::IrArray::Index& index,
     const KernelCodegenInfo* kernel_info, llvm::Value* y_loc,
-    llvm::Value* x_loc) {
+    llvm::Value* x_loc, int64 x_iter_num) {
   VLOG(10) << "Emit tile element for reduce " << unnested_hlo->ToString();
   HloInstruction* reduce_or_tuple = unnested_hlo->opcode() == HloOpcode::kFusion
                                         ? unnested_hlo->fused_expression_root()
@@ -2639,8 +2706,11 @@ void IrEmitterUnnested::EmitTileElementForReduction(
   // Record the linear address for the current reduction.
   const ReductionCodegenInfo* reduction_info =
       dynamic_cast<const ReductionCodegenInfo*>(kernel_info);
+  int partial_result_index = reduction_info->IsRowReduction() ? 0 : x_iter_num;
+
   Store(index[reduction_info->GetKeptDimensionEnum()],
-        reduction_info->GetCurrentOutputLinearIndexAddress());
+        InBoundsGEP(reduction_info->GetCurrentOutputLinearIndexAddress(),
+                    {b_.getInt32(partial_result_index)}));
   if (!reduction_info->IsRowReduction()) {
     llvm::Type* bool_ty = b_.getInt1Ty();
     llvm::AllocaInst* output_inbound_addr =
@@ -2687,6 +2757,13 @@ void IrEmitterUnnested::EmitTileElementForReduction(
       reduction_info->GetKernelMappingScheme()->GetUnnormalizedIndex(
           index,
           GetFirstReduceInstruction(output_instructions)->operand(0)->shape());
+  int num_partial_results = reduction_info->GetNumberOfPartialResults();
+  if (num_partial_results > 1) {
+    // Clear the linear index field of the IrArray::Index to enable the use of
+    // GetElementPointer with array types. This enables the vectorization of
+    // the computation for different partial results.
+    input_index.ClearLinearIndex();
+  }
   absl::Span<llvm::AllocaInst* const> partial_reduction_result_addresses =
       reduction_info->GetPartialResultAddresses();
   absl::Span<llvm::AllocaInst* const> reduction_input_addresses =
@@ -2699,10 +2776,12 @@ void IrEmitterUnnested::EmitTileElementForReduction(
   for (int i = 0; i != reducers.size(); ++i) {
     llvm::Value* const input_ir_value = input_gens[i](input_index).ValueOrDie();
     Store(input_ir_value, reduction_input_addresses[i]);
+    llvm::Value* partial_result_address =
+        InBoundsGEP(partial_reduction_result_addresses[i],
+                    {b_.getInt32(partial_result_index)});
     TF_CHECK_OK(EmitCallToNestedComputation(
-        *reducers[i],
-        {partial_reduction_result_addresses[i], reduction_input_addresses[i]},
-        partial_reduction_result_addresses[i]));
+        *reducers[i], {partial_result_address, reduction_input_addresses[i]},
+        partial_result_address));
   }
 
   // Emit code to generate the output for the non-reduction instructions in the
@@ -2713,8 +2792,8 @@ void IrEmitterUnnested::EmitTileElementForReduction(
 
 // Emits a kernel for the hlo instruction using the given tiling scheme.
 void IrEmitterUnnested::EmitBlock(const TileGenerator& emit_one_tile,
-                                  const KernelCodegenInfo* kernel_info,
-                                  KernelSupportLibrary& ksl,
+                                  KernelCodegenInfo* kernel_info,
+                                  KernelSupportLibrary* ksl,
                                   llvm::Type* index_ty) {
   KernelMappingScheme* mapping_scheme = kernel_info->GetKernelMappingScheme();
   absl::Span<const int64> dims_in_tile = mapping_scheme->GetDimensionsInTiles();
@@ -2747,15 +2826,14 @@ void IrEmitterUnnested::EmitBlock(const TileGenerator& emit_one_tile,
           llvm::Value* num_tiles_in_block =
               Select(ICmpEQ(last_block_for_dim, block_id_for_dim),
                      last_block_size_for_dim, block_size_for_dim);
-
-          ksl.For(loop_name,
-                  /*start=*/index_typed_constant(0),
-                  /*end=*/num_tiles_in_block,
-                  /*step=*/1, [&](llvm::Value* block_dim_induction_var) {
-                    IrArray::Index tile_index = starting_tile.AddOffsetToDim(
-                        block_dim_induction_var, dim_id, &b_);
-                    emit_next_block_dim(tile_index);
-                  });
+          ksl->For(loop_name,
+                   /*start=*/index_typed_constant(0),
+                   /*end=*/num_tiles_in_block,
+                   /*step=*/1, [&](llvm::Value* block_dim_induction_var) {
+                     IrArray::Index tile_index = starting_tile.AddOffsetToDim(
+                         block_dim_induction_var, dim_id, &b_);
+                     emit_next_block_dim(tile_index);
+                   });
         }
       };
 
@@ -2899,8 +2977,7 @@ LaunchDimensions IrEmitterUnnested::EmitKernel(
   auto emit_tiled_elemental_code_with_bounds_check =
       [&](const IrArray::Index& index, const string& loop_name,
           llvm::Value* tile_height, llvm::Value* tile_width,
-          const std::function<void(const IrArray::Index&, llvm::Value*,
-                                   llvm::Value*)>& emit_elem_function) {
+          const EmitElementFunction& emit_elem_function) {
         EmitTiledElementalCodeWithBoundsCheck(mapping_scheme, index, loop_name,
                                               &ksl, &b_, y, x, tile_height,
                                               tile_width, emit_elem_function);
@@ -2913,10 +2990,6 @@ LaunchDimensions IrEmitterUnnested::EmitKernel(
     const IrArray::Index input_tile_origin(
         Permute({0, 2, 1}, output_tile_origin.multidim()));
 
-    const IrArray::Index input_index =
-        input_tile_origin.AddOffsetToDim(x, KernelMappingScheme::DimX, &b_)
-            .AddOffsetToDim(y, KernelMappingScheme::DimY, &b_);
-
     // If shared memory transpose is needed, wait for all threads to reach this
     // point, lest we copy a value from tile to output before the other thread
     // copies it from input to tile. This is `__syncthreads` in CUDA.
@@ -2926,9 +2999,10 @@ LaunchDimensions IrEmitterUnnested::EmitKernel(
       // Note that tile_width and tile_height are flipped here because we are
       // reading a transposed tile.
       emit_tiled_elemental_code_with_bounds_check(
-          input_index, "input", output_tile_bounds[2], output_tile_bounds[1],
+          input_tile_origin, "input", output_tile_bounds[2],
+          output_tile_bounds[1],
           [&](const IrArray::Index& index, llvm::Value* y_loc,
-              llvm::Value* x_loc) {
+              llvm::Value* x_loc, int64 /*x_iter_num*/) {
             for (int64 id : tiled_param_ids) {
               IrArray& input_in_logical_shape =
                   param_in_reduced_shape_arrays[id];
@@ -2948,18 +3022,15 @@ LaunchDimensions IrEmitterUnnested::EmitKernel(
     llvm_ir::TiledParameterInfo tiled_param_info(param_shmem_buffers, y, x);
     kernel_info->SetTiledParamInfo(&tiled_param_info);
 
-    const IrArray::Index output_index =
-        output_tile_origin.AddOffsetToDim(x, KernelMappingScheme::DimX, &b_)
-            .AddOffsetToDim(y, KernelMappingScheme::DimY, &b_);
-
     // Write to output[index] by emitting code like normal, except that values
     // for the tiled parameters are read from the shmem buffers.
     emit_tiled_elemental_code_with_bounds_check(
-        output_index, "output", output_tile_bounds[1], output_tile_bounds[2],
-        [&](const IrArray::Index& index, llvm::Value* y_loc,
-            llvm::Value* x_loc) {
-          kernel_generator.GetTileElementGenerator()(unnested_hlo, index,
-                                                     kernel_info, y_loc, x_loc);
+        output_tile_origin, "output", output_tile_bounds[1],
+        output_tile_bounds[2],
+        [&](const IrArray::Index& index, llvm::Value* y_loc, llvm::Value* x_loc,
+            int64 x_iter_num) {
+          kernel_generator.GetTileElementGenerator()(
+              unnested_hlo, index, kernel_info, y_loc, x_loc, x_iter_num);
         });
 
     // If a tile block contains multiple tiles and shared memory buffers are
@@ -2977,7 +3048,7 @@ LaunchDimensions IrEmitterUnnested::EmitKernel(
     block_prologue_generator(unnested_hlo, kernel_info);
   }
 
-  EmitBlock(std::move(emit_one_tile), kernel_info, ksl, index_ty);
+  EmitBlock(std::move(emit_one_tile), kernel_info, &ksl, index_ty);
 
   const BlockEpilogueGenerator& block_epilogue_generator =
       kernel_generator.GetBlockEpilogueGenerator();
@@ -3026,17 +3097,19 @@ LaunchDimensions IrEmitterUnnested::EmitHlo021Tile(
     element_generator = [&](HloInstruction* hlo,
                             const llvm_ir::IrArray::Index& index,
                             const KernelCodegenInfo* kernel_info,
-                            llvm::Value* y_loc, llvm::Value* x_loc) {
-      EmitTileElementForCopy(hlo, index, kernel_info, y_loc, x_loc);
+                            llvm::Value* y_loc, llvm::Value* x_loc,
+                            int64 x_iter_num) {
+      EmitTileElementForCopy(hlo, index, kernel_info, y_loc, x_loc, x_iter_num);
     };
   } else {
     DCHECK_EQ(hlo->opcode(), HloOpcode::kFusion);
-    element_generator = [&](HloInstruction* hlo,
-                            const llvm_ir::IrArray::Index& index,
-                            const KernelCodegenInfo* kernel_info,
-                            llvm::Value* y_loc, llvm::Value* x_loc) {
-      EmitTileElementForFusion(hlo, index, kernel_info, y_loc, x_loc);
-    };
+    element_generator =
+        [&](HloInstruction* hlo, const llvm_ir::IrArray::Index& index,
+            const KernelCodegenInfo* kernel_info, llvm::Value* y_loc,
+            llvm::Value* x_loc, int64 x_iter_num) {
+          EmitTileElementForFusion(hlo, index, kernel_info, y_loc, x_loc,
+                                   x_iter_num);
+        };
   }
   KernelCodegenInfo kernel_info(&mapping_scheme);
   KernelCodeGenerator kernel_generator(std::move(element_generator));
@@ -3351,6 +3424,7 @@ IrEmitterUnnested::ComputeMappingSchemeAndReductionKind(
   std::tie(num_reduced_major, num_kept, num_reduced_minor) =
       GetReductionToVectorDimensions(input_shape, first_reduce->dimensions());
   CHECK_EQ(num_output_elems, num_kept);
+  bool dilated_x = true;
 
   if (num_kept == 1) {
     // Scalar reduction is a special row reduction with depth = height = 1.
@@ -3363,15 +3437,31 @@ IrEmitterUnnested::ComputeMappingSchemeAndReductionKind(
     height = num_reduced_major;
     width = num_kept;
     is_row_reduction = false;
+    // Assume unrolling is beneficial only when we can vectorize the loads
+    // of small data types.
+    auto is_unrolling_beneficial = [&] {
+      // TODO(b/122468062): Need further investigate to see whether we can
+      // remove the constraint on IsPowerOfTwo.
+      return IsPowerOfTwo(static_cast<uint64>(num_kept)) &&
+             primitive_util::BitWidth(
+                 first_reduce->operand(0)->shape().element_type()) <= 16;
+    };
     // Column reduction without transpose doesn't require communication among
     // threads processing elements in the same tile. The current implementation
-    // only support the use of on hardware thread block to process one block of
-    // tiles in the KernelMappingScheme. We try to maximize the values of
+    // only support the use of one hardware thread block to process one block of
+    // tiles in the KernelMappingScheme. We try to use one thread to compute
+    // the partial results for two tensor elements and to maximize the values of
     // num_threads_x and tile_size_x to allow a bigger hardware thread block.
     int64 hw_threads_per_block_limit =
         ThreadsPerBlockLimit(ir_emitter_context_->device_description());
-    tile_size_x = std::min(hw_threads_per_block_limit, num_kept);
-    num_threads_x = tile_size_x;
+    if (is_unrolling_beneficial()) {
+      tile_size_x = std::min(2 * hw_threads_per_block_limit, num_kept);
+      num_threads_x = tile_size_x / 2;
+      dilated_x = false;
+    } else {
+      tile_size_x = std::min(hw_threads_per_block_limit, num_kept);
+      num_threads_x = tile_size_x;
+    }
     int64 kNumElementsPerPartialSum = 128;
     tile_size_y = kNumElementsPerPartialSum;
   } else {
@@ -3400,6 +3490,7 @@ IrEmitterUnnested::ComputeMappingSchemeAndReductionKind(
   llvm_ir::KernelMappingScheme mapping_scheme(
       dims_in_elem, tile_size_y, tile_size_x, req_block_sizes, num_threads_y,
       num_threads_x, &b_);
+  mapping_scheme.SetDilatedX(dilated_x);
   return std::make_tuple(mapping_scheme, is_row_reduction);
 }
 
@@ -3454,8 +3545,9 @@ Status IrEmitterUnnested::EmitReductionToVector(HloInstruction* unnested_hlo) {
       /*tile_element_generator=*/
       [&](HloInstruction* hlo, const llvm_ir::IrArray::Index& index,
           const KernelCodegenInfo* kernel_info, llvm::Value* y_loc,
-          llvm::Value* x_loc) {
-        EmitTileElementForReduction(hlo, index, kernel_info, y_loc, x_loc);
+          llvm::Value* x_loc, int64 x_iter_num) {
+        EmitTileElementForReduction(hlo, index, kernel_info, y_loc, x_loc,
+                                    x_iter_num);
       },
       /*block_prologue_generator=*/
       [&](HloInstruction* hlo, KernelCodegenInfo* kernel_info) {
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h
index d217ee36cf..eb0ea90e14 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h
@@ -76,7 +76,6 @@ class IrEmitterUnnested : public IrEmitter {
     void SetLaneId(llvm::Value* v) { lane_id_ = v; }
     void SetIndexType(llvm::Type* t) { index_ty_ = t; }
     void SetTiledParamInfo(llvm_ir::TiledParameterInfo* tiled_param_info) {
-      CHECK_EQ(tiled_param_info_, nullptr);
       tiled_param_info_ = tiled_param_info;
     }
 
@@ -89,7 +88,7 @@ class IrEmitterUnnested : public IrEmitter {
     }
     llvm::Type* GetIndexType() const { return index_ty_; }
 
-   private:
+   protected:
     llvm_ir::KernelMappingScheme* mapping_scheme_;
     llvm_ir::TiledParameterInfo* tiled_param_info_;
     llvm::Value* lane_id_;
@@ -109,10 +108,12 @@ class IrEmitterUnnested : public IrEmitter {
   // y_loc: The y coordinate within a tile.
   // x_loc: The x coordinate within a tile.
   // kernel_info: Other information to support the kernel code generation.
+  // x_iter_num: When a thread process N elements in the X dimension, x_iter_num
+  //             has a value of 0..N-1 to identify the element being process.
   using TileElementGenerator = std::function<void(
       HloInstruction* hlo, const llvm_ir::IrArray::Index& index,
       const KernelCodegenInfo* kernel_info, llvm::Value* y_loc,
-      llvm::Value* x_loc)>;
+      llvm::Value* x_loc, int64 x_iter_num)>;
 
   // KernelCodeGenerator records the code generator objects that generate code
   // for tile elements or tile block prologue/epilogue.
@@ -243,26 +244,29 @@ class IrEmitterUnnested : public IrEmitter {
                               const KernelCodeGenerator& kernel_generator,
                               KernelCodegenInfo* kernel_info);
   void EmitBlock(const TileGenerator& emit_one_tile,
-                 const KernelCodegenInfo* kernel_info,
-                 KernelSupportLibrary& ksl, llvm::Type* index_ty);
+                 KernelCodegenInfo* kernel_info, KernelSupportLibrary* ksl,
+                 llvm::Type* index_ty);
   // Emits code to process a tensor element in a tile for the given kCopy HLO
   // that performs a 0-2-1 transpose.
   void EmitTileElementForCopy(HloInstruction* hlo,
                               const llvm_ir::IrArray::Index& index,
                               const KernelCodegenInfo* kernel_info,
-                              llvm::Value* y_loc, llvm::Value* x_loc);
+                              llvm::Value* y_loc, llvm::Value* x_loc,
+                              int64 x_iter_num);
   // Emits code to process a tensor element in a tile for the given kLoop fusion
   // HLO containing parameters that are 0-2-1 transpose of its outputs.
   void EmitTileElementForFusion(HloInstruction* hlo,
                                 const llvm_ir::IrArray::Index& index,
                                 const KernelCodegenInfo* kernel_info,
-                                llvm::Value* y_loc, llvm::Value* x_loc);
+                                llvm::Value* y_loc, llvm::Value* x_loc,
+                                int64 x_iter_num);
   // Emits code to process a tensor element in a tile for the given input hlo
   // that is either a unnested kReduce or a kInput fusion.
   void EmitTileElementForReduction(HloInstruction* unnested_hlo,
                                    const llvm_ir::IrArray::Index& index,
                                    const KernelCodegenInfo* kernel_info,
-                                   llvm::Value* y_loc, llvm::Value* x_loc);
+                                   llvm::Value* y_loc, llvm::Value* x_loc,
+                                   int64 x_iter_num);
   // Prepares for the code generation for a tile block of a reduction kernel.
   void EmitPrologueForReduction(HloInstruction* unnested_hlo,
                                 KernelCodegenInfo* kernel_info);
diff --git a/tensorflow/compiler/xla/service/llvm_ir/ir_array.h b/tensorflow/compiler/xla/service/llvm_ir/ir_array.h
index d6d84994ee..a483f7051f 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/ir_array.h
+++ b/tensorflow/compiler/xla/service/llvm_ir/ir_array.h
@@ -189,6 +189,8 @@ class IrArray {
       return llvm::ConstantInt::get(index_type_, c);
     }
 
+    void ClearLinearIndex() { linear_ = nullptr; }
+
    private:
     // Changing the multi-dimensional index invalidates the linear index.
     std::vector<llvm::Value*>& mutable_multidim() {
diff --git a/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.cc b/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.cc
index cebbc42901..cd8dd72cd7 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.cc
@@ -123,7 +123,8 @@ KernelMappingScheme::KernelMappingScheme(
       dims_in_elems_(dims_in_elems.begin(), dims_in_elems.end()),
       tile_sizes_{1, tile_size_y, tile_size_x},
       num_threads_x_(num_threads_x),
-      num_threads_y_(num_threads_y) {
+      num_threads_y_(num_threads_y),
+      dilated_x_(true) {
   DCHECK_EQ(dims_in_elems_.size(), 3);
   DCHECK_EQ(req_block_sizes.size(), 3);
 
diff --git a/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.h b/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.h
index fb633b12e6..f802cc27d5 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.h
+++ b/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.h
@@ -117,7 +117,10 @@ class KernelMappingScheme {
   int64 GetNumberOfTilesInOneBlock() const {
     return absl::c_accumulate(block_sizes_, 1, std::multiplies<int64>());
   }
-
+  int64 GetNumberOfTilesInOneBlockForDimension(int d) const {
+    DCHECK(d >= DimZ && d <= DimX);
+    return block_sizes_[d];
+  }
   int64 GetNumberOfBlocks() const {
     return absl::c_accumulate(dims_in_blocks_, 1, std::multiplies<int64>());
   }
@@ -147,6 +150,16 @@ class KernelMappingScheme {
            GetNumberOfThreadsForDimensionY();
   }
 
+  bool DilatedX() const { return dilated_x_; }
+  void SetDilatedX(bool v) {
+    dilated_x_ = v;
+    if (!dilated_x_) {
+      // dilated_x_=false is for the purpose of vectorization, which requires
+      // GetTileSizeForDimension(DimX) to be a multiplier of num_threads_x_.
+      CHECK_EQ(GetTileSizeForDimension(DimX) % num_threads_x_, 0);
+    }
+  }
+
   IrArray::Index EmitBlockIndex(llvm::Type* index_ty);
   // Returns the index for the first tile in the block with the given block
   // index.
@@ -186,6 +199,13 @@ class KernelMappingScheme {
   int64 num_threads_x_;
   // Number of threads used to process elements in the Y direction of a tile.
   int64 num_threads_y_;
+
+  // When num_threads_x threads process a total of tile_size_x elements in the
+  // X dimension of a tile, each threads process n=tile_size_x/num_threads_x
+  // elements. When dilated_x=false, the n elements processed by a thread are
+  // contiguous. On the other hand, when dilated_x=true the n elements are
+  // dilated by a factor of num_threads_x.
+  bool dilated_x_;
 };
 
 // A class to represent information for tiled parameters to support IR emission
-- 
GitLab


From 021a8fe8a70b880031958f596b6c96e819e14386 Mon Sep 17 00:00:00 2001
From: Nick Felt <nickfelt@google.com>
Date: Mon, 7 Jan 2019 12:23:54 -0800
Subject: [PATCH 0315/1095] Move contrib/tensorboard/db to new core/summary
 directory

PiperOrigin-RevId: 228213314
---
 tensorflow/contrib/cmake/tf_core_framework.cmake     | 12 ++++++------
 tensorflow/core/kernels/BUILD                        | 10 ++++------
 tensorflow/core/kernels/summary_kernels.cc           |  6 +++---
 .../{contrib/tensorboard/db => core/summary}/BUILD   |  2 +-
 .../tensorboard/db => core/summary}/loader.cc        |  4 ++--
 .../tensorboard/db => core/summary}/schema.cc        |  2 +-
 .../tensorboard/db => core/summary}/schema.h         |  6 +++---
 .../tensorboard/db => core/summary}/schema_test.cc   |  2 +-
 .../db => core/summary}/summary_converter.cc         |  2 +-
 .../db => core/summary}/summary_converter.h          |  6 +++---
 .../db => core/summary}/summary_db_writer.cc         |  4 ++--
 .../db => core/summary}/summary_db_writer.h          |  6 +++---
 .../db => core/summary}/summary_db_writer_test.cc    |  4 ++--
 .../db => core/summary}/summary_file_writer.cc       |  4 ++--
 .../db => core/summary}/summary_file_writer.h        |  6 +++---
 .../db => core/summary}/summary_file_writer_test.cc  |  2 +-
 .../tensorboard/db => core/summary}/vacuum.cc        |  0
 17 files changed, 38 insertions(+), 40 deletions(-)
 rename tensorflow/{contrib/tensorboard/db => core/summary}/BUILD (98%)
 rename tensorflow/{contrib/tensorboard/db => core/summary}/loader.cc (97%)
 rename tensorflow/{contrib/tensorboard/db => core/summary}/schema.cc (99%)
 rename tensorflow/{contrib/tensorboard/db => core/summary}/schema.h (87%)
 rename tensorflow/{contrib/tensorboard/db => core/summary}/schema_test.cc (95%)
 rename tensorflow/{contrib/tensorboard/db => core/summary}/summary_converter.cc (99%)
 rename tensorflow/{contrib/tensorboard/db => core/summary}/summary_converter.h (89%)
 rename tensorflow/{contrib/tensorboard/db => core/summary}/summary_db_writer.cc (99%)
 rename tensorflow/{contrib/tensorboard/db => core/summary}/summary_db_writer.h (89%)
 rename tensorflow/{contrib/tensorboard/db => core/summary}/summary_db_writer_test.cc (99%)
 rename tensorflow/{contrib/tensorboard/db => core/summary}/summary_file_writer.cc (98%)
 rename tensorflow/{contrib/tensorboard/db => core/summary}/summary_file_writer.h (89%)
 rename tensorflow/{contrib/tensorboard/db => core/summary}/summary_file_writer_test.cc (99%)
 rename tensorflow/{contrib/tensorboard/db => core/summary}/vacuum.cc (100%)

diff --git a/tensorflow/contrib/cmake/tf_core_framework.cmake b/tensorflow/contrib/cmake/tf_core_framework.cmake
index d7b2a1339e..9ae5831f47 100644
--- a/tensorflow/contrib/cmake/tf_core_framework.cmake
+++ b/tensorflow/contrib/cmake/tf_core_framework.cmake
@@ -302,8 +302,8 @@ file(GLOB_RECURSE tf_core_framework_srcs
     "${tensorflow_source_dir}/tensorflow/core/common_runtime/session.cc"
     "${tensorflow_source_dir}/tensorflow/core/common_runtime/session_factory.cc"
     "${tensorflow_source_dir}/tensorflow/core/common_runtime/session_options.cc"
-    "${tensorflow_source_dir}/tensorflow/contrib/tensorboard/db/*.cc"
-    "${tensorflow_source_dir}/tensorflow/contrib/tensorboard/db/*.h"
+    "${tensorflow_source_dir}/tensorflow/core/summary/*.cc"
+    "${tensorflow_source_dir}/tensorflow/core/summary/*.h"
     "${tensorflow_source_dir}/public/*.h"
 )
 
@@ -317,14 +317,14 @@ file(GLOB_RECURSE tf_core_framework_exclude_srcs
     "${tensorflow_source_dir}/tensorflow/core/util/*test*.h"
     "${tensorflow_source_dir}/tensorflow/core/util/*test*.cc"
     "${tensorflow_source_dir}/tensorflow/core/util/*main.cc"
-    "${tensorflow_source_dir}/tensorflow/contrib/tensorboard/db/*test*.cc"
-    "${tensorflow_source_dir}/tensorflow/contrib/tensorboard/db/loader.cc"
-    "${tensorflow_source_dir}/tensorflow/contrib/tensorboard/db/vacuum.cc"
+    "${tensorflow_source_dir}/tensorflow/core/summary/*test*.cc"
+    "${tensorflow_source_dir}/tensorflow/core/summary/loader.cc"
+    "${tensorflow_source_dir}/tensorflow/core/summary/vacuum.cc"
 )
 
 # TODO(jart): Why doesn't this work?
 # set_source_files_properties(
-#     ${tensorflow_source_dir}/tensorflow/contrib/tensorboard/db/snapfn.cc
+#     ${tensorflow_source_dir}/tensorflow/core/lib/db/snapfn.cc
 #     PROPERTIES COMPILE_FLAGS -DSQLITE_OMIT_LOAD_EXTENSION)
 
 list(REMOVE_ITEM tf_core_framework_srcs ${tf_core_framework_exclude_srcs})
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 3891caf5d3..136902e249 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -31,7 +31,6 @@ load(
     "//tensorflow:tensorflow.bzl",
     "cc_header_only_library",
     "if_android",
-    "if_not_v2",
     "if_not_windows",
     "tf_cc_binary",
     "tf_cc_test",
@@ -6888,11 +6887,10 @@ tf_kernel_library(
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:summary_ops_op_lib",
         "//tensorflow/core/lib/db:sqlite",
-    ] + if_not_v2([
-        "//tensorflow/contrib/tensorboard/db:schema",
-        "//tensorflow/contrib/tensorboard/db:summary_db_writer",
-        "//tensorflow/contrib/tensorboard/db:summary_file_writer",
-    ]),
+        "//tensorflow/core/summary:schema",
+        "//tensorflow/core/summary:summary_db_writer",
+        "//tensorflow/core/summary:summary_file_writer",
+    ],
 )
 
 tf_kernel_library(
diff --git a/tensorflow/core/kernels/summary_kernels.cc b/tensorflow/core/kernels/summary_kernels.cc
index b287f0cc2f..5e3465d1dd 100644
--- a/tensorflow/core/kernels/summary_kernels.cc
+++ b/tensorflow/core/kernels/summary_kernels.cc
@@ -13,14 +13,14 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/contrib/tensorboard/db/schema.h"
-#include "tensorflow/contrib/tensorboard/db/summary_db_writer.h"
-#include "tensorflow/contrib/tensorboard/db/summary_file_writer.h"
 #include "tensorflow/core/framework/graph.pb.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/resource_mgr.h"
 #include "tensorflow/core/lib/db/sqlite.h"
 #include "tensorflow/core/platform/protobuf.h"
+#include "tensorflow/core/summary/schema.h"
+#include "tensorflow/core/summary/summary_db_writer.h"
+#include "tensorflow/core/summary/summary_file_writer.h"
 #include "tensorflow/core/util/event.pb.h"
 
 namespace tensorflow {
diff --git a/tensorflow/contrib/tensorboard/db/BUILD b/tensorflow/core/summary/BUILD
similarity index 98%
rename from tensorflow/contrib/tensorboard/db/BUILD
rename to tensorflow/core/summary/BUILD
index 6507546ee9..a89175cdb1 100644
--- a/tensorflow/contrib/tensorboard/db/BUILD
+++ b/tensorflow/core/summary/BUILD
@@ -1,5 +1,5 @@
 # Description:
-#   TensorBoard database code.
+#   C++ implementation code for the summary writing APIs.
 
 package(default_visibility = ["//tensorflow:internal"])
 
diff --git a/tensorflow/contrib/tensorboard/db/loader.cc b/tensorflow/core/summary/loader.cc
similarity index 97%
rename from tensorflow/contrib/tensorboard/db/loader.cc
rename to tensorflow/core/summary/loader.cc
index 6439328022..68535feacf 100644
--- a/tensorflow/contrib/tensorboard/db/loader.cc
+++ b/tensorflow/core/summary/loader.cc
@@ -15,8 +15,8 @@ limitations under the License.
 #include <iostream>
 #include <vector>
 
-#include "tensorflow/contrib/tensorboard/db/schema.h"
-#include "tensorflow/contrib/tensorboard/db/summary_db_writer.h"
+#include "tensorflow/core/summary/schema.h"
+#include "tensorflow/core/summary/summary_db_writer.h"
 #include "tensorflow/core/lib/db/sqlite.h"
 #include "tensorflow/core/lib/io/record_reader.h"
 #include "tensorflow/core/platform/init_main.h"
diff --git a/tensorflow/contrib/tensorboard/db/schema.cc b/tensorflow/core/summary/schema.cc
similarity index 99%
rename from tensorflow/contrib/tensorboard/db/schema.cc
rename to tensorflow/core/summary/schema.cc
index 3c7bc87e4a..822e2fa3bf 100644
--- a/tensorflow/contrib/tensorboard/db/schema.cc
+++ b/tensorflow/core/summary/schema.cc
@@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#include "tensorflow/contrib/tensorboard/db/schema.h"
+#include "tensorflow/core/summary/schema.h"
 
 #include "tensorflow/core/lib/core/errors.h"
 
diff --git a/tensorflow/contrib/tensorboard/db/schema.h b/tensorflow/core/summary/schema.h
similarity index 87%
rename from tensorflow/contrib/tensorboard/db/schema.h
rename to tensorflow/core/summary/schema.h
index 3da4504225..6305f8eabd 100644
--- a/tensorflow/contrib/tensorboard/db/schema.h
+++ b/tensorflow/core/summary/schema.h
@@ -12,8 +12,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#ifndef TENSORFLOW_CONTRIB_TENSORBOARD_DB_SCHEMA_H_
-#define TENSORFLOW_CONTRIB_TENSORBOARD_DB_SCHEMA_H_
+#ifndef TENSORFLOW_CORE_SUMMARY_SCHEMA_H_
+#define TENSORFLOW_CORE_SUMMARY_SCHEMA_H_
 
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/db/sqlite.h"
@@ -30,4 +30,4 @@ Status SetupTensorboardSqliteDb(Sqlite* db);
 
 }  // namespace tensorflow
 
-#endif  // TENSORFLOW_CONTRIB_TENSORBOARD_DB_SCHEMA_H_
+#endif  // TENSORFLOW_CORE_SUMMARY_SCHEMA_H_
diff --git a/tensorflow/contrib/tensorboard/db/schema_test.cc b/tensorflow/core/summary/schema_test.cc
similarity index 95%
rename from tensorflow/contrib/tensorboard/db/schema_test.cc
rename to tensorflow/core/summary/schema_test.cc
index 4d3f2880bd..fa21b45b62 100644
--- a/tensorflow/contrib/tensorboard/db/schema_test.cc
+++ b/tensorflow/core/summary/schema_test.cc
@@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#include "tensorflow/contrib/tensorboard/db/schema.h"
+#include "tensorflow/core/summary/schema.h"
 
 #include <memory>
 
diff --git a/tensorflow/contrib/tensorboard/db/summary_converter.cc b/tensorflow/core/summary/summary_converter.cc
similarity index 99%
rename from tensorflow/contrib/tensorboard/db/summary_converter.cc
rename to tensorflow/core/summary/summary_converter.cc
index 93c1183072..e6e34e9602 100644
--- a/tensorflow/contrib/tensorboard/db/summary_converter.cc
+++ b/tensorflow/core/summary/summary_converter.cc
@@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#include "tensorflow/contrib/tensorboard/db/summary_converter.h"
+#include "tensorflow/core/summary/summary_converter.h"
 
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/summary.pb.h"
diff --git a/tensorflow/contrib/tensorboard/db/summary_converter.h b/tensorflow/core/summary/summary_converter.h
similarity index 89%
rename from tensorflow/contrib/tensorboard/db/summary_converter.h
rename to tensorflow/core/summary/summary_converter.h
index 329c7f9f2f..dc005d2604 100644
--- a/tensorflow/contrib/tensorboard/db/summary_converter.h
+++ b/tensorflow/core/summary/summary_converter.h
@@ -12,8 +12,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#ifndef TENSORFLOW_CONTRIB_TENSORBOARD_DB_SUMMARY_CONVERTER_H_
-#define TENSORFLOW_CONTRIB_TENSORBOARD_DB_SUMMARY_CONVERTER_H_
+#ifndef TENSORFLOW_CORE_SUMMARY_SUMMARY_CONVERTER_H_
+#define TENSORFLOW_CORE_SUMMARY_SUMMARY_CONVERTER_H_
 
 #include "tensorflow/core/framework/summary.pb.h"
 #include "tensorflow/core/framework/tensor.h"
@@ -35,4 +35,4 @@ Status AddTensorAsAudioToSummary(const Tensor& tensor, const string& tag,
 
 }  // namespace tensorflow
 
-#endif  // TENSORFLOW_CONTRIB_TENSORBOARD_DB_SUMMARY_CONVERTER_H_
+#endif  // TENSORFLOW_CORE_SUMMARY_SUMMARY_CONVERTER_H_
diff --git a/tensorflow/contrib/tensorboard/db/summary_db_writer.cc b/tensorflow/core/summary/summary_db_writer.cc
similarity index 99%
rename from tensorflow/contrib/tensorboard/db/summary_db_writer.cc
rename to tensorflow/core/summary/summary_db_writer.cc
index cfdc884277..7a5d796821 100644
--- a/tensorflow/contrib/tensorboard/db/summary_db_writer.cc
+++ b/tensorflow/core/summary/summary_db_writer.cc
@@ -12,11 +12,11 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#include "tensorflow/contrib/tensorboard/db/summary_db_writer.h"
+#include "tensorflow/core/summary/summary_db_writer.h"
 
 #include <deque>
 
-#include "tensorflow/contrib/tensorboard/db/summary_converter.h"
+#include "tensorflow/core/summary/summary_converter.h"
 #include "tensorflow/core/framework/graph.pb.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/framework/register_types.h"
diff --git a/tensorflow/contrib/tensorboard/db/summary_db_writer.h b/tensorflow/core/summary/summary_db_writer.h
similarity index 89%
rename from tensorflow/contrib/tensorboard/db/summary_db_writer.h
rename to tensorflow/core/summary/summary_db_writer.h
index 746da1533b..5669afe7f6 100644
--- a/tensorflow/contrib/tensorboard/db/summary_db_writer.h
+++ b/tensorflow/core/summary/summary_db_writer.h
@@ -12,8 +12,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#ifndef TENSORFLOW_CONTRIB_TENSORBOARD_DB_SUMMARY_DB_WRITER_H_
-#define TENSORFLOW_CONTRIB_TENSORBOARD_DB_SUMMARY_DB_WRITER_H_
+#ifndef TENSORFLOW_CORE_SUMMARY_SUMMARY_DB_WRITER_H_
+#define TENSORFLOW_CORE_SUMMARY_SUMMARY_DB_WRITER_H_
 
 #include "tensorflow/core/kernels/summary_interface.h"
 #include "tensorflow/core/lib/core/status.h"
@@ -39,4 +39,4 @@ Status CreateSummaryDbWriter(Sqlite* db, const string& experiment_name,
 
 }  // namespace tensorflow
 
-#endif  // TENSORFLOW_CONTRIB_TENSORBOARD_DB_SUMMARY_DB_WRITER_H_
+#endif  // TENSORFLOW_CORE_SUMMARY_SUMMARY_DB_WRITER_H_
diff --git a/tensorflow/contrib/tensorboard/db/summary_db_writer_test.cc b/tensorflow/core/summary/summary_db_writer_test.cc
similarity index 99%
rename from tensorflow/contrib/tensorboard/db/summary_db_writer_test.cc
rename to tensorflow/core/summary/summary_db_writer_test.cc
index 2e8d4109dd..c4e9ddea2c 100644
--- a/tensorflow/contrib/tensorboard/db/summary_db_writer_test.cc
+++ b/tensorflow/core/summary/summary_db_writer_test.cc
@@ -12,9 +12,9 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#include "tensorflow/contrib/tensorboard/db/summary_db_writer.h"
+#include "tensorflow/core/summary/summary_db_writer.h"
 
-#include "tensorflow/contrib/tensorboard/db/schema.h"
+#include "tensorflow/core/summary/schema.h"
 #include "tensorflow/core/framework/function.pb.h"
 #include "tensorflow/core/framework/graph.pb.h"
 #include "tensorflow/core/framework/node_def.pb.h"
diff --git a/tensorflow/contrib/tensorboard/db/summary_file_writer.cc b/tensorflow/core/summary/summary_file_writer.cc
similarity index 98%
rename from tensorflow/contrib/tensorboard/db/summary_file_writer.cc
rename to tensorflow/core/summary/summary_file_writer.cc
index 22b6f09d0c..593ccdd684 100644
--- a/tensorflow/contrib/tensorboard/db/summary_file_writer.cc
+++ b/tensorflow/core/summary/summary_file_writer.cc
@@ -12,9 +12,9 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#include "tensorflow/contrib/tensorboard/db/summary_file_writer.h"
+#include "tensorflow/core/summary/summary_file_writer.h"
 
-#include "tensorflow/contrib/tensorboard/db/summary_converter.h"
+#include "tensorflow/core/summary/summary_converter.h"
 #include "tensorflow/core/framework/graph.pb.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/resource_mgr.h"
diff --git a/tensorflow/contrib/tensorboard/db/summary_file_writer.h b/tensorflow/core/summary/summary_file_writer.h
similarity index 89%
rename from tensorflow/contrib/tensorboard/db/summary_file_writer.h
rename to tensorflow/core/summary/summary_file_writer.h
index 73b0a5542b..7d964516da 100644
--- a/tensorflow/contrib/tensorboard/db/summary_file_writer.h
+++ b/tensorflow/core/summary/summary_file_writer.h
@@ -12,8 +12,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#ifndef TENSORFLOW_CONTRIB_TENSORBOARD_DB_SUMMARY_FILE_WRITER_H_
-#define TENSORFLOW_CONTRIB_TENSORBOARD_DB_SUMMARY_FILE_WRITER_H_
+#ifndef TENSORFLOW_CORE_SUMMARY_SUMMARY_FILE_WRITER_H_
+#define TENSORFLOW_CORE_SUMMARY_SUMMARY_FILE_WRITER_H_
 
 #include "tensorflow/core/kernels/summary_interface.h"
 #include "tensorflow/core/lib/core/status.h"
@@ -40,4 +40,4 @@ Status CreateSummaryFileWriter(int max_queue, int flush_millis,
 
 }  // namespace tensorflow
 
-#endif  // TENSORFLOW_CONTRIB_TENSORBOARD_DB_SUMMARY_FILE_WRITER_H_
+#endif  // TENSORFLOW_CORE_SUMMARY_SUMMARY_FILE_WRITER_H_
diff --git a/tensorflow/contrib/tensorboard/db/summary_file_writer_test.cc b/tensorflow/core/summary/summary_file_writer_test.cc
similarity index 99%
rename from tensorflow/contrib/tensorboard/db/summary_file_writer_test.cc
rename to tensorflow/core/summary/summary_file_writer_test.cc
index ffbfb9533e..d3b19c3abd 100644
--- a/tensorflow/contrib/tensorboard/db/summary_file_writer_test.cc
+++ b/tensorflow/core/summary/summary_file_writer_test.cc
@@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#include "tensorflow/contrib/tensorboard/db/summary_file_writer.h"
+#include "tensorflow/core/summary/summary_file_writer.h"
 
 #include "tensorflow/core/framework/summary.pb.h"
 #include "tensorflow/core/framework/tensor.pb.h"
diff --git a/tensorflow/contrib/tensorboard/db/vacuum.cc b/tensorflow/core/summary/vacuum.cc
similarity index 100%
rename from tensorflow/contrib/tensorboard/db/vacuum.cc
rename to tensorflow/core/summary/vacuum.cc
-- 
GitLab


From bd2610f638bd35dca30bb00a6c16da775b3bfcdf Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Mon, 7 Jan 2019 12:31:43 -0800
Subject: [PATCH 0316/1095] [Grappler] Remapper for GPU
 Conv2d+BiasAdd+Activation

PiperOrigin-RevId: 228214620
---
 tensorflow/core/grappler/BUILD                |   1 +
 tensorflow/core/grappler/graph_view.cc        |  23 +-
 tensorflow/core/grappler/graph_view.h         |  10 +-
 tensorflow/core/grappler/optimizers/BUILD     |   2 +
 .../core/grappler/optimizers/remapper.cc      | 384 ++++++++++++------
 tensorflow/core/grappler/utils.cc             |  18 +-
 tensorflow/core/grappler/utils.h              |   3 +
 tensorflow/core/grappler/utils/BUILD          |   1 +
 .../core/grappler/utils/grappler_test.cc      |  27 ++
 .../core/grappler/utils/grappler_test.h       |  16 +-
 tensorflow/core/kernels/conv_ops_test.cc      |  20 +
 11 files changed, 353 insertions(+), 152 deletions(-)

diff --git a/tensorflow/core/grappler/BUILD b/tensorflow/core/grappler/BUILD
index 4d0f02f4d6..55d642612b 100644
--- a/tensorflow/core/grappler/BUILD
+++ b/tensorflow/core/grappler/BUILD
@@ -27,6 +27,7 @@ cc_library(
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:protos_all_cc",
+        "@com_google_absl//absl/strings",
         "@com_google_absl//absl/types:span",
     ],
 )
diff --git a/tensorflow/core/grappler/graph_view.cc b/tensorflow/core/grappler/graph_view.cc
index ba9d2eb321..be9b9c36c7 100644
--- a/tensorflow/core/grappler/graph_view.cc
+++ b/tensorflow/core/grappler/graph_view.cc
@@ -66,28 +66,27 @@ int OpInputPortIdToArgId(const NodeDef& node, const OpDef& op, int port_id) {
 bool HasSingleFanoutNode(const GraphView& graph_view, const NodeDef* node,
                          int port) {
   const auto output = GraphView::OutputPort(node, port);
-  const auto fanout = graph_view.GetFanout(output);
-  return fanout.size() <= 1;
+  return graph_view.GetFanout(output).size() <= 1;
 }
 
 bool HasFanouts(const GraphView& graph_view, const NodeDef* node, int port) {
   const auto output = GraphView::OutputPort(node, port);
-  const auto fanout = graph_view.GetFanout(output);
-  return !fanout.empty();
+  return !graph_view.GetFanout(output).empty();
 }
 
-bool NoControlFanin(const GraphView& graph_view, const NodeDef* node) {
-  const auto control_port = GraphView::InputPort(node, -1);
-  return graph_view.GetFanin(control_port).empty();
+bool HasControlFanin(const GraphView& graph_view, const NodeDef* node) {
+  const auto control_port = GraphView::InputPort(node, Graph::kControlSlot);
+  return !graph_view.GetFanin(control_port).empty();
 }
 
-bool NoControlFanout(const GraphView& graph_view, const NodeDef* node) {
-  const auto control_port = GraphView::OutputPort(node, -1);
-  return graph_view.GetFanout(control_port).empty();
+bool HasControlFanout(const GraphView& graph_view, const NodeDef* node) {
+  const auto control_port = GraphView::OutputPort(node, Graph::kControlSlot);
+  return !graph_view.GetFanout(control_port).empty();
 }
 
-bool NoControlFaninOrFanout(const GraphView& graph_view, const NodeDef* node) {
-  return NoControlFanin(graph_view, node) && NoControlFanout(graph_view, node);
+bool HasControlFaninOrFanout(const GraphView& graph_view, const NodeDef* node) {
+  return HasControlFanin(graph_view, node) ||
+         HasControlFanout(graph_view, node);
 }
 
 }  // end namespace grappler
diff --git a/tensorflow/core/grappler/graph_view.h b/tensorflow/core/grappler/graph_view.h
index a17d17524a..dc4ab93894 100644
--- a/tensorflow/core/grappler/graph_view.h
+++ b/tensorflow/core/grappler/graph_view.h
@@ -369,10 +369,12 @@ bool HasSingleFanoutNode(const GraphView& graph_view, const NodeDef* node,
 
 // Returns true if node has at least one fanout node at given output port.
 bool HasFanouts(const GraphView& graph_view, const NodeDef* node, int port = 0);
-
-bool NoControlFanin(const GraphView& graph_view, const NodeDef* node);
-bool NoControlFanout(const GraphView& graph_view, const NodeDef* node);
-bool NoControlFaninOrFanout(const GraphView& graph_view, const NodeDef* node);
+// Returns true if the node has at least one input control dependency.
+bool HasControlFanin(const GraphView& graph_view, const NodeDef* node);
+// Returns true if the node has at least one output control dependency.
+bool HasControlFanout(const GraphView& graph_view, const NodeDef* node);
+// Returns true if the node has at least one input or output control dependency.
+bool HasControlFaninOrFanout(const GraphView& graph_view, const NodeDef* node);
 
 }  // end namespace grappler
 }  // end namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index e2b9de7cc5..7e29cee86a 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -700,6 +700,7 @@ cc_library(
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/costs:graph_properties",
+        "//tensorflow/core/grappler/utils:symbolic_shapes",
         "//tensorflow/core/grappler/utils:topological_sort",
         "@com_google_absl//absl/container:flat_hash_set",
     ],
@@ -711,6 +712,7 @@ tf_cuda_cc_test(
     deps = [
         ":remapper",
         "//tensorflow/cc:cc_ops",
+        "//tensorflow/core:framework",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
diff --git a/tensorflow/core/grappler/optimizers/remapper.cc b/tensorflow/core/grappler/optimizers/remapper.cc
index f0c81f29e6..0869e3b49b 100644
--- a/tensorflow/core/grappler/optimizers/remapper.cc
+++ b/tensorflow/core/grappler/optimizers/remapper.cc
@@ -23,6 +23,7 @@ limitations under the License.
 #include "tensorflow/core/grappler/op_types.h"
 #include "tensorflow/core/grappler/optimizers/constant_folding.h"
 #include "tensorflow/core/grappler/utils.h"
+#include "tensorflow/core/grappler/utils/symbolic_shapes.h"
 #include "tensorflow/core/grappler/utils/topological_sort.h"
 #include "tensorflow/core/platform/logging.h"
 
@@ -60,17 +61,30 @@ struct RemapperContext {
 
 // FusedBatchNorm that can be replaced with a cheaper set of primitives.
 struct FusedBatchNorm {
+  FusedBatchNorm() = default;
+  explicit FusedBatchNorm(const NodeDef* fused_batch_norm)
+      : fused_batch_norm(fused_batch_norm) {}
+
   const NodeDef* fused_batch_norm = nullptr;
 };
 
 // Conv2D node followed by a BiasAdd.
 struct Conv2DWithBiasAdd {
+  Conv2DWithBiasAdd() = default;
+  Conv2DWithBiasAdd(const NodeDef* conv2d, const NodeDef* bias_add)
+      : conv2d(conv2d), bias_add(bias_add) {}
+
   const NodeDef* conv2d = nullptr;
   const NodeDef* bias_add = nullptr;
 };
 
 // Conv2D node followed by a BiasAdd and Relu.
 struct Conv2DWithBiasAddAndRelu {
+  Conv2DWithBiasAddAndRelu() = default;
+  Conv2DWithBiasAddAndRelu(const NodeDef* conv2d, const NodeDef* bias_add,
+                           const NodeDef* relu)
+      : conv2d(conv2d), bias_add(bias_add), relu(relu) {}
+
   const NodeDef* conv2d = nullptr;
   const NodeDef* bias_add = nullptr;
   const NodeDef* relu = nullptr;
@@ -78,6 +92,11 @@ struct Conv2DWithBiasAddAndRelu {
 
 // Conv2D node followed by a Squeeze and BiasAdd.
 struct Conv2DWithSqueezeAndBiasAdd {
+  Conv2DWithSqueezeAndBiasAdd() = default;
+  Conv2DWithSqueezeAndBiasAdd(const NodeDef* conv2d, const NodeDef* squeeze,
+                              const NodeDef* bias_add)
+      : conv2d(conv2d), squeeze(squeeze), bias_add(bias_add) {}
+
   const NodeDef* conv2d = nullptr;
   const NodeDef* squeeze = nullptr;
   const NodeDef* bias_add = nullptr;
@@ -85,6 +104,11 @@ struct Conv2DWithSqueezeAndBiasAdd {
 
 // Conv2D node followed by a FusedBatchNorm.
 struct Conv2DWithBatchNorm {
+  Conv2DWithBatchNorm() = default;
+  Conv2DWithBatchNorm(const NodeDef* conv2d, const NodeDef* fused_batch_norm,
+                      float epsilon = 0.0)
+      : conv2d(conv2d), fused_batch_norm(fused_batch_norm), epsilon(epsilon) {}
+
   const NodeDef* conv2d = nullptr;
   const NodeDef* fused_batch_norm = nullptr;
   float epsilon = 0.0;
@@ -92,16 +116,23 @@ struct Conv2DWithBatchNorm {
 
 // Conv2D node followed by a FusedBatchNorm and Relu.
 struct Conv2DWithBatchNormAndRelu {
+  Conv2DWithBatchNormAndRelu() = default;
+  Conv2DWithBatchNormAndRelu(const NodeDef* conv2d,
+                             const NodeDef* fused_batch_norm,
+                             const NodeDef* relu, float epsilon = 0.0)
+      : conv2d(conv2d),
+        fused_batch_norm(fused_batch_norm),
+        relu(relu),
+        epsilon(epsilon) {}
+
   const NodeDef* conv2d = nullptr;
   const NodeDef* fused_batch_norm = nullptr;
   const NodeDef* relu = nullptr;
   float epsilon = 0.0;
 };
 
-bool IsFloatOrDoubleDataType(const NodeDef* node,
-                             const string& type_attr = "T") {
-  DataType dtype = GetDataTypeFromAttr(*node, type_attr);
-  return dtype == DT_FLOAT || dtype == DT_DOUBLE;
+bool IsInPreserveSet(const RemapperContext& ctx, const NodeDef* node) {
+  return ctx.nodes_to_preserve.count(node->name()) > 0;
 }
 
 bool HaveSameDataType(const NodeDef* lhs, const NodeDef* rhs,
@@ -119,91 +150,165 @@ bool HasDataType(const NodeDef* node, const DataType& expected,
   return dtype == expected;
 }
 
-bool IsInPreserveSet(const RemapperContext& ctx, const NodeDef* node) {
-  return ctx.nodes_to_preserve.count(node->name()) > 0;
+bool IsCpuCompatibleDataType(const NodeDef* node,
+                             const string& type_attr = "T") {
+  DataType dtype = GetDataTypeFromAttr(*node, type_attr);
+  return dtype == DT_FLOAT || dtype == DT_DOUBLE;
+}
+
+bool IsGpuCompatibleDataType(const NodeDef* node,
+                             const string& type_attr = "T") {
+  DataType dtype = GetDataTypeFromAttr(*node, type_attr);
+  return dtype == DT_FLOAT;
+}
+
+bool IsCpuCompatibleDataFormat(const NodeDef* conv2d) {
+  DCHECK(IsConv2D(*conv2d)) << "Expected Conv2D op";
+  const string& data_format = conv2d->attr().at(kDataFormat).s();
+  return data_format == "NHWC";
+}
+
+bool IsGpuCompatibleDataFormat(const NodeDef* conv2d) {
+  DCHECK(IsConv2D(*conv2d)) << "Expected Conv2D op";
+  const string& data_format = conv2d->attr().at(kDataFormat).s();
+  return data_format == "NHWC" || data_format == "NCHW";
 }
 
-bool FindConv2DWithBias(const RemapperContext& ctx, const NodeDef* node,
-                        Conv2DWithBiasAdd* matched) {
+bool IsCpuCompatibleConv2D(const NodeDef* conv2d) {
+  DCHECK(IsConv2D(*conv2d)) << "Expected Conv2D op";
+  return NodeIsOnCpu(conv2d) && IsCpuCompatibleDataType(conv2d) &&
+         IsCpuCompatibleDataFormat(conv2d);
+}
+
+bool IsGpuCompatibleConv2D(const NodeDef* conv2d) {
+  DCHECK(IsConv2D(*conv2d)) << "Expected Conv2D op";
+  return NodeIsOnGpu(conv2d) && IsGpuCompatibleDataType(conv2d) &&
+         IsGpuCompatibleDataFormat(conv2d);
+}
+
+// Checks if we can rewrite a pattern to the `_FusedConv2D` on CPU device.
+template <typename Pattern>
+bool IsCpuCompatible(const Pattern& matched) {
+  return IsCpuCompatibleConv2D(matched.conv2d);
+}
+
+// Checks if we can rewrite a pattern to the `_FusedConv2D` on GPU device.
+bool IsGpuCompatible(const RemapperContext& ctx,
+                     const Conv2DWithBiasAddAndRelu& matched) {
+  const std::vector<OpInfo::TensorProperties>& input_props =
+      ctx.graph_properties.GetInputProperties(matched.conv2d->name());
+  const TensorShapeProto& filter_shape =
+      input_props.size() >= 2 ? input_props[1].shape() : TensorShapeProto();
+
+  // FusedConv2D on GPU with 1x1 convolution is marginally faster than
+  // in-graph computation in micro benchmarks (see kernels/conv_ops_test.cc),
+  // and significantly slower in large scale benchmarks.
+  bool is_spatial_conv = Rank(filter_shape) == 4 &&          //
+                         IsKnown(filter_shape.dim(1)) &&     //
+                         IsKnown(filter_shape.dim(2)) &&     //
+                         filter_shape.dim(1).size() != 1 &&  //
+                         filter_shape.dim(2).size() != 1;
+
+  return is_spatial_conv && IsGpuCompatibleConv2D(matched.conv2d);
+}
+bool IsGpuCompatible(const RemapperContext& ctx,
+                     const Conv2DWithBiasAdd& matched) {
+  return false;
+}
+bool IsGpuCompatible(const RemapperContext& ctx,
+                     const Conv2DWithSqueezeAndBiasAdd& matched) {
+  return false;
+}
+
+// Returns true if the given pattern is supported on the assigned device.
+template <typename Pattern>
+bool IsDeviceCompatible(const RemapperContext& ctx, Pattern& matched) {
+  return IsCpuCompatible(matched) || IsGpuCompatible(ctx, matched);
+}
+
+bool FindConv2DWithBias(const RemapperContext& ctx, const NodeDef* bias_add,
+                        Conv2DWithBiasAdd* matched,
+                        bool check_device_compatible = true) {
   if (!EigenSupportsContractionOutputKernel()) return false;
 
   // Root of the pattern must be a BiasAdd.
-  if (!node) return false;
-  if (!IsBiasAdd(*node)) return false;
-  if (!NodeIsOnCpu(node)) return false;
-  if (!IsFloatOrDoubleDataType(node)) return false;
-  if (!NoControlFaninOrFanout(ctx.graph_view, node)) return false;
+  if (bias_add == nullptr || !IsBiasAdd(*bias_add) ||
+      HasControlFaninOrFanout(ctx.graph_view, bias_add))
+    return false;
 
-  // Input to the BiasAdd must be a Conv2D in NHWC format.
-  const auto input_port = GraphView::InputPort(node, 0);
+  // Input to the BiasAdd must be a Conv2D.
+  const auto input_port = GraphView::InputPort(bias_add, 0);
   const auto conv2d = ctx.graph_view.GetRegularFanin(input_port);
-  if (!conv2d.node) return false;
-  if (!IsConv2D(*conv2d.node)) return false;
-  if (conv2d.node->attr().at(kDataFormat).s() != "NHWC") return false;
-  if (!NodeIsOnCpu(conv2d.node)) return false;
-  if (!HaveSameDataType(node, conv2d.node)) return false;
-  if (!NoControlFaninOrFanout(ctx.graph_view, conv2d.node)) return false;
-  if (!HasSingleFanoutNode(ctx.graph_view, conv2d.node)) return false;
-  if (IsInPreserveSet(ctx, conv2d.node)) return false;
+
+  if (!conv2d.node || !IsConv2D(*conv2d.node) ||
+      !HaveSameDataType(bias_add, conv2d.node) ||
+      HasControlFaninOrFanout(ctx.graph_view, conv2d.node) ||
+      !HasSingleFanoutNode(ctx.graph_view, conv2d.node) ||
+      IsInPreserveSet(ctx, conv2d.node))
+    return false;
+
+  // Check that data type and data format are supported on assigned device.
+  const Conv2DWithBiasAdd pattern{conv2d.node, bias_add};
+  if (check_device_compatible && !IsDeviceCompatible(ctx, pattern)) {
+    return false;
+  }
 
   // We successfully found a Conv2D+BiasAdd pattern.
-  matched->conv2d = conv2d.node;
-  matched->bias_add = node;
+  *matched = pattern;
 
   return true;
 }
 
-bool FindConv2DWithBiasAndRelu(const RemapperContext& ctx, const NodeDef* node,
+bool FindConv2DWithBiasAndRelu(const RemapperContext& ctx, const NodeDef* relu,
                                Conv2DWithBiasAddAndRelu* matched) {
   if (!EigenSupportsContractionOutputKernel()) return false;
 
   // Root of the pattern must be a Relu.
-  if (!node) return false;
-  if (!IsRelu(*node)) return false;
-  if (!NodeIsOnCpu(node)) return false;
-  if (!IsFloatOrDoubleDataType(node)) return false;
-  if (!NoControlFaninOrFanout(ctx.graph_view, node)) return false;
+  if (!relu || !IsRelu(*relu) || HasControlFaninOrFanout(ctx.graph_view, relu))
+    return false;
 
   // And input to Relu must match Conv2DWithBiasAdd pattern.
-  const auto input_port = GraphView::InputPort(node, 0);
+  const auto input_port = GraphView::InputPort(relu, 0);
   const auto bias_add = ctx.graph_view.GetRegularFanin(input_port);
 
   Conv2DWithBiasAdd base;
-  if (!FindConv2DWithBias(ctx, bias_add.node, &base)) return false;
-  if (!HasSingleFanoutNode(ctx.graph_view, base.bias_add)) return false;
-  if (!HaveSameDataType(node, base.bias_add)) return false;
-  if (IsInPreserveSet(ctx, base.bias_add)) return false;
+  if (!FindConv2DWithBias(ctx, bias_add.node, &base,
+                          /*check_device_compatible=*/false) ||
+      !HasSingleFanoutNode(ctx.graph_view, base.bias_add) ||
+      !HaveSameDataType(relu, base.bias_add) ||
+      IsInPreserveSet(ctx, base.bias_add))
+    return false;
+
+  // Check that data type and data format are supported on assigned device.
+  const Conv2DWithBiasAddAndRelu pattern{base.conv2d, base.bias_add, relu};
+  if (!IsDeviceCompatible(ctx, pattern)) return false;
 
   // We successfully found a Conv2D+BiasAdd+Relu pattern.
-  matched->conv2d = base.conv2d;
-  matched->bias_add = base.bias_add;
-  matched->relu = node;
+  *matched = pattern;
 
   return true;
 }
 
 bool FindConv2DWithSqueezeAndBias(const RemapperContext& ctx,
-                                  const NodeDef* node,
+                                  const NodeDef* bias_add,
                                   Conv2DWithSqueezeAndBiasAdd* matched) {
   if (!EigenSupportsContractionOutputKernel()) return false;
 
   // Root of the pattern must be a BiasAdd.
-  if (node == nullptr) return false;
-  if (node->op() != "BiasAdd") return false;
-  if (!NodeIsOnCpu(node)) return false;
-  if (!IsFloatOrDoubleDataType(node)) return false;
-  if (!NoControlFaninOrFanout(ctx.graph_view, node)) return false;
+  if (!bias_add || !IsBiasAdd(*bias_add) ||
+      HasControlFaninOrFanout(ctx.graph_view, bias_add))
+    return false;
 
   // Input to the BiasAdd must be a Squeeze.
-  const auto bias_input_port = GraphView::InputPort(node, 0);
+  const auto bias_input_port = GraphView::InputPort(bias_add, 0);
   const auto squeeze = ctx.graph_view.GetRegularFanin(bias_input_port);
-  if (squeeze.node == nullptr) return false;
-  if (squeeze.node->op() != "Squeeze") return false;
-  if (!NodeIsOnCpu(squeeze.node)) return false;
-  if (!HaveSameDataType(node, squeeze.node, "T")) return false;
-  if (!NoControlFaninOrFanout(ctx.graph_view, squeeze.node)) return false;
-  if (!HasSingleFanoutNode(ctx.graph_view, squeeze.node)) return false;
-  if (IsInPreserveSet(ctx, squeeze.node)) return false;
+
+  if (!squeeze.node || !IsSqueeze(*squeeze.node) ||
+      !HaveSameDataType(bias_add, squeeze.node, "T") ||
+      HasControlFaninOrFanout(ctx.graph_view, squeeze.node) ||
+      !HasSingleFanoutNode(ctx.graph_view, squeeze.node) ||
+      IsInPreserveSet(ctx, squeeze.node))
+    return false;
 
   // Squeeze must not squeeze output channel dimension.
   std::vector<int32> dims;
@@ -212,67 +317,72 @@ bool FindConv2DWithSqueezeAndBias(const RemapperContext& ctx,
     if (dim == 3) return false;
   }
 
-  // Input to the Squeeze must be a Conv2D in NHWC format.
+  // Input to the Squeeze must be a Conv2D.
   const auto squeeze_input_port = GraphView::InputPort(squeeze.node, 0);
   const auto conv2d = ctx.graph_view.GetRegularFanin(squeeze_input_port);
-  if (conv2d.node == nullptr) return false;
-  if (conv2d.node->op() != "Conv2D") return false;
-  if (conv2d.node->attr().at("data_format").s() != "NHWC") return false;
-  if (!NodeIsOnCpu(conv2d.node)) return false;
-  if (!HaveSameDataType(node, conv2d.node, "T")) return false;
-  if (!NoControlFaninOrFanout(ctx.graph_view, conv2d.node)) return false;
-  if (!HasSingleFanoutNode(ctx.graph_view, conv2d.node)) return false;
-  if (IsInPreserveSet(ctx, conv2d.node)) return false;
+
+  if (!conv2d.node || !IsConv2D(*conv2d.node) ||
+      !HaveSameDataType(bias_add, conv2d.node, "T") ||
+      HasControlFaninOrFanout(ctx.graph_view, conv2d.node) ||
+      !HasSingleFanoutNode(ctx.graph_view, conv2d.node) ||
+      IsInPreserveSet(ctx, conv2d.node))
+    return false;
+
+  // Check that data type and data format are supported on assigned device.
+  const Conv2DWithSqueezeAndBiasAdd pattern{conv2d.node, squeeze.node,
+                                            bias_add};
+  if (!IsDeviceCompatible(ctx, pattern)) return false;
 
   // We successfully found a Conv2D+Squeeze+BiasAdd pattern.
-  matched->conv2d = conv2d.node;
-  matched->squeeze = squeeze.node;
-  matched->bias_add = node;
+  *matched = pattern;
 
   return true;
 }
 
-bool FindConv2DWithBatchNorm(const RemapperContext& ctx, const NodeDef* node,
+bool FindConv2DWithBatchNorm(const RemapperContext& ctx,
+                             const NodeDef* batch_norm,
                              Conv2DWithBatchNorm* matched) {
   if (!EigenSupportsContractionOutputKernel()) return false;
 
   // Root of the pattern must be a FusedBatchNorm or a FusedBatchNormV2.
-  if (node == nullptr) return false;
-  if (!IsFusedBatchNorm(*node)) return false;
-  if (!NodeIsOnCpu(node)) return false;
-  if (!HasDataType(node, DT_FLOAT)) return false;
+  if (!batch_norm || !IsFusedBatchNorm(*batch_norm)) return false;
 
   // V2 has a separate data type for the scale/offset/mean/variance inputs.
-  if (node->op() == "FusedBatchNormV2" && !HasDataType(node, DT_FLOAT, "U"))
+  if (batch_norm->op() == "FusedBatchNormV2" &&
+      !HasDataType(batch_norm, DT_FLOAT, "U"))
     return false;
 
   // Check that batch normalization is in inference mode.
-  const auto& attr = node->attr();
+  const auto& attr = batch_norm->attr();
   if (attr.count(kIsTraining) > 0 && attr.at(kIsTraining).b()) return false;
 
   // Check that only 0th output is consumed by other nodes.
-  if (!NoControlFaninOrFanout(ctx.graph_view, node)) return false;
-  if (HasFanouts(ctx.graph_view, node, 1)) return false;  // batch_mean
-  if (HasFanouts(ctx.graph_view, node, 2)) return false;  // batch_variance
-  if (HasFanouts(ctx.graph_view, node, 3)) return false;  // reserve_space_1
-  if (HasFanouts(ctx.graph_view, node, 4)) return false;  // reserve_space_2
+  if (HasControlFaninOrFanout(ctx.graph_view, batch_norm) ||
+      HasFanouts(ctx.graph_view, batch_norm, 1) ||  // batch_mean
+      HasFanouts(ctx.graph_view, batch_norm, 2) ||  // batch_variance
+      HasFanouts(ctx.graph_view, batch_norm, 3) ||  // reserve_space_1
+      HasFanouts(ctx.graph_view, batch_norm, 4))    // reserve_space_2
+    return false;
 
-  // Input to the FusedBatchNorm must be a Conv2D in NHWC format.
-  const auto input_port = GraphView::InputPort(node, 0);
+  // Input to the FusedBatchNorm must be a Conv2D.
+  const auto input_port = GraphView::InputPort(batch_norm, 0);
   const auto conv2d = ctx.graph_view.GetRegularFanin(input_port);
-  if (conv2d.node == nullptr) return false;
-  if (!IsConv2D(*conv2d.node)) return false;
-  if (conv2d.node->attr().at(kDataFormat).s() != "NHWC") return false;
-  if (!NodeIsOnCpu(conv2d.node)) return false;
-  if (!HaveSameDataType(node, conv2d.node)) return false;
-  if (!NoControlFaninOrFanout(ctx.graph_view, conv2d.node)) return false;
-  if (!HasSingleFanoutNode(ctx.graph_view, conv2d.node)) return false;
-  if (IsInPreserveSet(ctx, conv2d.node)) return false;
+
+  if (!conv2d.node || !IsConv2D(*conv2d.node) ||               //
+      !NodeIsOnCpu(conv2d.node) ||                             //
+      !HaveSameDataType(batch_norm, conv2d.node) ||            //
+      !IsCpuCompatibleDataType(conv2d.node) ||                 //
+      !IsCpuCompatibleDataFormat(conv2d.node) ||               //
+      HasControlFaninOrFanout(ctx.graph_view, conv2d.node) ||  //
+      !HasSingleFanoutNode(ctx.graph_view, conv2d.node) ||     //
+      IsInPreserveSet(ctx, conv2d.node))
+    return false;
 
   // We successfully found a Conv2D+FusedBatchNorm pattern.
   matched->conv2d = conv2d.node;
-  matched->fused_batch_norm = node;
-  if (!GetNodeAttr(*node, "epsilon", &matched->epsilon).ok()) return false;
+  matched->fused_batch_norm = batch_norm;
+  if (!GetNodeAttr(*batch_norm, "epsilon", &matched->epsilon).ok())
+    return false;
 
   return true;
 }
@@ -283,21 +393,19 @@ bool FindConv2DWithBatchNormAndRelu(const RemapperContext& ctx,
   if (!EigenSupportsContractionOutputKernel()) return false;
 
   // Root of the pattern must be a Relu.
-  if (node == nullptr) return false;
-  if (!IsRelu(*node)) return false;
-  if (!NodeIsOnCpu(node)) return false;
-  if (!IsFloatOrDoubleDataType(node)) return false;
-  if (!NoControlFaninOrFanout(ctx.graph_view, node)) return false;
+  if (!node || !IsRelu(*node) || HasControlFaninOrFanout(ctx.graph_view, node))
+    return false;
 
   // And input to Relu must match Conv2DWithBatchNorm pattern.
   const auto input_port = GraphView::InputPort(node, 0);
   const auto batch_norm = ctx.graph_view.GetRegularFanin(input_port);
 
   Conv2DWithBatchNorm base;
-  if (!FindConv2DWithBatchNorm(ctx, batch_norm.node, &base)) return false;
-  if (!HasSingleFanoutNode(ctx.graph_view, base.fused_batch_norm)) return false;
-  if (!HaveSameDataType(node, base.fused_batch_norm)) return false;
-  if (IsInPreserveSet(ctx, base.fused_batch_norm)) return false;
+  if (!FindConv2DWithBatchNorm(ctx, batch_norm.node, &base) ||
+      !HasSingleFanoutNode(ctx.graph_view, base.fused_batch_norm) ||
+      !HaveSameDataType(node, base.fused_batch_norm) ||
+      IsInPreserveSet(ctx, base.fused_batch_norm))
+    return false;
 
   // We successfully found a Conv2D+FusedBatchNorm+Relu pattern.
   matched->conv2d = base.conv2d;
@@ -355,9 +463,7 @@ bool FindFusedBatchNorm(const RemapperContext& ctx, const NodeDef* node,
   return true;
 }
 
-void CopyConv2DAttributes(const NodeDef* conv2d, NodeDef* fused_conv2d,
-                          const std::vector<string>& fused_ops = {},
-                          int num_args = 1, float epsilon = 0.0) {
+void CopyConv2DAttributes(const NodeDef* conv2d, NodeDef* fused_conv2d) {
   auto* attr = fused_conv2d->mutable_attr();
   auto src_attr = conv2d->attr();
 
@@ -367,53 +473,65 @@ void CopyConv2DAttributes(const NodeDef* conv2d, NodeDef* fused_conv2d,
   (*attr)["dilations"] = src_attr.at("dilations");
   (*attr)["data_format"] = src_attr.at("data_format");
   (*attr)["use_cudnn_on_gpu"] = src_attr.at("use_cudnn_on_gpu");
+}
 
-  auto* fused_ops_attr = (*attr)["fused_ops"].mutable_list();
-  for (const string& fused_op : fused_ops) {
-    fused_ops_attr->add_s(fused_op);
-  }
-
+void SetFusedConv2DAttributes(
+    NodeDef* fused_conv2d, const absl::Span<const absl::string_view> fused_ops,
+    int num_args = 1, float epsilon = 0.0) {
+  auto* attr = fused_conv2d->mutable_attr();
+  SetAttrValue(fused_ops, &(*attr)["fused_ops"]);
   SetAttrValue(num_args, &(*attr)["num_args"]);
-  // Required only for FusedBatchNorm.
-  SetAttrValue(epsilon, &(*attr)["epsilon"]);
+  SetAttrValue(epsilon, &(*attr)["epsilon"]);  // required only for BatchNorm
 }
 
 void AddFusedConv2DNode(
-    const Conv2DWithBiasAdd& matched, GraphDef* optimized_graph,
+    const RemapperContext& ctx, const Conv2DWithBiasAdd& matched,
+    GraphDef* optimized_graph,
     absl::flat_hash_set<const NodeDef*>* invalidated_nodes) {
-  VLOG(2) << "Fuse Conv2D with BiasAdd: bias_add=" << matched.bias_add->name()
+  DCHECK(IsDeviceCompatible(ctx, matched))
+      << "Unsupported fused Conv2D pattern";
+
+  VLOG(2) << "Fuse Conv2D with BiasAdd: "
+          << " bias_add=" << matched.bias_add->name()
           << " conv2d=" << matched.conv2d->name();
 
   NodeDef* fused_conv2d = optimized_graph->add_node();
-  fused_conv2d->set_name(matched.bias_add->name());
   fused_conv2d->set_op(kFusedConv2D);
-  fused_conv2d->set_device(matched.bias_add->device());
+  fused_conv2d->set_name(matched.bias_add->name());
+  fused_conv2d->set_device(matched.conv2d->device());
   fused_conv2d->add_input(matched.conv2d->input(0));    // 0: input
   fused_conv2d->add_input(matched.conv2d->input(1));    // 1: filter
   fused_conv2d->add_input(matched.bias_add->input(1));  // 2: bias
 
-  CopyConv2DAttributes(matched.conv2d, fused_conv2d, {"BiasAdd"});
+  CopyConv2DAttributes(matched.conv2d, fused_conv2d);
+  SetFusedConv2DAttributes(fused_conv2d, {"BiasAdd"});
 
   invalidated_nodes->insert(matched.bias_add);
   invalidated_nodes->insert(matched.conv2d);
 }
 
 void AddFusedConv2DNode(
-    const Conv2DWithBiasAddAndRelu& matched, GraphDef* optimized_graph,
+    const RemapperContext& ctx, const Conv2DWithBiasAddAndRelu& matched,
+    GraphDef* optimized_graph,
     absl::flat_hash_set<const NodeDef*>* invalidated_nodes) {
-  VLOG(2) << "Fuse Conv2D with BiasAdd and Relu: relu=" << matched.relu->name()
+  DCHECK(IsDeviceCompatible(ctx, matched))
+      << "Unsupported fused Conv2D pattern";
+
+  VLOG(2) << "Fuse Conv2D with BiasAdd and Relu: "
+          << " relu=" << matched.relu->name()
           << " bias_add=" << matched.bias_add->name()
           << " conv2d=" << matched.conv2d->name();
 
   NodeDef* fused_conv2d = optimized_graph->add_node();
   fused_conv2d->set_name(matched.relu->name());
   fused_conv2d->set_op(kFusedConv2D);
-  fused_conv2d->set_device(matched.relu->device());
+  fused_conv2d->set_device(matched.conv2d->device());
   fused_conv2d->add_input(matched.conv2d->input(0));    // 0: input
   fused_conv2d->add_input(matched.conv2d->input(1));    // 1: filter
   fused_conv2d->add_input(matched.bias_add->input(1));  // 2: bias
 
-  CopyConv2DAttributes(matched.conv2d, fused_conv2d, {"BiasAdd", "Relu"});
+  CopyConv2DAttributes(matched.conv2d, fused_conv2d);
+  SetFusedConv2DAttributes(fused_conv2d, {"BiasAdd", "Relu"});
 
   invalidated_nodes->insert(matched.relu);
   invalidated_nodes->insert(matched.bias_add);
@@ -421,8 +539,12 @@ void AddFusedConv2DNode(
 }
 
 void AddFusedConv2DNode(
-    const Conv2DWithSqueezeAndBiasAdd& matched, GraphDef* optimized_graph,
+    const RemapperContext& ctx, const Conv2DWithSqueezeAndBiasAdd& matched,
+    GraphDef* optimized_graph,
     absl::flat_hash_set<const NodeDef*>* invalidated_nodes) {
+  DCHECK(IsDeviceCompatible(ctx, matched))
+      << "Unsupported fused Conv2D pattern";
+
   VLOG(2) << "Fuse Conv2D with Squeeze and BiasAdd: "
           << " bias_add=" << matched.bias_add->name()
           << " squeeze=" << matched.squeeze->name()
@@ -432,13 +554,14 @@ void AddFusedConv2DNode(
   // has single consumer (only the squeeze node).
   NodeDef* fused_conv2d = optimized_graph->add_node();
   fused_conv2d->set_name(matched.conv2d->name());
-  fused_conv2d->set_op("_FusedConv2D");
+  fused_conv2d->set_op(kFusedConv2D);
   fused_conv2d->set_device(matched.conv2d->device());
   fused_conv2d->add_input(matched.conv2d->input(0));    // 0: input
   fused_conv2d->add_input(matched.conv2d->input(1));    // 1: filter
   fused_conv2d->add_input(matched.bias_add->input(1));  // 2: bias
 
-  CopyConv2DAttributes(matched.conv2d, fused_conv2d, {"BiasAdd"});
+  CopyConv2DAttributes(matched.conv2d, fused_conv2d);
+  SetFusedConv2DAttributes(fused_conv2d, {"BiasAdd"});
 
   // Replace BiasAdd node with a Squeeze.
   NodeDef* remapped_squeeze = optimized_graph->add_node();
@@ -461,7 +584,7 @@ void AddFusedConv2DNode(
   NodeDef* fused_conv2d = optimized_graph->add_node();
   fused_conv2d->set_name(matched.fused_batch_norm->name());
   fused_conv2d->set_op(kFusedConv2D);
-  fused_conv2d->set_device(matched.fused_batch_norm->device());
+  fused_conv2d->set_device(matched.conv2d->device());
   fused_conv2d->add_input(matched.conv2d->input(0));            // 0: input
   fused_conv2d->add_input(matched.conv2d->input(1));            // 1: filter
   fused_conv2d->add_input(matched.fused_batch_norm->input(1));  // 2: scale
@@ -469,8 +592,9 @@ void AddFusedConv2DNode(
   fused_conv2d->add_input(matched.fused_batch_norm->input(3));  // 4: mean
   fused_conv2d->add_input(matched.fused_batch_norm->input(4));  // 5: variance
 
-  CopyConv2DAttributes(matched.conv2d, fused_conv2d, {"FusedBatchNorm"},
-                       /*num_args*/ 4, /*epsilon*/ matched.epsilon);
+  CopyConv2DAttributes(matched.conv2d, fused_conv2d);
+  SetFusedConv2DAttributes(fused_conv2d, {"FusedBatchNorm"},
+                           /*num_args=*/4, /*epsilon=*/matched.epsilon);
 
   invalidated_nodes->insert(matched.fused_batch_norm);
   invalidated_nodes->insert(matched.conv2d);
@@ -487,7 +611,7 @@ void AddFusedConv2DNode(
   NodeDef* fused_conv2d = optimized_graph->add_node();
   fused_conv2d->set_name(matched.relu->name());
   fused_conv2d->set_op(kFusedConv2D);
-  fused_conv2d->set_device(matched.fused_batch_norm->device());
+  fused_conv2d->set_device(matched.conv2d->device());
   fused_conv2d->add_input(matched.conv2d->input(0));            // 0: input
   fused_conv2d->add_input(matched.conv2d->input(1));            // 1: filter
   fused_conv2d->add_input(matched.fused_batch_norm->input(1));  // 2: scale
@@ -495,8 +619,9 @@ void AddFusedConv2DNode(
   fused_conv2d->add_input(matched.fused_batch_norm->input(3));  // 4: mean
   fused_conv2d->add_input(matched.fused_batch_norm->input(4));  // 5: variance
 
-  CopyConv2DAttributes(matched.conv2d, fused_conv2d, {"FusedBatchNorm", "Relu"},
-                       /*num_args*/ 4, /*epsilon*/ matched.epsilon);
+  CopyConv2DAttributes(matched.conv2d, fused_conv2d);
+  SetFusedConv2DAttributes(fused_conv2d, {"FusedBatchNorm", "Relu"},
+                           /*num_args=*/4, /*epsilon=*/matched.epsilon);
 
   invalidated_nodes->insert(matched.relu);
   invalidated_nodes->insert(matched.fused_batch_norm);
@@ -680,13 +805,14 @@ Status Remapper::Optimize(Cluster* /*cluster*/, const GrapplerItem& item,
 
     // Remap Conv2D+BiasAdd into the _FusedConv2D.
     if (FindConv2DWithBias(ctx, &node, &conv2d_with_bias)) {
-      AddFusedConv2DNode(conv2d_with_bias, optimized_graph, &invalidated_nodes);
+      AddFusedConv2DNode(ctx, conv2d_with_bias, optimized_graph,
+                         &invalidated_nodes);
       continue;
     }
 
     // Remap Conv2D+BiasAdd+Relu into the _FusedConv2D.
     if (FindConv2DWithBiasAndRelu(ctx, &node, &conv2d_with_bias_and_relu)) {
-      AddFusedConv2DNode(conv2d_with_bias_and_relu, optimized_graph,
+      AddFusedConv2DNode(ctx, conv2d_with_bias_and_relu, optimized_graph,
                          &invalidated_nodes);
       continue;
     }
@@ -694,7 +820,7 @@ Status Remapper::Optimize(Cluster* /*cluster*/, const GrapplerItem& item,
     // Remap Conv2D+Squeeze+BiasAdd into the _FusedConv2D+Squeeze.
     if (FindConv2DWithSqueezeAndBias(ctx, &node,
                                      &conv2d_with_squeeze_and_bias)) {
-      AddFusedConv2DNode(conv2d_with_squeeze_and_bias, optimized_graph,
+      AddFusedConv2DNode(ctx, conv2d_with_squeeze_and_bias, optimized_graph,
                          &invalidated_nodes);
       continue;
     }
diff --git a/tensorflow/core/grappler/utils.cc b/tensorflow/core/grappler/utils.cc
index cc9a38b2d2..375c3e56c8 100644
--- a/tensorflow/core/grappler/utils.cc
+++ b/tensorflow/core/grappler/utils.cc
@@ -20,6 +20,8 @@ limitations under the License.
 #include <queue>
 #include <vector>
 
+#include "absl/strings/match.h"
+#include "absl/strings/str_cat.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/function.h"
 #include "tensorflow/core/framework/node_def_util.h"
@@ -166,10 +168,10 @@ string AddPrefixToNodeName(const string& name, const string& prefix,
                            const string& delimiter) {
   if (!name.empty()) {
     if (name[0] == '^') {
-      return strings::StrCat("^", prefix, delimiter, name.substr(1));
+      return absl::StrCat("^", prefix, delimiter, name.substr(1));
     }
   }
-  return strings::StrCat(prefix, delimiter, name);
+  return absl::StrCat(prefix, delimiter, name);
 }
 
 string AddPrefixToNodeName(const string& name, const string& prefix) {
@@ -193,20 +195,26 @@ bool ExecuteWithTimeout(std::function<void()> fn, const int64 timeout_in_ms,
 }
 
 string AsControlDependency(const NodeDef& node) {
-  return strings::StrCat("^", node.name());
+  return absl::StrCat("^", node.name());
 }
 
 string AsControlDependency(const string& node_name) {
   CHECK(!node_name.empty());
   return (!node_name.empty() && node_name[0] == '^')
              ? node_name
-             : strings::StrCat("^", node_name);
+             : absl::StrCat("^", node_name);
 }
 
 bool NodeIsOnCpu(const NodeDef* node) {
   string task, device;
   return DeviceNameUtils::SplitDeviceName(node->device(), &task, &device) &&
-         str_util::StartsWith(device, DEVICE_CPU);
+         absl::StartsWith(device, DEVICE_CPU);
+}
+
+bool NodeIsOnGpu(const NodeDef* node) {
+  string task, device;
+  return DeviceNameUtils::SplitDeviceName(node->device(), &task, &device) &&
+         absl::StartsWith(device, DEVICE_GPU);
 }
 
 int NumOutputs(const NodeDef& node, GraphDef* graph) {
diff --git a/tensorflow/core/grappler/utils.h b/tensorflow/core/grappler/utils.h
index 1e820977ae..9053ae4c07 100644
--- a/tensorflow/core/grappler/utils.h
+++ b/tensorflow/core/grappler/utils.h
@@ -242,6 +242,9 @@ string AsControlDependency(const string& node);
 // Returns true if the node is assigned to run on CPU device.
 bool NodeIsOnCpu(const NodeDef* node);
 
+// Returns true if the node is assigned to run on GPU device.
+bool NodeIsOnGpu(const NodeDef* node);
+
 // Returns the number of outputs of a node according to its OpDef. Note that
 // some of the outputs may be unconnected.
 int NumOutputs(const NodeDef& node, GraphDef* graph);
diff --git a/tensorflow/core/grappler/utils/BUILD b/tensorflow/core/grappler/utils/BUILD
index 89417f85c2..ff15541cdc 100644
--- a/tensorflow/core/grappler/utils/BUILD
+++ b/tensorflow/core/grappler/utils/BUILD
@@ -143,6 +143,7 @@ cc_library(
         "//tensorflow/core:test",
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:utils",
+        "@com_google_absl//absl/algorithm:container",
     ],
 )
 
diff --git a/tensorflow/core/grappler/utils/grappler_test.cc b/tensorflow/core/grappler/utils/grappler_test.cc
index 576494cad5..5ef1cf444b 100644
--- a/tensorflow/core/grappler/utils/grappler_test.cc
+++ b/tensorflow/core/grappler/utils/grappler_test.cc
@@ -15,6 +15,8 @@ limitations under the License.
 
 #include "tensorflow/core/grappler/utils/grappler_test.h"
 #include <memory>
+#include "absl/algorithm/container.h"
+#include "tensorflow/core/framework/attr_value_util.h"
 #include "tensorflow/core/grappler/utils.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/protobuf/rewriter_config.pb.h"
@@ -125,6 +127,31 @@ void GrapplerTest::CompareGraphs(GraphDef want, GraphDef got) const {
   }
 }
 
+void GrapplerTest::CompareNodes(const NodeDef& want, const NodeDef& got) const {
+  EXPECT_EQ(want.name(), got.name());
+  EXPECT_EQ(want.op(), got.op());
+
+  std::vector<string> want_inputs(want.input().begin(), want.input().end());
+  std::vector<string> got_inputs(got.input().begin(), got.input().end());
+  EXPECT_EQ(want_inputs, got_inputs);
+
+  const auto attr_name = [](const std::pair<const string, AttrValue>& attr) {
+    return attr.first;
+  };
+
+  std::vector<string> want_attrs;
+  std::vector<string> got_attrs;
+  absl::c_transform(want.attr(), std::back_inserter(want_attrs), attr_name);
+  absl::c_transform(got.attr(), std::back_inserter(got_attrs), attr_name);
+  absl::c_sort(want_attrs);
+  absl::c_sort(got_attrs);
+  EXPECT_EQ(want_attrs, got_attrs);
+
+  for (const string& attr : want_attrs) {
+    EXPECT_TRUE(AreAttrValuesEqual(want.attr().at(attr), got.attr().at(attr)));
+  }
+}
+
 bool GrapplerTest::IsNodesDirectlyConnected(const NodeMap& node_map,
                                             const string& src,
                                             const string& dst, int position) {
diff --git a/tensorflow/core/grappler/utils/grappler_test.h b/tensorflow/core/grappler/utils/grappler_test.h
index 0cfd740dcb..20fd04c1c6 100644
--- a/tensorflow/core/grappler/utils/grappler_test.h
+++ b/tensorflow/core/grappler/utils/grappler_test.h
@@ -49,13 +49,25 @@ class GrapplerTest : public ::testing::Test {
                    const std::vector<std::pair<string, AttrValue>>& attributes,
                    GraphDef* graph) const;
 
+  // Checks if two graphs are equal. Both graphs must have the same set of nodes
+  // with the same inputs and attributes. Nodes can be in different order.
+  //
+  // NOTE: This function uses EXPECT/ASSERT macros to check node properties
+  // equality, and adds all failuires to the current test.
   void CompareGraphs(GraphDef want, GraphDef got) const;
 
-  // Check if node 'src' is directly connected to the input($position) of 'dst'.
+  // Checks if two nodes have the same name, op, inputs and attributes.
+  //
+  // NOTE: This function uses EXPECT/ASSERT macros to check node properties
+  // equality, and adds all failuires to the current test.
+  void CompareNodes(const NodeDef& want, const NodeDef& got) const;
+
+  // Checks if node 'src' is directly connected to the input($position) of
+  // 'dst'.
   bool IsNodesDirectlyConnected(const NodeMap& node_map, const string& src,
                                 const string& dst, int position = 0);
 
-  // Count nodes of the given op-type in a graph.
+  // Counts nodes of the given op-type in a graph.
   int CountOpNodes(const GraphDef& graph, const string& op);
 
   // Get a random tensor with given shape.
diff --git a/tensorflow/core/kernels/conv_ops_test.cc b/tensorflow/core/kernels/conv_ops_test.cc
index 3e59219f8f..fc93915e16 100644
--- a/tensorflow/core/kernels/conv_ops_test.cc
+++ b/tensorflow/core/kernels/conv_ops_test.cc
@@ -1497,6 +1497,26 @@ BM_FusedConv2DWithBatchNormAndRelu(32, 32, 32, 128, 3, 3, 1024, cpu,
                                    "3x3 /b 32");
 
 #if GOOGLE_CUDA
+// -------------------------------------------------------------------------- //
+// 1x1 Convolution
+// -------------------------------------------------------------------------- //
+
+BM_Conv2D(8, 32, 32, 128, 1, 1, 1024, gpu, "1x1 /b 8");
+BM_Conv2D(16, 32, 32, 128, 1, 1, 1024, gpu, "1x1 /b 16");
+BM_Conv2D(32, 32, 32, 128, 1, 1, 1024, gpu, "1x1 /b 32");
+
+BM_Conv2DWithBiasAndRelu(8, 32, 32, 128, 1, 1, 1024, gpu, "1x1 /b 8");
+BM_Conv2DWithBiasAndRelu(16, 32, 32, 128, 1, 1, 1024, gpu, "1x1 /b 16");
+BM_Conv2DWithBiasAndRelu(32, 32, 32, 128, 1, 1, 1024, gpu, "1x1 /b 32");
+
+BM_FusedConv2DWithBiasAndRelu(8, 32, 32, 128, 1, 1, 1024, gpu, "1x1 /b 8");
+BM_FusedConv2DWithBiasAndRelu(16, 32, 32, 128, 1, 1, 1024, gpu, "1x1 /b 16");
+BM_FusedConv2DWithBiasAndRelu(32, 32, 32, 128, 1, 1, 1024, gpu, "1x1 /b 32");
+
+// -------------------------------------------------------------------------- //
+// 3x3 Convolution
+// -------------------------------------------------------------------------- //
+
 BM_Conv2D(8, 32, 32, 128, 3, 3, 1024, gpu, "3x3 /b 8");
 BM_Conv2D(16, 32, 32, 128, 3, 3, 1024, gpu, "3x3 /b 16");
 BM_Conv2D(32, 32, 32, 128, 3, 3, 1024, gpu, "3x3 /b 32");
-- 
GitLab


From c7c4a42c4372ca560ea415fe3a798e18286cedec Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 7 Jan 2019 12:36:38 -0800
Subject: [PATCH 0317/1095] Fix an error in keras input_layer.Input() dtype
 type checking.

PiperOrigin-RevId: 228215444
---
 tensorflow/python/keras/engine/input_layer.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/keras/engine/input_layer.py b/tensorflow/python/keras/engine/input_layer.py
index bc2cf2fb6e..c6dcedfce2 100644
--- a/tensorflow/python/keras/engine/input_layer.py
+++ b/tensorflow/python/keras/engine/input_layer.py
@@ -77,8 +77,9 @@ class InputLayer(base_layer.Layer):
         dtype = backend.floatx()
       else:
         dtype = backend.dtype(input_tensor)
-    elif input_tensor and input_tensor.dtype != dtype:
-      raise ValueError('`input_tensor.dtype` differs from `dtype`.')
+    elif input_tensor is not None and input_tensor.dtype != dtype:
+      raise ValueError('`input_tensor.dtype` differs from `dtype`: %s vs. %s' %
+                       (input_tensor.dtype, dtype))
     super(InputLayer, self).__init__(dtype=dtype, name=name)
     self.built = True
     self.sparse = sparse
-- 
GitLab


From e7f7f01352be4054a7fb2a872cf2ef335292600c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 7 Jan 2019 12:48:37 -0800
Subject: [PATCH 0318/1095] Prevent the interpreter from continuing execution
 after an error occurred.

PiperOrigin-RevId: 228217675
---
 tensorflow/lite/core/subgraph.cc | 6 +++---
 tensorflow/lite/interpreter.cc   | 7 ++++---
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/tensorflow/lite/core/subgraph.cc b/tensorflow/lite/core/subgraph.cc
index 75212cac22..4be80d143e 100644
--- a/tensorflow/lite/core/subgraph.cc
+++ b/tensorflow/lite/core/subgraph.cc
@@ -670,7 +670,7 @@ TfLiteStatus Subgraph::Invoke() {
       TfLiteTensor* tensor = &tensors_[tensor_index];
       if (tensor->delegate && tensor->delegate != node.delegate &&
           tensor->data_is_stale) {
-        EnsureTensorDataIsReadable(tensor_index);
+        TF_LITE_ENSURE_STATUS(EnsureTensorDataIsReadable(tensor_index));
       }
     }
 
@@ -683,8 +683,8 @@ TfLiteStatus Subgraph::Invoke() {
     EnsureTensorsVectorCapacity();
     tensor_resized_since_op_invoke_ = false;
     if (OpInvoke(registration, &node) == kTfLiteError) {
-      status = ReportOpError(context_, node, registration, node_index,
-                             "failed to invoke");
+      return ReportOpError(context_, node, registration, node_index,
+                           "failed to invoke");
     }
 
     // Force execution prep for downstream ops if the latest op triggered the
diff --git a/tensorflow/lite/interpreter.cc b/tensorflow/lite/interpreter.cc
index 2abe062ec6..60fa2130fa 100644
--- a/tensorflow/lite/interpreter.cc
+++ b/tensorflow/lite/interpreter.cc
@@ -102,15 +102,16 @@ TfLiteStatus Interpreter::ResizeInputTensor(int tensor_index,
 }
 
 TfLiteStatus Interpreter::Invoke() {
-  TfLiteStatus status = primary_subgraph().Invoke();
+  TF_LITE_ENSURE_STATUS(primary_subgraph().Invoke());
 
   if (!allow_buffer_handle_output_) {
     for (int tensor_index : outputs()) {
-      primary_subgraph().EnsureTensorDataIsReadable(tensor_index);
+      TF_LITE_ENSURE_STATUS(
+          primary_subgraph().EnsureTensorDataIsReadable(tensor_index));
     }
   }
 
-  return status;
+  return kTfLiteOk;
 }
 
 TfLiteStatus Interpreter::AddTensors(int tensors_to_add,
-- 
GitLab


From 95d708d7b9e3f22c09ca77004e68f2cadb295226 Mon Sep 17 00:00:00 2001
From: Shanqing Cai <cais@google.com>
Date: Mon, 7 Jan 2019 13:12:01 -0800
Subject: [PATCH 0319/1095] tf.keras: Remove unnecessary backend check in
 PReLU.call()

PiperOrigin-RevId: 228222042
---
 tensorflow/python/keras/layers/advanced_activations.py | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/tensorflow/python/keras/layers/advanced_activations.py b/tensorflow/python/keras/layers/advanced_activations.py
index be1039a2ac..5095287430 100644
--- a/tensorflow/python/keras/layers/advanced_activations.py
+++ b/tensorflow/python/keras/layers/advanced_activations.py
@@ -121,11 +121,9 @@ class PReLU(Layer):
   @tf_utils.shape_type_conversion
   def build(self, input_shape):
     param_shape = list(input_shape[1:])
-    self.param_broadcast = [False] * len(param_shape)
     if self.shared_axes is not None:
       for i in self.shared_axes:
         param_shape[i - 1] = 1
-        self.param_broadcast[i - 1] = True
     self.alpha = self.add_weight(
         shape=param_shape,
         name='alpha',
@@ -143,12 +141,7 @@ class PReLU(Layer):
 
   def call(self, inputs, mask=None):
     pos = K.relu(inputs)
-    if K.backend() == 'theano':
-      neg = (
-          K.pattern_broadcast(self.alpha, self.param_broadcast) *
-          (inputs - math_ops.abs(inputs)) * 0.5)
-    else:
-      neg = -self.alpha * K.relu(-inputs)
+    neg = -self.alpha * K.relu(-inputs)
     return pos + neg
 
   def get_config(self):
-- 
GitLab


From 38491a84a9e38357e400457dbbe408b66e786672 Mon Sep 17 00:00:00 2001
From: Pooya Davoodi <pdavoodi@nvidia.com>
Date: Mon, 7 Jan 2019 14:21:21 -0800
Subject: [PATCH 0320/1095] TFTRT: Expand lambda and inline ifs to functions
 with if and else

---
 .../contrib/tensorrt/python/trt_convert.py    | 36 ++++++++++++++-----
 1 file changed, 27 insertions(+), 9 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/python/trt_convert.py b/tensorflow/contrib/tensorrt/python/trt_convert.py
index 3d2b6b499d..f5d624ef64 100644
--- a/tensorflow/contrib/tensorrt/python/trt_convert.py
+++ b/tensorflow/contrib/tensorrt/python/trt_convert.py
@@ -45,15 +45,33 @@ from tensorflow.python.saved_model import loader_impl
 from tensorflow.python.saved_model import tag_constants
 from tensorflow.python.training import saver
 
-if _six.PY2:
-  _to_bytes = lambda s: s.encode("utf-8", errors="surrogateescape") \
-    if isinstance(s, unicode) else s
-  _to_string = lambda s: s.decode("utf-8") if isinstance(s, str) else s
-else:
-  _to_bytes = lambda s: s.encode("utf-8", errors="surrogateescape") \
-    if isinstance(s, str) else s
-  _to_string = lambda s: s.decode("utf-8") if isinstance(s, bytes) else s
-
+def _to_bytes(s):
+    """Returns encoded of s if s is a sequence of chars otherwise returns s.
+    """
+    if _six.PY2:
+        if isinstance(s, unicode):
+            return s.encode("utf-8", errors="surrogateescape")
+        else:
+            return s
+    else:
+        if isinstance(s, str):
+            return s.encode("utf-8", errors="surrogateescape")
+        else:
+            return s
+
+def _to_string(s):
+    """Returns decoded of s if s is a sequence of bytes otherwise returns s.
+    """
+    if _six.PY2:
+        if isinstance(s, str):
+            return s.decode("utf-8")
+        else:
+            return s
+    else:
+        if isinstance(s, bytes):
+            return s.decode("utf-8")
+        else:
+            return s
 
 class TrtPrecisionMode(object):
   FP32 = "FP32"
-- 
GitLab


From 46f030b6e275bd9950da91e480bda5f2fa44ad80 Mon Sep 17 00:00:00 2001
From: Michael Case <mikecase@google.com>
Date: Mon, 7 Jan 2019 13:26:16 -0800
Subject: [PATCH 0321/1095] Clean up example/learn/BUILD a bit

Some of the data deps are targets that no longer exist. Pretty sure
should be cleaned up.

PiperOrigin-RevId: 228224240
---
 tensorflow/examples/learn/BUILD | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/tensorflow/examples/learn/BUILD b/tensorflow/examples/learn/BUILD
index d6ec1f393b..a22d55e5af 100644
--- a/tensorflow/examples/learn/BUILD
+++ b/tensorflow/examples/learn/BUILD
@@ -28,17 +28,8 @@ sh_test(
     size = "large",
     srcs = ["examples_test.sh"],
     data = [
-        ":boston",
-        ":iris",
         ":iris_custom_decay_dnn",
         ":iris_custom_model",
-        ":iris_run_config",
-        ":random_forest_mnist",
-        ":resnet",
-        ":text_classification",
-        ":text_classification_character_cnn",
-        ":text_classification_character_rnn",
-        ":text_classification_cnn",
     ],
     tags = [
         "manual",
-- 
GitLab


From d2dd3bd43ebbfd2d14f2f7e658b5b5aae201607e Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Mon, 7 Jan 2019 13:35:32 -0800
Subject: [PATCH 0322/1095] while_v2 fixes for better TPU integration.

PiperOrigin-RevId: 228226029
---
 tensorflow/python/ops/while_v2.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/while_v2.py b/tensorflow/python/ops/while_v2.py
index aaed86c1ca..f9fea0e5bd 100644
--- a/tensorflow/python/ops/while_v2.py
+++ b/tensorflow/python/ops/while_v2.py
@@ -115,12 +115,15 @@ def while_loop(cond,
             loop_counter < maximum_iterations,
             cond(*_pack_sequence_as(orig_loop_vars, args)))
 
+    # NOTE(skyewm): we set read_only_collections=False for compatibility with
+    # TPUEstimator.
     cond_graph = func_graph_module.func_graph_from_py_func(
         cond_name,
         wrapped_cond,
         loop_vars, {},
         signature=_build_signature(loop_vars, shape_invariants),
-        func_graph=util.WhileCondFuncGraph(cond_name),
+        func_graph=util.WhileCondFuncGraph(cond_name,
+                                           read_only_collections=False),
         add_control_dependencies=add_control_dependencies)
 
     # Add external_captures of cond to the list of loop vars.
@@ -171,7 +174,8 @@ def while_loop(cond,
         wrapped_body,
         loop_vars, {},
         signature=_build_signature(loop_vars, shape_invariants),
-        func_graph=util.WhileBodyFuncGraph(body_name),
+        func_graph=util.WhileBodyFuncGraph(body_name,
+                                           read_only_collections=False),
         add_control_dependencies=add_control_dependencies)
     # Add external captures of body to the list of loop vars.
     # Note that external tensors will be treated as loop invariants, i.e.,
-- 
GitLab


From 879af3c38d2405db3ff4c96d6104cdf82a1ab368 Mon Sep 17 00:00:00 2001
From: Frank Chen <frankchn@google.com>
Date: Mon, 7 Jan 2019 13:38:36 -0800
Subject: [PATCH 0323/1095] Change dataset rewriting in distribution strategy
 to avoid a unbatch().batch() call.

PiperOrigin-RevId: 228226620
---
 .../python/data/experimental/ops/batching.py  |  8 ++++++--
 tensorflow/python/distribute/input_lib.py     | 20 ++++++++++++++++---
 2 files changed, 23 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/data/experimental/ops/batching.py b/tensorflow/python/data/experimental/ops/batching.py
index f0cf7f0a99..ff8a9182c3 100644
--- a/tensorflow/python/data/experimental/ops/batching.py
+++ b/tensorflow/python/data/experimental/ops/batching.py
@@ -553,8 +553,12 @@ class _MapAndBatchDataset(dataset_ops.UnaryDataset):
     """See `Dataset.map()` for details."""
     self._input_dataset = input_dataset
 
-    self._map_func = dataset_ops.StructuredFunctionWrapper(
-        map_func, "tf.data.experimental.map_and_batch()", dataset=input_dataset)
+    if isinstance(map_func, dataset_ops.StructuredFunctionWrapper):
+      self._map_func = map_func
+    else:
+      self._map_func = dataset_ops.StructuredFunctionWrapper(
+          map_func, "tf.data.experimental.map_and_batch()",
+          dataset=input_dataset)
     self._batch_size_t = ops.convert_to_tensor(
         batch_size, dtype=dtypes.int64, name="batch_size")
     self._num_parallel_calls_t = ops.convert_to_tensor(
diff --git a/tensorflow/python/distribute/input_lib.py b/tensorflow/python/distribute/input_lib.py
index cbe6518e5c..022595146d 100644
--- a/tensorflow/python/distribute/input_lib.py
+++ b/tensorflow/python/distribute/input_lib.py
@@ -574,12 +574,19 @@ def _split_dataset_batch(dataset, split_batch_by):
         "The batch operations can be followed by a prefetch.")
 
   batched_dataset = _get_batch_dataset(dataset)
+  prev_dataset = batched_dataset._input_dataset
+
+  num_parallel_calls = None
+  map_func = None
+
   if isinstance(batched_dataset, dataset_ops.BatchDataset):
     batch_size = batched_dataset._batch_size
     drop_remainder = batched_dataset._drop_remainder
   elif isinstance(batched_dataset, batching._MapAndBatchDataset):
     batch_size = batched_dataset._batch_size_t
     drop_remainder = batched_dataset._drop_remainder_t
+    num_parallel_calls = batched_dataset._num_parallel_calls_t
+    map_func = batched_dataset._map_func
 
   prefetch_buffer = None
   if isinstance(dataset, dataset_ops.PrefetchDataset):
@@ -587,7 +594,6 @@ def _split_dataset_batch(dataset, split_batch_by):
   elif (isinstance(dataset, dataset_ops.DatasetV1Adapter)
         and isinstance(dataset._dataset, dataset_ops.PrefetchDataset)):
     prefetch_buffer = dataset._dataset._buffer_size
-  # pylint: enable=protected-access
 
   if tensor_util.is_tensor(batch_size):
     batch_size = tensor_util.constant_value(batch_size)
@@ -595,14 +601,22 @@ def _split_dataset_batch(dataset, split_batch_by):
   if tensor_util.is_tensor(drop_remainder):
     drop_remainder = tensor_util.constant_value(drop_remainder)
 
+  if num_parallel_calls is not None and tensor_util.is_tensor(drop_remainder):
+    num_parallel_calls = tensor_util.constant_value(num_parallel_calls)
+
   if batch_size % split_batch_by:
     raise ValueError(
         "Batch size %s cannot be sharded evenly across replicas %s" % (
             batch_size, split_batch_by))
   new_batch_size = batch_size // split_batch_by
 
-  dataset = dataset.apply(batching.unbatch())
-  dataset = dataset.batch(new_batch_size, drop_remainder=drop_remainder)
+  if isinstance(batched_dataset, dataset_ops.BatchDataset):
+    dataset = prev_dataset.batch(new_batch_size, drop_remainder=drop_remainder)
+  elif isinstance(batched_dataset, batching._MapAndBatchDataset):
+    dataset = prev_dataset.apply(batching.map_and_batch(
+        map_func, new_batch_size, num_parallel_calls, drop_remainder))
+  # pylint: enable=protected-access
+
   if prefetch_buffer is not None:
     dataset = dataset.prefetch(prefetch_buffer)
   return dataset
-- 
GitLab


From 7bc843afd6696fc802fa44f65f0fdf67b0bc56e6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 7 Jan 2019 13:45:54 -0800
Subject: [PATCH 0324/1095] Unittests to demonstrate three bugs in flex mode: 
 - forwardability is a property of a subgraph, not of the global buffer map  -
 input tensors should never be forwardable, even when they    come from
 another subgraph. That's because they might be outputs of the    model, or
 inputs into another subgraph.  - input tensors shouldn't be removed at the
 end, because they might be    used by another subgraph.

PiperOrigin-RevId: 228228094
---
 tensorflow/lite/delegates/flex/kernel_test.cc | 175 +++++++++++++++++-
 tensorflow/lite/delegates/flex/test_util.cc   |   5 +
 tensorflow/lite/delegates/flex/test_util.h    |   3 +
 3 files changed, 179 insertions(+), 4 deletions(-)

diff --git a/tensorflow/lite/delegates/flex/kernel_test.cc b/tensorflow/lite/delegates/flex/kernel_test.cc
index 647c199d4f..9de3ca67de 100644
--- a/tensorflow/lite/delegates/flex/kernel_test.cc
+++ b/tensorflow/lite/delegates/flex/kernel_test.cc
@@ -36,13 +36,38 @@ TfLiteStatus GenericPrepare(TfLiteContext* context, TfLiteDelegate* delegate,
   return kTfLiteOk;
 }
 
+// There is no easy way to pass a parameter into the TfLiteDelegate's
+// 'prepare' function, so we keep a global map for testing purpused.
+// To avoid collisions use: GetPrepareFunction<__LINE__>().
+std::map<int, std::vector<int>>* GetGlobalOpLists() {
+  static auto* op_list = new std::map<int, std::vector<int>>;
+  return op_list;
+}
+
 class KernelTest : public testing::FlexModelTest {
  public:
+  static constexpr int kOnes = 1;  // This is the index of a tensor of 1's.
+  static constexpr int kTwos = 2;  // This is the index of a tensor of 2's.
+  static constexpr int kMaxTensors = 30;
+
+  static void SetUpTestSuite() { GetGlobalOpLists()->clear(); }
+
   KernelTest() {
     CHECK(delegate_data_.Prepare(tensorflow::SessionOptions{}).ok());
     interpreter_.reset(new Interpreter(&error_reporter_));
   }
 
+  typedef TfLiteStatus (*PrepareFunction)(TfLiteContext* context,
+                                          TfLiteDelegate* delegate);
+
+  template <int KEY>
+  PrepareFunction GetPrepareFunction() {
+    GetGlobalOpLists()->insert({KEY, tf_ops_});
+    return [](TfLiteContext* context, TfLiteDelegate* delegate) {
+      return GenericPrepare(context, delegate, GetGlobalOpLists()->at(KEY));
+    };
+  }
+
   template <typename T>
   void ConfigureDelegate(T prepare_function) {
     delegate_.data_ = &delegate_data_;
@@ -54,9 +79,13 @@ class KernelTest : public testing::FlexModelTest {
                                         TfLiteBufferHandle buffer_handle,
                                         TfLiteTensor* output) {
       auto* delegate_data = reinterpret_cast<DelegateData*>(delegate->data_);
-      tensorflow::StringPiece values = delegate_data->GetBufferMap(context)
-                                           ->GetTensor(buffer_handle)
-                                           .tensor_data();
+      auto* buffer_map = delegate_data->GetBufferMap(context);
+      if (!buffer_map->HasTensor(buffer_handle)) {
+        context->ReportError(context, "Tensor '%d' not found", buffer_handle);
+        return kTfLiteError;
+      }
+      tensorflow::StringPiece values =
+          buffer_map->GetTensor(buffer_handle).tensor_data();
       memcpy(output->data.raw, values.data(), values.size());
       return kTfLiteOk;
     };
@@ -225,7 +254,7 @@ TEST_F(KernelTest, SplitGraph) {
   AddTfOp(testing::kAdd, {9, 16}, {17});  // => 16
 
   ConfigureDelegate([](TfLiteContext* context, TfLiteDelegate* delegate) {
-    // All ops by #3 are TF ops, handled by the delegate. However, because #4
+    // All ops but #3 are TF ops, handled by the delegate. However, because #4
     // depends on the non-TF op, two subgraphs are necessary:
     //    TF subgraph 1: 0, 1, 2, 6, 7, 8, 9
     //    TF Lite Op: 3
@@ -260,6 +289,144 @@ TEST_F(KernelTest, SplitGraph) {
   ASSERT_THAT(GetValues(17), ElementsAre(18.0f));
 }
 
+class MultipleSubgraphsTest : public KernelTest {
+ public:
+  static constexpr int kInput = 0;
+
+  void PrepareInterpreter(PrepareFunction prepare,
+                          const std::vector<float>& input) {
+    ConfigureDelegate(prepare);
+
+    SetShape(kOnes, {3});
+    SetValues(kOnes, {1.0f, 1.0f, 1.0f});
+    SetShape(kTwos, {3});
+    SetValues(kTwos, {2.0f, 2.0f, 2.0f});
+
+    SetValues(kInput, input);
+  }
+
+  std::vector<float> Apply(const std::vector<float>& input,
+                           std::function<float(float)> function) {
+    std::vector<float> result;
+    for (float f : input) {
+      result.push_back(function(f));
+    }
+    return result;
+  }
+};
+
+TEST_F(MultipleSubgraphsTest, ForwardabilityIsLocal) {
+  AddTensors(kMaxTensors, {kInput, kOnes, kTwos}, {12}, kTfLiteFloat32, {3});
+
+  // Only TF tensors can be forwarded, so we build a small first graph
+  // to produce tensor #10. Here #10 is forwardable, because it is only
+  // used once, as an output.
+  AddTfOp(testing::kAdd, {0, kOnes}, {3});
+  AddTfOp(testing::kAdd, {0, kOnes}, {10});
+
+  // The second TF graph, separated from the former by a TF Lite
+  // multiplication, will consume tensor #10, which is not forwardable here
+  // since it is used by more than one op. The existing code will forward the
+  // tensor anyway, because it was deemed to be forwardable by the previous
+  // subgraph.
+  AddTfLiteMulOp({3, kTwos}, {4});
+  AddTfOp(testing::kAdd, {10, 4}, {11});
+  AddTfOp(testing::kAdd, {11, 10}, {7});
+
+  // And a simple TF Lite op trying to access tensor #10, which was removed
+  // from the buffer map. It will cause Invoke() to fail.
+  AddTfLiteMulOp({10, 7}, {12});
+
+  auto input = {3.0f, 4.0f, 5.0f};
+  PrepareInterpreter(GetPrepareFunction<__LINE__>(), input);
+
+  ASSERT_FALSE(Invoke());
+  ASSERT_THAT(error_reporter().error_messages(),
+              ContainsRegex("Cannot read from invalid tensor index 10"));
+
+  // TODO(b/122457581): expected result for this test:
+  // ASSERT_THAT(GetValues(12), ElementsAreArray(Apply(input, [](float in) {
+  //               return (4 * in + 4) * (in + 1);
+  //             })));
+}
+
+// Subgraphs should not remove input tensors from the buffer_map, since
+// they could be necessary for downstream graphs.
+TEST_F(MultipleSubgraphsTest, DoNotRemoveInputTensors) {
+  AddTensors(kMaxTensors, {kInput, kOnes, kTwos}, {12}, kTfLiteFloat32, {3});
+
+  // Only TF tensors can be removed, so we build a small first graph
+  // to produce tensor #10. We make sure it is used by more than one
+  // op, so it is not forwardable here.
+  AddTfOp(testing::kAdd, {0, kOnes}, {3});
+  AddTfOp(testing::kAdd, {0, kOnes}, {10});
+  AddTfOp(testing::kAdd, {10, kOnes}, {15});
+  AddTfOp(testing::kAdd, {10, kOnes}, {16});
+
+  // The second TF graph, separated from the former by a TF Lite
+  // multiplication, will consume tensor #10. The existing code will remove
+  // from the buffer_map all tensors that are not outputs, so #10 will
+  // disappear. Note that we are using #10 in two ops, so it is not forwardable
+  // either.
+  AddTfLiteMulOp({3, kTwos}, {4});
+  AddTfOp(testing::kAdd, {10, 4}, {11});
+  AddTfOp(testing::kAdd, {10, 11}, {7});
+
+  // And a simple TF Lite op trying to access tensor #10, which was removed
+  // from the buffer map. It will cause Invoke() to fail.
+  AddTfLiteMulOp({10, 7}, {12});
+
+  auto input = {3.0f, 4.0f, 5.0f};
+  PrepareInterpreter(GetPrepareFunction<__LINE__>(), input);
+
+  ASSERT_FALSE(Invoke());
+  ASSERT_THAT(error_reporter().error_messages(),
+              ContainsRegex("Tensor '10' not found"));
+
+  // TODO(b/122457581): expected result for this test:
+  // ASSERT_THAT(GetValues(12), ElementsAreArray(Apply(input, [](float in) {
+  //               return (4 * in + 4) * (in + 1);
+  //             })));
+}
+
+// A tensor is deemed forwardable but it happens to be the input to
+// more than one subgraph. It should not be forwarded, otherwise its
+// contents will be overwritten.
+TEST_F(MultipleSubgraphsTest, DoNotForwardInputTensors) {
+  AddTensors(kMaxTensors, {kInput, kOnes, kTwos}, {12}, kTfLiteFloat32, {3});
+
+  // Only TF tensors can be forwarded, so we build a small first graph
+  // to produce tensor #10.
+  AddTfOp(testing::kAdd, {0, kOnes}, {3});
+  AddTfOp(testing::kAdd, {0, kOnes}, {10});
+
+  // The second TF graph, separated from the former by a TF Lite
+  // multiplication, will consume tensor #10 and will think it is forwardable
+  // because it is used by a single op. However, the subgraph doesn't have
+  // enough information to make that judgment, as the input tensor could be
+  // used by another graph further downstream. The existing code will forward
+  // the tensor and remove it from the buffer_map, causing a failure later.
+  AddTfLiteMulOp({3, kTwos}, {4});
+  AddTfOp(testing::kAdd, {10, 4}, {11});
+  AddTfOp(testing::kAdd, {11, 4}, {7});
+
+  // And a simple TF Lite op trying to access tensor #10, which was removed
+  // from the buffer map. It will cause Invoke() to fail.
+  AddTfLiteMulOp({10, 7}, {12});
+
+  auto input = {3.0f, 4.0f, 5.0f};
+  PrepareInterpreter(GetPrepareFunction<__LINE__>(), input);
+
+  ASSERT_FALSE(Invoke());
+  ASSERT_THAT(error_reporter().error_messages(),
+              ContainsRegex("Tensor '10' not found"));
+
+  // TODO(b/122457581): expected result for this test:
+  // ASSERT_THAT(GetValues(12), ElementsAreArray(Apply(input, [](float in) {
+  //               return (5 * in + 5) * (in + 1);
+  //             })));
+}
+
 }  // namespace
 }  // namespace flex
 }  // namespace tflite
diff --git a/tensorflow/lite/delegates/flex/test_util.cc b/tensorflow/lite/delegates/flex/test_util.cc
index aa24675a7b..a67aeef231 100644
--- a/tensorflow/lite/delegates/flex/test_util.cc
+++ b/tensorflow/lite/delegates/flex/test_util.cc
@@ -90,6 +90,8 @@ void FlexModelTest::AddTensors(int num_tensors, const std::vector<int>& inputs,
 
 void FlexModelTest::AddTfLiteMulOp(const std::vector<int>& inputs,
                                    const std::vector<int>& outputs) {
+  ++next_op_index_;
+
   static TfLiteRegistration reg = {nullptr, nullptr, nullptr, nullptr};
   reg.builtin_code = BuiltinOperator_MUL;
   reg.prepare = [](TfLiteContext* context, TfLiteNode* node) {
@@ -114,6 +116,9 @@ void FlexModelTest::AddTfLiteMulOp(const std::vector<int>& inputs,
 
 void FlexModelTest::AddTfOp(TfOpType op, const std::vector<int>& inputs,
                             const std::vector<int>& outputs) {
+  tf_ops_.push_back(next_op_index_);
+  ++next_op_index_;
+
   auto attr = [](const string& key, const string& value) {
     return " attr{ key: '" + key + "' value {" + value + "}}";
   };
diff --git a/tensorflow/lite/delegates/flex/test_util.h b/tensorflow/lite/delegates/flex/test_util.h
index 2cc2dc30e9..1913a406e8 100644
--- a/tensorflow/lite/delegates/flex/test_util.h
+++ b/tensorflow/lite/delegates/flex/test_util.h
@@ -103,6 +103,7 @@ class FlexModelTest : public ::testing::Test {
  protected:
   std::unique_ptr<Interpreter> interpreter_;
   TestErrorReporter error_reporter_;
+  std::vector<int> tf_ops_;
 
  private:
   // Helper method to add a TensorFlow op. tflite_names needs to start with
@@ -112,6 +113,8 @@ class FlexModelTest : public ::testing::Test {
                const std::vector<int>& outputs);
 
   std::vector<std::vector<uint8_t>> flexbuffers_;
+
+  int next_op_index_ = 0;
 };
 
 }  // namespace testing
-- 
GitLab


From 2d8379badceae7beae681eb2321ed575ba8ec718 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Mon, 7 Jan 2019 14:17:08 -0800
Subject: [PATCH 0325/1095] [TF:XLA] Bump open source llvm revision to r350438

PiperOrigin-RevId: 228234219
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index b43312abb2..834ec28447 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -500,11 +500,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "llvm",
         build_file = clean_dep("//third_party/llvm:llvm.autogenerated.BUILD"),
-        sha256 = "3a1c8a817d2d9a85b85e1ad776a90faa51f475bb924fbf70383892bb1faceac9",
-        strip_prefix = "llvm-6da47bee848304d215e3ca08b2a4a4dc9c954310",
+        sha256 = "20c0ebd54f433d68652a54e328d70b0ec7cf37ce734645a0259cf904d14c2c53",
+        strip_prefix = "llvm-0e2aef924e02d00a4d3454d63c1921a243d316c6",
         urls = [
-            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/6da47bee848304d215e3ca08b2a4a4dc9c954310.tar.gz",
-            "https://github.com/llvm-mirror/llvm/archive/6da47bee848304d215e3ca08b2a4a4dc9c954310.tar.gz",
+            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/0e2aef924e02d00a4d3454d63c1921a243d316c6.tar.gz",
+            "https://github.com/llvm-mirror/llvm/archive/0e2aef924e02d00a4d3454d63c1921a243d316c6.tar.gz",
         ],
     )
 
-- 
GitLab


From 30e5f93366b7ef7f24f7f267d2ad1d75afe4583b Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Mon, 7 Jan 2019 14:24:44 -0800
Subject: [PATCH 0326/1095] [XLA:CPU] Split out the logic to emit tiled
 GEMM/GEMV into a separate file; NFC

PiperOrigin-RevId: 228235618
---
 tensorflow/compiler/xla/service/cpu/BUILD     |   16 +
 .../xla/service/cpu/dot_op_emitter.cc         |  993 +---------------
 .../xla/service/cpu/tiled_dot_emitter.cc      | 1014 +++++++++++++++++
 .../xla/service/cpu/tiled_dot_emitter.h       |   55 +
 4 files changed, 1107 insertions(+), 971 deletions(-)
 create mode 100644 tensorflow/compiler/xla/service/cpu/tiled_dot_emitter.cc
 create mode 100644 tensorflow/compiler/xla/service/cpu/tiled_dot_emitter.h

diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD
index 5c72e1ff71..86c5f71096 100644
--- a/tensorflow/compiler/xla/service/cpu/BUILD
+++ b/tensorflow/compiler/xla/service/cpu/BUILD
@@ -364,6 +364,21 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "tiled_dot_emitter",
+    srcs = ["tiled_dot_emitter.cc"],
+    hdrs = ["tiled_dot_emitter.h"],
+    deps = [
+        ":vector_support_library",
+        "//tensorflow/compiler/xla:xla_data_proto",
+        "//tensorflow/compiler/xla/service:hlo",
+        "//tensorflow/compiler/xla/service/llvm_ir:kernel_support_library",
+        "//tensorflow/compiler/xla/service/llvm_ir:llvm_util",
+        "//tensorflow/core:lib",
+        "@llvm//:core",
+    ],
+)
+
 cc_library(
     name = "dot_op_emitter",
     srcs = ["dot_op_emitter.cc"],
@@ -373,6 +388,7 @@ cc_library(
         ":cpu_runtime",
         ":ir_emission_utils",
         ":target_machine_features",
+        ":tiled_dot_emitter",
         ":vector_support_library",
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:status_macros",
diff --git a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
index 1525a33af7..bdd688c2b9 100644
--- a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
@@ -26,6 +26,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/cpu/cpu_runtime.h"
 #include "tensorflow/compiler/xla/service/cpu/ir_emission_utils.h"
 #include "tensorflow/compiler/xla/service/cpu/target_machine_features.h"
+#include "tensorflow/compiler/xla/service/cpu/tiled_dot_emitter.h"
 #include "tensorflow/compiler/xla/service/cpu/vector_support_library.h"
 #include "tensorflow/compiler/xla/service/hlo_module.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/kernel_support_library.h"
@@ -41,930 +42,6 @@ namespace xla {
 using llvm_ir::SetToFirstInsertPoint;
 
 namespace cpu {
-
-namespace {
-// Provides tiled access to an in-memory rank 2 array.
-class MemoryTile {
- public:
-  // Constructs a MemoryTile that can operate on tiles consisting of
-  // `tile_size_along_major_dim` vectors from the matrix `matrix`, starting at
-  // `major_dim_offset` in the major dimension.  The tile size along the minor
-  // dimension is the vector size, and that is implicitly determined by `vsl`.
-  MemoryTile(VectorSupportLibrary* vsl, llvm::IRBuilder<>* b,
-             llvm::Value* matrix, int64 matrix_size_along_minor_dim,
-             llvm::Value* major_dim_offset, int64 tile_size_along_major_dim)
-      : vsl_(vsl), b_(b) {
-    pointers_.reserve(tile_size_along_major_dim);
-    for (int64 i = 0; i < tile_size_along_major_dim; i++) {
-      llvm::Value* total_offset =
-          b->CreateMul(b->getInt64(matrix_size_along_minor_dim),
-                       b->CreateAdd(b->getInt64(i), major_dim_offset));
-      pointers_.push_back(vsl_->ComputeOffsetPointer(matrix, total_offset));
-    }
-  }
-
-  // Load a tile consisting of `tile_size_along_major_dim` vectors from position
-  // {major: `major_dim_offset`, minor: `minor_dim_offset`}.
-  //
-  // Note: `major_dim_offset` is a parameter to the constructor.
-  std::vector<llvm::Value*> LoadTile(llvm::Value* minor_dim_offset) const {
-    std::vector<llvm::Value*> result;
-    result.reserve(pointers_.size());
-    for (const auto& pointer : pointers_) {
-      result.push_back(vsl_->LoadVector(pointer, minor_dim_offset));
-    }
-    return result;
-  }
-
-  // Stores `tile` to position {major: `major_dim_offset`, minor:
-  // `minor_dim_offset`}.
-  //
-  // Note: `major_dim_offset` is a parameter to the constructor.
-  void StoreTile(absl::Span<llvm::Value* const> tile,
-                 llvm::Value* minor_dim_offset) const {
-    CHECK_EQ(tile.size(), pointers_.size());
-    for (int64 i = 0; i < pointers_.size(); i++) {
-      vsl_->StoreVector(tile[i], pointers_[i], minor_dim_offset);
-    }
-  }
-
-  // Loads a tile of size [`tile_size_along_major_dim`,
-  // `tile_size_along_middle_dim`] from position {major: `major_dim_offset`,
-  // minor: `minor_dim_offset`} and then broadcasts each element into a vector
-  // of size vsl_.vector_size().  The (i,j)'th element of the return value is
-  // the (i,j)'th element in the tile broadcasted into an LLVM vector.
-  //
-  // Note: `major_dim_offset` is a parameter to the constructor.
-  std::vector<std::vector<llvm::Value*>> LoadBroadcastTile(
-      llvm::Value* minor_dim_offset, int64 tile_size_along_middle_dim) const {
-    std::vector<std::vector<llvm::Value*>> result;
-    result.resize(pointers_.size());
-    for (int64 i = 0; i < pointers_.size(); i++) {
-      for (int64 j = 0; j < tile_size_along_middle_dim; j++) {
-        result[i].push_back(vsl_->LoadBroadcast(
-            pointers_[i], b_->CreateAdd(minor_dim_offset, b_->getInt64(j))));
-      }
-    }
-    return result;
-  }
-
- private:
-  VectorSupportLibrary* vsl_;
-  llvm::IRBuilder<>* b_;
-  std::vector<llvm::Value*> pointers_;
-};
-
-// The base class for the classes representing the GEMV emitter configurations.
-//
-// The IR emitted (modulo the LLVM values representing the input and output
-// buffers) by the row major and column major GEMV emitters should be a function
-// of their configuration.  This is important because their configuration is
-// used as a key to cache the generated IR.
-class GemvConfig {
- public:
-  // Mixin for convenience.
-  template <typename T>
-  struct User {
-   public:
-    PrimitiveType scalar_type() const {
-      return derived().config().scalar_type();
-    }
-    int64 tile_rows() const { return derived().config().tile_rows(); }
-    int64 tile_cols() const { return derived().config().tile_cols(); }
-    int64 m() const { return derived().config().m(); }
-    int64 k() const { return derived().config().k(); }
-    int64 has_addend() const { return derived().config().has_addend(); }
-
-   private:
-    const T& derived() const { return *static_cast<const T*>(this); }
-  };
-
-  PrimitiveType scalar_type() const { return scalar_type_; }
-  int64 tile_rows() const { return tile_rows_; }
-  int64 tile_cols() const { return tile_cols_; }
-  int64 m() const { return m_; }
-  int64 k() const { return k_; }
-  bool has_addend() const { return has_addend_; }
-
-  string GetCacheKey() const {
-    return absl::StrCat(name_, "_", PrimitiveType_Name(scalar_type()), "_",
-                        tile_rows(), "_", tile_cols(), "_", m(), "_", k(),
-                        has_addend() ? "_with_addend" : "");
-  }
-
- protected:
-  explicit GemvConfig(string name, PrimitiveType scalar_type, int64 tile_rows,
-                      int64 tile_cols, int64 m, int64 k, bool has_addend)
-      : name_(std::move(name)),
-        scalar_type_(scalar_type),
-        tile_rows_(tile_rows),
-        tile_cols_(tile_cols),
-        m_(m),
-        k_(k),
-        has_addend_(has_addend) {}
-
- private:
-  string name_;
-  PrimitiveType scalar_type_;
-  int64 tile_rows_;
-  int64 tile_cols_;
-  int64 m_;
-  int64 k_;
-  bool has_addend_;
-};
-
-// Computes a dot product between "[M,K]{0,1} lhs" with a [K,1] vector (the
-// layout of the vector does not matter).  This implementation uses a tiling
-// scheme to improve performance.
-//
-// We logically separate the LHS matrix into four segments:
-//
-//   +----------------------+---+
-//   |                      |   |
-//   |                      |   |
-//   |         A            | B |
-//   |                      |   |
-//   |                      |   |
-//   |                      |   |
-//   +----------------------+---+
-//   |         C            | D |
-//   +----------------------+---+
-//
-// where A is the largest submatrix of the LHS that can be evenly dividied into
-// tiles.  For each tile in A, assuming tile_rows_ == tile_cols_ == 4, we have:
-//
-//   +---+---+---+---+       +--+--+--+--+
-//   |M00|M10|M20|M30|       |V0|V1|V2|V3|
-//   +---+---+---+---+       +--+--+--+--+
-//   |M01|M11|M21|M31| and   |V0|V1|V2|V3|
-//   +---+---+---+---+       +--+--+--+--+
-//   |M02|M12|M22|M32|       |V0|V1|V2|V3|
-//   +---+---+---+---+       +--+--+--+--+
-//   |M03|M13|M23|M33|       |V0|V1|V2|V3|
-//   +---+---+---+---+       +--+--+--+--+
-//
-// (Legend: rows are horizontal and columns are vertical; and each column is one
-// llvm::Value of a vector type)
-//
-// where:
-//
-//   a. The left tile is from the column major left matrix.
-//   b. The right tile is an elementwise broadcast of a [V0, V1, V2, V3]
-//      vector loaded from the RHS vector.
-//
-// As we iterate through the column dimension, we compute the change to the
-// result vector by an elementwise multiplication between the two tiles above
-// followed by a reduction along the major dimension:
-//
-//                     +-----------------------------------+
-//                     | M00*V0 + M10*V1 + M20*V2 + M30*V3 |
-//                     +-----------------------------------+
-//                     | M01*V0 + M11*V1 + M21*V2 + M31*V3 |
-// Result[R:R+4] +=    +-----------------------------------+
-//                     | M02*V0 + M12*V1 + M22*V2 + M32*V3 |
-//                     +-----------------------------------+
-//                     | M03*V0 + M13*V1 + M23*V2 + M33*V3 |
-//                     +-----------------------------------+
-//
-// Where R is the starting row for the tile.
-//
-// We have an inner epilogue loop to deal with the "C" submatrix and an outer
-// epilogue loop to deal with the B,D submarix.
-//
-// TODO(sanjoy): We should investigate if using gather loads and scatter stores
-// can be used here have the same inner loop for both column-major and row-major
-// matrix-vector products.
-class ColumnMajorMatrixVectorProductEmitter
-    : public GemvConfig::User<ColumnMajorMatrixVectorProductEmitter> {
- public:
-  class Config : public GemvConfig {
-   public:
-    explicit Config(PrimitiveType scalar_type, int64 tile_rows, int64 tile_cols,
-                    int64 m, int64 k, bool has_addend)
-        : GemvConfig(/*name=*/"col_major_gemv", scalar_type,
-                     /*tile_rows=*/tile_rows, /*tile_cols=*/tile_cols, /*m=*/m,
-                     /*k=*/k, /*has_addend=*/has_addend) {}
-  };
-
-  ColumnMajorMatrixVectorProductEmitter(const Config& config, llvm::Value* lhs,
-                                        llvm::Value* rhs, llvm::Value* addend,
-                                        llvm::Value* result,
-                                        llvm::IRBuilder<>* b)
-      : config_(config),
-        lhs_(lhs),
-        rhs_(rhs),
-        addend_(addend),
-        result_(result),
-        b_(b),
-        ksl_(b_),
-        vsl_(config.scalar_type(), /*vector_size=*/config.tile_rows(), b_, "") {
-    CHECK(tile_rows() > 0 && IsPowerOfTwo(static_cast<uint64>(tile_rows())));
-    CHECK(!has_addend() || addend != nullptr);
-  }
-
-  void Emit();
-
-  const Config& config() const { return config_; }
-
- private:
-  void EmitOuterLoopBody(llvm::Value* column, int64 column_count,
-                         bool is_first_column);
-
-  MemoryTile GetLhsMemoryTile(llvm::Value* column_start, int64 column_count) {
-    return MemoryTile(&vsl_, b_, /*matrix=*/lhs_,
-                      /*matrix_size_along_minor_dim=*/m(),
-                      /*major_dim_offset=*/column_start,
-                      /*tile_size_along_major_dim=*/column_count);
-  }
-
-  // Load a tile of values from the RHS.  For the RHS a "tile" is a contiguous
-  // sequence of `count` values, each one broadcasted to the vector width.
-  std::vector<llvm::Value*> LoadRhsTile(llvm::Value* offset, int64 count) {
-    llvm::Value* base_pointer = vsl_.ComputeOffsetPointer(rhs_, offset);
-    std::vector<llvm::Value*> result;
-    result.reserve(count);
-    for (int64 i = 0; i < count; i++) {
-      result.push_back(vsl_.LoadBroadcast(base_pointer, i));
-    }
-    return result;
-  }
-
-  void EmitInnerLoopTiled(MemoryTile* lhs_memory_tile,
-                          const std::vector<llvm::Value*>& rhs_tile,
-                          int64 columns, bool is_first_column);
-
-  void EmitInnerLoopEpilogue(llvm::Value* current_tile_col, int64 columns,
-                             bool is_first_tiled_column);
-
-  Config config_;
-  llvm::Value* lhs_;
-  llvm::Value* rhs_;
-  llvm::Value* addend_;
-  llvm::Value* result_;
-  llvm::IRBuilder<>* b_;
-  KernelSupportLibrary ksl_;
-  VectorSupportLibrary vsl_;
-};
-
-void ColumnMajorMatrixVectorProductEmitter::EmitOuterLoopBody(
-    llvm::Value* column, int64 column_count, bool is_first_column) {
-  MemoryTile lhs_memory_tile = GetLhsMemoryTile(/*column_start=*/column,
-                                                /*column_count=*/column_count);
-
-  std::vector<llvm::Value*> rhs_tile =
-      LoadRhsTile(column, /*count=*/column_count);
-  EmitInnerLoopTiled(&lhs_memory_tile, rhs_tile,
-                     /*columns=*/column_count, is_first_column);
-  EmitInnerLoopEpilogue(column, /*columns=*/column_count, is_first_column);
-}
-
-void ColumnMajorMatrixVectorProductEmitter::Emit() {
-  // See the comment on the class declaration for the algorithm used here.
-  int64 column_remainder = k() % tile_cols();
-  int64 column_limit = k() - column_remainder;
-
-  ksl_.For("dot.outer.tiled",
-           /*start=*/0, /*end=*/column_limit, /*step=*/tile_cols(),
-           [&](llvm::Value* column, bool is_first_column) {
-             EmitOuterLoopBody(column, tile_cols(), is_first_column);
-           });
-
-  if (column_remainder != 0) {
-    EmitOuterLoopBody(b_->getInt64(column_limit), column_remainder,
-                      column_limit == 0);
-  }
-}
-
-void ColumnMajorMatrixVectorProductEmitter::EmitInnerLoopTiled(
-    MemoryTile* lhs_memory_tile, const std::vector<llvm::Value*>& rhs_tile,
-    int64 columns, bool is_first_column) {
-  int64 row_limit = m() - (m() % tile_rows());
-
-  ksl_.For(
-      "dot.inner.tiled", /*start=*/0, /*end=*/row_limit,
-      /*step=*/tile_rows(), [&](llvm::Value* row) {
-        std::vector<llvm::Value*> lhs_tile =
-            lhs_memory_tile->LoadTile(/*minor_dim_offset=*/row);
-        llvm::Value* accumulator =
-            is_first_column ? (addend_ ? vsl_.LoadVector(addend_, row)
-                                       : vsl_.GetZeroVector())
-                            : vsl_.LoadVector(result_, row);
-        for (int i = 0; i < columns; i++) {
-          accumulator = vsl_.MulAdd(lhs_tile[i], rhs_tile[i], accumulator);
-        }
-        vsl_.StoreVector(accumulator, result_, row);
-      });
-}
-
-void ColumnMajorMatrixVectorProductEmitter::EmitInnerLoopEpilogue(
-    llvm::Value* current_tile_col, int64 columns, bool is_first_tiled_column) {
-  int64 row_start = m() - (m() % tile_rows());
-  if (row_start == m()) {
-    return;
-  }
-
-  llvm::Value* columns_llvm = b_->getInt64(columns);
-
-  // for (col = current_tile_col; col < (columns + current_tile_col); col++)
-  //   for (row = row_start, row < m_; row++) {
-  //     result[row] += lhs[row, col] * rhs[col]
-  //     // Also take into account that if col is 0 then result[row] is not
-  //     // initialized.
-  //   }
-
-  ksl_.For(
-      "dot.inner.epilg.outer", /*start=*/current_tile_col,
-      /*end=*/b_->CreateAdd(columns_llvm, current_tile_col),
-      /*step=*/1, /*peel_first_iteration=*/false,
-      [&](llvm::Value* col, llvm::Value* is_first_scalar_col) {
-        llvm::Value* rhs_element = vsl_.LoadScalar(rhs_, col);
-        llvm::Value* total_offset = b_->CreateMul(col, b_->getInt64(m()));
-        llvm::Value* lhs_base_pointer =
-            vsl_.ComputeOffsetPointer(lhs_, total_offset);
-        ksl_.For(
-            "dot.inner.epilg.inner", /*start=*/row_start, /*end=*/m(),
-            /*step=*/1, [&](llvm::Value* scalar_row) {
-              llvm::Value* product = vsl_.Mul(
-                  vsl_.LoadScalar(lhs_base_pointer, scalar_row), rhs_element);
-              llvm::Value* setting_result_first_time = b_->CreateAnd(
-                  is_first_scalar_col, b_->getInt1(is_first_tiled_column));
-              ksl_.If(
-                  setting_result_first_time,
-                  /*true_block_generator=*/
-                  [&]() {
-                    if (addend_) {
-                      vsl_.StoreScalar(
-                          vsl_.Add(vsl_.LoadScalar(addend_, scalar_row),
-                                   product),
-                          result_, scalar_row);
-                    } else {
-                      vsl_.StoreScalar(product, result_, scalar_row);
-                    }
-                  },
-                  /*false_block_generator=*/
-                  [&]() {
-                    vsl_.StoreScalar(
-                        vsl_.Add(vsl_.LoadScalar(result_, scalar_row), product),
-                        result_, scalar_row);
-                  });
-            });
-      });
-}
-
-// Computes a dot product between "[M,K]{1,0} lhs" with a [K,1] vector (the
-// layout of the vector does not matter).  This implementation uses a tiling
-// scheme to improve performance.
-//
-// We logically separate the LHS matrix into four segments:
-//
-//   +----------------------+---+
-//   |                      |   |
-//   |                      |   |
-//   |         A            | B |
-//   |                      |   |
-//   |                      |   |
-//   |                      |   |
-//   +----------------------+---+
-//   |         C            | D |
-//   +----------------------+---+
-//
-// where A is the largest submatrix of the LHS that can be evenly dividied into
-// tiles.  For each tile in A, assuming tile_rows_ == tile_cols_ == 4, we have:
-//
-//   +---+---+---+---+
-//   |M00|M10|M20|M30|
-//   +---+---+---+---+       +--+--+--+--+
-//   |M01|M11|M21|M31| and   |V0|V1|V2|V3|
-//   +---+---+---+---+       +--+--+--+--+
-//   |M02|M12|M22|M32|
-//   +---+---+---+---+
-//   |M03|M13|M23|M33|
-//   +---+---+---+---+
-//
-// (Legend: rows are horizontal and columns are vertical; and each row is one
-// llvm::Value of a vector type)
-//
-// where:
-//
-//   a. The left tile is loaded from the row major left matrix.
-//   b. The right vector is loaded from the RHS vector.
-//
-// We keep 4 vector accumulators accumulating the following four vector
-// expressions as we iterate over the row dimension:
-//
-//   +------+------+------+------+
-//   |M0I*V0|M1I*V1|M2I*V2|M3I*V3|  for I in [0,4)
-//   +------+------+------+------+
-//
-// In the end we do a horizontal reduction over these 4 vector accumulators to
-// get 4 values in the result vector.
-//
-// We have an inner epilogue loop to deal with the "B" sub-matrix and an outer
-// epilogue loop to deal with the C,D submatrix.
-class RowMajorMatrixVectorProductEmitter
-    : public GemvConfig::User<RowMajorMatrixVectorProductEmitter> {
- public:
-  class Config : public GemvConfig {
-   public:
-    explicit Config(PrimitiveType scalar_type, int64 tile_rows, int64 tile_cols,
-                    int64 m, int64 k, bool has_addend)
-        : GemvConfig(/*name=*/"row_major_gemv", scalar_type,
-                     /*tile_rows=*/tile_rows, /*tile_cols=*/tile_cols, /*m=*/m,
-                     /*k=*/k, /*has_addend=*/has_addend) {}
-  };
-
-  RowMajorMatrixVectorProductEmitter(const Config& config, llvm::Value* lhs,
-                                     llvm::Value* rhs, llvm::Value* addend,
-                                     llvm::Value* result, llvm::IRBuilder<>* b)
-      : config_(config),
-        lhs_(lhs),
-        rhs_(rhs),
-        addend_(addend),
-        result_(result),
-        b_(b),
-        ksl_(b_),
-        vsl_(scalar_type(), /*vector_size=*/tile_cols(), b_, "") {
-    CHECK(tile_cols() > 0 && IsPowerOfTwo(static_cast<uint64>(tile_cols())));
-    CHECK(!has_addend() || addend != nullptr);
-  }
-
-  void Emit();
-
-  const Config& config() const { return config_; }
-
- private:
-  MemoryTile GetLhsMemoryTile(llvm::Value* row_start, int64 row_count) {
-    return MemoryTile(&vsl_, b_, /*matrix=*/lhs_,
-                      /*matrix_size_along_minor_dim=*/k(),
-                      /*major_dim_offset=*/row_start,
-                      /*tile_size_along_major_dim=*/row_count);
-  }
-
-  void EmitOuterLoopBody(llvm::Value* row, int64 row_count);
-
-  void EmitInnerLoopTiled(MemoryTile* lhs_memory_tile, int64 rows,
-                          std::vector<VectorVariable>* vector_accumulators);
-
-  void EmitInnerLoopEpilogue(llvm::Value* current_tile_row, int64 rows,
-                             std::vector<ScalarVariable>* scalar_accumulators);
-
-  Config config_;
-  llvm::Value* lhs_;
-  llvm::Value* rhs_;
-  llvm::Value* addend_;
-  llvm::Value* result_;
-  llvm::IRBuilder<>* b_;
-  KernelSupportLibrary ksl_;
-  VectorSupportLibrary vsl_;
-};
-
-void RowMajorMatrixVectorProductEmitter::EmitOuterLoopBody(llvm::Value* row,
-                                                           int64 row_count) {
-  MemoryTile lhs_memory_tile = GetLhsMemoryTile(/*row_start=*/row,
-                                                /*row_count=*/row_count);
-  std::vector<VectorVariable> vector_accumulators;
-  std::vector<ScalarVariable> scalar_accumulators;
-  for (int i = 0; i < row_count; i++) {
-    vector_accumulators.emplace_back(&vsl_, vsl_.GetZeroVector());
-    scalar_accumulators.emplace_back(&vsl_, vsl_.GetZeroScalar());
-  }
-  EmitInnerLoopTiled(&lhs_memory_tile, /*rows=*/row_count,
-                     &vector_accumulators);
-  EmitInnerLoopEpilogue(/*current_tile_row=*/row, /*rows=*/row_count,
-                        &scalar_accumulators);
-
-  std::vector<llvm::Value*> accumulator_values;
-  std::transform(
-      vector_accumulators.begin(), vector_accumulators.end(),
-      std::back_inserter(accumulator_values),
-      [](const VectorVariable& vector_var) { return vector_var.Get(); });
-
-  std::vector<llvm::Value*> horizontal_sums;
-  if (row_count == vsl_.vector_size()) {
-    if (addend_) {
-      horizontal_sums = vsl_.ComputeHorizontalSums(
-          std::move(accumulator_values), vsl_.LoadVector(addend_, row));
-    } else {
-      horizontal_sums =
-          vsl_.ComputeHorizontalSums(std::move(accumulator_values));
-    }
-  } else {
-    horizontal_sums = vsl_.ComputeHorizontalSums(std::move(accumulator_values));
-  }
-
-  for (int i = 0; i < row_count; i++) {
-    llvm::Value* result_value =
-        vsl_.Add(horizontal_sums[i], scalar_accumulators[i].Get());
-    llvm::Value* offset = b_->CreateAdd(b_->getInt64(i), row);
-    if (addend_ && row_count != vsl_.vector_size()) {
-      result_value = vsl_.Add(vsl_.LoadScalar(addend_, offset), result_value);
-    }
-    vsl_.StoreScalar(result_value, result_, offset);
-  }
-}
-
-void RowMajorMatrixVectorProductEmitter::Emit() {
-  // See the comment on the class declaration for the algorithm used here.
-  int64 row_remainder = m() % tile_rows();
-  int64 row_limit = m() - row_remainder;
-
-  ksl_.For("dot.outer.tiled",
-           /*start=*/0, /*end=*/row_limit, /*step=*/tile_rows(),
-           [&](llvm::Value* row) { EmitOuterLoopBody(row, tile_rows()); });
-
-  if (row_remainder != 0) {
-    EmitOuterLoopBody(b_->getInt64(row_limit), row_remainder);
-  }
-}
-
-void RowMajorMatrixVectorProductEmitter::EmitInnerLoopTiled(
-    MemoryTile* lhs_memory_tile, int64 rows,
-    std::vector<VectorVariable>* vector_accumulators) {
-  int64 column_limit = k() - (k() % tile_cols());
-
-  ksl_.For("dot.inner.tiled", /*start=*/0, /*end=*/column_limit,
-           /*step=*/tile_cols(), [&](llvm::Value* col) {
-             std::vector<llvm::Value*> lhs_tile =
-                 lhs_memory_tile->LoadTile(/*minor_dim_offset=*/col);
-             llvm::Value* rhs_value = vsl_.LoadVector(rhs_, col);
-             for (int i = 0; i < rows; i++) {
-               llvm::Value* old_sum = (*vector_accumulators)[i].Get();
-               (*vector_accumulators)[i].Set(
-                   vsl_.Add(old_sum, vsl_.Mul(rhs_value, lhs_tile[i])));
-             }
-           });
-}
-
-void RowMajorMatrixVectorProductEmitter::EmitInnerLoopEpilogue(
-    llvm::Value* current_tile_row, int64 rows,
-    std::vector<ScalarVariable>* scalar_accumulators) {
-  int64 column_start = k() - (k() % tile_cols());
-  if (column_start == k()) {
-    return;
-  }
-
-  for (int r = 0; r < rows; r++) {
-    llvm::Value* total_offset = b_->CreateMul(
-        b_->CreateAdd(b_->getInt64(r), current_tile_row), b_->getInt64(k()));
-    llvm::Value* lhs_base_pointer =
-        vsl_.ComputeOffsetPointer(lhs_, total_offset);
-    ksl_.For(
-        "dot.inner.epilg.inner", /*start=*/column_start, /*end=*/k(),
-        /*step=*/1, [&](llvm::Value* scalar_col) {
-          llvm::Value* product =
-              vsl_.Mul(vsl_.LoadScalar(lhs_base_pointer, scalar_col),
-                       vsl_.LoadScalar(rhs_, scalar_col));
-          llvm::Value* old_value = (*scalar_accumulators)[r].Get();
-          (*scalar_accumulators)[r].Set(vsl_.Add(old_value, product));
-        });
-  }
-}
-
-// This class implements a tiled matrix multiplication algorithm, intended for
-// multiplying small matrices that don't need cache tiling.
-//
-// In the future this can be used as the innermost GEBP loop in a GEMM kernel as
-// described in "Goto, Kazushige, and Robert A. Geijn. "Anatomy of
-// high-performance matrix multiplication." ACM Transactions on Mathematical
-// Software (TOMS) 34.3 (2008): 12.".
-//
-// This only supports canonical dot operations (i.e. where the lhs contraction
-// dimension is 1 and the rhs contraction dimension is 0) over row major
-// matrices.
-class TiledSmallGemmEmitter {
- public:
-  // Describe the dimensions of the kernel.
-  class Dimensions {
-   public:
-    explicit Dimensions(int64 m, int64 k, int64 n) : m_(m), k_(k), n_(n) {}
-
-    int64 m() const { return m_; }
-    int64 k() const { return k_; }
-    int64 n() const { return n_; }
-
-    string ToString() const { return absl::StrCat(m(), "x", k(), "x", n()); }
-
-   private:
-    const int64 m_;
-    const int64 k_;
-    const int64 n_;
-  };
-
-  // Represents the configuration of the emitter.  The LLVM IR emitted by the
-  // emitter, modulo the LLVM values holding the input and output buffers, must
-  // be a function of the instance of `Config` passed to it.
-  //
-  // `dims` holds the matrix multiplication dimensions.
-  //
-  // `max_vectorization_width` is the maximum vector width (i.e. the width of
-  // the largest vector register we will use).  This can be larger than the
-  // largest vector register supported by the machine -- LLVM will legalize
-  // these large vector widths into legally sized vectors.
-  //
-  // `max_vector_count` is the maximum number of vectors of size
-  // `max_vectorization_width` that we will attempt to process at once.
-  //
-  // `min_vectorization_width` is the smallest vector width the emitter will use
-  // -- below that it will devolve to using a scalar loop.
-  //
-  // The innermost reduction loop executes the matrix multiply in tiles of size
-  // [`tile_size_m`, `tile_size_k`] from the LHS and [`tile_size_k`,
-  // <vectorization width>] in the RHS.
-  class Config {
-   public:
-    explicit Config(PrimitiveType scalar_type, Dimensions dims,
-                    int64 max_vectorization_width, int64 max_vector_count,
-                    int64 min_vectorization_width, int64 tile_size_m,
-                    int64 tile_size_k)
-        : scalar_type_(scalar_type),
-          dims_(dims),
-          max_vectorization_width_(max_vectorization_width),
-          max_vector_count_(max_vector_count),
-          min_vectorization_width_(min_vectorization_width),
-          tile_size_m_(tile_size_m),
-          tile_size_k_(tile_size_k) {}
-
-    string GetCacheKey() const {
-      return absl::StrCat("gemm_", PrimitiveType_Name(scalar_type()), "_",
-                          dims().ToString(), "_", max_vectorization_width(),
-                          "_", min_vectorization_width(), "_", tile_size_m(),
-                          "_", tile_size_k());
-    }
-
-    PrimitiveType scalar_type() const { return scalar_type_; }
-    Dimensions dims() const { return dims_; }
-    int64 max_vectorization_width() const { return max_vectorization_width_; }
-    int64 max_vector_count() const { return max_vector_count_; }
-    int64 min_vectorization_width() const { return min_vectorization_width_; }
-
-    int64 tile_size_m() const { return tile_size_m_; }
-    int64 tile_size_k() const { return tile_size_k_; }
-
-   private:
-    PrimitiveType scalar_type_;
-    Dimensions dims_;
-    int64 max_vectorization_width_;
-    int64 max_vector_count_;
-    int64 min_vectorization_width_;
-    int64 tile_size_m_;
-    int64 tile_size_k_;
-  };
-
-  // Creates an instance of TiledSmallGemmEmitter that matrix-multiplies
-  // `lhs` with `rhs` and stores the result in `result`.
-  explicit TiledSmallGemmEmitter(Config config, llvm::Value* lhs,
-                                 llvm::Value* rhs, llvm::Value* result,
-                                 llvm::IRBuilder<>* b)
-      : lhs_(lhs),
-        rhs_(rhs),
-        result_(result),
-        config_(config),
-        b_(b),
-        ksl_(b_) {
-    CHECK(max_vectorization_width() > 0 &&
-          IsPowerOfTwo(static_cast<uint64>(max_vectorization_width())));
-    CHECK_GT(max_vector_count(), 0);
-    CHECK(min_vectorization_width() > 0 &&
-          IsPowerOfTwo(static_cast<uint64>(min_vectorization_width())));
-    CHECK_GE(max_vectorization_width(), min_vectorization_width());
-    CHECK_GT(tile_size_k(), 0);
-  }
-
-  void Emit();
-
- private:
-  // The HandleResiduesOnX helpers split the iteration space for dimension X
-  // into a multiple of the tile size on dimension X and an epilogue.  These
-  // helpers ultimately call into `EmitTiledGemm` for emitting the
-  // tiled GEMM kernel.
-
-  void HandleResiduesOnN();
-  void HandleResiduesOnK(VectorSupportLibrary* vsl, llvm::Value* n_start,
-                         llvm::Value* n_end);
-  void HandleResiduesOnM(VectorSupportLibrary* vsl, int64 tile_size_k,
-                         llvm::Value* k_start, llvm::Value* k_end,
-                         llvm::Value* n_start, llvm::Value* n_end);
-
-  // This emits a tiled GEMM kernel.  For a detailed description see the comment
-  // on the implementation.
-  void EmitTiledGemm(VectorSupportLibrary* vsl, int64 tile_size_k,
-                     llvm::Value* k_start, llvm::Value* k_end,
-                     llvm::Value* n_start, llvm::Value* n_end,
-                     int64 tile_size_m, llvm::Value* m_start,
-                     llvm::Value* m_end);
-
-  llvm::Value* GetInt64(int64 value) { return b_->getInt64(value); }
-
-  Config config() const { return config_; }
-  Dimensions dims() const { return config().dims(); }
-
-  int64 max_vectorization_width() const {
-    return config().max_vectorization_width();
-  }
-  int64 max_vector_count() const { return config().max_vector_count(); }
-  int64 min_vectorization_width() const {
-    return config().min_vectorization_width();
-  }
-  int64 tile_size_m() const { return config().tile_size_m(); }
-  int64 tile_size_k() const { return config().tile_size_k(); }
-  PrimitiveType scalar_type() const { return config().scalar_type(); }
-
-  llvm::Value* lhs_;
-  llvm::Value* rhs_;
-  llvm::Value* result_;
-  Config config_;
-
-  llvm::IRBuilder<>* b_;
-  KernelSupportLibrary ksl_;
-};
-
-void TiledSmallGemmEmitter::Emit() { HandleResiduesOnN(); }
-
-void TiledSmallGemmEmitter::HandleResiduesOnN() {
-  // We can only iterate the `n` dimension for an extent that is divisible by
-  // the vectorization width.  So we emit an outer loop that first processes the
-  // largest extent in `n` that is divisible by max_vectorization_width, then
-  // the largest remaining extent that is divisible by max_vectorization_width /
-  // 2 etc.
-
-  int64 current_vectorization_width =
-      max_vector_count() * max_vectorization_width();
-  int64 current_vector_count = max_vector_count();
-
-  int64 n_start = 0;
-  while (n_start != dims().n() &&
-         current_vectorization_width >= min_vectorization_width()) {
-    int64 n_end = dims().n() - (dims().n() % current_vectorization_width);
-    if (n_start != n_end) {
-      VectorSupportLibrary vsl(scalar_type(), current_vectorization_width, b_,
-                               "gemm");
-      HandleResiduesOnK(&vsl, GetInt64(n_start), GetInt64(n_end));
-      n_start = n_end;
-    }
-    if (current_vector_count == 1) {
-      current_vectorization_width /= 2;
-    } else {
-      current_vector_count--;
-      current_vectorization_width =
-          current_vector_count * max_vectorization_width();
-    }
-  }
-
-  if (n_start != dims().n()) {
-    VectorSupportLibrary vsl(scalar_type(), 1, b_, "gemm");
-    ksl_.For("epi.n", n_start, dims().n(), 1, [&](llvm::Value* n_i) {
-      llvm::Value* n_i_next = b_->CreateAdd(n_i, b_->getInt64(1));
-      HandleResiduesOnK(&vsl, n_i, n_i_next);
-    });
-  }
-}
-
-void TiledSmallGemmEmitter::HandleResiduesOnK(VectorSupportLibrary* vsl,
-                                              llvm::Value* n_start,
-                                              llvm::Value* n_end) {
-  int64 k_start = 0;
-  int64 k_end = dims().k() - (dims().k() % tile_size_k());
-  if (k_end != k_start) {
-    HandleResiduesOnM(vsl, tile_size_k(), GetInt64(k_start), GetInt64(k_end),
-                      n_start, n_end);
-    k_start = k_end;
-  }
-
-  if (k_start != dims().k()) {
-    HandleResiduesOnM(vsl, dims().k() - k_start, GetInt64(k_start),
-                      GetInt64(dims().k()), n_start, n_end);
-  }
-}
-
-void TiledSmallGemmEmitter::HandleResiduesOnM(
-    VectorSupportLibrary* vsl, int64 tile_size_k, llvm::Value* k_start,
-    llvm::Value* k_end, llvm::Value* n_start, llvm::Value* n_end) {
-  const int64 m_end = dims().m() - dims().m() % tile_size_m();
-  EmitTiledGemm(vsl, tile_size_k, k_start, k_end, n_start, n_end, tile_size_m(),
-                GetInt64(0), GetInt64(m_end));
-
-  if (m_end != dims().m()) {
-    EmitTiledGemm(vsl, tile_size_k, k_start, k_end, n_start, n_end,
-                  dims().m() - m_end, GetInt64(m_end), GetInt64(dims().m()));
-  }
-}
-
-// The loop structure is:
-//
-// Iterate over dimension M as m:
-//   Iterate over dimension N as n:
-//     Iterate over dimension K as k:
-//       OutputTile[m,n] += Dot(LhsTile[m,k], RhsTile[k,n])
-//
-// I.e. a just a tiled version of a "naive" GEMM.
-//
-// The tiling scheme is as follows:
-//
-// Let the LHS be:
-//
-//   +----+----+----+
-//   | a0 | b0 | c0 | .
-//   +----+----+----+ .
-//   | a1 | b1 | c1 | .
-//   +----+----+----+
-//     ..     ..
-//
-// and the RHS be:
-//
-//   +----+----+----+----+
-//   | p0 | p1 | p2 | p3 | .
-//   +----+----+----+----+ .
-//   | q0 | q1 | q2 | q3 | .
-//   +----+----+----+----+
-//   | r0 | r1 | r2 | r3 | .
-//   +----+----+----+----+ .
-//     ......    ......
-//
-// and let tile_size_m=2, tile_size_k=3 and the vector width (implicitly denoted
-// by `vsl`) be 4.  Then we want to matrix multiply this tile to get a [2,4]
-// matrix that we can increment the result matrix by.
-//
-// First broadcast the rows row in LHS to 3 vectors of width 4, giving us a rank
-// 3 array, L, of dimension [2,3,4]:
-//
-//       L[0,_,_]           *      L[1,_,_]
-//                          *
-//   +----+----+----+----+  *  +----+----+----+----+
-//   | a0 | a0 | a0 | a0 |  *  | a1 | a1 | a1 | a1 |
-//   +----+----+----+----+  *  +----+----+----+----+
-//   | b0 | b0 | b0 | b0 |  *  | b1 | b1 | b1 | b1 |
-//   +----+----+----+----+  *  +----+----+----+----+
-//   | c0 | c0 | c0 | c0 |  *  | c1 | c1 | c1 | c1 |
-//   +----+----+----+----+  *  +----+----+----+----+
-//
-//
-// Then we FMA L[0,_,_] with the RHS to get the first row of the result and
-// L[1,_,_] with the RHS to get the second row of the result.  For example,
-// L[0,_,_] is computed as:
-//
-//   +----+----+----+----+   +----+----+----+----+
-//   | a0 | a0 | a0 | a0 | * | p0 | p1 | p2 | p3 |   +
-//   +----+----+----+----+   +----+----+----+----+
-//
-//   +----+----+----+----+   +----+----+----+----+
-//   | b0 | b0 | b0 | b0 | * | q0 | q1 | q2 | q3 |   +
-//   +----+----+----+----+   +----+----+----+----+
-//
-//   +----+----+----+----+   +----+----+----+----+
-//   | c0 | c0 | c0 | c0 | * | r0 | r1 | r2 | r3 |
-//   +----+----+----+----+   +----+----+----+----+
-//
-// to get:
-//
-//   +-------------------+-------------------+-------------------+---------
-//   | a0*p0+b0*q0+c0*r0 | a0*p1+b0*q1+c0*r1 | a0*p2+b0*q2+c0*r2 |  ...
-//   +-------------------+-------------------+-------------------+---------
-void TiledSmallGemmEmitter::EmitTiledGemm(
-    VectorSupportLibrary* vsl, int64 tile_size_k, llvm::Value* k_start,
-    llvm::Value* k_end, llvm::Value* n_start, llvm::Value* n_end,
-    int64 tile_size_m, llvm::Value* m_start, llvm::Value* m_end) {
-  ksl_.For(
-      "dot.m", m_start, m_end, tile_size_m, [&](llvm::Value* m_i) {
-        MemoryTile result_memory_tile(
-            vsl, b_, /*matrix=*/result_,
-            /*matrix_size_along_minor_dim=*/dims().n(),
-            /*major_dim_offset=*/m_i,
-            /*tile_size_along_major_dim=*/tile_size_m);
-        MemoryTile lhs_memory_tile(vsl, b_, /*matrix=*/lhs_,
-                                   /*matrix_size_along_minor_dim=*/dims().k(),
-                                   /*major_dim_offset=*/m_i,
-                                   /*tile_size_along_major_dim=*/tile_size_m);
-        ksl_.For(
-            "dot.n", n_start, n_end, vsl->vector_size(), [&](llvm::Value* n_i) {
-              TileVariable result_tile_var(vsl,
-                                           result_memory_tile.LoadTile(n_i));
-              ksl_.For(
-                  "dot.k", k_start, k_end, tile_size_k, [&](llvm::Value* k_i) {
-                    MemoryTile rhs_memory_tile(vsl, b_, rhs_, dims().n(), k_i,
-                                               tile_size_k);
-                    std::vector<std::vector<llvm::Value*>> lhs_tile =
-                        lhs_memory_tile.LoadBroadcastTile(k_i, tile_size_k);
-                    std::vector<llvm::Value*> rhs_tile =
-                        rhs_memory_tile.LoadTile(n_i);
-                    std::vector<llvm::Value*> result_tile =
-                        result_tile_var.Get();
-                    for (int64 r_m_i = 0; r_m_i < tile_size_m; r_m_i++) {
-                      for (int64 r_k_i = 0; r_k_i < tile_size_k; r_k_i++) {
-                        result_tile[r_m_i] =
-                            vsl->MulAdd(lhs_tile[r_m_i][r_k_i], rhs_tile[r_k_i],
-                                        result_tile[r_m_i]);
-                      }
-                    }
-                    result_tile_var.Set(result_tile);
-                  });
-
-              result_memory_tile.StoreTile(result_tile_var.Get(), n_i);
-            });
-      });
-}
-
-}  // namespace
-
 DotOpEmitter::DotOpEmitter(const HloInstruction& dot,
                            const llvm_ir::IrArray& target_array,
                            const llvm_ir::IrArray& lhs_array,
@@ -1062,32 +139,21 @@ bool DotOpEmitter::EmitSmallGemmIfProfitable(
   std::tie(tile_size_m, tile_size_k, tile_size_n_in_vector_width) =
       GetGemmTileSize();
 
-  TiledSmallGemmEmitter::Config config(
-      /*scalar_type=*/primitive_type,
-      TiledSmallGemmEmitter::Dimensions{/*m=*/m, /*k=*/k, /*n=*/n},
-      /*max_vectorization_width=*/max_target_vector_width,
-      /*max_vector_count=*/tile_size_n_in_vector_width,
-      /*min_vectorization_width=*/std::min<int64>(4, max_target_vector_width),
-      /*tile_size_m=*/tile_size_m, /*tile_size_k=*/tile_size_k);
-
-  VLOG(2) << "Emitting GEMM kernel in LLVM IR with config "
-          << config.GetCacheKey();
-
   const bool enable_fast_math =
       hlo_module_config_.debug_options().xla_cpu_enable_fast_math();
   const bool optimize_for_size =
       options::OptimizeForSizeRequested(hlo_module_config_);
 
-  KernelSupportLibrary::EmitAndCallOutlinedKernel(
+  EmitSmallGemm(
+      /*scalar_type=*/primitive_type,
+      /*m=*/m, /*k=*/k, /*n=*/n,
+      /*max_vectorization_width=*/max_target_vector_width,
+      /*max_vector_count=*/tile_size_n_in_vector_width,
+      /*min_vectorization_width=*/std::min<int64>(4, max_target_vector_width),
+      /*tile_size_m=*/tile_size_m, /*tile_size_k=*/tile_size_k, /*lhs=*/lhs,
+      /*rhs=*/rhs, /*result=*/target, b_,
       /*enable_fast_math=*/enable_fast_math,
-      /*optimize_for_size=*/optimize_for_size, b_, config.GetCacheKey(), lhs,
-      rhs, target,
-      [this, config](llvm::Value* lhs, llvm::Value* rhs, llvm::Value* target) {
-        TiledSmallGemmEmitter small_gemm_emitter(config, /*lhs=*/lhs,
-                                                 /*rhs=*/rhs,
-                                                 /*result=*/target, b_);
-        small_gemm_emitter.Emit();
-      });
+      /*optimize_for_size=*/optimize_for_size);
 
   return true;
 }
@@ -1177,41 +243,26 @@ bool DotOpEmitter::EmitLlvmIrDotIfProfitable() {
   if (is_column_major_matrix_vector) {
     VLOG(2) << "Emitting column major matrix-vector multiply with m = " << m
             << " and k = " << k;
-    ColumnMajorMatrixVectorProductEmitter::Config config(
+    EmitColumnMajorGemv(
         /*scalar_type=*/primitive_type,
         /*tile_rows=*/vector_register_element_size, /*tile_cols=*/tiling_factor,
-        /*m=*/m, /*k=*/k, /*has_addend=*/addend_array_ != nullptr);
-
-    KernelSupportLibrary::EmitAndCallOutlinedKernel(
+        /*m=*/m, /*k=*/k, /*lhs=*/lhs_op, /*rhs=*/rhs_op,
+        /*addend=*/addend_array_ ? addend_array_->GetBasePointer() : nullptr,
+        /*result=*/result_op, b_,
         /*enable_fast_math=*/enable_fast_math,
-        /*optimize_for_size=*/optimize_for_size, b_, config.GetCacheKey(),
-        lhs_op, rhs_op,
-        addend_array_ ? addend_array_->GetBasePointer() : nullptr, result_op,
-        [this, config](llvm::Value* lhs_op, llvm::Value* rhs_op,
-                       llvm::Value* addend_op, llvm::Value* result_op) {
-          ColumnMajorMatrixVectorProductEmitter emitter(
-              config, lhs_op, rhs_op, addend_op, result_op, b_);
-          emitter.Emit();
-        });
+        /*optimize_for_size=*/optimize_for_size);
   } else {
     VLOG(2) << "Emitting row major matrix-vector multiply with m = " << m
             << " and k = " << k;
-    RowMajorMatrixVectorProductEmitter::Config config(
+    EmitRowMajorGemv(
         /*scalar_type=*/primitive_type,
-        /*tile_rows=*/tiling_factor, /*tile_cols=*/vector_register_element_size,
-        /*m=*/m, /*k=*/k, /*has_addend=*/addend_array_ != nullptr);
-
-    KernelSupportLibrary::EmitAndCallOutlinedKernel(
+        /*tile_rows=*/tiling_factor,
+        /*tile_cols=*/vector_register_element_size,
+        /*m=*/m, /*k=*/k, /*lhs=*/lhs_op, /*rhs=*/rhs_op,
+        /*addend=*/addend_array_ ? addend_array_->GetBasePointer() : nullptr,
+        /*result=*/result_op, b_,
         /*enable_fast_math=*/enable_fast_math,
-        /*optimize_for_size=*/optimize_for_size, b_, config.GetCacheKey(),
-        lhs_op, rhs_op,
-        addend_array_ ? addend_array_->GetBasePointer() : nullptr, result_op,
-        [this, config](llvm::Value* lhs_op, llvm::Value* rhs_op,
-                       llvm::Value* addend_op, llvm::Value* result_op) {
-          RowMajorMatrixVectorProductEmitter emitter(config, lhs_op, rhs_op,
-                                                     addend_op, result_op, b_);
-          emitter.Emit();
-        });
+        /*optimize_for_size=*/optimize_for_size);
   }
 
   return true;
diff --git a/tensorflow/compiler/xla/service/cpu/tiled_dot_emitter.cc b/tensorflow/compiler/xla/service/cpu/tiled_dot_emitter.cc
new file mode 100644
index 0000000000..eb6c44b70a
--- /dev/null
+++ b/tensorflow/compiler/xla/service/cpu/tiled_dot_emitter.cc
@@ -0,0 +1,1014 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/cpu/tiled_dot_emitter.h"
+
+#include "tensorflow/compiler/xla/service/cpu/vector_support_library.h"
+#include "tensorflow/compiler/xla/service/hlo_module.h"
+#include "tensorflow/compiler/xla/service/llvm_ir/kernel_support_library.h"
+#include "tensorflow/compiler/xla/service/llvm_ir/llvm_util.h"
+
+namespace xla {
+namespace cpu {
+namespace {
+
+using tensorflow::int64;
+
+// Provides tiled access to an in-memory rank 2 array.
+class MemoryTile {
+ public:
+  // Constructs a MemoryTile that can operate on tiles consisting of
+  // `tile_size_along_major_dim` vectors from the matrix `matrix`, starting at
+  // `major_dim_offset` in the major dimension.  The tile size along the minor
+  // dimension is the vector size, and that is implicitly determined by `vsl`.
+  MemoryTile(VectorSupportLibrary* vsl, llvm::IRBuilder<>* b,
+             llvm::Value* matrix, int64 matrix_size_along_minor_dim,
+             llvm::Value* major_dim_offset, int64 tile_size_along_major_dim)
+      : vsl_(vsl), b_(b) {
+    pointers_.reserve(tile_size_along_major_dim);
+    for (int64 i = 0; i < tile_size_along_major_dim; i++) {
+      llvm::Value* total_offset =
+          b->CreateMul(b->getInt64(matrix_size_along_minor_dim),
+                       b->CreateAdd(b->getInt64(i), major_dim_offset));
+      pointers_.push_back(vsl_->ComputeOffsetPointer(matrix, total_offset));
+    }
+  }
+
+  // Load a tile consisting of `tile_size_along_major_dim` vectors from position
+  // {major: `major_dim_offset`, minor: `minor_dim_offset`}.
+  //
+  // Note: `major_dim_offset` is a parameter to the constructor.
+  std::vector<llvm::Value*> LoadTile(llvm::Value* minor_dim_offset) const {
+    std::vector<llvm::Value*> result;
+    result.reserve(pointers_.size());
+    for (const auto& pointer : pointers_) {
+      result.push_back(vsl_->LoadVector(pointer, minor_dim_offset));
+    }
+    return result;
+  }
+
+  // Stores `tile` to position {major: `major_dim_offset`, minor:
+  // `minor_dim_offset`}.
+  //
+  // Note: `major_dim_offset` is a parameter to the constructor.
+  void StoreTile(absl::Span<llvm::Value* const> tile,
+                 llvm::Value* minor_dim_offset) const {
+    CHECK_EQ(tile.size(), pointers_.size());
+    for (int64 i = 0; i < pointers_.size(); i++) {
+      vsl_->StoreVector(tile[i], pointers_[i], minor_dim_offset);
+    }
+  }
+
+  // Loads a tile of size [`tile_size_along_major_dim`,
+  // `tile_size_along_middle_dim`] from position {major: `major_dim_offset`,
+  // minor: `minor_dim_offset`} and then broadcasts each element into a vector
+  // of size vsl_.vector_size().  The (i,j)'th element of the return value is
+  // the (i,j)'th element in the tile broadcasted into an LLVM vector.
+  //
+  // Note: `major_dim_offset` is a parameter to the constructor.
+  std::vector<std::vector<llvm::Value*>> LoadBroadcastTile(
+      llvm::Value* minor_dim_offset, int64 tile_size_along_middle_dim) const {
+    std::vector<std::vector<llvm::Value*>> result;
+    result.resize(pointers_.size());
+    for (int64 i = 0; i < pointers_.size(); i++) {
+      for (int64 j = 0; j < tile_size_along_middle_dim; j++) {
+        result[i].push_back(vsl_->LoadBroadcast(
+            pointers_[i], b_->CreateAdd(minor_dim_offset, b_->getInt64(j))));
+      }
+    }
+    return result;
+  }
+
+ private:
+  VectorSupportLibrary* vsl_;
+  llvm::IRBuilder<>* b_;
+  std::vector<llvm::Value*> pointers_;
+};
+
+// The base class for the classes representing the GEMV emitter configurations.
+//
+// The IR emitted (modulo the LLVM values representing the input and output
+// buffers) by the row major and column major GEMV emitters should be a function
+// of their configuration.  This is important because their configuration is
+// used as a key to cache the generated IR.
+class GemvConfig {
+ public:
+  // Mixin for convenience.
+  template <typename T>
+  struct User {
+   public:
+    PrimitiveType scalar_type() const {
+      return derived().config().scalar_type();
+    }
+    int64 tile_rows() const { return derived().config().tile_rows(); }
+    int64 tile_cols() const { return derived().config().tile_cols(); }
+    int64 m() const { return derived().config().m(); }
+    int64 k() const { return derived().config().k(); }
+    int64 has_addend() const { return derived().config().has_addend(); }
+
+   private:
+    const T& derived() const { return *static_cast<const T*>(this); }
+  };
+
+  PrimitiveType scalar_type() const { return scalar_type_; }
+  int64 tile_rows() const { return tile_rows_; }
+  int64 tile_cols() const { return tile_cols_; }
+  int64 m() const { return m_; }
+  int64 k() const { return k_; }
+  bool has_addend() const { return has_addend_; }
+
+  string GetCacheKey() const {
+    return absl::StrCat(name_, "_", PrimitiveType_Name(scalar_type()), "_",
+                        tile_rows(), "_", tile_cols(), "_", m(), "_", k(),
+                        has_addend() ? "_with_addend" : "");
+  }
+
+ protected:
+  explicit GemvConfig(string name, PrimitiveType scalar_type, int64 tile_rows,
+                      int64 tile_cols, int64 m, int64 k, bool has_addend)
+      : name_(std::move(name)),
+        scalar_type_(scalar_type),
+        tile_rows_(tile_rows),
+        tile_cols_(tile_cols),
+        m_(m),
+        k_(k),
+        has_addend_(has_addend) {}
+
+ private:
+  string name_;
+  PrimitiveType scalar_type_;
+  int64 tile_rows_;
+  int64 tile_cols_;
+  int64 m_;
+  int64 k_;
+  bool has_addend_;
+};
+
+// Computes a dot product between "[M,K]{0,1} lhs" with a [K,1] vector (the
+// layout of the vector does not matter).  This implementation uses a tiling
+// scheme to improve performance.
+//
+// We logically separate the LHS matrix into four segments:
+//
+//   +----------------------+---+
+//   |                      |   |
+//   |                      |   |
+//   |         A            | B |
+//   |                      |   |
+//   |                      |   |
+//   |                      |   |
+//   +----------------------+---+
+//   |         C            | D |
+//   +----------------------+---+
+//
+// where A is the largest submatrix of the LHS that can be evenly dividied into
+// tiles.  For each tile in A, assuming tile_rows_ == tile_cols_ == 4, we have:
+//
+//   +---+---+---+---+       +--+--+--+--+
+//   |M00|M10|M20|M30|       |V0|V1|V2|V3|
+//   +---+---+---+---+       +--+--+--+--+
+//   |M01|M11|M21|M31| and   |V0|V1|V2|V3|
+//   +---+---+---+---+       +--+--+--+--+
+//   |M02|M12|M22|M32|       |V0|V1|V2|V3|
+//   +---+---+---+---+       +--+--+--+--+
+//   |M03|M13|M23|M33|       |V0|V1|V2|V3|
+//   +---+---+---+---+       +--+--+--+--+
+//
+// (Legend: rows are horizontal and columns are vertical; and each column is one
+// llvm::Value of a vector type)
+//
+// where:
+//
+//   a. The left tile is from the column major left matrix.
+//   b. The right tile is an elementwise broadcast of a [V0, V1, V2, V3]
+//      vector loaded from the RHS vector.
+//
+// As we iterate through the column dimension, we compute the change to the
+// result vector by an elementwise multiplication between the two tiles above
+// followed by a reduction along the major dimension:
+//
+//                     +-----------------------------------+
+//                     | M00*V0 + M10*V1 + M20*V2 + M30*V3 |
+//                     +-----------------------------------+
+//                     | M01*V0 + M11*V1 + M21*V2 + M31*V3 |
+// Result[R:R+4] +=    +-----------------------------------+
+//                     | M02*V0 + M12*V1 + M22*V2 + M32*V3 |
+//                     +-----------------------------------+
+//                     | M03*V0 + M13*V1 + M23*V2 + M33*V3 |
+//                     +-----------------------------------+
+//
+// Where R is the starting row for the tile.
+//
+// We have an inner epilogue loop to deal with the "C" submatrix and an outer
+// epilogue loop to deal with the B,D submarix.
+//
+// TODO(sanjoy): We should investigate if using gather loads and scatter stores
+// can be used here have the same inner loop for both column-major and row-major
+// matrix-vector products.
+class ColumnMajorMatrixVectorProductEmitter
+    : public GemvConfig::User<ColumnMajorMatrixVectorProductEmitter> {
+ public:
+  class Config : public GemvConfig {
+   public:
+    explicit Config(PrimitiveType scalar_type, int64 tile_rows, int64 tile_cols,
+                    int64 m, int64 k, bool has_addend)
+        : GemvConfig(/*name=*/"col_major_gemv", scalar_type,
+                     /*tile_rows=*/tile_rows, /*tile_cols=*/tile_cols, /*m=*/m,
+                     /*k=*/k, /*has_addend=*/has_addend) {}
+  };
+
+  ColumnMajorMatrixVectorProductEmitter(const Config& config, llvm::Value* lhs,
+                                        llvm::Value* rhs, llvm::Value* addend,
+                                        llvm::Value* result,
+                                        llvm::IRBuilder<>* b)
+      : config_(config),
+        lhs_(lhs),
+        rhs_(rhs),
+        addend_(addend),
+        result_(result),
+        b_(b),
+        ksl_(b_),
+        vsl_(config.scalar_type(), /*vector_size=*/config.tile_rows(), b_, "") {
+    CHECK(tile_rows() > 0 && IsPowerOfTwo(static_cast<uint64>(tile_rows())));
+    CHECK(!has_addend() || addend != nullptr);
+  }
+
+  void Emit();
+
+  const Config& config() const { return config_; }
+
+ private:
+  void EmitOuterLoopBody(llvm::Value* column, int64 column_count,
+                         bool is_first_column);
+
+  MemoryTile GetLhsMemoryTile(llvm::Value* column_start, int64 column_count) {
+    return MemoryTile(&vsl_, b_, /*matrix=*/lhs_,
+                      /*matrix_size_along_minor_dim=*/m(),
+                      /*major_dim_offset=*/column_start,
+                      /*tile_size_along_major_dim=*/column_count);
+  }
+
+  // Load a tile of values from the RHS.  For the RHS a "tile" is a contiguous
+  // sequence of `count` values, each one broadcasted to the vector width.
+  std::vector<llvm::Value*> LoadRhsTile(llvm::Value* offset, int64 count) {
+    llvm::Value* base_pointer = vsl_.ComputeOffsetPointer(rhs_, offset);
+    std::vector<llvm::Value*> result;
+    result.reserve(count);
+    for (int64 i = 0; i < count; i++) {
+      result.push_back(vsl_.LoadBroadcast(base_pointer, i));
+    }
+    return result;
+  }
+
+  void EmitInnerLoopTiled(MemoryTile* lhs_memory_tile,
+                          const std::vector<llvm::Value*>& rhs_tile,
+                          int64 columns, bool is_first_column);
+
+  void EmitInnerLoopEpilogue(llvm::Value* current_tile_col, int64 columns,
+                             bool is_first_tiled_column);
+
+  Config config_;
+  llvm::Value* lhs_;
+  llvm::Value* rhs_;
+  llvm::Value* addend_;
+  llvm::Value* result_;
+  llvm::IRBuilder<>* b_;
+  KernelSupportLibrary ksl_;
+  VectorSupportLibrary vsl_;
+};
+
+void ColumnMajorMatrixVectorProductEmitter::EmitOuterLoopBody(
+    llvm::Value* column, int64 column_count, bool is_first_column) {
+  MemoryTile lhs_memory_tile = GetLhsMemoryTile(/*column_start=*/column,
+                                                /*column_count=*/column_count);
+
+  std::vector<llvm::Value*> rhs_tile =
+      LoadRhsTile(column, /*count=*/column_count);
+  EmitInnerLoopTiled(&lhs_memory_tile, rhs_tile,
+                     /*columns=*/column_count, is_first_column);
+  EmitInnerLoopEpilogue(column, /*columns=*/column_count, is_first_column);
+}
+
+void ColumnMajorMatrixVectorProductEmitter::Emit() {
+  // See the comment on the class declaration for the algorithm used here.
+  int64 column_remainder = k() % tile_cols();
+  int64 column_limit = k() - column_remainder;
+
+  ksl_.For("dot.outer.tiled",
+           /*start=*/0, /*end=*/column_limit, /*step=*/tile_cols(),
+           [&](llvm::Value* column, bool is_first_column) {
+             EmitOuterLoopBody(column, tile_cols(), is_first_column);
+           });
+
+  if (column_remainder != 0) {
+    EmitOuterLoopBody(b_->getInt64(column_limit), column_remainder,
+                      column_limit == 0);
+  }
+}
+
+void ColumnMajorMatrixVectorProductEmitter::EmitInnerLoopTiled(
+    MemoryTile* lhs_memory_tile, const std::vector<llvm::Value*>& rhs_tile,
+    int64 columns, bool is_first_column) {
+  int64 row_limit = m() - (m() % tile_rows());
+
+  ksl_.For("dot.inner.tiled", /*start=*/0, /*end=*/row_limit,
+           /*step=*/tile_rows(), [&](llvm::Value* row) {
+             std::vector<llvm::Value*> lhs_tile =
+                 lhs_memory_tile->LoadTile(/*minor_dim_offset=*/row);
+             llvm::Value* accumulator =
+                 is_first_column ? (addend_ ? vsl_.LoadVector(addend_, row)
+                                            : vsl_.GetZeroVector())
+                                 : vsl_.LoadVector(result_, row);
+             for (int i = 0; i < columns; i++) {
+               accumulator = vsl_.MulAdd(lhs_tile[i], rhs_tile[i], accumulator);
+             }
+             vsl_.StoreVector(accumulator, result_, row);
+           });
+}
+
+void ColumnMajorMatrixVectorProductEmitter::EmitInnerLoopEpilogue(
+    llvm::Value* current_tile_col, int64 columns, bool is_first_tiled_column) {
+  int64 row_start = m() - (m() % tile_rows());
+  if (row_start == m()) {
+    return;
+  }
+
+  llvm::Value* columns_llvm = b_->getInt64(columns);
+
+  // for (col = current_tile_col; col < (columns + current_tile_col); col++)
+  //   for (row = row_start, row < m_; row++) {
+  //     result[row] += lhs[row, col] * rhs[col]
+  //     // Also take into account that if col is 0 then result[row] is not
+  //     // initialized.
+  //   }
+
+  ksl_.For(
+      "dot.inner.epilg.outer", /*start=*/current_tile_col,
+      /*end=*/b_->CreateAdd(columns_llvm, current_tile_col),
+      /*step=*/1, /*peel_first_iteration=*/false,
+      [&](llvm::Value* col, llvm::Value* is_first_scalar_col) {
+        llvm::Value* rhs_element = vsl_.LoadScalar(rhs_, col);
+        llvm::Value* total_offset = b_->CreateMul(col, b_->getInt64(m()));
+        llvm::Value* lhs_base_pointer =
+            vsl_.ComputeOffsetPointer(lhs_, total_offset);
+        ksl_.For(
+            "dot.inner.epilg.inner", /*start=*/row_start, /*end=*/m(),
+            /*step=*/1, [&](llvm::Value* scalar_row) {
+              llvm::Value* product = vsl_.Mul(
+                  vsl_.LoadScalar(lhs_base_pointer, scalar_row), rhs_element);
+              llvm::Value* setting_result_first_time = b_->CreateAnd(
+                  is_first_scalar_col, b_->getInt1(is_first_tiled_column));
+              ksl_.If(
+                  setting_result_first_time,
+                  /*true_block_generator=*/
+                  [&]() {
+                    if (addend_) {
+                      vsl_.StoreScalar(
+                          vsl_.Add(vsl_.LoadScalar(addend_, scalar_row),
+                                   product),
+                          result_, scalar_row);
+                    } else {
+                      vsl_.StoreScalar(product, result_, scalar_row);
+                    }
+                  },
+                  /*false_block_generator=*/
+                  [&]() {
+                    vsl_.StoreScalar(
+                        vsl_.Add(vsl_.LoadScalar(result_, scalar_row), product),
+                        result_, scalar_row);
+                  });
+            });
+      });
+}
+
+// Computes a dot product between "[M,K]{1,0} lhs" with a [K,1] vector (the
+// layout of the vector does not matter).  This implementation uses a tiling
+// scheme to improve performance.
+//
+// We logically separate the LHS matrix into four segments:
+//
+//   +----------------------+---+
+//   |                      |   |
+//   |                      |   |
+//   |         A            | B |
+//   |                      |   |
+//   |                      |   |
+//   |                      |   |
+//   +----------------------+---+
+//   |         C            | D |
+//   +----------------------+---+
+//
+// where A is the largest submatrix of the LHS that can be evenly dividied into
+// tiles.  For each tile in A, assuming tile_rows_ == tile_cols_ == 4, we have:
+//
+//   +---+---+---+---+
+//   |M00|M10|M20|M30|
+//   +---+---+---+---+       +--+--+--+--+
+//   |M01|M11|M21|M31| and   |V0|V1|V2|V3|
+//   +---+---+---+---+       +--+--+--+--+
+//   |M02|M12|M22|M32|
+//   +---+---+---+---+
+//   |M03|M13|M23|M33|
+//   +---+---+---+---+
+//
+// (Legend: rows are horizontal and columns are vertical; and each row is one
+// llvm::Value of a vector type)
+//
+// where:
+//
+//   a. The left tile is loaded from the row major left matrix.
+//   b. The right vector is loaded from the RHS vector.
+//
+// We keep 4 vector accumulators accumulating the following four vector
+// expressions as we iterate over the row dimension:
+//
+//   +------+------+------+------+
+//   |M0I*V0|M1I*V1|M2I*V2|M3I*V3|  for I in [0,4)
+//   +------+------+------+------+
+//
+// In the end we do a horizontal reduction over these 4 vector accumulators to
+// get 4 values in the result vector.
+//
+// We have an inner epilogue loop to deal with the "B" sub-matrix and an outer
+// epilogue loop to deal with the C,D submatrix.
+class RowMajorMatrixVectorProductEmitter
+    : public GemvConfig::User<RowMajorMatrixVectorProductEmitter> {
+ public:
+  class Config : public GemvConfig {
+   public:
+    explicit Config(PrimitiveType scalar_type, int64 tile_rows, int64 tile_cols,
+                    int64 m, int64 k, bool has_addend)
+        : GemvConfig(/*name=*/"row_major_gemv", scalar_type,
+                     /*tile_rows=*/tile_rows, /*tile_cols=*/tile_cols, /*m=*/m,
+                     /*k=*/k, /*has_addend=*/has_addend) {}
+  };
+
+  RowMajorMatrixVectorProductEmitter(const Config& config, llvm::Value* lhs,
+                                     llvm::Value* rhs, llvm::Value* addend,
+                                     llvm::Value* result, llvm::IRBuilder<>* b)
+      : config_(config),
+        lhs_(lhs),
+        rhs_(rhs),
+        addend_(addend),
+        result_(result),
+        b_(b),
+        ksl_(b_),
+        vsl_(scalar_type(), /*vector_size=*/tile_cols(), b_, "") {
+    CHECK(tile_cols() > 0 && IsPowerOfTwo(static_cast<uint64>(tile_cols())));
+    CHECK(!has_addend() || addend != nullptr);
+  }
+
+  void Emit();
+
+  const Config& config() const { return config_; }
+
+ private:
+  MemoryTile GetLhsMemoryTile(llvm::Value* row_start, int64 row_count) {
+    return MemoryTile(&vsl_, b_, /*matrix=*/lhs_,
+                      /*matrix_size_along_minor_dim=*/k(),
+                      /*major_dim_offset=*/row_start,
+                      /*tile_size_along_major_dim=*/row_count);
+  }
+
+  void EmitOuterLoopBody(llvm::Value* row, int64 row_count);
+
+  void EmitInnerLoopTiled(MemoryTile* lhs_memory_tile, int64 rows,
+                          std::vector<VectorVariable>* vector_accumulators);
+
+  void EmitInnerLoopEpilogue(llvm::Value* current_tile_row, int64 rows,
+                             std::vector<ScalarVariable>* scalar_accumulators);
+
+  Config config_;
+  llvm::Value* lhs_;
+  llvm::Value* rhs_;
+  llvm::Value* addend_;
+  llvm::Value* result_;
+  llvm::IRBuilder<>* b_;
+  KernelSupportLibrary ksl_;
+  VectorSupportLibrary vsl_;
+};
+
+void RowMajorMatrixVectorProductEmitter::EmitOuterLoopBody(llvm::Value* row,
+                                                           int64 row_count) {
+  MemoryTile lhs_memory_tile = GetLhsMemoryTile(/*row_start=*/row,
+                                                /*row_count=*/row_count);
+  std::vector<VectorVariable> vector_accumulators;
+  std::vector<ScalarVariable> scalar_accumulators;
+  for (int i = 0; i < row_count; i++) {
+    vector_accumulators.emplace_back(&vsl_, vsl_.GetZeroVector());
+    scalar_accumulators.emplace_back(&vsl_, vsl_.GetZeroScalar());
+  }
+  EmitInnerLoopTiled(&lhs_memory_tile, /*rows=*/row_count,
+                     &vector_accumulators);
+  EmitInnerLoopEpilogue(/*current_tile_row=*/row, /*rows=*/row_count,
+                        &scalar_accumulators);
+
+  std::vector<llvm::Value*> accumulator_values;
+  std::transform(
+      vector_accumulators.begin(), vector_accumulators.end(),
+      std::back_inserter(accumulator_values),
+      [](const VectorVariable& vector_var) { return vector_var.Get(); });
+
+  std::vector<llvm::Value*> horizontal_sums;
+  if (row_count == vsl_.vector_size()) {
+    if (addend_) {
+      horizontal_sums = vsl_.ComputeHorizontalSums(
+          std::move(accumulator_values), vsl_.LoadVector(addend_, row));
+    } else {
+      horizontal_sums =
+          vsl_.ComputeHorizontalSums(std::move(accumulator_values));
+    }
+  } else {
+    horizontal_sums = vsl_.ComputeHorizontalSums(std::move(accumulator_values));
+  }
+
+  for (int i = 0; i < row_count; i++) {
+    llvm::Value* result_value =
+        vsl_.Add(horizontal_sums[i], scalar_accumulators[i].Get());
+    llvm::Value* offset = b_->CreateAdd(b_->getInt64(i), row);
+    if (addend_ && row_count != vsl_.vector_size()) {
+      result_value = vsl_.Add(vsl_.LoadScalar(addend_, offset), result_value);
+    }
+    vsl_.StoreScalar(result_value, result_, offset);
+  }
+}
+
+void RowMajorMatrixVectorProductEmitter::Emit() {
+  // See the comment on the class declaration for the algorithm used here.
+  int64 row_remainder = m() % tile_rows();
+  int64 row_limit = m() - row_remainder;
+
+  ksl_.For("dot.outer.tiled",
+           /*start=*/0, /*end=*/row_limit, /*step=*/tile_rows(),
+           [&](llvm::Value* row) { EmitOuterLoopBody(row, tile_rows()); });
+
+  if (row_remainder != 0) {
+    EmitOuterLoopBody(b_->getInt64(row_limit), row_remainder);
+  }
+}
+
+void RowMajorMatrixVectorProductEmitter::EmitInnerLoopTiled(
+    MemoryTile* lhs_memory_tile, int64 rows,
+    std::vector<VectorVariable>* vector_accumulators) {
+  int64 column_limit = k() - (k() % tile_cols());
+
+  ksl_.For("dot.inner.tiled", /*start=*/0, /*end=*/column_limit,
+           /*step=*/tile_cols(), [&](llvm::Value* col) {
+             std::vector<llvm::Value*> lhs_tile =
+                 lhs_memory_tile->LoadTile(/*minor_dim_offset=*/col);
+             llvm::Value* rhs_value = vsl_.LoadVector(rhs_, col);
+             for (int i = 0; i < rows; i++) {
+               llvm::Value* old_sum = (*vector_accumulators)[i].Get();
+               (*vector_accumulators)[i].Set(
+                   vsl_.Add(old_sum, vsl_.Mul(rhs_value, lhs_tile[i])));
+             }
+           });
+}
+
+void RowMajorMatrixVectorProductEmitter::EmitInnerLoopEpilogue(
+    llvm::Value* current_tile_row, int64 rows,
+    std::vector<ScalarVariable>* scalar_accumulators) {
+  int64 column_start = k() - (k() % tile_cols());
+  if (column_start == k()) {
+    return;
+  }
+
+  for (int r = 0; r < rows; r++) {
+    llvm::Value* total_offset = b_->CreateMul(
+        b_->CreateAdd(b_->getInt64(r), current_tile_row), b_->getInt64(k()));
+    llvm::Value* lhs_base_pointer =
+        vsl_.ComputeOffsetPointer(lhs_, total_offset);
+    ksl_.For("dot.inner.epilg.inner", /*start=*/column_start, /*end=*/k(),
+             /*step=*/1, [&](llvm::Value* scalar_col) {
+               llvm::Value* product =
+                   vsl_.Mul(vsl_.LoadScalar(lhs_base_pointer, scalar_col),
+                            vsl_.LoadScalar(rhs_, scalar_col));
+               llvm::Value* old_value = (*scalar_accumulators)[r].Get();
+               (*scalar_accumulators)[r].Set(vsl_.Add(old_value, product));
+             });
+  }
+}
+
+// This class implements a tiled matrix multiplication algorithm, intended for
+// multiplying small matrices that don't need cache tiling.
+//
+// In the future this can be used as the innermost GEBP loop in a GEMM kernel as
+// described in "Goto, Kazushige, and Robert A. Geijn. "Anatomy of
+// high-performance matrix multiplication." ACM Transactions on Mathematical
+// Software (TOMS) 34.3 (2008): 12.".
+//
+// This only supports canonical dot operations (i.e. where the lhs contraction
+// dimension is 1 and the rhs contraction dimension is 0) over row major
+// matrices.
+class TiledSmallGemmEmitter {
+ public:
+  // Describe the dimensions of the kernel.
+  class Dimensions {
+   public:
+    explicit Dimensions(int64 m, int64 k, int64 n) : m_(m), k_(k), n_(n) {}
+
+    int64 m() const { return m_; }
+    int64 k() const { return k_; }
+    int64 n() const { return n_; }
+
+    string ToString() const { return absl::StrCat(m(), "x", k(), "x", n()); }
+
+   private:
+    const int64 m_;
+    const int64 k_;
+    const int64 n_;
+  };
+
+  // Represents the configuration of the emitter.  The LLVM IR emitted by the
+  // emitter, modulo the LLVM values holding the input and output buffers, must
+  // be a function of the instance of `Config` passed to it.
+  //
+  // `dims` holds the matrix multiplication dimensions.
+  //
+  // `max_vectorization_width` is the maximum vector width (i.e. the width of
+  // the largest vector register we will use).  This can be larger than the
+  // largest vector register supported by the machine -- LLVM will legalize
+  // these large vector widths into legally sized vectors.
+  //
+  // `max_vector_count` is the maximum number of vectors of size
+  // `max_vectorization_width` that we will attempt to process at once.
+  //
+  // `min_vectorization_width` is the smallest vector width the emitter will use
+  // -- below that it will devolve to using a scalar loop.
+  //
+  // The innermost reduction loop executes the matrix multiply in tiles of size
+  // [`tile_size_m`, `tile_size_k`] from the LHS and [`tile_size_k`,
+  // <vectorization width>] in the RHS.
+  class Config {
+   public:
+    explicit Config(PrimitiveType scalar_type, Dimensions dims,
+                    int64 max_vectorization_width, int64 max_vector_count,
+                    int64 min_vectorization_width, int64 tile_size_m,
+                    int64 tile_size_k)
+        : scalar_type_(scalar_type),
+          dims_(dims),
+          max_vectorization_width_(max_vectorization_width),
+          max_vector_count_(max_vector_count),
+          min_vectorization_width_(min_vectorization_width),
+          tile_size_m_(tile_size_m),
+          tile_size_k_(tile_size_k) {}
+
+    string GetCacheKey() const {
+      return absl::StrCat("gemm_", PrimitiveType_Name(scalar_type()), "_",
+                          dims().ToString(), "_", max_vectorization_width(),
+                          "_", min_vectorization_width(), "_", tile_size_m(),
+                          "_", tile_size_k());
+    }
+
+    PrimitiveType scalar_type() const { return scalar_type_; }
+    Dimensions dims() const { return dims_; }
+    int64 max_vectorization_width() const { return max_vectorization_width_; }
+    int64 max_vector_count() const { return max_vector_count_; }
+    int64 min_vectorization_width() const { return min_vectorization_width_; }
+
+    int64 tile_size_m() const { return tile_size_m_; }
+    int64 tile_size_k() const { return tile_size_k_; }
+
+   private:
+    PrimitiveType scalar_type_;
+    Dimensions dims_;
+    int64 max_vectorization_width_;
+    int64 max_vector_count_;
+    int64 min_vectorization_width_;
+    int64 tile_size_m_;
+    int64 tile_size_k_;
+  };
+
+  // Creates an instance of TiledSmallGemmEmitter that matrix-multiplies
+  // `lhs` with `rhs` and stores the result in `result`.
+  explicit TiledSmallGemmEmitter(Config config, llvm::Value* lhs,
+                                 llvm::Value* rhs, llvm::Value* result,
+                                 llvm::IRBuilder<>* b)
+      : lhs_(lhs),
+        rhs_(rhs),
+        result_(result),
+        config_(config),
+        b_(b),
+        ksl_(b_) {
+    CHECK(max_vectorization_width() > 0 &&
+          IsPowerOfTwo(static_cast<uint64>(max_vectorization_width())));
+    CHECK_GT(max_vector_count(), 0);
+    CHECK(min_vectorization_width() > 0 &&
+          IsPowerOfTwo(static_cast<uint64>(min_vectorization_width())));
+    CHECK_GE(max_vectorization_width(), min_vectorization_width());
+    CHECK_GT(tile_size_k(), 0);
+  }
+
+  void Emit();
+
+ private:
+  // The HandleResiduesOnX helpers split the iteration space for dimension X
+  // into a multiple of the tile size on dimension X and an epilogue.  These
+  // helpers ultimately call into `EmitTiledGemm` for emitting the
+  // tiled GEMM kernel.
+
+  void HandleResiduesOnN();
+  void HandleResiduesOnK(VectorSupportLibrary* vsl, llvm::Value* n_start,
+                         llvm::Value* n_end);
+  void HandleResiduesOnM(VectorSupportLibrary* vsl, int64 tile_size_k,
+                         llvm::Value* k_start, llvm::Value* k_end,
+                         llvm::Value* n_start, llvm::Value* n_end);
+
+  // This emits a tiled GEMM kernel.  For a detailed description see the comment
+  // on the implementation.
+  void EmitTiledGemm(VectorSupportLibrary* vsl, int64 tile_size_k,
+                     llvm::Value* k_start, llvm::Value* k_end,
+                     llvm::Value* n_start, llvm::Value* n_end,
+                     int64 tile_size_m, llvm::Value* m_start,
+                     llvm::Value* m_end);
+
+  llvm::Value* GetInt64(int64 value) { return b_->getInt64(value); }
+
+  Config config() const { return config_; }
+  Dimensions dims() const { return config().dims(); }
+
+  int64 max_vectorization_width() const {
+    return config().max_vectorization_width();
+  }
+  int64 max_vector_count() const { return config().max_vector_count(); }
+  int64 min_vectorization_width() const {
+    return config().min_vectorization_width();
+  }
+  int64 tile_size_m() const { return config().tile_size_m(); }
+  int64 tile_size_k() const { return config().tile_size_k(); }
+  PrimitiveType scalar_type() const { return config().scalar_type(); }
+
+  llvm::Value* lhs_;
+  llvm::Value* rhs_;
+  llvm::Value* result_;
+  Config config_;
+
+  llvm::IRBuilder<>* b_;
+  KernelSupportLibrary ksl_;
+};
+
+void TiledSmallGemmEmitter::Emit() { HandleResiduesOnN(); }
+
+void TiledSmallGemmEmitter::HandleResiduesOnN() {
+  // We can only iterate the `n` dimension for an extent that is divisible by
+  // the vectorization width.  So we emit an outer loop that first processes the
+  // largest extent in `n` that is divisible by max_vectorization_width, then
+  // the largest remaining extent that is divisible by max_vectorization_width /
+  // 2 etc.
+
+  int64 current_vectorization_width =
+      max_vector_count() * max_vectorization_width();
+  int64 current_vector_count = max_vector_count();
+
+  int64 n_start = 0;
+  while (n_start != dims().n() &&
+         current_vectorization_width >= min_vectorization_width()) {
+    int64 n_end = dims().n() - (dims().n() % current_vectorization_width);
+    if (n_start != n_end) {
+      VectorSupportLibrary vsl(scalar_type(), current_vectorization_width, b_,
+                               "gemm");
+      HandleResiduesOnK(&vsl, GetInt64(n_start), GetInt64(n_end));
+      n_start = n_end;
+    }
+    if (current_vector_count == 1) {
+      current_vectorization_width /= 2;
+    } else {
+      current_vector_count--;
+      current_vectorization_width =
+          current_vector_count * max_vectorization_width();
+    }
+  }
+
+  if (n_start != dims().n()) {
+    VectorSupportLibrary vsl(scalar_type(), 1, b_, "gemm");
+    ksl_.For("epi.n", n_start, dims().n(), 1, [&](llvm::Value* n_i) {
+      llvm::Value* n_i_next = b_->CreateAdd(n_i, b_->getInt64(1));
+      HandleResiduesOnK(&vsl, n_i, n_i_next);
+    });
+  }
+}
+
+void TiledSmallGemmEmitter::HandleResiduesOnK(VectorSupportLibrary* vsl,
+                                              llvm::Value* n_start,
+                                              llvm::Value* n_end) {
+  int64 k_start = 0;
+  int64 k_end = dims().k() - (dims().k() % tile_size_k());
+  if (k_end != k_start) {
+    HandleResiduesOnM(vsl, tile_size_k(), GetInt64(k_start), GetInt64(k_end),
+                      n_start, n_end);
+    k_start = k_end;
+  }
+
+  if (k_start != dims().k()) {
+    HandleResiduesOnM(vsl, dims().k() - k_start, GetInt64(k_start),
+                      GetInt64(dims().k()), n_start, n_end);
+  }
+}
+
+void TiledSmallGemmEmitter::HandleResiduesOnM(
+    VectorSupportLibrary* vsl, int64 tile_size_k, llvm::Value* k_start,
+    llvm::Value* k_end, llvm::Value* n_start, llvm::Value* n_end) {
+  const int64 m_end = dims().m() - dims().m() % tile_size_m();
+  EmitTiledGemm(vsl, tile_size_k, k_start, k_end, n_start, n_end, tile_size_m(),
+                GetInt64(0), GetInt64(m_end));
+
+  if (m_end != dims().m()) {
+    EmitTiledGemm(vsl, tile_size_k, k_start, k_end, n_start, n_end,
+                  dims().m() - m_end, GetInt64(m_end), GetInt64(dims().m()));
+  }
+}
+
+// The loop structure is:
+//
+// Iterate over dimension M as m:
+//   Iterate over dimension N as n:
+//     Iterate over dimension K as k:
+//       OutputTile[m,n] += Dot(LhsTile[m,k], RhsTile[k,n])
+//
+// I.e. a just a tiled version of a "naive" GEMM.
+//
+// The tiling scheme is as follows:
+//
+// Let the LHS be:
+//
+//   +----+----+----+
+//   | a0 | b0 | c0 | .
+//   +----+----+----+ .
+//   | a1 | b1 | c1 | .
+//   +----+----+----+
+//     ..     ..
+//
+// and the RHS be:
+//
+//   +----+----+----+----+
+//   | p0 | p1 | p2 | p3 | .
+//   +----+----+----+----+ .
+//   | q0 | q1 | q2 | q3 | .
+//   +----+----+----+----+
+//   | r0 | r1 | r2 | r3 | .
+//   +----+----+----+----+ .
+//     ......    ......
+//
+// and let tile_size_m=2, tile_size_k=3 and the vector width (implicitly denoted
+// by `vsl`) be 4.  Then we want to matrix multiply this tile to get a [2,4]
+// matrix that we can increment the result matrix by.
+//
+// First broadcast the rows row in LHS to 3 vectors of width 4, giving us a rank
+// 3 array, L, of dimension [2,3,4]:
+//
+//       L[0,_,_]           *      L[1,_,_]
+//                          *
+//   +----+----+----+----+  *  +----+----+----+----+
+//   | a0 | a0 | a0 | a0 |  *  | a1 | a1 | a1 | a1 |
+//   +----+----+----+----+  *  +----+----+----+----+
+//   | b0 | b0 | b0 | b0 |  *  | b1 | b1 | b1 | b1 |
+//   +----+----+----+----+  *  +----+----+----+----+
+//   | c0 | c0 | c0 | c0 |  *  | c1 | c1 | c1 | c1 |
+//   +----+----+----+----+  *  +----+----+----+----+
+//
+//
+// Then we FMA L[0,_,_] with the RHS to get the first row of the result and
+// L[1,_,_] with the RHS to get the second row of the result.  For example,
+// L[0,_,_] is computed as:
+//
+//   +----+----+----+----+   +----+----+----+----+
+//   | a0 | a0 | a0 | a0 | * | p0 | p1 | p2 | p3 |   +
+//   +----+----+----+----+   +----+----+----+----+
+//
+//   +----+----+----+----+   +----+----+----+----+
+//   | b0 | b0 | b0 | b0 | * | q0 | q1 | q2 | q3 |   +
+//   +----+----+----+----+   +----+----+----+----+
+//
+//   +----+----+----+----+   +----+----+----+----+
+//   | c0 | c0 | c0 | c0 | * | r0 | r1 | r2 | r3 |
+//   +----+----+----+----+   +----+----+----+----+
+//
+// to get:
+//
+//   +-------------------+-------------------+-------------------+---------
+//   | a0*p0+b0*q0+c0*r0 | a0*p1+b0*q1+c0*r1 | a0*p2+b0*q2+c0*r2 |  ...
+//   +-------------------+-------------------+-------------------+---------
+void TiledSmallGemmEmitter::EmitTiledGemm(
+    VectorSupportLibrary* vsl, int64 tile_size_k, llvm::Value* k_start,
+    llvm::Value* k_end, llvm::Value* n_start, llvm::Value* n_end,
+    int64 tile_size_m, llvm::Value* m_start, llvm::Value* m_end) {
+  ksl_.For("dot.m", m_start, m_end, tile_size_m, [&](llvm::Value* m_i) {
+    MemoryTile result_memory_tile(vsl, b_, /*matrix=*/result_,
+                                  /*matrix_size_along_minor_dim=*/dims().n(),
+                                  /*major_dim_offset=*/m_i,
+                                  /*tile_size_along_major_dim=*/tile_size_m);
+    MemoryTile lhs_memory_tile(vsl, b_, /*matrix=*/lhs_,
+                               /*matrix_size_along_minor_dim=*/dims().k(),
+                               /*major_dim_offset=*/m_i,
+                               /*tile_size_along_major_dim=*/tile_size_m);
+    ksl_.For(
+        "dot.n", n_start, n_end, vsl->vector_size(), [&](llvm::Value* n_i) {
+          TileVariable result_tile_var(vsl, result_memory_tile.LoadTile(n_i));
+          ksl_.For("dot.k", k_start, k_end, tile_size_k, [&](llvm::Value* k_i) {
+            MemoryTile rhs_memory_tile(vsl, b_, rhs_, dims().n(), k_i,
+                                       tile_size_k);
+            std::vector<std::vector<llvm::Value*>> lhs_tile =
+                lhs_memory_tile.LoadBroadcastTile(k_i, tile_size_k);
+            std::vector<llvm::Value*> rhs_tile = rhs_memory_tile.LoadTile(n_i);
+            std::vector<llvm::Value*> result_tile = result_tile_var.Get();
+            for (int64 r_m_i = 0; r_m_i < tile_size_m; r_m_i++) {
+              for (int64 r_k_i = 0; r_k_i < tile_size_k; r_k_i++) {
+                result_tile[r_m_i] =
+                    vsl->MulAdd(lhs_tile[r_m_i][r_k_i], rhs_tile[r_k_i],
+                                result_tile[r_m_i]);
+              }
+            }
+            result_tile_var.Set(result_tile);
+          });
+
+          result_memory_tile.StoreTile(result_tile_var.Get(), n_i);
+        });
+  });
+}
+
+}  // namespace
+
+void EmitRowMajorGemv(PrimitiveType scalar_type, int64 tile_rows,
+                      int64 tile_cols, int64 m, int64 k, llvm::Value* lhs,
+                      llvm::Value* rhs, llvm::Value* addend,
+                      llvm::Value* result, llvm::IRBuilder<>* b,
+                      bool enable_fast_math, bool optimize_for_size) {
+  RowMajorMatrixVectorProductEmitter::Config config(
+      /*scalar_type=*/scalar_type,
+      /*tile_rows=*/tile_rows, /*tile_cols=*/tile_cols,
+      /*m=*/m, /*k=*/k, /*has_addend=*/addend != nullptr);
+
+  KernelSupportLibrary::EmitAndCallOutlinedKernel(
+      /*enable_fast_math=*/enable_fast_math,
+      /*optimize_for_size=*/optimize_for_size, b, config.GetCacheKey(), lhs,
+      rhs, addend, result,
+      [&](llvm::Value* lhs, llvm::Value* rhs, llvm::Value* addend,
+          llvm::Value* result) {
+        RowMajorMatrixVectorProductEmitter emitter(config, lhs, rhs, addend,
+                                                   result, b);
+        emitter.Emit();
+      });
+}
+
+void EmitColumnMajorGemv(PrimitiveType scalar_type, int64 tile_rows,
+                         int64 tile_cols, int64 m, int64 k, llvm::Value* lhs,
+                         llvm::Value* rhs, llvm::Value* addend,
+                         llvm::Value* result, llvm::IRBuilder<>* b,
+                         bool enable_fast_math, bool optimize_for_size) {
+  ColumnMajorMatrixVectorProductEmitter::Config config(
+      /*scalar_type=*/scalar_type,
+      /*tile_rows=*/tile_rows, /*tile_cols=*/tile_cols,
+      /*m=*/m, /*k=*/k, /*has_addend=*/addend != nullptr);
+
+  KernelSupportLibrary::EmitAndCallOutlinedKernel(
+      /*enable_fast_math=*/enable_fast_math,
+      /*optimize_for_size=*/optimize_for_size, b, config.GetCacheKey(), lhs,
+      rhs, addend, result,
+      [&](llvm::Value* lhs, llvm::Value* rhs, llvm::Value* addend,
+          llvm::Value* result) {
+        ColumnMajorMatrixVectorProductEmitter emitter(config, lhs, rhs, addend,
+                                                      result, b);
+        emitter.Emit();
+      });
+}
+
+void EmitSmallGemm(PrimitiveType scalar_type, int64 m, int64 k, int64 n,
+                   int64 max_vectorization_width, int64 max_vector_count,
+                   int64 min_vectorization_width, int64 tile_size_m,
+                   int64 tile_size_k, llvm::Value* lhs, llvm::Value* rhs,
+                   llvm::Value* result, llvm::IRBuilder<>* b,
+                   bool enable_fast_math, bool optimize_for_size) {
+  TiledSmallGemmEmitter::Config config(
+      /*scalar_type=*/scalar_type,
+      TiledSmallGemmEmitter::Dimensions{/*m=*/m, /*k=*/k, /*n=*/n},
+      /*max_vectorization_width=*/max_vectorization_width,
+      /*max_vector_count=*/max_vector_count,
+      /*min_vectorization_width=*/min_vectorization_width,
+      /*tile_size_m=*/tile_size_m, /*tile_size_k=*/tile_size_k);
+
+  KernelSupportLibrary::EmitAndCallOutlinedKernel(
+      /*enable_fast_math=*/enable_fast_math,
+      /*optimize_for_size=*/optimize_for_size, b, config.GetCacheKey(), lhs,
+      rhs, result,
+      [&](llvm::Value* lhs, llvm::Value* rhs, llvm::Value* result) {
+        TiledSmallGemmEmitter small_gemm_emitter(config, /*lhs=*/lhs,
+                                                 /*rhs=*/rhs,
+                                                 /*result=*/result, b);
+        small_gemm_emitter.Emit();
+      });
+}
+
+}  // namespace cpu
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/cpu/tiled_dot_emitter.h b/tensorflow/compiler/xla/service/cpu/tiled_dot_emitter.h
new file mode 100644
index 0000000000..0a82326cc3
--- /dev/null
+++ b/tensorflow/compiler/xla/service/cpu/tiled_dot_emitter.h
@@ -0,0 +1,55 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_CPU_TILED_DOT_EMITTER_H_
+#define TENSORFLOW_COMPILER_XLA_SERVICE_CPU_TILED_DOT_EMITTER_H_
+
+#include "llvm/IR/IRBuilder.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
+#include "tensorflow/core/platform/types.h"
+
+namespace xla {
+namespace cpu {
+
+// These routines emit LLVM IR implementing tiled GEMM and GEMV routines.
+
+void EmitRowMajorGemv(PrimitiveType scalar_type, tensorflow::int64 tile_rows,
+                      tensorflow::int64 tile_cols, tensorflow::int64 m,
+                      tensorflow::int64 k, llvm::Value* lhs, llvm::Value* rhs,
+                      llvm::Value* addend, llvm::Value* result,
+                      llvm::IRBuilder<>* b, bool enable_fast_math,
+                      bool optimize_for_size);
+
+void EmitColumnMajorGemv(PrimitiveType scalar_type, tensorflow::int64 tile_rows,
+                         tensorflow::int64 tile_cols, tensorflow::int64 m,
+                         tensorflow::int64 k, llvm::Value* lhs,
+                         llvm::Value* rhs, llvm::Value* addend,
+                         llvm::Value* result, llvm::IRBuilder<>* b,
+                         bool enable_fast_math, bool optimize_for_size);
+
+void EmitSmallGemm(PrimitiveType scalar_type, tensorflow::int64 m,
+                   tensorflow::int64 k, tensorflow::int64 n,
+                   tensorflow::int64 max_vectorization_width,
+                   tensorflow::int64 max_vector_count,
+                   tensorflow::int64 min_vectorization_width,
+                   tensorflow::int64 tile_size_m, tensorflow::int64 tile_size_k,
+                   llvm::Value* lhs, llvm::Value* rhs, llvm::Value* result,
+                   llvm::IRBuilder<>* b, bool enable_fast_math,
+                   bool optimize_for_size);
+
+}  // namespace cpu
+}  // namespace xla
+
+#endif  // TENSORFLOW_COMPILER_XLA_SERVICE_CPU_TILED_DOT_EMITTER_H_
-- 
GitLab


From 0ebce89605326c602b3dd4d8cfe3f3a29284b390 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 7 Jan 2019 14:39:51 -0800
Subject: [PATCH 0327/1095] Optimize test combinations to prevent timeout
 during testing.

PiperOrigin-RevId: 228238454
---
 tensorflow/contrib/distribute/python/BUILD | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index de6b6f1f84..7ee12812af 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -674,7 +674,9 @@ cuda_py_test(
     additional_deps = [
         ":keras_correctness_test_lib",
     ],
-    shard_count = 16,
+    # Shard count is set to an odd number to distribute tasks across
+    # shards more evenly.
+    shard_count = 19,
     tags = [
         "multi_and_single_gpu",
         "no_oss",  # TODO(b/117919883): Fix python error.
-- 
GitLab


From 67a7f04d010c50fb3d6d1bde7793d3f8d350decf Mon Sep 17 00:00:00 2001
From: Russell Power <power@google.com>
Date: Mon, 7 Jan 2019 14:43:32 -0800
Subject: [PATCH 0328/1095] Graph partitioning: use a sensible hash function
 for device memory info.

PiperOrigin-RevId: 228239154
---
 tensorflow/core/BUILD                    |  2 ++
 tensorflow/core/graph/graph_partition.cc | 44 +++++++++---------------
 2 files changed, 18 insertions(+), 28 deletions(-)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 5e6c8d4dd8..e6af9211b5 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -1750,6 +1750,7 @@ cc_library(
 cc_library(
     name = "mobile_additional_lib_deps",
     deps = tf_additional_lib_deps() + [
+        "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/strings",
     ],
@@ -2817,6 +2818,7 @@ tf_cuda_library(
         ":proto_text",
         ":protos_all_cc",
         "//third_party/eigen3",
+        "@com_google_absl//absl/container:flat_hash_map",
     ],
 )
 
diff --git a/tensorflow/core/graph/graph_partition.cc b/tensorflow/core/graph/graph_partition.cc
index f213eb7c10..be0cac50a1 100644
--- a/tensorflow/core/graph/graph_partition.cc
+++ b/tensorflow/core/graph/graph_partition.cc
@@ -22,6 +22,7 @@ limitations under the License.
 #include <utility>
 #include <vector>
 
+#include "absl/container/flat_hash_map.h"
 #include "tensorflow/core/framework/function.h"
 #include "tensorflow/core/framework/memory_types.h"
 #include "tensorflow/core/framework/node_def_builder.h"
@@ -58,29 +59,24 @@ struct DupRecvKey {
   int src_output_slot;       // Edge's src node output slot
   GraphDef* dst_graph;       // Edge's dst node is in this subgraph
   bool recv_output_on_host;  // The output of recv is on host
-};
 
-struct DupRecvKeyHash {
-  size_t operator()(const DupRecvKey& k) const {
-    size_t h = Hash64(reinterpret_cast<const char*>(&k.src_node_id),
-                      sizeof(k.src_node_id), k.src_output_slot);
-    h = Hash64(reinterpret_cast<const char*>(&k.dst_graph), sizeof(k.dst_graph),
-               h);
-    h = Hash64(reinterpret_cast<const char*>(&k.recv_output_on_host),
-               sizeof(k.recv_output_on_host), h);
-    return h;
+  template <typename H>
+  friend H AbslHashValue(H h, const DupRecvKey& c) {
+    return H::combine(std::move(h), c.src_node_id, c.src_output_slot,
+                      reinterpret_cast<std::uintptr_t>(c.dst_graph),
+                      c.recv_output_on_host);
   }
-};
 
-struct DupRecvKeyEq {
-  bool operator()(const DupRecvKey& x, const DupRecvKey& y) const {
-    return (x.src_node_id == y.src_node_id) &&
-           (x.src_output_slot == y.src_output_slot) &&
-           (x.dst_graph == y.dst_graph) &&
-           (x.recv_output_on_host == y.recv_output_on_host);
-  }
+  friend bool operator==(const DupRecvKey& x, const DupRecvKey& y);
 };
 
+bool operator==(const DupRecvKey& x, const DupRecvKey& y) {
+  return (x.src_node_id == y.src_node_id) &&
+         (x.src_output_slot == y.src_output_slot) &&
+         (x.dst_graph == y.dst_graph) &&
+         (x.recv_output_on_host == y.recv_output_on_host);
+}
+
 // struct used to store the recvs, so that start times can be properly updated
 struct RecvInfo {
   NodeDef* recv;
@@ -88,19 +84,11 @@ struct RecvInfo {
   int64 start_time;
 };
 
-typedef std::unordered_map<DupRecvKey, RecvInfo, DupRecvKeyHash, DupRecvKeyEq>
-    DupRecvTable;
+typedef absl::flat_hash_map<DupRecvKey, RecvInfo> DupRecvTable;
 
-struct PairIntHash {
- public:
-  std::size_t operator()(const std::pair<int, int>& x) const {
-    return std::hash<int>()(x.first) ^ std::hash<int>()(x.second);
-  }
-};
 // A map used to store memory types for the inputs/outputs of every node.
 // The key is a pair of ints consisting of a node id and input/output index.
-typedef std::unordered_map<std::pair<int, int>, MemoryType, PairIntHash>
-    MemoryTypeMap;
+typedef absl::flat_hash_map<std::pair<int, int>, MemoryType> MemoryTypeMap;
 
 // We collect the following information about the graph before performing
 // graph partitioning.
-- 
GitLab


From 106b3c183546d2f4bde8c209ef9d01ba0fb2904b Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Mon, 7 Jan 2019 14:55:46 -0800
Subject: [PATCH 0329/1095] [tf.data] Implementation of a "standalone" C++ API
 that encapsulates TensorFlow runtime internals. The new API facilitates
 efficient execution of a dataset input pipeline (represented as a GraphDef)
 in C++.

PiperOrigin-RevId: 228241484
---
 tensorflow/core/common_runtime/data/BUILD     |  35 ++++
 .../core/common_runtime/data/standalone.cc    | 128 ++++++++++++
 .../core/common_runtime/data/standalone.h     | 122 ++++++++++++
 .../common_runtime/data/standalone_test.cc    | 188 ++++++++++++++++++
 4 files changed, 473 insertions(+)
 create mode 100644 tensorflow/core/common_runtime/data/BUILD
 create mode 100644 tensorflow/core/common_runtime/data/standalone.cc
 create mode 100644 tensorflow/core/common_runtime/data/standalone.h
 create mode 100644 tensorflow/core/common_runtime/data/standalone_test.cc

diff --git a/tensorflow/core/common_runtime/data/BUILD b/tensorflow/core/common_runtime/data/BUILD
new file mode 100644
index 0000000000..124862dbb7
--- /dev/null
+++ b/tensorflow/core/common_runtime/data/BUILD
@@ -0,0 +1,35 @@
+package(
+    default_visibility = [
+        "//tensorflow:internal",
+        "//tensorflow_models:__subpackages__",
+    ],
+)
+
+licenses(["notice"])  # Apache 2.0
+
+load("//tensorflow:tensorflow.bzl", "tf_cc_test")
+load("//tensorflow/core:platform/default/build_config.bzl", "tf_protos_all")
+
+cc_library(
+    name = "standalone",
+    srcs = ["standalone.cc"],
+    hdrs = ["standalone.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//tensorflow/core:core_cpu_internal",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:session_options",
+    ],
+)
+
+tf_cc_test(
+    name = "standalone_test",
+    srcs = ["standalone_test.cc"],
+    deps = [
+        ":standalone",
+        "//tensorflow/core:all_kernels",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ] + tf_protos_all(),
+)
diff --git a/tensorflow/core/common_runtime/data/standalone.cc b/tensorflow/core/common_runtime/data/standalone.cc
new file mode 100644
index 0000000000..b05bff566f
--- /dev/null
+++ b/tensorflow/core/common_runtime/data/standalone.cc
@@ -0,0 +1,128 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/common_runtime/data/standalone.h"
+
+#include <memory>
+
+#include "tensorflow/core/common_runtime/device_factory.h"
+#include "tensorflow/core/common_runtime/device_mgr.h"
+#include "tensorflow/core/common_runtime/function.h"
+#include "tensorflow/core/common_runtime/graph_runner.h"
+#include "tensorflow/core/common_runtime/process_util.h"
+#include "tensorflow/core/framework/dataset.h"
+#include "tensorflow/core/graph/graph.h"
+#include "tensorflow/core/graph/graph_constructor.h"
+#include "tensorflow/core/public/version.h"
+#include "tensorflow/core/util/ptr_util.h"
+
+namespace tensorflow {
+namespace data {
+namespace standalone {
+
+Status Iterator::GetNext(std::vector<Tensor>* outputs, bool* end_of_input) {
+  return iterator_->GetNext(ctx_.get(), outputs, end_of_input);
+}
+
+Iterator::Iterator(IteratorBase* iterator, IteratorContext* ctx)
+    : iterator_(iterator), ctx_(ctx) {}
+
+Status Dataset::FromGraph(Params params, const GraphDef& graph_def,
+                          const string& fetch_node,
+                          std::unique_ptr<Dataset>* result) {
+  Graph graph(OpRegistry::Global());
+  TF_RETURN_IF_ERROR(ImportGraphDef({}, graph_def, &graph, nullptr));
+
+  // Instantiate enough of the TensorFlow runtime to run `graph` on a single CPU
+  // device.
+  std::unique_ptr<DeviceMgr> device_mgr = MakeUnique<DeviceMgr>(
+      DeviceFactory::NewDevice("CPU", params.session_options, ""));
+  Device* device = device_mgr->ListDevices()[0];
+  // Clone the `FunctionLibraryDefinition` to extend its lifetime extends beyond
+  // the lifetime of `graph`.
+  std::unique_ptr<FunctionLibraryDefinition> flib_def =
+      MakeUnique<FunctionLibraryDefinition>(graph.flib_def());
+  std::unique_ptr<ProcessFunctionLibraryRuntime> pflr =
+      MakeUnique<ProcessFunctionLibraryRuntime>(
+          device_mgr.get(), Env::Default(), TF_GRAPH_DEF_VERSION,
+          flib_def.get(), OptimizerOptions{}, nullptr /* parent */);
+
+  // Run graph up to `output_node` and extract the `DatasetBase` stored in the
+  // DT_VARIANT output tensor.
+  data::DatasetBase* dataset;
+  {
+    std::vector<Tensor> outputs;
+    GraphRunner graph_runner(device);
+    TF_RETURN_IF_ERROR(graph_runner.Run(&graph, pflr->GetFLR("/device:CPU:0"),
+                                        {}, {fetch_node}, &outputs));
+    TF_RETURN_IF_ERROR(GetDatasetFromVariantTensor(outputs[0], &dataset));
+    // NOTE(mrry): The dataset is currently owned by `outputs[0]`, so acquire an
+    // additional reference.
+    dataset->Ref();
+  }
+
+  std::unique_ptr<thread::ThreadPool> pool(
+      NewThreadPoolFromSessionOptions(params.session_options));
+  *result =
+      WrapUnique(new Dataset(dataset, device_mgr.release(), pflr.release(),
+                             flib_def.release(), pool.release()));
+  return Status::OK();
+}  // static
+
+Status Dataset::MakeIterator(std::unique_ptr<Iterator>* result) {
+  // Create an `IteratorContext`, which bundles together the necessary runtime
+  // support to create and get elements from an iterator.
+  std::unique_ptr<IteratorContext> ctx;
+  {
+    // NOTE(mrry): In the current API, an `IteratorContext` is always initially
+    // created from an `OpKernelContext*`, so we need to create a fake
+    // `OpKernelContext` with the appropriate subset of parameters.
+    OpKernelContext::Params op_params;
+    op_params.function_library = pflr_->GetFLR("/device:CPU:0");
+    op_params.device = device_mgr_->ListDevices()[0];
+    op_params.runner = &runner_;
+    OpKernelContext op_ctx(&op_params, 0);
+    IteratorContext::Params params(&op_ctx);
+    params.function_handle_cache = function_handle_cache_.get();
+    ctx = MakeUnique<IteratorContext>(std::move(params));
+  }
+
+  // Create the iterator from the dataset.
+  std::unique_ptr<IteratorBase> iterator;
+  TF_RETURN_IF_ERROR(dataset_->MakeIterator(ctx.get(), "iterator", &iterator));
+
+  *result = WrapUnique(new Iterator(iterator.release(), ctx.release()));
+
+  return Status::OK();
+}
+
+Dataset::Dataset(DatasetBase* dataset, DeviceMgr* device_mgr,
+                 ProcessFunctionLibraryRuntime* pflr,
+                 FunctionLibraryDefinition* flib_def, thread::ThreadPool* pool)
+    : dataset_(dataset),
+      device_mgr_(device_mgr),
+      flib_def_(flib_def),
+      pflr_(pflr),
+      pool_(pool) {
+  runner_ = [this](std::function<void()> c) { pool_->Schedule(std::move(c)); };
+  function_handle_cache_ =
+      MakeUnique<FunctionHandleCache>(pflr_->GetFLR("/device:CPU:0"));
+}
+
+Dataset::~Dataset() { dataset_->Unref(); }
+
+}  // namespace standalone
+}  // namespace data
+}  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/data/standalone.h b/tensorflow/core/common_runtime/data/standalone.h
new file mode 100644
index 0000000000..ecea5ba21d
--- /dev/null
+++ b/tensorflow/core/common_runtime/data/standalone.h
@@ -0,0 +1,122 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_DATA_STANDALONE_H_
+#define TENSORFLOW_CORE_COMMON_RUNTIME_DATA_STANDALONE_H_
+
+#include <memory>
+#include "tensorflow/core/common_runtime/device_mgr.h"
+#include "tensorflow/core/framework/dataset.h"
+#include "tensorflow/core/framework/function_handle_cache.h"
+#include "tensorflow/core/lib/core/threadpool.h"
+#include "tensorflow/core/public/session_options.h"
+
+namespace tensorflow {
+namespace data {
+namespace standalone {
+
+// The purpose of the API in this file is to facilitate standalone execution of
+// a tf.data input pipeline graph.
+//
+// The API exposes two abstractions -- a `Dataset` and an `Iterator` -- which
+// encapsulate TensorFlow runtime.
+//
+// The `Dataset` abstraction represents an input pipeline as a collection
+// of data sources and a logical plan of transformations that operate over the
+// data.
+//
+// The `Iterator` abstraction represents an execution of an input pipeline that
+// can be used to enumerate its elements.
+//
+// Example usage:
+//
+//   // Create a `Dataset` by running the `graph_def` graph and fetching the
+//   // output of the `fetch_node` node.
+//   tensorflow::data:standalone::Dataset::Params params;
+//   std::unique_ptr<tensorflow::data::standalone::Dataset> dataset;
+//   Status s = tensorflow::data::standalone::Dataset::FromGraph(
+//      params, graph_def, fetch_node, &dataset);
+//   if (!s.ok()) { /* error handling */ }
+//
+//   std::unique_ptr<tensorflow::data::standalone::Iterator> iterator;
+//   s = dataset->MakeIterator(&iterator);
+//   if (!s.ok()) { /* error handling */ }
+//
+//   bool end_of_input = false;
+//   while (!end_of_input) {
+//     std::vector<tensorflow::Tensor> outputs;
+//     s = iterator->GetNext(&outputs, &end_of_input);
+//     if (!s.ok()) { /* error handling */ }
+//     if (!end_of_input) { /* output handling */ }
+//   }
+
+class Dataset;
+
+// Represents an execution of an input pipeline that can be used to enumerate
+// its elements.
+class Iterator {
+ public:
+  // Returns the next element of the input pipeline (if there is one) and an
+  // indication of whether the end of the input pipeline has been reached.
+  Status GetNext(std::vector<Tensor>* outputs, bool* end_of_input);
+
+ private:
+  friend class Dataset;
+
+  Iterator(IteratorBase* iterator, IteratorContext* ctx);
+
+  std::unique_ptr<IteratorBase> iterator_;
+  std::unique_ptr<IteratorContext> ctx_;
+};
+
+// Represents an input pipeline as a collection of data sources and a logical
+// plan of transformations that operate over the data.
+class Dataset {
+ public:
+  // Parameters for `Dataset` creation (e.g. TensorFlow runtime configuration).
+  struct Params {
+    SessionOptions session_options;
+  };
+
+  // Creates a new `Dataset` instance by running the TensorFlow graph `graph`
+  // and fetching the output of the `fetch_node` node.
+  static Status FromGraph(Params params, const GraphDef& graph_def,
+                          const string& fetch_node,
+                          std::unique_ptr<Dataset>* result);
+
+  ~Dataset();
+
+  // Creates an iterator for this dataset.
+  Status MakeIterator(std::unique_ptr<Iterator>* result);
+
+ private:
+  Dataset(DatasetBase* dataset, DeviceMgr* device_mgr,
+          ProcessFunctionLibraryRuntime* pflr,
+          FunctionLibraryDefinition* flib_def, thread::ThreadPool* pool);
+
+  DatasetBase* dataset_;  // owned
+  std::unique_ptr<DeviceMgr> device_mgr_;
+  std::unique_ptr<FunctionLibraryDefinition> flib_def_;
+  std::unique_ptr<ProcessFunctionLibraryRuntime> pflr_;
+  std::unique_ptr<thread::ThreadPool> pool_;
+  std::unique_ptr<FunctionHandleCache> function_handle_cache_;
+  std::function<void(std::function<void()>)> runner_;
+};
+
+}  // namespace standalone
+}  // namespace data
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_COMMON_RUNTIME_DATA_STANDALONE_H_
diff --git a/tensorflow/core/common_runtime/data/standalone_test.cc b/tensorflow/core/common_runtime/data/standalone_test.cc
new file mode 100644
index 0000000000..7e7a7a9b61
--- /dev/null
+++ b/tensorflow/core/common_runtime/data/standalone_test.cc
@@ -0,0 +1,188 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <memory>
+#include <vector>
+
+#include "tensorflow/core/common_runtime/data/standalone.h"
+#include "tensorflow/core/framework/graph.pb.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace tensorflow {
+namespace data {
+namespace standalone {
+namespace {
+
+constexpr const char* const kGraphProto = R"proto(
+  node {
+    name: "Const/_0"
+    op: "Const"
+    attr {
+      key: "dtype"
+      value { type: DT_INT64 }
+    }
+    attr {
+      key: "value"
+      value {
+        tensor {
+          dtype: DT_INT64
+          tensor_shape {}
+          int64_val: 0
+        }
+      }
+    }
+  }
+  node {
+    name: "Const/_1"
+    op: "Const"
+    attr {
+      key: "dtype"
+      value { type: DT_INT64 }
+    }
+    attr {
+      key: "value"
+      value {
+        tensor {
+          dtype: DT_INT64
+          tensor_shape {}
+          int64_val: 10
+        }
+      }
+    }
+  }
+  node {
+    name: "Const/_2"
+    op: "Const"
+    attr {
+      key: "dtype"
+      value { type: DT_INT64 }
+    }
+    attr {
+      key: "value"
+      value {
+        tensor {
+          dtype: DT_INT64
+          tensor_shape {}
+          int64_val: 1
+        }
+      }
+    }
+  }
+  node {
+    name: "RangeDataset/_3"
+    op: "RangeDataset"
+    input: "Const/_0"
+    input: "Const/_1"
+    input: "Const/_2"
+    attr {
+      key: "output_shapes"
+      value { list { shape { unknown_rank: true } } }
+    }
+    attr {
+      key: "output_types"
+      value { list { type: DT_INT64 } }
+    }
+  }
+  node {
+    name: "MapDataset/_4"
+    op: "MapDataset"
+    input: "RangeDataset/_3"
+    attr {
+      key: "Targuments"
+      value { list {} }
+    }
+    attr {
+      key: "f"
+      value { func { name: "Dataset_map_<lambda>_10" } }
+    }
+    attr {
+      key: "output_shapes"
+      value { list { shape {} } }
+    }
+    attr {
+      key: "output_types"
+      value { list { type: DT_INT64 } }
+    }
+    attr {
+      key: "preserve_cardinality"
+      value { b: false }
+    }
+    attr {
+      key: "use_inter_op_parallelism"
+      value { b: true }
+    }
+  }
+  library {
+    function {
+      signature {
+        name: "Dataset_map_<lambda>_10"
+        input_arg { name: "arg0" type: DT_INT64 }
+        output_arg { name: "mul" type: DT_INT64 }
+        description: "Wrapper for passing nested structures to and from tf.data functions."
+      }
+      node_def {
+        name: "mul_0"
+        op: "Mul"
+        input: "arg0"
+        input: "arg0"
+        attr {
+          key: "T"
+          value { type: DT_INT64 }
+        }
+      }
+      ret { key: "mul" value: "mul_0:z:0" }
+    }
+  }
+  versions { producer: 27 min_consumer: 12 }
+)proto";
+
+TEST(Scalar, Standalone) {
+  GraphDef graph_def;
+  protobuf::TextFormat::ParseFromString(kGraphProto, &graph_def);
+  struct TestCase {
+    string fetch_node;
+    std::vector<int64> expected_outputs;
+  };
+  auto test_cases = {
+      TestCase{"RangeDataset/_3", {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}},
+      TestCase{"MapDataset/_4", {0, 1, 4, 9, 16, 25, 36, 49, 64, 81}},
+  };
+  for (auto test_case : test_cases) {
+    std::unique_ptr<Dataset> dataset;
+    auto s = Dataset::FromGraph({}, graph_def, test_case.fetch_node, &dataset);
+    TF_EXPECT_OK(s);
+    std::unique_ptr<Iterator> iterator;
+    s = dataset->MakeIterator(&iterator);
+    TF_EXPECT_OK(s);
+    bool end_of_input = false;
+    for (int num_outputs = 0; !end_of_input; ++num_outputs) {
+      std::vector<tensorflow::Tensor> outputs;
+      s = iterator->GetNext(&outputs, &end_of_input);
+      TF_EXPECT_OK(s);
+      if (!end_of_input) {
+        EXPECT_EQ(outputs[0].scalar<int64>()(),
+                  test_case.expected_outputs[num_outputs]);
+      } else {
+        EXPECT_EQ(test_case.expected_outputs.size(), num_outputs);
+      }
+    }
+  }
+}
+
+}  // namespace
+}  // namespace standalone
+}  // namespace data
+}  // namespace tensorflow
-- 
GitLab


From 17efadd4556c242efaa68b30134d3e79d2412922 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 7 Jan 2019 14:57:23 -0800
Subject: [PATCH 0330/1095] Internal copybara changes.

PiperOrigin-RevId: 228241768
---
 tensorflow/lite/experimental/swift/BUILD      |  1 +
 .../Configs/TensorFlowLite.tulsigen           | 20 +++++++++----------
 .../project.tulsiconf                         |  4 ++--
 .../Base.lproj/Main.storyboard                |  6 +++---
 4 files changed, 16 insertions(+), 15 deletions(-)

diff --git a/tensorflow/lite/experimental/swift/BUILD b/tensorflow/lite/experimental/swift/BUILD
index 3bd288a1e1..53bcb0ecbd 100644
--- a/tensorflow/lite/experimental/swift/BUILD
+++ b/tensorflow/lite/experimental/swift/BUILD
@@ -83,6 +83,7 @@ ios_application(
 swift_library(
     name = "TensorFlowLiteAppLib",
     srcs = glob(["TestApps/TensorFlowLiteApp/TensorFlowLiteApp/*.swift"]),
+    module_name = "TensorFlowLiteAppLib",
     tags = ["manual"],
     deps = [
         ":TensorFlowLite",
diff --git a/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/Configs/TensorFlowLite.tulsigen b/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/Configs/TensorFlowLite.tulsigen
index 4010fab49e..16bc6cbfe8 100644
--- a/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/Configs/TensorFlowLite.tulsigen
+++ b/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/Configs/TensorFlowLite.tulsigen
@@ -1,16 +1,16 @@
 {
   "sourceFilters" : [
-    "third_party/tensorflow/lite/experimental/c",
-    "third_party/tensorflow/lite/experimental/swift",
-    "third_party/tensorflow/lite/experimental/swift/Sources",
-    "third_party/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp",
-    "third_party/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj",
-    "third_party/tensorflow/lite/experimental/swift/Tests",
+    "tensorflow/lite/experimental/c",
+    "tensorflow/lite/experimental/swift",
+    "tensorflow/lite/experimental/swift/Sources",
+    "tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp",
+    "tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj",
+    "tensorflow/lite/experimental/swift/Tests",
   ],
   "buildTargets" : [
-    "//third_party/tensorflow/lite/experimental/swift:TensorFlowLite",
-    "//third_party/tensorflow/lite/experimental/swift:TensorFlowLiteApp",
-    "//third_party/tensorflow/lite/experimental/swift:TensorFlowLiteTests",
+    "//tensorflow/lite/experimental/swift:TensorFlowLite",
+    "//tensorflow/lite/experimental/swift:TensorFlowLiteApp",
+    "//tensorflow/lite/experimental/swift:TensorFlowLiteTests",
   ],
   "projectName" : "TensorFlowLite",
   "optionSet" : {
@@ -52,6 +52,6 @@
     }
   },
   "additionalFilePaths" : [
-    "third_party/tensorflow/lite/experimental/swift/BUILD"
+    "tensorflow/lite/experimental/swift/BUILD"
   ]
 }
diff --git a/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/project.tulsiconf b/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/project.tulsiconf
index 14cff94453..82ac8aa381 100644
--- a/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/project.tulsiconf
+++ b/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/project.tulsiconf
@@ -8,7 +8,7 @@
   },
   "projectName" : "TensorFlowLite",
   "packages" : [
-    "third_party/tensorflow/lite/experimental/swift"
+    "tensorflow/lite/experimental/swift"
   ],
-  "workspaceRoot" : "../../../../../.."
+  "workspaceRoot" : "../../../../.."
 }
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/Main.storyboard b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/Main.storyboard
index d0e91d63c4..10cae6e855 100644
--- a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/Main.storyboard
+++ b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/Main.storyboard
@@ -1,17 +1,17 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="14313.18" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" useTraitCollections="YES" colorMatched="YES" initialViewController="BYZ-38-t0r">
+<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="14460.31" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" useTraitCollections="YES" colorMatched="YES" initialViewController="BYZ-38-t0r">
     <device id="retina4_7" orientation="portrait">
         <adaptation id="fullscreen"/>
     </device>
     <dependencies>
-        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="14283.14"/>
+        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="14460.20"/>
         <capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
     </dependencies>
     <scenes>
         <!--View Controller-->
         <scene sceneID="tne-QT-ifu">
             <objects>
-                <viewController storyboardIdentifier="viewController" useStoryboardIdentifierAsRestorationIdentifier="YES" id="BYZ-38-t0r" customClass="ViewController" customModule="third_party_tensorflow_lite_experimental_swift_TensorFlowLiteAppLib" sceneMemberID="viewController">
+                <viewController storyboardIdentifier="viewController" useStoryboardIdentifierAsRestorationIdentifier="YES" id="BYZ-38-t0r" customClass="ViewController" customModule="TensorFlowLiteAppLib" sceneMemberID="viewController">
                     <layoutGuides>
                         <viewControllerLayoutGuide type="top" id="y3c-jy-aDJ"/>
                         <viewControllerLayoutGuide type="bottom" id="wfy-db-euE"/>
-- 
GitLab


From a0dfedcca7c766404735720b9d675e1a04f40350 Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Mon, 7 Jan 2019 15:02:16 -0800
Subject: [PATCH 0331/1095] [XLA] Use DynamicDimensionInference in HloEvaluator

- Use DynamicDimensionInference in HloEvaluator so that it can correctly evaluate GetDimensionSize.
- Change ComputeConstant to first calculate DynamicDimensionInference and pass it into evaluator.

PiperOrigin-RevId: 228242693
---
 tensorflow/compiler/xla/service/BUILD         |  2 +
 .../compiler/xla/service/hlo_evaluator.cc     | 34 ++++++++++++++++
 .../compiler/xla/service/hlo_evaluator.h      | 12 ++++++
 .../xla/service/hlo_evaluator_test.cc         | 40 +++++++++++++++++--
 tensorflow/compiler/xla/service/service.cc    |  5 +++
 5 files changed, 89 insertions(+), 4 deletions(-)

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index 3c18c450e6..9bc6218f75 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -236,6 +236,7 @@ cc_library(
     ],
     hdrs = ["hlo_evaluator.h"],
     deps = [
+        ":dynamic_dimension_inference",
         ":hlo",
         ":hlo_casting_utils",
         ":hlo_query",
@@ -697,6 +698,7 @@ cc_library(
         ":compiler",
         ":computation_layout",
         ":device_memory_allocator",
+        ":dynamic_dimension_inference",
         ":executable",
         ":execution_tracker",
         ":hlo",
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc
index 3a97b56c66..b8d95fb244 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc
@@ -361,6 +361,31 @@ Status HloEvaluator::HandleBitcast(HloInstruction* bitcast) {
   return Status::OK();
 }
 
+Status HloEvaluator::HandleGetDimensionSize(
+    HloInstruction* get_dimension_size) {
+  HloInstruction* operand = get_dimension_size->mutable_operand(0);
+  int64 dim = get_dimension_size->dimension();
+  if (dynamic_dimension_inference_ == nullptr) {
+    return InvalidArgument(
+        "Evaluator cannot evaluate get_dimension_size without "
+        "set_dynamic_dimension_inference.");
+  }
+  HloInstruction* dynamic_size =
+      dynamic_dimension_inference_->GetDynamicSize(operand, {}, dim);
+  if (dynamic_size != nullptr) {
+    evaluated_[get_dimension_size] =
+        GetEvaluatedLiteralFor(dynamic_size).Clone();
+    return Status::OK();
+  }
+
+  const Shape& shape = get_dimension_size->operand(0)->shape();
+  Literal output(ShapeUtil::MakeShape(U32, {}));
+  output.PopulateWithValue(
+      static_cast<uint32>(shape.dimensions(get_dimension_size->dimension())));
+  evaluated_[get_dimension_size] = std::move(output);
+  return Status::OK();
+}
+
 Status HloEvaluator::HandleParameter(HloInstruction* parameter) {
   CHECK_LT(parameter->parameter_number(), arg_literals_.size());
   const Literal* input_literal = arg_literals_[parameter->parameter_number()];
@@ -1080,6 +1105,8 @@ Status HloEvaluator::HandleCall(HloInstruction* call) {
   }
 
   HloEvaluator embedded_evaluator;
+  embedded_evaluator.set_dynamic_dimension_inference(
+      dynamic_dimension_inference_);
   Literal result = embedded_evaluator.Evaluate(*computation, arg_literals)
                        .ConsumeValueOrDie();
 
@@ -1113,6 +1140,8 @@ Status HloEvaluator::HandleFusion(HloInstruction* fusion) {
   }
 
   HloEvaluator embedded_evaluator;
+  embedded_evaluator.set_dynamic_dimension_inference(
+      dynamic_dimension_inference_);
   Literal result =
       embedded_evaluator.Evaluate(*readded_computation, arg_literals)
           .ConsumeValueOrDie();
@@ -1132,6 +1161,8 @@ Status HloEvaluator::HandleConditional(HloInstruction* conditional) {
   auto* false_computation = conditional->false_computation();
 
   HloEvaluator embedded_evaluator;
+  embedded_evaluator.set_dynamic_dimension_inference(
+      dynamic_dimension_inference_);
   Literal result;
   if (pred.Get<bool>({})) {
     result =
@@ -1186,7 +1217,10 @@ Status HloEvaluator::HandleWhile(HloInstruction* while_hlo) {
   bool keep_going = true;
   int64 iteration_count = 0;
   HloEvaluator cond_evaluator(max_loop_iterations_);
+  cond_evaluator.set_dynamic_dimension_inference(dynamic_dimension_inference_);
   HloEvaluator loop_body_evaluator(max_loop_iterations_);
+  loop_body_evaluator.set_dynamic_dimension_inference(
+      dynamic_dimension_inference_);
   while (keep_going) {
     if (max_loop_iterations_ >= 0 && iteration_count++ > max_loop_iterations_) {
       return InvalidArgument("Loop %s exceeded loop iteration limit (%d).",
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.h b/tensorflow/compiler/xla/service/hlo_evaluator.h
index 4fc5d10aed..604b861913 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.h
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.h
@@ -23,6 +23,7 @@ limitations under the License.
 #include "absl/types/span.h"
 #include "tensorflow/compiler/xla/array2d.h"
 #include "tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h"
+#include "tensorflow/compiler/xla/service/dynamic_dimension_inference.h"
 #include "tensorflow/compiler/xla/service/hlo_computation.h"
 #include "tensorflow/compiler/xla/service/hlo_instruction.h"
 #include "tensorflow/compiler/xla/service/hlo_module.h"
@@ -123,6 +124,11 @@ class HloEvaluator : public DfsHloVisitorWithDefault {
                                   const PrecisionConfig& precision_config,
                                   const Literal& lhs, const Literal& rhs);
 
+  void set_dynamic_dimension_inference(
+      DynamicDimensionInference* dynamic_dimension_inference) {
+    dynamic_dimension_inference_ = dynamic_dimension_inference;
+  }
+
   // Enable the fast path for certain operations like dot or convolution.
   void set_use_fast_path(bool value) { use_fast_path_ = value; }
 
@@ -161,6 +167,8 @@ class HloEvaluator : public DfsHloVisitorWithDefault {
   //
   Status HandleBitcast(HloInstruction* bitcast) override;
 
+  Status HandleGetDimensionSize(HloInstruction* get_dimension_size) override;
+
   Status HandleParameter(HloInstruction* parameter) override;
 
   Status HandleConstant(HloInstruction* constant) override;
@@ -295,6 +303,10 @@ class HloEvaluator : public DfsHloVisitorWithDefault {
   // RNG engine.
   std::minstd_rand0 engine_;
 
+  // DynamicDimensionInference is used to evaluate GetDimensionSize, which
+  // returns the dynamic dimension size of its operand.
+  DynamicDimensionInference* dynamic_dimension_inference_;
+
   TF_DISALLOW_COPY_AND_ASSIGN(HloEvaluator);
 };
 
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_test.cc b/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
index 8056193d99..319f0db97a 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
@@ -62,8 +62,7 @@ class HloEvaluatorTest : public HloTestBase {
     if (use_bfloat16_) {
       HloElementTypeConverter(F32, BF16).Run(m_.get()).ValueOrDie();
     }
-    return HloEvaluator()
-        .Evaluate(*m_->entry_computation(), arg_literals)
+    return evaluator_.Evaluate(*m_->entry_computation(), arg_literals)
         .ConsumeValueOrDie();
   }
 
@@ -75,8 +74,7 @@ class HloEvaluatorTest : public HloTestBase {
     if (use_bfloat16_) {
       HloElementTypeConverter(F32, BF16).Run(m_.get()).ValueOrDie();
     }
-    return HloEvaluator()
-        .Evaluate(*module->entry_computation(), arg_literals)
+    return evaluator_.Evaluate(*module->entry_computation(), arg_literals)
         .ConsumeValueOrDie();
   }
 
@@ -115,6 +113,7 @@ class HloEvaluatorTest : public HloTestBase {
 
  protected:
   explicit HloEvaluatorTest(bool use_bfloat16) : use_bfloat16_(use_bfloat16) {}
+  HloEvaluator evaluator_;
 
   const bool use_bfloat16_;
   std::unique_ptr<HloModule> m_ = CreateNewVerifiedModule();
@@ -2874,6 +2873,39 @@ ENTRY main {
             static_cast<int32>(pow31 * pow31));
 }
 
+TEST_F(HloEvaluatorTest, GetDimensionSize) {
+  constexpr absl::string_view hlo_text = R"(
+HloModule Test
+
+ENTRY main {
+  size = u32[] parameter(0)
+
+  data = s32[4] parameter(1)
+
+  sum = s32[4] add(data, data)
+
+  ROOT dynamic_size = u32[] get-dimension-size(sum), dimensions={0}
+}
+)";
+  TF_ASSERT_OK_AND_ASSIGN(m_, ParseAndReturnVerifiedModule(hlo_text));
+
+  // Set up dynamic parameter binding.
+  TF_CHECK_OK(m_->dynamic_parameter_binding().Bind(
+      DynamicParameterBinding::DynamicParameter{0, {}},
+      DynamicParameterBinding::DynamicDimension{1, {}, 0}));
+
+  TF_ASSERT_OK_AND_ASSIGN(DynamicDimensionInference dynamic_dimension_inference,
+                          DynamicDimensionInference::Run(m_.get()));
+
+  evaluator_.set_dynamic_dimension_inference(&dynamic_dimension_inference);
+  Literal size_arg = LiteralUtil::CreateR0<uint32>(3);
+  Literal data_arg = LiteralUtil::CreateR1<int32>({1, 2, 3, 4});
+
+  Literal actual = Evaluate({&size_arg, &data_arg});
+
+  EXPECT_EQ(actual.GetFirstElement<uint32>(), static_cast<uint32>(3));
+}
+
 // Check that we get a useful error if we pass inputs of the wrong shape.
 TEST_F(HloEvaluatorTest, EvaluateWithWrongInputShapes) {
   constexpr absl::string_view hlo_text = R"(
diff --git a/tensorflow/compiler/xla/service/service.cc b/tensorflow/compiler/xla/service/service.cc
index efefe9003f..4a8d272261 100644
--- a/tensorflow/compiler/xla/service/service.cc
+++ b/tensorflow/compiler/xla/service/service.cc
@@ -29,6 +29,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/compiler.h"
 #include "tensorflow/compiler/xla/service/computation_layout.h"
 #include "tensorflow/compiler/xla/service/device_memory_allocator.h"
+#include "tensorflow/compiler/xla/service/dynamic_dimension_inference.h"
 #include "tensorflow/compiler/xla/service/executable.h"
 #include "tensorflow/compiler/xla/service/hlo_computation.h"
 #include "tensorflow/compiler/xla/service/hlo_cost_analysis.h"
@@ -1097,7 +1098,11 @@ Status Service::ComputeConstantGraph(const ComputeConstantGraphRequest* arg,
   TF_ASSIGN_OR_RETURN(std::unique_ptr<HloModule> module,
                       CreateModuleFromProto(arg->computation(), config));
 
+  TF_ASSIGN_OR_RETURN(DynamicDimensionInference dynamic_dimension_inference,
+                      DynamicDimensionInference::Run(module.get()));
+
   HloEvaluator evaluator;
+  evaluator.set_dynamic_dimension_inference(&dynamic_dimension_inference);
   TF_ASSIGN_OR_RETURN(auto result_literal, evaluator.Evaluate(*module, {}));
 
   // Since the result layout is non-effective to the Evaluator results, explicit
-- 
GitLab


From b94da8cca6fc657a77ab920e7feeb05cd57f37d9 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Mon, 7 Jan 2019 15:04:18 -0800
Subject: [PATCH 0332/1095] Print a more ergonomic error message when the CPU
 JIT is not linked in

PiperOrigin-RevId: 228243176
---
 tensorflow/compiler/xla/service/compiler.cc | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/xla/service/compiler.cc b/tensorflow/compiler/xla/service/compiler.cc
index 8f08c24490..653f4555a7 100644
--- a/tensorflow/compiler/xla/service/compiler.cc
+++ b/tensorflow/compiler/xla/service/compiler.cc
@@ -98,10 +98,17 @@ Compiler::GetPlatformCompilers() {
   auto* factories = GetPlatformCompilerFactories();
   auto it = factories->find(platform->id());
   if (it == factories->end()) {
+    string hint;
+    if (platform->Name() == "Host") {
+      hint = " (hint: try linking in tensorflow/compiler/jit:xla_cpu_jit)";
+    } else if (platform->Name() == "CUDA") {
+      hint = " (hint: try linking in tensorflow/compiler/jit:xla_gpu_jit)";
+    }
+
     return NotFound(
         "could not find registered compiler for platform %s -- check "
-        "target linkage",
-        platform->Name());
+        "target linkage%s",
+        platform->Name(), hint);
   }
 
   // And then we invoke the factory, placing the result into the mapping.
-- 
GitLab


From 35938dbf10a7695ace5fa1707b5e6ccdd22869a0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 7 Jan 2019 15:34:00 -0800
Subject: [PATCH 0333/1095] Fixes for flex issues with multiple subgraphs.

PiperOrigin-RevId: 228248671
---
 tensorflow/lite/delegates/flex/buffer_map.h   |  3 ++
 .../lite/delegates/flex/buffer_map_test.cc    |  2 +
 tensorflow/lite/delegates/flex/kernel.cc      | 11 +++++-
 tensorflow/lite/delegates/flex/kernel_test.cc | 37 +++++++------------
 4 files changed, 27 insertions(+), 26 deletions(-)

diff --git a/tensorflow/lite/delegates/flex/buffer_map.h b/tensorflow/lite/delegates/flex/buffer_map.h
index 6c1df4c836..d4724a011d 100644
--- a/tensorflow/lite/delegates/flex/buffer_map.h
+++ b/tensorflow/lite/delegates/flex/buffer_map.h
@@ -66,6 +66,9 @@ class BufferMap {
   // be use by TF's forwarding optimizations.
   void SetForwardable(int tensor_index) { forwardable_.insert(tensor_index); }
 
+  // Removes all information about which tensors are forwardable.
+  void ClearForwardable() { forwardable_.clear(); }
+
   // Returns true if this tensor has been explicitly marks as forwardable by
   // a call to SetForwardable().
   bool IsForwardable(int tensor_index) const {
diff --git a/tensorflow/lite/delegates/flex/buffer_map_test.cc b/tensorflow/lite/delegates/flex/buffer_map_test.cc
index 2148bfe8e2..7b4acbd69d 100644
--- a/tensorflow/lite/delegates/flex/buffer_map_test.cc
+++ b/tensorflow/lite/delegates/flex/buffer_map_test.cc
@@ -278,6 +278,8 @@ TEST(BufferMapTest, Forwardable) {
   EXPECT_FALSE(buffer_map.IsForwardable(0));
   buffer_map.SetForwardable(0);
   EXPECT_TRUE(buffer_map.IsForwardable(0));
+  buffer_map.ClearForwardable();
+  EXPECT_FALSE(buffer_map.IsForwardable(0));
 }
 
 }  // namespace
diff --git a/tensorflow/lite/delegates/flex/kernel.cc b/tensorflow/lite/delegates/flex/kernel.cc
index 0005b6e5b0..84745ba9cb 100644
--- a/tensorflow/lite/delegates/flex/kernel.cc
+++ b/tensorflow/lite/delegates/flex/kernel.cc
@@ -350,7 +350,11 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
         buffer_map->SetFromTfLite(tensor_index, tensor);
       }
     }
-    ++tensor_ref_count[tensor_index];
+
+    // Input tensors should never be forwarded so we increment their ref counts
+    // twice: once for this graph and another for the possibility of them being
+    // used by another subgraph, or being an output of the full graph.
+    tensor_ref_count[tensor_index] += 2;
   }
 
   // All output tensors are allocated by TensorFlow/Eager, so we
@@ -372,6 +376,7 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
     }
   }
 
+  buffer_map->ClearForwardable();
   for (const auto& x : tensor_ref_count) {
     if (x.second == 1) {
       // This tensor is referenced once by a single op. We can allow the TF
@@ -429,10 +434,12 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
   }
 
   // We don't need to keep track of internal TF tensors any longer, so take
-  // them out of the buffer_map, but make sure we keep all the one we might
+  // them out of the buffer_map, but make sure we keep all the ones we might
   // need for other subgraphs, or as final output of inference.
   const auto& outputs = op_data->subgraph_outputs;
   std::set<int> keep(outputs.begin(), outputs.end());
+  const auto& inputs = op_data->subgraph_inputs;
+  keep.insert(inputs.begin(), inputs.end());
   buffer_map->RemoveTensorsNotInSet(keep);
 
   return kTfLiteOk;
diff --git a/tensorflow/lite/delegates/flex/kernel_test.cc b/tensorflow/lite/delegates/flex/kernel_test.cc
index 9de3ca67de..cef8017e6e 100644
--- a/tensorflow/lite/delegates/flex/kernel_test.cc
+++ b/tensorflow/lite/delegates/flex/kernel_test.cc
@@ -25,6 +25,7 @@ namespace {
 
 using ::testing::ContainsRegex;
 using ::testing::ElementsAre;
+using ::testing::ElementsAreArray;
 
 TfLiteStatus GenericPrepare(TfLiteContext* context, TfLiteDelegate* delegate,
                             const std::vector<int>& supported_nodes) {
@@ -340,14 +341,10 @@ TEST_F(MultipleSubgraphsTest, ForwardabilityIsLocal) {
   auto input = {3.0f, 4.0f, 5.0f};
   PrepareInterpreter(GetPrepareFunction<__LINE__>(), input);
 
-  ASSERT_FALSE(Invoke());
-  ASSERT_THAT(error_reporter().error_messages(),
-              ContainsRegex("Cannot read from invalid tensor index 10"));
-
-  // TODO(b/122457581): expected result for this test:
-  // ASSERT_THAT(GetValues(12), ElementsAreArray(Apply(input, [](float in) {
-  //               return (4 * in + 4) * (in + 1);
-  //             })));
+  ASSERT_TRUE(Invoke());
+  ASSERT_THAT(GetValues(12), ElementsAreArray(Apply(input, [](float in) {
+                return (4 * in + 4) * (in + 1);
+              })));
 }
 
 // Subgraphs should not remove input tensors from the buffer_map, since
@@ -379,14 +376,10 @@ TEST_F(MultipleSubgraphsTest, DoNotRemoveInputTensors) {
   auto input = {3.0f, 4.0f, 5.0f};
   PrepareInterpreter(GetPrepareFunction<__LINE__>(), input);
 
-  ASSERT_FALSE(Invoke());
-  ASSERT_THAT(error_reporter().error_messages(),
-              ContainsRegex("Tensor '10' not found"));
-
-  // TODO(b/122457581): expected result for this test:
-  // ASSERT_THAT(GetValues(12), ElementsAreArray(Apply(input, [](float in) {
-  //               return (4 * in + 4) * (in + 1);
-  //             })));
+  ASSERT_TRUE(Invoke());
+  ASSERT_THAT(GetValues(12), ElementsAreArray(Apply(input, [](float in) {
+                return (4 * in + 4) * (in + 1);
+              })));
 }
 
 // A tensor is deemed forwardable but it happens to be the input to
@@ -417,14 +410,10 @@ TEST_F(MultipleSubgraphsTest, DoNotForwardInputTensors) {
   auto input = {3.0f, 4.0f, 5.0f};
   PrepareInterpreter(GetPrepareFunction<__LINE__>(), input);
 
-  ASSERT_FALSE(Invoke());
-  ASSERT_THAT(error_reporter().error_messages(),
-              ContainsRegex("Tensor '10' not found"));
-
-  // TODO(b/122457581): expected result for this test:
-  // ASSERT_THAT(GetValues(12), ElementsAreArray(Apply(input, [](float in) {
-  //               return (5 * in + 5) * (in + 1);
-  //             })));
+  ASSERT_TRUE(Invoke());
+  ASSERT_THAT(GetValues(12), ElementsAreArray(Apply(input, [](float in) {
+                return (5 * in + 5) * (in + 1);
+              })));
 }
 
 }  // namespace
-- 
GitLab


From 6d26622733a3f55861c135b857b2cbce981b2467 Mon Sep 17 00:00:00 2001
From: Karim Nosir <karimnosseir@google.com>
Date: Mon, 7 Jan 2019 15:34:34 -0800
Subject: [PATCH 0334/1095] Fix some tests failures.

PiperOrigin-RevId: 228248776
---
 tensorflow/lite/kernels/BUILD                                 | 2 ++
 tensorflow/lite/kernels/dequantize_test.cc                    | 1 +
 tensorflow/lite/kernels/internal/BUILD                        | 4 ++++
 tensorflow/lite/kernels/sparse_output_fully_connected_test.cc | 1 +
 4 files changed, 8 insertions(+)

diff --git a/tensorflow/lite/kernels/BUILD b/tensorflow/lite/kernels/BUILD
index 6809a4a3d0..50deac9bb6 100644
--- a/tensorflow/lite/kernels/BUILD
+++ b/tensorflow/lite/kernels/BUILD
@@ -352,6 +352,7 @@ tf_cc_test(
         ":builtin_ops",
         "//tensorflow/lite:framework",
         "//tensorflow/lite/kernels:test_util",
+        "//tensorflow/lite/kernels/internal:types",
         "@com_google_googletest//:gtest",
         "@flatbuffers",
     ],
@@ -513,6 +514,7 @@ tf_cc_test(
         ":builtin_ops",
         "//tensorflow/lite:framework",
         "//tensorflow/lite/kernels:test_util",
+        "//tensorflow/lite/kernels/internal:types",
         "@com_google_absl//absl/memory",
         "@com_google_googletest//:gtest",
     ],
diff --git a/tensorflow/lite/kernels/dequantize_test.cc b/tensorflow/lite/kernels/dequantize_test.cc
index bb5f1e74a8..6343745eef 100644
--- a/tensorflow/lite/kernels/dequantize_test.cc
+++ b/tensorflow/lite/kernels/dequantize_test.cc
@@ -14,6 +14,7 @@ limitations under the License.
 ==============================================================================*/
 #include <gtest/gtest.h>
 #include "tensorflow/lite/interpreter.h"
+#include "tensorflow/lite/kernels/internal/types.h"
 #include "tensorflow/lite/kernels/register.h"
 #include "tensorflow/lite/kernels/test_util.h"
 #include "tensorflow/lite/model.h"
diff --git a/tensorflow/lite/kernels/internal/BUILD b/tensorflow/lite/kernels/internal/BUILD
index 92b4705908..885f650dd5 100644
--- a/tensorflow/lite/kernels/internal/BUILD
+++ b/tensorflow/lite/kernels/internal/BUILD
@@ -545,6 +545,10 @@ cc_library(
     name = "test_util",
     srcs = ["test_util.cc"],
     hdrs = ["test_util.h"],
+    linkopts = select({
+        "//tensorflow:windows": [],
+        "//conditions:default": ["-lm"],
+    }),
     deps = [
         ":types",
         "//tensorflow/lite:string",
diff --git a/tensorflow/lite/kernels/sparse_output_fully_connected_test.cc b/tensorflow/lite/kernels/sparse_output_fully_connected_test.cc
index fb71cb7f7b..8152ae6685 100644
--- a/tensorflow/lite/kernels/sparse_output_fully_connected_test.cc
+++ b/tensorflow/lite/kernels/sparse_output_fully_connected_test.cc
@@ -19,6 +19,7 @@ limitations under the License.
 
 #include <gtest/gtest.h>
 #include "flatbuffers/flexbuffers.h"  // TF:flatbuffers
+#include "tensorflow/lite/kernels/internal/types.h"
 #include "tensorflow/lite/kernels/register.h"
 #include "tensorflow/lite/kernels/test_util.h"
 
-- 
GitLab


From 3b74e1c16ab280a4e0cbd5138a0c73710cb9804c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 7 Jan 2019 15:46:39 -0800
Subject: [PATCH 0335/1095] Clarified that variables_to_restore function
 chooses whether to use a moving average version based on provided parameters
 rather than based on the existence of a given name.

PiperOrigin-RevId: 228250786
---
 tensorflow/python/training/moving_averages.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/training/moving_averages.py b/tensorflow/python/training/moving_averages.py
index 72670f0ca3..4b267dfb98 100644
--- a/tensorflow/python/training/moving_averages.py
+++ b/tensorflow/python/training/moving_averages.py
@@ -505,13 +505,13 @@ class ExponentialMovingAverage(object):
     ```
     Args:
       moving_avg_variables: a list of variables that require to use of the
-        moving variable name to be restored. If None, it will default to
+        moving average variable name to be restored. If None, it will default to
         variables.moving_average_variables() + variables.trainable_variables()
 
     Returns:
-      A map from restore_names to variables. The restore_name can be the
-      moving_average version of the variable name if it exist, or the original
-      variable name.
+      A map from restore_names to variables. The restore_name is either the
+      original or the moving average version of the variable name, depending
+      on whether the variable name is in the `moving_avg_variables`.
     """
     name_map = {}
     if moving_avg_variables is None:
-- 
GitLab


From 0de09568916f31d77ef0093285a058e901195919 Mon Sep 17 00:00:00 2001
From: Peter Ma <pcma@google.com>
Date: Mon, 7 Jan 2019 16:09:22 -0800
Subject: [PATCH 0336/1095] Replace op_features with op_info to correctly
 reflect the data type, and some minor fix on function argument names.

PiperOrigin-RevId: 228254456
---
 .../grappler/costs/op_level_cost_estimator.cc | 246 +++++++++---------
 .../grappler/costs/op_level_cost_estimator.h  |  34 ++-
 .../costs/op_level_cost_estimator_test.cc     |  17 +-
 3 files changed, 143 insertions(+), 154 deletions(-)

diff --git a/tensorflow/core/grappler/costs/op_level_cost_estimator.cc b/tensorflow/core/grappler/costs/op_level_cost_estimator.cc
index 55eb391d2b..96bac8d0cb 100644
--- a/tensorflow/core/grappler/costs/op_level_cost_estimator.cc
+++ b/tensorflow/core/grappler/costs/op_level_cost_estimator.cc
@@ -72,25 +72,25 @@ static const Costs::Duration kMinComputeTime(1);
 
 namespace {
 
-string GetDataFormat(const OpInfo& op_features) {
+string GetDataFormat(const OpInfo& op_info) {
   string data_format = "NHWC";  // Default format.
-  if (op_features.attr().find("data_format") != op_features.attr().end()) {
-    data_format = op_features.attr().at("data_format").s();
+  if (op_info.attr().find("data_format") != op_info.attr().end()) {
+    data_format = op_info.attr().at("data_format").s();
   }
   return data_format;
 }
 
-string GetFilterFormat(const OpInfo& op_features) {
+string GetFilterFormat(const OpInfo& op_info) {
   string filter_format = "HWIO";  // Default format.
-  if (op_features.attr().find("filter_format") != op_features.attr().end()) {
-    filter_format = op_features.attr().at("filter_format").s();
+  if (op_info.attr().find("filter_format") != op_info.attr().end()) {
+    filter_format = op_info.attr().at("filter_format").s();
   }
   return filter_format;
 }
 
-Padding GetPadding(const OpInfo& op_features) {
-  if (op_features.attr().find("padding") != op_features.attr().end() &&
-      op_features.attr().at("padding").s() == "VALID") {
+Padding GetPadding(const OpInfo& op_info) {
+  if (op_info.attr().find("padding") != op_info.attr().end() &&
+      op_info.attr().at("padding").s() == "VALID") {
     return Padding::VALID;
   }
   return Padding::SAME;  // Default padding.
@@ -107,11 +107,11 @@ bool IsTraining(const OpInfo& op_info) {
 // TODO(dyoon): support non-4D tensors in the c ost functions of convolution
 // related ops (Conv, Pool, BatchNorm, and their backprops) and the related
 // helper functions.
-std::vector<int64> GetStrides(const OpInfo& op_features) {
-  if (op_features.attr().find("strides") != op_features.attr().end()) {
-    const auto strides = op_features.attr().at("strides").list().i();
-    CHECK(strides.size() == 4) << "Attr strides is not a length-4 vector: "
-                               << op_features.DebugString();
+std::vector<int64> GetStrides(const OpInfo& op_info) {
+  if (op_info.attr().find("strides") != op_info.attr().end()) {
+    const auto strides = op_info.attr().at("strides").list().i();
+    CHECK(strides.size() == 4)
+        << "Attr strides is not a length-4 vector: " << op_info.DebugString();
     return {strides[0], strides[1], strides[2], strides[3]};
   }
   return {1, 1, 1, 1};
@@ -359,21 +359,21 @@ OpLevelCostEstimator::OpLevelCostEstimator() {
 }
 
 Costs OpLevelCostEstimator::PredictCosts(const OpContext& op_context) const {
-  const auto& op_features = op_context.op_info;
-  auto it = device_cost_impl_.find(op_features.op());
+  const auto& op_info = op_context.op_info;
+  auto it = device_cost_impl_.find(op_info.op());
   if (it == device_cost_impl_.end()) {
-    if (elementwise_ops_.find(op_features.op()) != elementwise_ops_.end()) {
+    if (elementwise_ops_.find(op_info.op()) != elementwise_ops_.end()) {
       return PredictCwiseOp(op_context);
     }
 
-    VLOG(1) << "Missing accurate estimator for op: " << op_features.op();
+    VLOG(1) << "Missing accurate estimator for op: " << op_info.op();
 
     return PredictCostOfAnUnknownOp(op_context);
   }
 
   std::function<Costs(const OpContext&)> estimator = it->second;
   Costs costs = estimator(op_context);
-  VLOG(1) << "Operation " << op_features.op() << " takes "
+  VLOG(1) << "Operation " << op_info.op() << " takes "
           << costs.execution_time.count() << " ns.";
   return costs;
 }
@@ -430,39 +430,38 @@ DeviceInfo OpLevelCostEstimator::GetDeviceInfo(
 }
 
 Costs OpLevelCostEstimator::PredictCwiseOp(const OpContext& op_context) const {
-  const auto& op_features = op_context.op_info;
+  const auto& op_info = op_context.op_info;
   bool found_unknown_shapes = false;
   // For unary or binary element-wise operations, op count is the element count
   // of any input. We use the count for the largest input here to be more robust
   // in case that the shape is unknown or partially known for other input.
-  int64 op_count =
-      CalculateLargestInputCount(op_features, &found_unknown_shapes);
+  int64 op_count = CalculateLargestInputCount(op_info, &found_unknown_shapes);
   // If output shape is available, try use the element count calcuated from
   // that.
-  if (op_features.outputs_size() > 0) {
-    op_count =
-        std::max(op_count, CalculateTensorElementCount(op_features.outputs(0),
-                                                       &found_unknown_shapes));
+  if (op_info.outputs_size() > 0) {
+    op_count = std::max(
+        op_count,
+        CalculateTensorElementCount(op_info.outputs(0), &found_unknown_shapes));
   }
   // For binary ops, calculate the output shape possibly resulting from
   // broadcasting.
-  if (op_features.inputs_size() >= 2) {
-    op_count = std::max(op_count,
-                        CwiseOutputElementCount(op_features.inputs(0).shape(),
-                                                op_features.inputs(1).shape()));
+  if (op_info.inputs_size() >= 2) {
+    op_count =
+        std::max(op_count, CwiseOutputElementCount(op_info.inputs(0).shape(),
+                                                   op_info.inputs(1).shape()));
   }
 
   int op_cost = 1;
   bool is_known_elementwise_op = false;
-  auto it = elementwise_ops_.find(op_features.op());
+  auto it = elementwise_ops_.find(op_info.op());
   if (it != elementwise_ops_.end()) {
     op_cost = it->second;
     is_known_elementwise_op = true;
   } else {
-    LOG(WARNING) << "Not a cwise op: " << op_features.op();
+    LOG(WARNING) << "Not a cwise op: " << op_info.op();
   }
 
-  Costs costs = PredictOpCountBasedCost(op_count * op_cost, op_features);
+  Costs costs = PredictOpCountBasedCost(op_count * op_cost, op_info);
   if (found_unknown_shapes || !is_known_elementwise_op) {
     costs.inaccurate = true;
   }
@@ -542,17 +541,17 @@ Costs OpLevelCostEstimator::PredictOpCountBasedCost(
 }
 
 int64 OpLevelCostEstimator::CountConv2DOperations(
-    const OpInfo& op_features, bool* found_unknown_shapes) const {
-  return CountConv2DOperations(op_features, nullptr, found_unknown_shapes);
+    const OpInfo& op_info, bool* found_unknown_shapes) const {
+  return CountConv2DOperations(op_info, nullptr, found_unknown_shapes);
 }
 
 // Helper to translate the positional arguments into named fields.
 OpLevelCostEstimator::ConvolutionDimensions
 OpLevelCostEstimator::ConvolutionDimensionsFromInputs(
     const TensorShapeProto& original_image_shape,
-    const TensorShapeProto& original_filter_shape, const OpInfo& op_features,
+    const TensorShapeProto& original_filter_shape, const OpInfo& op_info,
     bool* found_unknown_shapes) {
-  VLOG(2) << "op features: " << op_features.DebugString();
+  VLOG(2) << "op features: " << op_info.DebugString();
   VLOG(2) << "Original image shape: " << original_image_shape.DebugString();
   VLOG(2) << "Original filter shape: " << original_filter_shape.DebugString();
   auto image_shape =
@@ -563,7 +562,7 @@ OpLevelCostEstimator::ConvolutionDimensionsFromInputs(
   VLOG(2) << "Filter shape: " << filter_shape.DebugString();
 
   int x_index, y_index, channel_index;
-  const string& data_format = GetDataFormat(op_features);
+  const string& data_format = GetDataFormat(op_info);
   if (data_format == "NCHW") {
     x_index = 2;
     y_index = 3;
@@ -574,7 +573,7 @@ OpLevelCostEstimator::ConvolutionDimensionsFromInputs(
     y_index = 2;
     channel_index = 3;
   }
-  const string& filter_format = GetFilterFormat(op_features);
+  const string& filter_format = GetFilterFormat(op_info);
   int filter_x_index, filter_y_index, in_channel_index, out_channel_index;
   if (filter_format == "HWIO") {
     filter_x_index = 0;
@@ -594,8 +593,8 @@ OpLevelCostEstimator::ConvolutionDimensionsFromInputs(
   int64 iz = image_shape.dim(channel_index).size();
   int64 kx = filter_shape.dim(filter_x_index).size();
   int64 ky = filter_shape.dim(filter_y_index).size();
-  std::vector<int64> strides = GetStrides(op_features);
-  const auto padding = GetPadding(op_features);
+  std::vector<int64> strides = GetStrides(op_info);
+  const auto padding = GetPadding(op_info);
   int64 sx = strides[x_index];
   int64 sy = strides[y_index];
   int64 ox = GetOutputSize(ix, kx, sx, padding);
@@ -623,14 +622,13 @@ OpLevelCostEstimator::ConvolutionDimensionsFromInputs(
 }
 
 int64 OpLevelCostEstimator::CountConv2DOperations(
-    const OpInfo& op_features, ConvolutionDimensions* conv_info,
+    const OpInfo& op_info, ConvolutionDimensions* conv_info,
     bool* found_unknown_shapes) const {
-  DCHECK(op_features.op() == kConv2d ||
-         op_features.op() == kDepthwiseConv2dNative)
+  DCHECK(op_info.op() == kConv2d || op_info.op() == kDepthwiseConv2dNative)
       << "Invalid Operation: not Conv2D nor DepthwiseConv2dNative";
 
   ConvolutionDimensions conv_dims = ConvolutionDimensionsFromInputs(
-      op_features.inputs(0).shape(), op_features.inputs(1).shape(), op_features,
+      op_info.inputs(0).shape(), op_info.inputs(1).shape(), op_info,
       found_unknown_shapes);
 
   //  in DepthwiseConv2dNative conv_dims.oz is actually the channel depth
@@ -641,7 +639,7 @@ int64 OpLevelCostEstimator::CountConv2DOperations(
   int64 ops = conv_dims.batch;
   ops *= conv_dims.ox * conv_dims.oy;
   ops *= conv_dims.kx * conv_dims.ky;
-  if (op_features.op() == kConv2d) {
+  if (op_info.op() == kConv2d) {
     ops *= conv_dims.iz * conv_dims.oz;
   } else {
     // To ensure output tensor dims to be correct for DepthwiseConv2DNative,
@@ -658,32 +656,32 @@ int64 OpLevelCostEstimator::CountConv2DOperations(
 }
 
 int64 OpLevelCostEstimator::CountMatMulOperations(
-    const OpInfo& op_features, bool* found_unknown_shapes) const {
-  return CountMatMulOperations(op_features, nullptr, found_unknown_shapes);
+    const OpInfo& op_info, bool* found_unknown_shapes) const {
+  return CountMatMulOperations(op_info, nullptr, found_unknown_shapes);
 }
 
 // TODO(nishantpatil): Create separate estimator for Sparse Matmul
 int64 OpLevelCostEstimator::CountMatMulOperations(
-    const OpInfo& op_features, MatMulDimensions* mat_mul,
+    const OpInfo& op_info, MatMulDimensions* mat_mul,
     bool* found_unknown_shapes) const {
   double ops = 0;
 
-  if (op_features.inputs_size() < 2) {
-    LOG(ERROR) << "Need 2 inputs but got " << op_features.inputs_size();
+  if (op_info.inputs_size() < 2) {
+    LOG(ERROR) << "Need 2 inputs but got " << op_info.inputs_size();
     // TODO(pcma): Try to separate invalid inputs from unknown shapes
     *found_unknown_shapes = true;
     return 0;
   }
 
-  auto& a_matrix = op_features.inputs(0);
-  auto& b_matrix = op_features.inputs(1);
+  auto& a_matrix = op_info.inputs(0);
+  auto& b_matrix = op_info.inputs(1);
 
   bool transpose_a = false;
   bool transpose_b = false;
 
   double m_dim, n_dim, k_dim, k_dim_b = 0;
 
-  for (const auto& item : op_features.attr()) {
+  for (const auto& item : op_info.attr()) {
     VLOG(1) << "Key:" << item.first
             << " Value:" << SummarizeAttrValue(item.second);
     if (item.first == "transpose_a" && item.second.b() == true)
@@ -735,23 +733,23 @@ int64 OpLevelCostEstimator::CountMatMulOperations(
 }
 
 int64 OpLevelCostEstimator::CountBatchMatMulOperations(
-    const OpInfo& op_features, bool* found_unknown_shapes) const {
-  if (op_features.op() != kBatchMatMul) {
-    LOG(ERROR) << "Invalid Operation: " << op_features.op();
+    const OpInfo& op_info, bool* found_unknown_shapes) const {
+  if (op_info.op() != kBatchMatMul) {
+    LOG(ERROR) << "Invalid Operation: " << op_info.op();
     // TODO(pcma): Try to separate invalid inputs from unknown shapes
     *found_unknown_shapes = true;
     return 0;
   }
-  if (op_features.inputs_size() != 2) {
-    LOG(ERROR) << "Expected 2 inputs but got " << op_features.inputs_size();
+  if (op_info.inputs_size() != 2) {
+    LOG(ERROR) << "Expected 2 inputs but got " << op_info.inputs_size();
     // TODO(pcma): Try to separate invalid inputs from unknown shapes
     *found_unknown_shapes = true;
     return 0;
   }
 
   double ops = 0;
-  const auto& a_input = op_features.inputs(0);
-  const auto& b_input = op_features.inputs(1);
+  const auto& a_input = op_info.inputs(0);
+  const auto& b_input = op_info.inputs(1);
 
   // BatchMatMul requires inputs of at least matrix shape (rank 2).
   // The two most minor dimensions of each input are matrices that
@@ -801,24 +799,24 @@ int64 OpLevelCostEstimator::CountBatchMatMulOperations(
 
   // Build the MatMul. Note that values are ignored here since we are just
   // counting ops (e.g. only shapes matter).
-  OpInfo matmul_op_features;
-  matmul_op_features.set_op("MatMul");
+  OpInfo matmul_op_info;
+  matmul_op_info.set_op("MatMul");
 
   AttrValue transpose_a;
   transpose_a.set_b(false);
-  if (op_features.attr().find("adj_x") != op_features.attr().end()) {
-    transpose_a.set_b(op_features.attr().at("adj_x").b());
+  if (op_info.attr().find("adj_x") != op_info.attr().end()) {
+    transpose_a.set_b(op_info.attr().at("adj_x").b());
   }
-  (*matmul_op_features.mutable_attr())["transpose_a"] = transpose_a;
+  (*matmul_op_info.mutable_attr())["transpose_a"] = transpose_a;
 
   AttrValue transpose_b;
   transpose_b.set_b(false);
-  if (op_features.attr().find("adj_y") != op_features.attr().end()) {
-    transpose_b.set_b(op_features.attr().at("adj_y").b());
+  if (op_info.attr().find("adj_y") != op_info.attr().end()) {
+    transpose_b.set_b(op_info.attr().at("adj_y").b());
   }
-  (*matmul_op_features.mutable_attr())["transpose_b"] = transpose_b;
+  (*matmul_op_info.mutable_attr())["transpose_b"] = transpose_b;
 
-  OpInfo::TensorProperties* a_matrix = matmul_op_features.add_inputs();
+  OpInfo::TensorProperties* a_matrix = matmul_op_info.add_inputs();
   a_matrix->set_dtype(a_input.dtype());
   TensorShapeProto* a_matrix_shape = a_matrix->mutable_shape();
   for (int i = std::max(0, a_input_shape.dim_size() - matrix_rank);
@@ -826,7 +824,7 @@ int64 OpLevelCostEstimator::CountBatchMatMulOperations(
     *(a_matrix_shape->add_dim()) = a_input_shape.dim(i);
   }
 
-  OpInfo::TensorProperties* b_matrix = matmul_op_features.add_inputs();
+  OpInfo::TensorProperties* b_matrix = matmul_op_info.add_inputs();
   b_matrix->set_dtype(b_input.dtype());
   TensorShapeProto* b_matrix_shape = b_matrix->mutable_shape();
   for (int i = std::max(0, b_input_shape.dim_size() - matrix_rank);
@@ -836,7 +834,7 @@ int64 OpLevelCostEstimator::CountBatchMatMulOperations(
 
   for (int i = 0; i < num_matmuls; ++i) {
     bool matmul_unknown_shapes = false;
-    ops += CountMatMulOperations(matmul_op_features, &matmul_unknown_shapes);
+    ops += CountMatMulOperations(matmul_op_info, &matmul_unknown_shapes);
     *found_unknown_shapes |= matmul_unknown_shapes;
   }
   return ops;
@@ -894,16 +892,16 @@ bool GetTensorShapeProtoFromTensorProto(const TensorProto& tensor_proto,
 
 // TODO(cliffy): Dedup this method and CountConv2DBackpropFilterOperations.
 int64 OpLevelCostEstimator::CountConv2DBackpropInputOperations(
-    const OpInfo& op_features, ConvolutionDimensions* returned_conv_dims,
+    const OpInfo& op_info, ConvolutionDimensions* returned_conv_dims,
     bool* found_unknown_shapes) const {
   int64 ops = 0;
 
-  DCHECK(op_features.op() == kConv2dBackpropInput ||
-         op_features.op() == kDepthwiseConv2dNativeBackpropInput)
+  DCHECK(op_info.op() == kConv2dBackpropInput ||
+         op_info.op() == kDepthwiseConv2dNativeBackpropInput)
       << "Invalid Operation: not kConv2dBackpropInput nor"
          "kDepthwiseConv2dNativeBackpropInput";
 
-  if (op_features.inputs_size() < 2) {
+  if (op_info.inputs_size() < 2) {
     // TODO(pcma): Try to separate invalid inputs from unknown shapes
     *found_unknown_shapes = true;
     return ops;
@@ -911,12 +909,12 @@ int64 OpLevelCostEstimator::CountConv2DBackpropInputOperations(
 
   TensorShapeProto input_shape;
   bool shape_found = false;
-  if (op_features.inputs(0).has_value()) {
-    const TensorProto& value = op_features.inputs(0).value();
+  if (op_info.inputs(0).has_value()) {
+    const TensorProto& value = op_info.inputs(0).value();
     shape_found = GetTensorShapeProtoFromTensorProto(value, &input_shape);
   }
-  if (!shape_found && op_features.outputs_size() == 1) {
-    input_shape = op_features.outputs(0).shape();
+  if (!shape_found && op_info.outputs_size() == 1) {
+    input_shape = op_info.outputs(0).shape();
     shape_found = true;
   }
   if (!shape_found) {
@@ -929,13 +927,12 @@ int64 OpLevelCostEstimator::CountConv2DBackpropInputOperations(
   }
 
   ConvolutionDimensions conv_dims = ConvolutionDimensionsFromInputs(
-      input_shape, op_features.inputs(1).shape(), op_features,
-      found_unknown_shapes);
+      input_shape, op_info.inputs(1).shape(), op_info, found_unknown_shapes);
 
   ops = conv_dims.batch;
   ops *= conv_dims.ox * conv_dims.oy;
   ops *= conv_dims.kx * conv_dims.ky;
-  if (op_features.op() == kConv2dBackpropInput) {
+  if (op_info.op() == kConv2dBackpropInput) {
     ops *= conv_dims.iz * conv_dims.oz;
   } else {
     // conv_dims always use forward path definition regardless
@@ -944,7 +941,7 @@ int64 OpLevelCostEstimator::CountConv2DBackpropInputOperations(
   }
   ops *= kOpsPerMac;
 
-  VLOG(1) << "Operations for" << op_features.op() << "  " << ops;
+  VLOG(1) << "Operations for" << op_info.op() << "  " << ops;
 
   if (returned_conv_dims != nullptr) {
     *returned_conv_dims = conv_dims;
@@ -953,23 +950,23 @@ int64 OpLevelCostEstimator::CountConv2DBackpropInputOperations(
 }
 
 int64 OpLevelCostEstimator::CountConv2DBackpropFilterOperations(
-    const OpInfo& op_features, ConvolutionDimensions* returned_conv_dims,
+    const OpInfo& op_info, ConvolutionDimensions* returned_conv_dims,
     bool* found_unknown_shapes) const {
   int64 ops = 0;
 
-  DCHECK(op_features.op() == kConv2dBackpropFilter ||
-         op_features.op() == kDepthwiseConv2dNativeBackpropFilter)
+  DCHECK(op_info.op() == kConv2dBackpropFilter ||
+         op_info.op() == kDepthwiseConv2dNativeBackpropFilter)
       << "Invalid Operation: not kConv2dBackpropFilter nor"
          "kDepthwiseConv2dNativeBackpropFilter";
 
   TensorShapeProto filter_shape;
   bool shape_found = false;
-  if (op_features.inputs_size() >= 2 && op_features.inputs(1).has_value()) {
-    const TensorProto& value = op_features.inputs(1).value();
+  if (op_info.inputs_size() >= 2 && op_info.inputs(1).has_value()) {
+    const TensorProto& value = op_info.inputs(1).value();
     shape_found = GetTensorShapeProtoFromTensorProto(value, &filter_shape);
   }
-  if (!shape_found && op_features.outputs_size() == 1) {
-    filter_shape = op_features.outputs(0).shape();
+  if (!shape_found && op_info.outputs_size() == 1) {
+    filter_shape = op_info.outputs(0).shape();
     shape_found = true;
   }
   if (!shape_found) {
@@ -981,19 +978,18 @@ int64 OpLevelCostEstimator::CountConv2DBackpropFilterOperations(
     *found_unknown_shapes = true;
   }
 
-  if (op_features.inputs_size() < 1) {
+  if (op_info.inputs_size() < 1) {
     // TODO(pcma): Try to separate invalid inputs from unknown shapes
     *found_unknown_shapes = true;
     return ops;
   }
   ConvolutionDimensions conv_dims = ConvolutionDimensionsFromInputs(
-      op_features.inputs(0).shape(), filter_shape, op_features,
-      found_unknown_shapes);
+      op_info.inputs(0).shape(), filter_shape, op_info, found_unknown_shapes);
 
   ops = conv_dims.batch;
   ops *= conv_dims.ox * conv_dims.oy;
   ops *= conv_dims.kx * conv_dims.ky;
-  if (op_features.op() == kConv2dBackpropFilter) {
+  if (op_info.op() == kConv2dBackpropFilter) {
     ops *= conv_dims.iz * conv_dims.oz;
   } else {
     // conv_dims always use forward path definition regardless
@@ -1001,7 +997,7 @@ int64 OpLevelCostEstimator::CountConv2DBackpropFilterOperations(
     ops *= conv_dims.oz;
   }
   ops *= kOpsPerMac;
-  VLOG(1) << "Operations for" << op_features.op() << "  " << ops;
+  VLOG(1) << "Operations for" << op_info.op() << "  " << ops;
 
   if (returned_conv_dims != nullptr) {
     *returned_conv_dims = conv_dims;
@@ -1032,9 +1028,9 @@ int64 OpLevelCostEstimator::CalculateTensorSize(
 }
 
 int64 OpLevelCostEstimator::CalculateInputSize(
-    const OpInfo& op_features, bool* found_unknown_shapes) const {
+    const OpInfo& op_info, bool* found_unknown_shapes) const {
   int64 total_input_size = 0;
-  for (auto& input : op_features.inputs()) {
+  for (auto& input : op_info.inputs()) {
     int64 input_size = CalculateTensorSize(input, found_unknown_shapes);
     total_input_size += input_size;
     VLOG(1) << "Input Size: " << input_size
@@ -1044,9 +1040,9 @@ int64 OpLevelCostEstimator::CalculateInputSize(
 }
 
 int64 OpLevelCostEstimator::CalculateLargestInputCount(
-    const OpInfo& op_features, bool* found_unknown_shapes) const {
+    const OpInfo& op_info, bool* found_unknown_shapes) const {
   int64 largest_input_count = 0;
-  for (auto& input : op_features.inputs()) {
+  for (auto& input : op_info.inputs()) {
     int64 input_count =
         CalculateTensorElementCount(input, found_unknown_shapes);
     if (input_count > largest_input_count) {
@@ -1059,10 +1055,10 @@ int64 OpLevelCostEstimator::CalculateLargestInputCount(
 }
 
 int64 OpLevelCostEstimator::CalculateOutputSize(
-    const OpInfo& op_features, bool* found_unknown_shapes) const {
+    const OpInfo& op_info, bool* found_unknown_shapes) const {
   int64 total_output_size = 0;
   // use float as default for calculations
-  for (const auto& output : op_features.outputs()) {
+  for (const auto& output : op_info.outputs()) {
     DataType dt = output.dtype();
     const auto& original_output_shape = output.shape();
     int64 output_size = DataTypeSize(BaseType(dt));
@@ -1080,10 +1076,10 @@ int64 OpLevelCostEstimator::CalculateOutputSize(
 }
 
 Costs OpLevelCostEstimator::PredictConv2D(const OpContext& op_context) const {
-  const auto& op_features = op_context.op_info;
+  const auto& op_info = op_context.op_info;
   bool found_unknown_shapes = false;
   auto costs = PredictOpCountBasedCost(
-      CountConv2DOperations(op_features, &found_unknown_shapes), op_features);
+      CountConv2DOperations(op_info, &found_unknown_shapes), op_info);
   costs.inaccurate = found_unknown_shapes;
   costs.num_ops_with_unknown_shapes = found_unknown_shapes;
   return costs;
@@ -1091,12 +1087,12 @@ Costs OpLevelCostEstimator::PredictConv2D(const OpContext& op_context) const {
 
 Costs OpLevelCostEstimator::PredictConv2DBackpropInput(
     const OpContext& op_context) const {
-  const auto& op_features = op_context.op_info;
+  const auto& op_info = op_context.op_info;
   bool found_unknown_shapes = false;
   auto costs =
       PredictOpCountBasedCost(CountConv2DBackpropInputOperations(
-                                  op_features, nullptr, &found_unknown_shapes),
-                              op_features);
+                                  op_info, nullptr, &found_unknown_shapes),
+                              op_info);
   costs.inaccurate = found_unknown_shapes;
   costs.num_ops_with_unknown_shapes = found_unknown_shapes;
   return costs;
@@ -1104,12 +1100,12 @@ Costs OpLevelCostEstimator::PredictConv2DBackpropInput(
 
 Costs OpLevelCostEstimator::PredictConv2DBackpropFilter(
     const OpContext& op_context) const {
-  const auto& op_features = op_context.op_info;
+  const auto& op_info = op_context.op_info;
   bool found_unknown_shapes = false;
   auto costs =
       PredictOpCountBasedCost(CountConv2DBackpropFilterOperations(
-                                  op_features, nullptr, &found_unknown_shapes),
-                              op_features);
+                                  op_info, nullptr, &found_unknown_shapes),
+                              op_info);
   costs.inaccurate = found_unknown_shapes;
   costs.num_ops_with_unknown_shapes = found_unknown_shapes;
   return costs;
@@ -1204,26 +1200,26 @@ Costs OpLevelCostEstimator::PredictFusedConv2DBiasActivation(
 }
 
 Costs OpLevelCostEstimator::PredictMatMul(const OpContext& op_context) const {
-  const auto& op_features = op_context.op_info;
+  const auto& op_info = op_context.op_info;
   bool found_unknown_shapes = false;
   auto costs = PredictOpCountBasedCost(
-      CountMatMulOperations(op_features, &found_unknown_shapes), op_features);
+      CountMatMulOperations(op_info, &found_unknown_shapes), op_info);
   costs.inaccurate = found_unknown_shapes;
   costs.num_ops_with_unknown_shapes = found_unknown_shapes;
   return costs;
 }
 
 Costs OpLevelCostEstimator::PredictNoOp(const OpContext& op_context) const {
-  const auto& op_features = op_context.op_info;
-  VLOG(1) << "Op:" << op_features.op() << " Execution Time 0 (ns)";
+  const auto& op_info = op_context.op_info;
+  VLOG(1) << "Op:" << op_info.op() << " Execution Time 0 (ns)";
   return Costs::ZeroCosts();
 }
 
 Costs OpLevelCostEstimator::PredictIdentity(const OpContext& op_context) const {
-  const auto& op_features = op_context.op_info;
-  VLOG(1) << "Op:" << op_features.op() << " Execution Time 0 (ns)";
+  const auto& op_info = op_context.op_info;
+  VLOG(1) << "Op:" << op_info.op() << " Execution Time 0 (ns)";
   Costs result = Costs::ZeroCosts();
-  result.max_memory = CalculateOutputSize(op_features, &result.inaccurate);
+  result.max_memory = CalculateOutputSize(op_info, &result.inaccurate);
   result.num_ops_with_unknown_shapes = result.inaccurate;
   // Assign the minimum amount of time we can represent to the identity op since
   // it tends to be really cheap.
@@ -1233,11 +1229,10 @@ Costs OpLevelCostEstimator::PredictIdentity(const OpContext& op_context) const {
 }
 
 Costs OpLevelCostEstimator::PredictVariable(const OpContext& op_context) const {
-  const auto& op_features = op_context.op_info;
-  VLOG(1) << "Op:" << op_features.op() << " Execution Time 0 (ns)";
+  const auto& op_info = op_context.op_info;
+  VLOG(1) << "Op:" << op_info.op() << " Execution Time 0 (ns)";
   Costs result = Costs::ZeroCosts();
-  result.persistent_memory =
-      CalculateOutputSize(op_features, &result.inaccurate);
+  result.persistent_memory = CalculateOutputSize(op_info, &result.inaccurate);
   result.num_ops_with_unknown_shapes = result.inaccurate;
 
   result.compute_time = kMinComputeTime;
@@ -1247,20 +1242,19 @@ Costs OpLevelCostEstimator::PredictVariable(const OpContext& op_context) const {
 
 Costs OpLevelCostEstimator::PredictBatchMatMul(
     const OpContext& op_context) const {
-  const auto& op_features = op_context.op_info;
+  const auto& op_info = op_context.op_info;
   bool found_unknown_shapes = false;
   Costs costs = PredictOpCountBasedCost(
-      CountBatchMatMulOperations(op_features, &found_unknown_shapes),
-      op_features);
+      CountBatchMatMulOperations(op_info, &found_unknown_shapes), op_info);
   costs.inaccurate = found_unknown_shapes;
   costs.num_ops_with_unknown_shapes = found_unknown_shapes;
   return costs;
 }
 
 Costs OpLevelCostEstimator::PredictMetadata(const OpContext& op_context) const {
-  const auto& op_features = op_context.op_info;
+  const auto& op_info = op_context.op_info;
   Costs costs = Costs::ZeroCosts();
-  costs.max_memory = CalculateOutputSize(op_features, &costs.inaccurate);
+  costs.max_memory = CalculateOutputSize(op_info, &costs.inaccurate);
   costs.num_ops_with_unknown_shapes = costs.inaccurate;
   // Metadata operations are so cheap we assume they take the minimum amount of
   // time we can represent (1 ns).
diff --git a/tensorflow/core/grappler/costs/op_level_cost_estimator.h b/tensorflow/core/grappler/costs/op_level_cost_estimator.h
index 84dd9213f7..f8ba8c6637 100644
--- a/tensorflow/core/grappler/costs/op_level_cost_estimator.h
+++ b/tensorflow/core/grappler/costs/op_level_cost_estimator.h
@@ -16,10 +16,6 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_GRAPPLER_COSTS_OP_LEVEL_COST_ESTIMATOR_H_
 #define TENSORFLOW_CORE_GRAPPLER_COSTS_OP_LEVEL_COST_ESTIMATOR_H_
 
-#include <functional>
-#include <map>
-#include <string>
-
 #include "tensorflow/core/grappler/costs/cost_estimator.h"
 #include "tensorflow/core/grappler/costs/op_context.h"
 #include "tensorflow/core/grappler/costs/op_performance_data.pb.h"
@@ -79,24 +75,23 @@ class OpLevelCostEstimator {
     int64 sy;         // Stride y.
     Padding padding;  // SAME or VALID.
   };
-  int64 CountConv2DOperations(const OpInfo& op_features,
+  int64 CountConv2DOperations(const OpInfo& op_info,
                               bool* found_unknown_shapes) const;
-  int64 CountConv2DOperations(const OpInfo& op_features,
+  int64 CountConv2DOperations(const OpInfo& op_info,
                               ConvolutionDimensions* conv_info,
                               bool* found_unknown_shapes) const;
-  int64 CountMatMulOperations(const OpInfo& op_features,
+  int64 CountMatMulOperations(const OpInfo& op_info,
                               bool* found_unknown_shapes) const;
-  int64 CountMatMulOperations(const OpInfo& op_features,
-                              MatMulDimensions* mat_mul,
+  int64 CountMatMulOperations(const OpInfo& op_info, MatMulDimensions* mat_mul,
                               bool* found_unknown_shapes) const;
-  int64 CountBatchMatMulOperations(const OpInfo& op_features,
+  int64 CountBatchMatMulOperations(const OpInfo& op_info,
                                    bool* found_unknown_shapes) const;
-  int64 CountConv2DBackpropInputOperations(const OpInfo& op_features,
-                                           ConvolutionDimensions* conv_info,
-                                           bool* found_unknown_shapes) const;
-  int64 CountConv2DBackpropFilterOperations(const OpInfo& op_features,
-                                            ConvolutionDimensions* conv_info,
-                                            bool* found_unknown_shapes) const;
+  int64 CountConv2DBackpropInputOperations(
+      const OpInfo& op_info, ConvolutionDimensions* returned_conv_dims,
+      bool* found_unknown_shapes) const;
+  int64 CountConv2DBackpropFilterOperations(
+      const OpInfo& op_info, ConvolutionDimensions* returned_conv_dims,
+      bool* found_unknown_shapes) const;
 
   // Calculate the element count of an input/output tensor.
   int64 CalculateTensorElementCount(const OpInfo::TensorProperties& tensor,
@@ -108,17 +103,17 @@ class OpLevelCostEstimator {
 
   // Calculate the element count of the largest
   // input of specified TensorFlow op.
-  int64 CalculateLargestInputCount(const OpInfo& op_features,
+  int64 CalculateLargestInputCount(const OpInfo& op_info,
                                    bool* found_unknown_shapes) const;
 
   // Calculate the total size in bytes of the all
   // the inputs of specified TensorFlow op.
-  int64 CalculateInputSize(const OpInfo& op_features,
+  int64 CalculateInputSize(const OpInfo& op_info,
                            bool* found_unknown_shapes) const;
 
   // Calculate the total size in bytes of the all
   // the outputs of specified TensorFlow op.
-  int64 CalculateOutputSize(const OpInfo& op_features,
+  int64 CalculateOutputSize(const OpInfo& op_info,
                             bool* found_unknown_shapes) const;
 
   // This family of routines predicts the costs to
@@ -205,4 +200,5 @@ class OpLevelCostEstimator {
 
 }  // end namespace grappler
 }  // end namespace tensorflow
+
 #endif  // TENSORFLOW_CORE_GRAPPLER_COSTS_OP_LEVEL_COST_ESTIMATOR_H_
diff --git a/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc b/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc
index 9a59877ac5..6a9bf13b93 100644
--- a/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc
+++ b/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc
@@ -29,8 +29,8 @@ namespace grappler {
 
 namespace {
 // Wrangles the minimum number of proto fields to set up a matrix.
-void DescribeMatrix(int rows, int columns, OpInfo* op_features) {
-  auto input = op_features->add_inputs();
+void DescribeMatrix(int rows, int columns, OpInfo* op_info) {
+  auto input = op_info->add_inputs();
   auto shape = input->mutable_shape();
   auto shape_rows = shape->add_dim();
   shape_rows->set_size(rows);
@@ -39,8 +39,8 @@ void DescribeMatrix(int rows, int columns, OpInfo* op_features) {
   input->set_dtype(DT_FLOAT);
 }
 
-void SetCpuDevice(OpInfo* op_features) {
-  auto device = op_features->mutable_device();
+void SetCpuDevice(OpInfo* op_info) {
+  auto device = op_info->mutable_device();
   device->set_type("CPU");
   device->set_num_cores(10);
   device->set_bandwidth(10000000);  // 10000000 KB/s = 10 GB/s
@@ -413,15 +413,14 @@ class OpLevelCostEstimatorTest : public ::testing::Test {
     return estimator_.PredictCosts(op_context);
   }
 
-  int64 CountMatMulOperations(const OpInfo& op_features,
+  int64 CountMatMulOperations(const OpInfo& op_info,
                               bool* found_unknown_shapes) const {
-    return estimator_.CountMatMulOperations(op_features, found_unknown_shapes);
+    return estimator_.CountMatMulOperations(op_info, found_unknown_shapes);
   }
 
-  int64 CountBatchMatMulOperations(const OpInfo& op_features,
+  int64 CountBatchMatMulOperations(const OpInfo& op_info,
                                    bool* found_unknown_shapes) const {
-    return estimator_.CountBatchMatMulOperations(op_features,
-                                                 found_unknown_shapes);
+    return estimator_.CountBatchMatMulOperations(op_info, found_unknown_shapes);
   }
 
   void SetComputeMemoryOverlap(bool value) {
-- 
GitLab


From 7cadc29b011e2dd73f70034f280a13931c73900e Mon Sep 17 00:00:00 2001
From: Doe Hyun Yoon <dyoon@google.com>
Date: Mon, 7 Jan 2019 16:10:33 -0800
Subject: [PATCH 0337/1095] Add PredictCostsAndReturnRunMetadata method to
 CostEstimator.

PiperOrigin-RevId: 228254629
---
 tensorflow/core/grappler/costs/BUILD          |  1 +
 .../costs/analytical_cost_estimator.cc        | 37 +++++++++++--------
 .../costs/analytical_cost_estimator.h         | 14 +++----
 .../core/grappler/costs/cost_estimator.h      | 14 +++++++
 4 files changed, 42 insertions(+), 24 deletions(-)

diff --git a/tensorflow/core/grappler/costs/BUILD b/tensorflow/core/grappler/costs/BUILD
index 15dc7074b9..92294379b5 100644
--- a/tensorflow/core/grappler/costs/BUILD
+++ b/tensorflow/core/grappler/costs/BUILD
@@ -171,6 +171,7 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = [
         "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
     ],
 )
 
diff --git a/tensorflow/core/grappler/costs/analytical_cost_estimator.cc b/tensorflow/core/grappler/costs/analytical_cost_estimator.cc
index b7804ffaa5..eac6a0de3f 100644
--- a/tensorflow/core/grappler/costs/analytical_cost_estimator.cc
+++ b/tensorflow/core/grappler/costs/analytical_cost_estimator.cc
@@ -104,17 +104,15 @@ AnalyticalCostEstimator::AnalyticalCostEstimator(Cluster* cluster,
                                                  bool use_static_shapes)
     : AnalyticalCostEstimator(
           cluster, absl::make_unique<OpLevelCostEstimator>(),
-          ReadyNodeManagerFactory("FirstReady"), use_static_shapes, nullptr) {}
+          ReadyNodeManagerFactory("FirstReady"), use_static_shapes) {}
 
 AnalyticalCostEstimator::AnalyticalCostEstimator(
     Cluster* cluster, std::unique_ptr<OpLevelCostEstimator> node_estimator,
-    std::unique_ptr<ReadyNodeManager> node_manager, bool use_static_shapes,
-    RunMetadata* run_metadata)
+    std::unique_ptr<ReadyNodeManager> node_manager, bool use_static_shapes)
     : cluster_(cluster),
       node_estimator_(std::move(node_estimator)),
       node_manager_(std::move(node_manager)),
-      use_static_shapes_(use_static_shapes),
-      run_metadata_(run_metadata) {
+      use_static_shapes_(use_static_shapes) {
   scheduler_ = absl::make_unique<VirtualScheduler>(use_static_shapes_, cluster_,
                                                    node_manager_.get());
 }
@@ -128,6 +126,18 @@ Status AnalyticalCostEstimator::Initialize(const GrapplerItem& item) {
 Status AnalyticalCostEstimator::PredictCosts(const GraphDef& optimized_graph,
                                              CostGraphDef* cost_graph,
                                              Costs* costs) const {
+  RunMetadata run_metadata;
+  auto s =
+      PredictCostsAndReturnRunMetadata(optimized_graph, &run_metadata, costs);
+  if (s.ok() && cost_graph) {
+    cost_graph->Swap(run_metadata.mutable_cost_graph());
+  }
+  return s;
+}
+
+Status AnalyticalCostEstimator::PredictCostsAndReturnRunMetadata(
+    const GraphDef& optimized_graph, RunMetadata* run_metadata,
+    Costs* costs) const {
   GrapplerItem item = item_;
   item.graph = optimized_graph;
 
@@ -138,7 +148,9 @@ Status AnalyticalCostEstimator::PredictCosts(const GraphDef& optimized_graph,
   }
 
   gtl::FlatMap<string, CostGraphDef::Node*> name_to_cost_node;
-  if (cost_graph) {
+  CostGraphDef* cost_graph = nullptr;
+  if (run_metadata) {
+    cost_graph = run_metadata->mutable_cost_graph();
     // TODO(pcma): Clear nodes in cost_graph after we make sure we always pass
     // in an empty cost_graph (a non-empty but incomplete cost_graph will cause
     // problems, e.g., no node_id in cost_graph)
@@ -179,18 +191,13 @@ Status AnalyticalCostEstimator::PredictCosts(const GraphDef& optimized_graph,
     }
   }
 
-  *costs = scheduler_->Summary(run_metadata_);
-  // run_metadata_ gets step_stats and parition_graphs from Summary.
-  // Note that cost_graph could already point to the cost_graph field of
-  // run_metadata_, since both are set by the caller.
-  if (run_metadata_ && cost_graph &&
-      run_metadata_->mutable_cost_graph() != cost_graph)
-    *run_metadata_->mutable_cost_graph() = *cost_graph;
+  // run_metadata gets step_stats and partition_graphs from Summary.
+  *costs = scheduler_->Summary(run_metadata);
 
   if (VLOG_IS_ON(1)) {
     bool verbose = VLOG_IS_ON(2);
-    if (run_metadata_) {
-      VLOG(1) << GetStatsStringFromRunMetadata(*run_metadata_, verbose);
+    if (run_metadata) {
+      VLOG(1) << GetStatsStringFromRunMetadata(*run_metadata, verbose);
     } else {
       RunMetadata run_metadata;
       scheduler_->GenerateRunMetadata(&run_metadata);
diff --git a/tensorflow/core/grappler/costs/analytical_cost_estimator.h b/tensorflow/core/grappler/costs/analytical_cost_estimator.h
index 2629672459..ade61f4051 100644
--- a/tensorflow/core/grappler/costs/analytical_cost_estimator.h
+++ b/tensorflow/core/grappler/costs/analytical_cost_estimator.h
@@ -39,16 +39,10 @@ class AnalyticalCostEstimator : public CostEstimator {
  public:
   // Does not take ownership of cluster.
   AnalyticalCostEstimator(Cluster* cluster, bool use_static_shapes);
-  // Does not take ownership of cluster or run_metadata
-  //
-  // When metadata is provided, step_stats and partition_graphs fields will
-  // always be filled during PredictCosts, and the cost_graph field of metadata
-  // will be filled only when cost_graph is not nullptr when invoking
-  // PredictCosts.
   AnalyticalCostEstimator(Cluster* cluster,
                           std::unique_ptr<OpLevelCostEstimator> node_estimator,
                           std::unique_ptr<ReadyNodeManager> node_manager,
-                          bool use_static_shapes, RunMetadata* run_metadata);
+                          bool use_static_shapes);
   ~AnalyticalCostEstimator() override {}
 
   // Initializes the estimator for the specified grappler item.
@@ -61,6 +55,10 @@ class AnalyticalCostEstimator : public CostEstimator {
   Status PredictCosts(const GraphDef& optimized_graph, CostGraphDef* cost_graph,
                       Costs* cost) const override;
 
+  Status PredictCostsAndReturnRunMetadata(const GraphDef& optimized_graph,
+                                          RunMetadata* run_metadata,
+                                          Costs* cost) const override;
+
   const VirtualScheduler* GetScheduler() const { return scheduler_.get(); }
 
  private:
@@ -70,8 +68,6 @@ class AnalyticalCostEstimator : public CostEstimator {
   std::unique_ptr<ReadyNodeManager> node_manager_;
   bool use_static_shapes_;
   std::unique_ptr<VirtualScheduler> scheduler_;
-
-  RunMetadata* run_metadata_;
 };
 
 }  // end namespace grappler
diff --git a/tensorflow/core/grappler/costs/cost_estimator.h b/tensorflow/core/grappler/costs/cost_estimator.h
index e3b3a36b09..725d81a881 100644
--- a/tensorflow/core/grappler/costs/cost_estimator.h
+++ b/tensorflow/core/grappler/costs/cost_estimator.h
@@ -20,6 +20,7 @@ limitations under the License.
 #include <cmath>
 #include <unordered_map>
 #include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/protobuf/config.pb.h"
 
 namespace tensorflow {
 class GraphDef;
@@ -223,6 +224,19 @@ class CostEstimator {
   // not.
   virtual Status PredictCosts(const GraphDef& optimized_graph,
                               CostGraphDef* cost_graph, Costs* cost) const = 0;
+
+  // TODO(dyoon): Delete PredictCosts() with CostGraphDef as RunMetadata is a
+  // superset of CostGraphDef.
+  // Same method, but returns RunMetadata.
+  virtual Status PredictCostsAndReturnRunMetadata(
+      const GraphDef& optimized_graph, RunMetadata* run_metadata,
+      Costs* cost) const {
+    CostGraphDef* cost_graph = nullptr;
+    if (run_metadata) {
+      cost_graph = run_metadata->mutable_cost_graph();
+    }
+    return PredictCosts(optimized_graph, cost_graph, cost);
+  }
 };
 
 }  // end namespace grappler
-- 
GitLab


From acfc65f0110e272abc8bd1abce867f2f534ba263 Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Mon, 7 Jan 2019 16:11:20 -0800
Subject: [PATCH 0338/1095] [TF2XLA] Use dynamic dimension in SizeOp.

This is needed if the number of elements in a tensor can be dynamic.

PiperOrigin-RevId: 228254723
---
 tensorflow/compiler/tf2xla/kernels/shape_op.cc | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/shape_op.cc b/tensorflow/compiler/tf2xla/kernels/shape_op.cc
index 12830816ec..85b0367f73 100644
--- a/tensorflow/compiler/tf2xla/kernels/shape_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/shape_op.cc
@@ -20,6 +20,7 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_helpers.h"
 #include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
+#include "tensorflow/compiler/xla/client/lib/constants.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/core/framework/kernel_def_builder.h"
 #include "tensorflow/core/framework/tensor_shape.h"
@@ -91,14 +92,20 @@ class SizeOp : public XlaOpKernel {
 
   void Compile(XlaOpKernelContext* ctx) override {
     const TensorShape input_shape = ctx->InputShape(0);
-    const int64 size = input_shape.num_elements();
-    OP_REQUIRES(ctx, FastBoundsCheck(size, std::numeric_limits<int32>::max()),
+    OP_REQUIRES(ctx,
+                FastBoundsCheck(input_shape.num_elements(),
+                                std::numeric_limits<int32>::max()),
                 errors::InvalidArgument("Size does not work for tensors > "
                                         "int32 max."));
     Tensor size_constant(DT_INT32, TensorShape({}));
-    size_constant.scalar<int32>()() = static_cast<int32>(size);
-
-    ctx->SetConstantOutput(0, size_constant);
+    const int rank = input_shape.dims();
+    xla::XlaBuilder* builder = ctx->builder();
+    auto size = xla::One(builder, xla::U32);
+    for (int64 i = 0; i < rank; ++i) {
+      size = xla::Mul(size, xla::GetDimensionSize(ctx->Input(0), i));
+    }
+    size = xla::ConvertElementType(size, xla::S32);
+    ctx->SetOutput(0, size);
   }
 };
 
-- 
GitLab


From ec569e692435fb68191553fd7ffa5b633fdf1ef2 Mon Sep 17 00:00:00 2001
From: Andy Ly <lyandy@google.com>
Date: Mon, 7 Jan 2019 16:20:48 -0800
Subject: [PATCH 0339/1095] [Grappler] Replace NodeMap with MutableGraphView in
 DependencyOptimizer.

PiperOrigin-RevId: 228256183
---
 tensorflow/core/grappler/optimizers/BUILD     |   4 +
 .../optimizers/dependency_optimizer.cc        | 456 +++++++-----------
 .../optimizers/dependency_optimizer.h         |  24 +-
 .../optimizers/dependency_optimizer_test.cc   |   9 +-
 4 files changed, 202 insertions(+), 291 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index 7e29cee86a..5a328a91c5 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -303,14 +303,18 @@ cc_library(
         ":constant_folding",
         ":graph_optimizer",
         "//tensorflow/core:framework",
+        "//tensorflow/core:graph",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core/grappler:grappler_item",
+        "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/costs:graph_properties",
         "//tensorflow/core/grappler/utils:topological_sort",
+        "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/container:flat_hash_set",
     ],
 )
 
diff --git a/tensorflow/core/grappler/optimizers/dependency_optimizer.cc b/tensorflow/core/grappler/optimizers/dependency_optimizer.cc
index 7fee3ae9d5..9bceb6b906 100644
--- a/tensorflow/core/grappler/optimizers/dependency_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/dependency_optimizer.cc
@@ -18,10 +18,14 @@ limitations under the License.
 #include <unordered_map>
 #include <unordered_set>
 
+#include "absl/container/flat_hash_map.h"
+#include "absl/container/flat_hash_set.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/graph/tensor_id.h"
 #include "tensorflow/core/grappler/costs/graph_properties.h"
 #include "tensorflow/core/grappler/grappler_item.h"
+#include "tensorflow/core/grappler/mutable_graph_view.h"
 #include "tensorflow/core/grappler/op_types.h"
 #include "tensorflow/core/grappler/optimizers/constant_folding.h"
 #include "tensorflow/core/grappler/utils.h"
@@ -38,20 +42,15 @@ namespace grappler {
 
 namespace {
 
-bool RemoveInput(NodeDef* node, const string& input, NodeMap* node_map) {
-  bool removed_input = false;
-  int pos = 0;
-  while (pos < node->input_size()) {
-    if (node->input(pos) == input) {
-      node->mutable_input()->SwapElements(pos, node->input_size() - 1);
-      node->mutable_input()->RemoveLast();
-      node_map->RemoveOutput(NodeName(input), node->name());
-      removed_input = true;
-    } else {
-      ++pos;
-    }
+// Builds a map from the &graph->node(i) to i.
+absl::flat_hash_map<const NodeDef*, int> BuildNodeToIdx(const GraphDef& graph) {
+  // Set up &node -> index map.
+  absl::flat_hash_map<const NodeDef*, int> node_to_idx;
+  for (int i = 0; i < graph.node_size(); ++i) {
+    const NodeDef& node = graph.node(i);
+    node_to_idx[&node] = i;
   }
-  return removed_input;
+  return node_to_idx;
 }
 
 }  // namespace
@@ -68,7 +67,10 @@ bool DependencyOptimizer::SafeToRemoveIdentity(const NodeDef& node) const {
     // The output values of this node may be needed.
     return false;
   }
-  const NodeDef* input = node_map_->GetNode(NodeName(node.input(0)));
+
+  MutableGraphView::OutputPort port = graph_view_->GetRegularFanin(
+      MutableGraphView::InputPort(const_cast<NodeDef*>(&node), 0));
+  NodeDef* input = port.node;
   CHECK(input != nullptr) << "node = " << node.name()
                           << " input = " << node.input(0);
   // Don't remove Identity nodes corresponding to Variable reads or following
@@ -79,20 +81,23 @@ bool DependencyOptimizer::SafeToRemoveIdentity(const NodeDef& node) const {
     // Don't turn Identity nodes following Switch into NoOp or remove them
     // if it requires anchoring a control dependencies the Switch node, which
     // is not valid.
-    if (str_util::StartsWith(node.name(), kConstantFoldingCtrl)) {
-      // TODO(rmlarsen): Try to remove this artificial contraint.
+    MutableGraphView::OutputPort control_port(const_cast<NodeDef*>(&node),
+                                              Graph::kControlSlot);
+    if (!graph_view_->GetFanout(control_port).empty()) {
       return false;
     }
   }
-  for (auto consumer : node_map_->GetOutputs(node.name())) {
-    if (node.input_size() > 1 && IsMerge(*consumer)) {
+  bool node_has_multiple_inputs =
+      graph_view_->NumFanins(node, /*include_controlling_nodes=*/true) > 1;
+  for (auto consumer :
+       graph_view_->GetFanouts(node, /*include_controlled_nodes=*/true)) {
+    if (node_has_multiple_inputs && IsMerge(*consumer.node)) {
       return false;
     }
     if (IsSwitch(*input)) {
-      for (const string& consumer_input : consumer->input()) {
-        if (consumer_input == AsControlDependency(node.name())) {
-          return false;
-        }
+      if (graph_view_->HasFanin(*consumer.node,
+                                {node.name(), Graph::kControlSlot})) {
+        return false;
       }
     }
   }
@@ -126,7 +131,7 @@ bool DependencyOptimizer::SafeToConvertToNoOp(const NodeDef& node) const {
   if (!SafeToRemoveIdentity(node)) {
     return false;
   }
-  if (NumNonControlOutputs(node, *node_map_) > 0) {
+  if (graph_view_->NumFanouts(node, /*include_controlled_nodes=*/false) > 0) {
     // The output values of this node may be needed.
     return false;
   }
@@ -134,61 +139,56 @@ bool DependencyOptimizer::SafeToConvertToNoOp(const NodeDef& node) const {
 }
 
 int DependencyOptimizer::NumEdgesIfBypassed(
-    const NodeDef& node, const std::vector<NodeDef*>& output_nodes) const {
+    const NodeDef& node, int num_fanins,
+    const absl::flat_hash_set<MutableGraphView::Edge>& fanout_edges) const {
   const bool is_multi_input_identity_n =
       IsIdentityN(node) && !IsIdentityNSingleInput(node);
-  const int num_outputs = output_nodes.size();
-  const int num_inputs = node.input_size();
+  const int num_fanouts = fanout_edges.size();
 
   if (is_multi_input_identity_n) {
     // multi-input identity_n with input/output control dependencies will likely
     // increase number of edges after optimization.
     int num_edges_if_bypassed(0);
-    for (string input_node_name : node.input()) {
-      if (IsControlInput(input_node_name)) {
-        num_edges_if_bypassed += num_outputs;
+    int num_non_controlling_fanins =
+        graph_view_->NumFanins(node, /*include_controlling_nodes=*/false);
+    num_edges_if_bypassed += num_non_controlling_fanins;
+    num_edges_if_bypassed +=
+        (num_fanins - num_non_controlling_fanins) * num_fanouts;
+
+    for (const auto& fanout : fanout_edges) {
+      if (fanout.dst.port_id == Graph::kControlSlot) {
+        num_edges_if_bypassed += num_fanins;
       } else {
         ++num_edges_if_bypassed;
       }
     }
-
-    for (auto consumer : output_nodes) {
-      for (int j = 0; j < consumer->input_size(); ++j) {
-        const TensorId consumer_input = ParseTensorName(consumer->input(j));
-        if (consumer_input.node() == node.name()) {
-          if (IsControlInput(consumer_input)) {
-            num_edges_if_bypassed += num_inputs;
-          } else {
-            ++num_edges_if_bypassed;
-          }
-        }
-      }
-    }
     return num_edges_if_bypassed;
   } else {
-    return num_inputs * num_outputs;
+    return num_fanins * num_fanouts;
   }
 }
 
 bool DependencyOptimizer::BypassingNodeIsBeneficial(
-    const NodeDef& node, const std::vector<NodeDef*>& input_nodes,
-    const std::vector<NodeDef*>& output_nodes) const {
+    const NodeDef& node,
+    const absl::flat_hash_set<MutableGraphView::OutputPort>& fanins,
+    const absl::flat_hash_set<MutableGraphView::Edge>& fanout_edges) const {
   const bool is_identity = IsIdentity(node) || IsIdentityNSingleInput(node);
   const bool is_multi_input_identity_n =
       IsIdentityN(node) && !IsIdentityNSingleInput(node);
-  const int num_outputs = output_nodes.size();
-  const int num_inputs = node.input_size();
+  const int num_outputs = fanout_edges.size();
+  const int num_inputs = fanins.size();
 
-  if (NumEdgesIfBypassed(node, output_nodes) > num_inputs + num_outputs) {
+  if (NumEdgesIfBypassed(node, num_inputs, fanout_edges) >
+      num_inputs + num_outputs) {
     return false;
   }
 
   // Make sure that we don't increase the number of edges that cross
   // device boundaries.
   if ((num_inputs == 1 && num_outputs > 1 &&
-       input_nodes[0]->device() != node.device()) ||
+       fanins.begin()->node->device() != node.device()) ||
       (num_inputs > 1 && num_outputs == 1 &&
-       output_nodes[0]->device() != node.device())) {
+       fanout_edges.begin()->dst.node->device() != node.device())) {
     return false;
   }
 
@@ -197,12 +197,12 @@ bool DependencyOptimizer::BypassingNodeIsBeneficial(
   // cost before and after.
   const string& node_dev = node.device();
   int num_cross_in = 0;
-  for (NodeDef* input_node : input_nodes) {
-    num_cross_in += static_cast<int>(input_node->device() != node_dev);
+  for (const auto& fanin : fanins) {
+    num_cross_in += static_cast<int>(fanin.node->device() != node_dev);
   }
   int num_cross_out = 0;
-  for (NodeDef* output_node : output_nodes) {
-    num_cross_out += static_cast<int>(output_node->device() != node_dev);
+  for (const auto& fanout : fanout_edges) {
+    num_cross_out += static_cast<int>(fanout.dst.node->device() != node_dev);
   }
 
   if ((is_identity || is_multi_input_identity_n) && num_cross_in > 0 &&
@@ -216,10 +216,10 @@ bool DependencyOptimizer::BypassingNodeIsBeneficial(
   // Make sure we do not increase the number of device crossings.
   const int num_cross_before = num_cross_in + num_cross_out;
   int num_cross_after = 0;
-  for (NodeDef* input_node : input_nodes) {
-    for (NodeDef* output_node : output_nodes) {
+  for (const auto& fanin : fanins) {
+    for (const auto& fanout : fanout_edges) {
       num_cross_after +=
-          static_cast<int>(input_node->device() != output_node->device());
+          static_cast<int>(fanin.node->device() != fanout.dst.node->device());
     }
   }
   if (num_cross_after > num_cross_before) {
@@ -228,45 +228,32 @@ bool DependencyOptimizer::BypassingNodeIsBeneficial(
   return true;
 }
 
-void DependencyOptimizer::OptimizeNode(int node_idx,
-                                       SetVector<int>* nodes_to_simplify,
-                                       std::set<int>* nodes_to_delete) {
-  NodeDef* node = optimized_graph_->mutable_node(node_idx);
+void DependencyOptimizer::OptimizeNode(NodeDef* node,
+                                       SetVector<NodeDef*>* nodes_to_simplify,
+                                       std::set<string>* nodes_to_delete) {
+  const string node_name = node->name();
   const bool is_noop = IsNoOp(*node);
   const bool is_identity = IsIdentity(*node) || IsIdentityNSingleInput(*node);
   const bool is_multi_input_identity =
       IsIdentityN(*node) && !IsIdentityNSingleInput(*node);
-  const string node_name = node->name();
   // Constant nodes with no input control dependency are always executed early,
   // so we can prune all their output control dependencies.
-  if (IsConstant(*node) && node->input_size() == 0) {
-    const std::set<NodeDef*> output_nodes = node_map_->GetOutputs(node_name);
-    for (NodeDef* fanout : output_nodes) {
-      bool optimize_fanout = false;
-      bool data_connection = false;
-      for (int i = fanout->input_size() - 1; i >= 0; --i) {
-        const TensorId input_tensor = ParseTensorName(fanout->input(i));
-        if (input_tensor.node() == node_name) {
-          if (input_tensor.index() < 0) {
-            fanout->mutable_input()->SwapElements(i, fanout->input_size() - 1);
-            fanout->mutable_input()->RemoveLast();
-            optimize_fanout = true;
-          } else {
-            data_connection = true;
-          }
-        }
-      }
-      if (optimize_fanout) {
-        nodes_to_simplify->PushBack(node_to_idx_[fanout]);
-        if (!data_connection) {
-          node_map_->RemoveOutput(node_name, fanout->name());
-        }
+  if (IsConstant(*node) &&
+      graph_view_->NumFanins(*node, /*include_controlling_nodes=*/true) == 0) {
+    for (const auto& fanout :
+         graph_view_->GetFanouts(*node, /*include_controlled_nodes=*/true)) {
+      if (graph_view_->RemoveFanin(fanout.node->name(),
+                                   {node_name, Graph::kControlSlot})) {
+        nodes_to_simplify->PushBack(fanout.node);
       }
     }
-    if (node_map_->GetOutputs(node_name).empty() && fetch_nodes_known_ &&
+
+    if (graph_view_->NumFanouts(*node, /*include_controlled_nodes=*/true) ==
+            0 &&
+        fetch_nodes_known_ &&
         nodes_to_preserve_.find(node_name) == nodes_to_preserve_.end()) {
       // Mark the node for deletion.
-      nodes_to_delete->insert(node_to_idx_[node]);
+      nodes_to_delete->insert(node->name());
     }
     return;
   }
@@ -277,33 +264,23 @@ void DependencyOptimizer::OptimizeNode(int node_idx,
             << ") with NoOp.";
     // The outputs of this node are not consumed. Replace its inputs with
     // control dependencies and replace the op itself with the NoOp op.
-    std::unordered_set<string> ctrl_inputs;
-    int pos = 0;
-    while (pos < node->input_size()) {
-      const string old_input = node->input(pos);
-      if (IsControlInput(old_input)) {
-        if (!ctrl_inputs.insert(old_input).second) {
-          // We found a duplicate control input. Remove it.
-          node->mutable_input()->SwapElements(pos, node->input_size() - 1);
-          node->mutable_input()->RemoveLast();
-        } else {
-          ++pos;
-        }
-        continue;
+    int num_non_controlling_fanins =
+        graph_view_->NumFanins(*node, /*include_controlling_nodes=*/false);
+    if (num_non_controlling_fanins > 0) {
+      absl::flat_hash_set<string> non_controlling_fanins(
+          node->input().begin(),
+          node->input().begin() + num_non_controlling_fanins);
+      graph_view_->RemoveAllFanins(node_name,
+                                   /*keep_controlling_fanins=*/true);
+      for (const string& fanin : non_controlling_fanins) {
+        TensorId tensor_id = ParseTensorName(fanin);
+        graph_view_->AddControllingFanin(node_name, tensor_id);
+        nodes_to_simplify->PushBack(graph_view_->GetNode(tensor_id.node()));
       }
-      // Replace a normal input with a control input.
-      const string ctrl_input = ConstantFolding::AddControlDependency(
-          old_input, optimized_graph_, node_map_.get());
-      ctrl_inputs.insert(ctrl_input);
-      node->set_input(pos, ctrl_input);
-      node_map_->UpdateInput(node_name, old_input, ctrl_input);
-      const NodeDef* old_input_node = node_map_->GetNode(old_input);
-      nodes_to_simplify->PushBack(node_to_idx_[old_input_node]);
-      ++pos;
     }
     node->set_op("NoOp");
     node->clear_attr();
-    nodes_to_simplify->PushBack(node_to_idx_[node]);
+    nodes_to_simplify->PushBack(node);
     return;
   }
 
@@ -357,110 +334,80 @@ void DependencyOptimizer::OptimizeNode(int node_idx,
 
   if (is_noop || ((is_identity || is_multi_input_identity) &&
                   SafeToRemoveIdentity(*node))) {
-    const auto& output_node_set = node_map_->GetOutputs(node_name);
-    const std::vector<NodeDef*> output_nodes(output_node_set.begin(),
-                                             output_node_set.end());
-    const int num_inputs = node->input_size();
-    std::vector<NodeDef*> input_nodes;
-    for (int i = 0; i < num_inputs; ++i) {
-      NodeDef* input_node = node_map_->GetNode(node->input(i));
-      if (input_node == nullptr) {
-        LOG(ERROR) << "Invalid input " << node->input(i);
-        return;
-      }
-      input_nodes.push_back(input_node);
-    }
+    auto fanins =
+        graph_view_->GetFanins(*node, /*include_controlling_nodes=*/true);
+    auto fanout_edges =
+        graph_view_->GetFanoutEdges(*node, /*include_controlled_edges=*/true);
 
-    if (!BypassingNodeIsBeneficial(*node, input_nodes, output_nodes)) {
+    if (!BypassingNodeIsBeneficial(*node, fanins, fanout_edges)) {
       return;
     }
 
+    int num_non_controlling_fanins =
+        graph_view_->NumFanins(*node, /*include_controlling_nodes=*/false);
     VLOG(1) << "***** Rerouting input around\n" << node->DebugString();
     // Now remove the node and re-wire its inputs to its outputs.
-    for (auto consumer : output_nodes) {
+    for (auto fanout_edge : fanout_edges) {
       bool updated_consumer = false;
+      NodeDef* consumer = fanout_edge.dst.node;
       VLOG(1) << "consumer before:\n" << consumer->DebugString();
-      for (int i = 0; i < num_inputs; ++i) {
-        const NodeDef* input = input_nodes[i];
-        // Forward dependency from input to consumer if it doesn't already
-        // depend on it.
-        if ((is_identity && i == 0) ||
-            (is_multi_input_identity && !IsControlInput(node->input(i)))) {
-          // Replace regular input from Identity node.
-          string new_input;
-          const string& input_to_forward = node->input(i);
-          CHECK(!IsControlInput(input_to_forward));
-          for (int j = 0; j < consumer->input_size(); ++j) {
-            const TensorId old_input = ParseTensorName(consumer->input(j));
-            if (old_input.node() == node_name) {
-              if (old_input.index() == i) {
-                // Regular input
-                new_input = input_to_forward;
-                node_map_->UpdateInput(consumer->name(), old_input.ToString(),
-                                       new_input);
-                consumer->set_input(j, new_input);
-              } else if (old_input.index() == -1) {
-                // Control dependency
-                new_input = AsControlDependency(NodeName(input_to_forward));
-                node_map_->UpdateInput(consumer->name(), old_input.ToString(),
-                                       new_input);
-                consumer->set_input(j, new_input);
-              }
+      if (fanout_edge.src.port_id == Graph::kControlSlot) {
+        updated_consumer = graph_view_->RemoveFanin(
+            consumer->name(), {node_name, Graph::kControlSlot});
+        if (updated_consumer) {
+          // Add all non controlling fanins of node as controlling fanins to
+          // consumer.
+          for (int i = 0; i < num_non_controlling_fanins; ++i) {
+            TensorId input = ParseTensorName(node->input(i));
+            if (graph_view_->AddControllingFanin(consumer->name(), input)) {
+              nodes_to_simplify->PushBack(graph_view_->GetNode(input.node()));
             }
           }
-          updated_consumer = true;
-        } else {
-          // Forward dependency from input to consumer if it doesn't already
-          // depend on it.
-          if (node_map_->GetOutputs(input->name()).count(consumer) == 0) {
-            consumer->add_input(AsControlDependency(input->name()));
-            node_map_->AddOutput(input->name(), consumer->name());
-            nodes_to_simplify->PushBack(node_to_idx_[input]);
-            updated_consumer = true;
-          }
         }
+      } else {
+        updated_consumer = graph_view_->UpdateFanin(
+            consumer->name(), {node_name, fanout_edge.src.port_id},
+            ParseTensorName(node->input(fanout_edge.src.port_id)));
       }
-      // Remove dependency on node from consumer.
-      updated_consumer |= RemoveInput(consumer, AsControlDependency(node_name),
-                                      node_map_.get());
       if (updated_consumer) {
-        nodes_to_simplify->PushBack(node_to_idx_[consumer]);
+        // Forward all controlling fanins of node to consumer.
+        for (int i = num_non_controlling_fanins; i < node->input_size(); ++i) {
+          TensorId input = ParseTensorName(node->input(i));
+          if (graph_view_->AddFanin(consumer->name(), input)) {
+            nodes_to_simplify->PushBack(graph_view_->GetNode(input.node()));
+          }
+        }
+        nodes_to_simplify->PushBack(consumer);
       }
       VLOG(1) << "consumer after:\n" << consumer->DebugString();
     }
-    node_map_->RemoveOutputs(node_name);
     if (fetch_nodes_known_ &&
         nodes_to_preserve_.find(node_name) == nodes_to_preserve_.end()) {
       // Mark the node for deletion.
-      nodes_to_delete->insert(node_idx);
+      nodes_to_delete->insert(node_name);
 
       // Disconnect the node from its inputs to enable further optimizations.
-      node_map_->RemoveInputs(node_name);
-      node->clear_input();
+      graph_view_->RemoveAllFanins(node_name,
+                                   /*keep_controlling_fanins=*/false);
     }
   }
 }
 
-void DependencyOptimizer::CleanControlInputs() {
-  for (int i = 0; i < optimized_graph_->node_size(); ++i) {
-    DedupControlInputs(optimized_graph_->mutable_node(i));
-  }
-}
-
 Status DependencyOptimizer::OptimizeDependencies() {
-  SetVector<int> nodes_to_simplify;
-  std::set<int> nodes_to_delete;
-  for (int i = 0; i < optimized_graph_->node_size(); ++i) {
-    const NodeDef& node = optimized_graph_->node(i);
-    if (IsNoOp(node) || IsIdentity(node) || IsIdentityN(node) ||
-        IsConstant(node) || SafeToConvertToNoOp(node)) {
-      nodes_to_simplify.PushBack(i);
+  SetVector<NodeDef*> nodes_to_simplify;
+  std::set<string> nodes_to_delete;
+  for (int i = 0; i < graph_view_->graph()->node_size(); ++i) {
+    NodeDef* node = graph_view_->graph()->mutable_node(i);
+    if (IsNoOp(*node) || IsIdentity(*node) || IsIdentityN(*node) ||
+        IsConstant(*node) || SafeToConvertToNoOp(*node)) {
+      nodes_to_simplify.PushBack(node);
     }
   }
   while (!nodes_to_simplify.Empty()) {
-    int node_to_simplify = nodes_to_simplify.PopBack();
+    NodeDef* node_to_simplify = nodes_to_simplify.PopBack();
     // Discard nodes that were marked for deletion already.
-    while (nodes_to_delete.find(node_to_simplify) != nodes_to_delete.end()) {
+    while (nodes_to_delete.find(node_to_simplify->name()) !=
+           nodes_to_delete.end()) {
       node_to_simplify = nodes_to_simplify.PopBack();
     }
     OptimizeNode(node_to_simplify, &nodes_to_simplify, &nodes_to_delete);
@@ -468,17 +415,17 @@ Status DependencyOptimizer::OptimizeDependencies() {
 
   if (fetch_nodes_known_) {
     VLOG(1) << "Deleted " << nodes_to_delete.size() << " out of "
-            << optimized_graph_->node_size() << " nodes.";
-    EraseNodesFromGraph(nodes_to_delete, optimized_graph_);
-    node_map_.reset(new NodeMap(optimized_graph_));
-    BuildNodeToIdx();
+            << graph_view_->graph()->node_size() << " nodes.";
+    graph_view_->DeleteNodes(nodes_to_delete);
   }
   return Status::OK();
 }
 
 Status DependencyOptimizer::TransitiveReduction() {
   // PRECONDITION: optimized_graph_ must be sorted topologically.
-  const int num_nodes = optimized_graph_->node_size();
+  GraphDef* graph = graph_view_->graph();
+  auto node_to_idx = BuildNodeToIdx(*graph);
+  const int num_nodes = graph->node_size();
   // Set up a compressed version of the graph to save a constant factor in the
   // expensive algorithm below. Also cache the set of control outputs and the
   // highest index of a target of any control output from each node.
@@ -487,20 +434,20 @@ Status DependencyOptimizer::TransitiveReduction() {
   std::vector<gtl::InlinedVector<std::pair<int, int>, 2>> control_outputs(
       num_nodes);
   for (int node_idx = 0; node_idx < num_nodes; ++node_idx) {
-    const NodeDef& node = optimized_graph_->node(node_idx);
+    const NodeDef& node = graph->node(node_idx);
     if (ModifiesFrameInfo(node) || !HasOpDef(node)) {
       // Ignore function nodes and nodes that modify frame info.
       continue;
     }
     for (int input_slot = 0; input_slot < node.input_size(); ++input_slot) {
       const string& input = node.input(input_slot);
-      const NodeDef* input_node = node_map_->GetNode(input);
+      const NodeDef* input_node = graph_view_->GetNode(NodeName(input));
       if (ModifiesFrameInfo(*input_node) || IsMerge(*input_node)) {
         // Ignore edges from nodes that modify frame info and from Merge nodes,
         // because we cannot know which of it's input paths executes.
         continue;
       }
-      const int input_node_idx = node_to_idx_[input_node];
+      const int input_node_idx = node_to_idx[input_node];
       inputs[node_idx].push_back(input_node_idx);
       if (IsControlInput(input)) {
         ++num_controls;
@@ -566,16 +513,12 @@ Status DependencyOptimizer::TransitiveReduction() {
 
   for (const auto& it : control_edges_to_remove) {
     const int target = it.first;
-    NodeDef* target_node = optimized_graph_->mutable_node(target);
+    const NodeDef& target_node = graph->node(target);
+    const string target_node_name = target_node.name();
     for (const InputSlotAndSource& slot_and_source : it.second) {
       const int input_slot = slot_and_source.first;
-      const int source = slot_and_source.second;
-      const NodeDef& source_node = optimized_graph_->node(source);
-      CHECK_LT(input_slot, target_node->input_size());
-      target_node->mutable_input()->SwapElements(input_slot,
-                                                 target_node->input_size() - 1);
-      node_map_->RemoveOutput(source_node.name(), target_node->name());
-      target_node->mutable_input()->RemoveLast();
+      const TensorId tensor_id = ParseTensorName(target_node.input(input_slot));
+      graph_view_->RemoveFanin(target_node_name, tensor_id);
       ++num_controls_removed;
     }
   }
@@ -584,26 +527,17 @@ Status DependencyOptimizer::TransitiveReduction() {
   return Status::OK();
 }
 
-void DependencyOptimizer::BuildNodeToIdx() {
-  // Set up &node -> index map.
-  node_to_idx_.clear();
-  for (int i = 0; i < optimized_graph_->node_size(); ++i) {
-    const NodeDef& node = optimized_graph_->node(i);
-    node_to_idx_[&node] = i;
-  }
-}
-
 // Suppose there are cross-device control inputs to node C from multiple nodes
 // that are located on another device, e.g., we have control edges:
 // A->C, B->C
 // where A and B are on device X and C is on device Y.
 // We can reduce cross-device communication by introducing an intermediate
 // NoOp node C' on device X and rewriting the control edges to:
-// A->C', B->C', C' -> C
+// A->C', B->C', C'->C
 void DependencyOptimizer::GroupCrossDeviceControlEdges() {
-  const int num_nodes = optimized_graph_->node_size();
+  const int num_nodes = graph_view_->graph()->node_size();
   for (int i = 0; i < num_nodes; ++i) {
-    NodeDef* node = optimized_graph_->mutable_node(i);
+    NodeDef* node = graph_view_->graph()->mutable_node(i);
     if (node->device().empty()) continue;
 
     // Creates new noop nodes for devices on which multiple control inputs are
@@ -614,66 +548,50 @@ void DependencyOptimizer::GroupCrossDeviceControlEdges() {
     // that device.
     std::map<string, NodeDef*> noops;
     int num_noops = 0;
-    for (int j = 0; j < node->input_size(); ++j) {
-      if (IsControlInput(node->input(j))) {
-        const NodeDef* input = node_map_->GetNode(node->input(j));
-        if (input != nullptr && !input->device().empty() &&
-            input->device() != node->device()) {
-          auto emplace_result = noops.emplace(input->device(), nullptr);
-          if (!emplace_result.second &&
-              emplace_result.first->second == nullptr) {
-            // This is the second cross-device control input from the same
-            // device. Creates an intermediate noop node on that device.
-            string group_name;
-            NodeDef* noop;
-            // Creates a fresh node name; there may be conflicting names from
-            // a previous iteration of the optimizer.
-            do {
-              group_name = AddPrefixToNodeName(
-                  node->name(),
-                  strings::StrCat("GroupCrossDeviceControlEdges_", num_noops));
-              noop = node_map_->GetNode(group_name);
-              ++num_noops;
-            } while (noop != nullptr);
-            noop = optimized_graph_->add_node();
-            noop->set_name(group_name);
-            noop->set_device(input->device());
-            noop->set_op("NoOp");
-            node_map_->AddNode(noop->name(), noop);
-            emplace_result.first->second = noop;
-          }
+    auto controlling_fanins = graph_view_->GetFanin(
+        MutableGraphView::InputPort(node, Graph::kControlSlot));
+    for (const auto& controlling_fanin : controlling_fanins) {
+      const NodeDef* fanin_node = controlling_fanin.node;
+      if (!fanin_node->device().empty() &&
+          fanin_node->device() != node->device()) {
+        auto emplace_result = noops.emplace(fanin_node->device(), nullptr);
+        if (!emplace_result.second && emplace_result.first->second == nullptr) {
+          // This is the second cross-device control input from the same
+          // device. Creates an intermediate noop node on that device.
+          string group_name;
+          NodeDef* noop;
+          // Creates a fresh node name; there may be conflicting names from
+          // a previous iteration of the optimizer.
+          do {
+            group_name = AddPrefixToNodeName(
+                node->name(),
+                strings::StrCat("GroupCrossDeviceControlEdges_", num_noops));
+            noop = graph_view_->GetNode(group_name);
+            ++num_noops;
+          } while (noop != nullptr);
+          NodeDef new_node;
+          new_node.set_name(group_name);
+          new_node.set_device(fanin_node->device());
+          new_node.set_op("NoOp");
+          emplace_result.first->second =
+              graph_view_->AddNode(std::move(new_node));
         }
       }
     }
 
     // Reroute existing control edges to go via the newly introduced NoOp nodes.
-    int pos = 0;
-    while (pos < node->input_size()) {
-      const string& input_name = node->input(pos);
-      if (IsControlInput(input_name)) {
-        NodeDef* input = node_map_->GetNode(input_name);
-        if (input == nullptr) {
-          ++pos;
-        } else {
-          auto it = noops.find(input->device());
-          if (it == noops.end() || it->second == nullptr) {
-            ++pos;
-          } else {
-            node->mutable_input()->SwapElements(pos, node->input_size() - 1);
-            node->mutable_input()->RemoveLast();
-            it->second->add_input(AsControlDependency(*input));
-            node_map_->UpdateOutput(input_name, node->name(),
-                                    it->second->name());
-          }
-        }
-      } else {
-        ++pos;
+    for (const auto& controlling_fanin : controlling_fanins) {
+      auto it = noops.find(controlling_fanin.node->device());
+      if (it != noops.end() && it->second != nullptr) {
+        TensorId tensor_id(controlling_fanin.node->name(), Graph::kControlSlot);
+        graph_view_->RemoveFanin(node->name(), tensor_id);
+        graph_view_->AddFanin(it->second->name(), tensor_id);
       }
     }
     for (const auto& entry : noops) {
       if (entry.second) {
-        node->add_input(AsControlDependency(*entry.second));
-        node_map_->AddOutput(entry.second->name(), node->name());
+        graph_view_->AddFanin(node->name(),
+                              {entry.second->name(), Graph::kControlSlot});
       }
     }
   }
@@ -681,22 +599,17 @@ void DependencyOptimizer::GroupCrossDeviceControlEdges() {
 
 Status DependencyOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
                                      GraphDef* optimized_graph) {
-  optimized_graph_ = optimized_graph;
-  *optimized_graph_ = item.graph;
+  *optimized_graph = item.graph;
   nodes_to_preserve_ = item.NodesToPreserve();
   fetch_nodes_known_ = !item.fetch.empty();
-  CleanControlInputs();
+  graph_view_.reset(new MutableGraphView(optimized_graph));
 
   const int num_iterations = 2;
   for (int iteration = 0; iteration < num_iterations; ++iteration) {
     GRAPPLER_RETURN_IF_DEADLINE_EXCEEDED();
     Status topo_sort_status;
     // Perform topological sort to prepare the graph for transitive reduction.
-    topo_sort_status = TopologicalSort(optimized_graph_);
-    // Set up index-based graph datastructures to speed up analysis steps below.
-    node_map_.reset(new NodeMap(optimized_graph_));
-    BuildNodeToIdx();
-
+    topo_sort_status = TopologicalSort(optimized_graph);
     if (topo_sort_status.ok()) {
       // Remove redundant control dependencies.
       TF_RETURN_IF_ERROR(TransitiveReduction());
@@ -709,9 +622,6 @@ Status DependencyOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
     // nodes.
     TF_RETURN_IF_ERROR(OptimizeDependencies());
 
-    // Dedup control inputs.
-    CleanControlInputs();
-
     GroupCrossDeviceControlEdges();
   }
 
diff --git a/tensorflow/core/grappler/optimizers/dependency_optimizer.h b/tensorflow/core/grappler/optimizers/dependency_optimizer.h
index 7b032673fb..eb7fc89080 100644
--- a/tensorflow/core/grappler/optimizers/dependency_optimizer.h
+++ b/tensorflow/core/grappler/optimizers/dependency_optimizer.h
@@ -17,6 +17,8 @@ limitations under the License.
 #define TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DEPENDENCY_OPTIMIZER_H_
 
 #include <unordered_set>
+
+#include "tensorflow/core/grappler/mutable_graph_view.h"
 #include "tensorflow/core/grappler/optimizers/graph_optimizer.h"
 #include "tensorflow/core/grappler/utils.h"
 #include "tensorflow/core/protobuf/rewriter_config.pb.h"
@@ -46,24 +48,22 @@ class DependencyOptimizer : public GraphOptimizer {
   // Returns true if bypassing node does not increase the number of edges or
   // number of edges crossing a device boundary.
   bool BypassingNodeIsBeneficial(
-      const NodeDef& node, const std::vector<NodeDef*>& input_nodes,
-      const std::vector<NodeDef*>& output_nodes) const;
-  int NumEdgesIfBypassed(const NodeDef& node,
-                         const std::vector<NodeDef*>& output_nodes) const;
+      const NodeDef& node,
+      const absl::flat_hash_set<MutableGraphView::OutputPort>& fanins,
+      const absl::flat_hash_set<MutableGraphView::Edge>& fanout_edges) const;
+  int NumEdgesIfBypassed(
+      const NodeDef& node, int num_fanins,
+      const absl::flat_hash_set<MutableGraphView::Edge>& fanout_edges) const;
   // Returns true if node is not an Identity node or if it is an Identity
   // that is safe to remove.
   bool SafeToRemoveIdentity(const NodeDef& node) const;
   // Returns true if it is safe to convert node to NoOp.
   bool SafeToConvertToNoOp(const NodeDef& node) const;
-  // Removes all duplicate control dependencies.
-  void CleanControlInputs();
-  // Builds a map from the &optimized_graph_->node(i) to i.
-  void BuildNodeToIdx();
   // Tries to optimize the node with the given index, possibly additional
   // optimizations by inserting nodes in nodes_to_simplify, and pruning nodes by
   // inserting them in nodes_to_delete.
-  void OptimizeNode(int node_idx, SetVector<int>* nodes_to_simplify,
-                    std::set<int>* nodes_to_delete);
+  void OptimizeNode(NodeDef* node, SetVector<NodeDef*>* nodes_to_simplify,
+                    std::set<string>* nodes_to_delete);
   // Eliminates redundant control dependencies by computing the transitive
   // reduction of the graph.
   Status TransitiveReduction();
@@ -76,9 +76,7 @@ class DependencyOptimizer : public GraphOptimizer {
   RewriterConfig::Toggle opt_level_;
   bool fetch_nodes_known_;
   std::unordered_set<string> nodes_to_preserve_;
-  std::unique_ptr<NodeMap> node_map_;
-  std::unordered_map<const NodeDef*, int> node_to_idx_;
-  GraphDef* optimized_graph_;  // Not owned.
+  std::unique_ptr<MutableGraphView> graph_view_;
 };
 
 }  // end namespace grappler
diff --git a/tensorflow/core/grappler/optimizers/dependency_optimizer_test.cc b/tensorflow/core/grappler/optimizers/dependency_optimizer_test.cc
index 8d70d9d5c7..673fc987a8 100644
--- a/tensorflow/core/grappler/optimizers/dependency_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/dependency_optimizer_test.cc
@@ -91,7 +91,7 @@ TEST_F(DependencyOptimizerTest, DependenciesDrivenByConstants) {
   // The 'z' node should have been optimized away leaving only 5 nodes.
   EXPECT_EQ(5, output.node_size());
 
-  for (const NodeDef& node : item.graph.node()) {
+  for (const NodeDef& node : output.node()) {
     if (node.name() == "id1" || node.name() == "id2") {
       EXPECT_EQ(1, node.input_size());
       EXPECT_EQ("add", node.input(0));
@@ -125,8 +125,8 @@ TEST_F(DependencyOptimizerTest, ChangeToNoop) {
 
   EXPECT_EQ(item.graph.node_size(), output.node_size());
   int found = 0;
-  for (int i = 0; i < item.graph.node_size(); ++i) {
-    const NodeDef& node = item.graph.node(i);
+  for (int i = 0; i < output.node_size(); ++i) {
+    const NodeDef& node = output.node(i);
     // "add" should get turned into a NoOp and removed.
     EXPECT_NE("add", node.name());
     if (node.name() == "id1") {
@@ -164,7 +164,6 @@ TEST_F(DependencyOptimizerTest, ChangeToNoop_RepeatedInput) {
   item.graph.Swap(&output);
   status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
-  LOG(INFO) << output.DebugString();
 
   EXPECT_EQ(item.graph.node_size(), output.node_size());
   int found = 0;
@@ -748,7 +747,7 @@ TEST_F(DependencyOptimizerTest, Identity_DeviceCrossing_ConsumerOnSameDevice) {
   GraphDef output;
   Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
-  LOG(INFO) << output.DebugString();
+
   EXPECT_EQ(3, output.node_size());
   for (const auto& node : output.node()) {
     EXPECT_NE("x_on_2", node.name());
-- 
GitLab


From fed73306b650d0b5c99689e8b142a610fd8a3488 Mon Sep 17 00:00:00 2001
From: Jian Li <jianlijianli@google.com>
Date: Mon, 7 Jan 2019 16:32:46 -0800
Subject: [PATCH 0340/1095] Update unidirectional rnn test case.

PiperOrigin-RevId: 228257798
---
 tensorflow/lite/kernels/unidirectional_sequence_rnn_test.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/kernels/unidirectional_sequence_rnn_test.cc b/tensorflow/lite/kernels/unidirectional_sequence_rnn_test.cc
index 494c449589..de1f7818bd 100644
--- a/tensorflow/lite/kernels/unidirectional_sequence_rnn_test.cc
+++ b/tensorflow/lite/kernels/unidirectional_sequence_rnn_test.cc
@@ -255,7 +255,7 @@ class HybridUnidirectionalRNNOpModel : public UnidirectionalRNNOpModel {
     tensor_type_ = tensor_type;
   }
 
-  void SetWeights(int weights_idx, std::vector<float> f) {
+  void SetWeights(int weights_idx, const std::vector<float>& f) {
     if (tensor_type_ == TensorType_UINT8) {
       SymmetricQuantizeAndPopulate(weights_idx, f);
     } else {
-- 
GitLab


From b9762fed18234b2d87efc9786384e467008fc674 Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Mon, 7 Jan 2019 16:50:53 -0800
Subject: [PATCH 0341/1095] Add benchmarks for pack_lhs

PiperOrigin-RevId: 228260001
---
 tensorflow/core/kernels/BUILD                 |   1 +
 .../eigen_spatial_convolutions_test.cc        | 226 +++++++++++++++++-
 2 files changed, 214 insertions(+), 13 deletions(-)

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 136902e249..701bda7ef6 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -2511,6 +2511,7 @@ tf_cc_tests(
         ":eigen_helpers",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
+        "@com_google_absl//absl/strings",
     ],
 )
 
diff --git a/tensorflow/core/kernels/eigen_spatial_convolutions_test.cc b/tensorflow/core/kernels/eigen_spatial_convolutions_test.cc
index 22f71d6260..03002adec4 100644
--- a/tensorflow/core/kernels/eigen_spatial_convolutions_test.cc
+++ b/tensorflow/core/kernels/eigen_spatial_convolutions_test.cc
@@ -14,6 +14,8 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/core/kernels/eigen_spatial_convolutions.h"
+
+#include "absl/strings/str_cat.h"
 #include "tensorflow/core/kernels/eigen_cuboid_convolution.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/core/platform/test_benchmark.h"
@@ -1540,22 +1542,187 @@ static void PackRhsHelper(int iters,
     pack_rhs(packed.data() + packed_offset, sub_mapper, depth, cols);
   }
   tensorflow::testing::StopTiming();
+  tensorflow::testing::SetLabel(
+      absl::StrCat("patch: ", patch_rows, "x", patch_cols, " D", patch_depth,
+                   "; num_patches=", num_patches, " patch_size=", patch_size,
+                   " num_inputs=", num_inputs));
+}
+
+static void PackLhsHelper(int iters,
+                          /* Input dimensions: */
+                          int input_depth,
+                          /* Filter (kernel) dimensions: */
+                          int filter_count, int filter_cols, int filter_rows,
+                          /* Block dimensions: */
+                          Index block_rows, Index block_cols) {
+  // Set random seed for benchmark repeatability.
+  srand(12345);
+
+  eigen_assert(block_rows <= filter_count);
+  eigen_assert(block_cols <= input_depth * filter_rows * filter_cols);
+
+  tensorflow::testing::UseRealTime();
+  tensorflow::testing::StopTiming();
+
+  using Dimensions = Eigen::DSizes<Eigen::Index, 4>;
+
+  // Default Eigen::Tensor layout is column major, so we configure dimensions
+  // starting from the inner most (`filter count` aka `kernel filers`).
+  Dimensions filter_dims(filter_count, filter_rows, filter_cols, input_depth);
+
+  static const int packet_size = Eigen::internal::packet_traits<float>::size;
+
+  // We are going to reshape filter into 2D tensor.
+  using NewDimension = Eigen::DSizes<Index, 2>;
+
+  // Contraction dimensions.
+  using nocontract_t = Eigen::array<Eigen::Index, 1>;
+  using contract_t = Eigen::array<Eigen::Index, 1>;
+
+  // Input to the ReshapeOp. It is the tensorflow TTypes<float>::Tensor
+  // with ColMajor layout, instead of RowMajor. But that doesn't make any
+  // difference, because TensorContraction swaps LHS with RHS for row major
+  // inputs, and contraction mapper always works with column major data.
+  using ArgType = TensorMap<Tensor<float, 4>, Eigen::Aligned>;
+
+  using Evaluator =
+      TensorEvaluator<const TensorReshapingOp<NewDimension, ArgType>,
+                      Eigen::DefaultDevice>;
+
+  using InputMapper = Eigen::internal::TensorContractionInputMapper<
+      float, Index, Eigen::internal::Lhs, Evaluator,  //
+      nocontract_t, contract_t,                       //
+      packet_size,                                    //
+      /*inner_dim_contiguous*/ true,                  //
+      /*inner_dim_reordered*/ false,                  //
+      /*Alignment*/ 0>;
+
+  using SubMapper = Eigen::internal::TensorContractionSubMapper<
+      float, Index, Eigen::internal::Lhs, Evaluator,  //
+      nocontract_t, contract_t,                       //
+      packet_size,                                    //
+      /*inner_dim_contiguous*/ true,                  //
+      /*inner_dim_reordered*/ false,                  //
+      /*Alignment*/ 0>;
+
+#if defined(TENSORFLOW_USE_MKLDNN_CONTRACTION_KERNEL)
+  using PackLhsImpl = Eigen::internal::mkldnn_gemm_pack<float, Eigen::Index,
+                                                        SubMapper, ColMajor>;
+#else
+  using PackLhsImpl =
+      Eigen::internal::gemm_pack_lhs<float, Eigen::Index, SubMapper,      //
+                                     Traits::mr,                          //
+                                     Traits::LhsProgress,                 //
+                                     typename Traits::LhsPacket4Packing,  //
+                                     ColMajor>;
+#endif
+
+  Eigen::DefaultDevice device;
 
-  std::ostringstream stringStream;
-  stringStream << "patch: " << patch_rows << "x" << patch_cols << " D"
-               << patch_depth << "; num_patches=" << num_patches
-               << " patch_size=" << patch_size << " num_inputs=" << num_inputs;
-  tensorflow::testing::SetLabel(stringStream.str());
+  // We will reshape kernel into 2D tensor.
+  NewDimension reshape_dims;
+  reshape_dims[0] = filter_count;
+  reshape_dims[1] = input_depth * filter_rows * filter_cols;
+
+  // We are going to contract along the 'in_depth * filter_rows * filter_cols`.
+  nocontract_t nocontract_dim = {0};
+  contract_t contract_dim = {1};
+
+  // These values computed using the algorithm in TensorContraction.h, with
+  // 'nocontract_dim' and 'contract_dim' values specified above.
+  nocontract_t nocontract_strides = {1};
+  contract_t contract_strides = {filter_count};
+  nocontract_t i_strides = {1};
+  contract_t k_strides = {1};
+
+  // We use tensor of the same dimensions to store packed data.
+  Tensor<float, 4> packed(filter_dims);
+
+  // We generate multiple filter tensors, around 512mb in total size to measure
+  // realistic workload when input data in not in L1-L3 cache.
+  size_t input_bytes = filter_dims.TotalSize() * sizeof(float);
+  size_t mem_size_bytes = 1024 * 1024 * 512;
+  size_t num_filters =
+      std::max(static_cast<size_t>(1), mem_size_bytes / input_bytes);
+
+  std::vector<Tensor<float, 4>> filters;
+  std::vector<Evaluator> evaluators;
+  std::vector<InputMapper> input_mappers;
+
+  for (int i = 0; i < num_filters; ++i) {
+    filters.emplace_back(filter_dims);
+    filters[i].setRandom();
+
+    ArgType tensor_map(filters[i].data(), filter_dims);
+
+    const auto reshape_op =
+        TensorReshapingOp<NewDimension, ArgType>(tensor_map, reshape_dims);
+
+    evaluators.emplace_back(reshape_op, device);
+
+    input_mappers.emplace_back(evaluators[i], nocontract_strides, i_strides,
+                               contract_strides, k_strides);
+  }
+
+  PackLhsImpl pack_lhs;
+
+  const Index packed_total_size = filter_dims.TotalSize();
+
+  // Round up row/col/memory offsets to make them multiple of packet size.
+  const auto round_up = [](const Index idx) {
+    return (idx / packet_size) * packet_size;
+  };
+
+  // Block rows is in the [0, filter_count) range.
+  // Block cols is in the [0, filter_rows * filter_cols * input_depth) range.
+
+  const Index max_row = filter_count;
+  const Index max_col = filter_rows * filter_cols * input_depth;
+
+  tensorflow::testing::StartTiming();
+  for (int i = 0; i < iters; ++i) {
+    int filter_idx =
+        num_filters == 1 ? 1 : internal::random<int>(0, num_filters - 1);
+
+    Index row_offset = round_up(internal::random<Index>(0, max_row - 10));
+    Index col_offset = round_up(internal::random<Index>(0, max_col - 10));
+
+    Index rows = std::min(block_rows, max_row - row_offset);
+    Index cols = std::min(block_cols, max_col - col_offset);
+
+    // Write packed data to random memory location to emulate cold caches.
+    Index packed_offset = round_up(
+        internal::random<Index>(0, packed_total_size - rows * cols - 1));
+
+    SubMapper sub_mapper =
+        input_mappers[filter_idx].getSubMapper(row_offset, col_offset);
+
+    // NOTE: Eigen gemm_pack_lhs accepts contraction depth (k-th dimension) as a
+    // first argument (aka block cols). MKL-DNN pack is generic for lhs and rhs
+    // and accepts block rows and cols in the same order for lhs and rhs.
+#if defined(TENSORFLOW_USE_MKLDNN_CONTRACTION_KERNEL)
+    pack_lhs(packed.data() + packed_offset, sub_mapper, rows, cols);
+#else
+    pack_lhs(packed.data() + packed_offset, sub_mapper, cols, rows);
+#endif
+  }
+  tensorflow::testing::StopTiming();
+  tensorflow::testing::SetLabel(absl::StrCat(
+      "filter: count=", filter_count, " dims=", filter_rows, "x", filter_cols,
+      "; input: depth=", input_depth, "; num_filers=", num_filters));
 }
 
 // -------------------------------------------------------------------------- //
-// Macro argumentnames:
+// Pack RHS
+//
+// Macro argument names:
 //    N: batch size
 //    H: height
 //    W: width
 //    C: input channels
 //   FC: filter channles
 //   FH: filter height
+//   FW: filter width
 //   SH: stride in height dimensions
 //   SW: stride in width dimensions
 //   BR: block rows
@@ -1563,16 +1730,16 @@ static void PackRhsHelper(int iters,
 
 #define BM_CONCAT(a, b) a##b
 
-#define BM_NAME(prefix, N, H, W, C, FC, FH, FW, SH, SW, BR, BC)           \
+#define BM_RHS_NAME(prefix, N, H, W, C, FC, FH, FW, SH, SW, BR, BC)       \
   BM_CONCAT(BM_##prefix##_##N##_##H##x##W##_IC##C##_FC##FC##_##FH##x##FW, \
             _s##SH##x##SW##_B##BR##x##BC)
 
-#define BM_PackRhs(N, H, W, C, FC, FH, FW, SH, SW, BR, BC)         \
-  static void BM_NAME(PackRhs, N, H, W, C, FC, FH, FW, SH, SW, BR, \
-                      BC)(int iters) {                             \
-    PackRhsHelper(iters, N, H, W, C, FC, FH, FW, SH, SW, BR, BC);  \
-  }                                                                \
-  BENCHMARK(BM_NAME(PackRhs, N, H, W, C, FC, FH, FW, SH, SW, BR, BC))
+#define BM_PackRhs(N, H, W, C, FC, FH, FW, SH, SW, BR, BC)             \
+  static void BM_RHS_NAME(PackRhs, N, H, W, C, FC, FH, FW, SH, SW, BR, \
+                          BC)(int iters) {                             \
+    PackRhsHelper(iters, N, H, W, C, FC, FH, FW, SH, SW, BR, BC);      \
+  }                                                                    \
+  BENCHMARK(BM_RHS_NAME(PackRhs, N, H, W, C, FC, FH, FW, SH, SW, BR, BC))
 
 // Number of input channel (input depth) it equal to the number of patch
 // channels (patch depth).
@@ -1645,4 +1812,37 @@ BM_PackRhs(/*batch*/ 32,        //
            /*filter*/ 3, 3,     //
            /*stride*/ 2, 2,     //
            /*block*/ 36, 432);
+
+// -------------------------------------------------------------------------- //
+// Pack LHS
+//
+// Macro argument names:
+//    C: input channels
+//   FC: filter channels
+//   FH: filter height
+//   FW: filter width
+//   BR: block rows
+//   BC: block cols
+
+#define BM_LHS_NAME(prefix, C, FC, FH, FW, BR, BC) \
+  BM_CONCAT(BM_##prefix##_##C##_FC##FC##_##FH##x##FW, _B##BR##x##BC)
+
+#define BM_PackLhs(C, FC, FH, FW, BR, BC)                              \
+  static void BM_LHS_NAME(PackLhs, C, FC, FH, FW, BR, BC)(int iters) { \
+    PackLhsHelper(iters, C, FC, FH, FW, BR, BC);                       \
+  }                                                                    \
+  BENCHMARK(BM_LHS_NAME(PackLhs, C, FC, FH, FW, BR, BC))
+
+// Number of input channel (input depth) it equal to the number of patch
+// channels (patch depth).
+
+BM_PackLhs(/*input channels*/ 128,    //
+           /*filter channels*/ 1024,  //
+           /*filter dims*/ 3, 3,      //
+           /*block*/ 256, 56);
+
+BM_PackLhs(/*input channels*/ 128,    //
+           /*filter channels*/ 1024,  //
+           /*filter dims*/ 3, 3,      //
+           /*block*/ 56, 256);
 }  // namespace Eigen
-- 
GitLab


From a253b9eab582307315567ba36d2b25033ad29b8b Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Mon, 7 Jan 2019 16:54:22 -0800
Subject: [PATCH 0342/1095] Change tf_upgrade_v2.py script to wrap labels
 argument with tf.stop_gradients in tf.softmax_cross_entropy_with_logits call.

PiperOrigin-RevId: 228260462
---
 tensorflow/tools/compatibility/BUILD          |  1 +
 tensorflow/tools/compatibility/reorders_v2.py |  1 +
 .../compatibility/testdata/test_file_v1_12.py |  9 ++++
 .../tools/compatibility/tf_upgrade_v2.py      | 45 +++++++++++++++++--
 .../tools/compatibility/tf_upgrade_v2_test.py | 28 +++++++-----
 5 files changed, 69 insertions(+), 15 deletions(-)

diff --git a/tensorflow/tools/compatibility/BUILD b/tensorflow/tools/compatibility/BUILD
index 452e6db941..31dbc02963 100644
--- a/tensorflow/tools/compatibility/BUILD
+++ b/tensorflow/tools/compatibility/BUILD
@@ -69,6 +69,7 @@ py_library(
         ":ast_edits",
         ":renames_v2",
         ":reorders_v2",
+        "@six_archive//:six",
     ],
 )
 
diff --git a/tensorflow/tools/compatibility/reorders_v2.py b/tensorflow/tools/compatibility/reorders_v2.py
index 3f05aea6ca..5c11388516 100644
--- a/tensorflow/tools/compatibility/reorders_v2.py
+++ b/tensorflow/tools/compatibility/reorders_v2.py
@@ -65,6 +65,7 @@ reorders = {
     'tf.nn.moments': ['x', 'axes', 'shift', 'name', 'keep_dims'],
     'tf.nn.pool': ['input', 'window_shape', 'pooling_type', 'padding', 'dilation_rate', 'strides', 'name', 'data_format'],
     'tf.nn.separable_conv2d': ['input', 'depthwise_filter', 'pointwise_filter', 'strides', 'padding', 'rate', 'name', 'data_format'],
+    'tf.nn.softmax_cross_entropy_with_logits': ['_sentinel', 'labels', 'logits', 'dim', 'name'],
     'tf.nn.space_to_batch': ['input', 'paddings', 'block_size', 'name'],
     'tf.nn.space_to_depth': ['input', 'block_size', 'name', 'data_format'],
     'tf.nn.weighted_moments': ['x', 'axes', 'frequency_weights', 'name', 'keep_dims'],
diff --git a/tensorflow/tools/compatibility/testdata/test_file_v1_12.py b/tensorflow/tools/compatibility/testdata/test_file_v1_12.py
index 5ce4dd49ad..2663762aa7 100644
--- a/tensorflow/tools/compatibility/testdata/test_file_v1_12.py
+++ b/tensorflow/tools/compatibility/testdata/test_file_v1_12.py
@@ -70,6 +70,15 @@ class TestUpgrade(test_util.TensorFlowTestCase):
         [0],
         tf.argmin([[1, 3, 2]], name='abc', dimension=1))
 
+  @test_util.run_v1_only("b/120545219")
+  def testSoftmaxCrossEntropyWithLogits(self):
+    out = tf.nn.softmax_cross_entropy_with_logits(
+        logits=[0.1, 0.8], labels=[0, 1])
+    self.assertAllClose(out, 0.40318608)
+    out = tf.nn.softmax_cross_entropy_with_logits_v2(
+        logits=[0.1, 0.8], labels=[0, 1])
+    self.assertAllClose(out, 0.40318608)
+
 
 if __name__ == "__main__":
   test_lib.main()
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 9d9c5878f7..2dbbe27598 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import ast
 
 import pasta
+import six
 
 from tensorflow.tools.compatibility import ast_edits
 from tensorflow.tools.compatibility import renames_v2
@@ -94,6 +95,10 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.convert_to_tensor": {
             "preferred_dtype": "dtype_hint"
         },
+        "tf.nn.softmax_cross_entropy_with_logits": {
+            "dim": "axis",
+            "_sentinel": None,
+        },
         "tf.nn.softmax_cross_entropy_with_logits_v2": {
             "dim": "axis"
         },
@@ -682,6 +687,10 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.norm",
         "tf.reverse_sequence",
         "tf.sparse_split",
+        # tf.nn.softmax_cross_entropy_with_logits *must* be called with
+        # keyword arguments. Add keyword arguments in rare case when they
+        # are not specified.
+        "tf.nn.softmax_cross_entropy_with_logits",
     }
 
     # Functions that were reordered should be changed to the new keyword args
@@ -714,6 +723,8 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.to_float": self._cast_transformer,
         "tf.to_int32": self._cast_transformer,
         "tf.to_int64": self._cast_transformer,
+        "tf.nn.softmax_cross_entropy_with_logits":
+            self._softmax_cross_entropy_with_logits_transformer,
     }
 
     decay_function_comment = (
@@ -950,10 +961,6 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "'deterministic' arguments. Now it takes a single 'seed' arg. If "
             "'seed' is zero, the execution is random and deterministic "
             "otherwise",
-        "tf.nn.softmax_cross_entropy_with_logits":
-            "tf.nn.softmax_cross_entropy_with_logits behavior has changed. "
-            "'labels' needs to be wrapped with tf.stop_gradient to keep the "
-            "old behavior. Also, 'dim' argument has been renamed to 'axis'.",
         "tf.test.assert_equal_graph_def":
             "tf.assert_equal_graph_def no longer takes 'checkpoint_v2' "
             "argument. 'checkpoint_v2' now defaults to True.",
@@ -1228,6 +1235,36 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
                                                                     dtype_str)))
     return node
 
+  @staticmethod
+  def _softmax_cross_entropy_with_logits_transformer(
+      parent, node, full_name, name, logs, errors):
+    def _wrap_label(parent, old_value):
+      """Wrap labels with tf.stop_gradient."""
+      if six.PY3:
+        new_value = ast.Call(
+            ast.Name(id="tf.stop_gradient", ctx=ast.Load()),
+            [old_value], [])
+      else:
+        new_value = ast.Call(
+            ast.Name(id="tf.stop_gradient", ctx=ast.Load()),
+            [old_value], [], None, None)
+
+      # This copies the prefix and suffix on old_value to new_value.
+      pasta.ast_utils.replace_child(parent, old_value, new_value)
+      ast.copy_location(new_value, old_value)
+
+    # Check if we have a labels keyword arg
+    for karg in node.keywords:
+      if karg.arg == "labels":
+        logs.append((node.lineno, node.col_offset,
+                     "Changing labels arg of "
+                     "tf.nn.softmax_cross_entropy_with_logits to "
+                     "tf.stop_gradient(labels). Please check this "
+                     "transformation.\n"))
+        _wrap_label(karg, karg.value)
+        return node
+    return node
+
   @staticmethod
   def _batch_gather_transformer(parent, node, full_name, name, logs, errors):
     # Check if the call already has a batch_dims argument
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index d8bc0ba75a..80d86d7a2b 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -684,26 +684,32 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
     self.assertEqual(new_text, expected_text)
 
   def testSoftMaxCrossEntropyWithLogitsV2(self):
-    text = "tf.nn.softmax_cross_entropy_with_logits_v2(labels, logits, dim=2)"
+    text = (
+        "tf.nn.softmax_cross_entropy_with_logits_v2("
+        "labels=labels, logits=logits, dim=2)")
     expected_text = (
-        "tf.nn.softmax_cross_entropy_with_logits(labels, logits, axis=2)")
+        "tf.nn.softmax_cross_entropy_with_logits("
+        "labels=labels, logits=logits, axis=2)")
     _, unused_report, errors, new_text = self._upgrade(text)
     self.assertEqual(new_text, expected_text)
 
     self.assertFalse(errors)
 
   def testSoftMaxCrossEntropyWithLogits(self):
-    text = "tf.nn.softmax_cross_entropy_with_logits(labels, logits, dim=2)"
+    text = ("tf.nn.softmax_cross_entropy_with_logits("
+            "labels=labels, logits=logits, dim=2)")
     expected_text = (
-        "tf.nn.softmax_cross_entropy_with_logits(labels, logits, dim=2)")
-    _, report, errors, new_text = self._upgrade(text)
+        "tf.nn.softmax_cross_entropy_with_logits("
+        "labels=tf.stop_gradient(labels), logits=logits, axis=2)")
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
     self.assertEqual(new_text, expected_text)
-    self.assertIn(
-        "tf.nn.softmax_cross_entropy_with_logits requires manual check",
-        errors[0])
-    self.assertIn(
-        "tf.nn.softmax_cross_entropy_with_logits behavior has changed. ",
-        report)
+
+    text = ("tf.nn.softmax_cross_entropy_with_logits("
+            "labels=foo(bar))")
+    expected_text = ("tf.nn.softmax_cross_entropy_with_logits("
+                     "labels=tf.stop_gradient(foo(bar)))")
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(expected_text, new_text)
 
   def testSparseMatmul(self):
     text = ("tf.sparse_matmul(a, b, c, d, e, f, g)\n")
-- 
GitLab


From 09713e439363d763ca7c12d0c279b8d55d5b6053 Mon Sep 17 00:00:00 2001
From: Davide Libenzi <dlibenzi@google.com>
Date: Mon, 7 Jan 2019 17:10:18 -0800
Subject: [PATCH 0343/1095] We should be using on host shape as the device one
 can have tuples in place of complex or S64 types.

PiperOrigin-RevId: 228262394
---
 tensorflow/compiler/xrt/tests/raw_api_test.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/compiler/xrt/tests/raw_api_test.cc b/tensorflow/compiler/xrt/tests/raw_api_test.cc
index c8479cb778..be0c4b9392 100644
--- a/tensorflow/compiler/xrt/tests/raw_api_test.cc
+++ b/tensorflow/compiler/xrt/tests/raw_api_test.cc
@@ -956,6 +956,7 @@ TEST(RawApiTest, CompileAndExecuteWithS64Argument) {
   xrt::XRTExecutionConfig e;
   e.set_release_input_handles(true);
   e.set_release_compilation_handle(true);
+  e.set_return_exploded_tuple(true);
 
   Scope root = Scope::NewRootScope().WithDevice(DeviceFromFlag());
   auto e_config =
-- 
GitLab


From fbbecc81be3b1eab0c15020ce3b0566e60e25c7f Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Mon, 7 Jan 2019 17:48:53 -0800
Subject: [PATCH 0344/1095] Dynamic Padder

- Change HloGetDimensionSizeRewriter to use DynamicDimensionInference,
  when trying to get the size of a dynamic dimension, GetDimensionSize
  is replaced by the hlo instruction the represents the dynamic size
  instead.

- Implements Dynamic Padder, which uses DynamicDimensionInference to
  analyze the graph and inserts certain operations to make sure the
  result is the same as if there is no padding in the bounded shapes.

PiperOrigin-RevId: 228266743
---
 tensorflow/compiler/xla/service/BUILD         |  41 +++++
 .../compiler/xla/service/dynamic_padder.cc    | 161 ++++++++++++++++++
 .../compiler/xla/service/dynamic_padder.h     |  44 +++++
 .../xla/service/dynamic_padder_test.cc        | 152 +++++++++++++++++
 .../hlo_get_dimension_size_rewriter.cc        |  26 ++-
 .../service/hlo_get_dimension_size_rewriter.h |   4 +-
 6 files changed, 421 insertions(+), 7 deletions(-)
 create mode 100644 tensorflow/compiler/xla/service/dynamic_padder.cc
 create mode 100644 tensorflow/compiler/xla/service/dynamic_padder.h
 create mode 100644 tensorflow/compiler/xla/service/dynamic_padder_test.cc

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index 9bc6218f75..ec63ae1ac6 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -1938,6 +1938,46 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "dynamic_padder",
+    srcs = ["dynamic_padder.cc"],
+    hdrs = ["dynamic_padder.h"],
+    deps = [
+        ":dynamic_dimension_inference",
+        ":hlo_dce",
+        ":hlo_pass",
+        "//tensorflow/compiler/xla:literal",
+        "//tensorflow/compiler/xla:literal_util",
+        "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla:status_macros",
+        "//tensorflow/compiler/xla:util",
+        "//tensorflow/core:lib",
+        "@com_google_absl//absl/algorithm:container",
+        "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/container:flat_hash_set",
+    ],
+)
+
+tf_cc_test(
+    name = "dynamic_padder_test",
+    srcs = ["dynamic_padder_test.cc"],
+    deps = [
+        ":dynamic_padder",
+        "//tensorflow/compiler/xla:debug_options_flags",
+        "//tensorflow/compiler/xla:literal",
+        "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla:test",
+        "//tensorflow/compiler/xla:test_helpers",
+        "//tensorflow/compiler/xla:xla_data_proto",
+        "//tensorflow/compiler/xla/client:xla_builder",
+        "//tensorflow/compiler/xla/service:hlo",
+        "//tensorflow/compiler/xla/service:hlo_matchers",
+        "//tensorflow/compiler/xla/service:hlo_runner",
+        "//tensorflow/compiler/xla/tests:hlo_test_base",
+        "//tensorflow/core:test",
+    ],
+)
+
 tf_cc_test(
     name = "dynamic_dimension_inference_test",
     srcs = ["dynamic_dimension_inference_test.cc"],
@@ -2981,6 +3021,7 @@ cc_library(
     srcs = ["hlo_get_dimension_size_rewriter.cc"],
     hdrs = ["hlo_get_dimension_size_rewriter.h"],
     deps = [
+        ":dynamic_dimension_inference",
         ":hlo",
         ":hlo_pass",
         ":shape_inference",
diff --git a/tensorflow/compiler/xla/service/dynamic_padder.cc b/tensorflow/compiler/xla/service/dynamic_padder.cc
new file mode 100644
index 0000000000..4db280f817
--- /dev/null
+++ b/tensorflow/compiler/xla/service/dynamic_padder.cc
@@ -0,0 +1,161 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/compiler/xla/service/dynamic_padder.h"
+
+#include <algorithm>
+#include <vector>
+
+#include "absl/algorithm/container.h"
+#include "absl/container/flat_hash_map.h"
+
+#include "absl/container/flat_hash_set.h"
+#include "tensorflow/compiler/xla/literal.h"
+#include "tensorflow/compiler/xla/literal_util.h"
+#include "tensorflow/compiler/xla/service/dynamic_dimension_inference.h"
+#include "tensorflow/compiler/xla/service/hlo_dce.h"
+#include "tensorflow/compiler/xla/shape_util.h"
+#include "tensorflow/compiler/xla/status_macros.h"
+#include "tensorflow/compiler/xla/util.h"
+
+#include "tensorflow/core/lib/core/errors.h"
+
+namespace xla {
+
+namespace {
+
+// ChooseIdentityValue looks at the instruction and returns a identity value
+// which, when padded, doesn't change the result of the instruction.
+//
+// nullopt is returned if padding doesn't need to be reset.
+StatusOr<HloInstruction*> ChooseIdentityValue(HloInstruction* inst) {
+  HloComputation* comp = inst->parent();
+  // Padding on elementwise operation doesn't affect the result of the effective
+  // data.
+  if (inst->IsElementwise()) {
+    return nullptr;
+  }
+
+  switch (inst->opcode()) {
+    case HloOpcode::kReduce:
+    case HloOpcode::kReduceWindow: {
+      // Because of the way we do reduce, we already require the `init` operand
+      // of hlo reduce instruction to be identity value. Here we reuse the
+      // operand.
+      return inst->mutable_operand(1);
+    }
+
+    case HloOpcode::kConvolution:
+    case HloOpcode::kDot: {
+      // Use 0 as padding value for convolution and dot.
+      PrimitiveType ptype = inst->shape().element_type();
+      return comp->AddInstruction(
+          HloInstruction::CreateConstant(LiteralUtil::Zero(ptype)));
+    }
+
+    case HloOpcode::kPad: {
+      return inst->mutable_operand(1);
+    }
+    case HloOpcode::kParameter:
+    case HloOpcode::kGetDimensionSize:
+    case HloOpcode::kReshape:
+    case HloOpcode::kTuple:
+    case HloOpcode::kAllReduce:
+    case HloOpcode::kBroadcast:
+      return nullptr;
+    default:
+      return UnimplementedStrCat("Unimplimented padding for instruction: ",
+                                 inst->ToString());
+  }
+}
+
+}  // namespace
+
+StatusOr<bool> DynamicPadder::Run(HloModule* module) {
+  bool changed = false;
+  VLOG(2) << "Pre DynamicPadder HLO:";
+  XLA_VLOG_LINES(2, module->ToString());
+  TF_ASSIGN_OR_RETURN(DynamicDimensionInference dynamic_dimension_inference,
+                      DynamicDimensionInference::Run(module));
+
+  for (HloComputation* computation : module->computations()) {
+    for (HloInstruction* inst : computation->instructions()) {
+      for (int64 operand_num = 0; operand_num < inst->operand_count();
+           ++operand_num) {
+        HloInstruction* operand = inst->mutable_operand(operand_num);
+        if (!operand->shape().IsArray()) {
+          continue;
+        }
+        for (int64 dim = 0; dim < operand->shape().rank(); ++dim) {
+          HloInstruction* dynamic_size =
+              dynamic_dimension_inference.GetDynamicSize(operand, {}, dim);
+          if (dynamic_size == nullptr) {
+            continue;
+          }
+          VLOG(1) << "Has dynamic dimension of operand" << operand_num << " @"
+                  << dim;
+          TF_ASSIGN_OR_RETURN(HloInstruction * identity_value,
+                              ChooseIdentityValue(inst));
+          if (identity_value == nullptr) {
+            continue;
+          }
+
+          // For each dimension, first generates a mask representing the
+          // effective area of data and padded area of data using iota and
+          // dynamic_size. For example, given a dimension of 7 elements and 5
+          // effective elements:
+          //
+          // iota = [0, 1, 2, 3, 4, 5, 6]
+          // broadcast_dynamic_size = [5, 5, 5, 5, 5, 5, 5]
+          // mask = lt(iota, broadcast_dynamic_size) = [t, t, t, t, t, f, f]
+          //
+          // Once the mask is generated, the input data is then padded using the
+          // mask and pad value.
+          //
+          const Shape mask_shape =
+              ShapeUtil::ChangeElementType(operand->shape(), xla::U32);
+          const Shape pred_shape =
+              ShapeUtil::ChangeElementType(operand->shape(), xla::PRED);
+          HloInstruction* iota = computation->AddInstruction(
+              HloInstruction::CreateIota(mask_shape, dim));
+
+          HloInstruction* broadcasted_effective_size =
+              computation->AddInstruction(HloInstruction::CreateBroadcast(
+                  mask_shape, dynamic_size, {}));
+          HloInstruction* pred = computation->AddInstruction(
+              HloInstruction::CreateBinary(pred_shape, HloOpcode::kLt, iota,
+                                           broadcasted_effective_size));
+
+          HloInstruction* broadcasted_identity_value =
+              computation->AddInstruction(HloInstruction::CreateBroadcast(
+                  operand->shape(), identity_value, {}));
+          HloInstruction* padded =
+              computation->AddInstruction(HloInstruction::CreateTernary(
+                  operand->shape(), HloOpcode::kSelect, pred, operand,
+                  broadcasted_identity_value));
+          TF_RETURN_IF_ERROR(inst->ReplaceOperandWith(operand_num, padded));
+          operand = inst->mutable_operand(operand_num);
+          changed = true;
+        }
+      }
+    }
+  }
+  HloDCE dce;
+  TF_ASSIGN_OR_RETURN(changed, dce.Run(module));
+  VLOG(2) << "Post DynamicPadder HLO:";
+  XLA_VLOG_LINES(2, module->ToString());
+  return changed;
+}
+
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/dynamic_padder.h b/tensorflow/compiler/xla/service/dynamic_padder.h
new file mode 100644
index 0000000000..509269f7f5
--- /dev/null
+++ b/tensorflow/compiler/xla/service/dynamic_padder.h
@@ -0,0 +1,44 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_DYNAMIC_PADDER_H_
+#define TENSORFLOW_COMPILER_XLA_SERVICE_DYNAMIC_PADDER_H_
+
+#include "tensorflow/compiler/xla/service/dynamic_dimension_inference.h"
+#include "tensorflow/compiler/xla/service/hlo_pass_interface.h"
+
+namespace xla {
+
+// With bounded shapes, only part of the shape contains effective data and the
+// rest contains padded data, whose value can be anything depending on the
+// source of the data. When a bounded shape is directly consumed by an
+// instruction that collapses dimensions (reduce for example), the padding data
+// would affect result of the instruction.
+//
+// DynamicPadder uses DynamicDimensionInference to detect bounded shapes in a
+// hlo module, it then inserts certain instructions to reset the padding into an
+// identity value so that in doesn't affect the result of subsequent
+// instruction. For example, it'd reset the padding to 0 before a bounded shape
+// is consumed by a reduce-sum.
+class DynamicPadder : public HloModulePass {
+ public:
+  absl::string_view name() const override { return "dynamic_padder"; }
+
+  StatusOr<bool> Run(HloModule* module) override;
+};
+
+}  // namespace xla
+
+#endif  // TENSORFLOW_COMPILER_XLA_SERVICE_DYNAMIC_PADDER_H_
diff --git a/tensorflow/compiler/xla/service/dynamic_padder_test.cc b/tensorflow/compiler/xla/service/dynamic_padder_test.cc
new file mode 100644
index 0000000000..55a11286e4
--- /dev/null
+++ b/tensorflow/compiler/xla/service/dynamic_padder_test.cc
@@ -0,0 +1,152 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/dynamic_padder.h"
+
+#include "tensorflow/compiler/xla/client/xla_builder.h"
+#include "tensorflow/compiler/xla/literal.h"
+#include "tensorflow/compiler/xla/service/hlo_computation.h"
+#include "tensorflow/compiler/xla/service/hlo_instruction.h"
+#include "tensorflow/compiler/xla/service/hlo_matchers.h"
+#include "tensorflow/compiler/xla/service/hlo_module.h"
+#include "tensorflow/compiler/xla/service/hlo_opcode.h"
+#include "tensorflow/compiler/xla/service/hlo_runner.h"
+#include "tensorflow/compiler/xla/shape_util.h"
+#include "tensorflow/compiler/xla/test.h"
+#include "tensorflow/compiler/xla/test_helpers.h"
+#include "tensorflow/compiler/xla/tests/hlo_test_base.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/platform/test_benchmark.h"
+
+namespace op = xla::testing::opcode_matchers;
+
+namespace xla {
+namespace {
+
+class DynamicPadderTest : public HloTestBase {
+ protected:
+  DynamicPadderTest() : HloTestBase() { module_ = CreateNewVerifiedModule(); }
+
+  StatusOr<bool> RunPadder() {
+    hlo_graph_dumper::MaybeDumpHloModule(*module_, "Before padder");
+
+    DynamicPadder padder;
+
+    return padder.Run(module_.get());
+  }
+
+  void ExpectPadded(const HloInstruction* inst) {
+    EXPECT_THAT(inst,
+                op::Select(op::Lt(op::Iota(), op::Broadcast(op::Parameter())),
+                           ::testing::_, op::Broadcast()));
+  }
+
+  HloComputation* GetScalarAddComputation() {
+    auto embedded_builder = HloComputation::Builder("add");
+    auto lhs = embedded_builder.AddInstruction(HloInstruction::CreateParameter(
+        0, ShapeUtil::MakeShape(F32, {}), "lhs"));
+    auto rhs = embedded_builder.AddInstruction(HloInstruction::CreateParameter(
+        1, ShapeUtil::MakeShape(F32, {}), "rhs"));
+    embedded_builder.AddInstruction(
+        HloInstruction::CreateBinary(lhs->shape(), HloOpcode::kAdd, lhs, rhs));
+    return module_->AddEmbeddedComputation(embedded_builder.Build());
+  }
+
+  std::unique_ptr<HloModule> module_;
+  const Shape scalar_shape_ = ShapeUtil::MakeShape(U32, {});
+};
+
+TEST_F(DynamicPadderTest, ReduceTest) {
+  auto builder = HloComputation::Builder(TestName());
+  auto input_shape = ShapeUtil::MakeShape(F32, {1, 2, 2});
+  auto reduce_shape = ShapeUtil::MakeShape(F32, {2});
+
+  auto data_param = builder.AddInstruction(
+      HloInstruction::CreateParameter(0, input_shape, "data_param"));
+  builder.AddInstruction(
+      HloInstruction::CreateParameter(1, scalar_shape_, "size_param"));
+
+  auto negate = builder.AddInstruction(
+      HloInstruction::CreateUnary(input_shape, HloOpcode::kNegate, data_param));
+
+  auto init = builder.AddInstruction(
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0<float>(0.0)));
+
+  auto reduce = builder.AddInstruction(HloInstruction::CreateReduce(
+      reduce_shape, negate, init, {0, 2}, GetScalarAddComputation()));
+
+  module_->AddEntryComputation(builder.Build());
+
+  // Set up dynamic parameter binding.
+  TF_CHECK_OK(module_->dynamic_parameter_binding().Bind(
+      DynamicParameterBinding::DynamicParameter{1, {}},
+      DynamicParameterBinding::DynamicDimension{0, {}, 1}));
+
+  TF_ASSERT_OK(RunPadder().status());
+
+  ExpectPadded(reduce->operand(0));
+}
+
+TEST_F(DynamicPadderTest, ConvolutionTest) {
+  auto builder = HloComputation::Builder(TestName());
+  constexpr int xdim = 3;
+  constexpr int ydim = 2;
+  constexpr int zdim = 1;
+  auto xy_shape = ShapeUtil::MakeShape(F32, {xdim, ydim});
+  auto yz_shape = ShapeUtil::MakeShape(F32, {ydim, zdim});
+  auto zx_shape = ShapeUtil::MakeShape(F32, {zdim, xdim});
+
+  auto* a_param = builder.AddInstruction(HloInstruction::CreateParameter(
+      /*parameter_number=*/0, xy_shape, "A"));
+  auto* b_param = builder.AddInstruction(HloInstruction::CreateParameter(
+      /*parameter_number=*/1, yz_shape, "B"));
+  builder.AddInstruction(HloInstruction::CreateParameter(
+      /*parameter_number=*/2, scalar_shape_, "size_param"));
+
+  auto dnums = XlaBuilder::CreateDefaultConvDimensionNumbers(0);
+
+  dnums.set_kernel_input_feature_dimension(0);
+  dnums.set_kernel_output_feature_dimension(1);
+  dnums.set_input_batch_dimension(0);
+  dnums.set_output_batch_dimension(1);
+  dnums.set_output_feature_dimension(0);
+
+  Window window;
+
+  auto* conv = builder.AddInstruction(HloInstruction::CreateConvolve(
+      zx_shape, a_param, b_param, /*feature_group_count=*/1,
+      /*batch_group_count=*/1, window, dnums,
+      HloTestBase::DefaultPrecisionConfig(2)));
+
+  module_->AddEntryComputation(builder.Build());
+
+  // Set up dynamic parameter binding for non-contracting dimension.
+  TF_CHECK_OK(module_->dynamic_parameter_binding().Bind(
+      DynamicParameterBinding::DynamicParameter{2, {}},
+      DynamicParameterBinding::DynamicDimension{0, {}, 0}));
+
+  // Set up binding for contracting dimensions.
+  TF_CHECK_OK(module_->dynamic_parameter_binding().Bind(
+      DynamicParameterBinding::DynamicParameter{2, {}},
+      DynamicParameterBinding::DynamicDimension{0, {}, 1}));
+
+  TF_ASSERT_OK(RunPadder().status());
+
+  ExpectPadded(conv->operand(0));
+}
+
+}  // namespace
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter.cc b/tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter.cc
index c919dbd82d..862b202971 100644
--- a/tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter.cc
+++ b/tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter.cc
@@ -17,6 +17,7 @@ limitations under the License.
 
 #include "absl/algorithm/container.h"
 #include "tensorflow/compiler/xla/literal_util.h"
+#include "tensorflow/compiler/xla/service/dynamic_dimension_inference.h"
 #include "tensorflow/compiler/xla/service/hlo_instruction.h"
 #include "tensorflow/compiler/xla/service/hlo_opcode.h"
 #include "tensorflow/compiler/xla/service/shape_inference.h"
@@ -25,7 +26,9 @@ namespace xla {
 
 namespace {
 
-StatusOr<bool> ReplaceGetSize(HloInstruction* instr) {
+StatusOr<bool> ReplaceGetSize(
+    HloInstruction* instr,
+    const DynamicDimensionInference* dynamic_dimension_inference) {
   if (instr->opcode() != HloOpcode::kGetDimensionSize) {
     return false;
   }
@@ -36,10 +39,18 @@ StatusOr<bool> ReplaceGetSize(HloInstruction* instr) {
                           instr->operand(0)->shape(), instr->dimension()));
   TF_RET_CHECK(ShapeUtil::Equal(instr->shape(), legal_shape));
   TF_RET_CHECK(ShapeUtil::HasPrimitiveType(instr->shape(), U32));
-  uint32 size = instr->operand(0)->shape().dimensions(instr->dimension());
-  HloInstruction* new_instr = computation->AddInstruction(
-      HloInstruction::CreateConstant(LiteralUtil::CreateR0<uint32>(size)));
-  TF_RETURN_IF_ERROR(instr->ReplaceAllUsesWith(new_instr));
+  HloInstruction* operand = instr->mutable_operand(0);
+  int64 dim = instr->dimension();
+  HloInstruction* dynamic_size =
+      dynamic_dimension_inference->GetDynamicSize(operand, {}, dim);
+  if (dynamic_size != nullptr) {
+    TF_RETURN_IF_ERROR(instr->ReplaceAllUsesWith(dynamic_size));
+  } else {
+    uint32 size = instr->operand(0)->shape().dimensions(dim);
+    HloInstruction* new_instr = computation->AddInstruction(
+        HloInstruction::CreateConstant(LiteralUtil::CreateR0<uint32>(size)));
+    TF_RETURN_IF_ERROR(instr->ReplaceAllUsesWith(new_instr));
+  }
   return true;
 }
 
@@ -48,10 +59,13 @@ StatusOr<bool> ReplaceGetSize(HloInstruction* instr) {
 StatusOr<bool> HloGetDimensionSizeRewriter::Run(HloModule* module) {
   bool changed = false;
   HloProto proto;
+  TF_ASSIGN_OR_RETURN(DynamicDimensionInference inference,
+                      DynamicDimensionInference::Run(module));
   *proto.mutable_hlo_module() = module->ToProto();
   for (auto* computation : module->computations()) {
     for (auto instruction : computation->instructions()) {
-      TF_ASSIGN_OR_RETURN(bool replaced, ReplaceGetSize(instruction));
+      TF_ASSIGN_OR_RETURN(bool replaced,
+                          ReplaceGetSize(instruction, &inference));
       changed = changed || replaced;
     }
   }
diff --git a/tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter.h b/tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter.h
index 30f44c23a8..9aa79fe66b 100644
--- a/tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter.h
+++ b/tensorflow/compiler/xla/service/hlo_get_dimension_size_rewriter.h
@@ -21,7 +21,9 @@ limitations under the License.
 
 namespace xla {
 
-// Pass to replace a kGetDimensionSize instruction with a constant instruction.
+// Pass to replace a kGetDimensionSize instruction with a hlo instruction
+// representing the dynamic size if the dimension is dynamic, otherwise a
+// constant instruction representing the static size.
 class HloGetDimensionSizeRewriter : public HloModulePass {
  public:
   absl::string_view name() const override {
-- 
GitLab


From 5ed2c498eddc0b95ae43d6f4c39e29011cd97694 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Mon, 7 Jan 2019 18:49:39 -0800
Subject: [PATCH 0345/1095] [XLA] Add testcase for ptxas bug.

This ptxas bug caused the wrong output on this testcase.

We never received a clear explanation of what was going on from nvidia, so I
can't say more than "it used to not work, now it does work."

PiperOrigin-RevId: 228271902
---
 tensorflow/compiler/xla/tests/BUILD           | 15 ++++
 .../compiler/xla/tests/ptxas_bug_120501638.cc | 82 +++++++++++++++++++
 2 files changed, 97 insertions(+)
 create mode 100644 tensorflow/compiler/xla/tests/ptxas_bug_120501638.cc

diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index 9433264917..106b3fd6c9 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -2118,3 +2118,18 @@ tf_cc_test(
         "@com_google_absl//absl/synchronization",
     ],
 )
+
+xla_test(
+    name = "ptxas_bug_120501638",
+    srcs = ["ptxas_bug_120501638.cc"],
+    tags = [
+        # Disabled in OSS until nvidia publicly releases a fixed ptxas.
+        "no_oss",
+    ],
+    deps = [
+        ":hlo_test_base",
+        ":xla_internal_test_main",  # fixdeps: keep
+        "//tensorflow/compiler/xla:debug_options_flags",
+        "//tensorflow/compiler/xla:test",
+    ],
+)
diff --git a/tensorflow/compiler/xla/tests/ptxas_bug_120501638.cc b/tensorflow/compiler/xla/tests/ptxas_bug_120501638.cc
new file mode 100644
index 0000000000..0e5d7db97e
--- /dev/null
+++ b/tensorflow/compiler/xla/tests/ptxas_bug_120501638.cc
@@ -0,0 +1,82 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/debug_options_flags.h"
+#include "tensorflow/compiler/xla/test.h"
+#include "tensorflow/compiler/xla/tests/hlo_test_base.h"
+#include "tensorflow/compiler/xla/tests/test_macros.h"
+
+namespace xla {
+namespace {
+
+class PtxasBugTest : public HloTestBase {};
+
+// Checks for a bug in ptxas, tracked as Google bug 120501638, and nvidia bug
+// 2459377.  We never received an explanation of what exactly was going wrong
+// here in ptxas.  Known-bad in ptxas 10.0.145, known-good in ptxas 10.0.249.
+TEST_F(PtxasBugTest, DoIt) {
+  const char* const kModuleStr = R"(
+HloModule test
+
+add_F32.14 {
+  lhs.15 = f32[] parameter(0)
+  rhs.16 = f32[] parameter(1)
+  ROOT add.17 = f32[] add(lhs.15, rhs.16)
+}
+
+ENTRY testcase {
+  arg0.1 = f32[2,5,2]{2,1,0} parameter(0)
+  reshape.2 = f32[2,5,2]{2,1,0} reshape(arg0.1)
+  constant.3 = f32[] constant(0)
+  pad.4 = f32[2,6,2]{2,1,0} pad(reshape.2, constant.3), padding=0_0x0_1x0_0
+  reshape.5 = f32[2,3,2,2]{3,2,1,0} reshape(pad.4)
+  transpose.6 = f32[2,2,3,2]{3,0,2,1} transpose(reshape.5), dimensions={2,0,1,3}
+  reshape.7 = f32[4,3,2]{2,1,0} reshape(transpose.6)
+  reshape.8 = f32[4,1,3,2]{3,2,1,0} reshape(reshape.7)
+  transpose.9 = f32[4,2,1,3]{1,3,2,0} transpose(reshape.8), dimensions={0,3,1,2}
+  convert.10 = f32[4,2,1,3]{1,3,2,0} convert(transpose.9)
+  constant.12 = f32[] constant(0)
+  pad.13 = f32[4,2,1,3]{3,2,1,0} pad(convert.10, constant.12), padding=0_0x0_0x0_0x0_0
+  constant.11 = f32[] constant(0)
+  reduce-window.18 = f32[4,2,1,3]{3,2,1,0} reduce-window(pad.13, constant.11),
+    window={size=1x1x1x1}, to_apply=add_F32.14
+  constant.19 = f32[] constant(1)
+  broadcast.20 = f32[4,2,1,3]{3,2,1,0} broadcast(constant.19), dimensions={}
+  divide.21 = f32[4,2,1,3]{3,2,1,0} divide(reduce-window.18, broadcast.20)
+  convert.22 = f32[4,2,1,3]{3,2,1,0} convert(divide.21)
+  transpose.23 = f32[4,1,3,2]{2,1,3,0} transpose(convert.22), dimensions={0,2,3,1}
+  reshape.24 = f32[4,3,2]{2,1,0} reshape(transpose.23)
+  reshape.25 = f32[2,2,3,2]{3,2,1,0} reshape(reshape.24)
+  transpose.26 = f32[2,3,2,2]{3,1,0,2} transpose(reshape.25), dimensions={1,2,0,3}
+  reshape.27 = f32[2,6,2]{2,1,0} reshape(transpose.26)
+  slice.28 = f32[2,5,2]{2,1,0} slice(reshape.27), slice={[0:2], [0:5], [0:2]}
+  reshape.29 = f32[2,5,2]{2,1,0} reshape(slice.28)
+  tuple.30 = (f32[2,5,2]{2,1,0}) tuple(reshape.29)
+  ROOT get-tuple-element.31 = f32[2,5,2]{2,1,0} get-tuple-element(tuple.30), index=0
+})";
+
+  // Create a module with the true-default flags, not the default-for-testing
+  // flags.  In particular, true-default flags enable unrolling, whereas for
+  // testing we disable unrolling, and this bug doesn't trigger without
+  // unrolling.
+  HloModuleConfig config;
+  config.set_debug_options(DefaultDebugOptionsIgnoringFlags());
+  TF_ASSERT_OK_AND_ASSIGN(auto module,
+                          ParseAndReturnVerifiedModule(kModuleStr, config));
+  EXPECT_TRUE(RunAndCompare(std::move(module), ErrorSpec{0.01, 0.01}));
+}
+
+}  // anonymous namespace
+}  // namespace xla
-- 
GitLab


From 9b884d8392706a82af674adbbff862ca2de593a6 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Mon, 7 Jan 2019 19:02:54 -0800
Subject: [PATCH 0346/1095] [XLA] Don't constant-fold instructions which
 transitively contain kRng or kAfterAll.

Constant-folding avoids folding kRng/kAfterAll instructions, but it also needs
to avoid folding instructions which *may contain* kRng/kAfterAll, because e.g.
folding a call that contains an rng is morally equivalent to folding an rng.

PiperOrigin-RevId: 228272774
---
 .../xla/service/hlo_constant_folding.cc       | 66 +++++++++++++++----
 .../xla/service/hlo_constant_folding_test.cc  | 46 +++++++++++++
 2 files changed, 98 insertions(+), 14 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_constant_folding.cc b/tensorflow/compiler/xla/service/hlo_constant_folding.cc
index 799cc9fd91..e7ed858e8c 100644
--- a/tensorflow/compiler/xla/service/hlo_constant_folding.cc
+++ b/tensorflow/compiler/xla/service/hlo_constant_folding.cc
@@ -35,6 +35,34 @@ limitations under the License.
 
 namespace xla {
 
+// Checks whether instr is or transitively contains an instruction that we
+// shouldn't fold.
+//
+// Specifically, we don't fold kRng or kAfterAll instructions:
+//
+//  - kRng is already marked as side-effecting and so is skipped elsewhere, but
+//    we check for it here.  Even kRng weren't side-effecting and took an
+//    explicit seed, we *still* wouldn't want to constant-fold it, because the
+//    evaluator's handling of rng is not guaranteed to be identical to any
+//    particular backend's rng.
+//
+//  - kAfterAll needs to be skipped because a kAfterAll op with no args can
+//    currently materialize a token "out of thin air".  TODO(b/110532604):
+//    Remove this check once AfterAll requires at least one operand, in which
+//    case constant folding will be impossible.
+static bool IsOrContainsIllegalInstr(const HloInstruction* instr) {
+  if (instr->opcode() == HloOpcode::kAfterAll ||
+      instr->opcode() == HloOpcode::kRng) {
+    return true;
+  }
+  for (const HloComputation* c : instr->called_computations()) {
+    if (absl::c_any_of(c->instructions(), IsOrContainsIllegalInstr)) {
+      return true;
+    }
+  }
+  return false;
+}
+
 StatusOr<bool> HloConstantFolding::Run(HloModule* module) {
   // Limit the constant folding to 0 iterations to skip folding loops. This
   // retains the behavior from before while loop support in HloEvaluator and may
@@ -52,26 +80,24 @@ StatusOr<bool> HloConstantFolding::Run(HloModule* module) {
           computation->root_instruction() != instruction) {
         continue;
       }
-      // Skip Constant, Parameter, Tuple, AfterAll, Rng operations.
-      // Tuple constants are not directly supported by any backends, hence
-      // folding Tuple is not useful and would in fact be expanded back into
-      // kTuple by Algebraic Simplifier.
-      // TODO(b/110532604): Enable AfterAll once AfterAll requires at least one
-      // operand in which case constant folding will be impossible and this
-      // special case is not necessary.
-      if (instruction->opcode() == HloOpcode::kParameter ||
-          instruction->opcode() == HloOpcode::kConstant ||
-          instruction->opcode() == HloOpcode::kTuple ||
-          instruction->opcode() == HloOpcode::kAfterAll ||
-          instruction->opcode() == HloOpcode::kRng) {
-        continue;
-      }
 
       // Skip instructions with non-constant operands.
       if (!hlo_query::AllOperandsAreConstants(*instruction)) {
         continue;
       }
 
+      // Don't fold Constant, Parameter, and Tuple instructions.  Tuple
+      // constants are not directly supported by any backends, hence folding
+      // Tuple is not useful and would in fact be expanded back into kTuple by
+      // Algebraic Simplifier.
+      //
+      // (We do allow folding subcomputations that contain these instructions.)
+      if (instruction->opcode() == HloOpcode::kParameter ||
+          instruction->opcode() == HloOpcode::kConstant ||
+          instruction->opcode() == HloOpcode::kTuple) {
+        continue;
+      }
+
       // Broadcasts dramatically increase the size of constants, which is often
       // detrimental to performance and memory capacity, so do not fold
       // broadcasts.
@@ -80,6 +106,18 @@ StatusOr<bool> HloConstantFolding::Run(HloModule* module) {
         continue;
       }
 
+      // Check for instructions that we can't fold even if they appear inside of
+      // a subcomputation (e.g. a kCall).
+      if (IsOrContainsIllegalInstr(instruction)) {
+        continue;
+      }
+
+      // Don't constant-fold side-effecting instructions or instructions which
+      // contain side-effecting instructions.
+      if (instruction->HasSideEffect()) {
+        continue;
+      }
+
       // Don't constant fold unless it's a net positive or the output is small.
       if (instruction->shape().IsArray()) {
         int64 elements_in_removed_operands = 0;
diff --git a/tensorflow/compiler/xla/service/hlo_constant_folding_test.cc b/tensorflow/compiler/xla/service/hlo_constant_folding_test.cc
index 92b748d813..4bdc980c9a 100644
--- a/tensorflow/compiler/xla/service/hlo_constant_folding_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_constant_folding_test.cc
@@ -268,5 +268,51 @@ TEST_F(HloConstantFoldingTest, DoesNotFoldLargePad) {
               GmockMatch(m::Pad(m::Constant(), m::Constant())));
 }
 
+TEST_F(HloConstantFoldingTest, DontFoldSubcomputationContainingAfterAll) {
+  const char* const kModuleStr = R"(
+  HloModule test
+
+  Fn {
+    tok = token[] after-all()
+    ROOT root = f32[10] iota(), iota_dimension=0
+  }
+
+  ENTRY entry {
+    ROOT call = f32[10] call(), to_apply=Fn
+  })";
+  TF_ASSERT_OK_AND_ASSIGN(auto module,
+                          ParseAndReturnVerifiedModule(kModuleStr));
+  HloConstantFolding constant_folding;
+  TF_ASSERT_OK_AND_ASSIGN(bool result,
+                          RunHloPass(&constant_folding, module.get()));
+  EXPECT_FALSE(result);
+}
+
+TEST_F(HloConstantFoldingTest,
+       DontFoldSubcomputationTransitivelyContainingRng) {
+  const char* const kModuleStr = R"(
+  HloModule test
+
+  InnerFn {
+    c0 = f32[] constant(0)
+    c1 = f32[] constant(1)
+    ROOT rng = f32[10] rng(c0, c1), distribution=rng_uniform
+  }
+
+  Fn {
+    ROOT fusion = f32[10] fusion(), kind=kLoop, calls=InnerFn
+  }
+
+  ENTRY entry {
+    ROOT call = f32[10] call(), to_apply=Fn
+  })";
+  TF_ASSERT_OK_AND_ASSIGN(auto module,
+                          ParseAndReturnVerifiedModule(kModuleStr));
+  HloConstantFolding constant_folding;
+  TF_ASSERT_OK_AND_ASSIGN(bool result,
+                          RunHloPass(&constant_folding, module.get()));
+  EXPECT_FALSE(result);
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From 5265604a31dcad7e37a326b1f7d5f1ff006968fb Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 7 Jan 2019 19:17:05 -0800
Subject: [PATCH 0347/1095] [convergence_tools]: Moving the norm, nan, and max
 computations inside of TPU.

PiperOrigin-RevId: 228273700
---
 .../contrib/tpu/python/tpu/tensor_tracer.py   | 124 ++++++++++++------
 1 file changed, 83 insertions(+), 41 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tensor_tracer.py b/tensorflow/contrib/tpu/python/tpu/tensor_tracer.py
index 66689cde2f..c7e29860e5 100644
--- a/tensorflow/contrib/tpu/python/tpu/tensor_tracer.py
+++ b/tensorflow/contrib/tpu/python/tpu/tensor_tracer.py
@@ -377,10 +377,7 @@ class TensorTracer(object):
       return True
     # Reasons for not including following op types:
     #    Assign: cause incorrect result with CPU tracing.
-    #    others: compilation problems.
-    # TODO(deveci): Check if 'Pack', 'Shape', 'Reshape', 'ArgMin', 'ArgMax'
-    #               are still unsafe now that we have handled int64 tensors.
-    if op.type in ['Assign', 'Pack', 'Shape', 'Reshape', 'ArgMin', 'ArgMax']:
+    if op.type in ['Assign']:
       return True
     return False
 
@@ -471,7 +468,7 @@ class TensorTracer(object):
                 temporarily_marked_ops, sorted_ops)
       # pylint: disable=protected-access
       for ctrl_output_op in op._control_outputs:
-      # pylint: enable=protected-access
+        # pylint: enable=protected-access
         visit(ctrl_output_op, cycle, permanently_marked_ops,
               temporarily_marked_ops, sorted_ops)
       temporarily_marked_ops.remove(op)
@@ -737,6 +734,59 @@ class TensorTracer(object):
       self._write_report('%d "%s"\n'%(i, l[i].name))
     self._write_report('%s %s\n'%(_MARKER_SECTION_END, _SECTION_NAME_GRAPH))
 
+  def _preprocess_traced_tensor(self, tensor):
+    """Computes NAN/Norm/Max on TPUs before sending to CPU.
+
+    Args:
+      tensor: The tensor to be traced.
+    Returns:
+      A tensor that should be input to the trace_function.
+    Raises:
+      RuntimeError: If the trace mode is invalid.
+    """
+
+    def _detect_nan_inf(tensor):
+      """Trace function for detecting any NaN/Inf in the tensor."""
+
+      if tensor.dtype.is_floating:
+        output_tensor = math_ops.reduce_any(
+            gen_math_ops.logical_or(
+                gen_math_ops.is_nan(tensor), gen_math_ops.is_inf(tensor)))
+      else:
+        output_tensor = constant_op.constant(False)
+      # The shape has to be 1. Set it if it does not have the information.
+      output_tensor = array_ops.reshape(output_tensor, [1])
+      return output_tensor
+
+    def _show_norm(tensor):
+      tensor = math_ops.cast(tensor, dtypes.float32)
+      output_tensor = linalg_ops.norm(tensor)
+      # The shape has to be 1. Set it if it does not have the information.
+      output_tensor = array_ops.reshape(output_tensor, [1])
+      return output_tensor
+
+    def _show_max_abs(tensor):
+      tensor = math_ops.cast(tensor, dtypes.float32)
+      output_tensor = math_ops.reduce_max(math_ops.abs(tensor))
+      zero = constant_op.constant(0, dtypes.float32)
+      output_tensor = gen_math_ops.maximum(zero, output_tensor)
+      # The shape has to be 1. Set it if it does not have the information.
+      output_tensor = array_ops.reshape(output_tensor, [1])
+      return output_tensor
+
+    if self._trace_mode == _TRACE_MODE_NAN_INF:
+      return _detect_nan_inf(tensor)
+    if self._trace_mode == _TRACE_MODE_PART_TENSOR:
+      return tensor
+    if self._trace_mode == _TRACE_MODE_FULL_TENSOR:
+      return tensor
+    if self._trace_mode == _TRACE_MODE_NORM:
+      return _show_norm(tensor)
+    if self._trace_mode == _TRACE_MODE_MAX_ABS:
+      return _show_max_abs(tensor)
+    raise RuntimeError(
+        'Tensor trace fun for %s is not yet implemented' % self._trace_mode)
+
   def _make_tensor_trace_fun(self, tensor_name):
     """Makes the tensor tracing function called by outside compilation.
 
@@ -787,29 +837,6 @@ class TensorTracer(object):
       with ops.control_dependencies([print_op]):
         return array_ops.identity(tensor).op
 
-    def _detect_nan_inf(tensor):
-      """Trace function for detecting any NaN/Inf in the tensor."""
-
-      if tensor.dtype.is_floating:
-        output_tensor = math_ops.reduce_any(
-            gen_math_ops.logical_or(gen_math_ops.is_nan(tensor),
-                                    gen_math_ops.is_inf(tensor)))
-      else:
-        output_tensor = constant_op.constant(False)
-
-      return _print_tensor(tensor_name, -1, tensor, output_tensor)
-
-    def _show_norm(tensor):
-      tensor = math_ops.cast(tensor, dtypes.float64)
-      output_tensor = linalg_ops.norm(tensor)
-      return _print_tensor(tensor_name, -1, tensor, output_tensor)
-
-    def _show_max_abs(tensor):
-      output_tensor = math_ops.cast(math_ops.reduce_max(math_ops.abs(tensor)),
-                                    dtypes.float64)
-      zero = constant_op.constant(0, dtypes.float64)
-      output_tensor = gen_math_ops.maximum(zero, output_tensor)
-      return _print_tensor(tensor_name, -1, tensor, output_tensor)
 
     def _show_part_tensor(tensor):
       """Trace function for printing part of the tensor."""
@@ -822,16 +849,17 @@ class TensorTracer(object):
 
       return _print_tensor(tensor_name, -1, tensor, tensor)
 
-    if self._trace_mode == _TRACE_MODE_NAN_INF:
-      return _detect_nan_inf
     if self._trace_mode == _TRACE_MODE_PART_TENSOR:
       return _show_part_tensor
-    if self._trace_mode == _TRACE_MODE_FULL_TENSOR:
+    # The input tensor has a shape of "[1]" for _TRACE_MODE_NAN_INF,
+    # _TRACE_MODE_NORM, and _TRACE_MODE_MAX_ABS, as related computations are
+    # performed within TPUs and only their results are transferred to CPU.
+    # Simply, print the full tensor for these trace modes.
+    if self._trace_mode in [
+        _TRACE_MODE_NAN_INF, _TRACE_MODE_NORM, _TRACE_MODE_FULL_TENSOR,
+        _TRACE_MODE_MAX_ABS
+    ]:
       return _show_full_tensor
-    if self._trace_mode == _TRACE_MODE_NORM:
-      return _show_norm
-    if self._trace_mode == _TRACE_MODE_MAX_ABS:
-      return _show_max_abs
 
     raise RuntimeError('Tensor trace fun for %s is not yet implemented'
                        %self._trace_mode)
@@ -889,9 +917,18 @@ class TensorTracer(object):
           op_id, _REASON_USER_EXCLUDED)
       return True
     if not out_tensor.get_shape().is_fully_defined():
-      self._instrument_records[out_tensor.name] = TensorTracer.reason(
-          op_id, _REASON_DYNAMIC_SHAPE)
-      return True
+      # If trace mode is nan-inf, norm or max, then the tensor will be reduced
+      # to a scalar before the outside compilation call.
+      if self._trace_mode in [
+          _TRACE_MODE_NAN_INF, _TRACE_MODE_NORM, _TRACE_MODE_MAX_ABS
+      ]:
+        self._instrument_records[out_tensor.name] = TensorTracer.reason(
+            op_id, _REASON_TENSOR_GET_TRACED)
+        return False
+      else:
+        self._instrument_records[out_tensor.name] = TensorTracer.reason(
+            op_id, _REASON_DYNAMIC_SHAPE)
+        return True
     rank = len(out_tensor.shape)
     if rank < 1:
       # scalar
@@ -1000,11 +1037,15 @@ class TensorTracer(object):
         if self._skip_tensor(op_id, out_tensor, user_included,
                              user_excluded):
           continue
+        # Create the list of consumers before calling _preprocess_traced_tensor.
+        # Otherwise, adding control input below, will introduce a cycle in the
+        # graph.
         consumers = out_tensor.consumers()
         tensor_name = out_tensor.name
-        out_tensor = _cast_unsupported_dtypes(out_tensor)
+        processed_out_tensor = self._preprocess_traced_tensor(out_tensor)
+        processed_out_tensor = _cast_unsupported_dtypes(processed_out_tensor)
         trace_op = tpu.outside_compilation(
-            self._make_tensor_trace_fun(tensor_name), out_tensor)
+            self._make_tensor_trace_fun(tensor_name), processed_out_tensor)
         if consumers:
           for consumer_op in consumers:
             # pylint: disable=protected-access
@@ -1050,8 +1091,9 @@ class TensorTracer(object):
         if self._skip_tensor(op_id, out_tensor, user_included,
                              user_excluded):
           continue
+        processed_out_tensor = self._preprocess_traced_tensor(out_tensor)
         trace_fun = self._make_tensor_trace_fun(out_tensor.name)
-        trace_call = (trace_fun, [out_tensor])
+        trace_call = (trace_fun, [processed_out_tensor])
         trace_call_key = 'tensor_tracing_cpu-%s:%d'%(op.name, i)
         tracing_calls[trace_call_key] = trace_call
     self._post_tracing(succeed, sorted_or_cycle)
-- 
GitLab


From d95400ecd815c493e4241a96bea1c483387d0059 Mon Sep 17 00:00:00 2001
From: Pooya Davoodi <pdavoodi@nvidia.com>
Date: Mon, 7 Jan 2019 19:42:00 -0800
Subject: [PATCH 0348/1095] TFTRT: use six.text_type and six.binary_type

---
 .../contrib/tensorrt/python/trt_convert.py    | 24 +++++--------------
 1 file changed, 6 insertions(+), 18 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/python/trt_convert.py b/tensorflow/contrib/tensorrt/python/trt_convert.py
index f5d624ef64..ac39a570d8 100644
--- a/tensorflow/contrib/tensorrt/python/trt_convert.py
+++ b/tensorflow/contrib/tensorrt/python/trt_convert.py
@@ -48,30 +48,18 @@ from tensorflow.python.training import saver
 def _to_bytes(s):
     """Returns encoded of s if s is a sequence of chars otherwise returns s.
     """
-    if _six.PY2:
-        if isinstance(s, unicode):
-            return s.encode("utf-8", errors="surrogateescape")
-        else:
-            return s
+    if isinstance(s, _six.text_type):
+        return s.encode("utf-8", errors="surrogateescape")
     else:
-        if isinstance(s, str):
-            return s.encode("utf-8", errors="surrogateescape")
-        else:
-            return s
+        return s
 
 def _to_string(s):
     """Returns decoded of s if s is a sequence of bytes otherwise returns s.
     """
-    if _six.PY2:
-        if isinstance(s, str):
-            return s.decode("utf-8")
-        else:
-            return s
+    if isinstance(s, _six.binary_type):
+        return s.decode("utf-8")
     else:
-        if isinstance(s, bytes):
-            return s.decode("utf-8")
-        else:
-            return s
+        return s
 
 class TrtPrecisionMode(object):
   FP32 = "FP32"
-- 
GitLab


From 8c3f0f594c8323c8f5bd004200e738ffd4a5a64a Mon Sep 17 00:00:00 2001
From: Siju <siju.samuel@huawei.com>
Date: Tue, 8 Jan 2019 09:28:38 +0530
Subject: [PATCH 0349/1095] Update ceil.cc

removed space as part of clang-format
---
 tensorflow/lite/kernels/ceil.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/kernels/ceil.cc b/tensorflow/lite/kernels/ceil.cc
index e0ea061f25..6bb763255b 100644
--- a/tensorflow/lite/kernels/ceil.cc
+++ b/tensorflow/lite/kernels/ceil.cc
@@ -42,7 +42,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
   TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
 
   optimized_ops::Ceil(GetTensorShape(input), GetTensorData<float>(input),
-                       GetTensorShape(output), GetTensorData<float>(output));
+                      GetTensorShape(output), GetTensorData<float>(output));
 
   return kTfLiteOk;
 }
-- 
GitLab


From 95b18ddcbf0527e8da892e5a665e4d2547b47d67 Mon Sep 17 00:00:00 2001
From: Siju <siju.samuel@huawei.com>
Date: Tue, 8 Jan 2019 09:31:36 +0530
Subject: [PATCH 0350/1095] Update ceil_test.cc

updated using clang-format
---
 tensorflow/lite/kernels/ceil_test.cc | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/tensorflow/lite/kernels/ceil_test.cc b/tensorflow/lite/kernels/ceil_test.cc
index e120105082..74d2bfb483 100644
--- a/tensorflow/lite/kernels/ceil_test.cc
+++ b/tensorflow/lite/kernels/ceil_test.cc
@@ -55,18 +55,11 @@ TEST(CeilOpTest, SingleDim) {
 
 TEST(CeilOpTest, MultiDims) {
   CeilOpModel model({2, 1, 1, 5}, TensorType_FLOAT32);
-  model.PopulateTensor<float>(model.input(), {
-                                                 0.0001,
-                                                 8.0001,
-                                                 0.9999,
-                                                 9.9999,
-                                                 0.5,
-                                                 -0.0001,
-                                                 -8.0001,
-                                                 -0.9999,
-                                                 -9.9999,
-                                                 -0.5,
-                                             });
+  model.PopulateTensor<float>(
+      model.input(), {
+                         0.0001, 8.0001, 0.9999, 9.9999, 0.5, -0.0001, -8.0001,
+                         -0.9999, -9.9999, -0.5,
+                     });
   model.Invoke();
   EXPECT_THAT(model.GetOutput(),
               ElementsAreArray({1, 9, 1, 10, 1, 0, -8, 0, -9, 0}));
-- 
GitLab


From 4cff2f464cf195fedd8165a11319e8379746456f Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Mon, 7 Jan 2019 19:59:50 -0800
Subject: [PATCH 0351/1095] Organize the dot op emitter a bit better; NFC

This CL pays down some technical debt around dot_op_emitter's organization.  It

 - Extracts out a GetDotImplementationStrategy that decides the lowering
   strategy for a dot operation.  Earlier this logic was scattered through the
   lowering code.

 - Removes calls to PotentiallyImplementedAsEigenDot.  This is an implementation
   detail for dot_op_emitter and exposing it like this has aged poorly in the
   face of the various tiled LLVM IR lowerings we now have.  Instead, we expose
   two more specific hooks: DotOperandsAndResultMustHaveRowMajorLayout and
   DotImplementationCanHandleTranspose.

 - Renames xla_enable_experimental_llvm_ir_gemm to
   xla_force_enable_experimental_llvm_ir_gemm since its behavior is that it
   enables the tiled LLVM IR GEMM even when it does not look profitable.

Note: In parallel_task_assignment the call to PotentiallyImplementedAsEigenDot
was redundant since we already check for non-loop fusions and dots.
PiperOrigin-RevId: 228276282
---
 tensorflow/compiler/xla/service/cpu/BUILD     |   5 +-
 .../compiler/xla/service/cpu/cpu_compiler.cc  |   3 +-
 .../cpu/cpu_eigen_tensor_alignment_test.cc    |  16 +-
 .../xla/service/cpu/cpu_layout_assignment.cc  |  64 ++--
 .../compiler/xla/service/cpu/cpu_options.cc   |   8 +-
 .../compiler/xla/service/cpu/cpu_options.h    |   2 +-
 .../xla/service/cpu/dot_op_emitter.cc         | 301 ++++++++++--------
 .../compiler/xla/service/cpu/dot_op_emitter.h |  40 ++-
 .../xla/service/cpu/dot_op_emitter_internal.h |  88 +++++
 .../service/cpu/parallel_task_assignment.cc   |   2 -
 10 files changed, 318 insertions(+), 211 deletions(-)
 create mode 100644 tensorflow/compiler/xla/service/cpu/dot_op_emitter_internal.h

diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD
index 86c5f71096..1eb42cf5ad 100644
--- a/tensorflow/compiler/xla/service/cpu/BUILD
+++ b/tensorflow/compiler/xla/service/cpu/BUILD
@@ -382,7 +382,10 @@ cc_library(
 cc_library(
     name = "dot_op_emitter",
     srcs = ["dot_op_emitter.cc"],
-    hdrs = ["dot_op_emitter.h"],
+    hdrs = [
+        "dot_op_emitter.h",
+        "dot_op_emitter_internal.h",
+    ],
     deps = [
         ":cpu_options",
         ":cpu_runtime",
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
index 91c64e45b9..a6d92ce10e 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
@@ -304,7 +304,8 @@ Status CpuCompiler::RunHloPassesThroughLayoutAssn(
   pipeline.AddPass<TransposeFolding>(
       [&](const HloInstruction& dot,
           const TransposeFolding::OperandIndices& candidate_operands) {
-        return PotentiallyImplementedAsEigenDot(dot, *target_machine_features)
+        return DotImplementationCanHandleTranspose(dot,
+                                                   *target_machine_features)
                    ? candidate_operands
                    : TransposeFolding::OperandIndices{};
       },
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_eigen_tensor_alignment_test.cc b/tensorflow/compiler/xla/service/cpu/cpu_eigen_tensor_alignment_test.cc
index 8727c72b6e..823bdf259c 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_eigen_tensor_alignment_test.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_eigen_tensor_alignment_test.cc
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/compiler/xla/service/cpu/dot_op_emitter.h"
+#include "tensorflow/compiler/xla/service/cpu/dot_op_emitter_internal.h"
 #include "tensorflow/compiler/xla/service/cpu/ir_emission_utils.h"
 #include "tensorflow/compiler/xla/service/cpu/target_machine_features_fake.h"
 #include "tensorflow/compiler/xla/service/hlo_parser.h"
@@ -23,6 +23,10 @@ namespace xla {
 namespace cpu {
 namespace {
 
+using internal::DotImplementationStrategy;
+using internal::DotInfo;
+using internal::GetDotImplementationStrategy;
+
 // Test that we don't call into Eigen with tensors too small to be aligned
 // reliably.
 
@@ -47,16 +51,18 @@ ENTRY DotOperation {
   TargetMachineFeaturesWithFakeAlignmentLogic target_machine_with_no_alignment(
       [](int64 size) { return 1; });
 
-  EXPECT_FALSE(
-      PotentiallyImplementedAsEigenDot(*dot, target_machine_with_no_alignment));
+  EXPECT_EQ(GetDotImplementationStrategy(HloModuleConfig{}, DotInfo(*dot),
+                                         target_machine_with_no_alignment),
+            DotImplementationStrategy::kNaiveLlvmIr);
 
   TargetMachineFeaturesWithFakeAlignmentLogic
       target_machine_with_full_alignment([](int64 size) {
         return TargetMachineFeatures::kEigenExpectedTensorAlignment;
       });
 
-  EXPECT_TRUE(PotentiallyImplementedAsEigenDot(
-      *dot, target_machine_with_full_alignment));
+  EXPECT_NE(GetDotImplementationStrategy(HloModuleConfig{}, DotInfo(*dot),
+                                         target_machine_with_full_alignment),
+            DotImplementationStrategy::kNaiveLlvmIr);
 }
 
 TEST_F(CpuEigenTensorAlignmentTest, EigenConvAlignment) {
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_layout_assignment.cc b/tensorflow/compiler/xla/service/cpu/cpu_layout_assignment.cc
index ff11409482..95b8025f87 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_layout_assignment.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_layout_assignment.cc
@@ -93,60 +93,38 @@ static Shape ColMajorShape(const Shape& old_shape) {
   return new_shape;
 }
 
+static bool OperandsAndResultMustHaveRowMajorLayout(
+    const HloInstruction& instr,
+    const TargetMachineFeatures& target_machine_features) {
+  if (instr.opcode() == HloOpcode::kConvolution) {
+    return PotentiallyImplementedAsEigenConvolution(instr,
+                                                    target_machine_features);
+  } else if (instr.opcode() == HloOpcode::kDot) {
+    return DotOperandsAndResultMustHaveRowMajorLayout(instr,
+                                                      target_machine_features);
+  }
+  return false;
+}
+
 Status CpuLayoutAssignment::AddBackendConstraints(
     LayoutConstraints* constraints) {
   ShouldMakeOperandColMajorCache cache;
 
   const HloComputation* computation = constraints->computation();
   for (auto* instruction : computation->instructions()) {
-    if (instruction->opcode() == HloOpcode::kConvolution &&
-        PotentiallyImplementedAsEigenConvolution(*instruction,
-                                                 target_machine_features_)) {
-      const HloInstruction* convolution = instruction;
-      const HloInstruction* lhs_instruction = convolution->operand(0);
-      const HloInstruction* rhs_instruction = convolution->operand(1);
-
-      // In order to implement `convolution` with Eigen convolution, the layouts
-      // of the input, filter, and output need to be row-major.
-      //
-      // These constraints are not hard constraints. Ideally, we should decide
-      // which layouts to choose according to some cost model.
-      Shape output_shape(RowMajorShape(convolution->shape()));
-      Shape input_shape(RowMajorShape(lhs_instruction->shape()));
-      Shape filter_shape(RowMajorShape(rhs_instruction->shape()));
-
-      // Set layouts of the instructions' shapes.
-      TF_RETURN_IF_ERROR(
-          constraints->SetOperandLayout(input_shape, convolution, 0));
-      TF_RETURN_IF_ERROR(
-          constraints->SetOperandLayout(filter_shape, convolution, 1));
-      TF_RETURN_IF_ERROR(
-          constraints->SetInstructionLayout(output_shape, convolution));
+    if (OperandsAndResultMustHaveRowMajorLayout(*instruction,
+                                                target_machine_features_)) {
+      TF_RETURN_IF_ERROR(constraints->SetInstructionLayout(
+          RowMajorShape(instruction->shape()), instruction));
+      for (int i = 0; i < instruction->operand_count(); i++) {
+        TF_RETURN_IF_ERROR(constraints->SetOperandLayout(
+            RowMajorShape(instruction->operand(i)->shape()), instruction, i));
+      }
     } else if (optional<int64> op_idx =
                    ShouldMakeOperandColumnMajor(&cache, *instruction)) {
       const HloInstruction* op = instruction->operand(*op_idx);
       TF_RETURN_IF_ERROR(constraints->SetOperandLayout(
           ColMajorShape(op->shape()), instruction, *op_idx));
-    } else if (PotentiallyImplementedAsEigenDot(*instruction,
-                                                target_machine_features_)) {
-      const HloInstruction* dot = instruction;
-      // In order to implement `dot` with Eigen dot, the layouts of the lhs,
-      // rhs, and output need to be row-major.
-      //
-      // These constraints are not hard constraints. Ideally, we should decide
-      // which layouts to choose according to some cost model.
-      Shape output_shape(RowMajorShape(dot->shape()));
-
-      const HloInstruction* lhs_instruction = dot->operand(0);
-      Shape lhs_shape(RowMajorShape(lhs_instruction->shape()));
-      TF_RETURN_IF_ERROR(constraints->SetOperandLayout(lhs_shape, dot, 0));
-
-      const HloInstruction* rhs_instruction = dot->operand(1);
-      Shape rhs_shape(RowMajorShape(rhs_instruction->shape()));
-      TF_RETURN_IF_ERROR(constraints->SetOperandLayout(rhs_shape, dot, 1));
-
-      // Set layouts of the instructions' shapes.
-      TF_RETURN_IF_ERROR(constraints->SetInstructionLayout(output_shape, dot));
     } else {
       for (int64 operand_no = 0; operand_no < instruction->operand_count();
            ++operand_no) {
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_options.cc b/tensorflow/compiler/xla/service/cpu/cpu_options.cc
index 92debb83e3..ff654c83d6 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_options.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_options.cc
@@ -23,8 +23,8 @@ namespace {
 
 const char* const kXlaOptimizeForSizeCpuOption = "xla_cpu_optimize_for_size";
 const char* const kLlvmIrDotTilingFactor = "xla_llvm_dot_tiling_factor";
-const char* const kXlaEnableExperimentalLlvmIrGemm =
-    "xla_enable_experimental_llvm_ir_gemm";
+const char* const kXlaForceEnableExperimentalLlvmIrGemm =
+    "xla_force_enable_experimental_llvm_ir_gemm";
 const char* const kLlvmIrGemmTileSize = "xla_llvm_ir_gemm_tile_size";
 
 }  // namespace
@@ -57,10 +57,10 @@ absl::optional<int64> LlvmIrGemvTilingFactor(const HloModuleConfig& config) {
   return absl::nullopt;
 }
 
-bool EnableExperimentalLlvmIrGemm(const HloModuleConfig& config) {
+bool ForceEnableExperimentalLlvmIrGemm(const HloModuleConfig& config) {
   const auto& extra_options_map =
       config.debug_options().xla_backend_extra_options();
-  return extra_options_map.count(kXlaEnableExperimentalLlvmIrGemm) > 0;
+  return extra_options_map.count(kXlaForceEnableExperimentalLlvmIrGemm) > 0;
 }
 
 static absl::string_view RemoveSuffix(absl::string_view str,
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_options.h b/tensorflow/compiler/xla/service/cpu/cpu_options.h
index 47c7eb13b6..99e6702d14 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_options.h
+++ b/tensorflow/compiler/xla/service/cpu/cpu_options.h
@@ -26,7 +26,7 @@ namespace options {
 
 bool OptimizeForSizeRequested(const HloModuleConfig& config);
 bool VectorizedReduceDisabled(const HloModuleConfig& config);
-bool EnableExperimentalLlvmIrGemm(const HloModuleConfig& config);
+bool ForceEnableExperimentalLlvmIrGemm(const HloModuleConfig& config);
 absl::optional<int64> LlvmIrGemvTilingFactor(const HloModuleConfig& config);
 absl::optional<std::tuple<int64, int64, int64>> LlvmIrGemmTileSize(
     const HloModuleConfig& config);
diff --git a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
index bdd688c2b9..70d8fee265 100644
--- a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
@@ -14,6 +14,7 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/compiler/xla/service/cpu/dot_op_emitter.h"
+#include "tensorflow/compiler/xla/service/cpu/dot_op_emitter_internal.h"
 
 #include <memory>
 #include <vector>
@@ -42,6 +43,18 @@ namespace xla {
 using llvm_ir::SetToFirstInsertPoint;
 
 namespace cpu {
+
+using internal::DotImplementationStrategy;
+using internal::DotInfo;
+using internal::GetDotImplementationStrategy;
+
+namespace {
+// Returns true if we should call into multi-threaded Eigen routines.
+bool ShouldUseMultiThreadedEigen(const HloModuleConfig& config) {
+  return config.debug_options().xla_cpu_multi_thread_eigen();
+}
+}  // namespace
+
 DotOpEmitter::DotOpEmitter(const HloInstruction& dot,
                            const llvm_ir::IrArray& target_array,
                            const llvm_ir::IrArray& lhs_array,
@@ -76,43 +89,9 @@ DotOpEmitter::DotOpEmitter(const HloInstruction& dot,
   return dot_emitter.Emit();
 }
 
-bool DotOpEmitter::EmitSmallGemmIfProfitable(
-    const DotOpEmitter::MatMultDims& mat_mult_dims) {
-  if (ShouldUseMultiThreadedEigen()) {
-    return false;
-  }
-
-  if (!EnableExperimentalLlvmIrGemm()) {
-    // TODO(sanjoy):  We should make these numbers micro-arch specific.
-    bool small_gemm = mat_mult_dims.k <= 128 &&
-                      ((mat_mult_dims.m <= 32 && mat_mult_dims.n <= 128) ||
-                       (mat_mult_dims.m <= 128 && mat_mult_dims.n <= 32));
-    if (!small_gemm) {
-      return false;
-    }
-  }
-
-  if (mat_mult_dims.lhs_non_canonical || mat_mult_dims.rhs_non_canonical) {
-    return false;
-  }
-
+void DotOpEmitter::EmitTiledLlvmIrGemm() {
   PrimitiveType primitive_type = dot_.shape().element_type();
-
-  switch (primitive_type) {
-    default:
-      return false;
-
-    case F32:
-    case F64:
-    case S32:
-    case S64:
-      break;
-  }
-
-  if (!(mat_mult_dims.lhs_column_major == mat_mult_dims.rhs_column_major &&
-        mat_mult_dims.rhs_column_major == mat_mult_dims.target_column_major)) {
-    return false;
-  }
+  MatMultDims mat_mult_dims = GetMatMultDims();
 
   llvm::Value* lhs = lhs_array_.GetBasePointer();
   llvm::Value* rhs = rhs_array_.GetBasePointer();
@@ -154,21 +133,13 @@ bool DotOpEmitter::EmitSmallGemmIfProfitable(
       /*rhs=*/rhs, /*result=*/target, b_,
       /*enable_fast_math=*/enable_fast_math,
       /*optimize_for_size=*/optimize_for_size);
-
-  return true;
 }
 
-bool DotOpEmitter::EmitLlvmIrDotIfProfitable() {
-  if (dot_.shape().dimensions_size() != 2) {
-    return false;
-  }
-
+void DotOpEmitter::EmitTiledLlvmIrGemv() {
   PrimitiveType primitive_type = dot_.shape().element_type();
 
-  if (!primitive_util::IsFloatingPointType(primitive_type) &&
-      !primitive_util::IsIntegralType(primitive_type)) {
-    return false;
-  }
+  CHECK(primitive_util::IsFloatingPointType(primitive_type) ||
+        primitive_util::IsIntegralType(primitive_type));
 
   MatMultDims mat_mult_dims = GetMatMultDims();
   bool is_column_major_matrix_vector = false;
@@ -209,9 +180,7 @@ bool DotOpEmitter::EmitLlvmIrDotIfProfitable() {
     }
   }
 
-  if (!is_column_major_matrix_vector && !is_row_major_matrix_vector) {
-    return EmitSmallGemmIfProfitable(mat_mult_dims);
-  }
+  CHECK(is_column_major_matrix_vector || is_row_major_matrix_vector);
 
   int64 tiling_factor = GetGemvTilingFactor();
   CHECK_GT(tiling_factor, 0);
@@ -264,8 +233,6 @@ bool DotOpEmitter::EmitLlvmIrDotIfProfitable() {
         /*enable_fast_math=*/enable_fast_math,
         /*optimize_for_size=*/optimize_for_size);
   }
-
-  return true;
 }
 
 Status DotOpEmitter::Emit() {
@@ -306,27 +273,41 @@ Status DotOpEmitter::Emit() {
     return EmitScalarDot();
   }
 
-  if (EmitLlvmIrDotIfProfitable()) {
-    return Status::OK();
+  switch (GetDotImplementationStrategy(hlo_module_config_, DotInfo(dot_),
+                                       target_machine_features_)) {
+    case DotImplementationStrategy::kNaiveLlvmIr:
+      EmitNaiveLlvmIrGemm();
+      return Status::OK();
+
+    case DotImplementationStrategy::kTiledLlvmIrGemv:
+      EmitTiledLlvmIrGemv();
+      return Status::OK();
+
+    case DotImplementationStrategy::kTiledLlvmIrGemm:
+      EmitTiledLlvmIrGemm();
+      return Status::OK();
+
+    case DotImplementationStrategy::kEigen:
+      return EmitCallToRuntime();
   }
+}
 
+void DotOpEmitter::EmitNaiveLlvmIrGemm() {
   CHECK_EQ(addend_array_, nullptr);
 
-  if (PotentiallyImplementedAsEigenDot(dot_, target_machine_features_)) {
-    return EmitCallToRuntime();
-  }
+  const Shape& lhs_shape = lhs_array_.GetShape();
+  const Shape& rhs_shape = rhs_array_.GetShape();
+  const DotDimensionNumbers& dim_nums = dot_.dot_dimension_numbers();
 
   // Reduce along dimension 0 of the LHS and 1 of the RHS. Vectors are a special
   // case where the reduction dimension is 0 for both LHS and RHS. This results
   // in a vector dot product producing a scalar.
-  int64 lhs_reduction_dimension =
-      dot_.dot_dimension_numbers().lhs_contracting_dimensions(0);
-  int64 rhs_reduction_dimension =
-      dot_.dot_dimension_numbers().rhs_contracting_dimensions(0);
+  int64 lhs_reduction_dimension = dim_nums.lhs_contracting_dimensions(0);
+  int64 rhs_reduction_dimension = dim_nums.rhs_contracting_dimensions(0);
 
   // Verify the reduction dimension in the two operands are the same size.
-  TF_RET_CHECK(lhs_shape.dimensions(lhs_reduction_dimension) ==
-               rhs_shape.dimensions(rhs_reduction_dimension));
+  CHECK_EQ(lhs_shape.dimensions(lhs_reduction_dimension),
+           rhs_shape.dimensions(rhs_reduction_dimension));
 
   bool lhs_reduction_along_minor_dimension =
       lhs_reduction_dimension == LayoutUtil::Minor(lhs_shape.layout(), 0);
@@ -441,8 +422,6 @@ Status DotOpEmitter::Emit() {
   // Set the IR builder insert point to the exit basic block of the outer most
   // loop.
   b_->SetInsertPoint(loop_nest.GetOuterLoopExitBasicBlock());
-
-  return Status::OK();
 }
 
 Status DotOpEmitter::EmitScalarDot() {
@@ -489,7 +468,7 @@ Status DotOpEmitter::EmitCallToRuntime() {
   // The two transpose_... parameters are actually booleans, but we use int32
   // to avoid target-dependent calling convention details.
 
-  bool multi_threaded = ShouldUseMultiThreadedEigen();
+  bool multi_threaded = ShouldUseMultiThreadedEigen(hlo_module_config_);
   bool use_mkl_dnn = hlo_module_config_.debug_options().xla_cpu_use_mkl_dnn();
   PrimitiveType type = target_array_.GetShape().element_type();
   llvm::Type* float_type;
@@ -600,12 +579,52 @@ DotOpEmitter::MatMultDims DotOpEmitter::GetMatMultDims() const {
       LayoutUtil::Minor(target_array_.GetShape().layout(), 0) == 0};
 }
 
+// For vector-matrix dot products, it is always profitable to make the Rhs
+// column major.
+absl::optional<int64> ProfitableToMakeDotOperandColumnMajor(
+    const HloInstruction& hlo) {
+  if (hlo.opcode() == HloOpcode::kDot && hlo.shape().dimensions_size() == 2 &&
+      hlo.shape().dimensions(0) == 1) {
+    if (hlo.dot_dimension_numbers().rhs_contracting_dimensions(0) == 0) {
+      return 1;
+    }
+    return {};
+  }
+
+  if (hlo.opcode() == HloOpcode::kFusion &&
+      hlo.fusion_kind() == HloInstruction::FusionKind::kOutput) {
+    auto* fusion_root =
+        hlo.fused_instructions_computation()->root_instruction();
+    if (fusion_root->opcode() != HloOpcode::kAdd) {
+      return {};
+    }
+
+    for (auto* fusion_root_op : fusion_root->operands()) {
+      if (fusion_root_op->opcode() != HloOpcode::kDot) {
+        continue;
+      }
+      if (auto operand_num =
+              ProfitableToMakeDotOperandColumnMajor(*fusion_root_op)) {
+        auto* operand = fusion_root_op->operand(*operand_num);
+        if (operand->opcode() == HloOpcode::kParameter &&
+            operand->user_count() == 1) {
+          return operand->parameter_number();
+        }
+      }
+    }
+  }
+
+  return {};
+}
+
+namespace internal {
+namespace {
 // Return whether the given shape is rank 2.
-static bool IsRank2(const Shape& shape) { return shape.rank() == 2; }
+bool IsRank2(const Shape& shape) { return shape.rank() == 2; }
 
 // In a gemm operation where output = lhs * rhs, check whether the given shapes
 // are valid for the operation.
-static bool AreValidGemmShapes(
+bool AreAlignedGemmShapes(
     const Shape& lhs_shape, const Shape& rhs_shape, const Shape& output_shape,
     const TargetMachineFeatures& target_machine_features) {
   // The inputs and the output must
@@ -634,88 +653,106 @@ static bool AreValidGemmShapes(
   return true;
 }
 
-bool PotentiallyImplementedAsEigenDot(
-    const HloInstruction& hlo,
-    const TargetMachineFeatures& target_machine_features) {
-  // For certain types of Dot, we can call Eigen
-  if (hlo.opcode() == HloOpcode::kDot) {
-    const Shape& lhs_shape = hlo.operand(0)->shape();
-    const Shape& rhs_shape = hlo.operand(1)->shape();
+bool IsAlignedGemm(const DotInfo& dot_info,
+                   const TargetMachineFeatures& target_machine_features) {
+  if (ShapeUtil::IsZeroElementArray(dot_info.lhs_shape) ||
+      ShapeUtil::IsZeroElementArray(dot_info.rhs_shape)) {
+    return false;
+  }
 
-    if (ShapeUtil::IsZeroElementArray(lhs_shape) ||
-        ShapeUtil::IsZeroElementArray(rhs_shape)) {
-      return false;
-    }
+  return AreAlignedGemmShapes(dot_info.lhs_shape, dot_info.rhs_shape,
+                              dot_info.result_shape, target_machine_features);
+}
 
-    if (ProfitableToImplementDotInTiledLlvmIr(hlo)) {
-      return false;
-    }
+bool CanEmitTiledLlvmIrGemm(
+    const HloModuleConfig& config, const DotInfo& dot_info,
+    const TargetMachineFeatures& target_machine_features) {
+  CHECK(IsAlignedGemm(dot_info, target_machine_features));
 
-    // If gemm can accept the operand shapes, use it rather than a custom
-    // kernel.
-    if (AreValidGemmShapes(lhs_shape, rhs_shape, hlo.shape(),
-                           target_machine_features)) {
-      const DotDimensionNumbers& dim_numbers = hlo.dot_dimension_numbers();
-      // The size of the reduction dimension should match. The shape inference
-      // guarantees this invariant, so the check here is for programming
-      // errors.
-      CHECK_EQ(lhs_shape.dimensions(dim_numbers.lhs_contracting_dimensions(0)),
-               rhs_shape.dimensions(dim_numbers.rhs_contracting_dimensions(0)));
-      return true;
-    }
+  if (ShouldUseMultiThreadedEigen(config)) {
+    return false;
   }
 
-  return false;
-}
+  int m = dot_info.result_shape.dimensions(0);
+  int k = dot_info.lhs_shape.dimensions(
+      dot_info.dim_nums.lhs_contracting_dimensions(0));
+  int n = dot_info.result_shape.dimensions(1);
 
-// For vector-matrix dot products, it is always profitable to make the Rhs
-// column major.
-absl::optional<int64> ProfitableToMakeDotOperandColumnMajor(
-    const HloInstruction& hlo) {
-  if (hlo.opcode() == HloOpcode::kDot && hlo.shape().dimensions_size() == 2 &&
-      hlo.shape().dimensions(0) == 1) {
-    if (hlo.dot_dimension_numbers().rhs_contracting_dimensions(0) == 0) {
-      return 1;
+  if (!options::ForceEnableExperimentalLlvmIrGemm(config)) {
+    // TODO(sanjoy):  We should make these numbers micro-arch specific.
+    bool small_gemm =
+        k <= 128 && ((m <= 32 && n <= 128) || (m <= 128 && n <= 32));
+    if (!small_gemm) {
+      return false;
     }
-    return {};
   }
 
-  if (hlo.opcode() == HloOpcode::kFusion &&
-      hlo.fusion_kind() == HloInstruction::FusionKind::kOutput) {
-    auto* fusion_root =
-        hlo.fused_instructions_computation()->root_instruction();
-    if (fusion_root->opcode() != HloOpcode::kAdd) {
-      return {};
-    }
+  bool lhs_non_canonical = dot_info.dim_nums.lhs_contracting_dimensions(0) == 0;
+  bool rhs_non_canonical = dot_info.dim_nums.rhs_contracting_dimensions(0) == 1;
 
-    for (auto* fusion_root_op : fusion_root->operands()) {
-      if (fusion_root_op->opcode() != HloOpcode::kDot) {
-        continue;
-      }
-      if (auto operand_num =
-              ProfitableToMakeDotOperandColumnMajor(*fusion_root_op)) {
-        auto* operand = fusion_root_op->operand(*operand_num);
-        if (operand->opcode() == HloOpcode::kParameter &&
-            operand->user_count() == 1) {
-          return operand->parameter_number();
-        }
-      }
-    }
+  if (lhs_non_canonical || rhs_non_canonical) {
+    return false;
   }
 
-  return {};
+  if (dot_info.result_shape.element_type() == F16) {
+    // TODO(sanjoy): This is probably easy to fix, but I want to keep the CL
+    // adding this comment NFC.
+    return false;
+  }
+
+  return true;
 }
+}  // namespace
 
-bool ProfitableToImplementDotInTiledLlvmIr(const HloInstruction& dot) {
+DotImplementationStrategy GetDotImplementationStrategy(
+    const HloModuleConfig& config, const DotInfo& dot_info,
+    const TargetMachineFeatures& target_machine_features) {
+  PrimitiveType element_type = dot_info.result_shape.element_type();
   // Any Matrix-Vector product of floating point or integral type, or
   // a transpose-dot fusion of the same can be lowered to a tiled LLVM
   // IR implementation.
-  const Shape& shape = dot.shape();
-  return shape.dimensions_size() == 2 &&
-         (shape.dimensions(0) == 1 || shape.dimensions(1) == 1) &&
-         (primitive_util::IsFloatingPointType(shape.element_type()) ||
-          primitive_util::IsIntegralType(shape.element_type()));
+  if (dot_info.result_shape.dimensions_size() == 2 &&
+      (dot_info.result_shape.dimensions(0) == 1 ||
+       dot_info.result_shape.dimensions(1) == 1) &&
+      (primitive_util::IsFloatingPointType(element_type) ||
+       primitive_util::IsIntegralType(element_type))) {
+    return DotImplementationStrategy::kTiledLlvmIrGemv;
+  }
+
+  if (IsAlignedGemm(dot_info, target_machine_features)) {
+    return CanEmitTiledLlvmIrGemm(config, dot_info, target_machine_features)
+               ? DotImplementationStrategy::kTiledLlvmIrGemm
+               : DotImplementationStrategy::kEigen;
+  }
+
+  return DotImplementationStrategy::kNaiveLlvmIr;
 }
+}  // namespace internal
 
+bool DotImplementationCanHandleTranspose(
+    const HloInstruction& dot_instr,
+    const TargetMachineFeatures& target_machine_features) {
+  DotImplementationStrategy impl_strategy =
+      GetDotImplementationStrategy(dot_instr.parent()->parent()->config(),
+                                   DotInfo(dot_instr), target_machine_features);
+
+  // TODO(sanjoy): This is not quite right, it should be `impl_strategy ==
+  // kEigen || impl_strategy == kTiledLlvmIrGemv || impl_strategy ==
+  // kNaiveLlvmIr` but I'll fix this in a later CL in the interest of keeping
+  // the CL adding this comment NFC.
+  return impl_strategy == DotImplementationStrategy::kTiledLlvmIrGemm ||
+         impl_strategy == DotImplementationStrategy::kEigen;
+}
+
+bool DotOperandsAndResultMustHaveRowMajorLayout(
+    const HloInstruction& dot_instr,
+    const TargetMachineFeatures& target_machine_features) {
+  DotImplementationStrategy impl_strategy =
+      GetDotImplementationStrategy(dot_instr.parent()->parent()->config(),
+                                   DotInfo(dot_instr), target_machine_features);
+
+  return impl_strategy == DotImplementationStrategy::kTiledLlvmIrGemm ||
+         impl_strategy == DotImplementationStrategy::kEigen;
+}
 }  // namespace cpu
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.h b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.h
index 4c2041b556..eaa226a4c2 100644
--- a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.h
+++ b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.h
@@ -30,9 +30,16 @@ limitations under the License.
 
 namespace xla {
 namespace cpu {
+// Returns true if the two operands and the output of `dot_instr` must have row
+// major layout.
+bool DotOperandsAndResultMustHaveRowMajorLayout(
+    const HloInstruction& dot_instr,
+    const TargetMachineFeatures& target_machine_features);
 
-bool PotentiallyImplementedAsEigenDot(
-    const HloInstruction& hlo,
+// Returns true our lowering strategy for `dot_instr` can fold in transposes to
+// the either of the inputs.
+bool DotImplementationCanHandleTranspose(
+    const HloInstruction& dot_instr,
     const TargetMachineFeatures& target_machine_features);
 
 // Returns the index for an operand to `hlo` that should ideally be column
@@ -41,10 +48,6 @@ bool PotentiallyImplementedAsEigenDot(
 absl::optional<int64> ProfitableToMakeDotOperandColumnMajor(
     const HloInstruction& hlo);
 
-// Returns true to indicate that we can generate a tiled LLVM IR implementation
-// for |dot|.
-bool ProfitableToImplementDotInTiledLlvmIr(const HloInstruction& dot);
-
 // Helper class for emitting LLVM IR to perform the dot operation.
 class DotOpEmitter {
  public:
@@ -81,10 +84,6 @@ class DotOpEmitter {
   // LHS and RHS) and store the results in the target.
   Status EmitScalarDot();
 
-  // Emit an LLVM IR implementation of the dot operation if we can.  Returns
-  // true if an LLVM IR implementation was emitted.
-  bool EmitLlvmIrDotIfProfitable();
-
   // Emits a call to the CPU runtime to perform the matrix multiply.
   Status EmitCallToRuntime();
 
@@ -121,7 +120,15 @@ class DotOpEmitter {
   // of rank 2 as well).
   MatMultDims GetMatMultDims() const;
 
-  bool EmitSmallGemmIfProfitable(const MatMultDims& mat_mult_dims);
+  // Lowers the dot operation as a tiled Matrix*Vector loop.
+  void EmitTiledLlvmIrGemv();
+
+  // Lowers the dot operation as a tiled Matrix*Matrix loop.
+  void EmitTiledLlvmIrGemm();
+
+  // Lowers the dot operation as a naive nested loop that computes the result
+  // one element at a time.
+  void EmitNaiveLlvmIrGemm();
 
   // When doing a tiled GEMV in LLVM IR, a "tile" consists of this many vector
   // registers.
@@ -142,17 +149,6 @@ class DotOpEmitter {
         .value_or(kDefaultTileSize);
   }
 
-  // Returns true if we should use an experimental implementation of GEMM
-  // (general matrix matrix multiplication) if possible.
-  bool EnableExperimentalLlvmIrGemm() const {
-    return options::EnableExperimentalLlvmIrGemm(hlo_module_config_);
-  }
-
-  // Returns true if we should call into multi-threaded Eigen routines.
-  bool ShouldUseMultiThreadedEigen() {
-    return hlo_module_config_.debug_options().xla_cpu_multi_thread_eigen();
-  }
-
   const HloInstruction& dot_;
   const llvm_ir::IrArray& target_array_;
   const llvm_ir::IrArray& lhs_array_;
diff --git a/tensorflow/compiler/xla/service/cpu/dot_op_emitter_internal.h b/tensorflow/compiler/xla/service/cpu/dot_op_emitter_internal.h
new file mode 100644
index 0000000000..cc28918ed6
--- /dev/null
+++ b/tensorflow/compiler/xla/service/cpu/dot_op_emitter_internal.h
@@ -0,0 +1,88 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_CPU_DOT_OP_EMITTER_INTERNAL_H_
+#define TENSORFLOW_COMPILER_XLA_SERVICE_CPU_DOT_OP_EMITTER_INTERNAL_H_
+
+#include "tensorflow/compiler/xla/service/cpu/target_machine_features.h"
+#include "tensorflow/compiler/xla/service/hlo_instruction.h"
+#include "tensorflow/compiler/xla/service/hlo_module.h"
+
+// -----------------------------------------------------------------------------
+// INTERNAL HEADER.
+//
+// This file exposes internal implementation details from dot_op_emitter.cc for
+// unit tests.  Please do not depend on this!
+//
+// -----------------------------------------------------------------------------
+
+namespace xla {
+namespace cpu {
+namespace internal {
+
+// Represents a dot operation.  We use this in lieu of an `HloInstruction`
+// because we want to be able to create this for the "inner" dot operation in a
+// batch dot, for which there is no separate HLO instruction.
+struct DotInfo {
+  Shape lhs_shape;
+  Shape rhs_shape;
+  Shape result_shape;
+  DotDimensionNumbers dim_nums;
+
+  explicit DotInfo(const HloInstruction& instr) {
+    CHECK_EQ(instr.opcode(), HloOpcode::kDot);
+    lhs_shape = instr.operand(0)->shape();
+    rhs_shape = instr.operand(1)->shape();
+    result_shape = instr.shape();
+    dim_nums = instr.dot_dimension_numbers();
+  }
+};
+
+// Dictates how a dot operation is implemented.
+enum class DotImplementationStrategy {
+  // The dot operation is lowered into LLVM IR that implements a naive nested
+  // loop that computes the result one element at a time.  This is our
+  // "fallback"; we don't really want this to kick in for any non-trival dot
+  // operation.
+  kNaiveLlvmIr,
+
+  // The dot operation is lowered into LLVM IR that implements a tiled
+  // Matrix*Vector operation.  This strategy also allows fusing in a bias add
+  // into the dot.  The matrix can be row major or column major, both are
+  // supported.
+  kTiledLlvmIrGemv,
+
+  // The dot operation is lowered into LLVM IR that implemetns a tiled
+  // Matrix*Matrix operation.  No fusions are supported.  The two inputs
+  // and the output have to be row major.
+  kTiledLlvmIrGemm,
+
+  // The dot operation is lowered into a call into an Eigen routine.  No fusions
+  // are supported today.  The two inputs and the output have to be row major.
+  // However, we do allow transposing either the LHS or the RHS as part of the
+  // GEMM -- we expose this flexibility as flexibility in the contraction
+  // dimensions, but we can also see this as flexibility in the input layouts.
+  kEigen,
+};
+
+// Returns the implementation strategy for a dot with the configuration
+// `dot_info`.
+DotImplementationStrategy GetDotImplementationStrategy(
+    const HloModuleConfig& config, const DotInfo& dot_info,
+    const TargetMachineFeatures& target_machine_features);
+}  // namespace internal
+}  // namespace cpu
+}  // namespace xla
+
+#endif  // TENSORFLOW_COMPILER_XLA_SERVICE_CPU_DOT_OP_EMITTER_INTERNAL_H_
diff --git a/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc b/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc
index 3b423f6391..6121d1ca9a 100644
--- a/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc
+++ b/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc
@@ -146,8 +146,6 @@ int64 ParallelTaskAssignment::GetTargetParallelTaskCount(
       (opcode == HloOpcode::kConvolution &&
        PotentiallyImplementedAsEigenConvolution(*instruction,
                                                 target_machine_features_)) ||
-      PotentiallyImplementedAsEigenDot(*instruction,
-                                       target_machine_features_) ||
       (opcode == HloOpcode::kFusion &&
        instruction->fusion_kind() != HloInstruction::FusionKind::kLoop) ||
       instruction->shape().IsTuple()) {
-- 
GitLab


From 8c91712adfb586da675aeaa59730fd4c6e8c5ded Mon Sep 17 00:00:00 2001
From: Siju <siju.samuel@huawei.com>
Date: Tue, 8 Jan 2019 10:02:21 +0530
Subject: [PATCH 0352/1095] LOG the result in case of error.

---
 tensorflow/core/common_runtime/gpu/gpu_debug_allocator.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/common_runtime/gpu/gpu_debug_allocator.cc b/tensorflow/core/common_runtime/gpu/gpu_debug_allocator.cc
index 77737a76b6..2879b00f55 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_debug_allocator.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_debug_allocator.cc
@@ -46,7 +46,7 @@ bool CheckMask(se::StreamExecutor* exec, void* ptr, int64* mask) {
 
   Status result = exec->SynchronousMemcpyD2H(gpu_ptr, MASK_BYTES, tmp);
   if (!result.ok()) {
-    LOG(FATAL) << "Could not copy debug mask";
+    LOG(FATAL) << "Could not copy debug mask, " << result.error_message();
   }
 
   bool ok = true;
@@ -66,7 +66,7 @@ void InitMask(se::StreamExecutor* exec, void* ptr, int64* mask) {
   se::DeviceMemory<int64> gpu_ptr{se::DeviceMemoryBase{ptr, MASK_BYTES}};
   Status result = exec->SynchronousMemcpyH2D(mask, MASK_BYTES, &gpu_ptr);
   if (!result.ok()) {
-    LOG(FATAL) << "Could not copy debug mask";
+    LOG(FATAL) << "Could not copy debug mask, " << result.error_message();
   }
 }
 
@@ -176,7 +176,7 @@ void* GPUNanResetAllocator::AllocateRaw(size_t alignment, size_t num_bytes) {
   Status result = stream_exec_->SynchronousMemcpyH2D(&nans[0], req_size,
                                                      &nan_ptr);
   if (!result.ok()) {
-    LOG(ERROR) << "Could not initialize to NaNs";
+    LOG(ERROR) << "Could not initialize to NaNs, " << result.error_message();
   }
 
   return allocated_ptr;
@@ -192,7 +192,7 @@ void GPUNanResetAllocator::DeallocateRaw(void* ptr) {
     Status result = stream_exec_->SynchronousMemcpyH2D(&nans[0], req_size,
                                                        &nan_ptr);
     if (!result.ok()) {
-      LOG(ERROR) << "Could not initialize to NaNs";
+      LOG(ERROR) << "Could not initialize to NaNs, " << result.error_message();
     }
   }
 
-- 
GitLab


From 8808014ef840f98aafd93f1e54a069601a184586 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Tue, 8 Jan 2019 04:50:04 +0000
Subject: [PATCH 0353/1095] Update protobuf to 3.6.1 in install_proto3.sh

This fix updates protobuf to 3.6.1 in install_proto3.sh.

The protobuf library in workspace.bzl has been updated to 3.6.1 for
some time, though in CI build protoc still uses 3.6.0.

When build a custom op (in docker image `tensorflow/tensorflow:custom-op`)
and using grpc's `grpc_defs()` (`grpc/bazel/grpc_deps.bzl`):
https://github.com/grpc/grpc/blob/ac6795a57e05523b8fa220bc5cef26abb876aae5/bazel/grpc_deps.bzl#L5-L7
The issue arise because grpc's dependency is 3.6.1 (and an immediate earlier version is 3.5)
so there is no match to fit grpc+protobuf no matter which version of grpc is used.

This may not a problem for tensorflow itself as tensorflow does not call `grpc_defs()`. Tensorflow
links to protobuf 3.6.1 explicitly. Still, it would be good to match the
version of installed protoc (3.6.0) with protobuf library (3.6.1).

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/tools/ci_build/install/install_proto3.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/ci_build/install/install_proto3.sh b/tensorflow/tools/ci_build/install/install_proto3.sh
index 821d50baff..3cb1008567 100755
--- a/tensorflow/tools/ci_build/install/install_proto3.sh
+++ b/tensorflow/tools/ci_build/install/install_proto3.sh
@@ -17,7 +17,7 @@
 # Install protobuf3.
 
 # Select protobuf version.
-PROTOBUF_VERSION="3.6.0"
+PROTOBUF_VERSION="3.6.1"
 protobuf_ver_flat=$(echo $PROTOBUF_VERSION | sed 's/\.//g' | sed 's/^0*//g')
 local_protobuf_ver=$(protoc --version)
 local_protobuf_ver_flat=$(echo $local_protobuf_ver | sed 's/\.//g' | sed 's/^0*//g')
-- 
GitLab


From e7b6486adb822bb83a9ff669dc4d2d477d911b3c Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Tue, 8 Jan 2019 04:57:32 +0000
Subject: [PATCH 0354/1095] Update install_pip_packages.sh to protobuf 3.6.1

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/tools/ci_build/install/install_pip_packages.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/tools/ci_build/install/install_pip_packages.sh b/tensorflow/tools/ci_build/install/install_pip_packages.sh
index eeadabaa73..5ae840cfa0 100755
--- a/tensorflow/tools/ci_build/install/install_pip_packages.sh
+++ b/tensorflow/tools/ci_build/install/install_pip_packages.sh
@@ -60,8 +60,8 @@ pip2 install --upgrade markdown==2.6.8
 pip3 install --upgrade markdown==2.6.8
 
 # Install protobuf.
-pip2 install --upgrade protobuf==3.6.0
-pip3 install --upgrade protobuf==3.6.0
+pip2 install --upgrade protobuf==3.6.1
+pip3 install --upgrade protobuf==3.6.1
 
 # Remove obsolete version of six, which can sometimes confuse virtualenv.
 rm -rf /usr/lib/python3/dist-packages/six*
-- 
GitLab


From bb447c4db4c1c5990f476b7b5fdf1d3c5b3afacc Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Tue, 8 Jan 2019 04:58:19 +0000
Subject: [PATCH 0355/1095] Update protobuf to 3.6.1

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 .../tools/ci_build/install/install_python3.5_pip_packages.sh    | 2 +-
 .../tools/ci_build/install/install_python3.6_pip_packages.sh    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/tools/ci_build/install/install_python3.5_pip_packages.sh b/tensorflow/tools/ci_build/install/install_python3.5_pip_packages.sh
index 0654fb81b3..a58f49af28 100755
--- a/tensorflow/tools/ci_build/install/install_python3.5_pip_packages.sh
+++ b/tensorflow/tools/ci_build/install/install_python3.5_pip_packages.sh
@@ -52,7 +52,7 @@ pip3.5 install --upgrade absl-py
 pip3.5 install --upgrade six==1.10.0
 
 # Install protobuf.
-pip3.5 install --upgrade protobuf==3.6.0
+pip3.5 install --upgrade protobuf==3.6.1
 
 # Remove obsolete version of six, which can sometimes confuse virtualenv.
 rm -rf /usr/lib/python3/dist-packages/six*
diff --git a/tensorflow/tools/ci_build/install/install_python3.6_pip_packages.sh b/tensorflow/tools/ci_build/install/install_python3.6_pip_packages.sh
index 3edf4571fb..b1c2a0ab00 100755
--- a/tensorflow/tools/ci_build/install/install_python3.6_pip_packages.sh
+++ b/tensorflow/tools/ci_build/install/install_python3.6_pip_packages.sh
@@ -64,7 +64,7 @@ pip3 install --upgrade absl-py
 pip3 install --upgrade six==1.10.0
 
 # Install protobuf.
-pip3 install --upgrade protobuf==3.6.0
+pip3 install --upgrade protobuf==3.6.1
 
 # Remove obsolete version of six, which can sometimes confuse virtualenv.
 rm -rf /usr/lib/python3/dist-packages/six*
-- 
GitLab


From 246780be77509fa4bea195d05fa73faa99467c58 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Mon, 7 Jan 2019 21:32:32 -0800
Subject: [PATCH 0356/1095] Hide the DotOpEmitter class and make it use a
 DotInfo instead of a HloInstr; NFC

 - Hiding it in the .cc is good encapsulation.
 - Making it depend on a DotInfo as opposed to an HLO instruction will make
   implementing batch dot (in terms of non-batch dot) easier.

PiperOrigin-RevId: 228281578
---
 .../xla/service/cpu/dot_op_emitter.cc         | 159 ++++++++++++++----
 .../compiler/xla/service/cpu/dot_op_emitter.h | 130 ++------------
 .../compiler/xla/service/cpu/ir_emitter.cc    |  16 +-
 3 files changed, 156 insertions(+), 149 deletions(-)

diff --git a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
index 70d8fee265..e8a84ebe6b 100644
--- a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
@@ -53,9 +53,107 @@ namespace {
 bool ShouldUseMultiThreadedEigen(const HloModuleConfig& config) {
   return config.debug_options().xla_cpu_multi_thread_eigen();
 }
+
+// Helper class for emitting LLVM IR to perform the dot operation.
+class DotOpEmitter {
+ public:
+  explicit DotOpEmitter(DotInfo dot_info, string dot_hlo_name,
+                        const llvm_ir::IrArray& target_array,
+                        const llvm_ir::IrArray& lhs_array,
+                        const llvm_ir::IrArray& rhs_array,
+                        const llvm_ir::IrArray* addend_array,
+                        llvm::Value* executable_run_options_value,
+                        llvm::IRBuilder<>* b,
+                        const HloModuleConfig& hlo_module_config,
+                        const TargetMachineFeatures& target_machine_features);
+
+  // Emits the IR to perform the dot operation.
+  Status Emit();
+
+ private:
+  // Emits instructions to perform a scalar dot product (a multiply of the
+  // LHS and RHS) and store the results in the target.
+  Status EmitScalarDot();
+
+  // Emits a call to the CPU runtime to perform the matrix multiply.
+  Status EmitCallToRuntime();
+
+  // Represents the dimensions of a matrix-matrix multiply operation.
+  struct MatMultDims {
+    // The number of rows in the LHS.
+    int64 m;
+
+    // The number of columns in the LHS, which is also must be equal to the
+    // number of rows in the RHS.
+    int64 k;
+
+    // The number of columns on the RHS.
+    int64 n;
+
+    // True if the LHS matrix is column major.
+    bool lhs_column_major;
+
+    // True if the LHS contraction dimension is not 1.
+    bool lhs_non_canonical;
+
+    // True if the RHS matrix is column major.
+    bool rhs_column_major;
+
+    // True if the RHS contraction dimension is not 0.
+    bool rhs_non_canonical;
+
+    // True if the result matrix is column major.
+    bool target_column_major;
+  };
+
+  // Get the MatMultDims instance for the dot product this DotOpEmitter
+  // represents.  Precondition: the dot is of rank 2 (and thus its operands are
+  // of rank 2 as well).
+  MatMultDims GetMatMultDims() const;
+
+  // Lowers the dot operation as a tiled Matrix*Vector loop.
+  void EmitTiledLlvmIrGemv();
+
+  // Lowers the dot operation as a tiled Matrix*Matrix loop.
+  void EmitTiledLlvmIrGemm();
+
+  // Lowers the dot operation as a naive nested loop that computes the result
+  // one element at a time.
+  void EmitNaiveLlvmIrGemm();
+
+  // When doing a tiled GEMV in LLVM IR, a "tile" consists of this many vector
+  // registers.
+  int64 GetGemvTilingFactor() const {
+    const int64 kDefaultTilingFactor = 8;
+    return options::LlvmIrGemvTilingFactor(hlo_module_config_)
+        .value_or(kDefaultTilingFactor);
+  }
+
+  std::tuple<int64, int64, int64> GetGemmTileSize() const {
+    // Tuned for broadwell - Intel(R) Xeon(R) CPU E5-2690 v4 @ 2.60GHz
+    //
+    // TODO(b/80093688): Tune for other architectures and centralize this
+    // information in one place.
+    const std::tuple<int64, int64, int64> kDefaultTileSize =
+        std::tuple<int64, int64, int64>(11, 9, 1);
+    return options::LlvmIrGemmTileSize(hlo_module_config_)
+        .value_or(kDefaultTileSize);
+  }
+
+  DotInfo dot_info_;
+  string dot_hlo_name_;
+  const llvm_ir::IrArray& target_array_;
+  const llvm_ir::IrArray& lhs_array_;
+  const llvm_ir::IrArray& rhs_array_;
+  const llvm_ir::IrArray* addend_array_;
+  llvm::Value* executable_run_options_value_;
+  llvm::IRBuilder<>* b_;
+  const HloModuleConfig& hlo_module_config_;
+  const TargetMachineFeatures& target_machine_features_;
+};
 }  // namespace
 
-DotOpEmitter::DotOpEmitter(const HloInstruction& dot,
+DotOpEmitter::DotOpEmitter(DotInfo dot_info, string dot_hlo_name,
                            const llvm_ir::IrArray& target_array,
                            const llvm_ir::IrArray& lhs_array,
                            const llvm_ir::IrArray& rhs_array,
@@ -64,7 +162,8 @@ DotOpEmitter::DotOpEmitter(const HloInstruction& dot,
                            llvm::IRBuilder<>* b,
                            const HloModuleConfig& hlo_module_config,
                            const TargetMachineFeatures& target_machine_features)
-    : dot_(dot),
+    : dot_info_(std::move(dot_info)),
+      dot_hlo_name_(std::move(dot_hlo_name)),
       target_array_(target_array),
       lhs_array_(lhs_array),
       rhs_array_(rhs_array),
@@ -74,23 +173,8 @@ DotOpEmitter::DotOpEmitter(const HloInstruction& dot,
       hlo_module_config_(hlo_module_config),
       target_machine_features_(target_machine_features) {}
 
-/* static */ Status DotOpEmitter::EmitDotOperation(
-    const HloInstruction& dot, const llvm_ir::IrArray& target_array,
-    const llvm_ir::IrArray& lhs_array, const llvm_ir::IrArray& rhs_array,
-    const llvm_ir::IrArray* addend_array,
-    llvm::Value* executable_run_options_value, llvm::IRBuilder<>* b,
-    const HloModuleConfig& hlo_module_config,
-    const TargetMachineFeatures& target_machine_features) {
-  PrimitiveType type = target_array.GetShape().element_type();
-  TF_RET_CHECK(F16 == type || F32 == type || F64 == type || C64 == type);
-  DotOpEmitter dot_emitter(dot, target_array, lhs_array, rhs_array,
-                           addend_array, executable_run_options_value, b,
-                           hlo_module_config, target_machine_features);
-  return dot_emitter.Emit();
-}
-
 void DotOpEmitter::EmitTiledLlvmIrGemm() {
-  PrimitiveType primitive_type = dot_.shape().element_type();
+  PrimitiveType primitive_type = dot_info_.result_shape.element_type();
   MatMultDims mat_mult_dims = GetMatMultDims();
 
   llvm::Value* lhs = lhs_array_.GetBasePointer();
@@ -136,7 +220,7 @@ void DotOpEmitter::EmitTiledLlvmIrGemm() {
 }
 
 void DotOpEmitter::EmitTiledLlvmIrGemv() {
-  PrimitiveType primitive_type = dot_.shape().element_type();
+  PrimitiveType primitive_type = dot_info_.result_shape.element_type();
 
   CHECK(primitive_util::IsFloatingPointType(primitive_type) ||
         primitive_util::IsIntegralType(primitive_type));
@@ -258,11 +342,6 @@ Status DotOpEmitter::Emit() {
   // which performs the sum-of-products (the reduction loop) before storing
   // the result in the output buffer.
 
-  // This routine assumes that the dot operation is not in a parallelized
-  // enclosing computation.
-  CHECK(
-      dot_.parent()->root_instruction()->outer_dimension_partitions().empty());
-
   const Shape& lhs_shape = lhs_array_.GetShape();
   const Shape& rhs_shape = rhs_array_.GetShape();
 
@@ -273,7 +352,7 @@ Status DotOpEmitter::Emit() {
     return EmitScalarDot();
   }
 
-  switch (GetDotImplementationStrategy(hlo_module_config_, DotInfo(dot_),
+  switch (GetDotImplementationStrategy(hlo_module_config_, dot_info_,
                                        target_machine_features_)) {
     case DotImplementationStrategy::kNaiveLlvmIr:
       EmitNaiveLlvmIrGemm();
@@ -297,7 +376,7 @@ void DotOpEmitter::EmitNaiveLlvmIrGemm() {
 
   const Shape& lhs_shape = lhs_array_.GetShape();
   const Shape& rhs_shape = rhs_array_.GetShape();
-  const DotDimensionNumbers& dim_nums = dot_.dot_dimension_numbers();
+  const DotDimensionNumbers& dim_nums = dot_info_.dim_nums;
 
   // Reduce along dimension 0 of the LHS and 1 of the RHS. Vectors are a special
   // case where the reduction dimension is 0 for both LHS and RHS. This results
@@ -317,7 +396,7 @@ void DotOpEmitter::EmitNaiveLlvmIrGemm() {
   // Create loop nests which loop through the LHS operand dimensions and the RHS
   // operand dimensions. The reduction dimension of the LHS and RHS are handled
   // in a separate innermost loop which performs the sum of products.
-  llvm_ir::ForLoopNest loop_nest(llvm_ir::IrName(&dot_), b_);
+  llvm_ir::ForLoopNest loop_nest(llvm_ir::IrName(dot_hlo_name_), b_);
   llvm_ir::IrArray::Index lhs_index = loop_nest.EmitOperandArrayLoopNest(
       lhs_array_, /*dimension_to_skip=*/lhs_reduction_dimension, "lhs");
   llvm_ir::IrArray::Index rhs_index = loop_nest.EmitOperandArrayLoopNest(
@@ -561,11 +640,11 @@ Status DotOpEmitter::EmitCallToRuntime() {
 }
 
 DotOpEmitter::MatMultDims DotOpEmitter::GetMatMultDims() const {
-  CHECK_EQ(dot_.shape().dimensions_size(), 2);
+  CHECK_EQ(dot_info_.result_shape.dimensions_size(), 2);
 
   const Shape& lhs_shape = lhs_array_.GetShape();
   const Shape& rhs_shape = rhs_array_.GetShape();
-  const DotDimensionNumbers& dim_nums = dot_.dot_dimension_numbers();
+  const DotDimensionNumbers& dim_nums = dot_info_.dim_nums;
 
   return {
       /*m=*/lhs_shape.dimensions(1 - dim_nums.lhs_contracting_dimensions(0)),
@@ -754,5 +833,27 @@ bool DotOperandsAndResultMustHaveRowMajorLayout(
   return impl_strategy == DotImplementationStrategy::kTiledLlvmIrGemm ||
          impl_strategy == DotImplementationStrategy::kEigen;
 }
+
+Status EmitDotOperation(const HloInstruction& dot,
+                        const llvm_ir::IrArray& target_array,
+                        const llvm_ir::IrArray& lhs_array,
+                        const llvm_ir::IrArray& rhs_array,
+                        const llvm_ir::IrArray* addend_array,
+                        llvm::Value* executable_run_options_value,
+                        llvm::IRBuilder<>* b,
+                        const HloModuleConfig& hlo_module_config,
+                        const TargetMachineFeatures& target_machine_features) {
+  // This routine assumes that the dot operation is not in a parallelized
+  // enclosing computation.
+  CHECK(dot.parent()->root_instruction()->outer_dimension_partitions().empty());
+
+  PrimitiveType type = target_array.GetShape().element_type();
+  TF_RET_CHECK(F16 == type || F32 == type || F64 == type || C64 == type);
+  DotOpEmitter dot_emitter(DotInfo(dot), dot.name(), target_array, lhs_array,
+                           rhs_array, addend_array,
+                           executable_run_options_value, b, hlo_module_config,
+                           target_machine_features);
+  return dot_emitter.Emit();
+}
 }  // namespace cpu
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.h b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.h
index eaa226a4c2..105bd3005c 100644
--- a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.h
+++ b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.h
@@ -48,118 +48,24 @@ bool DotImplementationCanHandleTranspose(
 absl::optional<int64> ProfitableToMakeDotOperandColumnMajor(
     const HloInstruction& hlo);
 
-// Helper class for emitting LLVM IR to perform the dot operation.
-class DotOpEmitter {
- public:
-  // Emit LLVM IR to perform the dot operation on lhs_array and rhs_array and
-  // place the result in target_array. IR is emitted at current insert point of
-  // the builder. Upon completion of the method, the insert point is set to the
-  // end of all instructions emitted for this operation.
-  //
-  // If `addend_array` is not nullptr then it must be an array of the same
-  // dimensions as the result, and the result is computed as `addend_array` +
-  // dot(`lhs_array`, `rhs_array`).  A non-null `addend_array` is only supported
-  // for Matrix-vector products.
-  static Status EmitDotOperation(
-      const HloInstruction& dot, const llvm_ir::IrArray& target_array,
-      const llvm_ir::IrArray& lhs_array, const llvm_ir::IrArray& rhs_array,
-      const llvm_ir::IrArray* addend_array,
-      llvm::Value* executable_run_options_value, llvm::IRBuilder<>* b,
-      const HloModuleConfig& hlo_module_config,
-      const TargetMachineFeatures& target_machine_features);
-
- private:
-  DotOpEmitter(const HloInstruction& dot, const llvm_ir::IrArray& target_array,
-               const llvm_ir::IrArray& lhs_array,
-               const llvm_ir::IrArray& rhs_array,
-               const llvm_ir::IrArray* addend_array,
-               llvm::Value* executable_run_options_value, llvm::IRBuilder<>* b,
-               const HloModuleConfig& hlo_module_config,
-               const TargetMachineFeatures& target_machine_features);
-
-  // Emits the IR to perform the dot operation.
-  Status Emit();
-
-  // Emits instructions to perform a scalar dot product (a multiply of the
-  // LHS and RHS) and store the results in the target.
-  Status EmitScalarDot();
-
-  // Emits a call to the CPU runtime to perform the matrix multiply.
-  Status EmitCallToRuntime();
-
-  // Represents the dimensions of a matrix-matrix multiply operation.
-  struct MatMultDims {
-    // The number of rows in the LHS.
-    int64 m;
-
-    // The number of columns in the LHS, which is also must be equal to the
-    // number of rows in the RHS.
-    int64 k;
-
-    // The number of columns on the RHS.
-    int64 n;
-
-    // True if the LHS matrix is column major.
-    bool lhs_column_major;
-
-    // True if the LHS contraction dimension is not 1.
-    bool lhs_non_canonical;
-
-    // True if the RHS matrix is column major.
-    bool rhs_column_major;
-
-    // True if the RHS contraction dimension is not 0.
-    bool rhs_non_canonical;
-
-    // True if the result matrix is column major.
-    bool target_column_major;
-  };
-
-  // Get the MatMultDims instance for the dot product this DotOpEmitter
-  // represents.  Precondition: the dot is of rank 2 (and thus its operands are
-  // of rank 2 as well).
-  MatMultDims GetMatMultDims() const;
-
-  // Lowers the dot operation as a tiled Matrix*Vector loop.
-  void EmitTiledLlvmIrGemv();
-
-  // Lowers the dot operation as a tiled Matrix*Matrix loop.
-  void EmitTiledLlvmIrGemm();
-
-  // Lowers the dot operation as a naive nested loop that computes the result
-  // one element at a time.
-  void EmitNaiveLlvmIrGemm();
-
-  // When doing a tiled GEMV in LLVM IR, a "tile" consists of this many vector
-  // registers.
-  int64 GetGemvTilingFactor() const {
-    const int64 kDefaultTilingFactor = 8;
-    return options::LlvmIrGemvTilingFactor(hlo_module_config_)
-        .value_or(kDefaultTilingFactor);
-  }
-
-  std::tuple<int64, int64, int64> GetGemmTileSize() const {
-    // Tuned for broadwell - Intel(R) Xeon(R) CPU E5-2690 v4 @ 2.60GHz
-    //
-    // TODO(b/80093688): Tune for other architectures and centralize this
-    // information in one place.
-    const std::tuple<int64, int64, int64> kDefaultTileSize =
-        std::tuple<int64, int64, int64>(11, 9, 1);
-    return options::LlvmIrGemmTileSize(hlo_module_config_)
-        .value_or(kDefaultTileSize);
-  }
-
-  const HloInstruction& dot_;
-  const llvm_ir::IrArray& target_array_;
-  const llvm_ir::IrArray& lhs_array_;
-  const llvm_ir::IrArray& rhs_array_;
-  const llvm_ir::IrArray* addend_array_;
-  llvm::Value* executable_run_options_value_;
-  llvm::IRBuilder<>* b_;
-  const HloModuleConfig& hlo_module_config_;
-  const TargetMachineFeatures& target_machine_features_;
-};
-
+// Emit LLVM IR to perform the dot operation on lhs_array and rhs_array and
+// place the result in target_array. IR is emitted at current insert point of
+// the builder. Upon completion of the method, the insert point is set to the
+// end of all instructions emitted for this operation.
+//
+// If `addend_array` is not nullptr then it must be an array of the same
+// dimensions as the result, and the result is computed as `addend_array` +
+// dot(`lhs_array`, `rhs_array`).  A non-null `addend_array` is only supported
+// for Matrix-vector products.
+Status EmitDotOperation(const HloInstruction& dot,
+                        const llvm_ir::IrArray& target_array,
+                        const llvm_ir::IrArray& lhs_array,
+                        const llvm_ir::IrArray& rhs_array,
+                        const llvm_ir::IrArray* addend_array,
+                        llvm::Value* executable_run_options_value,
+                        llvm::IRBuilder<>* b,
+                        const HloModuleConfig& hlo_module_config,
+                        const TargetMachineFeatures& target_machine_features);
 }  // namespace cpu
 }  // namespace xla
 
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
index b352848824..91e3693354 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
@@ -970,10 +970,10 @@ Status IrEmitter::HandleDot(HloInstruction* dot) {
           << llvm_ir::DumpToString(*target_array.GetBasePointer());
 
   // Dot operation is complicated so we delegate to a helper class.
-  return DotOpEmitter::EmitDotOperation(
-      *dot, target_array, lhs_array, rhs_array, /*addend_array=*/nullptr,
-      GetExecutableRunOptionsArgument(), &b_, hlo_module_config_,
-      target_machine_features_);
+  return EmitDotOperation(*dot, target_array, lhs_array, rhs_array,
+                          /*addend_array=*/nullptr,
+                          GetExecutableRunOptionsArgument(), &b_,
+                          hlo_module_config_, target_machine_features_);
 }
 
 StatusOr<llvm::Value*> IrEmitter::EmitTargetElementLoopBodyForConvolution(
@@ -2203,10 +2203,10 @@ Status IrEmitter::HandleFusion(HloInstruction* fusion) {
     llvm_ir::IrArray addend_array(
         GetIrArrayFor(fusion->operand(addend_param_number)));
 
-    TF_RETURN_IF_ERROR(DotOpEmitter::EmitDotOperation(
-        *dot, target_array, lhs_array, rhs_array, &addend_array,
-        GetExecutableRunOptionsArgument(), &b_, hlo_module_config_,
-        target_machine_features_));
+    TF_RETURN_IF_ERROR(
+        EmitDotOperation(*dot, target_array, lhs_array, rhs_array,
+                         &addend_array, GetExecutableRunOptionsArgument(), &b_,
+                         hlo_module_config_, target_machine_features_));
     return Status::OK();
   } else {
     return Unimplemented("Fusion kind not implemented on CPU");
-- 
GitLab


From 2ca0bf7cf4e16cbed6affd7dd844fea4ed3a62df Mon Sep 17 00:00:00 2001
From: Andiry Xu <andiryxu@google.com>
Date: Mon, 7 Jan 2019 22:40:38 -0800
Subject: [PATCH 0357/1095] This CL checks StepStats, records the output slot
 for each Switch Op execution, and adds the output slot vector to GraphDef
 Switch node. When VirtualScheduler schedules the Switch Op, it will take the
 actual output branch.

PiperOrigin-RevId: 228285499
---
 .../core/grappler/costs/virtual_scheduler.cc  | 113 +++-
 .../core/grappler/costs/virtual_scheduler.h   |  12 +
 .../grappler/costs/virtual_scheduler_test.cc  | 516 ++++++++++++++++++
 3 files changed, 623 insertions(+), 18 deletions(-)

diff --git a/tensorflow/core/grappler/costs/virtual_scheduler.cc b/tensorflow/core/grappler/costs/virtual_scheduler.cc
index 7071c679dc..bb8425a5a1 100644
--- a/tensorflow/core/grappler/costs/virtual_scheduler.cc
+++ b/tensorflow/core/grappler/costs/virtual_scheduler.cc
@@ -15,8 +15,6 @@ limitations under the License.
 
 #include "tensorflow/core/grappler/costs/virtual_scheduler.h"
 
-#include <math.h>
-
 #include "tensorflow/core/framework/allocation_description.pb.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/node_def.pb.h"
@@ -38,6 +36,12 @@ namespace tensorflow {
 namespace grappler {
 namespace {
 
+// Optional attribute name for Switch op as a vector of int that tells
+// which branch the Switch output is taken on every round of execution.
+// We use this side information, if provided, for scheduling ops after Switch
+// correctly (e.g., While loop).
+constexpr char kOutputSlots[] = "_output_slot_vector";
+
 Costs CombineCosts(const Costs& left, const Costs& right) {
   CHECK_NE(left.max_memory, kMemoryUnknown);
   CHECK_NE(left.max_per_op_buffers, kMemoryUnknown);
@@ -402,6 +406,8 @@ Status VirtualScheduler::Init() {
     name_to_node[node->name()] = node;
   }
 
+  // Traverse the graph to check if the graph is annotated with Switch outputs.
+  // Also record _Send nodes.
   // TODO(dyoon): Instead of identifying _Send node here manually, add _Send
   // to _Recv as control dependency when creating GrapplerItem.
   std::unordered_map<string, const NodeDef*> name_to_send;
@@ -410,6 +416,11 @@ Status VirtualScheduler::Init() {
       const auto& attr = node.attr();
       name_to_send[attr.at("tensor_name").s()] = &node;
     }
+
+    if (IsSwitch(node)) {
+      const auto& attr = node.attr();
+      if (attr.count(kOutputSlots) > 0) switch_outputs_annotated_ = true;
+    }
   }
 
   // To reuse _Recv ops.
@@ -771,6 +782,82 @@ Costs& VirtualScheduler::FindOrCreateZero(const string& op_name,
   return it->second;
 }
 
+// Check Switch outputs in updated MetaGraphDef, add corresponding nodes to
+// ready queue.
+// Fallback to add all outputs if fail to find the actual output.
+bool VirtualScheduler::AddSwitchOutputsToReadyQueue(
+    const NodeDef* node, int curr_iter, const Costs::Duration& curr_time) {
+  if (node->attr().count(kOutputSlots) == 0) return false;
+
+  auto& node_state = node_map_[node];
+  const auto& slot_vector = node->attr().at(kOutputSlots);
+  if (slot_vector.list().i_size() <= curr_iter) {
+    // Sometimes we encounter infinite loop. Fall back to add all outputs.
+    return false;
+  }
+
+  int slot = slot_vector.list().i(curr_iter);
+  for (const auto& port_num_output_pair : node_state.outputs) {
+    if (port_num_output_pair.first != slot) continue;
+
+    for (auto* output_node : port_num_output_pair.second) {
+      auto& output_state = node_map_[output_node];
+      output_state.num_inputs_ready++;
+      // Execute a node as soon as all its inputs are ready. Merge nodes
+      // are special since they run as soon as one of their inputs becomes
+      // available.
+      if (output_state.num_inputs_ready == output_state.inputs.size() ||
+          IsMerge(*output_node)) {
+        // This output node is now ready.
+        output_state.time_ready = curr_time;
+        ready_nodes_->AddNode(output_node);
+        VLOG(3) << "Node " << node->name() << " iter " << curr_iter << "/"
+                << slot_vector.list().i_size() << " Add Switch output " << slot
+                << ": " << output_node->name();
+      }
+    }
+    return true;
+  }
+
+  return false;
+}
+
+void VirtualScheduler::AddOutputNodesToReadyQueue(
+    const NodeDef* node, const Costs::Duration& curr_time) {
+  auto& node_state = node_map_[node];
+  int curr_iter = node_state.num_executed_times;
+  ++node_state.num_executed_times;
+
+  if (switch_outputs_annotated_) {
+    // If the graph is annotated with StepStats, reset num_inputs_ready so we
+    // can schedule the node multiple times.
+    node_state.num_inputs_ready = 0;
+
+    // For Switch node, get output branch from updated MetaGraphDef.
+    if (IsSwitch(*node) &&
+        AddSwitchOutputsToReadyQueue(node, curr_iter, curr_time))
+      return;
+  }
+
+  // Increment num_inputs_ready of the output nodes and maybe add to ready
+  // nodes.
+  for (const auto& port_num_output_pair : node_state.outputs) {
+    for (auto* output_node : port_num_output_pair.second) {
+      auto& output_state = node_map_[output_node];
+      output_state.num_inputs_ready++;
+      // Execute a node as soon as all its inputs are ready. Merge nodes are
+      // special since they run as soon as one of their inputs becomes
+      // available.
+      if (output_state.num_inputs_ready == output_state.inputs.size() ||
+          IsMerge(*output_node)) {
+        // This output node is now ready.
+        output_state.time_ready = curr_time;
+        ready_nodes_->AddNode(output_node);
+      }
+    }
+  }
+}
+
 bool VirtualScheduler::MarkCurrNodeExecuted(const Costs& node_costs) {
   // Update graph_costs_ and per-op costs.
   graph_costs_ = CombineCosts(graph_costs_, node_costs);
@@ -798,6 +885,10 @@ bool VirtualScheduler::MarkCurrNodeExecuted(const Costs& node_costs) {
   // Node is scheduled when the device is available AND all the inputs are
   // ready; hence, time_scheduled is time_ready if time_ready > device curr
   // time.
+  // TODO(andiryxu): Current node_state result only records the last execution.
+  // With annotated MetaGraph we can schedule a node for multiple times.
+  // Refine NodeState structure accordingly, e.g. record time_scheduled in a
+  // vector.
   node_state.time_scheduled =
       std::max(device.GetCurrTime(), node_state.time_ready);
   // Override device curr time with the time_scheduled.
@@ -831,22 +922,8 @@ bool VirtualScheduler::MarkCurrNodeExecuted(const Costs& node_costs) {
           << ", scheduled: " << node_state.time_scheduled.count()
           << ", finished: " << node_state.time_finished.count();
 
-  // Increment num_inputs_ready of the output nodes and maybe add to ready nodes
-  for (const auto& port_num_output_pair : node_state.outputs) {
-    for (auto* output_node : port_num_output_pair.second) {
-      auto& output_state = node_map_[output_node];
-      output_state.num_inputs_ready++;
-      // Execute a node as soon as all its inputs are ready. Merge nodes are
-      // special since they run as soon as one of their inputs becomes
-      // available.
-      if (output_state.num_inputs_ready == output_state.inputs.size() ||
-          IsMerge(*output_node)) {
-        // This output node is now ready.
-        output_state.time_ready = curr_time;
-        ready_nodes_->AddNode(output_node);
-      }
-    }
-  }
+  // Check outputs, add ready nodes to queue.
+  AddOutputNodesToReadyQueue(node, curr_time);
 
   // Increment num_outputs_executed of the input nodes and maybe update memory.
   for (const auto& input_port : node_state.inputs) {
diff --git a/tensorflow/core/grappler/costs/virtual_scheduler.h b/tensorflow/core/grappler/costs/virtual_scheduler.h
index f500e4ab51..ffc5e9187c 100644
--- a/tensorflow/core/grappler/costs/virtual_scheduler.h
+++ b/tensorflow/core/grappler/costs/virtual_scheduler.h
@@ -70,11 +70,15 @@ struct NodeState {
   // Each output port uses up memory space from time_scheduled to its
   // time_no_references.
 
+  // How many times this node has been executed, e.g. in a while loop.
+  int num_executed_times;
+
   NodeState() {
     num_inputs_ready = 0;
     time_ready = Costs::Duration::max();
     time_scheduled = Costs::Duration::max();
     time_finished = Costs::Duration::max();
+    num_executed_times = 0;
     // Note that num_outputs_executed and time_no_references are not initialized
     // here, since we don't know the size (i.e., # outputs for this node).
   }
@@ -330,6 +334,10 @@ class VirtualScheduler {
                           std::map<string, Costs>* op_cost);
   float Round2(const float x) const;
   bool IsPersistentNode(const NodeDef* node) const;
+  bool AddSwitchOutputsToReadyQueue(const NodeDef* node, int curr_iter,
+                                    const Costs::Duration& curr_time);
+  void AddOutputNodesToReadyQueue(const NodeDef* node,
+                                  const Costs::Duration& curr_time);
 
   // Scheduler states:
   ReadyNodeManager* ready_nodes_;  // Not owned.
@@ -360,6 +368,10 @@ class VirtualScheduler {
   bool initialized_;
   bool track_mem_usage_snapshot_;
 
+  // Whether the input graph includes Switch nodes annotated with output slots
+  // information.
+  bool switch_outputs_annotated_ = false;
+
   VirtualPlacer placer_;  // owned.
 };
 
diff --git a/tensorflow/core/grappler/costs/virtual_scheduler_test.cc b/tensorflow/core/grappler/costs/virtual_scheduler_test.cc
index c5651715f4..5e044b2037 100644
--- a/tensorflow/core/grappler/costs/virtual_scheduler_test.cc
+++ b/tensorflow/core/grappler/costs/virtual_scheduler_test.cc
@@ -869,6 +869,439 @@ versions {
     grappler_item_->fetch = {"while/Exit", "while/Exit_1"};
   }
 
+  // A simple while loop strengthened with Switch outputs.
+  void CreateGrapplerItemWithLoopSwitchOutputs() {
+    // Test graph produced in python using:
+    /*
+      with tf.Graph().as_default():
+      i0 = tf.constant(0)
+      m0 = tf.ones([2, 2])
+      c = lambda i, m: i < 10
+      b = lambda i, m: [i+1, tf.concat([m, m], axis=0)]
+      r = tf.while_loop(
+      c, b, loop_vars=[i0, m0],
+      shape_invariants=[i0.get_shape(), tf.TensorShape([None, 2])])
+      with open('/tmp/graph.pbtxt', 'w') as f:
+      f.write(str(tf.get_default_graph().as_graph_def()))
+    */
+    const string gdef_ascii = R"EOF(
+node {
+  name: "Const"
+  op: "Const"
+  attr {
+    key: "dtype"
+    value {
+      type: DT_INT32
+    }
+  }
+  attr {
+    key: "value"
+    value {
+      tensor {
+        dtype: DT_INT32
+        tensor_shape {
+        }
+        int_val: 0
+      }
+    }
+  }
+}
+node {
+  name: "ones"
+  op: "Const"
+  attr {
+    key: "dtype"
+    value {
+      type: DT_FLOAT
+    }
+  }
+  attr {
+    key: "value"
+    value {
+      tensor {
+        dtype: DT_FLOAT
+        tensor_shape {
+          dim {
+            size: 2
+          }
+          dim {
+            size: 2
+          }
+        }
+        float_val: 1.0
+      }
+    }
+  }
+}
+node {
+  name: "while/Enter"
+  op: "Enter"
+  input: "Const"
+  attr {
+    key: "T"
+    value {
+      type: DT_INT32
+    }
+  }
+  attr {
+    key: "frame_name"
+    value {
+      s: "while/while/"
+    }
+  }
+  attr {
+    key: "is_constant"
+    value {
+      b: false
+    }
+  }
+  attr {
+    key: "parallel_iterations"
+    value {
+      i: 10
+    }
+  }
+}
+node {
+  name: "while/Enter_1"
+  op: "Enter"
+  input: "ones"
+  attr {
+    key: "T"
+    value {
+      type: DT_FLOAT
+    }
+  }
+  attr {
+    key: "frame_name"
+    value {
+      s: "while/while/"
+    }
+  }
+  attr {
+    key: "is_constant"
+    value {
+      b: false
+    }
+  }
+  attr {
+    key: "parallel_iterations"
+    value {
+      i: 10
+    }
+  }
+}
+node {
+  name: "while/Merge"
+  op: "Merge"
+  input: "while/Enter"
+  input: "while/NextIteration"
+  attr {
+    key: "N"
+    value {
+      i: 2
+    }
+  }
+  attr {
+    key: "T"
+    value {
+      type: DT_INT32
+    }
+  }
+}
+node {
+  name: "while/Merge_1"
+  op: "Merge"
+  input: "while/Enter_1"
+  input: "while/NextIteration_1"
+  attr {
+    key: "N"
+    value {
+      i: 2
+    }
+  }
+  attr {
+    key: "T"
+    value {
+      type: DT_FLOAT
+    }
+  }
+}
+node {
+  name: "while/Less/y"
+  op: "Const"
+  input: "^while/Merge"
+  attr {
+    key: "dtype"
+    value {
+      type: DT_INT32
+    }
+  }
+  attr {
+    key: "value"
+    value {
+      tensor {
+        dtype: DT_INT32
+        tensor_shape {
+        }
+        int_val: 10
+      }
+    }
+  }
+}
+node {
+  name: "while/Less"
+  op: "Less"
+  input: "while/Merge"
+  input: "while/Less/y"
+  attr {
+    key: "T"
+    value {
+      type: DT_INT32
+    }
+  }
+}
+node {
+  name: "while/LoopCond"
+  op: "LoopCond"
+  input: "while/Less"
+}
+node {
+  name: "while/Switch"
+  op: "Switch"
+  input: "while/Merge"
+  input: "while/LoopCond"
+  attr {
+    key: "T"
+    value {
+      type: DT_INT32
+    }
+  }
+  attr {
+    key: "_class"
+    value {
+      list {
+        s: "loc:@while/Merge"
+      }
+    }
+  }
+  attr {
+    key: "_output_slot_vector"
+    value {
+      list {
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+        i: 0
+      }
+    }
+  }
+}
+node {
+  name: "while/Switch_1"
+  op: "Switch"
+  input: "while/Merge_1"
+  input: "while/LoopCond"
+  attr {
+    key: "T"
+    value {
+      type: DT_FLOAT
+    }
+  }
+  attr {
+    key: "_class"
+    value {
+      list {
+        s: "loc:@while/Merge_1"
+      }
+    }
+  }
+  attr {
+    key: "_output_slot_vector"
+    value {
+      list {
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+        i: 0
+      }
+    }
+  }
+}
+node {
+  name: "while/Identity"
+  op: "Identity"
+  input: "while/Switch:1"
+  attr {
+    key: "T"
+    value {
+      type: DT_INT32
+    }
+  }
+}
+node {
+  name: "while/Identity_1"
+  op: "Identity"
+  input: "while/Switch_1:1"
+  attr {
+    key: "T"
+    value {
+      type: DT_FLOAT
+    }
+  }
+}
+node {
+  name: "while/add/y"
+  op: "Const"
+  input: "^while/Identity"
+  attr {
+    key: "dtype"
+    value {
+      type: DT_INT32
+    }
+  }
+  attr {
+    key: "value"
+    value {
+      tensor {
+        dtype: DT_INT32
+        tensor_shape {
+        }
+        int_val: 1
+      }
+    }
+  }
+}
+node {
+  name: "while/add"
+  op: "Add"
+  input: "while/Identity"
+  input: "while/add/y"
+  attr {
+    key: "T"
+    value {
+      type: DT_INT32
+    }
+  }
+}
+node {
+  name: "while/concat/axis"
+  op: "Const"
+  input: "^while/Identity"
+  attr {
+    key: "dtype"
+    value {
+      type: DT_INT32
+    }
+  }
+  attr {
+    key: "value"
+    value {
+      tensor {
+        dtype: DT_INT32
+        tensor_shape {
+        }
+        int_val: 0
+      }
+    }
+  }
+}
+node {
+  name: "while/concat"
+  op: "ConcatV2"
+  input: "while/Identity_1"
+  input: "while/Identity_1"
+  input: "while/concat/axis"
+  attr {
+    key: "N"
+    value {
+      i: 2
+    }
+  }
+  attr {
+    key: "T"
+    value {
+      type: DT_FLOAT
+    }
+  }
+  attr {
+    key: "Tidx"
+    value {
+      type: DT_INT32
+    }
+  }
+}
+node {
+  name: "while/NextIteration"
+  op: "NextIteration"
+  input: "while/add"
+  attr {
+    key: "T"
+    value {
+      type: DT_INT32
+    }
+  }
+}
+node {
+  name: "while/NextIteration_1"
+  op: "NextIteration"
+  input: "while/concat"
+  attr {
+    key: "T"
+    value {
+      type: DT_FLOAT
+    }
+  }
+}
+node {
+  name: "while/Exit"
+  op: "Exit"
+  input: "while/Switch"
+  attr {
+    key: "T"
+    value {
+      type: DT_INT32
+    }
+  }
+}
+node {
+  name: "while/Exit_1"
+  op: "Exit"
+  input: "while/Switch_1"
+  attr {
+    key: "T"
+    value {
+      type: DT_FLOAT
+    }
+  }
+}
+versions {
+  producer: 21
+}
+  )EOF";
+
+    grappler_item_.reset(new GrapplerItem);
+    CHECK(protobuf::TextFormat::ParseFromString(gdef_ascii,
+                                                &grappler_item_->graph));
+    grappler_item_->id = "test_graph";
+    grappler_item_->fetch = {"while/Exit", "while/Exit_1"};
+  }
+
+  // Create a FusedBatchNorm op that has multiple output ports.
   void CreateGrapplerItemWithInterDeviceTransfers() {
     tensorflow::Scope s = tensorflow::Scope::NewRootScope().WithDevice(kCPU0);
 
@@ -1942,6 +2375,89 @@ TEST_F(VirtualSchedulerTest, WhileLoop) {
   ValidateDependencyChain(start_times, {"while/Switch_1", "while/Exit_1"});
 }
 
+TEST_F(VirtualSchedulerTest, WhileLoopWithSwitchOutputs) {
+  // Init.
+  CreateGrapplerItemWithLoopSwitchOutputs();
+  InitScheduler();
+
+  // Runs the scheduler.
+  RunScheduler("");
+
+  RunMetadata metadata;
+  scheduler_->Summary(&metadata);
+
+  // Nodes in topological order:
+  // * const, ones
+  // * while/Enter, while/Enter_1
+  // * while/Merge, while/Merge_1
+  // * while/Less/y
+  // * while/Less
+  // * while/LoopCond
+  // * while/Switch, while/Switch_1
+  // * while/Identity, while/Identity_1, while/Exit, while/Exit_1
+  // * while/add/y, while/concat/axis
+  // * while/add, while/concat
+  // * while/NextIteration, while/NextIteration_1
+
+  int num_next_iteration = 0;
+  int num_next_iteration_1 = 0;
+  int num_exit = 0;
+  int num_exit_1 = 0;
+  int64 next_iter_start_micro;
+  int64 next_iter_1_start_micro;
+  int64 exit_start_micro;
+  int64 exit_1_start_micro;
+
+  std::unordered_map<string, int64> start_times;
+  for (const auto& device_step_stats : metadata.step_stats().dev_stats()) {
+    for (const auto& stats : device_step_stats.node_stats()) {
+      start_times[stats.node_name()] = stats.all_start_micros();
+      if (stats.node_name() == "while/NextIteration") {
+        ++num_next_iteration;
+        next_iter_start_micro = stats.all_start_micros();
+      } else if (stats.node_name() == "while/NextIteration_1") {
+        ++num_next_iteration_1;
+        next_iter_1_start_micro = stats.all_start_micros();
+      } else if (stats.node_name() == "while/Exit") {
+        ++num_exit;
+        exit_start_micro = stats.all_start_micros();
+      } else if (stats.node_name() == "while/Exit_1") {
+        ++num_exit_1;
+        exit_1_start_micro = stats.all_start_micros();
+      }
+    }
+  }
+
+  // Makes sure we run the loop body for ten times.
+  EXPECT_EQ(10, num_next_iteration);
+  EXPECT_EQ(10, num_next_iteration_1);
+  EXPECT_EQ(1, num_exit);
+  EXPECT_EQ(1, num_exit_1);
+
+  // Start times of while/NextIteration and while/NextIteration_1 should be
+  // different, so should be those of while/Exit and while/Exit_1.
+  EXPECT_NE(next_iter_start_micro, next_iter_1_start_micro);
+  EXPECT_NE(exit_start_micro, exit_1_start_micro);
+
+  // Checks dependency among the nodes; no matter what scheduling mechanism we
+  // use, the scheduled ops should follow these dependency chains.
+  // We have to break the loop into two parts, identified by Switch outputs.
+  ValidateDependencyChain(
+      start_times,
+      {"Const", "while/Enter", "while/Merge", "while/Less/y", "while/Less",
+       "while/LoopCond", "while/Switch", "while/Exit"});
+  ValidateDependencyChain(start_times, {"while/Identity", "while/add/y",
+                                        "while/add", "while/NextIteration"});
+  ValidateDependencyChain(
+      start_times, {"ones", "while/Enter_1", "while/Merge_1", "while/Switch_1",
+                    "while/Exit_1"});
+  ValidateDependencyChain(start_times, {"while/Identity_1", "while/concat",
+                                        "while/NextIteration_1"});
+  ValidateDependencyChain(
+      start_times, {"while/Identity", "while/concat/axis", "while/concat"});
+  ValidateDependencyChain(start_times, {"while/Identity", "while/add"});
+}
+
 TEST_F(VirtualSchedulerTest, InterDeviceTransfer) {
   // Init.
   CreateGrapplerItemWithInterDeviceTransfers();
-- 
GitLab


From 94c604bdd72717b0c18a1648927529b27aeea320 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 8 Jan 2019 00:15:37 -0800
Subject: [PATCH 0358/1095] Documentation fix for the unsorted_segment_max op
 RELNOTES: n/a

PiperOrigin-RevId: 228290885
---
 .../core/api_def/base_api/api_def_UnsortedSegmentMax.pbtxt     | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/api_def/base_api/api_def_UnsortedSegmentMax.pbtxt b/tensorflow/core/api_def/base_api/api_def_UnsortedSegmentMax.pbtxt
index 7a60e4387a..ed4a2bd558 100644
--- a/tensorflow/core/api_def/base_api/api_def_UnsortedSegmentMax.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_UnsortedSegmentMax.pbtxt
@@ -3,7 +3,8 @@ op {
   in_arg {
     name: "segment_ids"
     description: <<END
-A tensor whose shape is a prefix of `data.shape`.END
+A tensor whose shape is a prefix of `data.shape`.
+END
   }
   out_arg {
     name: "output"
-- 
GitLab


From b504d86ca21f9ce57700d017ab8f6d411ca5afde Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 8 Jan 2019 01:03:21 -0800
Subject: [PATCH 0359/1095] compat: Update forward compatibility horizon to
 2019-01-08

PiperOrigin-RevId: 228294335
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index dbad494e40..5008741d55 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 7)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 8)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 84cf1da2b9552f0af564519e8b200ac8859a17b4 Mon Sep 17 00:00:00 2001
From: James Keeling <jtkeeling@google.com>
Date: Tue, 8 Jan 2019 01:34:31 -0800
Subject: [PATCH 0360/1095] Split tile functor GPU code into multiple files

This file was a bottleneck during compilation, often taking many minutes to compile. In local testing this change reduces the wall-clock build time for the tile functor GPU kernels from 217s to 71s.

PiperOrigin-RevId: 228296771
---
 tensorflow/core/kernels/BUILD                 | 11 ++++++-
 ...e_functor_gpu.cu.cc => tile_functor_gpu.h} | 28 +++--------------
 .../core/kernels/tile_functor_gpu_bool.cu.cc  | 31 +++++++++++++++++++
 .../kernels/tile_functor_gpu_complex128.cu.cc | 31 +++++++++++++++++++
 .../kernels/tile_functor_gpu_complex64.cu.cc  | 31 +++++++++++++++++++
 .../kernels/tile_functor_gpu_double.cu.cc     | 31 +++++++++++++++++++
 .../core/kernels/tile_functor_gpu_float.cu.cc | 31 +++++++++++++++++++
 .../core/kernels/tile_functor_gpu_half.cu.cc  | 31 +++++++++++++++++++
 .../core/kernels/tile_functor_gpu_int16.cu.cc | 31 +++++++++++++++++++
 .../core/kernels/tile_functor_gpu_int32.cu.cc | 31 +++++++++++++++++++
 .../core/kernels/tile_functor_gpu_int64.cu.cc | 31 +++++++++++++++++++
 11 files changed, 294 insertions(+), 24 deletions(-)
 rename tensorflow/core/kernels/{tile_functor_gpu.cu.cc => tile_functor_gpu.h} (85%)
 create mode 100644 tensorflow/core/kernels/tile_functor_gpu_bool.cu.cc
 create mode 100644 tensorflow/core/kernels/tile_functor_gpu_complex128.cu.cc
 create mode 100644 tensorflow/core/kernels/tile_functor_gpu_complex64.cu.cc
 create mode 100644 tensorflow/core/kernels/tile_functor_gpu_double.cu.cc
 create mode 100644 tensorflow/core/kernels/tile_functor_gpu_float.cu.cc
 create mode 100644 tensorflow/core/kernels/tile_functor_gpu_half.cu.cc
 create mode 100644 tensorflow/core/kernels/tile_functor_gpu_int16.cu.cc
 create mode 100644 tensorflow/core/kernels/tile_functor_gpu_int32.cu.cc
 create mode 100644 tensorflow/core/kernels/tile_functor_gpu_int64.cu.cc

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 701bda7ef6..fee30fa6c3 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -1012,7 +1012,16 @@ tf_kernel_library(
     hdrs = ["tile_functor.h"],
     gpu_srcs = [
         "tile_functor.h",
-        "tile_functor_gpu.cu.cc",
+        "tile_functor_gpu.h",
+        "tile_functor_gpu_bool.cu.cc",
+        "tile_functor_gpu_complex64.cu.cc",
+        "tile_functor_gpu_complex128.cu.cc",
+        "tile_functor_gpu_double.cu.cc",
+        "tile_functor_gpu_float.cu.cc",
+        "tile_functor_gpu_half.cu.cc",
+        "tile_functor_gpu_int16.cu.cc",
+        "tile_functor_gpu_int32.cu.cc",
+        "tile_functor_gpu_int64.cu.cc",
     ],
     prefix = "tile_ops",
     deps = ARRAY_DEPS,
diff --git a/tensorflow/core/kernels/tile_functor_gpu.cu.cc b/tensorflow/core/kernels/tile_functor_gpu.h
similarity index 85%
rename from tensorflow/core/kernels/tile_functor_gpu.cu.cc
rename to tensorflow/core/kernels/tile_functor_gpu.h
index 84a5060fc3..0de32e730e 100644
--- a/tensorflow/core/kernels/tile_functor_gpu.cu.cc
+++ b/tensorflow/core/kernels/tile_functor_gpu.h
@@ -13,6 +13,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#ifndef TENSORFLOW_CORE_KERNELS_TILE_FUNCTOR_GPU_H_
+#define TENSORFLOW_CORE_KERNELS_TILE_FUNCTOR_GPU_H_
+
 #if GOOGLE_CUDA
 
 #define EIGEN_USE_GPU
@@ -80,28 +83,7 @@ void TileSimple(const Device& d, Tensor* out, const Tensor& in) {
 }
 
 }  // end namespace internal
-
-namespace functor {
-
-typedef Eigen::GpuDevice GPUDevice;
-
-// Register functors used for Tile functor.
-#define DEFINE_TYPE(T)                       \
-  template struct Tile<GPUDevice, T, int32>; \
-  template struct Tile<GPUDevice, T, int64>;
-
-TF_CALL_bool(DEFINE_TYPE);
-TF_CALL_int16(DEFINE_TYPE);
-TF_CALL_int32(DEFINE_TYPE);
-TF_CALL_int64(DEFINE_TYPE);
-TF_CALL_float(DEFINE_TYPE);
-TF_CALL_double(DEFINE_TYPE);
-TF_CALL_half(DEFINE_TYPE);
-TF_CALL_complex64(DEFINE_TYPE);
-TF_CALL_complex128(DEFINE_TYPE);
-
-#undef DEFINE_TYPE
-
-}  // end namespace functor
 }  // namespace tensorflow
 #endif  // GOOGLE_CUDA
+
+#endif  // TENSORFLOW_CORE_KERNELS_TILE_FUNCTOR_GPU_H_
diff --git a/tensorflow/core/kernels/tile_functor_gpu_bool.cu.cc b/tensorflow/core/kernels/tile_functor_gpu_bool.cu.cc
new file mode 100644
index 0000000000..c7a814c7a2
--- /dev/null
+++ b/tensorflow/core/kernels/tile_functor_gpu_bool.cu.cc
@@ -0,0 +1,31 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/tile_functor.h"
+#include "tensorflow/core/kernels/tile_functor_gpu.h"
+
+namespace tensorflow {
+namespace functor {
+using Eigen::GpuDevice;
+
+template struct Tile<GpuDevice, bool, int32>;
+template struct Tile<GpuDevice, bool, int64>;
+}  // namespace functor
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/tile_functor_gpu_complex128.cu.cc b/tensorflow/core/kernels/tile_functor_gpu_complex128.cu.cc
new file mode 100644
index 0000000000..4dfa4bac1b
--- /dev/null
+++ b/tensorflow/core/kernels/tile_functor_gpu_complex128.cu.cc
@@ -0,0 +1,31 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/tile_functor.h"
+#include "tensorflow/core/kernels/tile_functor_gpu.h"
+
+namespace tensorflow {
+namespace functor {
+using Eigen::GpuDevice;
+
+template struct Tile<GpuDevice, complex128, int32>;
+template struct Tile<GpuDevice, complex128, int64>;
+}  // namespace functor
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/tile_functor_gpu_complex64.cu.cc b/tensorflow/core/kernels/tile_functor_gpu_complex64.cu.cc
new file mode 100644
index 0000000000..525ede938f
--- /dev/null
+++ b/tensorflow/core/kernels/tile_functor_gpu_complex64.cu.cc
@@ -0,0 +1,31 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/tile_functor.h"
+#include "tensorflow/core/kernels/tile_functor_gpu.h"
+
+namespace tensorflow {
+namespace functor {
+using Eigen::GpuDevice;
+
+template struct Tile<GpuDevice, complex64, int32>;
+template struct Tile<GpuDevice, complex64, int64>;
+}  // namespace functor
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/tile_functor_gpu_double.cu.cc b/tensorflow/core/kernels/tile_functor_gpu_double.cu.cc
new file mode 100644
index 0000000000..25e024083e
--- /dev/null
+++ b/tensorflow/core/kernels/tile_functor_gpu_double.cu.cc
@@ -0,0 +1,31 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/tile_functor.h"
+#include "tensorflow/core/kernels/tile_functor_gpu.h"
+
+namespace tensorflow {
+namespace functor {
+using Eigen::GpuDevice;
+
+template struct Tile<GpuDevice, double, int32>;
+template struct Tile<GpuDevice, double, int64>;
+}  // namespace functor
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/tile_functor_gpu_float.cu.cc b/tensorflow/core/kernels/tile_functor_gpu_float.cu.cc
new file mode 100644
index 0000000000..f0f31370e4
--- /dev/null
+++ b/tensorflow/core/kernels/tile_functor_gpu_float.cu.cc
@@ -0,0 +1,31 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/tile_functor.h"
+#include "tensorflow/core/kernels/tile_functor_gpu.h"
+
+namespace tensorflow {
+namespace functor {
+using Eigen::GpuDevice;
+
+template struct Tile<GpuDevice, float, int32>;
+template struct Tile<GpuDevice, float, int64>;
+}  // namespace functor
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/tile_functor_gpu_half.cu.cc b/tensorflow/core/kernels/tile_functor_gpu_half.cu.cc
new file mode 100644
index 0000000000..6c3810a0bc
--- /dev/null
+++ b/tensorflow/core/kernels/tile_functor_gpu_half.cu.cc
@@ -0,0 +1,31 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/tile_functor.h"
+#include "tensorflow/core/kernels/tile_functor_gpu.h"
+
+namespace tensorflow {
+namespace functor {
+using Eigen::GpuDevice;
+
+template struct Tile<GpuDevice, Eigen::half, int32>;
+template struct Tile<GpuDevice, Eigen::half, int64>;
+}  // namespace functor
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/tile_functor_gpu_int16.cu.cc b/tensorflow/core/kernels/tile_functor_gpu_int16.cu.cc
new file mode 100644
index 0000000000..2280dcbc82
--- /dev/null
+++ b/tensorflow/core/kernels/tile_functor_gpu_int16.cu.cc
@@ -0,0 +1,31 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/tile_functor.h"
+#include "tensorflow/core/kernels/tile_functor_gpu.h"
+
+namespace tensorflow {
+namespace functor {
+using Eigen::GpuDevice;
+
+template struct Tile<GpuDevice, int16, int32>;
+template struct Tile<GpuDevice, int16, int64>;
+}  // namespace functor
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/tile_functor_gpu_int32.cu.cc b/tensorflow/core/kernels/tile_functor_gpu_int32.cu.cc
new file mode 100644
index 0000000000..b05403bada
--- /dev/null
+++ b/tensorflow/core/kernels/tile_functor_gpu_int32.cu.cc
@@ -0,0 +1,31 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/tile_functor.h"
+#include "tensorflow/core/kernels/tile_functor_gpu.h"
+
+namespace tensorflow {
+namespace functor {
+using Eigen::GpuDevice;
+
+template struct Tile<GpuDevice, int32, int32>;
+template struct Tile<GpuDevice, int32, int64>;
+}  // namespace functor
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/tile_functor_gpu_int64.cu.cc b/tensorflow/core/kernels/tile_functor_gpu_int64.cu.cc
new file mode 100644
index 0000000000..2d83c6b3a1
--- /dev/null
+++ b/tensorflow/core/kernels/tile_functor_gpu_int64.cu.cc
@@ -0,0 +1,31 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/tile_functor.h"
+#include "tensorflow/core/kernels/tile_functor_gpu.h"
+
+namespace tensorflow {
+namespace functor {
+using Eigen::GpuDevice;
+
+template struct Tile<GpuDevice, int64, int32>;
+template struct Tile<GpuDevice, int64, int64>;
+}  // namespace functor
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
-- 
GitLab


From f09a6224bac04fd8b45d72b5babee08212dcf836 Mon Sep 17 00:00:00 2001
From: Gaurav Jain <gjn@google.com>
Date: Tue, 8 Jan 2019 02:23:41 -0800
Subject: [PATCH 0361/1095] Make DebugString() in ResourceBase constant

PiperOrigin-RevId: 228300469
---
 .../compiler/jit/xla_compilation_cache.cc      |  2 +-
 .../compiler/jit/xla_compilation_cache.h       |  2 +-
 .../compiler/tf2xla/xla_compiler_test.cc       |  2 +-
 tensorflow/compiler/tf2xla/xla_context.cc      |  2 +-
 tensorflow/compiler/tf2xla/xla_context.h       |  2 +-
 .../compiler/xrt/xrt_compilation_cache.cc      |  4 +++-
 .../compiler/xrt/xrt_compilation_cache.h       |  2 +-
 tensorflow/compiler/xrt/xrt_state.h            |  2 +-
 .../contrib/bigtable/kernels/bigtable_lib.h    |  4 ++--
 .../kernels/stats_accumulator_ops.cc           |  2 +-
 .../decision_tree_ensemble_resource.h          |  2 +-
 .../resources/quantile_stream_resource.h       |  2 +-
 .../kernels/v4/decision-tree-resource.h        |  2 +-
 .../kernels/v4/fertile-stats-resource.h        |  2 +-
 .../contrib/tensorrt/resources/trt_resources.h |  2 +-
 tensorflow/core/framework/lookup_interface.h   |  2 +-
 tensorflow/core/framework/queue_interface.h    |  4 ++--
 tensorflow/core/framework/reader_interface.h   |  2 +-
 tensorflow/core/framework/resource_mgr.h       |  2 +-
 tensorflow/core/framework/resource_mgr_test.cc |  8 ++++----
 .../core/framework/resource_op_kernel_test.cc  |  2 +-
 tensorflow/core/framework/resource_var.h       |  2 +-
 tensorflow/core/framework/stats_aggregator.h   |  2 +-
 tensorflow/core/kernels/barrier_ops.cc         |  2 +-
 tensorflow/core/kernels/batch_kernels.cc       |  6 +++---
 .../quantiles/quantile_stream_resource.h       | 18 +++++++++---------
 .../core/kernels/boosted_trees/resources.cc    |  2 +-
 .../core/kernels/boosted_trees/resources.h     |  2 +-
 .../kernels/conditional_accumulator_base.h     |  2 +-
 tensorflow/core/kernels/conv_ops.h             |  2 +-
 .../core/kernels/data/cache_dataset_ops.cc     |  4 +++-
 .../data/experimental/indexed_dataset_op.cc    |  2 +-
 .../data/experimental/threadpool_dataset_op.cc |  2 +-
 tensorflow/core/kernels/data/iterator_ops.cc   |  2 +-
 .../kernels/data/multi_device_iterator_ops.cc  |  2 +-
 .../core/kernels/data/shuffle_dataset_op.cc    |  2 +-
 tensorflow/core/kernels/fifo_queue.h           |  2 +-
 tensorflow/core/kernels/map_stage_op.cc        |  2 +-
 tensorflow/core/kernels/meta_support.cc        |  2 +-
 tensorflow/core/kernels/mutex_ops.cc           |  4 +++-
 tensorflow/core/kernels/priority_queue.h       |  2 +-
 .../core/kernels/random_shuffle_queue_op.cc    |  2 +-
 .../core/kernels/sparse_tensors_map_ops.cc     |  2 +-
 tensorflow/core/kernels/stack.cc               |  2 +-
 tensorflow/core/kernels/stage_op.cc            |  4 ++--
 tensorflow/core/kernels/summary_op.cc          |  4 +++-
 tensorflow/core/kernels/tensor_array.h         |  4 ++--
 .../core/kernels/tensor_forest/resources.h     |  2 +-
 tensorflow/core/kernels/variable_ops.cc        |  4 ++--
 tensorflow/core/summary/summary_db_writer.cc   |  2 +-
 tensorflow/core/summary/summary_file_writer.cc |  2 +-
 tensorflow/python/framework/test_ops.cc        |  2 +-
 52 files changed, 78 insertions(+), 70 deletions(-)

diff --git a/tensorflow/compiler/jit/xla_compilation_cache.cc b/tensorflow/compiler/jit/xla_compilation_cache.cc
index d5a1ab0850..bff4cc57ee 100644
--- a/tensorflow/compiler/jit/xla_compilation_cache.cc
+++ b/tensorflow/compiler/jit/xla_compilation_cache.cc
@@ -62,7 +62,7 @@ XlaCompilationCache::~XlaCompilationCache() {
   // about?
 }
 
-string XlaCompilationCache::DebugString() {
+string XlaCompilationCache::DebugString() const {
   return "XLA JIT compilation cache";
 }
 
diff --git a/tensorflow/compiler/jit/xla_compilation_cache.h b/tensorflow/compiler/jit/xla_compilation_cache.h
index 846d0c963d..02aa8f8839 100644
--- a/tensorflow/compiler/jit/xla_compilation_cache.h
+++ b/tensorflow/compiler/jit/xla_compilation_cache.h
@@ -88,7 +88,7 @@ class XlaCompilationCache : public ResourceBase {
   xla::LocalClient* client() const { return client_; }
   const DeviceType& device_type() const { return device_type_; }
 
-  string DebugString() override;
+  string DebugString() const override;
 
   // Describes the types, shapes and any compile-time constant arguments
   // to a kernel. Key that uniquely identifies a compilation output.
diff --git a/tensorflow/compiler/tf2xla/xla_compiler_test.cc b/tensorflow/compiler/tf2xla/xla_compiler_test.cc
index 32a4301840..492010f731 100644
--- a/tensorflow/compiler/tf2xla/xla_compiler_test.cc
+++ b/tensorflow/compiler/tf2xla/xla_compiler_test.cc
@@ -82,7 +82,7 @@ namespace {
 // compiled kernels.
 class DummyResourceForTest : public ResourceBase {
  public:
-  string DebugString() override { return "dummy"; }
+  string DebugString() const override { return "dummy"; }
   void Increment() { ++value_; }
   int Get() { return value_; }
 
diff --git a/tensorflow/compiler/tf2xla/xla_context.cc b/tensorflow/compiler/tf2xla/xla_context.cc
index a69af70503..6139bf3cea 100644
--- a/tensorflow/compiler/tf2xla/xla_context.cc
+++ b/tensorflow/compiler/tf2xla/xla_context.cc
@@ -61,7 +61,7 @@ void XlaContext::set_args(std::vector<XlaExpression> args) {
 XlaContext::XlaContext(XlaCompiler* compiler, xla::XlaBuilder* builder)
     : compiler_(compiler), builder_(builder) {}
 
-string XlaContext::DebugString() { return "XLA JIT context"; }
+string XlaContext::DebugString() const { return "XLA JIT context"; }
 
 void XlaContext::SetRetval(int index, const XlaExpression& expression) {
   if (retvals_.size() <= index) {
diff --git a/tensorflow/compiler/tf2xla/xla_context.h b/tensorflow/compiler/tf2xla/xla_context.h
index 0767d1faac..eb4ad3fe6a 100644
--- a/tensorflow/compiler/tf2xla/xla_context.h
+++ b/tensorflow/compiler/tf2xla/xla_context.h
@@ -47,7 +47,7 @@ class XlaContext : public ResourceBase {
   XlaContext(XlaCompiler* compiler, xla::XlaBuilder* builder);
 
   // Virtual method defined by ResourceBase.
-  string DebugString() override;
+  string DebugString() const override;
 
   XlaCompiler* compiler() const { return compiler_; }
 
diff --git a/tensorflow/compiler/xrt/xrt_compilation_cache.cc b/tensorflow/compiler/xrt/xrt_compilation_cache.cc
index d1405eae46..8bf0f28d22 100644
--- a/tensorflow/compiler/xrt/xrt_compilation_cache.cc
+++ b/tensorflow/compiler/xrt/xrt_compilation_cache.cc
@@ -273,6 +273,8 @@ Status XRTCompilationCache::Lookup(
   return Status::OK();
 }
 
-string XRTCompilationCache::DebugString() { return "XRTCompilationCache"; }
+string XRTCompilationCache::DebugString() const {
+  return "XRTCompilationCache";
+}
 
 }  // namespace tensorflow
diff --git a/tensorflow/compiler/xrt/xrt_compilation_cache.h b/tensorflow/compiler/xrt/xrt_compilation_cache.h
index c43d0fc478..7398e847d8 100644
--- a/tensorflow/compiler/xrt/xrt_compilation_cache.h
+++ b/tensorflow/compiler/xrt/xrt_compilation_cache.h
@@ -118,7 +118,7 @@ class XRTCompilationCache : public ResourceBase {
   // EntryRef holding the program is returned in entry.
   Status Lookup(int64 uid, std::unique_ptr<XRTCompilationCacheEntryRef>* entry);
 
-  string DebugString() override;
+  string DebugString() const override;
 
  private:
   // An entry in the compilation cache. The entry is deleted once it has been
diff --git a/tensorflow/compiler/xrt/xrt_state.h b/tensorflow/compiler/xrt/xrt_state.h
index 3e3d502412..4aac37737e 100644
--- a/tensorflow/compiler/xrt/xrt_state.h
+++ b/tensorflow/compiler/xrt/xrt_state.h
@@ -172,7 +172,7 @@ class XRTTupleAllocation : public ResourceBase {
   // ownership of the device memory is transferred to the result.
   xla::ShapeTree<xla::MaybeOwningDeviceMemory> ToDeviceMemoryTree(bool release);
 
-  string DebugString() override { return "XLA allocation handle"; }
+  string DebugString() const override { return "XLA allocation handle"; }
 
  private:
   // Creates a new handle with (tuple) shape.
diff --git a/tensorflow/contrib/bigtable/kernels/bigtable_lib.h b/tensorflow/contrib/bigtable/kernels/bigtable_lib.h
index 4652021fec..e3b4535bac 100644
--- a/tensorflow/contrib/bigtable/kernels/bigtable_lib.h
+++ b/tensorflow/contrib/bigtable/kernels/bigtable_lib.h
@@ -42,7 +42,7 @@ class BigtableClientResource : public ResourceBase {
     return client_;
   }
 
-  string DebugString() override {
+  string DebugString() const override {
     return strings::StrCat("BigtableClientResource(project_id: ", project_id_,
                            ", instance_id: ", instance_id_, ")");
   }
@@ -67,7 +67,7 @@ class BigtableTableResource : public ResourceBase {
 
   ::google::cloud::bigtable::noex::Table& table() { return table_; }
 
-  string DebugString() override {
+  string DebugString() const override {
     return strings::StrCat(
         "BigtableTableResource(client: ", client_->DebugString(),
         ", table: ", table_name_, ")");
diff --git a/tensorflow/contrib/boosted_trees/kernels/stats_accumulator_ops.cc b/tensorflow/contrib/boosted_trees/kernels/stats_accumulator_ops.cc
index e446c411a8..6faf696301 100644
--- a/tensorflow/contrib/boosted_trees/kernels/stats_accumulator_ops.cc
+++ b/tensorflow/contrib/boosted_trees/kernels/stats_accumulator_ops.cc
@@ -96,7 +96,7 @@ class StatsAccumulatorResource : public boosted_trees::StampedResource {
              TensorShapeUtils::IsScalar(hessian_shape));
   }
 
-  string DebugString() override {
+  string DebugString() const override {
     return strings::StrCat("StatsAccumulatorResource[size=", values_.size(),
                            "]");
   }
diff --git a/tensorflow/contrib/boosted_trees/resources/decision_tree_ensemble_resource.h b/tensorflow/contrib/boosted_trees/resources/decision_tree_ensemble_resource.h
index 94aeb2c7bb..0fe57c0a4e 100644
--- a/tensorflow/contrib/boosted_trees/resources/decision_tree_ensemble_resource.h
+++ b/tensorflow/contrib/boosted_trees/resources/decision_tree_ensemble_resource.h
@@ -34,7 +34,7 @@ class DecisionTreeEnsembleResource : public StampedResource {
             protobuf::Arena::CreateMessage<
                 boosted_trees::trees::DecisionTreeEnsembleConfig>(&arena_)) {}
 
-  string DebugString() override {
+  string DebugString() const override {
     return strings::StrCat("GTFlowDecisionTreeEnsemble[size=",
                            decision_tree_ensemble_->trees_size(), "]");
   }
diff --git a/tensorflow/contrib/boosted_trees/resources/quantile_stream_resource.h b/tensorflow/contrib/boosted_trees/resources/quantile_stream_resource.h
index fdaaae7f47..574e3065e7 100644
--- a/tensorflow/contrib/boosted_trees/resources/quantile_stream_resource.h
+++ b/tensorflow/contrib/boosted_trees/resources/quantile_stream_resource.h
@@ -43,7 +43,7 @@ class QuantileStreamResource : public StampedResource {
     set_stamp(stamp_token);
   }
 
-  string DebugString() override { return "QuantileStreamResource"; }
+  string DebugString() const override { return "QuantileStreamResource"; }
 
   tensorflow::mutex* mutex() { return &mu_; }
 
diff --git a/tensorflow/contrib/tensor_forest/kernels/v4/decision-tree-resource.h b/tensorflow/contrib/tensor_forest/kernels/v4/decision-tree-resource.h
index d3edb43733..3100a5a0e5 100644
--- a/tensorflow/contrib/tensor_forest/kernels/v4/decision-tree-resource.h
+++ b/tensorflow/contrib/tensor_forest/kernels/v4/decision-tree-resource.h
@@ -32,7 +32,7 @@ class DecisionTreeResource : public ResourceBase {
   // Constructor.
   explicit DecisionTreeResource(const TensorForestParams& params);
 
-  string DebugString() override {
+  string DebugString() const override {
     return strings::StrCat("DecisionTree[size=",
                            decision_tree_->decision_tree().nodes_size(), "]");
   }
diff --git a/tensorflow/contrib/tensor_forest/kernels/v4/fertile-stats-resource.h b/tensorflow/contrib/tensor_forest/kernels/v4/fertile-stats-resource.h
index eea0be27ca..44f2b3f473 100644
--- a/tensorflow/contrib/tensor_forest/kernels/v4/fertile-stats-resource.h
+++ b/tensorflow/contrib/tensor_forest/kernels/v4/fertile-stats-resource.h
@@ -40,7 +40,7 @@ class FertileStatsResource : public ResourceBase {
     model_op_ = LeafModelOperatorFactory::CreateLeafModelOperator(params_);
   }
 
-  string DebugString() override { return "FertileStats"; }
+  string DebugString() const override { return "FertileStats"; }
 
   void ExtractFromProto(const FertileStats& stats);
 
diff --git a/tensorflow/contrib/tensorrt/resources/trt_resources.h b/tensorflow/contrib/tensorrt/resources/trt_resources.h
index aac9e5c7bd..8d877b392f 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_resources.h
+++ b/tensorflow/contrib/tensorrt/resources/trt_resources.h
@@ -48,7 +48,7 @@ class TRTCalibrationResource : public tensorflow::ResourceBase {
     allocator_.reset();
   }
 
-  string DebugString() override {
+  string DebugString() const override {
     std::stringstream oss;
     using std::dec;
     using std::endl;
diff --git a/tensorflow/core/framework/lookup_interface.h b/tensorflow/core/framework/lookup_interface.h
index d33945fd1b..7e5dbe5632 100644
--- a/tensorflow/core/framework/lookup_interface.h
+++ b/tensorflow/core/framework/lookup_interface.h
@@ -131,7 +131,7 @@ class LookupInterface : public ResourceBase {
   // - the default_value tensor shape matches the table's value shape.
   Status CheckFindArguments(const Tensor& keys, const Tensor& default_value);
 
-  string DebugString() override {
+  string DebugString() const override {
     return strings::StrCat("A lookup table of size: ", size());
   }
 
diff --git a/tensorflow/core/framework/queue_interface.h b/tensorflow/core/framework/queue_interface.h
index 4ca4416c5a..9395cce164 100644
--- a/tensorflow/core/framework/queue_interface.h
+++ b/tensorflow/core/framework/queue_interface.h
@@ -85,11 +85,11 @@ class QueueInterface : public ResourceBase {
   virtual Status MatchesNodeDef(const NodeDef& node_def) = 0;
 
   // Returns the number of elements in the queue.
-  virtual int32 size() = 0;
+  virtual int32 size() const = 0;
 
   virtual const DataTypeVector& component_dtypes() const = 0;
 
-  string DebugString() override {
+  string DebugString() const override {
     return strings::StrCat("A Queue of size: ", size());
   }
 
diff --git a/tensorflow/core/framework/reader_interface.h b/tensorflow/core/framework/reader_interface.h
index f894acbe1d..e47644cb8f 100644
--- a/tensorflow/core/framework/reader_interface.h
+++ b/tensorflow/core/framework/reader_interface.h
@@ -76,7 +76,7 @@ class ReaderInterface : public ResourceBase {
   // Note: Must Reset on error.
   virtual Status RestoreState(const string& state) = 0;
 
-  string DebugString() override { return "a reader"; }
+  string DebugString() const override { return "a reader"; }
 
  protected:
   virtual ~ReaderInterface() {}
diff --git a/tensorflow/core/framework/resource_mgr.h b/tensorflow/core/framework/resource_mgr.h
index 8a7c25da92..9c381e7d6b 100644
--- a/tensorflow/core/framework/resource_mgr.h
+++ b/tensorflow/core/framework/resource_mgr.h
@@ -77,7 +77,7 @@ namespace tensorflow {
 class ResourceBase : public core::RefCounted {
  public:
   // Returns a debug string for *this.
-  virtual string DebugString() = 0;
+  virtual string DebugString() const = 0;
 
   // Returns memory used by this resource.
   virtual int64 MemoryUsed() const { return 0; }
diff --git a/tensorflow/core/framework/resource_mgr_test.cc b/tensorflow/core/framework/resource_mgr_test.cc
index 7c7f0af0ce..1c785736e6 100644
--- a/tensorflow/core/framework/resource_mgr_test.cc
+++ b/tensorflow/core/framework/resource_mgr_test.cc
@@ -32,7 +32,7 @@ class Resource : public ResourceBase {
   explicit Resource(const string& label) : label_(label) {}
   ~Resource() override {}
 
-  string DebugString() override { return strings::StrCat("R/", label_); }
+  string DebugString() const override { return strings::StrCat("R/", label_); }
 
  private:
   string label_;
@@ -43,7 +43,7 @@ class Other : public ResourceBase {
   explicit Other(const string& label) : label_(label) {}
   ~Other() override {}
 
-  string DebugString() override { return strings::StrCat("O/", label_); }
+  string DebugString() const override { return strings::StrCat("O/", label_); }
 
  private:
   string label_;
@@ -245,7 +245,7 @@ class StubDevice : public DeviceBase {
 // Empty stub resource for testing resource handles.
 class StubResource : public ResourceBase {
  public:
-  string DebugString() override { return ""; }
+  string DebugString() const override { return ""; }
   int value_{0};
 };
 
@@ -305,7 +305,7 @@ TEST(ResourceHandleTest, DifferentDevice) {
 // Other stub resource to test type-checking of resource handles.
 class OtherStubResource : public ResourceBase {
  public:
-  string DebugString() override { return ""; }
+  string DebugString() const override { return ""; }
 };
 
 TEST(ResourceHandleTest, DifferentType) {
diff --git a/tensorflow/core/framework/resource_op_kernel_test.cc b/tensorflow/core/framework/resource_op_kernel_test.cc
index c1e503dc57..7a2a87045b 100644
--- a/tensorflow/core/framework/resource_op_kernel_test.cc
+++ b/tensorflow/core/framework/resource_op_kernel_test.cc
@@ -46,7 +46,7 @@ class StubDevice : public DeviceBase {
 // Stub resource for testing resource op kernel.
 class StubResource : public ResourceBase {
  public:
-  string DebugString() override { return ""; }
+  string DebugString() const override { return ""; }
   int code;
 };
 
diff --git a/tensorflow/core/framework/resource_var.h b/tensorflow/core/framework/resource_var.h
index f5de5dba88..9387b6c23c 100644
--- a/tensorflow/core/framework/resource_var.h
+++ b/tensorflow/core/framework/resource_var.h
@@ -67,7 +67,7 @@ class Var : public ResourceBase {
   mutex* mu() { return &mu_; }
   Tensor* tensor() { return &tensor_; }
 
-  string DebugString() override {
+  string DebugString() const override {
     return strings::StrCat(DataTypeString(tensor_.dtype()), "/",
                            tensor_.shape().DebugString());
   }
diff --git a/tensorflow/core/framework/stats_aggregator.h b/tensorflow/core/framework/stats_aggregator.h
index af53ed0a3c..7c960840d7 100644
--- a/tensorflow/core/framework/stats_aggregator.h
+++ b/tensorflow/core/framework/stats_aggregator.h
@@ -83,7 +83,7 @@ class StatsAggregatorResource : public ResourceBase {
     return stats_aggregator_;
   }
 
-  string DebugString() { return "StatsAggregatorResource"; }
+  string DebugString() const override { return "StatsAggregatorResource"; }
 
  private:
   const std::shared_ptr<StatsAggregator> stats_aggregator_;
diff --git a/tensorflow/core/kernels/barrier_ops.cc b/tensorflow/core/kernels/barrier_ops.cc
index aa91235822..d5bd36b4ce 100644
--- a/tensorflow/core/kernels/barrier_ops.cc
+++ b/tensorflow/core/kernels/barrier_ops.cc
@@ -300,7 +300,7 @@ class Barrier : public ResourceBase {
     ready_queue_->Unref();
   }
 
-  string DebugString() override { return "A barrier"; }
+  string DebugString() const override { return "A barrier"; }
 
  protected:
   template <typename T>
diff --git a/tensorflow/core/kernels/batch_kernels.cc b/tensorflow/core/kernels/batch_kernels.cc
index 35ddda0ec0..5ba461aa9d 100644
--- a/tensorflow/core/kernels/batch_kernels.cc
+++ b/tensorflow/core/kernels/batch_kernels.cc
@@ -233,7 +233,7 @@ class BatchResource : public ResourceBase {
     return Status::OK();
   }
 
-  string DebugString() final { return "BatchResource"; }
+  string DebugString() const final { return "BatchResource"; }
 
   // Ingests data from one invocation of the batch op. The data is enqueued to
   // be combined with others into a batch, asynchronously.
@@ -878,7 +878,7 @@ class UnbatchResource : public ResourceBase {
     timeout_enforcer_ = nullptr;
   }
 
-  string DebugString() final { return "UnbatchResource"; }
+  string DebugString() const final { return "UnbatchResource"; }
 
   Status Compute(OpKernelContext* context, AsyncOpKernel::DoneCallback done) {
     const Tensor& data_t = context->input(0);
@@ -1094,7 +1094,7 @@ class UnbatchGradResource : public ResourceBase {
  public:
   UnbatchGradResource() {}
 
-  string DebugString() final { return "UnbatchGradResource"; }
+  string DebugString() const final { return "UnbatchGradResource"; }
 
   // Flushes the information for one batch, given its context and done
   // callback. Clears all information about it from the available_tensors_.
diff --git a/tensorflow/core/kernels/boosted_trees/quantiles/quantile_stream_resource.h b/tensorflow/core/kernels/boosted_trees/quantiles/quantile_stream_resource.h
index 1c31724272..965bf2c924 100644
--- a/tensorflow/core/kernels/boosted_trees/quantiles/quantile_stream_resource.h
+++ b/tensorflow/core/kernels/boosted_trees/quantiles/quantile_stream_resource.h
@@ -37,15 +37,15 @@ class BoostedTreesQuantileStreamResource : public ResourceBase {
         epsilon_(epsilon),
         num_streams_(num_streams),
         max_elements_(max_elements) {
-          streams_.reserve(num_streams_);
-          boundaries_.reserve(num_streams_);
-          for (int64 idx = 0; idx < num_streams; ++idx) {
-            streams_.push_back(QuantileStream(epsilon, max_elements));
-            boundaries_.push_back(std::vector<float>());
-          }
-        }
-
-  string DebugString() override { return "QuantileStreamResource"; }
+    streams_.reserve(num_streams_);
+    boundaries_.reserve(num_streams_);
+    for (int64 idx = 0; idx < num_streams; ++idx) {
+      streams_.push_back(QuantileStream(epsilon, max_elements));
+      boundaries_.push_back(std::vector<float>());
+    }
+  }
+
+  string DebugString() const override { return "QuantileStreamResource"; }
 
   tensorflow::mutex* mutex() { return &mu_; }
 
diff --git a/tensorflow/core/kernels/boosted_trees/resources.cc b/tensorflow/core/kernels/boosted_trees/resources.cc
index 2798722536..42df484881 100644
--- a/tensorflow/core/kernels/boosted_trees/resources.cc
+++ b/tensorflow/core/kernels/boosted_trees/resources.cc
@@ -31,7 +31,7 @@ BoostedTreesEnsembleResource::BoostedTreesEnsembleResource()
           protobuf::Arena::CreateMessage<boosted_trees::TreeEnsemble>(
               &arena_)) {}
 
-string BoostedTreesEnsembleResource::DebugString() {
+string BoostedTreesEnsembleResource::DebugString() const {
   return strings::StrCat("TreeEnsemble[size=", tree_ensemble_->trees_size(),
                          "]");
 }
diff --git a/tensorflow/core/kernels/boosted_trees/resources.h b/tensorflow/core/kernels/boosted_trees/resources.h
index f961ed3814..3c7b2df9b0 100644
--- a/tensorflow/core/kernels/boosted_trees/resources.h
+++ b/tensorflow/core/kernels/boosted_trees/resources.h
@@ -48,7 +48,7 @@ class BoostedTreesEnsembleResource : public StampedResource {
  public:
   BoostedTreesEnsembleResource();
 
-  string DebugString() override;
+  string DebugString() const override;
 
   bool InitFromSerialized(const string& serialized, const int64 stamp_token);
 
diff --git a/tensorflow/core/kernels/conditional_accumulator_base.h b/tensorflow/core/kernels/conditional_accumulator_base.h
index 4a5ec6f0fb..2618ffbb09 100644
--- a/tensorflow/core/kernels/conditional_accumulator_base.h
+++ b/tensorflow/core/kernels/conditional_accumulator_base.h
@@ -68,7 +68,7 @@ class ConditionalAccumulatorBase : public ResourceBase {
 
   const DataType& dtype() const { return dtype_; }
 
-  string DebugString() override { return "A conditional accumulator"; }
+  string DebugString() const override { return "A conditional accumulator"; }
 
   // SetGlobalStep is a modifier method for current_global_step.
   // It returns an InvalidArgument error if the new_global_step is less than
diff --git a/tensorflow/core/kernels/conv_ops.h b/tensorflow/core/kernels/conv_ops.h
index 7ec878e0b2..7ccbaf4bf2 100644
--- a/tensorflow/core/kernels/conv_ops.h
+++ b/tensorflow/core/kernels/conv_ops.h
@@ -63,7 +63,7 @@ struct Im2ColBufferResource : public ResourceBase {
   // the buffer memory held by this resource.
   mutex mu;
   T* data;
-  string DebugString() { return "Im2ColBufferResource"; }
+  string DebugString() const { return "Im2ColBufferResource"; }
 };
 
 // Convolution parameters specified by Op attributes.
diff --git a/tensorflow/core/kernels/data/cache_dataset_ops.cc b/tensorflow/core/kernels/data/cache_dataset_ops.cc
index f00b38e732..535f49cff8 100644
--- a/tensorflow/core/kernels/data/cache_dataset_ops.cc
+++ b/tensorflow/core/kernels/data/cache_dataset_ops.cc
@@ -614,7 +614,9 @@ class CacheDatasetOp : public UnaryDatasetOpKernel {
      public:
       MemoryCache() = default;
 
-      string DebugString() override { return "CacheDataset::MemoryCache"; }
+      string DebugString() const override {
+        return "CacheDataset::MemoryCache";
+      }
 
       // Marks the cache as completed.
       void Complete() {
diff --git a/tensorflow/core/kernels/data/experimental/indexed_dataset_op.cc b/tensorflow/core/kernels/data/experimental/indexed_dataset_op.cc
index a07eaebdf9..83eeed7892 100644
--- a/tensorflow/core/kernels/data/experimental/indexed_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/indexed_dataset_op.cc
@@ -106,7 +106,7 @@ class MaterializedDatasetResource : public ResourceBase {
       const std::vector<PartialTensorShape>& output_shapes)
       : output_dtypes_(output_dtypes), output_shapes_(output_shapes) {}
 
-  string DebugString() override {
+  string DebugString() const override {
     return "Materialized IndexedDataset resource";
   }
 
diff --git a/tensorflow/core/kernels/data/experimental/threadpool_dataset_op.cc b/tensorflow/core/kernels/data/experimental/threadpool_dataset_op.cc
index 8ae45ed5c9..fab3cab7da 100644
--- a/tensorflow/core/kernels/data/experimental/threadpool_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/threadpool_dataset_op.cc
@@ -51,7 +51,7 @@ class ThreadPoolResource : public ResourceBase {
 
   int32 NumThreads() { return thread_pool_.NumThreads(); }
 
-  string DebugString() override { return "ThreadPoolResource"; }
+  string DebugString() const override { return "ThreadPoolResource"; }
 
  private:
   thread::ThreadPool thread_pool_;
diff --git a/tensorflow/core/kernels/data/iterator_ops.cc b/tensorflow/core/kernels/data/iterator_ops.cc
index 9f5881563b..81e26d35c0 100644
--- a/tensorflow/core/kernels/data/iterator_ops.cc
+++ b/tensorflow/core/kernels/data/iterator_ops.cc
@@ -231,7 +231,7 @@ class IteratorResource : public ResourceBase {
     return Status::OK();
   }
 
-  string DebugString() override { return "Iterator resource"; }
+  string DebugString() const override { return "Iterator resource"; }
 
   const DataTypeVector& output_dtypes() const { return output_dtypes_; }
 
diff --git a/tensorflow/core/kernels/data/multi_device_iterator_ops.cc b/tensorflow/core/kernels/data/multi_device_iterator_ops.cc
index ba2125a66e..05528e0ee0 100644
--- a/tensorflow/core/kernels/data/multi_device_iterator_ops.cc
+++ b/tensorflow/core/kernels/data/multi_device_iterator_ops.cc
@@ -59,7 +59,7 @@ class MultiDeviceIterator : public ResourceBase {
     DCHECK(lib_ != nullptr);
   }
 
-  string DebugString() override {
+  string DebugString() const override {
     return strings::StrCat("MultiDeviceIterator for ", devices_.size(),
                            " devices");
   }
diff --git a/tensorflow/core/kernels/data/shuffle_dataset_op.cc b/tensorflow/core/kernels/data/shuffle_dataset_op.cc
index db0cc6fa4d..4c380c1fa2 100644
--- a/tensorflow/core/kernels/data/shuffle_dataset_op.cc
+++ b/tensorflow/core/kernels/data/shuffle_dataset_op.cc
@@ -412,7 +412,7 @@ class ShuffleDatasetOp : public ShuffleDatasetOpBase {
             parent_generator_(seed, seed2),
             generator_(&parent_generator_) {}
 
-      string DebugString() override {
+      string DebugString() const override {
         return "ReshufflingDataset::RandomSeedGenerator";
       }
 
diff --git a/tensorflow/core/kernels/fifo_queue.h b/tensorflow/core/kernels/fifo_queue.h
index 697ee81c39..4d3a7c1971 100644
--- a/tensorflow/core/kernels/fifo_queue.h
+++ b/tensorflow/core/kernels/fifo_queue.h
@@ -49,7 +49,7 @@ class FIFOQueue : public TypedQueue<std::deque<PersistentTensor> > {
                       CallbackWithTuple callback) override;
   Status MatchesNodeDef(const NodeDef& node_def) override;
 
-  int32 size() override {
+  int32 size() const override {
     mutex_lock lock(mu_);
     return queues_[0].size();
   }
diff --git a/tensorflow/core/kernels/map_stage_op.cc b/tensorflow/core/kernels/map_stage_op.cc
index dd89597369..27a8696e54 100644
--- a/tensorflow/core/kernels/map_stage_op.cc
+++ b/tensorflow/core/kernels/map_stage_op.cc
@@ -480,7 +480,7 @@ class StagingMap : public ResourceBase {
     return map_.size();
   }
 
-  string DebugString() override { return "StagingMap"; }
+  string DebugString() const override { return "StagingMap"; }
 };
 
 template <bool Ordered>
diff --git a/tensorflow/core/kernels/meta_support.cc b/tensorflow/core/kernels/meta_support.cc
index 39e60c9fce..44f2997e18 100644
--- a/tensorflow/core/kernels/meta_support.cc
+++ b/tensorflow/core/kernels/meta_support.cc
@@ -54,7 +54,7 @@ class Scratch : public ResourceBase {
 
   uint8_t* buffer() { return scratch_32_aligned_; }
 
-  string DebugString() { return "MetaGemmScratchResource"; }
+  string DebugString() const override { return "MetaGemmScratchResource"; }
 
  private:
   std::unique_ptr<uint8_t> scratch_;
diff --git a/tensorflow/core/kernels/mutex_ops.cc b/tensorflow/core/kernels/mutex_ops.cc
index ddb7a606c1..1603a2aa86 100644
--- a/tensorflow/core/kernels/mutex_ops.cc
+++ b/tensorflow/core/kernels/mutex_ops.cc
@@ -45,7 +45,9 @@ class Mutex : public ResourceBase {
     VLOG(2) << "Creating mutex with name " << name << ": " << this;
   }
 
-  string DebugString() override { return strings::StrCat("Mutex ", name_); }
+  string DebugString() const override {
+    return strings::StrCat("Mutex ", name_);
+  }
 
   class LockReleaser {
    public:
diff --git a/tensorflow/core/kernels/priority_queue.h b/tensorflow/core/kernels/priority_queue.h
index 8e69b5b699..a719c518c3 100644
--- a/tensorflow/core/kernels/priority_queue.h
+++ b/tensorflow/core/kernels/priority_queue.h
@@ -68,7 +68,7 @@ class PriorityQueue
   Status MatchesPriorityNodeDefTypes(const NodeDef& node_def) const;
   Status MatchesPriorityNodeDefShapes(const NodeDef& node_def) const;
 
-  int32 size() override {
+  int32 size() const override {
     mutex_lock lock(mu_);
     return queues_[0].size();
   }
diff --git a/tensorflow/core/kernels/random_shuffle_queue_op.cc b/tensorflow/core/kernels/random_shuffle_queue_op.cc
index 31e8ce944f..02b9b022fd 100644
--- a/tensorflow/core/kernels/random_shuffle_queue_op.cc
+++ b/tensorflow/core/kernels/random_shuffle_queue_op.cc
@@ -59,7 +59,7 @@ class RandomShuffleQueue : public TypedQueue<std::vector<PersistentTensor> > {
                       CallbackWithTuple callback) override;
   Status MatchesNodeDef(const NodeDef& node_def) override;
 
-  int32 size() override {
+  int32 size() const override {
     mutex_lock lock(mu_);
     return queues_[0].size();
   }
diff --git a/tensorflow/core/kernels/sparse_tensors_map_ops.cc b/tensorflow/core/kernels/sparse_tensors_map_ops.cc
index 74fa3a15f0..939638b370 100644
--- a/tensorflow/core/kernels/sparse_tensors_map_ops.cc
+++ b/tensorflow/core/kernels/sparse_tensors_map_ops.cc
@@ -43,7 +43,7 @@ class SparseTensorsMap : public ResourceBase {
  public:
   explicit SparseTensorsMap(const string& name) : name_(name), counter_(0) {}
 
-  string DebugString() override { return "A SparseTensorsMap"; }
+  string DebugString() const override { return "A SparseTensorsMap"; }
 
   typedef struct {
     PersistentTensor indices;
diff --git a/tensorflow/core/kernels/stack.cc b/tensorflow/core/kernels/stack.cc
index 5c70a2d62d..2af6b4b814 100644
--- a/tensorflow/core/kernels/stack.cc
+++ b/tensorflow/core/kernels/stack.cc
@@ -96,7 +96,7 @@ class Stack : public ResourceBase {
 
   DataType ElemType() { return elem_type_; }
 
-  string DebugString() override {
+  string DebugString() const override {
     mutex_lock l(mu_);
     return strings::StrCat("Stack[", stack_name_, "]");
   }
diff --git a/tensorflow/core/kernels/stage_op.cc b/tensorflow/core/kernels/stage_op.cc
index c91bdc43cf..65174e163c 100644
--- a/tensorflow/core/kernels/stage_op.cc
+++ b/tensorflow/core/kernels/stage_op.cc
@@ -132,7 +132,7 @@ class Buffer : public ResourceBase {
     notify_inserters_if_bounded(&lock);
   }
 
-  string DebugString() override {
+  string DebugString() const override {
     std::unique_lock<std::mutex> lock(mu_);
     return strings::StrCat("Staging size: ", buf_.size());
   }
@@ -170,7 +170,7 @@ class Buffer : public ResourceBase {
   std::size_t capacity_;
   std::size_t memory_limit_;
   std::size_t current_bytes_;
-  std::mutex mu_;
+  mutable std::mutex mu_;
   std::condition_variable non_empty_cond_var_;
   std::condition_variable full_cond_var_;
   std::deque<Tuple> buf_;
diff --git a/tensorflow/core/kernels/summary_op.cc b/tensorflow/core/kernels/summary_op.cc
index 1f4e3418f4..1053aa7d53 100644
--- a/tensorflow/core/kernels/summary_op.cc
+++ b/tensorflow/core/kernels/summary_op.cc
@@ -124,7 +124,9 @@ TF_CALL_REAL_NUMBER_TYPES(REGISTER)
 struct HistogramResource : public ResourceBase {
   histogram::ThreadSafeHistogram histogram;
 
-  string DebugString() override { return "A histogram summary. Stats ..."; }
+  string DebugString() const override {
+    return "A histogram summary. Stats ...";
+  }
 };
 
 class SummaryMergeOp : public OpKernel {
diff --git a/tensorflow/core/kernels/tensor_array.h b/tensorflow/core/kernels/tensor_array.h
index 384a63e945..507ab459ca 100644
--- a/tensorflow/core/kernels/tensor_array.h
+++ b/tensorflow/core/kernels/tensor_array.h
@@ -261,7 +261,7 @@ class TensorArray : public ResourceBase {
     return Status::OK();
   }
 
-  string DebugString() override {
+  string DebugString() const override {
     mutex_lock l(mu_);
     CHECK(!closed_);
     return strings::StrCat("TensorArray[", tensors_.size(), "]");
@@ -376,7 +376,7 @@ class TensorArray : public ResourceBase {
   const DataType dtype_;
   Tensor handle_;
 
-  mutex mu_;
+  mutable mutex mu_;
 
   // Marks that the tensor_array_ has been cleared.
   bool closed_ GUARDED_BY(mu_);
diff --git a/tensorflow/core/kernels/tensor_forest/resources.h b/tensorflow/core/kernels/tensor_forest/resources.h
index da258e5017..f0a78f9726 100644
--- a/tensorflow/core/kernels/tensor_forest/resources.h
+++ b/tensorflow/core/kernels/tensor_forest/resources.h
@@ -34,7 +34,7 @@ class TensorForestTreeResource : public ResourceBase {
  public:
   TensorForestTreeResource();
 
-  string DebugString() override {
+  string DebugString() const override {
     return strings::StrCat("TensorForestTree[size=", get_size(), "]");
   }
 
diff --git a/tensorflow/core/kernels/variable_ops.cc b/tensorflow/core/kernels/variable_ops.cc
index eadea18f76..00994bbe8e 100644
--- a/tensorflow/core/kernels/variable_ops.cc
+++ b/tensorflow/core/kernels/variable_ops.cc
@@ -35,7 +35,7 @@ class LegacyVar : public ResourceBase {
   mutex* mu() { return &mu_; }
   Tensor* tensor() { return &tensor_; }
 
-  string DebugString() override {
+  string DebugString() const override {
     return strings::StrCat(DataTypeString(tensor_.dtype()), "/",
                            tensor_.shape().DebugString());
   }
@@ -116,7 +116,7 @@ class TemporaryVariableOp : public OpKernel {
     mutex mu;
     Tensor val;
     string name;
-    string DebugString() override { return name; }
+    string DebugString() const override { return name; }
     ~TmpVar() override { VLOG(3) << "TmpVar " << name << " deleted"; }
   };
 
diff --git a/tensorflow/core/summary/summary_db_writer.cc b/tensorflow/core/summary/summary_db_writer.cc
index 7a5d796821..b203d439cc 100644
--- a/tensorflow/core/summary/summary_db_writer.cc
+++ b/tensorflow/core/summary/summary_db_writer.cc
@@ -972,7 +972,7 @@ class SummaryDbWriter : public SummaryWriterInterface {
     return MigrateEvent(std::move(e));
   }
 
-  string DebugString() override { return "SummaryDbWriter"; }
+  string DebugString() const override { return "SummaryDbWriter"; }
 
  private:
   Status Write(int64 step, const Tensor& t, const string& tag,
diff --git a/tensorflow/core/summary/summary_file_writer.cc b/tensorflow/core/summary/summary_file_writer.cc
index 593ccdd684..711a7d3d10 100644
--- a/tensorflow/core/summary/summary_file_writer.cc
+++ b/tensorflow/core/summary/summary_file_writer.cc
@@ -148,7 +148,7 @@ class SummaryFileWriter : public SummaryWriterInterface {
     return Status::OK();
   }
 
-  string DebugString() override { return "SummaryFileWriter"; }
+  string DebugString() const override { return "SummaryFileWriter"; }
 
  private:
   double GetWallTime() {
diff --git a/tensorflow/python/framework/test_ops.cc b/tensorflow/python/framework/test_ops.cc
index 99e184a8ac..1d0145f61c 100644
--- a/tensorflow/python/framework/test_ops.cc
+++ b/tensorflow/python/framework/test_ops.cc
@@ -157,7 +157,7 @@ REGISTER_KERNEL_BUILDER(Name("Old").Device(DEVICE_CPU), OldOp);
 // Stubbed-out resource to test resource handle ops.
 class StubResource : public ResourceBase {
  public:
-  string DebugString() override { return ""; }
+  string DebugString() const override { return ""; }
 };
 
 REGISTER_RESOURCE_HANDLE_KERNEL(StubResource);
-- 
GitLab


From 8624a703ebd914e9d91bb7992570b52946fad970 Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Tue, 8 Jan 2019 02:47:46 -0800
Subject: [PATCH 0362/1095] Try to make resize bilinear test more
 deterministic.

PiperOrigin-RevId: 228301953
---
 tensorflow/lite/kernels/internal/resize_bilinear_test.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tensorflow/lite/kernels/internal/resize_bilinear_test.cc b/tensorflow/lite/kernels/internal/resize_bilinear_test.cc
index 1c5ac1992f..4a19b69a7c 100644
--- a/tensorflow/lite/kernels/internal/resize_bilinear_test.cc
+++ b/tensorflow/lite/kernels/internal/resize_bilinear_test.cc
@@ -76,6 +76,7 @@ void TestOneResizeBilinear(int batch, int depth, int input_width,
 }
 
 TEST(ResizeBilinear, TestResizeBilinear8Bit) {
+  RandomEngine().seed(38291);
   const int kTestsToRun = 100 * 1000;
   for (int i = 0; i < kTestsToRun; i++) {
     const int batch = ExponentialRandomPositiveInt(0.9f, 3, 20);
@@ -91,6 +92,7 @@ TEST(ResizeBilinear, TestResizeBilinear8Bit) {
 }
 
 TEST(ResizeBilinear2x2, TestResizeBilinear8Bit) {
+  RandomEngine().seed(38291);
   const int kTestsToRun = 100 * 1000;
   for (int i = 0; i < kTestsToRun; i++) {
     const int batch = ExponentialRandomPositiveInt(0.9f, 3, 20);
@@ -106,6 +108,7 @@ TEST(ResizeBilinear2x2, TestResizeBilinear8Bit) {
 }
 
 TEST(ResizeBilinear, TestResizeBilinear) {
+  RandomEngine().seed(38291);
   const int kTestsToRun = 100 * 1000;
   for (int i = 0; i < kTestsToRun; i++) {
     const int batch = ExponentialRandomPositiveInt(0.9f, 3, 20);
@@ -121,6 +124,7 @@ TEST(ResizeBilinear, TestResizeBilinear) {
 }
 
 TEST(ResizeBilinear2x2, TestResizeBilinear) {
+  RandomEngine().seed(38291);
   const int kTestsToRun = 100 * 1000;
   for (int i = 0; i < kTestsToRun; i++) {
     const int batch = ExponentialRandomPositiveInt(0.9f, 3, 20);
-- 
GitLab


From 07c6aa1a40f4515506d53262d7da2e8663c049cb Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Tue, 8 Jan 2019 02:48:31 -0800
Subject: [PATCH 0363/1095] Use int32_t rather than int32 for compatibility.

PiperOrigin-RevId: 228302002
---
 tensorflow/lite/kernels/sparse_output_fully_connected_test.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/kernels/sparse_output_fully_connected_test.cc b/tensorflow/lite/kernels/sparse_output_fully_connected_test.cc
index 8152ae6685..7d5fec192c 100644
--- a/tensorflow/lite/kernels/sparse_output_fully_connected_test.cc
+++ b/tensorflow/lite/kernels/sparse_output_fully_connected_test.cc
@@ -62,7 +62,7 @@ class BaseSparseOutputFullyConnectedOpModel : public SingleOpModel {
     PopulateTensor(input_, data);
   }
 
-  void SetLookup(const std::vector<int32>& f) { PopulateTensor(lookup_, f); }
+  void SetLookup(const std::vector<int32_t>& f) { PopulateTensor(lookup_, f); }
 
   void SetBias(const std::vector<float>& f) { PopulateTensor(bias_, f); }
 
-- 
GitLab


From 81492c074aa134d8756b361557ec2dd9aad3cdc1 Mon Sep 17 00:00:00 2001
From: Chris Jones <cjfj@google.com>
Date: Tue, 8 Jan 2019 02:50:54 -0800
Subject: [PATCH 0364/1095] Add DistStrat `ReplicaContext.all_reduce`.

PiperOrigin-RevId: 228302155
---
 .../collective_all_reduce_strategy_test.py    |  44 ++++-
 .../python/mirrored_strategy_multigpu_test.py |  47 ++++-
 .../python/one_device_strategy_test.py        |  22 ++-
 .../python/parameter_server_strategy_test.py  |  38 +++-
 .../distribute/python/strategy_test_lib.py    | 162 ++++++++++++++++++
 .../python/distribute/distribute_lib.py       |  54 ++++++
 ...nsorflow.distribute.-replica-context.pbtxt |   4 +
 ...nsorflow.distribute.-replica-context.pbtxt |   4 +
 8 files changed, 363 insertions(+), 12 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
index 0fb672dded..4c8c01a216 100644
--- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
@@ -397,9 +397,11 @@ class DistributedCollectiveAllReduceStrategyTestWithChief(
         self._test_complex_model, self._cluster_spec, num_gpus=num_gpus)
 
 
-class LocalCollectiveAllReduceStrategy(CollectiveAllReduceStrategyTestBase,
-                                       strategy_test_lib.DistributionTestBase,
-                                       parameterized.TestCase):
+class LocalCollectiveAllReduceStrategy(
+    CollectiveAllReduceStrategyTestBase,
+    strategy_test_lib.DistributionTestBase,
+    strategy_test_lib.TwoDeviceDistributionTestBase,
+    parameterized.TestCase):
 
   def testMinimizeLossGraph(self, num_gpus=2):
     # Collective ops doesn't support strategy with one device.
@@ -428,6 +430,42 @@ class LocalCollectiveAllReduceStrategy(CollectiveAllReduceStrategyTestBase,
     self._test_input_fn_iterator(None, None, num_gpus,
                                  input_fn, expected_values)
 
+  def testAllReduceSum(self):
+    if context.num_gpus() < 2: self.skipTest('Not enough GPUs')
+    distribution, target, config = self._get_test_object(None, None, num_gpus=2)
+    with self.cached_session(config=config, target=target):
+      self._test_all_reduce_sum(distribution)
+
+  def testAllReduceSumGradients(self):
+    if context.num_gpus() < 2: self.skipTest('Not enough GPUs')
+    distribution, target, config = self._get_test_object(None, None, num_gpus=2)
+    with self.cached_session(config=config, target=target):
+      self._test_all_reduce_sum_gradients(distribution)
+
+  def testAllReduceSumGradientTape(self):
+    if context.num_gpus() < 2: self.skipTest('Not enough GPUs')
+    distribution, target, config = self._get_test_object(None, None, num_gpus=2)
+    with self.cached_session(config=config, target=target):
+      self._test_all_reduce_sum_gradient_tape(distribution)
+
+  def testAllReduceMean(self):
+    if context.num_gpus() < 2: self.skipTest('Not enough GPUs')
+    distribution, target, config = self._get_test_object(None, None, num_gpus=2)
+    with self.cached_session(config=config, target=target):
+      self._test_all_reduce_mean(distribution)
+
+  def testAllReduceMeanGradients(self):
+    if context.num_gpus() < 2: self.skipTest('Not enough GPUs')
+    distribution, target, config = self._get_test_object(None, None, num_gpus=2)
+    with self.cached_session(config=config, target=target):
+      self._test_all_reduce_mean_gradients(distribution)
+
+  def testAllReduceMeanGradientTape(self):
+    if context.num_gpus() < 2: self.skipTest('Not enough GPUs')
+    distribution, target, config = self._get_test_object(None, None, num_gpus=2)
+    with self.cached_session(config=config, target=target):
+      self._test_all_reduce_mean_gradient_tape(distribution)
+
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
index 9cbc34412d..9821828b2d 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
@@ -66,8 +66,10 @@ GPU_TEST = "test_gpu" in sys.argv[0]
         combinations.core_mirrored_strategy_with_gpu_and_cpu,
         combinations.core_mirrored_strategy_with_two_gpus],
     mode=["graph", "eager"]))
-class MirroredTwoDeviceDistributionTest(strategy_test_lib.DistributionTestBase,
-                                        parameterized.TestCase):
+class MirroredTwoDeviceDistributionTest(
+    strategy_test_lib.DistributionTestBase,
+    strategy_test_lib.TwoDeviceDistributionTestBase,
+    parameterized.TestCase):
 
   def testMinimizeLoss(self, distribution):
     if context.executing_eagerly():
@@ -117,6 +119,24 @@ class MirroredTwoDeviceDistributionTest(strategy_test_lib.DistributionTestBase,
   def testGlobalStepUpdate(self, distribution):
     self._test_global_step_update(distribution)
 
+  def testAllReduceSum(self, distribution):
+    self._test_all_reduce_sum(distribution)
+
+  def testAllReduceSumGradients(self, distribution):
+    self._test_all_reduce_sum_gradients(distribution)
+
+  def testAllReduceSumGradientTape(self, distribution):
+    self._test_all_reduce_sum_gradient_tape(distribution)
+
+  def testAllReduceMean(self, distribution):
+    self._test_all_reduce_mean(distribution)
+
+  def testAllReduceMeanGradients(self, distribution):
+    self._test_all_reduce_mean_gradients(distribution)
+
+  def testAllReduceMeanGradientTape(self, distribution):
+    self._test_all_reduce_mean_gradient_tape(distribution)
+
 
 def one_device_combinations():
   return combinations.combine(
@@ -128,25 +148,42 @@ def one_device_combinations():
       mode=["graph", "eager"])
 
 
+@combinations.generate(one_device_combinations())
 class MirroredOneDeviceDistributionTest(
     strategy_test_lib.DistributionTestBase,
+    strategy_test_lib.OneDeviceDistributionTestBase,
     parameterized.TestCase):
 
-  @combinations.generate(one_device_combinations())
   def testMinimizeLoss(self, distribution):
     if context.executing_eagerly():
       self._test_minimize_loss_eager(distribution)
     else:
       self._test_minimize_loss_graph(distribution)
 
-  @combinations.generate(one_device_combinations())
   def testReplicaId(self, distribution):
     self._test_replica_id(distribution)
 
-  @combinations.generate(one_device_combinations())
   def testCallAndMergeExceptions(self, distribution):
     self._test_call_and_merge_exceptions(distribution)
 
+  def testAllReduceSum(self, distribution):
+    self._test_all_reduce_sum(distribution)
+
+  def testAllReduceSumGradients(self, distribution):
+    self._test_all_reduce_sum_gradients(distribution)
+
+  def testAllReduceSumGradientTape(self, distribution):
+    self._test_all_reduce_sum_gradient_tape(distribution)
+
+  def testAllReduceMean(self, distribution):
+    self._test_all_reduce_mean(distribution)
+
+  def testAllReduceMeanGradients(self, distribution):
+    self._test_all_reduce_mean_gradients(distribution)
+
+  def testAllReduceMeanGradientTape(self, distribution):
+    self._test_all_reduce_mean_gradient_tape(distribution)
+
 
 class MirroredStrategyVariableCreatorStackTest(
     test.TestCase, parameterized.TestCase):
diff --git a/tensorflow/contrib/distribute/python/one_device_strategy_test.py b/tensorflow/contrib/distribute/python/one_device_strategy_test.py
index d46cd6f529..2403dc8f12 100644
--- a/tensorflow/contrib/distribute/python/one_device_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/one_device_strategy_test.py
@@ -25,7 +25,9 @@ from tensorflow.python.eager import test
 from tensorflow.python.framework import test_util
 
 
-class OneDeviceStrategyTest(strategy_test_lib.DistributionTestBase):
+class OneDeviceStrategyTest(
+    strategy_test_lib.DistributionTestBase,
+    strategy_test_lib.OneDeviceDistributionTestBase):
 
   def _get_distribution_strategy(self):
     return one_device_strategy.OneDeviceStrategy("/device:CPU:0")
@@ -57,6 +59,24 @@ class OneDeviceStrategyTest(strategy_test_lib.DistributionTestBase):
     self._test_input_fn_iterator(
         iterator, d.extended.worker_devices, expected_values)
 
+  def testAllReduceSum(self):
+    self._test_all_reduce_sum(self._get_distribution_strategy())
+
+  def testAllReduceSumGradients(self):
+    self._test_all_reduce_sum_gradients(self._get_distribution_strategy())
+
+  def testAllReduceSumGradientTape(self):
+    self._test_all_reduce_sum_gradient_tape(self._get_distribution_strategy())
+
+  def testAllReduceMean(self):
+    self._test_all_reduce_mean(self._get_distribution_strategy())
+
+  def testAllReduceMeanGradients(self):
+    self._test_all_reduce_mean_gradients(self._get_distribution_strategy())
+
+  def testAllReduceMeanGradientTape(self):
+    self._test_all_reduce_mean_gradient_tape(self._get_distribution_strategy())
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
index d8c6c50db2..7836687e7d 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
@@ -603,9 +603,11 @@ class ParameterServerStrategyTestBase(
         self.assertEqual(expected_value, computed_value)
 
 
-class ParameterServerStrategyTest(ParameterServerStrategyTestBase,
-                                  strategy_test_lib.DistributionTestBase,
-                                  parameterized.TestCase):
+class ParameterServerStrategyTest(
+    ParameterServerStrategyTestBase,
+    strategy_test_lib.DistributionTestBase,
+    strategy_test_lib.TwoDeviceDistributionTestBase,
+    parameterized.TestCase):
 
   @classmethod
   def setUpClass(cls):
@@ -782,6 +784,36 @@ class ParameterServerStrategyTest(ParameterServerStrategyTestBase,
     # Verify isolate_session_state
     self.assertTrue(new_config.isolate_session_state)
 
+  def testAllReduceSum(self):
+    distribution = parameter_server_strategy.ParameterServerStrategy(
+        num_gpus_per_worker=2)
+    self._test_all_reduce_sum(distribution)
+
+  def testAllReduceSumGradients(self):
+    distribution = parameter_server_strategy.ParameterServerStrategy(
+        num_gpus_per_worker=2)
+    self._test_all_reduce_sum_gradients(distribution)
+
+  def testAllReduceSumGradientTape(self):
+    distribution = parameter_server_strategy.ParameterServerStrategy(
+        num_gpus_per_worker=2)
+    self._test_all_reduce_sum_gradient_tape(distribution)
+
+  def testAllReduceMean(self):
+    distribution = parameter_server_strategy.ParameterServerStrategy(
+        num_gpus_per_worker=2)
+    self._test_all_reduce_mean(distribution)
+
+  def testAllReduceMeanGradients(self):
+    distribution = parameter_server_strategy.ParameterServerStrategy(
+        num_gpus_per_worker=2)
+    self._test_all_reduce_mean_gradients(distribution)
+
+  def testAllReduceMeanGradientTape(self):
+    distribution = parameter_server_strategy.ParameterServerStrategy(
+        num_gpus_per_worker=2)
+    self._test_all_reduce_mean_gradient_tape(distribution)
+
 
 class ParameterServerStrategyWithChiefTest(ParameterServerStrategyTestBase,
                                            parameterized.TestCase):
diff --git a/tensorflow/contrib/distribute/python/strategy_test_lib.py b/tensorflow/contrib/distribute/python/strategy_test_lib.py
index 6e5280e356..4fbd630cf7 100644
--- a/tensorflow/contrib/distribute/python/strategy_test_lib.py
+++ b/tensorflow/contrib/distribute/python/strategy_test_lib.py
@@ -19,6 +19,7 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.core.protobuf import config_pb2
+from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.distribute import distribution_strategy_context as ds_context
 from tensorflow.python.distribute import reduce_util
 from tensorflow.python.distribute import values
@@ -31,6 +32,7 @@ from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.layers import core
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
@@ -292,3 +294,163 @@ class DistributionTestBase(test.TestCase):
       global_step_tensors = strategy.unwrap(value)
       global_step_values = self.evaluate(global_step_tensors)
       self.assertEqual((1,) * len(global_step_tensors), global_step_values)
+
+
+class OneDeviceDistributionTestBase(test.TestCase):
+  """Some tests that should work with any one-device DistributionStrategy."""
+
+  def _test_all_reduce_sum(self, strategy):
+    self._test_collective_comms(
+        strategy, _all_sum, inputs=(4., [42., 43.]), expected=(4., [42., 43.]))
+
+  def _test_all_reduce_sum_gradients(self, strategy):
+    self._test_collective_comms_gradients(
+        strategy, _all_sum, inputs=[4.], expected_grads=[4.])
+
+  def _test_all_reduce_sum_gradient_tape(self, strategy):
+    self._test_collective_comms_gradient_tape(
+        strategy, _all_sum, inputs=[4.], expected_grads=[4.])
+
+  def _test_all_reduce_mean(self, strategy):
+    self._test_collective_comms(
+        strategy, _all_mean, inputs=(2., [21., 22.]), expected=(2., [21., 22.]))
+
+  def _test_all_reduce_mean_gradients(self, strategy):
+    self._test_collective_comms_gradients(
+        strategy, _all_mean, inputs=[5.], expected_grads=[5.])
+
+  def _test_all_reduce_mean_gradient_tape(self, strategy):
+    self._test_collective_comms_gradient_tape(
+        strategy, _all_mean, inputs=[5.], expected_grads=[5.])
+
+  def _test_collective_comms(self, strategy, comm_fn, inputs, expected):
+    inputs = strategy.make_input_fn_iterator(
+        lambda _: dataset_ops.Dataset.from_tensors(inputs))
+
+    self.evaluate(inputs.initialize())
+    outputs = self.evaluate(
+        list(map(strategy.unwrap, strategy.experimental_run(comm_fn, inputs))))
+    self.assertAllEqual([expected[0]], outputs[0])
+    self.assertAllEqual([expected[1]], outputs[1])
+
+  def _test_collective_comms_gradients(
+      self, strategy, comm_fn, inputs, expected_grads):
+    if context.executing_eagerly():
+      self.skipTest("`tf.gradients` is not supported with eager execution.")
+
+    def step(c):
+      x = constant_op.constant(42.)
+      y = comm_fn(x) * c
+      return gradients_impl.gradients(y, [x])[0]
+
+    inputs = strategy.make_input_fn_iterator(
+        lambda _: dataset_ops.Dataset.from_tensors(inputs))
+
+    self.evaluate(inputs.initialize())
+    self.assertAllEqual(
+        expected_grads,
+        self.evaluate(strategy.unwrap(strategy.experimental_run(step, inputs))))
+
+  def _test_collective_comms_gradient_tape(
+      self, strategy, comm_fn, inputs, expected_grads):
+    def step(c):
+      x = constant_op.constant(42.)
+      with backprop.GradientTape() as tape:
+        tape.watch(x)
+        y = comm_fn(x) * c
+      return tape.gradient(y, x)
+
+    inputs = strategy.make_input_fn_iterator(
+        lambda _: dataset_ops.Dataset.from_tensors(inputs))
+
+    self.evaluate(inputs.initialize())
+    self.assertAllEqual(
+        expected_grads,
+        self.evaluate(strategy.unwrap(strategy.experimental_run(step, inputs))))
+
+
+class TwoDeviceDistributionTestBase(test.TestCase):
+  """Some tests that should work with any two-device DistributionStrategy."""
+
+  def _test_all_reduce_sum(self, strategy):
+    self._test_collective_comms(
+        strategy, _all_sum,
+        inputs=([1., 3.], [[39., 2.], [3., 41.]]),
+        expected=(4., [42., 43.]))
+
+  def _test_all_reduce_sum_gradients(self, strategy):
+    self._test_collective_comms_gradients(
+        strategy, _all_sum, inputs=[1., 3.], expected_grads=[4., 4.])
+
+  def _test_all_reduce_sum_gradient_tape(self, strategy):
+    self._test_collective_comms_gradient_tape(
+        strategy, _all_sum, inputs=[1., 3.], expected_grads=[4., 4.])
+
+  def _test_all_reduce_mean(self, strategy):
+    self._test_collective_comms(
+        strategy, _all_mean,
+        inputs=([1., 3.], [[39., 2.], [3., 41.]]),
+        expected=(2., [21., 21.5]))
+
+  def _test_all_reduce_mean_gradients(self, strategy):
+    self._test_collective_comms_gradients(
+        strategy, _all_mean, inputs=[1., 3.], expected_grads=[2., 2.])
+
+  def _test_all_reduce_mean_gradient_tape(self, strategy):
+    self._test_collective_comms_gradient_tape(
+        strategy, _all_mean, inputs=[1., 3.], expected_grads=[2., 2.])
+
+  def _test_collective_comms(self, strategy, comm_fn, inputs, expected):
+    inputs = strategy.make_input_fn_iterator(
+        lambda _: dataset_ops.Dataset.from_tensor_slices(inputs))
+
+    self.evaluate(inputs.initialize())
+    outputs = self.evaluate(
+        list(map(strategy.unwrap, strategy.experimental_run(comm_fn, inputs))))
+    self.assertAllEqual([expected[0], expected[0]], outputs[0])
+    self.assertAllEqual([expected[1], expected[1]], outputs[1])
+
+  def _test_collective_comms_gradients(
+      self, strategy, comm_fn, inputs, expected_grads):
+    if context.executing_eagerly():
+      self.skipTest("`tf.gradients` is not supported with eager execution.")
+
+    def step(c):
+      x = constant_op.constant(42.)
+      y = comm_fn(x) * c
+      return gradients_impl.gradients(y, [x])[0]
+
+    inputs = strategy.make_input_fn_iterator(
+        lambda _: dataset_ops.Dataset.from_tensor_slices(inputs))
+
+    self.evaluate(inputs.initialize())
+    self.assertAllEqual(
+        expected_grads,
+        self.evaluate(strategy.unwrap(strategy.experimental_run(step, inputs))))
+
+  def _test_collective_comms_gradient_tape(
+      self, strategy, comm_fn, inputs, expected_grads):
+    def step(c):
+      x = constant_op.constant(42.)
+      with backprop.GradientTape() as tape:
+        tape.watch(x)
+        y = comm_fn(x) * c
+      return tape.gradient(y, x)
+
+    inputs = strategy.make_input_fn_iterator(
+        lambda _: dataset_ops.Dataset.from_tensor_slices(inputs))
+
+    self.evaluate(inputs.initialize())
+    self.assertAllEqual(
+        expected_grads,
+        self.evaluate(strategy.unwrap(strategy.experimental_run(step, inputs))))
+
+
+def _all_sum(value):
+  ctx = ds_context.get_replica_context()
+  return ctx.all_reduce(reduce_util.ReduceOp.SUM, value)
+
+
+def _all_mean(value):
+  ctx = ds_context.get_replica_context()
+  return ctx.all_reduce(reduce_util.ReduceOp.MEAN, value)
diff --git a/tensorflow/python/distribute/distribute_lib.py b/tensorflow/python/distribute/distribute_lib.py
index 76cbdd53d9..4ad8cc00b8 100644
--- a/tensorflow/python/distribute/distribute_lib.py
+++ b/tensorflow/python/distribute/distribute_lib.py
@@ -33,6 +33,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import custom_gradient
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops.losses import losses_impl
@@ -1554,6 +1555,50 @@ class ReplicaContext(object):
     require_replica_context(self)
     return (device_util.current(),)
 
+  def all_reduce(self, reduce_op, value):
+    """All-reduces the given `Tensor` nest across replicas.
+
+    If `all_reduce` is called in any replica, it must be called in all replicas.
+    The nested structure and `Tensor` shapes must be identical in all replicas.
+
+    IMPORTANT: The ordering of communications must be identical in all replicas.
+
+    Example with two replicas:
+      Replica 0 `value`: {'a': 1, 'b': [40,  1]}
+      Replica 1 `value`: {'a': 3, 'b': [ 2, 98]}
+
+      If `reduce_op` == `SUM`:
+        Result (on all replicas): {'a': 4, 'b': [42, 99]}
+
+      If `reduce_op` == `MEAN`:
+        Result (on all replicas): {'a': 2, 'b': [21, 49.5]}
+
+    Args:
+      reduce_op: Reduction type, an instance of `tf.distribute.ReduceOp` enum.
+      value: The nested structure of `Tensor`s to all-reduced.
+        The structure must be compatible with `tf.nest`.
+
+    Returns:
+       A `Tensor` nest with the reduced `value`s from each replica.
+    """
+    def batch_all_reduce(strategy, *value_flat):
+      return strategy.extended.batch_reduce_to(
+          reduce_op, [(v, _batch_reduce_destination(v)) for v in value_flat])
+
+    if reduce_op in [reduce_util.ReduceOp.SUM, reduce_util.ReduceOp.MEAN]:
+      # TODO(cjfj): Work out why `batch_reduce` doesn't return the correct grad.
+      @custom_gradient.custom_gradient
+      def grad_wrapper(*xs):
+        ys = self.merge_call(batch_all_reduce, args=xs)
+        # The gradient of an all-sum is itself an all-sum (all-mean, likewise).
+        return ys, lambda *dy_s: self.all_reduce(reduce_op, dy_s)
+      return nest.pack_sequence_as(value, grad_wrapper(*nest.flatten(value)))
+    else:
+      # TODO(cjfj): Implement gradients for other reductions.
+      reduced = nest.pack_sequence_as(
+          value, self.merge_call(batch_all_reduce, args=nest.flatten(value)))
+      return nest.map_structure(array_ops.prevent_gradient, reduced)
+
   # TODO(josh11b): Implement `start_all_reduce(method, t)` for efficient
   # all-reduce. It would return a function returning the result of reducing `t`
   # across all replicas. The caller would wait to call this function until they
@@ -1564,6 +1609,15 @@ class ReplicaContext(object):
   #   to that point that the first result is needed. Most likely this can be
   #   implemented in terms of `merge_call()` and `batch_reduce_to()`.
 
+
+def _batch_reduce_destination(x):
+  """Returns the destinations for batch all-reduce."""
+  if isinstance(x, ops.Tensor):  # One device strategies.
+    return x.device
+  else:
+    return x
+
+
 # ------------------------------------------------------------------------------
 
 
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-replica-context.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-replica-context.pbtxt
index df707e8920..22f8160c96 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-replica-context.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-replica-context.pbtxt
@@ -26,6 +26,10 @@ tf_class {
     name: "__init__"
     argspec: "args=[\'self\', \'strategy\', \'replica_id_in_sync_group\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "all_reduce"
+    argspec: "args=[\'self\', \'reduce_op\', \'value\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "merge_call"
     argspec: "args=[\'self\', \'merge_fn\', \'args\', \'kwargs\'], varargs=None, keywords=None, defaults=[\'()\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-replica-context.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-replica-context.pbtxt
index df707e8920..22f8160c96 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-replica-context.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-replica-context.pbtxt
@@ -26,6 +26,10 @@ tf_class {
     name: "__init__"
     argspec: "args=[\'self\', \'strategy\', \'replica_id_in_sync_group\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "all_reduce"
+    argspec: "args=[\'self\', \'reduce_op\', \'value\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "merge_call"
     argspec: "args=[\'self\', \'merge_fn\', \'args\', \'kwargs\'], varargs=None, keywords=None, defaults=[\'()\', \'None\'], "
-- 
GitLab


From 0ef4b190443a2f7b3b2330f053df5503af6b0191 Mon Sep 17 00:00:00 2001
From: James Keeling <jtkeeling@google.com>
Date: Tue, 8 Jan 2019 02:53:00 -0800
Subject: [PATCH 0365/1095] Split topk GPU code into multiple files

This file was a bottleneck during compilation, often taking many minutes to compile. In local testing this change reduces the wall-clock build time for the topk GPU kernels from 155s to 48s.

PiperOrigin-RevId: 228302319
---
 tensorflow/core/kernels/BUILD                 | 16 ++++++++++-
 .../{topk_op_gpu.cu.cc => topk_op_gpu.h}      | 12 +++-----
 .../core/kernels/topk_op_gpu_double.cu.cc     | 28 +++++++++++++++++++
 .../core/kernels/topk_op_gpu_float.cu.cc      | 28 +++++++++++++++++++
 .../core/kernels/topk_op_gpu_half.cu.cc       | 28 +++++++++++++++++++
 .../core/kernels/topk_op_gpu_int16.cu.cc      | 28 +++++++++++++++++++
 .../core/kernels/topk_op_gpu_int32.cu.cc      | 28 +++++++++++++++++++
 .../core/kernels/topk_op_gpu_int64.cu.cc      | 28 +++++++++++++++++++
 .../core/kernels/topk_op_gpu_int8.cu.cc       | 28 +++++++++++++++++++
 .../core/kernels/topk_op_gpu_uint16.cu.cc     | 28 +++++++++++++++++++
 .../core/kernels/topk_op_gpu_uint8.cu.cc      | 28 +++++++++++++++++++
 11 files changed, 271 insertions(+), 9 deletions(-)
 rename tensorflow/core/kernels/{topk_op_gpu.cu.cc => topk_op_gpu.h} (98%)
 create mode 100644 tensorflow/core/kernels/topk_op_gpu_double.cu.cc
 create mode 100644 tensorflow/core/kernels/topk_op_gpu_float.cu.cc
 create mode 100644 tensorflow/core/kernels/topk_op_gpu_half.cu.cc
 create mode 100644 tensorflow/core/kernels/topk_op_gpu_int16.cu.cc
 create mode 100644 tensorflow/core/kernels/topk_op_gpu_int32.cu.cc
 create mode 100644 tensorflow/core/kernels/topk_op_gpu_int64.cu.cc
 create mode 100644 tensorflow/core/kernels/topk_op_gpu_int8.cu.cc
 create mode 100644 tensorflow/core/kernels/topk_op_gpu_uint16.cu.cc
 create mode 100644 tensorflow/core/kernels/topk_op_gpu_uint8.cu.cc

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index fee30fa6c3..3df63d34e3 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -3862,7 +3862,21 @@ tf_kernel_library(
 
 tf_kernel_library(
     name = "topk_op",
-    prefix = "topk_op",
+    srcs = ["topk_op.cc"],
+    hdrs = ["topk_op.h"],
+    gpu_srcs = [
+        "topk_op.h",
+        "topk_op_gpu.h",
+        "topk_op_gpu_double.cu.cc",
+        "topk_op_gpu_float.cu.cc",
+        "topk_op_gpu_half.cu.cc",
+        "topk_op_gpu_int64.cu.cc",
+        "topk_op_gpu_int32.cu.cc",
+        "topk_op_gpu_int16.cu.cc",
+        "topk_op_gpu_uint16.cu.cc",
+        "topk_op_gpu_int8.cu.cc",
+        "topk_op_gpu_uint8.cu.cc",
+    ],
     deps = NN_DEPS + if_cuda(["@cub_archive//:cub"]),
 )
 
diff --git a/tensorflow/core/kernels/topk_op_gpu.cu.cc b/tensorflow/core/kernels/topk_op_gpu.h
similarity index 98%
rename from tensorflow/core/kernels/topk_op_gpu.cu.cc
rename to tensorflow/core/kernels/topk_op_gpu.h
index 2fbe1fe7cb..6f3bec20f6 100644
--- a/tensorflow/core/kernels/topk_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/topk_op_gpu.h
@@ -12,6 +12,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
+#ifndef TENSORFLOW_CORE_KERNELS_TOPK_OP_GPU_H_
+#define TENSORFLOW_CORE_KERNELS_TOPK_OP_GPU_H_
 
 #if GOOGLE_CUDA
 
@@ -561,14 +563,8 @@ struct TopKFunctor<GPUDevice, T> {
 };
 
 }  // end namespace functor
-
-#define INSTANTIATE_TEMPLATE(type) \
-  template struct functor::TopKFunctor<GPUDevice, type>;
-
-TF_CALL_GPU_NUMBER_TYPES(INSTANTIATE_TEMPLATE);
-TF_CALL_INTEGRAL_TYPES(INSTANTIATE_TEMPLATE);
-#undef INSTANTIATE_TEMPLATE
-
 }  // namespace tensorflow
 
 #endif  // GOOGLE_CUDA
+
+#endif  // TENSORFLOW_CORE_KERNELS_TOPK_OP_GPU_H_
diff --git a/tensorflow/core/kernels/topk_op_gpu_double.cu.cc b/tensorflow/core/kernels/topk_op_gpu_double.cu.cc
new file mode 100644
index 0000000000..8a5a7e71b1
--- /dev/null
+++ b/tensorflow/core/kernels/topk_op_gpu_double.cu.cc
@@ -0,0 +1,28 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/topk_op.h"
+#include "tensorflow/core/kernels/topk_op_gpu.h"
+
+namespace tensorflow {
+using Eigen::GpuDevice;
+
+template struct functor::TopKFunctor<GPUDevice, double>;
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/topk_op_gpu_float.cu.cc b/tensorflow/core/kernels/topk_op_gpu_float.cu.cc
new file mode 100644
index 0000000000..0b69396bb1
--- /dev/null
+++ b/tensorflow/core/kernels/topk_op_gpu_float.cu.cc
@@ -0,0 +1,28 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/topk_op.h"
+#include "tensorflow/core/kernels/topk_op_gpu.h"
+
+namespace tensorflow {
+using Eigen::GpuDevice;
+
+template struct functor::TopKFunctor<GPUDevice, float>;
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/topk_op_gpu_half.cu.cc b/tensorflow/core/kernels/topk_op_gpu_half.cu.cc
new file mode 100644
index 0000000000..e53586aeca
--- /dev/null
+++ b/tensorflow/core/kernels/topk_op_gpu_half.cu.cc
@@ -0,0 +1,28 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/topk_op.h"
+#include "tensorflow/core/kernels/topk_op_gpu.h"
+
+namespace tensorflow {
+using Eigen::GpuDevice;
+
+template struct functor::TopKFunctor<GPUDevice, Eigen::half>;
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/topk_op_gpu_int16.cu.cc b/tensorflow/core/kernels/topk_op_gpu_int16.cu.cc
new file mode 100644
index 0000000000..5bd310523c
--- /dev/null
+++ b/tensorflow/core/kernels/topk_op_gpu_int16.cu.cc
@@ -0,0 +1,28 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/topk_op.h"
+#include "tensorflow/core/kernels/topk_op_gpu.h"
+
+namespace tensorflow {
+using Eigen::GpuDevice;
+
+template struct functor::TopKFunctor<GPUDevice, int16>;
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/topk_op_gpu_int32.cu.cc b/tensorflow/core/kernels/topk_op_gpu_int32.cu.cc
new file mode 100644
index 0000000000..55b393a0c0
--- /dev/null
+++ b/tensorflow/core/kernels/topk_op_gpu_int32.cu.cc
@@ -0,0 +1,28 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/topk_op.h"
+#include "tensorflow/core/kernels/topk_op_gpu.h"
+
+namespace tensorflow {
+using Eigen::GpuDevice;
+
+template struct functor::TopKFunctor<GPUDevice, int32>;
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/topk_op_gpu_int64.cu.cc b/tensorflow/core/kernels/topk_op_gpu_int64.cu.cc
new file mode 100644
index 0000000000..3e4a775056
--- /dev/null
+++ b/tensorflow/core/kernels/topk_op_gpu_int64.cu.cc
@@ -0,0 +1,28 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/topk_op.h"
+#include "tensorflow/core/kernels/topk_op_gpu.h"
+
+namespace tensorflow {
+using Eigen::GpuDevice;
+
+template struct functor::TopKFunctor<GPUDevice, int64>;
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/topk_op_gpu_int8.cu.cc b/tensorflow/core/kernels/topk_op_gpu_int8.cu.cc
new file mode 100644
index 0000000000..ac73cd170b
--- /dev/null
+++ b/tensorflow/core/kernels/topk_op_gpu_int8.cu.cc
@@ -0,0 +1,28 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/topk_op.h"
+#include "tensorflow/core/kernels/topk_op_gpu.h"
+
+namespace tensorflow {
+using Eigen::GpuDevice;
+
+template struct functor::TopKFunctor<GPUDevice, int8>;
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/topk_op_gpu_uint16.cu.cc b/tensorflow/core/kernels/topk_op_gpu_uint16.cu.cc
new file mode 100644
index 0000000000..8d5f8ceb06
--- /dev/null
+++ b/tensorflow/core/kernels/topk_op_gpu_uint16.cu.cc
@@ -0,0 +1,28 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/topk_op.h"
+#include "tensorflow/core/kernels/topk_op_gpu.h"
+
+namespace tensorflow {
+using Eigen::GpuDevice;
+
+template struct functor::TopKFunctor<GPUDevice, uint16>;
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/topk_op_gpu_uint8.cu.cc b/tensorflow/core/kernels/topk_op_gpu_uint8.cu.cc
new file mode 100644
index 0000000000..fc1a8a2c8c
--- /dev/null
+++ b/tensorflow/core/kernels/topk_op_gpu_uint8.cu.cc
@@ -0,0 +1,28 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/topk_op.h"
+#include "tensorflow/core/kernels/topk_op_gpu.h"
+
+namespace tensorflow {
+using Eigen::GpuDevice;
+
+template struct functor::TopKFunctor<GPUDevice, uint8>;
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
-- 
GitLab


From 6c74d3a5893ff6e3ba35b471841ccc349848587d Mon Sep 17 00:00:00 2001
From: Siju <siju.samuel@huawei.com>
Date: Tue, 8 Jan 2019 17:01:58 +0530
Subject: [PATCH 0366/1095] Ensure variables are initialized

'ReadVarint32FromArray' & 'ReadVarint64FromArray' doesn't ensure output variable 'value' is filled always. Since 'unused_ok' is not checked for true/false initialize the temp variables, so that random values dont propogate.
---
 tensorflow/core/util/proto/decode.h | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/tensorflow/core/util/proto/decode.h b/tensorflow/core/util/proto/decode.h
index 8dde14dffc..188830cc1f 100644
--- a/tensorflow/core/util/proto/decode.h
+++ b/tensorflow/core/util/proto/decode.h
@@ -91,7 +91,7 @@ inline const uint8* ReadVarint64FromArray(const uint8* buffer, bool* ok,
 // the 64 bit version instead of copying the code.
 inline const uint8* ReadVarint32FromArray(const uint8* buffer, bool* ok,
                                           uint32* value) {
-  uint64 tmp;
+  uint64 tmp = 0;
   const uint8* buf = ReadVarint64FromArray(buffer, ok, &tmp);
   *value = tmp & 0xffffffff;
   return buf;
@@ -106,7 +106,7 @@ const uint8* ReadFromArray(const uint8* buf, TensorType* value);
 template <>
 inline const uint8* ReadFromArray<int64, WireFormatLite::TYPE_INT32>(
     const uint8* buf, int64* value) {
-  uint32 temp;
+  uint32 temp = 0;
   bool unused_ok;  // The Counting pass would have failed if this were corrupt.
   buf = ReadVarint32FromArray(buf, &unused_ok, &temp);
   *value = static_cast<int64>(temp);
@@ -116,7 +116,7 @@ inline const uint8* ReadFromArray<int64, WireFormatLite::TYPE_INT32>(
 template <>
 inline const uint8* ReadFromArray<int32, WireFormatLite::TYPE_INT32>(
     const uint8* buf, int32* value) {
-  uint32 temp;
+  uint32 temp = 0;
   bool unused_ok;  // The Counting pass would have failed if this were corrupt.
   buf = ReadVarint32FromArray(buf, &unused_ok, &temp);
   *value = static_cast<int32>(temp);
@@ -126,7 +126,7 @@ inline const uint8* ReadFromArray<int32, WireFormatLite::TYPE_INT32>(
 template <>
 inline const uint8* ReadFromArray<int64, WireFormatLite::TYPE_INT64>(
     const uint8* buf, int64* value) {
-  uint64 temp;
+  uint64 temp = 0;
   bool unused_ok;  // The Counting pass would have failed if this were corrupt.
   buf = ReadVarint64FromArray(buf, &unused_ok, &temp);
   *value = WrapUnsignedAsSigned64(temp);
@@ -136,7 +136,7 @@ inline const uint8* ReadFromArray<int64, WireFormatLite::TYPE_INT64>(
 template <>
 inline const uint8* ReadFromArray<uint64, WireFormatLite::TYPE_UINT32>(
     const uint8* buf, uint64* value) {
-  uint32 temp;
+  uint32 temp = 0;
   bool unused_ok;  // The Counting pass would have failed if this were corrupt.
   buf = ReadVarint32FromArray(buf, &unused_ok, &temp);
   *value = temp;
@@ -160,7 +160,7 @@ inline const uint8* ReadFromArray<uint64, WireFormatLite::TYPE_UINT64>(
 template <>
 inline const uint8* ReadFromArray<int64, WireFormatLite::TYPE_SINT32>(
     const uint8* buf, int64* value) {
-  uint64 temp;
+  uint64 temp = 0;
   bool unused_ok;  // The Counting pass would have failed if this were corrupt.
   buf = ReadVarint64FromArray(buf, &unused_ok, &temp);
   *value = WireFormatLite::ZigZagDecode32(temp);
@@ -170,7 +170,7 @@ inline const uint8* ReadFromArray<int64, WireFormatLite::TYPE_SINT32>(
 template <>
 inline const uint8* ReadFromArray<int32, WireFormatLite::TYPE_SINT32>(
     const uint8* buf, int32* value) {
-  uint32 temp;
+  uint32 temp = 0;
   bool unused_ok;  // The Counting pass would have failed if this were corrupt.
   buf = ReadVarint32FromArray(buf, &unused_ok, &temp);
   *value = WireFormatLite::ZigZagDecode32(temp);
@@ -180,7 +180,7 @@ inline const uint8* ReadFromArray<int32, WireFormatLite::TYPE_SINT32>(
 template <>
 inline const uint8* ReadFromArray<int64, WireFormatLite::TYPE_SINT64>(
     const uint8* buf, int64* value) {
-  uint64 temp;
+  uint64 temp = 0;
   bool unused_ok;  // The Counting pass would have failed if this were corrupt.
   buf = ReadVarint64FromArray(buf, &unused_ok, &temp);
   *value = WireFormatLite::ZigZagDecode64(temp);
@@ -280,7 +280,7 @@ inline const uint8* ReadFromArray<double, WireFormatLite::TYPE_DOUBLE>(
 template <>
 inline const uint8* ReadFromArray<bool, WireFormatLite::TYPE_BOOL>(
     const uint8* buf, bool* value) {
-  uint64 temp;
+  uint64 temp = 0;
   bool unused_ok;  // The Counting pass would have failed if this were corrupt.
   buf = ReadVarint64FromArray(buf, &unused_ok, &temp);
   *value = temp != 0;
@@ -290,7 +290,7 @@ inline const uint8* ReadFromArray<bool, WireFormatLite::TYPE_BOOL>(
 template <>
 inline const uint8* ReadFromArray<int, WireFormatLite::TYPE_ENUM>(
     const uint8* buf, int* value) {
-  uint32 temp;
+  uint32 temp = 0;
   bool unused_ok;  // The Counting pass would have failed if this were corrupt.
   buf = ReadVarint32FromArray(buf, &unused_ok, &temp);
   *value = static_cast<int>(temp);
-- 
GitLab


From 3436665db2438dd339a0edddd0d0a3d49b5b2424 Mon Sep 17 00:00:00 2001
From: James Keeling <jtkeeling@google.com>
Date: Tue, 8 Jan 2019 03:28:39 -0800
Subject: [PATCH 0367/1095] Split scan ops GPU code into multiple files

This file was a bottleneck during compilation, often taking many minutes to compile. In local testing this change reduces the wall-clock build time for the scan ops GPU kernels from 107s to 96s.

PiperOrigin-RevId: 228304727
---
 tensorflow/core/kernels/BUILD                 | 10 +++++-
 .../{scan_ops_gpu.cu.cc => scan_ops_gpu.h}    | 16 +++-------
 .../core/kernels/scan_ops_gpu_double.cu.cc    | 31 +++++++++++++++++++
 .../core/kernels/scan_ops_gpu_float.cu.cc     | 31 +++++++++++++++++++
 .../core/kernels/scan_ops_gpu_half.cu.cc      | 31 +++++++++++++++++++
 5 files changed, 107 insertions(+), 12 deletions(-)
 rename tensorflow/core/kernels/{scan_ops_gpu.cu.cc => scan_ops_gpu.h} (97%)
 create mode 100644 tensorflow/core/kernels/scan_ops_gpu_double.cu.cc
 create mode 100644 tensorflow/core/kernels/scan_ops_gpu_float.cu.cc
 create mode 100644 tensorflow/core/kernels/scan_ops_gpu_half.cu.cc

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 3df63d34e3..b934c64bfc 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -3279,7 +3279,15 @@ tf_kernel_library(
 
 tf_kernel_library(
     name = "scan_ops",
-    prefix = "scan_ops",
+    srcs = ["scan_ops.cc"],
+    hdrs = ["scan_ops.h"],
+    gpu_srcs = [
+        "scan_ops.h",
+        "scan_ops_gpu.h",
+        "scan_ops_gpu_double.cu.cc",
+        "scan_ops_gpu_float.cu.cc",
+        "scan_ops_gpu_half.cu.cc",
+    ],
     deps = MATH_DEPS + if_cuda(["@cub_archive//:cub"]),
 )
 
diff --git a/tensorflow/core/kernels/scan_ops_gpu.cu.cc b/tensorflow/core/kernels/scan_ops_gpu.h
similarity index 97%
rename from tensorflow/core/kernels/scan_ops_gpu.cu.cc
rename to tensorflow/core/kernels/scan_ops_gpu.h
index ed66c02dc5..976b221540 100644
--- a/tensorflow/core/kernels/scan_ops_gpu.cu.cc
+++ b/tensorflow/core/kernels/scan_ops_gpu.h
@@ -13,6 +13,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#ifndef TENSORFLOW_CORE_KERNELS_SCAN_OPS_GPU_H_
+#define TENSORFLOW_CORE_KERNELS_SCAN_OPS_GPU_H_
+
 #if GOOGLE_CUDA
 
 #define EIGEN_USE_GPU
@@ -290,17 +293,8 @@ struct Scan<GPUDevice, Eigen::internal::ProdReducer<T>, T> {
 };
 
 }  // namespace functor
-
-#define DEFINE(REDUCER, T) template struct functor::Scan<GPUDevice, REDUCER, T>;
-
-#define DEFINE_FOR_ALL_REDUCERS(T)           \
-  DEFINE(Eigen::internal::SumReducer<T>, T); \
-  DEFINE(Eigen::internal::ProdReducer<T>, T);
-
-TF_CALL_GPU_NUMBER_TYPES(DEFINE_FOR_ALL_REDUCERS);
-#undef DEFINE_FOR_ALL_REDUCERS
-#undef DEFINE
-
 }  // end namespace tensorflow
 
 #endif  // GOOGLE_CUDA
+
+#endif  // TENSORFLOW_CORE_KERNELS_SCAN_OPS_GPU_H_
diff --git a/tensorflow/core/kernels/scan_ops_gpu_double.cu.cc b/tensorflow/core/kernels/scan_ops_gpu_double.cu.cc
new file mode 100644
index 0000000000..adce37e473
--- /dev/null
+++ b/tensorflow/core/kernels/scan_ops_gpu_double.cu.cc
@@ -0,0 +1,31 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/scan_ops.h"
+#include "tensorflow/core/kernels/scan_ops_gpu.h"
+
+namespace tensorflow {
+using Eigen::GpuDevice;
+template struct functor::Scan<GpuDevice, Eigen::internal::SumReducer<double>,
+                              double>;
+template struct functor::Scan<GpuDevice, Eigen::internal::ProdReducer<double>,
+                              double>;
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/scan_ops_gpu_float.cu.cc b/tensorflow/core/kernels/scan_ops_gpu_float.cu.cc
new file mode 100644
index 0000000000..b72415822d
--- /dev/null
+++ b/tensorflow/core/kernels/scan_ops_gpu_float.cu.cc
@@ -0,0 +1,31 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/scan_ops.h"
+#include "tensorflow/core/kernels/scan_ops_gpu.h"
+
+namespace tensorflow {
+using Eigen::GpuDevice;
+template struct functor::Scan<GpuDevice, Eigen::internal::SumReducer<float>,
+                              float>;
+template struct functor::Scan<GpuDevice, Eigen::internal::ProdReducer<float>,
+                              float>;
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/scan_ops_gpu_half.cu.cc b/tensorflow/core/kernels/scan_ops_gpu_half.cu.cc
new file mode 100644
index 0000000000..f9fb528be9
--- /dev/null
+++ b/tensorflow/core/kernels/scan_ops_gpu_half.cu.cc
@@ -0,0 +1,31 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/scan_ops.h"
+#include "tensorflow/core/kernels/scan_ops_gpu.h"
+
+namespace tensorflow {
+using Eigen::GpuDevice;
+template struct functor::Scan<
+    GpuDevice, Eigen::internal::SumReducer<Eigen::half>, Eigen::half>;
+template struct functor::Scan<
+    GpuDevice, Eigen::internal::ProdReducer<Eigen::half>, Eigen::half>;
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
-- 
GitLab


From 137c142341c9f10a1e105403fbfb1123a1802eb9 Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Tue, 8 Jan 2019 03:57:48 -0800
Subject: [PATCH 0368/1095] Make toco_port trivial initialize-able for test
 cases.

Problem is toco_port indirects a way of initializing google.
One impl has a static way of finding out if it is initialized (internal).
One impl has a non-static way to find out.

In the non-static way, test cases won't work since we initialize the tensorflow
google emulation elsewhere (gtest override)

PiperOrigin-RevId: 228306710
---
 tensorflow/lite/toco/BUILD                     | 2 ++
 tensorflow/lite/toco/import_tensorflow_test.cc | 2 ++
 tensorflow/lite/toco/toco_convert_test.cc      | 2 ++
 tensorflow/lite/toco/toco_port.cc              | 7 +++++++
 tensorflow/lite/toco/toco_port.h               | 4 ++++
 tensorflow/lite/toco/toco_port_test.cc         | 1 +
 tensorflow/lite/toco/tooling_util_test.cc      | 2 ++
 7 files changed, 20 insertions(+)

diff --git a/tensorflow/lite/toco/BUILD b/tensorflow/lite/toco/BUILD
index d77234c80a..40bceedd6a 100644
--- a/tensorflow/lite/toco/BUILD
+++ b/tensorflow/lite/toco/BUILD
@@ -342,6 +342,7 @@ tf_cc_test(
     name = "import_tensorflow_test",
     srcs = ["import_tensorflow_test.cc"],
     deps = [
+        ":toco_port",
         ":toco_tooling",
         "//tensorflow/core:framework",
         "//tensorflow/core:graph",
@@ -386,6 +387,7 @@ tf_cc_test(
     srcs = ["tooling_util_test.cc"],
     deps = [
         ":model",
+        ":toco_port",
         ":tooling_util",
         "//tensorflow/core:lib",
         "//tensorflow/lite/testing:util",
diff --git a/tensorflow/lite/toco/import_tensorflow_test.cc b/tensorflow/lite/toco/import_tensorflow_test.cc
index 70382f546b..de7f4cdb7e 100644
--- a/tensorflow/lite/toco/import_tensorflow_test.cc
+++ b/tensorflow/lite/toco/import_tensorflow_test.cc
@@ -13,6 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 #include "tensorflow/lite/toco/import_tensorflow.h"
+#include "tensorflow/lite/toco/toco_port.h"
 
 #include <gmock/gmock.h>
 #include <gtest/gtest.h>
@@ -569,5 +570,6 @@ TEST(ImportTest, UnsupportedOpWithMultipleOutputs) {
 int main(int argc, char** argv) {
   ::tflite::LogToStderr();
   ::testing::InitGoogleTest(&argc, argv);
+  ::toco::port::InitGoogleWasDoneElsewhere();
   return RUN_ALL_TESTS();
 }
diff --git a/tensorflow/lite/toco/toco_convert_test.cc b/tensorflow/lite/toco/toco_convert_test.cc
index 3730c53ae1..739b924607 100644
--- a/tensorflow/lite/toco/toco_convert_test.cc
+++ b/tensorflow/lite/toco/toco_convert_test.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #include <gmock/gmock.h>
 #include <gtest/gtest.h>
 #include "tensorflow/lite/testing/util.h"
+#include "tensorflow/lite/toco/toco_port.h"
 
 namespace toco {
 namespace {
@@ -176,5 +177,6 @@ TEST(TocoTest, TransientStringTensors) {
 int main(int argc, char** argv) {
   ::tflite::LogToStderr();
   ::testing::InitGoogleTest(&argc, argv);
+  ::toco::port::InitGoogleWasDoneElsewhere();
   return RUN_ALL_TESTS();
 }
diff --git a/tensorflow/lite/toco/toco_port.cc b/tensorflow/lite/toco/toco_port.cc
index fb8c1b8337..b222032e61 100644
--- a/tensorflow/lite/toco/toco_port.cc
+++ b/tensorflow/lite/toco/toco_port.cc
@@ -57,6 +57,11 @@ void InitGoogle(const char* usage, int* argc, char*** argv, bool remove_flags) {
   ::InitGoogle(usage, argc, argv, remove_flags);
 }
 
+void InitGoogleWasDoneElsewhere() {
+  // Nothing need be done since ::CheckInitGoogleIsDone() is aware of other
+  // possible initialization entry points.
+}
+
 void CheckInitGoogleIsDone(const char* message) {
   ::CheckInitGoogleIsDone(message);
 }
@@ -152,6 +157,8 @@ constexpr int kFileWriteFlags = O_CREAT | O_WRONLY;
 
 static bool port_initialized = false;
 
+void InitGoogleWasDoneElsewhere() { port_initialized = true; }
+
 void InitGoogle(const char* usage, int* argc, char*** argv, bool remove_flags) {
   if (!port_initialized) {
 #if defined(PLATFORM_GOOGLE)
diff --git a/tensorflow/lite/toco/toco_port.h b/tensorflow/lite/toco/toco_port.h
index 2f39e3d6d5..231612ecd4 100644
--- a/tensorflow/lite/toco/toco_port.h
+++ b/tensorflow/lite/toco/toco_port.h
@@ -55,6 +55,10 @@ double round(double x);
 namespace toco {
 namespace port {
 
+// Things like tests use other initialization routines that need control
+// of flags. However, for testing we still want to use toco_port.h facilities.
+// This function sets initialized flag trivially.
+void InitGoogleWasDoneElsewhere();
 void InitGoogle(const char* usage, int* argc, char*** argv, bool remove_flags);
 void CheckInitGoogleIsDone(const char* message);
 
diff --git a/tensorflow/lite/toco/toco_port_test.cc b/tensorflow/lite/toco/toco_port_test.cc
index d80d423ed7..997da58b8f 100644
--- a/tensorflow/lite/toco/toco_port_test.cc
+++ b/tensorflow/lite/toco/toco_port_test.cc
@@ -61,5 +61,6 @@ TEST(TocoPortTest, JoinPath) {
 int main(int argc, char** argv) {
   ::tflite::LogToStderr();
   ::testing::InitGoogleTest(&argc, argv);
+  ::toco::port::InitGoogleWasDoneElsewhere();
   return RUN_ALL_TESTS();
 }
diff --git a/tensorflow/lite/toco/tooling_util_test.cc b/tensorflow/lite/toco/tooling_util_test.cc
index e44b94b712..faa6fe412e 100644
--- a/tensorflow/lite/toco/tooling_util_test.cc
+++ b/tensorflow/lite/toco/tooling_util_test.cc
@@ -19,6 +19,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/lite/testing/util.h"
 #include "tensorflow/lite/toco/model.h"
+#include "tensorflow/lite/toco/toco_port.h"
 #include "tensorflow/lite/toco/tooling_util.h"
 
 namespace toco {
@@ -208,5 +209,6 @@ TEST(FusedActivationTest, DefaultsToUnfused) {
 int main(int argc, char** argv) {
   ::tflite::LogToStderr();
   ::testing::InitGoogleTest(&argc, argv);
+  ::toco::port::InitGoogleWasDoneElsewhere();
   return RUN_ALL_TESTS();
 }
-- 
GitLab


From c503c6254608a622b06340f1f7ddd8f31565ab5e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 8 Jan 2019 04:37:51 -0800
Subject: [PATCH 0369/1095] Small clean up in in saved_model/load_test.py

Temporary `function` in checkpointable under test is not needed in most examples
as `cycle` is passing signatures explicitly.

PiperOrigin-RevId: 228309220
---
 tensorflow/python/saved_model/load_test.py | 34 +++++++++-------------
 1 file changed, 14 insertions(+), 20 deletions(-)

diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index 9eac3e6555..a1c81d3f14 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -42,9 +42,6 @@ class LoadTest(test.TestCase):
 
   def test_structure_import(self):
     root = tracking.Checkpointable()
-    root.f = def_function.function(
-        lambda x: 2. * x,
-        input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)])
     root.dep_one = tracking.Checkpointable()
     root.dep_two = tracking.Checkpointable()
     root.dep_two.dep = tracking.Checkpointable()
@@ -52,19 +49,25 @@ class LoadTest(test.TestCase):
     imported = self.cycle(root)
     self.assertIs(imported.dep_three, imported.dep_two.dep)
     self.assertIsNot(imported.dep_one, imported.dep_two)
-    self.assertEqual(4., imported.f(constant_op.constant(2.)).numpy())
 
   def test_variables(self):
     root = tracking.Checkpointable()
     root.v1 = variables.Variable(1.)
     root.v2 = variables.Variable(2.)
-    root.f = def_function.function(
-        lambda x: root.v2 * x,
-        input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)])
     imported = self.cycle(root)
     self.assertEquals(imported.v1.numpy(), 1.0)
     self.assertEquals(imported.v2.numpy(), 2.0)
+
+  def test_capture_variables(self):
+    root = tracking.Checkpointable()
+    root.weights = variables.Variable(2.)
+    root.f = def_function.function(
+        lambda x: root.weights * x,
+        input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)])
+    imported = self.cycle(root)
     self.assertEqual(4., imported.f(constant_op.constant(2.)).numpy())
+    imported.weights.assign(4.0)
+    self.assertEqual(8., imported.f(constant_op.constant(2.)).numpy())
 
   def _make_asset(self, contents):
     filename = tempfile.mktemp(prefix=self.get_temp_dir())
@@ -72,19 +75,16 @@ class LoadTest(test.TestCase):
       f.write(contents)
     return filename
 
-  def test_assets_import(self):
+  def test_assets(self):
     file1 = self._make_asset("contents 1")
     file2 = self._make_asset("contents 2")
 
     root = tracking.Checkpointable()
-    root.f = def_function.function(
-        lambda x: 2. * x,
-        input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)])
     root.asset1 = tracking.TrackableAsset(file1)
     root.asset2 = tracking.TrackableAsset(file2)
 
     save_dir = os.path.join(self.get_temp_dir(), "save_dir")
-    save.save(root, save_dir)
+    save.save(root, save_dir, signatures={})
 
     file_io.delete_file(file1)
     file_io.delete_file(file2)
@@ -110,18 +110,12 @@ class LoadTest(test.TestCase):
     with open(imported_output, "r") as f:
       self.assertEquals("contents", f.read())
 
-  def test_assets_dedup(self):
+  def test_dedup_assets(self):
     vocab = self._make_asset("contents")
     root = tracking.Checkpointable()
-    root.f = def_function.function(
-        lambda x: 2. * x,
-        input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)])
-
     root.asset1 = tracking.TrackableAsset(vocab)
     root.asset2 = tracking.TrackableAsset(vocab)
-
     imported = self.cycle(root)
-
     self.assertEqual(imported.asset1.asset_path.numpy(),
                      imported.asset2.asset_path.numpy())
 
@@ -154,7 +148,7 @@ class LoadTest(test.TestCase):
     imported = self.cycle(root)
     self.assertEqual(4., imported.f(constant_op.constant(2.0)).numpy())
 
-  def test_nested_func(self):
+  def test_nested_functions(self):
     f = def_function.function(
         lambda x: x*2.0,
         input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)])
-- 
GitLab


From 114f0d81404e7f9a93297f11ebc2fbceee8dbf7c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 8 Jan 2019 05:20:40 -0800
Subject: [PATCH 0370/1095] Add fine-tune capabilities to object based saved
 model save/load.

- Keep "trainable" bit of variables.
- Wire variables in restored FuncGraphs so they are automaticaly
  captured by GradientTapes.

PiperOrigin-RevId: 228311974
---
 tensorflow/python/eager/function.py           |  2 +-
 tensorflow/python/saved_model/load.py         |  8 +++++-
 tensorflow/python/saved_model/load_test.py    | 25 +++++++++++++++++--
 tensorflow/python/saved_model/save.py         |  1 +
 .../saved_model/saved_object_graph.proto      |  3 ++-
 5 files changed, 34 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index 58d1f6b886..05fccfbcd9 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -105,7 +105,7 @@ def _parse_func_attrs(attributes):
       attrs[key] = attr_value_pb2.AttrValue(i=value)
     elif isinstance(value, float):
       attrs[key] = attr_value_pb2.AttrValue(f=value)
-    elif isinstance(value, (str, bytes)):
+    elif isinstance(value, (str, bytes, six.text_type)):
       attrs[key] = attr_value_pb2.AttrValue(s=compat.as_bytes(value))
     else:
       raise ValueError("Unsupported attribute type for %s with type %s" %
diff --git a/tensorflow/python/saved_model/load.py b/tensorflow/python/saved_model/load.py
index 3e85fade50..1ee1b69b03 100644
--- a/tensorflow/python/saved_model/load.py
+++ b/tensorflow/python/saved_model/load.py
@@ -58,6 +58,11 @@ class _Loader(object):
           bound_inputs = [
               self._get_tensor_from_node(node_id)
               for node_id in monomorphic_function.bound_inputs]
+          bound_variables = [
+              self._nodes[node_id]
+              for node_id in monomorphic_function.bound_inputs
+              if self._proto.nodes[node_id].WhichOneof("kind") == "variable"
+          ]
           if name in seen_functions:
             if self._functions[name]._captured_inputs != bound_inputs:  # pylint: disable=protected-access
               raise NotImplementedError(
@@ -69,6 +74,7 @@ class _Loader(object):
             # concrete function, note that we did not modify the FuncGraph
             # itself.
             self._functions[name]._captured_inputs = bound_inputs  # pylint: disable=protected-access
+            self._functions[name]._func_graph.variables = bound_variables  # pylint: disable=protected-access
 
   def _get_tensor_from_node(self, node_id):
     obj = self._nodes[node_id]
@@ -123,7 +129,7 @@ class _Loader(object):
   def _recreate_variable(self, proto):
     # TODO(andresp): Can we use the checkpointed value as initializer?
     dummy_value = init_ops.Zeros(dtype=proto.dtype)(shape=proto.shape)
-    return variables.Variable(dummy_value)
+    return variables.Variable(dummy_value, trainable=proto.trainable)
 
 
 def _load_saved_object_graph_proto(filename):
diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index a1c81d3f14..35e384dbfc 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import os
 import tempfile
 
+from tensorflow.python.eager import backprop
 from tensorflow.python.eager import def_function
 from tensorflow.python.eager import test
 from tensorflow.python.framework import constant_op
@@ -52,11 +53,13 @@ class LoadTest(test.TestCase):
 
   def test_variables(self):
     root = tracking.Checkpointable()
-    root.v1 = variables.Variable(1.)
-    root.v2 = variables.Variable(2.)
+    root.v1 = variables.Variable(1., trainable=True)
+    root.v2 = variables.Variable(2., trainable=False)
     imported = self.cycle(root)
     self.assertEquals(imported.v1.numpy(), 1.0)
+    self.assertTrue(imported.v1.trainable)
     self.assertEquals(imported.v2.numpy(), 2.0)
+    self.assertFalse(imported.v2.trainable)
 
   def test_capture_variables(self):
     root = tracking.Checkpointable()
@@ -248,6 +251,24 @@ class LoadTest(test.TestCase):
     self.cycle(m)
     self.assertEquals(4.0, m.f(constant_op.constant(2.0)).numpy())
 
+  def test_basic_backprop(self):
+    weight = variables.Variable(1., trainable=True)
+    bias = variables.Variable(0., trainable=True)
+    g = def_function.function(
+        lambda x: x*weight + bias,
+        input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)])
+
+    root = tracking.Checkpointable()
+    root.weight = weight
+    root.bias = bias
+    root.g = g
+    imported = self.cycle(root)
+    with backprop.GradientTape(watch_accessed_variables=True) as t:
+      x = constant_op.constant([3.5])
+      loss = imported.g(x)
+      grad = t.gradient(loss, [imported.weight, imported.bias])
+      self.assertAllClose(grad, [3.5, 1.0])
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/saved_model/save.py b/tensorflow/python/saved_model/save.py
index a76b370565..d79b5fe3a6 100644
--- a/tensorflow/python/saved_model/save.py
+++ b/tensorflow/python/saved_model/save.py
@@ -610,6 +610,7 @@ def _write_object_proto(obj, proto, asset_file_def_index, node_ids):
     proto.asset.asset_file_def_index = asset_file_def_index[obj]
   elif resource_variable_ops.is_resource_variable(obj):
     proto.variable.SetInParent()
+    proto.variable.trainable = obj.trainable
     proto.variable.dtype = obj.dtype.as_datatype_enum
     proto.variable.shape.CopyFrom(obj.shape.as_proto())
   elif isinstance(obj, def_function.PolymorphicFunction):
diff --git a/tensorflow/python/saved_model/saved_object_graph.proto b/tensorflow/python/saved_model/saved_object_graph.proto
index f46927d6e8..1e2514b7f7 100644
--- a/tensorflow/python/saved_model/saved_object_graph.proto
+++ b/tensorflow/python/saved_model/saved_object_graph.proto
@@ -104,6 +104,7 @@ message SavedMonomorphicFunction {
 message SavedVariable {
   DataType dtype = 1;
   TensorShapeProto shape = 2;
+  bool trainable = 3;
 
-  // TODO(andresp): Add "trainable" and save_slice_info_def.
+  // TODO(andresp): Add save_slice_info_def?
 }
-- 
GitLab


From 0ad1d56cd974c1feef19ebf81b202b9bd4066247 Mon Sep 17 00:00:00 2001
From: Grzegorz Pawelczak <grzegorzp@graphcore.ai>
Date: Tue, 8 Jan 2019 13:44:32 +0000
Subject: [PATCH 0371/1095] Make RemoveUnusedOperandFromSort a seperate pass

---
 tensorflow/compiler/xla/service/BUILD         |  32 +++++
 .../xla/service/algebraic_simplifier.cc       |  75 -----------
 .../xla/service/algebraic_simplifier_test.cc  |  68 ----------
 .../compiler/xla/service/sort_simplifier.cc   | 125 ++++++++++++++++++
 .../compiler/xla/service/sort_simplifier.h    |  35 +++++
 .../xla/service/sort_simplifier_test.cc       | 102 ++++++++++++++
 6 files changed, 294 insertions(+), 143 deletions(-)
 create mode 100644 tensorflow/compiler/xla/service/sort_simplifier.cc
 create mode 100644 tensorflow/compiler/xla/service/sort_simplifier.h
 create mode 100644 tensorflow/compiler/xla/service/sort_simplifier_test.cc

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index ec63ae1ac6..7be180c624 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -3462,6 +3462,38 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "sort_simplifier",
+    srcs = ["sort_simplifier.cc"],
+    hdrs = ["sort_simplifier.h"],
+    deps = [
+        ":hlo",
+        ":hlo_pass",
+        ":while_util",
+        "//tensorflow/compiler/xla:statusor",
+        "//tensorflow/compiler/xla:util",
+        "//tensorflow/core:lib",
+        "@com_google_absl//absl/algorithm:container",
+        "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/container:inlined_vector",
+    ],
+)
+
+tf_cc_test(
+    name = "sort_simplifier_test",
+    srcs = ["sort_simplifier_test.cc"],
+    deps = [
+        ":hlo_matchers",
+        ":sort_simplifier",
+        ":pattern_matcher",
+        ":pattern_matcher_gmock",
+        "//tensorflow/compiler/xla:test",
+        "//tensorflow/compiler/xla/service:hlo_parser",
+        "//tensorflow/compiler/xla/tests:hlo_test_base",
+        "//tensorflow/core:test",
+    ],
+)
+
 cc_library(
     name = "tuple_util",
     srcs = ["tuple_util.cc"],
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
index 5ac746c9f3..b76f2febc7 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
@@ -371,11 +371,6 @@ class AlgebraicSimplifierVisitor : public DfsHloVisitorWithDefault {
   // Tries to convert slice(reshape(X)) into reshape(slice(X))
   StatusOr<bool> TryToReorderSliceAndReshape(HloInstruction* slice);
 
-  // If the sort instruction has a tuple shape then looks for unused output
-  // values and removes them from the sort instruction. Returns true if the
-  // graph have been modified.
-  StatusOr<bool> RemoveUnusedOperandFromSort(HloInstruction* sort);
-
   // Current HloComputation instance the AlgebraicSimplifierVisitor is
   // traversing.
   HloComputation* computation_;
@@ -2819,69 +2814,6 @@ Status AlgebraicSimplifierVisitor::HandleSelect(HloInstruction* select) {
   return Status::OK();
 }
 
-StatusOr<bool> AlgebraicSimplifierVisitor::RemoveUnusedOperandFromSort(
-    HloInstruction* sort) {
-  if (!sort->shape().IsTuple()) {
-    return false;
-  }
-
-  if (sort->parent()->root_instruction() == sort) {
-    // Can't analyse users of the root instruction.
-    return false;
-  }
-
-  // Index 0 is the sorting key used by the sort HLO itself.
-  absl::flat_hash_set<int64> used_indices{0};
-  for (const HloInstruction* user : sort->users()) {
-    if (user->opcode() != HloOpcode::kGetTupleElement) {
-      // Can't analyse users other then get-tuple-element.
-      return false;
-    }
-    used_indices.insert(user->tuple_index());
-  }
-
-  if (used_indices.size() == sort->operand_count()) {
-    // All operands are used.
-    return false;
-  }
-
-  std::vector<HloInstruction*> operands{sort->mutable_operand(0)};
-  std::vector<Shape> new_shapes{sort->operand(0)->shape()};
-  for (int64 i = 1; i < sort->operand_count(); ++i) {
-    if (used_indices.count(i)) {
-      operands.push_back(sort->mutable_operand(i));
-      new_shapes.push_back(sort->operand(i)->shape());
-    }
-  }
-  Shape new_sort_shape = new_shapes.size() == 1
-                             ? new_shapes[0]
-                             : ShapeUtil::MakeTupleShape(new_shapes);
-  HloInstruction* new_sort = computation_->AddInstruction(
-      sort->CloneWithNewOperands(new_sort_shape, operands));
-
-  // Map from original get-tuple-element tuple index to new HLO instruction
-  absl::flat_hash_map<int64, HloInstruction*> result_map;
-  if (new_sort->shape().IsTuple()) {
-    // Old sort key maps to new sort key.
-    int64 new_index = 0;
-    for (int64 i = 0; i < sort->operand_count(); ++i) {
-      if (used_indices.count(i)) {
-        result_map[i] =
-            computation_->AddInstruction(HloInstruction::CreateGetTupleElement(
-                new_shapes[new_index], new_sort, new_index));
-        ++new_index;
-      }
-    }
-  } else {
-    result_map[0] = new_sort;
-  }
-  for (HloInstruction* user : sort->users()) {
-    TF_RETURN_IF_ERROR(
-        user->ReplaceAllUsesWith(result_map.at(user->tuple_index())));
-  }
-  return true;
-}
-
 Status AlgebraicSimplifierVisitor::HandleSort(HloInstruction* sort) {
   auto operand = sort->mutable_operand(0);
   int64 dimension_to_sort = sort->dimensions(0);
@@ -2895,13 +2827,6 @@ Status AlgebraicSimplifierVisitor::HandleSort(HloInstruction* sort) {
         sort, HloInstruction::CreateTuple(sort->operands()));
   }
 
-  // Remove the unused values from a key-value sort.
-  TF_ASSIGN_OR_RETURN(bool removed_operand, RemoveUnusedOperandFromSort(sort));
-  if (removed_operand) {
-    changed_ = true;
-    return Status::OK();
-  }
-
   if (!options_.enable_permutation_sort_replacement()) {
     return Status::OK();
   }
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
index e39321f1f3..a2bbc4861f 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
@@ -2709,74 +2709,6 @@ TEST_F(AlgebraicSimplifierTest, DontReplacePermutationSortWrongDimensions) {
   EXPECT_FALSE(simplifier.Run(module.get()).ValueOrDie());
 }
 
-TEST_F(AlgebraicSimplifierTest, RemoveUnusedSortOperandArrayResult) {
-  const char* hlo_string = R"(
-   HloModule permutation_sort
-
-    ENTRY sort_computation {
-      keys = f32[64,8732]{1,0} parameter(0)
-      values = s32[64,8732]{1,0} parameter(1)
-      sort = (f32[64,8732]{1,0}, s32[64,8732]{1,0}) sort(keys, values),
-        dimensions={1}
-      ROOT gte = f32[64,8732]{1,0} get-tuple-element(sort), index=0
-    })";
-  TF_ASSERT_OK_AND_ASSIGN(auto module,
-                          ParseAndReturnVerifiedModule(hlo_string));
-
-  AlgebraicSimplifierOptions options(bitcasting_callback());
-  AlgebraicSimplifier simplifier(options);
-  EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
-  auto root = module->entry_computation()->root_instruction();
-  EXPECT_THAT(root, GmockMatch(m::Sort(m::Parameter(0))));
-}
-
-TEST_F(AlgebraicSimplifierTest, RemoveUnusedSortOperandTuple) {
-  const char* hlo_string = R"(
-   HloModule permutation_sort
-
-    ENTRY sort_computation {
-      keys = f32[64,87] parameter(0)
-      values.0 = s32[64,87] parameter(1)
-      values.1 = u32[64,87] parameter(2)
-      sort = (f32[64,87], s32[64,87], u32[64,87]) sort(
-          keys, values.0, values.1),
-        dimensions={1}
-      gte.0 = f32[64,87] get-tuple-element(sort), index=0
-      gte.1 = u32[64,87] get-tuple-element(sort), index=2
-      ROOT tuple = (f32[64,87], u32[64,87]) tuple(gte.0, gte.1)
-    })";
-  TF_ASSERT_OK_AND_ASSIGN(auto module,
-                          ParseAndReturnVerifiedModule(hlo_string));
-
-  AlgebraicSimplifierOptions options(bitcasting_callback());
-  AlgebraicSimplifier simplifier(options);
-  EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
-  auto root = module->entry_computation()->root_instruction();
-  EXPECT_THAT(
-      root,
-      GmockMatch(m::Tuple(
-          m::GetTupleElement(m::Sort(m::Parameter(0), m::Parameter(2)), 0),
-          m::GetTupleElement(m::Sort(m::Parameter(0), m::Parameter(2)), 1))));
-}
-
-TEST_F(AlgebraicSimplifierTest, DontRemoveUnusedSortKey) {
-  const char* hlo_string = R"(
-   HloModule permutation_sort
-
-    ENTRY sort_computation {
-      keys = f32[64,8732]{1,0} parameter(0)
-      values = s32[64,8732]{1,0} parameter(1)
-      sort = (f32[64,8732]{1,0}, s32[64,8732]{1,0}) sort(keys, values), dimensions={1}
-      ROOT gte = s32[64,8732]{1,0} get-tuple-element(sort), index=1
-    })";
-  TF_ASSERT_OK_AND_ASSIGN(auto module,
-                          ParseAndReturnVerifiedModule(hlo_string));
-
-  AlgebraicSimplifierOptions options(bitcasting_callback());
-  AlgebraicSimplifier simplifier(options);
-  EXPECT_FALSE(simplifier.Run(module.get()).ValueOrDie());
-}
-
 TEST_F(AlgebraicSimplifierTest, ReplaceEffectiveScalarKeyValueSortWithTuple) {
   auto builder = HloComputation::Builder(TestName());
 
diff --git a/tensorflow/compiler/xla/service/sort_simplifier.cc b/tensorflow/compiler/xla/service/sort_simplifier.cc
new file mode 100644
index 0000000000..3280672574
--- /dev/null
+++ b/tensorflow/compiler/xla/service/sort_simplifier.cc
@@ -0,0 +1,125 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/sort_simplifier.h"
+#include "tensorflow/compiler/xla/service/hlo_computation.h"
+#include "tensorflow/compiler/xla/service/hlo_instruction.h"
+#include "tensorflow/compiler/xla/statusor.h"
+
+#include "absl/container/flat_hash_map.h"
+#include "absl/container/flat_hash_set.h"
+
+namespace xla {
+namespace {
+
+// If the sort instruction has a tuple shape then looks for unused output
+// values and removes them from the sort instruction. Returns true if the
+// graph have been modified.
+StatusOr<bool> RemoveUnusedOperandFromSort(HloInstruction* sort) {
+  if (!sort->shape().IsTuple()) {
+    return false;
+  }
+
+  if (sort->parent()->root_instruction() == sort) {
+    // Can't analyse users of the root instruction.
+    return false;
+  }
+
+  // Index 0 is the sorting key used by the sort HLO itself.
+  absl::flat_hash_set<int64> used_indices{0};
+  for (const HloInstruction* user : sort->users()) {
+    if (user->opcode() != HloOpcode::kGetTupleElement) {
+      // Can't analyse users other then get-tuple-element.
+      return false;
+    }
+    used_indices.insert(user->tuple_index());
+  }
+
+  if (used_indices.size() == sort->operand_count()) {
+    // All operands are used.
+    return false;
+  }
+
+  std::vector<HloInstruction*> operands{sort->mutable_operand(0)};
+  std::vector<Shape> new_shapes{sort->operand(0)->shape()};
+  for (int64 i = 1; i < sort->operand_count(); ++i) {
+    if (used_indices.count(i)) {
+      operands.push_back(sort->mutable_operand(i));
+      new_shapes.push_back(sort->operand(i)->shape());
+    }
+  }
+  HloComputation* computation = sort->parent();
+
+  Shape new_sort_shape = new_shapes.size() == 1
+                             ? new_shapes[0]
+                             : ShapeUtil::MakeTupleShape(new_shapes);
+  HloInstruction* new_sort = computation->AddInstruction(
+      sort->CloneWithNewOperands(new_sort_shape, operands));
+
+  // Map from original get-tuple-element tuple index to new HLO instruction
+  absl::flat_hash_map<int64, HloInstruction*> result_map;
+  if (new_sort->shape().IsTuple()) {
+    // Old sort key maps to new sort key.
+    int64 new_index = 0;
+    for (int64 i = 0; i < sort->operand_count(); ++i) {
+      if (used_indices.count(i)) {
+        result_map[i] =
+            computation->AddInstruction(HloInstruction::CreateGetTupleElement(
+                new_shapes[new_index], new_sort, new_index));
+        ++new_index;
+      }
+    }
+  } else {
+    result_map[0] = new_sort;
+  }
+  for (HloInstruction* user : sort->users()) {
+    TF_RETURN_IF_ERROR(
+        user->ReplaceAllUsesWith(result_map.at(user->tuple_index())));
+    TF_RETURN_IF_ERROR(computation->RemoveInstructionAndUnusedOperands(user));
+  }
+  return true;
+}
+
+} // namespace
+
+StatusOr<bool> SortSimplifier::Run(HloModule* module) {
+  VLOG(2) << "HLO module before SortSimplifier:";
+  XLA_VLOG_LINES(2, module->ToString());
+
+  bool changed = false;
+  std::vector<HloInstruction*> sort_instrs;
+  for (auto* comp : module->MakeNonfusionComputations()) {
+    absl::c_copy_if(comp->instructions(), std::back_inserter(sort_instrs),
+                    [](const HloInstruction* instr) {
+                      return instr->opcode() == HloOpcode::kSort;
+                    });
+  }
+
+  for (HloInstruction* sort_instr : sort_instrs) {
+    TF_ASSIGN_OR_RETURN(bool result,
+                        RemoveUnusedOperandFromSort(sort_instr));
+    changed |= result;
+  }
+
+  if (changed) {
+    VLOG(2) << "HLO module after SortSimplifier:";
+    XLA_VLOG_LINES(2, module->ToString());
+  } else {
+    VLOG(2) << "HLO module unchanged after SortSimplifier";
+  }
+
+  return changed;
+}
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/sort_simplifier.h b/tensorflow/compiler/xla/service/sort_simplifier.h
new file mode 100644
index 0000000000..8c6f313aa0
--- /dev/null
+++ b/tensorflow/compiler/xla/service/sort_simplifier.h
@@ -0,0 +1,35 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_SORT_SIMPLIFIER_H_
+#define TENSORFLOW_COMPILER_XLA_SERVICE_SORT_SIMPLIFIER_H_
+
+#include "tensorflow/compiler/xla/service/hlo_module.h"
+#include "tensorflow/compiler/xla/service/hlo_pass_interface.h"
+#include "tensorflow/compiler/xla/statusor.h"
+
+namespace xla {
+
+// HLO pass which removes unused operands from sort, where an unused operand is
+// defined as an operand at some index 'x' at which the output is not used.
+class SortSimplifier : public HloModulePass {
+ public:
+  absl::string_view name() const override { return "simplify-sorts"; }
+  StatusOr<bool> Run(HloModule* module) override;
+};
+
+}  // namespace xla
+
+#endif  // TENSORFLOW_COMPILER_XLA_SERVICE_SORT_SIMPLIFIER_H_
diff --git a/tensorflow/compiler/xla/service/sort_simplifier_test.cc b/tensorflow/compiler/xla/service/sort_simplifier_test.cc
new file mode 100644
index 0000000000..5a81e537de
--- /dev/null
+++ b/tensorflow/compiler/xla/service/sort_simplifier_test.cc
@@ -0,0 +1,102 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/sort_simplifier.h"
+
+#include "tensorflow/compiler/xla/service/hlo_matchers.h"
+#include "tensorflow/compiler/xla/service/hlo_parser.h"
+#include "tensorflow/compiler/xla/service/pattern_matcher.h"
+#include "tensorflow/compiler/xla/service/pattern_matcher_gmock.h"
+#include "tensorflow/compiler/xla/test.h"
+#include "tensorflow/compiler/xla/tests/hlo_test_base.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
+
+namespace xla {
+namespace {
+
+namespace m = match;
+
+using SortSimplifierTest = HloTestBase;
+
+TEST_F(SortSimplifierTest, RemoveUnusedSortOperandArrayResult) {
+  const char* hlo_string = R"(
+   HloModule permutation_sort
+
+    ENTRY sort_computation {
+      keys = f32[64,8732]{1,0} parameter(0)
+      values = s32[64,8732]{1,0} parameter(1)
+      sort = (f32[64,8732]{1,0}, s32[64,8732]{1,0}) sort(keys, values),
+        dimensions={1}
+      ROOT gte = f32[64,8732]{1,0} get-tuple-element(sort), index=0
+    })";
+  TF_ASSERT_OK_AND_ASSIGN(auto module,
+                          ParseAndReturnVerifiedModule(hlo_string));
+
+  SortSimplifier simplifier;
+  uint64 num_executions = 0;
+  do {
+    num_executions++;
+  } while (simplifier.Run(module.get()).ValueOrDie());
+  EXPECT_TRUE(num_executions == 2);
+  auto root = module->entry_computation()->root_instruction();
+  EXPECT_THAT(root, GmockMatch(m::Sort(m::Parameter(0))));
+}
+
+TEST_F(SortSimplifierTest, RemoveUnusedSortOperandTuple) {
+  const char* hlo_string = R"(
+   HloModule permutation_sort
+
+    ENTRY sort_computation {
+      keys = f32[64,87] parameter(0)
+      values.0 = s32[64,87] parameter(1)
+      values.1 = u32[64,87] parameter(2)
+      sort = (f32[64,87], s32[64,87], u32[64,87]) sort(
+          keys, values.0, values.1),
+        dimensions={1}
+      gte.0 = f32[64,87] get-tuple-element(sort), index=0
+      gte.1 = u32[64,87] get-tuple-element(sort), index=2
+      ROOT tuple = (f32[64,87], u32[64,87]) tuple(gte.0, gte.1)
+    })";
+  TF_ASSERT_OK_AND_ASSIGN(auto module,
+                          ParseAndReturnVerifiedModule(hlo_string));
+
+  SortSimplifier simplifier;
+  EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
+  auto root = module->entry_computation()->root_instruction();
+  EXPECT_THAT(
+      root,
+      GmockMatch(m::Tuple(
+          m::GetTupleElement(m::Sort(m::Parameter(0), m::Parameter(2)), 0),
+          m::GetTupleElement(m::Sort(m::Parameter(0), m::Parameter(2)), 1))));
+}
+
+TEST_F(SortSimplifierTest, DontRemoveUnusedSortKey) {
+  const char* hlo_string = R"(
+   HloModule permutation_sort
+
+    ENTRY sort_computation {
+      keys = f32[64,8732]{1,0} parameter(0)
+      values = s32[64,8732]{1,0} parameter(1)
+      sort = (f32[64,8732]{1,0}, s32[64,8732]{1,0}) sort(keys, values), dimensions={1}
+      ROOT gte = s32[64,8732]{1,0} get-tuple-element(sort), index=1
+    })";
+  TF_ASSERT_OK_AND_ASSIGN(auto module,
+                          ParseAndReturnVerifiedModule(hlo_string));
+
+  SortSimplifier simplifier;
+  EXPECT_FALSE(simplifier.Run(module.get()).ValueOrDie());
+}
+}  // namespace
+}  // namespace xla
-- 
GitLab


From 0f9478c3c2a71f26fd26083b2e2ed76fb69d1030 Mon Sep 17 00:00:00 2001
From: ktaebum <phya.ktaebum@gmail.com>
Date: Tue, 8 Jan 2019 23:19:38 +0900
Subject: [PATCH 0372/1095] resolved incompatible dimension reshape before feed
 into final dense layer

---
 tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb.py b/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb.py
index 15776c694e..9b5a2c947b 100644
--- a/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb.py
+++ b/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb.py
@@ -128,7 +128,7 @@ class PTBModel(tf.keras.Model):
 
     self.linear = layers.Dense(
         vocab_size, kernel_initializer=tf.random_uniform_initializer(-0.1, 0.1))
-    self._output_shape = [-1, embedding_dim]
+    self._output_shape = [-1, hidden_dim]
 
   def call(self, input_seq, training):
     """Run the forward pass of PTBModel.
-- 
GitLab


From 7fa90cddf07c6aac532306dac15e02216a83b9fa Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Tue, 8 Jan 2019 07:29:42 -0800
Subject: [PATCH 0373/1095] [eager]: Disable benchmark that is currently timing
 out.

This timeout seems to have started after
https://github.com/tensorflow/tensorflow/commit/0445684a64d1bea8490a99eb9ce278176133df75

PiperOrigin-RevId: 228326334
---
 tensorflow/python/eager/benchmarks_test.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tensorflow/python/eager/benchmarks_test.py b/tensorflow/python/eager/benchmarks_test.py
index 31a7efca82..9191c8e689 100644
--- a/tensorflow/python/eager/benchmarks_test.py
+++ b/tensorflow/python/eager/benchmarks_test.py
@@ -867,6 +867,10 @@ class MicroBenchmarks(test.Benchmark):
     self._run(scan, 100)
 
   def benchmarkScanDefun(self):
+    if context.num_gpus():
+      # TODO(b/122081934): Re-enable this after figuring out why this became
+      # really slow with control flow V2
+      return
     elems = math_ops.range(1600)
 
     @function.defun
-- 
GitLab


From 107b7c4de6727ad02a0b4fc24aa1ac17bd697776 Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Tue, 8 Jan 2019 11:37:37 -0500
Subject: [PATCH 0374/1095] Fixes for rnn_keras_estimator.ipynb

* wrapping `range(batch_size)` into a `list` call so that it's convertible to `Tensor` in Python 3, where `range` alone returns just a generator
* specifying a `dtype` argument to `LSTMBlockCell`'s constructor

These should fix the breakages in #24701
---
 .../autograph/examples/notebooks/rnn_keras_estimator.ipynb  | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/autograph/examples/notebooks/rnn_keras_estimator.ipynb b/tensorflow/contrib/autograph/examples/notebooks/rnn_keras_estimator.ipynb
index 44532cb078..831c613f2c 100644
--- a/tensorflow/contrib/autograph/examples/notebooks/rnn_keras_estimator.ipynb
+++ b/tensorflow/contrib/autograph/examples/notebooks/rnn_keras_estimator.ipynb
@@ -186,8 +186,8 @@
         "\n",
         "  def __init__(self):\n",
         "    super(RnnColorbot, self).__init__()\n",
-        "    self.lower_cell = tf.contrib.rnn.LSTMBlockCell(256)\n",
-        "    self.upper_cell = tf.contrib.rnn.LSTMBlockCell(128)\n",
+        "    self.lower_cell = tf.contrib.rnn.LSTMBlockCell(256, dtype=tf.float32)\n",
+        "    self.upper_cell = tf.contrib.rnn.LSTMBlockCell(128, dtype=tf.float32)\n",
         "    self.relu_layer = tf.layers.Dense(3, activation=tf.nn.relu)\n",
         "\n",
         "  def _rnn_layer(self, chars, cell, batch_size, training):\n",
@@ -241,7 +241,7 @@
         "    seq = self._rnn_layer(seq, self.upper_cell, batch_size, training)\n",
         "\n",
         "    # Grab just the end-of-sequence from each output.\n",
-        "    indices = (length - 1, range(batch_size))\n",
+        "    indices = (length - 1, list(range(batch_size)))\n",
         "    indices = tf.stack(indices, 1)\n",
         "    sequence_ends = tf.gather_nd(seq, indices)\n",
         "    return self.relu_layer(sequence_ends)\n",
-- 
GitLab


From 02d4333dfc5a9fe2c3929ba6288e9a679725da85 Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Tue, 8 Jan 2019 09:18:09 -0800
Subject: [PATCH 0375/1095] Give bytes constructor the appropriate utf-8
 argument.

PiperOrigin-RevId: 228342070
---
 tensorflow/tools/git/gen_git_source.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/tools/git/gen_git_source.py b/tensorflow/tools/git/gen_git_source.py
index 0c37508650..c2449da923 100755
--- a/tensorflow/tools/git/gen_git_source.py
+++ b/tensorflow/tools/git/gen_git_source.py
@@ -29,8 +29,8 @@ from __future__ import print_function
 import argparse
 import json
 import os
-import subprocess
 import shutil
+import subprocess
 
 
 def parse_branch_ref(filename):
@@ -175,7 +175,7 @@ def get_git_version(git_base_path, git_tag_override):
       # two "-" are those inserted by the git describe command.
       abbrev_commit = split_val[-1]
       val = version_separator.join(
-          [bytes(git_tag_override), b"0", abbrev_commit])
+          [bytes(git_tag_override, "utf-8"), b"0", abbrev_commit])
     return val if val else unknown_label
   except (subprocess.CalledProcessError, OSError):
     return unknown_label
-- 
GitLab


From 3ba26026545be700c4aed9bc609b25fad9c1531e Mon Sep 17 00:00:00 2001
From: Sergei Lebedev <slebedev@google.com>
Date: Tue, 8 Jan 2019 09:25:59 -0800
Subject: [PATCH 0376/1095] Removed ResourceVariable._cached_shape_as_list

_shape_as_list is no longer on the hotpath of EagerLinearRegressionBenchmark,
the change does not seem to affect throughput estimates. Note that both
estimates have high variance.

Before:

    entry {
      name: "EagerLinearRegressionBenchmark.eager_train_cpu"
      iters: 2000
      wall_time: 1.24479007721
      extras {
        key: "examples_per_sec"
        value {
          double_value: 102828.583183
        }
      }
    }

After:

    entry {
      name: "EagerLinearRegressionBenchmark.eager_train_cpu"
      iters: 2000
      wall_time: 1.25307798386
      extras {
        key: "examples_per_sec"
        value {
          double_value: 102148.470924
        }
      }
    }

PiperOrigin-RevId: 228343127
---
 tensorflow/python/ops/resource_variable_ops.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index 1d98fb2c89..d27e38c735 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -504,7 +504,6 @@ class ResourceVariable(variables.VariableV1):
       # all in graph mode.
       self._handle_deleter = EagerResourceDeleter(
           handle=self._handle, handle_device=self._handle.device)
-    self._cached_shape_as_list = None
 
   def _init_from_proto(self, variable_def, import_scope=None):
     """Initializes from `VariableDef` proto."""
@@ -562,7 +561,6 @@ class ResourceVariable(variables.VariableV1):
     self._caching_device = None
     self._dtype = dtypes.as_dtype(self._handle.op.get_attr("dtype"))
     self._constraint = None
-    self._cached_shape_as_list = None
 
   @contextlib.contextmanager
   def _assign_dependencies(self):
@@ -632,12 +630,9 @@ class ResourceVariable(variables.VariableV1):
     return self._distribute_strategy
 
   def _shape_as_list(self):
-    if self._cached_shape_as_list:
-      return self._cached_shape_as_list
     if self.shape.ndims is None:
       return None
-    self._cached_shape_as_list = [dim.value for dim in self.shape.dims]
-    return self._cached_shape_as_list
+    return [dim.value for dim in self.shape.dims]
 
   def _shape_tuple(self):
     shape = self._shape_as_list()
-- 
GitLab


From aa5e17a2dc94ba47d41a74021d6701a8c10de4a5 Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Tue, 8 Jan 2019 09:37:02 -0800
Subject: [PATCH 0377/1095] Fix linkage for several lite/testing tests

Use the tf_cc_test rule for targets that have TensorFlow deps.

PiperOrigin-RevId: 228344897
---
 tensorflow/lite/testing/BUILD | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/testing/BUILD b/tensorflow/lite/testing/BUILD
index fa25cfaa69..19c950b4f8 100644
--- a/tensorflow/lite/testing/BUILD
+++ b/tensorflow/lite/testing/BUILD
@@ -257,7 +257,7 @@ cc_library(
     ],
 )
 
-cc_test(
+tf_cc_test(
     name = "tf_driver_test",
     size = "small",
     srcs = ["tf_driver_test.cc"],
@@ -286,7 +286,7 @@ cc_library(
     ],
 )
 
-cc_test(
+tf_cc_test(
     name = "generate_testspec_test",
     size = "small",
     srcs = ["generate_testspec_test.cc"],
-- 
GitLab


From f0fba8b59b3e3fa6b068c0c2715104153983c554 Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Tue, 8 Jan 2019 09:54:11 -0800
Subject: [PATCH 0378/1095] Use correct _t integer types to fix mac build.

PiperOrigin-RevId: 228347592
---
 tensorflow/lite/kernels/dequantize_test.cc | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/kernels/dequantize_test.cc b/tensorflow/lite/kernels/dequantize_test.cc
index 6343745eef..be7caa3189 100644
--- a/tensorflow/lite/kernels/dequantize_test.cc
+++ b/tensorflow/lite/kernels/dequantize_test.cc
@@ -12,6 +12,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
+#include <cstdint>
+
 #include <gtest/gtest.h>
 #include "tensorflow/lite/interpreter.h"
 #include "tensorflow/lite/kernels/internal/types.h"
@@ -59,7 +61,7 @@ TEST(DequantizeOpTest, UINT8) {
   // [-63.5, 64] -> scale=0.5 zero_point=127 for UINT8
   DequantizeOpModel m(TensorType_UINT8, {2, 5}, 0.5, 127);
 
-  m.SetInput<uint8>({0, 1, 2, 3, 4, 251, 252, 253, 254, 255});
+  m.SetInput<uint8_t>({0, 1, 2, 3, 4, 251, 252, 253, 254, 255});
   m.Invoke();
   EXPECT_THAT(m.GetOutput(),
               ElementsAreArray(ArrayFloatNear(
@@ -70,7 +72,7 @@ TEST(DequantizeOpTest, INT8) {
   // [-63.5, 64] -> scale=0.5, zero_point=1 for INT8
   DequantizeOpModel m(TensorType_INT8, {2, 5}, 0.5, -1);
 
-  m.SetInput<int8>({-128, -127, -126, -125, -124, 123, 124, 125, 126, 127});
+  m.SetInput<int8_t>({-128, -127, -126, -125, -124, 123, 124, 125, 126, 127});
   m.Invoke();
   EXPECT_THAT(m.GetOutput(),
               ElementsAreArray(ArrayFloatNear(
-- 
GitLab


From 0ad4e9ea2b41ad1d9c8c59844fd814b01820bab2 Mon Sep 17 00:00:00 2001
From: Jian Li <jianlijianli@google.com>
Date: Tue, 8 Jan 2019 09:57:31 -0800
Subject: [PATCH 0379/1095] Update unidirectional lstm test case.

PiperOrigin-RevId: 228348125
---
 tensorflow/lite/kernels/unidirectional_sequence_lstm_test.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/kernels/unidirectional_sequence_lstm_test.cc b/tensorflow/lite/kernels/unidirectional_sequence_lstm_test.cc
index 274f907076..bc35d90773 100644
--- a/tensorflow/lite/kernels/unidirectional_sequence_lstm_test.cc
+++ b/tensorflow/lite/kernels/unidirectional_sequence_lstm_test.cc
@@ -252,7 +252,7 @@ class HybridUnidirectionalLSTMOpModel : public UnidirectionalLSTMOpModel {
     tensor_type_ = tensor_type;
   }
 
-  void SetWeights(int weights_idx, std::vector<float> f) {
+  void SetWeights(int weights_idx, const std::vector<float>& f) {
     if (tensor_type_ == TensorType_UINT8) {
       SymmetricQuantizeAndPopulate(weights_idx, f);
     } else {
-- 
GitLab


From 011a8094c182101a65fb84e98a60a152e0f58f0d Mon Sep 17 00:00:00 2001
From: Sergei Lebedev <slebedev@google.com>
Date: Tue, 8 Jan 2019 10:06:30 -0800
Subject: [PATCH 0380/1095] Aligned ResourceVariable.__reduce__ with
 ResourceVariable.__deepcopy__

PiperOrigin-RevId: 228350078
---
 .../kernel_tests/resource_variable_ops_test.py     | 13 ++++++++++---
 tensorflow/python/ops/resource_variable_ops.py     | 14 ++++++++++++--
 2 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
index 67fa350317..2594055119 100644
--- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py
+++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
@@ -286,12 +286,19 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
       tmp_dir = self.get_temp_dir()
       fname = os.path.join(tmp_dir, "var.pickle")
       with open(fname, "wb") as f:
-        v = resource_variable_ops.ResourceVariable(10.0)
+        v = resource_variable_ops.ResourceVariable(
+            10.0,
+            dtype=dtypes.float16,
+            name="v")
         pickle.dump(v, f)
 
       with open(fname, "rb") as f:
-        v = pickle.load(f)
-        self.assertAllEqual(v.numpy(), 10.0)
+        new_v = pickle.load(f)
+        self.assertEqual(new_v.name, v.name)
+        self.assertEqual(new_v.shape, v.shape)
+        self.assertEqual(new_v.dtype, v.dtype)
+        self.assertEqual(new_v.trainable, v.trainable)
+        self.assertAllEqual(new_v.numpy(), v.numpy())
 
   @test_util.run_in_graph_and_eager_modes
   def testScatterDiv(self):
diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index d27e38c735..2d46e21074 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -20,6 +20,7 @@ from __future__ import division
 from __future__ import print_function
 
 import contextlib
+import functools
 
 from tensorflow.core.framework import attr_value_pb2
 from tensorflow.core.framework import variable_pb2
@@ -595,7 +596,8 @@ class ResourceVariable(variables.VariableV1):
         trainable=self._trainable,
         constraint=self._constraint,
         dtype=self._dtype,
-        name=self._shared_name + "_copy")
+        name=self._shared_name + "_copy",
+        distribute_strategy=self.distribute_strategy)
     memo[self._unique_id] = copied_variable
     return copied_variable
 
@@ -938,7 +940,15 @@ class ResourceVariable(variables.VariableV1):
     return assign_op
 
   def __reduce__(self):
-    return (ResourceVariable, (self.numpy(),))
+    # The implementation mirrors that of __deepcopy__.
+    return functools.partial(
+        ResourceVariable,
+        initial_value=self.numpy(),
+        trainable=self.trainable,
+        name=self._shared_name,
+        dtype=self.dtype,
+        constraint=self.constraint,
+        distribute_strategy=self.distribute_strategy), ()
 
   def scatter_sub(self, sparse_delta, use_locking=False, name=None):
     """Subtracts `IndexedSlices` from this variable.
-- 
GitLab


From 35641fa70182fb9ee4c7e015b4ed38d0148cf340 Mon Sep 17 00:00:00 2001
From: Francois Chollet <fchollet@google.com>
Date: Tue, 8 Jan 2019 10:15:50 -0800
Subject: [PATCH 0381/1095] Make dataset instances passed to `fit`/etc
 propagate to the `model_iteration` loop. This is a necessary step in order to
 start resetting a dataset of known cardinality as part of the loop.

PiperOrigin-RevId: 228351978
---
 tensorflow/python/keras/engine/training.py    | 185 ++++++++----------
 .../python/keras/engine/training_arrays.py    |  23 ++-
 .../keras/engine/training_dataset_test.py     |  55 +++---
 .../python/keras/engine/training_utils.py     |  67 ++++++-
 4 files changed, 192 insertions(+), 138 deletions(-)

diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index bd06751428..1eda8cf797 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -19,14 +19,12 @@ from __future__ import division
 from __future__ import print_function
 
 import collections
-import weakref
 import numpy as np
 
 from tensorflow.python import tf2
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import iterator_ops
 from tensorflow.python.eager import context
-from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import tensor_util
@@ -122,10 +120,6 @@ class Model(Network):
 
   def __init__(self, *args, **kwargs):
     super(Model, self).__init__(*args, **kwargs)
-    # Create a cache for iterator get_next op.
-    self._iterator_get_next = weakref.WeakKeyDictionary()
-    # Create a cache for dataset - uninitialized iterators
-    self._dataset_iterator_cache = weakref.WeakKeyDictionary()
     # initializing _distribution_strategy here since it is possible to call
     # predict on a model without compiling it.
     self._distribution_strategy = None
@@ -1247,7 +1241,8 @@ class Model(Network):
                                 'compiled with DistributionStrategy.')
     # Validate and standardize user data.
     x, y, sample_weights = self._standardize_user_data(
-        x, y, sample_weight=sample_weight, class_weight=class_weight)
+        x, y, sample_weight=sample_weight, class_weight=class_weight,
+        extract_tensors_from_dataset=True)
 
     if self.run_eagerly:
       outputs = training_eager.train_on_batch(
@@ -1316,7 +1311,7 @@ class Model(Network):
                                 'compiled with DistributionStrategy.')
     # Validate and standardize user data.
     x, y, sample_weights = self._standardize_user_data(
-        x, y, sample_weight=sample_weight)
+        x, y, sample_weight=sample_weight, extract_tensors_from_dataset=True)
 
     if self.run_eagerly:
       outputs = training_eager.test_on_batch(
@@ -1359,7 +1354,8 @@ class Model(Network):
       raise NotImplementedError('`predict_on_batch` is not supported for '
                                 'models compiled with DistributionStrategy.')
     # Validate and standardize user data.
-    inputs, _, _ = self._standardize_user_data(x)
+    inputs, _, _ = self._standardize_user_data(
+        x, extract_tensors_from_dataset=True)
     if self.run_eagerly:
       if (isinstance(inputs, iterator_ops.EagerIterator) or
           (isinstance(inputs, dataset_ops.DatasetV2))):
@@ -2103,13 +2099,6 @@ class Model(Network):
       self._make_predict_function()
       return self.predict_function
 
-  def _get_iterator_get_next_tensors(self, iterator):
-    get_next_op = self._iterator_get_next.get(iterator, None)
-    if get_next_op is None:
-      get_next_op = iterator.get_next()
-      self._iterator_get_next[iterator] = get_next_op
-    return get_next_op
-
   def _distribution_standardize_user_data(self,
                                           x,
                                           y=None,
@@ -2231,7 +2220,8 @@ class Model(Network):
                              steps_name='steps',
                              steps=None,
                              validation_split=0,
-                             shuffle=False):
+                             shuffle=False,
+                             extract_tensors_from_dataset=False):
     """Runs validation checks on input and target data passed by the user.
 
     Also standardizes the data to lists of arrays, in order.
@@ -2275,6 +2265,10 @@ class Model(Network):
       validation_split: Float between 0 and 1.
         Fraction of the training data to be used as validation data.
       shuffle: Boolean whether to shuffle the training data before each epoch.
+      extract_tensors_from_dataset: Boolean. When `x` is a dataset instance,
+        this indicates whether to extract actual tensors from the dataset or
+        instead output the dataset instance itself.
+        Set to True when calling from `train_on_batch`/etc.
 
     Returns:
       A tuple of 3: inputs (arrays or dicts, depending on whether `x` was a dict
@@ -2287,60 +2281,30 @@ class Model(Network):
       ValueError: In case of invalid user-provided data.
       RuntimeError: If the model was never compiled.
     """
-    if isinstance(x, dataset_ops.DatasetV2):
-      if context.executing_eagerly():
-        x = iter(x)
-      else:
-        if x in self._dataset_iterator_cache:
-          x = self._dataset_iterator_cache[x]
-        else:
-          iterator = dataset_ops.make_initializable_iterator(x)
-          self._dataset_iterator_cache[x] = iterator
-          x = iterator
-        K.get_session().run(x.initializer)
+    if isinstance(x, (dataset_ops.DatasetV2, dataset_ops.DatasetV1)):
+      # Graph mode dataset. We'll pass the dataset as-is (unless
+      # `extract_tensors_from_dataset` is True, in which case we extract
+      # the tensors from the dataset and we output them.
+      training_utils.validate_dataset_input(x, y, sample_weight,
+                                            validation_split)
+      is_dataset = True
+      if extract_tensors_from_dataset:
+        # We do this for `train_on_batch`/etc.
+        x, y, sample_weight = training_utils.extract_tensors_from_dataset(x)
+    elif isinstance(x, iterator_ops.Iterator):
+      # Graph mode iterator. We extract the symbolic tensors.
+      training_utils.validate_dataset_input(x, y, sample_weight,
+                                            validation_split)
+      iterator = x
+      x, y, sample_weight = training_utils.unpack_iterator_input(iterator)
+      is_dataset = True
+    else:
+      is_dataset = False
 
     # Validates `steps` argument based on x's type.
     if check_steps:
       training_utils.check_steps_argument(x, steps, steps_name)
 
-    is_x_eager_iterator = isinstance(x, iterator_ops.EagerIterator)
-    is_x_iterator = isinstance(x, iterator_ops.Iterator)
-
-    # Validate user inputs when data is given as a dataset or dataset iterator.
-    if is_x_iterator or is_x_eager_iterator:
-      training_utils.validate_dataset_input(x, y, sample_weight,
-                                            validation_split)
-
-    # For eager iterators, when we have to process multiple batches of samples,
-    # we will standardize the data when we actually loop over iterator and get
-    # the batches. For now, we just return the iterator as is.
-    if is_x_eager_iterator:
-      return x, y, sample_weight
-
-    # If input data is a dataset iterator in graph mode or if it is an eager
-    # iterator and only one batch of samples is required, we fetch the data
-    # tensors from the iterator and then standardize them.
-    if is_x_iterator:
-      try:
-        next_element = self._get_iterator_get_next_tensors(x)
-      except errors.OutOfRangeError:
-        raise RuntimeError('Your dataset iterator ran out of data; '
-                           'Make sure that your dataset can generate '
-                           'required number of samples.')
-
-      if isinstance(next_element, (list, tuple)):
-        if len(next_element) not in [2, 3]:
-          raise ValueError(
-              'Please provide model inputs as a list or tuple of 2  or 3'
-              'elements: (input, target) or (input, target, sample_weights)'
-              'Received %s' % next_element)
-        if len(next_element) == 2:
-          x, y = next_element
-        else:
-          x, y, sample_weight = next_element
-      else:
-        x = next_element
-
     # First, we build/compile the model on the fly if necessary.
     all_inputs = []
     is_build_called = False
@@ -2349,40 +2313,51 @@ class Model(Network):
     # rather than list inputs (e.g. FeatureColumn-based models).
     dict_inputs = False
     if not self.inputs:
-      # We need to use `x` to set the model inputs.
-      # We type-check that `x` and `y` are either single arrays
+      # We need to use `x_input` to set the model inputs.
+
+      # If input data is a dataset iterator in graph mode or if it is an eager
+      # iterator and only one batch of samples is required, we fetch the data
+      # tensors from the iterator and then standardize them.
+      if isinstance(x, (dataset_ops.DatasetV2, dataset_ops.DatasetV1)):
+        x_input, y_input, _ = training_utils.extract_tensors_from_dataset(x)
+      else:
+        x_input = x
+        y_input = y
+      # We type-check that `x_input` and `y_input` are either single arrays
       # or lists of arrays.
-      if isinstance(x, (list, tuple)):
+      if isinstance(x_input, (list, tuple)):
         if not all(isinstance(v, np.ndarray) or
-                   tensor_util.is_tensor(v) for v in x):
+                   tensor_util.is_tensor(v) for v in x_input):
           raise ValueError('Please provide as model inputs either a single '
                            'array or a list of arrays. You passed: x=' + str(x))
-        all_inputs += list(x)
-      elif isinstance(x, dict):
+        all_inputs += list(x_input)
+      elif isinstance(x_input, dict):
         dict_inputs = True
-        keys = sorted(x.keys())
-        all_inputs = [x[k] for k in keys]
+        keys = sorted(x_input.keys())
+        all_inputs = [x_input[k] for k in keys]
       else:
-        if not isinstance(x, np.ndarray) and not tensor_util.is_tensor(x):
+        if (not isinstance(x_input, np.ndarray) and
+            not tensor_util.is_tensor(x_input)):
           raise ValueError('Please provide as model inputs either a single '
                            'array or a list of arrays. You passed: x=' + str(x))
-        all_inputs.append(x)
+        all_inputs.append(x_input)
 
       # Build the model using the retrieved inputs (value or symbolic).
       # If values or generated from a dataset, then in symbolic-mode
       # placeholders will be created to match the value shapes.
       is_build_called = True
-      if is_x_iterator:
-        cast_inputs = nest.map_structure(lambda v: v.shape, x)
-      elif training_utils.has_tensors(x):
-        cast_inputs = training_utils.cast_if_floating_dtype(x)
+      if is_dataset:
+        cast_inputs = nest.map_structure(lambda v: v.shape, x_input)
+      elif training_utils.has_tensors(x_input):
+        cast_inputs = training_utils.cast_if_floating_dtype(x_input)
       else:
-        cast_inputs = x
+        cast_inputs = x_input
       self._set_inputs(cast_inputs)
     else:
+      y_input = y
       dict_inputs = isinstance(self.inputs, dict)
 
-    if y is not None:
+    if y_input is not None:
       if not self.optimizer:
         raise RuntimeError('You must compile a model before '
                            'training/testing. '
@@ -2390,23 +2365,24 @@ class Model(Network):
       if not self._is_compiled:
         # On-the-fly compilation of the model.
         # We need to use `y` to set the model targets.
-        if training_utils.has_tensors(y):
-          y = training_utils.cast_if_floating_dtype(y)
-        if isinstance(y, (list, tuple)):
+        if training_utils.has_tensors(y_input):
+          y_input = training_utils.cast_if_floating_dtype(y_input)
+        if isinstance(y_input, (list, tuple)):
           if not all(isinstance(v, np.ndarray) or
-                     tensor_util.is_tensor(v) for v in y):
+                     tensor_util.is_tensor(v) for v in y_input):
             raise ValueError('Please provide as model targets either a single '
                              'array or a list of arrays. '
                              'You passed: y=' + str(y))
-          all_inputs += list(y)
-        elif isinstance(y, dict):
-          raise ValueError('Please do not pass a dictionary as model targets.')
+          all_inputs += list(y_input)
+        elif isinstance(y_input, dict):
+          raise ValueError('You cannot pass a dictionary as model targets.')
         else:
-          if not isinstance(y, np.ndarray) and not tensor_util.is_tensor(y):
+          if (not isinstance(y_input, np.ndarray) and
+              not tensor_util.is_tensor(y_input)):
             raise ValueError('Please provide as model targets either a single '
                              'array or a list of arrays. '
                              'You passed: y=' + str(y))
-          all_inputs.append(y)
+          all_inputs.append(y_input)
 
         # Typecheck that all inputs are *either* value *or* symbolic.
         # TODO(fchollet): this check could be removed in Eager mode?
@@ -2416,13 +2392,13 @@ class Model(Network):
                              'TensorFlow tensors. '
                              'You passed: x=' + str(x) + '; y=' + str(y))
 
-        if self.run_eagerly or is_x_iterator:
+        if is_dataset or context.executing_eagerly():
           target_tensors = None
         else:
           # Handle target tensors if any passed.
-          if not isinstance(y, (list, tuple)):
-            y = [y]
-          target_tensors = [v for v in y if _is_symbolic_tensor(v)]
+          if not isinstance(y_input, (list, tuple)):
+            y_input = [y_input]
+          target_tensors = [v for v in y_input if _is_symbolic_tensor(v)]
         is_compile_called = True
         self.compile(
             optimizer=self.optimizer,
@@ -2440,7 +2416,7 @@ class Model(Network):
     # Note: in this case, `any` and `all` are equivalent since we disallow
     # mixed symbolic/value inputs.
     if (not self.run_eagerly and is_build_called and is_compile_called and
-        not is_x_iterator and any(_is_symbolic_tensor(v) for v in all_inputs)):
+        not is_dataset  and any(_is_symbolic_tensor(v) for v in all_inputs)):
       return [], [], []
 
     # What follows is input validation and standardization to list format,
@@ -2462,12 +2438,14 @@ class Model(Network):
       feed_input_shapes = self._feed_input_shapes
 
     # Standardize the inputs.
-    x = training_utils.standardize_input_data(
-        x,
-        feed_input_names,
-        feed_input_shapes,
-        check_batch_axis=False,  # Don't enforce the batch size.
-        exception_prefix='input')
+    if not isinstance(x, (dataset_ops.DatasetV2, dataset_ops.DatasetV1)):
+      # TODO(fchollet): run static checks with dataset output shape(s).
+      x = training_utils.standardize_input_data(
+          x,
+          feed_input_names,
+          feed_input_shapes,
+          check_batch_axis=False,  # Don't enforce the batch size.
+          exception_prefix='input')
 
     if y is not None:
       if not self._is_graph_network:
@@ -2541,7 +2519,8 @@ class Model(Network):
                          str(x[0].shape[0]) + ' samples')
 
     # If dictionary inputs were provided, we return a dictionary as well.
-    if dict_inputs:
+    if dict_inputs and not isinstance(x, (dataset_ops.DatasetV2,
+                                          dataset_ops.DatasetV1)):
       x = dict(zip(feed_input_names, x))
     return x, y, sample_weights
 
diff --git a/tensorflow/python/keras/engine/training_arrays.py b/tensorflow/python/keras/engine/training_arrays.py
index 9887fe34a8..ab7d455bfa 100644
--- a/tensorflow/python/keras/engine/training_arrays.py
+++ b/tensorflow/python/keras/engine/training_arrays.py
@@ -23,6 +23,7 @@ import functools
 
 import numpy as np
 
+from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.eager import context
 from tensorflow.python.framework import errors
 from tensorflow.python.keras import backend as K
@@ -146,15 +147,15 @@ def model_iteration(model,
 
   Arguments:
       model: Keras Model instance.
-      inputs: Either a list of arrays or a dictionary.
-      targets: List of target arrays.
+      inputs: Either a list or dictionary of arrays, or a dataset instance.
+      targets: List/dictionary of input arrays.
       sample_weights: Optional list of sample weight arrays.
       batch_size: Integer batch size or None if unknown.
       epochs: Number of times to iterate over the data
       verbose: Verbosity mode, 0, 1 or 2
       callbacks: List of callbacks to be called during training
-      val_inputs: List of input arrays.
-      val_targets: List of target arrays.
+      val_inputs: Either a list or dictionary of arrays, or a dataset instance.
+      val_targets: List/dictionary of target arrays.
       val_sample_weights: Optional list of sample weight arrays.
       shuffle: Whether to shuffle the data at the beginning of each epoch
         concatenation of list the display names of the outputs of `f` and the
@@ -186,6 +187,20 @@ def model_iteration(model,
   if 'steps' in kwargs:
     steps_per_epoch = kwargs['steps']
 
+  # In case we are passed datasets, we extract symbolic tensors from them.
+  if isinstance(inputs, (dataset_ops.DatasetV2, dataset_ops.DatasetV1)):
+    inputs, targets, sample_weights = model._standardize_user_data(
+        inputs,
+        steps_name='steps_per_epoch',
+        steps=steps_per_epoch,
+        extract_tensors_from_dataset=True)
+  if isinstance(val_inputs, (dataset_ops.DatasetV2, dataset_ops.DatasetV1)):
+    val_inputs, val_targets, val_sample_weights = model._standardize_user_data(
+        val_inputs,
+        steps_name='validation_steps',
+        steps=validation_steps,
+        extract_tensors_from_dataset=True)
+
   _validate_arguments(steps_per_epoch, validation_steps, kwargs)
   if mode == ModeKeys.TRAIN:
     _print_train_info(inputs, val_inputs, steps_per_epoch, verbose)
diff --git a/tensorflow/python/keras/engine/training_dataset_test.py b/tensorflow/python/keras/engine/training_dataset_test.py
index 646ce31909..40d42c13f0 100644
--- a/tensorflow/python/keras/engine/training_dataset_test.py
+++ b/tensorflow/python/keras/engine/training_dataset_test.py
@@ -25,7 +25,6 @@ import numpy as np
 from tensorflow.python import keras
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.eager import context
-from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util as tf_test_util
 from tensorflow.python.keras import keras_parameterized
 from tensorflow.python.keras import metrics as metrics_module
@@ -99,29 +98,6 @@ class TestTrainingWithDatasetIterators(keras_parameterized.TestCase):
                                  'the `steps` argument'):
       model.predict(iterator, verbose=0)
 
-  @keras_parameterized.run_with_all_model_types
-  @keras_parameterized.run_all_keras_modes
-  def test_get_next_op_created_once(self):
-    model = testing_utils.get_small_mlp(1, 4, input_dim=3)
-    optimizer = RMSPropOptimizer(learning_rate=0.001)
-    loss = 'mse'
-    metrics = ['mae']
-    model.compile(optimizer, loss, metrics=metrics,
-                  run_eagerly=testing_utils.should_run_eagerly())
-
-    inputs = np.zeros((10, 3), np.float32)
-    targets = np.zeros((10, 4), np.float32)
-    dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
-    dataset = dataset.repeat(100)
-    dataset = dataset.batch(10)
-    iterator = dataset_ops.make_one_shot_iterator(dataset)
-
-    model.fit(iterator, epochs=1, steps_per_epoch=2, verbose=1)
-    # Finalize graph to make sure we are not appending another iterator
-    # get_next op in the graph.
-    ops.get_default_graph().finalize()
-    model.fit(iterator, epochs=1, steps_per_epoch=2, verbose=1)
-
   @keras_parameterized.run_with_all_model_types
   @keras_parameterized.run_all_keras_modes
   def test_iterators_running_out_of_data(self):
@@ -172,9 +148,6 @@ class TestTrainingWithDataset(keras_parameterized.TestCase):
     # Call fit with validation data
     model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0,
               validation_data=dataset, validation_steps=2)
-    # Finalize the graph to make sure new ops aren't added when calling on the
-    # same dataset
-    ops.get_default_graph().finalize()
     model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0,
               validation_data=dataset, validation_steps=2)
 
@@ -292,6 +265,34 @@ class TestTrainingWithDataset(keras_parameterized.TestCase):
 
       model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=1)
 
+  @keras_parameterized.run_all_keras_modes
+  def test_dataset_fit_correctness(self):
+
+    class SumLayer(keras.layers.Layer):
+
+      def build(self, _):
+        self.w = self.add_weight('w', ())
+
+      def call(self, inputs):
+        return keras.backend.sum(inputs) + self.w * 0
+
+    model = keras.Sequential([SumLayer(input_shape=(2,))])
+    model.compile(RMSPropOptimizer(learning_rate=0.001),
+                  loss='mae',
+                  run_eagerly=testing_utils.should_run_eagerly())
+
+    inputs = np.zeros((40, 2), dtype=np.float32)
+    inputs[10:20, :] = 2
+    inputs[20:30, :] = 1
+    inputs[30:, :] = 4
+    targets = np.zeros((40, 1), dtype=np.float32)
+    dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
+    dataset = dataset.batch(10)
+    history = model.fit(dataset,
+                        epochs=2, steps_per_epoch=2, verbose=1, shuffle=False)
+    self.assertListEqual(history.history['loss'],
+                         [inputs[:20].sum() / 2, inputs[20:].sum() / 2])
+
   @tf_test_util.run_deprecated_v1
   def test_dataset_input_shape_validation(self):
     with self.cached_session():
diff --git a/tensorflow/python/keras/engine/training_utils.py b/tensorflow/python/keras/engine/training_utils.py
index 5edf1de1b0..949f1e400d 100644
--- a/tensorflow/python/keras/engine/training_utils.py
+++ b/tensorflow/python/keras/engine/training_utils.py
@@ -31,6 +31,7 @@ from tensorflow.python.data.ops import readers
 from tensorflow.python.eager import context
 from tensorflow.python.eager import def_function
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import tensor_spec
@@ -1081,10 +1082,10 @@ def is_feature_layer(layer):
 
 
 def is_eager_dataset_or_iterator(data):
-  if context.executing_eagerly():
-    if isinstance(data, (dataset_ops.DatasetV2, iterator_ops.EagerIterator)):
-      return True
-  return False
+  return context.executing_eagerly() and isinstance(
+      data, (dataset_ops.DatasetV1,
+             dataset_ops.DatasetV2,
+             iterator_ops.EagerIterator))
 
 
 # pylint: disable=protected-access
@@ -1199,6 +1200,64 @@ def assert_not_shuffled(dataset):
     raise ValueError('Could not assert that dataset is not shuffled.')
 
 
+def is_dataset_or_iterator(data):
+  return isinstance(data, (dataset_ops.DatasetV1,
+                           dataset_ops.DatasetV2,
+                           iterator_ops.EagerIterator,
+                           iterator_ops.Iterator))
+
+
+def extract_tensors_from_dataset(dataset):
+  """Extract a tuple of tensors `inputs, targets, sample_weight` from a dataset.
+
+  Works only for graph mode.
+
+  Arguments:
+    dataset: Dataset instance.
+
+  Returns:
+    Tuple of tensors `x, y, weights`. `y` and `weights` entry may be None.
+  """
+  iterator = dataset_ops.make_initializable_iterator(dataset)
+  K.get_session().run(iterator.initializer)
+  inputs, targets, sample_weight = unpack_iterator_input(iterator)
+  return inputs, targets, sample_weight
+
+
+def unpack_iterator_input(iterator):
+  """Convert a dataset iterator to a tuple of tensors `x, y, sample_weights`.
+
+  Arguments:
+    iterator: Instance of a dataset iterator.
+
+  Returns:
+    Tuple of tensors `x, y, weights`. `y` and `weights` entry may be None.
+  """
+  try:
+    next_element = iterator.get_next()
+  except errors.OutOfRangeError:
+    raise RuntimeError('Your dataset iterator ran out of data; '
+                       'Make sure that your dataset can generate '
+                       'required number of samples.')
+
+  if isinstance(next_element, (list, tuple)):
+    if len(next_element) not in [2, 3]:
+      raise ValueError(
+          'Please provide model inputs as a list or tuple of 2 or 3 '
+          'elements: (input, target) or (input, target, sample_weights) '
+          'Received %s' % next_element)
+    if len(next_element) == 2:
+      x, y = next_element
+      weights = None
+    else:
+      x, y, weights = next_element
+  else:
+    x = next_element
+    y = None
+    weights = None
+  return x, y, weights
+
+
 class ModelInputs(object):
   """Encapsulates model inputs.
 
-- 
GitLab


From e5319d26bb5b3fed5e3c9c865193db41f75cf958 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Tue, 8 Jan 2019 10:17:23 -0800
Subject: [PATCH 0382/1095] Add mean relative error v2 metric.

PiperOrigin-RevId: 228352265
---
 tensorflow/python/keras/metrics.py      | 80 +++++++++++++++++++++++++
 tensorflow/python/keras/metrics_test.py | 56 +++++++++++++++++
 2 files changed, 136 insertions(+)

diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index 9a2e057780..2c2700ffa3 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -51,6 +51,7 @@ from tensorflow.python.keras.utils.generic_utils import serialize_keras_object
 from tensorflow.python.keras.utils.generic_utils import to_list
 from tensorflow.python.keras.utils.losses_utils import squeeze_or_expand_dimensions
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import confusion_matrix
 from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn
@@ -346,6 +347,85 @@ class Mean(Metric):
     return math_ops.div_no_nan(self.total, self.count)
 
 
+class MeanRelativeError(Mean):
+  """Computes the mean relative error by normalizing with the given values.
+
+  This metric creates two local variables, `total` and `count` that are used to
+  compute the mean relative absolute error. This average is weighted by
+  `sample_weight`, and it is ultimately returned as `mean_relative_error`:
+  an idempotent operation that simply divides `total` by `count`.
+
+  If `sample_weight` is `None`, weights default to 1.
+  Use `sample_weight` of 0 to mask values.
+
+  Usage:
+
+  ```python
+  m = tf.keras.metrics.MeanRelativeError(normalizer=[1, 3, 2, 3])
+  m.update_state([1, 3, 2, 3], [2, 4, 6, 8])
+
+  # metric = mean(|y_pred - y_true| / normalizer)
+  #        = mean([1, 1, 4, 5] / [1, 3, 2, 3]) = mean([1, 1/3, 2, 5/3])
+  #        = 5/4 = 1.25
+  print('Final result: ', m.result().numpy())  # Final result: 1.25
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile(
+    'sgd',
+    loss='mse',
+    metrics=[tf.keras.metrics.MeanRelativeError(normalizer=[1, 3])])
+  ```
+  """
+
+  def __init__(self, normalizer, name=None, dtype=None):
+    """Creates a `MeanRelativeError` instance.
+
+    Args:
+      normalizer: The normalizer values with same shape as predictions.
+      name: (Optional) string name of the metric instance.
+      dtype: (Optional) data type of the metric result.
+    """
+    super(MeanRelativeError, self).__init__(name=name, dtype=dtype)
+    normalizer = math_ops.cast(normalizer, self._dtype)
+    self.normalizer = normalizer
+
+  def update_state(self, y_true, y_pred, sample_weight=None):
+    """Accumulates metric statistics.
+
+    Args:
+      y_true: The ground truth values.
+      y_pred: The predicted values.
+      sample_weight: Optional weighting of each example. Defaults to 1. Can be a
+        `Tensor` whose rank is either 0, or the same rank as `y_true`, and must
+        be broadcastable to `y_true`.
+
+    Returns:
+      Update op.
+    """
+    y_true = math_ops.cast(y_true, self._dtype)
+    y_pred = math_ops.cast(y_pred, self._dtype)
+    y_pred, y_true, sample_weight = squeeze_or_expand_dimensions(
+        y_pred, y_true, sample_weight)
+
+    y_pred, self.normalizer = confusion_matrix.remove_squeezable_dimensions(
+        y_pred, self.normalizer)
+    y_pred.shape.assert_is_compatible_with(y_pred.shape)
+    relative_errors = math_ops.div_no_nan(
+        math_ops.abs(y_true - y_pred), self.normalizer)
+
+    return super(MeanRelativeError, self).update_state(
+        relative_errors, sample_weight=sample_weight)
+
+  def get_config(self):
+    config = {'normalizer': self.normalizer}
+    base_config = super(MeanRelativeError, self).get_config()
+    return dict(list(base_config.items()) + list(config.items()))
+
+
 class MeanMetricWrapper(Mean):
   """Wraps a stateless metric function with the Mean metric."""
 
diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py
index a02b95bb46..4f8406f0e3 100644
--- a/tensorflow/python/keras/metrics_test.py
+++ b/tensorflow/python/keras/metrics_test.py
@@ -1603,6 +1603,62 @@ class KullbackLeiblerDivergenceTest(test.TestCase):
     self.assertAllClose(self.evaluate(result), expected_result, atol=1e-3)
 
 
+@test_util.run_all_in_graph_and_eager_modes
+class MeanRelativeErrorTest(test.TestCase):
+
+  def test_config(self):
+    normalizer = constant_op.constant([1, 3], dtype=dtypes.float32)
+    mre_obj = metrics.MeanRelativeError(normalizer=normalizer, name='mre')
+    self.assertEqual(mre_obj.name, 'mre')
+    self.assertArrayNear(self.evaluate(mre_obj.normalizer), [1, 3], 1e-1)
+
+    mre_obj2 = metrics.MeanRelativeError.from_config(mre_obj.get_config())
+    self.assertEqual(mre_obj2.name, 'mre')
+    self.assertArrayNear(self.evaluate(mre_obj2.normalizer), [1, 3], 1e-1)
+
+  def test_unweighted(self):
+    np_y_pred = np.asarray([2, 4, 6, 8], dtype=np.float32)
+    np_y_true = np.asarray([1, 3, 2, 3], dtype=np.float32)
+    expected_error = np.mean(
+        np.divide(np.absolute(np_y_pred - np_y_true), np_y_true))
+
+    y_pred = constant_op.constant(np_y_pred, shape=(1, 4), dtype=dtypes.float32)
+    y_true = constant_op.constant(np_y_true, shape=(1, 4))
+
+    mre_obj = metrics.MeanRelativeError(normalizer=y_true)
+    self.evaluate(variables.variables_initializer(mre_obj.variables))
+
+    result = mre_obj(y_true, y_pred)
+    self.assertAllClose(self.evaluate(result), expected_error, atol=1e-3)
+
+  def test_weighted(self):
+    np_y_pred = np.asarray([2, 4, 6, 8], dtype=np.float32)
+    np_y_true = np.asarray([1, 3, 2, 3], dtype=np.float32)
+    sample_weight = np.asarray([0.2, 0.3, 0.5, 0], dtype=np.float32)
+    rel_errors = np.divide(np.absolute(np_y_pred - np_y_true), np_y_true)
+    expected_error = np.sum(rel_errors * sample_weight)
+
+    y_pred = constant_op.constant(np_y_pred, dtype=dtypes.float32)
+    y_true = constant_op.constant(np_y_true)
+
+    mre_obj = metrics.MeanRelativeError(normalizer=y_true)
+    self.evaluate(variables.variables_initializer(mre_obj.variables))
+
+    result = mre_obj(
+        y_true, y_pred, sample_weight=constant_op.constant(sample_weight))
+    self.assertAllClose(self.evaluate(result), expected_error, atol=1e-3)
+
+  def test_zero_normalizer(self):
+    y_pred = constant_op.constant([2, 4], dtype=dtypes.float32)
+    y_true = constant_op.constant([1, 3])
+
+    mre_obj = metrics.MeanRelativeError(normalizer=array_ops.zeros_like(y_true))
+    self.evaluate(variables.variables_initializer(mre_obj.variables))
+
+    result = mre_obj(y_true, y_pred)
+    self.assertEqual(self.evaluate(result), 0)
+
+
 def _get_model(compile_metrics):
   model_layers = [
       layers.Dense(3, activation='relu', kernel_initializer='ones'),
-- 
GitLab


From dff89fe60f5bbf1aa6c124fe295c9af3d9e3a868 Mon Sep 17 00:00:00 2001
From: Tong Shen <endlessroad@google.com>
Date: Tue, 8 Jan 2019 10:18:15 -0800
Subject: [PATCH 0383/1095] Support outside compilation in function call.

PiperOrigin-RevId: 228352466
---
 tensorflow/compiler/jit/BUILD                 |   1 +
 .../jit/encapsulate_subgraphs_pass_test.cc    |  98 ++++++--
 .../jit/extract_outside_compilation_pass.cc   | 196 +++++++++++++---
 .../jit/extract_outside_compilation_pass.h    |   4 +-
 .../extract_outside_compilation_pass_test.cc  | 216 +++++++++++++++++-
 tensorflow/compiler/tf2xla/graph_compiler.cc  |  28 ++-
 .../compiler/tf2xla/side_effect_util.cc       |   4 +
 7 files changed, 475 insertions(+), 72 deletions(-)

diff --git a/tensorflow/compiler/jit/BUILD b/tensorflow/compiler/jit/BUILD
index b9a87ba296..ba9fa93654 100644
--- a/tensorflow/compiler/jit/BUILD
+++ b/tensorflow/compiler/jit/BUILD
@@ -634,6 +634,7 @@ tf_cc_test(
         "//tensorflow/core:framework_internal",
         "//tensorflow/core:lib",
         "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:session_options",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
diff --git a/tensorflow/compiler/jit/encapsulate_subgraphs_pass_test.cc b/tensorflow/compiler/jit/encapsulate_subgraphs_pass_test.cc
index 8617beec00..1f8ec09e19 100644
--- a/tensorflow/compiler/jit/encapsulate_subgraphs_pass_test.cc
+++ b/tensorflow/compiler/jit/encapsulate_subgraphs_pass_test.cc
@@ -25,6 +25,8 @@ limitations under the License.
 #include "tensorflow/compiler/jit/encapsulate_util.h"
 #include "tensorflow/compiler/jit/extract_outside_compilation_pass.h"
 #include "tensorflow/compiler/tf2xla/side_effect_util.h"
+#include "tensorflow/core/common_runtime/device_factory.h"
+#include "tensorflow/core/common_runtime/function.h"
 #include "tensorflow/core/framework/function_testlib.h"
 #include "tensorflow/core/framework/graph_to_functiondef.h"
 #include "tensorflow/core/graph/graph_constructor.h"
@@ -32,6 +34,8 @@ limitations under the License.
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/platform/test.h"
+#include "tensorflow/core/public/session_options.h"
+#include "tensorflow/core/public/version.h"
 #include "tensorflow/core/util/equal_graph_def.h"
 
 namespace tensorflow {
@@ -513,6 +517,18 @@ Status Encapsulate(GraphDef* graphdef, FunctionDefLibrary* library,
   s = PerformStaticShapeInferenceBeforeEncapsulation(graph.get());
   if (!s.ok()) return s;
 
+  // Create FunctionLibraryRuntime.
+  SessionOptions session_options;
+  std::vector<std::unique_ptr<Device>> devices;
+  TF_CHECK_OK(DeviceFactory::AddDevices(
+      session_options, "/job:localhost/replica:0/task:0", &devices));
+  OptimizerOptions opts;
+  auto device_mgr = absl::make_unique<DeviceMgr>(std::move(devices));
+  auto pflr = absl::make_unique<ProcessFunctionLibraryRuntime>(
+      device_mgr.get(), Env::Default(), TF_GRAPH_DEF_VERSION, lib_def.get(),
+      opts, /*default_thread_pool=*/nullptr, /*cluster_flr=*/nullptr);
+  auto flr = pflr->GetFLR("/job:localhost/replica:0/task:0/cpu:0");
+
   std::unique_ptr<Graph> graph_out;
   s = EncapsulateSubgraphsInFunctions(
       "_encapsulate", /*outside_compilation_attribute=*/"", *graph,
@@ -538,7 +554,7 @@ Status Encapsulate(GraphDef* graphdef, FunctionDefLibrary* library,
                                     std::map<string, int>{}});
   }
   s = ExtractOutsideCompilation("_encapsulate", "_outside", clusters,
-                                graph_out.get(), lib_def.get());
+                                graph_out.get(), flr, lib_def.get());
   if (!s.ok()) return s;
 
   GraphDef graphdef_out;
@@ -941,7 +957,9 @@ TEST(EncapsulateSubgraphsTest, OneFunctionOneOutside) {
             {"key", "host_compute_channel_F1_O1"},
             {"shape_inference_graph", shape_inference_graph},
             {"shapes", absl::Span<const DataType>({})},
-            {"_outside_compilation_subgraph", "O1"}},
+            {"_outside_compilation_subgraph", "O1"},
+            {"_xla_token_input_nodes",
+             absl::Span<const string>({"_xla_token_arg_node"})}},
            {"c"}},
       },
       {{"f_0_retval_retval", "F:o:0"}});
@@ -1101,7 +1119,9 @@ TEST(EncapsulateSubgraphsTest, OneFunctionTwoOutside) {
             {"key", "host_compute_channel_F1_O2"},
             {"shape_inference_graph", shape_inference_graph2},
             {"shapes", absl::Span<const DataType>({})},
-            {"_outside_compilation_subgraph", "O2"}},
+            {"_outside_compilation_subgraph", "O2"},
+            {"_xla_token_input_nodes",
+             absl::Span<const string>({"_xla_token_arg_node"})}},
            {"F"}},
           {{"outside_compilation_O1_host_compute"},
            "XlaHostCompute",
@@ -1112,7 +1132,9 @@ TEST(EncapsulateSubgraphsTest, OneFunctionTwoOutside) {
             {"key", "host_compute_channel_F1_O1"},
             {"shape_inference_graph", shape_inference_graph1},
             {"shapes", absl::Span<const DataType>({})},
-            {"_outside_compilation_subgraph", "O1"}},
+            {"_outside_compilation_subgraph", "O1"},
+            {"_xla_token_input_nodes",
+             absl::Span<const string>({"_xla_token_arg_node"})}},
            {"D"}},
       },
       {{"g_0_retval_retval", "outside_compilation_O2_host_compute:outputs:0"},
@@ -1244,7 +1266,9 @@ TEST(EncapsulateSubgraphsTest, TwoFunctionsTwoOutside) {
             {"shape_inference_graph", NameAttrList()},
             {"shapes",
              absl::Span<const TensorShapeProto>({shape_proto_expected})},
-            {"_outside_compilation_subgraph", "O1"}},
+            {"_outside_compilation_subgraph", "O1"},
+            {"_xla_token_input_nodes",
+             absl::Span<const string>({"_xla_token_arg_node"})}},
            {"D"}},
       },
       {{"e_0_retval_retval", "outside_compilation_O1_host_compute:outputs:0"},
@@ -1269,7 +1293,9 @@ TEST(EncapsulateSubgraphsTest, TwoFunctionsTwoOutside) {
             {"shape_inference_graph", NameAttrList()},
             {"shapes",
              absl::Span<const TensorShapeProto>({shape_proto_expected})},
-            {"_outside_compilation_subgraph", "O1"}}},
+            {"_outside_compilation_subgraph", "O1"},
+            {"_xla_token_input_nodes",
+             absl::Span<const string>({"_xla_token_arg_node"})}}},
       },
       {{"g_0_retval_retval", "G:o:0"}, {"i_0_retval_retval", "I:o:0"}});
 
@@ -1397,7 +1423,9 @@ TEST(EncapsulateSubgraphsTest, TwoFunctionsTwoOutsideDependencyFromOutside) {
             {"shape_inference_graph", NameAttrList()},
             {"shapes",
              absl::Span<const TensorShapeProto>({shape_proto_expected})},
-            {"_outside_compilation_subgraph", "O1"}},
+            {"_outside_compilation_subgraph", "O1"},
+            {"_xla_token_input_nodes",
+             absl::Span<const string>({"_xla_token_arg_node"})}},
            {"D"}},
       },
       {{"f_0_retval_retval", "F:o:0"}});
@@ -1419,7 +1447,9 @@ TEST(EncapsulateSubgraphsTest, TwoFunctionsTwoOutsideDependencyFromOutside) {
             {"shape_inference_graph", NameAttrList()},
             {"shapes",
              absl::Span<const TensorShapeProto>({shape_proto_expected})},
-            {"_outside_compilation_subgraph", "O1"}}},
+            {"_outside_compilation_subgraph", "O1"},
+            {"_xla_token_input_nodes",
+             absl::Span<const string>({"_xla_token_arg_node"})}}},
       },
       {{"i_0_retval_retval", "I:o:0"}});
 
@@ -1527,7 +1557,9 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationNoInputs) {
             {"shape_inference_graph", NameAttrList()},
             {"shapes",
              absl::Span<const TensorShapeProto>({shape_proto_expected})},
-            {"_outside_compilation_subgraph", "O1"}}},
+            {"_outside_compilation_subgraph", "O1"},
+            {"_xla_token_input_nodes",
+             absl::Span<const string>({"_xla_token_arg_node"})}}},
       },
       {{"f_0_retval_retval", "F:o:0"}});
 
@@ -1615,7 +1647,9 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationControlInput) {
             {"shape_inference_graph", NameAttrList()},
             {"shapes",
              absl::Span<const TensorShapeProto>({shape_proto_expected})},
-            {"_outside_compilation_subgraph", "O1"}},
+            {"_outside_compilation_subgraph", "O1"},
+            {"_xla_token_input_nodes",
+             absl::Span<const string>({"_xla_token_arg_node"})}},
            {"D"}},
       },
       {{"f_0_retval_retval", "F:o:0"}});
@@ -1716,7 +1750,9 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationNoOutputs) {
             {"key", "host_compute_channel_F1_O1"},
             {"shape_inference_graph", shape_inference_graph},
             {"shapes", absl::Span<const TensorShapeProto>({})},
-            {"_outside_compilation_subgraph", "O1"}}},
+            {"_outside_compilation_subgraph", "O1"},
+            {"_xla_token_input_nodes",
+             absl::Span<const string>({"_xla_token_arg_node"})}}},
       },
       {{"e_0_retval_retval", "outside_compilation_O1_host_compute:outputs:0"},
        {"f_0_retval_retval", "F:o:0"}});
@@ -1821,7 +1857,9 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationControlOutput) {
             {"key", "host_compute_channel_F1_O1"},
             {"shape_inference_graph", shape_inference_graph},
             {"shapes", absl::Span<const TensorShapeProto>({})},
-            {"_outside_compilation_subgraph", "O1"}}},
+            {"_outside_compilation_subgraph", "O1"},
+            {"_xla_token_input_nodes",
+             absl::Span<const string>({"_xla_token_arg_node"})}}},
       },
       {{"e_0_retval_retval", "outside_compilation_O1_host_compute:outputs:0"},
        {"f_0_retval_retval", "F:o:0"}});
@@ -1949,7 +1987,9 @@ TEST(EncapsulateSubgraphsTest,
             {"key", "host_compute_channel_F1_O1"},
             {"shape_inference_graph", shape_inference_graph1},
             {"shapes", absl::Span<const TensorShapeProto>({})},
-            {"_outside_compilation_subgraph", "O1"}}},
+            {"_outside_compilation_subgraph", "O1"},
+            {"_xla_token_input_nodes",
+             absl::Span<const string>({"_xla_token_arg_node"})}}},
           {{"outside_compilation_O2_host_compute"},
            "XlaHostCompute",
            {"F:o:0"},
@@ -1959,7 +1999,9 @@ TEST(EncapsulateSubgraphsTest,
             {"key", "host_compute_channel_F1_O2"},
             {"shape_inference_graph", shape_inference_graph2},
             {"shapes", absl::Span<const TensorShapeProto>({})},
-            {"_outside_compilation_subgraph", "O2"}}},
+            {"_outside_compilation_subgraph", "O2"},
+            {"_xla_token_input_nodes",
+             absl::Span<const string>({"_xla_token_arg_node"})}}},
       },
       {{"e_0_retval_retval", "outside_compilation_O1_host_compute:outputs:0"},
        {"h_0_retval_retval", "H:o:0"}});
@@ -2082,7 +2124,9 @@ TEST(EncapsulateSubgraphsTest,
             {"key", "host_compute_channel_F1_O2"},
             {"shape_inference_graph", NameAttrList()},
             {"shapes", absl::Span<const TensorShapeProto>({})},
-            {"_outside_compilation_subgraph", "O2"}}},
+            {"_outside_compilation_subgraph", "O2"},
+            {"_xla_token_input_nodes",
+             absl::Span<const string>({"_xla_token_arg_node"})}}},
           {{"outside_compilation_O1_host_compute"},
            "XlaHostCompute",
            {"D:o:0"},
@@ -2092,7 +2136,9 @@ TEST(EncapsulateSubgraphsTest,
             {"key", "host_compute_channel_F1_O1"},
             {"shape_inference_graph", shape_inference_graph},
             {"shapes", absl::Span<const TensorShapeProto>({})},
-            {"_outside_compilation_subgraph", "O1"}}},
+            {"_outside_compilation_subgraph", "O1"},
+            {"_xla_token_input_nodes",
+             absl::Span<const string>({"_xla_token_arg_node"})}}},
       },
       {{"e_0_retval_retval", "outside_compilation_O1_host_compute:outputs:0"},
        {"h_0_retval_retval", "H:o:0"}});
@@ -2214,7 +2260,9 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationClusterDependency) {
          {"key", "host_compute_channel_F1_O1"},
          {"shape_inference_graph", shape_inference_graph},
          {"shapes", absl::Span<const TensorShapeProto>({})},
-         {"_outside_compilation_subgraph", "O1"}}},
+         {"_outside_compilation_subgraph", "O1"},
+         {"_xla_token_input_nodes",
+          absl::Span<const string>({"_xla_token_arg_node"})}}},
        {{"outside_compilation_O2_host_compute"},
         "XlaHostCompute",
         {"D:o:0"},
@@ -2224,7 +2272,9 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationClusterDependency) {
          {"key", "host_compute_channel_F1_O2"},
          {"shape_inference_graph", NameAttrList()},
          {"shapes", absl::Span<const TensorShapeProto>({})},
-         {"_outside_compilation_subgraph", "O2"}},
+         {"_outside_compilation_subgraph", "O2"},
+         {"_xla_token_input_nodes",
+          absl::Span<const string>({"_xla_token_arg_node"})}},
         {}},
        {{"outside_compilation_O3_host_compute"},
         "XlaHostCompute",
@@ -2235,7 +2285,9 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationClusterDependency) {
          {"key", "host_compute_channel_F1_O3"},
          {"shape_inference_graph", NameAttrList()},
          {"shapes", absl::Span<const TensorShapeProto>({})},
-         {"_outside_compilation_subgraph", "O3"}},
+         {"_outside_compilation_subgraph", "O3"},
+         {"_xla_token_input_nodes",
+          absl::Span<const string>({"_xla_token_arg_node"})}},
         {}}},
       {{"e_0_retval_retval", "outside_compilation_O1_host_compute:outputs:0"},
        {"h_0_retval_retval", "H:o:0"}});
@@ -2354,7 +2406,9 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationNoInputsOrOutputs) {
             {"key", "host_compute_channel_F1_O1"},
             {"shape_inference_graph", shape_inference_graph},
             {"shapes", absl::Span<const TensorShapeProto>({})},
-            {"_outside_compilation_subgraph", "O1"}}},
+            {"_outside_compilation_subgraph", "O1"},
+            {"_xla_token_input_nodes",
+             absl::Span<const string>({"_xla_token_arg_node"})}}},
       },
       {{"e_0_retval_retval", "outside_compilation_O1_host_compute:outputs:0"},
        {"f_0_retval_retval", "F:o:0"}});
@@ -2465,7 +2519,9 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationShapeInference) {
             {"key", "host_compute_channel_F1_O1"},
             {"shape_inference_graph", shape_inference_graph},
             {"shapes", absl::Span<const DataType>({})},
-            {"_outside_compilation_subgraph", "O1"}},
+            {"_outside_compilation_subgraph", "O1"},
+            {"_xla_token_input_nodes",
+             absl::Span<const string>({"_xla_token_arg_node"})}},
            {"c"}},
       },
       {{"f_0_retval_retval", "F:o:0"}});
diff --git a/tensorflow/compiler/jit/extract_outside_compilation_pass.cc b/tensorflow/compiler/jit/extract_outside_compilation_pass.cc
index 8b01768c49..2a770c527b 100644
--- a/tensorflow/compiler/jit/extract_outside_compilation_pass.cc
+++ b/tensorflow/compiler/jit/extract_outside_compilation_pass.cc
@@ -30,6 +30,7 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor_shape.pb.h"
 #include "tensorflow/core/graph/algorithm.h"
 #include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/gtl/cleanup.h"
 
 namespace tensorflow {
 
@@ -308,6 +309,10 @@ xla::StatusOr<NodeDef> BuildXlaHostComputeNodeDef(
     host_compute_builder.Attr("tpu_core", core);
   }
 
+  // Set input tokens.
+  host_compute_builder.Attr(kXlaTokenInputNodesAttrName,
+                            std::vector<string>{kXlaTokenArgNodeName});
+
   // Populate inputs.
   std::vector<DataType> input_dtypes;
   TF_RETURN_IF_ERROR(GetNodeAttr(call_node->attrs(), "Tinputs", &input_dtypes));
@@ -398,8 +403,8 @@ Status ReplaceOrRemoveOutsideCompilationCallNode(
 }
 
 // Resets "device_ordinal" attr to placeholder value for related nodes
-// (XlaRecvAtHost nodes; XlaSendFromHost nodes; If nodes containing
-// XlaRecvAtHost/XlaSendFromHost).
+// (XlaRecvAtHost nodes; XlaSendFromHost nodes; If/While/FuncCall nodes
+// containing XlaRecvAtHost/XlaSendFromHost).
 Status ResetDeviceOrdinalToPlaceholderValue(Graph* g) {
   AttrValue device_ordinal_value;
   device_ordinal_value.set_placeholder("device_ordinal");
@@ -429,6 +434,10 @@ Status ResetDeviceOrdinalToPlaceholderValue(Graph* g) {
         n->ClearAttr(attr_name);
         n->AddAttr(attr_name, branch_func);
       }
+    } else if (HasNodeAttr(n->def(), "device_ordinal")) {
+      // Function call node containing outside compilation.
+      n->ClearAttr("device_ordinal");
+      n->AddAttr("device_ordinal", device_ordinal_value);
     } else {
       return errors::Internal("Unknown node marked with ",
                               kXlaHasHostTransferAttrName, ": ",
@@ -1217,20 +1226,129 @@ Status BuildHostGraphForWhileNode(
   return Status::OK();
 }
 
+// Builds host graph for func call nodes.
+Status BuildHostGraphForFuncCallNode(const string& func_call_node_name,
+                                     const string& xla_cluster_name,
+                                     const string& func_call_host_func_name,
+                                     const string& host_graph_func_name,
+                                     FunctionLibraryDefinition* fld) {
+  Graph host_graph(fld);
+  AttrValue device_ordinal_value;
+  device_ordinal_value.set_placeholder("device_ordinal");
+
+  // Step 1: add key placeholder node.
+  TF_ASSIGN_OR_RETURN(
+      Node * key_placeholder,
+      AddHostComputeKeyPlaceholder(xla_cluster_name, &host_graph));
+
+  // Step 2: rewrite `host_func_name`, replace key placeholder with an _Arg
+  // node.
+  TF_RETURN_IF_ERROR(ReplaceKeyPlaceholderWithArgNode(
+      xla_cluster_name, func_call_host_func_name, fld));
+
+  // Step 3: build a function call node with `host_func_name`, with
+  // `key_placeholder` as input.
+  NodeDefBuilder call_builder(absl::StrCat("oc_call_", func_call_node_name),
+                              func_call_host_func_name, fld);
+  call_builder.Input(key_placeholder->name(), 0, DT_STRING);
+  call_builder.Attr("device_ordinal", device_ordinal_value);
+  call_builder.Attr(kXlaHasHostTransferAttrName, true);
+  NodeDef call_def;
+  TF_RETURN_IF_ERROR(call_builder.Finalize(&call_def));
+  Status s;
+  Node* call_node = host_graph.AddNode(call_def, &s);
+  TF_RETURN_IF_ERROR(s);
+  host_graph.AddEdge(key_placeholder, 0, call_node, 0);
+
+  // Convert `host_graph` to function, and add a "device_ordinal" attr.
+  FunctionDef oc_host_graph_fdef;
+  TF_RETURN_IF_ERROR(GraphToFunctionDef(host_graph, host_graph_func_name,
+                                        &oc_host_graph_fdef));
+  if (fld->Find(host_graph_func_name)) {
+    TF_RETURN_IF_ERROR(
+        fld->ReplaceFunction(host_graph_func_name, oc_host_graph_fdef));
+  } else {
+    TF_RETURN_IF_ERROR(fld->AddFunctionDef(oc_host_graph_fdef));
+  }
+
+  return Status::OK();
+}
+
 Status ExtractOutsideCompilationForNodesWithAssociatedFunctions(
     Graph* g, const string& xla_cluster_attr_name,
     const string& outside_compilation_attr_name, const string& xla_cluster_name,
-    const std::map<string, int>& host_compute_core,
+    const std::map<string, int>& host_compute_core, FunctionLibraryRuntime* flr,
     FunctionLibraryDefinition* fld, std::vector<string>* host_graphs,
     std::vector<string>* shape_inference_graphs,
     bool* has_outside_compilation) {
-  std::vector<Node*> if_nodes, while_nodes;
+  std::vector<Node*> if_nodes, while_nodes, func_call_nodes;
   for (Node* n : g->nodes()) {
     if (n->type_string() == "If") {
       if_nodes.push_back(n);
     } else if (n->type_string() == "While") {
       while_nodes.push_back(n);
+    } else if (fld->Contains(n->type_string())) {
+      func_call_nodes.push_back(n);
+    } else if (n->type_string() == FunctionLibraryDefinition::kGradientOp) {
+      // Only gradient for user-defined function should be considered as
+      // function call node.
+      NameAttrList original_func;
+      TF_RETURN_IF_ERROR(GetNodeAttr(
+          n->def(), FunctionLibraryDefinition::kFuncAttr, &original_func));
+      if (fld->Contains(original_func.name())) {
+        func_call_nodes.push_back(n);
+      }
+    }
+  }
+
+  for (Node* n : func_call_nodes) {
+    // Extract outside compilation for the function call.
+    bool func_has_outside_compilation = false;
+    NameAttrList func;
+    func.set_name(n->type_string());
+    typedef protobuf::Map<string, AttrValue> AttrMap;
+    *func.mutable_attr() = AttrMap(n->attrs().begin(), n->attrs().end());
+    string new_func_name = absl::StrCat(n->name(), "_oc");
+    string host_func_name = absl::StrCat("oc_func_call_host_", n->name());
+    TF_RETURN_IF_ERROR(ExtractOutsideCompilationForFunction(
+        xla_cluster_attr_name, outside_compilation_attr_name, xla_cluster_name,
+        func, new_func_name, host_func_name, host_compute_core, flr, fld,
+        shape_inference_graphs, &func_has_outside_compilation));
+
+    // If the function call does not have outside compilation, nothing to do.
+    if (!func_has_outside_compilation) {
+      continue;
     }
+
+    *has_outside_compilation = true;
+
+    // Change `n` to call the new function directly.
+    NodeDefBuilder replace_builder(n->name(), new_func_name, fld);
+    for (const Edge* e : n->in_edges()) {
+      if (e->IsControlEdge()) {
+        continue;
+      }
+      replace_builder.Input(e->src()->name(), e->src_output(),
+                            e->src()->output_type(e->src_output()));
+    }
+    for (const auto& attr : n->attrs()) {
+      replace_builder.Attr(attr.first, attr.second);
+    }
+    NodeDef replace_def;
+    TF_RETURN_IF_ERROR(replace_builder.Finalize(&replace_def));
+    TF_ASSIGN_OR_RETURN(Node * replace, ReplaceNode(g, n, replace_def));
+    replace->AddAttr(kXlaTokenInputNodesAttrName,
+                     std::vector<string>{kXlaTokenArgNodeName});
+
+    // Build host side graph for the function call.
+    string oc_host_graph_name =
+        absl::StrCat("oc_func_host_graph_", replace->name());
+    TF_RETURN_IF_ERROR(
+        BuildHostGraphForFuncCallNode(replace->name(), xla_cluster_name,
+                                      host_func_name, oc_host_graph_name, fld));
+
+    // Record the host graph.
+    host_graphs->push_back(oc_host_graph_name);
   }
 
   for (Node* n : if_nodes) {
@@ -1251,12 +1369,12 @@ Status ExtractOutsideCompilationForNodesWithAssociatedFunctions(
     TF_RETURN_IF_ERROR(ExtractOutsideCompilationForFunction(
         xla_cluster_attr_name, outside_compilation_attr_name, xla_cluster_name,
         then_branch, then_branch_xla_func_name, then_branch_host_func_name,
-        host_compute_core, fld, shape_inference_graphs,
+        host_compute_core, flr, fld, shape_inference_graphs,
         &then_branch_has_outside_compilation));
     TF_RETURN_IF_ERROR(ExtractOutsideCompilationForFunction(
         xla_cluster_attr_name, outside_compilation_attr_name, xla_cluster_name,
         else_branch, else_branch_xla_func_name, else_branch_host_func_name,
-        host_compute_core, fld, shape_inference_graphs,
+        host_compute_core, flr, fld, shape_inference_graphs,
         &else_branch_has_outside_compilation));
 
     // If then/else branch do not have outside compilation, nothing to do.
@@ -1316,12 +1434,12 @@ Status ExtractOutsideCompilationForNodesWithAssociatedFunctions(
            body_xla_func_name = absl::StrCat(body.name(), "_oc");
     TF_RETURN_IF_ERROR(ExtractOutsideCompilationForFunction(
         xla_cluster_attr_name, outside_compilation_attr_name, xla_cluster_name,
-        cond, cond_xla_func_name, cond_host_func_name, host_compute_core, fld,
-        shape_inference_graphs, &cond_has_outside_compilation));
+        cond, cond_xla_func_name, cond_host_func_name, host_compute_core, flr,
+        fld, shape_inference_graphs, &cond_has_outside_compilation));
     TF_RETURN_IF_ERROR(ExtractOutsideCompilationForFunction(
         xla_cluster_attr_name, outside_compilation_attr_name, xla_cluster_name,
-        body, body_xla_func_name, body_host_func_name, host_compute_core, fld,
-        shape_inference_graphs, &body_has_outside_compilation));
+        body, body_xla_func_name, body_host_func_name, host_compute_core, flr,
+        fld, shape_inference_graphs, &body_has_outside_compilation));
 
     // If cond/body do not have outside compilation, nothing to do.
     if (!cond_has_outside_compilation && !body_has_outside_compilation) {
@@ -1469,17 +1587,27 @@ Status ExtractOutsideCompilationForFunction(
     const string& outside_compilation_attr_name, const string& xla_cluster_name,
     const NameAttrList& func_name_attrs, const string& new_func_name,
     const string& host_graph_func_name,
-    const std::map<string, int>& host_compute_core,
+    const std::map<string, int>& host_compute_core, FunctionLibraryRuntime* flr,
     FunctionLibraryDefinition* fld, std::vector<string>* shape_inference_graphs,
     bool* has_outside_compilation) {
+  // Convert the function to graph.
   const string& func_name = func_name_attrs.name();
-  const FunctionDef* fdef = fld->Find(func_name);
-  if (!fdef) {
-    return errors::Internal("Cannot find function ", func_name);
-  }
+  FunctionLibraryRuntime::Handle handle;
+  TF_RETURN_IF_ERROR(
+      flr->Instantiate(func_name, AttrSlice(&func_name_attrs.attr()), &handle));
+  Status ret_status = Status::OK();
+  auto cleanup_handle = gtl::MakeCleanup([&]() {
+    auto s = flr->ReleaseHandle(handle);
+    if (!s.ok()) {
+      ret_status.Update(s);
+    }
+  });
+  const FunctionBody* fbody = flr->GetFunctionBody(handle);
+
+  // Check if we have outside compilation nodes.
   *has_outside_compilation = false;
-  for (auto& node_def : fdef->node_def()) {
-    if (HasNodeAttr(node_def, outside_compilation_attr_name)) {
+  for (Node* n : fbody->graph->nodes()) {
+    if (HasNodeAttr(n->def(), outside_compilation_attr_name)) {
       *has_outside_compilation = true;
       break;
     }
@@ -1487,16 +1615,6 @@ Status ExtractOutsideCompilationForFunction(
   // We cannot early return here, because we might have outside compilation in
   // If/While function body.
 
-  // Convert the function to graph.
-  FunctionBody* fbody = nullptr;
-  TF_RETURN_IF_ERROR(FunctionDefToBodyHelper(
-      *fld->Find(func_name), AttrSlice(&func_name_attrs.attr()), fld,
-      [&](const string& op, const OpDef** sig) {
-        return fld->LookUpOpDef(op, sig);
-      },
-      &fbody));
-  std::unique_ptr<FunctionBody> fbody_deleter(fbody);
-
   // Preprocess edges between different outside compilations. They will be
   // restored in `ConstructHostGraph()`.
   TF_RETURN_IF_ERROR(PreprocessEdgesBetweenOutsideCompilations(
@@ -1553,16 +1671,11 @@ Status ExtractOutsideCompilationForFunction(
     TF_RETURN_IF_ERROR(ReplaceOrRemoveOutsideCompilationCallNode(
         graph_out.get(), n, host_compute_core));
   }
-  if (VLOG_IS_ON(4)) {
-    dump_graph::DumpGraphToFile(
-        absl::StrCat("extract_outside_compilation_for_func_after_", func_name),
-        *graph_out, fld);
-  }
 
   // Handle nodes with associated functions.
   TF_RETURN_IF_ERROR(ExtractOutsideCompilationForNodesWithAssociatedFunctions(
       graph_out.get(), xla_cluster_attr_name, outside_compilation_attr_name,
-      xla_cluster_name, host_compute_core, fld,
+      xla_cluster_name, host_compute_core, flr, fld,
       &outside_compilation_host_graphs, shape_inference_graphs,
       has_outside_compilation));
 
@@ -1580,20 +1693,31 @@ Status ExtractOutsideCompilationForFunction(
   FunctionDef updated_fdef;
   TF_RETURN_IF_ERROR(
       GraphToFunctionDef(*graph_out, new_func_name, &updated_fdef));
+  const FunctionDef* original_fdef = fld->Find(func_name);
+  if (original_fdef) {
+    for (const auto& attr : original_fdef->attr()) {
+      (*updated_fdef.mutable_attr())[attr.first] = attr.second;
+    }
+  }
   if (fld->Find(new_func_name)) {
     TF_RETURN_IF_ERROR(fld->ReplaceFunction(new_func_name, updated_fdef));
   } else {
     TF_RETURN_IF_ERROR(fld->AddFunctionDef(updated_fdef));
   }
+  if (VLOG_IS_ON(4)) {
+    dump_graph::DumpGraphToFile(
+        absl::StrCat("extract_outside_compilation_for_func_after_", func_name),
+        *graph_out, fld);
+  }
 
-  return Status::OK();
+  return ret_status;
 }
 
 Status ExtractOutsideCompilation(
     const string& xla_cluster_attr_name,
     const string& outside_compilation_attr_name,
     const std::unordered_map<string, XlaClusterInfo>& clusters, Graph* g,
-    FunctionLibraryDefinition* fld) {
+    FunctionLibraryRuntime* flr, FunctionLibraryDefinition* fld) {
   if (VLOG_IS_ON(4)) {
     dump_graph::DumpGraphToFile("extract_outside_compilation_before", *g, fld);
   }
@@ -1610,7 +1734,7 @@ Status ExtractOutsideCompilation(
     TF_RETURN_IF_ERROR(ExtractOutsideCompilationForFunction(
         xla_cluster_attr_name, outside_compilation_attr_name, xla_cluster_name,
         func_name_attrs, func_name_attrs.name(), host_graph_func_name,
-        host_compute_core, fld, &shape_inference_graphs,
+        host_compute_core, flr, fld, &shape_inference_graphs,
         &has_outside_compilation));
     TF_RETURN_IF_ERROR(
         ExpandHostGraphIntoMainGraph(g, fld, host_graph_func_name, n));
diff --git a/tensorflow/compiler/jit/extract_outside_compilation_pass.h b/tensorflow/compiler/jit/extract_outside_compilation_pass.h
index e07e7c5dd0..d64cc2a103 100644
--- a/tensorflow/compiler/jit/extract_outside_compilation_pass.h
+++ b/tensorflow/compiler/jit/extract_outside_compilation_pass.h
@@ -89,7 +89,7 @@ Status ExtractOutsideCompilationForFunction(
     const string& outside_compilation_attr_name, const string& xla_cluster_name,
     const NameAttrList& func_name_attrs, const string& new_func_name,
     const string& host_graph_func_name,
-    const std::map<string, int>& host_compute_core,
+    const std::map<string, int>& host_compute_core, FunctionLibraryRuntime* flr,
     FunctionLibraryDefinition* fld, std::vector<string>* shape_inference_graphs,
     bool* has_outside_compilation);
 
@@ -101,7 +101,7 @@ Status ExtractOutsideCompilation(
     const string& xla_cluster_attr_name,
     const string& outside_compilation_attr_name,
     const std::unordered_map<string, XlaClusterInfo>& clusters, Graph* g,
-    FunctionLibraryDefinition* fld);
+    FunctionLibraryRuntime* flr, FunctionLibraryDefinition* fld);
 
 }  // namespace tensorflow
 
diff --git a/tensorflow/compiler/jit/extract_outside_compilation_pass_test.cc b/tensorflow/compiler/jit/extract_outside_compilation_pass_test.cc
index e9a89e34e0..7c3a24feff 100644
--- a/tensorflow/compiler/jit/extract_outside_compilation_pass_test.cc
+++ b/tensorflow/compiler/jit/extract_outside_compilation_pass_test.cc
@@ -23,6 +23,7 @@ limitations under the License.
 #include "tensorflow/cc/ops/standard_ops.h"
 #include "tensorflow/compiler/jit/encapsulate_util.h"
 #include "tensorflow/compiler/xla/test.h"
+#include "tensorflow/core/common_runtime/device_factory.h"
 #include "tensorflow/core/common_runtime/function.h"
 #include "tensorflow/core/framework/common_shape_fns.h"
 #include "tensorflow/core/framework/function.h"
@@ -31,6 +32,8 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_shape.pb.h"
 #include "tensorflow/core/platform/test.h"
+#include "tensorflow/core/public/session_options.h"
+#include "tensorflow/core/public/version.h"
 
 namespace tensorflow {
 
@@ -222,7 +225,42 @@ TEST(RewriteOutsideCompilationSubgraphFnTest, ShapesInferred) {
   EXPECT_EQ(shapes[0].dim_size(), 1);
 }
 
-TEST(ExtractOutsideCompilationForFunctionTest, Basic) {
+class ExtractOutsideCompilationForFunctionTest : public ::testing::Test {
+ public:
+  void SetUp() override {
+    SessionOptions session_options;
+    std::vector<std::unique_ptr<Device>> devices;
+    TF_CHECK_OK(DeviceFactory::AddDevices(
+        session_options, "/job:localhost/replica:0/task:0", &devices));
+    device_mgr_ = absl::make_unique<DeviceMgr>(std::move(devices));
+  }
+
+  Status ExtractOutsideCompilationTest(
+      const string &xla_cluster_attr_name,
+      const string &outside_compilation_attr_name,
+      const string &xla_cluster_name, const NameAttrList &func_name_attrs,
+      const string &new_func_name, const string &host_graph_func_name,
+      const std::map<string, int> &host_compute_core,
+      FunctionLibraryDefinition *fld,
+      std::vector<string> *shape_inference_graphs,
+      bool *has_outside_compilation) {
+    OptimizerOptions opts;
+    pflr_ = absl::make_unique<ProcessFunctionLibraryRuntime>(
+        device_mgr_.get(), Env::Default(), TF_GRAPH_DEF_VERSION, fld, opts,
+        /*default_thread_pool=*/nullptr, /*cluster_flr=*/nullptr);
+    auto flr = pflr_->GetFLR("/job:localhost/replica:0/task:0/cpu:0");
+    return ExtractOutsideCompilationForFunction(
+        xla_cluster_attr_name, outside_compilation_attr_name, xla_cluster_name,
+        func_name_attrs, new_func_name, host_graph_func_name, host_compute_core,
+        flr, fld, shape_inference_graphs, has_outside_compilation);
+  }
+
+ private:
+  std::unique_ptr<DeviceMgr> device_mgr_;
+  std::unique_ptr<ProcessFunctionLibraryRuntime> pflr_;
+};
+
+TEST_F(ExtractOutsideCompilationForFunctionTest, Basic) {
   // Build the XLA computation func.
   // "const0"
   // "identity0" = "const0" (outside compilation cluster "0")
@@ -256,7 +294,7 @@ TEST(ExtractOutsideCompilationForFunctionTest, Basic) {
   NameAttrList name_attrs;
   name_attrs.set_name("cluster");
   *name_attrs.mutable_attr() = attrs;
-  TF_CHECK_OK(ExtractOutsideCompilationForFunction(
+  TF_CHECK_OK(ExtractOutsideCompilationTest(
       "_xla", "_oc", "cluster", name_attrs, "cluster_rewritten", "host_graph",
       host_compute_core, &fld, &shape_inference_graphs,
       &has_outside_compilation));
@@ -362,7 +400,7 @@ TEST(ExtractOutsideCompilationForFunctionTest, Basic) {
   }
 }
 
-TEST(ExtractOutsideCompilationForFunctionTest, NoHostGraph) {
+TEST_F(ExtractOutsideCompilationForFunctionTest, NoHostGraph) {
   // Build the XLA computation func.
   // "const0"
   FunctionDefLibrary fdl;
@@ -384,7 +422,7 @@ TEST(ExtractOutsideCompilationForFunctionTest, NoHostGraph) {
   NameAttrList name_attrs;
   name_attrs.set_name("cluster");
   *name_attrs.mutable_attr() = attrs;
-  TF_CHECK_OK(ExtractOutsideCompilationForFunction(
+  TF_CHECK_OK(ExtractOutsideCompilationTest(
       "_xla", "_oc", "cluster", name_attrs, "cluster_rewritten", "host_graph",
       host_compute_core, &fld, &shape_inference_graphs,
       &has_outside_compilation));
@@ -406,7 +444,7 @@ TEST(ExtractOutsideCompilationForFunctionTest, NoHostGraph) {
   EXPECT_EQ(host_graph->num_nodes(), 2);
 }
 
-TEST(ExtractOutsideCompilationForFunctionTest, XlaHostComputeRemoved) {
+TEST_F(ExtractOutsideCompilationForFunctionTest, XlaHostComputeRemoved) {
   // Build the XLA computation func.
   // "const0"
   // "const1" (outside compilation cluster "0")
@@ -432,7 +470,7 @@ TEST(ExtractOutsideCompilationForFunctionTest, XlaHostComputeRemoved) {
   NameAttrList name_attrs;
   name_attrs.set_name("cluster");
   *name_attrs.mutable_attr() = attrs;
-  TF_CHECK_OK(ExtractOutsideCompilationForFunction(
+  TF_CHECK_OK(ExtractOutsideCompilationTest(
       "_xla", "_oc", "cluster", name_attrs, "cluster_rewritten", "host_graph",
       host_compute_core, &fld, &shape_inference_graphs,
       &has_outside_compilation));
@@ -489,7 +527,7 @@ REGISTER_OP("XlaRecvFromHost")
     .Attr("key: string")
     .SetIsStateful();
 
-TEST(ExtractOutsideCompilationForFunctionTest, OutsideCompilationInIf) {
+TEST_F(ExtractOutsideCompilationForFunctionTest, OutsideCompilationInIf) {
   // Build the XLA computation func.
   // "const0" (bool)
   // "const1" (int32)
@@ -555,7 +593,7 @@ TEST(ExtractOutsideCompilationForFunctionTest, OutsideCompilationInIf) {
   NameAttrList name_attrs;
   name_attrs.set_name("cluster");
   *name_attrs.mutable_attr() = attrs;
-  TF_CHECK_OK(ExtractOutsideCompilationForFunction(
+  TF_CHECK_OK(ExtractOutsideCompilationTest(
       "_xla", "_oc", "cluster", name_attrs, "cluster_rewritten", "host_graph",
       host_compute_core, &fld, &shape_inference_graphs,
       &has_outside_compilation));
@@ -651,7 +689,7 @@ TEST(ExtractOutsideCompilationForFunctionTest, OutsideCompilationInIf) {
   }
 }
 
-TEST(ExtractOutsideCompilationForFunctionTest, OutsideCompilationInWhile) {
+TEST_F(ExtractOutsideCompilationForFunctionTest, OutsideCompilationInWhile) {
   // Build the XLA computation func.
   // "const0" (bool)
   // "while0" (input = "const0", cond = "cond_fn", body = "body_fn")
@@ -714,7 +752,7 @@ TEST(ExtractOutsideCompilationForFunctionTest, OutsideCompilationInWhile) {
   NameAttrList name_attrs;
   name_attrs.set_name("cluster");
   *name_attrs.mutable_attr() = attrs;
-  TF_CHECK_OK(ExtractOutsideCompilationForFunction(
+  TF_CHECK_OK(ExtractOutsideCompilationTest(
       "_xla", "_oc", "cluster", name_attrs, "cluster_rewritten", "host_graph",
       host_compute_core, &fld, &shape_inference_graphs,
       &has_outside_compilation));
@@ -782,4 +820,162 @@ TEST(ExtractOutsideCompilationForFunctionTest, OutsideCompilationInWhile) {
   }
 }
 
+TEST_F(ExtractOutsideCompilationForFunctionTest, OutsideCompilationInFunction) {
+  // Build the XLA computation func.
+  // "const0" (int32)
+  // "fn" (input = "const0")
+  FunctionDefLibrary fdl;
+  {
+    tensorflow::Scope s = tensorflow::Scope::NewRootScope();
+    Output arg = ops::_Arg(s.WithOpName("arg"), DT_INT32, 0);
+    Output identity = ops::Identity(s.WithOpName("identity"), arg);
+    ops::_Retval retval(s.WithOpName("retval"), identity, 0);
+    std::unique_ptr<Graph> g(new Graph(OpRegistry::Global()));
+    TF_CHECK_OK(s.ToGraph(g.get()));
+    auto node_name_image = g->BuildNodeNameIndex();
+    node_name_image["identity"]->AddAttr("_oc", "0");
+    PartialTensorShape shape({2});
+    node_name_image["identity"]->AddAttr(
+        kXlaInferredShapesAttrName, std::vector<PartialTensorShape>{shape});
+
+    FunctionDef *true_fn_fdef = fdl.add_function();
+    TF_CHECK_OK(GraphToFunctionDef(*g, "fn", true_fn_fdef));
+  }
+  FunctionLibraryDefinition fld(OpRegistry::Global(), fdl);
+  {
+    std::unique_ptr<Graph> g(new Graph(&fld));
+
+    tensorflow::TensorProto tensor_proto;
+    tensor_proto.set_dtype(tensorflow::DT_INT32);
+    tensorflow::TensorShapeProto shape;
+    shape.add_dim()->set_size(2);
+    *tensor_proto.mutable_tensor_shape() = shape;
+    for (int i = 0; i < 2; ++i) {
+      tensor_proto.add_int_val(1);
+    }
+    NodeDef const_def;
+    TF_CHECK_OK(NodeDefBuilder("const", "Const")
+                    .Attr("dtype", DT_INT32)
+                    .Attr("value", tensor_proto)
+                    .Finalize(&const_def));
+    Status s;
+    Node *const_node = g->AddNode(const_def, &s);
+    TF_CHECK_OK(s);
+
+    NodeDef fn_def;
+    TF_CHECK_OK(NodeDefBuilder("fn", "fn", &fld)
+                    .Input("const", 0, DT_INT32)
+                    .Finalize(&fn_def));
+    Node *fn_node = g->AddNode(fn_def, &s);
+    TF_CHECK_OK(s);
+    g->AddEdge(const_node, 0, fn_node, 0);
+
+    NodeDef ret_def;
+    TF_CHECK_OK(NodeDefBuilder("ret", "_Retval")
+                    .Attr("index", 0)
+                    .Attr("T", DT_INT32)
+                    .Input("fn", 0, DT_INT32)
+                    .Finalize(&ret_def));
+    Node *ret_node = g->AddNode(ret_def, &s);
+    TF_CHECK_OK(s);
+    g->AddEdge(fn_node, 0, ret_node, 0);
+
+    FunctionDef *xla_fdef = fdl.add_function();
+    TF_CHECK_OK(GraphToFunctionDef(*g, "cluster", xla_fdef));
+    TF_CHECK_OK(fld.AddFunctionDef(*xla_fdef));
+  }
+
+  protobuf::Map<string, tensorflow::AttrValue> attrs;
+  std::map<string, int> host_compute_core;
+  std::vector<string> shape_inference_graphs;
+  bool has_outside_compilation;
+  NameAttrList name_attrs;
+  name_attrs.set_name("cluster");
+  *name_attrs.mutable_attr() = attrs;
+  TF_CHECK_OK(ExtractOutsideCompilationTest(
+      "_xla", "_oc", "cluster", name_attrs, "cluster_rewritten", "host_graph",
+      host_compute_core, &fld, &shape_inference_graphs,
+      &has_outside_compilation));
+
+  // Check host graph.
+  {
+    FunctionBody *host_fbody = nullptr;
+    AttrValue device_ordinal_temp_value;
+    device_ordinal_temp_value.set_i(0);
+    protobuf::Map<string, AttrValue> host_func_attrs;
+    host_func_attrs["device_ordinal"] = device_ordinal_temp_value;
+    TF_CHECK_OK(FunctionDefToBodyHelper(
+        *fld.Find("host_graph"), AttrSlice(&host_func_attrs), &fld,
+        [&](const string &op, const OpDef **sig) {
+          return fld.LookUpOpDef(op, sig);
+        },
+        &host_fbody));
+    std::unique_ptr<FunctionBody> host_fbody_deleter(host_fbody);
+    Graph *host_graph = host_fbody->graph;
+    auto node_name_index = host_graph->BuildNodeNameIndex();
+
+    // Verify we have call node for outside compilation in `fn`.
+    Node *call_node = node_name_index["oc_call_fn"];
+    EXPECT_NE(call_node, nullptr);
+
+    FunctionBody *call_fbody = nullptr;
+    TF_CHECK_OK(FunctionDefToBodyHelper(
+        *fld.Find("oc_func_call_host_fn"), AttrSlice(&host_func_attrs), &fld,
+        [&](const string &op, const OpDef **sig) {
+          return fld.LookUpOpDef(op, sig);
+        },
+        &call_fbody));
+    std::unique_ptr<FunctionBody> call_fbody_deleter(call_fbody);
+
+    // Verify we have _XlaRecvAtHost and _XlaSendFromHost nodes.
+    bool has_recv = false, has_send = false;
+    for (Node *n : call_fbody->graph->nodes()) {
+      if (n->type_string() == "_XlaRecvAtHost") {
+        has_recv = true;
+      } else if (n->type_string() == "_XlaSendFromHost") {
+        has_send = true;
+      }
+    }
+    EXPECT_TRUE(has_recv);
+    EXPECT_TRUE(has_send);
+  }
+
+  // Check XLA graph.
+  {
+    FunctionBody *xla_fbody = nullptr;
+    TF_CHECK_OK(FunctionDefToBodyHelper(
+        *fld.Find("cluster_rewritten"), AttrSlice(), &fld,
+        [&](const string &op, const OpDef **sig) {
+          return fld.LookUpOpDef(op, sig);
+        },
+        &xla_fbody));
+    std::unique_ptr<FunctionBody> xla_fbody_deleter(xla_fbody);
+    Graph *xla_graph = xla_fbody->graph;
+    auto node_name_index = xla_graph->BuildNodeNameIndex();
+
+    // Check that we have call node.
+    Node *fn_node = node_name_index["fn"];
+    EXPECT_NE(fn_node, nullptr);
+    EXPECT_EQ(fn_node->type_string(), "fn_oc");
+
+    FunctionBody *call_fbody = nullptr;
+    TF_CHECK_OK(FunctionDefToBodyHelper(
+        *fld.Find("fn_oc"), AttrSlice(), &fld,
+        [&](const string &op, const OpDef **sig) {
+          return fld.LookUpOpDef(op, sig);
+        },
+        &call_fbody));
+    std::unique_ptr<FunctionBody> call_fbody_deleter(call_fbody);
+
+    // Verify we have XlaHostCompute nodes.
+    bool has_hc = false;
+    for (Node *n : call_fbody->graph->nodes()) {
+      if (n->type_string() == "XlaHostCompute") {
+        has_hc = true;
+      }
+    }
+    EXPECT_TRUE(has_hc);
+  }
+}
+
 }  // namespace tensorflow
diff --git a/tensorflow/compiler/tf2xla/graph_compiler.cc b/tensorflow/compiler/tf2xla/graph_compiler.cc
index efb7574972..0c2bb02239 100644
--- a/tensorflow/compiler/tf2xla/graph_compiler.cc
+++ b/tensorflow/compiler/tf2xla/graph_compiler.cc
@@ -22,6 +22,7 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/dump_graph.h"
 #include "tensorflow/compiler/tf2xla/literal_util.h"
 #include "tensorflow/compiler/tf2xla/shape_util.h"
+#include "tensorflow/compiler/tf2xla/side_effect_util.h"
 #include "tensorflow/compiler/tf2xla/type_util.h"
 #include "tensorflow/compiler/tf2xla/xla_compiler.h"
 #include "tensorflow/compiler/tf2xla/xla_context.h"
@@ -191,6 +192,9 @@ Status GraphCompiler::CompileFunctionalNode(Node* n,
   // into the functions.
   XlaOpKernelContext xla_op_context(op_context);
 
+  XlaContext& context = XlaContext::Get(op_context);
+  auto* b = context.builder();
+
   XlaCompiler* compiler = xla_op_context.compiler();
 
   NameAttrList func;
@@ -219,8 +223,12 @@ Status GraphCompiler::CompileFunctionalNode(Node* n,
   TF_RETURN_IF_ERROR(
       PrepareArguments(&xla_op_context, graph.get(), expressions, &arguments));
 
+  bool add_token_input_output =
+      HasNodeAttr(n->def(), kXlaTokenInputNodesAttrName);
+
   XlaCompiler::CompileOptions compile_options;
   compile_options.is_entry_computation = false;
+  compile_options.add_token_input_output = add_token_input_output;
   XlaCompiler::CompilationResult result;
   TF_RETURN_IF_ERROR(
       compiler->CompileFunction(compile_options, func, arguments, &result));
@@ -234,9 +242,19 @@ Status GraphCompiler::CompileFunctionalNode(Node* n,
     }
     handles.push_back(expressions[i]->handle());
   }
-
-  XlaContext& context = XlaContext::Get(op_context);
-  auto* b = context.builder();
+  if (add_token_input_output) {
+    std::vector<string> token_input_nodes;
+    TF_RETURN_IF_ERROR(
+        GetNodeAttr(n->def(), kXlaTokenInputNodesAttrName, &token_input_nodes));
+    std::vector<xla::XlaOp> token_inputs;
+    for (const string& node_name : token_input_nodes) {
+      auto token_or = compiler->GetNodeToken(node_name);
+      TF_RETURN_IF_ERROR(token_or.status());
+      token_inputs.push_back(token_or.ConsumeValueOrDie());
+    }
+    xla::XlaOp token_input = xla::AfterAll(b, token_inputs);
+    handles.push_back(token_input);
+  }
 
   auto output_handle = xla::Call(b, *result.computation, handles);
   // The output handle of `Call` computation is a tuple type. Unzip it so
@@ -251,6 +269,10 @@ Status GraphCompiler::CompileFunctionalNode(Node* n,
       ++computation_output;
     }
   }
+  if (add_token_input_output) {
+    TF_RETURN_IF_ERROR(compiler->SetNodeToken(
+        n->name(), xla::GetTupleElement(output_handle, computation_output)));
+  }
   return b->first_error();
 }
 
diff --git a/tensorflow/compiler/tf2xla/side_effect_util.cc b/tensorflow/compiler/tf2xla/side_effect_util.cc
index a9144cfc4f..412f31adbb 100644
--- a/tensorflow/compiler/tf2xla/side_effect_util.cc
+++ b/tensorflow/compiler/tf2xla/side_effect_util.cc
@@ -58,6 +58,10 @@ Status SetDeviceOrdinalAttributeForNode(Node* node, int device_ordinal) {
       node->ClearAttr(attr_name);
       node->AddAttr(attr_name, branch_func);
     }
+  } else if (HasNodeAttr(node->def(), "device_ordinal")) {
+    // Function call node containing outside compilation.
+    node->ClearAttr("device_ordinal");
+    node->AddAttr("device_ordinal", device_ordinal);
   } else {
     return errors::Internal("Unknown node type to set 'device_ordinal': ",
                             node->DebugString());
-- 
GitLab


From 1b2e900f3cadc1a61d0c0a65c239e53afba7a7ec Mon Sep 17 00:00:00 2001
From: Michael Kuperstein <mkuper@google.com>
Date: Tue, 8 Jan 2019 10:19:40 -0800
Subject: [PATCH 0384/1095] [XLA] Start moving over the DynamicSlice /
 DynamicUpdateSlice API

* Updated operation semantics document for new syntax.
* Added XlaBuilder APIs.
* Marked some of the old APIs as deprecated.
* Moved over a couple of slice tests to the new API.

Other uses in XLA tests and in TF2XLA will be moved over incrementally.

PiperOrigin-RevId: 228352804
---
 tensorflow/compiler/xla/client/xla_builder.cc | 62 +++++++++++++++++++
 tensorflow/compiler/xla/client/xla_builder.h  | 30 +++++++--
 .../compiler/xla/g3doc/operation_semantics.md | 59 +++++++++---------
 .../compiler/xla/service/hlo_instruction.h    |  2 +
 4 files changed, 121 insertions(+), 32 deletions(-)

diff --git a/tensorflow/compiler/xla/client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_builder.cc
index 433033cae6..328abbc7b9 100644
--- a/tensorflow/compiler/xla/client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_builder.cc
@@ -705,6 +705,34 @@ XlaOp XlaBuilder::DynamicSlice(const XlaOp& operand, const XlaOp& start_indices,
   });
 }
 
+XlaOp XlaBuilder::DynamicSlice(const XlaOp& operand,
+                               absl::Span<const XlaOp> start_indices,
+                               absl::Span<const int64> slice_sizes) {
+  return ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    HloInstructionProto instr;
+
+    TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand));
+    std::vector<const Shape*> start_indices_shape_ptrs;
+    TF_ASSIGN_OR_RETURN(const auto& start_indices_shapes,
+                        GetOperandShapes(start_indices));
+    absl::c_transform(start_indices_shapes,
+                      std::back_inserter(start_indices_shape_ptrs),
+                      [](const Shape& shape) { return &shape; });
+    TF_ASSIGN_OR_RETURN(Shape shape,
+                        ShapeInference::InferDynamicSliceShape(
+                            operand_shape, start_indices_shapes, slice_sizes));
+    *instr.mutable_shape() = shape.ToProto();
+
+    for (int64 size : slice_sizes) {
+      instr.add_dynamic_slice_sizes(size);
+    }
+
+    std::vector<XlaOp> operands = {operand};
+    operands.insert(operands.end(), start_indices.begin(), start_indices.end());
+    return AddInstruction(std::move(instr), HloOpcode::kDynamicSlice, operands);
+  });
+}
+
 XlaOp XlaBuilder::DynamicUpdateSlice(const XlaOp& operand, const XlaOp& update,
                                      const XlaOp& start_indices) {
   return ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
@@ -724,6 +752,31 @@ XlaOp XlaBuilder::DynamicUpdateSlice(const XlaOp& operand, const XlaOp& update,
   });
 }
 
+XlaOp XlaBuilder::DynamicUpdateSlice(const XlaOp& operand, const XlaOp& update,
+                                     absl::Span<const XlaOp> start_indices) {
+  return ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    HloInstructionProto instr;
+
+    TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand));
+    TF_ASSIGN_OR_RETURN(const Shape& update_shape, GetShape(update));
+    std::vector<const Shape*> start_indices_shape_ptrs;
+    TF_ASSIGN_OR_RETURN(const auto& start_indices_shapes,
+                        GetOperandShapes(start_indices));
+    absl::c_transform(start_indices_shapes,
+                      std::back_inserter(start_indices_shape_ptrs),
+                      [](const Shape& shape) { return &shape; });
+    TF_ASSIGN_OR_RETURN(Shape shape,
+                        ShapeInference::InferDynamicUpdateSliceShape(
+                            operand_shape, update_shape, start_indices_shapes));
+    *instr.mutable_shape() = shape.ToProto();
+
+    std::vector<XlaOp> operands = {operand, update};
+    operands.insert(operands.end(), start_indices.begin(), start_indices.end());
+    return AddInstruction(std::move(instr), HloOpcode::kDynamicUpdateSlice,
+                          operands);
+  });
+}
+
 XlaOp XlaBuilder::ConcatInDim(absl::Span<const XlaOp> operands,
                               int64 dimension) {
   return ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
@@ -2751,12 +2804,21 @@ XlaOp DynamicSlice(const XlaOp& operand, const XlaOp& start_indices,
                    absl::Span<const int64> slice_sizes) {
   return operand.builder()->DynamicSlice(operand, start_indices, slice_sizes);
 }
+XlaOp DynamicSlice(const XlaOp& operand, absl::Span<const XlaOp> start_indices,
+                   absl::Span<const int64> slice_sizes) {
+  return operand.builder()->DynamicSlice(operand, start_indices, slice_sizes);
+}
 
 XlaOp DynamicUpdateSlice(const XlaOp& operand, const XlaOp& update,
                          const XlaOp& start_indices) {
   return operand.builder()->DynamicUpdateSlice(operand, update, start_indices);
 }
 
+XlaOp DynamicUpdateSlice(const XlaOp& operand, const XlaOp& update,
+                         absl::Span<const XlaOp> start_indices) {
+  return operand.builder()->DynamicUpdateSlice(operand, update, start_indices);
+}
+
 XlaOp ConcatInDim(XlaBuilder* builder, absl::Span<const XlaOp> operands,
                   int64 dimension) {
   return builder->ConcatInDim(operands, dimension);
diff --git a/tensorflow/compiler/xla/client/xla_builder.h b/tensorflow/compiler/xla/client/xla_builder.h
index ebef8e0879..8908d172fa 100644
--- a/tensorflow/compiler/xla/client/xla_builder.h
+++ b/tensorflow/compiler/xla/client/xla_builder.h
@@ -359,11 +359,18 @@ class XlaBuilder {
   XlaOp SliceInDim(const XlaOp& operand, int64 start_index, int64 limit_index,
                    int64 stride, int64 dimno);
 
+  ABSL_DEPRECATED("Use span-of-indices form instead")
   XlaOp DynamicSlice(const XlaOp& operand, const XlaOp& start_indices,
                      absl::Span<const int64> slice_sizes);
+  XlaOp DynamicSlice(const XlaOp& operand,
+                     absl::Span<const XlaOp> start_indices,
+                     absl::Span<const int64> slice_sizes);
 
+  ABSL_DEPRECATED("Use span-of-indices form instead")
   XlaOp DynamicUpdateSlice(const XlaOp& operand, const XlaOp& update,
                            const XlaOp& start_indices);
+  XlaOp DynamicUpdateSlice(const XlaOp& operand, const XlaOp& update,
+                           absl::Span<const XlaOp> start_indices);
 
   XlaOp ConcatInDim(absl::Span<const XlaOp> operands, int64 dimension);
 
@@ -875,9 +882,14 @@ class XlaBuilder {
 
   friend XlaOp DynamicSlice(const XlaOp& operand, const XlaOp& start_indices,
                             absl::Span<const int64> slice_sizes);
+  friend XlaOp DynamicSlice(const XlaOp& operand,
+                            absl::Span<const XlaOp> start_indices,
+                            absl::Span<const int64> slice_sizes);
 
   friend XlaOp DynamicUpdateSlice(const XlaOp& operand, const XlaOp& update,
                                   const XlaOp& start_indices);
+  friend XlaOp DynamicUpdateSlice(const XlaOp& operand, const XlaOp& update,
+                                  absl::Span<const XlaOp> start_indices);
 
   friend XlaOp ConcatInDim(XlaBuilder* builder,
                            absl::Span<const XlaOp> operands, int64 dimension);
@@ -1319,10 +1331,15 @@ XlaOp SliceInDim(const XlaOp& operand, int64 start_index, int64 limit_index,
 // The size of the slice in each dimension is passed in 'slice_sizes',
 // which specify the end point of exclusive slice intervals in each
 // dimension [start, start + size).
-// The shape of 'start_indices' must be rank == 1, with dimension size
-// equal to the rank of the 'operand'.
+// The shape of each element of 'start_indices' must be scalar, with the span
+// size equal to the rank of the 'operand'. All elements of 'start_indices' must
+// have the same shape.
 // Slice index calculations are computed modulo input dimension sizes to
 // prevent dynamic start indices from generating out-of-bound array accesses.
+XlaOp DynamicSlice(const XlaOp& operand, absl::Span<const XlaOp> start_indices,
+                   absl::Span<const int64> slice_sizes);
+
+ABSL_DEPRECATED("Use span-of-indices form instead")
 XlaOp DynamicSlice(const XlaOp& operand, const XlaOp& start_indices,
                    absl::Span<const int64> slice_sizes);
 
@@ -1338,10 +1355,15 @@ XlaOp DynamicSlice(const XlaOp& operand, const XlaOp& start_indices,
 //   [4 5 6]  => DynamicUpdateslice(data, update, start)   => [4 10 11]
 //   [7 8 9]                                                  [7 8  9 ]
 //
-// The shape of 'start_indices' must be rank == 1, with dimension size
-// equal to the rank of the 'operand'.
+// The shape of each element of 'start_indices' must be scalar, with the span
+// size equal to the rank of the 'operand'. All elements of 'start_indices' must
+// have the same shape.
 // Slice index calculations are computed modulo update dimension sizes to
 // prevent dynamic start indices from generating out-of-bound array accesses.
+XlaOp DynamicUpdateSlice(const XlaOp& operand, const XlaOp& update,
+                         absl::Span<const XlaOp> start_indices);
+
+ABSL_DEPRECATED("Use span-of-indices form instead")
 XlaOp DynamicUpdateSlice(const XlaOp& operand, const XlaOp& update,
                          const XlaOp& start_indices);
 
diff --git a/tensorflow/compiler/xla/g3doc/operation_semantics.md b/tensorflow/compiler/xla/g3doc/operation_semantics.md
index 78d4617017..7377ed729b 100644
--- a/tensorflow/compiler/xla/g3doc/operation_semantics.md
+++ b/tensorflow/compiler/xla/g3doc/operation_semantics.md
@@ -944,21 +944,21 @@ dimension: [start, start + size). The shape of `start_indices` must be rank ==
 
 <b> `DynamicSlice(operand, start_indices, size_indices)` </b>
 
-| Arguments       | Type                | Semantics                           |
-| --------------- | ------------------- | ----------------------------------- |
-| `operand`       | `XlaOp`             | N dimensional array of type T       |
-| `start_indices` | `XlaOp`             | Rank 1 array of N integers          |
-:                 :                     : containing the starting indices of  :
-:                 :                     : the slice for each dimension. Value :
-:                 :                     : must be greater than or equal to    :
-:                 :                     : zero.                               :
-| `size_indices`  | `ArraySlice<int64>` | List of N integers containing the   |
-:                 :                     : slice size for each dimension. Each :
-:                 :                     : value must be strictly greater than :
-:                 :                     : zero, and start + size must be less :
-:                 :                     : than or equal to the size of the    :
-:                 :                     : dimension to avoid wrapping modulo  :
-:                 :                     : dimension size.                     :
+| Arguments       | Type                  | Semantics                          |
+| --------------- | --------------------- | ---------------------------------- |
+| `operand`       | `XlaOp`               | N dimensional array of type T      |
+| `start_indices` | sequence of N `XlaOp` | List of N scalar integers          |
+:                 :                       : containing the starting indices of :
+:                 :                       : the slice for each dimension.      :
+:                 :                       : Value must be greater than or      :
+:                 :                       : equal to zero.                     :
+| `size_indices`  | `ArraySlice<int64>`   | List of N integers containing the  |
+:                 :                       : slice size for each dimension.     :
+:                 :                       : Each value must be strictly        :
+:                 :                       : greater than zero, and start +     :
+:                 :                       : size must be less than or equal to :
+:                 :                       : the size of the dimension to avoid :
+:                 :                       : wrapping modulo dimension size.    :
 
 The effective slice indices are computed by applying the following
 transformation for each index `i` in `[1, N)` before performing the slice:
@@ -1009,19 +1009,22 @@ the rank of `operand`.
 
 <b> `DynamicUpdateSlice(operand, update, start_indices)` </b>
 
-| Arguments       | Type    | Semantics                                        |
-| --------------- | ------- | ------------------------------------------------ |
-| `operand`       | `XlaOp` | N dimensional array of type T                    |
-| `update`        | `XlaOp` | N dimensional array of type T containing the     |
-:                 :         : slice update. Each dimension of update shape     :
-:                 :         : must be strictly greater than zero, and start +  :
-:                 :         : update must be less than or equal to the operand :
-:                 :         : size for each dimension to avoid generating      :
-:                 :         : out-of-bounds update indices.                    :
-| `start_indices` | `XlaOp` | Rank 1 array of N integers containing the        |
-:                 :         : starting indices of the slice for each           :
-:                 :         : dimension. Value must be greater than or equal   :
-:                 :         : to zero.                                         :
+| Arguments       | Type                  | Semantics                          |
+| --------------- | --------------------- | ---------------------------------- |
+| `operand`       | `XlaOp`               | N dimensional array of type T      |
+| `update`        | `XlaOp`               | N dimensional array of type T      |
+:                 :                       : containing the slice update. Each  :
+:                 :                       : dimension of update shape must be  :
+:                 :                       : strictly greater than zero, and    :
+:                 :                       : start + update must be less than   :
+:                 :                       : or equal to the operand size for   :
+:                 :                       : each dimension to avoid generating :
+:                 :                       : out-of-bounds update indices.      :
+| `start_indices` | sequence of N `XlaOp` | List of N scalar integers          |
+:                 :                       : containing the starting indices of :
+:                 :                       : the slice for each dimension.      :
+:                 :                       : Value must be greater than or      :
+:                 :                       : equal to zero.                     :
 
 The effective slice indices are computed by applying the following
 transformation for each index `i` in `[1, N)` before performing the slice:
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.h b/tensorflow/compiler/xla/service/hlo_instruction.h
index 2827eed0df..789ace6a26 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.h
+++ b/tensorflow/compiler/xla/service/hlo_instruction.h
@@ -556,6 +556,7 @@ class HloInstruction {
   // Creates a slice instruction, where the first operand is sliced by
   // start indices specified in the second operand, and by size specified in
   // 'slice_sizes'.
+  ABSL_DEPRECATED("Use span-of-indices form instead")
   static std::unique_ptr<HloInstruction> CreateDynamicSlice(
       const Shape& shape, HloInstruction* operand,
       HloInstruction* start_indices, absl::Span<const int64> slice_sizes);
@@ -567,6 +568,7 @@ class HloInstruction {
 
   // Creates a dynamic update slice instruction, which updates a slice
   // of 'operand' with 'update' and 'start_indices'.
+  ABSL_DEPRECATED("Use span-of-indices form instead")
   static std::unique_ptr<HloInstruction> CreateDynamicUpdateSlice(
       const Shape& shape, HloInstruction* operand, HloInstruction* update,
       HloInstruction* start_indices);
-- 
GitLab


From 3b6f11930a3c6fe91c43fc92aeb303e8cefffc5d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 8 Jan 2019 10:22:53 -0800
Subject: [PATCH 0385/1095] Automated rollback of commit
 ec569e692435fb68191553fd7ffa5b633fdf1ef2

PiperOrigin-RevId: 228353495
---
 tensorflow/core/grappler/optimizers/BUILD     |   4 -
 .../optimizers/dependency_optimizer.cc        | 456 +++++++++++-------
 .../optimizers/dependency_optimizer.h         |  24 +-
 .../optimizers/dependency_optimizer_test.cc   |   9 +-
 4 files changed, 291 insertions(+), 202 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index 5a328a91c5..7e29cee86a 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -303,18 +303,14 @@ cc_library(
         ":constant_folding",
         ":graph_optimizer",
         "//tensorflow/core:framework",
-        "//tensorflow/core:graph",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core/grappler:grappler_item",
-        "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/costs:graph_properties",
         "//tensorflow/core/grappler/utils:topological_sort",
-        "@com_google_absl//absl/container:flat_hash_map",
-        "@com_google_absl//absl/container:flat_hash_set",
     ],
 )
 
diff --git a/tensorflow/core/grappler/optimizers/dependency_optimizer.cc b/tensorflow/core/grappler/optimizers/dependency_optimizer.cc
index 9bceb6b906..7fee3ae9d5 100644
--- a/tensorflow/core/grappler/optimizers/dependency_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/dependency_optimizer.cc
@@ -18,14 +18,10 @@ limitations under the License.
 #include <unordered_map>
 #include <unordered_set>
 
-#include "absl/container/flat_hash_map.h"
-#include "absl/container/flat_hash_set.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/framework/op.h"
-#include "tensorflow/core/graph/tensor_id.h"
 #include "tensorflow/core/grappler/costs/graph_properties.h"
 #include "tensorflow/core/grappler/grappler_item.h"
-#include "tensorflow/core/grappler/mutable_graph_view.h"
 #include "tensorflow/core/grappler/op_types.h"
 #include "tensorflow/core/grappler/optimizers/constant_folding.h"
 #include "tensorflow/core/grappler/utils.h"
@@ -42,15 +38,20 @@ namespace grappler {
 
 namespace {
 
-// Builds a map from the &graph->node(i) to i.
-absl::flat_hash_map<const NodeDef*, int> BuildNodeToIdx(const GraphDef& graph) {
-  // Set up &node -> index map.
-  absl::flat_hash_map<const NodeDef*, int> node_to_idx;
-  for (int i = 0; i < graph.node_size(); ++i) {
-    const NodeDef& node = graph.node(i);
-    node_to_idx[&node] = i;
+bool RemoveInput(NodeDef* node, const string& input, NodeMap* node_map) {
+  bool removed_input = false;
+  int pos = 0;
+  while (pos < node->input_size()) {
+    if (node->input(pos) == input) {
+      node->mutable_input()->SwapElements(pos, node->input_size() - 1);
+      node->mutable_input()->RemoveLast();
+      node_map->RemoveOutput(NodeName(input), node->name());
+      removed_input = true;
+    } else {
+      ++pos;
+    }
   }
-  return node_to_idx;
+  return removed_input;
 }
 
 }  // namespace
@@ -67,10 +68,7 @@ bool DependencyOptimizer::SafeToRemoveIdentity(const NodeDef& node) const {
     // The output values of this node may be needed.
     return false;
   }
-
-  MutableGraphView::OutputPort port = graph_view_->GetRegularFanin(
-      MutableGraphView::InputPort(const_cast<NodeDef*>(&node), 0));
-  NodeDef* input = port.node;
+  const NodeDef* input = node_map_->GetNode(NodeName(node.input(0)));
   CHECK(input != nullptr) << "node = " << node.name()
                           << " input = " << node.input(0);
   // Don't remove Identity nodes corresponding to Variable reads or following
@@ -81,23 +79,20 @@ bool DependencyOptimizer::SafeToRemoveIdentity(const NodeDef& node) const {
     // Don't turn Identity nodes following Switch into NoOp or remove them
     // if it requires anchoring a control dependencies the Switch node, which
     // is not valid.
-    MutableGraphView::OutputPort control_port(const_cast<NodeDef*>(&node),
-                                              Graph::kControlSlot);
-    if (!graph_view_->GetFanout(control_port).empty()) {
+    if (str_util::StartsWith(node.name(), kConstantFoldingCtrl)) {
+      // TODO(rmlarsen): Try to remove this artificial contraint.
       return false;
     }
   }
-  bool node_has_multiple_inputs =
-      graph_view_->NumFanins(node, /*include_controlling_nodes=*/true) > 1;
-  for (auto consumer :
-       graph_view_->GetFanouts(node, /*include_controlled_nodes=*/true)) {
-    if (node_has_multiple_inputs && IsMerge(*consumer.node)) {
+  for (auto consumer : node_map_->GetOutputs(node.name())) {
+    if (node.input_size() > 1 && IsMerge(*consumer)) {
       return false;
     }
     if (IsSwitch(*input)) {
-      if (graph_view_->HasFanin(*consumer.node,
-                                {node.name(), Graph::kControlSlot})) {
-        return false;
+      for (const string& consumer_input : consumer->input()) {
+        if (consumer_input == AsControlDependency(node.name())) {
+          return false;
+        }
       }
     }
   }
@@ -131,7 +126,7 @@ bool DependencyOptimizer::SafeToConvertToNoOp(const NodeDef& node) const {
   if (!SafeToRemoveIdentity(node)) {
     return false;
   }
-  if (graph_view_->NumFanouts(node, /*include_controlled_nodes=*/false) > 0) {
+  if (NumNonControlOutputs(node, *node_map_) > 0) {
     // The output values of this node may be needed.
     return false;
   }
@@ -139,56 +134,61 @@ bool DependencyOptimizer::SafeToConvertToNoOp(const NodeDef& node) const {
 }
 
 int DependencyOptimizer::NumEdgesIfBypassed(
-    const NodeDef& node, int num_fanins,
-    const absl::flat_hash_set<MutableGraphView::Edge>& fanout_edges) const {
+    const NodeDef& node, const std::vector<NodeDef*>& output_nodes) const {
   const bool is_multi_input_identity_n =
       IsIdentityN(node) && !IsIdentityNSingleInput(node);
-  const int num_fanouts = fanout_edges.size();
+  const int num_outputs = output_nodes.size();
+  const int num_inputs = node.input_size();
 
   if (is_multi_input_identity_n) {
     // multi-input identity_n with input/output control dependencies will likely
     // increase number of edges after optimization.
     int num_edges_if_bypassed(0);
-    int num_non_controlling_fanins =
-        graph_view_->NumFanins(node, /*include_controlling_nodes=*/false);
-    num_edges_if_bypassed += num_non_controlling_fanins;
-    num_edges_if_bypassed +=
-        (num_fanins - num_non_controlling_fanins) * num_fanouts;
-
-    for (const auto& fanout : fanout_edges) {
-      if (fanout.dst.port_id == Graph::kControlSlot) {
-        num_edges_if_bypassed += num_fanins;
+    for (string input_node_name : node.input()) {
+      if (IsControlInput(input_node_name)) {
+        num_edges_if_bypassed += num_outputs;
       } else {
         ++num_edges_if_bypassed;
       }
     }
+
+    for (auto consumer : output_nodes) {
+      for (int j = 0; j < consumer->input_size(); ++j) {
+        const TensorId consumer_input = ParseTensorName(consumer->input(j));
+        if (consumer_input.node() == node.name()) {
+          if (IsControlInput(consumer_input)) {
+            num_edges_if_bypassed += num_inputs;
+          } else {
+            ++num_edges_if_bypassed;
+          }
+        }
+      }
+    }
     return num_edges_if_bypassed;
   } else {
-    return num_fanins * num_fanouts;
+    return num_inputs * num_outputs;
   }
 }
 
 bool DependencyOptimizer::BypassingNodeIsBeneficial(
-    const NodeDef& node,
-    const absl::flat_hash_set<MutableGraphView::OutputPort>& fanins,
-    const absl::flat_hash_set<MutableGraphView::Edge>& fanout_edges) const {
+    const NodeDef& node, const std::vector<NodeDef*>& input_nodes,
+    const std::vector<NodeDef*>& output_nodes) const {
   const bool is_identity = IsIdentity(node) || IsIdentityNSingleInput(node);
   const bool is_multi_input_identity_n =
       IsIdentityN(node) && !IsIdentityNSingleInput(node);
-  const int num_outputs = fanout_edges.size();
-  const int num_inputs = fanins.size();
+  const int num_outputs = output_nodes.size();
+  const int num_inputs = node.input_size();
 
-  if (NumEdgesIfBypassed(node, num_inputs, fanout_edges) >
-      num_inputs + num_outputs) {
+  if (NumEdgesIfBypassed(node, output_nodes) > num_inputs + num_outputs) {
     return false;
   }
 
   // Make sure that we don't increase the number of edges that cross
   // device boundaries.
   if ((num_inputs == 1 && num_outputs > 1 &&
-       fanins.begin()->node->device() != node.device()) ||
+       input_nodes[0]->device() != node.device()) ||
       (num_inputs > 1 && num_outputs == 1 &&
-       fanout_edges.begin()->dst.node->device() != node.device())) {
+       output_nodes[0]->device() != node.device())) {
     return false;
   }
 
@@ -197,12 +197,12 @@ bool DependencyOptimizer::BypassingNodeIsBeneficial(
   // cost before and after.
   const string& node_dev = node.device();
   int num_cross_in = 0;
-  for (const auto& fanin : fanins) {
-    num_cross_in += static_cast<int>(fanin.node->device() != node_dev);
+  for (NodeDef* input_node : input_nodes) {
+    num_cross_in += static_cast<int>(input_node->device() != node_dev);
   }
   int num_cross_out = 0;
-  for (const auto& fanout : fanout_edges) {
-    num_cross_out += static_cast<int>(fanout.dst.node->device() != node_dev);
+  for (NodeDef* output_node : output_nodes) {
+    num_cross_out += static_cast<int>(output_node->device() != node_dev);
   }
 
   if ((is_identity || is_multi_input_identity_n) && num_cross_in > 0 &&
@@ -216,10 +216,10 @@ bool DependencyOptimizer::BypassingNodeIsBeneficial(
   // Make sure we do not increase the number of device crossings.
   const int num_cross_before = num_cross_in + num_cross_out;
   int num_cross_after = 0;
-  for (const auto& fanin : fanins) {
-    for (const auto& fanout : fanout_edges) {
+  for (NodeDef* input_node : input_nodes) {
+    for (NodeDef* output_node : output_nodes) {
       num_cross_after +=
-          static_cast<int>(fanin.node->device() != fanout.dst.node->device());
+          static_cast<int>(input_node->device() != output_node->device());
     }
   }
   if (num_cross_after > num_cross_before) {
@@ -228,32 +228,45 @@ bool DependencyOptimizer::BypassingNodeIsBeneficial(
   return true;
 }
 
-void DependencyOptimizer::OptimizeNode(NodeDef* node,
-                                       SetVector<NodeDef*>* nodes_to_simplify,
-                                       std::set<string>* nodes_to_delete) {
-  const string node_name = node->name();
+void DependencyOptimizer::OptimizeNode(int node_idx,
+                                       SetVector<int>* nodes_to_simplify,
+                                       std::set<int>* nodes_to_delete) {
+  NodeDef* node = optimized_graph_->mutable_node(node_idx);
   const bool is_noop = IsNoOp(*node);
   const bool is_identity = IsIdentity(*node) || IsIdentityNSingleInput(*node);
   const bool is_multi_input_identity =
       IsIdentityN(*node) && !IsIdentityNSingleInput(*node);
+  const string node_name = node->name();
   // Constant nodes with no input control dependency are always executed early,
   // so we can prune all their output control dependencies.
-  if (IsConstant(*node) &&
-      graph_view_->NumFanins(*node, /*include_controlling_nodes=*/true) == 0) {
-    for (const auto& fanout :
-         graph_view_->GetFanouts(*node, /*include_controlled_nodes=*/true)) {
-      if (graph_view_->RemoveFanin(fanout.node->name(),
-                                   {node_name, Graph::kControlSlot})) {
-        nodes_to_simplify->PushBack(fanout.node);
+  if (IsConstant(*node) && node->input_size() == 0) {
+    const std::set<NodeDef*> output_nodes = node_map_->GetOutputs(node_name);
+    for (NodeDef* fanout : output_nodes) {
+      bool optimize_fanout = false;
+      bool data_connection = false;
+      for (int i = fanout->input_size() - 1; i >= 0; --i) {
+        const TensorId input_tensor = ParseTensorName(fanout->input(i));
+        if (input_tensor.node() == node_name) {
+          if (input_tensor.index() < 0) {
+            fanout->mutable_input()->SwapElements(i, fanout->input_size() - 1);
+            fanout->mutable_input()->RemoveLast();
+            optimize_fanout = true;
+          } else {
+            data_connection = true;
+          }
+        }
+      }
+      if (optimize_fanout) {
+        nodes_to_simplify->PushBack(node_to_idx_[fanout]);
+        if (!data_connection) {
+          node_map_->RemoveOutput(node_name, fanout->name());
+        }
       }
     }
-
-    if (graph_view_->NumFanouts(*node, /*include_controlled_nodes=*/true) ==
-            0 &&
-        fetch_nodes_known_ &&
+    if (node_map_->GetOutputs(node_name).empty() && fetch_nodes_known_ &&
         nodes_to_preserve_.find(node_name) == nodes_to_preserve_.end()) {
       // Mark the node for deletion.
-      nodes_to_delete->insert(node->name());
+      nodes_to_delete->insert(node_to_idx_[node]);
     }
     return;
   }
@@ -264,23 +277,33 @@ void DependencyOptimizer::OptimizeNode(NodeDef* node,
             << ") with NoOp.";
     // The outputs of this node are not consumed. Replace its inputs with
     // control dependencies and replace the op itself with the NoOp op.
-    int num_non_controlling_fanins =
-        graph_view_->NumFanins(*node, /*include_controlling_nodes=*/false);
-    if (num_non_controlling_fanins > 0) {
-      absl::flat_hash_set<string> non_controlling_fanins(
-          node->input().begin(),
-          node->input().begin() + num_non_controlling_fanins);
-      graph_view_->RemoveAllFanins(node_name,
-                                   /*keep_controlling_fanins=*/true);
-      for (const string& fanin : non_controlling_fanins) {
-        TensorId tensor_id = ParseTensorName(fanin);
-        graph_view_->AddControllingFanin(node_name, tensor_id);
-        nodes_to_simplify->PushBack(graph_view_->GetNode(tensor_id.node()));
+    std::unordered_set<string> ctrl_inputs;
+    int pos = 0;
+    while (pos < node->input_size()) {
+      const string old_input = node->input(pos);
+      if (IsControlInput(old_input)) {
+        if (!ctrl_inputs.insert(old_input).second) {
+          // We found a duplicate control input. Remove it.
+          node->mutable_input()->SwapElements(pos, node->input_size() - 1);
+          node->mutable_input()->RemoveLast();
+        } else {
+          ++pos;
+        }
+        continue;
       }
+      // Replace a normal input with a control input.
+      const string ctrl_input = ConstantFolding::AddControlDependency(
+          old_input, optimized_graph_, node_map_.get());
+      ctrl_inputs.insert(ctrl_input);
+      node->set_input(pos, ctrl_input);
+      node_map_->UpdateInput(node_name, old_input, ctrl_input);
+      const NodeDef* old_input_node = node_map_->GetNode(old_input);
+      nodes_to_simplify->PushBack(node_to_idx_[old_input_node]);
+      ++pos;
     }
     node->set_op("NoOp");
     node->clear_attr();
-    nodes_to_simplify->PushBack(node);
+    nodes_to_simplify->PushBack(node_to_idx_[node]);
     return;
   }
 
@@ -334,80 +357,110 @@ void DependencyOptimizer::OptimizeNode(NodeDef* node,
 
   if (is_noop || ((is_identity || is_multi_input_identity) &&
                   SafeToRemoveIdentity(*node))) {
-    auto fanins =
-        graph_view_->GetFanins(*node, /*include_controlling_nodes=*/true);
-    auto fanout_edges =
-        graph_view_->GetFanoutEdges(*node, /*include_controlled_edges=*/true);
+    const auto& output_node_set = node_map_->GetOutputs(node_name);
+    const std::vector<NodeDef*> output_nodes(output_node_set.begin(),
+                                             output_node_set.end());
+    const int num_inputs = node->input_size();
+    std::vector<NodeDef*> input_nodes;
+    for (int i = 0; i < num_inputs; ++i) {
+      NodeDef* input_node = node_map_->GetNode(node->input(i));
+      if (input_node == nullptr) {
+        LOG(ERROR) << "Invalid input " << node->input(i);
+        return;
+      }
+      input_nodes.push_back(input_node);
+    }
 
-    if (!BypassingNodeIsBeneficial(*node, fanins, fanout_edges)) {
+    if (!BypassingNodeIsBeneficial(*node, input_nodes, output_nodes)) {
       return;
     }
 
-    int num_non_controlling_fanins =
-        graph_view_->NumFanins(*node, /*include_controlling_nodes=*/false);
     VLOG(1) << "***** Rerouting input around\n" << node->DebugString();
     // Now remove the node and re-wire its inputs to its outputs.
-    for (auto fanout_edge : fanout_edges) {
+    for (auto consumer : output_nodes) {
       bool updated_consumer = false;
-      NodeDef* consumer = fanout_edge.dst.node;
       VLOG(1) << "consumer before:\n" << consumer->DebugString();
-      if (fanout_edge.src.port_id == Graph::kControlSlot) {
-        updated_consumer = graph_view_->RemoveFanin(
-            consumer->name(), {node_name, Graph::kControlSlot});
-        if (updated_consumer) {
-          // Add all non controlling fanins of node as controlling fanins to
-          // consumer.
-          for (int i = 0; i < num_non_controlling_fanins; ++i) {
-            TensorId input = ParseTensorName(node->input(i));
-            if (graph_view_->AddControllingFanin(consumer->name(), input)) {
-              nodes_to_simplify->PushBack(graph_view_->GetNode(input.node()));
+      for (int i = 0; i < num_inputs; ++i) {
+        const NodeDef* input = input_nodes[i];
+        // Forward dependency from input to consumer if it doesn't already
+        // depend on it.
+        if ((is_identity && i == 0) ||
+            (is_multi_input_identity && !IsControlInput(node->input(i)))) {
+          // Replace regular input from Identity node.
+          string new_input;
+          const string& input_to_forward = node->input(i);
+          CHECK(!IsControlInput(input_to_forward));
+          for (int j = 0; j < consumer->input_size(); ++j) {
+            const TensorId old_input = ParseTensorName(consumer->input(j));
+            if (old_input.node() == node_name) {
+              if (old_input.index() == i) {
+                // Regular input
+                new_input = input_to_forward;
+                node_map_->UpdateInput(consumer->name(), old_input.ToString(),
+                                       new_input);
+                consumer->set_input(j, new_input);
+              } else if (old_input.index() == -1) {
+                // Control dependency
+                new_input = AsControlDependency(NodeName(input_to_forward));
+                node_map_->UpdateInput(consumer->name(), old_input.ToString(),
+                                       new_input);
+                consumer->set_input(j, new_input);
+              }
             }
           }
+          updated_consumer = true;
+        } else {
+          // Forward dependency from input to consumer if it doesn't already
+          // depend on it.
+          if (node_map_->GetOutputs(input->name()).count(consumer) == 0) {
+            consumer->add_input(AsControlDependency(input->name()));
+            node_map_->AddOutput(input->name(), consumer->name());
+            nodes_to_simplify->PushBack(node_to_idx_[input]);
+            updated_consumer = true;
+          }
         }
-      } else {
-        updated_consumer = graph_view_->UpdateFanin(
-            consumer->name(), {node_name, fanout_edge.src.port_id},
-            ParseTensorName(node->input(fanout_edge.src.port_id)));
       }
+      // Remove dependency on node from consumer.
+      updated_consumer |= RemoveInput(consumer, AsControlDependency(node_name),
+                                      node_map_.get());
       if (updated_consumer) {
-        // Forward all controlling fanins of node to consumer.
-        for (int i = num_non_controlling_fanins; i < node->input_size(); ++i) {
-          TensorId input = ParseTensorName(node->input(i));
-          if (graph_view_->AddFanin(consumer->name(), input)) {
-            nodes_to_simplify->PushBack(graph_view_->GetNode(input.node()));
-          }
-        }
-        nodes_to_simplify->PushBack(consumer);
+        nodes_to_simplify->PushBack(node_to_idx_[consumer]);
       }
       VLOG(1) << "consumer after:\n" << consumer->DebugString();
     }
+    node_map_->RemoveOutputs(node_name);
     if (fetch_nodes_known_ &&
         nodes_to_preserve_.find(node_name) == nodes_to_preserve_.end()) {
       // Mark the node for deletion.
-      nodes_to_delete->insert(node_name);
+      nodes_to_delete->insert(node_idx);
 
       // Disconnect the node from its inputs to enable further optimizations.
-      graph_view_->RemoveAllFanins(node_name,
-                                   /*keep_controlling_fanins=*/false);
+      node_map_->RemoveInputs(node_name);
+      node->clear_input();
     }
   }
 }
 
+void DependencyOptimizer::CleanControlInputs() {
+  for (int i = 0; i < optimized_graph_->node_size(); ++i) {
+    DedupControlInputs(optimized_graph_->mutable_node(i));
+  }
+}
+
 Status DependencyOptimizer::OptimizeDependencies() {
-  SetVector<NodeDef*> nodes_to_simplify;
-  std::set<string> nodes_to_delete;
-  for (int i = 0; i < graph_view_->graph()->node_size(); ++i) {
-    NodeDef* node = graph_view_->graph()->mutable_node(i);
-    if (IsNoOp(*node) || IsIdentity(*node) || IsIdentityN(*node) ||
-        IsConstant(*node) || SafeToConvertToNoOp(*node)) {
-      nodes_to_simplify.PushBack(node);
+  SetVector<int> nodes_to_simplify;
+  std::set<int> nodes_to_delete;
+  for (int i = 0; i < optimized_graph_->node_size(); ++i) {
+    const NodeDef& node = optimized_graph_->node(i);
+    if (IsNoOp(node) || IsIdentity(node) || IsIdentityN(node) ||
+        IsConstant(node) || SafeToConvertToNoOp(node)) {
+      nodes_to_simplify.PushBack(i);
     }
   }
   while (!nodes_to_simplify.Empty()) {
-    NodeDef* node_to_simplify = nodes_to_simplify.PopBack();
+    int node_to_simplify = nodes_to_simplify.PopBack();
     // Discard nodes that were marked for deletion already.
-    while (nodes_to_delete.find(node_to_simplify->name()) !=
-           nodes_to_delete.end()) {
+    while (nodes_to_delete.find(node_to_simplify) != nodes_to_delete.end()) {
       node_to_simplify = nodes_to_simplify.PopBack();
     }
     OptimizeNode(node_to_simplify, &nodes_to_simplify, &nodes_to_delete);
@@ -415,17 +468,17 @@ Status DependencyOptimizer::OptimizeDependencies() {
 
   if (fetch_nodes_known_) {
     VLOG(1) << "Deleted " << nodes_to_delete.size() << " out of "
-            << graph_view_->graph()->node_size() << " nodes.";
-    graph_view_->DeleteNodes(nodes_to_delete);
+            << optimized_graph_->node_size() << " nodes.";
+    EraseNodesFromGraph(nodes_to_delete, optimized_graph_);
+    node_map_.reset(new NodeMap(optimized_graph_));
+    BuildNodeToIdx();
   }
   return Status::OK();
 }
 
 Status DependencyOptimizer::TransitiveReduction() {
   // PRECONDITION: optimized_graph_ must be sorted topologically.
-  GraphDef* graph = graph_view_->graph();
-  auto node_to_idx = BuildNodeToIdx(*graph);
-  const int num_nodes = graph->node_size();
+  const int num_nodes = optimized_graph_->node_size();
   // Set up a compressed version of the graph to save a constant factor in the
   // expensive algorithm below. Also cache the set of control outputs and the
   // highest index of a target of any control output from each node.
@@ -434,20 +487,20 @@ Status DependencyOptimizer::TransitiveReduction() {
   std::vector<gtl::InlinedVector<std::pair<int, int>, 2>> control_outputs(
       num_nodes);
   for (int node_idx = 0; node_idx < num_nodes; ++node_idx) {
-    const NodeDef& node = graph->node(node_idx);
+    const NodeDef& node = optimized_graph_->node(node_idx);
     if (ModifiesFrameInfo(node) || !HasOpDef(node)) {
       // Ignore function nodes and nodes that modify frame info.
       continue;
     }
     for (int input_slot = 0; input_slot < node.input_size(); ++input_slot) {
       const string& input = node.input(input_slot);
-      const NodeDef* input_node = graph_view_->GetNode(NodeName(input));
+      const NodeDef* input_node = node_map_->GetNode(input);
       if (ModifiesFrameInfo(*input_node) || IsMerge(*input_node)) {
         // Ignore edges from nodes that modify frame info and from Merge nodes,
         // because we cannot know which of it's input paths executes.
         continue;
       }
-      const int input_node_idx = node_to_idx[input_node];
+      const int input_node_idx = node_to_idx_[input_node];
       inputs[node_idx].push_back(input_node_idx);
       if (IsControlInput(input)) {
         ++num_controls;
@@ -513,12 +566,16 @@ Status DependencyOptimizer::TransitiveReduction() {
 
   for (const auto& it : control_edges_to_remove) {
     const int target = it.first;
-    const NodeDef& target_node = graph->node(target);
-    const string target_node_name = target_node.name();
+    NodeDef* target_node = optimized_graph_->mutable_node(target);
     for (const InputSlotAndSource& slot_and_source : it.second) {
       const int input_slot = slot_and_source.first;
-      const TensorId tensor_id = ParseTensorName(target_node.input(input_slot));
-      graph_view_->RemoveFanin(target_node_name, tensor_id);
+      const int source = slot_and_source.second;
+      const NodeDef& source_node = optimized_graph_->node(source);
+      CHECK_LT(input_slot, target_node->input_size());
+      target_node->mutable_input()->SwapElements(input_slot,
+                                                 target_node->input_size() - 1);
+      node_map_->RemoveOutput(source_node.name(), target_node->name());
+      target_node->mutable_input()->RemoveLast();
       ++num_controls_removed;
     }
   }
@@ -527,17 +584,26 @@ Status DependencyOptimizer::TransitiveReduction() {
   return Status::OK();
 }
 
+void DependencyOptimizer::BuildNodeToIdx() {
+  // Set up &node -> index map.
+  node_to_idx_.clear();
+  for (int i = 0; i < optimized_graph_->node_size(); ++i) {
+    const NodeDef& node = optimized_graph_->node(i);
+    node_to_idx_[&node] = i;
+  }
+}
+
 // Suppose there are cross-device control inputs to node C from multiple nodes
 // that are located on another device, e.g., we have control edges:
 // A->C, B->C
 // where A and B are on device X and C is on device Y.
 // We can reduce cross-device communication by introducing an intermediate
 // NoOp node C' on device X and rewriting the control edges to:
-// A->C', B->C', C'->C
+// A->C', B->C', C' -> C
 void DependencyOptimizer::GroupCrossDeviceControlEdges() {
-  const int num_nodes = graph_view_->graph()->node_size();
+  const int num_nodes = optimized_graph_->node_size();
   for (int i = 0; i < num_nodes; ++i) {
-    NodeDef* node = graph_view_->graph()->mutable_node(i);
+    NodeDef* node = optimized_graph_->mutable_node(i);
     if (node->device().empty()) continue;
 
     // Creates new noop nodes for devices on which multiple control inputs are
@@ -548,50 +614,66 @@ void DependencyOptimizer::GroupCrossDeviceControlEdges() {
     // that device.
     std::map<string, NodeDef*> noops;
     int num_noops = 0;
-    auto controlling_fanins = graph_view_->GetFanin(
-        MutableGraphView::InputPort(node, Graph::kControlSlot));
-    for (const auto& controlling_fanin : controlling_fanins) {
-      const NodeDef* fanin_node = controlling_fanin.node;
-      if (!fanin_node->device().empty() &&
-          fanin_node->device() != node->device()) {
-        auto emplace_result = noops.emplace(fanin_node->device(), nullptr);
-        if (!emplace_result.second && emplace_result.first->second == nullptr) {
-          // This is the second cross-device control input from the same
-          // device. Creates an intermediate noop node on that device.
-          string group_name;
-          NodeDef* noop;
-          // Creates a fresh node name; there may be conflicting names from
-          // a previous iteration of the optimizer.
-          do {
-            group_name = AddPrefixToNodeName(
-                node->name(),
-                strings::StrCat("GroupCrossDeviceControlEdges_", num_noops));
-            noop = graph_view_->GetNode(group_name);
-            ++num_noops;
-          } while (noop != nullptr);
-          NodeDef new_node;
-          new_node.set_name(group_name);
-          new_node.set_device(fanin_node->device());
-          new_node.set_op("NoOp");
-          emplace_result.first->second =
-              graph_view_->AddNode(std::move(new_node));
+    for (int j = 0; j < node->input_size(); ++j) {
+      if (IsControlInput(node->input(j))) {
+        const NodeDef* input = node_map_->GetNode(node->input(j));
+        if (input != nullptr && !input->device().empty() &&
+            input->device() != node->device()) {
+          auto emplace_result = noops.emplace(input->device(), nullptr);
+          if (!emplace_result.second &&
+              emplace_result.first->second == nullptr) {
+            // This is the second cross-device control input from the same
+            // device. Creates an intermediate noop node on that device.
+            string group_name;
+            NodeDef* noop;
+            // Creates a fresh node name; there may be conflicting names from
+            // a previous iteration of the optimizer.
+            do {
+              group_name = AddPrefixToNodeName(
+                  node->name(),
+                  strings::StrCat("GroupCrossDeviceControlEdges_", num_noops));
+              noop = node_map_->GetNode(group_name);
+              ++num_noops;
+            } while (noop != nullptr);
+            noop = optimized_graph_->add_node();
+            noop->set_name(group_name);
+            noop->set_device(input->device());
+            noop->set_op("NoOp");
+            node_map_->AddNode(noop->name(), noop);
+            emplace_result.first->second = noop;
+          }
         }
       }
     }
 
     // Reroute existing control edges to go via the newly introduced NoOp nodes.
-    for (const auto& controlling_fanin : controlling_fanins) {
-      auto it = noops.find(controlling_fanin.node->device());
-      if (it != noops.end() && it->second != nullptr) {
-        TensorId tensor_id(controlling_fanin.node->name(), Graph::kControlSlot);
-        graph_view_->RemoveFanin(node->name(), tensor_id);
-        graph_view_->AddFanin(it->second->name(), tensor_id);
+    int pos = 0;
+    while (pos < node->input_size()) {
+      const string& input_name = node->input(pos);
+      if (IsControlInput(input_name)) {
+        NodeDef* input = node_map_->GetNode(input_name);
+        if (input == nullptr) {
+          ++pos;
+        } else {
+          auto it = noops.find(input->device());
+          if (it == noops.end() || it->second == nullptr) {
+            ++pos;
+          } else {
+            node->mutable_input()->SwapElements(pos, node->input_size() - 1);
+            node->mutable_input()->RemoveLast();
+            it->second->add_input(AsControlDependency(*input));
+            node_map_->UpdateOutput(input_name, node->name(),
+                                    it->second->name());
+          }
+        }
+      } else {
+        ++pos;
       }
     }
     for (const auto& entry : noops) {
       if (entry.second) {
-        graph_view_->AddFanin(node->name(),
-                              {entry.second->name(), Graph::kControlSlot});
+        node->add_input(AsControlDependency(*entry.second));
+        node_map_->AddOutput(entry.second->name(), node->name());
       }
     }
   }
@@ -599,17 +681,22 @@ void DependencyOptimizer::GroupCrossDeviceControlEdges() {
 
 Status DependencyOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
                                      GraphDef* optimized_graph) {
-  *optimized_graph = item.graph;
+  optimized_graph_ = optimized_graph;
+  *optimized_graph_ = item.graph;
   nodes_to_preserve_ = item.NodesToPreserve();
   fetch_nodes_known_ = !item.fetch.empty();
-  graph_view_.reset(new MutableGraphView(optimized_graph));
+  CleanControlInputs();
 
   const int num_iterations = 2;
   for (int iteration = 0; iteration < num_iterations; ++iteration) {
     GRAPPLER_RETURN_IF_DEADLINE_EXCEEDED();
     Status topo_sort_status;
     // Perform topological sort to prepare the graph for transitive reduction.
-    topo_sort_status = TopologicalSort(optimized_graph);
+    topo_sort_status = TopologicalSort(optimized_graph_);
+    // Set up index-based graph datastructures to speed up analysis steps below.
+    node_map_.reset(new NodeMap(optimized_graph_));
+    BuildNodeToIdx();
+
     if (topo_sort_status.ok()) {
       // Remove redundant control dependencies.
       TF_RETURN_IF_ERROR(TransitiveReduction());
@@ -622,6 +709,9 @@ Status DependencyOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
     // nodes.
     TF_RETURN_IF_ERROR(OptimizeDependencies());
 
+    // Dedup control inputs.
+    CleanControlInputs();
+
     GroupCrossDeviceControlEdges();
   }
 
diff --git a/tensorflow/core/grappler/optimizers/dependency_optimizer.h b/tensorflow/core/grappler/optimizers/dependency_optimizer.h
index eb7fc89080..7b032673fb 100644
--- a/tensorflow/core/grappler/optimizers/dependency_optimizer.h
+++ b/tensorflow/core/grappler/optimizers/dependency_optimizer.h
@@ -17,8 +17,6 @@ limitations under the License.
 #define TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DEPENDENCY_OPTIMIZER_H_
 
 #include <unordered_set>
-
-#include "tensorflow/core/grappler/mutable_graph_view.h"
 #include "tensorflow/core/grappler/optimizers/graph_optimizer.h"
 #include "tensorflow/core/grappler/utils.h"
 #include "tensorflow/core/protobuf/rewriter_config.pb.h"
@@ -48,22 +46,24 @@ class DependencyOptimizer : public GraphOptimizer {
   // Returns true if bypassing node does not increase the number of edges or
   // number of edges crossing a device boundary.
   bool BypassingNodeIsBeneficial(
-      const NodeDef& node,
-      const absl::flat_hash_set<MutableGraphView::OutputPort>& fanins,
-      const absl::flat_hash_set<MutableGraphView::Edge>& fanout_edges) const;
-  int NumEdgesIfBypassed(
-      const NodeDef& node, int num_fanins,
-      const absl::flat_hash_set<MutableGraphView::Edge>& fanout_edges) const;
+      const NodeDef& node, const std::vector<NodeDef*>& input_nodes,
+      const std::vector<NodeDef*>& output_nodes) const;
+  int NumEdgesIfBypassed(const NodeDef& node,
+                         const std::vector<NodeDef*>& output_nodes) const;
   // Returns true if node is not an Identity node or if it is an Identity
   // that is safe to remove.
   bool SafeToRemoveIdentity(const NodeDef& node) const;
   // Returns true if it is safe to convert node to NoOp.
   bool SafeToConvertToNoOp(const NodeDef& node) const;
+  // Removes all duplicate control dependencies.
+  void CleanControlInputs();
+  // Builds a map from the &optimized_graph_->node(i) to i.
+  void BuildNodeToIdx();
   // Tries to optimize the node with the given index, possibly additional
   // optimizations by inserting nodes in nodes_to_simplify, and pruning nodes by
   // inserting them in nodes_to_delete.
-  void OptimizeNode(NodeDef* node, SetVector<NodeDef*>* nodes_to_simplify,
-                    std::set<string>* nodes_to_delete);
+  void OptimizeNode(int node_idx, SetVector<int>* nodes_to_simplify,
+                    std::set<int>* nodes_to_delete);
   // Eliminates redundant control dependencies by computing the transitive
   // reduction of the graph.
   Status TransitiveReduction();
@@ -76,7 +76,9 @@ class DependencyOptimizer : public GraphOptimizer {
   RewriterConfig::Toggle opt_level_;
   bool fetch_nodes_known_;
   std::unordered_set<string> nodes_to_preserve_;
-  std::unique_ptr<MutableGraphView> graph_view_;
+  std::unique_ptr<NodeMap> node_map_;
+  std::unordered_map<const NodeDef*, int> node_to_idx_;
+  GraphDef* optimized_graph_;  // Not owned.
 };
 
 }  // end namespace grappler
diff --git a/tensorflow/core/grappler/optimizers/dependency_optimizer_test.cc b/tensorflow/core/grappler/optimizers/dependency_optimizer_test.cc
index 673fc987a8..8d70d9d5c7 100644
--- a/tensorflow/core/grappler/optimizers/dependency_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/dependency_optimizer_test.cc
@@ -91,7 +91,7 @@ TEST_F(DependencyOptimizerTest, DependenciesDrivenByConstants) {
   // The 'z' node should have been optimized away leaving only 5 nodes.
   EXPECT_EQ(5, output.node_size());
 
-  for (const NodeDef& node : output.node()) {
+  for (const NodeDef& node : item.graph.node()) {
     if (node.name() == "id1" || node.name() == "id2") {
       EXPECT_EQ(1, node.input_size());
       EXPECT_EQ("add", node.input(0));
@@ -125,8 +125,8 @@ TEST_F(DependencyOptimizerTest, ChangeToNoop) {
 
   EXPECT_EQ(item.graph.node_size(), output.node_size());
   int found = 0;
-  for (int i = 0; i < output.node_size(); ++i) {
-    const NodeDef& node = output.node(i);
+  for (int i = 0; i < item.graph.node_size(); ++i) {
+    const NodeDef& node = item.graph.node(i);
     // "add" should get turned into a NoOp and removed.
     EXPECT_NE("add", node.name());
     if (node.name() == "id1") {
@@ -164,6 +164,7 @@ TEST_F(DependencyOptimizerTest, ChangeToNoop_RepeatedInput) {
   item.graph.Swap(&output);
   status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
+  LOG(INFO) << output.DebugString();
 
   EXPECT_EQ(item.graph.node_size(), output.node_size());
   int found = 0;
@@ -747,7 +748,7 @@ TEST_F(DependencyOptimizerTest, Identity_DeviceCrossing_ConsumerOnSameDevice) {
   GraphDef output;
   Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
-
+  LOG(INFO) << output.DebugString();
   EXPECT_EQ(3, output.node_size());
   for (const auto& node : output.node()) {
     EXPECT_NE("x_on_2", node.name());
-- 
GitLab


From 93f760a1d83404cadf2ce1bc8cd8268c47e2f4f8 Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Tue, 8 Jan 2019 10:41:24 -0800
Subject: [PATCH 0386/1095] [XLA] Fix VLOG(0) usage in
 dynamic_dimension_inference that caused a lot of LOG spam.

PiperOrigin-RevId: 228357053
---
 tensorflow/compiler/xla/service/dynamic_dimension_inference.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/dynamic_dimension_inference.cc b/tensorflow/compiler/xla/service/dynamic_dimension_inference.cc
index f2b7e9a186..2b158d7a6e 100644
--- a/tensorflow/compiler/xla/service/dynamic_dimension_inference.cc
+++ b/tensorflow/compiler/xla/service/dynamic_dimension_inference.cc
@@ -433,7 +433,7 @@ Status DynamicDimensionInferenceVisitor::ForEachOperandDynamicDimension(
 /* static */
 StatusOr<DynamicDimensionInference> DynamicDimensionInference::Run(
     HloModule* module) {
-  VLOG(0) << "Param Config " << module->dynamic_parameter_binding().ToString();
+  VLOG(2) << "Param Config " << module->dynamic_parameter_binding().ToString();
   DynamicDimensionInference inference(module);
   TF_RETURN_IF_ERROR(inference.AnalyzeDynamicDimensions());
   return inference;
-- 
GitLab


From 2ce3e3758d0fc1eea31b3e16c94321198c121953 Mon Sep 17 00:00:00 2001
From: Pete Warden <petewarden@google.com>
Date: Tue, 8 Jan 2019 10:47:29 -0800
Subject: [PATCH 0387/1095] Fix for Raspberry Pi build, removing default
 dependency on Intel MKL

PiperOrigin-RevId: 228358313
---
 tensorflow/tools/ci_build/pi/build_raspberry_pi.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/tools/ci_build/pi/build_raspberry_pi.sh b/tensorflow/tools/ci_build/pi/build_raspberry_pi.sh
index 864278c647..987f0769b2 100755
--- a/tensorflow/tools/ci_build/pi/build_raspberry_pi.sh
+++ b/tensorflow/tools/ci_build/pi/build_raspberry_pi.sh
@@ -107,6 +107,7 @@ bazel build -c opt ${PI_COPTS} \
   --copt=-funsafe-math-optimizations --copt=-ftree-vectorize \
   --copt=-fomit-frame-pointer --cpu=armeabi \
   --crosstool_top=@local_config_arm_compiler//:toolchain \
+  --define tensorflow_mkldnn_contraction_kernel=0 \
   --verbose_failures \
   //tensorflow:libtensorflow.so \
   //tensorflow:libtensorflow_framework.so \
-- 
GitLab


From dd355d3b10a73474fb34972ec0ec708868bd0a24 Mon Sep 17 00:00:00 2001
From: Saurabh Saxena <srbs@google.com>
Date: Tue, 8 Jan 2019 11:13:29 -0800
Subject: [PATCH 0388/1095] Fill zeros for DT_INVALID tensors in
 TensorListStack.

PiperOrigin-RevId: 228363806
---
 tensorflow/core/kernels/BUILD                 |  1 +
 tensorflow/core/kernels/fill_functor.cc       |  5 ++
 tensorflow/core/kernels/list_kernels.h        | 71 ++++++++++++++-----
 .../python/kernel_tests/list_ops_test.py      | 41 +++++++++++
 .../kernel_tests/tensor_array_ops_test.py     | 13 ++--
 5 files changed, 106 insertions(+), 25 deletions(-)

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index b934c64bfc..aaea09bdc4 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -2256,6 +2256,7 @@ tf_kernel_library(
     ],
     deps = [
         ":concat_lib",
+        ":fill_functor",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:list_ops_op_lib",
diff --git a/tensorflow/core/kernels/fill_functor.cc b/tensorflow/core/kernels/fill_functor.cc
index 7090417dfd..9c4c0487f0 100644
--- a/tensorflow/core/kernels/fill_functor.cc
+++ b/tensorflow/core/kernels/fill_functor.cc
@@ -51,6 +51,11 @@ DEFINE_SETZERO_CPU(uint16);
 DEFINE_SETZERO_CPU(int16);
 DEFINE_SETZERO_CPU(int32);
 DEFINE_SETZERO_CPU(int64);
+DEFINE_SETZERO_CPU(quint8);
+DEFINE_SETZERO_CPU(qint8);
+DEFINE_SETZERO_CPU(quint16);
+DEFINE_SETZERO_CPU(qint16);
+DEFINE_SETZERO_CPU(qint32);
 DEFINE_SETZERO_CPU(complex64);
 DEFINE_SETZERO_CPU(complex128);
 DEFINE_SETZERO_CPU(Variant);
diff --git a/tensorflow/core/kernels/list_kernels.h b/tensorflow/core/kernels/list_kernels.h
index 559e14b03c..fd1be80f11 100644
--- a/tensorflow/core/kernels/list_kernels.h
+++ b/tensorflow/core/kernels/list_kernels.h
@@ -28,6 +28,7 @@ limitations under the License.
 #include "tensorflow/core/framework/variant.h"
 #include "tensorflow/core/framework/variant_op_registry.h"
 #include "tensorflow/core/kernels/concat_lib.h"
+#include "tensorflow/core/kernels/fill_functor.h"
 #include "tensorflow/core/lib/core/coding.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/gtl/array_slice.h"
@@ -65,6 +66,15 @@ struct TensorList {
 
 Status TensorShapeFromTensor(const Tensor& t, PartialTensorShape* out);
 
+// Allocates a Tensor of requested shape and dtype and fills it with zeros.
+template <typename Device, typename T>
+void BuildZerosTensor(OpKernelContext* c, DataType dtype,
+                      const TensorShape& shape, Tensor* zeros) {
+  OP_REQUIRES_OK(c, c->allocate_temp(dtype, shape, zeros));
+  functor::SetZeroFunctor<Device, T> f;
+  f(c->eigen_device<Device>(), zeros->flat<T>());
+}
+
 template <typename Device, typename T>
 class TensorListStack : public OpKernel {
  public:
@@ -94,7 +104,7 @@ class TensorListStack : public OpKernel {
         !tensor_list->tensors.empty() ||
             tensor_list->element_shape.IsFullyDefined(),
         errors::InvalidArgument("Tried to stack elements of a empty ",
-                                "list with non-fully-defined shape: ",
+                                "list with non-fully-defined element_shape: ",
                                 tensor_list->element_shape.DebugString()));
     if (num_elements_ != -1) {
       OP_REQUIRES(c, tensor_list->tensors.size() == num_elements_,
@@ -106,34 +116,59 @@ class TensorListStack : public OpKernel {
     // Compute the shape of the output tensor.
     // If `element_shape` is fully-defined it gets used. It is assumed that all
     // element tensors have the same shape.
-    // If `element_shape` is not fully-defined the shape of the first element
-    // tensor is used and it is checked that all other tensors have the same
-    // shape.
-    TensorShape resulting_shape;
-    if (!tensor_list->element_shape.AsTensorShape(&resulting_shape)) {
-      const Tensor& t = tensor_list->tensors[0];
-      resulting_shape = t.shape();
-      for (int i = 1; i < tensor_list->tensors.size(); ++i) {
+    // If `element_shape` is not fully-defined the shape of the first
+    // initialized element tensor is used and it is checked that all other
+    // initialized tensors have the same shape. An error is thrown if the list
+    // only contains DT_INVALID type tensors.
+    TensorShape resulting_element_shape;
+    if (!tensor_list->element_shape.AsTensorShape(&resulting_element_shape)) {
+      bool resulting_element_shape_initialized = false;
+      for (int i = 0; i < tensor_list->tensors.size(); ++i) {
         const Tensor& t = tensor_list->tensors[i];
-        OP_REQUIRES(c, t.shape() == resulting_shape,
-                    errors::InvalidArgument(
-                        "Tried to stack tensors with unequal shapes: ",
-                        resulting_shape.DebugString(), " vs ",
-                        t.shape().DebugString()));
+        if (!resulting_element_shape_initialized) {
+          if (t.dtype() == DT_INVALID) {
+            continue;
+          }
+          resulting_element_shape = t.shape();
+          resulting_element_shape_initialized = true;
+          continue;
+        }
+        OP_REQUIRES(
+            c, t.dtype() == DT_INVALID || t.shape() == resulting_element_shape,
+            errors::InvalidArgument(
+                "Tried to stack tensors with unequal shapes: ",
+                resulting_element_shape.DebugString(), " vs ",
+                t.shape().DebugString()));
       }
+      OP_REQUIRES(
+          c, resulting_element_shape_initialized,
+          errors::InvalidArgument("Tried to stack list which only contains ",
+                                  "uninitialized tensors and has a ",
+                                  "non-fully-defined element_shape: ",
+                                  tensor_list->element_shape.DebugString()));
     }
-    resulting_shape.InsertDim(0, tensor_list->tensors.size());
+    TensorShape output_tensor_shape = resulting_element_shape;
+    output_tensor_shape.InsertDim(0, tensor_list->tensors.size());
     Tensor* output;
-    OP_REQUIRES_OK(c, c->allocate_output(0, resulting_shape, &output));
+    OP_REQUIRES_OK(c, c->allocate_output(0, output_tensor_shape, &output));
     if (output->NumElements() == 0) {
       return;
     }
 
     ConstMatrixVector inputs_flat;
     inputs_flat.reserve(tensor_list->tensors.size());
+    Tensor zeros;
+    BuildZerosTensor<Device, T>(c, element_dtype_, resulting_element_shape,
+                                &zeros);
     for (const auto& t : tensor_list->tensors) {
-      inputs_flat.emplace_back(new typename TTypes<T, 2>::ConstMatrix(
-          t.shaped<T, 2>({1, t.NumElements()})));
+      if (t.dtype() != DT_INVALID) {
+        inputs_flat.emplace_back(new typename TTypes<T, 2>::ConstMatrix(
+            t.shaped<T, 2>({1, t.NumElements()})));
+      } else {
+        inputs_flat.emplace_back(new typename TTypes<T, 2>::ConstMatrix(
+            const_cast<const Tensor&>(zeros).shaped<T, 2>(
+                {1, zeros.NumElements()})));
+      }
     }
     auto output_flat = output->shaped<T, 2>({1, output->NumElements()});
 
diff --git a/tensorflow/python/kernel_tests/list_ops_test.py b/tensorflow/python/kernel_tests/list_ops_test.py
index fc0e270667..ec6906f20c 100644
--- a/tensorflow/python/kernel_tests/list_ops_test.py
+++ b/tensorflow/python/kernel_tests/list_ops_test.py
@@ -189,6 +189,47 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       t = list_ops.tensor_list_stack(l, element_dtype=dtypes.float32)
       self.evaluate(t)
 
+  def _testStackWithUninitializedTensors(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=[], num_elements=3)
+    t = list_ops.tensor_list_stack(l, element_dtype=dtypes.float32)
+    self.assertAllEqual(self.evaluate(t), [0., 0., 0.])
+
+  def testStackWithUninitializedTensors(self):
+    self._testStackWithUninitializedTensors()
+
+  def testStackWithUninitializedTensorsGpu(self):
+    if not context.num_gpus():
+      return
+    with context.device("gpu:0"):
+      self._testStackWithUninitializedTensors()
+
+  def _testStackWithUninitializedTensorsInferShape(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=None, num_elements=3)
+    l = list_ops.tensor_list_set_item(l, 1, [1., 2.])
+    t = list_ops.tensor_list_stack(l, element_dtype=dtypes.float32)
+    self.assertAllEqual(self.evaluate(t), [[0., 0.], [1., 2.], [0., 0.]])
+
+  def testStackWithUninitializedTensorsInferShape(self):
+    self._testStackWithUninitializedTensorsInferShape()
+
+  def testStackWithUninitializedTensorsInferShapeGpu(self):
+    if not context.num_gpus():
+      return
+    with context.device("gpu:0"):
+      self._testStackWithUninitializedTensorsInferShape()
+
+  def testStackReservedListWithNoElementsAndPartialElementShapeFails(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=None, num_elements=3)
+    with self.assertRaisesRegexp(errors.InvalidArgumentError,
+                                 "Tried to stack list which only contains "
+                                 "uninitialized tensors and has a "
+                                 "non-fully-defined element_shape: <unknown>"):
+      t = list_ops.tensor_list_stack(l, element_dtype=dtypes.float32)
+      self.evaluate(t)
+
   @parameterized.named_parameters(("NoMaxNumElements", None),
                                   ("WithMaxNumElements", 2))
   def testGatherGrad(self, max_num_elements):
diff --git a/tensorflow/python/kernel_tests/tensor_array_ops_test.py b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
index 3f1bd60d1a..e8af998a70 100644
--- a/tensorflow/python/kernel_tests/tensor_array_ops_test.py
+++ b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
@@ -185,8 +185,8 @@ class TensorArrayTest(test.TestCase):
       self.assertAllEqual([[0.0, 0.0], [4.0, 5.0], [0.0, 0.0]],
                           self.evaluate(ta.write(1, [[4.0, 5.0]]).concat()))
 
-  @test_util.disable_control_flow_v2("b/118890905")
-  @test_util.run_v1_only("b/118890905")
+  @test_util.disable_control_flow_v2("b/122324791")
+  @test_util.run_v1_only("b/122324791")
   def testTensorArrayReadOrPackNotAllValuesAvailableFillsZeros(self):
     self._testTensorArrayReadOrPackNotAllValuesAvailableFillsZeros()
 
@@ -202,8 +202,8 @@ class TensorArrayTest(test.TestCase):
     self.assertAllEqual([[0.0, 0.0], [4.0, 5.0], [0.0, 0.0]],
                         self.evaluate(ta.write(1, [[4.0, 5.0]]).concat()))
 
-  @test_util.disable_control_flow_v2("b/118890905")
-  @test_util.run_v1_only("b/118890905")
+  @test_util.disable_control_flow_v2("b/122324791")
+  @test_util.run_v1_only("b/122324791")
   def testTensorArrayReadOrPackNotAllValuesAvailableInferShapeFillsZeros(self):
     self._testTensorArrayReadOrPackNotAllValuesAvailableInferShapeFillsZeros()
 
@@ -1321,8 +1321,8 @@ class TensorArrayTest(test.TestCase):
     with self.cached_session(use_gpu=True):
       ta = tensor_array_ops.TensorArray(
           dtype=dtypes.float32, size=0, dynamic_size=False, infer_shape=False)
-      v2_msg = ("Tried to stack elements of a empty list with "
-                "non-fully-defined shape")
+      v2_msg = ("Tried to stack elements of a empty list with non-fully-defined"
+                " element_shape")
       v1_msg = (
           "TensorArray has size zero, but element shape <unknown> is not "
           "fully defined. Currently only static shapes are supported when "
@@ -1385,7 +1385,6 @@ class TensorArrayTest(test.TestCase):
       self.assertAllEqual([10.0, -10.0], read_vals[1])
       self.assertAllEqual([[2.0, 3.0], [4.0, 5.0]], grad_vals[0])
 
-  @test_util.disable_control_flow_v2("b/118890905")
   @test_util.run_v1_only("b/118890905")
   def testTensorArrayWriteGatherAndGradients(self):
     with self.session(use_gpu=True) as session:
-- 
GitLab


From 9c091ca7134fc51f49ea930aac7a48bbdf84e7dc Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Tue, 8 Jan 2019 11:15:25 -0800
Subject: [PATCH 0389/1095] Automated rollback of commit
 acfc65f0110e272abc8bd1abce867f2f534ba263

PiperOrigin-RevId: 228364189
---
 tensorflow/compiler/tf2xla/kernels/shape_op.cc | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/shape_op.cc b/tensorflow/compiler/tf2xla/kernels/shape_op.cc
index 85b0367f73..12830816ec 100644
--- a/tensorflow/compiler/tf2xla/kernels/shape_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/shape_op.cc
@@ -20,7 +20,6 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_helpers.h"
 #include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
-#include "tensorflow/compiler/xla/client/lib/constants.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/core/framework/kernel_def_builder.h"
 #include "tensorflow/core/framework/tensor_shape.h"
@@ -92,20 +91,14 @@ class SizeOp : public XlaOpKernel {
 
   void Compile(XlaOpKernelContext* ctx) override {
     const TensorShape input_shape = ctx->InputShape(0);
-    OP_REQUIRES(ctx,
-                FastBoundsCheck(input_shape.num_elements(),
-                                std::numeric_limits<int32>::max()),
+    const int64 size = input_shape.num_elements();
+    OP_REQUIRES(ctx, FastBoundsCheck(size, std::numeric_limits<int32>::max()),
                 errors::InvalidArgument("Size does not work for tensors > "
                                         "int32 max."));
     Tensor size_constant(DT_INT32, TensorShape({}));
-    const int rank = input_shape.dims();
-    xla::XlaBuilder* builder = ctx->builder();
-    auto size = xla::One(builder, xla::U32);
-    for (int64 i = 0; i < rank; ++i) {
-      size = xla::Mul(size, xla::GetDimensionSize(ctx->Input(0), i));
-    }
-    size = xla::ConvertElementType(size, xla::S32);
-    ctx->SetOutput(0, size);
+    size_constant.scalar<int32>()() = static_cast<int32>(size);
+
+    ctx->SetConstantOutput(0, size_constant);
   }
 };
 
-- 
GitLab


From bdeda4c087a3515578e6287c647bc00fe5f35491 Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Tue, 8 Jan 2019 11:27:47 -0800
Subject: [PATCH 0390/1095] Internal Cleanup.

PiperOrigin-RevId: 228366686
---
 tensorflow/python/keras/BUILD              |   2 +-
 tensorflow/python/keras/optimizers_test.py | 157 +++++++++++----------
 2 files changed, 81 insertions(+), 78 deletions(-)

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 3ffff61c2b..9f8da126bf 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -377,7 +377,7 @@ tf_py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:training",
     ],
-    shard_count = 2,
+    shard_count = 4,
     tags = ["notsan"],
 )
 
diff --git a/tensorflow/python/keras/optimizers_test.py b/tensorflow/python/keras/optimizers_test.py
index 18a20567ce..de24bf0536 100644
--- a/tensorflow/python/keras/optimizers_test.py
+++ b/tensorflow/python/keras/optimizers_test.py
@@ -44,116 +44,119 @@ def _get_model(input_dim, num_hidden, output_dim):
   return model
 
 
-def _test_optimizer(optimizer, target=0.75):
-  np.random.seed(1337)
-  (x_train, y_train), _ = testing_utils.get_test_data(train_samples=1000,
-                                                      test_samples=200,
-                                                      input_shape=(10,),
-                                                      num_classes=2)
-  y_train = keras.utils.to_categorical(y_train)
-  model = _get_model(x_train.shape[1], 20, y_train.shape[1])
-  model.compile(loss='categorical_crossentropy',
-                optimizer=optimizer,
-                metrics=['accuracy'])
-  np.testing.assert_equal(keras.backend.get_value(model.optimizer.iterations),
-                          0)
-  history = model.fit(x_train, y_train, epochs=2, batch_size=16, verbose=0)
-  np.testing.assert_equal(keras.backend.get_value(model.optimizer.iterations),
-                          126)  # 63 steps per epoch
-  assert history.history['acc'][-1] >= target
-  config = keras.optimizers.serialize(optimizer)
-  optim = keras.optimizers.deserialize(config)
-  new_config = keras.optimizers.serialize(optim)
-  new_config['class_name'] = new_config['class_name'].lower()
-  new_config['config'].pop('name', None)
-  if 'amsgrad' not in config['config']:
-    new_config['config'].pop('amsgrad', None)
-  if 'decay' in new_config['config'] and 'schedule_decay' in config['config']:
-    new_config['config']['schedule_decay'] = new_config['config'].pop('decay')
-  if 'momentum' not in config['config']:
-    new_config['config'].pop('momentum', None)
-  if 'centered' not in config['config']:
-    new_config['config'].pop('centered', None)
-  assert config == new_config
-
-  # Test constraints.
-  model = keras.models.Sequential()
-  dense = keras.layers.Dense(10,
-                             input_shape=(x_train.shape[1],),
-                             kernel_constraint=lambda x: 0. * x + 1.,
-                             bias_constraint=lambda x: 0. * x + 2.,
-                             activation='relu')
-  model.add(dense)
-  model.add(keras.layers.Dense(y_train.shape[1], activation='softmax'))
-  model.compile(loss='categorical_crossentropy',
-                optimizer=optimizer,
-                metrics=['accuracy'])
-  np.testing.assert_equal(keras.backend.get_value(model.optimizer.iterations),
-                          126)  # Using same optimizer from before
-  model.train_on_batch(x_train[:10], y_train[:10])
-  np.testing.assert_equal(keras.backend.get_value(model.optimizer.iterations),
-                          127)
-  kernel, bias = dense.get_weights()
-  np.testing.assert_allclose(kernel, 1., atol=1e-3)
-  np.testing.assert_allclose(bias, 2., atol=1e-3)
-
-
 class KerasOptimizersTest(test.TestCase):
 
+  def _test_optimizer(self, optimizer, target=0.75):
+    np.random.seed(1337)
+    (x_train, y_train), _ = testing_utils.get_test_data(
+        train_samples=1000, test_samples=200, input_shape=(10,), num_classes=2)
+    y_train = keras.utils.to_categorical(y_train)
+    model = _get_model(x_train.shape[1], 20, y_train.shape[1])
+    model.compile(
+        loss='categorical_crossentropy',
+        optimizer=optimizer,
+        metrics=['accuracy'])
+    np.testing.assert_equal(
+        keras.backend.get_value(model.optimizer.iterations), 0)
+    history = model.fit(x_train, y_train, epochs=2, batch_size=16, verbose=0)
+    np.testing.assert_equal(
+        keras.backend.get_value(model.optimizer.iterations),
+        126)  # 63 steps per epoch
+    assert history.history['acc'][-1] >= target
+    config = keras.optimizers.serialize(optimizer)
+    optim = keras.optimizers.deserialize(config)
+    new_config = keras.optimizers.serialize(optim)
+    new_config['class_name'] = new_config['class_name'].lower()
+    new_config['config'].pop('name', None)
+    if 'amsgrad' not in config['config']:
+      new_config['config'].pop('amsgrad', None)
+    if 'decay' in new_config['config'] and 'schedule_decay' in config['config']:
+      new_config['config']['schedule_decay'] = new_config['config'].pop('decay')
+    if 'momentum' not in config['config']:
+      new_config['config'].pop('momentum', None)
+    if 'centered' not in config['config']:
+      new_config['config'].pop('centered', None)
+    self.assertDictEqual(config, new_config)
+
+    # Test constraints.
+    model = keras.models.Sequential()
+    dense = keras.layers.Dense(
+        10,
+        input_shape=(x_train.shape[1],),
+        kernel_constraint=lambda x: 0. * x + 1.,
+        bias_constraint=lambda x: 0. * x + 2.,
+        activation='relu')
+    model.add(dense)
+    model.add(keras.layers.Dense(y_train.shape[1], activation='softmax'))
+    model.compile(
+        loss='categorical_crossentropy',
+        optimizer=optimizer,
+        metrics=['accuracy'])
+    np.testing.assert_equal(
+        keras.backend.get_value(model.optimizer.iterations),
+        126)  # Using same optimizer from before
+    model.train_on_batch(x_train[:10], y_train[:10])
+    np.testing.assert_equal(
+        keras.backend.get_value(model.optimizer.iterations), 127)
+    kernel, bias = dense.get_weights()
+    np.testing.assert_allclose(kernel, 1., atol=1e-3)
+    np.testing.assert_allclose(bias, 2., atol=1e-3)
+
   def test_sgd(self):
     with self.cached_session():
-      _test_optimizer(keras.optimizers.SGD(lr=0.01,
-                                           momentum=0.9,
-                                           nesterov=True))
+      self._test_optimizer(keras.optimizers.SGD())
+
+  def test_momentum(self):
+    with self.cached_session():
+      self._test_optimizer(
+          keras.optimizers.SGD(lr=0.01, momentum=0.9, nesterov=True))
 
   def test_rmsprop(self):
     with self.cached_session():
-      _test_optimizer(keras.optimizers.RMSprop())
-      _test_optimizer(keras.optimizers.RMSprop(decay=1e-3))
+      self._test_optimizer(keras.optimizers.RMSprop())
+      self._test_optimizer(keras.optimizers.RMSprop(decay=1e-3))
 
   def test_adagrad(self):
     with self.cached_session():
-      _test_optimizer(keras.optimizers.Adagrad())
-      _test_optimizer(keras.optimizers.Adagrad(decay=1e-3))
+      self._test_optimizer(keras.optimizers.Adagrad())
+      self._test_optimizer(keras.optimizers.Adagrad(decay=1e-3))
 
   def test_adadelta(self):
     with self.cached_session():
-      _test_optimizer(keras.optimizers.Adadelta(), target=0.6)
+      self._test_optimizer(keras.optimizers.Adadelta(), target=0.6)
       # Accuracy seems dependent on the initialization. Even adding tf.Print
       # nodes in the graph seemed to affect the initialization seed, and hence
       # the accuracy.
-      _test_optimizer(keras.optimizers.Adadelta(decay=1e-3), target=0.4)
+      self._test_optimizer(keras.optimizers.Adadelta(decay=1e-3), target=0.4)
 
   def test_adam(self):
     with self.cached_session():
-      _test_optimizer(keras.optimizers.Adam())
+      self._test_optimizer(keras.optimizers.Adam())
       # Accuracy seems dependent on the seed initialization.
       # TODO(b/121051441): fix test flakiness.
-      _test_optimizer(keras.optimizers.Adam(decay=1e-3), target=0.73)
-      _test_optimizer(keras.optimizers.Adam(amsgrad=True))
+      self._test_optimizer(keras.optimizers.Adam(decay=1e-3), target=0.73)
+      self._test_optimizer(keras.optimizers.Adam(amsgrad=True))
 
   def test_adamax(self):
     with self.cached_session():
-      _test_optimizer(keras.optimizers.Adamax())
-      _test_optimizer(keras.optimizers.Adamax(decay=1e-3))
+      self._test_optimizer(keras.optimizers.Adamax())
+      self._test_optimizer(keras.optimizers.Adamax(decay=1e-3))
 
   def test_nadam(self):
     with self.cached_session():
-      _test_optimizer(keras.optimizers.Nadam())
+      self._test_optimizer(keras.optimizers.Nadam())
 
   def test_clipnorm(self):
     with self.cached_session():
-      _test_optimizer(keras.optimizers.SGD(lr=0.01,
-                                           momentum=0.9,
-                                           clipnorm=0.5))
+      self._test_optimizer(
+          keras.optimizers.SGD(lr=0.01, momentum=0.9, clipnorm=0.5))
 
   def test_clipvalue(self):
     with self.cached_session():
-      _test_optimizer(keras.optimizers.SGD(lr=0.01,
-                                           momentum=0.9,
-                                           clipvalue=0.5))
+      self._test_optimizer(
+          keras.optimizers.SGD(lr=0.01, momentum=0.9, clipvalue=0.5))
 
-  def test_tfoptimizer(self):
+  def test_tf_optimizer(self):
     optimizer = keras.optimizers.TFOptimizer(AdamOptimizer(0.01))
     model = keras.models.Sequential()
     model.add(keras.layers.Dense(
@@ -188,7 +191,7 @@ class KerasOptimizersTest(test.TestCase):
     self.assertIs(optimizer_weak(), None)
 
   @test_util.run_in_graph_and_eager_modes
-  def test_tfoptimizer_iterations(self):
+  def test_tf_optimizer_iterations(self):
     with self.cached_session():
       optimizer = keras.optimizers.TFOptimizer(AdamOptimizer(0.01))
       model = keras.models.Sequential()
-- 
GitLab


From 35279e5c61d3ff5882c31d3d291256321ba27267 Mon Sep 17 00:00:00 2001
From: Doe Hyun Yoon <dyoon@google.com>
Date: Tue, 8 Jan 2019 11:37:17 -0800
Subject: [PATCH 0391/1095] Replace the references to PredictCosts() with
 PredictCostsAndReturnRunMetadata().

PiperOrigin-RevId: 228368607
---
 .../core/grappler/costs/analytical_cost_estimator_test.cc | 5 +++--
 tensorflow/python/grappler/cost_analyzer.cc               | 8 ++++++--
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/grappler/costs/analytical_cost_estimator_test.cc b/tensorflow/core/grappler/costs/analytical_cost_estimator_test.cc
index a9a1abfa98..8a56344378 100644
--- a/tensorflow/core/grappler/costs/analytical_cost_estimator_test.cc
+++ b/tensorflow/core/grappler/costs/analytical_cost_estimator_test.cc
@@ -98,9 +98,10 @@ TEST_F(AnalyticalCostEstimatorTest, SimpleTest) {
   AnalyticalCostEstimator estimator(cluster_.get(), true);
   TF_ASSERT_OK(estimator.Initialize(item));
 
-  CostGraphDef cost_graph;
+  RunMetadata run_metadata;
   Costs summary;
-  TF_ASSERT_OK(estimator.PredictCosts(item.graph, &cost_graph, &summary));
+  TF_ASSERT_OK(estimator.PredictCostsAndReturnRunMetadata(
+      item.graph, &run_metadata, &summary));
 
   EXPECT_EQ(Costs::NanoSeconds(9151), summary.execution_time);
   // Note there are totally 17 nodes (RandomUniform creates 2 nodes), but
diff --git a/tensorflow/python/grappler/cost_analyzer.cc b/tensorflow/python/grappler/cost_analyzer.cc
index b474e19894..bb8c6d5b85 100644
--- a/tensorflow/python/grappler/cost_analyzer.cc
+++ b/tensorflow/python/grappler/cost_analyzer.cc
@@ -42,9 +42,13 @@ Status CostAnalyzer::GenerateReport(std::ostream& os, bool per_node_report,
 void CostAnalyzer::PredictCosts(CostEstimator* cost_estimator,
                                 CostGraphDef* cost_graph, int64* total_time) {
   TF_CHECK_OK(cost_estimator->Initialize(*item_));
+  RunMetadata run_metadata;
   Costs costs;
-  const Status status =
-      cost_estimator->PredictCosts(item_->graph, cost_graph, &costs);
+  const Status status = cost_estimator->PredictCostsAndReturnRunMetadata(
+      item_->graph, &run_metadata, &costs);
+  if (cost_graph) {
+    cost_graph->Swap(run_metadata.mutable_cost_graph());
+  }
   *total_time = costs.execution_time.count();
   if (!status.ok()) {
     LOG(ERROR) << "Could not estimate the cost for item " << item_->id << ": "
-- 
GitLab


From a849894f02e41fdeb08f86aafad51a2f02b454b1 Mon Sep 17 00:00:00 2001
From: Karim Nosir <karimnosseir@google.com>
Date: Tue, 8 Jan 2019 11:59:53 -0800
Subject: [PATCH 0392/1095] Reduce the number of tests inside unit-test and
 shard it to speed it up.

PiperOrigin-RevId: 228373086
---
 tensorflow/lite/kernels/internal/BUILD                      | 2 ++
 .../lite/kernels/internal/logsoftmax_quantized_test.cc      | 6 +++---
 tensorflow/lite/kernels/internal/softmax_quantized_test.cc  | 6 +++---
 3 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/tensorflow/lite/kernels/internal/BUILD b/tensorflow/lite/kernels/internal/BUILD
index 885f650dd5..74a11a72ef 100644
--- a/tensorflow/lite/kernels/internal/BUILD
+++ b/tensorflow/lite/kernels/internal/BUILD
@@ -641,6 +641,7 @@ cc_test(
     srcs = [
         "softmax_quantized_test.cc",
     ],
+    shard_count = 3,
     deps = [
         ":optimized_base",
         ":quantization_util",
@@ -657,6 +658,7 @@ cc_test(
     srcs = [
         "logsoftmax_quantized_test.cc",
     ],
+    shard_count = 3,
     tags = [
         # TODO(b/122242739): Reenable after fixing the flakiness?
         "nomac",
diff --git a/tensorflow/lite/kernels/internal/logsoftmax_quantized_test.cc b/tensorflow/lite/kernels/internal/logsoftmax_quantized_test.cc
index 889a726f3a..945300dad1 100644
--- a/tensorflow/lite/kernels/internal/logsoftmax_quantized_test.cc
+++ b/tensorflow/lite/kernels/internal/logsoftmax_quantized_test.cc
@@ -225,7 +225,7 @@ bool TryOneSkyscraperLogSoftmax(bool small_depth) {
 }
 
 TEST(TestQuantizedLogSoftmax, UniformLogSoftmaxTests) {
-  const int kTestsToRun = 1000;
+  const int kTestsToRun = 100;
   for (int i = 0; i < kTestsToRun; i++) {
     while (!TryOneUniformLogSoftmax()) {
     }
@@ -233,7 +233,7 @@ TEST(TestQuantizedLogSoftmax, UniformLogSoftmaxTests) {
 }
 
 TEST(TestQuantizedLogSoftmax, SkyscraperLogSoftmaxTests) {
-  const int kTestsToRun = 1000;
+  const int kTestsToRun = 100;
   for (int i = 0; i < kTestsToRun; i++) {
     while (!TryOneSkyscraperLogSoftmax(false)) {
     }
@@ -241,7 +241,7 @@ TEST(TestQuantizedLogSoftmax, SkyscraperLogSoftmaxTests) {
 }
 
 TEST(TestQuantizedLogSoftmax, SmallSkyscraperLogSoftmaxTests) {
-  const int kTestsToRun = 1000;
+  const int kTestsToRun = 100;
   for (int i = 0; i < kTestsToRun; i++) {
     while (!TryOneSkyscraperLogSoftmax(true)) {
     }
diff --git a/tensorflow/lite/kernels/internal/softmax_quantized_test.cc b/tensorflow/lite/kernels/internal/softmax_quantized_test.cc
index 743ce0355c..8ac62d9af7 100644
--- a/tensorflow/lite/kernels/internal/softmax_quantized_test.cc
+++ b/tensorflow/lite/kernels/internal/softmax_quantized_test.cc
@@ -210,7 +210,7 @@ bool TryOneSkyscraperSoftmax(bool small_depth) {
 }
 
 TEST(TestQuantizedSoftmax, UniformSoftmaxTests) {
-  const int kTestsToRun = 1000;
+  const int kTestsToRun = 100;
   for (int i = 0; i < kTestsToRun; i++) {
     while (!TryOneUniformSoftmax()) {
     }
@@ -218,7 +218,7 @@ TEST(TestQuantizedSoftmax, UniformSoftmaxTests) {
 }
 
 TEST(TestQuantizedSoftmax, SkyscraperSoftmaxTests) {
-  const int kTestsToRun = 1000;
+  const int kTestsToRun = 100;
   for (int i = 0; i < kTestsToRun; i++) {
     while (!TryOneSkyscraperSoftmax(false)) {
     }
@@ -226,7 +226,7 @@ TEST(TestQuantizedSoftmax, SkyscraperSoftmaxTests) {
 }
 
 TEST(TestQuantizedSoftmax, SmallSkyscraperSoftmaxTests) {
-  const int kTestsToRun = 1000;
+  const int kTestsToRun = 100;
   for (int i = 0; i < kTestsToRun; i++) {
     while (!TryOneSkyscraperSoftmax(true)) {
     }
-- 
GitLab


From 68727289bca144c9d1a8f618d81474e4a7322256 Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Tue, 8 Jan 2019 12:01:56 -0800
Subject: [PATCH 0393/1095] [XLA] Move ArgMin/ArgMax out of TF/XLA and into XLA
 client library. NFC intended.

PiperOrigin-RevId: 228373518
---
 .../compiler/tf2xla/kernels/categorical_op.cc |  4 +-
 .../compiler/tf2xla/kernels/index_ops.cc      |  5 +-
 .../compiler/tf2xla/kernels/index_ops_cpu.cc  |  4 +-
 tensorflow/compiler/tf2xla/xla_helpers.cc     | 67 -------------------
 tensorflow/compiler/tf2xla/xla_helpers.h      | 10 ---
 tensorflow/compiler/xla/client/lib/BUILD      | 15 +++++
 .../compiler/xla/client/lib/arithmetic.cc     | 60 +++++++++++++++++
 .../compiler/xla/client/lib/arithmetic.h      |  8 +++
 .../xla/client/lib/arithmetic_test.cc         | 67 +++++++++++++++++++
 9 files changed, 156 insertions(+), 84 deletions(-)
 create mode 100644 tensorflow/compiler/xla/client/lib/arithmetic_test.cc

diff --git a/tensorflow/compiler/tf2xla/kernels/categorical_op.cc b/tensorflow/compiler/tf2xla/kernels/categorical_op.cc
index 7199b9b6fe..c2b4c28d15 100644
--- a/tensorflow/compiler/tf2xla/kernels/categorical_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/categorical_op.cc
@@ -99,8 +99,8 @@ class CategoricalOp : public XlaOpKernel {
     xla::PrimitiveType xla_output_type;
     OP_REQUIRES_OK(ctx,
                    DataTypeToPrimitiveType(output_type(0), &xla_output_type));
-    xla::XlaOp argmax = XlaHelpers::ArgMax(softmax_entries, xla_output_type,
-                                           /*axis=*/class_dimension);
+    xla::XlaOp argmax = xla::ArgMax(softmax_entries, xla_output_type,
+                                    /*axis=*/class_dimension);
     if (num_samples == 1) {
       argmax = xla::Reshape(argmax, {batch_size, 1});
     }
diff --git a/tensorflow/compiler/tf2xla/kernels/index_ops.cc b/tensorflow/compiler/tf2xla/kernels/index_ops.cc
index 843b6bb4e6..978e9480ea 100644
--- a/tensorflow/compiler/tf2xla/kernels/index_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/index_ops.cc
@@ -18,7 +18,6 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/kernels/index_ops.h"
 
 #include "tensorflow/compiler/tf2xla/type_util.h"
-#include "tensorflow/compiler/tf2xla/xla_helpers.h"
 #include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/client/lib/arithmetic.h"
@@ -66,9 +65,9 @@ void XlaArgMinMaxOp::Compile(XlaOpKernelContext* ctx) {
   xla::XlaOp input = ctx->Input(0);
   xla::XlaOp output;
   if (is_min_) {
-    output = XlaHelpers::ArgMin(input, index_xla_type, axis);
+    output = xla::ArgMin(input, index_xla_type, axis);
   } else {
-    output = XlaHelpers::ArgMax(input, index_xla_type, axis);
+    output = xla::ArgMax(input, index_xla_type, axis);
   }
 
   ctx->SetOutput(0, output);
diff --git a/tensorflow/compiler/tf2xla/kernels/index_ops_cpu.cc b/tensorflow/compiler/tf2xla/kernels/index_ops_cpu.cc
index 3e7e8eae6e..30b993045c 100644
--- a/tensorflow/compiler/tf2xla/kernels/index_ops_cpu.cc
+++ b/tensorflow/compiler/tf2xla/kernels/index_ops_cpu.cc
@@ -16,9 +16,9 @@ limitations under the License.
 // Native XLA implementations of indexing ops.
 
 #include "tensorflow/compiler/tf2xla/type_util.h"
-#include "tensorflow/compiler/tf2xla/xla_helpers.h"
 #include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
+#include "tensorflow/compiler/xla/client/lib/arithmetic.h"
 #include "tensorflow/compiler/xla/literal_util.h"
 #include "tensorflow/core/framework/kernel_def_builder.h"
 #include "tensorflow/core/framework/op_kernel.h"
@@ -74,7 +74,7 @@ class ArgMaxCustomCallOp : public XlaOpKernel {
     // shape isn't supported.
     if (!ctx->compiler()->options().allow_cpu_custom_calls ||
         (input_dims != 1 && input_dims != 2)) {
-      xla::XlaOp output = XlaHelpers::ArgMax(ctx->Input(0), output_type, axis);
+      xla::XlaOp output = xla::ArgMax(ctx->Input(0), output_type, axis);
       ctx->SetOutput(0, output);
       return;
     }
diff --git a/tensorflow/compiler/tf2xla/xla_helpers.cc b/tensorflow/compiler/tf2xla/xla_helpers.cc
index 00035d24b7..04a5d93406 100644
--- a/tensorflow/compiler/tf2xla/xla_helpers.cc
+++ b/tensorflow/compiler/tf2xla/xla_helpers.cc
@@ -34,63 +34,6 @@ limitations under the License.
 
 namespace tensorflow {
 
-namespace {
-
-xla::XlaOp ArgMinMax(xla::XlaOp input, xla::PrimitiveType output_type, int axis,
-                     bool is_min) {
-  xla::XlaBuilder* builder = input.builder();
-  return builder->ReportErrorOrReturn([&]() -> xla::StatusOr<xla::XlaOp> {
-    TF_ASSIGN_OR_RETURN(xla::Shape input_shape, builder->GetShape(input));
-    xla::XlaOp init_value;
-    xla::XlaComputation reducer;
-    if (is_min) {
-      init_value = xla::MaxValue(builder, input_shape.element_type());
-      reducer =
-          xla::CreateScalarMinComputation(input_shape.element_type(), builder);
-    } else {
-      init_value = xla::MinValue(builder, input_shape.element_type());
-      reducer =
-          xla::CreateScalarMaxComputation(input_shape.element_type(), builder);
-    }
-
-    xla::XlaOp input_max = xla::Reduce(input, init_value, reducer,
-                                       /*dimensions_to_reduce=*/{axis});
-    std::vector<int64> broadcast_dims(input_shape.rank() - 1);
-    std::iota(broadcast_dims.begin(), broadcast_dims.begin() + axis, 0);
-    std::iota(broadcast_dims.begin() + axis, broadcast_dims.end(), axis + 1);
-    // Compute a mask that has 1s for elements equal to the maximum.
-    xla::XlaOp partial_mask = xla::ConvertElementType(
-        xla::Eq(input, input_max, broadcast_dims), output_type);
-
-    // In order to make identity elements for a bitwise And, we:
-    //   Left shift the 1 to the leftmost bit, yielding 0x10...0
-    //   Arithmetic right shift the 1 back to the rightmost bit, yielding
-    //   0xFF...F
-    int32 bits_in_type =
-        xla::ShapeUtil::ByteSizeOfPrimitiveType(output_type) * 8 - 1;
-    xla::XlaOp shift_amount =
-        xla::ConstantR0WithType(builder, output_type, bits_in_type);
-    xla::XlaOp full_mask = xla::ShiftRightArithmetic(
-        xla::ShiftLeft(partial_mask, shift_amount), shift_amount);
-
-    // And with the vector [0, 1, 2, ...] to convert each 0xFF...F into its
-    // index.
-
-    const int64 axis_size = xla::ShapeUtil::GetDimension(input_shape, axis);
-    xla::XlaOp iota = xla::Iota(builder, output_type, axis_size);
-    xla::XlaOp product =
-        xla::And(full_mask, iota, /*broadcast_dimensions=*/{axis});
-
-    // If there are multiple maximum elements, choose the one with the highest
-    // index.
-    return xla::Reduce(product, xla::MinValue(builder, output_type),
-                       xla::CreateScalarMaxComputation(output_type, builder),
-                       /*dimensions_to_reduce=*/{axis});
-  });
-}
-
-}  // namespace
-
 xla::XlaOp XlaHelpers::Zero(xla::XlaBuilder* b, DataType data_type) {
   xla::PrimitiveType type;
   TF_CHECK_OK(DataTypeToPrimitiveType(data_type, &type));
@@ -148,16 +91,6 @@ static Tensor MakeLinspaceTensor(const TensorShape& shape, int64 depth) {
   return linspace;
 }
 
-xla::XlaOp XlaHelpers::ArgMax(xla::XlaOp input, xla::PrimitiveType output_type,
-                              int axis) {
-  return ArgMinMax(input, output_type, axis, /*is_min=*/false);
-}
-
-xla::XlaOp XlaHelpers::ArgMin(xla::XlaOp input, xla::PrimitiveType output_type,
-                              int axis) {
-  return ArgMinMax(input, output_type, axis, /*is_min=*/true);
-}
-
 Status XlaHelpers::OneHot(xla::XlaBuilder* builder, int64 depth, int axis,
                           DataType index_type, const TensorShape& indices_shape,
                           const xla::XlaOp& indices, const xla::XlaOp& on_value,
diff --git a/tensorflow/compiler/tf2xla/xla_helpers.h b/tensorflow/compiler/tf2xla/xla_helpers.h
index 4858dfee55..490923526b 100644
--- a/tensorflow/compiler/tf2xla/xla_helpers.h
+++ b/tensorflow/compiler/tf2xla/xla_helpers.h
@@ -53,16 +53,6 @@ class XlaHelpers {
                                absl::Span<const int64> shape,
                                xla::Literal* output);
 
-  // Returns the argmax of `input` along `axis`. `output_type` is the type to
-  // use for the output.
-  static xla::XlaOp ArgMax(xla::XlaOp input, xla::PrimitiveType output_type,
-                           int axis);
-
-  // Returns the argmin of `input` along `axis`. `output_type` is the type to
-  // use for the output.
-  static xla::XlaOp ArgMin(xla::XlaOp input, xla::PrimitiveType output_type,
-                           int axis);
-
   // Converts `indices` into a one-hot representation. `depth` is the size
   // of the new axis to add. `axis` is the position at which to add the new
   // axis. `indices_shape` is the shape of `indices`. `on_value` and
diff --git a/tensorflow/compiler/xla/client/lib/BUILD b/tensorflow/compiler/xla/client/lib/BUILD
index d863ebfea1..df1ee330f1 100644
--- a/tensorflow/compiler/xla/client/lib/BUILD
+++ b/tensorflow/compiler/xla/client/lib/BUILD
@@ -34,6 +34,21 @@ cc_library(
     ],
 )
 
+xla_test(
+    name = "arithmetic_test",
+    srcs = ["arithmetic_test.cc"],
+    deps = [
+        ":arithmetic",
+        "//tensorflow/compiler/xla:literal_util",
+        "//tensorflow/compiler/xla:test",
+        "//tensorflow/compiler/xla:types",
+        "//tensorflow/compiler/xla:xla_data_proto",
+        "//tensorflow/compiler/xla/client:xla_builder",
+        "//tensorflow/compiler/xla/tests:client_library_test_base",
+        "//tensorflow/compiler/xla/tests:xla_internal_test_main",
+    ],
+)
+
 cc_library(
     name = "cholesky",
     srcs = ["cholesky.cc"],
diff --git a/tensorflow/compiler/xla/client/lib/arithmetic.cc b/tensorflow/compiler/xla/client/lib/arithmetic.cc
index 33ff3971d7..3b875135af 100644
--- a/tensorflow/compiler/xla/client/lib/arithmetic.cc
+++ b/tensorflow/compiler/xla/client/lib/arithmetic.cc
@@ -123,4 +123,64 @@ XlaOp Any(XlaOp predicates) {
   });
 }
 
+namespace {
+
+XlaOp ArgMinMax(XlaOp input, PrimitiveType output_type, int axis, bool is_min) {
+  XlaBuilder* builder = input.builder();
+  return builder->ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    TF_ASSIGN_OR_RETURN(Shape input_shape, builder->GetShape(input));
+    XlaOp init_value;
+    XlaComputation reducer;
+    if (is_min) {
+      init_value = MaxValue(builder, input_shape.element_type());
+      reducer = CreateScalarMinComputation(input_shape.element_type(), builder);
+    } else {
+      init_value = MinValue(builder, input_shape.element_type());
+      reducer = CreateScalarMaxComputation(input_shape.element_type(), builder);
+    }
+
+    XlaOp input_max = Reduce(input, init_value, reducer,
+                             /*dimensions_to_reduce=*/{axis});
+    std::vector<int64> broadcast_dims(input_shape.rank() - 1);
+    std::iota(broadcast_dims.begin(), broadcast_dims.begin() + axis, 0);
+    std::iota(broadcast_dims.begin() + axis, broadcast_dims.end(), axis + 1);
+    // Compute a mask that has 1s for elements equal to the maximum.
+    XlaOp partial_mask =
+        ConvertElementType(Eq(input, input_max, broadcast_dims), output_type);
+
+    // In order to make identity elements for a bitwise And, we:
+    //   Left shift the 1 to the leftmost bit, yielding 0x10...0
+    //   Arithmetic right shift the 1 back to the rightmost bit, yielding
+    //   0xFF...F
+    int32 bits_in_type =
+        ShapeUtil::ByteSizeOfPrimitiveType(output_type) * 8 - 1;
+    XlaOp shift_amount = ConstantR0WithType(builder, output_type, bits_in_type);
+    XlaOp full_mask = ShiftRightArithmetic(
+        ShiftLeft(partial_mask, shift_amount), shift_amount);
+
+    // And with the vector [0, 1, 2, ...] to convert each 0xFF...F into its
+    // index.
+
+    const int64 axis_size = ShapeUtil::GetDimension(input_shape, axis);
+    XlaOp iota = Iota(builder, output_type, axis_size);
+    XlaOp product = And(full_mask, iota, /*broadcast_dimensions=*/{axis});
+
+    // If there are multiple maximum elements, choose the one with the highest
+    // index.
+    return Reduce(product, MinValue(builder, output_type),
+                  CreateScalarMaxComputation(output_type, builder),
+                  /*dimensions_to_reduce=*/{axis});
+  });
+}
+
+}  // namespace
+
+XlaOp ArgMax(XlaOp input, PrimitiveType output_type, int axis) {
+  return ArgMinMax(input, output_type, axis, /*is_min=*/false);
+}
+
+XlaOp ArgMin(XlaOp input, PrimitiveType output_type, int axis) {
+  return ArgMinMax(input, output_type, axis, /*is_min=*/true);
+}
+
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/client/lib/arithmetic.h b/tensorflow/compiler/xla/client/lib/arithmetic.h
index 632e8cc8bc..d4a7812c44 100644
--- a/tensorflow/compiler/xla/client/lib/arithmetic.h
+++ b/tensorflow/compiler/xla/client/lib/arithmetic.h
@@ -57,6 +57,14 @@ XlaComputation CreateScalarOrComputation(PrimitiveType type,
 // Note: if predicates is zero-sized, Any() vacuously returns false.
 XlaOp Any(XlaOp predicates);
 
+// Returns the argmax of `input` along `axis`. `output_type` is the type to
+// use for the output.
+XlaOp ArgMax(XlaOp input, PrimitiveType output_type, int axis);
+
+// Returns the argmin of `input` along `axis`. `output_type` is the type to
+// use for the output.
+XlaOp ArgMin(XlaOp input, PrimitiveType output_type, int axis);
+
 }  // namespace xla
 
 #endif  // TENSORFLOW_COMPILER_XLA_CLIENT_LIB_ARITHMETIC_H_
diff --git a/tensorflow/compiler/xla/client/lib/arithmetic_test.cc b/tensorflow/compiler/xla/client/lib/arithmetic_test.cc
new file mode 100644
index 0000000000..a13839f9db
--- /dev/null
+++ b/tensorflow/compiler/xla/client/lib/arithmetic_test.cc
@@ -0,0 +1,67 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/client/lib/arithmetic.h"
+#include "tensorflow/compiler/xla/client/xla_builder.h"
+#include "tensorflow/compiler/xla/literal_util.h"
+#include "tensorflow/compiler/xla/test.h"
+#include "tensorflow/compiler/xla/tests/client_library_test_base.h"
+#include "tensorflow/compiler/xla/tests/test_macros.h"
+#include "tensorflow/compiler/xla/types.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
+
+namespace xla {
+namespace {
+
+using ArithmeticTest = ClientLibraryTestBase;
+
+XLA_TEST_F(ArithmeticTest, ArgMinR2Axis0) {
+  XlaBuilder builder(TestName());
+  auto x = ConstantR2<int32>(&builder, {{1, 7, 4}, {6, 3, 5}, {8, 3, 3}});
+  ArgMin(x, S32, /*axis=*/0);
+
+  std::vector<int32> expected = {0, 2, 2};
+  ComputeAndCompareR1<int32>(&builder, expected, {});
+}
+
+XLA_TEST_F(ArithmeticTest, ArgMinR2Axis1) {
+  XlaBuilder builder(TestName());
+  auto x = ConstantR2<int32>(&builder, {{1, 7, 4}, {6, 3, 5}, {8, 3, 3}});
+  ArgMin(x, S32, /*axis=*/1);
+
+  std::vector<int32> expected = {0, 1, 2};
+  ComputeAndCompareR1<int32>(&builder, expected, {});
+}
+
+XLA_TEST_F(ArithmeticTest, ArgMaxR2Axis0) {
+  XlaBuilder builder(TestName());
+  auto x = ConstantR2<int32>(&builder, {{1, 7, 4}, {6, 3, 5}, {8, 3, 3}});
+  ArgMax(x, S32, /*axis=*/0);
+
+  std::vector<int32> expected = {2, 0, 1};
+  ComputeAndCompareR1<int32>(&builder, expected, {});
+}
+
+XLA_TEST_F(ArithmeticTest, ArgMaxR2Axis1) {
+  XlaBuilder builder(TestName());
+  auto x = ConstantR2<int32>(&builder, {{1, 7, 4}, {6, 3, 5}, {8, 3, 3}});
+  ArgMax(x, S32, /*axis=*/1);
+
+  std::vector<int32> expected = {1, 0, 0};
+  ComputeAndCompareR1<int32>(&builder, expected, {});
+}
+
+}  // namespace
+}  // namespace xla
-- 
GitLab


From 989c8b40a9b3b7bd45aedef43315675b0d4baf8d Mon Sep 17 00:00:00 2001
From: Adam Roberts <adarob@google.com>
Date: Tue, 8 Jan 2019 12:03:30 -0800
Subject: [PATCH 0394/1095] Disable writing examples/sec and global_step/sec
 summaries on TPU if save_summary_steps is Falsey.

PiperOrigin-RevId: 228373932
---
 tensorflow/contrib/tpu/python/tpu/tpu_estimator.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 98babbb543..2317300041 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -2654,7 +2654,7 @@ class TPUEstimator(estimator_lib.Estimator):
         if self._log_every_n_steps is not None:
           examples_hook = ExamplesPerSecondHook(
               ctx.global_batch_size,
-              output_dir=self.model_dir,
+              output_dir=self.model_dir if config.save_summary_steps else None,
               every_n_steps=self._log_every_n_steps)
 
         if ctx.is_running_on_cpu(is_export_mode=is_export_mode):
-- 
GitLab


From 67d8a0330b5bacc0bb45675b1e3f211bd930ec7f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 8 Jan 2019 12:18:59 -0800
Subject: [PATCH 0395/1095] Fixing flaky test in quantile regression

PiperOrigin-RevId: 228376536
---
 .../estimator_batch/estimator_test.py            | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/estimator_test.py b/tensorflow/contrib/boosted_trees/estimator_batch/estimator_test.py
index ee052ac603..47d910d42a 100644
--- a/tensorflow/contrib/boosted_trees/estimator_batch/estimator_test.py
+++ b/tensorflow/contrib/boosted_trees/estimator_batch/estimator_test.py
@@ -487,8 +487,8 @@ class BoostedTreeEstimatorTest(test_util.TensorFlowTestCase):
     self.assertTrue(frac_below_upper_0 <= 0.98)
     self.assertTrue(frac_below_upper_1 >= 0.92)
     self.assertTrue(frac_below_upper_1 <= 0.98)
-    self.assertTrue(frac_both_below_upper >= 0.92)
-    self.assertTrue(frac_both_below_upper <= 0.98)
+    self.assertTrue(frac_both_below_upper >= 0.91)
+    self.assertTrue(frac_both_below_upper <= 0.99)
 
     train_input_fn, test_input_fn, _ = _quantile_regression_input_fns(
         two_dimension=True)
@@ -516,8 +516,8 @@ class BoostedTreeEstimatorTest(test_util.TensorFlowTestCase):
     self.assertTrue(frac_above_lower_0 <= 0.98)
     self.assertTrue(frac_above_lower_1 >= 0.92)
     self.assertTrue(frac_above_lower_1 <= 0.98)
-    self.assertTrue(frac_both_above_lower >= 0.92)
-    self.assertTrue(frac_both_above_lower <= 0.98)
+    self.assertTrue(frac_both_above_lower >= 0.91)
+    self.assertTrue(frac_both_above_lower <= 0.99)
 
 
 class CoreGradientBoostedDecisionTreeEstimators(test_util.TensorFlowTestCase):
@@ -806,8 +806,8 @@ class CoreGradientBoostedDecisionTreeEstimators(test_util.TensorFlowTestCase):
     self.assertTrue(frac_below_upper_0 <= 0.98)
     self.assertTrue(frac_below_upper_1 >= 0.92)
     self.assertTrue(frac_below_upper_1 <= 0.98)
-    self.assertTrue(frac_both_below_upper >= 0.92)
-    self.assertTrue(frac_both_below_upper <= 0.98)
+    self.assertTrue(frac_both_below_upper >= 0.91)
+    self.assertTrue(frac_both_below_upper <= 0.99)
 
     train_input_fn, test_input_fn, _ = _quantile_regression_input_fns(
         two_dimension=True)
@@ -835,8 +835,8 @@ class CoreGradientBoostedDecisionTreeEstimators(test_util.TensorFlowTestCase):
     self.assertTrue(frac_above_lower_0 <= 0.98)
     self.assertTrue(frac_above_lower_1 >= 0.92)
     self.assertTrue(frac_above_lower_1 <= 0.98)
-    self.assertTrue(frac_both_above_lower >= 0.92)
-    self.assertTrue(frac_both_above_lower <= 0.98)
+    self.assertTrue(frac_both_above_lower >= 0.91)
+    self.assertTrue(frac_both_above_lower <= 0.99)
 
 
 if __name__ == "__main__":
-- 
GitLab


From a3d639a5a46df8f3523bd514cade1faaf602b66d Mon Sep 17 00:00:00 2001
From: Blake Hechtman <blakehechtman@google.com>
Date: Tue, 8 Jan 2019 12:22:54 -0800
Subject: [PATCH 0396/1095] [XLA] Canonicalize dot dimension numbers on CPU and
 GPU backends cases that previously failed shape inference.

PiperOrigin-RevId: 228377214
---
 tensorflow/compiler/xla/service/BUILD         |   3 +-
 .../compiler/xla/service/dot_decomposer.cc    | 174 +++++++++++++++++-
 tensorflow/compiler/xla/service/gpu/BUILD     |   1 +
 .../xla/service/gpu/nvptx_compiler.cc         |   2 +
 tensorflow/compiler/xla/tests/BUILD           |   2 +
 .../compiler/xla/tests/dot_operation_test.cc  |  34 ++++
 6 files changed, 208 insertions(+), 8 deletions(-)

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index ec63ae1ac6..728adb67a0 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -1873,8 +1873,9 @@ cc_library(
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:status_macros",
         "//tensorflow/compiler/xla:types",
-        "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/core:lib",
+        "@com_google_absl//absl/algorithm:container",
+        "@com_google_absl//absl/strings",
     ],
 )
 
diff --git a/tensorflow/compiler/xla/service/dot_decomposer.cc b/tensorflow/compiler/xla/service/dot_decomposer.cc
index b2ba261790..855424067d 100644
--- a/tensorflow/compiler/xla/service/dot_decomposer.cc
+++ b/tensorflow/compiler/xla/service/dot_decomposer.cc
@@ -15,6 +15,8 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/service/dot_decomposer.h"
 
+#include "absl/algorithm/container.h"
+#include "absl/strings/str_join.h"
 #include "tensorflow/compiler/xla/service/hlo_computation.h"
 #include "tensorflow/compiler/xla/service/hlo_instruction.h"
 #include "tensorflow/compiler/xla/service/hlo_opcode.h"
@@ -156,29 +158,187 @@ Status DecomposeBatchDot(HloInstruction* dot) {
   return computation->ReplaceInstruction(dot, new_dot);
 }
 
+// Convert a dot into a canonical form where non-contracting and contracting
+// dimensions are reshaped together and batch dimensions are the most major
+// dimensions. The requires transposing and reshapes the lhs and rhs and
+// reshaping the output batch to the original shape.
+Status CanonicalizeDot(HloInstruction* original_dot) {
+  auto computation = original_dot->parent();
+  const auto& original_dnums = original_dot->dot_dimension_numbers();
+  const int64 num_batch_dims = original_dnums.lhs_batch_dimensions_size();
+  const int64 num_contracting_dims =
+      original_dnums.lhs_contracting_dimensions_size();
+
+  const auto& lhs_shape = original_dot->operand(0)->shape();
+  const int64 lhs_rank = lhs_shape.rank();
+  const int64 num_lhs_non_contracting_dims =
+      lhs_rank - num_batch_dims - num_contracting_dims;
+
+  std::vector<int64> lhs_non_contracting_dims;
+  lhs_non_contracting_dims.reserve(num_lhs_non_contracting_dims);
+  int64 lhs_contracting_size = 1;
+  int64 lhs_non_contracting_size = 1;
+  std::vector<int64> batch_dim_sizes;
+  batch_dim_sizes.reserve(num_batch_dims);
+  for (int64 i = 0; i < lhs_rank; ++i) {
+    if (absl::c_linear_search(original_dnums.lhs_contracting_dimensions(), i)) {
+      lhs_contracting_size *= lhs_shape.dimensions(i);
+    } else if (absl::c_linear_search(original_dnums.lhs_batch_dimensions(),
+                                     i)) {
+      batch_dim_sizes.push_back(lhs_shape.dimensions(i));
+    } else {
+      lhs_non_contracting_dims.push_back(i);
+      lhs_non_contracting_size *= lhs_shape.dimensions(i);
+    }
+  }
+  // The canonical form of the lhs is
+  // [BatchDims, NonContractingDims, ContractingsDims]
+  std::vector<int64> lhs_transpose;
+  lhs_transpose.reserve(lhs_rank);
+  lhs_transpose.insert(lhs_transpose.end(),
+                       original_dnums.lhs_batch_dimensions().begin(),
+                       original_dnums.lhs_batch_dimensions().end());
+  lhs_transpose.insert(lhs_transpose.end(), lhs_non_contracting_dims.begin(),
+                       lhs_non_contracting_dims.end());
+  lhs_transpose.insert(lhs_transpose.end(),
+                       original_dnums.lhs_contracting_dimensions().begin(),
+                       original_dnums.lhs_contracting_dimensions().end());
+  HloInstruction* transposed_lhs =
+      computation->AddInstruction(HloInstruction::CreateTranspose(
+          ShapeUtil::PermuteDimensions(InversePermutation(lhs_transpose),
+                                       lhs_shape),
+          original_dot->mutable_operand(0), lhs_transpose));
+  std::vector<int64> lhs_reshape_dims = batch_dim_sizes;
+  lhs_reshape_dims.push_back(lhs_non_contracting_size);
+  lhs_reshape_dims.push_back(lhs_contracting_size);
+  // Reshape the contracting and non-contracting dimensions together.
+  HloInstruction* reshaped_lhs =
+      computation->AddInstruction(HloInstruction::CreateReshape(
+          ShapeUtil::MakeShape(lhs_shape.element_type(), lhs_reshape_dims),
+          transposed_lhs));
+
+  const auto& rhs_shape = original_dot->operand(1)->shape();
+  const int64 rhs_rank = rhs_shape.rank();
+  const int64 num_rhs_non_contracting_dims =
+      rhs_rank - num_batch_dims - num_contracting_dims;
+  std::vector<int64> rhs_non_contracting_dims;
+  rhs_non_contracting_dims.reserve(num_rhs_non_contracting_dims);
+  int64 rhs_non_contracting_size = 1;
+  int64 rhs_contracting_size = 1;
+  for (int64 i = 0; i < rhs_rank; ++i) {
+    if (absl::c_linear_search(original_dnums.rhs_contracting_dimensions(), i)) {
+      rhs_contracting_size *= rhs_shape.dimensions(i);
+    } else if (!absl::c_linear_search(original_dnums.rhs_batch_dimensions(),
+                                      i)) {
+      rhs_non_contracting_dims.push_back(i);
+      rhs_non_contracting_size *= rhs_shape.dimensions(i);
+    }
+  }
+
+  // The canonical form of the rhs is
+  // [BatchDims, ContractingsDims, NonContractingDims]
+  std::vector<int64> rhs_transpose;
+  rhs_transpose.reserve(rhs_rank);
+  rhs_transpose.insert(rhs_transpose.end(),
+                       original_dnums.rhs_batch_dimensions().begin(),
+                       original_dnums.rhs_batch_dimensions().end());
+  rhs_transpose.insert(rhs_transpose.end(),
+                       original_dnums.rhs_contracting_dimensions().begin(),
+                       original_dnums.rhs_contracting_dimensions().end());
+  rhs_transpose.insert(rhs_transpose.end(), rhs_non_contracting_dims.begin(),
+                       rhs_non_contracting_dims.end());
+  HloInstruction* transposed_rhs =
+      computation->AddInstruction(HloInstruction::CreateTranspose(
+          ShapeUtil::PermuteDimensions(InversePermutation(rhs_transpose),
+                                       rhs_shape),
+          original_dot->mutable_operand(1), rhs_transpose));
+
+  std::vector<int64> rhs_reshape_dims = batch_dim_sizes;
+  rhs_reshape_dims.push_back(rhs_contracting_size);
+  rhs_reshape_dims.push_back(rhs_non_contracting_size);
+  // Reshape the contracting and non-contracting dimensions together.
+  HloInstruction* reshaped_rhs =
+      computation->AddInstruction(HloInstruction::CreateReshape(
+          ShapeUtil::MakeShape(rhs_shape.element_type(), rhs_reshape_dims),
+          transposed_rhs));
+
+  std::vector<int64> dot_dims = batch_dim_sizes;
+  dot_dims.push_back(lhs_non_contracting_size);
+  dot_dims.push_back(rhs_non_contracting_size);
+
+  DotDimensionNumbers dot_dnums;
+  for (int64 i = 0; i < num_batch_dims; ++i) {
+    dot_dnums.add_lhs_batch_dimensions(i);
+    dot_dnums.add_rhs_batch_dimensions(i);
+  }
+  dot_dnums.add_lhs_contracting_dimensions(num_batch_dims + 1);
+  dot_dnums.add_rhs_contracting_dimensions(num_batch_dims);
+
+  HloInstruction* dot = computation->AddInstruction(HloInstruction::CreateDot(
+      ShapeUtil::MakeShape(original_dot->shape().element_type(), dot_dims),
+      reshaped_lhs, reshaped_rhs, dot_dnums, original_dot->precision_config()));
+
+  return computation->ReplaceInstruction(
+      original_dot, computation->AddInstruction(HloInstruction::CreateReshape(
+                        original_dot->shape(), dot)));
+}
+
 }  // namespace
 
 StatusOr<bool> DotDecomposer::Run(HloModule* module) {
   XLA_VLOG_LINES(2, "DotDecomposer ENTRY\n" + module->ToString());
-  // Gather all batch Dot operations.
-  std::vector<HloInstruction*> batch_dots;
+  // Gather all Non-canonical Dot operations.
+  std::vector<HloInstruction*> non_canonical_dots;
   for (auto* computation : module->MakeNonfusionComputations()) {
     for (auto* instruction : computation->instructions()) {
       if (instruction->opcode() != HloOpcode::kDot) {
         continue;
       }
       const DotDimensionNumbers& dnums = instruction->dot_dimension_numbers();
-      if (dnums.lhs_batch_dimensions_size() > 0 && decompose_batch_dot_) {
-        batch_dots.push_back(instruction);
+      // A dot it not canonical if there are more than one contracting
+      // dimension.
+      if (dnums.lhs_contracting_dimensions_size() > 1) {
+        non_canonical_dots.push_back(instruction);
+        continue;
+      }
+      if (dnums.lhs_batch_dimensions().empty()) {
+        continue;
+      }
+      std::vector<int64> canonical_batch_dims(
+          dnums.lhs_batch_dimensions_size());
+      absl::c_iota(canonical_batch_dims, 0);
+      if (!absl::c_equal(dnums.lhs_batch_dimensions(), canonical_batch_dims) ||
+          !absl::c_equal(dnums.rhs_batch_dimensions(), canonical_batch_dims)) {
+        non_canonical_dots.push_back(instruction);
       }
     }
   }
-  // Decompose each batch Dot in 'batch_dots'.
   bool changed = false;
-  for (auto* dot : batch_dots) {
-    TF_RETURN_IF_ERROR(DecomposeBatchDot(dot));
+  for (auto* dot : non_canonical_dots) {
+    TF_RETURN_IF_ERROR(CanonicalizeDot(dot));
     changed = true;
   }
+
+  if (decompose_batch_dot_) {
+    std::vector<HloInstruction*> batch_dots;
+    for (auto* computation : module->MakeNonfusionComputations()) {
+      for (auto* instruction : computation->instructions()) {
+        if (instruction->opcode() != HloOpcode::kDot) {
+          continue;
+        }
+        const DotDimensionNumbers& dnums = instruction->dot_dimension_numbers();
+        if (!dnums.lhs_batch_dimensions().empty()) {
+          batch_dots.push_back(instruction);
+        }
+      }
+    }
+    // Decompose each batch Dot in 'batch_dots'.
+
+    for (auto* dot : batch_dots) {
+      TF_RETURN_IF_ERROR(DecomposeBatchDot(dot));
+      changed = true;
+    }
+  }
   XLA_VLOG_LINES(2, "DotDecompose EXIT\n" + module->ToString());
   return changed;
 }
diff --git a/tensorflow/compiler/xla/service/gpu/BUILD b/tensorflow/compiler/xla/service/gpu/BUILD
index 5e81281bae..30a8bda606 100644
--- a/tensorflow/compiler/xla/service/gpu/BUILD
+++ b/tensorflow/compiler/xla/service/gpu/BUILD
@@ -699,6 +699,7 @@ cc_library(
         "//tensorflow/compiler/xla/service:call_inliner",
         "//tensorflow/compiler/xla/service:conditional_simplifier",
         "//tensorflow/compiler/xla/service:convolution_group_converter",
+        "//tensorflow/compiler/xla/service:dot_decomposer",
         "//tensorflow/compiler/xla/service:dynamic_index_splitter",
         "//tensorflow/compiler/xla/service:executable",
         "//tensorflow/compiler/xla/service:flatten_call_graph",
diff --git a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
index 9be7609508..d1522280ba 100644
--- a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
+++ b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
@@ -37,6 +37,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/call_inliner.h"
 #include "tensorflow/compiler/xla/service/conditional_simplifier.h"
 #include "tensorflow/compiler/xla/service/convolution_group_converter.h"
+#include "tensorflow/compiler/xla/service/dot_decomposer.h"
 #include "tensorflow/compiler/xla/service/dynamic_index_splitter.h"
 #include "tensorflow/compiler/xla/service/flatten_call_graph.h"
 #include "tensorflow/compiler/xla/service/gpu/cudnn_batchnorm_rewriter.h"
@@ -165,6 +166,7 @@ Status OptimizeHloModule(HloModule* hlo_module, se::StreamExecutor* stream_exec,
       // We need a cost model for GPUs. Currently, do nothing.
       return false;
     };
+    pipeline.AddPass<DotDecomposer>(false);
     pipeline.AddPass<ConvolutionGroupConverter>(
         cost_model,
         /*convert_batch_groups_only=*/true);
diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index 106b3fd6c9..0fd0fc108a 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -711,6 +711,7 @@ xla_test(
         "//tensorflow/compiler/xla/client:local_client",
         "//tensorflow/compiler/xla/client:xla_builder",
         "//tensorflow/compiler/xla/tests:client_library_test_base",
+        "//tensorflow/compiler/xla/tests:hlo_test_base",
         "//tensorflow/compiler/xla/tests:literal_test_util",
         "//tensorflow/compiler/xla/tests:test_utils",
         "//tensorflow/compiler/xla/tests:xla_internal_test_main",
@@ -768,6 +769,7 @@ xla_test(
         "//tensorflow/compiler/xla/client:local_client",
         "//tensorflow/compiler/xla/client:xla_builder",
         "//tensorflow/compiler/xla/tests:client_library_test_base",
+        "//tensorflow/compiler/xla/tests:hlo_test_base",
         "//tensorflow/compiler/xla/tests:literal_test_util",
         "//tensorflow/compiler/xla/tests:test_utils",
         "//tensorflow/compiler/xla/tests:xla_internal_test_main",
diff --git a/tensorflow/compiler/xla/tests/dot_operation_test.cc b/tensorflow/compiler/xla/tests/dot_operation_test.cc
index c5d8b663f4..2e02968ac5 100644
--- a/tensorflow/compiler/xla/tests/dot_operation_test.cc
+++ b/tensorflow/compiler/xla/tests/dot_operation_test.cc
@@ -25,6 +25,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/reference_util.h"
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/tests/client_library_test_base.h"
+#include "tensorflow/compiler/xla/tests/hlo_test_base.h"
 #include "tensorflow/compiler/xla/tests/literal_test_util.h"
 #include "tensorflow/compiler/xla/tests/test_macros.h"
 #include "tensorflow/compiler/xla/tests/test_utils.h"
@@ -1147,5 +1148,38 @@ XLA_TEST_F(DotOperationTest, DotRank2AndRank2NonDefaultContractionDims) {
 
   ComputeAndCompareR2<float>(&builder, expected, {}, error_spec_);
 }
+
+class DotOperationTextTest : public HloTestBase {};
+
+XLA_TEST_F(DotOperationTextTest, DotReorderedDotDims) {
+  absl::string_view hlo_string =
+      R"(
+HloModule ComplexDotMultipleNonContracting
+
+ENTRY %test {
+  %lhs = f32[7,17,10,13]{3,2,1,0} parameter(0)
+  %rhs = f32[7,9,10,13,6]{4,3,2,1,0} parameter(1)
+  ROOT %dot = f32[10,7,17,9,6]{4,3,2,1,0} dot(%lhs, %rhs), lhs_batch_dims={2,0}, rhs_batch_dims={2,0}, lhs_contracting_dims={3}, rhs_contracting_dims={3}
+}
+)";
+
+  EXPECT_TRUE(RunAndCompare(hlo_string, ErrorSpec{1e-3, 1e-3}));
+}
+
+XLA_TEST_F(DotOperationTextTest, DotReorderedDotDimsAndMultipleContracting) {
+  absl::string_view hlo_string =
+      R"(
+HloModule ComplexDotMultipleNonContracting
+
+ENTRY %test {
+  %lhs = f32[7,5,17,10,13]{4,3,2,1,0} parameter(0)
+  %rhs = f32[7,9,10,13,6,5]{5,4,3,2,1,0} parameter(1)
+  ROOT %dot = f32[10,7,17,9,6]{4,3,2,1,0} dot(%lhs, %rhs), lhs_batch_dims={3,0}, rhs_batch_dims={2,0}, lhs_contracting_dims={1,4}, rhs_contracting_dims={5,3}
+}
+)";
+
+  EXPECT_TRUE(RunAndCompare(hlo_string, ErrorSpec{1e-3, 1e-3}));
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From 495b3eeef0386b1b89b7aa9df42f2cf438de6ebc Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 8 Jan 2019 12:55:56 -0800
Subject: [PATCH 0397/1095] Enable fast GPU code path for solving many small
 linear systems in matrix_solve.

PiperOrigin-RevId: 228382682
---
 tensorflow/core/kernels/cuda_solvers.cc    | 12 +++++-----
 tensorflow/core/kernels/cuda_solvers.h     |  5 ++--
 tensorflow/core/kernels/matrix_solve_op.cc | 28 +++++++++++++---------
 3 files changed, 26 insertions(+), 19 deletions(-)

diff --git a/tensorflow/core/kernels/cuda_solvers.cc b/tensorflow/core/kernels/cuda_solvers.cc
index a59baaa96f..39d0a998fd 100644
--- a/tensorflow/core/kernels/cuda_solvers.cc
+++ b/tensorflow/core/kernels/cuda_solvers.cc
@@ -692,8 +692,8 @@ static inline Status GetrsBatchedImpl(
     SolverFnT solver, CudaSolver* cuda_solver, OpKernelContext* context,
     cublasHandle_t cublas_handle, cublasOperation_t trans, int n, int nrhs,
     const Scalar* const host_a_dev_ptrs[], int lda, const int* dev_pivots,
-    const Scalar* const host_b_dev_ptrs[], int ldb,
-    DeviceLapackInfo* dev_lapack_info, int batch_size) {
+    const Scalar* const host_b_dev_ptrs[], int ldb, int* host_lapack_info,
+    int batch_size) {
   mutex_lock lock(handle_map_mutex);
   using CudaScalar = typename CUDAComplexT<Scalar>::type;
   ScratchSpace<uint8> dev_a_dev_ptrs =
@@ -714,7 +714,7 @@ static inline Status GetrsBatchedImpl(
       cublas_handle, trans, n, nrhs,
       reinterpret_cast<const CudaScalar* const*>(dev_a_dev_ptrs.data()), lda,
       dev_pivots, reinterpret_cast<CudaScalar**>(dev_b_dev_ptrs.mutable_data()),
-      ldb, dev_lapack_info->mutable_data(), batch_size));
+      ldb, host_lapack_info, batch_size));
   return Status::OK();
 }
 
@@ -723,13 +723,13 @@ static inline Status GetrsBatchedImpl(
   Status CudaSolver::GetrsBatched(                                             \
       cublasOperation_t trans, int n, int nrhs,                                \
       const Scalar* const host_a_dev_ptrs[], int lda, const int* dev_pivots,   \
-      const Scalar* const host_b_dev_ptrs[], int ldb,                          \
-      DeviceLapackInfo* dev_lapack_info, int batch_size) {                     \
+      const Scalar* const host_b_dev_ptrs[], int ldb, int* host_lapack_info,   \
+      int batch_size) {                                                        \
     return GetrsBatchedImpl(reinterpret_cast<getrs_##type_prefix*>(            \
                                 BLAS_SOLVER_FN(getrsBatched, type_prefix)),    \
                             this, context_, cublas_handle_, trans, n, nrhs,    \
                             host_a_dev_ptrs, lda, dev_pivots, host_b_dev_ptrs, \
-                            ldb, dev_lapack_info, batch_size);                 \
+                            ldb, host_lapack_info, batch_size);                \
   }
 
 TF_CALL_LAPACK_TYPES(GETRS_BATCHED_INSTANCE);
diff --git a/tensorflow/core/kernels/cuda_solvers.h b/tensorflow/core/kernels/cuda_solvers.h
index 2c30d036df..1fc344731c 100644
--- a/tensorflow/core/kernels/cuda_solvers.h
+++ b/tensorflow/core/kernels/cuda_solvers.h
@@ -235,13 +235,14 @@ class CudaSolver {
                       int batch_size) TF_MUST_USE_RESULT;
 
   // Batched linear solver using LU factorization from getrfBatched.
-  // See:
+  // Notice that lapack_info is returned on the host, as opposed to
+  // most of the other functions that return it on the device. See:
   // http://docs.nvidia.com/cuda/cublas/index.html#cublas-lt-t-gt-getrsbatched
   template <typename Scalar>
   Status GetrsBatched(cublasOperation_t trans, int n, int nrhs,
                       const Scalar* const dev_Aarray[], int lda,
                       const int* devIpiv, const Scalar* const dev_Barray[],
-                      int ldb, DeviceLapackInfo* dev_lapack_info,
+                      int ldb, int* host_lapack_info,
                       int batch_size) TF_MUST_USE_RESULT;
 
   // Computes matrix inverses for a batch of small matrices. Uses the outputs
diff --git a/tensorflow/core/kernels/matrix_solve_op.cc b/tensorflow/core/kernels/matrix_solve_op.cc
index 169f3dae76..f3919a16aa 100644
--- a/tensorflow/core/kernels/matrix_solve_op.cc
+++ b/tensorflow/core/kernels/matrix_solve_op.cc
@@ -214,9 +214,12 @@ class MatrixSolveOpGpu : public AsyncOpKernel {
     auto input_copy_ptrs = solver->GetScratchSpace<uint8>(
         sizeof(Scalar*) * batch_size, "input_copt_ptrs",
         /* on_host */ true);
-    if (n / batch_size <= 128) {
-      // For small matrices or large batch sizes, we use the batched
-      // interface from cuBlas.
+    const int kMaxMatrixSizeToBatchSizeRatio = 128;
+    const bool use_batched_solver =
+        n <= kMaxMatrixSizeToBatchSizeRatio * batch_size;
+    if (use_batched_solver) {
+      // For small matrices or large batch sizes, we use the batched interface
+      // from cuBlas.
       const Scalar** input_copy_ptrs_base =
           reinterpret_cast<const Scalar**>(input_copy_ptrs.mutable_data());
       for (int batch = 0; batch < batch_size; ++batch) {
@@ -230,8 +233,8 @@ class MatrixSolveOpGpu : public AsyncOpKernel {
                                &dev_info.back(), batch_size),
           done);
     } else {
-      // For small batch sizes we use the non-batched interface from cuSolver,
-      // which is much faster for large matrices.
+      // For small batch sizes or large matrices, we use the non-batched
+      // interface from cuSolver, which is much faster for large matrices.
       dev_info.push_back(solver->GetDeviceLapackInfo(batch_size, "getrf"));
       for (int batch = 0; batch < batch_size; ++batch) {
         OP_REQUIRES_OK_ASYNC(
@@ -279,11 +282,7 @@ class MatrixSolveOpGpu : public AsyncOpKernel {
         /* on_host */ true);
     auto transposed_rhs_reshaped =
         transposed_rhs.template flat_inner_dims<Scalar, 3>();
-    // TODO(rmlarsen): Enable the following branch when I figure
-    // out why it causes a segfault.
-    if (false && n / batch_size <= 128) {
-      dev_info.push_back(
-          solver->GetDeviceLapackInfo(batch_size, "GetrsBatched"));
+    if (use_batched_solver) {
       const Scalar** input_copy_ptrs_base =
           reinterpret_cast<const Scalar**>(input_copy_ptr_array.mutable_data());
       const Scalar** transposed_rhs_ptrs_base =
@@ -293,13 +292,20 @@ class MatrixSolveOpGpu : public AsyncOpKernel {
         input_copy_ptrs_base[batch] = &input_copy_reshaped(batch, 0, 0);
         transposed_rhs_ptrs_base[batch] = &transposed_rhs_reshaped(batch, 0, 0);
       }
+      int host_info = 0;
       OP_REQUIRES_OK_ASYNC(
           context,
           solver->GetrsBatched(adjoint_ ? CUBLAS_OP_C : CUBLAS_OP_T, n, nrhs,
                                input_copy_ptrs_base, n, pivots_mat.data(),
-                               transposed_rhs_ptrs_base, n, &dev_info.back(),
+                               transposed_rhs_ptrs_base, n, &host_info,
                                batch_size),
           done);
+      OP_REQUIRES_ASYNC(
+          context, host_info == 0,
+          errors::InvalidArgument("The ", -host_info,
+                                  "'th argument to cublas*getrsBatched had "
+                                  "an illegal value."),
+          done);
     } else {
       dev_info.push_back(solver->GetDeviceLapackInfo(batch_size, "getrs"));
       for (int batch = 0; batch < batch_size; ++batch) {
-- 
GitLab


From de86e2fff13c5b2c41297ef923c2d60e5a7297e8 Mon Sep 17 00:00:00 2001
From: Thomas Deegan <tadeegan@gmail.com>
Date: Tue, 8 Jan 2019 13:29:33 -0800
Subject: [PATCH 0398/1095] Fix broken image

The image tag in this markdown file seems to be broken when viewed on github.
---
 tensorflow/core/profiler/g3doc/profile_memory.md | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/tensorflow/core/profiler/g3doc/profile_memory.md b/tensorflow/core/profiler/g3doc/profile_memory.md
index 6eda5abdd9..a2c7cc6289 100644
--- a/tensorflow/core/profiler/g3doc/profile_memory.md
+++ b/tensorflow/core/profiler/g3doc/profile_memory.md
@@ -14,9 +14,7 @@ Open a Chrome browser, enter URL chrome://tracing and load the timeline file.
 ******************************************************
 ```
 
-<left>
 ![Timeline](graph_timeline.png)
-</left>
 
 
 ```python
@@ -77,4 +75,4 @@ _TFProfRoot (--/74148.60MB)
                   seq2seq_attention_model.py:320:_add_train_op:tf.summary.scalar... (0B/64B)
                 seq2seq_attention_model.py:360:build_graph:self._add_seq2seq() (0B/25216.74MB)
                   seq2seq_attention_model.py:192:_add_seq2seq:sequence_length=a... (0B/21542.55MB)
-```
\ No newline at end of file
+```
-- 
GitLab


From 01dd371b8441aa96536732b804f1c5853fb8af6e Mon Sep 17 00:00:00 2001
From: Mark Daoust <markdaoust@google.com>
Date: Tue, 8 Jan 2019 13:29:19 -0800
Subject: [PATCH 0399/1095] remove JackOptions block.

PiperOrigin-RevId: 228388690
---
 tensorflow/lite/g3doc/demo_ios.md             |  45 ++++++++++--------
 .../g3doc/images/ios/bundle_identifier.png    | Bin 0 -> 64902 bytes
 2 files changed, 26 insertions(+), 19 deletions(-)
 create mode 100644 tensorflow/lite/g3doc/images/ios/bundle_identifier.png

diff --git a/tensorflow/lite/g3doc/demo_ios.md b/tensorflow/lite/g3doc/demo_ios.md
index fbf1dd6392..f4b481dc61 100644
--- a/tensorflow/lite/g3doc/demo_ios.md
+++ b/tensorflow/lite/g3doc/demo_ios.md
@@ -7,22 +7,23 @@ instructions walk you through building and running the demo on an iOS device.
 
 ## Prerequisites
 
-* You must have [Xcode](https://developer.apple.com/xcode/) installed and have a
-  valid Apple Developer ID, and have an iOS device set up and linked to your
-  developer account with all of the appropriate certificates. For these
-  instructions, we assume that you have already been able to build and deploy an
-  app to an iOS device with your current developer environment.
+*   You must have [Xcode](https://developer.apple.com/xcode/) installed and have
+    a valid Apple Developer ID, and have an iOS device set up and linked to your
+    developer account with all of the appropriate certificates. For these
+    instructions, we assume that you have already been able to build and deploy
+    an app to an iOS device with your current developer environment.
 
-* The demo app requires a camera and must be executed on a real iOS device. You
-  can build it and run with the iPhone Simulator but it won't have any camera
-  information to classify.
+*   The demo app requires a camera and must be executed on a real iOS device.
+    You can build it and run with the iPhone Simulator but it won't have any
+    camera information to classify.
 
-* You don't need to build the entire TensorFlow library to run the demo, but you
-  will need to clone the TensorFlow repository if you haven't already:
+*   You don't need to build the entire TensorFlow library to run the demo, but
+    you will need to clone the TensorFlow repository if you haven't already:
 
         git clone https://github.com/tensorflow/tensorflow
+        cd tensorflow
 
-* You'll also need the Xcode command-line tools:
+*   You'll also need the Xcode command-line tools:
 
         xcode-select --install
 
@@ -31,35 +32,41 @@ instructions walk you through building and running the demo on an iOS device.
 
 ## Building the iOS Demo App
 
-1. Install CocoaPods if you don't have it:
+1.  Install CocoaPods if you don't have it:
 
         sudo gem install cocoapods
 
-2. Download the model files used by the demo app (this is done from inside the
-   cloned directory):
+2.  Download the model files used by the demo app (this is done from inside the
+    cloned directory):
 
         sh tensorflow/lite/examples/ios/download_models.sh
 
-3. Install the pod to generate the workspace file:
+3.  Install the pod to generate the workspace file:
 
         cd tensorflow/lite/examples/ios/camera
         pod install
 
     If you have installed this pod before and that command doesn't work, try
 
-        pod update
+        pod repo update
 
-    At the end of this step you should have a file called 
+    At the end of this step you should have a file called
     `tflite_camera_example.xcworkspace`.
 
-4. Open the project in Xcode by typing this on the command line:
+4.  Open the project in Xcode by typing this on the command line:
 
         open tflite_camera_example.xcworkspace
 
     This launches Xcode if it isn't open already and opens the
     `tflite_camera_example` project.
 
-5. Build and run the app in Xcode.
+5.  Under `Project navigator -> tflite_camera_example -> Targets ->
+    tflite_camera_example -> General` change the bundle identifier by
+    pre-pending your name:
+
+    ![pre-pend your name to the bundle identifier](images/ios/bundle_identifier.png)
+
+6.  Build and run the app in Xcode.
 
     Note that as mentioned earlier, you must already have a device set up and
     linked to your Apple Developer account in order to deploy the app on a
diff --git a/tensorflow/lite/g3doc/images/ios/bundle_identifier.png b/tensorflow/lite/g3doc/images/ios/bundle_identifier.png
new file mode 100644
index 0000000000000000000000000000000000000000..398763916b353e61f236392e2b8898aad2aafe8e
GIT binary patch
literal 64902
zcmeAS@N?(olHy`uVBq!ia0y~yU_Qmbz|h6P#=yW(bbs?v1_mDaOlRkS%;aPS29M6E
z)7c|}Pl`1>pQfUsvc-XANy|N#m!Vl1j4VsI%$$TgG!#`>1g$r%I1m-EF~K$5H7A=f
zWn-d#cfxyqZjDpBA~r1iq{g{m_5JGi)z5!FyJw$%ZjR;iJ3A*^GpsmXaO6==2y=r>
z;64>IlL?lJmaU#uECLP;A`7^EEMit#D5$9kIY{pMAAIj_!?l2Qw@%moz4Jfbb@K7)
zMT`zf3d=t(xg&Feo$<ik9Z8;c3@vgi%x6#2Z}VvhWXk2_5!keIj`5T?iqB0{-L-Na
zB}YyOW(-(ZWHE_{;mVUjL+%}t7y6@EQ`iOlHJa}5+?=lc;DbVan~j#by7;DDrYWzt
zy#JH(=WzP14gP%U$up+z)NoPt&=5^8UiN?f(;xRIP1cp3b(qI&m4lF&_W8)T?Q8CE
zSOzQ37hdY>>U^!K#`i?<jng~56}Y`6w){UZbyLDw24;pSryjGoI+!)&Xr1colDYDG
z8MpI|35jnnRwUY4$61O!^(;ze|GT)gCbQs<K(R{K&L?K6o7S{6IL?=5srfeH{4yoS
znn^}}Th=<vkYJ7E{bZXN7IS35+0ROh+F2*ul)9%L=b3NGc1cxnqnGp<8+WTI;?GnC
zHZ5keG>R9HUnbM-`>;r4<4dP=PrTJ#_>P`@%+9L)`B2x-MJ)$^Z1gIuOL(*3Z&+d3
znFY4j&Yb6)`r21<yBGJPwqxI(9X`73j5zzFKQk&)b9R)esoS_oGUstCF><pos!cdJ
zjjhO0O<>zMlP?G2Pt6lJz$i3<@yCI>0Amv+&x^`U*DV5A{xuz8zx1%Mu`A9_qv4nS
z;i#uq<(6nyaf>K#au9zaBfz-nVgAgdDH#>pTt2X5wb-9M*Saz7u-g0Ip2tJvF9|kx
zIvz1d;<(6m{djF*zKu}a-^AiWwgu-G?$~!zco)ap7WbCdEqsUGt~``<l7FJ{L5mBF
z$vl#0^xpn{Q{esLswn@F+6DD5q|+b#Iv`ixXw3Sct{^Xb#b3Lyn+3=2XIZ}Je&gVF
zQu)Ncl-Kgg0h>dMWxpw{zw&)+z_F=jF}l|Z&i7Yt*tc4q|DJN)mLp<G>>D=U;Cy3q
zt;du>t&Au8h+d-m6PrC|^E9p<yO6W7t76jv`G@|W3)!Ae3HN;!@chEt50xAEpM2*2
z{8XP?JKQE#>5RY=W9A2WXVW*i-nqDVh96&Ff%w$-e^0G_%ImywxpEKp{gTS#j30EG
zPxa(m@!e_WD`#}5|L&XqO-k?gQU;mK$EQ?Sy=3+FpZFhm<>Bl_>^%?erScjVG36bQ
z{?VKuz_-ZJ#)B<5VJ>UIT(1Kr2Eu2UmoAXeVbwkuwLvh4(fUB%26>Z)v;w9%jN*>c
zJNVum+IopIt|9IN-xp;LC#MV!*^ACSg4{>=3Y}~O;#DL%U6LlIsED3)aOvQg*?GqO
z)I_fo-A$4!UF+QWl+`DE_OLFJuIwmLUO&-aqb(>f_lm$)m*WfMG8m*9k1d{jk;jBz
z*U9_B(HB`;*zUGoTdaPe`$g#%=Gw+@i44I<C1!A@^;ah7=ty}Vyky8}CaryV)dr~?
z{%b7rzl5$~etppHk;|Mu<HG74afd&!%;|pqaPvp58r^-&>yP_C4FA|(q5p5HgR+Fc
z8;&>@vqUDvW`o`UuN6u;$}&oG+TQ!AKa`#5ks`3R<K2h5-fC$==>pmvuRE(dk9WR3
za{P#$Q0Ef&P0BwfEb^2IvRgT2OWT(2lAe;*E1tg=ELxSc=Ft+NMUxgO1!;z@47$0p
zDCB9^so<$~o7iqPy>abR`+VZ^N$(TUCw-rse&SbT?>afj_2a@L!A6>&HCAg@Yb@9N
zu8|)S8tNNr8+v>-SLpLq>%#6Xuk!o4@Rj>2v8&TgE%IA*YDM?T+bfT++`ckD!(^7~
z+4<9)PA}J8cJ-3qCAkZBD=uDjTfOmWMON{ZRaakKUAJOyi2Ca273?eg1NDQ}2iY&E
zi`?tpye4tw!AlP^&Rkn@&0<Yq@M8bT>t3#qTsw1d<ief7pZ7d7)}1tMY1jra|J5$p
z-Pb(V9xv(1b4$Ot^UKdKYrkCn;+MN<+f=h?>)_on)!WNfm#y!c*4OQ~$Zw(F)^mb$
z`PhzaNZBBHvq!b(Z;y13zcP<IOV5M;2$Q7q9oowGUiV%3dj09^X)oVh-Mj9$Xa7p^
zHPe@F_gU}wexKdC{MGhx^|Su5{k{ClR(Y#?RsWMd9qFC*Ph^>|9Jr=%Eurn8ShF`<
zZJS%$!nUNtrfurH)jbk!|J-J}Y4`L=l}cakt?g|+rrpyoX`lYefO*EvGgr?9pNSJ@
zo@^jI^JLoOu*r47+Lu`_CtcpV?E6`bG`X~wY1(I^&QzW4J$roSd6V_V`;#5Feb^un
zv1XG+_Q!jdP8ppnYD{f@YOEcry)ATN=-jKe*|WCJFP&G~`_)u>QE$8Nw6j&S&7!Yd
zyBd}dom+jAZ)@>p;ah9BXUn@vJhk*PS(SFhw|Z4=Lv3U2+`oq}_FSB}xZB;lr_S<N
zjnAnWsmbU1e!hC}YQozMS0AohJS}-g^2X%JkF(XI)z<}Yy=<C3tGMH{%yYNr;d*6y
zHPd<Zw#U3&_c%s;$F?1Hg~jEQcf2ipU1nbHUmE|s?y~c3<ztHvJ)Wa}(D*`eU;5_f
z?)v8IYu5jbtFOLYonHC=OP$%ZES>r{*EaUo`qWz3&91eonYnM}zS%X_e|-K})S1@x
z|2g}o^55-0^Vx2&q_Dl=s*rpkpuuS)@?+l9qqB}$$+fVyvQ1^(%9hQl&a$4jP9RBY
zl3<YdCFxgER?1&0BouX&aw2c|Zd06ndR^-Gw%zULx$H$9dpo)#q(5q0G7A#AX~HKf
zZkP9SM|GfB;Gz|683Jef;^b=Wk7=ptKkNT^<fGx^)yK<^S{*yrV=L|6KU-?M@w?cX
zgg=vh?B=}2xv6EE%hw~Gom)HKPi~r|HQ7q7XQn~+p`$5(Qxc1kxinP`Q=`4^A9>KJ
ztzPC=G&jWR<*}62CtD|<_rExQlI_zIAzxR{S;-#6xzcl`b+CQ$B1_Xb`%KDi?b;Hj
z*S)*zchmFp(Fb2Y`~5-tr~l>JOWK#`r~XLMIsZcJ2j?mlTc>FSYd?ghMLaup=6mGD
z$lIIQHcj62{n?5qOP<MyWp-|ISzgpz#9e&*XxXEnLic&$_fGAJ^K40(Ei%z>n%T4L
z!~Ji(jk0g*Z)#`V$9k1jewnJDb-;>94Xr&Ucgub)dg<To)KxdT@%)2x88b7Ditn9i
z-^`SjJFoYQ-|V|a$8-En&N;VpVR8AH#b-{>`kubgxiIzP?Vp~XXM6E@KbvOrPxZ_6
zT|rxeJYSw=y~k~)cSM{^e4AcT+?KS9*Nx(CwidO2TJ`C?=$$U^{Z{{W{kq<G^ksMS
z^c}i4H%!?PvgOM$qy4GYTBh3FTJrkx+xOJ57Q5c;+OEBPolX4Hpsj_XsZ(=8kH0!`
zwKx0Bb)DGHm1`5;etmV|)!x@U*L0)4*V*R(oyz-9`dg04orrxOFRMLV7Cqb9{QC8X
zYazLcd73vaZ8a@?ee=Fe-Rir`_OGqEJ^k&Xx3bnr<#Se*u3vjMb3?JGY?S)y*Y`{_
z!*9P^aJRSm&G(#pr*A~><9{>xZS}wESMSZgxxdM`d{CHh-m$y+HQz=4saKW1UVdFI
z&B80&d$RX!`(gg=CY9#0zmmVm-=6I|Yjyt%naH9kMO&V{IIwY1a_#Y>$E*EP=2YA<
z+WPd}Nlx|kbEWE6eJwfivN+x8+#DO}f^Ro0=NF%SzW1E`z6XU1A1=Cm^q2R#>3n}*
z{(8PJZ2Prer{7JNzHM`R)~(~)<o7-*e|qHVg4OBkHm;A|BUifjuiK&CXmj<uZ+0Br
z{q;@Rx3}TtT<^ZzpIhBuu>Q@yV}Dn%v+-yBOZ~iluWX*IyBwSCo-Z2jF5W$U=eYhn
zkNGw4bY3nh4Yf6`i~4=$Wn^)=NBQl~o#(CNJL125pYm_%&+EIUU(@IR_xAtyrS~}Z
z&Rwsvo-HnX&&BG^f7hPAJ#F8NeeCax?tR^_|9ipr#N+P${pt3qwN<z5-x|OC{7T;}
ze%3y>n!V-|j13f?d02G$GxC_3*vd9szs<Nn^>BW>8H0xK)O6!kg;SrGG##i<div8>
zF^?hVJG(-n9>c^B(y7cjViQ(=(EHEQ|DAos=FbJvBAL53Prf8BIxm7lo{jCE1d~G+
z(}V7mga!3~4;+}j*5Jl%2AT80^EjU>**E7&DKFo4Zu$fFpXM3&3bkHv1?^^FU{FZ*
z2=ZlMs8VBKXlP+z_{G4$(C~tRq11qZ;Z*_ygVhWM2JwP9y8>+(7#P@+yxm<G{)6Cx
zy~`&vFfecyctjR6Fo>ywFr#OX$_EAp2KEw9Usv`=JR;0SIsvx3wHX*17(87ZLn>~)
z*;_tC<oMPdf6Kl<vDs`naoU{49ZXXkgcP*hwbf5g(mj21-@b?6Z(iT`ZENk^W~Nh{
zR090m8dO=7I#ebWx1B94da>?(|4qrVo3RlpEZrgRKkJ++2@egQ_0Fo+Z|>qZHbP=G
zX}jWDELg*TzuEdiWBDaVMfLDU&u^HVeBSzLrGMGp8)5M->6v}umzJ+TmY(a@KC?7d
z{l=9O9iN=mh+R%!Gsi_~qKAr5r^}_k0ru5`cb#qC$ZehAu}g9O%j1gmkwTqd`APcn
zYKJe_Ckw_#dzAb8e(L11<n;ad{JPD_=e<Rj)ctm4EG(&ex5Vp}#^e_^KUXe%yOr13
zUEu<+taJF2JJZ4DD^09<Xb^h++I)vor{{$9Syuhh*><$BUt{jFUpm)q%r0NdQLnT;
zd9$;KOU3tT3;V7K%NGZ`+jH85Tkbz>)G9DT#IZQoIB;^;51ac<o*)N<+?~#FxsmN{
zi_(40=?5QGw7WikzH7t2cOf3<{*~7p44rd@ap~n2_4Nhsl4|XE(`Uzgs^rK~-xV=C
zyN+w>>Y_i*Ug2Gq6IoW5sLsDs7_@Zr%+)19e(Bd=X4%X5F4tt+dslN$9Dms6muqJC
zLBm1lWT$~@?_>Qb{yGYBf0cL|6D4%rFCQ0W4GGO&yioYK?w^Y5d9O=WzpY%#{nNkZ
zY>8#=*OK`cwSI3-`8kKn^6Z>I$DPJ=U#AOC>8}%(J@;0a`;wP9>(g!QQPZ`pbk`{#
zFOi)xd+nRTUyFR3FQr|ZIX!sQvV~bvaNEUuAMaZ8c2mQO1_K^GGjri%2c|?W=k8Dm
z-XS=p$|hpXH=j?7e_CmNcA53W$MX1O>DzPdH&)F&zs}RHbKjI2zLKrW`?I&)`D$}Y
zQ|GnulF(ZmTht@v6<^F+SsCSGxAwg)EL<lUA2@lm%(8F6%H@-eoYC6*Yf6Ch)xd@+
ziX5(w-mF~d%6Tn#8Owz;-`>pHlkK$W#gkj=Czo^lzM{JGSIX%LCCYumYgRf(eh%sG
zkbVFE{f}EOn`3XWeXQg-J^xv312{nmb-ENyOL~>#t7m0=`OS+4kJ+)&K54T`w6mor
zM{W-3&Qg16^svZHqnxWO-2H-1pWoGkvYaQo7A;tnx!lo-L&Wu<0gp(iE9)1r;`e@*
zqE4TdKPVQQ<iEttzE`8GYtaPL?e=-@CDn?rgTpo%s}_fM<tks1oBY}0(5H>^*51of
zzAJQZx&G5f&ed|)=KWih*YUMJ<xA=*|GjYvG_iWvNvmb-{ude$aOK*p)zkdfeox%S
zv1YC3`op0CJ{r2*g{&T2jzNswu?yY#txF#szQX1lk?VJ>`cmG`DXSU;ZcYd?b?4jg
zZt>O~pQ<nH-!tK@HNRoem4a`1G0v`&&+WXtJYn;!OH98FzXsiUllqTG=$U8QTeVGZ
zp0_R1kh!yU>ZxTc%I<tem&|jhvg-S^?e-<Fyf-yAQy<r4U;XBP`PunTO^zIkE*fl%
z5bD0$csT9sJg;L)EsG6JHuFwDd@s!G+&>OicXt0Pl8BVkl<c!!G%G9n=GN?RZQX~4
zP0LqZ+MXX@R9w8-DKpGERpUil)4J^(5==R2qLWXaVV}ww7HXoLr1kTp0&|U4+3dx+
zH@lo{gi>xka-G5>dHlgAqemAN@@<W$>;9f?@xN>J3fnESTlIAp`B&zb<^P+Z+aA<2
z-)LgjT<IHyD<)<>_dlF=ZPTphJ(Cv;XWELodM;;gG1%l(^=(H+>7J=>et}aK|D7;V
znP>6T`AZf*O?zG7Q+L?l(($Oi@-Iq@YHKgwx^pe6Y(Z#jUtRufvx|DZugwa4WT%7$
zg{Z|o`_Z>mJ7WL-nU2ov$=hb#no{Yc`}w)AwBqOBwl7xm+{FFPo%-35Ic4(s>qYAp
zt8`6FIotkzQ_<hNDjvDxsp9^J^MVd|Y$|){G;RHT$Go2&>*m$k`E}p=mwIXW6}k3D
z0zKW6r@d!=uuW^n8ugihpi&1Es6S;av^2J_*>*O~*t-1Pl0}a^q!l?22sABPq{PN2
zbtN&ejUl!@MN2m)Q!6GUM`>C1o@xi>RniHclQx?N3uiAlljIzDbulYP^Qz|Rh%cvB
z^K{P8<l2$3%I?6sx$KfQsbY_h_bn)UU)7=*V9Fh=x3^2MP}bP<ntt@5Prg&XXz4F6
zKQDAfZR)RAT3OkzH{6I7nI^K_`DTeygV~jkz~Jcqqt6yBSiv#-Y?A;R@7AkVu3lR_
zGgm}J^znx1&bHZ{ZHp3?WMpwYJgt2oGkB$4P~*Z#4K2OAllq&UM!nVJ^6uI&%lGA<
zgB*uE1i4%_UtJUy)?B5n)p<ndN71JfXSqw1ax^5|Ce2cI&){<V@S|8m<Bam9D;^p`
z$`huydj@B&S*@sQ!em)@=xpzmkg&-@T$Qb^6?^r*q%^htViO5nxN;xk?%Upe2fTAr
zRWDX_?Rk`BlgEDW7@N?p#)%ylugDyFdRo!me)-Ww;aj>4bTd}0%)5TgSSoq@)1_{&
z_q~pZuxs4BCg<EXp%iJ>H_w-v+O4}VYvt5SuS#}p`<T)4EWx;Xd;Y@{Hzt13<r2tR
zlXm)=tO?ufjjfC3e5gI<X~kLn@v%sK{5+@N-T$JT7)zMn%}sgK_fh@1nIk-MZ49=?
z{ozvlP_?&5)_Phi_kGowK54>S2hz-p%icuP#O1RyO*mknuvA-R(#ucBo~-`zS?f)9
zTTuI{J~jcTg-Z;khm?vq`!8CltEJ+qD9QWrVUzia)%i(xo33w^GF&t@dgZ#6>U*Bd
zpP{Rz{QGD0w#ZdEe={_t`&(w3tm1ziwqE^)#N(;z^PaW+ey>_op{;NGS!rWS<Gf3f
zHl=Fev9{8?m*(CHcWu<op7!a~=Fc6=4tS`&Fw9Xhzf&gbzHM#dz03L&gwldOYS^~>
zKUux~{)u`0_D_B;k3X1GKB4RQ1(~=%a@xFGzq%*|w#Ao7UXQb#9s7Fv$5Wd>-@5Yf
z)$4tS-hGw-^k=eu<ISeq)6a*$+tJy7zjmsx^!G%gY>BR^>+h7QC%f_HTtD9?m;O?E
z{e~a&-!IP8vJ%tV@ou$<Mbeq-Pp;>7fAnMf{^`S!_y2v~cN)*MS;nt#^M}LqYof@)
z$ES23|N8yzXx^32-{b$z+p%!rD@~q-fiYnYxuvET6FLs<TfF1xzvzf>-@MnveB=!4
z=Cqxt&^gs|`@MrL#clF;l~*6QHq9>Q?lo=e7w<2hOTQC-JIW+y_4IcRyZ=<qj_6Ky
z_Ubgtdb{mVkyL14C)?{AyOQ;5yZ*@B**bSs8^^3!+dM21Mb`5D-uPLN_ZX}DEEYTV
ziZw5}IST(=7e2USf#IZo|4(-*Doio(c*(Tf^=Q_wDZ<}GE{a_Ce9>f+{=L3%;cKgh
z1ro8>3wOrMIQuO4zPzumiV)|gPoHkhUA=0Rl34e}&FTJGSy>w*bS&ojmA$>?8W}0+
zw|ukjuLJ`RS69~BUtbQUy!!X|c4&*hk)(}lOeY>q`WbWVQOQIfHBS{Ghv?O-H}8#&
zb>whVns^{-<CfqVTaFp<@X6cxC{0`tzCJD^J#xo^<mYQ{<{V4fIL9E~eShWp>$lZ<
z*S!u24V^f7a`V+&_d>KLdI)f}8qD<BarfQhZ^hvmvCH+D)3e?@`e?B-go|PF$tkw=
zi$4C_bm{J~mla1I7ruJ+N<%{<qgdZ~YpLzrI*BRMr*BU8w~XGNmz%ul_HuvuQ>RZ`
z-<i*oxpu;$X5YYq84I?1C)Nv_s4tlAy;O_kqKL){6<*PcQFB#U(q>Q2+`K;J^Rm4>
zyMIWfot@+3{4}8~@9yyn-Ro1&9SyeF^LO!=uOc2UhgoLNdTCiABdnI6_OgvDZ^!ez
zKV}t~|GX`{+lpTEZoauUPjda<KW5DCeCc;C?7R6|tIKJ}&OX-9&yr8RKCQbw&iU-T
z-aS8GIhD!J=@RlcEIH*m$2il5-AwQK*2?acGm1H@j9hnEo>C3J>s8q~S-t*mw$Q$f
zcfKC3j{N+sdrjQdwo{&qE51LSUO0{C>9lE9d;Wd8%_C>ar8`wgsLSZ@uh%ndDtROX
z*UWR;S9Ja8lxLPRzklEV;pD5*9tq0{*ZO5$Ls#C~-F!5E-{}wC>mS|mRQ~Z`YW;>E
zORw+v-p+535un-qZ10I{PS$}B(;hwfsPg)I(5-Tf={~cVE=tY&cUn2;=C1yWf9IH7
zGdC)@wKT>mnQ6W4FYz+@6_5M!YbRfL=zFIAneVso5aIr|%oUtPZ?Et7`)^md=hut-
z72ltlOPK74Kh4eE>ak&7d3@IWYmLh7#@j;c%lZ8dNk^$KJDa=pbnM57yI-?wSCz>A
z{=sx0r|?19%p03?rLQ`ly0I&lJuF`Dor|#Sp4V1|6)*K7I3y16F7~}+am&To<-xYC
zGeXKITwQYT{`BQPPPeb*JMu@L{cK_LbTiY>o~}JDdM8RVVhoQQUXgTLa`k2QYUMY%
z9-n$U8`rt=Sv#AUm>A^Wv)Pt&(@AUUBKLl|nDV=&%WF37D0vz5?d|R3f4|?qpXazc
z>#CNPw)W1?=d5R%=gXz0ruxjcoBMoz{k++EyE;F8Dl*Hz=i}+g+33*l_xJbDuU4=B
zbT+?Exc_+ZvfUr1T)c4MKz{x2=#p6X(9kn!#=*<|Qg3a^OrOiZ`r#nE{Dy*uhrU=T
z${c%HvPgqVJif*-c)6eL<*UDAR<2+F{N?ib&%WKx&);s5d-m1U)s<f^y3e$$wMtD*
zO}}d)Z=G)Hn|y0arq6u4zdLd*znRBJMV(5W9(zsAS3~5HfBi4v@_Uu)PoF-`J@@wB
z-r3LRRre{o_azw3oMBO@6dqH^TDE)d<!rabfoA#l&TL3Lydi4s+kM;DUA}3?H^0r;
zu<T>SoSntbH>IDS_lA4(#l`N$Csn5l^t%20JpcchwcGD`{r~s(e989`>0O5{G`kL6
zSm?a-&8E`@7BX|=K0i3vd}globooCI*mK`iAK!EC+#W+y<E$$hpu(ZJ*qG@<g-wr~
ztyK5XODiqcu4@hs+_9s*>-Zu89}W}aCsCqFvmUJ4C&g>%80(|B>hh_a>0<r6wfwe=
zZ$8;_`0e4xg%f)!3{&^-w!Gw)<e9xjC(PP!IrG`1qthR&M{S>e*!|Uq)uDU-f4lAC
z+<Qd0aMfz5fVQn?<#X07+@yF+H9Xeiyj<!O5zzp<9GkHAX14Z&2Wzz}|9`!nk+tHO
zfzj`3R=;>bWj9{s2Mu@r#B}}UTavzh-j!+J)878_dX=qZQM5#EebCWKVWlfp<t>O{
zT)w}hSZ%Iz+`jo0CN7?tjMdlHr)1kW=EfSYo406g%t8&Wt?l20`&w4s+1`AV*L}tM
zY$d5R%9BK5Dz0vQ^XsPn%?;-sEPG$u|ENa(?6Rak`|hs!d40Fb;zXs7DsHEL%zVSg
z)+gy3xjyd6G?B|0;V<uQ=Re!WvdUFaqEC@Ugir3+jE{>yTIKrK{f~+JbNqM9;fZ3K
z(^{V$yHyf>HThOmPJKDI6(>vGo+qb1id<G#l$1AL`NAQ&W)G9g@8Gq2`?t!N+`iKz
zajV-jd;6xMr(WMeOfTmjsc71L?NY;w9dm?B+?U_I@oLWXWgANR&&k_)6yAUGcy>s+
znoGUp1>S>`v!!na&wp_3qFs@!6nA;+&$E}T{@rh_obr6h{XNHmn=(!vNDSMU;q`6C
zvSt6xS$61TNk1>1*y4DKVY;hhuxLxe4n2dF0=`^t(pV?267c3+<jK4{^60fiC;OJN
zJZ!nRR`%NRAkAmz=6Z*Qik9Ck757^{Sv<aG;{89*%>R5@Zh!Rk*GWEVob88a_$*tq
zVDW3Kl_6ZdmpfdPB6P%Zwk_6IwK^r-eYC?xNoBI<qKg`3yKR4m)%^PMa!HV8NT{j|
zpK$llg57s(-e$)u3vgVxapT11^LD3?ipQUsq8a?+`g-~D`?cnJ)3t5GoCG>uE|khH
zzdSS6hC`8~{qT|?%{zJJ+vESZDG8Rnxp9!0-=^XG^W@9Rd~JR{nask(=+-CmbLB0`
z{^Ohf&b`QyvSrWAfBnZFXUu9_mU-lHp^sXyS^vqDR~vfHSfn4@ugsYG;=;lwPoD+`
z2OpOI|HECb=7VEVQIUp@jthsQfdtEer$r05<tU!s;u<LOrp&t2g=zKGu0<LulRcdl
z3WTlxx}$sRI|mb~)VH^`-l=%p8~JwAs#PvZf;nc=VXH-BuSaf7V%=N){n@6|dZ)8)
zTEBk(&f~S^>+ipBZb)SAxBq96nwt9M{r&kFCS7OKPTw=1bW+7^_S(buYhLg3UK+F{
zNORWg*_)%*@@$OgIWAv6$3fx1_kG`OSFKuA@#o`lO&y&le?FhjzueB(u55i?TT83r
z$HVrYAN%Wl`sHkI)y}u8wK|z1<a>G3W6}QOhCIyEPhWL=`PFsC;`F39lTNDK-j<vE
z=ElYzN#nM|?fjn~b?YZ>jhbnaDReMlLWq{DiqJuWo;RPFFI^K~7{HNHecphlIl-V~
z)vDZMCk=YuSZvjMo>Qkcr?XJXw#tN^UoPd}pPw1Cj+tJMnOvJT!{_DOg&!+qW}j_a
z_d0Xtc2~DkJ&x_G!xdRtTU&V~t)_@-hb@>;myseT@b-}IR-OgxoGsWkTD0giEwgyD
zPFRCWWaok_%)4J5$a*wm@5DzLQ*NBuciksLTCs203!mAlmwqpg_BNii>fuTKB@5Q@
z9M)L4;k*Oe7u}-<eT9C@R%mbp?V6{_b@xK2yOYk3<Cou9$|RrJ(it&Fi~U=O?Ahx|
zwvvYwls!3>->!^ybaZtLHNEq*^~v@t3l}WqJ8R9C=Eacke%aMMpPqdFB5ghK(LCD~
z78`b7-lo;C>gEYeJw^Qoipy7YeA+1X>5p8H@4_!%Tm5uoToX2AJ&fM7If3=@GT-G2
z`UlqO-rn@~%sXQ<moR579%XHh^xfAjUft1|dPz{}@#Q}m@hfKCN{sG*`+oBz;aU0H
zy*|ub(xu0BTD{=I)m<Eo3zqoTe|vx9-0{`dAAJxt+>oz-?M=+xRrh2ow*Kq6G?B^V
z_=AAOKZ0h5%<rh}^}Ht2_v!J=ojfPJ^|kV)ULMU3?p%K9@3ysS=l1lTH95-jDOSq7
z??ccV!_S|dr5)mI?n>x7mC!ZwZ|oVl7jBHx-tc~Ycg}z_)<EO_sjr7^C79$Q?>1as
zRJ1_xWbEUbBSB5i%}p*#T>W?}Bv0yPs>3`}Vb?X$oAdA6ndRL%;a~Sj_}90$+z0NK
z-#;szzsFHWOnIWmhc}zg7d<$@XyTiEZjPnTT&t<IzrSrvILLG-&pdit&cURO1&4Xf
z4GJDOtXjRg^4-qox1I)uPQ7(2>e;!ur{CS(ee(SI=d0uYYPq_)u8H2>_v=^Hx;>w~
z6#hCHOZ9%b<gH(LGj;mLnx942*2Q|)*4n0Rp1En0(Y?y&vU&S{roAbv4lADNp(0_D
zA#gpeT9>cAx&H6#c%7&%9U)q-|Ns5n`LIoT)1lJxqj}qn3m!Op`uthaBtsy2TTbHb
zZMi)%mY<f~-kE#b?AiJG{>jP8cJuqQ*X?X$WM<n?@i8fArAYC!GY1WL;_7~;{`&Ir
z@P&oWl}{$R7g)`GbASJQ>-T#mhiDz;*5Bh08!KD4fBx%ryZacK*#d%t4|nVDJMr)D
z?~S+L#uOeE?UAwE<QvV!#igO4v0%|6rtP=SJ~-H{siC1D*8TC#=JTIE9+yv+V0(P5
z_wlOL>kg$Dfr3v=Kd$HY+q4G<8n0ZxUi^CPb|J3Tif=d5EsLITgsq->T7UnXjmgL7
zy?kh;w`$cQ4X)qc-abBSe*Z|?=8dJV!)om0zuzh5um1KX@#G}c89r+FYCij}+xIK$
z+nbw*_f&qq)f-v9?CTs|UES2Ludb?0_MA0)cIBIm$9H_a7QK0z0=Kx{lWn*2lK=kt
zdgbcXtv}B)=iJ#bQ9Q1KG4}e)+uQX&R_vL}>SZIhe9@vN&&g^Rm;1~A`ts5_F)^`B
z`q;O(w`bbd+xhzXK6(DU_%N^egYUm9KOPmoxvNyW-}akCczmrXx0p`C>uYOme!W<H
zr}Ft+8Jmg;w{Au0?f+90SNT-*&_e@JQPGgFFr(ySJZWcVv3_WswRm~;8--k}JInp$
z+4$woJUG~FS@3{CzW$G)u$qqp$H7ghrz<`l70=Acc`_?|UE-%FCyyi<K081E{=0dJ
z5^SHJpHDyB#;ehFXo_a=jxU$IJ(mXU`Soh`n)v;4u7MWc?-b9pELK~!YTnoLt(R^6
zoZEO3PfSn@2@Bg)`a0~<M+^OZKa>u)@g_exF|ox-abCqE&UL%r?K11@7w%-y-~Y#G
ze$6M&tk+GhtL<GwC9O&_z9kx#ytwd1dII0|J-SKL7#B*IxLuapApTx#&+gL+2lWnS
z-)4~y=g4w;Y2wcMbla@pvnj{2f>zwxJ^R<OWhJ#O6TfZa(Uq3#&C$?EToV4(Y<F);
zGB5jg*#i$h1&7Hfh;?16{}8~%%%}D3?ZJ$zH@8fA`t^Bm*n@|Qx8F;<{cY}=s6S%+
z|NpN$_-yWP&a37V;=|7E=zMITp0wmqXU@JarAIHmf3)I4q6AxByk6?jx<?B&nA!N3
zYCKp_bYE`MqY5|cwHhw7q%Q_b9ai^QvS&?0MRM}$3EOjD&%VBx-*0#4l!Zn)M+$P3
z+@3aS1${Jlf38~Y=>6#*pZ6tP^w!^y{$FsGl(gZ_VoB>%qq_D^4~<JkCv;0QLd(9z
zHlNrtMPS7$ucym+b?WOScW=*2IJrpG;?w8C-7>n5lOBA08Wbh-V#C_wAML-gKC#d*
z-gT$!;?{pBEi{)mD`e~LpZewf;REkZrY7u<dvQ`kbuW8>NI`1+s}hkXYfqP2Fmf%>
zs>$?SwfoS&HWkL*ZY#~#_Gb1;n;a`y_O0xab%L6RdBu;#Y`>mG{J3{TMka0QbGuBX
zh1_Q0M{fqsez0s}oSy81q<hEGvJPvOU01L8_E;^AzvSatPsXxq$12OyIqnLqv64`g
zG2c<z)34M&vzFJ>P5kzchhJi9%6g}pE!~=uet3ecO~r&cb7Z=@x;7nWYGl}cuS%Ph
zl~u;7q+`w;nSFoX<{Q`kDw$XP&a&yC!oq-#?f2`XXP-Ux_xpYS%uLPV=jT+#x(lD2
zm{?Znv`}Exs!g@O%h-4%4lwiEIG9L1Z55AWcyMB(@|&BRm;2tGbW&xuS+0^?zj5v@
z6E<F{BjNi#i5}~bbPf&{j^38zc>U`d&Sz^PH&;ApWWTYqSe;v3Z^_D)o$2!`+q(63
zb+F4-FocG#T)EQna%`PbN$m1<>-uipin_Ko+P$>YbZ^zyr1SG^WAoB;b9EmU%y5^h
zY>C@j_3?r;|HZ}b{Bo5~1d}#OwDZX({r~sZ;g{U@^z(A7R<HJ48nh;AYgd5Ak=*Tf
z+m1he_`d$Xb!uv=&6f+#BI4qopUut}>ONZWuvI)GFYnpy`~S=yAMbzOsXlMPzJ2?=
zb8|8>6r_5KzTYh`KCQcbL&im=vbVR6o||h8Dp87`oq2dIIzRPzpX{2LokEH%AP+5C
zq-4FGPtN9sANM8MK#|sG&(i*VU0;83b-4b!{r~Hl4l3mD{VJAxyzl9p;&X|YmUy0-
zXFEH?<kVq)`#X7OjH0%@^H^5<`T6<hC)MXa`M&?Z?cd+um&fm~JMy^DEboqojU4-e
zojYgd-rlzGb=B_sHOFlJUv!sWTL1swo%;WOXPiyrlefE*C-}Z>_e`@~sq;3UdG`N#
zsvo%_fl;o%ebXkRrh^LoHlH|Ze}8*weg7w4+3vaZe;&)vIGgtD%uMI?aeFuEe@#0(
z%XNL+-W?y0Nx!_gSsfIl@9*v9P`t4vQ`oxf&4Xjo`A;T!>m~mC^YhBJYnzgf_vxRJ
zu@z2ce_8FPHT6_h8Na+;&-K@5qheHqIN$Af%-6;%ea!g0jkAWxqKz9j-nC8s{_bwh
z-Ca|^->dGwoxguBs8jvwRo0`A7T(iz5`TVrdZ+YyEU06?=kK@MId^w`T~h;U?wxWG
z&dSJGux?$StaaIj)YD>B-`>pZyRkW$y|lE{;aB9|%Fk&^6Fs)cgoalB|NA|1XOZgm
zyJfcndy96z?Q(kXvBDz#PX&Wbo_}}24YknO=Xde7t=q6aaKp0&j=^tMEC`?6;IDN_
zGggiDXvWn4EB18WeVTY<$3&gzPjc^{{+wY|YVxm%=}noo6NgWj^}PoE(BKWb?^Nt%
zVs%`9(D`KKrb*`WDyK>B|5>WH^C4SY)vHRm-&g-DuzZoU?X=lHyTayPn8Eg)JD&WS
zUGef|dgZU}`#s-Zlq=VBX8(44hE*?XzV4%m|8F*3eW|j10mI~Fi5smyJo^=BD(Wh_
zgSqaFo;63~(P?(iKRq}4yi2;`=ga3af3N<3Fk_{2;(^l#!+y3V2ACDF?ESU=+1H)*
zN7KraidYMF&QY1{*^?Zn@<e31@#j;PIyIkT|Lx1-+?AB{XzBFez3JS)`OFMIZ>!eO
z-xB!!xBbSdqgHLbvggm`p8xzNzrOe8Lxq5lhO>DGQ#L&~&zZ1&+1V-?KYqJd*H96Y
z$*l1T?7K~K)ABz@L~j02I&Ht4YvGTbe~z90XpwtkZk~-?K?PGx&E2ni{(Rxiy*=yk
z+n9S+^Ax`A)Vt)L)L-OYYA0H1ooE?*QRnbS)4N<|yScaDb})^UFsb25yz1T0=)7^J
zsf?bv=5*$*m*1VpSa+VO^QDejCWp_erd3O1tM=|RJ<-)1m~~iFPx?t(%*wfUOF7$~
zL`Uwk@5(zc?UlvB9eu9B5}`SZ{SP$VU-$9aHi2Hl=Ch%NPMpomy`HO_tlAoULkrcu
zmps_G>WtizW%YT#%I){H&-I;J7$(~z)FW+j{IcJkb8Ug=XGpGEb?f0zMrO7I3AWeQ
z*DqhSs_WOUUu%AbWoBxM#}qI&Iy9Wt+ubAo|Htu=&``sI2M#5%`P1Dw4sJ+1Jj15a
zNUXc{wBBwVZI%_QZY9bZNm;JH-hKI{in+OY_M^EcHy1v3b6P0C&M&uQ)he#T2B16!
zGIi<Fr8OT8vLAU|n09W?&1bC+<u0M2uU@_Sarb>4`+-Jg_8p(kSxeg2*{t99D{K1g
z$e^G@-`?KtbWw__dbyN^i81Z$tfTGnbtfh&yWjd+o~_$({Bh#9H#fh$y*)km_BKJT
z))}_dW|o$gcdA~mbx{J9oFKo&@2^`EvvX2{fk$epYCFHYngGX~4Re3~{5hxk-Oe}n
z_Re0mOl{Ihm9JmFhJ=P*++Ci(%x7lPvdl9x44Y4#@@f&7GIi?4l#@dLex9#4N;<-!
z<};%p%(6&ZR~J;)9en-ODDRHNw|95Dk0u>_Tb8{2cH!>3Ik&g1y=?gUUA38U7vJ*B
zCmWgBZ){9vZ*6UT`z-QV_3QI9--8??*KeGCO{b`+Xoh96+WWoV<94s@TD7VO)RHfG
z5fHXE>geqJeJA%+ey(`2uzf>>4#=^J&TSi#j&kuFPB=YH*F|aKn!}6s?6FxBrd_w+
z{<m%M`jdb7bi}w9yY((Ae}C`C-@1EMuXSh5nssAK=4J7`zIo?128*=1C>{Ls^YhOa
zi~CosT<LhdG;jA?vlf9HZ_AeX%yjCPvz=j>%yxR3uCh>PNyYgUYu7IIpKlkTBPOO3
zabSjF@{T{BPH(*Zc1`4Fwd(#WrE?>6jy*d&`{niZ_anc@SO`C6$voYgKEHMvE4SE&
z>hF2W{O8ZR>X%-pFWI-ZZ{6!PvAe|%8{F7cs=e%So~cOFzNb$e+zV@6clWn3n(<~i
zGx;{kXy=N3?0xtpZkMB_z;)|AN~=zu>dDSay>w=^!EKovvSxYzuaz4_zFiVzrCRG=
z@VuG1V5e-4<$IyFr95k-M83)$`uqQ1yvk%Yz4hY74=vvP-d;3oR?`e!Gp3z8-~6gD
z)ec|R_@+*YIp0xUZ)UAv{O^zc8Cg6@M$sRiK1(p2J;UI!5nJC}AEkwlzf>GB*swdM
zV~*w>;{qAI?aZ!?tM`BNH7UNeZ}0sV#XHkKRK(<|?>v0$<JNlZ<vTSyP9`0mK3Q*j
z%@vV%aus}f+l3DsaEN?+vaNlee{r(0+GgI_cfE9k9^HLDf2o~>-7a%SL&LILJZiBu
zlFaV@Dnh-9#k)ST|4~_-`0oG7f;)Els}>bp=dW||TCwAf-n-W~=X+Rmoqc*}iRKI)
zGaa?5>udkB-<YFU;{5Q>>GjUy30DsUU6}lP-~ZY-de*zAFFWzpddBT{J2)qHB$roz
z{JlqF*SQy;(+z#hH_Qw<)js#|^?z&gcW;|_@k79Izw2d%Q;pSkX)w-LIJvTS^&Qj1
z$dZU@9_Oc?G?uiBdr)%Apt8E)-%ncx1D?5(;deVOoVjuK)yH`0Ez<kXe^}&rsZXeX
z<}Ht|wiS*W-Jfncn3wjY{Nr!_*B-B$mMCt>;7ap#6mJq*-`Q9h_eM=LRPnWJbm)O4
z%jO!Yl{Vg8vS=mOE=7ywSN!<rONs=Ds_s5+IZfzKrK|15B?&1LMZ1^H3BC1y(ITZ+
zuU=(jWu1Cw=({xN&E4J87cEk{SMiv4ntuGbA0HnZ&OVzf*ORu{a`*PrDMkxdWnH`T
z_AtNw8-xC%Nhj?Bbj7&a5=$)OYSO<SOfl-bo97;Sb&Y4-vEXmc{I*9nrJj!1UuV01
z|39nv|G%zFnytBQS$iT$g3Z#>vgUR4{X<z>Kb?;MC$#?h?AVyHt#j}HIA+dpi{pr-
zy1`5jnVJ7YoED^S^W~gasx)mT_sp7^g=H2pEX|HB0v2=6z4>o`w?vqgl{Gf&IcW6z
zQK!0uZIwyfdUI_b4#is>+B0{(-gR#0^LgFJA18i&b=BtIkH?^4tm^lBw|n1<-_0Dh
z+Vu9kb1ln#>gBD|?nY|Q4CII{K5P2q`E%wE%kBSmvT}<RoYvjG#dG!RUxn9=CHJ$h
zeLnkUPTq9&nLd}UTsd-LqOzoImC3_`8S#G}iSPM*&N`;(q-unYn2(xpTVg;^(4huK
z<_)*sx+qQ5+wL8D_07Km*T79rYbQQ?F;l{t!S}L;zJB_R4T%;qe80ZFZomJ2`O1}?
zzLz)s&D$7p!GzZ}Fk*8WZ}IbUxy9#C^t}JR|9@@Rnuvqe@An)Am0#uif5-A1KJiX4
zT%^@ofA5rq&h1HCqt5SoerTqY`s%Bve9iA3v3x$~@csY)%5%)5^`>jTe!S?bvC34h
zt(Di(=V{L@bv?fRQFLU&GGFVdZyL|+mo;@YU$EUj(M`UWyDB^8LiE$7xS967+|}7<
zUM#+JdwSe~(zcsf>Jz57pML8S8hR%#zWZ75f|)uM(JhYMcW0KZ_OCM9xwnpKr{xSE
zDJPCQ_ZXJiRtE+|G~8SwsG*beVum4)tnI9Yi`|WLzDTg~yy`x=<k6=enlEooZd$~o
zqqbHxY;)(FKTchzygpRisi=wbnbmbO$@{uvXw<sbhaYZOW?LTp_Ga-+t0`Nq9J~~y
zzgczivyAW7F>+kqr%#+cfAneD<FCJ8Sm#JMAMZ&#@!}$nB-pY^^Pe+^F#Tkm^6iyt
zUY^3<gtA>9SpTfhZGQXZ$A>E06<tl{6PyG4jsDch=k<&4zS^{E=7(Q@BhD<F_Imf`
z+R86;B)99gJLJjzyJO>3{Y~Y;_ep|N*7Pm@dp|OIx9IW>R&(<O!){qk`tS4Al2iCa
z*-Mc%W$bd>x!0<0@qFeV7&0f1U$6b|iiG=5_q|L@u9_AsXAu0v?DHLU-z#mUAqnNN
zP0D9A-9&F|l*~L)I_t&T?FQ)!W`Am6-MlQ}(v&xRTps;hj=x+}{@ZYyZ9eouNP0R?
zfnY&Kgiq%2*fxu*_k5~6+ro~|ym|lV9LKypW>cq3tgM(*A%1Y?vORm|?Ac=zSMiYb
z)$7-pd3n#~mfw45zW*mLFE4M-y*)D(o!biDZoPhKv3viUrr1D{Z}0A&-c|aVqy6xj
zn4Lz_RY{kZ`ObZIb4Q`_?d|#M5<H&{vdag=#Ppngdg|Xgqb+p|%#ja2oS3K_dC+R|
z^7{Y(ZseG`^~;?-Djx5nAaEdS>!mw)&P-DEmawn00gcS7P41MnE}QdG&t7rWs!5Y4
zJI}YPonf5Lw|?I*t=`^V9tnd5r{CV$mMg8Qsw&ZT@YdGsP8TImrRK3;=3~`f8H<93
zTeqV4WGoKMv#q{mao{A#EdovbcE36bEY4KF-+SEnyiN1bq=otS_wBfwr?>A%(y<=N
z$)Yo?7k~YccY7P}^wUSbUXMTj;^N}V)8g9>URdaSb9cG^>aewsK0G|E6SYMn``q@d
zt6GO18f?E)#BH2@F6D3=ugl_##<nrXmwvT*mr*DGd+&-BD-IiK>*&~A_O;Yo8Pav^
zn46Tev}Mf?19pD7Bac5OY>m2T(z`YqG~RV%M`7}{otHNLO^{%7b#+yl><JnLw2(>r
z`s%9etX4A<lMk2Y|C_>Z|7SvghKr7v_xj?yd0pMzCr_VVT>k#vnOUaV(oTAxjr7I3
zQ+;#$|K6)wV<ly*cm&kkc(+m5X+fscVz=I;UteC%DL!ZU<L|$++WS^}7iq}eGZt}8
z1ofJ3Z|k)ze)i$&`nsvj?EIH5T$r#o-fzC$+?SV^CvS`>u#hn^Gn+T{e5<jcU)Q1s
zpMRb>fByNa)$7yl?kdf>w?}gI)vmK?$-lq7ZE;fEmV0~JUtZ3M_pGvwX1=(;-@a`3
z+?B!0%O0G0ZPBe<=+ks<+Wf9X0{%;GR2j5*RsBy+Kec?9TdPZq|8cc9FQgW&RnRly
z+Z-j7V!b|g7hm3kwSA&fzAf3XCVcm=lzUrJzdcQpR-M8(XZqItVYgp*nQ{AEeApGd
z#d^xeWpj5<JDHRGV#UU{Q|9N^b>!+_$&Byt3)`z+_hY5c{9;p0<~6e}#Z_OrXuA9N
zt6g(<e!4i_!9B2V_TI-kPUU{)+$9nIGh}Asc9q<>UoBos@ITFnSbnDR;=$J4*H|Y7
zcidf5bl$Q~?%#^W{}Ca*ch?AYN!O%qn0=7d%!GBF^t;4+%hK-M5L1~Y6k~X!*ORqB
z^l)C@#d{iDb-UY^Y4M!iYOt|=?*00Repm0=rS~n*^)GWcurM*J(*F78l`$8tpS=89
zy0X4sax&LXJz=f~8&*`T$=W`Tjjz3V_xpXbL4y+i|D6ASrfU73hi%dt+S-r*d_MpD
z=kxi`KRi7A<8R%mQ>Ow#MHN|g?%cU1Vq+64x7dX{ch2yd-*L#x%UfRgrp)^9udk2a
zZohwS7T^8i^R_Eibsc_q;B6W6%fIb>?aZ!$AAbLh<UKsYFuCIG*6W+%CL4FJUg+Fz
zlz52c--mYll*19A;me4%VW7M}N!5G9&73FCo*89cT4H?n>2cS~CQOYE-&j9v$-Eq~
zuf|e4eBF~p-FmNztkwPIbX<S^6x6j}Iz2AwU=yo~5NF<=k8U<{{FBvulWuNG{quYO
zf9t)~-<K^|ps;!^9}81tZ@PlO1JGFc=Vxb=4>qwLNitlw_ghqqp7`~cVo)vjbpOBC
z`#ang_sCjryL{t><U{+*>rPHq-<)^Xs_xt7`4i{Q_fJez6zgtnWM(gVIyL;$r_=h6
zKUSPLfByWj<DsEbLG#z!@7K*PusD-CJ$Bk$<GT~DC1-Ao`0%sl=FyDTKi?JY+;p3>
zsHn(FX7joko9Am9CI8!f|1V$nQ75CBOO`BYu`GV};A6#(4~MvCT9@l}i|ZeI{yFt*
zO#k1rx8Iiisj~--(SwF!FD`Ng1yja_1!Yn{?%avd+wp+u*}1vh#~&+7^-h{T{rH=k
zn?L^k8&iDN^vryF``1%Gd^)Y~9vZqQPQRI*|JlQK`DfDif3PiGy7cCIe!CwEdegP7
z@9&Ar50k$3EAGnn{QHm*lI5$k?B=g`&VM(vcxF}VJNK?d2AP*suC0l@{A}Jt!5elz
z9x&gl|6d!lGUQs0wY~lP+TY(kUY`F?M3IG+mDOjK31|1wqHDI|yAE6MN*FYJe0==#
zzu)hTQ%(qg-23m>_5GWwzk|w~mseMZhn;@^{db^9>rCVHP5JljK;gacOl--9y-#kR
zY&_?p(7MGkJ-1f*>)D552e+yE-~L*8T6}85b<cJ8+ZSaW-|6NqwCd8!o#&MnZGGMH
zhVf~OrC-dm<*PilFIa4DJ^#O}zjNZv!;gZWzSfxXabx<FGc(t{`D~jgwAB0MrgM+{
zOFH?^Uf3O`KUwL;>&uqu0i0pQx23PVxA*w8`l#l=w@SuLKW^^!tgBeq$-aD+^<EdX
z^Za@SpU<wmTkn76xuZ#D!46ATmBfH2h7(^K$kx78Twu5AvcT1kH%*J)NOesr`;&T6
zBu9<a*`YY*cxIsCYA#LZ0O4z$D!iZOs@(|_b#@5$Ii4G8xKekXY^h_aLF+OtmN#0S
z%y+`3Mi|LPoyrZhcV6wsFWIj3v0~1nM~~h-I%2?+m6g?TH0jB+XOn!?KK`s(6Tg2R
zXkhYv{r}%b9v8~+v0GYN&VBZ=!sf{1!jjnI20X7`zyA39?@f#DnE?uV)4LN4I!>mX
z%JPoY6IYwunPTJ_8!OA#&fI?3QA6a=<BtngY3*7U92+|~)*xMs{bR+PRjag0VwW#i
zu;6xk_Qle<%a*Ct*u{sf$jQrNyHwroziqA)hm@35Mpo9L*I(1b4Q3V@&s$?GW!`<%
zskpd!_5{tDMpDw!AFKA>+{q}w!?tqe%7b~^Cr_V#eWhpNve>xw;=bO|Ua4D@XI3U}
zjp{s_v?gxvtWTecY~=V)PuEWeW#{Mh|J2KN&;9kQ3gnXtn<vknmF?QT&-y}@PHC*V
zQ0Ie>6`SPl8ZXRrS{PtfbGGx?8tX-`t-gN!8acUc^UXO|OB|zw!$Lz_1)Ao}krC>2
z3EN@&%{+hmZ9%7i=xA>fsk8U&=lG@X-kB=FcJ=Djkg%|%?YA|xwX==izkdCC^Sr~+
z=Wk}1csV;WH##uPK6~w1^|2U}VD-&6d(NgEHt2b?S3{_?X5q6JFXgNiS<aq4ds86u
zcwgr0>N>S60mm+eKeL;%Gd)5_Y)zQ<!hnufud;N+xPSiq85#UNLdUJN)O5nP?g^EO
zrDk@Z+~T@8FlnQN-F*IrTeoh#x%dCfdpC}Y&u-4ycq8DMo|Qr$*Zku}YxsWtkXbGI
z_KIWK>8yQocm1)rZ};?}-etD{yM+^K{%F@N+f`!h__L?LYToKtQ6GC}+1qcwzwz2V
zMQ_fZDNi-#%Dvd7pzpzb>aU-I#{VBm#&XiJU5j;CQ%}B3d3|l|o{z_*K?AhMO}9_=
z@M{auI5Pj=7xh=KUl;C-2?+};x?6hPF!$CL<5<&9mI=inN<U3ztXXy~zo)OXXv)>j
zxgv{Bv-eMb!^3eZ_e|jKS4BUrTyy3+HAl9o$a~Vcx6($>?OHd~-1!o#<M7gnZ|P<I
z<G$-Pk8wWzxok$lwFF^j?$A4Ct)~)>M@#rfwLF@?C|+ffzvuqCzdPRV`@QS$?23~&
zTTefI1S-LHm%mpN>I?`Ab1N+TICI@Jmr1VY0z02PG30&!)bes^hIs3;(wzHxsdl|z
zr_`MMafK=7x{=m=v)9ixBj!15GAo%Iv18|D`4+A}$+2DA&0c337Zlboe@}>6P?Y0R
z+WF-3!4#v?*L%w+Z5CgC`Q(R(htHRos!Z~mSRA5cY3UR8dFK7*1?3WH&I@L45qp2}
z_{o}C*RK_6{#SimEHY(kvR%2<idQ}v(bjhtt(_3W^3ku3GjFxi#&E~rg{3RLoD<Nk
zlzH~$KWG6oXxE5m`GcN34v{5WgHL7%tL>FbOnLNHQ)1QRj=W=!3X(V9wAO!May|T$
z>f_+*CcBNNUti<f;TkSH?_8O-tgKy~&dO}}SvyiDlwArv%W($TOwLm?mzHTQO<vmR
zlq$MvwMuQk#S5nTTNSxW&lVaKZK(D8eR9bM?UzRCFBNvzvMu#qX*G9ptjI0@nLBPy
z{SbEi!KNl}u02o#CvBc{SwUlac>C74_tJ+Llv}S}<}SY2lf&fPzi^jw#O%c_!k&vJ
z8kp_goV#Rha!6sO{;jvkBIRO_R`lLkxJoy1U7snVy%?*j?W^+<633pa<!E<xEh<^+
zx123B)RCj7FXwLbqc@$?W?YtK{c&zt@RZA8&oBGAzkR*Qq4LwD^yS41CNJ{a*-@2a
z^QG^QpEqoSMZAIEW45PXUhP^OaJFj_pXgGpCI3SVX68oZ>dEj~yH?6=32E`LxRGfS
zZ##9tvTf4}bV_?RC~%mW23)_<bMfS(xPu9&UoG9U_l5H#%|e-YH!e4=a|>&CgO;9H
ze}AJs%O|zf%{A0N@{EIP;EEf&=ax!un<O-Uf1976=C-_u<Jb01eS2$x2G_-9UiWH#
zrk2U9aeU#Km3_&OL0emU<+`=XOQlz=^4hU+5`XwQ<w+i;-0sI_?krt(>eZIoSTUL1
zdWrK2_~z&(hL(Jags+)yDv~leb^P6dH!5o;T`*j9l|xx}N|fPEgPAomeAFIol<?EO
zVs=$H*z}F0@tYo@;Lw%UTV~Gx`^@m%%t~AHvopJtctl(k<<>m?c>lkC=sCte4_<G7
zxH0>sO;?iulXFAUxp`~lH-FkNvs3u|!8}7}?X@cfG8yefv!C^f&r3Miwf07inb1d#
zr^?HVe*1<$G**xNb#K3hgiWWS?dJ=;%mD#c!&0|!{ER%fYboE%V=|$qv{uYIbnolw
z$f$+J2_F>J%xb#0s>f%SS*eKo)jG+w<<GC13apI_(x~csDzD?DIp@fNPf9KKuAMW#
z7IV5?Ugv0xjmZAO8T0=?d#}<Wxp~<H4=J9R1$O4Kw(SL1<DTu`G$rmY+k<V<f^8Sw
zobN`?jlRyya>(LW(9&GF#lg?>B;RH)ixHh`sy$6``xE2BHv$<KPo-VjdTm~MYo@EC
zp!@o}yXsc0x)idt{pWfe>sxk-2@<iAlf#}zJ&ScOR$RQ+KYo2GpUlOV_8AIky?pT|
zTc_RV4`tZ?Pt*&Rz$UHxsLA@&e>>0n7|Ep#^ArNN%T45IT%-`>cxGqv#~*D85^Qnn
z*=KKR?J7`iyE@HGIIr|;h>>6ISwXu$37W>sdKVu^b`BKzmLGloRR8t|(otQC4lg{j
zR;|v+;7cz5t)X+`@S2%3Z615Isa^W5{W^QKp@{IqXKYED8ai6XUY7-I{GEI+a#Le>
z{+`({7ym9yc<p|s_L**2zs%_uO)B~Oe_8!~9M|<X`QwhcFFG~a`$7admw$LJHevc>
zR%KV`SX(QphtJ&3l!>dk9(?!lVc879Hpkjn*|gp4BFu>|Os<3!>{uzeT)+L$9~qHL
zheAxQxVWZ@YWj$Xh&?P_I(MGx<G@u~+MLHHfwn80{;2$g<?Gj84K5Q?_K#0RCqCX7
z7#{1H>&&tKcw?SKhU|*Ftp*=NzC4|`{(krVCF~2EvpdvG*6n|t_e-qm=H7?O^J+i(
zdR2c(k14!0$L>nR@pHQl&v=v9)&0qErPuP(Yr&jSucR;fXz_mis*%R-dbIV1kCX=2
z#RO&1saIdDYF+e5DW>eT;WSn*C;y<>P{(AAw%gO49pf52f@S4(s@f;4w%=Plb;A4a
z3l;^(7Iyr4X5i=Fu;`GDk5<bp>CR)pD#?kD4z=7)JeacaVd?I`s5L@DhgvQw#k9|s
zx)EA0arM!X9XkZNJw2~)ZT`loDBz}L{a!}*^SLD*X2k~|eF~g0-+9H#d8b}Q<>={q
zDwjUm;CcK_p1@oc%c+MpISBY4aeT3SLkMJ(pHSzL>JOSMPlfdl-(GI9FFwA}!SLor
zvDhj9@>DwBRyW?5w>;o?J@3_<(wlE~osaQeylTRwQk|^o)rH+Rz3wi2yXNMm={G00
zB^+^H6SJ|aWl`ca5dlxTm>*X9`!=#nzU!tl>CvCl>z{YcoP2X{nWpw?p1j>@V(V>w
zx?hv!NH%xWkXW+fzstVD`)5@a2bMS=e7PmLe3yjfOQn7BcjPypJ@|XQ{lR&!thDae
zF!o2Lo~t}u@ptA<oyh&%n_Zn}*u)fnmi3XDa9rE}-^9?}QU<e4j`_}U`uooJN#Cv+
zv-M{?Ey>g8|9^kJgyHKlrdgfN(e{t{_uqB2ka4^o6Z7}#ZOi&U0<ZYrKKZz_fAjKX
zlDxA1ahBWfS6bULOX=JGHSo5#Y&g4S(xo{u^~d+}oV@(=@&7;aUpX6-3X7hu{T?*y
z)_G<BbIMD!Z^-gFEi@4BUcN=%)>Tn5@6lTE`A_G(e!n;*&|qe+$n<F!SDQ?qWF!B2
zn*QSpjm<rJAB8imPAJ#^=dQAGvRGC8vUI!A^QMm`B>~T^-|d<k6U<!y=jnUS)i1?a
z7Y5AW6puga8^5!o;Aoy){Tt4*&@0Srs(;TtTxY#*`GIVEEuC(a)Awpy!~gB%=|1bE
z!8JX;p8MLew`=b2X<xE(((<`-?{;VM$?I3nt<2xspSD@3U(#p)7p--BQjAWE?>Te2
zJ1o&`rlyoppHVQgXq6oQol@a4r9Gd{9ByGcytAZde#U1<x!<?tJ6sMO5wdz7S7Um;
z_9>H7=<dot8<USc>AU?R`pv`U^7yoSv*lBs-(?NIbKuR%=Kgc6zKN%fh8O)iVI)`m
zL-1QDd!fafj4Fxv|An#lnQGshUw1kFWqQF2UX{t7W$$iuuJVY^>2$dyr^vE0glqE2
zl2hCEgZJbWSy{F&yT0h@{qz4Ed_wm6zj`oP!)?xuK8<gQ|9i8_7Hcv2hI*7e(eUD!
zb5H5l^W719YgC{2%DTL`z-RV$UGk$Hf+6axZbqGXD`Kj?p6!vm*8VjAU$g4!YXuj#
zd{jwJY)X_>WO=wu?aaKmJLT_tS-!7*n(x2wSl?lB<3;~H+`h*j`&|Fue%h3;i|#i#
z&M{6@YUbv)ct7d%%?*X}%zArgT8j51eqU7{@#SLPy1kFHd)~Z#{xx0x>C@@=QXZJ+
zE6L;>n|Qgq`rV#Bxu11n=j{(&`kKvgZpML$b7xx!gzVV0XU4B6$7JWGnb+?hu>AX}
z;`Pqs6>qneOPKCZ_ZHuG@}67oZ@E8makDDE=+{5)WtA`ZJyqDS(rd@ld8c2@&3?Gk
z{ok=$SLa{&|H#qA)Z}9Jr=>CWziVv7=DOE^*4nn|qRoe^H?O?=(_#PjX{FWIGdd+s
zXM2xKwpzi}HgVC)jT5Wd4^51fz7Z{(@@PtoNlDO017-F2(`P!FE1t{ue>&K{+M@o^
z2ftT)rbwsrEm^p4;=N$~$H#64NA50=Tdx~=a_MFD4X^H+TMBF|_+MV%Vp%d_|Npm|
z|Gqv9$eU5i_ku~JHDyKTQ}1a3HgA~n_SP89zP?T+@rb1DQ=4-)e;6u0cX+oa(QM<c
zyL<lJQgS=HW7f^f?i>I8HJ|bQmo|^I<%ARyAD<gPU(Nk)v?ZROFSnv!^#6~B!==R<
z3%Y!x`3nx}h9`!;?r@cQQTdK%-JSxb`8u`T;`b}(1}<jbobJi~wz^#N`&|>^yBBVq
z+07reF7elle=W14&$im=bH~;6G%IR<uYX<_v(oC{x9RqiJU+g(=#%=nxodmc=9zW2
zzY|U8U*sqj>Rd8=l84HT{q^>?)!&XJZJhJ+6(m_+n)5_!yY}hVlcSr;SIqAVYmz%0
zFn7hq!wZfdd}H@3EUF}?FUiRG-yP09KVR*?vF|;{*6i>FqF+)y{H3SfcGgJKw*CF(
z;o*L>54YC&UXihAG5G57zA{wAXy2OFtmGRL3ctLLbULx{vhjun7p>0A_q%t?{oxZf
zt2uOS?d_Wz(m1{E#h;7iubW+5^<zV%Z-oux9zX8t^7qR(+&56`mF_#fs_<gZ=ZNpy
z%U5LcHD@K?IKj#D@;Y}Shg+}o@kLiZ->7|IU~QhWY|DR+Dv=*6FY;FL%OBnHaq&lM
zpBRhBHP`;V)m@d9Wl^!C>EYvaqc!#VAw0|a<&Rn4{=Rs%_@(70^LewUZ8Dtts36yP
zZ<VA1OW5*m&dKVRS8wUK<s5BsX+pxaQt?Z>x6Igdpu|o8_lq6(Rz<&G-rJT{@oM7!
z2isPeTr-~#Z`b!yxGf=M{c+J3XQy~3Yv_D2OnVZrO~=|rN$A;ug~~kg^@X#o{udu-
zRN7c~BPQ>dpGEtnMM^=7+dscMe%iy$NychRf5n+2UEjqPNfZWEHQk!U73AcYEm<-7
zs$b=MIi0BAcK+^K8t=^~bUL#?EStIEcJ2#5amM}avC-cqZV)dx_~oR?v+5Z(Z<(gm
zZJ1Wu{+i*1Z3aiO8{e^&jj0zjxN^IHq@0#Jb<_1y<e?LdwimQle`NVpaPjA5O}$T#
zXVo#Dp0+OW`#fhEtB{i)J{sTZ_4K*^BDC(z9^>yUiUO;qCLTU!yd+*fXrUFKtlgoG
zkJAm367oX=kNte?zIoR_<BG8N3H-&g4gXF#s-+R38@5JFQ^Q+j(l?E+re&ECYr{Z1
zq0T33j$HZrV1s(xs`_0=_dJMr@AK?W@kQyd<d*LVTKlA~zRI2&Q?e^PSc0u}Z~KM)
zDXiW1s^-2}IJal-leC(HybCm1r29U6N$#I`;^py{hs=S>Td$au-TSqrST!Jej&|tV
z`)3!nsVr8Tcl?Lu^T6r?_WeI^)^9F69Om|2;-XT)_9x+?C4W~u*0f!!EW+yQJAK`j
zE7uKJTKBRA#U&))e6;k+`*kUIrcQrou=MQJ(%AE}Bwp|8Q;?nKadOJdOYK!&?iYPz
zZtk0QeBKQA{~LuRFRY%is(H7-+V2~*7P(6Jg@>4bnI_bu*Ipvzl(6;p48zw(RnK-9
zM#@f^HuYhtwMB$#fUnsZuUkK7EX&MfwBFmgJ$7zj^tp+4`5cV_QG438xfVS9_$2V!
zVv(Z;8ai5wmz-C)a!qm7nX<IFYU{mi`KiCRz1{Qw7<Z0!!1?Cw=TDu}ZT%v_b@#x9
z%WqAhWO_9vjq6xj6b(WOesN1!uHk=uU4QW;wo~o>is#dpFH(9OpcC76ftQClB-GJy
zf%?~Jl^2ZER=Mq1*tA@(wkPG?#mey8C2wMkH_hk0n=$k8qNAEI*H5<TT0c`Si*oXW
zpH&bq&?hYCCG+|E&zZrpVWGuekKKCjc;%4xhOc|rY-<>7i(a(+Y)MSmR6Wz?ql{hL
zFMmF@J3aEY0zF#}%<6Gpk==E<{z-)!?|b(hI~!e2d2oBIwGl~Yd15xNKeV+=R{rz!
z`+w(^--|vuv5SlE&28sonLn16Gfa)ugk^5*?cViFqW1Uqg=x>(#Ogl?{{Eixc&)kk
zBOMLp>u--3L{;XAF?C-)e<RR2;_tugknR{az84esEnBoH|8SPrzU^~rtG9jrHv4_j
zY2Aei_Ve%UJ#}M4mf&jpo5!~Khixvn%2xI5UGppsA^(|@_K8A&Pk+kD%CNC=I~I8U
zmY=_#?WeUmRgZJGy}fz7>$~`-Uz2S$RxxShwyr2ET){f)<go@vXGzm45oflq#~*J=
zZ4J8Uwd3@c)oG=dmWTJ3mA<>UnxAj=wQc?a5^WBftzO(*ZoaU<_^0hJqXm1WbTPAY
z9A&w9OxfmtfBlBC--(YNR>jTzE_Bt<+Bp4uT1KT|=+&!BcvyEIb$U9{S!T_eqw$y6
z+vCpOey{iJK;vJXy6?HGg85E)7jL>QuAAJHmG))F#*n<&!?SXy_Q)DD6?dE5kJ;XS
z{eIo_y}!S2d~tXCN}q_A*SD+hwa$uvZx=De;_KN$wjP=DFCPBwNO=~!dS@HM*;fm~
z&l}mr?VQkWck|e@QtgO6HKv>sx0b&6w0`Ye0iEqa@G*ecWw&qnW}OQPS>6BQU-*T?
z?iUt^MtnZrFzLSDS_RqioAM_dgH3+*NgVG`)IK3;H}g@$WUcn*ISmJ9M<2T4sT}d;
z*yc6S8OI-+OrHMSnKehxJg&cQ)v5}nx;J{#6Rw%<`fImlmD~Tq;2GO$`SbS}OlM6q
zUKujUZt*gs+CvFCe8P*5^*)%|n*Y(SZ}$2ai{8_EHb-{r+aFaDeKXH#-TtRWfAqB9
zHORN0zUEbSQNkq4<XmO`<@_54pKADc6gODR`J$=+@bS|8jZfC5h3uZ6oj<ep{L6$>
zJ(e=7Y9^`w|5oh~aG~0ySZ`KOzgye4{~hUaKW)pFvv(`A&p+L`Z01LHrW?9yEd~bG
zHD~ty%6evJ+JFD=`}H3x_D=csw)t|R>Qt{~TleZ}ba8dHnF}09yLKT&x^BOGsAsnQ
z=7;m6=G6RT-nS*ZNYUW-n;(1gLe8GqDSPGTCdubXqNZzKO%prxw~l{RTH-6$VvB+w
zo^pR5=U)(M{UgV`bA`$LiURrUY3q-K8MIui{XJQJ!^d-{{WpH#6`pnU>>Ta$$KP%-
zuKA%6X8+yj{GKm-{CA2h&10%1_4j<!`~Uf3(uFr?c0Q{*#r^k9@s5wL%dYg4?r%60
z*Q}vmP;lVN{6<@D10LqHDN9$ZnRB7h_s{+RddkbRV~UQNe$hR({Kul3GHWuf?z3Aw
zv-qOOAGy|*lifD#xVq=``Ok~8+@H^KXh=#s)pPTWoym0df1j*#^!VTHNh+$%n^NYt
z*{)Uob@1C?QA(h3w@w$OpG}M2SjgC9#$H|-;^(^9Fz2N6m!fxv=h(BlxJj>6_@d#q
zs57P3DrZXC`Q-X7_J+lrWwY1%T2^T}8i*WzT9$Z8<j_KaRX5uXr>V*D3i~&neyTLJ
zConv`KftHr4qp)4jyZloA2~Q>=kQri3=lYDHph(n&;`!crU==*HEF^tU3y-v(rS9B
z5w?7GQ&ZELhUs3%GPAgnHd+_%mXTP=Qd(MiWU^IWzx@2;Hy>TK?6?$Ux5V^r^!0t`
z5BvX}yuR*ryN^^%QzD1ZMh?zcg&@vZ&Vq0B^cm+{OFy<0<PMI$-=G-s$8x9cQ_e*X
z6?l6ceZ;K&W=k%dwc+-emRZd=V)AsB`RHVFm|eRp()FmIa>J}pp^F-)#q}N}6ib*2
zUyj&nl=q5<O~lgh=A0P@8GNn<JMYdg*?QNaMTs*;Pkq*^30_A!j(mOdz3)}d3(uyr
z3v=#%3291D>AU^((2|uZixRV5zY<)<Ip3ar`Q4_8UYG939C}rfxP6vkkmsw{uRD%S
zwzw1bCyt-%<p!U#zP=KxSl1u#UB)5xDx1^wQG~2s5mTLss{n7ihZL9I^vy^ASRQ@2
zH~Po#c)7$6t>%S~-*_fSy!_@mc}A#Xaj@{JqkevV48h&ocE<jGGp&T@oeTTtbBXFw
ze5uc)s(eI}9x9afo}HmncE``VdwHolyV1M!$yHO!<af2Ld(ARiO7i0RA5-22UJK^T
z%6{#!b;i-vvpDzLH=NCKld$Ujy4B^8<oYx&zUkuguUBq)<#Zt+`sl;1tP&BXpROX}
zPZpFu$Th!o$Hiclin=+wvf!e13uj&nyr@$qZdsTA%QZmD@90A35Ob5{(jA(b+DBh+
z*t#<R^^Ties~&Pcd9Y!Yzvq4rm2~Y+m!`YB%Y)<M^uB9=3b!X~o&<BA`XJfzG&$*#
z%VNVfCl61N;10}fmAw++xMY#WDo*9N(}`b|7Rhp)dbz~wnoYz>j><^)*ew}$qJ6Q}
z%g@f4`exq~v&w!=aS@TPD>T;_M7^J37ghGUD&<poh<Rw}%?(d{<?283{_@!~#d_V$
z*28O-oet@n<scu;cJuSgy&E6R_$4PjN&B*1;hDM**S<1e5H>8!z3*47Q}|^^)}l53
z_FYWx-q=0sJM}6<#;8^)Ds!&&>f+a*Zq?+kOg`qe)7Sdw#eeTttndHk9)3*avcJ|=
zcC$~r_n&e7`t4zM{O-9)`!~nl@{uo3^|_RrVgJ5wZuW8M{pTz#J;N<u3g`Z`VR?GW
zHhbs)h4stVS#}-GQJU#xn089%-TwD-r|t;WeQf?6xnZMR`&}Qcc}f8$6QbYm?e|{J
z?`%A2P0jPre?M+aezTrGIh2PdTyOimGV$y6scL4yucuo!)!Y55xA}BTukP!M^_=Ve
zvfKZ=f4no{=$ch8s=5wdUjIPc{*T(W5ZTZ3s#=!Mt8d?trL^?*l>S*q%J+Zu{p8Ae
z|IfYpz*u>esa^{;7R>aSF}H5t*ZKP&O)`ACd`Y~@q|KtxW@gUuE1`|wr$pqGCD^Xg
zSfE|@XxcxHy_0LY*m)Y0I2XMB=J)zi)C`Mn#jEp6G>W%)6|V`n^zHY?Lxs25WTVTh
zAAXzhiRbWwnkypvuIKaYbKooMzLoaBD81CGYF}-vOWNhvg};_>{<Shqo4LLJjY4d6
zdi-(WJGZ`u&t89gb+P(=>lcU5eX6-tlKZ^7ar=Tf>~eqP__p(JKYj1yomanRC|J!i
zo_qbfi{H0f-wMO)7asRj-8JdOXT_;K!GX6Ad{`O(@YBn~H#R-1omcr!a!zB=zWU!G
zQdwFbzRDc5HIH*#lcoQ^;+baM=imE{YmVdx@UMHazy9NXmruV6|9_WTbVKOb%ANfm
zcE|sf@pqIA_mjN!Eb3!|vFTOQB|&VR8OFQ^1vcmHpDeg^=Zh+?{6zmZYR%u{EBo&M
z`zp^fvr2V)e6?Wv_H~ag?zcU6zVO3p`8x;aZoltbyklaezx}bk1(Scj-!n6&zqwJ>
za`}E{Rp|W0@+o?&o1@k*^k4dN>&=v9nkmbSg8B@;_+9372sM4fXB_wY<V@4D(!2G`
z{>a|E8z+;O-|bM*rggDvUDf`7O39mUXy|A$`7Js7w&Ai;#<KcO^<7t%-<nqA_RWa<
zLJp6NX>WJBU&=2_XV7U1x4*wr5t`4#HRZeffivD|396oJip=Y4Q>D+I_09QnJ&Es#
z&z~pN_K#+HW-i>}zHj#Lca<IWf3ppz`$=`J-5L~``BCQnCAZmU4_?U>Hz+NT+@%)~
zDykfH`Zu?D!b{6(@_#-=^XPp%bR=-c+_JE@eG6~Z#F%7@9cj*q{%FR||D^Ea()fi(
z(rxd`s>5nA8H2fN9#nhztczZlDfQ1m&#wNu*SQ&I=VhH+QKNNa=H7ivoc*F4&ZXu2
z{9i0+cDs6k{k>GXQc=I%8`dmH5a2PrbRp#Yz25m7)?0a<Ob}0-%+=-ec4G6IsE>l3
zZFhAdcCtmEOxRP_t6aL(!mF+D+yc9sm%M)SBu4D7`F7n*$e%0cnxE8S4HxILHh*T%
z|GewGr`L5!%dbM~EXoy1*|R0h>a6y1?R~jL&3f~?{j<AEWjEivbIDg%?$hp*-g?kc
z(~~O?vMi7DuG#eJ`FZajwli0GsVtv!StY0I&Hh~dZ!`64ty)uAw}iA@pZ7yF``(7j
z0m1z99AoUN#r!rmty+;2x3?$$&269Bvx-SolJ@_8#(p_bx{UX(@S_#-4wsnTTs(2*
zcxHU}e1>{IzMqzILA(0yzjjgGtLq|>`(9wxEvsYYzs<no5K0q0_DRh7^&w!<<nwnY
z<i*)`K2gckzPW7w+>G^~Jw+?O+*tYVD~Cad@!{h>$;Vn8XXx^=@yee)@JT4hxBUWd
z^To^h508J3Js({Ega2NP;Nm4F{2|<`jW$YCxg)n`itk;<%l_a&A8*c`z@rP+|4_Vl
zU=siJ;0x6p%+=3>#CazMT%D%ziD~uaqhHJG=ALBs+I01za_N(AXVNeJ@_l1@<-mn`
zbr+I#D!#H_d)HZT+;)4=lhT?vW=&{1usl2`%JFCLy1BfE9j@Pg=N~4*8X9`}V}<IZ
z%~P*6J<BYgR@&8av#e?351D&LoFzIuvi8ld*U!H4?$aTIf*p6hXvm-7HNSJ>k?8-E
zo6qk*w8D|!;O!yFGu4%h^*^7U?+m}^HF0xi`n-xsQfJSb>+KO({L`TR$5DU7>2nX4
zZg$`N;PBOms=kAdw_Q|F5aZkYU~2fy=vS`(DjF{=8@SKUKf)@0_St;<zsCE2{I5IL
zdp*wM?L1$bk1v8ne0Iom|6Cy+uh_F?=DwQGY7p;yYF&6bZoU7B<vJebB|#en(vD^X
zw<Jom2w92kmY#jK?v%9N^(~VwyRqN9XK$albG7mA)l)5O_lr!v>E@Vdmls~09?`Yx
z;dyO`@4FZ!41U$*?Ra8)?`Mi?^tZUfI}(L|ey{&8xWE31r`*ra>kU?GzWEol@Bd<X
zqvAUbXWV*b=kICSZ~K$$|M%UZGi+Gz6o0-dV&M~FahIhgZ`vN?`AfB4tQ0wXZhzOw
z=YDCQ-n{-%7d%6Ke&y8o|3$o-&KJsW<?O5dZ~Mf;N4GTi<FhA$6V5mqd)@}EE1c+|
zGD)B7)VFm<0__4huAkp9t?&5b8&@oBxtMP*uySfxw|vFAm5a9-c3qV{kXBxH^6+|>
zPpuEOoQ#?KHv8S5>r;9c-`W3U%27~T<(2fT4ZHX3c$1sFj%(rNI|1zmmLIBOG#pH>
z_!L`o-4;KPXxc1bVs<4cG`2fWgXI&~iSu6<Z!<aJp}NARXvVPsk#dFpMNKh2T~113
zi#BSyyk2`kY&P@!>&>egE=&#ymszMNaxi7%<oJ-5<9WHUmQ81SkMK-wINw~>R-@~o
za<})qb>Aj|rVNu?=cLw}&RUkZ{kHYj#zpFnuUxs7ym9x3-*<A~wJC`vzdW@4rhH++
z&o$hIxBbrUeRtwp>6!X$PG_~VPyTQ=FU@A|bWwVFY|f8!^-3>48+$ExlF;aCI+|sx
z7PhwakIjY|vpLl#9Z^cm>FaBK8my?Gy{fB;r&r~r+8i_YsF~bzZ(jGAb<!y=F0TE#
zi0fhL6Zyqw-hE0auZ&VZB!1$s_=_DpS5p@5aL@1Rx6IqP^ThU@R%@p(nyIPr*3)r0
zr^uZ+`*+5-s)bHY=Ra%^vA4?f_O@K+3}v45YeLNId?}}<XlBg1mMD4k_x0lY&Hv9W
z_`Lq1z1%ML>eX}Olq8cszCOPpA)fK_a{u2`nu@woyYHu5eRepe>~8qW@CQ#nY}>l$
z(<A$gZDNJ@S{!S3{Mc&t98_Uls<rqx_kYQnxkm%nUq8O@>(+=h7BXcvamK&-uM2QW
zTFhA&{7x|%G&jwE^Zt^edsjGb_;g+I^WNuj;-=n2k8^q^zKi4c*X7*a*8A+)v$Cd5
z#~&|@-(NSE>*R!M8sXvLK{xKqIXAs)bN2BvtyfMW;?EYW&|0-9?S7<Lh<wuLX>VO7
zY3=>ZzhsZQpqtU5d9l@oma!_$yfSN6EM8Gw&o*_s;414w*6YtceyV!=<ZJ6We~OP^
zd_U)N<qP(mQ~0I*cf1Rj{rO*U+l7c|&!CNgard~pvkEfq=Sd|u#XWtxW6d+;yKjX$
zT|O;cRP4A@dd*y4HDR^MofSH5r<VoYzTw~#y3u;h+tn*IZnn={q;b>y)Xetf27T`u
z&v89<O5g~%aC7t5<S;oi<HARlH*_<D&*j7^TJ@})Z8?AEJb?u(URI@ilBuoGepz6=
z_O&WY8_zzimpzx|PDdzB^f)Ki)zu|omLs7ZzAoo;Pgv;G#KUcYAt5D)j6_srKluu>
zamUwAn*aY@;UUfwx_WN;c1_PTTmnP+a({RH|9&}sL#DrJn(u<DQ$N1(|C_s#`OKg1
z-}j$gIHhZvPr&_vcIA43|L>>Ut8e*ddM|d9oHAqmpP%)M?Y{({)3rSm|L>Fex?Sqi
zZ6}uA{9OO*=y}e&Mds}GRrC7!4(9*+CeF6w@Sj)h_migIYPSFT@%oNWOBU!%^iWwY
zY%1(+@|*pxlSkW^!{^?XO`bg2*|X5lwyW51Qs5W9GmC{MDqJ<)dL<-uL-KFGEa~PW
zN{-*&-8%VhFMEetaAxB*kHFZ%j@;YZ3LkigY94mbNK-q%W76)tS<R<c1;j`D#?~_b
z{{AlQ_qn?!?a6N+Et9I6H0xE?tL*F>TZ`G-wX!|cl4b7gpDt@%ep#ezk%mB4-jZ$G
z`nsBb-`JSVf6${RAz9XXd;aqS6SYbf&o0@vq*}@9=VVX`X4$pKpyWjW7Z=y2rcDA(
zO-)Q~e6qJvUmLpya$J*ra#Vf(LbIf&)2jC$y(-?bY~3=$zi<3oD*m<}|M_9Eb4=}T
z^Pd-;zbCA^dFamaXMxvcl&@X!h`n-#H@<qpYkfKYv+w*P_Pr5yn_F!s>^6OI<lLCT
z({JaPuasX|(X?w<l~dr{!smYH=k5RV?Dmc?lRSAO1I|tSYaO|}P(JkcwvB1m6RY?n
z`~n^FK1}dXnRMB=y7yCNT+4h7m29`)*=Dm(o;|tHaE{VMuSJo4R#yT;d!1)b|NMJ-
z-0__4lY3kYb{1)Tid?Z~m8wLa;m1qaKVQwsTvPL1^x58>(D~OUg7dHM_J4OPikY_M
zT0|h{Rrx3V`~NU2CLZ!o^GsS+e%@)7mUi?fpGOCOuYbOnb^V7|)vGt&mT^%^Eb6LG
zGCXwUnz@kotR3IJtuH+Iqmg5$c~p_noWR`8A;sP*lioERf1LQ`#YLT%9TQxX8jmJ{
zI2AVY)`>6DXpvfdBV2gN{}#)dugkJGy7O%+yJ&xNfwPU!lLPZ?IfT~4?(C4A<#^@7
zh1at?js$VA96sjz<d(MnV~4yc;`d|xl;oCfsS$WuTeUHz*mm-{xi0IK<fF?|)?5hT
z`YltuZ_9)yQ;IolzKhK^TX(Q@$;6Jl<V#ad-q^a%+#NKWvb-%%bcw~x3!X=Ujx6$g
zv{8iNW5%Qklg>pN8ME3vgRMohf={aW`Wh}fyKCZmyC0@LyzCiaTtB|?|L?U7ZrhM{
zHR{!DCkdygmsaOLbGQG)Xng+V;ZLF297lsdvx*M|oH!2o`p&RjdguFv?>}$pP7mH#
zt7_eUWm~0n?hDOXucTiZXKC60xmYe4l9BZKpIYAD_loa!{xs^#i9g)<icRREfMMeE
zW7Y4TXPcjYcwn2qhK$Wr`TswvdGxkzD7$MV6y!Dy)Bt=Ed~thz{HfEY1)T!o;{5LK
ztMy;Uz}S`L6fE-SqQu7~enx`I(>&HYi?}+5hU!dXczoPH?bw-_ck15ny;F16b;nz^
z7KZ7kn>?9agD*!PG?M$ytGxH@D&f}~lh1#bFqyQG**kK3mgak%*v{1@2EuJq=fBjO
z2FgyV%hm23Om}TeiFtmsJ$ypP(hqDhA+lL1A8rImtvVbcm-Xb)Yqv|Q>*LDw3#?lX
z$i6ta<6*OAiHe4<-hsR29JZ&uIKRI=+f@46le2eT?XM3H-K8sXud8M8$vDlZT5>vA
z#Ax>%^og(~>n>lj-f-YmiDA(liK;Ie3*9{pOa0Vqe}7SV*YjZGK4$|lcMiw?f6Wp$
z3-mv(Qem~U<dHL+vFTD$)pXI#`wU&Y#3M{H1GOzZ&&>#nJQvdGa;er}_SqgO)2?UF
z(%gEbT5sO@x_6J0gv6l((i&+I>z(e!Y@QMS_k4YFzf8i}dr~&FtYXWjGd8z<y5-(&
z5hB0nNb-g+@&Aupe9yFOesEi&goLEMhSFPM)wlI7N7u~kWSu?Z`<LJ!@Ah!FGr3)R
z?|J+4yxV!{Zw@G_`AeREWdE;OYhlqv-}{Sn;%CKrs|cN(xga`YN~^$(wNeY0X|yTs
z_;4z5fksYbb;oCY`KFtTHol7A?HA#9t#g{9z!r-eMm=kGmMmG=v|PTrC)Zs{Q%iHH
zg8A}b%f~C6^#Vj$qmE=Oy*m5Rv|G#NJ9Pa|ta+?|QETF*Upq1?4Y)P)J?yND_&2*K
zO$-PKIM5?$tf8UtqDby+&f*0dI%3vKr!Sar``!NjLT0YWeHDz>t67!Z`PJ?3_h%Iv
zJU_O3=d(%05ns=X|2!N&f5FKU4>xt|sT_Z+qQ8Ga!}(^e8E^f#y1PE^yxVZBGC0kk
zFu+oJdi{xCYuh*MiV+bLeN-0B7<~KlZ9Yb=$LhXn_mf#yPv@Mhp72^s+Y~gy`RSsq
z$tm3(d&*++wB`2Coiwj*(L@%x{mo55@@IJ`K7J;;<*S13EsjN*il&nM!T0ARoMC+|
zX=^fn`n7%TN~!Ei*QUSjY0!z9zEq<{s_(-s>-WY9UqXIWOr5s<(3X>vcgDF1<W20o
zEpz_a!ndU=lY-m6%+Q_avCk!AmYbj7InSpj-<x)|%U4Tv-|du?u&NZRmob@_pwf8z
zwX?FP&PsuT*%$Zx{dv*9{;X{1Ru$e4tVNtZ?krgT<EL_Q;99;j4(BI6|9Ina@<z+c
zQ4bU^>pkOFRuk}EQczWM!=!m<fB(UUpX7?;jz9exw7Fh7UpLCB*23}Tfs;ORg_F{r
zfQAMvQ%kfRBW1KJw!OJ2=%)Vc+S&~_WLb)8SIlBs)ph*+@rRof88%;Sw7=eNzJAKe
z$!(io^rlB{ztDF)Q%(PTJJ*evyghQVp`jmH{@jQ)b6BuKD{sI2?wc+)5vOlG)h-m_
zTN>Xt+d-k>ZW0q?*>dUq1xvP`n<ZI)p#N^tv4>wCTwnY0@hZuBHU5`2jE5W5Sfx6j
zOi|=ntSWQo;%v97{)N}hRqu6C`uS<mOr0lBw_kG5*4j6xre^Jvx7XaxhwYv%xA)F*
zF((Cm``=eV&F}>Ks3nUYeP#c!?bM|C`LDnHY3rZ%@ssxZpbP0N<@c-AZHr{mcg<E2
z>U`qez3FejhaN4H#TPa7io2NR&p%%O*Dh~;Z}z*nC%Efl{Y)l3SjN8W!>h@wIn9pk
z_`Cb^%B&oPDJF^Am&?1%JhfcN&}pMe-gIu?#sH0uq-_tkM0*}x+>+#JlV`3Zx_sL@
zxrH-6?2ui;Q(!rF$s<7_D+}2&rG*MJaxHSjS{4d(v2L`wGegv|xR_)0<<_e#2cHR;
zO_yHnsVv1g(L^LhPu<C1kgILO?mH5l5}eIu$E{B(Pk%q@eCy;n5wV`@URxbp=H~WA
z|6tb3u=lqF+77l?ueuQ{fAmVyIt`X1!B%^1)sB7FJMeM>w}0moP+xIF_4m9}r%!K;
z&`IBAmCn1cElMj}y5J|%lc#JKo}9MrbWyTQF5z~(ndx_N_mYNXZ5*psJ^B>EX?j$E
zX^x!s!~mwl1|NROzOjvQ-QgOL7ya?ey@x$NOKv%rx(VO?HRqDw$|;6x&%N=^n{6(9
zYGLS3rM0WG%I9pmZXXr)`?2iPi5`B^r#xlzn%BKHF*h~ZB+Yf|q|2P==Ajpt>|D_;
zt>JJ=;EMaB7U|bBVw0XtT4-BnF*kkJDxs6&Qm~Q2=RNC-L|KD0=T<27U7g7%BH?|+
z(E8G=1uj~RH`g&peRP=<;>CXS%E5j2wmrL<+j@T6?{@3YvvjS_-2PS_Znyp4y{ns4
zCS5)yUc$dQQ6j=8Z5FTe>)Dee?Y~8Y7;~=f{CkpPVS$7qi{JFuKH=e@np&vS<&v-E
z<EBkFa~5ti^f;<+ZFuA(GnaN(*Ws6k9#38<q%uiIvhlmi`&hrF33qy1a~iq2G~X`V
zw{(@KqQ}YwGei%qn)~V%n^!6G{Ofn09=j4QpK`Qa{-Nf(^Bs(=Id=jNz6fl-xjBo|
zdX|aa-U_R*mjw^D1n;f4ZlA@f+{QQMpB!X$<s74PI~J?~jaSLp)xRseoSM>5QlYj}
zYLdsMlB)&}L$W6B-e}k&z}3dM>)xZ?-VJA$BpJ<&jGFpjahKW@mnp@c58p9ZCNh8b
z-n7Vm^Uv$c?=Pr0!qfKUL~}^XbGg6D-e%^-pJP?8guZ!n-e338^}nZYe*CC)&o<}b
zpYC&u%Z|6Nv6V<a_VanN_2csE`s};ys@hqrS)1E<j_Aqrf{Kob9)5y%lQgf3KHM(7
z{<P{OeICikU7PRU{&=ob{F2iqC!?v?G_p~SrRG@OUiwg=xu0d`t@DkGw(*8URcDBQ
zo?kg>;YTO0zmGewzDs=l?5)q7O8(&Ee2afxs1iCUvdf_G;MMI$MvQ!_r>fuYJzOl$
zuKC`2!arMX!)z|ztye>?UP&yh(A3pDl=MnRW7UF<>)KAOiaJv!uBv+UhK`y*)3IMt
zt5&_fu%^t*%WFgQS(QbOQilpfY&~n8jV)P^?pRy!<bxp3<lak@CY98=w3m9gG+!}4
z9^lgSEMd;3I9tO`6?Z;)H4$H-6Z#7jj^AIoxFB)K3cH#ayAoLP{p61R+<C-_*Mv>@
ziRhEWOCqygWqT$lnLkjM%X;!*qShp*#S2!j>_6T&(JLrEynmVIm1|d%zdi|=@_5;&
z6M~00n-@(A+T$nva2eYpHi?rOUwIwxG}dqx|JB{6KIuhO+2b>wU$#TevG;sl!tMCe
zrR00-PM4x<7g*PB`Z4ACF77r)O<AFnCC<m6pqveT$fqu@VO56k$`GN=N2|6pq_k|0
zn|u7$+o?I#kNhnj6}ntCvb(iux~^}3Vi>QUtgu6%LuhE(y1MS~d|EY6?<m^*|JMFy
zP3Nc8_y6?%fB*k)!<B_o|Ga-M|Iovk=V-vr7q4#bczw+BhR<8}#OvzaKd-)*TdK%b
zWpnI8=Jd~V*WdN|yY2grpZW62leR2$mws7(@amrj+2(;;Gd1mhyq$bQmQTpHcSh{a
zFFG5FKNQ%0<>ig3`dnM{_Selfk^5)F|9^PVtaQioXQwT8WNP03|1^1pnz3KF|G&5P
zatTLQ*FT^7z5MZJO&8s~cR$bdTh>RM=d-dr`Dpt5rx%yUDK2ZuI{on5?|r8Y;zj>|
zJ*~du)x!G6O>t>;|K#nL-#=<zboI!+KhK{3KYQ&JN1F5T+2%9qf3JT(Ve+a6`{n;A
zew{TpMaeN@qrr-Nel0$Em56Zvk2`Mu`Qu)`De0cD&E^c%vh8ydG<@zndARTWpN?*~
zZ@$(^KJKh3;q19@*X#!kT_{cT=yTs*aWzlE>SW5Mv+-I}4;l1qPD<39I486t?0@;U
za<RqRGp6KrIUQJY;r7xE`#EPlzHoZ!Bh%Fpp}dMFi#%BqWgbf(eDh#ac3#q>3r2an
zp5KnCKiV2o_qEqYBJ3D%dHrnbb+bAxg_lMKdK_soF*6OaDX`fdV>fqU^7*Ig_W#6M
zw@LWh|76+5`26GZ{ePyuzi->Fdfxu5diAPX*DZsN7x$lY)t-G|{r~s;e~xc|zbNbR
z?>Ck~z6Lo3?^LgUT>byI>HXdR-L~o1%s*_J*b_2o*DH<bcM9zP|GxjfaciA@>6eG%
z`+wTMn_=4decHMutG3O0KCf<5v(=S^B|%rNAKmff*Y6)s_3JzuZ?`;qb+G*Z-~B)C
z-~ZE;7}v@#zW!~2)fM?$=Y;*wrQbTs|Ka-nf32t0zCBraeBRM0t2S=+X`c?=mwUFJ
z|NmU6iN{qht4-!Izm$Ao=bty}`ybW+`@CDDH#%bPUjvT0zh*_T&8z0z|M%~Iqm4(-
z9D7?@^IU!Z1NHkm=PJwp?cO`}tI8qAZy_!-YF}^vaW}vI<dK@W>hr1^mxVl2TsDEn
z{L`CY|FV^^81p+Kd0pf6E<Y`~Lu;gR_XNIfb5vQFym9x&TW`{L%@R5(-Yv{|IC42d
z`mROl{p*@pcVIv7*4IUGY55(|yngZNhnwzC@4lRP@X5zVn<78_E-tvGdu98%rAxN=
z&p500$4;EDUBTYmVN;&<ZZml|jrNMKOMf0|{;Cp`$amGuXkUHe?ANW*Sr$bGM?%YE
z+|D1Wsd^HB=s(}wma|V)Gj!80Z)5%Q_~vpClh8>$%LSC)?!A9KS>Sn}q}xnmzw?Z>
zg*_79o{n<PZaivR^Gv@^6@7E7hGlN@>kBRXjq9%q=*_X3*VMqPEV{gYev)0ye_cDL
z4bKZ~YpbMWr4+A(y;-az@#w>~t!LKXadQr3e)vnkx3G0}{+{lu+Hy~JmYdf7J$^sv
z(Pjbj=|PjWx0c8KV|VhuAkf*U=)f^YPJZ&TdESYIlFX;70^XMzE{NwWJf6U%!|8NU
z!-?aV->fsN!tRf*JlLeYKINvecLvw-&41-HS0rA4c32|zMLuMzW0G*;E!L;W*YB?k
z;hG(}&^<BE&AnM*!E4P`*{@dQu5p!M%;A&#9-=gHj>O@H0vRDsjz|{Y!W*&b0U{bj
zjw{*At~N%p9==s+`SQoB<2CjNrP&mt)+9vT$&EHOKDzgp!Ge0jU5N%6Pb%8JWa@9x
z>}pbCsq)I#<mj2wIFIA=o#guDlhZVc*X`nPzjJU`X{r0`i&@MY9Zp^g=LlfY(1`F+
z>*`y1c!M3QOL-2H>shhFg4bUEeorr!EIjBs$9Ri+sP*>Cku27(jFV3~&;L|6<%xyi
zsow?XO3YpBtXD1*@f3B@@=P{hJFv=p-s6jv@&y-Fi(gEAobvlu|LreJFMWzs)V0~3
zb@Af2r91`<x_@f~+>R~(8K|>t-B)N;aI({4?Xv5SW}Ig-XlY!e5wRvrXWEzfSF?`h
zl~0&I-MMfc`|N2-Vx03g{L!mC93~Rt>wMYI_p+#~Yi=RSl2ux(s>D`3P?;3AW#+;3
znUdR_x);a<XY@1~h={#MKS5BSYw6{(FWpyt!a}Yd>g&Co$#?W}tYY7>1aaN_my70}
zIOei_=btNbt5UANbB?gt`9(Iy#npF$opW2D%!Un_uhW8LV=hQ)EO5UcGp{;JF1qY}
z`qf9%f1Y!;PkOvhODgO2%Zuqe#>;w?C9@V73OPG6CMHO%eY{CJH)*qUTch#>_xp~r
zR~9Z6Z7ba3d+}M-B2RU3zugW0R2ln1!WLgH{QqRyw}lt7FNj>1G%I4679l*lT;9>&
zt>WRyc2BXs=j&dpefo2_KGnZem^YNyg4uRw!Z9AEZ!A|!d2ddN&fjR*av<Y!+n3kD
zp`np~|FQ4cm0|h2JU{K;XMI79Q%B!#QUKk$W5N7Wc6;c(Nvh!XP7T+&KXR#sS6-J!
zUGVvzIjikz*5QX6g6BMUHCA#t<Y*OK_yV*h<*Ulq$48RFL>?_<o2FOQZvJoQlmeTh
z>76D^HMpi$Z{J*Y*i5KX!draL$xo%X1&?cn1cpl9^SZQFaIUPtT-6tGFRUc2-mGp8
z-!5U|sxURkLv`UYNxPK|w*qcGoW8C+rtRd*^gmOIBftF$K2u$+8X@yU<m-`C^X2EG
z-#mHt>!JzkY-|4c=h+?3FBI0?(*EP_|9`DrD|%<Y`)74-RcX!l#@|2d_t#EYvb;s6
zpE>sW=K7T(Tt{C$T~;lppucajeEo07x}Vc4zgU*DHJ_?B%01Q^@~&oTe$D6GH9KA!
zJhkmI$kY;RpS-83L^dF##&YKl+n2doX{#sSVX`+X=QF=|b06Q%IB~Jv&3l;amx;ND
zAN_mWAhT!o{eR!T|G4@8?9qp36|X;cVgC}c%6D#U>)#)@=9+*0xcl>t|F7epU;Oz#
z=+<WO((16yCGH*3`TJX~_f>J%#u(oFbKR>g-a@!P@aT8>KWzQ`H}%Z1+gotMM`Kp&
zojo;go^2_deE6yIgYOF%R;4Mb>+`p()~;J{;LxL={huB<+b4wIIea~Tzu<PZuyQ@S
z`Qn=opYAO$y%Pp4ZYI0ltkbBtHEpkKtcF<cqfIZq=$uoP>)&pfF2S}iprgXZ&q8LI
z<*LUY#I_h84hxxj>TdRr%XhZ#c-xqNY5j@$SJ!BCO}hFcLu$|eSC?)6Ueh|}KlkX?
z@R*L{iHuXGRjkh4D!F#u1kn{!OKgOA6a!4B@;|pfcZsXHdVdw``G5be&-gw6wCwfx
zCeDeX9jzYOu3sW1q`u$pe(-@O&&!$Zc{1)RrhDwt+_g?PCod)CztEa##_ygj;O&f<
z(jLxoQ}T4;QATFn&wi_VIZLam-yD(C`kPaF$M|CJQf7OOl`cKI+SGFR_~oYe1{4dc
z?dINnv}K;x1&OYQpQ|oB7EJPF-hR7PiRJ9sUYBIH#u)oFU)4Jy{<+S=YO}ll{E6XP
z8+mTF{2GOot1_k2oR3G_O?sUk+aQ&@WZ6E>vyT+??Zvx2T_rSMWoLIBDXpoqSDJe0
z;kO34<(iWYJTG0$c=n*0$Y$p4cb&8Zt*iV3Z(UK;H)rqmw4HVAv0L`WmQL4ME7xWp
z$tiwV7}3${RP*84>o-|E3s=3|C!A*eJg!Rg`IP5@@xi{q8eOL!tExXxm&{d@xcc#T
zZH89Mtl1tMqWh2g7AM>K9^}yL=ils^u*qUW=7l>qTJF9+oOG$9b(W_DS6;vL7ZKlH
z-yO_yalg;{M_zpMWH<jkUlWNVZ@C3N><nyh(ecsIy!5wU;m(x{>()-X?5{L2;rqM0
zQ>IT(-WYMNq*Miz@hvm`Zfkw7xU^%D#+%#Q`(M4vGVwi}wmH#kHYZ!NL|fzSw`yYD
zjz+B>M-6$ZuicTnl>H#Y$j{kXS@W>RadF;b0iJ>J_gfANFIb^4-J{lYe$G9&#QTXc
zB?l{H!h{cb1O`Vs))q@FS9b_Zlv7)pTRVB9fbT<#G~a;u`yD4=En1-=znU#Ufk$@p
z;lrZa@AHbutFfGnXzA+mO44kevxkW(PJFBTw~yaeufJ^nZs`qsvHsJAyN*XI{XDez
z^@ZE3AG}&ymSBEIh%?Gzp0{IvNnGE_q{RykO#A!p+pkUYO((a?#8-V>e^VyH=%fg1
z|MmXk)mvnin>SpRK2i65>bHbLJEh!K^B(=?peW{h@Xg1|hacxhCtr2(s}r<59u}~9
zVzOnFegE@}zdqBpJ?~rV?4HSx!^gjr&;Q%*t4T=@ZWM|f5pv%hWO`BgI4fv|mE(!*
z+5FOq4cm6D-5GWD<m>YXU*})v+GFY7e=|mIlj$X&A4NqBjoWwhByAUtS8#CjyImHM
zb8hk9{!59`kW$#gZ<@cV#{3?xsFz7qvHpohGdD!&MCxuhn&imQcsA{Dicz8jTiWiK
z5k{RyZ{6eezf}|c_`$NLfBv1kZg~6ci;of$m$bgl+sz#OHZ81vQ}dLc2LATRPxtL!
zmw5Wq_L<+W{mwAc{y5{{7w`3tK6RfzdPk{!^{QhB&uag;Y@DB1dZY9BA{QmibF+(e
zub-JUEzU-ZtIOll!A08xJ{>E(>dIXCva>yM-&+nb--fH_b+_L=^X>cnH2+fBsrmaF
zjxQ;=x+ZglQ`S_~bpE$@)l2m+aYfCo4HYz5n!4O=*`ebVLU*LYUOc^EfB9RtfAjtw
zJ%vY8(|4`j)snT%FyY7w;~4H%&G36iSe%bunyWtF1vFMMNnb==S7ZK~x!+DMIX}aj
zTX93w+M5-pW-XiOwUjl}H`wsYWA5oyMxM#ZGuGEU6+0&zcBH%i-%S4Swa=u@&!_!4
z=FefpUUXeIJkjLc!>8B946}}1Ju}a)>+Add2lrf$PyF!VAkXWGvrDH;_Byh1<*Fxt
zR=-d9c4gw4m>)vx_kPq$KE~soX&4zI&~jKYq<qq`S<HL(muVa4@0jk%%oVXS$Fj5K
zQ04TvbL!IHj2>K?J=3&yipV6@blJPMXEtSRGDx3rYT>J{#d0gGy;rZ8@~k?*_ma+*
z-N{=mH`IMg0_VMn9=m+L-%6OVa>~h;n>lWQB0Pr`ggkG%XU*~wSK7kBRPLv*)+6T@
z*w{U>`Pb)z4^_h2xp&`nJM~gY{Xz2eYkyVVu*MkF8nynEQ}f^5uvC6c;{AtJZQoB=
zS1Brfc=rA2p_h|2PklG|`iJ{x!BaIYsip|)3+pXyEx+h-efT$9KXBt_d*-&jgn<1)
zTQYdfR&(|r??0ZZzSP?>Qps{^#w@q{n`(Yddph@n*~H4DrME&)fy#@C9x9Xa&U;o`
z{=8%2d0b>pVbju8T4lRsW3NZI+6u<5a<{7Ka*%i#G3TXGKi58|C3AB*@1Dq7!F$-F
zy*smT(_)Rak0zyE)=ZGl{b`-iC2+%*^-kS;=9tQpC0ksS^=-bD-;3MC$E^C6$Nl9y
zwlzX0BYPHYIl^xDPrgg5H3d?s33;}!kYvqGuP>-NquZ+Ju{20!^2wYNQ(TlBEo9Eg
z2|xO=H6wS|-CNbA$6l9OeOozEEUZ;w`4$<pRqLhfwWo_L_aw{y|8d;jB1YaRG;~&W
z`NwO!yKnBU=jXm(Ih*aUr)Qt|V%4(DUHvQ?TwB?<eYmtb|Dn#i<8y9`2fw+xV%<7-
z!<mM&A3S&$d&Run?{@-d?It+s3Uxj?GUvjVUnOtcML3FziWi3Jq`C%vsIZY}oA|Zh
z?f2iAJxi?CEIyfYSpLMpH7hmB*c)TyS1nTzh?t{ilgAty__3-eK*mpi?cX80nEJCa
zaW%Dyh_%zd#D3oV^pAZ_%*M8blhc#JMb`M~x0|njv|V0TNloX>zF%%tPk(0=?D|+$
z65@LB?Z?K`U6-e~n_fL5T>slvZE~0OcKPS?t=V=TK6|22xgzyp^dI8~k6zup{M0V>
zq`27L%?d|_I-hViF3U{c#RZP^nmmo$e#tUV`#M|~Tb$MW^-ag=dTCZxmcdM)8*j^$
zCUUF{;R4@XuwvDzb$>-$CcXT;_F^z+(?SWoe(5hP3p1FupKe{IrIM8RNV8z)oFj)6
zd5*SRp4g|pDB(gtyt{_b@mB>Cg+DCTY)Y|l(~vp-`18i%qp$xI-mBX8_igpYY)_N!
z$#-PzXK{VZXxYK1w|s7fMb8DD9dE)WedH)D<yGPR@QbM;rmRI^$)v(rvzj8__}%gQ
zz5Jz7=aZgGQ!P%lC0@94MWr>h^9gs?vdoIT_skYbMJqpsEVa#f-edh)=w|&T{%f;l
zWwkK0ANI45aSRlR(Yt=Jy-+=AZ`!5@-=i--yS|Ed|IultN4MRGjqscuZ5{Kj#8A>W
zH{_4yPnM#oi`8y1`G{H9t(J8S+-P-IMC{qak3Tg(+%kw-y7|W3o15n9nNP4w-(;ns
zQ?_7B%ZigR=To*#oSmK>%-;E=yKU8~3Y&T7<0iN$z1&u!rUFhzpIjgGaG&a8(u`FX
z>YU`EcJi#Y!bA^&?xO`eZ8aJ$KUKP-WU{<O*7fC%Te&iIoo4EN@3y~lusF+c;<!w%
zU_t3AHvcEHYmYrFnt74;@r^UG)92n5_HBRar4nDVByt%nmF7I{;XakNx%JpF1$Vof
zodUXXOK)H75nu`pEwqTOsCjoe%KdE4(c50mx{!rgkaAfsJtp9q32(dt|9boF-Y=*6
z+*o$GM+DlH08gc!J}4<Z<>495{owQi5^_<RDC50dV2Y~H$?1}g8&*zy7Qo`M-{SO}
zkK3(cSZd-zcxFkx?l{6*R8+ii#--o$zlO}Yb*n0<>Q;HfqO6L)b#u(JS3hF?YAPzT
zZq2cPGf%`t0w+9o51mr>lhNd?mtWuXf?(T+6<i{$tv#WgO{X6pf4yz7x^Re`rHik#
zvDohJOPh>r__D=s>p(s7@*HSlz@VgL@7l>aOGAB9J6%4px~WvYd};sriT8h}goGDt
zvz9H?FnF>@`{z&o{f(9(2mjZ6R}7Qu`gC~yUwPx>Us)IYI-n+3@k0HI`Ei5UXO{$N
zvh~e7Q~SNPH}R=zf=cAibJ~9%UR1ZdAaJm$@oe}3-|hDgmrjrMc-!cE=l=!&71>=$
zMzaOm`t<WH?Ek*}yXV{U{|OJ<v|1)QnTwc&G9Nu8y1lObciyZ6d)(`e2Y<frY(BG7
zebN_c(^mPq-_c*Rx#zr;n+lpi7WMV@WodTIoU1X>!>>btY5VQo>#uDSZyD<$hJKvv
zU$!|~K!$!&_9S*b;qG2!Sog{Ghj;tN^v_<;_I(f&*5BI~Il1k@$@h(Sd3KxWrr%V*
zy^>Gz&+EOSbIjN2i#hloTe%`P<;a%PJClBSe9MVDoa}GkU%>j~ZM}`l*<<`af9luy
z>eMUT|M9Q<#Puc{)9-?HpS$A&?_>)w%W`UENlTigifukG+qm#`MwtE2ot1OykIM4c
zt)JO{uWq9GwD(VL-F}~*Q7N~VjrY-szOR4&-n{<O^k>$-!pEv_V*cCJ|N8xT&!=x1
z`>NlIp158u)ag?6i-WCshL4&I-|?C2x?GfAK66nLY&!VhrlQ93Ak8;vt#YlEDNMSY
zjJxJ7+O@3j*0HD-!2{0<=Dl3yr84O}Yp>(tW5yXGZ*&4G{`K-pnEa~QS9J5pmrTL*
zo9w3xzQ~<k!*tO7wEcsCx1nE7mNGDzNHe?nB_HeIR8j9w`+jU{j%A3mja>T6{r`?0
zoH6tN```8{=UNUje0dvr#yemC_$;GK<uYg8{s&&3X<W_Kw$8cR)cf6*ZidE9`P}c<
z+8?Rhte5twL5N|Se!REN!r-VW?)~%jN?3jHYvY;UaWlMNnRPH|Not8kSJUB#2OfW%
z^KzAk%B0N=2?jTomZZl7h#hhW6wwLGmOg*UDdNO==I)&{1C%^XgpQ?^7b&ftb*=B9
zRbPM2vlk~+Uie(~4-IWG*d=m#XKDSsm39FlqOOdOZ&+MQ=K!C<l4r7iA4kWu2anf2
zcUe(oFTd{Afg7LSJ^ZA&;n)6uhfekGeo=g{z`@ZlNaoJI#{MwvH#fIT+bYd(nDMRk
zM##a5yQFed4@=lBFjo>yK78xymA1(|@|JV<#0a~|Yzdii^!(qc%Gq%TFO=4(Dpx+p
zHlO&|C|z4sK=<M6OHaN_9hu>1Tlx6o{f$*0vw5V=<`k>>B_3NaF=)09B&U921r>xG
zm(CSgeW=OZ7*S~PreaUpH~DpnLS_XPv2))!28y&MB<r^Ghfi`+Ro~yy6l0$i7GBFX
zJM)E!shNQHEBDu{S5?$9u1R}qa?Qoj-zDPzM|F{}@@^CFmohn;zRfrpaHitUeBagj
zEq=ky${Ss@E;H`96L-h&b|GtmYv77IyXPt^HYYl>B?b9C%+TUmwP?;1hIJfs*Gk(n
zrZX~Ei<hpRZW!;ft$t6JZu=}v4%5O7ehQBir`2z0TD^?lJ<!4Qkxun<OEdl1#}_)c
zYe?7dmESEfHRFD{;SDD<&xK^)KcA28-<;vhx^-(}-j3(?J032Q2?&*GWcc}$RicW^
z|3JH^?LTSD`oj9UZ>R5X%D9+hmV5WuuC3mYAFkbgv+ghF{r!)-@(PltLW|iNv0Zno
z^ul7j4<Da>a<&cM<hyMrQ<UVoA8uNC#mr0~IP{I|j6`j&H+Q!*sn4&Qc<LF8nDCMi
z>*+O5t@P%ZiA}Crvu&sA>|2j6n#HzF60O?wl1EuPMgFY50nfy}1+V{PHhMb-MpvI|
zjaboly#BX+&RYKm8)gYQB@|Zvd}YgTuqvF<O<OnWnUqt_HY@qsg$pK>sm$@4&SAdk
z#nnomU0bI-YLHfR2}{=g_y5~DYrig~>Ie7V&71T18PDlS#X<oF%4s5jhUtdsdmr5W
zt2g8Q6PG|kZMF~x2SLL|4z^8`E>ALR=d`Ze_51troR*`;a~GK3=Rf@JTuJEOx~q4C
z*T-G0{(VxA``YI_Q`lFSmp}jZTJK<*K=jijd&zIplmC|`H?qb3xL%vH!uzGdOpk=q
zI!~q;A2VL#AMO%$s(gy80BhNrxMP#JW^R(Zd(mr)-@1|)-Ov6BeE;0AefsLXeF7{^
zU#own+%)fzFlJkSwPVq$xv%%EWej+v6TOE+F`LEK`*T3*(+$ykKg?E?O#65#)kJkt
z&DWPY*J}7ULlfUlpC24&%eh)%frp#ltP<nreP<r{m*4e^czxvYw}u6Z*Pp7!Z0*qB
zclXqhi?uuc`S%wdxx{+s)A4#`sk0p`PAr-?@$tFG(~8|{@;7R2%YATr?$sW<w@l}D
z2AsdwtGzS%lj)k+-F*RXc;5UfahPe6d8wv8#C6%ib+ZyI+Ez7bWENEJiST7(IP|r;
zvi4XG+q@dHzq?cZA2>C6P3%^tH9x<+p7uL<$&u>;{!F@Z)AyFj2<3hf?l@Tyqj%)+
zkBip>ws!m5|COnZ<7fMRCh2S2ULM(7{kidVr_$yhS*b4D>{u)@|3LnfQ*WhjJPnty
zSzw;|b-Cpoi#I$^A8ac!^Q~k{-#<^{jM|zoZLx0F-FN#=K3Tj-uV;~jwaZ~Kmd0hj
zZ1?;*<n40i?#`dHHWx^Ri0M~!Z;!9+oYTbAHYM#sK)7>mt@5OE9${r>YI~<eSajHn
zI0p#*_@g*Sa%MzHKxkyI(#w<QP99ounMrHkFR83oQWquHK1oSmblGd=r-K|TJewA3
zSpH<q@Cb;Fbk&%3;NiDP4+|HbGCCO0vQXV8<;{e3CpLk?s9M&<?21sZX~i7AfJs-b
zUR`K<tKbIDk5}3CEoW{@m@w@-<`&}Vz2&68#;hZ6&mPLmpP4ct*iEBBDL8hkczm$L
z>ch@yZQL6p*Na+T+gufFX_(uww%R|fAX9ABs!LD43kHW=aS@qs-*WV!+Tq780Z*H^
ztv=z&bdbZjfBuOZ3zz<Qt^G6Mp3X9RJKo20%MCVUn|T>+oUfA?^sj7b?#tB<>a~hY
zOdmajMMbBou*%ByH@n0fdswhws)fUJx3wqE`DV}jeXd4krz_8^i>#Xi-Bn&J-n5D>
z^2)@ylY5WeD2!%cTB&-yQ{s2oRAHXuAJbAFZ<zHXF8k(X84=N?hnED_9yOb$7k&JN
zvEK7}ug@J-IJl}j{^|GNv<)Rs*^W(XT*P%_j%3Mu<EGE&Ej#W16pL-X7k|q4x$Nn{
z<8tTEL~>U?I#@5Tc81F|)g#jD_w}WmO1d#2uOrJ+W$~KWT@$81whlVDXU)wyCS^Q%
zlleTpZe6uPcGtG7!(9LVZ9o2^_RY>Y^}hn&-2K5Sx{t-#=(Azg<@;YUPfxJl#K4gM
z-P6S}<kFI#iAS14|73o0vAgl$c%In=AC(i@(kD!}-#ND9;a(2&Fr(~evoorD{|UAg
zHJQiPb1pnBX@52#?6J+)_4Wqev+d8@828Fm@-gh2Wj!~2nz_N#h5H|y$jF`FpK|5=
zqVLZp8b4t9C7SM_bmX+Z?V;nZn`25+1;2_$&-ix7$*X$G#>%rrV$)t*{%c8Iad*oU
z{c?B9pDoFT%hqm-Z_zz{@A=-M)_s33w-wDg-0z&Hv3l+Qr3T_plxE&q6tlhJ+m<L@
zx$V;N!ILVK8vWMim-n2wcQ*UR<H<8>-tPSJW>W8}Lz%*Q2NL3&;<Pu^-Zfgb-(qsd
z8dXXDV4DvT>vk2`T$P??`1I(`j9xdPhXzvJvvb<}R=u)nTBzaPGy6d6a(VZkyUtf8
z9*Z_<o4jJvsR_w`Pyc*OzPY<jGI-wX-c?tVwxt*U3>Dv0_0UIb{moWK?MYM4%&C5P
zmFL7&r7o#E^K?Fa{v7;r0}G>PK*X<)PHDO~Pi-~7*R!q8*5Bqo-|~2ynK^~>m*amJ
z8qGgksm_!5tvfg}T+mra=!lb|&OM=ZJAQc;@iMJ6UCFd+(u2*dn=7jSEkCiSLu;nR
z+d179bt&=zNo#&(v9{Ls9S~XnaJA@bi3_Iey)zzsZrvQQ;g57rRi}rETg--ndrQNO
z7R{a>sqgo6_PJZ8Tpf$p^3(WSTUj+&Sw)^zRKCdnzPaXS)~{(55!(vH^53eT?sWf_
z`z~F8^~Sz$#q;j3*FR_d*`>upcD?;~lhfw!j;5F`yzH&AH0Rx(zgbSZ!;StPHIFWh
zG<p%C)+c-7Po?~&3jv2;>+f$Vc)9h<S&vU|zC6Dfc9TKl$uz#FU8m2fUe0@ypyQo1
z>8bbq)X!QIJro+0*Nb0|E4SR~SA1*pAK?p<XH%Z0ByL_lk8!HC*+i4wmd{!qpE!St
zA>l;f@r&Q@x=zXeJ1r%$e$ytS4YPvJHyb9a7d=tW{#kW0XqBJcq&;@-AKLHVKRYEo
z?!ioTIY;GF5qEyxDZ5@ObJ2MAu1{W@jOV<G`K)<<U3Al`Q#Xv+404WiggM{LDV;EN
zYNzM*Z#&+Ue(8+aR4Tpx{Ot4d!*x{V`2>|*Ss7zdD6{K}+EbIvOMe`$q%hSh9I7<`
z+sfVl);syDUR=!^#=M>1tk$eQ^I*lMeKwzdh}gt>F>;t*3*48k-<7gt!wP}fHyt)C
zoVQ}pq9e{%uKX;lC_Zuan*x`Xr)B4jIc>pHg+t!m_4RdT*nHSK&{3~2p1GHE>Kh;a
zNemk&^h?E0S+j;`OVDX&U&ROcDK`&o-<khPVA<cVX=kD)Mrb(1?p<G68eUtf`de%D
zL>{rOE-pjqk|T%eUNHr-Si0rT?eDq6B;(c?`*imNQ?;({E`yh*3szmZ9<pKkBDTFp
zJp%(7+1~Qku2^;D-Mv%N`g<JS&zSD?okPj=_K&i=W^MhuXDx33uDs*7_9Rb9*Q-T;
zZcaBeU{g~(w8-`S$4|2u?(QmG?q0|j<+)(vwyv&MRbSGqe^%~%!LQc8>e1@e%NFh8
zTYJ(gE|1S`uUN>`W4p3eWxbmCMeWn4PX>}(jwmxqH3hP$I@UTfl!xh=*}l2Go8L8A
z#onHOvESvGe-4t59x{CW$I&EpT)rnLAg0I7H(+DA{y`6>ayPM=_nNDJhFUsa-M9A&
zyVH%kGoPeYcCt!ma=4v5amq!T^T(^~j)lsH->L}Zq}<=9CA7*VEKg2vHm9oO@^$MN
zJw;ZmUVl8{^MuD6gCaNfFI(Z{S7*o9Jy9)QIndR$w|!+kPf}^>q`4QBX7^l@Vs`oa
z@0Z`?XW|bgNS|U?;NV!)A=4|E|7G#U8G>6i4OKmzCb>*GQuo`Fb?UKKR%d3milpoe
z5RnXJ-Fx(OX8PRLhC{Q~n_c*SVzPu~jX<dM^pAI>y;s;gW${q?_<Ql3(r-_{oWE%B
z^`Se5g?rKKyWyL&9=_i5|EiXs%_9EK&(Ao{x9_M{lQ}<c>->+8qCda1&QK_`?A)(;
z>g4nI0~ejE7Yi$QT$9n;^(v&x)+%o9KNan;H4FXk8C*%3a`gS#l=riy^oY3|rq8$S
zYdqYacJq05#ow!v8BM!CPT@^-=(pLnPVaxAQT@(Ilij|kKFL?mvEEVlDP>KO?B#hC
zlUU{J&K*_f_gd*IlCkE-sX064jVkU`%Ght|Isa_`u{}3Gzqr3##CuWK*4bsoKhCT^
z!L{!3S5@tmvgWg7+S@92be@`JF#T?UEi>2JJ%8^^J~QvLob7u1{t53D-p-qp6&@-g
z;_4a}k~2@H>WfBFcCu;c?(IhD_u6^vRxZ|I{x#Lie+I`9=iW23ZFf)JC()31-pKCv
zvZmtobA0YIX>Xoi-}Uq<uSS=T!7Lkx=?8_67hJokzVY|WpD`szMR|12CLCPtzsY*Z
z<6XDkJKo;XzI<LyXTSaXvz5zvSa+5R+O3jtPcJF@`Q)^K)OO>qC#OelPL=<8`1FL!
zZJ+wp_Ftc9W4+g2En6$4{^f<q72j?e-k9_5TJjuA+aJ#-pI_|3mXpTp?_VvmShaMI
z?PhI$pUSQRo|rUV4y9&)o95$fJxg{ilamzOZa$f3JF_E0eqH{&Iscs6ZQd<fa`<Uh
zWTj%klbzEq{@?O>!|q!t7ov@KzBqQ@xXekl_n?4O^UO7Ga&pxE*4v#l$-K1WC5z``
z)%^=v*2V62y1uSN|K##dYq)0^S93b&NgVzh{I6d)zk15Qa~kD$D%h=GF`qaoF!S!Q
zk3SZdZ~k%9y)x%p`;510HH`{}y_4<jK4_%#=;vMUIyEPJ*{26rreCy*VaZ+;Q+a;A
zgu>fL{Q9*AFQ5NATQ5#ne*Y(l^EL0DpHzQ$JX`Sg)N2=4Ul+Kx=dX@FkHo1`VfU?h
z;X<wx&&Hp5{OrE1L+~L7qY(f5m-U`6ydQsl-+J+akMHG<rfd|*UE!X7dXdTbQ{U!>
zUb%L)@Lg~Hlk)3xjxT%qdB^X2>^DNiodQ&j#n*9{$C<xmuPnb>(eZP0Tykus>FxLm
znP{7qsIQjV+utRAKJ!w-gljop(K5CBsuR!7-*?(w)Sxaz(pX$?=^njP&iuB={{MZ;
zuTrl2{Gg_P@YDXqr?igGE7iOnQ<Ap)Q%>YpM*ct9R=j}|9tT#*m#US=)pM4<wtw{g
zf6c4_CwC*MB{^pz|GaDSUJ><Iu-x{OQq|;{5e5zCra3lVKQPbsstMaHiRyzXc{?7M
zY+L)Iq5Z$@-e#W3`jY40naUSFuRY)7*M2qb#QT3G#`E`E_Lke-?s8QZx_ooTx4!$u
zG5;qqPgPO>@nEg^%zvM(kGKYG<<r~!kMCDh_RjAI(`OW%-Yaw6^?3Zh58=D!RwxR)
zi+y|$|Nq#sR{~P1jkdeR>wlcSTk(YJU&-UUCR|ldDta`ks!FsI-p{+sA|<6MVbS;d
zUkPvb{hB^W`I^R)YgBKDsI4nB`1$R=VOhcUu!$^;+iLzY^1m%#$9}ndmdSpW_LC;Q
z$5YpyOb`y0uRV6meO}6KW8(m!PQK%1S#2CnpC|cRT~X~);!xUI$(phFtImwyeDl}G
z_y<jkc6yUkDr&SO-~SVL*NM|7mn_NF=lc9v<8@SzMDOF(>(idUtNOz`Rj2B`?X{e{
z&N^~S7nw0MT#Y@m;$icNFIF2PWJI2)Z74jNerE3e3ALq+E}^Z@YWJG$ZtmgHmz0#;
zVO!=Rc;;P<+8pb7vD$X2=?4y2rq5}KT(0=s{gjSNvWk#r$i)K-9=c!NzNSOZ_E+(f
z1#wrdg$ej-Y1VI2xDc?jyDdOC)Y|;Ry=v~8(c%hHD-;DfHx(b2DhvP3aqaCgErBJ=
zlDO~Z-M+p{eP{H3#vZ9V?Qizz$&}y8NaEG(Hw$|H_LvY?$)`2JjDb1RmcDd|{IgJ_
zX~HCPku~YZ=l%VqB)D|ZvPRq8-zEiLYZq*;nmjlB&`0&2gJ~BRuTt7s*~<PRXoATN
z88ee8UA^hjb7v)-pI4N0k>Rv`<dO9Fb*h@jSFT>GvXbj}Y;)Bw9-Ah)Snd~5*V->J
z@qe7*Gi!dETe`yG*GH2)DvQ6WX~*TWRry9Auy&kMu*0~(GACn;)t)$gvBm42I<Mg~
zpE>iLnSO`DgM##cYm3=!tEWtSS-8dR{B#qmJ@<GvZQ2wSFBvB;wQ}u#UNP-^Jv=3|
z`W(Njtey9!NvZ1FE7$FMu9?b~zOzj1=aw}~WS1E#w|!r-E6etxi%@+s@5UvEwyhWc
zn3~#SZ}&;$@^af_4MEbr8#y*QC`W&5vH1UN|Ho(9`o%HpPEFDDf5@#KJo#)BLl%$g
z)uOL&zZ(}H<5_24t?*e~=fKt7_E*-Y-n{iz;QQu;kK#|hzUsey>Ee8)(@gGjs|?NK
zN`?G=e{!!byRzjNE91;#4s$j>k2$Hme*dZF?{UX>?R*~?C+py+vSMYA!N*(GGe5q0
zJmXb)fO^T}pV#j_`1ZN~`QFy~0fC_>x76FW?R>9!W}fx*_IJA`{JgyU)6r#n?LAtX
z&K-Iwte@tw-R9iRPho1l7Bh|2<IYTYSLw6k6W=<kbeX7yxjK^FESX<^o_uTD#qDo9
zYg66gABW}NDclx#di42x_xoQnmibjrvR-E~xBq@c=hWL;{29iQs~%5XKHv3(>^bHC
z^G?mr+dF4fY4^=t>t{K*q+RiR{_(^V_npO`cIN!~;r#6y>x9XiuU;$l%r0T>|MNJm
zr1W;?f)4g&K2I+d9{%xUa%xCexZ&fY;(SvQt4+O(Je&1T?I>RV_#pc~k8PpN8QPDw
z+5emBF28qv9N*0IcK;@wbDDbX{mld1`umQ~_^kf<$rg2s#~n3sbN(6qTI4v#U`@mI
zWws};9A6%|I<4T8p|Yo$Y0#Bwk^24r1?7I6o~u#%I&xo~zJ}+k?AI>}pPclmd?VQQ
zBs*g7(s{T4e_{B3ua0wfc<rHYr>Y(;WU$tDnp1T$*QfSNVq5wNoBxNhe+X^qa~JmI
z-Sg)V_kHjCnV0?q$<)cEin6R_dw*i#4z};fclYuK7d*4ziM;n-L;1m}GYT=EO`lFJ
zO_}m^(JBG%tDj#!KYjA;t=am|&L+54HYvOFAI_bAW6$1oTPwUT=!OOcX7c$gUD9O1
znf0o;S7qYNnZ9v#YF(4O9X<W4L@PV<@3&02wS|l0&Fwu+>G3l>Z%z=LBKkC7Z+i6X
z%=H(IQ@&W7le?__^k7i%>4hPplRio`DNOJ=)^Q|RgLC3f<M_m@@1FEelMHqBJEePb
zWuT~PYQeV4rVT1HPn<fZdO1-^UA<eb`OJ>5M(@`|v>to4py)T-sw*mHbB~-_A8uHc
z5U3S8`HoPG&eZQwp;7xftn;@oO>`8Qt!@zVXX@`88#d0_BiMW~`-xwQ;lxDapyk`<
zZV$8BpkF1PzM*ini;0w~qBcj9i-n9^n8=lij`I68ODF&Hf4P2Z5~IVRw9k{`C$uln
zNJ)J@Ir<XwM#~f7w^h{_U46B%a_0mU&$)g~OFSp5f%2;u>(;y9w>7MCDsz;KyqI43
zVtxIx`1y5n3O;3uRLbZ~6m^p=w@H_&W)b7{+4V$8%yvUVTY}10*)^*aBwk!yub+SJ
zXT$MpdTIBU`Kt8To;>mT;6`Sz$UhI&Wo8<8e6?P5r}p{_zdf2-s}?-|qn)4l^Z4W+
zm;YYgu_1ob&(-lumQ?$Pt*!3P?Dto^wPAXO3Zs|SyO!lk<dfZbm)?+CnEl=;$Jsq_
z&TQX;6AY8)>n<yMpJ%q_UZa-C$4{Hj3w^vU)X#P!^5%()dq3AtxR+C=p4QB_{BrZn
zCGMLn4PTd7%*{LFX8ZMRjNaPlKP6Ff__W;?JGEVR-czSM+dNc>bN9@z$6mWM^B#GW
zSQh$Q;;QX(=W5?x(>JR)j6Zf6Yo0vmbk?f3{;PEF!6fws?@#^W<*xktRJ`+tR&XCr
zsB1u|%+0%!woB|;pJ^P8?b`4~&c(4z%(r%za?M9m-q7DS_N?S;{F}1k(?hqT6K{75
z_t`b+^mD38F03vzJ8fS(eZ9y`S<&7p(@N|b<9C-!haP08He37P>ndN9u%8^}KRz61
zIk)qL*1la2tL{apd^~VP^NBUDi)rhV3bRl5Zm(aYqGZzc_`)3BBPqoOMPDX9c_JDV
z72&e>u2=sirle^Md4E^4C<%syrXDyEIL9DMK#X@z;kB1K^^b)A_+ES_%6Z{R#OY_g
zQ#<ZEL$aNk+|xb}He0LZvdLj=i~PEk)c1d`2n{`<{n~w__k*v~;}Y*alMQ)u_v9Y)
z^beoTu$q{-ez<BrX>R<niHF%0j~#AV{PEc)+2V(*&fdHKY=3|V>*lk*(`(fKCG@Pb
z{XOZm*{1Ar%OdY9Pt)}En;cXgDC=cQn$@t}{j~RobnV*Bm0fd|B;{L5o9BV<mdd%c
zYwofcJ{z{pF~2v*y+7G+;d=47>SR6Mg9iWF<kdIddJ`$$`T6T7*P5Ma%&)_B(?4E+
zee#s%fd!&b%xXJ}y9;NjF3Na$d%eD!>8W*lOMk2?+gRppx_nMOtNFbGTk|^=#^Q4e
zO>SRbpOhcV5IE_^wklS6-Jg@@MqfAn|MzIbCZi2Ac{yG=-n!bMQ8a1RDchX8+on#9
zuRXR!eOchd4aP-37>tU#3*M-E|D1R0*_`vbwGS7n+%I*AS7GJ7TQ&V+#6RBr^Oe(|
z&aXXlrc(ZrNN~VYm3_b3{7P3|l?+Hq+?J=Wz9>smOLLi~h{ECp+NrUhR@~V;d)W%F
zBjE>Muity@ly7zL=I2?nC$8Q0&9Ula=`2u#{U_&^*%Nr*+&#BR-+V&o1^&IuRFj*1
z;_hFx`T1z`oa#e@SMvULNw1rCX8HYhjzX?eTJ9Ltdk6U~(8%3<rAb35SL1p9_PeJ}
z%`Khs^^NcLJ2on=ju&QIew>ngW8b&zXM4W!^=+!-(X;k=pdWhh$%&`$_TP1W@Y=lM
z%hH=iLO2yK9uemMJ6HT(q;J`DqpI(k&Wp463oX}GZd<kO<Wi*uu0;w9uD^H@c5AL(
z?Jt9jBN^}h-JUOb_3?2Qh2Yt9MVyZB|9|-UmQ`9Sa+4NKHW7IyesODdc-0r_Ggr-h
z(ju;}@!RF=pz!ohMjoGQ*o0)qG@HNqY4tyN=iPDIHBD^MMUS^nGRztMt^1ue`?0i~
zOVv1=cTZUF!X)<8@8;e=k2_z^?0Eg+(psMN*Xu4Mb}BS1*%%mcW$L<EZwZr67HaY3
znwEaTofVTTR@$CvbYN*%TAcpj%IVD)Z`zb->mFToxiloTTe+nyS#j&){|>q3xu=Sl
zLi6h1o&9yK#=dsB=V$Li38_<BS-bx>z50G4fAuP@v~WK0*g8qQt<zTiubEJP=exrE
zHTRCb6jnD};V#}LQ>kKVJ-zULE&rw@Z(d8unHegAb1$}>e(W%XMZh)3PhG8>QS`zq
zUkh749y9ahJ{ID+kHzG6{+GA;dg->tG##TiX<ODlbq#S<+`UD9@9v$IslP;)?X}=u
zd$RX(lG_B68*kIX>P%*91~2<?WB>oN!RKwxuX|N>t#0p>35t_9AD@3VQuv%o+|I_W
zw_i(`rJd=YeDk%-r4EHx7rMDN$JKPz+yCWycBpYq%^i;|lh}5jZuL`sc;?k1%g6p{
zpErH~dHnwWgKzk!O^lt<zP^ZGEuYi5s@!q?^~KkvUl>+s>@3r`AKcr}aCT+-&qdoS
zKRm1t7Tj*|aZPZcgz1V!+dh|>YRcC9y_h3cTf%kT?!S~^vf1@J1-iD}cPxy&`6j4p
z+w)J3>%NzCG;i|)V{41Ab2{0-v>wk|wBftp9Q%JRWozP2{c)W?d1ide(Iu`XGbF37
zykIWX?7Csg`sMwsle_cx9=kp}TcY^U%P;3HeLj=i|KY?G^_5(jPoAd=a9`V)bW{Az
z$_$3ULrLWchSE{fuSCBs^ErFE^)jTglkZYezY)A|YOl2Uhnwo$J6`^JePc>N!8ggp
z6Q?}$5mj3A=Yj09vp*XC9iM+PG~CZg|9a3}35Q84cGd4|<K}JdSSua&=}fTxgo*Pz
zZ%^ZLTglW}FyYoLgT=?@9eknuD<FRFY!4^5zy;4IB%kcff3|S?^P(dc9~VejNh+D$
zcH|1MdSQ{bw@T=Bc)o&2c;F<?SF2xZytVB5sNSb0w_>@XNT%)$CFgF%t71QlOh4v)
zOu4+dp0RpWO^Lk46sPr#q1j;*{|K($s?y8nsk|}wFxwyBi`Ty0%D<OA_f6jP%!3JA
zFGlR_&%Ir~aC_y8xM`lb&K;ZnHEVp4y0N8FxJdotMxAv!yTq(#{ydajv%6INwB3x8
zz2^7My?Q&<<l4313#AO|vc^J6j7$?VPVwcLKis!pCTsKJV~@_fh-sT<vgqxp_P<|U
zgxUJ6-?3DQSFc>T@@ax{M!uuD(OM?ot`iFsoek7x-`KKONcG;4?DM|LGcBi_R_94S
z_GM+x>Z}C|m&xs2-SF7oigSJB!gUAlSJ`T;x|H>!+-K^_ZF!v{s-0K8#f;M*3YF#k
zZMz;<&t6?;=C_*BH8en!b+vKxySw}5@4LL@=TeJ3)8bZZ9b9Y8$T9VOX6+o-?eC6E
znatW$aN(pwXzQO%jdyCoIYUjio><j!_u;X}WeQvdZ(anyyq3>$aYxhDG}g#t8HUZZ
zD%+>=9L&0?$YG@xV<qXB?ck?&^1bn-Xz%u<%`YbG;(xSlw|B-D_KPoicHGr;zI>tB
zrMS|zYf-}IXT@K8w_kkWEt_tZ5$MY6wz$tb|EKN7mODD@yEMFRyfJ%pZt_c0ukO1^
zOve-6{fpIHbg$*+g{hb94%cruvwbSx;U%7vCa|*gN*w?3xJsx?rC^6mgm0C|%@a#j
z=R2iuQdpe$!caZ<R=&l-0-nh?yDn#{3-T5oKd^XXbnn^C)3n0!InD2vv0E)}Y|R#W
zQuhA5srWo!C81+E#Sb>!o^ZJLMAwbV-CY|)*>!i$J{u;TwwTW~FtGJp>coH-wNEU(
zQbp%32)a;fu>J1crOxp$wo1MFma}}5;u+27F%I*tA3Sxm_~s7%PPf(RKQ1m0xb?k*
zxn$*37rkBo%+B!~H;8^SgC|OF|G%(TucQN9dHI%GtHy7#uK#g*{>9n48mo@1e5}5(
z%St5D?Mk%8>!#HczNkHUZ@3ZMz|6d~#r?{Q!W(P%*nRz9@jX&@lg`Dh?-iEat2RFD
zW!&*wt047h(v_Dd^z@dL{W6ZJdT8Czb!Ne1{bx7+#YX&Sbn@%#vpjEi<nmO}8=Jj<
z?BDNmEbmHEd6mE#wl$GQ=h%I8nOF5e?w*<TiIWBsqh~aBbX^QSutNFw<|Rq8$K`An
z7;6^p*y9lvy?J>+(w)}u!qZ)%HQF~X&dTYLt+)FA_jTs6`Crv`?zAk?*K}Atae`0S
zrXt%66Q&NKb3Ht3{Dc{-7Ee7BW6tKzcKu!<|Mx%d@_nT2<xeI&I$^A@ays?*!CzOR
z=JZ%(vH0)*Xk={5r)RU(e%_rLowGN2OstgW@rX}SN!VanXc;{3!KAvwCu@HT%hmoc
z{P*p_{2d=&y{`B$-@o8CulGb1-oO3U%<J#A%<<V4f1-WG_Tq!4<{!?t%RinMo$ut5
zbm0c)mzQBqf0j(1@$HY#y8Zu>@5qU(R$6ALa-PgNID76S#=s*FKPGH4e|;q3;OwV|
z622R~T$Nq<Vg6;S6B8bq1g9HqpUPfT&6E72q5tRe`|l1t`jD`t;^Ph76RWzE%43?|
zo_TrBG(7IWWc_`|o_~&aUTs|bto3@p?p#*y?Pi5P-)=WB2~YigZNI_$!}=CqZwUV=
z_!{zV-^(hSICDwW))~A}E}ja9CtsW;dMKy#!LcR}ErVspUmsm_J!pNCy7T$=eBbA1
z+t_;6+2rlY6f)zLSmQT`Pjc?flS@{seZH{YYF_rT!$m9d&zt-|%rW~L8~@9yiykUX
zk&~Bf(U1<A>au{-Qp)x693?HuJ_Sjhj9f7RmTl9*g}ky1q<ZIGyDn5K_-+2pMTg}s
zPdo3JaW8uDHtS`o8aEo}Z)Mq(wR-zw!`#v(uQKmn`}g+!|GIReKN>S#87+kpjurbS
z>e^1*e2g#a?*2Rb&ipefw=j3ixXZqGf1Ru2!^4rrHzW)SkG7^KPso_78Mf%){~Pb>
zq%0CL1D03a^Pl&(Uy!A7SzE1*d9GA+ma>J9lvGn-pLMd*a#>p@&j9{Kr^}4Co+;ZN
z<9ycp)omAnsQ#BLKKIT4du?Ls%!C8(|MPafC~dzU6Fti$laK$=M~mCr^Ot*1*LzX&
z@~YW?&d`Y;O|M)O>0GXMNmW;G`|i@ElUAMhQQIrAhL0zT_n`yxRMS!pDdw9uZ%TZX
zirbNHBySqE<G+5n(6`?kcJHmY%f}M(r_Mg3=f>G8p^Y3<r-o`Y@%10~vssw1edfo|
z_fHO5=cTSMRyl7|CGPf;LG$D9niY%W+RtA3m1prleNITBr7TBQ)2vlmvle;U$mwS(
zHQr2g_c)f3(e<dv>H>$BUA*$5drfDTs0(smmgss|W--Bq!|Y06=>3js8LHOC%zMLr
z_5ZrPJzqX-wdm}%XIJ<gS!81J%V0+C4IQ>7MtiAd*2!_*b9x?fA1k=A_QXbmiA%c9
zW;q6gGW~N?GGv{8yfsXS<CpY`><^1gXPi89CgFPce8caTQ&n6HP4xm(WK@}FlpN#i
z2$VPdT4!RVe2+)JYHiCZl@+Vks>)0*w3s`=Cu!47fhQLK0(jc*wyCUOVX9-9E4ez*
z-AL@9eBW|A;g<@{*B?8spXzZ;`G>Ra@g>I$Cr+5Zt4dyKFYD|`|23K?j=w*X7M&~*
z=+hf4(mT;`xlckxq}W%}t3PHd&B!g=lPBKk(U1^v?4{RN*Wd&0F%t`JtYwjY{eksR
z#2da6sR<=&JyMMO&-Y(^=CFQh2MhC&P{!G3yR_6l{ARilW0q$3DSeu0)Q;!Rv^z~f
zXEE%UyOMKy#ga~sNmKp>TH9VX$*l8ZYGgQ=FyYRf|9jRxo*;edVg`$>A=jq6YuTcf
zn@BtF)_&c&_;`ZBzYT5wt_Or3dMME3HviP)@5aL2EuJfw(k5?A=2`u}X!reA1{Wts
zEv?lZ$5M0r%C*!}s&1Y;-=%Y!<&WL3lCwt^Twbz0ZMo;<Wz0<q0{zD|_4WOwPwkh`
zdT;YlA^z8zJJsh?c<eu9>&vZqblyDv;3Qk6{}oRR`(6cCtnh!qesSi)Klc~az1Q3J
z=2o-QZI#_8r~1C%q$iVSHdm&9)63cU89m2tAIn<Q_Gr_+CtH>|{>fjw;la|(HQ%Cl
zeErp3|6%J)wX^5Xu0ItpEp=wY`gsK_IG@_<M2or#PQKZ6_MxAvq~GVcf5P|9VgG3H
z|N8B}a?kJFb6&G#?T-1fk<&h!UYVRJY`rAV3slR9r9`U9@LgUTy?x8VKiq=6?H$@o
zVcjp;)e{4rs$3QQeIP;g@?)zN=N}b3oo+Ja@}GjFi3Xgm2Xf4o3AZe<eDX)Y`R9c7
zv!|$Vwx904`|@Yp@p6NfOEv1R>Mwiz;dB16|JJ+j7SEk;{`=3B<z3=$TC2azbMi9>
z5m!(9?)pR?HRCq<$gl|}DNq0J+H>M<VCd8Wi<%WjJ&Fxx&DB2W@F2i@k{a*Bf*;lO
z_OeCJl9!I<&obHXR_9y(=3n)v%S${z-&g;;ciyw<hqAT`@vw2SHMd7iG0D6n^K8Yc
zuB%yp<BrRTwH1Lnfq%Ap9DDp@wN}Bir^;O~b)_>eE%E#ueK>D>^7h;OTO~3tE&177
z81W=9Ok}2u(#3~UeboLxo(Gy>Uh-3q#Zf`9dt>!Sr&-_LItuW0yT^e{gfxr~&vCZP
znvy!xWeKO|&5uiu{b~7AdSubW`S(0#nPmP`4`iHsv*+5oC1p329$n<HH}mRM;P9B#
zqS2+mv2uQcLf6{;%YQx<WKo>@O~n^#=(F@E6ZH4`sZEwBWnztti#p`IQtL#~q*=GB
zUe%PY=;CgEIh`wczMAi>`&^!p%RWx_H<%V45WHP9`+8gokKE6_r@9v1(DOI{@k%RV
zYmw}<GfGt=)i?HRZi}<^zPb0C_o>+`Zt|8q&2CNtAtI`K<$q0g5MX)r?cIV^o=dB@
z>v=duy$6L2C~3@ZcYenf7|QA>aN*_G7bPpF$E`n}VKVVy>Ey>-7fu&)IkfQo)b*hm
z-@bG#s<^=_l*t)&?O*8%kC)HePu6ObDsW`3kDVporob`f{jKIT8Xg-hc%`KOofp3l
z68-t^^|=ZQRA&8nbJ#LxWrl;0>x9K;=Kd7Q-@ku~gG$r-GV%3i{&Cc+`29S#Rp;+7
zX`wEMtKUR*qBiyHP};&~on5W=_j&f-zZ;#Fzwdr~FMoG?)co@Q8*AFjZ%-A!b?-Oh
z`Sa6$Z~A)U@4J`ZzW+El?=1TqW*G(N-_Ko-KV~Ug8-1p|+g^3$vU5M%Us%I~Jwc|Z
z-$-k1^Xk9sX;yRPCZFs$yDs(mzNkZrtdnmRU0|G}z`;^m`sQ}HtB_#1o7qe`b)|N-
z<@xQGLsI_ib6do@(>E^g$h(4t|6Pr*aG&^Su=@HM)!otZo90gYFTfk2`r`XSj;Ne#
z^PgP}50$K6+@E1{a}k41n046Zht}U0b2m9$J$E+5MDxqbtFmX$zcdwMadGr`-~NBC
zB8SUiMy0RO8~Ej1dJns<3|sQ>>Tv_T{{oLs3J0?lq_5E8ow_^1AYJTklH8^>e>;Dc
zZLU4-$0k>*=lpBR#gMJ@T#PywdG%Oi+Soo$P&oGK>3pMn=Z>aBdLqv&E@Zj<;oW-W
z+V2aJYg5uw7hL6->-US3iAC!5ij~<e*#;A*PVJ0a%@}(0euR50WPMtg8r$4y6<av#
zHTo8)eE98bu}3oED|CI@sdI-GOmtWvEO{j4X9CX^7T<*yH!F97)})2{TFg4|@Y|!p
zTMI8QU(S#<Y1gS*lQyLY**v2?dXIeT{w%b}>zD4;5cqI)a)9X$nX*NS_E$|yRYEw|
zn+xy0+A+uHLG~BLPNrTvd#lxxd6u`{TBW7L$(kP~W;S_Vzybf5z`)6dHUHFf_{`(B
zFW$L-l1A69PoYikt6zcQcYjfcq^YFx$`$*6R>z2%oLJNsAHJ{CB$F>+g|XyP`TG;r
zU)nyEFSwX-A;aXzqCg$CiB|E-fq_SEoHWXDY?ymr@A=W`=|Q3$i!VC_nG{`mDJ*GS
zWgxY2ak+Eo)GN)QFJAo2G?a?&51w)@F)8|InYZ?L?fFJKSX$Eh{k&^0E~}dN|M}mX
zoQ~v6*@2;=bB~^54UawFu*hdh+0XjuTLD++s}<jFX=+f|a#D!@=Z}JOvwL<}^vg`&
zTA8`>MwQR+Ur7h6=FR@Z@b%fdJ9T$eZ}23oj(MYg_+QbR`*}JS4sie9)z4e1DXSvP
znKQF>PSLT7wy!U{JiLGQZQ@vTpzrGGh#<w_Ym2R;I?Lv*{yS}_ne@Ibmustc-g?d9
zs1UJ#y8pD=!ySgnAEJ6SR^8fpj(^9!mNNU$5ZBzPnsZP83{c!+`!>Ym%G$(=^{Q_)
z?mO4rJ6U10C*(<)`j$V=HFwz`X}-Q$CA-8b()a2_p3g5Av97rvdAcuBKe^|4;Op=W
zWxwUNnVoe}*6Mlr)IlIf+xXP=+Ee0F3+_A!dRD)y>r<P?<@vQ!XXo#lm2x#}*?xn`
zQuQJt69d)f9<^L9+w3T0HDi90!iMa_&u#u5lKzpba6Gw0x{PW1{BpK^Z%<c#+G*aY
z^7X_<23Pl<r>{PrTafqpy3LO}Iy`R59}^hXp7iWH?vOf_zh2?M3}dyzdn=D`O8*-)
zuX5vJp6BxlZKlrN|EzKOJ(mj^1~ZFh)*NnK6Zb_*zGkQN>^Oy0rp51`PL{l>{Jml6
zlcq@zKJ!2ObnFd(2dCDedna!`k9VB8cAlpH{-Za%)s2<nPj9KWb-XxnxnWs?+<Bkb
z$G-jIh;lvlNGtV|@AStW*9_jS=YKevM@6_$scD*CtL2rj&`VD~PrmtFB5!vY(`xZO
zXXm758_f9H;*|JkQtRjSJEmQ%-L<7p+BGoY`L9&=IoavI{3ffNT`#}%ne>yX=Y{%?
zc>J68d{=0zRkpp^lQ_2^g9&xBmU@2voN&QuV(^kP%0&kApC;yNbT8_-nsqd3W5V{^
zhBIqIuCxY<B)6yTXi~TkvcG+Ld_`x+PHXe{VlltwQ;%5lKYr8JpBNl)V8SIHsaIJm
zazCC*Nl(6%-Pzjw!`YBUweh_Cp_vmqU9UfzO2k4`&GkO1q6^d3`?MWz)jG}MEZ8>B
zVB$vST}r397A-mWw)Db{0B@hG4+^auJcGi++plFDPBeb7di`HPqx9qp96zL_166e|
zs@X5h5ZZ3ey?k%#Bu78Vg$!LjrB~x>d=uC3@z0dhJCe3H>`lb^SH4e=Y*F0y&UWU(
zrI9OEc}>3W?h_a&99Z*DK&*W>PrtC^RQ_A*l+-y{lzD9Hl#<)0JzBs1Npt1q8UKEm
z^pxK_cwpU4jgv>}Htq0?-OaV?)*oMu3!d>E4GuP!TwGmMJssPRKUNT&UT7hcp~`n*
zX`uF|;>%Ib_8H5&=Lt4fl^o2uBFOn`#hO))Vx=k1Q<4o|SxfC(<&*5Jyh%au)9(Nm
z+nYDzoO(;I{C2ju6Srqw$*it;F-FZO=7m`-d*^xBGzG>qX|xK=a(W_arg&`jiC57c
zM~mf^Yd=VteYce8sR|IiP(49ad_mK(6C2g%9&6*B{i5tr>Kvn0@)O%+%yRFzY!}vl
zye07OhquwQrB-n;bw&s`F7Qg5{%=m@AA@K49ELw{d4G7Z*v9g8ZpqQZKR*9>yy8b0
z=lSz9eCIRPN=8hYI4dS|ow8(J%DdV5N@@HL-gz#6bbo7K`2Dy8?`8#h2(}(v?4Nez
zgW{FExO12Fb(dzn`Pfk;6kpp@yl&?t#hXI^_x*RQnRoHdzum7RcYi2-wl?DMN9Er}
z<>%Hgr@S!FU@(h3_PoFF*->%Bw19G+AfAg;4vSu&@7NiBKHJ~!<f^CljrO?PF})2t
z#G%w1YF$x%jj3&JrQvbjSEnw0Pc!;`cITgu{NaX02ZHX^el0B%%g6|bjj_oQ@ceI9
zw43kQmTvcio~s|8Y)Su-<}T{~boCEush381O6juJGa@hlKYQJ}=1$%7w<o4%S3a6q
zelxn-rn5k}I!h_0_O|Jl^XA6CXRbUm>#yG5U9r=v+P1zZ`1bum`7D#ny001z&bhqX
zWalSMKHTt2+C+gh#L*!@)Twal1Zz*HteIDQC##+HSCwdPE6A$3tR4KOWbZLW^C?%a
z{kK|vbocVBSs!=Y{V{9O5n;}fs@mPjF5mLv4$I9y@+ABG()8&$A)<b>3mDmc3w?=d
zeEWKu;oqY(T~c&UR?K+Rc)Z|h>*JO8HuPxfpL5MNnZ(h4G2%{&bxBC*bwM6i%cMIj
zdNzM8PF=4($8CIXk>MPdsiAE$#ahq%<xk)HdV0pEJ2@e1#Wvf{KD~a|gsa@?n<~!g
z@kC8rlJe}1_xd!4bz*^69(+rgIDdY>&T^Z`Neec7dE9y?YSYy0{5_Jnf2-zI{FC+D
zUtx6fk=?TWACgn-+CTeA9C{<ZeA7CmHWihQR;P$PsxgIMr#_Kl3CfdT4E6kQ^HAg-
z+yB8Sfi6>h=9Hatm9gK%uVENgyS3?%oqIyp^$%CON_k$)YIgJa@bgflPQ*W6x4kYZ
zQp$P}7Vj@<OI|#CdX4z<Prm7IrC418o6U{e7ad}n(!|77yvOFt_TQ2FYD{arZFHh7
zy0?Fu{Pc+OZ?o^l9ZWg#zb((dzb^iKZl#jblD_Nnt7kox{+?ts`@&1rNkL}i_v%a+
z-qoJ@@6qhFyL2Q=bcB!drTp8&pP8}Z;n(T$PCLFt{Yq1*Ec^92XLSyPNRfAY(b3fB
zlQPYh^`tsFbhgd-`)GgUM(Y;O#R6PMH8nI9L*iTaYqlJlcGmQIy#IF@aj#pO|D0Cf
zXvr>L9JbCeY@z0+-gk=^uZfpwikNg`TI!0fo?wp6r!pMAzjS{fr1>J!$41Ka?2GoT
zDHbN{*KLT%oxu|&QzjnP&FOon&2#zD#PX!=vp#I!a(mYyXV=BX4>rs+kP5NU6F+(E
z?VDRs%~~NJAO7a<XsuCs!V${r*fPl{O0I^9X`+MeRnf#_einU)-teTnmw5NiOknRx
zr^)LkthJtIkpC`6=wOr(r{t_~-H75{23zv^i+T);ytoX%zVNQg6PqZvRO9}-qw&$F
z*R8#(v6TH(g7eX7w|fH*vrT$=WpanRrrwE!tJCLi;+txFweC~#RKF122dnRT`v!Ns
zdi%O^e;NC~Ukr)$GnVZ*;JMsC<?gKgM(>R$?*9L!x@ZoY(kvaLIkzJ=709icbU?m$
z`8h^|h&4P?RxNw>#CUBg{t)R~qIpg|kLTgFW1L5mR%m^ZwCI?~T%CN8OUC+}OW2-&
zq6Z7sRI0MOF!6%?|4IJKTs7ZW@%Dz2-`GN`IF!1UEWUSV?gqW)`<H6<RvEop#yjJ&
z=?bUUvvxb{hRrhB&lxoNjn*cnr@dTD^8{<v!#*mSPq{t0<MfKITf6>*=3fx#R$;gx
zz13q<i%O?P=U=w`Ca>gD*=J{*xHiRTboH!iy7%bmmd)p!91_olOTVap;@Kl(*RiVi
zbj7<~)sT5bOe&Lp{0%&3^}FcT^Nbn4J}9%yQQrCW-Rj7#A4>22&k|LayV(A-_}PN<
z@<(ii`_j+fI$!Z|?awc=CIY+X9Jw#@^vP%a2QL=xz1S(}lFcU`TdH+bPIkRt{UlNC
zYZuh>C9;f8J^G%usodAgt$Xo|&FAHgA9(1#`ORn3J^x<um>j%YajMN=fxBFE`1Jo4
zGxx`w^i=*O^vPUcnrw`HrBT?<zD$3clTG~g&mT=@UwTE2%Oor(_1qnPNt<^zeezMq
z(m%&1-+NQ}<$Z9MYT(gZzPlxBo`t5cUjO)O($p(`(!PDp&Ng>kQhjHf&f77Mu{G>o
z-8;{;^OgPcuFrQrQ9hx}-*f9~Ev~LbO4?dx{(QDoA{)1EoVDt1^i1RT&TR9q1Wsf8
z>ot421vhQhiT>n{2+jP&Zm`p9=fAGaH$Qxg-t*~Qmfr54X}c^%m-)>-wu*Q6i+RuU
zv?o8?-xnt|(L`wL)~gk}e}25a%H`FaDPq<WXN4=z$Wg26e{=fV`m)48nU0ATWu7W}
zTjPr-8##quxwiDq+vy+WOCtBqsw!J*b9UwSEt6c*^6ZWtoq3|mX}a8BP2CSAMvY2J
z3zXD@qs;?CMIs$NU1pen4&8T4?(q#__oNQ#i2e7jubXpWkF7)QM47F3>ueb>cuMX0
zeKH{If#SAe@9au}6?eAHQlD4XE$7~|B!7F%t5fAQ&-dATCM&LA(4o^VxO727-QhD8
zZ`md}&RpV<#p<cirm$ko&k8+b@tU(ME*_uo+F(=i-?nQXb54tix`r~CraEY!y{#7|
zqS-JjWY+W(+!|h!LLwUb4xZdH$>rGhOW-Q)tOl3M?89?UH1)|wKK{6~P4Po?WV5b<
zX5z1$dX8de(<;-h@U>21zU@&POfu{KuF&#bHM1jy-BERx#gBr&9}K3jdK^hfv)enT
z!ld(V>Y}Rk{qveXXRP52+<N`^jn2QCub#Cwvpy4Qc~-km_O<WQipD#8n9AP7oNL<o
zUV!h)gegz2)pmx3D?0!BrgB<DU0+=|`+A(nv*&`ZqRy~qUwZ^uuec>-(~G^|JKAcG
zJPOpxV!pw+diRXL!|W4ZnoaC@sd?_~gW00<Kl&P-+_#ItwN=aH{p+naWm8fr0!3IC
zpY79Vx;giBOY!l~-}nC)Hr%bge|C*f+T6A|dziNU=a?zJF3wqLqKhamk5rvW?ddLo
zt2TWySGw0{Yj>!zZ*gnSU2(_Nc}3@{xzZKu*U8k!He@I`C<rYrI}KW}sImGs??wev
z(dZv#x9y7BMHa5oy8DYa;mQNM8()Nu%bL&1kN-U>L4ymtXpz}!E>EcETm2l6iI*EU
z{csKpVytC#I#^ZqW~ICOme<`bQ@UocEs3o@srmYFhvVLjuT=Z&6M0_8|CI_AJ$Y6;
z<#%7IY>B3a)EYi%skLI$Pxt!yZb<jH+_+-e)A>410ZA%PElRgeRS9k6TQ((QLHc^e
z5?w8YxS$T}&5d!XOH(8tKiQevQoGQ(HvI2-!O5&ly@4Wc?ToxkbmnR&Z+UWAqV>uq
z8wY_CS~?5vT@zioL%jO+wMT~CFI;o8|F3FTaZ$0usx0ST-}-bpD@Kvf%<pe9E3cVI
zm#tXA@xs?kYIS*FT%2R%y5>deB;Vw5Ossd3ocJnNCvxY+N%vO%USAuV);@cVllteh
zz24jYnmFyg{_)qGrE}`fPW>VM_sJ9fq$>~h)ojc#YxL+z`)rh|u>IWqEw=wdOt+Tb
z&nOSParN}NzbP+nl?O!sOk3p3n8Ycp+}3ws_4m(lxwb!9S1!ACdiA15XS5U!uev1S
zf0XyewpZL{VI~*X*SCDXUpIHjc6s67hyGkozp(g+7}HnAp1znqiBgru+|w(+vqov>
z<{v4W(Gd`A<?d^HeE-i+*>*p2Sev`n?y=5S<ja>Vo7|`6>KJ$+r9AM)%6-4(UwIq*
z&NBJ$8p6_(G-;BMxZ2`>=0Z2T9X#4DT$c!Zbs+bu(W=?9CTVrvai2{x>nd3mO%#ZK
z$0hj8K7X=}4Nq#Q>+0zlpWH8Gn97*%<2ROzKGin0dS=OCje9k}xcB|3h~B*LPGlux
zcpBToS?e{|uTfsCBOclsv0|RTeY{`Z^?Nmj`E1&!gk>zNB=UA2dG@ySZOAnnhv1;*
zyI0<?RPbw>^(#i%GbJr;^7Q$gi=G4pT$&u`8u#e7`O-adUZ*Z^%>T?auQpLF)!8i|
zY>M;OdAYgE_PU*)aPhH4wa@x1m$E}uHPr({J&q-FG&vYaDhu^5RB?HBzNR&F_hkF1
zv=C8C&C9Q{gQiS5cyrbEEBu~D6Q|DgUUt~U&ri&jZD!%?)O(Q|xq7cUKK;fUveGlh
zs&i`m`lK%hlvo%=cB<}((0adh7024fiB{R$uUcN2^NZE-X#CE8Ug^^(r41$ay1jD|
z6P7enE`7VlKkDOweJ^6?96YvG=wil{mNg0-LJtchA_D^+E8eU9Jk{@4eC~GtEv|N)
zv(I|hIm$IHnlN>)f1W4*k!PWNZ=D}=zAfQf@@vsf`%5ol>Jvj)earNZL^RaAXOq(6
z*-`C(XxdHHt?WBx*%*JhCtZ1Wr|Vj1*M+6(i%woDo3m<>>8%&`83yam&h5RP`20(<
zcVXI-B%>vAGbdeZxSZ)M#9ET3ZkEmD7<%%p8AJa+@3@L*qIr8?TlrZ(XPQ@=$M*g2
zQ*M{Jjd9BTM-vSv_^4U)l)B8FdRZ;*1o!#9XO!#zYd0-ee168p63r8;MNc^5>VL=n
z`!xOhhl&`HClwbvm=@h@Ii2i2iABI=jh{MOw_L!*SG_xI%u2-6m;22)vibbpTaIkY
zuevNg*l{}9eM->z{xgr_cwX$C@_d<X!_jxg5=tL!k_=3dbj`cHxJM`XJLonR<BHsK
z)iqx~*6>tuAD>%j*IRw<V`*ha=k$1=`8+G`e3Y246MxuH+3n-aW#JVc*4H0=T=@9R
z`NIn?D|j*aP3LY7a|%6}uq{!%XXW!LOO8G)O1$9tBebA;i%xaD+I!P0$z`d<A6=Ix
zUG6^n%YTufl||_xm3h_w4ix|G?U{3ViD$6@i{pU|lcbG*w!D#P7v&TTUF_)Uy7=Ov
ztv4KIf74j*`T267qesnpWqy9=os!*mJC3e8{>)@j^a~ZCzU1OrC(aZLWbnxUDG6=e
zv#CyG_r1qor^g*ExX5xxtua;p+~d1TXYYJ<Oz+1l>67Q=k4!ww{_*Le?VRq9&!%2w
zwvozJju3SXG`M}I<Yak}V9&*^>G4ieqa!vYG2dnH_mI6TWEd{c!BTT1RrhdPzk>7Y
z=a1dwuRXgZV(EGGUXfYel%R_nU3nvSRtnrXE1iFC@o~fD3SoH<wwgaHx}3WFQGSMs
zb^QnaX*-W@RuOvs`f0bs4R!h5F~=XRzkd1aip_=@t=BYezq^w(C#_Q>>esi|r=NwM
zzPZKRY5SkpHFtx~YCexYz9aE2L&Vk>y7S_9bnPuYuIMFnJ8b=f#j5c}3yz%o^Ibgo
z<g2E$ISv8*m)5ztOq_fr<Xz%xGu?f^yu{8{w|BM7?L5prVd3h-8|Ujte{Ye{>^duF
z#Jc&cv**6<Q_Q~}aqRl4_xP=U{NuNl?=LMsaBNSd{3Vgki%xmgeC3_{DrmLwf+L4=
z?#B7;+idNsdNd~{)YZ)~JSJ#{tGv#^d6(yQtZ%C1wp&{AAdp96%Nudt-v|DA@Sm=)
z6mj|EJJrHRiMN-z@cWBymCm`}TF>b`Sbd{cJF;J<Tfx#swnsDcLeZC#&y5nUSd_SN
zN9~>WS#LjU??DfvnMdZ{o_l+}&atdZ3G!xrha9z}CRuFSU3C2YdsV6Mf*U*;v43~`
zo#!M~<EYZix&FB8(_K^de7>4B=`15>Xz0PP3zLKUBoF46|DRVC&k&e><Z<D`s~tSj
zwtbuI4tE?8%_+0aGU_~-<FJAM;kiHmBF;*MI38}{)^oog<*Alhw2S}gxkZc-zELKz
zzl(p{x~?^<i1FJpsbk~qfOlu)8?LT#4HQ}PE@s`%hkQ~sj3!mWV$FRO7H8D+_nlkk
zutRsj4!83U43i~mzMP!$Mb^zg=4i^shb1>W-J0(dhv&a3lb*yd`TdUW7OkZ1Q9B;L
zOJZ8ck=D)IyF@9I_k}4__`OPTwr){N&clAT?TxD%+<X@mJ}tUBw>)uM^o{900avmG
z!eeUqHU@BPzT0>?&C&1F6khw8i<We;&D?Y8+&R;^a$3HSNuke&E!Xfo{X3m+TgC4y
z|0gK>hlZY6C!@FPk5OFR&(dZ8J|DdDz$5w5m(Ix&R~B=>n&@s+{I2~@`F*b(Gi}H!
zx`P>qo)#*2F?A@JEf=1Aw1F$*tJ)kncDC8GgVpAo{;a?Epz`<s%#H#Y(p4ssWyYmQ
ztLWA~nEEo<U{>x6pS(F@#!X8O=9zE)wD@LD#46TM*Yv`olctkDu8sFPG2x`^<MVDS
zezqN-U)jC&c7MW+8HRfdpEt|vZNA>QM9}fdLUGZwy+1?_7O>=BbnvswQ;E`Sy8F~=
z`NV>>(uZ?Csv8&I`(3lSL~+rQ=QD!UAFRI9d@}mbgnO|UUVm+H=s)Ou@^6yu)9O`%
z0vjy9>coFB>u&OWu}0EoR*4qhVz$sLW%t&pb=}Q-_PRgq^V@W(Crf(F_j6yrQ=n$c
zeWzp9TZYJEJ=WZlAxr4I7eSWL+5EqBhOhSF##tM_*fo7VZvNq7y7$ekH~rG;1i$Wn
z)WK%#+%I)j`n}Gp72bhI-f1qVzwUFd_O<DnYp+2o>E5iYXiS%@X7Lm_q4DC}$4bc~
zO;6WEd}D1p6UU>oExCB-t&%*0hhHp$|0;IeknuYcc6i~wx&M|6urOK*s@@Vw^e$m(
z-RtKgd6K=dfP=}RsCz}sUheHXdN*8bJDK)ALB~7FRxUqFw0qU6h}>=07(ym3IhuKW
z;VP|3_gc<f*wi0+AgefGX|CkPldlfs6mL6~boSK3{|TI1@BT|)e78A_VeM7Fd1o(1
znTb019eR24$W6bx&``%qkFQAHdbM>~k=*K4U2eC(-TY^?Wsgek@lUH(Ei#WhX?8PX
zX65DACR4t=YgV6@6YZnu@x;PMgTvxZ+?;vares7li%#E|yXi#E*6lCPKRoiRWZTLe
zKWAL2UU~nkn0DK>&zBR5tpr(Fz1_@aswtN(m)?8(YGUWMZFB!w+h2FCEnNm#I-gnB
z3t6sWpd{3pa8d1KkwxXdhyELH@%rre#c^-OwjK|Qb4y;k8$Lg}p7YnDn5y3rt7>{p
zR81s$k1_kZ32?C2hFra(v|6dGUAq5v;|-aJWg8dH%ZNFcwYf<rc}s+2=Vg)3<gF1C
zSTeG>lrnf{^j1yU$l-gD#cVaJr${E3$)y+ki!MqQ?hbBCTl7<6_0c93GxvC2*32(U
z`;4WxtXRZ0InX`$fWOYfs6#7OX)V%7+<d#k-C5;g()JmSQ#pV9>8KJ;YkuZ_Ql?AI
zFmsa4#|yO)RXxdn3T-)Eg)Cy^)~MNb=?G_WEi#V0kZHnk!sOa@VV>g#GuK+oEfcyZ
zFtMfUZdT@Jm6cq+jSQewY3uEdX3d|Nt!FdaMWyHFvjeHy72BmN-yEzjxwlSf`>O@2
z<(qG29nQJ3^;xE90N(^xAw@Ax;mM&oX~v5bWG;wTt!>+7D;@dc<n$?Dq&y%?(jve3
zPO$&`Pi&7^=dWLXOEVp{J6)80Sq>YX5wlino>rma5?jl>YQ@2aCCi_9t8uW*x-eT;
zqD08|;NNwVJ(yN+U2E^MmT|IUxkmol|5BTDF5d`fzrIGPv)!G=&r#rm>9x&ePt&J-
zm1?^B%whRuMbDu2Irmqsn&hL_dHk`<qQGryBKizt{Qqr8{`l<Kf2E)6Z$E#Tr+y<*
zy1+|--)CmQhCh$~>pt43;puuH<I3{negC`yL|P+s+_vZ4z3{Sb_W481fgjDyF1+C1
zmgc$PMoh&VvzGZU58d7VRJx2kq>b;*(e11Evt`@;-SxC?{ic&HQ+DlQ&|A+w*X_-R
z$l$|q)7IZUEa=X2aMAyjb|S5!+|NJdE<Um|eSTM;MtuFg8Ts3<6}<60zB%tBQ_G!y
zbBwAE^zWPR%=shjrxROEvAV*BonO+r+<gxGnjKXo>-P77*p;>N3RC7a6dc;mH+$*z
zi_5I+BDZwhS~A(#&z5iY!w1)Wa*y)a{do{9!@Aev-;2bIOZ*?_vzwm06{=l#!C1k!
zYgt0W(b<2maNKO@3RO2)x0`M9w*$%i-2I!lB+go?-L>Ef6uBsE(z8EDV-=U$>UqX`
zlC!w#Qm-xiwQupfw0tM=*_*$<7wd0Q@Y=C=Cc8|3%8M6MH*WoW>P3&m*JIbJbj;Ml
zcCT&@U-s8<rPaZVZHXqgUka>q58Y@PGxyY*%~rRb2)ew{6W2Z1RM&Hcuem{n^TDYT
z2VQM@eDcJhyL+aGg@&>O9?2><nI0Y}SXgCLz?nNwaqr1X&u(VAWqg~~)zozL;W6!5
z1-on{mWc6B;#ySlp^SB-u<VMpZbh%wGp7cIUXVQd;`Z0f^xB2ykt=WSotmnB{h>i-
z28WyFyPi%9-Kr4Bw;|KD5)2{@XRKMPS-!>1*z)JJYX>q{ty1!klF>_FE&e9qy65qK
z7B1#j{wu6nm1HpMrR@WaFIBQqy~~`}ef`$4s6)l`MX8n7FFqBK56R!t{wTQ>b+6eU
z5^dkTr>p41cA*zM$FBXh_nmcE%=BDv|Gm?bjqQ%S-J+;2B=Pp=os#eKXBJ#qFh6xt
z`r$`6_v}?Yc;H{^p0CHe1B#A)ms1JP`Q4?RKlhxn|KHj7SkLjzzxpojeEjk~hr9Rw
z{;%PDGkkBeWPbg*AD_>u-Cmh`_mh9c@|g3IkNs2c@9N+2?wP&uj~RtBhE*2x_gJ%^
zW#Co+&*E%(c46wp)t7gCKUr@1wCVN4+pXNwl<S_ItD4)CxZH3>wwcQ8ia!UdEAyAm
zf1Q8c{>gpS{hC);^7s8Sa=L!`a@(i+iOzW%+)OHyjW$;B?6njvH<RjLXu6bb=f@P8
zwSf#ONgpLnF6mK_iFxy0s-^5-oqO}L$A;{6WeKwEQZBk|FTQ90Kk)tb|G$e<!a`i;
z&fKiGOUiMvyWc7HYZp9cXC3#lu3fNlRo5D>9*aU7=McuV3rv=0dVX}iZ9Bv63iH%k
zj`RNZ2ZoAv8@Frc@0#3oN^4P}<m}77Wy{z-1NLuz>vub7|L*_)E|*SvHr;*w^<)Ey
zeZl{m6ik-SoNOS$b0f#>#98OiI&DqYy2VSr|6Tqwbisvy>nA2GKL2?A_n4kLA*V0b
z@9BQ2zvm3+aUK^gbAe#fikPwh(N|fk1bNG3(s)g;Zhi4ZD0FJ*g6%;uAuWN^rG#0f
zvo!7pH(q?FsHh$=B}8D>k~uCDZZ`>BLpe7g_xQ;}DK<X7rLvtVt9*3sxcod<G-*;}
zipsrNi)9twtX^L8Jub!5KQU0yY%`;(<og&u-kgY?Q&#rd`o{Ig$yB?@pWE>#&F$EM
z6=olw9+zKOC|>kYRXiXl;s9@boj^~Vi^EjmkT+V(kKJ#%JlokZPff|F)by&#1%Vao
zk3RV@Y5L@ke|xh+7p~hTWwLC_y2fK|QHwS?OuO!^a>JqWr<_p6No}n|xu%NhkFSK7
zKRESa(RHsQ%8pr5hf-2g1wNV!bRF$%WpZ&5{wWz&<6Eer>KE!O8y2qCS{a$)!Wilo
zu(RCUl$Y@gpMERr&KD|;3C^|;E1DvL8RA2hEm*;!_$>YUyeuC-e%GItYo7-H;&@nc
z>twjKub9)0uQDZ1#6=%%T4r;$Gh;^V)2A%ToJK{TPMj=JUSU+a)l`L5w{cfi%1MXB
zM7zdYwISD8KRkIddGf<oH~Llu#D_PpG8J04{+PJmq`Bc8*E5%`tdoDQAL;3MKY7QC
zIl>#IqK`e%3VkYF@$Tbs36Zmljue|beC4|)c4y;bwL2$0-FYonm|F^RZ_ZA)JebjR
zW=_qfy?QK+k1iTim%UXI=zDnfoU-}lX07GxoO@LQVuQO6&h)Hno4a>+nPAV!C*q>z
z7u+i=Eqx7V&e`K+ba&g8(wNX@4bCgqexEp@e=PfI(8lKn>Z~PXU+XGe&Rw`}S(k=Y
zk<lHFqDiZ2MKf~qZ1{{HI@s<md#}9C<B6Ea!@?Pzt$(kuF5)Qaxn(6Uxs%nRhiUhz
za>Z;P5pmJPOW7P|SA;`Drc}(BWhKv-Z?^MhoC~8+k<iAUb#@=(PC5M9Ic?J){^Aq7
zscXa9?~6a_-WnauxNaYFporm2-5WC3-beoaZ_w0mGwJVoUW2b{pBd$z$1l43-iftG
zOgeVAX?wMO&-Bgz#4Kb?e><yd1~q=*J8tYb`PudJ{~s+`*u}QoeEvGsqa0p*wce+0
zgefna81l^4bn00z&hP);IUR{;sQ-6A_C|Z`xl;9%qR%beE6itmu439c_3E7ByYF|T
ze^Qa!zV_*~+4<>b`?Pe*#Klhfzq?S#9^wCb`uWH2s^t^HV>uirO4R?<y;FHtrAJn0
za>z3_M-h*<te?;2-#vKBAC_PeU9i*VM>1>2BA3JKc<<D{zFV_1ReaToOW8+m+SJ^?
z8{vF=N97?^*(Y-<{>tz8_9fBg>j#svY4xY)*VU`<s<-bgJuUMr*T3K)=iiDqOSebH
zi#~mK|9_{u{7-iIJ>O)q4o1A&_eA4f<-<a`IiHvk16@s03r@d(pMI%usm-tF(v|71
ztWuF_kEWIvKHvIy|NezKC(cxTV%+y3_45tq^29J_pP#qyvKaL}>ML&lxO{hB`qO)C
zUyc+$<9)P{{lDY7kfZYJDqCjD)z8@1s<*B7r`~qI=wqAx@1GYxZ+Ggi{yE3R7Xzv;
zsjQQne0pttEzjbUO#)wnC!2iQz2E-SPxUWP=Y}u)bl^((1*<tM*^6V!e($|gaFBD)
z|1<l4Jn#8A<E@@l4R6ZSuSchR2@Vc*t-W(VUcPEhasSJUtNJIbNZM*Oqi9l_<Fr0U
z&DUk?p7WRAKN|jY>kE%ZdU8jc`|tRhPd}Ah|KC=B?{8JL$$X*jpRL<(d*)^H=ZY`-
z=AYhe|7x+sg16N17oU{Ff-L#atzTc*w))goS=Dme<X$h-IkEVEIk$X1PlCWAmfEj3
zirGKjxWazp=e^fwqCaRFzRmi#=K9C;?e*@<LfJnGtl#^?Wd7Q_=VsOaW%btS+LW$1
zf6cuUr;f2-Tv7P1;IZqAZ?8|d?C3i#zi0aM=XH%YSGXEf?0Eg2zxY!_j@<v-v3s^W
zh`!@CvE$yk_ws+t-P4Zm$UJoPikF(uM-4}wFYm9m+FF-ih=}(782J3-m&4`B$B$q6
z{qF$x=V{H0CInuXU2pS=<L+9!yg3De)svY#_fFj{U&HgZ&-G98Det$x-uC<KdvJs=
z(=ocE{Z+-EfA(c7zW;NVTkvZ2^~#iE&wVOiS(Z#az4PU{`r^CH{sk$4lWynjYAqG@
z&6vuv?v0~Q<tIiV>HE(YRsU-}nWT6tqW|11*UWiSWvy4QT)Fz`9qIok<kwmDf1m$T
zJotG{%gww)&-sO}Tm3xD++TF;!=q~<>lesW=>L5uA5(B@!k1OO^E-ZAUi~A}V^YiE
zr>fTLJMJ`1dS8?JT=Zc<<wTX2%gz7TAE-C|^{M{D9r;xc1HDUfTmvt;eX18^Wb9qy
zxY%?5rS)I5|5|<6T3^%fUsT2L;3HQ3!rN=N7anz;{A2&j&yt0Q3L{LE4C@<~If!`F
zhX1UsV|r+@YR-aXo7ye~rhoX<bA89>MUyLjzikaMH#2<wbNP)&n?HS7o&Whw^8a?B
zUV*Naxy|v84H3-!|9{_~@q6B(H#>_fU##PvX<w@qpL6T>%AQ3Uo4!4c+_NV`_`A&?
zlg;t<(;EFcGYon(R4RVoRKFo%Wcar?a*xSdMXm6&&%f@BZ`6`kcpja<*WF|OQQ!T$
z8}(w@i|-!}w#Z8`oH(Js?U)-EyUgjc)90LApnTk@^wj<r(JvNWyzXHzX~Du8+2DVF
zb(Y^L(Jy|!?{Mq!X`g;9zI-EEUh%?(4%N-_{`+m3v(<Y4Ha3UqHn;O7ycM6naj#Ls
z{vzx6`j5KKZC}#<efHn5&FT1K|NV!w{rz3zH?|$0Ki^enrb@(X<(qqYECjoQZa;7j
z3sGuxbBg$JcmMon^UGtKJv{=p>K~Xn{qE81{BsF8D;vK*JF6trzj#H|x_y7`?#(c4
zetll*NHzC;<p~oy*`zo*#s2+xA#783e{Ih1Kgy^2tmd2Fn{%)**0<)%KP9cQn#;@j
z%kMqhRQUHpXwI_7Y~?ez&!2m2RjpF+{L0qwW%g~hK?mEl%{IS#eAC9hp11t&f0q2c
zAGOq{NLGG{zr2F;$j9rZNoQZ2-|{^E@wd-?Pbc$-75&q#Pd{edpUC<4Y4iH+@0L4@
z#$UB#64b5#ef{ztlfQ;fW?l8E{wV9_zh^@5aoMvU(|w=K|G$H^e`jU#@^gn+!(+~G
zY(6!k=DPikUym;De9`sVvdlyB`n^*5?XhKMDoG}$teII^i&kp3Fs=EUVRkuv?UQZw
zK2O)0rxn!rTg&UlG}wzcKG%5Iczj;MvF!7PH*`N8+_O|gWah3_QorVw@U1^L=io-w
zs<1WdAMaav{^O0>?VGAk)@%#A^6~7W@``uOU;q4!zg+qJ?erIFp+>pCvRjfkU#-r5
zS^UUykJV#Uzc-VeZ8YCr&*%Q&9x|y@;^5yWvEN_LVSY9D*B2kRITjO>G*8s>-#K3U
zf0oy|6_YLVGQ<!4WAkF4cjEH-eP_7)?K^I4=sJIG>SC42S3=xoRGi&=rs7$l&sXae
z!n;1+$zEUl`{(h`?+*E|=snWOUYm1M`q7`+|BuaHzyHj}om#=mY+REqL{owVT%}H3
zFWWI)&uZJ0<MOqW_VR9Y{_FAQYVir%Qw0+qm}OpCVwoeT+PUV95091aEydYtzO(o^
znjCE8mhai~C!qUN_u^Gd>w03UzP{^~IqBMI_UYvs$vNex1UnXWEa;d!g|Yka(H+75
zYWl{L1Jz~D3eVr`yhroEvAw@7vK}q%d2^pv<LRo+$2Tolq?^~Ez+rMVbYtREK9+?u
zrq8dP*0FzDg~@?)r`-D9&8>H+X7V(v`K!B^t^dK9$L<Li53q*E96Qdhqa>M^boiUE
z$<M@lul1W{R44Xq-d!Wkx%GWxk>1Sat`_af&3^ex*4EOwyA}Ga=8E-S?mId~L0y~K
zQ#yBA^pdLcVr8bkZtSemJpV4v`|ds~rmK#xm=!NXN*ez%Y5Vf(*WzCbk1x15W#+u}
zkKwN+Pk0^bo7eMZ;%R05_s>og>m^pK7h2HLeNOW1XYK2kMP#oSu4`3LF!6U^(RHd}
zqI=Ps6^b)lZY*C_v5V))lw|h_FHMW?1Ugq_7xRQBF9_ei@64RfjKS9-#})M64K9ol
znY!<=yHQC(dD8AhCFf72T)F6c;+%G(!}KJ(kJmSCuQA>&{a3U0O_cYZb$vmLw>A45
znf0UWyIkL#3Q6T(Q<!2aUp);MD6P8p?>5gaeu;xCkIOZ>GEdgq*HFH$e&VWK3NLhe
ztRAuK5`QKve`Wfbd8ItEH>M|Lw@f-D^D3K(J!VSRxnnnq^&fWJi)|3nv#eBk|Mq4}
z#y*DGce|2`CqLX){vpC@j$DoybDY0g*k;DHhkMV?_;owa^y9_Z=RZGs<GNx|$GJC8
zmfio~=o`Ow@$&-{d+hZ}b??>w4|bWP6*hU=y}R5$KHR+?xn@<4*7}9%Gbi23dR|cY
z$5VY${Eb$-qp>RrBVJx!J!8gE|80713Nj1+tuDP{$`<+QV9d1_3(N$TZ_Lv7y{IDA
z-hKJle8=mDVjLER@87p)XZ-%orbFrP_Z<7Y+3tnH`)TX<+b+1*cFOjF#)Ea$?-LGX
zK36QUZ!`Z_D(S}Aaz{@jZC%>4;>!+wLBc%^YZi2I&6`x$V_T$g^cqKOb@_3#FCJaR
zn>qPp9g<F}yDZ`qaZSCl@p!SD_x`vd!EU)2hh&3WS@xx6`af4QUtvG|WcQc$Cy_s*
zA8s(10@^<D+DX7cIdj62FUp|(2kf8r@A~8&X!=FksO_}=ZV^U{uP+YHsrzQS=Efd&
zM-QPcQw3(JP}hmKA60NLb-WGQz}Yn8lNh7dn%wAFbEa6|{l4h>3k|Kee)H{GPjvbG
zf0!(B<#7UI9gB;j!?fE;JsMs{q7#A{Sa>BPS--rz&Ym(;&i41krRM`!mcC@LjOOuj
z4c(t?f9~MrFPUHcPE~zbrg6CI&Sv&xztRcs%@(du_|xLr@0M@3`1qIPkL{apI;%1t
zo8vL>I?t*uncT~}WhMp+GB&zwYAmpM+88{^Xw|J{<$((x7nM6^{0*MO;^L_ivU&3U
zdv)3e?;M;{dMM+}L9bP>ZfEaR{hzsN)wEyHS2;7j@_}qMINY_QWZ{kH&b@hg)?OKX
zZ-qU##TIYsnH3_yDyX{hg3>mbU<YL-58XrCm#BzXZCS<Slpeceg-Uek(nV@dukFj{
zJi2((f4zojy7^AgmQ%0k`o;d@c@(o)JXBypiP|R{vwearQ&Kj4jS5L+brrm^@mF%*
zuFr{m7GHh(r0z`JD}C+l_Mi=_(O$cA<#eleu?mEqC`iAw++Ti8)QgLI+56_Ki`e4y
z`YXG@s+%Y8=k1=F5cj*z`(*5;bb;qmHgBJ7-4qwfzvBA5&?Z-I(TyBU3KKfmIoh2Y
zTKm&&PBve@pZe#>XN|KPHhuf7f9oB`!esj`^JZ!(tjN7SW734^-=>=C0Z;m(-oJ^H
zs_mLv^ZlWq>fh^U6yvLS7ibuDx-2@ZJ=5pkaof|Gmmu3;Oxn4bI9F+J`|$SI*){Ro
zXPMutYqvV;)OWk{<cid%pO)5atrGUOpLhIu{CfYj%U-L@Yg?t!!KPh%s_RF|HQ|~c
zN29{@Vh>Et-{<X->34Hk)1nm-dv1Pt6MtZymi%3IzNHtk6Ryq*RpF0VG5wa&(iK8`
z-M^$C4xj(=)SUhsyAoCX{cC6K`XwN>xYe$<>QVL$=P=LmUsFXlReqJqYVuTRWPTbF
zywjM+%J^1>Zua$jl}meScBTr4Wq&wZ<|^u{xOoSArC&zRDIQ5X9_Jm6SEb7oJCs(O
z^1t&~ukg{0$v3|Jv_2ENZSG`03$DQIh4KA*HD>8j^0gbMZH=!vA^g6^Yx=y}!l<)H
z4x8@z{!TsT{<e9Mo6{e29}|c&DlmDyd#~KpZTlG-D|U45dS&3=XWM+y{ma6ARpt|S
z%rScMqGHdc_<{ge-<QhQIe7zTE<D_x=*g0~W~Ji<msMrC&la}-KbyDz)PlIBI`h7=
zyszKamA~&t%W3u41E1foU3&k8VouCQOa1ddC%m0~{llBb{ttCEm5%2+hCcmP8Ikdt
zZ|l$b3|=$Lm)Rf6{w$`<Ewt#)k!oZ4^oyUau8H|&cYg10OUu~m<O^F;RRtSk@9RDM
z_gg+KJy)RjT!Y)<r?W#}XYjZ=FihPhD&Ttex6!*P!4^NCOqQ(vb8&}{Tb2CGNxxgo
z#W&@iw)*#enfA}mQzU=X@E%FEUOvCZs<zAM#@_WzJD={d_b9k3c2{dv+dO5q1xJ`P
zzR9%lzH|S3@9diuYuu(B;Y)q=P+_Yn`?uL$4vCTT|2{Fj__*UA`<30BceO2h_4d!>
zea9^xt-5Wo&f{>r{0ujPi3{rDO~0)P^0Uvjk$h$$y7lXpI|p*FPrPOI@xnhT<_Rhj
z&)=(_KmEB*((AUVHlMF_r#wl2v`sR4yY#j@-n?($Ej0y2@Bi7t)9lsT@5rj0HaeLh
z9JO(AppG`z!W}*5GF<Zw{++mW8Je&&>o{Fv<jhy6?>V~kwVjvbgg&0UeV_H%%FZtS
zxQ6qI-p<LZ^=5g*)ybV(XIr@IOox>1%$bohN>-+Bn{>^?ZJD&^CF6gKPOiC8AiX`V
zR#D8KZ>H;AjT0q`97<)=*$<{M%$=P5eoy=T|Nq{jY>R%aFxM?zO<P;KSLH(OnVB)x
znWp{!zlOzanW1s!^rGrayAG>!`(8wu<^E(gk1J!83S1EVKiOixg>>NE@`gDxXXdF0
z+E^}~V`4I0^ZQ@M@@t!S%!#{mPJk()D1GuwO{LBTg$D)I2i6^*dAt4o-7?$y`9^cU
zr}-RCDJptc`aQu|x*#<zq3EAUenegW!7tidHpiJyXiTkaIB-)u_VhQ|??u12PG<-(
zOOctZFVk=Ae7;5d+_Qf#zkj?KZB|sYN2Em8LAf*HjozLaQF~WfZEjs7!$0epLnhm5
z+2yvKwyS4cK5siwU4Bpd_OpCNMrWRgr7CbFY}_4O%CKn3@!2ywJ?CFqeDmEQ<yj|n
z-kg1(zkA+(-<ju{BrkusvBsug4r2>LuW9I>&j*#<diPJ^6puf?EqDL(b+7fDg<MXt
z$)ERh|GI4TYX0pp+d5y??{9v|w4=D{!|~ti4kurIeCEV~Hy*u9OVzSB|54%)^2=v4
zo6e~!miUsLdt&tr$@)7CEOR**D;{+I?z3#s!p=zt)r-$A+01aFFn!V1B}~i1-rqPg
z=O(wst25l1dd0@SekA`qF5ExqeQELEpU$9hs4S}ry{~cZa}TZ!&w2Qg{oSL=zylL+
zH}B53XxtRjXn6SEvw5@ijyybj@X-mi%emGM>I`qkRIn6#ZI`mWx5mduYT~`*W5stZ
zXV3WdTe`wY`QZuu$9?vGhZY2%$OykTE%eMwVgDy9|Nio|Inj3^WN)<9mFnyA=TlD}
zn)##0_0Q9`=n3EW9{#d^m;O-vxk1*SihI94{onE7&E^xQ&#G>&))Z?No4f38sfve%
zvC#&rpzC|iWvFUj?{-sOv}(fS#mVP3^M5KyRsHrx=-+$&$s47&-k6ievU=g+y;~N1
zjoY<5*5#9ly;$Pqdk+-vCQ5W4HRv+r+7$CPP2O?Z<SSRM1wZdzuziQYj5=q}CzqG}
z6jeTLbg-fS-`%2%uP^aG1Z~Fju#rpu{q1eZ%S)_t{k~t;{u8=D=!5Aa{x*-F+gN<W
zto7QrTKaZ(t=~887MpxnyL5A0^9s|1tDYv;%oECXUC1_75jK=!5{&V)mI_q3^kng-
zU3|S4+m7d{=&inyYsOS358kcqpn4;~J+xLiYVp)JXL!Ec>btijMaj)MnPd6w#xNnz
zV@e?*VM>b+&VHyU=9^G{<3b11#scBrTB`zFd1v0d8PQWEV%fNU-^T|<Z8au8oGk80
zW?YqcxTEQAp0XgXV2@hit+yRbUG@*$FGy^h`DMi_r`S-{-bR72_U;}YbD;;H&6QrA
zx#2VS{dykN2eY4YOMW;zx8cCEOS3iQ3>jCenubl8Tv)c||E$~jdzuBs|F`bev);Eg
zN9U4DYO1KGNs8%w=?0npP3spmwP-#1TGgSN62m9`al!l*N)HN_e0!{xBq%j|(T+KN
zX|b#v>LD^^)+ZxcPMy*cacn=HtUfQG;70C|rhV6}LM$DVYn7*^*|z<)OUim>9Q8$z
zC)xD%$94a#A8m6z;p%?T;K}+On?7CeV%q2-EGim$!s&Q%fRN!TEp5)DlP&H>?wBja
z5_<6YXT?{+zkH4yGBG<E^od&wwrTzF^$QGA!ezNrUa#N7a{9yS9e>}{{ctig+Elc0
zHgAL1iTCQ7FE-rvQArY%s!5E=Q;+KPP!e0XY#rm|qm75tj=axyN#C^9kEg`V>mApu
zMMir0o9{R*SvhgltEv(;Yrf5LlDuM@`Qmr~Ub^?+mA~r(Ed>{EN|Jxr(sg=v=k2`4
zw7=VBoQvm|y)l<;zgriOT=~Dsw#<0nv8qW;V%rb%b@@2w%1%1jb~tOn#&vB~@xo=x
z+2^`BCU00EKFNe-@!T)Y#qzC>Qg-C*DL7wRb0y>9zx{VSZ@fSJ?%ShMYk`1|zyGd1
z-|+YS(lb2eR<>6hGq~6y_T2REoxr;yqLk_Ju55$SqE5LYXG^Kxglj4<gC>+Z%)8~G
z(sN#Xf#y1)saqv_(*6a{-_FnTE!J`>KkHmp9qW)!or^kFX|eMjxf0j@d;j`X3SBGp
zYh8azJ`r!(Hv6Z*vXZQu!pk(g+V9tA#kO~Cd!73~<~~#ItA7u7`AMGwZAte3lP`5D
zvrhBh<b^Ri-)8IWdllAWdHBco#q|p>eAX`6rgCV7{GFy<UzuAj2@ClB5YMQeX}dg2
z=fL0kzsleKJvaLKMwxG`+Q0U|%IBT!KQ5&|yjQK3>~VXqJ-7Xl0J|@XW2S}#wD>8-
zzb>irQByYCJuUWk7H7MNPugim<rMBFhYR7y_tl1~2o|QNOnzCYRP^GK=+=45lpcRK
z74qG}eNVCLhU_}#wA%l>IZ!sd`t_)`Hr(&mdbKg~mfWc#qdxl!EhkqbKfR>tac>=u
zbU@n&VV-D5kF!1RZcpxV$td4>^5iL()q?e3R1O{N`SU+yS=RG3tyv<!zP<B{{h6lK
z#8q2W>aP69!1C7hugRzW+pKUoq`2rJi`8Pm$qRGC>;$=myIdx2Vbf82TtBA<ysO<L
zv(C4IyUOtI&+rX9_gd(_Y;dy|aZG=DZ1=ye6~6hkHd{D)_I|PIo9O!G<;Giz4=+fc
z`YpJ2<>JD#ReO}%7Z^&Bv=y|lKEHUg`PSGcho&vy)!px`wZQGS2~X38`Rb+nKWXwW
zYEa;kRr=^U`&H$?`QJBhRP6Iy?;09fB4#ZfQ)<iHWfXX;azA7z=;jN0v*NVkJYMan
zIB~qa_+HA9qm@r~tXZ+@)+@%)P=P5d^5<(?eHqowV=Os0hdBtinyvpg<#~Gj$)(|O
zM{em)U#L4(!f1iDR7i&I{USNp+jsuYiON`W>txULxF@sUhDz{QC0$hc+b?yx^6~K`
z^Q^$og*U737PdO1KcB{>qqsTgXHe2Eo<4~?+#*ZA-k7p(N}6QQr)?D*SOXJ|ze?3v
zIpO8a-Rjoi`pF;dW^RI9A@Xd?``=-bEQ|~1s_$94?z~pYuJ3Oe<Akifyw`7Bpce7*
zCwtAtG;3Ddwt`zXE=rg(X%vRlZR<$Q-u6KA<@uY-Wtt<^<vw*T0iDz4Ir-V~MM?sF
zWoIN`e@kmvC6vP@oAt5o%1@3?fnO;?yi6xkK+|0&Mt_{{-)K@$P**ZhRceot{ePQ-
zNr`J=!h_}4r?LrjxE(sncD~;3*9PM$&9_h8G<hmH=WkV#!R!rDYa?{T-jrFZP44g7
zWzd!MX2N?v0hU>>zJK_{tuJU;{`^_?xkH%&k$WEK-m816TExXA5j4xuIhJ?p*Io}R
z<;a^iXMBlDdZ_R(fXC7E!RPjZw3MWaj-jchT}#xh)wvh@d1vJZ@|u?RzUT=n+GKn&
zb7{!W1ePl*2Lq<e`NL$QvSQsDB@y2T1=S}%><CDmnV@m#g0PxVnMZg^q~jEm9eeqt
zq_a9#O4<0aYk0nT_3DCDu0>AUmAYl}I>lyey$$O(3Wi<+-CEM6R#?5c;8wJn<dZ3l
zJPS^Qgugv{UF6uS4-@7-UASe*1lIL6r{2w){Bq`dH($4ipI5JY^yPdiDz^MQN!#(c
zLvz=v6IZrq&!76xt*<@fi+jPYm<*SxZM&*W+@#IA8w)P@TQKr<w}H;+Jo}DgdR(z_
z>Q6nhy1g@=r=K}AcjD<5;dTXg=QnFtsK|If`jYaf^tNEgRJHy65kYE_+z+$69(_uC
zSb8fXYfbvecJ7Q#C5v}xi1%OZIRD&P;D9{GqYuv>Y|2=<@{YH&Ld(r(O4g^nPc2`+
zL@C#)_=xY)r}JJddG9)l$xdmbN@qrggw>}aql;VeG6DlaX8wA`_05!Z<E}j^J7n)a
zpZ7=7<7~x|MTty}4A$oP3z7=zccocpe!99L@v>M=BSTvN%Tm>D=UYL3ppy(t_B**u
zl3<En5oMacU&VTVo{Oqmoil4$-~zQ731`LZH%+m)lC;$G^K19e%%HRPvwu{d;C<Ss
z`2Aqagzl8A=vNmXZM~eewc<`*jj!jUP8O@Z`|cVlRhzF*?DtRlwDtL=&E96KxR(3w
zU}&!8tXg~ETj}W;{~q}$ZJL&1<MZ)A@{U)BK8EB+9GfEke37Y?XUpLXoxDx5R;yOM
z=4e|aW0=PzKCe#Z>1(^^t&;f{CHxkh(sJ&zocj5Af9m^RsXmo?W~;)wZ_YdWG(7gi
zt5)O4J!#@>YdwODir*~_H&zJm*2pTT-gM&pdG~3e0xT+yi7$_H&)k}#ef?gA@vUE9
zZ9e|7o>TWRWz*4GpZbT#Pt@=|pKH)^Q{OK&)|JQ8Lx8(=LxuN_N3XO`nr?Y|C3q^!
zeBC)mZ-2IT6>|+adE}+Aexl3ygDH~Y8?)YuRf+Yf1Z?lm-}WZ?#LVSMmo?A){t(OZ
zUf`T{s)F}7{rY#RUru&?Jo2S{M#*1a5s6t9Z~nDwN)%d6pJ&_V|9tPkXC0G^{okBk
zm0w<b$a1oz=?e7~TJIkFyH4PFU?7m2^<b{#`HdA{&9+!^h9*CL!>yP4<$%z%T7&M&
z?(ig&SsgA;FW%gYG5sl(bzLv*>n~Xqlh8d$8;kFC{+;pfMe>#7p)7M}8mq;&ElktR
zKl7jE<N|fEPzmFAI>-8@@6M`T|8|$xQ@=a=z7@}Vf8qOb!^}U1I?)!hBjxKF!xpP<
zzRr=iqeQ7Jc@<Nj$j!}(to`rr`l-k`U9;ZtV^y|f)U@B9-@N`euVayiO2qkNy4IQ}
z%QmIo7F#8K*dr*q?;N!Jy56GEw6Kz8b>O2ZA^V$yYEQIpu{YL|GUapNVczN_=6TSm
zD$Uw=)_jlGKg_32KD~hTr2_M#!|zurEt&T7)J2JpZuU~$k{d0x=BjvlOzN0)L*HT2
zf=dyx?UtJ-H0g+UDJ@>LV0(q-+cld70wO+NW&0&xDWu%azbNam(Y)$p&Z*1iFm$Ta
z)lb=cZr_}LmQu^?{%|lp_sHA(J1b1LX72a>QAa!7za;*fWBKAUd*S(($2T{BGmfjt
zwyS05J))=n(cIPY=O1p1s*L=SB$kV_4!efW_1frueD&q|DM!0ZznqkAo7J=F$KIK5
zVkb|!_(<c~o6W}!3w~VG`CY@ie!ms7Z-M!|O^Q4(CLX_b^{KtzGyU+QOC9@K{_wV>
zvS0U~*Au!f`IfN%rU!@5@BHz^QFCi2Q?F?BOylp)Z2i{#kK3nxy!q$)4C`#YzyB+2
zn}3z0{(Lo+#b4ssIo@oJ4n4MS@0eCOX^6ONQp;Fzqs-xF<>qIBRexD{y5{>@uQr&m
z`o_L;{)4`=Gjm>?-}ctJ@ayYlsjq_9qh16~<6kw&NI}(hv7ZfZU_<HrGtUmM-&TAs
z_4<YVH%{J>Ja6#(?awc|yFu3@B*YwVy&mU0P49?szwhJjskbfP-$;)9d^T#<nY%mR
z+?g!cwELs!FHI@SzG8n{v02|fp19mCQE=$y9HTUb)%J6b&n`~CT<wtG!hA@Y3$g*O
zdjG!p?)x40db-T&Smma-<BwKV*~#OJ#I4sn={Qn4$NXcEm#xxe`RX41eGz95Y^rfd
zsngtIB^=h>COg|a@Ur&&-r3t0lvivx_E*NO@Evs2?R{@tQ%t*9q>)g5?f329mrfHY
zIkAW>GN+Jl<^GeOeI`GfzU*kAc4xp75zC02TZfMX<dt9Mp1@-k`1NAC$=e&fhUVSt
z!sqm_xVla5_Df}_jGOFU?Yn|5&Q&yD-ymx}eRIM)>lS8Hl~l{EcCw<YS5Hu^t)7(H
zHF^5v&aL1h<~d4rpEa9if4myLxwzamN-Ofqy=s@kc1a&KM7J`&X@DLvUozX{$QP04
zX&=sZo&TXWGbKK@VbP&0J*^5HH{S3W8ST;NYPy)_E^_eAxzs~DAOHM)|9|I<yOZzx
z+Hr37bM}j4yj|aNhC6({;zW-{U2JT;_Ote-x^`sAx%0g`dVQ;?%Vg%De<_X&78Z%~
zE>v4pmiJ)quA)DsjEfo>_H~>&u+TZ;%aPkTw|91%X75|Nq1wLaZdiz^OIH4d8<{^l
z>^+>4mt`07Rd3I8xng>0?RkSFU(=jh8>eQ)PZHegB{22V73KK5`?XnJ9Mhin-A$SL
zE%C^NeW0yg^9xK?&#USG<&d>MFtC5#mzRNEGX-M3xhEuh_M~yEEoQu#bMWbgNBKb)
zXX^6ETTXhkC&x_ZZ`TgP$;roiRJ0EtJ8|*&r<;>Rr@YTTuy=lar~nIMTi9=xDYd3`
zE(><J>$81*-|G2BVr_JwNLSUjw1oL`H@<pSmFuU*D}Cdx-@7t{H|*C}y`-b3dTTyD
z6@RmWFW^?b#KBzc44?X&kTtgPib@ak-Z=ja{GTvi-_fLUh3c>OiYo*zRvJq3u6S9R
zka_7(j>HyC<(+4g<-VM=^5J<}wn8Dpw8EY5sOb***Y9d)G#MXHJn^Z{U6O<iN^6fs
zH|4GL(t4lx>ie?={%7T!da)iI-r#UL?R@R*@DFdejWwl>nl>$xS@k_9+xV?%Ra=kT
z@tet2OF;)#dRlg_ax#*07GRllQet8SkE)_k{WI+~x90x2Xta3M3D2FX)~2f#mBi%T
z)|sY)a&)*)#acF<Ywtv=x9U{o3z|OtI;Y~J0*_R{t^A3CR{HZ_7F}8#ZMfPl)_t*C
z;)fTPSHx_d;BWU&<mN=@z-8RI7y9ik9%<vXei=TcQDfDLqMr43e???06+AVHqPes!
zH5YRw?S2q*)=ELNXU)?ox0=#8bN_Q^X<eAbuxibM6DvbGyB00bD0$?Yx7tx#?#B1M
z-9{1HHJ4_D1_rWj^<DRDQ`O!nS>mFBQv|NY|Ks*^Ro%WsdiS=Rw6oW>4R3CKd2efn
zX!QPnM^j;orS)7ki<h5I-f*IT!`Sd&(h}b%CYg0{DvTvU_h<fn`oVY#<H-f?&$lou
zyxOAisoY<5)hegBNPDk4+!c?XN=oS2NN@K{xat16GXG`Tx?Kyo|FOwivNBs1{ZQEV
zr$B6C#G<b&FCSdL?fxpU&``tMcS<x10&j{e&OX1dW%afVPhR(ak+5mvlgo2d5c)DX
z;0uqu9n<=2=hf}(w@sV+q08s#z1W;v8j9LnO)C_Q&H0<ED;}@?`Nig0NXx3Pd+dt!
z%;T!08r|eAS)1!NoYBqiJQBI_kMq7)!41Cal23N=me_vM;fdO~v!ly;J<D`gXA#}n
z-(OQtztX?4!~Be3Q^&nGNgo>Id}<TTByHU;ZZ}zf_+M0LXyJ3Q@~Ao%rJr}J1S>1~
zt{!vJdl}lc^P%*;>VL9sa|;d2y5)N9`V(ure-^xxPO;?uqVZ`{?pdR_s$9F{zPtMN
zrN++K{zTU9-y3bVvcgaIcFQN7JaatRSbjqGPF1z33LO$9RolYfCFhsgs%`Z(UA5}b
z$=6MhGY?nG*PMCj&YyHtoqI*-nUb@g1NK+%w+a>AeAan7(|5+&`^6t7@Ezwmu_kI?
zi|<vg<@0Mwj?Z3KS$kaZ<NZ}yZnN9vmL7exBEmIqy3wY)*S@@33vM@jUhU4(BlN*L
z?WWIUwX^?u+Ltc8)A(0er?hF+su{7CbILEO%$xH~=KJ5D{V%pKafY5hDXf>`&@ZIa
z%>L<-^!uXkxB55S%#qk?6)^j<Mdp!dH^Mk3aZI`~EzVPU(dHc%FTWU9yg4cU<FW00
z#T6HdpLFiXP0LymQW@&8H1(9uJ5R@FN^;dN^d)r5N;WmPi8zV~PLbfgU(>eqbIFQT
zP4`}bMi!UUef|BWTVluOo{t=%t#|&fpZ=rhvA{20=|kM>&;EXBl)dzdzbE%ch4X$p
z=O<V=xt(P2v#DZ{Qk<f-xXE~P%Vj^Q!`;U5$uD;sJ6<S!6e*r?B|AXy<-P6ley7>L
zytypC{py7|O`Z-DR73)g&b;|_!Bf_7Gu54Xt8!I$$eOiW(z_v3*0p=eLgCPmwVlG}
z?fOqGpZDmb`}^Y=7X*7%Uetd%DPz2dzfAMg#=~X?pJ!_BQRS4#xfxUS)<Q;2w@FK3
z>t;7rM;~8KPphgD?c|GJ!YdxRa$oO}Z&TsuI6HBHY^3U(%2ONnd^k3{=HtQJ8jBi(
z)#IEzr!`v{K3yxu;5bQug>~)ClQ)ddC1m~HCqMO7bz_<&3nSx9-__GMW;CDTJT7}R
zYTwc0Ym^rKn<SU&=#asALoMs&wcYZig|e!9s}@~fa;7ZM%l$KSZ~|11FEjoppVQRz
zbZ=ya7Xt%>YKdz^NlIc#s#S7PYGO$$gOP!ug|4BIu8~2Ap`n$jft87gu7QP>fkDXU
zg`y}La`RI%(<*UmP+cdvl7WFi17t&Ha<Wx&enDkXW_m^mgENe!;8;+Qld9mEm#n9w
z0J2%%ht-jRfk7OmCndEwxhS)sBr`vc!6mb}ASba>Tzcvb1_lNOPgg&ebxsLQ054{@
Ae*gdg

literal 0
HcmV?d00001

-- 
GitLab


From 9a9f67ec10c4ad5e695df0f7d1c63a69f763fd56 Mon Sep 17 00:00:00 2001
From: Grzegorz Pawelczak <grzegorzp@graphcore.ai>
Date: Tue, 8 Jan 2019 21:37:08 +0000
Subject: [PATCH 0400/1095] Add SortSimplifier pass to compiler pipelines

---
 tensorflow/compiler/xla/service/cpu/BUILD             | 1 +
 tensorflow/compiler/xla/service/cpu/cpu_compiler.cc   | 2 ++
 tensorflow/compiler/xla/service/gpu/BUILD             | 1 +
 tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc | 2 ++
 tensorflow/compiler/xla/service/sort_simplifier.cc    | 7 ++++---
 5 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD
index 1eb42cf5ad..e325b75260 100644
--- a/tensorflow/compiler/xla/service/cpu/BUILD
+++ b/tensorflow/compiler/xla/service/cpu/BUILD
@@ -133,6 +133,7 @@ cc_library(
         "//tensorflow/compiler/xla/service:llvm_compiler",
         "//tensorflow/compiler/xla/service:reduce_precision_insertion",
         "//tensorflow/compiler/xla/service:reshape_mover",
+        "//tensorflow/compiler/xla/service:sort_simplifier",
         "//tensorflow/compiler/xla/service:transpose_folding",
         "//tensorflow/compiler/xla/service:tuple_simplifier",
         "//tensorflow/compiler/xla/service:while_loop_constant_sinking",
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
index a6d92ce10e..60e39506bf 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
@@ -93,6 +93,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/reduce_precision_insertion.h"
 #include "tensorflow/compiler/xla/service/reshape_mover.h"
 #include "tensorflow/compiler/xla/service/scatter_expander.h"
+#include "tensorflow/compiler/xla/service/sort_simplifier.h"
 #include "tensorflow/compiler/xla/service/transpose_folding.h"
 #include "tensorflow/compiler/xla/service/tuple_simplifier.h"
 #include "tensorflow/compiler/xla/service/while_loop_constant_sinking.h"
@@ -285,6 +286,7 @@ Status CpuCompiler::RunHloPassesThroughLayoutAssn(
         [](const Shape&, const Shape&) { return false; });
     options.set_enable_dot_strength_reduction(false);
     pass.AddPass<AlgebraicSimplifier>(options);
+    pass.AddPass<SortSimplifier>();
     pass.AddPass<HloDCE>();
 
     // BatchNormExpander can create zero-sized ops, so zero-sized HLO
diff --git a/tensorflow/compiler/xla/service/gpu/BUILD b/tensorflow/compiler/xla/service/gpu/BUILD
index 5e81281bae..175f2aafc8 100644
--- a/tensorflow/compiler/xla/service/gpu/BUILD
+++ b/tensorflow/compiler/xla/service/gpu/BUILD
@@ -717,6 +717,7 @@ cc_library(
         "//tensorflow/compiler/xla/service:llvm_compiler",
         "//tensorflow/compiler/xla/service:reduce_precision_insertion",
         "//tensorflow/compiler/xla/service:reshape_mover",
+        "//tensorflow/compiler/xla/service:sort_simplifier",
         "//tensorflow/compiler/xla/service:transpose_folding",
         "//tensorflow/compiler/xla/service:tuple_simplifier",
         "//tensorflow/compiler/xla/service:while_loop_constant_sinking",
diff --git a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
index 9be7609508..e1bf0d0547 100644
--- a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
+++ b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
@@ -79,6 +79,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/llvm_ir/llvm_util.h"
 #include "tensorflow/compiler/xla/service/reduce_precision_insertion.h"
 #include "tensorflow/compiler/xla/service/reshape_mover.h"
+#include "tensorflow/compiler/xla/service/sort_simplifier.h"
 #include "tensorflow/compiler/xla/service/transpose_folding.h"
 #include "tensorflow/compiler/xla/service/tuple_simplifier.h"
 #include "tensorflow/compiler/xla/service/while_loop_constant_sinking.h"
@@ -200,6 +201,7 @@ Status OptimizeHloModule(HloModule* hlo_module, se::StreamExecutor* stream_exec,
           [](const Shape&, const Shape&) { return false; });
       options.set_enable_permutation_sort_replacement(true);
       pass.AddPass<AlgebraicSimplifier>(options);
+      pass.AddPass<SortSimplifier>();
       pass.AddPass<TupleSimplifier>();
       pass.AddPass<WhileLoopConstantSinking>();
       pass.AddPass<WhileLoopSimplifier>();
diff --git a/tensorflow/compiler/xla/service/sort_simplifier.cc b/tensorflow/compiler/xla/service/sort_simplifier.cc
index 3280672574..ee91762253 100644
--- a/tensorflow/compiler/xla/service/sort_simplifier.cc
+++ b/tensorflow/compiler/xla/service/sort_simplifier.cc
@@ -26,13 +26,15 @@ namespace {
 
 // If the sort instruction has a tuple shape then looks for unused output
 // values and removes them from the sort instruction. Returns true if the
-// graph have been modified.
+// graph has been modified.
 StatusOr<bool> RemoveUnusedOperandFromSort(HloInstruction* sort) {
   if (!sort->shape().IsTuple()) {
     return false;
   }
 
-  if (sort->parent()->root_instruction() == sort) {
+  HloComputation* computation = sort->parent();
+
+  if (computation->root_instruction() == sort) {
     // Can't analyse users of the root instruction.
     return false;
   }
@@ -60,7 +62,6 @@ StatusOr<bool> RemoveUnusedOperandFromSort(HloInstruction* sort) {
       new_shapes.push_back(sort->operand(i)->shape());
     }
   }
-  HloComputation* computation = sort->parent();
 
   Shape new_sort_shape = new_shapes.size() == 1
                              ? new_shapes[0]
-- 
GitLab


From 530d1b267f124fdb7dd6ce65b94423b8801152cd Mon Sep 17 00:00:00 2001
From: Tim Shen <timshen@google.com>
Date: Tue, 8 Jan 2019 13:29:30 -0800
Subject: [PATCH 0401/1095] Add fine-grained BUILD files for stream executor.

PiperOrigin-RevId: 228388744
---
 tensorflow/compiler/xla/BUILD                 |   2 +-
 tensorflow/compiler/xla/service/cpu/BUILD     |   1 +
 tensorflow/compiler/xla/service/gpu/BUILD     |   1 +
 .../compiler/xla/service/interpreter/BUILD    |   2 +
 tensorflow/compiler/xrt/kernels/BUILD         |   2 +-
 tensorflow/core/kernels/BUILD                 |   3 +
 .../core/platform/default/build_config/BUILD  |  39 +-
 tensorflow/opensource_only.files              |   2 +-
 tensorflow/stream_executor/BUILD              | 703 +++++++++++++++---
 tensorflow/stream_executor/build_defs.bzl     |  11 +
 tensorflow/stream_executor/cuda/BUILD         | 365 +++++++++
 tensorflow/stream_executor/host/BUILD         | 108 +++
 tensorflow/stream_executor/lib/BUILD          |  62 ++
 tensorflow/stream_executor/platform/BUILD     |  47 ++
 .../stream_executor/platform/default/BUILD    |  25 +
 15 files changed, 1277 insertions(+), 96 deletions(-)
 create mode 100644 tensorflow/stream_executor/build_defs.bzl
 create mode 100644 tensorflow/stream_executor/cuda/BUILD
 create mode 100644 tensorflow/stream_executor/host/BUILD
 create mode 100644 tensorflow/stream_executor/lib/BUILD
 create mode 100644 tensorflow/stream_executor/platform/BUILD
 create mode 100644 tensorflow/stream_executor/platform/default/BUILD

diff --git a/tensorflow/compiler/xla/BUILD b/tensorflow/compiler/xla/BUILD
index 45623c2718..aa43fc8dab 100644
--- a/tensorflow/compiler/xla/BUILD
+++ b/tensorflow/compiler/xla/BUILD
@@ -152,7 +152,7 @@ cc_library(
         ":status",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/stream_executor",
+        "//tensorflow/stream_executor/lib",
     ],
 )
 
diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD
index 1eb42cf5ad..de62aa60aa 100644
--- a/tensorflow/compiler/xla/service/cpu/BUILD
+++ b/tensorflow/compiler/xla/service/cpu/BUILD
@@ -241,6 +241,7 @@ cc_library(
         "//tensorflow/compiler/xla/service:tuple_points_to_analysis",
         "//tensorflow/core:lib",
         "//tensorflow/core:stream_executor_no_cuda",
+        "//tensorflow/stream_executor/host:host_stream",
         "@com_google_absl//absl/strings",
         "@com_google_absl//absl/strings:str_format",
         "@com_google_absl//absl/types:span",
diff --git a/tensorflow/compiler/xla/service/gpu/BUILD b/tensorflow/compiler/xla/service/gpu/BUILD
index 30a8bda606..2be5d918ee 100644
--- a/tensorflow/compiler/xla/service/gpu/BUILD
+++ b/tensorflow/compiler/xla/service/gpu/BUILD
@@ -731,6 +731,7 @@ cc_library(
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:regexp_internal",
         "//tensorflow/core:stream_executor_no_cuda",
+        "//tensorflow/stream_executor/cuda:cuda_diagnostics",
         "@com_google_absl//absl/container:node_hash_map",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
diff --git a/tensorflow/compiler/xla/service/interpreter/BUILD b/tensorflow/compiler/xla/service/interpreter/BUILD
index 2e1d2cfbda..9b4cafa4b4 100644
--- a/tensorflow/compiler/xla/service/interpreter/BUILD
+++ b/tensorflow/compiler/xla/service/interpreter/BUILD
@@ -119,6 +119,8 @@ cc_library(
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/core:lib",
         "//tensorflow/core:stream_executor_headers_lib",
+        "//tensorflow/stream_executor/host:host_stream",
+        "//tensorflow/stream_executor/host:host_timer",
         "@com_google_absl//absl/types:span",
     ],
 )
diff --git a/tensorflow/compiler/xrt/kernels/BUILD b/tensorflow/compiler/xrt/kernels/BUILD
index 67f475846e..78d093ec4a 100644
--- a/tensorflow/compiler/xrt/kernels/BUILD
+++ b/tensorflow/compiler/xrt/kernels/BUILD
@@ -62,7 +62,7 @@ cc_library(
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:protos_all_cc",
-        "//tensorflow/stream_executor:stream_executor_headers_lib",
+        "//tensorflow/stream_executor:stream_executor_headers",
         "@com_google_absl//absl/strings",
     ],
     alwayslink = 1,
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index aaea09bdc4..5794ba520e 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -2313,6 +2313,7 @@ tf_kernel_library(
         "//tensorflow/core/grappler/clusters:virtual_cluster",
         "//tensorflow/core/grappler/optimizers:meta_optimizer",
         "//tensorflow/core/grappler/utils:functions",
+        "//tensorflow/stream_executor:stream",
     ],
 )
 
@@ -3819,6 +3820,8 @@ tf_kernel_library(
     deps = NN_DEPS + if_cuda([
         ":reduction_ops",
         "@cub_archive//:cub",
+        "//tensorflow/core:stream_executor",
+        "//tensorflow/stream_executor/cuda:cuda_stream",
     ]),
 )
 
diff --git a/tensorflow/core/platform/default/build_config/BUILD b/tensorflow/core/platform/default/build_config/BUILD
index da1f66dc67..f5a194428c 100644
--- a/tensorflow/core/platform/default/build_config/BUILD
+++ b/tensorflow/core/platform/default/build_config/BUILD
@@ -32,15 +32,25 @@ cc_library(
 
 tf_cuda_library(
     name = "stream_executor",
-    deps = [
-        "//tensorflow/stream_executor",
+    cuda_deps = [
+        "//tensorflow/stream_executor/cuda:cuda_activation",
     ] + select({
-        "//tensorflow:using_cuda_clang": ["//tensorflow/stream_executor:cuda_platform"],
-        "//tensorflow:using_cuda_nvcc": ["//tensorflow/stream_executor:cuda_platform"],
+        "//tensorflow:using_cuda_clang": ["//tensorflow/stream_executor/cuda:all_runtime"],
+        "//tensorflow:using_cuda_nvcc": ["//tensorflow/stream_executor/cuda:all_runtime"],
         "//tensorflow:using_cuda_clang_with_dynamic_build": [],
         "//tensorflow:using_cuda_nvcc_with_dynamic_build": [],
         "//conditions:default": [],
-    }) + select({
+    }),
+    deps = [
+        "//tensorflow/stream_executor",
+        "//tensorflow/stream_executor:dnn",
+        "//tensorflow/stream_executor:event",
+        "//tensorflow/stream_executor:multi_platform_manager",
+        "//tensorflow/stream_executor:scratch_allocator",
+        "//tensorflow/stream_executor/cuda:cuda_platform_id",
+        "//tensorflow/stream_executor/host:host_platform_id",
+        "//tensorflow/stream_executor/platform:dso_loader",
+    ] + select({
         "@local_config_cuda//cuda:darwin": ["IOKit"],
         "//conditions:default": [],
     }),
@@ -49,9 +59,10 @@ tf_cuda_library(
 cc_library(
     name = "stream_executor_cuda",
     deps = [
-        "//tensorflow/stream_executor",
+        ":stream_executor_no_cuda",
+        ":cuda",
     ] + if_static(
-        ["//tensorflow/stream_executor:cuda_platform"],
+        ["//tensorflow/stream_executor/cuda:all_runtime"],
     ) + select({
         "@local_config_cuda//cuda:darwin": ["IOKit"],
         "//conditions:default": [],
@@ -62,23 +73,31 @@ cc_library(
     name = "stream_executor_no_cuda",
     deps = [
         "//tensorflow/stream_executor",
+        "//tensorflow/stream_executor:dnn",
+        "//tensorflow/stream_executor:event",
+        "//tensorflow/stream_executor:multi_platform_manager",
+        "//tensorflow/stream_executor:scratch_allocator",
+        "//tensorflow/stream_executor/cuda:cuda_platform_id",
+        "//tensorflow/stream_executor/host:host_platform",
+        "//tensorflow/stream_executor/host:host_platform_id",
+        "//tensorflow/stream_executor/platform:dso_loader",
     ],
 )
 
 # Dummy stream executor cuda plugins.
 cc_library(
     name = "cublas_plugin",
-    srcs = [],
+    deps = if_static(["//tensorflow/stream_executor/cuda:cublas_plugin"]),
 )
 
 cc_library(
     name = "cufft_plugin",
-    srcs = [],
+    deps = if_static(["//tensorflow/stream_executor/cuda:cufft_plugin"]),
 )
 
 cc_library(
     name = "cudnn_plugin",
-    srcs = [],
+    deps = if_static(["//tensorflow/stream_executor/cuda:cudnn_plugin"]),
 )
 
 # OSX framework for device driver access
diff --git a/tensorflow/opensource_only.files b/tensorflow/opensource_only.files
index e00d063cfe..bade64dcf8 100644
--- a/tensorflow/opensource_only.files
+++ b/tensorflow/opensource_only.files
@@ -241,7 +241,7 @@ tensorflow/third_party/tflite_ovic_testdata.BUILD
 tensorflow/third_party/libxsmm.BUILD
 tensorflow/third_party/zlib.BUILD
 tensorflow/third_party/eigen.BUILD
-tensorflow/stream_executor/BUILD
+tensorflow/stream_executor/build_defs.bzl
 tensorflow/api_template_v1.__init__.py
 tensorflow/compat_template_v1.__init__.py
 tensorflow/api_template.__init__.py
diff --git a/tensorflow/stream_executor/BUILD b/tensorflow/stream_executor/BUILD
index 3c45bf314a..7cf7e75a55 100644
--- a/tensorflow/stream_executor/BUILD
+++ b/tensorflow/stream_executor/BUILD
@@ -1,126 +1,663 @@
-licenses(["restricted"])
+# GPU executor library for data-parallel kernel launches and cross-platform
+# HPC-library APIs.
+#
+# Throughout this file, all targets are built with the standard crosstool and
+# do not link against restricted binary blobs.
 
-load("@local_config_cuda//cuda:build_defs.bzl", "if_cuda_is_configured")
+licenses(["notice"])  # Apache 2.0
+
+load("//tensorflow:tensorflow.bzl", "tf_cc_test")
 load("//tensorflow/core:platform/default/build_config.bzl", "tf_proto_library")
 load("//tensorflow/core:platform/default/build_config_root.bzl", "if_static")
-load("//tensorflow:tensorflow.bzl", "cc_header_only_library")
+load("//tensorflow/stream_executor:build_defs.bzl", "stream_executor_friends")
+
+package_group(
+    name = "friends",
+    packages = stream_executor_friends(),
+)
+
+package(
+    default_visibility = [":friends"],
+)
+
+# Filegroup used to collect source files for the dependency check.
+filegroup(
+    name = "c_srcs",
+    data = glob([
+        "**/*.cc",
+        "**/*.h",
+    ]),
+)
+
+cc_library(
+    name = "launch_dim",
+    hdrs = [
+        "gpu_launch_dim.h",
+        "launch_dim.h",
+    ],
+    deps = [
+        "//tensorflow/stream_executor/platform",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_library(
+    name = "device_description",
+    srcs = ["device_description.cc"],
+    hdrs = ["device_description.h"],
+    deps = [
+        ":launch_dim",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "@com_google_absl//absl/base:core_headers",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_library(
+    name = "event",
+    srcs = [
+        "blas.h",
+        "device_description.h",
+        "device_options.h",
+        "dnn.h",
+        "event.cc",
+        "fft.h",
+        "kernel_cache_config.h",
+        "launch_dim.h",
+        "plugin.h",
+        "plugin_registry.h",
+        "rng.h",
+        "shared_memory_config.h",
+        "stream_executor_pimpl.h",
+        "temporary_device_memory.h",
+        "temporary_memory_manager.h",
+        "trace_listener.h",
+    ],
+    hdrs = [
+        "device_memory.h",
+        "event.h",
+        "kernel.h",
+        "kernel_spec.h",
+        "platform.h",
+        "stream.h",
+        "stream_executor_internal.h",
+    ],
+    deps = [
+        ":dnn_proto_cc",
+        ":host_or_device_scalar",
+        ":stream_executor_headers",
+        "//tensorflow/core:lib",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "@com_google_absl//absl/base:core_headers",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/types:optional",
+        "@com_google_absl//absl/types:span",
+    ],
+)
+
+cc_library(
+    name = "kernel",
+    srcs = [
+        "dnn.h",
+        "fft.h",
+        "kernel.cc",
+        "plugin.h",
+        "rng.h",
+        "stream.h",
+        "stream_executor_pimpl.h",
+        "temporary_device_memory.h",
+        "temporary_memory_manager.h",
+    ],
+    hdrs = [
+        "blas.h",
+        "device_description.h",
+        "device_options.h",
+        "event.h",
+        "kernel.h",
+        "kernel_spec.h",
+        "launch_dim.h",
+        "multi_platform_manager.h",
+        "platform.h",
+        "plugin_registry.h",
+        "shared_memory_config.h",
+        "stream_executor.h",
+        "stream_executor_internal.h",
+        "timer.h",
+        "trace_listener.h",
+    ],
+    deps = [
+        ":device_memory",
+        ":dnn_proto_cc",
+        ":host_or_device_scalar",
+        ":kernel_cache_config",
+        ":stream_executor_headers",
+        "//tensorflow/core:lib",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "@com_google_absl//absl/base:core_headers",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/types:optional",
+        "@com_google_absl//absl/types:span",
+    ],
+)
 
-STREAM_EXECUTOR_HEADERS = glob([
-    "*.h",
-    "cuda/*.h",
-    "host/*.h",
-    "lib/*.h",
-    "lib/gtl/*.h",
-    "platform/**/*.h",
-])
+cc_library(
+    name = "kernel_spec",
+    srcs = ["kernel_spec.cc"],
+    hdrs = ["kernel_spec.h"],
+    deps = [
+        "//tensorflow/stream_executor/platform",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_library(
+    name = "kernel_cache_config",
+    hdrs = ["kernel_cache_config.h"],
+)
+
+cc_library(
+    name = "module_spec",
+    hdrs = ["module_spec.h"],
+    deps = [
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+    ],
+)
+
+cc_library(
+    name = "shared_memory_config",
+    hdrs = ["shared_memory_config.h"],
+)
+
+cc_library(
+    name = "stream_header",
+    hdrs = [
+        "blas.h",
+        "device_memory.h",
+        "dnn.h",
+        "event.h",
+        "fft.h",
+        "gpu_launch_dim.h",
+        "kernel.h",
+        "kernel_cache_config.h",
+        "launch_dim.h",
+        "stream.h",
+        "temporary_device_memory.h",
+        "temporary_memory_manager.h",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":dnn_proto_cc",
+        ":host_or_device_scalar",
+        ":stream_executor_headers",
+        "//tensorflow/core:lib",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/types:optional",
+        "@com_google_absl//absl/types:span",
+    ],
+)
+
+# It implements :stream_header
+cc_library(
+    name = "stream",
+    srcs = [
+        "host_buffer.h",
+        "stream.cc",
+    ],
+    hdrs = ["stream.h"],
+    deps = [
+        ":blas",
+        ":device_memory",
+        ":dnn",
+        ":event",
+        ":fft",
+        ":host_or_device_scalar",
+        ":kernel",
+        ":launch_dim",
+        ":platform",
+        ":rng",
+        ":stream_executor_headers",
+        ":stream_executor_internal",
+        ":stream_executor_pimpl",
+        ":temporary_memory_manager",
+        "//tensorflow/core:lib",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "//third_party/eigen3",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_library(
+    name = "timer",
+    srcs = [
+        "device_description.h",
+        "kernel_cache_config.h",
+        "timer.cc",
+    ],
+    hdrs = [
+        "blas.h",
+        "kernel.h",
+        "stream.h",
+        "stream_executor.h",
+        "timer.h",
+    ],
+    deps = [
+        ":host_or_device_scalar",
+        ":platform",
+        ":stream_executor_headers",
+        ":stream_executor_internal",
+        ":stream_executor_pimpl_header",
+        "//tensorflow/core:lib",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "@com_google_absl//absl/base:core_headers",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_library(
+    name = "platform",
+    srcs = ["platform.cc"],
+    hdrs = ["platform.h"],
+    deps = [
+        ":plugin",
+        ":stream_executor_headers",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_library(
+    name = "rng",
+    srcs = ["rng.cc"],
+    hdrs = ["rng.h"],
+    deps = ["//tensorflow/stream_executor/platform"],
+)
+
+cc_library(
+    name = "temporary_device_memory",
+    srcs = [
+        "event.h",
+        "temporary_device_memory.cc",
+        "temporary_memory_manager.h",
+    ],
+    hdrs = ["temporary_device_memory.h"],
+    deps = [
+        ":device_memory",
+        ":stream_header",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+    ],
+)
+
+cc_library(
+    name = "temporary_memory_manager",
+    srcs = ["temporary_memory_manager.cc"],
+    hdrs = ["temporary_memory_manager.h"],
+    deps = [
+        ":device_memory",
+        ":stream_executor_pimpl_header",
+        ":stream_header",
+        ":temporary_device_memory",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_library(
+    name = "fft",
+    hdrs = ["fft.h"],
+    deps = [
+        "//tensorflow/stream_executor/platform",
+    ],
+)
+
+cc_library(
+    name = "blas",
+    srcs = ["blas.cc"],
+    hdrs = ["blas.h"],
+    deps = [
+        ":host_or_device_scalar",
+        ":stream_executor_headers",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_library(
+    name = "device_memory",
+    hdrs = ["device_memory.h"],
+    deps = ["//tensorflow/stream_executor/platform"],
+)
+
+cc_library(
+    name = "host_or_device_scalar",
+    hdrs = ["host_or_device_scalar.h"],
+    deps = [
+        ":device_memory",
+        "//tensorflow/stream_executor/platform",
+    ],
+)
+
+cc_library(
+    name = "device_options",
+    hdrs = ["device_options.h"],
+    deps = [
+        "//tensorflow/stream_executor/platform",
+    ],
+)
+
+cc_library(
+    name = "executor_cache",
+    srcs = [
+        "device_description.h",
+        "device_memory.h",
+        "device_options.h",
+        "event.h",
+        "executor_cache.cc",
+        "launch_dim.h",
+        "plugin.h",
+        "plugin_registry.h",
+        "rng.h",
+        "stream_executor_pimpl.h",
+        "temporary_device_memory.h",
+        "temporary_memory_manager.h",
+    ],
+    hdrs = [
+        "blas.h",
+        "dnn.h",
+        "executor_cache.h",
+        "fft.h",
+        "kernel.h",
+        "kernel_cache_config.h",
+        "kernel_spec.h",
+        "platform.h",
+        "shared_memory_config.h",
+        "stream.h",
+        "stream_executor_internal.h",
+        "trace_listener.h",
+    ],
+    deps = [
+        ":dnn_proto_cc",
+        ":host_or_device_scalar",
+        ":stream_executor_headers",
+        "//tensorflow/core:lib",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "@com_google_absl//absl/base:core_headers",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/types:optional",
+        "@com_google_absl//absl/types:span",
+    ],
+)
+
+cc_library(
+    name = "multi_platform_manager",
+    srcs = ["multi_platform_manager.cc"],
+    hdrs = ["multi_platform_manager.h"],
+    deps = [
+        ":platform",
+        ":stream_executor_headers",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "@com_google_absl//absl/base:core_headers",
+        "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/synchronization",
+    ],
+)
+
+cc_library(
+    name = "plugin",
+    srcs = ["plugin.cc"],
+    hdrs = ["plugin.h"],
+)
+
+cc_library(
+    name = "plugin_registry",
+    srcs = ["plugin_registry.cc"],
+    hdrs = ["plugin_registry.h"],
+    deps = [
+        ":blas",
+        ":dnn",
+        ":fft",
+        ":multi_platform_manager",
+        ":platform",
+        ":plugin",
+        ":stream_executor_headers",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "@com_google_absl//absl/base:core_headers",
+    ],
+)
+
+cc_library(
+    name = "scratch_allocator",
+    srcs = ["scratch_allocator.cc"],
+    hdrs = ["scratch_allocator.h"],
+    deps = [
+        ":device_memory",
+        ":stream_header",
+        ":temporary_device_memory",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+    ],
+)
+
+cc_library(
+    name = "host_buffer",
+    hdrs = ["host_buffer.h"],
+    deps = [":dnn"],
+)
 
 tf_proto_library(
     name = "dnn_proto",
     srcs = ["dnn.proto"],
     cc_api_version = 2,
     default_header = True,
+    provide_cc_alias = True,
 )
 
 tf_proto_library(
     name = "logging_proto",
     srcs = ["logging.proto"],
     cc_api_version = 2,
-    default_header = True,
     protodeps = [":dnn_proto"],
+    provide_cc_alias = True,
+    visibility = [":friends"],
 )
 
 cc_library(
-    name = "stream_executor_impl",
-    srcs = glob(
-        [
-            "*.cc",
-            "host/*.cc",
-            "cuda/cuda_platform_id.cc",
-            "lib/*.cc",
-            "platform/default/*.cc",
-        ],
-        exclude = [
-            "**/*_test.cc",
-        ],
-    ),
-    hdrs = STREAM_EXECUTOR_HEADERS,
-    linkopts = select({
-        "//tensorflow:freebsd": [],
-        "//tensorflow:windows": [],
-        "//conditions:default": ["-ldl"],
-    }),
-    visibility = ["//visibility:public"],
+    name = "dnn",
+    srcs = ["dnn.cc"],
+    hdrs = ["dnn.h"],
     deps = [
-        ":dnn_proto_cc_impl",
-        ":logging_proto_cc_impl",
+        ":device_memory",
+        ":dnn_proto_cc",
+        ":stream_executor_headers",
         "//tensorflow/core:lib",
-        "//tensorflow/core:logger",
-        "//tensorflow/core:ptr_util",
-        "@com_google_absl//absl/container:flat_hash_map",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
         "@com_google_absl//absl/strings",
-        "@com_google_absl//absl/strings:str_format",
-        "@com_google_absl//absl/synchronization",
-        "@local_config_cuda//cuda:cuda_headers",
+        "@com_google_absl//absl/types:optional",
+        "@com_google_absl//absl/types:span",
     ],
-    alwayslink = 1,
 )
 
 cc_library(
-    name = "stream_executor",
-    hdrs = STREAM_EXECUTOR_HEADERS,
+    name = "stream_executor_internal",
+    srcs = [
+        "dnn.h",
+        "stream_executor_internal.cc",
+    ],
+    hdrs = [
+        "shared_memory_config.h",
+        "stream_executor_internal.h",
+    ],
+    deps = [
+        ":device_description",
+        ":device_memory",
+        ":device_options",
+        ":dnn_proto_cc",
+        ":kernel",
+        ":kernel_cache_config",
+        ":kernel_spec",
+        ":launch_dim",
+        ":plugin_registry",
+        ":stream_executor_headers",
+        "//tensorflow/core:lib",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "@com_google_absl//absl/types:optional",
+        "@com_google_absl//absl/types:span",
+    ],
+)
+
+cc_library(
+    name = "stream_executor_pimpl_header",
+    hdrs = [
+        "device_description.h",
+        "dnn.h",
+        "kernel.h",
+        "kernel_cache_config.h",
+        "shared_memory_config.h",
+        "stream_executor_pimpl.h",
+    ],
     visibility = ["//visibility:public"],
     deps = [
         ":dnn_proto_cc",
-        ":logging_proto_cc",
+        ":platform",
+        ":stream_executor_headers",
+        ":stream_executor_internal",
         "//tensorflow/core:lib",
-        "//tensorflow/core:ptr_util",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "@com_google_absl//absl/base:core_headers",
         "@com_google_absl//absl/strings",
-        "@local_config_cuda//cuda:cuda_headers",
-    ] + if_static([":stream_executor_impl"]),
+        "@com_google_absl//absl/types:optional",
+        "@com_google_absl//absl/types:span",
+    ],
 )
 
-cc_header_only_library(
-    name = "stream_executor_headers_lib",
-    visibility = ["//visibility:public"],
+# It implements :stream_executor_pimpl_header
+cc_library(
+    name = "stream_executor_pimpl",
+    srcs = ["stream_executor_pimpl.cc"],
+    hdrs = ["stream_executor_pimpl.h"],
     deps = [
-        ":stream_executor",
+        ":blas",
+        ":executor_cache",
+        ":fft",
+        ":kernel",
+        ":platform",
+        ":rng",
+        ":stream_executor_headers",
+        ":stream_header",
+        ":timer",
+        "//tensorflow/core:lib_internal",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "@com_google_absl//absl/base:core_headers",
+        "@com_google_absl//absl/strings",
     ],
 )
 
+# The stream_executor_headers target does not prescribe an implementation.
+#
+# TODO(b/25131218) this is OBSOLETE/DEPRECATED -- get rid of this target altogether
 cc_library(
-    name = "cuda_platform",
-    srcs = if_cuda_is_configured(
-        glob(
-            [
-                "cuda/*.cc",
-            ],
-            exclude = [
-                "cuda/*_test.cc",
-                "cuda/cuda_platform_id.cc",
-            ],
-        ),
-    ),
-    copts = select({
-        "//tensorflow:windows": ["/DNOGDI"],
-        "//conditions:default": [],
-    }),
-    data = [
-        "@local_config_cuda//cuda:cudnn",
-    ],
-    linkopts = select({
-        "//tensorflow:freebsd": [],
-        "//tensorflow:windows": [],
-        "//conditions:default": ["-ldl"],
-    }),
+    name = "stream_executor_headers",
+    hdrs = [
+        "blas.h",
+        "device_description.h",
+        "device_memory.h",
+        "device_options.h",
+        "dnn.h",
+        "event.h",
+        "executor_cache.h",
+        "fft.h",
+        "gpu_launch_dim.h",
+        "kernel.h",
+        "kernel_cache_config.h",
+        "kernel_spec.h",
+        "launch_dim.h",
+        "module_spec.h",
+        "multi_platform_manager.h",
+        "platform.h",
+        "plugin.h",
+        "plugin_registry.h",
+        "rng.h",
+        "shared_memory_config.h",
+        "stream.h",
+        "stream_executor.h",
+        "stream_executor_internal.h",
+        "stream_executor_pimpl.h",
+        "temporary_device_memory.h",
+        "temporary_memory_manager.h",
+        "timer.h",
+        "trace_listener.h",
+    ],
     visibility = ["//visibility:public"],
     deps = [
-        ":stream_executor",
+        ":dnn_proto_cc",
+        ":host_or_device_scalar",
         "//tensorflow/core:lib",
-        "//tensorflow/core/kernels:ops_util",
-        "@local_config_cuda//cuda:cuda_headers",
-    ] + if_cuda_is_configured([
-        "//tensorflow/core:cuda",
-        "@local_config_cuda//cuda:cuda_driver",
-    ]),
-    alwayslink = 1,
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "@com_google_absl//absl/base:core_headers",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/types:optional",
+        "@com_google_absl//absl/types:span",
+    ],
+)
+
+cc_library(
+    name = "stream_executor",
+    hdrs = ["stream_executor.h"],
+    deps = [":stream_executor_headers"] + if_static([":stream_executor_impl"]),
+)
+
+cc_library(
+    name = "stream_executor_impl",
+    deps = [
+        ":device_description",
+        ":device_memory",
+        ":dnn_proto_cc",
+        ":dnn_proto_cc_impl",
+        ":event",
+        ":kernel",
+        ":launch_dim",
+        ":multi_platform_manager",
+        ":platform",
+        ":stream",
+        ":stream_executor_headers",
+        ":stream_executor_pimpl",
+        ":timer",
+    ],
+)
+
+tf_cc_test(
+    name = "stream_test",
+    size = "small",
+    srcs = ["stream_test.cc"],
+    deps = [
+        ":stream_executor",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/stream_executor/host:host_platform",
+    ],
+)
+
+alias(
+    name = "cuda_platform",
+    actual = "//tensorflow/stream_executor/cuda:all_runtime",
 )
diff --git a/tensorflow/stream_executor/build_defs.bzl b/tensorflow/stream_executor/build_defs.bzl
new file mode 100644
index 0000000000..a7ddf5a0d7
--- /dev/null
+++ b/tensorflow/stream_executor/build_defs.bzl
@@ -0,0 +1,11 @@
+def stream_executor_friends():
+    return ["//tensorflow/..."]
+
+def tf_additional_cuda_platform_deps():
+  return []
+
+def tf_additional_cuda_driver_deps():
+  return ["@local_config_cuda//cuda:cuda_driver"]
+
+def tf_additional_cudnn_plugin_deps():
+  return []
diff --git a/tensorflow/stream_executor/cuda/BUILD b/tensorflow/stream_executor/cuda/BUILD
new file mode 100644
index 0000000000..58c9480884
--- /dev/null
+++ b/tensorflow/stream_executor/cuda/BUILD
@@ -0,0 +1,365 @@
+# Description:
+#   CUDA-platform specific StreamExecutor support code.
+
+licenses(["notice"])  # Apache 2.0
+
+load("//tensorflow:tensorflow.bzl", "tf_cc_test")
+load(
+    "//tensorflow/stream_executor:build_defs.bzl",
+    "stream_executor_friends",
+    "tf_additional_cuda_driver_deps",
+    "tf_additional_cuda_platform_deps",
+    "tf_additional_cudnn_plugin_deps",
+)
+load("//tensorflow:tensorflow.bzl", "tf_copts")
+load("@local_config_cuda//cuda:build_defs.bzl", "if_cuda_is_configured")
+load("//tensorflow/core:platform/default/build_config_root.bzl", "if_static")
+
+package_group(
+    name = "friends",
+    packages = stream_executor_friends(),
+)
+
+package(
+    default_visibility = [":friends"],
+)
+
+# Filegroup used to collect source files for the dependency check.
+filegroup(
+    name = "c_srcs",
+    data = glob([
+        "**/*.cc",
+        "**/*.h",
+    ]),
+)
+
+cc_library(
+    name = "cuda_platform_id",
+    srcs = ["cuda_platform_id.cc"],
+    hdrs = ["cuda_platform_id.h"],
+    deps = ["//tensorflow/stream_executor:platform"],
+)
+
+cc_library(
+    name = "cuda_platform",
+    srcs = ["cuda_platform.cc"],
+    hdrs = ["cuda_platform.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":cuda_driver",
+        ":cuda_gpu_executor",
+        ":cuda_platform_id",
+        "//tensorflow/stream_executor",  # buildcleaner: keep
+        "//tensorflow/stream_executor:executor_cache",
+        "//tensorflow/stream_executor:multi_platform_manager",
+        "//tensorflow/stream_executor:stream_executor_pimpl_header",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+    ] + tf_additional_cuda_platform_deps(),
+    alwayslink = True,  # Registers itself with the MultiPlatformManager.
+)
+
+cc_library(
+    name = "cuda_diagnostics",
+    srcs = ["cuda_diagnostics.cc"],
+    hdrs = ["cuda_diagnostics.h"],
+    deps = [
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "@com_google_absl//absl/container:inlined_vector",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_library(
+    name = "cuda_driver",
+    srcs = ["cuda_driver.cc"],
+    hdrs = ["cuda_driver.h"],
+    deps = [
+        ":cuda_diagnostics",
+        "@com_google_absl//absl/base",
+        "@com_google_absl//absl/container:inlined_vector",
+        "@com_google_absl//absl/strings",
+        "@local_config_cuda//cuda:cuda_headers",
+        "//tensorflow/stream_executor:device_options",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+    ] + tf_additional_cuda_driver_deps(),
+)
+
+# The activation library is tightly coupled to the executor library.
+# TODO(leary) split up cuda_gpu_executor.cc so that this can stand alone.
+cc_library(
+    name = "cuda_activation_header",
+    hdrs = ["cuda_activation.h"],
+    visibility = ["//visibility:public"],
+    deps = ["//tensorflow/stream_executor/platform"],
+)
+
+cc_library(
+    name = "cuda_activation",
+    srcs = ["cuda_activation.cc"],
+    hdrs = ["cuda_activation.h"],
+    deps = [
+        ":cuda_driver",
+        "//tensorflow/stream_executor",
+        "//tensorflow/stream_executor:stream_executor_internal",
+        "//tensorflow/stream_executor/platform",
+        "@local_config_cuda//cuda:cuda_headers",
+    ],
+)
+
+cc_library(
+    name = "cuda_gpu_executor_header",
+    textual_hdrs = ["cuda_gpu_executor.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":cuda_kernel",
+        "//tensorflow/stream_executor:event",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+    ],
+)
+
+cc_library(
+    name = "cublas_plugin",
+    srcs = ["cuda_blas.cc"],
+    hdrs = ["cuda_blas.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":cuda_activation",
+        ":cuda_gpu_executor",
+        ":cuda_helpers",
+        ":cuda_platform_id",
+        ":cuda_stream",
+        ":cuda_timer",
+        "@com_google_absl//absl/strings",
+        "//third_party/eigen3",
+        "@local_config_cuda//cuda:cuda_headers",
+        "//tensorflow/core:lib_internal",
+        "//tensorflow/stream_executor",
+        "//tensorflow/stream_executor:event",
+        "//tensorflow/stream_executor:host_or_device_scalar",
+        "//tensorflow/stream_executor:plugin_registry",
+        "//tensorflow/stream_executor:scratch_allocator",
+        "//tensorflow/stream_executor:timer",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "//tensorflow/stream_executor/platform:dso_loader",
+    ] + if_static(
+        [
+            "@local_config_cuda//cuda:cublas",
+            "@local_config_cuda//cuda:cudart_static",
+        ],
+        ["@local_config_cuda//cuda:cudart"],
+    ),
+    alwayslink = True,
+)
+
+cc_library(
+    name = "cufft_plugin",
+    srcs = ["cuda_fft.cc"],
+    hdrs = ["cuda_fft.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":cuda_activation_header",
+        ":cuda_gpu_executor_header",
+        ":cuda_helpers",
+        ":cuda_platform_id",
+        ":cuda_stream",
+        "@local_config_cuda//cuda:cuda_headers",
+        "//tensorflow/stream_executor:event",
+        "//tensorflow/stream_executor:fft",
+        "//tensorflow/stream_executor:plugin_registry",
+        "//tensorflow/stream_executor:scratch_allocator",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "//tensorflow/stream_executor/platform:dso_loader",
+    ] + if_static(["@local_config_cuda//cuda:cufft"]),
+    alwayslink = True,
+)
+
+cc_library(
+    name = "cudnn_plugin",
+    srcs = ["cuda_dnn.cc"],
+    hdrs = ["cuda_dnn.h"],
+    copts = [
+        # STREAM_EXECUTOR_CUDNN_WRAP would fail on Clang with the default
+        # setting of template depth 256
+        "-ftemplate-depth-512",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":cuda_activation",
+        ":cuda_diagnostics",
+        ":cuda_driver",
+        ":cuda_gpu_executor",
+        ":cuda_platform_id",
+        ":cuda_stream",
+        ":cuda_timer",
+        ":cudnn_version",
+        "@com_google_absl//absl/strings",
+        "//third_party/eigen3",
+        "@local_config_cuda//cuda:cuda_headers",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+        "//tensorflow/core:logger",
+        "//tensorflow/stream_executor:dnn",
+        "//tensorflow/stream_executor:event",
+        "//tensorflow/stream_executor:logging_proto_cc",
+        "//tensorflow/stream_executor:plugin_registry",
+        "//tensorflow/stream_executor:scratch_allocator",
+        "//tensorflow/stream_executor:stream_executor_pimpl_header",
+        "//tensorflow/stream_executor:temporary_device_memory",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "//tensorflow/stream_executor/platform:dso_loader",
+    ] + tf_additional_cudnn_plugin_deps() + if_static(
+        [
+            "@local_config_cuda//cuda:cudnn",
+            "@local_config_cuda//cuda:cudart_static",
+        ],
+        ["@local_config_cuda//cuda:cudart"],
+    ),
+    alwayslink = True,
+)
+
+cc_library(
+    name = "curand_plugin",
+    srcs = ["cuda_rng.cc"],
+    hdrs = ["cuda_rng.h"],
+    deps = [
+        ":cuda_activation",
+        ":cuda_gpu_executor",
+        ":cuda_helpers",
+        ":cuda_platform_id",
+        ":cuda_stream",
+        "@local_config_cuda//cuda:cuda_headers",
+        "//tensorflow/stream_executor:event",
+        "//tensorflow/stream_executor:plugin_registry",
+        "//tensorflow/stream_executor:rng",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "//tensorflow/stream_executor/platform:dso_loader",
+    ] + if_static(["@local_config_cuda//cuda:curand"]),
+    alwayslink = True,
+)
+
+cc_library(
+    name = "cuda_kernel",
+    hdrs = ["cuda_kernel.h"],
+    deps = [
+        ":cuda_driver",
+        "//tensorflow/stream_executor:event",
+        "//tensorflow/stream_executor:stream_executor_pimpl_header",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "@local_config_cuda//cuda:cuda_headers",
+    ],
+)
+
+# TODO(leary) we likely need to canonicalize/eliminate this.
+cc_library(
+    name = "cuda_helpers",
+    textual_hdrs = ["cuda_helpers.h"],
+)
+
+cc_library(
+    name = "cuda_event",
+    srcs = ["cuda_event.cc"],
+    hdrs = ["cuda_event.h"],
+    deps = [
+        ":cuda_driver",
+        ":cuda_gpu_executor_header",
+        ":cuda_stream",
+        "//tensorflow/stream_executor:stream_executor_headers",
+        "//tensorflow/stream_executor/lib",
+    ],
+)
+
+cc_library(
+    name = "cuda_stream",
+    srcs = ["cuda_stream.cc"],
+    hdrs = ["cuda_stream.h"],
+    deps = [
+        ":cuda_driver",
+        ":cuda_gpu_executor_header",
+        "//tensorflow/stream_executor:stream_executor_headers",
+        "//tensorflow/stream_executor:stream_header",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+    ],
+)
+
+cc_library(
+    name = "cuda_timer",
+    srcs = ["cuda_timer.cc"],
+    hdrs = ["cuda_timer.h"],
+    deps = [
+        ":cuda_driver",
+        ":cuda_gpu_executor_header",
+        ":cuda_stream",
+        "//tensorflow/stream_executor:stream_executor_headers",
+        "//tensorflow/stream_executor/lib",
+    ],
+)
+
+# It implements :cuda_gpu_executor_header
+cc_library(
+    name = "cuda_gpu_executor",
+    srcs = ["cuda_gpu_executor.cc"],
+    hdrs = ["cuda_gpu_executor.h"],
+    deps = [
+        ":cuda_activation",
+        ":cuda_diagnostics",
+        ":cuda_driver",
+        ":cuda_event",
+        ":cuda_kernel",
+        ":cuda_platform_id",
+        ":cuda_stream",
+        ":cuda_timer",
+        "//tensorflow/stream_executor:event",
+        "//tensorflow/stream_executor:plugin_registry",
+        "//tensorflow/stream_executor:stream_executor_internal",
+        "//tensorflow/stream_executor:stream_executor_pimpl_header",
+        "//tensorflow/stream_executor:timer",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "@com_google_absl//absl/strings",
+    ],
+    alwayslink = True,
+)
+
+cc_library(
+    name = "cudnn_version",
+    srcs = ["cudnn_version.cc"],
+    hdrs = ["cudnn_version.h"],
+    deps = [
+        "//tensorflow/core:lib",
+    ],
+)
+
+tf_cc_test(
+    name = "cudnn_version_test",
+    srcs = ["cudnn_version_test.cc"],
+    deps = [
+        ":cudnn_version",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ],
+)
+
+cc_library(
+    name = "all_runtime",
+    copts = tf_copts(),
+    visibility = ["//visibility:public"],
+    deps = if_cuda_is_configured([
+        ":cudnn_plugin",
+        ":cufft_plugin",
+        ":cublas_plugin",
+        ":curand_plugin",
+        ":cuda_driver",
+        ":cuda_platform",
+    ]),
+    alwayslink = 1,
+)
diff --git a/tensorflow/stream_executor/host/BUILD b/tensorflow/stream_executor/host/BUILD
new file mode 100644
index 0000000000..59472b14c1
--- /dev/null
+++ b/tensorflow/stream_executor/host/BUILD
@@ -0,0 +1,108 @@
+# Description:
+#   Host-platform specific StreamExecutor support code.
+
+licenses(["notice"])  # Apache 2.0
+
+load("//tensorflow/stream_executor:build_defs.bzl", "stream_executor_friends")
+
+package_group(
+    name = "friends",
+    packages = stream_executor_friends(),
+)
+
+package(default_visibility = [":friends"])
+
+# Filegroup used to collect source files for the dependency check.
+filegroup(
+    name = "c_srcs",
+    data = glob([
+        "**/*.cc",
+        "**/*.h",
+    ]),
+)
+
+cc_library(
+    name = "host_platform_id",
+    srcs = [
+        "host_platform_id.cc",
+    ],
+    hdrs = [
+        "host_platform_id.h",
+    ],
+    deps = [
+        "//tensorflow/stream_executor:platform",
+    ],
+)
+
+cc_library(
+    name = "host_platform",
+    srcs = [
+        "host_platform.cc",
+    ],
+    hdrs = [
+        "host_platform.h",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":host_gpu_executor",
+        ":host_platform_id",
+        "//tensorflow/stream_executor:executor_cache",
+        "//tensorflow/stream_executor:multi_platform_manager",
+        "//tensorflow/stream_executor:stream_executor_headers",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+    ],
+    alwayslink = True,  # Registers itself with the MultiPlatformManager.
+)
+
+cc_library(
+    name = "host_stream",
+    srcs = [
+        "host_stream.cc",
+    ],
+    hdrs = [
+        "host_stream.h",
+    ],
+    deps = [
+        "//tensorflow/stream_executor:kernel",
+        "//tensorflow/stream_executor/lib",
+    ],
+)
+
+cc_library(
+    name = "host_timer",
+    srcs = [
+        "host_timer.cc",
+    ],
+    hdrs = [
+        "host_timer.h",
+    ],
+    deps = [
+        "//tensorflow/stream_executor:stream_executor_internal",
+        "//tensorflow/stream_executor:timer",
+        "//tensorflow/stream_executor/platform",
+    ],
+)
+
+# TODO(22689637): Rename this target.
+cc_library(
+    name = "host_gpu_executor",
+    srcs = [
+        "host_gpu_executor.cc",
+    ],
+    hdrs = [
+        "host_gpu_executor.h",
+    ],
+    deps = [
+        ":host_platform_id",
+        ":host_stream",
+        ":host_timer",
+        "//tensorflow/core:lib",
+        "//tensorflow/stream_executor:kernel",
+        "//tensorflow/stream_executor:rng",
+        "//tensorflow/stream_executor:stream_executor_internal",
+        "//tensorflow/stream_executor:timer",
+        "//tensorflow/stream_executor/lib",
+    ],
+    alwayslink = True,
+)
diff --git a/tensorflow/stream_executor/lib/BUILD b/tensorflow/stream_executor/lib/BUILD
new file mode 100644
index 0000000000..133ff2b161
--- /dev/null
+++ b/tensorflow/stream_executor/lib/BUILD
@@ -0,0 +1,62 @@
+licenses(["notice"])  # Apache 2.0
+
+load("//tensorflow:tensorflow.bzl", "tf_cc_test")
+load("//tensorflow/stream_executor:build_defs.bzl", "stream_executor_friends")
+
+package_group(
+    name = "friends",
+    packages = stream_executor_friends(),
+)
+
+package(default_visibility = [":friends"])
+
+filegroup(
+    name = "c_srcs",
+    data = glob([
+        "**/*.cc",
+        "**/*.h",
+    ]),
+)
+
+cc_library(
+    name = "lib",
+    srcs = glob(
+        [
+            "**/*.cc",
+        ],
+        exclude = [
+            "**/*test*",
+        ],
+    ),
+    hdrs = glob(["**/*.h"]),
+    deps = [
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:ptr_util",
+        "//tensorflow/stream_executor/platform",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/types:span",
+    ],
+)
+
+tf_cc_test(
+    name = "statusor_test",
+    size = "small",
+    srcs = ["statusor_test.cc"],
+    deps = [
+        ":lib",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ],
+)
+
+cc_library(
+    name = "utility_headers",
+    hdrs = [
+        "ptr_util.h",
+    ],
+    deps = [
+        "//tensorflow/core:ptr_util",
+    ],
+)
diff --git a/tensorflow/stream_executor/platform/BUILD b/tensorflow/stream_executor/platform/BUILD
new file mode 100644
index 0000000000..702b2cdfe0
--- /dev/null
+++ b/tensorflow/stream_executor/platform/BUILD
@@ -0,0 +1,47 @@
+licenses(["notice"])  # Apache 2.0
+
+load("//tensorflow/stream_executor:build_defs.bzl", "stream_executor_friends")
+load("//tensorflow/core:platform/default/build_config.bzl", "tf_platform_hdrs")
+
+package_group(
+    name = "friends",
+    packages = stream_executor_friends(),
+)
+
+package(
+    default_visibility = [":friends"],
+)
+
+cc_library(
+    name = "platform",
+    textual_hdrs = [
+        "logging.h",
+        "mutex.h",
+        "platform.h",
+        "port.h",
+        "thread_annotations.h",
+        "initialize.h",
+    ],
+    deps = [
+        "//tensorflow/core:lib",
+        "//tensorflow/stream_executor/platform/default:platform",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_library(
+    name = "dso_loader",
+    hdrs = ["dso_loader.h"],
+    deps = [
+        ":platform",
+        "//tensorflow/stream_executor/platform/default:dso_loader",
+    ],
+)
+
+filegroup(
+    name = "c_srcs",
+    data = glob([
+        "**/*.cc",
+        "**/*.h",
+    ]),
+)
diff --git a/tensorflow/stream_executor/platform/default/BUILD b/tensorflow/stream_executor/platform/default/BUILD
new file mode 100644
index 0000000000..f1ae7d86ff
--- /dev/null
+++ b/tensorflow/stream_executor/platform/default/BUILD
@@ -0,0 +1,25 @@
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//tensorflow/stream_executor:__subpackages__"])
+
+cc_library(
+    name = "platform",
+    textual_hdrs = [
+        "initialize.h",
+        "mutex.h",
+    ],
+    deps = ["//tensorflow/core:lib"],
+)
+
+cc_library(
+    name = "dso_loader",
+    srcs = ["dso_loader.cc"],
+    hdrs = ["dso_loader.h"],
+    deps = [
+        "//tensorflow/stream_executor:platform",
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+        "@com_google_absl//absl/strings",
+        "@local_config_cuda//cuda:cuda_headers",
+    ],
+)
-- 
GitLab


From 60bcd1c1fd2cf412b217d24e7672bc3699458a43 Mon Sep 17 00:00:00 2001
From: Duncan Riach <duncan@nvidia.com>
Date: Tue, 11 Dec 2018 16:56:09 -0800
Subject: [PATCH 0402/1095] Add cuDNN deterministic env variable (only for
 convolution)

---
 tensorflow/stream_executor/cuda/cuda_dnn.cc | 40 +++++++++++++++++++++
 1 file changed, 40 insertions(+)

diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.cc b/tensorflow/stream_executor/cuda/cuda_dnn.cc
index a34aa9354d..e91caf3bdb 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.cc
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.cc
@@ -794,6 +794,19 @@ bool BatchnormSpatialPersistentEnabled() {
   return is_enabled;
 }
 
+// A helper function to decide whether to enable deterministic functionality.
+// TODO(pr/24355): Support all cuDNN functionality (currently only convolution).
+bool Deterministic() {
+  static bool is_enabled = [] {
+    bool is_enabled = false;
+    TF_CHECK_OK(
+        tensorflow::ReadBoolFromEnvVar("TF_CUDNN_DETERMINISTIC",
+                                       /*default_val=*/false, &is_enabled));
+    return is_enabled;
+  }();
+  return is_enabled;
+}
+
 // Turns a ConvolutionDescriptor structure into a cudnn convolution handle
 // within a scope.
 class CudnnConvolutionDescriptor {
@@ -3056,10 +3069,19 @@ bool CudnnSupport::GetConvolveAlgorithms(
     algo_types.push_back(CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED);
   }
 
+  if (Deterministic()) {
+    algo_types.clear();
+    algo_types.push_back(CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM);
+  }
+
   out_algorithms->clear();
   for (auto i : algo_types) {
     out_algorithms->push_back({i, /*use_tensor_ops=*/false});
     if (cc_major >= 7 && CUDNN_VERSION >= 7000 && TensorOpMathEnabled()) {
+      if (Deterministic()) {
+        // For determinism: always use tensor op math, if it's available
+        out_algorithms->pop_back();
+      }
       out_algorithms->push_back({i, /*use_tensor_ops=*/true});
     }
   }
@@ -3104,10 +3126,19 @@ bool CudnnSupport::GetConvolveBackwardDataAlgorithms(
     algo_types.push_back(CUDNN_CONVOLUTION_BWD_DATA_ALGO_WINOGRAD_NONFUSED);
   }
 
+  if (Deterministic()) {
+    algo_types.clear();
+    algo_types.push_back(CUDNN_CONVOLUTION_BWD_DATA_ALGO_1);
+  }
+
   out_algorithms->clear();
   for (auto i : algo_types) {
     out_algorithms->push_back({i, /*use_tensor_ops=*/false});
     if (cc_major >= 7 && CUDNN_VERSION >= 7000 && TensorOpMathEnabled()) {
+      if (Deterministic()) {
+        // For determinism: always use tensor op math, if it's available
+        out_algorithms->pop_back();
+      }
       out_algorithms->push_back({i, /*use_tensor_ops=*/true});
     }
   }
@@ -3135,10 +3166,19 @@ bool CudnnSupport::GetConvolveBackwardFilterAlgorithms(
     algo_types.push_back(CUDNN_CONVOLUTION_BWD_FILTER_ALGO_WINOGRAD_NONFUSED);
   }
 
+  if (Deterministic()) {
+    algo_types.clear();
+    algo_types.push_back(CUDNN_CONVOLUTION_BWD_FILTER_ALGO_1);
+  }
+
   out_algorithms->clear();
   for (auto i : algo_types) {
     out_algorithms->push_back({i, /*use_tensor_ops=*/false});
     if (cc_major >= 7 && CUDNN_VERSION >= 7000 && TensorOpMathEnabled()) {
+      if (Deterministic()) {
+        // For determinism: always use tensor op math, if it's available
+        out_algorithms->pop_back();
+      }
       out_algorithms->push_back({i, /*use_tensor_ops=*/true});
     }
   }
-- 
GitLab


From df17a7db51000ccf7c6b062e37b5c3acb2cd286b Mon Sep 17 00:00:00 2001
From: Pete Warden <petewarden@google.com>
Date: Tue, 8 Jan 2019 13:42:38 -0800
Subject: [PATCH 0403/1095] MacOS support for simple speech example

PiperOrigin-RevId: 228391393
---
 .../micro/examples/micro_speech/BUILD         |  24 ---
 .../micro/examples/micro_speech/Makefile.inc  |  27 +---
 .../examples/micro_speech/audio_provider.cc   |   6 +
 .../examples/micro_speech/audio_provider.h    |  10 ++
 .../micro_speech/audio_provider_test.cc       |  26 ++++
 .../micro/examples/micro_speech/main.cc       |   4 +-
 .../examples/micro_speech/model_settings.h    |   1 +
 .../examples/micro_speech/osx/Makefile.inc    |   6 +
 .../micro_speech/osx/audio_provider.cc        | 139 ++++++++++++++++++
 .../micro/examples/micro_speech/timer.cc      |  22 ---
 .../micro/examples/micro_speech/timer.h       |  31 ----
 .../micro/examples/micro_speech/timer_test.cc |  49 ------
 .../experimental/micro/tools/make/Makefile    |  35 ++++-
 .../micro/tools/make/targets/osx_makefile.inc |  10 ++
 14 files changed, 241 insertions(+), 149 deletions(-)
 create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/osx/Makefile.inc
 create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/osx/audio_provider.cc
 delete mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/timer.cc
 delete mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/timer.h
 delete mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/timer_test.cc
 create mode 100644 tensorflow/lite/experimental/micro/tools/make/targets/osx_makefile.inc

diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/BUILD b/tensorflow/lite/experimental/micro/examples/micro_speech/BUILD
index 70eeac1458..702b893e6f 100644
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/BUILD
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/BUILD
@@ -196,29 +196,6 @@ tflite_micro_cc_test(
     ],
 )
 
-cc_library(
-    name = "timer",
-    srcs = [
-        "timer.cc",
-    ],
-    hdrs = [
-        "timer.h",
-    ],
-)
-
-tflite_micro_cc_test(
-    name = "timer_test",
-    srcs = [
-        "timer_test.cc",
-    ],
-    deps = [
-        ":timer",
-        "//tensorflow/lite/c:c_api_internal",
-        "//tensorflow/lite/experimental/micro:micro_framework",
-        "//tensorflow/lite/experimental/micro/testing:micro_test",
-    ],
-)
-
 cc_library(
     name = "recognize_commands",
     srcs = [
@@ -259,7 +236,6 @@ cc_binary(
         ":model_settings",
         ":preprocessor_reference",
         ":recognize_commands",
-        ":timer",
         ":tiny_conv_model_data",
         "//tensorflow/lite:schema_fbs_version",
         "//tensorflow/lite/experimental/micro:micro_framework",
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/Makefile.inc b/tensorflow/lite/experimental/micro/examples/micro_speech/Makefile.inc
index 5623eabdde..6f7563c02f 100644
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/Makefile.inc
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/Makefile.inc
@@ -5,6 +5,7 @@ tensorflow/lite/experimental/micro/examples/micro_speech/micro_speech_test.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/tiny_conv_model_data.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/no_features_data.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/yes_features_data.cc
+MICRO_SPEECH_TEST_SRCS := $(call specialize,$(MICRO_SPEECH_TEST_SRCS))
 ALL_SRCS += $(MICRO_SPEECH_TEST_SRCS)
 MICRO_SPEECH_TEST_OBJS := $(addprefix $(OBJDIR), \
 $(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(MICRO_SPEECH_TEST_SRCS))))
@@ -32,6 +33,7 @@ tensorflow/lite/experimental/micro/examples/micro_speech/yes_power_spectrum_data
 PREPROCESSOR_REFERENCE_TEST_SRCS = \
 $(PREPROCESSOR_TEST_SHARED_SRCS) \
 tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.cc
+PREPROCESSOR_REFERENCE_TEST_SRCS := $(call specialize,$(PREPROCESSOR_REFERENCE_TEST_SRCS))
 ALL_SRCS += $(PREPROCESSOR_REFERENCE_TEST_SRCS)
 PREPROCESSOR_REFERENCE_TEST_OBJS := $(addprefix $(OBJDIR), \
 $(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(PREPROCESSOR_REFERENCE_TEST_SRCS))))
@@ -51,6 +53,7 @@ test_preprocessor_reference: $(PREPROCESSOR_REFERENCE_TEST_BINARY)
 PREPROCESSOR_FIXED_TEST_SRCS = \
 $(PREPROCESSOR_TEST_SHARED_SRCS) \
 tensorflow/lite/experimental/micro/examples/micro_speech/fixed_point/preprocessor.cc
+PREPROCESSOR_FIXED_TEST_SRCS := $(call specialize,$(PREPROCESSOR_FIXED_TEST_SRCS))
 ALL_SRCS += $(PREPROCESSOR_FIXED_TEST_SRCS)
 PREPROCESSOR_FIXED_TEST_OBJS := $(addprefix $(OBJDIR), \
 $(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(PREPROCESSOR_FIXED_TEST_SRCS))))
@@ -71,6 +74,7 @@ AUDIO_PROVIDER_TEST_SRCS := \
 tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider_test.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.cc
+AUDIO_PROVIDER_TEST_SRCS := $(call specialize,$(AUDIO_PROVIDER_TEST_SRCS))
 ALL_SRCS += $(AUDIO_PROVIDER_TEST_SRCS)
 AUDIO_PROVIDER_TEST_OBJS := $(addprefix $(OBJDIR), \
 $(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(AUDIO_PROVIDER_TEST_SRCS))))
@@ -93,6 +97,7 @@ tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.cc
+FEATURE_PROVIDER_TEST_SRCS := $(call specialize,$(FEATURE_PROVIDER_TEST_SRCS))
 ALL_SRCS += $(FEATURE_PROVIDER_TEST_SRCS)
 FEATURE_PROVIDER_TEST_OBJS := $(addprefix $(OBJDIR), \
 $(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(FEATURE_PROVIDER_TEST_SRCS))))
@@ -108,30 +113,12 @@ feature_provider_test_bin: $(FEATURE_PROVIDER_TEST_BINARY).bin
 test_feature_provider: $(FEATURE_PROVIDER_TEST_BINARY)
 	$(TEST_SCRIPT) $(FEATURE_PROVIDER_TEST_BINARY) '~~~ALL TESTS PASSED~~~'
 
-# Tests the timer module.
-TIMER_TEST_SRCS := \
-tensorflow/lite/experimental/micro/examples/micro_speech/timer_test.cc \
-tensorflow/lite/experimental/micro/examples/micro_speech/timer.cc
-ALL_SRCS += $(TIMER_TEST_SRCS)
-TIMER_TEST_OBJS := $(addprefix $(OBJDIR), \
-$(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(TIMER_TEST_SRCS))))
-TIMER_TEST_BINARY := $(BINDIR)timer_test
-ALL_BINARIES += $(TIMER_TEST_BINARY)
-$(TIMER_TEST_BINARY): $(TIMER_TEST_OBJS) $(MICROLITE_LIB_PATH)
-	@mkdir -p $(dir $@)
-	$(CXX) $(CXXFLAGS) $(INCLUDES) \
-	-o $(TIMER_TEST_BINARY) $(TIMER_TEST_OBJS) \
-	$(LIBFLAGS) $(MICROLITE_LIB_PATH) $(LDFLAGS) $(MICROLITE_LIBS)
-timer_test: $(TIMER_TEST_BINARY)
-timer_test_bin: $(TIMER_TEST_BINARY).bin
-test_timer: $(TIMER_TEST_BINARY)
-	$(TEST_SCRIPT) $(TIMER_TEST_BINARY) '~~~ALL TESTS PASSED~~~'
-
 # Tests the feature provider module.
 RECOGNIZE_COMMANDS_TEST_SRCS := \
 tensorflow/lite/experimental/micro/examples/micro_speech/recognize_commands_test.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/recognize_commands.cc
+RECOGNIZE_COMMANDS_TEST_SRCS := $(call specialize,$(RECOGNIZE_COMMANDS_TEST_SRCS))
 ALL_SRCS += $(RECOGNIZE_COMMANDS_TEST_SRCS)
 RECOGNIZE_COMMANDS_TEST_OBJS := $(addprefix $(OBJDIR), \
 $(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(RECOGNIZE_COMMANDS_TEST_SRCS))))
@@ -153,12 +140,12 @@ tensorflow/lite/experimental/micro/examples/micro_speech/main.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.cc \
-tensorflow/lite/experimental/micro/examples/micro_speech/timer.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/no_features_data.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/yes_features_data.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/tiny_conv_model_data.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/recognize_commands.cc
+MICRO_SPEECH_SRCS := $(call specialize,$(MICRO_SPEECH_SRCS))
 ALL_SRCS += $(MICRO_SPEECH_SRCS)
 MICRO_SPEECH_OBJS := $(addprefix $(OBJDIR), \
 $(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(MICRO_SPEECH_SRCS))))
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.cc
index c0365d5690..52db18e686 100644
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.cc
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.cc
@@ -19,6 +19,7 @@ limitations under the License.
 
 namespace {
 int16_t g_dummy_audio_data[kMaxAudioSampleSize];
+int32_t g_latest_audio_timestamp = 0;
 }  // namespace
 
 TfLiteStatus GetAudioSamples(tflite::ErrorReporter* error_reporter,
@@ -31,3 +32,8 @@ TfLiteStatus GetAudioSamples(tflite::ErrorReporter* error_reporter,
   *audio_samples = g_dummy_audio_data;
   return kTfLiteOk;
 }
+
+int32_t LatestAudioTimestamp() {
+  g_latest_audio_timestamp += 100;
+  return g_latest_audio_timestamp;
+}
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h b/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h
index 7e2442a5e8..b690673641 100644
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h
@@ -33,4 +33,14 @@ TfLiteStatus GetAudioSamples(tflite::ErrorReporter* error_reporter,
                              int start_ms, int duration_ms,
                              int* audio_samples_size, int16_t** audio_samples);
 
+// Returns the time that audio data was last captured in milliseconds. There's
+// no contract about what time zero represents, the accuracy, or the granularity
+// of the result. Subsequent calls will generally not return a lower value, but
+// even that's not guaranteed if there's an overflow  wraparound.
+// The reference implementation of this function just returns a constantly
+// incrementing value for each call, since it would need a non-portable platform
+// call to access time information. For real applications, you'll need to write
+// your own platform-specific implementation.
+int32_t LatestAudioTimestamp();
+
 #endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_AUDIO_PROVIDER_H_
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider_test.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider_test.cc
index 5f7c7605f0..85fbbb80a6 100644
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider_test.cc
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider_test.cc
@@ -14,6 +14,9 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h"
+
+#include <limits>
+
 #include "tensorflow/lite/c/c_api_internal.h"
 #include "tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h"
 #include "tensorflow/lite/experimental/micro/micro_error_reporter.h"
@@ -41,4 +44,27 @@ TF_LITE_MICRO_TEST(TestAudioProvider) {
   }
 }
 
+TF_LITE_MICRO_TEST(TestTimer) {
+  // Make sure that the technically-undefined overflow behavior we rely on below
+  // works on this platform. It's still not guaranteed, but at least this is a
+  // sanity check.  Turn off when running with ASan, as it will complain about
+  // the following undefined behavior.
+#ifndef ADDRESS_SANITIZER
+  int32_t overflow_value = std::numeric_limits<int32_t>::max();
+  overflow_value += 1;
+  TF_LITE_MICRO_EXPECT_EQ(std::numeric_limits<int32_t>::min(), overflow_value);
+#endif
+
+  const int32_t first_time = LatestAudioTimestamp();
+  const int32_t second_time = LatestAudioTimestamp();
+
+  // It's possible that the timer may have wrapped around from +BIG_NUM to
+  // -BIG_NUM between the first and second calls, since we're storing
+  // milliseconds in a 32-bit integer. It's not reasonable that the call itself
+  // would have taken more than 2^31 milliseconds though, so look at the
+  // difference and rely on integer overflow to ensure it's accurate.
+  const int32_t time_delta = (second_time - first_time);
+  TF_LITE_MICRO_EXPECT_LE(0, time_delta);
+}
+
 TF_LITE_MICRO_TESTS_END
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/main.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/main.cc
index 515f82fcbc..3a9a5a4df1 100644
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/main.cc
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/main.cc
@@ -13,10 +13,10 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h"
 #include "tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.h"
 #include "tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h"
 #include "tensorflow/lite/experimental/micro/examples/micro_speech/recognize_commands.h"
-#include "tensorflow/lite/experimental/micro/examples/micro_speech/timer.h"
 #include "tensorflow/lite/experimental/micro/examples/micro_speech/tiny_conv_model_data.h"
 #include "tensorflow/lite/experimental/micro/kernels/all_ops_resolver.h"
 #include "tensorflow/lite/experimental/micro/micro_error_reporter.h"
@@ -76,7 +76,7 @@ int main(int argc, char* argv[]) {
   // Keep reading and analysing audio data in an infinite loop.
   while (true) {
     // Fetch the spectrogram for the current time.
-    const int32_t current_time = TimeInMilliseconds();
+    const int32_t current_time = LatestAudioTimestamp();
     int how_many_new_slices = 0;
     TfLiteStatus feature_status = feature_provider.PopulateFeatureData(
         error_reporter, previous_time, current_time, &how_many_new_slices);
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h b/tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h
index 1d8f3123a5..f48252d14d 100644
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h
@@ -23,6 +23,7 @@ limitations under the License.
 // frequency information. This has to be a power of two, and since we're dealing
 // with 30ms of 16KHz inputs, which means 480 samples, this is the next value.
 constexpr int kMaxAudioSampleSize = 512;
+constexpr int kAudioSampleFrequency = 16000;
 
 // All of these values are derived from the values used during model training,
 // if you change your model you'll need to update these constants.
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/osx/Makefile.inc b/tensorflow/lite/experimental/micro/examples/micro_speech/osx/Makefile.inc
new file mode 100644
index 0000000000..89d107cfe0
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/osx/Makefile.inc
@@ -0,0 +1,6 @@
+#Settings for Mac OS platforms.
+ifeq ($(TARGET), osx)
+  MICROLITE_LIBS += \
+    -framework Foundation \
+    -framework AudioToolbox
+endif
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/osx/audio_provider.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/osx/audio_provider.cc
new file mode 100644
index 0000000000..892757e799
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/osx/audio_provider.cc
@@ -0,0 +1,139 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h"
+
+#include <AudioToolbox/AudioToolbox.h>
+
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h"
+
+namespace {
+
+constexpr int kNumberRecordBuffers = 3;
+bool g_is_audio_initialized = false;
+constexpr int kAudioCaptureBufferSize = kAudioSampleFrequency * 0.5;
+int16_t g_audio_capture_buffer[kAudioCaptureBufferSize];
+int16_t g_audio_output_buffer[kMaxAudioSampleSize];
+int32_t g_latest_audio_timestamp = 0;
+
+// Checks for MacOS errors, prints information and returns a TF Lite version.
+#define RETURN_IF_OS_ERROR(error, error_reporter)                       \
+  do {                                                                  \
+    if (error != noErr) {                                               \
+      error_reporter->Report("Error: %s:%d (%d)\n", __FILE__, __LINE__, \
+                             error);                                    \
+      return kTfLiteError;                                              \
+    }                                                                   \
+  } while (0);
+
+// Called when an audio input buffer has been filled.
+void OnAudioBufferFilledCallback(
+    void* user_data, AudioQueueRef queue, AudioQueueBufferRef buffer,
+    const AudioTimeStamp* start_time, UInt32 num_packets,
+    const AudioStreamPacketDescription* packet_description) {
+  const int sample_size = buffer->mAudioDataByteSize / sizeof(float);
+  const int64_t sample_offset = start_time->mSampleTime;
+  const int32_t time_in_ms =
+      (sample_offset + sample_size) / (kAudioSampleFrequency / 1000);
+  const float* float_samples = static_cast<const float*>(buffer->mAudioData);
+  for (int i = 0; i < sample_size; ++i) {
+    const int capture_index = (sample_offset + i) % kAudioCaptureBufferSize;
+    g_audio_capture_buffer[capture_index] = float_samples[i] * ((1 << 15) - 1);
+  }
+  // This is how we let the outside world know that new audio data has arrived.
+  g_latest_audio_timestamp = time_in_ms;
+  AudioQueueEnqueueBuffer(queue, buffer, 0, nullptr);
+}
+
+// Set up everything we need to capture audio samples from the default recording
+// device on MacOS.
+TfLiteStatus InitAudioRecording(tflite::ErrorReporter* error_reporter) {
+  // Set up the format of the audio - single channel, 32-bit float at 16KHz.
+  AudioStreamBasicDescription recordFormat = {0};
+  recordFormat.mSampleRate = kAudioSampleFrequency;
+  recordFormat.mFormatID = kAudioFormatLinearPCM;
+  recordFormat.mFormatFlags =
+      kAudioFormatFlagIsFloat | kAudioFormatFlagIsPacked;
+  recordFormat.mBitsPerChannel = 8 * sizeof(float);
+  recordFormat.mChannelsPerFrame = 1;
+  recordFormat.mBytesPerFrame = sizeof(float) * recordFormat.mChannelsPerFrame;
+  recordFormat.mFramesPerPacket = 1;
+  recordFormat.mBytesPerPacket =
+      recordFormat.mBytesPerFrame * recordFormat.mFramesPerPacket;
+  recordFormat.mReserved = 0;
+
+  UInt32 propSize = sizeof(recordFormat);
+  RETURN_IF_OS_ERROR(AudioFormatGetProperty(kAudioFormatProperty_FormatInfo, 0,
+                                            NULL, &propSize, &recordFormat),
+                     error_reporter);
+
+  // Create a recording queue.
+  AudioQueueRef queue;
+  RETURN_IF_OS_ERROR(
+      AudioQueueNewInput(&recordFormat, OnAudioBufferFilledCallback,
+                         error_reporter, nullptr, nullptr, 0, &queue),
+      error_reporter);
+
+  // Set up the buffers we'll need.
+  int buffer_bytes = 512 * sizeof(float);
+  for (int i = 0; i < kNumberRecordBuffers; ++i) {
+    AudioQueueBufferRef buffer;
+    RETURN_IF_OS_ERROR(AudioQueueAllocateBuffer(queue, buffer_bytes, &buffer),
+                       error_reporter);
+    RETURN_IF_OS_ERROR(AudioQueueEnqueueBuffer(queue, buffer, 0, nullptr),
+                       error_reporter);
+  }
+
+  // Start capturing audio.
+  RETURN_IF_OS_ERROR(AudioQueueStart(queue, nullptr), error_reporter);
+
+  return kTfLiteOk;
+}
+
+}  // namespace
+
+TfLiteStatus GetAudioSamples(tflite::ErrorReporter* error_reporter,
+                             int start_ms, int duration_ms,
+                             int* audio_samples_size, int16_t** audio_samples) {
+  if (!g_is_audio_initialized) {
+    TfLiteStatus init_status = InitAudioRecording(error_reporter);
+    if (init_status != kTfLiteOk) {
+      return init_status;
+    }
+    for (int i = 0; i < kMaxAudioSampleSize; ++i) {
+      g_audio_output_buffer[i] = 0;
+    }
+    g_is_audio_initialized = true;
+  }
+  // This should only be called when the main thread notices that the latest
+  // audio sample data timestamp has changed, so that there's new data in the
+  // capture ring buffer. The ring buffer will eventually wrap around and
+  // overwrite the data, but the assumption is that the main thread is checking
+  // often enough and the buffer is large enough that this call will be made
+  // before that happens.
+  const int start_offset = start_ms * (kAudioSampleFrequency / 1000);
+  const int duration_sample_count =
+      duration_ms * (kAudioSampleFrequency / 1000);
+  for (int i = 0; i < duration_sample_count; ++i) {
+    const int capture_index = (start_offset + i) % kAudioCaptureBufferSize;
+    g_audio_output_buffer[i] = g_audio_capture_buffer[capture_index];
+  }
+
+  *audio_samples_size = kMaxAudioSampleSize;
+  *audio_samples = g_audio_output_buffer;
+  return kTfLiteOk;
+}
+
+int32_t LatestAudioTimestamp() { return g_latest_audio_timestamp; }
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/timer.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/timer.cc
deleted file mode 100644
index 6c96a61ab5..0000000000
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/timer.cc
+++ /dev/null
@@ -1,22 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/lite/experimental/micro/examples/micro_speech/timer.h"
-
-int32_t TimeInMilliseconds() {
-  static int current_time = 0;
-  current_time += 100;
-  return current_time;
-}
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/timer.h b/tensorflow/lite/experimental/micro/examples/micro_speech/timer.h
deleted file mode 100644
index 162952844a..0000000000
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/timer.h
+++ /dev/null
@@ -1,31 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_TIMER_H_
-#define TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_TIMER_H_
-
-#include <cstdint>
-
-// Returns the time in milliseconds. There's no contract about what time zero
-// represents, the accuracy, or the granularity of the result. Subsequent calls
-// will generally not return a lower value, but even that's not guaranteed if
-// there's an overflow  wraparound.
-// The reference implementation of this function just returns a constantly
-// incrementing value for each call, since it would need a non-portable platform
-// call to access time information. For real applications, you'll need to write
-// your own platform-specific implementation.
-int32_t TimeInMilliseconds();
-
-#endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_SPEECH_TIMER_H_
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/timer_test.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/timer_test.cc
deleted file mode 100644
index 0487a12b25..0000000000
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/timer_test.cc
+++ /dev/null
@@ -1,49 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/lite/experimental/micro/examples/micro_speech/timer.h"
-
-#include <limits>
-
-#include "tensorflow/lite/c/c_api_internal.h"
-#include "tensorflow/lite/experimental/micro/micro_error_reporter.h"
-#include "tensorflow/lite/experimental/micro/testing/micro_test.h"
-
-TF_LITE_MICRO_TESTS_BEGIN
-
-TF_LITE_MICRO_TEST(TestTimer) {
-  // Make sure that the technically-undefined overflow behavior we rely on below
-  // works on this platform. It's still not guaranteed, but at least this is a
-  // sanity check.  Turn off when running with ASan, as it will complain about
-  // the following undefined behavior.
-#ifndef ADDRESS_SANITIZER
-  int32_t overflow_value = std::numeric_limits<int32_t>::max();
-  overflow_value += 1;
-  TF_LITE_MICRO_EXPECT_EQ(std::numeric_limits<int32_t>::min(), overflow_value);
-#endif
-
-  const int32_t first_time = TimeInMilliseconds();
-  const int32_t second_time = TimeInMilliseconds();
-
-  // It's possible that the timer may have wrapped around from +BIG_NUM to
-  // -BIG_NUM between the first and second calls, since we're storing
-  // milliseconds in a 32-bit integer. It's not reasonable that the call itself
-  // would have taken more than 2^31 milliseconds though, so look at the
-  // difference and rely on integer overflow to ensure it's accurate.
-  const int32_t time_delta = (second_time - first_time);
-  TF_LITE_MICRO_EXPECT_LE(0, time_delta);
-}
-
-TF_LITE_MICRO_TESTS_END
diff --git a/tensorflow/lite/experimental/micro/tools/make/Makefile b/tensorflow/lite/experimental/micro/tools/make/Makefile
index 6a7c875888..f9a2dffae2 100644
--- a/tensorflow/lite/experimental/micro/tools/make/Makefile
+++ b/tensorflow/lite/experimental/micro/tools/make/Makefile
@@ -21,6 +21,38 @@ HOST_ARCH := $(shell if [[ $(shell uname -m) =~ i[345678]86 ]]; then echo x86_32
 TARGET := $(HOST_OS)
 TARGET_ARCH := $(HOST_ARCH)
 
+# Look for platform or target-specific implementation files to replace reference
+# implementations with, given a tag. These are expected to occur in subfolders
+# of a directory where a reference implementation exists, and have the same
+# interface and header file. For example,
+# tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.cc
+# defines a module for supplying audio data, but since no platform or OS can be
+# presumed, it just always returns zeroes for its samples. The MacOS-specific
+# tensorflow/lite/experimental/micro/examples/micro_speech/osx/audio_provider.cc
+# has an implementation that relies on CoreAudio, and there are equivalent
+# versions for other operating systems. 
+# All lists of source files are put through this substitution process with the
+# tags of their target OS and architecture, so that implementations can be added
+# by simply placing them in the file tree, with no changes to the build files
+# needed.
+# One confusing thing about this implementation is that we're using wildcard to
+# act as a 'does file exist?' function, rather than expanding an expression.
+# Wildcard will return an empty string if given a plain file path with no actual
+# wildcards, if the file doesn't exist, so taking the first word of the list
+# between that and the reference path will pick the specialized one if it's
+# available.
+substitute_specialized_implementation = \
+  $(firstword $(wildcard $(dir $(1))$(2)/$(notdir $(1))) $(wildcard $(1)))
+substitute_specialized_implementations = \
+  $(foreach source,$(1),$(call substitute_specialized_implementation,$(source),$(2)))
+# Here we're first looking for specialized implementations in ref_dir/$(TARGET)
+# and then ref_dir/$(TARGET_ARCH), before falling back to ref_dir's
+# implementation.
+# The argument to this function should be a list of space-separated file paths,
+# with any wildcards already expanded.
+specialize = \
+  $(call substitute_specialized_implementations,$(call substitute_specialized_implementations,$(1),$(TARGET)),$(TARGET_ARCH))
+
 INCLUDES := \
 -I. \
 -I$(MAKEFILE_DIR)/../../../../../ \
@@ -67,6 +99,7 @@ tensorflow/lite/core/api/op_resolver.cc \
 tensorflow/lite/kernels/kernel_util.cc \
 tensorflow/lite/kernels/internal/quantization_util.cc
 MICROLITE_CC_SRCS := $(filter-out $(MICROLITE_TEST_SRCS), $(MICROLITE_CC_BASE_SRCS))
+MICROLITE_CC_SRCS := $(call specialize,$(MICROLITE_CC_SRCS))
 
 # These target-specific makefiles should modify or replace options like
 # CXXFLAGS or LIBS to work for a specific targetted architecture. All logic
@@ -115,7 +148,7 @@ $(OBJDIR)%.o: %.S
 	$(CC) $(CCFLAGS) $(INCLUDES) -c $< -o $@
 
 # The target that's compiled if there's no command-line arguments.
-all: $(MICROLITE_LIB_PATH) $(MICRO_SPEECH_TEST_BINARY) $(PREPROCESSOR_TEST_BINARY)
+all: $(MICROLITE_LIB_PATH)
 
 microlite: $(MICROLITE_LIB_PATH)
 
diff --git a/tensorflow/lite/experimental/micro/tools/make/targets/osx_makefile.inc b/tensorflow/lite/experimental/micro/tools/make/targets/osx_makefile.inc
new file mode 100644
index 0000000000..0e8aad1993
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/tools/make/targets/osx_makefile.inc
@@ -0,0 +1,10 @@
+#Settings for Mac OS platforms.
+ifeq ($(TARGET), osx)
+
+  PLATFORM_FLAGS = \
+    -DTF_LITE_DISABLE_X86_NEON
+
+  CXXFLAGS += $(PLATFORM_FLAGS)
+  CCFLAGS += $(PLATFORM_FLAGS)
+
+endif
\ No newline at end of file
-- 
GitLab


From f9b6cccd90939bf9516d0ae0e3bbdc4a7c4f4a54 Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Tue, 8 Jan 2019 13:50:00 -0800
Subject: [PATCH 0404/1095] [tf.data] Update `map` documentation -- calling out
 the fact that setting `num_parallel_calls` results in asynchronous
 computation.

PiperOrigin-RevId: 228392788
---
 tensorflow/python/data/ops/dataset_ops.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py
index 61210f3a7f..45e732be0d 100644
--- a/tensorflow/python/data/ops/dataset_ops.py
+++ b/tensorflow/python/data/ops/dataset_ops.py
@@ -970,8 +970,8 @@ class DatasetV2(object):
         shapes and types defined by `self.output_shapes` and
        `self.output_types`) to another nested structure of tensors.
       num_parallel_calls: (Optional.) A `tf.int32` scalar `tf.Tensor`,
-        representing the number elements to process in parallel. If not
-        specified, elements will be processed sequentially. If the value
+        representing the number elements to process asynchronously in parallel.
+        If not specified, elements will be processed sequentially. If the value
         `tf.data.experimental.AUTOTUNE` is used, then the number of parallel
         calls is set dynamically based on available CPU.
 
-- 
GitLab


From 4aaaee6112c40791e5dbc9730c2a06274ff35b3f Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Tue, 8 Jan 2019 13:50:17 -0800
Subject: [PATCH 0405/1095] [TF:XLA] Bump open source llvm revision to r350570

PiperOrigin-RevId: 228392842
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 834ec28447..ae26ba6e68 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -500,11 +500,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "llvm",
         build_file = clean_dep("//third_party/llvm:llvm.autogenerated.BUILD"),
-        sha256 = "20c0ebd54f433d68652a54e328d70b0ec7cf37ce734645a0259cf904d14c2c53",
-        strip_prefix = "llvm-0e2aef924e02d00a4d3454d63c1921a243d316c6",
+        sha256 = "83a4f199742f3d6892994dd6dc46d6a53019aedaa28590b460c120f3dfc7bc47",
+        strip_prefix = "llvm-671f057a2cf137914be6c786daf8000469adebab",
         urls = [
-            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/0e2aef924e02d00a4d3454d63c1921a243d316c6.tar.gz",
-            "https://github.com/llvm-mirror/llvm/archive/0e2aef924e02d00a4d3454d63c1921a243d316c6.tar.gz",
+            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/671f057a2cf137914be6c786daf8000469adebab.tar.gz",
+            "https://github.com/llvm-mirror/llvm/archive/671f057a2cf137914be6c786daf8000469adebab.tar.gz",
         ],
     )
 
-- 
GitLab


From 731ae198b0d16e2b51299c6fd5050a428bc18152 Mon Sep 17 00:00:00 2001
From: Duncan Riach <duncan@nvidia.com>
Date: Tue, 8 Jan 2019 14:13:09 -0800
Subject: [PATCH 0406/1095] Renaming and factoring

---
 tensorflow/stream_executor/cuda/cuda_dnn.cc | 78 +++++++++++----------
 1 file changed, 41 insertions(+), 37 deletions(-)

diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.cc b/tensorflow/stream_executor/cuda/cuda_dnn.cc
index e91caf3bdb..16d644629e 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.cc
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.cc
@@ -796,12 +796,12 @@ bool BatchnormSpatialPersistentEnabled() {
 
 // A helper function to decide whether to enable deterministic functionality.
 // TODO(pr/24355): Support all cuDNN functionality (currently only convolution).
-bool Deterministic() {
+bool RequireDeterminism() {
   static bool is_enabled = [] {
     bool is_enabled = false;
-    TF_CHECK_OK(
-        tensorflow::ReadBoolFromEnvVar("TF_CUDNN_DETERMINISTIC",
-                                       /*default_val=*/false, &is_enabled));
+    TF_CHECK_OK(tensorflow::ReadBoolFromEnvVar("TF_CUDNN_DETERMINISTIC",
+                                               /*default_val=*/false,
+                                               &is_enabled));
     return is_enabled;
   }();
   return is_enabled;
@@ -3049,9 +3049,22 @@ port::Status CudnnSupport::DoFusedConvolveImpl(
   return port::Status::OK();
 }
 
+inline bool TensorOpMathAvailable(int cc_major) {
+  return cc_major >= 7 && CUDNN_VERSION >= 7000 && TensorOpMathEnabled();
+}
+
 bool CudnnSupport::GetConvolveAlgorithms(
     bool with_winograd_nonfused, int cc_major, int cc_minor,
     std::vector<dnn::AlgorithmDesc>* out_algorithms) {
+  bool tensor_op_math_available = TensorOpMathAvailable(cc_major);
+  out_algorithms->clear();
+
+  if (RequireDeterminism()) {
+    out_algorithms->push_back({CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM,
+                               tensor_op_math_available});
+    return true;
+  }
+
   std::vector<dnn::AlgorithmDesc::Index> algo_types = {
     // clang-format off
     CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_GEMM,
@@ -3069,22 +3082,13 @@ bool CudnnSupport::GetConvolveAlgorithms(
     algo_types.push_back(CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED);
   }
 
-  if (Deterministic()) {
-    algo_types.clear();
-    algo_types.push_back(CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM);
-  }
-
-  out_algorithms->clear();
   for (auto i : algo_types) {
     out_algorithms->push_back({i, /*use_tensor_ops=*/false});
-    if (cc_major >= 7 && CUDNN_VERSION >= 7000 && TensorOpMathEnabled()) {
-      if (Deterministic()) {
-        // For determinism: always use tensor op math, if it's available
-        out_algorithms->pop_back();
-      }
+    if (tensor_op_math_available) {
       out_algorithms->push_back({i, /*use_tensor_ops=*/true});
     }
   }
+
   return true;
 }
 
@@ -3113,6 +3117,15 @@ bool CudnnSupport::GetRnnAlgorithms(
 bool CudnnSupport::GetConvolveBackwardDataAlgorithms(
     bool with_winograd_nonfused, int cc_major, int cc_minor,
     std::vector<dnn::AlgorithmDesc>* out_algorithms) {
+  bool tensor_op_math_available = TensorOpMathAvailable(cc_major);
+  out_algorithms->clear();
+
+  if (RequireDeterminism()) {
+    out_algorithms->push_back(
+        {CUDNN_CONVOLUTION_BWD_DATA_ALGO_1, tensor_op_math_available});
+    return true;
+  }
+
   std::vector<dnn::AlgorithmDesc::Index> algo_types = {
       // clang-format off
     CUDNN_CONVOLUTION_BWD_DATA_ALGO_0,
@@ -3126,28 +3139,28 @@ bool CudnnSupport::GetConvolveBackwardDataAlgorithms(
     algo_types.push_back(CUDNN_CONVOLUTION_BWD_DATA_ALGO_WINOGRAD_NONFUSED);
   }
 
-  if (Deterministic()) {
-    algo_types.clear();
-    algo_types.push_back(CUDNN_CONVOLUTION_BWD_DATA_ALGO_1);
-  }
-
-  out_algorithms->clear();
   for (auto i : algo_types) {
     out_algorithms->push_back({i, /*use_tensor_ops=*/false});
-    if (cc_major >= 7 && CUDNN_VERSION >= 7000 && TensorOpMathEnabled()) {
-      if (Deterministic()) {
-        // For determinism: always use tensor op math, if it's available
-        out_algorithms->pop_back();
-      }
+    if (tensor_op_math_available) {
       out_algorithms->push_back({i, /*use_tensor_ops=*/true});
     }
   }
+
   return true;
 }
 
 bool CudnnSupport::GetConvolveBackwardFilterAlgorithms(
     bool with_winograd_nonfused, int cc_major, int cc_minor,
     std::vector<dnn::AlgorithmDesc>* out_algorithms) {
+  bool tensor_op_math_available = TensorOpMathAvailable(cc_major);
+  out_algorithms->clear();
+
+  if (RequireDeterminism()) {
+    out_algorithms->push_back(
+        {CUDNN_CONVOLUTION_BWD_FILTER_ALGO_1, tensor_op_math_available});
+    return true;
+  }
+
   std::vector<dnn::AlgorithmDesc::Index> algo_types = {
       // clang-format off
       CUDNN_CONVOLUTION_BWD_FILTER_ALGO_0,
@@ -3166,22 +3179,13 @@ bool CudnnSupport::GetConvolveBackwardFilterAlgorithms(
     algo_types.push_back(CUDNN_CONVOLUTION_BWD_FILTER_ALGO_WINOGRAD_NONFUSED);
   }
 
-  if (Deterministic()) {
-    algo_types.clear();
-    algo_types.push_back(CUDNN_CONVOLUTION_BWD_FILTER_ALGO_1);
-  }
-
-  out_algorithms->clear();
   for (auto i : algo_types) {
     out_algorithms->push_back({i, /*use_tensor_ops=*/false});
-    if (cc_major >= 7 && CUDNN_VERSION >= 7000 && TensorOpMathEnabled()) {
-      if (Deterministic()) {
-        // For determinism: always use tensor op math, if it's available
-        out_algorithms->pop_back();
-      }
+    if (tensor_op_math_available) {
       out_algorithms->push_back({i, /*use_tensor_ops=*/true});
     }
   }
+
   return true;
 }
 
-- 
GitLab


From c52d92f10021f8caa00ea271755d47a9c86259bd Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Tue, 8 Jan 2019 14:06:17 -0800
Subject: [PATCH 0407/1095] Internal change

PiperOrigin-RevId: 228396077
---
 tensorflow/lite/experimental/swift/BUILD      | 101 -----
 tensorflow/lite/experimental/swift/LICENSE    | 202 ----------
 tensorflow/lite/experimental/swift/README.md  |  54 ---
 .../swift/Sources/Interpreter.swift           | 265 --------------
 .../swift/Sources/InterpreterError.swift      |  99 -----
 .../swift/Sources/InterpreterOptions.swift    |  29 --
 .../experimental/swift/Sources/Model.swift    |  40 --
 .../Sources/QuantizationParameters.swift      |  38 --
 .../experimental/swift/Sources/Tensor.swift   | 138 -------
 .../Configs/TensorFlowLite.tulsigen           |  57 ---
 .../project.tulsiconf                         |  14 -
 .../project.pbxproj                           | 345 ------------------
 .../TensorFlowLiteApp/AppDelegate.swift       |  24 --
 .../Array+TensorFlowLite.swift                |  22 --
 .../AppIcon.appiconset/Contents.json          |  98 -----
 .../Assets.xcassets/Contents.json             |   6 -
 .../Base.lproj/LaunchScreen.storyboard        |  44 ---
 .../Base.lproj/Main.storyboard                |  95 -----
 .../Data+TensorFlowLite.swift                 |  13 -
 .../TensorFlowLiteApp/Info.plist              |  46 ---
 .../TensorFlowLiteApp/ViewController.swift    | 299 ---------------
 .../swift/Tests/InterpreterOptionsTests.swift |  54 ---
 .../swift/Tests/InterpreterTests.swift        | 315 ----------------
 .../experimental/swift/Tests/ModelTests.swift |  59 ---
 .../Tests/QuantizationParametersTests.swift   |  43 ---
 .../swift/Tests/TensorTests.swift             |  83 -----
 26 files changed, 2583 deletions(-)
 delete mode 100644 tensorflow/lite/experimental/swift/BUILD
 delete mode 100644 tensorflow/lite/experimental/swift/LICENSE
 delete mode 100644 tensorflow/lite/experimental/swift/README.md
 delete mode 100644 tensorflow/lite/experimental/swift/Sources/Interpreter.swift
 delete mode 100644 tensorflow/lite/experimental/swift/Sources/InterpreterError.swift
 delete mode 100644 tensorflow/lite/experimental/swift/Sources/InterpreterOptions.swift
 delete mode 100644 tensorflow/lite/experimental/swift/Sources/Model.swift
 delete mode 100644 tensorflow/lite/experimental/swift/Sources/QuantizationParameters.swift
 delete mode 100644 tensorflow/lite/experimental/swift/Sources/Tensor.swift
 delete mode 100644 tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/Configs/TensorFlowLite.tulsigen
 delete mode 100644 tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/project.tulsiconf
 delete mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp.xcodeproj/project.pbxproj
 delete mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/AppDelegate.swift
 delete mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Array+TensorFlowLite.swift
 delete mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/AppIcon.appiconset/Contents.json
 delete mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/Contents.json
 delete mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/LaunchScreen.storyboard
 delete mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/Main.storyboard
 delete mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Data+TensorFlowLite.swift
 delete mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Info.plist
 delete mode 100644 tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/ViewController.swift
 delete mode 100644 tensorflow/lite/experimental/swift/Tests/InterpreterOptionsTests.swift
 delete mode 100644 tensorflow/lite/experimental/swift/Tests/InterpreterTests.swift
 delete mode 100644 tensorflow/lite/experimental/swift/Tests/ModelTests.swift
 delete mode 100644 tensorflow/lite/experimental/swift/Tests/QuantizationParametersTests.swift
 delete mode 100644 tensorflow/lite/experimental/swift/Tests/TensorTests.swift

diff --git a/tensorflow/lite/experimental/swift/BUILD b/tensorflow/lite/experimental/swift/BUILD
deleted file mode 100644
index 53bcb0ecbd..0000000000
--- a/tensorflow/lite/experimental/swift/BUILD
+++ /dev/null
@@ -1,101 +0,0 @@
-# TensorFlow Lite for Swift.
-
-package(default_visibility = ["//visibility:private"])
-
-licenses(["notice"])  # Apache 2.0
-
-exports_files(["LICENSE"])
-
-load("@build_bazel_rules_apple//apple:ios.bzl", "ios_application", "ios_unit_test")
-load("@build_bazel_rules_swift//swift:swift.bzl", "swift_library")
-
-MINIMUM_OS_VERSION = "9.0"
-
-SWIFT_COPTS = [
-    "-wmo",
-]
-
-swift_library(
-    name = "TensorFlowLite",
-    srcs = glob(["Sources/*.swift"]),
-    copts = SWIFT_COPTS,
-    module_name = "TensorFlowLite",
-    tags = ["manual"],
-    deps = [
-        "//tensorflow/lite/experimental/c:c_api",
-    ],
-)
-
-ios_unit_test(
-    name = "TensorFlowLiteTests",
-    size = "small",
-    minimum_os_version = MINIMUM_OS_VERSION,
-    tags = [
-        "manual",
-        # DISABLED: Following sanitizer tests are not supported by iOS test targets.
-        "noasan",
-        "nomsan",
-        "notsan",
-    ],
-    deps = [":TensorFlowLiteTestsLib"],
-)
-
-swift_library(
-    name = "TensorFlowLiteTestsLib",
-    testonly = 1,
-    srcs = glob(["Tests/*.swift"]),
-    copts = SWIFT_COPTS,
-    tags = ["manual"],
-    deps = [
-        ":TensorFlowLite",
-        ":TestResources",
-    ],
-)
-
-objc_library(
-    name = "TestResources",
-    resources = [
-        "//tensorflow/lite:testdata/add.bin",
-        "//tensorflow/lite:testdata/add_quantized.bin",
-        "//tensorflow/lite:testdata/multi_add.bin",
-    ],
-    tags = ["manual"],
-)
-
-ios_application(
-    name = "TensorFlowLiteApp",
-    app_icons = glob(["TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/AppIcon.appiconset/**"]),
-    bundle_id = "com.tensorflow.lite.swift.TensorFlowLite",
-    families = [
-        "ipad",
-        "iphone",
-    ],
-    infoplists = ["TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Info.plist"],
-    launch_storyboard = "TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/LaunchScreen.storyboard",
-    minimum_os_version = MINIMUM_OS_VERSION,
-    sdk_frameworks = [
-        "CoreGraphics",
-    ],
-    tags = ["manual"],
-    deps = [":TensorFlowLiteAppLib"],
-)
-
-swift_library(
-    name = "TensorFlowLiteAppLib",
-    srcs = glob(["TestApps/TensorFlowLiteApp/TensorFlowLiteApp/*.swift"]),
-    module_name = "TensorFlowLiteAppLib",
-    tags = ["manual"],
-    deps = [
-        ":TensorFlowLite",
-        ":TensorFlowLiteAppResources",
-    ],
-)
-
-objc_library(
-    name = "TensorFlowLiteAppResources",
-    storyboards = glob([
-        "TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/*.storyboard",
-    ]),
-    tags = ["manual"],
-    deps = [":TestResources"],
-)
diff --git a/tensorflow/lite/experimental/swift/LICENSE b/tensorflow/lite/experimental/swift/LICENSE
deleted file mode 100644
index d645695673..0000000000
--- a/tensorflow/lite/experimental/swift/LICENSE
+++ /dev/null
@@ -1,202 +0,0 @@
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
diff --git a/tensorflow/lite/experimental/swift/README.md b/tensorflow/lite/experimental/swift/README.md
deleted file mode 100644
index deb16c1e91..0000000000
--- a/tensorflow/lite/experimental/swift/README.md
+++ /dev/null
@@ -1,54 +0,0 @@
-# TensorFlow Lite for Swift
-
-[TensorFlow Lite](https://www.tensorflow.org/lite/) is TensorFlow's lightweight
-solution for Swift developers. It enables low-latency inference of on-device
-machine learning models with a small binary size and fast performance supporting
-hardware acceleration.
-
-## Getting Started
-
-### Bazel
-
-In your `BUILD` file, add the `TensorFlowLite` dependency:
-
-```python
-swift_library(
-  # ...
-  deps = [
-      "//tensorflow/lite/swift:TensorFlowLite",
-  ],
-)
-```
-
-In your Swift files, import the module:
-
-```swift
-import TensorFlowLite
-```
-
-### Tulsi
-
-Open the `TensorFlowLite.tulsiproj` using the [TulsiApp](https://github.com/bazelbuild/tulsi) or by
-running the [`generate_xcodeproj.sh`](https://github.com/bazelbuild/tulsi/blob/master/src/tools/generate_xcodeproj.sh)
-script:
-
-```shell
-generate_xcodeproj.sh --genconfig tensorflow/lite/swift/TensorFlowLite.tulsiproj:TensorFlowLite --outputfolder ~/path/to/generated/TensorFlowLite.xcodeproj
-```
-
-### CocoaPods
-
-Add the following to your `Podfile`:
-
-```ruby
-use_frameworks!
-pod 'TensorFlowLiteSwift'
-```
-
-Then, run `pod install`.
-
-In your Swift files, import the module:
-
-```swift
-import TensorFlowLite
-```
diff --git a/tensorflow/lite/experimental/swift/Sources/Interpreter.swift b/tensorflow/lite/experimental/swift/Sources/Interpreter.swift
deleted file mode 100644
index a14b5966b1..0000000000
--- a/tensorflow/lite/experimental/swift/Sources/Interpreter.swift
+++ /dev/null
@@ -1,265 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import Foundation
-import TensorFlowLiteCAPI
-
-/// A TensorFlow Lite interpreter that performs inference from a given model.
-public final class Interpreter {
-
-  /// The `TFL_Interpreter` C pointer type represented as an `UnsafePointer<TFL_Interpreter>`.
-  private typealias CInterpreter = OpaquePointer
-
-  /// Total number of input tensors associated with the model.
-  public var inputTensorCount: Int {
-    return Int(TFL_InterpreterGetInputTensorCount(cInterpreter))
-  }
-
-  /// Total number of output tensors associated with the model.
-  public var outputTensorCount: Int {
-    return Int(TFL_InterpreterGetOutputTensorCount(cInterpreter))
-  }
-
-  /// The underlying `TFL_Interpreter` C pointer.
-  private var cInterpreter: CInterpreter?
-
-  /// Creates a new model interpreter instance.
-  ///
-  /// - Parameters:
-  ///   - modelPath: Local file path to a TensorFlow Lite model.
-  ///   - options: Custom configurations for the interpreter. The default is `nil` indicating that
-  ///       interpreter will determine the configuration options.
-  /// - Throws: An error if the model could not be loaded or the interpreter could not be created.
-  public init(modelPath: String, options: InterpreterOptions? = nil) throws {
-    guard let model = Model(filePath: modelPath) else { throw InterpreterError.failedToLoadModel }
-
-    let cInterpreterOptions: OpaquePointer? = try options.map { options in
-      guard let cOptions = TFL_NewInterpreterOptions() else {
-        throw InterpreterError.failedToCreateInterpreter
-      }
-      if let threadCount = options.threadCount, threadCount > 0 {
-        TFL_InterpreterOptionsSetNumThreads(cOptions, Int32(threadCount))
-      }
-      if options.isErrorLoggingEnabled {
-        TFL_InterpreterOptionsSetErrorReporter(
-          cOptions,
-          { (_, format, arguments) in
-            guard let cFormat = format,
-                  let message = String(cFormat: cFormat, arguments: arguments)
-            else {
-              return
-            }
-            print(String(describing: InterpreterError.tensorFlowLiteError(message)))
-          },
-          nil
-        )
-      }
-      return cOptions
-    }
-    defer { TFL_DeleteInterpreterOptions(cInterpreterOptions) }
-
-    guard let cInterpreter = TFL_NewInterpreter(model.cModel, cInterpreterOptions) else {
-      throw InterpreterError.failedToCreateInterpreter
-    }
-    self.cInterpreter = cInterpreter
-  }
-
-  deinit {
-    TFL_DeleteInterpreter(cInterpreter)
-  }
-
-  /// Invokes the interpreter to perform inference from the loaded graph.
-  ///
-  /// - Throws: An error if the model was not ready because tensors were not allocated.
-  public func invoke() throws {
-    guard TFL_InterpreterInvoke(cInterpreter) == kTfLiteOk else {
-      // TODO(b/117510052): Determine which error to throw.
-      throw InterpreterError.allocateTensorsRequired
-    }
-  }
-
-  /// Returns the input tensor at the given index.
-  ///
-  /// - Parameters:
-  ///   - index: The index for the input tensor.
-  /// - Throws: An error if the index is invalid or the tensors have not been allocated.
-  /// - Returns: The input tensor at the given index.
-  public func input(at index: Int) throws -> Tensor {
-    let maxIndex = inputTensorCount - 1
-    guard case 0...maxIndex = index else {
-      throw InterpreterError.invalidTensorIndex(index: index, maxIndex: maxIndex)
-    }
-    guard let cTensor = TFL_InterpreterGetInputTensor(cInterpreter, Int32(index)),
-          let bytes = TFL_TensorData(cTensor),
-          let nameCString = TFL_TensorName(cTensor)
-    else {
-      throw InterpreterError.allocateTensorsRequired
-    }
-    guard let dataType = TensorDataType(type: TFL_TensorType(cTensor)) else {
-      throw InterpreterError.invalidTensorDataType
-    }
-
-    let name = String(cString: nameCString)
-    let rank = TFL_TensorNumDims(cTensor)
-    let dimensions = (0..<rank).map { Int(TFL_TensorDim(cTensor, $0)) }
-    let shape = TensorShape(dimensions)
-    let byteCount = TFL_TensorByteSize(cTensor)
-    let data = Data(bytes: bytes, count: byteCount)
-    let cQuantizationParams = TFL_TensorQuantizationParams(cTensor)
-    let scale = cQuantizationParams.scale
-    let zeroPoint = Int(cQuantizationParams.zero_point)
-    var quantizationParameters: QuantizationParameters? = nil
-    if scale != 0.0 {
-      // TODO(b/117510052): Update this check once the TfLiteQuantizationParams struct has a mode.
-      quantizationParameters = QuantizationParameters(scale: scale, zeroPoint: zeroPoint)
-    }
-    let tensor = Tensor(
-      name: name,
-      dataType: dataType,
-      shape: shape,
-      data: data,
-      quantizationParameters: quantizationParameters
-    )
-    return tensor
-  }
-
-  /// Returns the output tensor at the given index.
-  ///
-  /// - Parameters:
-  ///   - index: The index for the output tensor.
-  /// - Throws: An error if the index is invalid, tensors haven't been allocated, or interpreter
-  ///     hasn't been invoked for models that dynamically compute output tensors based on the values
-  ///     of its input tensors.
-  /// - Returns: The output tensor at the given index.
-  public func output(at index: Int) throws -> Tensor {
-    let maxIndex = outputTensorCount - 1
-    guard case 0...maxIndex = index else {
-      throw InterpreterError.invalidTensorIndex(index: index, maxIndex: maxIndex)
-    }
-    guard let cTensor = TFL_InterpreterGetOutputTensor(cInterpreter, Int32(index)),
-          let bytes = TFL_TensorData(cTensor),
-          let nameCString = TFL_TensorName(cTensor)
-    else {
-      // TODO(b/117510052): Determine which error to throw.
-      throw InterpreterError.invokeInterpreterRequired
-    }
-    guard let dataType = TensorDataType(type: TFL_TensorType(cTensor)) else {
-      throw InterpreterError.invalidTensorDataType
-    }
-
-    let name = String(cString: nameCString)
-    let rank = TFL_TensorNumDims(cTensor)
-    let dimensions = (0..<rank).map { Int(TFL_TensorDim(cTensor, $0)) }
-    let shape = TensorShape(dimensions)
-    let byteCount = TFL_TensorByteSize(cTensor)
-    let data = Data(bytes: bytes, count: byteCount)
-    let cQuantizationParams = TFL_TensorQuantizationParams(cTensor)
-    let scale = cQuantizationParams.scale
-    let zeroPoint = Int(cQuantizationParams.zero_point)
-    var quantizationParameters: QuantizationParameters? = nil
-    if scale != 0.0 {
-      // TODO(b/117510052): Update this check once the TfLiteQuantizationParams struct has a mode.
-      quantizationParameters = QuantizationParameters(scale: scale, zeroPoint: zeroPoint)
-    }
-    let tensor = Tensor(
-      name: name,
-      dataType: dataType,
-      shape: shape,
-      data: data,
-      quantizationParameters: quantizationParameters
-    )
-    return tensor
-  }
-
-  /// Resizes the input tensor at the given index to the specified tensor shape.
-  ///
-  /// - Note: After resizing an input tensor, the client **must** explicitly call
-  ///     `allocateTensors()` before attempting to access the resized tensor data or invoking the
-  ///     interpreter to perform inference.
-  /// - Parameters:
-  ///   - index: The index for the input tensor.
-  ///   - shape: The shape that the input tensor should be resized to.
-  /// - Throws: An error if the input tensor at the given index could not be resized.
-  public func resizeInput(at index: Int, to shape: TensorShape) throws {
-    let maxIndex = inputTensorCount - 1
-    guard case 0...maxIndex = index else {
-      throw InterpreterError.invalidTensorIndex(index: index, maxIndex: maxIndex)
-    }
-    guard TFL_InterpreterResizeInputTensor(
-            cInterpreter,
-            Int32(index),
-            shape.int32Dimensions,
-            Int32(shape.rank)
-          ) == kTfLiteOk
-    else {
-      throw InterpreterError.failedToResizeInputTensor(index: index)
-    }
-  }
-
-  /// Copies the given data to the input tensor at the given index.
-  ///
-  /// - Parameters:
-  ///   - data: The data to be copied to the input tensor's data buffer.
-  ///   - index: The index for the input tensor.
-  /// - Throws: An error if the `data.count` does not match the input tensor's `data.count` or if
-  ///     the given index is invalid.
-  /// - Returns: The input tensor with the copied data.
-  @discardableResult
-  public func copy(_ data: Data, toInputAt index: Int) throws -> Tensor {
-    let maxIndex = inputTensorCount - 1
-    guard case 0...maxIndex = index else {
-      throw InterpreterError.invalidTensorIndex(index: index, maxIndex: maxIndex)
-    }
-    guard let cTensor = TFL_InterpreterGetInputTensor(cInterpreter, Int32(index)) else {
-      throw InterpreterError.allocateTensorsRequired
-    }
-
-    let byteCount = TFL_TensorByteSize(cTensor)
-    guard data.count == byteCount else {
-      throw InterpreterError.invalidTensorDataCount(provided: data.count, required: byteCount)
-    }
-
-    let status = data.withUnsafeBytes { TFL_TensorCopyFromBuffer(cTensor, $0, data.count) }
-    guard status == kTfLiteOk else { throw InterpreterError.failedToCopyDataToInputTensor }
-    return try input(at: index)
-  }
-
-  /// Allocates memory for all input tensors based on their `TensorShape`s.
-  ///
-  /// - Note: This is a relatively expensive operation and should only be called after creating the
-  ///     interpreter and/or resizing any input tensors.
-  /// - Throws: An error if memory could not be allocated for the input tensors.
-  public func allocateTensors() throws {
-    guard TFL_InterpreterAllocateTensors(cInterpreter) == kTfLiteOk else {
-      throw InterpreterError.failedToAllocateTensors
-    }
-  }
-}
-
-// MARK: - Extensions
-
-extension String {
-  /// Returns a new `String` initialized by using the given format C array as a template into which
-  /// the remaining argument values are substituted according to the user’s default locale.
-  ///
-  /// - Note: Returns `nil` if a new `String` could not be constructed from the given values.
-  /// - Parameters:
-  ///   - cFormat: The format C array as a template for substituting values.
-  ///   - arguments: A C pointer to a `va_list` of arguments to substitute into `cFormat`.
-  init?(cFormat: UnsafePointer<CChar>, arguments: CVaListPointer) {
-    var buffer: UnsafeMutablePointer<CChar>?
-    guard vasprintf(&buffer, cFormat, arguments) != 0, let cString = buffer else { return nil }
-    self.init(validatingUTF8: cString)
-  }
-}
diff --git a/tensorflow/lite/experimental/swift/Sources/InterpreterError.swift b/tensorflow/lite/experimental/swift/Sources/InterpreterError.swift
deleted file mode 100644
index 5de58b997a..0000000000
--- a/tensorflow/lite/experimental/swift/Sources/InterpreterError.swift
+++ /dev/null
@@ -1,99 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import Foundation
-
-/// TensorFlow Lite interpreter errors.
-public enum InterpreterError: Error {
-  case invalidTensorIndex(index: Int, maxIndex: Int)
-  case invalidTensorDataCount(provided: Int, required: Int)
-  case invalidTensorDataType
-  case failedToLoadModel
-  case failedToCreateInterpreter
-  case failedToResizeInputTensor(index: Int)
-  case failedToCopyDataToInputTensor
-  case failedToAllocateTensors
-  case allocateTensorsRequired
-  case invokeInterpreterRequired
-  case tensorFlowLiteError(String)
-}
-
-// MARK: - Extensions
-
-extension InterpreterError: LocalizedError {
-  /// Localized description of the interpreter error.
-  public var errorDescription: String? {
-    switch self {
-    case .invalidTensorIndex(let index, let maxIndex):
-      return "Invalid tensor index \(index), max index is \(maxIndex)."
-    case .invalidTensorDataCount(let providedCount, let requiredCount):
-      return "Provided data count \(providedCount) must match the required count \(requiredCount)."
-    case .invalidTensorDataType:
-      return "Tensor data type is unsupported or could not be determined because of a model error."
-    case .failedToLoadModel:
-      return "Failed to load the given model."
-    case .failedToCreateInterpreter:
-      return "Failed to create the interpreter."
-    case .failedToResizeInputTensor(let index):
-      return "Failed to resize input tesnor at index \(index)."
-    case .failedToCopyDataToInputTensor:
-      return "Failed to copy data to input tensor."
-    case .failedToAllocateTensors:
-      return "Failed to allocate memory for input tensors."
-    case .allocateTensorsRequired:
-      return "Must call allocateTensors()."
-    case .invokeInterpreterRequired:
-      return "Must call invoke()."
-    case .tensorFlowLiteError(let message):
-      return "TensorFlow Lite Error: \(message)"
-    }
-  }
-}
-
-extension InterpreterError: CustomStringConvertible {
-  /// Textual representation of the TensorFlow Lite interpreter error.
-  public var description: String {
-    return errorDescription ?? "Unknown error."
-  }
-}
-
-#if swift(>=4.2)
-extension InterpreterError: Equatable {}
-#else
-extension InterpreterError: Equatable {
-  public static func == (lhs: InterpreterError, rhs: InterpreterError) -> Bool {
-    switch (lhs, rhs) {
-    case (.invalidTensorDataType, .invalidTensorDataType),
-         (.failedToLoadModel, .failedToLoadModel),
-         (.failedToCreateInterpreter, .failedToCreateInterpreter),
-         (.failedToAllocateTensors, .failedToAllocateTensors),
-         (.allocateTensorsRequired, .allocateTensorsRequired),
-         (.invokeInterpreterRequired, .invokeInterpreterRequired):
-      return true
-    case (.invalidTensorIndex(let lhsIndex, let lhsMaxIndex),
-          .invalidTensorIndex(let rhsIndex, let rhsMaxIndex)):
-      return lhsIndex == rhsIndex && lhsMaxIndex == rhsMaxIndex
-    case (.invalidTensorDataCount(let lhsProvidedCount, let lhsRequiredCount),
-          .invalidTensorDataCount(let rhsProvidedCount, let rhsRequiredCount)):
-      return lhsProvidedCount == rhsProvidedCount && lhsRequiredCount == rhsRequiredCount
-    case (.failedToResizeInputTensor(let lhsIndex), .failedToResizeInputTensor(let rhsIndex)):
-      return lhsIndex == rhsIndex
-    case (.tensorFlowLiteError(let lhsMessage), .tensorFlowLiteError(let rhsMessage)):
-      return lhsMessage == rhsMessage
-    default:
-      return false
-    }
-  }
-}
-#endif  // swift(>=4.2)
diff --git a/tensorflow/lite/experimental/swift/Sources/InterpreterOptions.swift b/tensorflow/lite/experimental/swift/Sources/InterpreterOptions.swift
deleted file mode 100644
index 2365fd7ade..0000000000
--- a/tensorflow/lite/experimental/swift/Sources/InterpreterOptions.swift
+++ /dev/null
@@ -1,29 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import Foundation
-
-/// Custom configuration options for a TensorFlow Lite interpreter.
-public struct InterpreterOptions: Equatable {
-
-  /// Maximum number of CPU threads that the interpreter should run on. Default is `nil` which
-  /// indicates that the `Interpreter` will decide the number of threads to use.
-  public var threadCount: Int? = nil
-
-  /// Whether error logging to the console is enabled. The default is `false`.
-  public var isErrorLoggingEnabled = false
-
-  /// Creates a new instance of interpreter options.
-  public init() {}
-}
diff --git a/tensorflow/lite/experimental/swift/Sources/Model.swift b/tensorflow/lite/experimental/swift/Sources/Model.swift
deleted file mode 100644
index e8c49ff1ae..0000000000
--- a/tensorflow/lite/experimental/swift/Sources/Model.swift
+++ /dev/null
@@ -1,40 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import Foundation
-import TensorFlowLiteCAPI
-
-/// A TensorFlow Lite model used by the 'Interpreter` to perform inference.
-final class Model {
-
-  /// The `TFL_Model` C pointer type represented as an `UnsafePointer<TFL_Model>`.
-  typealias CModel = OpaquePointer
-
-  /// The underlying `TFL_Model` C pointer.
-  let cModel: CModel?
-
-  /// Creates a new model instance.
-  ///
-  /// - Precondition: Initialization can fail if the given `filePath` is invalid.
-  /// - Parameters:
-  ///   - filePath: Local file path to a TensorFlow Lite model.
-  init?(filePath: String) {
-    guard !filePath.isEmpty, let cModel = TFL_NewModelFromFile(filePath) else { return nil }
-    self.cModel = cModel
-  }
-
-  deinit {
-    TFL_DeleteModel(cModel)
-  }
-}
diff --git a/tensorflow/lite/experimental/swift/Sources/QuantizationParameters.swift b/tensorflow/lite/experimental/swift/Sources/QuantizationParameters.swift
deleted file mode 100644
index f367875644..0000000000
--- a/tensorflow/lite/experimental/swift/Sources/QuantizationParameters.swift
+++ /dev/null
@@ -1,38 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import Foundation
-
-/// Parameters that determine the mapping of quantized values to real values. Quantized values can
-/// be mapped to float values using the following conversion:
-/// `realValue = scale * (quantizedValue - zeroPoint)`.
-public struct QuantizationParameters {
-
-  /// Difference between real values corresponding to consecutive quantized values differing by 1.
-  /// For example, the range of quantized values for `UInt8` data type is [0, 255].
-  public let scale: Float
-
-  /// Quantized value that corresponds to the real 0 value.
-  public let zeroPoint: Int
-
-  /// Creates a new quantization parameters instance.
-  ///
-  /// - Parameters:
-  ///   - scale: Scale value for asymmetric quantization.
-  ///   - zeroPoint: Zero point for asymmetric quantization.
-  init(scale: Float, zeroPoint: Int) {
-    self.scale = scale
-    self.zeroPoint = zeroPoint
-  }
-}
diff --git a/tensorflow/lite/experimental/swift/Sources/Tensor.swift b/tensorflow/lite/experimental/swift/Sources/Tensor.swift
deleted file mode 100644
index b738d87549..0000000000
--- a/tensorflow/lite/experimental/swift/Sources/Tensor.swift
+++ /dev/null
@@ -1,138 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import Foundation
-import TensorFlowLiteCAPI
-
-/// An input or output tensor in a TensorFlow Lite graph.
-public struct Tensor {
-
-  /// Name of the tensor.
-  public let name: String
-
-  /// Data type of the tensor.
-  public let dataType: TensorDataType
-
-  /// Shape of the tensor.
-  public let shape: TensorShape
-
-  /// Data in the input or output tensor.
-  public let data: Data
-
-  /// Quantization parameters for the tensor if using a quantized model.
-  public let quantizationParameters: QuantizationParameters?
-
-  /// Creates a new input or output tensor instance.
-  ///
-  /// - Parameters:
-  ///   - name: Name of the tensor.
-  ///   - dataType: Data type of the tensor.
-  ///   - data: Data in the input tensor.
-  ///   - quantizationParameters Quantization parameters for the tensor if using a quantized model.
-  ///       The default is `nil`.
-  init(
-    name: String,
-    dataType: TensorDataType,
-    shape: TensorShape,
-    data: Data,
-    quantizationParameters: QuantizationParameters? = nil
-  ) {
-    self.name = name
-    self.dataType = dataType
-    self.shape = shape
-    self.data = data
-    self.quantizationParameters = quantizationParameters
-  }
-}
-
-/// Supported TensorFlow Lite tensor data types.
-public enum TensorDataType: Equatable {
-  /// 32-bit single precision floating point tensor data type.
-  case float32
-  /// 8-bit unsigned integer tensor data type.
-  case uInt8
-  /// 16-bit signed integer tensor data type.
-  case int16
-  /// 32-bit signed integer tensor data type.
-  case int32
-  /// 64-bit signed integer tensor data type.
-  case int64
-  /// Boolean tensor data type.
-  case bool
-
-  /// Creates a new tensor data type from the given `TFL_Type` or `nil` if the data type is
-  /// unsupported or could not be determined because there was an error.
-  ///
-  /// - Parameter type: A data type supported by a tensor.
-  init?(type: TFL_Type) {
-    switch type {
-    case kTfLiteFloat32:
-      self = .float32
-    case kTfLiteUInt8:
-      self = .uInt8
-    case kTfLiteInt16:
-      self = .int16
-    case kTfLiteInt32:
-      self = .int32
-    case kTfLiteInt64:
-      self = .int64
-    case kTfLiteBool:
-      self = .bool
-    case kTfLiteNoType:
-      fallthrough
-    default:
-      return nil
-    }
-  }
-}
-
-/// The shape of a TensorFlow Lite tensor.
-public struct TensorShape {
-
-  /// The number of dimensions of the tensor.
-  public let rank: Int
-
-  /// Array of dimensions for the tensor.
-  public let dimensions: [Int]
-
-  /// Array of `Int32` dimensions for the tensor.
-  var int32Dimensions: [Int32] { return dimensions.map(Int32.init) }
-
-  /// Creates a new tensor shape instance with the given array of dimensions.
-  ///
-  /// - Parameters:
-  ///   - dimensions: Dimensions for the tensor.
-  public init(_ dimensions: [Int]) {
-    self.rank = dimensions.count
-    self.dimensions = dimensions
-  }
-
-  /// Creates a new tensor shape instance with the given elements representing the dimensions.
-  ///
-  /// - Parameters:
-  ///   - elements: Dimensions for the tensor.
-  public init(_ elements: Int...) {
-    self.init(elements)
-  }
-}
-
-extension TensorShape: ExpressibleByArrayLiteral {
-  /// Creates a new tensor shape instance with the given array literal representing the dimensions.
-  ///
-  /// - Parameters:
-  ///   - arrayLiteral: Dimensions for the tensor.
-  public init(arrayLiteral: Int...) {
-    self.init(arrayLiteral)
-  }
-}
diff --git a/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/Configs/TensorFlowLite.tulsigen b/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/Configs/TensorFlowLite.tulsigen
deleted file mode 100644
index 16bc6cbfe8..0000000000
--- a/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/Configs/TensorFlowLite.tulsigen
+++ /dev/null
@@ -1,57 +0,0 @@
-{
-  "sourceFilters" : [
-    "tensorflow/lite/experimental/c",
-    "tensorflow/lite/experimental/swift",
-    "tensorflow/lite/experimental/swift/Sources",
-    "tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp",
-    "tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj",
-    "tensorflow/lite/experimental/swift/Tests",
-  ],
-  "buildTargets" : [
-    "//tensorflow/lite/experimental/swift:TensorFlowLite",
-    "//tensorflow/lite/experimental/swift:TensorFlowLiteApp",
-    "//tensorflow/lite/experimental/swift:TensorFlowLiteTests",
-  ],
-  "projectName" : "TensorFlowLite",
-  "optionSet" : {
-    "LaunchActionPreActionScript" : {
-      "p" : "$(inherited)"
-    },
-    "BazelBuildStartupOptionsRelease" : {
-      "p" : "$(inherited)"
-    },
-    "BazelBuildOptionsRelease" : {
-      "p" : "$(inherited)"
-    },
-    "BazelBuildOptionsDebug" : {
-      "p" : "$(inherited)"
-    },
-    "EnvironmentVariables" : {
-      "p" : "$(inherited)"
-    },
-    "BuildActionPreActionScript" : {
-      "p" : "$(inherited)"
-    },
-    "CommandlineArguments" : {
-      "p" : "$(inherited)"
-    },
-    "TestActionPreActionScript" : {
-      "p" : "$(inherited)"
-    },
-    "BazelBuildStartupOptionsDebug" : {
-      "p" : "$(inherited)"
-    },
-    "BuildActionPostActionScript" : {
-      "p" : "$(inherited)"
-    },
-    "TestActionPostActionScript" : {
-      "p" : "$(inherited)"
-    },
-    "LaunchActionPostActionScript" : {
-      "p" : "$(inherited)"
-    }
-  },
-  "additionalFilePaths" : [
-    "tensorflow/lite/experimental/swift/BUILD"
-  ]
-}
diff --git a/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/project.tulsiconf b/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/project.tulsiconf
deleted file mode 100644
index 82ac8aa381..0000000000
--- a/tensorflow/lite/experimental/swift/TensorFlowLite.tulsiproj/project.tulsiconf
+++ /dev/null
@@ -1,14 +0,0 @@
-{
-  "configDefaults" : {
-    "optionSet" : {
-      "ProjectPrioritizesSwift" : {
-        "p" : "YES"
-      }
-    }
-  },
-  "projectName" : "TensorFlowLite",
-  "packages" : [
-    "tensorflow/lite/experimental/swift"
-  ],
-  "workspaceRoot" : "../../../../.."
-}
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp.xcodeproj/project.pbxproj b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp.xcodeproj/project.pbxproj
deleted file mode 100644
index fbbf9a1de2..0000000000
--- a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp.xcodeproj/project.pbxproj
+++ /dev/null
@@ -1,345 +0,0 @@
-// !$*UTF8*$!
-{
-	archiveVersion = 1;
-	classes = {
-	};
-	objectVersion = 50;
-	objects = {
-
-/* Begin PBXBuildFile section */
-		4A7304B421500B8400C90B21 /* Data+TensorFlowLite.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4A7304B321500B8300C90B21 /* Data+TensorFlowLite.swift */; };
-		4AA72B732146ED64006C3AEF /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4AA72B722146ED64006C3AEF /* AppDelegate.swift */; };
-		4AA72B752146ED64006C3AEF /* ViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4AA72B742146ED64006C3AEF /* ViewController.swift */; };
-		4AA72B782146ED64006C3AEF /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 4AA72B762146ED64006C3AEF /* Main.storyboard */; };
-		4AA72B7A2146ED66006C3AEF /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 4AA72B792146ED66006C3AEF /* Assets.xcassets */; };
-		4AA72B7D2146ED66006C3AEF /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 4AA72B7B2146ED66006C3AEF /* LaunchScreen.storyboard */; };
-		4ADDE0CE2176600E00FF07A2 /* Array+TensorFlowLite.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4ADDE0CD2176600900FF07A2 /* Array+TensorFlowLite.swift */; };
-/* End PBXBuildFile section */
-
-/* Begin PBXFileReference section */
-		4A7304B321500B8300C90B21 /* Data+TensorFlowLite.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "Data+TensorFlowLite.swift"; sourceTree = "<group>"; };
-		4AA72B6F2146ED64006C3AEF /* TensorFlowLiteApp.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = TensorFlowLiteApp.app; sourceTree = BUILT_PRODUCTS_DIR; };
-		4AA72B722146ED64006C3AEF /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = "<group>"; };
-		4AA72B742146ED64006C3AEF /* ViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ViewController.swift; sourceTree = "<group>"; };
-		4AA72B772146ED64006C3AEF /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/Main.storyboard; sourceTree = "<group>"; };
-		4AA72B792146ED66006C3AEF /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = "<group>"; };
-		4AA72B7C2146ED66006C3AEF /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = "<group>"; };
-		4AA72B7E2146ED66006C3AEF /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
-		4ADDE0CD2176600900FF07A2 /* Array+TensorFlowLite.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "Array+TensorFlowLite.swift"; sourceTree = "<group>"; };
-/* End PBXFileReference section */
-
-/* Begin PBXFrameworksBuildPhase section */
-		4AA72B6C2146ED64006C3AEF /* Frameworks */ = {
-			isa = PBXFrameworksBuildPhase;
-			buildActionMask = 2147483647;
-			files = (
-			);
-			runOnlyForDeploymentPostprocessing = 0;
-		};
-/* End PBXFrameworksBuildPhase section */
-
-/* Begin PBXGroup section */
-		4AA72B662146ED64006C3AEF = {
-			isa = PBXGroup;
-			children = (
-				4AA72B712146ED64006C3AEF /* TensorFlowLiteApp */,
-				4AA72B702146ED64006C3AEF /* Products */,
-			);
-			sourceTree = "<group>";
-		};
-		4AA72B702146ED64006C3AEF /* Products */ = {
-			isa = PBXGroup;
-			children = (
-				4AA72B6F2146ED64006C3AEF /* TensorFlowLiteApp.app */,
-			);
-			name = Products;
-			sourceTree = "<group>";
-		};
-		4AA72B712146ED64006C3AEF /* TensorFlowLiteApp */ = {
-			isa = PBXGroup;
-			children = (
-				4AA72B722146ED64006C3AEF /* AppDelegate.swift */,
-				4ADDE0CD2176600900FF07A2 /* Array+TensorFlowLite.swift */,
-				4A7304B321500B8300C90B21 /* Data+TensorFlowLite.swift */,
-				4AA72B742146ED64006C3AEF /* ViewController.swift */,
-				4AA72B762146ED64006C3AEF /* Main.storyboard */,
-				4AA72B792146ED66006C3AEF /* Assets.xcassets */,
-				4AA72B7B2146ED66006C3AEF /* LaunchScreen.storyboard */,
-				4AA72B7E2146ED66006C3AEF /* Info.plist */,
-			);
-			path = TensorFlowLiteApp;
-			sourceTree = "<group>";
-		};
-/* End PBXGroup section */
-
-/* Begin PBXNativeTarget section */
-		4AA72B6E2146ED64006C3AEF /* TensorFlowLiteApp */ = {
-			isa = PBXNativeTarget;
-			buildConfigurationList = 4AA72B812146ED66006C3AEF /* Build configuration list for PBXNativeTarget "TensorFlowLiteApp" */;
-			buildPhases = (
-				4AA72B6B2146ED64006C3AEF /* Sources */,
-				4AA72B6C2146ED64006C3AEF /* Frameworks */,
-				4AA72B6D2146ED64006C3AEF /* Resources */,
-			);
-			buildRules = (
-			);
-			dependencies = (
-			);
-			name = TensorFlowLiteApp;
-			productName = TensorFlowLiteApp;
-			productReference = 4AA72B6F2146ED64006C3AEF /* TensorFlowLiteApp.app */;
-			productType = "com.apple.product-type.application";
-		};
-/* End PBXNativeTarget section */
-
-/* Begin PBXProject section */
-		4AA72B672146ED64006C3AEF /* Project object */ = {
-			isa = PBXProject;
-			attributes = {
-				LastSwiftUpdateCheck = 0940;
-				LastUpgradeCheck = 0940;
-				ORGANIZATIONNAME = Google;
-				TargetAttributes = {
-					4AA72B6E2146ED64006C3AEF = {
-						CreatedOnToolsVersion = 9.4.1;
-					};
-				};
-			};
-			buildConfigurationList = 4AA72B6A2146ED64006C3AEF /* Build configuration list for PBXProject "TensorFlowLiteApp" */;
-			compatibilityVersion = "Xcode 9.3";
-			developmentRegion = en;
-			hasScannedForEncodings = 0;
-			knownRegions = (
-				en,
-				Base,
-			);
-			mainGroup = 4AA72B662146ED64006C3AEF;
-			productRefGroup = 4AA72B702146ED64006C3AEF /* Products */;
-			projectDirPath = "";
-			projectRoot = "";
-			targets = (
-				4AA72B6E2146ED64006C3AEF /* TensorFlowLiteApp */,
-			);
-		};
-/* End PBXProject section */
-
-/* Begin PBXResourcesBuildPhase section */
-		4AA72B6D2146ED64006C3AEF /* Resources */ = {
-			isa = PBXResourcesBuildPhase;
-			buildActionMask = 2147483647;
-			files = (
-				4AA72B7D2146ED66006C3AEF /* LaunchScreen.storyboard in Resources */,
-				4AA72B7A2146ED66006C3AEF /* Assets.xcassets in Resources */,
-				4AA72B782146ED64006C3AEF /* Main.storyboard in Resources */,
-			);
-			runOnlyForDeploymentPostprocessing = 0;
-		};
-/* End PBXResourcesBuildPhase section */
-
-/* Begin PBXSourcesBuildPhase section */
-		4AA72B6B2146ED64006C3AEF /* Sources */ = {
-			isa = PBXSourcesBuildPhase;
-			buildActionMask = 2147483647;
-			files = (
-				4AA72B732146ED64006C3AEF /* AppDelegate.swift in Sources */,
-				4ADDE0CE2176600E00FF07A2 /* Array+TensorFlowLite.swift in Sources */,
-				4A7304B421500B8400C90B21 /* Data+TensorFlowLite.swift in Sources */,
-				4AA72B752146ED64006C3AEF /* ViewController.swift in Sources */,
-			);
-			runOnlyForDeploymentPostprocessing = 0;
-		};
-/* End PBXSourcesBuildPhase section */
-
-/* Begin PBXVariantGroup section */
-		4AA72B762146ED64006C3AEF /* Main.storyboard */ = {
-			isa = PBXVariantGroup;
-			children = (
-				4AA72B772146ED64006C3AEF /* Base */,
-			);
-			name = Main.storyboard;
-			sourceTree = "<group>";
-		};
-		4AA72B7B2146ED66006C3AEF /* LaunchScreen.storyboard */ = {
-			isa = PBXVariantGroup;
-			children = (
-				4AA72B7C2146ED66006C3AEF /* Base */,
-			);
-			name = LaunchScreen.storyboard;
-			sourceTree = "<group>";
-		};
-/* End PBXVariantGroup section */
-
-/* Begin XCBuildConfiguration section */
-		4AA72B7F2146ED66006C3AEF /* Debug */ = {
-			isa = XCBuildConfiguration;
-			buildSettings = {
-				ALWAYS_SEARCH_USER_PATHS = NO;
-				CLANG_ANALYZER_NONNULL = YES;
-				CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
-				CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
-				CLANG_CXX_LIBRARY = "libc++";
-				CLANG_ENABLE_MODULES = YES;
-				CLANG_ENABLE_OBJC_ARC = YES;
-				CLANG_ENABLE_OBJC_WEAK = YES;
-				CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
-				CLANG_WARN_BOOL_CONVERSION = YES;
-				CLANG_WARN_COMMA = YES;
-				CLANG_WARN_CONSTANT_CONVERSION = YES;
-				CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
-				CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
-				CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
-				CLANG_WARN_EMPTY_BODY = YES;
-				CLANG_WARN_ENUM_CONVERSION = YES;
-				CLANG_WARN_INFINITE_RECURSION = YES;
-				CLANG_WARN_INT_CONVERSION = YES;
-				CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
-				CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
-				CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
-				CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
-				CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
-				CLANG_WARN_STRICT_PROTOTYPES = YES;
-				CLANG_WARN_SUSPICIOUS_MOVE = YES;
-				CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
-				CLANG_WARN_UNREACHABLE_CODE = YES;
-				CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
-				CODE_SIGN_IDENTITY = "iPhone Developer";
-				COPY_PHASE_STRIP = NO;
-				DEBUG_INFORMATION_FORMAT = dwarf;
-				ENABLE_STRICT_OBJC_MSGSEND = YES;
-				ENABLE_TESTABILITY = YES;
-				GCC_C_LANGUAGE_STANDARD = gnu11;
-				GCC_DYNAMIC_NO_PIC = NO;
-				GCC_NO_COMMON_BLOCKS = YES;
-				GCC_OPTIMIZATION_LEVEL = 0;
-				GCC_PREPROCESSOR_DEFINITIONS = (
-					"DEBUG=1",
-					"$(inherited)",
-				);
-				GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
-				GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
-				GCC_WARN_UNDECLARED_SELECTOR = YES;
-				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
-				GCC_WARN_UNUSED_FUNCTION = YES;
-				GCC_WARN_UNUSED_VARIABLE = YES;
-				IPHONEOS_DEPLOYMENT_TARGET = 11.4;
-				MTL_ENABLE_DEBUG_INFO = YES;
-				ONLY_ACTIVE_ARCH = YES;
-				SDKROOT = iphoneos;
-				SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG;
-				SWIFT_OPTIMIZATION_LEVEL = "-Onone";
-			};
-			name = Debug;
-		};
-		4AA72B802146ED66006C3AEF /* Release */ = {
-			isa = XCBuildConfiguration;
-			buildSettings = {
-				ALWAYS_SEARCH_USER_PATHS = NO;
-				CLANG_ANALYZER_NONNULL = YES;
-				CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
-				CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
-				CLANG_CXX_LIBRARY = "libc++";
-				CLANG_ENABLE_MODULES = YES;
-				CLANG_ENABLE_OBJC_ARC = YES;
-				CLANG_ENABLE_OBJC_WEAK = YES;
-				CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
-				CLANG_WARN_BOOL_CONVERSION = YES;
-				CLANG_WARN_COMMA = YES;
-				CLANG_WARN_CONSTANT_CONVERSION = YES;
-				CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
-				CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
-				CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
-				CLANG_WARN_EMPTY_BODY = YES;
-				CLANG_WARN_ENUM_CONVERSION = YES;
-				CLANG_WARN_INFINITE_RECURSION = YES;
-				CLANG_WARN_INT_CONVERSION = YES;
-				CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
-				CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
-				CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
-				CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
-				CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
-				CLANG_WARN_STRICT_PROTOTYPES = YES;
-				CLANG_WARN_SUSPICIOUS_MOVE = YES;
-				CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
-				CLANG_WARN_UNREACHABLE_CODE = YES;
-				CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
-				CODE_SIGN_IDENTITY = "iPhone Developer";
-				COPY_PHASE_STRIP = NO;
-				DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
-				ENABLE_NS_ASSERTIONS = NO;
-				ENABLE_STRICT_OBJC_MSGSEND = YES;
-				GCC_C_LANGUAGE_STANDARD = gnu11;
-				GCC_NO_COMMON_BLOCKS = YES;
-				GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
-				GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
-				GCC_WARN_UNDECLARED_SELECTOR = YES;
-				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
-				GCC_WARN_UNUSED_FUNCTION = YES;
-				GCC_WARN_UNUSED_VARIABLE = YES;
-				IPHONEOS_DEPLOYMENT_TARGET = 11.4;
-				MTL_ENABLE_DEBUG_INFO = NO;
-				SDKROOT = iphoneos;
-				SWIFT_COMPILATION_MODE = wholemodule;
-				SWIFT_OPTIMIZATION_LEVEL = "-O";
-				VALIDATE_PRODUCT = YES;
-			};
-			name = Release;
-		};
-		4AA72B822146ED66006C3AEF /* Debug */ = {
-			isa = XCBuildConfiguration;
-			buildSettings = {
-				ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
-				CODE_SIGN_STYLE = Automatic;
-				INFOPLIST_FILE = TensorFlowLiteApp/Info.plist;
-				LD_RUNPATH_SEARCH_PATHS = (
-					"$(inherited)",
-					"@executable_path/Frameworks",
-				);
-				PRODUCT_BUNDLE_IDENTIFIER = com.tensorflow.lite.swift.TensorFlowLite;
-				PRODUCT_NAME = "$(TARGET_NAME)";
-				SWIFT_VERSION = 4.0;
-				TARGETED_DEVICE_FAMILY = "1,2";
-			};
-			name = Debug;
-		};
-		4AA72B832146ED66006C3AEF /* Release */ = {
-			isa = XCBuildConfiguration;
-			buildSettings = {
-				ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
-				CODE_SIGN_STYLE = Automatic;
-				INFOPLIST_FILE = TensorFlowLiteApp/Info.plist;
-				LD_RUNPATH_SEARCH_PATHS = (
-					"$(inherited)",
-					"@executable_path/Frameworks",
-				);
-				PRODUCT_BUNDLE_IDENTIFIER = com.tensorflow.lite.swift.TensorFlowLite;
-				PRODUCT_NAME = "$(TARGET_NAME)";
-				SWIFT_VERSION = 4.0;
-				TARGETED_DEVICE_FAMILY = "1,2";
-			};
-			name = Release;
-		};
-/* End XCBuildConfiguration section */
-
-/* Begin XCConfigurationList section */
-		4AA72B6A2146ED64006C3AEF /* Build configuration list for PBXProject "TensorFlowLiteApp" */ = {
-			isa = XCConfigurationList;
-			buildConfigurations = (
-				4AA72B7F2146ED66006C3AEF /* Debug */,
-				4AA72B802146ED66006C3AEF /* Release */,
-			);
-			defaultConfigurationIsVisible = 0;
-			defaultConfigurationName = Release;
-		};
-		4AA72B812146ED66006C3AEF /* Build configuration list for PBXNativeTarget "TensorFlowLiteApp" */ = {
-			isa = XCConfigurationList;
-			buildConfigurations = (
-				4AA72B822146ED66006C3AEF /* Debug */,
-				4AA72B832146ED66006C3AEF /* Release */,
-			);
-			defaultConfigurationIsVisible = 0;
-			defaultConfigurationName = Release;
-		};
-/* End XCConfigurationList section */
-	};
-	rootObject = 4AA72B672146ED64006C3AEF /* Project object */;
-}
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/AppDelegate.swift b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/AppDelegate.swift
deleted file mode 100644
index ffa90a06ad..0000000000
--- a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/AppDelegate.swift
+++ /dev/null
@@ -1,24 +0,0 @@
-import UIKit
-
-@UIApplicationMain
-
-final class AppDelegate: UIResponder, UIApplicationDelegate {
-
-  /// The main window of the app.
-  var window: UIWindow?
-
-  func application(
-    _ application: UIApplication,
-    didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]? = nil
-  ) -> Bool {
-    return true
-  }
-}
-
-// MARK: - Extensions
-
-#if !swift(>=4.2)
-extension UIApplication {
-  typealias LaunchOptionsKey = UIApplicationLaunchOptionsKey
-}
-#endif  // !swift(>=4.2)
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Array+TensorFlowLite.swift b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Array+TensorFlowLite.swift
deleted file mode 100644
index 56df1ce659..0000000000
--- a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Array+TensorFlowLite.swift
+++ /dev/null
@@ -1,22 +0,0 @@
-import Foundation
-
-extension Array {
-  /// Creates a new array from the bytes of the given unsafe data.
-  ///
-  /// - Warning: The array's `Element` type must be trivial in that it can be copied bit for bit
-  ///     with no indirection or reference-counting operations; otherwise, copying the raw bytes in
-  ///     the `unsafeData`'s buffer to a new array returns an unsafe copy.
-  /// - Note: Returns `nil` if `unsafeData.count` is not a multiple of
-  ///     `MemoryLayout<Element>.stride`.
-  /// - Parameter unsafeData: The data containing the bytes to turn into an array.
-  init?(unsafeData: Data) {
-    guard unsafeData.count % MemoryLayout<Element>.stride == 0 else { return nil }
-    let elements = unsafeData.withUnsafeBytes {
-      UnsafeBufferPointer<Element>(
-        start: $0,
-        count: unsafeData.count / MemoryLayout<Element>.stride
-      )
-    }
-    self.init(elements)
-  }
-}
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/AppIcon.appiconset/Contents.json b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/AppIcon.appiconset/Contents.json
deleted file mode 100644
index d8db8d65fd..0000000000
--- a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/AppIcon.appiconset/Contents.json
+++ /dev/null
@@ -1,98 +0,0 @@
-{
-  "images" : [
-    {
-      "idiom" : "iphone",
-      "size" : "20x20",
-      "scale" : "2x"
-    },
-    {
-      "idiom" : "iphone",
-      "size" : "20x20",
-      "scale" : "3x"
-    },
-    {
-      "idiom" : "iphone",
-      "size" : "29x29",
-      "scale" : "2x"
-    },
-    {
-      "idiom" : "iphone",
-      "size" : "29x29",
-      "scale" : "3x"
-    },
-    {
-      "idiom" : "iphone",
-      "size" : "40x40",
-      "scale" : "2x"
-    },
-    {
-      "idiom" : "iphone",
-      "size" : "40x40",
-      "scale" : "3x"
-    },
-    {
-      "idiom" : "iphone",
-      "size" : "60x60",
-      "scale" : "2x"
-    },
-    {
-      "idiom" : "iphone",
-      "size" : "60x60",
-      "scale" : "3x"
-    },
-    {
-      "idiom" : "ipad",
-      "size" : "20x20",
-      "scale" : "1x"
-    },
-    {
-      "idiom" : "ipad",
-      "size" : "20x20",
-      "scale" : "2x"
-    },
-    {
-      "idiom" : "ipad",
-      "size" : "29x29",
-      "scale" : "1x"
-    },
-    {
-      "idiom" : "ipad",
-      "size" : "29x29",
-      "scale" : "2x"
-    },
-    {
-      "idiom" : "ipad",
-      "size" : "40x40",
-      "scale" : "1x"
-    },
-    {
-      "idiom" : "ipad",
-      "size" : "40x40",
-      "scale" : "2x"
-    },
-    {
-      "idiom" : "ipad",
-      "size" : "76x76",
-      "scale" : "1x"
-    },
-    {
-      "idiom" : "ipad",
-      "size" : "76x76",
-      "scale" : "2x"
-    },
-    {
-      "idiom" : "ipad",
-      "size" : "83.5x83.5",
-      "scale" : "2x"
-    },
-    {
-      "idiom" : "ios-marketing",
-      "size" : "1024x1024",
-      "scale" : "1x"
-    }
-  ],
-  "info" : {
-    "version" : 1,
-    "author" : "xcode"
-  }
-}
\ No newline at end of file
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/Contents.json b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/Contents.json
deleted file mode 100644
index da4a164c91..0000000000
--- a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Assets.xcassets/Contents.json
+++ /dev/null
@@ -1,6 +0,0 @@
-{
-  "info" : {
-    "version" : 1,
-    "author" : "xcode"
-  }
-}
\ No newline at end of file
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/LaunchScreen.storyboard b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/LaunchScreen.storyboard
deleted file mode 100644
index a07a1321be..0000000000
--- a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/LaunchScreen.storyboard
+++ /dev/null
@@ -1,44 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="14109" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" launchScreen="YES" useTraitCollections="YES" colorMatched="YES" initialViewController="01J-lp-oVM">
-    <device id="retina4_7" orientation="portrait">
-        <adaptation id="fullscreen"/>
-    </device>
-    <dependencies>
-        <deployment identifier="iOS"/>
-        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="14088"/>
-        <capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
-    </dependencies>
-    <scenes>
-        <!--View Controller-->
-        <scene sceneID="EHf-IW-A2E">
-            <objects>
-                <viewController id="01J-lp-oVM" sceneMemberID="viewController">
-                    <layoutGuides>
-                        <viewControllerLayoutGuide type="top" id="Llm-lL-Icb"/>
-                        <viewControllerLayoutGuide type="bottom" id="xb3-aO-Qok"/>
-                    </layoutGuides>
-                    <view key="view" contentMode="scaleToFill" id="Ze5-6b-2t3">
-                        <rect key="frame" x="0.0" y="0.0" width="375" height="667"/>
-                        <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
-                        <subviews>
-                            <label opaque="NO" userInteractionEnabled="NO" contentMode="left" horizontalHuggingPriority="251" verticalHuggingPriority="251" text="TensorFlowLite" textAlignment="center" lineBreakMode="tailTruncation" baselineAdjustment="alignBaselines" adjustsFontSizeToFit="NO" translatesAutoresizingMaskIntoConstraints="NO" id="3Gq-PV-hia">
-                                <rect key="frame" x="16" y="315" width="343" height="38.5"/>
-                                <fontDescription key="fontDescription" type="boldSystem" pointSize="32"/>
-                                <nil key="textColor"/>
-                                <nil key="highlightedColor"/>
-                            </label>
-                        </subviews>
-                        <color key="backgroundColor" red="1" green="1" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
-                        <constraints>
-                            <constraint firstItem="3Gq-PV-hia" firstAttribute="leading" secondItem="Ze5-6b-2t3" secondAttribute="leading" constant="16" id="aXL-9T-5Pf"/>
-                            <constraint firstItem="3Gq-PV-hia" firstAttribute="centerY" secondItem="Ze5-6b-2t3" secondAttribute="centerY" id="cDf-Go-1FR"/>
-                            <constraint firstAttribute="trailing" secondItem="3Gq-PV-hia" secondAttribute="trailing" constant="16" id="fB9-BX-A3B"/>
-                        </constraints>
-                    </view>
-                </viewController>
-                <placeholder placeholderIdentifier="IBFirstResponder" id="iYj-Kq-Ea1" userLabel="First Responder" sceneMemberID="firstResponder"/>
-            </objects>
-            <point key="canvasLocation" x="52" y="374.66266866566718"/>
-        </scene>
-    </scenes>
-</document>
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/Main.storyboard b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/Main.storyboard
deleted file mode 100644
index 10cae6e855..0000000000
--- a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Base.lproj/Main.storyboard
+++ /dev/null
@@ -1,95 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="14460.31" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" useTraitCollections="YES" colorMatched="YES" initialViewController="BYZ-38-t0r">
-    <device id="retina4_7" orientation="portrait">
-        <adaptation id="fullscreen"/>
-    </device>
-    <dependencies>
-        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="14460.20"/>
-        <capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
-    </dependencies>
-    <scenes>
-        <!--View Controller-->
-        <scene sceneID="tne-QT-ifu">
-            <objects>
-                <viewController storyboardIdentifier="viewController" useStoryboardIdentifierAsRestorationIdentifier="YES" id="BYZ-38-t0r" customClass="ViewController" customModule="TensorFlowLiteAppLib" sceneMemberID="viewController">
-                    <layoutGuides>
-                        <viewControllerLayoutGuide type="top" id="y3c-jy-aDJ"/>
-                        <viewControllerLayoutGuide type="bottom" id="wfy-db-euE"/>
-                    </layoutGuides>
-                    <view key="view" contentMode="scaleToFill" id="8bC-Xf-vdC">
-                        <rect key="frame" x="0.0" y="0.0" width="375" height="667"/>
-                        <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
-                        <subviews>
-                            <textView clipsSubviews="YES" multipleTouchEnabled="YES" contentMode="scaleToFill" editable="NO" selectable="NO" translatesAutoresizingMaskIntoConstraints="NO" id="7Mj-sL-hrd">
-                                <rect key="frame" x="0.0" y="367" width="375" height="300"/>
-                                <color key="backgroundColor" red="0.0" green="0.47843137250000001" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
-                                <constraints>
-                                    <constraint firstAttribute="height" constant="300" id="YUb-MC-D5w"/>
-                                </constraints>
-                                <color key="textColor" cocoaTouchSystemColor="tableCellGroupedBackgroundColor"/>
-                                <fontDescription key="fontDescription" type="system" pointSize="14"/>
-                                <textInputTraits key="textInputTraits" autocapitalizationType="sentences"/>
-                            </textView>
-                            <toolbar opaque="NO" clearsContextBeforeDrawing="NO" contentMode="scaleToFill" translatesAutoresizingMaskIntoConstraints="NO" id="Qwg-EP-bd6" userLabel="Bottom Toolbar">
-                                <rect key="frame" x="0.0" y="323" width="375" height="44"/>
-                                <constraints>
-                                    <constraint firstAttribute="height" constant="44" id="jhT-Q0-E9N"/>
-                                </constraints>
-                                <items>
-                                    <barButtonItem style="plain" systemItem="flexibleSpace" id="P3q-uA-YUa"/>
-                                    <barButtonItem title="Invoke Interpreter" id="A4J-Mg-nmd" userLabel="Invoke Button">
-                                        <connections>
-                                            <action selector="invokeInterpreter:" destination="BYZ-38-t0r" id="lZU-x7-PsJ"/>
-                                        </connections>
-                                    </barButtonItem>
-                                    <barButtonItem style="plain" systemItem="flexibleSpace" id="Qad-Pa-ySg"/>
-                                </items>
-                            </toolbar>
-                            <toolbar opaque="NO" clearsContextBeforeDrawing="NO" contentMode="scaleToFill" translatesAutoresizingMaskIntoConstraints="NO" id="Gkb-TR-PCB" userLabel="Top Toolbar">
-                                <rect key="frame" x="0.0" y="28" width="375" height="44"/>
-                                <constraints>
-                                    <constraint firstAttribute="height" constant="44" id="hSD-2q-fUE"/>
-                                </constraints>
-                                <items>
-                                    <barButtonItem style="plain" id="LKw-TX-bbH">
-                                        <segmentedControl key="customView" opaque="NO" contentMode="scaleToFill" contentHorizontalAlignment="left" contentVerticalAlignment="top" segmentControlStyle="bar" selectedSegmentIndex="0" id="rhA-nW-xzT">
-                                            <rect key="frame" x="16" y="7" width="343" height="30"/>
-                                            <autoresizingMask key="autoresizingMask" flexibleMaxX="YES" flexibleMaxY="YES"/>
-                                            <segments>
-                                                <segment title="Add"/>
-                                                <segment title="AddQuantized"/>
-                                                <segment title="MultiAdd"/>
-                                            </segments>
-                                            <connections>
-                                                <action selector="modelChanged:" destination="BYZ-38-t0r" eventType="valueChanged" id="YnG-Ov-B5D"/>
-                                            </connections>
-                                        </segmentedControl>
-                                    </barButtonItem>
-                                </items>
-                            </toolbar>
-                        </subviews>
-                        <color key="backgroundColor" red="1" green="1" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
-                        <constraints>
-                            <constraint firstAttribute="trailing" secondItem="Gkb-TR-PCB" secondAttribute="trailing" id="4Cr-Sf-I7n"/>
-                            <constraint firstItem="7Mj-sL-hrd" firstAttribute="bottom" secondItem="wfy-db-euE" secondAttribute="top" id="6ot-zD-sze"/>
-                            <constraint firstItem="7Mj-sL-hrd" firstAttribute="top" secondItem="Qwg-EP-bd6" secondAttribute="bottom" id="ELA-C6-NiG"/>
-                            <constraint firstAttribute="trailing" secondItem="7Mj-sL-hrd" secondAttribute="trailing" id="HDO-xr-mBl"/>
-                            <constraint firstItem="Gkb-TR-PCB" firstAttribute="leading" secondItem="8bC-Xf-vdC" secondAttribute="leading" id="Kmo-6K-gS4"/>
-                            <constraint firstItem="Qwg-EP-bd6" firstAttribute="leading" secondItem="8bC-Xf-vdC" secondAttribute="leading" id="hGu-lm-fMG"/>
-                            <constraint firstAttribute="trailing" secondItem="Qwg-EP-bd6" secondAttribute="trailing" id="iXR-LK-nTO"/>
-                            <constraint firstItem="7Mj-sL-hrd" firstAttribute="leading" secondItem="8bC-Xf-vdC" secondAttribute="leading" id="nr7-jW-ZYf"/>
-                            <constraint firstItem="Gkb-TR-PCB" firstAttribute="top" secondItem="y3c-jy-aDJ" secondAttribute="bottom" constant="8" id="uCF-VW-rR0"/>
-                        </constraints>
-                    </view>
-                    <connections>
-                        <outlet property="invokeButton" destination="A4J-Mg-nmd" id="UxZ-Ft-E45"/>
-                        <outlet property="modelControl" destination="rhA-nW-xzT" id="KKf-TT-BQ2"/>
-                        <outlet property="resultsTextView" destination="7Mj-sL-hrd" id="T4I-z4-tYA"/>
-                    </connections>
-                </viewController>
-                <placeholder placeholderIdentifier="IBFirstResponder" id="dkx-z0-nzr" sceneMemberID="firstResponder"/>
-            </objects>
-            <point key="canvasLocation" x="125.59999999999999" y="133.5832083958021"/>
-        </scene>
-    </scenes>
-</document>
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Data+TensorFlowLite.swift b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Data+TensorFlowLite.swift
deleted file mode 100644
index bc8a70c848..0000000000
--- a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Data+TensorFlowLite.swift
+++ /dev/null
@@ -1,13 +0,0 @@
-import Foundation
-
-extension Data {
-  /// Creates a new buffer by copying the buffer pointer of the given array.
-  ///
-  /// - Warning: The given array's element type `T` must be trivial in that it can be copied bit
-  ///     for bit with no indirection or reference-counting operations; otherwise, reinterpreting
-  ///     data from the resulting buffer has undefined behavior.
-  /// - Parameter array: An array with elements of type `T`.
-  init<T>(copyingBufferOf array: [T]) {
-    self = array.withUnsafeBufferPointer(Data.init)
-  }
-}
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Info.plist b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Info.plist
deleted file mode 100644
index 3ca3875f04..0000000000
--- a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/Info.plist
+++ /dev/null
@@ -1,46 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
-<plist version="1.0">
-<dict>
-	<key>CFBundleDevelopmentRegion</key>
-	<string>en</string>
-	<key>CFBundleExecutable</key>
-	<string>$(EXECUTABLE_NAME)</string>
-	<key>CFBundleIdentifier</key>
-	<string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
-	<key>CFBundleInfoDictionaryVersion</key>
-	<string>6.0</string>
-	<key>CFBundleName</key>
-	<string>$(PRODUCT_NAME)</string>
-	<key>CFBundlePackageType</key>
-	<string>APPL</string>
-	<key>CFBundleShortVersionString</key>
-	<string>1.0</string>
-	<key>CFBundleVersion</key>
-	<string>0.0.1</string>
-	<key>LSRequiresIPhoneOS</key>
-	<true/>
-	<key>NSCameraUsageDescription</key>
-	<string>NSCameraUsageDescription</string>
-	<key>NSPhotoLibraryUsageDescription</key>
-	<string>Select a photo to detect objects in.</string>
-	<key>UILaunchStoryboardName</key>
-	<string>LaunchScreen</string>
-	<key>UIMainStoryboardFile</key>
-	<string>Main</string>
-	<key>UIRequiredDeviceCapabilities</key>
-	<array>
-		<string>armv7</string>
-	</array>
-	<key>UISupportedInterfaceOrientations</key>
-	<array>
-		<string>UIInterfaceOrientationPortrait</string>
-		<string>UIInterfaceOrientationPortraitUpsideDown</string>
-	</array>
-	<key>UISupportedInterfaceOrientations~ipad</key>
-	<array>
-		<string>UIInterfaceOrientationPortrait</string>
-		<string>UIInterfaceOrientationPortraitUpsideDown</string>
-	</array>
-</dict>
-</plist>
diff --git a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/ViewController.swift b/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/ViewController.swift
deleted file mode 100644
index 73c74fd19c..0000000000
--- a/tensorflow/lite/experimental/swift/TestApps/TensorFlowLiteApp/TensorFlowLiteApp/ViewController.swift
+++ /dev/null
@@ -1,299 +0,0 @@
-import TensorFlowLite
-import UIKit
-
-class ViewController: UIViewController {
-
-  // MARK: - Properties
-
-  /// TensorFlowLite interpreter object for performing inference from a given model.
-  private var interpreter: Interpreter?
-
-  /// Serial dispatch queue for managing `Interpreter` calls.
-  private let interpreterQueue = DispatchQueue(
-    label: Constant.dispatchQueueLabel,
-    qos: .userInitiated
-  )
-
-  /// The currently selected model.
-  private var currentModel: Model {
-    guard let currentModel = Model(rawValue: modelControl.selectedSegmentIndex) else {
-      preconditionFailure("Invalid model for selected segment index.")
-    }
-    return currentModel
-  }
-
-  /// A description of the current model.
-  private var modelDescription: String {
-    guard let interpreter = interpreter else { return "" }
-    let inputCount = interpreter.inputTensorCount
-    let outputCount = interpreter.outputTensorCount
-    let inputTensors = (0..<inputCount).map { index in
-      var tensorInfo = "  Input \(index + 1): "
-      do {
-        let tensor = try interpreter.input(at: index)
-        tensorInfo += "\(tensor)"
-      } catch let error {
-        tensorInfo += "\(error.localizedDescription)"
-      }
-      return tensorInfo
-    }.joined(separator: "\n")
-    let outputTensors = (0..<outputCount).map { index in
-      var tensorInfo = "  Output \(index + 1): "
-      do {
-        let tensor = try interpreter.output(at: index)
-        tensorInfo += "\(tensor)"
-      } catch let error {
-        tensorInfo += "\(error.localizedDescription)"
-      }
-      return tensorInfo
-    }.joined(separator: "\n")
-    return "Model Description:\n" +
-             "  Input Tensor Count = \(inputCount)\n\(inputTensors)\n\n" +
-             "  Output Tensor Count = \(outputCount)\n\(outputTensors)"
-  }
-
-  // MARK: - IBOutlets
-
-  /// A segmented control for changing models. See the `Model` enum for available models.
-  @IBOutlet private var modelControl: UISegmentedControl!
-
-  @IBOutlet private var resultsTextView: UITextView!
-  @IBOutlet private var invokeButton: UIBarButtonItem!
-
-  // MARK: - UIViewController
-
-  override func viewDidLoad() {
-    super.viewDidLoad()
-
-    invokeButton.isEnabled = false
-    loadModel()
-  }
-
-  // MARK: - IBActions
-
-  @IBAction func modelChanged(_ sender: Any) {
-    invokeButton.isEnabled = false
-    updateResultsText("Switched to the \(currentModel.description) model.")
-    loadModel()
-  }
-
-  @IBAction func invokeInterpreter(_ sender: Any) {
-    switch currentModel {
-    case .add:
-      invokeAdd()
-    case .addQuantized:
-      invokeAddQuantized()
-    case .multiAdd:
-      invokeMultiAdd()
-    }
-  }
-
-  // MARK: - Private
-
-  private func loadModel() {
-    let fileInfo = currentModel.fileInfo
-    guard let modelPath = Bundle.main.path(forResource: fileInfo.name, ofType: fileInfo.extension)
-    else {
-      updateResultsText("Failed to load the \(currentModel.description) model.")
-      return
-    }
-    setUpInterpreter(withModelPath: modelPath)
-  }
-
-  private func setUpInterpreter(withModelPath modelPath: String) {
-    interpreterQueue.async {
-      do {
-        var options = InterpreterOptions()
-        options.isErrorLoggingEnabled = true
-        self.interpreter = try Interpreter(modelPath: modelPath, options: options)
-      } catch let error {
-        self.updateResultsText(
-          "Failed to create the interpreter with error: \(error.localizedDescription)"
-        )
-        return
-      }
-      safeDispatchOnMain { self.invokeButton.isEnabled = true }
-    }
-  }
-
-  private func invokeAdd() {
-    interpreterQueue.async {
-      guard let interpreter = self.interpreter else {
-        self.updateResultsText(Constant.nilInterpreterErrorMessage)
-        return
-      }
-      do {
-        try interpreter.resizeInput(at: 0, to: [2])
-        try interpreter.allocateTensors()
-        let input: [Float32] = [1, 3]
-        let resultsText = self.modelDescription + "\n\n" +
-          "Performing 2 add operations on input \(input.description) equals: "
-        self.updateResultsText(resultsText)
-        let data = Data(copyingBufferOf: input)
-        try interpreter.copy(data, toInputAt: 0)
-        try interpreter.invoke()
-        let outputTensor = try interpreter.output(at: 0)
-        let results: () -> String = {
-          guard let results = [Float32](unsafeData: outputTensor.data) else { return "No results." }
-          return resultsText + results.description
-        }
-        self.updateResultsText(results())
-      } catch let error {
-        self.updateResultsText(
-          "Failed to invoke the interpreter with error: \(error.localizedDescription)"
-        )
-        return
-      }
-    }
-  }
-
-  private func invokeAddQuantized() {
-    interpreterQueue.async {
-      guard let interpreter = self.interpreter else {
-        self.updateResultsText(Constant.nilInterpreterErrorMessage)
-        return
-      }
-      do {
-        try interpreter.resizeInput(at: 0, to: [2])
-        try interpreter.allocateTensors()
-        let input: [UInt8] = [1, 3]
-        let resultsText = self.modelDescription + "\n\n" +
-          "Performing 2 add operations on quantized input \(input.description) equals: "
-        self.updateResultsText(resultsText)
-        let data = Data(input)
-        try interpreter.copy(data, toInputAt: 0)
-        try interpreter.invoke()
-        let outputTensor = try interpreter.output(at: 0)
-        let results: () -> String = {
-          guard let quantizationParameters = outputTensor.quantizationParameters else {
-            return "No results."
-          }
-          let quantizedResults = [UInt8](outputTensor.data)
-          let dequantizedResults = quantizedResults.map {
-            quantizationParameters.scale * Float(Int($0) - quantizationParameters.zeroPoint)
-          }
-          return resultsText + quantizedResults.description +
-                   ", dequantized results: " + dequantizedResults.description
-        }
-        self.updateResultsText(results())
-      } catch let error {
-        self.updateResultsText(
-          "Failed to invoke the interpreter with error: \(error.localizedDescription)"
-        )
-        return
-      }
-    }
-  }
-
-  private func invokeMultiAdd() {
-    interpreterQueue.async {
-      guard let interpreter = self.interpreter else {
-        self.updateResultsText(Constant.nilInterpreterErrorMessage)
-        return
-      }
-      do {
-        let shape = TensorShape(2)
-        try (0..<interpreter.inputTensorCount).forEach { index in
-          try interpreter.resizeInput(at: index, to: shape)
-        }
-        try interpreter.allocateTensors()
-        let inputs = try (0..<interpreter.inputTensorCount).map { index -> [Float32] in
-          let input = [Float32(index + 1), Float32(index + 2)]
-          let data = Data(copyingBufferOf: input)
-          try interpreter.copy(data, toInputAt: index)
-          return input
-        }
-        let resultsText = self.modelDescription + "\n\n" +
-          "Performing 3 add operations on inputs \(inputs.description) equals: "
-        self.updateResultsText(resultsText)
-        try interpreter.invoke()
-        let results = try (0..<interpreter.outputTensorCount).map { index -> [Float32] in
-          let tensor = try interpreter.output(at: index)
-          return [Float32](unsafeData: tensor.data) ?? []
-        }
-        self.updateResultsText(resultsText + results.description)
-      } catch let error {
-        self.updateResultsText(
-          "Failed to invoke the interpreter with error: \(error.localizedDescription)"
-        )
-        return
-      }
-    }
-  }
-
-  private func updateResultsText(_ text: String? = nil) {
-    safeDispatchOnMain { self.resultsTextView.text = text }
-  }
-}
-
-// MARK: - Constants
-
-private enum Constant {
-  static let dispatchQueueLabel = "TensorFlowLiteInterpreterQueue"
-  static let nilInterpreterErrorMessage =
-    "Failed to invoke the interpreter because the interpreter was nil."
-}
-
-/// Models that can be loaded by the TensorFlow Lite `Interpreter`.
-private enum Model: Int, CustomStringConvertible {
-  /// A float model that performs two add operations on one input tensor and returns the result in
-  /// one output tensor.
-  case add = 0
-  /// A quantized model that performs two add operations on one input tensor and returns the result
-  /// in one output tensor.
-  case addQuantized = 1
-  /// A float model that performs three add operations on four input tensors and returns the results
-  /// in 2 output tensors.
-  case multiAdd = 2
-
-  var fileInfo: (name: String, extension: String) {
-    switch self {
-    case .add:
-      return Add.fileInfo
-    case .addQuantized:
-      return AddQuantized.fileInfo
-    case .multiAdd:
-      return MultiAdd.fileInfo
-    }
-  }
-
-  // MARK: - CustomStringConvertible
-
-  var description: String {
-    switch self {
-    case .add:
-      return Add.name
-    case .addQuantized:
-      return AddQuantized.name
-    case .multiAdd:
-      return MultiAdd.name
-    }
-  }
-}
-
-/// Values for the `Add` model.
-private enum Add {
-  static let name = "Add"
-  static let fileInfo = (name: "add", extension: "bin")
-}
-
-/// Values for the `AddQuantized` model.
-private enum AddQuantized {
-  static let name = "AddQuantized"
-  static let fileInfo = (name: "add_quantized", extension: "bin")
-}
-
-/// Values for the `MultiAdd` model.
-private enum MultiAdd {
-  static let name = "MultiAdd"
-  static let fileInfo = (name: "multi_add", extension: "bin")
-}
-
-// MARK: - Fileprivate
-
-/// Safely dispatches the given block on the main queue. If the current thread is `main`, the block
-/// is executed synchronously; otherwise, the block is executed asynchronously on the main thread.
-fileprivate func safeDispatchOnMain(_ block: @escaping () -> Void) {
-  if Thread.isMainThread { block(); return }
-  DispatchQueue.main.async { block() }
-}
diff --git a/tensorflow/lite/experimental/swift/Tests/InterpreterOptionsTests.swift b/tensorflow/lite/experimental/swift/Tests/InterpreterOptionsTests.swift
deleted file mode 100644
index 54b4f59b28..0000000000
--- a/tensorflow/lite/experimental/swift/Tests/InterpreterOptionsTests.swift
+++ /dev/null
@@ -1,54 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-@testable import TensorFlowLite
-import XCTest
-
-class InterpreterOptionsTests: XCTestCase {
-
-  func testInterpreterOptions_InitWithDefaultValues() {
-    let options = InterpreterOptions()
-    XCTAssertNil(options.threadCount)
-    XCTAssertFalse(options.isErrorLoggingEnabled)
-  }
-
-  func testInterpreterOptions_InitWithCustomValues() {
-    var options = InterpreterOptions()
-    options.threadCount = 2
-    XCTAssertEqual(options.threadCount, 2)
-    options.isErrorLoggingEnabled = true
-    XCTAssertTrue(options.isErrorLoggingEnabled)
-  }
-
-  func testInterpreterOptions_Equatable() {
-    var options1 = InterpreterOptions()
-    var options2 = InterpreterOptions()
-    XCTAssertEqual(options1, options2)
-
-    options1.threadCount = 2
-    options2.threadCount = 2
-    XCTAssertEqual(options1, options2)
-
-    options2.threadCount = 3
-    XCTAssertNotEqual(options1, options2)
-    options2.threadCount = 2
-
-    options1.isErrorLoggingEnabled = true
-    options2.isErrorLoggingEnabled = true
-    XCTAssertEqual(options1, options2)
-
-    options2.isErrorLoggingEnabled = false
-    XCTAssertNotEqual(options1, options2)
-  }
-}
diff --git a/tensorflow/lite/experimental/swift/Tests/InterpreterTests.swift b/tensorflow/lite/experimental/swift/Tests/InterpreterTests.swift
deleted file mode 100644
index e98da5f951..0000000000
--- a/tensorflow/lite/experimental/swift/Tests/InterpreterTests.swift
+++ /dev/null
@@ -1,315 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-@testable import TensorFlowLite
-import XCTest
-
-class InterpreterTests: XCTestCase {
-
-  var interpreter: Interpreter!
-
-  override func setUp() {
-    super.setUp()
-
-    interpreter = try! Interpreter(modelPath: AddModel.path)
-  }
-
-  override func tearDown() {
-    interpreter = nil
-
-    super.tearDown()
-  }
-
-  func testInterpreter_InitWithModelPath() {
-    XCTAssertNoThrow(try Interpreter(modelPath: AddModel.path))
-  }
-
-  func testInterpreter_Init_ThrowsFailedToLoadModel() {
-    XCTAssertThrowsError(try Interpreter(modelPath: "/invalid/path")) { error in
-      self.assertEqualErrors(actual: error, expected: .failedToLoadModel)
-    }
-  }
-
-  func testInterpreter_InitWithModelPathAndOptions() {
-    var options = InterpreterOptions()
-    options.threadCount = 2
-    XCTAssertNoThrow(try Interpreter(modelPath: AddModel.path, options: options))
-  }
-
-  func testInterpreter_InputTensorCount() {
-    XCTAssertEqual(interpreter.inputTensorCount, AddModel.inputTensorCount)
-  }
-
-  func testInterpreter_OutputTensorCount() {
-    XCTAssertEqual(interpreter.outputTensorCount, AddModel.outputTensorCount)
-  }
-
-  func testInterpreter_Invoke() throws {
-    try interpreter.allocateTensors()
-    XCTAssertNoThrow(try interpreter.invoke())
-  }
-
-  func testInterpreter_Invoke_ThrowsAllocateTensorsRequired_ModelNotReady() {
-    XCTAssertThrowsError(try interpreter.invoke()) { error in
-      self.assertEqualErrors(actual: error, expected: .allocateTensorsRequired)
-    }
-  }
-
-  func testInterpreter_InputTensorAtIndex() throws {
-    try setUpAddModelInputTensor()
-    let inputTensor = try interpreter.input(at: AddModel.validIndex)
-    XCTAssertEqual(inputTensor, AddModel.inputTensor)
-  }
-
-  func testInterpreter_InputTensorAtIndex_QuantizedModel() throws {
-    interpreter = try Interpreter(modelPath: AddQuantizedModel.path)
-    try setUpAddQuantizedModelInputTensor()
-    let inputTensor = try interpreter.input(at: AddQuantizedModel.inputOutputIndex)
-    XCTAssertEqual(inputTensor, AddQuantizedModel.inputTensor)
-  }
-
-  func testInterpreter_InputTensorAtIndex_ThrowsInvalidIndex() throws {
-    try interpreter.allocateTensors()
-    XCTAssertThrowsError(try interpreter.input(at: AddModel.invalidIndex)) { error in
-      let maxIndex = AddModel.inputTensorCount - 1
-      self.assertEqualErrors(
-        actual: error,
-        expected: .invalidTensorIndex(index: AddModel.invalidIndex, maxIndex: maxIndex)
-      )
-    }
-  }
-
-  func testInterpreter_InputTensorAtIndex_ThrowsAllocateTensorsRequired() {
-    XCTAssertThrowsError(try interpreter.input(at: AddModel.validIndex)) { error in
-      self.assertEqualErrors(actual: error, expected: .allocateTensorsRequired)
-    }
-  }
-
-  func testInterpreter_OutputTensorAtIndex() throws {
-    try setUpAddModelInputTensor()
-    try interpreter.invoke()
-    let outputTensor = try interpreter.output(at: AddModel.validIndex)
-    XCTAssertEqual(outputTensor, AddModel.outputTensor)
-    let expectedResults = [Float32](unsafeData: outputTensor.data)
-    XCTAssertEqual(expectedResults, AddModel.results)
-  }
-
-  func testInterpreter_OutputTensorAtIndex_QuantizedModel() throws {
-    interpreter = try Interpreter(modelPath: AddQuantizedModel.path)
-    try setUpAddQuantizedModelInputTensor()
-    try interpreter.invoke()
-    let outputTensor = try interpreter.output(at: AddQuantizedModel.inputOutputIndex)
-    XCTAssertEqual(outputTensor, AddQuantizedModel.outputTensor)
-    let expectedResults = [UInt8](outputTensor.data)
-    XCTAssertEqual(expectedResults, AddQuantizedModel.results)
-  }
-
-  func testInterpreter_OutputTensorAtIndex_ThrowsInvalidIndex() throws {
-    try interpreter.allocateTensors()
-    try interpreter.invoke()
-    XCTAssertThrowsError(try interpreter.output(at: AddModel.invalidIndex)) { error in
-      let maxIndex = AddModel.outputTensorCount - 1
-      self.assertEqualErrors(
-        actual: error,
-        expected: .invalidTensorIndex(index: AddModel.invalidIndex, maxIndex: maxIndex)
-      )
-    }
-  }
-
-  func testInterpreter_OutputTensorAtIndex_ThrowsInvokeInterpreterRequired() {
-    XCTAssertThrowsError(try interpreter.output(at: AddModel.validIndex)) { error in
-      self.assertEqualErrors(actual: error, expected: .invokeInterpreterRequired)
-    }
-  }
-
-  func testInterpreter_ResizeInputTensorAtIndexToShape() {
-    XCTAssertNoThrow(try interpreter.resizeInput(at: AddModel.validIndex, to: [2, 2, 3]))
-    XCTAssertNoThrow(try interpreter.allocateTensors())
-  }
-
-  func testInterpreter_ResizeInputTensorAtIndexToShape_ThrowsInvalidIndex() {
-    XCTAssertThrowsError(try interpreter.resizeInput(
-      at: AddModel.invalidIndex,
-      to: [2, 2, 3]
-    )) { error in
-      let maxIndex = AddModel.inputTensorCount - 1
-      self.assertEqualErrors(
-        actual: error,
-        expected: .invalidTensorIndex(index: AddModel.invalidIndex, maxIndex: maxIndex)
-      )
-    }
-  }
-
-  func testInterpreter_CopyDataToInputTensorAtIndex() throws {
-    try interpreter.resizeInput(at: AddModel.validIndex, to: AddModel.shape)
-    try interpreter.allocateTensors()
-    let inputTensor = try interpreter.copy(AddModel.inputData, toInputAt: AddModel.validIndex)
-    XCTAssertEqual(inputTensor.data, AddModel.inputData)
-  }
-
-  func testInterpreter_CopyDataToInputTensorAtIndex_ThrowsInvalidIndex() {
-    XCTAssertThrowsError(try interpreter.copy(
-      AddModel.inputData,
-      toInputAt: AddModel.invalidIndex
-    )) { error in
-      let maxIndex = AddModel.inputTensorCount - 1
-      self.assertEqualErrors(
-        actual: error,
-        expected: .invalidTensorIndex(index: AddModel.invalidIndex, maxIndex: maxIndex)
-      )
-    }
-  }
-
-  func testInterpreter_CopyDataToInputTensorAtIndex_ThrowsInvalidDataCount() throws {
-    try interpreter.resizeInput(at: AddModel.validIndex, to: AddModel.shape)
-    try interpreter.allocateTensors()
-    let invalidData = Data(count: AddModel.dataCount - 1)
-    XCTAssertThrowsError(try interpreter.copy(
-      invalidData,
-      toInputAt: AddModel.validIndex
-    )) { error in
-      self.assertEqualErrors(
-        actual: error,
-        expected: .invalidTensorDataCount(provided: invalidData.count, required: AddModel.dataCount)
-      )
-    }
-  }
-
-  func testInterpreter_AllocateTensors() {
-    XCTAssertNoThrow(try interpreter.allocateTensors())
-  }
-
-  // MARK: - Private
-
-  private func setUpAddModelInputTensor() throws {
-    precondition(interpreter != nil)
-    try interpreter.resizeInput(at: AddModel.validIndex, to: AddModel.shape)
-    try interpreter.allocateTensors()
-    try interpreter.copy(AddModel.inputData, toInputAt: AddModel.validIndex)
-  }
-
-  private func setUpAddQuantizedModelInputTensor() throws {
-    precondition(interpreter != nil)
-    try interpreter.resizeInput(at: AddQuantizedModel.inputOutputIndex, to: AddQuantizedModel.shape)
-    try interpreter.allocateTensors()
-    try interpreter.copy(AddQuantizedModel.inputData, toInputAt: AddQuantizedModel.inputOutputIndex)
-  }
-
-  private func assertEqualErrors(actual: Error, expected: InterpreterError) {
-    guard let actual = actual as? InterpreterError else {
-      XCTFail("Actual error should be of type InterpreterError.")
-      return
-    }
-    XCTAssertEqual(actual, expected)
-  }
-}
-
-// MARK: - Constants
-
-/// Values for the `add.bin` model.
-private enum AddModel {
-  static let info = (name: "add", extension: "bin")
-  static let inputTensorCount = 1
-  static let outputTensorCount = 1
-  static let invalidIndex = 1
-  static let validIndex = 0
-  static let shape: TensorShape = [2]
-  static let dataCount = inputData.count
-  static let inputData = Data(copyingBufferOf: [Float32(1.0), Float32(3.0)])
-  static let outputData = Data(copyingBufferOf: [Float32(3.0), Float32(9.0)])
-  static let results = [Float32(3.0), Float32(9.0)]
-
-  static let inputTensor = Tensor(
-    name: "input",
-    dataType: .float32,
-    shape: shape,
-    data: inputData
-  )
-  static let outputTensor = Tensor(
-    name: "output",
-    dataType: .float32,
-    shape: shape,
-    data: outputData
-  )
-
-  static var path: String = {
-    let bundle = Bundle(for: InterpreterTests.self)
-    guard let path = bundle.path(forResource: info.name, ofType: info.extension) else { return "" }
-    return path
-  }()
-}
-
-/// Values for the `add_quantized.bin` model.
-private enum AddQuantizedModel {
-  static let info = (name: "add_quantized", extension: "bin")
-  static let inputOutputIndex = 0
-  static let shape: TensorShape = [2]
-  static let inputData = Data([1, 3])
-  static let outputData = Data([3, 9])
-  static let quantizationParameters = QuantizationParameters(scale: 0.003922, zeroPoint: 0)
-  static let results: [UInt8] = [3, 9]
-
-  static let inputTensor = Tensor(
-    name: "input",
-    dataType: .uInt8,
-    shape: shape,
-    data: inputData,
-    quantizationParameters: quantizationParameters
-  )
-  static let outputTensor = Tensor(
-    name: "output",
-    dataType: .uInt8,
-    shape: shape,
-    data: outputData,
-    quantizationParameters: quantizationParameters
-  )
-
-  static var path: String = {
-    let bundle = Bundle(for: InterpreterTests.self)
-    guard let path = bundle.path(forResource: info.name, ofType: info.extension) else { return "" }
-    return path
-  }()
-}
-
-// MARK: - Extensions
-
-extension Array {
-  /// Creates a new array from the bytes of the given unsafe data.
-  ///
-  /// - Note: Returns `nil` if `unsafeData.count` is not a multiple of
-  ///     `MemoryLayout<Element>.stride`.
-  /// - Parameter unsafeData: The data containing the bytes to turn into an array.
-  init?(unsafeData: Data) {
-    guard unsafeData.count % MemoryLayout<Element>.stride == 0 else { return nil }
-    let elements = unsafeData.withUnsafeBytes {
-      UnsafeBufferPointer<Element>(
-        start: $0,
-        count: unsafeData.count / MemoryLayout<Element>.stride
-      )
-    }
-    self.init(elements)
-  }
-}
-
-extension Data {
-  /// Creates a new buffer by copying the buffer pointer of the given array.
-  ///
-  /// - Warning: The given array's element type `T` must be trivial in that it can be copied bit
-  ///     for bit with no indirection or reference-counting operations; otherwise, reinterpreting
-  ///     data from the resulting buffer has undefined behavior.
-  /// - Parameter array: An array with elements of type `T`.
-  init<T>(copyingBufferOf array: [T]) {
-    self = array.withUnsafeBufferPointer(Data.init)
-  }
-}
diff --git a/tensorflow/lite/experimental/swift/Tests/ModelTests.swift b/tensorflow/lite/experimental/swift/Tests/ModelTests.swift
deleted file mode 100644
index 025db18906..0000000000
--- a/tensorflow/lite/experimental/swift/Tests/ModelTests.swift
+++ /dev/null
@@ -1,59 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-@testable import TensorFlowLite
-import XCTest
-
-class ModelTests: XCTestCase {
-
-  var modelPath: String!
-
-  override func setUp() {
-    super.setUp()
-
-    let bundle = Bundle(for: type(of: self))
-    guard let modelPath = bundle.path(
-            forResource: Constant.modelInfo.name,
-            ofType: Constant.modelInfo.extension)
-    else {
-      XCTFail("Failed to get the model file path.")
-      return
-    }
-    self.modelPath = modelPath
-  }
-
-  override func tearDown() {
-    modelPath = nil
-
-    super.tearDown()
-  }
-
-  func testModel_InitWithFilePath() {
-    XCTAssertNotNil(Model(filePath: modelPath))
-  }
-
-  func testModel_InitWithEmptyFilePath_FailsInitialization() {
-    XCTAssertNil(Model(filePath: ""))
-  }
-
-  func testModel_InitWithInvalidFilePath_FailsInitialization() {
-    XCTAssertNil(Model(filePath: "invalid/path"))
-  }
-}
-
-// MARK: - Constants
-
-private enum Constant {
-  static let modelInfo = (name: "add", extension: "bin")
-}
diff --git a/tensorflow/lite/experimental/swift/Tests/QuantizationParametersTests.swift b/tensorflow/lite/experimental/swift/Tests/QuantizationParametersTests.swift
deleted file mode 100644
index 65648c2698..0000000000
--- a/tensorflow/lite/experimental/swift/Tests/QuantizationParametersTests.swift
+++ /dev/null
@@ -1,43 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-@testable import TensorFlowLite
-import XCTest
-
-class QuantizationParametersTests: XCTestCase {
-
-  func testQuantizationParameters_InitWithCustomValues() {
-    let parameters = QuantizationParameters(scale: 0.5, zeroPoint: 1)
-    XCTAssertEqual(parameters.scale, 0.5)
-    XCTAssertEqual(parameters.zeroPoint, 1)
-  }
-
-  func testQuantizationParameters_Equatable() {
-    let parameters1 = QuantizationParameters(scale: 0.5, zeroPoint: 1)
-    let parameters2 = QuantizationParameters(scale: 0.5, zeroPoint: 1)
-    XCTAssertEqual(parameters1, parameters2)
-
-    let parameters3 = QuantizationParameters(scale: 0.4, zeroPoint: 1)
-    XCTAssertNotEqual(parameters1, parameters3)
-    XCTAssertNotEqual(parameters2, parameters3)
-  }
-}
-
-// MARK: - Extensions
-
-extension QuantizationParameters: Equatable {
-  public static func == (lhs: QuantizationParameters, rhs: QuantizationParameters) -> Bool {
-    return lhs.scale == rhs.scale && lhs.zeroPoint == rhs.zeroPoint
-  }
-}
diff --git a/tensorflow/lite/experimental/swift/Tests/TensorTests.swift b/tensorflow/lite/experimental/swift/Tests/TensorTests.swift
deleted file mode 100644
index 4540043a16..0000000000
--- a/tensorflow/lite/experimental/swift/Tests/TensorTests.swift
+++ /dev/null
@@ -1,83 +0,0 @@
-// Copyright 2018 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at:
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-@testable import TensorFlowLite
-import XCTest
-
-class TensorTests: XCTestCase {
-
-  // MARK: - Tensor
-
-  func testTensor_Init() {
-    let name = "InputTensor"
-    let dataType: TensorDataType = .uInt8
-    let shape = TensorShape(Constant.dimensions)
-    guard let data = name.data(using: .utf8) else { XCTFail("Data should not be nil."); return }
-    let quantizationParameters = QuantizationParameters(scale: 0.5, zeroPoint: 1)
-    let inputTensor = Tensor(
-      name: name,
-      dataType: dataType,
-      shape: shape,
-      data: data,
-      quantizationParameters: quantizationParameters
-    )
-    XCTAssertEqual(inputTensor.name, name)
-    XCTAssertEqual(inputTensor.dataType, dataType)
-    XCTAssertEqual(inputTensor.shape, shape)
-    XCTAssertEqual(inputTensor.data, data)
-    XCTAssertEqual(inputTensor.quantizationParameters, quantizationParameters)
-  }
-
-  // MARK: - TensorShape
-
-  func testTensorShape_InitWithArray() {
-    let shape = TensorShape(Constant.dimensions)
-    XCTAssertEqual(shape.rank, Constant.dimensions.count)
-    XCTAssertEqual(shape.dimensions, Constant.dimensions)
-  }
-
-  func testTensorShape_InitWithElements() {
-    let shape = TensorShape(2, 2, 3)
-    XCTAssertEqual(shape.rank, Constant.dimensions.count)
-    XCTAssertEqual(shape.dimensions, Constant.dimensions)
-  }
-
-  func testTensorShape_InitWithArrayLiteral() {
-    let shape: TensorShape = [2, 2, 3]
-    XCTAssertEqual(shape.rank, Constant.dimensions.count)
-    XCTAssertEqual(shape.dimensions, Constant.dimensions)
-  }
-}
-
-// MARK: - Constants
-
-private enum Constant {
-  /// Array of 2 arrays of 2 arrays of 3 numbers: [[[1, 1, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]]].
-  static let dimensions = [2, 2, 3]
-}
-
-// MARK: - Extensions
-
-extension TensorShape: Equatable {
-  public static func == (lhs: TensorShape, rhs: TensorShape) -> Bool {
-    return lhs.rank == rhs.rank && lhs.dimensions == rhs.dimensions
-  }
-}
-
-extension Tensor: Equatable {
-  public static func == (lhs: Tensor, rhs: Tensor) -> Bool {
-    return lhs.name == rhs.name && lhs.dataType == rhs.dataType && lhs.shape == rhs.shape &&
-           lhs.data == rhs.data && lhs.quantizationParameters == rhs.quantizationParameters
-  }
-}
-- 
GitLab


From fd54d400ba60e57daf431afe209133226e03acfc Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Tue, 8 Jan 2019 14:10:55 -0800
Subject: [PATCH 0408/1095] Unify RMSProp w/o momentum.

PiperOrigin-RevId: 228396933
---
 tensorflow/python/keras/optimizer_v2/BUILD    |   1 +
 .../keras/optimizer_v2/optimizer_v2_test.py   |   2 +-
 .../python/keras/optimizer_v2/rmsprop.py      | 155 ++++++++++++------
 .../python/keras/optimizer_v2/rmsprop_test.py | 106 ++++++++----
 4 files changed, 181 insertions(+), 83 deletions(-)

diff --git a/tensorflow/python/keras/optimizer_v2/BUILD b/tensorflow/python/keras/optimizer_v2/BUILD
index 964cf5fcba..2b990c9334 100644
--- a/tensorflow/python/keras/optimizer_v2/BUILD
+++ b/tensorflow/python/keras/optimizer_v2/BUILD
@@ -217,4 +217,5 @@ cuda_py_test(
         "//tensorflow/python:state_ops",
         "//tensorflow/python:variables",
     ],
+    shard_count = 2,
 )
diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
index 58c8b1f343..b41e4aa064 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
@@ -507,7 +507,7 @@ class OptimizersCompatibilityTest(test.TestCase, parameterized.TestCase):
       ('adadelta', 'adadelta', True), ('adagrad', 'adagrad', True),
       ('adam', 'adam', True), ('adamax', 'adamax', True),
       ('nadam', 'nadam', True), ('momentum', 'momentum', True),
-      ('sgd', 'sgd', False))
+      ('rmsprop', 'rmsprop', True), ('sgd', 'sgd', False))
   def testOptimizersCompatibility(self, opt_str, test_weights):
     np.random.seed(1331)
     with self.cached_session():
diff --git a/tensorflow/python/keras/optimizer_v2/rmsprop.py b/tensorflow/python/keras/optimizer_v2/rmsprop.py
index 3f153c4a41..83b81cbe5d 100644
--- a/tensorflow/python/keras/optimizer_v2/rmsprop.py
+++ b/tensorflow/python/keras/optimizer_v2/rmsprop.py
@@ -17,8 +17,14 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import numpy as np
+
 from tensorflow.python.framework import ops
 from tensorflow.python.keras.optimizer_v2 import optimizer_v2
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import state_ops
 from tensorflow.python.training import training_ops
 from tensorflow.python.util.tf_export import keras_export
 
@@ -114,77 +120,122 @@ class RMSprop(optimizer_v2.OptimizerV2):
   def _create_slots(self, var_list):
     for var in var_list:
       self.add_slot(var, "rms")
-      self.add_slot(var, "momentum")
-      if self.centered:
+    if self._momentum:
+      for var in var_list:
+        self.add_slot(var, "momentum")
+    if self.centered:
+      for var in var_list:
         self.add_slot(var, "mg")
 
   def _resource_apply_dense(self, grad, var):
     var_dtype = var.dtype.base_dtype
     lr_t = self._decayed_lr(var_dtype)
     rms = self.get_slot(var, "rms")
-    mom = self.get_slot(var, "momentum")
     rho = self._get_hyper("rho", var_dtype)
     momentum = self._get_hyper("momentum", var_dtype)
     epsilon = self._get_hyper("epsilon", var_dtype)
-    if self.centered:
-      mg = self.get_slot(var, "mg")
-      return training_ops.resource_apply_centered_rms_prop(
-          var.handle,
-          mg.handle,
-          rms.handle,
-          mom.handle,
-          lr_t,
-          rho,
-          momentum,
-          epsilon,
-          grad,
-          use_locking=self._use_locking)
+    if self._momentum:
+      mom = self.get_slot(var, "momentum")
+      if self.centered:
+        mg = self.get_slot(var, "mg")
+        return training_ops.resource_apply_centered_rms_prop(
+            var.handle,
+            mg.handle,
+            rms.handle,
+            mom.handle,
+            lr_t,
+            rho,
+            momentum,
+            epsilon,
+            grad,
+            use_locking=self._use_locking)
+      else:
+        return training_ops.resource_apply_rms_prop(
+            var.handle,
+            rms.handle,
+            mom.handle,
+            lr_t,
+            rho,
+            momentum,
+            epsilon,
+            grad,
+            use_locking=self._use_locking)
     else:
-      return training_ops.resource_apply_rms_prop(
-          var.handle,
-          rms.handle,
-          mom.handle,
-          lr_t,
-          rho,
-          momentum,
-          epsilon,
-          grad,
-          use_locking=self._use_locking)
+      rms_t = rho * rms + (1. - rho) * math_ops.square(grad)
+      rms_t = state_ops.assign(rms, rms_t, use_locking=self._use_locking)
+      denom_t = rms_t
+      if self.centered:
+        mg = self.get_slot(var, "mg")
+        mg_t = rho * mg + (1. - rho) * grad
+        mg_t = state_ops.assign(mg, mg_t, use_locking=self._use_locking)
+        denom_t = rms_t - math_ops.square(mg_t)
+      var_t = var - lr_t * grad / (math_ops.sqrt(denom_t) + epsilon)
+      return state_ops.assign(var, var_t, use_locking=self._use_locking).op
 
   def _resource_apply_sparse(self, grad, var, indices):
     var_dtype = var.dtype.base_dtype
     lr_t = self._decayed_lr(var_dtype)
     rms = self.get_slot(var, "rms")
-    mom = self.get_slot(var, "momentum")
     rho = self._get_hyper("rho", var_dtype)
     momentum = self._get_hyper("momentum", var_dtype)
     epsilon = self._get_hyper("epsilon", var_dtype)
-    if self.centered:
-      mg = self.get_slot(var, "mg")
-      return training_ops.resource_sparse_apply_centered_rms_prop(
-          var.handle,
-          mg.handle,
-          rms.handle,
-          mom.handle,
-          lr_t,
-          rho,
-          momentum,
-          epsilon,
-          grad,
-          indices,
-          use_locking=self._use_locking)
+    if self._momentum:
+      mom = self.get_slot(var, "momentum")
+      if self.centered:
+        mg = self.get_slot(var, "mg")
+        return training_ops.resource_sparse_apply_centered_rms_prop(
+            var.handle,
+            mg.handle,
+            rms.handle,
+            mom.handle,
+            lr_t,
+            rho,
+            momentum,
+            epsilon,
+            grad,
+            indices,
+            use_locking=self._use_locking)
+      else:
+        return training_ops.resource_sparse_apply_rms_prop(
+            var.handle,
+            rms.handle,
+            mom.handle,
+            lr_t,
+            rho,
+            momentum,
+            epsilon,
+            grad,
+            indices,
+            use_locking=self._use_locking)
     else:
-      return training_ops.resource_sparse_apply_rms_prop(
-          var.handle,
-          rms.handle,
-          mom.handle,
-          lr_t,
-          rho,
-          momentum,
-          epsilon,
-          grad,
-          indices,
-          use_locking=self._use_locking)
+      rms_scaled_g_values = (grad * grad) * (1. - rho)
+      rms_t = state_ops.assign(rms, rms * rho, use_locking=self._use_locking)
+      with ops.control_dependencies([rms_t]):
+        rms_t = self._resource_scatter_add(rms, indices, rms_scaled_g_values)
+        rms_slice = array_ops.gather(rms_t, indices)
+      denom_slice = rms_slice
+      if self.centered:
+        mg = self.get_slot(var, "mg")
+        mg_scaled_g_values = grad * (1. - rho)
+        mg_t = state_ops.assign(mg, mg * rho, use_locking=self._use_locking)
+        with ops.control_dependencies([mg_t]):
+          mg_t = self._resource_scatter_add(mg, indices, mg_scaled_g_values)
+          mg_slice = array_ops.gather(mg_t, indices)
+          denom_slice = rms_slice - math_ops.square(mg_slice)
+      var_update = self._resource_scatter_add(
+          var, indices, -lr_t * grad / (math_ops.sqrt(denom_slice) + epsilon))
+      if self.centered:
+        return control_flow_ops.group(*[var_update, rms_t, mg_t])
+      return control_flow_ops.group(*[var_update, rms_t])
+
+  def set_weights(self, weights):
+    params = self.weights
+    # Override set_weights for backward compatibility of Keras V1 optimizer
+    # since it does not include iteration at head of the weight list. Set
+    # iteration to 0.
+    if len(params) == len(weights) + 1:
+      weights = [np.array(0)] + weights
+    super(RMSprop, self).set_weights(weights)
 
   def get_config(self):
     config = super(RMSprop, self).get_config()
diff --git a/tensorflow/python/keras/optimizer_v2/rmsprop_test.py b/tensorflow/python/keras/optimizer_v2/rmsprop_test.py
index 4d61cfbbc5..67662aae20 100644
--- a/tensorflow/python/keras/optimizer_v2/rmsprop_test.py
+++ b/tensorflow/python/keras/optimizer_v2/rmsprop_test.py
@@ -58,14 +58,18 @@ class RMSpropOptimizerTest(test.TestCase):
   def _rmsprop_update_numpy(self, var, g, mg, rms, mom, lr, rho, momentum,
                             epsilon, centered):
     rms_t = rms * rho + (1 - rho) * g * g
-    denom_t = rms_t + epsilon
     if centered:
       mg_t = mg * rho + (1 - rho) * g
-      denom_t -= mg_t * mg_t
+      denom_t = rms_t - mg_t * mg_t
     else:
       mg_t = mg
-    mom_t = momentum * mom + lr * g / np.sqrt(denom_t, dtype=denom_t.dtype)
-    var_t = var - mom_t
+      denom_t = rms_t
+    if momentum > 0.:
+      mom_t = momentum * mom + lr * g / (np.sqrt(denom_t + epsilon))
+      var_t = var - mom_t
+    else:
+      mom_t = mom
+      var_t = var - lr * g / (np.sqrt(denom_t) + epsilon)
     return var_t, mg_t, rms_t, mom_t
 
   def _sparse_rmsprop_update_numpy(self, var, gindexs, gvalues, mg, rms, mom,
@@ -78,12 +82,18 @@ class RMSpropOptimizerTest(test.TestCase):
       gindex = gindexs[i]
       gvalue = gvalues[i]
       rms_t[gindex] = rms[gindex] * rho + (1 - rho) * gvalue * gvalue
-      denom_t = rms_t[gindex] + epsilon
       if centered:
         mg_t[gindex] = mg_t[gindex] * rho + (1 - rho) * gvalue
-        denom_t -= mg_t[gindex] * mg_t[gindex]
-      mom_t[gindex] = momentum * mom[gindex] + lr * gvalue / np.sqrt(denom_t)
-      var_t[gindex] = var[gindex] - mom_t[gindex]
+        denom_t = rms_t[gindex] - mg_t[gindex] * mg_t[gindex]
+      else:
+        denom_t = rms_t[gindex]
+      if momentum > 0.:
+        mom_t[gindex] = momentum * mom[gindex] + lr * gvalue / np.sqrt(denom_t +
+                                                                       epsilon)
+        var_t[gindex] = var[gindex] - mom_t[gindex]
+      else:
+        mom_t[gindex] = mom[gindex]
+        var_t[gindex] = var[gindex] - lr * gvalue / (np.sqrt(denom_t) + epsilon)
     return var_t, mg_t, rms_t, mom_t
 
   @test_util.run_deprecated_v1
@@ -117,14 +127,17 @@ class RMSpropOptimizerTest(test.TestCase):
           mg0 = None
           mg1 = None
 
+        if momentum > 0.:
+          mom0 = opt.get_slot(var0, "momentum")
+          mom1 = opt.get_slot(var1, "momentum")
+        else:
+          mom0 = None
+          mom1 = None
+
         rms0 = opt.get_slot(var0, "rms")
         self.assertTrue(rms0 is not None)
         rms1 = opt.get_slot(var1, "rms")
         self.assertTrue(rms1 is not None)
-        mom0 = opt.get_slot(var0, "momentum")
-        self.assertTrue(mom0 is not None)
-        mom1 = opt.get_slot(var1, "momentum")
-        self.assertTrue(mom1 is not None)
 
         mg0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
         mg1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
@@ -137,8 +150,8 @@ class RMSpropOptimizerTest(test.TestCase):
         self.assertAllClose([1.0, 2.0], self.evaluate(var0))
         self.assertAllClose([3.0, 4.0], self.evaluate(var1))
 
-        # Run 4 steps of RMSprop
-        for _ in range(1, 5):
+        # Run 3 steps of RMSprop
+        for _ in range(1, 4):
           self.evaluate(update)
 
           var0_np, mg0_np, rms0_np, mom0_np = self._rmsprop_update_numpy(
@@ -152,10 +165,11 @@ class RMSpropOptimizerTest(test.TestCase):
           if centered:
             self.assertAllCloseAccordingToType(mg0_np, self.evaluate(mg0))
             self.assertAllCloseAccordingToType(mg1_np, self.evaluate(mg1))
+          if momentum > 0.:
+            self.assertAllCloseAccordingToType(mom0_np, self.evaluate(mom0))
+            self.assertAllCloseAccordingToType(mom1_np, self.evaluate(mom1))
           self.assertAllCloseAccordingToType(rms0_np, self.evaluate(rms0))
           self.assertAllCloseAccordingToType(rms1_np, self.evaluate(rms1))
-          self.assertAllCloseAccordingToType(mom0_np, self.evaluate(mom0))
-          self.assertAllCloseAccordingToType(mom1_np, self.evaluate(mom1))
           self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
           self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
 
@@ -191,10 +205,12 @@ class RMSpropOptimizerTest(test.TestCase):
     self.assertTrue(rms0 is not None)
     rms1 = opt.get_slot(var1, "rms")
     self.assertTrue(rms1 is not None)
-    mom0 = opt.get_slot(var0, "momentum")
-    self.assertTrue(mom0 is not None)
-    mom1 = opt.get_slot(var1, "momentum")
-    self.assertTrue(mom1 is not None)
+    if momentum > 0.:
+      mom0 = opt.get_slot(var0, "momentum")
+      mom1 = opt.get_slot(var1, "momentum")
+    else:
+      mom0 = None
+      mom1 = None
 
     mg0_np = np.array([0.0, 0.0])
     mg1_np = np.array([0.0, 0.0])
@@ -222,8 +238,9 @@ class RMSpropOptimizerTest(test.TestCase):
       # Validate updated params
       self.assertAllCloseAccordingToType(rms0_np, self.evaluate(rms0))
       self.assertAllCloseAccordingToType(rms1_np, self.evaluate(rms1))
-      self.assertAllCloseAccordingToType(mom0_np, self.evaluate(mom0))
-      self.assertAllCloseAccordingToType(mom1_np, self.evaluate(mom1))
+      if momentum > 0.:
+        self.assertAllCloseAccordingToType(mom0_np, self.evaluate(mom0))
+        self.assertAllCloseAccordingToType(mom1_np, self.evaluate(mom1))
       self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
       self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
 
@@ -325,10 +342,12 @@ class RMSpropOptimizerTest(test.TestCase):
         self.assertTrue(rms0 is not None)
         rms1 = opt.get_slot(var1, "rms")
         self.assertTrue(rms1 is not None)
-        mom0 = opt.get_slot(var0, "momentum")
-        self.assertTrue(mom0 is not None)
-        mom1 = opt.get_slot(var1, "momentum")
-        self.assertTrue(mom1 is not None)
+        if momentum > 0.:
+          mom0 = opt.get_slot(var0, "momentum")
+          mom1 = opt.get_slot(var1, "momentum")
+        else:
+          mom0 = None
+          mom1 = None
 
         mg0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
         mg1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
@@ -341,8 +360,8 @@ class RMSpropOptimizerTest(test.TestCase):
         self.assertAllClose([1.0, 2.0], self.evaluate(var0))
         self.assertAllClose([3.0, 4.0], self.evaluate(var1))
 
-        # Run 4 steps of RMSprop
-        for _ in range(1, 5):
+        # Run 3 steps of RMSprop
+        for _ in range(1, 4):
           self.evaluate(update)
 
           var0_np, mg0_np, rms0_np, mom0_np = self._sparse_rmsprop_update_numpy(
@@ -358,8 +377,9 @@ class RMSpropOptimizerTest(test.TestCase):
             self.assertAllCloseAccordingToType(mg1_np, self.evaluate(mg1))
           self.assertAllCloseAccordingToType(rms0_np, self.evaluate(rms0))
           self.assertAllCloseAccordingToType(rms1_np, self.evaluate(rms1))
-          self.assertAllCloseAccordingToType(mom0_np, self.evaluate(mom0))
-          self.assertAllCloseAccordingToType(mom1_np, self.evaluate(mom1))
+          if momentum > 0.:
+            self.assertAllCloseAccordingToType(mom0_np, self.evaluate(mom0))
+            self.assertAllCloseAccordingToType(mom1_np, self.evaluate(mom1))
           self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
           self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
 
@@ -420,6 +440,32 @@ class RMSpropOptimizerTest(test.TestCase):
     opt_3 = rmsprop.RMSprop(learning_rate=0.1)
     self.assertEqual(opt_3.lr, 0.1)
 
+  def testSlotsUniqueEager(self):
+    with context.eager_mode():
+      v1 = variables.Variable(1.)
+      v2 = variables.Variable(1.)
+
+      opt = rmsprop.RMSprop(1., momentum=0., centered=False)
+      opt.minimize(lambda: v1 + v2, var_list=[v1, v2])
+      # There should be iteration, and one unique slot variable for v1 and v2.
+      self.assertEqual(3, len(set(opt.variables())))
+      self.assertEqual(
+          self.evaluate(opt.variables()[0]), self.evaluate(opt.iterations))
+
+      opt = rmsprop.RMSprop(learning_rate=1., momentum=0.2, centered=False)
+      opt.minimize(lambda: v1 + v2, var_list=[v1, v2])
+      # There should be iteration, and two unique slot variables for v1 and v2.
+      self.assertEqual(5, len(set(opt.variables())))
+      self.assertEqual(
+          self.evaluate(opt.variables()[0]), self.evaluate(opt.iterations))
+
+      opt = rmsprop.RMSprop(learning_rate=1., momentum=0.2, centered=True)
+      opt.minimize(lambda: v1 + v2, var_list=[v1, v2])
+      # There should be iteration, and three unique slot variables for v1 and v2
+      self.assertEqual(7, len(set(opt.variables())))
+      self.assertEqual(
+          self.evaluate(opt.variables()[0]), self.evaluate(opt.iterations))
+
 
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From 0ce305b6ce8560945be3a92eae1bcddd60af5e10 Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Tue, 8 Jan 2019 14:53:40 -0800
Subject: [PATCH 0409/1095] [XLA:Python] Add Gather and Scatter to XLA Python
 bindings.

Refactor handling of repeated fields in *DimensionNumbers SWIG rules.

PiperOrigin-RevId: 228404715
---
 .../xla/python/local_computation_builder.cc   |  16 +
 .../xla/python/local_computation_builder.h    |   9 +
 .../xla/python/local_computation_builder.i    | 281 +++++++-----------
 tensorflow/compiler/xla/python/xla_client.py  |  12 +
 .../compiler/xla/python/xla_client_test.py    |  40 +++
 5 files changed, 183 insertions(+), 175 deletions(-)

diff --git a/tensorflow/compiler/xla/python/local_computation_builder.cc b/tensorflow/compiler/xla/python/local_computation_builder.cc
index b0f0e9e570..c153603105 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.cc
+++ b/tensorflow/compiler/xla/python/local_computation_builder.cc
@@ -927,6 +927,22 @@ LocalOp LocalComputationBuilder::TriangularSolve(const LocalOp& a,
                               conjugate_a);
 }
 
+LocalOp LocalComputationBuilder::Gather(
+    const LocalOp& input, const LocalOp& start_indices,
+    const GatherDimensionNumbers& dimension_numbers,
+    absl::Span<const int64> slice_sizes) {
+  return xla::Gather(input.op(), start_indices.op(), dimension_numbers,
+                     slice_sizes);
+}
+
+LocalOp LocalComputationBuilder::Scatter(
+    const LocalOp& input, const LocalOp& scatter_indices,
+    const LocalOp& updates, const LocalComputation& update_computation,
+    const ScatterDimensionNumbers& dimension_numbers) {
+  return xla::Scatter(input.op(), scatter_indices.op(), updates.op(),
+                      update_computation.computation(), dimension_numbers);
+}
+
 StatusOr<LocalComputation*> LocalComputationBuilder::BuildConstantSubGraph(
     const LocalOp& operand) {
   TF_ASSIGN_OR_RETURN(XlaComputation computation,
diff --git a/tensorflow/compiler/xla/python/local_computation_builder.h b/tensorflow/compiler/xla/python/local_computation_builder.h
index 5e83415921..98759cf984 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.h
+++ b/tensorflow/compiler/xla/python/local_computation_builder.h
@@ -418,6 +418,15 @@ class LocalComputationBuilder {
   LocalOp TriangularSolve(const LocalOp& a, const LocalOp& b, bool left_side,
                           bool lower, bool transpose_a, bool conjugate_a);
 
+  LocalOp Gather(const LocalOp& input, const LocalOp& start_indices,
+                 const GatherDimensionNumbers& dimension_numbers,
+                 absl::Span<const int64> slice_sizes);
+
+  LocalOp Scatter(const LocalOp& input, const LocalOp& scatter_indices,
+                  const LocalOp& updates,
+                  const LocalComputation& update_computation,
+                  const ScatterDimensionNumbers& dimension_numbers);
+
   StatusOr<LocalComputation*> BuildConstantSubGraph(const LocalOp& operand);
 
 #define _FORWARD(method_name, return_sig, args_sig) \
diff --git a/tensorflow/compiler/xla/python/local_computation_builder.i b/tensorflow/compiler/xla/python/local_computation_builder.i
index bf5d667c6a..cc96d84724 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.i
+++ b/tensorflow/compiler/xla/python/local_computation_builder.i
@@ -34,6 +34,8 @@ limitations under the License.
 //  PaddingConfig proto                <-  corresponding Python proto
 //  ConvolutionDimensionNumbers proto  <-  corresponding Python proto
 //  DotDimensionNumbers proto          <-  corresponding Python proto
+//  GatherDimensionNumbers proto       <-  corresponding Python proto
+//  ScatterDimensionNumbers proto      <-  corresponding Python proto
 //
 // Arrows indicate whether a conversion only ever occurs in one
 // direction, or whether it is maintained bidirectionally.
@@ -167,8 +169,41 @@ bool HandleStringAttribute(PyObject* o,
   return true;  // Handled string attribute, ok!
 }
 
+bool HandleRepeatedInt64Attribute(
+    PyObject* o, const char* attr_name,
+    tensorflow::protobuf::RepeatedField<int64>* field) {
+  PyObject* seq = PyObject_GetAttrString(o, attr_name);
+  if (!seq) {
+    return false;
+  }
+
+  int length = PySequence_Size(seq);
+  if (length == -1) {
+    Py_DECREF(seq);
+    return false;
+  }
+
+  for (int i = 0; i < length; ++i) {
+    PyObject* item = PySequence_GetItem(seq, i);
+    if (!item) {
+      Py_DECREF(seq);
+      return false;
+    }
+    const int64 dimension = numpy::PyIntOrPyLongToLong(item);
+    if (dimension == -1 && PyErr_Occurred()) {
+      Py_DECREF(item);
+      Py_DECREF(seq);
+      return false;
+    }
+    *field->Add() = dimension;
+    Py_DECREF(item);
+  }
+  Py_DECREF(seq);
+  return true;
 }
-}
+
+}  // namespace swig
+}  // namespace xla
 %}
 
 // Required to use PyArray_* functions.
@@ -657,128 +692,27 @@ tensorflow::ImportNumpy();
 
 %typemap(in) const DotDimensionNumbers&
     (DotDimensionNumbers dimension_numbers) {
-  int length;
-
-  /* lhs_contracting_dimensions */
-  PyObject* lhs_contracting_dimensions = PyObject_GetAttrString(
-      $input, "lhs_contracting_dimensions");
-  if (!lhs_contracting_dimensions) {
-    SWIG_fail;
-  }
-
-  length = PySequence_Size(lhs_contracting_dimensions);
-  if (length == -1) {
-    Py_DECREF(lhs_contracting_dimensions);
-    SWIG_fail;
-  }
-
-  for (int i = 0; i < length; ++i) {
-    PyObject* item = PySequence_GetItem(lhs_contracting_dimensions, i);
-    if (!item) {
-      Py_DECREF(lhs_contracting_dimensions);
-      SWIG_fail;
-    }
-    const int64 dimension = numpy::PyIntOrPyLongToLong(item);
-    if (dimension == -1 && PyErr_Occurred()) {
-      Py_DECREF(item);
-      Py_DECREF(lhs_contracting_dimensions);
-      SWIG_fail;
-    }
-    dimension_numbers.add_lhs_contracting_dimensions(dimension);
-    Py_DECREF(item);
-  }
-  Py_DECREF(lhs_contracting_dimensions);
-
-  /* rhs_contracting_dimensions */
-  PyObject* rhs_contracting_dimensions = PyObject_GetAttrString(
-      $input, "rhs_contracting_dimensions");
-  if (!lhs_contracting_dimensions) {
+  if (!HandleRepeatedInt64Attribute(
+        $input, "lhs_contracting_dimensions",
+        dimension_numbers.mutable_lhs_contracting_dimensions())) {
     SWIG_fail;
   }
-
-  length = PySequence_Size(rhs_contracting_dimensions);
-  if (length == -1) {
-    Py_DECREF(rhs_contracting_dimensions);
+  if (!HandleRepeatedInt64Attribute(
+        $input, "rhs_contracting_dimensions",
+        dimension_numbers.mutable_rhs_contracting_dimensions())) {
     SWIG_fail;
   }
-
-  for (int i = 0; i < length; ++i) {
-    PyObject* item = PySequence_GetItem(rhs_contracting_dimensions, i);
-    if (!item) {
-      Py_DECREF(rhs_contracting_dimensions);
-      SWIG_fail;
-    }
-    const int64 dimension = numpy::PyIntOrPyLongToLong(item);
-    if (dimension == -1 && PyErr_Occurred()) {
-      Py_DECREF(item);
-      Py_DECREF(rhs_contracting_dimensions);
-      SWIG_fail;
-    }
-    dimension_numbers.add_rhs_contracting_dimensions(dimension);
-    Py_DECREF(item);
-  }
-  Py_DECREF(rhs_contracting_dimensions);
-
-  /* lhs_batch_dimensions */
-  PyObject* lhs_batch_dimensions = PyObject_GetAttrString(
-      $input, "lhs_batch_dimensions");
-  if (!lhs_batch_dimensions) {
+  if (!HandleRepeatedInt64Attribute(
+        $input, "lhs_batch_dimensions",
+        dimension_numbers.mutable_lhs_batch_dimensions())) {
     SWIG_fail;
   }
-
-  length = PySequence_Size(lhs_batch_dimensions);
-  if (length == -1) {
-    Py_DECREF(lhs_batch_dimensions);
-    SWIG_fail;
-  }
-
-  for (int i = 0; i < length; ++i) {
-    PyObject* item = PySequence_GetItem(lhs_batch_dimensions, i);
-    if (!item) {
-      Py_DECREF(lhs_batch_dimensions);
-      SWIG_fail;
-    }
-    const int64 dimension = numpy::PyIntOrPyLongToLong(item);
-    if (dimension == -1 && PyErr_Occurred()) {
-      Py_DECREF(item);
-      Py_DECREF(lhs_batch_dimensions);
-      SWIG_fail;
-    }
-    dimension_numbers.add_lhs_batch_dimensions(dimension);
-    Py_DECREF(item);
-  }
-  Py_DECREF(lhs_batch_dimensions);
-
-  /* rhs_batch_dimensions */
-  PyObject* rhs_batch_dimensions = PyObject_GetAttrString(
-      $input, "rhs_batch_dimensions");
-  if (!rhs_batch_dimensions) {
-    SWIG_fail;
-  }
-
-  length = PySequence_Size(rhs_batch_dimensions);
-  if (length == -1) {
-    Py_DECREF(rhs_batch_dimensions);
+  if (!HandleRepeatedInt64Attribute(
+        $input, "rhs_batch_dimensions",
+        dimension_numbers.mutable_rhs_batch_dimensions())) {
     SWIG_fail;
   }
 
-  for (int i = 0; i < length; ++i) {
-    PyObject* item = PySequence_GetItem(rhs_batch_dimensions, i);
-    if (!item) {
-      Py_DECREF(rhs_batch_dimensions);
-      SWIG_fail;
-    }
-    const int64 dimension = numpy::PyIntOrPyLongToLong(item);
-    if (dimension == -1 && PyErr_Occurred()) {
-      Py_DECREF(item);
-      Py_DECREF(rhs_batch_dimensions);
-      SWIG_fail;
-    }
-    dimension_numbers.add_rhs_batch_dimensions(dimension);
-    Py_DECREF(item);
-  }
-  Py_DECREF(rhs_batch_dimensions);
-
   $1 = &dimension_numbers;
 }
 
@@ -861,85 +795,80 @@ tensorflow::ImportNumpy();
   dimension_numbers.set_kernel_input_feature_dimension(value);
 
   PyObject* o;
-  int length;
 
-  o = PyObject_GetAttrString($input, "input_spatial_dimensions");
-  if (!o) {
+  if (!HandleRepeatedInt64Attribute(
+        $input, "input_spatial_dimensions",
+        dimension_numbers.mutable_input_spatial_dimensions())) {
     SWIG_fail;
   }
-  length = PySequence_Size(o);
-  if (length == -1) {
-    Py_DECREF(o);
+  if (!HandleRepeatedInt64Attribute(
+        $input, "kernel_spatial_dimensions",
+        dimension_numbers.mutable_kernel_spatial_dimensions())) {
     SWIG_fail;
   }
-  for (int i = 0; i < length; ++i) {
-    PyObject* item = PySequence_GetItem(o, i);
-    if (!item) {
-      Py_DECREF(o);
-      SWIG_fail;
-    }
-    const int64 dimension = numpy::PyIntOrPyLongToLong(item);
-    if (dimension == -1 && PyErr_Occurred()) {
-      Py_DECREF(item);
-      Py_DECREF(o);
-      SWIG_fail;
-    }
-    dimension_numbers.add_input_spatial_dimensions(dimension);
-    Py_DECREF(item);
+  if (!HandleRepeatedInt64Attribute(
+        $input, "output_spatial_dimensions",
+        dimension_numbers.mutable_output_spatial_dimensions())) {
+    SWIG_fail;
   }
-  Py_DECREF(o);
 
-  o = PyObject_GetAttrString($input, "kernel_spatial_dimensions");
-  if (!o) {
+  $1 = &dimension_numbers;
+}
+
+// GatherDimensionNumbers
+
+%typemap(in) const GatherDimensionNumbers&
+    (GatherDimensionNumbers dimension_numbers) {
+  if (!HandleRepeatedInt64Attribute(
+        $input, "offset_dims",
+        dimension_numbers.mutable_offset_dims())) {
     SWIG_fail;
   }
-  length = PySequence_Size(o);
-  if (length == -1) {
-    Py_DECREF(o);
+  if (!HandleRepeatedInt64Attribute(
+        $input, "collapsed_slice_dims",
+        dimension_numbers.mutable_collapsed_slice_dims())) {
     SWIG_fail;
   }
-  for (int i = 0; i < length; ++i) {
-    PyObject* item = PySequence_GetItem(o, i);
-    if (!item) {
-      Py_DECREF(o);
-      SWIG_fail;
-    }
-    const int64 dimension = numpy::PyIntOrPyLongToLong(item);
-    if (dimension == -1 && PyErr_Occurred()) {
-      Py_DECREF(item);
-      Py_DECREF(o);
-      SWIG_fail;
-    }
-    dimension_numbers.add_kernel_spatial_dimensions(dimension);
-    Py_DECREF(item);
+  if (!HandleRepeatedInt64Attribute(
+        $input, "start_index_map",
+        dimension_numbers.mutable_start_index_map())) {
+    SWIG_fail;
   }
-  Py_DECREF(o);
 
-  o = PyObject_GetAttrString($input, "output_spatial_dimensions");
-  if (!o) {
+  int64 value;
+  if (!GetIntAttr($input, "index_vector_dim", &value)) {
     SWIG_fail;
   }
-  length = PySequence_Size(o);
-  if (length == -1) {
-    Py_DECREF(o);
+  dimension_numbers.set_index_vector_dim(value);
+
+  $1 = &dimension_numbers;
+}
+
+// ScatterDimensionNumbers
+
+%typemap(in) const ScatterDimensionNumbers&
+    (ScatterDimensionNumbers dimension_numbers) {
+  if (!HandleRepeatedInt64Attribute(
+        $input, "update_window_dims",
+        dimension_numbers.mutable_update_window_dims())) {
     SWIG_fail;
   }
-  for (int i = 0; i < length; ++i) {
-    PyObject* item = PySequence_GetItem(o, i);
-    if (!item) {
-      Py_DECREF(o);
-      SWIG_fail;
-    }
-    const int64 dimension = numpy::PyIntOrPyLongToLong(item);
-    if (dimension == -1 && PyErr_Occurred()) {
-      Py_DECREF(item);
-      Py_DECREF(o);
-      SWIG_fail;
-    }
-    dimension_numbers.add_output_spatial_dimensions(dimension);
-    Py_DECREF(item);
+  if (!HandleRepeatedInt64Attribute(
+        $input, "inserted_window_dims",
+        dimension_numbers.mutable_inserted_window_dims())) {
+    SWIG_fail;
+  }
+  if (!HandleRepeatedInt64Attribute(
+        $input, "scatter_dims_to_operand_dims",
+        dimension_numbers.mutable_scatter_dims_to_operand_dims())) {
+    SWIG_fail;
+  }
+
+  int64 value;
+  if (!GetIntAttr($input, "index_vector_dim", &value)) {
+    SWIG_fail;
   }
-  Py_DECREF(o);
+  dimension_numbers.set_index_vector_dim(value);
 
   $1 = &dimension_numbers;
 }
@@ -1151,6 +1080,8 @@ tensorflow::ImportNumpy();
 %unignore xla::swig::LocalComputationBuilder::QR;
 %unignore xla::swig::LocalComputationBuilder::TriangularSolve;
 %unignore xla::swig::LocalComputationBuilder::CustomCall;
+%unignore xla::swig::LocalComputationBuilder::Gather;
+%unignore xla::swig::LocalComputationBuilder::Scatter;
 %unignore xla::swig::DeleteLocalComputation;
 %unignore xla::swig::DestructureLocalShapedBufferTuple;
 %unignore xla::swig::DestructureXrtAllocationTuple;
diff --git a/tensorflow/compiler/xla/python/xla_client.py b/tensorflow/compiler/xla/python/xla_client.py
index 378bbdcb17..4e71121c09 100644
--- a/tensorflow/compiler/xla/python/xla_client.py
+++ b/tensorflow/compiler/xla/python/xla_client.py
@@ -1477,6 +1477,18 @@ class ComputationBuilder(object):
     return self._client.TriangularSolve(
         a, b, left_side, lower, transpose_a, conjugate_a)
 
+  def Gather(self, a, start_indices, dimension_numbers, slice_sizes):
+    """Enqueues a Gather operation onto the computation."""
+    return self._client.Gather(a, start_indices, dimension_numbers,
+                               slice_sizes)
+
+  def Scatter(self, a, scatter_indices, updates, update_computation,
+              dimension_numbers):
+    """Enqueues a Scatter operation onto the computation."""
+    return self._client.Scatter(
+        a, scatter_indices, updates, update_computation.computation,
+        dimension_numbers,)
+
 
 def _forward_methods_to_local_builder():
   """Forward remaining ComputationBuilder methods to the C API.
diff --git a/tensorflow/compiler/xla/python/xla_client_test.py b/tensorflow/compiler/xla/python/xla_client_test.py
index 002a20e60a..874e087eb6 100644
--- a/tensorflow/compiler/xla/python/xla_client_test.py
+++ b/tensorflow/compiler/xla/python/xla_client_test.py
@@ -1129,6 +1129,21 @@ class SingleOpTest(LocalComputationTest):
     self.assertFalse(c.IsConstant(non_const_expr))
     # self.assertTrue(c.IsConstant(c.Sub(c.Add(x, a), x)))  # TODO(b/77245564)
 
+  def testGather(self):
+    a = np.arange(9).astype(np.int32).reshape((3, 3))
+    indices = np.array([[[0, 2], [2, 1]], [[1, 2], [2, 0]]], dtype=np.int32)
+    dnums = xla_client.xla_data_pb2.GatherDimensionNumbers()
+    dnums.offset_dims.append(1)
+    dnums.offset_dims.append(2)
+    dnums.start_index_map.append(0)
+    dnums.start_index_map.append(1)
+    dnums.index_vector_dim = 2
+    c = self._NewComputation()
+    c.Gather(c.Constant(a), c.Constant(indices), dnums, slice_sizes=[1, 1])
+    g = self._Execute(c, ())
+    expected = np.array([[[[2, 7]]], [[[5, 6]]]], dtype=np.int32)
+    np.testing.assert_allclose(g, expected, rtol=1e-4)
+
 
 class EmbeddedComputationsTest(LocalComputationTest):
   """Tests for XLA graphs with embedded computations (such as maps)."""
@@ -1186,6 +1201,14 @@ class EmbeddedComputationsTest(LocalComputationTest):
     c.Mul(c.ParameterFromNumpy(NumpyArrayF64(0)), c.ConstantF64Scalar(2.0))
     return c.Build()
 
+  def _CreateBinaryAddS32Computation(self):
+    """Computation (s32, s32) -> s32 that adds its two parameters."""
+    c = self._NewComputation("add_param0_by_param1")
+    c.Add(
+        c.ParameterFromNumpy(NumpyArrayS32(0)),
+        c.ParameterFromNumpy(NumpyArrayS32(0)))
+    return c.Build()
+
   def _CreateBinaryAddF32Computation(self):
     """Computation (f32, f32) -> f32 that adds its two parameters."""
     c = self._NewComputation("add_param0_by_param1")
@@ -1568,6 +1591,23 @@ class EmbeddedComputationsTest(LocalComputationTest):
       execution.join()
       self.assertEqual(want, got)
 
+  def testScatter(self):
+    a = np.arange(9).astype(np.int32).reshape((3, 3))
+    scatter_indices = np.array([0, 2], dtype=np.int32)
+    updates = np.array([[10, 20, 30], [70, 80, 90]], dtype=np.int32)
+
+    dnums = xla_client.xla_data_pb2.ScatterDimensionNumbers()
+    dnums.update_window_dims.append(1)
+    dnums.inserted_window_dims.append(0)
+    dnums.scatter_dims_to_operand_dims.append(0)
+    dnums.index_vector_dim = 1
+
+    c = self._NewComputation()
+    c.Scatter(c.Constant(a), c.Constant(scatter_indices), c.Constant(updates),
+              self._CreateBinaryAddS32Computation(), dnums)
+    expected = np.array([[10, 21, 32], [3, 4, 5], [76, 87, 98]], dtype=np.int32)
+    self._ExecuteAndCompareClose(c, expected=expected)
+
 
 class ErrorTest(LocalComputationTest):
 
-- 
GitLab


From 1ad09edf03a1c4b96eff12a7d6c765a430603b7d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 8 Jan 2019 14:55:09 -0800
Subject: [PATCH 0410/1095] Use nsync, version 1.20.2, which has a bazel BUILD
 bug fix for aarch64 systems. It requires bazel version 0.16.0 or higher, but
 TensorFlow already relies on higher versions than that.

PiperOrigin-RevId: 228404987
---
 tensorflow/contrib/cmake/external/nsync.cmake | 2 +-
 tensorflow/workspace.bzl                      | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/tensorflow/contrib/cmake/external/nsync.cmake b/tensorflow/contrib/cmake/external/nsync.cmake
index 479609458c..b15143bfc1 100644
--- a/tensorflow/contrib/cmake/external/nsync.cmake
+++ b/tensorflow/contrib/cmake/external/nsync.cmake
@@ -16,7 +16,7 @@ include (ExternalProject)
 
 set(nsync_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/external/nsync/public)
 set(nsync_URL https://github.com/google/nsync)
-set(nsync_TAG 1.20.1)
+set(nsync_TAG 1.20.2)
 set(nsync_BUILD ${CMAKE_CURRENT_BINARY_DIR}/nsync/src/nsync)
 set(nsync_INSTALL ${CMAKE_CURRENT_BINARY_DIR}/nsync/install)
 
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index ae26ba6e68..5b0f6d4c84 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -395,12 +395,12 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
 
     tf_http_archive(
         name = "nsync",
-        sha256 = "692f9b30e219f71a6371b98edd39cef3cbda35ac3abc4cd99ce19db430a5591a",
-        strip_prefix = "nsync-1.20.1",
+        sha256 = "704be7f58afa47b99476bbac7aafd1a9db4357cef519db361716f13538547ffd",
+        strip_prefix = "nsync-1.20.2",
         system_build_file = clean_dep("//third_party/systemlibs:nsync.BUILD"),
         urls = [
-            "https://mirror.bazel.build/github.com/google/nsync/archive/1.20.1.tar.gz",
-            "https://github.com/google/nsync/archive/1.20.1.tar.gz",
+            "https://mirror.bazel.build/github.com/google/nsync/archive/1.20.2.tar.gz",
+            "https://github.com/google/nsync/archive/1.20.2.tar.gz",
         ],
     )
 
-- 
GitLab


From 9f0ab258f7d715171f878080e6ed5c880aea004c Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Tue, 8 Jan 2019 15:17:41 -0800
Subject: [PATCH 0411/1095] Prevent OOM in decoding GIF images.

PiperOrigin-RevId: 228409275
---
 tensorflow/core/lib/gif/gif_io.cc | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/lib/gif/gif_io.cc b/tensorflow/core/lib/gif/gif_io.cc
index 9a5215320f..3fad8c8b14 100644
--- a/tensorflow/core/lib/gif/gif_io.cc
+++ b/tensorflow/core/lib/gif/gif_io.cc
@@ -82,9 +82,20 @@ uint8* Decode(const void* srcdata, int datasize,
     return nullptr;
   }
 
+  // Don't request more memory than needed for each frame, preventing OOM
+  int max_frame_width = 0;
+  int max_frame_height = 0;
+  for (int k = 0; k < gif_file->ImageCount; k++) {
+    SavedImage* si = &gif_file->SavedImages[k];
+    if (max_frame_height < si->ImageDesc.Height)
+      max_frame_height = si->ImageDesc.Height;
+    if (max_frame_width < si->ImageDesc.Width)
+      max_frame_width = si->ImageDesc.Width;
+  }
+
   const int num_frames = gif_file->ImageCount;
-  const int width = gif_file->SWidth;
-  const int height = gif_file->SHeight;
+  const int width = max_frame_width;    // gif_file->SWidth;
+  const int height = max_frame_height;  // gif_file->SHeight;
   const int channel = 3;
 
   uint8* const dstdata = allocate_output(num_frames, width, height, channel);
-- 
GitLab


From bef89aba9c73880ed0db8109e2d029b8e65cb367 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 8 Jan 2019 15:17:50 -0800
Subject: [PATCH 0412/1095] Improves Flex performance.   - Avoid recreating the
 EagerOperation every time   - Avoid buffer_map churn by keeping the inputs
 and outputs along with the EagerOperation in OpNode.

PiperOrigin-RevId: 228409298
---
 tensorflow/lite/delegates/flex/kernel.cc      | 179 +++++++++++++-----
 tensorflow/lite/delegates/flex/kernel_test.cc |   7 +-
 2 files changed, 138 insertions(+), 48 deletions(-)

diff --git a/tensorflow/lite/delegates/flex/kernel.cc b/tensorflow/lite/delegates/flex/kernel.cc
index 84745ba9cb..d2853cf9f8 100644
--- a/tensorflow/lite/delegates/flex/kernel.cc
+++ b/tensorflow/lite/delegates/flex/kernel.cc
@@ -51,6 +51,15 @@ namespace tflite {
 namespace flex {
 namespace kernel {
 
+struct OpNode;
+
+// Represents the origin of a given tensor as a reference to the output
+// of an upstream node.
+struct TensorSource {
+  OpNode* node;
+  int node_output_index;
+};
+
 // A list of inputs of a given node of the TensorFlow/Eager graph.
 class OpInputs {
  public:
@@ -65,8 +74,26 @@ class OpInputs {
 
   int TfLiteIndex(int i) const { return inputs_[i]; }
 
+  // Given a map relating tensors to the node that originates them, populate a
+  // list of sources for the tensors in this class.
+  void InitializeTensorSources(
+      const std::map<int, TensorSource>& tflite_tensor_sources) {
+    sources_.clear();
+    for (int i : inputs_) {
+      auto it = tflite_tensor_sources.find(i);
+      if (it == tflite_tensor_sources.end()) {
+        sources_.push_back({nullptr, 0});
+      } else {
+        sources_.push_back(it->second);
+      }
+    }
+  }
+
+  TensorSource GetTensorSource(int i) const { return sources_[i]; }
+
  private:
   std::vector<int> inputs_;
+  std::vector<TensorSource> sources_;
 };
 
 // A list of outputs of a given node of the TensorFlow/Eager graph, along with
@@ -81,6 +108,33 @@ class OpOutputs {
   }
   ~OpOutputs() { ResetTensorHandles(); }
 
+  // Stores information about which of the tensors in this class are also
+  // outputs of the sugbraph.
+  void InitializeGraphOutputs(const std::set<int>& subgraph_outputs) {
+    subgraph_outputs_.clear();
+    for (int i : outputs_) {
+      subgraph_outputs_.push_back(subgraph_outputs.count(i) > 0);
+    }
+  }
+
+  // Returns true if the tensor given by index 'i' is an output of the entire
+  // subgraph.
+  bool IsSubgraphOutput(int i) const { return subgraph_outputs_[i]; }
+
+  // Returns a handle to a given tensor and, optionally, remove it from the
+  // internal vector.
+  tensorflow::TensorHandle* GetHandle(int i, bool remove) {
+    auto* handle = vector_[i];
+    if (!remove) {
+      handle->Ref();
+    } else {
+      // Don't increase the ref-count. Instead, simply take it out of the
+      // vector.
+      vector_[i] = nullptr;
+    }
+    return handle;
+  }
+
   int Size() const { return outputs_.size(); }
 
   int TfLiteIndex(int i) const { return outputs_[i]; }
@@ -102,6 +156,7 @@ class OpOutputs {
 
  private:
   std::vector<int> outputs_;
+  std::vector<bool> subgraph_outputs_;
   tensorflow::gtl::InlinedVector<tensorflow::TensorHandle*, 2> vector_;
 };
 
@@ -111,7 +166,9 @@ class OpNode {
  public:
   OpNode(const TfLiteIntArray* inputs, const TfLiteIntArray* outputs)
       : inputs_(inputs), outputs_(outputs) {}
-  ~OpNode() {}
+  ~OpNode() {
+    if (op_) ClearEagerInputs();
+  }
 
   const string& name() const { return name_; }
   void set_name(const string& name) { name_ = name; }
@@ -130,6 +187,8 @@ class OpNode {
   int NumInputs() const { return inputs_.Size(); }
   int NumOutputs() const { return outputs_.Size(); }
 
+  tensorflow::EagerOperation* op() { return op_.get(); }
+
   tensorflow::Status InitializeNodeDef(const void* custom_initial_data,
                                        int custom_initial_data_size) {
     if (!custom_initial_data) {
@@ -162,10 +221,8 @@ class OpNode {
 
   // Build thew new EagerOperation. In case of error, the returned 'op' is
   // guaranteed to be 'nullptr'.
-  tensorflow::Status BuildEagerOp(
-      tensorflow::EagerContext* eager_context,
-      std::unique_ptr<tensorflow::EagerOperation>* op) {
-    op->reset();
+  tensorflow::Status BuildEagerOp(tensorflow::EagerContext* eager_context) {
+    op_.reset();
 
     const tensorflow::AttrTypeMap* attr_types;
     bool is_function = false;
@@ -179,33 +236,47 @@ class OpNode {
           "')");
     }
 
-    op->reset(new tensorflow::EagerOperation(eager_context, name_.c_str(),
+    op_.reset(new tensorflow::EagerOperation(eager_context, name_.c_str(),
                                              /*is_function=*/false,
                                              attr_types));
+
+    op_->MutableAttrs()->NumInputs(inputs_.Size());
     for (const auto& attr : nodedef_.attr()) {
-      (*op)->MutableAttrs()->Set(attr.first, attr.second);
+      op_->MutableAttrs()->Set(attr.first, attr.second);
     }
 
     return tensorflow::Status::OK();
   }
 
-  tensorflow::Status BuildEagerInputs(BufferMap* buffer_map,
-                                      tensorflow::EagerOperation* op) {
+  void ClearEagerInputs() {
+    for (tensorflow::TensorHandle* h : *op_->MutableInputs()) {
+      if (h) h->Unref();
+    }
+    op_->MutableInputs()->clear();
+  }
+
+  tensorflow::Status BuildEagerInputs(const BufferMap* buffer_map) {
     for (int i = 0; i < inputs_.Size(); ++i) {
       int input_index = inputs_.TfLiteIndex(i);
-      if (!buffer_map->HasTensor(input_index)) {
-        return tensorflow::errors::Internal(
-            "Cannot read from invalid tensor index ", input_index);
-      }
-      auto* handle = new tensorflow::TensorHandle(
-          buffer_map->GetTensor(input_index), nullptr, nullptr, nullptr);
-      op->AddInput(handle);
-      handle->Unref();
-
-      if (buffer_map->IsForwardable(input_index)) {
-        // Take it out of the map, so Eager/TF can reuse the buffer for an
-        // output tensor of the op.
-        buffer_map->RemoveTensor(input_index);
+      TensorSource s = inputs_.GetTensorSource(i);
+      if (!s.node) {
+        // This input is not produced by this Eager subgraph (it could be a TF
+        // Lite native buffer, or could be produced by a separater subgraph). We
+        // need to fetch it from the delegate's buffer_map.
+        if (!buffer_map->HasTensor(input_index)) {
+          return tensorflow::errors::Internal(
+              "Cannot read from invalid tensor index ", input_index);
+        }
+        auto* handle = new tensorflow::TensorHandle(
+            buffer_map->GetTensor(input_index), nullptr, nullptr, nullptr);
+        op_->MutableInputs()->push_back(handle);
+      } else {
+        // If this is a forwardable tensor, we will remove it from the previous
+        // op's list, giving TF the opportunity to reuse its buffer.
+        bool unref_handle = buffer_map->IsForwardable(input_index);
+        auto* handle =
+            s.node->outputs_.GetHandle(s.node_output_index, unref_handle);
+        op_->MutableInputs()->push_back(handle);
       }
     }
     return tensorflow::Status::OK();
@@ -214,11 +285,12 @@ class OpNode {
   tensorflow::Status PersistEagerOutputs(BufferMap* buffer_map) {
     auto* handles = outputs_.GetTensorHandles();
     for (int i = 0; i < outputs_.Size(); ++i) {
-      const tensorflow::Tensor* tensor = nullptr;
-      TF_RETURN_IF_ERROR(handles->at(i)->Tensor(&tensor));
-      buffer_map->SetFromTensorFlow(outputs_.TfLiteIndex(i), *tensor);
+      if (outputs_.IsSubgraphOutput(i)) {
+        const tensorflow::Tensor* tensor = nullptr;
+        TF_RETURN_IF_ERROR(handles->at(i)->Tensor(&tensor));
+        buffer_map->SetFromTensorFlow(outputs_.TfLiteIndex(i), *tensor);
+      }
     }
-    outputs_.ResetTensorHandles();
     return tensorflow::Status::OK();
   }
 
@@ -236,19 +308,23 @@ class OpNode {
   OpInputs inputs_;
   // List of outputs, as TF Lite tensor indices.
   OpOutputs outputs_;
+
+  std::unique_ptr<tensorflow::EagerOperation> op_;
 };
 
 // Executes the TensorFlow op given by 'op_name', with the attributes specified
 // in 'nodedef'. Inputs and outputs are given as indices into the 'buffer_map'.
-tensorflow::Status ExecuteFlexOp(tensorflow::EagerContext* eager_context,
-                                 BufferMap* buffer_map, OpNode* node_data) {
-  std::unique_ptr<tensorflow::EagerOperation> op;
-  TF_RETURN_IF_ERROR(node_data->BuildEagerOp(eager_context, &op));
-  TF_RETURN_IF_ERROR(node_data->BuildEagerInputs(buffer_map, op.get()));
+tensorflow::Status ExecuteFlexOp(TfLiteContext* context, BufferMap* buffer_map,
+                                 OpNode* node_data) {
+  TF_RETURN_WITH_CONTEXT_IF_ERROR(node_data->BuildEagerInputs(buffer_map),
+                                  " (while executing '", node_data->name(),
+                                  "' via Eager)");
 
+  node_data->mutable_outputs()->ResetTensorHandles();
   int num_retvals = node_data->NumOutputs();
   TF_RETURN_WITH_CONTEXT_IF_ERROR(
-      EagerExecute(op.get(), node_data->mutable_outputs()->GetTensorHandles(),
+      EagerExecute(node_data->op(),
+                   node_data->mutable_outputs()->GetTensorHandles(),
                    &num_retvals),
       " (while executing '", node_data->name(), "' via Eager)");
 
@@ -259,6 +335,8 @@ tensorflow::Status ExecuteFlexOp(tensorflow::EagerContext* eager_context,
 
   TF_RETURN_IF_ERROR(node_data->PersistEagerOutputs(buffer_map));
 
+  node_data->ClearEagerInputs();
+
   return tensorflow::Status::OK();
 }
 
@@ -286,8 +364,10 @@ void* Init(TfLiteContext* context, const char* buffer, size_t length) {
                             ->GetBufferMap(context);
 
   CHECK(params->output_tensors);
+  std::set<int> output_set;
   for (auto tensor_index : TfLiteIntArrayView(params->output_tensors)) {
     op_data->subgraph_outputs.push_back(tensor_index);
+    output_set.insert(tensor_index);
   }
 
   CHECK(params->input_tensors);
@@ -313,6 +393,8 @@ void* Init(TfLiteContext* context, const char* buffer, size_t length) {
     status = node_data.InitializeNodeDef(node->custom_initial_data,
                                          node->custom_initial_data_size);
     if (!status.ok()) break;
+    status = node_data.BuildEagerOp(op_data->eager_context);
+    if (!status.ok()) break;
   }
 
   if (ConvertStatus(context, status) != kTfLiteOk) {
@@ -322,6 +404,26 @@ void* Init(TfLiteContext* context, const char* buffer, size_t length) {
     return op_data;
   }
 
+  // Given a TfLite tensor index, return the OpNode that produces it,
+  // along with it index into that OpNodes list of outputs.
+  std::map<int, TensorSource> tflite_tensor_sources;
+
+  // Find out how each tensor is produced. This does not account for
+  // tensors that are not produce by eager ops.
+  for (auto& node_data : op_data->nodes) {
+    node_data->mutable_outputs()->InitializeGraphOutputs(output_set);
+    for (int i = 0; i < node_data->outputs().Size(); ++i) {
+      int output_index = node_data->outputs().TfLiteIndex(i);
+      tflite_tensor_sources[output_index] = TensorSource{node_data.get(), i};
+    }
+  }
+
+  // For each node, resolve the inputs, so we can keep pointers to the nodes
+  // that produces them.
+  for (auto& node_data : op_data->nodes) {
+    node_data->mutable_inputs()->InitializeTensorSources(tflite_tensor_sources);
+  }
+
   return op_data;
 }
 
@@ -370,6 +472,7 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
                            node_data->name().c_str());
       return kTfLiteError;
     }
+    TF_LITE_ENSURE(context, node_data->op());
 
     for (int i = 0; i < node_data->inputs().Size(); ++i) {
       ++tensor_ref_count[node_data->inputs().TfLiteIndex(i)];
@@ -392,7 +495,6 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
 TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
   auto* op_data = reinterpret_cast<OpData*>(node->user_data);
   BufferMap* buffer_map = op_data->buffer_map;
-  tensorflow::EagerContext* eager_context = op_data->eager_context;
 
   // Insert a tensor in the buffer map for all inputs that are not constant.
   // Constants were handled in Prepare() already.
@@ -414,7 +516,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
         reinterpret_cast<profiling::Profiler*>(context->profiler),
         node_data->name().c_str(), node_data->index());
 
-    auto status = ExecuteFlexOp(eager_context, buffer_map, node_data.get());
+    auto status = ExecuteFlexOp(context, buffer_map, node_data.get());
     TF_LITE_ENSURE_OK(context, ConvertStatus(context, status));
   }
 
@@ -433,15 +535,6 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
     tensor->data_is_stale = true;
   }
 
-  // We don't need to keep track of internal TF tensors any longer, so take
-  // them out of the buffer_map, but make sure we keep all the ones we might
-  // need for other subgraphs, or as final output of inference.
-  const auto& outputs = op_data->subgraph_outputs;
-  std::set<int> keep(outputs.begin(), outputs.end());
-  const auto& inputs = op_data->subgraph_inputs;
-  keep.insert(inputs.begin(), inputs.end());
-  buffer_map->RemoveTensorsNotInSet(keep);
-
   return kTfLiteOk;
 }
 
diff --git a/tensorflow/lite/delegates/flex/kernel_test.cc b/tensorflow/lite/delegates/flex/kernel_test.cc
index cef8017e6e..5b3a6d1647 100644
--- a/tensorflow/lite/delegates/flex/kernel_test.cc
+++ b/tensorflow/lite/delegates/flex/kernel_test.cc
@@ -144,12 +144,9 @@ TEST_F(KernelTest, BadTensorFlowOp) {
     return GenericPrepare(context, delegate, {0});
   });
 
-  SetShape(0, {2, 2, 1});
-  SetValues(0, {1.1f, 2.2f, 3.3f, 4.4f});
-
-  ASSERT_FALSE(Invoke());
+  ASSERT_NE(interpreter_->AllocateTensors(), kTfLiteOk);
   ASSERT_THAT(error_reporter().error_messages(),
-              ContainsRegex("while processing attributes of 'NonExistentOp'"));
+              ContainsRegex("Op type not registered 'NonExistentOp'"));
 }
 
 TEST_F(KernelTest, BadNumberOfOutputs) {
-- 
GitLab


From 2c6316172f2c17929d34aa64df45f08210f08a83 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Tue, 8 Jan 2019 15:29:53 -0800
Subject: [PATCH 0413/1095] Add a new experimental initialize method only to
 TPU strategy

PiperOrigin-RevId: 228411389
---
 .../contrib/distribute/python/tpu_strategy.py     |  8 ++++++++
 tensorflow/python/training/session_manager.py     | 15 +++++++++------
 2 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index 9cb2e461ab..f498f58b25 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -455,6 +455,14 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
     with _TPUReplicaContext(self._container_strategy()):
       return fn(*args, **kwargs)
 
+  def _experimental_initialize_system(self):
+    """Experimental method added to be used by Estimator.
+
+    This is a private method only to be used by Estimator. Other frameworks
+    should directly be calling `tf.contrib.distribute.initialize_tpu_system`
+    """
+    initialize_tpu_system(self._tpu_cluster_resolver)
+
   def _initialize(self):
     if context.executing_eagerly():
       # TODO(priyag): Add appopriate call here when eager is supported for TPUs.
diff --git a/tensorflow/python/training/session_manager.py b/tensorflow/python/training/session_manager.py
index 1ec9639177..cd22e7e7de 100644
--- a/tensorflow/python/training/session_manager.py
+++ b/tensorflow/python/training/session_manager.py
@@ -182,13 +182,16 @@ class SessionManager(object):
         set.
     """
     self._target = master
+
+    # This is required to so that we initialize the TPU device before
+    # restoring from checkpoint since we'll be placing variables on the device
+    # and TPUInitialize wipes out the memory of the device.
+    strategy = distribution_strategy_context.get_distribution_strategy()
+    if strategy and hasattr(strategy.extended,
+                            "_experimental_initialize_system"):
+      strategy.extended._experimental_initialize_system()  # pylint: disable=protected-access
+
     sess = session.Session(self._target, graph=self._graph, config=config)
-    # TODO(jhseu): Delete once tpu.initialize_system() goes away.
-    initialize_ops = (
-        distribution_strategy_context.get_distribution_strategy().initialize()
-    )
-    if initialize_ops:
-      sess.run(initialize_ops)
     if checkpoint_dir and checkpoint_filename_with_path:
       raise ValueError("Can not provide both checkpoint_dir and "
                        "checkpoint_filename_with_path.")
-- 
GitLab


From d4985a621d8e71b128d34fec2a0260109bb4fb7d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 8 Jan 2019 15:32:32 -0800
Subject: [PATCH 0414/1095] This CL isolates collections between instances of
 wrap_function calls.

PiperOrigin-RevId: 228411886
---
 tensorflow/python/eager/wrap_function.py      |  3 +-
 tensorflow/python/eager/wrap_function_test.py | 39 +++++++++++++++++++
 tensorflow/python/framework/func_graph.py     | 34 ++++++++++------
 tensorflow/python/ops/cond_v2.py              |  7 ++--
 tensorflow/python/ops/while_v2.py             | 12 +++---
 5 files changed, 74 insertions(+), 21 deletions(-)

diff --git a/tensorflow/python/eager/wrap_function.py b/tensorflow/python/eager/wrap_function.py
index 2c65e97871..6f978feb3c 100644
--- a/tensorflow/python/eager/wrap_function.py
+++ b/tensorflow/python/eager/wrap_function.py
@@ -197,6 +197,7 @@ def wrap_function(fn, signature, name=None):
           name,
           holder,
           args=None, kwargs=None, signature=signature,
-          add_control_dependencies=False),
+          add_control_dependencies=False,
+          collections={}),
       variable_holder=holder,
       signature=signature)
diff --git a/tensorflow/python/eager/wrap_function_test.py b/tensorflow/python/eager/wrap_function_test.py
index 17e204b4d8..a6e1931fcd 100644
--- a/tensorflow/python/eager/wrap_function_test.py
+++ b/tensorflow/python/eager/wrap_function_test.py
@@ -104,6 +104,45 @@ class WrapFunctionTest(test.TestCase):
         fetches=f_wrapped.graph.get_tensor_by_name('fetch:0'))
     self.assertAllEqual(6.0, pruned())
 
+  def testCollectionsIsolation(self):
+
+    v1 = variables.Variable(2.)
+    v2_holder = []
+    def f():
+      v2 = variables.Variable(3.)
+      v2_holder.append(v2)
+      ops.add_to_collection(ops.GraphKeys.LOSSES, v2 * constant_op.constant(3.))
+      return array_ops.identity(v1 * v2 * constant_op.constant(1.), 'fetch')
+
+    f_wrapped = wrap_function.wrap_function(f, [])
+    self.assertAllEqual(6.0, f_wrapped())
+    self.assertEqual(
+        len(f_wrapped.graph.get_collection(ops.GraphKeys.LOSSES)), 1)
+    f_var_collection = f_wrapped.graph.get_collection(
+        ops.GraphKeys.TRAINABLE_VARIABLES)
+    self.assertEqual(len(f_var_collection), 1)
+    self.assertIs(f_var_collection[0], v2_holder[0])
+
+    v3_holder = []
+    def g():
+      v3 = variables.Variable(4.)
+      v3_holder.append(v3)
+      ops.add_to_collection(ops.GraphKeys.LOSSES, v3 * constant_op.constant(3.))
+      return array_ops.identity(v1 * v3 * constant_op.constant(1.), 'fetch')
+
+    g_wrapped = wrap_function.wrap_function(g, [])
+    self.assertAllEqual(8.0, g_wrapped())
+    self.assertEqual(
+        len(g_wrapped.graph.get_collection(ops.GraphKeys.LOSSES)), 1)
+    g_var_collection = g_wrapped.graph.get_collection(
+        ops.GraphKeys.TRAINABLE_VARIABLES)
+    self.assertEqual(len(g_var_collection), 1)
+    self.assertIs(g_var_collection[0], v3_holder[0])
+
+    # Both have only one value, and their values aren't equal. So no sharing.
+    self.assertNotEqual(g_wrapped.graph.get_collection(ops.GraphKeys.LOSSES),
+                        f_wrapped.graph.get_collection(ops.GraphKeys.LOSSES))
+
   def testGradientsOfPrune(self):
 
     v1 = variables.Variable(2.)
diff --git a/tensorflow/python/framework/func_graph.py b/tensorflow/python/framework/func_graph.py
index 9528a24b46..9603c1536d 100644
--- a/tensorflow/python/framework/func_graph.py
+++ b/tensorflow/python/framework/func_graph.py
@@ -18,7 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import collections
+import collections as py_collections
 import weakref
 
 from tensorflow.core.framework import attr_value_pb2
@@ -80,7 +80,7 @@ class FuncGraph(ops.Graph):
     seed: The graph-level random seed.
   """
 
-  def __init__(self, name, read_only_collections=True):
+  def __init__(self, name, collections=None):
     """Construct a new FuncGraph.
 
     The graph will inherit its graph key, collections, seed, and distribution
@@ -88,8 +88,13 @@ class FuncGraph(ops.Graph):
 
     Args:
       name: the name of the function.
-      read_only_collections: whether to not write function graph collections
-        back to default graph. Defaults to True.
+      collections: a dictionary of collections this FuncGraph should start
+        with. If not specified (None), the FuncGraph will read (but not write
+        to) the outer graph's collections that are not whitelisted, and both
+        read and write to the outer graph's collections that are whitelisted.
+        The current whitelisted collections are the global variables, the
+        local variables, and the trainable variables.
+        Defaults to None.
     """
     super(FuncGraph, self).__init__()
 
@@ -97,10 +102,9 @@ class FuncGraph(ops.Graph):
     self.inputs = []
     self.outputs = []
     self.structured_outputs = None
-    self._read_only_collections = read_only_collections
     self._weak_variables = []
     self.outer_graph = ops.get_default_graph()
-    self.captures = collections.OrderedDict()
+    self.captures = py_collections.OrderedDict()
 
     self._building_function = True
     # Map from resource tensor name to last op (in program order) which uses
@@ -122,9 +126,7 @@ class FuncGraph(ops.Graph):
       # specialization (currently used in cond_v2), here and in the cache key.
       self._colocation_stack = graph._colocation_stack.copy()  # pylint: disable=protected-access
 
-    if not self._read_only_collections:
-      self._collections = graph._collections  # pylint: disable=protected-access
-    else:
+    if collections is None:
       for collection_name in graph.get_all_collection_keys():
         if collection_name not in WHITELIST_COLLECTIONS:
           self._collections[collection_name] = graph.get_collection(
@@ -132,6 +134,8 @@ class FuncGraph(ops.Graph):
       for collection_name in WHITELIST_COLLECTIONS:
         self._collections[collection_name] = graph.get_collection_ref(
             collection_name)
+    else:
+      self._collections = collections
 
   def as_default(self):
     outer_cm = super(FuncGraph, self).as_default()
@@ -338,7 +342,8 @@ def func_graph_from_py_func(name,
                             autograph=False,
                             add_control_dependencies=True,
                             arg_names=None,
-                            op_return_value=None):
+                            op_return_value=None,
+                            collections=None):
   """Returns a `FuncGraph` generated from `python_func`.
 
   Args:
@@ -365,6 +370,13 @@ def func_graph_from_py_func(name,
     op_return_value: Optional. A Tensor. If set and `python_func` returns
       Operations, those return values will be replaced with this value. If not
       set, returning an Operation triggers an error.
+    collections: a dictionary of collections this FuncGraph should start
+      with. If not specified (None), the FuncGraph will read (but not write to)
+      the outer graph's collections that are not whitelisted, and both
+      read and write to the outer graph's collections that are whitelisted.
+      The current whitelisted collections are the global variables, the
+      local variables, and the trainable variables.
+      Defaults to None.
 
   Returns:
     A FuncGraph.
@@ -376,7 +388,7 @@ def func_graph_from_py_func(name,
   if op_return_value is not None:
     assert isinstance(op_return_value, ops.Tensor), op_return_value
   if func_graph is None:
-    func_graph = FuncGraph(name)
+    func_graph = FuncGraph(name, collections=collections)
   assert isinstance(func_graph, FuncGraph)
   if add_control_dependencies:
     control_manager = AutomaticControlDependencies
diff --git a/tensorflow/python/ops/cond_v2.py b/tensorflow/python/ops/cond_v2.py
index 7d09e32e24..5dca8e501b 100644
--- a/tensorflow/python/ops/cond_v2.py
+++ b/tensorflow/python/ops/cond_v2.py
@@ -68,14 +68,14 @@ def cond_v2(pred, true_fn, false_fn, name="cond"):
         true_name,
         true_fn, [], {},
         func_graph=util.CondBranchFuncGraph(
-            true_name, read_only_collections=False),
+            true_name, collections=ops.get_default_graph()._collections),  # pylint: disable=protected-access
         add_control_dependencies=add_control_dependencies,
         op_return_value=pred)
     false_graph = func_graph_module.func_graph_from_py_func(
         false_name,
         false_fn, [], {},
         func_graph=util.CondBranchFuncGraph(
-            false_name, read_only_collections=False),
+            false_name, collections=ops.get_default_graph()._collections),  # pylint: disable=protected-access
         add_control_dependencies=add_control_dependencies,
         op_return_value=pred)
 
@@ -554,7 +554,8 @@ class _CondGradFuncGraph(util.CondBranchFuncGraph):
   """
 
   def __init__(self, name, forward_graph):
-    super(_CondGradFuncGraph, self).__init__(name, read_only_collections=False)
+    super(_CondGradFuncGraph, self).__init__(
+        name, collections=ops.get_default_graph()._collections)  # pylint: disable=protected-access
     self.if_op_needs_rewrite = False
     self._forward_graph = forward_graph
     # Maps from forward intermediate tensor -> the unwrapped captured
diff --git a/tensorflow/python/ops/while_v2.py b/tensorflow/python/ops/while_v2.py
index f9fea0e5bd..f5a51bb1bc 100644
--- a/tensorflow/python/ops/while_v2.py
+++ b/tensorflow/python/ops/while_v2.py
@@ -115,15 +115,15 @@ def while_loop(cond,
             loop_counter < maximum_iterations,
             cond(*_pack_sequence_as(orig_loop_vars, args)))
 
-    # NOTE(skyewm): we set read_only_collections=False for compatibility with
-    # TPUEstimator.
+    # NOTE(skyewm): we set collections to the outer graph's collections for
+    # compatibility with TPUEstimator.
     cond_graph = func_graph_module.func_graph_from_py_func(
         cond_name,
         wrapped_cond,
         loop_vars, {},
         signature=_build_signature(loop_vars, shape_invariants),
-        func_graph=util.WhileCondFuncGraph(cond_name,
-                                           read_only_collections=False),
+        func_graph=util.WhileCondFuncGraph(
+            cond_name, collections=ops.get_default_graph()._collections),  # pylint: disable=protected-access
         add_control_dependencies=add_control_dependencies)
 
     # Add external_captures of cond to the list of loop vars.
@@ -174,8 +174,8 @@ def while_loop(cond,
         wrapped_body,
         loop_vars, {},
         signature=_build_signature(loop_vars, shape_invariants),
-        func_graph=util.WhileBodyFuncGraph(body_name,
-                                           read_only_collections=False),
+        func_graph=util.WhileBodyFuncGraph(
+            body_name, collections=ops.get_default_graph()._collections),  # pylint: disable=protected-access
         add_control_dependencies=add_control_dependencies)
     # Add external captures of body to the list of loop vars.
     # Note that external tensors will be treated as loop invariants, i.e.,
-- 
GitLab


From f71e77471c140bac2f4dcb9a003139fb5c657302 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Tue, 8 Jan 2019 15:35:49 -0800
Subject: [PATCH 0415/1095] Don't check for alignment before calling into a
 tiled dot (Eigen or LLVM)

The alignment restrictions on Eigen was lifted in cr/226091175.  This CL lifts
the alignment restriction on the tiled LLVM IR GEMM.

PiperOrigin-RevId: 228412524
---
 tensorflow/compiler/xla/service/cpu/BUILD     |   2 -
 .../cpu/cpu_eigen_tensor_alignment_test.cc    |  38 ------
 .../xla/service/cpu/dot_op_emitter.cc         | 117 ++++++++++++------
 .../xla/service/cpu/dot_op_emitter_internal.h |  88 -------------
 4 files changed, 77 insertions(+), 168 deletions(-)
 delete mode 100644 tensorflow/compiler/xla/service/cpu/dot_op_emitter_internal.h

diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD
index de62aa60aa..a197bdddc8 100644
--- a/tensorflow/compiler/xla/service/cpu/BUILD
+++ b/tensorflow/compiler/xla/service/cpu/BUILD
@@ -385,7 +385,6 @@ cc_library(
     srcs = ["dot_op_emitter.cc"],
     hdrs = [
         "dot_op_emitter.h",
-        "dot_op_emitter_internal.h",
     ],
     deps = [
         ":cpu_options",
@@ -1029,7 +1028,6 @@ tf_cc_test(
     size = "small",
     srcs = ["cpu_eigen_tensor_alignment_test.cc"],
     deps = [
-        ":dot_op_emitter",
         ":ir_emission_utils",
         ":target_machine_features_fake",
         "//tensorflow/compiler/xla:test",
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_eigen_tensor_alignment_test.cc b/tensorflow/compiler/xla/service/cpu/cpu_eigen_tensor_alignment_test.cc
index 823bdf259c..485769a373 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_eigen_tensor_alignment_test.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_eigen_tensor_alignment_test.cc
@@ -13,7 +13,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/compiler/xla/service/cpu/dot_op_emitter_internal.h"
 #include "tensorflow/compiler/xla/service/cpu/ir_emission_utils.h"
 #include "tensorflow/compiler/xla/service/cpu/target_machine_features_fake.h"
 #include "tensorflow/compiler/xla/service/hlo_parser.h"
@@ -23,48 +22,11 @@ namespace xla {
 namespace cpu {
 namespace {
 
-using internal::DotImplementationStrategy;
-using internal::DotInfo;
-using internal::GetDotImplementationStrategy;
-
 // Test that we don't call into Eigen with tensors too small to be aligned
 // reliably.
 
 class CpuEigenTensorAlignmentTest : public ::testing::Test {};
 
-TEST_F(CpuEigenTensorAlignmentTest, EigenDotAlignment) {
-  string hlo_string = R"(
-HloModule DotOperation
-
-ENTRY DotOperation {
-  arg0 = f32[5,256] parameter(0)
-  arg1 = f32[256,1024] parameter(1)
-  ROOT dot = f32[5,1024] dot(arg0, arg1), lhs_contracting_dims={1}, rhs_contracting_dims={0}
-}
-)";
-
-  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
-                          ParseHloString(hlo_string));
-
-  HloInstruction* dot = module->entry_computation()->root_instruction();
-
-  TargetMachineFeaturesWithFakeAlignmentLogic target_machine_with_no_alignment(
-      [](int64 size) { return 1; });
-
-  EXPECT_EQ(GetDotImplementationStrategy(HloModuleConfig{}, DotInfo(*dot),
-                                         target_machine_with_no_alignment),
-            DotImplementationStrategy::kNaiveLlvmIr);
-
-  TargetMachineFeaturesWithFakeAlignmentLogic
-      target_machine_with_full_alignment([](int64 size) {
-        return TargetMachineFeatures::kEigenExpectedTensorAlignment;
-      });
-
-  EXPECT_NE(GetDotImplementationStrategy(HloModuleConfig{}, DotInfo(*dot),
-                                         target_machine_with_full_alignment),
-            DotImplementationStrategy::kNaiveLlvmIr);
-}
-
 TEST_F(CpuEigenTensorAlignmentTest, EigenConvAlignment) {
   string hlo_string = R"(
 HloModule ConvOperation
diff --git a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
index e8a84ebe6b..b018e0cd46 100644
--- a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
@@ -14,7 +14,6 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/compiler/xla/service/cpu/dot_op_emitter.h"
-#include "tensorflow/compiler/xla/service/cpu/dot_op_emitter_internal.h"
 
 #include <memory>
 #include <vector>
@@ -43,17 +42,63 @@ namespace xla {
 using llvm_ir::SetToFirstInsertPoint;
 
 namespace cpu {
-
-using internal::DotImplementationStrategy;
-using internal::DotInfo;
-using internal::GetDotImplementationStrategy;
-
 namespace {
 // Returns true if we should call into multi-threaded Eigen routines.
 bool ShouldUseMultiThreadedEigen(const HloModuleConfig& config) {
   return config.debug_options().xla_cpu_multi_thread_eigen();
 }
 
+// Represents a dot operation.  We use this in lieu of an `HloInstruction`
+// because we want to be able to create this for the "inner" dot operation in a
+// batch dot, for which there is no separate HLO instruction.
+struct DotInfo {
+  Shape lhs_shape;
+  Shape rhs_shape;
+  Shape result_shape;
+  DotDimensionNumbers dim_nums;
+
+  explicit DotInfo(const HloInstruction& instr) {
+    CHECK_EQ(instr.opcode(), HloOpcode::kDot);
+    lhs_shape = instr.operand(0)->shape();
+    rhs_shape = instr.operand(1)->shape();
+    result_shape = instr.shape();
+    dim_nums = instr.dot_dimension_numbers();
+  }
+};
+
+// Dictates how a dot operation is implemented.
+enum class DotImplementationStrategy {
+  // The dot operation is lowered into LLVM IR that implements a naive nested
+  // loop that computes the result one element at a time.  This is our
+  // "fallback"; we don't really want this to kick in for any non-trival dot
+  // operation.
+  kNaiveLlvmIr,
+
+  // The dot operation is lowered into LLVM IR that implements a tiled
+  // Matrix*Vector operation.  This strategy also allows fusing in a bias add
+  // into the dot.  The matrix can be row major or column major, both are
+  // supported.
+  kTiledLlvmIrGemv,
+
+  // The dot operation is lowered into LLVM IR that implemetns a tiled
+  // Matrix*Matrix operation.  No fusions are supported.  The two inputs
+  // and the output have to be row major.
+  kTiledLlvmIrGemm,
+
+  // The dot operation is lowered into a call into an Eigen routine.  No fusions
+  // are supported today.  The two inputs and the output have to be row major.
+  // However, we do allow transposing either the LHS or the RHS as part of the
+  // GEMM -- we expose this flexibility as flexibility in the contraction
+  // dimensions, but we can also see this as flexibility in the input layouts.
+  kEigen,
+};
+
+// Returns the implementation strategy for a dot with the configuration
+// `dot_info`.
+DotImplementationStrategy GetDotImplementationStrategy(
+    const HloModuleConfig& config, const DotInfo& dot_info,
+    const TargetMachineFeatures& target_machine_features);
+
 // Helper class for emitting LLVM IR to perform the dot operation.
 class DotOpEmitter {
  public:
@@ -190,9 +235,8 @@ void DotOpEmitter::EmitTiledLlvmIrGemm() {
   }
 
   int64 size_bytes = m * n * ShapeUtil::ByteSizeOfPrimitiveType(primitive_type);
-  b_->CreateMemSet(
-      target, b_->getInt8(0), size_bytes,
-      target_machine_features_.minimum_alignment_for_allocation(size_bytes));
+  b_->CreateMemSet(target, b_->getInt8(0), /*Size=*/size_bytes,
+                   /*Align=*/1);
 
   int64 max_target_vector_width =
       target_machine_features_.vector_register_num_elements(
@@ -696,40 +740,34 @@ absl::optional<int64> ProfitableToMakeDotOperandColumnMajor(
   return {};
 }
 
-namespace internal {
 namespace {
 // Return whether the given shape is rank 2.
 bool IsRank2(const Shape& shape) { return shape.rank() == 2; }
 
+bool IsSimpleLayout(const Layout& layout) {
+  return layout.tiles().empty() && layout.format() == DENSE;
+}
+
 // In a gemm operation where output = lhs * rhs, check whether the given shapes
 // are valid for the operation.
-bool AreAlignedGemmShapes(
-    const Shape& lhs_shape, const Shape& rhs_shape, const Shape& output_shape,
-    const TargetMachineFeatures& target_machine_features) {
-  // The inputs and the output must
-  // 1) be matrices with no padding, and
-  // 2) have an allowed element type.
-  PrimitiveType output_primitive_type = output_shape.element_type();
-  if (!(output_primitive_type == F64 || output_primitive_type == F32 ||
-        output_primitive_type == F16)) {
-    return false;
-  }
-
-  if (!(IsRank2(lhs_shape) && IsRank2(rhs_shape) && IsRank2(output_shape))) {
-    return false;
-  }
-
-  auto is_aligned = [&](const Shape& shape) {
-    return GetMinimumAlignmentForArray(shape, target_machine_features) >=
-           TargetMachineFeatures::kEigenExpectedTensorAlignment;
-  };
-
-  if (!is_aligned(lhs_shape) || !is_aligned(rhs_shape) ||
-      !is_aligned(output_shape)) {
-    return false;
+bool AreGemmShapes(const Shape& lhs_shape, const Shape& rhs_shape,
+                   const Shape& output_shape,
+                   const TargetMachineFeatures& target_machine_features) {
+  CHECK(!lhs_shape.has_layout() || IsSimpleLayout(lhs_shape.layout()))
+      << lhs_shape.DebugString();
+  CHECK(!rhs_shape.has_layout() || IsSimpleLayout(rhs_shape.layout()))
+      << rhs_shape.DebugString();
+  CHECK(!output_shape.has_layout() || IsSimpleLayout(output_shape.layout()))
+      << output_shape.DebugString();
+
+  switch (output_shape.element_type()) {
+    case F64:
+    case F32:
+    case F16:
+      return IsRank2(lhs_shape) && IsRank2(rhs_shape) && IsRank2(output_shape);
+    default:
+      return false;
   }
-
-  return true;
 }
 
 bool IsAlignedGemm(const DotInfo& dot_info,
@@ -739,8 +777,8 @@ bool IsAlignedGemm(const DotInfo& dot_info,
     return false;
   }
 
-  return AreAlignedGemmShapes(dot_info.lhs_shape, dot_info.rhs_shape,
-                              dot_info.result_shape, target_machine_features);
+  return AreGemmShapes(dot_info.lhs_shape, dot_info.rhs_shape,
+                       dot_info.result_shape, target_machine_features);
 }
 
 bool CanEmitTiledLlvmIrGemm(
@@ -781,7 +819,6 @@ bool CanEmitTiledLlvmIrGemm(
 
   return true;
 }
-}  // namespace
 
 DotImplementationStrategy GetDotImplementationStrategy(
     const HloModuleConfig& config, const DotInfo& dot_info,
@@ -806,7 +843,7 @@ DotImplementationStrategy GetDotImplementationStrategy(
 
   return DotImplementationStrategy::kNaiveLlvmIr;
 }
-}  // namespace internal
+}  // namespace
 
 bool DotImplementationCanHandleTranspose(
     const HloInstruction& dot_instr,
diff --git a/tensorflow/compiler/xla/service/cpu/dot_op_emitter_internal.h b/tensorflow/compiler/xla/service/cpu/dot_op_emitter_internal.h
deleted file mode 100644
index cc28918ed6..0000000000
--- a/tensorflow/compiler/xla/service/cpu/dot_op_emitter_internal.h
+++ /dev/null
@@ -1,88 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_CPU_DOT_OP_EMITTER_INTERNAL_H_
-#define TENSORFLOW_COMPILER_XLA_SERVICE_CPU_DOT_OP_EMITTER_INTERNAL_H_
-
-#include "tensorflow/compiler/xla/service/cpu/target_machine_features.h"
-#include "tensorflow/compiler/xla/service/hlo_instruction.h"
-#include "tensorflow/compiler/xla/service/hlo_module.h"
-
-// -----------------------------------------------------------------------------
-// INTERNAL HEADER.
-//
-// This file exposes internal implementation details from dot_op_emitter.cc for
-// unit tests.  Please do not depend on this!
-//
-// -----------------------------------------------------------------------------
-
-namespace xla {
-namespace cpu {
-namespace internal {
-
-// Represents a dot operation.  We use this in lieu of an `HloInstruction`
-// because we want to be able to create this for the "inner" dot operation in a
-// batch dot, for which there is no separate HLO instruction.
-struct DotInfo {
-  Shape lhs_shape;
-  Shape rhs_shape;
-  Shape result_shape;
-  DotDimensionNumbers dim_nums;
-
-  explicit DotInfo(const HloInstruction& instr) {
-    CHECK_EQ(instr.opcode(), HloOpcode::kDot);
-    lhs_shape = instr.operand(0)->shape();
-    rhs_shape = instr.operand(1)->shape();
-    result_shape = instr.shape();
-    dim_nums = instr.dot_dimension_numbers();
-  }
-};
-
-// Dictates how a dot operation is implemented.
-enum class DotImplementationStrategy {
-  // The dot operation is lowered into LLVM IR that implements a naive nested
-  // loop that computes the result one element at a time.  This is our
-  // "fallback"; we don't really want this to kick in for any non-trival dot
-  // operation.
-  kNaiveLlvmIr,
-
-  // The dot operation is lowered into LLVM IR that implements a tiled
-  // Matrix*Vector operation.  This strategy also allows fusing in a bias add
-  // into the dot.  The matrix can be row major or column major, both are
-  // supported.
-  kTiledLlvmIrGemv,
-
-  // The dot operation is lowered into LLVM IR that implemetns a tiled
-  // Matrix*Matrix operation.  No fusions are supported.  The two inputs
-  // and the output have to be row major.
-  kTiledLlvmIrGemm,
-
-  // The dot operation is lowered into a call into an Eigen routine.  No fusions
-  // are supported today.  The two inputs and the output have to be row major.
-  // However, we do allow transposing either the LHS or the RHS as part of the
-  // GEMM -- we expose this flexibility as flexibility in the contraction
-  // dimensions, but we can also see this as flexibility in the input layouts.
-  kEigen,
-};
-
-// Returns the implementation strategy for a dot with the configuration
-// `dot_info`.
-DotImplementationStrategy GetDotImplementationStrategy(
-    const HloModuleConfig& config, const DotInfo& dot_info,
-    const TargetMachineFeatures& target_machine_features);
-}  // namespace internal
-}  // namespace cpu
-}  // namespace xla
-
-#endif  // TENSORFLOW_COMPILER_XLA_SERVICE_CPU_DOT_OP_EMITTER_INTERNAL_H_
-- 
GitLab


From cb0ecda8555f827c4fa4d9c160c68ea028bdc305 Mon Sep 17 00:00:00 2001
From: Tim Shen <timshen@google.com>
Date: Tue, 8 Jan 2019 15:41:12 -0800
Subject: [PATCH 0416/1095] Unbreak windows builds.

cuda_deps seems not to work well with select() on windows.

PiperOrigin-RevId: 228413463
---
 .../core/platform/default/build_config/BUILD     | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/tensorflow/core/platform/default/build_config/BUILD b/tensorflow/core/platform/default/build_config/BUILD
index f5a194428c..c0c1022c16 100644
--- a/tensorflow/core/platform/default/build_config/BUILD
+++ b/tensorflow/core/platform/default/build_config/BUILD
@@ -32,15 +32,7 @@ cc_library(
 
 tf_cuda_library(
     name = "stream_executor",
-    cuda_deps = [
-        "//tensorflow/stream_executor/cuda:cuda_activation",
-    ] + select({
-        "//tensorflow:using_cuda_clang": ["//tensorflow/stream_executor/cuda:all_runtime"],
-        "//tensorflow:using_cuda_nvcc": ["//tensorflow/stream_executor/cuda:all_runtime"],
-        "//tensorflow:using_cuda_clang_with_dynamic_build": [],
-        "//tensorflow:using_cuda_nvcc_with_dynamic_build": [],
-        "//conditions:default": [],
-    }),
+    cuda_deps = ["//tensorflow/stream_executor/cuda:cuda_activation"],
     deps = [
         "//tensorflow/stream_executor",
         "//tensorflow/stream_executor:dnn",
@@ -53,6 +45,12 @@ tf_cuda_library(
     ] + select({
         "@local_config_cuda//cuda:darwin": ["IOKit"],
         "//conditions:default": [],
+    }) + select({
+        "//tensorflow:using_cuda_clang": ["//tensorflow/stream_executor/cuda:all_runtime"],
+        "//tensorflow:using_cuda_nvcc": ["//tensorflow/stream_executor/cuda:all_runtime"],
+        "//tensorflow:using_cuda_clang_with_dynamic_build": [],
+        "//tensorflow:using_cuda_nvcc_with_dynamic_build": [],
+        "//conditions:default": [],
     }),
 )
 
-- 
GitLab


From 29579a2e548800994f1acdf37fc2f8bf42b043ee Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Tue, 8 Jan 2019 15:50:48 -0800
Subject: [PATCH 0417/1095] Fix tf_driver_test

Use the proper path for the tested model.

PiperOrigin-RevId: 228415121
---
 tensorflow/lite/testing/tf_driver_test.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/testing/tf_driver_test.cc b/tensorflow/lite/testing/tf_driver_test.cc
index 363d162d56..e79704d616 100644
--- a/tensorflow/lite/testing/tf_driver_test.cc
+++ b/tensorflow/lite/testing/tf_driver_test.cc
@@ -93,7 +93,7 @@ TEST(TfDriverTest, SimpleTest) {
                    {"1,8,8,3", "1,8,8,3", "1,8,8,3", "1,8,8,3"}, {"x", "y"}));
 
   runner->LoadModel(
-      "third_party/tensorflow/lite/testdata/multi_add.pb");
+      "tensorflow/lite/testdata/multi_add.pb");
   EXPECT_TRUE(runner->IsValid()) << runner->GetErrorMessage();
 
   ASSERT_THAT(runner->GetInputs(), ElementsAre(0, 1, 2, 3));
-- 
GitLab


From 0b53b291f31d9ac9d1bb9cffbc037fe978f593fd Mon Sep 17 00:00:00 2001
From: Reed Wanderman-Milne <reedwm@google.com>
Date: Tue, 8 Jan 2019 15:57:52 -0800
Subject: [PATCH 0418/1095] Automated rollback of changelist 205730535

PiperOrigin-RevId: 228416180
---
 tensorflow/core/BUILD                         | 11 ++++
 .../core/common_runtime/gpu/gpu_device.cc     | 51 ++++++++++++++++++-
 .../core/common_runtime/gpu/gpu_device.h      | 11 ++++
 .../gpu/gpu_device_kernel_check.cu.cc         | 37 ++++++++++++++
 .../gpu/gpu_device_kernel_check.h             | 32 ++++++++++++
 5 files changed, 141 insertions(+), 1 deletion(-)
 create mode 100644 tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.cu.cc
 create mode 100644 tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.h

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index e6af9211b5..6a842a432f 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -87,6 +87,7 @@ load(
     "tf_gen_op_libs",
     "tf_generate_proto_text_sources",
     "tf_genrule_cmd_append_to_srcs",
+    "tf_gpu_kernel_library",
     "tf_opts_nortti_if_android",
     "transitive_hdrs",
 )
@@ -3162,6 +3163,15 @@ cc_library(
     ],
 )
 
+tf_gpu_kernel_library(
+    name = "gpu_device_kernel_check",
+    srcs = ["common_runtime/gpu/gpu_device_kernel_check.cu.cc"],
+    hdrs = ["common_runtime/gpu/gpu_device_kernel_check.h"],
+    deps = [
+        "//tensorflow/core:stream_executor",
+    ],
+)
+
 GPU_RUNTIME_HEADERS = [
     "common_runtime/gpu/cuda_host_allocator.h",
     "common_runtime/gpu/gpu_bfc_allocator.h",
@@ -3200,6 +3210,7 @@ tf_cuda_library(
         ":core_cpu_lib",
         ":framework",
         ":framework_internal",
+        ":gpu_device_kernel_check",
         ":gpu_id_impl",
         ":gpu_init_impl",
         ":gpu_lib",
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device.cc b/tensorflow/core/common_runtime/gpu/gpu_device.cc
index 010fdff4e9..891b6e0e2a 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_device.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_device.cc
@@ -31,6 +31,7 @@ limitations under the License.
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/common_runtime/device_factory.h"
+#include "tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.h"
 #include "tensorflow/core/common_runtime/gpu/gpu_event_mgr.h"
 #include "tensorflow/core/common_runtime/gpu/gpu_id.h"
 #include "tensorflow/core/common_runtime/gpu/gpu_id_manager.h"
@@ -388,7 +389,7 @@ Status BaseGPUDevice::Init(const SessionOptions& options) {
     }
   }
 
-  return Status::OK();
+  return CheckGPU();
 }
 
 bool BaseGPUDevice::RequiresRecordingAccessedTensors() const {
@@ -907,6 +908,54 @@ Allocator* BaseGPUDevice::GetScopedAllocator(AllocatorAttributes attr,
   return gpu_allocator_;
 }
 
+Status BaseGPUDevice::CheckGPU() {
+  se::Stream* stream = tensorflow_gpu_device_info()->stream;
+  TF_RETURN_IF_ERROR(stream->BlockHostUntilDone());
+  Tensor device_tensor(gpu_allocator_, DT_FLOAT, {});
+  if (!device_tensor.IsInitialized()) {
+    return errors::ResourceExhausted("Failed to allocate ", sizeof(float),
+                                     " bytes on the GPU for initialization "
+                                     "checks");
+  }
+  float* val_dev = device_tensor.scalar<float>().data();
+  const cudaStream_t cu_stream = *reinterpret_cast<const cudaStream_t*>(
+      stream->implementation()->GpuStreamMemberHack());
+  {
+    se::cuda::ScopedActivateExecutorContext scoped_activation{stream->parent()};
+    run_test_kernel(val_dev, cu_stream);
+    // We have to use the CUDA runtime function cudaPeekAtLastError here,
+    // because 'stream' does not provide a way to check if a kernel launch
+    // succeeds. Calling 'stream->BlockHostUntilDone()', which internally calls
+    // 'cuCtxSynchronize()', does not catch all kernel launch errors.
+    cudaError_t cuda_error = cudaPeekAtLastError();
+    if (cuda_error == cudaSuccess) {
+      cuda_error = cudaDeviceSynchronize();
+    }
+    TF_RETURN_IF_ERROR(CudaErrorToStatus(cuda_error, *stream));
+  }
+
+  float val_host = 0.;
+  stream->ThenMemcpy(&val_host, se::DeviceMemoryBase(val_dev, sizeof(float)),
+                     sizeof(float));
+  TF_RETURN_IF_ERROR(stream->BlockHostUntilDone());
+  if (val_host != 12345.) {
+    return errors::Internal(
+        "GPU kernel for initialization returned wrong value: ", val_host);
+  }
+  return Status::OK();
+}
+
+Status BaseGPUDevice::CudaErrorToStatus(cudaError_t cuda_error,
+                                        const se::Stream& stream) {
+  if (cuda_error != cudaSuccess) {
+    return errors::Internal(
+        "Failed to run GPU kernel for the initialization check. Received "
+        "error ",
+        cudaGetErrorName(cuda_error), " after running GPU kernel.");
+  }
+  return Status::OK();
+}
+
 const int BaseGPUDeviceFactory::InterconnectMap::kSameDeviceStrength = 1000;
 const int BaseGPUDeviceFactory::InterconnectMap::kStreamExecutorStrength = 1;
 
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device.h b/tensorflow/core/common_runtime/gpu/gpu_device.h
index d002d02c51..86622dad49 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_device.h
+++ b/tensorflow/core/common_runtime/gpu/gpu_device.h
@@ -26,6 +26,7 @@ limitations under the License.
 #include <vector>
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "cuda/include/cuda_runtime_api.h"
 #include "tensorflow/core/common_runtime/device_factory.h"
 #include "tensorflow/core/common_runtime/gpu/gpu_event_mgr.h"
 #include "tensorflow/core/common_runtime/gpu/gpu_id.h"
@@ -121,6 +122,12 @@ class BaseGPUDevice : public LocalDevice {
   se::StreamExecutor* executor_;  // not owned
   std::unique_ptr<ScopedAllocatorMgr> scoped_allocator_mgr_;
 
+  // Returns a Status corresponding to a cudaError_t. The CUDA error must have
+  // been obtained from a CUDA kernel launch used to check if the GPU is
+  // initialized properly.
+  virtual Status CudaErrorToStatus(cudaError_t cuda_error,
+                                   const se::Stream& stream);
+
  private:
   struct StreamGroup {
     se::Stream* compute = nullptr;
@@ -161,6 +168,10 @@ class BaseGPUDevice : public LocalDevice {
   Status MaybeCopyTensorToGPU(const AllocatorAttributes& alloc_attrs,
                               const Tensor& from, Tensor* to,
                               StatusCallback done);
+
+  // Checks that the GPU is capable of doing work, by running a test kernel on
+  // it.
+  Status CheckGPU();
 };
 
 class BaseGPUDeviceFactory : public DeviceFactory {
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.cu.cc b/tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.cu.cc
new file mode 100644
index 0000000000..017565195b
--- /dev/null
+++ b/tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.cu.cc
@@ -0,0 +1,37 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+
+#include "tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.h"
+#include "tensorflow/stream_executor/cuda/cuda_activation.h"
+
+namespace {
+__global__ void test_kernel(float* val) {
+  if (blockIdx.x == 0 && threadIdx.x == 0) {
+    (*val) = 12345.;
+  }
+}
+}  // namespace
+
+namespace tensorflow {
+
+void run_test_kernel(float* val, cudaStream_t cu_stream) {
+  test_kernel<<<1, 1, 0, cu_stream>>>(val);
+}
+
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.h b/tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.h
new file mode 100644
index 0000000000..064fb7a49f
--- /dev/null
+++ b/tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.h
@@ -0,0 +1,32 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_DEVICE_KERNEL_CHECK_H_
+#define TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_DEVICE_KERNEL_CHECK_H_
+
+#if GOOGLE_CUDA
+
+#include "tensorflow/core/platform/stream_executor.h"
+
+namespace tensorflow {
+
+// Runs a GPU kernel to test that it functions correctly. Sets 'val' to 12345.
+void run_test_kernel(float* val, cudaStream_t cu_stream);
+
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
+
+#endif  // TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_DEVICE_KERNEL_CHECK_H_
-- 
GitLab


From 03ec3f7b11572b70ef7f33f2a9af9bebbb6ce473 Mon Sep 17 00:00:00 2001
From: Toby Boyd <tobyboyd@google.com>
Date: Tue, 8 Jan 2019 16:24:00 -0800
Subject: [PATCH 0419/1095] Installs TensorRT

---
 tensorflow/tools/ci_build/Dockerfile.gpu | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tensorflow/tools/ci_build/Dockerfile.gpu b/tensorflow/tools/ci_build/Dockerfile.gpu
index ff1fefe892..f5a28ff163 100644
--- a/tensorflow/tools/ci_build/Dockerfile.gpu
+++ b/tensorflow/tools/ci_build/Dockerfile.gpu
@@ -7,6 +7,12 @@ LABEL maintainer="Jan Prach <jendap@google.com>"
 RUN cp -P /usr/include/cudnn.h /usr/local/cuda/include
 RUN cp -P /usr/lib/x86_64-linux-gnu/libcudnn* /usr/local/cuda/lib64
 
+# Installs TensorRT, which is not included in NVIDIA Docker containers.
+RUN apt-get update \
+        && apt-get install nvinfer-runtime-trt-repo-ubuntu1604-5.0.2-ga-cuda10.0 \
+        && apt-get update \
+        && apt-get install -y --no-install-recommends libnvinfer-dev=5.0.2-1+cuda10.0
+
 # Copy and run the install scripts.
 COPY install/*.sh /install/
 ARG DEBIAN_FRONTEND=noninteractive
-- 
GitLab


From f3b8bf9aca77b6264d791425593bd8cd02aa8843 Mon Sep 17 00:00:00 2001
From: Doe Hyun Yoon <dyoon@google.com>
Date: Tue, 8 Jan 2019 16:23:50 -0800
Subject: [PATCH 0420/1095] Add aggressive_shape_inference option to
 VirtualScheduler, and enable it in analytical_cost_estimator, but not in
 VirtualCluster::Run().

PiperOrigin-RevId: 228420860
---
 .../core/grappler/clusters/virtual_cluster.cc |  6 ++++-
 .../costs/analytical_cost_estimator.cc        |  6 +++--
 .../costs/analytical_cost_estimator.h         |  3 ++-
 .../core/grappler/costs/virtual_scheduler.cc  | 27 +++----------------
 .../core/grappler/costs/virtual_scheduler.h   | 12 +++------
 .../grappler/costs/virtual_scheduler_test.cc  | 10 ++++---
 6 files changed, 25 insertions(+), 39 deletions(-)

diff --git a/tensorflow/core/grappler/clusters/virtual_cluster.cc b/tensorflow/core/grappler/clusters/virtual_cluster.cc
index e80d6b54d9..118f74e8b0 100644
--- a/tensorflow/core/grappler/clusters/virtual_cluster.cc
+++ b/tensorflow/core/grappler/clusters/virtual_cluster.cc
@@ -73,7 +73,11 @@ Status VirtualCluster::Run(const GraphDef& graph,
   item.graph = graph;
   item.feed = feed;
   item.fetch = fetch;
-  VirtualScheduler scheduler(true, this, node_manager_.get());
+  // Note that we do not use aggressive shape inference to preserve unknown
+  // shapes from the input graph.
+  VirtualScheduler scheduler(/*use_static_shapes=*/true,
+                             /*use_aggressive_shape_inference=*/false, this,
+                             node_manager_.get());
   TF_RETURN_IF_ERROR(scheduler.Init(&item));
 
   if (metadata) {
diff --git a/tensorflow/core/grappler/costs/analytical_cost_estimator.cc b/tensorflow/core/grappler/costs/analytical_cost_estimator.cc
index eac6a0de3f..09cddad8ba 100644
--- a/tensorflow/core/grappler/costs/analytical_cost_estimator.cc
+++ b/tensorflow/core/grappler/costs/analytical_cost_estimator.cc
@@ -113,8 +113,10 @@ AnalyticalCostEstimator::AnalyticalCostEstimator(
       node_estimator_(std::move(node_estimator)),
       node_manager_(std::move(node_manager)),
       use_static_shapes_(use_static_shapes) {
-  scheduler_ = absl::make_unique<VirtualScheduler>(use_static_shapes_, cluster_,
-                                                   node_manager_.get());
+  // Use aggressive static shape inference to minimize unknown shapes.
+  scheduler_ = absl::make_unique<VirtualScheduler>(
+      use_static_shapes_,
+      /*use_aggressive_shape_inference=*/true, cluster_, node_manager_.get());
 }
 
 Status AnalyticalCostEstimator::Initialize(const GrapplerItem& item) {
diff --git a/tensorflow/core/grappler/costs/analytical_cost_estimator.h b/tensorflow/core/grappler/costs/analytical_cost_estimator.h
index ade61f4051..6275c62198 100644
--- a/tensorflow/core/grappler/costs/analytical_cost_estimator.h
+++ b/tensorflow/core/grappler/costs/analytical_cost_estimator.h
@@ -34,7 +34,8 @@ class Cluster;
 struct GrapplerItem;
 
 // Estimate the cost of running a Grappler item based on the theoretical
-// performance of the hardware that will run the model.
+// performance of the hardware that will run the model. Note that this
+// internally uses aggressive shape inference with static shape inference.
 class AnalyticalCostEstimator : public CostEstimator {
  public:
   // Does not take ownership of cluster.
diff --git a/tensorflow/core/grappler/costs/virtual_scheduler.cc b/tensorflow/core/grappler/costs/virtual_scheduler.cc
index bb8425a5a1..0aac0348b5 100644
--- a/tensorflow/core/grappler/costs/virtual_scheduler.cc
+++ b/tensorflow/core/grappler/costs/virtual_scheduler.cc
@@ -310,29 +310,15 @@ ReadyNodeManager* VirtualScheduler::ReadyNodeManagerFactory(
   LOG(FATAL) << "Not a valid ready node manager: " << ready_node_manager;
 }
 
-VirtualScheduler::VirtualScheduler(const GrapplerItem* grappler_item,
-                                   const bool use_static_shapes,
-                                   Cluster* cluster,
-                                   ReadyNodeManager* ready_nodes)
-    : ready_nodes_(ready_nodes),
-      graph_costs_(Costs::ZeroCosts()),
-      graph_properties_(new GraphProperties(*grappler_item)),
-      cluster_(cluster),
-      grappler_item_(grappler_item),
-      use_static_shapes_(use_static_shapes),
-      placer_(cluster) {
-  graph_costs_.num_ops_total = 0;
-  initialized_ = false;
-  track_mem_usage_snapshot_ = VLOG_IS_ON(1);
-}
-
 VirtualScheduler::VirtualScheduler(const bool use_static_shapes,
+                                   const bool use_aggressive_shape_inference,
                                    Cluster* cluster,
                                    ReadyNodeManager* ready_nodes)
     : ready_nodes_(ready_nodes),
       graph_costs_(Costs::ZeroCosts()),
       cluster_(cluster),
       use_static_shapes_(use_static_shapes),
+      use_aggressive_shape_inference_(use_aggressive_shape_inference),
       placer_(cluster) {
   graph_costs_.num_ops_total = 0;
   initialized_ = false;
@@ -343,12 +329,6 @@ Status VirtualScheduler::Init(const GrapplerItem* item) {
   grappler_item_ = item;
   graph_properties_ = absl::make_unique<GraphProperties>(*item);
 
-  return Init();
-}
-
-// TODO(pcma): Merge with Init(const GrapplerItem* item) when this
-// deprecated API is deleted
-Status VirtualScheduler::Init() {
   initialized_ = false;
 
   // Clear all internal states so that the VirtualScheduler is reusable for
@@ -372,7 +352,8 @@ Status VirtualScheduler::Init() {
 
   // Construct graph properties.
   if (use_static_shapes_) {
-    TF_RETURN_IF_ERROR(graph_properties_->InferStatically(true));
+    TF_RETURN_IF_ERROR(graph_properties_->InferStatically(
+        true, use_aggressive_shape_inference_));
   } else {
     TF_RETURN_IF_ERROR(graph_properties_->InferDynamically(cluster_));
   }
diff --git a/tensorflow/core/grappler/costs/virtual_scheduler.h b/tensorflow/core/grappler/costs/virtual_scheduler.h
index ffc5e9187c..d96371bcab 100644
--- a/tensorflow/core/grappler/costs/virtual_scheduler.h
+++ b/tensorflow/core/grappler/costs/virtual_scheduler.h
@@ -260,16 +260,9 @@ std::unique_ptr<ReadyNodeManager> ReadyNodeManagerFactory(
 // dependencies, device, etc.
 class VirtualScheduler {
  public:
-  // TODO(pcma): Modify power_analyzer.cc to use new API's.
-  // DEPRECATED
-  VirtualScheduler(const GrapplerItem* grappler_item,
-                   const bool use_static_shapes, Cluster* cluster,
-                   ReadyNodeManager* ready_nodes);
-  // DEPRECATED
-  Status Init();
-
   // Does not take ownership of cluster or ready_nodes.
-  VirtualScheduler(bool use_static_shapes, Cluster* cluster,
+  VirtualScheduler(const bool use_static_shapes,
+                   const bool use_aggressive_shape_inference, Cluster* cluster,
                    ReadyNodeManager* ready_nodes);
   // Initializes the scheduler for the specific grappler item.
   // Should be called immediately after the c'tor or when the scheduler will be
@@ -367,6 +360,7 @@ class VirtualScheduler {
   bool use_static_shapes_;
   bool initialized_;
   bool track_mem_usage_snapshot_;
+  const bool use_aggressive_shape_inference_;
 
   // Whether the input graph includes Switch nodes annotated with output slots
   // information.
diff --git a/tensorflow/core/grappler/costs/virtual_scheduler_test.cc b/tensorflow/core/grappler/costs/virtual_scheduler_test.cc
index 5e044b2037..128cb986f1 100644
--- a/tensorflow/core/grappler/costs/virtual_scheduler_test.cc
+++ b/tensorflow/core/grappler/costs/virtual_scheduler_test.cc
@@ -30,8 +30,11 @@ namespace grappler {
 // Class for testing virtual scheduler.
 class TestVirtualScheduler : public VirtualScheduler {
  public:
-  TestVirtualScheduler(const bool use_static_shapes, Cluster* cluster)
-      : VirtualScheduler(use_static_shapes, cluster, &ready_node_manager_) {
+  TestVirtualScheduler(const bool use_static_shapes,
+                       const bool use_aggressive_shape_inference,
+                       Cluster* cluster)
+      : VirtualScheduler(use_static_shapes, use_aggressive_shape_inference,
+                         cluster, &ready_node_manager_) {
     enable_mem_usage_tracking();
   }
 
@@ -68,7 +71,8 @@ class VirtualSchedulerTest : public ::testing::Test {
     devices[kCPU1] = cpu_device;
     cluster_ = absl::make_unique<VirtualCluster>(devices);
     scheduler_ = absl::make_unique<TestVirtualScheduler>(
-        /* use_static_shapes = */ true, cluster_.get());
+        /*use_static_shapes=*/true,
+        /*use_aggressive_shape_inference=*/true, cluster_.get());
   }
 
   NodeDef node1_, node2_, node3_, node4_, node5_, node6_;
-- 
GitLab


From 46137ebafbb56857622403115ba99051d896351d Mon Sep 17 00:00:00 2001
From: Martin Wicke <wicke@google.com>
Date: Tue, 8 Jan 2019 16:25:48 -0800
Subject: [PATCH 0421/1095] Remove disabled test (should live with pasta).

PiperOrigin-RevId: 228421179
---
 .../tools/compatibility/ast_edits_test.py     | 23 -------------------
 1 file changed, 23 deletions(-)

diff --git a/tensorflow/tools/compatibility/ast_edits_test.py b/tensorflow/tools/compatibility/ast_edits_test.py
index b2e9f67638..4accd8fe29 100644
--- a/tensorflow/tools/compatibility/ast_edits_test.py
+++ b/tensorflow/tools/compatibility/ast_edits_test.py
@@ -39,8 +39,6 @@ following new APIs:
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
-import ast
-import pasta
 import six
 from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test as test_lib
@@ -418,26 +416,5 @@ class TestAstEdits(test_util.TensorFlowTestCase):
       self.assertNotIn("not good", report)
 
 
-class ManualEditsTest(test_util.TensorFlowTestCase):
-
-  def disabled_test_arg_order(self):
-    """Tests that generated arg order is sane."""
-    text = "f(a)"
-    t = pasta.parse(text)
-    node = pasta.ast_utils.find_nodes_by_type(t, (ast.Call,))[0]
-    arg = ast.keyword(arg="b", value=ast.Num(n=0))
-    node.keywords.append(arg)
-
-    # This is only needed in Python3, and I think it's a bug (but maybe in ast).
-    arg.value.lineno = 0
-    arg.value.col_offset = 0
-
-    # pasta.dump should never put kwargs before args, even if the col_offset is
-    # messed up.
-    # This fails if run with python3, but works find for python2.
-    # In python3, the dump yields "f(b=0, a)".
-    self.assertEqual(pasta.dump(t), "f(a, b=0)")
-
-
 if __name__ == "__main__":
   test_lib.main()
-- 
GitLab


From 5439aec55231818d345767b5038410bf647844e2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 8 Jan 2019 16:36:37 -0800
Subject: [PATCH 0422/1095] Internal change

PiperOrigin-RevId: 228422956
---
 .../g3doc/images/convert/sample_after.png     | Bin 185267 -> 166567 bytes
 .../g3doc/images/convert/sample_before.png    | Bin 155610 -> 136956 bytes
 2 files changed, 0 insertions(+), 0 deletions(-)

diff --git a/tensorflow/lite/g3doc/images/convert/sample_after.png b/tensorflow/lite/g3doc/images/convert/sample_after.png
index 6c451f97903f7f70a9f28dee8abf6daeb7ec5693..db09d0a6ca70695205833acfd2bd8ac6682cb065 100644
GIT binary patch
literal 166567
zcmeAS@N?(olHy`uVBq!ia0y~yU~Xh!U`yj*VqjpHxzUV`fq};))7d$|C9}97C$W-&
z!J~6(czH<hck%ywCbqN~2n$RQ{3hfolESg6eSv~>OpA(xLdy@2fWqi(?WpY*1y=$^
zH{P;h(9~LVE%M0P(mw&+T|wPZ-yeK;XWI6yyl(&J-Ou-Yf4B4g`M-D0?|nYE{2pu1
zq$65Wn@rfcre>*Gs>Mv1IaT-39acuMiVY2WBZ5|Od3$*=G|YQ`|Ls#z0fvH$-fMz1
z|5g5rKkK5UJ(a;ls39hE<q`>nZ;V^!yBuA?7^co}yfbU6RDv|KfYHpzn+&%(8w}!K
z|7I3=mi3o~!AZ^3k}2UiLsiYj87J9!L>XLCC)JoH>|kiv9;6k{)Yi-3q{5n{neao-
zVgI8?LYfIr*beA^*SGmu|M<+?ry|{-t7~)h85s6@RIc<=ocE``?{R(fMyJcchXon-
zH+1^4DX?_Zx~hIW$?$`rFleT=kzj+SL(C*^#<}(T9fW5uJG_6|6t@&+19p#nAu^Ku
z2c#5ar*uu^+*RQ*&&G<c!O&sZsjG!6Z^Z<inxgXh$4SpkmsU)Xi$B9)<<q)ML)Yi{
z&V3GI*AL$eh?_8L;Z@IV(@tJ?XEI<pF?*@2=0~eDi$67WOth77n>>f@0Ox`U$6eI3
z4W|SbX8Om6BsALOEji~OAim^SR`JS0p~4U@2JQx}DO1zbzPy-nN@?+y+Q7}Jr%dM7
zWv%p*oww$2exT;0l8RM<%Q`&S9xw#xE=|?6-5Hyu7HVEErBxcVDok2eM}*)1Qq}_7
z2@M=BPAt4GCtNwX6!TxMdEMesF2rjn@x+-;KX!?_GINqafK!jil*i>7iq4t&aTZ4s
z6r3fe{J0=}_1@kI?1mCk9NU6K9@<HWD9dr3Sis0EIc4wFOCO~rPW%syZ4%N5U}F1J
zf6T?r*04;(dXntvzO>z!!%8@0UDPiqmT1U2{l4(8#G!wkf5o-hYn9hxucd}bzy4M6
zGv}G{Q>7jUX_H+mPfYlI{auFmt@dwA55G*XYT;|>Fk8O%fh+&Azb_4{BxGHFUt+Ak
zyz1rKAD4AXV`B34PQE?N`*c_Bx&s#uU)Vm~mF?;Mw1rov-nL%bac}bZ-yJ_r|5>8p
z_~*IDtL!ZO(k3gXxr?M$T6L^A9AaBK&C28I-=}wf@bY}BUajiPrrNcj;g9^+BfDnt
zq-cCy_wn6_y7ku0QD&aJ41%hwPWIUBchKbuUMXS}I49`R{`n`4S1+)iFmtNzrMjqj
zvzr+lX3YO5GW!AxO9+F@zvVM-2wD`#3+(wLQe4CKL7bUAT2*S9kMv^323ENqg%;JE
zhaBb=2ifa5oHQ00$aH$RiXC%oViIXS=;Cu!=giR>onu^U4zlGOE34_ZUE!?U_aJ?a
zqqg2-wp$naj=wnMy+@g&$yq{x>CmJN%5ymV54C?#ck0kkYV=Us)FGoZKR`$-N$Vr4
z;rs}%HHv0JSC8}`VSL0<*k$1#qyA1n_K5K#UC)U%K_0gz=sjV$G@nQL@yU=PotgfA
z3a3w6K5;4%-Pw7^^V%f$6UI+cH}%%|>|5aQGWgjP+o!HSg9SsBruse&<T0JQYJt{D
z-z$2n8m$~-m)u(6_lo!IGS)2}-xf8$h_LFNc0u^X%P&k-ynE-aTiSn7{H1k?_}_`C
z2R5ZuZeofoT;mgFm#vhow|dgnsaxll&MBRF^<34-w9RL4u6v{SCZbGi_XOM4y$yT&
z_RhY0#P0mQ<8=og7U)>co0H(D<bEcw_*qd{aaj>xv0lL~%k^Kse0}q!=F6h5o4!t|
z+Wb}PYuO9gOS>0*cX%&*ea7oEyie4Mmfkse$Mufx9kzFtdDh=$&$m23#r<sQC$7rP
z6{{<DR~)Z2uRQ<t>sQ$?%wODJp8lHtYW=1Ej{leZll^O66TFYzzRA9A{X_jH`H#&%
zuKzq=T0&xnM2l34^a@@vsSs%y(;c}VZUrRVxNmSSAj{(LhEp4^eQ0_pth{dG=D-;j
zEfyCo_FBBx?Rk$?@6Ka-#{`chJ$`vC^cZ{3da3;ef-`tbJWVvsA`><pelq2xb8z3v
z?#wAGuaxw?OuM{z*>{#H2Lc5YCvfZ&$Pttg-@<*<DLiq`h5(EBAA2{r-bhW$GP|L3
zH)m4h)SHf*&5BRmIcjq*CUM`1n*N`npSRcM{A&3+@$1a%>EZLaYvUrnY~|QGAu=T5
z&9)smAFl<aNTx=n>^!pR+^1=xxla>KPh~B-TDAId*xcII+U8$gzrw!Uz5G2q`_iq$
zRyV7<zNWq2d`&+mCpRf4#B6nL->s5cvu?ND61%0h-8SWRqWT8zjgxQdZnw_)?vbL#
zC9LVP+U2-Q?6+&*Hhz=(-TO`V+k2T9*-J8M)^EyoJUvnTrTk2B%)1+JZahtSZgMDd
zRdDa+_2**J>z^|{pL$MxPW;XtyHa-4JPmsO>B7}RrsuZa*?R2jve}Kd4YqUU&Ay|z
zbMxK5cf0Rg-?_bNg4{kiR=H5Q=Qf`H_WnDcTb=*>H{vhTFU{Y%ugb4>zZZQL`YiNu
z>GJ)R|6BfT{(b#xz0rY8hi?t4%zivSUc7z1_QSnLwaE)!9(c%{&YvUk<IM5H?vH|>
z7H3X4xZs7OG22!CDytjC8*)DUI(WT?b)Rz0{!h8z{ys22(*N=H)9Z#uGCj^s*y_CQ
z7x&-Ef5d;x=Sg5~VL!u>!BZjpLVN|^9pNAD6T$-mFZ3;FcaCj8)s`n&rGG;)LS~QQ
z4qX?KNy49UJ#HKPPJEO&yX`J-DbLilJl<M?XL4Z@+l=3M?kKkq-e><O)Ft7_mX2i}
zT1IM{ls2iJGSSk_^1bplqy5F77qechlej-a)6jS0nj1fEsBR2@obyPf@M>Y~;dY++
zW`81V?Cn(ReE-~K>1R<}n8;$8^rItHIj^roZlz4*^o}Dtj-+;OpA<7$P35mim#tQ2
z$ki3UPclzlt~TF8a7mB<&556kOl?X7U-g=7H}Q=s-WB+3#+O%L!X+n4dK=#|Vo&iq
zZ8JG_s_OYw=bxo4PtQMFXK0o6rR(C`<eKAWWQ$YRyuRU??BBatYPm#t!Jdm1AAjt%
zTr;O<j;!VTqn(d@ABpcQ+?idxTSaiv)JexL1-vZEY}bj{7V=F&-y%Qr-r+wF{-pkt
zWj)NL#clOe@RRAM<*gfArMcooRmF0(UxlS;l!?C<eJ&y&FiWd@%5kq8Z?&m&!Yfxj
zTXjo~#V^8eVRvWxwrN63lf7SGb=vAx8}@kF&!FPvI!o59dh_+jm(FVSnLa^&R@OI@
zKkYfme`@jR{j;6?C;DX0%d#)McE$Sgp=G&q{Cv*Oy=QZ{@Yo#5x5mk7dGn9X4mEi_
z{pP*R)6f0;+})!u_3Xx(nU>G*O*f3ccE)th=`(wOg<c4k-nVT3wEDY7`4{$F{cIQJ
zq3w76`@DF||9_cIHeJ5Y^C#`9`CabMV%u~+MQ^Dytln7kajNR!X*uy(o3_-v+?%RA
z?W&%*?)r#-&$~Wq`-SqYyR%`^_Mpw1azhWT`gPTBwfOqF{mT;0K3(-;)!pp1*Uv@&
z&YD|txAp77*X6kzZW`1dzMcQ{e%x(|%^S-mpI&zA?3=Txv)|`?-Y<G&{r~-{?NPaN
z_kTTX_H^5}w<mA+hw`sncQ0{oav*Qy;*~FNSf-`F{yy>k_4l7{Z7tmX^W9JT2kdA0
z`|Hm9{CH~K58JcVf4{%G^?XnM+q#4Qx}p}973GG<I>tuVhW-7?e3;+FoW=Cb|CRs!
zZ%e(EUd=a4Hm0PbJmcKLcZZ+0$D6F2W!3*ge&=<YeLs&~JhOOuU$51*BAb6dPR?Ik
zZ&#)C=E%dz2i2whS?5_5-1=W~S8~7c*7Vx*@BSbCKK-w5o$l7>ZF5%V^Je5)*nj!6
zko&Nn%ub2QkI%ger=N?FtNi(A>K5(m>-u)yD)X&+`>^!r*X8@C{r&as*qPaDW{Ym`
z$q&7^ZQtIP@$dfMy8G;U*$2NTb2pa<zq_|XedqTd+b`X}xBdLi{DS?T8t*p0YPaK;
zwq9f%CEr_j<zK|tAF<Ex-1n`E|9#}`$$j=t|1EA$y?OQDl*(Iwa}E}tet!F(%%Akn
z-Alx`$BXT+`M&DW)~B!E=Kr}rr(*WUZwGgGzPEfIUR&`0!Uw~DkI%k)^e%7e3_)fF
z4Th9Bxeqzg2@_a3q8RUGyv<`{P}uTYWx-$epXc|L)iX2vV7%?rxMik;lkkRpoCWhZ
zKg@1WVOY6Pft~l&-jc9c<|o`wA5CYt_WIqs=VlDc8GjroDc>J`{qEmHZiiU*S!x;O
zP7b@9>#P|VqHfQBQ2g6G<6fcGyH?jp3=9mM1s;*b3=CqbAk63)r1F7*fm0(hB%&n3
z*T*V3KUXg?B|j-uuOhdA0R(L9D+&^mvr|hHl2X$%^K6yg@7}MZkeOnu6mIHk;9KCF
znvv;IRg@ZB<rk7%m7Q#vqGWHUU{hgLkz0_PT9T+xk(-lOY*k^a1UA<yuh<GCtgoa1
zR#K8}tK<l>P$9xMK*2e`C{@8!&rCPj(8N^1+)~fb%-F=zQb)naz|cb9z(U{9Sl7_Z
z%EZ9R$VdSSl<X92ic-?7f?V97b`_<h*(zm}loVL$>z9|8>y;bp<rk&v8(Lag>Khp8
z8yV>qrKIT=SLT%@R_NvxE5l51Ni9w;$}A|!%+FH*nV6WAUs__Tqy#m#BDcWT7jAG~
zF*Kl(^K<np5)1SV^$hfLb3y(Bd$_p7wX6ic>f(^ff>iyW)Z+ZoqU2Q9vedj1Wte5|
zp1uKa-5^h-XXX}wy-|@{0JkJ18Lp<Fv?vEGpOUPfnv<HFnpaY+Z>VPoS6h)=09TCF
z@i41B3rdnrDsl^4D-iC_EGS6^n_gU!qK~9N-^jqgLf61Z*T@oXQ9({=F|tji#i<}6
zu=bpkG<}c<K!F9bFSQ~Q5nm`S1Id8H6{fTxBfliSI3pj2PvFvUO$ENbRxbI;rJ&&O
zbb+P8l>Fq(6f3xDt7MC$q_h;nG+mQ414~`=WRnzKOJnmyUBgs^WK+{bi?pPa6p#}@
zrg`QSmn7yTr-DqY$Su&z%uKN|GBGnZGcd8xO*A($(=|ynG1Rq4GdI;Wu{2LLO-eIJ
zGfXmu>Gv<nOwY_q%t3Y)$efhS6suHIV*`uilqB8MM9UOi6N_YX-9+<L3*F>2GfTr{
zLqk&&LrWzEP|~n+4Dhs7GSV}EaszS_OVaX-a&47-6O$GEgQ3D9nYpRKC5fPzH#9Z2
zurM|?wKO#~H#M;|fvN~gEh^5;&jXodXrO1Nq>z$q<(6NRn^<D2l$o2Do~mDvmkt&M
z8ILAfky~KpT$Gwvl3x^(pPyr^1ag{!k)ELe*tZb5RF}k(R9ht@14Bz)14CT{ix5LI
zD<exQV@q8FQ!4|gQs4ZP%(O~uN{tPzOpH;Krj{h8B$gx+R18W!Rsoq6sW}lYnYpQX
zP>)(z7$_+q<RX#e3@t$JK_m?4{G9wE-~5zRTP0&91%$Y#vvWXEep+Tus;!a>WGn_$
zh=5B7E04_LlKi4dXt5NWS_oz3WGa9Q1*^nlP!VIRRGe6rnxX_1OwLHmOHWO)Rnmuv
zWMWBa#))QW29|~<x`}3Hrn)967RkCvCP{|6DJEt{mL_S*sfkHuF#W~(X(i=}MX8SI
zsd*)~O75At1>m?*(14^7O_;*+j8sqpFfg>xH8jvQFbgp-w=%S}GBwpTFb74DK2aXP
z=3L@31|(GAu0TpAC~*%_hUP0s-4OsQ7ZjYGL3M>C*pHccpuz&~O{5S8`A8pB#6w~S
zR3!QOB1Ifh31{V#pPUGaFK|LnO@W4mQ(`*U5d12!$p&ZSm*){-3QQfc8IDCI<@rU~
z{zd*tS*gh-cuc`miEK_#YI<gINl~RIKJTC?0+rxao}dCDxU`@kCo{DOYI$&KAvhqg
zDnJN=LoXk<CWJg#k8^%OWl?5&Mu~rU9(JoRRDkvQWG1KP6=SytNe&$MNY%Zaf(=+G
z$i)rB1h-5;trR;2MDr&#FU3}=NXgz#^}QH715?RUPZ!6KiaBrYmPZ6Xzs>&P{Yg&`
zryE?0n7B5%+*z@MU8{rn`@LIPvTOHkyR~qweRj0_+HHMVukUfobTGXOT>5B5hj7de
zN0&y2YbPuJd)!#Cz(L~s^H-&&k9%g%P=9WBKE*_emqW2dz)5S}_Wb){D}z$MyttTs
zeVu8Ej*GvB00)mOGf38h^HQoc&*6Yu!DSqhZH6U0Edov)imDpMw-;_%{${~$MkkOV
zLAP4k_!>YmOHyXeo_&9J`Fg*(R$*ab`~Uq~z3{Li!_x&IeJdy2+gJPh!NKM;%l+rs
zFbc@a|G!`VKXsqR<dZQmF<$m&Obrh!?_Dqi8Pp=O`PSBK`>HQ5_Rel?Yy0)<*EA;^
z74cX6UL1-o6S!8m^-8tgJah2i!X?UzKNrbN0qGIFaAKnJ{k@Eb4Zgm<zCL<;-peD-
zMGIka4UEh!hst}aA6C}g*pRqXjQ?b7Q2@yKAr7IdLMrp!&mU}NpYCU%qJPSfQ?W%L
z$dFgkXvu=@5^b4RSB0*P+UjL2H%S)ilNnXt-|0^D_*C01^@z6*YL1W5OdSOUh5Oqh
zrpa0qBxoLY-n1BM`?U)Toj=#^Eq;D3baj~S(*%#QDa}Go9EwX0efs>_*xda5Z!6Oy
z(lSufH}v%Mu<=Mtc<s6|;?C~!eCHPDO#yshpLglJfAJ%3Z`I6Mv!qlcCUrusPLZ`L
z*|2qMs)X9-spV&Gx_I$IjXJ@?bnwB!=E`~3ZB0#0zkdI2p5ws+v-`B2)xP@w|B^P|
zXgDY!3-$Iv)9h<!PM-AqzGK17Sst1&AH+XP@sYJIGs#hLanE?c0FL&yH#asW|NHY(
z=D4KQgrsF5T8uMh&b+!h{QQNDBKvohzGh=(O_eVX{bo2bwgeQUPHcQKJJQd~xhH2H
z&DOe_u~p~rl=Uua4c;wX%Ix9cp%JfbA84xqN_R!vnOCQ*E!Y|5YWh6w%)90KJhHA6
zL-)*<0Y#WV(6vBtXb3oYNkjapxMa)V5}JMC0wX9*E@3jv%(%67S<U&xwF{Xw<&31a
zbrsGFzr3|mzp}D(&V;r1uPv#K4m@SQ-fXVkMByot(73tb;nuVERc^9Kz3Im<#m>ir
zcv@tS-{xO2|9y0nUhLl==WaL8elk5?<nT-h>F%Ayy4lgM<`o)lJX&yOQ}3-upui6D
znZY*gY!1VWFE36@SWQb>X!+%H^Tp!(`vP9g%MxC<H)gZm$0M$;lY$O-9o;H@vu59x
zTR*-XEl{(wGu1Kp7{jx^I*i9%=d}jAYKwrA$47xO{s|ix4%{#?FI}l&$yWdT?ak_4
z?YEBX&V1glnjiJ6=GKe!hlabvm*yUyRVj8@ezj%2O=awkUyu1i{6eO@X9vY9Pq(4r
z>j%r~Ig~&D`^zisXLxGW<<!kqhFaV8z5E_A7#K{tR;oYG_+$Uo%%jCiHue94<PnQ2
z@+oO&vxMdyIOTVusq}q@&h5!+=dZ2O*(`W%%c-hU>X}J#ObM}fCI52!?`#pdJA2La
z{wwRGn5RDb`1H$Z{!R1UZ)iE3aX+d2B<O#c)5Y0a{vNlutM~P`*Q*=cE?IA9Pw!fs
zzQcx*>(zGCvhC?tGG|Ta7Zv|^`u4rQeUqOx`-j?in{GB?S6KCy?G6jWF}FKAmwFug
zkd+aC$8Ud)_4_)e>t;dx(JM>?mTLXCUgx=jhxh%@1(1Ay^uQK(Z?>3Evm)<2*uQpe
zuVh3-Psa3ZR@nvH7kth5o!DzU|C8r+IsfPbhGt^(&zWsbWBl+byKu5tM)uFp-$he4
zMm}r%YvQjoul~FB^9y&b?BAT}%&(k0t@`4Lk`=qJ@GzK^o|Ri)9sTyU;IU~*a#{yE
z7=I`-3)TH_`e%_`Vi$b6=-YO&bCK*%?L7nSDpp<Gz;yO+%TZ}=tCi1W>kX2VQj!-{
zujuMzO1P4~EBF1goNc}pPrI*Yi>amaMI}jv%(TCGM(?$Wgq@$L?6;ba$GD7o9RdTQ
zZ9s8pH*4km*4aJ&eSV!6m$k)Rz4}Jd^tIZlZJlN{7n`3RFF7f;e$V%cTTFZm^7DUO
zRWdzLtRGjmH6Z&{x_SPTD>|x~fA_F2-d|Cw%h+(?-0o+843qBP3)6iqExFm|z^z^0
zEt$JEbEt%cKJPK#CQ^L<yq%QktA{UkYI@2X4%<5KPh^_(u9N2Xqbz1;3hN#1mHPDH
zdb3nRqOa<?cWUqS+{BL698^F1H@zs*bLEEL&&76Kk$m~mT&|=2Z^-8IV#Z7N*ZhBX
zaD^g&{oU1U5*ycEeQy7{x|?fOc>L$ynR918J9%?QptD@hKbLYPW&ilyJ1hM{IHBcl
zzpC6FSB@ofEmyD>WR_<cF&G?*nzq-rD5pF#_R-uFhMpaLmEOgFyT8uf*zoGnzY@!Q
ztK0?-5Bn`2&c2L(@HlvyUSI*kfg9$D+v>j;oQmU1V31oieSUBC7Ml;bPg+~{2Ck7{
znDDEM{lI<C`e|GYVOPI~PBPo8bN39xlKIw$7<cS@{Pd&+gH1^1=XTqFc~d<Ps!u&t
z)bl=FN9Jgp?h@O!sQ+8Eiq9?kvSEMly|S$@U$Hn;etW;<@^{7$*O}+9u{g`jvm{J7
zjf>&tmOvrHigzbZzYi;9_;F>*();@=4>Bgm?hbnyv{&N4<tf&zZuJ%crxV8$@950F
z`e5q!xc56b8EXE#exPwVk(WW@_ZOiz`)BT5XleRkzSjLRVFtB*wej&c7#yBm?{Tau
zIuXe+YqpgXv%$NM_U(GRq}&ZxOK0@25-Q*PMf;*wYW+%y7#|O-2Jh<lcqR$qhs`XV
z-JO5#Z)Qli8<3;N#NfC1eTJrJs>-Ban$xbB%YBfQm5}h1-l=}yxbnQG{{FK(^R|9n
zWq4Kl^Gdac4{tv>dvbX`=kGuM93MRVIe(kK_TRh3RCVRvnW_oTw2D{czR6>4*e$>E
z)SU+Z+Ttghj?Q?-&=bAI*Jjsm=l_hSR3R1Lr$ClirX@)qHf;a(irr0##bnO~$M0gv
z&ob{_vU%wgCe^;%QIj+F?Apl2<l>oM-71reKUY1gs5APyQEcm*R-K9!mz4Ea8%yo2
z?42dprO+Z3<goCp^_32P-+xa(r!cI$yGHz+>Bh4!a!)?H7jS8X>bLvM2kzG$V{~$m
zozL=n*L&;Cun#qtmM&l2XBc{B!lVV21<|uNhD=(rF4t~8_xxE^%IDVfGBdB-QYx-o
zST}o4XGmX;UfRKHUlJPk)W@*8S%RxgJEr8Mds`V9;!d8j`5G+pQRevT<t9nKyTAMH
z+Mt+SZD;!PitlH}uaXfnN<Vv4zfoqJ)X&cBu&XYw?C8p=`HMfFK0YIg^+>?!N3$iT
z<z2J+u2uNeX5Zs2tMYCh`+FodXvx><lFkoq99@+9VK#Su>s;GbhV%Bmwtr>J-dksy
z-8s$vYWMYf|7WILu<<|JWq4or$8jmuBgYuD*GO#sB`yc4sef|t*`8e!Y4`8W;`L$A
zD)t_1xn*o+&~RYi+Rnert{ji#{WNQWbJxDFsyat}*}_XVKR4~4)c;ESTM7f4+a$KU
z{KfgtEM~3l_gs5@<?*=Z!VH%Wl$|vXGTg#BeO}GZ7G@u-XLlOvMgAYY^@{tLcZBlk
z&egtbs@*zgE0*lJyUi<f#$>s@v%YfuUU#IkG33@2!>=Fr=BzJds8}(De{#vy1*})g
zvmU?So6S?_>XJMO)X4etFs1nxSHqQH_oDrUwb4aF3<eXTzI;FbQA~cSh$Mf*#uBrQ
z!rIFHPoiWzUj3fAv2<IO<FdxL*B`ImR(95$!Ld4ir3Awzzjqg&mhD-mobk;?-}hwp
z0rlhi_^c<3RX_f^P`BT`c2>!kf9&&3rf>1)|H__uMep}(rQce+zMF1LzdV_d;aK0M
zH`*SC-;|OWF6{HvIMb;m1WmFFbACl$-TcI~|6bdj^MA$W|9Vz1<=w&!6Z6+GKI=O0
zcvqIaoTjE?`q%YpElbs2gvqVS`h5BPF}*YfmUVYm{r%Y661w-^Hg)6V({}EY4g{^T
z_bwG@Fv!e{Im~k6n*5ZNp<kz;3^KfQ{!PX1zvr)O|Et*jydbRU*S*P10;wVOML)yC
zvijRzv|Zwsxt*pPbZSSW$vQ>{VOc%@ikUKn^1DQru<@R~6L@9+YQY1_s+qwY3<g(Q
zcKy2Kb-{QFq>b{)G3VBeGcWa7DyF(6oK<*VWxxKhWdo~3$ku<~HZR|0*;(?4U(Nna
zc4+h&`~6d{`kY^`_SP!hY^g1?!J}JO)7D=sn$wWVsBvzcf8Rx}-Da14EwolV)0*U{
z%g(SiCcaxM)B57M;uRMz`AmOnl(c<T{o4bZC+rA)x$ETf-s&@jZzdLM^<CBqzNFL?
zmuvOw-DD%a+%;BLFTOo$Wiq)kZmRq9NT<&Js*T^a9B%o><gvPDYryUw?ms`j3cG)1
zO4mX4`;|K@<QcAhi@#T~X2S+y?$}+o{if`X+1vltYHL-wIKwCB`cuLc3^}{Sl<rQb
z>3$_z*CBNH^VE`74JQu8mM5kK4-VK?f4lSMPFw0dOOLjdp}A+79<SMW@6PP2lQ;L*
zPd^)z8+_j8w_MGq-s8c~)1QU<yw2QepF45W*Z*;;C2Us(OC-XJQ_>UNbk)x9^|bqO
z{q^7Q-sS7oJWHKgd-}`f)$wa=d5;}QdiQ?X)>6?W0mtWg{J6g4*M?_22bBMw{q-~7
z@4k8bj=QJK{@bgUet*+4p_+AW$c2ZiqmpMmx#e11egAR%r<v}a4@{Tb(unVjyw<*G
zSM8&t`t|8cC-wB+{GzWbeJkC}`SYo8&)d`X+1ASUbRPX9I<3r-?fmjL|K5e(<9cTE
z>`u?kzh7LPoSd$%3cbI(@TMcAoZ2K}#{RdWr*p^L<*!Oy>^eD;mA<S|7hvw%{_4f;
z){`MD>v&t2@ralAdOs-jcTE#IySCoa@`2frf4Ae-e0q2~-n=mMZHtj)@)3uWoRsHz
zx6ZKgJ3V@q_jkkOXY!rRvo9QD+B0eQUwQTO`P|3cUcA&0XbwNJ`g7Pyu~^Hqjw^NN
zfm-Hm2bOF3yNi68_UzaGpZh}@-rmh#HOK$!=ZnU?Iq6@L7J5bfiLL4pE6-2QQ)Azd
z)4o#eZL|NZdX>WRkiz$On)<KD^#A`MJhj|JtrKLN=yYe>jS@UPKEL~gPHl=4+p%rq
zoXy7$O{(7gzu{ri>MJYcAbr&EJG@IbO%LIjSRMX5G<N3}k+qY=XUyEFQ?Vk5Z*P5E
zRge=*rMUdpq;Hp=v~%v-_<haBl_xIwEb#RFt^elblEC%fVhjo@rb=9G26rNzy7$lc
zarmf<!-Ig?j6c4(<T`aNOjd&SB#JH^vo$H<P~qNsR2Y=*CR~D6795(n5G5@GA*5zf
z!T=f+&@BE}^&{edL0eyw$SH6~?@&9m^&I);?OR5j?%I?W4jzIM(qLfKwJ5P-o;hS7
zLQ`9W%eACrOW11D5{090US3{a`}>=$RmltS_?pBQ5nb_XqqeRJU7hwqqpR%|xKlsL
zRcT^HWu>H5*Q4_P|7sbot`5(?w`b?VWJQKvPbH9h6qoix`qTnJw;+8V4#lOtqe7rw
zAeBSH7dnpM)HUC0sh5YxiJD)Jj&d_-YHIq=w>w)Cs5LbzGE!2(g2}+<ytEY?IPoTh
zt&iI~+dO|?`R3r|ep~bJ?-O~XxD(nZ(NfqL5i{fLnKNg+tb``P#&H;XrA%Mll{Vmc
z{OZ-M9fglCb+$iof|NsA3mlu-I9cAGDz2C(Z<Z6W@~AU>z-YqSwYlwlviJ6$ouKIK
zr7AXwA5!TC8SY&?YjWa@Wxlg_?btCT%|^u<Ch_(4b$=D1r@zIJ#&TwyIeT_@`TKeA
zqj?VVNSUmd@>uaEWT;{SSN4kw3xDnd4T?#bW=$zFfSZ)~^~FVJwaHIw_b#|IWr`Hk
z{6x>Cm-f~EepVHzBJ}nB{rd_ZC-^{e@Diod){K+q&!4|^=~Blg#hH+V>*0EBskgYc
z_Ubq51DtCj9iV<TS{N{+mSazd*4wMAr!ziQw1S$^bUN<i&E@{`$;rvgY|bT6Ggz<h
zD$TaGw!WJjHGgxDfRt53k>~2GcGcf{o>w}v2PKBCjSAJ8DrLnLw69R+`=6i1TFX_B
zA3NrzBPMOd6;x(h{jKM*g|qv_rSAQ5tHal`J$U%=<ByNY&h3+zdQH`ux;bvzLirbq
zs(yW0xqp9sNXV3h%qvfvIdk^=``gzyCc8VcJDplx{r%m~($`|r!pScjv^K4}cCEds
zIeJS*;tL0lnTvx~{`&S-T1Y7Ag~KU!Ztm_48#X*UJ6qZcRHkKIytAY5@b&ZWUrtuP
z`s%yb>|0V+bGkY&YsCHA-u^Ijwt2psO~r<VhZUcmU+O)5*U#VktG=Gv=2)*BDeBxF
zY?vAHY5MhV9W7_(-Q8XN<wfF)0uRYW7c)SESo!z&P1B7w%e!;q!ow#MX3WS)OAEWe
z^@&Y!`swZY_w$mHihg~0`RnUzX{$Lwu9g{sb1aLOrLT|ETv^m#r=4xK@XiVrRr}p_
zrOEZFpSt7g|CS!>k!<5@pQ7&GC-d_9{{2BmO(pi6JGOcG!ow?1+_TyB<NmXQv%W03
zx;ni6(@AxZN=}>e{W6wCudl8B`|IoN9fgY>91i>4)3>pyd2pa{+O%ou=jWA{mYSAW
zEN$N>)4DbN{5-wbT}w1~s4XrJW{SFWChJQ6$D4*Tx4xbpUuRkSt7KQOQtOt)!)^0y
ztFsjHto|veE>JzYDC~2Z@ZlrRzRj=u^^)KIPr{1=kC_IUmzJ!J-d^_lTJGIlrBfQj
zg08QNHP5~E<<(VfdHMP7{c@64b1pf5UC{dJ_xt_+^X=}QII`>Gi@j;4(o$A)PC3n;
zS5kWUfjPf2C?G38r%C;FxR-lxPo@2z56xE&?fZDcL0H<{xji^?MNoWP<bC^}kymyU
zJ`P(Srz>(P;QFc8r}g*8Y)bL`{q1dYQ<I8N=fcC9>DzK|e|x!n{<W3C&)?kKe06oW
zd5Oi+#-a<~A5_*oJ2MkB4qX`fd|7hylCZ5jvVl(ztrWdp-q*ih@Xor}-OTKK5fKp*
zR!Tu)g)-)umzM1R_v`eQUFnkz8<UJ*t&Uw~a_UO<Z8w#Bwh!OG-(USbZ>CS$i;fP)
zKkw`R&$cel>lV{(YHogM{UPz+zu)gOFE6XCr~pN{*VL~2)8`glC=Xm2ve>=<-p1ti
zoz9!p7MFMSX#W#m#PewDt=k8U<>P<sj@w(caN)w4!g~Z4O`bN*tm4Ci2M-dopY!#$
z<qLY9;gJnYUGnsbIn#2R{hKbIKK=T`!^79EU2E&>n6m1A-S6CEJ(9fAW;R7nI(F{d
zx$v;&@o9RoR@L9$sQJ#CV^ewQ%$lbEvjX{|7Cd?K<lf%u`8Ji8ZXMo#aKhI$9m`gm
zEMZrleyru}+Q7|e5g99%FJHd!uwv@b)#2-_zP$MOe*gblW>QZq6P7mUIv<YJUO3~z
zPJTb#(vl)8DY>}iezMN(&R!?u>;IZ29%3;!-~QplhgG4grL5)@y+1BrAG0CB@%6Q}
zhYugt{<de~l;&seZ*ES%yQ{Rjt7}uX;kLX3>w6kZOI%pauY0#=WBI0)oo^mAGQYjO
z-Mqx&$Ok!_iU+S>OA8AJ@4ow7@9r~0p5zxD9FO$=)c$@D9=bZrRz~R3nn+`i1MY0g
zz3tW`5x6>RZSwKH&`EJUOB0@*nW-JV?#-p8-US5(ta99rmKk<=cXwUAdj0#!$H&jb
zrdj{2Nqte^G405V$Je*@iF{b48@-K3#^S<-hXT&Kik^1GZ_nf8<jk`bzmjEo@xsFi
zIv+0{475Jp*?VJe^|v3te@k1*1T}qofB*je`u|y1SIJtJtqIeW(hr=o%xC7N#KUb3
z4Gnv%zlU8}Soh}6&SG9!=bJ_E?pV%0H#dH-*P83~p=+;|Bx*<ZthNX(eEUhss^a;!
zUNJGTd-v|O^>s+NMMTVq-=3$r@+EIu-in(gGZr3Jv^<e>M`!LWyWQLBe_!i0&Ayg%
zX-VfoWhGT58=F1;^X;Zho3?P_!Yf}F-T3+Q=iS}q`WhM<W@g*`=i7;Hn%iD<L8hsx
ziHV76N|uS##=G0<8WtW_{3$RyboH~FudmKs-2UyYho|SuTU)bRb!I-1HOswaQt=^S
zt;4O9Gxt{I&E=KlJK28a(79cC+keb<Zz{_d<zTr|x$Kc;^|zdUIol;cnI0Y;?d|Og
z53g*HXnT2a@$s*(uQxX}&9kq+w>h14&W9OA7j(44)<_tq@n~<%GRoUOH8|V3-I+^e
zQdCI(mH%fCbE`Xlzqhk^dHDJ`9dFq(?QT$!^ZC5JxzXx}Ur#y~Tdzvc-rbWDnk)UK
zR<>G8r{eqO^z(oI{>?o0iR<yDrQZAY?PF+aZeBTW_Y)45rntRTv3n{E4Gat{EhE3r
z3|F}#FTzkU@BgpY>&11WQeuv-Jb9z`SZJ2CmC~oWIaan=$9J^MR+P0Wxv{_A{w!P2
z!_2L>G=rD@`T2bQvTa{y_3(<!zWAcO$YHV1&-~LLu5Vmk_3h2h;^*hOY)@>T)X~?M
zcYj}PRMf3~wZCs|%}#$&ptWzymMtJh>?lY)H^=hxv$Mt}7Dqda0#;mowK4fPC?($6
zm-+bO?%ouW^)5RPW^dijSM&49{nB~!<}F;f&@}s+Nr}snB}<ks=a;oA`Ty^4Sm`ZO
zsg1pAjJoby=2%soe<J?>ZPvX#l^z}(XYU5DJ9^~EmfYKBhK7aL*TvS=)wS_0-{j%v
z_wD=l{Z(IIEpqMN_iNQ7o~i5JSY~*6dEMVv+b?S!wsl?nVMXZ(<z4Rj_3!5YFg5#g
zd{$Z6!^FfZx+|Tk{{AZMm$TK2*^zL2!~W+D_N-3I&h9Q<ZO?zsxF9b#@890O&BYZF
zGbT@-yii$5UP(`Ho$qWjP-(xXvUt8-Esw0<(@8UCym<Y(`t`N7p{v7czuio4<6B-4
z@=E5*&(F{A?ke3~_BQJ5v&FVm|J^1xOpEu|&)+X;arDd?lOGQbZp*zbWhE0dN6xP1
z#q;Oo&(F=hmAib$kMqkFGoFUu`DwbnIXyBuTHdl|N9N^aZG7z_XS>DqZ*9w!mX_X~
zc6Qdo!|l>mb56bZ`T2S5?y|l0|NlKZJKMcSBJo8*m*dm3j`{cZ-MxDKdtpq>+;jJ@
zy!^9u;bBGj*H`Bz2i|t$T5xY$ZuI|ue|cs3H08qA$9;W$P1nxu-rD2mw?_FJl~_DU
z-rHkWQTx22GeTlVx&GEC)0Ef8@6Wrw?ryKNdEM`~+c|~RB&_C~($>>kSN#0k`T6$u
zH>dmG-&e~kd+zwEC5uHsBAY^2tNYKZ`V#HPnQ=7W>e|iMm$yqQcQ-P#>&5T8bK&8H
zq=GLmE^d2!Yp<72mPHD)>H<+`)AMn@$KQr$yV_QLxv`^g@xsH3t&`@=xwAF<y0m%T
znc3$3LIH=13JNT0eiU3?70S*pSM%hApmY1@jatckyXyb{^Ru(lEJ}Smarc{9p)I#^
z7A2=$UvV?teQD?%yZ?Va&ooXCyJhvSEAjQUwg3M8)9TALGFuVF{HijnFv4nEmYJbX
zN>uHc8HS&qpEobDcw&3$(xqMH@9TbkdOFiM{oD-0!xtV3SXf)Lv$M~hG2_eI+uy&u
zyxhk3d~wLGZ9hU+hxyJla#dM<tKv#tarLaLYnXyg#>d_@d69c(Y2x9w+#4GlRfHt1
zl!8*Y#dLl=o1L%n%v7qM(`4(?!ZgtdMn8W}{rWd3?#>L0!be@A+8JH<H+e1%`ugUk
zv5ig5r6r!-;`(MK7AmKoo}OOH{ZP)nZcpXsXKj4XnM2pfe7T*!|L@c3@ja_8%;bbs
zR(mlUYwP&UmD^hDb7-ydT<dZ^9v&0p#g#Ruc-Myg{`q|V`k0-UwnW`MmdvCoW^zBQ
zb!ni=|Lv(aURS3xF)@|BxuH1Ki&s|gq~qd?JTevq_xIH<cI%DWS)|I#u9JOkuC=&;
zfWlNS|M_;Y`|E0*+dnI6dGoCbTW$aI$z<QzW;1V_=Gmn6Pdsz$g5bnSN3(a%{{8>`
z%cH%#QYIX{yldm;J~P&57vVa(A@OiiQ<KizTPAWlV>K3*vTIqabrap2eyKaVde7dy
z{IXUdOT9#$-Ce$HiF$itqH?LTjd{)uhwPZCzV7{UvIYqaOP4;qxj8*HHkMb`Z(?Yd
zjESk~*82bd5)%_c@9s8}*)`F05-7mE-pNK+yuWe&8z0}p#Kd>+-nH?y?>YVY`ugf`
zZz})&`I*K0>ze$IC<PH|E2SW*ih2E(HoqU<6xGwyd-(9-*6izMB`z!sYCbbQT#v6W
zeSIzW`MJ4H3ol%FIDrW?3;yH7L({A)7Ul0^)<$jRk@dS6@<-;&)6>&I`M>IG7Vmn&
zufP8JO>PJ?^ViGWEBa;U&Yd?GK6r3#ZM3wN%%uH0c38y6-@m;*|9Y0~$*srFUwIXN
z)8^On^VaSAHg4R=X<M|U@bNKFY4`inY5nOsk(U-aw=Y<ruyMvaw$r=b-rf#sF8lfU
zJwG>BJHEG0)P{X;_4mB+`1|hs@>W036utYjcHv>gwvVsQUCh4i#wD3|yiaz^Hh0OA
zx3^69=iQC7sekv%et+5Sm2G_O630%jsJ&iZuQgTTEjQPzOG~>K9$vZU-}n9h&(5{p
z?%FMO@ZdosNmJ=*{Pq9;e&3RL*~Z4^+#Ji`zF%8Hp2*1w3JTuSjXq}E_4&nbcGU$_
zBSW|A#m#T|ptQN}@2|BnJ1<>$I6=m5j>W}gxwn7(2<L5Evz%p%(dugp{3mbx9U}Ak
z{rml?r^TGxgFSU4H>JG0vy)fa?9TRldDR6^PpHqY`E=Re{_dX2&EC`X^6&36EwN}l
zdTy@u^$UhmI-1$}zrDDq9J?^nq9*#Q+g~18K0|}f%F@|Yw()oQtOa(&t_=D0<3~Vt
ze&mMQ-{1cI{aw6g#m5)1*GhLwTFE3O1bMfwdbjVRIp6!6J0o?(F7MjCH0SIr)A{xP
zcJAK&`{CjCwQJV!$nt4Q<>%+md;a~+&EjWgW-2<jP3f-W)K25um3DU4L}hm)w`ab`
z-)%0+O9yo#6pXfQ$-Scg^5D+L?ceY0uix+9FK42<Fi|&Z%ZttD?Tn39UMu<I$ld<Q
zYQL;zT#|71ZMNFD<8y3^3ZAbHU;pRNpBH5tOLA^+%e}LsaBKE;K3?9nVQXiBO7TUp
zyUR|VJlWXTD6StTW0DbYQ{E?x@ptpNna1h={`?GDearV4%c(V>CXHsaMd+=qZx_ei
z<$Hf=sdqxcf|V;*E>vFCvnA^7mzS6S|9H$VEd5<AnMrk88K||bD|Pht>e=mTckQn)
z_n-gv_I7U-A!qlHgKzKe-(Ua#-_=#2)@5%Zc1lP47R@wDJ=Ml5UG?>qsDwnsvC9{@
zrZ#7p%<{MYTXMLKH_cSqRC4q26Gx6+5S+NF<9d<#H{WWpojU^;ySewtY}~rlw8Uj<
zchE{1yP6;O_Ezi4Mw?3Qj8)kRY82>P$_lw&?q4A(dSzAU>IV-JvZVAZ6B83{tG{j8
zuz`c0U%$WDXHv|TsJA-`AD6wqXPa_j!rJKV)1K~()%N4lntuBC_xJKrQc*|O7B5bm
zohcOXDersVzoPd)nA)SZ9y)yJ&Ye3vvVN%(r%VwM5GeTe=4RH{>g)1f#C@5(?lnID
zxx0N|?U(af=H}ZsY%o|gUz;=i{Jgifwyt(j;^gJkU7r1R!X6W;vl|i*^T}G7)cyGZ
z>Uy0i%UG#knNd@-XaD~HPft(JHj#aH@jE|b^fK-Fo{}F`iaxKcy{w~i#+gqx>*l7V
z3lFdSA!lFr2HbSeZ4VQ^Q>Bw|>cd@2O>4P%v4!SOc4uwdmbh>sue6y-iG}E?rQXx6
zN?$Fxn&s@|G|MzQ?L~n{VUX6;zrVlR7d|?&CG+w$z1XZ51)stNi!RKVF=O4nef;t^
z1+RWA-TYq0x!w8H<d*2fz}xPf5p~bc&1Ga{Y~x$5w7c%_uJ!Zpe@;Ii^XHjm*^fgP
z9!~i5<mlD|FKmT-mp=OY``gbSKP0T?a4A_??TX!9_VVS+-{0P5U+-gkALfvmnYp*>
z>#l-_hvrxoKRY`+{YAm2z;~||_2Ty2*c7_jonQXy+s#vi0)j%WmDcXHvs{wu`|I1=
z-Bn++oZEvF=h{?mDxGbnAFp;r{*Srpf+Z_*q`zewsRgV|`}60|w%prkFAB7pX3Usz
zZm#w6Wy|vJ@0%-hq3hI*8x|TGD+(VUv;X(w@uQ>Nyt00Fr@r6uQB+j?cX#*lrK_cQ
zZpSSC=m7GuYG)<y>nhv0yIgnom(Pw`yDVy2()R4@>-N?DUR9a6IM6!Brb@y}CP>e7
zZB}UM_uhWHE!D@<l9Q98%D*QcIeN78#f67QyTxal=fAty+?C?0Cf3c&#N_1cT>j=p
z;M%CIo31@xB(weK|G(ew3#<Eyc<D;_-pLH?Rufl|cbzI#pL)3A$@2fbQl?pNZ*5Jj
zvFk2MIXOu+zV7GJ8)vqGM+!2IF#pu(TB06q_Qkt*zisKOE2*caO?}F^KlK`@owp^k
z`2D@T`~UsQzOf+@G#b(-t{+$O^3v0Pe}B(5%ayV$TC(u)$)LF63-Wb867TJ)3_2a<
zdyMVWnphrLzLgvnQ~%uB7P$VM%-7GKZ*NZT@9gYcsH}CrV%~hi<YSkXdZ%7n=X;#Z
zQ|B$Otl&zHD=+i^-L_GAdGKf5-(R_RcTHW!r#tD$kt5&U-oC%1F!|-BrSt24tz39m
zaq07qkB{5g*`1$ne}8Xv`Tn}p)Q=}VO$X)sPGR+R(c90hacfpf{w_81LJ{AR8yS{o
z&(_A<?B24~bFx}Hzx=YJPIfbbR?3)WU0D~qTPLg9TyE=RHAdB=9;*Z6?tguG`t|SI
zTQZA3JaF{({#7RGRa8{8+;6U4{JuYr`|bbT|NnPB$PKz?X4|r_uUj9#->%?+!}h$?
zq}(S()0$5`{nW_JE@xM>qbqxP+u>~-M?->LyW(?x{Q0decD<~Gho8H+aADLprRvYm
zd>tGP%rs8VdG&RTeTA;}qFGu&XDZfyJGU_Z(Yf{OqPOQoM9kQ0dv()(_o%2@>({^E
zS^PZh?5wYUzu%X(l1Y*<&%5*C>sMcYe|}!x-240XE<CImZkgfa)U<#9|E1p3SD8!3
ztiR92sBKi@vgA<JpZ9IIfB!#!>C~qJ507p!T^?D#%1=*D3ftT7mp0G4QC<J&Ubdhw
zlb6@I=F9a*WM&5Mb?d#L>@H=Jk?^A6l-|;yo2$dufB5jhbF$jhvmY-mxe~Ll=H~tT
z@&`UXK0e#r{6oa7qRGwoc9*YTz5e~*>hChvKII>0XTK=$I6A%Hv&WS_LnXzs_qVQY
zPWN|i54Ll0cmMtCr)~AGD@V)MB_Ds;#@8NFbN>JB;^gz`$Bs>EX6Gw;bK~NLhf`do
zdeaUxFoMcb)2u6}rfQ#lGs$wvl{wbsdI|~-N)zYU)!y1*p8s@lmy(gurQNBZQg=7|
zA1_V~P1$R5cW3q4uMm~^{_gJY_51%Bl~_DI@bdEV=<S)G{~Yx`@0hK@sJe7<)ndlp
z<8MM&Jbe9ncky#S=l0-d8C!2%T<rcHQr&2Wty#b_g(2heva{#rT0bqSe12~3`+Iw{
z4|tq%<g5De;o&z~t37^Rc_umC&ZqrB3ExF0%QWB2&?hI)KJgICE61;SyUX9dyR|iY
zR#Uc7ReA5RW?osol`g9+$}`L!b~r?(o|vErD!&&V){M6M{bqCS?QK7Q|6U)xy>Hnj
zrJ1`*UT&)V{OtPr`0`g*UOqhB-o~|fO}+B<_3`&thwJkm@jWiT<>QiX0YSG?uIcHj
z$(yde_~5vFy^ML@9Jj@WB`&EKH>I9t<CQwnp}W`4Ht1-p+E4YRXSY=S6`L<RIdXU1
z*Q?=|m;3AM*PijZ)gx*A@7L?~eX`bhcXv&FW}~+A%d4x|XJ?uE&$qjJ_GZciiBl!@
z7b-tL`}^TAzYzCtwPa@BoI4;t@4-5FcZzG?^ZmbGtycA(CJ|nmXJzJ*6?*!D;KWnj
z=%bSxBXVAJ@MONbv-AJ&`~N`=Z#f$e`^Y5<+1J<oeblZ0>e|}sXJ=-v3SB+z)6LTv
ze5VTUEb*ND@7HVn%dh779%Jb`n*%CE-fWrk%<aLH=iy})m6ft~H8*bEx^+QNa-P>x
zDch<qeX`b|sIkdfequ-0LS?0?vdhC_?tk^H`Es!A?JZf0f&+&R9l9VW*_Kzk$*ork
zr2ozxP$$vo?f3ij|G(YNmp0F<`Sj#u(7$a{j>?$m>gIlUaPZ*4gJI`nrFw5@iOiO=
zQgZr|<n;ghgDd%cetr)V6VIJH*T#1ov?TZT?(+A+%l#s&f0#>bnkVYXD=VnTa3Fi@
z_U3fEeILsAR(*AHc7FTj4UcSKX7uK?vqw6G@7}$8_|PG*{b}7(-0lB<xqROK|D9E#
zs~5ZXOG!&lzg2lUhVR#_tE=DL-5tND;^MZ83g;QMGlc?zuG#cOl>a_3JuGxhpN!od
z@Bm)2#QAx)pFe*FRk^X<g{915#!S~P2ntR*+#Gy5$*Fz?($J))ZS?j$S<9l7x3{+T
z%h^Wlttti8XT9t9|Ev1>>FHeSa=Vfj0V}^IYerkXXlCc%_3PKMV@Iy!{cr8Q|35>F
zGsx`deU6>`iq}l^*ie36?rzHdsi)4)Hn08rXP!;nn>(d_!OQh|W%(3;R+p7N@7+7u
zH|5wtcK&Its)=3)_tpOX_5FQ*c6RljABF7f>>!J0t_)uOimi#AjcwXFg>6Ptn&bA>
z?3}&7-pQ${`g|+jS~UqP8K-+s&n~Y2f7@h6@V?m1X=is7KIW12dkGpkJ0G`q(&S*9
zUu##~ELpMeu;P<gp=7_9+g45oR&UF>X;kpQ!MQzn)1gC$s=mF+%*>ROlk@B5nasc7
zYS!_7`Tg0~*BxwT&%eEGt%%y^r03`6Mqhfa(X(NL!T!9vK|z<Lc_QO4XM=_gpV+F1
zm4&|krGCdsXiGKsVFOvK5`z+#rHTdy1_}xeOWhU=-p+l^4tDF|9bL(y*Y)%lHJ9hz
z&PYsr`1Wn?iw>_-H#ev6E`OhwlOrP}6tq1r7UX-?w!~fK@8x7=Wkp23yt%n~o^AEC
zlH!t?CmhXlZ<%CVP>55=HA>45-LBWh*FM3?a8l^T+#C9u-$HIIybFr+@88$o2zJnq
z-&gbI#zv>ismmVoMObY$DzRu0>AJBd%Jh80k=@6enzt=qvZUngt*wu~e(LJ!`SSIv
zC<n{Ig9m%1&CN<ImdFTmurM+*PS=a&;^C?J{H%BW{P@m!=RV|q1SQNjH#e(_mH8ff
zW4bcT?ftzqzBAv*mP`(+s5~fb9JDO$Tb+p1KhLU#=aTXdm*uL2+zrd^)P5lEsv>bS
zD&^>>^mD7%ElGc;n(<h1)x>3yMH98JWOjwF^LKmh=@s_JNHh88Isd~)+TLcLXNkM^
z<p0DjmcP>6XCv(%P5RZgE5b=HO)B+NeYD%2KHHm%-TPyAmrZ@vaVY5U=Ra#AH;d`V
z<-~0Dn{4O^9*11?C(u%tV`^~et=Fq(v+b=cDg5x@;B2#8BO@cD5|^XTetmuY_V#vv
zfB*aY>+5fC%k}<M;<eM}x09^U%r~*jLQ}mezm?jy2+iCP@L2rm`uQ=I#c_A`gnyQr
z^k@5nN&jr?{!P8LCi3|<vqxw4&bzxy<mZ(;+6!FI9-6bX;_0sx1_q~@I;)=)dhJyD
zEmmpA`|j&uZ~iTDzH>kQ@7b|rg}3>nt3mzKeAjwah-5@XML9Se`1$#H`uTZ(ANSi&
zyVJ?Dvw-P%VeoRlzn{<B`{uqjm6X;n)84H7V@4pO(fOR<KWfLd?Cs+&@UMMveB!a2
z&zU^CgZ->N&5M5;rmH&N?85)(BX+^eKjUx9+;N$E$AuxQ@Rs)S9f6FK`nLVbU-W6E
zn8x4i=y(2X3?j2`&KB`EnpgTlLGr<b(3nZ`8ut9x>!sADy?IgQ7Pn2u;@zKA-<&%2
zf?S1H7b92-1jSE1naV5B<vY9QW^wZSC(i9<ptkFWhlfRTj6}9Z*`}SDk$ZdFOzR3$
zxvi%?!qWa*X)R}6Sj07PW9a3N?2F5KZtpC9-Yu^G?fw0D!?d$Kot-c5>@3d8dbKWg
zx0fO#>y!59r89&ll{R0>zSubb?`x3_g@?4(pIiN7jegB@`7%A}5A!U!=FPkP!a8vO
zgIC*k*K(h749;)w`@MfY15cEc(em!xBfnf9*WX@Rz51!SHSfgcXa7<dIBL`WfBya8
z%p~#Z2hZz1^QG*&{d)Ocd-ff#<%&=G_r1Mq@OXRdn%`$OSI<53$@<{!)z$5Nzvl}2
zTGejN?EJ%eQ$kvrosI3+@86)N=#?ua&(6$LEm3MVm0h-c`S;h?-=DYtzh%|sbvKV(
zFq+fA^q~IkEUVP=U#okbr=2@g#+vi$R=||X#LFt#V(a~D-?#D}X#F3eF1M#%(4*C<
z{;ap}L&kr9La#2=_-mV{8Gmh!Z)tDjpY_qf-;bot{qt+)mWwy_au}TYmh|6TeqrkA
z+k%;r;${b@>^r8-=C!xvdYbNfjR!7v=bRW$@|&1<{ms{~V+wm>S<svOJ9Fa1iT8A)
z@9nR@e?d@m_LC<m_4WUAZ*L1-`}+CUlM9XJG&G&cj0^GGoa}nZ?eO+n)&2IRuR>;<
z<<7IK&3e&c7ZwvUr@#OF>hSeuxwoFYejTm#e9G5IiORS0Z4XQSmG%Bm&OTpc!^_**
z$Bl#ozBhh87t>vr&&0JLZS%YA=lkmeHU(rYst!2o#nxf_$1Okl?^<pNhNXK{rUb3a
zmY(o6QbKY^Ldmn&_N=adrt31YJ}~>R{PptfB`GWnbvNqtlo@@weiv6A4y!Y+yUp6L
z(LifK*W}$be<T{_71yrydUXE8;WKC6JUG}aCnvW#?QGQ6tWeImM-419l9G~^`_1K(
zu}HYRp?<Pqqm$=8&-$B_W*v#`yq;#0ynE}J@70G&*3UOxxbhWm_=z-4yV%`7msmZm
zoqhE9s%i56_sf-X52o@t_{ePk-t_9~?{8XVX6M|x+r5mQr}JB`>o~W=<mS~A&yy2d
zdN>Y9fBlp_Q(oc!qn%Y>_kD4H|EESm^L64uhCjV8jG}f<>P(cFp7+0)KceRSy}e@h
z?oHQ=1(iMzetv$QdwZL0*_(*7&lXGf--y#oTe!SvR_CeAxE_aq?car-ue<N=?!NuQ
zw|$kLmn~HORiGcY=f#zkmmeSR4__ZAD-f`XnI-b`uDR-4FW5f2`uf}7&r18(K1nbC
zGwtV*b?e*{XZIhycF%TKY5Kj%b3Eq!@n>Ow%f}~weTx>uW~pzyw@Y%q@7voo_fg%3
z@Y`=sO8Vces`-|zUVAh*<jv`tvI|S>A69-masRj3h7%PhS5*By_%OZCC;#8Me~Mkd
z6QlDR3t!&a^yQlKwoRM3`1sZZFF*I><Ye>wdoe=G4IKD>eRy~{Y;BaQi_4WP<E8%|
zY-LK^`EvT?33IM~3EMj_aIcN@3@Z-lwEeF&z502V72H+QJ6^<|wB_cml#lDVR?X5-
zR4QB6FL>o<w#$#R|5ugt>&$JfJU;hZmXZ+D_4O+@x9|PlWTg~0`_s89MRERxnhaaN
z{n-4dW6u-IEq}IL+@QOD@}%2a^W~+avR)L}oqBtFdvry$yQ}Nfmw$5i?%O|oapa4Y
zbC^Q>?jK!wxu=Rb%)a{HqR`jZ($CK`EwPYSGBmt+{P^>Gd#g`R)0H+m!(e<e;^yrX
zyMNU$7hbP4yI`~T!`Fp5yq8yfH&;lxz0*K?vsvFdxxSnE%g<z$>)dtCHQ}GmdM0i5
zjI?gYYq_7yce(H0Z~epOyqx*r@5dVc*Ezg?v)d%&@}jn>Q>UKX{b}R&i_e`abZ;;6
z7oHvT)+sYP8?^EwJ>5L*%#6L&-^G4^^tNXI`+oobJ<;3tNDDu{_E~(poc+}GA3NRL
zcn)Zn`bTW*Pk-mXK0jFSfcyPQcgCbBtIdq_oa)QB#rFRBU&hPc!NSPv`d6CE+M45&
znRbQM#<x26<=58onjHwfazA*+cb9p7YxXlUAGp`Xo1yXQz22GSt)WM@l&AkT7h&DO
z{nqG|sdDOn^N=V}1_|rnV}AE9wg0&Ns`{6dfSE%0*~uMm1A-a)xO*=y>bPaOnPJlW
zC-+&`=4|@-=4QzsM&2XvX_cQ2|NngUKOMGqo=xSei}k5<&us<`T?be$E&3H_I-4)k
zCTqKm@7<ln&nNBJG;PI`Cr{SJ?!LFDviRShpYv-zbvh;moW87(agB$E;qBYhh35C4
z-Fk58;8p>X+5c~w@A~$!MsxD~`+F-tzq`A;T;9en?AhkVi5ECxE}dO*Ixx^&-llr)
z_pldVAMfs;e1HCiQ;+5E?~xQO{rv3lk1KoJcl_27ZTRzb^6bFCz;gNL!W`58{CIbE
z#flZP%jHiSHwV~7n(w_IHt+GmD(&!7Pk;SY$%{=l6=!~M_4T)Bs}!%<+}ZqM#(~<X
z=;KUmQKB8)R&^2%-{+n6-hcJRy?b^lDo1YJsyZ=2(Y;Sb^Ir4xET`HpFD}~E{$8dX
z!rPv+;@SfL%z3+RXzU2~mxxanYbXv@-u+^YU1-nB3%A`G%wKI-zGr!~-R^7OMSQb#
z&)u728Fsd{O{>iAwBaVlH2tW1ekrp)mh5P{y-WRXgw&ec8*yJ<d$TxwT`MFemZ_~g
z;LVzR>KxyMYbVP>W#%mSr|a?a&?MRWQBSgzO~f20FDT5?`LO9~{JiQd5B}YlQm2~y
zTTfT@uvAK|`L8X{mHn5j?|HaiPHFAZ-RC-QbULRpB>nfCEx(ra$KQ*yBzs#KR$Vt;
zRjz2eIXdF+F(qyV&9K-jeMvo2ax)Yeen@ZcE@F;li@26|GbicB)USV!^+<kxa&q#`
z&FRusix%D7o_~LWqO*z3?TRPsR=s;#yv{^xd5GWr<*QcrUtiw7K2OS^;K70W_5W*M
zU0K=Iw<6%urAvSQ{#_rl^V8Ss@t|4Jg2ofAveka?9vS|6?aLsx)>7_6_5C&Pi?Y6K
z3pGEg9JZx&+WB+!FXDo3ygF(eCUfTBr4v7;_rG{MwU(*<$n3<;ziQ<Vz2=^HA^p-n
z*ZEh)j89$nSJIc^z4zPQW67`6Z=buRZ_R$r`}yIt%D>`mDO=e~=KcD;htqD;lXiE$
zwpRK7pIw$Ldp9NiShwGqh$@rEI>!C~>@Ha)|Ni}a+xJ;Nimx%U%1)E}v9fz{!S!@T
zZ+$Js0>SQG+ag#V+`cQ`RyBiZ*}wUIF%=)n8M4h=__J+|Hvdk0v~+QG<=X|Ev0Zl-
z1{eOHuugH`%+J5g^%&2cJ9qHl!MeJ-+TY)Jd3kk@g&y}>FIRGUnl5}=agW*phKM<~
z>o1jaOy>A)nOfTr8`ZVsz*p^6?LVhltY7v>a_!Ne58{2xdgCTrrN;l#W>C7ee)-3Q
z*)q|$th;<H|5Y}bJzm!(Yv=HYCv8UmlBJ6;gs{ALmB6KxB+fte+QiKBGQQG#-S*i}
zV@}A?{k3boV8bob)?-Q;3=6kjI6bXWVs+h;>>X1!INy4upC)VN(QYcBW*>Sse*Ukk
zc1OAS7%s1wT++h3zpVYweRCUz6+6G~e){?Q(%e0Z|CjEl3;e32+%PZF)W0V`R`J06
z-7!yIPwrm5ukn41qw#|&iE3tfx@xk<Ti2SWpPSPutX?#SaoOXa2HN54-dtZVuW$7E
zm&%eoY6}(=c|F|{aORDa>FkP-sG~1GpSPcHS-k9Ex>k+;(x98m{pZ*J`<edf$;skp
zXABEWimtr96MelfxcJZS%l(@_?fLCqKJUp^d!CHczAb;g`9zpsOF!1tJa_rJU*=X;
zR=;koV_nRcGtai_RNIL+w{_>+{_gPp?Z2$&O6ZQPzV)wci^GkcxKIE8e*0E$XQitQ
zJuR%;mu1Q6ec+1_xs`c#V*R4U`mfIw<@^+Xe9fiFfB%+(r=1?X+%uxJ_s#scyXx4X
z*30Y_%ok+hw|sn}e3$Xjp}M!GAD_qF|5N)f>dVv1*2i{tEwDA+d;LZC`rhjQm7n?_
z8ZqQ_XS$#JU;XdQOa}A6wwa>u<sM6jF^F$`@#E|nz9*lIZ(nz~u&{i!()xwJzV+=>
z+~>&Kd^R;?ecWDVcD@dmMRK-PF?%XD?%<2|vV76R%I(%CqdDbfg)hg}BCXb+%3eJT
zT_x|%aR246dYq>8ckcX-x35k(ot5o)xU_PgO3xJ8cdASuY`iu;+b8CbWn*p}v67MD
z*!tBf)xxts^;Uh0`f2W%DZD+s=fhMEcJq{JZzt4m(*3*FSmN6^(QsBdH;rreuQUDm
z_qfYlhJj;kt8dWG%<Q1Wq5@0-#nIbOJeLdqw?X+YCp$yhGaiOxK^7-^S&!>lE$IqX
zR*c)M6?DDoxXA|rwI!bmPew63`0?AbCTGt#&ua=SZ$I`lZhzO?Q&K<ClXu~AmWsuk
zweKzZY;Ns*E&b!vCy(3Z0<&+L&6datUH*5LY4)#gZ)gAeS8sLr_p{mg%Y0@kZC!OY
zCuzsdT4Cq*&w?xF1X=3JJl5J8?PHbo?!nfIkB=T6ZeJau<?J4!(CcPg@!`RT4<9Zr
zcGr*Ea^k|nhWgb)UNLQc*(<B9(vRE!(ii)l^8Ne4HQ&$h^*Q<0Exz(jK5FxqMP3i=
zUhd|6o5FD5+H1qV=RbaBh>0+I_Mv%A%(dz#>&2U<)f*N*YwNhXYvPYtk3aLfuB`s^
zq5MMMUD^BQOfQf1*>C%Fr$@Iq>h^5rM^(0Vv)^ek75qM*eBSrfxxIB?kNZuM_>j}>
z*{(Fb|K+{fJN)~7%0wA={MKP*xaX$-ugLhd-RIqjx@RLgR$r0-f7pcKL9$<bpK$lt
zEUQws12bks%shE!&6-WVQ=)7iOlREXZl7<tbhq!-J$p=^w>a6<ENEY|B35AMihx?@
znwLEt`W}<je3_V;E&D!y{o32rr8O;iS_R*#t69r^W_~(ret$;T=M2M_J65iB_ng0R
ziD$!Jaid@1(H!hI865r|TeRxH<BU+93ELO9*UBzrW%wYzXtS2vmh)Gxmn?pnl2d(o
zO|Sg@fY-Npe~P^8um7UDqiyfM+kf|Ean){)&CP!~YopQK`mJAfbG?dUmW(aFG%vgF
zWq!a<^DKqrYZCtA3q=@CpF4Y_cHZ`V(`S~Nnyp@`x<66oxKZ}JlpK3@HizwMN(|c<
zZ|X17nDXiNf$4JtlCGuXEXX~~$j&bZ+J>E!v}mES)`Y0Fa;8~VnpnBdyq#?_za%8=
z>d6a&wK`hM8Kz!#5|j-8`0L4p(z|)Lx97jVwl=z`=+p1-nx#=|%dV^l1dU}IBpmqk
z^fa$*-#oUA3)Oqp9baB_x~t>5O~a8B%F&g1pVVW|e$waobYej*gVoo}^toS^Zf{z@
zFFE_i&oy;N=T>dGRzK7CmEX;uTi-oA`uf*A)A!MQDi6=>o6JxrYgV-8+>C49?^iK?
zn3cP+Q_5lfo3byjL!#Gc7arSIrel;=?zN}%q00KFdHPZd@7J&SzTxPu*s{3VDPN^0
z#&0ZqbtgXDd5ztce}{~D8>AnyGt^w(?K#oCTxyr1-R^5^qu<}z8LT(G{N0_O=jK|=
z*B_X)_Ijv>h;uvN-R=4Fmj=&1ro5e7RW#8rfuUxP+N0iWy{!xhQCm!+>K*Rh4(2FO
zWL?t`bl>iO%>P5qzL(jY8dgSoH&p-35h!>6ylU$Tt%RIYVXeRRYVstX3Q1)Au(ABg
zdZVPkYy}}Eow$_u(s2%!E0UEN7StS$?Ro4Xd7sZzfoZ|Tebsq#c5|&iJKeq;#$a%E
zk4yHW9oI}*AME%L?JD<lLH>isr&5{Y%b3DK>Ob5UD!yHRUgvw>O(PqfGZ!yjyx{nB
z+N4RB*2nK(?A{+%`u<kI3$<%2nsmOpJT-|A@%x<QddW?B+ub|%kB@Zn=HA}M3tBv~
zH^(>9w(!xBx3{;KzrR-tnqJHDO<3kN`M>nTk2j}DKS(x{+i|U`=k&w<VaAIJZKH2I
z+u!`?<J1#1S?;ZTU)6SZh20CCze_^*{dI%KPuIL((pi1`A<yoq&(GbdUeJ<$uXKNr
zcWtS0Tl0dO=KK5BEZpxXZCEzPV{MpoZ`RGrkC~3W+265~b-~e&$mFQoE3YybBprzm
zypqMRNbCL1*OPYe&%I`p^3wi6au{3u+YgyvYrFe6@2GDrH?7Qnu=9n~Bi46!d2a~J
zk9~e_?)IFUMmaYO<mKlVKR<VMUF`4VzObhHORq7OrPTZ{UnX-?bY<q{qZ|!4!YtNl
z2D`T`iSTt1<QEG6P{C@u^;;`bsA|Ku0{f3rwFj$8g14&L-PgT$nQOtb3WhbaqRSsT
z-&*o5F5ve!L2kx{TR)s$$HcAhvhmre>hjah_YZvEvd8R0>T!df)&FZYR-UzP-}f=^
zrqP!Z6OFbm_}ne7zpM21xBK<~uV$ESikqMMG=p#1p#?>=^tF~(8ZvE*yPqobSt`)I
zKI`FISvj$1@80FTSnwt7%#4rU@7I^Vxlwp}nyypgOb?e$iP3zLy}I#?9Y1RC@BHN}
z78k-7b>+3JX3f4>7e?b~Yq9dHdt#>kw^ho1c`dL-SVmd%j_;-k7jvIKy1H;@=swn0
zwqO4_Z4`0X)LJ~-OqaX#*c_=Za!)sXsp0u?d-b<B>%-FZ*UOq*yZ-C%@289MJKh_v
z=uMP<wE6YT&CfyWckBLC#7g}#G|y2jwf_3z;^MHiQFHC<_l3Ro<ZWBCK7muCVao4U
z8GpqiE?o0w$?%V>dH?53kNesh`+|Rgf2#G0GK+E*pYIT3VhC3E@8wHA$+kn}+x+N^
zqtDiU$$x+HTJ`-W*)@B9ZOSuXVVT$G)bRhi*Tm&(&t*I~JXiYZjz<0IQETn9ElM7*
zKC)!n+gq*e?b22<d#%ggZTax+-~0Xl=hUQCev6JfC=?K2smn0+a?_QMGb7jkU#0Fp
zZ(GjQSM%%t8J1Wud^pT+|L5E7{CoHA?Jj>W)_wZX)Q}e!-W`7z#IQ%L;L(fs64&<E
z|1K|mc5!Ljay_@_vzg7Z!>+%X`#m8yJ52k@@(FAX(v_bb-&HQ2@aM_1=0o3lb`^hd
zyLV2qKmXpD^kpmN%+S(Z*7t$oKyiEe`8Ynu_IJ5m_l|pB7vL|+$^Fsx;|&|X+@Is}
z^>_By$8Qh1&Mv>)rRYNT^>tf6eA`$2yl?gYn(g!UPg(EjaNqGrTE66i^8eGnu3lTS
z&@!T+{ND+QzuT{F^y`a`NYB3e{QUk?w&{^-jcZvLgo@+-+%vr=#t?AZ`;z7Hy^9jc
z>@(&2%hM!o_MhA|xwa_gc#L${@h#JKcz?QWvHZa4n|f&{XFXW*y=7Lo8?$`*syXH#
zo(8S0d#{%KeWgX>_jh-_GtPD@?JRicbUJLU+;Q7)FMhWdPtWI-_1n0x$m{8s<SSpK
zu2o!FC$jU>@#E?D_tm!XKD!=YKi51zZn0Z$Q&Uq<@96+_$zCp>u=`e#(<lC){_{xn
ztqZ%}OOz=W=G>0uaVR!=<5%=8-|y4m$yp3$+gPuB|L9rnw~DdC;`t2q>mEPmSwB~O
z{I6!knYu+wbQf3mT1_qg_u~!w*{T(L_rCdl*mBo{J$tt;zP;f6yfmGY(I0Lt$XU&}
zcr)M4{r=9Im7Y80zmMU1xRvR9rtfJt=bEzQ<9&aBKA)d|ch}XW-qW||e^&X`8vMud
z#mC3TZ*R$+Jz>I)gcHARy*+ijC%g5qMYy2Dk(6^6gv8gLdok_If9aW5mDEn^)Jcjr
z{D|ny^WM02Mf|lbS9|)q^CqT#otV=rd~<zt+*#L9#)3QN=GfjaYPVzPY1McgQ)4AQ
zHE^$C&4w+TRT(0pHx;)@Ur+pb!T9Dnt4cjTw;;9yGkdgnz0*JT=e8O#B<$KUrRKG6
z(7pinWe!J-7!(987zH^0+%2B||3}2vxjluw8?SSlpHfUWcpDSJkSV&TQD0npx!GB@
zHm=mGZoN`pKcBbXU*9fN{NO;N-&`xz<x9+^c3xihQ-j$p=nkma_ctQ+?yid3DhZ{}
zb&%y`0f}qk_WoMEe&4F-?e7jYv!6YC`bq}Z)4L~5-+6j{ChMUk^XBKBu#}j7@vd=o
zag+DS9r?zmd=?(j{q4`lko)A>gY22n`}US)$(}b{^Iu~Bno~}Tb53jQ`}6%o*gWAK
z{;$`+&s_E2D}LwHohoU!xbHp@+%;E4ueR+v>w)ZTtUgiS4I-HxX8tKSy+)5aS$f^e
zaQ)T#ivr($dH3?Q2t#R9`BmGyrp5_pPyUu+(0D)p_Pweb{<{C)xBoJ7TkOMrU~b(6
zmIm)#)#l$H%@JB|R%|rw-JebSuUk)k+9OevvL)uq$-fL%Q<t{=c(d=vqwf6uf4}Yj
z|L^s+wX<X5t{C<@Y<#}p_V)bmpFan$nyHq|oGblJ`(CrO$mM-IC+=opus)z(Y%7$z
z;hoonH8-bj^q9L<)%ry6|7{z4FWh%4pL_YA)VdHRhRd7#uiO_BFckjn^}2&$XPWO0
zDTbC`O^@${$UIQkFwH17%4?a47}KARhkvbHGB<nUu8V9*#f`EI2kv!lO=D~@y|{nV
z(;T<|S%TIz3r^npxalbK($bU)xurK|vIt9Wo5sS>qx)rTMbF9wt2h{1!g8!NgC1RZ
zF7&n3@|)bGslqAQ>L=Pv1)3h4$%HXhEUFJTsF?8Oy5KS9y`R?IJ;eTMdhatn_x-Ei
zUb@w`^|GkH;Ny9Qk7i8VG%u8eA@7c+d2yClr{~sPpkC<OXCLpD-}jwo6S-RVlf#7m
z?Nwi2?W_Hr6)C*lpD*(ISKYr1UOZESwU!?}d}m$Z=JSV^hV|{LjP<I2bzyDv_PTF3
z)BXJX4jw!hx++BT>E`{Djo#gVa8`ZFo~o;T6Qm_yvqgTJVaL~Su|f0RuS+bcpAI+A
z{(r&NuJm2Zne4y5{-Wz2*8AnLKDeQO`1+hZhZ*zb=ghPHBhGL?f96xAv;_|zSIQkY
z$6K9e=N+Q;^!xeT?~?zmS!A!RlUjTF*gd<hy{A7jJd(G)`qXY^p}GFOs|+5s|AP1!
za&}*ge{t+h@qfM77gq(|-&Z?(+pCQRE{2I$`+d(y=zUB(;wjo-yglj5+P9xt-2E*W
z%KPGK7d>e;z5ODshiOAi*7~-uzrTF^^2+pMdgJdUmCDkLE23_5GOWM3t>WQ)Zo8Kz
z8HcahWb4fQ)jNl)LD2WrqmS!$bEH1Ho2?$ou*E-K{e<zpoUY%;uis_5aCdjv!+)7P
z4|c!b_xsQ1^ZGG6F02mMUuv}f>biTKn#UGx&A7NIa&wyJnN{X;o3C%#m0l=&?SX{m
z@8cdTGyAUpSn&2!vnIohcy*8Y^F5-47PDShwf4(po3|O6{!=IOtz^7$MWF7UXU_Ds
z6O|PGzWx>rea5Btc6Rsg>MoznTi0sd&)6m#Hpi3kK#kdn9j&h4X4Smid);U5oY=MN
z=X*vMd1pUX`Lk{E5hY^*KWWK@F_LF3cSxVj&Y$v^qrGoc%`T5ky^F5q-JMjX(<^z^
z`O?O;Yx0x+IPLwoA?SKy&&HJLd{d%c+?$a7@5a&->y>9F9Ars8&3=mWA#16T@EOU}
z81?IKEVA;x{`!(W?@{GiSBC8szEww8$DdphxM%OK%7{yUH}9?fes5px?vP*a7M?l!
z5)?*1pU>}In`JI1t+G05`IU3gmq4YLZkg84)vKl1_tn-^9_bK#eQm99r&p%srv(cZ
zFfy}+tdIM9V`DNOpX<C!dssB{&S#k!6hBmA5Gc7(oLCtv$^7qR6WiU_a(~PZ1TsB%
z*%@DJe}6{y`*Zi-#F+U^m)2O{niFM~U72!B%PXh*Vg32Jm+r*b=h(cv&#WDOp7*3#
z)4ro8EdS58x!`w<;gm49d)+?O*VV^2)IRH}H_SiV`sVpBO%0_3S6x2-c+{kCcegsc
z{<wO<>+MgDpWj^(;<>l@!tVGmb``f9`L_?gt1p+YH+t-!-14{S|J>;4Ibs{XZvV(#
zmYR3-^Z#i+@=voDYnSEcpUpMS&|lB0@XFFU=U>9?npIZ9)2_X*DgL)^KQouqqiGwS
z7VmFWFG|VJ>CRmE`q=DyHy+x)P<JzV`Q@r->Sj5U%U4zggVxz>PCvh3!2%`T+1m4D
zLRMe(n{W5G-~OM$#=>kPHIJms?;nq^S+T}fl0BI1!0DDZ!tT#EAG|G+n{#n_&$%Z#
z$_C<0Qvb?q{xgO?QCrqCTdzkYH)oyXc2}R^Y_<pb_Szp#C+8{~_@CUIyzrd%hjk}c
zOxl{xWmj=@!POP(-p~7Cq_rXS^t5(9*-e|YKFUk>x*6B~`2kwsel_Px${9Ij;o0Bc
z2S)~4*50qw_O^T)l6iewe_g5d8t>_Pjg5`VmMwdK|M}yVwzhAtuAaVh$;ig$&e5Yw
zm-wd#oO*ISUsQ^t)ku4F&8|?brzi8Daxn-+iW=Xn{#7g1<6Sp*#RUc@Pf_j7{1#m5
z{(e_l6XTBhoOxEIZ}(9*At^cdfz!KF*FSpA?2WwGKDqGIj~&bNTf|J?<aXIC4%WP!
zwe|7w{`*_AuRl20ygm1J*pHWoyA;_&HAD`#^YioYY$?gPn(?G;;+0Q-e+xJ<%zOU6
zt>*iSJ6`6X6|Wnjwq{Mevok0-n4L#L0KSSgaxs(jC1IyckGI}ATYGuFTVB?c6&EKe
zyXWL`&-l5?Vvc3;vFq2vwWhYUwN+Jx&3Wb$^f1!|v`K}XoxNY${N4Ng|Mi|EeCa>y
zIC;{fs;{p~-`&{>S#}$Kbar#+gh}6C+&N|4zHe<>wsh4KkHv1iUXMid<M(~}{(XPZ
zQ?K4-hi!iSQ%pF0N5I?Z)Q;-oVt1^+bvv((dYG7)mzSqn*mdH>3BS2kjG&DnmzVi^
zKN4w~A~V&?G~qx)^6|dfk4ME%Pt(;tqbEPL*{=HAn&Ri@w&&d~$}qjS_PD-JWYnI<
z2}Ms_Lju0M;|%J&xbVRPK3OY~_Xp)|DlVKn{`_GxyWiY>MenS3SMd9JJpG7R&f71O
zd2dhUN4Y=8dZlx3Z}au{w=a0$FxxD*>-&S2DScnQe2LpzHP^nr4zfk6`u#Wgsiy0q
zw!XTyHX3B)`>hj%0-Wq-<Tmf~J;ru2IC%fRU$2wwe@s&Kmb0sPaIKh8bwNuF&+as3
z>E1gtzx?|8{eFG?8}|QSug6cGG-+GjUC?s7{Y!!}6A~6IUi{eK{_m7oQ4e}f&9u`h
zyyDBAdVP(OubxLoVua<^sbPh`HfUbE=UaYemT$Y*?}?l%bJpD5Ep>9$G#5)>e}DE1
zJByzu*&8V?J$dHLn%dvrqSl5jt@`hED5TPp19To1(n)j|KSUg!>?wJA&woCtf1VfB
z-)iejOi0O|D0+In<@#Np3e#7<*U8D#_3(aFc4ys;(@O;wIX|5;;nD%kr(C}4uB&X~
zpD+8RcDeVj?>~NqXKhZKd;3<+ojrEz<zx2${Id4Svb^ByMqAf~&UCX}@vt;QTk^q-
z^sCb>o-R>l)s6eK5qSwk(e!VN1ipNqc!g<Ryzl+Mn1g~rYJrxjOP;9a%$!$!QK9YY
z{o1<3*vN&?X6vlFa(m(9fA7^9ny0m$dHKoE*hHQ6tJKDx>pl_nU1?XfY_ht|_e}Wk
z<zn`wf7O!>J;OFV>;Jk^wLFn|LBNaiQ)f*oPSX=i4xeJG>??aCieZ7Ru<umuB~6Qq
zygW{Bmiu`8X+4wp>uZ(lk3Sv$_W1gGU9E*IA`Jnn;+s}Pt!tfSv2>3{_x$X6Zn}?6
zdp69EcV5M!%e0DJS9y!0xk+zSH^Y*>4?jiU+-U)t@^0Ro{3F@$=Cr@8#<eTXyxPCT
z^7<CmcQb>%%M}^UO26}o;&k!!m<b9~q4amx@(PL<%j;`NHg+$MKB=dvQ~1eV@Xa?a
zwY<L2&3o7I%)9sN=a1z1W*an~6h6|EV|`$E<+x`|(95D$m#+&_+te1XTD)Y}-X{f?
z)2sy+pS9cX(atxiP3S~-K%i1a?s2AyqUhK^pQi`}nSldVa;5=8g&nW_WzMAm5z_AA
z{Kutka+k#1_>}Lbq0`!`|HM<hnV<Q^yXy-=>Lj%IR<Y}@U!%)<Hj8V)!-Egs3G127
zSLF8bPIvU3so=Y^xB7_d^P2}QcW(81bWVlQVfD007xQ!)w5Bm-2g`BUIr1`GGiypS
zW17h3A6KMr>f!a!SU-6gUyiOI!`uCat!w)gmuy;C<mKTRtY%;TMTOHz(Ix1%)VH*s
zJi0-4)dlZEyWEsEu8CURHr1kh(T`srV|VVHymzL+fy>3xf75eVK74qZovO>IvZ?3n
z)bnySRg>3EoE5fp<3z>V(zBKadh4%}xVC@m?8=jWXNn3vUGQ{bf>->;>7{)^Q$9Fs
zHHi7D^>}6P<P1OYjhWkcC*3bNKY5GO$_GDfP2+uUteAEoq@`UjGkww)zITuOrC;pl
zye2p4s2s!cnR~-tUUt5;QH{ILpIteX@x#H}Pa8FVF~5@f;lNwd@}&QirlP8@X3p0~
zlG=P}^_frp^u~YrcFM;^wEz772lj?7=XbHcwoEEY-u`OU^_6}1?)_=re6adIM`uX=
z&cAnFu!*fKFZ<#5Cv^MGWh?KRGv?gA`TRSNnEmb_bKn0Eh`YTf_;a_q9#e*8^6UG_
zQZKh%F+C#9`*~^Gk41}Qw%if?lfR}=SMZY16?=Bg`3!q&m+YMzneD!KK9j-21}=u=
zQ|DePnX%S?b@}z)@D}6UC3j^bjJdC!X=1I&_TIR0!bMY~3=4*i-OGPz_Ws_VC#3n|
z!>XjWy62oY8kYwipS8xw(NxicH6erl@jBC)Ie+i7=)Q?y;7>Naa=_g~F2s*hGn4c6
zv0r+7SG=+*z3MM~>*RsE7F$X#T$eUKFDE;B%ft7EXBiZ}UgQ5K>>Q9D_(81ReRB5h
zebV0^H$KqSZf4}T)$^uG>CwHVyA7il+??1NI*adreptHivuypO>*pfMD;6woos^vF
zantnoW`Side?6`@-!7Q-NO}41o?wa8W&WQ3S>zcFt}a(JJt8jpm0dS|*N=|uPKLf{
z=_NHx3^(d+{>Cg4Yq)Xk1-Ce3gPv}$595lR6C`vSY~82$TQCHut#DgEv3_6Rzmwa(
z$mrH(*_kz7|GMkqlFa!_K3w#ynS4u!A*ICnc45p)mLB781}i6CJW$^4c1viI^weM{
zuUmhev!}&{aNbd`HD5TFhv7z*?Xv%t`{Ww9y^qP%Enq14|6-d!oOX`kir&SX4Q<WK
ze9J@`Kg`>uXqEN#O#F5Bw>R!iJUNYHW0JA9aV)3M?5I;IX<=zbOYE$~SF)uzadJCk
zT+TW5X4?Oyuc}vN#ksM*lzL>F=~6Q>kS}VVLEP2|hD?2v@J)MnX(u<Vn=Q@luw=LR
z;{_M?ukWo~%&M?0VXkpmWxoA;^OZIX8`oaEB=@LF>BGxAD<i(%?NQ$LNrcN0lz6*A
ziPu+7atpu1a=W){HpitzfBWn5UE@!U4#U=2><p35B%D?sdpn!4qi4_5IUD;8xh7q#
z(KlSl<WQaUWya3`HXbYwZWSInue<v7y&mg&6E+*uBlQxe98U|oX{}H^_5ACU-*eWn
z9o@l|&E8RByxQ}In5694dFr1TCakNCdr=&_YxTw0|2o-S`=qpIUyRxDc!$<wsa5$8
zs(nC?b9_2wg3$7c8danDH!rsEeJwGayIjxhG$ZePK8NB@cWQU$xp7}Rc94zXbwKd6
z^5;z)Z~q><%DP~$u*Jgk-yiqCuhz=Gv((X)VJ7cK;T4MingjTv&WT)ToL;|JyGKWv
zah=VD*r_wt#U3$YNRYa*fnnZ#?L$j7zRs+=mzjD@X0OlRW0QIhtNJpeWIwR>2f3#m
z6j*B;PSt|OJ{k1G3mw+Ftz~qme)WBAx+R0hPBB5Ygr4|`L2lQoZ{1Q=j(RoE<&W+0
z^{#(y1NA4aRotg+dZb?9l;de$_VSDTxvjfaPg^Ias{d44Ppo72+F!3%u{GR?+ETRU
z<EM{C{*sg8SA@su99nwf$P~{#Yv=8-d>iC+EY9fR62_#+H0F3C(+`4>bQtG#<Jp4}
zzwVx2=l0ApU7gJ~W5-wS=-;dj9Zx6E>X~c0I{&t%;!E4~HBz_tWYt-VwoE?VTbus3
zw&wSxcan46|C{GLvi~R*>%3=$yy=m8g;S2FxexC6_UP8LKee%$lQ-n*Y~Fu5WGX|0
zQK)_6wywvFHhWgu2rk}P=)Ti5;@?lkzc)JGm{jVO&0JsdPUY{~)gAuV*Kv5wuXOp?
z#mum9<pL{-TITz)Sr3#|#I=^UOxbb7+tK*njTaYPVPkSfb0*8r3t`@{aqA^pajsN`
zG@bb0*E#I>ugq@pcll$qEcRYo#bO)dOX+(jRb^MK&g8cY$-a>K@au`*`~6G|cF!)Z
z7T?wOqx0@c#ed9!d{JQ@M(-DWipiH)`QWtUr6`9tv1d0%?>c6)EH-=b8QvJ_t{g^(
z>L+ug8hSTh6ww#7$a@|6_1WfCSJST^pZ9SmKNrJ_#QA{*Ex)cE&brFN@XC6#navSC
zr&wLTnmNzvmxugH$l@sFt(jQ9vavtJ3RDDE-ntSZ$8cDJTXpAa@%eGq7Ik0NJH5B8
zb+6%LxOL*}!Rto%YpdU0v6`10@!zn{k3r;n|Bs%3GfU^+T)()yyHD1-EazcH@}$od
zzYdi?z2^OwTPp5)QNim#rH1SL@9ct&%5GhL{{Ds9UD0aZKYxz<2{F|CF^zSui7wCT
z`0rq1pCPHxw3X9(#l$+1=8e5CPV@Op5>is=bw2wlM>aZ)gW=YS-?v_Co+{?ck(O>O
z^j$1}J4<}d)hh*JzTd?dcgQZZnZ(Ahwq$eA<~MIRBU^nBf7D#Rv({ZSJ?KG9_f#f^
zBhQ$uA8;O6{vn@HGvTG$zl(QH+zb2g$do_KVDapW{_Y{`8SK@?8jL0upUHS&J&(yr
zOAr*zOn#E(niUUSRT<)>H|HPUm$r57!m#e1-mCeK|JiQIe%*PS!RmL_wQs9_KYvnh
z7hU=9hW_E}S2$$uU)242@a4_J^^70>PK(q_Vmq+6zK`iao_^NTof@Bc)~!1$Rr=Pn
z%q8Di^7FUAH?i~B1^&NSDN~xl;G;9=z#Nvcvk^JAXA_U+&lYFRW&LpaLj2vs`!kc2
z9}2yXvtHS>p!w!yZNmq>ze<ay20H&?UnFzo!yYN`WhUpo@hb{e?2>%Cd2UR#Dl5aa
zci#1{mEXP5+46nDnlB6tE4Ci@*W7mJy1z={gDtDR2Y9aA+p3eCsVj3k{8Gq4vn?0I
zB{ojY_{UOqecBe~%v&AoG5PLd32$;|-tc~@-s3VqYU4CMxBY$>*eCyS`g@%5>x7SP
z8jm(`I%zS0Ld)Xl+sLzQX=^i&9M6d}NH4CFF3X9KPPkpSt!DP&WvQE|>CeBFZ^_V*
zYgqI5j^^37U%!`r{l#<e(1)1c_x4Ixe$Pv}pnLVaDN}mh&Xv`MkJfwMd;OdBul04;
zn7yy}Fus2Frsdm}uKOR(E_?j1bW4BCjpcv#O`OYADs8;xf7jp3^5<XhCG^Y}H$75s
zaLSSQ{Uz6<FS=cCHQz6e+4$Vzan9ix+zYRKxcJd>)`wr)1sN+ULN}VaMPHsYQIfG?
zW6#>xDyx!@9%DT@{aLu_CVQK|n|crUHMOwh=2$Vj`@_m0neKVfZ8^h%hr%zf_7r@V
zGyO2}R4b?;^?S7ML)Dc-N^cbMZv4H>d)Hlcf8FW}4jNM;BcHx_Rr$6+qCxUC-}lwp
zmA}L0&0EK|z@u}2S=swTUl)E8FLOPAYB|^DxcN8#Z~OmcEt6VkPRZ)?^<pJojU6=o
zr=1p+;y9bT=E#Fb_53*#^|q`HUh>dah5eM{X?;60yZQH$*YO^`D;we4l=aS)x8cT;
zGvyY~4EHbWpSbJ6>PwF+-ah_5_vxqFU8@9xDnL0rGs#$6db7>?eS5bh-`F#^ba|e{
z<wJ}Ok=Ml5ea$ZBzMd~4Hiz-#nw(F4(L$%s9NBbc@dmSReP;5K6-(WdCuvFXSBo`V
zh_X1O`P*La@}<}>?(&90SJ*Qbi8=A~f<of0=M_zc?f-2`K7PENqpv;xuI<;-_LY-P
z&cC*lb1F~oD@W~Fv-@}6Gi7Z!u)KRC|8aHgiVZz8TNoGAEdHq+k@|vxQxW9M!ZO|{
z#{VB?9oTr6iFMuCKe5@$x*soe3N!3+|5sE|<L%7up~1ek;9Xqsx(8dDXG?`nHAs*&
zmSp~LGJgkmc!+h<ipstTMvWjBo!#?r#pB)CIT3q{Uw77TEm!8h+spF!Qt^>B|Cs;I
zwz`z#mCE#Vf#`*$8DE!Xn7Z`LWnq{UR3Rnp?GXv;<4g)WyZH3=`jVfsf=`J`IcQAm
z%;c$_>13I4bm_DCD|ls@6`$N)F-IuqQp83dHM0@{6?y21g)LLeAP0*%d1Nl$_CrT)
zSyS288>@~nOqh1ezV@uE?c=8jzs$w&?OL@jaM#<^d44&QS1~c<Pu@|Vv}OIh?R&eK
zZ||M=;gsgnnO~IVpN8&&&^xFa+Vxn(XZkXye;a>A)YRp}_e2C`{`a03<o0ySky9_N
zyCXKrw4BOJJ)r7(*P|%=L5NfDeXpG{lb^dT3{cWrxNhE@KR5SUi|gBcmb!5Fo%6Gz
zAiEuZ<Yi`CMb&!j`ugkO*V+E}y>effm9Cx9yh4MWl|fj#tjgU}w%aCNzt4Zt;Y&P!
z)?WI3G-b(~>w8u6(jpIj$y{$MJa3<!-u#*C7}IJRmh`S&_vT}N>nWwVr;kjMy?gG&
z>1nLT=k8zA`L|<(_ZN}3v!%Xf=A;~aw6M)uZ2j5YjoHkB{l}CVx_3B)Z{zAdm3eaN
z<?mt}lPo!=UOx5U<;zlM`};dyoKUU*eaHKv82itM`M1tnE<EvE_Sn+k@89Zbcm3Q_
zpgrwN_{TSY_=DA2?_6bQSa7q??(W;JvKYO0pE|$V*KYm3?%C<L&t=x?bjdjd1*{i5
zU2(SM>+)G*wa<$hivqUfYf4^r_-|>`7y54Zik*uetkt>s?riU#>`l|s7`80^QLXu-
zaK@EBRtq~{?qoq`hBXUf&0ns5o5}t;a(Dm!LU+|CJ$te@F?0znPw<NGoL=gd&Z9d&
ze&#=qqXk^z<rn{N756rtm&G+>>#^gJ911@hpSfN1cRa<awcMd|%4H|0SG^Q8OOBc`
zNSDQYnR{~`OO{${?Y|vYUjEydxp$d+Q~QFO{?X4(CN8=6>;0j=N^L)xF3Z9U-P=h~
zFXA7suK%)A^{UNsof~g<{yW~<p2xUJDb+jwz1-}(@BjOy>D1(_{F!j_gQ@@GX&<HB
zZk%7AcytX5>sR)#S@S9rFU#30%`c9rRc)QV`lH3X=vs}x%P(v3G8kOd_0BYH4a+@R
zrZew;pii9k{4H^I&pRIV{<|LY;JVwC_j=-a#kce750p07e~j5%75{6o(4IW;!<r&m
z{|?!|lwS8NxBBEEk4S!Qwg>qx!B4%8x;)+T<Wy#27c_-VsCs_n((WvV?c4HpOq-KG
z+riMFJnQD_$r22QBX;kyytUixg;l|!wrusZQX%FC|6;%WeRd+rf;T(u{jCMr-@nOk
zTD|4ghgf&M^?yGlpOY{DsKM}kp`D*-x$SbVxy?pFWg*Fb_ny5f+@)J7bulS#^|kvC
z8IIjrQ?ifICtF->edzr;m&$)u$t?f5|FGBJ_f-eWOQ*>P%{}{m?}EOL>&vD*sJU^p
zpzHW|)7gu6MxT72B~$WIU=~wAcEA1a55g0Tx=&?xPP=?a1d>j@8g}of-~Ml*Sy6FB
z=k@Z>HBv_TH4(2B<MpoC?)h<k?UN*{)}6hF+25ol#qV!*_m%d(+;35Q>EwgM47+}t
zt~gLUE9}*_A9}gHU*ZLJrbYYw+duF5|G?gY$!Y&*pbld^zrp0D^&-q&Ky+=TQqryK
zf)U-UAJ3bnKdNL9Fg&MbY1Zfai19|W(JntVFOICPACH$8>s(%5yKLuLf$C?M*SBT9
z*V`b*eJN<ian+yaZ(QjtH4$C1H1ly>=Hkg+ulG$W+;-;UpS4WSB<mL|nSSr-eE%-=
z%DI`W6aFl5`E^pTS^H^GR_@6e>9>Ra=bj0xh}Y0cu+&}95ooFEdO$rca^KTs=Q~bL
zs=wZH=c>&A%=3r$&D*+JJz$Pe<h;GcTW*~3;c=KZXU*=DNk=#3Y(Akq-*(4$Em!4<
z5n*dX53dq%?rSm4Do<xo+!I#)@}9S9zFHcC#^m|-Rg+j23Mj42<*nXa^Q%N_zHP-9
zHCZRcC1TsYnWf#le_3SZdmq2|c`+0A*M0kPRJ+wp!*J`0ml4j>vhO2}ru_RI&N0=J
zJyj~DM53}HHuOP5nCW(7r<q0W<xhouUTS~abS_XxNN$B#+v+&R*je4RuQ+ngN;Is`
zK6Ncm;nWUhq4Hf*?7kbN2YZ{`KCY2{KIQBi*)H`2rd?9iFWVwDOybu~eBRlzQGn5M
zcZ8)y_r8xOU)6>>#oTe7zg=*4+VA4aLRa>)YIZ*sI>hrbIYxc;#Yxs*_^&VBI`Q3c
z$InrJzsfzWZkKq?uaYVq#li6J*xUXrAC=Tl&FR5!4eTQq=UiX#I{nhLJC|<V;7%{U
z7{7bne2>}N)ZX7H(OUlAMquh?uLsXI${O~3HlOcnvR+Vllfj(pEblCTpA46MV`Ls+
zV|8)W_3)Ygy=?!AzWttOq;dYQW!<r9vl(3G)^|o+3+B;~oB#h{eG<p~((^Zqk32b+
zcQtf<aO~e-=IbZcGt@k~yxWGq;p?hs=Q9kRiL1rWM-&?THdB6{|7X$m@HC}a6`yb1
zag3BTXWXEBEceIL&1z<<h6}WsKb@S$S}gvqvthx^?r%Xn6^*wy95@@l`^2WGq7vSZ
zZij2M3!H7VN~8C9B+RaKUSD;+ZTb3@>sK&x{4>9H<Mb?F?|}1hldY<JU;T^VYuLK@
zqCcY>Ly8zvbZxLg@czaVFL=MluW>6s9)8>Vj$g}<h^sdeTK~xfoIQJQVFeSzpRU-K
z>lq5}{CeB9$NojnN{4d8T9MWJcUbO_IQv=9`=I+g59z&XrD+=;g>sgxFUrmM5?j4d
zb2YEElKft^(!;)d4I8&+INeUl6ua<mhuzndV<&Dtv3|Ap;;PS`Q4HIaFC7>Eb%^c2
z(a`$`%Ju$#Jh5bs_S>RU`nM;3J=^nFJ@=%pZo`k?uT%qN8lIb9p8h0U-23U5yDR2|
z)y!(IekL#cRpR!Ap2D|M517~vOcPzWesiu;;j6O0N7~X77&hE9K3{h~n2jOneq?r=
z@tOm*F&`$)OJZ2Lz3#(&i6*P=Cgnwo7jHky$sn*N_356Qo5gjy1D>z4K3z3U=+pCX
z!v~BdFJ9<5AKJCi&NxiwjZNv*{TDA@ym-A#SAsX;^P7voW-qF{_wCuT^6!C6^_gYb
zeJ(EU?(x=}(=84c{a+`2^XF;p_1^EgSJ`}Vp5I!nw8uWSR5$DJz89al*(x&L=SXjQ
zm$Oa!#*+G`euf#ZE^lBmKeE{9`GOu6$<X_^X7}H_FTSd~f5$YA2RpVpo>FIM_<Q{F
zdDSe&j@;gT1%hw*{pVN3)K9FDSUO{acJ0l=qg=J>wa22E7;dG{YA%#v=(xea{`V|z
zva$iI(7P4qo|JdKx)Y<q_Q7p$RQ*%op6q;<2S0s}o|4dQ_>n$q|2omnM|&r3n!Kl%
z!S0jCuRaU812g|$+4XwM($I}lJA8kO=rb^+opTGe@~{dMRIv}x+Yz><FaPbGue>!T
zm$o?>d05_+S#^asw|t%csT_sX`G*;9F{gk2Ul(LqYp%6i^xkRi_HUnVt#8{R%IstM
zzR*^MVaxXEt@mnX*BT#NcJuoE?@R|=b*%pT#A}{YKX&ioR^??p8<H6x{JWT|xF_sJ
zQh>C(Fas~U@9Z~POb={JolCBCT{GSEe8DI6?{dfOt7S^x9k{bEcmKWFYz<%M{k*%u
z?K}V4$f`iqg85(Wwv-t$B)tAQ<KOciRSbGMlEpt7nNGFW_eC-#9Gbt``o?UAgtf2E
zS$$j1Es^2h8(p&^!+3>gqnF%z)&n6Irv?T0v#n#e63A~nJKRKnaZ?HJ$9bY<E_?i*
zo^{-uxGHK-z4ZDhorW#qv$Su<-7R&k4=gNBo@n~a;rDyf+KW}QFM8U4epGDXR6jA}
zLd5+eSC>xo;Q9Ba{^y@%#%729ozq!xS9HylTI<|7ty?1){w&S8t^MR^&GBcC6t^%e
zcy`E0OZSJ!*>9^~wfmb|pWn0DXjR6R)g^OZMKn&IEl_k;{-Nt`k6`Bb)S#-TQ~aOn
z?YY3zFmuClRR)8gZA;fG==#Y{d>^@T&OYI(m%S8JYYwwrt4`}@6JuZ~eD1&h`reHd
zr)S2ToHIMWJ~s5lyA7*%h0D#c6wu7Lzc|wOZZKrLkHH=~-gn0Ml83Zq?SlH{>+ZJs
z{mFYD%X@pz@19oa^Xp_Yc`H*c*fGrgwPo|UTeEU+eiy#G_`~OoD>sUN`xM1tRgxE?
zTg<&Is=M>=QidyviRIn952P8CeV2ao>e@zs?R18KeRV&A_!=fkrrxtSHrHN?dEX)P
zBa4H+vtM=K^k8gQnW%bTb9v}SgCNJ=KWQ<ZJI~ubOgMgM2Uqs=sp&?GnqDjXP&r$l
z|3gfy=SJ9#vk|2#z8B6-JenO^SIlK(yvy^5!xPS<6HWelpPduI5Rje!d~@x_k1NhG
zd2~PDRIX`V8(6t(hrdX9wCFpvH(A;Y6AJueugK;6VzcX$`k|%AJoWN})vwmwJ;HE#
z)y$0a9k({`J0bFC=INi$;zIoHH=oLEUB06}@1d(Ii@Ni7?~s*R2TpiYI)0z_>%uSY
z;N7Js;wm%mB^=@CKcQ=`RA(<@eC4$3AFVIH6HcnHex3HE?aGIj3?}O5?@s4mrJu|I
zAHV#4CPJj-&h4f^2BDcdr|K-8zZ|><)@${Jh4)gzFP_@{Dp-7Ra_hyPe>HQI=lgH?
zUY_9}TR$b{{$iD5%o};x>qJciJnDA5|Gxj(pSc%fO;=Cexlggq$u+;jJ#`)X)5}}+
zjh9u6N$>w3wzwf@ZvEol^EUfDv^HpeG5ZP2TN~D8eood8Ue*5$;qT#^P~}zdcm9^C
zH-FE6xo;BFlnX`~R#|=Q2{~FTq9&zZwEXT8ADD92;mzd#SGrg)nSZcd%C?W|bih*+
zRX^di3=GEZiM9)hV-=s=Ir!(={3S)d;zInit@Mh{yD_Yb%YS}$-u8rpO|0C{&hl3-
zc`d|cmzMqE>YUq~&sOs@AIN5MUcdM15!W@3c2?R-r|*kh9hl8<VcxIW)$87SpW51C
zz>xf>=y?jy>-Q0-kDWTUtgY8KjERfE!+GDCoVO7SdtyQ_yj`=iH+12@7yU~8H}>Ax
z9NI5`Zbly8|Mcr~8s5B5H9Jzi<^}J^6=y6p4jfl|wZG*XZ_lf}`Ps3PGGbb{o_-(u
zRC?WqQs%<#ru`NSE8^U37ETn9Dc3o#(avbwV||M&&wYu0=ehMuHZZxLu8Gr2OUpTI
zBQEVfJ1JL6VB=(|^m)z(OO{{UZFWv`aU*|uUI24EYubvUQ|&L82Z?`qsZcik!9(5B
zdx{nze)qdiWgeYgmHl_x>_-z<H(g_QP4D31;<DQ7HoeVl&7BhM^xfM!qi?@lr^F!L
zC&R;MsZlWh*;|d1-}3fv*c#8!5n;FT<c-M9zqZX4o^-GH*s6_Zi|^+?ogr_cp3HDy
zW%tF(=NlL<tci46yZT$mm&I4+<aaSRTsU*{n$G#Uqe_{lZ<IZLo?9gU^L6Wi{8?8F
z&Ts9Qv`#VZ%KAs<M-~@7UvQ&3YU%sshc-3O7LRy-cj?+6#o>3Z*G^Q;nJptOx@;8-
z!<3b-S;?2$?mlEV(DQYRn9cshr$TKn+;zXTcbS5jl5O?zdPm-is{)0D|8n<CuMRsf
z^Yr)5u9f+%R>>EqH}qNR<*mKtwf~KBq29d73<p-oB}GrOFIIUwONOE4np^yJ-8bb%
ziyQl`U3qInXDvMS^6TkY;S3hXZu+XQl`CnL_vdOYZ*_dS<<Rc>l6sXN@2kyEezgi-
zdi2V_V+;#yw?6&ycTRrPFJDPEhpp)s=S&s1oB3$^>UV(}Qy6D#ta%%+{O8ZIb)0sB
zA5YIHtCW7kt%6u1+V^zUB<OJTs;5uaR%`ybfBR;+Z;cdtgP=vksRp4J8&-65M^prI
zCKSB?wrjt~lIS<#|DP{?A@xYTZ2b@3k1L*7YRq4BbGG5cuG-|2*L^?#>F#C`X6|XP
z?8xaYd^2rbP)p`@tJQo5wk|v&#<0cx+)73bBetK4k0(v3yt;B$@uUOX(?cZx9@!#Z
z)aRCUKg{UB%+?s;iERx23*YU3$J<-?Uel~<@!h}k-O4Au;#E;Ldgk$Y!_~jmMqg#^
zzIf06{Hf}T{nmFnX?>5D-tDnj+R!2SK>6pD=GvOjuZ^Gio=Fxzy%lihqT2+q<x%p>
zW5w3Z-+F5A+#Rv=VvX%?$V}GXR-DMVB#=4YHGN0Xskt==UPfKL{K#bIvb8G`o|@cO
zTIT$8%AW2=-)6lp=f1OiYpLnU&vLi3p5)wJ{2;&F{^ZZAt8Qv7uvTU`u;JfaV>jiO
z{FVPZ&cB!+`F-YJtE%lUZfs0GF4kwZlW$$<(oYN$$0BCbPtwS{S<l6gp={0`f4b)D
z+pWQ`EBX@u9{*wKIp@}kS4sLhcS~%w{e3MrmNGZY`!D@SDUD%I@A(<At6tn(5wPUH
zsF6^bXr*u0zO-+ZHA)NxoYxn<WBT=P!JpIGyV`zidndYIQEqCLj}XIy7avOsCMD)M
zH57iIv~KO`*u6{l&0CP#Gjms;l0N%FfsVaX?EV|E7d)4_VP?kkc$S;}wySy;t5-hM
zne|SWsleK4rF8d&xcKJ>cUWxfc*bC|<yw4c%qtGT`B&2xNLSpx8n&Qv?MXYSm9^d{
zCHNH?PP81he){YBL;aZ0&maGEiZ5EXjj7?|OU=(6k0&i<GXEPHa65X}#))a))b5^H
z{60PR@U53p4lX^rHRq+c96FxOXZxVadJ}_=Y|lQKmHHmD%wOHt7t8gV*5sG?c){;4
zTW0sKC{4d3eVVs8jj`;Bf@ji%M{d_t=V!iOq<`R9C~JclgL10vY7T}wJA#|KdnXwk
z3V8aZV9BoO%Gzx8n$Nd*iLRUTEci&tiaC?Y1MTjAJ=`G>r@ba~Nr=+?cjwi%$x0sd
zj60pZ?QHDrU*9=zzA{qPV_sI-u4DV&Zk>N@WS`sf>i;sV2kypQIaM6^tQ%=qzVqA5
zs~hC?wHQC#c_{q+xaF4O)4Mx@z-uo5J>NX_dH-Ld8sFXnA7$?bp{%=PI31`RHt)5S
zVo-I~k1zJ$_5NOQomTefy}VdlNLSUle?|vn85H(cyPD5j@h5f9o^`eNY<|biKR4I=
z*O%`<FHb*l;$myi-{V`-7Tv$hX#4IB=f$&`t2$@cm!4tTnfY4cgWF%Lou{<woNXA|
zp4;!KP5JfZ<>_g<+2-0&Nty9IhEwfkOTB5?HpxnC=l_$J?`81K4Ust~)p3C5gJ5}?
zSemHG)^9&Fk52m?)$lWCLSc7~yGK{3o0n^BHrJ;8Mt>`=@o!t%oKb21@vCrVWN=*f
zjrikU+%j<r*U#KrwL8Obag~SFwKF=!f4a>#Y)vrXdb@V_s|^{YB@5dgaXsrXS~oen
zp>0<5UOU_3Z!;NVRqx7fUbmjJa8;{2SKNwa8q0UQ&)c$M@4u>_7FxCESkE2v{E{qQ
zc#ZvZ;p*$l+!?;k?n<wC`tHrQ4_Uu{nLONcO{pPzkJ{ctSJuyoy|crw^zrNK>&4$E
z9`BLdT=@8yU-9FXNuTyUo9%e&W%^;?s1399%~PZ87bQ)-+|+gBPWK$%sh9uE`L)sS
z^Raj7=jZ9(JIp6%Q*nJ=Z2rAHf}LF!&rT;i*?nx=<)=*y2X?ICxIMY{&+osro2$>>
zoL|cxc)s#^-OJfC+p^}#Wk0cgyZnJSW1r>zM=K}JyO;dQcIk!mbN_Xw*K9X`-l6Q+
zZYw_R`dRhN;5hD^TXL>_@_6xCX4}yZ`=9$|&eNS5ax6an|LynM{HbSmnSJ`bbuxR^
z>4q!T?0bLL{_gv6U-9FmZ{HTzpZof#r}X&oEi2dx1Yhry+5ct5yQ7Ww|Jd$ZT2r+C
zrQrj`jLX}f{akST7Uye!-?Nv0Zk3s7Q~Bvwue7kR@Y*%^j1P1dUC=Soe4KGIWTK&w
z*aLy@P1+BX4qDtcc;=*GT2u1J{cAziOO=jY>(6dK{-xS5XxTi?WGzRhCNs0SAGaKI
zXPay<_&D=yN8@zQTk0?0-`{g(#tg1K*+#2U-p))HT|Qaz>4Mh4Q%|oP33|Yp>vD<x
zo!`?fkERBna_r{+{-!X=w0d^T)EU>y&dZzSKkq4R|9y4po?2~x|8VAjB}d||cb7l>
zwcT28&HDH)R!*s(UT^2(ZE>18HS{0nlAw8i!qzdgY|;wb6!fxfrBB`8u*NelmrOai
zURYG>vsUPn^N;Q=;m&=1WA@bN_ZGCCchs(qt||LeVrRA2z<x?ij$y2;-ktjYyYyMv
zCMrkSMa6R-@sUh<qxkT1*{f43DKpczi+tw38M6G@;^H_PnV6n^dD@p&aQ$P6a9i=n
z?0u-qlmn+)W(EasIi9z_-Z<#vlp;%C-%HP|^=lQMM*DT|7YMXeeJT`v_@3Hz@BH;|
zCCu{E@2-?Swxxg}VVbGg-K}-Gc@KZ}vX(H^vTvL@_3ZBU9o?yEW}CkEHBE2V{iF9}
z<&hgZ4(MLGbSa8ww&@zzvrOAH6x24(4COUum~hkj**_O8lT(h`3dx+G-^pz0TB$H!
z(W!Oug9>HA<S*+E=JW(883?9EOLqxO^3@bk(ptV`PL)t8*MYb8{IOrkbMxlkELmJr
z_WjWNzjo7oB7?6d7>jW-{_LFG9}~;P^Wfdx-Ph{kr=<Bjm3_KElxKg^_4ciA#l!y8
ze*Ixua#iWc;i*oFi;BEXY3^oQ+fZNO``|`iBh&S?f9tkRoL2GT2q(im<$6n-eM=uY
z3I9I#>vs9Hhv(u21HpI0PB1!%e_kc5>t&agC}ajCC@kdji%(zWWA{CqQq;;O;FVID
z9;YPA0=cqIGdW_T&$RMmfk6vHUX_Axg>&j~46w~;4Z0uuB6zZI<;qC$*p|fi*UXyo
zzPkQl{yMFHhIW@gQob_u;HaP@PmgT+y&))u$Hk|bKX7%{pZgDAH7(m=9+9)8sN(N~
zv_(57cTautcXk~^?c+K<r%snM(_E76)$7aNpF7%peh*v8cH<in|2Fj%UFd3gdDktd
zopsL58P=KUT!tFkLcLGkVEc9RT2lNajVLEO9%C(sOBciHWrOXm)_u5>E+5d<-PkeL
z-YIMOY{|KgbpitTvN~>fsDAphKF+!9>kRpi-sdM}H2e4Rp5FQ{P%V6W-`m~4%U)l+
zd3Bk@rTC+pKim7v6e$X4W@t*Y{#CMlu^lVxf^9#zRAcKy9|&`oS9(;eva1W8{Qa1L
zywWkhUk3{XgEoGAzJ3*->&xz-iI>mMX<fZy=?;qw|M;-I7v}n%*5<l*PGIB2Q%WC$
zeuT{nmXE4e+!r9ebdgcvYf;(Shy6V_=iYJuCnQss@_X0P$?O&P_ZEM8e}CV*+b6RR
z?_4flujFZZUX$DG{Hs8528G3|_LVP?*`o9?oPFurf9@Nr$`hxj^>scjb6LT?ahtqx
z?TVZx?b>LLlkuR_ahwFhGS91pKh!GH)MLx~_rmmQr}}%}dAx$sD|Mcvo9VMN*rmPR
zw_3GK>~h%MKPCNdWMXBqZ<uFXe7afwn%w;r>*ZsOGQUl|F28r56nuQYdX87arY<g3
zQ`c&XsNf}A*Uoghv$*tyLZ(LJ9IFPwPkMh=mxm~dDda!=)esb;<02XR=-x$%%oR60
zT9<e3HhA8$?~luxJ0;$6Hrz?iix-`7U$v(F>K-;-$p?%Kw@$23&+mTf^306yXwjUm
zUad}E_ahdwQ*47ZeyA8+-Te2Dw#lE#A`Bb0r<Ba~oTvVAm+OQ#*&BS<uqb`_xPGUx
z<@L_AB~5u-XI$RCbIxg1L*2Qorm0OPOU>R1iqFkItIp1#JNM=+BSwpVAzR$SS6*JV
zG-Zo4_kyma2X0MiZ}JaFm%Z$E=)~nM+vk5~_`gIzbX7}@fpD<n?x;0}KYpaVJ!_G>
z&dWX9+i~+Ori_Y$9lu+;&;HC!S<;kO$`W{Smb%iiTG_^}5d!~Xiq+<^CqRbeC!Vqo
ze6BHX=F@k3(@&ne8D!7(Fn86y6DeXTtxoX~Tv{>jHIAAIxvhN|JbCL*E;|N)^=poc
zZDo$%zP-VB6}OAmzbm=E6IZ8M?qRf<d*j!%CKJ{A_Y>8uRMQ12lZ6d+vpKU)v1Ikt
zOP+T<@|@$$Ew>A2+!QW7IO+XvDVwKe&B=97+)gd`R?Jp>VURlKW!L`Zg|=J2fB9?q
zuXNMBXH!lVrEWWWG%YQO;nS0pyKkknr=`wz4f*-z;NEaesm>#tvert4wa$v2Qkwao
z%hsInPVEOKULB*p&o6wfTAVy|bKZWAE{>b|=g-B-Djcl>p1IjKW~aXXzH(t|0e53j
zK=a0xs<YlEPx)!`u<y}?Ad6*=*Y<mSVt)GkVOB8vMgF@-oERQ7Z#&T(SytAtrBCOz
zMYAwyG(P#@mhU2EtJGJeDl#0uyLYbK>`Eo|i*xjnF6S)wt2=D#e~D{pQ+v2*w6Ia!
z)`aIg=I3mR`k&qTrTdcqb%*^~j+3{XH{D=<V6yqT);%kZ`gdxEtW&t+bP^df;#a82
z1c*L<?(AW9<_JUQ_S|pTobr`cwNG>J`tUVr)*g+!X1UIUlRt#}aOnC5y|YK{N?W$2
z@f^5rnwo7ckUE96aJl!(V*Tv)pC_eKckhU?pS<nKv}?!WEOvMQt19~Ia%+;r+ui3U
zS;uHvZ=AnnRbKSYXQ$V8xoBD^FSz3Le_z?(Ut4Op!xLJrt)6F)uet8M=Bz7GbM|_F
zi!c?JT6N=RkNL)w<J-k{-s0$Wy`YjYM~Zt-&R5<T+o*#29IySK{`H%?zgf(@WZq2q
z(rlZ3D`ymkH8a&E-r7E;CqqGm$;(t<%#h*NQLTMe>sBWwxqY2bvgTyd<I4v<A{lBv
zlvO5p?R%~|HR9FU+|_wrc9G^EgD=<rw-e#4`(Dm|{I~!5c^~&Zy4V@Eci*WycQPL6
zr|*BS+5G9w{Kx4zJ2@S${G8sAYJ0Td(Z`?if7IJPL>BM)B4fC7V|3}&*CFBi|DDYi
z2|0b<cCYl}>L)LQgdHB8O)h5td;et4@x<t!+aFG+`)*>>Rc&1xXOn#N;2U>I%lgOS
zd-rBA1U(Fl-gEHC@l|zw7w)U8+6UB?dKLd_N^+gDuk^vkrC-h;zRD2v=t{TVO2#Aq
zCf5a=)sxd&t9*H$<t($R-kN$H_5(MfzLdr*vzK4&xaPK1hlOFmIjPVCGd30P>hL`n
zZ#&Vf$~VhHhW)^qrqW`Os~bf-HXVI@$05dj(caqN(&xPV3`*PTtpo4aUiEhu*|lq4
zd${Q4?WgN`L+luhmu@~H^5^4&E1a{R1UHGkWY{q&%Sv(Tya%%bjq;<-H2uClopt5n
z>&x+s442pJJtED(@?SdpP3qZYsw>jhd&=%r3k}y>BimcYWx%AcS>VR3m#b7}{^E6U
z%ir-o$#j?0)LU_n%9QGE_Fb9$@t3ym8~aS}yiCR;_rvu&A2HbMpSjN4;ePG3m>(tY
zPH&ZLH*I1)u)O&4wI%QBS6K@t>pI>$wDq5s{Dr6~+eNBPMbBsGPHWm<V3PY~))md)
zm*N=(HrCnpvfpNKcst7}S<s~6Lez`Y(>6V!_Fm5z9S-qjMjz>7<9wZYdfDEGaVaKk
zx2iq%eG_@|n>+b&ipjFZ0FHk?1+QAZetPe}njylv`}j2m26q3&eWsDAO<7-}ex_#!
zAD*AZxJpXtXNR?D$)xM&UPsCcec$-lmf^#z(u31C=e<+$ySr}wjSD<H4A-ik-F#nt
zioxNHb@thUHqjmZ+;NwJc@Rr?3tUP{zHGB(WN6yHW{!FFx29CVS20ub+#>yh-?!9k
zd&<VZVRdD@ab;y@@bAXUenxM@WI8+g`u3HjoiAv6`Y+ae)92}udtHUE^?seX@lnF_
zm8<{t_VxAc+m?MWgJE`h|GZ6#n^f#qb+r`*IPa?V{oBFJu=&**V}=KpSf}PCH0<4M
zuzB6)xr+rJd~?mN{@t~jiC<ad_4{nby8nf(dF2bYZun+<^@r&v+rlV@6Jf!ZmoA;C
z$zUojw3v0n^NyhM8c>i({rt6K&5=Om>wh+C<{pf^s=xcvyOoRw&bS4Ho;|MiVb`9B
z$;)ad9dcLy*rWMcfr;U6#<DBvvz!Z~HVU4tck^;#m)|&>WtRD?o1issD|Qx3&W!O5
zF8L^UU5HU;yP3yN7KRP~D}L>C(FHBH71wL}`|xFO|Lg1*@@|@HZ@l8mVm3ZnwQte6
zC?<y4&Qo{ovV8t)(V3#~`)!O{*v}Om_<ep-{*(10zdlW!=^NL1^?u4K#wVIF>3Z6q
zrzN?a;`=VdSRuDGqBx#g!F1ki$u(CP7#?2_Rq}kaY^`T+m;;By+1$HI^Q&Wi9_}lN
z^)xru&pP--*?)=et*QUMhW(ruQEMLh{aMoD+duyt^5`-CCB1~f;f}2yE5i(_FeV1y
ztj!MBOc<^Ew5K){3NBWh>Tqabe%a=G#<wlJ7))N9Mty&^>g@4N<+l@ZX4U?!)n|w~
zvgq^qFHbxCd-fjAcK4A?IL>t7@8a7|6>B#fU0~Ly#vsKv%j{+-%Yo`wZ7)`cuFZaO
z!ZCQheT|s>{~vi*_FesK^gQQR(cwA29=YS5mhs@cW4|TsrR&rKwU^3;Z=GalSbIJB
z*Zjw?8Fy?*IrEU+^}R$~WFTKw#I4Ev77P<M*zxU(FU~z9t;-O>@z;m3!BEs}^&yqJ
zf%hBsb*|mEz2t1f%y;{veswo4Shw|^<GWKHe=V5xK5MP8-*>}l{@1jwy=!(&3Ur^i
zX!V?lMq)o%&uUkwG8okT?K!?{14Ca-u<O0Wir!09IIOHTG05EuNROMqHz(}h_mJYR
ztBg!qKdw5;<Warw<h@6Gv>&g2^XwSYhAZ(^TmHvgj^A}p?X|XD>Ftj%@3e|XF&GGc
z+rq-&c52%e(IrP$Dl;qyNY}Kp^x9Y}3qHSPlP~xB+GYEu@i0u-_|}*4Me)Hc&u1lP
z{%YP6$SiLVIAxZ#z~cV}PJP<%Ui@>Jc;K+f{2Ot{i`R?P+>$=~QiNeey!(W`em{J*
z_#JYxw`^(OF^{9hs$46qcKW(=D$BQD?_2w5=gaxuFMaKpXi@rW<8G0LXFvX2ww!Ef
ze*fRDRTu6i{%ZWpe@>=DWXt=DYnHuR8FiMqLFcpH$yJ5F(-@SL^HrZ)<{ycVFME4s
zW%T`p#bIyQ9vCP6dZzrpCof9Z^xL7hIBv)GbG@rl^Uj218*bb$(qN_XW?c+ofqCEY
zZEt>ez6<>EGh)J_h5r_?mHq0f>M+)3yS$-MpfKxL(2H2t^5g>*=Avx#E^W!z6Kdw)
zzb4V=rj`za-M7A9p~qTX*EV`^Y^Y#V%V*+pShCx~^M=^63%vzZ^Uf^hny{{RUa@!L
zwbUv7o=>D^ZafiIKe>6<?pZ6|Mz}JBU064Hi>vb4Z2?D5rOSVJ6qqD`Y|6$&#t+-&
zQcKS~3f__0Zo$wqWv$|=^z~DI-x203)9G2gp)B(3{+<;NVhn$*+Ldi?cwkGG+AYa-
z-`40GvDQD^B*maCD)LOn-|k0$&GKL0eqI-esaTPD%Ynn7G@DakZs0e2JIQksw|rWj
zY+=qXmLQYM>Tz;!UsTqfXQzFo7-DC2SDT8>d{*=QQta6R^DvHxYa+F#VHJN@6`d@6
zW#m^BIPdO9>4swU=HA5;q6sNUB|kUR{kS;CR=hTS+v#6Hv&|(84=ER&?(MkGf0XBI
zk>Gx=zjhIyPQKb@^w)Mqn?L&k%gT5AY65B{nKzj6nK4AP&)Rh7dF#ak@%_Hp>pVjp
zl9KZBrdbxfeA)Zr&!ZISpEXT;_d8c>X+28|NwJ&j)q;Hdiu|*m59K{moxUz$JEorb
zulg{v!8*-PY`ZGDjoM`+tT|UVy?k-zQLmZ_>xCEUQ!7e>xBgo5Z-1$H$m;W}!sZ9b
zo-p<Zdwz<!c6Rc0y(CEn1KyknhV$<;@0nVCtqtGgTYXu7@5T9nIjS<&SDw6XQ{8w&
za^;sV@zux9nf;ev&6KzJ>(+*ZnNE7ulZ2LchA}kET)aW<!p?O9Meh=74CgPEII24J
z%&A?LrwenM^wgi4vWBd<WzAVHn{BjaqlfVFS^NDb*6s*=W&Cx|<x@Qe&%hS<ciATe
z^<~Dnt+8Gruw&OgyR8-XnpnBTbY_@W$MtmYo9ef-^zc)e&83_0&rZFL<YNe0=i0(3
z@p$_AuPbBk-U>?#Gc(P)5iwIV?Zuv*y`j$@Z~0?xxOr<b<A)+qQ*JS#rwiJ|i+syv
zg-bH@UcEYyB@|*mH-NFBWA2U6815aiOY>Vrz$cgN<w!osCUQYp_uaNv>sL!LBxPSb
zwei^L<EL#>J?a+5{S<s1$-Ke%Y51E->pKF{;)?F<Ro&Id(ITS0(C+{0w-bJA9h|}S
z%HxZ9+Og1x-G9D??R&^|SL;~2>uc|%`Fu&`oO1V0MQ!@D_*{_9bme1<!}g`lsTYYi
zI+@|uFZ{{U`)}JL`Q09?x_+Dx&b-{e;@#2@Hz$`02djPDR`uxF7H>{T`9>d$%m$16
zUXhhs@>=9O66gGn*lV2;A5zk`V8Mc|lcO0M%!+<blzFW<W5?FH@3fdUJUnsm!Gva}
z@0s`dCgr^UhI6Ze-Scy1_8E3tmM@ZD`mDy2g`p`;Z<|r<_GiK3i$AYDm3eJVp50}O
ziGKq4vMLI<o9?>E{I2(C=t1|mpyNj};ushnztL!!#l3&(<LmpkO<z*n*Y~&S=HDf?
zHk)D>7RP-Kk>P*v(D!Ml#Mxf!Q)>@?I?u21d7JdJ70*w+Uo~?lW3tv9>yT&4hn8l)
zTABUiVc{<GFU*1*ntoC*v@Yq)b=|vZ``4};N1c1!Eg$TAr=*a0J0k6V?@BF3hSV&<
zW!XFGB0qC69JkordE|kg%gg;AYj%BGd~VX-X(4m|b1baMk~EI%?73)Pv}W>Rvuih_
z!teYD->El+Rp$9&i`UN<J=)^G>3YGd{*xu^^qCk~w`;p`H(Xcv9lEzz{bhcvYt61Q
z*MMJVl(}O0l@_jCV8y}6ppmpq-%9O`$)cIQ@20V>k7F+7@6*5c+2DBjjZ68b3>a3~
zc0XIQ=fg=6`_*%-xmt}FjHj*to82=1{uR@M>PmHzm)Ci0x81sDPbvSQ|9j{D{d#11
zA%jF?^U~D0+wLA^Fi}7JP9<Ic_ep;LRg5=oL^W1fiSeoOrZ#@_iE&f(zLwMbnDOn+
zp72R>FWVhIyo{N$?`O0D#vvqG3|V*mgdG3&s9ZgEGox!dhs^e`YpOm+UVXLsRWj?A
zTj48b{Y*`mvnoaU!mL2S(6zG|9BR|IwYG#eXTDsvpC!+J(FM~aG3R&_r=mxfcXt1(
zvU9xecuYAaLh`{Qg%|^`DA7CnvhQq+-gU}*S?udI9}U*=x^81wU|ZzJ>oC!<!^~Lm
z^qUtkroX4me&A>Kpd|kB7K^hp*T}`!B%Wh35Zjfvc;_X%i4NwECK#q3H$D3zBFf?6
zC$qD$U5}GZcfHg<vTadaq@=;)$4V>{{y5aCRPRoh*K^q;>ItXzrmsxmKXz|qkc$s}
zS=U;0RPCZ@rQqUhtFNCDHrLO|5(VuZIa>(cJ+i`B<K)A02YVIwl-)W}Q{<9(cG|9q
zo_o{^PItQBypJ|@rtwbxqvT)X*LIuZ-QSDJT9>`Kap>YPmYo?!``X=prgsW0H`}%9
z8n?H3?0GN4V85KZ3=5wBZ`;vv6zRAWYs7IW_Ua7zkK@z-7{33lpk;W<@pQ}LiCM9!
z=f8UYO8b6zx~UYygY&T`cM0WL&c0Zokkrm^5@{W`tzyBJKA!W7MXV<?aM&0fGqy}A
znV0=kkA-2{T-$>w$;axmkDs3QZR!?dR)*cLel1)x&GgILo}Q2e@AvB_-SIuR{KKxP
zr*-Zgv1gMw|6$^`4T1~>dAgD@&5Q|2hi`c@eArmH^rD)F)@<41M!RY@FwOV7G*#GV
z-)psvsXw{hr-nDYo<8;Yqpb`**;0RRglRCx%yGTE*Oej9U8D48WOaG$x+6O*UcR3!
z^-5b%a{8R9-R)mhC+;uNJwBuP(43?9I<JOGZBk>34czt5!M;7@blx#<cFBARt+h-2
z9cS)SPKh^9{j3>tEoWZPn!lzo*H=$^dDHz>K4-^_&f2A=E)6l8)na%L9BEst%E<7z
z=ck@Tuw$qo1IOAMWtN}bFm*<_ec#`y+d6Mq&t6{Z^c@>zySIq1Vx3;PGyPq>L~C8I
zRN145$Ek}`g7<t4FHO&Xd5m)tn^Hq+dWrF+t<AqIH!@7=Y<(~5sI_VN^l#s0moqRd
zIP<<XvGlsxF8AGOXKcS(MCa~5l6t@-pW)r(UnzZVmv(+XAINa$t?OEw&0pSHr`>y1
z>;J@t!SA0<iuZ)P$HC9f9o=(oyWqj1D_v919&iB9q9so<eK5&?&A+;bu1i-uf1}TQ
zhcV=%)W7FX!WjDg8r7UVX)W{N8_IT@3HSFdnynG?&uHE>Hf@2!pXKeg{`m6p@^szk
zYk6T?j$GO&k-TcoGCuRntR*w|#!bBFb*{!VEKEPvwzA#m_xJ0^Jb&b>ce<waDlhNt
z30T+jEN703eYs%z{yDFvMJD;QO1Fs_#5pazStPE!Y3&&n)9t^N%r}&N`x*LU!P_E7
z@63GnznaPVOV9T|dfsoe?X2OHxax;3NxnQ!7sx+-^XORVD(U$Bb$cuLt<%oT=#@4v
zn<cG2Wyb#c*G4CgxLNFOKVnu~uq&$iPsFcjYi{PwWVCy9?%bg@Pd%^Pdc0gD`dv2T
zv(1jjbagiAg?Mf}<*{-_wy)#2@MZGnPg&SY=Ur;LTC>x0V*1y`<#SjU|B@)*a<b67
z{;$uiNzr@i7Ai0_7&-3!F4tDseQDCp+22j3Cz%L(*1Z3p{{GzKq+7o&8tTvMoV~JT
zy2_Iv&B#qD_cIrUXie(Mxn39h`t0MV^{?ZsukY*5S?l6y?YfQ2b482T($MP5I?sP5
zJ9Rjnow@S;@6!15!Ix(i+hv`b`Tozje8b-rMSH6bW-mYd=aUW_1F!C}MS*L|w%#lJ
zd};N6?p5blE|^XDnO*U=I7~ty%J$Zj*wvpz=6zhj@%xlPSO0$-)?Y@&Vdh(J{pb30
zllk4V>KcX>pSd<`O}vx#^`8S%;uT)K(;Zjid)XL7Ep7J499qi$_wb%ThJZ<jZ$Cai
z^_$d}Uv6{j_VO%RyjZ$?dtYDQym|B9-`ksP^m<n7!pfCiyOVPM?&rV1X_?%P@ajL+
zp<U7Ozi(yBG9FOdZ*4X=TgvojWTNd^@%&vv*_S_FF5kZC_gyxh^`AR>F4nv1OqNaC
z_-ONS-)pfqY&Z59zPWSfxO&;Y?e-7OwYx7{XS?k{qQhFX=y!_`1n<84?OwgTGVk8~
zr!Rix{=Id&es$f?!!3)Id{*zz`<H#asLDw{V#9&EclRz>;1C-6^>%a;ua@SM(2Tnd
z70X{dR$gn8@m9O_W9i!L<*OZolNdgnOyBucXP-^snl($!{$6cfWvqE<n-+^Z8-qii
z^ZQGkD;IO|#+$F-%eGcl`qQVDt>P#0mO5++Q;1+#f4^p5h50YWh~~QJYSCG3It4#d
zlJe$;>pF!u2PD-bH?#fx<n!m`;w?c6TSPhXRA)xG2yJo+H&YR-wPMbQRljat#>$ZY
zxn-vBmPZ1UFN7*eRa7!EFf5ua#wcGe?P9LmbX5JZ-M^xH|BrdKcJl7N<zcbLQMKWL
zubboRZ&LHW{;zuSb^GUd@oBRAv+lRX#>B)$o@8!Vb?J`#MZ0J}k@@>;7g<05y=z&)
zY45Jns~IF@Ohf(d&ON(Bwe@IdO@e{Ns#UtnPpol2oE`9Hxq{}Yu9kAUS{4298R6fz
zO}MpT;)c1s5z}s!)Wv=MRb9j2@%P6zEd`?!leZb|Ve1b+aY8h$=3_#f=FIRe?-C`$
zlNv&4HCs(s7>vHEd!6UsXCqwyNn@+O@TN&Ve->R>^ZCiD@^D@^ojEG$C;q*9;>B=a
zfpGr3u)n&qw41;0{qJD(akAdC`L*1=U&|d=i0uli?maK3RabKF=X&ngI%~_l`^^|M
z>SSi!+xc6+Nmu=j<o)>f8!Wn{=H&MmrfaUsdZ6(`zU{B$Y~#b<N*EHhB>B7y`OWXk
zW0LT0W2&Rv`=Z+I&jp;$HptBW@vT0K>qlI}lIXNUr3@8i1`p!g3fH#p%%8M=z061H
z>Vt3e;%81}3go!$ID7LFo`XNOnK0bS2;O;-Y15J}7ORCP)EF9m_w9eq#TV?l;->QV
z%10A8uKd~h+K}Uc*vap64X;hVwsXDf^Y{ge6cm)07}$LNet)&5F@)Wv^nlbBOYgcP
zchv=~zD*2F7j{q=v1DP0{Nz_)z3*4esq?<l9Rf}qo;<3`*9-L5PHxwm)44<2XP^9p
z8hH-e)tY(2GCIFXLKa)eu9CjC;A~8elD^qQcArJE(hLh$Y*B3H5pd#A)VzG_LdjFP
z%Tt;1Dn;cR4n8wU;q`M-UE>(dCU-MrafYB1hvG@c^g@d@zs#M57=ABQZUJd4JY06+
z*pZ2<AyVI@r=F;ObmYiFqq`xQp;r$rVsboSYq!u*I_bofRS(|pXrJpI__^`9rjzgb
zXMGiaRX)zE{JHqVh8@#cKYd!Y)~e*?^>}%%`!>CM=Pymkxx7F3?|C=gHq#7cB{AP;
zFU&K_5;r}Ze7Ku)^1j5b9Co$H5I$Lp5=-W3R|SmL&tYN6o3^1ZbcNugu!?y~SDu`j
z7cZ5tf1P49UrgD<7hkNxv=;R!eEzZi+U}%YwLaaid^M+ZYr|bmWom6=NO<sHFM;>g
zjty+NJ~oUD!VT&*(~kM;3TbUk<9qAD&hXFr-&S>|WiktIH?Cal#n7;%^xT}(<L?f%
zqz6u&J%7^aSWCB(hwZ7Cq<-f1dTvRcF!SqL_U3@g2S0DAtC+XQ?a_vUvn%$+ip*RZ
zUaoIb+|Ryc-)lad7ppw>oUM3PUXz`&sX;mTzuyrLfxWqh)>U79{g=<@h0dEm%RlD>
z9JQ0B=dy>#Gu%A!T7h%N)0!~u_dB~xGm;;3ZkxNJhKYfJYnREz>gdwW1L_mm84A|l
zD%rAT^SYGM!>S@LYNv2ABwor^WO(yo)rNOIr#*zNEKX*ceOoxqqA)~0Y)xg0_Gx9W
zhn2t1IzO!Jk~;b+`1<m%f$R(n@3<?i82B45ug#yYr*mk7`}~Q|ny-J~bZ(Kw<K?F&
z?L8MJP}03l_69@C+ie1WpIg4N@=9lqsdhAb$8P&>s%rD*^wzWQ7T#I0d#cf-(uq^c
zHsxq|uB`j??{<Ifxva0HoV$-RD?B+O{pR+q?MqKHF$gT0ld|-}e?j|wQ_if<dcxc*
zanLNu+47!&%alW(zuf*{<J!A$R>F<h*F_dE1gy?}(XKDKzi0l*%7|+V;ub#&oHkX&
zlgo3{A~hB7s~g%Evbq19K9Tp;zRLwK4_5yd4_ta^!j`@bkJ{cY=<O)sUwcJ=_48A{
z)428|@BGQ*6#n#a^Ga701_4mP+Ho&nQ+F1dbIEI$>(98?FPmIj@>15)#g%pD2d@t|
zUA*6JeX%z*(?_A0n>BF0-nBK$ea)vRKbrT^__fbrm0KGfn}atu&78z_k&ofW{eA52
zJXe-xZs3%gucIyKR=dy4gkiG8jmB%!oF8+Z*uJyL`u+4SIg<&tzVrDQvb45@ajAq%
ztejM~TzD(Tr1E)2oWZqL-@7Mo|Hf7^PwA!7di!&qzcie`Uh>~beEH?-%`;V1Ro}}e
zXliX@VA%1rHHMLyq2XNMlMbz|hkBmW>?%vwO?!7*+;VBnwkD_Ox8LTRUuEIwd93@)
zA}zb-wU->9Po6(*Pvy+WV=pf3ti8Ma3hSLuS{nLKyY^}8>gls98d%(y(?6<vvTkl9
z3lEz?s_E8S+I3~gu>t?oY_<1q={lI8oxSa9&|1r)Yrf^(sqYv*yvkMF&djS)5HM{=
z0`ILEMweJ5zvjEL9s4+aM?sI>JoYDVFPu7@>BKOrd+!=GS%wEU|E{tTb^Ff8ld9&h
zGeO|O?Y;N!F}6Gm*Rr|Cz#zc&<*L~xtF3;T$8GNU+2?67Flb~L|Cc$<#Gw86Rph_Y
zHD^|?)&G3`0l)nFj*H2biy0X__7^{${#vT*;$G?Ee+koa*1D&CV7b07G`8YDpLvZ!
zwHH4dbHi!nNFOGKH^HmJT0U>(jo4g$WzUT$$xYiOnK*u|?hm~hE2(^CippD?x#{ZH
z<7YA`Y~6E8U*xJF>-k`nRuP_RjVrqay`8P=pHFaPd~fe}@z*<U8;eD@(hFXEF<G5_
z(uN_T{@J3mYkLjmzqR{bd8Y7l`egNQ?AJ{G372#~UikmZ<rS3|m)*5VxBWi9zj}G^
zji>uhZdokE@L_Wvk5N1WL)+<d6~6C(rcT<FF3#q2E9`UchhIFK85h~y+wNboa(9L<
zm)(}!oBK|0m7n!%`|tOUdtF;Iew{7efBvsI>xsI*U*6xX=3`hFT@w}|HO1CG->!Uj
z#_!<$k@J`qlt;=wetYA@T;HWE_qvy}?0oH08PoaVZ$r<nh&l^j4u+Ja4-VAIF+5QX
zZ4>-bCo6es(RW$Cy|V3RP6$1@ZsKvlE&3#HGLyl}+jCM%Ee@@?(3RPKT7M}QgTp$}
z;-#^^^7*1iXYJQ_o_THEI+>T}SO5Q}W&f!mezMAnn%?lA-?~K~mhRms`8`or`phKO
zPh}ZLw;WF8)(?Be%rM#RnAf>LW(J4Flh1sS<6)Z+7=G&69F6$u#ZnAWFE^!SIR5H>
z<?9%_==7P@?4hgAPtE`S^pUjmq?|{V{+v%|vH3j`Kf6o+M8&*C&$><XKA)8p=4E8y
z*n9Z8e^17FHim?)v$y6MGVFM_+ic4J7e_v>wfkPlaDdxlgMG$<LsPY5e;G11h{Vjz
z+@>4Mz%W5>&)uqb_SgPfOMTWbe0S_;<lLDGd-ffvy0zCyh+);ey@Im(KNF+o3a>jU
z?bFe@Q@SR{l7S)j$ioTI<;h8l*8RW!^Y7nw|EIF^4XbYMJFT`~cE{`)GiLlaQT*A2
z!D4UKtwWQ)GcM4wJ(2XzUeo09VL^WXIUn{u{>{%YC+g?^=$$o{vAbtX5LV`tV$iX!
z`}*$gj1!N;bT>1wu5n!Hyz9QGsgKZK_Bt6ci%r6^4i#_v6wLappNA}cake4;(EYL<
zHNFl{KM6nWG)?C}o5fJO$I@H&!PU8|gfpEOJ_S`=&1Pg^SS*?>A<Urg*HZuE+zHXD
zd+N9EdS@;qH#5p**|aM?HYfCzjJr-8xnuRrMJ45>-Z3AB6Mid%_6kpb%Jpx@p{L9Y
z0?B7?ykf}NIjeZ<4iN^0zpV>I7#c1FEoJQYIjiiNzwi%NVw&d6h%Zxb_i=a%7|vOq
z@<KUIvtQ@b<DWKvPe~})_y-mrld`QfV_=y1=$-uUhZm2`P-sfDFU;6|?U=A%#m}xT
zGv@aFi-f+NEiH_!RLWYsREJ^0pUuw~8!xmdy#48NNwfXG61f!~?9-yk139@<PWA36
zVVGcUaMR-Gs{1QAZ;SP;+x=j1v52T;mX>Mm_Elb8GOn(np+~LqCoN}{xmUV=R+_>8
z^OlD#7#O~+dpu2M&Q(!{y!SKezNU)q|02u4aOI2ggzJV|R5Bcv?K!8<Zsi>q8hGXU
zoTUp6J=9pGwe5zG$or$g%T0R!ub7j1gmIeM<+z2nwu^8??%K%KJ3n0Z95aK0gz4Tz
ztM7B(>z}Vs-crziYm)?n)-0Z6RVju)Mh@Q!<S+Vney{b)h>dc*BF!+bV0XIfUe6o6
z9=GG#7xFMT-1GZ$W1`=Ww(l!iuDT00ILohNIJmRvk5*fZ=zlj&28VLlSt)0CHeUEH
ze7I6d`gm{e<7=~7&IezKG2GIgC#SyKXu+p%>w^1bdv+Idtnk`dp>Q(c;ih;Ob?*84
zTDNYM8iuT`oOI}i{Xc723w72L)tr*7Z;p5bah%`gH?uHSl7Zpf-QDW9-W_Vb?(J)R
z;pN(#yZ^SX)%QHy=05fQ_uU)}i*wqy{`#R%yWsBnf*eDJmLp4M^&1vQrcSt<H)n<9
zRs~(->1#eej`5yf#grYuHF4j@t8Rx=89E*^&)lbIW9!_;bfEco{2bP;b{!LU{t>sC
zW+yD+RpM*KFk$8S%AfD==T+1s7%{w<z|7*1Y?e@xA74KC=iBf<cD51>F4Or;?z{Z^
z6kNk#@b>tocSqOk*A{RJVmKe%GD|A^$d{=C8~9qTpKrH)pO+jkM?%<xYx}Iy|22nS
z^*@(gc<t`S`Dcz*-+9t9MO@3DalwC)?Jt{ef2?3&2wrw3fvKOJ!N>HUdirz!)u%f4
z#~=5wZ7aLZI(O#p$(7gNtA<%xue<Y@nIW_MFaP-{VUKNJqCRJDU|SmW%lKZmqmh+Q
zz?%~?414l+@d$7n3VKz!@!FXUXW8v!B^cIEUS(T!bn<q-m#*`d<lgW6_?u&!m2lUk
z)&Kj1|K*)NR&Js4_JfV<ij^l{&))g<clv4DcT-En>o@<5&${6_>)<0s+5LBa`A?px
z&d~7S#EBCpm>fDcy_=xOe00J$?d{^CrnfID&UBv^&dqSj=-u0B^XF%5=N-N^*>LgA
z*=`oes=rj<@O$Zgx}<#ZxaO?05!)sj1%8_Jf62V%S00FQdNMFLJ@1~z@6?*N__|Ad
z<e?4DbF0<jpL6Y&-e@*!)76<z{&sY!I$qWH^8C}$Jd4%FUob0+OYE~aBg0Ai*KbP=
zALJa}A*metx>ksR;mg}}1#boh8~)m$Pe=WYCr#p#uHF<Kc)a2DccIrZ_pWTu7r(do
zGxM=?7RCRk$A-UKQ~F}{iNtI7wb!{GlA4m#|G2`Ef#HJN?1c;|J~exbKMKCeVQ852
zX;yT_eu>S#b~C+ZZ@U$D?b?Qk>C(IB2NesIU-|ij*L(Za7pgALrVFawoyouu%5+{t
zkzvB0UB9n=D14W#k_!&km6Eg1s=Qz*c(UHK(PciTv(?q_J(I#G+AzFW6KLGM{=?08
z|JUtXvDfwE;?K$Ie`khXUE%!thHQJ(CWZ!i$v=CRe4Cg3DI??5eY^d_3=)f%@x0dH
zXE=~>W#@xdpYHjK<+3lRg<M_m?(Ct>{P*(15Am<gE<12**Z0IMCI*i6cOIzi5#IYN
zQ?GB6nid1Y+K?C7F;7ZfJlU!t#~`rj3$Nas!^RfcziVE%{xf{8{@%*vY>zP$L&Ker
zeHjM&-jg%`7hcOt+9Joy;1&0&NvlEEievHxRtAP-=c>1oT#bR3KQAuLmSQNmzLfvK
z?|KiWfVz2nQYY+enu`;Rc9%03KdS6n!oT>OzWvp-R~wfuTA!lk^J-PZ^CcegqDR_f
z)LB^$SAJT-XVrYxg}>@wVzGshF9X8?-`$mY8$K^<^uAvDT;RZ2n|+_YOB#z+9)DV|
zt@`;d$6kM@uqPeQ>R0_#%zn_)%D`~w_ww~NzcvJZT>SdDC!^!`8Fp^F_cXCEB!>AZ
zGBE6^oiruJSgn(d;efj9*+u^G;tUQCPd_o&EsZr=<2bu(57XI&>1S6*tkW&tWcG3E
zO>w#VtHt*2iC6vn+Q#z9txebXQ%iys*rZPW$veZ+kiLCKpv{Mly8E`XJ<AIx*qe24
zKI<6P&U}+IbkWRpw_h7r6>x}6Hh6JtO>tcS1A~GOk4wJSkpp(}{=2`;eX;fh14BvW
z#XE}*y;Y3SV_-;3)y`hXbU@|Vt&^_|7*G9dcdlRvP_^oO@nxs+ld|1KVtf+>-};p|
ztobd|Cp+ON1H*zv=Wk^wZ_nA1^FQpzCufn#4WFk-+-*IWBvjPJvsv%Hh4Kd**C20&
zir*@aide)M94hYKw%^tgU4C-uwfjkq>1qr*n@rNKZgsq|zfWCxzR0xV&2q8xOmnRn
z*4UcwmX(#rZSPp-#?-KPf24Hj_N{Nza%{iL&pP;L|FUl^4V$jKh-G~E{he!$m2Bv9
z-Vz3e2WrRe2)6Qcd;b2A5m|L<Z;Bnml+^vAw&7=QJblpQ$RO}-{pp7%o-;K(nr<Gp
z`wurmw(XrS5j=e|j0}g;*^iwmh~A%ZDC2pkrUTP~XIHy-eU11rBj6Pa3qyt8?Xnj~
zt|f+*@pDJNC`zd)%oLiIoci7Jq~z+_A73w>IbQT__on?p99)~dJQ)&84u#uu{!Ka&
z_wL9{V}~|HfqQ)`Zp_hOXxL=FCv;)x`+N4Cj_M8yd+Jwzs^6Y1r==s(b?=WB7fb0m
z{#9$$AKSGjU)#29>Xu%`+qHb@eP(B`hUG2R^e*8{SZVZsp7Q=R+-ptdZhz(#nr6xA
zv#Ysz&u^Y{4PjgMe0g;>ebqb$h6(P<S!vtZ)O=@py19nj(-mN1xX{@p^RGAP;oRwq
z5?rc%qq1_I)@R;uln(SXIu~rTJMEFbk?Qhy#!TmfRgSKjxAW^8)2d5dzuzx!`0y+H
z*8)4Yd!D7b8yQyp`kTVh_P1l+tJ1$B3<<~cC%1ilpZspmtJZyY_i1lelUl{dz)<_*
zk?`>_zg0Iod^{O1Wxeb(IR7*E1p`CQSH3C-h65dSb6@|N6FIGHXY_Wv%Vxj3_u2lL
z?Ymv>^7{Rn`-=;{F?6#o`cU5Y=idHk^Z!qb7=B%-tUjL?c}2Wq@BW+8rLWCyWI2Y|
zFBf(!SNj&goX{It<zTXK<%75d@nP2whc|sc^NNe%(39V4-ZECjTh9EO#lfJkXJ7vF
zm;dK-ZO^zrVdwQ(icM>(&#hvM+>~<i-QC^oJ5@J@-ZD>GCf4qG-`Y9M?D^-XCY+JK
z=B?bs#h|Lc^MCgf=`BhP{&uUgld@uYX9+RC`je?GI%^%1$`bDV|4zQ&y%jtT$xty{
z{mh>~hovw2b~D(`jM)6g<f3nP!pD<4zON44etp5#Waqef24ADSW48xfyj}Hj%f#wG
zKMEheEBp23<-2$97GGO#e8a1E_w#8>InM_#>Nh*6E2MVq=dwSGKiK3dT{nOA<#non
ztnA$#yGu(-zI^#o^ZlLd<dbXaUq9>me`@XYucwzTw~zPR{Ay-&e7i$X#@>_Zk1f_V
zHEAVmxcX|ALtH>yP(U0jLyoHHQUCD7=hL2uO)-qhUTKyk?~~`5SSTtl!PB?uRr*og
z?FQQ=?Cj3zp3!|~Y$5gDTwDK)?mDJQ)86ypGpE;ai|G`Um+Nb21ca_lkCEi-STgUW
zfh23gZT@|?M9xoWERvJ_{%rMpv7qApw*n@HtDITA{{5}*r?gc+d+ir%^ZBFlJpI0t
z+qw;m-lfVag|j}{s6M_VU2mym<3I6==k7AK2_o-|-+$Z6t99x}$*x6hZI7;-EhrW-
zwJ7~RW$B8AU#~v@YcOrmtLv9GgqWJ_`=?tFVjA`Cm&f}=UN7EWzcZ%FDKD-b>R}3U
zpUtPb$M}AQaMr@PQw0}E%v;W3x^-30jIXczLNmTdT#EkoCw`t0-+$h#zp7G|nq9p&
zH`V)P@ff9O8E__C`fADG(AHkOe^2r8Kt-444=GPF-!i<4`yi`$B?Vl8?%h$R_28au
zL$x3y!<K#OVtv0p#aZrO60+r5YFJsAr9{H##LbP#lU4{$K2p8k&gAD0zpK-^SMW%k
zx|e=b?5j}X3UDq?&*6Tj(jp)VHu5D;`73S~28M;(U!M6^w`#us$tj{6*FRtJa#5h8
ze?KTQY!AJ6{>Xe`Q`Q{zrEYmr`R5YjtXY{>ozQRNW?(qt62A0EU6s}+>HWtgoL)$S
z&D^-PeZEgkL|hze4&U;151n$|b$hyhf1TVe5wXie_(%Japl9LkJPZs^LLbeRbh>i*
zv&cd<x90`>N}J~%%_?&~F39lb$=(XH*}M!l`D>*#d7N~^3#)Qc9cR6mlNGb5_L)jd
z))!mlE0;v}b*$1mpWpR4BqZ(buF};PzxPbA2n`82a^wg@Lr;&-Ug`f={w#R;@};s{
zU(L*E*9wmW?KON|S7ASC!BnQe-^F6vyUzUna&7h=w!7<WA4(rzU%cpSBO8ycsFd)D
zKiir9?^;$eEqr0`#l_keIpek(*mchTF*Wl0zS=2miax&$PVmIt5U;&{WSelG&cdM8
zt#eN=I4h-F{68x-!qvIo>BJ=+tIJm-7?_MKeG8VaxVCzV=a$vAf1gjDdC9&t@!sVJ
zEye46tSUb~I%=4F%;ep<`S$g8MORk5G0*>UEa!zqQsLuwds2VB5#=l0GPUkCPu7~#
zu5;agz6lBoi8}7H^u_TfHlKXI%u-fAzE4WRqU6PeV|RZxv-7v{O0zLEHZ~qRcC2^z
z=Vj^-rqyqHrQ3Y=XGxi#fXKeCT1}I8mtU$cd;Vl!Pc?_^?~;(eAES5Ji#rFz#eID!
z&DQ+!+rz0JCdFO$E#3E6<IbTc&-J^tejW(OSigL6Xm(jZUS;Ll`wyq@chU@qjkQfY
z)bioOho#=r-`&`__}<(4S8;o`ZTmL&_I6zzok#25uhBaBE^GgEx9O+;{b*tOo3DQJ
zmiW8ya;<Cjt19y+gfH}$R!d&v$=;}-);Isx%i^b|60eVMygHMCX_CLX1cO-Xp4@wP
zZq4>JNscZ{VPKf<zG=o0jpR>`Ay4c5?JM8^{1zycQ6VpRJI?4-NySm|mfG4&7u4=?
z=AI3)JbPa5G3Oq8U5EXJQyl-^Fs`b7^WONB#GFm5{&TH3$IhTI&F7KC)m@o~&ptf&
zar(Q^b4KsyR#g>l>{{iO;BcUUky)zt|F2)a%HQ9SG*0umw6*lkBc0mSx-7ix%Kt<z
zU)=Qg_O)NKTrBnStN#?7t9!ilZp8HN^|G$c;c2g5+{)#DGPU^WzFw=ZwcTH}vhqJ&
zy*exRRtz_nk&#i>m%ra%U48xP>gs3Do|&1MU0)ab`rh8^#qx_dr>dTpeefYsRfQp<
z?%lI3a;FxpXH}h<Xs15??UyCee+1uNe%b!)g~*cQu5)f|Iix;w|DQW&`&ON@-_$+7
z@1xnT2QMny?Wzl|?wrkQ?{)2NF?agE*+H+xQ~&N?{CW=iCWcj))=5PFfB$&dmVM9j
z=f61jv!-R5%Hz*JgO~Ym^7GF(&A#^S?QQ8#Q|7<rU$t;y;p1bx?CkBU!_S9L-)ma*
zEn&lx`l9(F51dUIxKI2RoccSsMMllRH`F;g<G9(s4KbT0toik&vPI`_e)KjY2iLH#
ze;TqFZvBsW`XzMcYW0If{_8RVGcRA<G)?v;!y)ZB(S5ELZhE9$WtbUvI7Pfb`}=j1
z#w*2B+AbI;=*ga&^!Ee*uZ1hQq*K-J{x-1JcVIM<3$3r1{`XrT8*j?_cLy05@<ct;
zg%^phNQ$1hHSNhF0Xc@sBb#{`ILvCFs@yCq5%BoAq-W*A|4Ul)Uw$~PuXym_T<h|@
zMT@uO-~V@QZS=Kk*DhXkU*%Q2%iqc?ecOel52aZB9heyY%jWM@-`>H=ppcdQ=Uevf
ziF)Gqt-SKGzBXOI`qW0kZi?;Q35@dVQeLP}P?;&S;nml*%xr9I2M->+b?cUI$%2K>
z?Qy$GGVkmt3=0cexNu=rRh4haf@8-OOgNYv%CD!EU+(+F#K;hPr6pXX=EM5b-_EYl
zf~Sn^FKh^(BeU&RU8YLp=U>wk|2RZm<1l3CVk(pU|3=a2*>}VD-fye=CyDMoUvp29
z;e~ltNTAD-6+(OeykD>I{&RmX<L=2v7rTp<SS@jv*%DF0(V!grxU|LT;hUSAnVFc>
z{O8sD_>kC?abf;S!CB50GJX5^|35R+n0u0P5`X_%z3Shee=qTJn<l_;s`A6XccL>V
zpS3R8_~~o7-V?LMk*63G1Z21cbnmO*5jyphxxuJ-p`a`S2W#70Jz0iL!aMf4GBBL%
zU!~05!r*d0aLqbiYcJ=J(8=5LL+8~TSYLm6*<DV<dmWrM2ajGjHCb?Ky37C8@IT_;
zKh}Pl?Y=Pbdjf<1gQuZp&qIE$Is8=drKx|&n-enH><YhLe^{_4<cHs>`?0M|b&Jj)
z+VVknZ`Idb)zAGdi!M+)b?Vf@g$sq%{m%IAktwgSS-)xH^z&JjtGrugzK@@vbl<L7
z>Sa;j`9PP%dCwjyz3+8C@my`r{Exq`H6CncKYrwhNi3sQQd-)xmoFnDA~G^FXU>~<
zZ^`kj8kgH;QC$orR&)RD`!gx?@?Ap}tqjlozjm|L@y2f3+MRykYVzrNrAXx?8g(VP
z;YRu2{LW8Uut57gYrF5ttrAbS_fL9j`d?%3|3BS^{Qhrl78`!-`yJ(ek$>G9+lhx%
zUH6+a>~gzpW-l|zY5)I!yO%FtzIE%@mzS4k^6tBO&0_l#37%sclaF6p7aRRa?cuw+
z%b)g4al5I@YWKgu<=35OcTb8=yLR=4+lfo=Tb439xQf~T``diT{7-r)1H=B3^i|9=
z_)<<WF>vJD|Ju>Lz{QS%VQ$_c_toDuc&AJ}W<OV>BI;9czK#8U;auire8OHvKNB;X
zq&1riF4nI+^rubM^>)KT&Xfv=r&nh&b=ruBRBm?IqN=D~8k@Y&FCg~gh2&4OBlwMt
zH?LT+;@aBi?uQF>#9kk2<?a^Kt@{4z+u!DAp{^G5f1UsNi1lah^|HsYdQv6xlC;Yg
zKL{5+K6gX)><8)T>EdEy%N8wCYHPlA>(;K_yVFxsBO@YG($l*icJ#jd?C&;5&iPmN
z%2lgQ{jb=lcz<7Y{o!p=T3q&mub(GRahJL8ctO{P{YCcH{QDnmb+6u9ertPQ&^McB
z7A01@a_7xkw4!!$@|z969;|O>Xi(A4`o8q<%4sTHYhreOI?^eurluAb7uWNCrsZnG
z+g0W{H!iG;-MzE;`91rLYnQJE?{wbpc4sF`*8RqHJPZu^Mp@$hDOrshqNAk>UNnnz
ztOIo<Y!ZXk-kVSv+2q9#$xvHswd6|hhm+a@v$p>83JpDw+HmQ&dr-HXB<IJe=5kKo
zFKee4ifZh=zh%{%&z=0Q?|2?g68Lt^h+)BviCgRWRzI4n@T>0X&n>&y_g<|O`Ma$o
zPwxHZnFsXMKehAx|H!{yXvHp@_l>)q3*G))SsC0fYke%GYrVK>uiNkU`~TnCn*AMg
z;hS~FrR-hXoupU3DpTWqRg-J}>nYRfGt0uh>b*Yb`TNp#e{r3N4e96SwVA*DV<Rp;
z{nMvUJBy$9Nt^3gSVY9e#^$t(Dd%}|Zojs@M2vgKomrck&jtR;S$AQ5NZS4FldU*U
zoj#pi`fAbU2%dlKwHri4BN+mwifvU}#P#6S5y$Y%x$e8pmTr~GUGu~CtkO<Re|DXi
zuQz%A)E@6?UTk=Oqj=|*%*)I6R)6P{GI?=nsrP;B?l9%==0Cn(kH5Vo)A-^+#dkA8
zzn@e$<<XTY|1adaf9(rak7a4KLGEg)9LKgVWfZV2k$jcIz>wFt;oA)6O;c(PGs~)-
z$h=V6`2Foecjn(4=a$;<yYzWpar_D+_A4cFs`>1pa(?T#-<&ifM|=6_jpqbe?g-7`
zn<lt7=tt|GsrT;uS#SHbI!U7>>6)Cb%5oob^QSy#AHV(n*luP1f2D;Fr!I`QS+O>v
zSY+Om^1mXQ=l!b-F0EVi;OYN=-}l%5``JE;M>}jy#`Se`SFe8k_i%ExpXKUR#&ae~
z=yp};iLcl=<(Z1DMP_{HpR~`<&gSLiJ>oY=KR0L7rcF;zPrrTRM$F!-P)|?KKa#T|
zI{!a1jDB}DWCBa^vzx|u%Y37>q;6bE{Z^kY)|=;Gd6~nyQnvC!)zaVX`LX5m-hNK{
zzv!<a14GNbeGfh>Z|RVBzgBm{VTR1l_m@u8XDXNd%rSVo=GuY!+~tL<`)fZY6kd#G
zP~fn9Ik)8h{{R1eKc8RkqWo%~?aA!9QvaLp?kIe`&fQH*vq-MeBK?l-%@@C?%g8U=
zx9Z0i<|xtcwWsA}UDnq%KG?j?VN+0uALoL558W3&oXWkv;C}rD;a_pjW&74VV|K8t
zKmYI(o5%tNg*z3CbUyt(adXmhoB2mx&0tbwu-P4KxtftRgE@Ih>gV(KR&{wi+FSSC
zaB7V9GcNIO=WDv&-)LiF2vF_$S(ByyGI)+){r?G(sSj8G{rx>Sw4uvqp3To&TeEZ2
zj?CY1zOp6wkD2Ai7`=eS8a}%vt1Ec++gv|<xOoy&NXpGksjsiCt^V<0;oP~guB~6<
z|Jhy7I@MWyT#QGkHhFJpQvbST#c7gE2X+-dyvM8MAf2v0_n2<IM4ii)IE{lF$_<UB
zo_<*^C*ZUC>coEiz6(jr8-D41`eapJbNW{5wA%aT&pFoXZ<Mb4ntJp5-^KP*KdXo_
zNGP|LC3#-{@twa<-hN-!)m3`2yINXW{@ggeL~ZBJ3;XN;TW`-V|FQJ&<?im_W^=Qn
zic6Xo?CX=|%|5ALeb{=I!Y$#m_j<P8n{}39;qv&<8r$=~kF4wtwPaveutIxt0Za7~
zskKj3%ReuEpAogjoGnt5$M?znzGv$`ubEM{NXhE;-*c8d_kaIQi1%}vW*0G;&EZ~O
zPQY57lb`0b9+BIsQlYwU-@eb!&o8&)>ya{jl~%H)t*z?N&zbu9Zx&klbzk?G&i!Tb
zJ++t~`$TPL@tD0;SC1ZTy=CzF`ug}?B`@b%m&?e=?1;(x>p#Kh3*%?AGobO>rxV;g
zKAem^J})Bw)65%>cmFzWe#lQ^Eqkt=1v`VszV*kKK6t$P`7)z7d#?FseXU~-UE9}Y
z`zzYY=YG+rgT31A>Lu$kx5w;!RC4co%3qhzQ%CP^Z@h5(>+y)p7`9h`etzEn|6lcT
zzqyAFAFkiA+atKN;Ns=WyNiF9y}!5jveI_-s`KJpOEykl%hI{`tgj9`L&nvKHye4+
zzYAfl{k}ZuVWad_28NPZ8||1+{j7G{SDo;){Kx~&e)bT#O8b&<=f~{#?bT;*oWkf8
z7bfpF{hg?w@%w5Uo3|?tB)Yn|+_-fsNOpPK#EBR0T9@<i@B}qov)R4rQ+n7MhZT|)
zwXY8>F1{SKa`nBQJ~LC(rB-|l2l`~K)6UP6m6Uw>=+U9)=KISpneG3T!@}_J^Jeeq
zZxf={#$_fuob-$0pS$m3kQjsC`nYY{lXidmBD8n?7T;B+GJEdrPW!Oiz*2MhA9nrO
z8`HP{oZ$3&|9{zqs<q{h^yBy0RDa9S)m_`s(P5TbSEEobT(&9i?yj%B(&oo{BtNUY
zd((V*GRxJoxm)?vu3WG1TfcvQZZo^s1Z$^N_tsBS;;8(h?0o(5{LuC%S8NR5AMG#I
z`MpoV@cXJqUJMKoSzF5P*BiXeGfJCfQgQtE>))qSJ3lGq6i<48sO;^ndwrdfCK&=(
z3+)!JT`Mar47#@aOEkOiIbZQlv*V+?T?F1gWvVrwq%xoV&(A-n{#%v5t9kzI?L6D+
zpt&y=H?#9wSzE{Nttx$XW~NhO@$6FXxcj$iJ$-k+H_1D~e>UO1hV{dq|Bp|2G_f(T
z6hG_e-&}XC_1>}4$6*W%55K0buYXcjb5nSJ=I*43;O*;wPn<P7Y2m_!!OQ(Z!^8VK
zI$m5@c=-IARhjI+PM$s;zq2Uy)|O1x^M~KbZ(E`Fd*xGQ)w8=#&GNS|n!m4fL$;mG
z#7b7v)lV+IoWJ+4z@`!ghC0ieA3DuX#5g*ho&H60aeaZx*RyA4{eHZ*YC~O){C(-i
zRYv>_4gc=u&;JzFW6yrt!Sa8;zvez}e=b)4KXvtj-`=;2xvvtw>gejKs-tt}+_`se
zA9oiX&#j#Leudu^p3a{|jr+PM#Q*(qXw7x=@4PY=25M^0rpMPUw7RICVKQs??%zK?
zKCZ2)sr&tQ`?K=>hYomze>wV+RXzUiiyN2c-{)Mo>^3V0gF@wT{@B`&Z+j(ke|OAs
zs{C+wKG%<(-|J^@Y0Uf@{zN``$veG2YooWz$jVyE_+@0gIPaf*_sL{m1Bsr=>i++J
zKA*1|$^Tx?K==Im=kK*#CuO~|blUz_eZSk`Z>HbpZobd8=@cVFM3%|jC}{yM6J0e1
zhtkds4|@+QyKS&DQC;=ol+VQ<|Bm?l`V%L=%I#LlYWbNTUQ{QG6np+iT9Ln(gZG%3
zw%O{g{cnAye&J&g3kkRRP;hG3^Sq=aAz9hK#^yS+q-LM}_U&8Tj)KICr=vHv9}C%g
zdXZINvGTLitXURyp`Ld*ch7ORGcz-L`0!y?7O(6}9X}qn!<R2#K6B>G-MfGL?f)&X
z@>?vFBKLcF+6jx3_PZb5$X!)7Nv`M>LuByYs&#*U$35JhUE8+DhBrfb*Y@B0!ooK_
zI5_E|v3$=z<3I7c<+Hym4Ty=Evv;p;^Fa<izIPWFA3y&-z(;uA-@o7QxAVz9+ULE$
z{=ymy+b|pU+^n4|Yc6WYul=^w+FEE=mI*_HP+bM9<liQKh676^Pu1R8R2w}jWA0vc
zf2m)Iy~~c4TulGKy}I0`BP>GEtv|K?{oUN_g*v`glA<9WSI96lT=Ysyo&LU>@sC4~
zViga=3XR`={O?+rYG><4Z|mCsHYD_|=iD^k3DI|bv&5{{>z(hnO`H1v%dGD1(^prA
zi;C2xHKi6VxR_D>{oUO?mBlY_FaORrYsxc|0w>)o3Xk^0ENFW?E#<zw$ubSys(X7X
z_wC!aY15{!_U%eWMn>ZLaWY0JCmNaA|NVNczv{7x*ulBVr~8-wDY8&W`fL%&P_gTK
z`1~jD_wK&SVXb)o*q?o03Z~z5&yTss#-OmrzWVUd{MuLAetX3Zs%B(y-QT*Ty7JUC
z!OM%>*LM9lzwsKAchigi)8=?<{abh7(exQJGLn-IKmN$X#PsWH_8iU!N5A*krJtMA
zEv|owC3U0w@u#-DVz&#WFF(&`JoI8Z6T^hdX*|{c1$<{G9$;grkU6c*-cfx*o0VY!
z@7vEGdyil8*mB6^{^oNtr@pFKyzsAO+fjyxPqVM~2d~Obi%c@ElmGS8VLpGhZq3ds
z)#=LDJb0sAtG;%YJbX7nD{#4ebK;uB`T0M3=Ux#$yhZ+XecGCnRr0}tQaul)Oc)*>
zU2)#Jr0%Uoi_YJ8^~lnO<NQB!d0v>^l`y+2A;$Kl)ohdVm5%ez!lp!svaWw8YohJf
zUs_yjUHfZ`Mf$@3o_TqBa&mHiKHa(DoXmB3(aJvMsrx?N%6oG9sJUO+6^GRP`S)xS
zf@XbAxxCDmoloY+jsr`UOqx8In}_Gk<>lw6=|-2myW?3pr>*)v<J0b!%H8snXK&5V
z*i`m7L6`TmD7%ZWO74u3FQ-lKNSv7^U2fEK{=X0Jy_4a|3?g%X-p^n8Wn=wWnFZG}
z*JNx`ulsR}`MNP9gI64DRVAz2F|lh_*Wb;tu8=>ma(DgTr=Kn2?YX{RUB@VVZbB2U
zakZ$rdZ&uII{W+x<M%(To8Ra7i*CMhB}7|$^`=cm_V)W99+uAa%G|h3D>6FT-OWu)
z>x7-n+ZmPbr&#d+HIB->aJ$pW*od>)UwSi>^6Fv+h8O3`jAR%RE?t|sL4x6r)4heD
z8Q!T)pFL|BGEz^>Tj<MhU_srRf2UX&Dy?5l&g+Y}=X#iR_*$Y<oY40oO@;$6v+tBI
zS}w7v{{1J<*{42y|2lutFZKFJ-rQ%C7=0KXzEqH7n6R^Ee#pg>F7vHA?|z-k^fKFe
zr8=9U_x9IEue?8DBCoppc$=w1tHh1o$^+BRG8t*D_|z@5<=tX|O<#H^r<yV{Ecnf^
z?Q}up?SnbD59YK#+;nVKNyYK~O1#~3rl;8LslTvglY;M+H68*%W*=vFcY6l}OyF^r
zs6V8qez5<QMbPur%M+F3^!62sTyb#KPktvaby)jsS$TQ<)-2Jhg;HLDfrct7OR}%8
z%f7b8vgirN)k3M3=m~!`Hc2tGM7CCLb(<Kzw=?qXC;fv3WwUSaGH`6&mu=?H#4xG6
z=HoT(*7-F%9@+KRs{GQjex3VP-lRr7zMJvD-r_&&b-(Wj{vUm-)tTWy!%J;70TTv`
zuebBVCv(qZe_Qzbwf^?|%Xr`ab<YS2tMn}Ua8Pygx8pN(tT$vFd2Y_Mze*(U*}=-P
z<DQHRe_p2Kr#@FmN>AUO^E`E0clX3aPxb|;q@+YdMLm1|Jbp((;+q>Adv~Ne{n<Nv
z^5n^9&zz0lUv)Ke%j|V4UFQ63(WyNj63MFY_uSNOt~K8?vSXf>*h%{y7M8IJtXzJr
zIJ`gowT2Lb3Iij9YvfgHrd8SXOw#xIZI_BSGae9UXy#K8aB~fr>+$=-s*k;k1y09W
z+;x#(e(LA#b0M~jN%vT8{>zfTU%E+H>wkIlg}waQGe17>&0Q}SvTncn1%nAUf7Pu$
z-P3Z)bUUlm+!@lo*=tv*3dmT6RkA;LZ7pb~a;$sX8pW*ndt|3Q`EbX1$H%Gj-+Yh`
z)IEO6_sYLl(ytz`=#urf|8uG1>AE?2IV(;)I>$d@LEz_~=MM{=^IIzXr}+7~n>TL;
z%054CTl1sf%#Drt#YJC^|Gk`ef@#%b53$o5HwkZj8Nct_zK^%WYVS%+@BY=##Bhtv
zsGfIvqt&`h4u%JBlm8!|w|`S4k9~Z6adGjg$2HsM%$c*$xqaEPWpb89M{eGn`O1PJ
z^!%M)RxCn?_vF8S`|0G~mp1*b$=h6NjBc~-{r@QYc=}44{W&@D!3=-OUfi0a_;%H$
z!_Ojh1a%8p7)+Xf9<=*>n?dBrJZZbPpG9q_$@0&?x4$&hx%|^riQEN$7TxYJZJf$*
zU`v$jzx9i?3Y5$AV?tKDzwGgqmjA!=+hIG#hGqFXlkWyyKEGVy@3FuKJD&&r?p2Kc
z{Wtg1?&QmnabHSr7T;(5_PW9BQFpwWX;D#;kFW3LOPBoSTA7x<x^nEUC9A0h!;>dZ
zdStD?Jv`hVRlB@w>s{Xq8vB?(eU;j>@XMALRb32B9_IyBuI*%3GyEei)y|hHni5mB
zSvLAYT=TN^E+;N5+%>Cas;<x4HUBm3y@awatl3fQxsO{^RFr95jHFQt$JN5RYp<`b
zKYRA<eEa%)i6@%QO`E-Bo#XbGHEY>&6%1qaHn9C|`!eJ3`E@(CW?g+16%g?L-rkjc
zpPMgTzTD0uIce(D(qCUPUoDsUm&JQ4hT~N6eUWLOwJU9w?b$SInw#)F^XV(3wk%k1
z;Le>tFE1}==aZQMx<CH?{8=*tO-)S~FIx2G?OSJO|M2P0BrNAHI(_2me-pzW7w3g)
z=Cf$#@7u}Rzh~m5KfiNRpKuw^y2=uBUs1OvI_t&#x3{+Q?<+fd;lhMh7QG9c+xZL)
z4Idxt-CbK;AL{8>-tKI2tpCi?20P`p)2@M^XP@3<I6-Bm(3WqZSL9-C_fB~5pikC1
zXzq*Snc3Ox?d|-sRwcK#WEK?_O}aEoyLR(hbDp`2FaFyt?)={{XW5dNG;c4joSd9_
zcD1`oUte3M;m~;KdUsQkQ$oUnGc%2CZETYIqrT00rg}It^#A2q9L`_571rdYY~9St
zBdKJ(@z!(S;@Qn>D-GVNUk+vbQ8FvTpk1X$NujlA`M=Y~@8`X^xA*tf?CV>$Y+2!V
zB_kv>^zh-sbLY&_i~Y1#@AO;KuR6_tmAjeDjeV>b8H#dKugw<xd6mUk!9w`#^p?n+
zr=~A1?=p69*p?Msyg4q*?Xaos!iMQDZ0(;f4FCV{FEbNUT}7$m$%V_8i;Ibw)%++>
zS6}WuU5{7RYRV>!noUd$E!VH-*cnzQRL8OUS6_JXPHM-BL*)V@%UbUHv8fp=KQm!q
zm=gTkanqtjht8dQcd(f~``VhwjY+P;vaS2-{{B+CRr>e%vdg)-xvN*NKFqYYcz#Ua
zW%J1sgBPlBI;!(*$(Xt{MU!7dG=nQg>96Yh%HqB8dUrb}?wxU~%Y<*ThMwwatyc4-
zRV)8`b5>04KezLl%@--lq7)Ag595mD-#n8~mJ}46IDfvM@8j8esWpii2NS%1*#!Ig
zhR(b;i_vwdO0#e9B=s3n4@s|YnRG$fWvA)kn0~$Eiv+Je*==NGzxtQYCKr~)GV`N<
zJ(!Xl7qnpN)KCY9hDkxjKQ)#vTxeMJ<%Mzjxr(Z)J_*A`xwp4nd7|($W?9mDMdvS5
zDt<;<Rvuk&{q%~dI(^fA^IiRUetoO@w6JMhp}cK8Z;X=;J((D5vf>wS(at-blhs^Z
zU3KI3RGgls>szjn^{ct=*?~r8ZgIVoqAS%qt2DPxzjn`Z1?!_wkt4hPwfy$mSAI$<
zFE5vr^xWBgNd(kau>b#O^4crgzVpQYycWBnuQ|ZE(7Wi7|E=k1>gVR!Mr%Dw_!W2Y
z;>Bmrp6Q5bYiO(pOyhIcm7IURyr_tahi8sy_O*+P-DiJ^5DIK&I&idGT-|@(8TFTL
ze#@`&`mF404*09Z9e;f8hSewhWB1kkytXzv`p|^A!OQ*5&NA&zIQ4x_mEY3G39B9}
zgzVpR=lMtVH>>ZZ{`q=6KKt65g;svSoSdAJ=6Q2AZ7TBcIB|acf{rMig#i}D&wTXt
z*PooMKHsvq%`N-ZvIx(Wj*Jhkt&M)XW=&VO_pTdnGt=KVmCCg2TdMN<BJ+37tErW{
z%ijL_`~CjPK4rUu2M?Bg`xY7#_ubPyx?J8LtnkIvx9!W-xp%6%_sh+-tF^jXIP2O{
z@9AnjGcN2ZeZ67B20mG<09kg=rLElJ^K2?F-MV$_`t|7tGd?Lg3xzZ@9oU?H{>+)P
z#+NRa`h>llu_I8Hec>+AB`x+=##7zG=35r0Jv!1Ea$Ioxj~_n_laD`p`t)UV?mL4F
zy{m-+tD^HBO>N;ejx;KXm!E%ZZbWp_T-)lhQ&Tke|M}#dk&!XmEO$j;^OBy~vv2RK
z{q5u9!^FfCvGtiizSX4AW+sQSvTrjClZAwYZhf1bW>Kg9G`L;q%gi-DxIOoAi#)Lm
z*Nc}kdvkiLUv$fK{rFikXD?s2<d4xC^Sv9Tqn+$oRAMjX%h#4ww(i<hb>ZsjW5<q#
z92cCg?A~`~rm=f`{QgM3i7vj+-|zqb@B96Fc3vqFDJd;=^<^)ORM)YwHcXl%R9*dh
z!-frEvB$m@Sb3(O3YyEYP%3kxb#SwJ#@1YEA(=Z{v%^<C4p_Nh!GTMcKE1oUdzPE<
zZeN>*;C7{x6N_^Ho_@W%q@<*&xw*8YWJRBHo^{!qh<!CXCr%6u2ng6blaax6<=x%o
z|9?K8zqjgZS5J?~^K3Vv>;Q%bA1f{|_qVV9wr0|{Lj7=mNxr%n&LYaKhtjA2Ke3~D
zOZ4`<>+50{=kSSm21Q1`eDR{<=cm-rxeslZ&q>@cyWp)ri<Fbb1KW?WGi2)*=HK7f
z*x0x>D$x9`$<k}<<M$^XZu|H5cl^$x*299ApS<$$0&UFxe!t$_#H6RID=0Wv`1!(>
zbL78By9Bp~g@w7rd_CvO6>{A?{gh^~EZ5c;rnsw%KVRM?A0uH`Q<0I8(OsMTyC>t-
zh3)zG@9}&+TQ9aKG2>*yNwC_QyqkwFPcEL%5)l=}#>S?ltsOkK!${e!M?+it^S5vN
z>i_?1;S@e%u!F^OW%2WKXXo3;ySqRC_4T!slvI-4u9j;H7#svzR<2w*Yt}8%{h#yY
zf37N%Iu>+giIqrJXN73%NA<Vd!Ak1Bv!1+u-QCNz^<u`B!pFx9C3>>MZSLEB3x91R
z^*&0$tHAsIG5O7v-+#SZ6S?`^JX`H~f})eooH?VTu73Rbb#q0<gXhkzTd982@zRO=
z_v?@KNao(&wzu%H+wQWrN8}xx-XFdG?Dgx{Z{FxANZQWvU;TP^fGqn~ovE!aKdQey
z^*LyoPUNQK<9sqTpH0>M=Yev0rq0oM!RJ^1F`DqoVnwRvY2UhnOZzry)ZX1yDs+2;
zQk~Pnf>&2ork|fT*C6rG$;s*)d0e-)wzODSSgcsF!o}S^*mU_Ckt+)r1Y~7%Q*&d3
zqS7+9ug{%-GUT|UuItI!$NBwk6?@-*cVlDn%S%f`j!!JPzAkq2=IzhJxT`GWuD?ig
zEtR<vw5PiE`$J9lx~sbnxAUK$W!nA9Vp8k*=hhY$F?%X5UcFlS|KHyqno^y2_EcUz
zcFZj-?Ax(k>ASnjmut0^Tv@;{Vak-0w7hv!XRgfNw(RnXry<83&&E4uJy~oV%;7k-
zz5VzS&&ezLlx35WlaEJ5JbC_HIwJq7oW<-{7At<9t`++6;}DNp^M{Y$wr$z6#Hvqa
z)w*@twr%6%<C|w+|8HCF?G=5^E5yac(-RXJ83Y9dYkz-RYS4KFWbf3ere$w#9PgLk
z{@Q+X_t`S0eC-8llmA|Lz43ZXW#65tr>DF3NH~_tWKNnkEiEnYpOWd;#~+UFO1_r7
zn>))&vfSTh_ZFW}->+E-3H$5*uIN+Vsim!b_RN{QdwYH!m#<$Fy}j?1MTb{sXJ_)U
zo|S9X@bK|1tLe(l)OvhtTW<B|XJ-YK-BRw^{`=6=-4O0ss&vI>@1A|XKeonO7Hyhq
zRhH8@nfu=ozqwX-`#N9V-v0iy^lst3&p23ZRtYbBrm6a4{c`c?e=dETZoNG}nekWd
zrcIl;#dI!Qy?S?7>FYjO>lJ;?8j~kYN=i))4GSy#_{jAXcff>oQX&i@qN2^s&9btx
zpC>Ur?u-1a#kJC}WtQkgl?9sDuSrTt?Xs{|S5f)$?yj|k#f(=mLCZo{hh1MEzkSP=
zGj$In>x-1;M|+hjU0K7y`set)&v9<=UR+*&e(vq<rKP1S{Zw9ldwcu&n>S~k7V+}(
zUSAhm{o_O8Dq%&gCCisbN5MK!XJ#0N9A})cOG<<xB_-wDT<h&QH#e29e}C6HBP?=;
zU#Zd+AC8}Q+@CL7dR#`%s^rDm=<R&6R!^QkKmPcmj*iZ?wbAM#Tr2rBe4jjd^5^g0
z`oCYVAG&hlhI8`HrTHMwaN2Eub5OJW&5q*a<9&MzAGiHTH<_ZR;;9$E@6Y!8b(gPR
z?d|E|;pe}8^=j;{l9w}$(^vGh9nn%(KYr<wk+$~fyLa!NJJ<Kh!gR?>`xoh;A@r1!
zLerdH$6c-wx&3W{72l-1J*O76FQ3~Lcg$yw#l^#iA0K|wq#w6uhC$+?!|nXOzP{(@
zTDQN}uV2qAZMG(Ue_WXB?&~G-mAqNY{yUs?E4_L9hp@2sJ)2o}wZA4RyBECrn*X?H
zXYewgkKew@nPgmedV0F6_k$(Nm-9;+sc37z{_^tjwr$%&(p??}1Oz1H<^B8e@-j0!
z-<jFw{jV(KJqvlc7~<mMf|vOS3K_ZS$z@e7232>Vp@C_CJa=!p^SoP3_tw_z>*<?c
zgocLd#_ag;<KyGEx3+%%^hwG>Kz;uoyW{=x^NrK_<}ST*`D;^A`@fLmj+a*1=$rmq
zQk(j4wQ2S>P-z1y09spH&CJZq^Y6_uNNj3vXP<rc--pFlu3S-cZW9p^si>;*I$;}m
zrNqn2OGigX&Zc5R>FaBv+F=2*&xJ3Q>+dsTXK$Y{LBP_?_FnCy+3eyVX9i`KJ=)u>
zmb&x&_Po2l?$`gXwd^&Xeb%n*&5e_j)w#L2+W6)7Re#UZO7hw`(>OhDU(HOu(8A+u
zis!Qgf!Y)0FTcN+H+sC!+rH+9f!KntUf$l)76lK^&NiQ}7yIhz)6m@vtPHWScQ<S>
zaCLQ6S6{wjg~lo;CBqdfR&3b3x%$I{gR@MtRn{`FX6Cbtdiwa7$QwWN4VyOa=dV8H
z>7W+N<U``-a+Nz87JRz0GI+B3_04}AotzG}a0&+n1*!SWDERZE(ACxTV@r<QyZig?
zuN_*nVZnt1Y1O|OjaT+HFY&y0Ih*tOGDFD+PhV!Hww{*1^7!7~>aeh|pFe-@E`R^;
z?e_cowngUTyjc;rxWy?kAwgm5WS2#1hn^Ox`^*p!6x^6{a?)Ju@*vsed}kN3u{NAJ
zbH*_Fm`&xUC4UdZ^RC(U=*q$sibZliRjSTTot*qTewRsBNb|3vDO0ALnPYkR_;GiC
z|MSO>FL&pE)A2_(C^#52V6kjzsm#WZd>+4`4k?w#^L}sc_6`XNWnyMx<B@PEmGLb9
z`RVD&$?B6;D&O7N`SJ1b`s&BC&GTo?oVhV#&4L9AduEv$*Vx@Zc~aBB;KTL!dRy>7
z&Z}!aSzF8A-nzT1H2cPeM9J;_c2?G*2Uah#5}Z`Be$h9TIrCR8Tv%spwX2a;gF!<}
ztE#&C{jII9udWVfX5+CaeHAjfEhnv{wDkR*eYrU~e?I*>cRa4RmC5dE=Y%ERyf?pG
zd0+mP@7DI@qzw#fukWk<os*kuDdU%tlCqaogMmlV=*gv}-lC$S*REYVdGe&Myg+AX
zfW{ot?68D{1(PRVUg+GuqVIF)*(I_eEvKLQ&9%CE`n31Y>5O}0^bJx&SNOH42zpLB
z&-`zC-RlL7jgAEc8@6rx#{cHYx3{<X`1oedoN1hX?#}l7`{{SrUNE0BeLARq+gla7
z_Opg^IzOBDyjLzBLY%CBmUQpm-)Vb!b@=)P3l>~k8y!5?W6?yFok>T#WUb3yynM;Z
z%36E<k<@g(*jWaNPOq=6J$&epRmqDBC5jVo=)~;!@Z{uVGcz+gJG)6XwJw`iIWjh=
zoCGZnytjAu<yFP=CF^pgehX1|n&f)ox!t!ppTB)Pda#*2erJ)Z>(k@*E1o=kYH4lV
z-qu$3{@&fomp6M)*Xy0Jx$^PnpT*D4SQb8NSu`#1z2rTkPZ4no{5e*gmY-XCwq7nP
zFYn#s$IANs@*(?{E?RWx>eZ{qk1ziw<8%M%o12@1m-)QBu+TXoB4V~#?kpu0A!&2-
z?X|zZ&73tWZfDWc!#2&6I``H6ty*PlqSmBvtVeS3f&~H<FKf;v+ne4M)M2fub~de;
z`d)5_^%Hykjhi=r{{B5YAmGQ$y@eJs>FMcg%?}?xE`D^R^V-_z@9*yJ-nhst-$d%{
zv14u;B5iC{duwYVWABHwE%qwaP|;j<`tzgye*LP-(^s#4{qytlrcIkd)F)*~N=l}s
zr3nfO3ak6=DS3IRzjx}Ji;LT*O`F!<?p{!^Vej6(bLPY>e(v&UW8vdtbFIty<ZLVq
z3<5r#pZn|@->-wM+~Ujp=6YSa>86){rNlizmOXI&tJrJJlfNy^pRX6QW5YSi8FdjQ
zC0jOcE_QP}w#0LCpNwVG<EPT^PE1t({OoM<iB(@pzJ^>oHgPA%sn78?7K!13KeDc_
z3Y~41yU40<$?n?U+t#kl&C7cS8j4sK>n(8P{F~h8=jNWCrfY0u#KpxG6cm)TW`ajf
z+1p!NtG>Rvx3}8e&CTnmb>QZ;j*JRMMwin5{P^_g)H-4QZ7(e+g4z{JE&b)@%n5)m
zl<3{}|IndBpaODN$;#m6e$UU(e}8>_{9^e<J9ia5?K-@~)5h-0jG9eTA4aTtte_>D
ze#Ew*{6VP5oBR9Y7ljp1^jYpdf1Yi%*<3&S(pOhnIEAHB=PznK`|Im#aRGq?4-2GH
z=X==n$=U9z_?YDEe0ar*74k3o9Xp?u>F?XtCu`j&ZT{}j(r*2{`AZF-gw!20oT?j=
z81<!Dxa`oX^5uSWXW7@+1<N>vg<ZRNF)%jv?)B^6e|&tLV^)1(fusFRwTn(KUca6_
zYsQaXRr_~Uhq*1>8ninuMBVAiiE6P=52i3P@3?pGUhp!Xg;sqo!d$KOe?A_Uum7`f
z=~7WCDXXe4FOJ>%+##rZ>eQ*4ni^0koOp=EQ|??-mUa2NJ58+IB_$>6;`gsxxX`h5
zPFvQww+oacB_$W725n_OdsY6szMt%^OHWjOs8*ewn0h*9SH;C6N0zMQ|Ni2lvb+27
z^z-v(&YnHrt~RP`9qSb@PftT*^ZKu!QkMs6mz)g?mSyjZe)jjWzrJ2rP!JCfhlB}e
z`d1~&)3Y<<-tk6e_P9M28#U8-W|-&STj<<=tWUQ3-=9jg$v0p8cJDoX|Ni}>M_s+W
zUq3rL`;zHq$It5>85gWrapKmkU(e3YUc0;hz1>TWUd4-tj|3=J)fWH%+z`6f)O0;7
z>(@6ogKgM#_4VU-6fE4md-t+s%i7!9CAjWCd-(SDb}=!rj~_oq9n)*u_W$CG8w{6&
z_Hg|D?)oF`EuU=D`dDKFg9TQ7F4Z3&wQk*78WeO1v^q-0@{-Hqd7hKzSe3q-X`KG%
z?c2L|?@Dj{Zj)7V@ZiD8YQ8~%fjcWdFPk?{?rPya|L%&!l`B@Xw6^-rFnIX(v3KF|
z<Da$e2hQa%wFJ#>uc?yTu&Vg^x$?aGV$qCT3^IJrpPZZwx+z0n-`w2%|NZ~}jz52#
z^5$zcWMclz*|YbL%&V-p6&Mf;Y5_j+Soomx{UY&)vwM4b-rd>P%lcxwtDD=m_xJzb
z+?*c0J<qoERmf%E-Od6nyLaziyLRpR_3xWlxucw-r%vhW>nkfNnlybnKR5T}T}&cT
z9@Z;50yJXwRD8U*xB96mPu2Mk$=chOS_x)pIn{klUcWP6xQ3lirsC--(N$IH%a<*)
zva&jM^eC^4#fLjPi}(Ne#I0($gnRyZ&<fv`!OKN%e|uwlH`-0k+Snwz=~b8e^JP!%
zxy=j>KYsrF`{d3`v-UQ#^Xo-z@dyk1_UGs4Y15`9StON~Zr!@Iw4mU_p32YL^6%@l
zDKC02BrUC7rRw71^5w1e)X!Ze&ka52D{>^gxv{aRs3`oe{vO{C!B=Gu1|8VGJ*^`?
zROY>%=Kq^pKR!MlzprNIp`V%|p`putXNLs_K79Q6@w<2HR<F*^$nXf2$aGwM@mRn7
z`RAWOV|&8uSG=5k^2Ea{3(p3WR@VLDyuD~khPb%6zP|p+`6nhg)!5ZPJk%;}mIJzV
zsG{P>gM-aSi({<j=H=(hi;6yd{rdFdk8V8@it`>XYODD3<KvFP$E#MaZf<H?vYoq>
zdzGUr8^7F|<;$NxuhskAI6bI3IWjc!)!yz`M|JCj%dhORFwVQTr*gU9+({EBE;Qqx
z>t}9m{{Hs%_fxgQ@7%pxTvX)NFV|c9=e23p6%AkCv#0Bx7t8%xQ6hCH?9w7D!7Lw9
zR{iAu9_`bfMa9Lp_t)Fc^;>Q=@rjm>&X#T4zJ2>92Of9qnBhLhVCET4VYQaFwqHMf
zY>3c#xwdrH%qdf*Y)n2rZQ8WBeKk7;(_IDji<Ri_+qPkYLA3SmuC7&ieC?*aZ&vm-
zOS#{?y&N=t*0tFz_tuuIt6Hlb*E~8q+gwjiZ{oy>cXyS3{_^Dr+r_pI-@lialsw_g
zv{l=^{Z*1*smv9NE@OF{g83)3&)?sie*RD^_sTxyi}UPivvPCi&YERa{_f7+>hD3O
z!9j_M4?jFSEG#Ts{^o{Z`MVe|qXqnzE?@Tb@!7O_^WMFC-@bXXqVIEX)R7rW^k>eP
zai^y*@2Z#X!#_RY7k{tlYnF1ncslw!fA7PoPj~Fz?H$F)#Sj%0m6(`V|Nrmz%gg<H
zdwct3ES*Y~G!v4Og#`r{YMigDxMleHU*@XE3*09Bw=sFwp?XmJ<HwH&nZDd^YHDKR
zk@)cH>gw9x-(+nnCLCI66&f0P_|Tz}($eX=(a#<}bPVNTVYqqorkI#mWMt&KckeF6
zH8^$WvWr&!{k3)H&Yz#3pSR|jcfazEy~W}kU#fopxRiW8{-(~d`Rmv3-05OI<L<7~
z(<e`A+S=Ydc<|wgiOM?^;`Yw7ug^=%oA>Y6)^jhWS}ps(*oyC0VdTd2$$#pCLqa%q
z6s4Y+pcry|V#e+5`Q^pM!C_&bNH)z9xmw7;U}R*}#v?gt=FG}hS5~G4{PkG;=g%Kd
zbGY``mj?%%&n|Y{<$8SqLqJ4CMrx|4u<+$w?Xl(UhfUay-eteYUm;uk;^)+<YooUA
z5)-+yyF7o@<16b|tjKtFW~R7)+?yL4AOHOP{P*|w{-sKu(<e_hE_-vMonJod>)cyc
z%gh}QziYnq-1Wyb@rAQ*ZOxY#lDV_gTYS}Hk9DtJy|OHRwqy71@|TyMetLR(*REY5
z>c2EhO-oNqP-JCg71xWAuoeryCc663qa<$b?sk6pyxZGye|~xzay(N;_e<O#_EoD^
zJ$(3Z+qP}_!hat5-T1V+Yo%YyK4#BJC&R<-_rF=TdUf^pbFwQn92gy3U5}nWpa1{g
z-<vmYI=AuM+?49g{a+(O-`Dr-iWM5c!I#ay``cUCMh3(>mMVRb1r?+BABiuVUG?Qf
z;rDa0Iu834m6elUTv+(--8)eCxcGUWi|7J|1*=y}%gL=<vEs#>H#Lus@veHjML|?l
zw5+U*U*0ZecUkUcO$Wx?j;<|E4;Q(1e|vv_{e@c_zVpP3@g17V=ESIfvAXV@^ZI{(
zf3vf&tXRD|c<zhcswyfzvrJyz+?<}4mbTn~eqNX9_gTVn{pQ8beE9kKmwww@?fZGs
z!!3)f1Xs;(2%lMyS9S05g22UYv3n{e9@=?HCEmrQW%1(VxVU+<XWzE}|3`V_YT=_t
zkM_z~25E?}v9Nr3eZBvc#aa33r?+n3-rv&`vnl1|xw+OW`!3H=`J$?qW+Ih)e_!qY
zf4}oz-d?hs-N&nV(h9$pR|nT?b=q4hK3_O}>(<hglqDu&3m7gecJJS@V~3=1T1!id
zgrsC^TU*dvk41(v&s<*azdmN?CJW_%_ZT;KHBApucl!DuROE=<)>G9|Po6vx5fN!&
zePNz+Z_mv$XL?RQ{q^%_?!7&cN!$SppplWQtHWEJ9{%|FSl+hErF2eO-}&d=&q98F
zy&gY1l9B5M-!Dmi{(ed0Gy$K=#hU4%)``<AIgjd|?6gk|pY=KA+nby0d@>nqley>F
z*VnzfvvcXvrP^U@ZfwaEZe>nbdg$NZ-}S%Wnyb0*oxXZ{Q1#0f4Hub0MfP;rXNFJw
zpLDcKG<sXkLC5*=!9hVrxwp1-i|b2CN&Wiy^X2Q;!E;wQIypJ5S+hn&RMgqYX^HCQ
zD}r3DmzVp0|M;=6sOZy&4-=mFtJ$+<UpU{|)^_XG&CsyW@^XtyyV`ZH7G8LMs%ZZ4
z`5V?(P5*d3zJ6`w=C)TBy=uC;XD?p-`0wxUzrVkO2A2E!_^x)U+(OK2oLheNtMAIb
zX1nl7Utcu-Zh5t7&)wbS`cYdtURg}i*3>+C?%cbdpP%#b@u~aGc~D>xqW+8T-@kv~
zzI~IDl1fZW+#y!DSlP4a;i1-d@7_H=-v9mEx4db&!e`em`&YSa*|Lj)fv>W(`p@~-
z*S=P`T6jUdEAH;T`V(`)e?2?WDSUP%18aldTq{ry_2;Lj-{0LeHZroR`C)LiQ)SlL
zwYe1)JDA$PKDe@E&5g|=#~tUN@aOBdO>!){Us+X^#Q2MM-n@B#e|@d~_~>Y#thJ4;
z?b6i@tPIl9voA0AfBxi2L{wB-dU~+z@|}gn#l_|2=BB253m!Uc)&b?4{m-th4uAIS
z*~yb9_w&TANdIVl&vg>Z&dzWly)!8%FVxo7%=)-RK~PZe@-pAkCr)hGvc;zC&5UKs
z)UI}_T*sUmVPr@-I@kYG^a_RZJBxmP+K_m7MW6D*A2t8FM72R(zg?x-`uh4S|GCQA
zR+W5vbMx1)Uk48!TpRV!<KQt+eC6KUl=|VpK}AKyqN1XZ<C$6!&2^!Ujt6%XK7My+
z=i*G0jkk|5Ta;>Sy_7FsS@x5^wDjwO)#1Ipy({}JZ|Ump7T1fZcy>mzwDhZUJKxRQ
zkBc_NPn|p0Hto!emX?-nf0s%3-ERzT57O36s?IubY}?ygtns_Afu}J%R&39|Z)IWO
z;_AvPWpd*7?PzcBSLsPNw`O0zwA8!$%L_pfktJ)^_>{`5++O_roN4y8FYoSJ8yRh?
z?0<5&`eXdkWy{0_1rw8#Cx1R+{qG4&T`1?(&M1vZD#1<HY(K{@yu9-AUF-5$wTxU0
zUS3{ua&q_X-Ba=On_-an>dH!0)gPZ$M}s<Lix-2YsY>5Japq5FTtDfRi>`>Kh~3^T
zI;JVf+w<<eySKBM^~Lo&yGmD=zrPn47`XrMx7%M|U!U#B$i?72P3Pu@L}w?brnPHx
zBO@iR7B0CxQQ7_4n#kmg3<)8jOMAEj4wPM};$&%R<rbH-D)DgKoc&&MpUzRU5cRCc
z7N&c`^I325+x-6aR$4;h#-+S3De39w=U5tRXm~_N-@bdd_Q{EfA?lL?Y~=EPeR-LF
zeqQJ$vv*HlX&#<<YUi#c(;LF47Sy@L1wHui@bKTie+yY}t-rZ7d;09z(vwfx6g}zK
zyxI6_q14STQSEK{_w8za6xiF_o6Ytum01Z&3O6^Wzkc!J&YsH6swGDjt&iV7?-h?w
zuiN5YY4g65DRH~s{QUM&)pYMgqfbA+sj1cEy=)Gi{CN_Sj;<~<GjsF<T{%`(*7Wpr
zUtiz)`u~rQ_lt{*_e&Tq+F9oJU$VsN?zL;%7A;azI-zc^84!Daj^=Wo(iPL1!UHAh
zFYPlhExNSCv#zc#$b5sFO;=Z!jI8YUH#ZkAUMwsml$3RgN4?m<*tq)lH&A2#@$vrL
z+}zm**Hk8zy}Yz^rg8eV+}mbaTB~wzZwomt$ezQVy8<*1I>&Od+TshvuPUeZHLvKq
z{HM_SzMVy4{G`7X*VaU~^T}pCoWA+Wl`ERT%kJzfUY>n@UHZ8>FK=#EzuKvCWu9%d
zn0}nj;moVqv$iMgo)>aF^Pd*?<;VWFraoV)6T7SA_qVqeHJkmm=ie_YDe>|5FE1*x
zs`;^D&mNnrowK++C#_qzPEJm)tgP(ayLY;OcKwZQYHpU5k-2i^%A&=Kzkd0$qEA`v
zhWwU=F|o1COiVLO;_iI>m@lj!EW3QB_G#aJ<p-9i{Qmgs>+60w+b(gV*x0+X&GYl_
z>@c*n+!?>W?*E_9{SFEXtfr_$ZONFpeY^RGuH^oRC+h8B{U^P4lwVk4qo4Zx++6Fj
zHwT!$c&m7V8WuM;Bvw{dy7$RgR)5Pe)^lKNQ0SPf?tkm%&7VJi>PBo}klrb!VP$pe
z<VjCQM@1c-HBUtrG`wAKUt3QvZ%wOfT->}c`Sfj0pWm-`<YZj`%J+^W<Mt;X9v<GD
zc6O0fUrJ|xe|ceH<*zR<7dSTi&9z#(dbM|{lBWo44r$rdbgu5jZl!acb;ig3Q@hse
z7B=6mw(8On&yeF2U4H)jd3l*HHxG}Afx(AQpI+VDYkjrz$dX;VZXG-J?AF$7H8r)`
zUtbnl^)1oX*0wHu#4^_pv~c<McKxe`N4j0^zdYJ4zDxIIGMDO;Cr_A|m{$A?_Fi%7
z)TyK0;=(M7>gwCm&&w?g@F-RC{PN`F<g&N7ga10^JrkKaMM+tCB}nau4+R%zeAf(K
zHf8G6Kw0-YT3T9d{PKQZwL`K_oIH7VM`80TdnwipvR~Xo!o&Ahd}OLG^>z35Ud`I)
z@JsLd`uO>V$;VDl*LUxcD13CJ6TBkD<55Ux=-!&2K~d)}-`<{oewJzW*;%H2eSIP6
znJdcP-ZK5YGr5jo{`vULX=kskjeh#%$+!3S?Mq*UtP&1vRyq0PNy;13D+0RTbosLb
zT9$-HM&A7N^t7w93@E{uzl&KVyii_TOe`%ctLnjl#>&sn0_U>+-LYlMmXnj!@9!=@
zf8)lFKR-X;h-O=uf56a_!&AGm@~3k<Uu#RtiWMu)%r-xN=8TTAa`USIj;%52=jXk>
zy88Ox-`~U6L<kBCH?wLu-ETA1aJqE%$`uns!@^@dk_#6uyt}K^y6%s~)lPw|!c(VC
zE%TfE>d_-5b#-wWnLVYi!_+2wmTI)9$X;8(;c4sYdi3$}es(stQ>Ra#pJjUb(xp#l
zXPbw#JFQsS(cAm>>gwt3?d`3tu1-!)o9EQaEnL=Ms^N6DrM30wWOe^#J~NBHzPh?D
zc6U`pMZjE+g=u{%C)Y-A=aVwgu(hqdzAl!Lkx@6-HnF9S?b-qkQ~6V;PA&JJzpvz_
zQ1{VUR;8<YdwI<|l&-iG6&0O1d-m>~J0&G0@9yo@o@LnkDuBaNc#WfARvu^>kWJ+y
z@C^1-z1UqcHWdQP_dBd8oNHUXE&KYqg$oxNr=RQT?mqmqSP*n}t<n{ql`B`ywXgs8
z>FMdNuCC9|&&yktytusFe<fdQK=Fjhla1^CREUd<Tg`p<_;K-@8-|vamaCi<Ri<uW
zb6s(B)22;cUS3f<3P58q)!*Lito!@x)vGL!f}<{H=l}cnkB^Vf)6?_YH_%wFM^_?m
zS3vQy1q(KmzrPo~J+JiLoyx;)ytdWf_EdgWySh+dlAZi4dr6+Q{{H^u%a`xkWd#z;
zi)dE5@`qQ#K*7$g?(wnS%*@QgZM?Is%kv5g4M7T|<mBYq+S>m8`!`w5x9ZaqPP2NY
zD;gjNx+WOSTphmN&%;B)Ea%3<!|lAhynp}vS;5yjVae%p=eBLzHqWY5Yx2o2A3yFa
zc*q3O7Xi}8`tr`s;*^vp7Z<y0YiYfC`!+W>xA2o%%f4fW4=>KXzAk)SOkqjMk$dww
zOszlxAIZ@)!!-Na<>mhBesd~bUQ(U2hjXERZN|ST6DNNB{Mowx-=0~sq^>R$$jSp5
zuX^dql`o$@9XdH%;Mb*<!OLCF&Y!IAU-#)r=i0TpptLv-r0VIfoKMS2UtfFp#CL@%
zXdZP(;p1h#v(>Wx*_6MN0lBpVq|9}-ZRO?o`oF0uDJ_%D7f$oM@<&oa;>D9EQ>IL5
zIXj=j)O3xbpp(3Me$~%Us($xcqGqjGqZ9a9efHUXB`+`C+?*a78VZW+&+VofP8<yC
z`A)i5Pfk|fUHV!~LgK}nH-Da-6qc5jUL~w}B`ATnOW+p=NQrw`*tFKxqdk(wH*&V^
z-~WH6ar%az!it55q(l_IaL(Doxs^rTf8Lv0TeAZKCUkdu2L(;qy44iqx55o<t{hh%
zZaz7ijZa2GT-?1>qeaRwEp6F~pTT~9?>;>}Ehi_ZrKJVRSgK&-LKmH!Ev6qQBQ1S<
zef<8PpP!41iU!JZEtFRg;Q06Zy}h&Z;ROp8Xa+B{sr+>0)Tu6T0T9uw)Uwaj(J}G*
zy4a5&KQ^=RT2*{luy5bLlatjyUbC5GRsR0o+uPgUKRVhS8XC&X#N;>MuJ`PGjzI40
zfR=#2RbO5Rii$pc_s&n?NROm(8;|6t7Z;URIV)aKaCLPpetu5YBqJax>CvgF+LI?u
z($Upj$=50Xa_toZ7RMdCcG=bct2x-j3bN$#^7D&ay9=ZORUX>d*zoZ1@X6cRq@0-0
z&M*J(_xt^zhH1MK2O~Ia^-i5S_50h~-Sz+L?(eJp{_gJT_3Ph1eCSxJ(emoUmzS3(
zD!cp5wVJBpSzi8K+B{D}N-9W}OYw`~wFMjt-(OoDzJA7x8Og`{I9Zxj1~2E6Hq$XQ
zocJn$<LZhvYht!$O`SOLVz0D$+WC2Z-`q3~4-a4Etk_})iuV`p0xX8b&(1t~@&vTa
zbyw-?@bz)t)Ae+9b(dO+XT4-%W(KW`dcXg_l$6xHU8S!-JUndi*~V!SXLdl#h5q2c
zK+B2`59V5zSABc4GyA$;ZSCLU=jQ@txwZyKN=lZMl|4Szd;0Y0+dGTXV`A<cXk-Qj
z*Dozl%Cujde}7;3`+K$}F9fEaE-fpQGtIgJDS)B`?B?%JKF;Ute7IX&|JLo>_SN5V
zPEJxSv6}lThGSv=K`9Z%D;yanx8_(D&$F+$D}Hw7_4W1Al9GQv9+yu|O$~YObT;0{
z$LIaMz0>vM*DYLF`1Mt&y!`xaxwk>V7Ge*wzt=4`_V102$%cl8cXyZH-<BIaT`zXe
z-n}dNT4$|oYHEtyoW?6@^yKH~=b)0r$4ADxOy}xC0Vm#8Qw^sT6L}>J9{l_J`}eoE
z+S=O11qGmn-M>GUR~H6scAE60Q&^pcho?{8-md7$iI$d@UTO1Fr%naU<yQP63(5zk
zc2d3D5)L*!KR<u>u3c@dtzTbUOuo7*R8&+H<ao!Vq(>VP52u}*<GK9u`8k%ta&qg|
zulFz2Xt_{+807g76BCm=ckeD<ym)uX%TMq3|1W!a>FD|M{jUN%x~5K_{{Q1~`9&`?
zpP!%q|IJO~^z?L4gx3jxQp+;aiVqKV?AUR4clrM6@8|B_yLaRWi~1IK`v;endbhQ;
z-QJdKY-F^n`1!eie}7M%Iu#sYg`iwDOJ=U$|G(ewgAO6dy|tyIqhp?JwO+)A1sgU@
zcopMeWmEs}&ZSG2u3h_fcXxU9_jh~C-^bnGR|_^E5@f&;m*C*fPfkw${_gJU>+A0~
zGP8gA^5xsRyQiOjUU{xn<@)aO_tSJDFYT@VzB+upm8GR^^|zddhgwaf_7y8$k$Et)
zM?gqBIr;I788eL2&z(4RN-t{5iuCjI5)%_c+MR^l6$E6gN<94i^S{5lYg_%Tr?V5(
zKAdfqyW(7{K$Z!M=cJHy2ac^SPoF-WZJvK`Uu|`0=+v!Szn+_G4O&>nr)sxx<HnUM
zSIY0J{qy6a{r^A4$;WttgD>;*uUffs=8PEvERL=YTCQ$xdU1O!6crcx&Nc%zU{|ls
z=KiCqaOF)RZ<oNLORK}zhpmZt`1rB1vhwDNkB_dd4%gPu2#{TMa_ZEnrXS|doH66Z
z*6i!c&M8}hhHP_gZt51--xRTCsrU4CQCq#@;^w^y;8>{EXsY4VQDvTY=f~gg_vLM?
zmaJU)^Su3kpV?+>Z*H&7%*;$pPrtr9GBs6o_E|A!xl>c8PR+fwCG+~a*ciR*=g#H*
z{`OYZx@?J+h|>zr#QK01fv;`+@^bR>^)D_c+Re95KGyT^@9+EjYG=O+@Xfxrr}Ar_
zap|io(*hbMsl0sm?%vMg=Vxb|zkdDN-Nog{uU|=wT>+ax<%NgRg^Vqh#m|;3S)voS
z=f}gt?al1`Wo2dOPMun06&_qzRFpL(cS)>|kI(;qf0z5uKlk_}pPY?^yu5wU6Aw@#
zJi_??sB(+Sxy|Y4r|Co%y}f11*Z%m~vu#O7yV7pT+*lvK|Hh3Q>tkE4UcFjeRP>@`
z*Nos!-?X%6^7VfT!P81R6Am_kD#tmWIksxt;Oi2YWWmP9=IH3i$jF#`dt2=8vfSx<
zu{AX{EBLPdm@{Y2zKdK2nU|J4i)si93(LE)!7)Dm|DT_qlaKd-VtHNc?hOpCE0%&v
z6qSWKy1K%`!rSxj$AyHL6h1mKQQ6(D>PyF~fZmfAFMgDEUO($~?+<QqJqZa3(4v><
z=-;odf(p#5*RL<P5^?Ha0;O^l@yRD+w&g??6@6M0xp~o|MZdnhG%kO4=g*&-RmG-!
zzP@Lze%lGLEL^+RHsgZANB69t!oou5cD}hbl|`POC)dU977`LV)+g&-s?j2l3Mwo-
zt$2BR7rXZ#J9Z4TN&dsb!&OyPyUX4_`dAV2Jd35&tZb6X%`;Mt_xIIWCm-vn`}^ze
zzS`f1PZ)!mv3e~6Qww~AoIGY#R8{@^cwGMF%a>JERngmWHg4Ve_t-HTIelGS-%0sj
z_f>!Qb9R1wy2?sXabflMcT-%lVy<4jy1VS{tu2|snVBz7OjIs^fA8*McYaVrsxWP9
zR%%&ta*5~UdA8NxzTdA;Pfx#n^JcGvp^}~5zijFMzh19rV`DqsCo7zO=>4KKYjP43
z1AhxPDeNkHdy9uFV3LY!tbO>pn3e0+y?gyyT2Qc&Rm158GpJ||0xhBgrFKuxnWouc
zB_&%nZrr$f_3P=zopu!;9$Z-&%mCUn-_XFIHOcHm`u4l3|J`aF9S<&WY`$~n&c2Pl
zB_&@P7@5`l=fwmBe0abAzg_LGE9cJn<!}l*1%VUbk_acKL!X|Wc6N4t`sB$p-RNh}
zo*jEw;9@UjS@h)V>+6gS3LIkHrGI~Y{rC5G@XW8;TH4ytTeCvr;{H9HxMhn;XRPXq
zJ$q`NoS69i{eAWZc7C}pZ*G3x%sbD%{@t@@&)&a3pJMd<+}zz|Z=*^}OSzhqT2$n~
z;T6#@XY1tT#Kg>OUGic>;bS*j+r5vE_j9)%I&i=tkdcc)#j|K<3^#Z8lP4)&hp+jD
zg?)2gAJ(Ms>-&5CjjS5|$2V`?Dk{2w;X`C_Mqb{#KR-Y3E`9y%{rmj<{Qpl*PPVQ7
zmcZC0@Jk9DjOPy>Qc_fWc%YHF#;*R_n#jFXUqRE<Z&#XoPSVlQF)=niee&c&gXh<4
zcKr04v}E&U<E@GN+Eh-i3|`*W)>c=cqN1Xrq{PIarKR=n-|zg`*t^Z_{NLW*{(fz3
zw4I$@RaMoMR#y&B#y#Tw0$FxmUS6P?rQ+v)cJt$(vw@1>uCAbf0D<BS`Hw&UTpPVz
zPe*6T#f<!Wdou6tGQHaQYD31wMSH8i*L_QPe{b*e^Yi!T-QBfjjZSRr-LtdJZ{NNh
zEX1#<QVPlfrrN2gsr&zY@=i>AcwH}Vv3oxs7uPCV%haeSsp+Rh!w>AzjozlCq2Uo4
z`t{k_*;mvQIh@?xpKm^C{r}(Z_oms`YTjzTdi5&%`ntK+<$8WUI|Z^VKtXi&)UB=A
zpi=g2?wq-EZEb8^%wLzkzqdE#q|io}$x_|o`uDbGUvJOvm$5WTKR3r}y9$SstE;M|
zWu&jKZ_G;pr?V{0uNXKKuVmcYQ@J_!_O>c>>*{Z7rcJwcZEZB8g0ZnMgWGD&?31TV
zD);3Jad}F;U48m^cdvU|UR2br_Wat~+UMuydJB}AM|4=-s;a6wH{U+KM^ez~EF-9{
zewKW8mT72cDDTap+uL&A-r5?yC1c`?+c#xRw%%B?tVhlCT-wvRA67<2MW3FWWcbi)
zoyOw0CVv0Dt5?4sUK7EgctyZMPfqcQkBErK9E-w5jncv_8!J90<>$Xw>s7h>=<(yV
zYu5az`L}&{&P0`&)2DAwKF+t1)tSSTA5@Ob;;gLv$;vHO^74|a+p>SVUoF~o>gG*L
zW#z>$7kkdXawVj=_;XiR7h9iX#P+;cM@Prt<xU)?>J|DR8#hF(sr~(pvB5#1;^QMw
zKUhduc)m^Lryn05KY8**KRsK?bJFq0m6er&f?<n7LqliInDONKbMrr%pFVv$di1D;
z5vSsn0<h!%e5}~><jIp;p)b3|^;?`Cw)5IlODEI^OiVQFe<CX4<?Wqq|KsKi6}8DL
zmoLA*)O)&K>@EojiH?8aetv#$5(S-tct8z%?;}r(=G)c&dUA5|+1cjmYHCTmzYboQ
z;XdDX(SijD85t3~AHBJ<Qh2W4auqqJ6*mu5f8<y=t>b9Y*Ecser=Oq4FK6T7<Fmx7
zFD21H;`6h!zXJ-hva*8aawuNm0oD6XFO;Pi#l*y_zr9&##o@GKCa9IxA~kW+nwt?z
z%cNL@8^RBs_0{j+d;DzWag+HgPkC0oE}#9Xf<sY76O_Iex-DF?ts?gK!K(T3p|OqO
z5gx&zs+V4X;^6{2nDoz1Uj6?)4=W!7L*%@fFCRx1JZ_w@cws`;@?67~rK`9YZ1PNx
zDhfJzOnM;ml|%7MamL{<<<~?O?Ah|c{qnQDygSb6>xw;FB2t}G`Q}S{-8u$_7hP*V
zG<6F&$v;pPRBWm1y7lTN7Y_r&rXL4|x8IHTX`TP!)%L|x8Ct5M)dWPIICCg=wtRmi
z==4H;%gig=c29b7y2<;vy}pBMOJlyA|LZHfEnDT1C1z=r<~x5{wQ$#-KXnS<bKlB0
zt(m!4=ey&>-sflTJ)Xj2+S#?;wJ!XJslUl8TjS*h%8lV(O-*vE_sTFZM4qgk_2=Y1
z0VfY7aG2WnW!{P1Vqtr-cB1Tre_PLX)t+0b{_H~o3&Vo1m+$@C?N)uX-@a^a>_mYn
zdp<T?xDa;xXo%--+sUklc$_P}YkT`lyZJMt`|HedJuMer{r7$U)ew6-)Bk_u*Z-Mk
z#BNzK-*myViNV@&tTx$C>eZb%OwB+AnUlTWI`7%D*8iA#KciMeE%uBsZ`X3(%l3QS
z7VlNk|9fn4=e3Q(v2nI1SQ&Q8CGXCSWN6vb6j!ar?ZFWBJ5EZrbJC9=lbrMz1Y~62
z9=+y}ws%hQSG8vCTe%a@gDhrZ(60!-)EmQ~v3sXHcR<U8kONX8iY;|A&L(1<4gak+
zE}Qvzc1YN{fCR>q=Y3+gHM6TY6#ldKF247HvEgE-&+op#e)U}<)0i0@ij0Hni+%o;
z$9~#l`?Q9EAxvBJqTjuo&*Hrs!{su6KU7+`WUYr&-L?9Iy?^vwI2JAgH7Ys&_RP{-
zP@=l_*ZSox^JXoSV)$_R-L!h0i0;knk9F+}lCu!v=$-X<-;SSp?i>sbX~jDk0#+Y7
z`|U(^Br8Kp)cmkm2?mB-wXLsCJhR^$c&oumLa{{!R9Y#1dE7G9)Kp{PGRO7hH+DZ`
zund#s=XQGE6=fxvyW7a#=2XJ9jW4TrI`T6zIJh0YC&ut3r|o2e!uv-v+1TAV6hVVT
z0#5ep{#5#%4d<zx$(9rU-E?}5Zq`c{w#|*=sRv8fA1Y#Cm?^h7m2>Ge<}D{5wcKGZ
z?LY0#p~%?+8p8Oy<5jxmqFwz7Te8iQ6ZYQjKBjAUt88uYx`2bXMe>;WeU?3OS<cAd
zn6WMP-Lf-{lV)T`G%B?$;kdSdL-C97Jel)#k*dFf?v-#o&))vHG^cc1-h(NwQ>HO6
z2+Z+Y6`OVTlV?dnF#`jO>(2XD(;iHIujSVk!Ze{~SJD1=CMPOi*FJCmFUX;&QU}Sd
zmAhZZs4_4J^pyQ-->#KWR~I{VbNTC?m2E}tx>70odp(&M7$kkCOUj!iSocq5X3#im
z^^7|p@UzDJ?>FmS$UODmC(icFMo&$Gf#J{c?OtAmX;m!~Ob%MU5_Ed;JSDznQ_!;0
zY~p;}8LQZSRsTIT|Jc)e)e*70Yx>+?{q1=w`ZDF@<|jw<mM(R&*k`u<b>Z1>I%(<J
ze~+eW?mmAqVy9$8Ia`C%|Gll2PEqpOm3#~g&cX9pS;ReFT<JZ_#GtU{i>PU)s-=AP
zS*H6LZ{IzBp>Se%)wEgl)=nIMH9!^Ti{*Pi)L1=T_UlvX+p5=w5+#1q>tuI-ss6wD
z)S=hY<KEhT-&dq_F6LYH*HiYlzvZnrs3>}Wa57`VlV|Dk?oAT@<a6xbd$~ThMQ=H-
z#@*fHzf^wr{jzB@UwHW%+<N)8wEpAmE#>m3E~txhc^%ujNcG9RpL;HfD7L7`fFkRP
z#ECV3a=kx0c4lTAxR&8Mefsi~4ErC1-t4uWe_vUBbJ72~^S!sEe!Y}>qG7`9;GYk6
zzEHoqZ*_95{r$qY5BJu~Z?z~sK2OO=%w1WrWeFeH$d0Z`&+<x7-WwCnJUW(cS|al2
zI_G33@A@irP?TDzaVmmtP3n{eDQ*#P^0)&kwLxN!K!eL1iY)>{=AdxlP;8m716+D>
zD7GBoeV*pTvGADV>)hMB)Zf14ye}VbGC5<l(cO791<&p_AF`^~OE>j)jqaQ;`}x{?
zGX{gI*{i<2aY|?U9~*Xa?}t;<--o2>D9O#y{B!EEpp!=tXspP|zB|og-Q6vnQnI0*
z_qq=W*TpF8ub21oGOoXD_VRp^L*du6yRrkK_J0e`6=68#l)qo~*0!5p{G)BJ_S?@j
z=#7pz6cnnXx4+X=!^z_l*s1o7*Zix08})uGGQH1LVSj<+)VBEC2RHs7ej4#(YN>hV
zVK)ti2?0~8fA*dF@3-Y@FzX7vJ?~~etZq&-SoJrhIyKalVS%Q0Z9NMp6yET62{_p|
z>a40NeEoBOyLWxkNoB{#vcB{CrhM3vpTEpx4LiexeEI#SWv?!qF2%rcS^3hp`H!Eb
z6t1sca<1^Kk0v8S<Xb*_IabA%Bdpm0EdswdQ@q~&vt7zR?{i(!$+EZ)A7#!z=-P8C
zKiB2;@{5t23|{YBxUIKM6twkNzt^?!+^fS+?dLHmwjALCbu|Nm7nZ&{lXAN5M_9xR
z|5Gb3-(+BL5G$CUcJ1DbAK%us^#&v`I%GQDa8~%@QvN41*L|zJ_&mW6Z~ihax@R+Q
z{pzKz%s)OB268B>h=DS8KyYAD$>o<zKR&$ORXyGN_KfX$=b0E79MA1@ooRkade>@x
z^GC%k4oAWZVpAd*6fWN?ojvE>&nb7GySS!*j4zgRFce$Bp?I<#RFke)De~9YxGueg
zfq}t8<J65ZoyR{)g&K4!zm+V?J<J`j;A`;vH7pFOAI)1B9G30jtoZb>ovT^>yH#;e
z_8kWX1|0`MrywR!5M?<Xy1hFnZx15_Q^>9Lk!$xKl3DQL>HNH>_l@g?a}RL1UhMs`
zy4<BdBqn0!lix26>v2rmHSM+5;<?$WuXir(7jT*cF(xYY_jd7V3=Ai}zF2;sQR)BP
zR?GX>BUYwdYW`86{BLg;_v*)Y_@1Uusi|nWxo`c$pUhlb6^ZVk9zcj<OMtL)@FhN$
zu488>Px$=pO!NEdU%#b4rY@5HxOFM(zNlA!3YLc-{XKIbhuiLRx5JD#nu;i@d<12q
zh0Da{l~)OIUU>6kKj;0gSNP2zeZF_UBc0bgS+i!1(a-I+la}S_)ZCZzS|O=fek9x4
z$>Y+)QZB`oS#vIRvCjD(zq#5v>)WqBOV6ylBjPI8?sTZ%%KG~&di&S?b(<!Bh`Y7^
z|J3Ve0<{+3IKM*BNx3B;7&KtpGV8_x)m^n)TOuENE>C7;a9H;Cy|i@I?l~_4i<SS~
z`#MYdV@lY`(?w3VAANQTlbH5-OT?Gd(!jdpRCkVrYE9r8XHVCx|Kc?tg`@2EvM%P(
zsQr8YUYEA?y_5pRh6g2=|K9zxnfK(YS!q6#!m|%9X-sx8fAm@TeO*P*>`(@W{M|X<
z)eraIE0<PXxZ=w<_09K{TPCo8T~;Ty__{>ji#TiVdBMB&)h=#3`z+N<M`M5Gng>f6
z81ChK7Axg>$}{u!!MudGx}m?eiPvf_zWep?VJ)>b7KX?VYhFA({%M=ldJzVWu=aE6
zcW&>FFgCrtiOuhl<&SnDZiWSae(fw@YT>B?3P&qXQ0<p-_>13~fU5g9Cns#Ve7CCe
z$v(fbpKGUFcw3UQe3cSILrnk1O#+dg8C&BmCx&j;-Fx*(=%!h_=gD?Vn7#9GfzNaP
z-BaY>WZB%<YLSzEYNF3qy)1?RIr*^NE7sI~6Izgc^>m1n_x7!4*RRdB`@*~S%xC)_
zzkE`J|IZe5S|JE2)Ep(I8tJZjCqGd$(|GMAyGj-9$KN?6<8vYkYa-u#v?^ct<=K;y
zOqFq!FW&mg`E_~Nechy^;KFghL`Lnl=Ug@go!a2C$9&uBm0Fgtf(mJsja{dm|D4)i
zGs8!d(?hWKq<7aN)lPE}Ck|6Ka1jMApFx?&i9=BZG}i<w;1vtO4R--24w8%d>z+;=
ziXTp%yPkA9V6KxAhvzC`g_bBbP^F+dYuYq1hKls(!Llw*D^{%vl4Vr9@<&jmbAF|u
zlWurv>DRx%zyGYU+iIAUoIG7WeqY7MM;{*_51!rPwwPDKpy6ankTkR6l{bmj8cr)*
zCr_LxX;WcvRkPvvV@8MP=jOKa%lnmzwFn5Y%S;A)qU8uL6+Jd;6ps3y?JA>y(+bs7
z=g*%%ckbVZx2LD;Gb(6nzyA36xV&6QU|{C;b+W7v1)NsA6jbpHn#-uz5;d)>t1E16
zRA@-ZmEEaDMVsc$n<t^9*fL?p!@U(P0z&4Ik&#UbQkzm&hpnAu#pA>g2rid}%$pRR
zotqo|KF!R`?9{1KzVhsfEgGO{W{$uECr_>n+~&xksM-40QP8PFYOZy;pRe!VTdp$>
zKYa4^>B&>4x-3PVI6B!u<4*#nbLP(7e~~L8JG+}z!)b*iIJvL*_%`}&g|eLDl|5j0
zEb7^~v9PA*&-Ckwp`ll|=ifJyuxK@QT5<DGSwM@xqMpf<gXi)xPFC|h#3Z74We+G>
zas)m*JKOy2t*yU5S;@%C+NPWk@B(?PBEee2i6hX>+Z(iJYOA~=$D-KXWzFpT%d~_*
zjsj&>7uEFi=eO=pQhE90$&~}4Qv{qmsvzzN3JeT<eQj-}&AoPaIlG#SloXdll@<XZ
z<{Wlc4n?h%D^~PK8oPZlbrgu*RdNvI7$I<JBJ$fnbj1rdo2`igP98QN;%uBa0@o~B
z<P;mbche>zhM?eJ(1^cZi-40a-}CvCEtnb?tXR?UN+hU3fy30)bXBmzN!Lj!iHQrh
z)SQtsOmYbbn6PTqs!5ZA>gw#S7D~B9MoOw1x^nzA$YGzVGjrxlW%ph$U(bbB9VHSL
z1rN4nUl$P-UCGy^;^*epwsh&!$H)6u@?Dy<HTSleg2ICI^YcR39XUKz*Ep(r<mJ7)
zu+Vw(<jK*JQ6~zb0%hGVg^KOdsSz|yIWghEg9OmD`(p)8(7=gd^07UYpO2k7<rNz{
z_mu^2XjGJzzP`ThW}~T7r^fCo={%Ygy9czjqve$ahvLg-(-liJw6vrQlU$sfn)Y3N
z#@GBjP}bebX7`(Ea>XakoOyF<s<x4lQOI#eAw?sjqCY=APMtc{y<cvwMWNEw!daVN
zym;X`S#9d{>GN$WFC9JF3Nlr*`SBxFr^v{gGr;Ewl=y}xGu|rr6*hb8FF#&R&Y32e
zLi0h#N-nzB$}N88>{-SG_wUPdUsz^V^5VjV4F<fty_&(xa&B&d8fL2TRZCm@c&~K%
z`+Ix)4vUvd?QMExaj8Y2W8Jp@nNq$xN?u;tl6kqsN%3l@z|^ugHw-I3rIeO_y>Q{e
z!^5CeLaqiTCVT4s?pnC;;i;+G^}pYm+pr5d@wR_|q<QS|$7B8S^<Q6I{l;AXHfi(R
zR~A8@8V~HP9#8silJV`$&31nIWmbHP7(6F=PuIJ<w|aYHW8?n)^}Ea7uJF6ENkm-S
z+QQ<_p32MT&b@ncb90W_>{l)vikb~?7la6Lu=x4;g|Cmxy?p%Sfhy^%g-b+x&lwci
zEnQ{(=)^?j)6?~rTk#2*>rH?C;>C{Y@9!L&*;-p$6B)JYyu7@o>&I`~y!rFz&&Gy^
z2X8(VaN>20-(P2YRT8wyFa696&^$+Po|&zhtYXHG$_K{<P267RU31MeS8m?Jm~f;+
zFht!c>C?Nr*52NySBJ0flefQj`EoF5;>bl;r1{{6ty^_9H9vm+`t|$w?c2761qCg#
z5^?fal*sEf6SQ2<y38l`{*@C?O?vx2+x*v&id|M({<`VOhY1Y&tgNk$&1|48E<4Yh
zITN_p?dY*%=jK{}e{itbM{V*e7hNewfo02<adC0Y1A}GDmaUE6z6@l1QDS{S)|!}|
zMW?6fYRzr?Ht(HtdeZ%sZo#qh<g0Hi()s>#p>zAQGc%n_Wv;Xc2@4<Zk(@kr>ej-?
z$Ci3e50YiCee(A0-`DH+Z@T#gw5LzXv<sBnI9p9MSV4ozRbR7Ib06Co|CQ9)Q+ngX
z4pqzV{4>MF_I~^Dp`fg6nH67_ka^yn4coS@Td~5T=EsJ`i<LoZc=PV;c=+N)L}Vmr
zIsM7W>cO(?iYlEO_!sT?`~7}BXsYhQn_Jbp^u-qVHZEOpGyR(Av_;cauYP^BTO71y
zV*d2$(`U|<^qeHFA9sW)#E4hk&c?(fBsTW$$&;D4&14EX1y(U%TaZ-x=f}sR-QvYj
zFTc(#?bKZDQ#vQjJ1lhfd7b*UHMf>{PHt*$2AOwxUF>c(-&spmt&$QFYWiNO8o531
z?z;H>c?k(0j?35U#O;w-&gaTe%l7_g_xV<C@i}wm+&FdX@3XjM&CEdA<v+F0*2z?-
z$jn{C#mOmYRibgV@WOc|&!V57o?4f`dvj;!=KcHkZ`)>ewKL!^Xz`(pOj1&kn%|rk
z4<8;(+PK26MPQOhM03~4soLQ=xw(>a!fKz#Ob&kSQ95VZA`4rycRP5a+d6tW49v`y
zf%Yt1y?S-sx^<w%{RRmQ+qZ9DwMy%1XMl6z&O4IEX)(J>GGAO+I9)$JEG%r9m7tS{
z(u2Jfg&h3+^9>T2=B~YzzRX^1!J{Qha_?OJ>Xa&a{>qs*A0HnNIqqn`V#kgfckcW-
z)XE(e7S`C<7}4M*F4q0@;X}jxdwX8IcyX|qJv}8QU@nKEir@ye+@kGycQ<X?ruK5v
z-t-?Qx0@}n^6OrFD|5xf&od`Y+H_G(O;b}d<hbK~U2W~`6B86cE7)wSzwIo3ULfm{
z>9Y9Zw*32`gGtm?Re2>080-a<S{U9RHQ)N<#}7ee_pmU|s`GnR+*xRUwJ>Vk_Ri_T
z=XPz&z5eam+1WSx1G-dHRcFqgot~DKcVh#icv3E#jjiqTGc%R7wO_w~-~W21;*~>*
z))~THUS39~#@g#2Ncq~fvubROD&K52r$E72Ir++pKub%@kmHWLO1nyDPo7--<wc;5
z!|Zm?$!efY04kow#>W2s{;M7<w5(_`%?S4N_Lj~~3i`6{#)+R>X1(KB^;jX}cKW7+
zKV<de_81r%YHDaKu;N?Q-rwKf*~vNo{P)kFi;IdDS@C6=D7*LZaB>C)2N#!=+}T;I
zezj1*Y05#VEuJZ9X?-%LTI%Y{cORY@7JDo}mi_7lIRo$fGx7fOYz)oK^))pYTJil_
zWn29Xa&iU>JA1#J?W|W8D<*z@aZ%aGC@4O@{^q7s@9BC+^%6O@`b0FF?Ynd5&c(&<
z%lFsiX6*OBd}7`!ix<MO?tkuOUg_Gg!-AW;yQ_<9)#C-{A)ABm*Z+?V4>vb5+61cp
z)SY~tqNA;CY-%1HXxv@)c9u<LQE4e?nUK??Hq#7cZ7nS$LqktLziZp?T{?H8^_9g6
zzFldnx3Zt#UHW?3+O@jVPcOCN`?X@%u3JZsrvCo+_V2H+-2D9SFDzuf+8H1m6d1_K
z!2#Oe{N;;@v9Y$E9yo=r*}#_TlbV{Ee5|MP&yS7kZqDLseg-OgrZ!$pTe~mV*~v*p
zR`zF&-PJ+?1_w{ilQ%XdpPy@;o|#$s^;IaST_C(r?ReVe^Yd)2tG~UGHqYacv$4?D
z?gmv19}h}xxfl`_78MnBZ+H3n#~+T)b$cig=Xw3=H+$d3oS^;CX1P-9Qhay5dGltW
zbGw_1%b7WrpJ97^osyguUj%KAO+4JD?mv%5!r;Nl$?75MP8^+_*B10H-nHx2y?b_A
zTC4sZi2r&l50nVx9T%4tTiiKy;nAC$n~R?(EjFDx&#u-g;Xp%6OUu^m>+f!EZueUr
zJeQ;M_NS+(K@mE6GU&Ko(D49|6<UrkW?yhq5a5_IXHHX7)0STr(h>QKt@yGuq~<0*
zd9rSm^`jRT7oVP{yV%N)yY%z3vyRPdsi~>5b~QK7p6z{Q(P8B0=T}!($JBV>%9Sg(
zZ_j>Z!Ld-y9a57mk@lQ)dAa}of`?A8-#y-KEu)qZam`z}=~%|yv&%Hzradv0XE@Lr
zedNsf^XHEpV`I>Z*^!W)?OiJ4ve9p@)y$bQfBybG->TFrGP3eO10yIDOcixz)LlE&
z%I!bjZf#z|wYwiB91mA_Tx!=-zFxg+`=V(V+jyn7<=^)!og<d=|KDF>HJ^Z>ATB;W
zGh^e$R(z8<+}zwIPMo-F*RF1H{kVNKmVUBb0mf~wkNy`iH{ZTCdi$ypsaw^%_~)Jk
zmwkJ>ntr$IoGO|xYf~YxyWPxmlFI+@`~T<O+~n%+{`}L^)1W<*S34)9XliK8uqb?V
zcXzqE|GYQv-sveTJC`c81Ux=0_2oF|IN9uLYXUEsy)(W%|KOzMR(!uA?@XHWyZv~G
z$eu4BJ`@xl+!;HYtN-|H!{lQ#jnjL2duwZJuU@^n(yv7&Fg11Q>ebl^2@g(8RMwB(
zwPeK#50KHyho!!-gX*ttas9B2%V#Uf6oSKMzOq<R7$UapzP-(|T_(4WA7|gKZq|C~
zQc!jE?+X_$Y)n2r&#pGg&#&*5i$@VBOVd*C>2lU(YpTD$yS^^=@+HttM5h&oiM+p*
zL5r5Muj%+{@7-72oMN$dMPIX2aAlm=n?!EK?JrJE)qeKuS;+Csu%e<O&|%XrE-scd
zPJ452uk`Z-C*F;vudne+n{COzZ)ah#!+W}3$Z<ywQ>itM`){MxXAhr$N?T!KZTRhl
z#jXuT78VsZHYA1|&r}2Lp3BSo_v`Cx`~QE6udWIOoz(H<%uM6Y&(DVk1nek!dTN5A
z^NPM^fh?Jb=DPW(r|aLodDGBR>R55plPgw1vh1c)ANB0mk$g=@&Be{-%iG)2Us(j{
ztEnA3b!yd)9To5I?OnHS9mr616O$`9Z_Ygbe7<e<wR`vGfzq#AL$Psf`-Tl0I)&9Q
z-PGEf{xg~J)*j8R?=N4zy1DDLxv_cupC5)-3zy8^zyJS>i;GvUTD51-o(&AH_Q$SV
zF|n|S@bt`lbfok5x3`O}_`pLOw*o-b*S3v{pz14T%JmD&t@yHvR?Tgb*f~d2Q&T!O
zY0{)gPNg!Lr5_$RT3cIJR{qSszOGl&xb2mNN7vU^SGA3dg52GoZ@*txTwHv3b_Iv0
z^%_TM6+64SHLa}u{$FN2TYF<0`_;k;GgiEwT{gc)$zH$y$%?@B@%xuq^?ixlnsxQh
z&(GE+F9ZYyFE00&U-ejF<@eM2`&-)EzrVd*{`%V5;N^Z#KURS9*k6k`{89UtELpPf
zaNAyOz1OB^y?x7Bt`<(%k$N<~;*qJlyZiHJXJ^lvH7n%!L_Wxpg5`d5U)|lkeg6FT
z)nOL3ELkQ~r%nak7R1G61l}$j91yU;O3>+rdxCX_ZQPy;!>lha7L^}$J0};>u+1m8
z+FGjT{>Mj0yDu(w7Z(u;kag#pGIeUNwE4R$D}$dsdv<+Y>|rJmQ@zN@NPqwHpxtY&
zt-P{UCf3%V*7!pCM$;GDq4nGM+`R9e?$M<OS1Lv8y=C0|<yfANmzR;Tu{NlB>vO68
z{q60B#KTR^&9Skud-v?Ikm0-98DQ+P{4(fNhN`cx-23G~i~4SEP6q`>t@#?q{jc8M
z-oDta_mqeK`=tHho7-it7784-;)}nMc}4!?$B$xSVxR?k$0r7yJb99tji=(ngM+I=
zSM$kO97r%&;ny;2^Z&o^>)H8aPMkU=Ws(u#<HG~0mG84>U)UuoDmru4tYu4<yok=v
zOz-h-S7u;{-E42oGq3*jwY47~A1~e+v+A+O9P9FTJdm~G_xIJ-|NE(awJ@Oj!K+uV
z-o8Eiw8*_zs`TY0RZ#n3AM>>Zr5d`rrskkTx^dmjTaen#Q=oQr{q1k1yu7?~q204S
zmYp>}Kb@Ees%LF%ZI}Dc2kr80nlXDeJ3G6!me#GCH)Smf8bCRs?U2-$;;B=of{wAQ
z|6g}p(A%mv%B@sJrI07LJX^~5%lGf)rKO@=ts%!Ja=7=&Y}~r_X@SM{b+MZ_Y*=8$
zmsMEx{hh3!;Khx}$62|>9(Zlw@Z?_OXdf6Gn+x83z0|f~+r1fILykL6>R51bm!(Wr
zYDVR|J3Dt4Z=5M}ZB3+c`Z<~0+_xPa9LdSas~#^9TNS+AZ&7B(|G(ek!^6YpScxc}
zZ2#O`ry{_i20p%Q_0{7$U$^{aaSWKt;i;!MU;4e>N(0lPlao}f%ik@r;xqkG^Y6{g
z&HnRjZmtg3x3{<F<>g%^yl~pOMT?S7P0_r*F4oxC`19AVQ>RZ~4obMIH^^>b^PFUy
zcILy^ueo>cZd?5~PRHTm_KOm0;)2g>{x?dvzApClwY7___*hS!KD~C$8ozlqno%dD
zxdT)<pFDZ8XV0ExzO%)Igp6`-Y?w1gW|gpF%f2qic_JPytlVNU_H{9L58OF%bFSNQ
z&eVXv86|12^UPRJ3Mw+jefspNnT>am)x;l9o;{m2YnGe4`|ce(=2(?>olOe{9iwL=
zb@tLFA%;DB_tw_d>gwoRxpHL%U#q|>=55VP3_Eu2l(ne{2u+_c@pGosT+dP&6;p|v
zqjsD1PGw81iP-q)!b0be<ARf~T)D!?%y#7H(VaVY%G%XPs580>y*O|G|IYUO_^PU3
z2O62*-`VM0D$^1myTM)~=gyA8x3{)>-8xpA_RBcpMkPnBdEU$0zuV7fbH&ZGtF1aU
zMKk1hX4Ln0cQ<d@qNAe&+8lmnCg`l73)@*(S$$_3wW@gf`}=2SXM@tM-Nj4U7eIR?
zBR9D$^fiBcv(YCqb=6~q%+Sy&h3_X+mS%3-zJ2@L*v+w_+6D#%kB)Szc!KJ(lHJ0J
zoZ`~b+4=eL5fK#+4m4U8KMRm$SNsy#y-&RU&5ez!-qQ-AO5Yj377|<VB7H;Xt?x5S
zXPyi!FDcnm_Ezd@;jAMYlaH@iyEZj7wYjNj1B2_OLpN?%C@C#+Zs#k0elB)zl_@B@
z%5S|>cH#5$^VZhZo2PF5eRl7$pi5b=4sQ)oh>+j+aPrKly!Rv)teB#ce<Sc#_Jy-m
zMyuwBRxi2!H~nqi0g1cQ;(kr5w+Re*9l1rZ&1WswzrN}wtNB-#*!@1<l`^gF%=w9`
zAz9@Xe;#dDU+=g*+yBS0kFWpemR?W&lI^-$LugL@wI}~~U7Dl5(Ou`|p;HeYJV-p;
zX8QHvk<7opzJiW4xUjH!|Ni~pGb>&l)74+Uw?uUI;f3|t*?E<3r>>L8;d>oDyFMUp
zhNQgc;aBN*{a<S<Z)9;TKb-ujva8m&Eo+0F&Fiz5Z~Tt*I%f9ww%HSn-q)M&^Dr<N
zSj_29pVtr_7h2t|UnkvkD14)S>#<s^tADM9L{yL1X+AtD#rJ0Mtp)cN?Rfg^*{z#5
zZ7V(`+}e`q-Y<95?+k}$lUS8p+`gL1Q&Ti`SkKyR*6Uu#aP`HJJ6Tb?F9=o5S;6uD
z6wj%<zc)_*y6SJa;IFx>cf>@UR|MbP7WjG16?2;gtL%WJKLXqMWF4l@`?N=?%YDbK
z4enXj_9Pc<Zee(|-jOq0+Ryyl^8V0|;qzCWs`-DnL-0-KE&2P`yyA+)Td(|i_ayYi
z<+hU&*_Rv@IyxUOzqYadR?EdVd<-}Fwa%Nn{_)q$D0^#r*Zkd#t*uK_Q(Rm?>tX-R
z@a;SQ{QO+&?>|01wypm5<@4w4yP(x-`<DHA6|a1arS4Bd(%!58<+<ms&zVzx;Kmb%
zpC4KCH@!RbuJi7`+J-f=-#shd-n}pGY3jeNQd{2K+&a_SobkYt*v4!h)Aw_3SY<C?
z{=%L4e|ck<)R)iqc^C}b*6;f||8cvq{?-5TzxV&K|8u2k@_BnRHjJzNJU6am7P0?O
zIP0hU!8!Gh9p|h46#rww%iH_lL4vrrczOBvIdf#><>mR>ga2O^aPnQLdas+Gn|roJ
zq0-#7m#U}jpL0t;$s#FFD*x6)qxV<%^DDL(GVi+<=Fs#0+d>9a`TMKpl<oM(B*Ktq
zzoxGu^8cr&Q%#RbM;>5iFnZ=#>fq!XJt5+>zuHB=ZE7dK8E>sO{$Q~xDQYc~2m@2I
z*7Eo@LVN4&b_ajl$FNj)%jDb76|!|!Eo@B*jXfTCVOco?gMgcA*US6Muk7>p4?nM^
zcmK?tKOY|-e`J|)dF6@~683c<o_oKXtc;A#K~@chf}HJl>Pt@FZe3xybKRdx%g@@g
z#q7C$2Q)}o{1N@CvGE?02*a&!TW!9T|CQyE++^x39lg4UA)+mL*<9N#Ui=&k0>xJk
z>x$o%esCji=lk82x7v@J|GZln0ov1C+wR++U(utI!LZ@oowMIrGXg&xRCQ}UrgQrx
zCqv7oGvRvCO*=jqFl>m*Uw2F7e~;UthXs4~?sYZ3!7pw0=I!n6A?i*mmSz{4f{yN6
zvu3U5uAKLh;!jP#9Ibq@`lHRe-|M4ap8m2fTbeg^Szkt;1|!1+%}n04b53|YeCZKa
zdsM6P#QH<8!>%f>Kc&vVFkzD3hn=x=?;l7ukxJ?mPWOA>zkct5537!RS52{AYjy7#
z2irlGhVngo`|Z}8_j{J|nJ;!t&EK=EKiQ-V7#jZj34~shI4JG5^|FBS^v$ZOsf&Gm
zCQY7vwEV%bY15~hCmrcHaKJ%bT|G0hsBPVqqbYUu7q6?l|8X?B)GshjuJ+&KOJQ$!
zNtk{4X1MxlLt(ORXO-UUH-2J*;eYn5YdP~~F&{%$)Z5;+9+|NFo26nyZ;3u$_}uGW
z{c8KXd&dGzdAD<}ES(zv=akQ+6yLw+*R_B8-JWu*q@1UdE9#Q}v)k@mcArH!Y&SQ`
zN~u>(PBf5c=aY@vS(I8+^XEV#b8t}53cr?s?rWQ}F8uiMW22Z@4ddGDCh|wO*&eC=
zp|wehf#C-egVo(#{<80@zt!G5@8`wNz_f39y6DA*KGD>rhgNs~v3A`mul)ISj#G}2
zvfGB<8v*}3OMU0<)Bbh0p-V6MfVlt0cV|-=Z2F#Gef7~3dPQE{YOm$IdG72DXVUku
zm95CtdG$-pR&D;7^ZF8|RbO6TUw^dv!!c01w2_(p-v0XfpP!z(_sgXkEls-@bM>2i
zc#hRB?FApYd4EK14`;iw%P=<fKW|Hvf8L2>CB1!fd9APgvsn7$w8zJPOh4C`-Cnco
zd)SJpjmBjM|GUIj{;jn9-JPz|`eO;h&ay`jZu!47T(W=Ks#UB0Y`>@3C7jIUsknE4
zZa(kJ7k9R%ot?F}`a7t#@ZdmW`TKiIt@xaD!?{;EPM$Q$=j@qN+B<Ax&uf>zaQ9?m
zW0>x4^n!t5zl33_{a?+62VX4qDfs;3y!SrCfF6kz-rHXu+hDMO|MIG8$H09I4W63~
zll%;lW@KEp+x^%5#3db`7hmrSrwAS9eIeX4!;Gz`EPhXQ*=v{W7t|OQ>}9)qti$qP
z!@_?@`EF%j%-v<EUXpy}oMnbW^(;GChASZ%YehQ3-hZF-yXX6&Wv-L+j6YnyG}}}D
z;xsm<gIh%uYxr4ntJu{TINsMi7nW{yGhe+<m_H>otxw7}>XjJJp8Eg)4m2`fzH|vP
z0>ZZHagBUjCF_M>-&mO#TF#fndu#p6X})p#c*C{t@8|v(zAL|;`}^6`PnIls0y>HQ
z&Lpet6B17}EYQ$Y)KgS^`E>RneS0p^G?nxzSK>EJd!DAEs;Z)*n)+l~?auZm=BW$;
zaWiI4nJ{6(oP&Qq{cM`l^JGa&OUsfcJ^k}v@0fXf`i=RDtSTYm(~D1^&YpMtntAc-
z<4g>b%I`eiESB_7`~BKPljft&)^+bEg*UZ)Sj~`-d3F7{b&Dot9_x{uH*cP(sHn1g
z9}fq|0#Ia4DXGc%Td5znN5ZtK<l^o}^YosFl(q$nHnV;B6u5Plp|>o<44&C~TV(z1
zr@5Q`O_5;uo?KaU_|?DNCGU^Tv7ezD{d%IL&B3kNq2l*yf;Jc3GXBn&thML=wWVGK
z$95a7X_C0tYq&~#bF&dIOVg9|sinOSRwi0aczfq@GTZHkPyTkC`m5<wDf(;8!87Ni
zcr4XEeEu=F>SB*+w%xIw*ZMb#R~gAMFmSN`%94(p8&$e4SuTIM{PtT7UlSjMXghy1
zb7o+y|7sZc-v3jOQoM=8YG3i0>$YcCaWgo4kMl`wSQK(k_=nvh>5mHcq(5%W{8}VZ
zaC&}R6zIgV6)Q9{BMm^u?+B~=iEywyKR0*sxy-$%C6(8T&5-houH<6aCdaq^=lOS!
zyEp25eAB3MZAz_k>FMz6rX}kYG;{tgSvv8_apB4L>YtsgfByAaLjA6r|8MSozhT)q
zeZB|xXV0GCz`)8X%4n|iC?j@<=$(!oxp$e}?}oizwYqQB)m5dRg};a0;p#G)v!J8v
z5mSr?U&sVO50AWe@B3R8HZ5XGoOAy2bIm(<_I&5y{QkuDy@m1G<8y*Dmvhc_lmGjC
zdsmX1;BO_14{tcnUtQibHTYYM)SSTU|7^}Iw|@RTrb=ru&&9Mm^`G6XJcZ{~$=g4E
zy!)VjL}>DxB=d>)pG-}^yw2$Bn|kK6m(JDKWcp7JtWP)5$k_Vn#-eRgTYlFU$5#3>
zHgNx19=o`n;f#&IjqB$)3|^TeUAdI~pYg+uSC`knJn-e=?Dy;Ee|>Vb)@|G0XN+?{
z9alGP{lONxzplvf(#l|Uef{<GY^!(e+Ld=_N8yhT4|nx-Oh{TD@uk_W{@<Rhi<8gH
zo)J}RHuGx6`o|eU7W36_{p*S<b$xlay~pl#NF2k?tnewb&aPsK-#@RSD0rjYj{}=e
zHs76mOwO~C-Qh@Cf#Hv%e-<0B)#~iA_;)qBH0nM3?fuh_91JsAurjW75)(r}nr*ZD
z;!y3sbz56GFRtX?uJX6g<=-sk111Gm<P-Q9L{D?`x&$yf87KbTbmp?hQ)A!tYg_(q
z-6_D}@@CV%U!hB0ai>pa-+j8zt%!l=jLYTSuU&1uVxO#?8~f$UvYKmhH$ORdeTh^q
zU$-=?I?ZKGckM6j$(ixa93Q4T?OU3_#n9w7O{Zb`#K`qGr`WHNX$yV5<H5bX)o*WY
z_4YWi%YCf_WVAZ=NYcjX)59P7I5098UCFo{YFrTf)cX0K&pp?FKi>TNPvzG;alT#(
z-Ti9$>+<t&Rxh%@oV2z)AR;2-MQih4$!WWHRVK{VG*7bkinxAe*3;OvPd{vAHkDnc
zb^mMZ@^}6XZ{Ez>A6Bu(Msy9!0`)WN=8Fc{#y)!R>wKnr`TNs}iHg6c^RN}<ooz{c
zckn^*I{nz^;%ip1J^Ob{a`OLRzqnl`LHg_rTKDU>{mgDJbD0}?FZIg(*tDEoLQ-Ny
zb?ayMa=dErjAe`OHhr<{V0~%5Cd0OiF7LI&*T?L55Ec7-=9VeW&l7pL8@PWR{~q^?
zq3C9-?K^=juWlXr6<vSnBmdUcAGgF>*SjxHJlwW#-@dT0FlD!%8P?_N)*V({azpc%
z<%`+o`7(3o%yW0W`86VX+qQS+8w9%w%?xsjCT-l%zV(~l)g2S(2?|tdhqBe!74%$8
z|9xGYv7$8k-M<K41<khERZa_;7?wF(dp--=@u<0IT6@m1hfe~lgSLF`RXJpqG<Ejw
zHwjau4v4qSG*UZccQvx$j=g%Y^3)3H#ZmHWR!%hD`nJWh@Q%y(B?gu)3{&o`o4cuU
zU)Rp+;J5;YgqSAA3tRW@F!^BOvG3_dsR#YPPR%c5P%shR#vuQ5+Vv-sR;7IHcCpjZ
z`&=9QcD9-0y>08}J&RAtP~0+|K|W-8@dgWrv^KHK)*rvEBEFbx_R>8bUN)!d@v&aN
zxmK#`uUB}Rr9V4A-#+Jt0XH{ye0)56`}yNf#iVL#Rs{sDeShTGto-!%d)L$o7U=Vx
zu{>zylN;)};#ZB?>_;!#6#p|x>`V;P)^=uCli8d3+$@Bdx%9w>RCVL&hh*9G_-*#2
z9a^z6x|uH{?a}HQ@p*gJzOgC4{BSRORDCYnrMr7qmw&wTWbWqKQBxU?m|FigJL$&q
z;9y(vdY<evyNin(7ppMY+|KxN@`0)IoVo31V{MMPu_*jWRbn{Ot~+~E^VYe)()S-b
zrgLOz*3PqYIT&h={G6b%@y3ptr)CTi&d$NfD>tspT-F+NdZ*sIr9WT2pUXI_e&VaM
zUvJhG75=|hcInOJXKM<#`)6~1WUX}jBObCk?Ckyfpp7@5pPvUENA@&hkG`TxxyVNu
z6Ft3kxwp4X6cpSR`%g1||NM1_FFq0u4No#rwOQt<?0ZP_MaWdv<CU$e7;EZ^x2#gq
z;bf>uSZ3OGMStqTS@TRaZ(q`Q{N#E0RQ)aDkHX@wnWZHz-N{{>>do{!%c+(@$2xz-
zm0A!g+1G5QYNowd(sk8dZ`J)lb1$of{9#vJ5PR1%dhYrc3*OJW=EVB;uYbT{p7^qN
zUaQQmPT4f;&7Q~Sl^FQ*>OU`8r1s58u_UtN{F<!Y_Sz=W#XmniT^GAsOC*x#)n&`R
z$8LQxmgVp6+}fJ``}gng_4C$yyFFnQ4cNVUcG{8UNzZRBh>$z*JK=q-jd@yyb@GGH
zG0K`o7bb^0Us&aPw?}ow45MDRuUFU?*p#ZpO#Aq)@Xei&==<ht49o79oSlCCUB>RO
zy!(Xi-sX1y!>IG#ykNz#3Cpw(Pi3gM`tHPcEr;8Mw|6xsyKa0Yn;vj&lIyKke(&G2
z|N5Nu?w~#Mmh<wjF67UArujS1?(`4oiC1qgE1ErTmR!=)le&%ip$zYS$uJu%etrJ_
zo?SHyb&sTnE%TXaoPKUg=H;}{&(5w}x6WuLAGdmoh<@w(s!7G2ot<xQZ{J`1-0x~x
z@tWBpYjT_T+cl!gMVJL<b=4VqeO<4xDdpq0xL3V7=hmlA{kpvM!17g9TU!}|7dJ8V
zZ?aH)weVGJaA4+v10A|^U5kFN)OyCm(8YgetA^i{Z?6KfcBO@8{XJv8b<Ml*9>L%)
z)&uT#A^W^{3u}c6a59Rm-#nLT(b44v;&H3zp5|g$=2$ziTIBkcJq$9>i~mQfi)M0v
zJYO2saehtJ^ZlVKH*HSe{`KYM<CiaQcF4UvY0jKKr}g)P_9&IUzE=JHovD%0uV2Mn
z%*(Ev-_O6{ckc7p4}!P*TFqpG&c3dfmb_>%-7R&qjs35@h_LpXyZjD6{3(Bhv3#ED
zl~r@<Y#y6tX58LmfB4;_o!tHNG!p**+!sIJtEybTsd>S{o~O*ZTjW2o@yMzLX-y4S
z<n!!{#ca#p>y_UzXP6}1ny_q|mns8~gh2?)QlW+wSHgFFQrT9kZf3XjlhKckQuVxT
zQi1PI*gt34wmAA$&F5Ruvz+!ExVF<k?2)mtiOH8QUu2Ba&b+(3J9=x@)2B~U)4px>
zKhrWnK5WPTf*MCJuddCTiwg=0mWEB%vi7@HYBArdC#Y4W<FzDn!Yq?(+Sy7>SC%*L
z+nFUJ`hDvYpOtmG?wOnm&Rtx##;cG0fY#2ljVtd<@i45|^8I}J_n`7)DIYBs&p&$b
zj1BYDx4S!A8!q@*ES<YRMc}*3)Yz<fbH6zK5sejpWW4$Hu1Vg$c4co49B5=Ne|t++
zK%_(K$nNs@pvyVGzPcJ;_cQhJvEJ4U-eocg(*La_tp76ze7Qf-duv@>mG8yp^YxzY
z+c;I|S^R3_&1zSpyB{1koV32QA&W!uY~|Hg+p{ueJ!Rj#zNSQ|>no$mlr^{3L_U^@
z{g}LUH{YslpS)Opu)pS)(+xg4IsCiit$(+R^>P*W70*v=-6Al5riaR}`~Ux?Cnj#p
zxoO1C&Mq%MUn_Le?UZhTN!DCPVhye@UAC<3{k^}Bj&@)3bqjlK;D7yL%84lkpMK|O
zO}2ULRaNS(r06yC?z_1&4a!6}{h9jleA2#2J!+?~SjKn9Wt^&ui^|SCf9;H8oe~Sf
zvQz1@+j}@)N4<T$-rUYj@{{9aJ?=N!oQ(fo^{)RPW4BQEh`LCJQ;n6I!iM)-51u(!
z`TpMC#csWqN-Y>Z{P>uBR;Y8bn(wC%A2gykH5fS7M(Y$@V?VH(uUG1i`r)!yH#dC>
zpSPHgqx#!PBTpp;0h1L`-?Ufro9f3udp&mnLqiI~s|VHocCqGMocix<E9Y-E`|#n-
z-Sf+L+U_sc@1DBI?%0$ple`95hIyZ7nP&5{terEv_U-XcmcEK|8F#k`h(EFiEszLZ
z9cEkk$z`(zxEk-!ZnD%kUbz3?uh%axFV}tbb@jVVA=e+KDE-Np=X8eo>EWIyUHgT;
zEcMa0aH<Yp8S-@Ty4S1HW~u!wpTACbhF?G!6T^FB^_B5&Wm0Y$`pGX<HY+{iJj*L*
z|DKYBWog>irf>PV!rQ}Ob$gxp?w((JwSG(RwWj<ER9WG7B(*xtOd?m&ZqEXdN0&Jx
z66fx_?G>86H}mo`&|LragDovBCV6*$TwLsadOGM3U&X^UuWP0-c}?Y7=JKoj(l%z}
zx^sNOzptIByXD*Z>sDrjUATOT-9I}kE(YZdW+(F^W9{B8+%MD+d$s@C$*aP3sSQu}
zA2zcIP0<TD@>b?fYQTZ#uaBqanLPIX+8p}7vetw}-L546xLv*5>C3myl*uM+^IU&W
z()Zh&u;V{w*Z=;=zkdFst4y~$zj9S7{u4gj&VPS*d45vTqGcNEUl(v_p6ZI?JN5a`
z1Vv{)dApcpVcTcfUK3k!dsSp*qr?QMCGAJ#xvr{3uZnoOug9m`^Y-nQ@MWr5h7+~U
zx-b_;2J%ntF`m1MlTpPcuqyr4htqTJUCr0f;%#{RE%q@(hV&GUtW=c_pOZaz9Sa|>
zE7O_j8Tzs5nbTR7CEI#cjb&ZBuAg$AslL2^n#-$~#x-2!)@#K#|JVPO>%(y1Ov~K6
z_ZM)Tp1*HFOMvyi#XgdZ4N;+y?dRF#kGW{cFI&4*xKcfR^F$+7<qZN^TptfUWL+UJ
z-%sJnYU_l4&!fk*!`IC*PVZZH2y`pb>1n!;A3rXCfA8<@{QWPrdTbVO@bf)%O1gYs
zpv}}eW7(YMm1Z|~`u(bmNxJpi%r=)xb>$tO;<N{+-j*AE>-6|?d#h-}>G=I&!s&l(
zFAArrH0Q6YdjH6>@$Hd$w<a#TW&WY}_si%cwf)z0<Q7&=S`xMWpDDwKRd?4dHn;Z;
zU<^>Pxof%h?WGe|?`*TudL~`^ZL+t!&zB`&X{Pnr?Z21r+Vwko+m}$AeOs1i?sx62
zpC9dgYK4d<!;a7Xj~EIHA3mI)cTRKVot&~;*X9OSUHhN3X^z-ocS+sy+y6Bfjw=Or
z{BP;*KD{z{`H`f;>3Xqd`S;d<Vs=wk6rWYvnHh7f%cpJ2jekAe*XN)_K0AZbjF}q_
z*9kFdgh#JWzqQZtp`zE>-j~lpbB(k_4RfVmhh8Xi<dV87yzH&YlDKz^X9k?zQF8fx
z3RB07z`I{pcQDL6b=RGf!RamUx-8p`tG6ETnV)$(!swg2;(^~)nbT^-Z%n=P$4BDz
zmaR|MYHd!_Vc5{d<gH-nRrzx7w&Pb$`EANuawU1mmiSyni@eq0S`5y`+nQJyc)YS+
zf9;vLU<(IB%d#Bq+BbV<nDsgx`Sw2d#Y45&Ytj3=MAqk>{(iL4|B2~U_sOpwv94pA
zJok(A%-Vn1kN8_N>lEgM-<W<(V#dtP(`MaX(Xr9@bn5LTqK|6#7g{c_Is8iUkI9dQ
z!k@di#dIRJ<wX8t-%$IzEImDaqDRT6C!XQq*Z=S<oYz>iZr+WD%O6aA{e9l9+1v6b
zcs}<$%<$r?)a!(GwXZdW=E>dP81=dIz*29)jkEY|)taxZ%VuDE)$nh!nTyJb1FAoA
z=WPBm%jt<xcGqfMh7~^~=CN<ce|@l8|Gu`!rsLDUS?v7F?x@}Wwb}G$#GlP43g)p3
zeAbEj%&DdM>fY7@(S{}A`6&zswn^AH8mG_Sy=ztR<Xv}nm#@FR<nyKAPZM8@|H(O1
z@cupP&*e-z3hbStbEogE{no_%;lrD=;#_eN`);yrpEb+<TF>jC<zd_NzJ1-@^D8>R
zy6WG7T?}5~Yj_&M-^Oh#IXvlMU+uPge{w9@7V|M5XH0NeeEZhaA72ihvA_KN;lh6_
z8LnL1n14B(ZG!!`yEXC+o1Kr!{LAAKXJ==RkH7!v>FM1&cb=VZAAkL!K#+-{&HTkH
zwOS8dSs6TCH##k9=~4e?{!NSW?-uIuU(H`}lGRwnUDK^vo6#h5o&6#s`_$brn;-o7
zwD8yb+^UoE_mr*YaWFg+JM^(H>9o($BijR4{;yoVO#kz}O$&D{-+MwMSTy#d<JQ~W
zb&7(|rn=T|7ft6=df2JtqLr2VFOaL^xznfp-+wyJRb6{J{Y#I^p0K>nL3aWU-3rUE
zJMFX8VTJ!Z-_k7YlO`8E_GYJldb{b+%)UvJUh)>adZ2K0*Ot(wyS80^ALaQw;>p)3
zt%{M~!nmgOm>5(pl6X1!t^n)7H;!zfc0X0MclJg<$-TQIG-A)LnMRMJ9N)0}F88Uc
zzkAp6<JrGoUtfQo?*C+wYj@h&S*e+sm0w<5lrqhFa=Y~NoIeM*u<i9vy>*|XDU6-@
z`jrVr58`!JzKwbOBlNnL5W~f~)3^KYe>=JP_L&>oSIxI&a6L0uSNmUdxsk@-g^k5F
z$G((#yl3gzRh4u1Smp92<G7i<Z0G)E`{a9TGHd*q#wWAiIN4jj?XB7Ew_j_%A2@S8
zs>I^N)nftw{@j|xscdt2tNcxttJQyYC)7mm4)WyhxBfC?=EbHB$KO3ZH+#<AOWX63
zT5SDd=7tB(k=c^ndVA8}m+8Cy{CmH|@$FgD_iKOr{ygjJ$0t`AT=X`d)A@7MrhDDF
z?b_K}cWy8EujX4dw{^Sq@$!gQ-vfW0GMiOVwR!f%yL<U+#S9l+e4gK@SM_(n%eeh@
zclTC*zi<JxWI6xFhQ_l+zXhC5eAlwFe9^?p{p`%l;6u9{Yquw_xp#fLv;LklJrdc<
zYzJm<e<dE7n(<Qoi25q!{oC~~T<BQ2E>p!%cIGEr-4IjpDFRGd{~US#d)nST6_cQO
z@b>ojtea<I0+c>h>&31&igx3F*uUdM{=Cyu?u&HhH<_1t)cvnG-Y3h<#&hEJ=|UOv
zvNtz+rOj{MzAbNCwIya}{#=E)9*q>vk4BaXQ>LBTbKv)Lbq6Kg)HzQjEf1+4**ASP
z!@?y~R(>qsa_FScz3f98fu|zQ)P0IL^Sk1}g*!I4SB7-mkZ8Sf$D&<$rCpx2*01Ib
zOIoM?G++NBWQU5H+PAm2ub)4E{^-%v`}=Cy*es)eH3&>%c2!}I-I{fEiRa{=(^prS
zKU(6kf62PTiYm*W6rNwdv)seCqVmIogYWKsE|6b8Wy+NJ`oCXqY)saT-gaku{``#}
z!<SVEJDE)Jdb%QTG01`qiHC!i`%TsR8@aJd;8QHunU0h48!|61^PH^ac{Zo6%-V@l
zLo-H2E$nNXtgxr||4*m&m;28@w|MoQ`v3PXUi^4tWAfgruU1uGG*lO~JelL6^6A5e
z88(%lwq{?KGSB<-<70BHyNicl;EUgm-{0Q8zCAy_)3dd0v0r!Z%55H8OOISHU;qF7
z%+Td?EQ^=9c8l%2{p0iV^S8HT3d_maB_3*V?G_VVaa4&z)8)?HyXyY)a?;Yuetmg4
zzwVbOs1$i2`b%cZo~2J`Z<n9t!*j)a?Sj`496#$mp3jeoy}qmR^D}Y1m=kxW9q*Oi
zUiS7DE4SE-8rL8vuHDJU`@X)ty*>GO-<A>$Mf-IZu0KjOnpyaB*OZ1yht{=RJwItu
zm{H_A{nhiQOkqj5u_1BiJtM`Vy{lJezr3_mSeX0Oq5KvRtyVjBRq*DaR=%eTzzmiB
zfqY&ZpgpdiZZDrLC@d>rI4v^wm5BQD)XHtAs!X%5?OgW3t>KXqmx2&aj+RmCPww!b
z{F>`UUmWyIKgDK+ikg`Ss~j(^mKS-^JJDm|G0l{stLg5oLMtuKe>?Z^(Yjx8mCH@?
zE-c&mbGyFs{Kz+xt-mjN{Pk@1mCc${y$=dHo#<aA6Y_n{>JGbm493k}kMHk$s>i0i
zC+^v`Ppmz&S1*Z-JR3W?UN3}U!#cg9HF`_|jluUhe*7_;t-SNniE_a&C6huJ;_mv~
zoZTO;denEx^+$4>`|FA}oQ*jjx_8s<7oXf!Z7poye0OE|wZ_JD_uGD--96TKQV(dw
znHG6F?7hR)C*V{karIHD{wv2<3FSAhe!q2gL8yGl?mfYN_k3z69^j5#789A6FaC1R
zv7J}1AG%*VLrP@rTm35~n@>Hx#bU1cG1@3rFnMO?{2%Xgp0rCwZWd*5s8#H-a`cp2
zbuTwD;>J<czi&<Mzxq1wy|~o0a!w6~h5O2+&loL}&$m0HnbM_m?tGutuWwp$M>(oX
zRyfaH&@zFk<(DHj$G?CXs{*}k8CG)M*{Q)7IOqA+E5E)TNq*hU5VA)1=hKNf@jiMn
znjaX&mpzg>m9DbYaN^4B1Cd65c5;;V%}!MNI(61PuX(bo{x!N4IVMLdGF)YU=TuY|
zH%-J}ZEei1#Ot;Ux>+}7K93ZPa}L@$`F`iC_X!bugPxsL6jfI<t$kXS=kj)vWsdr*
zTagUgYUgToXI3*gc`!QtUBK#ktmA@@`L@@JT@zS*w<{$_>s_ylRGVM>g`@LHpX-X^
zeI}`^!zNh?ov?dX`RAHjQS*Y8s@xBbGN=BFn=TT5XiK8;%AW=gT1?lcB!{0=DqbgD
zns%1QerKn_^Pl;1{&VUxoRiIZUU<%jL-BswFGucO?eDLC?TKVqmKx!}yI_`SG;h{=
zjV)V5XK>D3U~rZD>`J9&siKWDBNl{wQ0x!Ux0`$Z2uQY*^<?Lr&ui-5ew!M3+2Wnk
z9d~x#UKU-g4QvmRg|2*Gbmct*gRhh~TX;I-6eGbIk{>60&Yv2e(UKVxnxL0BMaxdp
zvG(ES#<kz3MrKdFrQNmND7{`!T|o0k1N*A-6N^|QH~l)OE%zXP$=OPyO=;R43@g?@
z%YUw|Xt~6r;_Dk%nHHyxFQ=m-B-0JIdVKtJa&dX5R_Bo=C9K?+&!<@&wd0>Ybwy69
z4vRo-X7uutw;60|g;%doaJbc?;rixu`g;2nEx(qYp5-0svE<U1u#k(}!{fYDZ}|DH
zo_=C?y_#Z64f`oaZmVqJ+@Eh^b?&d;xg#(&F_Uv~N2qC*;u-~326yeF8TqNVk4>uE
zYFsz_Zq%oPb(*g)85zG`wMf!0>F;?d_Jp)^Wh+%zMLTAw9J;*V33uVklQPDqLLUW~
zeU_cbZ#Tb)X^~*2xY6dxI-8j-u5Nnl)xJh*Wvaf^-7kw@^Mw{I_dor8?-O3%i0#Xi
zq>C3ea;)0E>Xbd_tLazEzLe!1S$l6@5i>(ViteA?i;r*woT%K`!nmMfuGOcARnL5U
z73D%Y)L8@X%<Mg_w%&H$&kF}yZ>yE8oj<SR)cp9Fw==J3+C3JYa=O9LK2(3Y)a;Ox
z2OH~)KDIo66@2iNX0;ygx>%!^5AS_jJT2Srw%xf(>jT$jls-+^ZoT{2Ze2^Y4Vf2&
z%>Nv=<dAy9dnd?y)!*F<HIKahn(g0O;pE}yY?*Oz^2!6bxvQ_QS)ykAIJvjwZElN`
z18awGtLT$GTs;C8E+<EAz4ShOu5X85Y^|E<Jr|3e-Ai;%gJwMh^lwWtzs{=qEc=xq
zH!Lepan3Erpr|D~nJ%CAOwvy^nQ9w)Y$>ZTn`Fpz1wF0OiRG`X1$Wo1sLM3qjq-BZ
zwnWG4S>1b$i|q66O{u=~+>2}GHrX%s`6{^v+VT}<NB?fTbz#oj>%RN9bi93#)S$d+
zvR8<Xz6nD}!v&uR`9GIERFn1AB%E<L^zT+o#p6X>3CFfglxX~Jt9bjTP@CAHR6QZa
z>GvjUWu%;Bs*qZ_$jL@C;a;BHt-7aMub4f)rZ08ViR<0FlT&M#<=-~UE^xVX&DU9q
z!Qs;1opashI!&$jd-Epbq(RC5`G3V^tTz_l<z#3|W4`^9)pnPh+(Nbm>M0BoVQ=e>
zE<ZK@?cG0>U!R<f+xymA%VOI8>KnNy4ciN6+qjvr?RahdUt*7r>E-!fcbrz=!`M^2
z${7;yj~%&Xt_57U;dJ3rn8niFb-!l$`maBI{nzQc+m<ZJus8o;z5TS>JP*#Q8N1%E
z+vQ&>CiDIL$txbJ_l~W&#-8o<*eBOoQ9<Hb5dVSI)kVJBGtPKl{qJ=7^_M@Z_HKH(
zJJ>Gr!se-yckE(W#eXXD-pcn38aur&&RQq6Y)?yn&FSp6Ln|j;)ff7?!Q$)lUV~pb
z|9>eltj{}rSAI34%F%*ge~l1^10@;i6OR-M-<^Eui2zeT(eAQS&wgq>d*@qukg=hV
zlZ_$yNK`k2!%QXyPyJB!#aEJSbIbb{x<)Dfvpd_N&U*Us<agTZmvZH}|F_)KD#CDM
z)?D@6g4LV~AMPv0`9EQhn}4^qE1X;UYi8I<0f)JE!pCoPFj(~5mMW`$eC260-+S8&
zvrc|dpCu|iTS}UN;fR}B(8`S~H-5R<o50XBZ`q9l*WQ1PX125XJ9kA@@$U`G1oHij
zA8{(O@;zO^+IJ(w<kHau5AV*slH3YEnaiK8$%*4&XYl&!t;(R{zWf)Hr~b}f#+CU#
zt-lx|%I*cM{0%}cYuIzumq$uo+cj$g6L-V@I&I~^$*1jPr~Lf*h0F1^mrld-a$~h$
zGS=&Nh_2mL8C+;(c>9F0c~Yorgb>4z13unrE914Cg&7!vwk;K8$Tge!$E(yY+UL2w
zM&p)!332DN85&+*IGm7SD%Mb*k@Z12x7qhit5bmDJQ0UY4-UICSd?WaPAi;{d#85C
zGwu1>6B(v>$uTT>xmm!sKI<lfL&1l?EnA*Tq*Qa}{=9c$k4VFT`rqg8ml{rER+)NS
zM>1d2Q~OA~r8MI_u8%DfnmV&Svidg_{mfi?gJZ(yz(UE(wI)Fp4qq-#L>}b+9g=k}
zXr*-6r4p4@@+tGJ>zWLc&O|c&`I|0z_{tBLysMtMpQcaV9jUu|jr~0qyU?qvby;Re
ziL!8bd$(@Z7S`H1`~B6m-kb-xqAXsSuUdVpHfrCyrOym=75`a^KJs)_uCG(J=+-^g
za?SMkjgD)r+RW=(PyV{O`iBUE)_?E*)Y>dh$p+KyX6y%|rcbZ?_arCcN^UIgZ>0nB
z5ua{1)<@~yGP!znXX&lxTTXvGw2lZnajCpw*DaW2zvA$tD~f@yCd}{`zM!LNDaWw-
zi5dH{yge)a=5K!1?GW1a?3wZb^PI2CFMY`{nwDDp|EZwp_gvu{+pCww{9oW3dUV;L
zg|)oPuP07iDNx_a^>N9AlgqO2?O3DA-7tM=kwfb)Uq%Pn5BJw~%P^?4vCU;PSaR5W
zNzWl8-!i$ln+~532FQMRzQND_?2YeY4$GWsvLS9e<;Xqn{i{s^vo?j=Ti>`e&-C)2
zwn#_S$#b?ZeJi&<<8P)$&%^~;(>A}}eDp@>iBG1-?7L3}8f9f3S$|hQT6&4c`(s;6
zC-7|b&_1FLN)<Yu51((F(|6&@1?F|FsW(0EFL2~dKCq#x-&jtVfn}}a-_I8^FYhv+
zU#oRA_0z9~yx!?wm#ir?nz_$+_F6@||9x*4SuV2KSn%Z)cTALG+?8ffx?t)my0EAz
z^|(cC$dc_VmE|slOuEK+=*sr99vV+G<2&r+x);v9w5};l*<tdkMS5;C<-|6g{w2k;
zNwB8s-*FT6X5}sYp{qV!G+mW7Iq6^QCGr23G7JW8hbn8A`YFZmhxQdu|LKx>D@DgA
zGFQ=V4%f#e4xP8JC!hMYLg?(=qYNSc(&f}r<f`X>xDwBKV9%PB&#D&&a3y?_>i^IC
z;zHt_-<<mw9CePWiR-xjJID3c7iIo$JO7rw5j^trR{yWxyUzFhGjn2ixPAVq2@5oj
z^cO92vQb>put;Xh0fA*-Lu*!UJ2PckP1U*9-M4l+sD&Qop2fYC;lP$$otZBcg7$>1
z;|R-?(+iI_I^gtM#5#B4+?8Jo{g=0jayR%tT$gFB$$Iy<1kZ|NUX5AT4yiVO@ou&j
zTIKg%LpRusVL@2vwSzA%yCo@wp0{8Sm@fMD#EYiXIk4f+cl-amnZIWFd4`6r@TSa|
z*xB3OeVo>wc`03Z;W7bv8Nb$wN$V{tZ?&uMJ<oPoJ5Dyzs#J9Suf3sfkJ%n)YOty}
z-g)86@yE{0W}jsamq}e(XvFX{Ie+=V`pw&dtGAS8TmfA%*sC>Dy6(=RZ*MCf|C4^p
zD7wD#{+;@7at?1Me~6fP@#@^^rt^=eh}ru`?sKSqHhcCgr<&quaH`apH915<LW=X(
z)vN#Xc+RxWJrf#dINvXpJz%ZY>XnyXPEE<~no>TMVS>ccJzwIkteL{lFm-Oq+_`0U
z0}Aehtq+Og+<$#hSt={T<0nd6yB`UCVZFL_?!*-@7VKZUF6(jV)@29u^b|L+#4%K#
z*OdBx)tu=7*BqIzb7f4Ie{M_{HD^%y^s&J%At-jwf|hxqkq6IiKKcIZjehAs{qKP~
z?={4l3tujqU!-zWyHoP%qIg%96Cv;HycpV^<*;%xNK8pt?Ig)*;Kp?J&{C)1ZLM~C
zT#O5{f=}vi)chUysqNXCr7R5dqns9K9{K&OVv(q#eb-}=P0x>;ufJ9!Uhwh7MWJ3V
zX$Q`UjUGSW<bAj&YOQ?0IA{6%RZ}!n-|kO5^Ys3;Sj+7@_U}Hbxc%DI$@AvSzglnD
zU}?QPRM?n7F4m5_q56Zr@bCId`bo)`7v$~Mb7n8uI>qSYd8NaZat>!UXGpH{V3e4)
zhUb|44g1QOH(g$D)h_Fiy*sP=!}<N}Sw64tI`F*9%g|_O^UdDByU2)XPaMCyw5BMi
zni4tuIr52BtB0qsaHHp0Hs!q~vM0V9uV3@sKYFh}pBiZFtlmU<WvF)TS-zPs=iT?4
zDZ4_~pZ}FkW~|-gi!Ikr<;HbM-(s_?oKpXnG1c2hYv-@J9h25BNl%~hQ*Y_L{l+(~
zCEK1Iikz#OdinHkudVNQPMI5cE8P2f*{@jbOzECI^FHlcwIuMQj+EhcfxdE)=(Jtj
z`+B|pp11zWnvlDC`o3Q)pJ}R}UuLy+YKm0es(V}A@2%>0j8q8L43d4Z+#r8teCyHG
zvo(#^iLR`>!0r9!t=8<;mX;@Zo1NQuZf?uHeR{gS_ml;9rS{*Sy};Km?|<!AlZmGv
zd3VPyVOYRnyS4V}7Sn?l_GRC2`)hty`t}8_wVRDKZ`@1tnAXcNH|>9JVaBU-)i2#8
z68jH_KMxf)F7MZQV_ZG|*@K?9`_@k4-zNW-+fM1P&-DA@=Ou2vi1_}@`*5&(9;d{2
z-NoLXejzIY4qm(V?a|ThU3@JQocfC{EINFlV`bhEhK~MV_hmJ|G!~z~Vv^LOQt>-X
zcIKfQvz2vLue;_SF_YuW*M$Ke=g-ec)!T4o+qbE1h@q+TWgE{68Y?gB5maWIw>M-z
z^Q+Lb(=QbN2#WuVT`L+frCH3itK~+;PA;c~SFU_nyl~z^cCl}>_ve2~TQ6s-^6}%Q
zDI%&2l}SD?T+c6!eq{W9`%liu9G^WhZA;G>zqMQM(J}R8sMf+QQ&vu4-7({NpnOnE
z>gou|%NZB$xLIl_?rhEa=;|J%BtL0JZq&);9}+cR_J2w=w9NWkIp<epj2ZvY74x6#
zY}nrU=k^=s-+f|Ea^frB`3EiO?Daj{Yi}(jzia#bKeeXEa%C@_kZLGACTX@+tH^tM
zj&R!l?U7ko|8DM4-MDe3#pTe5;?-Wy?8VF#Tb`KCn(V@%`19@2w+syYW^?v@v$4CJ
zc=Ys#hL>xPc^>Oo;mEBe=+tHPie0s3!k@Ir-JAt8!)2vzeVlM$nWOsbE|0qUOXv9;
z4j(ugm)Bb=tDxBO#PXCQr=rUKsVj_6d~Y~<t@n4?&)Cp*_W3;gY{6$!4{VvY*ZS&e
zm(`krPF<`Zr7rBJZfu>qZ`a!rKgG$b85X)|Eqm$QG9jt8D8Py1qHLJX)UN$?Qi?4i
zmL}?*0!|!?El<o}vFrLQ`PFmsxirhEpWAot_F2}E8Pl4ZbvEa7;J(vW+xFWEIi8>2
zq8efMVpmzPhLc6)o5kt=kAy-QzV!NdP1lP)c4tQGn{QXyr+s}r^I5*F^|#83Bd0AG
z+@6JnYDLzpwV&)=@$<){r5~);&DGgod&^q<_m^M7rGKlYett85MTJli_taORMt>*%
zJ|r&v@KIj;{@DwzY|ox^f4`=a2cwH+#=^uD-Kd*^bI$7b`2Se2@3+EL+b_q(&WUWS
z|2i?`!kV75g{li5%UoHuZN+DQr3P-UB|K9ax_fVI&b)hR?sGd!y$zXn7nglH=)AGI
znQ!*9+t;RaiT|GdXWxT_V_|MhuWZfM%)EHTY0}f1;@i*s&pm3$AYgLi^_`QSqEEe#
zIn-S|FIs*5jq|ts_P&Vx`T43_y@a^vqwT2|<>x(IrCF^PYNR{Uz4yrdFK?{h*t?nh
z`g69*UPNDUiN^w&Ef+j`f_Cj(eP`E8b7OIa0M!+I@u6D0ET*eDUQbKy_1@Gg`=x%}
zqCcNEJYpAe|9Ulg<)^5stG$vdDl69={Wo{cDlX7W;oYmHDlX3#>E*sQ$=dnsb)on6
z((KUHQ(QFq>r1a@zy6{6o4>6&(faA*`J2O~>q;ZV*NNOP$p|$%`2LnN!>jgow^AR=
zg4E1jolgmgQ`7#2z4kn&f0N&1zv{Ui#?sIH<I2N6^9Bd$U$5qWeE9s?S08uVufJBa
zTK&&F6}$A_0>ypL8Frk>&i_@rQOK!FOo~5h$A{zZSD3^vt$VOXt~mYhl+}Gpc`TLu
zSr|Iby5xPc|6m>ZDK7EUgzv9nI=-|U{#q03$@buVTcCPI;dix+$!d(LH~gY&*7cXH
z%hwa#tfOtZwd9Tc?@tR;uO9gDs>A%_&n=B%TkgLreRxVxoA0j8p3<+=oE`0~t8Q^H
zewZ_T_w~bT<{gZ>abs&n(D&Q=OT*V))3+1fqi%ZT_r?jEkG`7za{v9?r<1+rnJd-T
zq+PEyxO&TU-3R6CN-ZLiQ<}w$CVkPH;`3Lrrsq}36tluLAs^kXy<9SRQzC=Um|uO<
z;>Iu~>;81B-sTr`Y?6+CFrB48U2g9E51W(9J739O|7XonbL2GtrW#I<veN5uCp&x}
z-wwULC!9yOZr<+1T2ljt39C-M%PdNLwLRWDYImeEcW`j<;XS{ib2CC$PmHZB7GAt3
zPsQ%dDb2XEX0N_&w!C_uLy_U?qi!D6>2KTKgo(X5zl=%n$Bc+~hfeqV85@h67>IX-
zRD5i!UhEX3H2<gEKSl!+L5r#Bt~yC^EA@`l)Nl%3-qEyRPsoBNKg<ndE*$T_6`r;>
zG;6)azPQkBlcsq3J<GR$8dq~^p=MH4nBa^~RxeihPaa=SF4kSOi9slCh0^|q#~r#B
z9KUzl|6>+|()6Q#Q#buf3Awj2J3Gx?%P6j;t0nw=+!05O7LWaFTBm+gf2=RM@nQeH
zsoOVCEZX$Ef8NoPp=#62!j=g!bhWU!E)$Mixp4}sMPyUYWw{I0mDWqX{d&qb^LFy7
zE7C7C=Dq#?lR@m}=MU+HMN^L~snMMJVs{;b<-wyi^<S*~^bCzOV^7`ra=HHP_0-80
z^}h}abT16Zu%5lue&%}fMZF)ktrurVpWaq!FmI|rkVYV1l!?Jr@66qvLXV8KUiJuD
zpAOyMW%77W{5`F3nJ^b72ICc*my}kD$;@YRo6mY+{tad8DKpBlv~HPA+p>P{KQ6)T
zdv)i_uGo8eOOB3YK(q#nh*z9W62q5{DgPg?HZo4%eMRW%Dg_-)rj!`r4bw8i-}7#L
z-}z*2=G<Ai(%!#PP5Sb5e<tjoQYv`sfn@DAKTpO3_iH~K{Vk>WD`nxo<)-nzKekqG
z+c|Z^r7L%LAGzwy#PH}`PK;UQ+mC1Z`D1qEI$i&jRM5YEo@L#yp5u=323N)M3b_B|
z9$UF3uqSxiZ@zWmpWbwtUACQkdH#+U1rt{HF0`@Q;JLi>ma=-&v(~)7I!!It^(`3K
zswCIHTB80uy+m@lotIJnk#_b!wzZE|e&{LpnqBz+to+L<(ktT*ZS0ht7I%5s!k&y9
z?_$4c|NbM#Bj?a7`2Vfc`ze=m!vFnPvP|Iqf-Jowt{j>|E5zb6miE>dn+Ww3O}%1#
z&4(!<p2xG=;fUVr)$y*Y_7#{_ZrJ@MLxugo0UwjSyX6?NtM^`e8hP~gm4kAd?%PbN
zS#*`b?p4PWuG`ne)Q+rbl3Qi6U19aDDnBI#gY|37muw5*_;;vt$;9ZC;&aQH3|4E&
zWi@hz9cr7}qH22AHt1T=Df|9s_cCvbTI`hABEaadI_%fqEN<aN!Qy$IzaDq|F=cPa
ze6Yg)Kuq<=EVElqf0nMu`Cab0aY5DZ3-Q01jHc8)G&ar--M!6{`@!313;iZvm&-TK
z`+If6h7B7o{PkvIzG0aC;l|p73_)>mA1z*RGt?Ye6L|UF*ZCVtUOr(<VEAL5bDG=l
zPUSt*enVfA-g7nrFS3$39Fp!n`_rb#;I_{8ZPZDF?$_#jcZjWHy0D_o^=;$vdwhDy
ze`Z!``^WHcb8~;ZeS3f6yYJ1%+N;do+Fg6S@ipK4*NKUVajV&xEw-mMRUhcSlW{NS
zz_gzEZqw_luCBXTwq&+!!_Mj&{%1CftdHcgxIb>yUGcbCV3Dh3Mq+A$?yS3!o3bX9
z<i<0%Z_;5nvD5gT=JCr_1-rJ1PiLFQc5=H-81tXN_@Bmn3@vGg#FYN5ku;k%ZT-8i
zp&AU${p$|6$N5Xv2kbL+_?JIHjZ@%86eoiR&z8w?kA)c(UI|X;VCZRG`Jk#J>B+Y#
zA`A^$j$6;fn*B~WHAQ!mL2hsSgGUFjtjx`vzcP3EUi+;Zu8KX%bgL;;_wQumZ%DMh
zx<=vetC#)$yVmcSrnN!#@ADI4K@17mW%ITiT&vb_puWArHt^b;Z6B_E+;5`pfA7WC
z%3B8M&lOMG9%FuR^<rU>T*KiB_vcD7Hawko{ZhK6Eoc1+j^AH%QfGcGW%grj7U%!7
zeskFthdci?jTw7l_Aba>D#UQ)+15H~+h4!y`67dCVsF;l&B)T;llDJHkul-!hGw<-
ze07teU$5H8xBstcPa@--8#^vPKNDu&zdKf}^4qG|XVFp*^X6XzdBa9gMYdK&cgvKX
z)XKyv`y;IRH>dUQ;$i)Inu{T%<U;xNbw`db$#vIVk=WMmX~R$$xp(SZ$$sOm-I_5D
zON?wjCzx4oH(cxewb|$PrcOqNBgYEfKXGE-`Km^V;mRtd6E>;S1+&^)m)1SFWaRNL
zUjFE^yd=ehplMsfIleG%c(Z3=&{<ZKt2GR1sS%nXCqp{a=RdqExbdURKBKv=Vl_WF
z7>?)Zo-yLv)VrI%{kRK*UAU5U=%(q~j17s_rr8NgEuC`~`SxyE>zz~WTVk>Glh>-}
z>n?Qm*-INwaF>Y@ePWVcvr{LzT7FN$uR6CI$@f{@&(8Mw_v6y30#%l^QKlO61=|Dl
z)Du3IpR4|qwa|IK+snD#ZQts1<Er$W-rhKB_kCS(-j3%@XKm9~$-nzoYviK0dD-q;
zb<xZr9qKJ9twk3OY5fd+^`iKdsq<G;<G|O~#1yrXujwWqnUci#VIu$3_m`@Qrf7Av
zFmzu}Sl7P#S9p4BgJ<PMdFR&hJgt9U|5jYmn{BB7FV|!`#}fZa->yL4o9mb^Y+JbF
zwEv=<<%jBQivLdxiiu@eC>VWy4&#C!2V%P)xn$})s46fm2zt%@)U>stkLx3gz{}~E
zY7V;FRI6ua#Xa&?KRy3G$GId`{e`7x4b2vnK0MWsow%v)){~y+lS8!>W)~IR<=eaQ
z6W5i4^ZxbMm#zKuFYJJ3NMZS^nOApqy8r*BEAi))a(#&R?Y}P~UxYH`H1B%)x#;X4
z@$b%zTe5QJSH4(Yy4cTI`st@up4H_y%}&;TIJKGKS6SZIwaUJxtrd-HjX|B)nHS?*
zoVr=3+}ZSb<>%;Ej0Sq^x9>1CS2obSAChUUJpZouUCGH$KTrL9{_*)~NrnstTk9s}
zYz|y|Pj&yq&E3(VVtyH_nz~LgA+HVzd~L~_w0_;2MNb$64}2`*V~9}QZfWp(Y1XbR
zu8$W(I@B9(tZ~b>{c(~}MRgfN{29l_Cu{hnr*BOWT=yh(Wz0D@1<upvaSt?RzE!t3
ze1Asd`o!>WJG!mTug(8e>2#7ccb?5{|9vkGJl!_)c&An+LrqQAJiQ1lk=M*SKD=mU
z`gY$&_@DI6Yb&|tux>~`b1hEMcwZ`~1!T1*I3c(_cI4i^f_>#~<*nOy7H$2i9&-9e
zD;HzK%A~I^*UvCaj9tyi(>vok|CGgLD)IHR%cf7^a{n$=Uwc3{V0B-GW3*?cp!}QZ
zHYcsW1}e?}+;7zNJ?re5>V=xWD<jt}waw07qIu-;!&t2&!VE#T*>p|f*T$9^ealYI
zK4!um!+pAF=><04tS3A8{cj~buwdBpYR&fZormkXIBlit=RI$I_vdi#)2TbF&+WdI
zI#Fiz;pTYtCzbL0K7BYFK7ZM+n4BD^ytwA-hab1MKl6^;`kTN1Kz_x#Wdi(L^1NCr
zoKCoFu<L#~_v@7Nr95+cORrrucN8Nx?pt!!J-C-;bJdIT&B?bdUaikie-wNrn}4~_
z*J;_WzB(y89xK?MV_h)AY^w`{!t-zLD;Phl(qGTLqBbm(lR-sg>Du7Ko08|)%~Hz=
zcDGrwa!DfZL_vl=?gdUZj0{&^nuQpjcl(<#_4(E7H|Ku4@qa=1jx9SkOEjGNf97!i
zr2Q?+|9LSyOWnUveO-v{@_#(1Lu$AD_#S!P)%o@n-M=R}`Th0YZ@-fMY~JU+S<}7j
zPwcm}uRCP7Gp%E@mBoW|v%j>rT-f&@bmtXoEuojp4?=hUFaPBD?D54q?dWYc=U01b
zh=19yDJLj;xAxDpl56Y*^=Avte^?@8XZi2p*Wf)-3Dz6sK<Qh4vCNhUCuf*WDU^PE
zy<*vtyE9quR=>Q?@<Zg^H}<o(5f}c8|20!%NJu+$?p=OB>r?;u0IgufZDLu=ejIaI
z`H$<z8Xdhvt>sHQ+Mcq^Iulm>eoDrv0wuT0YXk51UHu<*W9zHVQ%nuNj-*~~-NNTL
zdDSD<v#mi43AfCqohcE}jMHYl#>4<BF<<h$P1l!P^0R2`?R<9UMfSFh-JfsUtUSGA
zznrQ;Z*?ClL+OV;;e{{zkGHesJhN<jYqG>SmBDGE97CI-hkW;xZ}pKm&qQxZp3eIm
z`-8)jVfoBlUHf%*?mySQHLRU|BDF4gaZ7d7!-w0?`{f(IE4{0=HM-<d&g>`uZXe1v
zyd_?9X0rxE0DFr~?f+i;rY#Ki>w-hFF2)s4eO&)-PQ8Lg*uh3`4%?l}{{F~)Y__B3
z@(SOx=UE!&eYmVuUGsu*dud|sGo6gA#CLy=SDk&{xc^tqubPj`*GN6g`z_D<zyp*5
z(>2(2eRz6VnCma8wXr>IStr)>ZQ7D2VwKuk%&i1}yVrYi3#UCldgY7q-)iZq9iG#_
zUR$qpboK2|wS@wV2{%v5el@AJmetzKAo6~vhrv^oS&sQlQKDP*`{$p|ey^eG#m#VH
z>+ho*7Pm>Qky4R;y&!h#?p4kIwoSbBF0=U&&#CU`rsB7x4SQd|@Li$L^x?s#M@>SV
zfv3DPqc26@^t{$PrK)Gjx7E5V$8*0ve&fx_aHn>LzjXa%r-en&HIIk)urt^_TJQL_
z+9$r$?8&>&)Ai@K-~Vj=;NDul?vL8%&)uj?i!nYO^!0bX#ATKPJCFXo`&Yim;m_pH
zn}6xE8Kghol9j~vKrW7x>A-5`L-Lu~zx5Z#Gd5WLIJWos%=OWCH@Y($B&_<H_48uR
zk`>*vt$OcsI8-L}b$5H`uYGs-x1Uw3*5cd$k4(w>^n8gK!va2uS68yXHY&55oqO};
zx7*pv)4%72mLJ+T=e_o-3VFF#^SK;kWq*fFySV@Kl&lQ}jjNx%X8wFP>dNZm_~oAa
zU%oSC`)9r)@TQD8zr)85yUbGSMgOXOX8triJnd`z8IIq+uT$6WeDp@Omf^#xv%>!^
z1y9$m75tfe|Lk{03+s?S(<(R=k1`$p+`9Ww<fS7n2|=3=dg#ZpPkO#4_p8a}ek1RS
zMH259&*g~OzcA&EzvR_*v+GV)TORar_dMC7$5~o1At_dZeSv54$&On(6AX_lJ=|LK
z)$nP8dGndSw*L92?uSphqCK-v#nQ{rbV~d)rHlUApAS{G?&#YQsL9yy<oc<dcB?Fz
zzuM1TbYvy-`O|lmFPZmv-d?JC`Revc&3mEZ=kv9gcBH=3Ji0;t%ORoa-KWwN6%}{h
zeltPQd0X!7Z=ar?7E;}3v~YPt@R2+A8?M&AJXbtlrd-ORdQEhH*0PuSb*?W~zFoa+
zEvMh@U6XUK)L-7Sk#{W*W9$7(jgQOI_U!njbN9Mo+35#M?Ihj*S(@z*p1S3|#r>DP
z%TJ`UuPr($!E<VFM9}q0@%dkms~|=}Z+-2*J?ZZ)b{5{Y@2Tc*xmLZqQJ?U>?`-`h
z^Qq5L;?Ccm``B9N>*+hT+}kc5xjg&srM9yL3*E1OpZ#>+zZVBAqt(CUP57+o^?v>P
zOv@kd{rUXvy|G>rDEag7<FD0cuE<Dk|M=xZ+O$tAiuae!n)0t%^hTfEW6NDJ-+I3o
zeJOsixkIqmuD-Zr;!F0m0SlYfu6=v7Tf9`3L$Ouv=>pgI&|*hU#RZS#k9?S4y4N7|
z>`Wmi4WX-vosqVd(LH-vnMLK7-Bp{naeKlMEjyLA^e;EBGJoxDXi@Ky{HGOdtTx*l
zdB8QJk3;5^@k-8D{@=Fht7)*G<@8hR?(d&|Qbf~B)#caHv$~r+pNVj8{8%K~;<R#Q
zrr)z$sl_{6!t)c-694Q?mK5C-G1Et{JyEkbd83Y*n7xqCHjBtN)ib%AGC6}Y<IYd`
zFE&R<4Y~lYGHkZDsNa-kp-Ish6Dlr>OMY&ByXxnO8ShVj`>~<_(h>iCtPBg~QoQU`
ze&-lzPUQD|VsE#|?z_V7%bbi1VLGO!)1NH}_RY!3`uDhc+2&(Ub?Tn{Oj?<FEK4{j
zIGDks>Wyvj!tISa9k-qL^7GTYTk>mu?Wv~)oGM{iS^omFX8)f4LvPKMJ(GHWUO%tg
zBH}%zd6JE@vlZvbt^kdyy0h~)uAKH(vL@w7YtiIo9~1T$cmA%~!J^pmN8yyCr{3gQ
z2D1WJSG6&DKlprdbBjo^&}%dIL+?bsc?&q5;9n#&<;gO&v#QE(w&-guT$%CEBhYC|
z(w=g$+v-A29EwXU7K0RMi$4{-Zm;)tW!K|3N}w^(8p)>%RHpCsUASa2XpA)8G5d-z
zNPNN{H_MDd)g-CgT+6oz*Y09BQ)~%7v1vho#-_;EjorIn6~DT&vfHq0jr@PdjsEXk
zkDS_H7guG`)zTimch9=IZ7VaE1q;Qw&9N$Y)!MX5tVLk%!l;^$%1QrUDX&yWWoUTp
z<@D3<x}adt1dVe#G6p{%?A4JMzG3p>=B16!$NhM7y_U-RR25X+c;C(NC9*QB>^9d<
z+4HOam;NlRf8OfY9lLMWpFMw`PTiU8KA$U`F~cS~e!1IIdu!Li!qb25hsiKAI5zJ+
zH&=)I(yVWwG2vg1-HM=bM}FZ7hJeraFYI>m*r<?a|J@<t#7ob$TW{Vx(0Y5%-o5<t
zH})mu<d;@_X!P}NFU#NgCgfNims9OqHHXws3H(dGU#~6uIOo%oX$K^(hWm?2hDm*S
zzG|MC6T{6Gr8Sq<+<*15mVqJYRf|H)3xTH#j`mOb&K2CY=l`K+o-HEPNvE&%$(qK8
zZQHzc)22-uS6|P)GUs*NYpDjt4YkKENps$E+P0)feZO!!|C|q!VfxpPZ#$v)CW4uv
z=Eyo_^Ziw3*)iwV#XYu~7JouFl3RMWnD47+VKprRpYCd~>n?N2l>XvbBE_c2IxT3W
zuVHYg<Y5QZYr4yQvbh;vJz&=ty2Q8n+0>jjT9HqFZJWMo#eti3x1wAx?0Wk>)lzcb
zO0U1SB#WMg`zy9guyJ2HL#aO#ytXUl<SX{|6%F+hKV3X5&)_WF7+e{4MCsYoSBKPZ
zAMc)VOqAv6QYO{o`#T>t+0|XyXOz7*sM_`8*HxRQF7YWcs{gI|{n2CAmuq!y%=cwg
z`e3;6PV1ZRed<d#TrfXVCH(Ei^M{*+rmpQV{u}r-MSt<4`>Ll}ojgx&da7%;?uvF&
zyj}HQN7XC(vDacwIVnx8n|CPA{wbs5>WzncuH{)VmNq>9d3>{V=*ds7Hr)_EA9rMx
z!p1d0QU3O)-<qeMKRo~F$xxxmpO%I!lKEfEsi?9(fG=u_WF+_hcN&IFB1LgJi3}QX
z%lmAmt>jh_&6W1Pb=|LCvg7u>Y3KQ_@L6Zyj4)cTEr3H~%ASZL3@-#V{~r!dg)9RL
zb&LpNIk7`6LHF{zwR;zSvz@r-vnqpu+Z&zmzgjc%cJJR4y3S%v$C=0Vriwdti}$`*
z3tFg`uqsqTmvuo$t)NWBH{|76LB(-bmf8xt>Gm<)-;nq<`??&H#Fi_c`O~8r8k91w
z*KAik&SJ(J^{B1hg59xsZ)0?pr~NtE$?x9B)%zROd|n~w7H&S5hpqm>`Nw&4)u#S7
z+2gXmxBK{d+kdU~4woi>Hs00q`FSP_pW+gRg;6zv$9i&=4Vm6oD`h?@yLm(SlL*6|
z<F*rn5`W)1)DjVT!Fg(0s3eg?Shkw*A*|i@tbTfIx-JbJvDd=)cZJkG%=wg1e)FOS
zUqj=}jH>dze||j|o@6c3S;!UX<mB|`b2X!P+qP)Q7)FO)#^l$txfyTXG_7CzA)-<%
zg44l&g|E`nz<YDu{>mz@c>k_E`}8zt$M%108)|>`oHo>A@GR%wt!w$rAn)?q!_n7e
z_HE3K+O4yk-IZ15+w-fRW3wEStgoEqV_1KuM(N4qQk@j_=a#+k)prt)e9}%#OiWbV
zEza|6$=oGa2D22uHVZkG^cP)F>NZST>&mdehp}PF^i`7svo_vSceS$nRCVe{^QT_M
zPxZ|vkxLF-^I~vFT3~BzC&;i!uCSBgK*S8Lb?q8%dHxrtd<o@TmHKYeP4(bqITKq?
znw%HdVX-0hcDFaPfAf=Qxz&vSR;>Rc$<E+poXEZF??k7k_Y>#7GR?ZQ#Iw6^@9}rL
zma)hyN^Y#XAEIu^;9-6KZb`-a#%pKX>%Bue<rx@)k~O&xZ0MS6tjr+)@y%SG1DmA}
zGDz6gGB8B%zn;(g<3<1R6ATsAMaLMgC1!2*5@O)p#dA$2cK_o!I+s1)udnv|?vv(n
zSghxL0uRF@tGjp9Zl=`rAALLDNcg~stK7H3v=|+7;$CmRxVZk$aVHK%`wmc$O#jTq
zkYM%2Kq9J_^?~dxsXs3syKZ}W=a|6TiTs(DWqo)U6wKPH)t^tcUvW;I$ziQl(;3+r
zJp!|uYG1Lx%e%$;|7&0d->fO3*Yvd+0xmk<uU!%HIiB@xg1IB_=9queg(fXOw)V!`
z+D8Yz^=dEcmyO`%Vfgvq?fUEo6Du3P`?qzxn!fez`;6CD_HXW;!ua9e$8HI`$0^F6
zgBY0pzqPNwel&t_LWtbvls!wIEI!}<apCi8pYJUB7~8|hFzHku_nfbcD{92&)lb^D
zXqM82VqJ#zk86!ZUwrk^HFleJN<KbHa$U=U(A8{`*{z%0mfipVSfOP?jMHPsZavlV
z&wF>3YBEMRa=ZU^D-GT4fBO2~KR;y5lH>l|&d?Jy@cM10>1LX`o6F&L+ao>u&YlUq
zK9@CqNL*E3x?>l|A)%Z|F$Vdc0YCM0Uxuzy5Mt7aYi7(S64EsB3SE;Fdh)`mNwIs=
z-u~i$xx2FKwO4)Xsq)b2OQ$s!TczwgVjcZz`ODR|j5hnK&fLxYwz8i+De6it<}#?S
z$9XmM-o-LLnDn2s`L>VNg7g*QkCxtT&HQ#S`|a{=35+Ke>&**d$cVaawEMyQ);QJ!
zW-FrOW<7OLb>g@vdCIYS)7yEPZnZqGrl&A;__jL#mNMh?iWHi4wyt^p##K6VE_ek+
zwd`1NrB(42=UtVXX^S$a$S5#{1Z)&wI{$9zyPlsL>zNgIO{>gR>(FgK7gczvK8NK;
z$oa_+C*Q8OnyBG>>2r4W^S(Q!zBjASUDNiP<hu6P()2}#z-y)~*Zp3S@F9TxkEKq+
zywf`KkM3U*&JmK|WHx^acTP?c^R%=57wk?SJI40#A%n>7ZEZVD^D|B{#B4G<^k(|{
z@J|dseq5Qe?=7dI%KiY|eHxoh9$kMoeJyX0$}-uFpcPXmE*58B71Ynr*3^B}{CARi
z<#Ic(C&m9NT?LmvgACSi_kad#F0SqqN%G<3mVTyP+x6c0yH$zIF6|wUP6;~}SD713
z>zZV2d+KD$FSFcjZ9o24Z|i>6@#)B+i>vjQEblC@?Q=B<4bQhJ_;KRx29tJv`EL@J
zSs1PytlsmUslhASpJU(mU#CB=INThs9`LbIAZSB?SwM;*`qHSpf}YmQW${(pYL$Iv
zM=3E(DE_T{g`q-p-@ZVF8JC0@JUpbMqQIjt%ugYsFsHvx-NSj(zUI&Ef_dx*4A>6!
z+s}N-X}Uhw^}#(+hL%3&s1PF-hJ^;7e?K^AxBY&fZPSjJ-+}e<-)~;L@5Z6<=UedG
z6y?ez^XkuW%Cegn{QpuM|5tCXY5Wg8>mO52XP9Md{eS!XqbZK84k}3u4ze-@&wYP6
zJ7uWru3S+*MPQNjlxDG5tvJqQg{5cL&$$0sm;dsaloN9py=I)sG|u2X&1tf<>};;*
zB1_*RA*>8nwkLnxUv@C+t4k`QL+ztIkG`?{N@>0L-(4py%wS-gnEu^-#i38j^)>ci
z;80}koo8^l#^&ACb*_(<4loOR&B#~2R??AmKg9f9_9vUEIy2c9^wr67GgK6RkK0k^
zH;JL)!i68N&Yhms6P)z%^OUFDM)B_+e#~87@%2cfuHF3)u@U<=rlkq3x7u>~y`;+3
zUA(Iq8^ZJAwwMOY_Sd>+SACFi$Fq8Cu@zSqO-_@W`a7R{{nED<@}G6M7_KlX`rrS5
zp}$SQsgB|40@vvR46Dqh{z*E>U|^E2tIF`>gof#XDVO~YiC$T?qL%khOkB0RZeSFH
z!@8XdH@;c1kZm=a=W(kSnZHt2S-r@Y)_j|{VsD3I<;OkGoT`<V70T}A{OZae7ZVl}
zko5d$$lJGmnEM-_{@p)!g8ZSAN$+HCmV15f{VsUkTTwAw|L2=qKm4~_y`A<;lfmGS
zoV5GaD|^x(&3DZTIx{2vq7j3D7q0}vEB4pcJaf(RYy6kj8JlLlo)r1`ZFc|O<pvMr
z{(OGln83<VQ10x^Xp$3kTcTs%yPL`X3>g}fRv&12nlmeN-!1#fxC5V;xG^uw{<hx!
z`}+^~E{Dt7?#hn(t0(u+YuPN&!9A^e_dcH;RqvO@%<!sVq2Jyxd5fZi$7lTZ=U*||
zv?qV{F5zz{H|-Jvts7~3&i!3s;UqORrpdE--Kt(%O=xmgiLf}aKu6)siytBEzfRs&
z-f-cp!sSQ06;;9i_Dxz6cd|$MWMJ3Ig@261JGZ@`92{@4Uf{Z*xt1!!gg;T1*WFFi
z7yMfuJjJ5)LFLZsTOwSaPl31e<s4gl(6OsDG33wbYptr^%BG%ma&ju#epAu8E$8;O
zw@*(`SM{25;_kF=jhhoc>Rv9-{2#OU+~3_L@2)M2EY)OyY^M91_iJ&_hT{)cf7Uu0
zKmUn|&u0CJ3#$5gPrGaM{k6OISg*ls@^$|eRku^O=>O>WlD_lLzN%vZ?t71`zbc<O
zu`a--;IHY~KjplYN4Bup+RaSb|DqtE`f#=PvAz#H*Xv$nn;h`nTU=}qT>4;1qwKA1
zD>6g7T9q=c+vj<I%bu^CVX#MkcV>7%>G6Ne)#W+=ChyG@+ubrLClqrH)A=(Un#o5x
z8q@W!zTa0@l}5N9&ZY(0Fjv{`1@6vos-BYKW|?tuQAclQdqCEzyJ~TQ(!%S7yv`bz
z^@gh)IrT(qYCzPh<C`5Ko!0p1wO_b!=Sqa2^lGh-k(0S1*Z8PSKfGbe&W#ab(%iw9
z&#(Ub<h2I7s>=2Yw|rdh-*)+z`{;Y^r3Z)OKHaS6W^l4SF{S22`ti%I$1aFnT~Z-5
zd)Dmq|GEqtH?F*4F<np1)NJaqt*U0GXOpf5ZM+gPXI_byZ|2l0r;c?OD*`?xuh`<p
zq1ZAZ-`!H<r(BS*n{jBw`G5&;`E*-AUD%roqHKPq`OfloyO9}h=@fkN(8NV88b>Sg
zqYA;xfGk}tHGZar>zz_wSHH)<XH~)Z9A(fIhIWf(rquLr*PK!U>MuX}eX$|{)Dae#
zbbp2XmIVgJNt+`4o)&BUbM=ZlJ}G!>POIP3n@7x7o!z<d)1IlUx6cGAFijE$ZFxCA
z-6-`7>)f0hK}ijd56c%j7G4qe-p9T?;6&&unK>F$JdU#Fe=Un#@qgz8_4mBIzGt<r
zWZYO`l@xz(zno0$qJ7zmKke%46>!?e_;dkl)7G{3{!FWix91Po6V_uB=8*Q^Aa=Ir
zstpdIW{(m-@#iw>3;DcUmmhDG+>`ly+N;a%lfEBKnR98$!ju>OhrV22om~5+*KcmU
z(O2p4yXr2_enxk*Uf*<w^K)_&TQ&3cE0c`k?Tzmj=$rAs-}I?!Zk((z|GZu6J^lzB
zm!4uO#++f3EdO*t%Y;L%Ssz)w??hic`nqef<I4A2G-dxk*!IX{f|5qi)I*F6;S;Vs
z_VMKY^mn`30;OX?_J5_*yyTgGPx3hw9eTkz^_G3ig@v1!WZjbVmGXA`qawn@WjX&|
zeQHcxzMu8Q@9zFx+P2PY47WEtpCc;$?TzJz#DfZF3vV-S{w{N4bM5C@nR&Sk440U`
zNzUE9W6kqZ*(*QWTt6SXtG03Rz7q#dtV#W@mB8@i(=_)vPScuo<_b8~xiJ2gj^1Ot
zcb<{H%j9Gu(buo0iZLG88}RgF3WHb2jZo*WU-w^euzPGS=a(1gC?LId*YB^Bjg#Sf
z!Po-M#+{GrI`LYQ+w}jphAa20UYTpDHgv2_ZaLw?(4@DuHK2&SZ%M^b#)MyGQ?_hQ
zW0VLNDm4kHQh)w!e~(D%n!_RW{f)EMHifkHo3QI@UtgJ<&b=UR@ipJodT)$AZ+*vf
z#w+MY<K2@yw^ydIavms&eir=q;p<DGUT?}Q{&t+ddwyA$@Z?$himc5Sm7R>qohIO9
z2ij1^!o0%2J*8CHQ@(%M6TSV<k5;j+Fq$!C#W${C)}ytzH%D!qbY-H$v~T%qBU^pz
z7DdeXnE%|rwbmr;{H?GEu|4lSeVMo#?mwPe^m1<+cfhd)4|=aFv&JMftSYWQ{$;P$
z(^DaD!&ge`?>V)0&O$y71LyUO3Yx#>*{^^0GbQ7Y$n0BJw;4~Lw&sS_^wo@k>DSJD
zm^_=~<yu#clhIfBqZC|hJ#3ZI-q*a{@;Us8lW>UXNe}0zo`1Mi86swK9C^Ta;^{3<
zZ`dZW_2E<3s-#Yu{PVW|PVxC7XI*_xYcUybbXfVoS#G6~ebm`mTa6Q(<W|Jh7^=9%
zU)ac0{Et!nbj^~k2W2x}xkf!UTpuEywB>XAER$(G>&`K0q$(L0KT>?&FXwIdt8li^
zSq__}zZ{hqLiEpT-P0;RR#S5Bqe1BAIlT*1pCn&;xBT0_snh<Rifk)u-=1$Crmm*4
zL|#Z*x|`8Ku{*TDi6LR>u^``NV!v(v$Y$pzD|S_WdQrN_D=8)9-Yu_Rg<|U6mH*zv
z&f#*JeCb-)vo=Zo0#1LqeFAkw2F8AhlbxrnSlg$5>~u<iv&P>?J(@>PtDm=9;-Jw|
zcXCtMv-xkm&!3EQbKUPfg}Zgqq}0>VGqSpWUYx$Ibk-i;1DDM=c5yL@oXCu9nXss_
z=mJxdz1Xj>ox2S6E#sN@)>oyAJQ3Y^%AQ~A9^=%vU)yeQG8%BbdsTdIS<{9k91{wC
z4@52N6?qL=!lS?S{<d75m*LirSud<rm47pRlb#drffG?*(%lUiroUR5cRXk!>$0VL
zFKk@4!pW1-Lhp~Gm&%203vaCK+OWou`GaZCjjpY(Znq{M?YkcQthM0Uwnu`Q;U>_1
zZaVyx&rGIYOn1K`&ba6OOA+7B|7-G3-8`?CpS$Meq32UsGaGa?MR?X`rA&UEs<v%q
zdVNvV=9lW1t(;kTug$x4;glECk+%|W9to~vG049@anr@?=PLES^MrL-^~)X1k1Y3E
zXE`^~I@j)ZxpT21!-;wC=G-sNyyeVrtLJzAKM(a}h6jJ9ZI#{c&F0uFJAK=qoH?!w
zm0upTYqvYS<^Brhwr#~1juaadc=xV3*0^tfd0yQ7`+qA-e+h28xM6K~{qLJGi+-BD
zx-^e-v6y>9{*4X4wr2A)@<hG**}ggV(&4>t-F2_e*<3s4b6LY$!`G({#urs?&{S+;
zSP)gSP*`}T)z$FXBCC>u-Q_|u)Y|6nUr}*{@qyp{(CaHzbXKlB=Naj(`QK)i_xu-+
zpCnyyVqsVt_44~_U7rrW*d3}b`;|Xkmtk1excDDKF@wRyv<iV~XR53#vZF6HEUOar
z;$m<(nVq-x(xj$Mx2!dSS8wxp&s?q@wDje!l@alGoLW}6uk7CQbXN4Q%d<a4R;Qfi
z*ECXJur<eZHT$dSYz(DGZtMS*(V8mE&=6DG%GNw-kJ7t6PcH4>`-Y)m<ukbrOQ%j{
zVK_1^uUMT)V#~LZ;?Nh<&i=1p(`MB0Ul;%UtwGtZFaLO587j_YN9<13w{ZAc(%Jml
ze!1-3z54B~t#9IuBo6E}6>i(f&dl&6|8OY70=~LGQPW&S8W!fB*A8Dd@8b+6hNiH0
zcGK6#?JQD!U_X1!vp4&9?%SLCzVn|?_ttxkiHV5|#reLaN4$IWtTA`losOrBCl>Fq
z?)_!$6mb9U`)kv5+nf0Q@+SQLA~`)dq%CxJ{KuP@W(hLP*mLvstE<}oX2)tUs<u24
zJmttee}+e@2NMH_uj~F98T>}i4sq!SDTP)yiz;tlp7F^hv?*)m?woHYr)Wl}B;MLR
z`@qr0RG*gGRhu8|sP|juruEUxZL3|x<fTSi-?lI;lG8eDw?ee2^uyIFI@uos`GfA>
zNo8Mr#o<`T)x~z2k1u_CynOpp<?qYae44s`@5-0=Q_rY0OzN*$J@>{sudw|dPYz$Y
z{zGJ=x0{q-o~GYxt5kPh(-#>NzWO&@OrJeXl%Zz9{-$ebN|l~(Oj1}FK*x8*`rAvD
z9i1)s?ZwqQn|%Jxi{_KLq>{UB@ALQ_d&@%p|GT_BCgQ;w{)X_k_o_c0#d%Jiz6Haj
zu;gcDc`t5M-`W2*XVc61sY{O?yA-5$%e+04GkN94jVo6cin$gYz0B+UkJX^)@S3o&
zHin4HQkT!=u3%$()sT4FPm3Y+5j#UenM`A%sCBv<gWaQ>){+q$H?G|HELYvkXrfoI
zZ?7h+YRePRRY%3${``AzrOL00;qs!D8TUO^8_c%0IDI%+K7~PIVc8|8SL}MZ$E^OZ
z?@tOg-zZ?9dHR+%3v?shvMIV^nbsj^PHhNUk>d6B(_hP%34cnaRtUf4_Nq0_&2X$~
z+NSVh>eJa%UNzr4*_O1-O*QbZ*Q`<p)>zrEd;HT3g&tfldo;=6(BzQs8d1yUJ<(lQ
z`dM}E*X8qlIAeDH^?Uzo>k^(Pg6`MuZ<e**xk!yYWTlDc>{Q;}TPpqBVweBh{`krX
zjt|%K?wj#^VBfIk{sC@YABKkgg}O1U4dHM5YW}w`-MPVMub%g|lAl>BmXV?Mh?DP(
zUb4>wpL{p}&CE46rD`={>AN>=`zxCpA6a&-WbwZGKT2)qKh-Wd_X@O_UFwu0_tF_2
zPY*Avx~giRu|9H+$AvXQ+%r%8e%xt&;)?&F<16Gek6v{ZklOh5j3h%B^jJBe045~{
zt*v$wN?kul8!XD7cl<!R%SVQD?gh`Lx^my@I@)k!&RQ;)x|(@g_r~qs#8eY-?z^T!
z_SeaaeD7qPD48>dyY#@lV*c!%w~n>PS{_~fKQsLGWnp9A)#+YWe=S{juPM9h!-U(*
z9%^w&or$o!;ACB2d*DK%($XMK)l&{rch|Qpe&3c_I%Dl?6PbDc9c2riM>^^5p1*AU
zH<k}SUKy5ezq!2Plqsj;E=S7@#iR1Omd<&pz{K)aYSZ%foLU<y>f(PsJ+x&KvkSxV
z)0PYgQ|~-Gpz~b%WyG1}ryn&E^|Qrf^#A`*bmo3I-}kFRu<jlI(27a^Jgc|QzV_k$
z((5AX&*UVZTPc6K!`ArLrQK_izMbcSNY43&LY|Ur4qJb%m74V`CvmILvt={)`&QNE
zzP&SX_d3n$^Ea!GdTv?~YJbXwQ}Jj2v1YNbhYL@o%P`Ccxi<Sw1jh>IyyqERJ09(|
z5@b*?NnKs{EyKHdTN>ky>KEsPeP^9X*>~Rf*`ZWDp^Y(HbEexE%~lth#NK1oD+kJX
zb`=j!oV<H-Zu>T~zpo}UFnoO-9>x^Fe!p7i|I1{^v1Pj(vsN-6nx|S@<H5qv#1?v5
zYCr$o*B1(3tb5R<?ay-d<=q1jT8k=cTLk7VklFGfX#S4XObmkA@zX;0g#D~rYI1hU
zGzJH!ps05C(|v`%H12N!@4HL-{q*15^R?Gsd92d6^Hxe@SYcQ2a;IAI5xv)H7pJeD
zmODND-n5fzmmk!ru9BR*Cv;Wa?nEB$2lb!R=O?}TniKXj`@BcRtMx1i$;)DAe7>J3
zJx|Mse|OJq%N?f|G=%SzPH+9iH%q@}o!yC=KeZ(y3~mXzO1`{zb~80xD0JxdYGY6k
zNwfU!-+yg|ddm~TU6tY$H=k!+XAYjf)VAvVx$rcdE34GCxEWpvPY)G;m*sD^QY-F4
zj0P*ivfSj8KI{`8ZT%FWBvJcq{lB2^3opI=y01k3(z>&nf>*qDcb?qJ`}t+!a`DUF
z*<q%lS;y<AXeu)Vtk$}{N9lLAZ6NpOGpSyV`>T{1`1h^zT`gH#Rv7>H(t)|rtBSU*
zS-tnd)wmlcr^&uvb$^YBu%vIQ%cs9Bi{|8%1>`L_{(jc_FIC@G&b+EMZ|2_p?>5hT
zE|Iu@pKMg=>v(2{^|#(yReiq9BFf-;=<?jlxAc0~JifPBt>K0D?KVDHtHgi*^!41Q
zX`btE3sBu(caX6`j=@9A{Ibt3eo)EcRCQ|hi|yIk4Ab}5tyK#<IjNH2)r;#J85+#X
z-?wHzSK(xEIGVl7^}T&Hzrr`qAKsE{4|;Q)6gH9D@<4E5U1^|cZo10lT}$unZMbq`
zjX7^f^}R6Pf1VtD*556<ZO$y%QxR8u@3UBXNTkV{eH;gxPj_*9DS(c~S`n$q$e?X~
z(#krH)s3U!@wTd^OdD1ytx-s2xN)mn$9YH2HZP$YQ!mxDC9XNm^&x_H!P2#A>8Yzi
z(*%z(u1H>HyF5x%F)5@*NuK`&v)a|8++Hstr;47qI)S6%*Q30Pr~Xb4Ui_6cFZZi<
z$A>@LVvOI;y_hckXVv<f*OpCdesX!Hg8UTqeTAC?oGuk^n(_H~fA+p_v1K3JvR7M)
zsc!o7AaqsWaznG#*IKoYTVEGXFFmyL=>D9oH{RdckmdDc=Emmf%riEpx2T?;CTp|s
z*13s-4|c9zZDk&rnw%^wC&!oY<QS{Hw6VjY`_oLXuhT4e8}&JUPIcOL8)1>e@FoTa
z*@Abc9!srVozbV}diU~~P5xD1-Po!%ec0B<man{fBB9jbzo)cKc;5;&-*2}~%3qqc
zo=CkW5+3cKXqv#VLVnGAgD;Whhu!DZ-IBYky7}0DS<SoB7MqPE>J$DyNlg=8dwt!F
zkNw|Exi!Cb3ON<Ye{B}i>9O0Lw&LN9WmPZDErql9Ml$m*PY=$k`!$PM;JZty`RVq?
zuChs*wM&&{zqHJ;x}aa_Ib)mm;lsv9?XC7!Hw!XxuAjPk`*IddMg_C5wJ$ID)Xgtt
z%8HA8Vt8pm-mR`Ly|<;-<f;E!+ka)}9y9i}S3~NTPF|6p-dtC9>%8UDEblO@PKMG6
zX=~IaR&Q!L@mjN8X-ndaDdN4abYmYZiVr&{YpFO}c7^S|l~ZpR{Z?F6o~K=3A!e_+
zOiu2{W<TZQC*>Mv<yT%^*!XtVwb>i=XNOP8{MuI4v%!7!uBDr^Kin^@jWStvec$~v
z?pJrOw=LW`Jyw(9kdg1Vs{D7Ei))oz+S|ETeUGSo@MlNbtiSR+TG!TbF*Jo;J)O2!
zBK^|i$KSqvWz)A@_VZfpoYl2&6&dwY9i`nYN~DB&<Qy*iS2NkP-Ck^-%9e~x7xT~0
zILCMA>esuou54X@Ipxo-&CJg$Vxs<ZYch0L|JfI}d4Xp6+ba)W%zCGOZcW*0P6t`J
zJ841r(koqe|J=*qAiJULXrJWmTI<N#*Dqw8I8F+-ov%vW;k0;3XXk+>Y0VSkcJaBG
z-?Do7eYKWFt;06`;CO|_=hw-t$Y)Wp2|V*5{N^mffRFr1KhD2j)8^(=s?6q4Gw0Xj
zbz#QBj-kqT`huRjwxy@)ZJ6@lZS-5$vpy_kOC<#v`s#nZy6n-_k88uC#>Cya;`*zW
zU#@%g>B)Cl_O>g3U+jRK=JoNFUbl^S$+n9BaeNDxgotsUDCXSw{;5{V*A+_7o>nJz
z8ZFy->+XsPsz**gX$XjV^|?)Sqv1yZFW;@Vk7y?)KMHkPS$S*S$4xh8R9NzO&y-HT
z`80YD`XFD{$JUu0rRz@BEtJr`((H4xeb1S1$!<ZQZGT@E=RdxZ|9Q=^zcy}v|2)o~
zyS&Ou^M>5~d!=3B>vQwx-{m~%sWr{i%-sCkv%+gPOm-hX@=Q!@r@$liUa#J>OK-9&
zo0*!Q{}=7ydp5IP#mOUXvCNhWiFaaDdA&H3?ibGyj27bHJGW25x0sXTC*!B*;m^)~
z^)Oq1_f7iWy`R6o{`znIPS<bs#$R8Y*Hmn&Vg9u^@`Qm~PV=TU5*p!Q91b7NO7Es`
zp14G<Cyi<CyoCpZPNcaBDn8McW6iOzd@EIO_PvTi#)i~2kDnhc3UykPw*K^s@QX$_
zg1##%wmcCy>#3FU!{yzz>fB2!y#m(S{dwu;?mNr-+Aa^1<{d4|7PTt#hL`Iwbe|Ir
z;t0*Wxt&phAz_v7t*w7P_x3INd+l1O%%Yj1CI9P%rEcCnZ}a29=Q0k(mYTj_j@+_!
z&FP=`YLB^eh)421-BT1OSNQj__0m-Z{?^Ja;VRd3x3}G$bn?jrkIi%RL+%ykc%1mq
z&&T<)UrVv&3CG`^I#W_KBYleeznisA@^dQU)}C6o@_?S$<tv3w96#k+_ld1vzI^(Z
zXZwsS^S8ySsq8;-&vx4J{TD8uJ~_|!zT0G#%1Y78Ur%$KbPZo0SNr{)Z0!dHmGBLp
zpPjv2%46@MB2+ooMoL;RV4EL*<Ai^DHR{I7$3o6XEdQ(iY~mcV%P034-Cj6rgJ0#g
zMU3~3XMO+3%pjPodA6d<u)Ae_j=0w*)&Byuf9uc9QF^%l|G(9Ni{0Lt#jOld_4hwt
z`}^C>S+lOKkGC(E)~~Q=)p1>T=D_BvL;d#I#T)&b-yfZ}+2Chm|AQ`dMMZg)RNpWA
z_xJAMYxb@XEqfhz<nr?M_tYe(7+pTU=*7{!Jp8-M?H^@cS*_xBe~rtu-^bF6r~22O
zng4%E_oB(_{@-3*_4fDA|NHCfRPFF<r4}tBRjqZsRSd<?&O~lVa6H+?_q9zf@$07F
z#$KC_N^F$={VM(3{rPOF=ea|Tf4%v#Zqlon+mAiFB3=`;&E+G5@yh*M{FVtVeY7>{
ziDBx#qcKyKs2yd=f1^<?zBBdoG)3pOCr4H1*;HOy7rXn!i4*th|JVNe^ONWE)VJrj
zH!k_@EK`-O#+m7JvpBQh?W9@jceaQ>x7qF&w{iZ~xm6d76Uw^N>jV*-9?$63M+;ZV
zv58le)b5Xb#~@|4rta-6XAY@9k9V$`?0Za&_0DhW)BhxPyq*=5aCw2(kFOjb_>`xu
zm$-k>!aY22&YXKWzfSg?Rgv<)w>{HioAz=sUO73r$!fkvrlwy%eagD9pb>QTyoLtX
zW5?&`=g*%pV@KB2RkN*s{j;gC|Jqh(EE>dU_#*+d_bO{8M*x$6&Wq*wCz)jCuW4)S
zU}OlpuxRnW-L@x>3;+E8)}QHs$Bmb@Uvwi2wurqf4_tPnwW4I|-tL&aRi$#O3mCLQ
zSBXeSym<3wPtwt@>H6_`cXkA-Xf0Z;${|_Y#<*&o&Azp{s?L4P5oc2t!L~~liv<V0
zx8A1G#cyxE`uO{Qd$zePQ2)~xVrPG-qNb>Df8=k8ZI|bDduK};e_S}v_xIV^w~{xU
z$j*;Gdu;Zn4%W55DnGq4GcH^EDqOtdw*9`hP5*r}b;V<6zEb)3EA)u0IBRLl5vPA&
z>X~<~<qr2MuIKPPXq10%PwMGuq3dFPo|$PJx;pGCD9H%AE~(m7^z_uj!|li2ezm`o
zTk>p$%AdLO+*dQ6m@<LUcUHG<<en-G`NxXVdtJ?LNB?u2oyuipBAn?TtfU$Ex3S88
z|8%d370KS;k6bSo*NrmCyR*Y)Z`jLAOaK1;8ygb?IxKH(^!6i1j@ay-(`WcpSGR|8
zTI$U$D*o$Z*2dT9RrqK9`1?|5b^Nmbb@}@Jn)U*b_u40lY%4m($e?`ff86OAi~)=M
z{QDpJ9&q=WS)w4uQ1JZ$n}3|{2HC&q`*$%gOukpYZXKK3#^giyZoe-o5<TlbZR)J4
zThE;R>&mS#)yp*LNJr}FX;;_B+iPfa2n9G51qz;YWX!p<V`KjPefjUJ+hrwAuC|Ze
zHus6gvAZpqQN6A?uX-4!rZN5dta0^y9M{$5%O}1+t{I${_geV>pJP3e#%br~%;XO@
zF)*+wdUB#eQ2Ec_zqhw$e}8p#b<K_y=W2chhQRl2zW={C9lmc<o<sZm;fk7XE9}>&
z-SEqOHQm!bV!LxS!=2{(rg<k6`z?N)w`Az6l{tIKFX7Y`$rl=IAMW3CaOGPU<fEdh
z`t{At;_vV7&b2D#;@~haHvavqm_yMrOeE{06jNiu?{9C7X1=*$=pW`EwfT5ftb0g@
z;jgAoO8371ap!)pI)0VhiKE|yVp&6%tO#AXz3gj$by4M=9fkG(|CaBIot_$5UjF^l
z)6?hYT5m6XeeHOk?CQ9^Rr_N1E(lUl(J-n#-E}lTW6RxtYp(p5Jk@dY%FQ*=%8`#!
zj3Spg>aK~<30b;b(ah{$#*{#_#1l<_`<W7?|HQ=23ACL)WqbBnjz5)K&rdxe_Eo;>
zQx*en_O#PqTvqxvi+%VUIM3>HMc8wxpl99Tnqv3%R9;@}-ap$spHJE>=kc*#r>LJ(
zC)^2JaxcVbVZr}@f9G14+m+|-4)Bk<o4RqGveud(B6*8`Ki8gMr0UABV9~<)_HE(C
z+rRTx|Nn4jf}-=k-|zST;ZHa>$8y@VY07Rr1w}=lzI-VuC^+$7f@@Msvaq$z-eqSM
zBrk5byZcS}x1X`b8&@9d_+Qpre(Ipa=a`HqGyk{gUEaR5-a6yxyQ=<cz47I&3^TH_
z&%en{{kXNip=<m1zrl<q8CkzCzEL{7iC4Pg&w5R!KZoU}cCauwYDb^$_uIH8mi<Gc
z)q`i7TcpZV1NOMOxs|=V^z`QD^y25|bl<E#E)bL;IyY)hTZE2X*_(*TYQFPg<K$m%
z4!HhMM6+k&fgaX|^odhi>g?w|KWyWEyhZl?)z#OR`OYpeb*+7IVWGO;oB$1xbLY;f
z`^~xF$-~-qvqAT|%`uaUHJ6{xkK3BPP4;CPd%`=p`^}bK4#mqSo|?P-&Zocg)5-(f
zZY-|fZPi!X9?q;>H1lcw^Ap#F0~k(;oDDh|wAcTa_!k~tNrnkc(=(rMeqDdZ^t2aG
zu|V=h0U4P$FJAok^YgRsY_mWnFOPegS(X|M%*@RC`s?S{|NFV|{w8%ss~0I+t8STy
zbTZZi<n=6gxb6P?e-}@#&wEo`R(5Z1b@{Ot$=DY+Ha`CO`FU?oPu<_I*L`Q3y)>A_
z)KL24?RrpG`^d4mcIn%8Ro;2K>BQ#nw?^@`Y46OGbS|fyY@5hXadqKb_wzaPOtpe;
z{5`P8&Qi8rBW2$vtGnW65y}h<`ugkpWUbZI)#a_r&YV5#d;Ot6&=k>wG9nJ0LaH@2
zHE-U$QSv+A$@S?%sf8<Jvfi7>sjOFi`)j<=`*!!x-n9KYcI_;EeQn;nd8~=0OMZQQ
z&CbRawl1df(UH#L=jX1LTI>qV+wq)<;Z^(Y`uTs>pGdt{5*|I*Bxu>s39RRSF_fK|
zwdYd2VfyhUS&tU&o|nzIIra23yV_q85)v#dEG6&mM1tINMD(nTh{MXDrRU~ahcES-
z%I>c2e*ebu*9BbGX+`dx5**(qSF6|m&6%{+ecykZy&HA~E_OTKFF)Ob>(rffTA{00
z*x0VEkFWpv>8Y|?Pk>hU`=IEytGjb%UkUqMdjH(oec3O6#ALeMDt+}o%9x3vCnI8Y
zeiU2$(fY5vv%03NpBv<(8?hnb&W=J+tq=|#o}3#S7J}-^7uuUFH5!VZp8EOg*R<_h
z_LP6vZk~9&RMt<?>_q7OxF+lAldlJg{;Rg#z2)7-o35p=u2|Oo+Ojb5pj1VATAG-E
zzye9R3!Br=FIl1z>%PQzTj`e7_U|M%e|`C#ulnCdw;4g9*$s>bw9WIsRDIz+f5lvj
zpRGaX>txBdD?L4J8nVpN(o}U#+Jo%+dM7VuVhBFFC}rdAXU<E5{(ifi?>$XtW%&BI
zfPjFlSyxMB1r?7v1)g(c1f4f>{`&SUkuR;Q+xJyhgk66q;<eXGdW)WRg}{_g`4XxN
z?!P>`^)0{uxixWnf8E@i4l?L<!6t>NUj4GxW;r)D6hA+=HUIv<v$M^2^?8Wz5SOdZ
zZh!Y-)!q6EmRGALYRrH8x-z$INByru*&8kI1<o|xTz-vNpysEe)3e8(7q2%>WjGbC
z6~8X3lp%oKrsCs8P5nG}W`^K>J}VzhZ9bK<DQ<7o+Z!7fJGb*aKR@5x-28f}ON)rO
zs|`DALUQus>UVQCZcKc!?%&3K{a03VoQ~b}yuWjgf{{k5XNKGS&I3+c>L#?5ai3fm
zzMb!$wb4SSqh0&<?PIO%&7PKOYh&~5+}!Hd*VbNJ8}07m^1^CS%`w)$`CaZVuI2Lk
zRp)+Xyc2JH$M{e5?v~<(g8$A=<=nsfy2Qqpe?F!%EZ~#-czvP_{~<B+0y&lj?tOx5
zPn?LEyhuK)>~2Y3RAA7~eQA%bY~4N6TysJzdpyH}>zil$7E0Ag2@7A|S6eMFKmTAe
z`+SQ+C#|WktdyL(CN7%a%(NnO^|bRdjfFOTxb`jn(c*Qlt>!o_PA&Faze*=%)q<2A
zx42FJTVB8T`@HL#?~bx7Uw`0golzZ^$nG{xJN(@J^XqGVe%i(N{PnZ5v$tnnUiSR_
zd}}MKxZP#G*B?IFwMDes?zG7xXZ{^)xwDPa9fP@xpW9!{7W4SE`usBS`R0n3@;0wj
z|5Fm#EMq<?SHj`WI?2aMH~T)fTTWSZCvfwP$*0QC{c7;PeL}Cti1(JYG1I)yR+18P
z%X)T~ePLjjc>0l;n9zcl<v;ZpYW{!o%b(J}Z{J~m#)NkbZZl{9K4ia*qvFepiHjFM
zK0Dhy{p_r*rLV)HTIWru3DYSGVBnQ9`CwJ*bz((c&g+>!w0y2V6k*bp_6AKLI%vIl
zzvBMdl`fy-_1SHeKO8jget%}_oND=p<<;jEf`f(a<L`S-)p}_)XHHRZv31>_3NJ6O
z<9)LJbFG%XE-0G(sJ3E7y%s}h$(!g%H}i#mo=%PYD{g$K?sC;l<^(U(8GD!TGsNwz
zd}OblWG`jDvZBH^<nz_lFZ$hdp6r?U<`9F3<Lu}9dzVT4xc`3E+%B26KNlwpGTgAs
zn-l)y_na0+&#fQ-_cCm#{WIyb{!@lU&1HS}E5&`bPJL{Zwd&2)-91vrHtZ?-Z1dfL
zkKudP={KH&53+kA?(dwG*tag|M{7%qg_V_E^*2ym%`K*L0o0syKdQ>kdSQ`kx3YVm
z!_1Q8V)L7~ABCm=aZ|dswCr0flhM17kUsVmqTz-9%=vF$WNTctFaNe??c4G-pB8(c
zJ9mzek+H3<ZU3K7r=OjjT`KFhu3~p_&E6{A`l~s?E9*AwD_$H`+E%%T=?Q=T^?dU~
z^TR){x_exIFM~qA)%Jv6b!RWjy_s=w)%hyli@*MUwf*+(>#wud*KK}ickb%Fdv&j`
zn*V5X+r-N}L3ZY~Gr3PA8LH2*9ARoWl>gX%mhnM%pSSml&OW^!&*!{&&*O;oXN^~{
zJl~%azQ20=t+#ilaj*Jb@cq`!Q%Y}r&qn#}s>_NxFXfosc75g5v-|vRPrV*geg4Ll
z+?wt$_s*@4TN?2B+qZWY7CHw7U7D!u?$#%>5_CbpGKqQmaVL*FJltOW_0`h!kaY)7
zT>iD}+AhB34hfL0JO{vMwEsAJex8v>mcC=rmZUG9uVk)zRTOF;IqZ6*Xy5a5qW9(+
zC@U-9-k$&e$H&K+mzSNLY3zRe;gcT7fQg|lAKN<5`(O6VcqBB>eXn1j)BO!wT3a)x
zE&b9m<N12~B@RpCZ{0a$q8@##-m5I7=Jd_FPiDs7*1k&rznJm)^xWIqdZo?RrJS6!
z+;6Vdo_v3&u7;#5{HreRt^S^Sd)vv!i=Ay^L^#)%%K9-nR~I~MTVnn_Kj*F915tJV
zIT}~WcdQCs&Bn%dZFPA5%S%fayY&XG42iGzJLu^AJ0f|tmzkNFS=ddL1q?lB*XHEp
z<Y-0KhBI{?`VkzrCF9}@!{jw<*D^CRbAP=(d4ke{4}V#I7BaK(czAh9*;bWwv}ZR5
z#wj%i?dtQm-ZJYt<K?5F_kW41UrCN@Sz5L4j~9F8?Y1h@-UUgl+k_Y{yj(Y#wd2j*
z#{&G}Tb2I3xVU(2^!96|7WSv^-1+n8=jXjuUvKR!e*XIU`ttYpVlSxkWs7Bov!99E
zl<JbKox17LuFhcH&rO?@9Rd&9?t4}KUsgZJ?&6u0NlRZ@Es}X)0Xn4n_O{&LFPG2X
zvv=?5DL<F>MsO<bTyODg0YlyIx7(L2S+X*Cd0R!vYWqh^0zTD7FA@>e`gbw<TnBT*
zrlpb74hmllch4}m_N>Y17te9UU#7=5)f6a*K2q$j7JPKkRkM0``TMx^^z_*F$<|AP
zUVeOh+`UI)W9jQ_-`?Jqx2xIVpua#v>!n577yqg%hgXk(e-|}aduP#%e^XC;uX@q(
zd*b`bjS+7?{a$h<B=hw9o7YX^Rn^t`B@7;Xd3pKmt*zE2F9IBIt2a$}wDLwT``Mgx
zb1b>VbTlFt2JTICkF;fuZFlC{er1XIb+a;`y6+jMr|CMk@mwsm*qQ(B%}s7`J<y%r
z8<UT(3S0U$(R1O)Z*OnQ*;ci*wD8DSTsVB#Sw-ljm68)j0MGH}G_(4De>^9v)mD^!
zebyI$RjYN~Va3)NMyh&87Ds>D?Vf#YjpigyjRsb3u^qd2ySuttT3W{L_j~<j(xHbR
zA0O}Q>r+%zG)_BnV_&Vc>Vg&#nWMYpLz0hl2+lUkHClafqu>8nrfH|jp3Gh;(y1sH
z`)L0CAHU!4Pd_tb;p>7=*FHWz{`<?z!v<&0&9$~Jd$Zzo0msit(&l+3@9+J+ySqI5
z`Z`(Lsx9jdE4I`uiTK`}h1M_3^e*|7Yms{DY;NtN`S)iSCc91acVH}edFkl=`}YqV
zSWy1{Ui7w{i=`GVDX-7WG=6(~yT6BrMac^RkV_q=G&?B><lNtP_v+QJr7ND_RTf^6
zJmblnsHs)i>HBx6Pe0x({r&s>`qx%-Y92j!Q1JAWXm<AN!pFySV|D~we<(1iRo1%f
z%d4y2I%2<n{w#fc&G-640jEt4TPt*1ogO}VRP^ml<m+YoZ|>5$SZc9E;K9|H7w;Sm
z%l&yIjZ!{8JIfQiC(Nb(^E2P$eX{xY_E=_ISa5Z9_~p3UOCCh*srY!Pm3!^lwddzp
z2EQoNR9^C9fsBcBQBl$EvbTEi`(kc>>|gaZ?WL7W(1MRP@?`~AR!je``u=Wj`TMxW
z1s2mpCQX`D_3cgNvokaKWUb!Z*?IXugV=Nht7C<Cj&urd&$$`2+)p+?|9#QZQ=o+4
zu}y>huZNS9)9Y(%^Y82kTzmMB*IKm~Rx&{acdbAD?p{6Hy8K<uj}H$Y*<21=8Nw%T
zSM&SZ+pn*#T9>?-FmvY2b;?Rd!;6YO9cX0UyLa#HEt$nXJ|rqDE62LKaQvL~=tkvd
z)?>$x+1c5Zm6ZuDf9}Fs2{PNEwfbyTVQKN}Yis%CY$E0e2QVm2%(%TR*Eszg4<~0~
zNlDG09~(htUjf}vRr>15+qZZ3*VpUo>w7tHYW~n*{~NM0XlYGt?dp(~pB}t;@?yf?
zU3~2#7X9aRYpW7_o0|`J3ahIK-C_}95Yvs?Qup_ly8pZkv7{@d7E5ka{`#_V^5o!x
zf)7_#2A99N5qQR4QYa`QK-VriFfh<}mRYUj-%Vxrb5pZ+zOs@D3b5I|<=w-RGfQ7z
z`}+O6e&nVl>key1^vT)&dcA%>7faJz>+-S}7aXrYobc$;v$M1B?k=zY{q1f2|G(yW
zcPwJtojFcwzG6Q+YwOmn`{&1JZoX9hjko&ygEi|8E3zK?oPAqv-ki+r?AMPUJ9EBx
zzN_r*tJkl!#k#A%zuQ~;yDYZdxvO<^`uT17_wDTL?p?k-St!7XOZ=4M=FqiKruU1V
z_w@2Qmw)fCtqHvTP(Zf2^!0(i%EH=VtDd}m?QNXD>GBfK$=c!TBBG-D<ZO5S(3#;?
zl6rkztZnr-kxrNA=jOh?w|6%<i%G9KTJKs=P>}8J-YWF9O;$TG7Idr0-Hx5rS|#5b
zHl?19*;%tQ>1fw3zU2$5zrG4R-Y4tq?A*>LyDRIe*0j`8SwYRFX?n3%_5W(LwO3zW
z?tlB{&6id(PAAM4%Y1pQ7qjESzS`=(iC5$cFRh5tjcs@Sbo)2Q{@J&G&zNGeujHfB
z+VZ*1ot>R`cbD6jzl%BECtLpJhNE};q?jdPtJQsHJ$d*raCO+)%gg<vBO+c{DLI|^
zz98z)^{p8fAHBG^IN+p{S^4C>y%$O?RN{}?^6uHXc>ThV|95U~p3?Aa&culmr%w<6
z|L^bKy?d?8-u!rX*E-hSrK<ySnZ(t@hZmQ=zBXMib`i^z35U9iF34ow-}iT4?eCD2
z@11R8f|C}P$nu?3|EQmL@7A^GHR=|{ze?`zDg|}Fl!ZFW-rO+Ey|v}__4U?PR(6%2
zT&@dFbka97yLR*D&nG7*Loem(?d>g<6;yokJCJWy@ZH_z_xIKAwz0E1a@cj^hp6ih
zC)iB?@$24M-Mg%N9zE$4=Huhr#kV{lNh0m!B-QL|YbyWz*a&Iif`a+TJlpDLPo9)~
zeRWkcc-fnKd!zmR{jUoOI@NJJU2uOv{r`W`W;q^xwNj5B&eB?ZD=AQ|`umGx&(^)V
z(sH<+e|z5DsOt|!@<10xeEHJT-=BYf-`|^?w_K9^DYrfEZr0shr7kW<R)wyfrW<|C
zpl6-3Qp+EWQ;z%3t_WN_O*eX)$ma*gp5<v(?pb$OvC>KI`=hq@)j{9N&d;-*YgKxw
z)WWqft!|Qs%BzcukFSf}T_Ve;nNm|*dwYAnytw%G`v3npg+Zl@lgGP+=Vxaxe=Wf^
zQ_XkQ2b=fBpB(N?)=qu>MPS1g&6tUkW_?ks2!1~0#`5TGIWMoTpTF*~rq!CYYhz<$
z`(>@SRe#T07;s}vq;agf3s?Kz>hF9K1`ax6)z#IP)192UgkG`#m8<{z_4@n$|NlL_
zd3CmRUFd3&U3~3Nq*9A=PApq@^1a;J$jxaV9vs}ow|vFQz{PF>0ReAsZGC=zzJ1-F
zioVAsvVxi~rs>DOd-g1Bna|AA)AhG!Tyz4J@<qaH`G5U&>y!Dp|Nr0fYut`o{VnH=
z0QJ-Dy_S0L+<YSR|IxEYN4uB%&tLbtpleM_OUt_0-M@bSwzjs;zrQawK0f}s;KU}6
zwPF7AY<|}N{~dpJmZ|@IyWGsom9INE6nFOga@>EvjaT~D*Vo^_?JK|0IB`$Z+Sdgq
z_#aNuG~fQUufDMO*_oMB8lGt;CqKTt-2eOA+uyIO44!LMTJ`_m-gU}KtbH+h@2{`F
z|Nr0j{nK=#<M!3e6bf+aQaR<gf4+mljcvKn*Vo0yw^!Y>HEOJy!mYnm>+7|1b%p-x
zdgSfvE-rH2#kc&$<;$0EZ_U0gU;n4@?JZMv_2ud3=e@L23TilV^r)q^wY+`Z9{>4v
z>F4HX1}<s=C6XtcryT1$0yOSy&yU|-_BM{~eMLo4+q%Pw_nSdOe|)v_?^fTu6LbBc
zh&a#TKcCOr|Nr|vJ|!guRPWr~we)p|N8^V-Ka17XmoHebz;m+N+q=8Nb;Mp<DLHw>
zE&RA(r-G&B&f4GKrt8J7dh^5mYT4!zS-ziaHs{-AC4Rno_Rp^N*PEN0-4z!2r~Z8V
z;9&FL-`}razI=OgdjI6flh++q+*R`X+uOXmyY|-o{dK?o|KGpg@0ZH*DXQdOlr<NZ
zkdT;TRk|u}Z&m-_X||jZOJ5h9FyB#M@@e(#-`3^t_7p$&^KvL%6uv$#{rtSt$Zgf%
z-&KEmbFtLIC7p?h>DJcl@BjY(uKoQDH1JmFV1D;bVopxYvon&`*1J0{7cXWqe`zH%
zN%>*i=D3)<Z(h8p`0&7SV#l@0t24!Pqhj{e?992j>2N!L{I;B#>kcbgZK(hMZ(Zze
zBOXvR+tvPhVI||_QMVwfCa>!6uhPwFXXR|GLN;GN_UUWg)l!Qm-g0xJKfSn<#LnJc
z{QR8f(xBLOX8~E+y9=G$H*DA-sO<LV_xt@<bYpAQL~Km@_U5LTm>4K&hOV9lD)^rW
zpK>gY3<(LjoOX6myvy}>yVv^YhR3!$Z~FG-(X9``_1D%$U%!8Uf6B>8yZG8Ojvhb0
zd)Kac_VxFU9!<TyEmx=$RNn1b8n$}5-`rh^huN&HcZ164$VhPG;@`q~Z82%Nxo@Ap
z&Q8lKOTWLNod4V_E15~H&#!N5o&W3dojWy8PE4HAAf}<}HRZ#{k1sDTKi?^=K4<RS
z<mBX1S;3ReT&+n58W^R`ax(7i`MK14x(1`_k{t`{F4zc)iCx>{+I?inl=*EHv%>P$
zy)N)*3*XLn=cfIkQ>VNXr)!IJZP;KC^>5dLhn01wrf6!1t=X_~qh;Bf8C$n*U8k(H
zQ(sSSo&S8h>H6{SuCBg*ZmxAEubNYr;w$#E!r#7qvr9kcapX|+?fTCCUF%*KbcCI`
z8=T1QHdQ<P-JzeKXPf8u_4mu0<;+;utTaDK;u+|^!;ML<6%`er6n%f++SeT(det8v
zxz0Ar)r;R}Q}^cwG=+S0vCN3eys)5g`}FAn2kzdurTp=#Y;3#p2fIVxzuh~y)NAIi
z-#<gwL~Ja6ey*9F|J%opH+K{+UgxYFIm0CL(!Sc?Ny*8#w`M<o|9<`J0*`f^+}zn$
zS6yvp=Qo)xa^Z=wQ<v_lqtA8a<>#N9Yi;~!60cj8_t{l1t>!$^es^-Jbo4H1ucObI
znO<F4x!7{Xgb4}xppv=#_qVs8(GD}GNlP0V7{bG^KY5Z;Uj9A4?x*YZhZE#W|NZ$H
zU-wg0RTUKLwZFfCW~c;W7x6EsTphc+?DzNg&x4X+m8GAd=>JF0&c5Ayw5rK?x&LzY
z1q?mCz2^D%=1iGll5#@emqD1p)~u_qK0Q61dTNTKS<a0vQEgCmR%GXwD=95~T6ypG
z?b$*BP8`<x_iSR12OPP&B5-lh(^Kr<^Xkj*@Ag@n=d^L*I%Tamq4@i|A3oFyIK9_z
zZq~ImH`V*YRt7yiJKMaSPgW~((~{=qW<MpB__Vydd%MfuA8zNDw=7c8(_7~|+l=S%
z3oDIJXF*dDg^!O-)eJs%{rdL8$H#W@wF@k&4(UFx{_fqo#csVuChOzsmrS3xWy`w5
zntCbk<zma<A6R-s&bn+(@N&P%`O1^8u8aMBb#?gmyt|t=Zan#%!_#k1{r`LC&gtpt
z`K=5AUCw@UQ|mN7U6+b^^UZQ^?J9kplbd^fj-@lVtbmir3bFbAZhbO4Ykq!OGNawx
ztUUPS<5F3_jS8^N=JxXU_udEvFa)hEd3|kd(axF+3mj#RKTU5@X$N&?XUv$vD{c1W
z)z#NGHYV%o=@k|h?&9lu(&4mF!lK~8uF}`D%yO%qo)YzP;8c9MLTrBhER#&81J9o{
z?obaub?v2MLT2+Pn<=xtsQsuvr&)M@Ti)GYS0|>6YK8px@UVIL^5u<<2cLhfS`=OV
z{ax&a1jp@pcc<ya-rAh*e>VTr>C>lg-n@C^M#S}t6ZL<7c<9_OXM1jr<>kHA-;<B`
zNlHqB3js!_tv~*lRepLRsvR~XEcff@2h2>0YrCeaPdT~q`rWz0_wPjSE_*w}FuCZH
z$j+F!?Rj^X`_Bhmn0D`;-Fjse%P*gvdRJFhPxKHG6H7}=TV}bV=I1B-e?Jz->@0Hc
zlUexoDEFEdFJ7#R-F<I&dH$IhhO5KZFEgFNp*VkmJ=f2-_v`=58Ya2qq&U2I=a}WU
z^mRd(!c?V?$ELmO%zvjFx5uKLQ-h(?WmE0%vZ-FC85b0$cGc;#9^H_5_|c<BckbT3
zEF8cPyv*n3=JfNs%irJHmRtSrPo->|`zFwwU)b8HySvNtLCv)5>z1-i5%^^JYxe?X
zX|tRc*Vo^_{rg_|xBdrLvtrwWmlT|uGW&Mb@%=k2i=Ul&d3kyL<zi{mtS`sq>+kF+
zOuo3twNgaxr`o%B@0R<|mov!-xV|pdzVgNO_<GyoXFQ-B+04%WO2J~%-fnUIZMnD2
z)YQ_RpPPGmneXLNi<SvYU0EM@Eeu*&_4QThmluKEY|PnZ@xIw-pKxu}`ueQT?)~Av
zYCbbIWL;hLK10#ebZgz;U;J`599&$#em<XHZ0+!g7gVb(_md3{zHD9oZi+^r(&^&7
z`};uENcj3VP<<(^=5yjXhvzgtS*s_{o-JFxy!^+9hud;*zXH`y2VMUz*rh1K_5J<*
z`|DzNPd{iaH*e0u*9BcnacA$|v;I=M<Nk$(&f)7~e%AKBc=<B-{yy0Q3mlt+f`f}q
z9X|11$@==>VDse3leg#HofXT(wV>?nEk{R3)2u5qEQ_CASsDCNc<Z^fvAfG29B5p&
zY}u@J0iceR<^>J*tCF_1dzIb$OtP=(y!zC3!kD{6*6-o5&wu7J?>~CS_TqB?`M$Hw
z{(9{H<Zu7?%k%m5ZLO_m=USVKE34d3`T6PTbiG(9>#{YICtsd^e#^FP`)Ym`U0&vU
zeQmUP!2^d@r-}B0ll(u}*w_C0l6iUA%^NpBGdydf!$IwBE~`~XUq?7OH7#D8?BU_D
zm9IWD%QW!%Ly@rWeLhu%rQO}#rx!Z6yDXBA{o=JW$kM{%&aTqemzH{OsSNtZ&CR_!
zet+H3F46pZdsePrKY#M_6Hh<=`}<qos$@mv=Ct46-|I(iN|8J6VK-^kEUTIy1^)i$
zkM&B=w<uf`vNC8_Uxz>tM*!cd>Wv#W>PBtZQT=`0+Sf&|j~y<R^?N8>_v-v|@#|KW
zmUFF2y>!maeY)=5y}i-v<MwiiYTemYs@<L`a8lH3>88ZPZRPLo?5z1&wAih;=%35h
zDBHwCEnBm%-@17dH2hNY^HZg`%1IqNyMNQ;>vk4AY+Akg^}W5-FaJ7moK#$O^!2=z
zD_`y`e!ePX?XL$nrhvQQ%O_Np<ej|Soh@yiSM%n^#yegPj91n~eqO)-9~VFWe9Pjr
zyse%^kJj)12fD=J%gf9AH5pkI1URm(iQHWB@>2Epcc9AQ-QC@lB8n&9JUtB>_uO0X
z@KFB#zhy5jD1r(yOLx#UW0<X$U<;eQ8`SP(huz&(%FM*H&!G6}?-f^HrJbF{%F4>h
z!m=gfqEflEdgQLEuUVIu`DR~TRr>bU*7tJ38>_#+JJ`%FEg>OeQLv!$^Ru1OiYMRv
z`}=#{x^)*XUhEdrUF4tLG9lm9GJ{b_Q1IfK%Fo;1--|Xax@>In`<HOn_9r*g_1C{&
z6}b3VE4R4UbiKLa(>eJ0*GF&9+gbF~$=R8gm-q7{A(dD!FR#11O6S{Fe|sDqI@h{<
zUG(<6S&m(mUv4QF&CI#H%-7S?bGCWDoJqz6Q1eX0$wt%T-JPAuNm;92i@PoiTv_z=
z#fuj<e|s~oNy?gLh0Kd<{a|CCcgKRkd%E7)sn_!E?TOr$6Dhmjq1$R@NY$?|FJpez
z3W8fTx&@COKUP*&E|qmvJYe9{D`onr>W}34&wYJ;jEszSc6M9tCEok^`Exd?Oj+i8
ze!l(u=-*aqs;aH+?U(28*b}@l$#oW!KgZ9!Da}HkME&R4tc=~gt+Vr_xY-*;dnG%&
zePL^(Hg4RwaN)wxRUx2}Hvy+l%xAmRpJ;1^uG&)b^V9S5^Zn;o1YX}L;w!A?Q}OT5
z&reTJYci^~OnB7%%aQZsL_S%oIhMuGetdkar>D28Z>fi6)t8Lzd3UW!UOagFcJ1p3
z(0TUqr$9=@_2X{bzWw_3>-&4FuTOc_`*hOw{QK9|L<TPnYHe+e1Pv0%IB_WcY|r}G
z@}y(w(x>(R|9=1V^|fL0u@_d;79~G9F%i@o+ny(@x}ZfM=uUvWhT{DLSFe8k@$oS`
zzucRfo0qRk*8BqM-e+7`5VyBV)}p}S`bGgKj+6Ny|MTA4QyIKD&9^P_-Mzipd7$Pe
z*T(1P=U-nH>Kz|%ucOm*?Ym;jgrttFk1aLLMyaRPL~nn0cXzq6d*78(omPqSpPk$J
zdi(pofB3L~Wr~1Pm-;D3&YxzrzrMV@yZiga#qQ;AZ!LWt!L_>d^|iUy<#{(YB%V8W
zZdYHcfKx|afbP5r`QQHi{XNSxTT7&CclrBEDYNoL1qB1Q=fxHmf4;pvUm8^Vf<j=I
z<KG1>HJgK$dhOi3yZ*<AgP^JRjUq-e$9ZKe3?L1<DGfqS9)+M=749FtdbRYyfyR6H
z?#ao??dn@<5V@-)Ga%r?kxt?0Z8<B!QSd_O>Lbpde)4uT8@7XnRd*CW_iN=6Es^z8
z6p)g-wIT8Fs?gPKEiE1@LZD(=(PjDy`6(0LhipnYd3LtBx|$lFtko2|)3u>lS+5!x
znXhDh?US_zZEzBBveZz#zrObOH)Xe;51&6rZ_k^%E?M)%vokY`pP!o>xjF6X>TrK>
zEL^l*byWD1^Q1|WvaYTY6&4njl8V}z6&l+f>=3)NXldo=XZHX96elDs2++7vYSAL#
zRMP*;k@KfwTU%T7_B>xtPf3%E2~lgWm0GY)^ih+xC|EFW-aVKz7CBmKDB7<IT^;uL
zSnuwlr>DRTRBuO6ZEaKa<;DB`|JMaBZd;eE*dh=V0E&&636mxTZO^-#etzEF9fgO_
zo%6f?P(;Db&hG8)?fF+%O`Shq{;&b4ZRzCU2r_)9vVp;d$jxacPYgGIdAd69{=V3a
zNv`0~TEeg}%0_YL;=;$rf|vR1to!@x(b4WwS#L+fv@<haym&EHJ3NTVi$n2=@D+Y9
zk9C0>B41x!<>u$#U-mW%JTBGOS6Nl{?bX%Q#m~=y)}NH>v<NtH$$+wruX)~`j4LZ9
z&Yc@uP_SX%ym{*mUz};4fA7Za+n;UZ#l*t4<wQapaRBUym(S<d&$BFk_V)Jn|G!@A
z#~xqgeYl;!{`<ST$9knfIagqk>MM3tmFb#OPr3C<-Q8FF`_xozUTL!{r508jQcq7y
zJlyv6?d|gK@9u(=t_Pzl=;{uA9i26Cd#k2uhwrQWoK|>S`tQrj%eQ7<H#0Kg;^sCs
zG=!8MTpFv63Qzj|=Iz_(XJ>zZaIo3_|DWLN8=q*;d*03?Icdg>9jT|qK=naO4dd5l
zp-Ji%ec89RWEK>B`1$$yY}4#zuM0k%Tot-n&3Beb)|C~!(q=EL#GE)3S*4yXXn8WV
zonQXnm&^WQVqyyyE(}^3a{b|xoxi`o|NrOn`KPC+>#O~b104d?1S&wLzPh^l|KIQT
z`{itBS(UC@mwd8VYiibm1C1+JuDrcHe>o@zp9l}+^YW0JWsvC9Ev9RhbEDwh9n09`
zleYi+_xt_p>+Ahz8ZFJdybKZ!iVj;VoIYK9e7yhlm6gd!NkU>`X*oG-UKf0F@$m3?
ze5^M&H@CUD`DoHcP&ELm>g_cYD_0jiJ#~7TZuP%EKaX?@m&$t2n{fImGdmy4hUDXX
z;DC4{0P@<-z{PG`v#;m9zP5I`-`q>3Iy>K=xBtJV{QbQ<J3)i)tHIe=L@K0PUBw<W
z<8bfZy#ognbac*KxdIA?S={1!Tk`JiN<7>)T`x8%G7L2E#i3~e%Eo^ZGBbB(U0oHv
zKCV~RI_&z!C$p9;S#q$M-CJeS+uPfN!AWy=0H2qKoup00hVJgu_x4uv%UUgY9Z__A
ziRa{bHkF%7UIs}^OUE8};!s>N0hG{wKL7gqx@q>clt?ojoilep<3xW1^!4@A)6><}
zmv2r#Z^X0sbpeNBi--~E7{m47-roNHdj0;g*Vo>DczAeM-_zfF>;K=och62$_362}
z)?r~`ki7QL85GP{_4og|v^spfvU?wBxb*tQn*A3SyYH|2TlMwTRZ!W-@zV~Jm+Gp&
zzni;vZ*5jqRz$>%)>hYFAJfmz+qrwU{~U{r52n0SR#M{O<vl%Bd-~6Wi!V;8wg@bW
z2J6#~-L+-+?&_GBIkRU+|8nH#@9z@T){EQo<9_}BtBE-?Pd@!Lzy9CNLx&FCy=(jT
z!RhJx$9p7`^YZ5XR8VXY__S7oUDbs>CN`FxN8-WTw{IUl{P^kV={{L&K6$$s<IT$-
zywRzDd1>jAB`OSy-Fjc$*!cMLbbZDhJ9i#Ec5K(~-Lq%Td}-z2#G%+KarF`B#a@1S
zyBoJ}OG`=ZtNUwZXJ_}<;;_N{dwaDxUf=MaZT5D4{XftD+fq(WvaR}ZV^8Jh1C7kv
z^Y5=ad;k=@j92)*I`Wn*TXyT#ExVc@28xP`p9|+b|NG}piD+nA`nfqL_q+(IsrmEl
z>}*~ho@ZxgFW<cYq)%{4v(O^udGqF(W?lL4^(*U#O5H^YQywQLB{?}e8y{cx)5_9v
zXW`>xadC0GxO)YhJOUjoH57kpnVEgN9$)|V>gw>An3!qm20X>z-rTI|dsRH^kA=6P
zq2al6=MJ~?`}_EuIX?kpk19C&tx8^O$iHuwa`o@ZgP;FAKHfjywtCwB!rE8dl^fKG
z`1$)+g|6OL@li>>S*b-Js05T4_Z5|uv3<CDR<|^+&%XHCneg+Sot>4xzvbGoICCg2
zd9hGt%7hxz@^^QVkM~X02z+#UdF?`xdHixV6$>IeUvi(Cu3!K0(b4&JzclNeK!zRx
z8Cv)H($dp!Z*P}2&->F_d~TlY>cgd%YiepXReM}+e*NlI)$ea_)6UL%njFv~;M65@
z%8~OTqqg>HZgD-I`F655uROiKx4QiKxw$p{-mkfv9b26q&N9vJle3Ky2s2P^5eTXX
z<n!vt<7ho}v|D_>UG1)rl|hm77OYlW=(W^nH>=m`(A8oxGB#yzZq)E~3OIRObhgw`
z+;wGN?eE*$^S5VRef7D$#8kyZLqo&T((<#Qum9Txj?JJJ;&MOPxh}38ic2~c%1jZE
z)ec`bXZ`y3dn!Lmw4IEZ&d<RSpf&ZU{Q2jt6}7dX#hrP#xA|&KeO7M+@`eh?8@7!O
z2IcQ!;^X6si$8y9K6&z_qod<b`JVoM|7RgTmd~%-Rr)&Y|G&SV?Lp^?xJo=-&>|wT
zHhTNM;^%(5%idn<e^PJx?{GW+!Gi~F&Szh`z$vW8!pa)DI!t%|VTBffpeKe_G5=d9
zyxCLtH)>(P3oEX>5B{IOSId6n&;LS(`r3Ev*6o_owOGU{^|Px;<W1L(HBR5;ZvM%M
zuYSMxyI5CGO6!NMlTyq*R+_BawQKkJ_x;}9ny>A8pFf`e{_gX8^Upsoz5e~v8(Ta1
zX?OQje%_RNdW!%3{q^flZQi|mcfYiGUq?x4Y52OBo4ZQ0L+9;vFf=g<2@ik1i;JN_
ziCa>HYtG%><@_=h6JBdA*zx+#&d)QA(|4?6VpwoOs34ZXq3r#=y~>M@{o3_Rwd*UF
z#LlLv8`n%&e~Ht0Qb^^FkgQcpPd<KHXnt`~?)mn9M@Pq)a7KoJDT>WbjHY)NZQ5D3
z@!Wy~3#BI-#!P2m2so1HE5yLSz_3EfNMlgYvl?DAG1M_HK5MLb^vUwnpeZ&+|MI7Q
z4xVvIv~t$ZjOik({XNgGXRW%sXJyo%xBAuV|If|Z@>6Z;KI_s+N*|52e=b%R3KiLw
zvozROZfi~dqkcIF1_mzilmLw*EZ_3f{H}h!9Q5y<&xbpU>vzwauOwx#wuJZYo~Vx!
zjZIUR+D_bG^*7uram&|fIhv8v=0EP{%B(rH`}sxoYo5+G?pt-2t=`nlp7H8KXUEjN
zor+FutPC3GCO9tC@Zjb3srVPAcll!GU7uJFouwXmao%02S9`UyuNyu7+&T4cT-`5j
zp=rGS30c#l?3TJ@+6ihZ&9AroK4D9%%aLVEIC@rf{9Czvq570o+dsd&-0vUjQT!GC
zzR|a1<}0D=NfkR8&;E3A`NWd{z2!UStF*0KUjJXbF4Sr9s;Lk6f8Cci^V`>R%kNwG
zX6*T%>iT(_lC9!N?tAB%85lx9$!V|4`ls{x|GK*z=`f8p3szT}q`vdLZ%0a}#x9rY
z$-I9*o5roLdUJXG_WM&_B>A~?tzbC$m3wA<&+YKr1?Nk?9{I3iZn^i387u!kje2t3
zU-f&A-X*1JLX&O$Ra`fDUe`bJ?#i{fJA?Lj@j1Uz{mtvW!#I4}_lZRlKLi#1Qk@yu
z{iUUTLk<Ij#*z(coLr`i-toy(7^ie-T>j(Lo4?WcUs8_6>B~QE3%(0qTE|`b=lws0
zl=;VO`8;d>%(*e)ii_x!N&jAQ8rj*JDVDuf-T7%_Sf%ge`+s{6a$M?*arm#rz`zjd
zXs;oXWYq5*Z=${Y*;So~KLWLc=lq*;f5Fscg_3PSDT@4)<@TMv6SUSfz4T@9$+_i!
zqSl(Mo|Y)2tRfd~<RAUyYSyaAojj9g&G@9{Fwdv!!S|WRr&K;On|1Yc(5jvwp(Xw6
zPbqEYv&fFl*t$;5wzzHWRj<`ql1Fb_cSfF{$C|v*`h->CR}oG@L(!t_sh=fXlN>Go
zelg4OnbE}aqe@CLdeWgMiXV?;d_LPFskA7i^1|;s?8k$PYdvo}ZLZX9oMts!V&aox
z*Ljs+g_Om%Z3`1#shQ-ol<}d<&Fd|i9TLUTfnPuMNS%zUQ2*JUUN<#<^-8x?;dxQ3
z9=(44<h1#dNdMK>7H!`3W!{s#rmELrhbF!1T5ouAU*!3JhvyqHFo>|6SP(E{qrPz8
z{xh4WK5DF<^C__PTdLaE>z6$1m(JGt@^1P6wNq!88@#v~boq#jf{JT~Y45i*E77e}
zba||{{jf+%;}SG{G|5=}Md-!Er<z<r&p&rAow8yk^CM>w7Z;!0=6d&+66rFtV%V1c
zsIHCL=YB9__4dLGxk;+4B4%l*Ufi;9#X3#FjaPo1UgCIir|8`qj{7rhTd(iCG9hN#
zyC-#SS&y<F{qg^m_VO=t{L*cyIX#!QFZm_U9a}eRqg;Z~8bjwJL6@d`HVf^0QX>~v
z+MT~&Eh+V)WMPO&%Dx-NbpC4IUhIFT@#)3s@dARED<*v0aE(c+a?zCaswzr9t@|JB
zWH;i|S}No$_VV_OWwvi`Xuf>5?t|>~&W<B%Pn~(OdZ~JfW~;|s#iVL$*K?BUoPi&*
zwqDA;8xwNY$JZlgPOqPQa&P|od8ID}H|G_-IUKPlrvB7^yKV*shJQ>wN1ZnYF53I?
z&%aq#9}Rau+g~jE{i&ylAZO3oQ-7@+By`VesWB;Tbg_ACrK)A**)Okt)T6WG;;NXe
zifARF63M0EuI_S0Z~q?ncd)t5WxuD&q~)qcan6U=UM}q^O<8E({QG;|b1Rq1g0HjQ
z_B`iP2vq*NuI!emx3NmPM{LM-6Vt$(>yM|K7`ZKx+!~bitn^Cu)zH<U?l-p8-;Dbj
z;dN>&^QZRokeDd3S0BA@8hpKZuwU9nXuJQP3hlM(7ALbSUwpdka^%-K{@q`ubqM*d
z_c(DWRXcfV<m8KC?=6<yX{b82_S1pco|CquMI~SB$~_YlU)|pG=5ONBn<AEmOIE)A
zWPf*7O_I)$b!&f45!#zq@qy#|_t)!Bbb3siw6n(TxYPY(CZ;w9-~MT*9=VqBYK3Z2
zhv4*CrV-x$8vjc^JOBUpKKA*s<0mBRR$f}{X5f;wb@r4Msms(t<PJuxm^|gruhOn5
z9UTjWs*?X-6*08`rOC|T;Ml~&b=0-<qhFSb=kw`xd;c}v==gC}d-78kmmW{s-V4Es
z>zq|Oj|gnMbfLqu`B1n%@B9s(TIxzdQ?xu?W|{VOcBFK=xa2s!5i6f|W$JzTi$DHJ
zIVC+^IRF24Pd}F&r$4VG;}1V|Ibxohoggb<cS!Zq(v}W?mrvsC&hC8wAJwW~d%Z#>
zSYCa9Qt;#dzS`!Qt3EEtbUXLf)c9+DoqoUZHbWoVI;WrSSf)yAR2wd}_Sd@K=eBp*
z{beax+1JfdZ}J<aGa1RxSt#_&SZr$Su9b)8y?r`IYWdrD?*+I02#MnEzPD7WBl6~K
zf9`FkRhCqg{rG%z`zdq9tczWzt_yQ@dYn6UR9NtLLi(4}Pu@R!d`dO$wbSx*-S-TB
zFFqcvl(b@rTk|HDs+y-wOI!4H1UKqWwo?-jd@22@a|!!%_I~R%XWn<sc%Jx|y)Mc?
zJLkro7K_xIOTYXw`+d+qGPuQZ1p@;s$FWBXD;}8YB(8FGiIROjHGi?pRMAgAU0isk
zURtqg@|SqeROg?&tvgToIey#h<K?t=i?-6q{c6IQAAR5UroXI-+p1}L>G$i$T_-Pc
zC23`ZUsYH7dHc6yZEen>>)ia_d>I}tQN6R=V(rvw|4!676XDfm_ri0HQr-JqTXtRg
zK4EX?jhD_!rCy$ir$V%E?%EiYcl!9}@?#PIGkUFl^@L3oy0*)(JjwInbO}w-X`(?d
zt!}C9`60kM_5G6IX`XxE|GnE1Ww|(i(aNc1I|II5UOORE+W7p5knQVEt1CICKF+P4
zBF`5;BW!gQtFhWIUhg9lmi_zEyXWxjeQ|YD3SQ3I^>X$Z^#tMgddm{^w<+<-Q~6n{
zs$33wzE?6mvqklLo{-?wb;?&ll^6A7TV}4(l-IChUo!RVzMebplxN*&QVcTQsu%dH
z-TrC&#+^0G7#J30omdcHGh+tFk0U`|m;ScD)V>%QsihpC%(yfyF5v&)UnNRPUb}wx
z<X)Jg7_|D*mOIPXIPbOZUlZr<qLQ~@UT{Q8{+;RHvNfIv?~f_96P!Kc_$AGouiHEK
zh3~W7JxO9^?8U8a=VZ&5el9)c(jVPr#b@&<dwZ66@pFmDFV{4+Rg~IHyi4+A_dlEz
zW-i?p^ytp|6nj2Zw?2tevp2b}+0(KwQ8v&wZce@1^tBh3hzOokjz7-sW|TQUOv^iq
zM^L*vK`8HN`|cN0pBn#}RHYYO5@p|iXM@2n$7`9ae|kzkb{%gtdR~8t@37htT{&rY
zmuIqG`g2#uhN<tZEt^^IbG~NM65TB=N2VBso_@T%hlh(HAV8tn>Enbc*XGWAv)S-k
z{5<F8JJ;uFZ~VT!Mee%J^D7gU_Y`hp-YRYK(;_X;D{s1FlJk_iD~^aiN#~jIaCNHf
z-LpG(%~#o}<8kV`Wb?VB?zdh~d-nP4tW}o{x8~_yYMxW|_L$AH<~-Tjt{qb+9M)uE
z5{~=x>FCGT)23WdZ1Z~IA2G|Opr~rTw)S*E!O1d0o0FsaT5INRYuI12%P&(=@Z;%g
z8o%E!Z2A0JssH}Tr)$$IA0G+Pu&iA)&owjbpK8|Wnr%;i1c}+tneN~+<&0MS5+h4(
zg=g&csSFGn1_tsXUEFOQilx1`#BRnL<S)JZZ2p{YCw5i;?E3V1wY|?h3GrFmPRzTw
z(ICdi=<hezQ}>R)*HpUq=Tz65mgvCM#d{Z5?K^iu?a0qPDoSl$FTcsJNnf~_<0FUV
zk0~7<8BtY1^}I@*=a0%vx83M+r0&P2g!P+>+VXR@q$O>O7wbt}`u=~*)i;-7eM6e^
z{}^UX%AV04|4aJ%JBw8*e{J%8ypCMa+g;i@&o$HaN%Mrp<VWxJ{8)KgQ}dAWi|=)(
z7#J29omddCCu5uR!cePVbEV+jZ?+s<I%~%N40%hziQgWus!g^II<j`ff8O64zB;Wc
zTH3u&(Ed}}B%>)=GrP_h)h?6}Hq}*9I(pqhDe2|;($i1h_qeEh{t~p#`O>!BrEZ@r
z`VVi?HV^KQ2)e#1^OV>9<5hbW%6IIL*mhh0?Tsqlo@L)p{83U~5pZp7@-y8m+h^0?
z-f~@><^H(-dRFI*DaosyV~)GBYjr<dJ$uUOj9E_)pO~@qK=wVqU=Nom6}q`^18@0;
z)&F@|TmAUcdd^e4`#%M<F)&<pVryNbqWL(n=$hOXkM%7cmV$;MUV92APMgUm`0<SO
zlt}Z~=J`UkA1aRR`}Ah-G@C$ArKHKJdsc;=s&j9Bzi)%G|K+$e<wx(d!pl~_{`O|_
zzUfilx4m-FIePrfR|TcUg^YrNCpU>6%c$L5@~8ay*()o094r|_{t2h6C<XmE%64qY
zRLRdTUdvi)<{jPV^ZNKy)|k^p2bb^|eT$yRHUH}T_jNwjbywG={<@;9)Y)HuYuc`J
zsqMFoU5<3PIzN-XJ^SSS*dBh*M9<*ezN)D!H+uh@s*(R`O67@I@9@93wMv&&rKZ+v
z&DMJQa;fhbQHfcCf|=%5j4r92%CyT+d$rekh498Ns#UkI?EytEIO7E>UJ85vXw^zr
zmm?F>P99v>bV=1HE>$?G?_Lm_wp+x;GiTO{mdUR8u)i+KN@ed(?RQg~R$h%;;p%ck
zKY5<nk(hl^x0Y)3o-y$@<~b?pn0ojnbA`*1jOi<~om{&58;urCaqiyV=GEzukQQH7
z5#Hu;p{!@&UP(dEbNSO;HXVuRSS2AK=o}^G`)Zl^8RPt=_qR;?B)Naj(y+6eEcY4L
zZFb8#b@Z+(@6<${_r`U~h5J{_T=4&3vE<IIf`9hYI+iYp==WJYQ6hKY<jrBXmWMlC
zKia8(DQw@Q9jSevR{6Uuxth0WN4@m3lkdLh&bp>2`dx0tj`X!Rj`d7!uZyd5*tySp
z^=p;6zfwCpR^Iel;i{vmYR{*#tZA0VPQ6uI(jIT&zLFhg9A+mlBP7)0(k~a&S$yVk
zo3}o4GtafP6Et+{F1pP(eg5l7aqADP$)5SpFYD=Pj_NZ8S<#&xD$C1NJs;Ko2|oAn
zF)IUupMlQrVBvKP|90KnoUR|S;lZ3z+v427y6=MJ#*z2_{(jlXxjfx%-bsnPXImvg
z*2QNZ-0o)@_RsX)n{6+Q`&neuXINbI%0G7L`ii^!U)Z)hOTYE$s=nv?g%-vAF|U5_
zeptNkZf(2Z>m9zAR!=UpFAe*p`s~508?W;{-z03>Q~vB$#LM6P&dZJ0b@xQg3JTkF
z;Id+Of2vLT7Vm$%9nV{wHu~`+`Ty@nAHTV$pXs@17O{PA%ZJnx7j|ZDoVZfq-YysK
zqVLVOqs(egFH==otMhxKh`eQHMAV$8Hzo2UmwsH!W%kbV!;QmM#>-!YUOfFJ)21V0
zn~|&2lHBAic~hhW`{x&~xWT+Sd7YHRTM5-OR~FBFQ5ynUy7lr*{{NUgZrXeE%~yKW
zzLl-?zkOln!cS>_@#TMie?4k{|Nnx>pVx03ex0PZKfvzn+vPs?ix0nLKg1=<%*6C5
zkBfm}KZ~RY*PQs13w|EY;Lf`CfU|Q_a`0c~`d|O++;moCo)FI3FrPWd`B8Dz)hX&K
zW|m4<<5j<FoIfrp`PuE&8}65@cYM*^6XnYqFTcDd{{Q)Td-k->+Myz-X~%LxX|sH9
z_`N25y^|Y)ZXP~vf0g;9@p@&AgD0BjpHPl`8f+@dd$YT`$n%)I^vZw751FyA2-ThW
z`TR%o`co$sFfi21uRF_flxMa=b=iCKvi=FS-9L_o|8J|AGm&?f`q^EsAK#?u89Ua0
zdU8-dxT&}{RPayG#NVs!7jRTfn)G$$m)&MP0lmHzGe0S8KUMdBIS<IdXU$9wieH*l
zV;C3$BovyN92Op&qQCOb+@so=pAK*R#J~`6AyG$;fs4IYP@8vc&y@bk)U~JHZ}sA}
z5!>~{WP2zh!-4gVY^)836e~X;;a5MEa&wBt!xdr-3_oH|EMN%8&``23j%8wC_#-)k
zp9?hVv#O1UzO>qdT?`F#zE*$O(&f?pT5G|Ks@XF_dS6R1Fm#!v1ZWh@GR+QKHN`Q@
z>y`Gjhr6aZc`gi4QBz&HdgaR1tBYg36PK=De0g7O^}5vz3>v2-rlsy&$H?$crM~`u
z)Yh!C^K7f{?Wv5p&cwhl!T1b2E69hT&TOMJXsCqTa1bttWnlONTG{sZ`~CWt{>jP7
zTeoiAyJyd#Lx(Pzzk2cF%pA+$`1twVOpFW+o021@J>A8|@IyB$D$3K-v(jvL)z?*P
z*RJJy#mK;LLipGtMy|bnbFH4<Y&-wlJnv2f>rn;<h9~Ss8V(DeKYBDNT{=BIotKwa
zRD_v<VS@aL1q`imzrVfpo~HBjF=z@q&{LOzf#GKp4_Cm8l`B_TT3Y^H;J7kqY1G!N
zrJJf47#Kda@Nfl){Mj*kbNcytOpJO>jG(^FwWG`;tX!>!Zrrf2v9URx|K;uN?Hd(1
z85p=kzBDt5u-eFhHiX@04`@p~HC;bHH}|Y(C<6n-PDeIY4W%NPzVyg#{`2j44zJiJ
z$H34ae;~rf;bLN1+P7zCXXnJfTJAsp+N#j*)sq+)7>Ziz0vZ+_zIoHKm?2eSnOm<E
z#{ve100zZoCK1`2ZI7oie~b}jW?=ZhEYZ!b5%lW%`u{I3F8=cV>+S9NzP`RYbvYRr
z9#p?zXVnP$_2uPao&`LIO|q_VFp4lRaH*vPG$<_%UmtfW^n=c6D{JfGSUv^@hjRvE
z`~g=?^6&jQ)+_BIA}1HO%tvzFX;64G=G|xi5N%^?3tCeVps_<wr8z`PRY|VjJolE#
zy6lsO?%b*Q^yK7@Yz+nmfnSb{b;oy=zCPA3->)C|@8<2>^44WO-rn7>ofsGzl20sP
z*x!_TQ!{wknuv{yyryck^UIr=nLXRJs7cS%wDi;zP4_;Tja#;S`S3v_c9+R@QAP%a
z=@T3o>!Q58PBkzx^T}GJWM)==dg95<%)Cxp%jDUanVXZ2cAc84U0qojx-Mqs>mmk*
z2ICV880^*c^v+$p`0>@%)n#R6zrVe`eC5iMU5lL3USC@)tnQ~1x@yYu<>}3AygT&R
z85njn@^F0+wyXUGnx9ziKc7#=qTuW-(;B<^ue-RGZcabHYuBz%KmSZt_urO(e;(Kp
z<{fq#4t1<-Y*CSswza=Xo}HQb`ucizhQFzNe0*|pa<5*!lC>_ovpJpr`cno52KEV#
zjGvA|mT@K~CeAj?on=>Rb^WPG)wOHa!q>-{nwZEKBpf)@%Dofhpb*~Vj|`Q?<>l|M
zt`_e<etUnt{jt9P3;gHXm4bJX)gS2)ls3zG@bN>qA%jC<<NY5Dl{Yq}b}wGMc-yvb
zN4v!Z9scV$J0CthUEe(WTF&QZXYcMR_4e|5v`dMBAwV!Cph147W%08PL1nFw6%XFN
zjon{YyY94RQ%On5ox68;m%p!jc4lVS+9+3d_u^PV28I<(#~v{jsZBoF#v{2Y{rtT8
ze?OmJSs9$PamMSeE~9Cw%l+r?tNUBU&CRW^zn)7}s{o|w52(hloZz8ycX#>zijPUN
z&2stVY&3*AcdT0x5wXAS@0QHVr>5yve|&V*s`OO>C?)=JJ+Xk{rmKs~jhi=ZYkpif
zdNlR%v0kRehSyzP6NT0NcqENnG(_gv)$R&i9Tqd4nSr66`&lzn%Ew<{v$L~bxAV*U
z%`{4VeQhnr0<CL%tINyF%i9ufZO^ZNaX}H3o*VYFN{T2P{`>p;+O=!%@2}rq_V$)#
z@w0+hKf|kQA~&C&u0LHb_SVgtl|Mf{g&5Cz>=EOoimECrOG{0WuD#XY%ii3W_`0ZT
z_k{}=&ds%EXJhM=HqX1dDs)GQ27`lObNp)t7Z>aDcO?Y{0$i<fa&jCCSQ|HP{5VxR
ze9hXm*Vo0~ULCH#GUQaLD+9v^b;V|;3x9roX6Kj7Nl7XB`YKdgTYKH<Mw6<ls^{nD
zzrVBd^T)@>LHk9&z7Aif&BV}P&zAf#V9m*=pC&51=iT16w(RY#<$iOGOiZ5aQaa(c
zFu-S)N#^ToYfZDR?5Oy-XyZmhP$oJc-@wClG(0FMXnow?bMtI(FLLcZ+|K{{#fySi
z!6#WqyF`nhozV<mH)s0v@LnlXaF(bOH_{LZ4h#(Z_U+sEcXwZ3Tzq_6?(HjAuIyOH
z@u}d&g@uC3ZXGU*Hm9FA%f8kF$`TK@C)jC-xWB)@fBUv=`^w+n`|<H{`P*Bq3{e_i
zUS0+zm!h3}d}o^-@0WkRONN2rhwh050dH)ozMMF9>ebiR*H2H=jowk9c-SB&oaN`$
zRiWDY`ueuEd*|6!uZ!N^2e#pQqK;hG^wZPzt*xw<Em;EEL3r`v#Mea+!uQnw-&g$n
z-1_+a^Gq@?ZAv};A)JN5A+e<{K*Nns#v&m-{rL0G=jYjWGelK{u8FvK`}XZimx_*d
ziPrx9=IiVG6qGN1WS&?M5Tg^jYfJw9eMuXCe0qA?cedHnT{1s(+1S{`^<rLJSa|qg
zGyB^&Zyr5<ymK7~14A93k%mYuS8LLl8HSga`L2%JyQ{ySf8Cp4lZ>n^Ioqm|2L~F%
z*TrPs+VXO_|NI?bElkgvodl=Pne*n+qoPkwJf)>)FLv*rrV}~ob=L#_=q(u+54Cb@
zhpc$;^l55h;=xi&28IdxCl&-uIeO%X1$dd}>g?<5tjphtaJ9ySv(z7*XIp)&Pj>a{
z)u6?{+1c6PbiM}^Qfnqmn4so6>&~vy>?0k5zrVe8X3#wp6C2wvXFF@kl$7l3*YDn)
z16yF85}+|_o^5qnYU<ZVN4r4>{2c8PwUPT5uK&oY<VC^9N3OZKx$f@Ab)&a|GwKt@
z{|3&2PaZ!`PEHmU6x>+++|S(HeBEhAQSk18?d7dpqHaABhQ-f(!0wShSQDU;)hBCx
zZF_!vY3bL)?fkcI-71RZTM^~r(z1B*<Hhd%F|o1R^Y7~kaDc<9sG%-EBk0<i$jh74
z{cCIgdQaC=Y!LHL+IZvi>D9io&8ok>d3j~!WEZ7}ptw81Xrv(`sB_wHmdVMpXV0EF
zbLRH#-F|beV!~PG2zI)t`^_mREZkY~G3oQOv&u?JU>i0$v$ZZ#P*pv8^JeAKQ&V}R
z&GuA%&1&Tm-La0tLQQ~UTi#u()KgR1`Q@XcqxaYSo%OnlfkA=$4122+XnThDbiM2A
zVmEK!et&oQ`en=1u8TS{f>vsqo0~H;GyD7ddwY9>{ciGtz4g%B$H)84%*?K@i<LIX
zxbX0ByP$*OY(?ib8M_(_Ma6}#-D30R%>x%XnG+lr3RHi5)H-$Q)y?VW)6UP^n|IgB
zZvOdF*8qXiS65CRKJ2U^Vr60R=GNA3P_}5`kQCuse|vkrdET8LkB|39Z_Al!RjPIU
zsY7F0YHI1%SE0(viyfQUbR#x2fYdX5c3fCsRr{;t{k^?EYyKT<X1~2H_jIXcfJM!(
zFDtXJuRA}_R@$oM#WLU77Yg|p4k&(Uc6zuu^YXI1ySrY#cu`SVso3E3Pf|)MZdb`l
z*KV=f+j5IbN_-%-OT#13tl>;a<Fp<r(^FTjT)BOFcii5pnCZ-|J$-%WW*VngeR)y%
z?aj?;y3uZ~u7$CJ3=W2^8!I$SQjBgsIN1FC-CfXH;NazcQ}tqJz3y7DAt|!#*_oNy
z*VY&s8}BZ88FaKu6qNTC+;C)TJ*23pxUl;BySTWxx3{+5-e14pL*>yfC5<m;d3P*I
zUxjRpsCjs(wf^7Fc96%rKvTIacJ=>q($l}+umAt`^>xrWc;M<~L5D@@t1Al}o3C8G
z>Oa@&>M~zxP~>U2O>kTY-c-7~?Crh1)#kakwnT1DTls8BP^ZhZmn+*mzAP5K6137~
z+RB{{&MIfFH)VU4yr?&?Y2}rblFEE?$4RdE{i^rVLu5~1^F4a#%U-*E6$SP_+Z~^~
zJ^fTX&0hPqy0eQ5(?pLM|6KH@s-)+Ayt`bnzwZBoxr-eCdRf-HeVVbHf8Oz>Sz+7P
ztzSPW^|g8aJ<$437p0qftG9P|cWe1^GBCKj@NZ%QIq%!IZ-$13udlC{w<uV!%1`^_
zocl@B+>b5FY)^gH{YLBgC5xiq=4-PfDt~>xR@}FyaMjPp!Yy}iUC#T#_4Kji@mo))
zISMoB^Uc~Yp`&Wuw;5NLy}28Enr*$+s+yBkzXbJ+GQRn@b+D{WyBBv`yEgAg_IKC2
z&vq~0|7ydBBa6<3PVIkMxG>!B&Ac72H1@xmGFx5i<&%=88}91Ac%n|P=-u;g#lQGU
z_WPjH@agI4yGveP+Ma)Z!v=$xY%vB0(+zrZT<bnRKM&dmt~61`vdE>CaY~0v?DTu}
zMZb4&PrfEt{;odVxj0km_Y<!whPAhn|MjZ>t7HpM(n`<V|M-4lrI~O>mbUzqi{gTu
ztGAci?nrHuG;~?==;wx@&X$fT^Us`AUH9^~_`F@eErJ<TmAd7)=f~c^Kk1?1#W~K~
z_px4(SbsLJRBHSD9j`P54OOoezkD>seos|vht=8K<^Rp^x#?wHS@H1k<KVSXTQe^&
z11&IH7_j2Q69$H(B1Re_0oG-2UfkKadD*gOt=!^o-o6cuXNcb(^?G9Ol>VDn<}S}~
ze}2WQ=<|nFU(}CXUG=>DM?}P}RmtqtYW?~P+*GaK7ccLdyW3<%ou<+yL!*_+mp4Vm
zRsStKnSb6iqe|o5?cnY{Ww9H}O7Ga!{<UnEP4Qf)qy1+4wT)#b^Jib-&HlQp<Z)B9
z^H=%#Ctn}nPC9cdD*T7>(&y1Ti=M8F*?DP+=j5-iuYZ4WQQ6Wm@+d0<gNXTw1pxwQ
z=USJan4oy<*s*)}?(s^SJz2$5D_r>N@!Qqs-RrMAh?S+^`su@bS@XW{)IW@obyA;?
zE_-&>x7SEDdTy8Q)w)MI`~APw`97cdr%Ki6s=_OStX-cjZu~#@@0YuBjMM%0D3xss
zo_uEam2H_ROV+Pn|9A0y%Z>l`$*pO+zarb#ko|It*EF5VcXxKKi`_lXrgD?-Y%@bc
z!&U1U85aC-U~6r1)tk;QX_RtrPi5(=D?7{H&YC$>vi;~1FQLx#Oivy`O{VKHJRN7R
zJvew%_Q>Wx<yT9d{;2R<B7YyWnx2)FwbkXw(;H5)3;aIKXjsla-;TH9yJ6J7*ya}#
z7EIW&hk3P%l8VBEmzNtFCUmsC?-%%D*w~?RaUrAsd^=vQiFNz;+vuriJa~C|IisWE
zq%TTK7FyrkC7bY6XXz2IB|@IJ>ZHVSmz#+Q3C`a2*=3tt(#_wGPp|v1sxLol)@i5J
zF(>E8eN|EV?Z;a@M{5<6)53!L`)ZZldUlk)4x6mz>*VBQp~b?m;KkutDjE(#!ot<x
z-c<hkQyCq7+qs=@szzYb>n@e1X>;ASu6$>et1qZ~yK3v_sN+35`#Jxyo3nEK&dZ2$
zHJV(sW1maNfvG!f)nk&4)x<wZ^WNH9;bObd?OIw~@b0tUeq8gpko9C{s<IZBWM;{W
zzt+>w&y9FqXkU71Ve~r5$G7fJ|5^9{&pU5hqqhF&{z&Wk2<^Z-zq!-*>c!rDQyU+a
z`{mKWtb4-Y@&C%EX->cX&3pf;|F@&tW!m&lStzVOn)SUk*N4^Jy(I3fx~kI8yQ#U-
zy2^_rkH7n}GOlSw%iRkPz5iYly4ZJj`kdKcyyx$e{QBS%XV}H!h}rY!U6}dj+nyqY
zdz((3fA@?1W5pg(?XW4@;d!8LM_8D+e%zYS)ne^m85l%DU$D0}Jvu#IUpIQ2P5r-`
z^>Luye<%Z!hM=J5im2Cmr#feETD^IFd&-kN+o$)(B#Rc$k$N5U==<;5x((Ya`#)Zv
zktVHo=j-?H^+k_&a37w2|Mt!GUzhw}w3+4FSL5_JgKWE$68qHl)o)%c(+lo9Hu1@)
zUzMxN)_mJB-^=Ay)16<Dohk39N0i2uT`YUudi&o${(ike)kWH`kMrKDntx*nySI7O
z{pm9WKPMgj(wQBst{402S&FO6rq!pv#r$4i<FMg6SNp?mnPvIkYajjp@ay5hXIV==
z6c!hJ?Y{PE)6ZL0;(~%7rF-{3s6PMwSnoNpOnY(W)qTwpc7iA0>+O~jnE$WxuEmzA
z6}r1j^!{(2U-IVkmc}2q-0u9(ee>+uw&LgKIyyL{rKRQN<-yfe(*(zb4i(?u#XdgP
z`}fbEJ9qAsy}7Y)*|J-j{X3JBgIT1~YL7*0&p&5XW1TxAO0FvBo8`W%Q^b}rMZPNE
z?Rh_6Zo{OGw%mC)54p_l`0?xNvbdW$ZakVbY`XUID>d${y5D8J{93=XUF!F{$)~3N
z*tGipy!u(U3eSBDZtvfECY$xr=fki0Hhk**oo;G)QB~;L3yVW%V|qK7%;SG9iOqet
z*_}JbG*0kzzR9OQf4?%ndCPt_(&ki5Z%32eF=4@@zx3yC>5nfx^?%OV(jO`&D@-fS
zRVXQayZW{LpwmLXonC7@J1$<kc3tr0i`VJ90#+qHKk9eNz5Cm5jr$Ydd~S}schYIP
z+*Q||?a$_%oFC^S-4Xt@>xcY{tE;cSyu4hL%XN3z+df(AW1t4FE3c78!TbIH|83s1
zsp{{quU(?r|NefDZ~J-k$EWg>9a-g)Yvz2^yu1Hlo{6_h;CFqQgr|$!wp>;GRkteb
z&}(b;2-c5LTh>LZ)hY)syZGVVESD*-9*U}*e8IK#?G2UOn)uzfUMOFl!vA;P(pVlZ
z?fzFv`E&NUxoFS%%^M<hA*D1xOK{_+=JktHTry?1>)yT{CZc5@v3Aq3nqOt5%U6h(
zuiMB~`C!w&%-{T5isIJYd3s_ozvSd#Rm(ZMgI>QoCHLsc=iTpWrU+jA?ETKgWUu$$
z`TKU&9F*udf9%?|Z<m&O|N8aIw)$Jl<}_Z=*n^1Xo&HKk!}6DxRNdW=U%7JS@9*#B
zZ*NU~T~u^T;>zp`+kbWZJ>OZo_4r@U$&6|9Zf`TlYPK@fnw|W6!}EWZxzDcWnzs0@
zQEffHnZIzW*3$g*GYnS6Ze7WJ*#5w&{5_TGOJ^OL(>KZW(Da7zqDPT_@fXi)tbM=Z
z{Q<|Ul`B_H@k%v1{y65^oA&)xjxN2w{A4d2u`r$5u_FI|W7d>eUM^W|m1|lqt+<uG
zXY27-w~y?*Rq-ycdtux1>9G$(WpCbzu|9ujb0FW5&#P{Uw>17Z^XkgV$8X=>-C3M&
zZod7(g$qwlPZ!^^e<A||*UF}^ujRUC|NZ@)U&bOpMQE;7>8WGK+FlpcHRRvDJ@uQv
z+M?ar{`Xgf=C7Q;<+&02;rTXQ>nzq6rtO>O>TUXInz2-6X@eH0M5^^>HeIP#&u8=&
zy*hBFhGC(a;L)jTl9H3>t~X8#-R)h!`Rcy)9lUo+{`ky1r5@pWzVWTs^S$wAnam$$
zt8a-+X#8<TD`Z8%*H>5H-QB%DX6L8J$NQ_ly$KBe|1i{#kwHP;K#YG*<erMc{dIp&
zP0=iVdTMHutV3r9%hTDq8?LJ;g_RZUc>P~uU+nj1Q$O8sfB5*n;_cK)S2{W>riGq*
zR`_+{vZtq>+~xFgap`nfvP1rZ-tW@I8hR!VVnhD*IqK`%|NnO=P?3?*GDcHN``Chu
zGxlBW?3h}8ZqCkm6}oS7KmCf>w_K(}J?rKY2Hqp-X$c7rUcSuC&#(XS;o-Npw~rrv
zS~F9Zf#HOGO2C5#r-c%>RbP4}ji-F~SQEF`>bmGtpJ`4#dB2PUXV(c|njg{oK3MmA
zs=26Cgr<DVF7X+a)gciEf`)rORqec+HMzI4Qg>>{s&LJzm78}d^m9(rX3wqDKfC9S
zUE2QD&h_)(ua}?ybAcqkXk+%Zs(mN_B)yq@Shnh)zRu6vhQ)i$HTG|uoPA7jclp}I
zpD#OK-KZ~Lv4Jb@>c_9#HymoxX3d(_&Mz-#l5t^c_Vrn2xxYR>b_b1+Xxy3L=y-L~
z<jK3s-|s7Yd~BKT?6`e3H&2}Cc->W`%)9G<z&``4zlXeUncmW#K1VqDSysII-nlma
zqSKdGMYUX7k+L|aGX3kl^um`H4etFukf-#@Wc~J;2Ffc|avL3b@Vr`_Z$nYs)sJ^C
zr_EaCdj5ahlKVo&hBhI7UvwR_W|Wq$baPqrCgJ%+>-)z~c6dmAuls++%sc-2&0A~T
zj@3wiee+RI$7)`OmU-stnyM17=WDOuJJs2-BtKs^>(QPIRfpz8DY>cnDk=T^T5jl~
z($C3ts;=0t*UN14zPr1MgTAxb^PIk)bz)M+Q~&jI6_su?SqjXToPR!kd*0nQH#g6>
ztIY!U-?_m7ab$v{<I<$0q^PY~rN6#py1ToVmzT%J-re*4VVlX1dpm3I1(}szPV9ev
z|DE)b>lbqc1Xo)3ZWCJG>XCWp_vYW%H06Ka+4WT`YA0Xp`u}ac(~e9ybo^-U=hYUH
zCm-+GekSgfdgyDD;-KfNTwILIno>JooOt(`QGW4~pp&<w+oO2ny!VJ%o9)!P9Py_8
zx}%P_i=h0WDW9!`o0$JJrMOJ_@N#oYN5}m$rE`kT?)!6ZR$uRHD=VvO`F9Fex_{c>
z%l+!5!KW1syRQdzyg1e!p85Frq!auLi<GJ=e+I6YB06RBO`9EUpQe6ImU(sM)vc{*
z%<T`mD?;7Ry$_$>;T5{Z#bwQ#2cMSCp3bxJ?atp9%fClxUEFtHLMiEr+qRYe7R#?N
z(Ry)mtB6uii>v#xcu9qyZ?DeP?>KR9Rgzu3*_~NdrLU&xM%&f>xv``0@$K#Tpiy#%
z#G@N>_`a7j{_t+&lidY6Fk!KqZtSit57z1X@y*G)d06@K-`<YvkABSkdewzz<3jJO
z+u_#L>o>1t6Kt(qpJ86+zin@{zx}3TJJkd|mFK;U+FJJb=kAyKUuW-44&h(@#qYp_
z>UYxH&xFRc#;pw8ANkqstAUuIOVInJe~-PZpZ8Db@O!K5f**g@mK)8_ez<b?+_PIJ
zJO7#1BVDDWkt=F_e$nkytjV`S%snb9udTY>)1%C>OV%cP|J}NRUu$PeIBqZg{wz#H
z?Vjz1tPhhGztpw%jd*!&|J;b1OC^6Nt@AxEyr!w+iq+XKzqU`eXkT~k#MP})?0>3D
zb{oY#S19^_<D2*1%}2uP7Jn2k=hOT6XG7@u)}C7@UiZ6+eW<%|MlW{PmA%!V)YZ|!
zVfTNzzAGyO!-)k8g63M4TGjvC<1*=Z(%VHYTAkffRfIWR*k<Z~_*ct(NkH<&lN(C$
zPYN#vo-|TwY-`+iPG81Vud_R7+QRGw%1_d(b(OTHc|_}+E7hE8V&$YI^!iD$V(7JG
z_V!xQDJDjO|G$XN-EnofzkGT5_a#eIczJvGL^Crugt3;hGKhpSIez@L-%sO`V0YaX
zF1zz=KM!BOc-~g=S^9tZDF?+nkM7j-3KDwNU2j*vO!oGbm_OdTi+I8_m>7P@&LDAa
zGi=YgNz5PIJyeBmmP@lT@h<$JAG<n6?)%5LXQLfUt!_(lODc77ecJt7Ytxd<X0@xQ
z|GZL6ei!@o@1#}9H+Q|%j|+J@{bceyy9&AKt0M32+GnTtj*lTAMzNXck51?FFV8mb
znOt1aYPfl8{=I$K?P0w-{{q%czIVs8{)kq&lG41ViPhH0ih`at%X#CarX1p!qi-`O
z?~k{kp5li`3plwuS3J+^kIdEjGc)P4`nQXncFos}vXs}`uUdcVGZTYDW7EcphWV<m
zcD+0Jc5}8)r=rrMNIjd9Z--VouTJaVm=a$-|Gr^r!1lK0vYJW%Cl+&R-L_ai`&w}2
zy)X8XTYD>AT=oP-*$X=?otfvadckM6W}xI>8$G-K%TLP3?R9dQ`)r%GOZhVvh6a1S
zzY7@lJ8hq|wR!D_edR`nr<iRwFcmaXd$#p<S@E6s*XmYo52?#P`=+*Z8~=tsZ$E$N
zJ+r!-HNT@T?UqJS#GA;+hr@Fpd;RPOb&Dg*`1k+4FYt8j1QV^BmGd|MaJnV7Y-!!@
z6Z0qjxd!b6uio-ml!1Xu>Ps`zAF*k^OTL_3_~+7Ib=^?AFT4LLz2l#&WNE5;TWp_4
zPTbu)9_sh&olo6c|I%i1{CbyVdBOW`T-md0<-{)alw-esS8d>`TfFY;ksr(w3=1L-
z+cPuNPdcz-k?rBn38G!>`F7rsvWmjrKZaUJ&XQi)%Cfg}VrKZdIJ?{r(Q@LS)J)VO
z)h=9E9dOfSiY<?qQ<L2hmD46)PD!4)Qe-jN#Oj4}QR-(M#UQ=TiqDERGflpDO%sxS
zx%{l>)V6nXuDoT?eje<##>LA;De#u|MMX!UyIyLMo7b4UyWD<3LWsk|ck`NwoqpG5
zuDz(}82`iVncJ;NFD~#eylBL0%UwKqjqeiq`?Wu2sa)0x^wJR%%<M3lmId#h)<3`Y
zje#M6MWLDLk7rIugurPb!S#MSGwyHp_Iq#tKsLiPW!l%{PgmPc>Rj=4j&SyzRr~gq
zo=E($sON5lZrE=9oa9xjYo>j8_p7Ac@Op%+U;2bL7MmR<@BX~{`}{}f@;f#c-$&2r
z{(f(3#hX=|Bl{|o5~Yg|UKE@g{@b(Dqpm#iw%h5Bj8ET$r!Ia!$9m(r$91okIB$>1
zp8jW>f6w)4Ple6roUPec_y3*b^`(sp@BX%Kj+?$N@7q58)n-*66HMQCuGZI<=7{p$
zxo2B^rjpX6+AqK3cOR5nvuI(ivwgn3(x!P83BMb5WghpctB$i2$d7y)ZWqJApy6a7
z#{c2A(Z4C(-*4Fb6y4}@Wkb@BNruM!73{MFi_3pEm3m%FJ0fuLc*~Q7^`d--#VvjA
zzx#dU|HH3`o6lbLc=-6)gID5bg#r)V<_??d7MuKPy<Pe3`}hAZ{GK?yP4e#oPW`>d
zzQt^;`fsU}bZTY&>x3Rlt8Gtjgr--Yx%c<N{rmq<x{9~?bIs?pHLFz$oPPHDwVfRu
z9rrW-u->oUXRenOAG0y*cvk=Kl`c#4686bj7d<}^CTu<@*Ea34ZGC<He|7iiJA^Lo
zo4Iar`F(Y>mj^HP-us@Kc4W^DgW5;8u5@}&-(?voIPuM^O}5wVH17X9IaAqY#;@Ib
z<ljeaf4%YhyJ<ZASB&lE-k6@r{BdjVhwJ(b4u$PJTpwnuEX;ROvTL*1`_%6G#+O@!
zJ6(=!pS3CD-kZz){o7M2)_17RTl#y~*W-Kp*T!$HTe-I8=aSk#tAhVM&slp|eWByp
zgT=>$1vjo=XV0V7WoebkCdeDT-QlLLmdJG}!=jX5C%5Rz_}N?67KKGCEy_QAr}=z-
z)PuP(YwzFs|8;|ikl_6tmqoOdl0+TjRn`Qo2woSIt(3H7>(^2jt6#_NK6`6X`Xj<y
z?ez8kZ-1^`{Wqv0)cs$6gt4HY;C?sv?Pr^_1uo8=%DiWh?{mBAEeFCLi*EemrhNW>
zy#0ZDbr<FwU3}(hhsKL!8^ORs?a|+ZJ6)J~?F2cGOk12DW$))PUst=n$}m?!PWurf
z1H)cHhW+k^_t&<^{xYs}{{8XpoLx7~FDNQG?rfWub^Xthwafk1c(^>u(!Cwly=(Vh
zsnrV(q}H<knxdklWN&Z(bf3FitwiL{*RK|YJldL(7^(MohU&i4IX!91)D5SX&sA8o
zPF8x!`Q2+CT4ZZJK0L#8onDfWl$VC_TAlMM-~H#RDcxRo)+;^dNwhWJLfOZiFDAXu
zwhj({dTrX<Ty3tY?N_z3&n#bXWsBrGsatF=OFk6J8kB$T+UV=UdnTGMS|o2qnE}rc
zW(Ef}(2RTb&6TUeRA;6r@lIT~Pvi41nR(mHCa*30v0B{a%8K__7=3$!gF8EZeUn|`
zwsNJ*lC%%+PO2(BioT|+erjLxI-x`I`zHyncU{Xh^V-jjp0}|vb&qUS+W(Z>fBk*7
z>65A3%+<L(USB_co^G)`HNflVKfS<H*VNv!hgxrZyz|l1lZkWvqIe?>$_u8OZM}T2
zqex+1_*FUPj|>bBAh&fsygf5a;5GMWX-QTmU!^YX$fL)O{rVjqFPGfe(d+xOQ~M6L
zgsxK1j}{k??3r0l>i6de72jL4_|3}{)t{Q3`KEQpy<WC-{(diT@$TErxANzkKEDvU
z{8s8E_D9y=wj6Y@VPFt=*39&0na*|gv)iXnpI=}9^}>XPh6qc+iApc(Dr9EgPd*uu
zT2u4mQSrCwDi;ec{gHFCG<pB{x&FzQq28yne(snxDMsmHmU-##tIn(6?)NpD``qSF
zS&L4t!n_r!!k|!Q5H!+YP#1x8V)m$hJNb~$_WyU)w@-iDoi}1y+HmRG(bJnsUA2}c
za<Q<vxRm4^TPHrr=VYkdt|P0SPMjXUdHZ><*aPZHmke^V^i`B@ny#N7`6&O+@9dQ>
zDtf;Uzw<6tQu3;Jc&Xw~wdK^6chg>O{#~8+U0o?DzV?SwXYJ1e&7mJw^{wx3tYBbJ
zP)Z4Ch+pdAnb6shxP@isu~l0brvLoqGIjQrP&*5&rJt{5=Rb9ExwL&dpJ`!ShHau>
z`?hbVye)ENKdz5=iTa;)q&-1NsrCHf?}4Xuv$A*uPp;qpLnGDm@S4XbUb!4O<@fE4
z`Stg)Q|4&Ad%W4yZ!RBKTj|=^NgXMdRFo#g*NZ#5e44Rf!FiX=*VF%bzqS>PDwp?l
zS+ZmQ`#Y1rWF^o4@h0Ud&;2}y-(Hfz<ySWEf4JdAg_2U^fr!Gp-&6N|YT&KviA`AT
z=%O;&es5V@M~<NYKLf*kP}O9!)8<X%btZdF!Hq@!`S%qv{U)m>-?!U;P^&}Z%hA;@
z@_AXF=1dW{QCftpOJJDs$A0@Y7Z<MijceyzleMnRInA@1GkVI~@GqHx<#z-*kFwbC
zDk+5t%7`f{E@G>Fu5o;ysJ=hnn=5<#&d%z+{q5ezRgZpcW}e!8#~ZzK5qPBi`l;-f
z?lm02eEXzL3U1o0G8BxBe6#86EpN-fH@jaO$BGGGeDgQ>Y)g-g)`NFjMU-}Zd=z@i
zg~{_tdw`|j-aUUbLQYNUQC)PjHtdi1L?4$=D;B6<59+wSDC?$vc=<jBrRvpP^>5r6
z7#Lb*4sZH7uZ-=(ZL88(B_AIhy>TOA9rsV!p!GX_)#shq)z%K_ei-Sd@qdfTF#lNa
zOf7j^j7(axkBiHs-Rxn!aYZJ7m--2Z?fROp?)Q4ulzX-tbRLJ!dpg1H&s+9)Q8#vd
zd3|wV;L7vLyZl4q*y2wnKJ0XzyZY4Pd28q2sc(<JHTUVy-k5Fr5_d0nSg#Fy)4kKo
z)_((+QqXIY(rM>*-ehOK)PI0u*W6v^*L_Y1c+RhQ6Kh=h_iz63=jKH>?aE7>h2OSJ
zPhY$1?1??64{rW;`z`zO`)4`@ORS$;ziI0Jn;Itf<<)JseRqEu|IeQABJ*mo;$pK;
zzZOq)T`7MBa{h^@r>B(E{(T}m3=3{Jv9V6o5|aFKeuDR-MJgv<mVi19XX>_BO!8W!
zI_XKuB{PB79I6*DE?GbSa>=<hDuov(rOb3tIJIzNjF6zH_Yz&P=R0?JNQukN)>%BK
ze=>iN=Ny+wI?9dfBH}$%rn|7TLpmAR&-WHTY~<l$_+Y)kj-f%iW@2B&lxi7~t}m@G
zfBG+(zyF`}#J;+9?N7RoJgQ;yTG&^2&+)7K@9vV8nXevnA6fS7(R~X!1_lvFQhOEf
zm6flXnN~E!D0!aS=c{l*faB-=*g5~q&wu`Sws=27T)Q4`OZ}Gv3=9nQVn!MV?yK$J
zx$Jz0?D^uFvx^qi-v0L1>gINnQzi4y$6I;6IQM_&TD$*E9X~>s-%x61VsJRvwy~mN
zfA9U0KNok4Z`+zu;o{PBbIx3M{TPew<)@9NPkW{R*XG)J`P<j39=z?pUpnQV%-m&v
zmz^!zq@=WL$*=iuV_6v(emtJw$Qa^yd&$0B?^30tiZs*LVdjR0Yx9%MbN~Ley=)~c
zsQIoxs!Hu$x!-~7hU??*Zz{WKAEe8`aNvF$57z=Iqo?%|W*=`mD?KuQaWDL)z1Hrs
z*Uc?|&lWFPwoKOX-)wd!28JJ}4dg`{qHEUPiJN%5bq{B+=QK`}FIpjHf~k`6)9Y%!
zKg^!wv)M&+V}#2RMWM9HNDr0%-(QSOg4C34lZ%CGd(W)*T4dkWS`X6aHAN>qOXXD2
zgBJ_T<4=W}U!UNzr&3r?IAe-NYKqdeb78Yol%5=(E+NVJ#cS&Sf1wNv4K|9+Oa+>j
zkG8U}R#0k-kMvhgj=Th&4#_y4z42CUqI7U9&L-%~+?ki2)EE2z7u3GAz31j5%QMSt
z1Sgu5{GYVbyZ^rGyrnZA9n?;$6g#K>-Pc%nX$y<Z$DK8WFTW+bm{i?*!Ru|d<o!E?
z+V4#1r?*VduH0W=_kF3H<mPbbyv<zupDP&{94@wRtYDa$B%$jVb#00}%g-|#=dZeR
zFUyVnaenN~gK^q2ZTdws_TBy7`Ty|icJ|q;CmuR}_TW|hS*l(3mc=dX!S&O>KaPD}
zX8Y~l-ANB8R!y^43qJq%cKwkqKj*Zy@uqhtNhsYisCr&)Dd>6r{L5Wme#EX`ID4MW
z-P&9C%R7ThkDhMcTYhiK>BY7E@*!*iO1CzoIIrHfc$x2ug*8=W9~)e!+Y1Puyr;kM
z)t)zt-d^M0{`arET<jw+m+))-(p$d%yZ`qJ1H*z7j%=&}nqlvDCjLGboBf=BMdbBs
zg<CcWm%P}|KFuoBH(T)a{+dg6byqe27jL`vaPox(*B)3t?(Db`eqEj|7)#r3LaHTu
zvRUo>t4AMKZOeJR=h*`lrKD8#_*x$|uZb=i9UX=jo(l_3T@iPCPDjxEebYbiR-cQk
zE-FiZ`mUHSHr~3l<@AA@+xwS&Jt{Z5=#}tAoRb`!M@2#y7=B1fbh8KO9@F`^^ZBZ5
z{i)|o+VXbUZHs&+IWtiIT}-m5$c{Dd9p+DRae4Cm`SO~VCl>PyM@=(VP-^Xe-LP|Z
zcu#IpR>Hla8;c4KpGg<E@$F6A{+>@8H?&0jR4|J3S`xdpR{!&j2kUxQ3kZ5HzF$90
zKv4Nqk*<1Mde~*B_@5Em+Ya{~WM_Y0aqiUoeZ3s@RllSJ*A~1hKOEHG(ed}Zz6WTC
z^1<sBCyyolv14Rt_;YB#B!kw`4`<TW$1k(6y_|SWQfbmVoBx-VX4?MknYrNM`Z$5W
z#ruNUO$_~fRh3@dE?*$EbeYh^S&cpkm+Atx3R}<9j+-ZQ?d6iI6IhGc%QJ(mUw?h8
zs-heew>a+biZ0v3!QZwCeSH08M$Ibrz&pP$XKz1ytIlk1ZCa-G_x-Vt_odC&+w#~_
z^WFY>ahVTZem19HYX6d4sTu#qx?n<7R+iS%AjwG&zjHD$Jdyu*mLWiMns3ka4O@5l
z#TT8=zV%Zj?A=SNEsL&qPJC%QlL;|e|5$L%vG)<0a}5gr%PQ^KT^siM?dE4PyIzIt
z_R&*6W9Fu}vpVPUgP+xvV&{JA-4(kRbFH|sT27Sv%4InY$)`ef;8}qG_hRQVFf6!n
zxF&!hR9HZ;)9Re{y}1tsCn^<8{`;r<(wf}3xVnFTRbM+ZF)eyp)akPN?v=OU{A;(R
zU)s0-^PKkkeG_W0|EV_o&2A=bQ=fmePW#=QIdSJb!YA~rXeyX0DaCc)QoGgZ0v^aW
zt6se0LPL4rPta)!3cN-d4DKzW%PZFe+4)$SW!@|Ne@0)6>H3<lHIpm7VMFCw2bO;P
z@x4eZc=y>$>EFMfdFStPWZ$n3s+Sw8rg7J$`F!=^+qe6Qzn9D2T_)jXfx*f9vggM_
z2I-4b_s1reJY8`1{nDBG%jGxUzq)Y$`Q>}=889$7u%!fmi;;udH4UVmMpv(sURP67
z7PsD(f4@NBQ?=Zw(S3d|_k6t-Z7DeM%iFcizH`K*{AUDvyPpL&<!l56Ii<fy>^>0I
z+~F}zkn5?r-O(E9#_6s#0e)<u;vZ5!W?Fs?b`cSrX!GghWram=K09`LOf&RUfA**`
z*5yiarM&RO%JAt~E=NxBmC5b=Tsx<yV@cur@BX=)zm(2hdpGOPJ9lOVhNGND8Vy`0
z4$q6Z`&?I5DePV4L$3yxd16ARZ};7>vC!SS_Qm~25B4ThM$Z1C`}(-Wm6cbwW}ju_
zJ1qWjRo|TZ>(?1&>%H6jXn#kF(j<3Q+nv*#_K1aQzY_9PTBOw3G4*bR;G6|EhmK!w
zJjFi$kR7+l<!h3;viIt4?a7RNedC;!oP*1z-T(U}1IkalJDzz|tLB&AP00_2URpm(
zs!ndWTBn&cH?1!#xDh<cEotbYvVZ%s<j1@E`ue^~m&d%CXp?ifsmf}GA{PULePiOy
z`+HqY9sZTRy7KbT(e8~K@B3=#oKjhS+-&~SxpOLo4{x`Ay8VjJiZi+M%lNj$zn#9Q
z#&hlEN4vXo<z*vGD^6bg@@=l#nv&P|{m!UQs=xInrF`a8_Ps$%<g@<H{J+(}_s*M)
z?S>krwbMJT{iR}Gn}2k0Q%`?w$zFZ*t>m6xn?7aszr4*J?d@i{W3SS*3f*m0I+~^F
zh1-ALoUZY8#*WNC4QVg8J9_bbj5f-9(YavD%jGuRdD`MVxxsfIY`0r@=p<6>OsaA_
z8$-hy(DXxBD%)SzBF~s9Dl<DwN_a~)-Q%0rv1F1(r%I$7Q|!il2FvdjEBAKy`KnD^
zx6fn7HxsSMW!Va1A74J*_`h`a`*WbCr0wb+6|0#Z&M6C&Pt|29RmPNrZ0hk?khoy|
z=f9_qvKVPF9C&|V>r4hNjeQ>OI;Xyxy>Xh@XYf6|-eJXJ{c9&bUD0@*7~vXX^s)5F
zqdWcQ&R6j;Ff`0>`?-K&!d!;>IZO-(;z129uTL|6xF{)2s_FZYx6+1z;edSWJ3a;l
zuR7DKo1$;&+KPU6FW->#qwILi^f2MXt=0?-e?(6#U~pIxnE9{y_vg)d!W@DQj0_L*
z9obkJlpb;KkFMfpVEFJo!48z28CFPv7RR^8Xn9V1x_qI7bI1cLDOLY<v6VmcyrQG`
zm$YW_N=F8KIWErsuKlIii#sc1u5V>za1aEI%Dfheo44_mnYz91T6MXbb_<@}U-w-7
zgmXDxVZ`4BeZlQN9?R)3`52!Zv38S{lhUSnbu5x13=IFwQUV(67aV^Pxoz6jk}HpF
zulLwJnC@kB+H$tr)eDJs?JV<d?R*n8A!f<B($2^e`Fij7-O^8ex6=3SS<$VNXZ?S_
zX-`S>xvBD%`xL{zwEL@0J<+mg{;sLQ3=AU7puT(dr8xQPzm|4%c>Hpk?LWm!WmRV6
z{#g@N#_g^<UFQ4j-(!Jeg}srgEV9-!PJDQq6>>o#X_wyql#fPxLQ?NGhl$_c@X5YP
z?WN^*wfSuEl2N;#$lK~LFo;w|OlJ9D9VFqny)2V)N{5jzuW!A^iW{jDUKBd*OY2?b
z<`i+jH0<D}&P-RXudA=G(-WJz=0?!l^Yb)rWdALBpJMCiIGcl;f#Hs`y~cs{m$$vB
zD49RiVtv}xUvHxSd+O@xy}Qk}u&-}Ihlf_@v{S4uE;SSTJZ4^5<lMCH!|B=5D_!?g
z3(M(W)(Bh_Qz;y{O+>w)yHZ$cvW{@Zi_jHwo?hM2|K3TMMf8kskzCMpZ`G43ozu1E
zA3g4+vP7ZrMf>Zekik|LCdSNZI@zE8_C9(#ssCVN{G-XNOuds*PfNaBKG*WiHM!lL
z)g3SXKF`Qxl@mSS8p*(LKsF_y;eOzvs<%RwW<p)3>mySy|ELMNRXp!0TmM;IACr!x
zZ#&Cw2~P2HDdwM}C;hDa>5(qs@4;8C`eaVMyY*t?nw)bV-j>D1T?v@;Ew{G%^@PJ8
zTqWI~Z;g1nZ$s}&J;9TQ)|uy-cXh1tw<*YRxcgq)lfx$IYH{T);nwp9wuPOl>brk4
zRCoCWDba$|yz`vbr+oKUQc|j0?6_OJe}($B06X4&r5SfV99`d$d*i~P^Y2v!zs6T3
zZK}Q6GQIsaZ~VfX&E-9R<CN;Q&fIsr`tQS&%NQ5}G7@#<99Uaqr{9Ts88Ww{!|?X2
z&8u5G*#swEsQ57Fqsx{0!$+5GQr}&sXI*GNwIk?!{;i6GSzoqf>hWIf=$Kl~m$lm_
z#Lq?K&DKMSKa|>E-mabh!+uX)#fF_ynl*Q1V*W0QRPy@wE3k6kg5&wcdmisDyZUb5
zzVCa=e3X<PZM{+T@|$h^eQCSwv&FmApFd}>p8Z{-zBhB*!$Yy4F;w%u-3B#3_s;zr
z<^RjSa@PayvcuwgADP~G^|#{sy2<7KdP<A-US4*;Z$f5x;g`KzR;wOXUesxA`~L3d
zz1ubmGifj|EOBOI4d6{&&FBnjepcU+Uy}dz``!Z_yDUvj^)4^?y)IdO>cX{I{Xfm;
z<w?AIWR!com-X8$P_-tvPtt4Sztie(ABWHVWS^cbYsbl(rKzj~89P;ayK1fYm$z3g
zZ2SEwaBUG^P0hr<z7r1mYu|^(hr2IXym<Zk$%hqoe);>1QM)ta#<Q8aK3h6F*2m|*
zntC?v$b>`b7e0LsQi}W&uB7BO+kHXo&Rw>LmA_qW*B5%kUHYYT%ar$fTIP8yS-ifP
zgNuPd64ZNh3E1~?Qr|3>6430o(6kPXpI2Y{GKRXBtXXlU!Dyz@7Rm6{>*j9@?$isu
zHcQairOw}ZQQV!K$F!a_p8mz>ll*j!*@n6&7uhWZKYEwX-BodY*UsECrS$=xb<^Bc
z_v|Zf3KdxI`g&LDC3B-xDc<+%JY81G#W@OvZO#c5zrAX0jFr~zz{U64+&1L+u?Iid
zBFMndWqO94H9#`4oAL6CKQci-WcGd96!v@-cnM)r<n5@N(#ukoxExy$w!ZBZ_n#$>
z{_%S+R^wP?VzkJ9$20XeJL~;s|16E>T_~LG`S9}_=a{vx{`*9iH+9^*SA5^9Vj6q*
z8cqg=pRGJx3#@K|7Wl<%x#C}yckZR(+~xcCZJHTgx@YRjWBlchR%U~?TVJ_))wEM%
zU5co*boSIMvzcQz);(L!saPldjK})H^jpc_^6rO8aYtJycQP?J)FtT1IZXAs(JSK(
znw|cby!^bp&l`&kOY7!Y3kpt5J-libkKo1Ar_Vp$S8b}Ipz!N_Tx^^tX#I!cYc*3P
zrFp4SH(3Sx^F^0RR6M9VdiD9cbMF#YDhGN0;Z|i}aG2P_!?hrYjrSH`(hrv{x&L(d
zm(`We_Nsg<`2Am$mf+0puT#!LS6bXrf(%lMPn+{N=1hFZ!p5ne4lb>3oVne-l~q{q
zu=aX&ubsseqJo?`zs-!VswoL^Y$>vuVzn+1G#uy1#`>d~#nSG~sSWr0I8V91x?6qi
z!pVn=JZ8Gu?$|PUsxi~^d)q{{%rjTtsldL(p>4@R>*c-D76orUw#V0nEWNdT_w{s5
zBg2@=zv`t*O1)lYdC#(z#5jshpO^P|FaI00MSh>FW^<-aa+%g4lKAkVi_4dvl@0;7
zW~s;tPV5OS>)!W6z}4mA`zAZDe{SKr3=AS(Cl)Z&cf|brx$}EuzNX;BL(g~oZnnrg
zH}Uj>iFY-MO_ijw?>o#0Zr1Lue!K6RO_UOEby3-d*IY`oYBN?C&S!a=W5&O3Vb12d
z9_P7HvAeJ3JTRycT>t9C&%dvP0uQ~Oz50Co*Uz8hcD#4%yUwnvq_lGX``FZp;WhV{
z<@<8*ec+xX7<g#;c_AgGz5k@mqJ*5e<i)q2UH>RWrT_Ciqw1COL)KnC<2iS=5T{EN
zkDw-#`Br(0(swuBt(}v;M`>Eev@<f%o4>tC@5xh5>Ui<u$kXUD-Rx^8KNgm}@~>UQ
zz!1;^ni%Fh*%<u#-{n=$pZuP3Ewv)&U&FDluVwf<&ad3M&(*oq?ZR5uyFr$Fy<D~w
z`J7MdeYL%K-p6By3J>qR?RS+IG}%=0XPxZjs%550@q3lFU2d5g-q&aHL|8C$%Blr>
zCIol4J+qAKHGcT^<}B0KyY&jg?e1DFD?WRRxyH`xU7lQ4s<~I>pS!t_rLL}dDK|yI
zAZyRj-xpT&g?*}?y~}J}^k%=u8>RKW&))m^+M~E%E-r_@37cu^XJ~!@Y#|z!{MvGT
zX>+y!Xbe7kyWBIS#oIp5?dTA(lMZ`3d*Am}b5Hh%$1^fG7#`YS8GoMt!TJ|hR$e~Z
zEuNg5yi>0wc+>ab|AiIiiSgfVXR5LLZGH3bpq7l-yod`IHMF#5vPiL6#yZ65b#_lu
z5uVt#uF>z>?hW3Ll@<xHr1pII|3JoPrHWqIBp($4$rtJSi)}p@D<^p>T=L)TyT`;z
zDNIeEbN^k_I3ZcV**>?}lIF_nuHiNg55NBSc)z-;YG#H)G&92=lbG3@4p;BR**rL?
zWwvGY%;@GLERyrrD~r8;a;-uCl3;gHUFZsBvs0z}<5kZ5Epc(G-DvTBxymK+$NxMS
z7(Uph1Tchp@O_+Xo$kJ9$2MQvvxh(TulgQw`-;Z*{P|Y<W`IuDdpO^UA++N0=I?vv
z%TJvDy;5o3({&fO-S3}z`q87Q-*@r+7ExQc>V+K(LxU1e_9TXog+-t3rX3LwG)&{>
zUz4$7{@+PHiT9*m-Qo*a$5pj4H)ioH({vgBKmRNk7#Lice}M+<7T0{f`>$VhQpzQP
z+U;xFJ66nj`=Xt{J>{m#f$PEy3|%^Zl-U;;)l7Wh&iuwHabmO;9|OY#?w+H}3=9kn
zs}{<OJkX!yz4W9}<T75Si5k)A%fH+`d^{!Ue#rmp`+GjU_|AV%Q|ik4DVG)Y|4F{^
zc|ymIkf6)!1D5XUbmY^0WM^A1C^LQ5s%0zl!#3}#)7!hza-OT!#FVOPCf<nx-`m(3
z8vZ!2vHs|0>i2!W^GndL;JT}t=Kf!Pmo4?J{NVgPmRo&3$0T2?lG7pkpXkfP%ZXP#
zJ@T$|b>FAeemg4{1#Qc&IJJ15?)*FN8{)Oj-`P}jp*j3W6mRUtteoFIXP@|=%PE?-
zb83f_*_IEVt37Q_r>bAOG4u7a=Q^M@_XRWNUF=J*{Bdqw)S@@1*D^3L>;tvL#W`DM
zxrD5}f6F#FD)Yosmzq@b#GDILXWmtdNfteo_NpoVM6O}n?C1V>Dk6WjXs(>Mm)Yp&
z)h4lX>JhH5U$S4bxUSk|X|?TaQ|{y_wbeiUGMB!2T~l4R@$(sFp8M}EoL#!LWX273
z^;s6@=UzG+diHa{>@B~u{=LwVn)+s*&Bn){HE&dJzI#L0Md^<dBSS;Yp_+gI&9=l8
zqsWR1i|ef`LM$&ADJj{_KmUK)gCDa}WBTGg+%bG<-nS|*V)gZ<(&BfdRz3gm>GR{+
z{-Bj~6aN>l^LBBmvi-OHT#v-deTD7w>lBr|KE&Jpl@R2;YqD?}pR>!Z(p~Q*F3q?R
zKKV|c>+{z-r?wvvpKko8*E~o=Q>5$q`gmc%!wc?<Gdu`aXm0W<eRpSON7S|)NfD7F
zrMg;ElumDRyuBoB@vEmts#NP9FIaTTtheXO*RNZPe~GE@x7zoHOYr8mExLbmrGuCG
z*+(l1uC=Xk_xkef)6DEOO%HEx%?nVg-M{PYZNZ5)1vl@OpRAn!boQJ4^(!WOE&coJ
zYxS2GfxFAz7ML<KG{iUaaA}!lTv+h-_V)I+wryEgU)|eV9W(v##+XI3P4u`sL$pp-
zD4oAPQP|GL_MleBip1VCwo{afPp5&_?tTqZ+FAdWNARQW>#4$NU!G)6S$z+*F+}zL
z_5I(Lb?sRnzrXC&m6u;$UcS4lw79$+w1KhV5SOG#NbZLR2h-2b^Y!xDw0ZOA4<8=v
zn!}`VuD0g%vHK>O``$jyzF*ioTVwvNyggcHk1Hz0?Nv~Ev~@++oYc9K3g23RrlkJI
z?X4@+ZSByR9dh=P)9xH+Z&RgDJD<saZ@=dsC?qC!ZCB~*+uQT|<!pc5*qHo+g_Yq)
z>WKvh7^|zR<>lokdR$o@zCLz$Sz)Z7WALu`di!M;OKH!y&?#OOyZG#egI7B<&MjPh
z{P?z0{;xcJeS+*RoY@uBxqji?Zyxi6TwPq0zUI8{xUzMtq>_^0LoM%{HueAZxOR)_
z$L)#OlydUd*VmOcfBvyB{3!h)%XKH?#)gM;t;@ASRv4t6nep@U^Ws=P$HUvD4D$sJ
zt1l{!{uck~mTkrH;`UbcIa$+B7N4(~-?*jjj>D(pexW9J_Ec{6ooyy4DERyP`}tO-
zTHwXWRVNl4kTf>lY+e5D3gq%vWw)LQC%bN+nkH{7Q<TIgs2L|W$DnX&)bg2^t-_z~
zTJq@im6gGKe0)Va?_9oo`RLJB@YXJ-lz<08GmTQ$#qYnjE_QeA?{8mUTy$1)d>+k~
z%iV69mpi5ZcS+8|&yTM~p1iTdChCNy;riw$`9Fg%g079O`}^zV%a?a|6fX9iZ5A_~
znIXVMp_%FUmaSXQ&$qu1zTFXYuCu$FTTyHvW6)MV(WlL|i_7-^pXm{?XvxG+jzQP<
zJ~^rB;?fgoxiQW%sl#K^-(7*4oSZs>GiOd0`x|x9mE-g#<Fqp~Y$`X+oO$!^?(+XX
zpUZ>SX<AKiY^-K#OgKN!);R6VkN5lk=j7(z-&cFO)Rnc_rsVh{*;DOn-aL3BTmEVP
z>KkX8Iy@c<OUz0wt$uO&w~LF<PNSsBv%bA@DfwTPFingzvF>|t=ZZO*JCpXQX{xHG
zJ~+^reSO{5O`AS_{kk?{qY`L8u!!b~1pxs%F*^z#AM1U8Z|~-<TWf!OP`v)Mz^|_S
z_7d6kT6ue)>|L^0O>l37{S%h}C%f`%E3P=~TypaL&*eU|RUN<Q3rwu~zV4aR!lkwK
zN|WyY*Y`QKY`X_X^~ycXD{3cq)xYQ!@hU7VjNM&kTlQu~b93|d?b{n08NrsDo>&mj
z&?95{>B-5-g@uK0Z*Twp>C>ZKbN2f!m{!K@Q|kE5?)VM$<81j~@@=afg_Sq{ed0Uq
zh`_}?^KLuuw|2W#znOoI^sTe+U0lM&V?FM-@>Xv+AMNE*{LAc-@YIbt>E}O0?0o*y
zI`RIj6MxUiC@XFC>z*x{`Q)*V>*eMC^KWg-ef{{camkAd!OQ(Jv$Bdn8xRFsc(_=-
zudj<;AGcRaglq1cIiR)BrIz(mzdUVLKjl9EkX?G6;?KqJeWoAnIZ(MO*jFiuM<-qP
zrKd~It8*VupL0#0^+>aqtEx!1wL>KF;zS{{&yG4WZTv|;a$cR=8U6l5r^mm{>pI)R
zkAz9@TDf?onc&2>n>V&PcV@i5p}O5(O*rtz-M`7-?bVOve4W8JbG_5A=FK9u`tkd6
zZf<(IIsN>z^YizYz7Dh7Kf~XXf#FB=1jmI3GA}JT`S$kq-ku(}9*K=9CxzCX{;9o4
zPG(i)=6#?4D4$v$eY|8{<mPAZs^@oV{Cs+oEw?=`S#-^t2REY^x=fL&Nj<n_@u_Kz
zzn5))vGb*{VBnK~8^69bf4_E*`f{a$;NxyvU-8_UX!zxvd0F)X!>!d@cfW224WOpY
zHCz2RsLJR5zVw{UUqV99m2G)`PBN|Q_2FA)1#hBu-+drz)14P>`+d8C*ZzXqxOB~^
zclY+*UKP5!U*7)Rot?$k)<lAH*c3yZ=p)sp3nu*f@iDobPj;SNtrg_9S@BEnR!l$s
z?|SaO>enybuUTAwvZ`%I$Ak~-J(nzt^vYSHf9hrOl;f=|?dnS>Ei2PhR{k6RV}Wqo
zye+TH+ADPX?nlP^9(l6YxpB!JJLdl5)^&e&%$+L>E^P{%_RMA|IC$d3i)UwN$L=b*
zxwAOEqT<KBz14=g@2|h;@GUuFpL6iKq2b?^r~A{Ef6PtFez-OHIG?$>d2MYiVo$He
zuL+I|A6&Y4ac9NHMXOi8et5Y3?c2Bhb1W9FUw^(d(B56mS80*9ciLMqUj5~&-l3~+
z>`LwZbLYd}fC=vVcU7r(T)E;@Gu60Uwe?F__~&P5qqpTmF7=waXV0EhAuA7+S~4&k
zWjgj~;ezDIvQJM=YKN~gF*h$SFV~OXXLDWjZ_9@2chBWqSFR5B^6CoFy1Me(uK3$h
ztwmvuev_PQw#)6(zalzm*DkB29KEx@Y3024Q=8l?W%}wsBQxmI`41liy6f*G^Dr>f
z^B8G}@MT|HQ&?QA9lh<%%HZX%uB<#<Y8fD+qN1{6$BvkNH9MW#`HuI?i+6S%U%H%u
zfr~9AK*I#Ok9%Q2#g7jUr|Cu)#qx>d2?`4K_4W13*-9BC9N3n7n}x~8hlwFTGiEj?
zSFfCHRYhf`A<yNd-qV%c`wC+DM3VCF?a92k>FHwk{<L#*c4lAKV+mkjxa!Q-dT5Dj
zx7hal`(oWko7s4ywq}W5f9kOC%ZG=~(b2abJa}++wz)oZFLRe>&r#=x!a_oz%jX&z
z8U&TybfUNQ96vliXhOWWe%zbe+t)iSv?zMw@%-Fe(9s$TejKvD7w}`oj2U-!m7YF!
zEba5Nvwwemo$R$VCY+`9@I2e<sxL1VE?oHV?b}`kE`|fTDFGT&zI^!t+C{lGdi&+&
z{^hT(ynKCqeM~qD>)l1J-EO^7sW~}sUcb)H%{>c_IoV^6777Gwh&(>ldwWBo^VY1Z
zckbRjJyrWSXao1jr=LDPKK}mJ*42J<t;F?W3`{^rH-M&GS`X#l+>{!;+|Sg=$g21m
z=n|YAdhA@&mMmFP_xD$2Rn@k<yT7ii3|94;@&L5uUBQX1b<x8Ejm*q!JOZ6A?)`GP
zx3*|rf9l})@#*R5@9yrtzrX(e&6}0q-`&m1%8ChRVPMDv9W}@#E<T-GT<^rwB6jxn
z;N^aWv3w#N@9*!=zq;z`-{0TW{pane|6j)vz|g?NDk;*{acyn%{XLbz{{HLZ_t(uf
z%YC#<=E04}$NO`0bMNn|++6ndR_f_#hCGKsCt6HkJ@#m!N9F&2wY$sSHZ?UJ>y_SK
z@X$%6|Iy1=S6A2m`r_&7X<6`~Ve({QP=0jayTH%o>IFK6p|*DInl;zf#ab6UI51_3
z$U1GNP|!8bix(@;Kfk}`rxB9|gF|C050`6iV`Jm??c1B14{uC9zN_r5l(_iuQr87m
z+F@%Xj8Z&qZc5F+zwhpvNMq1`xQ3WS9l0*GoiTPrPfq0C-gb3W=<6>pFYi!EnYcVa
zL(O;AmeSYPq)fA7_EcQFabw2oE(V4N?jSb_&p&^ESLy3NKR?g6t-iJ?_4IW8_@Y?B
zKdvWFp48LRi{74>du`3lx3{+oIxsSD*?nnt65O7BeceLm_E#@n%&{mubm~;s>!Jte
zyE{8yUR>P1em$uD_U-L$brCMm=5>ZoZFK<}kJ$KRD&F0(Tp5y;pI`s)PvyGPjrKYw
zCLl-c+qduWv0m$<CmbN3CpPkMxyHBi$=1}=Y}vZC`un@R$;bIX%8%W@zu&c6tg*2X
z<iyQsyn07j83H;IbmY3audk2ao_+lt=;n)MJ~NHe`K~`b@P1d_->UU-dnZkr^zF^f
z%gg=2`vx24H}h~E^>A@%*}L~|Gdq7wOpICH9S!hN40RVfh1J>k<<=}*SorJ9%doXk
zh2W!FOnQzw8!Cx)*Vfd4j)-`0pz+<^-QFrf>$IEVH`e|A_3!U*X$grtdn$z)MHm)L
zaI)7Bu{N5S^Wwt7-R1B9J)d9iH`hv4tb6A=j!)gGH(y>}zCM2cy+el%oj$$Vsg(<y
zUiWkST@dhwgQY2UciGuFmY**!cHg#bTmHR04|ge@m_B8SNO<`5udlD0=iRw+{kk~F
zngC~e4UzchZ8?&rSyT4y`v)r0uCG5|YWYFAu&~g!`rDqu$8OX0V)<mPmaJTP5|m~?
zL|@?NI(q%X!^8e_t)@<xaA9BV?{0DZqFBBsuP!h5&%Lz;bU8$apt9dQn@$EU@LFl7
zjn{S-r|0MA^YVg<wl_C7x4$k@U~g<})Q#Hm;rsXa9R&}U`OdDe`E!klL4(UcjNbuN
znO|Rb7j&`9_x=CR&M<6#U8K;we*OB%lY?t&{+yd@{rmg-^PmFc!*PXXCIP|PUtd-_
zxAXbWvyrqYcreQ}`$vu4by3EhyLa<T7&s^h%&{n3RQvlIs4iuw7joOm(9yxe%bS01
z&&^$>uVZ(YX};fgxYYIN-D_*3=gys5T3Wh3et#dx{a+NCnU;X=#$CRA`R2`=moHCm
z=aXHuXwi;!D}Kzft^U@+DXbN~uIBkU*_cfX3=2G**jP_2Shj3i{r`U_C#!=>m220&
z9d7532@iR*WZ5!4Nu!jfr>0JuHf`U&eN(kU!DkHp&^C}4QDA?4ZLPF@-JS>?yV6%7
zcXyTU)LXv8!^5MXtZZM=)2_XH|87h^zGcf6@Npdf#8Ltp=E&OD-8p*HH8%F|g@w*L
zG<R~<{QGnB;>C|2A0KaK<Gr=3RC`)0BZEk;L^u11nUg19etLTP&zgOU7e9V@xP7PI
z@{a1ize-O})0MZcv#I{}M%KDap@E4(r1Hc9hMk~WIN#phetL?gam9xRQ?<j3V*MsA
zol`3<B(y2__O{B;&%g<L1=F!djGsW4MeVEo{rvoVerdCuqg|pq{LjyS`ts$?_3`#f
zR#jgzmix^GACwcq@_!c7lY=*I{P_0v_Sspcw|AAU?(X(3Dl)n*Dk{6X?Cq<Yo0m_U
zc5Q3+bpZ!PhAz>RfQCIUU%dhyp5fjv_wL=hxmKlJot+PNEn4&dbRnd1n$OQqPit#y
zrA@PrfYQT)AdqMEr={M#bLZN+*yycUq2=Y}$;rv<w6)eeDf)S1WAgVmH$m4H$LuIr
z=rhv@oOeLy@_i@{TN7be`6<QO`LJ>NxhrcTC%-P@nt$ZTk$ttl<Mvc+%)Px$-GAPd
zYuBEEYLo{IMj8kFLB;Os)vK2+dv<ZL`)t$fW2Kg@C#UPj%gD;g+tu9Io_~L?b@{55
zD?vwhFcdI<2A$6}->z0KX2*d|InU3{^`B>RvedHm*zWT8Ik~y=b~P5cx3<*&{-y|u
zi1ZiitUp}+=h<Z5-&b2v@#8=vGw94Rz2ytMy}WL1%Z&~Rx$^7lYxiEMqad#zFa)h<
z{r>Q9d)eDtdux79dhNu;zA&KT#RWxoclUMNj0_C#Kx3g|D?@&Lc^Ta5bg)!W^4^}x
z$8X-)6g)Vv(7C-R)>}+eD?}rDo6hx61_p;l1<>(<hkP|eqN1XTVqL|QCU$i6^!65B
z^krbs*swv36LjQX2rp;`m4RWDCNUg*50o-8eAs<$UF`4s_5YXJ_jYzFE;3F(x2OKU
zU2Srwkm|Fuv(pn27X11I8d5vHLFaTSFT;oEEnBusnl$NCPTAL2q0!ONqE|p?_o??B
zWfrN5-BogOb8lOl+hjH0e|3nn`z)=kpWj>-zCO-rVF1z*fY;Z>TG!V8mEZpI^78(y
zF3=Hn{3!tqPW}o4UtU~%%+BjIU9Wd0=s>*%3ZQfQ_EvxYcX0LIs;`ee|J2fEV_;D5
zj}Ye%5ZR!Uos<M>s>$rXSpM$L#!Z`oYD2*X`q^tZ1RgN(d4F&3e*OhKheK8bFfxM9
z){_FAt*3N6)LrNFv-GPr3=9oji8^u%UToN~q2lYS&~5r#_wTO<9j~Ru$-toC{(_xV
z!)nglxyGO)1GBEJ0aYywtO7<F4i_ipT7Z|x2PA+D@W{Nd;9&j#ztjKit^58i_Gp)=
z<ysB~1_gGI2|<_IA4|4v+_ue)ft8`5iBnQ!fyc|MtHlo+w1C!wYuuRN$k-*~?0opy
z+1c9i)|Qrq(L4+c4tXpw;tT;As_N?TI|>v{O{czgDhgN`q$)2jzcAp%<Hv_fbuCxN
z?k>B!Ds<;MCI*HbEvB9f`;$O@k+3xnc6A*(b?A=Gho|14Lo^xmKm)<yDr#y%7kRq9
zmR`BKbKTMjGe77UXlZr5)?#3Q9Q=1~ozKckPKwt<MT~woA9mWYj*)@EqTSRJl-(9&
zflfnZU>G=>X&pC%hMC013+uMJY3h3Udewfnd3i4R+8$@m?6@~^{NL33kL2Hp-XEL0
zPbYGoz~qX@bDlihy+6o+f#CpX=y3tpo@33Yc=L20EYE*&g0EwGAZO#R*+2HmeVT7M
z<=6z*HnU})W=PD7b8Yk3op<=f)4$1o>t9JQFfhm*j<7j!U%gDje9hJd|4%cf3%*XP
znPKw%^xAM0orP*DlhRKu{dvGdE;%Fll-fQXS$m@=6%2xcA6eOwggSpNow+{<v{og)
zfrsnEbxvz@aRI@T``69Uul(UB_|vHF_Z$A#`X@R(6vSrzIdVN<x2vquBzF5yfs^O;
zIqudPeL8UKh4iOO+SjJ#1-^Yhg`a`p!}CNPxd-cm^`nD+He5V$<AmLf??0nt7f)MS
z_sGGA)mK<hP|*JE?Afo1m8N=RpNY!V_41hdZ)OoA1H(VNlz@i)eN&eS2@1xSZ-_p0
zW$H^~^I*TYY1`lE8|XPIZhYB`elN;pF`bWRS8qDGbjsnkJ+t)om$<t1KXH8D>MS-j
zQ$bB>>!zudPoIi>jZ;(vtqfkESf#)7Uz1mKhS?g`3zIzG?^@on!R1SC_b#O`6Q7*x
ze)F_Ti%a@?puT2OuyAm^{ls7Um6ei?Fd6^#>768Mc;vG8=_@%A3t~PUx6@-_(BPAp
z_V=dhP6mg?ZoRj*WL{pI#_P@9T4^R$GAH#>NbJuUzC67#TOB@W`Td-t8Qav`<DY%}
zirJ6yDSaFLp4A5b*=whJe{sSQw$;~Py-2m+q@>ojeEL#b@A<+^!lpi_S9AXCy0AO`
z@adQ420y<Aow~03IVg)yFFASEJ-eUh-uXxCEw#TYWfY&2d`#fv(xQD6{9QEr`&-xA
z`R-{;-M%A}>+*Raxl*-={oc+`O_*GN+?}v8TePxHz{Tat>Gy>l4z{08Pyb`e&d{)k
z^Rpx4KdYJT)lN~*RNPKS*c4~)e5riZ#l=OEamw`z1+PnFlVu}coPF?b@BN*8KR>BV
z>bR1<&3K-5!;KD&?S%)q%T+u(JD$AX>zsC$J9eW=&gDlF)`m}!UEO(acT@V=-#h+u
z`bS4AJP{T=f5U5kW8kL$bJz3V+vvix@-=UK#Ia*0svq8o^WPWI6r6Zr*Xn5+xrV~)
zk3KOnOLNgQeBH**U&?9uuczb6>oV=g_~{~b4-dB;X<5emefh7so%VaJ%wqgpIG<my
zJyFiU%D@od#K!vL_l@Nflf~7Kq{P^H?#;SfG`s9&`A_xQ?u?pxUd4@{E}Z{=TSZ;T
zOtJ23&-#Z&N{?o5GtEf$$Z7d=HM@0AM~83kR95z7cGXJURr=L`LoKHrJhpEtJNvSj
zo3e(*Zb?dlg4+Lu3^(6j{`7UyweuZSb63~|9{QP@*nRz&e6)O>Swn|}mS$F<=T9Mj
z|JfZKPd;^2C{2p}K3!1o<5k~Nf`S`4<I~@IC@Lv!{dJf3n9e2b)Ba!W#VY@ti~n_l
zfnfm$XiW&`x6CIM)BUH*o!rM7*k)3?+fUPahozuku;cDkc7iAG-}66tL-pb7*7j5F
zQN6RCJ(hX7nThApKkdJBWF}p6ocT+&u_L0(|5N3fJ4*FmqawK%PusTm&)L~iTz1-g
z+Nt8==#q6-JLNy?!X00K{xDWb`mwY`YR0BQ{lnAepL0)SU|_hZ*vz!T@6*9l#pYwR
zm7g?gyVEW|{FSQ}_>%3FrLT5s{+;QkIy-(mUG@J;s>|gM#$vL|+G^C4lo&H>cK2lX
zcx^lR;LYXQMCBMKwY67D<|c0XzF}kV<M;KBC(h}2)y==Qbk?UQlirB0Vgij^u>N@M
zbzSh<`dJ^R^mfFXD!w_h`3vtYUn$|hbJwS*JExqIoKbTyKdxck!kX5Ix^Bm(+hoPI
zuQ|#V#}!#<cDzHA^Vh1H1^2d;|B49$EzN7<;rdYi@!v*cDeK_lI;YEZpMF`?zrE?Q
zltieYfaY$M;@9s(T=oPy_AjpL|M`@COPKhnWd1+;B_H!{KA$4Pw5)8k(b}FV{E}Ls
zRlg?ej~8cP5OD;p2blh-aKfA)Re#yjZ|m6>S?h+W9n)6ISJM<fmp!w~XnE){7vJs<
zkB_EC=Y==!uk5XITXXqJPsfcT2i8x&FC?hCG;M#8^y1i}U8}9)<u<40*2oA7o?nuh
z`TNxV(l`bNjcF4c8UJn4pPyRvXyz83%lj*zoz3FwowC_9RX$GZM#qlb`JEk#rFZAm
z8r>?%U+s4!sOC%@zmk%dSGD83mkzTsAOCo@Y-*>+THo}AozK3Xt=M(y^~uv3mbMXE
z>*vgui!-e})zq@<)c;lGD}JsGHq*Vk%F_3uvDgWw^L2ZRzn}i5ed?*-BDuxJcmIK!
z7x7@n<QH3Q&1siDFTFhS@|5sfPPa?Tc5a*6(J{5%ZH=A3i)Oo6J?~VZi3iT?d-Gu(
z&%_09C7i2`l_t%(uhW^**^z$N6Lf^X`Q|HH@k(!hD9)c67<_iq^oM&&kLm5*a@P0f
zsShW_xm;Xaj&y8YHC<V$ZA%O9<veZuEt`YVd`*L&mRwb7?dj-vC@r0PJ8jQT2g~14
zr%F~oHQ4>=UaVEwVcxfABYu6V{Aj4OXu0)%n@%;Qwo}t04Sg$j1~Y)>Hftv_)E)J|
ztL|BTQE~64K+pL<qK-wrD^cOD7uuM#^2_bJ7M+m8?2{+`Tdt`6SE#t4^v&1pM?y5V
zRwZ)o)tT)2^kKR4@^zsCQ)g^i8PdB&S>3jJZQyFJUp9Wzf|B~HyXV|moq8)!@aL1)
z?!teqB){KFc|5PBCvs7S!HE=~?5CCyKd-NE=l>kVe^vLix>DpCmj$!Mj3?*iao#VS
zd;QSa6o<n-K~ERTv-|li-1E@RlL4})@!4`Cjs1?=U!VN;{8y;5YDL`gE{7$4Qzl7O
z=$oq8wO{Lr2)H~&9dyooXyy8EH4m<5sa#O|nZ5tZ@6-R4+}4EMpK{6nY{%7CvyDy)
zYW`fO;ty&YIk<pEt`EGQXuo5Y*~H*ac1OS0%FS)fX?YWQ#DD&yo3@qr)_!X2^{5q?
z_!<;^_AH+r85tND9Ij3n<TRuu`mw@BY17V!>Q5f?@0_nZRohEaq9B%)fuVp$QshDS
zm)F<d-`!nav~$jDr=kfHCKSBCXB!-R*=wp+VXXJLTif%`mvS;N958H4ja-+_&|rV^
z{{8)7YokC1c$??TUEj*U!0?3UaL3?iYJ#G3Ae*|B+Rt-(Q_OV+hW)7z4m6tQ-@9_<
zN{F4Wrza;r|MjaN5Sr)d<t5a8l$~F0O~gi}^iU=Sh7|B|>n~ruI(GEv)0=Out&R4b
zWfJIU%fP@;c__lhVd3@l@%!b?6C{@9+}w2S+O<_H*+D%q@Zs(6@9nL>(~x@e-rnlZ
z6+iZYZt`l}SkbU(LDJE#g9i`V|F8lb<m}kYwn_(d@;UnjeWn1BFE1~vZ)Z4V;Pdp<
zRL~IZ0te7R>|AvQGUqp~*}QqPRyU|;2s%u@t7DdFHX94elOMf&JUm-AYzU|Y^+|R*
z+G{unPJCQobJ(Et^)**;_liwYWI@M^%gfJymj-W5Wd$3+m!18(`un@o-$CsyEfUhw
z#i2qB3=Vx4__+c~Ql$IRC6;AgUWRmryKVKiGc}4;puNN3A?<*Yj-!vl)<!M07v*5_
z^YfFr2I~DvgSwNdpHD3*dwc8U>(|8*;C>spJLm51URqlE^s!9%DQ3tOeGIN$eSLn%
z`(&r~TbI8(lWPd-sS1KF{8T))=}i+Wcb}~FvH*=8>r|a{a&!b&u3jAt8vXDV6B84=
zy**z)bd|{URt5%!?*+914e?WE&9W+ZaDbVeFC#1K)vH%Uv3_jRQdNaI+uGV30vi6E
z@bUEQ?CV<>w6tsb^zfx#Q^5ma7HvFSAG9Amek?01yFO-T&~m@Ig34}7{13Ls@2L9v
zYN2zx*L1zNFJDIP2c0#}$iScn8oAl>;lqb-Z*SXIep*ub`C0AnZ)a;f{eEn0=a<jB
zz0Ef^_V1IEli%Lj`WSR#^aCCvjRWT^e|}2cUG~<l=EsGd#pyb~ezH0`I=;TXe!ACE
z>+*MRuCJE|O)W5(U#MsPApGvm&dWOrA5WMd5FQ@Rav?rBFV8ObmWizF+<$+6>ql+r
z04*M9kbc3=dSd<N^z+}|+zehA&?8~E2y}Mv^xz#wj~oG=d}U%%a%Dx}>uYNd*RU}(
zJYtg+QHbA`d)qAO2**SZle{|?I<NmJPntC8+WL5T35gq9v#&e1^JQjafX_i+!Dpo5
za4+rbtgr9)|L;1QWNyCQDD_l9tlve&$tTO++<5r>`Sl%zkBgt5dw6aU1H(~vBaH+7
zCsH;obZ&olZ}0A;qg{`W_g}ww5i~;BmU?rsdp~IGFzf27xVX5dr>8#$4c!H>rUW#^
zc&Z4gt1ssk*SoSNviQo1K&OQhUUzlr=<Bav?A|XWB^9@$fU!82k)ewjG}P+q=637m
zP1}kO3p^*Qty{MaeB6DPg}wcLS?jWltgLyK#mBB)n+6J@2k$|Hzz1iW=il0v`}^D5
z+kLXuR#sL;v3`XgA06%Hl{Pz(vT1Mi_iyj+&IYU3KlC+#VbkvV|NFLX{kpsSeN0S@
zfJ5lYsI6I-m-z~Jy43#q61lTT6%<nk;z1W4=;+1lxUjdn{AtjVs;{pKWBm$6L`An&
zeSOt0Z@+8j&b2W+CnZXN)=M<l^Mj6Xe|L9xJCEd|g$oVy?(EPCT~!e4_fd7C$F;T5
z-(Ou_J<qoK-QC^ds}C|TbjhUzG?@JV_xt^t$jv-FJZe5O5;8JQlv*B*&B}VUE%&yZ
zUCoU>mBrx0v0@=Ma7v0aJSqD5;^N}%IX9D%k{&&K7PcwHbDcJml2WTv;fo83`T6f>
znP!8AP-Au$#enA4z>}%vo}Qk2tG}0(m#+_7d+Wu;#ipjFJJ)e+`tad{Mdha@8#jJD
z+AW@aWd-<H?+@t;%}yJ|L`AP|Og_H5{JmM$m5d7u9KpvCE3og|w{Lg(d%eI#M{aIT
zpKX?V3N#-2fnA~5>EqXbe}9*~y~WDPx;6j)y^|+7U$5sjs;~dwD{a0>r~1W(g?V>(
zfrqmH=%oZ`ER(aZyR*=_{rr4;P|!dYxK#fB@bK`ftE(SBetdmxw0YH+4DfW>eb60=
z7pH26?<#vcYt9^-loJBrQ>+{RboKPaY|Xm*{QUgoJ~KC^oD?dFWo8hG10CTmV^eYA
z>ebNL*xD~IE{3iSd%8>JhvmnQAG5EotNs0LZFl$Sb+NmzfKQHRV5nm?(h$)<JxzDE
zRq3jE^XAQ-JsWgN{nG>MCwMK5-j-AO>51oTvs}M<Hj#^5IKj~|zm13MXsgr0g5u)n
z?Rj%2PQ2K{Dg5Q@*PZJ)>gGBvys;mAlJo0pYpWj~YTcB2x(IyvfJ{n&hS-uI&^U9y
zynS9?o?Y3S2wh#>b=pn#2kwHvUZ+;BxVShmF){EQian#GNY}pg@%ybTEI@0O9Glsk
z+jtK099s0`Sg-W&Z*R3DH$D0J`T5$lYe5sq44=3qMY?uexNrd!6)Y@UvaYTwe}B(L
zt{-%cd-bO$C!d|2U1@Xg;$nAM85wX1`3W?i3W}(2Z*RA^wwAuXclX@6zSl($@+*IT
zdwXkZwyF?FQ#+q5II<_0|GTrZc;`CC2kUp#{jIvaEqC$a#h|MKva?T@T7D3J`SRt}
z)#2|?PF8pCmz#R}>5oEY29cT<?5&Gl{`mM<PfyQAF8|7kK+~)%4|d4}wC>ok<Lc`0
z`X3(>9~@|m-kvAQ!2(XWJDu2Cl`=qO?D2m6$W2dvd<0$Le!kRlfz`>ACuf`Ge)|4>
z`{vEoWp8GH57%a3I3a1IA)={s+Ha;&>d`LI)YH@E8YDWc3|hKl9Y=_--t_nP_TIjJ
zeY;~b+q`-6mU>SIPcDE@XqVO0)MRGku_$>F@buKw<Gs?$*RDNV>Uv=2WOe_lii(hR
zF*i@2US0I`6gWn1DmFU_3W|$AfAON?>#M7~N?(I6osW&3`?{+k`QE*I?EG?Q7kO-Q
zU~66U;PP_+x3{;~|N4^o`Ptdu-`;k2H#A+U|NnP;?rpP1hk{2(I{9R+3P6*XDGqF{
zPWlE08)A2t9Xxn2c6Zs^D=USsKXo|x>DSlSTeGjLsjKJT-&cE}ff4MY%!1kgjXO29
zwb5I%L?tC(zIt^Fd{B5JlY4yp|M~U*D!;tAI8oW1g~@Sq+F9_lR>uW?uGU$*cUP~E
z+Z!=W)-cKC?X9gl_1L-At&iWoE^e=tib~3l4-Y}Nd3^o}ijaUiip@@mTT@R@3ta4$
znVsD)ZGP^`l_{@_7J%kRqW0I-CL|<0KR1_O-fj(a3iGIxk%owuar!yX1W<&|JcGoh
z2%TrUWHe+_jBcKrYyJN2?)2Q;+JAp4Yinykc}ip71jmIFCZ0@jZsWPRIlX`7%9lF|
zAD5Jr>{!RaB|I&)U)K5=`2K;1PoIV^bYcbTSvJ9Op@FA|M?q21u8NOIYooTh^~q?0
zC$zbidQaDrwJchab90lpUd)Q<?Ruc1UBvywf`AMc7Z>Pt9><O?tN;HmB8+MO;q&M7
zkN3&Wnl)>=|NOkGt3<)dtxqfnu-H@gcUSK1ZSU{x1ugWuzAkpB9(z~QfddD4rOlQs
zS@P!Q=HoYSf^T~10-xvp?*9J#^mK7q*|(1#U0USIz3z15B4rbklCQ6>8X6jUPuH7k
zl*$D*Uirj=fD4u2yF5;&Y)U-b_V@F7`&E{it3p<t_!8o)@#o@Q`{3ZET|Q?kvz)fp
z+|>!0CGgem!`20-?5BD^D=M#cJ@oqddf_Ltre;OFSh4i|*RPLvU%xnO>g(`}r_V?J
zTaz=NM^Nx%i__LG>Rln3S<l4o*GhG@bZnoPqk7rMvfiz$<-ztu&!VqIY0GYD?R=V5
zQu1Y0=xV#VKNe+gZd_d*{@86j1H-CD9xm2f=jK|s^GYvUxze)i%?+*4RV!X?x%Asz
zY)-|LS^xIbz2~%jHSOfm!aq|pyY)6pZd^U>vBY$@WA=8AXHH9dmsM)Wm&;i%FSJ}M
zGh4O&-ptMacmyMN`n-$s{(klAXKS1HTJc8362?0`<<ft0Z>l$PIkNs{sbBo+FIN-i
zzmttFewFxZ_18JazUL&Ys9EFD_38WW!V91E&aZg<rqH)|k6rqs-PxbF#iaaLo359&
z_;2_<CqH#xpP21=u>k=SCQrWn@9*!8*LWEmSU-r{i^P0>eqP?DV#B<7_rAQm94g_^
z>5*~s_5Gv8$4wJ2?Nj(KTP-ow`|^>5|6drgQm)osd$hEYEkNng>X7n|$+`2+{&<y{
z=n)?6u_P^K`OOuLbEh`?op`eI`}`?dQ?>lFZ@#|&aj$V+`OU+*_ZImuPU+y0{u*JG
z?NXZ5vEud$%lk|jhc2&XUv@sWEMVrstt>nLyb9d&`Mdg}dYSCiIt@Y-r%k(dq*GYh
zB%|QZkB?@#w`%_W3T;(qXV3s8b%jZ%pB`@Goo$@n*V%cpN79&wmp3$?+5hs2wAou8
zoIbVQzCv<#u=Y#u`14=X+1!r(l(h@HyfSEgUhFAlk5!$o|K%^beWUYo^o(f=N@pjr
zJ=M*MFj)2P)-tz0=GpV|FKu7mRvKNtexlhAIhlEvk1z7*R1CYYCBFI7%fGv`RD0Jh
zJ3s5v!c7m_)4OV_w0mZ3+IrUe@4{Q3y?uRie|&hjHfrmx%Fk(_v&5a9Lt_{i915Fx
zxLEyyf+j6q{CHdL?N#CH?Q(BznWXAn6xw-0;W5(>J8^l{n(Gc9zh0f_uKChGa_1lQ
zzIp49*~k{&EP6C)=DrnDOINM_wkV|Q1o!Gb<7HAavjnbX-j4kF;@-yg`2Dw)zs$U^
zu5`RDZDzS<e&Et2+qQW;cHd}iX!62v<DOH~L`yApd=TqDUR+l8?%v*N2Zx5KQ>X6V
zzyDP>1A~U$1jmI5on2j7H#a@KxjB99+O@~~<<GzM-79)YBuOh&sjWjqUf+h-`^>cm
zXPxDfv@Ca8Jh|sT`<JPbjgF3ujg5_#p5}2erhN?}oQVhDH#Hwt+NV~t{ZYHnqC<-(
z?Ylo`u8E$Oj*f+ej*gb!ufKwa_Iq`D%n}e4laY~;oBHeTzpf=KK5TgK;K7CuD|%Ah
zS6;MZzjh~0)a8>(=Bkic+vZo@oHbXAyQ5=j{Ql`{+am-k-><PhzoNO-VtQ-X&LEcG
z_G*iE{SSHeVR>G9x_QNi2OmBZNK4OteSLj#I0u7+;P=CYh74D$zP|%iA8~uDu7Vow
zcXu22WIeZerZ6*paop{wW6^x~6&8KIcmDPFt!HNZ*5LiPS41o0Rn*p=FBN{)>McL#
z$CN8H7)|^8an%-yYnww|=juGvvHSA!;Z=3H`s`z?FCJT4p1o0OshZ2)+P{A$-aqP|
z|K!`^XC~V7KkrgiQp$X?_Hy?5)F`cx&AI#Et<B<?tTg}M_b)F$uReeO#ObYXuR4AB
zv*5XLDs$agzstXFEh+z8SNC#v*7WLQj5A-=s#{zY`TYJ}T;;>wo(_@q=XK}R_H|hG
z&%M554%@^u+1uBi--}UEnlF8RR>X}Z{`Y(8U%z)>Gd1_@{CO8<O8xY(6o_~Ivw8Qw
z$X;plYgez{-BFl)ds{BQjKzZ|Pmbi;GBCIb{c>EmpyJDmiCedReSLlX?=LSOKY0=Y
zKEi$W8L6uo3+C+5{BAEOG1c(Xzn?efG9<}L&&u4SdH=n{?|09dN`EuY$XXZtOgDB{
z$;UN?z7q2uZ9V_Ywf(+&%1()+OCKMfFA<h~^P1Y)!(V?sHFbZdHu1{;OE0<Y=Jnq@
z(dqG_d10pOlbe>S&)Kdt%s4e?srP@`%ey57f5zY1e)qR-;H%l2r_bL%QCg@q)~e`3
z)17tY`McdhJ3QXizjHev{*-Z_&F6@%r}xe~{9tGI^ncdbCEU(-8ySCx=hjYM5LbWT
zOr39w%aPTW5>MUvS)6`$RsM>FcUHSht<sf{6V!~G&&}ceFRa((!gnphtI~h|w46KQ
zpUM1DO?Bsf=CEEV(`CN1mn~fiy5^<y^)*nF-2uE5`OwqT(`TDzi;0Sw*8KQzZEbXM
zte=`^(Y&ac$I8-9r5}H{$4h!yn9F_Zj~(GlgG&wL|NZ@XHb{AW&PN5M+l;ZRBr^mA
z{coR0JjURxl*GKP(&n6vNZitupZ9$HdwcsUlbnAMPrYh({(tFy@Ke1Ed%wKtwyL<-
z)12N{+)Wexro3yGbcWzc$?Lgy4^HrRNu2&Sxy9}7G`&}G&ToYU*L%(G<o;#8K6?Ak
z<n?B4HA-Clo@z>7Tpuk8CZ}K7=q+sQE4MOp_Lh=24lYO9_E(56UAAmx{4zB`Ln*0S
zvz&Z$-};DqU0r*9^~6sXXGZH!dG8_k{o9exA&tI^i|lJlm0dqfKJF6oeg3IcF@pb@
zpPrsR-!R#2rcvs#UTO2nPbo3S_!u6XcVc6G_xAq&`{&MquDqIKaq-fnNw14G1q+0(
zxbI~7IJoTfrPnOeIv(A(+j&;&)|NF(J(v8@o1;=+_tss<@M?D0m9sZi1s~lu(Ms^7
zye;o}ljP0E?4M;e`es}y+tU2$;p__;Pn-8V3pvJqk6%Hl-*@)*r8BD^PM*`@QE08?
z=r={IJp0NzMZf6N>(;G4zc@d<yo<kD)?v+~$3MUPeV${pKBxTo;pty07v>c4ajx;{
zZOgPh(q9u-KUFyJ#O8LFD?1&J&i{8dD1q&fv3BUHD?O6NTH)*VBpu}v7oR?5ij37B
zMurF16YMk!a!yWCy}aE2`t|GO#l_m;>twD!ed2U`%ii9P@1hqTX}kYJ|Hhuko9g$%
zCF(v`g_acO$O(&0x||l*Dz&ZTz0yWT#dslKqfKGQs+4(^yVZE!o2};fBQw+MSeB)%
zDtBhh4Ruz-sWNkBzgpUxaN8vR-1O5=Rn+JDxSz{CTleO<fzqm^%PVK@+_$N-*SC0I
z(M7v>>;9#`v)p>$o{_6&6O^v|xu5@Pj<=n~>+=#@S4M>?pFgzOQ@X?a&vFHxBk9Z5
zuC=ZF^h7s$n~YV-i_g!`FZY{!>Egwc)#eNgC%BC?3fvY3^hlfM9d6@&dTMHMaq;Er
z*PriNB>!Nhd7j^PKB0@Nx%uN}h1h&|OS`(e?&N%%u4Q>~kI&dlOf0pt4WAJ8Pi*Sv
zOIBW|XN9d^Gxza|+Ha<xZhl~W<uXM@D>*T78tAO{g>}Ks=NEud@O#?}nvdPrgno{)
zpMP;5?~&+8@Hy>Qu3mNTm)l$a|KIBH^=7%ZzC1eGJtt;^7{h{&gAq0b%9fTp-TURF
z&2lzu*bwloxhOVJu7XYM?*Bk#rJx^gZx!`aluC!4e*13j{SAzX@sl>^JrWi)oO<fT
zv%;?nSN+;mS>7Zh7|12KG5@gA{@ovfgF+&Xe4lP|_U7{R#QX6UT3R|f?b@!Xi({Ae
z_8JOKeDv|-<G3Rs|BpX<&VG3+^G6rcy2SXY4?b4>{r&y@t*x&wFZW-&W(^ZFGiU+n
zf(y=Utb9j~9NCh2`Pq|`lc!CaCamUj;nJzUtDNUdI^AKkY0-vhy|urDH}mz@Z+Nq8
zO6D!zY4i5|oKd-3<rlZ0V6fMe*K5sG=QBNV3Gs_j`W(h+-@ks<@on0Ik?VfWEXaO+
z>TIK-{f*X>CNI|hsCfA3@%fqYzC8Oj=ltC`Bma0x(cWz~=l292TWihQ`)d02KQFnZ
zRO+gJ&xn{8*xtpOqhL4NZ>|-ugn@#I$(F*$$M)^p2kJfr>`-iOTGiUpVv>33$+fl7
zYooWz+12cr6=nDPcZmMi%d^Yh>~8sTFo*YX)G5{HN2cV4ZLg`ab}foZS3m6E9gvk&
zoO(KZp1Jd+OW~hYZ1<PW?RZt19b<NRw%7;bWk-(et1Frv`k}|l#AkNqtg-{;3twJu
z^zZBq3#e)e(Y}3PnT+7XN8euA+6j7cDS7#J?I}$;x$^^8vu^L@{f}R6UD`cOMKDmZ
zJL=T(a{l$7_<pQWQ1bfw=TiB-W5#J^eQHjVP6u^($O(M>|LvWP-l|pqnSVULmnh+O
zJTW+>(9^-t!MRx5G&uOqSySEXMzyDkR>t>-r9~}U_H@?H=)9TF_BD8`>2-WsGJz}2
zXciZb!fxfm2508pmv8RsXzJoQ{rs<g@pt`iPWHv;Y@fei@YvPTa)#;q#N8$@iql`7
zd#7t7|Fitm+@yEjTP}TlVbAzu>dC$zOt-e>TIb!_v1ZMhK3VH`x3;c+!QIjELUD%U
z%8<w~$2}DvA0239zF+hCtdS9CMq<xNL&1fCZku0z-*DoF^=r13vr1}38+=du-Fvm?
zRPV3Z>HY7w*Yy2*9@KR1QqhX4)FT}J90u<HcRVa;ifdZZZ?z}4UoHIF7XR;Qc?=2%
zCp|mV*LHmA>mE1fM@uIghreroV4al`?JaeAxr)jm9tC67vqG)Bnj59ovN+72mdL>H
z@n>1%wCTc2XFPbnymnW@!6wbi`)&&-zA-3zQ0(P+>F80Hr%XE}=1;hOJMV;#-&BSp
z^Y>3nYI*U%cvG$J>50tQ&ugtJ|226AKU;fKbw~ND`^)qivaZH5IB?zF{`~vh)oFTG
zzuq%v>u5@E=Vehi5%>9fMablBfzu0YwZt2ytUA3z`facI;=Ut{93RT`e${(RUf5Ci
z`2WA}`-^to*|ccQ*3*hgTI*T-s@#`mURdBb+dLn1d*AA?wQFKeAC`P<?i(pGGrv}u
zW8dRt{NDo@Jnq_7t~>62Ilg2g>jbgA`QL6Ov#$I0{_n#J)3X^3((YMEZTTPIa^|jc
z?(e==S2q2-Yxn2L4CDR(H=Z@(-rHI}+pgAndr=sJf!o{2ZSP#(?ET$!_V-r*`(;0h
z{-yKIvH0s6Zk^nf_Wf7r!&`fn=<L`oacTeN{qCmDhi<O?Z2o+=bN9yo;sq(^PE~uG
zck0^xTQ$G?PJyLK#?^Ph<^DOh-mYDjE4^AbKJe?u+skyh+9ey-WX?ay9JP}B=c_91
zice>D{Cn(XT74s7ZnXNfv-gCj&aR*S;%@1s-Ro~1ZL8f`v0R}#_tY7`)cvww)*U%=
zWc~hsR_5m4zrDR}W+rpT)8)$zPzw9{=BBZQMa9!oQ?KbQ`69tvwf7)LN7rw)iuw;M
zCq%VU7p9AJFS-7*M@ZpRsMb^AJFZ4AN~al3(OMd#Q=k*PoLwaAt8;shWa-SETw6bV
zj9)W>XXeSw$W2;(=UyG3cm3Y3Y5MVXzrJMdjmuB}exo=Lbja^MVcY1U)}VD2S2TVo
z`D@4puB@tX(mD5)Yw54V)DEuFK*qpzLH0Mh_bzgJ`{<OUd1r@~>^^=k7Z(?orM;3<
zIyyQ!yrhja1O){Jr<$b&C@Co^Y2}_-0Fn>6HN&y9qoZTRmcl%>MaR=j?nGK>$_Gw4
z{q+0`@3~9-{dbjUE-bsukZ#HFLde9BbNdw2S3fIm*uT^*H4HSgntbW<`RAPKp;bwf
z-u;?tvG?S)#dWr#`&oqrL&MH2aOpoSu=D5ZwSoL=A~+oizH(>W*b&(<e|t3NQ>(}K
z-OK-ol&~?doabj^nxo75XIEAKHI|KE>-WEW`p3mk&-&k=Q@qxUTckdD&Ud@&z3j%L
zle@m{SJnQ`z3KY0Gi$<UomW*_WYj7-<wc?9EdJRK=a(ej{Vu)k0MEhV2fx?br)PKU
z7b`CMeQ-rb%Zd(xi@RShDUU1FXmqoix!V8rL$gm?cl$GzmA_wJAhghw(`oDRsIZe6
z1}8)-gU_DM`D}M%wO3`S(#uz^+zhk6<i_-cJyTIyBy^m|>x;@1y|1%>-HhopatheJ
z^J>Yo!#B73Eq->^FXu;@f5<u!;XPkn<_4~>u=`r<!XDnAl@$9RWZ&E_@%umW-#*c?
zR-gXpxJB}%oBCoYHI|$ArfRC}^gi%he)s*RxwE~uF4y~1D3L$UT9EzV*23nX+!|G-
zMVs34lLc*Mi~D}QJH2qxucz;tb>&T`Kl-Zrg=O){kc9pf0lTYqSc%j}R=iC6u;+eD
zRoPEHfvbA!<n&Z8hW)nKl=Q!|CGtzwzd2E1yZqIZ7Hw*MyTC=&vt*4RgQ8H{ORuCh
zK`+ZMFF7R(N=gNSg(l_Aev?&Y@w~Lo``45uS8iLX&CI>}O79&1msQe1LX+~J&&XMA
zwQ|X!rbU6%Cro<%s4Bi=3eQZ>lP@hdtc?m?S{Cqh@8m2W-kVz#uIO|cyySIb;Bvhu
z80*!vu&ev?-^gn#{QW)m^3LR0bm!j1$jbk-*Y29aGWX2ItB#2mmZ~hjXLd8Jw77KN
zf05*>pi{YvJ@dlP#;s*|(pBxeb;)aSB}NvLk5#jlzU}N-!IZ$Jsuj5Fipvbfj`Nmx
z_x*W3&)(VJ<!JTx_rdnH`2{V44vY5*w{0o$-cwR^Z&vL0bwV}mbtVVyFYQV_J)z{|
z_4n7^4#(VJ?$Im^*PqkNVO1Db^4_qt=C*aBpL}V_d8^YemhS#O$Kb&I(6U#{-#44?
zWLPr8cJs1#j2DuxR7}5ZDERQVvdOY{$@^=T`c2(^b-wkU{Kp43|D1ol?7)lzY4eQ!
zo-Iy4KeyXTW~pIiiQJb}zE_Sd_&iNG<)5uOf9_&729KAs&TbBKJM`x1^QE1a7*^c>
z`!{g?mA%2g&vJOX@7aEO^52(2f`P3EdAz=CX{kQj@vGy0{Y55*J*$?#kK7&4y@S!v
zy)5SA=VIq6VjiYXROa6|&p*6z*<QD=RV(-TUhBOduFHGTZT<Z1HD9K1#%Ud#@a6iI
z_w|hrEWa#X_)jmiXQp{meXP`~ldt>CRp!L+|FPoQ90rGiUAGUcu-=;YtxqF)^}gh)
zEnlX1uiR}vTRN*RZvThx{^?(<3ZG2LI=J~-tCY+8b^pzuEY(_e`#FCO_r8S}v(7T)
zyesXz&lPEC9WnX!{mLf?`glbrBq=wG=UbKi+5VI>GWlL!-twEZ8eD1S3hy4Qmu9`6
z*UbL-)c+>G<1_T$Sc=`<(4JrFAipowcS%70m-NMZt{7<sZ~ixXs@u)I>?xU_tsZ4t
zZ&04sv0~HX-wRy)HLXoGEsbrT{#t5$Tdd~m>z6VNi^|V8{{7`M!}OkblUT!ppBFbI
zKDLv$uQ~C-ao?qVyvJR?%dOwOZoP<N_|?4o+qD^H2<yf9GL|In&tbcec*DVa^NN_*
z-&_6WzO(arc&p^ln@-_hW^4)P&Sh)(_Wh}KpK@{W+YR@0ZJ#o@EMI23m+`=wy{Qc`
z>q=I=<agMxIapKFYie+J-FrU?>vQ5x|KyboKNW0H3om$oWp}bOD?`Utrd@qG@!SWV
z)}B6pf7QMBw`czS`y|TY%eyZhuOGh6%ChCP^@_=}c^sUg%orpl>1X;h%=OV@61W;S
z_xH8`*9*R<hQ0h1KaqiHRz;zUjiBI0*(onx&RD-D-sJncX)i=CEOk9C#yHQ5VbS90
z({-XIzrMe^nwKG~{Jz_}?MZzuhkJhP^$w5OQqq`hB6WIe;`K?^*UxZWo2BsMSC!?x
z-OKMb&(!@dJ!2QQ;zNd6rYp<8Tc@r~t7UtvlvP#zWOn7tytJ(w_P%dxTmA4q^MmVq
z7XG^Dx6$$QH;YYj2X@X_A|8_!@$HxCx8jr4`_}HA%(2!p(Q|W=wJ1aJmrbI#@5|lU
z+30P${kiqNiSH(q8<!ms77VmIv%tmw_1Vt(m7#(~jMA+^OHa(_b8Vd)@Zd?$5etTU
zxmEiv9XM%pZ?Es{GNTzj3@NLQynfEm5mzePZJ)hd@$;;(rU%ixtEN_4i~2FHjkZ5_
zKEY?_Dou-Dat$wHc2A!DX2P|SK;i5POa10q@)FDIzrQ*5*7i{Uky(!qhu6!$ySuOc
z{<pKzS5I2cxGwnqT8r#Q7Z;YafJMty*{&393m2`9RGHbU%TRlDmYv;Kd7D+XybRCQ
z++n}YR9nHgfi<De@9c?{Z<C^*yyTx8{3FlW+k3g?ZVT1Uue*DyPVXpu+<BTIsV`@4
zaZ2|7jq|szNUmS6yl*PM^`p)XjvDzXFEXFbm)~+@%k#%_b8Jmi_4K@%7&eDZUZ`IZ
zP?NZ1&yNaKyUh<2oBiT%Oj^{sY035OSrN{O?zVgHbQ{i|lwzPE^<C!c&l?LF%j3-M
zetSHt-pIF7_G80*i1`U@s++hp?JDY1lK$C7UO)Uy=iZ{tD(0W3zIgqeX{pAO(m3x+
zwFg5slzizn@;?1&<yrT~pPsvGmVTdpza?`s*Svar>6si1pQLzapJ1G^=-;J#;`N&+
zFZN})@@iH8kHSw}CP#GVZnN`j+ST{}LH>TZ%iDDsW(ZHpm)x`^Tq#lP?ryu|eHBVd
zOb2+pzARChoZ8W!pk4i9ePCRJ@Tar>A(xj=e0q4L@8^FVep46@%&2(z<cq5H>+;vX
zo9Y{+86Nb{FUvCe9Jl+-o|!xh&t9xHFP&z)h=;-FVqvD6L)>Q1=TQs_t@?4>bk$x?
zV|j5opGE)IyCpXkM$Xk@@Y!j};B(Q_ieZwTqN4B#?Uz|OYzz+X?rt(>n6X~QzWQ(8
z*2wFNozDmzX(-dbvgh}8*MIWImA-Df7c^~giJ{x<+?@;4PapVKSHxi8d42JJmIq*e
z>}r1BEM#lv>9xD8u3b%>VabUXr`KDVmrS;w;PTbeTyLf-gXTw0tCPN}Oby_M``Y<^
zN0=-&l-*-Ic=ha@1*|OJ{r=v*(%KPsS4x-Piof_v*Uau)-!osYxSX@!IP=dwRpW2J
zzyB;+QUB*F3xk3B^3=55R<|p+y_7B}2vj-q?8k?x_uHS_#udnZ^t3*$@Ez9vU*6UH
ztv~o%v)Xp!=x3MC`*<2JSjMy>Z2i|0=cd<LRs1~kPX2$voDZwk&sKc-DBrt`@x}Kz
z{%}i`*chJ>y@u|OoK-tC86}J+2dgrCe7=6^mbLG77!RzI|F*OK)@RGDYnOQMkekYV
z@cP}r@@F104Gj6u(?9=j?&vrre=tYyTXwpO|JwBP^9~+7s2#p;OPQko;flQ9;nM#1
z?c3Zx$ePzy{Og=B>HPaSyA^f*Yc}Tlzk858v+yLWxz2C;_FLHXZM|13HLq1zmb~A)
zxGDYjwMP?$4d2zRjo!}lAuRpRo`hzz`?<6EW_)lnHuo%ld_dpZ`CU)0_}-{~sT(&>
zAJluVQu}9HX7`~)Hihg<tNo`MM>{`$ch@P$@Z;*bbwv!vUQayUY%aT}D7{nd;_9y}
z>gMj9^OlD*efhcjbN)|)btSqjJ9X{mUHf?FlBKvpWZHgq2E&9SNA7(s`uD5oqo3oq
zTT^Dv)e5qH@#*Pl|M_-nufKl#=1rH~?2e9i$9cS#u5wDbFiBDJi&e8=L(Iw`yBi%@
zCNFr}?!;)US~az(*^hndsuwS3@l?${sL{D-(S*sW&ddDxHLko;Vi%YEYPC1e>EU-x
zw#AP51`?_rUR%BRJlf7Y*`gCE>Yn~}@}0G3lGs$0s-)g@t8d~9To|G)C&&JjYwDl8
zq~H}@*XvdUtdgtp4qT|0$1+7b^ixt#Q~i&xTuW`@WkDT*Rdq*xy10aJ7WxR`Z}}TG
zgA@d2CbOw3DJd<wbZqC$j(L+(3?#VU3A^s}(VO-r+~(wjc_Pxn+IudX365K2UT4<h
z8NPmD+w$(N_>#_!6#}4g-1%4af1%TdX7d+)=F0fu^jMtvkIcNHtTIj0l+$ewIxH9#
zRrBxPG|zGAl+_m%cbm0kFMXH1za%hpW0|dsONf|Zvyd(C#>DCT`?LO?UtfI9<uK2|
zt@l?j#`SHRdGh0P|NmlAqARbu$lZDP^W2k(yfQiSA8DRy%dNh3^WVP5B|NKcOnF-7
zT6?Hf_p|w(*aMf{((V>@bgX!zb^8At!KwUQ&#U&;nK3BrxV9?#^v#<~=lMIVPmJ$>
z`BjNw#`-^dp0D|rDW{+nYy0Zz>g0H#qju4Y?(F-z!*l+g+jnoZ3J3;9f|~wcmhAaI
ztN6^NyK@(%cbn}KV3@H;PgQ!}FUe(0{My|j=eF*(mASfVdF0E<9UWS%-;N5~T5oDE
zJAm4b?)tr-N2>ItrsQ^Cqe|22>Q;`OkM=z_YgpoceOcNppB1m3JX^Et|J#6*6XwmC
zHgm4><eOnz%S!_8e%D*2r|PI)Tvj?ERZVqi`s|#yZ)8@B*Zf+Xt<87x&yDE<f`O@r
zdAz=?kuVlkPGnQ)FkSV8yQ_1$-?`|`ukUSpVEtt=<3GJ=8?BF>UT7Wm)=qgx<*!Hg
zHP*5m{M`I9IqSaXq^78jy(Kv}XVsRk=d1ZWna}R3?)=I6bMI_On=ba@Q_<y&EeyB#
z=3P)eaCU|FPNCwLPCxDy^m<-f63pHr)3w8RT0D=$-<J;hTh~Zz1$8wgz8w{wDs%9f
zntry|OiuCb=YAgD^zi=f&d+n>w?6s&``Ox?SGQbWk-2%_+>b}u*Y7%+&=+T)ztb(9
zZG!CY=d13f?_qX7JI^5PMoa#&FIg-%p8nT0oqKm<&e?s--u~v1zjRxese##E-ggqi
zff+@4?`FkD9sE4^e0rap?Zefb`3x5SUa$Y3GHX*_R$tp2&g(N6W+?H=cCER;GkpEE
zI3tD$O5ta2O;-|)l@=5X<V<2y)v^qFa!}d5|K0tKHJ8%wpZS0D634B?1HbdPl$^G9
ztWUn*_IOcs|MlqW;aw9}XKjCfTa@7pr}+A13?BQIF)G}M`1Qf{yx3fuiDmcSEN;qv
z|G(y&+w`z~y>W6@+pHNp9(rb4GWh(u&A?#2)QQdE;Uhr?#%C>0qS<~|>ALmLPN=lJ
zx;ON{ej5M#8@o!^UyT2fuf)!vY4Ya%7Nu-uB_%D#H{I%5mQAbe^o%sL?rloFocQF*
zfwrIbkM9u`7v?p7^j$Bu@ZzPtYcFQ#v>#fhTAm$p*KaLTLFN5^>l9_{y;lNM*I%p*
zNMBy__lME`>9ek%mR8W;9vp3#lJ=Zw+cv(u+&_0N&z-eHi@{;~lvy+Pn@v6`lr#5k
z_R3FXnzO%4h)HdX;*QxlvF5bdw~Jdpa%&3;PE|+?STsp_$2Uo{h+oC0_uN0naDun<
z!nSXfeyVqD=S`GgKd{^Pe7KawOdo~|%Y3aRebg9?rg9dndi}c1K(`?4yZ`cUKlo&>
z{yoZY|Bt^x`hDq2{qhpt27$_SBfr3}G3OH>GpZb&W4Y(k_T9g^4QBg%Hw)PsU3v4(
zg(q3-)zwyZtpJZ~cnKJ51fDc}o+2HvJMz7w;q&_56OAG}b4q(pK8`QmQ5=(&%<T}r
zby>gv-$U;g<bAkzWTE}}`vt6NGZt@O({u0Imh(@{qgHa;-wsK*Jz<gMhTPH>$F8h;
zKO?j5)7*Id$_Syo&Qor3GO$=n&xS;VWzH=HEl(u|jagr$uQM?*6#rSc)B2ofgkM~>
z-MfRDuY(^wDiKm?c6|5BKJD0(?N;Vn-<Z$6`|)$*{%&LS7i(W920vfJu{S9B#qrC(
zw|oiFO0hhucl+|?v)fkO0u9p$7;9WKR#<=W^?qj^`C|XvJ9WSIu|8e;<H6sFoj<>)
z#PZMHT;V-S@Z<8Zbw7;-i`tLL-k<UDW%b&-DGXb#zdu``e}nVqy3_kbmKi=v5nz}Z
zFD)-_%`m~XrtDJnCQt(|=4xp7n%<6%6)c9$LQ`c9hLt@J_j>o{|H%)1)6S=F*S9PC
zD%LQ|u2-kpFL%~=_mV}aUQ3U7t1s8T8M#0D{w=N(Z>G$;P_NS&{O&`{ck|!k6-_0E
zU-Kp}nNoTA($zd=VaAgalGzlNrbV<JPd|1{?OEv&`!Co3T)SGVn8um(ditML;*rrG
zPo`8pyy13ie_}_+iULqmSkv?6b^R7MNk)gRKiaF-CMTD-YA~&M_xsj~Z`13ngt!y#
zZ;~{7Uw7%w<@A)_QPXVMubs12*s*J-@Ro)L_g2==-o@avVe+HzY;(W8xOgd;PxITa
zQieFYty2wRE~hdXcu(JQQg%-48-^un&v*~`d!KDsJ^7H?`tSd5i<-~AtvYv~u}%H=
z{Ooeo8@+!YnVx>u8r$7(S##sc>a@H^JLgY-%X;B`^S5(K4+30rkEZqPud<C#Ec5@|
z?(O0tIyJ$p_*<ryt!?!;9zMRfokdHxa%*a~{a^dLD*MV`*{ewhj;v?f{mgg86v0hN
zejC)TZ&J**%J?O{?|&QH+FP}+-o5nQz4z+Y+~lns44KpU&i&Kmde&0o-u^W!T)X7p
z_Vz8`-4sge%bT3PT<Tr;U2I|gmG6hm=bNwjpO(5MHC64qm@<3$8Qa};cPB<L{JZPM
zR~5Op^wc6I=P4}T#SXKV*Z#X_<k5DTfBjmEyjWYy<eTf5tatDKeN+41x1)Pqe{QV#
z`RVd<|JPrutgWBrC@ngE4Ah`GlzKs31>T@JCEruww)b4Z6qUe*LALj*-nzvulUw98
zb-T}=!09WeyR=3vZ7{66JjHdQfw`<fqC+tI+;bd(>w+}p_f!XN4U|g@P+GM7;Jyka
zZ2?iQtxi8bJT6-5^{?Y_|BisA)9h=68H$R07Oe<bEnoGUJ=<92@7Bua@!MXt_ZOZz
zd2;2o1>x_*_<Gj3xP;i8S>Q6YTVb+j?WMmiGj&3RS<Sb%&!4nl?~4md+1)LL1Op2V
zn}tHTCdK7P?lLyjy~?otxa2dwotyu+<nOOlOBEChOg+lu74mVZtNfh3-!?ogzB92j
zxPL+Bea`$XH7>i=1O+c{1}m`qRW{FX@!#6ra{JxqL32tdS5f7`gPtxfE@4LmSzKIP
zTtaT$;Pd)oqO$m)hG?yn!YR?zymh{ZzrNrO&wusi&8o2CZy!FDHS5{_%ko)u>9uRV
zd-ioV7Z*_(V~vZGPhXq&v*_TpM{^?g|7KqtxAO0r*Ofd$StnM#Z(e^TQ*uJBmD0Hx
zbEl?;-O$cuxbpt1B53T^Hv7y17u5x+&+Y2wz4MxBSM>i@Krd%g`u&M>vo$A9T=myD
z#eYgz&<&1j>kRE*zOUtOGTr%CtuN<c*1EZ$zve$*nl<li)<5Zq92#ERf=*@_oD{8*
zoC;~ge%Z3?fBGryud5gtdOFLL5}4j(RbMNfet5B;?fqk4x7YrvjhY!He0@T}FBiTe
z4N>Q|G0fth_vT-g+tyj9F6jUFG7*1$wkgTjEoa8OZ?Dp2UxjG?icwNp<#Xy5cb3aj
zZ@nX8y`Pyh8mbxJ#Z42=HpyQ9<9dSD(^GPa)~0$=X)}sHbxm1q^QvNnXz1$S(>}EL
zo_x;#PbxL!<d5$>f`Xz)$^sUtr#}}ncxS6$8n}Gt{y+CVl+27<zkl;#x0O?v8!j$5
zJvD_vL2C+^chKdfDlbybt>fEtX33H-Ig{7Uots>m?%F7#{xWz{Q2z57DceF<@4izW
zGJoEbS-Pv=y^8ta<uU1ELW|<<jGL#frAsHYt*CfXp$Q&)R8BEEvHFI!{f-NJQat?q
zPM<k9@np{GTh>-9KFym`_VUIZ-s21^sV{o2vQ9bZ80q4YVhe5szYKid=<|(X=dDQb
zFITs8%-NHBu=t3&^8E5`ta<YB|6)6?u`IY>E&TTK`WTM;((4RQ&MJA;Fu_@{(o-<$
zzzcu&s+FImG==r&&e@)JeeZ7Wea`k$58el{HcY82ytQUNV?{}Q{T=O6`86um|E^5$
z{{Ad?#^UfZzb*xx-|;fphKqact{vRB#d152?KS6gc^`g!&&7|HTi2!<|GadPH@hs8
zQD5J@*NuC@+XtMsOm{YIP27C_&E`49HE-`cZg}zW*Oo$k+iOw_U0gyMLG=Su<z&a;
z)NS78^ZJ?}{5m(~B5w#2!-=`KzQ2{)l#+k@;pEyg)6Lu8T+C2qSk~d(&T{|U-aYy5
zZ@I6|P;gy0BlPS3`(KL0Cf3y5-@|)+);FoEub2KZi!`r`-)iC_y|?b(r;EXA6)Rh0
zVs|~Q<h(wKXT|S?2b+uY4RfQq-|YBsQ~SHqhuZW{4ZFAwW;jhfv?=k})yKxu-?l!m
zel*#9Uir4kB_FHPb@+NYBdv^I?$tazFPQV#l;Yy+Z<-$bJeR&$YDH>W%jV$c`F8(a
zujjQnZo1-T&aLHo(P?XY+IIflJ%7uZ*sB_yJPTZ0O2mI3>+ov7vp<%d!9ZQcy87$8
zd(zYYH{Dp|>g*0_pgrJbY^YxSFaC_&y^W=>;}-9Kao>fHAt>X<{mcvZZ)|_K<Hfzl
zU$1?gs`O;lt+&6+w%+;t=&A@q*}J;Gw@to&%Kn(9zW)&O{1pLOqM@N>)7Gv!rNy;0
zXlanxC-KIQ&)%Q@#`wbE)7R&7-!gP)eUF>pTf&^s7q?OW632`$+Zh?kyjpo04joly
zU^>HjayyT}(^IRwypvb<1}zQH2=#IcP*U2}{&s=OQkRl%l2?v=yWTP9wBzyoEnn8G
zW$5Xwm+Se;@#b05$p*E_s#`2~Uk`d3yCmmcazw_FEZu^qZl82--^iFDxWD{mSNF8S
zXHvb#isEi;eUjI^s(aC{%Jb&4#6vqndK`b<>@<96;QZlh{kebl;<lEzPunK#|9pnh
zIn_P)jvPL?ws(H@u3+JCR)!aof{f)q9auEq-Nj`pkFiGJ%8B;(*NH1_J0w&4Y0tOb
zw7kfT|6LCKSysQ6>G@3GpKKRbXKnwq>E^<dSL6JBf{rqN$h-V`Dx2T5;9FZV)tgVU
zI4m=onz7$4>Q3*WvhLEqRu8_^%~+hTl797D#k%JkJ~B*7IwuqNGg{@;6_yYemyl(k
zgezxl{(oMa_4kaT^LfAH6E??jit}~8i`^TOBVRdDW3w;=N12(K**1nRq1nHFT+yFe
z#l3J#_|y9)OEUO!qGeYZZhd8)Hfw&(^XuL-Tsk{eSp2Zl2+R!HcVQ(rgU7<<e*Yy4
zm6m(%eje%?oBZ(Q%e0fLZhbEN#oo8<$kCVcT>H0MTIpH-{ugV+AYhYyOFKW7f3{V+
z>x-#U&i3=}Enm7kNb$7h8#xyj7k*H`BlPMk)4EUrMwy%cqL&1$Z|(OE3{O#apUtpi
z)#|8i(5Bz{6T9Th(mzf!NmdlM-}vh59$oo!nU6ld-#>R24};U!Rb6*xOm<{Qsmra2
zdBfnKroF}WdRy@oVZn>cKk}IaId^^MR<-$3etO^86?xPA&2-~G1&D35`n`7bbEh*x
zN3QAr3|;^7@`+8KuPm#qcTx9ZNbuQHQuJ%m+^@U8u3gK}vr=*Yy7-Mru|KXx_wQKl
zqo%#)#GIZG&ISc$hMnHc>+*~krZ@N9zi|I;+;#W47F-K5_fL;~I!W<3kAv6Q4Ltql
zdwPA(yzUTZbZA-{Y4UCvUxS{e;@tzF`gBUW<dhXH9zkg_Uv935VK8u?Z&&S_uVYgb
z`kC*=6E$W5i2(n&8=T^0x^`l+tBoZtub&^eH%GsrPJT<dn4;B>o}BsS4KEMes+{jP
zZ-2t2DV5n}obTrT+P;!o!TMuR!4cM8wXElz`+50wuT(rP>REc?^Yr&ymrDgD>299<
z>ar0J!vee${tYWOh%(H0%wM}Z>GHQy?){OQT)7!;{Jnk4bMp0j4x-|Y3<X(__Z@cs
zUibI!pX^<Cj{js<Ra(^5lRT-zs|nmJ`q}<?&aHiau9SU|Hs9OYewR5ty!7)H$4A|~
zufwVz1k9_=7LN{-6uJ}LBXQ?zrTFO>pQOWji`_%HqqhDQVVEJj#o=;ZerfLQ2wkRz
zwYP5aUSK+3y;Iamdb#4uH&@=9-@p4O`RyKimh*y>+i%agx#QIX`)P9x-LLuDHZ=+_
z-~Zy)gSY38%kkc-*LZZ{PiDK0`|WSCS68OivND{Ae_{40=E$jx8LGZ1JF2_4+sn<X
zF8jMnd1KmJ(f@hMv7&rV|EDSLZ#^heYrwJh|L2?B=lbPpU0qzJb|jb;3usTBH_x`3
zjh%gc%+5=<S|g^G6klF`|NfhQaR;B8yl1(&j9HB*<JE%8l8UFe=Q_k0y(nc<X4o0D
zbWgo#*<!oT+Z;ozS9opcKb&#lHLsgP$chlw@OABh(^qbn*|}S@lk4l;(8s*Hd}{SX
zLa&~3nq6~xGiXhmU2W9*>$h*({xEIs=;%677O?2jv<f{Fp=W}<+<x3u;eLIdB2%-z
zI)^V37xmh@=)>ZorMDKo5Pc)xAG2_Z{gc9m7;vMLYwE1m%7TIyn`bz7dNmt9ou7N#
zsp!hq+E+K*a``uBnqE2X_-(^v)|<}!f`WpfJZBcTELHsY{;{AG?}E-Q2A1;|=l<b(
znRxxRZg%XlWlN<3esIrqad8o4K7LeqYTGQ{_-`!?3{9^Oe82xCNOaC5P2FD)mfv5q
zgUPsIhw|%XezUgx+_LS<YZWCWE!iA)RW074=g*JVSYJ*sa9UT95W&x20c(NYQ1*9m
z5fuj&qJpAb?xh<}KNZb;VIe4Z(RqXZJk)u8&~TIUq>C9-KCQo@sU^Z{b*C3HMt3^c
zM@#?Zq;+>ps;{k%yrH|~OUsQXJbO8fLQBK>(?b>0UtCUpJn5_2<TGJUqfR@2onX$y
zRoiuOLOpLq;JOK)t91nhMcF~!QW5!ml^@@IE;o1mE*E@!`(ESQyQdVHTBTe+R<QO;
z?VIU(tF-(J>OJQdDgL=}Y(em%>i(6Fv&Ck($6dO6<M-)H0ll6DpG&UBU1YlV_P*4m
zrG@)0FUhW#`}J|VJwrw1&P<aij(Pch62F?Vr#`YdI-{h_Yu{9R#fmzETA$o=nyxM`
zM?upGJSTH<7%R5Onk{ReA2W&J;w7II{yR3L{%d<-{rdE6o0A+BcXuAPHkaEYT(J6v
z&4!M&M}6z3w;X<R|MlO6YhN?%3VxmT-eOas+q#@6etYle^rR>6<m<VcB|aRy;<$CH
z)3vW3UV3l)yD4Gko5>shRc7zDf2CP8y&-ZllZ)WVi~G;VR!eM1R@M*{6czq|azT*l
z^wYj-lie0?EK^*1YGvDGCIQ#LJ)gPNPrZ%qpZ(?J?z{6Jtt)DeT5I?B(f-)IT>sT1
zrRy|g(hlwoet&IR=*!Txaq`dDp2)?Em;DuEkjQ&;zl6)7UGQ#}HY-Eut1S{&_t~A{
zls2DpP4u<qG<nYuYfHPM{6TuDsi{Yg9N}mxcwet2C}_*~@2%(3SBLrSL$szAKR?&o
z)3c>acj@sQBhed+3)a1T`*1~=?b%cBx~%N(@o9yI?ryq#m$||4<Ck~b41c13u6+M4
z<a+K~b@mPRI>$Iv=GhDL8XswJoA<5Uf2H&XCI-#A^=g{SUo5|)ICK7^=Nj#oE?r7H
zJ8SBsNuNF*m%n}MR@818rA0-Zl2g1we}8-H?Buj6boIBx{Pry^EfKqAw5vTzj9<M^
zHjv`|ZnnfpUsJ>SXO+bn2Ak}g&u_0S_?*RYK%`LbWWo-qzv>JZKU*HJX`YZHYg?np
zU?D#FON)J|tGKxM_p|2rj~qRE7PPzZ&dzIk?Hw!jJo>#L<W|&{3_&q5F$swsd3Seh
z*|Np1_E*I2MVr)qpZRsO+GOW6UdL<i{%$O763^YB^Z4b(x*OH8f4b(Jwo^9bV_=Cd
zU;cYm)ZDD=tmh8P)aojFN>AlJaQgV)ea0eL`udSu)Xc43wf%b<<$u{PS}}H4N#@5#
zM=dQaUteD@Z<NvjUO0992A@}0!2Y_w8#ZiM<~#e{&CSo3&#yDGy$~9^Gc?=y*XzK!
z(h+8}`07^nwb!?QlVkYR`!^@XkV%8#WWr0&_Lx2E`D3TtzH50nqw;ZZV1dn)m_&63
zfj2Dw-Ub{1t<ieF|9_mw(FF?>*g(DLBlW2#tg^1H0qwOqaNxk><Nf+^du}8cMC@K<
z<hthm#{a9EI3B!_KYo6`{kB&gIhPrQqpob!|8y$Jp0U5K_u`z`llMOAoO-|i|Gu1?
zM(OF#laKe6zPfVq)>8q&snur|q?l?=%{o8N_HY}ob@{tHb1aLEjE&dE2F^^npLmFY
z;n|N1ywcxJ&ieN9>Y1~*R6~Q(HdmgWrn~>or|#|B%OfHpCad|bi4_zKbUnc1bxmdZ
z>Feuaf4^KlKW=YT>9sYHo}M!7#r=o2zMPTQt1~~Iuj1dvxU(h)BaTQ46?N_M{Mcq6
zTCj8H&edUSZ|$$IKRr$N@S#JmuC7jh+ttyb#gP`UY2Qra^nV`?^Lu)F8mFK8^ZkDP
z+E_oyM{&oGetP)hy!&K1b)j3Kg&S%N?%98fGkV6<@ZeQO{MA4K28kl>Jw4ebDYF@O
z>|D9?!Nj=r@f#8xBO)Ts&$I29v3&IAO%B)+zO(=>;bVQWyK`@wH6JucKR4&|^Yh}h
zj;RVu<!*`=JuiN-UU7PN{mNQja|Q$V<I7)te(%3GF;e!x$$hhe-!e!u%<}&DeQu)1
z<qRK&IawB=pZ3cd85yOWpC>CTYg_f@g{*a%#?(0~4IL{~LO<1~uHbojdHMN=huh!2
zc_XG5BOxV~^|ok@->#Y`(<ih@y|4N8bAj46hDUsBxcpD(Jv&+?u77&MGVP6$Kg*=I
z%-YX*V@u7&P`&Rtm&6)Mp5EV5^Y`1$m7LQ=U6^aMN@vdxTmARh?EJhtI}B@oeR+C%
z`soz^O|w8{1(UJHRK>N?+wCepEopB*{_5)L?{97<zb%@#bN9((ZebtH4PXCy8Ook-
z^2&HYWAiT0e!+PH3_0;%+`SG~KZ(^4i8+44Zb7{;gLs+io!u&by)vc<I^4f@PbuO3
zPbUTzkBRB}%kNq`U;0zY)v)HJf_(d*$Is-yP3h89Q%k$HCNeWKb9MOoc~+&XK-p*2
zj2VtAb1MJ+eC|Kj>gtNX#j@6AYvT9EP15>sKqC3i)z{PfGgnG}m-YDne7ovyg>COH
zZW3cyu_sRc4p+m?l9I#oIAZtN3oX`rSQVnjpcy*7@c);vJm2{0P=T|{7T^BALx*vK
zk<+%E3rupdZf1oWnHeh2>K(Rbu-KRP-TtK|!=)sr>vKC=A6S`eDEt@cA^&kom%5VD
zquuZKO;*YL{_bx6{=eJ6&fLJVdFEz@pqZ0r&iwi7^?G^Bq9vY_)sm0(Xzuqe%J$EX
z-0-)j{yl&Dz8LNQ|DwbF&qZuoec<HX`+whv=Kr44vN`_D{hXV^KfXQTv+FxHg**G%
zleLps917l>RQ}24+fkZy_l<4f&Y$lUs><hndAFLce0%Ro^=~;himmvXw9jxdM17UI
zy0-N1pXT04$$oheJH5Gg?BtAE8(EP5@!N!VymQ|Bq{h!#&T#YA|Jt1i+AQ}AFSl1u
zdA`s`QBhIaET^ETh>MFW>)xK7V85K0;kdHr)X9^QhDk@3`Oe<6XV0;I`Sn37*Ti0&
zToJ*h>-+1>lpY;rmz}5X1uoTC^V<2+a+P!m)e~<+`uJ9<zmQoeAzAYK+uQ2z@4kNd
zVpH&-Vf*&&V1pm*h_jg*xNX}uY4g02^78-xe!ovYJIl22(UDhIS4Zz&QRKO@ENqp0
zW!s99vdUGC_dgZcr5sN&S{u9j+mTLT{e3@@0s<y<cXw~qR#IB^V}|3(H$p5Acfa5F
z`NM}FFBXGN%C<~A)M8coDq{DF<H@JaG8QC$&v(y_FAkdFv*}gczO7xGA5`yDcqz0v
zY0HF@LHhF<7*eJm-(JqkcDUdEU&X&aKlNgF<=ox1HS6lCYwJ2XRs<!)+e~E!EqFD{
znc<`M`$04Ry!wBY+0#EuTuM4OXYaDj8y)lV)?Lh4^XTFIWr1tCjD;@D_;mi^<nSk<
zPd04a=(w(T@_Fy4Z<n7wb0%g-!NaSo!`*u%HWoegl9sO1RaeqdJ+mOhuBD~r%e%YQ
z+S;$*Zoi**b=B0Zrx$i97#bG-`}5P&({q0Pzn?vl#%p5v1OuNQ;_-UgwsgCRon767
z1C6n}N;Z~RhU|;oTUC0niPdUu-pNU-@ArP62M+F%8ICJ$b{0N%OGtQdv|HS|^p#1*
z1qH_yBEidoJ3+<wrKR4ijyJYsUVgoPf8N`ojt(v5v;eL2OP4NHeS0G*C3S00<>$b~
zZYQ6LrnT@-o%Hy4|N7Y7*Y4e`dw*}Q{{BCkz_u)z;kfe8$#duAj8aZ)%e`GwQ)Bn%
z!(pw^RS~<Dw9L!i-`iXK{G3(kD;bM|1&N2-z%}!yB(|kRo|Be*e0==>hr|4Ga&p4z
zekDako61~6u6a({Qup`QV)y=i9}aOJJa{lKFAowK$!trD-rUaLKi8&m)8fUC-Q{ba
zJbak=wrIthQzuVGZp(>GNl7{0C%ZlW{<<*xs%v#hN>$HhIIgU@y&>`NiHXX&xw-a#
zJ~+$E%V$q_PMzqtJbFjLLbt`0pU;}t|NEJ~&)LT%M9iQ$XlMD03k&zv{!U9z-~apF
z?z6Max0hLlY*SHJKR?@CUs3VlWPiJ#-)`qyPG=Gflsw4eCF<zn;&Odm?Bq$4*!boC
z+{#`bxm!j{dtL1AZ3zdP&YU^(^Yin5dHZ{yO7x=bnFS$^F?*{_4GcbfdU`r`SBa!S
z0>kXx8d_ReCnu>+QpvrsA@M*1BPg6Y-W}%g5_R|T^7{VnuK#?yzn8rAdu1#qW!HzN
zsr%3S^K$w85Us20Vt3!#nw<`czZD8;0jmPORQ;9C-?K6Oyj*<z{f)`TwWfMyhr5Ic
zh=^<{eI3Tn-~adb_us#NNAH$VQqp2E)(ACGoBZ<c_xtf1lU(24+N$n1=f#5u32(br
zeDRpHCG+yK!|nXPfBt-Xe?RElAW&3vaH-EW%bm3~TS3cnkp}2o5Rs#?yUXTU79YER
ze}B~0tZRB*Ud<OXVm2nZW@Nn3Ucaa4#RWxiaq;YM(16C#vVc_^!q&&xrk#;k8f2Mw
z$09#JKYP0K`LM9CYl~dF|NVSEpOKjjRLn07h}f;9q*Nu67O?8Xx3{;=6Amy;K53G6
zW`>&Ytccw*RbfHF!Ozdm7MGW|&%3kZ=jZ3$i!wkp^$LzN3qlUIwzrqRzh`S|T6$uF
z;`)8RR&71KaFV#Rv~~TzKU=nJu`GU8^Xa5IIH<c?C8ucZn=xa?fddD$!`FRzadC0}
z{e9Q;+PyB_um8Wd;$za{i$6Y}x8MKuS~NsMx8xM9)+I}p<lWsh*Rpt-?`*Tf?flbE
zrd(Us;pI?K@nco!>NjuRl)bre@ax8oj#J`k0jqTK?(e(1zyAN<-`~x%uDtmB{r>YA
zQ$M|XaIks*zhA3Yug)$gC~$CS0OhMi#}D&(P4&pjd$;@jzTcnE+h1QDp8w~^$E>Zl
zZi!x;Y%u$5^tK$(aq<2A=Np;XUtL?94f6MkjfTxZON~=cP08Q)^Vrd&sgI6ynr2_y
zQfB$3qNunye0^MQ<iyhV_xAev`GKQy9w-60fp(}{m#y(yI?Eu@X>ZlnHL-l6)8p31
z@2U8BzyAN--R19nXPXu3hAA!j1oHOExz^=;ayB<|%=Ul3SKTXZzOBqMWXqN<TUfcp
z?(8UhZ2$k~d4Bsp2O3jdT%Jl9YlOzB`_FrGak0Cb+p!N158vLFoBg)vMc|{ykDs5P
zKY!9BqvT^f=k5RB0q2m*No-5K-oD@epO1%UPu<^Nuh;Ltw>n%ud%AP$?wX&U7P)qB
z+Ppb@ZIo%!5svKX&Mq#h4U$u|tk;I!Ug|x)`1!f5+1K-4Tv!<MY~vcs;%8^NM75t3
z{oIy&dsoTJByc84F=!T2G&bIxe}7*)zdRoY$B*;%|B|z_v)^`UoR_cvv$62;u~%1D
zv$L|AW?wr4a#LVuBAY4`!{Un;Nk=;3>wX?ReLDNkkB?eYPu&s~+?ID&N<`$z)2F2m
z4lwHLgB<GOqS`4rMWg>_&ODRMOLuk_$HvCm|9Y_)T+J;ybvni9@0ZK|>i+X`uC0j-
z2ngtvG6gk@1*aOM1uXil6SD(!4nx+~szWWDF%8GHHf`QKT_;jVRP^fF=<T}E+d|ex
zUA-kLC>VG$nN3ye#rFGk(ncvA8ePf9`@Vkuyt&MBs<)flu_jjTb+Nm@owI(g6TR)s
zv17+TS}qzN<napWSnS^a?cLqz<(GS9EI&P+9>1;3a;j@m(xW|<pY!hR`FT`4zUJ%I
zaBw=Gr<fM7=uqsAf`#7G^$H6M|Nj2|{@U8vTTie2VIp<*P%HQ5O`E3a#adN;$pFX3
zc`;*+i_S-n9b;o-b8~kue{-Yo!-Io!tiIgr?CFWwmNQeMYqoj*vm{0rm#K`#8W$HY
zS-$-HtE;bne0+R%mTC3(yX8-dc5c;P)zBwvy)Eyq6+3%-?eA|}v#y>3$3!M*MR(J|
z1PP;*6W7*8OPgdE<lO-saucz85m)N-b8~l>zh4)<{oTQ4c4?E01aO2;F=!SN^_{eY
zQ&>$WZqJRocmFQ#w<`iyH=>Oyp4r#e+1CB3aBy%aDA=%i^=eS_V$r3;JYFF&#m~;n
zG|2>Q8$Z%1oPBjw>f54I=FJB;<lNi@%CW)A{cdl|T^+V|7FbWvQ68@^5@vaKERv3J
z?2P&MuwC9P=f;IwPp5=$&$}D7qu}9G?eI1p$xUfzrS_$IyM%Cp7BtNRl`sDD?QY(@
zx!A2Y>*Au;t*15XCrz64?EL)sGiF$1URn~lIgPU^0hGT_aLsV+l#5PDNx8l*R#-@g
zjaTZ*&f@g!>A~~<{C>ZmTTEw1>1$9SYWL#-GdMoNK<j-}{{8)a|G<F*_5XgV8yFNk
zIMA4LM!Sz&Tu&!*6Uzb65iVa}gKLlzvt~GU&I|DJ^6Hnf^_y!Iy4-IrIMvSCbS}fh
z&DC|~)mL|Smv7I#eGL>H6Q_cFV7WKWzUaw`v$M^kcTf0q@b>NM>+53o?b~-vucf17
xO~2$6$SBnH{c4c4V>~2oJHc3dM#zBw{5S6?y<4VsViD+AJ5N_Xmvv4FO#tx&pu7M8

literal 185267
zcmeAS@N?(olHy`uVBq!ia0y~yU~Xh!U`yj*V_;wqySy-vfkD!>DkP#LD6w3jpeR2r
zGbdG{q_QAYA+;hije()!*4kO=lVmI>wf=uBQq7{#Zo#+Y=Dmde{r8>2f=!pVZ{D^h
zFU`=%n5p1ni~Ekx_Wx_|)_*)+bzUzzcj@#;KPOkeEBxU7dA|Mp>kmHv{#~r`<4ft6
z<&XP<_e}or!e0Jo&6(++D|wqAfBVaAS}!}_UcS7vwyyK-YnS-^Kc$D3XuH?_{1x-t
zDzIz*<A<;BudA=ke<Hu_^Vj;w`B&TiZao$MZ?5*i8^?YIm3&+>!KPk3R&`_3l}i^4
z|9xBeL2>>o>q|dx2mPyk6Z1Rj)B1zIzunf~`1$ecieHu@@3gP1`8{2tR+j(U?;CqL
z{_a@x;qNK?y@{40{~ynnPbz<(_kG^^cW!zvt9GpS{kwJQcWL=KEk=9Sue8znFTMZz
z_sc&sa|$PJ^fgnrKiw*)G;yzDyR6%ehkLGzig};;c;r-ekM^?iHA|(cWArs6?H?ZV
zlG>*1Zm7FynuW~%wW@nwe=FZBDiwaO(f`<ilT7!2-0uH<_u&82=eG;rU2Vm2aJ&B;
z4eo2Ap_g@>XI{Qv)78@QTQs=(W3BzW_FW(CRm>(jRww4=6#V#Ud34K%z1uFGE`C(}
zZ1d;&*EmY+=Qaow<|Z2|PGpRC+7!^V-td}0N3ybg^VH*09;+}P)Oq(nZ>ML8Q?Kr;
zTQ$quzTeFFSS*pDC^(I4?WHRkD%$Cb>$38$T-cCub7thmth2j%Cwb|Gd98GnFwI)E
zGfH>swXkYg-K}@lObv_9-rFlz&2(tZ^tjmkz26OYym)Y>IiKIo@K}%WS)1Hxk?FH*
z`&@sn*^@IhG(5VtSGN4tmj8OYcik<!|G33jb8gL>DZ&2Me|x;w-YxmOH9Bwa+i%wU
zQgv&#zV(mf-k<Vr{${a#o0j!F{l4>;ZMMw1uj_+Z4t8p$r|3L$IJVlz!O{IG&y91I
zTE(YkzMr^kgU7WdO-pNy<P?MX0&m;$cmAHYv;NHN&41*d>i%1O#&YSq|H{RFD}SH5
zbN}<V*=yc@lu+HvvXWagc)s3&W9@D^f7s>zzkIEk8EL!m`_4-~(#w`UkD9&s+{1kC
z;)@!O-yH8--WMBrK}pW~W2@mQ6N#nktPfll5PoshHgfjeNT025LQfpaY_$-)U-5iS
zt#H`s+UYjOVqeD153`;swk+hkZr8H4cPkH1e<Swe_QB%?))70xUMvv4$+=ZE*rWLN
zOe3G~E?4996u--<-H+coNzhK{dFMNyZQE|G3p>WQ`ixxk$y*EU|6aS4%d}2Ca7y7+
z>F+&X=lL<eo%K%5J@sAD<lURwjq`QfE~M`&J$stZ%+V)Ve_x@(Z0VJa0p~iyZbhk{
zxx#TguVv$!*gvJdEOyuVjHf#tJn+DHKCkGOJ*VEyi`u7pW68<hY2C^5f}^+lewJX3
zUw_}c+Wge3iRT((U(8~>YifVenc+}q=FXs-TT*YQ%=C&$Zu6;ie-s(LVX@YNr3_||
zHU;0E+Z!OGy0|9IVrzN5_nq%7xA)%cn#tB<e7)$HRb`HK%x$yR(hK9ZFPd3*(@wr#
zygJ0HR$Xk_{Y_U+F5S=18lV)qQT^^kTh5Fn+0SH8`OW>pzJEv3i-{t#`K4!eSD*M=
zHqH6a`*rOeN2<N%ZV~>kVb&tIdvj>gj9_iQM)TQ^e_0>CGv~<s+mAKnQcm{%`mU}M
zXm^oob#Hb{cy{g+vzv2Ub)LP^%5c84F#XlHYXxq%(%0=4n10dde@6R+*tN&GTxE}2
zyq)xDtMVqr=Gv0@rLJD{J9}4s(A%#dp~Y3QzI0kZutdR9QT?gRaa%Y;w!}&0Gq)~(
zd*a#l&u^q}ytvkPiS^c#o8GfjWJ@|HFL;0BTVnE3JykZPHQT?q27h&YzTw#0<-3==
zDiy3ac9rQwDMye*&E;@0V}6Gc3+CSK$|;jyPI5YZ-r9!!{F5bbp6?1ixW=LQw&dSS
z@_Xl>opelxA@aRx!(o@1!L6KS@(U^_&tfbvvr>{<WOcm6>2cDu9TuJE9JCk;&uaX+
zb?ipWf{XQ{Pj03&Ojcj_I{a47T>tAu<uli^9XAzU;rM%s<dt=`suTK+S4L&T@Xunr
zb|pRk!QBQa9;+ztBh!=8m@jZX;Ag#%&YpW+#D4y+TUB>=X2)M`;azxX&Qjm4yQXeF
zs>>k8>+^3n`^H;^ji*?y_v~X)*X-sBWN1-e{Q3Q|o40dx&u=>ER}t>{yUp&9MYGb^
z1qKP%?^q<9&soYf)kws!e~G_O3FpFWcbDkvpVs()4Qbsu@5b6ntA)p|U$|dx$z=A6
zNu7Hu<1I#JgAjI>L#(HJt2}co92h^8%=zqa+$G~)NMr3az9&oqddhsK_QrPCUSG89
ztC#81!i3LuN7n3pyVUKg>6`9F4nG7JSx01vl&P^hHaacPKf|cv*y>R4w4KrHD)U>0
zlkXgub(n=F<)jws-9OD({N{o;NB9ShX|pao*D00ZTCa3)H)Fi9(BEr}*OF=k?^g3Q
zo}6?y^i0~O*%Qx2^A<jgsARd6bXX@R$w_Gb33f(-mj^nv@*`Q+@^A3xdcQ>_Kt72x
zYW<u8lVUQ~Z;w27Ztm)i0<OP?2MbHG87K5^JNZEI>Z})e1r2>}nuUA+?p*QoS=dsc
z_KWgQmPy)j9F=?_{Jf(w=>7t6U(er%ud=Vp%lVjbzw%4KuWu6-9x8m4&Yk-#bAiqV
z<{+zBUcF{{jpm$!TWn3)`xOhdEIuDfP^g&p?T(23h6g+*?-><3vzJ6nbPG#7ZTY}x
zZhvlKy2544l_D=@uba;$@rHlTN`;%dOt~iYEs}a+J8jXt9|xE_E^lBK4PAO`%86VP
zUa`(!%!k;{Fbc?Sb$AkM9<j^5j)(v48fO`XopKiy4K}F9zE=3cvF8K3RC|g*N=%vT
zFN5%`yK+~!A|Lg%9-0!8$U5PrVKdK>J&Hn7Pdvgld1S~L?=1<7nvt}eXJrJ-g2i8x
z9Bj{orilMk`hV%?g5`HJpSZoS?&*HQbSUyC>&oLEbGhF0Z!!~lWpzaIo6!OLF2ygL
zf{k-0ty4bcv@-6?D__N~;8%BchB6*|eSFr7wKvp7-?3z?<S+!h6}ToBz_Rn5!_u%#
z{>4`ex}99+F`jI?*LQD1#TVTfMQT4}yP9PJo~|t28hu-?lDmF$=mfji)CY^i4=~u=
z(YSQ7bk(;5+!?|@wHLZLTdbNK^Y-NOXl|h?9eS@i#6L}c^6c7^8ab0DwikkaJsB$Y
zvtNXBNY1=JZS||97kcRq41oulG#{HYPF&h~K!!_T#Vv-Z%Xeo*7Z^QHy!up5B&uoM
zgE|!r$L|aVZ}>Uo6VJ75NIthkG-lIMKgYXXN4Gi!Z^>rf*-+!M+G%UZ%D@RJg<@N#
zI!+0%+Pyfuf#b_`-TEKTUL9fm<#OAbKR~rk&xBh=Klh^k+{b*g7InT^D|CO|r$v1Z
zTnAT`2;RQaYpe4_;+?kti-dDsGowzt{{5hG(;}9N;|&4-6t?O2tA3MSoS?4|{gQ=`
zrPrrGPG!#Ku)W>O9OM~__PkY?P`q$C?*q@NKZCwc+Zm89-@=!{@$@0X3ulcI=EWQ@
zgL5}+t*~9tz$YMkwRZ`xn(5uYZf!30#)@xmN!&Qql<+%)+swn@xsQSVTJ|DI--pV&
zwSALs@Jw+F75>>FYJ0&{x5rBFMnc#7%+HZq*G}enGyU?3X&N%k8u!&^JHJ*F`LJki
zM$svQiGr847#2GVPHAJGX*$*RV}!s9HrAI*lxO-z2AW)d{<7f-=S-zf%&S~<rYOj0
zO;2$7%)L^Y!*S{-#WD{u7uky|S1^`2K4E$yW@}~b|3odNJ%P!1r-{#-jhnU~`pkRw
zVA+I)rOB_%Y`p(m^p@N%)U{^rg6b@@pv({lZn@(Zr&KXVzMUb|VmN<s@`Xz-T8F2+
znYS-zy3>9IH_!HWNB2+HD4%j+Aw$6SFE%$099LL>F|a_DbMKe$d(GnK#`c_v)UdUw
znDo=NpSLMx#;jK5=7ba_<84u|Bu^Zh^UzS>#)AY`xhujMs)ypuS}*<(o8#nlHDva#
zJASVmUsar#WfEe4M{7=_Y1099WiOL03$jm5VrYA-e8?++{ml_J&qEtdCRVeFzU~z|
z;NAG=fKq09`IJ9xQVVO6HijxNNX&NPUB0Ox?S|t9wmDX_ekooq$k?kSSRCuY{c6V9
z@7f=yYGtjA{kBtj#=Vfz2VW;0-^ggj&G7JRoXm8#_l5d3|CF~pv9?gvXgqT)r0FK3
z?JVgN8u2>US>3Zjl;85CJ)6sQH=gUBShhi-qGr@m3k9)#1xC+~UhXO|oSN}+vAk!6
z`?{6!E~b*tTyH4-b*nvY64LMZ=Z>p}0jJo0foq;eFSC?r++-;;Tz~TW_WL~FW0)NR
zbNsbW{JC&`f92FAjx(Az#4>rz>F($ae8V|W@!$7_SF&1N-K4Yb7j(=@+a?vIpyA+R
zW62|Z+u>5+`YkFa?ydW@=iv3J3Db0HJ0=%P*WI^OT`|Wk_sERB8y0fSRLD|k+;DMc
zj_f-QjV@JAEn&%ZoVu<5-0mGxTC-&RTz7{^Q_}^{Yb2}~m|k}>J36lHbO@K$xZf4g
zTy|Pnm`BjZJ78gsR&J8TN7rh(oxNSM@3j<5Dw^1p7sQ$FWlOklEArO~`^_^_^ct>;
z?74m0yu*1_#HD7Lw1nx`4w*68PW)5WdwxT8fPKa-u{Cn45gXOE<lD1_>YHCQOs}2P
zl36u**}~f@_0_fx>Bs)*yPVtUw87D8W#qnTyP1!e8Pu>#E>%mIq;-p%b%IOM$*iE3
z;P(w3E23^KJYr!I)wPLR!y%%^f?M`y+nf1YbQk4!v~ApEc0PIA%&Ub{O!Gv07d&j9
zrNM8pMl|!O$K##{7w&!8bNs@}fF{njHupaVd{{6sQ!X-_`NE29C5FEI8EU?nbzIUn
zn)kG6&%bk+Pef>g%IT>*hHC@w&lXc)TAZiy!*$Jix7LZXx}GrGv%M8gvO2YOJLBnw
z%YHWuR3v@(F#Jt(TYgT%LgkY5ALRot{svjiwY_CmQI+p|c<nQ$R)^#6GJ0W4B3f*p
z9Dme0rQ`gxyGte;#!S$z=GimB<!;BK6~_)7nXvVh#%>YSa}&g_sIat6kkc-4F}NbQ
zL|gUg{WC{PgwNE~vUd0v>F9XRR6cy$?Q?1GCV^9t--DafpP86nm~_sxVE;Dfh)E0d
z6VzAq_h+5tJI%CEVMg+1oeCL$#(6rOH#757gij`1P8GMBx$9+QQJ>0V|Gzcmm)E(i
zVb;AmCGbvZ+ZjR68=()PKYQ=%ob^m-i{bxcYcID&vi-m2_W#mi$7iaRQ$(ZuH*@~a
zs?mMLG4V>DMNN#RWk+|JlE;)kDZ#bR=5inK_m1wCn07^$`;~^@jrM0#bdGWdUumej
zsvxYr=;NaXYsbhd(;eSF+j*pE;=0!Iy}Nfgx2?D%&Nju%VLS8L&=Q}v#O|JT*Vb%1
zaKmuHOZm$(KX1BD&Hm&k92mm4WkCSjZuiY>28?qJpG@&*)Stj46=r|`pjFA?1zi^%
z|EapE?|Bit_x(AU_stnjmIcCXndh8$2Su=TT`~|f->mV&c9Po~RjJwu>0YkepQX7r
z?b0h_ke|?Cp6%1ybAy%js&SOqgGHAv{`$M;Bx{t(caw*)VLjJxN;yw%oftcXaY8Yt
zruJQl(%Y9@E&Y4>8f8w&{MBZ#uMwFlls9eefny2NZ^Rp&KBuq4(lPCb!vg*yfyU-1
z20RX`9_ME+>~Ys}Vf-0b>^4icM_Y2=Dx;ldn~o)!yw&8;-@sPpl=*BXE5q!(J?(lH
zFDBe){BY&xg!#hdEgJKs!X7a_joLlEqN)4C#3^<h3?C%+9N>Dnpl#Pu(WvLEwkvO3
zIeFe+!*;`f&`c+p0}+*}4C1;yyB4l0cvRBw`D{6Ngm!qc^J6#FHw!hZT23Vj9%0Mi
zI>x`|)?EjQMaJfhc?X`aS35QRh2JIqDPo2Dc5N2Wmpmst(JUY$wET|>D~H;{Ide@j
zMEXB}%RT&m`;z4yY(|EPOzn9rN(G7%EY?ivN)uMBeaA9CcAvb7!r2f-heDIP3!bJ}
zX9`=1ocA+N<C=DQN;&(Q%k2}G%&G!*_I>wS`@*y0k?8*qj?O_fis#swKVH#%;p2Me
z&VgkMK5}0Y$^Vn%AHQSHGY<QysX9!57HbKdR89#L*yg#JqyO$Vsl|)0e(WeoVKVpG
zq}{lj!7+N%A{obL57>%cDeN~<$?2(SQvNw9yn<Qa*7n#I{u!p3d-HnZ8=iKzOxexV
zC&7K>hWNK^w-TSAQ2X}2v^Sxb(?0Do{jj{U$;X53y5+)UQCl1*+%Ng-72JG+`LgrY
zpX+u%<;dtr?NVmDxV3GytN(N{E7<^LfnRN<yIQx-cInDqY%(Rk<J!cY!h;GC8ae9!
z6V@Ki_A`1sHKgU<&$ohF>sr|kg`0WtP3hOGT+(ynkNo=;sxdnpB_%sDr8Wc~Y`k!}
z<Ffb`&r%UThw98Uw(bil!OUNE_5VKM(P>j&;Fh-Fc$d6%-)X5^9vdFc4D?i;`Yyug
zF1MclmAx<7mc9Hr<C&?H59{Q+HPeN+H*Q$)rLgwXBf%xpE0ZcqB95$kti8BRVv$MT
z8ZT|hDNMikdDnRzD`vFcqBEf{W9IukC$xH3eDVD8Lg)LEy?K6*WU^b5**rg6ir!(6
zZCLI+dDAZM(*<i9(gcH?7-qK5z3B5m-tFi^UcqyzZ%j@1ax1?Q?3&RbeyKn2@LR<b
z{0G)n3(mN(@ONhF+y&A#hntq2PTZ}q+1ym{YD*>OQ33V0kCuMbaChDl)W-Xe@wr#k
z#98IDyNcto_!AZ^ZH(I1W$&2zxpA5F#kj*?xMt=?iOuDC_}J*9^Na5C=&2sAcN-0S
z6z16U<|yANjn?(DYm|Az%)8*wx>+l4nb|I5%9DMO>M{9A5MLPEiwm{e<U~r1EZqyQ
z$?fflTVeBH-QVU*|6We@ihp34I_asp%(v|49$^vlCT6v`TZQiJSRfdts$(2~GCk`9
z`^?2ji#b<6ozz~`<QR8jO^4vs<u-G)|8$8?-0{7sO+%LXVE}ilTKBBilQ>SO7aKAZ
z3%&J`K6pH&EV97-*>W9$bssI)*+1ALEu=WP_F=t9xNOjw6RA6@_MJ)eI2R>r<(6?b
zVuzT9&9+5u|1#2(e#><<tY}=t*d4I0hT(}rp{j*$6~lyS*{AH=xU_GXifZqAzdOO`
zO{v!x#%kjUOsefgwl_ScwU?}s3Ym0GAv5dJ)(SPj3ob4R&l+8r89p>^t?3bRx$q|b
z<-_^$oC*pHJ}OqtQh4<(W?$%`2Oj@s>Cb3gdQWk>h20ws`Fkwdww`n5Eo@oYeusJ0
zv`?3KmMeB0`^vK8upa9kW*x(Y9_0zUGYecT7pnza-NwCBjzK*Bfp@@^-8V`lv+v3u
zTe946>OCh`9?AL4E|2B(nlio4D_lG-kuuT3iE(CHpOmHO^6ncvOwwQMk5rY;z9ela
z`tgB9kKA;zoC?1wUycN^Of-1g`^dw8UR3$QMOpV&RQ`30|MSaA;H7wU&T&Px2}yQu
z-K-``Z8=vxOZy?eRM~0oa>h6Zm7XHU=pVeHEeWx^b{BVa*#EOq)Zcw9B=7Rw6iI(y
z{*@9bj~k1eZVRe=tX^<d>$sq;@5Y>nmU4f6r@4L=;fkM5{`5R@Naw+QbH(Flvui$9
zRy5^J(%dgKCGPG=b~Tlo+ur?OoH42TyS?f4t6%2Db}xE(hVPAX5Z|Md;Y=JJ^KK-t
zF!ja%m)hU^M>p=BJ-2~UOnKIii%yp7=gqxv=K7qHJ@c$TEDU+jQQo1h<@fkoN{Pg|
zidWwPFA8fEc=1b3o{^X5-!;>iaZ~kf+Z*eag+%z-bzX8-miBR&`jXAyR>>WK#mk(z
zB^~c7zAt$!`GPaJIr@mt@=c{5mmU0*vt3d6LG;SY6MtXkk=|K7!S`{D?M_FgK8vl=
zml9uxnO)rzoSo-el`G;BbLlhJ*S>wnKObD?uJcFUsiFKnpGbY$rGU69FXdj%gRXlQ
z^)+{G&b;KFo)LCRx-i2->G8UYw+k=$wRrn$b(~zu&K)7(+RD+W>iPAExvb5r&X$Ir
z>=O&p%Y|kgy}l^&MEbF3zpp5>7lvgjlvJ?_EO)d1eL#dEWS_>*vZw=BKG(f|Vd%@2
zHkWnhX|H8+osMVS9<<(BBdWA#L3O}~Dv2tGuC<8_%lIRhbp+=WoYS+=bT%ngb1Z4R
ze>T?khHFOE-=2wGX}>LGc8JvKU*J&@^iP_b8^rBj#@SQ-+wjr5{(zrX${)Av&8_U`
zjyX_O()Ye<&XV8LWTfiq<gNt#e`zRFG5IOm2AQ)ZOD9d_6}fpK*l2z=U+Co?=4CRU
z+*_4pWQ3TiSJ!zcUB0kW=2N?bd<n0GYo~0CMP1z<oAglC4|6t#o_IMgQ08~a2M=wQ
zN~i0+iUwsFG5rTD0<^EL`5swv+m-*gM5{1s(1QZ!$0wd%m}VC9u;lqprM`=kIdac@
zpD8UH?E3d;$>lEI%y~S#35<5<nOT$^zt(zYe1Go0KJ%WcsignWhSQGxFL{zYmp|Ur
zuf|Z3mo@EOz9sV>PTQ#olNU=na>$kHKc8Xwu#{hB^N*@L>3X{la{~FR0~fZi*ml2p
zT~+Yh&E>Jm>fmR;UDhw$*e{oBa@{6IqP8br=yX<#X_H`uKoMKii!bHkZ=D#vU;MC7
zi;ZUm&-uBL{7*{m+H6~Od`H>+fG3CkcK$YHE^#hTD!c0IVH;_2vR^Ln#IBW>zWkXj
z$9~CH*>APF+(i}#nbj5N7m4>?nBQ6Sf7!B+PP=#iXBFe<7h2!{E}VgZM?KToIl$A|
zSs@_E-_6s<m4TsR&eZAb5y2<L8lF#6QBm3Az_O&}p3BS7tPDn$C0u4sLLM55DlCH5
zn^qi%3fP$78t$5t&6u(=QNKIkJwLa`sa+8p7JgFWT(J6n_514Qzn|T+Pd_)u^7);e
zldTz694|QXC?|xuK_+mYikZm-OGV38&ngxH2L_P^+&&gDD=ie%)Px))cl{5(cemkM
zz`9$fYyaN)AMZN(`1B%1ha`pNAD7&bIl<0&;O>qjPdkPdxfSNKr|Gx(v;;Eca`Fgl
z+BwH~${WS!CaUgQIggSfCj>JFEG)8^#KUmqNueS44#^AsQLHKKg8mv!cX)12*M9Ip
zp}x&VOI=-j(=OAL*IVBIN%?a){niG5KK0}oQ+H~(sCsCKrWY^!KmX~E`;#W?O3ymX
zW46jcNKE^DWZd>OcQ`DA73T{tb#-;V)>PwrBKXGXo!$!E-V$5>ADFr+;Vc6)!<18x
zSzH~=8gjHw^>xWy`Mr$W`No9Aw-+lC?X2T0#h!W=CA0rs+**@aa7Un6rEBLCv(!y%
zS{fYZOS9B`n{a-al4H#zBfl+c9cD<dM)H2L%?yh<vf%7zB}VP66K+b~Q;+k^w`9Ad
zs<_ch`izac)fDk(ssfu9vsoI&3&<~%>GpkCB(m|P)43<!>MndoPd;X6)&6{_>*u1D
zgFiNU71kxZS@1Wku<XnN+iPde^G$v2tGL~Z`%&AmZ_f@NU3NyC{n4Kp6{$Hp%GA_t
z+$5RvxRn^WSs2wO9Gu2h<ftaF?VHJ$1M#Qk2^?S)n!xzuKwW^biIV3<<)-Tv0WANT
zj<8>PSlHMVXQ$EdOaE}x)2nhzw5zy9ls7quKamk&-1IPiX3~_5ift|*Sh8B|&z@`D
zn08q0{cq3XA@Y|5n>!tk7$k9AWV?R6wlLpDDDH1!@gduS^9y(EyD7Yj<86z3%j*`t
zLvL3eN;=6u(fFXn1;%6^$uoLyf4?d4esNWl|48kE`WMpa4}KkxD{nMreNb1B7rx@J
zUD(ZnWB0QxUv$55a674d;$O;ZdF6o3p~bS_l-6JQzBS<3RI?b}YX#@~D>v+0Ezf^X
zxo*o5u_X2ln{RNwvANb`%Ai)plYK-l(fx_d9<zBG*N$Dt+1ORFX@UGh|IdYN&!>d@
zz6yAL;q8aY4g60&bANuS&#fJ96RUJa;E6HwgS@lpn_TZ)Ts*^%udhIS>ifT^);{HR
z-nd-3hx>j><#EOjI?bng@~!ypH1m}+I@EvnP5&mP_j@UWOy=WLDy&|zdiziO54`el
z_9FJ42lrBWjf<G_4oLrKP7vT*<Y?o;7Mw7bwP3E-0TTn^Gt5gD$mp<YAB@@{n8Rp&
zAa8@bNkduz(;P-|N9i4W?+$If#2MER_kr(=GKZ5>28ZlL=N>`sBYcHUHUjY~5}htd
z6H`<~Pdd1C@XYKy<9=$QSBmZ?$(62k?tIGX6Fz%b7fDxklqj#C=&#Wh6qtKOV5`gV
z1#%e-QjNzJPrk@w!msP(ec|YftSxMJTdyrvztH`nbPIEB<F`bH;G+^VIMezo6LfT>
zyboS7<TR7kKD=s!R1W_&mib>o*D$|6X!povPM>jM^^Um1A6VveKYzISBUg>?KIZku
z{U3&ZY_HJ&H`PH|Lf{QY9E({ZlVY<$Z-Cbdr5t4$r8#Zy{nQ`IPV`6-SljXL!(DH+
zG@*0>?T*)-)t$#X-yS)B#7?MliTftypA!~&$^_Z1oU)~DOLs|6N$VBQUketkN?P-1
ziO`}+i<E*i!&U~}Tv-(IwChyx)VfV<H=EwL_Nje7arvb8iRhEQPfkDaE3$W;oaFj(
z;gMh?&CeREHLEq2Ykt?r4+#zR4YdtDzM3oa`Konccb8ZBeO>s<eU;eNX{Q$XEjqQL
zd*$tw$5(D&nV(@YOZDvhX-=n?>n^)`$?uZf1-lg&uez<?c(o#{_{yrQudc3Ju{T70
zb@U4M75;(xLF<F;7t}@Wb#Gpixbono2N`Fst+-~frZ9N1|KxQqS4gg%xj1s+&fw2`
zo*C;-nzl4-gP8wnm+bCqo@<Yn^yImvU)=fS=a;o#E`RaMU9@egS+sTV?wIQBWvk29
z_f6~T_FLq)&~NKG!MS{F$2O#Fki6NW+Vi(Zy2oFc$DO6;L4Slv()kW;<$JIDu6(`z
z^!2ot@2>7$_uI37rTCiZOSk*1cYMFkZe9Lr`?&g9|JeRs{$;DY)xE0!NuQ4N&iW^^
z%vTOvQ@EDUc2KO@o2|CZt!-gj(qYp!b>8Y83AcZ4Gu^a%`lL#wFZb5=wjR^&>6f%m
ze`UZt<K~&GXM)ef2{TVN5T1E5ZF1P;x?t_gESHlm?_Ku&tVWt#+RHTUGf`)%&i0-?
zKJ&cEdgJ}cj@v$L5Qtc_$s+sXy-TNzP8Kz$Ha|7ij@8~4Ix%$aRom=YTj!U~EA9Pi
zD!r(;-FMpAs@Z1ISFT+RONh>`zR9<>c(d@WwcE4hT_v7cdYP<ByW(5Ds<xrFv3Bm?
z!xwulPF&pWZr)R8d923g)Qr^RbA3NwJ$N<Y?S`ul*Daowyd!yI^5n<a>e1@!g125a
zO`lcV@mc1%+w*X}GQFDVyn5SXUaor_BfewXj=I9)^2s~i7QQYsFZVBve_nUl`L^=0
z#fKixQ9o#Wp|~%7^K*B7^Yu0B|Hjo<->y!teE+4+>{^yi{hMnW`)hq_E$n93TGh<l
zw{qX?8tXqke=O=u>-ztk{Zskx_MiD|H&{~G-f&e&z7Wvhv=R9+@9EK5N3G;qSX<eq
zvTkL|W>sfd&s!&uBsEDeNc@uYD=91GFBKAsI!ZZ_H+;7#&OW^^^?Tdy_VZl!qK>^C
z-4W6sH7=P23Eec|lNGni`?;e!P%LoKina`avwd-Lwf4uf)byYAe?0Qh@bT*7<wvcK
zo$IlccJH4pwcYq#Y)!(SNk4XTUgO-<GR@`d5zo%8o$n_%P12fdrPedkAp6kKl)ov7
zMaf*6s)ni2UiXhY=+ss(^DCMgV)gP^%IcG?lh6BKoIlC->4}i9E9b0a58_<uxzak=
zzIc(P>70EgWw&;1iPP)e-SxZa`T6LBub=(?p#9VTa_uGU%kxuzr0AS~A@+lF6^pIY
zw1TxCLenCi9Xs<q@?zxe&1{<{Z~Fdh#giq^WW+K%H@Pe?>Mi0fzJ0XpQBa}#yzqOc
z_QZL%q|6qX=r_&mS@z-nH{M3sH}yBQv+iTP$|}E1)z3O$MWlw-9+SIezZSjp?{?~{
zo85T+!MTi?nMTF;&a`i4O3R(sd&Y0}U8CbUekbRg+qtl~{LJDrr)Pan-{@SJ`tkNp
z&(E{Hc)XuYv-zj`W%{n5twEkI&$8a*Hq$#I&LzH0uPAOy+QsWeaW`9w+CQ!ObYApM
z7x#Xvf4hENZ#??4J9_#K-J2Vx><HQN<(Sd_RBJ6$?QSi3efjNs>R5|iZ+30hUcSyI
z{%O$GLebQzIibg2ow(YYedfAO?B~k032(o?I`C@mYo2SmQQzxq^Z!oe{U`k`$K+1L
zzK@sH9xjWX?QDMidc?JmT*W-i8<)147QViD-==Q$-DUgN*4&={cF|i|>!k8It4i0e
zy_>nA*i$x2ef8^mrkUZl-z~V?Tm9yH&b`w&qWAH?nf$i;U-hf^=HJ}k<Xb){OgQh@
z-Ta#GBLCE@%3m+Pu9jxumF+#*`?mcs|8|o~bJ<_XU*vDk_MNr5|AkCs(UhVsPhK3@
zxG1^y_|fClekpS*?ig)-`tBsB`ue$2^{c*?9C=xs?sRUBjda1c8<z8n&pzLKPJZ8m
z!i5hP-9Gxud);)tzb}71Ul_Li+ON~^rc2+pxjpOF@on;ZAC*5na&^J#^mQB8$L^6U
z-TT+=P;a!k`rS7>j_&^YrtI6>@N%wqU+&MX?k`yXX5X>DtJvB2v;L)iUcXm1Pu5+I
z&34ZhjdvIC9=~&3f1bzuns+)c7nO$En$|`AKJzlNxZI=s_UF#?*6|(jU%pTIxAf=r
zUDL1W^Z$GM|NGK=oO|c4S6R;%7ry6W_2$28Pv4%lZ^l0M_eJ-<?$`gl;Cte6_x}EL
zd)3;iTlQ~_Uw(e2Zx%mmpIgmd^9jZV3eP+&y8Ib=%uH-$8?N7GT%dY5zuk;MLwIVs
zajU|q&r6yP)F(at>8qH>kn^2gAyJQE;s@ze<{YsJD?jM{XX*dWzGCy|0%?)VU7IIg
z5*M8p!6DDac29!IA&cojcS^#7`o9MbOkZno<2HlL`QUk+PnGPO^Q4rQZ#y^rf%{MM
zjC+Mzt@|g`F);8S%?ybsiSYHYO3u&KOH9d6O4X~#Enolv8~cia#N_PM5{0DH^vpb4
zrT4q{D=B2A*eZpa`WpBaIHzW0dQ=sq23ProBv)l8Tc#-4+i}@cSXJZ}<ffJ+Dpcg=
z<P}?0*eZd|waP2D0txFYDS(xfWZNo5_y#CA=NF|anCO}48R)uJWR@8z*>Ne@6s4qD
z1-ZCEjVMY<vsKC{DJihh*Do(G*DE*H%P&gTH?*|0)Hg8FH!{*KN=ef#uFNY*tkBIX
zRt6aXF~cRbIJqdZpd>RtPXT0RVp4u-iLH_nmx6)<)bPxLl4RG461W8*KG^u;k`#T<
zf|6vDirfO%iV}Sz0|N_P10!7{OMPVh6}bhzzHr@n#n4bp&d=4aNG#Ad)HBe}%|+2s
zT;f`Wun((_;*iRMRQ;gT;{4L0<W$$P)Vva9WJ?QjN{f;0EG<q234z_6lai)ikqfaI
zS&zG?Zvb2mC>YZ-a|^&aK&p{drX<7F6_gg`fYqcV>!;?V=BDPA6zd!68KQUyv}6yi
z1I0fe8E_CF8()!IfL%2#yugaV;cDfQpIi#E)YHXQ3FKa@l>Fq(6e}<@Eyc*d(!xAN
zH_gyIN!KLN(o{FeJjGBq#XQ9%$;dR>(jYkz$tcgf;*!L?<W!JR6}bg^nVBh8#wJN7
zCWa<Sx~VA!sk$cSi7C2?#+IqN2B{{7Nk*xrmPQtaNJjV<Wu|B5CFUTz3S?ADW{Op+
zsj-1Ya!QhJYNBO|u8Bpmxo)C)s)cTHnwh0xvZ0}=iJ>LfC{R3DIR<#zDjDe+AVdOk
z5=+wZi*jw1d=rxu{DToPA(^?U!6k{HP%|_&wy-cZHMKM~H8(Y}G(ji|OD!tS%+CWE
zYG|Nm29`)kwsOla%1tb>Rm#jwOi$G>$V&%HfTG1JAhRMhC&DE&H#HBc)4~Gc9)xrx
zlC+_P0YthYx4_D|C^fMpzbGU>KgU)H<Rb+mJwpR<##68XWg*v!5-Xql<U~-`11nBV
zu><EiaNc%GOxFi<ZS?Ue#xN;3BfmTktPzsG@tA{93OB{EsH8l<DBHitKPf9UxdgvC
z*c8J}3QA4SEG{Xk^d#UZG<7zRMD3ATT#{c@iDZ9pY9WLN@m5YIIH(j9z(s{sVlpTc
zZIy}>%TiOo5vTx@O3p~kOHWO)Rf1+nn0O|(#GYuDW?*S(qMK-DW~ytFVv($yWRhg4
zn_^;SWNDI?oSK*fNjGrQi}TY;$`gxH9n(|uN^F(fGjj{T@vNW$Nu!#mjw#Pb1tmZO
zLknF)16>2N5Cd~7LrW`DQ(XgdD+2>1NR%ts=!1%4n3rupg_W<b6>^~1fC@7!Pf(f-
zE-fg?$xJPR1P?gz1g924OtR6(p$<tuIPK*V)Q+qSqT4yYpt2}4J)^|GJP)t+SQJ7`
z@X1V0%`3)hH?k5)upre~c3jY)3UYC?<Fe5QSDT<J6cQGoDvXvGngnTSg@VGUB_xIK
zXmE`N7fB&NlH$?SH5yzbg#by4M^hKof{P2$H&4w=u~jNpvbUQh$t}Xbz`&N|?e4<x
z9|RZdT|SwCfq}EYBeIx*K};2d89jqkJ}@vau$OrHy0SmwV&fAwi}6?&%D|w&;OXKR
zQgQ3e-SU{wuaA#^y#92PIY&qJ`@P@899#+&))hsknf-|}+xP4AnP1y-BeM>hY=7h=
zA~SRGnkzo8#^2*CW-^uk`F9{B#K%`y@BQ<~7at3j7ry%~$N#+c{PUW=gDFOlf?zPa
z=;x<YE)flf|Ns8}e6@PLNJmU^etP<H5f>21ITQc;CtEY)+t1$Hgfa{yrZjbcK<~~%
z`}p5~ij-?r6_h|=QvVHYCa`(8A3uG1w2_%zBX}9lzJ2@l{Qvix%Qb|_6>Qqkr>Cb|
zmb?hqcDqmBp6|fT&FMe?e!su@?(=8Q+;$c{6^XG+*$}mMmTU+}wMxHs<R%sOKAD}%
zpC37POh`s%&4*8tbA>fP7A7%siR;yzKd%|LXGc>orwGXIiF;P$-``jMH=_BV!`7^;
zD?(OILXxWf{_e)@`*)|vhpnD!SNluDWU2#5li<nKTp}6;{?EUEeRDH-;YyY+kOw{D
zuB{GVzrVKUV8VjJ$H%5<1UfY$q;?jk&zU>-{SOIVR#B)a(hr|KbE~NMAuFG3BGuW`
z6B41-2y&-V5_i@2ceTfV-`<wnJ%9fErXYx01)~Logqo&L5AXY)t+8SS#MT=E?(Xip
z%HP-B{yk^ToELX?ZgzqzpLAJg;ascILx&GLGeTX`p__SglWJpQqXbXeZmsr3Lg0{5
zs;pGMf39zyZFSPtsP%WBJ$&exk+I^CAk?AJC7*Q`3JMBZSXydI1VS9M(e>-AtBXrs
zUb39#1QimLE`Iz}A!=*ZQU?}>YFBV9S6+Q{bF-v<olV;2$izEtRaI6D^XAP9SsCQo
z_+&xQQZJpD9UrE3mgG!5SoQekBCzi(JyKFsK0iOde9G$XwjECoq-;zuk@)0$aA!Ds
z&}9Y>503-q&+EtUNtXpXWTK7!vO?J&Ez2VLV{h&q-y^*I^VRBW;%Z={l%N5s1oJ0V
z1;{Y`KOZz)ikQDkRcz6BI`!vi#1uEN?F>DvF~vKKc~Ac|jtKtu_oz`=c3X9KnuW^W
z6FW0sf3bu{-yQ4c$JrlUH()rv`L@OWcT<=c-$e5+i};>ZUej}Qj@z@1_B;2KNNGm<
z+wW;LWO-m)remjCyY6nvz3uVxmuxQ-1b~w7#BA2ZR(4;R7z+4bcR6-4Bt(4^YyZ{%
z^wY~Pub1j4TBobkpZ)p1{^_ZsC2xB4ET*pUJg`|<!(zwL!_gA2BPTsxWU;?c{@VV$
zOYTb`$*0Kvjdjc8oISVLnO{8)^xJ*<`swTQW9}6^sN%HQKmE%5u*nv=Ov~rJKYw!S
z?{#;dU&>LK7b+iDwru{j2|vDi8GT&|P0M%8o2}>RYRCK+bl;<-H05-!;N2h1hQDW~
zzTA9I_2Z(SQDu5lwd|e*RrQ<+sa5;#{2Y=67(!k*F5fR0e5mk)Zr5JH9?|HBugvCO
znKGMAW*2Kh{E_d=f{Fw$?=z14yQ1c3cI(Nik1Wqxmb|{ZGEX|DOYDdP+p^?n&)Zcm
zuFd^&?4d<u`Gil!TzU)7ZY<y7^6Jsm8UL$#v(6Ui1@l<9`{+zRSCD%3SN8Y2XVyg-
z$7qWko-niP<(b5FIlpX=Kb%~?ImdGPrj&Jae2!f{&dtz1Nwnc^`0Zo{zg+2u57G|I
z;+%H5+T->v_2bhwY2O!_eyVXoW>ZhfrTX}0O~+=tP)La<r_jH_O~PQZ#ZT2Uw)>Cl
zU!%>o=VRR-%axNiowv!nzQq3MuPeJ&GUq+N@i{-)jOD@KBN;Y-Hy@iC6}+vu<i_m?
zua<xht#v!9?)_MkH>2<WnU(+RdUjUS+Bn^Q6u8)^Z=Ktj<=Kxe?pD$Zm(O4I<mATn
z`oSNr9xwb_c*u+GK?mc8Z?F6gWuKo|^rr2n53lM6$-i3nQf)WuKcAp-iGS{a6r-ec
zE006f_y4`!H1nOBL$h@s<Gbs1XKrj~W;m{=%<(2h{_)wlhJQ`O)%H6->YaQ-VpVX$
zpUV|uXZM}4G3)60Vd1y;``WaeS_TstaFKmPndLoKUWD*t&s*t=mSwlME>7+Hdh2n)
z<-@w#_n%%^zIwf=Q1-9JHB1}{wx@rJ-&YmhFw4LA+0pV16)gcJG4?SwlXv_$Yr*%m
zzOVSUFvAA@TN9KXvp*2IBYEHLYC(x<5^GPRXyG>hz1w8U&h7irE7`NUU-((Dw8f?M
z^6kI-RbMWWY`)0<fBF1VW-*ulPF=&4;d$fG#k#!~U;YL8G#q@<SbVc<N4QS=;_cek
zc1LWxZIr&g?VIepqrcZb@|Mcn&G+!ibKkC#b^WrLF~6Semw9!)`|x{x1z`gV)<?hJ
zL|;EIyJfZi!jtPCL|zfT{E;{E^qu+>TT=_yNcmjc$h>9UDo7IkQ|xDZJCT3My7*b4
zZ?>ILwlsWg*Idh4V->#dt7r7SXMg5L@*7!9J-B+}^{>BIT|LZ^@@LnKEh5@-=gW^X
zdPINYn||#t<F(ByzCRkY!;cq8bzMDX@Or<D+&%{5$mjPWPd+#k+-&hFQ)Ie%zEa{r
zz8=5uc>A9*VU}C|dEPqtWX<gw?pXU<Vi!2)b49vezGSxUgw}HF^y0L)7gww5Hh+Hn
zyWq|K`H$<wPd|H9_S$A8+xsAktGNx{5&3`a|FOBPs&eVkK9!e60*sDJ89nZQP!+!}
zd*;_&>%|uTZd$ErU=**JFViaf;9HIToVf?G?9R>nUsty^Kz*{w!%)4s=b|K%S1NFA
z(Ox3@P5u(3fI4}fNA$zWaEa_+jCyDGSMkmMsw2Lr{K3WLRt)QMtrE9aeAjgroV>Vh
z$$`~xH!|O8-d&V<IfUVk)2l><-Xk-36T;>DgCfPF%6Y5jK6?Als_9wF$MrKkJ|5n=
zNIi4^(a96#8oPyOd_OvYZ9^)f_%;T%@;Cgd4`R;Eof2=BVta8vn_pabfAB2d>*v1~
z7w&kt;=$L`Rw?hSTDFFMT+<ny6{hmB;rCtr+t*ItytmtMgI??Wm9yWP1s_V(bN5#-
zyeogByi&G$Qy-Vw(Z5B(jq@BY9Tlti#W3$_p>~|m^{&Y05wrO7)E*x@9yNLAmL%<a
z%&*dQ)0@-g#GCrRUhB00)RH*ycVo7AU$=0CX50Jj^@#=B*nJ}2cE`W(oEQ2r>&elh
zzBjuCs>LTw5onpdUvPKO*TT>Lr%n0%^~0)1tjn)WsyV}?+Lf4p#_HF!ZI#+H&3hJY
znPp^n!@9vX<q=cN!AX~S^sl7|{r|4h$?){{<NxO(?l9+Dl%K2kz1;G84omlDy|)Q>
zi-kU46`D}_O-W>R3IFM7fA>b~=RG{~e6^v-hn6KQKJDKUJa<0)9>#UXvLxH+<nF#l
zH@8@@#S~;-;&WeaZBz3%?b7dS$4_z`Q<`e%)F=F~1L7}_b3E@VSW{)+ZeAu`xVq}^
zCbqE6KVMBax~A>v#f949>*Zda4m(qKqc{9}mc)l+n`={p`nMkB^oS9j`*p^a*W$+e
zKOJ__JAQxRJ_)ZsXFp9l@O$dSA0Z!(XW1S9>G^oWwko;(T;{$@ib}Y{gOm%3S$7A$
z{g5YecB$|5jrq~0uV2jh%5w6xpW^{ro!h==()lLtEdO2Wo^LgE&fm3nUmm=cJmb>A
zlcEN<?*H5fZ7xh&{7ClW7xN_B-4m7d{q}tQ_sT!0D8-wnA$DWh4)NL3N=x1!GZNn1
zqaC~YVGhgkTecrp2k@+Z$<evmFIC%uO{Q>NT;u7FyYv6rKE3@Q?h(_;)%tf9GK8rY
zeA3=w(WsipeLVgLgYkoVugaFb<VcVI-Q|?ef9&Rqj`J5hcYOCaa`3pYwr9+|pFg)M
zPA}h^vwB<V{<{_Bp@$l{^-8@a*1S1z*3>9F@T9`u53c{L1b&pe|I$h4zjHTd=_F-e
z9%WDi=;yTP{<Is6C+*kl_{&^&PUYjHuSSn$1*>Ks+S=Sb`+S=1vE@4NT<3b+-_vyF
z_MSQ3AA)!MlFrjD*XKXB-t(vR`yJakjy=r^_KLOs9p$-c)*AD&Rj-3CZ=7{t){gny
zg|9vD?(gP4z3*|ve)SzYUOixr-FDmaZqbil+1GRGPXFaQ%Jt`~htF^G31y{9axvZy
z!`>~Mm88+u{Q9a;g{GtSI!K~8QOoh7EnifA%`)YfKFf;V$0H{1+){IVtNhz+w)<tv
z&S(c_ualj-X_8COnYP#EYDXr{N!S;q@|UCHvu@1kRNnhrwC~Srn!l%Mvtcv0@$+<T
zYxDS~gXfMv{Z%BQHT|poyRU`1HD7Pln9mc=&}h8>f8Md9oR_YhfA(pO{*Dhf)nn%E
zx2X=_Gd-2LV8_E3^M9T@{#a*q(z>$-S>dZB<;=ybomux}i?SUM**|GYi?PbTnh&pv
zEFN85T>kBlgka0#t2z39-8+9tv2OhT`ols$fj#O{d|Kd;-@ss|n&9Jd?PsB++M7ST
z6B73RP1c`tv}S9v<cGumuP(RvcwLp}Us0K6XZ!9)(U&-1uhhNS-V-%>r$xE)yu;Bc
zyn9QuJP%%+ym7|a8?kenKA&pK_qdaO&~C-Ot=U4iM0NU`rBmYO<{X_=^!LW1;>`<1
z*4~n?mOr2U{J7*Ct$>tMITn(KFP^`=F80S=(;S|qhw>t1HcNU}39Yo(e{yF_X!+ip
zwnx4nb7P;~dM(j?{qWjXW<ihmp9U-_FZi{(RHS|rYuS!~2S>{{?D@{7xt#0a?^7$*
zoU9P~|Kv<{H^Ye;U+wim|M9$E<acBGYljCHM9%p{wHUsT<SIR(wO8}>K82^Ro~Pv8
zV`$%>X7E*S$;qU5jR&)~Mu|3qO1CGj6DLkQGv9vx-Q3wpCU>5eD7FcS%=u%s?q=Fz
zlj6)vm$#q!_tGp&h%t0e-dpp|-4@@@{K#2)_`;J)FT0BkOC<k4yu<sP`QOoXPZhep
z<aY1<c+9D+^T^WR9UCg7*PZuSk`Q}8dfxPHi9*ZwN3pY&?>S!A<L`Mr`+MrU|MObn
z?`U3G{_kow_p-v5vp>c}l?$vdPkSg`@!V(9^IkQ@X9w=<&5k`Su4%a7TW<W}E$M>C
z|4sj_D*fiww?zs26Xw`%_WwU~<MV32|L;$~IeJ#0;dzJnhn0akw<ow}|FYCOH|N09
zQufV768|ObT=VDpnPlFpx14@C@UBeH&At7V>VA_aOlX)te?BDTcbuJipnY5V+rHq>
zt5VkdmV7NMmmI&9tIY4f)YB=)-am{hKlb+2^#k_~H~f52b|$>P>*mz;jm^uAm)X}H
zoWm6I!gXV~b)t{`nVH|0Cw|lJ+B;jC&7Ij|$D@cXN^)OM9b72U*Ejd$foX61PRH(H
z`o?%K!T0!{=ik>nJ!80>;mzu4ht6@X2|qV!=H-jW*cI%jIq<TJf_ev)0&7k#S^xWI
zzI^hnpQr7AnNHh#dvm<c-&eWU^*Hla_&qq`Cvoz@qw4>vx${}Sy*v4Bjz~j)+T%p8
z+6AY}jz6{gr7r3I-t6?|8D;aXT78|6kt6{!RWLee50{@`r0|*wmoJC!y<Nr>uus!2
zS?%%L<Y%8Ash-l7zcdljw~j6<5r6q|Zb2badzYigZyS!{ETL&7+GX?RN%Tfuf94%k
zmM`8eetV%Zq_4c~$gbwgi`6GoSTyc`reV6P>cNA@opUuiuiSrZVYo*A`Lq0>u%A=<
zAkC~Bvo`N9`nNUSlK+ON{!G*2j9ZmDoSvuLU}Ft4pT0<JRk)d9u>zD(JD}B8M+dZj
zLR|r9BEU5KiQ=~|2heE7r(K%u{fX%Y8XP;*Kl{xI19x}@Pil9GYO@`1|J|>_von9)
zx^+8t?Rqx*d-jH?wPCeyK!wjF@z|Y3tv`NL=!kV6+R-bl?ss8pc6inIcV~;w+b(w~
z3RvXA$t9*UBmdrdNWgf^d-~>$&)r?6qAq-VayA?e3!Pe5#P8q7B<LV~7!njT2?Cg*
zFq$R-K1dArSeLk-0~D(+pZ4|i@CXSBZMs|b>q{oXq)C%33Lm*_`yCS#^Wp2)(1M>6
z5QU}3y|ATTqGj*z)%u_R`LJC+<Km*$L#NYWeJ3^xOUt{Ti*0OeMdv`%(-YUfzrOB%
zeO!u%?bLLA|FuzDMG#W9RbLLAd-tyF{n{|?+FxG+GoWJ_lfqfq*e-2KJ-xfGW_{dV
zky$XqQp<}=&L<cIEccT=HC2200ccK9(SLnytu!AWUwKVWTT>TQb?=8ypB6c{g9HyI
zEO76a<MLJw0DC#d@%_EM?<?xN#dI}dc5FZz(GnADn?3t>?Q|FDAksveBS(+6u8ZA$
z@0%SvJG+vSlE@^e@i!*T(fbb?0_*SZXMzuMU0N4wT~k||zcbd^(G_Cd5w@GRZ{Ig!
zUeVpnJ=dx<$O77)OX|+Edv$%Ow|Go!tY{B3UEElt<~u83OUA`)`{yO|7Cf>4)}`=$
zQ-sbOn@XY8S63P2bj+VGucxQS<@CvP^UX6WgO_iq-E7Fn$Osw@<JhRF?mzEHj@k2@
zPx-mIlP@jtWC&RoGc);k-_x7lgBCb2%ATxo;0gQ7dTy%U91F*c3=QkDHvtl6$1b1l
zKH4pIaaE{xtK!c~jm+#EnfK@Y{rydoNBZ&e_vbgg$dEG0SYV)X{`83x3$m}TJ96ZR
zMo-M4+qY-$*kSSN`ug<-AxydF*TwGU;NyIF<l|$JHP<B7`yE=7l+P^Fp7XQTjxC$*
z@^b&>>F4Khxqh0e8@=s<oynBq=jQ|?Y?<7*GjSbtn_(1WvANcc?;G2*^RwL}B4#*E
z+_T$mwvUUO+op_*N>Zj-E^DK<e){s;vi_gVym|91EG(Yf{61mQq(f^WHy4zbr{CRG
z8nQafS9_J$g0~U->uhDMOO-S&TTk6p3tF<xt7)yL&CbcW_uVGUtC;oue*OH3%I;bs
zC&k0oL^y8m;J9RbCglB3Q^rthEfMMBmnYW$vD>S@Y)iM#ER#+}=Qa(Ilf`Sov}=EV
zOFh;jS@QZ?Z~p$jV%FBaiziC2iP<^n#0ig@nwlFM5}BW#o*u68-7~vH&#3-i&8@B3
z?yp3RxBfmK%(x|RUhC?M4_02?@pJm~R`IxmBOQWP`d&HruB;4xd1Irp+ME@}6)U+G
z^nTI{oL}YqxZso4F=2n3gW2o%E(=gOFKLz|VVrhmLGABvHGjWe*J{}55fC77d71Cy
zJ3EWFWL|DlcJI@e@_UJL<IW?;<?GLM2r3tpd|w{+pS414<-&lRn6z82+k9W2Tzc4E
zSW@!jmzS4K7dG8kQuz4T$6ML!A3lohiDF({z`tTaKu(O?C&{|sYE4HbTT7T^2uxP<
zJ#;Uw_@d{#?}xbcKOAJ2zp%)aTTCyeWAo<CPA6N_&doU~>~G^Z(<l`b0<B!4K^j_(
zmEBjfQm?EC1dX5X_%(IuxdW{`eoPB6_}OClnalb)&$Bn{GG>|QpPQuW{o&iUFoPuN
z*V}KN*^qd6hEXb)T<fZ!Y6lU&c~-|Q_Zuu#-(36b@tpg=Z5J+FcxJXazqz?NtMblk
zuh;KCcVlC6=7j}}7cX9niZ9-&zyHsq=kx35otmn>x$bWjm&gUnXIhPwE^cmYe}8{Z
z|Muo)h3I<sxBrR@3$Aj_dB2ZiL&ObX+pgr<`<8f3)jD$QSlaGo{T^{sX3g?CKhL(a
zukYCQ9Pcc<Z?fOc&Ery=zj21~O;OW1!TZlH%)7fwCw`wyeSJNv^3HSZ@^vRpOjKSO
zxR^~lY|Vkwr@a?GVH8&LN%->O;+1u=v$tknfAn(Yeq*i1oiRyCN^`Bt)85|N`Xiig
zY1R6C1FILe3V+^QxK;RuxcRh0u3ZA+(%x5Bg^G4P@qB)6uCk6!Pki0a)&sL_SKO*q
zDsy9Jo!tMnTlVxqd9{F5C7EGsV?1XXrHXVtkqTbs^YEnl{3TAUTx()>3fbG+w@zx`
z^YNJUp1<F2n`B;UnW`QB%Bo4Lap(1ulhrft@3XZod$S;;xiN&RUcPy5;6V=c?Pn9O
zH!c5IP@ep9lIoU>i%ttY<`uoTpxD&hd~<ugyy|Rw@15Ts8%XeQxmoOb>HSToR?yzw
ze%kRP0qf&zTb26noSA7{`RC(t&A>%0tHak{+nHK<WkulP85V^~Z*Of?wz7&U)>*;w
z%&z{=M|Q`92W9NbdN#dv3*L3Mo6B{R<^Fu$+v#79o||>B{C=%_O3D(ai8c?n->*|%
zQd=z~B(&nf$6PNHxm5uwvp0Xd_-2MNhq}8m_p|frCMF>ZPbhD$_?V=mq@)qCL1A_H
z`eiPhXHQO;&MR*>XJznmAvw8oO|0A|l2`9vn6v%=qF!n9ODhVK{pMJBPJI`@g5gxI
zsK)iSUH)qhJb2>VFK7GX#o~UEt|wK6g@q>B_vU;S*9%^JzC_ev=h}u<vMY@%G}Y$3
zxUy0>zW(o4r-|;5*1tac^VcsSVd2G08O(M+p9m`{D@Sfh;jI1r?P$_QjVaa!eX`bN
zYmzE{i0Q?6R8&~JoEh->V<bx$ceutRW%ct-B5Q7S7T2#!y1D44imIxhh{%$}(n;FB
zv(0)z^B^0G7k+DC{Slh6IzYu;@1E<&^;hO?-W|T7;2zJLH*ZY7#QqDFHp@}a(cuvn
z7f(t`S`o8zlhZ~<w&unS8w|?c-8p%vmD{rPm58kMt65ttpDlg3fd78_{Muui&)an`
zo^5aX`~2b68M>`X<*LSCYp!i&>Hh;t|1-^UrTY5%xZH|*&F|MtzPQ-^@tfEwx0uBq
z#BG@5G;#N&c@?wnFDp}i8Dity#`CaSf8T+OD?dMeD#*RLsrB00=;VzN9T#`5+_p2<
z(|>vN_PjN*yQc{%yM1`rF0Umr^+5gft65K<PLF?fxBULmtqz%4{4p)rjgz?C-gR0U
z_sVW&pSZ!gZJKWMrizbAzOlOfbB-n@{`l~)=HpTE3YM^i-wtTZu`ylPbVoY4N@#!m
zcM0)t{~eC^$<8!R?@RkOxn0f9Zl0KK)Pz~HPAzin&bYT{r_;v3l`B`4yt>lqJKM}K
z?To~+Ug_llAx!^eXU?3-BV#c^(m3rw=o-GQzs?`N=o`#;y0&g%!p0BBCy879JAFN5
zZPe5oHzK-3G#4d2ndsr|y}11SJ)O8c68DYHuKfAKCzd(j-G6S@VzrN-=Rf=MzIbK)
zemg@$!&!^s<#yEm{&sOqr181=_UG^HEav3o4Ad~wGc=rd;X=T++}p=?zu$MdTR%&W
zZ|Z?|9Zk*7b+NlQRejC+qtzO)_<BfB-><abe><D+??3-<YJR|yjKHvCM_<~o{k$H#
zqW~1aX1TXq5_&mSmHlV3uI3G#R(~vO>n8ngrhMyv&oD|oy~#+g@6u9lZHY>SJ~>;j
z%gcN%3m>t7s$=izdR(gWE$aW-96WgN&&T8PHUEA-xA}S{n9Ei5!$XVGS0Zn3Z%_aI
z?XAel`}0rDwZCG@@U+otW2xBg#chvYzI6O@X{kx}HJ^ni_CNaY@Ni2@%ZKOl>w`Ak
zEI8HLcRj`RQ$qQ*`dMe@y_o*t`}g#tpPpV>8SK9BgniMkFPXDUvyc7z`+KHMrBT1_
zw;4_++kZcEbaa%lshIHc^779=9`|dDOg+G!eSV%T3oCo^n-_tnw$!#7<;I6;d<vNJ
zyV%q^?c32lpR{vxJY8H`oF=ws-QJelvL*lg&CTg61-^-?&f*CMsqBnDb<VQ6uxyih
z)<GuT+mnBnJv_t;s(04K?*8%NFn{Hzlj@>gcBNll1m4<~dwON?@)`E^cCow5&VHZy
z!m5$~gune?m&M0=E$jZUOx<0(j7fukYJBUqE44|MZ0`9=%E~{#T=p03vN2g5wsukE
z=Vx=Q>nBR>UABthlym5YEcI7BQ9jF#f+A_(-*3^aO5b%14JR&GpiuVq*3lh>k3$$f
z?|gK$dvndtqCbEBytuhpT{~<|0OysgkLPvc_RP4Jduzg!Nq%eh6>CQMx6gd~@Uu?T
z<SPHQhn6~I&a3?v>DDXN%J=4Uvsv;n9w%q#lLpc&Z3`4xSHF&ts{B#oWxDL%1q&G)
zF2jTa49EKA*9SaVw{+REOADRZx8>bEwIXn_gk{kZC&5$GbiGSTOy0eFSMv7O)Y8}2
zp1!s*k@+J3>(A%&l6Ex~e0g<OUYYZ+PueQl^=GwF(7BDdQT%%Hcb(?j)y}Xg)#Bsh
z<8u4Aar^zc+2L`Otm*3yU6B%Kbzh*SG->Bc_Fu5I0QKQjk2LFkFO&9MUT#?OBEatV
z8{=5-fOmh)4V9bwWUV(9J@snil|EMee(&-CmDy{fw+l&0aUGatnhomwc`djzJ3PL2
z>i@s*>vxE0xUOa7F8qAiC+$<l4CBi;46_+H>(?yEy}ixF)%D@am72ew{)^BN1BH>A
z-<%HjgI*vfZQicusnp;0(o1Q7{q=Tk_G`j&a_8RM++0{#7-&%RdRLN>prqu<MXud1
zuCJFDR`<K|B(x}6+B{Fi#zsb5T>RSFX!osIS3{VdzE4d{GfFuju*`RMTlmGOD^dpX
z?&kHZ%KcNWCZ&CU^uOTI_577_d#yHZ+$f^8q`Um|-kn8HxoUrXNql&y^~aPvueJL&
zZz%s3Wbi3t{|tjv0sDH1<=f9MPdwbF6TPj6iIq!3V&}fT2%Rr)Z-2j)k{P_*@8~(d
zbX(7JVQZsYzrVZt@#DwD!)?4%^yA}H1B#~2nd9T)!ctgR_#>2Ok=y%xL#thy5+_ah
z*3`8#ufD%1;^U(=J>83wj&^amzB|>*E&k%hMrHfIUxZm#moJTaqVnd=T#=q9Tw$xH
z)}D4Rt~i`NMLWE2s&;tLrW8+BWd@s{PbT}!G;00!_O@~QIhi+a-->qqS*z+j?Zb`a
z{=gjtiDr3sJUWF`Lzte1rJtL#u=e-2oZH)a50%O<_qyI+%Ik1)YG3}$gYw+-kDfj~
zIrZX2lgvvl3s1<(+Ez_@@F3yao14y^YKJxCzfEfk^juf+<hcHi;+xtmEGz=MlUTQA
zT@BLsd52qn&jEJ%8i&)<bR%~ZDEj&R`sDTQ-kQkG5=JQ;OTDM3JwDd^GfiZw*Mrzw
zyUX=K4GYVf8JBY77Uk>?7wvkI_55GV(Qh+rH=56wX<Ka;yDKBiedpe{x3>#l+Go4m
zf9|517ZqM+^==D2)|I6lS1LPK%_N@gE~XpfvAgW8h?m{hU$562m%WLQGRbf#DKW{*
zlbWmg{oQH({ZD>ApTE55>8UmG`|aX+%&#ZB-2LJ6dHc&70+svZEWMtto4<m=Yu<zK
zg|emXM-oauEQ#Kp=QGD5aN!BvegFSe&oawCc60Od7583Ci#nVX<_df7^tQ44gn=|C
zFYnXc@An03{Jg*}UvuEb#$-_2vRh0yh;hnqvy>A8_i8@-mc75{ySwb|j+&pF&hAft
zd}Cws@mGt_-@LTco0FSc`E6IHRR7r}0cZ2LTtD%ouXkbzd3j=|ZJ|!*EZ3?ktK3^#
zE*jV~ZfZN=;^LC9%_?B^|Cm48Tf;RbrSJG5a(sP~L3+7v`oB3ZEi5fTbr<Vof1i0a
zGt=kSPU{lY-c<9msHV17wCj&Due8~btgWEl&b9UN{_ErR=KPuzt<_k+x$))g?fPu)
z`mcCmTJmo`<8u8Z!yYcrlwJ1t(Zb#}k#_T&+4+UU#Fk~}J}rK~@3)@6?N^b$cdVes
z)xD#ywM9-A&$;uLbIrY0?%&roCB2k6H`m(T!=q#0i%o^b>E}`|EO4Bn6FKR|jR-C=
zofQF38Z++gvFz;Zba8e4_<H^RON-t4XU|RXc+T=H;mi!f9(j8?@%xjm$O*L0u8(Y0
zs+?u1JoWA7Uxj(+y+FORkPs0$IXO@pKSHNR!m!ET?kCIlhAxo$&HjO`%9XQTo=V;3
z`~K|FX6cugmU3@SJA3Hn&B%i{798ys7ZwzBJlMn<xwoox->kEN|7;Jp^QXVM0&Z%o
zSfMdnr%9{P{J)~%%!)^y>YyTXo@-W6^?@3hyUZdzH&|95J+sdBd~wm|lB{cMPIifE
zbMWwl82ogps<P_0{U*UJrgLC<NOIP<xvkCJ{Y`?i6AV)~6yKXX=jO%<$9koOWo6eY
zHkqmE>Gk!=T0eUG_Ux&t+Mn;1-hC2(XI_L3kCaJA<>zM?SBLATotcqXe0^>R=N?Bd
zFD^kr!6V0x1ubxj&iXcUT{c6PuhoWE^)eBk&5n9cNl1;65!DI_&@g-V=xBFKd;8|Z
z!)#n(uYI=41()zTY_tqpJ$3KtxTh!j=LauzVtw@JQOS}MZaFtLG%i@6FlX-E&feao
zQCqXPT;;STpLCgTR~xY{N3xlnKkd{MO*7LgF{&R{ZB9QgBqep~+S+K%nC1Lach@z|
z4HTCWy<J-Kx#VBO7n^(c;*RJ<yDfI>73u1=DtjXVZKH_f|4|V=bnjTg@{<dXmdW1x
zAX%OK&->s3het`EOw#6b^6<8tn@+E<t=$-*bL0rLWAeZ2%a;-*C8Zz#e!s8$_w)Ho
zgTyAU1-hU<4XCuRE9tnjHf}-A2Z^;`C+VyF5j@@(`M1CzYNl~I-@}IwxsD!tb7$w|
z2NMMR=Gu7foKQF=>*pM&i8T+ueA9UJod4+6qn)7flV!frt&`Y~9zTA0Pi66?OP3<H
zWC)&}oju9!XE~_iT^X!ymV4{S48vpz^Sn8}d)C+d5B;!y|35EK`=aOv$JAXu0gO{>
zj_waKC~em{oK&(r`}#VcStgzfZ%o*lef`ps%;0l#JQqZr*AHb-jptl*vG*49B9HSX
z0zyKE&df9x^^)71f8Xxbwp{O`BBRY|XM-50xF5QDb!scOc+l!F-Fp>}dGA%fx1C)P
z|MQoSprN5*OW6yDtSGjtzkQY$to2y?a?*Q?YSCl+y(FIh>q!0BG&3bF?Ui(~v_bj1
zm|bOWy_Q{_H|uYjaDb)HLXUlqA4My@j%xaNdWLzm)w*@-K&AS@Cyeg>a!XTBPn%;`
zJL~1;<-%gia%b3B`T6;^w6_aOOP_vobF*f|hJ<<Y)8^#2H`o1qI-N(>N~Bs`KI_}u
z=5o$hW$#bhd!B6mW46`0Q@w6^{QkO{uUEs(s&i$czaM0m|8R&~|G^VUkG21}Kj^IC
z6j7<)UG&jnzwcYOPd9CCKA$nJ`Et?S%>43vUCY{ECcd-Ho*rmq23205^>QL7O<$+&
zE_-WH@j>D8GT+7d_xEWvR-VjWzxUXV!p8+4H67nJ2JZMd{c@Bp(+uOF8+M!JpZ|+-
z_3+^M{QUg!yLV#`J`oICJ$0h8yHVmHmOFn=2d6)9np4}QDIus}{8nhc|NEmyo$W8K
zjW$m^Gh<;k=QF40=jShX?G}4;Yil>Bu-b<s!u}c}Cqw&Wt=(pu<${{mpP!vw9KAhH
zt8wStlPO6rE-VBM?No@Z^LyLKT~MOGVr%3RHnGi}cf033elkrGG?MW2^z`5j3;xZ0
zQugxY%P+65_wOuz{wQ{n;?`ehFPbvA=H1&j^U_Vr@0{n}#Q0oZ<{P=IMAOdhSNWu9
z6*aX>yGpY`EkEh}JqPRmeU@)k+Ii{U-`|{ET#pX5a_dBHYWeo|_G*@AVVatnpkcR~
zpG90#@_l>l?kB!qqAa6infD;9@A-r8?*li5Xoej-8n|vw-;C#B)AeGNY;5LyeSKXx
zV5#!es=C&Tj_wvNdB?Lhet5nq+~U_|6|K-!E-^7ON4s8j{9HV3)-11r0t0FDyeCVi
z$36Q0@BM#~t|wW$%ibzkSj^b6#Uv>?IrHM8R<DG-$rC0xWMpVWZ_8O2@OECv%Wslp
zZtQFKD%C%}p&xv6^AGzUKYnenQEOx4mC~3J^X$jR$1N=_8?vwKb(zXI9rb(vN~%Gr
z{NaUN{x`NK@w9S@tCbz)S{1&2U4Y!gFJDTmN?#p0HC0>EEN8~qGy62Q<-fkRwxz9&
zX+zpssaMy}TNn4-sNu9Jc)$QEA#4g7f^^*j^itMbi<nqFVM8^mbsGPHgL2Q#&h<Xt
zCoAfuCnzuPU*N%^?muTi%!dfCtx^jET;3gAdH?z5?0>;Kc3EvMIVl8Mj~MVTRc&(T
zySuwV;}yTYzCM1WQ`k^~hs*6#Pp`DOkepoK=FP^hudO|89ItWn%k`tP&GVNfE_Rc#
zshH69@cn9rt~>1iPrY4fT%jpu9d1?r?#vX;U=c694S9F1W|`!kT6sBr#jWH`TnkRr
zYJ_aI-pZ!)z)OvVogLJ8YyG6ZDMIJV`}^}DwbsG(`JgUMrQhXczM!H`Q&V$G?(J!N
ztG};evGY1TU4Oaj>1i=LW_X;nyC1mcqQKgvO5acJcBy^E+k5!rY6*Y4pG(eNtvK@R
z>}>ad0D+a^XRf@n??1G)N=szo?~nyK-mh$TGFeY&`ts^(_xJnt@eBWW&#QW+3F@?E
zWN3iKPV{aoR)*~^dppCf)=FG2W<lxeYch5<5vm^oe>|I=4;o<s4~uFthMLy+KAB(i
zS>L^RdHBP~9ai~!KC(5l@g7Ps(wtF$(_OySrC-k0z~X1X_fAm}jq7d;JN`J!wXT8<
zxE>XcPdU;d_~i0wjsMwheKIGfXa;9oT_tLqcIHB9rt10ES5^v7R`-8)q*M61p5nyW
zH}6<BK0em_c<c4J!#8(JY-M+QC>P+Ez4QcgikE-d?=#0O_kXW;a(13+o-db|m&f%?
zadXt#JwG0G*L*n0jy&F}#2<h$@>*C_6lB1E<HpA1%D-Q)XWrPL=sjKU>V1KYn(?(?
zMbF!Qmzk;+n)LVA*B?KBu4J+EN!vU#@o<}A>M0Q$uF$358Z_qEFmF}svSAV1+!-5q
z*z)V`oLAS@p8oXo^vCbt!wvX7k;Xfxub5xT>u{Zk%k4?x8~>EJ#OHSu=k&&VeSUT}
za#ITDy?gh#o-zJ>(9B=)uvL6T<Yu+g)Af&o1~5C${`~Q{zw+zV@Rgyf#j>xjJ3ICD
zPTwg9u3QPJ`u4`Nq{L)jYIM-&kCHd6D>HaSDo=bny6@aVdAlVcFE4@m8#y;OaXn?+
zUGvjuu66mdcKNyk`-?QLR@XLPJm`9`!^S17yVdsMVvB7ry;p^-bjr!maqSihHTZww
z$;ru*RwWvu+F>esdV1I86a_26t=)T7uXUx&^UkaaU0qOG8p>HS0n{LD=ac>M^?E$#
znw!n5?y7_G`~OAB_Vs7ysp>z7J}_PM^z>lRz~rZfIX0D(CQJ}0dw0k4B3ttck$-Y*
z){C12e@>VA^-%fS(S2?Ym0UZzdzWTiUB&g(?(7W1=6$ulL0uqFpKGC0t4P<4J=5ds
zX8POzow6<W_NJVhn_^7&*!WLB;M~q9C?a#_U+r&=n~y9{weDHBvei|tar5q^8<yz`
zeSBp)GwlE0Y7Kt+@9n+4vmZ<l=##g0yScz{O5WclC&5Y<zr5Lp*RlFPe)r7l%cZ5@
zG1ohbUAx5u1qB@w5)`tptyx&{@e!Bnr{45=mCtT1nHm@vxHC;?iA^-q_Po1Zzb^!U
zhDL9El<hLF3(B5(Lb>GS3H?8IYt^59ISp#EJ2tavPWinh@i3cJ*(;Z%B%!IhYuzSV
z?G^1hp<GgT|9j5-j}~Ruc=r8%XRV~HEP5)otGjz~^6|bOKYnZ|dg=w5X$w$^-x#68
zBWE*XXYun(o74T%&dr&rt=FC;V_o*<#-1IHoSFCMEqP+H>ie%YrPALSyuy`7=Irf$
z6s|d?#Xf99!M$<`9=51$IV%(11%`x(Jbd^tCB^CN@pQom-I#?PHc9JPy#g21So<A!
zH%@=Hy`sAM@`glb*3af<IX4{I`D6=9OI6j>+PtP}X^NaI=V4=(wJK4tvzyn-Enf8L
ziKkWBn;$3K^`Gq&vdFz<QuXCU<JEJmpkd1Jg>$dzO^i8oaV<lm>8`@U|NKXEXCFT~
zS)G%Y_bO|{`%BB`*PS}h$h@&!(_yZo_Xo|p5eqw>2>H!p)r>Hm^SL@)cyEE2r>7?i
zE9*;>DKa_t_RM_nAmPoMHywR_%fi-1ak+ldwYIiCbolViO{v_0fq^l*N;)S@nBX4m
z_xV!61@uAF3uXHPKYceS_`GtF_~bQ9?woCZk+k5%*~8twVSUT*{w@;XHV_KAJg>)A
zvZnk?ei$#i^)Iu+1J^TB|N2Y*nO64r^^J>LHZivb`__C|ZR!8?-}J|xH}CJC-OV_E
z_nDiv2RyDPu`Sv*>)jtos~duWr{(k)Po2L#Ws~-0vuo*)AyNzRZ|hZcbb59cKYz9>
za$}M3>Z_-`_4gjBe!sVUb-ZL2e?&)i<D^^rGJh|78?85GW>sVE<*I_0C)R&?b8~X$
z<z)|Fyja0|lNXeSy2W%oyuFh@K00cWeeKOe{opDuu4S|Hipz>0uUmG@%U}Iy$>q&&
z=N1<4`rf%KNKfYZ_OJF2L-q@Q<+i^6^Wd}YKbxMZpG&O@TAypoG$*fS?&LokQ;pU4
zyb`*%z++vkc;58iR~ZV6Pw#6rTE4vC=`;4{6E^=ml(<c1Q^0n=m)7-d&${2-*wlT*
zM_kK%Rr7o0dC|J2rc*zBD5(4OQr)dz?yPkFo{uM{9ktWX@i~)beBS0WPkhZsR=LK|
z6}ARk>i*HQLQgsDFEC$quVn3&BY$7o?0)wCU)HUUZt>xH^LAK$&9FKgcIR*WonyUo
zDi{6wdNuL?NuLMjtUZrvTU^-Xbj@dt>KU=R-v@p67o<nA#{CzcR(FVL#^;cC3^9g=
zRkuX@?iU1~o_4N8e_rkDi4_`i)5GJ=-@JR=<Nve1E4(I6H_!M-`@Y$KSZK=ncMcJ{
z><7-ApKX+z*1N6z(}HgeKi@=sFgZQ#ET|o=6TdI!a^wBHb|d4-jLd8X#m{{9bX{K>
zwtuI^uA4!Apr}fVzHQDP9{-9b?o{8+<{8H6e9Qgkr+s{Mbl1JRp})VqU7UM+n~J)+
zarwJDF@X$<txq3E8L6rL-YzxwUhJXhg{l9Z?c}V9+par5dh`G2{NJ}^>|WJ<-+Xl5
zjUT!Po$v4e_IdT^rvKYm)ZTZdUZ}svvfy}&#IY^hPoC%BOa0JZyxw-%MeX9ELe4$4
z?-;)2x_y3p+;(T7@Yed1(IwK`Req;`dB${)YkkFDVHLUbuT3j^{uJmP>sowc!vd3e
z<q=J%Z;M^1zHnSZrfp+h^j!CO9jjMs2L=YVw6z)4{i(3Av5_#%x}qF8EymtWTUWO?
z@9wUTN5$hGL@%@a+IH+#1S89aPyc#S+Rl4FzQen{YTDN2KkwUYuHkLZvG!QzBK*E~
za*e;W2gB^T^ACPIFRQLPH&y$3`Jzda*ECF>zb$n`PW#JS+wFE1em`;l@C4oOKYwr5
ze|qlg$}jsf**O)`T&0-BVjr&ey*Y1waQ+R)P01DE;db36CvQ12ZHU<U$^7)CWo!Ok
zovSOnJ11xDHe;&*(;}m#Q8wqFot>R|V}&3)zuXnMOTE@NJ3B$%DSqYy8qY63VI-~H
z0FJlWx6KzfbDPS~cHG$33|>xgYm273-<%WQZ*N)b-hXMCuk^XO*2lMGUgqHAJE!XX
z%x!wP+2+5;f7+z&_?PPbrGEOoPx%k032m$`-NqDQSS0g$`|<n1j4T>K>zU(w_iy*)
zc^i9Y{oi`Wc=H3DrxgFrw0s$QDsjiJb4ByNOqr6i(|zw<``dr>_5RGfpRd)xAgXjf
z<@v7n8L1~$sjokKtHPOakH;Nx<rNn#S|)~jZ#a1SV}9S0SvPm3nKH;8XL~%``_soS
zUs7IPS_)cXFg>oS^Su547}XDHpiWQe>uaFy!~rQm&;ZKb&CAc`*-X}-_4DnTr^oD#
zx8#>Bzk1uDJ?rNB&KZR#YvP^r)?|t;TYpyNrQE#YvU;}p>pYV$eqXs~uW2^(&i5;3
zvd%Wz^!CkDHO5rg%Jtig&EqcFq@5Lglz(HkW|YPAGrZr}WEk{PKfNsT-X?qZ@^Yh@
zTQqu=%V+LCc+?{`pE)5lyk?p2y31_MjYV(cmnZ$Gma!;kVB?kElzLihtNnv3D}zn4
zuk~DA9Ui=-u0G4w?({N!ruA=UZ_78^Ub}ho;-KgJ8q#0tj~qXKc|oFc+53BQ^S5U%
z_n*HE)L6AH>+zhdCMY-Wx7t>ts?bGmH?Up5{^rd3$S;pyTH7eM@@}?2Q|5iqYwG)a
zkA;)%c@N*Ze6ZmA?tAB!I{&n*QvAFq+-Qa0HSv~tt=)l(Gqq<)7d_YeGqw5Rm0Yv#
zBhsg1Zv^kT`0H2TdD-9RpZsB3X3`hE*5d2m8SAd@&EIi&qSMu?O+|mX+^&@Gx!tOm
zcT7G(!#!ZJ&`vG&`1trmix+DKFXQpI|7&u2xqmq4pVJQyw`bnjVQByVkFn}3zU5Kp
z*KgRo{jcG7&d!_d8ltNnc}$g&dmVUZo|Jig?<f9aR?_d13qD-?n>Ovu<~t>E9aD?L
zixPM6xw&mEY&kP~|G{lWf2a3k+y9)hReibTeXFZw2jp)2jDDv3zVhHQpE|p2#g}^g
z{8Luv^s_rGYc*BRt=_A5?*(Vpx1W97;=-Bt7O`&Jxbf#MhIcnNw}WCM`8eOHEwxj%
zV(rg~i#J~X=6l;bI%L;Zk5wh}kDLr%o_S{pr?gqliSpZ9K0iNyd6};?pNz!;P{V(|
z-OloBW%rX9Z`S=QI%K5x<chmr&ZDQP=DewA4DSBdH^2M3xR+&$mjBbB*8E$4DuXsn
zb$-Wvy<|rJA@}4pZwoJf+I8{Y%kXu7P2TVKww!v{`%T==hT4dq```KIRrbE}l@7RZ
zT|EB#?$!F!AJ;1#%Q|>t0rytsqWh8sjg5_<5vDB}fowby2bx&9LzwFS|1Fm?%Q|we
z_V<blAL~Ons*UfT+4uCtjIU=ie{HS*@3t#s@89EBw)Qb6sQ&X8uHPSNcwxnwi62&d
zYY8*CwCBUz&n7b&*{u&{TL=Ump11Aj+aj5wfUR35KC}D&&8}P2OepSZW5=p*mU`Y-
zFS$8&r{4|NIWS|Zj_BKyg&XEHH;Re&x?Cwx*t5viHUEa4R7}sFd$*syD&`gWdg9l6
z{q*lCMGBvPxtaedbJvjR*`kxgo@Tu9x6c0`oAct#UYD!L?f+b9x=m?8misH7xF$~Z
zLx&Fi_;x%0b1EZfdNA|K4#VU9^A{8yaXGc6R`dRz^4$AdYBz_vzWTB#Y=7mFC25zo
z<=(EZO@9~Y<>RBGq0w=7ce!!hpNcjf$w{lHn;WZ&zIhe+{#Ak3>3RJfcWZxc>%Y6{
zi}r@q$A9q0?=<`K$?uY||Hks~dEegMJ$=vlxQEaF!{0946Feh+b8oV?b=jL|7fy0p
z@)_<g$k9J8^nR_wvN^WL-rcpnv9a3x_&K%bKXz67r`?@i^w45=`J05}_mZFNxbe)T
z@9o{QeX`a!_I{VpG}kNmRk%;?Vy|SFahTDLIg|XnPJIn8S|@PC`|D=4rj2QJXPa)z
zHk!(&{oB2I<LVzaXT2={ANId>rqA>=<BJO$?RJ*RByLPP+R@p0@c#YzaeJ#aCLQgn
ze_Q^1SJa-Yt6IBC-g@1d%9HhFme;{ov7he8JrXsxmoXQp`P!dx)2dIT_k0~+`@zHW
z+8E0UG$-#64qFv=;^)&<p`{5C!s>Z)8D<8TeRAH2KAW?rw25VV5?{@&JBqJo-&rlU
zJ1OnhACs`<P4x{E{jSQah&&|6&u;OjSnIX^zrTIW2M?ckQ6PTm&X(g>jTI;Ji!Up_
zb^OEwHNGkQ8;)OP^x;(Vy1r>~-h;2QM>nvg+0C4GEZ^K_f2?x)uaNWmtjaSJQ$K&t
z^tqG%^l6;ll!6`mHXCm#zsB_Dzv0x6AH@!wdQ>5-oSnMYZJ}3R$jvF!-r7vd+A8yb
zP4C_LKgXYMmx$@P@?jhEnm4zSPnoaY=6}-dinP@6{QD)z2O1b<?Ca)ST<rdQ_V@i6
z7nQ2MzG^+0wIuBR&L5#U%Rxo#t$jB+_J7J<@%u>8Hs9BNDJd%Q^?wY{+kEcv-T(Z`
zl@KRq=bvx4->>-jbh@Nrl8ao#E0H(RrB7a#O?~<QlL^DUxqI3^^zTmlZ=_*4Z|3~>
z$6kIspAlx(da-<t&nu<E$@h|<q<iP@T>kz?e&xF2yU}T{Z1-FDP2crb-tgM%+hV`X
z4oCcX7d)lJXhPjXi`nLDm8uS^AIUlQq%~6d<^0@=6u<oU2KRfrAIzQcCYkA$-Dktc
z^KzP_A0>wA^d>)@rT?RDbEntg|Hl*B-%DC1UgbS)*cB@O{pkO;=kj+fy8WE*pZd4^
zO_WjM$=O@9Q_tU7erV>0?68-{|NrPY^4?>8aJ6#wi`bjY9~8cHF5L6^^>Kz8(^A#i
zSzpeU&AazgcXdR<)(ekMmi_<pXRkJU<hSbq7phW=ze+`J%ZXgHcrhm@=R(j@yko~M
zEpqK{|5sgnb>e~d`+oObxDa4;#x09KW`;etLW7{$pDBfZ|A?kn@0xn<U*Oj4Ck@;i
zK5ysYyPvo_WP6{m@@$KTO;deW2fehnocXYOo!^BA2`Wn#YuhsCeLm}=$!>Ed%uh99
zUA^*?oF^OKottvhVZPZp6RoLRn_0s5NB+0wbG!HYt&^lV_v`Sbm+FhSy4U+fJ$Uv0
z!0Fh{ZNI1K6qT}bZuq`^dESD?!@rF!KV1(nN>|=HEB3x(*3C<;c30ne+Dtt7BY#q5
z;YCru*^E3_vfmzh%ow(B*4#t-)7RPAUp{00IY?r^#?<`(d-i?2`!kzud49yTzMFjk
z5*+JN_#<xa_&Wdnl_}cwwJWY1TYRl%Yt?S{kOc=fT&-XUOFT2f5H!5J`DV`N_tlS%
zbe6on#=E*%y|n$vlk-d~K*`VS+~y45SG=*U-i?=Re!tn=BW2oUQ=+=AdeXdke$(}0
ze|$VHU-|3ha!KPfpL>x%L{?nc_Ic~)&pz4z`b&>*n!c=g^|KAvvL0@I*ZWcFE1&+f
z1J{{v8+_WlbW82C;=0a$b-Oz2;`M%QMztmNZT@#_PH(o8wVfsUG|C{Q;(oTfs&BU4
z^3N-Fs!Tex%0K@3;q__)n=M}(>|6Xdwx`2b_SxIEhkK-N&pN4Hr(#$4dh64Ozb}h<
zX6=z&lbKVQu5>@&ERc`6zUCX3vi-S~C)V!`7F8c!efp!`{dKlQZ&Uk^?e$;2F#qDq
zsro-Nt8JpMpG_;MWZ&@TK!@MopI@7_=kAX<t=^w{H6ywCdF02-`hT|6**+9qSAREb
zL83(SI=?f?|9-|^d*gTN+rl;TO(ow%UD{drvHp)>{J+otZ(4uev0~M~{{8RoNTxA3
zIyx4)yB6FR{m%Mbclw!mch0K&=6>9`Y2KD!H<#)9FUy^_a%E=NTc6@m*jj+B*6O^O
z+?|<PdI~!q^l@|XoG~bV{_o$8!#oNKT1u}f_s8u^c~|vr@AJBM->c7kuj#iI*%G(&
zk-enT+gn?IKAj$KAi)C~RJ^;p+}_6T!IzhpLCYhn-aE|diEOXGYG$+L#eqrdHoR^6
z&3tde%7b^EQYX9=(rNa6I6YZw-mg8;$?YZ=*Ub3Lu5-q6ovglT;KQdLtBsF%Es7Su
z%&X{eE!%9_8Pm_Nx=fDgot$*_{F3LLzl4mZ&-(ej{N<ffA+fc)r8Jr6^lUPCzhiT0
z)#TP-KW1<F1<M0h9pWwy>fC6tdu1+HxA@G;;>lCZr!6Z!wsUR1r!SAHy=}A8){>|1
z*Pl|j$2MK)ZH-)Rv84Ietd~2dteZ69&7S#Pr#`Ao{o%r%zUirU!3IrlwY?kI<ktNT
zS~O>#mD+<@b4xYL|8?AM+v&WN&$cD>&I@Um%Nt*RSbwe0viRAB{q^;iE?<_v@w)cw
z)$o}HiA?kUg=SclF{^5u3H?(0;``z@b65CFEfe|g(yH2~51aYz5>8A|Oi4+x3AmoI
z{dVP}PW2lb5}B3VdLI1!et-Y^Q|sf-JLGdcJSiTvW14U5uEVl@E9X3Hzpgy-aQw2`
z4RY0XJA3=q&pLd3xABG6-p5-FbLU2>1@FG@vQJ`dZ`Hl87d@`cIq@!~jsN$ASnagW
ze{M{lDgU)uJ9}mA<=bnl5*A;Vj^CX0l7F4gOxA}NE=B5uiK^+&@><=c%BZ}x+V=1f
zsaauif0LA-dtZ5Sw)uC(>cySOD^8p}zEP{nOu72ola=Q8lJ^wf-cjslxI^wRyS|<a
zU-iuEY}*fnvn9XW)l>JF_4!$wWAE!#6HI;u?aNa4o#{2@!2Rpe@%8n4HoMNZKeYHg
zZ^_@P@{3{N&U5}gl3SLyEkWj!#nwIVOE_j6YAk&h5b*!^k~p)hD;-ZyPhYIT6}!6(
zG?~P|P54ED#SDW)ruF-NY1yS_U#z*KR@0RAaYHgw-L0t)<+y{r&MTyMm)&h+JH@a(
z*{GvDZ??4P>X#Rp*h9*WrsSV0o3nYNy>aQ}?LOA?-~66umr~rCzEYEGbJ-5j4Kc3Y
zrE~YUu9_zKDMa<oWXr{SV*hKMewUlYmeJ;`mihc`PKoJ(l`)%B_uZ2!b@~z|F+=Bb
z+O%Xp?sEPaoEqBc{2x!K?>usBjjhN%Bj!2DWxLFaxLz<uH>&M8*WYd$qNQFnS-;m!
zSAE4lhq}LC!>vkQ9GI#dF1I3k%gr1fdAm6;FE0<?l%zbX_c!l*qtMoUoVCKgzBMF&
z(DI#Jzg}rUTKP1cNT=`b?t1$9)hVny`>?>E=0`!@zn|%Bd@?8A+}!-$-(76Q_3|y*
zYvz|;T<#KkwJLYL%lg!fs|^0#kNt2qocoN+`Up;k<8`^4Tm74Sy8pfVI8nMRO<wl)
z#&GTLyQ3GW3Gb^vx;5{+V|l}#@W%Bogl>oBF+Mna@UVnw)|7AgDcW{@myft^t#|(;
zYMtUKe|2HB`zi_5>o=RUGL$AoZqs%z+vr-lK&E#7!kNcqOuIfDl;6?!@$bIio5zix
zDXWB9+dg~MxOMHU<LioV-L>7~I9vY1)qTA?9-Cx-57fx;i`trX^!xq#{+l_=f|vUp
zId<%xt@WF@xD`i}K+9@Vo}QZeBQ)3daH98x*+)`V>rGa;=y%1GeXaLPhR$cYHZ#6=
z_wFnG(s1b2-Ji!buCAT2zkmPhC3BB`{nFra_LS8R_4m2LNpEBFmP)3_*%_s7DAazo
z>xpt;)HR;DKO&^3Zx*?m#@-xf9H;k0rlM9@b%of9*u`IG9eT&&YVG%|(9yd*uRP$w
zk>9H}w=YVcP<DQ{nW2HY<HD)NiHF*h%jX#!6xCX4Q?oG0e8#y=jl2A7dRL|h)aT9i
zt>_lZiP%5y!Gj0C|J@1-3X(8LVA%inn{?`-T><}3=1;WU)3N~^&l41+Z88nMSo8h2
z={!68IjDzqZEf^%<^DVOi)H&|t*5DaPrI-=-T&FSx!r!tcjvEn)w<j9pJ(k+<8Hpj
zsvg$;Z<b#@A#knctLmhOnpNifyE+xCXYIXMSuXra`)XPZFZ2APH*f5+X8YXmEAsBk
zoZHhlvJVS*rfpoc;q83kh?T{MEe_Rq?mhD`nd`-@Pp{e!Ydp`=mO8a!InRw8vki_n
z=jgrl-847OqKu{CnX|g_rcl<LbEjpkN<`RlZZPmH<-V7Z#`XB5c+`epk7XCgJ>Gnx
zc1!&)!LWV%1!Hem+`6h!c>KJnu}Fycwa@!FSkHPc_nRBBB}1^8onK8!sp;+Q?ZHdE
zrm`RB+*9>cYj5@UwCCsMPFZ_>{;vM|PQQGY33F2~mPM6cJ2pw8`HRifh()s)Cp~)W
za`)jz;YoQ9oAZkc4l<?sG(0x5ds9~KeQV`mNgb~<;nTR|bX>K&bAo@Qg+FY${b$Q*
zx9fFoUosS9^vf=0atiz1*sPkpBQ|DXP`doNfR)nowy6XzK7L!mdzyOL)zvNAGY;<B
znw@!L!^7HYU(oDW_4jvw{(iqdCEKd!*55QckzfaoTH#-6Uv!UOzIO7;JA2{alSMK=
z-pbAbt=ITlb?DHccc)*^G|3c-um5W*YgLkPagpnbufKl(;XJwJ$90Z~FG37~^Vg(Q
z-*r24BRBQJ?^=zH-SeiskBVQLu6wR6>zu;<Wx6-}FYvM5SfemAZ}GtZ{hd|3&a<{G
zpIKiQ75u@jes5rQnwMVa+-qy2MNY-8zg}*kDF6P6gqFMg|B3TYhg}vr)bncAV)L(M
z1?^4Wq^|th$H{uu^ZmWOpmTa8Y^%1M7Swsewxs$}P4TNMno-+wPM(`<z4A_$%-4UW
z{uT|D6N>J*_1yfs<Ccjb1K+kc`=*{OSIr7N_dam-950<62dAW!-MN->vMN1t)zgbj
z`ae(FNEKCw2EEsx9Nu2P+sbHbexjAE>BVF5Gd;yF1gb}P&-n2Cp7s0>H)OR~H@3f7
z)RexkXm$FE|4*3o;yoAn-rkmTW5dIJd-`tNh?v#h{`vd;`e4%}qf)jrVtflOm@mxd
zx4fzpx9nT;{NiV4BBQqCoa~deez!Zj;&HF}kNf}sxmQ$J*#G&!?Dn`har;Z{txwm^
zo3(iHt?9;PN3uPN_~o+0H(dDZb3FU?yBRgHtC#s^KZ-u!w!rM^=_2cLbKV1jXCDgh
z^sjXMB*`~_$KFKojr(Pl);B!6+|FP8RqWGl&!w|JzX`~FF@4pV#8XplF39#jf34LZ
zM#)%2<i&+mziMO6=T|)9OglU4=*P#$J3Bfae0qAiwd#*jIn(#|_x(SAC`d|9UU~6x
zu2c2??-$N6MhNg2St_xF)cvsjef(zC!E@_bCHBbIp84y>7QTDq)^$}4=SurduQXq;
z<#_I(Wq6U{iZ|Ood<xG@S~U53@A3IY2PW5T<;u``ZZluU*zS>=_X@XKW##CZ9L7&K
z=j-YG*KT*5lA~_)^4$FO)*)Vrubt(zrb?&2{<ieuim53YRaM{KoIKRZ{eJbQ_qVoo
zi|IxeSv@q_`s@6ns^|Z^+O@v;zL?F>@X43$a={Oypfh1gN=gwsi=Ng7f8)3RGa>Wx
zvWp8GnZwpZG)}+%SNc}oc?QkfCzSK8_<#Fv%RLldF=-a7r(Y7+GGG6*v77TF{d0=W
zRn0$hQuo$xF^Q7NcABQ05zn`6U3K3gc}4ra2fdy0wqIG**IJ&QbYziWt;D{sH3#_~
zP5it3@e3cj$l~OfzhD1)zqET{dM@h9n(bfIPHd5}Q2PB+?zMaEee3V+>yE!GY-`%*
z@VMyL?>KX}Ua6y?)!Y-6-Cx|<X?%Ts{C@kn2YhFnUEJj>trxq?X>a}6#a_4BtN+c3
zc3)JLDJ!9}Db3WsW8+SvGmWR_U%Yv^$tSW^tDA9xM1J|(bRHvv1Ny&rDmiVLuKcs5
zcWdTPX<Oe}nU7NgME0FeUE_Bxg-apu)P%h@S-TE3>a0EQJnzGj$(wB$>x_ciD*}DP
zBDbHNw%>4yrr5k&runBY-Fg##_e6B=7r(a?U)(u#r2a5(g#L|)c}#3KE9d)ItzwcW
z`NDBJHbnSw?vXUlC!+Nq%!JgQ1by44c<bjSmlYxNeuU@xUf%fnLiP02A0Hm>`TcJ9
zch{ZuKc9+QmA&yO^0MM>tMzY9JACx82&lz5q1R~P+{{m3qztB{9B5#y`F1n?=)N=Y
zbw5@0_kIyF&A#^JdVKxTl#{<L%bTa}c(Fg=->zA@J)2$L=B?wI<)PT~BGSIP)Zg>$
ztKQVe<RwRwh3CWtnQ$wAe_k+Q+o#@D3MXwPZm!MZTduczdg(`hUCCa(H>`5Sy{;wu
z4_EicoY_<=C2n+?GhObOh|_*WQ&Znf^~V=K-uG}xaZ_dXCr>focV~Kj{5m5UFgs`Q
zL7SrTDd*y@Mm=7&_0^-OZEv2s&)ocU$LFuxi)wxgMe)|Pi~X-`-gQr?Z1uF@$e({%
zJQ=)%3Jb-Hzt3O2EiKEvocmgLzo}+|lSHlExn+kMb?uoie6-_ZJ$v%=dHeJA|9{8-
z`}aG4SNZ$Cn>l(@9*fNs-jQ-rXjkd$wySEE#!P$y@y#-9LA%rB&WXj&Y!s~0+96dZ
z>yjqLe6noX+~W7557doU^zClFp&j7g?{ISK{dVa!=f3O~+WWim(qVoBkDj;hJx!St
zwi%~4pZyw_n&tWRn48?%CCUeajm^|dt~j-stu6F7h}pa8jh9D)&dSA2pWk<drZUaR
z-xPCbZ_P#d$W4q=DQz}!O*<Fw{rR9Ux9I6-cAc8}EUHYYTGpipS5L9#@%nTz@L1)_
zs`{DR=1rKCko;S;{B-TnC^b9j$7k;t-n;eWRM(+Drn3Sok8PdXr_NUS>0yArT>hfv
zKJR>sbow{$bV>E~Ie6$K6H}^IO4ck#B^~cDr<1|U{4(E8wH8_3p*5dh_0+X1&xCZM
zyRXMp_kMhQ{BeOrS8nD0KcBpReShEYtoQH?$Bj@MmxcLG_gQuZZE|v+Ww)8p|9?Vw
z*_TIF(;qxK+Fki<X8MY#ty&W&PMo3_`)gkrgYdtel!s~BtKJq*TW8mDGc@$?3)7F9
zXVs-F8b3IcKb~y*Q)R#2oTtC`mi{j5i!I&0XiA<@i~h0gED4XRoo8%T6=U5|_o#ds
zPbGiDb#Y&{BN~mdsuzE**+2ika{Kw%gIT7<HNS<F-0GQjeLM5FrsZ_y&N{7m7SF$R
zE<d+RDmzG+Z?m;u@SH23#pj+*z13H7Z|d8SpT%l>qs$i6|Nm!_ce7h^wQkSPKe|#s
zy)?r4gtxk0?#p%W+j-dL>~xE#C3?yAw~tG%?%G$AeLX~+Ve|aA&VP6PeG$H#v-a4&
zZoM3t%Wb9?q%2p3lv*y7Kl7vM<^Ojl7X18u=kvz2!|i#8x13WrY0!6WE7!Fh344W(
z*QMW-O<!Mha?aDc8)ecjn@4G6od2N3KP7L^w_DjZ-|rNImZ{m*{z}*y_5RQ4HDZmQ
zKwI~G=GjP2ey?mI^HbQa|B7Hd<EDC>bl%TL)Ti@*?NUuk&zN$iJ=w-`zh!TKU#9$<
zX}LC)R{b9qZP^@lY;94S!j#LOW89TRi(1TLyiPoryj60}$xUlwjg|Ya8tzc@EZbXh
z{Y}mGV~4hhoPAK0HbMNX_tQ7q%0HYB?abQA<7T%*x3|B~f@969=lf-Sr<{t)IqFz4
zW$O7P5v|rrgSB&y^B#}=8qFp+*Vl5>1Ivg#8oj-lLWiqog7y~d_1Kq?d{9bzrgN-{
z{mg#(?<LEoUgtRyd0OX*t%%$4FNSeH=58=bJAC%o=@&Ds?sRAEe7bqNW{su(+~U$6
zpX^iy5xW_a^`cI3H{30nTg%Npx#SQ3uWxTpgSIPv`Jxk+TJv`6^@{Iz%Rhcv?{qj&
z`-5JvuZ;yLak^aHuPOQL^qc4p7T5j%AHA^9`RB9Q`34d^N0-%ThpkC?aG=rV(+TCA
zJ3A(NP1U-)-~4vbl#{;l>04(lKInXXo_zV)Yfko8jeXW%pFC^-zmtm(My1;C{^`8S
zZo9&D=d!A=)oFa|9v2yWU;l5~%*pA(y|>p)o6Z05wYmC*HH)ktryiQW|HIAc`McB_
z&oBA8VdbCuKaS6hN=yCRlW4PD<M(WplYeJe{bIWO?9Hx)Ht*MTJh~{Qx?#?}vtORi
zf41DBT%}R3gh77qSDCrlQ`Jnrg&$&1kzb|olKsQo^|n8EJ<r&dveSK19mB->eV(Gp
zQi0w7kH2_%dehtMzwPA>3oF8zjGw;zC3r8#wqyRB;|5>m9s0F>e!=?vjovje+lAZj
z#~r)ge@`d!lB$^P^<K%@`6YEvp6PFRH~ZobnJBLx3(6<(JpNI8e+T<Abp!45cE5E%
z%OlI*-&+&0@loF-<GZ2i7akTE6h3lEJ3Gs@^FV-9Z%pIK^-Tf)s$ClTT~y`IHtej>
zeC~3nGQZmPsAG-S!)Z&a7W7T3I<T(DByg#L)i&=R9xZ9pEKfSt<Qdyat4OhXG@Ocm
zo)`C(nW=P&&&kIT+hgoxA3jwznk1WJt$ysZNQC<vNtr*1I^z5~poN@kqPB9?|F716
zTef@d#l`NQ-)_JED0&}buiL*4_fCh82U@@#=CFh*Jtp7uB@aGb{QBjcoyMo9>mNUU
z{Q11;*>`u9etCJB-QM0Fw6Mj)qoX2%^^n)6tFI&8EVy3%`2GCv4=h$&&U~G=*8Sp>
zSgH8;hkO*4O{n?GC1$<AJn+{2>4z+KCHXz`N}K6D?bvmV=i3B3!`CsZa`1fF`%`xJ
ztd6VJ0`bX-Q}idcY`%VBx|aLV148^Myu}tWJ#x0QQcq9&ctqI$!`<@xcfXnFmY?O^
z5~0H*Y1H!c^z_H#cP6nNo|f?8i|{W60ggP$(9q84zf#M$)w^Y6XuQ6@etFT;Q{UIv
ztX#Pgv`?<|^|e5&EyBEP`Yzu-i1B+V2;AAV&r*bM{hSjwU*v=K<7Qu9r>ddxV&2sB
z+uL$W-rf@R_xA_wpP6f=ROPLp;dkS0SsRa}Q%Z^oXw`pw{ok*-syFv&`y~Vg34!*>
z{QmZK%G%r0cQx-53U=UNX|jHH<=gp=*st-Y*jP8_y^WGG$q0Cox})Nwl5N$Og!lLM
zPRX{J5%o8H;pDH69)i?An%{f*XZqvnkX11=d0$@_mXeBkk_uY3lYG37VNK-bwqw20
z-R#N&BF;ueMiLeU3ZUhvKcCM}e|Bc(_j{Z+<?m$XT9rOJF;O`r%W~R<xc7=LOCDu%
zvNSn8Jk%(9<c|TH|77;nO3In{_tj3E`};_za7j<cgD0KBA#Z;OUES>?$nT;cAR?8%
zNW3WJDfbku&`G7)*DtM!+^l5%eW7!E#^q(aFJHd=^Xatymb|-BHUf<cBqq<B=QrQ3
z_QuX)_2A`xskgRdZr@VBusVHy{Xa|hKADAC$ES0=Hg>jXU}{_tz`5%Be7D7634Xp$
zr-sLAUMjBta?yQ`O<~i8dzV|8H3T@g=5qPAUrf01T_tw!sn_fG$1QpO@55n!PcN^e
zkB^QXVie(c$82M38@aE>^5DUPdp@7DUb1Z2y-WXE-z`i%7aO>6<qTzS1%W%Ve^zfh
zxVhx2(`@bMVPDtQJXSyIEWp81-rBH1ucBhQW>KxT|8*ml(pOhJZCDx~*d4rneR|p3
zTMr8?PMkQgW4qr12L%BR7N$lJ)un%kQv}9gX=>+~c*(C}`|_6fIMajciq!tfiJn=I
z9HGtpZ_}x5b0y#G<$k&%I@W(PgKVZJmv_I`A|nnKr~57YI9Zwwxo=F`cB42p_0u=E
zed?)RkIwsA@67qUG;_Hb%e_{Yf8qWzPd?myUBPy3Nn!7$MRFYK0ReL|?jJjH;K7}z
z>beI{?by1lI%u8OLN=>5DTWnmEN<++8h0Z7!pDe~&Qk||*1psc4Rury0JUNj1VpT+
zN0@AsHa3gSk5E2o`y=r=Pga$;Yuu_Ui(gl~vVYHD@Z#}RpSLkLKEGGnxafR9+N!c+
zXKr`RC|FXVc*v1;&AkcdIxI8))astjQ9N0ecGi1uI-5|#(MNjzSzda^GbL2CD(?OJ
zSgUn1qv&};bWet=I0v{O;b19mW@tP2t6TQ8y#mYf&FZZmj!p8Fx%2;@Z6O2WmwBo&
z_0zw8-&e?dwCiTo@8jF&+aB<_+P6gBVf(K|;v04-Rxz>q$*i$RS?;L4Hr49wtV40P
z_v!{F)v#?2T%oWs$L86IjbTZTCg1z<f9|m>fBSA*)pZ-J{dIf2W{$V8sRPXScWk}d
zwmt1kliRcPcGnW_g?<?iY<2rrX7QW3w-wD4YGIpxa^p3xCr85O$nTxF{PombYagq>
z`N!L^L0x%8))W_3jkJz=4vo^))2?OgtPpv*zM?h0&+^1fjor1?!e4z8!t(aN+P-Q*
z?30;`&CWb}$qkF)-97G{XC;HKnryiL`qBv&w(ASR3U*4T7$*MBiMyL~(7%3n)665&
zPM9)$c@wsN$BA_c+>6u$0y0>Byv&G-TP_*Rb!2W}$9dzMjV|4<tWH(G_SkIYVX;(T
z^6_n@t4{q;*Ua(0RJT1RG0A0xn)=eeB`iNyBn0k!l5>-v<H|*ofcArY>3Ui*TbzTs
zW}OxIv{EJbCc}dV=acz9eqdp0ykOqw5U^&W+WhBNW~y9xeei*2z|O88&t{%EdTOWe
z{{qXSeJ7u|&ph??gZZ;}U9q;Z?K@LKwSHyg9lZIL`-Nd9BZInr+|j5=cD4LFyQ3}3
zABkMQ^tR&H6h(sz(gqLbKK=1+Gy9K<a<)l#n!^^V9z87Vx8T7+m*Xw}+-9-d(a@Xj
zmtv6Y%r<?q4)>a@gsgOBJIigp`<4~hoou`peEes9&WhBpCM(t)KhB`O_|lDOPZk8t
zn7I7)&egdNy2A2Tm_)3DpG9@PNL}PyDJe3wM8zhRVd@Fhv}wzCoS!PG)ygqd2$aiM
zm>f5%b$*SnJFvvResAL04co;()ZAA~`O2!hy!?|}|Hd_8o6VWN%2~~8Ti8FL&~)m9
z4+U3*>ji#Z)(LaJT~T<l%<YP{Yu1~fU44HqY(2mI{I@634(lvi=i1vfe!P<x_u6XK
z)m6=Dva>f<f6ntQSJQp=dLnPk@6@)YDxP;*PL8d@8WjP&Oa4ngbxmbn61(Z+J1*_?
zhJAwbzO2;~7I<^ey8qCOcq4`hH7Uy*mu$)}wwf<fY_MUsx~=xfAJZ=VYxHWEFQ~uQ
zB~7I1uB6!15B_ZLmuUW3zMIqk!4o^~;(m=Omzvl0vCE!19ig9Ukk(qZPrbVG;?g}W
zy6!V)TRSiHJ9gW3#T0NJ5a8(2oc<#%B;4-aT*u|<_Rr@U=6-a0cwtq)<d;)SVt$eb
zH!k^EW4mePw#rw6XBn4By?57J<X*t}O~1PLVE$jHCyUN^uw2-h7BT&-%iqoI7U^3W
z+T)6^c!j(_e~$NR#oE^y+g9+Y)oyM({(8!xRVOMYAKIjOenV8&nLWPomvsXAZ4GC2
z3i>C0ypUcTRqnEm^})@KUrHidZqz+wX|i5W+!Dxb9VGJ1idUs!zWLJ4f4kIljw$fZ
z<G;8n>baO)6!%Ww#I3;<burmbdoFWqiaV8XM8^2MuEFQ`?=S6CY&x|}sNS=*_2TWz
zGIKlwJ3WIkLyl@1M;^PVb2sHES4FC&OJ`A})6%|<HI+<_4liaqC}_-+i8pQUFHc{<
zEqy#Y`PQPRAEvClo2?YzvyS(`x_-A!&stn>|J(jPVy)E$b9tB73A(@2|2pa$>}#7Q
zq5bFP&m%HFCQY}WbG&P*{RPKU54ldb)%>lE-Ies^<Kf%4#p~nqJ#3GD%aDH1dv8hR
zB)bio!ppWicpvhe!TtA9<~1jHSl23VYT{B4_`slaamgk%rcCiSZ_aKki=6hQdf_=k
zy|y#Y6j*iEZRP8WEtU#dnzAb0^4Qv*nfs=#+SRkEzglu-E`w=oV)mY1i|;2hBHh=Q
zHg*SSHaaML@l_BIiH1*3aGrR#_Tr-d`(yW;z3E{;kXEdA@YA_j%%|tKz1=eJ(2Ez!
zD(RM8*L_b<P+f8*r0Ehv-`XCDpK8{AF^mh`+70*Iek6E3jnlgRvBWA4R#CqN4;oD;
z+LTT)ezMo@>5fNPjNi3%gBcA{&h;lx%AA*a;mpsJ<rzz@E$U}Ky(yetE*V<)ly{HG
zso9=qzv$eZGEr4_k9EkaEr^U+c(e#KtbRUy@%oV4R}XJ5E$2x)urAAx?VIjx%e=5y
zzT^beTq~}fL6v**x7+;cDyd$z?3vUty>y@VWj+5=RMYoLU)mWL`P}VYzy(7kkr0c<
zCidxn*r%?FIekSh@({}#+liUSJW5{XlyFUS+3Vn|cJ6=i?>6sOmeQB&t{+-#bUtwV
z!cJCaACn@zlTun+gIl~fS(p~)Gl7~g_3iug9nZDNU!Bv)6q0wrEBfh;Q|jIS3*06w
z-(a6*%b>jbZfCNZ@V`Sbc2c(vZB=>odSyv=+;n;Vm@9M6>nvO#>!PuufQgk~Zt5YP
zJr~{HXU#m9!+D<jwP|aG!ov)<__vq88L>(H$eAL(*}XdWxb!6b8$WWszm?Wgtx;Jg
zv(>OHQL0)m)v|B<yh3Kv<};#NQ+1q{!iww|&&0onJT8gq${256uCG7KY;C80fKmy=
z>*sdsEi*PBznF0)MeTe3;VV~Pewp{ha`CLGJDZI(XPk{Infu3d_Ou$G^E!_?|L1>*
zC{wu=yKq4pBQ)?f-LkoQ&SUB%t>ea`Csi)r(tlnil<MQBta;Tou()l}-`Y<brk*|i
zMy6})CW}jFiudmj)_-zj-JP1s<$d%1tnX$ty?b!E_NB_owXLVmeXWw0?N`|8y!4R^
zq5x74|F$OK(i+w)uh#hfOSr#j-i;K~<6$S~+*#DB!ov1*_q11Eqht;(xmo-(E5m%3
zt#$i}vw?feH|5S-xInJbhiiUr<txFqEj5jkxu&eWK2wr?apZ?z1-WypdV;6x?{IQF
z{!L`zf<v4lcg$yO+IHhf;uMd0R&g><JX00y&Ye4^Y&PBT%C@QQ+j6t7m<zBpG;h5<
zS1I&vGp8@l0u7yihmReyH~R63Ut0S7%AeEL#vIiNT2$ViIaPOc=)+xCQ<*{y!Y=KO
zdLu0S=Fk0WJmoh}zl>cNr2on>9JUtVVxdxq7p%1*qRJOEx7Tgblh%7vs{L=?{JY<%
z>PY*G!^<z#zbHNzyEsx}r~U1+>)-RAN{2d5sy5gZ9m=3{{JN0gqu3`Krb}K>obyc9
z>cT?j;wEPACq-)8zrC?7y14hox?P6?`sZAlv_HmH@VMn_7uMtN-}IcFGvkWK<m2-n
zukDNKo-u33eNnDkYCbJZV&z9{8kr0n95`n>^4^IQ*u%uT=SG~;NiH@8-^Ne<CqlR<
z&7FGeVB=#xzGV?N<2;v@F5p}LNp8>8V+X(3znQW+FCl41*`CFLVb`p#E=hf+a%|I?
zDTX&$Pp(_BBqlWC=j_&8&49g|deWDjnQ|qN(Smt}rq*>v{{#A0PDHEPu8>&r_)w^5
zitOZF)gR79<uX0b3pH)r92dxOPDpF3rmIaEs2m2jKwLD!gub~LC2u#|6kaNm{PfaP
zleIz&X3}i>`Z?Eye-vLe63e&aa#+5ubj775YVkey4;s&Zn*tix3eK}!Y~3=c3VCSj
z=7q24`En<fJ$><e%bL%1-;S-8^3A-klz(yCx1~+nk9-aOcx2_Q;8_)~bkbI3+P12C
z1&Vg+9@yk5FYv)h)$Z?or+e`mB^tiZ`!it!@A^v`7g~x``~F57Ib4^!zqxwrngz^O
zL5HJTKxKEpo6JiWj<mE{u4?jhpBvD>r`*W;#%mLkXX|{PSG7#^yZSX`P3oh~QRi}V
z_{?&*Zg{c7dhZAQ{plCJdj`Lo8aV9(_XXos`#X<`&HT0W4pY(>^AF!7W~O#uikSUO
z;Z=*%{RGJgD<o6|_p8r6E5|nP+w_mCpB!vYzLV;|FCqDwzS%$1sOU2Flak<40aR=~
zn6^PCwdnHk$hB!R^BirD9eNoYIpchu?5l9blv_5&v)^mgo<4JO^US|*7}Bh-pFZ1e
z@ct~P$?6w*C9Cefk2gqmUU$5^yIW09Z<eULQN{K<MR9Q@H@51%T3y(3Uiv_n{e6`q
zKR9BS@jjWo@_uu&_WXbRaevE%ZC6=Zh$<C_#h%>IcRKdBR}sJ6-%6$B>R(wBQnsr>
zN|xV^w!3wEZq%x~2mCCY+$wMG8NxqBV<r3QSXH%;#{^a!kIG*AT>X1wM9mq;N_)=y
z<+hKX>IQFK)tL3qh*SGa%^d&iZ|9{i{nQR{XBCdw^+dr=)*<n^UW3k4k3xaRHs@xY
zeL3Z8!;X4zQ3INEaZu1u@<}{nQPCi|y_P}wYqj0kKl2WI?yHjRj+OrY{C)jnj{SeL
zx9BXGyf5-_!z{IR@#d4~t!rHq9VB$~|8f5_Gq(42?VcuRZ2A7)u3LYXeMsNEd}m&x
z&{F1)SKHJ+o$0EzVU9R^a>iM)H!n0LU-S#M9`T=c*(TOMV5`xincV6D9|{%<&7bl8
z#&oH#y<gU-B=4VcaQ5!4JXIe(Bew@k@LPT6{qx4P3}t6ty09<LUno;AD}HPuw3YTQ
zyywR+_D8di-+KLfx>3cg?^kNd?38DRGaWF$eav)4v}@wAJ*+RdA23$@W`8!*<JgP0
z-50H2A53M^W)s$syHwlk!K18Ulw~<t{sr?Fp0%?!zw+F|VQjE^zM$UCAE8Uk#Uah&
z#svrFSZ)w)SRB75>HEvqme-;z#kWrlzh&Q@`RDQ#k;~>bUpnSFI8H45vBl~BCo2bc
zmvnt4k*O)&T$@_f8#_<?I%{s^M9D{K&??iRfIDQxb)g%LS@GvqygPmIYKB+8RH(vR
zuGMDwYzj3us@F)~b-JU|Sir>U>VBtAd%hOmyo@)Fl9QK6y5AF8Fsa5j@%lZUX>!7f
zzR(7mz#T2s*Zk?1`Jc}C|Ma`gy+7)GJ2JwKx;BeAy<a%F^36v%6F#wEbB-2ajUDW7
zIyY`qKgPz^$#qVp%2RXIi8OTqQ0J3{>7Z=Twy^ZLnOA3qi)g2H%U($SWY(24m+2u3
zR~M(*RAaM3X>bN=bWqSJNm0Ic&#h3NrBX@gFuZ@y1adT_;}7cpe^5|5<k;OHWM_3N
zY?|idrGNdrgDtB=A0Dt2H5TeOy<D6!zt{PwLCp%k(3x(jTeda5_T;kev)oa`{kuJ6
z(#y!`^Y_+<ZCfgSXfMx{m+^r~KjXhuo1J_1ZmQ#=;$+)bnbOk2pKqR7@OWcemDee)
z!0N8_y@H_G@iO~Y(Tt`ZFJ?b*boYF8V4;fB>(HlhEFn>6)814{zN->;du%K5aQO_&
z@BTWw%jYjXd0<|M#uQDpHFJKv&gaPU*--y*hYIp|0`Ei?cZ<gF7b0ieIOwu9sB(`+
zm)9i8KQ)fFL3`}VCM`dsUF+NOD^0(8e;MDlT#xj)rdjWg*L_*67R8?0HRV+1-mmB0
zEuDX+W1E}_=Oo9#8$YMd{jsQSUin6zw^G#?)-PmVnqrX|eq+Xq6UDcszVdv@%3u&#
zRL05esqn@3MfAn22^Y){HlMqBfB(ai&&59cyJh=jMTXX+&9fsvXIU@H+xA4+S+V5J
z&Az<9yM!;=d|79;YFe>%W%415u;Z3@gJOfHeS7WMYHhXB=}`RancvTO{}D-lepK}~
z=QsUOj-|dyuJae333=NybNkd|i?q1aKg3;F*V;Mp>z(c2OZH5iy8G>ssE|K@Q>x59
zE9}|RGhe@<`O(*vmp@)h@YgyqOL6)$2L0yC5tB-{ynT1_scPn*S(C2+(`rvLzMy@}
zl&QR+)=KS*`9&24?<bp_j|Lt2{y1-{qRN>A^|vHHp4oJ!ZnDlN!K+)Vrq&!fr}%Ld
zCuk5vJBe55ms-UJ&!oG{?=u?Izs%dU@6NUk<Ls)-`PbyE<nEu~u2mE2<=VAu&*loP
zUZ0gRKdZ&Pm)D+p7JB@~o-Dnh&*Bl^uV%>Y^lmkoy7q9(OYW7%?{(AGSTR-HpZK}J
zHcvU2?N7Vzs@;YsoZ~Houiri|m#{3${SyCz=-P|i6`%DdEVz()NoRHAlZ4Ar#=+({
zZ_HklwC&(amsb7orJv8empu2buG9U`lBD2&({g)4COoWJ)B3UaY|rfUv{zBw*G0;=
zzPxs8!h6Fq+0Y;FG7=+%w8YQd{!x@_wfx;o)ynUu5>5u(+^RhH=kGVHx8gocSsCgz
zyZTK4cnqjk_*d8#DU~n29~MlO+C8)X@s?lZDIaUTsjvRNL`HS>wgvv~Yj^ujVp<_6
z<#SZ(>x)l6B%Zy$e<tO*)zkAc_v|zH^-k2~+tqWoH^)AdIk#EIdD`04vv<U~_padF
zUC*rW<|M}|Db>K4t-=voJmwi4mfm&S<6G{w4I2Fy7A3nI&k$eDAjG)u=+aGITev1Z
z%wk)gSK9AwIXB_CefFAn5?`6NSo{?Avfq5le|L_Hs+Gsi^v^o?_ZrQy%S%t}IW+M}
zQ)^u7l}nGA6N)#yoKtZhJgOtQmW{Jk*ye`Dn}0vqzx%tJa9_Xw+b%QISNXPrqh&+=
zUW>zDi@*0NtK83BZn7b0q0DlZqq|Jjht6}bpYNoXC={)9o#DdfXj?=5Bu0UU3tD}i
zo5}51we{w?^5dWW|9E?6;=v06=cVOb8s_&cds><h?%24)nCs)qosy|myTUIQDjYi}
z+3d1A%ZjPTO6dIUo6}yHoZ}I?>V0QNe$Aw9`xecA(xtwA(c(Rhsh8hNn&-T?NPoWK
zsE2IuJ+`pj25<jlYH}@qpZGQ_CRFW)yL;5E=9bfZNt51{dCzkx-7h^oZo9)to$se7
z&HDW(kyW<v!nR#%*(cMx>ds7X2$1=zd%%BDO5fj@_mO;WcP9w`nkKHcFrSI>)zrOb
z;!0+8EPQPE*RuU;VVT+9&&uk*uP^lSTko`SQqDO`xf>np*QW)}oILqt&h3f4_bSr8
zpCvuL(^<b`PSWv|Pm6t?7X7%Nv`M^nr`DG#C2PWDEx!FZ?7F&Z{pRndXVuILPM_qk
zp1oP!@u}9M+0P~Hw4eLUc@(v4+s^cr3R-Nx?roU5cLMWoZmH^oeN}y%#r2XmzBc?h
zWl>Fm<(^{edpDVD_E<A}zH?lCVbUTaZuJc-Ca##Ss`2~D!JgUavo>qi$fv5Rnq8i8
z$#CKRiE51IT?dy6OP;y0Kz4Jnv9jRRFDyT^78;w@rmVVRyo1-Ir{vD$bF=F;S;EW=
zR1YctJ~-*EPh{3Z9-;oSkBdChirBm7ia($6S!274X8M$nbA_@~S4mfxD*e9EU+w5I
zqw#r^^v9Ehzc1<C{BfV_b*zbw%D=1UyR&XCIw2DFHsHmbQ-`0X++KgH;;H*<-Bg7V
z=_5v8y<aj+b#{OIs8pDv_V?+75AyDroJd?Rb7$HycXt_<uq(z(H{bqbxx=zo{Mn4p
zd+b1ycP#wo?dEz<XN1fRpD*csvgmk(Z8Gz!T1)$XAM@j~;uWitFRwhi<;~}yZK-no
z`;W`7|9(tb`}Tf`vrAmt84XIR`d7@{YsfV1vUTO+;}K!2Z%SXR;+OS5e8K(1j#*y4
zdHomH+&L?AZS#?pQOpzWEKz&fBQ)X2FMa!jOUw7)+wd1U%64<oi5Sa06V2IE<>#|2
znAdmxd~fT#*x~ZtYKd#cI<x+%8Q1kN#RfacR9m!l>6I9q7T@R8QT~l%@|l2u{)4fu
z8X1g&4;Qp{I=Tm33%l6U^!D5DI_c`%TaSP24~#S6oZz`TCHd5x5Vtd)PoGzuRpR9M
z_F%5NY5KX{Q_f{7WXiwT^_8b0CisW-J-@WwAyd`dZb>N^+BFAG6uVxO({-7VCE#aC
z<gRZQ!$h9xZhZ2Sae7Tj%B{TRkM4#AEfwHj<|^^gMYXC}O8ME6Pen^wPrKSIS?pHJ
zAD5ke#Jy}o-P)Og#X&RcBea)?H#8dNzDp};Xx{#8=_Xb0YrCIBNZTJx+3>k*+nbCK
ztHt@39hf*)KIEHvYu<vc@4gCO&Mq)2&iwXWude+W1Cxr}_Kz1C-~M>czx;Eco$Yk+
z)PP*k8}*+b-}XvW^)A}C-lbQsU#MxBbGKdW@vAbkXRj)I{@~e9lhQop_Wf^47`C>p
zS~vN*fy<<et+uXD+-EU+z((-g1cikUmj}zUDimEz_nB?f`qFsuhNsuuC+7xot=?bJ
znVj+7>o;FwMR8}oo2$Xg%keM2ym|gdQrOw&Ox-kJdBy{WG?ix<+_x?J@MvCs=Jy}m
zITLSvdKtA==lAx%Cj>m*etwp2E3KNiH~jt8-5IG@rT@<Om@M+pzy91m=FgFmn<u|y
z&VJbOx39kTGyBckuO{6P+$q+OuKcPyAmAa3nC+xao!BdUeDim0H58kx+!JD!aqh+)
z(Xi!{+3fSZ(szG$&gS{_;HQABeZI%@o4$G4Z=Nu6Cr?{yTs*neKeqXUD$^Xd*^lSU
zjTfG?zQ^&|MXz^ZLYe{X`~{k3&AC4%%;wILG!fjL_V{%2yJ>%y->S6e)#09RIbD6;
z=FeCEE!sRu#eZhuL%9a8FSkmjS@m}0*58hCTRi{&-RZ6Q;ZZvt{JFh-7rXGS*)Ia3
z+&^(@U;cjo=7Ra#65j=APBisb_;Pk_KKJ#=lRnQP_pVcnUJ-EDfwSPlY5AY~>md=C
zr}r-EsQ;?w>Hn^>DJx9neZDeo)(MNgb18QpzBb=CyV1Au=~p`jjllPtKhN1M9=%t{
zf6c3e7cGrZW=Riz<lo;IRk|SGuH)Lls5MoPamQ<C<^H{?N#d*VT3uv0*TFdbpQZ1k
zotI^9{Nv95mVa2=e(BCR=dZJwKlriA5>(#B-mr+1zI<(Ev(eUS-zVJ{w=&)UFJ|A+
z@Kq(g=MP7NVZ6`6TGzEYo9cbl^8Yv$*U3g#zG@DQcz^Qpi34wPN+x;zww`MxpLKOL
zW5e46Cv%Ed-qM##a81+5kew(RV$t~hUdke$mC?8Nd8V$NwLEsiX7l5>QmgN>xo51{
zHrLv)vDyCn?$+m3(;R<9o}FSEnAf~8&bd6s|2Z35&x$`E%kRk-Y6hH7zM6XA;3kjN
zub*<SS?8|3?bJM(-_d7WuQ!PVdHZWJ%s%u>XYGlsuDPnebLQA&KmVO8=^7x!K0m$B
zd{U_i=hfz$=hpC+rx|nwXf|HhzUQm2Z}^SUgFEkp`{&z#ufDA0;U3l0yW)?RI;dCp
zKvBJw_w#e*_WeHVt7bk*TXOmG`rX@pBwXAUBfIFry8fM2%=Hob&u{&6WLtd4wXKW8
zFMrBAyUFr?{QEUOm;cGUu}+iW;>Ajpib5CTk00eq-dviieth3C|H85-X|^*>k{RpI
z$htA4Eec=5vLJc?HkOpzk;`2F*YB>&y&>tJ^+&HRNOnQC|8lm}&HMXPXB&Sr)ctXH
zTa|6$v+9e|FI*$GtFI8e)mW#yI=g#KxMgKvaQeyH%F+4phtA!%Xw!0X%oj9N+hDM<
zS#-{0H<eRAUq6UB7<F3JU-s#zlV0<RVjlJ|iKcwe(hXLeU9Fy|;h?->R?74D*;;44
zJtdmIlzcyxkUOhy+rg-9d-gm(I%&q0yDfqp_i_?*XI1-NUUg3Rwv3GL&hqX1rWLOV
zids~3c*l(U<<UR)a-5!2VA9OHuxb}eX0Fcke;e<9jF1gqF-6H~DT~wnBITm_y?S?h
z8(dB=W7~H>VX<E<v)ziE->0XoZw4=ZeDLsCQH`~6S+wK!)>8Y~Cp^wF*nHUVuE@Gz
z^|OtyZ@$}d{-$^MCOPN!Yf4oY6&Kk>9^I%uTOeuAnO)bF4gdc3&PtQqoff{8CjvYq
zEawuN&pCl}jmV0f*;7wFpY?_1!C%qY9Qvw>j6U~NKYl-FwCg;0Ncr`geL=a$t{Z%a
zoYT8@cXG$|<4=msQ>0w}NZ0*hk%_jf(3-hxYnRo<E}y^0<CBg)cKGBOR`*FXc>g)0
zbB4Ds<dti@kPzgm^6r>-z{$n8yuk3Z&VIM`(b>Hx?kk7yoc6Qsw9t0zkn*kCEuQYT
zt*mNRy;{le=C}3xYu6w4R|OqsVJ}~@^U%UgdzP=8m9nS(`@LluZOzmB`;3k*(95xZ
z{LYkPMY_tNg7o9XtzL6Q7X~pc%-?A>iL)$e&ePAl*Mg;Agaq@awZ;nzg6C~dFt@&t
z7iS1qe6n<z@62OgUS6JQS1Xmj-tMWCct=&w(TTr{N+V`{m0gpY6n}2l-|HLIT|Vwy
zWw5sVa87hs-UYAUhc{037fbKU_D-z7QpMm_cKUAT#3ynr;e1Ez4JH&dHr|wVP|y4s
zw^-lt+>DT`a}J)J_1=r^mw5B@B(+!V{3|YR63@^1ocOR!R;Rdc{u!@p0T=kd!#dwD
z_$dAB`h9g(#QEl1+c%UilU}y9<$3SdSw**X*K@kRol{h|Uw&m;iVw4`2AA|_j~NCV
zri;rqmB{pZJzqSdCdK)h;G1wBxyJQAe{FasSv9(fC?!mgynbQAThrg+`Te);=2(Yt
zCGmGcvi7O%cE|3XxVnDtL<8YDHc6+ABRfHBnVA?@y3g4D!r)lq{LM-CxRR2RV*c~3
zi8$DG|M?D?gDcMOd+MS0aH6WyZG(%J>+=8Iitmj1tG#Q3I0wJk(;ImVpdoYF>~9uQ
zy|d0d`IkFmm(|>RlkL(&Hs6_DnKwh-Kzz@f|M_=in=R+R1r4*WF21yJ>Wtu(JNLdg
zcYU|!)#jS_2c12WF0R<tGw+*j;$gd#>z~3iL)NMMib>Hk7P(^J02(R$Aouz5d&ze*
z3L~F&rkTH=@$SyC6UC+vDxO7WPs*%qKByRYY^Cb8E7qLT3pdz>6-tVfGL_8fKYL>9
z*RAO->b9p}cl3MhPrR^y-JHhr?Xxu&?l^s2Md$a{s7<F%#`4*0(|o7(H=|v8?Gn(U
zwT<&TU%3Xbw|zGfQdyl7J^NVQbK|S*6(79L-mboTXJHUmG-yKbMf5zGKd0ZFDbWeE
zz0x4qdX~LN{<RLMjC5cq__KX}r`yx7`+AG=S7_XwlP>?c$kMmF&VJtMu)31j3%;_Y
z{rO=RY=8Uce)SUN?f*>n#&Q3cV^Xab&->PGd-`*Mot9nS-L{=I7WuMlMpNeH;McNS
zAd8pFHD^wa-r~0DfI_{NIcNFWn;)KEl;$gcn$uMFc9+)=OYYDqkB{qc21ZrreQ>#7
zw(fV5fyG>#>%CrutxbJzH9@nf75#z1Z~jbC-CJ?AO)V~Sj{8=V7ZSf$h&N|Ta(rs)
zb<fdjUYxu7jqYmGSJE6sR)yKyvu{VntaZB8a(btvdTP<k8(G)q@dRzMXx&}p$#HpI
zZ)fAxRD;`{Ud`^WU&Q{ey7v9S<a=&5>nwjf%-_Ab@YTdBwYIdG*O!G&I9F<0sxA6x
z^~`SH@3-%pg<IA%_txCG+iW$j!7(YJrE_`q$vO9?2UdK}cskK}xm&HWEK}(c)f5B$
zg9p#>zFTQ|=$2IG?CB3uGasI|bdS6(@Vr7nP)k+dSDA2a@Qos;!q&oNsyB+yR_J_r
zd(8wi%+q+ZXo}_f{|$O?SZ`g?n0VaHNVna$@Nd_j_xy8CK9$wH|0%G_YnGlz5x-Tc
zC&y!}yC1i@^i+sjzDZ>(;aFbT&R6o{!kh;O?wot8aQLQ%nrGaL&tDJqY^lg;D&C$U
z=JxLVwr{`wt*u$C<x#}%R<^opWmb0h`KRmp_diWATW;8Tw);8rh2Q`Gxpr(We_NF_
z&C`hMpz;;zJNFs+Ys+}IF5mIn=)8JdVb){SZ}J_N|COx^(|=dm+O&;Rc;SNWe|V1Q
zYSh%-p2FAFd2nHsgVLH#_2O);H`YmyAE>fVn(lHof!SB<<wfR?)t4%N>$3*jz&`!(
zZIfK@rd>kfTZ*^(c(;U1Rdfqp`in8|vC5KJazdMr3#6o7dM&i%aNX59XQvrux;{N?
zp01I)SJhT>X6>vMZ*HYK`-a@CIrzXwNI~%8?b%VBTqoyQIWgbK<q17x8+qZtxheU&
z>#ls*?&Pq*r}fMVu4b1R{BuqQ<(XbQw@R3+wt4PMR_)3Uj|{dSxUv4`kCg9omm2Jk
zlw{IQwNq#3zJ4p_=`YQ=#DY@E36f?fj!YF5<4ZO6QN7i=gLOlSq^yPVH&y?cR!g`3
zuFwb*)SvF~;<oRXT;DCdb_b2zLcTpQ>TSNT{mzN#=9Yb&auI8uM}9jNkbA1_lU&5w
zC$>Mfy9mgt$VP{-ulj3!Z_e#5|6@;H>@;pQ_|RGqR=;QRhULFfy*VbIw71^Ed3NgJ
z_m?-F`+ZY7GRtJC<8-?osaw;z)Y#L%uYInYo<C#Nn#9lNman*CCXy_d>$h|^&%^GI
z))mh-&&{dH=O{j<n_ki$5aj>o+!V$4j30h2I_bCkT>r!Qb;rKxC3lqNq@FmU|KsK3
z<(wb=l0RS1H%Q(t>ik}F&yNg=nQ`HF?qyuRFD`uc%9@BretC8qtMBx^`ng$Lv0HfE
z_Y(UK%jDgKWeNI8^)YW}iC>iaYIeSFPT$StQ>TfRIJjq3bO&6B4yy|d<u?77^yo<E
znK_n|w@pqq_!(NWB0n#|a;9OfPl)!^>Z3lp)6%bsac2u{t~X$JlXw{tJ3qD?HmAZo
zp#rqvwphko^Pbn1<&AnzOU-rW@6g<B_$2t`fj95m`11Tt+cj=-e0Df+p1HqW!Zzd7
z!es}Q4fjm%<a_!w$L&gMwm^v2p}fKZm2<~_Nk|)RpOff+{K~_)kkh)sExr#=SH8R%
zw4eV?(xUW&vc>O$oY;)7rQP|aa*S2;@J9pwd->1JPWj1uq`%T@J+O-H-m9E5|8zuK
z^`B=wU3dG&i6v?~`9E3i)Sp|KS+#7d5d)X}8BcNDr;$G%H@AgUAL(vp=NG#PS{8S*
ziIw}sk2I#LV9$m5M?>z<lMHHlwrQ^T{;etX?_2lXJ#g0K#o4=DjZXKw=B&*RT^xDX
z;NOqO{Yuu)L6@s+N<F=!?5z}A^Fzk{htsC@X1y-jv?biO#sB*M3g>U<A8RkakpAkt
zxy-=})l#?bzn*_qaLfF=9?k3YKlnY7ljA-2^r&C&GQS@uYP^^A&p7q}XYG+}`=^s$
zxq{d6p1XSROv$=$@P)d2rb8F%>a;&j^R+p#=YQ>?^wpdE7a70#y0fR=MkZeYyyW)J
zc8BZn0_)yHI_(yG|7=&@CG*)5H}~i)5cqnwyXW5z^Xt2g$N!ymQTI&Gd*i@jw$imP
z<LlJ5{j4R7^R>^LX@9=(>gbP;k0UoEFov&-Ie35n1JBwP;Ru$BYA=@Go>RU}`!J1>
zRY*_6+)zSg6}#7zE6rEe2~Cdpk*pdOa<%Zm4(0odN48&C^guK4w)>&uEYtNYJN&1w
z5IzyBS-np}<rRyEL}Rxn-?=6C_m?hj>F@jSNl5Y}-?q8Jk5gNOHNLEMc;7p}&&s01
zOKVRu7ifFh9EXEv11-vbaiwarJ+rdQ{m-9oTyg8W&#yFI=B!ye>hAhf?J{PuQ=Ry?
z{?Cz~%IlH4vgZ~5>bQLrvNHCTgiHPZKe48(EoZ(?<6dxl_m=(3o*re${uH_{`oQib
z_A@yH*PaaDnY1x@Q%UBT=R!NCty5nS<ys^Eh_P||YnHBp3yU)Hy}~;^_h03o<Mk)j
zOs`C~T;$cqE7I$NRn;!$e({xf;;K7&*7J(VIikVSEXt1@y1iZaQA%*<$)gKah&(!|
zoy&8>n5!T@m9w_((4qM{E;rxB-u3h?-0*QlMLUbr{k)fZJ&T@LTh=7Lj-Hs+eZjnO
zfrN7Kx}=14+3ViNpL&paKlt;_pFKHooDBDKI?b~Kws#%sspQTx`nO&|;^SBTB?~HT
zd++UIsk-ka7GV53eb2F1o7ML0$=g>N6(x0+-9N41^QY})qWk`_{JOVe&79(y<wp~k
zT$k?7Z4uVc2-@Ql)HwIRB|EE6XATreM~H~>H@CPqUQ+aY;cMk3!7#fbkEwA%MK;)3
z6~)gEgfaBhZS1;hn{{DL(zOgB?dOlmKAvoNZs0q^`D~K?)kZmk&6TG$vs&)#SyNb8
z$Wl@r7Xq5}k@E|&i%r|NT-C!SFZqj1;ztgaCU*a5&}z)1v$?)TF<A$^JuP{))^eAW
z@5{i&TuWjTuRc4=(86EcZ@M`84tsd=lGjhxzqE?934h4K1)2~7tqX06(Tuc>U6=`5
z3s7${yH4ZyAN~F5AD`J59&z2Qr6;vSpo)L)ndJWO3ko;Pc;=um$Dz>yq?f7jLF2-l
zwdeL2tu|h-5q3t*dcpB0xgwM2aJL9+z}B4FTsYyjXK(K#JC1jnR)yR!dzl&^JaqiR
z!qwHXE3r!H)afNLTR!`qd7J)+hvUk{z3Ujiww$%xxWz$1KtxWIxpd3A)CDoK=JuCq
zw^qG5bm`#kHpfM=y3D$#SJmu2SazB7MfcyY94tp$R;%+BFHH$``s3?bUljf#^IcxU
z$rKm8R}~>g&+M#Ey1Dnaj4#`_B~Ba9w`t9BXnbHc@x|?1E~b(z^on)NdULnWII=F4
zTX^xSm$9N|{}!ESQWxOh@?YF$pS8$&!fl~V^GX9|1YZ$p<GW}3-2d6zh4VsA%#n`?
z)Hr%}&Vi&FTPDYik8I+<{uDo5wb%Db^<K?a3Ky>Y_}aq%BY)rRD;#efcF&7_d2h9_
zDbxFTi?b}ZPc~QUNiLmpS@WuCYUciVAp#ROerFNn)6x*S6nm`k)YRJsB6@STZhW~x
z_P4IIxsa&M(ZAvxEI$`+DvoDc&9}q!S>N6LF<YMdm40~l@WJ=}Mmr`%s<n45<tT|&
zsF2vPDV@1Hl=b$i$~RtH#DD(uZuzFDmL|S=laH;S*&ACW^EIg_&E<{j5-bWgM*sL#
zzy8AhWZ`G|_SM%<C22*sb6yrV+8OTqTqysdwCWs(gUxvfwL6u1<;x;|d}=S@yL(Kq
zPgO&wchkf=?`lt<noF|{O<hdIIatiE&zN79ddGz2)3>e9&y{{yml*srx1MeG`8!>&
zqkGo7D`=R_xVf!h+vlm4#~w}n{Vve5CgbMb*{{P}9|$y^4qX!W`JBz?V=-pB1>Z&L
zO!By@*RLvHF0n~pH2cb8@AGAgvN+TOK5UpU>rsMl&)gMDtPNJ@ZQQv6vXWKIIh2LT
zkx_Q`>1o#XUU9dAC9EpU4O1k=LnbtAcvNT0ba3ync^kS{&c3@y=<ChxGKQ<aFVl&v
zPf2+{QRjAwfIRP-rGE=tiXWW$x;tvxq{gW(Pc_&&;%>ByHoJTkx+xy1qvh#gGwJc`
z{Ee&UcwYRs?uGi!M>o%_ubeTrK3qrr@`?C%#;HeEbKO37SEFds^1gRdRqO0m?f7<j
z(=@TGrBl9No0wKQp}<Q~ZGWEq^1`c&cKKB1%W7yvC%;)2_Gg-|PY=gcomrRate9#-
zHmr!*lxiQp!1K_YXAfRfiUwFt)?3x)5Fp_7<3&bv<T384M^f0^g=KbrTf_=pZ_8dL
zD$-%SSYZC?YPHH&FXDWUH%eHi^9a3N^^(Kjm4{7!di0`KHq%^A%KVND3O#ot@5+`n
zze=>9p71QSFF(}udSxE($yKdl-O<*0%WPRM9(0)8a(3OMS#M5+dNaNXnYuEie1n?f
z(xe4Gi``2k)@Vid-+I5tJh(CYT=A8!mnWu${(L@V?hFyGb+6ev#ipA2Zr9Wj7ZzP4
zl)q<MgM+LubE(9ef@g~kxmv3%U9t0ybmN16$>(3~T#$XfWqHkyFRwica-J@~-*3Ft
zeXGp-Ia5<Mn;yUCR#R&_)lgl1x6_r>#pd^pt(h6KW{&ko{^`MQX86=iSsBAu|9EX$
z)kMyf;mmb5fBwIjTV5e@^?Jkon>WwqZSquLR5IPdz^M7@B9n(cQxJ<tpktSYMub2}
zNXU^kC41?k8omw_)Gk%O5a8@+^zhDDIE$fCWp>W|_x*F<ESsF*p!B_Vo{fK5*{QiR
ztIy9Xez)^EL&2vrqATK!F8@4Vs4TlB>Vsb7G%v@0mGe$cs+n0VJ}Jif^&ZXWeM0|!
zy<uC~AusN>;6c3dicNhVW%q2#@wQ1!>vcUn;aB8t*H4n%>vJ|e6kqeWd0o}k%Nl>I
z?jOmDOsmn)UnRYH-v0VUz5oAaDe`rim2TZqAK?CUl2n0(MWxN$9cOY^E_=M!uj<dH
zt%g6_nooQBZMi3PPR?kvg5l-qs;T#+A54C_>+Wo;8MgLwH(&j_XTnRlQ&$WPcJe*g
zU|hFs=Inz}c5kx|@NbwQ`h{)!r1YIhsXx3{zmQ>a+~?8rB2{5cPMNB+VwStr<-YWk
zw^t2T_0Kz&TBN1inZBlG-$~J`n=4n&<GZ?b%Gc=(t+(ooLSMZ$I>~JNmVb%y!v&Mm
zcS%%g%zjsxk}eh8XUVy0NotLH|FS|ArC!aA=a{xDU$_1`P4eRP*_R~SXT9l(O?WnI
z`<WSa_okG*f0^>_&6b+f^PfD_d*!b@(`0{r_3FnUZ`+&G4fwZO-q>*PjQ(_y8s?O?
ztycn#J-Yk(-gK6&GQDzP#~Sq&%d3sq@Bf=}PX7M~S^jrVeea%mze9D7eb%$Rv1L12
zWAZQGIpAPAar)6^3$E`>$+>S~oS(CG_j<z>yBo_yQwlVeC_mKQ{p;eZR<&u%J0_}W
zJX<37`1wnBLmT^ULH!>efA?p1I3=Fky;|a0uE|p)>&@p}FD!a49m)6o)pyC=_ue-O
za_>&(Nj_6Ey>Z#=ACeQ}cMF}FasK_&<~-$y)hTDE-AjJpA$3iw{cDw$>24dp%b&K-
zx?NuIF=fs1lOc<l7_Rr_B_5i*ezWfFpHI`y8$RDG{&T;MXUe1-KmMj1^NF4M_|5q`
zo9W^5**z!z7v(D+a(o{w^k_|uROacV7kf9}J=a&8woL9@3+EjB3FowD$NXM;<o1t8
z4X^+2uzk5^(@CGYTeDxesau;bH?+8OB)?EEeyYEu?Bib*f-<X;6TJ1OFJeeIJ*&Xs
z)8tU*fQJDtQq6~hBc<P4wby&jj0!Cc(O>O2eeGYxuU|vVEiCJ<oa&TbRUWDSxguNk
z<MK5%o9?}dv#<CzVdh6}-<q9w4za4Zm6|+gKXmQtN}m5GkF2=*Y9F6#2~Xn#F?-2;
zej}l<Fg4A4r*3=*VLR^^-mB56mdn$!@^`JpmG)z)DVKzDd-(lR3o2Gc+f7;d>ZtnO
z7(b0<&)zJCUFQPV@8er9&G)Zo^W=x(>n}z4XU{!!y*udUv<HE1kG}2TacF3{Y_?3M
zZ0`S07fZG?MqDU+(l+T`!kaTb*S|DPdiCx4q2Qly7hS&l&WP>P6x+@GnqRV3o(|?S
z%UhD}<86CS<IU|ilbRQYtvDa5etBs-|KH{->4A=>42-#wy1l2p=JakZ)Qg-N&o*;;
z)L-ZS>l~NmMb6wJ&9^RvD|U6r@dG<{&8m4?6=160cIEma*Tm<qLfX!oef-Bay?Cup
z$GnA4V}G+=|GFraRnmLW)Pu&pdn}Bn`o8)sQq0oypXuZ6`qbHvQj+?s5;#&{7=-Uw
znl$_9tLDg9|F*1HOCiH{wH+x7Wxg%TV%Q~<*uO{g9D}Eu@GqIw%U4Lc1^4gdaai^F
z=qrz;q@*7|_r@`-nP=bI`S`f;%hw(`JI{WvNWD9)DoI!WTzL4mtXo0r7wOl_|L8RP
z8<#x$%B+>IW1sim+c|}CQqSY>kv6vrbCzvw?mQ!PtI;qx{Nl%_FaO91JNx*UbgT2S
zCEWXQuxFOAs#U826YF;_^~Je!f**7LIeoHa#clpKn<q__Ul7zW?abtwwX3rj*|n4A
z*nI1lcj>@Q597`L){L$9x6Qiy{K~Y8K~JYlO?;>}?bzy5L1yVS%b&&R_3uA>@K(fi
zJMZh!cdl<&iISAbODd}gV?LX=>GNCvOrKk+Pd6XUoD;N_?Y{!=(p;{>&MxbJcjrBO
z;?(8jGpXr`y6~E)DQ2O;-g{HhefJ3OoxUPiS>TELZN923$pdwVkNx=iif7i@gYSHf
z1+dz`xZ>!aD*K?o;Bz>)_?7(n+q2HS%U)OeHFa;r*0#3$W=lVR3Omp9=jP$GHhsV6
zuh*t6liQSTxQXFe?uH45o+3g1(_QVZa!+DV*Dl}5;*of3qI~#Sxu277R{T~z_t;?b
z%3}d}o6?^<de18g_%u!O!VT3adh%w%>#o`R`WAe6mepOQ*Qqy`S7K7kvGkru-p3m)
zBMj~zX;~{~7cu4CwAueu|E<&s?&$J(y!7bHp6G<6%sJ_W$r-`6amPPTd(AfA=i%m8
z-Cg!i1Mbgs`RQ<d^S}LCuN-Gy^VPeRb&%;t$3iuYZz`Lfy;VJtnE$CDboz&^OT`<0
z-+uXd+Y^q;X-%^X-u<1oE>Z9&YqfsmlkOtjKH;v(jf>|kSY%b=m;Z0()-K`P&FYmq
zR%wgvJ2-j0;ihHY^IG=Ww9hykoEd4iTyRF!Po3#PLKgL(47B&npRu&%$PLT)f&TS<
zlFrK*3~G*5+zKn28_&1qY;<&505j*D+m(5rq;w9vG<{gJ^?sMy7u9>R_oAkrHsUXn
zKiiYY&$J=`l)$yQhTZEFdmIk#ZJeB&ekDGk*{40><X+ovZNDDJ1$m2`>Un><d}+xP
zmrI<|^$$56R|!4y{SmPE_xeq5UdC-bT`{HWPJy~oK}y&<o9ox7f00-|v(q}`X}{0A
zM^kvN@7_K|(?whS>ajH|HggNb{`zIGrEN9i>&0tb<_bxsur&Rt)4LT~+B<!7{mo4p
z2HNEjKRz#->)fuB<1sN~4Oin9?Qb%Po!!Q~ccNb1+R3ms{mp;5w}CZzYSx>jB6p-c
z{1zLy{de|D!3T!RW=+33K`dW6NcM4S?(w<m%VpMt#I8Q2(^`9zYn`?J%?`gXdx_Lr
zKX$&;Zr*abY}=B`bG3)39bUJ~His|lSk=70x7MclrP%1rZL_MYynp^H3w!uku1xbc
zZvuJ`R9!VqnV-LR;;J3JAJ*@BT^aDaYWwZSIdep+7A=@jY+S{CS(JTo{7X+amfZ`Z
zs_m+-h<?bu|I5(2$gm^J_S24Pvx1Xu{%RMEt;2<uW;%aY6SX;fWm%tna+<x50`F}T
zMg^@`5+^IS=znV8Sdw>t-n!a(&-SkLW_V#xVDkCe`^199>>j<hj~gB3oVBf_=0)S0
zW3QOcN_^b$+Ab&Cyy#Mj(yQyD3|rpcon;jHZCd8N=at`=`&K`8dA_4vAe-rZaoL~5
z$14RSZH0JWWh>rm6Ir0TCZw(2L21pzX-{vz-IBS=x&P3z<}wxgnL)39_3jWhTbFjp
zDyQs1CZps!g{PSeOfQT?q?&z<?eB1A=&5T5aCwv$S#IysP7hs^wBEAf#Qm)oYq+02
zyI`vR)+^KD+J#+v&Ckkh`uv&A__AK-KLvp&txwbim-PKie!giPe_)VI&E8`V&5oUV
z{mIV%R3^K)Oh6#p{2(LI%`E>tBaK3y9ew3&W>tQbSx9%*iyx_zdv|>;`LS@r=LNpG
z?aMB_-SI)+=6coCPv4Kev(8=~SgWfLxO)>@(v&m5uFgF;@p8po!Pg7IWF;r8hzeuY
zJQu_7>$}1DoBDLc`zu$@o5x#fd99jR#v(J_L03{&Ai(jswDVRO%ap`ZVH@?L1MY0x
zkbUt#!;QVUrgrVBYZ|7n7iYg-Icfi%?Irg8dv032HqZ;4-^l!0!1TYq(*8*2H<x=?
zF5G{r?R$7~#lPw-X<I*q-H$%i-QNE8*_M}{uOw9+)HH60TI;cgADH3Jz4uq`$!XVT
zCHb2+ADk|v`sd^v?`MzV4S(L5f5usMUVG~OhGOgX<o+THt@jFfxgVN))~rmv<tcsp
z`|*UTW$RujedkD0`(Wqpx#{!rUCYibZ|XBx5IjTAyQs<N>Xs>Pi()%?%-`gAFus{r
za`<^))X%!<&-1U@ef|G7EG+I+{oK3q={67J=32iDZBz}szcjb#=ajt(k&n0*^8DSR
zGnd)i=Itf+EXiM6^dC!|SmeyXQu$EsvykZNXa6rx++TjtIJs3vbcKi`AG6*)zn)ib
zKdxW()Nw&_uI>z(HC{Wa-|mn;qHg*ApKGyTYhfqf_u|d>lU8jEGS!Xe`}%Zdo7J0-
zVRmteYSr_mU-33md$%Fs<P(M!*DoBn!^rl}zsJn6P;6?wyHR$pr0>~TG576ec1~6^
z%)F$c);HfHhu1%6a_T`DzGo|rt6%=IF+E24?7lg?7tJ>wcQ=YHG}!2VSf@IyZpG5q
z^3TimMQ-r^dUuI{>`uW3#aO8)E2Xxsi*0rI@6dE=y1ItJYQ_iwFBaCcpL3?YzxQuS
zXmW(;WKqFl)gFg~wnuJd%-(IiC3n^0;tf5v7f-1Ru5u4kH(s=P-l+p056?3&So`a6
ztS!I)HW7u(uf?)+AN5E7bd3GJSbuTytpNFb(rHW92~JvnW5%}qFOIsp+PALk_VwxA
zk`n0SbeQMe5>}>0h2^Q!6GK8d&&4KB_+t`cULoVMPIB9xjduzPst-DHNhjMBtuBwY
zUC3rpeP){K<@AZ$ANOVH3zceRf7<)JnDboqBJDZ;XLieem3r}W#pK3IR@ZkOy7+qS
z9IwD4i6{H&gs%79mN3-sW$v6Qw<hqkxBrXJa}Ek{yes{8Ys;hv&6fQA@rTxH^9c(J
zKmPao>-G4VmYZ6$_PA~=_#1E8_kHcvu8XgB?X!FxzOm}9mdV2Pf?-!D<nG^>Gv}uk
z!^Hc0kNoGY?cV-t%cH$pUw)A=p1aW3`}y4yIi*aWkL;W~qj=@cnuTqV8M6WwuerFl
zul)DF88x!seP>-c7GENm$inqb_EG=syPT{}(yugoovtuSZ`@e6ZpA52k5JtkGC5Y$
zJ|ExjmCP(#TU568W9{CTvF`hp3Z;f_E}CbXrKoi6T)euDPDJjbsZ*!f9Nt`1vFLN)
zIivU5KQm{3*fNpN%0eLb@zVm~*^gxNRCSay8yyr**i9%ne~YE%#fzD?$Nbfee`d<F
zu^xWOqkBN%bAf|j^71XKPDcdg+bh4H*qmi+)9&J-D_bSwBJ+)>O!aQ$m$xVEl2@F!
zXY`(DoHlct`_U@9khSxQ-x)3GIhM;3=d1hR@J{AqSsrekyzQ4|x6S`tB_#WL<*9`?
z4<?nyE=)QfwnFN~jeE2ImtI@A#CqnhY^(fr=DVkhX8bjYy!4!#<tSUR4)^{WP22uD
z%wl;e*L`6}y8ZY6(dzn7bNW6?IZO{+chlkZCH;#0TXys0ueKhDIk)uct~;v9`(~fy
zXWo6`>C=KRcb88$CtUk~Vq(zF!WqB+`|g)N|MBnE?vLvP*K?{VEU;QDS1MY+A^-M?
z&L?j^natdLJH~JKf=%mo3uIqx5!Bd`Ua-m7ZnB8iw#^E0jC0<c+G6{1j{44~N$(zP
zuwD{$xZ&Tul8~uuj2N?c?IRNp`&~b<J$25p1q<WUnhiCMbO-j9J}B6bFuk{O_nMmh
zr;kmGZkaeuYwgKJXF|27>Rygn7TL`%SMWAk-(}fM$AfJypQGfCW<9LhY`uj?P*6+Y
zN!O%z3H43O*%Icf*Gfb<ALF>fam^@mg89<I4NA8TPD?iXek13uQMT8FS-d4vLZ6=d
zb$8w0GkM&OCzB4O#NCg+Wxagq=C?hEZPPA4di&<gCLLGJ5^IM=OZ4QOy_OfMUzqeK
zWnT7@)0rGk6{`>ZDVpBQSy)!?mRcJrTg$vjYL#r(j&sv$jvf28rAe|n?%sc!%Ns3c
zue#vtJFocPgsXq8C*1xkVU)7MuSorCR*d_(!2cK0SZg<2vvr;=kpKMf+a%is3o43}
zuEt!`@{6lB%UAY^UbxfRu_r11+@FqO%Wo?$EE4bc()+1+^6rZnnu<NuJ@exH+YioH
zYTxI1zbw%t`+8s5+gX98C(QZRhb;T}bn1S|v=!erRGSsu54SH==}w&Pbfn3W;meN?
znprQtn$*`!S>@+um|t2lt@v)wy>mK88&%4aJ*H=^%jLXoY%6dvzeP~vNBB*d)Z*%&
z{W2d~YSvk2gxl`m+Wr5;t`Ft^^j06=y?V0bh4a&A9`5mZ_gqV^{^y37@snciuJK#u
zVtjDsHQ}usq3-*2jO%vj7aRXQE0gwQwH;g3YGXdp)0IYNSf_k%Dfhf5r6pn8<-S5*
zLzBzg_}kj0&lktp^}KxGyIJA-R_7?G7xDL&YW!q78Dg^IouS@6O{->y3GCh5Qn(gQ
zGcCz@e|F2meS2<Sb%=~iyr0Hh&*07%bz`kmkDchAA1kJ=zT(p>AiMZ*OxMwm9^JQB
zWD0BTe(KztCFk$cY$UfcP<eXfrqaBfOA8dN(vzAFZCe~<I<tpU|7Q5(X@>6SD!dX*
z-oH8=?*Gf+d1ik2PtMZn`HgPJOecMwW}EuN=E;J|`sR^$BD<@<d|MXX;61JP#be)6
z*@R8P>#k*g<(;&0ji39klvDF`er9T{-R3T>arxHn5AVa(z4I#V7VVt2t4?TYQ;gr<
zE~6!}8;xErIK0;GNtyk#p7p6S1)XbuR85!|m>8F5tFLlq_Rhn-SFZ2;CVBa4ch7>Y
zdarhGkJ)|KAhGem{l||QQ@_mJ5%f3ZWV*lZ(=59;|F%?|Vc9nSyxH0O6$gda?@W8_
zH2?KnRYf1R<==A3em3>hJ9h6XT{-*ZdjI)~K7Z<1#jjY+=ewMG?cD#XwQs)N-QBr&
zUVMflL-R7f;#95Dyz~PvB#)@wJ~+|m$oKc3f7WsDl6x$9TU(?dsF&TqnYZ}vr<)9i
zw$DAlc*kaQ^G9Yo<_CX&<_9|KGphOhn;2+(;mCbXwTqSiR8?F9yS6xedvS5`&tI?C
zXZ{W2X>0I$#3`;EC-=ZxS>O6``fTIYM6W%DS57p)G;`D2^rYh}->PL#=YC`JN}K0c
zY|+`Z#X$F%V}Pmb{IW~W3%;n#-}vE_n)Iak(=7e(E2%c$Jojg^!G^kRn{UYI+<d<v
z{`8xZ$3^R6Gn_8pnSFd!m6TYy!rXp-?HZdC?VD208@x)1Z<+ZaB9TQ*yv#Sx<7<(f
zjAhZ2b+N@KCMcenYd!s&@S}@TUU#OSziaMs`o}xAQ)%b+WEO8K^|t!F`O4=h6^okZ
z^(jV$Cq6#&JI~zs{}$m16(2q2XH7k`>VeA9$c|6fF79knJO1rr^k%7QeZCJ%XXd28
z@lRX#?4NOXL)n1^_w2TCpL!{7yUDm$^SB6GsriL<P3{>HLb*EDtmj)^dU%%=e3>$r
zd1IN<WD&X7b3(t(u}xi^R2n<4NI+r5o@wE?Uay?td(*pT`=_&2+Tsll0`%3koz#0j
zXU)x=rOxWk+isevb3a?2s-+mn(0af>?Tm8Ot7(^>OjvoGKXS&F`|D2q-ShO9wISE!
z>XfYR!za(lihbezViJ6QcG=Nm>p6v(U+v!1n`U`jRQmq9EqA`VJ^0qY>hhLEjooE$
zcV%CHbAP{mx48bZW4+QZN~}IA2XmZR9)5Yp^V^ZzUe2F;?%$i6_GdTlxw=F@d+MBB
z$JF<`U0S^?UF>{aAveG6C#m^s>>Spw+WEqBW#AQF=}Gy!x5%mq86KN`?e4u3Uw`+>
z|2cHs|H<67mnRjp-2DH$Ei+qbb58xYzcYK@h)>JicG>C6t@%HGt$p8dYKmxFo>_yu
zde5Rv)%YO)HK&3mUwGT_%5dV{`CkRI8}lbmRCYHie;2bZcDGwX!U6*(*6j=Lux702
zo_u1-n%k#YGFMD__oRQyo_=1=O94*YH$?l?HAJqg|LhW;FULF0viiiCrU-RWCY7F5
zeUFo;tc>+_|LPF2_rj4+Qx4q^UaG(=$~$4k3l%-v+qFMEZ62sEWIq>M^lYI6SBs!V
z&w-fZVGV5x`5SJCCJBA|Ejf{cbCSkF-4pv<)wjQMu|D;~ljXN(#3j+BNg^uwo$R~<
zlb48ieG_>i#Za$qwKL(jc-%TaB^8HVx-;i}T=RO`bcNH!Wx1Dci}I;y?A$+HMBPx+
zX&a|<z=EoP9VP#y_(Ql(l`i%AGO52Z?U8=Z%&mR<w({QDlc{7I%e&FvQ=#g*k=^{-
z$*#ZCRzFqFlke7?Ia|$m!O;jUf#iMO={hR$`#W#Gi`pz9q2aT;`YMOW#2rli!I~#M
zQ}k{N3(rxx_vprs+b5#JH#g36zv{C@IE24ACMb+iV{Z72%ePN&{<dVo9RGqMyZ-k=
z!6HJzA|D^IHr<wt_VhcaE}G)sx}(L~{M@HW(dCO|Z>F|QS|>0$uHt6!jO;sdLW}2p
zS=jgR+EW>ZEt`%kO-{2rcj8#i@8##c6xELJdUt2r-Jkyy1w`UgPK9%w`Z{~|#qNr<
zNBURlD*2peu`)lHulhF4_f=@&ho5hDW^_ATzApWj(@4x@{c^eg2NWi<Zjn&9#=`Z_
z^i@dF&PiKRxA3f4c5>kn6aK3obMtLgZbeL0n>QUa=g21C9P(g)P@R*k07n<MIA1>d
zgC9MnZ|v)hZ*LdVn|tTnj&1tM5<i)?ew|d2&=ezR+}Kp#yZE`RO_(Ed#)}Il+xhrf
zQ#LbgYT18Kb&HPqo0{EGDLuj*EKN<1SG8&haQvGcvgSV1jpn5Ia~IZ~zIY|aY5S~D
zg||}KN=ghI)!&S-@n-AEO>>ZFY^v^`d{Jzf)toQ+Ph2ybpBVk?5n3>z=J3IN_G@&y
zN;}w9A@i&Yf)*WA*rum>e$M;)M8_lVE&Sa!K1&htFnwY8L09hm^<KA7?@o4A0k;JY
z?mczr>18%FyWp8Mlg-0RGe-!zMX>32JNv7Wq&wT^N<80?V04>Nmu+#}6l*aK<$!=Q
z8#Mp>iP=EreVZP2tvQ}?rTe*phDnm@4(MF4gMz>v_l}tdl6o#iD?Ko~_n=<+PSB&y
zu(&I{R}!{adG30aaJJ4mXhK{2>lYgH)6!4XZJZaNAn?iegn6Sw!ETEUGtS)LUK=ZI
z@T$8yVOi&k+-Qr6XHxt2HBMO>Gke+Dtb<>@Wvm5`9u?|+d38<G3#CIGB7da44r%*K
zt_*!qweD1|c-;Q<3nyzVF3dh<)v6*KVs`3B%d3|kH#jXi8Jd5p=prYFswdpO>Lv4j
z^lsQEFY;^0v*fym#}!MvH@>ZIj6DDS$!ccaibdI6-q+<0Pbz9wIoiJI-?|HzpXL8o
z>TyV9;rgeW!{@+$V~&h{uDaey&rbG|EvsIAvb&jo<K~Cy%yau)JW^$^GRLNhcm2M!
zJF%80<l4cHb6@&cEi)*aFp+JOuxjw$U%Qvdy;RZ464hJM{ZB)H<EPq_*$yA#m)>9T
zJ>&iovwyopuGQLhvHs3oka>vxviqk6`;Xjz-v9aa8vPq(ml`jwwz$8tId-!A^VNUD
z)c#n_J)Ui~a{j%xa&v2q%R9uc@~?d?!gXx-H5qQT-Ete>1WGhE#h5HTx%u)>@4qj^
z7ZqxIUQ1y=ofI-PWB*h?**%qNODgo4r~XNinmUC+{oA5bUtwzj+|TdavPjeW(u>I8
z!#^+k{jz=Qd~^48DL4CEl@I4l?sb=S?SngBKIM)7ZOJ<;w){GINHbn?`stG%ch!1V
ztew6hc)#FVxy+L-ZMHij=H#1wRq?L#bDp&OV?rvk!o})OYFTr4IK6zO`ol7ZdHIyx
zPrEj4s4DT+<G+1~`>S_XU&N7)BL+t$@|jhRN8RNTno!E?yRG1h?(f`&7JtKi&%H!b
z_Jp#kKK?hQL|uFF+dmT*6<n?Pe(Tb<wT*0S2AwN(8XZoUE8b^m`mYeCk{d5qDBa-j
zO|)sIeb(v^H;Uc<`D_)xZSA+WVoGk8u&d|iJG1R3SbcphkfP%`wQSe!Ytw%8-v3zE
zGLvDgMopo*fN|qdvDTflFRz>$^&vh(_2|J%%wp@ZRGpYM)rn7k5|yn~>1~tIz!Uqp
zXp;I#r@L832g}r7KN6K{JrQE+wPdNLjNR)KTlY9G{bclCe1&7X)1gv!Ykf`2Zz~0L
zf{(jDdQfOB{waG;l)J$Rjh*IGIVP;&NYKyNG0Ek#0Iwq(=j>QT)0;E1XB(}%zGqV#
zmm9BH`=R@sd`iMLzqe)NB^2mR56}`_u&`%7Q(CJ_;4>#sTA!*oH|u1?rwO%A`#4#?
z+wI%BYjZ`Mgk!74vTv6^|9H2l@5z%VKG9!)tj>P9VaATU!WCRQdM@67@hvmfzJAi`
zA_LRYuRSb^yR<dSc<SmxIM%fonp>Zkc;;uYxQEg0i&9K0Vr-w+ot%EIKrz(&>8z=r
z_#c);7|c94C8;pqYiH)67U3<&nACS|REd9fr&Z-bmg95Zo1dnBIIlb9a^j<)Kd$SR
z&A6DwJ4GkD@l)eGvA8?7)AN#_F%^7!|2je|`#^$={u{?c7On?>GTEPVuZ&o0o00y7
zL7BHFa{K$+bNqf}sxL@%h|fQ@s`*XDl$l2ubiZn4Yp!JW;fqoTHoW}tZ^5S0QmN1S
zD_3>QKl3B*UdVa1++XUqe%xO#_UOv_x`S5t-oBc|W1AHzx$V-d^wZ7Rm4+La>UAHw
zym0kIzO3K%u^CCBb|)2Hfu|$)zDoUWaOGLTVK(FaDM8b2nV1`F*e9l&uu=0xsov2w
z`EzEPZ@b0GclEPV?fDCu!N(6gv1&Mc?#Ik$e|jSu9SY)4Y#0Ci<HDLHEE-edlO9c1
zX{izB=#s6wvhscS3gs`32V?%;KfEPk^2zGOnTM`yHO`1W$)IAGU%;O>ucR_mO((+f
zX7}%#o8EX&bG><?w7ye(%1xi>*E4@jf0B3R)*nsZW}D>CU*5HTk@=W;tvmbHr>m#@
zUxaLBVffZ<o$>L62g9qL$ODrU`Jdi=cjfVZfp1Sz&a*u)Yb)a`-d_K5-_0xMu6Ow)
zJ^nsf(zrCVFMr<O3cdAq*JkiU8`OPwij(~)xlzL;H>&&ZZ+`}Z7oSu2e9X{TsV?^M
zM2G&}<Nv>WTKtFQ#Ma}R%QOG)-Y)mizK?@_@B4rsS&f~|C%b-sJ}0Uc>d2&e_uv<e
z=?Pvov(AZmF3SuJb!0L)6=I?_Xa9=_tIl5Twfgd5;a0v2nA4W|`ykVnVczUNs&~!b
zw&d8;ujiV72#49lB|U0)F7w}geRf5DXQzJnl8vVFtSbwZZIh#3eRNB_|ICD~eYsJV
zt5ta6*6rT>^1oK+?OTu)$Fp+1YxkjZS6A}P&wcb)R`2l^@Kl}RzbRAwugyLw@IIY?
zrsVz$N2Z_Fb#fHqXsKaV@!FOCRb|zbv;UWeoiJ~Fu)k&AyUAt7PdlwvajbCs@$Kd+
zMvfQSMsJ$0$GDxp!n<XYyK(h3ah>4Q$+>4smaXMazOti`C*zfnUstm1`O<l3SWPXI
z>K;%3BJ)wx_Wr+#doLQw+}YTXXj=N<&bb%!ns3(MRb@VFuD3I~^wasvKc}7l$Xh#S
zZ*_h8<1-ucnvH5M%hlcAJ$>%&{R`8Ud9=<^G0M(hKjstbdVKow&(%IWhK3h)=9x$|
zR6b@m&hneG+V|t@mu8QGlT3R#`+rQnxp}(q&O51R{+YbX{ibr`Bk#+@_k1^Js_#sB
zJ|pt=kH6pd=FGn8EYWa?L*!4eYv#N6VcVN7t*N|b683i0$%e_i6Z!aLwix_bUb5XO
z!GD{v<((td*_R}(Z@cuIygaq4<s`=RpXG8cy<Ew)-ZPmV`QPZ+`S3_mnb6nWruD3c
zmkCT?A~5}pXF!3e=fj#wPCiceN)M{!&t&Y472i{H<4qV_u&~UYowpXX?e;EtnX|wo
z;loFcowHXx60JYw*tIMEr$g^*-OeuSkbAI6L8V>O92A~NFW6vt%dN{-QS;)lP08Cm
z)+m8$*ZIaQ);yJtqMjSi#8xVt-fmcZT{-jjCzsV-HWz*rrs+q%Gplg=?VIxa;mRHf
zgNDM*<)1DtUT0Z%^q+BeS!}~>LmxS7e#<Yu_iC3}{xf>veYIG6n#RUA782{F%4C0L
zB=FDLRzK&Yvh|aEhm&r9{`OAc`f#R(ZQ6(9FW&nnEU^FSnm8{`_+0q!wHZ$FkM2HP
zd-e4Rn;W$z#%;D72li<?N0cAyv3=^+cAdB0wCw8kDp|RyCJT-fy<5eZV*9W0-#gQL
zRr?uw-0GSBwsJ;H=_t7Hd&-o{Ws7{Pq$C{U!`$QTwND4fc)31S;Qi*J9UbbwW69Sw
zyxZ;-q<CFfV30bwRlfT5MJC47ttNU8g_1%K?cvOS5*BK7v><y$EL&L|Q;y%hm^oo9
z%A~m}=TBXgt+{erX}r(1ude63%@$4mYVwJ_$0+wtLUL2{>NH)uG;8Y+)s=Cjym5<8
zbFet=6Ls1rYmxErfMfTBe5QkcS0%7-PYNo}KepC09W=(YBtH3-q^iO3O-a83o!omD
zdbPdTaQiNs|Iu5e93}IQ_8-6Zr>!!(UAX(uzr)XWrgbFFe$?_f-k{(`nNd;NyZig^
z*Iv<YKlia}`Kc2{l2!Y^O77*p`0mj1Erx<$Uvn|s`kv=IbIml>v%hWljbqtfE!$gY
zb4hgy<6GyQF&lSRmwmkTl3PXNr{ye{wxmQBt}cPrr|fG?G9I&fY{^`g{8LTZ^i{@3
zW<J@I&O9l_teTu<emB2dxzxC=WzD*+fgRR~kF(U~9!X=Ld#&`3Xw2)BcWhs^p4#8s
z`+m(_x360pT;86Nns)q#<nPI^?^&B(y}WrLpZ&#17AD2}paPzy)ye<1;^Ty~><{jo
zTcoSL;=1;f*@o4Tt1W&mD&BjCe{<SY|DcssU-Dlab^on9$42J#%7$bH6X_pf&QW1U
zmRW6K<CE#)o4q=0)jE-HbClkmORzgHAGoci=OMdfWbpAVlg+lh??3p*r~X*&Zk~qA
z``4?0&MOIRnW*05a4=T&-;Um3-90PTG5=p>b!bDNxeGTRdvp@Fox)w$xo;Du$Ig9q
zCs6ydaMwX&k5iy%&#c_Kw(*jcN7wdGA05@#e>3Aqg*TW35^AzbpRHXs|I^Pp?$!1Y
z2T$+4r<Qk`2Rv@SbwTlesXaBZ#%Yr-a_j4dS7a2O6p*Xi@zb_&{?FqjKZCa|;!)4l
zP}Am9`1+@AS)IWg*5A2bs_XfxttEaed24(~V$bOg)5h<nT>D*b--%rq9?s0`<)t!z
zWBSUop#eIcepmnN2HNuTZxvCvy!zPwi*Hq4g<N~`Tj!jr#megr=iTS>Ou44%;=07J
zYVXuw&8`pAK&g`Rr<wo<SEu~lxT>S3TyO0+mz?Cf_GgaU_l0kbT5H7SsQEg744eLK
z-i<%vCoA8S|9cU*?*37!`H~CQ-2Q(2$?TeCo|A9gSR^iZtyF`1(W(zuUwPbnf1Ra%
zai7V>NYI$}ZW+sr$6debcP8l8X}tNn*x={_6|dFj)}%~UXITZETi6bsTX_CXw6e47
ztH;4rSBqC!DW;m_slLow<#N0#>zdT&=A*B28k5f3yWP|4(&T1w+6Q)b(ZZ0&=WNt>
zJ~_lMd_Q^Zm5$@`>%uyWRf@}Id*@x{`6S|4D))a@{<XQMHhw?*!nLIKQ>@;s_d!PL
z&Y@AZueAOp+}}1)_@vjoxBObKPw#X6&+Kr!PGRlpU%SdKW_>*QTqtf)#2(P#$FZ0f
zqI>=7?54b2kYXJ@iL0(;ubb7WRnH^$9zOI?x@&hf?~C`o?xAs0IVV)+H}GFgZRNF&
zJ>>9d+RuW9i60h%20pp#>-*+#&9=U}`&X>foV#;(`p1X4-)~s+yL`pNpgEJy--wvc
z*4C5uLdEGyz4Y~o#aHe-z3yCE_ki)b!76bjVYQ@HpzKm08pAK2mGUc??ce*dcmAMi
zR?c<y`{Q$_Jia+|eKpUl<MW?5)~`GezPl_zc(Vxmq;Hb*vf^jH(zV{bKl)Fa{-N!2
z!(5wMFYgw=^l7Gl#=l?e>r_mpai5X;uI1dm@6e^@aPG2ltMxVSbbGZWOSSIoewq32
zheqfV@KCqe;*#B~{}!3Im2e)L@@&%LbKT)P*1h}uJ>K1NrADN$@a{+RWNpgcNikeM
zKj+CL&x&O&f*L0p*UxXO<F9mE<IgJV&+5H>&B~&b{a0hFpS|(4{@KYnpQ}o3%cY>}
z{);M^_2WV>PnN%*oVc9#+51gZdrn!+Wc*t*)AG!TCw1!{otn-+`>yL`?a=E%H#HVs
z_iVX#^=Y)af%D3j5?Z_e=ik#<U9u{~cUjM`t#4)rmR~qIeYQ@-mE<bO05hl5#pAPI
zSsh4GubJ#!Xq<jREHdp#fc!qWl$4Me)1&#?S}KjpcTBX?xD&FS^~$Bsf%@{tN<J)$
zlr&rd3U(E%E7ulX_x$lA`(?~-g(62#xX(NO@WK22N)e(9)Bcs_rN2DItP=gn+{erG
z^|2GP^eR7{T0C#j%bgpaZvJ=X<~Q-{zG8ZA>yvIIgkSvfU3{N|tK=3@xw7f83_spI
zooBM*`s{n9(wB>mEYEO?PkOX@-J%aG3wQs>Ive)wegA{WmDZnmuezA3B!6?;^*;CX
zy~~mNe0%Lr{hO?(Iiu=l{`a3=rg8^*ChA`K#;;Iwn^We%Sz~Jn`v=D1zn|}S<=rc`
zIBM0g${MTKwtYui!sV79HhwR#n(dCi)PJth*Zq$=LYlI6e2cY7GrE8NU(+?)Umu>A
z&VJE;h(jdh*~NADd}?pCA6se?8TVc}w%12GWk+k{(<ez?Ocpmwe_WHBrd7(*Q+w;g
z2Lt^reCofIF3ni7Ox=HP%Hz3b)ACZRdc&q2c&L))Z<YW4b4}sjU6<DeER8LG60L5q
zqfNZHUHWCrLF43zb$yLH5BKu$Uo%*-onOB6(r(t(b^*taAFh=Qj&}Ci)B8Pa_RfSL
z2ldPA=j#`@OL1y1Ry4eK>IZ9DY43zX{ett9PQEaiaz*n^kH_=x*{_mPBbDBE&6U;k
zQ#x;PhbwyNCaFh!OFo~?+_)kV)Y*HuY4@}FPH(3EdD=7~pUKhiK+T-(a#<ej>h~42
zg4rIZ3b$Wa6v#YH&iCZ=2M$@LU!5*}Tl&^{X@2EOO?`XM*J-^KPiB@|gs$cf4uAi!
z(`u>F9*Zqixn{=4ZC_eCbgGKA%B}ZLJpE|8#_}#9=a;9h1n{oh=9~TOwTZXw-mYDx
z(uNh6yuNSyFugi(`#*tg@kWA{k691x>Mgln{e*x0r8Rl6Ve@=Sx4u03{>RVH;TcUw
z%Jsi5b-cT4%NxT<bA9Khef#AxZ$&)Uh09OBzI->UqwjC_&##Y~Th`w@v3*tIYa2<|
z;}`#~$kg$F-s!)C_tMJ$vyb1l&s`d=+Wey1wPlHh*F-knemUD&>-YbgbuIsB0>=uK
zc?E?I>Ywih<nLJ@0qW#!o?LOd=Jy)&I)_h^-1+&Ng`?b<b|3m!!zJ0ylBuy!_5R;e
zEEyahn-70H$5gT6pJnPFm8QQjOdsF7sEZesYIt7JeZOn}3>MYXW_x2FCN!m=2;sUE
zaFL<2)9=wG*YI7l-^{u*;nmX`i=rndVs}5Apy)iqx?JzA`jN$^Qv~!KPi(LGHaSB?
zC(*M=XmU|`Ug(MK|M_m)ukyRxw(!lZt*_U`Zm$3TPbY5A3~BSc1(#nQ@z(HqlU}(x
z(f`~YyM9$?xl?x!<Xr0C+BJFJe4WRO-#y(~c9tiKdFdz3EAI?ci_fN%N!F-(JzRf&
z=W<oc#_W|*avJg#<z8+Hia&q(o+-PK)yu*rRebk()gj*#pY~?Yj?GrlxNNxWHRm0Z
z=KPfkf4(l#etPW0MCF;*<$BlG#VQ+Yc%a$t5@f(QH>&y}`y1<$*P3N(#b@MjvFqQx
zRW2~SAbWjb(xk~pmh3KH->J#fskuFCx06rb@6)-`pt0Y$62(S`6We2+oH$tVblttx
zptj-OlP`I>BUW3?{N-^s`rYhlS?<&Ra&mpz|8(D`i}Mwd|NVaXw^`$Tz4`mF{`v%g
z?HQttj*hH1*p3}o5Z(5dJC8q+<z72W=;e=b(fTQ?Cx5;!vWormgq&nYM-j1&a^VV~
zku(7gmdYf%{nI(+&#36Gu;c$f-@IE*@7U`5=3BJl8MfZ`Uh#|xv_3#U_tlFJmpvCu
zUYW`gaAViiN$EEx8#)xJH+~8d7K#z*4}_U!X|eUM`n5Or`QNA~@h-?b^d|7_XP)C*
z)R#|Ku<qQhil0hqPDkHNf6%8abofsD8{4el%_nz?EK#1A_DucB!ho$t23y+?&Yj?}
zT21B0w$%~(3IZJ9reV{Yko?akzE@gwJWPM29Gq^#duFfBk)s_eZLZXHSU)^vX!`J!
zwRe9vyXKC?9WxL7p3rjHtgZd_iWlmQktdZa8&(CpH4?5?5I>k2exz9h+~8qhax~*P
zm^(xGv%1=3NglVJpD8ITFVu;!^V`kuZqo}XX*6JBE$`sgWDPD6y4>od0&_}}a<Yk-
z&Whmm;+gua<@1&J*-a&4zFO=3-|1r$+vrf>zC*IU;Y;@NyPuxrO)T82-Ci%O@2DVf
zM{wQzTb)-+zOFmHoyR%l{5;!7svImwTefU5`S$KEcf<7Q;S2k8KUp<?{P=N8?)5(7
z<^I82-@d-`;?u#}zeT5hc&2=RA904Ob7u=^F89kym5qNE*xsL2dDeOB6^;uFoFBfr
z&lmPA_`r2@2j}ThMye^Cv#*|%cYk~2&Z2w4j<*9VRIE%dEt<1^qw2yH^Ghzyzx_g2
za6<07JBzLwtZ5bxe0}NWq_XO_6ZV*vtkw+n;<pg-n^z_<b4sqnD;2G@|9`h@#|UiQ
z<zcgJ;$d#~g_A|Y)<if?*Nc63Ypeep3&-8x-tMe<DP+2<#^dU?^_}8DXHMkCIL8^c
zvrg77k5YSnz3)z4@}!_CyMxL@1HQy`#`&F@^zg?IiHPlGx@)7h?#jMySMow&ZS;1w
zgtxc0n&jR2VORBuPe;x8TTb*=HszgD?j0`ttHXZIBJ<9QD8cLZHWxg5Fh5|cU?_99
zS;YDmx8D@B?oO+Vi_)$CzjM~wlf06bR{nas{czFT$?E4OOvz(kD?c?WWO>atrAD#I
znzw&nN-}u$K1laSHS_ALrylp)pS!j;y7KSW>obi~y%zTAe=4}};6Z}Y_qU6qx97>+
z`DcIT<n+o7@5?iu{LWvv_lf%FkC*RR)-5>iw#-oR)=W>$36ghibstMjEwH#zeZeng
z-LaX|IPdaKxwf;1eYeg+)?KmN4J%iw9yjzaG5e4adR=?l4UsQao!@#z`QG|GpPhUz
zX3FHIt&htxE}!VW$GcZU^Kj(uW?j}5ia*}Zi#}X^{#St6_Vqb@)0W+hvyL(`HQ2Jy
z?DT=``x5i^Wu!im(hd)8&HpmlO)8~ZX=BVrgPNtizh`p)u77l+>pBMqPr}<-rY)_l
zcdx#_zACi)zOwSmOG~-k&1)K-K6B0b`a{N3xhO|e+^Y1$vyEIg?@d2^@VN2i%_nv@
z^*p!Mot%HQZ-IdRz84ejeqOzyYGLP7%}24~=I7r1>6`reSn#$OOS3ic{>^N>?_PbC
zHp@A&tMqlnlZoyxuC5lBHp^KMx3}v1@x%VMCp$LOy?de~*%P~C_Q9{-RW7cdShwlS
zK0M>Ut{Y$N<k-`z0!>tZ$Q+UHN%*Fozv0c2xf{95gH1Q2T%7ZhZ|l$ZvBobxJo>(~
z?)$kzcemf|>)x-Qa;12_;Ow`tUuu54+f^*HtU6FrW!hW4Z?11{_Mt_!n^&e!dDpF%
zz5TYNMS()}_Po>^8xm`NKAqlbcU(47SVBbPNT00ri|gy<UtiNbH0zf+@4T7)VehN!
zJxi=37;U)%1z-AfmnL0dH0xhzxT?SLSgMhxiXUrn9^2CGXO8dXox*ZSFRC)LSM<V%
zZwm5Rtgj^gtMtU?1om=FU-UU6RI*@Gm8aC{$qUk$_3v?hUX+-f?-q4w=Cf~M;(KIU
zm%6R!yMFt{$8&!ox>kr?pKDXT*KOAwUWW%YTTHu)!Y`C<$;f-q`fC+aM%V=2N!1L$
z!aqFvdyZ>;&L&1KdmX>hz2(hkcBt=N`C`-Q_t}rk>QB8~J3n}*m#A6p9gnlKO#e?U
z3t1J?X<Pm6!sc}UvbVn&i%*HZvi+&N>0HBud&i#@Xo`K4HdE8tHetyxb>CK*^W|Y`
z;%j6%3&Y&zZBE+cI6I83?Mk7i$n1)<N4Z=BEp|SWJ@<B|Uq*(8w0WM2g~gBfOz+iU
zYk$1me*eY2z1Fr>UlKk)J9}kC;N$)Og|-MePGV{KaCOzrl2xx>|2jL>_}SD+H!PBs
zl9%OvJJB#H&eiC3$>%3x$G+Fj&FxD6Q=Qp;Xu{6o+%}bKHKy9v^HtV|zYN;{BzNoM
zN9$JFERve$dwaXOX43525;0$IetxjK$G^HYX~nhP7ccTo7#(pxv*5P&edc4o)h>2k
ze)*(TJTBqvEK{4WSAtvG+m$_A-y4^osm(Q!iriVGdfxWC%ymDv0);)=*RJe;AFWvb
zZ&7x^?3@&?%Yi$JZXWxteaW)CJN|8)?3XDl1r{@Q9-DQM_eag<qo%TPLC06Hty5I}
znfX%q(XaoX61h)Lj;WsCvu)GcS;7-CF2$X1-hQ#~P|`N7H@7FBo8$aVseW1cyE{Vt
zrWYE|9_%{%@=T{ky5q6?*G%?VzSz-QmhmDop4*Jwd)gACKkCAV5BcVFeV=oB|Ej)q
zb``%md4q!I+}Uy7)N1)*zcZTx=iK{c$t!L4<ok&e^=BE*EJ!&yDd*mvnd<W@np$Rr
z%FOvHvF(e>!n0z^Yfk#<rATGBPSplQ=pI?gT>EJ+MD({>oHF0tdbqo-DDQj;=*G%>
zd#nGOG8;6Xn`^zf;$zaf`2FV=_uF~Z)cpDT*=Y$^h|YWd6DKy!IT`SAQOw?dIk$Gs
zyXd>O>&v7~-<f$L_RK3aJ}&n0?46xv_j~Rg2v^VD^I`wdoT&ZtuC5N>{ogE4(&Ity
z*2hNPalZt2-t=B`<G1?e{eP#48csY@yMKRm|JC>B)vhhtai9MLOQ=Tq{gb`n@wHQr
zc8gyIUDJAYcKb84#S`CU@JtR{o%-y|%sv1A{Z@&$I`jUrRianVUVGKrb?d}iQ-9~U
zs9AWPOrPfdX<Fv$HvOfsTRyu#lb_&S*0(yiEcDe&4c?Nhb-{a&`+ajwxBd7_B6Q(|
zRI6(jL{`aePyI7*o3Z|}gBzbL3J-X9WoNT<X~=cQPpsC_e%3bxUnh7Ui=T7(?M;0j
z>n7)|Q(O~Y&nmg5T6ypPp|3CB?0Z%sJ>Sl#$;V<%<{!)dntHM`{GXqjdvj;8I={__
z1GX2<_Qsvp-~Z>3_WC`CdN=>>nDvq4nMz>Vo{G~?ntGpG>rdv^zkkH0<m2JIeJ2mz
zto`x(!}>2TFSqMPZxav@DDdC1w*37)(88H+as5kcqs{f=_tnVnopHxZYUTd|<5TQx
zLNB7XtlFHvw8~_4aN`1o{Yv{LT3<~Q7Tml{;pE*HB4OR1rfLOl%?kY<|I^`c?pEcm
zN8-`~d*#*ld+^q;{e9!u-($v)e~Rg?_4GWPqkrJ_FMi{^I~Hl@=lLG*la1Vz!g;Gc
z!0XAnGjr;GJZukH6ERUx*-gN7u`xeyl+VdizDM;P7*~cwswF4=y;SMW=WO(D*5qBH
zCJ9qAAF6*$ungV3(b95O?%SwGuIw)x%mY?W<Jrh+zM$r>%Bi&1hH1tTTLP3Wf7_w5
zbOKkGi-+F+YV)%JN|zg-UAdE8C)u^-tWd4=Zky15p<jKZ*Qvj4YmN;}6Pp{B7tOgk
z)bZG)jMG~)zOIt6=HLENNY++5ELf@i*|*2fe%e=jyO}=IvRKXP?%u82A{Jl#aMt{O
zz{Vukdo`bZzny>Kaahs&qjtFeo&!mJ59KDFJhbC?e$u|eiO$8pCVl#D=+-B5a{K+d
z>f87JSRZX*WHy-T(<5Qn^zQEN&4rKMyr%2v#>LHx@bKIA<b&DEuL&C~p0wnAIlnb;
z!C$_4*7h=L!pm}A`=;!bTYI<Z`IV|aSC8mCyKG&le~4$z<4&>H7p~+Sk5*>8xY#-Q
z&;LecYsGI8iHq+o%uG19;h0+NuEYJ+?|;jEJE=Y&bjQ`jMXua+zu#`JkBNG+uJyd#
z|39CnXovTy|N8p)!5O(L)meOwrMoz11~O!YPOv?7^ynP7+ZR9cnO<yk`75BiG<5C3
zqO&uV4I3vdDmck@we*3oig44~11IBWnVWv_@wg^9uj0_5l@79rsWzLteqUQIEEhlH
zre){ys^0;&-F5F1&dRaRjjGxy{QAcF+Y;NaOiET;r*ZpDxbJjj#cqGkneJASnNu^8
zZ><;mx2v~&M`gCv?9V^+Zrs1jH{1D3Im6|vmt|&MZr_rfUh(_w_Mcxa`@etrZQsXZ
z(x6o2*vuv<dT(kw+drR#uV#;2xn|b5r*2wMF{$XOM*FibM{7O{O;+=Lw08TwUFo;2
zYJL<11qVx7m+9=S`kHiiSLqauK&L%NVm#BHX2<@h*?&A@a>@CRpPP2oe0}oz-4f%i
z@ju#F*Oxfd_s{u#XPe{~8FTZ~+Z_M@p6RxH*~e2RS3gY3GAMnlbZgr58y^q<j{H_S
z@6NIG_4$vDeWUk9eN&v9HEq}0<>708SEN1rc=xQz^&%tBPma~|!z}(ko1OpT%Vqz&
zoO8IMnvLw=t-8Ik`un?_``dbj{<$rBZh!da{pTDr0~w5_|MfV1G&F@ZJeL`^7GRsk
zOj+l52j3SwF)Lm=;a$P)$X2g^n<}%d7KcTD=ul8Uz5O=F{j}1`4L)W2c+EvGF_g@G
zzWC<KoYPaj>iul#O*IVrwrb&|<I0yM!#tQjl--(Bb@SZ$>N74fTMOU0UHkn(bMfO|
zMz*e7{83xySn-_o`?jp}U)|?F@-DUaXMTB|KGQ7smTf*m+J`H_{u>hyv+-E{NG^+C
z=h7Xu(n;-p<V+onx)}StvzZ^pZkR1J!@mBXZ7xHaLE0G!yB`mj|NZ&w|Le<3=crjf
z7M_}G#4~5_{F%zpJGOuM^N916{f*9thhH&#ICyBSj_wx5iuCibYkx=lyYq|Lx8Upn
z&s+O=UfgN;=5O!Mt5RZnZ1dX_T(_o~GgQsmv#CXuJN@I&-^mqQJk{RsEw*0#drRWo
znOBtRH--PRVrbYFtG(*w(%<XS&(1RKF27gVers#?=FH2>l!}ks`Ff){?d&YUo%!-p
zSBE^i(RID!_K%4Zq+L4~3w*q|-7t54)yZT1?Qh?`yS#n#JWgBQjKWPn!i%SEHEeyl
zTr_RnivPdcD-2hyZ7f@AqB4J@f#$Do%TI7!PE);m@|lG7=d#Ej358P4>B~31PFj{N
zy>-v6Eic60dEPV9J)OnST6>!-|H;4W9ya?rneFYmQoPfCetNp+%O&rltNy;dy*>T=
zx>y$0sn0$vdTz1#akS#(7N32KjUoeTe*FHx^>4;qY5kiAca^@DFi3FNnEAVfQ@G;q
z*Xx}<Jxk*E*G1@veRTRPaU@Z{?nLpVY=iax>g;BlolCrNX8((yCynN<_<z8*GVd+3
z?hKRU9p3z#^wa<4OV2c|V=264r!(_|N~+OY>4<$=rLW8%U#xTfQFBhQYz^P@kF(~z
z+*z1^MtP%#{7TDdqVeyUSF*+Ly(#*)_S#}G-KZn0*YES%nsxQYuF~v@Z7Kh?8DE%Y
zT~R2yZ};#0|9}0LZ|_q{d+BW<Tc&8;#mW9HvLi5f&a{=Q7H?wgbDtIYzkhDC-?H}s
zo8O$2lDd9rSK2GHoB#Hj-JWwMYVtY$nMPk&Sw-hA`xcTO-gfYbaoC~9Q-AM@kL{Rs
ze*WiQkJGOR3ko{EzqhyhX{|}dg@*ge%ACBss%3wxW#Vo5TmxGk2ynbh2DLW796C5f
zQ})EW*xfI#tPI|`yInJQ*@r{i`Wtd@o3ZgoG~Ctuvh9xM@z1AYgk{bN+kgHwL-R#d
zPkPMy3)R2w@Nbbi<let4DB`TNecSqNkrnm#GP3R@JzE=cb(&AatCN3T+Eh-_Ubg7>
zS?dp<Q>JP9Rn`5G%sqWSe&d4GVQU|?ipMRe`T5Dk#U)@}%*8aPzt_VQ?LQn~esg1E
zvtI13f|n1@%gp+&!nfM<L|@E>z>byqI}5mN^Yvdg9Gc>rcIobX*_<xDSv+$;9$%ZY
z>yc&nqm_%I<UgOkwWoal%;uBp{(oI+5~9;22HNUc9LalUgJI$2zp?C(u1xckiM8wM
z64l<2|2{5iOGe<D_v#iFGxk=0zqBdU`_cC^4?Y}Hw0L`{STE(%+({>!szqG`EjB)j
z^;=UndzNXo(9V2$e%mhr8~4taG*0_)#`ye(yt`KOs^3}8@e7=#VqTN|HRWAK|HA(z
z-?zDjg#7cHcl$(g{yzunlc`c3yPju6?{8pz{j`YZ*9-3J|IfXZu@5&<T)47cH~B+&
z^+o&Ns`4V8g0VF>_EZ|5xBIQL)O)&`l2X&DsoLT{P5)gD3E8)A-<r6+vko@17r#9s
zw&>9AL|<QlOUlybmu%%k=f`Z}Y`*QictVA5;dR9+Pw$9`a^{yRoz4r|ad*jiucqeg
znOo*G%692xKYqtO@55=MuU|#3uKb@ld;9z?JE!idU$x@rwCkVpy!8!k{;c@;SDI~=
zh3ucNuUF5x^(k_i<d*+TjEsyqH#fD~)&BbMeWIw$oU78&zjzfdwD{~(Hi|5G|8PsR
z<)_mI=ibiq`uyzd&E4htp{qhPuYF%~F#|N_*l+ty;`B7#$4{rni%sOK6u6~u<nRAI
z6N@cg@A!URcPiJyZLgjL<!t^@cXuU|{OYwAPS{FkJf3gO{)RJf&z~plFRFY}ZqJT=
zx7y$WsMoc>zP`szdA51}zT~YF|9`DH618@kW%08Q?{>eRRP)ilZvH*a%U6S)s!|@U
zxUDgXW95X;Q?IUk!s)-~&-Ui-RmT?UFI80e&G7wlK-|{aH7EV@%HM6xjjo+`b3=oG
zJ{z0p!LzSY;<8qpzT`UpRiMdJ;eVO4^H-geD?0rlTtiWMNoo;4bJga#C%Mmj&sB41
zH|=%)Jpb9Fe>+Rw+?W`<yG*ol#o_(E2NM!rURv5CW!kmGxZ2`Ft@s&pJ+6gnDMb@c
zryShddUwv5gEwn;2<`2O+gr7<^mUlmRIRV`cbKmYJN@Wr_scswjf<b3dwSOVzKFif
zasE53X?3qomae=}|EzcB?{`0okLUmDeA#t;QH;&I<Zto%D<-Z#dOe(Z*0w9Ff1H1&
zHZ{9l;_gFBzS4^6Yv+H}+!T4`(Sd`_>@RO`*N<8|tykJSXn);b>*rT?hUfiy+;9Kv
z_xt_Fi<IZBkes_yvfhD1aB7RiZtFy^pI$ewFZOJi5)_}jY3|xguHAks3x8Shmey=~
z7rU~_O*e|C{%>jZI%VHVS>~+Fjq%%!R1dw%b8lbQRxhJwxnlRcO`8s#`<m&p`qqSP
zJ>rkE{%>9vC6ZgWqwVz5`O~U?$N8W1Pb<o2zL#b_x!&lxhPmOJLiuCYul{-dd&||5
zsm#XJ1z&HlvGGV8*j4)a!{^Vx{qL=ITl{eM`+dommUu4tQ`_^`*lJSIQ+*aEHSwuF
zVt1k=Cg}@*tref-_x{mQ@%WUVpPp7!SN~uB?&9gw-J;rIA3hxBH?I9vqUJY8BW}-e
zFXd0|8_)Etb^5W4L#?!Y@9*UMw&yoZUitma&Z+yqe|q)$;g!0F?qzFcy}9+G=TR$v
zxX2mHQsX{eK`|9y%|AW=6`Gf8PCI{_FD?9cdB=}s_Ki2+)n7MX99i+@S^cKxM~)o1
zusU2n$4ol=+M0!#m#w3N|IWK)*3K_~u1i$=f<=(!hvagh{@gwCsqT6fH%pIwZS~(U
z?b^)vgbmXIOwB%?V3byS@L;}K(3R`AR|ST#KE24c>h#T&^uoHOGBTCrGs}0sxcDgL
zDx0=@(%N~K_}FyUZ#!k1*=2e8S7Ei>^K~~J{C6wxW_}DyJ8u26YVyt4TZ&t@+MTWy
ze|Lv9zj*nws(bI&JYOjl`QDl9(j_$yW{b#;m&>BxPS}%bSAA>tr2`Y~|Igq1yKmK_
zsNWX<Y(+G=!~}M`wXV2P-K!cXuz|6Qql;B2yT|m&*<Rga895cbdm9xNnYcQ#2)>C8
zaS>b-C>V0UZDQ3udy{9gW<)G1YIweR+Uqz!znMSJ&awP_=HBLVxzJauFK?g7Rx6#E
z5E(zWu(tF?%7Tigdy<cOroH0zv)O;)@vB!|%a<=d<;(P6amTJ*kDfktjXP%RxAv9i
zt*y;P>y{k)_w)Ckk9{Zei{6N=QvRDKb3EK;_XXqhb24FTA{uv9eU$kY8Xi7<!UO?O
zpB&U_d~~$i`+(TL`Ty?M8~ff~T)c0GPXE&1Uv8GH$!fWJMO1%T{0!3q-R=M1%{Bf1
z;%kVP^IzM2-(OFcl-uy~|GsBkyIAe5wjW;5ceMMZ#;0$7=|}fHfB)wPo8rEGEiQAF
z-ufS3oIE?cNJ!Psd*2h`ujMg)Z=`o@eSKan)o$~g*`GBYDv9gc{hmHyg2Ror%$~Ua
z6MkLoj=%k2s&@E`ySvR_zI@5T#&&7x>Gt{A5l?5yfSQLPAtJx3tmb^(^88z8@6mq~
zf-ZW@eYO97%CeOU{%K3OI4vrg7hf56{Nl1Jo8R{@lQQ0wz3|}X&MV=ub!AKo&zw5`
z{LyCP_O4~OdRF{=!*Fq7uUYPCO$JqW{y$&swp`b5Z(3Npb>(W#@A_$*AK#4Rows}Q
z&%b`lyO!Omx%^;#__5u8`~yR;eOSMHZlJHP+P2FqTpxI9!%{TgKYdvL^!wr-D?jF~
znJZ34@B2Pmf3uUj?CfW=uB=|$`SZDz&fC?lyFYQ=GQX74y45U7!%|?@(bd@p>f`uY
z`>ZyzzYO^K^M3E`OyeEheAi^RZhW_*H&$FWDm1LUz4A>>K+X5B$G4v^DoW+w6tsSC
zhHS8g*SR_N&FuVYCMHvYq*z_@Hz-=F1)Q4|dg*n4r2OL<f7DJZh8%Bx?QXGimHJNq
zNgqCZc<}rEe*JH+*Q{UfUr}MfD{Yo?YfI*u$j!?l*5{XHF+E>2@2i@YhtK1G^W6^C
zR6Xag=b6{-bXf2@WAD7$Zr&*4+<OPok2l@<>N8{iXMI+Y6yf?;ce?ja5_<pdYtsem
zecxYCliXyv?Vo#{<;330_39TNRBFfVei;1nf6as$z47&D*R_N^a-RO^*W>;jb+$cv
zy8BYkA9XR3Qy1J;q@;Jc|3d92u4CnE|M}gVSUY>_;@pV;d#qp8%jCSwO>UoicXPd&
znVC=A9_F<Nn=@C%?>Q%(cXwCo)TyGHnwmK`H?eYaclUqxULJDMx=L6`$iUeA`F3gZ
z6`xzRkJ)zJI~cNX@wp&%*)IV?J1if&GJe```eVz&T~}(aYAEmgx#re{qZ{q_-sk_j
zW##JsZ<aJZ53iR~dA32r;<<i!!>%nC(>#iM)74deE?CI-M7%)Zlcw}K(LGO#G-ep>
zFzKz^Up4vTo4Re>8WMNo+qbdl&!3}mH05Kx`{&F%LER_U#@#(t{Mbj#*#Aj#$bON}
zyZ;ojz6tXYyWICIImCVPP1`MT`y?&w=ZC(Y_w>`R?|K!lQ};#rcKfl_+g_}iGpF0V
z!t7#X{ogg_k?z{uv20gr{&L>=$vji1OS|HwO2MfMmrO3TojH<oj79uzpiA&0<&KYK
zPO%e@KRY{n<Hn7e8`Znh&(B-Byp^?l^5m7*U#Dl4N@=HGknfz@%Ez3w>i^sCFMao&
zm;X28PuXe4fX#K9b#H>W{(bxIR^w5Ap1rW{_4JJFOaTdjhMhYrJEQ(Td6HuH`;GCs
zn4O1anPwN1l;}ihfA}J_ZtYu+z>;I{U#!>XT4;agcy^6{UFV<I8s}f-s&A;?%l-6m
z`Gr&8`irlw4}Y7Mviz0JUde+Qr#t67>A#%2^;a^-XL)B)g|AoAcJciFIknrg@^4(*
z_O(YYmsh>Nu3s!_bmQ5rshNuB{qjtz{~x%w^IM3*>=UYwCH^n3=5;?a^X$+6KJ_m@
zGhBHUxiRCIO4XjCb%)F&`8w`fao)c9aQ)qoVy6C!-!B{K|MTnTUoiJAPr&P6=C}R6
z?gtGaKe)9uJLk?0L3Q=z{o%_sLMtMcKY5a(qoZR{{Y@t{+WO*!Do(Dgi!QHy^wssj
zZo`$U`a10<@S9Km-1&F;_p2cno(h|Kt#+<jHtDi?>VwAXD-)Nv?cK!XwbI$IZ<4ep
zYs^WlEqhl)h%|OCH!n6!kooniisk(s&*gq|z5djFyPdy3F)z=rrp9KgRqxZ_)t0KO
zLPJ8k4=k$knWnVi!yTJVv%IZiS4M19+U@L|d2x~J$uke1`C65}a!E>3nm2ErYxcV9
z*Totiwf^CGch^%{S-<;EjPI!_#tT*jJ}f=2bILU#D@*Iyvu8(+9^II9l*_GO?r)%d
z{L=K;H8DGl*!g4*T)Q^Sw%X=o<s_fr_TClKFX=YL&-(s7Z2J2r#XrwU+gXO6{CB!Y
z`Q+^#rRy7_je<{B7w5ibTKDE>=PT1Jfo;|1%a$z@;tE(DrrRfD>9js>uSHdcrN6xT
z<r{gTSyD??<|w&|_MPNUx%aN$Z)Hf4Wr>wJpU%-6M_N8?Fvz;TE;i}&ipb4s)!*Me
z-F)6I`NoFC7q4E4ofQk@y}AAW#H!4mtSMnLk6yf{Gi(3MetG+4`S<r7IeOHv{@<Ve
zI0muK*jRu6{s|KV=FFY@@zbYASE96M@qb<#^0LeIk;5mQ1lHPLuL6uU<$Hs;@{T`y
zQ1AZis$l(k_t`-vSDI}~w~9CWh-}@+I)(M;5mw)a-#+r1<lUKZcXxU4)~wK*JN_?r
zYUO%!cegk@JNveLeg*Zva%bo6xp+ZO`(cg3tn<%bU0xg3H8YJXs7NH^$@9%G6XGV$
zUcb3#qSaxkY=?c5x<dcGuv(=#`)9NEiSw%0@_A0C*Jj?{rhD+<K?y0TX+L6oe0(}O
zIu>|O*E`>zef#H|&F4ev4}PAK!X)x0`PLrZnydTRcFx>+<;@4y-rg;#?cs($GpFCI
zHeoa|-C26%^UuYSB2%*yXU?2?<Hn7GXJ;gr`OZFeWo0n;@_F+ui=X*S-Tz|2y1uPj
zO)V`gU0Qw=zCYy0^J-?G*Y-VL`rDmi9sjP%O0f<6q$2+1>y-<a!wNoE>3zJU9{lI;
zJm>AtW?i`wzIe;BCU^fSOVUL*{P*yicr*V~@n(zdl^Q40YhPYlEB)}{!=k1y<*uE}
z)XK`rmMmFPU|`cQWrsnJkM6|_*CrR(xjc;Z>-CcQ@SszOH`nykw7>;3Ll69uW)<0b
z?%GqW%!Ir}ee0%4b;+pmU!J-|<=F<!g)i262Tq#)x5Zfc?aq3`mV0}vg=J)XZf+`i
z_^jN`&8_C+QSqExTRPW9Z@;uPJA75>>SedM*Pc<14xayPu2o`?+4Lz#Qr_7!O@8LP
za)mOp(aKqKrc{)@?|tfdO_)pT%JSgf(@NgjcjWQ*1bkgPb?)4=2O60z3mz~CtNA>b
z9$%-qXR~kUKJ^cwVQV4|?kIeGK_}Pa^TK^o)?Rel$Lq9W!ksfAi)`6Gy?Exy#w*p*
z-p;<%%TxTg`Nz-a?LW`||A+nUrmGo-N)1L+r93p8R$St|^EhSE-Q3DaBB#USYeUy8
zZ}_pDPtK-e=T6J0sHm2<wz><R>svLrK7IOBP*9+tqQW9BE`DuYZ1jrL!B4sz8gg%M
zGt9W4@M+2`lhO}IHq`Oo+}skt7@4`hu&SANre^pIqg1ZX&(BXk=y&hZv0mxI!a~P|
zk!@*yvo+&)FK$t4;quro?*D}?dY;{$qy6^(PHf4%tTN>~2S5Mv@bz(^vE2_J3he%V
z3FdU2;=#(wYGG~t`1R}43mltoY{?YP%4)rx7pc<i&~Uh&|MKecd^y`HALiLhL-I`{
zz8NTTU3`3`c+STc{(iq|@7%fb<Js(dl_}*SVq$H*YjyoM7iZcO=uVaTJ~^O8!-YRj
z=INT}3S67yH<!JQYU7vJTexT8+uPeO@2{`d3SH&m;=-~u>*}Ncmrl@pz@58yUtV6$
z&&bG_ks<PLgO2Mq4wfAyFNM;cJ-f53a`LXb`<LgHhu=KiqQSCq`uC!|$--HuWXy7J
z9cf@>R+&<-X=OEQo|xF3{q^&!_C60>eOQCjwZo}Ezf9dQdd@|yrG5?j|GUpVvD4z?
zw{K}DCMec?I;sBV=H_;-&{ZB9i;fhOmZsj^lv?us-rNTd61HsF5|XsDt3;4v!jvg4
z=jYjWc6Kg|nw>b~TXf8lXF?*2R(SW^J9M;R<BO@9XV1x6zqqq=bIYfBG5hOejf;!d
z#q2wyK6n4IBT3sz){8z;-c?zmJH5>3=!FXtF2wH6xVma;%cs6;Ya$Pyn5eumeEmEg
zF*UEA`E|cKcNRatw9uLT+nbxs>(;&UDGq<q?NDKMy>0DUT|q&?kemcZVfOMXrll$8
zg&#5R7TY>QJ8|=hsI^iL6BA8xZUi`0o(~ER-nePgqaPn1hvZo`xG{&Fh-q@FVVPcj
z{>d*p^H<l_dPhV^l)k<;HQ?V;*KRSx)Kem}%yPXlGFJ4g((2l(>OJkl{QrO2pPij8
zEGoLR=y&x-4JO8-;$lg=8jE}P?oC;Hv+J4j_MlZ;Vp4@3IR}V|6l5OXe#dItG`C)<
zjn&`tuF5m*;W&Nj)S-L#=E+)@1uV=HpT&E#rES|j7Ud)C0Tbq*Zfz)3am`;?`ubW+
zOABbig!7p#XdY#A+Sx_9x3|5yvvcyLrQY5ei*7`2&y&4h^O<+qvSl(B1q{s0%u}O}
zDH-}dkY0P@^G=q{n>Kaad3t<rx1z)~WB1_gbKZ11?Q#FP)8F#BanBOt=ogoka#w$Q
zGf{X6|C{eSS>E5<dwQWQ<B}Mz<0r0g@vb!z>JxY$7*}SMZp&Wy-pL>=EDYqJb#vyp
z@9~e_U8ZVlD{F6W?>EoJ^Z&oUp-B^X6z@$xJKNki?To~$Yim#MtNm?YWE6Dv*2*0s
zjSO6^O>(wX2?+@Y&N;}xo5#JtY3{j<m*LZFgF3z|m>c(Ilt*9Eb2lt_;K0Ph^hE2O
z`=Vvb+}6kKy>Q_|!uNA)@7T(>99$mn(a)}UZ}J=U^3J&H)7$sU9zS}NH9tRps^o!%
zI}DADl`So2HZ(9KB_(axxKZ=V!%wXh@=Z-m5{5}ES6785|Nr+lWPMz0>FZ^WF3Em5
z&&0%3P*S22ztU0I-Jj=m>2%xRj(SDyw`v#PnWfy>f96Q1aO9SZi9A6;9(V5E{rTy%
ze&FRx%V$pJe`=qpzmVgj{*wFN4|;a%M)KzwB_HGIy7;wlYG^<}!{o`r_wL>M^8UVj
zd3m`ei|a%&Ik~>Mb7h~Np8ov7!DcOc{j`m`OpJVTHXiToYAgPz$Og|&4av6j_!gje
z@8DzW=a!H2yuVG}trNAS!*jBlr$%ee_jh-1+_*Jq?qq-A-(Mw6U+FI7_{b;suWZg{
z>9;4Ve@(c$I{fjYM@kE8R4OYgZ`{1OG55Av8=vf{Pft%jeY8kp5}&M<%l&<|7cX8+
z{QvK-N#31^r_*;vh&M8b>&G2Ab7#(x+uIGzX1mQjzx>2Jts~JMdxehf^mPt+Vt@4X
z>F!2mb}tRBM^{z`M?^$Tnl#DjuifXBH~(pKx^`?Yc<;pVe0z^Wuyf=7`SM|5VM3?e
zr%j&hJm0RCgPS|~%nU=Fhz$--g081tUS7U&%N7;|Bcn;z)<%nWKGpl?s^E~6q$DOL
zHpjkx-mBcXnHS0xxn3429LZLl`}OIb_Zp|BTuXi4CtLFNR;W{;;q`T~htJ%ZvpRfT
z!0yv5J=N!?XuovY(-?i+ydd*;ik3u3LbwM<?yW5o1GqfH!o<?j(qybkI`a4bohE6V
z=Ap63qocQXY5MtjDXFP~Vq#%So_XmBP7T<ab=9Keg+T1?vZvoZew}mIWX7L6IyZ}3
zf3RQwuKV!MG=0srtDWcG+M3#@BO1EgPqt6a(&_cJwLACKb8g(Hta;-6)P$BF?9&g|
zspQ>uv0M3<^LU@!RHh9RUcPu?Q1&LGr?+>eeZ5_`nC_&2mRIubdhyH3-``uZeEH_w
z+uM{v8K%up5^#u*pTA*)!JD_>s{QioUlV4}RIA<~Q2*(-OvCi@^9Qb6nR2kN4>UHk
z$d&uhp+hPPq9GfTT)(}&-9B}ysAu{%p|^God)}xE^$64lO_+bYa{tAc{wSBz({wG%
zzGxgfc1*}eF?w5$WM5xjN_u*7R@SL6FE2;#t=h_>9CGdKZ1atqHYus9w(3T2yRbR^
zywY|))m8o}D}$DT#*`<k`#*b_uWqsNQqP(#vp1=8fBelE6leC_jQ{boiC1gB-AtFU
ztC<nHI!sVP!XqzFZ`!nJXJ!~SH#ISFdLMo9>Xnd)$dT#sb)L7TYWS_)r11M>ET>z?
z^G};DTW-GCx8(BdjT<&3{Qvh?%ep??b76?qOrun;`1-%5$BrFyadq8T{yq*Af+m?l
zv(0iRO=vmg>F0Os)Tyo`M_g=dY(Tku((6E_cbqj-RlTPf6h3m<@<r=n%?$frS7)7w
z)AibNXAS$DuO$x-epTj?FksNvUqAiXiCtA+wf_G8{`}3&&0k(zY(6{NJUK(7_{z#f
zk(<*Di=X+FoVvc`=Ld#U617?jJA`!A|LR?oX8Z8toMjuo{JB}C*(pt*oTkm4dv=1N
z^M}u$7dyA}y}7ZGnVFgSL8P|#a=*DAK0Ze-Ulz{J&gS%4(Q@TV$eFWeJG;A+k9LXn
z$l2~XvCg|wmx-~awpP+8g@d&#$VB!{Yval1o65U|UcPL;?Xvs!ggX6sA0Hoo{OsAc
zQ$J_Tv#UL|qwsOW{yN*MtHY09zdrrj+tfa*&@FX;tK9lzI(LOd&f>l6v@<GKRU}X;
z_0E2$ExUCiGtFn1<x1KA`?0ts=)|Q<mtNf7uHVilyQu#EzlzGrg=^RTy;Qs|W~UR#
z!M?t!udl6j-L!kIMx2~|UC+{`s_EzEB>wyJ(`T;LR*&`mopMT-&z<Z0@S&h&>($P0
z5(nlQN2l<MtDc*@B*Jga8^2$BE1sR7f4=_z@B91bZU~Q#o?ZI-+QWD6{IasNVt17!
zo}8pAw1}&ytSs$#pX`dyQ!c;$Rq)=aqeaPUPP@5(QPEV%oBa8ks=j7vGK3!3nteUv
z{yy7wKG~#8OFV!4`t@%q1B-!~nVOOkQ*CXnjEsy$!GnetbuQnuG&M6bvpJT<ZJC#s
ziT-2~@T%~B@IEOy+3@|%$=02nGiID$QntNw)Anv5#ka5O&Q<cZ{n0VYyK`cJWAnWq
zTMZ2jzr4Rc|MYZyP=o8*`uP5R`}Vc0O7fd$)7iRIHFIU=mG`?RNWGl{3fDXRw|&n%
zOU%e>cz<o}mv?ugoitO_{pKtH<>n<zHl&;s+V}6*>+AcTU0E4i^Y7<#5fKpqA)%m@
zX9>bb-OI|%{{H^1Ze%p6Q&@dd#m7fU8{>jl?8<+C%dPtM=430k_==1FykC8vr(ew3
zm8AP~j{U}ci!WTgs+RG)%<li6x4WI~>;6cjr>Fn<``6IW@ZkOX^VNKx{EG@LetJsu
z%$c)0>)uLj-R*a1$?`J?RlBxFtK00@lB@Jody2Y>#f-AIw?K8s@jh7|DHD#RK_{P`
zo&ECq`uo!Rzg<}9JY(j}g-e&ZW@N1RbFQObO<$j%ot@pu$?3<Z)B2H{(|k8Zny)OH
zw>oaGRXe}@vf}6G?pW=bR(#Lz$#c7@2`(E;9v=uws+nVGWc1|1!RGfN42L$a4FB`z
zPeF0<<72(jTe7eB^-7z29nd;<<Hn3+_x?+(Lbcbuk3Lg*KTzb!b5Das5xa!!+IJkt
zUlF-l&Ane{B2xgXPV_dO?Cfl>>3Y5)Ay=YB8yMzVl_vfE_BJIg?b338c_ASom5`4I
zMMXp&JbL8h;laVj$G0YG>#B&g^S7}4^DHYf^Yioj@#|O6yUArH^1E1l=YA6E(GZ*B
zrTXz;hQo}lu(bgHkDooPy)u5kU1DP5l{Jx{_wM<Aak2Zx&6^*8dwbi$+In-&%}wF=
zo-1<syScHIm6g4@wUxWs-+uYZ#o|&?sVWkIHhwogecrL-t@OJ&w_H9yJIl$-yEJI2
zmynO*|7SjPt){wmizOx{Ib~#Ml)k=J`~3UueYMtqe}7*dwA2eUiMcvVmou`);PJ8E
z!*}n_-B<hj(fjxDZ`?$tDSgQ2i<+$Fn{;xL>XdaqG<Drs7RBysV^Qw<1}=NI_dR*{
zGZ!>*G}pR3Xy2dTdk+-;|5qy`D|>TuI)AUUdD`o1YfUmPI2;OBtgNi;k+Gb_D{ZzQ
zeygUg+mFCiJ##d=7wOuoHy7V?_UydhymP1JzkmOxXex_aa0REOsX4dv9ld}5zkaa8
zAL)Dd?(s+(wM1>r5)>44oT?S7lJoK2&Rx4U<=ix?`tsu7xw+Qyvy4{n2mp7iZ(gZi
z)g#JP_3cgPym@kpw`)D@<U1M*!?%2S;^HadyEBsC@^PZOSB-_4nHdW!>%!&Bzu&W8
zUh?vig_YH!g$o<^@3)sW%UKY#)N4z|MJKbSu7Kd+#rgO5iHL}7P(Lz#_w(|Ei1qAC
zy}~CHxJ;Zfxp4lAuYW5F{u+Q>eEc}OtgI~OGJ(6hO1X`Vjg^#@B@Gf7Zg0=8&u4Bo
z&AimIDs(kyIv^rKV%xTDFJ8U!dhuxQOxtR+ZgKr($;bOzT3SSIKknSZvhUb5-ROd%
zBBzz>d@moIZD8<DYhj1d)0c-`&h5DU=}Df<<72(Qk83L2Ti(ph|LDbw6KkWlOIQ>r
zWM5k|@uG9+nOUaU61G()QdUnh%}TVNW}RA5y(r_P$IeqTdCgLm-35&@r=Oc+DdeSS
zUG+ud+O=zUc9-`{8mAf5{wfjSI$HgI&e3jh<H}Ddke-J|>gj1yKOEiv?Afzp{Puqw
zy2W%my1KNkJ=YCl+1I#Yg~qRMuif3;#CX})Eb-dRDAc7PG*2b|K~Hu4JiA?o_f&o^
zC@uZFV*dPT)7my~Hom<*f4SdWs~<mpTv+BSebYcI<@vd}H+W?{^5je;pZK{3b91_i
zaQ5fTeb&=1yL0ABo%nroE-rTW)@VI*`SRrT`+jkimX@03-SPPU@2~xf=b%L)GmX<P
zE%TLbYHrqy-L)l$<>R?)>tav0a0(lknm#?y$ozik?@c0&M-Tk`{2Wv;FR?B+c==O#
z>+U+`g&b3-mAzGXzDFkTS=xhl@6J6q*vy?9IZZeE*c8oR4GoQib8{?xW*R+hj*%35
z|Ln)d#}QGH9~Is*e*LYx%<c9=p{{_;@A|F6n=3VDWwHMF@ng@|jLDDwYH4U3xO#PJ
zE4O&iVmIEVrlyR0dw#nAe4`n>Y(vIHrCnujy%G`<Hts3+uBn)9p8xE${{AI-cX!Fy
zRBUj$lda|YAbLmXYca^cVBqG%8n3?2(k|xYnmR>B*kjFmkCH9k8*6Wy?J9pC_u_QP
zyE~SkmRre7p=ZyYRa8|arKG4xd%MQ$tC2J=F3yYD?>G1S@}!(^%`HkHm)=gEcO-YV
z!_7^<T3FaGFLY+-Tq<B&_vgn^n=H_pmD}5L<!eK(U0m$GIqRyHj*iZUj~^X_gM-a_
zDv})?9eLzzW~>TbUG(C@LLuW<YeX9JUmk2`2dz|eZs!YJS!gfCAJaUmEv8gx)#L5X
zr#t!o&GVY7z1iAiO4`|3E6;37O-V7xyJKNvW3wV=r%_y7oL1n1uJm{J_I58_s(SI_
z#gJT6iQw1&7cE<-c#vaiVd?ppb?@f7pN-neaeMFVUteE)YqajPT^G01f4*JJo(e&A
zb@gj&B7-$_3%!;KsjI7-<=$#pyH;0DPEP*mz4E{>t*xz~4&ABg`sWuqw;LE5Y8t-Q
zyFX!j$iA9NB_*X7_xIbEpU;2(<yYO4G}|c&E(@#8*dKg9!d`!V`TV+58yCBWu3~6O
zU);WH_wL7U-}<__u|0eC?9cuGf8{GHjS6>atq55u^!xk!<9F`Nv0iRgc<u7MKXZ6E
z-9-L<T&pmjoAL3^UFCAVv(28~xVSioL2T2C6&km;<(^*ZJ$;6Ky`8jqp3ataGq_rr
z`1$$QtX<2=!IALh#zqh6_KLZnZ2jPY!%U;pj*bqEKvo40?+4b}8X5_QS~&mwdcA&%
z;VS=b!N!c~tJMy69J(q0xya(Zb=Ph~#x42x=gqY)U-gIgEa<|dd-vvDUhaQ+Uv0Hn
z{=GBm^J|o(gIS+EefsdxqeI7fr7y0()+2WKuSTfbu6JFBCeELHa_8B@9y>+czQ4Q6
z$;0zUA^1**`SzlxUdQ_7`}_L%?Ck6!A|qEGDSG<k$%W<q@`nx`0+ln|;(AkhRpVY>
zSsASV?e>Wi9#Px%Di<f8wrPAHWveX`5fQOq_3G2>7U<sT<y_JPEsSfP2!C7N-|<bt
z<J8i-UtVADm$fbv=~V314q0(vL*n5RXU-hCe_x)TpTDQK*U;2-YH!quD_5?77Uox0
zR)WS%^~==RzMj!os$I<4)n5}df1=YDwrsf=>k|(SHh=v3b*d!;_vi0w`u&BEj~QlP
z)5*HN?(Dt2)h};sT>NUzl~t>gv$Ib(GP84Va449YPhS_i`&65-zGeMCo4r+EmDJVS
zFI>1FR2{!@3(G!sP!%L?#?#uS=i6Qpyf81F)3rwAX8${Zy$7eMm+Ko<eaV=n8y%+d
zzWK)1Z1KZwypR9={oNyDDP(SLZh0|SaCO+)q&GJ<X1oYl@Y5jv)U^!Fg&qA$#$T;2
zO6Q#L+1s{ywRZ9Ib5jFbxdWr3q?p-wKz-D_ySpAff4+Q^t=~MG%0tiR*PmM%ynKdf
zwpj2opNYI04e8AbzP`BF+|Dn*srb3ynX_lTbHsvJ_NfaB3P#+%F=6J<DQ;z}SKhf2
z<|HT|^6Rd}BJ&%cz62^2KRfdfG|71Bb@B6avTXcvJ|Q6@y;7!0udl7Wq_sk+si|qh
z_U-E19eUN8ZC2bk5aT2$e`WFQbc@G^4R?A(rs>6kx&kUwoJ7RM-RIlY?kIZ7wb-rq
z(e?OxS7+y~?7d#z-iy7b>m50EY*WTXCAVHFRq0JFyDLAd-Q8WTuBpjsZ*QNIljBt)
zXJl}u`qJL&az0rbkE2~|M}G7?ICd~qsOv|9T#WshlcBaJef^IeKD_vZcmqS|>M&I$
zr6$k_+>RX<QCqW)g1SM;s~h&!{uYvy?5vZ_Jy&(B`1g|KElPGXru?v-6PMho^xg2^
zzki^Nci}?7t*zO|d!@}Iw&h4_hp!9y$s{oEZjpeXVB^M(hLhF(kDWg4ZI;G4ZOW9E
zrAt*iJ3A{XDyIBk5?G@8Vejv^+wU8go2%Q|&5M#f{vmVy{H4FUD%QW7cmLeawegGA
zF7-@saM<|fRKbrAi83-WJ1Ree2Hn?0Zf@I@divBc{gjU%KhBsryZH0=`zwAQEx+>i
zW4%yU#d;zC%3{M0*<ZeX<+!(Vvfo@QPmR`$hlg5s?AjHyF2?fDpFcnTd_LcO$Lh|m
z(r&-GR)xR6<(lQ)(NO=d8sO!{Wn2ACMNhA9s&=^7>J1`|@_tcKQhTevKRYo|IV3B|
zIeGWh6`4~eB^=qm|3pEG=tt}Qi<6Faz2V~p4JsZji<P&jXaH5l+j19|y}bphjJM@R
z-{j<qEGjYr4T#uQPg}KiZbrslmeyIjSd{;5U0waWxb(cts_QEpD<UL#rA#ISaC!Rq
z@hQ9a9XWisnT=O!L-KLHp5ET>H@{+b7Pb1#wVF6<)~RW_(Hk~wn9v*JWK;Ep!`s{Y
z+&tUUO|0Bc{$2R5c-PUsflE|tL;3r-HEY-IlomemlK0vZ+gg=_9p%eHv^;NoW&UU#
zV`ygf?8nCBqq`Vd(u<4O*w{AJ{4AO@Y0`@;D~0#}`P6;pZ^#L)@O5Vznb}`lT+CkX
zS5<VZSm3ASOrf5N?~kJ$?ZlbX3rf$+%reP5^fCGP5(XWY?Rj?}Z9Z>z`0!z7Zf@>A
zS?jPR+zt%M$;sc|-R<7D&#tep@6G-F@n$8Q6K2lr?Cs@EKR++^*O!+;4F5W}u+%xQ
zva<Hb+s}J?divt<lT*Z$nJ#EWhc*f3Z<l}iT<EoV+mp@vw&dQP_TWJR4==BG2%Djy
zA&-p31n=p3iD_wV%a^N9)e4<7cd@|JXU`tJdDAm}x_D~#7Es~fHFcfF!H(^d=FfIL
z^X%cBe(tBA&)cWp-j?f{)HVB&mXVQAb#?W-`}_M>uhzEz|EGAOOyRHhbL?to<=x#C
z7!cr4P+$NWfa;C&Nli~r&d&A@2@w%i_e=Ts=%~=C`5xX6+Ihl#ds9zOGc11>Gw;sc
z6CXwIR!*u3Y!b}ZdH*tb(cRyZYpzYVtNkS)DjNET%b`KdcUHl#FPUyV5{?xW7QVC1
zQqRw`-PA0i<m>BeV{7~K-R}1lCEBxuwa=XJ*{gT3!+igQ2{TWv+`4zCym(i0GxNfQ
z3xgQM7FB+JmYA02R#s-#*w}bxw)y%9(FO+H=xr(2*Tss6h+J47Z+~vCwYu~^i+R>h
zpYD~&{c~D>{}iSV`5xX6+#fxCdbC4OIUr+NrP=fZ*_}6I|7?D9{G?0%hKHV?Soi&U
zr43#=v+GXv)5nhwuLxXR@#zU?@N&P@b8{?Rla}myl=Jh`)0)3uuV+j%Tk<miR01n5
z?5LkIv#L3xT-qpjol)tlkh-5w#U)+VPOz!_Gb8bE8)yL2$JdvWhi6IL-m0Tja}OUr
z{PON@^IyMywY0a3|Ng4$$;HJbVOL}E@83TQ3yTL&pN78COY;znV0n9UGkdnQ^rm^w
zF5JlDojNPRTjSBxOYe``&u0GcP`B>SNB3pEv(*;<>Ce8tE-^p<{K?7cpsB}u_wHR;
z8y()H>2iBpZt{f%j!!hNX1;pSnYdR@sH>v%;>3zRW4XU4J@=oVr0OkUU$>_v$U`q~
zPe*Sr@5hfHL1V6ohuOTQ>%}Tvv(?bxh>ni#>FMd{=y)*M-)^GpqK;s#sXfx>eEj_U
zzrMb1pFe+ojLJXuAgOv6+o~@Q9yBsbNKJV@=X^n(^`0On!TOI6js^OyGgAM4Qr0Zz
zh63vY?JFyT+Yh(%8>gNU@tb4eczKyGcmOZBW6j#Npmmc~OSf8VE>!4TzDHH$qqrt#
zwdK}hLaE=Jx8>c{VogxIwX0M+_tut&udc2JjVrH>-ac&xlfb;Q^{-yO^eiee3JVKU
zQB^(Z7kFgnu3e8FK6LDtv%Rq?wOgcNy`pYo>w{yx(lchyo;+c~0rmO4hmU+xf61oI
zutR+J<Qcl>GNuNpAGjaiwy}<R;>3wO(q=u;+w&4rQ;+T{eZ3-P=O&h5553r3Ei-po
zZqK{7B=7tCmACJPItenU%=`8*>4C+UQ&ZAFqrC6#L^^4*R904ky3VTJ(*jlosn*oi
zdVkp#vAaw+zV@r=*H>4Sb#!{VL{l49FM4rYzW&Md__~Flu3LNi;?=9Q7kUV;m=zQp
z{POB*@#B56i;FHZnn-PMdM2dvFi$S_@KV|PNf{X)FV@9w$-3(0RO#F=XPb0qM`6yb
zEu7BI&S7h#TGy^!%M$DXD&1~wRyQ;>49PZ@xm?$sqIXc}UDOk&buH(YMSXqM^7fwZ
z;Wl2?DNY67-^IRp^Twj!0fVen$$|6d`&WggUMVs+H&^qW<uX~#mxGsgY0S=|O`Am&
zyT$d>Zf(ijl5w$V#R?6{gjc+t<rkci5)vFzQdDYxe^dQANB-iwzbC7IMKozPPVrKm
z?wnqrnW0ts`pUx}8<T?=_AT9%div4Rr>*nn%X=-I^zZNQ$4{Odd3^lSTrHl^RUr=_
z9Bc;71FOs{c)n)3^3R=-LOz~0`$R<qw$~bkA4<QxB2c+c-dc?{LD912hk<U?mIntK
znL!!z&F$^}TnC=3`OP`8CUSE}XXn9Nw`P6$Qo^~kBVuzJ@8xB_hp$|j^5H{)Cd2)v
zEi7W2{`{#q)+>E{Rp{yqtM7cc8PoJDRAq+HA_e>D-2ak2tu{}+wl-Q=NJuF*@ZZuU
zOH}5}oy*C~yEN_Wtcr??3u~jzIhP4MdG^ex=10Mvy?ZsIH;bNa&s!Rj>CCxo$)9~s
z?NpDYvwYWYyRg(-ys4>4BYN8!mSB!MckUFFmbwN9Pd3fI_Ta?}jfHy-U0E3%xuZZa
zc2|j_fq_77ZZ4;5Yh`uy=d0oIf!p(9`DCqBwm0oj{W{Y){oE7@uKroGr1n;QbqWri
zeAjwg(f2=PJU;stgh*NjYQ6ikX?{W4V}*C|JfQLA2@?cfHhq$N@$%)v7cV@Xo|?+Z
z#g+8;*H@dWFCC!9s_!HD-|TL&vf0_$M#jdSJYS#RJ1W5QdgkJl+P^C|?q`@V({0Oc
z-OTVspmBl5#zxhw3xc5^9~@+U^Y(4zt`g1C*Vj~SZ05+>raZZ~KRGS!*xlXboE#hv
z{`~wL5gmQ{ijP*-?{9A{tgVCBMw#--SU7CWx~ei|yNYh3R?6dJy%yHii^IJ7)BNnF
ztiS8%q`7R<{5wz2utm$M&pm$hXlv-|u#@~}*$bYY5^dv=Jhbt7>;<#jTP~Z^&Z^9C
z+EDT`NX=)4!^%sdnRaLR|GwHJ+qHVlmf4_j=feL+pkb$nhgyw(L@HWTebM;#?c1F_
zmBQ@o>|yKUd}kV^s?4|^pf#08)=ETQUq2@&=fj5&6Yi&r3X6%g&7CXD#>OUNQz0NM
zEX?^-_JnPtR?3SD3t8CN7q6Ul`sLTZJEV6zJ88CEdaqiL<XSGLJ~#c@nVD;1cZby_
z*a;*jC-cc#9eHzev!{p00^iwYfBt+v|K#~|ZwvXGKR!PG`1y159rrm=$(~+^;y9N|
zRLzk%+&T5`$sP4Gk9Lc1&bX+g$-up^19VyKT<h{@|Nj176S<i!KR;iS;r;}tg$M55
zm6ev3W@2UrZDG0;x5x^#;-I;ixvHwl#l^*g@kjo*(oD_}+vWcAj~%)+>G${d!CRLq
z->SNFeCLi6=W9>Ry*&HG`MNW&uC6XDESwl%H|fQT3{V3;CPqfqswCm@v0e#jX>Wtp
z56t&=`^(GOdoIs9t(Y;@Y(hX+=bkw{@8*1R@}1ede7QO^JKvN5yG>K3h_vxayLotU
ze0_Z#G^&5BSK8a)y@I-W`@Y)Wo2tL(?J9rocY2zx>J+DG^X8pv<CP8w4+o7QUt1gP
zt?|DtXvLD%;p??}eSM}h{LxPPUs75+_29vSn<k4lty!b<w1rc63RkPs%GIk^$LuOu
zxoN=yho+w=^kQ}_SQEFnO@_bgq5a=YsZ#?313^0WR)5!89kzBA%M|A}p1@_xmwRjc
z4@yg0w#j?C-lw}^8;e)`{r&y)xu0?#-V3_oTX$7`T~*?&k+v#qtrqhEkxjL~zZoVS
z;aC^H-|zam*qa*?nYU(NpO#>^Xu1FVqQAesCe_z~#=7+M^n7NU@m5#=PTC*4I()sJ
z^OpJ)P>Da=toP^V=ZjabKK<$G>B6F-NeTDjlQ}s#Ig6wl=QxX62eEJ|dM)|#rDU1U
z%uh)hT3ETo46?3haJuaY7Zel(P2-4ahZ$6T$>4O^6MlVNET{sXV^P=?wl?bF<HyQ(
z@3ac!yRfwhI!*CNJvAlq>Z;I|_V(a?HI;7ta=kNWN^Z})JE@>WK<oVN+q0jYon8Ft
ziRUuExu+&7yDwo7aq<w<Xc5R?<lZlL<k&Hzq$3>P-rRIv?A9Bxziuy!a!Y{c%a<=L
zEG-QyJ|vWsmU41)C;$8NGb1a@%fg96lc`nUk?+gbuQ%8It#Wd5(g<G0qo=1A5gA$d
z{G2SOJI6vJzqwYfpP!xm`03N3OP40im?1IqPb`O`>jD9%Kho2tO^ex4(5UF#wjukv
zo|IWm$MosbS(IBeLcEqv`uX`eXbdUu-k#3cv!xFnJQ%VjBJhwP#J&H+b{0MT@a0R&
z?{9B2GBY1Oc><aOUK$#}5vVa~@?=AE^W#^qPTf`dy6Df3LN%Wm4NeOu6toC{Li)(>
zE8Fwq^<s83?A>c?UG`>yV>8>Dn4Lk(as-`fOzz#gXOeScf^GG;2XEe-d3kwx=Cw65
zS(IBQC<SmR+KDhTGuPDA+}N1R{_EGT9!cZ2^LD>`cI>d=bm#bZP(@w+^7?ptHh#ID
zUAwIQ{rk5jdOKfiY%Hfc#KSc_pib1Iquo8y=6#FZ`!B7DG)_A=$J5WRuSL0~Vtrly
zUv4h0L+8%*b&KmKWoK(2<WO9~A>!20@9Xd1A7B4>>fv_&&o7tHH_E=IGs`sl*h1%a
z6&b}N`@ij9TQ4Og_2c*N&84ryczAhLWfWUP8l5--XWZJF?Osx1vN`?yvs+uUOWxg?
z8MxT(<BN;RAQ$`X*|W#M)KpbNqoa|Tz3AZ~*37@ckN{rg5*$2PzV1ikj2RME<?lcx
zgQ2OZs)0ekAwj1-8<#9m`Ss;x^W@3Gvu4c#*_(P=459bcWq*5BEv>Gt+1Dq{oY}c{
zt?spJ*No=22t3-lXV0FD8ygg@%ikTjcQ5W8%sSUrr-cWO9&O#R!veIs;oIBWsaBkd
zNACZ+e{FxzQZGRXiH`n$el|8Xp*{hpE(gUH4KJ<mbv^6XulG?`Z21u_ZJt+i@t1kV
z1qD#VZr!>S7G+(V!hO{ZlaKMt^mFPEdUI~B^_N#yyAuz$?alamdr#%&E&n(nVf^UP
z`OBVvemw3s$-Nb_Y_h;2mS20n>Z_=*u(Px4#qH5pTMtR@i+23F|59aL{C>U5o*aQq
zDr#!`GX7q^b}jAqx3?A5)xis$SYh!kk`D5-YX9!t*1b}uN-8QW?(Xh-F*_XY@2gdn
zfX2|I6D2}UBK4h}oe@z{pzOD0>sC;Ge0iDgrC;ihbmyf7a{Z&zckj+^<CPYYk~;O^
zVDrw3k4m+_zD!JjB<q%nuO&inZ*PD8<>h6eJ^|M=Z*Omp-}3L}t5=(nkMr%>vqvL-
z-<}p|LOyc3ZoXgAp%%_LR;68Lxwk-@Fy(A26drOceDvw_=b!KE|97ukskw0BLeTsO
zx47Pg-R1c(Z*}ZnzI=K6?Ag*6FJ3HpaiMXlc6iYGINLjS?=J0LA)xi@$H&K8^6t*!
z6jlpZ=)~I8-2Cy&mnqlmA^ugo7c*JS7u2MblatdB;R1F3mix`MD1SGnMY*M`;==>R
z#csWeoZI=196R>uZu$MfkB?l}tXZQ1P0@<?<~c1)cyVFjk>kgW%icuv$=idf5lQ>H
zJ!f<^TogZl`n0I-?=R4%FwnThw>Oc`&du#!w{9KW>-&S(#aMy{;X!?)ySq&H|NEu=
z_0`qGr%!ty5`3iG-Q5j3e#OGla$$(p#}6Ms^Y=4m&4QJ7PJ80d&NhF3Vxsbz=<R)f
z{?yEwGv~#foyHqCZWQVhkdFup6T4sk-xk!F{r~T8&DX2pF}q4S7dp4Y4XCl;YGq1K
zPZtsrk}%B@0ZlwsetzaN%cN7?f1ZbfQw@Wnq9Uja+FATODJ$#L!^7>M&XCV9JxF3W
zavW5&hpme_xiR^8#<ew)d3kwPR)_bWoo()I;j~A6d*0nc$Bwnd?X3b$n~CYiflIOb
z`}V>@z~y1a-Cd@~`{mQGt_n>_Pk;R4#fj(h>-}P4=Cmjuu|9SBbn?YTt}|xM+LU|S
zEGIX2XUR(;VRb*3*w|PmAxP=4XafJ8tSr#f+m9a=bLP&yxjkPVG#Gt5f4{1&Z7h?}
zBV8HGq82gTD1+2fB0W7lpux+3|Ne!n4)cW-;Z9p6q@*_G-L(QOAAWqS7u2@ll`=VS
z{=EMo!AGV+!NG-vg$sk1`yDxc{PSUcdjV<b?$*{;m@itgvahd`RrjBla&uGalI6>j
z)6?5$&XoN4@uN_mK>eZ+El@rE_4W1izrVhgyuQ{uV}?Z9`FXv4eSIt{kbra%w4AYe
zrD55dh)0hfOInp^2nq_CWL|1{zyJTdGiQ7_JvbE1%%+{2YyJ4m8&H8h+blQf{k^?c
zR)_n;{IDp)$jIo>;ls_%?EIG&I5IywH`lwFjhBOmXG(!f=hTT44{pC-_dDs`S;ORG
z1urkD%E-#Xie|;G_C8tbqF-M!r|Co<I^52`aqHHl)!*M4UEHHlxGn#F-^!Jm++sQl
zGA}Qau`cU*dU`r2-NN(WBiGp2x%vBkw*B}~0cvTbq^O*oWt#lv#>OQ}mwGxZis<g|
zKYr>IS9W%`oL$X{_51&w>J(PbxVvjBBmk8JoH`z@EqZ!t$L`(5kB)HO+LC#=OH|vy
z(D32)_<B`+eSapQqo)qH^Ut&>R02)9USAh$VPOIC#IauK_F1#$eB*%R=)k5eSy#K1
z-TO9VUREn9DUq<RvpGA<R9Q*M>5!mnTwR^r>aewkjvQ%G^`3TNO{8(3jAhgI?dGAY
zLMAq}2t2CSnBuiGC`rJnW0I<}a&zV9XB!J1I?c1I^_p*28?mRt5VT&g;NHcR!OM3P
zJw4^!Ki9r~-u(K1GX<61E-ZHMpZ;%lV<Y3;-R15J13W7J2Z08q)O-}w)Y`N{SAF<=
z-rl(6MZn20NKMM|G2J}xPQt4zD}Vg{9lR<;Q_W|_0e<^G4fE#BYxyj8>GI|KnwxLj
zh>$YN(O}%S`1SSmpqa{TxwqS<O%nqh7x4T2{^dS1L8}KL8R60VH~06?2gO72^K*ea
zi&E?U{Y-D;l~&W$%wL#wb(M;$>fWo_n>KB_aP8W(5BC<nm}i{MmzS3ZTB($reE3i+
z_sZDa+gg-c1o9o;af&!`fDX6n=<7?nx+?U^)2E<jCumyd;$nBzIp!u+UtVml+BbRf
zWRZq^&y^vM9zHw>noQbL`FTb7dbx=cC$g}xcra>!D+D8}%cZcj0E#UYujN@xP2Kur
zPCh+7eWpPo6KG#Z^6@?kE2~K}W@s?(ZEJ2m%wAjPS^ngNAjrlK4-cO_dp7mysi`jR
z?w3KmT6Ye`dy`7oMV&Z4f<}Up9v$g?b8qkLOG~{Uzj}4*%F5uSUth^@$-LYayxi~O
zuK8ik&d!r2PYw(ZpMD_@wC14Z_uK71e*9Rla^=Yz8<S@mCb#)~(`ym<-?W8=OR=S5
z>94P^g{7pt3JMHB%Nc)td1;byfuX*>p7VLWg@wfjwd-N*d@>rW53*BIQ$dq1udc3c
z@9*aa4Jg~!%bA;-YcjM#+6Dshlcr7;-MDe1PRx!8?EG>D2?rQ<mA*c<v-r8n9Q%#i
zx7RN|Sh7U>h5hYKsobC<;@O#*SJp&Mj@@0hG5fmStE;QkA3_tiq8*#BudkZ_JRc_~
zCOthp8H<93X}ZxySywcIf`U{QJl6_e_ox5+3QZBNtZQpL+xcWw)`VNs{xX?g`%SW$
zjrY;V$H#Z<+$kt5+zctZz)jE=4IVc)w>7c5rv)$fySUI9)cx%3?BvYL%X{+dnU}+#
zV(s}sE2o4V2;WitT~FDq2h?Gdv90>Dl(Eai)KpbLv8CdCN`i!9i@^Jw*Vop9f)><~
z6&4l-4K=P=yY~Kztvh$_jM-hb_D1Q)>K`8z-Q3*jqeYG%IdWk~Ve-w&hB<R&?%cUE
z!#JJKf1b_4=<Rtrk(*l7d}n!BIB|Sr^z>dJ;M7rWR`9?<FLu`vfBU~KK|w-hW@aUC
zZ%uvi;>GoS-$2vKpo%tn`?`o!(4>NnuI~R&JDe6SNIcvY#PDg2>zD7p|Lyzz&YFjp
zmy?4-!N@3x%Rv!R^tP<JeDUH#`~N@rr|ZWbJ9@MgRDP}s)o$aH{k8ow_dLtuwq9v-
zCWE3U9!<^7-JxphUww*<+1<2k*)q@)>6qBq`!9a7^UE!X-Cf2bWzuowN(iXvjfs&l
zGc)^?D&*AB@3N~Il&sb#B_)AY%vXPV;}{-3{p4iz$FE*_y}YzE-e!69_PjT@x4*y1
zKhddi+BC7>-`+0Hy}j+IQuCTMI$B|CJlx!ldH2WfF6+IuHCudk?%atJ9k=J*wW$2`
zr2Emn$jxc%Yw~JpY}Tz?2U_j1a^=Ytfs1F@)mpW+wS8JD=+x2g6UEdj;It>)cb18w
zjm?}169lrZt~&bW=4L}f!^FS8zCu<}eOkoOlCGO_^XAQnsHjO3CIsjhUATC$^5s2`
zRwu{X+j6@@8E4F%-Mw||*KQ_(iqz}}d#k@o7^m@onv{^SSiP7X2QDr3UdkW>X`(ub
zs85?V?Z}ZM8<LKKw;2=`F0B0gtVh!LnSSw0&8c3OE?qJJFDTv_vF88(vcDl(Q!T8l
z#AfHN4$&$sDoQ#(&sN`7PEO7*B}Ii{=FFLYzFhW~ke1$D^wcYAd*0cHhuc4X|9<?)
z5tn@mP8@+9ORQH2ICb#9`}p|ynVH7zN4v!*PoMt1|9tMXHInRnG7kR!>pj*N$HmD_
z)e2S8)#WucHGOwyr}I*;sa;3E$=KJ~?Em-cwf6t$ZZTbu$Bfg?C>R<FCMPF9J3G65
z?b@|9f-TUDt?F4-Wi`L%6K7^-=AL)UpLgr;Q!p|4qF;O@c6V9tn>TNiHlLfU?ho1(
z$t9{)@Z!S4Ma4x60s^+@-3>o?@%v?ed&AsYTdwbGU${{5@^b(5D=PwxuFn^Ml;w+5
zX4%zRZB9R5_xQQBmDQuCProjD{bG`;_Y8}|MJzuj`pht3G&kQqT`qca8n3c@UrjD!
z<i46pAt50H1A_?)tpZN0nI0@o9Ex^ZtjphRNI2NEy{Z>f)2$9m{rBf5!<~J#(hd4N
zrvgKlhfIw8-o&JumDQ!Rb?=_M`;TvJ&DOUS6A(DCB5-klhRFXLlYamHI~R0>+?g4M
zn?7?xs#-;_MVXhEef;wB^7gvBO|0B2;`UlSKHgvd@kaPot>DmTZ>PTvdxYQb{Zbwh
zBGSypTNU_ZZ=ch`glA`F^2l0!S-R4(MZjqVo1|<AhvE_Tsk3ICYU7o@_i`&&YZIri
z+JooM)g8q^BR^H&-}%PH$>mqEcLuF|^6&3&ov1AyOt927#RN3U@aD!wlZ*=uK}+wg
zlYai*wSDnoWzaZ-f&C`!@O24!d46eWYVVC&4)jPGd-(cJ1qIJhkMaP;76JJw)24}?
zo~FyJU~fNvb@+NvzcVR0d1t{xrswD9t1Bu#%wOIYpf&Z$vuBT9zt&EybMc>V2WlEE
zS+eBMYHo&pIonB$Ni70SM^C(1;KZTG6`Pl*$H~cAugSMQZm*Z4<3aE9cW*7)cSra~
zW2s$HpHWqnP35O2>(%+zOGvl{2MY@e3rm?~I5e~Iu880NulxVit5>Hw!0L`iXH(MC
zg~i2>w{nZmFwd9UxN&0;OC8I6S<COIj(`2~#lY0`ulL6A<$khPu3lBOP;Ajq0;hkD
zsY{nGb={nH7Stq9KQ~9w%q-05XM{S3lvIzL?XRUt>x-VA66%4bHK&e8ZoZ7urcKj{
z-4$|30Ge+*9<9@{N)wy1Hm}7=_{3S2-}iit{N^fH|5Tjy>|BUO?Zn-U?y6df;QC#$
zMZk6E591jI%~G<yaep6Mh!kDE6tFsg@u+DjXjlf4rkzA0L)WY{FK4M*I{)v=Gds@a
z{$IP~oxQVTrt+7n8v&O$2}g-)F-({}S9vC<VvEM6luZ?iEdr{4rf)F*D)v8+;aS}6
z(>oIv@08xtzel}a!(-nW<@NCv??1=~y?e^MU|oD*9Vp<yLr6{{u59)-t@l3H8J==a
z-|=8qOSoV3`ksf!u4kWQ5^1}7!A0>Q$c{<i;OdyP`jo>eQ@!7h&R==^IzE=+-#UdB
z>*wu(%LPUL{A9nng*{a|+;Qe?W=qMGIXsHL^A5(}b2@%TZtff7C2#xPH@oVenib|Y
zv2r?lqjh}EwzkaRe>Sh<4z86bny=w_o@K4ndG&<ww29Kw9U5~y;UTtu`OQ^^OXvNs
zPyH4;sVRToH`D$<=EcTlGh?QlmkY>tKHGXfA;kIrQ+2+-U1w$l2&JuGaNx?US<Te}
z+tPSVwk~)uO*3`PtHhA#??*3ZfBV_weOPVM!WCY<`E`dD96Fsp{n2M#bEd}^-Xypu
zPcN&t*;lA#-lTu>n1~a{$72w03wdvMoNE8?#ObQVVq2TKZ}0g3`^18*xhtPr@-CM8
z_2k~2^E(-9Eld*(%+KAa{`SvTC*#ZMh=3`M=buC}u3)*6WpaGQwx;73FWUH>W^)MK
zrDv-Cg)vp>y8E|h=hnUvpa1i~|42rT3DRcm-M{DWt^c*~X%GV!tkSIrmfGLX#{A^3
z*l+jO_oTJ{-{ttS#bIXoZ0*V7w)KCNgX-qFe43j+*V^>emnCj*nditG%-F&cz;No#
z?U<jp=C$lJ_K?^+rRVuyhpk%E>ObBq7i0POTVl!6U6K30G{<gRX6F^;@h5*qjte}z
z^WXJbGh96#R=Zo?wUv`?8Qb}LPj3|(#c3(XPu%kQO85R#Ti>&BEw;D&`btqw%|5?8
zL~Y{VJs00E6*77G|K{%ZA2d$w?R|XzY@XCY)d>v?J@qfm=A0|QbZNE23MQ^?;Q}Ju
zVlMsdO0H9s5*B{H=Oq5}s|TmYR5isZ9!^4&?w;LOpX0H(bzV{Zy3^j9b{kv&o_+J?
z&h&Ye^IpbS7N7nXwsX~kNUMK*ed1ORB5$yszIea0;=~0lC8b3I8cIqtqhd;uI82^=
zI`qT9w(i+i#qQ60=Wkngdul=J`6=&JWhUEY7WrI_Zdvy){?Gn7AF^&sHR$NRe$X;i
zyE{FDwX37UX+cNFiNhU#v{gI}PS3jFc}DoB;MTdmI}Wv<{5Y|$VD0SZg|<d^Y}4`^
zRx7es)^H{5E}U?IS!}I?pkN?}h@fDl#Kz-(DkdDJGd@k;zx=(kZ~bm%$=_4=RmZz-
ziL-M&vsh=rTNcIoIotH-E9(Db<5OBL-I5>QthAt`LxZKOqr+y6a_~-(`unmD*V<wW
zs{<xqShQBts)i%a++0XdPF~?|*MmT-qbKUxoOT@fZk^1_?R)oA#QPeJSJSt;Fq?Hx
zoOAW`LkEuO&);9!B`he|>L4gM(Y>(pi_W6_=VrlmUVc)3+xC?IuRL?9OhEMi?a6am
z_ZB`7n!bI0U;VMpJ+W14)oZ<<hi~X-k~nx&le=-n5p~^XyU*Q!G41m?_QzpDMK9Re
ze$*xExNjGfTlnyytILy%nHwqu1yA15(|vZr-EWHAC06tFf)5(5CkwkfY<Bfuxcgt(
zOl%H!xI+2YRViolj)>LuJpRijC@9DtY_I-g>B=X%4|<Y2pG|pFK3O<9?ctRgfu&JX
z7#{q(R2akGd3d2$@up+(k|s}eJB`_rHaJe)wBty*LGlMZ&u@G9TH00kE3b?Fb39md
z)*$KWsZ}$h(xJ)VQ-t5yry1ARK9RF%_%pRT*+{ZbJ9h4&rK{B|15K`*>py%ipROty
z@{diNZIAmK_xY377cp+S`o`tytho#uj>bGav?Z}+jcr@-?e&`XH#!{(Ve9vrGWWmq
z^JQ-o<KH*5$R{^z+8;gs_teVrY85_3#)1^S>xX7Pd|5Ez!rZjA`mm&sv#mQu_M=_+
z$;RZFK|3lclg`W%Wmn%2^LzH}{qI%Y$z5)(`z^Mxt;Ql;tL-G`qa>#eyAR=qypzQF
z(%v81z5ngh1%)dkc1XwkcvpQWr~0a54=jyDnj5gcKP2<#gQ8K||D*1&SxrPAHC-26
zI8|9m$-lv>O%M#g=@G;Nr2-I*Q()qAS7(=wj*cV6EAGs)5tZHX=kuZ6{hU8m@cb;O
zU#q)~+4#rz17CmdGx7d>Z-@Aq#;2Fc^z?5f{@=LnjzE9cQbVhzr}w+>+}Yds!ubEA
zd!PRs$jmtSmHkYxi;Ih5fQySt<`3=5J%4O(Jr2FWJo~4}_4+s$o^9#|&t;yx_^QYH
zH~!d77W1U-_v~4$LY6yhpO(U$x9;}ARA;-mAJ6QcXuq5Ir}S-=K(Vb%djFd{9d}lN
zGCEhIl9JL&@A|8n7DWtkteaHq(oF3qu5QrFWe;BU`1HxTjK^pABj#9rJ;8Q^L%HOp
z-yQpZmUaxsifTVQozwp#+a6RW^5V+0IdV7OUVd_B;a3eMCI0W9mkV`tcq~ihetd0Z
z?w_x3Rg{wEy;-x?Twie019rb0i@C2di>_$q$FDiv{=@uAzqr=ZP3^~vr7izH);Xgn
z-g(nz?&R8C#SdL97}l|JXWBxG6OUI;4>+Cb|IF)Pu20-%b2D;w^D-eL^NtnT7gzpq
zUDp@R`8#xP$6V<S?LSO@+vgiEtNF<5#OALQlpWyWBG)*p8B{E2%oo*cT-=?#>G-2{
z@kghrn!8QE-S|n$&|UwI;abJ5?*o_1-JHXyv}4=9`UH)67j&*F&X;GMmHXgNSF)#A
z0{4HJs_vwF_bd)fetW{r#YKe4)y2hVvy3cz;Fd|@n~pCk(^r1f>h?tQfz1DosT1V?
z^>go^U%lnK$1M>P6+WKdon0@RCL1>hSMKUp@ZNZHj)bwc;)lCGR`Ij%SjYehI@hy~
zt}ZTe3UBQ9-muS7zV>5(ZcAa^5s`ELcYAm@w8}ru_<gNkhM{nU-@6~n7Cii2zrEe+
zHs`;{LY14RK^bqU<o*YD&wMZA31JX{7F){|^Nwfu{rShwz;;LT!_5S*x(gf}kIyV`
z?x^WeRF$cDHSup*@P$xoVY&WUp9K$c|MxLVZ!XT;v%B7@HE{mpS+9;QQ&Lg_rxX6S
z&-#^j=DnXXS?B5M$!B>FP3wDDb>eYs|LKSEa?GByn$!eM(+z&J`968|^FvC%wb%-`
znKvJwoaSr)6jq>=-0&-?`s0;Xu9BtPa@v^B=9zwBQt*Y@+h;c|E2&bg-+zDR|K3^d
zA(IyB@GSc?MZ!?RJysCzm6ZQwdCv`)D>eTV?mqro(eIMT7q3an7Mv?(E;u*M=1ydp
zkNS?=7g|<NWN3>KJrNmU{~Ts-$dexra_>y~^EPOwa;3IyRr+mCU&r;bemPuOC)&Ti
zejoJh^!4|16!OfM$IE-C9NN+5m$E+2Q5zP=^DOTF6IuPt_s&sX7XRC`1rB}MYyRcp
ztQ4tJ>SnBW_gZe$T=7=vVY-r`NKVoB2FdO7CM-4=`q#s|dsV42H1Jl6PyONldT*WK
z>fk@Q_B<{HWp6ANP2+z!|MR!Dg>2h5iBzw@7^}YdXq3?Z+&(`+EopTV*1fwj!#6EY
z(u-s6c_MA_?A*EdNq5{clzTd4#kX*|LQ93MK2xP0m+kIdUL1BO=w5RAiR0Y=WgYm^
zJMLJuW~aaL{5W6mv+z6qlGRtIJLT5ApK)$)RAY<%T&{gn-ei{lH~Q%7*e_V7R{5)E
zuTbLr)?!7kKYz4YcWpUfm~S_y);gf%Q2Ub(`NeicpLPEAUdU)vQu-vYzgnxYqa!7X
zuT<mTXZ9x^MU&?roH9S@@brXk%hMnA-U}saxJ}et{NY4iy`A^Z@Pbs~i){H^HCN=G
zw`lfOl>K74zGtzeS)FoZRjvEXW2%$(#$KCscV*OOYx8UUkB<0@pNO@ZZhPWwe7*bn
z1sji_W6&>rea4gL-q916Kb3yk;#1Y1U=lvf@4=fM->80Q3{TQ3jJCL#eEM&Evh~52
zac3=Sk3SZ-{2yn1?tb#76)Tx<SbwcEK6lW5|DQu!*Q6<3f4k%J%LlzO|KIYMhp$oP
zE6@pMzpk{ikl~TizlYD>TVMQY53PPY&GQ6ymI+S$&N=^~wx7zMOul<fmJy-Cf)kG~
zuLk9R7ic-)0xhONG`L^@F+k-uh$bu`*XXG#AUN?$%E6_pD|TC~6z~t6<7+AaZ4RB=
zvv)7!jQW?Wx<2w$R8*|sP*RedaxiW4%bNKe9ZM=VZru3h=4STpqm%NcRldKs7qlp{
z?(eUQt3p?oY`(+C$M@sMj|VSatl;%`QDMF`)j?3uQ+&>xIV&PJr|pVL+kBJ9p;Jh8
zMfCP{VT(FCmT<n3a059SlE6SL!p<i{1@U1sbPd16G1*H&N$Jy%b#Z%7E%ly0$+qm-
z8A%4cxIHHpI=4@<eg5v9-@iXULoEaaE1&V0%|5%#sH3C9X4i@p8aB4Jon2j3D_(-8
z`93~68tNb@Sh?(qOo)ri6T_Xmb|o1|$W>YS`}>RfLsQ@->E?r=`BnL>S<`f*CmE;v
zA*2=`IB;P3uPQ;oiR!N3!7XG(Rj(-zo}8R)^J4kcvuAs?!`D4{^=j3k&W;nt%e5Po
zl$88Eo}ZiRd3NX3Yipym8DO<=)%06x^Rl?Kl$6d-hSaH!p{v7E|Nr~@eUH+MhYt@v
zJw4rUrb)>9jt(37T|yd4N}pI*+1T!F_snSLlNDibb$PNAoB}7RznEYDPqF^rpYJ?P
z^?yFHZ_T<Iw9v)n$<B+Q1Ugau#O>R&tx8|r`MBd?f<pbjKUda72D=JDtF@jb#m~=`
z+oWy2$-%>OB*9>XgObvxn3a|*Iyz1Wi;0Q3&9kXAn0@x7-2eOk|GBqviH0Z$3RbQw
zQ4Mf$dBRy-UcS8W@iESNX&%th&b6S*7hJ_jhF;ppkZ;Z^Dk$j54GC+_j;=1H@^^P!
z+})WAK0Wd564PCEsH5XV@$$y5C+pUPX(uKoYPm=y7)YF$sy&@cQL^i7+T&keU#EQe
zJmKrBtI1zqU9G67n4s*=w=Qn)tcCXZ=jK?NvMuWHsIu61WvYYc%R{Z)SH6FK{G?M@
z%T4Ox>({4enP#7uXS=&g@#VdrpPy&m+M?OcCmUpFG^6I{C(v<5T48HeB!L>SUR8op
zQeJOwZMCfXV=*sv)`KU{RxYwhzj*z8!QU%lSyxxN1_TI1Zb}IZRJm(a@<O0j+Wgu4
z`hTDS(I@BTS|6@|*3rSSDedg6hwW{k<@MgvbPnFU30iga`2G9!ym6oc;L@%Q8w_4u
zU48suGrQ)^Kl?>i{an^H$xSBOtfTn0x{!dt0m#Chpg@(oH@9R8uZ`NOWMwt$<YaZw
z_83vs=AV!I?O$A22wFk<=hJEZl9!iUT?GX_o3B`}VEAUd1LUR$2b)E*Eg~2VdO<c+
zgugy^Yist)3k#XWx?No-mVnk9n`B+#Xjtk!U5g<tU~BgEw7<W;R#aAA+?pNU#w)#S
z5jZrWL}X=sH>aJwvA5d1_4T*K9?kqAQ%sH}7<>qNz46J~3w@wjxW(@Lj~+b=k?<4;
zjn9LQlzMk(C+NV9(%08Q1C2HqrJmYQ`8n;>6wSl#^0h~Hzu&j|@6-;DDC=z@#TU+;
z%`v;YDpWgcZImnH{p*@{i<WdvYC9UK`^R!IcX;2rXT9e45-u(A)aqIiX*Rp}`~CX)
zhRMeQR)uIz(~I>2W%L<7kB;?9@A>s=b<W*gQ$foQil6z!#KeS12u?I{2xl@jTz%Cm
zAwgj^=vZ8>#wl5bPOhG><0BS#a?Pzjyg0(P?$=9oHa?jZi#%qD2n#zO?~|1@PUA7n
zx{~ns*H^BFnLd3omZ04jxwp0?UR>n*<M;3E7mA=vQ#mikOnPtC*QA3@tQAvpJ#TKa
zPx&$Lwb7aB*Pi$+&D(q;@<l2rNcR2t<SpuYX-fV7ztbBU7@nP-?Y=r}ZN$bT)_3pT
zY5Ax<dGqGXmdwi)&*zrk*p@48nsw#Dot?#6J)lm%4gdQ+pZWIx{U*KB+iXeA@|6-Y
zJz8E?NhL*#KV^SD`7U;zqI27ayXE&+YD`iCEq{vInzb?YwAin2Z%==DdAYExEG*H;
zu=yZpr_zsaxAQH_-bmQl*~!>c1hfhYR?hpKc2{`D&zdh6-Fc+Vc)lxbUHRu0Q%~P6
zcjdiZy?ajG^Yq;z`Ab|_-EWEid^@c!8*|VKgTlwhV)j%_T<F~X>EG}7t2HK_`SbHL
z=<1om!a~pn_8&iguG9dv(mrje{e0FOv=>i?b#>6v>p?!}B15N?=&9MX*a}Mv%9c%g
zd3pKeoyF;*uJd-@um3++OgBnERP^W)&&e-tY+T%>^!-klRoNSlk`fb85X`kI4GMH|
znP+Iqe%9q!pX})e2b)h$`>v|Be_H^fL36>mdDFG_PJcY-?rhNE_Z1(LL|x~tto-~;
zP(YyJY+CZiM@P9DisDOOUrW8atMtjaxw8)(aB%AhSe^oQYY1pDb=}`z51&QO(b_xx
z!HZeJt|uL0Y`KD#eh>0lw-c1*9336Cx@?3OyZ0}1Zs)70sMt{UHp=w<<Pgp1Z922f
z^OuFJ466C@u>H&H>;B1{T^%;gS$wlNLGu^~4miXVi7lzRZ))-ULs+2EiVpAg8^42%
zd$vV`rtWTS&0ekXDda|u85^I>i4}p1H3Ap0cu&_`nsBg5tH)-_h7AUBwO>QG<=^-F
z{q615i4(wygY)!s{pF>vug$n)v+`OxZ)A~ZSfJ4k>F3`z<!5=v%;CwowPWI{(ABFn
zKDit{b_}$K>Dif?&HerSpP!#Ue)zC+>qK`UVPWU<^K3y^CxA+Uo10RfoLPBIOG(Lp
z@~q~x1kfJi*xh9kv3FKp%3jHopf+)FLa0{ReCIE^#=Oi-79}qxbSe42Q#sr#ZIX4R
z!?O6<g*}zUVe4X6GHCQ4y?Jxy%gf6@KbxKZ>9~A-z~(gHbMx)j9|Dyvrw-rRmMeWW
zddq>dlBGrGT~p2md%2!;vFW<9%;~vgOZ)vLjr;A@{pJJ&KG_(%r(z-~T~~fiyI1j;
zH|^}K*6!}M857kD3?4l=aAKnJ&tI?CbMo;WJAT|<eS?e3JS9{1vkl+g-}m49>ucw}
z)TKr1k3P})vZ-s5XRO_xo@V3SllANEw&mVkRlV)WL{@fo(0XsZm>mZuC^}166fEdc
z@;`cKXEEpqsdekt>BR1uQuXx}C`J3vSjUoffO%^hBcqGk(SU`wbyyGh==t9io04fh
zkNf%g$IqX;dw6toDe=E86W%*vf`FR;yfq9O{-N!BvY^!mX0v-i1;DwvtHI^#slA{L
zF>L?-wfwDa=9;^F;_3)nQP&gUONEz4)gE6Ny!^xW@6T^-&DQE#a;u4z`@{F|$CEaK
zw&gN0Glz;)zIt_i{rOv4vrkUd7MHayOS!TlPz&6sSW@!v5bLYs@h>&Mzw)n`mbYe+
zM^lsCUd?+yv_Z>b7Q6Lw@biZUK55+d|6esIS>3sNmy?TYkz+HPR+o+O)6>(_FE8_*
zq7ylZk(uqozu)f{TStIeM5=Ka85+FrBeu<ftp$)Wd2r|E<|&!KnJcxoga$t0Ykj@f
zU2s4D4bT#vStg2FT3)UbcW=6xQ}f{<`=0OjsztQJdVFV_1#V1o6?L1pvx${^Q{rJZ
z&<P*5RVH(-N|nsa!dgKcg)?^!9XbT^|BsK4r{oy+{0ZH?LZit4C=*-ohG(;LZg1-a
zEu0q?UcAV|Xye9>B`+_vPE>Xe+FMm>mUF{lp;K#!L}lKcyLTfuCb8;9ZAo}{XD15_
zOTpt~yrOO{E{X=lyDC4cfvU<23!O!{fAlz+{#vE+$zkcN|Kjhn?t->a$Xb^jN!qwV
zgUO+y!eafNPh6m5r7|xsTNAalD`scWN`p^#cJA0=Q2fjXlql=||1HnDw&vu~Zt+kF
zLBZAoXH>1MW-arby{Y<p9@q7^?t=DTuc@#6j#+U|@p*S?srQw2v9o)n%`dGE*9WaE
zTD^KTmvZLwhNA}?FD`aBPB|d}TItf?&+pzR6PO5!4CD42HzGibE2im2YdrXEbu?ab
z(V`R1h87k9->!OAOv^lUW~TAW+u`98jW;iL?|=08@!^Bb>_7i}KL7LI@ArmMy;?m>
zJek?~Qa(L72|BH6bDHnAoSTcl2{SDxZ1vOw2OPrI$Mu#^JqNBcC%XUsB<&l*@kg{j
z>FW%eN+V`Ao`683g=?d?gSL)?8sJMjCrg-QOz2X|j7vyR=oZsGl(qHJhD7IQ=jX47
znCoLUyLX;#^`?S{PM@Y^AN9^Xo@gKuB2oE8w&CPVue)iJH(xjnI?`Kz|DPb&lP+Sq
zQ7!xT+ox@w$;vHuVQIJ3v`JqUgI3ta?iMRAFSjarap1<r<WSIX#(|#abIb1~-rAD+
zL{HRVXJ4>^#fqKNZ0r^Xi)rmQoB!sXsDx1phq8NLNT5-peEpw^Wp8gyRCecUYHH%(
z;0Op*xw|r8A=7NL+(i)^lR&$C8=2X&bQ}dIh8Wz~vfW3VjaTZ(hlhs?^@C=gTCaQH
z#fwFYP6(&=?K(EE(|3ntRS&1I+6%jbJg3q&fBbk{-Z<xmL6?~BsZFV;D?T0-50&t|
zzAkpR+wX60Z*I*NpKDbL+J<`c<VjC(TKCz%@87R%yWek&i$vCVO^x$5v`h~Te6q5G
zlRxqNl4CBh)`!2mynNNhU@>d$=9%+stBsP6@vI74d+Jat_sO}|+qsk_ok~hf?(Qy6
ze|l=_mG$xSRlTQ8u&pkGB=x_bW&D4CeO12Aox1YZF)x|e;#C@-O8A$oo1c7s$+07D
zF`yRAmb|-Bt|u9kl$0`VZ_|B!ZSCW!;c=kwOgTBpwRIxT(X6eY+8(s4>eN*2<Fib&
zKYaTZ22P5X`&R}pzqGqN|Bg*}@Y?rTbMDxLid3p8t)G+deu>(Vqb(b2eiliYW-VE?
zBpI|EG4IX{`~QC?gFKRUMuL}7QS#P@hljyu{BVgVXlQg?TI#(T+<?fuKgY5d)E`kz
zcU<*7wz-&Z?#U_X+K(<Je)G#~Gt`^4V1dG`D=QZ-TH;*)_p5lXl<A{4H#gV(dbu2Q
zCZASU3YT%(nFnjP-&2a-mIFGUA!=(Ds37NR%)GuS^>oJNWxRcPaUpLv&(hh+wOZp-
z%pHsU{k6eS58iOhHp^QQwKYr2%W6~Y@3L9u`R972%_Yq9WbW=R4-Zrc2DNbKT9+S7
z+6XEFZf(m2mFWu<l`hTs`uh6ieYMp+dhbHs?G0fJn73MnbydK|-!XYdBRDdybU3&3
zt<-pV<kC{_%nJ(`x98nWy0@p2t6`F+Y0ixa3l=C8Jvrg{``g=}58LHKz)5>c>g{d0
zTQY76&AvGGl&$#WtE-oEDV?9|J~1lfg;Dc|53@}^e1CqVQ&`JO$~^H9%dIV$&ezsN
zMs7~y?UgnU3skwhDrBV)C~0S0T(sx+yWKoeCLW;HiPHkF31*-rXW_F<4(1pJtUO+`
z>ACj$MIN&z?oM$&Rr+h@>hN_z_s{olPCvg&BlC*-{F+12`FmX#I<?-|QK%fcDkLya
z<#P4MqvD|Dk)VZEKR!Gxd3}v{^UXDfIy{ywt!BF1@Zdp0*7cQ#_nYShtvqg1B)WR}
zB9B=mh5u#x&b)TMxXAV9u2SvF&(A^wFL87qb$Wkq@6Wf}?}H9q0F@t?gKfFGRYYZF
zeXp;Jjoe#h`tRRA(5Aeco10b|C{23#!s|hC*{ds>f22$l-rScCd0`XF7;tGz%~b24
zRaeVF>yg=br3&8OG8I+j-BI~j?d&Ym$J_7MEe>4lwkC438)Lu|y_nco(DkRb)!$P7
z{`v|khP|ijtu#<tB%pEr9O$^A_s`v5JPkNiI-@5&v-hZ4X1wG4dm{Cz8V3&^WMN?u
zkds@t$i{uTe*7}OxmGN!tQYrImxGp?%$_})OSy>OZSlmy$HzeX9sd3K>|aw;V^Q{I
z2A8rDSL3G~zB_;Z{IRHiC$sc+pK|_BGp(*A(yg!m$-DlsO934|mwBmW#R?74r<N9V
ze=I<)c8h`rf4iS8A0Ho2HkuhCF)`%FkH`Iz_H{Pm`f*F#dZlvi?~8Q>cTK7e9XbSB
z8V@>mDWyB{$@*KMe&6zl1ItBL{S>>sEq8I~>aZh6j;zo)f8pF*>y?3v*`A-9`?y`c
z?!je$`_MoaxuEaw?#{HYw=*;}{PFYW#*~vnqOL9?OixdCcXTWW{8N#Al*w*+K<_@S
zt|if`v+KS0W!(j33DDj?P_23Xz^2sGE8_Rt1qKG5nQJ}$dR(=yt6-%IWT8>Vo12?2
zFY}c?)-MmL8Jrf_SYEt%(Iop?k8Sn0fE{u5T|an1WwQ3=DxOO1E$J^WEd?E?xnqZg
z=u^#KFPF~;EjrH2%LDCQSM!~<LPM#4Wzo}9Id^tUJk-kl`P=RM&nMOAuh0O+V!ytb
z*|crBw>PDp7UTN0&++52_*EK{6eVP2G=9Hwu{mv$dTmYQm$$1|-}XP9KEKv&zFn<G
z#0CXv^SmWlS669uoe&OM;xVyhqv`&?-=cSwzhAeJ#Z~38-2Tit2bQ1D+6r1iCZ-qD
zQ9kwDo-6O%dDxn{lqJ39sJ8vxU-sbFNng;pS%Hh)R%-Zn98G%IYkm)OO6<a*rCvHw
zTR8Of^+nyDfCfYsWnW(h@=ZengP2~7$F(()p%Q{814>jsXl}ll^X1LW$*tVtla|M1
z$7NajF$NetJ9h-q$^5y9f3bW2vu^!;0fF-xPMq)n6;=BC|2Q3P<K^Vy3Stnc?Anxa
z^3a~j&!A1`+1J-C^_y$e($ca5oW@sv|8_gyIQN!G(ex~roqdxHEhIukPHyP&i(GLh
z+3x0E?GIPO;|+_Rc!)kVY-wu?+MXACZB3-}biLS!4GE3M7(^<?ej6>G2&z#*Nt2nK
z@4@TWr$0VE9x5Ri$T5Xo+9X3jSk33a;tW}>l6h^-7Xw`<@|=G%X|C2@P(dalU-N-c
zSlurqQ10N#$?6|JeoVZ-uNJfl&vUX`CXd6!^f#CN?Vp0u<(-|yKYsn%RQ%je)YZj^
z<*x9LDLk@PQ_kD}p9Ak*?waplG<`<L={GZPbgalu2c7E{ySpqjFevl;ySpa2x2Av=
z;ijFDsQUV9>7ouB_toL+mt|jHcO%CPwBoN@OgBUX<mxBKdL*6y|NDFLn^WYfeH(?`
zug}ryO7Yr$=-lS&B;_P!J;VBcHMh28E?#7F;M7#@%=`On&1UyzUS4)_OJ=aBs|xRp
z-R1hZx3@k0_4PGq`@LQ5uN4|flk!$tz7RG`Jtgw%`}_Xc>71*+#4Tdbm^rJvnsL>a
zxVFX7c~)g_PRugR4w=IK#y)CGM&s($+N;CYhcSqFI)m!_W4+RvAu9xGe}7B;@Zg|U
zSI3g*T|z%t&d#%)4O&?~X|uxB-|G%qwtp{<S394>HfOS3!p1_yUxLe74=Lri*c~bo
z`CudzaM9(>#)xI}cmG!lUpJ+b?Prfm%r=9s8%&SZ$gBuGCiq^1`~KT)%WmdA%-g#A
zt&_{A^Y5Q!9k6}6Htu_bf8^etEt&}l5A9CZUU%M^q_(QsN<cI%_1nhh@yk0VD8Ko;
zQuEl3jHafh6~W8<$|N*F7nXu{eK@spZ7g{i)FrMzFVQaACq-r26m^a}cRWJPBMl8*
zqJn&Kiw>Us`Ox}i*Pgn@ztP(s$LwXjx;OUS|D<e*-DWw@S6NzL{geIp>6{X~{{<!Q
z<@BDd@H##H^U~G3j*APJZ+}v)k?o_hJ!sz#xzcTM|7Y(_f3zZM>50GB7n=4oSN99#
zl`ZYb7qv0+*q^a*;X=^5VrF)}3ya<PK^?T;rLHP>FUGy*jNFzZ8N0jesAY8AsxNWV
z)R-283%a+N`$)Pzn(?n#L|V?^U+&IVdORyqb|v3(-MlD5Y~ruf6%St@l71@uy!ey6
zalw9pj0K^(;$2dWNp60BM30JXZhWU3yJEiiM-HWDNiq|{oEJ2{)4i)vz1>eHKT_h)
zZCyQ;$`x_8;<v<o>PkfF)Y|?>&)zX<(=O?kfyX;nRrV%q-Bdfjb@KO`diGnU1q%fV
zUOJd;zU<(->1l1}sr9-k_os-NWZs(MD{UUMHp=v^#fE;y8{6|ib&=Q6F44#>8G@Cc
zo}6@;Z!jq|{_5k+Q%|1H+9I)U%6}8hUk#b-4fd_Ff1G=A|LcG8XA*sWZ(14G)E!;F
zH1PhBd0F-|ufJ$i{P1j}>6fbi(hOETaVOJxKX3Z{{NWclQ}c(W`{d7i9#;JDu9V%t
zRN~CGjk2D*x~+D8vETAHEAvz7m$OX&E^i7+TE@Nqz%SPQ1_@VUdY0Nf>1TC4Q8!<8
zM_~eE<KAOQ1vTCG`9n?`L~ofsUEI;p5!8zV?M$06LE!5hZKaoevzqM~UteFpy!Q7u
z&=AC(YR$-^y))FNX@A)=Q*Lse(f=i;H?{KLd|q&QBU{u|Unz!MnOkDM{hs{d42)%e
z^r!Wv>+qZUZn6AWyME@||M%+@rirraa;9&+9P_8xWWs&Z!kzBtJR-|2wy1W=X)-As
zICb^no^3zGw)0=@d)wnB#2{O-+*how;_6j72F6D@cXq`;(8^d6P?K@L$l`5?mV2XC
z$Q!%q(S~A7Ql^hCEOee(cVAFc)OB^(+8I`*TD(#w3K|+3&8!X#=285&56{2-?DEkG
zJKjhBd%4MaA9s06hy8VR?uMmtf4g++<?2>2a7<Vq6}+tGZ^8HcnK8oFpHDr0$Yv3|
ztdKv=?@!AkiAVO4w}gcyXRdk2{r|S|&CmCBZk_xe#NyB}X?@+{h4+^S-%qgD`H^((
znhK*q&jQZBD+3lAGCy+Opt$@p^SUQdH@7F+GMwF?)8g)Sd)3`LGwkc@4lVdp`s&KV
zPft%<DF}i3tVy@G<(lMQWP&#|Q)WdU3;lieh|15~o0kYregC;?eNvxDsn4u!Idd%j
zW+WaGlRdu0H!WzsY_3m?$k9Ey+h<tl&r<%_KhGjP^<ZDbx@9NV9&q^YVE${0$G_63
zoe#y@CokFh=iYa_6r&kD7EFJ#)9%bKJo!qn=;WCrM=#DXU7nX&<9(BP&(nDA=tY;p
zmfCb(S>!Z*CS*XUewkeP>r+#;K|7SDYKIF+NSpw5fo7RL{-%_3CQ$1C_Bg*go6ZJD
z?s;3sT;Trc_?y6!awp?=omp(ES@P}3@|)Twrc3^*pI_=BY^E1~WY5#=m!YcW3^(_3
z2P_P$eI;?}ng8>(mrt*Zsup|bc{p~nx#X+q6<0q$IQg4J)Z0Wn$865--le`jmwexB
z{n2S>-<rFdX8pT5H)yv3%UKJqwnr>K#7<4Pz2#&IXgdRFO!dwE{qphg@qTkG7BVXf
z26CKv@$LQn^V+_%5;j@|tUS)N>ACiNGx0CedT-p`|1s=9#uC@5)$*QO*G@{Yab9fu
z<b%ZbA_=)o7eAiuun3dgw!!$#xrU0bVYjsN_g3kJ6f}Cgo_pFlH$3N1{JWz*Kkg*U
z>1tUgG4Z`MU%un|`D0D8X=QE41QUJlhVM$N`+aIx!I?iE-}CKqcY6N{-gj9)W^Wbm
zel}i<>TfyAZiCv&T=DVo-`?Cje4vqercI?$U!P>XOkh7#+{u6QY^qqVzn|)^_wK=B
zQO#LK$D-u^p4H{IUMsl%kg7)T4eO)t^sV);`8U5l^~pRritkv2Wpvlm(#KP`+xz{<
zzPJ4Jj*|XJmi4g*ICE<5?RzHsStI^+>K<F04<}~rI@~7RzJ2p0vG$KouZ4U&b)$S9
z=%ifGrq<2r=O4X(eR^H&?ve%-C9X!v(1HR3US_6_p00&S-)}zA39<FnU(eZLc-!FU
z|4sWN4o*xe=gx^!TlK_5p!n#a_=hWB8lPy$H&oU=Ju4&Vl!t8o;lKUCYZ?|a>uOtE
zTGzz+zlpCrHz>8Bbb+Vl<HM%gi%bHxPnlRIDw7#`%H?0R%<`$aNlQE(e;0{;FuFB;
z>6Fep-)H19<}2%-wz{-#&C9;?;fJrc%Bj9}J^S=sL%JFBJkNXPUuMh<es-k!)U<u&
zo|pUFUOg>lSUuHL{gGU@ZMt^Gk_pNVGP~u9<NPOw-+Z>`3aI5Cb6~wAvy+pP(b;{;
z=jYk}`1K34kISuJ4s>j;(b;r!ceh4o(^W^!m-kqkwV56?fA{Ow-qXt(oK5%1eJynQ
ze7d^(;e6xcoP2iYK0ZEfDKAxbKiqoeoC8vB9}d>a*hrk2S-<Z7=FXpAKOeW0|5w0h
zWc0=6;6Zjv37IoLFU<Zv=l#d}H|JgB_B(ty*eqjHXBWA;H)YMkIregA4mPuM+SS?p
z^jtoJ+vn~5X_98zW=T5*C7n}J9?gmIVl!B=>+ipU*_B${@5)%OeKP;tzW(^a5c@3g
zvdS~2xlbRog)*4r#{J#*oi|Mc)W3RnXQzv+YvBI6+ORbd3z<~~JwM-ckh=sL=*+&*
zu%SLbVB={Wo33lV;+g-?%-{HA-K!ja>pr!pX=M*~FFx7qbo!Owj0W54vneZ!|DRnP
z6X||$)ytmjQ}WK6r<!Kx89twH?YtzK(f!!fLY3%A$9LV?WRzTI@_I4P72WJZ?^WHo
zo^EMh^!WIq_m>&H7PaluE76-@QyG1I>&EnC6Z?4`%By!zyk{z_`7l+fTRCD~ghtTG
z##NQV=bKHcZ=U+T-*xl<GJ&&cDFxo)UCTTc$*!LvxmBa>M$oyFR(n6{JU-N6)Nu9C
z1I_L4|8Cy+WP|F$$#ccMtUq4rJCe4J;i;D8Bw-HgOuuhG7Vo$#GR5SEh>7ZZFWHU9
zT#8Cg{st|5*NxsbK|6dM&))YDTeC#p-rjyZYpYTAwKYB#4@y*){W@E@J5jQ#=8H7L
zVfEYvPj8r=^#5_k=FFLo2QA9~Hr_g_Bv~{6+>&*y>z|+Id%yYQw3_&|nw{eNYksRu
z-5c}H$XB5GR^7Q%Dwk(S?KwK>Vg9E@n;U1m7QeAMS@yJe&G}QdX>8|idG&u<De+GI
zQ~t90t9^;ulg%SP)R*hTf6Z)&{&9azoY;|Dyuvqj9CvzIzHjOF$8Kyrann;Q?NaXE
z$g5DB`R~iM$Sp5^&lj=&TGbQxi<g&s_Ke?Y-fr)=wcEWu_mE|ejZB08v$qNxe9RQ2
zeeb<Ez2R;$<BU(IqpYsKnfAtRZ+7+djR(Zs+wFMe|Nq+?y*#pTw`RuHq?u{2^DZsn
zTphM{QS9!rEjc$q#{z%)^?Lp51)eUKa?B>Q7ySEE`Reyi!79rsMY3v-BX<S<iu_Ta
zoqb+qo}FI!MZ=p{^_y2uWcw-eU|Or>^G62vmUCuY)jG$s;$@GvS=pNIMaP%S-m}@_
zR^9n|izgmCyZOS!*XLexiIl0#zItq_c424wp4gs}%=b<Hn(n)<m26^nVEFvk+F|F~
zINNZm>E?oUhSS#9?>oufoiCWgY7lVt{M2<HPk&6>_k)3BhK+B?+3W}JC#Rjex9`!*
zq$R;`e+WFyNO4)9?RxjojZJ!8a+8%Jh3;SU7V~$v%X)ldXYj_F%~l(or|;*#lk{PY
zhnrrH=e37N7v9~dGN-)U<kFs;OY7%0x-Io>=w16;r~hwho#j5w7x&e5GdF3zkiFgP
zuiv%bXv%{J2~12(9Q^#}H~j7c?QIm(jRGA%uqt%5Q6huSiN-14@~0Nm&r3c0{7;?T
zPxHTDc3I|ZI{MM|x73pd_tK5^-v900wf<Z3<b{0pYwn$Vc)0y%ara(ksn_dPmVA~!
z@$Ypx+lj(v_RYI0V&m$UM;{9HJl*zj%dY~xhc27)?g+T~N}fGs`|b1(KefZn_S2u{
z|9gFT(!=aOGqzMoxAVVG&Z}9j^ZNK+_w}CD2gJ_aOSIoq*x&j{!s6HA<2(}CX$LkW
ze#w6?ykF+<&(h!|;g`jZ%9$6x+i&?%VRZYy&6$1o(=YFCQ9hEptiD_B&Axf9VZvq~
zYsI(!_;9=b>h1E4wWr0Rruo?1opyS;<laZt&o;i3IoI}}BSky)^cv3x0sUd7Tz22I
z_I(Ub3cm2^-ptoA#a~}t1&v;@a)~?ub<Vr>_g!e5+@Zm;tQ@qPY`r7%x%t-aGq>4K
z*j_WcWa}Tb^oZ@sp6Z4-eYfb&s3>^+dz0|ZhROfCRfTq3kJV=jSB_nEYyRGn4`P%3
zZl7OvpJ(f*ho5Hrkh9@6;Vam=xHI+4{xmlK&L@vu1Ha4_oqOYAcQ<E!I`>|6SEK(i
ze{TK^xznCeV19AolG+<B54DplGuyNmiXD4zsNS;XQ>xD$p3Rr?FZuG_@7n(-cbidi
zl}dbG$HVTM7h^ikAG=w6<MqW=6aRL-yl<WV@$*ZI1IH%uZnu#BZ#8*&r_~LM%A?aR
zB(GG}>f;tP<7;1P^P(r&dD1*5@6Zyj72aX_!l|(h^}1_1EB1Vyebw4C_`>3AC7WZ#
z*8Q#*`S^Rn-{0RaFYy#!v}lphoqdm2uiv*Qe0^L+b@gV@>`hIL(c8K!Wy)fcy$hLs
z|5UU2cJobc=UKzG;xoSgEb6h74_d#0^YmUTm#&|)IoH>+JIv>7UKxJi-pam<i%puZ
z`EM{+?7i)?UQqj8M$*g6xhmFTy^r_qlmB%3TqWn;X;Ef1XS<ryX4M+bblv=1^~;o*
zQtoN>=XqvNWxQ;^IIU5p!C2q4dh$QjH@{CM3tubN%+B0rQ_`@oPH)bEwvSJ@M{G=&
zS#O%_wO)VU)3-ma%l%icOD}kApv>lXIO3JF-P<?bWy%)MwcOw9|2iuEnPvZb>-NpX
zWuHVpZh!xAQ`%AG-O@L0Z3?`%@z*v^yT8r<$7}n`=IU3D7FFuc&)w5nJUv|VaazS0
z)10RdW*u3!KyGG%w0_(ki`izmi;|9Zf%fcwd3RU3`)E@~hf{);h?tn$;Wl2&%110q
zqj=QgubzAPnL+2!p>G#<v^0dgO+DV?yU^3HdPbD!ee0)t`}T5)m?q{+q$ay}DRMme
zXZZQ~+iuG|w^wQ2c5i<wKWkZ?vW-Lge!h2&_<N@9{Si4U|JkH=&wg_@u-9$ho0rFy
z{P@4`lV{I{d6jwhqi0kU2$!Bb^s8k};^X+wZx4%C>*kpY*IqnsnmBKA)vmx3Gk#pv
zd^_{kuJ!ljX5VmLHQP|JW`B_7KZcJF9+y7~TD<0@T)}0>;0sf_SZll%M&)nLy=@k?
zHOtk<$EWQ^e#F|aUteAx7LTuS{Q2oAXx)mc_cWEwn;D))fA!mOT|ey?|C^ruN#(ic
zv(j#?+4HxG;pXG`YIAe{cL|NUd@`>d#vA_JAk}80e|h`dy$g3$rV1Qap8P)e&#bFH
zId!wPn!G>rQ{&;zyX`{jAE$eMRo``J+R@nz5;u2B+g5+u^8b63ubce#_c1xsX3p7@
z)~NDU`jz>&$FAwlhju^h-9M?m`sO;>y9d>xWv(S3epyvL&$jx}<w}=BHqT@%X8)->
z_WO-T*N^SqJD9TLJ{nv9+Zg`+MAYve4f5Mpok{M0KF@P@)3Pb*{&G!CO&`90Pk(h~
zC8#1gal*sJ<r3Fj;Tc?@z5ZSD*VjZGcrLQ$qIzuoJlEf6uGAmB=h<-b@Wj5GNjF1I
zO#I`vLr7+Jf&ZPoQ?v{I7qeUc+ZM2N52r!Et_jR@c74x2|2OqvmT&duD>^rOo*erV
zFn{Ta>9vtb|F&-WJxBDoO7xbQKUUA2W)TxCCw=$1$MFeg9rPC4%FO$7`pi`o_lPv^
zZ{H1^I9A#x=XGz}q^i=ltWx&P!`-WtC!dR4DS2P<%&o=!w>!O)f-ii2xkNL=OLg{N
zcJCaO*Vq1ffhMOnBrvK4|GMky>gsxNW3u}+{dm8>zrN12tv0*4DfKen*Kgl-!c*_P
zk7_etF3)(Ivo7|Rx@o-jxmEMkk_z6qexCmMUiq(XcfR}2Bb7y^_waEabGnnpH<x#L
z%*2%Ct0(QX%3gV^#86^p;^s|HuSh0obbmfvGh=o*r(GgzdsdCjCh2>e&ll{vw0XL6
z+Gfc!6Vy|e)H>uZS5)s~`cQP;aZi}B{j(o;ldrF{Ej*HQKicY<($(uTV!Q-zoS!~*
z@%F~IEcSZ#4tvex5<Y(TYB8m%dzY$QeaEERzK*h9re{lh9fMEJGS%i5(|PcA`~5V~
z{_NG^{h_PF9=?3JQnmq9)|Y}tpL1_bL5x0o{9pKOPHXMMb0H`B5*D<r7p}Z3+h}p^
zP0gRQmgk4$lrGG8%(0VoXKaJTZRh>dZ)VLuma)yK)cwt5%?YX>Umd>rAYxhjz2^?s
zOuuf9-Ei1v@wIa^Hkhl+N<2t-FYv7AY2(4)jqK8%vzqe!-F+Q{wJv9wT)Uw$O>eII
z_Po0@43pXT_+G9*c<Act@XTv#BzKp+U6g#h&u5N>VK5Kl$AX?8Wv>`7+cP9Qzi~82
zH+*K|?&KE+dsy{tp4};~uyt{ES!}KP+WL^2HrsR^{{Lxe&WAz|KMH*vc=vIgdPh)w
z*Rdx#_Ybx_*Ku1Lwa<UatEbVW%YVFhJLkZ$jUPWpPFlWVuk(y~y%m2lCfyL(64@R#
zbCYI<S6^QrXy<N4b+vKQ5ssRgnuy(Hx(<!~_h4%QT)V|W3tgV<3@X0x8+3n+?`*R_
zACJq6thw0!sCU25_W7!nI-3o@J0vBXio4O-WvTdQ&P-nZQpIx)Go8NQZpoR$>szbP
z)74Y`<lqBeIk$ILKdi`$RMxkB@^QQ7zt5t3kB9m<@18jAsQkOv$=s%!-^WKE3SXa-
zSlw>0``c{!>c46xti1m9zK)ZxUOm{j>8<*M$Yq+F)6awM`g{4#c#d7IRpOcr|Nj04
z&0w+~sIEQFbuBR?xcb7zNU=-Xk1v<#@y!U|e!}_99kb#eZ|=4wcC6p&Y})nI_pbu$
z*3VBXcdTyM{ym<du$sfJ_`9M$$9{(oGqd|cmhZnN`!Ft6Z7x6GvtQ10z8gNNnpwVk
z+Q#d5c6A+WKHlhTJL{U(ryts1%7OzH)SXSbbzm~+z6j6~v-ERwJSCJvK&ugWH$|;2
zd3B`|bY5TVQe)NG{NBgiquQ*)N?sTm*Uy~HzU{4bu(;c|IUY8TG-AG->ppqZMpo!q
z<=1D;<_GtDwA~T!Eh!=YtZCZC6SsNa*<8>IJUH`u)~*1pt%u{5R-6}Kz3ajJr{_}T
z?p&LCEp^g%&8<t4D#hP<2R<&~K7Clu+Dg@o)pnlU>?XC<>n^&q9*92~y2oy^y8pAk
z-|zdIpWc^$->#;%*0S`Kh_ZX%lh^C_A4@P$_;`O}guxS!UmeoCyH(@&+x=zIoZEHI
zlPlEsO-%aRb;;#6^=}{T{2|QET3COfVevMDf45Tp=S|Zrd)~3oZ`qamn%`sBbN-Q?
zyeZUK;&Szt1;3wpubkZ(+%=~<hbKAw?#*@oj?Ld(x%BYzm_4c9QJ2`?>a4uD&;8T(
z?CZ0*ZpznN@BMiF-j8Jq{;qhmw#`j*@4sKK&&;!(-O<s}vTPw}L04wS*|f)7ug8JT
zpLwEreb)3_|0b;Ce`535@&V6|R6`Z+TJHZoJSx*(r#)*?<rllz)4ASMozr5Wx-RG5
z6szZ@OV(C@dMPEc{r`{l`G%XVj)tzB;<z&F>yJdOHDS-*Y_p&FAx@8P%IQfp_bg>6
zw{O*T7kb}TsQ>KDg<H#7{1T*o?fTH!uWYE8CgwI_Rm00CSqH>&pC(MwE{nC=^+^7|
zr0w0_nro`uuDxd0atxa_H~N;&v-!3@cxRGZ&AY<a;V(Rm>s4R!{(rL2{e^(q(M{GB
zvxR0gy|c_aD`BlRSy*)UxkG<l<3XbY&*#_Adp6xtL!$$9LfPYEylLm=oSdD%Z{^%f
zBSV?{oB3bwdwr6v_QbEC$1QRuh9b+|FLRwV|K2g%^K!%?``E54UNiEJzqoH9GS8&K
zbwlAXeFhWxOP!C-wTUd$t30)9&*^)Tz0ND^ws`LlH#Ge5*1pQ}<m$y7JMYf@@b=~o
zk>#b+FMZp)G*wOa{;X5?KQf;=-tzzK7e`*nvQ12rZ|!s0wA*0LJGm)(aogW-=(@(^
zQIYemU3cb|mnDC$+qYegP7mDt+5E@8|Ey2%@I)lNX}%g!Z|^M8VA}Kexc>dpe|oN2
zA7{otnz^y@?W$#)oFy*T9$GfFdG<7h2g{c&_<jYl)U0#!W@FG2pI28_YIT81bJ6_$
zf6Xp0_g61p={DuHg=Dj{mG0!lwp+feO0f^9TDV8}?H*zNw!Zhq>O8##3E@c!CB-He
zE?n2Ni@0GvdHU)ZAAEjT_f5=BezaTM{KB5C+P-%f-I;4|?+Cgg*s;YRbFJmW9S^^4
z4BDu5Zi~)~hUKemm+TEbc~7P2@T%L)E35<5raL>m(4LWE@SVxEr{b9WN{P)@6XZKA
z($ijVJ)+&gdTPgqWO?&tUkW$#%jIwVbXu$W^7S=wZd^y@zo(p>d86jagtLuzKF&Y;
zBymZfw$FcMvyGcs-IZhG&YqfbFR!QKs$IpOrOn%C2(xYDklrp`zjJy?TE=M|o{R^l
zPS5*(D=kOICdbbrBdXM*^lYl_-kvA>YtKB(SP(L4!pr$fG+(rJ^nA88u}WPS)L)r$
zJ{+`)(<tdkhso|Wpe2^Rv(1iPyf{%*JM6-yRPVPQ!7K-gcTY3UP3AHCT#>fk#wq2{
zPp<M=U*D&mcX@mN;BoCjc~Rki=BJm{&j0ZJ+1hB!!oOjMO77PGxM+7S{!ZGiFY^y{
z7H~%#zG?BXxb1D<!Hqu?*tY42*0Y^*TX9xS<oO??`48&X2?`6Je7SaW#)WDfiz~^e
ze#mh?e0uEWWSRc@+7~ywIn}fC=5eQeFZ_M);j^WuKlV$ny!?1?%RRrfyN_L1<K_xl
z?=V&Bti-lYVztNkmll;VU&}uD=1$6e^<2v;rHwPwpNO72_I|?usPYDGLo=VBS%wS0
zdvngOzq4C+MZh}KwWq_Dy59f)AmEkmzE@#_4R`d#JUJFBxA{KUzy9~dhtrww?45r%
z$K>6&b+!MW{}AIn{lw#L)Sj|A_K&CAzI#-<PUUUs*K3((K3mOMb6<U4WcKs-hmRi<
z54Z6?IX%7q-d^w#t1C5>KAn-wF`Mn1>ha}ayFBP<Hk+<%3xoQ9O321udYis7>TlTz
zFTGbWJo#(4u}_rR8e6%zqqlMYT`!3jwJHALMfa5t=*$v5x#`D(hfi}K{r@xP<orc1
zuT3r9*b^%|JLkZcU03I3hF_bQqZyTR>`!*$vsKYKoeb-h`u;3m_*r(ZXzTpW#QC$8
zzwUj<6V|@X<>TEz>ASOc2H3}cD!%zpF|96E^l+WAh|bd^;ibV_HY)XrzD~-#aDnx@
zNdNg6az_vDQM3D4teAf3X4K;wn~mG#C-d}P)H*OVtW`L^q$uAc{{7~A-|w8R|DAI1
zuMbb{HM<p@>zL=4o_D&qJGU)OTeyCo>*0HOBBqCyGGDCOb9>gqeUqQ9$bK{T%9M`N
z?vnO(HM4df`gYd*zCp!@gip`UgU*fCh~2fNY>DSxt78kY)^EBr?NLj+v+c9@66yKB
zGIt-SKW)DB%cj^ZIyv`m=;z6JF8=@f>$=Tf{#WoNTTS2k-r0%&<L4C{|LjOg-*mm?
zcMVtm{#TRVG+o^p>;1Fv(R9w{1NXlbZ9gyZ>D%g$zc)#k@x4Cw-`9T9pW~Nq%sOAy
z{^9Bp+44N*?y^gpT*C6^-alJ^zq*<AxNx+F#gv@W<<9^6+fs`PKd(>yvE{GI&l=y{
znLF2LPM-ZOEqIcYq)%?-zSrJyC4EWHdjh}wdw(b8vT6N)>tE&hE!W@1p6^{;6S=y%
z_6E;^jXx8b_Pv|&<B#k*gI#m%55}KQ{q%OhnZ5DWfB#Dv=RBS}@zReX`)fXLeYYuZ
z{`m3(FWd3$KmHzw-X%1HOHok~G>HjXMRh&CelDl5T7robc*ecvS{tA2DbNXQeN9(a
z+^W`eQ#Th?6}8zcsWIcM9E*N@=i?^J()D`<1&^OpuXw7JZN{vcf8JWBZ}LU|w0%eS
z$ec)I(lP66z4h)^&E|&b%s*ovoSJ%N@A~D_nJty(oiPg2Oj79=ma$KLy2LU$FWFFV
z`c31FhJn}pKx+tgmA(CSXZp#kt)I?WzX#nhcWRpM=}D^I6%Sj*w>VeC-S$YaFjvZF
zRJ&8OR7T@(+<I2=?J+6k)sxuog&kTS|9Bd!!OjV%m971g-d#IUu<mZ$Gnt(y-sSAM
z-OaJLjQK>@^y|&mhdX!5dG*W|zd3jQkx#8f?;h=+QB)OtKzp^>iIA?24g~{)37~b9
zdn${cotf!8O(!x0v=E^t3^t|z&&hJto{dc4DSZ#ABJDReyVXtA%>KwfJ3rrlwps4H
z+nYd#RILnF2kkJpyQ_4jaXR14&FR;@geJD9nazHAd%J$|voi~;zP?%$vvbm&oyDsS
zls<*Load16>B-42@9)c-pBD;ww>N?@K!vxcP~Tx{oOfZ#Y2V|0vXPt9&K5`ZWnW+S
z@cHxOp!Hw(_t|<+(+Ok{ndtuP#fuYDG=n7!5*T)uzdv^U`t-}o{lP}3{?cwNes%_Q
zZnOAzr;zu%*`B}2I;T2mnrn+p;(LEz!(*<(A|ftxtxD(Jte$UQFIQh*-zRT>uKNAn
z<L38k7Aq>aJn5Wik}1T@#&h8EWzeC}0V{)4!OMB(x7GaoT>SIY{wpuGGl%y(-BNP7
z)F}P@`{vije}UFQ=|*p}+5Els``vQTax@96l8jwa`vY7q^)L0F4w`<`UcYD2Mi$p6
zD^Gqo>!_@(d~)9UMNA7ktR^jXNR25HQ!q06G;`nkv$MY$z6Pzin5v-U?|OQg?#a2<
z=7NHPT5XWS1v)x9IzR^;fN1b(105Y5C+^M&VLEuxp>BG$y=>WOw<pC?Eqf{)m3`xl
zHr5yOL@#1=XWw<T<jCQS<kR0GzPa0<w6Zx`X|MRSYvzh~FC9W=Ouc!0&8;$5xdT$d
znesdSuG`=6|NOm~?(TtW{{No&a^@M8$t*$&NeYhJ1d=(lthKBc<;1x>{3jAPLAy&w
zYsG>^F)pluA0M&BGdczc2)c$%lx=nBh}*%an8+)j;Ly<GnPznP@_zd&qsfYi_n+3D
zJ8Ajy6$q@F_bX)Y*J+}&U3a=kTA%xsnr-=tYkA%4-yNG`{zX4|^G`mSe~ya9tFl+e
zrcOSkz|o}8!6EYJNSL|tqR0!8k9UOjD!YeVTVfG8qjah4DXABKCrqu@7Arkx@@?jv
zs<`Z#nUSY?97L}kme;;JeZ9=Ogj>_umHW4B3SzbRk<Mtm<Q3N;%bOcN*zV%_z4A)&
z+WWK49`8REKe=Y^t!*=w$5*j8<Ryz6tU3F%%&%;Y<=lA>lF$0iKdHS^JJeBt#gXGh
zD3`hI%-dX*Sxs}l9=EBxSbpg27gyFZ|2I$gyHiT-=z+DOz9nxnk}Do-+ZL_cm|CrP
zlKqcytgG-uzV`Trd8c->7-pFTSXvmo*tE2^m7_cK!tPGFu;nLp7AVZ&-)hybqMOUR
zc$fN*LruH-5`He*lC8$T=yP!4zW@21Tc-9bu-a&?Z&&-h@odeSorW`ZeQ=PS$tuF)
zDB$;7c9N1nSIc$&%si>8bQbR7&}nI_Cag57c2@lQD(XeJ|9sWmx+hQBm*1K>v-WAx
z=DgWnv!a&<E_FCKbMN8_{sAj<E=0_Fxbwsn-aS8N^q%<t%s6NAbED<8PcvllKXH{z
zz4|jKnf2kH&IOgn&l%KeGx+BFD|qePX1OUW`}n8ItrlJyTUSl9f4gRKrT)Fc2k&`T
zs;zwEx^(`|hv5v5<g;B(yB$|GC~!0hG@N1K`se5QvH7x<hjj4VCC{?#1V!s#PhD>%
zm@cu%<L*Ku^TUUaz7x2<XXjL_unnuH+rQPm8S;MRn*CG$-j2DR_>bkG)$x!sPxgMO
zGWZ(Su}aF_>FSHi_S;kbed2VUZrx!JyC%r?Tvz2Uwa0JeJ)9<P*XL+bnDcM5hO5FH
z{oDPOE2P(zt1IOPGSqHcleWgm*GA_|OWH>_tr<Kj%Xi+4FIjlgVe>0Dt%#l-bEH;U
z-+iNh^<t0X{(DI(_Wr5&BeT+zOj}Pp)>_>^-9}t0ZAqqRtANR-V%wavUka{Nuk%dS
z_P=^*{=%4+iCl?N7AAc=&Z*}z_hw6UZjvh4Y3ld1Dx$XS>%`8hB1w1U#L}{kxM<#~
zT`jmdIQgmBc76S&OS8{^-|%(;TgDsxXEW4v!WMQ(9yt8#b-}bCfq(BTiupH{|1UI{
zc|gYJ=nIKM=U<#T>Ur(mT$!`Q$y?n5_Ia61OfPVlyjfqidS=#<me=xonD_fGyt_b?
z;f>{-=FMG#Ys|Xz%Acid{C<J+&a#A4;XQ|zHC#>O7N~m1JX>=2LV@N>ebrf0Cr(#~
z<T$-~H#qitUc4yN>;8W8g$=VhKZP-$lQHL=J!{3I*jty3j(Og9)w^Y~?Ndcig6Zo!
zKFX<;jPpf>jW?Q};mJ)^{Gs!Gy};57E4R(GUOj`u`0%_ZHDU7KSvppx%Wt~0;?c{w
zy1i3ES<f6kV|?^ekmA8pGgur-ji059_nw<n_SVI?_PyrJ{c(Czk9NzRIdh4lH{eBW
z|I8g4!is4H@`b@~^_0JA{Yv+>HVKh5-`Zeb^XS}>-TYCU%yam7yF>MppD%R!bw>9~
zY<LP+*Ni5vL!r+s_V>siUf^G^SGer>2~D0q?zUaoj>q?H-E;L_q*z8{v#IR-ig~-U
z?s_Gj(_Cei)v~Rc_j6i*vSU~G!TbM?1$(AiZmng@o&Wbh`PnP~-p=IG*uMYZt@qZS
z_rFwrVVk9QY{xH?$NXVSRwt}E-%~nyW$x$t`Q1K_&2stL%N8^#a5Qx|JY=#s8)9T;
z*HL|1s&<i<`Wcto7Q$)=cq5d2Lkcr?7$z_^HSw6_%bj_=Riubt#OU30CqK6t5?4-o
zxgK@+JMExN;iYq1ayr&Z9_g6oxGUmw*Wncz!s~4kmhQQ!|3NN4RrO-pf)h@sWGA>Q
zJJ07?I8C%KUF_4jMNWOQMD1RP`c(2<$xl-)En<BCWc8je3#KL{oS)C9_HSv>wUh;C
znGdG>+}<x2lzewRtIv(6iY8u(70(k+%@&n=A;DySYv%i=+gdX@K2Gag+0<E;@#f~k
zLjtLbymm0I;rjpHPPl$eAv4>rt=oj!j%}(eTbAmWv8B^ccCAFH?t`0BRlbhj-xPFf
zu26_E?K4;z9(^oNV@c`%^Oei}gL78o%(S1D=e+F0S84x=g@3g7y@`zPuY08*%Mdj8
z`*YvLu~!4kYQJ0c@3);^dD$*muHj=`)t&uwp3If@{La_HI<sV<Sk&XhYkDisz0R%P
z?DW|n<C4=ZwP{CJYGp(}yHI^4M(XSTNu9M)%v#dR_nrw(=JNloXn*egXEDYL`$J_7
zSw&eK1-x9Zv2ZP2)FPEBle;8xPv=2?cJ>^_jp>V<{lhI@*zRm#y6@3+Y|F~BxEbLN
zpT0Sz<$b)cb{f;FQ@=lR-+b7ld^69^zwoC^Y)})Y!KW?eZ=0iK_uV}BM#udB_LV!%
zt~hqw_|2c!(tqbIzr1H^YgAal{DP0i7CGI0a%t(O+l>A1>!z-evhK?eS<6=Ry!yuc
z<rB;96d&8ddywnyUcN%UyfTBt>GSTb%$XYg%yMt0WW|Fz?Q4&<^nEN`XC01K)0_J8
z@*AG`XHV8jvcCDwb^3!*Rr`5?Wdg57cj~?G^-tY7bN<VNLQmxP^7rbcas8TJ!7AFa
z`sli!!V<d|3MALW$GyKSa`%Gc!#+O!Pp<E+86v(X`is3Qd-1XUZyW0@_Ta;LccMeT
z|BGCHo7+Zyx%n5jzx$PPKUc*4OfV`=JF}?u;g<OcN1ZBCEEMlKdq=RCMO#dKX!h;j
z#6|VyO<ivUMPAwdlKHxoneo2v_lIZCXd1ElimZochYc>XUS2kAnRj-LO4x<tlRo>e
z`15J?+=naU6d!$>@bqftPj;?IwmGMQB=>KcaCoBH-IAa{iF>DyvAn<E;kIw3t=64I
z0@0UUL|w%zrT<H76v%wLbp7^D@0G`NS5>8-+Qwq#J9GD`llA;Netz~>@BV!5@15n#
zWrOd!M6TzEEx)ymXP%Du?4rHwX02(7yA*Vk{x6ul^k<!5^`Yo6*OeK3yy+7=-UvCZ
zpY_XF=T?ZvDZl@FXQ{k0UYEqB#St!Styx^hvLN;OvJj4tY?Hbl@%t95W|zO1?=t=S
z>{RV{(b|)C&w6Rxs;OB&F<NKpgyLrh^Mj<sLeFVi?KV#OB~=u1AgcGdJMV%`TW4s#
zxD|iWK{z<TQGn&)nhwhq0aiQp>>}PA5@NG3G3sZx^4DDCz<NgY@M`y$kGanYYe^f0
zo;jK|YxN7E?d@!#XYCw!>b`%pDT~9n;_lw;KU>WUWc#Kq|ManD|AJi?^a9_`*xWyP
z&F#m_@+NT_6jX~UFUdUmyEFax(ofF~`E^f8IvQTul4`*FzU1@SB=-5KKUwo8YQC^D
z1f`SqWcm6fJI?)iELqKd{`kzFI(Ltr;?2Av?|)?N%i^MUqWd?VzIS%FpTDA(nb&`n
zBVmWiCf@Aax0%&+rYD=!W;fg8YrfC^d~W*s4*|c$jwb!|b^P6-xk7=XsX#16EkI-H
zA7QymebpXkzf71FyNkhQ_r8lqCp-$8xn<pikJ*3fCwKjef4<D&<8_zQN7JtLp4Zu2
zbaZv}Qse&kS<BC@+{}HW;*h?K?nJdJTd`9Utf~%*n6t)$irRwG{mtpN(tfV*8G6(7
zU07G$sqC*iwRzHipZLdfzn)JH{-n?FrRKq-)|oG-srk${p7zVtIdb0AJ^xCq-$_|b
zR(73#C~4UO-6J9)0xXUKf*LUf(=$14imVKH%=3Jfs#b``s>ZH~s|3ZXh5t|amb-Cj
z`W=C^C2k*<+H^LyXr*WCNmXywj}ZvtyZqyewe@^8YfU>rsrx6q*1t$vwqPeX@pK2j
z^q#rL;&A%9okh;KBGVtfzP^8Mfx?zB@w%B70u4|6e-)Ll53l_C@VCUa1r2@8cFV(?
zKZ(y(znZ6RvCcGpsoulGb#rSTuB%jjX`N@PRTmskEDkBjTv)iShxlmO++Ve1Z$^wP
zx9$6lLF;CmSS2p6@j&M2KlABHJKlVL@bcL1ZG1{uGhT|6MY(J~G~>8ah0(c)Uti}~
zzMaBTzhu=_@3cj3tox5&S-kw*cJpu*A77sOGj7_(Id`5c`p;tRZ4??`>Ef<c7kr@v
zoJ<ZErj<#ac|J3B$NIf_^B=^0<i1_(AHR6c>3O2NI$c7Jdc?0dxkS2#?^lY_|L;eb
zZ7$R^eV(TNIqv+jbQ4xytHaAa96h}K?3=U2#pO*;!*1FA){~xI<NMv>@9Zi2D*4Nb
zYL_iIzfByRKO9{a6zrS9quO%nreVOnsH#6#qyOlIDP=`e{I;9CojGNJZ?g7!Q=`W=
zPaZ86*?wN~^jgobc^%)DPkdnf)+Au}6v=%rB-U$&HJ)O-uQO+2`|j2cF21bN0mj#k
ztWa;(>_62%+lFy-+x9z)mdEdIb6ULN(&cma9;U7Tep5&EY)L0a&A%|yuUCr7cSlT#
z-*rR#;zsUo(>6yRZf!ia^^eo1+pc_<x~}B6oJ{AMHA!#Tf~Eq^6@9@+uZF&yIEVdb
z;TP?iH9IR;RtojXzg*s49rUB>-m-=j>gH=xCM`<}xOMS!V-|x&neL*i!S5Fx+3z%6
ztT*GvB39iflO<Up8Jgyw>o5QN@p^;(jkQ0|1pL2M&C@XF?#m}L4oBF(Ex(^I!Sq)i
zTm6S^|92g(|K;QOz8ex=4`uGwX)-Y?n4CHBxct{@XOAhJWr2BOY1!#}mZkBwDm`Ad
zU=vsQjM$uy>txTIF8O%d(D?hb%dCDrO;gVc{rSzYXjRSD4FYFbPxH^*xqnfhc&Kcy
zO3JDyY|<-358AsGMlViScIRc<pQ`@+!)4x*%2ONv9_U%>CuJ|@BhiqwB=mGb)Yc>F
z>#qHMb;Ug6{ACMH7G}Y%I(uT(I%aga{rh7&KiSbsGqd_fLg!?Dxh)9?9gd$nDDo%b
zz@cEad2c0rN{tiBjT1ZnJy;mX6}sn;#ga8nyfJSa%-wZweZC;F@`8!PwTkZD$85v?
z%?w_UZy~=l;O465J2&)POy}CA<Ns=fLd=|B!G*WJfBd|paNmNd*>#VW8}Vw&GwGda
znGyTu?xaIK{mUjcwmD`VkG*<2;M}#j`t$zJeQp<h`qKAZ#)`k#tG2H=y7KPoip9UH
z8TMGd4b7_Z)i3|IF8|WHyz2RCo$uLGl=arxmEXJH8R!3}`IH&6`uBGVv6uI+zgf+n
zuzZ(YfLPSDEQtul8uJVu(YTJMm;3iEi@z_n;`Y7gha<Ot&bt09BH-pGPw{2E?>ry9
zxb<^a_ys-P&dr^WLb8l!*0N0VV#X=G9S0-|*}wkQ*^_x%VO5mC`|X@sUt{W)P2Sh-
zRn?k#&t}H*tf_P6-rl~mA@TJewO;?S4VM-LCG5H5thY?3<Wsx;%}b9mWvm!QPfHb8
z+x(gO&S|BTi_?vlbEWz3`<_k@v}<vjS+TfzYS+=V{;_h?bafM=mhGSQ|M9<se}Px)
z*UL(CTZA3<I=m^}&syu&k%nnvTJx{%w3-xoqFzjNlYB=nKi3WajT*f__lsRx)jRdU
z<wMKsUoWWKE}+HKcr8Khnr^w!Rh=2HxMcopZdfRtqV}OAtlvvzVH9gnaPazDw$<O>
zoSdQ=eB#upMLsi)J_+S9@Tu2(uD<tw=Ys4rviBcv|87;aR(wuQl)=>g_`fY8f9?w|
zO)TFhbFZJx@Mzjkzoipytau)I*=(!SvE3K#Kb+dKw$c9BGQBgA_H$;5F-O<Qcy3QL
zu6^F-7wh)<&-={V<&qhajMqzRGG&~rO;g?-{%&ZscY@vI@Js)NuE}v-XAfN!qG)F~
z&ouj*M<Pd)f>MCXgI>>!8I4hKAIm=NXqRP8U(s`r?Hz})Kw7J^yUiO7g9#F>nK#A5
zT$7c0Tv(egubunx)(j2hskPrN|8LQ5k?i<tclx2S+_&9&{*s{_#s%Ip7`Zo?pFSM8
zAkVZlx=EpS`6<5XSCj=C4{bf_nRm`pCa?Bxd`ikm(Yo1by-z+08cUjFD#qq*FZ#4(
zZWGhqk5->H-Q1F9wpe|B%CTK1@4Zz%*JG<bdomw~uDI)tr(bGr6z;e?Bj;(y7R?5)
zZ~J`wL}sjrux%@d6wpu#5MX&I(_y)y;Hc%)<r8O>=P521%~&ET(!YG-wAzQ?Z<pUW
zyzlYzj>&Hm@4MQZo@mpnTsgh|&XO%9!R064%vXr9kKpL<QToG^?w>d7;A7>qqU!(a
zU)(ehYxvM`qVUQ--_$QpG-sT9JpH<uWGS1wk=P++-Q`c66}6UW1%8^u;waGa*+u3s
zN0S1_A=ORwpS>8i)f&#{{p~le?3!oTFRwO}_x)dgrtgT|mJh0W*BKfKYk(9__jtNQ
z?ZEqtck(a3KH!&l{OYpaJF^q^$NoG$&6siJv8~v*Je^imNS_sCiS%5f(s?O2*JSLI
zvY-6&^!3M{{)u%}ToPHAK6ZvzF3Glj6*hIPG>fAEi(n(DXZB(H!)YyMJd)Sj6jurF
zKFm8~!!4ang}XuHPM{ET6!>>dLoCOvdrC8??Q&PCFA<~{G#Dh1r<9@=up=YqAe+*;
zfaW=Ty~^R)`PQ2ok6jgg*J)~y+S$F>+>3EmuT9bPPNOcjr3D#su(bf6+{B|(^c^0u
zxomyDq4<4X-}9Y}9DLmiYJPUz5fbfdntS&3_Q0KSu}bTmXP=l;`!Xfv<?oX%u15p*
zW|=&<uU|fC_SSCG{Y!(NFDwh$$0KKdb&hS$owp0)?q*#-(Q@VFyDR^luI>uyHHn?Q
z<?7>{dplZV!VNfk*PpLa2B+#BnVj9G#@`%ITWU>kd&kKiy-n08@v`WR&*#>Bc%K)x
zN+fsJnpIPt9usUV^h~q5lgbt{>3S!VNS?vR>5~s`kFPuSBGcR3Hoqs^ar%;)eg75P
zB~KQ2Y>9Qhtz_~2^W7EGD}Mi8+gQ2oYOB7T<SE;W*EV;}G@b1J;f{OpUh%8xZokVy
z_OZP7H!ZjKPOm?@pnsopXML;b=GAix?GI^xF+1rKd+tejTw2+#*$40cbGv$C({+`%
zlk3kt+qnFH^T9jQL+h_iZBv7IWZsF>B^RrkX5X9JywlukyH?rrT{Ab|Yt=q-LiVcf
zoyb7NJ!_mEyB)j}d-v$;;yKeJ7gR?4zwacowRweT9z%cIV>YeBVWwZ6P2YX{tk}w+
zL%c4}@`E$JzhU3^_}}8?7j{<Wo{IFf@RLjVduY9c!7J;1{_M?{-dX)#Xn8{X|9a*5
zOLrR_ySlvcX3Vtw8|&%=xBWk`nP1Ex`)x*^{o)MMz`ys7ONytY9$4|7;kK&H{ldDR
zNi5BFzpsiXX-JoZrZf64(|_8lr@ztZ^N&MKK5f}kwt2lWsts89_`^0AuV7&o#~n#0
zW;r|*y71#?ect`Rhn!PecZbyZ7JvMBhpFn{3x)I5=YkIUygza}=5CA>_vM%+a-Z)M
zsHSs-d1z!am~2~eQfEQI;{uDH-t!quG~UkGlcHRE>$>u(G{-=ZoQYF2Z`Mle|8LMX
zOX7l!q>SClIP;(O8~#Y!d6$1Y<8J@q`JD9<=0DjdO|LntT)1n><twc|5j+0sXll3|
zSG}<1<lO0s=8kuwe$Us5+}j~8%@RBLQ$SPx!r$fH5f+6Xw))@R*d4#@KWp2pJNI{V
znV5ky+@Y^sS4A$Eb4}^EnJ1S1i#6)%?R95<Y}*;J>6-MCJ6UQoIG776Bc`c-_1NK9
zkn-W;e!Gvdeg869c+VwRMJ~J6T=VPU<OQh{N)Oe|wP~39^>TVb?K5w-JMpK#pXBP@
zH+%cOA9H3^q_5<<`&^GJ{9)>e{@x|k(n7M5o+&T3{<P%lUFmzc#mmLDCHC_c|7B}a
z8=pC5-k7<2@sR*e0hWheO7W2oyB-B?Ek4%S8MedvQio*k8=0wBBURV06i8iCH$`x>
z^f&o1%QtP0E2Wl~>-sY+6m*LU4KmQ4P<!F*t2aTP63i~_=$hzdrmPqGxL})Ck>*FC
z)tX0k+3K9RvLf1f;i|I^b^DGzv(@_TJ^O&MbVk@+57utM>A}jS%XeR?@Xda4_xzo|
zd1W4^f_VaEy0cC(t<Rjl^Ktc-RaM^-r+cma=g58dx`@z?ZF6?A-`xG*Vunuw-?F3<
zo`*}<t}K2wVc%zssX~u7tStNX_Ri_T_Sqe$3Xix%?ah4p$mXrlQL#Q>#t`eB-xfuB
zuto=n%?_UY<4?1jYF_G&rFyneQ4-tk6uL3))Q{c2-*l1C(>u?yzleow`u_IGg+lgg
z8#qo)?^ZJUb7}ixi`{QZC(VvtG-YGNftMlP(|)rqeOtfo>}fHk3;U9rSMN~^5@2z3
zS)dT(#&bpMx%}O8wvrj!GC6`P9xqH-b(?AZujhi|@@EUHj!tY;w~ojR^kfWO%)oq*
ztKrz5{I`E5guglDlyUin*X<>buHJH*eU`6n?dNjmS*F`QxlG~|eo&Gbapu>H`suQK
zadFR<IZbg|ek(02+%WJO&yCPo3j%gV*zJClcymw9GSS&bvoGH`_1Lq~cU7@*@06#k
zHPar}mpN~6n!Cx(A^Eh~_7LHRYzNOR6YZ{bXa0Y`_W8FZnX67|-}gBlwEBeLiNZQb
z`!>h<|6-J~`X>fU{OGTW2)McRpLku|A8z6O@oa6gdj2dje?3q7mGA<F=H+^iSMEyv
z`E~I%o|$vyCtR~Go>Zx!Cvv>8)FktJlYLE`(P8u2(zY|tiZb@=WjHl4-Tt`p?rMo!
zE9(#3+LnFr<d>;?te>!oEn%-SkNW<fU-{RTT}_8Hr|;CaIq-tFPBFv4{;c_BVaeNX
zRfF~@{r{H!@9-DKwL3ICU)?zRM=AKiFViJEIhzz>5|7A)?5Gim{w6Gy_T*;!nk&5e
zMNAJ8eX}-lq_rCMvj*RNmiTr3PaD&N=lrer$Zp_DYqMFizQ5C#XOXKa>+^|D^JZl{
z-`Tl5qR8QW$J7}Kn`dUsPEj&H?)#~1dw!XMnr27XJ;&cGvK+to7$rI-f87)OWoFLv
zv!S~p!&{}qs$89?N(l3=s$x}W@LjmcH~Bu})Wv%Bmou*#wk*46adtNQ=ZZ<4cbHpy
z=G1geuP-U^xayh%8rKwjdOq!}#eU5f`7>u-k$!mWKyAxg?LTYvu2k72cW@bcdb(&b
zxX-w<TC~~j@62m|?bB<cq%t#S|H!)%%uw_9Sn!wMTC5FCYqsuUV7Xva@%U!A<*izU
z#qRuz_FYf;e0aI~=J>4_5`VC%+1)=<e*Ajc+Q$ob1Td`GxHj^ut3Si9nq4#6SGCQv
zT)q6;=7ZYnq^|8gE59dOu-D4&JePCqv2(xYZrNfIH2wU7>gLtf!WBiAbKX_%kmoM9
zJ#yXKY|^x8#|{-%-PHY?q97dhZhuqx`UAp~nNRP${JVcriM#Z#FMbRU*5*(D`1dH6
z!-FSf`!zFyw)FUOgQv9eB%G4Y^1M!X`dB@;=2rwquU6IDCK;{$I$95&CpCZlpOL-L
zW`p|8gHK}R93C%E<@;7ux@cun=d!dNyE9}H4gXps&lFY{2vTB~vNjVlcu>v$INa)o
z!Of?upM5l}3f<oJ!}jvrFXxof-i7St%i(cxm%2IQ)spj9n7F!r?c14m%x8mfP90;j
z`fbn6zi0a&cjIc~5WbtGxnR{E`~4GS^(-d6efD|EqTkXUULVS{W*z(O>v&XqMZw_*
z8V?%Ej~!pI<#C_ZHtt<D*Z*+#+zse3+^4_(duG>JZK3xsJ}AY1*~i>aD*r9zVV-z-
ziQLKh`hVNs-*S8`ZuIi%^341D)a89-7Hls&&)8@E<&Vj$pF(eWtEB%Yd~16&pSx^%
z{_%vWIqtK<?W+HU=_#=CdiVdoa4`K@)tvWi`<Hc}D9pdSEK8C>Cva=!y8QdA!@r%g
z?Rjg*#AEt1<DtR3Z|e^C<bG)I7g_#h-=*1)HB74B-M7iTus{CuG5_R~`tcX5ABkD*
zHJ<H0Id5mnmQXYOZ?l`a!UG%yTD1FlwsTJBpWb+Cst=>`!PVk7XT~ILJMFQztE&C&
zHn)R5A97h!4=noe_)Z+-w)vOVZw=u|%3)^~t}ooQ)FJgLpZLq=`gaRg^yr)2t~m5=
ze|}52;j@(0GNm&(3U@?46>Z$QTH5li{``Ya;*SSaIO(cgGqH<#^yKupUviy-qM|}`
z-I^FhS|9Gs*~gbLai*q2YqUttj+5fS0T1&Vo}E;9zJ%-f+dYT<PR)^L%(%HzQ`Apb
zyjT98xX+cdX4BW)b=<wG(_&Au2HR5E@7MZ29^K~RGVKrJ=3}2<aQ8+Sl<Br^UcUKd
z>$E$vc?|As_v`2QtTtyacztu;^9Zhd{-1o!E4hu?w|%kfDSY#*cjHSv1GD9of4v%a
z^6jn)divylm|^tigsdyQZAa5jItl;$zT~3L*$)>s1wRTsadPdS>~FVyUVW09_xApW
zO9}?%HO-3rN8=wcG|y<_f+X~aPyAw+%2&4gSFr{^;eK~c#W;nn>Y{Orq{j2u>)Vre
zcP~EL_BJ-pU^W9QUt&V?uG#Olio_O#8htL(7ykeI^2zqoULVgaXQ^VhEpnLuQ2X8V
zcOlzm9}F)2cy(E0t6%v{%jNfX1?`#lt|W3rDqHXjr@I=lZ!G;+<eUik`bXTBKY^{F
znuASCbY|14BrdHx2{W2CpJ%g3zOl`EK5vE5^P4B{AD7omzsGe{1a+YEJGYeY>EmzP
zif>BZ`{@uav0KbW^Gw{v?%ndM&Tag^E`{;x%nPsjHpk37?*G58PsHTdzaK`Xy8jN&
z-aGr+>-i18%7b1?<(2LKzuuR@B!6N=(%tx+!)0HxD!zUaidrok92aviIQi>dUisG-
z{!F^ABsQ~2>hW#RL<xAdsLh!%Mk_f?)M(L6yX|6enjPPzR`bM$`@G!wcXLklb}=K}
z-#MMTBsOxz<)?l8#>jhR-|16REX*G-v;Of|X_I8|*&_#@g`JxHup&)SIoOuxP0mv%
znI*eJ*6#{7mAU609HTY)+T6Rjf9$(_9QivmR}?fSt~~Hox9PxZ+s`3!i~-N~=%2TL
zeX~F-EKK2UYnr~x(?i#_W@wo{dpj+xO}E@2|A#(L-3tEf{c{_39*(}!Z8Nzc*mIsq
zN8)MOJ;txgZ_U}4c3~<1l?-+6oztvkXXzYdSaWZuyn1KEy28+o^});+TpwINUw?G*
zO6Q$FWE=i0TOhmZE@=E#fg_wVQdlFV%xJslr?Y(Z(_da&lE8M&WSY^hgauVTFGDW~
zEc_7QtGqCgtuMFu<D2V0b8LdvNT(TJ{2s`?jr&aWTPv}9dC5!nL@K9N+Ue_QrKHa*
zySP~KT(Vr3%#>M5qUTahT+s<sG+c8+L}XjOU~s@gx5I8Vf0ylB?0@`>`g5jrdQpdW
z7dcC%-!|P)_xHAbQQAqj%W{89FFyM>)z$l%VfxO=jr}Hei!*MSyiKj@k3AiIf8C>n
zEymnY{#9J<MeF~b*_(ZCzD$U>c#x&#&Tw_%S9NOptS^?Y*t!4V)6Um=c6*hA0~Q>$
z1&sxR#us)h{8>7)`ASmlq}@?bFGEg++|K>FW(%KYr0)F}LO#z~TCeFg|9nwr9M#zF
z|6Jt5DW&%FHCIH^cw(LxpIcbPyf(Yq_LpVIzR!8L7wkM=F0Wm0^>lC9@}0}OcJ_pe
z%*zkGl$;O)9tytnpltoim8&)d`8c*pPq3O>{ZnRQ#kSfrZ@#vlt310-^!{dpyT@wZ
zmTEo&4dHrotcv+oVcYK(y64Bv1x2ON7w;}eSg~>P_qeL&-e+-J_HC|NS-AcD|Ayo4
z`VY6}hkwin+SyQi>tQI5goUvC#a(g!T-yK3Efo4V>!Zu#A7(mjw!ERf<6r*2Z9>oa
zb>zJ_U%OxUG<AX3r~kZZNyl=1A~wwll>Jik_v^|1Gv?O4S`=invv%=ykuR*Ty1?UM
z9S#qV9Vm1YT=wzg8zw%{)mi-ZQfFiv!WL}$+47A0{lD`GQSL9?MU_O_O8FP=TC}lZ
zD|fhJM3S$_&#$M?m3)3bHQiw67O53sKFzK7vcAl$5G%d3##PNbtvUbNy11rA)@7Fd
zS=UX21K&#;y)QRk-F^0z>y-^3zSXL|=ULil*1LAkwG%y`CvYdPxp(K&QIU`xft<c6
zH&&?Ewm+`2KmL6s`?&?;eVc18b(g&QpI`P&D&vob{rm;Ls_rdwP~4EII@v_D`BL9J
z{*cSA(<03^%_W?e#Pojr-#WSY*T1K2r%L253ZAt-aQW3%vk<+bK8D3VmzPKyu@)Z}
zG~qj)`uf`1d;3c+ng2gE;o4uzlJwF^v$ic>7XHPrB2Ptn)8Aa%>ilKjE5j!3dVWJy
zEN3IT&e1&!vo3GjSM%r3FXf&YQ+@h^Sxu+=ezsruwn6jD|3!~g*PhkZ%5P8zez|R~
zy7}2f-_tX<)P2r<*V(t|@9Fu!*;xPGee3-#zTlSD?C?J<4&IR21woBF3V#n8DBHgK
zw0hlyXkQzV+z;OqW3=Au6#YuSd4@~BeETJnLkG5)wlY4vySvS3<Mv*|*so_F^XHWr
zeBN<=W!EBu7Y7x(j(wJ2v8-Ha&he14ga=`(4~EapzW;3frSkAk<z}a*TtD1>?fpUx
zo7>tg{qwh6Tpu`LJ$KmRK!f04H!61AxU^t`mss2VbS1+x4v(!IdDbkq|Ge$~+%;v6
zam%{z_x(&~NfugDbM5%CBX(xBpDRM2u70Z3`SgoT&dqgRPX+TLR+UPeIWN!qq3>(T
zhxh8YFQuM1{%6*%*4oI(H{SP8eTo!*5wl#A`Pjcx7gp}S@c-VaDXXu~)!*~`jBf3E
zYvHgQcUHf5aYv=^-e-PrHgkQI?B*3Y@w3<5KQ!6#m*4(}nb#Vd?F@e&zgDaIDSUOR
zc<c;`tWO!6C-Eg3U9a)n&G`7M|CK%FFK=vAj@@1M^`GsT!~J_tFs-`0{cWtoDbt5v
z#iM^8iCZ7mZ<Kef|BTdsskd=b@4Y`aAD>nH{J_j~(_&lcbIC7rq89S!Rq{P4I3i~e
zoTU*ye`3D<6YF`Vi_CprUsK%ZCoCs#)Ofr4PrJHw*q)Z{uTp-y?KiNmcu>$O(YI#n
zo{N8Nd6pfo{Cd$PeqUSft7ZGuudBpW)XQ1EVqNm~#?eilyEHR?yKVP;EX3ltqf5fX
zclXM-mrl94B=qcP{#7gHmAiu>>2Y(-xp+VI^6iruz5mQ(cx}4j())D|#`Eo`KR0$Z
z{S@V`oyyGCX1hei`lpWXdry<hoToEYSUF1GOsGC_bjMD;d9P%Hl9tq;pU>A9x;1Ef
zPC`Z9vdQ6QnLRHvzMlWHQZ8(LQPRrfg$u)%Je4o{cumMW{~L$i-N>F-iFdWwg^g^E
z&K3d>aZd_Tnjb7?Wf^t)P)GHdhEp>*S&f7%BqMCKCbndWdieV8&HDc4=4SQaWj>19
z+TBK}r|w*N`u@zn?#OuGgfCl`E}fTuHCf_sRr$iSC7KQbh!JLcTd%|KYApB2^Cmel
zs-JfMo^W8xfv`)tUwq`w%sdmkcsfhle*5W(Z&!)uFVK;`G~-}jqDuW8!vfm{yXFU$
zZ%f}*_N`H(`1to9Z};}yW7JhYdHMeG?<?N_5$yT;^LjE@yr2KFu+#6py!fT?{q9tc
zC2yVfE|y8W)xa*X_fF)ao=YEC#r$8zo_P|Rdf9*e=O5C-Opm@+eAZ?wTU&qeTvA`|
zpJ{3Sd!t|Nbeufx;^v3C7Tf;&?A2TU^U&}9c%S*z+K*4foGN}B$5S!=*pGdGjCSj9
zpXYw7zfEs>;P!HZ8NWBZI=|0IIHE@@bd`X-{P|g?*-H#qKogxxAKrWRYAC1rG(;yy
zZ!2rNSi)?&h-Zpnqw#s?sC%lR)k2GTA6)Y|eU0gmPG0Yo9A4)Qhb-1}oVoip<pr1d
zM^T>VCzM;-Bv*(o<Gwcexa}Ja4KDLbD<(A_is4_qImLEC$ASI)t1h2AIaz(<?YSa5
zViNe4)r5SpWSKbm&YZ~>cM^<lcJyqCy=U`O?umhD2jBjl>@#!9ir$pTKt{HY`uXuq
zw|)P-&cfRA_?sPuiQ2-QXN#Dx%T}AsWcN$ZlJ@lpSz)3E8q$q%O%dHzaQ5cJx0d}8
zQr4UFr&nC^&{jO+#r@fAlEQ}FCPC+&_sl(Y+2ibGFV629N@nlLe)zsUX``TI>h1%P
zGTaX*22S6z|BmEmkvi>*)2#!$dp*1h52h<$`h8(}neL;l6T3M&xh%`Ae*P4hvcFw%
zefi^S-8aJ=FP`3g#Qa&-vxh}{)uTm1A1_)W$O&%#w!9XIJfzNX=w_XVZ`!xqh(^VA
z&vq(jTc3!i)z;s)sxj$D^i%L~vY-Zsxyd#TW>;;wW~bheHsh*&!YmJ;9h15}OaAh|
z^2^WRs&stcZT{rbv_Er((%GIj$!g}8Pd2#fX)1sffX83{%#GN6%<<T<AfF|79$jz+
zji`g8nWNsjPo(34Fk}q--@FrPA3=lQpwVs)B_`H~RtIwy6z=If_<gpf{!&E_r_2?8
zZ_0j4{+nYl_j$p`)pF^7C$Z{8Y-r1Qe)W9axp!+*o>=HT{8)T*U!5ZNb$N^VH=d-W
zZb-Ps^v(L{BhZS5G}kkZt*=A=X#Uwfd#aY%`&%}lOLg@6yiS7F0#1&1vpcxp{q}jL
z(+t<y^_yg$Wai(t*fC~jL-F!$y3bxtzsI=X_6=5^?BJ(b&;Bd?zy9+7$Jq9XD*LZb
z<i7d)?!>Lrw*UH?>bOg~;h1y!rFDr*ji+6nbyB(F-Qu11m{y+e`Sx~2&O|8{j-~=*
zCF90JrCse`|LdkyuPa}2Yw^PgPJ%28S;TJbGwt4T<>;Opht_9Q^~^c6$hCWhakf_e
z_t#zOwylxzeao&NJ}a>~A=-g!*@J-l|F^r`T9@^%KR4!3iSJR<dZX8y?uwVJwcMiR
zzwe)6|B;uEcc!1;+4<_b-MJl#clR7OlVlZx1pIYLz24SLy`cWeB|0pQ7W}#qGsSe}
zzsQDpcc`nWwe4uxRZ{oO*G%tE&#xELScBX9p7qD<*IcqS%6i)Q*@xFudROdvch$sZ
z`{rM-{lC<2<yrRs$8*-$wJrC(obDdGva-gm$WPHquW_00yD7VrAk!i1^3QbruU{?F
zuE3Fg@aZS+-A?5XtQ@j)jM?_za<l)iWJ7L!UV}pycoAjBl+`oh%b!`b>3+X$p;B))
zb^jajU4G|%AF>`P+rhPT5ojtViB~Bmo5y5bqROeP1M}V|-ge}@a3emz=ozPi$sv!`
zUR-IcoA`G{&W-hIKbf<jP*izpj$6#CV-IttJ_&lH8W!A9S{*TCW#INUr_kAre`<Qq
zR)>6>zDd2tXUn0x6Wz{aq&#1-=A6)SuBHNIK^du{Yo9HDU)nf*rJK)>O#9wdbJv_<
zmN{FTU=*nty;11m(iy#0#hY$azhKD=Hh8M-mKb~>YxCT|h`bmB!Je}(zn4hNue0R0
z(Vn?COZ@PyQ2T7Vx0b(e?zU$)57P?Q_Yc_cFlXb<>HDTX<}<(6;Wedr_EJ!&aVRmh
zX1hnM=`5GuBE)7PD<!u2-o!#VttIF0Ty|T{z0fyVJ9lzdcqDhOnYF~WbNl<2MJ8+e
ztIRZD^qbyxbj|XcsqM|vr|(<|S^9V@)Mtt*bJ<iijS#(S^Ivzhr+)r4bK29XD-*f$
zZA5&&Uj9|{-{#Sy#g=|$CsvEJyq<8GH@0s>(mkjA#p^t$ellBMaNGaXuK1be)iT^c
zF+Bgina}gmdc9lg^Y(nnA8)Et7F=B5sB4yUgYUrP#p$1me*aMV>Hg`9;1@Yl@#L7=
zSs&T8FNNOyd1}|mE8C_jUGn>T)O>#a(O>I5ZGQVmyxu>@pG#_U-2a*P-o`z-7BpY8
z|LNoV$p@C(&kyIEb8Bzc!nwaTI|*;D0%hYj`zJLOOf|T$e(O{NlWAwIUFWYX`*pTB
zfMLNjqr+*&7tKz(<n_&d@;0w)&yM3NtQSvNZ;%bu?U?7O=995zUU`?@<hOfvdWlaK
z&5C#-${U*GW3u9vXaj>;>nXF|TK15>U4PCl3Oz0`YZfDDMM~4kyw4kcU9XupE8+Qa
z*<CXuJy?yM<@hFrYjRvyJh$hu#oL`vo;}-;|Juv#>#q45HFsS<e|!E5_q`gz3D2+l
z|2yk0e0-CF>wgB8nSYwjPTbj1c7C6C+eSIzY5%96oLsN)v%Q>ghE<Bj{%YBsb8Z>6
z-&_3b(u)7nbLM>(oqy(Mv*!AH=WotgpR#VwVXm`>JNmXBl=5GHkCP!|o#z>E-Lw3g
z=Kee!ZTR5w{LC+B*w^@aE9>e`&A(T-sf2L{)^S7TnVg^{+Mv}rUQBJ8N2-1+>8*eH
ze5H>0XJgxlCNJaMOE)%dY+$@^EwTU7!S<7S@uGeG7gujIJyL)7q)oni_HwUz;U#YH
z@w=`yzIV~b@7#{NabX!SoGv|6YevaN4UR*lN8A0kFU~uxW|Dbn3j6MF$-mgdGbAJC
z|C}>({gzL8M-vtwj*d4<Kc*8U;q^~u*Jr<q|C<H#F85kFuU5ZsFM&n3d9V9d{$uy-
zgTAcy3D{u1&uOpw{Kfmq8k&NN%Ugarr!O?AYOP-VYW3XL<yEK3zkOamJ<0xUSF&`v
zaY~sT+vm%BY#%TlV%!k0yf{Cu$NH~w31~Er&0#)dG|#pyBzNBH+u>U6pqfZ2V24=-
z-&yO~=?@z96u+!mS-wFmS4$-|T~pfHI^@{$Ux)ogisuPDjD0qF^4g*+3n$heI;*vw
zV@>Sc6EBNP*@Q(TCy31pxEbV=QMcpBOKb7Ki!OV+s)Tp04Kxt0x>-{nDpD`JYj4vY
zf5#p7Ui`mkt~1NLd*x&MB#-@aSH4uSmHvs3f5kT^B5CoLOJUq>;XTW%Pgg{hzIT{&
zeaYt=>;9%>H(qaZefgVj-R}+7%FEU(y0fmiT2Xg}K`YvuFKh0dEmO{$P8DY_`P5~(
zAuRjk?YE_ZFXPN6h3@Xo?|Pha_l-YehRK4XnadVAx`gq?l)kv}RXX`{Y(kFT=U<J#
z*|ZWT23dltXV5T+a*S5J+~*66<_L-?`$n@}{@`bRU#ILtWSHj1sSfWKxbwX*u1P$4
z$S-xbq)q7bf?TPH7bVjzAJnXz^K7Hz>9Wn2Qx{H?^hp)b(%NNlXXD+Y7n5#XdCIog
zbN@d9HitP!I1YVv*?D`FaIoo=8LK$2?EPN1>w8U)XS00L8!q*Csq&XIOH!}x{dpkL
zJU#r>-J=r<H#6SYTBj#m#h>h``TF&u_c@NfX_Fd{tqDAD`1<d&J)fUy+g#b_s=3lC
z()h75LtV6C|Aez_T#XAZC<KXH)oyI7-QT!vy~$Pkz&@XAIv390ogDIW&iQ@im#b|5
ze|^7X(piP6;3kZqMvT4B+*pT*?E<f+UbWVo<{`XSj(e$R5#wUBmEu#=d`yLzXGd%1
zw>_TO+T-)pV8iCGcajVO<~K<eG)G^QK6!6#tIz3a<!T(}T+8zX8<!gGz4xQzr}nRj
zjcyO${9MAH6}{{7w4^z=wm16Ue|7i^uimbGKSi^nBP*Bxe9V}1p~PEw*;z^N|LyTA
zth_ebm49DO<$3>u&HLVO^`pJ4ciJb<G`ZhssMj}VYO=({hyLn%dVH5Im#(z<s~uE*
z-@Vy`dBLRj)$2ar2)MU9w5@EDK<Fw^8!Lb-iBBuWSgSp==GR<d>i{3vT7bF!ljq(m
zt+x8uyXH&Rz7Joru5bKMCawMStd!>I&y|INiq@aEbj!Ugko#Wo{5wxbzvkYR5=tU+
zt2EDg2;{Nwq%P_-p7a0Ajg7v>8DA3gmo2|@>_+9uJvX=7Jv7_wKS6Zit+y=4w?12a
zMYZO}`pNO#XQe&w7p`V*xiL+(_@u#=ggZYX&-^Sj&dki*x8Jh(nZd`OYE>@&T=(@p
zNv+=fcN&{!H2>W4f3`0*Wlo7|vx4HsWkJE-l4FN_cu*GxcJK1qB|QB=+%?PKC8fFR
zi&@%FmEXHqdiHJn|MGd3i}zfAXR`gquEV!qMP|wdZTh+AqUwxuoLzeY9Ct8Y@RL)#
zyL;aG1df9tr#J#9tXyGh9r@h!w#n9-re&edUlS&;Jo4*cu5tC}J9p+L?3}vo{Jzu2
zRO2t(`uW?`ewN!mS=^oN{=WSp!Kc+%$cO&=5VF@Vw)Vfi^k>`ZD+j98`lAm{G`5+4
z_xD!g>g~r4PCj(%zbs!z@}c)G$8;~ZG%0|azN;QxD@a^WxOS1&ijQxV*fU-C3HV(P
z4P(CAWqE*m@6{K^A9Y@xi0EEtD7_-^%0jOIj(rU|SJ!1PTXe5F>7JJQZ2ydX%P#$$
ze{f6Rg?=p;4W8F>PNlBNAG4-|%9(Q<MsuGDdD))$QvPf5!@2j(%QL;_91VEX{P}pi
z>!Oprw(<S1Eekqi@?SjEjjnat^g{iLh}5}XnfreqnFm?iSJr8h`@fQl+k{{LOGejQ
zo}Xr+S#OF37VoT2xb<mo!dlPg%k!@vj+tEXe0Ax+Lc`~R-`-vCH(PVVhc94jruBT~
zIh^NvQ=GTeYv@$xFAG;#r&4;}aHR&+?^=&!S=JZ!Ppx-*J$LE_jn-3Pu>$W-d|_bA
z`&xO;IOw-oed*rallET@Tee5;oyIFsO|J9bxE?-M&;61u^2C^Z?M00lnzcDLGOy0N
zd8l$9es4TcB5gt8Tm`mkm#01h578XlV%qv|rpsoh@9P%&yva@W39tS9qj|RM85_Oz
z!2;`awZ6Ui|MBq|)_oIP?aw{iekJ7E|7|TVAKzbIJomTc-aD@+-?-BH@x0k*&;66D
z%YU@Qe!IsUXPAEI+1;{Zo8K18z4)qinBB%{_3`-6oF@bGc@zv+%16&tge<nxj9Y(P
zE7~lVX~98T&lYBOvvbQ%ckFaM^>&T?@#x3zzRK2`XIN-|pS$Rxu=9%dqGjIpf5c<k
zYtH!F`#!%~HFw60cZUKlcg*@@=PN8cvv!Bs`u#U<seh~5S~aC`b6v`{AJ;R!U#f_|
zC2w<Wjy3x^osF`s3LH%zz8_zZdCPZi!h#$ZmSY{(EAFQ+ywbM1^X7>Y>dq@TW-z7w
zwOJGWF2RYd=Z);4%z5W;eqP?39J=<3uC&Cw`7fVL^_5Dtiu!9@zpDM}>NydwA~GWU
z#JhgB$L!g9<m_zQChu~w#Z?j8PaSMYIw!Sg)y#^uOQ&w~O4u_cMScF#PTQ+nlfJHS
z5n2_hU~2kx)!ohM=jZv%G-_S5M(5NNO=bOuyAGV|XP#x4v-9yBL+P>?e#`mhX4dBG
zzW13k`RU?ngM?et4#>L&i$7Z<?{{^wbmhjYm)Cr-YBap|>X5j+O>^-(Z=M+|;xAWS
zc+RtC<IKL=>5bC_C-@zoe#~%7xOY?8*~M|o7cxHAv9(Kgp7{N|l8oe@sQun6<vi}s
zoa}S^!rGo!-+4A){%c&#JO4#*`Yn|p9@7^ellbK_E>E#eI<@eDO69NLU*x{dwHHjT
z{r7doq3M?=ue`d++IE-SiP~~ip-kb)-(L4(9m8lfVBX#$xq+d2vyIH&En2=wTNv+1
zW!{W(m~nLN#93<-Il?5Ki#G1uJ(nl;q1wKty0hDI-u5hhWV)zliecfoilz>Shwob^
z&v`RBVUwU_CO1d%>>L-@TTG9hJ%g^STQ_6JrsvNR$}J9vc!i``oSP$aCsj%FYnh+5
zWtlY}d&{(Lr8`OL>g`GAl{z>?SRSUiZ0itMXX*aQr0+@Wx{VyYTXf#q9NNRH!0|?}
z_sW{$`}OYI+~cMx+$o9Z=;^ZRH?Xy_x%B8Z!#}S??*1cTZ`7upIAQ$0JY%C^rl#JH
z9G6pdpdP!q?IyK=9XkcH{XhjWWX;8&y&GL6G(Jg7h-59<B+?0LNpUn42rE8YaLCtn
z^6xb^lOL@Aw#>s=`S*;IXG@F{CgyyFs8Q(P5NX*jntk2k#ElG%%0p{CAA-7j(6ZtE
ziHS^tHJ}<yf#cBG?lqjC)-0%j;<%%8!c2#Q?yT?QYQ<Zklj{%m{@k<mvG0~uOD2@x
zIJ-o9$IEJ;Er<3_?VZv1C#HAm?K|mrxGxIiEQy`tWXZiCYsvc{Pdh#1>4(m?zbbiD
zuv`C!`q`|_1&{M<^U^POEnV`?&MhNDMRe62zvi~B3hLea&6X7`@UfN4ity9|`%Wof
zhe^`oW{HbBFPhdD%N1Qpf7kmW;OGJScUyyE8M8D}a?YoJeOkw}dE!BS_Q?@u1-wb^
zX>wx!XPiIt=D-VIsfd@Q(=`_aZd=3NFRPJq<j)~vE_cTtzZ28H75i3JH-A2)XM8RF
z7^wFonV0Z&_k7R)ua4a4Ev@)+&}Lu!2k~=zgk`Qft&mv9f7w*dD%fR#0!LGUCuj>t
z(_{BLYi_<ssVh=+uRY|Ed^=6uUd`I`-os@FtP7XjJMjCBs7Up;<mp-rXW6va*}Kjh
zyzAtlldbT*WA>A`>ylD4pEUA?Wr>J4^scpeEZX4L?)LF#{|7I}J(c?-wU!kG3yQ4b
zwp@BpdC$*_;x)hD>g=od?Ro#_5B0W>Z`b6{w0bEV_xA2B>8#xc-OWO;^;EWOl;)DO
z`(1ob%697ZXtrcN&c#lf+gJ7-{5>UH+uHAq-^%VL1&(_66tw`3g!9Lk+Lo`fQ!Ktc
z*Kx}CSN&^e$NZVY|A1-pq2qg-IQJ#a4@v$rr)E}|;iTdNXQz34oj%arf3xFHzTIK7
zN`@6PC0rKqmgVRqqy$#3(|NMfy(u%|97kHKRpJI^fAQA2lYAZo&HS3o$ld=s)9%}X
zslSs;4{!UhDou{3aHB^2s}%*G7noGFiV9Z?ytFNAfA?kX>X@~Bo}3>UuH@cms?O?J
zayh#_?+v@%)N^a{KN)Jg-Lbe!)Zjwp8KJCPdzYWDQhzMFm{gV(p8e$P$vxulA11tC
zYBM<`Olz?M$D#W?GdV?SoY-u1{A|717yodQo_^~6aT6U!^L00`uC(FjzI;bTe*cz|
zS?1e=j&prWaxHf+ZT)lCZ$HBquJf#!s@E-!9-F<Q<JhL3u1o>_zuzums<VFfaqoNq
z*Q}KVpC@d;<lFvEx0XrZz(cJs@*-^8mS&y3-JGt<YW~4^zSV}QqO;h2PajX`OXyTl
z)NIs<j?Bw0{HnubRR6PO;b~cy8BY!?b2NP@cS&Pn{mWh-t!N)5@~@on`kJWM2SgZ6
zm6%!=Jf3o_ZvD$-pB6#xUCF}jhu^!XZJgz_=Jw%3KE1n4MONM7dGb6v=icI9>v#|3
zEQl;s7S(Z!@$0=Z_npwktYca#EfOAgpQZi(>2*IrnLo^ML)PrBzh6GQax(+%e0_88
zCi~qr=e}8IWnQ{*DtPM`zo4BQ%A$u%3Nrqfn9rZ@wR8T~jJ1<aAKqs3_kPq=kvZ-$
zKBx60|35b|{~mWF%GYtn#*H68G)c;A47K_1?Ib+IxK!)mJe#88?)e)8-o<*iU-s(@
z{`&8O=8Z$S`!=qfQ~OdTG^w-OZ1Ub}_4ZG9A`Tx~AtK$pRzx=C{JrD%79TH6R$5e}
zV;H!wQSeGkS^oY0m0OCJFSgofePG`6?R(EyXCCik_c6<v_E|8=Z&TU(r!I^$d<0^)
zDH#SnOy4mdZLDZwo$_Uj-LN`~*jXI~Tv)j3%}t)=?K1GMxf!W-+Nr5xfA)gF6~@Pp
zzP@j5$ra+wvnahO?ktbr)UUry9<7?8k<Io-%Qs*}OirVBxQ)*2>F2lT)JM!d7AEJ<
zwPw}a*_lgp^dD^Isa>X{bM4G){#UYl`?hkuwB7z8=Td)Kr&`p##ARNQ$}6;9-sZZ&
zxFs`TVQ8ta=waT5V{a3=wyVzM)V{Vb=cVb)-rnsWd^XmEdTcOKV+^>v$M9mHQ-@TO
z=)9HbY$nrm^*+p8@Nd_ehWPUnF5f6v{QmEbkd;dhopsJQ^R^}Kf~eS|S87J}C-(Bp
zjL)&V5Ha6hyvFgfgm%H4_48kyI8su}lKJC{{K|?1X7BX+18+pcpYhdJy3Ib5&Mot=
zZ1tApOY7fXQaE+{+S}?GW{(8j#Jia<?D}6XadElmt~WuEcFTB~m)$-XF@Jsi>@#;C
zrf-{Nu{Bq%cUSL6_KL7eum8F2`%}L^&uIJnRK*7y*Lq9dd#f6jF|VOu>BGx4U+)#)
zkxl*Dn)>gXF7y1_ejXDklNCqXcyBXS6n)8w%acF9@~v^k?>FrGegyyCp02gv&Of&7
z{r{g|jqV4n29F5@t#MP?R4(%Al+peNMbAYp9oJ@H3|m{cDfp_jYxnkw6~R}v85nc8
zrshih=3Bw&7aUc}vv8%k-{)rsZRVvYeRoh)R-gaqsl2RAP^n`0{!{~hmzN)v4*d<i
z8FJ0z>E<VvX7|nre`k5gByeD9;FY<Xr<zu=E~x9j<(byxx8hHE`|L%Aziz*Z%*bjx
zGq-im<`WN%?Qefx-fX^a>*s^~=Y*Ixl<k>WXxpT@{`-d$7DaVWqrZmb%gM@JJ5}Z2
zXt7-2jlnB(yBd>&w_CD}5}&55XJFBp`tt8J?p-f_mKc8DX>M3~>Gv7Gi}nude$M}V
zbCKH@X`_~RykGh!EYo}P^7ru#@vL{MC9b|uycd6G5@&avx~}fj_inbV2QK-uefrWO
zl5Zy?<N2L$Rb|kP(i3%M2c`dWWM5zRe0pc|-uVZg`9yrOetmsi>eo~M1<J+pm$#ey
zEseY`9+8-FYI*l0PjBz!%dg(Q%nJy(?!U<W^}#<AA8$Q&BF1HR`hC^I5&fK}|DS$x
z(tg69!@GCgkT#lffAigzYZWuLdaGSNw4lD4%{S`AudmsWhAS6yIDi-XHU-3R?DF_J
zD}JWZ*+0gI>dy(JF6v_1X?^F6N8zRC|2LQ4|7d!*Ugmhbxp}ZsN0)7O_r+Pdp^L(t
zgO#tW*c-JqYHgtF&y`xCQC(fWYggDh31lr%O6xgx&wApCxbNTHl~(vV9sB*WqQd6M
zz1hX*E9XzHKL5OC|MTB-g(rrX9Bn&dR2C(1e{JW2|7Q&T9DXM|t#`GgVT$qcR8Hk1
zM?YSkmEm{nK=12~f8I$?4p}nQJ15oVe?HH@?T<UVdX9ge|3bZt-L~Flmfqz<5mj~{
zSN*JwofDSRv*3EI>)V^o6HDS{4z`?~=yOYc_LqH@YlORwPd0zo`$SS&?)*_}wLkCY
zUq657ME~8lKVH?Y{v9RU`{l)y&bFEJ=Bz$=%;RfJT0`gE=*1?kEhmh{no6z8ynn2J
z$+9{8UG0O&|FN$PXP6&;ySL`ank#c=dQVdowNZ`kN%+0c^Gn9<X^qC)jW4M#oZhTi
z#>oAD--(H>`;G3491s7zZ*kQ>od(fVsam)Fk2qN$f7FcQJgT(q$}7trx6nD67hk8p
z60P@|z3ie`b#?Tn^xfYzA5J(b7hPQ~BlAYy{(a(=$YmveBpcoZw*_d|ANnmJ>t%Q#
zo9n27!yfaw$JZ%xb?hzv{qCvW8{ywOZtabf5^t8A;aBi3O+sSrhVl*1-yNSE`Yh+h
zty_;1)(Zu=sMrRU2Y9lniT!p9-q@#_upq6JTWqbT#uYn_>^@7i8h-UZIR^X9?>T;X
zyW_XpQ<IOMk7+D3-*`yo#I(~L!BZj+y;$s~eAxPC(RTACQ;+Ul{m8ONYNP734Kpf^
z>GIXT`&u_8$FpAa<DcSq@tY4ORL8EJSLxO{qhi`R{^RFPIjlS+7NS*CF6UruSGFzX
z@2unH(Yc}LyVf|In3=I`?)8?BO6UI^4SQf)my>e%fT`e{f2n-!(IH+ltPMV~oUXU`
zI;p>J(S^IGw9L#_J~$b^>Ps#2#gu1{^VN*XbIbh_lFl!5zEPRanh-O)ENA_kimj)&
z^xJl>ty^}3b<UozraxuQo{apcTsP<E#p1Q)Z;JEMU!1&KG2_v}HDdB1Cw?9%ZkrSO
z^5UJg^)`n(BVT(SnBX?$T<D~2=Tvu2S|1qn`qA3U;yd%IE<N|&!>1uADB7shU-mbJ
z=YQDtB`u~Yr>e`Xn2obU%$y>0UdtQ4I9acsKYQKrBCTUxLShN?ZhE*LZ<#!0#qGj3
zmh$aKYu&SX5*P)f`*{VXDMhCqc(jz=$>8Dihy%ISg6+wI+-}#l9^%?iWD+D|@w4N;
z)`Jw!y)kJ8)@_2)$t4Y)Hsab@r7|4ZZH{})x%d8Cx+dGR>2LYQYa-|Uzn-%Db^V|5
zqZNwPk#;MSPM)w0WceX`NKrxTjFxdmjP)lK-YY*f=UPPz2n#bCPMCZn{M*&4mmU9w
z=a~jA^b7wmgX7~p89Q;Oo0swpKZNr$`2F8JDKmCo@>);Vc^w^H4il$qlueWENGver
zKG0JY+LL3E<>sgyV<GnWOsr(`Bcav1Ercg_`qm$tVUhNx&rkm7r<W38bIp&hZ7@uf
zmj5WV+vZtK^j)DtNk8u=l`@M~3YMw9GF)(qzvrg#(N~G5Up}?!m>d^upnY?CR_ogN
zg_>!eldnk|MJHr?{HWjPwROd#3BQ&fKl$jzhYQII#lIP3MwCB$-?{MUg~&InlFzP)
z$_?S^>PRuzr5-TJ_rO7kbE;x_H@hM}UA2DZU~{j1UVX&r711B>?5tjTNO9rOH-Y!c
zje?FT<vuO>apvx;YX|E6U3vSS_2lOChNkEwKX<p=G&R6wo@<)jpFQRiF8CI|ob+@}
zq=4UIzpaH8jY~sUg{Ul)uGpjRexAktm#d<<W!1B5tM6Z_$p3S`)$UQ}5q|g0r$o-C
zo?qL{z4F=y;m@_)A5Uk=w{2U|vv)14RI!(fi|c|Dr{{ET_~nsQuEnkFx3kA3wm0>&
z@%J>74{k}JB0^R7Lo*jooL4Hh?n~G7?TeC2v%RABKgm+Re1&`C`5iLZt6RicZrm>9
z7pZJJ<95t!<1sa(X&Z}`-tUe#a-9Bbro^Kq_K_3Qr|u21wUQS4Rr25Pk@YnxtxJ2{
zCO(gF;z-}FR?{GQzA*pX-uc%44|A^6zhBIq6xZIL@xJ!y)ZF-kURNjm+y8O?V}9HE
z*Oz{_oEO{7l(M-u?fapR3^T0jz0Mx$Y`!{U_U9#rt8`2w_kfbe$p`Zmp18T&+G?e#
zz=IbL)8$_+$Z_Q^x~r>Jer(g<6MOHU4ZG4c@%>D7zQDW%@2s!!tV?sea#dp5T~7(m
z&%d{A^Ld?e`aswDvP+Xp-p|`Anfl(EyUV94`ZDwFn)>T!&ENU;X4I|sFFiMhv)(VY
zTdH#Xgo$CIx+`90RJ`zC=X<neCsXbBd#n3+&D`9!`^|8A^r11LXu8RxS1Em~CtJ!-
zGx^lIn%8ubv_++7>^7ChM%y>koV$O**ZRBudr_0K(Lu-l-P~Px@7C^#lFKZ3p5<oT
zT_EOrYns=Ol#bYj*UybDq$hny6cjC*R>}Oz=vhkTGE;446`vjVRtBs8>WMhhD}Hld
zd)_sU=dX?);a_rOdZhTui6%xb-^D*Guz2$C&=<bwXXlv~eEhyvBTdvhqT<l!kJHx)
z&G@k@+r3X{f{B68BTdIOnk!2`+}PPN?d$<##u&cKDNB`<UOux96+EFF>9ABtOvF1Y
z)i~gHt#@CQd$G|Y&;1hp#}`KIt2!<7=+OtOzdshK2FGc)?N2}S*g<Aycf^IAL5~bx
zTQ9yL$2_M$@whH`R*Juc;mP=0vraDcJ(A>MQSkWDjIDYP;|~7GpA~FSxMlYwKd~pD
zyQ?3W2qw6_e3RyRJ4|7^q{_P2zg2rZ`^B}BSpK&N&;MlkZtuPuv!4~%id~i!KG?uJ
zVQq`3^5%$1_R}RNeoV+^3)!^j<WdQ-_dU-h^-9LRx)pGNNl&KBN9yC{gyfVI?b%0|
zBu<5Gem3!N*qy$_(g}RNCucl5v~F|m)m2O8oA<|_+IQ=x=6rQ-|6Ok;?|s$&MaQo3
zB%|S%vXw1Yh2vhE{t`}m8rhRQb&<=QFHgGpI43XT|GQcGXOCM^=?mwLm7I%PkNq`n
zbbIf&pyJ~r78VwU3$?FK9PxbkenaZB{5Lm+S*yQ&xTmgXQh&=t(B<L&)9N3SuSl<t
zU9K7W`|Of*=kxNuQmrOmPMIDmI5aJ1%Y2iWvR`*SmVWPhyDs>{kEPN_a}M7$s+CCo
z=8|}^g5z(jH&ft^G_M~$J@)^sd9CIy?!2pMv+Yi{<DWv0{5p%w(}(UhYc@5mJ@9D#
zu}O>b8P$5{pZ0(KebVtT-DV}FmyJ<OPZimumG_r!dffeL8vh)FRPBV56UxeXYP_Z&
z+040lr*Mhdw}pDW|3g=7G5s5T`_kT7s?Q2PzPTrKuK(Znb!NM(T{e8Z^f!FIFxP<z
zs}28HsWZ(#a%{_i13#8Lym~m_b|*uLPEup_cb||Q`9VL*U!^^|UL{s@?ULB^J>R<y
zY`U8JV`(4jTItk9zAvk<`xH&z{A{|Fn1hX7`TLKD9sb3n&AB6eaFg)&YnIV*+*MmP
zq^2ZG7r7qWSW_x-|Lwl-f7-XMIq{w6yQ!R=$YiI*3%Zu%{nz_B?NjL7pQmceFC{O#
zk-BAd_}s2qG25r_4>byf>KM*2UmVtIKJQnY{r4=-v#Uf`-cK`F?q!mC>DAmFQ_t}=
zRGur3+)>ANw)_5X9S#nT18ZE2>moPx#+iP)yZ4Ce!lKS{quEy%xvWz^zfruZO4QF!
z^yP=gr<NRF_O<Bf${6=O`t1*mmr7r?d()Z!Xp5O${g;KSCJSpbI~kt;tQWlfkLP)}
zs~x|em(SW0&A9y2`%BfA)Nb*XrXPRdFZ8}~%c6tR_p6`nl-QoGTOX@+c+1s!e{QKR
z{Ay%(U^@S=tqXSK%Dw6>@YeEB@11{of#vL<2K%@2xd~4EQnWAMXr|E@mXyfNsfTr<
zCAG@qmoHYFd1Y;~^Xx@wIT1p~7A<J!J`;cY<^)L&OW~sn)^40~Cb1zx`1tzSa@9uZ
z({DO-p5b(jE(rLe9IafOV!Svb&7#Ea-&Aq$pR#99r1P5R-HF~VbMuVG-!{d>j5Z75
z^z}#Vmx_kYJN`|6u12g1dtF`K`Zu=LWp7Trxw&~~%}=AqO(_@sJh+q28$DRtZ}ItQ
z%(Rj<A3l7Zcw91ICu_Fv#~PDU%@cb+6}%2pnGrqt??&OR#|4Edv=_eqksL36^y}&E
z2InUUw})S;5)&13Du_@?=*ul*m;1XdkR^_vtvFgnCP%(P=h_`H_x5Vm?1%@YP09!L
zb}N4ivzijOw275FXj6*kzkk2;TNRZwo$flSRz^w`YG&0eTi5+;nt)jHq4%FIuXk>_
zXZW!E`N{RW1x3#HUcVCi>Co2-J)5N3Z7Y8-JCR*lb-3o*tVc6L?!R2Qo|nJ(XvUV~
z7cc(PTyK)}{=tdK@u{uJ^K90aM6b<xTezk6)~@J>4_^G>>2F-yth~*t#l<CrrOW1w
zgLrthR=T<JhcKDKnY}5jHMR2>zFu@pLSSNwSY@L4;r5-v?;`r7@^5b_IFh6j?ek=>
z&G`?d=_}W}2nq%YYJB=4WUzVSshvF$j5~`}W&8ro7tQ=LgE28@v*bIzoc5)2C4WB9
zWZu!yVY55<UU9_cbp8$Py4ehyPo4R5=;y}f<44-=Jgj;YD>!{!j8e~}tiM8n6HEM=
z51;ThyxaNsz}g?b{Z=IX{JQ>y`@U~?zpt5j<%U(}2FYS}?tj<s_Qgv_X?O^FiskRS
zKkJkV->(#nV=I0>Uh(L|i%dnm+?g({&-*-?*U5Ox6r_H*^q4Ei#pROeR+dwzr#1R~
z<VauF+nDpL;3Mbvw{L%0PZF+{FEkR%4(d^wq_c*5&v~8y%q^|1O<Q&Sr6#Mc|1AFF
zXKC#b^Tqq5T*1wf;8xce^D<q|NegQIa8X$*xp8alHov)HVN&0MTwILi@!Xfaa_Q2f
z3)#mzIQK7I?5NVqvAc5Ci3i;gQBnIYecu~xuqJxBW~z%y>YvZ&?eFZV3>MdLQE`=Q
z=a;v$QD#p_Ogy;DRGY)*q3^m+XAO+6x94R|SKiFXuFd@I``^t*jz4OXFItEiURp52
z^66(O%_*FDtg|@k6OK!tNdNu7=uNMH(CV7t-6w8Ozwx`ByD(7EXO6|hpru}WH+|QO
z*<m0jCkMKql%1VjD|FQo*-vL4b#QenXsNZ$`2BBY&$Jo#kBU-a9z@pkvrV%0-<Gm3
zO7f_ry?tKvbEVn7pY(o~t}l)`A^L}3=IN1}n`3qgdM$fnxxujh&%Yfq56)jqR=24B
z$F-16x+vx3mq)J!#a$y@t}y>LJC<_fQj9adO}}aOwFl3hwJlw$y8r(_Yu7H3Kn<ly
zN=AQDJ~BAB^QAsHF)>9qq_=DCwB#*kmMAN4mD@10*C#jkaJ~1}&2|rY>?QgS9y@WM
zf30TX|M}JYk00IKXV^6Jxo<;bU2d-C|M~Zl0>c--*k|OXxIF3N!s{RB-kPvsw)#xV
zVl`RovYMx_3!a^ke06p8@spF)e||cxFKL*>!q)6){XEmD|Ki4|?j?>9?_T|X&Sy2{
zbXB}!v7ml`WQ}Q^;ju~gKWulmelt0h^^wK<?X5Ex?>nZf@NMDu+LfyNzm@hqe$aJ4
z(>O(nA#~N0g!A+253Vt?uk?9v_)(->&DXVud;ac{{rvq%Lf9Uoe6}}FS=3gQN+17m
zk5_KvJOAi@jcYt>4jwQS<o|QcI$k%a%3WRd_xTfd@6N6J`|II@2M2bQzFrZtbCZ+c
zM3KeSOpSudZUxWJ$*TFxIPh=h+RELBleTg!3)?8b8<BXZ+?!>x|B>Wu#v=?f3^(=i
z{_Js>)A^y|M9Rd9qCAtuY0lDl|0K_?vR3;SAu}r=BuZNHhS9_FM<)4+tHgu_Tju{h
z-g4LG<of;penoLPcx3S2*;{R{r>EB=Z$Iz!bbaH}S0Oq&Ca2y`-k-~3A;sg6Ah4Pd
zwiY0;F4A=NuGY6g|DN@fyXSEpDn7-rZ`Pid_gv?%nI<v$;o4^AJrbARcyF8au2g)3
ziGSZA+tM|kUz#q?G)@s@d))r#=XZ%yx1>4jUO0X=JN$E5B)iTv-6eT-uUo|3=Ov`9
zHLIBM=x4%(8nrDe8CW*%D35jfcX9r$iDGYlE%Uq7x-C7LUt6m0)53=u$}aPs_WRE^
zuvB1+_Llv2KI_Mahdr{^)0AAh9=v?%d2x}eXjjJ)jjxwIzMR(Ie_?BOIFo<g(+5r8
zjvQYZpO97lFzAq{X9Q0R<NJN-u6D{XKg}o2dzV;aXWaAtntPG;yL%7sEoeKs_kL&N
z1?!J@!XC%JTK{+PyAAni!S7;h53LAX{Nn2B>)EmM&F@t-uZ`Y*X-8qQm~NEIWHsL%
z#n1c7u531Vq8cw(eW2rF#-8;Fn^ekg`rp`5Yx{SVO8(b`Ehjy1AAKG9{GR<xb2ZU5
z#VhRh^F%Q@Y`?#MNlub}htAX4kMDhrHJH1ge!nV<+P*(0E@@WGcx0&k)AIhdl@oqs
zm_^n6);?+=V7i^3oBwXjyRQaPy%N0%Th2PJt8tTke*V`@F=_AZd3PnO%XFq{g&sP7
z+?{nn#|n;`tIPc6dcD53c4uLZ?LY5b$zA>78Y^!yKbK@%C2s7LVA8xNd9s_<-U7R6
z`qy0rf1kKNaf^bUTGQXn(Lyze^380#Ngo~@T$!}jDD~8lPft(3ytY<4dVAi|xV=>|
zJBwVc-^@H$BmZ@Q?$fLg)&6E?zOs|Qucl1juM_J1{HS5{vy^)iuJ1p$+<a}qYE8N3
zkIr^0rhIOEu4S@f{s;4tKk7B-T%4Y8PW$(SwV~KMz39$=&RLsn-p&!e^<Pczj&Dd;
zy86O}3r+Ix&0*yh`|z+`{?XgFzI`&5B0U{@gkJGg)w|}T9h+lW%)!l_{M3uTXWF(Y
zncr47c{1~ur9Ihdu(9&?Vd1OC<NKagED`m)c1m+?XMgXr1An%$%)RNkK5p-ihwbt~
z*GoATPt3f$Y@&I7TpO>n+uK`PBX<_5s(MYiaL!vWu%*T2Z@GR#Dr;ci_g}@*xo_ny
z?Gtji^nQ1W8yz~t@!P+?c=1%xXBz|WL~jxO_ATuE&U1-B@?s}$*mEj>^7U5-^JZPE
z^bWqAa4am?M!J0NlC-7uA3fe&i(KmSuAS@Fqp8`4ru80K^YzJ>32sw1nc6(AI`Mf|
z*;}czvrLo!|NC3=?#|4W!ON4@{nT`Iaea`@bL7sPqut_{R|G2W+4C$RFVnm8*kVy1
zt@N&r#qYNqKO2&vkp1Xw)W%)!_20g^{KJAz_tlxT^?P63*r+V5?&q`hwN=TBfKN|P
zgKjKY5xksFPEIalb(pWU;my_!oAQ~Dz1bPQMpDOWbHYC1{agMRWnP#WbXU!2f4S_M
zpp%=<=gvLk&YW)lA>#K(?iCeTMtjxYd(~`C^7$0wvDW5%;a06zY{$2$#r#+mv-ini
z=3O_QF7is{-uABZQ1xp2<(uaCRNPr3rux~|=8yNjqNk0$0`JfMD^{GMHP^L0hRdSz
zlZt%p7s2P}<|g0RkeKqp@s{fo)4=iz?i)63N_u)~>Ym^4cF(!}(^}p9pZ~%oiEU@n
zo_Opjs(-g+)d7=0hFLy8s%y@b{tw-;bEjeLuP-6?OzF%0=0@zTG7St2+_7WFhN`bw
zZ%?F5Z}v}rcX03X-Jbn5UnWaGJ#}Si_4G>&Jim*Lz31;{@VS{Po_S?H^X&f%j5v>P
zIJieL_x04{J4?g65)Ul;D6mZ~?q3b7f`boBdHRm6TWhYE#(KYbBlJHn=b%Vg{ujU5
z?pxBN*1j$2SZWX-A3taA+?@pvnJzE)4`<X+(qamp?K9J8W8Ph>dv(8a{qiMGOTO>q
zQtkb<pv(1;=k2!=%jR9EotJrVqiB3#>V2h{n3#wS35;&Zw|30joOrk`=k~VVGc%1Z
zuL{*(=0AVk+?j8BLVmxgwvra=>%1ELeB;GpvDiEPA1-R}-#PWvXX<IQ^FQ_M{a(&1
zn5bMP`8lk@DA|MGZ<cxL$q&=^T$r2ijBQ?}(_#Zz%k{T+mA(dDJOA+EL(t*ada=7S
zL<9o`XGo=<oYdMYZ636%B-1ZP;#8%4dF!NO5;A8q<F0TV*<4XN<y`ZnhKADng&*fj
zURf6_&ClO&V*b@>sn^sSd#lY|yF?b|-`~e0ZRTTbdh_YB)yF*;9{rC!Fg^ZnkLzPa
zo)X=etESKWytl8XWFuq3y9w)mq-KjaKmWFP#?)ssecq>?o8x(Un(oOdn!@+)-80F$
z;<2!!qwByGo$7CIDk>{KKAW9iRC6>u=GJzPg-hlNO;|4#mCX7huQR1~hmhx6zubSF
zn{U@I&APg3P1IJd?AKc?XPuv&zwhL`ySrD0tQ2~Dtaowr_PirUGV~@rNeO?Xoc4Y7
z>@UTy-L_83Z2C4OW}B{j{b%;AyqxiS*}BhNshY|8I@#Ce{P8FgL92qe=84MgmsW@C
zbBXDk02vsxw95%JFq-N3?#|Aedn%19KR<o^;?nsY&-QyQT*CM^Hn(6?+hOa&GuPxT
zSb1-!>}1QWnIcb5PftJG#(U%D(zm}S7d<)AxP5zh(DZB1&dwH=l3FD@x5sPRe)IXa
z?#$b4AwB)q^xPA1Q-V6bv&~9MGbp{gQapZ!Z`vm{J#8JUZxd7FGcpb}m>unUuHb&n
z#`C<3O;YtEC1Lk)GZAI~Fm2fceN8j7Y3J>J_kgZ^?-JF1^!Ty!#-yVnJsm4Je7r&4
zn09t{rSfrUnYbIBv;XG>X{?+Pz1Zi<xzuBY^Ev9ig)`2$6?@0Hz5Y7SGM7%FoLgHu
z*F<h!*8O(V)~%^ePfaa(cSllRU;owh_2;){U*B?gvhy9+JM}xRyWSVs-LU>{`SM?>
z^Y2@<-}n8lwc#l9|Ksx<E_dsmnZc|+y~XoYZMmHKll|x0<C6OGow!Tx9jZJTt5f)D
zQbm4WJWu71_a7$YCkf7&KIdjA%f6ls8w~E;xiiDMT#r}EB;oe9Tnj5JEeTL^6%*Bu
zn{zfU?#Augg#~rdO#c7Z1_UWiw=&!D#PsnJe-?YWnh!<WBjvd3^7DNIk50*ovD`3a
z@#)_I8v+!~^6$-gr{--Y)w^T&ZsVLA27EFW4sUO5wJdzpQugO`Xy%VU7v;YTx6asB
z$Zc=twd4G|8~L#xIXHZ`o;HiU?K)q7dElzsCEM0Y&5oGx^U~YjPxUMIZxWpvdrN)I
z*7tULt~37yGygk%?MNkK>?-j?yF8XIam@cFv-8vaO|1;~-!ep)<vSJLUH;?Hy7>K|
zE4*&*C{&iUDoObI>gtVKw;<k|HdQxzTJOAh5z#Y?6V9o#%)h@nAZX(E`<sugT_JcV
z=_4n5xccWM^?FZrXWqEMb^7+lIUyk{AE&&0VfANviF*0FJ2`hZweG3>yzF+atX+-8
z<>mg*54CbzSX*ySKQAZC&|h*tguUM~pU-Ybi{9kFKNDVfta@H}<mv1Clb6N+*!Q$V
z&o1wz*fh`o+fB^X9&ehz@0`GqKaP7XGILIJoV$7=Zb{;$5|NFw+du98epEc_`JAb*
zA14_c7TMlsx?A5i;LN+ojVuh^#}==gd0gtm*>+_q<_$mFzw+mFzq}OpEYEX>a^a2T
zp@#b$%9xh>%{_GU=FGppzZ<8V5J)>aYpLf0R~OfWs=%E^sn^!VcJEZ2R=at2@$;_z
zPM*B153H}1=*<0*nw{+zoi05qWV_$5=V50jotg3B?s`V6pyp`xZ3QpRf3vViNk0AT
ztcYHW#ohek>3XrC3tI&R1x<2pOxT)z{guOy{uR-TGCp${Z2tu*m>-hMx!bj8Pq$*O
zd$K|PVFq(`2A-SRpYQhT3U|$%A^GKRrfDPp-Opl2_gC|){VjQV>G0j6_>!Cc(diwB
z(vNW*d(*k-?h$>>)Y$<cVXuFfo)u>AoOyY*_-m#-+wWG%u|4;McdP0w{@(v)j|}Gl
z)%lfQ=9RPX9R0C0{P2>Uyxuien$NK*FPkCBGk@D`i-sNAXa1~{S2qY;b^E=ER?hc7
zuN8m%{Q2<Bn=?C$pU<!=)lzov3(-(oq-3ynW#neH)6@0aX9RpZo8feWxAcA0TAdyf
zz3)8X(T5*JRNq^&CvNkC&ATlPwwso1GpoqG<#KY8s_~g^yLaq(@b+!*+_|#zYd&!r
zr=7XL$asMF+1GU*N54luoOq+v@ZXR3Caa>ReVTiBCzH#+-CbT^cWt~X(yv*@xVlVb
zORDR<?@3P%9QE|JvMU$Z8ujnoUB3S9-~M$@4tBlOv%`GX=PhyTEG#VzU(au8dUkZ$
z-!G+Y2GjUgI8KijdZ+yV^5xkFjvUH5eP#ZnI{wR?`!avo?SH%H*O}Kf;c?IPBg~Tx
zjy~Fcac`XJEHh2_zj}`iv@iB#zJ3;{eZF>n@3ERMw|7rtJnQ}Nb9DHr>A6k8KX?7v
zq&`cmd71I<^0VwwjPuiNjW&IKef{zRM`kBy=g56EmUnlThifQJS|#bDHu+pf{Gs^z
zzpb-pOHZ3w9K7>4TaFBu#*`^G3AsIgZk=m8GvmkW^9EseD_^Q!+P_!V@0viJX4m=a
zE2bJOZH=*Mm|vZ&(^Hu9?bD*Dtyz*r?UU90gZ9_e`pvgH+rY>Sx_;r;y$2^A^Q^ej
zw6$%P&fUM!LD@4@7Vi+yo3r1`h3CeNy^iy2_-s}_PCN0HJ7%veqnO#AhcnpOZ$EkQ
zO!C;Ld+$Hro#tUDv897Ech05SH=yg$!q>;8oSijwYxecD8|O^~J)^G({g^b%Eceu#
zo0~g%_O;BfPG01+PHRed*`dW8mu6kaK7PY|wPxBfjcaS-<}UP~)+MHU3RH1LY*b1<
z*0a#PU+&1UV`)9T*L8z54`=xQJ9JbtZjZ&!`9Ta!H>{s6ulV>#M1PL`Os)ex+l$?M
zrCiU?v(*e(z_7dgeHf#L(l4!Z9(I4f1Uor9|NQg${Kw+*rA7Uv5h<tEB?P_b@U_U`
zt$C8=+^AhRBU)<53;EvTPt)_StcyN6$!MnHwG6kDM}D5@m3pn<+IrWfcm8hekAVl*
zL~LxjwKY5ahmVY`?9FYt(s_A#XJ(uCuMS_oDdAw#7t2Wx?slGiy!Yq3<1<UD)$I)*
z8j4hG=S!7c`;~ptvZIB+=IhGr&OD^_%;yq=apJ|772S^?y|}yEd~^EwXKT0LOZxlk
z>ytAxgWH=rR&boju>1YS807Y6v-3X`=Q}OU%Qo1)E!;q}K&kSTo6Jn%XvX5biAORT
zj5nUYy!xgCqq}j@Es2iqX7BLF22#F1-i3u<m?9)uB541;zk_r4!9%Uwg@uKI8l5@0
zxshA5MDOk@O}@4!^2+-7`TPI<(*Ao^Gi{>i^Lg_>Pg|-gaQ0>L@s2}{N~T{;r|{X$
z*je&OLOp-xhE1CmeVqQcqi@^O+Us8`*XLJMSj?~cr73NevmkD7l}_|F9&2lBSvI9f
z%U;YokYZNyBEYRr#`9WJ=*k<FtZt`$xLl34ytt74X6CI4A7(F~0U0C-(dhqpxBULc
z{r~^=JGF9cEPCqIC3dxEn$pUXz0&3x*VahZ)zxjuzRp)(Uaq5~Bhu5c#B!Iq!OUI>
z!=}2wzY2ffv41oFcGr&^8m&r_VLmPbb?<yR_E&b@ulw!G`XIPR#&S~T<z<DSyPdDB
zJbZ8Oto9I<troSvOwOG<_v7bJ!;A|G&(6$T%oN~qNoOm|vI%*2cNy0HDk++h<?(ad
zXG7yvrzfZAYz!>_Q2f8+0rziE?@YoZV}etq8|W(O)YH=>c-Wquo$Y?OjaRhmMD~{a
z`*xr-e|EMx=%SSQ_5XG{2~ITe)os*F0bTc*cXyWb@oSIwFR@<sqoP$w(k;e^<#+ma
zl^=)f*m$LuFnzGB{c_Pg=kBhlJ~NFjE^_5gJ2z)#fJ>aPzrX*h>+AhJJUC8I(+vhU
z-JfzEoHf%h{hUlpZ0yde3(d->e=l$gKgsbZd9kIzdS1u$GEjT4?#Dy+)=x~ULRTMK
z7rR>{U;)G4s;`gg|NoxvBv^U$*}1vhvu8_(g@xtZ+|;V(J4-`E&{O=1tVt{A0?!?V
z$!58?TpC}^58U*zpUd@<Zg7nZyZDypzb0NSd2?f;q;Xn;K}OJr>4k4^nM#@EocQ<m
zcjo<lwxA^S?Afb0y-Alq7t8(s_uZaLM5AH(a&=YjY2a!hU|wuRg~iubSCgNgn=4ZI
zZ`Pu(Z5kq;^{fj{w*0>&7jtuahnQ~EhJuGqXU<%?sbJaC)^=%6WwDK|ZRUjqjGv#K
zRSs6`wb7X*^ytx}7q_?T8>gRJQuFiEn)v<m?(8hq1~tAk&g-kFwA|X7y*dBBU6-kZ
z<HM+0k*+1$?$hf}9<5Kiup&_T->=vDy|bpDNlHq3aciqK=r*3Dqg^`D+j?Xyi!>xA
zrijSN`Ry)y8?nF6R=(x~WBuQ+;xmoY!wi%ry?bHwS34;`|NOCDY0cPOChBH9Pxtii
zcA8kCoN!k_@1Joq$mWulL81+s2M!%-y16;sxZ*>?pFe*J%F3R7d3l-3O=aetJ(ZLB
z?f*;wRRh~{qpz)vUd|NY62fxo^yzuF*0as?pPe<o|Kvy4`W;WU2Q2iMRema)Cw=>W
zx2;)ML7m8J*RFA?vi3@ucHOW4Ki4?@oXXiqH6{J$Z*FeBvND)GK0bcm&u7xl&dpuj
z9jP>F-Ad~h*`Q0#@2!oSdwRNF@Vc)#Mbk<{H72Rey=^$N{yNVxr&cad(pwX`S!+so
z%F!;-Em>E&jE#*!3A~w&SETC%_l^Da_P(>to*wI!295K^?XA)j5%m0iMd*i=h(<ue
z+URW`-1^Kd_N*{A@6Qj?nDi{>hVg8W*(oO{<=oj3$ofG0+sEVbn=3!3ty!}s<KiOL
z;%8?9HI&Y0US8%4DvlmJNU*W731R5!=yLFkJlZ9iym`9#w{K-KF&CyCxLmI-;%U52
zCgRY>r?r|PI}B!<=U-z`U4L@-`+cX^#qK^aU0)uQ=bojARPud(etvo7=VuWS5f7d|
zb-ldISG21mMdB6TuUS8T{R+B$bLI+(6*nqXBe_C?G$yTzwcj+YncGe^eBGQkw{}i;
zYUR?Lay@oR{7xs(HOmzr6y{o$CLL<wl<0M1RsJNAe5@z&NQdB)^YiEPN}C1jD#>hB
zQqp32diB=6TI;W`udAO+@LighePYh;HBJ*lUS8|eeKY^^q@2jNH$Q&AKg-5abZ7B%
zKF}q3r%!jc@k$$&zl%Ams_{ww<HwI*USH=gFE0mOdDX@%ePXhDf0N)u9{0PB!Hu&_
zvo|Fj<?7%)obM)lC~r=H#-vv)bMKyQ+_`q+=B)>_FRO_*tn_(yW@hB}JlWja+fx7j
z`U+}J&Nk22l9=csC@2Vu(em>0tn2H}_Q_g9g1K@psFSqFwY!6Nf9w3++JcSGMZ1<r
z9TfjwoKVKSM(6B@I~y9gM71=ggq%2ZsA*^MbEAX<4By_}?cTiEIQ`t5K!ZsSS=rbg
zJ$iI#mT7jz)m5U>W;qA$-HSU2T82>h7gQk5u`KRDGy+5_-@c2nJ@s#<uc5K)#<Z_l
zt3p>V3kWi7K6s$t{@;oG{eO?Wyu7@ytSs&SzrS2=Mr(?np9@^>C;RW`bNOH2-=A+{
z<=(JmONfEeBBg+!{%yIpjZ#mE9AZBI==nRZk{P>-Preo|`z-Q1t;VOOgaedow`N}t
z)5ts`9$(Yg-p+pT;6YGQ^PO!X+M8m-&L^{=?Cq^N7KKfm!fFO(Zz6cunwvm<=Pre>
zFE1`;Pd?tK`sG(4sQH?cvcYL$xSr7qrQdgAJ2`hBoL~R%WV?LbihxTqb~7K1h={nb
zCerxX`T6SuR1V+RQ)vuJ3K|+6pz*Qg{_?F#peAwY>-+odo!j|TpL|mMcD6`a&njl&
z32kB7s@}cEc1n-Fv8J7w(KvhdY$wmfSyxsZ1f}cA>il<im#3eZVaVlrX>TjH_@ynG
z!98xuv(0iZrOyBjLTq{b`~Cjsr}g(g`L?tEhW&QYj~fHIT!UhKDirE2+pGO~rdjv<
zt+`dniv<Cfb_6Z;dieOU^Uk8D5{5}Ed#k^z>FN0$^w?#RcgJE^$xEkmb1ZM}ELJzo
zz80dPr1XDbCDY}GcXxMxJ}zHBVR4A_sqJ(7XI_k1=+V{DyQ}&Bmt!pPch0}KxLDGn
zV1bjTyOWdCj~|cwZ|*A9*3;8FGt-z|T3VV_`K41$jZJ**SJC=EAK6n+O@Ur*=zp-I
z!=@>U>2iZ+@Ujcbe5LbbqWk|(dN<?IW6>^^-8*J*d^?}#_~GbZ9yyx`)&#4R{QUE6
zywVwWcbR5iU-$Gdzx|VIYooc`F6}vS!sFJK%)|2ae-8eBzyJKXxz?c!U7*o~$JQs$
z^xWK>{`uo^`Qon&*xjdncjVnMX%3>MytunO->p|_X}~2-Z*T8s=jWfl9$$ZUo^3T~
zK=S#yxxogL(pXtpA3l0?XjSOyjGLQO=gpg!a&nUE!48lGzt5aK8@VMz@awCqk2k%K
z`4?;KvCw0a!O`P|8Hwt7tU1eqm;3q5FmPPBq?oO_aem#e&Q4+VqEAmeLHV$enO&r3
ziKVz+%mTl;RzH6HxUfFnUQ92hW9rnYO@e}zw{|wX<MfF>bolVf`2BX)Wp5UAe7-KS
z*4$;`iEi-*HQ|5WyNXl)Cq-?`IXS=nU+2u3>v&UDyr=6e&Aq)XA}Z?B^ZE5b%l%|q
zl`;=4bZ*bMw1hK!ecaRM^Xs3T*540`-}4^n)tNrad}c=MF4Gm&4tw%t+QvB_!&dk1
z*^#Wh@6oA$M@_P>bX0zRwvypY!<{>KZfwaEc3V7gneS|n*;*o=U-#_YyE1&e-1fY?
zNf#HnYB9LFgs?2D|Mu-$MO77asP5R$yO%|~Y_>O)I`8;#o@dV7xskg{G{wZkxKx{0
zMQ&EJE`O&IzAmQZ?X9UBlaFhOcy1Nbk2`Z*zP{&R0_d{2X}Zx`B7zg2FL}2h^TyTH
z;VT0dvn}_VyQt@O$fvNJr2$W7&iV9fVsPBzm0@e8;^N{!gA^hUFD>;p$+*B!Uthng
z^!2gx_W##hOY{sq)-Qj){C=(bJe$fJ8xomQPfrWg0F~ax^FBX6AH3I%S1)#t%g$??
zPext~U3fxx?%!%5#_at|k*Y@#PazqZp4F?hZ*R+8T=@9dnR&L-+1c5w%IiaGYHaS;
zeC7>a=CiQm<)xe(8yf%o{H!e@I8l6w_nyp{Mkdx9n<|YhWr_<b0@L&4?k@>=vQh8N
zOy2{y@(W8!mQ;U#r^WD8F>+H%;^Skzp!?ykt`2|v=#f)G!h)uW*VVMNy8P|`PSFfr
z_TgUj`;E!R`9vF(l=MAy8(+5U-*5l^>Y9^tFL2y@^`g;v;&+iQn+K;FRb}Gt^#1wz
zSy)ukGbBXh-@kve0wU#3rWjq^UtfQ$U*7-Yqobhd1y(MR0E16ntgNgTH>G-)y}Q#n
zd$#oFXJ;>q2q-C?bO6~fd9tvyt?kBbM$Zq-@#nDHAHL9I(rfo?kM=KN-&y=T?f16a
zGcydEA3R8~Du1`8;K@WDHfH<(e~cd=?^jn)Xt=dCyZFTgMONkYXO4D@|9sHQe`9B{
z`rhjAY4B8Ysy=jG%*?N^uO;S*9p4vuc1Q8k4Nii2b8nnk*eZKs-rdHv(c1!61Sm?G
zX1TQU$==ves2sf9FVy2x#NS_EK{qtBu(1WLkF(vDcXw5Q%F&;{e+RD))9sVDKj&}%
z_sEe>VN;KZ9UewO<rhja9v|y9$-2VP92%xo9^aqam&B#o%w@H?yViJiQhN4Lv;6#1
z)Ai?niSh8@@bmLyVP!3RaY2zwTyM>RCmZ+d-CI~%s%m31C-d^Mk0;gVYlwIX-rkn`
zc(T8pqLLD*y?1$6X?ClU(xfvbk-D<0!q>}ni|ecDDGAt@hbu9i*Pg<o@#9h5xAU2e
zQ<qB4lQy#nU~oEAGub%pjDm^Dls;MOMXwz{scg-<n)LkK+$U#dPChr++PLmdMXS=u
zyqA}j?%1^}$!KQ8{<^(f4xqZ~lg%uX%tL5>ZJsCRt}cEv|E9=d%~__a!-KbGU3K5L
zF7NKHFE1`O=ic6CSop|=Ro(x|lP4|x{rs%lVo!Fz-?zB_|Gy*0k84YKiV3UxEpcw=
z+w<|5^q;@q?_XIN?7pz0V~Id1yYKU5zO&u><!m<;>o_ib_j<#YTIYpJiWANKH<m{+
zKT=`>B`Ep29}g$T%Zs_~EPA@5=qVSdrmgt+h$~R;^|iIpQ}kkIEm)wCl$2CZTAKRo
z%*<Kd0dX@<rX(F`VDy=1Gjo!v_oJ6DS5C}ubqSgg#dLY+ySuxM^X^zUt^a>|pQLX`
zFHe}p%ZQW<u7B>>E2+Iv;}+Me$ztS4$j&~!CUSE}PtTIt-`|cLJ*w$Zxdt@O{QCNO
z<J40kckbMY*jc37D{Zc(q2clHL~(al*P__nWgshGU0uC7`??+|N3Cxt{PE%8VNvH_
z-`?$h&$OZNtWeXrH%j3@(`HEOiuhOw%U1E;w~y)IusXgba`U{`akH9Oxr=^(%eAqw
zIWa|3*fi@(z=BT`TDirIik^6Y;%0_na>l(qm6r-P=iRl6iHX^;eS3JoC!f6g`+PU2
zo&E6T%aY8?%htr~oOEWUF{r!<cev|lJYoI5UtJ#`AAcOMMJ&HOqG{&NXVG5zNr#wk
zTAyvTzTRh)c}b<}+nb+vPJ=w`)XFu-wtCv$>hF)9JXw;Udh+S%>4il_hYlTLs;{pv
zdU9gompeNOAOHCEdOhgo%+u5Lm*?Ev#N{*Z#iga*HGe)Hm$9vyGS9a9)9dy7v$!|}
zCx&~>-ywP8y#0Tl?Rj@OY+jx{$ojUkzdKapr9;jY*>7iaPR-4Bm$a?=!Y$gsaP8W)
zGc%3bZ`_EuwLRZIXH6gf&gI3=&#|zw8s^^G!p`N;@bB;M&(G)A7kz%_`|8@-)7x@y
zi_EaOHc{DqMd)fV_kOvj$K~r&E-mqd)F577vy4)?a&K*Ucx>ZWnYcT=lij{~Uw)LN
zbi;n*)W`J<3<?aME{-9)Sr3YT2X(I4_+)+ron9TaRV%*!uj%h^Z<DXA2;_=rXklDt
zmUF}5_qVq{e?0D&|2ZpcZPde;FF6}luh#aSt`}PHX~O>h|7L@3J^%Zw^x4_j#~T=#
zMY=jpWG{HPKr$dFM<*{YZ%xF;CY!0Z1DCyAFTumcI{Da<<8GVl^Stv|a~Ar~xAU1{
zpcpw#T2|I~sn=ACx<3|nc6NLIe!I<O&1_xz3N*2D{NTaH!|nW&RlTRF@B4RQp|eTG
zg$C{LbsrA%+g|{+4K)}*Jx{&ZT?v_)C*$k?y58MYDhhJNqG<<sPTT=cvIeh@v(@{3
z?&*EgW)Ux`<gWh57Y_N|;kx0p+;8rWr_<vn<=&h&Z5pV_)$r}@ZDCp2wFit}wYL4J
zuqt`sP*GvAIqhs!79&T%x){rvni`GJRU*yo{Lg0R?{mB^m3nE3CupRFoxOd@H3#s5
ztMxOME>#7k61QHdIrlAFC(lzB^@`GD@Kxi}|Jn1}J?!$PRPSZ}^XI+M-I>4ltJwXz
z-@5<){R7RfEldx1e5^P5c%N*|$D`usO$*CGHQ&x+^{cDH(?2~qxh4C0--;C)tg{^L
z*R%1-oM>QV)`;6<vAg^|s2j@V=JLcexcmZ-o{9=fdV2bwKYui?J*eio{%UnV(9XrD
ze;0n7Khq@d&WzO4(-IRW8eOhEa_m^p+9*>#8H)pbveqlY*3LR~-r>*Rzo415RxZ(`
z|Ns8pxN*bgyOENza^~%Ax~He<E_Umc(uvzM1JvKW!=q$nHEUh$?ne(F9<+YH=kTkm
zt543e-Q5HZhn0>A2@0m!*Pfi1s63@a*1fTOm)4XJt%SGR=lrb}Dt&b&@$Idx5s^#Z
zn8>^L$t+Ad+9jeBF@c$#Z%=@tLZ-iu4^QZ-kcS^09)5FY=j2tPtM`4_^yu;9mzS6G
zPge6?RPyo?D0S_r{H*n)M9;!vhQHm<mbbUJKYsP<)S*`HuRj%)CUvc_Uh&|;gNEtT
z#q-5<Pko&FcPD5}V2RW6J#|ms+@D<Wd~SKf&Z5*V@dk#}Q&S%Pe!m}7;~s2g&sI+n
zxL^O@mYq)~;lY7M{ZIy`88c^oyq&+_Fzt*)pPa4N+NiA-MNc}M{Eq9z@AJ94tMul+
zTI>0>-z1?!O_NToPz{(g$G(2vE7Pokmyc9WZJp<PZqqt0)u|S{|C%#h<6Y>~%2iQa
zZGV|T_N1zv-8?DNtOt)DH*eo=Zkm1V3ZLh3P{(am_<Fyan^JFXPWRtbW*)a8fl=0~
zL?L*Yk51&Kmhbnf{Tr7XAMcYreD&&7@9BDmSywb-ZG=F}{xT1C3ahX9c(3}-j>_&o
zp}P|@r#mm)llHJNeaD`1yQN-JlYW1D3#y~m*T^z4F|7z#$h5Qg`LhQHn{V8_nR$GP
zv!I}0$cli*;^*f!mcNgiGiT1EzqWh!?8&&a#PibMr1g(0rcDtP6Fc_h<z-G@-qd3~
zl3Vic$1Mc4?5AuwbpQVRS?~Az2P7;w<e|zXu6d|$|9>_Ac{b~FW!3!Vc--Gt3+f{V
zFY~$R=h<HM{oULvrJrZ#@AKUF-Ll{T1Fy8%lUG+)bJ@<_xOHpl%}uFS*2l*uLX1|k
zu$VDf-CwKY$_Mu>gMd{bE3eo2_nu5Sbn~WUety1L-klYd`!3DYjo$X*AiI3P-m21V
zxwqS9MxMH7Bqt^1<>1ipOu#3dOH^x#paZDI-Q`hL^z4k}sj1rT+j4GN)cp9Mz1KW2
z$giNlAocV#RRe<w&FuW2p3TnJ5)l-fSmLv|x$A_ncFYcg$H#gfA8O_P@^-bQ>g8^`
z+Fu0^53wHWl?IJQxu`G)crWO%+0Mo*_2A{plc4U;ogEvQJP(JhiEx~(=9_V00psiI
z>%$e4Kuu^@7rC;Jk6h=>nX_Tz#)l^+DqC1sXiPDAajaK*ie_-zjT;ev{{D@BeZW~s
zN$Jz871k>{Y_`|@`~+%<tc%<0b$*_$sE5?mz18J;cXk|na&j`L2iK9Jki{u7QT*Af
zSEp8mu8!DMqIq_<dALI6i90)sGw<)S&CAQviQMGU*Z~Sx5y8q`+1J+vF7uJ(VPkgh
zlR0?tV&K9hjG+Ei&dp6XZrphA{{4EzzpgGWMiQm$q7%iZ8K?WH`OiCZcDDJ;dwX{?
zc{2N`3D31ERkE;<C@(JuwFZ>5m|R`v`JbMqn|W~&t9IC$gnxg2a;Z$6ZBuC!7Z(Sb
zYZez5XH`;Cn#8!$dPT>cEz#TaKu%b+Xb~uHu}OL^^_yei*vchZP*U>b!b0bib8{jW
zc64;4IDq^%eVT1`*)reR$KKxF{_@h&ZYIyxM~@z5Twf<EC@6Slo^3Sif)3DNir~cQ
zk6*p=f{sbPd-sk@WvZlc8jqcw-Iuqwr+>d+9}fzJl!RA&T1qFMeS3S`!qQSuNa)bR
z!|j?88x}ZuvhvB<cuds_)ri`nQCC+N5gn}!N)0?dx{XScayF%%J=G&=yrcLz-(0Iw
zr|UD~_P=<M0h&m8^!V{i>+*G=#wI9FDxF+i_xD%H>ubHb(c26vKc%pKn#C(?HRW(S
zKWL6rD{RdQ2NxF?m%EOxDnDD>+Jd&_M6Qe9f9_y2yJpmu6;74BWp8gCt$x4v`0f1t
zXW!l3ZD?*DuA!u)v}l@xpl4lKS=pIcrrjMK9H6E|P0b#s%A<!49RdXisD7Jk9S#nV
z6&xZHr-ZH!)2;gY>gc-I-4gb7dzdPZ?)&#E+i$MbQGWYB4l9F}ih!;IR|zZ+5cD)e
zDhl%N+j&jbTN@D92wG6Gzpggx>Z+sDbfXO<c(`0$TtF+9PpsZp{XH-1%8G@*FU3v#
zz5Jy`$qRv2F3}*6OFuDyQjf^jbJp)K?60r?_U`WK7Z(?cdT_nIxmkU>UhJdk@pXZW
zAeH?~K%OcL3=9le6EU%&f#KVmo6fCVq9R^gA3l6IaQ1AkTd&l{s;^m~1O!TIPrlyV
zo-ZG|I&5jl%S%6g{YuKrTp2Lw;@0fz1w}=NQj8wHfA7C8W~WG3M@NUvq+QKjC$v9)
zyPXee1o6pQ9r=F0e!at_$m{E3cUFE@1ND!<&hQ60qcSxqDQQL2R;_kE*`(|1VnbF2
zxh_;`o@tzZX=ibI)b_lyO|0DD5CaA7mBaV%&+ih|4qD<N`0f4u{;gZLI#qI}pP#oh
zW@k~#xjB;c_4S)lPcI8_adCOlc||tFB~DXaUA>K8{@lmM$A5mioiBRA+hC>-Xtuz*
z{GCDZGaqnv`s8y(NJD9TOiYZ-)z#t0uV0^@dwW~pVmIE_PfTh)GaUBU{e5w9F?;Yb
zpFmK#vPdbwC1`#9|G(3>->d4jsr*#%;elgLZm#I5VvDjj68Zc8npJ*&7Fq`?RARuc
z=oV7-0=29PUR}`yje@g&l1WNV&b+=(_Tt5hXJ#6M79xU6i-}u6&9^ISA}4DGFMIIx
z=~9Pz7y0e~9Jsf)`sMxo_E%SjgG;tg5Mx(gUhW^fucmTS>gl%q`|ZDd`^NQ@^Ut3@
zpz2p6a+AuwefuIJBQ-??1qCbhKpFe`DvyauJ+ju*xW)A@Y|oEp{lqeH;zUq0t6^vH
z^HmHWm7<MGlgz%pxY%6v^_5}vHJxkgVpluJJ-(g4|LKd1i+QBYdO(A(0w6OLLHT;+
zp`+d6g+)b5!OMJp{Qmv;!Gi?>?>5)|F3Y;U&UdO-=#7oZ>~4z#L8Z$H?m+Ja9ZR@=
zetMdGsD(3TXHl#Ez8_AFchpTxrtryHCFJIwonu)H8V3d!FP`$ChT`L6y^%W#6hYO$
zLg%lquaD0(PM0uF<2l+b4hf7PP#6bIT^+tYXqk^>kDKzpfB!73tkPcCTw5F6-rda&
zY7$*u<}1O&CaVI9lps)auk>0MvvY<`rIC5wodxdwa*|fv(cAMb?krB<lydUWn#j!v
zXUw<z`9v7hQ`=W#$t|XHVuNMPzn{-z_Et@eum3ys&(F`V4+sbfPE1+&Zh_Jyt-E_F
zh1J#77cE}=@%wjm|9Lk2HudlBSb}<}_xIIa&Y6BuSlurnJ>C7?ot^Skc^4PCZppgZ
zrRF>9!}a+3mknT(7K5_r)O!_=c|qC&0t8}rl~g{kEH5wT64g5L;^N|;KcCOP{N$Uf
zo7<;vxAP~f`^zo!o&D`V@pS$8W$EYVSy)-weVGF)X}TN)J-5fi#{T?r**|iBo$b4K
z?_&1V%xvS8zO*&_`jX#|_w3no>B+XYH#Rn3xDdcl_2q@*;Wpl#H9w6QVt17&8W>E-
zyu56pvOAwF2dF+-0}ku0x3}j%e)6Pc?p)dM^>L~e7C#R18#Xu2HqT$S^W;-@9*G0~
z_J6+|NRHf`wls2c8fX-z?CmYX^m8(@3?Nh9fFmHPq{Kwkd)kxx_5W2>RGut+?yDvo
zy)DPH`1aG6FMnRhn1Ar#L6BCTnMQ^+KMLN~2!flxN|N2}?d+h2*5_wuJz4KBUb0nh
z;q~?L&mSCY=FdHUcmCnShbuu1(CvA1`Iw-3UdAURD6Eazdg$!g-USO5sPDU$V6Y+c
z@-maHwdLKnmWy`j>FI$+`>e~~y;uNB+@NewXPR-L!L?hAu|NOs?$aqog}=UJo||hu
zJ#E|j$gXR<Z@+!_&hO$P*PRs~7xD4HESlWi&3(Z6`x#r$s8>R-UcUSpvwnZwUn}#x
zI~N)iA*~XXS}C&}3GMK8D&Es{&i}DKf9a@^sYTtNic28^=ausB?EwvM?D={v+R6Y_
zYq>7ySP~Cf`Pt4V`)F?Yy~<_R^KWhG++y(f?6F?y`M+lE)?Kh}-8vbok`5`;tQVjZ
zBnWc;zo+Nt&wqP+yK%vT1MFRk-9g1~JHNc&*V|uozBA97J^S<R{QVn~j&_-t{|9By
z6&xahm733<J?oLPoy99{Ho>;~oBBSpt@p(xgoQs}vE=O*)oxqAUO&F(V{6$yNb_c5
z52)q^Ssk@C%l4b;%D?*@<mKfL9X>2y^(T)z{r9)GH}}`uKRrGDI-j^8XlO`D2{bnM
z`Po?>c{`crgC7FE2ju7HCnY5Xz5QjsPJ2afFYo7PXOq9Zxq0K(tt|I$XiH-Wzpk-y
z8?W@SXJ=;@mXy>y*!kf@K}}6f(A!=4cXoVSc<Afe?e`uXXk^}!dppcJ98zO;*!Y3g
zsMY*1`1<N<^@ERBZmE9x_VzZY4l|lLiR++U?XL%rg??L{U0qzBOf{Tw>ClW<N-s}b
zT^-)p)fFUg@<@u2Vd^Q7x?eAsFFAg0p6%>c*NUH>n)>DCW%dT0$W0$UpSS;fkX=55
zU2XB5oyEzSnJX0_b}y0JoOX6m@Nz#cCl%HmyLN%5%2*YZls<_(>vDK<)w1{*55s$_
zl!X;(ybP1>JbL7NWR_2aWN^Iy_YE^F*15R`T1@6mOJjSxCdKUcom3_!CXp6!P)xj%
z8PI92nYUaiKxJpne!(RIE-o(fCbG2&{<N?DE*l>D_juu-(w*XZ1%jfY;>*%3jHUSs
zrbG*W*9kx86+P>pOlsE5sf%M$XGKoaK0A?Flszi)osMa_-kcQ*C#F`u|NFP_qf%E_
z8tbn*JGuSG_4S_5`E5JD+TznQ>poC<(%}SZ`*n15Xs~pF`BW7k-S89p4=)kwNMUA^
zzti>PXu<N|N8^u_{VV!r_qFC|!RKAOKbuWk-+12Gv|#@qZt;CTKidDi)8qQ--jd#r
zH`dQNocMS8tt8F1onf*yr*G`ANMkGe8`25(n}^sxN8Xp-6U|cJSJ>aaC~sj{6SvK-
z-21xY6Yc4r`Mwn#PT2I=W9KrycE11Wj0WG`gHMDWs5pK0P3Lq^t6%DYo7E;sJlyg9
z?T3}ljr_YGJ$;?iuKZMgk)tB0aiSRDvZ?#Y<lmjYKO1ZliLZHW*V`^BqmuZy()Obv
zpAD1OgJ3&>#ey=LeL3~}3Kz$#o%<kBd7MYV-7bB?#`*szbDu72=)WnMGij26Xcfz8
zVgE-nZ~L5DDH~(;q&xnr1J8l_AXayoi_F$S(VtVf%4^xuPv;(K)mMBMAtTrL^kd~K
zPMbZ&4<fBPn%>)=T&mf;W6R4kws|K!_1f1T{O6;qv-50%_rZno%7TJ|tqy`G<<_<I
zKD;yM{+xJ;KOa|pHIZBSP^s>u`H{Kh|ElDtE>?84t9^K`y+19qFE-}aQ~e?n{g*tA
z;@^}07Cz;jeYEbk1Zz6avB#$(la}YJxqm%y<=3C`9D6;bg?Ei3_8dGY`Y2pS;oqL-
zAM)Ws#&_H2INKe2FBs2LF=6#*?<2WVey41GFS2jYvtwvHcuoM+faSN*<x!rbJENeW
z^TldLfri&=rz+!wf9kDQdbWny+`s?#a+Z_VU6dEUT$X6~?K{W1KmXJ%lsxuL5|w&4
z^<A;gcT3Ah9W!@H-M+ehul%9aC+v%(&1PJ>cYT4phM?d?_CMy{f+s%(IXy2pxSHi;
z`3C<x-#;t-?<`p_seSFBwt_v|B#+J;j(_?Vo4<J;w^>rIbN7ZktFDbTvU|LqA1_c9
z%6@T6`q{*tQ`a|!cm9!>w0=TKhW<8|4f8F^4i`9YpLF%3R`dbMCzBp|{Hc}Id~Pf;
z<wY>p`(3FXe;%^SJbnEj*2?~y^6VXx9If+xESH|RX*)yvF<SzwfymysdfRiRAC(VI
z-2X+=_?Jl8r-U<aXa4*8Blw1rp0#)rCyxhjB<Br2xzk53oPKgU|8X$a`t(!DpS~R^
zRGsvLH+y63{slb;<%{no+|hk^W3m6F9p7~(d_3e(b|7Gr)wE6gi3RObY9AhXZ&`h~
z?ARyM4?2M?=HFA^Og}eq!?WdE4pbdB-OT#>nT5@(YX|3?X8yf2L#XcTg1IyI@PT?K
ziUCg!KC^JR^PT-&hWPJqsz>7fF3NEJ^u3-j|9<xN4A)a9&(5i=X4*FC{G{cvx_i1r
z4yLgEK50~8#&cx*jP2R4ZF;kdc3l^k6vXVlPEq;9-wu7gXrVHjImH5=j~-1}7t`m_
z+ue5j#OHt7KiGaO%a1w!PUS7<<1;fPZ}8exT*>x2m*p7I^^|$ij-Wk8(~fsr`_J7K
z^Wy&8mfLd+t}gp=wL+-+#&#9=J83s3`)ujXiq*LEu(Ra8-yO4V@6N3!<M#&qlGvVZ
zr+!@Ln82N>O8TL`bK51(xBk1^xw~MtLe#b^rsj_%f)-tJck({}{lj{>8UKnhZqK_B
zncNYz`Fy2S=x*T;L9sp!Z@rY<@}F<;zkM;lk7xeWf4stq?&tYGtd6sJyY-lF$N!(&
z=jwJI`?<AV_S62WobT_X$*3^Oohz=|$NRi}Z=>m<zUZG;PuBeAnP$|uWA;J8XxaPU
z7%Sp}gWQjAbMgNpZ)lxlR4Tjdg?g-u%aizp4;LtX-z~$-S@E8APmR9t->cO!Z$Gh#
z|Fht4&{XI+^uC<6**vb_f_VXR0rv`rz9hCs_5bJoc;S+M;e(#zz3H#5zi0pB+A06>
zsMyAN$#>%(xO|`I^0$<A!_u!=J42T}xN`6Z+mA^_pVIyIe6Vrf{iS5@_T!6rIVA3h
z>V$sCzQ{2@>8C-$;Rg0;$Ga!jJk6RvS+F_%7h@0G_dPqNuW4}qdAv|Rp0W6^xW<P&
zlPx;-*RGFJyPX<uJ8Sa7_DfCo-rbz@&t!`B<zIzW$x-~-H8&*u`Xb-(*zWgxbmf&Y
zLrkR2HB)zEU#`B@isv6RyFb3Jap$Y&_43@`^=kKKZR`J@E&O4<oX4k!1xi)+=Ra=r
z_T1FSs9#i8GkZ?I)WHC*|9Yi{lU%MGnQnP;HTQ9~wWZ%~ELFKXqrTSrVfbE)lj+}u
zH@}^*y8QZPH}5^VYz99jf1fK?@!Txg`m^!z_=?8l*tTNJTWtqtOs`r0<q*RT`><Y_
zr*j!ERNi4Yv+tvdjGa$V(yQzz3g2J9f1GNV^3Dp>Z)$ZA+{rGv!C(4L<Bz9{cYJwl
zSKv}qdFQywk~AIXiRvG1c6M~=*=~$0JSSePGEZewefhaQ>7zHz)hEx`wN^7<k717-
zug6)Fm6PAe6dv)3Y+~lW?tApob5@(P4>rtwa_9EsxeJd79)DVxU-)v(=Z0gCzMd4`
zGkx=3tD=7mqU+Wz+?j8tUcLWJ>j&Q>EoTo`#h#zHF@l%ro}A0Wf6SaO>^q!fr5-(<
zy|XH1SL|1=Nq_z_h0pa-)!J3{MrgTy$<-sDPfBGLzu~d_VG}0P{@CK8{hPgp4HKF7
zZ?{`;V8uz1xrV)ma#t;rE%w(~ZWl9UTj9#*C!;UQyPt}mBWc(xqJ5+M{L?$Dl-CvH
ztc+xrs}kBQna<E(%J;V<XTIN+#+KP<IJYM9&OULn@x9R3mjQn5a-O--dN$f(3`NO%
zcGdaZVNG9ACZq7LWX@;P{Y?^$M{d3^eAK&h-NS|7Uf5fHbuvHKQ_&u*&#SKY?Um(o
z)$--bjZ0;#B2A(LU0j~rUjWJqU*}h9O<ugC*kgJ5<MPTC_pUOQr}Z4$5`W3o`?t{l
z=yh9W6qE>uultr{d2Zq)j<ok%+;wm0sWRM?=x!JI(5L_0Zl=$tp9Ok<<9b>&tR=c0
z1>drqf37S=`sJ_1GKDcwJDX!-wXJW5E@czHen$P{hfEF8j^>I*Y3mu<=bpar<6}KF
zn}6ab@rjqs8;&whnq=@q-+A4x(Ad2}=B46qyKV}%rnB1f-0Pp9Y@vQ}cG7`;Q?;K9
z8;Tn^@8kL*QdrE_Si6FAA1jYaKKK5Ue|)9c_A6CXF1NM5P?)AH*lf2uaot?^50B4z
z<Y+k_5I2t5YCh|--=)PF3T-R)7d&_)`$hVXWM1+92bX5eJ+d~g-~HqMyo|=jrxNwf
z+?;Z4k+)&ye}yOKc;_hmdr@h_T)QFf=gptJHKGR3OZ@LXoaOFQv2*vo+jV*Qw=Avs
z3hHjL>HN*Is^;xxpR63Ow*IiJ;jf*df`Wk@A`^FqE#X*pWoOmL+ht3l6E07j)N}9&
zpPI+Mx}|}S*XHwnoL&BS#gCrP;(y$;0xF7JA02vLlo*qh@p*m6ab2dFkC*4Fl&$Nz
z$#LvZsv%RK_RRQiYEM7#t^2d<+pO?wMNjweAKf)&_t#U?f4<uj8}4MDwElvKdC!k4
znGRo;{S0_jS;IXe_;9zl<*chuKP_MVF)vh>+4G2A<m`_hcOLo~r~k+-f$QJXBCB|v
z2*%Tk^Nzgpynj1WZ_@RdJI~2K`}FU#M2x(r5#OD;w%ar(N<1<v`kCWiByNyo@Jy-v
zIOC6Ne<y#Q-=~sQ&DNaU`!bw;>;HZ8r||hN`J4QpL;2g`>vKPQdfR#L3q1T$^TF)A
z|2(nvXW!g&s7dnOb2i~bw9I#(_;#*FB_$=UMx~<fk1kie7c#4HJ0o}d?Dn%qvSzJ|
zzq8%<;gj!iB6-1*+gCfTC^N6o3VZw|`>^LDvwp<_mmO8&hO^%Nn8?buxSB2Z*UTHo
z&)+d9`O;SVeoHL-^unJt9G0IY4cV?8X#Tll`>L6G+?6(AqUMJSi^4uMpXfbucMGqY
zkL#TsHqV^({Qq>w>|5#Va$m;PY5V`BuV<FrnB#eJf`H?f>G5|y&C%nPYAigHQ1jWc
zdF4h2ORhck?K_?dTmJvM>~_dy+r!z4*B{h9(VNQO{+3C6`JVR0Ve+4*ecIdXugQ4)
z>S^7hZHJ$%`J1~QG>P%^c&6Wi6Se0UBt9JYyfH{JKVTD|-Pd_LHYeMK@ZA1XDD(e*
z`jL4(zSVcPi?G_in!C$t+2yNpTW4={|5utH*4lIX#L+_I4fpj9+{@f8Q+OoLVnWpE
zJx^*DvhVc%nbgmrn)FTY`LSn@p1l6gG_Rrb@TzD%zsGZu%ja+_?<<_U{h)-y!jFHA
z_9#ko+0|U$f34CqzjwDP`>}@0F>mg-1}}`heD%`ukJ?YFw4Tex8T1DxymAM5>-W#M
zE+y5yH++8^eO%EUedSY5O4i?ZZ%aY#N>JQ!$MIG@oO{0DvHqXtstMbpPM(of*~)x;
z`Eib}KkE%$@}4}ptzTI4%4^bky$=t=IUfI<-?@Jl@80fY&OPtKk~NyDUtLyzGyUB1
zw#D@e%U|f-Vx6!_>C9Gv9{;yiJCc>nO7t|d>;2w-{c_;<#YYxG{er^3!k^s|+SwHx
zpYL1!`i`ik$&R<ZGtY!fVXBqB|6y14?e7{F7Fo_)DlPosAIlzHwn@+Wl^30sc6q!1
zR++M#D_8mt$z`7QJlEgOWUE-pH*@bMyWEH3?p=@7e&5&eS$6eOndZBDaqXayYypi=
z30?mGQ=QmqO`jj`uzvq%?OlhsKkju6<qL{E=0~4QKWSfRFz=YkqO|(?|0=fi{C;*v
z=iJd`8BWa|Q~8R%F1cC%cUtiMw_dj|2ztJGwtBT+)r8HlzrQdpE3LI*V4vRkPfdg&
z_@GU=a(LVGqz~Wr<o>?&euDhupSvCui@z<J_aH3koBQ&)abNuOayK>BDjTw2s&Wec
zwod)!v|g`+g=cr(H1BzQ=h@}V_wl<}f8Ms1mrT67LH+I5<JJKWo(eqw5YlWvZQgr9
z0ilk!FSPXBRURJTo?pek`<&sk#osr^?cDXb)c3==&f})53#3j>EBI<+HA%Se^jtIh
zD<KA7f4107{iye(*fcz?>-gvM%dUN53XJ#J+x!~T853c0ec~o2Z!@d>p!U|~M;pX<
zR+Su_qVN9etxTcI`P(m~PNf@&EB~LcQsi%znCty_J=^tF{{0c%*#1W1lzGXyR_30c
z-phC$YgYK&+Wde^<MQl(u(bfJkEWh~P&04-N)}f3^3r=QJa$hi_H-EEm~+pcYl6vh
zXT7;4b_VA}B%B(jM<+1&e2rh6&|w@CF}=R;>mFvFr5kJ}seg^x^Ks3$ie1H}+L7%0
zrTBNp8QFz!zIW%ov&rSy^3@OON}ryLVU4}{wC`3a@9Rlt*DYLDxUe|Jv^V;nG~XQl
z_I}R2d5#jz{Ljo}Ka|z_S)RDLk=L!eKYOxvaNABfU&aS_AJqTYAHaWDK7M)iUF{vp
zNB7Ni-6(l3o!u%&-O@ZwEd2VHk9im6?=PQYYG@OsUTS*1La6ROf7_xR(d+b5CiAaf
zmC4`rT}ya^*srcB$MSq%OMY4N(TMT!@}M`H_x*Tcwk*zhsm7n9F>+G(4DYU-#S}f^
z|5LLa_EY4GZ*SKKx&N~=QoP=E&)1@5kTT`DqzqrXbE?44avoW$DT2yw8+3kndi{!c
z_fFy8S`DLRUwxl<9@UjQ^i%#4zh}wnn{yA%eVw#xnfUR<-7aq~>394}R9|*1KX8ff
z&o4hGddK{!vfjD-n(y_Wj~@t4-*#;B?oIaB%ehtMZ=boDD17PJQprCb-`<{oN3?56
z{SoI!tc%k7m7g6~esF(^Qu4h04{KyPk6ZkGl)w{fw;*8cBguK$r~Vyvsj>Lq_F(Gk
zzy2p>3mzLSnErhK+13lnGqbPrmV3*8@|okt_nA>CPG+8&)};WE|10Gm=H5N?ai8#|
zXXhll9@+m?*12accj{p3x$pIwx*5Admp_;i{$bCulYKYrEk6E=scGj?`L4Wk{p!zU
zO78VOlSDtP^jp}u{sG?($;<oQ&#WnQS%2Djp?>_qnppmOQ$JKLnUde0zwdSSuAbBL
zN>9Y<tbKU0{jlZ!qwP(yTFcLU?|FVH{C#oeePQ#xVzS%3kDQt7@nu%+%zb?S`_k)g
z_U+6t{PyndYU$pNjt-{<Cu+r*-v3&Dm;LhMrv=8$oQ5jzZ=S#Z@1gmR<f<(Sd1?MK
z&%aHQDf}sRTlDx@p1u~3N#AA8DwQ9w&Yr$7Mbt*-w}*|$Y`c`{>l<HQdarW#&7c2y
z#`0RNyZ0Asto!>TF6dnPAG<vMsfS+7=WbS>I7N2<=ki}y?|6kBm~yz`?GgDKL5EjG
zpXA!{yK2&=d~UPsU-lj<?`NK0dH(yOIIHVx*RQXCWEH|8A}F|1xyq&GMDrm*_fzk5
zzyFvqamky$J>|FmzBb*txy+I;PtmaX`rF<0`_9H3*njhkFuR9U;qSL^)cF$Z3-j+?
zuKUn0eePu4hkl;(@&^*ijtiDIEP4C?Ah-%BcPVCab$RlB^U@V;klI|S$llmy%IUBD
z9ntHQX2x2CxVX6dOP1o(Qc@~Ev-$dQ5gx54zizE{5a6#BcKP)0=bS$U8@W`Jloo-8
zD!xx{Isdz`#_^x@{=%cH9zjRVPW(Ml_is*@f|AlC-b_E}a2n|&UnC5?;WRu&S#7Gr
zll#{zf9uI@tXM6e^!>y{?T=FeC0$%xM3`JR&00RcZq}a4&lBcMIiuL_$a?b7n=<S0
zM~^=ilpjBQ*qQa@qoQBGE^SD37IkuQQ4!|hYd^fusH0=a_Vn{|r>5(lzp&7`^84NL
zP|%3_&issl^(-|;EOCb902}@%*+;9f!{!BO<v{-azti4roiRhArly92oBQ#VmBBCf
zJ%9cB^aMrc51&5=e{^wiQMqksA;Z^xN<oR=C_g{{(c{POGv=4Rz81K<ELYUu#l>Zl
zEHAt0ME1DJYQFC>suy4MxVkDd)B-fg!671evd^mQO@;gI@)s8v{r&w#{UFK3Wz*s|
zUg=5N;p=?z@9(Rv7WIXRYnPXo-`bKnd1dbBXJ@_VT9t;Fz>GR)bbI&pb+OhyYPxH5
zl$4ZiP7h-j6=Zx<X3faR2wFcFeOIjh=TmWAE@hDHtSTl~7Y1{4^Lh3Ee$HldXg+wr
zncvo>pkPCnDm2G_Z;s~WEVYo?l7C;W`QQSl&W;X`ZOa}mP*S?i!Nc=p{r-QaHn<mD
zUl%(?KR(V?NKkNMj<0Q_lG61T7Zx7oWxSAJ09uj8>Hu}&&ig4QDqmg&1PBTWPW;{2
z*4)(*^L*v<d7zcBFRNJkWh|XOJ~|pIBPb|%^4k?*4W**F-`?Kd&9vZP!UC_UT3ii~
z)|XPz{;GF7pU13ju#({e?R;3&(a~|D*7>lbtBcWxGsfpXT=utTEco^&^3(J4>miZ{
zmk4#7*t;$1D3_RCOvl~4^V^<Jm@$LHLSD%}<HCZRzm-#TA}9U-`@VjL&#tGSnUX3d
z*C%qbEee%te|>S>K5dPvib~7R&(9yfeY-XZT>88+Hx>N(TZWIFnT;pm&5eyVpH3*B
znQJ}0p@Bj4nf<gmbIvp{GXHqD`~8ZLl|l;_E_`x&x_>JyAgVn*JwcOXmc`EkRtBkx
z>BV@=v#AW3p_6iBLn3HeSxD7OKupZ7Q%E%gIwUgDT3lM%`}8zj&>o;(Dbq){^Y<S;
zecHSAWE1Gro&9xxPs}#g*A82=An9nA7HCGnX+ej_v$>YVZEd{Lo3gL#ftJeu`1sgR
zs#nWLY+K38pj{;|n=WRgJUrBDl5-=#6*Lij;Du$QQc?J~_xJq+0t9xKy-hkXLD6TH
zi6^6hbjGbMnhO^$tf;8Cu%j?J?aYjYps)pH*E_e?@Bim@bycWk=_`@R>i%gT9vsy2
zG82-KIkT(uwMN7Sh1lI?sZUN!1i9-3v&&&e*C&0T<zdg~Rr8sfn}c>kK5V&pW2SL>
z#=Sk3)!*JM3|}8t^XujE9Xns<H!CSAeYgIbt*<oGEcKK~)V7??{(k<|VQYgJ&O8Gh
zC^5@4yW-c&<v;&?KA(AYmFUNhAGLbG)o=#~kEBsc+}^6dEg6A_4j-<3wQ_mLl-~{O
z*Xw&v(^**h`r4llhxxCp3~pbsLPONe#pQ_|D0LTkYieqKd2@5Jq;cAZBf|b0GA=6p
z`TI9?#_yiP4--B-IOsFWq%&x#m!O!K+r>q$p%ReM{mKauYr{Y*EmnoDUJ<@tZgcwi
zV@VrV7<}L8Iav*~Bb7@;;{Zr5U|~}iG%52pojBnkWtw%QPuBXyjg89lYrjdVP4;Z9
zWYCG&&@j)oI>~Hy<emz{(pOgk16^EP-YL%#)=)CHsQF=V@ZiBWH#RmiGP4<!z6xPI
z!0z$-+S<rXDV%K0jcvTrM(O8dL?I)}F7Lu`Z_8D-u$Ym2yw5Q45K9}c^s+@Z4fE=L
zWnNno>AbJz=ZgyqnUjz8K<p3(Rmysxq~tr>?8e(N&_W9-^Sm{SY##W{v+>-Vb{4c<
zFn`ZSw)r)mI_IfsC@D=k`!c>rkTK!cmzO5Fw?sDI^vTFr!L)!uKtSNYt*zOde0*t#
zS~zXK-zip6Q(I*KDPJbd-TVDsHE7K`Xp3;(U8|<1rVx?BlXI=jL5ou(H>cJ7|N9-Z
zrOSJ|-b#?Bn^W4FyKD~Rm`U%g{{F06f8T?R$K^oV14MKp0$L~X+<aRG+DCR}Ww2)W
zI+@jBYn2of9H6;w;u{4!yLr#&Rrf9Ro*uM0%@?#ucK!PGT*})`%ictQ&SQFbsFjnO
zdvVOpB2YpFH49Qw#r5M<R8(4YqqkkymK(kA$0P1LckhNuoZMnr{OrQ&aD5#yZtv-O
zPhT#d9|nq`O`K;rMJf--SeNxYc#yCy=cdzqyV@VGR<9Rfc-pr$|NcDJZn4D8H$my+
ze*OQwuyhc4_S{@+PA;xTA08eCO&U*Bc26>p2$9&SUHa;ZqK;0_ot?#(*Tq_^`OR4Y
z%5^EsQhZvU8X`8Q@#;ozQ_<Aye0g~}Xw|N$>$}e6<9!#`L>lwS+xh(e_gB)kYRkVJ
zpf*zLO7(yzj0V})bfiqPS}ta!yu7rOtD(qWR6DHa!-oQ`&{aonY)pQ6Wu@@RlP6(W
z=#RKtuaxU@zqud2e@{O<%k;{MKxfy9)q4K(Y(R_J)92StySX_XbV`(7{Ju4Zz-43C
ziTy|Owtv2ry&jb8p*tS8Oxgcu@t!?<W|-&8f%d7ty|wkqioj;$^m8jfQ5)IwjHT;N
zyO59&C?9Z(>n*AJ`s&PV^ZuBfMJo+Htp{x@k+3XMaqSjU)zRq*TI#hDI@IvVzoV~j
zS?1+sId^tU6w{3|sQp!PCT%m9^8a0+4E*c$`pL%We$({h{XlgaXw?L3hQ*2#wJJ(V
zO<7l0Z7h8323lvf{eInQ295tS^<sA!<lnP-^!V}0*xhE{({#X@0@SySySca8+|kkT
z&F$^|-|yGY-<W*-(X(apg+KFcf4|ubI{ZgoULLexYl>zttl0j!8?;9K>#M7sJv}ME
zzP$YM_I9}IM4iP+N4r|u+k>}c1cL0Ipy&+JF!BF{XDnSQ#h@*&!OQ(BDk==h-^FBI
zU$=IV$21X1Nzc>MbU_>VVt17!-rABm!$%EVXvHs^=P)rR=g*Iipan?w_V!`x<JK~0
zM9$j3-##%h@yhD(eo5oB3rjqO*MzMGRsNtR_4daF7BBAax3B&E4YY(56#pSB798@J
zmcB6pbQqMZby>>QRiQSY&ltP7xxun*kuj*|dU2t#-~Qi>Q&Y7;dmg&PbXOhn*p_HC
zbB0x^7N|Nr-Y3h#&R+cELn6d|i!2+JZhGY8=$xHxp8ogO*Or!+4`<EqgO+6-^4MmY
zd&}h4x3}K&Y$_u*Cb3S{3Jn6~q<@L8I7LpX`OUZU{rc)EXvGa^tK_>oJGmMrz5V*;
z=H#8l&xORr-OtW4jog^j3d>2qzr9t~(CE0kyWF_sMSz;$oQ{hbA&F1+fxPeJ>})vu
ztV!;zDeL$B@&Z+OfgB>01%E%C)(=@5HTCjxf6z`6Ha?jZi%!(4YiM+U+Hora6nB-s
zKi46s4347u4p0;&c8lvj`}utS@eC8A>Tfx(uCHIe=tS+?Gc%31WL#{zySx1Jqi%hp
z%1<d@tGYWn?j(b}vPjK;p3myAwI9BIRn^h)VGNMtVC5D|`TFWAXqlS+z8_7~bfbmj
z<<~<q(VhGEs$T2HRX!E%64&>usIc(2`?;h`iJvDvKECe9L-t$Sa!+4c>b){_wV1em
zTu36QdJyDg7d`0|6chwnW(V4@=5O~?rCUr_DSls#R*%gD89sLBHlD;kKR&Xsv2CjQ
zn$;z$y$oibJZNp|-Cd@MiHRl|7aC?7rzdTU2$86KQ2Xs>I**KnKz)6E*t(dL!u~d}
zsy`+F`MJ4^0~fpf`F1-Wbb_4cWVJ-2nIRGrL%PLu7gc|MS5Z*`sy|;{UA-JsaA~k~
zrTDk=$u4qiW&`a?acpM$c+UEL%FJ!5QEQjPe4n-asME4*Sw22TW)<DNr{=Ho%sT$%
zMXAzB_v?O%7C&7UqW7fob#Ly)oXGFm;*)AuhHPj1_bj`=cvkK0?S)$DuCqcl=cKOh
z?0GdW%0Wl0_}69MEt3lV9y4FLS@)I1?fzRUe_T1d{U1;6#W%P9sc$?PH-)#3YroDW
zi|6O<|NHDNd%GcOZOz}W*Gt~q2s8&RAltoQ9oxkJ7pCb(b8vDhT3OBVoUC?nZS;1X
zD^}Xt_g0^@UidOREyt>gQ{F_YaCYr^<DE&dYSuX~<JMZgu|BrF(&VzP#nwVL-Z;6}
zzkaR#$(tw5^!VHQMZae&GiI3UO(}mRn(#S%@ym}VZLXUgY~FwRkfOokUCc9H@jRT)
zdA{(i!o$Pfb_=dndta(|Qx^Z=xcO4%OucU^+2S?6`~9CK>ex2S7q{touNx(>xtQ_o
zZvO)h?#-4jx0{-(6};Gu7gV`_es&gAqA&NGd#Tvi)y3sqQy9DG&wicweRHC=W+iTo
z`uT46`%MuHEE_H)T`~Lj<7u^()TLcb)AsJ2GI{$=<7fR_Uos`U-1h$CW%=rN8VpP#
z>qHJsf4{%P{r8MLH8KC^-apSYL-Y0q?#;{hxGJ<=F+N=-wDwv_Y5b4C=_U8&cV-yx
zj~8cP^x7(W@|}Eo(CG(z=JL<p{Bah8gzt^#wJQU*D*g>}-k>=Bcy3iu?wg41Vhw7C
zr5{W``YL2yj3ub(oN1IQVUi)Re*ZtKvfZ)fkk;_THE&Pr?_UzRIjy3y5_GtTlate?
zoakQ5(*fJdMXz7)d-77}-_tj`-!^>ue8>ONUZH?xZ|?GHnmkut@o{sDgi-zKPc=IP
zAD$L7tiPGH%YLdj!+j^$6#;w2{G~TN)75`?!bt1d4CgmnAFeCZce~>E`s<z2A9v33
zPY^oq+0I=2oO4T!&UMrK9~LD1d>--lve&P$Zzpn+Ic9I==V`pcuQyBeb?A<Q#B1wf
zy|-pvonewGlzV%ds-mK!xfr;mIB`wpj}H$)4a|lH2GIWBn4Lw_@;Ao|{x6C5m)<^S
z#q6gOgJ(-@`Lk<bkEdq&|7V)zyVeW-f267zz2`!yZQMSQ)lcVr?V4TG^YLt3LXFY!
z(zjbbywPrsjXzsot(l%z)UkK9cmW5y^X-t`4<-6T*E=cN+yALCD&O6I^RxCsKI>yz
zx)qn~U!RjWF;!a}v}4A#TTCZr$Aqe{uLNaf*GfZL&mPwjH{X15bF=#UeZTcYwL%j9
z{`$(o#+KH-O+9*jL$s>aQoiL|-pij~wbxKGZ+Gp>=?}s>zuQZ`k*cdK?wn&+HDPZx
zf6<2rj!H_&`8&6^pJdLt_~!1`=GfieioTSHY@3}UEm}CC<Ayc&gli`_zP-D9blu+S
z&W@hlFAMqSmDQb>DxI`<ciG0*2`ruFe;*`wTI`q}ySr@T_XL(}+0t_=CcUhUHawkp
z*ip#vn#-DNTa+uOtvY|;)VbCKnUj|zCj|26oHE=ezgKMgrP>vNcRs(5TlywyJ5$31
zwW;}*`3Bksv(MK2e!Kn4tE=6hW#Vh2OqG<BN;V4$3hw09<x&2mzoY0W*V|iLlYf4C
zdSzYgY|G+j6O`Tia!T03e74T|;I(&CiR2evhQ-U*Exy?@?d9_oN8cuXd$M<ib*)#g
z`x>5y&${{J@9duT@a?^ny~=-U^pdB(tNEL#Kkv);GYRtz*tSV|y)gYf<JFgE?HBiI
zq?t{SD4mo$``U(_gDy_nRWH|{bKA8<TCQ^QW$R;SzDF5rnf#q_XxDn*2YaW#O6asb
zc4}$jnz-+l4{rx)e16sbrGS6={T-&)w<I%UtCgtTH~lSt)1rNue*J`hd+%f!<(>3+
zC2DTCKHT_NZfTp{?PnLStT&Qs*#0bkQSD6`2R^C2Ka3V{^J92$dVToig;CE-K7Y~6
zi$BcNKDSK&>hIs5zYFzk`12~@f>(b^xAx_;XL~m$9}n75kl4l}=@bwk06H+I=I5tf
zA3Hia?!3A0Kh@#M{L+VqSV7C~KRrEN`EKX)h%Fg`X50<yWISf>Ive}>bI>*3r^h?K
zEPQ!l*87O}8;sZZh87h)UDj^3{jc=A%-ii>+m`<<d7Ad^&CSE|_LkxcHs;Rsd%o-M
zPu*9?7%VeS2!yfOIiC-=-d&{{e9d>+&xXn)NvUU-l>dBky09hBo9)2ilpTL7m&!a|
zydbS7E_C)v?fS&?^JLF>$4*IoyTts3U8%IiqnmcGj1O&JD$vnbJD+da{g#wpy;83C
z%^X+0l{nk;^>}Q%t)SVoW%~6Fmk(b4@wMOf&5uGoTd!Tswam4%%M!Dvn4h(eKYyfe
z+LaHt+o$f&dw6|*-J-~qTV5;s2PE%Wu6HK+w#}^4YW?NE1L|fOi{8n)wC9y`{cq(y
z%L!&8Up6}3e4XD?`}<q!uP-lM+}wipR+XCN+*mNXAKY&}vH$&xi;F=`642Sd+w<f1
z{rTj5=kDFjPj^}*O){utyR^@L!|Sc(&;Cs^let_r>8JDbL>?)Jvyaq*Cgm3`IFej*
z?`twc+^-rA({79K&nCa7CO)`zey+2}Tj>{eX+L?Lb(*dEQ!g*_-1G4h!@heq_qS$c
zp4#<2a-a6hV~3Y*%y?IG-D~E|kI(OPo+-NV!u+%BD$$diGc>R75U=&NvS0pp{`#jq
z{*rC)@15+pHBaVH<3_LiY5B8Vu1^ry#s7Okc+hs+3M<nEN8cp+{LeR>CU@YrY|}^6
zCQJXyDxTMqqs}=0+3}bqZL@x>{5tN2gU8wJ<GJ_M)kR$LeWUHMng8~VeZLZ(v4zJ5
zoy&bb_XEor+1o+al77sdzo9pF^`*9^U#W+)zwCTIFZtu6qc`4`Ntt9E`1A8~sjv`u
zYW?STP)h`~sr$!=#AAK3r+>fS-+wUS#I@&Bm~5`?Y0q94|LBbW%{jM{%L{LNrCH|e
zxwP4C-^$x*D(^n1iCMgP5H@*N|8xKEw$A^w_T0I}-SFgs0bkVR?`+rdVsvwVtn`<R
z`<pl6b#-+4a@!q;tS_gR%$qN^o;@L_{anfGYiED%Uw2ia{KCv6S<TIsHHNGnufMLj
zy20$|8Mf=YML7zVZCjnYdq=+TYuU1#-|m-RzB1fhs=VT`#C5}cw<~S_*0{Ys8~wB3
z?zMGtH=aC=-}rLN=3Vvc%IxF|3O=j<ynVj-e4DeQanSCv+_3d=z4PbGg9@lWKR)h~
z?Cj`>aW7_a{l~aEeEqW2)6-Pc)s2gv`9y8WD7-&sw&MB8dv31O*_-x{v(NcV#3%Nv
zyVy28>eiIFv-|QAug2(i^{>v`ZvU}v;_=sCT3Ce->}$MxJX!PqhpqSh?dvZrFX6S4
z>N!(%<HyVOi&Z{pJhQf(xr6(SZ3^>tFCMwuGo9A?-SR7W{m!qb$VkQc-)rT!xnb6}
z)fw^(p;o+0vbQo!3a)mwE`8NzeNu&a$8*KYr>|8^TUI+i!Rh#4ratGIZ8_?u@j53S
zUCb`jzB@bJ>(`&R=fA!R2-to}@n)XJi?3QCD?poE&(1PkoPU4c9Gl8XhRMfLe%nLa
ztto+^W1^zB=SAp<fey+3`s(WY&Bi7|e`>^M{Iq8<^N%=l?32GyR<_bxH}j<r?%P!P
z{kN9r$T;J>GU}Z`-r)x?YoF(*8SSXzW!Ap7WYhh%PEDqt+fA*jjgyOQoxZ+0_<hN%
ze$)EK1BLsGIM{!y*PrMA>1`q2^75L(-6WH8^<0gN$7NSHKMlC>T2wb`O6lus7k8Cr
z+x`2I>^I-;Y>%YzhYuev_<Fgx$W=)3X%+F;{&?69YAXNv`FV5M+gra47?!ooX`h$o
z(qa6<B1r60(e&+x64Doc&1^TW7x>NgP3HUyL!Om)3-hK{EOB@y9~7{-;GgwtR-aPV
z1N(aSx*OL$Vcn>Hv--=U*sCuO%(CBk(^+5X+dI`$(|#sPehPWG)h~6f>Grh=Cnk74
z2%9|N761Rg{GdVD1z}s=-_3q)0_w;|Y;2k^LEzcBx!#A{csqN0ul9O@rY%C5Chk9<
zxBc_0)$2c9i_Q;(Hpy;oN_{{3v$rRc^SSK!jREUF&wja$^VQ{fMl&PkEXg>kaJ_#0
zy-oky-pAXN9+IrOBd4_esTq5*OL4;9;x(x^-9O#?kjo$ZEd5mNoxjC>s&V_3GEIzk
zn>=mhO*bl^@Y-tTo8QNc6OYE5=ZJki9y2fZyV-Ifnd{TOnDvC@URM2kduyH}>;3)q
zM+~1G_;zLCg4&8z7yg<$965cuduQ?UOM9!!udR>w@0BvOsQtAiFA6e&RIab6*m!Mi
z^yi!D^FjNOKrQ2iPOa}28~Z$Yuub+xnak4Z7tQ`V+t!Ji7_wYaUE;KBZ_f0`>wJ^f
z{r^(4V^{Wm-^cqB%inB0_}b{<=6!4nPVPQ0oM9Ft{_4{5HO0|$x-agS`p!A~dd!h6
z55C{tSA0FEwso_5?#sio(iL>ArWHOrIpwCwYumZWEBR;m_t+jg)A&`QbO%##^i<`$
zFDm<cnlG)?c>ZdE>Z+^dd(HA4jF+vmpR|j^N6Fpn+`_2#i7fjqdp?OVYj%D-SM|4I
zN(Mu;2|NG1(;t4!;}3s6_g3b4|2Ow{C8>$G7~WONxpiafKX!(!`|8e)`5ouoK&Nr1
zytuIN&8@B73l=DRdwYBN?CIc@*(c_9_w_CFn`;HyEj>*)dQ-|tp`vGJD&tuX2uDx*
zm|LOL_vGXm-oL`DvpJIUq`tp>cg7_*PW6Pn?grIka>*~Z3&+Oi-`FwHIgoSR-?$%N
zjGq_(dn5hg#`j{rqstm}r+v_LmfUJ8_+8EU#r-u~ia(36PvlDv`19`d`<vCjlsY@Z
z`f}zt&y^ECuvRVJ@7*IVeZ%AY#RjFnPdr{8WBl`x-<#Cej_C)wt&8=NjlVr)fSwER
zwN%-^yz=nYTPt%<S4f4|&Y7kg{VC+?u^ukp2TM;XS6sT@@b*!h*?NDb2g_^QFSS)h
zoiDaAjggDkJb7{_pHJ)R&z1c}>psu_5*4g9CEu~zUHj6Pr~C!lcV~aMvgdQSm&Cu@
z>BWq|`JV&N-`euK>N#)!@q$y1htI9IEZ$z7|7z;lJ6oq!d{n4kvh8l2R@dk2>vqj5
zXeqrsw^2i7#f{p(pREcWI4tv-dFbWk<twAMYVH5?sXH%8N$Hb=kp+03sHi-y=A$cU
zQ>ga(J)ooh->tvOF8s}SLr}Dn?91>cN7ky{&!6=~=GRM$lkWYk0ckJAW{H-63z*9#
z@7;VoJ*DEruIBy8HSO+m52_TI$JZJ)o}ZTQx83*o!uU^Kw-=PihN*AUdAM2ga4di0
z{#9Pj%PYQ5-Nq8Z<Dm3#vDN2aXY4QEf6}8{vHW=Cz1h27wtee~*)lUtwbrBK#wqvw
zJtvx)GhdhPnwoX=xN_X`xSKXdWFx!+R&T!~fBj0=O>gHJVwak~UcYi8A?xTS-NTw`
z`wl&eoPXi*;>wrbvOhgL{-t&y<G=9TACkTq?y0hVo@~tUUCo)J`v1-?@9#gbd3@@6
z>igfHXOx+06o1Z5-*<09;6rmaujHwAYi8YRmZ?9dy5g<}&%3=p3)A2K-lYDa3RIbu
zzP|S4%*@FrPI&aVEzV1W<mZ`}mv{<K(~s}FniXpJE@+YIG>L4cghP{_|6*TyKPURb
zmL~_V7wOkmZWoj;o!BCB;EAW4ef+zl7jv&g=*8t0pR@Tft5-7D<;hWr1MKzxZ|k>B
zmJsz^bW}e6^WKbj6?MPZLsM1@xc;e?oAG*G?eA}(xr!e@Dr{_Rzcy<qDe;TgDmguQ
z|4(4%m*g+yGj#X;h^~46b6fejj`j2HzyEr1?97&%83*s*pY(Iq&!5Tr(%*hwUg7zl
z{oSt1i<Mklo;WVE1g&s`41z0x0BArCM1xknfoTkZPw^LTeN+0h9W+XE=Z#wOrz1K4
z^yQA_Du4STSa+{t0@qn7!2~mgf{zcv%;c^7*F6$l8D{q1wq|$6wl%+}a-ZB6vMQRn
zzF6$ff*j<b4|`kXSQi%;#Q>Kl=OQBwe@Ncr^xyGQt<52;-NyN_*!iI4UYh@I)=W8Z
zK7OzJdjDw_`%6|Ixii6yWk&FAs~4$mvOb3rUL80k+swJc-BBa&zt*n(mO>3>kDV%)
z$6eI<?s@c0jKlAD!Uj9OKRZ6x|K%l7LBUFgOp6sAe|*KFeD0jn_dXuX*{ogBJ7@Eh
zsol>$oV;*-#S;skAD{FkFYFbay1V|Q<Zb(#a>A;s$`c+fC|ma}_V;zY`3+ZY&x?-R
z^WTlZXG_%O{i~<<zuy1(cjTrA&r_0*@7l^>B6*L&K6Zzak`lkpSC+05wi7=;Oqg}e
zY_qNnW3!^Xs<-I!j++h>C-n#J5A{y_mV08w2bY}aW5!=@bNRfEdMTgPInhb=^rQS=
z+%w9Z6LNn&<T@q0zs3K+;d0p)p+23(L9aLe`4sJ66CD2;JdEH~$SCh6dVcm-y9F)E
zEoX1Bw>i9G7r)A?t)%pcA=rOG$DLxn<D!>*g-o73{`qflOnlzoRQZknOP+c9ow87h
z`lA`SO{KNw{ipl{&EpdHb2yHv7bon|tNA6a?|Ji4>xr)5w@Kgrc&#|xx6zAx-k0n5
z|GT)j$ff-9b#VDNced}sq!x)g9Hlol=Kt?5?&PaE%a-IX|44^bGxEgTj^=v7xsRMX
zH}OA|3drnMeYAeEi~5p;PiK5D3b}4bnXze&i;Rt5i4)hno!pbvkN$nu>)G?@!ABOa
zDSHez+*~v9Pw5{W&WcIa;c<Vse=a?n=woy8%fGcVo^D7~>6FM;Vi5DbrE`2mM{`86
zozFxbr?jLul6~`g*d}w{*{cz-f@7bg&xD>XixaHLie6%$4oa0ydaS<K<Kr3Mm6KeJ
zX86xM@pAF;og(v_!u$68_#MsQuk%vu@y3nWcexmxW`&e}je|~Op0Gbw_)&yimXG0%
zmWbM`>q~34RH$t}^j-MMXW1tI{=}FS#f39>9Tqd#=bH(tQ!Y=p_o!FcKC36%Xj0YD
z;$3lF*1wdSXYV^!)&5sZ*Fz=geOZ`(^tl}~Pa8~=J8+)cDqgrlPx#-}_C5bjUXs3_
zf6(LAR1;{2nav*7VgBzM?a;ZxS0~zF;)UFkU*>O~sMr5=*E5BSExwY@`~9ciyRhoq
z`|}TapNSeCl}mnl$sla~?>QfGHK$a5b*|dcemVcntS!=Ia~apnyfCw7W&H9v_fFmB
ztrB>(A$Vt5>bACgR)rn+bwFzX-}=IP+%J+u1O+P(fOCxTrN5mojTX-T-|uYUobWU$
z{pX9Vif6d)`zx@S`AZ)>@*{yMSbUnL?DPYj8#bEM{$!rTH}BZ``ZtNk`-@)&UXpzy
z{o?i(?&!sHj(1sanzTS|{f2uVL>^E3duX0*^$kJ0>3vf#e`h%+%klVYS+b;gu_Qy=
zA-U^cnB0ute~sGn?_|&|zQ40%tG{h{-XB_GB5`nEqxo&WXFOuF>i5m-l$~2U<(O7&
z#Lw4ab$>HO?N%TAbaMJ*j=R%%8rn}5oVA{xUbWro%kL7+()ihDrrB2C*!-OTS?v|p
zhWz%Dhga8cZ&aT5cCnpuO*B`d#EtJS6Mp%6EW5uV@Jp`n6Q0`FZpO)f3&VZI`ChuN
zc&NFL>;K&KYIR57Ka3AP{eVsG<^O%+#R|C^FOploY_|uGunA7AK6mQmex(i@*@NqP
zy(R8#Vrv$QDQsK5fU)Moy8yBO|7^GN<ef`+5!R_Htp2g~M~zyoQD@-R+@6>Jb&p+Q
zuGsBtu+Polxu$=AV(pyva+ycRW!2|AT>C$oVUI~+%j)tAZ{p|0?X6y(J<)&VGM#&S
zXSBY*vj3LLbm7zL#!5DgMsw9al->3^r#Sr>!z`YHHvRU$3gT<{&ZNb2*~eU$RI07{
z@OkTl*WwPV%lD^Tyz}!ML)l#A)60$WvW4=NR)7BH?Z2F_*feNeVZ7b@K)wcF`EpgA
zRThWK->10b`tv<J5V6$!^mgAx*&ZBs+iOo~tIl2c?ZL~_D_4HAklwD!Eao%soJIDh
z^+#4aH~qAIcZ+kTS=m}IX|7eFj-UzQdLgli)#vy1?#q>4T75FYvc+!gtqWR978P?W
zTWK&=d*|l5Y^756Z;K@w>Ls(T>K{IO_VOHO>+k)`g17U=$VTieeI0iF<G$OKbMC3?
z8NN#}+HrMB+R{TW@9Q7;ebjgG^)B&6k)N+JF+`cHn~~R-uyfjbgHszAofAS&Me$sJ
zen+|@U7&5dZWNEhTQUFLZH3FWO?WWP_MvzBhS<Na6QxA;-~Gy4+qP9R&1^?sjKTp9
z_Q{u@c~$+Iz<yv~qh4(L=btyOuMD%{SIypgggu>m+1}o^CTJ<~N7_w4$~{1jnX_TJ
zs(RhD*V9c34<A21;o-h?{<q8vyp}hXtQB+ER6Ip>u2t#bxhazv?|d%!(tq`wMd-^y
zu4ta}>lb!x{!@2?EzeXl{>v`aU>(Mmk~bCKT|GJ;Y`>zoNb+8Tyw-7T)w@66Bp69=
zdw1}bqAts%DMhn)mkUaj=^t*=|9_+|K~di{(j(x!AkX`YbE^86UVQ}V$Vzc(LF)TY
z=M5yJ1NW@3uY0zeJ56o#=IwfB*Of!o%hg<d%Xg;y9CyhPmBQ=i=bKB;v90*V>*Aso
zv~S0A+qFlhM%P`xe)aJ44fkGJN^rjVtSYa*&vXCG*$g*q-pWe(RktS=U%U8h<DVF7
zku8P0UqLKvc*fFo!u~<qbC!EcnX~`xdHI<|`P(~HHB-}@{CSUa);xZ9Pvz)Ct}N?s
zb5B=DXNzp?EHD4D+S$cC=kDZ-96{5)rk=X<aIthw#M*l%5*EFs#m5AT&ej|$x88F`
zf1+llvtwh_RTB}=Qa%wTSC#I~@8!?8h}PfzP<2MWxN_Uw+c5>XJ9>F{UuL%u&RpHR
zEal44veKftC!frTW<OK^TKik%dhW^FCfA6Bbie0|IaTqWr(#A>x>%Lia<`6RiJViv
z4EXK{gd|&TyJs15Wxs>k@0T_~|LybSWmkWDID_|-eOQ0s?(K%jFSu7~{t-B?AJJ*K
z!M0Q*Lw?>R@FFA7R*Pzxn;u=oZ8FUVr7!0iyGJqX`Ztjc)X)0+eGlW=8N2>@yb3t!
z0;)B(@+ouH6-%7I#$RX~mXx*r`%S5t*ZxXW{&C6LY_{R?4cF_h-@QBcHrM{$<B~<p
zHq4oKcCRtEUaGSs_W8pc+bTh6f$A#r$G1%qx4l%2T4MKIU)9jA^#5WP<{OJFD?Z4?
zY+iCEf1RAG@xit6rL0%;r~fUCopJX-Njq=zM!{HVJD**GzvLvvKZIOK4X@zc@c+bm
zcD>#ARwv!QE*W_H^3<C0wf`9#Zb70Iw7}l~X3m^ja>qKZu&=bv(dPL7RiShHj3;8n
zPyFk=YIpoTS=-*RD=j+s;j4AW)fs%M)@q8L+P>iEV$sZBpV)UyeRsBcuKe){%eC^q
zuYJu`efQOt#7BRQo|*qitoPoWN4M&8e$TS`cdR_3GwuDl=5uDZ{0p=$SvqV^TAsM>
z;goO8+%K;^k<_yJr4jnt{`~azF2l@B^XsSPRd(CXk&;L^PP_8F=E`QXxBKpSKKXc9
zro<wM`Gw~73+HV_tv~(L4rPq^o#?)7oqYpuhvhG2XRA7n0~cm8bDBtRh`n?A*;h}7
zKU>}h+zN<zUD-FkMC`?ZS##(3lsI&K-fNd*KkdGf@ErNWN1Oj_wBx*!6}0OB>ownU
zK2}~%t!3Gugc!pESvz@WuA1+x4Ov&Ul9H0-9%!iuneAcwE0(f>r7PNZ-+hx#qw8z$
zR4V1D^!_@#H}UiJWosTSeZ?3)nSXYAty6#hv!&~Q&X&LYy_M~U^yPc?pVwS`xxIL`
z%lU6^%g)~NS$RwL<(+SltDe}pufO&;hJVfeRWDDl)?8R)vaNgf%rE=Je-vd;dR=`?
zOLnH&qfjmJPdnB2zDut7TBhT{yWyQL+o!(wMs4Tr&Hnc+E9>YY-NT$F(hV;EIR3VM
zWLDq!cGh?Eu$L#cJX{~NvxC2oeVWjfS8JvF468FaE<06O$j>Z1`}%ZUZRb>3=ZMD}
z3O`&hy>D!mbcsJd_~R+Ardz#7z;l4#!u%hU?kahC===S8|C$<`dGqFJK^A9pI3<K7
zZa!9Y@ztA`pU%u=kKg=JE%J)jl&fA&Oh2A{Jbpb#Rg3G-AD!|qb=Piatl`w0;WKUa
z`|Qm^m4<8|4&MLoH?7!H#Z0TMMI^8Gk9F_EYY}U%u>=`RynLVEIAOC}(W5J>K0CPr
zkKA_`p9Egz*&W6uFj41h#>b=gCv`4J-S!}TpIJ!chuz)p^t;!~Z?Cmp%v%3{H)law
z&EG%o3q&uRnX^m(*=woXEkA<yEZMK80GgC(d<NYm4_a>q-YyR|8tNk~0)OW3{mqxZ
z>d$?Py5*j;oxHSJPiYxVQQY*$D5W#t+ceu#aZMauT}?(TbqWey9Zd>TG*5`7a&6KK
zaB*b3x%$i|Z|Cr1@B5ET@p5%=h`P7_^`~8vCQqJRY-cG`{k-zrX9fn2Mg|53hKBaO
zN+w1dwl|x0skiL^tCc!w{Yjn09|PZZg{Jqv3qDe(f3<GulruG!3=9nd91Q$s40-?7
zY_0OSS$=#g*R++ZrM70Rs=KRoD$&rU<*vormGWOM-bk(YnAFpHFO)-3fPrC&V+lL!
z2YKUHOU<8W{yuebRZ>CxB%Yt96|Zd8#{6tft1*qrUG?Kd!jYn0`CHMZr%Eqf+g!+>
zX#eMH(C#{^8*A2Ni_iNsaiTsu1A_n)<AaAEPKO35hStO|p3&alHAQ1h%+61%QjS`^
zGOt^{?0fc$sEZqeZQW~^zhIx|`Zshzf#+1cX}$W)P|cew<t*o#=hteTToL4ZyDEw+
z_x1J{ORoM5_@-oNn7O-bp9llP1hztdg@%wrE57{S(q$Vp(cS-b)*N58`#q=r`shXG
zy}m5De)DnvS5a$xUmB%^1x#?Av|i`gZR`43sy7#(fBE2L_okamdJeKq(Z29~+t)SW
z=U$!p>D{$?=9AeEHJ_fD?L6(a-M212*}Y5Vv~r)gF12*A`}xyppF-QtSWeMCp?~!7
zMmN`2PnNFUbaF}0%=!Ore`Ns2)TKE)RwzfkEAg1bb1Z$zMf0Bz432L*6WIJtdD8xq
zt2Z6flU?_!l3B%b+h)OE3(~}O%dJAxmh>oHbM=^XYOif{e7VlVH*4E;F5NqO`Rcyo
zGg3}h*KT!MA69cq_KjcmwG;fNSqIBbuHN#=y6#${<<n4uknXf^!FAqEO5fjKY%^VL
z^6K@^NlUK(-n&lUt!`q@Y~Ek~mp5Fv<oR78%0J`Ug=JD(ZGP_UzgpFM`9k>WQ}Zt~
zGBEt%2F3dgLy1eLpKw-|xys&JaIp0L=RI*>|K8%-xAft=E3XW*F4VS2efb~tu=-op
z^<z6<1s#Yk$_bdyCM!~SRP$w<81IXPPunJOMQ@KzoZp`LeOKl>y=8XW|I1DLarpnr
z+J3`T_e&QXtG^Y$^y#hn`f>AD372_xUEg%D*Cgv;+0?gYj@5Oum#J1CHr*7NXvtT%
zI4s!wbl(0&A6V~KFfCB2HT=XKntMC=@4c*dk>CEh%bF&g^}H^8o3mHvcEIBB){IN1
z*52A(KK)tm8wJlO>wmnZ^R`SDW&x+9CAxmUxoSUt4QcI_Ue)L7rLpAW>v>@Z^8TAg
zFP<gp74`F1T<xE8K9byfSMxiaK6zhbrm<+~^n>}k%*!IA%jZr?^2<7X&c#z~W=$;T
zC6SP*_52#k?;36snUqoYb^gc6c2$d8S6$(_ntp%J<6CobC-TkjojgS$d&=%qqu>R*
zMfIDx#I+xAP24|s!qpkEx@`TI7di6U-j=aRUwmxub?00w-b>diJ$0vlz8|}$c+$7C
ze`7Y!E!%6Gy+G>;*MyX7pTAbE@Mp~Tywup4T{>&FC?msx{SM^;4E^%Xb$4%UcbdP>
z{zc@I+qq3<-`17f-di^P|BYo|{n|?sw%jVawP5+3)5=OpHm_@bzxLc4`ZP0jlFD(m
z(icA#s`b5G%T;N-(qU@2>7$x&xAprJb+2qm2|Hh0+@^5;aku#+)t@T^)=twZ_&Y^v
zYW=mZPJcw_?no$=ZF>4mQ;jF?XXuiI1<xxFeC9P-dfNPzQR@6T-|m;_yxLP2y(w;z
ziQ1OAW>e3-;M=rc<I#k(vQGOiPCQ*JDs$wiG$cl*^1PTD-+ZipTJlK-Q;8pQwc~wb
zmo?|cUHZ>7`962WzYW<%-^Fk5Tiz$yG<C*{sC(OT)ho3FJ10Gv>uDCOclFhuIbwb{
z)sG)^t=jreY3f@mANjIM*Q?!24Sw#qYU=QE{^X;-pH+mF^2W^*^E|oY(b?Zwg~gs1
z)wiVFN#XtT>vzig|JvtQEZ=^{A%4j(?MqW?CnYcUld)g=xBcy;@XW9Bm;EgF{Eu0^
z)~fVdSW?oWDH^F$drzA)FetDvd05XmH+xrN>8gyy|D<*%Z~dj)J@xWFC)r(3FTQGB
z|MFUy^<>MrDw@AmriX4!+G(A){mty)FP?v*tE#V=d0zUqo_*D|>&&JV=lrw6bY9!P
z3H7QqSbNK^zG)ZFT*dWG3upa&95&OuKdd{^`~5?$H<Q2I$obIIFl&EE>QqjX0wV^7
z2KK&cCPtfSG5Z4|L-J2BnU<dCe_>(ZSlYfWsD|sce9Y$e`RWt;@9d8ce^VB@_GRej
zz3abTp0jaPw#TkNyqs^>tqt5(edqA%Syxw1+Ql<>q0`%A8Mhbx`xW2EFq7wd)tq<F
z?AD*sns(aXT=nF};NSi6{y%vyF5mk9N2;Fy1A_wx%LKQXpI6-f<ZtUQ{A~YXMec9+
zw#|BcAYE$vqZhfS*T0{+`j)QUg_);+UtRWRwpHTvO=7Fp1@3=;<%{FnSZz!G*H4!2
z|8jp%?xOuq%6}Z4duv^w;5v(2pQcW?U0q!*{qNad&D}N6rl_2JFg4i!t`X})^||V=
zT=sna_Tc8i##{Lo$152jB`jaF;$ENaC1Kxdcg+f`xp(*TpWlkQrnT2qJuexHW>ri(
zU%h>y+FJW(?{-M8PmX@J!s$i9)avXFUB9ZeZ)^;F$e#aare5f(%6wk#LUsM<PX|)(
zm-vO}zhunnwtBf!{=)H;DR13^GHk@3=FQw3e`Vf%zMXQ>!FFb|H7i#{n_Bx!y7K4o
zwNt5@3*2(&6+aESZJzbxR><6rvaJEf+~;3d^L<tJm5GmLU3pUJCj<$!SvO15&il{W
zdG6|_XIs`^IJNb6>Q?oo8<sm)Wh6~s^1*mnZEEf2$5Y<i)KZJK+ZWC(s(;I)w`)pJ
z^E>4yp)X8%bHfh_f3nN6b=&W<Q!R0OyxkJp*TO36Piak2cbF6!I9-32^9{AXg<8ip
z-`(uar_#F5SUkmRy2-B7OWtk0lJ8+}swKCmk^9^3L-)S@4_@D=dZqN%yCsK9asqCb
zFVFga^K{qtd6y?mT%VQW(zNF_f57=yJ5?n^S00u=^?u&u3;P}$U)7Fx>XkMRJNfCW
zNmk{YxVZ*C|35#Eb7WvB_|N0F{IaKH!|&kzb+v1wwt8J(AHQ7U``J~hd*iPiC@tI`
zzo4gdi$&6uPqY8?*giMe8XVi(>+xaD<JkYox&Jj>_MiI~66<!(``_O^Ups$oS-(<Y
zvC3~t-<&Y_>^Wt!R&PAMNj~2H+hOYGdsmJeEB*3x;gOWv71ghlo<`jF%RldTZ%cUY
z2lFgR+t}D+>ptpa-}|3ks+k^oo~^bd!tdnjO=`OGE6?1v3!1!8lV9xepSPS<o0C<4
z-ruC7XXK=O|HZP__DS!8zi8cXna}fj?W<Pf!c#U3vQ{M*npn9-q@}gv<KwxQKp9_w
zWtm%9_7=V9*u+cb3X}hwJtlI!d%N4|tlr7KYOE}*_pFi++jjTwTlBhclBz<0Sn;Jz
zrwh$kQ@AD_)sUU@zi}2%tXA{NPO*I16>F|(cTIkheNv^q`{VmQw~OI=+UL|xK3VP4
z_|NE*s4x>FLj}VOB_@YiiCcbXpLq1Y|Mg7We@6F@R+S%qBfmo7|Ly5dJrY0r9Zx-a
zQse&ug}<MdOFxybFn`U+z;H+U9VbhJ`-vm|`H(hX=FHge{5vJ~+g>?MnL3Gqq2a&C
zeoy8DhlR=`-cQYYG&9PnZ*JAzvPj9EQWN`{Df2cPoKgoDbPl&3&h~e%zqfLsW^Bz_
z76))TZZJ8fv3aJUEhEE$$F0>&j0_A64>+B?6&l#jtgFbLlk>gC^wU(;tJ`NqZcYCi
z_13B`vaYW2)TG~*Q!6(vD0#Z(-;c`W=UAo$9TqPx>SADEa8hX4qWtVZxz_EJ54QW)
z-?@GN)$2ajpu%sqYgfifXS_Lo^y->#yt)zp+8^b|Tn<gIO8s1ZRw;hn|LrXh2Tdq5
zt*>3+<Ylj3|6V%k<IUHzWG8A=PQCVk)AsA0{f^>F{23pwT#Tweq1Wx7Wf!#Dto}|&
zva-;ZhmSb3q7E!f+Nl4!*|aQG^J>8PKf9dVpH6u?@vYyUHD;T2(#|=8z5JnLk8yId
z)s!=D>crMBu5<J3oTQ@_v}?&cW9g6`5uTwdBOT@UU2sSb`h4)1%%wm1d8ZQAEPh`6
z+skOh#pCv?9vu99>W@#uKF!@;?sLDnZ!!M(YoR~u+}E4J?snN8?c`u!$UA(%fI)5p
zi=>Zl$Yo&$)xTf)4o1(P_36dbb7F7%HZHxT+-JEt{`Fiw%em&0`_9U&ICPXt`^2Xy
zpF%#?rTi|+`daZdUVM3vnwu&s1B2#bRe=M{Ps2CNdv;c5^@8e(^Tlh^v;J7-ySlsI
z-*iQ%{`=mNuI;b9_S~&KXmsjP{>Ax%&%K@~#v0kJWwP8OTKZ?jx%(SVnme;St+AS}
zqxn>!dW%y3ylYo2X8Y`uocep_?d#ok)iKMHeNM$HT%WtU;MU{R-pOmsAK7;2#cV&N
z)t<8H&Qk~Tk5<)xmaIGKr8arWyOlYoM4$XOpPzp9W^j7ROSd^bYVo#Hb(a^hsJ(qS
zQ}=S{^S3W2EfE%9r(w5!`jc$I%^{F<(XP?VC4RcV|Gfdb?ON~oVIe<LH+pjK^?Kd?
zR5;Xf`uZiSXWd-5Z<&nO6qQ<w>dsAbJ1^X7P2N(FzG%<uz02~>8Px8yIrq7&d1K$Q
zb&?B>96hJ+1XTo%-vv+F-$8DK8%u}oOY`>(dbmA3yY9G9{qFmQ?E6A*ipi_%7UTph
zmrAw$y+0v(#=PCmU3>P_ueHnhIPIFX?8#};@o&~-%74A^zp2Ldv|sMb#pb)}mmDry
z<ukLXtHVOWGs`|};je4DOJ+0|hi?8YasHQa)zzy!p2wcEY1BBLb}zSAVqmCvb5U00
zK=Oe#Rm$66OipXP%c-|tEtjL1-NEx&LaF4RW!hhAHhSsb{HBrfDns|qORm<=(`qkH
zPfh(=wZASa_}JC2monoQo-?((mic{c#qzLzP$%Qcg8$dK?r#p?ZFVknI@7&L$<4n^
zUM%R)_VWz7vvu>ly;fU(&sbA_BYVG-uc+KU=he&iJm#D-x$e#?*@q8rZAqNN6~R;g
z`yRLb`3tAd-oH}itXjLVXg6rTvF&WJ6}<;Fe)p}+3V$p6_tSH`xlt>v?z+F&5U%MP
zwLLm9v3pDHMC~)@UR;llUYzr(dfUcB^*q6%&9688`*ub2)<)%+*UPV|GcbWO{&8cD
zu(vnPe!Wl`ae4Z`Id87+eW;ykU>dl2dehXcA;GO@<DYQaw9ewcy6pbGfbV9xSM+lx
zw_WvpeyiuL^x3;LQM>kq%kMW1KIZiPRr7_SPv&0Fd|Yi02Aw;6Ysc#;);B^gZ`k%V
zRBg#hyLXXa&oQ3Ln7Dtx>r5lHtu<_HKTbb-zy5$j{oh>uGe3l9-S{Z%d1<vU`-&nx
z`|Hn`=3lPPR@d6kGby>*YD?1J7qGPe^?Q6iPRcP~xAT6+#QnS9uexvC8hZVKDyUJT
zx;n+=wEcznm;cwKwe@U@_#^tlpM`~ip~ZorU-R;1{mc~G@P^xFWo>oacSRi%DUa}0
zdlSA;?(3B2=TlNQS2Fp3C~g&fanp-!@27Vb;S(l_e2iYA_kW}B>$!Y4`Fp}{`^~*_
z@^Adj3rp6`x&N;!sLazSEpYL7yPzA-<yTcTHtfFjzFPK@+Ord8x+PXB>rbr;Q_9|S
zohLFr<omL9QYw1!Yn{WEUrN<pVy4&{vennt|MjD%$;(BbmfU+7EHfv7fkENv{+$|(
z4aeIfr&@oTJdJmnjI>GlTSw8{2#@RE+p>3FUU<>8YuTS|El&*3Ojx-hYTCz(8#*dv
zf4$w%m-feb(wC&$->+<R+8`9W*W)U3TivriiD#<u?{D`WhA#FBd8?%>qi$*T;``)_
zT2Vh(zy7XSsPghg-_luEQWs3K=BxzQSPHx!v?H9~PnBF4DERSL;s>FxSvTU{ulY?0
zUuYg`w*Gx=-cfJ6x^-g4%}3qs?AA@&`eOOK(4$>X<$g=Cu(00SrlV%$6wgx^x>;H1
zYRv_c*6^=aKb%_F0&1z8f5FG~z;Lo-Y20hwlcoOmn|^w4c{6+Vs_QGenBFI6FFh?1
zWcB@r4*T^t!C&sL5#L$Af87oFj|;8LGTxt@Are<`Ui(F8)cLf}*Pq%Jhi)rVKYMSc
zdvg7ab&hAfPv3vEcJ>?{h!1p6gsf__l8%uzR_B*p@w9DHR+Qm=+Z@Fx^=rF-)rfJN
znlWW3U(K>Y?Y?{4zJ0YV4&8l}-Aed*jmq)qyF((+Kb9|-+41)4>GYW9D-wI=g}He*
zZ9DrnWKs%O=4!9X*ya-_Qa;@go+%`+*3Ax1V*h0?%g(>2xBHfFrMBh;liY>R|MHgq
z&75BUFEDs+nWWFcEYV5P=R;OJtjxW)o7Xh+ui<qCgRbivb&Eq3OLm;vYZ~FI%cl8n
z%d&6P&U3B4UTcb-{_fq2J+<!&a?DHrov+lov-;<=)hp~+W7gK*K6dWH`-|J#&Pzv`
zSI<&e`ENqlsr+pxt1f@t_4rI>Y~lLS#QSx!pLlp4@0)Ua&b(^}?fv4Wsl-~=d8Dr`
z^Mh5XH{R~rkUw`{y!iX7cQ+QWde?`Q{F&SAxlH`^isjZCU)M!@J+of3``hAav$oaE
zUl;MXH0}KNSC<Z&X8u}Wc~>N7ijSpsR?_t6wy*o&Uv)UzZ5FgxY3Y>(zd0&Jt9+er
z?^J8;x7-w&JpJ#Ij*e0(yW-2T&pD@9d%WYl|ITvT)`!O@?UgFd+ONd7lWkq6W%<Fq
zdwa6}ADyoCb{^Lv-}#+ZGP#rAT#WjDRiU@5V|w{t#;WCsik@nbmv%F4czl56>Z>pF
z?B3iB33a}IJ#nYggA<H<Lzmom{p+s4f|_@MfzS8;-MEiSdac)2>Fn=QUCuc}S`i#a
zWG_^!#sypztKAyxxi<E5ds)cONk_LP{O`5iV;R7D`p1cctVO3(js8D&uf4RoRcmiV
zkg4<i>ld4PzaHg}zPhGB?fYuCy<gI1Ec=o@@eAAYMA>P<U+@2N$$q|f*_(IwChwbB
zT)k}iikM{wWS?{PSbp2vC8D{KPkDOPG~ZWO16;dGue5Cr%bkD2H}&<GFEYQsoy>o6
zRLOPAmw+B0%Tupcmlj{y92a@B+$Ui1C)JQ86_sEACH3CoowZ0;erw5n=FG6Q+tn(b
zxomOQx&FxbeD)>&)n9j7->~9)8?x2zZ0LbW&Zcc<pII*H+s$41&$upVZp^+p-3$x|
z4nHWdnmdJYhyB;r*RQV%U0w3`S83PB1vmX(>=l=K@gv&UbboT^6y~&-&!XOytT_0g
zRVqxsT`#_Gvd_L#CGDNJ-d@Xy-F9pzQ@n%P&)NC$#&%X-)02Hp%~Jj_(MM7+zxP^c
z_8gVTIy0G4JwG}>E>(TCD`k_0!G%dVGtZ_31|$T<Y>n%@^wr{|e(;1pXXkinzPzxo
zSxhHl!HpXcKYs~LVr1|aTh_twA;M}~*0yQZvcDFkO{|akp4Q{*ws?P2)1Uu^v5Sw!
zJ5IA+pZ<+Mi0{99`I=|DZ>XPKb6~-O-4SY$OwI3ZG;xBvq#x#Yck^6tZ(jZ)Ex<7J
zPrTJxSu20Lv&Ua9l_@Wr`RQR08#p0MSS54U!1=V2l8S4_T_ex9rJ~=ePTVg$zQy%^
zz?9wQzh@Swr-Uj-?Pp+Mc*n$in4v<)=5tZ@^^BR5QckRYxcT@BpM+DFo#sS-Z}n;^
z`NFf~Q+xG+so%wA7EEAZVEErK&4Hnx_5SksYg_j2PThLUbKkW6k}^kC(@(s<wRLqi
zSGb8@CIh(4SlJOMcKUAVE0fuh--H+#9@Jfs6#)$~&r{gK!t~?(jmM8g(>LEOi`*0u
zl=Zax`p4MaOZTWwHUDrhZr7V|zZcIR^G(&7`JV4H)2-5P7wlZ!T(tx5ah>_QhU@vM
zFW<A>=U?7u@26>H7<lg`)71SQiLA5U*V{i@8+P+wka~Rg&xvP}IUu3tAT#5Bysq2d
z896gIUf6wW&xW~w@;T#|m#q!H@nK(7#ffF*&vF9q9at{s^P}5t?N;vPFW>q<-)V0?
zKWNt4^L0PVYwv8Io93=^YPCbAZSjkA*=fP4yL&G#-<$pSscqTYpLtDA>^_rJrfLQU
z2nw>RuD>L;_k;0OO@V^*zXi+RuusjpVE=RW9<S_o+YCgvN?1%}U|>)baM0SxF!QqK
zq<PuqewXjB)r#L8JL%%(U8R37MDb>P0=53;eg82-&gI3<Rab@EU%l18UNKE#-Y3tU
zp}Oxjw1x6rbz1Ix_1F1p=?hg=Vz2JGchxlKll3a0=W9QduYPsjZC~<fyPy{h`Kboi
zF3vf-IONLx1+S~cRyn<BuiEa(Ub%k8>Q_5!-e=m^e7<^do^5H0iP!RLz9Dn(#d~fl
zU}R@t_}*I0)L4Ho!Qe#N=9hndeinHge`2Qja-B&%Zi{=it0*MzT9aj*zckFJ@;PJu
zqW(?dxhv((*M`hDvMY}%zTeat^`hj&>I>Pw6t=WfXS$jGe^C5%y5mRV`xoxq*V`#}
z{LR%H|30gF-c{SPbkV<c2c!b_tXx*2t8dBwdam8)(x>yjAJ$lWQRC!_1#@m%CjL`x
zKfF-cy|3aQFEeO#sNlMkS&oM1s-UG^e6m(9j_<;obdMgb+u8N%=pjqdDTarc|2cGb
zh1LXGGBJz3C~rTVa3@h$VsF?akxNQ0k$Ha^6|xJrau;s>-#dG^@5f1fYk&H^p35iM
zQyQ{1<XXyXt-#gGR8H0<Y}qE?zkB=N9j(2apT>9g_lNH;d;8_}`u(fY&d%yF;s*8C
zcbLz$F28m?zJBfH<^HR)uCBVZF`3=9lH;9B%_C7+*}v2Ouj%UFt`?oP_5SbY$L9R;
z<-Qvmc-m(E-)#z!3rbWygC2dlYL&J%>-n$uDxSwK{SWpl{v@V$e@<)Y(FePp{FA6_
z(f?{+@b;GJ)%Ef5ppCne)qJ;<ybS6Bj~47GkKI*r@%#Py^*3%rTwNLLetNq8a*h6w
zEz|Zzx;{NU{rjKrhfhyhRj9<|Pg(yU^4i1G^F`|Wqx-D>wEy;sjBIVnmp4+|5m9)Z
z`8eC(q8}aqBIYZ2YQ-+R@Rs?^kJR9qm+nnk!u$HW)-3&Z6RnD$^?>d^T^qgq)z|Cs
z*Z=(doXN@pnt-{pe^=@2tix@*n<8|wZf;t7=+GgJQ_7QFa}O(-EKRRGKZ$Me?3Fh@
zU;Uaj{m`T(2aD@2HQn=U-hOB8k=YyP9e)`lVS4Pv>*-&P%v*5!UAx%)DQ!6)C(fV0
zyYzz6va*U6(Bj@xQ?+0B+yA?8VxqEX-Jgoe%8&c)L0Ra){*YxplDW6H`R*)!9=1Mi
zuSv;^fUZv`Bto|x7brE`vFIv){Mk$WtCz1lJ*76wpl0DL(bK;)${b(5FN@W_e`E2c
z%Fk(`>*Hbr0|RH7W>33U{oWQFut7`@4xXHEzkgfq?PdS}zOR=u&6-m6<%Qzx-i0=D
zy>DLaDwv^ivhaiS-ml-(KdrNO_$wv0$i#oe(Qfh8j?HYFZoiG%nACb}OQv%mc#wlX
z?%JBjt%Z->uC9p;Hcmem^7+}>)UB7keAf6OnzP_#?W*c|+t-C#9zA*g{j{@^)!(&m
zFP*(>`PGjCzB3FQ-`?69e5^-uYvyIO(DiY17lF%nxg?2?pD%ApJ$+?gZS~r?y-{1U
zu3kFYEv|V+Ibg+)&GU{MSNmP=te?2e$Y$!{&lYB_Z*M$~VLo_bLcC?Ok=F8<?|&kL
z=X_FfG&J<{4Eo0uneuRTmBt*u<@f6U*M5C_JN)P8=hwfzz5VqNw|;;Gs0cl9INfNb
zN$szaz5oCH{`GWvyoiv{qU7U!fiv1q)<p%o`FL`%Y|Fn=erntAdocn3n%1%R@mdx9
z`u5Sr|NpkjvP{KV*#UZ@D__t0Q2YEE^Y;FKlErR<*21cumY<)cB+O`6P*ilBYgKw_
zN8#g;Z8?#x?d{tB^K3-eKz+Cm8NBj#G4XXjm-flqubVJI;O(ug-HUAIu&j7H`ToTz
zhbv899xs)O)Ss*Jvj1c3)(-(^YI84|$7`NXFc$oM?(fsk)qHR7sCa(Lyyd}qCT;4v
z+xNDrTi0KlS@AsH=z-(f^7EDv+nEkNaqRBv3tQ~gYf}HO#x&!C!|Q8ngO~Zt)ZhWt
zWDoX#yp_E^^Wq}cuD-sowNYEQfHoQz@%!Afn|Iu9*_7KC*1ey(UN8P#=vK$WA&Tx3
zUVWRp;L^KyGo7Pq)?U`kw|tq{_1Adw>T?%<O<FSFZH>=1-sdytn-<T_es=I+Zv2$M
zplg>*LH)etwXM@~ttaa&aH}zopY$a5zjElQht>VR8?7zN-^~Ht@4_o>wjz3aUeun7
zjhB{syC;H^*#EU|y;3ii&##+wGbe0w+F6t0XFgp;;g3o`Jh%0%|1f`N!ELF+^uwkL
z&bk}TaQYgzS!t4r^Ri1)7MohNH>mnuoxQmJ!qokDu8Y_IobYXjdPVu1or}7|p6A}y
zyRl%mm2^*$n%~v-$=i0am28|JeB*$fmCo{cugx64e%ke%`^(w&_a5)^oRs!OZspqa
z$10xb+s+5QkNmCTT-W<H^!K)(T<_EDCv0kF=U?SDRqNN^@ApkhUIh5gGVyd%o}0@E
zDn|P|xkR-xUSC^#YNB#`)t46wCn~!KO|jnb=d+BcW>C;V=lx%GW?xyouuDX9QKofF
zqQScd-?c&)&%S0YTXbvR#qZHEZ%Yf8U(1TvUpeo)?2Mc7Q=@$*tPXzediGUpkZOG1
z($;r+v-+5YL*5@<q;;>dYf|YS6Smdo&!7Gpclr62+M00CNVscP`1KndET-@OZ(a6%
z-IO=~_RUdTpMCVo_o??c2Cl!RvGiDP*UGf%hM%-<xJ(W^6ZBMOyPrxu|L?}zIrsO)
zhOQ2~dUJESY3ZvFUKxuCix?OR#CiOdPo6)g(f!Svo10g~?%oEv*SPfEoyg?leYHx_
z`h4g9JNUNgg>L<K|JCLTO8tNTxw*T(`m;6Sr1e}Q_NM_0p7PzFX&W_h`8jQ~K(FW1
zBQ1+q>?RycNaE8Ax~bNzy8cwuhTJ{#HcQMqQSV&ex9|1Sr%PG6#a8&uwTg*gVA$cU
z=EY!BWm)v(!n3opzn;=wpYiI-N~`*Rd(zI#I9Th`Z~1>g%qRX`r>FM*J2mzC|Hk*t
z?{&Ys+pS#YGjow$?Jo@jg8=QYH32U#Ew!`+N7G7$fFHlJudWJxbzxyM=(cuH!T<13
zD;Ez>O;T1uce(GeePvG^d^eYbpZj8_TJ=0|tJeAZW%ut)K2!H}L*1=1k5AviQhv<;
zJ}D}-nkm~b<%B@5oNd(4Pfstuy1M%7o6YBoniv=g`d{#I{W(6zZ~3opZ&&-<{d8gF
z61i|`srT0}m;LPpkJqIg5BmGzb?VO_JDyc^_xDf#Uwrk-x?`WuEH3ZfaiYL?*5=@6
z44a=kR+d*iA@gTal$~SgrkgphZf;(lc7ER4pru}!KR-R?;^KPr1l*Bc5;wp0+st2I
zUM|kLxheDGqoZ1Jdn{ZTRgNAxvch+^S=8pVUeIwBtlVNJk`ncJL8YOCltKN!nytCF
z!_qcK7C%21I#nxFgKJ(!U|?X@ogEwN{(cQFeRCslvbz7YlSk(AFfja)_7P;XahqkH
zzwYYl@Y4wfFQ$gaUEF@Z&O7kK5v!^%D^`W9TohmbH<X!;Cm<q1f>8ie@J;v@vMwev
zdV5}IK)?jhHMOAYbQeA7+?;(~ueGJ+#g&!8vn&c1wed>31u`&HIK1HFvRS62tPHwl
z>h1mg@wQc8F1)z7`03N5@69}ZhpmlTx-IwisfPtGZs+e0J>15t$;H6HFHj%I)W0%j
zXHn+qX}YT-HZD^0o3o<q?JZ3^yLpEcd}rI$?wVzuuNNO5zczY%9|H%d-mTb_a%M(g
z_Vsn4pyo)*Ng+_{<odeU#ReXe-FhSz&a<u7vapDF`0$}t_&OO^MurE93#>IJa9POk
z?frO6`q!gw{V#Wl&u6~8wDi<WWA#?YSqa;3e|<JPUrR^F=V-V1^;1)|r`lGRaWOL-
zaCY)mm>~6UVO3@3pC2DVHSxb6kNHE_#YF!8_I9y>$JeCf<g1%fy<1ybS4M5ky0xd$
zm{EYCp}l9CgTmCAw$<CL%HC*nA6-=W`Ps{*)8iItcuduc-L+(y@9eDS=jL+p@?PDN
zdHKoPx4wxC3>7CbI9Y;bn&sZgczSB;rrU3`E-Yvi(~t8BobYOnMWGWYkDr@seRX?&
z{MlKi-H^aIaR2A#+cC#_B(JWEwf3E1(6~t9>W#hC+nt=8O1``Z?3Fh6TkPIH?O?(J
z0|o{gr4n}5ppC-nek;<>&ax_bG2vdtV_sf)yE*pDK1Te0xqSYWU8UKoDk>S@-rUT(
zwnh>Zc!G;n1w4*^dUA5{hYtl)b)&-`AL|WX?l*VJgb5CbA0lQMr}s&j=jklIxWY91
zng9bhtL~7WX_R_tSLy333mlox&bN>6l`;(~DKTk1n4EfMhT*GQTUUc_!&)1?eO1s>
zFA-LThW{LQ9UDy!<!yhx^?KapCnqO|ZpjGLjoj3-@Y~1v#TGJE&*zq#WL{FS{r~54
z)Yh!2P0h`kJPZu~6{DCMO%7$4WL;X~DP>i%qVE6S^3at*uI&7BOALOTKXUZwO5fRL
zvrIB4X@#y@vNn3V5CbDa1<wT`jwOzXiHWbSt)2bn@87Kj51pp!Mo(MxVB)T-ud6^g
z<->;p)BJle#_8v#+{{@93Ou$9PL@jx{N`G{Y?rSKsHxfW>e|}Xsi&s}ir5$}_n#lO
zHS4N~oZLF|dlk<2>wfD(a#2J3+OoH|Zf!_x{`2S0l~tkIzB7$l7d=>bdw2PIt?+eg
znA!PWJnpx@mOj7s*cMd=2Ai}Od|XQk-TPz$&(1O}eRIQb@9%fJrEDq;Tn{>fPS)9)
zb2Dh;#*MS=YIj9#&+~<}@qXyLxVo;)ySwYx&*$^6F7uVH{rPmd(M%r^HvTh*TDh+-
z^%np6>sQvvNvf-&wrYW5)zGy(-~p$Ah{zI9jb8Tl)-2oVZL`dBwN_vCO8nr^BX1w~
z_V)JhWj-^vRDMolQea@{w~S(HRPQ*O7Q8+0u1W4Klf54faf8|cO-)T2Jny<MzYO{R
z?=Pr@EUFU`usVEw*t<JBL2(4tcrqm@B4P#`kHi8{QKP7+XjSz^qxGPzw}(eSJD;pc
z-Jgo7`tflOA3g+4#07DJY`YR3U%M2PpA8HGy2bUwPEJzQ<a*~-QnDp#OU6Rq*=8?6
zCD|Oy;wgHuR$R;s4qA@h3Ljct-P^lcOh0bTj2RMFSA}|mHq>kUa0gva|Mk^XP+Ruv
zi;K>nLtGe`KpnaXQJ~}6USC@~dHwqJptyf|d3mtNpS8^Fd@G8cp1QTWe0>|g{5noy
zwG*ei|MxO5Ot^JHR-}PX&Su3d)9hDIPEP*ypqc;6zu)hJMO1|4<kqE~o%Qlquk<Xd
z(p8{q`5<xRarWF?>*cq$W@n$DXZz~P%E@-W-xy2wx^Xe<#_kG{wJw`-GUd{tR&G%F
z*~P@*pyg&S$1rKho12@nKR-LW3UpEHp+i?TrFtJdb}VRy`kf`7lSQPYRy8y*<lfma
z5mFjBL`_n3W?LJv(P`nrg;5)mT0wQk>FN5xGt{pvaBTi^Ejs_@%=CE|x8>fJa1;ZT
z@(o)~+`b*XIqhuF{e87ti=X>}iuNZbCyPAeoHS#`igmHOUrqM63w(KLsi>UXx|*M#
z0wow2E^)l$Wcje&?*E_Sy+5DL=HlYI^yp}J=pq-+u1Uv^9yy|6ViE!>pYQG}HLd?w
zv($h7yhGrg!-pji8xk6CZOK%YmzO^~)7U+9b=Xpk%qy+j;#YPQCcnM6HyYHVUE(=e
zLWYB(!KCBe0)}@hA~&aj>cJkj#h^peb{0Pmnqu<e#6)FK54`OCy|s?bY?-H~Xo8Xr
zM<c_$kf~asm#(dio@$yMR{HXi>-?HeoUWeE|Mq-0HZlsDtnPnpf}*nq7Xw44#9c?m
zJ6l10=diU=x3=fckE{I}+AC@778mzt+LT$KaD01f>tvO)v(5d#zP`R(gMnd5;h(Q;
z57x1=vuD4)wsuv>%1LUzvzD0U-qO(1>pQe0C^0cH>*Au;OUwP&e|>#j7*ZS_c+14Z
z^yP8C{i@2(&!TpftOOlz1iBP%M#!`O|K9(<vN~Mf_V1U=T>SjkyY=^Zfa+7H3$h{y
zY74FAzPh};A9Tm4Q!5v!%HrbVTV~L2x%=+AdwZ+1e|>pb^{`cZOVLxW)U&guLPEJ=
zw{hAT2~bb!=H~S4<@alsE4g+BOgVn#>}>O^+j65p3m8nZu3T8;+WqADb7&j>z}}h0
z>3yKnA4^_d%DlIya#hGmA=k<?F}q4Ud!@~<t&81#Wm|5v?_8_Ud-v{%urV~K3w&o`
z`f)vKd*0fq+To{OmaIxV+!nPZL(sLdN5;M`Mp04m*4FIlzrMa+KGQfIT<$hq6ykW$
zcyn`lf7aDiOFut9zdCri->KQ=`mR6wj~+Xg_36pUT{S;99qX50KhL&0NCec}X>nlS
zKX-Dndg$gfUs;=q2}w3;XVNx5J2P|fv0mw59!R=+aP;``*W2&cO+Nh4W9#vPFE0XH
z+uBwIFZX+OZEf_be|AOB&#le5xv4OTiGjg$Ql}6@#f39zo4uy%&0W=hFlnR5q^9QP
zna=zS3`;b5c$gU=<A)#$v|<!&1&AX6UJU@6@dELPm2y%5sRk`pWnchxH$XhdT2}@V
zrHHkWpgW0njGU>kD#M2*(&l+{>V7_*o^pC!++HouU+;FmFL`vNGi811i3yI*Z9IXX
zuBsdZgGVW3p$vn~Nl*^BapOjH)p>vWze^@e5a?oOU<i`k)y!1!Xz!;}+TXYATfcri
z=p=%N-~ma7pv4z1T&RxPpL%+l(d@G#eBf1d3Z|<fH!oviW`1Aw_uAU%U=wg$G+a5A
zVzjdI^Rx1;dDC<vcZdr>6mQMGy6S4|+UuKAy+H*l8v{d-t{XcmgQwr};GIQJFWuan
zUb_7BqiwmjL$>8aa)ITem>3-<9ZU#V9k%wL8G{rL+tYJ%XM;*pjz)$FVRteQh%dif
zd46#R69a=y-wQr2hV2>=8y4`%+s#SpesFJZ_18b2&j+h8Fico=QHX<KI|ncCUR{te
zqMAW1Fk`MQxuQB{-2w&%1>J?#8VrA2R|YNpa$0}CO17X)-Jczwo}6?pWMFVu<?gM(
z@aO2-SSMb#X3*`#43OznkJVAzax|L{Ds+O(0L5Kq+_yJ3zkWKcUp>z}`&!Q;28ITc
z!!Dc*HXMb8g;6_;mI?_AUq12?bVRDOS<Zx;IoCk9DxUcsvL?b%R4b%ISd5`TT>>=p
zt^n%g7$zSJ=r|s8kcqYA!2?GwMur2zJ<}W*_>Y{OZLaG1Ui{4RGj?_&PYl0Yd(U3-
z^HVApBd8tDAUDS<(pbCurjMeW^Jc%ffhwsC3=cYwA20x=a0d<+5JfO463$HuEKDGi
z@MM{P!D?O%4>l%pOxo#Z`ZLW>U-g*Q#~*e_^KR>=H{PmRHD~&|^AUQ__y6HY{u#!#
zCep_(dd?fe@Y3ST{F~lc`MrgDM|J(Dxl=cLo6hXnuD4b_{kZfd&1N4v_tNBJMK}K+
zP5)6QyYP+Z=TmiGSM4!+HqmiXe5cW_PleU7_p_gSGcYjtAG;qrh2cQG-ipSeeXqD0
zzO7~#{rdB_)z5S7yS6;tkaA8Tf3wPN&tvr;1=~LDow7G6&N5%_%1iV3;3rLg--Y)W
zrE*QuaWqOZ{16LX9>QRg^Q=vO_ioi2rEe!M-1N>$@A%Q?T_3+H%$3<6WaeqadUDR|
zfBKPiQ@eNDJ-m`%<grO#dfDa6VlN-PDcrT`<T{<?W18MipZ_poU~rh_ULL^k-*SGA
zf2Evs&E);h{!}j0Tcf{DCe-Nue|Cu??O46;*BcFAnof#lgRKSFG<`?nLLt?q?@j&c
zUM`(+q+%bWux)5RD>?r~T}p_<bi1y~_mfk;|5Me8Qm&r*&4@L9g3@~R>h<A`U0&f9
zv!`TC7D^J?|6Bd0o*J)wvQB`<xgBSIZC<?mptZv>oywFG8aMgpPT2XHBUM*a#q-m-
zZ#{eFZS$V)e<HDT|NfBNpHY4%c75d6^mbgT9UQXki`miq_*E?)-=9}Z`+cJDS9s+1
zzY{+-MtwRFvu{VuyiF%&Rs?stf77U2UmYOjJi+HD^Z!q67MAP0;v^?e+W7el<IS~m
zj2Rdn{F9vHw|ue@!;kG%-`=d`6jsadUTSnI$?X2+YTvJHTH%`yc{)CSmKpn}WX78I
z#gp~IB8^T|emtfdzCP*HmwMZ$@^9*Ie-rw)?%K}&<);%lf|hN3{A9l2)AOY<z0<;v
z-M;_+3hPms9h&#|_@BO=Gv)D@vRLc+!Hr9c?r!pHT32i8!s)t8Bm3x{lZ$U}%e4A_
zwg2R_rSGjm97R8$RyZk_KmFpHU55=5zt_x_-0kv7YjIBe*FdY!YkwEUP2X?5^g#Hj
zieK-n&*g_dseb%qNB{d3sYmi$XVsK^yrk+E!N4#t;f^rJ1N-#oeIbSyw?Ena<sg@o
zfv4r~{c8E;`sY#@FRbp_<o`<le?t9Dzx-<}zpX9V;~XA)t2l7dli(%Gn(W_bSS~Kv
zd0yJew5BWRuC47`^=q0%4+Q4Q83(1S)hs#TnQgna&-{Ap%PR*It*o>A_x$;jQp9e$
z{=p_LovF@+)!Sd*I)D7>l#~hmMrHEr)`))AdU&8LHcD<C%lGsdr%i*Zye}yj$4&jM
zm3{QjB%zY8pWj(sUE8Z9{^#AT>q5R?LQXQA6kPauec9Ekmpx;Hgq8PQ?o~}L{ktcd
zKY3d1y2;-q86b;e<>b%ZFWLCry=&*Jnpte~o%A)Iw-~VQ`{cCg=`{1(r?2Z+zFl7W
zu$gn6!1JDqy&K=Zo@H$y`T1JFvR5}%)2F;???3VI-WKDJyK<KEX?*-{yGhnc@%F^*
zc{|zXUhh9y_o_kYnqTmhhVP}HmsRilJ82QymdiW)?XO?``2O{$iL>X`c1D^-rQ1m@
zm;QYG^`GzS1l@V7yr!M{IZ1De<lj5}Z#KE_JhUeFaoqI=bE8x0;eOxQkG{BdK+~&W
z{oB}0D~-NNE$q43<8{Y6UR*rqaSS79h?gN#gJa)Kx94?#LbpY4THyFn<id;C)YqRj
zRXgVW6fL@Ioo>JFvhWNW#jK}#=P!TJ-&fLaU6^ft&Bv4fvA%|$(aOKNAA@>cKQ3|J
zC|y=P?KZ#s$1Pe?dnRv7wXX`{G=IA({*L<UYU_LY&;5Qfy}7Z7`(2n*v*G4H-cP3*
z|5|Xaeosp=bSHsr;r*Vll;`Fpndf(=oW8vGN^IlR&z_N9C!h0em$5Q`8?wi`Q?gj>
zUwAQhmBi%!g;OOe5B@!&^7Ee2_6Jv4EQ8N%H!fY~F73MSbLXYfiuj#rXF>}$f2yx$
z+@>qNP+NUcl(g`-KQHo3S-_1UkGGpdrteHyxPQqqjk`a-=+6E5?t4o2U#%(ARqhsg
zm8UPs`RMuZZKBWrydC`+nQQJuTB@H94Vb#9NbX1A#N>VZlQ(RZ+x4vS&F7%iDc>cS
z!0mt~_qs1|mrhyyo#E&6{eO$5Y)EN2muES3OI}}k>Li=Bx&N<Rdfs96D`LxnpC<CF
zmRpBS?!US2^=A7EFZ-9BEK_{=o_SS#XzJQokEdOj85#Mj{Amg#idN1~{;qg4HRRa0
z*q|EQf4A3qI=*|xYpItP_GZ3v=d$zKr{fPVR-C@)^6u@mZy(vd?f#tTBbLuG>A&~S
z*xUOn9j;%KDc0@x_1XA(1y5bos(SXH5^s0?x@vK^Kz?=jI;~Sd-zEJdzzv)Ur($<0
z%)TpHv}s%Y&u!E9O7B@=|Ng}0xi%W@#$Fe1$2y);kZ0YJF}LRLGSer@*B9u9r|XpK
z_xeU1{5PH7WBQf7SJc;Qr&NaLYZR|5ssAmvzQJBSf4gVlcDJ2JM3_>oD*sk8JYzV&
z@y8XtB5&=qpdVlIzO6H2o1J3)Veh)!+v~jJ*MIflOw9bZtHv@J+&ptwRd(98=B4v4
z?TQo9Ug0Y1_iPfG65sB4D)O-D)uzjJQGv<RJtql0t67r$?&lYupRcu|zq=WE7+rm_
z^i65(+PrJ2!l`>-aAm|jeRcXxo~+3;{k32G(>;FuIl(=z=Xva|rJMMF#vIIzI<?**
zPV&D`rrTzpcR#sOIf8y2vb?KkviD5tyw5qOVwSH_e^U8e_X+0$zVhH@2PM|XOjg&f
zc$D*WP4pFEyUXShOKgk}J&9kwd(-u^`pflR9&-O2tN+g|bo#fL!umL~RN2etjmwo7
z7!Le3u$b$op33kc|LdEZ!NO`j7p8foPC1&hZ~1q&o*9dKZt7`&n-lo5+4$bKU2p%$
zx%Nsf{n5QWW*v{?r}v-xjKt1QeREwW{f}zw-?TOB@@>>4i?l!Acz9~U$8Xl`Z=Y{}
zy`}omSDyEN0neUKwln|sUHOylubfTQ-{yE-_{#b^B{R(XUj5B!j~~g!TF>@wJjy)z
z`Ry0BU2jZ-`hw<8T^(-fe|rAqI^XNh&P@&9n#XnGSya~Uys4TBns<-p{quUhJZDPy
zQ@c}FH~M@^-ZKB%O~2a7{{4mGQEqR-mYj4K*N=VF!pZ<@C;pG+FqyVWYyOV8U3aB*
zBR`#3{$sOIWQ~fd#>WGEs&4mo91hj-Sf*6@G<DjX!a%P()qnPVyC3nzb><(H-xE1*
z>e;oNUZm)8PxZ6c_IHb`XDaG>aem)<AG9vHV7(8ht#kI@XMfM~w~_uDihqvW-Ckt5
zIOVtPtNZT{zqiiY9s8lN$W8O(^A+`RYzzz^CKUREdqM(Cpf%MDAXC6e3)HwHueFG$
zNk~wtp+0313xgbszW()mFHv*X6FE=5FFhW(=kCHYGmQeFD>vS6&A#q;ykCC$-8_Hc
z<HEwii)F3LH1hB5IW=8Be%8-<pd%SiaxokbJ_lI?#4rgouGb}|y9zWWabkj^2nz$l
z63H(&!7W%4dwY1ipeR6oprHgcD3MUwD}}fd#aP0Td%!ulz-q4FDTW80*H#9r=ib{B
z`SQ}z&pO^79v6Ooel98?u%MxV;ryo?HzHVBSzpf1-xoN~rcy%=TxmFD2@41;c=6&z
zb=CQ4y3rvEomi22ms_*1zgu_RRD$Q!G~L;tZWO$O8MZd+?$=#@%Y)ZOZJjh-e7Zja
z!vw2~vLXyBPft#E2c779H%6%Z-JO*|OT7Y37#JpKeGE2Z@W{Hm>#IHJK92?I^D5h5
zow{3heWwR3zr6Cog@7(r28JNhUCm4l(G!(ix77dtXT6O>&Z5A9m0N5{1OtP@)kUfT
z40qmLQN5C2u)?KN2sBjXz`@e6MTF@;U#ue|gF}>Kc>u!>*PTUAw<I6$6NzR~R#q03
zk@1-b>KD3sD=^$qTB^5@t=Un3?-wD^(6Rs%qr%$Ei;Irt3Lm^IDgv(f6neS1xPq>(
z3Z1NLJktj>fPWNHb9r?CvvTk`JIhp5Lc${vlIt!lI@T+F`uOqZ`Ob<^5rO*p`m3wM
z&wt*Nc(_f3m4RW2B&cKMSY$O<$}DHbv-9)y+Ye97>VN#}>+4A>^78Ui^<u3;>(yDg
z#jb!>p`4j%T-?OO!0?Qd<--FNH8s!x3}{^W(o*l`4Gj$;vT19>R@c<lrW!51cmK4I
zkWk6fQ=*`w-lO(bZT<B0^zlwE1_s3m0vr$S?=F1Y1{&?#ntgp$#m7glKK_4mx3VHQ
z_20jb<&&0WDok<OSdut#Tfou=)|)AUj*N~67#&?5k2qdDn0Rr1S6$)#QWjR($UBVY
zE~&cR8Xap+DP3T?bSNNd*99?+Mibx3fBt!RM4K@RGQPiFde-#q$w+&d^H;6U7ted%
z$gZ^~T)lp-s`s=HFPG0Rx|urNu>N1ox_!U0VAnr9dGqwilY+;+=AT}#-+yVLGdmlf
zOvRm;>QA7-FxkoK{%RH$GiDm6U)q`-4%c%g_x`@Q>tc6rs{Wprx9?{f@1i2c-*30y
zpJ9;5<hOh>==}TL?{;~E4pILpeJT8_Qqi@rwNb9K&2n#SPUkPbU%Q>@0qZ))X0{$V
z+gTSb1hDZ+9r4!Ry99Iy_#fVtssS$NPQAIgdFR`0w?9<a*nBv^98-EVRO^&$(*J*d
zd!)_#&YbZ9-BSPK<KwHfpj+rcXH-2=^PhKyncv3Y>8YubhDj}MbM|mp%J8ZA%s8+u
z_x6jsyUpj<{nAuaRD|7L;Nc2709}922PQ#5!5%5ou34tpt2D${Ub++nTEGH2qItGi
z?hNB}zN@Rl!$C)=i!iym)HFWa_xs)B!~FJ7KAqN2e{y1COIzD2198js*H4Sb*Bo5r
z+P&jZmv+RC0!7fp98hgf-oN?q@bH({*ZZG5NdevIu_5vBh6o+4Gpe8~R%$*T6?buU
zoj7OCnsOUg(CLb<Pu^$U-&b4m{@&d9x}UAva&H%XzgwPcG&5v|_k~+qvn${4{k|b;
zE$Eoyl2=zeK_Lw~&|?4OFJDSP8~e}AwSIYVF?;Q=FM)x2h0)vdp1$3Ff7zBTQ$T0)
z|NHxUHRyhX=@Px1dP?eFKA*QQK5Ke?L-ut&(3}Rt5A(vOQ^Sp_zGP^HuR9}Ke&?Wc
z{+@-P$cah1BBY^o+|$o*S^oWfjRzBEnPx9r^yjRIh=@Vn9gA=8?w(!|xVYkRuepH)
z4=kKxvh4nRaDKP<dz_P#lZ0Us3+NJ-Q}?etJlrm6UuR=o_U6I1+j);cQ%4OBkb_W)
zG%FvridTF(sm{sI|9oM)oRXH7m+Q~j#XDns=Gn|FeSPiXLTC20v$I^Ut_p?Sv9fc<
z&bxVXH6I++{N{MPy|q=+G)p8gF;T0B;lam>A73u}|NQs+edd)Fg4|*{3TkR@t%BfF
zVJCJ^Jp6D${r`VwW|?+tuirC?UA{(O_0?5}P82>nJNx*VnZ}?~^g((I4Fh2T-R<e<
z$Y@*jCE@F<t6Q?JcGds;JfDS?b)`X3kaqaGCx`j%mv~LpIy299_P4jU!GR4rmc#JJ
z{r~^mudj>!`F8vL9UqTLzr4PFzFYki&;lf}dzH^+_r&Rg66y9kMcuGF3&c0|N}FF=
z8*N@xT)gw|x7#o8?A+X?l%Dea-Ca;laBk-_tou^|@-{4c>bcyn{Vuz=`unmSJ7!eB
z-#cAQH)@4OQP{V)w~yc0n0)f=*{2uX<)2Osj|<G+1U^%^BgSKy|NL`z%kTRZ6d3T^
z{ZI&xDQxXhsz0&NxxM1`+U+-X7OOL}@jPhew>ywywi<LN2Unxgr|{~xTd!BV*?4?K
z*jlN@ZoP}b)<$V{%@OunKDkp^9bBU9EKW~5H^+0TR%i(HDx9C%6*h8iy;4Vie0<#5
z*Y|8;yWFCxudlRv=6wF0^Xke<o#<^nOO~iinmoC)rw4p<0qA~>nj)*YJTev&K$lf4
zaAa=dm-k!hH8n)y=Vef|MQ%<D+@2Q;x+}@a34Vdj^TUnI>@Tja76+~7tNi@zPWAh}
zRpH*}y5;NtO!T+=*|K4S!LxI7Pn%wkSqw_u8Z2EW=4%=oPhJ<hd(x~~r@mf~_y7Oz
zFX&RFo;l|G|NYViop$cFcp~UNhPb^|@JmM0Z){5CHosS)92pt;<?Ze1@9yqit?_9(
zDAe-zeiduymw$G*{QlE@zuyH1LN4oi5)-v8N7DR$jd9uTxz+D>y5IfqM1T71*{5%8
zOuo52Uq1O*&%>Y3=ZC|U&oE48V`k$ykYQq!bHhMLSU6N<r-F#6sA1k6i?a9k&f0#z
zb9jm8WDYK_AXt%;v-j(@=pGr%NgFm8xb?|+f-VFT*||VOT>SI>|No|gZmro{T@E@v
z8Z>(gJuvsg%nwz2ZGOF2{O7s-e?=psN$qk~9a5%QD-4Ppi+10w`SGxQP0Y?o&(6*^
zuK!n~xBHC|=o$~uA%TAGo}Qe|?EKFz``fGP>GcUJyCrPC86vUs!Pz;MlPf<z6BHLe
zeyEkZv%4D}3L#xxT`w*#=l7nbqo}Fb`TPC;`41i>h`Q-X-LL;|dwIG4a<^WoGxP1|
zKRY`co{N-j-n@CI;&E@ztu38=vQ`EK4;;=iYKSlW`ue(Y<)@UUrY6vxIxnuRoej%F
zPjhc?>jg#X#EFjY@9lkgbMx{pCGo9WwwQo!(dg>#o@tcI)y&Qx1`TeOgr}#betB`R
zxtX2cDC2^{t?l{i7oAX5w6>nTG5L5<K)`{|=k3qWFiZyB`XSirFmeC($jxbmzu#_u
z{I#m`K_h#{r6rwRO4Fw>v-5$=#Wyz(|NVad{O$bxy(d#(5%jN4&2P?$4T*<0+|22b
zFl?&&`f8;{k!O*GjL!^%#@yT63O}DU|9mrje&W_BXc;$gf4^b!u@8^B^@Ril4}z{_
zb8%_uQkuR6RA<%yuf4V=^04Xkn8SB=7O#xkyNgR1d?IGi%m0cs@ArQHbkSWtaDAMu
zTaSceM#c&*<$CV8_0#qD|C!XvEk0?&gae0KxmQMSUkB5cx#{*>P?g2XEe5*P>)+S)
z^@$rJLL`3Pj@*(VIKTd%<-~~-Rn*ldPniO)x;0q3Vxm5M`t;)7UhCJ_)-I0SUB)A2
z;?bOZ;{Gk9I_vXU^O*X-U!m71omi@<tJ|v=y9-qPRDFBn`TpMCP!YkJ&&m7#e!D$o
z%9IZm-Q^3P&n;gCs<OBmmBhEtum3mm-QC^65fLXE7@2px+x0p`WMYw#v9Y9ip3Lv>
z@1I`{j|UZTq0s2m^8;O)6TiQ1%Cu=lB`*SIndMd;oHP6M%Ok~Sj5Z%_`nbic?BN%g
z&08<_9L=$L{JG#3_mBJAOqw3e77`b$c{SCx>{8IB4ZGJKQBxDh(%V0u`OV#_ZVOHO
zRzF@HyW#th&R6gB6~wxCK6<)q&W^s=pDLx_ZhXG6^SpP?-dW54nSZ%)^2W^8!?Hd;
zf2(fvh`YsdadS&rm+4f0f0z2>!^0kFbH2qFSG0G4OX3qpAOHP+KmEmpg%NARK!sdQ
z%^%j1GQYZie=7Igda!E4mskJz3$ugH-oN=x;C^pi=h1szFP9X~uQy&0y7sbN^WWR`
zPp0xeW3Ag?`Q!4I&Xpyna=VPfKYnS+XUwpDAXsI-?o?lDab^C$R|l-VR{x!?n42BD
zp!{g(Yrcf6E3=ogZ(H`q!0~4Gz9(nbUk{B~AGv?xoohC$bS_(R-PrWaecAc=Uc+7N
z{=FIJSEk*!-t?M%-rOAw%`e%G?YSa(n>A#vvU{IF-Jc53^1NSPUmw4+GT5|WQb)%L
z$xGoYl<FrwpI6<d=-d{tHp*14{?A8l9R{X|jKbgCd;Xl^xFvN;PW1Ku825wk|1O-j
zaj6qSPu|_>>hX17dzcg)w%$;A_xAeBo^6J#(^;EVnQ<p9yB2!g=*_&@mQvk1ZQSqc
z3M}V}zOvi&f2*k7{?aR(_mucBuykaVYOlBZ$-w(D_KNP|d3xduZ*!uIdXLYq?0xC(
zT(ES({mKv3H@2p7Gx+{C*|~s8!C|S_)QTSu+cWR&u{2IQv!M9-IT?!r2YwOo$$=BO
zr{8{?_Ts`qo4;Q!Us)GBTYX+d(~1=<xW6kV_AU#2zNz6=ss7{XZ`u7fwoZPl{X9HE
zNG|sEn_VKaoICTUxd<(vf9s5(=b!tB7fby6@Hi%XJr~2Z4F#`MR3;_tFp#fYx96+P
z&o^E&PY=gl;{D>Q^ZM7!iN#&Bs=aRXPhaq@{Nf(JpVNYj6Sqcv{xP}#lCIth6aO^%
z*>$IwoK<|KuIx~FooY6F=l_4dcmDhJI&xo)CFo)iW8+|caj4__4?kSsKi>}2fNJBF
z1~u;v9Xh1<<MTS1xWYxhmU5dV*hk7)ZfHJlJ|!gLS?TWo9$9vek4I&=s2B<+9=Cbm
zwBq6ira5_sH}^l;Xl!tW+wxhBZp_uE&kp2g?~6Hp^44x6^YyCIFD`HTSNMCyWC8yf
zXWgDoxXgXY-KaiUIj*NrcZbcFLv!aH$uKE;KDYeR_Wbyu;NZ-=yG*11=+1!Lc~|79
zp`o!L`FJ0w<&?c{XB+6Qv#nA3UB%xmyblO?ui5W<ZnODJ-$-}A9eKZYElucHdcPs=
zPu=fj-OQDiDXC8<OrGrQIPvk6Z1$N=2a?sblf@+^BcI4_n4iKRx<UL&z=IESE|yw#
zbDcPKN<mq>yGiM}=l9vg2foj|9On?`#wKaWy2CHZhMm<iG9@Kt!qnh~&3`m^az$Fb
z^$n}AZIEui+<G~3+wun|+`fF-{-E;i>4@S*ViD)<uARBpWVlP3Q}AH@m-tydy3&ju
zSyIQ37L=F0zc*LWxees3-DPhhw&l!}GgDGh+OO=PD)93&X#MrN*xj3wkMr#+eeHIC
zU#+18kN8DVkz}tNx9P{W&iHY&pW)13yVlk0+gAMOn^nxOEuJcL<zL>q6)#hcX@1K%
zf9uehkC#1`KDrTd?8j?~s@q><<f>{GSWem5dHz`K(PKroC4{YwSN-R8nQ<^xKOsGe
zOZu8#g7dD~v)gRXZY|ZC>-OvRasT^!j?PJDOU}IVRPR^Z{Enj=`+Z-VZ`4fX)O?wj
zwCm2fQb$dn`TH$5tXmoA@5Z0w{?qL8owwSOzYnhII&Nf=^1%6ThDc)4*Y^AgbK+ig
z$7dE-{9I|taL49P<gFtu3}NeDl`t1y5@*=(<@nhxz8T>UZTC7`Z{HxMzt-CH*W>r^
zcYfRWFeCiJkB;tV?{Y3L;|*OM_H=Ify@lS>^?YWV_169U_41&yl2VaS7N^M1$+<T+
zG=fsbmMtdlc0A_$_4Rf6L{<m+{EgFJ9C_CG{qvpZeM!-dNAu?IoE<)~Yr388>*TM~
z=9I*<m+L<N`7zkjH0I>pe!J7F*Y7)Z+W!~Zf{#mdpWdBr_qhJsV|KkblVrC2Zn2)n
z_1>O}QVkaS{OtMa!Urj*7v+1zS<UBtxleD2<MPY1FK+I?>EEAvZhhyU&li_he!VLG
zV|#tTG~I5y#8cm`8O2|27N0!nof_APs4te6C*Li3e}DeFwU7ROopHE-<<G{u&+_LT
zzrmk&am{ssd6hpy<0?KmuIn~xG@rUMS$$$2|Fc&IlXt9MdsMz|_s18;=RcmA>|azG
z;*nq<`e=6H-2bs|{<ma;Hg}y~9=>gD3{zX1i9P%K_p&GTUUNlRpD78vJHNBD6V$Qx
znyOWJJ9j&%dth4wX?C95f3#b?`0du~pf(JsMZ#--r{H5;&*9#SF_X>izGfGz-I?Vc
zS?=Snn7!{-gW*1PUS*#h{$@XWvgPmle)0W%A@hU%4x>Bb6+PQBs%#B!TyMQH`_z+4
z>HgPAlK21l&6Bn4kY4}(<b~&L>~6ZH`9IruCG%!|lRR`H@i4P;>b$-)q1=%VzZP6(
z-kT{9$7v9({gQ2$>rbbEYmao_Y<DR>y;}dG`9&YIgr4_j4xi^=<Qya<`|FNh`u|;)
zzyC6x-Pm4vd0KMMyMy+(t{u-}*}T?F_IJr=^;a_6-f11k*%q`nO__m5QSrtau{CDW
z)42SftuooYH}t{w{Q6r`E4r6wsK+ne!I5>nYlgjzcT;;${OMgyxAYE0Z}FRJW%Kcf
zFzBi^L1niOCzSgaL{5N)*zA?T%Rv_h*vRqo+y61pjoPx~#rl)XSN8tebzpbhQhQyc
zv-gwAGoG(s^P%d+osEWJ2Z|@{-`ib$Iig#fhc~mN@@kiS+p)_lr>>V?QGHmgsIO92
ze`99zrgzJyW$d!Kq5UDUDlPHB^UL+d?*wkw`LQP4tG}Q6>B-4||81R%w$(dGo#*=W
zCQn1$Q+T$DP@<l7)YVt}++SW?sm_<WbYhqI`FlI}NZ)v#GOsQ@FF}5F+4Jy<`4^*W
zK5Ymx-k2K6{ibc{64?)PZ_his?<Zq^Ufoam`Wf*{VjcTK7rXJ!t9Zou=+UDe=j;C+
zZeV2A(A2!h?CIjNL}stB#wYcPr&Gf#{(L;XGH$QcVz*u;HMM7#c3MQ+=H-NU^VgqZ
z$>+0QcYeXatrP$J`dcpHBby->|8>gj_vddFclq~T@qNRRE!JQ@)%K_R3faOO9$jIf
zxen3I6Gh|rZx?-kxJc}o@lLVlikmjsCUh*lKKJcPu@{Ut`}kyicHGS?aDHKU{O{h2
ze*DkU7zOUHQ@?JuLQdkrme}2VGYTHF8QipZTj+oG;+B_j^LsznGw1VlUw{AXlwaZ>
zqiXK97w2LGB|o+Qk7SN@Ts$kTB-XKix2r5uw<v5aK(3iw<r6_r(|h^+I<LGuz4-dS
zrVAG?tg`6r=+I#4niDP8-wsL!-)`p%3kx4!5x96o+}>ZRWmDaxw!CDp_hnCT|D>O@
zzSgq%l6U6a|5^WDw1{6@m3~3+V#Lf9ZE7AL8!Uh1=&4@Fl8Aio^(bfgE4NoRZ|}}(
zs;MpwS?B5G^tU+Z>#p$a>RJEtQ<x*WuYd5)@V(z)AS?Cx_+sCk>lUmP);qOs!Er7=
zzGny7<x@ai;>b;)?OU4@4mNRD`S}S7PW=4+`kanA!bguDy;Jpi?Vi8iZmX!MutYKG
zpJaEhV5*({>-HU+?4JpbrYHC;dOFqAzh3=b$fi%-{vr!&{-<{=O}lz|{*nh>uR_nS
z5xuaU<-qRAYiiYJ&N~$75H|1cnpacyT6RzNtqPnlwe_!0+J+5BFYk=itv_pSlzhy|
zz+_40A<GxLDz|8BOqd~1`0~zjH|1tF-bLl_?}2(`*VaaF&b@7ByT%1{biAs-&&Qw(
zgP)(9JJYgQO*d-G15kfIXsOrqGhcUlFMPD5P)ut3?v;nEzV6;?x%QxE)|GDlKYKK5
z-xto8*Zb1HA#;Ue>go5qABtwY35j2RYDS0F@mi@AtH17d_ME&wvHVU-eOcYCFKqAg
zZr4ZsHjug8eQC4#OjhaQ{^<7}$F$;Re^C8>TK4>h-II4pXI^-9tLyspS6Qzj{{QgU
z?7pe|*RCeF^2eXL_h`T91YNmyu$g`G%$c6E&2l4l7A=*7-Vc(Ve}10rmfYLZ=GXn|
zl+N4H`1}2S`OP<LIAa%fzxfhdQGQdqq=RSWRSSbAwKk7+zN>-_8AQdWbGxS=JS1~b
zVZ+_NpQmp=*05LTYMwH`W^%3h?xnmFpCvoT9+i8;ez~UkqQv6e7mYsXCEeUumO5>I
z%-)vf)_GltXJtG;m;L*0YO5{r{bJCkKf-5r%l!WF@^ko%$-&bVb+bY%Vy5k%_lb-D
z_pQHKY9@R-(SL77?DIeSb${p!rSjii^J<?*zbb$4oXfq#yS91Sj^NK5m@D2qc_K2m
z;QzA@wl97<+NXSVll?y{=GnO3_%dPQFK&j;=^o)1gfs8&vpw9-|NO*6Wl%@$%q&xH
zetpPTnep^lv%FqkTYGa?srL4JRodlu3Xi*5vOieKlu&nzEo{k-54-j!Up?kCr%|uR
zY*yLsbK>WsO4aPPyxA4oSGy*DpTdXgQ~9?~ZrsguJGT1!-F=n+_N@F<eoLrGM(%@!
zN2JkY%a==M%=-Fl*|R`*V=i~)i>p$zPg*4$e_AwYcde?vZzMxo8;|52JApVER?8>9
z)F<m}-H_*4cdp6j(&k$A_c!V<?7wqjs_ZT{H|wuUR6idx?*II3cD~WGng0#ao4fs*
z^Pd*2X6(1^-Xe1U$h+w8&!&`qE)kpfo_~6^`TJ!DSdVRBVJlxFy5nD~eE-vr7av!A
z-e}nKH$klJ#knuJ;o=Qirb<kTCHLmMpYivecTfDc{g0O&|FH4WlG^ZkfxV4xhRd(t
zdb8>IgopO!{tM?eKfSysTVB)t$F3%|qwVpR{_i!*ywu_@UkkbddA?ok44cYLa!E=`
zpSGub$P!YWxVh~2+wGr^ipLk+ul;^;SLti-*N<YVf99O2Fe^HveCWc2hv)e|mH*XA
z`S#*^XXT5?b$uUr+cw#k%B`>Y^O&>z=8I?b#v6L|Wp;Lb+mQ2*Ni5%E^#;G+cFr&E
ztg?GrJ?+`@wU2kdPiIqWwAb1foByjM(|5;F@rdAOK2IjkH`127a-%8QN-p8qB-!J0
z&#zrP@9C=(0x_&F&&>NJm%aIP+$Cd|?I{lT^-Q`}HW{be`uF~RG~@5KlpFuOAGKd!
zHX&-C+p%djf9~kI%$WNo!f^igwDs#YL~edperav3uv32>>+>6@&!ivwz3eQ5&)F*D
zUGc9so$oL1cydvH;?;5s=UM;nDX6#qcsV(I#$)~lQ9t{A(*1X<Wff<xuBbY+UBbuQ
z;Mvl1({sENDtGSOY4iWj=PA>s75(^-*wkd=HVa&ApU8Z5bF=!%lP7&<8o3^|xV`3O
z$%BGj49e3Ut9jYq+?BBH#)hbKsp)L|e7j|*vm4CXcyrD3C97MGzdTU)WzJf&HEEAK
z!xo)*$Dq^F_U~VUg<iGb(rE!}o)s6?xjuPs-?TEw^pLY|{p{a)mzH?0iQLTAe%R5&
zqvM~mh~UKiqP2fDmFhJ&AMLuCWc&Q}JWJ)UXpQGp@7=nmUbt_azPR;*F-JmLw8pdI
zvbu#XE?NI<1O+GV7v3uj8qxp{(SR@Zz<t99{K}7dQC~qtrTQ75v5QIPd)$oEKF5Eo
zoy>l`(Pmrz`GsYhRSx|v*1A!A$@8>j%hyd+Dn~+W+uJu!o;IoNZmz1CVPn8&)*t*&
zH%{JY%-wE1W5%zq8(wBRfIEnS6OEszJQhFyIkfBblhqPRo<6&p)Xq-!=;?oS{;a#+
zHr?NC{Lj~ambae9K6w&{ebu`8`_IqGF_*ag=|%Tm4tsaU6&3H(mihb^Yq-WNx3K)N
zl=z+fS0>)I|9ox=D<lG|c9lXVH(buOwThMRby&Pdx8~C^ld?yw&N|!wy_S!Wdz141
zok>{T-i|4}mP>LXx(xS8_y7HDb?D!gFb&~}oVk)BDbF6?T-`W<>+e0?tGyH69!pr!
z^pt;2^XZpIax`Z@xGNI#tvGb{Q>H)VpEhKl7Gs$8Ewjnjd>wdjrQ?KrV($w7%|{f<
zmhCto;1v?$d+4lyTx9z(-C3V6Zwa3{IjBorcjo%KRfi{JC;wk!Y$zlq=`5NYoA9-;
z@$0MGa!bqp2|BHEpHnJO^mOg>lji)NzIe-S{55^1@}wX8+ZS89a0Cj5>g}mlxNc|m
zcM5Bp$1IZ-3#G$cT%P3n<hOL3*!f)dVDvv#$7GG%yjSu+OlyC~$<&uUE8Ok>=fdU|
zAxUA+8DDke>*v*~^v`Wv7<a;uKR|h*;KU;?EwK~$uPM*mzfn;C*bcE6xvWdJT1rZv
z%$H4b5VX|qxvHnk$t?Zx(ao8s9mONgKCqZ@*m0GhWZDu1_5Q=}1hS5`2)_t;Sr{-O
z{KP{c^JNcCUFqPk_cZrYXSMt(qgN;#v(w7#yyl`Ar~UjKH3cUgXE*EL+_C2m^ArJ}
zr>D%E4(Fu4H`v7$`RnHFH6N-B9vyr5WiNZ?#wmNBX6T<d7^&IxLAb?d`eO#x*7k!b
zOp9l(x_W5iya%kF)sB2+!KO-wbu8YpJ`tBYCL1$z#fcBXd;DKMa)@JWkIg%y7`j3#
zi}6f^7SFNAipi(l7$(g+#u}v$x`_ZZrFTAAUqyVqzCc{u+{4y&iRbjSe`u!af0V5b
z{(m55lKBs((8qSlk{7hPVBOK*hS2V4;7Xl;PjjDsHn&r|y8r)V6??8tWgAofvOW91
zMJ~C7f1cFEeK86}6E@}W&6kl4SoY_m+qcDmOFFO0?{h4wTB#ql^z6f@%bzh^NIt=#
z#J7B7;#(%OCA)WK-nbkSy*%h=clzZ?l|J!X{_WVDbw6JtcJ}e4qBqHxfBEKkMLK!y
znmt2rdvR<4*X8H=d~33^u1vnjd|R-}<d&?*^X}=(Hwfw%b`~yKzJxj8|FU|+C%1Mz
z)vJ}4?ki*M?Z2>B15z9my_3{3X|cTV;qLO3-}T+A3mYuu^%bN$m@d`*Xq(&F=`zvD
zEZ~KXV*O{EV}8FUrtNLJ@OsuS7TKGJ%ir%ked$fevc9=TYLnfB(v2US`8{>{y&obx
z8+mFn6nd4k9%|ik{`CC)CZ)Y=yjK{f8wacFDBqs_*n0n~XGLop%EIbDaou0CBjawm
zhHcc*q;DcdzdNr-KJ-m}6@CBXkJEd1RDF<GrLzBwy7nB2$GboNXNz%G{hny1KgaXa
zypm0uYZXmaIOU&q)|vZ<>*UL&Ca;WUM6ch^#4TrJu4epjvwWmovXG^FTakaaz@>D3
zHCFvVhii+@&&<p4`2O#2;`8%o;wQINpO&ff+1n(?o6W#?t?nnIH1F3P?>iNbw9fh<
zvefwNQOzAbqPwf*AYERcSVvbEmw)fR_$}!8!`Y>I=JfIx4vdnwuN!_o`}{|ZZ|D1k
z!uKaf=f6CAV!HbUfo6RPsievm>vl{0t5=pVNf6j>{>32wR#DGI+aK<?e?{~rY`eNF
zd4Ar$yVb=vcCBXEu=&+xeTEk$*Vi^)b1crt&3kcY=OM{RrP+p`rUv%BxqM`M_oGc=
zhyV26|LEJ*J>y$>Ve#sS`03usrL(^Fe3#ds?d)eTdu_=2n<vDToSimA>PF;V&8$4V
zw@1coPMbvEYL|D%R8^KwPwIGg=Ir(Ohwq<dEeQ)UXDeUBdZ4dz@{O7mY}@zz5~>wG
zo5FnT$2prne-x}#R4*<(|F)#>Q|tCQe~<nPnSK6SiMc@2$@^a~OGvp-xBL2Y{^mcw
zC&(Gq>AvP#6$lC*7u`msqIZ?$Mx}pVi;77;p0s0Qs&<sx(O<g~q*e=xrz@H6o#M7P
z?~zf%g$yaPzGnfq`1h@wTmAggyC-X9-bT!vH&5=y{Z)5=maMF7d6Hu;A$7@MUDVOX
zt#fqV$@~tRJ6G5HI@5(cnR~hJ_gu}oH~U{s>#UDQWQ<=s{}iYU`B3OR=S|daV}?%M
zbsQy+zsy(UjVfu8U&gdSQ@4M$k{)->tQ#s5If`%2Tm5VKwhc08XFd~%U+{UEQQ@cS
zcP{Ve=2coHQ2O-ioic?rZwy|+F3TvgyJPr@^O2UM25XZ}SkAQM!_x)5J07;&mOFEn
zNnu~C|GM0Oxr>==w`E^v6wbTDpeDUja`qp_&3aeQm1Wtb3+~=*n~>r0R%)tNUD`^&
zpxfKh<5|KPcVD`z{`ZDN-0X9KYGUcTZ_b+6x!E_8<IAqQ%kOPBoTDb29P!&(?cUz(
z%KyDvkCeahp5LdN5wkpJt3&vO70cj``=oBDRv1|kn)7V)?K>h*vV)5uUBAiertdu_
zKcjETBg@ytU$;M~4A!)XaC+PEpg5{)QgEG=P{za0V)I{`O02AZEby+xH~p8tHG{|R
zhuU$!A8O}Geq+&n9kSFZ{K7R(h*j?Y{|E|N9`Bp0{y`@-|D4s!nkLrBNSTlFhiA_{
z=h*)J4c{Y8?kwx?Vsk5Gm&>-lep6O&5|rnA{zzrl*QCdb&KJC5<M&(Ms%G2%q5d%2
zq>y*DM`r&2+i<_@PHL=U;}MopR#8afVCR9#Z)%$plHV1c?oV0sXe(!YlKrgeJ=UIQ
zS4ceQY1_1}U+(-z#)}18v{zje$xw{`d(>!CpKsPYr!@P&N~<^Z&#Cw=%whCXe(kf<
zI~J{6Z+T_zC+(ZZS+6F23a-6t=j^OKKg^eD`qMLy?xruB^t|zcRqu=cuRim<y?ige
zU*;3PrRAi(m#>SO?2?|oT6+cLWd9SEeK|Y&`KBdt9Q(4{fBGhd6pJ*k=2@J*%=YX5
z<>=gB{_NLouU#kZ?wgc;|L3${cYhazE>>Ce=XLG>hufEkPu=StzD57Od`HmDt>@#u
zUHB}2YVNF~b>a!@JTL2Q{WE>TG*034Nz=+X_#aqoXn*qYf8x@O!e{?}{<qWH`_Jv?
ziR>S4oII(?bZT8<_%p}<{%Mb*OF3DtK7DQ6$GLf>&ey5~GhEtNXN6U$$VX3J%*Aib
zBp!5j^VBVG9v#@26kWPBf6;#ab+?_}Ug<NatOEOJ%Hnb+*C#Sh?D>C}=RcGF@A9g`
zY|^gBR)^Mi9IBaQ@Z!Yn!)5jJFE;J9ykQ%7<IiiU!$J~wHj2sro4|SCQ2+5w7X&<~
zn78K~5V$*Y-`iJBf>ovYXO{e&eQ@@vC)1Bz-dJ9}>gMI2yY3u$^Z8z7BWrKf=Pw7`
z-kd%0v3*m`H*ry6NjayS-F7OvQs>?FRD55TdP+L`wBHqhlXv?|IC|bhE^XT%y)>Qm
z^Ro~oo!`}A;d|#^l-={>m-=dbtsDK5XGJt<eZ6J0<8+x_%$8=|&Hqn-zv@;Ldst&m
zW%SIC=X4nIDi^R%O#5(SvxoP`s|By`$;i0;+|^;#eM0}s)vWi=?uu+#@4sWe_M0`^
zp9}u2@Ymz#J(hf4QF*Bd4{ztXwDi0D`_?V+{P=K}yQ_=Kk~in2*I$1fD(Q00@BBPl
zPd~r3Q&TkSA8YNn@#Sl=PkeEcVgL`X<i9ry9xn?&asQ9*djIIsJFjBjOZqNvSsZL@
zadTsm-TSj~v59lj?!I3V{adX?>U7`!<fC`GUP~suym9RWzx`6dxi_Y5TRW+P-}Z!i
zz8|+wg#3+o@MIWQe=R?^eeLw6Pu)J79NU_DVr^^(H@D{WxbH@vXMSAsmEA7*a@?17
zTl&_ArG(zuxKVZSyxpxHo8QYAo9U_l-0b@1NOa-<?IntOHZ1Aqs^&?3KU(-(;@kg8
zKOSz?+^1=LO1J*U%-1)+yvxoxC_H1yz3-7i<^8*6R;lj)^ra==a7JFj8SlAQg(`YH
zC#!8tJ1Zr_#|~Q31j!K{P7hiSJuXQo|K*ly7@Zs8e{IW)qNH*wO<h*+rvYoy)XS`I
z<fNOGw_Hv<{_<0TNtEBdwZ}at-erv7;I*CnFeb+04PT6$-`@Ymy$}7^{g!j{yRBJh
zcBbKYq|3j5$}_Y=m2IvplJ-;#aB)#_xa;Wpq$zDn1n1T{ZWoNAH_SZUF3Y`bbH=|-
z5xIZfJ?s&4I<FNmbK|Wu=@wisUdHSa{cNb7yQM`)P;!;Dhl`6#-~V7YRRKW|2;=}w
zut5qM5ML0dz{LF>%Q^%m+BdQ<ug}~6SfY1^Z(7gu*V-EWlbI`LE=iTzv8qz%nfJD5
zdY{VPww+Gu-0<?ZtsrFV;<E(9cHN40{`wO(Pon}0Jx%tl;tW?;75bKbCiR_W%FF4O
zJ?>3-dV1bPTaP^|&I{U`Vpr<SHJ!Wlrd*!XHxt2aUb}dq4byYt%Hel@<j>3gC_R7w
zH^!9}2L(>*I{y+3*PXHG&)%Fk<7QD!>63o<Pb|t$`uY6DK@$#cyW`IjpLTt-<>dU=
zmw#=E<I)A*>;CUnPUnvAoBeF{$BXT|Jj)k!baec=7oV-9RL|sHVJoHfYk#_tOVH{(
z-9v|!S4^MavVQlx^o;eZYv)&%f4Vs5tHrsWm(O~o_}IR>y0Yq}-GQ^Anbv;{<#v8o
zyITO6PgANl^}pGFPiW7)uKsy9roGMRU}PviRrYCa{iVnq{uOT1Jua;Ct?@B+$_=}l
zr=+A*bnA+2h|51EwYu%hD~>xl#woUY-<i>*9jvS*bDDGCuA-a#0R<ln@+QyJ3SFIK
z*mN_=@?G>gg#(W*9Lf>}^UOom$+_-$%aYGoHAm*eegD8hPPJfzsjI`3&y@Y??*7&!
z^JZCr(qSKwr%wwP-0&^%;JBJK=ZB5|_o{OiFRDt-J0ab)m<KVgJ9ED5Kb>X}=e9I?
zQ^dP(Q+xg<gdJ^dYvZX|A>XUi$5-<x>3COuMgBJ{p1_SJPwJPXR38vH-F%*JY1z&@
zX&j~#cK&r&t20=CNN@MO&m41wvRLZQ-`_VWQ~r#SbVql$lC7<D!?(Zgmy|WRrtfW-
zn|b@@rZT<k%XeqYDqAM(9l2-Wos1R7RNhtp347$^n_giH>Y&%#m9mRYw4bjyV}Zko
z&jC|5PhB7P+vG#x;@Pu>;@)LW*<UG_Y(K+wbDVR<v!~&e4^Oi1)H}s;57ZWS*>zW3
zuG_f&;oh6CJKkHhguC%QT{ltRXiIpq!jm2Cv#bA_`}|3eJ8ShV?Lx!s&adUJu|F;v
zefsm)C6;w|#oy-06zQa&&;Rc|Qv1F4$<Dy-j2r%ZKL2y=iL(=HU+P3#wQqg7uWs2p
z-tNdf8|`ANzAnn0WB;Yn%;@LSN|B0lZ>n4%lNnDgJ-B&dk^HocIS0*ss@`zM$$Sr7
zAi5!Ve|=%}>J^(xWvgeH=UN_nA@^r-ZtWU3gWFk`H~O33e|`@%!SbPX_1%~IdL%_;
zDwZ7G_Qpr&w`JZmCubG&`~O+C-@P%bt@pM;!HjwX(eqKCzD@tZH23j_`lkYG#Nroh
zb~!Skg>jGN6ZxW`t#RJ(r0tA8*d>>|-e>dkl)}3F+tcE{+j`z#_HWDU$b+kLf1iH=
zcJDcMXlM($Uvqo5{Ecq)j}oaf!LjQf+%SL3m%8-lGMgLL)63QKpJuq9{PN|?0jt$z
z&oA#S<aKh&Wq32=d;C5_-esRn!xN0--KO8(9q@O?yQRyHWM>4f&sh>s?2+-{`JzWb
z3Ns#T(72`QR=l7pSn02N+Ij6IFK)ap+|B&nxXu0gA+a9^yt?JeHzpnaox*JW{YkNg
z#pf;S-S3}*rBoxu4c}w`e*E)EH936^^MaC>Y@2SY73@44ka)Fu=f0D7AK(7^<QCi2
z!r$)SbFQyrR=cx<(Z}qL!`EHBok^lM54^f5zp2X4IZw7oI9%P-=AFo%qxZNLH_uw_
zI;lnRf)Z!KmRP^@yWJQjOyAb08k`pL<B!Xpyz-;0+iq>?<9;;zOVB3cqLZ8F{`;!I
z(gn>sQhQWItQVGjp1spH*LZ%u2IF}*wSD!Hi{0-yMC=Up{kLvz-zig1j}`T={<fJt
zXbfcjnQ;d+0+X*5@8_ACw)tMqzO|+cm*2TjduZvorV_DpHS<MZ=0$y%JZ0ROwy*Zz
z-21%EsXa5CY7Sl#5OTRNGp4_zqvJ&70{wZePi*XRbWHZ9TIl|Lf4TCz-m|ayX0reP
z$xYa6`Fe@)3GtNF#P1EqUmlUU*|AW+=h->Ev!@>2{gwYC<KZ^rdEy@ej%m%S$~Dq1
zi#v9peg4w&)e&pNgjLfk&u-ip*qFC?c`-N>g4R*KG>ms!@6OIrryFZKcm4Ggx$U#k
z^NqO8p2=<h0~_^^-BA8j#O{Zx_?bkLo&U_@D*iCua{I|va`N1X<$wNsZWB0cSZyTx
zd3X1gsh?}-t87zNR64T%R_2-X2uH)8EBA#RJ8|JudI{XhOaqo=t?ye5+sz+MbiB@9
z7yth&OWmDPj)qyBz09Y>>&>RU4=(<zTcuaDef^W0<@=V_eYd-^twHmM*RQVoOo``Z
z$}G+AUi{v&{K3trh=p6v25-^7aMbuj3X6?}cg_B4F8-pDITse4@hM`Cb*%qCu`=Vj
z+O#U$gq+i+ou=nrRGpi4GDIw+*{5nvcSpyGl`rQy2>ukVu$Zv1RaEoeGu`uUb-tPY
zTh%T`^z54uwD=(}&$}IS`RhMyWjZ(e2b1G>JE!g&;F0Z!eIFig)G=`pdU{q-+Q6lJ
zS^n-_3_CavZZ6mReErCU#i2TBHYsT-3<A>wq_zFZZr3JPSJ~b^dG5W-j{U9c4Yubj
zx%9V{U2&7uv<I8=R;~BD=iWWR=GqkLizVNB`e#lRKD0qSpF`)iQ8tr(+uzw~g{GQk
z?;dd%e$G^Ix>>fiSiWuUzpry-#jDFxWr}qbcd@#<xI9_H6SqEmvq<+G(D5jf=FB-W
zMKd_0he6%G{*z7o(m4z27aMj4p8u+DWKet9ahhLU?~^Z=4rnb)+y41YF7MC#T|f7j
zSC_V)on5+ra^Bn70<ZGl+}d9&xV~lv)4e&#*LHWu$!2YQbT&M4PHZ8=jM?vGJtJQ2
z_KAPEX2YJs`Sr!er}{11XK~bgDRV+c-pL35t_Sa`)0+RoIcD*@=AC<eHR5|E>kl1F
z;<>0Rd1b$JxLe4ofXTAU3vJV1U064Dk3r!zPW#<2@2T%F+qWw3_~aQTnL^pu)*O6T
zARx5pf{qesNOzGyh2sYc*&B2B8Dws|wPBY3v}rLGZ~S89_}#jBXFu&P=`2aqFKW2h
zbT)0}hBL`q1hO_aWb4oW>71~+PWk0^9iH8HS>24yW`A8W*UkTb6=<!c%aikq-z`uo
zI%pKNrX_04@{P+RSD)?qI!ESekM5Nksna!HUUwvV!<?Mgrq|k@+Y|ru(d~FnKHFZ!
z3@<N@m6sag%(tugNCrV_1($RB%_qMcKfLawOJWt5i|gEDw=7fgc^@B>%gE4&w4DSk
z<!$)bUH{GJ{x1GVrOV!6hx)S*Zap<$Lq)@&hUl664i^7->bN)4{`9{eIQ{=Joc-1F
zWLsFE)&ywdG{$R{>6=5``Xx61nu{;!hPZ%d2wk2qrI^hQWf2q<+?kOX!3b{uP24ZI
zm8I*1<+FoNOCDNYKHHO~BmQ?!rJqsI_6^zd=T4Y#DulCNQO}O+4f~DV+PzNMGU3O=
zFYAITX>j?VaW3xo<@~zRW4EvFf9cp36a8E1+`d+w+V{MZ%C9X94KmC&OL?eecSvx3
z>#DH(i?`LSuh)i`pu3_<U-~BHn1oGk?&Y>ibKND)v+;}Moz|!@i-;L#^}U}@nDq9b
zP|A*u^;ZsWtIVG?C+?K)Vz$dhm%QIjdOmaGP3NgMo?ZEKpRI`T^v_)`{oKoM_qeXt
zgO<;8^v&<uw`IQ${rLZxlKByi9gPB6$FC{h;xR0Jb#a4dZJOObw%>cjKOVIzj`$qR
zDKUA4s2q<&_rJxR``#&pW<^DQ{IdQ@mF@GFDKR-SzUlm4U)|(as`b)ML{M<z<BS9e
zLCfP_2X9PLYh>V%`P`P!YLl|3=b5bTpYt#GWvt(^!=O84&jVe}-DaD6XIL4wL}>*t
ze)w~?ol^a0B|HDZMB|qq^^}y9)WPG*%bgZYlKAH!voL*wZ^>+SiIM{?haax^P_ZfM
z?_bGSzc)+eY64Gn9L?gaX-?HYaWIcd-RO)<V4NI#*IkzOmVmWx7Sn{ED)cuu?Q!Cp
zzQd3;e3{#kkFPEHo&-kZEOJY+m=-*D$MM$7tml%#BNa+S9v|U$JDtf_@)$BWyl*Aj
z*{Q39<g#o}enf9fhLyGb-v0l@>&ta43hx)lhVwGd(48@r9ojnoXA_q6D&l+wcX5Wr
zx1Z-fo@tx3skJTQU0sSncpE#jklBea1_qhmo-U3d?U8mb?RCz6+bVYN(*ccDayL#+
z+gcns``xiCTa4#MoO;BYn#hwZ^xd{jyKrIZ+Jc?8!;QW*M~d?2Wt=$B{_CQ~q3flW
ziW9WY@A)IhU}*lg`{DI#wl6Ei-^ZT4Ec|Sm?uTVYKc6mVes2Hg=E5U$lw5CZ-eyqz
z$0S}m$r8R?OG+kU+WuPA%XTwd%>=~7U8}0h7~cNwU*aAla`fivil@z$jbEqN|FJsF
zn0&^*BJGFia=y*b#(h!U%*?9a!84lfr~aCyW8m`Xyy5qY2cF4$9*vpXSG#NDm$^H)
z$eYY!dC#{m!m9mCPW*lE?`=nQ%uJSeudm=+e&#c;!{eJLo=n&N44T0)cl!6NvS;o$
z<+D}13$L4g<+R$={{CL|^rbgf`p8;${yx^DaxQJbkvfHfFw5Pa3xd8cVboIDrF!1x
z@}x&=w(c=X)sv9=bpGPs71@Rlikz1QtE_L{J?--)m-}Yr_C?a+ZS4;>zE6F}EgUg%
z+m!P7Q;!=B%L{j&ZCITkyuRLmn`h&n&z*`U9HtM?M4k_Ryf5MB^ZAz*$}D)Z%%0CY
zuKeTEQ_Urho-V7FUl{uK)D9arR?wu?N{6{EE>D^`n|iHEKK*@h_TllbyR~X|r5|6P
zkmGq;dsQH$wK=Knhf@+}b2D$rJkd3`w#mz2YxQnF@NwHatq<(GU4CuLc8f{!e)nQd
z#Vdx?EeTtrR^)JV>a5CH+j2DPXxip^e;8liI$kbZyS8jD+y7?m!?m}ib1#0WN-_w1
zkR3W_yP)`kG8sE5>y1%0y0fQsNK7(lFmM0k>}N1*W7xK$EpH7@WF1c0n6z1osjH*I
z!!?UjWFmK)p-toM*=(PGwh5J;i9a_lY~z(}4C`X*BByxzTz}^x;~SP&uJEA+)Iz<{
zJyp#1UDDCE?<d*H-kp3NP_S(M>8JO0?y;`ac0Yc`$KRhnXKPlu-CgPH2Qqk;x}LwC
zQONfEt>xi~zL<-YLJclopYq_|hR!+lCaP*WU1gmYWmepq;!(eB^K;Rll2@BA@v1_1
z=qxlYPHm6d@KF21vQ0(BX4_5sA9*~>&pC2hwl`~Tf%f{}GbXy`Zq1YxKVtrJX-fK&
zN$IB%qq3{F!v|&a&iQ}u_#0X6C)&Dl_6^Q!It=+*)t~wUte+KC-7{rs*t4ZnXQoke
zZ$;^D$P(#}m_JU7q%Io2i+;9b`lh;P=Sy<$b<ff5Hgqdj)>Han@;&K@$fMU<Ssa{7
z8Yi~TU0mcVB=jo&gXxB_t+LzCL{2z&Xy*IPN_Tx<+c{UNT|Be>{QkuTpn2GIpZ6?X
zCtAfCcdYu{yHDO$J7VuMakD=mPM7(@ldQY<9zEv9x~n%W|J<R%Nng1(ZQ3yZz1$SY
zs+tdt^)A8AI;N|mET?SjS{^t1OxN7$h94fLzr3p@w)&_`XPy|O9-ZjzIJL@m$GHd9
zHh({S<uQ|bb<s%i5zn^(pSo4+&!?wkJUtL}{$7e$&KcXhx5|>MD{>D_e^Wkp{_E9&
z`gR{aFZ;6hcx~MdgZuJK*#S3$yl$QPxi4eJjlLF+{|Y8*!ppP7c`v1Wgq9BCrxqXk
zu-kv$sx8u&_Uc^Nva3yE^}{XwWymed6*Dhbe7O1Q{IO59S?lU=h=}jJ9scQ!_5Gmz
z)$;!pOj>pR)@V&uuRnAvw%zY_h1%qoBG<zX9dx>Xk>}slGlIM?#Gd|6J^ttEo5CsY
zFTGTFx;&M^1hlv%jsKj`{pa%zHOt7VZ_zNGe8(X?;J>3SOBDO&Z#Nk?JbzknV5MUc
zwEjOQ-rv4xO~|$7Q-0NbU$!ajub<4iVD`ycA3aOHu`gZc`PE>4(p}xnY9Erb)XJ83
zGfTd%s9%)N^gUYcjmHkbo<vTbcM)bQ&K&AL{^`ey!aYB4NF1v(;C3!a|F!M2FIW7W
z-E*JtMtk`CKfFDE+rvM1xl1;tyGK2^W%ho>jY}_dPVk&be*s!HVlRFDUak2?i377w
z)i2joJpPcezj=zP@yy9_T`ME=^RnbvB{s7BX4+8xJ}GYQAIbRoPh$G}KC9mRmwaa4
zN5lT!e=}@Dlny<JF8}(aQYX@E`t$n|$MdRJzH=576y&^cz(DEK^TM}JHyb|gm)?`|
z_4?$9zovJL{zOI{YnhYs_>{|0Hph2YV*<CIdM}rC@3`=c=l^dC+?}-hclqN}9g8PB
zO6+`e?6*WvDrnsDEq{(>3iH`r_dU0N^X2$juPP=Z<tH+MH%3yt^i(C&+{)LLuX^>h
zbbkMPys;*~<LSGL%^XqYbH9uI75ky{w%xzt75k3TE1P3vZ#Go_%a=I)Oh4gc+tHrt
zw8DhF-)h?~><q2_zwvC$)xUXWxgUI)-h19ZaqGpjJD)!&cwP5@I$7T6W7_*Af!`0h
zL>)5tl6&-$<mv;nPc8ac@cFNQL1@K>cdj5G8&vF#<JDC!ng-g^QB-6Uyv!%i@SN#+
zJG<&Xyx;DAVK}^gZS|>!n``cS{4(V}oLAm>vF23gEn7eKvgPle+HbzvcD3fuEIoI@
z_PK6OhjX4U-Q9Bf<$;GKjBYw+($j?HuC(qw!RW?qHhbwIqyCFdE7t0)vwG`l;dL;`
zQe<`H_Dh{UvrIZov#&iUu;Ae240Hs|!MlPc!5mNB-O|)sv2(ZQ+WWEFmmKN6n4xNN
z<=5`lQo3PIy36Cv=kThYsrt9>YR<-_ZGY}Dp1t98cKJpgooyF_w<SOuaVO%>g?^AW
zPS;zivc-(^Yg6RcuUbt}{3?oPzU;m&<)bDV08QI^YCCUF+pK6S>!F^?oY3)f-?>ip
z_t#z)1U&h_v!GtlZ_$-spz^dx%EW3#M~pyL+{I(&QECl}lcN8ZvKMX>_UGPcUn<zY
z-(u$6e~jsy>VC8=m#gm4zm~Fk0;sLK-x=&9gZB877sBHYy)0?GSv2WjU=6&T&0NY2
zn!f>U4ejm}?0)*is&v*_sc)c$|C2ZW9tnXP+k&m2x)jv@{<6Y)MaQ318$EHo{^v86
zy6H?4F8x|EaeKCTjF@-J+dFmjPd7f_SUi9FjVe?Bk7sAJGh5DlB46_0Sj$>B6~l+y
zCa-^M;gA+)$G2TNcth`rD8n0P((C2)mJ6RZEH#Qg^GnSi(ph-&Xk+<Xai1A`cP4G=
ze0MNd`Bt*7_&%<(rJG8wm0U}!-jR9ZvdayvxwAh!*GM>hOMa$Z{nr0GOgGLmnDaLA
z-SYRFl<xK_YE3)wW^M}eN7?r&2YP+`ZXP@QE?(^OeaZXG&}NiG($3^34(qzslpK5h
z@Kp5q8Rq*|?3k!HjeWkXSMJ$vMQBB~(SVheS>W<Ny|!;2N8Ws7IOgzr8f0A7ZB@M%
zN5g_*BZd`oeZBXk{#sPTxFzU9XY|#Woi(5TSg$$iy*TcFQo`+B4Rf!~-*d>^-sgF1
zkns609#1lq<KG>-`KSDtmVw&G2f^EeZ)ZL5I2JXpHeGH0K1<Ha%7-&V<Z9fWC-!+f
zdOCfwfs~^CSL?X{6`TT#!a%WMsr^gZ=?TBV?6W)W=3TmcIkd-q!LQV-8yVad3nuSe
zKIvl2)NP05A5W~#x4QnnsO4(b*~k8NUfXvB*NacvS!S615j>xq`})`PTN!`*cXu#(
zO|L(+s>gZX(Q~#sKXObN=DvI9m%BQD-eIrOD5E9GwI7}t)ZUVMlU^g|*5js}pt7aP
z^sYhug^9}U9^T%E6AXU5)`F}|SpM?<|9|FTtEXO%ub<m1ZN5q&{p9Pi`2{;w_n3V&
zyvngP^880Z-7B6Q>Sxjx7j-;*vU+Dxq;WMjLvNJ2P}IN87j|v&{j&AHZp73!?h|+S
zoV%Wt>9eRX=PK9kRkKS(8RqRR_B^j`x%sok@yA#HesX_vfB*dH@pUtsnwaG3ek3+E
zH;0M{g60P%UcSCD*?pQ`tk+bn(2C!0w@2<OS=seT-J<kE;ll!pzMu1jr=N~YoO!wU
z*>T--=M&cM*UrjWyXrXW$rU@a84B`mxXinJa^02q8vbiHPOmIGqhN6GYEnjr{)=3%
z)0gdE#VcrQcZ<i@OkA@@=U;vGcRA70xgQGu7#~=mu~12~>!G1X(B=bIt{C0E!lqy8
z?G+tlps%U<z*08*ZPBZbVpAJDg0kWqd=GOk(8ycX(G<08^`lu`8Bz+;0w=gE3~cHj
zJl^#(II4r={q^|bpN8qrjEn2i?aq8R%D-n*`{km03lqbN=`(~m{&?QqT^=44HOohB
za@e|<l}$}dpd*eGvV2zZxt$Moa{IY8n&(=riRyvSXz?R2&Ri9G{%N}5tT%t>@2uXw
z=))ubaQO|3qpa`!$X@c8IagJ7s#gE&7{7I!F0Y=~Jlp-XwD^q72MXrq*|O&~u7CXU
zS?lFhx3WEzrq4j90=u}htc%&X=*5eSpTB+yaWO2o_h`Dq0x6p>7o5Gky@RLg#ZH+p
zVL|HYX)7Q7mweRms%-weSwHNh{>S{z-d-YkH+Y?1|BIXZcX&*S<M=Uq+1)h8DJtGK
zr)E#@4KwGvzQxFF&%1=rTX)v+=IyHL4dh+N_tV70q~Oz9$NP1^*V_GjA{-eRscC39
zk%@z00gDi0fNj*aoRv?Wq|EeDTe@V4hLVy~YHBJ=u=Daw?Z5i27`LqozV-e7k{2&7
z-i_MyxoOuAmX*7+Creq*zW3_Qrc*&`->yzszyDtp=wR`Cdv=13TK)I;w=fq&L%yF2
zLx_BJb+xLBO2(NPhE_#SI-H!GLPA1Z3``ew&g1>GQY$jDT2y$m>$TXnY>(?<F2Vev
zVPRQ!ca?(9dbo4<?$l}1)b#cBn;01ygg8R%7hhZ<svY)X?e=>?X=%$qr@P;Y5o21(
z$vbCv#-#)2uCM$4>CK1b&+dkvsCmsEy+nK0+QWtIQ>IVPJ~>JC*0$W)pk?_>mo8ly
zySt2qnSnu7V8>?A5uvTz;#%hB;l<C+T)eqCopUMMma^Q!w?(Hur9A!Dx$WwPz{}aw
z&zeUrx{|bSmqhQu$g{Id!i=4-&e^}|)9kbCrmNq)GPn@7R_02>_E&uxnwo(T5i>4b
zy7cAG=kwXm&&|!cx5x4z1A_x6OO~61L&LT8@#{BjG6J2T`{Lr_mLm1YPg{-iu9sf_
z{_1q>@_!!=n`Ty)YGghwOno~2PC@;@pFOLm%Upe8=WhETa_fVlWBif39=_Pbl$sJU
z)%o~uS<9dI7+YIgXPM{E16@7QDXgCL_*n0z&B?rB3=A5L4!qKKH9N}Q+*sJgE3IW?
z6I1>Do$3_p%M7Ji&$Hg1+8MuopK0*oJ*%c={?yFu<a;vZ-$HkjZ3Vlx-8eIoyLEqF
z>8W!Tik_P)9~D|k_f}n4U4GTrvN()&zm#XxtogxPC+<=4=dYW&D(~jao1im1&ds-9
z|NVY_Jm|RkHEY*$vN14tDHN33{CqOm%hz{mcX#(Rz1USPEi6tud9T;c+`e(&&&=1K
z`hRat`CVBm_iphwU)d+7H~jhI?#`(_+>}4%yY+GhXXnGd@A8$-PYmtul<Btqa=-uM
zY00;~>$9qq=ao#Kx<rlNPR(CZ<#b|g{i+B1xw4Lx|6Ud6Jw5PKMppIg^UV97etCI$
zmUa0$E-o(6wKSlUvRjzIk^dWXkZtYnZ!3$RpS!iAaPgwWi&HmlusQMGSX=w^@$Bnd
zpSbgF9!;P8OUuY_Q&mv&{EXnz#d~MB^}bW|T(!m{%;%7z=zp!e;=;bVJ-ZI>R^1=b
zYJT(L;?U`JRmT#Rp6W`;{qZd1YLnhHr?*k!i*r2n&i^{S`smfaOuf2KZNEi&2Aw)p
zG562Xtd%pGE>~|YI?sCY!J2TN{(BqZAFj3GzjJ4ybNiQ%$K_8aZPc){i+g;mS6PIC
zp{{-5rcIN0S2sw7goG?fJlvM~{M_76pFUk$;yF1bJ3Cr`^ND-={Su}5UQJvW{MBo{
znfaZ1P2<qbiyr0u+TN`8$Jc+i`NgSo-hcIbaVvZK>|=XZrrXXsIj7twzU9b^o`1z%
z##`R4H)sC6a;0A8`BRl%JEHciOMhzTV60}dc-jA}mp*M(S9Ih2T|bHETu^GC%F5#t
zK1%&~;%u(->C@+zAxCxkg10RRTg!9m+`1_XOUrvd%P#qLRyyuKSN{7-53RHR!rr$Y
z^=j*z_3dB8&MS{a)5_l7S~_Efgr$|0mZ4$b-Cd=RAvtr!+;uTKmF(^9*G6u3%gf8-
z;^CRHcdxDLhH2WGr-C1*Zca&^Qad?qQ~n3(W=$Q{Q@?ixq;;p{WUNo~eY-mK1*__Y
zQ*FG`pkv8xe>`aR^75Lrd$+Z^7z0DzL|#D#)AQG^g{=--8}$0x+OH?o=V!dTvy+RD
zZ`$O^!Vj+`mGPZTeO}Pb9({9VW%sN2nU^BYi(9W;vEo9PsP>lh^K!QTemwT_@d@b`
z)BRDv1kTy^%?B6s+y9Hu)6)YTx(+H;7A#QMxog+8`TAd;h892TJ6gXq<#t_BPQXI0
zZ1vJtH=eas7Jr?&^v9x0{^{P=uAQ>q`ux?}rC;+7|J=06sP@;(<)Awk_Uzp&DlNVG
zSg-VFMn;AQW&7KWWJGrSzP2uQwPEtH6Sr<ft>6D|msQD&0R6b~gD;zZ{fpU{le6U0
zr=po>(@cvhUHnV)4zCKmZuoremA)&+MrKA{W@eiLcjn00UfC@76qJHKd?-jwOFMP`
zynna2{<L1+|HoNCh3%cgpqN?c-2Up*)6-Jsd2_11yl~WLSkf)7uVrEqvUBIoS(e4i
zK<9*LFfar#HST)&zinq~YU<0Q;_*|qZ#Q4KAnx>;Ga(TXGrGFFMI|Lyo}8>M%*DVk
z#bH67W&OWBK5CQy{XAd4s{a2!(1i&+Y-R%Pe~X`-5Y&y@;=$JJn0;-{#SMvvIaom@
zNPxMhh{%hd&*xvix3~K0+Gz7$dHXmwW$&d6{&{$MUhERpzOuwq7<4?;-m0%kBH)7W
zZdg>5R(yQ?+W7r(a&mH@W7efivs?^Te17!!G3a>rox65vX=-|cww^5n7tgQQyN@n9
zaKJ$~W=DXYo*pPYEMBa9aKX8{dy5w@o;qjFn&WcSKGECrLRW{a<p39L0sleAHdK8+
zYkqZ2q_OXOyI4Ivy_P1cr>CZBcXf4LdU&|~>h}Ej+#4Gl`S|#tDSLlXQqq(6@Aadj
zqkZREh04jvwKQ2R+_3|68O5m+CoY`U-@oSR>FLJoVj#aWc>el*|NlBcWw#YuwuA_)
z`2=jwyUW4)earm%f03V`pAYx*>$_L+n70;mencBLs9-zrl~2}c2_rL`hPiq8V)y>A
z{dIpiSidhRetu3<N5{w2)s>5vH#8wZ0pjlh<C!yO8qM^vDt$F&&)&UHA3r|4UVX>%
zCCis<8yN+Ob-NlG8h-ltQ4t)j40Yadar4-CrI!5t{e5-%`FWqdf1lpl%X@IaHE&<v
ztD90!r{w3a-?78u>&wgS0SpWaSeSN*s;a71y<9pS)T-%|wboKla40J)V`+}kHZ%;}
zxN&1qVIk<S`7bAw`vt&Bgkk&QMT?w*m-$?LKEHk)yIh6C!Gj00Zf(&#xFBp}gbwJO
zcTkn)H`glj_O{%t%gcBn73>H1+7AcWtKRSZ{^ZG%B@Z4XnB8<+n`kzBYsJT;+V6ME
ztA4#)E-Eg5++GXpuoZmX)Ag=`%FXoj<(Ze4oxFQ@Za_hJ?7o^vZZVw;Cnu{vef)Uw
zix(ND85bH7AX@aIx8+Ri@9)>r(gNKyQCV3j$n~!4&z~C51s0K!k+aNlXE`}Lr>3U5
z8h|neW5{-9cD{@Y3mi|KKR<o_`t|cFpGi77_U<Zp*yQBw92yX?V8R4}y&sQBLn@^M
zb4z}G$(-q<HrKM)t(l$wT8E%A2dn9$Bb~xs9UY)Unm;`~{q_3(e^ViWyMlA)&Ye;g
z1q=A(Y*s9vSLKy_yl?8(t)>Sb^jQ`>XqcuGxu}g-8q_9ub$z`&B*FtG|NHyCKKsdu
ziJ(Ri=++j^;AI;O*xz-$y}kYQY5o0EHgDd%E$^<DzrVi|xasvEPeWh-dJCuUmYSbM
zTeGkGH8wUD6%;5)gmj-e<+V0?yPv10=cf-J0)m1j-P)Gx4e|P&-2DB2x22t*w^mFy
zO2fh;LN|JwipU?$r%#`P&bR;e{yyjc_pHy)&K8xFs7QdaEkjnfYqyxDf`Y@hZ{N7M
zxPo?<z2#u7_we-WR8mqZd31!6SJo;d``Vg`bLYxJB0b=C&aEw;pk$t!x>U`7-WqrL
z+9?4A(Og_yB4T38K0iMXzAfZXD|ciG2Pj81+|n^M4Fw%al9;&gSfA|bdwZ*ex$->2
z!lpSnJ5QZHeR<^Ow9IE`W<q0R?}H<q!bUTFetkSHZ<=$%fLGF}CE&wszqwXRL01`V
z*%IPCO=sf#`OwfU&_8+Z+_df6xA)0dg6=4KF*Q7H;?$|42OqGp^UGa%KEGaX@x>Ki
zUtd3c`}S-|B<>Iw6&1}q(jj<jV{*HVt?k!a+3QbUyEZMM;H>%mn#G~3!%m(&>Dk%Y
z`RdkIZAcsz=o`;I`|0y%(5*yIpFLZ4;DE!;+qZ>zM5LuxA35T(bLUP?O-;|dygY~}
z9AX0^BUefqr@dI*Z#QZ4=FQ9e=j*Aft2eD&uBD~5CFiEm(q+q5WnW+C<>PbZ@Av!u
zkQlsT`tadHP(j$-+}zaE1UlQ_X=S>Xw>PLn|MI1TSH>a$bP3txW4(~JUc;_etJm+F
zw0`~iHEY(02nc{~LJ{<`4UC9bv0;P3T<dbb_4|IUTC`|UN^-I@)Q-}VCr?UQm8|IM
z?tc3I{rb4QRf1l!<sTopg0j^AU)T44IcNR;MKiyh0;B|bH97fs-^tszqwW8Gne5gl
zv$BzyUC2vz>coi)KYS<v<=$>F-AhM0g*m~MN5iidhg!Ko<y+mK9lyT4U43z}`{wBh
zvc*3>B)Yn~o;-EREBpGos}mHRA(c(TuQ$`<>n=`Ebe=M4(xP>-yR#0pa2iSVvIMig
zxwki3JA7Tp<z>EGQ%{S1eQ~iF5`hgNVPR_V@$pBG9a~lV`x~eb09Q8OH~7vr)3mdT
z%e}qr>bJMIukNd@hU6%Q|A`wTL_|cEbaZfR&AS_QdAa{|P;GOk`TgTAQSB*{CMo&*
z`{&->6?*R6IY`C9kf*1j;$ob3CSY^gSrHkTHD<ZD1Wws5TC>I{Ffj1far=J@|NZ@)
z{rA_`UGMk(hEz}tf3|^A&-wHIZi@qhm-}5k%x~|bvGvYU@9CgxXpGa(ow$BIJig{5
zYq2#WsIWe;_V=&X>rHEZ6il5qE$h#ZkFW0SwO)87Xy?wIT)ezjk9LcjCLiNDJJ&j#
zt+^3WCLVZeS@UB98;`_-Idf!srA)of+kTf>c*Q9F{JhYJh#7jZyOywWi@o@IJzgCW
zfghfGwVh14baJwKs?p3ZkGl0gX$R{TzPVxO>gMLPO6t(>L)`k%ir~lRPoF-SRD4Ky
zdwaY8x^?S{ii(t`>^_rYrfq8*%g4vp)!!ff{@z|lOsfhweAU#{oVb7g{%h-Ey(PA`
z?B2cm(W6IKHYU3tVgR*0KnG&3@L$sbq8T(A!IYN*h-whx08tAd$H<c?HN^p>*CF=n
Y{-@dbB|mq!zXI_+UHx3vIVCg!0O{5<+5i9m

diff --git a/tensorflow/lite/g3doc/images/convert/sample_before.png b/tensorflow/lite/g3doc/images/convert/sample_before.png
index e5317ef295062e79c66430512ef1c45925858ce0..55440d324977f0ff5b795bc80898857918066e96 100644
GIT binary patch
literal 136956
zcmeAS@N?(olHy`uVBq!ia0y~yVCrUIU`yj*VqjqKm#ySyVBj&ybaoDK$t*6&NvvdG
z@aUWxULF$sUHt!^i7hP#!U7WnzX`dDq;PC%U!Wiz)1u;_(DK70pfEaHJ8HW{!IePK
zjkl~AG_@98i#)Qn^iM!{S5SA<_XpqInYMi^uiO85_wzm9-|f7A{_mahd!NrOzsK4$
z>4?_UCKI-<saa~4YB5u0PSt&Mhm}#RVnf5;h@e$m-d<h|4fCGgfBRHafT7@`_nIKh
zf0h5@&$?)7Pi1fsYKX~PxkN(Y8{?MwE=N}|hN&|g@64Jil_1S5U^FxGCc|yc27~z5
zznKM|W&LGga8fh1WJ-9>P*t;W#z}S_Q3jXPNj0VkI~W?a2Wf>fwe>PMsjwz#Cj5|d
z*#GE}kY>UYwgbA~^=*FEKR)yJsYv(d>e^g=28O*Jl`Fjz=l!Ygdt4vA(dlyVVL^ud
z4V}Jh3M?J9uBsnTGW=jD44P?eB-o(o5Hrb}ac=#72jSVv4)32f#Vv)|fZbzXh>Rrv
z0VxI9DP0pecU5@Iv$5i9FmzaU>T2Q2TQNbWrl`FBanf_sr4>`;;?FQx`Lr(6(DgaK
zbDx9Q^}{y<;wH>mc-3>;w3C<JnG9G?%wFoM`O)gk;!jN-6Ky5jCeL9zz`0<;aToP$
z!zsaqnf~!135_;+OV0TRh%Y&oRlKrLs4#?!fxAI#%G5NqFE6H?Qd+#FHgI$5DU-Q%
zSu4F{=dC%MAE-I0q+(UzvJOwS2MhtaOH(y%cgAL^g__q(X_W@83X|5=5#jg0l(oQi
zLIa136AQ1)30F=o#r&6RUblFZ3-KCCJaJ~zk6og!%$#Hp;M5~B<#D-&qH|_`oW+p@
z1!u`AKQ2gLy|;G)yP?Ds$F?AmhjtPo%5q#M7BDhPPT70)(no2D6aNEan}jq1nAkqm
zA9HcDH7paco+Nv^FKzebuo4bg7xfE@B^t6$zc2hNap+&?UvaJWTIIFaYpG$<uYXni
z%z0+~RH?^7+GN+t6BB-4f0rSCtNq*3!!Hx8TKF0|%$Bcx;L5-3?@NO!30arlml*3W
zuX_3R$7P+;n3#OMlW!06KHXKj?!bk^7q(A#WqW!*ZQ<3ax2@N9+?#y<cgK&@f0k%C
z{(0{4DmzQRw8_e8?jos`Rvjx2huD@*v+}t5_vzgqygZ+(SF1X+sdg=B_#^-I$gWvD
zDH>nbeSG(!ZoPGLl$j?lgP`iFlRY;39dx;ZSBe+~&I!7-fBuQ%)eGz=%$%xwsV-{X
z>}Cds8T0>%%)Y?F62jo}Z~2THf))kx0(<_56xXnQ5NBqOR+U=jBfXfhfmN<Yp+z<4
zA%}UzLH0ThCyhl0GMyf-V#getm_(Wny7(N`Idil|=NQ+TgKRm+%4+&;S2!#8JxHJ9
zsIB*y?be09<1Y?*?@{Jxa+VNaIy7m6@*Gb8L+u~bojNp>8a>oDb;u~q4-k?{()!41
zI6uN`jiQ;*)g%2!7$0#Ic3JqxsJ|1CJ!1Sw*K=Y`kjJeFdQTWG&F4{md@`g+XQscO
z!s(NiPn?QGcXr<Kyf(@Ggz=NqO}#Zf`xZF741PAn_NnX7V8IZjslHDGc}(Z7TA;Pk
z_ln-CMk~kICAU`iz2ZH)jCD)Lw?)k_BCL9+T@Zfp@(WWH@7}rVmiAv1e`#GJ{&!;P
zflX<Zo0uXC*Z73lWh-Utt)8@X>el(Cb4q7kJy&%yZS&ci>)t57i6|4>J;AnhZ^Pcc
zy|eEgu{*!-c-_H=1v-}V<|Ozjxt|FvepVD#Tvo(atXFW$a{bpYU*CMG`LgKirms_~
zHh<OnTJ}Qr((VP{9p1}cpYi$(?-RA6rFTx=alNB^hwYtZp7nRx^DWO$aX(x7iK}vR
z#p;UP6~`;hE6;!Z`c?J|^B4D*r@yAZT7T)k<Nqc9WdEAi1n*<FZ?bP&|4{!){$ulx
z>p#zzmXO#X(IS;1y@FRvDnwewbVu%oTLB3-?i-v7$g()R;nap}ADSKtE3cclIdH~B
zi^WBYy%z6vd){N!yYradF~MU=k6#`OJ;vU%UTVLA;0zuUPZLeE$b?OYpG-OF9Nf3E
zJ9Em)D<yp|(=IPw_MK(Qfj|Mp2^{+bas*|>w{YKd3QwG~A;2R3$KDODH&WBG%x>u1
z&6yNA^`_%yv*J^Cj@q1yN!)j$rvInt=k2vQzgoUd{5tb`diZ?q+PKIsTRFB)hzyB%
zvu#Jt$7?|;lBtm?JCAHS_i37F?$bomQ(23yR;_*<Hn+C5w)vOWudpw7FMkiuzI5xb
z)y=A|uW7G0U(?UY$xX@$F<YJ6cdO*qtlMq3#BS+rw@ta7sJ?-F<K)}A+pTlHd!(pw
z32VBnb~)}6`|aAdjo+kx_kPp;_Fg7N_L5AR^_#LCPfrwoDL+#j^X|r*8&6Z7n;gnq
z72JDy{kfR*`sYl~r=C-v6TfrEu9O`$PlKL+x^VT7>A9_UwjTSsY<A;qgYBGov+wBb
z+<f=%-R?WrcW$qmAh%DBRW4NSxs9j4z5mYVR_8zejrhy-OY?W`tMaSe??s=5J_~(Z
zx_p1-|CWE7e_#JvZ*(Bj;ah_$vmeio7jIv${c!J5ZSsPb2Oct~^XEwXICK24`=j8e
z#hDWhE_mT+%yyN(%IZe(hMW(-4qmTe-KSi$|5NU_zYolh^nbkl^t$1ZOpkLDwmPr-
z#r=2kAMqdac@kJ#*w1ie@KgxD5MRM}NBD>Pgz$jC3w;aPonzZiwdF}x>EBR{klACn
zL)S%QlJKWokJ|>n6CWkcZoA7{$}_bskGEFfnOvB}Hsd#*JIXDD_u2mmbxAm~rDK_g
zmXX>frA?}*Otf^fe6PICXn*nN#jF?WB<|19H1yrL=Ejd3svE-}=R8s=yjmE0xSeOd
z*`Ej-dpp%S-#>R*`dQQ#CbC#2{pd(l&g(0YTPYJcz2nG^BdML+C&f%wQ~7JsWvi7L
za&^V;lgyKstIhWiT+-uzbK)l>Q=8JjSG^|NO?;z@cLn~M@#WQ*aLI|1-p03#*i-yY
z+e}WKs(OCa`DZE1)AP^P8Cqq1>ALtfx#sv8+2YhSuWxuJ`}b~^S}svuu;*gM#~*tw
z*UageBWwBoXy+r}N8&pRcV-vwRuP;ub<*)m0WXU(+jSzgg?v-cx5&@DclghPKdC=u
zSr2n*aa%nV{ABuRdF#ejX|8xtRk2*{S79j{W#X?zpNq%`%+l(fa@;G&TW#u`@XA%s
zR^3u#@ry8A*xi}FZJN;1WbfBkowj<_hCN>PGpKmE&XP5&-h4gsrL$UnrcaQcmG#Z!
zPkT=CpIUr+|7<7!i9VV0vg}K*U9rA=Xj$$YKcDk+@7WwKJT^!2t#NW%-u$DpLrq>!
zzj<%-^mG3{clYQ^J-cycrseZ{(+%UVoiUwr`pn*6p%=oX_buB$t^Tf2{)IhPKih?Q
zX#1W2J}=(#|6it)O_%TU{7JiNewX{R*fyO{(Oc>at2Y*XoT_?wT26e{rY$uu_ogaO
zyQ(LyyFTLI^RAEDexZEp?rfN}J!tc$+|Wa-eqHrjExx{P|FVR$Pgi|dbvJwM^>fj`
zv*woEZT-6Nb$RZFn+ElVZ|6U~A9q_~^Tx8tr<a{N`{r!w?DzSe_lq7`|9`(~dsOb+
z{a+88J>9nL?aAByq5Lb?-Amk?9LO8Fc;(9*mTBp)zfZh>{r#s~TMM`UeD~A-0sC40
z{<<?iKc1TR!}e_T-|z2kJ>Qf6w(j7+uBb(2MY-Xzj<M0TVSj%zALchPXEA;Af8~Gw
z+fr|(SM$x1jVb9U&p5a6-QlP0@g^&0S@l1W-+A3;-_K(g&n%wa*K2jH$mZXVlk*qX
z+f^yOIr4DwL3L?=)_GP1xBi#hmE3Q<HNE!yyZ=YOPyef1r@Qre+nm+;ycxL`_Fw)i
z<UXt?vs0q-<8$xA>E~kPDu4c&x<&i?y1t#a%6zNdJ}f=@b@~2je}BC@c4qdP*`nKf
z@<Z=!+qd^+{Ja0R?moL-_QCJT+|A{|@9ynT-}(K=_DlEgZ9l&=zhM8T#=Fh0+U@wI
ztruBG$@kV>`4{o^N9^-E_kHW)e;;{!a-Y4^e~a5wZ(jX3rSjI_oP))upWpr`^C$gt
z_Y(2#@nZXHzOQ<;_37)k`G4-ushIuo+rizP?=9bl*B1Q0@WJrk<FoG`y~~?ALy(z4
zgCQkO?n91r!UR^1D8_pkZ}Zp~6t?_US@4(r=lOkQ^~?-E7;if@Zkg%eB)nlCXTdzq
z53?Io7*;M+VCQ|cw<K(q`3d*aN7EUuy?*!Zxf#Q9#vcbt%J)ZKzxy|l+aZ>HmRd%+
zlf&-jI%@`osN3@&6#q8QxL2t4uGMuC0|NtRfk$L91A~|<2s3&HseE8y;MB+ri71Ki
z^|4CM&(%vz$xlkvtH>>200A5Oih{)C?9>v4q}24xJX@vryZ0+8WTx0Eg`4^s_!c;)
zW@LI)6{QAO`Gq7`WhYyvDB0U7*i=|m<QC+nmLw`v<mTiRTUFR9fz7qbE4Bg&>nkaM
zm6T-LDmj8IREY2mP;kyKN>wn`Gt*5rG%-~$x70H<Gd8ib)KM@pFtpG&u+TR&)-^P<
zGBL0+GE#s7B|8P1qLehNAQv~NT}3Hrwn`Z#B?VUc`sL;2dgaD?`9<mahL)C=`UXb&
zMn<|tDQUXJm3bwJ6}oxF$}kgLQj3#|G7CyF^YauyCMG83mzLNnDM5{`$Sv^og&Uk#
z3=OE{{9OHt!~%UoJp=vRT#&!O9xg6%Ei1vVx;Uh=AXPsowK%`DC^^-&EH$r08D^Qg
zr*8mUH^>v|nYjgEZ&c(Kz%5BhhN~$kEy@AQrzGpA=A`DP=9Lud8|oRt)mG#dz!hV4
zJj`m(f|6vDirfO%3WWPJ3rdo~rWco_=p!l6H!?7=&^0j9HL`?TRFG3zjBHbBaVkg%
ztUV_sO&{a|P+-CAORdO6#21RoKr-NPg()q_$S=t+&dA5%6Sy>7Q-QCql}mndDJVER
zU0`W2B|kYc#R{(4D$UH$)YRP2OxME9(o)yZ!X!=C!o<Q{H_bfBGTAi6$jB@$8RP_z
zX`Xq-C5d^-sUXuTatrh_GgGV#ERxNP4NMGm4Gc}pbWPGy4RtNe3=?(D3`~p+EYr-4
zQVrqy{fjcwGxHL2kX;2bCnYn*D$O$4GSS2|NjK3X(OlOg&A>$0BE`U5*Vr`8(#$B?
z#3<RoKuH0VG^`v0JZ+VX^kCe8oWzo}{Gwc2CEvtk1^-~Es*udw)ZmgtP|O>e8e3Qx
zo0^(h7?~It7@I;>gryc0XXfXD%rZ34GgDGXNw#v!FUn0Uu~o{<O-xVKFUU&=i-L?t
z6RpTCuyQU+O)SYT3dzsUu~h;&O~FXd&;T4-5V=&B#FA87B_jhvOI-s)T?2~{Lo+KQ
zODjVIT?11qkcU9V_~xf%rd487YHVm_gsL>PBrzqiB$1$EQ1Y<~$gD`siEzoxP0fRP
z)WX6*NdX}ji6m!e0dfx_VL0dK<QMtor=;2{87nCu#66vz1B&v~GILUGm0TcWF`z;O
zTtZlRWEPj?7ga)wrQp;;C@UvZ0bD3pB_@N47+a;{#In>BC8%I>Mq*xiYKpCrK1?JN
zOG-00u{1L^u`tp#H%$YjG-C^0%M=p>-Lxe0L~~1%<YY@rbC~|({Irtt#G+Kk^whi(
zTP63*+yZdiC}=>^h$c*7c}6NI0T>us=z=1}EX2Uv%Fxov&_dV1927zNM0o(4bBW6s
zkWhiU0x6lG#63hAny(;rM*yr`P;hnz)fJXtKW65E3JbV5kwO^cBYjX24~ZR6k>u-(
z6mdu;oRv?0av~_czzIDy1sWDkiRoZN@T<fo8=R3}o=1c!Fm=dgI2M(Z=ND!B7x^b;
zr6!l)F$Gg4vN=Ji>6yhPMU|fTyn~_$RDxT1f(nG-(t?7V%+w;N<-w_i;DErY03iqt
zy?or75b|I>&iMtEMVaXtCI01k*sa1)0oLb}nVgzejNKk2IdI$~RrhuZHejJ37dH?S
z+%f^RQtT8E&7ah~6kDYtC40L#wZ|tIm>8`*T^vIy=DfLE8X*!b&G4bTY+FJmqpP5>
zj*FD0X!OSDjRH9bcRWikk9XzKd2{HPo``Dbo^46o9864`Hf%7pzAxY1p%HOF=fl5w
zziq<XW*Wag$M^f4WpUGl(@z@&oX*U&t={IK@Z<jff5%G{JhqzT-TCq4<YZnclLRY)
zV;drM#4Z~uv}kQ*<CVIzqmbFYJ3x`Ewdw4-E1q4x0xWR3oV&Y3p>irGPt_!c+<ULa
zp(vV|miFz@QSN1n9kkpR282B0GD&E2<1qA#kDss5apg+LGG@nBti~lv>ppTQie6mc
z$h^jFv6F#w_$%>#fgr(U%a+Yp7^pa7q4w8qcaD`DYn&IqPi$LvMPE*FiPD)fXOfbW
zXZA6z6gzWa>aU~9Eg>vr&)x1H=80VyZ{y_QI%C$XeU+cx76ur|as(j^F*P-1W^Vre
z{=PA?aNqa$_xa`R{@mJ{?Ze&_;_BwsHgV#_2%R)5fny*iujHzc>g8Jb<m!u4zfTD|
ztztVKv(QPavMtS6mcfa`v8LwF=jZ3|@2jo;{Os(rv$Owyzh8fKRp{a4m!?j3QJQH}
zxhX_T)pOF}hd+M(+H^BVZ~E@6tE)~;Tgy3X*~~sh#gLVEcNQPNcP}nFI$CMs6Z?{q
zl9HZ|js!D<BOEJ3PCfp(@BX~mvrj+&eA-TqEp**`We!Erl;q^#)nRMToH_I7i;sZI
z>aevEyj+HDr=Na%d3pJTV|KN_LbiS3nXypNsV8}3#Ov$p#W`4hMw^(L=H})~@^Bhv
zAL$UBHEY&S`>d?2J~`W22P<YYt-ZscI5i<J@86S?!cX^Ky?RyNzAk3DpKO}5lh4G{
zPpz!1lqP!oT-?{^7ZnwiTFt>5(eB2vlB2%9zO=OT>Hf@F$G*hO+<5bioNbi=3nPPy
zXH`W-$RUU;v<{s;d-l*Fr=RcHc%=-ad6MQVU%tGpjg8^M&l-Jw{bh#}+0rsvCak)#
zA+dRr)Y~Udj+E$hB>s$L{rct0nl)=AdRq-ao>_J1+_`gCuZl9r$jHp-V>RqiIa&Aj
z7vq5w6P0K7wHjI^b2?4+%gK53_xJaieUFac+L~Sd^3u{LPo5~d_Zi5Z)-Nn9Tp75S
z?Q-Ch)(;;(w6wHrvRS-wqankKSFfghJiEZ^z2>*4ZYwTr%()UUX--VId-qF+IvsDV
zt51q%_C1<@L}mG|(ia6Uj(9F()+`lq-FWlOxl*4~pO%>Trh6=@`XhfPE~x5VoloZF
zm7fnjK7RVs64!%PCQeiN+EvnL&Wgy)HkS3+DZb|5{$R^%Tb5<caus;}>UxsZgrM@M
z1O0&kKJF9Q=Q}I62so|cFg9`GP+ZbwKo0l0h%*Nx!-w|stCQC8Gnl9@e5%rYoQ=Wi
z^Tp#wZ+qul(pfa^-JFkEHBD@j5`-BXCfHcLUh9&zAaB)7!}^-CKlyXKvOgJ}&5~)I
zqoSaj``5@*i<jZfJUh0xx=W{DTQ;xi*K5~8+i79>>!%uBHCGo+cl=i@bvdl?(6ls{
zv}tEw{y6+UeCfn>PJAr0*%c=3*)zTJ#fq3OpP6sQ9(%Dns^F@y{STeMsV5J2HI!zw
z2rw`#_lphsWL0|mTui0uJj00_bBmAF&)>22_0%WnEPw5iV+|yvKUcgDHv75s&7E0A
zN?mKRy|^1yuGViic=&6rP-$`;@6qL{-m39Nt4;oBu2}s)>rC2Qljv)#46mH`m^_?o
zqJO-4^>vjxa|xC;*Q}nIZoN}I^(N=h-P2RMc`J^!Eeejdb5-Svsg}%-J1+d^kAs#~
z5LcAW*P;N&qddI=ObiPq&0EI)!#mz)Lgy2O^HcUz%q#wIzvp--149ej?KO(p|2O+D
zU}R|V%r&a#PCQ|lA7QX|qmR^vvsv|jBNs3-=uE!VqA_n$!=vI!Ht**CIa(a~q;tyT
z>BYgD7S2fScG@ql-q~<!*}jn35ox@Umh+#mu5#7c-1cVHtoWuY37iZpZ8?)yF)+9=
zA3o{KYFfh3uuE`)MY=Yt!;LD*uLY@}O&R~Hy*t1s!?0-CYPYlVmrTvSC&IwQz?1d-
zZfw&nna^!&oUElY#7ZqxCo%_wIIa1zr2f)QbC!_HC9AmYceQfmEew|TFg{xULF#mz
z<qx4o0WD@r#?@T;i#8iy&sB)-HGG%v>+R8}7rmoi@oLSgxZ2prwh4<@yE^9`Ydbk}
zlie}l9M$tdte1N(KdVSro$+ezjCBk3mK<nMKJjBq68rh&En&H_m3=L1&k4`{!pzWg
zmX$%lq(n?7=*^5-m)8W|lk8->FSENwsk&mpIz#>6Z^R1cGXC-{NnNj}u5)kc=A!+4
z-_PwnS;z2XIx}zSbe-#$guFCk?KqVfE)*EX7_@K)ZEM{3=i|vMM}CFgHc|ClprPO?
z;Les3Fqid}!}hreD=NF!y;fRr%x|{!T;l`1tV`}~S^8w@Z>hVtCQX=KTI72xvux|y
z%{w}$KNfqVHT(ObKZ;o|w_fqRUmAFN^J2gG=Wf0}$JFrc(A=u6S5wcQSJk<{{^$O+
zp=UNtS~maEynk<JU)r})>8tQ@hxv8YdLNh044j?ysAA(SlinWQ$;R(LmWb(GyK-*f
z)zuS2`kdy?+xBY9u_No-)z|O^&70%5oUtJ|l%3&;WK@-Qx!9i8P4P3ucCFkLueI-<
z<tpCS7dRF!65PsT6CGr@-2NTUG{*4N=NGJ55Y5GK^X=XIcKzv1Qv36GzfLR5x%l}?
z>E|H%dX*ms<R7mNHGSE9{t5rS<V`KVe*G^wyGO*C`9w0G$~>FNrSFw}8CUxqpA}tp
z?ar>)<=gxYs-JJ|pZm-H;WlqBh7*-jw^})HE%2yiX4n!lMRd#Vgc)}dH*a07s`|sn
zZ`PmnEDa0+@}?1U|C*&9UUq){QJK$qccRr*iWU^lQ|XcT@p5j<^)sQnl5_c3EssB6
zl5=<J&kxEB7fN0|KfTe!BUt@+n#}!~C9hZ+V{T4<&cN{D=jB)V=Q_BQqIe&r-8;Ky
zqx$E$Ex9)xca~&4JG3^4L7<+EL2AXE;OG+v@9)zrUw2X?Ve|dk5B77!{srAWDiSB-
zJ@5T&@0BaBu4VMy{p!lI<-W7#Z1||<9c%mik=~2nXCKZojbspEj9FWFnKwLdZPse_
z>e9$0^Gz3Ym;Afy8`^g%&XP-1G2|IfP9?*ViA(ctw>`2hH*#JqDjCGE@`aa(gPJSj
z2?g_KoC~k-DO+Iw{GZ)J?OMlV+h13mxfou&d86KW!2DtDFP~<U*Iz<Uu58}<(ZrGI
z>nE2#5$k^hGGCropY(D6oNj|3|01@|*?cG@>cTd4fnO>N*KhS+d9!Tg^e;OT%2f^p
zTs{`?@tYj$1aZ*}PM@uroc^23%ge4QCqB7)=SK`fz!BR}h6_ScJ8HyM2r{s^vonaq
z-?E83wQH{&Bg3cQ<3}A{Ree#-u`am$|FmGD`X;5!-R7y8=B9H@;}1`8Z`gOwAe({V
z=-Zy%FLO?t$xPcDmV5E}<;d3iQ>M#{GVC<>^L#(=+%BQHR(VOMW~*Jj{ys$AQIXjx
zcrt_D^*Jra<wY2-9KZAaUhvcH){{<oo!eWjSa@<ti|X?$-1nVIx9IHKy^`mYuRU|l
zQ#TG#-(GG9&T@TmEqmVCyNXJ-Ff=?6W^iD*!0@G6S)T97i|?7Mtt;fD#R4r&0x$Ql
zEMQ1^Z{OU#%xJS<&C_>YmGf;PPcLU?*xmSPdCXD%zO(0cee~JAzIN+V-zVqiZ<d}V
zrMphE|K`W1k()pL&r;#%_-0;vHbO-sNcr-$ST+U)sSjsL85vIe@wfRoJwU~EU)it3
zTVId7%A1?rGohRB{Ljg&!#%G^tynqzPv&vXR)@)4wLz&B`TL5^zs;*IYMAptpfz{?
zs?~d5q^B=g@a^yHPhaOLm~0W(xqh`;`n~n)Q(^W@4ShF0?p(h2+?L)y{W`8rA=hkM
z>mD9n%{am2$upVzlh?k{^2n$u=g!nDdhlC%)1D<6Yn+u^_Bo!`R51NyC-vy@Q-@b)
zV^^gu^4#`m<;%-6Jti?S=q_U{%4BF_z9V-}YLac|Vz#vB4YN}B-f7`^a6QyKm9gQ#
zQJzckPc-;F!|OzzUt&15FzlzsN*RV1Th_?S%=S8cK7LYAOr5D#>CZ!JLd2q{3(0i_
z$(?xgnW16X{pWjxg-fF{W^Ha*aZ;@@IZMS$VDoIgtHs-ucw%|X{lzmpG8h&-snq}A
zouqJuH!|&PhNgych=(eJ!wSiw94V%TlJ}mPM^jfXO>0w|`pQ)Qx=Gu%P#5{LeLJQW
zKi#A9=*IF6i>B)i$M;$>9QgfsCTCG!qeI0Tn+xyvR|mInSx)#B%E(|6vn6Y_w_MC(
zo;5pd7)o;mB0g;`YIQtr?3e2~<E?vb{SGz9(~Jx+=IAfHaP$6Mr|+%rFYVvIlItGx
zs-Bn)35=lL#i=Xb&d#2@Lr2+e{|9{zUf%hvp|hkmMMS;Q>pk+&Tr+b{%Wa3vfy`&G
z#co|wVie}{kVjtq!u3ZpuP#_+e%)y8sg<GH-<LYA|L?GFrSg8MkCpB5=Z;#pq)Ok-
zdsnmDZ`Ln)`*ph(njdaffAIOO^s{^7um9ywYMUpuFnd*4yKYw%6T^bU{0CP%)gKhj
zzH;i`i}e{n4{o@wULnKqt9g;9dfcCr=L6Sk%4YkXZJ)3Ahn*qn^8c&GSJfhRF<Eb3
z)2wL2UcFXq@7CS7Hb*klyuEYz`c2>29J4H6Z`r<lQCZrj+po=5ow~HfX>)d#m5IX6
zD3!lac?ruCA1=6gqvierHTT}*YnRWg{Ij!d_uRDZz9-My7rApf9KU!icIW#hd-uKn
z9!~8!E2jGD)puEg4JG#tL>B~TOw)<8S-8$~;oChAPp`Tu!4$7-{zpxf`9j9B+vj`L
z9~8{CxcJdr)l*3GZoq24m{$`$7$)pne08qQr~4v>e;sd`Zt;A4Oh?J|`^DgG#nruT
zeYf<!a)|Y(zdCU0$BMk%b=QtWUB2eJJYD|mvZ(pr?rB$4sO_JfSez`@eRy-eM&LYI
zhq~wQ%(vfP6go+|oGVw++UHvNio=hxHeQ`vET<|`<+}0aA{E<aJ7%^iCrkfwHZ2SN
zpUvX5LgnU6^O;%+(V5x*UokQ{Z)%hK`>1l_=T3pPoM78mM?e4Z7e2JWY$f+6@%R4}
z_SoA^OAV5~bWhWP*`@OEc8|$z6Mi>(oz;5eBbDOIC_4SmSvSYrAY0w{0*5Xnd`^0~
zKiN3`-;bs2^*;N8w?CX1yu9B0N|C$X^v^r)r02%g{q}HbUOI2BhQUOSiTPX$53bS-
zIHJm&=<#ZK(XTb;E2``h9|nf`*3QaLG`VV+dPV1^|NY=p&--_s_SXf^_4DdmdH=<S
zlfnD{33>C|wYzcr?a^qhnizD4ucYhGZP^!B%0FGPx4A7Zzq+HEoq-|VJv_MZe79||
zndtP`ip4KvRhcfl$#`QvLsebzX;GHoOUb`Nzh-~oIl4!)WSi5^vJ0A5)DLTgwW(Z<
z*zK=AGbZ-#HO6~xC%WwSB!HYyU+ioI?kd!(|IYQR*=1#Km}=SAn8q}B(x#O5ebaZl
zb#GJfJil+B`PUec%Z7#wpl(HNd!O(8_p{HHh)j*Tv(^9edP7;3!yEL?N$+qJ*nm1W
zzL!OKi&)S7_`Q9_sS;jMlUT`jl3r8O4jtp^4OkkyIZU^oTRgI5mHxryF{TXx4=qGZ
zY&R!|FI%j^t>U>UV$CwehpA<5%kI^(`ksGvWo?gtlt_#vv*MCPudc2x|N82xRp~1O
zS&pOwGYpe;bajvSNFFXx@HzDS0*9igssM+J(#7-Nokat^yt)=HRJ<H$z+-IU<l&mw
zw(g99Gc%}5dFl{P>`Ya4_2jl=JiP)zflQ4G*1vauogMhJp)}(FNGLL*eccunq0R`M
zYZosjS_!lzwz+Yv)HritYM7T-S7RgN<v;^|n6X<nbVhT+IR?(!aK&P28LDSM&T5t1
zb?Fe#+7?ME#U+ajoWrALESw7J$c2ccz3}2T-Xa8YR@Wh(*v!PXutOjh1kPBfz4pqL
zkTVw=K$<pzG+AhFM^Y#o1-EpLk+Jd47`^%D)2#%a!EF~V6F%Jz(hRcEZ^pvU5Sh}_
zuXE<eEL-f5%4=L=2X@}kq>U9ea%wM6u^E@xnO$a7XbDK{m$U8a?-%cMiHeH)_wOHz
zV@i7Z`8k%sv9WVsUI<uum}hSPg3MVjB?O#aNV7PuSh;fJ&74*z$G|Budecuo{rK_Y
zv17-4+S|P5_V@SCoH?_<pZ{{B0e|#PK_{*BCnqKbtu)cr_Er(<JessKCT_W(tk3zW
zEZ3lJl!=dzm+PPYUwQV~clY<(H-B8PsYST!>0wa7SU!99tn}5Dlm924fByZ|)$W58
z0g;ESARY<*a^C(w2Mg2F{hpJyY~7k_CKr-s;QSn{&y|^t=fwNtx0ZT~Hz~~MWuMC1
zwW!4@ajI763_cfC@Q{v@sp-~6X7))cD_5?xOgO->?6701im7R6c=+_AhwiXIBBG|Y
zma|Eq`)Ka1Es0iQA#NKwcPq@<$f6j+bm`KiMT-}E*VM7@j5+tT$l1xsYugPck4ZT<
zH!YntD{AFD*1ue5F0g`P;qTfD7cMw9v(212Q&dJNR7plghT*{d`}<S6b)S5Ge*Wgo
zo5%ZPvu|ujTse<JRMp!0_tn+mZEbCyo}SQX{A)RD*>v6LXYb$d&%M2ETkdTm*>m%6
z@2%dxdiCpFrLWoe<<6AoOucw&s`hl9$VJPRsZI2-kn#Ka>gvs$Z*we*-Fl^tmSp&V
zQra))`F6EZHWdY7VPO#w5no?joyqrnJ&(NIoom<1K0G+MaN)v#|Nhn3&41aUB_19g
zuB)rNFo1(Ws8dB#v-4<DYFe6+toY##%O#v%__wsTM@L4^oIN{!N5R59%hOehi$8DJ
zu;I)ZA9nWk>H6_TvVKn=J$f{0_U!0_?pd>L?I?Uaqs!eD6!?GTW-SX}7jtuUxW2vp
ze%EfXjW<P?Yx+4mCx3r;_ujpGetv%K{PJm5bG%$)WA`SgGw4mX&bqSVk7l;k+OE&b
zTLrY3U%q_l+{WYS?Y+D7b(m}HsY9nvYwtf={rA^b)q8(={{~!N7dzjs)+<niVMD~4
z%*)FRWd%QP;OiCm6&e^A7#0@x_4W1rH9w17eb0NHon^|+&+qT)Sy)n1^Y@qO<%c0$
zZ*Ol851SRK6E<Bhwx_3OMqfwe4m%U47t`0o?k;<JYU*sW+@kXGbgOBbHkZD>mU?<x
z?#)e0{pZ{H91l9^sUoDStGhB})v{$`qN1sZi3dwu&VZ8Tug>%H?fIq6bPNp(@9(Qs
zQ&aP455D&A_xt^d&TTt(?_M9fd)n!zMzVgcY3b?Z-{0MRc6PSAo12krx7UHNyq18%
zcXxIsr>3Un<kUPpHPvgzJjMg3PHEZL*x1?KyLt2H-QDFr?ZFdE-`)~s5Yve`P@=Gk
z$=Jkcg-2ao-Me@1PEXg5-&<9hy3Q^uJ6qnSVnh7?x^LgUE%%$7WF@0!Tk&B*;9@tI
zmox0^?HEel-1w+}TGA-RqpZyA^7K_K3EJjPFW8NYjoo`BK7M<9`|t1X*Vo0)epwK5
z^zQERc0So#o72ySt_~}-kXd$EGyC7af8AobQ9Fy4YOOb$HhZ>p?XNEvE?iJ{@4Ip1
zhJoz4c+j}miVfA@-$h18OG`+!w6r8t>7BTC&CJH;&(qV>*R5MuSXk)O9$a?m^5yC;
zFE&<xe>Y)*K&an&h6T%(fignE!6v`CR%>H-Z;QF^WC&_(Z_K>Bj8E1|CuYZo`}P0V
zp3%MSdhzO2UM{Xz_xA3dJNNF*&FM4wmh0TvReJi;rA=XLqZAbtzrMPfY&9qT((3T_
z7cN{_?A~AY?vACY>C~46{{*%bDYtw|NlmS+u0DS3*t^@?*YBwjFxIfLs=Bcu@$c{N
z@%w5jU1vqCRp?qIrXRPb^!2r~v(3w2Txfh*P&Ye9FMdzO$Aiu6-{0Q8zN<8Q+2NnL
zpy5(R2Ahfx2X5Vpa&~_F>+9<o^Xglho1H5vek^wHkJ^?qbHaoJB^Ijsii!u%o_+i5
z?CggRA3im-WoDRtHf~43!)>{@+jt~D?f?JRUvIjx?8Wy7K|@qeZ%y2se*W9{@5ep$
z>SL#Bh4#tY-`kk%9vywVTU_5r*3XlRk8j&XQ26*36clXDx;lgJ`T89@ce=W`9654i
z$F5!b>i^GsSy0Ektw=d!Wv9!cz{PI16(17*|NEPkmgdtQ9OvrdQc+dKCux-O@zGHg
zPb1lL@e@5%{{Q)W{^sWN^0&91K6n6fciXbe`uhL%|NoYMes=b0YMaFxXXTJe+qyqD
zjvZ?|{II31Evf3Cw4a|}Vq#+XyE~b8ca;VO30;2pXXo?t^Kak2t*)x->gE=<<O>f2
zi{p=9zvkIg79H!6T<q4XwdKartV2A#0ji=B5-*-TGb?>{<?L+p)hkyT%KGh0Pe>?u
zb!BDP+Nj9r==i-=p!jo1NLa9X_3IN8mA}8ey?pK3vn3Ww51l%trKIGfA#&^1ExWou
zGhP<dH68hRRbW+4QBl#Jy?gs*t;;?=@iaF#Uv_w<1xHgsVd2J2n?8N{a^}pLoSU0e
z9R+;Yzk0VgJ^b+Su>V}E(my{ydBIrL@8zS96?b=+hlhvDTa~O>wrttlxw4l9MLAzz
z;0V<_KhJja=FN*WUOhWIn^)Q_#cB@M3fFG2#Kgqyd3UpJZ_6zzGP?Zm&zz!HrHu{+
zudl6j>y<K;^^=sesrYbpb@=wYySH|izdt|U-dOfre`1^43Za|3%k#U%bZcsAw&mVl
zwr<_I5)09=tE)oK&#`>`;zdSkYU$r!rOOU$3i)1M9k#ZpygYtW%E>lf=^r(Amj$P4
z1_xi3HqQ%L9d`E8B_k!Jrk4eE4ADCUSBVrA7a#ADEPimH@$vEg?b+A+UKVKCH7P`G
z$+);B^YX*P?cFX);o;$-*&qEpJzw74{k@r;zp1(T?c29A`PwB_Jh$cF-{&{i>Tnyc
zaQ9I?Jw2_jW{I*=3%lMQ?H2#`?VHj>4IP~|o|8evUgwUTJMZo)ef{d{YFVq2fR#Jd
z*T?Kk%FN76O)dTT>FF%fY$I8|!&c?*WaQ-PetvptmU~M@P0dL5T=<NIf~y|z$=SSk
z^Twv?%ZmdC92y-0HnnyOzh_<Zh5N&2Yk_ZXjf7^N|MvUm&!1~;r2anqz4%emx%D5u
z+Ws=xe89}}1XH`G`+Vl9f<jxaZvWntzG(lfRS%uNY@Dk7mk9*?)jTp7PSg}$J%2ub
z>HUc;oV*UIc6&D!9ga+Fbe@!-`~QC9r1y8SQ&LLa-LcHh&eqmmJ=;8gTE)-(-fEM5
z=iA-Ae7X4fxw)U8oz;H7c)NGk#EBPwetsSs8@qPx+VgX*kC#|f8aOM5Bzme$y13Z=
z{A~00XJ#5JyY*=F27g!`-Tx&|aBuDLE>3o#73*)me=oH$>G!weQEPu&UHS1(V{+TF
z0^1N4)?$O*tL9FRE%j9>eJ#DwDem9LBmR@x6a7^fOl5XieN-~byA!cKIXqd8k>OSv
z-yGS-U3zPKDlhF`{w*)s_V0zmZ<ua+3hmQ&*uL-I{CV3gR6Nhkx39mv%=hoFuhH9b
zB%43G?5+KM?dZ{`A0HnV6cp5r-j<@pbM;8l#<X*DDjytZEPQ;-cb18#)>I=|!L*DP
ztrg+x<18&KmMmS`+TI==5i#Xnr$^%~HpdH}nAMf{pYheLtISx=!N4LF-yZ4h;<PYe
z%J=34Rk^{FPNy}ys7%rjw!V3!c1n9OFT(}nE7}qJTr@<wK25e)UC5x(8K9!Mp<QGD
zlN|!}{P#Ho85quAtzIW9;5dC>NcZOWyAONbB?pT(q~AVV{OrujD|?w*Q|#`RUtiRn
zdUgLkIoGV~>tZ7!Zrt3QZe9Lv&a`P^Rq9%&9v*Ig`}XbcFE5MV+%T;D8}w5zh|$8z
z%FV@PPxbe8m7kyK$L~uqlQ?sMBh<ms(Xp?uZ{ECl)!*M;y>dn8?qts=Va8$S3!dl4
z*9ZPwVEQggAp6rBI~MOL;hZPAKF+quD||35^{`Hre!bcf#)b=uVe3AaZZ9m@kh$&G
zoyS|2hl_vNB(wANKXcV!Pxr8MQ5#cF8?c`~x?E2EILq%Zo3>@9*)3YXcXD&@|6Ue`
z4%_W#%lG}i)TuX3`1`Tr?d#|5m_F^7Z*-|jC({K3dE0%SPj<<kV*6|!-uL><`w8>^
zaO*CdAXEHrZgstJ+{<r5tK#<lI@&ERZJxJgj``w{^PvHLetpxYU%#`n*nhsAZQY+A
zM%MY3OINS1{_x=7!^7>?mX>dCZf-y50ZO6^J(gbj`1ttwIhLQVtPECm@6(xFd}>pr
zsTKdHK%4mbcMsDWt~3|7%)7hx^Yf5D0-F*a$(`oaRy!L}m{NLqGV}5|Az`tW(+<nq
z*KMeF`)zCdeBZifmo~;8_W6;<dwd7?>@cr6$NVI$liMy&+*uqfwXNu4;^x~rw~8Je
zoOtq_wg0_oOI*Lq6Sb^4bwe{a{q*{^Rx;uFpB9y+EVi7z{JXglBSX&aUe=?%wF?Uw
zB$tb;@MT7C@0hN@dA~l%#s9WZh05u)&EooTd-Cq?nrB<RE^6y4o~S+Yb~PEv$;x8g
zhYugt)z$U>T_W1Hef#&F#m~dmL>Q)=nDFLJj$6_Jp5B0?AHThgc6UF1<j9eZ4i0VY
z)tjsfx9#!Td@hdVW%-M+JI$ZhALBE;tX%Y|PQ9|`XUO9E92N$KC8@I)UYWbXg44tP
za<Ekt^Mq}+?l&*4pI0aHI!k0$TF|ElpN>}DvizQZ$LsRzzwg^_Ze>wnxUtjZ8RG`i
zhJO?0{`$truxFQW*>~0Ir>E+zuU(wrxtUR6|HoOei+^_MWu>Rhum1HQYR7JOV@JWz
zCuf%beiyYRH)!dT^*(}DrLRg}Ut9b5cz^lZTc)<Qd%yT?DSUj)EccerT&vRG-`+||
zNi8j3#C7t{oj=#sM$7OWKXvNWy}i|`@(iFn+!4Io&(gwzN6O^IfktM~9B*Zo;Ryzw
zs`Zje*2b$pKL}pTX2$4Qr)?CySL)}ZU))cdlio_XF&toIV930ew08-cV?pDeyr*$m
z>a)$(-QBZy_wL=*zrW@2Zo0hm)ZtHc{spN;oCou+-o1NwuRDMF_iv%9PBS+hS>O60
zs`A{D)Y-o)btDY#EL@>gRCMNN`P$#JMW^ofZr`}^<I~gAUte9Fo|`KxD;xXhX?jY^
zmlqe64GauaR8k%sXjF}>Q(d@j-M+%d$0|QRD}8h0<Nf;ox{u@PW-Juc`krERbBX6<
zHUD`ve|~)2n0$QN*O#nWoX_WXGWbN5m8(bWQJkE+g(G1zqr-tz<_V?JrBm<oRVpuG
zeDL*a@|=6IS)Uu~mQFb3$snL^x8ISkBzw)uW2$k>(-;}%A7*7Ri%)o&R-JloiMIYF
z4bMx0_bvU8=<qQV++;ZLFpHsK-a<~@*u9@#)N*<xonIFzKhe*FkwHZ$vCBYzVzx<!
z?csUHvn<vvy=iXhp`|@HwEs9;fLS5GR7T{t#eEhs=RQ3>9lk#9?%L?>rLV6|O*omD
zn0R)Usj;CUCnx91@b&lRi+)mfez~G2+Fn8@e3SD_l|V=7Fg6yJA3x`w<8G)b$eTWY
z{{6NE8k`Q9JWF5S-CdrM@nWK~`<=UYPZz~lr)9L9vdzzbukJsurM><3jT=)wu3Y&g
zJ^I-i&dP0Bxff&~%;uYWr#vpcS}4eW-Q9}0VJ$MZN=%+P_wogA_j$roG*?h!L&m+s
ze$UgkFeJ?GExv7V!(8D){+%rw<~~<1o*#F&+AwNU{?&{B=gqN{;_0$({}yxUtzGG!
zyT4EFjWXMu`F-bolh;=+7;j&?T|cg)PD-}snU!eu%eEU?r>DGg<YvD;<2KKpbi3P!
z@7!1<`ZdSXyWKZ4Ui;Fnx9t2mclS3tHg32)FD`a_Cc9{LX7+`(B@eqR%YCe7O;6H)
z_h{$t-S>ACCO<zn_w;oA|GzJZa2-9=%KiJ>+t<&ZZ%;bPr8oWeao4{m1FV9!Pn-B=
z_rW4n>nVNL@9YhJx9sffe9oB9ObWAi&d9(2bJN`2%={`#O{T01TN@R=E++HRlFp?|
zQ$0LR+!y)!nuE>#nPQ8V%js#l>1StMUF6EWj9HUURj^ZK!BSO?Ev^exHTHed65?#s
z75LC~XLZ0yp@|$UO!wZms0p27joM%AqIqLu^6|;){@e2I{xWZ^`SBs~!U9JfvFzi0
zvTkl}ssht~?Cg)-|9VSO$pKe+@ni;ukEbQe_MX4kwe2(i`uExE?szK+d1(aw6zfRt
zzx{1*-KW0&oht2r@6MhrTfL|9{$_V&W@%0ih6x<4LPr0^7O04vWbO>$iTatpz(wWH
zr)S4n*A&iQC0muhM9$&U+TPq_Y&y2WSNHa@F{re!nK}7N;coXhn@2aC%b1r2es6nv
zS9s?CuO)Xkyzxs|Jn6-1c_AUAsxL1R54Y(?ZRt3h_Ww`Ae$b?TK%_!tQ`1Y!`qTcm
zr?E1vO}g~?yXxmUeOqRRlND8Zg6ZexY}~dj@9wUzGmX=|)NBvW=UF<Lx#5wU|LkRE
zetc(j)b4bg*Xvza5-Pj<i(&-#_troLhRAetH@^2@O08}O$$DMiRr<Q4qhrPliSY32
z-{1e=Vf$b@XkcOCCtszBKfpl7uBPJFmQ0;6&bkf%zXe!nUSRmJzd5mAK4xZ)v|;AM
z$hzd~ljOaQud^)(G=F(<ZQx6jzYp8?&ud>B7s$EeN&6*935hutg^QxM=arR}9q*IX
z-r2+r>cu-fjmXPe*W7%#onKx_Nl8XV#_J!;e#<F(acl>&Qm;Bldo0$Qr@X@YyO8Rg
zOA=n!7u;pA{&n@@>hi_S=PP=115=DWUh*te=16*Uq*L8*&J4TSUvF-1_TKiPX6fy=
zJ^`l{AxDoMUAlDX)O$RsKb>Ee{WD(uIr8#>4+|AK|9(2H@BGr{3~QzH%SRRr3`!H2
zKRDdJz!702D>pG{$CW<|OSUpFC}@9VcYfJ&YGKK#E3-eD1xB8@zG_3;&mSNA4vSaD
zC0EDDMW^U*2$J<m&&}PNeSICLu$oTnt}P~~j$4^Ht?=m-&Sziqg`0!p!*8poXV+^b
zw|^F1w3%CdZr$lKHUX*q=bujxySU$e%DI*aj8CrqOLD&U>DUelhAEc<tWM|2F<eM1
zyYA$F_jHj)tY%!znF|~XjgGC)yT5($!(+YDS7X>p{$D&5e)N}79=k%l<ZAg7H5)%y
zESs6gslPSy=AWAZR=mll4}SeGZIPRL;8f?o$jy5F+q|{6E-ZN?byj?P-o2%%hno^t
zUkTW?zh&`S0j=++j%KXR@0kDl#@Aw-qU?ajPW|fpCAUv*7vgjcSRrpwWBrZIVbVOk
zi`Om()fLrqMBaMZb@uL4+0z#<9j~{!rDbNe&3n2YznqOm?~1yM-<@BoC@o~Eh_Zhj
z7SGsr$+qgtiQBh-UtH|IG|&Pxx^JlPo4vGojllj&_DNYY9d9&FH*+sLo3q4qiLHfM
z(Z#vbzCD_IvFl#iuV=Yy<Z@?zU-<f+u53}5{~hnkU74%5i1DxAoOtc$v%Nmg&ve;{
zyu2Tx(0i{w>Bx+WPc>|(m#eHV5c-pMxSx~7e&)Pw0Viz^zTeh8foW|~jWQ^4uVi_;
zWA2LT!z>SycOEEwFoo@+wbdNyO-mj>JQ3I{TD3Rk@2dP+m(T4wwz~O0)9l#0nV%Mw
zne*Suop;av=cPHPx84ccv3=tNPy67?+Vbh9+PhN?1hiCt$Ve$J;i}ugBF4&4K5q}Z
zz>HhPYkNvOA3W{9eNw(U`0k#$CK-!d-#_&{TNRltSXo={UAZ*9<rJvJ5O7o3v?JeW
zg~i9zF1DKOEYs5351gFH{rd;&uiWaJ$zIRdKHMz{RAJa1^?8DEqw<#izIU%`e&tF9
zpH5IX(DzG~7c@Gx=HkrG>zS+@K3~#Mt=hiy^8P5^-L>0U#0398sM}t-o2~54?z+!D
zPkqk)d|dV<{@A@EZS(67Y@01okzbkp@9XV{zu!eoxxa7sQ`xdB`jH^tge+U}CH<fG
zU&aSFeCBIxUiE51@wXg~-9P7iu(SPYl+IT)*RXQUv7PC2&#ld3bT~Sfo42k|Rf2(m
zqwo5}z0VxQ7ps;}?l0rX$iLVXwR6eR!?*9=e&zqtDEZwXw?z3}CE_-Jj_*r<yeamj
zq-e;}#;MW=F9rNQsPXW<^(*fdCr<wbQ?E~-o%6`f%W`K~OvnD?A#X2K`&KZtTEEg(
zzqw8(-!LtsC19q~S1I3H1vx>Dy4qn+{vN)XefRG{rHL$V8umww<>ysi{+e@!Nx715
z!^4Y>sozgN<1YKVT_OK_qv6H7Q<C%LY+tB}cvXGBprpEJ_S+vfj+|W+|9XXfFIP;7
z#yyAccjvj!T4&1A>-J@Rebz;jH!I==Kr;?pnijj%tRFs<{rT}j$GS4pE5`T!mVZh;
z+^KBB{Vpcq4X?j<rNsO6!{q|Pbr%BbQ+?lm?Npc7TRZ>Roh?h3U++9REv=?-ciWA?
z{R%aFpqUgY-(NBx=Fczq9PwbG@%w`{?>;H2ZdszFm2CfF#ecUMoL$zMFHIJvC$P6{
z>took_+L3_{52@DDJ`#d_Rg=VRy<##yWXvcPiW-b`YJD)ExRXkZ^EYKA-B`_E>)E7
zdbgs!;Ny&ovHUu=!d`Ff4Kucct7bl81I??~c|W==tzEO%t>?qnuX80Uv@{Y9M?Rb{
z=DO7W_M_KVS5JSwig`Pim=kCGJf4UrrmtNlss<lof4(i(;_u}|%lk5ue|$N0YQD?w
z#kHH{&wsf!$K~(2m0H{5->x|CSNZ4AzxxX7`)Zf3mA&isa|e_4EAgh%jFt=fix!pn
zeT^$my>{;NWf>obDI$Az{cbug-1g&NWFlvG^f&XnLKYA2mfnAwEWGphweM%6|2JGQ
zx+reH^Y^JY$3s;;51ouvZ!*tIKVS6g>15A$^Q<bqeQ67R(!;rb=lS;A%k?vF>8!Qc
zx^4ewY1ywY!?)+h-@l^mQkif&a`&^S)OSbwe%%KT;H%E`Z`yM`R&wvxRqf8E3=EBd
z=UJEE_qWk!6!?>%z^l5@L(Zcm>BqxQoHP6LC4a_$D_F^<w{7?4q&<Ci#J=UOU_CXJ
z??#oIEC14`lHX;F=k443{KNLyu1bQl<F5$u*xZ(z_pJBFarS=|Ex8weduWB8f4Kkq
zYuT^%EpsmYQ~%Jod~1YCT5#v<Wi3K-_ihC47XYpP_&cdQ(7fU5s;Q|5&D<7G{9K*Y
zY;Yw1(~hH!(bIT0zu5a{<^;b!&TlVMb1$E%afz>1XP5pj&6!!=KXdl%XumtN`6m2)
zFk$D;pY?3E?%P)_({T-Mey}BD3G0M!4<htmw;t#F@io3?=88BUog_(zf2$%sO>x(D
zt9Wa%Dc@)FvVYg(#Y=q-rq9XO`#F2|?C+7K&m2!keO_kkZ}0Q3PQGE!yu-QLzULoa
za{I&*zwfZ;qVMv%3jg~(Iedfv{rvg%6&+U(hP}T%w=(|vhDVv1nG+Yp^eqlj1+U+z
z^St)?!)~3;8?WxmO%{3avX&$A*WE7h+}M&E+i#t@|LFSbtLHAtspt29dR?4yY^vk(
zM&1Q0^Dm@b`4)C{|CtO&RhD&CoBq@UpQwBDc4I-<5$9^%Lye!S0zT-osact&&A9hw
zLZF$|#-{(j<SPtLt-a?T9=huG!RIHQZ)a|P7w0p54VT@Oo#*(MU1w-WdHO_R>aynr
zl8?TAvOjBE^yt>Xy$iB$ZF~M=UfR?bFF&mZwM>4AzPhUWN}K6@{iF+Q_fvG9I?RfV
zXTRjr(KWe*fx&wEF&l<$Y!~D=sWS9hR|jj`9v1#RGiKMlU9a!_dA#_~f5q^5IpLG%
z|5bS=dVTvsF-C@*clyUw`}TiXb+e>6mm%Xq{p-K~_E*JB{<m38E_46H2)nHvUw9Na
z*QeOFpPq7H$7zoHdOq$8A8nW_t@uUzL{zViL*e72)fczlzWM6viT7c97#J8{PB%+#
z@@5hU)wxy{ID2d2(S6f>@9tAMdSkM8(`%iWYdhr_JR)NPR5(6Y`Tqa3=5&#iNcsI`
zPh`~umx+H~@l*Qd{J1?!Qqq^rtC;okv8>Vjce}GME;afp<F<;;7&JXIW6LKAp9#gk
ze(o1}wlvQ3)MD1S)s|m(*KB{pI-8jx#*Cq0ht!1Y5yy+=jGXJfMNHLOJLPiS(b%Y4
z-yOD2?2G#N@nhic@cEVQmp?rYe^<xQ(DTJ*);TdT%VVAM<LxqDL<p?;zxL)g9b0SN
zZ-;g~HRrPT{?~NMEB)%@+Dx-4b7D7bd;0S=yPCDW)E3LeUkB|@@;l$#x=!b_{r^o@
zlvnpH`F~At(e&Sc|A2-W|NIGC_vLoI)|1>_Kc@37F>-3!_Ul{R2cIv0WfoOd|EXb#
zwUpOsUwemRVf^9Mw#KhYdaqutvWmR5{aq4=etcJq*qV8_Li%q_`g7J{))w}I>3QDo
z1;rQ5PH~&J)9>iA_?lbmsvm1f`9@W3zH5CWjFqAAPV2e<GE5EY)?R03I4D{EcK<)$
z7XP;D(jV8Sxb6>`@NMg%ck17_{#?i9y|hinzVL6~o4#W!gMY<Z?=+RHta^JmZ}RWB
z+ddm?{aVidNqlSl_GRdQU+WbSr`I}~vVAYBEiSmWrrT6+{fa}mN#3Xb?($h#`0)P1
zJ2#qs-)LWNZSZ3L=Nm_2Cs>#jeY|%xh5x%<n$-TPo&#I5Z%du~EB@o1js3xpqO!va
z))f~`pV#>+?s>ZEgk4{IBNX-T_y7I4zCM*DcBj0K`vv<Mk7N`>=55KjS#)hpq;Av}
zjoFt2g(iBaOx#>q#VV$`NJVKP3zuGW$dQdAkvZitUP?mD@dub0KIQs+VgEVZteAm8
za>Hut8(01Yai}hAaM$u)?4q>O;Q;fADyhE;s}DFDNnXjw33h+KJzH65qQH;T>*W>4
z`(*v++1%V!`ue?`gtYYSrQXxo`Q@g#h1_4{BxdnbPPp**+Ri!GH{N(xF(p9b*R9V{
ze=Z$gxBc<N501U2j(_ePUpK$v*}=DR&m1~08EZDq6I=hF@#Rc~3t@6I5#2P=i<1=l
zY*Wkwq)PK6{^hEB`7hrm!>3?rc<ANf8Qyl^e%1fJ)cx(nMdkeb_wN02Q{5ikIMplg
z%T~pI!P4c+zvmk18BAFABe{OkM~`_92b38-7$$H%j@zU2!gWqS*aCr1-c@W2M)^5?
zD!<Q8&gZN8_$YPNjuq41KML8m(AM5@xvTtyGo1Gf1Eoy9=5>9Vq?Y?KHTB+>=NH>E
z8CF_XNA}(k<y<uVwR_L4P1&azPS4x7Z=dwKJ!}g$WlG)1VOZcA$!W57;lY0bwHw0s
z&vbNroqOgi{}B~thPnSTT-Vn{H#Mc*j;hSxtEOcpu{8hQ^)u<(RbTH(Y{}oY_UgJ9
zDGgl)5q#j~0;?LM@3ZZ%<~hIObcf(lL7u9rUo&1$m|7}bFXxl<#rpgF&C6vQE}r;$
z+N^Y5*xp%k2hvwqChS|DeCMX+?oGGu&bD~qyw~zw@#-^C;ruD*O*tL~o;Z5iKX}`j
zn?H^vK4NoEY`=E;o14w1Yf4p7ueM6sEerfHPyF8DO~=^ubJKa6qkma^`@j9&%#&*S
z1>?$`el(WfmFtxhTmE~^@ukO|WmnZ{{cnB>YR5ReNPqJF+q&IS)t?IjF3ik``k3=K
zFqK_zn`igVZ#Mh2{g22Wl8Y=eU^{WjgEdNvZN<$UoB~x*@wXH?c;9A~%KqOgQ<cAb
zd)}+;gqwvP3=ONM<-UCp&L4Lyu1az9s-?^f3j(j7?fa~u`g%Llm5fKSb=Ur-99VR&
z`>;siN~Na>JLRT5-6hKcTF<j^zDJ+UY_r70{@cUK`riJt-7sa@IeG40W(Eew+kx}^
z!`XiPZ2w+r&fcK5ciFa%*vGqzPM`W3o}Tx9g^UJ2!?SOFVq#*Yt6o1{?o$>gt+&!q
zfQ3_vW#W{HYyb52asGatu{VCQr1YCBg~4m<{&UCOX*|cW$%$tgsKN=d%q;(IvNh(P
z->2rfHJSQ(Yd(Cnjk==n?ETcwAI(-NK7R0HLSS?3>hSfE%hS$UoxNlJMUdsJ-9#1!
zhR`=vJj?QBf2HJSpI~cv)6#B!W1g|i?*51k2M;gPVz@AK<@HS0gPs{IS6|K6dKb5M
zHV;F|t>3R*xY;i+TYGm$<MMaj3=A#j`@3&PNqn}9VMzGgowH2O!b(r^jX-8ve0;x0
z<$Q))ms);b*E*1330n9v_3-MfSGL!p-n;t?tiH19{BAj|ru@m<Kiob1<Bh!7*N5L;
zde185JzW~DemO69=EmTUkB&`cKlR=+<ksceg+Wo<E~j7Y6ya(xd1tb6=k*(6r}S<v
z;y<5x^IY88+uQRVCgkPS8EM_hf4QHbAY<|Usp6TZm^!t}7}=lY%ib+Zb!b@s{$=U$
zX=Xwns`V^Qbjqe3KcxFQ-S4kSe*bjU+GiOC8t-Io6>C0qkKb?!WmJpn%j}XR^VSz_
z`oK3eWbvY~fA8z9*}v-E_r24wc&)%HhEo+=LNh0JwA}BxzDVQ5>(#n~T&nTA!@Bvt
z7lp3jkPq=!6xqC$#in7+<b8Q759{@9y2#GprE+FTMSR<bDMu@23kAx#MgQ$uGuc>u
zv)h`taoxI;HXr4je(jDyl-;d$9dlKk7Pc%8E!ZOJ(c)ye{z6g4eygUwNgYpY82KHP
zCNX%NO4|^#vTxg=eRnt(8ab-2-TX$v<C{fA<(JE<cg5N}p3C#Rjy6whd#vzS_xbJg
zdG|}SV|9O?P+fcSd3brxy-hjq&Yf|d*yzi8X~hB8_D8%W_up17c^Tz=A>&lj!WO>5
zw-;7vyZ7H^th%wQxcSHQ=WQL4oZPIz%XWG-FOPXxeWkuA;L7_I>lYX8IlJkT&AnxF
zPS)-Jp0j0Y<rK{;jcbmac2|z-pBMb)UhAe}!PzP|^XKjPapvzWAHAte+w(SC-G8b-
zNj)GfWnKgOhs$4sPceeWD;%YGZ`l}KQ1`ySp=$rlZ1MQ``1RY&GM-3no%iCR#=6$%
zC-+vr&R%lO{(AkvM(elk7SRXKN#`(Fy<PbI*OAHm)35m#=iRz=`Sp=Uw`@At6jp7$
z|FPoV<GG95WjkfIC%=%{tlK5Jnr)`$x2QK68y+T|5Bs}0JXVzH$+Z6;&7StO{p=HF
zT<#ZlTe{-zi8D)&|DSf{+OO9-!JY0GK&y$SPFz(c8x^?syv-bgDbqQ=oPK;!{tVCh
z)UA2m`}XoMtbf;{dh&*L_`mER_vYzS%~e%Y&#@exJEi1S$oT_&9J#WL4KBZC=Doc(
zh2=pu!(2{=B1s0-=?pJ!oGFVpU6}jn#L;f`Z&#R3hQ`cUbtcrX_}i?9XG%mwSwOwk
z$cMA}q_0;zIaq5s!8HHf235v?Q~G8dZ(I1tU8w2qs@=PHZ;CZ~n!D(Wx4S_Z?{?j0
z(_>rW3~mOmdb|1Fr3t(Wj0`LP{9b6G!eH|9^Zv)Cv(CsEum8L5`EzHhH@mYpp4ng1
zzg;sv$v@Y7ll;Y1zt*RVd4^ng_o8GLyO5{a2Ogg_v2Wg<&H8g`=Q?@Y%1>c?4ZDAv
zUHkXAAynquQ#X#OvfkzSH*cP0m)i0D_4VzSySc-qw&ZCkY~jtmc<HKk|GnuAD&OAB
z`YQM`N``65;hYsyj_beq&->}sjq+Q!F6Oc@Fy!zxg!mmY=T2`lF?Uz$xo!G<!^G1I
zCO?eM{MmTjP&9FmZ>5v^oH@HQFVFm6_go_GbKPDOz5Kg+Cg0~69JywY=(3(qx%}?W
zsmj^roBuej|JTK8Y~r-y%?4i?HS>aq`+GSBw(jOBn{oB96vL0w==_<NY%?AvNEgp;
z{C+0AOzHcZ-3(pU?S1zdqe^~%`+9P()%N`Ri)QZpxrDiVlKh;VU!1m`%)ftlyWzp~
zb;0NRWx74rC!W7)u;%QK=JP)n=}r!R$8A-1ZQ=bzM|9h@4phu8Qqg^OsBTu(woLzv
zuU(%_<Y;yB{KuhH$}2mqw0Bh$lkIK2#bVo^-M9Ph!@=OddHFK;8#V?7LC)4gdn9*P
zR$cA&(f)C}8`S*Vw8Zky1G7o)3=OZZH0U2*$+Av)n%##V@&8^$#C<NWILprPLtvBJ
z@A(JV=Uq#hIKx+U-E5&#A3xmtx}$%SPWOT8$y5Gs>g@OyxGwkVzWFXHlN{`I8(pr5
z{fJ8a&!w$>@u%*3X8kv|_bPwvyI_9W>N<17ZfkY^`0}N<|NK4uE6UE&sQN&c)b#gN
zd+!JDD7HH={pvnPy|c?08+@v_Bt2nTRJ*&ycXqM*n$OaA-54Sj>pt_TTs3Omz^tF!
z_h!~j5AGD6eRqr+C8ZQYmhIEacb#-UPrSzdoLbfYh0?QLP2^{7dvizK{QO3%lFQnQ
zLLBco%wGTW1Itt;+tn*q*8ceL@Z@B5-&rP!20WE99vRK=oU@`nt6B2>i=W{CJkpI}
zTipN8QSy_tF0e3|Ei`+5U_#b>`E9@V74Ks!G*^4F>D%Y(cJX)ngR8H8yq4}(roE->
z7)R0nBev55zqfH7I9}BF_<$zwqVkf24_}|{uKBrX)~qN^(LV4-%%iWL)>nBN|J>SK
z%|CZ%{*9_HU-nJ&X<t(0<Kx4`#AH?eZcpv+GPc7zstaDuy>Vf|f(5U?zrXZ}IX$nQ
ziGiW{UiPoKGiB%StrXV_&%7RCyuaA)<JMw32Q%LluR~XdZU21cLC0@(>+tix@<e;L
z9G~uQ`s=ux<>~FP*~{Sdru_jLGG;ju0Rb1@-QDf|QX~Sj-P544aJt~W*cp9JM|owf
zu3WshvHt(R>+9q1Z^@jzZe3oa`-Hzg9^C!cZuTX7vPXs9?@Fcqv$xmrZIEBUn(*k(
z>ZyDFubQeET2(2{;2=2nbFNfmRD{VH?OxxSNLS~_r?sTERGEG{`c+(sznHP1j90nA
z!q4#IlzlU1%y@XXeS7}>f9K{}*Vyf!at<^o6Hw_28{%o1@X|vfJu6E}R5WyVS+2VJ
z@)IXc7}-_7IqA^M$t)edK5n_++*ennze<~i$!&e|i`$T)!EJHkW2yOf|L;jYy^A3r
z<oTa7IUlb-`cb+s+I4Z*UANe$+nw)mWHMCL{*_o>KK+)B89T#`RNHb1-^>61{my@X
zZ*TbexV@>Tr(N<s51RGRVrWtj5*D7>*TlGFTIaqWC+~)*i7s%@PfAj9{B!kyf!_4p
zb$@@Ay}eagQPCr1n)U0;OP#RDD+ebn{`MuVa_7&0=_1$T+Ef^(3QGQBVR-gsZhA@T
z%)N@YFWp_7xTh-l=>~If!Qi{sY0B|!bGIfNJioT@_ND3NVtE(l+S}z@rpKLEZNJC=
z{`tlaYrWsaKHOuwIrod_j9sdU&lIjpPClMBr~2B<JFCR}&;8V|R7}n1+A#C|<uAJ}
zcX_`)s;1sAV;Q6)c7Jd6^&>}?Z1Hc`PFDzd$I8xLU0Hc{uJ!fXw`aeM_{YWS;N>E;
zt-+m%K~<z{-@56H3SR!pS#HJGFLQCSwElNxjpOk*%6)tSB7w_ZE!N0-c&OFAPsXzL
z*OskYzup(0F=NJyhYuG9XfQD|_xAMs+jn5Uo9Bu>d`<FB*VfeCFUoB^_j7Ssh4{ta
z@7Y&p1PD!hG5cxgsViHvue<fjy?y$0Yr;XMNjlx@TW_4|4cK{oZS?kzj*hw3<$Tg+
zI@Z6BcN?tQ@nlBA{AFUB!`GQ?FD$rOZEtOtwDA2K-Pg}<MO4OB&gPwYKJ3e@y@nb0
zbD~$rUOdTwHpFzU<V(Gxrsm4z<9+>dw!hx*|L>(@%k1#=)z$T}yRWT_wN6ie&MmGt
z#bai@`u_j>R6vEu_xCP#J%2y^`}woEJwi68+v_~Tfy-K3PKZvhVN`w@sW$oSOyl%7
zH#V~K%f;-eD7?8O{eYE;lW1Fn&bO~$_tyRW_3!WR*=D(?N-XwwdfmOb{jb{Tr{c%2
z{MNX%=l<U0XU~L{rGjrqeY^SCz&@p@>a^k<_wQ{-WhE~wZ+kyiGxWEKyoL6og?q(5
zrrdwwcrfqj7QWiAudZHP?A|YLU-#|J&H9^1{yIN=_^_Fc_tmYfuP3Q`?_j;wmNZG@
z;hTxVhWbK|*SF-J*E^T?G{Ns$`TKjk@^*JFTqyYd?(VN&sk?eK?b_W|G(GtI^Y*sf
z+xzS5r|ZRv=|ntu`SN6m%l_sNafXMtkKeoya5>;l&D^F{pWhwq^~syOb@lbo%oW#<
zsQh`ec`{@7vpk+;^Zcow>>k3#m9Cz!-tWCIKqqoj%Kd${?fmk2H#enf&OLhb=FfkB
zf5*ke<>lp7e|xhset(_j+!y9kwx9dy-Y+-Ts`S;Ho14{rXN7!AKhDz|&}vliAt5^Y
zwq5P7miG4FKYwzvJYAOev$RfQr%cDM*s5<E!oKNx`B(k7xKVrI>0a||?BQqMR0?>Q
ziHqd7h-6G;2*`^|*w-&_UsY9AWHIOezu)ICT-XqD9<&<f(UDHw=xtjvFQ?tylqx4D
zx3p?kmx{Xj^ZE7vI(vHd)co8ub*iYc;e+Cg7Oh!rywdk}7PlXNymQwsyV_q<UKZ2|
z#xHPQlGee%V6rOWfVBVpxtvOsWicz>e<?1!$nI(U`&X6AsoyLYrfEemMb-T|b(f!E
z!o+zVmv1_~QeV2dPue_h$BrF4cJ7SbSF>}=<>GxClaI&kudBVgt28by?&+50PLkC>
zJ}lh4*_h$Z?(+VXUwA;P;wD_Va;5C;Ez{y>XZ+?`y}iGG{>y^8cPDqw{hjCi{qOE~
zr!J>3EOz`Ye!chG>lX<xZ+yLTqJHn0StW~*x)-?>w+g=Bd6&J{%H(b4!@VyJ7k)K*
zQ)hH}{e$NsTt_c0_1?R8@AtR2w<jEI+9b&}_57VXHfm~a!NHd=Ui`TKe_i&^m0s!!
z3JbEYuPc6j?(XLF{<b!@=7|aZhAmo~)cxmud3pKxwQFU6e|=qQUR|(qfr`?^mbFvO
z!_8F%PgXL?HMpoe<g7TQoAiLYVnK{#)X}9&mh?y%IxP&4v8&k;wKa>!O-Mw<B=Jy-
zX7I8fKYmPc`@W?}*VeZ7&W^&)&dzpz`LNdG-y+)GRw!j(T5|I4?sCQjpylNoH~O@H
zw)U9k@cMO-l!|`-owL*bGoKXZs7TrAaXv9I@!Yv{etv$}*Tp8A*)luqult*Ke_!pd
zFE8WvRxMq&%t*S&i<6(fyr{^?)U>y!C!`e;-XB<)4(_Y{t>!=P&c52;huis$W&NHy
zUH-Bvbafk_?5^zV>r}m`O}S}t>(SM9vA?gajlOp6nxCJadET7_E1743B3nydUQ%>!
z%elL&^zE&!GJKnh>*`zA-r)!p>zA{Ab$NOJ=FOjHnPz*H8YM3I^768Je*XJAckUc+
z=fA!pu=!;{-Gf_Ovz^=dzJB;H!#JI9cC2Unx0)Xx4qm=I`S|0v_xHz}E=%~$1zLhC
z<~!T0x3lx)*|WJ<S6y8lzJ3PZ@}};do|3Y%ZQHi(D}3CRwJe<><MOhze}8|Mx2?Le
zE_U~h9Tv+DCmz^Y{5&Qm#;W{X&C^q&q2A{i9P}bKG=O%3q_%)hn`l?)czC$|`kKhi
zg^!Q<xQ8%aG})VXcbBAb+M632li%Ffn0hDucR42u6F)!y!hjcV-^Q*EgRF23l$_|H
za(TJ`{Y|N-Hzprv<CR)6YwoiRQ>7Q4dhze?@7iBq3X6-Qw`Pg<hhKaBJv8Cv%a@DY
zdVl@-`FWzUd)BozCrd2$E2^n2tN;H`H+oyl)~ry=*WdpCPD)O0=as&;z_D4@x-26*
z+gR3bV#u%4vQ8_eL_|g1+g<+t^Yim-qqo<6e#U$GVTi!?{QG5PWqEgYd^|hb{B(E~
z8-uj;?En9M`|F5JU)`Da@5sHq)wj3j+b14sS?u1ws;G)B2XscnqmLDTe|?>N`suuR
z^GaV`$-J>a@v`95d}*_sAJ6C4GchsM)YOFRN!_hsBe%Z%{XM3}gc}<YXU&?Wy7k($
zR{~tEi{1O}%HQ4j`T4nZ*_#t37WSZ_-q12bL&MtN-`-wWc=*%P)1j%XONyR;etv$j
zd%vHjCnp!zs?gP9mmf}P5Z90U^8Wt*vbVQpo8?{tZCjc%Nl;kWxaP-)Z*Om}kKVp+
zPfx1G^!P6VPO2U^_spC#=gzj=+u(Cg{49lqg&)6p6SF>U?}iN<x+czJJaFcWj+ND|
zBS(&$JgKRr)%CLA-?UBd;}A`pPm|^`f`;c94m|(-`}_O(lP5oZ`ZRQX++NUW9<qTy
zH&lLp)+wx>cV|c8rza=1jHEq16+JfvFZcWU`ntSX&Ws41vZA7(v-i%~Pn|UB(Zh!y
zzr4JBdb+-O{=GRb3+nhmOYQgVs{LKIzwYm@vbVcZP6}O}8R^NInwHilZLVi$_wU8U
z#ig&VBwEc0nlfXCMaBgMh8sJJmvi+==-q=f4{Vc;bhL7dPdmOo06Z*u>fXJ7`~Ux|
z{{IgYoR1$rp2_F#dUt>Q{hK#;y7fv43JQu^Luz(sC#PFmG9SNrQ}gf;>$1Z?LyczU
zoSS2re7w)L;6cNyGuwad2G3~lLN*MgrKR=BSYA4F=FF8VQ(kt2oZgmu`&hsH|M&a<
z`^`36JE!j5f4wG!sBJkfU%ar$y0T)&jvdo<qgValuK)fn_WixRN0UAt?G~SHl6k1a
zq8>cvX)h)zd2@gL|9g9@^<#Eyxa4;$CLk>A+K-Qq<M-Fqet8in8u@!gPijg^kA&f)
zx3{<Nt^S^7HBG70Wzk&g@^kZSt1m6_^f~@%J7|wt*OY0~%u-HF=;`U%Tm602n<tl2
zA75PT-ptOQcX?Uw{Q2>!^ILP@-`m?SZ?CuVl9IBrsj2DNvuC|mIf!*%-Ctj?t-X4Z
z)&xh;St~~uW>!5tHTCuN_52(h8Ta?uUVdnxY+z9E@KCF?S&oE7fx_jDAtrWqcDzz1
zGYk@$E;p`F%Gv3}A=<9cG1EAG*Nz=LG8P2~8W=x6I~)8leS)gDiqOv=KWu7$nb_Fe
zIdi7xWr3DL*P>G|MRHOZ)^j_(02MUv?(Y8n;o)JwxmK#Gs-Q*im7kt;c6WETwYim)
zm@Io9v?xHsz*(FXq<*@Dghb4qijB#~`}FkmK!vx}oS^OH@9&+PYkhv6ZT0a!S<$++
z>THQ^dTgK+RwwExQ2gvnVPWCN4<96~N?yFVxmo*3dbXe6xh0;HLECU<o9Bnk`#Nc6
zljJUmidX@s{mVmFhkbu{cX#dYvgvxUlO|7|$+x`4w*KE9*KRQt&wD$I*)JDM255*(
zU6p<~{~%AVz^~lU)nR*!o_d|0raO1;TxVzJnS9HS$XJ!EaBk-dUhW5~5qIsfy8JMN
z>qhVz32;&s))C8oc&Jr3YKw$X%83p^<+O8iBG=icA5Ho=MKgHOqDAZC_s2CQ7hm*G
z)Gjv!#qz(H*=D&(Nl9#cGB@^Ce+Q-e%TJfu*Z=!--u{0?M8uIJM>trRmK|OhW8f^#
zY;5B6V!ec<WbE#;+=qu+laKf5?v<RN(RJ#?i3``S>-Qe=ob=`6M@iGHkW7)ju6D^?
z5*EuPoL0zq`1x(yuwg}o?BxkX@4nal{beZIp>-%<k7Hp`!{LV~o)#@z?4ZhD-6hJr
zp4(}KOv2`yGZt#A<UG}5ixY1Nuso2qIn_w=a^QkVAR|G+`IKQ>QFzFVh24$USrtS0
z612-*TZ_9yK@MCYqrlO0G$~NzXo<p#mPA=d_&;S1Tdb|}=5#F29zBkQpBfK8Oi16@
z#s2!j(F3oW1hnKg@Wm>EVi#oJ&j#joS}Jce!+1brc%K;AV?sTC$iU3mRut}W;}p~!
z#^{~eDrew^Ox^8thzGaSzB@+`@brRg11SXY5O!;q8-CcO&~l2s2E`?7?}!Sey-0dc
z>c}yb|HdhZBONcBfVQ*;Occ|Pt9f}zm9Je`T6*>B)u*3-wy*tF^5TM`y1M!@XXpC7
zkB^S7T)9%kQ_iMhL-h7MHa0dLvFux0GH>0w<<q{*36ugtqE?5kb#iv*&fQb{d)wT(
zvdSM92sut#;<Qj>3#c9g6_zS;S6gzqITWXwRe#Iz^*!4Z{pIUdL)+=)o72udD%#1q
z-6Zdh#N}c|P-fR!>pR;l$ISY_`y>^19*F}9HY;3sB#o9la8Z5r>Q(Zwo<sw=tKc22
zmm+koeRz1-Ql{^dyQ%5cgU#&5(n7zgd8JHNgsqMG*DvDAD`PR?U`2>9Xwk_kY0ILN
zI@4LlA6r^j7)bML3C^20&qmJw=lf@8XTN^^+E}`u3$(5wsI*tgbXDYLHHHZ)H}}`q
zFFWk0did~RXUlV5!NJ9EZfx{vUkpAC=~Uy#$H!NLL9%dpYvA_0*r($9J7eyxjW%C)
zIM5a3nWqiy?e2bleLia0_x4y`-nc;M@zbZQ40m^x1_uWxTZx4p`S(+gL-EVnIdkUZ
z-rhF#qs*Kbz3ow3v$C(R(>*<zsbTWu;L16&u3A&`?(PaznrI;FQP|?7$nfIDi#3s(
z+g?U=End9%@zbZS0xpvC#nW{hIsR^2?A~9+BVn*)TVdq9kW~R27;dCmJ3qO6X3ZQe
zHvXjK)1Q{8KKk_Zw5(N$MBw$7mkxDGhUV8h&g=hv@;#dA{*j}{j?J1mbLz*+ji5b>
zxhB5bO%3M-t{1!9IAvG$eB<k#-2SzpdKXOIMs0RWt>`~>p!ZCUTh?aKdZ!kFAh54F
z6hlD~L^2ng*;6*m*|X}K5qqc1jT|w~t~32N&6_tn&Hr22*b-^}H&OQL79ru<X@5@f
zJZPEhZ~DAt$yN3LKYB#xCrTf#-&O3v;I(K1v-eIL<Ehj3r3L0?b)KDh^sRa9{bL=|
zPIEq8bx*~ULACDlyx`M4tLObVn9lk2wCl#*K4v+oPTo&g_+5_wKilKebbr6$CxN!A
z*MC?}x<8&i|Hs|%BO;*XK3;7~N$IDas_ffnxM%OL!rO(0S1!pLuD@k)++J@&ouq1d
z_QoqW_s_7}Vs-2DndsG=AysYZZK12ru6N_;XHT;U{?Oa|vi$mF{`)M=R(n>d&g5R=
zZr*)lmrb(s#g(}yt+y}Bu~uMX&dXaPvL)ix{&3wX^JBli+%L0l<3^pF=|;AP-`|(G
zGw<FjQ_cpN$e2AUro0Rd23J14Tm9>&|5s%fj;YdD{yrCB`1;ZEtCCLgUstY%=;d+u
z-gEG7i*%m0;K02m&@oJ`_RSm&1}o-n?_XdRJ%2)a*ZzOE7MR^$^X>XvRw)OGNv5Ws
z>&)J)jqUbr*jF}xzOH|yhR~r2$@ex)N-3Qt=U!6g9rmR5<nkPb392?z?%PhQdB(as
zkh$Ugx($tL3=`6}e%9wpyws2!^)<WDb#L>ln^Ro)90EBvUg>`RN&Mo>bbn6yf9kqF
z@`R+`EIWIHZPo|lE+0eQojV`KOC{HpB{w)QGjL44wS<A8VZ&FWR}W7;%VwH8_v`yF
zQ)Cx1EU-)4Hm{ra=jCE!UQioQwZoa=!Xg<41zG7y3`G+;7*2#G_xy^!EHr8HX|Z?l
zGwPo5*8f<vO1EdJXVSin>t0-H^EtFO>-Vy*EB-RfwZGo(JhHsB_fD_s`y*LfSxfjB
z8jd{)-Bo%uW1ijYs=tNDh3`I;{Sjp&#h{>_dgq0V;99#qb8U<?cGuaj6?^x_c485O
zg3g!gzSpm)9-TNnh1WV`g?i1c4Hn_`7tANB?3VQ3FxAFe)W_Iw_T#yt@p|GXf0TAf
zGiEX@@L9^h@`1sjuIT3S_v<#y+ivYYcgpjCoYG0>Dg^8^S@YjSH6Ob)r^LB5?#u6)
zx4Y`u?LGI;YrUJgF7WgAkHvObB|jz!&*n3k_r_0Mb>gvO69m2{R{Cu9eQf!`^6dWw
zv#wh-TBXd7ol~_t@~rFKC)=5V?3oxiYnU3IeVZMArgx%ADPR4mw7tJ}ZQ@aM5ASZf
z)5F_+k}*$5;p;Npr}z9rHg8pUbna;UwF~)q9t;v|XU<n%y*P5NdUd5<;>o(R|8v(S
zO`B>`x5_1C+k3Bzpb2O#Pm_kK?Ukl~%4b(ZoQ!j=y;CF95ydu7cjb)B^5tx&8&+RU
z7dU<B^og@3$vew0y(#RxF3c)>?6X|l<iG=I`+q;Ew|F}-t@OLw{OLVEE$)B6oNZXZ
zsB`Q^blBNDdo7<HGIrWKZ(eAUpVfs4Q=jd(e0!;KV+Y%`y?M@7whvN%J#A;%J+H4i
zmVx2d!{wUNDk25aw?3RK?_9hr)&8QcNaup3ve92`O7HHua$KI%?(+3MN3q5G_a1tG
z`Mv7p2~#)bWZb#v-m-4zF3;Sx&#pf0KfgpTLN!f#Q+GZ4;baC4rYj}S->ul0T7Gt4
zFXzTDuXlE^g{qe?PTp2O!&W!%o}ocrT<(rv%Qv_%?0oWW?UVxr3|>Y5Esq=zwtR2*
zmg8?#!rI28_a#EvtWw$T1~Ht-dtYZ=vs2{CkJ77l@qJ4jXIb6+etFv5NY5gsloA(*
zTf&06{k}7HeYm{oa1q0=VlIXY3_G81Tj5v525L$Osqd3kOa7d3;+M<@M>Qsc4Nn;-
zxJ}!YE8X?}&Pkr$w5!Go9lg348C0Gti#V*h`dEvhD1XkRd$w~|8?gP_`(h~*Lxv>-
z!}3g7h6meUEDN1~YFEAbYfl-~g^rVE6n_u9b}*QWq3rv+m>~D`YkyYhW-~RkMPJ|3
z7J2mTlD*p_cm4YE*f{8iql%@WKgZr@Tbch_e*exHANGN(#L&X`^&#_c3(sj&ct1_K
zQ^8tl9mXhfp>R=z;`NiqQ(W(#*PiaQ@qNOzsp$+VygOfR-uvh{U;XyiW}zGMXLMT5
zXL-4sg`r_bbQ0%=(h`;B&Y}y}U*9eld6l8zO3wRJ?=E+K{k3b&fx?{Z&-3p}cur7c
zozS(;g)t>aq=6?QhJnGUpTWvi@8s(Uw+)^F&-SnBmh^X#+3gpzXl7b&=Cg~}m)0z;
z2lwo}B>ujvI>ps`a#7wd?P=b7)4Z<kV3<<yy=dVk-kmE~&$47>*b;H`qWiKx@{AL%
ze{@%4cj{MGc%mfLz{DWd&Rg<ld8zhP%_+CfO=V~J=`YCXeD|B_CjGbP|1DsM+_vOw
z!rfnGx1K$Deq4m1AnOG)!|}`oxBi{D^Iqv;4d1T<1^4~_(RUAizG3kD@>KKG;?9t>
z-@|9LFojDnFl_oW?Q*l*i`^!MQJWRN=CFx;=Bso3z{xOc-^O=`dzqq(8`%<GOEc|Z
zJ2CM&!-ky`gr>~A5WfBBj8zB3k1ktvyCP(A@XYS@b-{P^??|j!|NL*D&HaBX{^Wj}
zm>nusY8+$B_h?<G<JJUODaDYYD?2LZYrf}Kom=I$AzO0o%oAPx)0tPAm#>`Fsdn?!
z@@4+%uTEAotV{d;jDcZUY2*a$UEduP*@O0LF=#AFVk+74YYPLzwg&bC%Y!d-EQ<RY
z*&Nxj@YodIHa<5lOV)V1M9$@juhxXm=bLw$ckcx0!q=a#ofer8RolkIa78L=E3bLs
zy*SAi(&wZ&@4Ft#eYjNd=46YkgfhMrk!ER+CK)zQI>p#9tv2kRhWDS4A6%<HH%_|D
z;84KH;M$gXRdn&u+t2q#mo+oghItjfSEyO}SnJY$&xu<;#d_~mJ;yk&SD2e&fnL10
zivB#Nh7YG~oleh~EjDe_BOBMOO~&%tyIBK*6dJs<g!h*uKRKoFezmh6uS0=@;#3BP
zg=r@?X0DjPtayQqp^2B_fKkQ))2mJ2+b_jGE#TSWBCa>}o$LSFCrzx}Ge8HWp0la^
z^yJ@FZQ0c?YBpb86TLOl)>1}n_daItk6|J;K6%&N7H?Y_=KZ#3w&yEesf=y!R;4bT
zyfMk^-RA0@chq*}?&f$i>-ClulWTK&7qzZ0<ITOa!?4@qtY!VQyenR(&qXeemN%6Z
zJK^!@aj#67tmEHhR<>3vKhIO0H$kc_>r7Ji$JYg}l48duM{T?j<61LA#gnO}?O6uD
zqiOE6Jln0a*lsc3n$#JXsor07`^!V=YZo(qSejb7Z@zfl<H)YHJZyYVW?6k&EjYjY
zhv#MP{`;On-iw}mIkEBasW-Q#O##iJSqb0qjsN~TUT~Z9YW^Y>OPw-V>22Sh-*M;s
zGP~l?rGKwQ?NnA-U6}VIYvL`YuX(u@QZ;$DW=pddzD_%t9Wj5&{N?#y&ujjY4Y|Q%
zy1_hcqlL}ywn-mmPvK!`xLv$ypR=r3c=y_GYG02?JHF=Im3Nl)U2bH)zH07H7KRUY
z&TCgEe>*4o=*jdY-r=v@63(6G$=m<ZZSJfIGGh0yb?(|4ye2vPOz+2s_0}f;CG{^{
zM>%HedBTpUYU{INvOB9xuVzb5x}_%L7gRQDVVSe>*>BG#%|3XkB<j~qm)mjEqzV?U
zco@w-Dadxuo-bd&dcS@9%dlygcg_?mrxlYpxVUcFCf};M9JRl&=fH$D3*Glu{QH`y
zpLoLTYOu(iAcn+%*{qpY)TLg$`J=l;??n8b-`xl5ML0q7i{otmKXiY%_x#M)N4j3L
z_y7C0B=y~;rQY7|Rc|vQzF+76`BGGGkMqX&5BxWtkrI;J7}7X1h=bYJ%9NYIVFBZ!
zH;a;fyp?tim>uH3c#7)t`FEF?9oZY~cJBMf%fWLz?uh(4yqIyp<-O|94=YQVWPIq>
zU%Lb}{2wq=U}=DU(7u~a%TI9UYW_HVrTj{H^pllK1w1z<hG^OEJ=9wMW4HJ6ijRMG
zf7-8dQR$NLpPRjkRqnOdoudW6^D&f6dw1oUGUwMd{fE1J&hvoJHfqt*{CN8N6r0z@
z4~#-zD=%v)&1eyDnmVQ8`=XY+KJ0-epoTHY2gM$DR&M#{;+2<Ilc#q%aLTS9F&lm2
z;<AnAYaOo+VKXY4v9kx%f7-t`@o-yy(GlgiV^g1crWWmVQM()vtbKZN+S4aIk|ry3
z&PN2jy1IJ0R``^U)n9gNbe)>$VRH5AOui@TEVpj8wY8<z$AQjp3b6xCc>VqLRa`$V
z#Y!L~>D}Gk=0#6V%$Omuj5#py>DI4oiY)<^%L6nxni8zIuC_`_DYgh0i*T`aw`Q*J
zsK^v>;xJ@3F8TQ3l0u6>5?k7fM-P`eaws0*IdkFYVee)Erw+*(3%lE=vw^f3IE(Y^
z^Ez=TrX{rL$;C;v2soWNz!Ph;$Am+%Wk#dqt_rFe=y=%#)WwImr)v>tpap8Xb~$|<
zoNO%1uxiGVpqDE)+HKEsHIO|aFT`<$d3*k*ty@#A1X{F=K*7Y*>oqSx<IR<&>2m_^
z?szQL<Z2*$ZoYt;ZRXu=mnS+`TbI5%Qlc~UZZkXov0mxr>(<SiI(6!-S+~~3?w&Je
zPT1Ng-Pm1IUS7Cx{}4~C&6x`;T2$54c5QvDvi{PZ!2eTScK*G6*(6JAMbP&B#~+lg
zo?cnmn_xA^OV`}oT<KAE@|oG@`3wyTr=FVIhj2jRPxa-^&FR<5`qo(OeO(?ITX%W$
zBtPBiZo%iv|K)$&!`eNgZ)s5Wy*)b@E=)YAJ8RY~<FqpgT5=&xkoeUyOiWDd(_70E
zlRkUPj3XsFs?G@k_lvGPZMds$yVnggfMhZC`h|tgmNI^z!HCi~Hzppe2=F`zP6;bg
zWbEtq6km_~TAjBx&$QuXgqKkG;;hSp&-L8*zI<{|SzW#TWr5bphlksrr)M-NRDF9B
zX^^=16E`>!E)aUK{b2nOjr(fOej$drXG?Tc`_Ep^cq8~l$@%ihtzVb>&!54!+$ihP
zlFr7)#!vU9q@?2F;?5jv_Tqwgy}otX%lc*KBDN$Y8^8CCGS!*Ew|vV6kFw@CM$292
z%*y=o@9#^snsX}S$%%<Ja{LV7<D^tCKU^>gl6)`BpL{aqFFT)Q!7I1T6Q^JK;xmJ9
z`IW1g=JAKK)~s#*ytn%MjJ~CpK7RP1z|o{~(y^KCVR}{T!T=4ANhzTGWp4G$IWjVG
z?d{UOntwNIDh%9X`(GBE>e$#k-Sha?yj-ieT_v3_BebU9+M0cRb$I_G4S#?CWyy}J
zZHqMi><i_Q;qU$D5)?Ek`nF$C>~6Qer~j^4bwBmSk-4iE-Y<V3uz#zzjcvQJY+n{9
z7njyvHa1;dT?UIX&^*v_uf;W{rl#`p^0s?VeJTwWP0%j?$N@h8e5&!rjT_hAD!E`L
zA9Xrc>+(heHS4EGFaNDo>-|2@KA-RM#uY7d=FHiceB905onOLW!IWjfff^!_k&(A<
z-C7!G!LX<9ua$|3iH63Bg9jH*@oSu7WMmY#r{ZAJMkZ$Ft5>eH@yVV_+L(2H-P%7w
zZJ!d`-1e(0DIMAyt^fEM_o~}GNmgQQo|`7kfAjkC+P#;*t(~n^Y5U;j=JeWMUp8*q
z6t%x@FL=QA&(7lK=X&Ms>z<vN8NAFV(P~=Pnah`pzrVYCN<aFPjEV?X`uTZ#Q%(vk
za}L&>`YW0B@4DdSex)Vv1GoPc^8u|1Sd;az^j}iNn(FIuJ9qB%X+J!p_V>4~85a*7
zKc0SjTkcaUTjqw{yQ?!YUMzHOuln-h<Im6O%MP#H@ayaA<ttat1OrVGu2!doZEbGT
z^<roAc|T2%mHNA?cedI8{Cktsn)$LNqm5*{xdH<=O<%6|;^O=DaaPec<r2G&KKl6h
zczbK>+xz?XL(X4GS`zf~&CSjFF*^iAL|R_Yp31f}X5He&%0iu=wuh_^)78}MJUAgl
z_}apY`ZLXvukBHNJ*zVI$d?$7?k6+8a-FzVv|(Fu)%SO@mmf}<E3O}R<M!=t7tmhW
z&?R4Z7-ICwzrMQaJzekZot>ZCc%^6XEoVyFd=s>t^8NY)Pm4Bo|Km?=bF){Reww|C
z=XUv<Sx;A-eifQ!EZe80#IeZlcPkTjel+{@-{0RG%lbuHRD4*ldGqHLfs2`#nN3Yh
zX7DW!DS3bI@2{_~cbC6^cX@e#fJTbdoUUV+FB|_qr|2nU{DnVJ+g!`9tZZBL_Ivr4
zPO#OMPn0oTb~v!~WPkJhd2@^Yyg0ZGbdH*L=0pz_VRgSfRbOBI`}<p0N9&DF(N7hj
zlRc8gPEJmJa<*BisYgpJMA=;fckS9085zlNVMF5K^z-u!WhZJWiR(lhDB<yNRaIBt
zy(MS%IkW5w+1lM_Q?2F{IUL^5c;H6f<v)t1Ss~fi)<|A{IAzcE_3_*D@6S8_c&<g^
zp(9HkomXOK$T8cUe7vu%t*x1z|Js!+TQ+Y_wpzs1bTkRHHsoM4dsbGKaoU*-&`74>
zN~N5qZuX!{6|$q<SGlEM&U?)%=6Cx0>XK}0v#&Ofty9lUT6%hB-_jOY+o~;<pPwCW
z<GsB-fBv#%YL_2QS$S`7b^PwK+|$!^nb~*(!hT=451PrDe}1_})`J6$KF1d=+Avl6
zuR};k$Z|c~p6K(d^NeS_j5wv*Y4GyoKD)hlqVsNUS;)jQb(+`G;AK8L3mzW2x3^lm
z*n541&OEc+TWh1Y_sLp^Nk;wHKYixRpD!*ho^(pe#CLnn%}riYwbcFR=}cr|1f4SW
z>(?*UP^rfT&WjH<H8<O~Rn7d$bs_&k-^++ohC3sEy!nuz9lZCkBm2)9JJ8`!+w<>#
zdwcu(jT;rGr|B*=pC4{n^yI|EmzS68M{mn{a$=&-@t|c+&d%)oaycm}Ax+)I>MANL
zR<E9Z#lW>|(Y?LZ+l!z3m6v}P(wtB@W8p<}CZ=tE^Y?D}c%b=~>DrlnOYZ~+Y!X*L
z_G+_T%-)i)@bKv`3;rEh8NA%BSIQJL3H|t(p=>u-<;O=y9~^Af)z#gecUMb*161+`
zifk=-cu3Z|jD>||MTP!}6DLjxD!YAobF=vBs?eCco$qaf-`&0+z&Y#Ze#`&SJ9f>y
z&gyeswaK+e>dVb}nj1E5EAHy<PPUqJUwQILHQ!k#m7ktGKHi^PrFZ1a866XoDJq`P
z(YMw8=NZWQz3l4m&%e4V)WyZ6qGCta@^pp*@GTJ)-`~Z4es(r~U(Lx9i+b+gMam2e
zJ7erpW?cWsE$D0O|9NY+fvn$AM}d~Ozt=2<e(qvjY`rBrcx{yFGUZ>Y@9*z_e`V$5
z5G~unM=p1FmCodI7cI1yqZhj?BO`-H(&)&60}B?}&9g6k6|z0=E*r1ZljqOF<Jk7C
zUb*t?ySvc~19+s()?{5>HEr57ANMcMH%ygo^iX+qY3=vZA{jUG%}?sy|E#ro%@g<K
z_vX#7`gG&rwYAa4vZwv$SQa0<eY^V8lau@Q?ejSv#Aj3eE$737gM55^v(5AM-d~<1
zt|HdG+-GLd$48*^<)&X==|A75(x~J`Ky>tN-RNyormq)oP0%h+ojz&OqX$<$#pZ22
z*36e!rDq#D`<ub*-Mii2H~!|ANwAuxbn@w^mBGvXW*9s?F;RKOy!yb!ZoPSTcd1T3
z`R2`=dG__^N-R_#-`t%3?c2AND_05%3O+TlWp<dX=6mSSp&c>nrcMogsT&=)YQxms
zLDgShO<i29y{T+_)!b=<EOoKkv01y+b}f5bwqA;tYv$^1<I4|4rcIb|VP|pr!hnib
zS62S|`r265ukz5tf?cJrk3Ig_#wVMVk`i)Hef7<KwZ9)6Y+mj^e_z5uCb|COB^LFq
zhaZ-_yR$QDYnF2xkD}+M%FoZ1ZmFo$JQ()w)a!3=Z_D1@9iDHwyX|hkvgDO6Dl<F#
zzkhvqd+k}Z|6f-bhx|I!$~|x1JYhAT88($gKYwuqYxc|A%h^;Ebc^YN&V!O`zroph
zXri*aUG=w|^Yd(ZLASijnIs}EZk}^vgI4IOH#avQKX9PoWkH>2+6%8&cXk#t+qPYQ
zzwzdR!1hVHH!rqbU#;iOS72`Pz2emMw|?LDtlJvF!adKW>-*8#>g$COak)FUnMR2B
zJz=SzxxO<%*_<czcY*P93tzsxQ!9@J>)g}#`|){4$HKW4zE?Roxs4tz$<7aExUik~
zepG)$PS#u2m+E{Mx3iv~wInTe+EaO>%mr4J=H+j_kItX}&Hc%vGwJO%IZ{$`)<q2;
zt+-qRH>aPsD|m3=-QC@?va&OImoc3<ckbKAkCorv+}xOa{LGm%Gx(mHySTga%UEoP
z-Cd>~wkF}vHXYugoi!gH9cAa2yR$ui{;P8Kz@Vw?KdjKny}hmWfx!D?a_(L>w#*JG
zJx_|PBwkLO8^gutmhyGeBZ-$2ABA5!^iq-`r7L{CWAC43uiQ?*JYw=HcA3&k+piPZ
z?#&PP=&<X{sa^2B^`r&cl8t;ZdwezP3*XGU<IyiFspcE3yu9q`_O<7)X$u%F^q!S{
z@N~cGoOz55XRqfOJvnl?;?~yC-RI}qPp|m7U(e9+;@Pur@9nLA`0(M3dEPB;ZO7Kd
z?sn^ydiv%~&5H|)%MSlkHSt{^u@O|~YKN_f*q#^L3TkDfZ5G#y*^zsDo1$~upZE3u
zC!c=m<Nl?1#zO76H;bR2pRxVWq1Su1d1jU}RDN(|WXNGY*Py?-WQOVE7t9P7zQ+ap
z+%1_Q^tJT!rcaxBSQ$=e7+g^Oq}DJ&bDI2V+sPa^cXsVq&%KwK;mUPY9xk7(lmm<p
z1iqT?SF3P(n|)F@_Ghe)MJVG7ovi=PmJ?FkFYmd1@7NsWp9;L^n5?Ep6)Sz8%ENWe
z{$0ko#O^28z0&#8#Ae@Kke?uR{?M^|mUedgLe70O4qq3uapT6!ySu)goUE>W$)$5u
z!jTTag9i^5KR+jHoYv!_lxQ{Qk&3$d_Wb+*-rU@rn3x#5s|2((<oUB_VuFH&@9)`y
zPX)PeEPKv>!%RMgsj@p`-n}&ab?NNyY0CPAraa#7U94<=#m>2NqUG))%jX|hB^fT>
zZ~OaW<()l+b!;lfz6EZ6ayjzArRSH``<~xnXHfXXfAIxBgM)kAN#FJ7-v+eL6_~pD
z!NKcMIcImxsgJQN(p<QnwcM#_@m7ntFd0)J6_F1f9#ylfUyD7x*Y-Z|$f6xOzFk~<
zH03YMU}3ob?@z4yk-+lQ`MNiJb1j>u3keFuN^e-yynkEfX2yMWwG4-DPVuSI`TK3R
zqo)$bqPpd&g^AL+Z|1IxwY$G1^Ky`NXd+i@(!W1H8yg#Um%lG7E!ExpIZW#;_!v`p
z^SqeP&(8MCT8FKVvt6eAtI>P99<RJz&5;g4<MeYXCr@p1=xb~|`0?@azkmO}ee=dF
z_tur|`SHt?wS=xM)b5@$XHMVT-SPXUCm24;S-C7-wQ$cHKdH5OhcXW{hQ<1=jrsYK
zKVEq4WbOM4x%O7nzq|5xk%ZvtaQ}TpyPk&!S{`ODzIU4Ac-Wo)hf?N0m_3KnfhYg?
z=~~6FyVd@2w9M819@}H$$EYyv{Jn{_uNEI#p?Cbigv;CQDvedj&*y4>IJxesi28l4
zhDqn`l=>zyo>-xsuX!QqyE)4=&gIjZxcnOBl}b*s$yj|d{Q5Y4x{UC2PPe@7n5b<#
z>;6_{&i8)#_;`Qrtu37VAaMLk%qkO8)9-I@o9EmxaCbkx*uCFK_MCZ&(akF>gF(v}
z-`q5={`Th7<atv$nm&B}YFqi~$(fnP?!8h|XU{%e;$q(>xhu%S*EjdwomX1})?Yl}
ztg@YFnV{MY71;`ZwgXw&r{xll*NN)>{qZ$UIVWr4C;p<;?dwmJ7d_Y@RmjQk>!CO6
z36b=9CmRnNvAU)${IlYi5!-|C#P4~h{5qz(*-ikh)A!{1%IToS`aqyJYZ61ia>46M
ztqT87HTl)t<jlnIqDJ#^%@QA`mbJX)9lX{r8{Y&A>K`~Z=djr6^-rhVQPqsvnzr}s
zFP5^`SO0%_cz9{;x(O2mva_?()1N<hkl=hc=>MWNhJ-6C0#ErEBpzZ}8FH$`f`MUu
z{Qk0{qN;y?DpOLHba!)KerRwEd@iPvw)W|3*S>vxeB7t~GxLmv+WW4rkB`1><@F_c
z=GQw%lukd|*(Q;Z(EO-ZCyd8UtVX;s>A!T8Z)F-6!;2Whvt}y2j0_5Tp^OZbxo1@v
z9Aes-8vOq0SqtSbHT3a`uHUbAv%-0oX<XvEZ(lVS7#3KnFlcOxpS)(jPGr6BuW8;Z
zp0<8T?boTy=yJN-@;Z%Ud2-h(CE>hfDX)wxUz!$*FA87pdimjUcmEmNimuKIvTAi&
z7_p<^;e&(C<!^3$1f7Xm%2pN178Vwkn3%}P$$9SFIZ$`nYz8wspGp3`ng<6O9UUE2
zx2_2Jxpd`9P;bt2vf9p_J9BStYJFM2b$I_Ju6O6=T9=)T-I7zAwS8`In!4#V)k+mc
z2l;v(=7w_q6i$YnMGOjAuY?&se3{H-v7}j#p&?k<jANxls;~ts!-Y+P>_IE58FGZ4
zh%hwtFfp7s8<fIuapU?ZZr>G=(>Jf=?w`&5N#9-bd*|r@h6$Uk3%@oN<S#Z`qo^5v
zZQ)gp+4^^{ojY=ZK|$S5?5ppUlDfbawVMsj?zY-r)!sFktTx>!Q(3vS{sGJVld5)X
zMyjeyo!j{o6%`W`6W2y<)w+3z@qx3?2QPbj`~7qKFYSLR|6B3bmih4w_fP3=7e3nB
zztAX;HTUb=y$@!tIM6+H&+Lm=9^L4Eb4L7OV_o;2w7EZa>w8XGv4B^bnThGoA3yi{
zfO%gJ?u@OgtCMb8q{869DL?1-wudDLI}f;|i~CvqHT%8Z@3-=;TesNx<=))izrVTp
z@XFxjUVHw;D++bC@yqYaxVXqT{alZvaoa?Rkjd@s$3cgg=H9-xyF5RwJzS(oa+i{I
z{lA*oWplI3<MMOfN2aN7-TII>+u_=lNB@(Z_jOFH6$utk*64lp=I1#Xh7CoJznism
zNI%-pUvj<f6?gT@@{0?Z{X8GppRf;fS-<a<_b={SMJunWFR8u#`8DH}Ef0G$&b`e!
z!!+s3HpM<y&!;j)M-+OGzO}gS_w3ZH!dJx&=||+U&SmDFVVd;jSa3`2k`mt?*>2pw
zq_o23PPOe_-jL!iyK35gIghY8e>Tpj`?SD8@8O1HC6BH@WjH^<`0CWV$0i+~7AWq$
zT)f=yhn1<$9=&-xN|vY3XH>nx_~Y5x*<oQ}W@cvT=jZ7zU22_mqvUn8e)jKgrQLo1
z!V`*L*}bp4y*>X*-tFuAew8nN>Tx&9ZPSG5_sf>0eqOWp_|>Zqv#(@@%&sf?epP-u
zr?*_2?L>z4JFA!9@ttjcUucJw#{K$DGrNC&o2!3&r}YOFPWA2Y<LnRr4=ii6R{VVQ
z+~b*PZA-tZWZc<Nc)U+`HfY}D<m6PNo9Fp8EWf|MzkS=bWtmnwI%f_YTBPG2t2blT
zthoJkv6iYa6%`hjA3ixK>8-U!*1F7Zrrqv~4S%h+zM0wg^f*gai=cd5YRz6|UK7sT
zhVaOv_a@6MTE_My?SiYV)4zQw%)U+MwDRg11XeU^h%+2$V4i=&#dblw;gxb1#py@n
z<U?b1A7A=;`PQvl^}!;(mrqaAy}hIG@wK(l-qm*<3g0N+zUSR|H7kF{{;gt+3^Vz^
zSyZoK3tPQ-;=f*&%I5njN()<*wr*izQ0TMGyZ!ywN$Y=VlVr9{Q~00A%Id&$L~iD#
z_cLmbTx4WoVDQoq`N$HrXU|*>PS;NkN+QkE9-Mv<+v%afn0m5MG5gN)toD1-E7>R1
zWGBzw$mo6ErF8vbMuwA$+0Rr;@4QP~r}O*>$M3_r-&mgh<&bxpR<<*#xV-Xs&|CAV
z6DJy`otZIZ%9O?K{j+AwnDT7pryoCm>PBu#IXlaA)t&rE*O(mziGO~4e0;orzHPPH
zWB%q1e6fbQal2}6y-^iYKDRQbij5&qFV?`8*&$2p@8mB$3`SP_ZhZMudvc|{_j!iO
zmy!%YsSeMLrp;q)IQq==&35alIXTJi=ggV&=lIi_Utd7ondj%{^K)}+H>cLB3LTtM
zY5Z1XVYR;Ib#F1&{Eo;?iDC|Q&ZS`;_n#DYGjG?CPP<tB(^yfLo#ANjJTu?iIWn^%
zeRc@W|NG#|wt3HdzuW#<qvKc^QC4%r>Hf{=YiGA^{IutjnYE?bjpfs~`Wk;0tbIFI
zCg)GS3WLL&ow@ry|DEnvvT@E1b3415uSfQ7xsrctm9nQuNa>qP?FqVf#N8MfQs&;3
zu-p?oSzoa;m4S61i|3;6^Cd1`-nM18>FZao<SdJpq@SNxR#pbe99N5%E?rvs`daS&
zeYNI!cMg2{qit3CN<>bs?&+zi&|`n4dD32VWxkxG`s}bDCy&oeK9;|nO$x7euNV02
zb8e31=X6!XY4m$4E}l5CBIWRJ!>Co6@5~NG7v#RYST1s*P;q(o*TU87r~d!VuPCkd
zZsVN|Ntb#=-NVdnEelS0qMr|U(`v5Oc7=%hewP)j9@_b>zrTXhPI%9Y>dVvOu5HYG
zy6AA)_Q)+yk2EV!o?AX|kFIC;3tR1?A5%64FFQ9aKDyLm-MuHSZ%?=toB6-bcVXI)
z%^!2cx2hyN#LBmR$HVo{6IX{HIr4a>|M$CFGj6Z(%8XmKzNWUeU&hj??$3|R%gd%-
z`V+Qr;lgRtrhR>Nb$8j@sJT|9UXMjYC!wDvoS-sep|(cTgRBQSflsCVH?MN~c)Pgo
zlF8npr>E}it<Jx(A@SlO*Hnvhp=y(}cYcX-INd+Lvqrc~qoQGXx5K>u3Xz|Fd|K)D
z{{{yGgU4wz=atrhAB`Bov{$p;a*!(hDC2l{=X9Q3Zp-eo>GbVhx4`J0?GC}8i%#CU
z`uf<dPYo<c$I1lXw_|JwFF*8dU&{KJpZUkFq>M7n7@pXl%)gM=Ap7^#i>tdY9!%eJ
zY@0@w-{~haKEIFpZvJ@c-9w7szI}W6@L|8aJ!l({)1mk?R{v$Y7Ck#R*E-=qgYRrJ
z&<w=khkyP^gfMg;on=}4tdW`hSg*8s<|UP7Z6_bjN#Y2V0o@1C&iD7r%gbkHoA>ti
zp8j-2r65Dnb;gc!xo6{TmPvyS0Gp*#sl&k0z{`;HHS0<DYke2FXG{!lrQ8?_Wf&L)
zPcT+2$jxBUnfdMe$zVP9MRn8X*{+%Dzv=SiOH(_;e=WV_S&&+!co1}eX>4W5soO`U
z?3u9df`PpKy}}<#=d*MAPbYW(dZxvBzdq?g#(JZ9JDvUg-{07n{QBD3&0DwL-CJFL
zyifN3zB505{BUe$E8KafqoZS0=<0v9jZ>#BUYzXWa^%Oy$Ft4z>z<qt48FW6Z|V^%
zlcaeIIjcSEx2o>FGVOcl@8kcz+bMFIUaH@6?2McJ-pQSw%&h-sggueAv$DE%>eQ;b
zzrUvI$Ll@*!ox6A{(h&1Wy6BU0rCtpOt!6MXgK?@>5-e06a(fVr#>8#4soW6W+}S)
zJ>7~8U#2#le#XeqzM7FCr*R+ao{wMd&QzF`e!TUUpS}9?jg|}tQlGzHoSxz29lqYC
zDxPW5jm_ED+?NOkF)(zx%uqVx{bbr*8-@jE->kd8?Z@ZO-A!*mr_`31q$|u9WHC%V
zHHDd-uSdobbbgD-*YLxC{N`GvW@q1C=-mF`!2^50^N)^HGiv{3`tygc<!J5a?lbAX
z|Ni;&Crs|F^xkj1J7#>=uBq*}JFBz7S7_6L#7h^SJEXSM*zb7vy|%+{ThY>Z2L}g-
z2PauLc^PKvulx1$=h`pUlN1&*t~jzhJS@*bTc^O|)7rQHtv@{9Jn_Y=SD*osA0Hn2
z`T13Ud-HSlgZ~b${|-1kb@iI_=R|SqMn=v0$=mgwtUn#k#Ld7FyUG8qym{cXdrqON
z76&i)J3UPobX@lA*lW4TCk+o4|GT;QVD-V5Gg5MIpSzcPtbD`8>C5j#Z?5BKU|I8}
zdWzoqi;1tMb<S~IXUxPbcDDM)%R4<{Qpct>zy5DtaO+gVRN=py(_-avzdv~LER^f#
z)>w9BW&O9jd<&EGS??$F-rBz-Z%e#8yZ7;R&)xpLcD374@@(6eZ%4T=`Y&!iAG173
z|E@rEF3P_0>KW3}TNcgJ&3w5*^R2h*tk}grA6+}UdrQs_(+n+Bm5k$Gtv6Y3_nW)x
z-;=j*Kj*)#HWM@0X0~nd;^c~oih_cIfPf40rz(hbAMcemPdx>y^ZaHSJ<Y%N^I7<I
zxtDj9s($y03iGLm{OI1ZKl@16+uQQlTi?A7c>H^@kJPL~Ujqb1&K!KRsw}STs%hhu
zwfRT1Y<9dWUt+uWub<2Hf)}rjbl=!fn7p?ywzranA!D{>K^e=dU0aNvdA^@_Yj^I~
zFV6g%xogf&Z+XvkVeS9z{eSlC-Ftq%{r;q*T?d=l)6=K_DfqeQLB{H?&h_ga+89qe
zdpX5wLcHsWiL={R)pDGV{IsJ$$n=8BmYetaPF+&na`b-5b}@#Ih8b2q3;tS^=kc&I
zNX>G|HM99wG-dMYTPvcvKK@osHS@i3zgpC$@R-5zz{PHVAN^x$eDL=6cF@_$Qdi>_
zsVH5XZW!n#G%<i_&-&(NU3NWdmxiA4YxSDe;i90iFW~76U7?0a0S6{`w;nhzf9~j%
z!pFyU?$}XLUH$)E<N0~EukY`#Z)W5D^Y`!8EnDj94)}M14zJwpr|rfb^QF*`jX_}l
zoW$$DW!7d`|Nro7U*+S=4MI!PgP3feMk`z0IUTz3(X$0nEgV0Z^7Yfo-h6ykdtR=W
zCxmbBWR6y$pbx1N?)JBx7nMj-_upx}>22El4R5bS<(#N{bF)U__x#%1wz`ZAoBn^!
z5#0Ihp2gEQ8AxZlvd=_1+cj>&?Ag|}NSAi}scp1xEzMY>Rac@fl>B>z{>yXnkJs(H
z9ldw99)rN)U8{B9oS1JAkbK<W;Io5S|4W{|mso4KR`T@8iRtIsXNCQ-?0Yk9-Jh>_
zf67L$c>7{?%!io;W~!6prrgz;w?ae3Kfu72nZbX-*U4Z0Jl*i9Tr0%refmpDh7)^4
zoIZI7arJD|E8muX|J|!sR%vHu*j9g=;#0-O(0M28*el6*f8M^<U&h4XaObMRwI?@g
zIi|hvUcZw6B761M*^(k#?j5`H$HB*lLBQ(BDban;%=q}s&W6pep7ZU?tERYV%H?<e
zFH6ySdE!*0_O=|!X9+uhTzM?dH~)88N9?V$vDxo5-c|U;3D47D<NbW`@oblMw-@FZ
z)E7V9aqbV_&&kupUJ1_!AGY54^<Qvs@aON}<2NKYF87;zZCz|M+t=tb7d%R9_p?lE
zRD2oMTly^E-IL8pX7f9GC+6&(x1A--Wr@nPrtpV5y_Hsq_1?66zckzL`M&gxuckkA
z3gwu(@ywYsb$@^T{QWzAXHn{p4-ZwNw&~0cU;J4?{^;kg&*kdaJZvr8U);QFwf|+8
z-|ou?vv1Ccl|I%pvpo0sjHic>-x0pVaA2d%zaJl)?;78FKhyGisp<5-w^C_!w^c3Q
z9a=V3X!%CrW9Mc(EJ=BHXG+_B-uc=#NA#{N%ePOsDSO-O@=hyx90#oHw=%xlTlJNT
zm)ExHOUCnabAy9}y|=tMWZ<kk^=q^02AQoK42Mn#yby_EnlN?ciF1E<v4}}G_%bp4
z)#$sFwPxE^{r6rQeuUoC3uz6AEGjFzcIgrqFK_OJ1&$}D&(mFhY0YMF$6vC$M87}y
zvwEZRnJ*oG?3FJb4?nOec%5wc)lF+<|Gr9en)b!mcIjeMhRoaLQ)K_Xdg;D#o!i37
zk1G6n@oSdPE&hLOt;ybd|MO<VdIwbenDb<}V%ZWN1_!-!t2b}THi^IJ`u5D5Vt@G<
z?(**$VIl7spE7(`Q?);^H*9Uxbp3cUbMx&RHvDK-oVcIQJz6m|@P(PVNuuP117<3{
zj?*S=XA#p@4;1@0^Y)_URclKY1f@>zp7G)NmC%i6rd_|>WSh3a%G}`k!6`nqI!D#=
zrYiaE*;^g2lJQwZ$n)*9+Fk!wJP(|x@;>cG*=B3Gi^tdeXn(h8Gk1Eh^K^~pzt>E-
zx3{|d?JZNe_seyE)jRPRflhO1d-Eqf?D^(pjSrn2g`zX&>+cov?A4dOyrgpT`Uz4j
z2U$PuUKz1sPR^|zWuL#j1D^tH7`w_w=lyq<>3{M%Vx5u}eoKkdK0ouz&7Xg55)O8*
zShVUyCG#Ec<=^GOm&+`iKYzZDn)cGy{1NSLT1^*HS32Ii6vVJ#p{L)=BjQVgTc;h^
zrImbD?7-aFm%1-L%2Zc5^{VurPy~a+Cl2P=qQLsPoFx@06IdEjSDZiK6#8QMkE?Hg
zmzmtzQMh>4>4y8>=RcG-@-i?y+}-}s^~+k_r>O>77L8Gdex@riOrCc4E~i-ji4<vF
zr<oI2W6T&hR$Vm@y|cH<T$16%mhaa)#rU>vc@fUFat;ea;iJ#ePaL)%o0S#ws5DEA
z;RE>aC)R0s+rppu?E0kCJ4r?M<J`|KXS7Ue-u+fATcF3lV5#cqe^T0G>BR+Kd8*}B
zKio0xex3K?O_w6SR^&A7VSLA|dpi3$6T<;3lOR^9wkLDSVi;DuU9;_G+cf5dstOKj
z_bVJPZQLDoz3!FR$^hrdd>oS)Ca}pI+QE9CkAb1c&SSmz$tngR>A3GEAD8RgIvAGg
zsT(Qd>Gyj5H$NVsg)t%XXSS}o;2)Zq@YHRC$(4Q4-(KBjHT$%Uqjl1sb1_Q-|92)&
z`J6L<&Smw|8?N26q!=7JTV{&*zxSTLiT_lb(;40cTKiXgNxvPombX%DtJ#W+d+N%Z
z)xVb(ZB|$O`s7yc^-R{+7nUt9P5-uaBe#XY{$BUnb2x>C*4ads=VU4~S{Kim{XHT6
zsLJWv&xOk8#h<9HDt&!4<K*2Zi{DP$pwHg?=-fiv>BqjzwE9=||K#2%rUw55N4Kq9
z7@Y5odZOU9ybJ%1n%kz8mqhqan9F}JDpTi&!Hjv97fzO4o0l!uc4}P#=y3KP&({e7
z3U9V75C3cac0rQvcDAb$cLLMz$xe3u_q6(um`rY{;ndw1!;Wg~2$(u~^{oWktXr8Y
z*QqHcZch91<Q><U+MtB=WM3u4ZoLYTg~y&ah1M^2>f83K?Z?jLx%bN)6Pu;PYeHs<
zJ~Dsr5Tt9&Vg2FKvlHuP_HJ1c*x1c?RQkof+n+ryZ*5x;sGfaoTdVYL`8`+lK4-LV
ztkW^xE{Sn;lybw9BOJ^5ll6Yyn7;gFetr3tscmh?USD5tYAX51dpbv}P+a4@Z>JLT
zo0z(%{+-~V!m`aFz~}!@C+9aB6FuT8Si(1IObpQbagw89|I)>aCr_OkTDA60M7!H6
zzb~arH4ojK60$tO!MIGw)x<3Mvgeh_`Q?5;HkpL|3oOz5yGrM<YrXr<18!ILCNh24
z81ulQz^nPiRMWlJ)#ugh*itvE`0})Qaq42**H8Ugb1^N!nfLm_;xpSc!q!NoEW44t
zaq+>nZrkXu=i^F4ITlR)S#m$NJ2iDHkDKZ-Z4<t}Cx3&YL^$fqwAKVTE_tAC`G5Cg
z)~KhuTBRB0J1;J3jVm_!{xAL39?k8s!ezgLUvA+(<CEU+b(UZ3MuqYj7SO3tvo08K
zV0Majclh*4{KkaS3`>eQ8xEvx`~8wLckkbI+veT~?(PJgBV8EL?)H>(uIpO+-r9Xy
zJ?mu~rXK0k4>nnFfk`^+>fLQT*CZ}i9Zd0Gx$S(Fw{uxz>grFY7#q}T|CR4?n>;i5
z+1C4C|2UnQa@$9zxiR46A)elU(-vDU-_9F8nWbw{@Fb0GVjF+Y_;Frp+T1sl&N??_
zSpUA+Sn~Gm`oktBB_|E@VuDMnzW$S+o|E8Fwf%wJ&MDG=UfUkBaJesLlRarxX=K=$
zxV~p|;(AZ4b>Bbl`n8GnTezN9Z`-l>!ukcq!s{<{xtGOBEp>Unt>oACcX#qWEtp__
zXrAxZj_cR^3#<Cqp7q=L0emd5nv~SxRj)O_|J!t+jfr8&PD`Q0V+WsxSLuJZl8n23
zr!vGXs=c;cS8VF%IH#w*FRuOj`+Za9JeA1?S(ba^Ki{u3)8>r4ptR%Uysx#=V*BGi
zt$kZp*mv6IO!xh@hYs-c{yVkd-K)H-5$pd6pWm;mZW5WiYsGACjgRu8sY&OgQ;%|W
zPP&$TDmUxs{23D;e>ckSDJg4vWZeDW<4gHzIT?q(l|GSteV{wuPJE%;zssH3Sq_KV
zld_&1S@6xKI(Zx43GO-53$l+)vpM$V&-N8xh(03w%j(<P*XO?yU1YQ2?V8wmugt?V
zpYZMGi}IBYI6UvM^_sJ9Isb1H%$>Ps(rw58GoAN8&Fa@T*v;d%RHZ=n#@mUh$8&4L
z-TeNn{lZhpo%2HD_ij(Qw!K*oE<JHq^*B_{`l9dMukRbDiO*PgalWzX^PQo|s<vlh
z7sX7_^4|C?;9OUtO>RTW`|v#v7*04iPre{{nbY2@;2m$IIR7)bb4(QrBJ87l&HwXH
zalVp%SY5E;T>oK@&dKLH7jN(TS9XD~-QbzCo7msOon9|uvd(ti$y7G!n%mxKR}opW
zRWU#G^!mG|Q-nL%9<wZ}@4b?3l6iQeC+oH~5fi`jhp#XF&fe`JB(YGHp@Cm<`Ii5j
zH4id%#X4_>JldtX(6rpA`cCnzqUUbL%8P2BFfsBksI~9(P-(1-Vw$jJ-^(+{)!uD#
z%L?6?eEW*=KgXiKX^p*mbe#hKd8wxee-HF!U{KIIazA-h*$dm$wrNY2dvE!=@5_&%
zZ~txwb<T1#a9(Vm9@)>b;C<cXbt1F%JT5g#+;%@Z$v>UZd%pjCi+PJ5uDQ04O;bc`
z%a#a+FI$b3PWkPvHJxVQ_o~|Q*5b)g{K7YOntu<AoBwuw>C<IrcL>kkY+ut7bu~gk
zuc*w{>FsR!Xsb`{CW(9fm%a{7j<r_4_5Jb6;~d|vYX&Wd)!ygMyzS5CKS4?l@6GtC
z;kD-*BP&Deu~)Zd6yH#q<FMt5=!;|8o%iZrv@NT8;@qfc{r%+G;x&ACzwEnfzQ6zd
z^~DwTZ#8;Oooj#5Y`0qJ_IBeNI}9^GHH2B{WjPt?oiF^nt3RI0aLqTa{cIN2zwONL
z3#H$u@ma1`f44OA=$6o^?)K|94PMuLj$2lo<|^0qC0|Ryt4y+|h|$>OSF+1rEu94`
zw%@n=)$1Q<v2&GW$inA2XMGtyNUpfJTD|_3XXNR+lwaJP`8<15wp7_NUu(LY)P3>Q
zs~{<c6o&PORO2rhOPD<gVa__7%)0R8yc=_*rKarFXRay<d>{Ds`k_my)lnP1u|A8L
zyfLgNZ2j`%O-JVY&#8Xe7O`O2y~S*&7mEE6pEUdafw&bp=D|js>H@mKd*c2r-|6lb
zW4%MX#P{|$nY)+Q2!$@xVtC3VU9Bt^nJs9m`}G!&{AG=r`;Qg9t>pS-u0|}pe){VA
zU&}1tuF2kG^+SKd&8u_O^s~x8FSz^c<=@HrXLrV&&TXxh=AQY*n6>hBLXTwZ%2{p&
zTSR{)m#hk4R5=~7_rf>%H5G~MCJeRjC0QN1HBzsJ78*{wwCa2wlTNs@#r*v`ChdRX
zy}z9`-uay&>zdL{2iG(Ph7)x=V{dNJ<D8-t#>UV!GkUXl&GrW+fqt<9``M@21z6Y3
znYVw<#8%;sX@=mFzCMSqnPGqUwB+UD%Fl~g&Fq*D%vFES$g=EPjJ9&DP{mK~oD)lK
zGAJ;v$h+q!WpZ)R_fJdG_O995aVeV3d1_*tTm8GM>QW5`YeQe?Eh;{0a(jXaTf-hv
zeFw{&$e-b}?&k?hDW7F6Dyg^o8CRyCFP;5R;NNL(%f?OyhYwE}973M|i3^ERVN{uD
z?>V*dYkaP-vI(cHki&tGhW|g_cfBVSY0;?Rv|sD-ztbfedk(a@VO~J6>m0@f1Z6Ar
znip;Y<uP$7-M=gswy^HsoOe&`@T)gpm9MNk-*rekE`_;fCHrYc4<UwzbBpqpYHTU#
zSh1%u^vIROSGzbjX(Vi2<E<yN=Gqij<_%HN6B!nqXJ<IHqa(-nG$X^5{XO2{3=<ab
zE;xI&W8d5>d!$6aYP3&tZe+jc;PdKSeW~jAJMON|>2=-u>W{b?G@ce09y&Y4r&jH_
zS{{Q(O;vL1`s*En`}XZKX7*=g&9Tt_b9s@Si~Q%8S66@UkX*j_F<0L57JYv<olmn$
z7p<72`cl9D2dka%-*bD8PWf<fXRfrLy?)(t&|p{H?=Hx7b;tJ?F7oQ)WtbUXFXH+(
zrX()u&s`?zTOaw~mCxRuop5~LYxDCtDhdn=^^LqcMHn2E1dk!#ve4W9Q8Yoa$G+kG
z!i)147q84@FMeL~@cZlK7h~5<U|8s<r8noS<vgbeP5EJaUfd4rWp_DVW~0TJHTl5i
zho<$jq*a@Z`Jb$v%kgxR*x{;GGe0xFDyV(Mz~Eu$@yvee$Lxu3*Ip@$W|+WW<f_eW
zq_wO-d2^eh<;~vn%!k%?N4?v6=kod`&tFY?SGps(y-0QCrkmn*R|@AGmbyGGdOEX>
z^SO}ls_(j$zxA@tvXmT<_FA&OVEr*ULl567gYYX2kFWMHtNis>zt^a-inHCW+4qiv
z>)Sc6_P#r{X72OlmNObOG%HHKv-Yd=eEd6k)28_{uPmND?z1(PlaDdJz1sNsnY-so
zKUqDW7^U{>k$QRj%)cL;Bj;39f8WO}&CkJKS^e+TNfl1@^0^%McFeUsy>iA}q?7$w
z&BC@h+SNCC+3v2r`{=gN4&!o{5?|-iuTM5Vf43orIlpA~jeu`&nWVdfpF1_j$8fSS
zFf=_bO<7+#p{TL9kMXT;RG{|sEhQDZtE2yF#-^Q#y`k~OSv;Tp=K{;8b)X2*P7m=(
z-BZ-&&}(o><%iL)LbktW-Rc@IhPfI%aLC%YxSKVrMehRVP6mdzITmUvA~)tTcx^O)
zwp%rWk;8Nm+Xc>@3l@5R%kp6QYRbdM8CahB{)WP-ze^cDUA?0EsQz1#*MzKfb0+?L
zGNJhY(#6~3z80Q)zjOMQx<@}3*2PubU+?J}=rC!Dhsv}66+ur0_J>;CyS#>#<pL+S
zj@5O$qT-5VS%#;p*x&pJ-pnSux!=O-!SUDbm)6_N)}1RPdBbG?)5()Mjs(1U#dW1q
z@LT)sH=J5!r??B>DDG~&w4(a(r*pG^#`|CAJ^hZwOzUslk*5Vud*VY98hg+Dw~3xN
zOMdSGk9o{F6(tEbyPp}eoBsPHq@pCq!Lmo+opZ~U6GoA2kNKy3J0*GJRIgWq#`L<W
ziz@!)eiM3n$=>%ogZ=rwMR)dqRw|c<`8sd9cKv_ERqpS>r=9C9y<#V?_<B}xNvC4u
zQ}3#OQyD%@O|1%xe7`7<VRfkX>87oci?QBtuy1$$#>&lU#^UF4o;`SwVEVAE>(9G$
zC!alg_TRcno?)KM%)3{zHs8GQ^4h6CULUq7>i=JAYjN##)x(X<b$`X<)mDkic0HM$
zpH`T-yZUQ%-KP_$(!_-Gmj8L+EzQ8tk;?G1R^(pHCv$W2_nTjxX`W#E%(r@8-HbJN
z;$A%PvN*9^<inPIYc@=r`{6_1pNa~XGr#%Yzi*#s^=;F_=*Ri{5<h&pRD3V+6svov
zf$en0oTqM2PwtGKRJ^~^!)s1WVNBrs>o3YvZTa_~Hj<8hdz;Ui$#yTlqBr+?&#l|q
z?{+O{i&a}c!#=tAa+tq;8vplS^RNG!5Yni|uz_=HBzv1g@n<iA<8izx{d}2gm~Zy)
zwG}(pRgtmY)uJGdpX=Y&9Y*hW^w(ECySM6AircGQVgB~1x20eG`LbY_Y)xNHowdP>
zcSlQ3u6p47llRU2RXxo$UQ^vB+p+(-s(;Vs8%N1nS?hDL_ud@ORRP^~|Dt?F`1Td+
zIrjYbbds(KULLZ}bL;EZvqFRyJz5~RAxHKMFJsXcA5TMR(8YBxpW7K%WIaD9t{49C
z#rLJpRO+`cdVjHDwIdtD&G*T1g|*UIf(!?qiP_Z#8pYpBpBX7VfnkC5&x_6R^|rph
z&d;h?^4Bft=N0+o&-Fh?gl=-QTk3P+%-Ve8?fZP%1Np$0B43}lRVuRK(XsCDt-H2t
zzPfs%{<{4Ec~{pZvc&KEcls&Ahpn$0I*ZqbeTrQj_D}ou)q4v0(XY;3{BIS{Q)HIH
z>>-k)ry<l*Df_8zYmw?SZ;|zv_OGdMUAp{F(e+Pn{Pp94HW>%|+noLL@V)kP1^FdP
z2F}W%3llxgy;{6)-DPv;RUFXSpo+z-xF(0mHFFmE+5ew@Xiqx(q$%#)d%}uel}|*u
z(e4$8#RPMKX=fQHXq8{jFv*E|y!rS1@|;L<28K<3zsjoamU<X_boJNoDt`9MqP1a~
z{Qjcv%nT_@)Gw86+pyx@=Nz*O^QZ2Nzr8_B_x7G=4k^1_zC1Xw$G6<L%cAW(BST)@
z?S)U?Oxmh@KHlEoX63~NXF22jTvC^Ae)aJN!zs0Qpv1~>^6rAQb&QORjCCNu$msYy
z?<n`onw&57-`?2>tkUCR_)%$mIN{AA-^}3730qf$Y`LVAd9}?ee(9xs?|-if<<<(B
z5^_siOLL(sw`(_7tHxo2+q1v_U8s_ITzbxZd-?o(yT4D!KX>Q%yUOh)Q}52+{rp;0
zm7MnT%cAekY|307CmK>)A<BLK$Am2!CnlvBb1uzSH`6qnI?;LAv%l|(4+bnvHlHb7
zob|1tx-ab5x_QOVtE_nEmmkeMeC8Y9<}&}OlJ_$E=AZv`l)rlZ+{=~^T_WbF<oO4<
z6#Z%KKDlV_z2o;MGB6zbrX2NsZp@yKA!pu6b}Ri@F5dU{+T!x}_nMP$-I9KBQ$FHK
z{p9HL+xptxS<4Hb(lnP(sgq}&Ia_2yrn}74O~s4jgVsf?i@d);)Y*T%<n^0ZiheJj
zxB2&rDZ-XBWd%3c8~KTdmwrllBFyMeb6`)UxYV}nB@5T)yLC3p?BUz1tf>C+)U>(3
zPo|51Z*h5AG+F$(>ATc`*IhMUiL-g-*3@ObpLf02t!jt$_bp;aa$o1B?ot1g`m}WB
zWQp0onvKfq+}8ZxC#zkvxXWJIByTPE#OswG?w+hXxo=^m?$%X*dGpq6TQzglj$5l1
zSy*WBFiiX{S;(zgF)w~q0r!H6Um<~=8HbuCxjs(ar~0Jx+0s|LPwk0%>yf@Slr!kS
z<Bu#!V$rcPKc97E)X@I_XYNFi31{YqxE#E`Ns}RA->z!)Se~WJ!dwnyC1l=OGv_o5
zliUW29JQ0az4bHLjN<i$I-lGsF$fYAFZbc{NHDHXPl&ndY8Moldz8EEZOzj13;XnE
zo2#=_^hT6thF19eU)H7KX*4rQuK(ym14fU$Jh_69Df|Dtp2RfiB%9oH{pEkQ-ddOA
zd2joX$il~O%6xb3Jn!<8{q&N3D>prsmA&D)>D#da*{1MS=k#oorXEhc{KIswR`sXF
zn^vzaSSm3|NrZFFFQeUeZ+`M&m{$?4u)mUj=RDuYOv6@hVU2S4pWkF&F&n(@bm4Ah
zJ!^fk_ln}YXTq0Lj=nqI@@-qux37{*beua?YWc)B-md?wb=N+U_ee_dl$)vdQ@r1L
ze!c8n|Ml3(lwz-0Vnz2=mUDZZUCVKk*HriP-k!gf3W1J!UPVSH>h^kQgybgWt@yX|
zaXfo~5QqCtvGWDf)&5v*&V6FD@AWV3<@&MbDoV?q&y<K&m}gu)f1i!fjbzP>i>q&)
zi*=p6Jx6ZI)9s0`{~4Ea#Ij9QN{sqfUUuCjx6HBS`=bTrR=N+<0=hCc?0?>Vm|Ojw
zQ0vi3?{n{qqIRU$dARKT&|5f3J$ZTlovlf;N}E4TdtV!!>b^qg^|h*XeZ8y=7hm_C
z+hoolaA&3F>Gy`t%9_f|3mI8Y_NI#6-f;ZJsWbgTGtE*NIa_QN?-f&iX_H@UcDY{U
zc97?dWxT5mHfAJGNx3Lc#;q&)T<i?@!`AaxQ`MqRTg}|KV`)5Vh2Qu7)_+l}*PXM!
zdu+E%5wA~7jIWle6bnO9uhp#6YkX6CuBHl{u{*8rzxdw1{%;a<?LU55_3qY#B;%!r
z1>UDU{QD(x-}B!sEbA^mJ?g)`G`eip?V#QZMyi$9pO?4BCe|1(e%tV|nC~aI<F!+j
z1}t+e_m!`Em#-cn`Ph)N?)M|+wBDJ!Pb@pk$(@$uu;*dc;#nL1$h~(~)>Jyo^0E5j
zRHus1A%A>WxIP60sGL-0WLTk8bnD?Ep4e^Q6U-Po8mhjuh2$ykKR&ber^)<_9gGaV
zUsJ5j6y$8g?|G_w@9%mkxv<NnwnkJv@8XiiWl9H4%rlo=J+NWwpQ|Z>*0P}t3`f&q
z&!)<yK0SSs>GGW=hV2QNGcBx6KIi0?kGEQBe)vnaXIzX=)<OZhevzX`zpETkVqh@*
zU!L#BX&)H3*N0(7+PTMPS3d82|JhD||9V0H83%cKmk5UjupH3&`hNyb@xuaE;h;B_
z6XIDEm<v8}{lDRC7(Dl>q1VBXHiijrc5B_J^4#!s)rYA%g0Zizo~z$sEjCHW@+13g
zsr8<sHUHE88opYxfBEf8_wSoAZ{N4V;ZJx!ljf$~g~|*Ix~2zi=5TGQ_`b2?t%>(m
z!_^gGe?G51a3$l)msG~DwVv~ORy^Uma^L?2SGV`ChLs)yn;fn!dHQ|E!igq{Oy1ip
zw2fzJJ7;Z@V2JP$n0#S}=xVtD!}6oLy%C=nj6U_aIzO3r_gO!;AcKK~4Ohd?1>19E
z72f1Nb>E&=Ii)J}GyC~P4ncjf;n!;ybyit6D?UxT^`>LT_0+Dqx;<<W0&m|hzP8A3
zsY~SNlP4MO=>PaL<IbGv*VVtx*0ruU%W=BOXj50foT#U>%3f=|F8|cXz|ip9WJ-Se
z3f=<?S1<`NC|D<DUwXoKcJ|A!y1|pK2YGX=6z;SMnp0?Q5y|43G_8k;p~LeafA=l#
z(3^}5Ut^R+j2Ro|fBYg<95Maadge#G*=Igg@8R!V;t?gqnYQWnoS8zZZgooasgsWE
zY_BP1Xw=*tyTw30<riQ0mY*tu%okV7TL+(+W9nizm7i(dk6+3QYmD8FAKN6Yeu61{
z%ZHAPCmA;<g};B$ckXI?!)Z|i%cI)$qSBGRa~&ovjZZ3`a?^()&6!!ac!$n$rcbAx
z5(~rboLRVg`5lu?CLM+cPoFWlKmGjZ-lOMDZ`UclzZL2&P+yequx5h%_NsTk!y-AK
zKkkg{uJL>O{zlKKUF8}Nw0_!rKX7x~yKlexwEOqS8CM+H6fWESXlFOSp26DBe_HoE
zc^P&+l{IZWf04NS7n6`wsdIPcnw3{9x4f*bHP81|ZCP!)n0588);BGA$JWZkY>75n
zbT00t;qt}YrsXrH-rUn1?w+AJy|e9%uhpJ>|H7$v&uA|GdL>f#|La%p9%;@@lc_d4
z{QZuRR@r53?&D^rPdSYvazl+nFK4Yz>b=6Ts=@1boin$j(`?tJqKxkI8kD-e-~AS8
z|9{)9iJ!7o@1Okr$&RlRmacVQd(*%pvvR-Ycb(Y&yxG?t{Jhw=_+9(`d1dLpzb#+z
zOF`j6%F~IiwXfUWMr=J{{{BSn{%taeJ@-E;=bK&);<pQWb!Eb0^V+=WXJy+{N=&Q8
z1i5)W+lrTQFvZw1&oj`CEn8b~_Llkowf7iKUUXfht$ch-;MtyttE;`T^8d{Z35)*A
z$T(d)O<O#_=A%x<p$Ey=COSIbND6zo_Uu0GlggDghV6&hgc>4?9Q0ln{>cx!DQf=X
zXY$;J<E)lX@5inA-@oA&*HtU~J5F9-EOS41+08$oy=b0zZSCLZ_mwgiEV^I5_VBKy
zT3=_G8PDIn;`fEyy0Z#7ygV;fe$#L}vaXbAHMen2oLR-oeHj-U-l<zfZ)d)rt;3-8
zeBtkFJsa9rM?HzyzE$#dS(0n;x7|!#MMd`wYNV6vZ?F4)?^(ZgBy-G{8y{Q#ysG}T
zHusAE@`*dXx`rlc*RS0s%<cGTQqsnobMil5FITNRbN=V-pJyBIuix~0nWNeAH#>G)
z9p>p3SmpJ`@37iucR8QA3I9G`)x9{?oAb`QriRsXm#^`Ss9IS0c^m&a*Ul?*&GLh<
zKNR3udj2z`iaozdVdTy!>vk=(6wq2F!@$rMxjXC?L#x-bGv1x=k4G_ztY-|n^5Wv3
zWt#(UOU`a?|9+ui&+i>yU9Tz^%iXQGc_#J5Z)Qb9VYco6yA7Ykhupks13CvG)a&Sl
zYpu6xPMmL7QadYIQg!#$(py}6udSY}&aLrQ?QQi9ar+I%_R7=ajU<vx-|V{WdU5gX
zOD~gKwSu=vWp8ZY57?v2F3sTJy6sDJWbO5a;<HTK&VP_9$~(0$cb6c@`sA!Fb9XG!
zy&D)(J!4^DsKS$(#cyxE@?qdqRcO?8W?=A8d6@pgTq{qhs$p#mzw9z0=}(_E)SNdd
zO_(2+n7g>kIpce5hIPu(KHlkarGGUd%~PE3A1=Om!0=txwxow!PFMZj{gR{qeAH^Y
z^4M@m28L-ej49V1r*xlH%=^{VwOD$EF{8r-d3J^GsnL8BQm3wAJXa}L{%77Zmj0`0
z$E0G_)Lgb$8`d;Q`xS<-YT5ixd>(hm4JJktIh$)6?W7qL)IZ*`n=gFUHq2ss;Lql5
zEGvu8{F>Q*PwGn0s_>8y-Zk5tLY+D@7BQCReLlZu*%s586EyEgt15)f(ACOo><-KR
z)*yByY5$?$mLYZmZ|<+#pmb$M^Q?yqtG%|qytF^`ck5hffuglJZ&Ka9n1rx4gfXZ4
z8P(g?F6f<@`Q5)cy>xPQslfvqk4Jqi#~6=x_x8@Q+u+|XH@BCEXPIml$LE{elXh(Q
z%=wa2oH?m=qxiY~Ym*;1-IY#DEqiGDvEt{0T*X91^~q}a7h;2NZ{N3q%k5k9*ZV);
z-BC8qx%2!B4};X%=nbduURzYt)NVeXgQ4N5S>gZnGeODA{OiNpS^Mp;?mu8?vWja~
zmrD}Q{~yww)0$X7v%x_|D!Y?h&V7}f@b&J#P$LGV3n5Ak{;Bnm7h}`Hd2&^ibpOx(
z|5>`2!D9vYm1h&2e&6Fd#}yx+weV;|&B^Pb$wn=WQhUxXvSVNnF#dINwT+$?pZ%@J
z0--jG*7=^0OJHJHee%_eoAcRs$lUF$`|kE}$AL#8;hsBnZ@%Bg!k}QAwD=Uy{+5ku
zHFEP^({euU%G+Fj|FPxK>5}cX@529Fy~DnJS;te0oM#>G{H!tS7VKYaG`pnp-HC&m
z@4Sv*%)Q?E(P?kW$!!g%S^J!7W7YOQ5n=3)&f+Zp_)XJBZ+%5@uesoD_cI|o>-^`R
zU1OiJik*{Tg8a6EfNw9j+j_I!CrBBatm2B)cfH5J5MZ-O_uuB~&k;Wk20nDCW;2g{
zQR!x}VEdnjV@Fo=U)aLutg|S>h=t*+#;pA>4I|USdwQO|d1_|#u$|j!zlhcBnVXK!
zJjrlCMf~TF=%z_`xj1#NwbkWlg)drhG9{%uY>vQ+nVXhxOWt>G*{tiyF}%<BUPzQa
z!MJ5|w1##><?M5@`Ps84&0TuxP0~Tlbj@`S{2olq_Tt(uU$iB{)jw}e>X$dm<LVa2
zcUSGx(K@zPe-+RB)8b7U^LIuEm%e!-|K3OJ96J-kyt#LG8*7Ez8UI-H>B|S!w@%@D
zx8*->iQXP2D0n`Fef`d7%QhyyQtX;^CThLSo}<fUyUV}W%Urd6e`67copSEXJ=OQc
zi__%Pt=8M`TIDb8*|7R<ub<uTW3w**SR5uf=c!wXRfXFGeg`oR1_zc2Qit!?Mys%S
zct7aB8T_NieM_$XZ4d8bJ6S)wxm-LLp?Jb@?wJ`ocRosgc_=*Ievb2>SjQLIoD&`$
zZFH^E_hx%~<W61Drt_?u{Y3tj@9<cpX}p;G^f?E%ORm~}qPEJN(`NNVZ)ZL4Q6U%G
z@-nh(BYU3x)=M5WUq38Ve#pONm+y?XIX>kcmlnnCt;)W(MzDM7i}mK}X~&;vzvWvy
z*R1el+kcK_0vcM+r(NE2J9&><)!7(@eY^f;bS>LlEZ%f8{n4z)9~Z7}J^m+db=i!4
zd#r==AFjLUyz9p8ay5wqO>J}CbaOPV%isQ*SX*_*#)^N%m#SN*7!*`~=}NfPS1r@X
zdVH+6tE<b%XzGuU#4{H<!jwK8((a7XmK07XS>f-Z;<3rz%VWVi)rky~7O*QN8134Y
zaL9ajk_XqO6Q=1c-}lun>Esb+aM*f9(5drc3#a5vA%+E?RP|jKEb+*(Ji&cJ;TqrL
zElEucEDQ;Z8w#eVKDD^Tw{D*IwRN#Z$ItFky%+ND)wj2|Po4xF8(;tV8SmYVBEm=J
z+hje=ILOE_A@q7G6T|nb!VDF1<;HfcTl4Ndi}MvV-C`oKi>=qPWsaYBv6Co+%ONHP
z0S1GcSzqn*vsJ|yIgT<h1YCJ<${V+qw^r%gp5ogp+=X}jZ?FCTulD}F+VwFzKYe)U
z%(htQvPN2l%jV+ktxO4hl1j%<3f=utw87%;r=KDWs*@aM#v0ars(2doIBUTX1;&6t
zt$pl2E4BL`{(X9UTk0E&pdT(R-zx&PE?>D4wEI=jxvk{g9ZMaZo@)_Z`3BvUk&|o^
zU;Y=IS+lVrq4Z^be!hJ|!`|BdT}*k34Np=nwNIQnz31rhD>C1As^{I@l67@wW8YgX
zyKnY|KR=~rXTLr-*V^69Eh{@aJS%?2!oaD&G}ssfrbkV6ecF9~x%ZS%N&o$PU7Vto
z3=Gn>Vl_(Z>n7Cf)@GZ${%FjT17<(V@;qIbYEE95zh7_Co(Il<r>(JKVCZQ(^+B)s
zRY3p8cdZ(+^8&-!9-N-W<72jP<*@^`qN{xT9&Oz^Z{BgC&sxiW{+VHa=jSRX{q<Qa
zjun-abM5QvUS3)nyv#>aYteJZsy|IU3<-?x`<F*vx^}p~NR#2+ZV@9()mT^WcA>n8
z@;8kPEa%VV+^NzIpB8^0<DA~Tl25`7b7U07m>4)%?&z#@`4OXZ`qv(%n1~RK2hX?|
zOm12GW_^6{<LowB>5JQh`#e-;+1LM@q8Z%O+&o<`_EbvPuLwC~lT}t-yb<1OQW+Ms
zw%!T3yuSX<@+8-!Nj9@T{*2~ycwWb)-Cpp<RAc|;rZvwO&3^B>71Sy(n6p*o_4$q~
zTO-a&pEA+?`ad(8d8XYho&=-i-_oolPKnjO%{z6YWY>$mbI#mK{FEL0#4^7n`|**^
z`73_4efrI|*;2*CaPs^sJs%b4XQim^-?ZANwBAXh`{=IH*J5`A|4g%AbNFYlS5?*5
zefNK^TBY~DI!5pN!DjZNqWZ~=KEA&1@9)<a;j%7$_2tRQ$z1(!p9}fr#qWB``O~Cg
z<$1}acAx(l?=@Oee7mvFwpP=~SnX5jT)o|Uf3@or8=4x(mYq+?)6Z)EuJ`HYQSNe~
zb8l|WSL(DDl3Jd7qtz!WGkNFdYcub@R(-VkU;E^ug$rBHY`eH^a-U5aBg2*VY59rU
zH?ha^f35lXY3bUvZ&!z}m$Rw3u(SC2s+S_==T^*Ib^V-VSnLy7M~l}|oD3~Cm(!=*
zSvuj+jePkFv1!sh?^my_a|-TF_`Gt9zGOsj=ZSO688~u(o@f8RVyo8Y<<{vgE-pL3
zS7w}@X}mn>eQ#|)O9EE~lUVy%o#+r|6@BymCoAe}3pRl&Hj6Vi`}P|p`AA>2&l7-*
z2ziycD|k*iDe}4E=xyT(cVg_^znwK%{q=GF@2yV-7*wX8{=G+CjA4R`!OSN8^y$|t
zW}aSsqM4oFtxx9W>hSft(c4a(I`!{Yle|UAiwixH#$virUw+@t4`E~MXD^a5o1nV-
z+y2`Ii}!e2q|STzH#L}-fx+>{m8HDlYqkX}TUe%NVNvnz%}wuVIx}r5i$cG3mp>Ai
z>t?N05~LPUH0A!h^TPLAq?&mZn6KRKzdt83O}fY4YsJm`c}FMy{dsG>Wi;32&+#v=
zt-XDBcX>OnblLB3x$*JwzP@L*UYLA-_i}2oSW%4I98Qi2^93bVh(-s66xV&$xcR&N
z_*zeijP)DKUQ6%Sj@I3B;{EG_e=81FRrX)osjpg9;(vSR>Woj*&NgppGOKuRBfOaV
zNO9Wx%Yl!Zj-L_zw&8i~Eiq2n|I8W{)!CQZE*D9;GA_KMyL_6r691VSGI!R_c6D)i
z^Jl{CW$XK<<n6JuPknnLUvIt9@9*#9HzpnJleKQ+k^I!}+bd<d%DJ7-xs9i@r{~Vb
z<n~#!qFP;kEWa`5k<XvfRd06l8rPM0X8Q28b3VGVwK-j7^`3++_qXpqb=kQ8_~%cb
zy1Kh}SAAWzaABh8_dZFfi<+u)O`aLZ$3Cl%Q&u}X<FcpPZ{gHi&T-Go>>v30ddvx3
zS@-u>`1-imJrx_bZr!?n|NgC8OH;o`T;I{WQLlLU0S%?32fuqlmIrR<kG*-zVE56R
zir&om<(aaZBGWQfbZ_Q+-E*?}JL}o_Uw->8y}R}?Q09k+?0Y4h{As_J^G2E8vazav
z^fL7B)`LgWyxJA-o%Q>64L%O_%r3&h=SgKzp?L19{KuR3HXjM7IP}l{Ypi<A?FpNX
zv!9I;El;lWzk613SM&69Uw^o0*}kz|@3SRUZ#@?)E9+!6UoTJ3owdKqy2W(0miWfT
z-kmXHhGp@yh{(vFKYy0KxUkS^)>2;Y!-ozAMHpN=ymwXi8X+Mew|=?1D}$F;e}AW`
zq4DQeQ&*ehuCzreLWP3iztVT+ALYtBI_+1SN~Ca%N3=<hTf_`+fqI*y1f!X&!`8mK
zv-9(biOTAJb1aIVo!Ou9#Q#_D1BQo|C!)N)JO4hDVzBAo{nPJA)b_mM$7!!}x4TN)
z@7Y*-^V#2%hkoiL{yDi=tdCK()Z&kQ&W6Chdvs4KRl^6b`qD)C!c=BFQ#-pwszBt^
zHW!YZJEDvY7cQ8wGyI$F!Fkif^kLMzHUF0JMw>)wuNGfrbmq+&Y10JrbKUp9=RUrf
zs=vObU(U9uu<+%zwb4F4XYSvv`zII~c~duf+l`wydnF8${`~m3IsN>)N7C}5UMpw6
zev=n^@AbOa-PhJeZ_mBGE%9($Xyx<yiEZm<F01hS$}f@eKuBwO*v_L2D<&<8i{#wt
zU-kXo&+_Tt-`xEC?(S~qHlCfOudn_4``gq+tML8NNvc0oT}_V`&x~Au@7}k&f9`+w
zGrztj|Ie*1?kC?%E$Xj5*ev~t;k}(0g9g)v5b4bh8f;<#OQ*G*J~g$<U-_uRmVyGA
zgrFL0%kQ!48k+R2lcr|c7=L7oeD=xf+57kVOI}_A9jGH=xM<R@4coWxFMofpLr{5L
z{Qh^(pNH?S`|I^;Mcc`fCudp|CWVDvn`>P@%QV|9S>@{YV9{SQdtYBa^IhtHLjCza
z<^3=Eo|Z?JCaf=<7H=G|ZnF(T!!ohrdkR16mZmYXl}x<-Y|)0?Er(7VJJR3R!1FLi
zd;2o?LRGQkqWYb>&LUwJC&CtcX>n@$@y%A5EnBJo@7w;6XtSIfA8u{UK6dQbwQJW5
z4Gl#l9&UePvC8F}yBxz5%~dyrrdAZa4h?75?t2hhG)JOK;{NrAe*&AMAB*lTPyRi(
zMuCChfgSJD+mczQPiQ%P3~=7qr+Dy2iP0qX1LrG5-D@8;O;X$=^ZMoNW5VKlcD}Rx
zy=KN?CWdJ;HTQRXw_OxF^K@mC*RQD`D{Rco&E@6i&o<8wi>uNDUHY;*e0|ybdv|Z$
z`t|4M=c%{C&kBf&rlzJ|UFJLc#fyxp%f<`_)AeG#@>y0)iYR??!Dsoyf1!&-UrB1M
zFkfgrC*IgJbvCo*)0PKYCALUyURE|Kdzqh2y^5C*6T^(s3Gz$NfB3F%qpF~G<D};#
zSFdHM{VK*Yjo;pU)*JDu^~uZ>PKK{fX8w?Ezqwac`P({+>F?w}luvNFclp7@T~dEm
zPd~qM)vABV**t8EtG~a~jov0>nkB*z5fv5oqv!kN&A+$H?WxsOUG5rvI7~Ns>(tY6
z`^(PWi>>xwwOi)H8?EW}#ZOoHhR=WAc;@rree6~~w=ey9+GeR;9<x)XMAd!Ksf#(8
z7yTxm{c)D<@A49(E$>c!|9a!slv6tj9z5P)I+@4*ZOq=Uo?QKRBeut$R$H4hD<<Bg
z{O-C~{<uA!{{CUtU)mMzsr<6->g{j8f5^37Z9lH|=7!9Sg3q~$f4{x`8e+0#f9{>I
z{coPiuK6k*_GUddgOITB?{9B!@2UK}VZ(;1udhNYe&1K-SXBJ{Tv%Ax<KzAHpP%*i
z_WFv7d<t_{@mv?RHS6Z4)PMi}$;ruG`Vf66Yqiw7UteGUm6=;wwQgnp8=rEW{RRgd
z4>oER*@qe(KY7R4@!h4wxC}c6gK5420o8gR_GmFFJlk3PYQIojj^gjP6CbTloVGe?
zwNB&jndje{<eiPnI~=iNxyP^XU1Fa$&1d0xck||r`{n%Ip%X8!&A-1-)}~^^u3f)C
zXI51gzTfU@x}-rpt@s85)4Yb*7}MZYe9H|!`Z?Lv)PCvlVBx(z!z?uY?+!->2Jm29
z@u_$Tod+L}$ndr6%yd>~wVBtjH0GbxqxVw(-+kf|6%g1^@bHlDY%|gNv{gy-?P`mP
zi>rTrdU~*#T{|l@s(0eFX=Y__BKX>c1q1@NuiIi1DVDx7+d^7;c68vxnb)W9?Y!ky
zu2bJOLI3Mkp+J5G?Z>AVpHNv4(;oP(qw3>}2?>r{?|4l5{_tq?UfVnFH`ryqtiADt
zhoM)>bXVbHx7XL!u3fwK@uNpt+ddl2$njg*a{SMnX^pXy0&X6y_BNWSpi%Kt#d7JC
zCr_s2)ZH|_a^H-D;Xv5Gz$(4GdCaM+Yw9ANeU5#6KQ_*hi<?{BXU2oKZ)+bP<Bhsx
z%&=nBs<yT^zZnJxuU#|K(CC;Zu_U;nqGGw<++W|{*MEHE`qJ}!gZ9xZ=Gxl7yEks%
zb>myry#MPe-(Q`&d(-ww9ga+@?`K{4tmC=*7f;$u<*zez8RlIUeYpQ*b+!4&4<C55
zr}@tKu=~9F&tM}XquFM;n>KEgw5j;eBWb+q$3gC=$=AyZN|#!9torcg=4L@<w;8i%
zA3uKla?qCGS?%ra$NOa4c_f7d1s}eByLLrHb$L7+gWY`l@^^Q%!`Fd!1V=<%0G%J#
zwoY`*)~)Z4xn+0Fx3M-0Q(Cv?tNzrE&|msfFI-D-2z<HQxMdp~bHyb8P}#7fC+ey+
z4Kq$WbpEZSZ#(_P_4W7XT9^09+uP;b*x*0kF0`zig8_5}&|>%gna1gIHWd@TeTg}9
z;DCaiU7V-q$y295C&`txJI`M3KmXQ^8zS~Op<I`Z6WZ3<hJ=KybB{5(mS}7FL__PB
zOSwgT+aqT|hTo^%#C6m<KC7^%_C4{oyCD8B-n-xWi&o92som)^GBT@{9ar<6b!SiI
zX3xoLvespL%HPLbEs3)J`16nFWVPHI8x)O<E`^>-<9_n=>GyYccc-0|a(6%e_V)Jc
zhgd!T{rhKI{cVm}?yZZ9-KS1{S#|5XenQ(i*Bd$CPF{G>XnR)p+xI8+TH7zUb_FCc
z{Mgjt;SyJ^pSk2X6NA&di#}{}wt`b_CwbKET{r7ahu_{^&w}4fyKy@(^v}=Ou&`^l
zZ&yD)*2~P!_vPcqm8aHa_fMH3A}uYwFhD|HK0hJBq1@tWaambkcei!vt0|Kw2bYzZ
z-F+A`anGJTfs5VV-ra4krFH7;+1_^rb!^5ZZ=D2=^yNlB^C_+0u-^UO?^X90o`2WO
zG8E^XlFWSZs^h->VhjgP7XP`v$>{d4(`zNw^s7`q)M@_w$8o~Irs}lmWX5gN&YbaE
zbKdkq(9Ua3lO~*8ZWH-l|8DpGwX0Wqd-H(yopyF|a&bvn7O6}>y;QYr)~g#ElgrA=
zCae4NadA!hB)!+l%E~<Nj)9@!#!Z`cmA#d^`!HnJjvW@-+N-O+zB+pB*tWd8v)&cd
zF|J;lwNlx=ujhY$*6V}+UY)hND0@2nerWVm=k1KmbM-6d{6DNLQ~5xYks;%h!!`~E
zhn1$)yQhRkPds!e{rItM7nU6ImtFq%ZM1?wMP+5|jsismg#$Nk%($13#n9+rkbP~<
z>+9>~?d$HGIMMO0;NO9x-Qw-M($C(#+gI@L5Z8LO32Y*+x%c)|9`BQ_{{HUm{r&cL
zAN~=mN%GXax-IuM-?TLA={q(|*(&<_)kRr828YLLI*ZwSPOJF@uKM9#{rTCm;K)Ps
zWd&<@ZU|sfI6ULp^t<ady&aW0Jp{Z@Dg>VR{o%=l1A1w`hn@Pv-YlN+pRfDqrtRC;
zhp)f4B5?7PDN}rWd|vdekgERqDfR!qzoMd|=jU2q-%*&n>+nxzR#w(|^XBPBZK?SF
z?(V(4)w@eyFI%+8sXRdR<=@}m54ZFCdwT9Hc{!;>k4sy;Z(;UU_kOvX-(D<EG}~C{
zuDN3GN>;AoI9tvFDf_Y)x2|fx>|449blQQGiAQ*N`SWvgxAebRy?Dch4WRS?A~&U6
zTH;x9{8Rd-&6|IJd6}G+wypTNUz5VNoSU1<-rmxU-ezHCbxAhr=#HA7o7&rtw{QxJ
z=|-vW`g8rRov|=`_xAkzw@O~0%($^Oa((F4O(DB}o$%g0zoO#CuBqMXeqR<?Py6)o
z=O55vfG#d9Y;4<dZ%=zyu!?tKfJOPcJEy1X%UhSNx#jD$r|$1A_kOv<ZM@pH^Q*qU
zySvc2z3knc%D=z9e){yueqYqBPM1Zh-qYq-6e<}QY$$$yuB83*@q(ohxz|?&F5bQI
z;;W|i^L@KLj~t4RZ}@krJ$?Fo79OrI-zVPs!oyHhR8;-#P2{eUmCen~`#}*M!oPCm
z%5B@W-Mo48`@6fV%U=Hc`1tr-tI|(jzUUP1zI^4%mwS7wnHm=aFZb*0>@;!xzCLbW
zO=foX?^jn>v-8V^90%WD$gpDNO4r!kIV*Vmnb^91R_@=&@ONR)ZbyB8|6MhTd*b)k
zl^p++t?bsbqw@2z{{H^myRG|=FSeFhDJCa(Z>jh6i;Lac+uFi*OK-n;@nTt7nY3As
zMtM}6r>CZYLBYE_JNH(9zqh}>{%ZU8{-);U-KDS1($35XUhcPY_3F#A=i(1V=6#vz
zqBPSa_14DA;k<5bn?j{7$oid~D7Dk>zvqRmtHq<Y<#?9s)Gt`EBIDDOlNT>uY;0`I
zy|rbbwaiLop-wTKh=Ol#ZcdmWP+VNR>+sLy^mB6xi;KB=c;4LG`}_QPA5Tw7qm+)m
zzH{g1TK})zzg6qN<Bxmx?0NL)QB!mC>gz?n-anL(sy&uqQ1<?gS@NY9?7yo-Qq(TW
z`uRF|o|tm^vP5S3`=fW)#l4hdc=GgVZEfxIv$L=7E`NXEfWxlC6ZP`%?Rj}`@9*2&
z^WWd!A1{0P^TB5J<Gs@5&(6$@UzW~r;p)}i&d!S$FK*nlY1NtDW7@|KxAU)EyEZi?
z<=wr#(RVK!Kah~B)i=<1vHxV;+IE?M>@44MovRX>T2*B^ImV?QXI@_D<>gh<{`tF*
zurNEXR7zIXuGG_Fnw4uz_MV$-ZJv2)$<58_=Vur;zl-=M;Tq^YO^1_*$EN;Y&FZkV
z|6bhMoPK`S&YhAH5+_ca`132rYJ$qo2M3w+^WP^PZo4S!ee?iN@2SGCudX(pJ3Q;#
z&4}%98FnSFoaWMM+1cp4D`Fd;#S87N`RV)DZ`g3*@L^>;J39@H7ykFJWeaVL*i-TG
z(S{8h9z1yP?d|Q$vghO<fBu<#tf%t#x3?!JtDBmd{$JF(X3d%%J1jEKNgX?O?AEPY
zCGLTV2F{B=sj8^l_`2nK*87!r9jmU(c`sjd=XyW$oI5dd%+|y^m+Oe0ynem>-JPA;
z*VnBzWS(L_Y2G}$q9-1_yuA$#4X<{5(l9pOT>t-{Zq$|!U%#equHUiOY?4-mywue5
zCr)Vm^DB9n{9jyL?9uTZ_mBJfd)&C6ee=&sQyp98hS{^DU0ja*`ubYkch-^TpI`DV
z_X3Ro1TXjN>+4HDH)mzv%byn)yT84?ef#$9{5(8IzQpuCkdXR&=6JvS?(eT=`<i9E
z+|hk-(o$=g%<qanvrct#vj6>dWk$PU?fL}^68`-7*e_?x#ly2C`+WZ@mq|}HrJinT
zZtj<}&B~3{4Nse6Utf1-hT-jPxza`{9CtS^uy0Pz_+p%um1Vv@-Y`SKk4Hx3nfJ-b
zYnO%`+q+obuPVl(dK267bcTSC5SyAGAD*0?{PxY8K3QwGa*L_2@9r+&Tm8N6&ySCP
ze}7+{S#@t)?(E&Wf4{uE+|V}M-J;EHzuH6(n}4%~%HFm8I}xz0$adG^iEiuH=boxA
zG5B<f_2vEZ>lUe~sPIUcoH%wY?Qk2fY0;xEudc2J9a>{k`RUKk&#NCZ7amRecx7eq
z;zf%N9X?!IQu2b2{V%VK^Wykq1BvR|GQZCf9~0ugH?2u8X;+Tb=zQ{X=FAVzuI$}$
zd#k!FbHl`m7xz|w-w~5{eO+w)z8cSR3sFy{iEAP@9y)yZ@r#R#xy5u=^u4sy*FS&n
zo}H<wsg;$~<>IfcyZsm4zq&K{INR4(G7+|JfwyDgwhG>T=)oFvq`oT0NBQTQ*Mj1v
zCMGZVmap1)GbcSgeUeISY;0*s$(JunlB6%!fBE<K_rYfN{q_I<eR_KO*RNkC?Vru#
z_Ex24XJ;oS7Ctz@n0e!&$X`<#=f(C0Sx-(}m~bH7-962gZ@IPDuTHMw`7$rAFWl;N
zWa{Uuzh<iZ&$zV2Q(P}5Vo$}!>C>kNMC>(>-&yuH>Uf{*?$Xy~FE6R;>gw(~{8M^+
z-rY^xw#_rioV0y=`PW;Sg0=kbofq4GdvK6>_q=bVYnNqjm-jE%*`j%>_Or@?%qtuI
z%{E`Rx^MFWQ9XVA{5Ll?u3QOfPpnzCf5P+g^TqRnZ>)_r-<AB++tJZ6E-sFdk<qs5
z%K~e$5Jm&%#rr4DnssYwwExXzwX;g%qkQl8{bx0Q9GG-rLTY!K|Lj+u<vN!@HFNp<
zdux{*D=sd+zppmBl0P~uAYg`RcG%vkt3|pgA0OK^H#HqPe7JD=WGQB2lV9GJmX>#`
zzn)bsDPez;V6ggcvu=I7pO;I~i;Dk$ch8+O=SAPrGSI2N*Vabs$Lw$@R|s1Eqi|kk
zW@chyV*Pguf0el(>`Z=nOG!yxG+(#vnvM@oi5T~%mwd~+_UxAbW0P}e=Lg{#^CnI?
zapugMOG~-CkM64as<rI76q~Whuj)^qKBfE5f026k)xvyv$2v7o!^_-!Tbk?IJ))vF
zHzvF9N_OO6P_VO;lYbuBoY>~}zvX1guXnd?kJc6(+V_q%@?=SSa3m-h-n{wb^yBZ3
z@*exzsi>B{yOWupzd!Z#G}msii?ZCSM09pBJD47P<jO6TX+Fp5uw#Bk!>T^jyAMMQ
zf{ys}$u3_zqyFJJ-_+F9U56bxK7IatyjObqq)A1;zr9U6JL{q>-zpu8HaF4aH#ZEA
z_sP^$rMO(1yt(9f&_owk`S~(Z3HO{H-nz;?W8OrQiZ3rF?%cU^)-2G;$Og98b7m|I
zYz3Wu|KuunzTO#+-H*M?bwr#08|o(3uRr)L?6@=E=~Iuc#dXyE`}c3#wr$;Fy0`XL
zo8PNg_pxA_bf}83@Z=xX7h`vQtvTwOeXEx9?!!Nm|CIO7W8sV4l+bqlOJ>ROCIJyq
zQQui6nr3F-etmsy`j~O`i|Gc=i@C0C$-KO~aQmAoD;w*F7f+<|Ifk6Bob~6sf83mJ
zclM_re{it*Wgn}eho7IENydbY8w+!C)-*LS-F>(~t4UHSbdOc(t2_H7E$^hhS$m*v
z>Xp_A9U;yipXQvZlG@hPbZBPqaqIGTF`jRgzx?Ktu`sCq_9k?7Sg*8s-@Ax_sfT!a
zwOaS=sfl=T@l^KzOG~|_&GYt@ybO9;+9>dLN6fnR_T!7)`<0ZHSHE<Nei0vXTGFFM
zVAaQWcXnQ07i(>5x^>463!n2_h5Hr?I(h8!P?@wf`+DB(ZMo;?+5Y|geYv%ai}aZb
z9EvKgLBYYXv9X$6yQ;tE9d6@&$;W<?GkvF^lZRG-$kyubdEDIHpd+yNR*7CVK4fU(
z#BovI)b#4vvuihO_;9ROdTsRfW!5q(q6Iv?0!}51G_tO*i+%R&+1~2!eSLk$zVvuK
zkdRVr$@u*0s<yegx!wHw&(F?=t&O@U%Qs2SqRov%Q7S-WYw>eGZ*TAF>S~|!D`G&R
z3s-%6byYh&JbYn5#ph>dr|CvtlI5Ee1d(zMywWFYt)rv!>)TstQPI|yl7Cr1-dxeO
zr{?FSvuAx(gk0U-lk?p6w<Kq@2&^h`T3GP))zxX!rv3f(b$8)owq@yGtikS``uf$?
z)sxkHv+nLHt*P15*T;AF;esq?V-qKi&}ltAJo@_UZ``<%dwZLug~bI~zEx=j)1(z!
zwB|+V+}l&RdE&&t)6;b4+f+8al>Eyo<E-2gz`DEg^RxQ@e?eo7(cAN0zIt^@mT%QM
z(Cw1}ttZc&n`fGR?ds}q^Za{n-n{WCw^&hiI5MwAU=@Q<XUg|?ciGt3X3d)A>FGIR
z-o^cgL4o~u^^YGvI)&A>G&ExNR$V=Cpy6FX0B<A6U%#s5?d#@P6ejJj`)gxsyFPw@
z-%H6*1yHPhG1ut&^!<DO<z>8+Px^RyNm&-D+!b6ni!FU;V5w8!mCw)5Yj$~gdq4hI
zaq{Fzh6T%(EnB(Lvpi$PrkEWC3(Mc%Gcqz_a0r~TX3ZK86(d8#oz>s<dIN21etg*f
z|DSO8(e(54Obrbe8VZGg%9N{~5fL*cPoBJJk&<hmi0jN*v&2M1Uc7j*WZAMyvQn#b
z)<$nn+ZdrVam5pk$tO*WjTdXr`}60|(Qa{hyPArlU80YV^(u2Tt@8-zZQp!nh1JVT
zOSxyC)zi>;@Ufz`wUw2Xb<?Izi`{w`S?h&#&N9giN_(GTX}PoT@v+ca6&25SH#aX2
zT^$yq_x<E#^~cBikALaW5>3kpnY&{3>hc#C7Jd;_o7^e6>zK5C-JU&rYEEwV^70C`
zZ?miZwr1hN#DankGmX=0YHMF#Ss8qkaX}P^vB|2mWBu~$mn;eSl)KoiH)>0UW3tT(
zE*@E{5T%JC^}5Q+#$|7Ata@XssCe+)xx8m*W@cYs*W1x?VE4HdrxM!SuKN7{_t!hm
z^VG?cz4G?+?o|YcUbu7#lrKJB*AZKND$V!O<;%x=BrmTH*H>3x?%pr=_3iELS-)IY
zNTg+ioYm0KD0zQx?~m-RMQyy&ZuvGVvV3M3I0gm^GDw+ZT-a6mx+^zEN38tSmCWz&
z?y9<TadC<1Mx~^rT-lI#_{ozeox<vyB6h433pG7+VZ|@anJ=zgzn*_#0ps0=3syaN
z@<c_bGbJ@uJGpm8clYVHx3{OCoi%m-{P;aGYf?{7TOYrFU;Y1ot3p?M-4AtKq4jXN
zQ^?;hZ*B%FP26xZho7H+SF&TM3wXRn%gCtcNQa>4?N3)$1|L3r*b}s;2{eBB<3r-!
zs;}qf*+v_!?-PjS=vwrmk8z>BS;2z?8#ZjXb0@~s)O1&}<9>m<Ut6BL@cXSidZYe%
zaNWV({PJIBnf*Lo=Fz#NbzR<%quvY~($3DBsvRySCMLFDMO%A!{r|ceD~U-$7Hw{+
zt!-_5QYIeGku%>p)VZmMo!%6&%XiiV*%RkGJPS83^L=ky<x{R;vRhR4ZcKnl%;)*f
zyF^15?t8WW_+fR`$$GJD4E6Q@FTLCuk&>d4mZ73*;JkR#Jex`*pWXMSJhLic4|@Bw
z+q2#J`^!EjEtAJ5wm$juT4L8Ck%>Z<kvBhl_~1ENO;l8LYMrNs$gkhO^&>VcsQmn_
zq_sVO^UQ@6GFh99q&96TaSwcZ(#WOA=tuSxH=)iY_5c4J{r)3DXIfKJlZ(>B&p*Gu
zzJC4MwPWh7ikgj*Qi>t3IgTbVWL#YpdV5>0uaD1{uV16L=iOZsX{@u-DTwvIjG)t2
zzG$~LeUVGDtWNtIB&8HpWH}C372kdR%ACQFQ)T+>s3Xgg=B=}d(Tsg)W44O#L#HZ-
zqGrJTjlL&VwW)bYT8pXlNKPs;@;tg><s6%PN8PtdrDe1T{FOQ*wUog%U#$OhUnLtu
z>%-advQ<nnd<+aLHr$Tf^m^sAIcHWZ=VCCb(*5|C`)ZEG>0W_Fvp4OHQWl$9r)}*S
zQ`BSgc=n`Y48rDrBWqWdzfY0QJbXLavoZ3l?QgAu8A@`RhZBQ#ewX*#b5SWntKIe4
zyr}P=Hu$Vw5cw&8o!yMhQ}@ZUE#zvR?<nY0ayeu=qrmgjn&Jsw4{Z~+&oFJ+v0MFO
z_M_&ViWg3-a^23K`Ee%0&&<<vg!<G3UbU^Vns}1^5#x+M&p$HM2-H+q9-qBTo_Qfx
z8z^){IV`VkYM&*kxpBdVR4s-Br#aR!+{k+6`kqT8Q0)6umy*|#oGJ_xrt{ScIqco1
zXJq#GiBWzr<NTDKEDlgPx@)hGh5*CnnKO-Qb|2X#purG%wu*rvq;ZBjqjy`w`*-I8
zH6_iBC3LG_?=m^dXB`*ns2uVV)SR=8+7@N}>$Cx*N23c5kC`+>!<mzJckI#FeW*=m
zQ*-)t!?pgMM_oQPE6?y@@-W;!A$0$|5093;ncTVm{*+H&u6lh<QAtu`w6u&4=ia2r
zpb+(Kweha$dy3qSO_};A-Yn2*b@*c5#Xl9fH=p14<FGu(U%6L_w`QuZ^80nx?0m(e
zt)46*3@a)s%rBmP`pMz!9^q{NB{ME|E%y8H&W4RuSwK9<fl=`=gMxhkLqNrkl8I~E
z_q^$TG9zYJy+n%^Pv63e?^j-TStoap_v`N&DibeXTNb2j+<%YZ?6NnzM1QKQ*lg=Q
z9-f??{n78;*)Mk|Z#D1WpLK>`-@);r|L%u*ca;M!q?u2tiP-D);i)mpR3YUwO=FRW
z{a<G`T?i^GPh)UMY<y5V>%m`khK6Z-r{>K1o2c{R`VaT#YHjx>nrt$(61#TIe8t^~
zACBqXSNDtkS5%uIc>GSm`MkZgn`e4H)R%kl@>`D236sn74Az~TGmqtj*$h)bhKA~I
zhqdkWjsEc4RVb=_E#T>`<93d1D^Dx`_ts^ZTRelnQd^PZ`%m!XuUvJVOHSP=YFnDf
z((e1um>C4Rcv_h_Gd7xVe4Kse(dn7Fy$}9+8(IG^&hCvie8TeR$(2dpI2pV)b1pWR
zD$>Cyk#fuU(^?jWPv=aMp4*3;y#8k@;wL<FS<#cn%O}lu_u2bw|6@%pkqLX&i*FL?
zI#OA);`7svG~sDdQ(7D|v;uGbJTuK#<>X3{hNk08rKcAy`g>f3)qL}_J1Ptf8B0xT
zMU=Cg|K0y5b<tl$F81dKjSaITPp&xSdFe6BOd*DbBM*)w9$aJQ=5kIsa2X53yd6L4
zb|%GVtvqd<Zk--hc!VR==*)!|+dbC(e8Rv`v(MjyL7L%6;%a-1vMaZCxQn&uoV9v%
zJn^Qa?yFndQjHo_8c(wP%bj0deJ?w2ruyWWE-$3I{nwwcab2~3?%EXxR2_rcK7Tws
zYo${6&eG~FQ&Z|ETwLS4*Xp<9kFDa<4lnxkHhypHmq1QC{eKgsH}A<y&wXgUZ_YNU
z_m8&qyWZW({=##U@Fx9_Chie}P2c6i&Ipu5+`6zY?%EknhBK3Xq_XR;%U`eS@$|I$
z!+=%>hOW!kSDQ{=exo$vN%!~P^H=P?y;*d|Bd1AA4=k}$Thdyy>T+B+!;1Zdo(I|c
zHpMMoWUT!u#JyOgVbkoG?e~l$m6O#(p0?+nu{^qJd-}YbV@u7w|E_*_fAKS+{XReH
zWOV#|7AT6eylC5e=f!-Dp7Owiy{B$%`+S*uUyq~x%@x8-h5@bqmtxv$Jw!M3S{X{s
z)>z@W<My@X@Avs>oj6%pp>@4>x?O73uI27_+x_Ow{Or1XU57`)#-PjF<<EBR<8}~I
zSidc!e(_y)Da-AOKbFeP%XtyDfAzcLrtjX1{h2N}^WaNM&wnTH)#wTQIdjPScJaKA
z4s#-ZggMn|2`!J&)Tu3xNsnNz-s`Vzdr$x4pASv^n;6>UN_$>Elf5yu)UGoiW8J-X
zrSJCoaDB30C-?Q)4P~dZ4*%ZXo1<>y^+zZ7*{1$FaaRTfrhor@TUA8b^v>Mb^X~4+
z6s8nc|9MOd2bS|o^M$;8FRK{BozUj?zv0587pnr8woMAI5BsP5?ya_Kem8@|C$&H(
z2TqL%Q<L}9YAZc8Etg%FK67@pO7CU2xmEururRDz@W7Mpgz2>D=O;0U#9C|i*iWsu
z<ZSSC41CA5eY^1SCBKeOdUE|<#98TJg(W{27@CT=s~nh;zj&VbS0Sm=N$d$LwFP99
zmrnZ{@@vb3xjeeEYNwAdthl~Mgp(m4iGg9}HeW`Eb;b5i#Zvw?@A#ws-sL@)!<^7B
zVNS>FCNpp{uyk`W@Gxlo-ua_g;u8yl;$kj_N0S%UB&_N!P&-p~>e)U0CyA~6PAi%o
z>X`o$|HPx_u_^y~&Q2Bvhb>9cD;OCJy#K8`GUds$-%A&mB`(>tGkNjUS9hGcGOkSG
zbYSX;t8eP_3i;^!BLBAP6Ap&1UE5UuN--=9{-xR#U$gGT=lZ^Q(cAr(BepR<aD8}u
z*{a}ao<}-g_IDe<n*K#kh<TyjVZ|dht9~drx3ONi%^!VD^!}M?Z|`+-E>%D3T^fGQ
zkdcAmtAV=FV#n)tys=03EatBNAKrFqdCn5k%qF(iX_va>^QQ*hv+SB&Znjqb`w8>i
zKYIeTw0r{3?N$;h+U|e4*7e^8*_KtDpdlD%uA>YNopy{20qZL>Yr<{^zWVl-V{Kp5
z?zCbNRuP5?Nel~)s4xVqb-OCNZsz6$r>YdOiyJyM-YTCk4Ze`dz_2{o`-HE?pAU{o
zY-w|M9epd_U#Plssd2I(E5joHN8<SmDN}!KXJ@#Od#pj}-reO2ohPb~P0m~tE3)g$
z;jh<uQd1wU{d}}?y5ey*1}+DsC&CL>*{m>T*kF;vV5w!@t}x4d#`8l8TSWG5%Z$7>
zUE>6!<}!wc391Y#J4FsMF(hPcn{;)m`i%Ix6b2{9?u(Z?7!_<p)KpR<Ki#anen0e4
z{vNXx`S(j+E@@Niy-{R;Piyg=;*Pecx6<AB_vG88H)(x+FtPh{x3qkSv}s_*ny8<k
zS{HOOg~N>AtOqOj8Myd9o3xcPG&D6QGBF0Het4rMt!DU6PAlVy#(6fk%YIsY@3eF+
z4rHy$J2&y7^qquzOuWIO|IU79wVw8LX^<3m?W?u7mahBx>+Y*qugl`v8yv4?_55o1
z_D1ltX;d-Cgz)%n4QGGeS$XNpPA;Db<ujJ5Sblq#Gbu*>eA{X>lOp3MZ8<s>+Kdf<
zPV=iYrO(fs{A8+zVa<;U9YKaE>G@&T69w0Z@rIq9+Ier{@1E5c&Mxk}FKC)8HsMv5
z7QfxnqSUC{uVp^0yZqzp@8C1L%6c2GhSYxPu!<{VIkBp;@1cFI_l79bI~I|tH|jqz
z6zyH6@yfVmr^5Zq!PQ$#Ss5nHzO$X_V>8E^4?n}-pWF9o)>L%{wm%iYCyLHzK3;H4
zENyPp@h~+F$B7&(4qZq+Ss2;1HY(Kd3P-3})1nXmB!3<I^z^hb|GW9O_MChle<<gU
zRM#c%u9f?aXWX3i@#D-Nr(fwhZRXq@GV}6$#VrxLUQK$EziKT9U%;M6)26PFZ*j~p
zOLXA-*~ql&f%)MClRT#Up#7S!l|*LtTBH<;N-kWwGrP6m^N~GMK56g&bac@=L(V&g
z4&DF%Fl$L=@v|o~wkBS?-<|hUDv4&v39FI)m2i+TA;RRYZM^f`NP*h81=pkJPM)?a
z)A8KjJNf4r85oYfWRH5ee~<mS=xX<dX<~|}cKk0od+E=2FQd}W-QJr|g}QkCx?3Nn
zA`tSzzcb+IVMCKu3{Pe+xA#2Hpx{}jrMGp*tW$n}Pk-TI2$^QFGsn>6>(UoL-Cyq9
z5o~0=U&Z#npTs1imy!%Cm_GTcX?(r(jXP)0@n}2sXYMa0|Jr)KS=~9iGAZDKad^s7
zwcUTF7w_Mr-YUqu)yK4AW4KDlz1fx0#l@a8+1^Iw+F#6ZYM;L(y=9f)nF}0$cQ%E$
z<`=8V3*TFRQ0m1g#hnvt6xtTXxqmXy`R{I>xTP>4nR~PTeMW`{_bMzC9Ufhros_WZ
zwr|$Eq|m7cl|noXoRz;kUy;7k+vf7KHKL#DYW03u=j7@#On7Y(6uI)xcFm)5vI`zO
zJRZKp=f3K$%fFZ%5?}0M&pyw^7;;N0By$Z%C?BYKtLc#H$>5;#>rd`y&RyF!oqzUA
z)N!tux9w@m_KX#&s{$p8otYH6>Q`piuFA6(V>s}4yGX+aW;+Ivt1J=xpe{hsb2sC5
zt<8-qldqRZz0S4R{?v;-?O5v{@kE<-tDTago1ZQ^E-hbM&F4MWci+AeW`>68+AKHQ
zjQLil)=bF0)p4?RqL6^!_x_du)&o4f0*j=-yZu_%o`1idDIig^(ZXK8Fe2ik_WHZO
z_Wdimw*9%op-0_|9vz=la9(Zayva|rH{afx_Th5(q)RV;ND8c~n6XgM>C5Cj|5+Fq
zDj&&TXPIK@K4r@HuS;vV6#NVR`#p8-2)KIq_XQ5cmI)?@K*LoWp30zL0C9g`SfRt#
zTxjvOR!{Zn{Cg@>TzmpI^?cg4c#@ROMveU|I^JmrI9&yeD@<!|Z(p`-*%bQ?wuXiu
zKRrDiyxec8^@R8tt5$oLb9|XRajNBvX<tJ=J&rZ3<JX+np(12?ZU4S(PEXEwjX%fT
z1XeNaC}vwVO)qxWmcqx!yr=73l;!xf_0OL_Cr+HG&wqYzZ}oEj`Ek2SUhX$vx_EJN
zO3Ih_ZxupTg2t;>#dvvn{r&az_xG}t)YRmpB&{WyEvr~p?z`S0;KXq?3KRewikU5v
zQi?4CliUWFdt{L{L(9K2%l+qTbe%eXUcA#q-oB27ft8ijY_@NC42NRKK?CQ-uby04
z8O+SaGhymfR)(%cudc3E7vY+1miz0|Q}12Q&K%&LT&Ox|q+;_;rp5(NIJ%EYSrjPr
zDk`=Fs2==%VTIMo=<Vw^ZY+GMylK;><YPSx)umblR{6|W7-(voc!<T-_2{I0Q&Ur3
z9-fRIcEuJgPS7Zeg{5Vu%c4i?Z{}>9JzIM3M+J~!JiV(VQ&XRwn`<pz?<;WR#6;zn
zy)YerL4z=!o}3J`OtY^o@e~G)b1(F2l#~j!|Mu-$(#9EQ)BI-`G`@=va9XhmGz!!|
zXU?3pYu7p`NLZC<+})_q5|DbZDkDTQC^)#CUtUgL-e15ab&LK)500jcD=Pw1P53~Q
z)~~OxKW=E{)ZxbB8d!3?A!}Cfa=%__^Som{l8=w|F1DVw|I4SRrwx;jNk~e*y}Nt*
z;fEJw{nm;}OIw$`2nY?mdg#z0|M_;N85b1n?eCvEm-qeM-IDgUfTxEHO;&j*P0Tnq
z$8zDqg~$73jm^!MThG~leVOm<*VosdpJ90T%9SraK0b~p@>Le={`%_b>Ph*=#>R{c
z;T(%@F}!zP%=P)%+1Y+`t!%5my}7nF`l^+!_~hx+?{Ce%erBfe?y|S9o<Dalx7a$p
zv(xi%8?U!~U{KI3%VM?Ij|w5F2F{DS3ctOHEH3_B_xD$8OUsItD=*6WwMyF6{5aYz
zZk~6?qUy_wLx-Gp9bUQgSf6b4o>(tVYisLWO^)Cpwf|l^V&~`CRzEt@>E0(Z(=0bi
zRhM^h;OlE^^Y83%oU~-;&dQ)5p}P;SxGi?;O){E!G~QUE=U_AY^qTGf-Zn|8zmvaw
zDUp_*-O|$X;>8Q_{HtGTU_ii)-R1A6Xa-xAzXRP5w(Ic9(vs3r?`v<hH8ghY+I2~q
z-zgL{$5Sn?7h_;*`u67L=bggpCg6FrOTAL2ukP$DULCf!sJK{LOKXw!oLi}Pb`%yB
z6)`BNt6yK}%)aZe;|i^WHn;j_m6Ja|Ki|E3H#-Z<nK_o5pPkM5-?49BT}X&X|MATm
zHhB2?Ewi2zmGa}mLlM`F5o-b$yIq~P-#p|r=rFK1cXyZH-BsG%+uPgMw{6?DW!7_6
zEnBhT#q;OpiHBISvtJ)>=f5oLmwR!6W3yTAt&XFQE-rSzEX(Z_$}&Hq>ED^D+Tm_}
zGB4k~vr9iG!?rlA{!sOg4-31yyVtILyDj(jjU2PP4==3Wu%Y1WEYp5@`*lmc&9|+d
zHf4%Q!hr^_toXRNeLI-;O}gd&!auRiZNInDM4i}OSFT?#|Nrl=nVDHhyRzt~&!5$O
zXMOqe^Yi25{o6AxHoYr&AtEPdSNSO=JNxy%+TUSu`mC(1U0q#MuT;3X9c$y2o-=oD
z_}ZwYYu3bg?p65m+rWA8{Yf)tTISp^=yZAX{(b!RyjakBf#2WWM(->F<w$mZIiGSJ
zlZxNp-aa`wdGe%5d-m>K{o8%}ojZ42yTxL6m1Lftrn~iXys+ogC|mE&y5N+QC7qp~
zzrVdbKgY89<xkLxE9=TnPwwn2R`;EyBEt2OZ~3c^qmO!}&9`M=KX>7RftlI1U7)$+
z@bLRPi_;w(4t#ogy87#@ub_fR*$FgK_QhDJGv()}r-$45?{7{&f2fr^`)BX-`5D*O
z#g>$mxVgEt^T~SIyX<1??(8%)HQihA@X*%m>(9>44u3W?{%X|kZ*TMO?z*}%Se=`@
zd#ZN$wN?JH9;+Tm>-}2z=FOY9y;ZUMYJNUDJNxdg(#6&?nadX~5)u(Hsrgah?R|Q~
z27_IPSAt4V8T-0F2O638?%Crv-wrf5xJ!9eg^pPEuP-kzUc6{%Xz1qVc71*Pa%-g!
z>BKg-r`=s$pt0xOWpB^TwGQ83SG()*N;5UTITBJ*ubw_NEqxU-(Zi+OLR4QtVS(>#
zGZx1?d#kVCygBn-fmZlpw_YbFr)hezRz*)v1TJ<<O;5ipE4V5aG+3(96}7i&>*md$
z&&{=-t`phB@9-~p*|KF9FJ4?9xA)bJjgMbnU;mPC`5_f`_43!(az8&id%REf_NLVC
zt}d>-f}s=l)&9P>w>sR{*Ec@C{{6kZJ9deMN}jp!LfX>Ovi8>(%i?Ee&dxU9UH;y$
z+~VrgRjXcIT<p%yCzFwwC@3fAS8nmY>-=;5s4XW>omy4?{vKrc*+T0@tByS^xV9#;
z_}7=r`1t#vt5xI9nma8uoURw^RnFrPb!S)U>gw<B%*@Pwe}BK;R(Z*(<jl;>)Kt^D
zKR>>F`I2{cSLo%^vg!Kq+qQ2{H;_<NRJ{6q`3iy3*VjO|!+NL`6&IgAb?ViPjf;DF
zc<wqboW=8cp&$c;YoN4w-klwVkJrWSuKNDYx7@<iD=+WekB^V-Yk!Fd2`yT?*0<cE
z{=whh-!Cuo?e6WpyCpOD{5;#2eC}N-si|jYo4^0|_O^A|n}ULZ1=cbl$ye|7A6{|f
zc)$Gbudl_Wq@+OeTsse~T)kS}y6nuod-u+qdGqJz=aTl%(JDeG`(&+eZOgsAB5*Ny
z)-N?9GxO)KU+->h1*HmRHlB*#-*iFqKb+}1vv*sUzq@hs=GOfC_s*a9_x0_4SCFOF
zq+nI@A|Wv`(8l`K*6i@idD@_PfCfh9Ns}fuH8stdJGW5jyZbDY%tx<Y{d#n?yYTTb
zLkS*kZtl!~>8@7=&lO$Iir$tJxjBvZ+?CLo{PK1?cJCG!6=h{-H_y1x@UGz0sq^RK
zcN9FlxY&KVemrP#3MduE?Wr)#y|sm3-tNua-PbQ&n)I$9;B4dkA6YB?=i6CYS+TLQ
z#_q3^y<9vMbbU`m#EiLf=dN7&a;f+97ktm9kN3-8zkK=kmzTjcH8#th+h5vK`T4_#
z0)2gbclYC*!fG|YzI49p2;gnpd}nK5P0gOUb7Oaxz2%d$S+Q_o;KttN_igw7zR|k7
zd|pA6)U~Pqe+e4=J*WBqLU#GO<A$@g=B_yM>zUN2<SQSYXN%3xe3Y<Q)4Rq_!(;B{
zb;a}Uo@lcyY-i?wV=*;3eff8O{%ImdTHbJV?fP$FWi>bBN>sFDgO5eE!H++^H_Q{(
ztUs%?lk+)$n@XWh^!1-bobS(Uy6I^p8@)R8$!8tTrVZP+nVFiFmX@A9dv@!_jSFkD
zzFb`$uI$#s!OeYpbNcxsox-9bA}@NE&$@K|y1Z>wNlMC+<;&CeR(<8+<$c-55n650
zwytzr{{4H){pU~B4$r%@W24c$rFSlDnP2?*Pwnq*I`(DzuRMIi@croTZ*K!O7H`W?
z2>&LvY?fbbpv$T4{NEEc`ri4Uy-aKQipAGks)YXkSwFv5r=sF>C&P`mZ>z=M-r1_%
zb@}9`#KQG%vb^7ZE-Tr-J%#(yp>MJ`tkQokX8L&j^`}X-&Yz>Q3jb}pKmX;8jgQaH
zHqXDmFE=~;_rJftPrZ0l-r=(7>gw>>vuBsTxbX1X+uLDbVI{6X#w%8=$ho`A)WV|T
z@v+`*+qRuNd9uX4WtI7v3tKs+pDryc`}X;BcTbOuL4w26(kl+zEg4ka+En!auAJ%|
zcW9?W7mJ(t>1*-rZ<yq!pMEM@zjkS;e8b_S#)i|%-Ta2H`DeaLTyivN(^ctf`4Ous
zC!K!Ebz=Q}E)RyF15Y_0%wA@{{f}DGZ;tcPHVhA5_}wv$`&5#BWp&e|=|=D4-Wwcu
z?ssO0zI`z{D4zS~o}jx+?%Ph!vR|Uw-f(K+(xs)ZuB_~nwVpL&hKq|!NRYn0pr~kS
zN{Wam2$&`x<B1Ur3p{h?%%Vk$s=mF+OikTd@iD2<;lt}}0WJQt7q=e$`}<o>O)V=c
zOHfeIcedHuEBjcO`mR6P`E1hbc0D^r28MetbIZf?X0spoUF22Q^8L&Iupe*ZY;Rr2
z^c9*e#N6;?*5b_QU~O;Z!zZub6H}RU=XJ&fvj-2CKN9QG*RH6}p834#ZnW6UZPL|0
zLmi*py&TLoYlf8M4y)*h$XR*&r?D}dKlkoy%%<9_SNe_x^Y5*d-kbPJ(d&h6cI^ji
zp;u)E5-tYQSnsGXGJHD!W{2&v?f<4v_@UGg<ew>chtuZatBkOf{Whrv5}%)+zrVkJ
ze|PukOG~{~g*;QgeGB#TJ9lGaay#TYy9*ZrWM${Bdb#~ug3-*$>i&J5ot0l+Ol)jC
zI8{6RMIT3~_l$+vt#@~qe}8c?`Q#+k=<Ru_n{Q6NC$Ab}^y1a4b31pviqPV`GP@(#
zs(IJ80KH0E+oao9nvdPS<~Mt@S%vfM4T*7yP74?k{ym)CFRbTpzS}hS=FDY#f0w99
zd^pRSeUJ0^|10eM^KDW$NtfndTjjlKS*+EnO%dBpm(Ta$n3wlfT1@-I&Zj2tze%j`
zj58MRSijEx*Rr2qp5C2V#a(_@E@(dIfh?n(O8bW!Z4GydOs&h^{foo+t31!d&qB<P
zc;EJD{ZOmjxvS)5(dTDpPn<X*XIqt$ni`rFFXH<1*RO4vm)j0MJaYW__xJbr2kl<{
zRLUenKtyCq-rZe=kB{+6o9P%EPkz@CVA?FXt4&)&L&B=$#k;$^K~?CJCn2GCyIhi<
z_AyTndUWB#ZT);XOWTqh_r4uZ+%`S<xO8%rv5zGKgHQ79z?7IX(Si(%y?2Lc9a!_;
z;M&xl-P1X<zLxHnJ^1Oqi`R+%m7Le})L-Y{+K{?PQhEkM!o7{W(hW;FLaH3UtyX9F
zVDm1mhAaE-uCse*X?3-*G6Wc%zp?6fK=G@e8?`@{`?c@(ee-9rzxK@X^GnO5FHMS#
z48D_kWp()bH#awniHW_vwRQELUOr)AVe9gDZyrA`{`BPJ*VoszH8n$n+*fRg+go+?
z_;K@`8y5~7NVvW(Hq<P*DU0FE1=E{{+xdNcd@N+nZOy(OwIw4k^{<g*o6tm;$9~<r
zPQLy4!d&&}wsPgR2o?sJ1V#q$g>uH0MmsB>MJ-LNWthOo@SstG!NGO4ox!WsHS=wb
zCMiT&SnzGwzWx2h#qQUa8SOmU8U7=+Qe076h?^s9`oC}AzA>}ixcz;F_Jc0F_AjlA
z4lePOU8f^&o#%hqaarl9vX3(rtUet1ka#R~=W6lhgAqFl7WVg_zp&6bd|gcBMt&^~
zEv;GB<?DKT&n|FmR#H~(?dj3lQ253B)2C0Tr|a)8c<2-m5YWyiD`!*TQ0}s#>Htq{
z^4r_n&tJP%c5e^p=ECWdf9L%yNj+6P|CI7khB<MImoXgJ(pe_MFs(<&W8KoH3=ErR
z#>E%r9P!p;WzbjZKW$-YqUOR7V6*(MPLlBL#f!xnKJ0Xvc9e}lGSZbHq41T&q0<}Z
z?aQ<&JUILHyv^skZt6%fOh}zNXYsnLGf+2jPZwZf0BzlV_uM7#PCg664ZRn?->$n}
zr0VPFnB=f0QNMZD)PJ*o^{rj|_UvqP{g@pWR)??Wm$wV?nEm|R-0aKC&YqsGpPZcB
zJ-;qg@$9r|b>F{O<>%+;-*?~nD8KK%<<qxsAMed8&tcpD#`MYY6%%ad`FT&SSZdeJ
z5%heEsm;8Y3DafXKHqyM_a0~5&C~xn6x32~Tgl7I=f4jwztmmrYw`O1+qZjlRooaD
zRLmc~xtF_q-n3mU<!Lv!1jk<!3e&N*-RnJFuUFQ3Tl)EVmzVi!Ypj?O7vS3@xhrj!
zLE<6M>bm%SHYq12XuY%g{O)C|F<)MBeATPGWnGc-<+nX;eP4HMewTZvN`Rr}TBZFd
zQS&#J#ot3_KAijP$4{O3KO4T^w45utwKq2Ej=Wj+-1ECu-v0bNc2>eXfo<7)U23lW
z|M2#<IRD%}h54~Iwf%-R1+JT|@9ov;wf<sjowr%q_+e7T8iQ|#ep+{y`?b4II<FV^
zc>C?QSM5_~cCT!FxuKek+f5=vx=e9$*sX|nXMP$@NHfjZzbLwzefG8ZX0^@APT$;*
zbv|0;H?ODWlh(I8pp%%UZtME}{MGgK_5c1<Iy)aOdwUDi^^TwA<?k<VQLte7^5@^)
z-sYFF2-u#NTKnvP(C+&#0!LN^E`Id*adC0+*Dqh3oSn5-Du)<92)55wtE~LFWQhuB
z-6iu+$)E#|IUOQ@zTUmC;kYP6^vutfx8^KqI4-<l%GCOzpqbNHZR85-?rBV5nshQv
z?0VkDqcwUbG&ip<nzUPHv3vi!dwYNX`1tt$qo_$Lpe><US66WgtAYBpT+`S;&6hPS
zj$-Z6YtPFw4^(4N(0{n#YX|Rv7e#NrmF?A+FK4k?*N`Ii|MG|Vt+OoM+m7bu>u98C
z7JJp*Rbdc*5V_&{gU4dq`*@!+FieQi>p!dc(_E2NWil_5p8URlTRAH3=>LC_uVi-X
ze8|qa{5e(*X;=7eciOE!l>Ey)l%?TJM=$sNueZz(E!Z(vc%7JgE`M9iKe0>7VU{sw
zz44b<bNzd}c=2LeTie;D+1rYrpHuUl6;ju};@3gLlCwurjGjJy`m*nl_>3ICmtS}o
za{4~$U-~@#bX{H0%xO3Gd!1)E@aNCH^C`wN1GGdhyjUMzboA)anKNh3wX6O0>gsCH
z+T5<Izax^qKVP$UZE{kQj;`+6GiR=?iBuNqT(u-RWNt#6+lpr=PM@9)S+upxXJ*m2
zH<7WicUQfZ_I~qkPEXI7Gc%1BFIu!Net%w0PR-|MXSH5Sg>uhWD7Z?|$=UhqmoF|#
z7eV{g`DCM>{!Q+@8-MBIMa#lRN19l<+xTRy%HG`g^XHEiAA_gvwe|jv9HH!bdV1=9
zb2e<;xNzzBOR8nsHorFB((lPTB4FRVNwrLw$6~(sn|n378<WMS-{`ChN=gr^QMdjc
zTd7)h=aJCvN2@o-rKZ&Kz2tUn_MB|h6KVHNEBnaOPJ7*rvmdx!f7{E*%y#3}Eh`I)
zmbSJ&N#nL<8WTzm8k#JUw^zQFK6iV6fB*OEt7hE)w$$!<`18;t(cUQu1`^AbEi3uj
zr&{*k&(qU0K0aPY?EkNj5bFmLQWyQ#HMSP77dKN5tCZVw>;4!1`%3$-JZE4i2$p|#
z)i7sC#JNoHahj{s!@@*;b;Dzl#E*NdeZ7Cheo!tyWTaa5b(&A!?c!(ew(LA?eRif@
z;K!Vx_nTi^Yk!&?6SYR#EPJl|;qs41_IPhTD}2ao&GZi4zx?KJm)CwQVfm@^{mb5U
zhnd6gNNij%{b^X!&F2MItkRq+D%O0EEBx12ckk^7k57^BO#gkl8~0aTub4BEZBw(k
zo35FFdWquowOcmUh!!R171}x%cFql&DYW~7@WjGJ7Dp$mzy4yn=!MQk=M^y#xuM3I
zT3Y)mJ}$Ddy*BZGN!QN@d<O#$-(E4La%oBJx#!9;>mJ;GdQF8v_|svV-D&;%4DM~=
zW0+u*CwDUJ|E3qawN__noqBR|^6l;U|9@W%li_=Qb#-`G*V9^uiBb_s-;4kM{T_e3
zPxkKJyDxc}Jvn8Zl`{ha&b1%?!}0#8<NK1RLuVI=Tu%7Ev+C3#iTzLA?`5r8Dp#}Q
z%1TA|_1ig<-*1xr%kN%#-g=ux(|M2RV;^T-^{lw{defZ_F@L_#Z1SJqeAwA_EZND_
z>%6Jx<g=QS)An0`sGk1!m5AB%>z`GZdc9$oA@ekTPG3OiVV>S4;f1w+XZBu{nUS<-
z%VC4{Vdd9Mi|)@7{8kX!E%SPtS@K=KH7^!U{$KJ_FTwG!-(s<CcXT6n*Hw6*R6P85
z?Y7v0H%3bm{)*}~6vZVu?0NV~Yt14@j)nE@+D9i`(OPczT-BT(bcC76!gZN^EnQme
zC-qjI7wl$eNn&I;Uefq?<;k*l_rl*Sv(ma8?%FKi^q<MtWRj4N>!aB`r_Otv3~Dio
zPB>})0&)Ub(Ei-&kYzch*M8Lrb#OBLYJS%K>&*F^+jq1-on<nqt#_4x(~J9wZEiu0
zr%rb)Q91DI(~Z=)SMTo4o*MmKJ9z%(mZ@JYZYC9XMC(6~oUL|4?0%-s%JYm2iGd3n
zejlrUc=4BPd%NrVg-0Cai|1X4b8X(&u3E<NSLj~5+ZW~H$=lZ7;$t}e+^;h;GMtg?
z{G85&t??4JCjxEX&;BK|VBYa#Ztr9y0&5oDwfVo1>w4Gg;#+5>Ze4IXR=g$m&6Bg+
z_T0UE{p7bblT$1!djCBB9O0_my5R`l!uUfxy)TZ3T%Wz|?T;-%?$_4+{o|^>U6((m
z!iI5&RxyKu?V}~9c=jV5dv!14|C-}{o4fq4&8~iZPQ<YNFxv~mhy#x~6E+yJv)s*#
zIQ>v$yU3-CtLwB6229aU|NJR_$_;QoD{zmn+l@nxHy7)Rq`%mtT2_93=`H@uPWf{I
zTU#ZiJa-3$nlxEAy06pJyi<4LVv?xs&cbOIH_!jI{dv*8H;F&KYMF&y^w8JY==?(6
zz<F_2i*3Pg&6^K|!};ZzOk=H&bN1<ed{^Y#tXjq~^#D)rD$%}e&8wF)-wj;g6&#cF
z{QuS?c|~Wu*A*3Z&V6@X$U3YxR!x4vjw5{{+zcVU;y2dM&zWwSca%$Kqq8aNx~Fdc
zwUj^Jit;xP5BK*!zq3x=xn|qkhyQYeUzYdzOIlnwS$b}zb-tIp`cBP6yVG}{&HWL0
zy6Mi8g#7&~_vYIin)XC(&fVQ_uUwu#f6By)%Dda`1JkbT`{~&Ke}VEB=ZK{C3+F+L
zESh>g&b^oX;fb5_v^Sq`Z%aKJvXx!B{Qil*ySE>0Qg^Sa+*2B9wY>k@Y1X{jb9*06
z-*Y4+FQ)y!%D<f(?o9hBx$)ocvNa!ie>LkivScRy=F6G8`ETGse?6fi^KG`2AAY0E
z&+YY9yW`vU!0eBUWqB^j9}u!`spCuEdGY?~YN1E_o_=bv4R?#U#?0{GN_kAxtFKpz
znHbJ}KV9f4G{a}o@yDm9>OBAD_GapK_WW~34+W;>Zr@{jTYYs0zXCgh!>1id)2jZq
z*>gFsYivE9^VQmM^*&24u8NuokN+g7vuxY8__&JJw+9#J|E<Z{%$Usk<%v@}r`O*z
zos1vbR+m4H{=fea$TggxJqjxguQyy=+FkgvB((mHkh|prw+emDy)x6wtDer-dXVoN
zUwYqMMmO>O+R>5^Hy_K{c_-iVL00->^@H4P>vAjpoy*CMWMELZq4nYCUQ_jlKXwFe
z<O_bCbFZc|HT&b?0G9hat@rQP8031ta;QFbYi(p*__nAi#`;_gHG&Q&-J<mM^WK{$
z`@as%7Mp1+ZD?8*I>EpG@Ggdcw2rxvF$bE<&HqbIyP^2Sd&a_x@u%yhWY1?<ZoIc$
zS+s9gRn?JIn@$&+^BTlUU%Hj}>y~P9+19s{e(X!=xm~k;t>8@=RfYt?PoJCI>io_#
z{7Dc06Ctd0YlYLn<nMx~pWUs>Sm+aRYL7(SukVjeb#I<OMWQahR`}WRH@dMx_977(
zEB}33lxDk}rM}8kN9CW>x`e+km1}2B3tqyl&~c1uo>rr383*INcDMfy3mX%;JwG+`
zHyOR&eCp=MN4y&v64Npluqv!xcbY|vA;T(HMrPlG61|l9`<GnYdv^NTHAh!w&+Y#i
zBeR|};KQ$X#gXT)onm3wkn+&{=~A17EhZOJm>O<<apz3Fwc_8i)?d!ge3$yktM2-u
zV{cnO@6oo)$C>7>IwH5@?-eiQk5>=3_qEtxcx9HjZL{HBMa2-;0-oM~Qo-)$1ytN8
z#ee=1HLdQM5~Oc7)BI4w=Ss_Wvmfm@-YC()yGZ}J={C6=t~M!ucj@#tzMD0_KlQiY
zjhtG=BT?O~3KKS@IDVSs-DD)pQ1|_2Z_P%BPcbS#Y8YqEJ{o-e*IvfZgY8d5f~MGf
zK5~NZTyDs|`oG1D{P*szNs35n&(fB0UcCSMj?*0CO*;!dd=q*1{PxzSHSXSb%}#77
z(AX@qUDKV7Awq{C;Uoh`w}1E*>A&%_jT`-|Z<;VOH2g6*e^38cpGi}B{(ZGy=kES;
zJGAh~!s(AMGd@VpV{;WPI4R$9D2egwN{jESGhPNPij{88d(KtCd9b_uSpA=arbR{!
z3elewLkrTLs?9E*=y-YUbpHEOzb#cZvY&o&&RHgnd5?}U%>%hJKooRI731!ccV+}{
zX1`Os#i;D-9P8_sGk@&S_LENARz72E^{O);{EGk7Ow+Hs@%zo3bDZjj9;p;oHz`ee
ztRLee_V$b2zdjZQgKhE`cfEh|$aK5m?sT252lL}Bwng{f+gr`wmzkZvAiX{z@V{@K
z?QJWmUGHmrG?w3~neek#BqcBCz%tpwkW=S+>pXWR?M$m}4&$-in--HEeERMDs&nra
z2m3uqDRVjT?}SBQ%$1XqC;GE6d}RFmEH>lDx`}Pu_DQPp6@9S!($_NQh<0?@JN5ao
zcZ7wep7xECxh}z70jg%SoCBWPh>M3-cXf5?$L$FS3i|bRrD&tbhYFj<8ArqPw_at`
zeDIMapz3q;Cw<kK_A5Gm-QtsfVv)m;06DkZC_Y|<adqR;%*{o5!W<O`9&MPZVs82`
zQR+m^UJsQ?*_RJ`xW?Qu)Kq9Z`)BSSr%yW`P5SB`n(*n#$+x$+%iGue$)9)rVL{y9
zs--I;?&<l9s=q%e_*!wb+MAt5M|a(up8S5BpRM_++rEF#d%scIbA9!$^F|K^9G2>u
zzPYUO<Jj|3W|e;Jw7s^L`9BJ~qmQ>u`)}@IE1$jP)az~Ww<rDPv)A9q%%I@EBT8ng
zt%T8`2kQ39|8EL@PCPWDda?cY+2!{)Ufy=Ku&!$A{<i=7|Nnb^W8>o8yQ@>nTUN2k
zI4|ZpkWxJ5>eZ|JeUJH=Jr92=$xt_OUia;X8)hCo-zx+<p?u1;KW*At|BcLc|LBbh
znJ|0OpR=wfPv&zUy)wOT=KksP7#kv|zKqQJyW)r+>kh$tm;XmU++O`P!+E)>XN;Cz
zL2Kj0i+56sdv$bmZ~OM06*~Ug&)mRZM*SN`1{=<}3)}9ln<kt-TjcrmeD61GvvjQG
zp4g@wRb6zj-}&ddBY6)Vm?j>4-NU1PJne7Bl$dQt&K)?AaN*a7mX2PwTv>nHKQ_+I
zs%0Kt54;RR=5BfQ>dq^E{fXHLj~-<OX<oeI7Ln8*aP^>}iPMYkiJh`eq3p&Wrs(t=
zvEFrOe}DTiUC5f@$EL?euM1gg1SdV;Uw1d{;sH5(y^Y?#H)Z_q_j*&b-)8RAGiQIj
zUHbH>YS}|$f0bube_Ktvp(q7vY&|@Cw>aC@w!ikb-b~^84QoHny0Yo;*8PoFjYL_!
z-w3QSs*soRY(2KNRIy0z{NvdvXV$HLZMSh=73-P{f-b**ukd~G`~2Gl=T*yi&Ce8V
zG1~Ux*#cHg%{_hI=4WzN`@K8&C+&E5i?DQ;_nUvlIy+|Q?0R0i=Y8DwUp}DB_G_)v
zOCjqHiRhZ$I}h(!oL5!FE_PS)@wMX3dd*%F4RlNL>`s)J=|(?sd!{CPZ~O8cNAjLc
zxmb1f7u#A7iH#X|UYcI0JGE}Eb6r4^c$JcopUCoxPbqWv?cM##o@<8rd#A(R*`_I9
z&uVPk-W-1T^KAE%cDpy#KCk;U^W(e;ZF+k{#E+ZWZIbnVBVfDX)~d8skFr*G9k90A
zFgfLI??Wq=_p_@cC$*gKUR$|yr;5Al&(q#}?!-J;F=dYCgCbU+KBMJ(!dGvITE^rF
zx<5c?<7%FW+*Yr{h9;B5TC_Iit$qILMg)Vz-ght7#FZW{TNCwUcKMTS@zEbIZ%q~#
zvb9?N_KWGw`pPv+oo;M;ng5l|qfaj=R?>1=XyAuqx&Dq{9UT^&Jf+;t_M+x<mUQTv
zX)Aer%-m<jnh8(8_3_d2>HqvX?@nLeddK|a%ch|2V=ZsmW=(q<ckG``Kw+os3B=+1
zTaUb(?pVjyvbe|iPKViyg%d?48p=(6uwCCS;o6Ly-8oisUgX^6t69I1f$P$u=AhrU
zpCtae>1G}{dVKMh*K$^h`+v==&6fUpIrGWv&(_yD*XW(uP<ivg(%*|*nb%C4UMIRb
zb(OWwM(3?!>qML{IsNGT^eUC<fJw1jzMrjs&HUqGzibj$uXrz4drav}dgXjkbJhG!
zl`_3@XY;M3x7tovaOrf2;loR<&$L?k*SS7jVV!$#n}hpizm?&p8=}}0Lwp}fNO=k-
z7H0Aph;llK?c=NZJa6u{vVA2lZe6=v`dNmbdGd<)QU&WG4}6o1b3W&L({4k<rMye~
z+G{^PTJWbQW@ga(cej^GH(oX3Son(L_yv!sfb*(OhQbU#B>mSLwwLBER#9hWI50{1
zQl4Dmwk;V8{ZjV&F^eu#_<pakxYE`(EdIXzx0mytuiZ5(HqPR0Q4ybJh4`DDA3VP`
zuV4R5#U){@kFQf*568{NFSmRNkB;J0nil0}{q3{u8i7e=GZs#qlCbs9`qyUS{_^(k
z|EJykVrw3&ZT>RhK;?uZvYfrE7vJx$*>&&Kg}=KN$VqOld;FBYe>(p?F(I)cqs&j+
z?x~@lES$gM)7PuKyRr+MSIuf!d2NxMveMJ9{c-0#dwib;GOXG4Ebkfbsz6<4r>6(9
zR?k`~6&B4{v-0ZCD5I}$y_3w^c@-XT-jFd9nI67zs&^RQ=Zctw!hM?ilC-rquigbp
z$dy~KzE`M>k=roU^3A08#h*R;s!r}_*HoC9BT(@3!`C^VqBLtm4{pBj?Ob}_-EWtF
zzdP4zp`AD9?mA8L?=v-~@2qz(D*n76WMy38Uh)3ZleORX_qxF{h0@X6iBo^~olL2(
z<Np6CK0#Z1`nf$aeMK*0+4t{>>NZ;<)aeo#8Cl{UsPuo)ca!z9j=Ps_F^{ZM5?A<p
z?qOr|^R|{8Q&R(uU9)Fae*ZG1QEizU+q!dNVbhk~UcdC|5qWb11A}>08sZB24?7wg
z4{qhDo4ML+YrMo$<GTk6in<zV%8GV3HXeL<WR3Jc>BlOC$;<O?%KyH()OhUiA9)6b
zslPMc9yqw$*-321*5docQ)hq9Sp8f<Iat}>U)@{V`}5O>d*uHxg!V65@#RZN(*aZA
z7sh8Ua9kDeo~HBi*4EeaY^zm-IPY$B_;s}O^QT``=Vw}^A7y*^@x%B0<mz2(<6i7P
zdpP;y)xUDt$10Y*E_=D+RMJho`?X4n6}z_PN32<NbHM}4Yavz}7^b|=xYc(*{cv5#
zvsKl1CRy6EewMq-P*G#J_-ez);;2(x;A<ZY&h)>Zd;9pYb>ZKyRUTkE!Cxjn*ZyL4
zU;FWIw-0`@URHfSxUB4*lfiGFm|`2dlC2+R+&>+E=h8Z*&d$!eyUWj?KmY#fYVo@p
z6*3zor4~vh9c*GX&%49H&Heh)($mfC{29Hj1@GUqCf<!=WVm?Y-ACvD$L@$9Fqodm
z#BjxQoAL5__CAjp8!C^R<uFWB3V6xN)L>S7=j8I&i;aJDiYxE_ntQzZ|DWBf_u1`?
z)e$=Myhvo@ndiz33`usZ2f~6w7#TD?cF)ksTE3WzVZn_xt9PDRyrza%m$z-5V=|xO
z^2?Xz1gu>Yf3jaxOiV9k$A<Fv_ZGYL?yCJ=2HHH^$KlC+=E4fMlb(CN{pFL{+CFdI
zy<TZ^HQ!lJUc9g<es<>1pPE@rg5qKUh3oBW|Mfjd`|R;!PdA%L_~qKVm!IpT7+Q3M
z8!i~!d7@ySfBRO$lx?y5bJ!!#h0eDYtoylY&Z>*&-)@3j1hL?U^wnc)9!;`4>Ah0I
zocX}c;^$}1oG~&so^Mn6=;6c0;5#c!&Rj5^Y;vaflGEZ!?y@JPavrCDzu&KLQa<<9
z7w+WaeZF2^LL4m1mMu$6Oca&A`u?HB_br|=yI)*6Tq0+Fw*1c_o~vTJCjHR<IFrHQ
z>7G6ntLRrNrsf!{DzE04e_7{t)f1VIGapOw#ZF<g__}Yum#d$f7(+x+%bZne9*>GD
zmsmW`E7-)*bwo#!l|f)~(;c04dvrc3S<64cyb0n@lYVq~r#<6|lNTq8T)up{ombkd
z{$Gu~{r&|D6m)fUqYgP-;xR5co06IN^4YU*FE6X_V*VMf!oX3{^e9C2LE+A&)4opF
zb0=oqf-5JP^uiAWtyr%e+FkrV$iu^<qGHFwg%9t?u3NY6-RsxYUtU}^&A!GXV_{%w
zy47j%@2z3Xd!CilZceE1_g)=W^j~*F-pzaJ87Jp)mc6d2v9a~fwcO|R`sdOQ5s7bF
zKbH%=pTpO$GCy|Dgc-~XUBArU=~T0C-o@aMF=P5pFaLMWKQ>-g=(H~Skd$Y8_@&q;
z;a!saH_M!Nt17>GGxbheZtRNc!)wm;-Q7{~;*hN4fnUk|_Ll2xjDFqRBY3(jf3EfC
z!;hMiXYINDaBp`_*~de3Ta2r}EeD_09{N$gUFBr%?QNhFD`R(;^~>45y0$i2l)YgU
z_nK|1X0|vzTobw3Z@!(ZfIvc0kgj~doXYQCP8o7r>M5UNVL5T0gXc=uUCF81^CfEn
zD}VhwyEx^&yxexR#07hg-CCRfXI;+L3rdS?p5^&o-8i{^&*z9Ue<r+mm>crqALl=z
z`RC`GW`~7@T>1F;_<t+s=>nZDv(0jE?I?Ww<mBY#{`2*Mvvx1nQg~3iJ23L5(9Tb_
z+rM7*U#C=Q0~$gSva@gQd-Czf{gTK_Ei?XZp0rL&SX})1%a<#cF5Q}aT`x8^cGqFY
zAYS7V-?B$XIQjYeEsLM|%(0mGF5;ie%(t)aBrpFsi(BsO{q!Y#@})m6OkK;nS^qX)
zq}8qGF|Q{yHqLcwo?F_-XYx?5w?OO5+iiiLuYnF2-LX1Xy!FBB<XbB){bl~sE3O~6
zW5<pynU~Y<@2eG!xcTST*Vmqt)%I3=OuD@-_u92<qE~f(DbKg7joMT3acA*!v%EVN
zH9rb=9d^9RaIWZj3Nt%jPiLoP&5sS;-Q3epFSVXye<TEUs%N<S94?0QkMoW#-*9d1
z?K`VyR^89NC>^buA8S^cx+!3yYL9*8l|vVsm>CpS$K|H)^q*sqn4SIl;lsiwCnk2)
zegK`gGt)RdY;DxjSFfxR54C`1-Vz?aD3`IVx^m@8h?|?5rY7fQ<6pir7H02AJ3H&{
zuF}<Ye}9FAg_VRq_+xl+y8e$O#)Gde-M@FQ?f^seZ@E%d`TXDK-j=J>-<kCFwC)Wj
z1<;Asiu<oDy{hBM&C&!q2`YBimq$mtr<y-BEO>BW$&w{jrLShpo-J=t!0`P1s$-v@
zpO=-9*_MC5?$wo*cXyYsUJ-KrxBiM(iEZnC-pKiOZm#w9HIbM1*Z;5m{q3Ucx%ZPF
zuMoN^U&*jw(Nbql^EKAr92lC?)6Mcay=-=VUA4Md)cu2=r5}$g&)<1zp3(gFr<OA8
zcpr4)$&`{^&HtI#$L-y;b?e!aCwG>=kK0}LR%@%itLstS=xs53Dn33q*lb(%#ly#E
z&6RgqmW>V@vaYVmySvNM!s5vB<I$i4>jYP&88}}T106S7`+M8|{r_KFT>Mgd@?U2E
zbHCV*a50<Z@0>G3^ZJ_GdF%fvo{sV1^=ZFr9AUA_+h2UIt?-*UX>a|mxArd1y}xDl
zzxgkxCeFRhSMuhi_r#fo6Fp--uKzuOX}RCrSGTslzOgYGv`BKzni$i$Qqt1*_toA$
zdUR>|`+K0%Wp#Ac%&N<t+R?#LT>N>-k|pf?aut=8hBL3`#Ibua$7qJ_{_qq1V61;W
zANNU}IyFcC{?waZk58XUR9I-WN4m>p(a+D%=iArYsi+(|bEapSgpslFa{u{tudl83
zoo)8@-QC&KB(mn6Idf)b@$+NHj=j6F@$kKS^WGH%m>!GSRib&<aKWm>NgIECdmFvn
zZ*JgXx0NeagnXNmbu?*1%FUdcLH9-NJzt%!XN=giNr#PLvD6u!y_KJjUA!pBaBjZ+
z{WX!syAB0}gp?E&85tPNu&b>K2?=TElU;SUE+fcQY2usv`|F>an0R+rDK{tQ#p~BW
ziw*aNHh(t?dDzA;U-$XhS=;JwJ9h2*RB>tZR{vbk(qC!m*$)r5&$p=z3Jsn5uHeAA
zbMHPrKJMn`_V)Jn`}=Bt$J?J?GtJw>19T$u#^mGcVs~#VdmF{d%DPM0>8jkhqU&+4
zuC9~S{r6RVPD@BosJpRVDrsZI!$Ymh{pa)X@on3(Wr_8iO+i6HYJPKO%$>Wp_IKGX
zwR@`<H#IehYKOJ7w5*BSd+X{|(fMn8wYWgW85tS<{rx>WFi_IAYD-&N+r){2cLlE&
zWM*de_4O%DeDUaLw{Fyy1=e$lGBPrHq)fdcBP;*>_^2PhZ_B1llO|7IY^{-5{_2Wm
zdiwJ=Ug>#OrK^&U_sy6W$a*O9-L1SUD+0~)?%dc{3z{h{^v!>|v*4lAa=*E3yiz5<
zzGQ;-z#slOQ&8DWMOD>srcZG2<p&QEb{(D=mw#`Mq)EnwE>Z2KrY1fai-6^Rvb&U>
zOnK8XSgNY33JVK;)HWACKj%N+ZmIR0S#COF`cYdlZfyY_C$l;2>?*NjozT!wdHMMp
zHW=jR=d-i7KR-YJvh2hX&AMBlgYT@%-prUV;lj@1bdagS=ZcgW7$hYn-QC?QD?w*D
zU0)r({+4Kx=h~hg9(DEQfByXOp02kw>uT4#f`2{Rwtd@C_}H%Q&x?2O;vyn4GBZym
zP2DV9oLl_*nr?b}y598aS64tsMJFyaUS(s^<`%zm*Dg0Vx4au09`67D@A2oKp!3qM
zJUu<#y6nx3?fLhMpP$Rj$haVT?tX&N%(%EXemNTtAD=B7Hgxp#EU{Lydivyv&m4=w
zGcycNPt&!o|5tN)neWR!j;jST7G5;x;o*7o__3?f#NJ+CFE6gU4@1ILJoTct<y>0g
zdAv_{wt0Ts))_bdapvZ}-LPSUZuB;vStcj1UNzO$?ta&iAs`@-ke9b^)v8%jrkG@0
zP}p_YaVt|=M#!{9i;@BYE=<u3e)Hyylatd6zU99vzrUNickkc7zd@^b?yn41-*tGU
z0UsY<XJ@CCm6e#77<esd>n#8IcA!P#|Nj0yKiB&C+qb>^rH-L&uNPkAS4BQID#vX0
zyMk5iCr+Fwe0<Dzjz!_YCf1%;A4)dw|M25uGCx26M2{zL-{xjyc$8aAU9n<C#@kz4
z+4*E5Hl>^doh=gl-*MuVL-rP5oZZ~qzQ4PB`uzF#x3<1MJzf7LpZhIOl}VsuWaRDt
zeRz1dTTFM=!S~PD*w_LC10NsjeSLrb|2sR2xw*Jr^sNYr*;8@x&YeGJXPfiO+s(15
zEaLsg?i$4X|JjA}8k(A{?CkyW_WPo@=Ord4mb5GHH8M8-{q1cv2!DT9`tnliy8<m)
zM}c?m-uch7IeGfDv9Ymr?JtwNf}u;w-{0H0W5=0Url*e|SGTa(QTO*(iF-?y=9vpG
zuK)Y}em}Rko=)7J8>dfuYlvKu?YpwG@Uh$RK3VJXcQsE>P2E-c`jV{Q)s?}^`}Xgz
zKRr$N`Ptde-@N(r?QQff<zL|{p6_mLJ$>m?P<;IVkB^VrR()AutrW64u~hk&u9}+K
z_jh-TOG~$I-O9_sk#T7W=UqWh)=rmAb$@@={{F_w!gAy0&5N>pywh|dohGFuB|Um}
zcJ|t}YnS`ZzP3KzepmA^@0TxMa*OE%1O|5Y^{p#A`g(=ip{YqP&gb0TmizS7RM6RV
zx3}xx-FQLA+WI#5v>DLK;4;6Kw$@hP`F6H?dVayd!VGbHt4wQueL3DQe_8f={G{pA
z=bL6<TM@YU1s}`8^Pt^|V&dZF6(16+tAB^C4m)|`#06PC+eM2PgMy*z%ZsO{r(Zv}
z+)GtWt?ccsuYZ1iR#Q_eIo=d-w&nZ6i}J?i=KJga*F{A|eSdfN{+>$ba*NR3MJsxx
z%|VB(UcX*`byetAW83;)UosCgFwQp1J@lm~;ORl0-hXYYR=qkoS-qW4HY+)~_}v}L
zU58g%Sz229`T0%Piv=A51YVWgE+^Cp+K%w;+c!NuJ&?t|;GNS)jvV2UG&*weV&TtE
zPfhEhTW8FeadEM`wzhWs&Z4Dd_bT+ZB^+#;F=Ixzxc;>t^@S}jT9Y%r_}|)|U;qE#
z-{<G&e}8dN+17UMy8<n4*TCrQd9r3XGy41c7cYMN;o;$<NrrbFFDjdueEIP3@WF!z
zXUr2^l><6Bs$tOzUTL$OdwVLszq@Oi|L*(a<Nc41_n$v|mY0(=^U8{eE=n)2uaE!y
z?CjpXdtdUcU1eMOX-W3=b>jMQIoH?8-rcAW^7)~JR47A}LfZLxvQkp7UcTH}@iA$V
z%FDhLhmIUQ8W{~bYU9_jUg=&bQzizsW<e2=l81*{o7wp<$;M9o_2c8?21e#>d3U>B
zN-kX0n4A$Z(XQslgGY}h&7WWY;=)3yg5_nPGRG|U*7p4S-~AtYczJ2Xsivo;xq0V=
z`Y28O^768Ke0=@oWxk-m%mth9>dMOEe}5{gtAF>&TF31yYJFGm@7{9%`E9(?Wq*Er
zv@U-)<@W^8%BQ{6->obyuTDBUNn9sFL9F}w`gs3x1&*sq7Hw`*A06wJKHSDTdGcgs
zj;41J|Bkiu%e(c<?M*z)78!YS{+w+UA0J7Z=f&(STAHRm+p6r%jX9RZj~+b&?PKX<
zT^Pk;Y_dv+m6i4Vy}gUAyY{Q->#twC_V4M-8eOmM@893ockbKU+p86{qHL<ZoVa%F
z+oz|e-Q3(jo_O0NDYdZv(9<Ga9i3<A=31wnk=T{IvO?az&cxh2erM6s3k#iJf@}bd
z3jcrg^73*sGqdz_a|}USkEV6GOiGz=SIfi8Yo2q%p<IXKs@skro`v;Sca^@b{r%0-
z(lRwI?UJmw>n!_vJJ-PM>+6<U3$$ddoV8~U>q4!|>*M$T{{G(D+B$x3)z-8(-$i9)
zYX1EAxMRnTTRTlToGvk>?-X1mA>wLjWo4Fgqu}|uxh2Oxt(UW_$w*Hxe|k#P+1Ytl
zvg3+xha&S@wAdU47#SG_1qJKsK<9V*7Zn$?^GIx{{axmBp5r3-@e3TGOP)M=;y2f7
zu3had&_eZye{7zU_Evp0GBk|bo(E1~w;xJKE$q6oDb+hZK0ZDDdDgD1oOgG2`luD3
zon>0W-Xw5nY7$4N2|If`$l;cju5NB&>qC>>`(!w|xx2f&lMOy>ws!ggIuJ@r&Na}_
z&#(0LwYz((&F^mfvyPo#?#qjd%1TOyKu17D2sklVobC-sRp5B^;9xVTrLn)>{_e&<
z)8@>Pad$sH*Sb8Z%FUzdfrQk;Q<El5y0kI*c<^$+t@-!oy^E-8Q#m=$wtCj=*^l3*
zw=CgG%V^QMxU=}Vn$HXYX=&}VyHl$xD{X6keK|AJxZ<~@;>xD~&PpMjzrMVD{P5w!
zCr?0oXZn`bG&VLqeE6`PU;fyao(Wq(a{yDj7ipxYr}Ogm+E#zNAnP6Z{r&y-H#euJ
zq%7ID&+hI<g_cu?{LgZP>NP3w$yf-;%IfAyT3K6HKRD26XJ<EKp5P=ui#E3vS~m6n
z?%cX%rK5A^)Typ_5ey7Rk_x}SySsMn+VFKTp5;0m3qK{cxvl7m*->!t?p<3mvu%#e
zY$eB;E-&--_VncB=Rbb?Jcp<5pZUTIwJxm=U++0t&GYc8jSUSA#m~<@d}YwGM5b?{
z;HsczcK)!iuw@@T)ym)BdwX~H_pRC2Z*9qR4(H)mc)xM}0>M>5|Ni~kxN&2NEw72T
zcJ_k<jn&`Z9sSZX;mct|6Q|I(Pft$|3ky5mFaQ3=Mr8&OF|lXQpI=`az1`^7Bo7s*
zg#mtkd<;jDHqM<J`~2M8>H6_;lC_F0T4Er}v(C;kwXXQ^V1lCarOTJ&_f!P#ulp;e
z8x;~156bq_r}y{tur$29yj)vTQ#X3snn}N$I6`$~oRwYrOJ866`SWLLO3H@{o65?c
zyUX8i%e%YlP_?0fLB*#hoqP8DInc=b`0?ZScXlpbv?%D|UWFDd=Dvl3i`oM;YX1Hz
zb#!EO4OE)=;y@#_=snl0tXJLQ`V0$JtkBTWS>ribZPTVrqW8F+R!mB0b6c^h^XMbJ
z*j;O4cZVqmh{&&8!Xqmn5MbmP77<bL_g5)1bMyB6`(YLC91Bl1NJ=eSHEsU<`oF)v
z8kx_iu(>zax?F49;=jMYPp{!%V6dD2|I^dcudlBU&(Y^t=mjb}RxMk$EG#6%B=wX?
zWo2cPLKnZ7sOZmMzrqsq`Q`1-oII)dGtAL(VdUnt?fLi13JX^rh!b#{3Mx%ZCQP0T
zDzNYFF7NH_wUpswn4nVm<wf8m7422r_4WTBF`7(1Ic56v`S$hyHm9GzwLL#R@U3!-
zmh`%(ZXJFiu5a(}zkm9)w}QZl6DL|)Tfe@zxY%jd)hk!doILrkz(P$;4OBl~T5@u~
zdAGnSuNeykS1DOqTE^&och}dA+VbL1D>vu{mI=R)Twfm#I?j00rcbA)YRk&VoU(0K
z;E<J_J1MiQWx|$&JiP&-3Z9dWCYc5W|2=8h>a?&!CG^SD|9f{UP1M-m5%2Cf@#ABb
zKi?iNul@aP=kDGAtIIhSo;m>9Ih)1W>2fH=VBL(ZORXoYzh+-QclK=WauJS&@}P1=
z5wvNUr&quUX%91r+zDTl)-1a+|7pZM?zJK5YxB00Z%<M_#^0~dvdZ$$d`ZQZOYfX+
z%Dvz_@AKWwTUVy<S>o@rjp^fLWsZeee8whDm%L7Uy0*JI$WC><pZCf~V-wG;F8RF^
z{ygDuTJh`fTw{SrL0rkZy7Svbwn(hGb8}wlhiyB4tnjmctJ?hVZuFu$VK$M#r*e)Q
zS2--&+&r`lb*@AO@3c$YIbBX@&9hbC&N47?=$_qqc*Tv{OzvlUzwTu|pu%x>llu!k
z8O4x517~HG-iW`yPaB1XKG$8DeVKznWcN2!h6RP{tgHghQ<xc+>{-g-@Oif~$HG}m
z={p4{*|kiI>ThEeXXd`7s-PusW7mqU2H}PYb*I=F7I3kq`u+dM;k1J7fT4*~Q2Pd{
zwFfHxJ^XNU(d*~kd9Jm$iaS}BeOeIs;oDjXhlQ(+g&Pv@%whCV`?&FqAm|`dcaBh=
zGZ#2KXNDc#a(uQx5${x=PzDALS5<LuHipfa{Z;Ee-Bf?yye#l>v-=iyNhSl!yVBEb
zFD*8|YtP`cBI$vSxl_=yn-jM$?yvbfP3y#c8|(kCzh4jb&oY$OoRA;H+3-mE{C#;x
zj;joyl$f_d=M%5P%alKV1zQ5Xwr{?JbwfLei9rCAkZNA3JB2nLOqd{a@<6-)|Gk=u
zo|`0i*c|^PTl*VrF4xrjTW!O!(90=6#O{>%!c`xhot-^r&Yb%E^e-<i#>K_Cxw-wn
z!RIK@*4D<u=C?#qWoZFVZ-8hfXj|N|-v4`L`Pwh93SC|H_SWC(U=GF1Hu?F2PJ?H^
zI&9lKWaqWOudR&CY$_@$8X6i;o;=ae)I53WRNbE+A0Hj<&b_^DskKQ<z|w?^vJ6fy
zw(qI^ZDwk^^~dWoXU;s*pE6}i1~-Rd$jO3f(hFZDzPqzC_0z_E`|2jE`DSpjD~7x@
za8?enwXm@0a9K1dUsOcoSiih~vJJ<=Q*Dw`3tu_L#qIN-Z>M^Hl8R-?ivX#|jvS$8
zXD)DrN<Ti<t3C6DnW5puOP3az%e4e7J;c)+ur)C~J^WAWqD4x6g~}n1K{ey8gpCpB
z&Yfd%>~K+1S6AP4*pVYt4`hV;x^?SBTuWbGVtu_vVAaPBa#9PqrcIi(DgC_M!hi_L
zUUB;*52c9}RaIebzZnv;vaGVMXf!Ekt*mXikbfw>OEDyoh3Vj{tE<c3-Ff-!S=e&F
zxn8f9u2|s_7q@Sc)N&0i6O#}(w`0eTA3tzl!K5ljj=y{|&dMQ%>*DrW6+UvQt^GS=
z#tb%IDNuiZwn^rs`Satq=iNQlBe}WcWe{YjXT`Kbzb|n7U47=vnTLnlckkZ4K6ba6
zhQ^Ah<>?Fx9EZ-H)ipEw_V)Jn+*?~Li=Xk_6;urIPHb~~@jeiASn0cMxwp+SE-2X9
z?F(NYSK=O$xMj<hn%dg(x3^4df0ekqAK$XYWS6qjifadXdN0I#d3e0Iu+TXn;la<(
z&q4Kb<@dJCD@(nn-?$M`TU(o){MdK4*+do3au<%PN}$H6eON%ihK!4g4!85`$L`uP
zO{co<<D;WnGA|bu6@7biGdVNUv)qE?Dwjo@+l%upt*tL#zI^)R$&us7`{itRnJr0Y
z$hfzsvZA8mW6$>X_T$U_=fCLVSU8K{*u-i7>#wh`*Z%%C*S>z={Q2|y`}^D4*zO93
znx>~e-(CLROGU`mb}#r4b;lKLhk1H0tnYMLv^D$stXZ>W&767k$dQQX=*zN#xgjAT
z+w$&8$;j*}eC%dzy?e?O5s-h69pdS|pzq@DzCHW;yIWhc&GYWeFwF)Xae7zq;`uXY
zbhNZqx%bP-T9rg>O6hb^SYWNxlBE!%*>-{d>9c3&W|_XewAB0cwYAa)2@H21dMxhg
z>51H&W?T0MbYFpLbiQJX7U&38t@-=+*UvV~ee(3_<HwH;ZU1{Tv+*`HHC<a94cZ<1
z@Nj!Ns7){EWXhhF(Q?Ym%WKodjW>4|gSORQTOZ%Q+v9K0`nbK19zT{g$w){@*pPi)
zuhYc|6#gt0r+Y7iySln+hphpP95u0W-?@8NlIIV{@AOHNCfV599!>f<&$ha!xA!F<
z`@&be_ZtL$y*bp%{p;5+O-;@6cXuT1>ui=i-@jn_^75CLmKr7>`|{>y@vkp0Sy@?2
z+*<-n+ul2JIJ&vDZQlI($;ruk_wMDFvx!(AXS+*z)iI7H3EQfYnwlCvzrK0%?sW>Q
zgJSL#m$AvJs~(fSeEVi*WOV83)!yFTty{M)wO%CU<m`O-`0?ZC&*vX(V&&uGTOYrF
znYB{OD)}=PR($K~@87;*0|z(v?wX&U4z+SWee&c*--=}$w{7d|>@+ktH#aqn-CefU
zLuC<2i*Sr)Tb8okbnE<kdxX{f*2L^Ay0Ia#q&@ia(q+r4Dl3;RUHbOs=H+Jg6L}Zf
zHzj9;yk=o%fBo|1$De<G{HX9bAG+9cvYNWzoEi4@|0XCp$Hc}K7Zrh=pT%HovdT4X
zPsPQ}>F1|vhqJM<<=x*G3)&Lgku{x>neEQr>hQR@xcK<{_wUbtS0FH{%;I!!7VE-*
z9eH=H^7G$I8mIZpGU+^;6zZhj^kV|^#rHoJ+$no=Bkxpv|K~+a%<rFm=(L(9?6xJn
zzM6OD>>`^3_B#)La6QMS%3l14zpU^)$A07Rg4Y!h5h2C>qUUl${#mX0Vtt%>{nFJd
zHaKoucesP`R*sVX&r^+`{eJ}Hw;WH%HB&s{Ybhy^`ob=(|8uJP*{0*g%kRzD^?m=Z
zGyC6t16?fh;N{Dc4-0B)YQB8=qP3L2WtF{*^Ws_WZ*9$PYirxMapU&9yR)oHvz{(V
zXGoaME4QZN?}2^3^QKSQe~q#GzKc)H?z`q*$0{>!e&J#8`Ff1EOG&Oa+6=s~=%UM9
zUlT*4H21jY2jlB{&Vd&;@aq1K+L-_L#>T~!>-XKtVPXi0iaPb?d+_TUuCskV7$^Px
z{x;&Fw5ncdV%3|K-Ck$6Xo1do-?;JP*6iytvY-(!oBDr$7A;zI>J{h0S3Jfht4?m(
zyxF}^#<J?m3sLPb7IyaR?CkKO_DwPh3<*=~pG-eH?}W5F6Jx`|RqJ9Veej6O)BAp!
zGiPm^o5E=(fr85U{hDT*W?3EVV?Gd;9}vGiWS+=}9Wiky%a=5;Fvv}Jud{7_QnZsv
zh{0iwsCn^=cQKP#63UC>Q}|p?96dh&yjF#rx^d!!%vgpA*$EE~tGj-h+qXXOd%$`9
zVcOJb3m6$Bc#i$}2<lu+o-}FE`t|h>54E0p!@Dqx)7WIyOpUHvyUW+_+gCT&s`Sz0
z$8X=hUA-il^WvuE*S{QG^r6z+=m>+u(>t7}3hwM*WIy%$&z>*V?d*K>q=hzJy>dX|
zMV=Z%gU<7tq5n0Rb=T<l-aczR%QdaF(%;yi#Q#YkLqO2{FK^z&omagb?)x^w=Ie#q
zYuux4*(GO6RaRL=Mn%rj=YP&9vS;tsGsOqr>2~|v?%n%G>v`M%V-}uwZoHIUZ2V(E
zhC$+_e{XevPp%MQ*t17w)vhJWt30nXU3#3}u;)lag|oGC@2P*Oj1GINzVgZ0>?nPG
z?c`*2ekl_TW#z?OR<0{{9pLHBTHxL<7a1M>`}gnpmc?mTSB1*(EkCtOEG_t1EjzE0
z#OJ>IObzXd>^gU=Od0Avt`6~w@!x)x`|tM33Y~qs?kAs>=&z39Vo>@1Bu%DLpg1Xe
z@`uo9{b%Ld^4J-cK7X`wPPE_Wy#9}~UYHg){&@9kvF5CE=VCe1=67cI9dBPJCil7e
z<m0&;{`=Q{_;E5=ZFk`Jw6(kPUfD=UJUjmV+}1n=c80XSYE4g~4tzCdkoVU;E0KBq
zjk~?mv%AGBzd1*3>Xe@w@bJx>H%ZCKrKP33yu5q&?)9AX=c_5lRi+sW1ABu)LQIOE
z`9w$Get5V&erM6q0|y>lzaFx|X<FO7&u{qt{8jvJBy{+jS*b~gD7U@8@8^5=>}@-F
zgc%mB+c?+q>z~A^c@ZZr&wh2M#aCn78<R^;-+mrf+BPX^rrXZ-^B4|%-ZA^5=9<#i
zX5X_`tysA9;04o+Gm1}2qGk!NI#4gHsc64UEoS%JHTyFUPwPzc5oBnX@&Bis+3LjK
zTlqdc<UXgS7Sp>;KJm!UKqfI4_IVO^2X+K4-?3j^U447T#YNiT>sne{BO@a2?5*CO
z_1oP;D)iDM$(9!npFb~udkb_Xy05SAt|W~|&b#g2J$oJF{&k00%IQeohCBJwt7;=x
z9t%-9wZwB;R<an!{<+0$3=I1s95(FwvVFULT=ulDT8jl7K5YuwyhNfyjIWQ`rz<%B
z-R><tMz2rbh&^_0={X%q8>VUVI({ZHo%81DFpt^0_g((~9il5!PfvS(Ztm^P>HU)@
z2WMqz=^84i*cR~gW{KF%&reNtjjx)<${;BwAoXmWo=nrt^o-}n7!<^q7&b6ie2seH
z^=SQ(bS8$WpRFdn$XU6V$?4tcKaY-{*|~AyAI59GPg58eB8pfTyxABcazu4yy>=wt
z-N?=)U}L?x@b%i5$U4Twma|zI6#lU&ERQ=b5ZCwhi|^x3?t3xc=hTXcSTHE4vF?n#
z^XP#{bm{EmgK1m-onQ8Jdvb{w_X_``=6j6g-IkO|U;5Op{ASgK?fLNw13Z!$IKcag
zLL5)LG5%e)ez$Pq&MWDcbo%=aPye<y+JCyQ&CV_54|heYxUA>CG}Cxdv;PI5+k$1S
zlhv2U)@^$FL(7(LV&;|U_GxcyvV^Q3di~8!bbgg0bE8g9rO@5!@f-_7?a8fc=Grau
zzEf8Hb^eYsswt8u`0iEY3T;ikruS*)$5!jAGZOzrJnu{t&J<=s-L94Ew~O1fe8$w9
zdz!=DGeoDSul`yfS!TXG{pET6l)Ot1YpWlvm@-%TZ<?RvtSzb;KYEHoZI1Q|Xo=fx
zWVxy_+bp;0@2{`7wr0QNd-9wmW?jSDH8HDfW<4paJP`QokBRvtCOLnGQ>X0Xez<W}
z*f6eMk~-<Q@ae)E^%DwMZS+oP{y!-ys;{iPxcK?G9qXhOL9O4f`_7(YVDRbNRP^)t
zG`r5Tn$>sDpHC>NoA>E-XV2%?1_Dl@%<r9*TLhdgaexk<S5)Z*9TyMczIRr3QH~2q
zc(9kLOZSBGtJ^)2G3TyH*#~Z74VygQ?!UHi*8K0Y`BuA!{5>2k;Pjv6_yvxO9hWmB
z>}Ss1{r#Eb-^{AEpy-5y({tCHvi25u`+o7YMd$V(?-E#*G-ILQBGKT;!*fz^#q6H*
z<<D8Mi23|eb7EPSZabj0wO5pZLEn6Ocu^2&=hebyL)-Hl&K!TaL7m*iMcWQMTAEWT
zyZ?6m_kd(~fB*Bj;*a*_G#=I8+8Y{n!>Ihe_V#bQ#?}*laK5;~#;h0;_`%M^DQI~G
z_ZsKzax*oA`;NWZmTdE7k^hyQg%|Ai|K7XO`SNW2E%8eX^SgKE%ys#&_ZZW*sh<-#
ztcq*i$J||Xyiff}`2EfKxr^(pdUfux9ln~(WT(snUNgzzIk#y0fm8oy@J^Y)e_=(h
zoxrP8J{mWfykj=?PF>{I|IB=D?=3lxfS0MOK2|PI|GVnOOG(9$hXp*n6Sn;Pp`a?h
z?>3Wn{ev^-UeC3;{;}lp<LB?9&oXLgJiqm89pm#^tU2=^nm)NRrS^8o%lmB1ieIi9
zI4is8i!nT@{X0$k@$Nr!lEh;>&rki~WBa=5XmrBg<%@$Rlzo-T&Uta;->Nl%QLkhe
zG;T6=t1f0>ND7+pS?S#m+q0Z<tIzhB?(*^J$S6<x!D2q=k=u${pabb@K{;yMnhsHi
zJ4)BBd$)*)7@eQfn>^!Z>}&I_TUWdhi+Jg;d2wRn-I{Lw*oXB7-|sxv-5_SY_3xhE
zt8(*?pJvZrJniHi?^$~iZ_Ikx+kaupWn-ro_aEL1Tj91+tvdSevP;nkPutJVJy&q|
zN__iK;mtoX#m(bZ?@w#`dA-;>H~H&Rqe;g_|7We{&1|WTRt(`eeu2YtZBTTA=pMV0
zXMWbr60sRvKP9L4-F;=2_|W|Mv|Gnsolf=XGdfqG9MbzRDz9Y~vupET^%|*V_bs11
zFSlHnaNy6>yR)?}$qGBIC<2X-zj9o>bn9)I>|Ki;I6`G*oRz;!KXGSD!uy}=)~%a2
zZ{F$JCg(p)lTXzfeS3eR|5NnC8`lb78Qph}e01q}vfr=jUAli0Ps8?y{|iiq?8=(&
z9(~S|wg369w^i={en_<hw1T=j_fJ$8-CD8vajNC*-raw<<wicfVv_yy+Uj~+b<xdN
z|0+c2-m!Hnx>H?o{_WCD3ntBr+uO0@`lBL!;U%qatv8gO`{vI2bbp*)o!grK`((9i
z7S|Q|?tH%F_`F>!uJ~~0=PHMUf{$T5vE{C__{lG^n~R&)T1hvYy!hTy^@aV_-}+n(
zr*3|15}Cko;PIzZ^7WtQ$35Tmrev$>+Z*-14|uw2el*706shjqX>+vAch^4+9?+($
zCHurSRo<6VtBld}5BR6bP{gZaljd*u|Ke$T8}|bB4=t|?jD0R}fDTIH_`9*`c{!72
z(W?!9OSNb6xak#TtP4py&TaJkV%T{{w`Feg=J`zvoA&JV;mRw!?=s((+)`@u&469u
zc6q|?LU+4!F?Y7_(400$$1y`|BSVE|Hap)vDIuHsD;qo>{HZ87C(gK{=C2v*YS-`X
z;+*^Xdb#i2n|1lxhP4;pSa)6wz9pq+|E=NAJDG;_)hRb6A2PcLuCfC4miMci{*`8P
z^2rMAr$-`dw?A&(Tr7V7^;xxbk=yNb@8nFle_E37@wRpMZl;GdSG{7Gzu@8p)nw}_
zCSQL1Ir)uyx@jW!;dEZ%%6Tz&TxY+!{Uhi7g8Orh`t9G57Z-QGx;uAe$=w6@9A_HK
zT&@3l!vCmS*y?k6cDK`SZ7o>#?2dEy+}_i+yx&hat#}1WclPSb7#yaBn#E4t{kXMl
z8k_U=1%~YtE@x)?tzcqUUHIHpEa&}c|9|Q4^_Uk$d^c$LpPtS=O>Dyk<`4mqkR$Kj
zy*prY-0#i11MlAXy*c#GAwVSBL4-wBY+Ks%x&sdD<+x`s&cEMYojRlTdt<Ym&))TF
zf1DOX>$dmn`|sGkH9BCjrS2S!hcD+{z7=DZ=5i#Z@W*@OOPU$^vR1d2tx^^=v|P=8
zH#Yjo^Ry?^o~_$g9rU<&wwy`kn_qh;>gV0MyltbjUm^1nbrX{V`oU2*1ut&w?%)6O
z=K8Qb1_F}v>q@V0Syk1q_dQ)__Wln^-BssK&gOZT{`==E|Mgk9jz?aWxTu8xd!?!8
zoWxZB_vmka`NG19h4z(i&bIEaaQU<N=j?SgzdvvMZhlWA?!KgO2h$R5kVTW9>Fj?M
z|NM0Et@nF=K6||FN3yY-oadRi6BUo2^qRX&+4IPCrp>e^o)bg@JUTmg-gi#%YU|iE
zCF{;S`~6>YGx@7E1wUTAemCpy?X(C}-;5WV8<>O5>@?4LJbPokSnblDs<OHD|NmO=
zpJJpmNxgjMskLECe_uZSX20~hKJMtd$zPPue&5HS`SJZ%dHbs3Ti+Y*&gkB~JTCG^
z?eo7s{@;IoHg?TKRt1KbcD5h9Ytr9;eZ5Kd#hsP6mK+M5C7Sk5rZ0bAQPJ)oOYJ<l
z=|xAo56xN-`FVN)|J}Yg-9vGazcnfsc28T{QM$tTK=9MBKu4FD8+%UO?Z`;D9CdQ>
zd7GleKBbe3*S1Z)X|Sjy)BTZr>=%BW>!1FgD=Q9MUjE*)IJW+7&V9SvY^AG2igRo-
zejIgMeCwK&t4Y$|l||p~|KL7%lqElM=O3&886V%6I{zx$#D6z@?fqSv)xRR1EvV{A
zae7@B_FFt`E06C+|JMok&*rkrAJfchC~2E&tPt$^eyXv>+NibHq?TySo}wMHB<ABI
z|BS0y&0b3%b!fc%*}FSYWpeGWIdSWCqOM)NbYLTw+0RNn*3)%WlA57h)8^gXBbB${
z{CY)O&*CYC=l{R9eB_b2&|`hqu~U)^CkuFZKd?u%o^o3Ft13Hdi?QHDr=>vxg5FC!
zJ1RwloQi%fKmW2MtkPR_`_F*cIj4SwtgutLbaXEd=k>Ze-j^blZ%_U4<Lkcf!Ou|7
z!OJ^=QKNWD&i$=V*6%yD(Tn@|77u2ImrUhdj0_ANf>Kfp3=Ta7)1(#VX6C<%C^u{i
z7s%YY(Pdg?S9Qpm7m0_3GUxMu&xpwSzVumZec9V|=N~t(ns54+^W@9bGQD>f8+D`q
z###LK%g*{ZSC>Jd$fAwSBb}>%n(pS<=$%iNY;oOgm-}kRotM|Q7OhqdbWsVN^5&a=
zo%Yl7{iV-eTylN<t?lpqE}1p1dC_P0O81w(`M#Z@;l#p)jF;wbjb7`;lbp6P=%%=y
zc|mS`@`<n4v*z7+v-a2bqQ5@`jF<OycLp!7m};h`_O4s+noWfLd%OG_H9wBM)^B@$
zr7&vK$$6jmZ)0e9v2dZ{=FZ;UzaRY$1q%zhyTsf%%Y9wKGqG4zctgIcaQgJ#^83P>
z=k?!SEV?v{H&|=ed7D*gj~1{0UEyzC{&PY0N4v&X{KXP`Y&L0rZ<@<}Yqi*UE4lB5
zcV0Ba`g;4_*jib)+Vl8Knfv#YiV6!0-4+MF5MelUxHRKL$g0rQ&tAX2y*hloX7I9<
zlS+@43Q0!mReWlwJHCGIJ~x-%8u#a^ovVYlKbmn=MfH-+?A<?#rawL{eJdm?H0#mj
z`s~PiH|lNtr@s8UO7!~cd-WE5eYZA8-b$OEe9mU^PG9A7Ehf1kH+qDcc5R()@aWRr
zOJY8fHz#jn?+{|ze@`jTy6laFWl_rWb8}nym=tVeoSm0FdiG3AMkXgY`SHVt8|(g7
zee|1oe1B?pPuz|K-+%*GOOvDD%SYs&{u>dNICrmrx66`}Tcuw^`fk6jY136wTRugj
zy_olI-@l`?eY*>Cqb@ZmE^4W654XKhT64Vb?D2fzi$`k1Ef=+jo6Dd6Z5e6cqB2!j
zaAo|RUs>H5pw%9Z&20Pj?W_Lw#&dt<Lk1TfUf!9#J9qxvQ~9}1)_Pmn+gr0tv)S0$
zx#zx^9Wte6+U(Fulib+WiF0=Tdh=O*=LXh!rsqzq_M30NjB6#Y%cY!o%lX&6%-Xp*
zy}9W3d{u$WztiW-Z~y1eyJ@MG>6>k*mQO5uyhz*7#`9?k&+m$jQLZlcZz$PW&CPpt
zWo789kd@Kf->r?_ZewfP>c^tMm635`&Ye4V*2V0+v@Q4ct*zPpvessrnw~FQ;yZeF
z9Ek|}bY+3RzpUk9X>ZYT1*N;mi(^)KR20;i25;IH{_e)^>E{JrZoT!w`Z&{)rEF_<
znyy`S>Hj-v=N~h-c9zcG_5A!)?f7*vmNv7?S2+1!d%It8>dcRAch&P+gfH%%5pFYo
zf<|ENR@Ha^^ZtlU+rPj5<)x*~O-*_C_t{o_NSJo?>@x=W1q&4~Px4Ut|L=ExLPA1C
z#g6v&;};jZxB5-#;l8oG;GNdihp)aZw6m+7y~|9dD*MBovvcp{GJn&XnQ1a7YWLHn
zOTBIXrOh_dyWH0%Vs!7to#eCCJM#kmKD7R})>bfXe|LLT(c45tb0#LG$<ynfy_;(#
ze(dw~=OvN<PnF)C;a0pSd#8!!z1%{9<c}|^<{S@?*1J>rxw+w`s*%yA?Cb0HR)3#o
zoPKU!?eC(er&LRn7zz*G3i}XlQTZw5{k^?&=gtLPVx;WuwqxPU%d2>UrZzp@k`=Tf
zV$l=T%^_l9_BO|!Y{}x)wAb79@vZKmN0oJFXPHh`^L=$?<>enAABSp9oL9##ASNaC
zPb_L}*{La--qUm>&2l2%-`iVg7OK(M*!cJN_w>lJpP!zdJ9p0MIHQ2RiOHW>Cnu+>
z@9$(~W%m|5bSf_|U-<Y>?3JvqOFSooCLvN!P072v%aBL8M2R8raB0Sm-6DE1Kd!6{
zc5dUzyt(OV10%E1i7#q14<2l0zkB!Y{r&az@9*t>cX#*Vd5m=&yu9^`zrDT9&d%<w
zB2-@fUDmpcLsoq<H!pAQ%}q~t6h1b~xv?SfFq^ft^+I9B6rPNXAKsCXk-E{_a?a1Q
z-M4Qaue4dsrzf2kI*k;Y4?cMPI(l7<rN96A-R1AY*2P4=aAA<-l#=?FHE-U$?Rj@s
z1uyrTsujA-Z|<tgFI)VkRL1NqI(qu_>pMG(udj=hwyoL{qxVkpEW;7@@-9XJ!Pd66
zZ*OmZ|M&Oz=VxaRFT5w;)!8X&oOb5p<Kwlpwb9#hG)rU}dbo~XU~qBSS@aaN9%_zd
z@xPzX=dX+1?dImjHCJQttv!{S+uPf{mU8j&x!E%R;WRNhkj^V*vZMZg-Mc$Gy{GH#
zt@@fZC-Ijmzr5X=*xlbYrJnxt=Z~0fR7zr^Vu?~i4{!QT#(jz!8Y|Mz&yzWRcE0`n
zqeop|SR^gKn2~mF&d#i>t2QPd|Mv2-yVAk+{}@&>)K6sCckt%TpNCqx@7=q{D`g^K
znA8%rR?KQ*#axTRMIkGL{O8;8^6<R5zFz*7q{7w*?}ZL*H{`jzxBC0<@9+25|G#(l
z?%xLoo1N5!=5};+@X1>7$XFD7e00>h?9GXJw!1HMG8l=Pm>j6zxOHpn{<^=%dZmx|
zN?+fUdiuhJfQ7=HO&2p_c9mq#G)kQ_XU@C3yTf;v<$~hQDI=pnes##oOLupd+uGWm
zo~r%)_3LZ{i55SVCAYWdhp!5Gd7zQ``T6<tjZ(WhJ3B%4sNd0K`@lWVuJ+dM^7kJ;
zd|>4ktEj5tl2zY(^ypFF83u{(@9n+3-2eTZotrNdG8kFPI5Yog`t$R%ymeX5n;RQt
ztxA4;zh9r6l+@xkrLOG#y|?e)t-JpE{r&y@Rcs#Y85s@n*EXe|e()f{*w~n#zyIuP
zb8cC6(2{P@`Vix^GdDISA72%^IwvQm#gFAfxPvqEA3kpG?pAJbIjfS4o10R%W?!%S
z@nPYG&Py$R%V!%VyDjyax+-jK)$?<6i=Lk1oXhb*zp0JwNA0_J@2;<p&%d{)vRBGf
z&2P?%iy19`Q)cPK?3l1;(}}aQ&D;6q^`?5Y`muZ{cXVd{BiQM3X?^_u0|yQ)_n$va
zEA-Wqlarm)y;$?_?|XZ3ar^e|-@m-P%r9p%L)~9)p)lj0-U5AdhgFtkZ*JKC|MU3S
zGqciHA)q|-_z&N_=i75{n<XA<`S$j<{~U{pXU_P%aABy|H!*R@x|CruS<N?ULxSTi
zUDLCLb6k`XCDeM4m6w;t?<!fjdUdu`FF%7lCok`WgHv>)%Rn={r>BWpaWE{=jcI3N
zU=UC?F=1e6kXX2ok%8d|2QM!JgMwQ|1_J|wp_4N+qG@#D0s{j>QcD}vwIo%{6Srb!
z_!BWzEA-vHy{m&(K6>=X$JcjhP-aL7XoAygiiTo|34@1l&_ZU01O69Ze>DUH7ZGXc
z*?TVDyBBvMfMG%#sO^?4VU*Ibb7$o#-MqX!%i?DuTKo)4OifH2j)9h@_dT|&&B)5S
zwK?6N@iT)%l(RE)3J+K7qQt{(Jcm{4PZY+So~GN{%fgV!#mhS(vE=o&vuDoe{5+p}
zYRblq8wJhy8J1|8m^icrF7*=aba8S?OG~pj$~)oZ!i9`RJe@9=mibC|x+ED$NSS7-
zl*ll6L}z3)^epk7uIILRqg$_()>N+-It&xG86<N$NOkr0u8mwdDQ>mrQNw)uiW%na
zW!c=_SELL)7t5L^*8J|98<g$9;31lk(O@DXBlG6nt6d9SE{4bir<<HoU0L}pJN$?A
z<x`>8b8gM)N@hAR|L~k7hD`0{OI-_3hgje9<vz&y>bbo9BPNEA2TI9}%tLdM7!FOi
z`hND-g}YmN+Ab8tt?ZLBy{k~7!XO|j<IK!)a9`p#u{f*$PxY()0*b$N<#?Ux=Vyp_
zbY|vo2)Z&+)#%th#ec4Ewiv#0sTQ_w>*z3?ylaQo+keH=a*p{#-SC{0^wHBZQ9V=d
z^Z%;d@^2X!lv+05VPG=2$752WnR%~>Yj(E#lTCXAh10!+Do<U$kQgJ)RhzR#Qfs?M
zPu%1Dd+QqK>=(b4z5MJNK`raYw=dqE#P#LGy;UmfQ&X>9+Tkmlr1yM-{C!71tCe%t
zDrV&0c(Xcky8ra!t&$efsb;};dJmp<aBtHrpRv{Xg}y99omk&O#ukD4FIR7EW>dPn
zN%!-b52oj(l6}rhE)9|XcWcvz!sNKMHvUT%EWe#3zuI$^%D;!VAM<xcBo*B*U3uj0
zo&Qx|8EsGM&e|UOH6vfER%~a&$KUEwGmD)+$EdZXKR?{Y(_8sxUPsb)ozS|y%>Pa?
zr%3#G&85I|u0Ap&eg2CE4!4QAr!Q!#weFe1<;Atb+2zZx+)t^QZl{DKJ3N%83kgP6
zX>{CplXvavT$AhTkI&ni9d%;Tb*(eAO(z{=KNf2@^UEj4FI!Y=3g5qZTKuB9Y|rjR
z0kvD5R|Ia*mtd$9JXgfbaiHdi&&h*=8y)o@J>xU_GI##f!}s=Vkrebi<I|bc`Q?7y
z%ow*LN7v34n;x@8@9XAp_2es0Gwwfjap_rn$a40>d589umm2M<FwWLD>=XL-zl@`5
zu3x+g!yidda&Y)QZR@{s7H=1qUlP26%X#b4RE$<%4f9f2|7K#%Y6)f4q^Uwmu6EH%
zlU^wC9=X*sGilQ8Icx8jhu<&qJZ~B_z0d5cm%+EEZ>RGF&on+2*Br9!j`jV(Dy7pO
zZGUdRwZ2T?!)msM_@?9x2F8MXVTsF0mg0I>j&4-ryS8hAbD_EaEB%nOQTLRu|61Et
z-svLaKc(VT*fgd%zSrAd9agGHI{W&0pOwgr#q+$kd1_a5_0PMb9T$GTBt-i3w8J{l
zL6?4}8v1)ZeU*B?*mITH<+CAWMwg!_2LD;AS-wL?@ym2R2K(-Zl{W316T+IBo2A3n
z6s&N$xZZx@rs*q`dspq*^k>?!>*YIF@YnN7?%pF*b<p+tslD6I?!FlN$hmjPrgY!Q
zJ68OdvOcx=)(1s?yV#(#V(Y-5AL74jUhXfyAt(27Ps+@z(&m4^P3L6DWITR>!70F9
zs?+0djQo$IX?CwH-H%j=CUvGxk^lDOO8ow5+?_r<8>@dl*nj_RyEQ`t$3rWVh7%%r
zGqz?5#)iy^%+CMwm*vOohZ0f_DNcNQ<cb_#2rwwv{ORK(zI1<BnbEN2>yO+opPnyt
zTK+@dJ$}D#Fw^G$OdWjBp0as3vo$9^`|@(*#`nR6Xa2JDRaH$pnUz1kD|uVa-D7v}
z+KRHXG}yPdv7JabHBHyL{N0^Rsi(PQSr{&ft)ok|LW9c7vz3p*{%C@MP2Hav&np);
z*Mz^jvolm{>IF>(1|xox61D|bvsPbQAzksl@z4>U7drQkNtR60I=WT+`P(;dTFx;s
zOz>Q=@aVY&1Cxw9HaZt?6`!zmQ@XcW=)Ax0%7C56rgr}e3k!2fXJk0RbL6+aoxsG4
z895hs)qc=kS8!Q0X<fR~sp#A(2iMg+&Dts~Bm_24vg4Qh-~BS@KhND&vcK$qZ}91_
zLT1c!-f9G-XU$pU(R+*6M5^~fBm;wyjmegZzqiW|ob3MIwdJPLM6H%{pEh%ueW+c!
zb^ZMnGvn5aTX8cu9BXS6TlrAg?d=!q=q-o+!sboPn~<h5>5%U<yPCr$`8TIMwpjR>
zfgvR!Bg1f(QR%N$z14H9-%ZrgZ9Vttw91+*E06AyRy35gs_g#x(SU(rqO)`Nxu(|c
zXF3rtFJ<nJN!2cyc3)-UsqQWJYMw=TtgrvYHJ6n^VX>2Q_vu?(Umx?E^YTvS{zFQ$
zFGT+1&^Fh-m3RE!;^}RC(uyTw3=KS-yt+!DQ+w@GQa1Kg`{WdH&s7!mm@0NowtW4~
zP5F<OXo5qp;=+Zb!XF<J-^<m`v3_H+Szo7QT8Ogo*+-Xt9@wLrUHkkj*spAp8Ru>K
zEB)j5@9lZlcV^tqeReap<=muGBF`dYt=`TzSh}ut=g#fr;b0F-WMn*y-CMOaaPqW1
z-Rn`$z6n@)&kBfoUi5BD`GpN^X=$s%)<$v4vNI?oABfCrWMyHwvg-DRgvU$Gerj{f
zRpnjzhD+@Aj@8xg|JBvifn6za;eu1uj}M7grAj|to0<DM#QB9z<~f__jf--(Z+&~=
z>z9|8!Ri<;TxhEN_owoxmT6Il-1MtUCtirW)S0?&lHb~E=6Tm%eR_Htq)x!vWQ#%E
z-m1`jNv6@~87=SJ=9sG*8~I#2e8tPITVJK_?yHS{p~JwisI5)x!0qk%r^DBzy}NXB
zp^y2(%^U}|6>R<UwPy9zHPSLNIwfKZ3?C2ez4KDn%xs&}Q>jZeUyH+K6iUPn-mo@*
z@TF69^_3k}FJu1w`PtgX3~DXDC|Me|dUeS9#it*IZulj!a5INab!XTdyNxwVv#zg?
zIWg0C`GrCT1{e19otOK=R<Dg*{?IydVN>aH&bh84GG65yOTOyGZ;T2FnF30xF6_rI
zygUNRC0A^8E*{$``E36mo$}VY^QD}kTGw|JUA%wll-CQ3MP+|}d`v$-PcX2i=Fh=q
z_V@Spo;L957Smm`c5Q1P8$)K(L9Tr-4`i5_W#9aEY_Y}iu-Pg)b=tpfO;%Pu`zSxg
zqWTi39j)^yH8FAGj2Rl!Puu5udU@@t__%2L^yv*N85G_=&?%qwxN=|c^mo_a?JvyQ
zDfjh{FY7Cx-e9q{fx8>s77JQ+SM*94I;l-oU7C=Z`t;>X%@REZ5ABQ>Cg1b#Gpg&~
z)4r6p+GUl1Rd<Ex8vE~4ukDLFysT<(+zTD0tLtKSPm$lGb9-H^btg-GE2EL_pRaz=
zhuwQ_y)56B(LQ(0osbtgi;j7yw9eMy?VMtg4as~Rf$#3@<QCQX^3r)}(9NaZ;?Sn-
z@`akg%h%u9aeU`vxs{iKUg$_Rv`n?y8c}YnvdmdM|Bl>8U)$<$CZ>Nste)4}+PZu9
z?hTa;3cd>$UOv^-+MTA4$e=vVN>jx%_x?Pyb;7*8zd7c*_C#&Xnp)H3s5ia*ZP(Fl
zj7F9wC9@)Sm)*V5X&h_yHasFl*vgyj@G0N!ZMQeRzwuPjxwhq;lGV0t+vMcrT#jtY
zIo>A=N-3G#+r^gen`>SEP2Olz#q?%1vz_&Suln4!;u1G*D>-)c_tyU4obT(_Yh)Y>
z5HdgR|NYzP>mfzqCI!>>?VV|2d*Vmt|9iP1dhfUXpYx<Dbo%aRQ)F(=t5En?zF$RY
z{iQCopDIzdA7$zv?RXX=Bw947v~;3Zr~QlNmeXBQ=cf90Z#Z3^WtW<EW+`{n3A22k
zU7xzY*Li{pkMC=ugRdVtRP$y=;C^{4<5xE~KVRxST}DQROI#0>89bshUWAmv3Y52j
zH${Jaym$XVzU89X7tWgSpElE9G+*w|!ostkPisAyB6M8$_V-d-;fp&3oSFG&emY&K
zq{JI3)Vsu-ZNK{YJ7@A+6_tt#r=`^<T2%Umn{4;2v_AP&`Bkx}i%Y+p#11d6!@Wz2
zJ$-J^>bk@iq;0gMCSui^*=Ob$v;Dia_NGy=^mD)8&vjQO=jE9dIQ+lQr@!OCfddyV
zTnLf5aOKLB=auE<-w(I*FMqTlXyulmrCwaDa^?&If+i(dpgY#@t}DK{?e>|?yZ;<C
zs3|&cx$|^xv+2p&JNu^ebeuTRUaMsz`8daSz4FiPE;V}$XDLrpKGt&5vnE1F@Z!GU
zS+y=MU*61}*zwn7*Id)&%m?!IHuJ(}PO$1MiZnKSU-9*LEmxHBM$Vck^Uf4`{GJj#
zhtpi(<LdsGURy)O^c5FRw^B+v6KVZO?9P*pjMIf@J=Kq%@zT_vW$=`1qoAXe>0h<h
z%CPBvztw$KX6L0{TRZ#vdp_kE6DA1A%g@&iU$<k&pOgHD%|-O%K$}26KVSarNl9U0
z;lqayh1LDu+}Nl*?*Wqlr%B1GomF467OUyXMz7m?=IzJj$(cR+k2L+wGFRU@^JuzF
zvVL)J(en9*q4x7kziX|16E&w+_OSI0Pr3UZvuZATxM@XPOUZR@>sTb0ZMG+IkFBU$
z;I4VUOXs&w^>i}{y(K&|+Vj!;%;R?+x?c}W2{!7Kx)*XXU{2zfFy6`WW;>eBEJ)7L
zeeo-^cjFQ#N1wN9%a*B6zP86CrpqMkq*`{yI%@%c(^IE*q_<VeAOH7>W1d9zwZLQY
z9KRMnUpD31&N<d^Wo`B}&YZcb^!2rRqmU}l3XdIykH5XS3ED(9RXcpvj2RqrUo4N=
zS5tXrhGFruGcyg7-R|xx1?8_D!8%h}pcT(#P{s4-XGV}EYuxpH>RO?0elBNEpHzDP
zBJNVV@ztKaTiiKcp1k|&VzrR?T{Dp{0o(lM&9J<8p|bMO$AZPNj;+y49+qvHxKw{$
z<f@p!hZ=@?Hgihs?qtmMkJCO`pwOahBzJeQTv?dASdL`zvSMD>ppLwMnrE|<yh10;
z-Lq8v(e`|wn9q~c7ulMBb@BOn$w<lbYR#nL>HDj>Zr_bm3zha${r7UaQ)>8zh~u%g
z|0EwgfBt=W^3uIOe*Cz&*j-#V>dKKLPtMJ?Zf55{r}5RtCH3?)+saQ*W*8>3@k)j4
zEP9%KeI00QCcnMSj}<iT7PZ*#-mc;t+3Y2kCiLiaedafx)VswuZ`&N(x4Z_Y?B}gG
zmwI04V->UD#j_53r%5cIBeb$bj{Rk4#%s;B(`GPz^Z9c9?a3(x3A}8dv~;)o8Qs~j
zp1WiD@%I&venfv1`Py_@XkySKhb3o}a-S*jR<GDClyG5_i_3hy!sGi*SXP`|v|!7`
z)n~$TEq>gMK0iaN^~MT;!};?c`JF_m&{rD9&ARZi<nyz$+7TNbe13jjKYpK0>8mR|
zyB4_q{q=Qy{Qi53T)UU~&Nj=x_vhkbcPDkGAJq#N{=f41`FZWIl@k|#+Og&wZ*XBm
zu<G0W%PtEAukPdgA5h-y@$D(!um0!#_ILN{rrPHl*1TqFo1_}6v;D=CvU{3$7ky@n
zu6Fa(Tpu8;oZ9hYb-L+izfAF;ucx(3eLl5cs`-Acv-=UVk5g>-t^6Oys6PLZo&21v
z;O59@S}(6;)jm#Js~5X#N>uVPv3bw8=iGes<Vnew7Z;n^`PW5kR0>}g<JA63S?6@w
z%S);X3JWSeK1w}34YWgTwt0RFAL|EmC+GhY!`H`6T_17wTCoG)ce9n&o8~FoT`Kr%
z>z`s%ch<qhXj+GdvxwA5$qvoer-THP)eRN@UE@~Xcx01{OPOV)u+cmoBef!xP$QQw
z=Nw#?T>sEG(d}~0r~2*<X=;-M=la~8%A?t_<B9ZTiD@2xul?woaZA@nYgX{OKbwwz
z6`kVZqB7a&oTc5I<ocfnKDhWFDLmfXtI4b#I=@Y+TzzIpvc*?+{_?-}uWO!inZ^b#
zuX}uZd;aogJL;c3IXU^=yLbEc?em_db8~Zg|K`n`FLVa=c6Oeepy>SW&Q9gYC%eRS
zLAz9c{0RB0&&6PWs88NL?gfwJRZt6S+UlIBEzawT{x7+;!02Sdu~UnlP1?g6x%qmV
zZ~jazt;dHH))l6v{;gT#qNF)>Zr+`1_GSU5<*7Sua)i@g&i(8;J1%hFouF{{nY){}
zI2wL?vvJ9Vm`!1RMoYaOr7W4GS5&P!+1R@HWs_ZcV&ud>n|7_+wB+}!qk)qiP8D4i
zq-O2C)LXM?hLPpw54W3lPO40Kr}t?UtBQNMx@_-~PpT=ZT_@i@-dQYoGJNY#7vpP@
z*%4oV6g%&>I9;{t^wgkZN8>*IYTcN|%lg@&TtjDS`a6@elj8H{)*syA$18nh^XARo
zcMsNIeS3Smb=jL8m7kwYRCYf%!%*3Op3R|Wp2})!YFAf>=il4&^WNTSc3!C^0UBP@
zbWU!1(;v^tP|x$)`Tw#THzLj#P3^sz{ba3O)B+A4$Ly;AegD5yY&q3(a@~>+5q{s=
z$<3C|ZT$WzwX=Iprn-5~cbjI&cj^C~|Nn2yG+y#>@ujqht2%DqR{o<Nv$sUA{@<U>
zPi*#|-rU?gQQ7_1@87?F{3v;O$yMubv|I-V508v>*_+45`<MI7ytFD*yY1)S_BV_V
z(ibeO4?e18dUz)j>+7=uRwwRjyt#F&JiIpErdnn4$+|t|uPT3kdwZl)*gXH<o!#Z{
z|NZ^FI%L(Cn(~Pj)!*LCFih^qcy?xH=;|<0D?W#`w$~T-zkGMMAx4y)bM7PkrvbZ8
z|Nj2|^5;z%NlA~Ecut-(XU><ZziXqnyY<QZEWc(f7`iIt<(r$EL5DS5T<m^!mg)OD
zJC#e68hp5Uc_o9Z<|dg+@SL8e3o0${?k@lP_wVyFGnId+792F>xx6NF^KA3{Yipyo
zm%Y99_SV+we}8sfSjb?cXtL!-<<zh<S&2qFSDxG=w(N4$3?2<_-5<s2WoOsiRP^<o
zwmn=-Wm&oOX4TTV4@TnUM|aIzcGb7uTGMzzZ&txVpRdAO)6dUKI@*<ce_!qMb8}x`
zUoZaZ)OL%LDVyYMtM2TrHqXDe=iR$^%xpXzeSKn9A`Z(Yt^8+uc8Xr?<$Ko*`N}5!
z4xe1cE-llo?OtyB@lWR@zl!u4{X`M3O9JwOnH68}{WyD9B70k9U0cV!{l;!Ch3D(b
z#SD(kD&4lgef|EwdXndUetvFiYpd=v!=U)t8RvGssTzS!FI<#T)6&ecuB@1Cp0B2=
zYH4NVH`7Sf@MC}aBxVLBC+BpIjaEvFZj}G&RoMGp)K)NBZ@o$-o3M4U+RvgDb4`OU
z*c{DH@o^Nc`u%a;yV~U@llAN8*&jJLXXEb+AJ05!Q(JdUNQtX<mHb9`>5ql-*2T*f
zly(`PG*VJBE<X0~vp##zX4}s%lX7l8$+R>6C%pOlySuC7_tzcoldb;pqVVOVrBbFr
z+4FXsIdkvcz3bP%pP6aAJ@2ko$O_N_DHeKHpRH#6k^NVfzbl2QidQg@FPl@)a!Sa$
z)mol$A0K@b<$bR5e~a9)1COi3!z=6kGk;{Cn(XT&m0Xv8b3*^>j2TZS^sQfe!Cg9c
z(?-MZrtG{y>8gR8PcxSKO^$sxr<E@+@O@u;mAQFg@TZR|8$&-;m#NR1YP@ph<lb)M
zs0fdaAHnQbXY*WQ{rXftPGT<irjz<V4^LflL_tR<=gbVlc7Ay|n~D$1d}lxTv|c@=
z`G42~hsIuM^Lx8Wv%kE!Sp59lT)Wz;$H#hG`B*13K9sO>e$aVhbyt#0@#S-UIWarV
zecyD>K}32{Y+a!6yx3_)wmr{oF8yo!G`(D<@Amv1`gc5xs-(VTit;YYk4+77xYY62
zlRZvqv-qU8m}_=VUVJ>d<fQo5<MSjtJp7-8I!3B-+h+z}n&A6tc9!R5wNt|4U!xYy
z5XgQ0v@`LD$cD)wT3W{bkJ9E}%Behda*6!1pL_nKYe#av`1||&;Wl2-ff}1Pb8~RK
zxV)VI)z9rybs|4~czF2oa({X2vNhAEUq9L{-s;CPA#mYBV=lMZDK%d_w&%3(nm=F9
zr0aRdewl<FGDjj)|0db?Y|vRVWyk6F9d_sMo>gDJ=->>GPn-UHO;8q;wVHIJviF(p
z#4nFq?rnVaME$YkcbheecLdyAT&B;L_E6JwwGph>o}v8cs*Iq!`FyDpPTSVaUU}H~
z?zZwL?{p?-1tqGk{<8dPR{Gglp(_FwzIgE>`}(?GdHZ{Z4mrIDnCW<SmZ^UHzBg~*
z-dz{FyNy>GycmE(;jyE0ItPzn{r6>c;SCQ@?LKdOM&<mKzgMdy!j|y}PJVnk`s9-N
z|NBo)e|z|9s&#hfTl0I}{+B)kRbOlD=!kQZ?DUxa=<THTu6_d@o28YxQ$p5cS?v&r
zUMe6hI>$;oa;fy<i#zh}?s977irrE0@W;o;+~Rs?=GpGPu+T`fE%DZt%;MkQ-WEMQ
z6}l?q<n7zmHa0OY0zlKVR?HrDBG;DBaGCsng9Mvk?OwUI$t~)WLUa!bO<X&vLxNpW
zb^W4)$3MUC@Cpeq+ms$N=gH*PCntwbO%d&xyR~HpfAYQhY!%7qW3xI=te!FD(T>TS
z^OGeyBBsT3own(nbADC}1FxutPVtnF;X(PTt4j)ATu=-Tzb>jBwkCc*=$NU#zP^=)
ztG*rUd;0nL`T6_5mD~}(ef7DyE@)46)7kJX|G!J=J-hhrr?UE69<QB-zm8RjZ_hlx
z_0M5r>*e2HZ~t8PX8sYb{kyV{_53_NO*eX5TK<e_>Wgknm<%$`u)g|J`>h%7x1X=)
z3|l@U&Oq_R@oG<(tUv0Z8X~p7zwvT#Y}mL_Gj7k0{(k?QH}bWX4E6jbTW;*!accIH
zX#$44v$X7OQdB4NWUdMk2wuN((~=&(xmVj>&I}Uu6)QC{<-K@j_afVWlTJq;N{Kb!
zto2RE$fz`1>vh`n2~*D;pBh=@wK{Z@@2nMU;gjCoU(cyrC4K(+Cxe+m!58m3yB)c*
zMsNR<^*cYz%D-@_#3}gludqLxYHx{Meae-Tbv@up@$~1Zn$veHEuW_udN{?!H#KqQ
z$9>Z#zS_B1@`>KN?Ca}7SA}Grn4tLhSg&>Er<9pSsX_TUt?s9%rKMkAJLmHI3Xem}
z*S1YxUhN;h<JX=wuG#)a|L<+vdUCGv%aVjc;?km#+@-H$FS#9jQd$~3$8z<u`*UWQ
z|N6J}d6r%JOt!c#Z_rs|0lOQUn-rB!A4*MGeJx`8`Zo&_OxCYk8@n^((yPyHBKkc0
zxBZ_{S@-)&w8G&_mx>Oxa9&>*TmAW&??R{6XJ=+EzWVR~dToXetY=RDoxNsLO6lub
z$HaM|FBIf!w#v1gYT3B#y<YjlgmW!NKi~iT;WEd~sb7BFt^ZSDz`9oRk?j18Lm~?Q
zKCahU`|<fX|8v6M9(DHio;|-`MMWj$NQWS3*??tH%I|M)FQ+xCeciT9RIU9*3h%Pm
znb#Z~U5?D1cX8YOW7R?DFN%k6{r90jYI$Gh(-1SUldXRXlFu7XYP7!fv^#im>#N|-
zVwsIPE4Q1jT`;p|S8<@WlG0i$KW{NH`Sy!NYnnojube8jsz07bTzq-h3LOz0Nu~AW
z>-Q~r-#S%QcgYliD@Ql?mV7uiHTF(NhfrSm%8JT!YwPW5e*ZWg8?&5e*PdgpQ)^gV
zb}wHYUa>y8*1P_-(FytLA0Hm>D16Ms#H8jo=f*;3_D+`LoDBEd_Dj!niTER4yT<R=
z`l~DUy?XqA{qrY+|2EI-?5O|K7cTtK^JVP${SW{B{jH~`mzS4U{_f7pD=UL%t(~8d
z!gXEBWW&;b4?2q1^SB&wkH7n8)yY+!y+8WX%MP#4%e-|d{diyHyL0De=kHTgnzUMO
zUexXFnVIX2-l@9fbU#u0eop>D%%0n|rv5JT_3~_Nwk*28qc!-~^zzsH6-|r^;`TpV
z_io+aU#8lRLzi7S(ww!Z<$|~HuimIv*2ng`Ocy@C==|EbwRbD_o<49>JU;WpnVrw~
zAJzN4&dYA<-!Dh<Ru^uy`(L)$xAKpVNOk?=-bYtsH!nH6y+cM;7Ie8&+1pz;H>dyq
z{eFMu7d?iV3{q16xYoz-|MumJj7`Oc&Q8x+`zGpj>0ez?S?p$%bX`?x*Y4z4$@jM`
zJtqH&wEI=1`+nu(>v}J)X1`e(x_4RWyHf2wb*0C;Pamu;U-xF_^zRR*R33k%Jx$v&
z-Y@KcdwiDJqoU`Id)av<Q!Px3Z!7N>)73K0xqI(&+1(qqq1%hk25+tM-&>y(_3g#w
z?fvHU*)OJSey1BXX~~s!qIOS<G&)Z#PRuv@m2dNV(u=$QKCZV4`Tve*S$>{eUEKQ6
zMAuvAYTmtkX_<Ft$At?QOlrj&czAhvC+reh8rii)A}nlk*73FB-@mUraec{-sSh8A
z=)Jn$?-``^bKQ?Mc~`wZ_P=jCDpm4I>2LItP=mBq?Gg>$`agP}iFx0yE+{<zPVnO6
z*W!6Llg`aB++Jq4S?61%O6#2B)e@=m4lR|?zO=_C`N=(7{tx?Qf^yF3+`lKcW?sjM
z1wRXvTxMM^uBx5fyqNDUzeVZBIP37M-?<{H3op;x)_eET3l(ODKQ?DH*(RKtWD>F{
zZ+npOuCg^R@1OcUK{oKr^zBui_J6h}pWPR?&^v2twAU{E^OnMb6OS!OT{2br>gwq=
zGp#<|T<xOt__E01BDN&1Yx$RUq!fKU=F+nF*DIygMMdKB_wr*-mzmU^pZZ>`;LYD&
z^Dh5bjcL+44FA$B+Rk;%=qsIad6t@fMXJ%-m7e<LFa7UaQvSGZWz40kTNeK8FaN99
zzv&d~*ZREPwfnp0bfg3&ZJP4xR@~7IrP1MMd|lXN1t(7Nn7TGW@p12D^Lqk<@_!%k
z1aI__^S*S{w%32(B>Cdp?aK;x)_6$@P7FCI;F;>85*l&q`NPu9X>ax{R(<Qd|K?}C
zqbE!`Coyqp3Ql}zJMq%7Z5Fm~FXflcxHaL!r=0zX?I%K9UD*7jg?~PLsBe6H)nRXK
z6~F%>T7RAz?h5hx^ZN<+z5I9c>f*MBK722i$n5aT(K+4aPn)vUT;A6U9+>~m^xJPx
zRiTl-J!nPU2d(4+lbNSxTzL_?%u8^knbBmO<f&OZH1?+&r^VRX=Kt9gZCLYj+hb{-
zjA)O8cdYj%ed+Oz`S3(?w*Ko)Q=+z2-uX1|YPwGJwe|Pw?#Bjqt&oVm&NfRl*6mEZ
z)xK=~*R0jiH}^&D|HvV^KL2j*-+K#1Cp(vX7Spdz3C(=Dz|C51OL=)&e(~v>%gy#@
z{1bZjaO2`|=0DTcSEcOPSLz>e+jsJ(__^`Bd!lc$weLFrwd};>OY`<$J$CxB$BX6C
zKi6b$%Macfx^@b8OtIlX>E}AT&e=!VRQne6{95$>**4vRE@=mm4_mJ03;oK}PICzl
zz2}kZZ*MEOH6tZTUpn{xj0%;B9VgDs+Gz5j-`;M)oYQfKj!Pe&)M1!7y)@+A?G<+C
zjl!ht-aL*yl=jRgPLs8EPjPgr(<_xzsb`Pud2H`19Cl$=r|*NslhXOOPFt6=(qcnk
zt;3E?yZW!0t1mD2_n%?#@We#r%vrCNXFh(kg^Rmeq~E>xm4)d_JA2cwzH`FX`hDDT
z_4~I8by72fSMz#JeI(8mzPe9&>CyFj9(|X!tEsrVtF*4J?&Zfc21$W+&gm&9gY8bL
zoQqU(xpUe@X=aK`aB5fSIv(d+Gtw&08(J=%rQIn!Z^FbGX}Te6r|<~Yzw>xBYx<S7
zot?|O>b-N!_SSupTo>)@Y9{I|Bj~;4`L!J1s%f!^A;*_Xj`sJwJIU>vb|7+x>*Ld9
zYD!6=_J;Bns{?BP?)rbPIrMM&(|YBZT$d*?cd(u-N{^7y+;n=`cB9y<S521pS(mvK
z_pc~&Jv#k$%tk-4lAlY`o+Re(X;M_;y!hFA*16Nd@dCjbvs(-&R^0M#I-lwG=Gug*
zS4>Y9e75yZxwF}J;dw{jNj<sYk@t4CDQUWx|DW>A)NrZRiQLYWK^?suClfXoFf*iB
zWW3<%R?S`V@L6=6mx$P!+rK8MCwW=aY&>^$F302SzRX)!b8-^v{+@lJocHgYa$m>w
z7bo?sPpcO)Z&TT)wPeQqAV-&~MJ|0F`FlkDIoI=)Kkk_7Ez>b2a>?14C5il{;Y>Ui
zK|!!0at9-Wk<y%}=Pvx1yWGU`ZD^jH+W&ptF@L7~-*GPVlW5wp%k!S)`{#r@KYX}i
zPTpeaf|kYo-XVg5$3LnhyKgs5l+XIL_PYMtl~YvbTm1i#efze{;t)TXkKyODwjC7~
zH2gM+EstyEu>!5;GlzH?CKNuDuv%;+py?x_RQGS+lI<=_voxe<)$ctwb@$pe@s(v4
z=Pj|^&KqQHHhWT_ioB|@WJky1JAIdOj>PZG)Oe$1syJz;ihc57jm{30cQ;keJM+Bn
zSb6u*l0}wJPw7X;#0P4{AB)ZiI&uZnq}`u$bJI`bY43j=I~D%_?PZ~e|9fxG{;chu
z$M@@fErUbLg$t5`m$&s@3Uo_Ro*fpPHtUG5#lowTmSjc;&vySc<#miM=hK{G?PXU3
zw`rYeV|yumJw(7T^z`Y|yH3fZE;*rB+Gln|`0|3ak@D)-9DLrkoth&wFXGzr-ji?E
zEnB8vx|xY*_qrE6M%%KZH+!0<Ebo6^oVuk|?Wk|)LSN^bkCtjD#aeIw19RJ#Ldn(n
zZ<p@fQ8u&mph+Y%Gs7j3b4BScf2JvWMhE3pta`um&*RO({9iS%?|ik_=HR(m8?UJ7
z=Y6c)wCXhDt|#^~af{0)d`k3OVw9-SxjyFj6pOrjc4w<K<;DD6N|YV0zujjyT~c-L
z<NHFfcMrBc5-Hb8(%m#=(k}6WA8xkwJ6Xa$E%Wyd7n~TWxBj24W#0*b%36=tOKbI~
zobe7)3Oasd!l_S(ox@HRx9A>~No0`Z?^|fRaQ(HNT~*)bXZydjf3wE#(bbdTD<dzd
z)UJ5$|GQ=1<NHE_fiL*Q_B37j*sHqXO8>Lcw;M~|Ub>d&`|IoL?GIjj?e|W|4AMy!
z-C6Fm{i?eD+MLH5-~FrhTAymbprHFeLdwo~^Yp42!B?t}?e42NCwMn{;pabFCrv+a
zt3Y$Z64uK{&Mu339_Om}Zlmq?y4;%H=)ckJ;u5j>KGOvEEq$`>d*;^tQ$*r)qb{ZG
zkJYZ>Qe`#0nQZhm%yzk#8T+9fPf`tbeS6hud5FPC$;6~)nn0$+-mlW9%^a+L?^oS0
z?LKd}?pM98in(nX;TMyRywHhwWUtql`eRGa)W~VGSoO9PluFu^SKpo$sqyXGG3ler
z#k^NfoyQ28Cp0z|SC3q7^l7{K<YnxMd~bQ4+@0%g_@&-EK{VTH$)huAf74cQx8DB{
zX&LG|<wNiFwD-F{7X0hwX6Mf;-SPeuYwHFcHV3x0wsU8i;v{&dPLiL?ba9eU;G5_C
zSHr{nZF0hPPAI&2QaZnD%IRr)SgzSu>Ifh5`{}*=&AUq{yl!2*=jQ4%f7*dbr+hk3
z9LQWeKjwbv>2>MrSIXr@YJKOkh)wdpJ?|c9IiBOfg~kj0wG%&h9*gLU=<_%x5_0Kr
z#lO|ElkERI43eA>wnJn8)P0lJWxo71V_kUh&Dpc|eY7r#eYEk|{=|Cq+O1_*g@1qH
zv@X=I<XmI^kiX*ZwPm5<>ibJB-TG?SZm-j5x^1R+^}Xf2(~oyNJ>8_XE#`p#gu7ht
zj8~uC(ztVb($e$>9`5YuziJDwZ~L{*OLU6tCj<URlIlAb-uC<3v2N8KPq$4IG&&Z^
zWwre(-oG)V?Lz!z^OO%;*VYA{Qr+<A`!Aj;C37!p^oQh5$U4+__S<>8wdpI*|6Uex
z?A7Db7Nv6@x_jqxaArp{{xP@7oqEzhwA)KK{zck*|0>0aB0l$)nes%<D|oT>qTc>1
z56h>WVpyo5W4E?(>W8G}`&S#X!{6=asQ>uj)3L47S6$QjbadaESC_RW%f}z<+BuK$
zLHmUZGkN5HOxmoGwP{c5qa%!Gla*GsI5qQTEpm~a*z)4~^iv0n!xtqy+mm&E)0~Il
z=4&F0l@u0!JaN+W(@f5}oD87FJ?Un@{j$Eub*UX)y4Pyfsp~3&6HT7Zb=to#q-`@N
z>)+!o>J#ENiReAI-81KcCWFG?2NG736OU~PvfH!f<vwR`m5srZ8aq-uJHE&WPW-xF
zf2yLp?SG!q8{Z_EXO+FRO8WQ8_w+TJU*AM--d|p8cW3F3brb$q*L78wR~`RUclH?j
z>Pn?9zK`<vw`N~o_xD-;tt~k-<72c^*%=BC8J46aC)`}P)@NR}hje<y{P=m(Ra<v2
zs$6vMu%naC<r7Zx_k_4D{eJApQX@Urnh34OdpWi1SI!nm&~VxnH}BKB+_N1$?By!=
zgnntC;@EL{^Q^NbbF1=vP0t=yo;r&+_R5B2#jE~*o_1B&nP1yhp=Ufv%d70S`u?l`
zZ6%LKiJ#H4{QJT+U+I#K)crN{pYIZWeD`c#&iq%kr<OXeH~A67;9vp@e*4_oCh!RO
z@%I%<R-zvgJrBqB+sVz3z5f4FkmLtpp~LTAeZFoKD!5YSwao0|eE%E2Pg&-4KJG}}
z`pP0Gs%B4j_Z36KIj1f;9TYnHUiPlds>>H2?L5<Q_TH-c_5R*%g|Bu*^ljZ3AULu2
zZ~O17rW5<me>nE)nU>O2_thI8=lj>MeA=xcp`@p$Hf>)_;^Y4{8=tTVP7IS>uFB1Q
z^?Tfc-tNnrw75m*En^elIaieK^h)L5$NDw-AG^I%5BM9Ljf@i8v!VJwE4%WRDVwhy
zShsFc@@Y4#-Mh@*-nzQCXZr5w$$!5u)-<cAx}4TsmLu~tFzM}8^M}v3_gb!vG)X=5
zj9Z`i+uDprZ#I2=SFQB+=Th4!zo^ZLUasF~x(Ba2y!K4s5;<kbt5=HO&TN;_oaG}i
zDgM-iUl+eDd-wU9Y?Sf3o!zfD9N4h?^R8(nueZ7T7FMU1l<VD7k2tH|5;yBY{*|En
zhVN8&&8e8?;KI+p_Jw)g!zurs-u;!|@!-ns-MhZe-&k^U597IW=TzNft`rI0PVbqq
zuOxH1QQqWliHVsE2Nx{-pQSnd)Pdm8`i8YuSMr0p7cCF#S@G~<%JZw&wOW0bs(s}a
zy!_$*Oh?D($FJ~8TYn2kxzT#|yfokYtMxB3*0&yS+7$Z2V%LT{>A%jDJuSArzA}2^
z#0gV8KIgo9bb9Wuukr?oQ_CKITCTZ1-TQN&<(@Ti$}=x-kMzvH@37VB;HvH4bJxCq
zZoVzM{hu@Y#~P8y-6lB!t@ZOAQZ6oVbyhMBd^E4>|BWqgE-ufP&;NJrg68q97pKqa
z-EpTf;Ju@>o21{_J0+Hfc75Bk^Ki9!p}vR9zS`G&EH_oOU0Yo~|K8#D>RfqawGZLz
zS28<jIXeHpDx}1_c)J?k-FX&F+L3=1udO?|LpJ@p&MGH=wTkPvip--jGm}+6YCN9P
zdSX}HamSbwGhghvtM_Z_z1=xUcRj^>^cP=Axgh+KCrT+v^x94CjV>-;%VWd+pYGUu
zF8zj5Qc+#0=qa7)*Lx0}p1u70lXdIzqD#Xv?p$7cy<}eNW<kNln>^?3b6M}L;O6JT
zwzsdt<MXGUiPhh24*W0j*l4-Arn)*~-d*KI$FJ+(sZg0~n4I_a&&{lDYZN;7-CO<r
zQ(JIZ=(&TeGWJmij6U}L|Cup6tysM0RCDHDMwX`67e0hf@LJk0Z(sNGQ|h}rJA-Qr
z&R$lIju(3yWa&Tc&{Xjs>%XtOlX?H_g>9`<T~Z7LG(EDE(p7f8eCoJ9?6~=l^VSng
zx1ALGRnhqUH22<^8%*b`+IF%pJHLKW+1v?53pZ9<Pg}kEyGHKfqP6$GZ}5pO|FEmL
zf0c0HqH7K+f2-#7SDbV3Irz(0YPyf!tr;Edj!|w=I|KJtot(Z_dH&u@Ld6wM@74YO
zhMN~Db@_Wdko{D3ZO!!dj*dt1M>BMCg6fOP`yT)K`*&&3A7Pdcu@-G&*2h|8FWxB$
z)4m+3>!W6FI`gX5&-16c)K-4u+k1P4?n>p~59{RKbuGPnCal!-tLEQ@JZIkAJUKUH
z^{$m^SHneo)>>p9K7IQ1?c2L!b{5&$*%=xduC(4^<i4-?@3H$`4?8<HZZuZZdXnJl
zlCno|xvElCMcVQ`xu%bouFZLPi8Xh|3Gu{~m6wZLTy`yS$&Klm=JEZ8#&j(qxjFG?
zr>uE9)%x4Jr^YT-Yr`}r3r`F=c_Z!m_Zb_^f1cWsD(0QUC3<SEF8^VlJ1^zkMgD#`
zR;r|A^y0)1l{jyapMN9;GnZUD`Z9BR-Ri?v+FGWk|K_z`Qgq_%>$I~sD&4(rwbyNa
z9@|w}UG;K?@y@WdAGRs(+L*TX%H}7X7n%K9T#9dXy#4ET|MHS|<=q`G9vp0zlau@Y
z?(S@x%A!~Od=nxUEOb3T=j80xb#d2z?0q2nCZcL_bYa}-$W5$ggJ1Lh=<qna`uqLa
zHM2A0!m_+m;sfp-4=wso@MvmAk=DYT3_q1*b#uduEidx+nB96hh4)uypZJ6jt+^J3
zkN*Dt{<UiF?Af>1L~fomDd@dp{>+)m_1T{$)SbSj^E!IZyMMo&t3aa;X*`#A?>@gt
z@~?H=s(t(GS#Kw8-N|mex2xQG_1R;wGc9kwPd#^O6YJbne|MX{xOY$0xVJ4hd;6X(
zB`c5L&j~)3dh6l@r(@D{ck4#%Pqvp_pC4PVbMn-&f~*|-`7g7l`RZJ+tUK=c_|Ds+
z(pW2x-H9F3H-B!a4etmETErB3TkMMNiWhJ8yqmd3<o~}XCp)&yR(rbNYJZvJ_D#He
zrSo6Eov{DbCCA<}pN#kK-rRdO?emJXF4n*19$c0&_cqoXYstH-ck_Cd^_%-a`fO%v
zpS+5DTzh(<Y0;8rj9;Ffp1wMCwOQ(^DWKaSEQ^obylE-Q&VC|gL(#E(ukBNf-+VrO
zYew;B=ALhUpJq;d!mp?1$8YqkY})n&-eRVI>%&rAsx}#?dwi~p)H>Fp`QhV3ZKWcE
z^h1yK|Cax|x^$gZsKLviDQUX?n+m3XEdLt*<;V<=>t!J_cNX!TS^pv-QKERB@}h0N
zKG|sx)z^Df%@Mobl>hVURO6tkFE1{32r93O*=dw?q@z#PdQ*;;O~8@DBb#DkeAYPr
z`nXa3Y_s0dbe>%sOzaC+_4At+Ms0Om7$lH+HLBHX;+}i0USFRk2h9n~ov=zoXL;yk
zrK6Yq=7gQr5}9MA7PdOzW=)mg^3ZILrIYO6zqVR<{Hdv)*0akT@jWZAdFk5OPdJ)+
z_0|8HTV8AXlvlp$cKmmynVtVwpRBZmghl12CuMJMd8%00F#KSQVG-JWV)?J1E-q7Q
z6ej<Xf0H`lU9Rxu(s`416vW!w$3C`F>n%1ue=o=`y3_HzcPfvdd_|zKddKQ;-$?!4
zEt9>^SmxcFwj`WYL|FSIU!LcUzaMXEiKoxpzWk7im-&xPg(b4CW~!4{JoKH)b5wF!
z%#$eb)Xo+0+D~uU&eE<@O4{>UQ^RY&^*jv+>u1R)ENlaHrssYA)6ONTwI%!dy6o%g
zcxA0hE-rEvR`XdAzyF_n-^oobN1~Se=xyxp-Fa^R^~p}DAAkC3e@lL>WhbJu`qzK<
zKKpk^v_j%@{p0u!A9MS9^_`ZI^N()hKfm?vOwXTOv|L|)`oI0W@xd!~{=L4u|IyXe
z;gi+<?P`8}`0(&>`TKio0}^cc8|F8~CmXN&T6e2LN$;SCOVy(1;U?btuX$Y(89~Do
z@~b4p{VVmVBfFd4?7Zyxdi_g}lV+b2l9PD&50{<~iizamF}l-dw&{qvixgi=fy=SQ
z)mILmdv&5o_n+VX?=vS_9A5Dj<o%f`KGmT=b$D)`w0hzp=z8qhDb1aAmp_!c6l%xr
z`f_t~`tvh0jWaJTd3AMl{LUg(Q}^deWoxW&F5LUPbLr`Id%c>ucgA;CZ>hU{lke?a
zJ()eLd_SB2X5F}PWAd>c&ERD|-`?Dmx2-a%_>geL_`vzYh9zxFpZD1rZ!4HG^^5tm
zv*%>&lX{F2Egt2w7Q1Zu*;kijnK`vdNARh(()nYT<{sa*p&;2w#r|yn?;-`esQ(Y$
z9w4n%n$$Dtxv}CVr7CZis>e%wInykcY?jjS7M++|^Yx)g)p9qke;dm_1{>-r|8*{X
zeQj;*?r+C>r9n%p!8iJ_sH-mzk(hq=Ue=ErArjBkBSk*>o!|bOGiq(w%}uG0Et@<%
zJZB~<AMcN6W{5vzSi)BJW0RkE&hs#@&?W!w9Pd3jch$uYE_?2IxE!0he9Py)?B{bY
z*-kdTKWEyL%j+|}etk2x*6e2#j+5kTmG-s2=NY$tS!eZ}efKqO1#fo#ire^pl84G%
ztJ113F9Mf(P0hNx>g)Uc|J__%IApb$jiq{Lo9CAm7w;~5dg|up^zgM&tyOG4Y%^Z)
zJf7Rwabm(Fh8g|lGMwg75uS2)epQ4^7*11jahclX`O>!h)!K<2vZ3yI-adP?)aK5K
z=(V+8wr|ms>7lbufQEDhH9IWgoL!D=aOs@YfBb@)&rznFuXj#f4_2LLsoMD6QfXDp
z%_XjRRsu&tPn|DYm49h-2HU^FAPo`cHlE5aFDCBYTe~u7>9JmEacODok`+DAZ){9X
zJv}XUciG!%y3ya>-;ZD7A-GVOF(tv`^jt&Gq`tPg@}j1tQ~&yI{hVK?@#^pOFJGm-
z`1k(WFni*PB<IV2KK3-9+L5-}%&2o`K#%VHXuI9Z=REqdc>Al|IijpZr#@+Joo{vb
zXk?iA?v|8I6Q7<q7U-sPf8~~*)4J1zf;aD~$<(-Ru{_7a{p8$&qSKn0Yt8nlSJj6b
z|6h60ZvURm%HA$-rDq5U3T{83?>z5?-^o*_vX1pg*8ct`YhAWx_3GCT4mMYRd!sr1
zbnCgPjCb}_7JqqhapJ^@mzVqJUs*8`v}KdoCim;?OK&q*|9=*Ga|!3mwqM0AM`mm|
zZKc)!-eZrO$gHHNJyX16FZR#d_N%_^8ISz={i>Uc(>Kk!@Y*AA&5Doe{?UJzRDXZ>
z^T!VsHnu)F+g+uvuT@yRsGet2X;k;;hj#e79(ntH+1J;t{PabNLEv0bdd%EeGuFLY
z^6KFUnJ@nfY|ch_i?|p)a^4?v<JIb=>*`j{>(tQt=d9{}^t0om70<r}u8G*#%+8;8
zXGh`JS6AcW;*R&ri(5^6v8VEL+JOef&{ZLY<>mE%eiVLxcQ^1n!%7a`xq-G^&mUR-
zyTf_5z4}PXq=%v@f$uB7&r*{uN?zF<61>o<_4IUo{}~1cj~`#2cX!wJyt_hH6E##+
zRD5O{9X)*b@R>6@Dk>>YPE1r&TWQ9x%b;>N^4<L@hZW<$No~%Zo5dmfB!5ZW?<cQn
z&X*frOiN39`E8lZ@wHJ~vmPDkoHuWtu$s?_`S$g@_HJGA;_I)gtHaOFvE1y|D>Yf&
z|J-Et{tKN9M!GW2|CgPWzHW2ty0zJtWfvUly22xmZV^+vJWES^R?Xf!t&w~$Tb`*+
zj-K&r%2|o+yjk~JS=Z+8skpf4%bIkhq{waQ=jZjy*=hwYa(Q}cDm$Nygrwxmd5kQ_
zKmS=5yZg@FyP(GLn;RQ@rOg-3W4t6H^~*eTWl-wtWm~_SZSU*n5VAVmfAjX{o0fS;
z<jz_izJ5AZWsgzM?WLU4z2>|zKEU6s_t5vjFSq0xwl6O(#!g^ZwrtsA_x`#c9~O3Y
zcJAK&dx~bTkdP3I=cIFg{{GbtTeBnKVAE9Xa5>wmD_5_EzHnif8PE2^J#x#dtE;zX
z-E3OWd-iJJqWKeNDKj4z4t%)i(vRPsll5Ou|EVh5Z!+uI6fqU0N6wFX*@S+7<*QBd
zd9dqPZFQN1O~!%^o|{rm8s*>HGuu4huKwSht5-{pc8Rh%v(5hU<@w#c)!PFXyZ!z3
zRobrR$Jf`_SBI=rDp6|K&noq+e@oWYQ{~T|Ea3H>loZD`_v!K$mW;{aix=rn-!1T;
zGq2gqF2i|qpYCg43H@m=bw58lySwu9GiCQa8M~StDJLgYe|sZn6{uDH^V8Dp+shvx
z>y<Xm5|NR)vnkcP?VxW5!%KGFzsvXR-Q6c|YEe+Docl+5;pRVGJX`OuF8AlUcV$!H
zS)=0TXGE+{U%zwbPM^H}zM7v!@$vV)rfSW#E`N7?z2M##FJ3%6+&*19d|koAL;dph
z?=COz_nBt+yoB|{i3=B2COG`BYyMTizBYS>e~*~e>GP)?PRB~{?s~BHtyYyt#KxSB
z7b5qqj@p`Ko_9yWrsBi?|Nm|;@tnM10Z&2sA#+t#)w8oqSG#lyxh*!du&6lH!uhhs
zU5=rqsm)I|*{%Q0rFDfDel273opT|wPKQ~y{GjyC_T9UtX=LoId>l1#ep0H$veMVr
z_~mR?gs+b~+{U}R{Jor%lvc@#_ydJ^Zf(u(7T2$Pe{ZjL_&U)2I`{6yy$E2aH~ko2
zf5sr~><lsa`5WqkyOU(UZz>92__!wgQ<3;=zT&B$Hdl6^DsIhEXYcspG3)uy_UbC}
zGmGwTE)I8_xyDOv^4Hhb*LzRbGfqGE<<qCE+uL$y8l|@IeNEoDaU(Z)HOudPwZFZl
zYN?jUFen`AQC->n{O32>-M{YF1a7%o)*0fWebszje$@P{<+s;wKd(POQu(Qsz@4V2
zp4&TKT{q7?^J@3o>-FUviZ)gT7v|O1JiBs{$148nZ1eoIb8~(^I@<k&-`UyOR0_0o
zp~d&BZ_vsu#m~=e&Au*X#lcY1*5;S$ws@oG=WE;Nt(Y5?J7q(XrR&F)3-|AKR=)g8
zWnEicQOT5-zgLT&pKD}pE^g(m)zQ_Z6}RWd!De<_8ynD`FG=Qg&dZHDT{dO^PFtRH
zX7S#=6^kRz$*I4pSUFdx(r@E>mGhcm+Ot&x)m=8duc)iq_2lc#&CB~!`>NCy)&2cd
z`sRk<R~`m?{^J)iofcmVSsQ5HbUJp&MW%(DQ*0KBi>cm=u77*#r%&<nOaK1<Zs(O2
z6A)MsqSbnii9wQ0%4)5Oib}|e80C9s_Bza6>6Cub<IwuG-8-LEzx`Cn&d$DY^OC;r
z@9$e%Tj$^1wbiMW%PF0)p@)TcZri>6_3_(kM3ehwRj%$8vhqGAP*`2v^1nq}{JfY(
zfWr%&mB()1zJ26K%Ed*lPV9^gJZ?JMjZ(eB!or@PlhKde_Q-o@<vXswEq%&m6Tkm)
z^6%SO@vbCo)pKEWzd3fbRt5$EFBBLie0<=yaZ74?dildQCwJ`5KD2I~m{s=*S9`0k
zSMm?6eB2INj8ZZ!Wb%Xw7Z$tsi)x1@MCmd7;I%kCS8kqdb=a}hN!PlPi>Bo0mWUZv
zeSdSQIajt%?%w;SPgP5%UAeM7KYnY}-n6q)3!51n+FCZ>d8rt`ujcu=IUiO9v`(F!
zR+^_#B2(DOyZ38C_3^#Cx2JBZ+E?)K(7nCY#_w3}GALM>lvouO6<O6j@_3rHHDYGx
z+GO5^k2_rdy#D=t`<8oK>O!|>U2QoxX_kpp@AmEAA06#pyJihI&PCTbFPAqr-@ds0
zoBh4p*3VwDF5KMF^y}^9YUx?J@o#Q#NKCU{Z;*Fqhmfk5N!Aq)P*^lRv?{5xE`H{-
zc-h*nZMXAgo{oK?<8)~cH*Z96`L!vAcXyWyTX}2kE`03P%*Olf-rnr=boCN3h7$)0
zc;?<)6|!<u>E&C$XGb4*|Fbt-W9!F!rt9WX)=3G^R;TZc1C6ilEPlRd9^(W4g$r2&
z*2nD)JuS|bo3pO3N~J`M@r7`5(>2pq*H>R&(8%0!Zql!1esgD;=if8Sy~QER!eC@G
z=c(A4S*F?Syz=3DUMC9AzT=ecY0s8h^WavS?5xoF#?@^XB7@=+5)!s%U43_F=VWDf
zzJ;3^9NOBecd}mInmzqTOUuV#>78ZPs|?O-mCX5a;N|2d^Q}2Cr;UTJ86Pi<-CZ`<
zELZB4BtsDExgzeB_51fBm4x22jHOEWw{2Y=vg*O7r8A4$UhOP?9u*l`SzXNyavkTK
zr(*XmJXF4xZ~yAj(*+@!iY3#wBt7BbFDofJz52%XYU|URoQ`|eDk>^IJvDWA-QTKv
zdn!R$WG0JLlt|V8+xhoDzCLR8F7@d$r*zL+oAdT}?rvTXdgtQd->F73FBAsJpZ`2n
zD>UokBG>!-YQY-W&lPnan&kR<b(Ixser~o&M(eprzvN^kG7j6@mlXIfTa&I7wJD|Z
zV#b@x%lj`lGAINeG~5y)(R)noZ1nxAMP*rA0}rLmv;HwpM$%Pq`4_3Ld#k@2Jr(lT
zkK6O%!-pLO51SS*UVI@Es?S4ZQtWMW>(peYZ(pADx-ERXr0@CiWg90xUl+eKVr$mb
zz{PHyvg*Di@9))qcyMq<I)g{4MVpv-uUqg|x4RsA<qzGsTh4u2?xe5dvuFFmTT?%O
z`}Xbqy}iX{W%u@0Z(qKA`GuW9mv3#&7T1rf`TFXrPvviE1`cjsU8}6EyK0uc`k3~P
z^_7pP)#>-#ySIJ2yS4JV-u&`{0*`y36Lbn6AG^D+Hd=2w_iWv*E-o%RcJ2h7dQ$Q6
z5vVZn;Yi=<>b^8+u4Sdmw{MR=2M4#D`!riAFTa0}#p^>igXh1R*)hvJKW<0CL(O2L
zSK?p)SABnXcXPV`6=Q}af-=tCAt50lFE@!!?aM6=f1_VAEnc|t{QIL54~n0ks~5E7
zMELxwudk-go_+h}<>hnd&SgD0t!rEUeY?y{OQN^urOpawVo)&4$oN?G^;M|m`A=c*
zYGrJU&oBk#JNwz*yM6ipm5oBGUNyz-Q?<j_#qIrdX{oo6C!1aU=WlPL#l@$C?w8Sv
zjRGA5k(v6SUe$Q7jAc=bO}W*puJ@~>Z?~NL6nxKi_m7P7{>AOz>etWfTv7e~-LhrN
zK0iP2e?DR5y7>KdZ*OgNwq<shb;xkb4cm$j2Ua^xy|Xnv6;$+0`!6J2_4?nlY1QSq
zYs1#ws(=6U$w^^<|MS_`*F8NwUB7nI(sj-3{CrX-FW%kVt)`|13ePh<ymJ}v>?~eh
zv^C1?=xtBYX-?@s&6Jl{7bjnx#=Ba|B;$ke@AI=vUmxidKHe|y|2ko%o|)OTn>Q;z
zKRYX`9VTMM&5%-*@uKJ9x3{;Wx8)g29sMo8I;}TrYu43YC+AxBX$LR!nQvEXRsL?y
zw1q*SOB9|I{hXxg-NMJjFjHPiszp#hAYgOa+1=&u)6ULHymrj4@>9z8yt_=y%xBM@
z1?LTUDJd4?tSc)fOqej&x|~b)$?_Ffv&6-xFZG`O@6Spmh6ek_HnxKqTcg6mufM#!
z+&u42#0wq!YwP3p@7Mu47MoL+h2fuAMuxzpR&MdA?RmBD@7a2LPrS!6{SD~m%;;@A
zTwI&BZ4<NNX85Cf;ex^j(9MO*FRzW+8T3No{?dyXckbL-<~#e^5>MfUn;8zov+(jV
zb@%s|mzBkB&x@U^720y{Q?8wzo&S8hyBm|+FE}zhc;D2<_E6TYW=H-1e~;!cKY4a`
z_SaWecUOO3w|DQ}3y~lNk@k)irLV8)#_sxZV`FkF-zQT@v3hfJ`r>(v3V$C;NVz<G
zdwcu2bLZ;n>g43)oYH@Su8Ry`ANTe1XJIQ2hCjU*E+~l2HqW1DntiQT+I&gS$_tSl
z9iS6?mU>N%-k!HK=BX5e!rcct<{hrP%ib0}I|I6{cww`EV4=+Qt=ZR4PF7z$kI_NH
zqK!?Elb5%&yxhM2-=4*b7jN)<c4lVrj}H$gtNR=ADCY<<{K#Io&@o0#KW<O%?QO-+
z&v8cd_RHJrMQ?j^b#*vsCsPO;1H+uBY=u9c&#&*uczJ256YGP@d$(^_Us(~jBAr1%
ztZ$)XiJV<c#6qXm+2;ArPEFPRd%yPOrKQs5d3y>THn}w2k7so7$jA^_tQov4W>?8e
z(3Na|_UXs(^Vuix?#YuW=kj?OK8VVwHc0SFo9Wot)SRBC`|{5|+v;yJ76l5gBpHq{
z9KWCtWS)Nyv?@3uVZqw9xxe;2{`K|s^YioT-`+BvXfuJAq2AKOq~rR9h0gJNtF~5t
zepXVk{`&g(-Me<lS(lwTdsbKW9v=f}CR8EL)6-K=PmfznXGQe(yr})*mzVi=i|JZb
zetNQ{PMM)$e#_=cg*Y`eHSO?qZ|>|Y4hWdAb7$qPy%*2TwU)Q3*iicV+Lmf=hKBf7
z>!S`=R)wq#+MMPqFF*eY=su2;@9*x0NP=z%<mBXZ5s{U>yTAUwY4$Y@k**L)Nd|>b
zP@LWU{r&yixpS|tiQHWJ`daDhYg4sCU)|c8U2pO1^mP5tPfjMMr*E(P{ES!H?8eQT
zXXjd*hhAc0SRyv(Dci=)n>M|=vNHJbvEE)cWx>1q>*qiJX+KB!%KiKIAy;S=K0any
z{Ors3@BVYwFfgoSPv7ZyWznKVQf4_8`S<oLS)#HwZ1$dy_x4uTTfCX5>^{lAP+d)J
zU(L@;H*Z$n-&b32p~m3w%E`GIbdlq`2M3!wJ3HrZljuEG{QTVChZ~mr&;R!&eQnrp
zP)XxE3v_P)e>*FKhigWLfN6aE{cX9oXU&>bcWcs=DJ4HXJhYcIwz09%(|dQhU8wWO
znKL=>@9nMp{Os@KL`H@naVaSm--Q7=_x9Ype*OB9BTp_L>lWAlcXGzRfB)vE@fsT%
zo}6!A@8IC@_*n1%6d#6$koLAV9?-$};(9R~wr`()`l-v3%P;5IRDQa>xONvuZdTT<
zC7zQH9B|Mv;bc&1X>03Qm~wK`!Gj0q&6~Gu*)q4qAOHOPyx%tDlWRtB@MTtRv7CE*
zB3FhiivyY6{CwfV<wuSm-(U6B>xIaX&wut*ex9ZuKTkh?pR#-3pJN;oCr%XA4%@PI
zYwXsnsS?*V1g#9=<KuH$4n9%ou>Z=ieRqCphZjXga>}-JXv8hPDBWwOnJ53dlI@(t
zQoj>fX=)QY9<f}y{9=Z<e%zCDbE97<Feq5f*vL}XrDj;@+nKa<?b;T;NjI<TueJZV
z$b#WOxJ#h~BLf5Y;2lyAY&tY2=>z+l)i?R`RFXR_)?RcFYHjN1=<b_oYpAJpHEr_L
z?J{Yv=gl>jnDInaIdYofIf3aKXXp8r=lEUO_j|uV$mGUt^LE<vr5d!f9g(}StXA^>
z{q;#PQa87*Te_t}^j^8qclCHqh6C#l7@GWP%&DAubH<Ll+4oqbr}FXr`D~oOZLad}
znyxh~e-<yjz0+mw#xRQ;tG}<U*j!clr0cBIrsVr}iouuTYo+h*o0`?T^^#)sqkW=*
z?s?wb;b-kK`y$uL^BiWsd(m50W!aBXF@{T`Qc`vEkGL)S7A;l(v+r9jli<t4@1*kH
zR23VzL>-yfq0%~c$<8zOZ_nJADJ&Rmq5nD4JI_6G;hOpF!B3}JS}*<6nbL7WZ~YOI
z#>-M27J6?K#XlVoC^Eg4AOGfRbla5Un|o$nIU1Uk?)Sgf|G^fvc_nQd+N=KS3hsVq
zl(O#ApB;y$#>Qnn+%v!Zk4S(0!v3q14}J*!x1M?RnGTOhw*wb>eboK+X*L7HhtCBO
z4ExW&*sn9IW76s=9g3`ilHdGlv%~JyeJiqjxuk4c+k(9h-EY46e<gUvrZyM9tW%%f
zGdZ#QgnyZr93C=VA~}E6j$en;TQ_|^vPM+4L{MsJ^VVNCnj)5YU7MwMY{hmX>*#+s
z7jEBm{=L##d$rxqH(xM(f2PxJj%Ru1V}lKH@`4AgECOy!{g7$CS@&PK+xx}hd4?bV
zxu<$@)=gftZs)UEH!c*^MShaZwQEyXGVb5wvV@;oU1@Wp@!HR`KX65Bepg94k+JQu
z_{WZto=r0j&+-+T_Vn?)*6(xAY?(b{Z<Lwuieo7&ZtiY9z0`MCltqNZ)~HS=$0JAI
z&;9&u$*H=UcK`b=8zQYQecaE(kiurs)^;WR*5bV@zeGN1I{JQoM#?$fv-k4*cYbYK
z{-ZqDdfy$%6A{+4pPkEoIP<VTi9yyMpWZ8RQFpAXr#ih~R@yD`_55n%^&wmTh2F7U
z`rJdSR&HNG^p~TvU1Xfqt|f;(joPIb-o2>D?fcKO635ptOMfyCZ|j%(RrE%$=);+7
zH^01X4R>LC+xvN9ww_SoEo*)alPTUc|H`i3e6lcSna+u=Y1v}#k%wOT8ct7MS>YeO
zea*(#(_G{@(|ov7GK0T&SGc(7_`ko;_x?l8Tis)aJZD<3PW|a5vg!5N9d({_Iz9LO
zDF2=J*+uF<>v=2x;Nu(9cxTP5{poA)|KiRPhKBVI;(0xCd-krHpt?SEhIzJ1FMrSO
z8Y9gsEBZYYoKu2UOx*qLf``mU{j2<spTyM``{#eGJ0_wvVYa&LdPSv6a{E$Lt}SV}
z<1yvQe*g01`ahisO4F7vnGtwz;a+R|`08Ho$~x=qn<uH>{cNDLX!>*0s7X<gncwG|
zPFm~R&6E>nt#oZq8f)fft4ZqVUqBxD`0@IDuZTq}8n(aY7o2!T)m~6j+iZ)*iKF*-
z%GK*zq}nhG3VOGh&(fH;|Nj*anU8^i|G)ItMClygb9I@H<kyrM!K0J?Y9E<>EKl}p
zpJ1}x_2T;doG<^}XJjz4n)9?RBwZ=-q(rIZt?w3sf{#}eoI55c`1`2e$Ir_@=I*{V
zO@CVLw<eAK<t|6EC;3Ocp8UW6XNk~nWw(<8ANTCf-(fE}`T5~%DvvssZ2RWo-yy`?
z=@Mk=`mTdFJ27@+cv#RXdAFInwx3RWkfc8MkHw^r{rPb%FTc!rYFpgb!7+QAv(jWC
z`Cr${-HN(TWrr9n>G=L^eqDThXV1*1KQgbGop<`(dty_~zvtSoz1lj=SMj<;Dcuae
z&*FUHwf)a+Ob70>9ltPv%VKV0N5$W2C8d9@RzAw5FEfJIvn0pgt2=2h)zW#H*CQSq
zLBpc96F=4kTzhG2s(svU{;Ts7)~bABop@)@M0LTyg@1&+g6?w}NOf%eTOco)ub^af
zY{{B8D)TzNyw`lSM{T0WyEq}GMSr}qFSXrKU$jGby976T<(JkIFWW7z#Yjnu$Y=dA
z5fOYD##b-CY^tT{$y+Pe{JR?6ra2+mrJ6H2-z=kX@AAp9T@^vnZRQ12{zyAdn%7a`
zYuUYL*{z1)IkBEQn@(Jr)Y0)|jjFRt+Eeb}Cv$g9|Nq^Wf#HYfn;#z^x1M8=JTU9&
zX>ZRTbJR}#|1a|RqM`fkO>2CV&OX=MS~FinVp;XM*;S(9JEv`46*+77reAN3r@zVd
z+FZnaEhzQYhFOPvH1!ppuIY{rFATO2P5H@gcv@WTRCi?f`t@2*GeZsI53Rd-tn05%
zanvS0S5==W7xylX&1CO%S*G!J$=-^-%ff;)`Ddlfyc;v?{M72YPj`>9gkFuh7HSjv
zb(Qx#+3tv{o{kqsttYdk3Wxb$-&1*HTK<8b>pg3}_}wd7vN!!xz|k$@=U4Ub54xh0
z9krup<&(OaB(YFO@zuvleMApFT+BDiB;%jvz1<1yyV>-FR<dS$b?qwk&GC8KDlGKX
z@@&_Xtvc}y6E#FyPkG$iJHcAq=;NBipoJSNr*)W5UZ#G}eAyp07g>&9&&<nCzg}@I
zcGr!YH<v!Fw_!LjLt>u))|sjb!rI}RT=J)B=jML>te;qH9T;RG{`Kxp2Z`N#)|g5<
z&8z=&BQEO1F^NwPzU$ra;<7e;FIS|Q=9T#7%+Dv~;WH;%J#1I5nfm;7Mg03*wUbMY
z_SHD|*k(%~HT3;_CQHINwL(dAr-=T~hvp0az5V_B+&t~5Eid}_J2Ct?{lLm(iRinh
z&HwK;_E+U~c>OF~qNws|`~7ngpT6GyIKS$n`m3M+Ki9~<f4x_@M_;(3<A;Y{*NQ1~
zTO!)}pUBmI<UYo~pOb;%e)H=K4E1N$?5SP-Xx9pf&-09RPN#`m@iL?cS+ueJNahq*
zYL5}HVqr)zvmoO*NJ;Mp%Qbv_cE#_WcluV)3kCZ$11a7)ftsf!>~Htqb6XvvwJ?~W
zK_c-f&kxPQ!or0M7dkEfq51FMKkKqLpbOtX6CGYlPt7)8e?gPMp~vW0AsffJLx&bc
zY)k?vJU`FY*xdZQMkXV}KKAPab*Lre^s$K{FBlkBIxoK*ySuFS`A^V=yho1Av9F)k
zu_^t0+=c|lg^L*({_wRuuUu%%u%s#X_O`8=mycb#Wc1U0R^p5sHzM5KkF)d3ameyA
zJScx4A!XEgYiqW)NY{-UH~gc0e0-|DyiiO~0d;?0Uzp$-5g9pEJ6!Ln*y4*fmibCA
z{KUZUN72M2MGkcCkGc8(FAKZ7y}!M=$$EyF;e+~x3ll0$GA=l*zpnm2a%s@lH#dWC
z*f22EGx74yZ13#!oNHA&>3@!a#4PiCy&O4^9c^tsORK-XJE^|w^i=Ki)Kt&_dc*y8
zkg!{?l+Af<5w1H&c^M>S=WSzp=<4DEx@mH&n|9=;Cp(Lue|mmiUP5Aq7PHsVAa!;1
zw#1-R6HpgQ(%Zqq(9rPlvEJKzDvP!5m8`v(@#yj6w{PAAWglp1VX3bEUHSQ0?N<|V
z2GE#?3CH0C1D&`%6{n_X`pz=p<ls0lNwxbz=f3XeKV_}Uwq#yDcKLGgwKb8IpPmR>
z2{QZ<Es&LBDfsarF+Dxq-hRKbd*6>=zq)qL>+DFJ5w<q!?!MaH1rHCYdQX$FEJ|^5
zYPztHLE-b^$UKkm@9*!+n`B(roPK_p@9Z$p3Am5{Yzki=*DGy)?*9GyOG`Xsca<DG
zn#tf{osr?8e(~bP<l}vJcNRb2ka!prF;mkXXzQ5g-vdnsg{_`zo*&oBCEC)*%rH~<
zT#@sv%dfxg+qciPOT<vBcYEI5U(e^)8<p4!Pu#ux_sPlXzrVk~zpwW9v$M19>;J{P
zaABChxp1Lk?$4h;Ev>EPt;^or*qB`W?99fE8wITbS7qMY^YiWP?cJi<&mKN}`1rA~
zyu5yi62pm*3l}DYfkp#1r=4Bs+<tD3W%2oWwtanlPU>E@UQ3I9e0UhXK8}~4zr3U*
zW^2~eiy0zTA`B@)JNS54zJGgr`(pS0d%Mfm_xJa2-(K!!zx?myy!-oXb8c)na^y(m
z=Vzg-LOfqsFid#(P(mtba@_j)w$)`5m0Xtuy?lFnd-b<BKc`<Y7M!XP_~=Nda5Fpq
zyE{9JC#(6+v#ZqtIqt@V3lp+H6XIJkE?!>lKi{UZ=-HW>_0JE9`p!1nTl#w1?Ag&T
zFD-57la&$|R|n}>apA%QE+%GX(5j-MCnt6mKYw>?>*|1oPEP7VlbjYxSe3lkntfeK
zS(#g0Z%@t7qL?{s4fh*fU-;m@Xwf3KUa771|Nl)?b}xH-YierbLgCKSbFItU_~qrq
z#X;v8XoargkY#5u5|VK~oPX(MiE;WlouDNhK`ZwZKlcMq4F5lW_Uzjm8<S6#pE1q4
z^5Wy;<4)>K4Lr=ey!(^i-`nfjC1P3r?#|20%a{Al&&$lzEKv%~zOv%t+1cjaQ?;JH
zd}&$n;lYg?5etPG9oQP$+TuaG>Q7D8?w2%9JJ`f(UG_#JV#9(9oknb}PKFs57J$Y@
zs=uGRcoB58fXs1DSuO?_-s=k=rthu%oc83zL{P|>W`#UIH@DF5_x8JctIL0UcxY&7
zxNqOStE<Dst@s!^MEVv!JiaCB?Uu~TU834)d3klu&&lTJ=U@3eUn6eMkBQ3eZvAq1
zSBI}><CU_h`r`4zg+aj$+6q{-E&KYqW4+R^OLp~2o15j|i&+@(=Uc!-bAFw>m0zA+
ztzLEL=;m29|K1w>dG&AW?!W8j?bSQ^e$wSXrzQP%ls_-eond@UKmV^+*{6PQ+pPKg
z`hkYqs@msVyuLjrYI{a~$=6q|Pxi}pZwW8cxmokC?84FRhdz4h?A%Iv4=((A>wYe_
z+B5H{%#xUx0~5bmH-)Co$+6$(^42^tqF~-9zia#dZ|m#pyLa#2`uP3tuB`>#Ens77
zJJsvy`+7l!2`?W=NbP%lZEduucGw(~%uAD0y|?Awo@QVF&tAr0dG%p2)<@y%zRcb6
zxvcAKp!DwE*qucaQ@@HWE&dbly75}&rq}-VRUePKtz49{z5Y|Q=GjjoCr{ts{W0SE
z?#?ac`gbl*I<z@%?vkV4)8>SSYD_l%85?`rdVS2_6>fWtxN>jYoBH}0Z@k2egdcx)
z-I#E7cQW_w)pg%%?I)am%FM>Iqw;fFP0gOgiytp??Jg*kVz6^`K5V|H?(Z(|>3UyZ
zUCq9_>gv&MaqX}*GoqBAYlJj(G>Na@FZ%r<TgQ!t!?P>;RHmM`Nd38Gc9^WV_;SC0
zoA=h1DNPOWx@GaT{a4cF?XT{iz8ZH*E2k*!{LG{ZJv}|Y-zy(m^mSNljhegZ>y}sk
z%KL5hZiwCQouw2c`fBfz_Kyemd_FHU$^PrJXE!$;t`-e6TvCv6>GgfKrB5~U_w((q
z{870#tTmKpDVw$RZqT{eHWdZu=h+_Z7PqhaQ?ZJNn?X`mN^0LX(1GKvTsLpr0F5Sl
zPuCOj%=$MWW9ur%<XSJu^LZ9!{T&@w*v0P5*^|IB@z-O$kISF)|4F|zYiY2xXug@Q
zMW4CA#EbhocX#g&m%RR{_K&kL6KpB??<azj;-?8zWjgH-^sYU=WBQAtN0EnmeJ9U}
zjjdJtv16KM@G_VA*%R56swy&E_ZM<6|Dyl?TD^0|i=&Z$%|Gn=>71U||3D-A=C8Ao
zCtG^OEtD2j?021ReO>dL;UqWzDUT|R-d^>s-evaD<F5Bd$bjqP<Nfcit-bx?;^N@a
z9EKAUE?oGqHU0d&shYuQmzH=wd-iOlz14c-@^zQC&VF@e%G~pokK*Uuo4>qrYLnP}
zJNddQ$NyjRxR>!u*wy~lnZ7rd?d?@Te;4KLF&tbLh8G@RYT&H<`RdBr6CN+Rxv#yl
z{rYcX=FP?a_wDZeS-UW}Z?{QCOqay+U(C%-=l&m_zyH5gWB<M@IY*wJYp&v#GCEa|
z^=#X8RyMyi2gM}!rbMiMqchPYe|9HtsrB^u&WAq>{<oUFwaM96+#v(oCSCJ;bN#Zo
zy;a8P=k`>8&x_*WW02&QlG>+O+CO{$lIzksr_5!~OP9&-`Saz_HRiRur-Vz|#d=(0
zlf56FvSD{uM@Q-EYZG_ZsfPHgtc*Eyang-9Uw4;0U3b6A`PxnH+Dewym6gX%?5&o2
zP;a^9#+=A~du>IyBt30TS=<&rnaj0NW$BLe?;S5be&sfo^GrW8BiefY%(&?eF1PxV
zEv{w6cGd6s{G@j^J73lJbwBo6Ke~6v=Fgt2)rY#SSRVcR|I(+~{0x#O?{6|v=wR_S
ziaT$(Q@dy7sySCyFT7Ix<!Fr1VP~bFiBme3EW3GQg0bSOvU6HWr{~=4j62oQaV13|
zXu{IkvPmy=|L?cqWrOZ2{<QVsj1CFi`Fm!HEKyfd>UBKrp>4h@T>t0QiRa^7>pH$n
zocVeG|7mY+HlEhZ-2HfO?#tU-#LeGk-o7mDKbz4Z>Tv$+3G0&L?Zb=NltSP3)plk$
zynm&ow9?1V?f-@hrKHc|nF%|}er3w|yZE1bGs$%I{Mi?UYk#DEdic_^EAHvr&XWG#
zqLob@DMiLglUC()U%ssOH>`anFQ@+bZ_;Ha7EBVF7~<s;BsuY%!?bUe6?fN2TEEZm
z{Cy$++a7Pjp1ux`)xF(4zYFGXi#}ZD5|SIz{3d=$*r^T=Jyj*I9s57JeY|xp^76lE
zR)%_O8E5BTmBpVAyg2xJz4yHISnFjI-S6gI70TXuTS_Va?7Z#sI;NeSF+Vp(etMMi
zw6>pH)*93#<ZpI9Sju;HO~x|Ly!ZmQ*$a)Up7!iG*kfV0Cgi}qNaNl%qwMWZvR@Ru
zIgt4-@l0cM=&ih2CF}04xRv_MFz2rB^K_RtGi&cw)c$;S`M-$2h1jC~ruE^?qPf%f
zvaVD+{gxH>xL5S;=f-nw4X5t(oSb^f=62z^<MTQ?mRQ=Vhki9#|Ky9MiFj$%_j$(G
zzg|E4?d|5=2dCug!ghIH4B6E!&G5iJ;hO)CTN9PtYieu1zrC%Vz~;9)Ec@i4N1vIG
zN0rW*CL*@#{~Z0+%d4(Nt!zD7|K`ip6H!}dh5WbGD2(b0-QY53nsTe7u)7zxtJJ(F
z-#&l7y(RN<BQyJ@ce)G;&Ib%lIs{h-)P`>Uzy0C!xNnPNMOMk5w>nh!B021!&_=Cp
z`8RwF6RZkk!M6sX9E~5-&c>Lz{!K)=X<PV&FlleD-FH+^PuGjJc-|lHE>e5H{^=%L
z)+*nuzz25=`rDS4#b?Fccw>L>#g`vl>|JZ;UqAHn_wPT;TNLL1d;V+v4|VH#&1?-{
znvydFCLG$F8oc@O>F{Hg<u5NEJZ!fn`*PIX>v8{QhNo!;1Qb47GwX-{>zLZ?^-q?r
z?(mqkbxQe4nf>;inLoDvJ$~l%ZP1*O?81eP5zErTCQY*39JM0H=GFHZA)%FT0}5mW
zj~BY!>hBJ!-I*jh(M-=z+w`x&$LGgO*V)zdU0!un{@dR@C88UTzWSOG{_gTcOL1}G
z$we-Hrv)ckz5A>bF~>?z?cL>?1-Dku{#KGZ`;>Po1H&J46O#^yPbIM_7gRs&(JSfg
z-@bZvws1PX|9Zji^J9HL7Jd7(sWEr?x=7>78&3r0&Y3agb<Ey98`fo>&aB>kzCich
z^PmSm{k_BQ^4l2hTN(X*Yk5aUKj)l#Tb^E$wRWqV@?zVckeHGOd+N@{-i`hJH9PFs
zYM+&z>{ng%{q?g<E05otYo{v~qvmycsSyLXH|Us=cBXgs^8RSM9b0n`Eqh&={_b6F
z{k?S4=<-LMj5Qfornb$^nfC5w)Rf8Zv+PPveN?_X@2h3xqf6`hw5pE3U$sX0-cG-}
zUCCmTr)Yeu+FSo#evhfK{%^fg+lx4?D<2$@dcJK+l+l{0O<T5RTbxzZ4E*ygZ|<XN
z`58fn8*hcbir#kh&mZ>P+g9xPmGkf3<+L+A)-NvBoV|Q7G@n1>>5c~(lTWP`N}JcC
z?Bx~5Yjt+oDF%igYBJ8v7lbN~T~UkP_NTO#FJ{-~?RkmP#l~&Ap+@^_`yL$?dH!>X
zc9`h?-TBck)s}|3$cL>_N)na7wzI3Y#64)o5?ODTYs}2dR^nT|HkM31bnDEd4h_XO
z5gq$n<g}HPHvb9rI?1H`c=L_)i6TYk)+SA<<eHwero-p1`_<d`msf57slNJqe$vD}
zA#Ta$a~`oZl(Z#h2+XS%-2Uk3{vD@h^{XygAHL>M^1QsR2T8J~nq`0P%$91t_9J4w
zYLK8{q2a_=Q>tqsB%M#qF!VXHn?=c3X=P}h=1U$nrN@``pY7AVR`cZDoTD{Wy=!-V
zJNWSOR<nB=cBP8bPBAom0a-LH#7$)OgiR&(df~1vX^}zeBCgioc-d6y&3gW}ki=o(
zqpmiOoa9<BA1_^}^Y-(HU#gqU<NGeIV&2vtTl09X;?YZ?YDLdHLu@oZhpttM-~7IB
zT8GBFgL%GRAMqhy2Y2d6@8a#L`uEG;AAPzf@$=-_%jbpXhDvy+GC1ryY-qyKv3q0k
z!DHFG+^?(aY>@k&pKLzugg|BX;bXyf9?vLkkJ)uq``gv7P`#Y5^?jzf!55bnD0*__
z>l8mdV^>=9$tqm6t?ZVK=7PI3-mx#6n0{ZiJNoIxnwVW#N4@%!-PdR3@3|RS9oc;{
zI=&?DX5OLApFGVU9b5PPq_koA3!!%lZ(m&PdP6%~9()YL^}xL)4|pndZ$B{7Ro@q7
zwe#%*K8vHg3@`b!qYqv`vSHg*@qhV8wbn-YdQEMzNcz9tYHO6KXQ=%3dB5&m3ca<%
z;&8p*l&_hbI%}=gUf=Xf>e@D@h7^l7wu`^_i>|iQ^Z2^<R)|K({TrYEfBSV_R4aJj
za;EG^#s&w}!fx{&1-Fae{qw8}{TBvgY>iTK{ic-n^z_tI3x6HFnxOaW;yc;s{=Efp
z25Vn#KcC0UkRoHz#%6dbXp5KS=IU-S`Og^_cRF7#x_JBF%BQAE5r3O^OGVC>xoP@6
z&*s`ve-DNRlh!u2!lbij&vcy3OwLT4e0u5c7o0ZbKVD6~!9V}Xs>hG6XqH_&cH~H*
zi;LFpH(@<X=j@ANYqlu$=_)nocs%L&k1W%^DeD;+1Y|(7hpSRLUOfD&?K0)^wVn<+
zxu}czcMfU(J-z$ueSw35^0jx5SKr!snmxArJ7>t}BR8bh^}M~XB~x;}k(~0kBLc4X
zZ(Q2)^vL1g({_r#FV;1Bbz{@SbL-am?k+3UfBf{_wUqyVGae~zJ1=2a`|^@s>K(PW
z+(JLN*%*FsTwmz8Mnc#~F5aZF%_!mZ!sAbmueA$MZl3~a%<-x}f7+u}vEu3v-|(z$
zUN?~LI4f3pK3O7jvt;z~=hekm*52#-aItiL{tWHxy}N6>ZG!esDvvvDWwtUt<Kn@#
z|L5N9X}fE;=h=;o?K0NwXD(F9%Er9R>|1{KSWjk5azA&P05gLis7@1|e`G^x^c8(c
z$wQjA%rm3Z>;;e7?sj+SUmw5YqlZ=Uu{*bZo++}pe&Owl_&qnba9m%esw<(ibxFWP
z87{Z1`RhfJU6*8Rc<?Ca_=HN~U;ca-XL?F5UD`YK$l{)2F{Rg4vh0P5%g@PJ#T-D{
zaaYC0P<X)5q^37<#&rF7H8r(avu1J4Zh0=edDqn$d)LG-jr)82$!)W(t(%X*j)Jkd
z@9|`Fcud~;%k0;4W?osH-h4KgFRt)c70<h4S@YlZYsbtmGycxK-r`rm`k4nEmME<3
zoRe7eX_iC#o_VIPpDHRX`Vnp@#Vwust9Iglbqk9bi65ew7#>JS{qt|-67`y<^YYoV
zZK<cl7}Z;+p0wW*GVRmd@V~E16P#Xe4Zk083;jqJl{=PV3c98VXV+$iPW@MYzb3BK
zdDb8MMC<2sUq#*bUfRpCGjD;4uHeLvZl7|K`Om+ZvVC>z+N_nri`nOJbUxc6c>8zn
zzU4pt7Hkn061ub{^Rn-3GhRs}mkk083Q`#vKW2F?y|pRz^t#yHpgXU27Cl`ZzJ6JK
zn>izUjrly4#s3#s-d48w^u{)|BJ`-t`z-Evw{&H%WE^k#_U_!f+&s(vtmkV2Ua?)3
z-<N#<mhi)`*1J}g=hz&zwJH1Z?C|I4>v86B_cuIEUHi>-+V?g0a$a2e;AFBpc&qS3
zDdV=6ckia(?D;3FAA0}({-)&1ivOoo8E(i_4gW6nKALasMCS(P7v=Bo@k$ykaqE>b
z&AI}*_o|gk^pZU@LkDlzxrX<TD(gN!JFD#0qY=F9%oNSwBGAQWi}>wyvp*I#o9X$j
z-n8MzxfZoKVYvY-L|me_hAn&~U!rN7e4%ye&(7dlxyCE6EGDfAdAZYtO;Avhr){cs
z__-N|iz7FuO;+>Wv~{cRa|Q;5D2ukX_em1VYJY#@m$xe^EBp58(=3C;rl__5KAyfL
zw>n_yigcx*h??#5CYSr~%UK(s*#X;y^z|smlAy>X^|tc%b$d!)UUKi3vn_sh=F(E{
z!1D|fg!&dfT;9^&&M$BG=gdsw<l}vyQ!m~2PVMZzVORaMI7lEe<mH8>TLjd%Ot!7~
zaNzjy=SRE6i=UlY8NYvDfFXm!nF|*_aIcHqeQkez{aiou<YPSg`uYnWr+63|8ctUC
zzqhyg`-zFlzrVaxo_w;!kA*?0b@QDM++Nf5YJYxus_5KS0$S45&Zi%@N5Cr3Ey+Ma
zOg}DWQ;O%bRC${UgMtSRSBx3{u$!3t*<_Y`%ckbXg{xP$*8KeR_*k#EhlhYwAX~wc
z6BC1$dd;<|ymbG5{l`Z~BR8dh20uGw&lMehU;Xvf)c*eSCnu{P?~%N`5Ohe`{tvaz
z1qB=W`~5RAUfkJP{QBBj?GhP=69ymrwf^jru`J5DvEku?M&{XOxvK&e9%|(lZ}C&n
z%fG)*)*!)Qu2t!)>+9u3L|Upqx3RRf**|^wu<+v}S9$sQ@9ys2o_+n@&(F`D)P?57
zt&iW3(71c|?x2-5A09aV|Mz!k(3uU~4Ex!5c~>qDT^$CB$nx^_0Sgb!Fid{*_;IVB
z$_(drzMZ>w&$ljr_vhzlZZVw=pdn5{h7^yC3=dPUrA1#}Ty%7Fym;|qUS3}I^>wCF
zy-w|3;f{`uUteFBH_yBC=x8^;yxp58U+TFl85sVUn3yct^ycQ~_0ikkU0CRRdt0t{
z__{kgi=VgnO_5u)Xwk24Z=?6u{axle8?<mSY;BYiXh@Hrmp5}!%E?L3&(D`PPV;$x
zZ}06bnV;X>G+rpYRCKPNdHy|{^78WH;?EN%Ot7o{^<a`Q!xAnNlO+p51GW_(4qU#x
zIcTZZVz=I|?rveLKrQQv4-4Ack1zF}t{1z@B>$d`wzf8C81PwRTiceU0UB;S5{2*Y
z?cK4%!rXlO)z#tYechKE8yibsUz=-^dFj$p@4tWkfDfZrVmOg=;lc#DO=)L$RegQ6
z*u5V#1e$Ym)4zZJoYa5SPMkQg_Scup8ygZ&Pt#>)V#>O*LJ>4{I_JWL33Yew+&MST
z);jS}i(T!nEjc$oot&)hr2fk^YHiuMIhJyAa$#X%ii(Q=|9s{@z3h9_RE7`V9h{x3
zvTtq4EGQ^w<q};Lvhva6$HJhUO^fDl%D-<{UjAL$JTGQnP35~gJEu*X1`0eMzRyL?
z2};V!?)`Fm>;M0Ic({E!=<rsH`=a9F=GEWc+_-TAG<du=D)fZ~!-wq-&dmqj-P>!O
zcV|aON5>VZfVE-Aj~#n<Vxsa^wv@<iph5NB<;O26Fie<t;lhNQKYsqyjox<V{CWQu
z3JeVOEXP@BbT1iX3P%ufF)T_K56gvVJm)_<r89W2n&;hladC0`%$bojHGghz&;R`N
z^zyLP+1J)s=H4=a%=WCjz{n8P?zen*?eA~P`41mD)N$h0t*9L?AkU=lRGj?b(<e}w
zxAW+oJ2fXJC^|i5WccBnk#WMts_c!%)Vji`6;)qfIkmz?%irltKRx-s&P0!>Z8<+f
z6PX!O6hO*jVq)y(zA!U2eS2wX_l8OahJP$lQbwZr`T4C*jxJtHFP%Q^{lbEQVLv}F
z?@X_DK3R#jiC#-j)(cqiF&udB?CiX3l79TYnxCH}dD_m+wdS77%J887ft{2<P)f>^
zE1trZ&5EMOO#(GKGM-*iU1^~vcy&eS?!?E(R$8ku9H>7qCrM$++1cjbzrMR=z1~@?
zFY`&I>G7G%U&Y?pe($L3ty}l^l<SKgXJPn}Z_&oq7<yvj+TeF{oR<G+RC=~*sf>0B
z7sHR;53I<&nl91LP(U;C@4iUsUEU`$IC!5=eYGO+@_x%o^XSd_-_+h21-m}_nVKIq
z*Z2Jt28K+w;};aXzVU9}{^8P3{ra25O|v$Krv%qk|Ek!WWqE6Ar}XWoXQ!sy%#13@
zZj{epIAQXEU#er`k`>C{DLu8$f4XmECSU3NrdGxyD448PS~&IA%=#Pp>3-8JHW~TH
zSQ$Bn-Cnlmcl*xGp=mBIWtB^JZWa<W{C4S3WpLm5CuLJ=cd{}#<P=PkK60YM_`-#y
zODa!m=B$&Qc>0^0?;K6fCnuNo9zIZ2<HGX#fkR^Wq37&3zpXnm;m)f2Jd>{nb{+c0
z6Zq}s<16nsM~G~?xVLX&+UCgu(<Q!y#Hs9`lG>a2JiagLkny)`lhdOj_FN7&+xgAy
z#RcJO`wpL38+rC=a@F#vv}K9;;yzv+UBbVl85kOpGX$3ZSiJt)JX3+0({g8~x2~_e
zCXu`-`C23I&)M3C-Fa&*%fu}!(y~wPliYmYW0%DI`g=Z-(zX6)rRCaqXA127xM^a9
z-Ie#Q&;NZaT;o}n>~j6v!r$|ZEI)g9o><qkwApEI>WkGU-6eJlE#q^{XV<+@q*7iK
zzUh4YSBb7FHin4~&dnZ$|4utu|8`+_G3d&PZ1XeJe6l9R^Sez?$BU<xz0;P2g{3Gd
zD;;Z7RB}5N^ohS@QeOwp`FR5JD?2ZJc=(;yWyy7c<mG2n7oNDQf9%IATXn0R^`|C-
z4s+bv!Fyfh+P0btj&l>@^%xFZzi>gp?U`~~l-1@V`#WY^zc}0L(Ux=eZf=Fr%C)wF
zf~rzNol8FbuYdSl`cw*+Qc{P<shW?Go3kv{ctc+PJ^b%(nu^)a;!Sh5xu|^3N?w%2
z!QdbRGIH1UHDC8DPs;y&>3BwmMeYMfUnR{udsSRqeono%gm<UE=MjOAhBktRK}t%U
za~Ioae{|XN&sH^jlcT@OoW72Z{)bCVwN816@BhDK&Ccm99UHeSw~^f(HhI6)Zb$!Z
z-;{T8e|*ZsP{aIAgxT=XJvD=6M~n1gR&*!(%B(4MY=6`g_jzA*@vki>P4lPC+dA3C
zLZr|k%X}ZBbJ1%~;mo^9XPX0-H6)+AS$oC5a+*iv#>HOiw&^WM*>TG(*Eixi6VJ)D
z|1!n*J=VLvW5e4gXMe3IcPW~!D6Csq?D?bcZMX8Q9erXuS6TF2X!niQ{$dWgK62jw
zZ_UdXUM75gcGmh>&zw+$^VjDG-1NRO(JeGCvd;VQ>)kO|p3e{ROjLh%_wbZxyLzqi
zxT>FbejF|jy}5arXH1Aq^7QG8=KVht-;n+Op8nGt-P`m2AKAyiu#y>+H(z-u3$C31
zntA`f9>XUGzsIH2{QRTN{``uIOU;$@Z}}M%SYpV&%<_yT8&ma%cYm*zPXm=d((fOc
zaDndh0<Ym#tgNVrcz$yGof)&97YPXoCqI5%oOX6rS8~>rwA@?Ue2+3RGSqQsr#qxQ
zeh#`E!N}g;eql2M!yiHN9fYNRq34C1kcw|&$P0$dN4K_S|Ni#=dGDSJj-ZW#tH2YB
ztPBht;c7WA?(Qxxe|hcpHW&A>ETxiZ`Hy?sj%r(-n_;u9{CyngPAZ0pr*e4rCOlkZ
z_;|VK>kY@1(hC=E{xkiJYIOgs8*f}QZhToe8MMHkA!*9Wb<X^f&y4GCF<sAdca1ym
zk}^wme)jj|?Cr9-*B5plOaN6&3>|)InHhWI?(fPy?z`K6cd30^(q#7+7WS^|HWj~o
zRQ}D+<?JoqD_L7F6f!V4#27g{tE;GltPlPFXm7@IJ@(XPku80nxg&OFUfdQvf8|@7
z)1iG`T|!oT3=AC}ZTT<E^Y2a3i#80Iz1icn?;4*a+uQ!KyB!kC+vAZQzjcZGO?Cfy
zpc!I@BTC8UUlzOfdri|i{d7(Aiib9Chqz>)e}B5x?9`>j@tWs9U0)v$x?+r>!-*~L
z#p>|&(>|`scx<VXKQ+-Q-ILYQyZl^qdEC}nrTsFNN}$`e1mfIsGwkj6OKp{YvU<Np
zdv*1~%}b<qFDmWYa`V=z*y&eSU+0+1%D})fO;Nh)=O@uz>oWaYdArvJ^Df-H#Avn6
z((ZCwlawb*J_@UGg4DH4`Ix37X<GH>Qt0gNYWc96Vggp)vjT347QL;j`8eyDzVqR5
zz4$YmB!B#>vg+>ceq0&1HthA+UpH<<C>a?VPCWh8={Xa_4_Th`-Q}NNtuK(desuTq
zjhnZ3Z@aH&zFFML@0L-<WBHu-FD6~jdA}}KO-=H>+}AJLtG~Z{c6RpZr$zIgSI(R6
zp<*@n+{u%g78VgNG8h>4D<7Wo)X&FfP3ZQAFJ~{*i~Va<Yb!i4Br$Hw-9GlfWly!j
zx;r}st^C*?ynMMbM61+lZrplv2_CDtetOf*v#<HQh+$yZubFJ}<@WY`+tZ<SLfOGf
z&W44p6SDHNnz+H}c-m=Uol{eEo__uFvv}d-j@WZ^EW1TCClzs@-Xv>PBA~_3a6tYP
z1MlB2FE3A$OWnF{Dz|y0PNMO>*3b4+)<0tj{g@}0dg|`ZV)qw1N!K^0`%iuKS--%t
z_*qFwiHHF|!-4q%)z0_#RBoP<Uio>_|7UCMqF(4U6@Ga0Fre6N=_*tIc{Z9QVu9Hg
z7ad)m>M_MDb$j03mb0LxHZFHB9Pg7|9kti*omH{ysS;7E?gN{w%u8~1F3jvMlQz#4
zvg)q5dE>?u&ERL3+uz>WDr~^Ra6n!t-+6uX{(Y{q{rqyxZi}zgE)fgdaPIljOHUo{
zJ1$=4H(Rkp?BlDOo0sq0X(?!FX=!O;0ZN_^zIT4fT-`Z&*$=&_$Sr^MjDH_;N*5H*
zeY1K&@7XAmHQOp4+H|_?*`L7ax;E_gty@A9*RRiCAGa5@H08r@m+KeY7H_<6U;ACo
zGJefpy;fJVg^xQXItokg**e#K;u`Dixj`BtPR9l3zPh@4zE!D~mDR0d$J}1%Ffi=*
zkKtW=F=N)Y4B-z?_6EQ94Qkzaq43D*j0>%AUd{}aEB$kV5!CN_{Ac1^>+(K1TPr0c
zrLAl}brQF(Trt_oHpOeHtW}AGechg|TT8QgIT=1UD`mZyF=K|*R_Q%YWx32Fl@e2(
zjtg4Pdp=)I_UL}A;%{Fz@>CuC`t0m%&`lKG-LuVdqdJ9DTlzl7e*E~+IQ^VW!Gi-c
z43n$Aztf%iueARUV}rtxOBeLx_pJ%pDmHty+rOyE5^aK3er<wPiHTp%K3(1?UshHo
zW_4P>qod=)$B&kEe}2p`Otva}Gec2t#r@so`9D591f3}T_3PTu)zdB%GAMi&KDxh$
zk&*G;(!D-irFsF?Mz4NOf1-8YJo@R!r`n;`)!)`!h^*^!TRhVwbJFI`o9E8GJJ-70
zs{Gv?#aA;sx<s{~J%4Uq_~?jT?XN2<gP$L4W(Rko{C?ES_|LbC-RRXj=j!rjpI+&f
zOykj<o&DMAY?x`rouuP^vQFtg8*^`Ov$V3hx3Bj1pP!$F)%|?xbhMIJMsI)DD{T(i
zak9{<bzkl8q*Emf9@b3rpWfS(>ixc+-D+D*_T>G4^7q`a+rA|}QnGLf=XL*mIX||{
z-I(ScK6$p0&l~QyU$XCnuV3|KUB&~Q+G^v>stF%2YgcU#+_~razv4-+Lu22~3NA{i
ze$@Em{+~r1CsuD+<Z0I#=5KR+zQsDr=`O8vQetD51U=&am3dAzFfu*zveKJRb7viw
zkAD;xcGHmi`g#BV)$b+b1uOD{Uw>TO&LeZ-|M%kMGhbg{&o5!1U}#wQ?99xkr=|*9
z`Du0Z^?~+o+}m62-Y0W&xxYMUVNZzq6T>e(GdKT#`~LBr%Kz`L8VkDaGx4!sy#4ec
zF&(}2GalXB^z7;s)=eACtPcObZY(&L+i=39z`BjS{T(KIs`6F#8vd}IxnY0(vb#qk
zqwdbpi@j|7M1tM=+t=K;J5`hxx%2bP;ay$1%6|HhEo)cighnN%`^`GPK6#bM#4P3B
z`_kXq&urekqtx1}^4q3=%<Uad?lq;XkK6m|!a`@zQTk<XDjptcJrj7|Ws0h|SbY3_
z(6InkB`0p){CQ!aGboDBxXoyLR=LlAf8%Dcb%k%s&e%j>w3qafJultap;Bsd<B4t$
zXS5-=v&$Sk*T9n!>D9-~e%^iDdr?NxBFRUhy<c1Zz4g+WH8YhLrN7^|!qYl?SwdIw
zMlE5bMKRZWPO)y|iZU)UQc~J1bII<h$hIJ*N8$39GdsJc|M<|&oP6$<;P$s_?nm-1
z*UhZ*^xt%TKfA%THH&-f<-hdaSFf9Q$!dD`_QlqJ^tR>Pja&c!Z1TsDDO0AHBpzyc
zbhP{ViHVQDy}d1KRdS-nTv9|cDCPgZzroA>WKA*x)<$i8cYi;4^_dNeh3WFES<zeH
zR_JQn*?QY><)u&l?ZHO*hrZ64y2in<^5kjH?HyW`Cx2dV`+jqYn!MKRJzm<o#nR)C
zf8Z;dbliNB+I^Er6S=0DMn+FmT(si&EZ%&HeE(L@DeIos-F-eWb&{a()Nd}&!Y2Jl
zzhzkd<6^1E%_%;M+RQ_ZMA9Yph8})0op+sr@e7G)?p<|J$vP_o1cG;Wu6%iNiK#yC
zbfw<VO(&&hz2-X-+`Gx>-OPvgmS2gl6MT2hXyvT}t&jfuC5?n%C(Zp9fA{WMudi0_
zf4*^*E}Z@xH2BaXVYsN|<t1^wm=klYw<{Q4@_l=2>+10J`;v}!otmm`o`0|A(h^Vb
zAe;z8O2x}s9$9JW=#^P(Z=e5q)aspYYn7Uk5^MS$@fGURa$jgEPP(FhLT2xp|2NJZ
zx#3+sWx~8WYvuN;7hcSDcH~ce{UlTU5%1jT>N5Q1w?spBxJu=@OGo~C^rYZv7ngfy
z;p2x(K0kDOUGpi*vyE9%X5z;ea$h%Ww%U|%dWP}bY>i1yC7=B><Aj4(HNAPFYb&_F
zHu=%4={ACIcdk@VS~Y!|py#S*ljrRIv-fe-GA~6lr6UXcdf&5Y2U~5u^WoHt{ZrCg
zL*}qOc@DbzH0S1~efvPiu-x5{=)7$G^OT^KU*^~U+gbWr?EGgxKfmJV=R^-@@-UpZ
zaB1tp!-o&so|@91)NQpbLVMz}dnHqg)C?T{&(c`my=T(uGd5oJ7N0(!Y>fG)Be>pN
zK=3ZlmF&*KQ<@XKUKCE4*RgHhrL0cPg!3--f6r?MA8Is}II()m9jlWCkN4>*%#+*y
zxbt!T(cmlGe%>x#oy-%By0^^iHhHt~omYe7X(7Rrc5({nx3bNRK0nF7FrM$<;@~^+
zJQ@>4R@u)kGB479`t`%Z!&jufwI4ck=+?H}+iN0?^Yh<tPCq|QH`?s(y(pEOoSbK8
zW)}bX@iF;$pQv`&8_+T*&={f4U#VaJ{{DWqbng<@Xwfz1wx#Rcy*fL3+~tdwOe*_0
zCC*}Ty3f9Q?*BLbvx`lcwrT!$iG8KJ;!jS}n-*Q-W&M6i-o?u1wYG~@l)ioD>zShX
zX#KzSdw=s~Jj#2_@GnMBkyBcPMN%tRP{gJ1N^7*E%!6;G*Fy`h1b$x;6&}sr7nL4c
z+Q2NXSJ3=`!F9sJ1nv{({NBC$%N%kitkL_y&p)5d9=<F07mDBa=bhgE=d*GSi~fAJ
zy6bIbN$uM2#}b;}X)&CrTU&AR`;_^!ly+Nx-eGig>iZp~e-^(gofEnG?42<C5)OX;
z^$!->D@^nd(~rBeCG&FC*H^X`9};eCNWA)f(&y{NB_%e6kB;2hn*IFD%*#u?r{A~{
zu}#~FBXfzC*9&R$ye*l}GjG*REZx0Q)Y?cnsl?aAyCJ(=M3o_6g;g9w4;MqjA_j&_
zMH-?4YiCblaF|iha^Q~t!bX-!EDQ!46<QtQ`<AMT6tJ=|h^$~>m}2xSkZ<i>hu6#9
zmaLm#oWEq-?gj=shNCAK7&wF+{&eXvEIQ7RSZ%vu)5F}4&zeOT8jh>Zc|GlE@GIe~
z!Kw_du|4(A*-RSG-Ck((_T-%(U6x1x&fE3QIj3e@+54-{udEE7YGA{_aM<AYx3|Rw
z1r?v3cve*WSmfGWQc|*SPc-j^wb9$H%in!@b#-;{azD`NJZI0Q?pA7<(55(DFE)y`
zy|6g{=7Mqu+39wo)@#;j?%dJ8Ky3cwG%m%DTKia6)-2gE&Hs#)!WIDmo7hc?j`w|*
z8HHZB8T-N6cGG-g^IvUW9MnuUJTohOvNO7<+FWgY?1W{JAK6TwaF$r`W@bEFl&Ynm
zbEfmQ`BX8!J#J=u;(RMFS=^i!b539O=dGASb6EPt{_MKd;;Q@1!{+K5x$}2jS5MYe
zSMiXV#GHG#BCmI=kN1k#kxRq3N_OhK)Cqs<Q#bp9-nl)T@1mIW9wwbSd9Hb%?}-Zr
zm3tlqum7|=<N7wk75D$0zZv)W**R6uPp7T_h(CPzaNoXt6DLlL-kvwtD79<fzPh7*
z>t9;P_<eqM*1b=r^7ps5o|DxCSuQSe?dJL~BH(m2RCH~`qQ&Vqiqd%ePb6h496HCr
z#XTcD-%Cj78Pf@^1Uvi6tyh#CjxO=G-^<#+;S%f1*%K>URKIV&U4P(IhlloxJN}9}
zg>w%z?cvJ#K4sP{tMqeoqPOSi#_lTl@Zg~7&pF@1y}Z0Yd;OD=9=+fH|K9fe_-&^h
zIWif(h5ucn<x{k?f{kJ06wiM_Hp~uHC#OFOJGo@4eB{sjk0cqitmkP@i_nSNQLu3K
z?AdGAzHMgbZ{v|{`uNB6&D*!k%*@x<$L~)#*yK0Y%C_dmgtuJ+POsKIQ!)7!$8-42
z-QCv@9#pinv{X_`I@rW2YN5Ga(W?BNjG*Af<^J>6#qR$1<8i;GrDaZdi$GBPj0-PT
z-`Q2V+O=Ek_O@K>k{1`=-QBILqjO`ONAN_IpO5?P-Q3-|Tb<O^mxK5JPA%qVa`HGF
zy8ZQqT$>+%Ull$PnV`aWAZ5As`8%Pi^+zuW?s9wL;;&nsn|7D6^7<t`spt0FXLnoI
zrxkxXP<Qfaehf#?3_F+Ed$j*5zr20@(Z8zu#j;kas;S@J+!PcPJUvZU+9cz`_bC$#
z^?w{V`<&^*+b{bk>K)79|6p3-^N<7=u@f_{{g-|l`2FtJYb%<h9o_q7PtMws@2NcZ
z`tr+9_3JX<T~>Ng7T&>WRsZyw_jBLXzv_?6KKazK`1bmz*^(!oK3n_e*On~|3zE*g
z+dV7BXU6t91{*}@YxQwj*7#nn%bMS|zFcsA<hBN>jf?!5@8;X<I209q`t$Si<HwJ$
zuZ_OGzyAMwNiM~ZzaiT#zr6MM$;_IVd8l)@GH+M7rJ10PjB?J=-=F#xG3}b*{DD=W
z`m*ceWDey^=i3+>4l!A2Pp|s8_H~Jn+mg+8Dtp&-Rs2g+nwatL&(E{7Oto_*f6o2L
z!yxv~jycD>>`lb~*BjZN&O3M4KJR0n>}=j$JQeoy7#mDhrLW`h^{RaCski^;j+16>
zOn0k#Um5qmSF8Ul6Y}rFvt-K;(Uo7P<lWr$wast$w58A4PAtovTmHuD)vK$M^!@EO
zCckfsXOio@XXf9$=t!)iE8nvFzxNthTkl@%-k*1COXkN%M`f+cdiwhQonsVqy6Rc#
zoxz|H#K0gTYEYg$^|kU8h9~@AEe|j=7+lnTUGXu_p67vJnN!8rU)wgC$C`LWzM5)!
zj6oso7sqU=LI#DmGmGvO{+#V<>idPmdU27t&s4KZ7nF7G|0{H`n7!}q_KUkp6yJ5l
zEnTw6fM;{{_dIs?_9;`QoH%i!^7Av!R{1qdpM785y3OS9XIlL`@#%B-Xm(kaH!N(p
zyfO9g_Uy>hM~@yonmWII<x0y2zUkbbk_vAwcbdgN%WUIH&FN31qi0OFzwak1E`Iuc
zO^FCA!;}aez5Ow>WB#N)xpJW)F*v`Q(cyUArE4F*Oy)J8*X>n5rS^T)>v%P@ONtHo
zH}fC)@3r01b*gt_;rlz>>mB*5M2oxUEMK1f^82-_FE0$s-rRWe<_(vmnBtO`t8#;*
zm+~+ySSQHNpfj8ETiK&2WqUmOCIs?sz4zttjKaW|mqN0mwDXR6uSh?}D4F~C%<Qkb
zy4&(%-(8ZcNtMgZaeH?Bq~Gct=a?AG4>~nErLB7!!|f+8#6DGE@}HJ<Ov_j-wz8i2
zsL=V2|4^P-N}=J}_2(HA*3A)<4=6OR%wD?W!nv}PmwNU$CP!}UIu)_>$?vC6o_u+6
z(OE%2#<J+h-Mf2}*r%T^t&6kyer$dAx7dn1ISs}Q78RHGiq3wfJE!jFS2fjz(&F-G
zRy}7h=U3Y)f7AFn7sHFyfA7RHFj&Z%&yUS5d4J(jD`PF+U5Tql1QxuB6<K=sUvlh*
z1=C9E13vx=W^~(rtv{;vNOi;anaSx5mkYl?<&|4@r)6_}{?3W|XY7g`>yN#R-1d6e
z+hmEVZ&80mjoD{z|MpG!U);9;%f3I7Q<*a7ri@+I^E1~~PflTYDmHQYeanKHGe<Z@
zf9y@0q;kAZ);i%p!`0Q{)|Qs6EG#^-R$CsmxpQPL$+Bo?OJG>gt6o2E{x=QxO%~1@
z?P3-(X?RV^->-Vw;Q7?|?GMw>&AsOJ`c~k{Nx_pQOPYLm5cy`do>WV{exr$-)%z(X
z7E{f(KUZrzEv(_OPc^pCyYk^qE`})#xwkl@t|d*ApK#7fVW0Z5FYncNlz3~hpYWg3
ze@oAk{m`|R%LhA`t1qqFuVnpY&ZEM3{Ud*C_o`{1-&_6NLgw7l)6=zW<}orzw9ec7
zd`6*Fc$i$ZBjaSLs#WvmFTZ<xsh^RPx$p1erI%m5+`q<d)48WdCd~KOXkcvk*`9ao
zSj?=8f4+T5O_%!e=IXo$dwxG(`eVoSl*f&am;dmWJbh>W-cO6X-|y3H$arC^Z1(L~
z$}_h+-hz|MZ<ojYSTSGyuT6I2t5dfQuiR!__twiSYDeigg<_$G==u}u?Yfe0D}Rzy
zdz{tqal^A?^JgfDAAP(3_7d;QUGL+Tu3!KEcK-gg5gQ+Ucz9UU&cey#ameWwHU=lX
zrIRO#)XR$c+b;GvovM*=C^_We#wm$&%YP*<xvsR+p;eq=Pwej5nr|05a#()#RMVQr
z*0A}x>f*A~af}7=`Y&Qt^lUx7cU)Hvm0VTAz`#+zkV$Sis{zBAzZF~!g24<TcdC~$
z)!w^Tcx&2u(RS`Q{Ud+x?p4$7-?*`GY46&QRbOs@i$ArZ$*<jh_s{$f4=(1GhUDw1
zoKDuCmt%4{mYt!Y<gHCpSC>aHQ-RekODTrTceYhLqNVoIJ!jtDpL_6B(BX_#4_wb&
z{cz-1{hDa|m6r3*Gy8p>c5WF%)Uj_W1q*-tOlI)t+f`}axjXIbd|Rm(TB1q}6ZXp;
z-uU`~n9rw;r_0~7%$KuX`FDBL?v(r1-xpn0V6RK9<(9DyS8DMJJ-zu33&ZCR^NfDn
z<#;lYA^&&clzYp4zkGPWEKph*@UJfa-R=L1QZocU^e{3AEAxGxxYSi*WzWM|lcp{3
zVfQ$@l+mKAs9jrjm5R4$L9672xC{FxZxi&e{FbtA*StsTa#TJ@sJcdMkG|r$p=J^P
z_N#}Y!(>947+x@lGwisPmXZ{)sA9%uzJ|tDO9w4}1_vH320MuvvzXm8KO|PH3URnm
z`Th0nHD7KO)P?PvqggBUBjDG~Qz1o7w>S1r4}W>j^1N~JDX%Ku>pz#zkLv$b`bG5J
zhVIE(^^ZhP<jY#vl^lwQ*(IzPa&y%~nO!AXG5aj_gRk*0Gu(UMesRKKh61ad6Bz=c
z85mkr`Mq{$7^wO%Hbh%H$_F1>y6_s8{@P2wdpzwDla4U1$~m(m+~mWFX>JS*xl+Xo
zXYA6cif-d-P&v4ibG@kV*}J-36C2kupPzW#B_w9O=h>hypIT=GF~0aB@sOY4vU=%7
z#s{CaH0+x-kDXzO@8Wx_PFzSo%EMs8%l+hMRkW~I!zTMOA*cTj{ByFo7I-Y%x}%2S
zLFP*)hVbQO!WCv!uhx}p^qNwY`)bZ+Em=K=f}IQpo-%5LBqT)j2d;@O)`{Z#)MC6p
zf5%*Z4F-lMpB~M5wCVSj*xEKmhv%$&&)#1db%2>cSC64VBk%uRHQPC_L;gj#3aom$
z>bA_Sr7Oeiv-udlywlgP(CZFh`e3p77xxc`im&2J<fqTwtCnx-aAVQl4X)K5)f%y)
zj=ZLiN{WTw<uW-iX}pZP%krz#;=tpxxn8+@mEteVyuW@vA4Bbz2`aK3;i^8TpB<W7
z7Gl=U>`?N0k?G?-XHGiYs9sPuIcCF!<bPIL@)ulBurG2Cx0}lM)$_sb&${m}>F(ls
zZo7PK<(Fj3i!q+9D_+OmnX2_W?!%#(|6K~>uH4+b!vAyJV=M1nyRU~mz4O2Jusy?v
z>$7^(yqE1LnZH%H_T7cD{byz@sW#j9{N~oTrKukezrDsYb8p%r*K4s!_l{cXCi$(2
z_Ff(J&hpBE<DZ-gG^S3Fueo3RqVehVcHI{ji<lk1S(POFUi)}|@rxC2*`3wR1!Fxz
z?%kKT@%z2v?YeiynKwTyKJAeG;Jvw+_39+6pzN@cjGupAeBHT>f#JZ`jIDcuwfFMz
z-WT(I+Quf!-yP)ka-L1(?(*PgcZ&P#{6tH{&7C-E-Abc3KGEKLo!{BrL*k(N^~>9?
zC|~lP;cr`G+_aNnlKu8WW{K%6k*qx~9ygdK|B=7`?w{ZP%%hV&KR>@de*eEGCnryh
zx$-=qXk!H5uV4B53v|TAPdBb#|9R!=!};ZpU%jqRTm0Q^TKCZfpMN?wP6}3;&$8f&
z$cNBPNdhlEYkySvKGQgzPsTz(Mkc1R-GL*s<&|0cKd<+$Ya)K_6q9<N`zO(6df$ho
z7g8s2GcY)uQmId!YASK^dZm7veZ`~KWxp?}{aODa&3D>U!;|Wa9<n(bI|WurUHxEX
z;>4jC!kgfv+#=xgibGgRu|*)r-Smr9iRSdSJsWekv@ra%G+BD|jqSP@7kp3Oog`f3
zBNU_je(SoTPaTV^-#;;~nQ*$hAwTKvnzilihr_>aJ=48=*W>k%w+T<@-=M4Uw)pAu
z4>QezKV4Y*wpsfB=0u0X{`sFH7v0!-d7c07>P?n){8EY`p;_-_W(iE1Z2ckr=n1tq
z(`*(UxFnI{aQu0rS^tl&u!t2-AEJ5RtgkNo-Iw(IV(`D~cT8%1{=c(-*ZFIfZT8Ju
z9}lnVY^=Qe{8e)F)NSjS81m+dB}ZSm`um1iysGl2E3CHC!9D3M0ivsRa^@!aaWO6t
zVr#H0i4)U)Y;w8ngT+!A2dBN)r*W<Eu`GL9W&M&t!0AxX`?u4RI3MobyY|BMOV>=F
zm~39LczyDnJJo+a`8A!`{P8G*h%4iXc=M%I+ml5b_Uwo_cU3R?$LG@dt2fOLEZVpu
z=l1rss~d9OnAY9k&rf?EQdn2_Q&7{AmEqB&?Y<i{3JdE#&i?A~w46^y-gt{8n~Bp3
zt*du7-?7XL;cW1(e!FyQ>g&7}|2QUH3u7#3WII&ea87}d@j#5d{>|V+k<YagK4r!V
z8~twl!gKWRMoa!9CMyp=Wb%m95}O~i<&Dk<qvrupGPYg6UQaM_`uTM6;_@ka*IBh%
zI}$}QcR%mDTluWdR^VjHd1dwAGST(Fe2=eM$6I0kZ`SUIu6y+k-O%mq`sr|F&Y$1D
z^H*P)UA=sL{l)v;o2#79-DhY2`+HaE)id*sxlccH?#15n;~C*<|1Zue^zO`yy}UhV
zs$b;)<?~;@OFdoIX7}>SNkbD8$Lt5Iq`Rx1x90u(IVoYr-N(+48jmmArk6YQW3=j)
zQw|&p&#%~7oDmR|t$)&+;gvP#wjRN6*OQB;GBC(Ah+o_kcw1G|;l}L3H99-CdRBca
zd^s^>*FIhIfAfsyG6ngay{l&>vHX(R<xi#8q!QfY_^198Udh1tvRioVf3N8~s`bA-
zIy!mv@h!1(se6BY_so6k+4SaX_KVi$-R-VF{)FsH4h=i<Omq6(Z)@k-WuDx1l4T{s
zv7f&4S1;07KmXA`n_VS8B>rza)9Z5Ssqbqo_37VicbVxh={cO7Sg7o+mis65(UFD^
z%7@Mxo87xEZ?Hhw@!PB|S|9Q*<aZ_Sp5J*ieEIv5H}CG|zgcg$Y5LKnYmN43Z46~|
z;;0Sb^~-43lJ52Y?W4^nwH{2|`#W>HzV+I(Ha?3OBxVZvoH=<V;^(Q)U6FrV7q78Y
zP&v~W$hfB?bRW|Wk%d#3uWVOiT;Qm@c3WxZi<F~td1hU`zNFEGXJKwmTH598ybo_*
z2=A5Y`C7uc$<<i;@-APg&-n&Vt;0+!D?V*-x968_*wh!wpt66>`Ov^1lj93#a&j`b
z8Ot5*DKgTzzN;&LNj3w6g4g~DD@s<Ub1*E(etfg(yT<Y*(=v7Y_HN$Fr?}$zg(d@*
z3uW;SO0&GC7fS1w&%EZpe@m!aeE6T!+=?Mbv))C%+h@yK^*!Zx@r>l~l$}!wrYPUM
zH?jD!=7HDSLKqwZdupB6vdUH{svqHx-P#`2%+tW_Fo}Vor9pwQL51OkZ1|J)3=9YE
z_61AN2w$^DrT6K*6I(qem%rPT*2VDPZ(pd-zs<~%Arlv`xzm<k9K3AO<&$2P4SWm?
zQ|Bw|?RSk#=h0i}EfC47#1PA{ZlS5Th?XeB-#2qhA4&Z_%fBhZru@;Pyy|&#xZm#k
zpEvnF(}qwsrxmNN-mxsHQY?9=xq*eD;f#<}gXhVGjt(CfRo34QVr1==-RW?#y!b$j
ze1jTK#R4V+10@EB>m3i6YJacfYKXZIyxL44Zf>ix^dyP%|5>NA2dHSt?^=I5l(8Xi
z?I+pQ%g%CceSKA_?(jnQpF9qQVM0@9OkrIAJ4SNGwd5($_4X3;^u>?MMDBT-*SIR?
zwbnwhk0#q$Sa_`}v)<ZC>asIv)CsA)zH~$=MX67BJ~zVxEfxU|uW$M`Irqw5bVlEn
zyn1l?_jP+-n4DmYPE}}G6?1m3^YbYN3J-Lp*8L4)`s!n`^StPBZVThzpBX#+a<_7A
zJ~)v*rg&YU5O3O@R^gH&ZpMl^ob&dCs7hFJUYlv)rgo(H^`*-nePpXQt=)R<z3`R=
z5j#J`@9}^0I<UMy@0R>ZkFKa~Co@j^*T#A>gr56b`+I%B$)DBdclCVE`V)6tOMKJm
zC)|JAx0Dxdc&=hpa=K%~<VTY(*@fqAc9)BhGtRl#e{l7JssFF(HorO<w|(8OYWL6F
zisetrH}`-2k$s@6KG=;{X4{*}=$CgN9sPf1?fkFL->eO3%04mIE#=5O6{}bG7qs_1
z$~C^0QMz9Ce$j*VGjE6S%>8L^bF^1rRh8+M?V=0}{VUd{#C_yQS6;NC<zGm$Hp2x$
zAt%kxTnAkGPv|zfJ}#LNJ&E^a&w?j!Z@yXoBKN3T@9W&Jef(BoCyyoxy!g<lwfkPc
zycj*Ub?08+3!a}9tE#UTR=X|m+HA3iqQcbgo2ul$ZFAz7x(ZZy8Lek<IC<n*jbhmS
zl6|Li6B#%dR%E+&J({|xeEZ5rZIet-u*<HFGhgy6BInajEBTwBn%COhUbyLx;m;eV
zxfG8~npG|PYqo%M^{21i85;%j*Rjw0;liO9(wX%^=2x^q`?=G68)I3VI2O)YuuIiM
zQRR8n$6il{SrdZ~JQrkOFsc33^QP+PoK`h8-iw8rQ~d4zw^|>Xwfpn&T^}n<=E`Wt
z%X%_8Nb~>uKdYe9L|A_H59Odx#j7F7m7U?tVeh?J?=G8b^F!(ILgBskRl6d;{JACn
z`nxRmUgOW}xfkEA3%$<UB5;Z0>N}n7a)JBB?r=_@>zSzI>v1CNPpz|`<b<;O_e>tz
z`h@jqZT=8^dXhlh+MQb&eco<aX7Y57oqFeq=C(X0dD{(T&-%+Bx43H>iL`XjPMB4@
zwfedCG@Z<q??Zmi4|{uW?<@AWKijky^~Z-SxHSFy%^THs&lNLn-s13amp_{mM=1MM
zU3H(5&K(?)30ku8KFkjHw5>EJ-;!ErV)Xxp_EScN87~CYp3X|ybDeo{O?6L@`uY=L
zEz?UQE(kI)L`VuQ{jOY^q5dK7&}!y{>;r$c##iZ0zE>I;SS7Q*ZO#PeOSOilRM}Hn
z8QkXY_g{Hz@>7*|!!7IP@iq8;cI=*hOKr(1@BA0l+h4Y7m<#ea_(fLtWi98Zx~t9%
z8nb`$D5)?r<#c!V(Wl3kA36H-y<M|_)2n5><~}`Fx$F)H1H;m@9NzJ9S!*7J+P~eq
z*5asS*UEH}+G?)fQNrir9h+~-Or7-b69>byi@G=EH%_^}+t$D+_viky2XpOIxT@~z
z-8wqce4A+ejMXzF{q*0QG#3^Sl3c8-upnaVCo$6-oh%J2n^~&+PAg4eaOnB^$a$U7
z3YHQ-&!ef&k4j1|&=<Ip`^2?C>`VSsc>$9{mp9(u6j`MAG467A?_!z${2Q}f+;e~C
zy0$;_T{La#*OL+nbFH$UzWv{u{WaxyqsF1UcSZNzfBVhx^~rd$|LcdAZz?D9b3Wgn
z@aRh4&HekYoqAmy)uub!;CsZK9}RoWuB+*LNS9XJ{o!`(>f8U~9E!dncb{FD|LMGB
zB`1UGB1Q+!4@yqkbYoUBdGt^0y`|?VAgs4`%Nn1`_g@d3I3KQ>qA_o}_Z9bztJo_|
zID0hQMdx@hUQj%Kw0NDS!~C3}1vVe&uz78L(4phRdEl{*)Vj;t#2D4~{J+|I{UgsZ
z-pVuw*5i*p-JO4Q?fw^c;^SDJ^Vi=`OTKO|^!ewL-@f|)_W1qHZ>;C`>yFf(-t*!0
z|7HGe{O{-K&PeII{J?f&d3g2G|8mD-3v4tVop_^Oe^iXI;myza(TCjq=9<;s^!xem
z?fD~Z_q@0Ld2(QP-QQ2XKU*fmPBxq;=#;f+nTm<V$_dZ6OjKSkSr9c_c%flQ9-}W~
zvHd)z1^jN>jZZ3W?Y+<JJjHpf)#4rh&)FvW+<V=Al9eam%wM4sz6-t1FP*0MA~&MU
zrRpc+#VMDKGzD~}_S*HHQ+{6lAkJQQZ#=7h<l&c5TO&Vf{CV(-^UB;<XX~}z+W$N*
za|f(sa8PAtsJN}(e43S^yJ{=Lk8i!REh{WuGc$xwU%1HV2oHx8+r6DZ-w)o~S!~9f
zc#xIB;Ai3CUFygAUX*+{N`JmpspW$El^HE;7UJtBFk5C#a8P%z{QGtC(wSP54H<eS
zwOx{vkk&h89usJDT`8+RXO6kC4r5Kt*`8188(n9v+AO(c+lDWK={Ntgcs^I%ZOqv4
z<42IxD@UEQpHgohFTSn(S(0JyvU~rY|C2bnf-kOKt?KR<rh6uPe-s?wF5l7Bbm{0-
z(fs;TtPEPU!K>FLM$HY&k9orR=#pYrkrBg#7hh~&ifr&x5vr|JZVBKG$Q9n@z#8x^
zdKXXdBCGk1uceFT?U^eg%;v<<z}LGoz$3QOb^mo{=QJ<9PtVLB?={}k!nCe|f#Csj
zz*6z4oI(yt)wb<oLT__V>Rns9eMVtaWP#k#K!zi0Cf;&;d5Ysf$p*#gyyDybuTr&$
zeDLVeq^VP%K7T%a%9JBvyj#yseLQ`x#$V%u3#Ks&eEW7yPd>BiV5<|uItGRZx$}8A
z7kUds#pr7Ers(Q1a8xib#4svW6(x&bT;neNdQtnsL<X;&GA{hn){4dVSQxii*(9f1
zaIgPnx-;hA@Av!9&oVuI`gHeEBLSzZMOAS{m)D+-{lv`A@WzJa@|LI4@29eUUForO
zJOB5bw3}%r4%PCxvxN=?wcq=@-p1YQPR<`cexBs?eQR0QH#+=NyBs^Ga_<?-1CP)C
zXY7%!bTCr3t(dY^OuQp6cFVS<fwj67o81Z)86UXXWE|lm&Lm<tCAY?{|IPBZIt&>L
zp5A9*aG0lPufOuu_NUsBQBii4pHdzl>z!*|E@xBm;m^;{yu7@tcd*Gmj#{PNz5JZ-
zpN0D#oj>dDzQ*D~p3aAg?Jtiqg+Dv;zFy^GVEx~^gh&0%I<u#IJsDxw&Xb(scjBp<
z*513>hv!>J#AR9dlqj>jSRuT8@$-_xuSsFNnJ*vx-Ou>#%iX}_Yd?h?BW=sBTwVMA
z$<nRb501z8e$$b=v8XFvpJ9$gVba4xt-iC(db_$dZQZ)|7Za$+O)EW@aXIh?zsTnG
zK@1KYC-=K?=*dlWVOla3G;zSs<87<7_lf-;SG6jZ;Gk=1i~9uPo~>|Q|Kk4@%g4QU
z?%ZkR7N2KRS@iDC&d0}kmF;ta11mLl&Ux}#v#@UBob|~u`;Hz>pA}cL%Kun<b9ZQr
ztexk{qX(`h*8aT4&M)`p(`kKGRaI%ToDW~Vr0kYynXqfcw)CC%jW^Bzc5b58W7*Ej
z;isQnZV_zfKftW^_HJ89l4kHe*Q@ni=?o`o?|#tQ-qIP`U-2(ZO-=3g_Wb=dKR?~v
zoF2U`r}F2gr>m^ZPCjR5SecxY^C#}fr?2)mCwKif{m8?hvVY4@(UWC1`)hs{Jv!1^
zS68R()>HA}0pl@arxkl<O}gMI8NzuqX1|K|Zug$!KbOktZ=X>3UhU8PufMY=E%+1@
z6QdWuZ%@_NS2K;%zrDY2Z)iC2ZPzO1nLgXf-bV5B^A{I?K62#9$H&KuEli*O@8MXu
zKE!stbN!35C4b6Nb{_Ytx+++@mv7CqnwpxbZ*O*HUS75?b~m4dLBhc%RyVg-SKdZ1
zW?^H~i`#SK?p<3gtyPZAY&p}L1%iTqq+9=rvXRSwc4p?sj~^KxJbSh+_qN$ouHW^4
z%)h_C|Nhq2*T25L{{Hs1xQIy0#~y)SJ+n%e-w4mTJnR1LNz5X}|MtAuCug~R`GFpG
zR{hZ1=TF36dwMSKurjC9lywoZ>kNN>$t~}yJ@)u`fBKmj6KBoZm34L1ySux?uWYy)
z5f*ms(b4Y1hYn5CkGHG(l3}yFSK!wotzV`GAAj`Pl(hO;#QpPErkE^WQF%P|dgzm`
zI;Y4*kKTFTI@0vL*ngfz@K%+8X$PO0m|UH4f5W3YDpS1zH<l-v=iT}7=xBHSpO44y
z>@5EM?d|U;Cns~s?3p#uYw7B+wVytG*syJzpO24>b(zjFW2YC{As6$zn16p$k&&GG
z`$^JOnRnHJ3l>Q*6cjv~cEmHhMf<;%^4=NF+0g}W85m}45sG3BEqwRzx9Bn#yE|{U
z79aPs+yCfxW@TmN(PPKHy}G)(`ujWMv@<78pI-gJIb+vi_kOeNYi~|YR{#F)ZuHiy
zsq5D1-4<1BF>w=B-LifA_odwF^XAW+R9IerRc51;%CQsTcQt;Og>4jC(BZ|#aCYZ@
zJ@t1nfA%}~$=$L}JM+TJilzVfY`fYjCnu)<<Js5N+<bO+_EfK@)5}*gzPq>g_Riwx
z4Gj&=?EG@pWozpG{sPVQUNQ(R-?ldO^tAH4c>Nupk9S|V8d`RN>#*|f=z>M>GS9T8
zGkEmhnOv0T_G0O0+o~@s-Xu#(OXuI)^Yho&*9#Xe)Q#F=QT{F_RX%WTYFb*KtaaIm
z35t)8_wTR%o_Bj&?#*=^ic2Q0`cgHcx%u$l6$M7dDle~?w1qXxvBu}7$lFcUx&O&0
zIm*p$|DHWR;v7PboPBn7cKA(4Y11qbA)!r$kB_-_i~aoZ<H*sYp*I&^ZE;bG+*y=*
zdYW!-PL7bUuzSCpufUJo>Xr-Ut6o`$y}GheShDV}*96J>3W=pt!aiQy-y3eeuzF+B
zD^XR386WoTusogNHMQcAdfmILzXKnx(RqJobNYGrez~7Netda#6|`$?eO%_3U0Lzn
zi8Jc||Gj?p?Aw=@m-EZnMC>eD+Sg}QuJ6o|$sqPl^Xk>Bdutw^;qT2p{_f@4DJy>O
zb(OW|Vvlex(7AfBP3EQUoCiy;l|KyPD*UtGIp@nKrMy^EL&L<?JbKfuOJ7}SX<^CD
z%`Gqg?mOEo^nLf`i4!Nzn>X+F_I&%ICnp{rZeP20Ehy+hWH(GUw~>4QVPWn6&u?Y+
zF5Yco-#S%lU)##wYZ~rC9z_-TZ!gBS&J!!pYiql-?(i#j#dEbgZr|QK^J(kngRcx4
zjg5?M?I?VFcD8x`ogEL~y{r57CUTqhs*0B{UoLj*m9ne(F-0@@-JPA6m-$MA*1?3z
z=iHmRC-1G6wsyCxceMqt^_u*PQ;vGssXm{XbfxTQ`}yL?{Pr)5A6KXMfB)?#V_kCf
z)YF{npZMI}k8e&tAGfEX@K}%J?QOZyd#ko?-fVo^HI!jv#GmK$>;L`ze&7D@m&^C|
zR%;iSO0)=Q9sOi^d*#32Z}0B>so0+%cS*ENTho70!Va}Pp?_K5w!Z)Gv0blx=KKx2
zCEY(we30fFc<XQb?y7ZBR_=En{p`)VyUVub$A#<Hx0k=a_wV1oojZ5tgnxM}(Pmlk
zVZrR#(OzC&B_&_h@Bi2Ju}5GPPw19c&!6AjReJjL^8KOLpO)Tb+Z@~dta56B=F1;P
zn17f3{PAN$?CI&EVq&1<;SV%28>gL_VN<zj+BC7-qJPZ=1qF+rol*2GdVH*RZPeDJ
z-7+l!rOU)Pc^N)_{5UVRO2SMwc>Q6XZP^!9W`5^e)~;t=|7=OxQrDoMOXuzX%gD>`
zFM4|F-rnlXvFE})y}hG1BpjS;T^_bR&bH)5z#l15rxlx4dOI;Yq@|@DKWD2k*=(C_
zxbTjGkKrDi(cyo79Js12E+ZpzW8Kqe(8k4MJ(63qu10N6>+SC5zWuajQiKjDctHob
zy?<|?eoh8dR<4rDnsR|5ASUL{;nn%OBJN)-XK#=G_}U{}n#Wgt{@kPbd&J9r?wL7L
z^7hjj>#jwh<M}c!F50ulrY(`9Dd7ChmlG77udRtpPE0&lrpU2S%4M&R6oUxY(Fcjj
z%1J@ee6~w8?`o}<SUN?Gr7t%@!@m00j(PLsZa)oSXiIeKl~NVp*p_qiP?@g$!G{GK
zq(z-pyjmIS#LRHm;Bwj-D~XD8l@${XoS5{s=t_6cB>D1!fZxx)v44O6@Z*)b*3IjS
zKRr2lb91`8yZh?H*LMgydHAk)e5OI6!&my>4D-jwU(CI0x?**S=!9p>kG*+ex5d9*
z`Tx81W*_HFo@`wC=}GPHZ;3X_EdiCUEZXZHe|UJ<-fz~ef&+QUx4-3FKeX>rPkSLd
zyKeNK(_b=!Z?0Ra^!4TC<%N%r?f?5t`gW^A%Y>@H*M7&F+4;?kjbC4`Pxw18`|PYQ
z>APjjo~pL1vi;jpeLT$G-hNy5N{{MqZ#;c{d3ktJb_=u!1X*X<w(EU;adGjvyH!{7
z>~5}l>z{M|P~Vyti&u(^&#{xdu`bpcqz!Vx2?sClTDjK^N-Y6LFEzBVHEh|kg>UYz
zVjb(-CGiXA-9NU{X61GBXZAM)%PfzaJlXj+N=wN1@?5LZPv5@HGtCyey;Y%Qf>qFQ
z>!Wrlsi~#<pQcSTu~ICMSsS!mbfdPg^W~H2&sWWp+bb^`clMDa!?pGC=MB!BxBq{p
zOp!xz$ug5<zF054>A(48ekptmJ$#UTCsXaV(<@`Pc+HbAxfc^r@bHc%Gc)tH(<>AD
z<?X+H`O?$VvnGDO9oX7gmwGHqH0|cg*F2h0?moSJch#xx_)W2XszF}d^6y_ikD6bu
z`^D*Ncly#G%bFh_uC5MGwoz`mke(6b#LUos_~yRc&tEzZFDfq)eX6n{)^F(q(KMSo
zR~N@`f2Q8t_u=Q&f|sDhc~Q~P{gTFQZ=D33Jl1AjihTF!+uQs0sjt?wHGh}9nt!%T
zS9Il*)R;Ssji1x}|EBtV-BYh+W3#9J|35=R!)42s<y>zPaLV!x7M5bri{16aykJJ}
z_p0FZvl;O}Tz{U=v$M3y`aO-;xt%X{_cSj>vF_N|SkQ8w?fLQBvK=`TOP3$y<YnNO
zvw3mr@NU1^^L3;Do+#73ul@UB=aS2ppP!w%v*4lAw(ONUlhysFd8kx>f0qhM1(z74
z%4C1p=H=xD3YxoKygffY=2V&Pebu+$w)JRpm*w4;5)ha$bLPo1%c)8#Do^h0ES_za
zn*`R$Gt0JJZ|d~v*LThiSG;~~-}yAYmDgU+sOw0akuENFDRcVabuCv`hrhobU(d_W
zZ(sk<=5}f5hChG)Ec2cH?)LWcmoHC#8ztc6VVv<zW>?7CsI7<c+bpjM9hH|_F{%58
z-gEl}nyF3~Dwse1tq$D&`s!+ShR@H>pFe;8{<d7{+e<??fKKuJ`4g0PE?i&D$H~j!
zwD7~7m%_2pw`^{`T`G_YD!u|wdOza(VQLn9XII_P6DL-D@zk5XJOBPZRqtsk>gt=L
z<5+)vzh5tJTeT(Ys@67cM-D~bAYUOV29HU1Hotz~x%ijs`xkoWLcdo`7T;g9V|Dua
zvfguZxP&-Zp4LBk@nT2D#YKY3ZUG`k!zx8Jl$DczeR=u&`}_L$sY)#ZzxKQ;ko{$A
zUH<M@<vFL%X8%<DPn7A(AHT3_&4;u0zx(6&WE?zho-iZt?ykzJs&n)0_jhHlF1Wk9
zJpcB#w~Ji6Z|JcowwU;s`eZQdD0z8lp|azvKbNY?-rHn6IB-5^!n}F!-o2}Pc!)JT
z{Q4@bRgZG-?kc^%uXcOhT_w*+;BY-0a=vAq9G{%cj;yPzs=vR>d~s{Or>AG`tt~rC
zUr%dpW(H@DX9rutvYFZWwrtx5Dwc2RJ+WV`@v2?EP6ATj-dM+~*kU52wk7KH>C@S_
zw!B>E+<t3&{`oT9|7TB4)m|UFyX@g1)+PnGx?R@g@9s>|3{FZ)`ts$=jdf46@9wYP
zpMHMc_xJa=Z`hEqTdYOkQpk)8QVEAzIJKr`U0)ZQBmOBg#c1a4-M^byxmRWX=Wh{M
zB{k!Ml!K0#dG4(*=k5Q0`S>w$_cZ<4mc?mRRlg=GyF)@pT5U_(j@sYb=FgA!@#*R8
z<h;GL=G@iQ;qCnL|2{l)PEJk+=Z&*%oNEoAot-^>!i0i9KMJ>T|6KCu(IYkAS%1D>
zkGHM*0&!Etj0;lE3j-=DD$dL>On!1=V$StXg^S($pMCyW|M#mnsIUzvU68m}D%(g`
zckSQb-^<_Kxq12W<y*IAy)C*|{^`ld+2;9f?(UmoIh;5`g)Q2z8Dw8y$IHpNv*>A8
zRPA{Sb@k<$mzOy@I_BQmk_ZZhRcfpJ^H)nHU0V~WH8tyS8*h&Ir#|`mKZRdjT$DJ^
zp%~Iwz_+$?o_+njO{u3>hp)G?wA>iW&2V6zZM9kMEfGn{laD`w!+jOMPe$*S;^*gb
zZ*OyTaoHf(1uwKq%gWNW3$+NWN;6n!8v6d;-q_gK>H6{Kl6Xyg%U@kNdHQtrw>LL+
zrZoyUT@_N>vh38kb7iHaww0e!PEXUlx$0rR{XY&qK0a>l#|0nS*jfZuCCs=W1v<g>
z^Ru(nA09Mr+g5f}{_n@f$3aW(dU|p$EO5LkFRR$H%GM|2tXa|#j@sJ4XJ?y-e=MBp
zwY0tc_!Q0HV|}vGN88;v7D_P*M{O1r6*bMglybaJ_U^9I)u$foMQl(oGy8UBWpHR^
zK#RbtK!b&*aTysePE1smx2wrWNeMA^=i%pne}8|yk5A9am6=y%1QlDd6ntL1O1QNp
zbLY;T?R>JUnvy$@CWVKC?%?_M`uch<zIFj8Q!%xYQiZ8rvrV(NmAt&<*vuCCZKIn1
zJRPy_|39D4KV>K2#BsIYhw98%4jLlw?(eU^wIy@r%$X*>`jMNK1TXg!HQnK%Vq|9a
z@B96FMg>S|$jHbjYHH!cvC#jZWy#l#IX5?5xNu?8q)9C;ES)ZsX3vh^UH109{I@SJ
zFaQ1ZRhWb2?d|RTot>5i4;q#)fBv1nTfphX=K{Xi+8MKExw*I;Ida59Mab56@9F9K
z;U_jM_nW&&!%IgjerM6r6)QXz285j0z@gY;$N&7oRmZ}@!hShhIlCGSvF`gemIV(E
zD7*L7{D1K5*)#q4eKns0qN8u$zWw{cLT44vy8jPA`dEc`S#|XHmlqW&c}`li!rWg<
zQu68Z=k?D|=*R8JxUyoRk6QI-p84m~CERqxig(_5FC(Maav}Xe<hxf30vEe!YHI4~
z={=qPKSV>Mu(0sH<)>C|@l)$}u3ftpbY}6QMM^#CEdr}@W?Z=1vUTg%ySvN({{7qP
zl=#wE#I?1hrRMntM@L7YP8J4_Nmnjkwk~*ZU|sBP7N&#Wr-6dO`-4t-Y4O`zTmSw2
zJ$=d)jj5;1a&O()Rr-2%`yvgm`1p9Gi5c11)>c;YtV&OvI<<<)+m%D{i=;*Sx;(3r
z7ZYa80JT@_=GXuJmb)|N-&to@S65FJp@jh!xwp2gSfK%SnwLfUx&(1&C$n|YtNEs9
z&fn)zIcLTX=WemBJv{q!TYfBAvgEmbUB<mV(+Wk`{`f1&a`5Jt@At1=yOtx~B;XX<
z@>gMHnjedt@KKu&n@!Sp3%pQL;hy`ZBvtiJ&c*G$7sA}T3(6j7ADy84@6)xZIle_+
z6%`h0CQd7!9cWosA?w?-dv@4?q_`D(CMbRT@8!U;a8=9og@R5Tilr@_y#h`gE0ujR
zS_GUtWD`+WPt3T$q1ZCP>QIXthhobWK{e14izR&qP_5Zrp`3@V6ui@!8^y%XHF33s
zf@O^!gT}l$=k8XnT*P-M(|DU-efose4X2+n7&r*2Tv+7f77^`jZG5<%^=q#4L<jM`
zBC-5>JH`J0+{-cDPOW)2_m}^-Hwrj$)QX&2xVmlPfqPy*UpRfcpMK!ZB!9cJKYuT9
zS+h8HzaOhy^S-^>`kP!5GEb_^t=^eheQw>_R!*aNd5uh8EN}gP>u>n$+qya3kxbs+
z6?H$K-gh)w5ng>JY4gOj_WQ+!rPn_B_dHp#Mc|i)&x=)xlke}J@cVpZXpqrTrl={a
zef9)zcsgZ`2O~p5(EQf{|IaRK>6Bxb!1uM&{CtG<r#GilEf=0H%{*m#vHEvQpPs<7
zB~#};%Hm;l*k9$g)GkofZek2aO{t<!pvvzv`{p~j+Am~SG2MT~k=d-$P8tkr*|z1g
z7iOHeSZTaW^ZZUlMh2g*O&*+DSF67^`_;}XKD_<^m68(1hUB+D%3ApEv9t&{S=?PI
z<l}Yl+1~f|Gll2u<ts^FW4cMu=+CW~S1(V`?e9Ha`Eje~)RsqClYJb0Zhg7>$FqKh
zg{I2SWw-uVYZm`!o31#`c+ZSAI+rd9T~apUe?Ft<z21fA#taOqmfd;`3<n=GGrTA>
zn}6Lf{gRR+_og=;@88Yd_-fwGY4a-rW{4MfSd?w*GwV2~bG|nx>+9<NSI6Qb585y_
z2zoO#<lLRW63}^+!J%wUhTRHDW%Ww8JCB%8f7)!TyWC{@@#nA3Y&#Qe(Z|TDn31#d
zH$%J1%tyy1|7mw5?w?g|HhJmEmd&NyOxuco_g>t7W%<)0zWMwPQ|J76eDx&z)B7!J
zBDtU5P-a$aDd0Q1NliFw_chI1yG@_h*euM=elu-Sz5VRQNlo|NmdI7RNLXe^U%#Fo
z8ZKIGRsJ$c^7<5Si)%9<9=>yX;>LusSC1Sv<b*DFo~R-(p>dM?#;o#(b9~r%8n%6}
zS)IH4Rj%Z6*P#5}_4OOp-_&>;Zf?vN@@cnuQQ3W_1y3&eE9mx4+EcOhvlN2?!!5gR
zU8ZkQ{Kdg)?tAW3F?Cd_Ew&E$?zi`j=o%K?+WiZyWTOg>`WIc-Ki6C9$-&T&vf#s2
z{r}Y)pW7|GACM4xe8E~<6}B|}nOaL*4za1Ss>>TNY+5~gUVoITo?2=3?@7LD)9Qt@
zcry>*I^HPY^om7zmamPtG=spF?-#c(_FA;qOD<h=v&({06PH~KNnzp@F<AMiC26hM
zziX@6Mb<^G+*BquVL`=2K86WZ&T8!I7V4>{JIvGES-tql1y894W(B)lD!G?#Ou6{_
zY0;;*?0+WK@;+U@?qb00iU!@t<17tVUSI2beSPw06Q7?|>NoGs_R^c*IO))SGtHw6
zYG;2kEZMm_aC+OO6)P5Hm#ci26F#=~)qB1NuJ85z7BPARq@FI4RC_1MD`K#7=j7#2
zZL(v^59{R$WvE`z+{vSHWV#%~X9If^!5v{$Ywy(yulf<}z@hlHC4J{=so;0}Ep7A+
zGS^>QzmSo^VRg2(5W|V`ZTdHNbV*t*E4zK{+5Y@faV1k3A6zqIt>97!^kmo(wLoKk
z_aw%KOPlYMJUP9ykcVOa7xkbYC-zyocVA+c*g8Lgo#jzWOz!u@G_6$`Kc8;Dx{vF0
zq;K^9SHaa^wT|A{$+b|7A#_i9XrUA@!-GqG;T0y27cAKpFTVI;_s$Iq?uh0yFesGp
zPFrVWR5#D8G;Zx%>HBe!;sOjjx?zkS5?wX#eIFbs{<ZsFa*Mz!z5>43qdYrm7`#G%
zIn4aorI7Rbob)+GpVIn!DYq`oxhBT2G>M_%f)c|Err%bp)lZb^)P1}i%9K_7aG4KN
z&37{y1_tRy!4txdB}7^Co9^FhV@a6Oy0l+nBL~ANvlDwiKHSF;Q1jAmVgAag<;Lk>
zExWEf7CSm^%ChwVtKZ1GpV}X({+Kh*_VfMag_kacI7sP7X>PfuZ2c?x=l4rl_F>`w
z{Ga_$RN~}_oqJDAIh2V(`M2Z#ecNoFaiw1J>E>UM%fP@8wS<w8VOOrKg8DX<x6S4Y
zC)qQ%2wdWLa%rje&2<Z-ETv`EO#Zg0v+<vM-Llt-oBpWh#>yq8D3@HAe5cOC-dj1w
zw&YC=&+l(no*hsBr@hYjNb{9T>bxaeYOOw(vPNx-Iklp5;&Sc%3uRB*oU|`;oU-J~
z%PV(JY@XT|E`IJaxBlW+S6;q6^6(Y6REFmf-m}Nbo_(^o!}e~1ysp1EV?)?}$*s|5
z4>DdBx1KNRx7XgEe{CB#1A`{(p?hmxcZZ$cH2K7~y(?GmR`q0k@F3Z2w|xKchqtyq
zxp4mI!A7n2bGyF2{c0^C5_A4iS;>{xSEBEq-H`D0ywT!5HRraBo7=@^2)a(+#A`d(
zGNv|2`p4cX&fcwCWn6@ow&pFq#+IAQ$u~d${-odui=O7_=UffFv2K0b+BI*UuAaJa
z<<t8uJwL5y_U|b#P~5*FbNae-=gv+2`BJfEf|R1fv&H-QFPhIX$y@Vs?drp8boSMW
zA6jCU$+GH*c>I3G;|mo7CpbK<JDvXh;l7qmxyjK?3=0->CtaN0e!);xw`b-N_0-9+
zho)rb`>5LetT1iSJMPu)cb4tYlr?j`1^>QY`}t)2&u$i$ke%UQj+!UB1)b`+n$6BD
z^+i-)P_bnd^B%slc52^SwZ8Aqbx^na`Rs7*j{Ha)=9*9&<_3#)uQQt4>-$#yIsDyS
z{Nl<*m(H$;X%JJBQQ%Z-_o{lH@=g3cON+p&%nw#3tDJV_`m&n_K6&=qHzN7MtvPSA
zK%)!~Wu!tCFP@zfX?|0WNwLL*`5d%KKuT+(PJY*;*c0b2KYSz|eehk{^PYs=pk}_8
zgo<d+g$s#4*T>mD$T)D4bES`1<$uR^v$Z$%o~$k@EXljQP4{-FLQBBagXbhoK|95M
ze|sAo6a?;>U2xAh5W=t6BCx6v)=VMAO=v9~YHRICkZ5}=6)@Gy*Z1v(h0T3^e6E3&
zm6gkuEfaBl`SPW!o7<*XDJPDF+YJ_G&w2DHiJ7_i`#cj<)3fK!r7e|f5pa@kP2ZU<
zQ(OCYUF_~@^==v>=VqJtudM<%nbb;rO%e~W*xK$j=g*uaXOzOx{Z*kw;FrL>r|UXX
zjAm}!xG_XaxAw)uhlS72$(DILaVUO~o8x@lg{kqt>C>yf>@O`TQS+H`AkP-mWN+b&
zb?%imU-!jjyVufBU%#gAmT3`idVjcO-NIcZFN1!`zrD5f_lIkFEdow!K!P5VKv#7#
z^vK)K0}TRj)Uw>uTxQ6{#TB+D;^T{pi@T&YY}jC+rL`(}xu1$>*og)f$BN2I-+4AO
zPd^Pi(ZHb?GC9#Hn_;Tg*3{F};_Lrzjoh5(=jSJCu_;FH`RAXarc$nf=K1$Tx{uB_
z%YAiobGj>Uw}6wWu-X=Izkf^S<&u(;KY#w{=;++gdwx18DXFQcNk&$dnTbitH0wy2
zC5PgYl?Ds1hR4L*xw0}?SV*X*wsy8z?x`}%|BYTtxwyHl>;L_EaIiUiO+?`8FkR3%
zzv9_u&b57?K7A6?i>dhW;UVZE@du@kwA6$;{pQ=%et&njm0SGR@86#*Y{1HWn>g3T
zP4P-~a5&H<stvjndv|$$Y;5c{?O$;-XU=SFY&?4OsI+<Bmv7(FK*4!QV#bB6Cue3F
zUtZ>W`TF(mpFgjT-mZ81X~>dxet9-l)|)qPva+z4<lUL^wn)IqgE_%zb?fnd`QN{P
ztE;R3{`qs~jvb(3!37$xUR_=N{M_8npFjV8yZwIF^>t^<Tsah%Tr*f037Sv)`T03J
z8ykp~e7sMp>g3PYokt&CUhePi>Uws*J*c-?Q7EO@V#g{x%hYRW(cv~;$SmK7Z{Mz6
zyqLIq(W=Q)r;18QL<9tED17WDFE0<;jS1>Job3x<?&mw-?(U5nH?Cj5e&^1dw?#Ju
zrKP1MByKElY;I_1cvY1ruh_CrQmrJk`pb)pQ#6C)_EbE4{`~oyH)qN$brxs-dUbVm
z`uTZp@9+P=Ds;7vkI#*Dpw0%XTFKRuXV1>HE`Qf4tiET@9;>o9Cmtr;o@6A{`Q*#X
z%jNIx%(O0l*CT1XWy=<@mK*Jyv7#beM}K{N9lk#9?y6Aj{QUgf+}v&2LGJqT`*^sx
zPMtjq+Tp&x?(YpfcEy&6HqO|Wese4gYk!pp3k%!Z@3$&_wIQ}|SM{GCg$W4@rcS;3
z?CfmQ?6tuy0;?isT$rk%rj~YUiss9gFZb=+x5MZ7^~IULetmtt+<*SN%gfKtGR^k!
z^#x5xwL~;<#%dN771jNGI{m?e1bzMWe6m&#RG;T7O!PQ6&sJJa&My7j9MBefrrjw@
zEfd@hw5&@g+WF_a{r{NVWx21ft=+q4&xc8$%TJv<$H&EW>fE`w{dIfSuh+l*6kP7L
ztn+YmbTm#ovm$tT-}?3XaYgGKK7RZNI*TTH`@3gnXWzYZ2kgaFOll=ZD?U6pI7Ktq
z$k>>jN8-YYz{T%nd?SB6`qis`fH|@GS+30Kweh-V_msX(<kaYWd#mb(<lfEvx?UY!
zdxQfoR4!V7^!De2r;Z=5uT3zltl-|eW5tQx@Be?Xzdp4_-6ZbF)i5pIw^6%pOEO+q
zdC~lFoEL}kZ9m<&l@8)*NA6ZEd2{Ko?~khai&{2Ju=(wntF!;-iltuqe)BDhkKMjq
zeRh^<T3XtDO9dy6m6m(#e(jChS7TZ9<b)ph`Ze8{9Up!_mSm`C`)X?|t=1lEyY+jS
z+~03Mr)1x5K78oXR_Xon=NS%MT${S0YR=bJchfzYrTwLy84F5$c{};*etn&?l2g*_
z!Ofk!wDjV3%~2@O(QZh7n;}?cdi&<pdF-d2m*>Y>R=+5lwdT%#t<*<bPg^akx?go?
zyM>J3@jlt<`tko>U0v<gE44OeXOQ0X`u`g^6qjr>Sh)Z6EYoZ$)2uJAuC9K1dV2U9
zB^8-{(-;_DM3+zIIyEmS?LOaR=7xs<WsdwyW3;ADW1YIMQLv%mNkfoUz3gil@9TcK
zGQA8A;s57`UG`e}<!zXQb|yoV=2Y#|xr+71pH}baj{fgCg_&W$-*m&dFV1}CwA^$_
zUevZe>HhA`tnv&MytB5yzq{M~v(wiyfy`rT)32R<C0X<S_QzQP8ecwto^4&8S5UBF
z?b^3DH>Yc7rL+hHna6D7zK}3;=1gvJy_k(ju1jlVJzTGTm&}@7_9e%|C;NEcFNXDU
z?Zs7Jzgm=UTA;(YwBNZYPxP8m`;q9Iynl{=Uw4$jpzyW$KPCnTi{4qbC%mTBJiKwR
zDNXqJ*7e`ZQg&%P%JERkSblF$WPU^a@~E53{aEeJEy_J>>A(4fpYrpva-&mSe0#sR
ztO=F*7jPqc+Vqz<>+=i_{HczzIIw!^!o9QETjcgneQ|9)C;Jk9J+<xf*I&QA@qS;&
zqUVN|d27#V?W&);a^=e-ox;jWN^1V|=1iItba1h9%Y>=}mS46n&b)Qu!iG!x)_R}Z
zam@F2$#UcK?#sOxyz=BHmKCgDU-9>G(~j=n`B$VZcci~R_|JLmA~X38E#7x__PUXq
zv+De3AE`Yq|3OD)_KpaTdw-|I+wQ4|dU|1LwD|gae1eU~yIxyaW)yp*@4t9C{O9+g
z9cB8FnqGGbELPU8*W0`2SJ1}g`OmMtG(6Cne6c*_eOQ|eL!akt#<L;i>r}cWG&=S#
zxV)=T?_=lm1wMNuZywz1sQOoV^Tv$_cWVeZU6oMV@+;}dlzCgT-J67k=1rI`HTUk;
zwKMfTS}mNK$^Sm;OV$B)h6h)&@ATDuGl@US@G#_ljI;$?o~+e{^Kl{?mAjZ8?p~~Q
zS?<^s!BfGHgpcf!JM8{t?b+uH4ZY2({0$-ull8UFon>fvlg@d5lZA5|PkxZQL$DD;
zMe(Owcb0DDom~6;<L4#6`K`(_znqlZWb*i$ad!G@t{oh`0#2p^^PYyqdo5u262+5U
z{q?}Zt^8+>>1T4ET65iAWh+m|nH@eq`KQ$|<T@)i*zhql^ekj#Xkl|>S$OH0<Hz5N
zFBM59=OjIU6jjx|V@tpi7KVVpRD1OuCaS6)%hT4aTY52iPH@B#hAIEE`|_`Do&ERQ
z<J-xW_GhBMo@CIglAC;a$5cx_lO_SD`z+5dyy{vXDlk`*eZ}1kJGBMwY_pZ-;8gHr
z5Ge3u@RL2A_T};q8BJz}oBr<#Y-6&8MHmER7#h~xFutS5`<1aGMrn5h7lX|t76!fk
z`1KplE>8ZP96#;CsY4yLjW4zMI2mGu7%Y}BFqC}FHeO|%{%(ne<4n=oZ#A1b7BDbO
z@ygt|<&9Hu?RPVV2bZ?be7HOG@+P50eAAf&s=9XjzW((?pGUD}AMehRmqFXIFN7RD
zdi3`8{P(xEX1DhHbQ`UoV<A0JLh<VM+?3qZC80@gZ(VY77b<=BX6uzXcMc`}wfg!-
zX71XG!ut1Jmz5?RzLcx}z5Kf9gi9^nj{ZkovYy0-{`Rz;BO<vm(R|&~qbU+MBKPj8
zSJwNgd3gQf8+Uvslzlz2b9J~={<YV($2$FXRVaSj85D58r=sTXqt78bGe4WjZB0pU
zYIfFdzjrm)dg1K73{S$>&)HKK`tsVr!1?YB42lLOXS}v%UB5Twzgd&3<VHos<(Gq>
zE?Ax&=Kb;dNpZVcE2)rMnNk}gWJE<xlaKYBvSM)J*vJ*G^Rw*b%a@RYno~EjMQKi7
z?VzEv-K{fb71O#2|66}Ae_Etj@T2CQ`tlv`d)G`k-S~g0<9%5Ny`?`6b?xijJ!j6G
zIl9r?mMmKqc27~jsl&BGHfnQW#ryUL?EW<}^Y#DyVSC?u=J)wtZHpSkc@Nz<7Us9_
zcSaepv^l+2aLUQ@Je6(<IgP184_;nh_dFv#J?)2S^`%R1es9mWuSm75neck^B|RzK
zv-ZUcj>q47_W22apZwC>d<EvBHLp216k8Yy_|_^`l<7uu$i;F_jVgU{?b<Pg{Nuku
zt=HTCo@Hz4lpLG-Yjck1b6@e^e!*0mvlE$X%4V-uI=f3u)6n=pg&W7h_pO|3HMg+b
za$#0xR*c`?n61A5*q-J5MoSn>e3><#rrH{0tPEMDv){x_ce<A2^%D{dr^+vCs7^oa
zZN98~z8udEorN#?w=G<%6!=pxQ<azDQ<1GX!xW{7U(OzG0+)Y<Q-xXte$`b>i(Xvd
z$-uBcQ}S@Y&+Gi}Icg{5@J%>c`1Suy+fRRL=dJscw&Y2@eSe9W=#(XH?{0m&vbO!+
z&SHMg^yosjH#4utn!I?l#kl=^w&ks5b9RT+FWLX}a`E{YU*7&c&y$t^*W`&u<Z|A?
zs1N5Jw{#c(JL=t)_h0&6(EnE-ABW4Qo8`rP|M=J^Z}q|M-#b6L`t!?oZToCsU1m{v
zYSYB9n$%BcPw>sPdU(H<L-EUBgN3H;4He)1d{lnh^ZeoQd|P8ltG%1!4o$ouasA)t
zDyO)Qm+iN`?3s7(V$(ynJe!JF57#T7DbMA6wYOyJ+6fz;rseNE{vyJ{tZ<_RzYIg=
zlXv?|e(Sur;$ZFlA?fnko}+eaA6)zU@^Q_pf+<##OK*E9m;F<HZIivATX0M6m-CN&
z6_;(c+LlvddT-0;1&Ynw@APjKm@c0_`@_w@?rObkyRD*EYG|IFqIbl$z(nszuYi*%
zgK$*w35g3=O$`i6zs|gQb<4B~%#oT0Hm;0!zdu2L%A9nG%i)<HgVsIWu<-w7)8$2v
z<0sj^;^JgbIBl0QZ=C~&NQ067v!$i^-nk40`Icp&-V;(?85_Jf57h7N-J_t^U1TiC
z#PGp)PR_}Wq?vuaa>solX7fjFJ(J@g&Cs)UZYcZrbH0BQ&rHzcb6TXKr?$LsHP?I7
zP5~!V;d%X6RLsP_u35Ww?b)oovD_|P3=Af$Ei<D(KU=qM9q1A`wHVIBsWWOkZ+?Ba
z>G!r%vyv|`ocVm`(I$7Vl`niRuiAdQz=0t^VG=U~!^+s%hXX`-4?Niu%<q0b@8Fq_
zvMqwU|3#GPySH!`Dz^l%9*SQ+tI3(k<4ITfd?AOI?kBEqZ+r9h_qt~%Gb`i%?5khw
z=*|7&#PRc73=S$<3~qW$<I68TcpV{p<l*K-Nu`}HQqSCbP|<XSiD6b)SJ%?i+aG-9
zNENTY>mA6%;IT<5^jz4n1ylCE>vHsD+)@!R>65|KEiWzqZ9mvsS?DjEXS+EsD>dM1
zK93VeXz&fo$G1b{p7-!yIRCCCX5U{WW6NH%HQB+}ConANj*qb^wX^j;w>mfM{Fy6f
z4r;tSw30h9{LrDq3H!ItTsU)W-NXIt{jrrbPwRXH6IZSF_nvp8W%<z*35y+D^TOXB
zT{&6t`r&o%ulncQI2qyZUG+5P!c}dj;KxTc<<7B<+M&Jf9oNYeuH?LqX#Q7W@p46v
zJhE2p39G&Qruw?lJaq%MZGCQ&dZpLBmVS0zRcgoWsV&}fe^t#8p8CIR%As?T91CCZ
zTBp}om%f<ukq=ZfS{&MbI`gvEPN9k#xtcdAoVwG4^Q#jt6j>hqy<uWokk-=u7uuXP
zr>^?pr0A|Woyq^l%P<*HKI?RamR06^g4am>T+qw*y!fx4#h;>H<@?oN*!+LAvPm=S
z*<{aI5&!CFqkz+k{RduSovZm^wM9)Z^~k-NDd!%4oZ_X*aQ?)id5fESAG+%+Ouzp+
zwdTgr6;^Zq+URW2-lMWq<M+R_>eJ@Q>J;eawFs;VuxP)gvP;a5?@&+obh%qwvhHtd
zt@ZpdFVwp~<;4Ci;cKn#?5K!Ty|P%RrporsqmQfoU+Cw9rquQyY`LZ*v(SMdVU6;Y
zYim>A-n=gU;>GftlQzj~Jq>z!d&jb;K@&qb84g@FG<sa%sdu4hWen%jZ30ts1Ak4>
zlU-a>uCp}cm!I<L@{c^Z*5BWy1sm$~Dz;p({-Cqn?a?O&hE4O-nHx6R$un$P<g9Ni
zm@V3FFY;}c@tU=-P9AOM-&$wjzh5`oa_er>jKhy_r_0^#tYP!n{c!W%*sSR``}b;F
zENxvK_u=QBkDq^cFFq%`r{eydYi6%s1}mIl`_*4;zFhfSTS`;etFyDevi<m;o%i!&
z(3Fs?dvjU@R(Tfitu3rLa3t2E^4ERd&rE(Ewe^2)-kiV^_^9o$=iBPimrt)w@3O5m
z*8F~QpTTFfytCFnc4>2aPk+beJYD|!nt69lKMZr3Bs1lt`MHT}cmKI?aQFPkBrnCA
zD_Un;I&5ACT1<B3ys5r{fz{`SM=xu&>)qW|@wKdj!)DsIrNu8f6kD{!)V3T8WIDce
zY4qPmI&HVNRTl5IVo*4V*38>qXTQDr|DSEjD%%Rbi#ah&ZOMF-a+L9c&=j-V4|euO
z7+uk^e)8j=aHJL^r_I836E<wze1-W!Y3^&+^rOdYJJu;rS$;&lqtv#o|3$QPK8rKQ
z!d0!Df8X-(@GJ>>d2zA3s?f>l+t)et9NIAN^aaDaq2}LYcTeVH@bdDU;CD7AHg>jo
zd1<L(h{x+KFN~9Y<O?GntU7wI(uAR*ORL^;W?k9a6Y>v(&i`xvo3k^&Dtc<3)2AH{
zSCT|}Hg?YKUcGwt>hA8zGh>#plz#g3^fWU&-<9?8^Fb?|R%}Xes<*ba-7Bsi7ZDYu
z7qvyh#AM2%Rgb!K+G}b{+|L)@m3b%6zhhp^lC|-(1Ls;6tGjZZTJvJ_^T}~{EyeO4
z+(`R%X`{paNg9?h3Lzrw_qH5f^!eSK?zazK9S9dovt8`CkcA;(p7&H9h7`M<hyO0@
z+`RcMN|SMw|C;-LlNc_o+9Reb*4-~-xhe1NE<t6tn0+-nx%ztqoI<&(qie-;b93e8
z<@NQ~E4%m2u_|3v_V(7YWy_|P?|5{jI$b~JtnNw8iLbA2-o9nSR`ImEcEUxznKyS8
z*FN$oD0uPo!tPzU>Lr^`?%e;qXw|e=p^G+!v5Wuh*>GXUZ`K%g?MI0bCq7?cvbDZm
z)ih=D@^i;hy*Z73U%UGH*}8W-AC~K@?o}#U+`sI``7hadS$U32+U8o6fAxBCwrcP3
zEnBYq-5?h=rSkWqryW<)TZQJ$oH})1*wcRp-2+1+B5vHT|9^K?DCj7((zmy!s+`n+
z-zMNRi?cd9bcS*IIiI;!S1(++aQ^)HJ9qZH;$ASdXeU>T(?56lx}4&!<Ifp2o_fj2
z8*SIOx|@AcB6z{ir)3jQFFo+^bLE^l&rh<7OgZh$<>9=WtyJs%(>>~DW@ZKk7mgiM
zQ&2c?^XAS|ecb|1IS2EjLnZR_-@m)No1KMaO~l4UuH9ny?%mt9OMYf~*u3vi(X|&|
zzB+u?d-)z4JN^X`UwBkR3Vt3c|7HC--h9oXB|r7Kr#$_)bHxs~UuB}9ztU4vQ(4*A
z#Kgq%Zf-hy`}S_HzD@zBH-{}tj-Eb!I(kRJ!->l7Ro~vceED)EV|vodMID<L?mWPH
zp)BkAxoZL4Z<|haxGXYGKj-4=svEQ8!u|X6kN>_CFT<f2ayG#!eA<p37WMW2mn>0<
zkH4?xJ8MI1U)w?rEfW(H6_u0=3mmn>)~pC!Ep}T}u_Yk%aLc-w=jZ0mwklmUZ{EE1
z>({rovfh50(KchojK|0OZ{NPn&cb2=T4%%O#IccCc-N(sA*;^LHrJ2ew<mIQnw_0p
zXiZ_%=9H8a7FO1&Q>NtH+OjfyeH>^9f>X|+mUT)$fBx(i)3qvj5%BV8rk?owi`DHO
zDqMVg`|AJyTjbik%x7lPdwwU5M5pkT>i+Y7{QhlU`Dsc0|9|V&t;?CdSS&;<^rh<a
zh0g7I@%!%FxG@8CFq(&Mg41RN6_u1%S5`VL1V#I$Pr|l~v)W#~c(Gu?0=Hf%(C}o=
zjS1l4$=(E~&CW}MK<mM7Zc5F|%d@kytE#HX2@kn?>GEY}X6B_qFPHhwuKM!gAb2#k
z%wXZokbr;<85b8dv-6j|zqhye`8m@kkFpC33kwPgN=m*=QuS8zpBLjCXX?c9H+;s0
zH{Ko|GptIpa&p$JU%!6#Z0XxiLzJd^mHzznlv`YnN8YaH=clJ`Zf>9@`%bU8g`;9)
z_EcQFe}De@=ll`|3!>uYfCA&=<KuJZ&Na`y_2k8i6JVFTGgx?)!!;1JAbVFy=H+F+
zwKX+gzJ9&AZbj0PrAt}a*iJqD)X>n7cX!vuSUx9?Og5j4t-%oyJ5o+gs{i+MdG+^q
zvAaqxzBs+*=+UFg{pMEv`0()3Qg3zNSxXqbrzp2fxOJc<EZEc2GkCdQZ)fMr`}_Cr
z-feyR>7R71sjn_A^%mEUlQGS@a%rhIFE1~6{Lq9)ZA;YdnxC8Y@3;4y#LULCVcWK}
z-E-ELzQ4Ek@$vrMyLRQ>*>UmSy?Nk(S59<VZQZr#-@o7QUAx70?%46=<>liC4m7;&
zvQsxT{d!|#vWRQ$y*)dNpPvIK7bza$C}R;(QP6pN=jYjmg@xr@|Kus+`t;c|z4(29
zj?32{@jl0)_+_raLQ~JItXG$p``=vmXfj7r#<ew(TX(X`Dz@Bcg-(s|-DXv25pa5;
kY(T~g5Z2k2#s8U8mwb))zEZx1fq{X+)78&qol`;+0AhmLRsaA1

literal 155610
zcmeAS@N?(olHy`uVBq!ia0y~yVCrUIU`yj*V_;x7Q2#E1fkE(ORY*ihP-3}4K~a8M
zW=^U?No7H*LTW{38UsVct+g{Vi)1$Pw*LPtV#yTVV8h2bDRie}-+X>s<&&SLd1*~s
zJJEe@o=`;vx9GY3|0)mL{|NrvvC!<Tk&9COs};)r`(Awf`uyvxq*}ZE_o58<-~aoq
z?t$2O-ydaNt=B*OYMXOCd)A)&`|2EC|Ni^+@81RAekpzb-h1!<j~$60)qhLfyZ-fi
zuUFmP`S<?wKmJ?1`QL`Q-|aVE&$?W<{?qdR;Y%fU7w(_rY^OT)MEb|&B4GhlGyloW
zPZreca-Cm(>~my&{BDJ}dne8B`u_8(|JLr$f9rn6b$!q?ef@8x+xzdqziZ#l<NCW}
z(U-4J^6$?O%m4H7Jpb0+<=?B%XP0m0nmXyjy(`DV=B?jR5P0g-pBaape_sC)HNXF$
zdwRFqakINIH4z@lsan;+zSd_;E?36sN_wyPnex;${q)ny_2OrhZ(pABO8#ZqQeNu|
zi(`7$iQYRmKil(s@$t_#-gDL8ADL=dxcP|xpHG5y-=F>e^s#brwd^j<qyJV3Z%Bx}
zX?WIXPw~H*t1~=yy`Qyh`M&e+KN!tFwjUF-x-rkF+|1%})!n46Us~6G`m(biw%hc`
z^$eA}@1+Bj=G~3((JE2!($e+pnzebEi>&qEnQLsO&k~pQSo?JDp{3@no)_+Hwx2!!
z;+L4&(RF)yw_fN7YYBX5r(v@C8e83#G6R$7Ow%pbmSn!p-J<Eb%0w`*uy0n>maO2w
ztLvQ8_4j_ewlw>F?H9{W8UYc~MkYOW$7W1!_sg^W`SPS_@wz>yR;|{J-1SQ;Tk6Bc
ztg}b1?Bbnm{eF|=f8FTA@2cNaUSjlGKCkS_i<jwhDi7Y$-F~m^>8rQpcYfa7`bF+`
zeyq2B>AUx_-tX5{9)DFlx308zd#HWh#5sPmzW4M!o%Lcz6+1)8w|PhBX_fdt-;-XL
z(zYmd*R)yY>FVz#WY?%aExVllexKF%>-$>6{;%Hm)V@@&;`KlIx!m`5l|Qe0dpm#E
zw&#mAo@P$)UwGYQ8u!M+i)ZHUzwqdLxW(~R=XNidekJT=?$fHW$mUGjw`q^3mCasS
z+0>J|E941B_0pj3dA`%;ELaxE_w~KOElux7wzsxz-FvhCcIc^_T$N8f?VSu;PA)3q
z%?ZD`Ip}_C(%Nb1(Rcn|xmj6c<@M2TXX>rYOr5z)&s;8FoF_EtjNtWMyk4t{Cf}Mh
z<@Yi-UCFA|JIyq^RnJag{iJc?u}#evTl2lC+&peK+H>caR6P+cvM`l>?8Y0Dpt-K^
z;`W|n`<}0Tk#Tj)^1FG#;rB}AZe2X;<oBxd{mxUfdXEQtEPXjw?v2gswu`Lu7^NO0
zeBTu@RdelH&(-TEIWl-Rmd4Gj*4^dUDN?v&7Q>Tg^HwKSEcqTJbb%q;`P<s;#qSq~
zO<=B=IeDMLw%}iFLiz7E&o}$Fv*77IKMwAaixwX$zOKp5zB)}q!vFQZR_Tf7uYdTZ
zbg*sGHHq78r+s_ZoclIm@|ruv%X$)5J$?4>b`<ZG^GuhO-e@<+{t5K{?lATF6W?^J
z&1Da>Ive)%E&g7~7j*ML?BnQU<6@0RR(rWzbCNGjY5ID4wwM0D%*}gLzRzmASbWU*
z>*fy(#n<13Up)KdMAl6c?IN2T1H(0cG7==FJ};~leRcKGrgJ`w!9IzNW!LWQp3xl_
z8MtECLAl)u&t|tWeYbgemp8%i^PRwV>(u?fx}UFUPY`|<dh!B)b+T@icUq^Y_y#xM
zWw%dPYrSi7W6inwB>u|RkpEXDwzu(SZHZoUGHt5Ps>?6B{@pfAzuZ}Vy7XDA$Klq6
zd2*!-92%#6J*IZ%?Y1b9{0sq(uQ3w)_HLafn0k(_ZA-6C?G~PvV<&urXMEvdVO*N@
z;nu!QYqQPVmTbO!_D{Uq)n|)huNNKuR=_^*a(3*VP8X90VmWg*vg({!C~Y=xZshsb
z>yG&*7Os_KTAw>_mjFYsglVUtKtL^<TTpkyg#-(p|M&BXxhr06-_*lsy=~u0m706s
zWC9rHh;3~&?U9(W@5hmU0W9x!f4J!!`{7l?ZZG?sV`i7P{rooT1momee4m(~ew}xF
zu^or!tv8J)bE8eYZEZKSr>=FFbHTEOb;g<7j8ix6*jM59cC(Y%9sZ*&o<DwGeeU&f
z)3Xy2vP<+92sO<8cd1F{txE4^-V}B1tH~*;+q#_|?SF8sYp$(_qIcYE&%7H`CNJ0}
z`N$&Vh^eaQ+u7$tU;JLsca57#StBU>c&aP+<!v=nZOZMK7%jj4&~Qx%+vMN6@ynSD
zd1tR1EEl-oHTP6%mruaXn(ue4m7ipOKF0g=u<7b|yv1K67o5}6$SsR|pW%IF?sc)*
z0^b^PyARFb(U|KHxRJ?w$H%=J_|^#RHEPQJTw*dy_r@AVhPFnP6WiW5-6;!+J@B(&
z@xfrLIYkSqOKc85-s84KG~nLV!w1<)9XHNkwV1Hbb%sP`f$U7?)>nL~ZHn90c}#IC
zDG-U_yn1x+2Z5#CmpgPe&kdf(e5-vsSA*D*^6p!15-}HB{w_!<36$zTa!f;f5oaZ%
zu<41G$mCCqfoWz2zp8J(VB#%gUv=PHq}i(ecc++Uh(C?UkY-Ci$8oRURsM=tOnLIt
zdBv+7*Ksn~p5geN=fj>;ad}o<d<4Vfo~Ip>j124lt?8S#sJ+7D$2*4YJVu2f?n`eu
zyx4xkmm`V$kHJiKzxjT8S1)Yt-B?hkKJU)GgsVH9WP{l|w3|-r9@R@;bSGak=vJa|
zvq@Fx8$Z^bDY5GkN*?fh*Z%6LoWLU=f1reW{=<}OH?E)cnP6c4BG-5}PsHvICvVyv
zH_MAZb6TwNe(L+s%>S!i$ljT>?%_$B_=Q}lee8mq9*2^i1-LCbxFmr21+(J57YX50
z_yu@mqW2wtn0Geyu2@vd_R?zBokl{}e}1~=!79$GcVpge2Zue$I&RyUS;bxlZn56_
z^`WovVY^M5ua3RtuJ*XS-+Fe;DX;h2eS-IJhMait{?)WO`<^d2y;4@pbN7esyDwL{
zMa(Mi7d-G}?K)%Q0~S0|Wph@g9(j_R{8c<H_aVm-SIG-HMc2#T`<$0ZZ0$<<`b6|t
ze*6xBt|iZUE-q{rxpTqAZmrx*(^HKm^6Z-(Z&hkEv)C&HOPIgev{|k0oGVN0;&lfa
zqWgm<8YkR~$lNTUbZE)-3xY~U9zl-|eR-oUtYmY>Y}ul-t3K}FZ)6s$JbYICsneN4
zd7G{C9=vl2jQeqPJI~a_uZ#Nia*n>5p)ue2$6*6a4#RoRg{$S)7e_|C`Y94|X2$z8
zCxwE5tq&4S<aim3$_!4YbZnV_cwz1j4&NBR`4u99kMCSKQ`~jleCtx79XTwsU(N|g
zN<Y4CTJ=L|!zdRfwd_SR9S*5CaPOSDi2dNsb|b6S<9{}BEq{=sEurJDz_>(&!*xFI
zhNF*fJ(-vpm@ig2BWsVZz>?cHUoxh?IXCI>?5xxZR;y!0j*YE}@8XzB@&lUQgw75K
zTzxJ?m7Pt<b{~h;?p76#hs=i+6mFZFpKy7}w`87bo#njz^?B!l3{*~AO;?ov{E=}Y
zZ>4^kx`pWZ57&eOd5#9RKH!&-uQO{731Yr=%E#^{ht1;0yB;{aak6{kelqv4*}5|?
z8Jrt60$IXJ)jg*e-E{fx%^rB>_Egab>(b?xoGUg~waC|d?OdR&$Z@AK>GLeLtjilb
z-u66`lq`8yy+GjTk*1vT7Pe+J^(+~ane2NLyjwQ4@y^RLI1<d*vh{`XOO1)j+pP=N
zGiEH^;PI`z=iiqo!G~UM#mz>XB4_wxtsaE@xp8uXqxj2{)$9j6`Zt(VM7X{E;?$sf
zBFcG3a9UhL%OYits)Guas%77Ix^TPOaC{V*Z5U-?CH5+DW*5T@=Es_an-uk%rzNtV
z<S3{rJACO#LCq@$#Rt__3afoLb~4^jVw-gPuttI0vZ!D8%m0|J6~6DFH`nsOp48dL
z<E}fLdMNgK+0G4X9~*2u87g1ww4~WOTGjSW<KnIN*xItf{HL~yCU)N4rXk~VVnr~Y
zms`i4%Up?4KXe>A-Yd3FNDyjqJ)zmdUd6Q7N-@M<a;ad$aoNs?#&bi%W7rRVEU>xw
z)VY8)tEF|`9P#PVb~|J?^L?4_8xm5*Ft_E_DdpGM6%K)aRIlv~mOXd)>M}#ERcCBc
zycRY1c1Bu==H@vpteMgz*$_KpM^~{bm*5=3ma~6-3ZEQI&C?3hQSe?PDEp*v$svEY
zOrdKEdV9EpdyHA~KQ#QFA-VA_LrR55+b83|1r<%sVQM1HZ=H%{A2L>bcG?l}wktZe
zX|I~1$PfSROl}<=Mwjn!$h>r}E}ZW4@9yLN4QH&F6!y&X)>OM!6c@37a_xmsk>w)(
zITHMLdMD(3inQq$OS#kWFsbeKTi)--8On^ZyuKaSolyAEXx*(kl^F-ur?Dk4$n7!x
zSDx%-vg@JZC$T*W4(i{%TW?e`R-REiueN`|tcr<#(Tn7pa&)FAnttDPiod_NnaSR+
z@wIhuTl`Al{|`(Rt_wYq{PBd1<GS#7!8Ix}e6ET1f*NUkPdu+Vxo+^;5zSI>wB?$F
z>FI!346jzbc6n8s7}H}fH(haUhiq$+^pnc!D>EB5dL7s#^#1!-<tcO6)>R}qpIqCX
z!*KZ`yOV%t`~zDy5&iQ&E~-!ZR$#7lZbo2}|2)+mMIRUY5BUOIJ#U`}W-?FgITb4*
z==f>5sGhIQ+TSgQV|w;q7i%n8)p}_Kk6qrb7p_q(|3lXLF}F7!3r<$w_+y^#&+g+5
zZ5tYxk}dwf<`7wSC`M&T<8qxP$D6PG^t|{<rEjIz?MGR%(nmbrn*4Hjp>X1p&$A8J
zd)n@6-dp}j_3#lM6Vd3VU-9OdUOjDMJoOwxTD>h>k~npw>%-Pb^Xgcd7#%y(al#<2
zL@;PU|FNs>Gpv&iwtsWpu~T^BL0;o<L+hWx-j7(ebjqZdInO-E_)>h)$NRBIPS2b1
zuvy)*Zr1iHKOdEeJiESLo?WHf%BSz`+%E9X{>_Ku*XLf$XggwaT}f?*uurnI?wVo`
z>+NE9B+j(Py-1Y4cv5U{Q|##?_tOE|X4`*XoWR8P$Hv}GCnLuD+4nuidUl<^)jzYv
z+VGpOa9dz^-mJ)<l@sSKRq8N^E-Mm~vkE%&_3}lTJ0gDX?F(mrC_K4%P0P1(k2G-~
zg}JK^`<mEsw!N<73XNsGDb5_TEBHKL!0*C%vAY5j5}pcq6fF8_<MMXnjejAgcRbaf
zJFQYVu}V4qziP*PF{Rjw_!kmtO$+&sJ<qXdJCi7&^iOuhk9FdK%4+Oo!Aj-vcTW6y
z;ilaw{%2dq-qh<8XK$$Xiu?9ZjpK=DjZ-<F&bb*omSjtw7jWS?RPW!NEn?60t@xeO
z)xi2BAxn{to&5pKbp=c3R=4cAYV)0Y8Rs@-9)`rpvK+^}zNlw+_)FQv$1+c!%p_Q!
zmKG{>r&s^LVG*S@u_@gFW!d64+B^>gw6uyU+*{pLYu^;G<#zwZFt+L=70Ys*D-6Re
z4J{QE9-ThJw@%SBS>Ld3@{YcJsXKT-Y>^h+STk3Q;a<{}zWM8;RFoO{D!BcW_}q2a
z(gegzwW7s7N~WC7kh~zuBsOhH<<2QR``8Z8`+MiT+4fk!4L_Y;2y6_!^176x@=u4`
zhj7LF_8fn&Yll<_tZeFBS<ondZTq_wN7lRiH*HAJmWa|(XL&l0ZAEcly3t>ec~7cm
ztlx7XclEQv)4UgsJzQKG@Wyh3=X&>LON39%f7iTQ{@Ioz3ldfquKoQp^i<*0?~DJv
zO?TW^v28c+1OHteKjuzlU^uy#-Nopk*`}YGKU^bbXP2wY-Mq)LTYc%%<tNYZ3N!f5
zDEFV=Y_?_Vry9p8GfWvYmb6<Gu3n-cTUXQYE%~4r8%N;Ns6AZwxUQ;MmrZ3{C}<P$
z{0>Xq5~B&rMErM4rX(G5+U%I9eEqrLNuitkn%o^<@0S>y70?PhcV4nk$i`?@-30rM
zPwi)hJLx8Q>ppYnvTki;*cc)kewF{G$cYKRInOtrjeGRrnW4f7{<sZi8;bSqGt%|8
zKK?u_ywdsaziJ+{>?6~*+O!><@aWh}6&t^k9h;@@o$ELC|9Hprf^m;P4bw{H>vDfK
zEZX^@<L&<alk8rE^-b6>A0t$q5q@M#f{cXa|Gf-+v#&&4z4PU!7H`A<qr2MsjSf6k
z{v-Y1{5j?6yr1+L7<kk(ot*<bot+f|g8bb)eOwtBD&|a`&K?nbQmo<mG!+$<Ee<S8
zTJE{L49&`5WLd&x<|O2yp{T+lXuWC0fvA9u39jL;IoXUU8x!@r6W;T4Yn<8@v0>pS
zHO>XA?^nOCe*XK}J^S=?b1a|V**V#oVa4%+Bad=Im>XmQ_o<kfOt4h6Z1t>S5pZA-
zS-|aM5wp@lK}}7_L2}ps;Cpu)t_7^Sb-MQNo&WK!laEg?VsuDSSpIRz9hnpCj0f)S
zNb<B}XpviCK6{#en@>w1Q!Xcuz^0vZjHkR&d~TxZu9fpBIdVcUW5B{9i%C2TSDq9a
za_^A5&>zK`!Y=5q(R7FB=5*}`9~A1_Y_!zX#W(FTO?kcL{hyRShtqFu@aI!co-uW&
zhKs6)hG=^6vj6j+{<uGBvaa;3!#rlI9E8NQ&qu~>Uvr1UGFWlG@KRS-=W9(hz9)ij
zoZjiJ!0j!u<^O@Hn-b14Ff&X!^_a!g!K@)i>r`Kt%$48ExSelINPK&-BGJw|&Qk2D
zXHhcy-^HypnFV(QidDLHJ~2z(w5FxOalSN5&9@2Xmnk{cOfvG@vesdS1ZyPkC)>=h
zm?I0$epX`C&N|_y)IIe$&wNX^OR9<+y`<0BxLZvTf2Jz1X)&9nQM`csGMR4QheaYA
zUpk$8;;rt&cl6|Ac2@1rhq`_)YB~60qgP>F!kYzu!wSpJEU>+H<~-lj*S?C|y|^E>
z9sBm|@X=*w#MvMHnNg9Nv!hH+-NsFlIgeY3k(-54ZNkB6Y(<W00^7csd^r$*YM#IW
zMxhCeKMvFd7@H`0UQ}+nZV|xpujvT;rH6%$U2%3A4ZrjcM?JkNw?w;&TSR%2gZL8}
z0me-a^JgYa$*9=o@_{9*#s2KM){SY0)!zU1JRTx{NwB%o@rXeZ$3?d5$7>7oZG__f
zCKeyEEjYh$$G)4wyExvqxVOA+;XCwp<)NgL{1c52T3ldE=8-(3_xAUj0`C`BMfs1^
zE~tMYo&Mn00lD%<W7Y?C1$p5s{@R7zEI4*Q%ko9{8wa<O$|wG%yp~rE*c@6c`%P*6
zmG4^vj!iX-(Y;o1zQ1zAzSZ*l_mu0l91%-m->~@x=Np@AJ*Et5WjxtO^b*~l*z7Tz
zr*ZArg`ACD6`K~wKlJ}x$o70nxbLfg=NI07sNBH+<TLl@r~2I5;Wn{KX9S)YGe5{X
zo4(2Q&c($u{P_9`#HYUhdur`dUgwR=m3z4FmsB2S{Gii(swdxy?@lvcIio}Uci;4H
zQhL9aGRR~;KBdCyC9Ajp#Q(r64`(l8?|E=9mDjk4Der*vkLCmczD15U9&Eu0b6E@K
zdL1w^5I)1abb*WxtM<XD4T3q0)(7%7$eT2z6)??V6nB)~!T0Xa)=Qjm4RIg%z9@4z
zIc0FjUUcpe<UYbz=wu@huOiXuk~A?zMf9YDO9#))&NJ?(CVHjlZjxN-TIbHEtUlqh
zhjo#3Wk-qf`icG;Z9###R|K}Y9A6-p!64OmZ1LoaJSP0QPTm)ezR23bcDMD~V)YB%
zFG{yC*EW7jWC%VgF@rO$zcN8bN6P!)B|}a#Y3;+SHb~|0Ut^j7C3Fq*>w|WWT;}u{
z7gq0xJN$uVPWSVNn?G{Z=<Z`)f876J_{a7N{eM#(lqCe-aKy2gB{C^C8}tTvtx(EQ
zmQkA1_TEqZq3lGD6oIuJ?>^l1R!b8~7tro_-C5mvyz}jm<45d-I+wU_QvNw%k*7?M
z-O4Fj+O~9;^pvz-@%*)5(W<01kCq56nzTqMNHc6@(9M-aAy2zb1y8Np#CEgkjccFU
z=M$GtdY_0s>HFmL6Tc#R*U3q)9~T}8Hq!j8v0AfQW4Y#cjr@?%P~TA7(BrGQLZ7c%
z7j}1fmEYHeuiRIOU7dDnk>8?IE4o+SUU_`w_Lcb=CbLw}&Y$LVdb#ehtC##P$z8Bp
zaq+6#>Wx<`vWl;)y87zsx)pmv)K^EZU|-=Ms2{XG$bLax<X-pYHHj+^UV4ym=GuyD
z7HbNF7yD0M_i}~g+L?<Z7w!!Hyyuy*?xbl;!#0TduXf4qzUI01cu7y5Tl&SFUw(dB
z`{nW%zuZOJrkX`t2k(xl-d?u4Y<=IfzHYxoehdA!o)etQ$98N($_B}sJ*qu_d!&2(
zm3iD*dLHygm?WL=&{n?py6?)@>rY=#d-?9_-gUn{`&WvunZ9(p&w9uA`|Q@`ueOh?
zpY@OJ@8w^%%3Iy5`k(aaNbjtFBFlW`z%_+y32g_(n!VX-+uYg~wj~`lZBys1?vZf&
z=Qh(#yQfd8RQhsnZEx!_?Vf%~`}9`^%rkDDxq2q}Oq?+DWCP)uC(|Z}O|A>pzRYqt
z>GIxX-_L5K$)&wa(>@b*rs{0(+2b?Mo2)n9pX|8p!v=wfHJdE5Ki<1^%IIWKV`}qL
zW9?Y&ZJ`rG=U%nVp0#y;>Ace3ucp$AdfR=covoT}7JcQ~)v$!<-0GWrTZ=af-&(sp
zTi#XTsil|6s<bP<)vIb7Y8z|k{yluL=i<c0-R|Z+b(Y6!d``_sO+MH6^VNe_6W(sP
z`f%OiX~{d1HzrShoUI<MzAkv{Wz+Oo#T}nzp1VB{*DKSjna-=XJ?7=Q$1&nNw(Y1Z
zEH0nC<89&VGV^l(()j0fmz{4bA6tCr@f`Jo#utkF(l<YM*Ee5Zv;J>fef90?^vd^N
z>ddZX>D0fuwz0p~r`E!5cCA&-%zZ2O&91Tj<MYR&&a|%o&)Gkf|8D=8&vt_)h3ySj
zh2#qX4Ne=8AM>6bopsbou7$OgZ7S<lwro~)mi4@K0!dPn1cSscNxzb^QvOmQp{S#j
z6M4gTo8s)#>r%hB?QTEMWiRU3+tD2%{ZZqRS&-086FymSyS$$}ssqIW7p-W^5IEZx
zCs%8KOiNAwS^vi)9}OR`K3;y*>e#s+TWR<H*;3n$-^JD>{F(G)H|I6ZO)b-0z8>-H
z+}inma?>QO$yRDTGYzs29ZmV0l30|?rKxI|8trxe$b(L8^)kPrxgl0BkEN_W**f{W
z|Hb)}Y@eP8`MPq>O7<Ymm7XiDgYAnKS(?t-XHs@+*OoZF?%iF#o1UMKKKT0C?+@BP
z{V&&E(!M-D^+$@%`4?h8I9IXQI!!BB`yn(f;@Pn?-y<(Z-rmf%Y4WD;&sIEH@=Qi7
zvvZTn@}k}%?&8}=%N_+4y3Y&0cWO_ZXG_X#k%@lO%${W*?tkNLlzmfwQ#<QE)~l@Y
z%T)cW16D+8Xzek%TlQ<wOaE@CuDaQc=O3KQn3-u*eD6&AW~Q{<dA(=+X5TeBp5u3N
z&bgfni_6a}K6853_w<d<g{dEJ|MdJk+l$Bh*)*Gfs$Zt>3fdav`SL95J#I6-BjQ}*
z+w_X!wxnIWZWMR3wW$5ms!!)d?{snRxB9p1*Y(DuFT10s@6f%uVakq>Enkiq?N7DV
zGS%+ZlGm5tzNe10*!5=DcJ1ZsY~r5=Z7mc{othJR{MCu8z1e53>%@MpT$}Lr>#GB=
z_P*x1rW^IW&Nlz=RNjBm-*QauMC|)`S?%Gn=-JNZ*RMxh3&~Z?)4XwMt7+luoA+(%
zR^MH=e{Id}>2DXkm9<VPpR=lT{o1>k8;U(;qtsWwzGs>le*4{myS>$KzUSOKeIt4w
z|C`BgtN&HMdT;*C{Y}2*gTjRKj@`|#`7ZKLy{i26^6P487GBxjlf7@-5A$y~sWg}U
zmHb8i_H5r-tNUNbL>5gc+VbSZfsKojYmXm2UhS7Mr{a#$)~D}Ia;mSND^<VhYsrz9
z#pzDx=GaIVe7j*ezxeF)z31fjJt$oGaMA6fzr5E?=llEe*Yky8+pqmP{cgJSZJXP(
zZXMqyzxPr3(<4_GtWICIaeeF_xzfFV-469eo2%b_v*YOQuW!n}y$vttdiUl2-0J><
z^>6ka`@4#rjX&#O>gV-)W%Fd+<=AZZe9?G!@$T_E$MxrV%&&Q;^KwyXsI6&T)bBGd
zBa6#D%5Q(}JZ~M}5&z};lz&TqUf(tSnm+%(xBtH{y~nwC?s}E=Y;oaxE>>^;yY}?$
zY5QjEV}D<C@9Td3-wVDc9(V8WPq$aCt-59Z*7)V;SNdl0v-Y{w>@}ZYY@qPW!=lTd
zk;lx$R<_~#ZN>$vhx6Ob7&L^ZrW>~^ocg?^=|Fwb)1SVIc?>z<*%cD?7$$y@PG!y!
zo3QeO-hY<<@9Zl!e=d*~$=tPh@+EQ6c@Z4)Y;5-=m>ja09(1Q9EU5o`;K1~?1~+aq
z$ea(J$N5yrzBx}ydHJ?;(;vA1G|#wKsMWfELLCDG|Iy5lh>{3jAFJg2T)o7U{G?R9
zirfMQ5U{bYC`e4sPAySLN=?tqvsHS(d%u!GW{Ry+xT&v!Z-H}aMy5wqQEG6NUr2IQ
zcCuxPlD!?5O@&oOZb5EpNuokUZcbjYRfVk**j%f;Vk?lazLEl1NlCV?QiN}Sf^&XR
zs)C80iJpP3Yei<6k&+#kf=y9MnpKdC8`OxRlr&qVjFOT9D}DX)@^Za$W4-*MbbUih
zOG|wNBYh(y-J+B<-Qvo;lEez#ykcdL5fC$6Qj3#|G7CyF^YauyW+o=(mzLNnDRC(%
zC_oL*EGS8Kttf$80OEs<FD^;Z_bez$HmS%haIGlOH!?7=&^0j9HL}!4)?blZ;Oh(5
zomUJE#pL{4{ffi_eM3D1{oGs>9mOTCWeEGQ>L?DWEJ)Q4N-fSWElN&xElbTSQAW13
zAg8n#+0N49RFDwZ-8m^~`W3klo00Xnd-?{z^?-sgJu|letOKMPS!GHxTwOtFQ4Uy5
zO0s@xPHJvyUP-aOp`Ia%hd|2);5tzJ1Cjv;0kZKGxdqr&!@>)!7#yxvF8Rr&AWJ=6
zY?VOnwMxlP&P=faGm{LA&5aX{Omq#*%nfx-EX~bylZ;HvbS;d{jf@hFlT3}0(vXbu
z%quQQ%u7xM8C8*6pqH7MVr7tQYMf$XXr^moU}&Oil4_EwYhh?%uA5|(WRhrZ0K&#d
zM)((Hrf23Q<{-NYWK>FKidC9rvSp%)X_9WDNus%~Nt%I)u0@K0xvsHknx&ahvWZc$
zfdSYkP&`;U26);k8R;R31mq-^q~#ao+A8@bCM);{Bh-du=B5UhB!WWC(A3z%!r0W*
z)WXQbz`)oPp(rf1s5mn}4`isJfu0#yA|=_%Ex#x?vBXv>GdD3kRlguF9V`Kg7OQ~F
ziqxD4m(1MMJg80!3y6CV(ve8gh89o{ROA*|ITxiSmgE<O<mczuDuH~YV5Dbg0M2*{
zHlQrzT2W%<lb@Uj%6eeMsVR2gTnEnEPKoLIV6KfmKE)U&1!v@!=Ycgs@;4rH5K7^u
zI2M(Z=ND!B7x^b;r6!l)HwT+yxJf~&>6yhPMU|ceJcXvt29l^fGK)*{iz<=q4^AzF
z@F3pG$pi<Lf&#dxuu4n@g`%xeabj6&3OE84U{c8$iFxU%DYi<`3<(p@#FE&}O)Sk!
zO)QLb%}vwHbWPHXEp#nYObm3>lFSp$ElrY>EiKJaO)t(*D=AMbN_9+6%`350a?i{y
z0LQa}1|*GYqAD-XNChQ8149d4Ljzp{vk(JwD?>{wLknF4b1MS_B}kMj*yw|bVwjh0
zK!ugBuN88j*nkQ%D^F0G4K6Jx$jMACf&>pZ@dT$9LQJyJ$Ds~MKRE5>6V#5Z45HgP
zzo4=xGd-ikzdR4G^;i@_Oz_D}PR%REYd5kINU$K)Saw{{pbBzvv*WVS2UnY*DijhH
zpel@(7@7oWX@!Eqs3jzY?`Uw11{X;oK$7Co)HNDhB!vJ;ibqoy)q;x)(Kk=cOR-fd
zSF*R8dZk5!fq{W7$=lt9;Xep2*t>i(0|NtRfk$L91A~|<2s3&HseE8yU|=ut^mS!_
z#Kp!ZEcnw&+Kz!ifx*+oF{I+wo4e&Pp<m^Xf4nYR(0E$lPPB31LL-(dNA?PRKQp!3
zWQmTp@9C+`zn3bVE1I(G*}ZAXS47<V4)sQ9EmGq+)yf^SWBTvkkChfW<)zll>v!*W
zk6U-Y_}urJ_218apY#5?MvvQKNyQd{pVwYoTzoNWYt6r(&o4<Tg|Cj<swHifvmj`x
z*ComG6Qfp!c<FN}y4Z!S3{w5}?rwLDe8(L<U0vR-QM$=LE8AULofP*fv<UpP(b0?8
z!0^*fpi)t1Rfv}4>GLyP@)cW7JdcTu-C6LksiftEaAIJZsHzT^VoSw2E>W!(r-ceu
zR#8^Y9uXo!VyyyBHfw^GdKvB3P|@My>~`W%+!VGk$(6I)>CmDH8#G!3l(LOBBjldG
zdgWD7v7@AA!Um9_V#^6Dol|E(Dpf!>oCrGFCHnLC`~9BY-jmM>2#K|}xGJ^?eBZov
z>(YXUhhp|tg(6w}=GE2JU*6uH9$)`A)XMpZSNXd;3oAZ8Qc+j;PWOlqapL%>bYgF}
z()P(WLGIyDj8o8DsH~#X#lCVWcZ+~i%<i(@&(F^n|NN9%_xr7R*7bE~<?H`AZcIA*
z#LjKvVo4shHhy_OFE1{wsa)FH+DuGL0jsY@L`NIv-ZJ6rKkt!fHv8-)UMG%;^|H2A
zBJ%R`bLPx3kl^`#_kHMcKUqFG8;!XtMQf%^5iybS-DdCa<#p)TF*mDMk{(yYoH}aG
zoj%>YDfP6FxOjPdK!}EjNzRP`D~~4`*REYFd3VS1^!(CycOsh>f&G8}kL!XEqvB^i
zTcY09uD7<Z2v{Aa`%+=z<FD`U%d@k$w;y{yLD6|h@jT|8Ji09cObq7c=I7>EPX3r~
zZEfv0-_G~vr>9GcUHZ~Yq%;E-FjVced%Ts2l}kWaSoyoQU4Og=I6N3w*x8fQ)2Dw-
z-+WWYG4Kgj*4A0y-rimupurHZGUUsfo53*W1TS{$T@kq0?bv+n&{ZmPm5O@n{{BkL
z&*yLG?d4@=<C)+$5ft^0DrcK!haG#9baz+jQfHTS6Dtj;b$4^i$;n;Pge0B(d-v}B
z`1zCZ!Ry!BvPwm~Q@vcTuZv}rFv*xua-vwpb!V%<BgxkG_Rr7f*FT&2IP=;XN$avV
z53a3^UK6!d%YXjvs;{pCH>G$=9`V;OF`07W#0i=0@k_m?G99>i)6%`~?j5UpDW+Ql
zoNSc8YfUx0v+HO!tIX#!hDVn#d61Ahclr6-AEPgtWL<2U`r7=2LLHOZWX`QozN<f_
z7Twu5^K9BXiJ*dw7oY5?wf^QW;(IUWXjkd$qPv#UXSyo3RJes{i7wWfcJS~?Pb+2d
zn>qQ1FK#h;*|D4>#jCYRa{ctF=PH<HZ!wecH-37mc6v1^r*J5?fTPoiLoqRsL$O7`
z37kq5TR;h2z=@!=%RF`e0zsz^@e|A5PZufCW@)&Yav&#U;gyJgFHMineKf!G{>MY}
z6&GoAf1dGu!`8qAw^paM8}_WZl+d@oM{%h^nW*dz<8QV?>umqtv*4TG<7}vXsYgrX
zh=6giQCr4o;ncO(Jc|p8%Y({loWC5~?7x@y^}NX%7w=45A;_rWcKUKwG4pT#Wr?K;
zLgjo%pDsB6^ts5IM?SLzMY4Do9549DQmB1B;fj6Yge==BPIaamw(UOnH8DQ%RP*zI
zG_h8JN35H){!3{cj(4fw-Jz0fXtGq~&d)Nv@_##eOLubVKbKGZ9Qy9e4W6Z53opi~
zX88#Oc8dvY{b#W1x>nM)Mz7TGXJ%hK8`E%boye?7nRn*dXew(kEoqs@YQ1~8b=-CH
z!i*1649i|$_vqZ?<mJX8+Yy`D%Ckjgwvklufwd-m|744=<f+!%&un>eD8ozCaetxQ
zcK-Vgudh!GDo$T;(JnH2nR2_wLBXZ5_vY452Blv82``^TCOv1LT%_0&vRAODP(p8x
zdg0NMcXRe7R!%-7w>3L;%FO*swtSl=KUK3`>+|a_rHGxO&kj~B(Fx|<n!WRPyidM!
z<T4qqzotHH9=ofBlaGk@h@YEWm0=qwT{cN%>W2^0ox|SRx@q|CTge{Gw9IF-vDfue
z8p8R>H3_F&Uq(rmX14nrPs@MFoGW-GwQJe_uui4J_uZm5H7vFNe(w3P8LTrHT-P#+
z<iBsfb<B}1iZAu<Ue>UNMO);*EtR(~vN16`VDyZ8`tR?>i*(zkMn7$D>bbPl=+z;<
z{+!Dx225%`6CW<g;du6J+y1}+p7veq|2|KDaPa!si-}8)s6K!BS!4P_P^5XyvDclQ
zFxBBsPvi<IzNI1N{%6=k!_H-u*!LX|sb<kzZ#}uny?yFt<+MpVjI%8YSI*V^TBpnN
ze9^_apspE)%$@yfI!<5l-?;s9;v$2vTh@2x?Go@kuV^u?_}GjLhur0k`xQ@oYF)kU
z97h`KF<*t;ha5uLVJE-z#=Ly^ZCOpjuJ=iEyfZy+SoJ^olpnZamhsBzWkQyPoUBas
zOB%iguH)fn3i-|&{e1nh2QOAAAAQo(%k_of+gA;vb>`QWEYOSLc>I#r<$lH7MZT*R
z2Q{8#(GpdWpLJd~SbIxZLf$VsSDA0gb!+8rC1oY;`ysM=yVOFL*3;^f1K$1+{=LmN
zaH7kfw*P{gJ$*N_F5a+j<K&J)?-Yf7TwR-Xm=vg$oz%DZmhIViZmq|RjiL<8KXw;r
zrn|rTnd{>BQ?UKKc)$hId9&x|9eH_K=a}2)S4&*)A6r$RcxL+c!pDoWkIoiR{uyPo
zZT^jDJ9%{z5eaia_T?^T%8mxi6!J7aCS;r|{7wGO{j?|7SO5R3aV#or<=U>OIz_{K
zg-28iE;`R|Z1EP|{3k~x#%Fi^o#&25n>AVl)_*v?rC>kXjK2Iw^Id0q9+f(N&bA|d
z?|c)5uixMAyi?Alc_Zm_l;<0%ipAIFt5+SH?lS)$dn@k^9^L~RU-+)$mp=EkVd0V2
zT!Akhn;m&;(I;sj7k{poH@d_ABY&E(#_8=#zU*<G68Axt^^|DEi@K9Kx1~Q??#eHZ
zxHr3^^T+z<lbM#d{?Gln$@Eoc{NKNaH+$8(Jq~%JHt}#<aq*85o|l`Q%bKebm+$kr
zvc~exUu{Y4?b{;W$M+xPcy`3t<=gvnZ?)M>O#gaB{^0FY^N2BTTfyqxVg0i|b@sE2
zckg(P`*2-6%)DcBo}jsToI|7O%Jg%~-ye+4tN**VXLGpe9_eoftlmi!R@W=M>tftm
z&3&mpR59-lQxL<X_fhB6)r8E?IbYT8-dOl0Jji!JSp37P5B^75;*PA0bKS9FT68kA
zhtbWX-%*|`?zJ&J&FT1AU@>uX`Qj~`l|N2bz3}v*nXH#KqcGQ+!kX{FT`YPFGIYg*
z^xqzNV5@QZd_%0+{N6tTOF&JSBdddUh)Fqw-jRL2Y^tYS)s!8z?jE}*Gc8$gzf_{*
za0*k9;>#bBB6?djw_8Z8wx7OmcL+yrsQ8)beo=X=%+)6pJY%~t#j*IeURqGc{Vmdm
zkBO?p8ZQoLeEFzq-CpB{?B?CV#sAE_W}mtLu61+nd9y<8T^Yu=mdQE<a)>6py~#1v
zIa($ruju>1qugCe78Rn3Z$EW@k^XnNpV@tj=v)J76N{3pcB_9PyPS7>?wZEp5OpN-
z62s<$JlD)LHo7uutkKzQmpN1KyXJe}NxLHSPDz`uQ<fAxXS;jBimOboZ`gZWIjuA!
z{OHO3R$j|riAb7?{Z^6^5jnK9fBDn)%>BjBBXqL9t3_zm7U(R0{%+ZiHOG$~t@mJ_
z^es>2=?%3<<+6&u?(%p1+AXD;rgzAXHFM>Hf&zy|(`$KJ7PWIt4t((sGJE{?-g-|h
zpO4!;tiBnW@B203#QJTC)1Rg<u<0}1^fO1~-0$yaPNiljwp4`XtYsA5^?vz+-QS;v
z-sNA-)Hh8aW%9l(Ndx9*8$x5^=XoqV=Ue#ht7?{7f|to+jkjG~28=7i-#gi|zjIab
zmVT!5|7QREPb;5J_4*#?sib@Rn^4F7YQE)qb|SNbcpv}#yh1yC$&VwKB>MHwY+?Fx
z@Bht3ml~Y@emEq#l<U$WenI2p^k05qPLH3>`FD5j&+L;kGSBn&zw`g>ne^?>0++>M
zQ{Lpe{J$(AnY`ZX<~xfWcOHE^^mR$5tifAr=Y#v^Zm3+W`%2y~fknu?eL~o3W7p-p
z2Ws-THE*O@o4u1h%2hq*nD4=xPmQiKxdj$DPI&s9UA$;}Ge4`{%-*y1UYC^kh3l1i
zog%kI@&qKRbsj0TpHPr*IPH+#?DGxEnVl)WBTv6B3<TG^s)o|O3pV}#E`Klky1}AD
zPrE&DOe=J_S|G8seXCQU@zqcNQj900{50Cc^!gj~M)N-hdJo?9^k9kp&ZXjdk8$?(
z4XZOUL!-}&CLQ+JbKPx?r837X#l<;(Y0E?wKizBi@#c2%++#ax*)`R&bGl}B%rlzz
zBSSSSU3k~j_Wyr_HeQUlWfZUa(aWgo)7Qd+;HE_;S)vi4I(>FVlBHj_7k>Qi5V%Cn
z^;uolwR;Ji%xaJK*G|<`(0J3epk!^QWoyr4FYk>k9<8~8YxZa{PuNt#tf`iLW!uFg
zZ+B}pzZ2b9zgsiY{n~_r0{hP0+mbDIOuNml9FU<Ecwm3aQ{V9Gg5sG+R&5Zwd+Tk&
zzFB8)T$q@~61wn<tN*vJ1v=^Nif_VwT5UI8V<_`mP<s5BA?Mv!rFU*x^q;usIAc@x
zWyi*wGwh}b8P|IVZ~Dper}Ldx&bIlz?8md?7E77u-La6o=EU*w%f8vx(|29->biX+
z<u6;S?YG0n0yk6%PVsVo^D0!)@{75Trl7<c?S}zNh4^}JKX?(?U>$6KXx01O8QhkO
z3UxbIy5!1eiyd8+6FEt5*~3d~t*?o`P6+lnc(>J2<zJCVYqjv4+rN?@|5f8_{r7Nw
zW_97uHR@7pdClymdj2w&e16{k$i1Utt;Mc6r&nK;x10R$=xgVPjQNW$W^Gs(Hm&96
z!6l*93-)qeHc(}H_IgQZweZ=kymxH>h%aNBXJIkt$+z?M3=e+1EQ!&wS{Pup_-mPJ
zohye_yu8fo@<S_o`Lk3L9=lvxX1ii_xO4PpUM`dG2TLjh3(d|Jt_u=b>g;&t)g_A`
zVNR)UJ5K!(KX&Nejq4wdF3gh5(VBYfR9JH<EBm8sbC2J<FWvr2H}I})=f4n@8OjFL
z+YbIZmG(~V-)sBx(G^Yeq|dd!_!L$ydBk5qSGRY)n%|SZ*H5MVEcU&BQslE1li4?w
z)_-l9q8rwC7GHPE*b){IzPkJ8t$c|*DG$T&V;gH&F0ST_JoHc0xU4k$j=9;S`1g&$
zp6Ve>S1k&0nOyQp=au=a<oKn0eqXf~1Y}(Z`QlpnT>g#C-3@D>bvm!D5_xe!vi()x
zow%w;3(X$q+zMJ!ra8q++r)0F;ho$52P(eYI6QwP=VgO8wzHnTUL|%|^GUSa(yhu_
zyQ5X+b)6~ObGJm(=lS+~r%Ip4B$~~B+Qz5YQo(jF@2bSY(8mv-d@140=w6`pvgzlc
zpYGx>!bBWnFS1^4{rvrGk_Ojw=chc!xWnh2pMJPF$3JqJ%#j#H-s%hgqnEjdE;CYz
zJuV{qzhmx3_t2<r%|~Hkwc&hIW6$i*)#wVSc;^zdY`^oJqyOKXJ^TO8qZipzBexb#
z=6R%^U?O!fBD#28=HBN97v2QkQl0S4LGg`yA8)JesY_9pl=2!EajbH>7*RUU^Wma;
zA<H`6Q<qu|ZeLmOqr=LDSI+g-<eu&BHEi>~ZQl6unQq_yxYqe*IcG|r$GnRaaq0;7
zm2Z9Nbmj5h>oTo|-QQ;|UZ2oYSGW6%&28`Q9p9H$)!VuUANhIw&P)~`LC%|B?uvT*
zy0=f8!u6{4(u$@Pb6w(nCKpbXo*~cI)o*HchI7eXY2jxYKHIwkE-dEI3J#rPwcm5q
zMZVop`wD(4tn|0KC-+gZ;@*5y@UX&>(-R729?9N#<K`WOtgb7eXX+Okcdb}27L?Io
z|LyDSCdFwjw%(oh$|8Qb9a<#d#KAb@aHp)a!<sKH|3p7(Z$JJ|>+c7q6ImIcZWpQj
zubn+tK;BPJF`I2T^WY3MfB&<-K1KWHMa{aY^Z2jl@o2reU7J8%y9&3Zr>8A_et!Rg
z5RFec5^MoaHa_+?o-KKEQ!md0^(8?o1GJ`w@N-Tr{MGU}_sZ<t;-{A3GtQ>{yz0QA
z=(0^KWW|AZcXw;X@7q(-c4A{2pKRCe-PX?Sd{fU=oO?14G!pXPbM@6l*IzGHcIgun
z6?L7c<hn!|)a(F_r?j*{hB;JpxIVJv-QA^{7zhf9j&M;iVIiR<$}W9s#@*n4_nt!=
zG>p5I)Qu&0RMgdj7dSA!<N*h=iVj!d{uL`UxWx5zWFei@7FWeheBDKyQ6RH8IJ=!Z
zBBp@4M^1+p2?&XS$Av)SJP5lLA=YA7Dj+20swX5XtBX*S;yaI97u+8PS)~efqDO>?
zl6PTYVMKIva&ofrT)1Q2sdq=C`?yHU+S*!3Sa_*2EDT)4;zhhvrFz|%7EjnP!yhu>
zlk)V+%HWvYWxRXi`WG)&K6vn;h6tCCkdTCBkxE%v*(Fb>LrHcE3}&BAd&#EQBEVnk
zs3CIW)~!Wht3AEF52qL<nn)Q+^@4_tf|h!Ds&|Ix-Pn-0W7jUDoEru&d&FJzoVgWS
zPFz~x*zDoysc2^>w>QqesL04|@x)J0PkRT5Shk(f@dA%ab+NFpEV!8QuG)8P*yXj+
z<}YO?CZD(r@mGcGojZ3l;`jZjU4Qe&jSCx--NCYFAVWdxFC3Sz?+IId_Qz(grHjJW
zMqM&=Is}nox>xmD*Jyon&fQ(6T&+%4ZcjMWCUeg9b3fR`y3`mPW)-42xw$Xz?X}jL
z%Eivk4$|;aWMZ<5yZhreZ+e0>3&DA(BV0{EfuYsOv8c$%cb3USK0(X2n5H_*B^uiW
zoSqmN8XC%27PU-|RJ)#Gl9Ze4>)~-i+qP_~muoW{?}n{gr#336K>fL(_V>4oi{1HS
zVq@jCvOg>R{Pl}TKt|?|SO5QacXz9(sB9>E8>JP#PKQ}Z-pj+|!KJ0%BAP)hv(59p
zAgPS$K#rMl<)@UU=H{Q@Zs%v-+LF2NZTI{A|K;rM?bFW8a9kg^H>Kc-absg+&DX2p
zH*VcZGLYD?Y11N?PN5Z7vySyh9)57JdFf=3?_K^C{r;9)_4So&O^ppZpUeW6PNAQ>
z?0<cG>%BH=>x-+a#eaW$t88H*AuTN}sqwRA{(SkqzP=YFRt!9B%;x#`Tvmn%iHobt
zeqlPnB(B(U;-HhW^T!V#4xByPJHP&)<$d;})pzdPu_$=JP*_;_<JYgEkB?kmPN`WG
zx3@|~Rn_*(uC;4*-|zdqu4DqJs5l{iCJp55k~cRdE?A(T<};%qN>{wuX!G{%&wsz)
zpZ?%L<CV41)92Uy^0ZQw->`Xe^^3aD?Cfkdez}@>#>$U>PVBdF;+S~&$gyLWc9-WT
zB_-Y1UvIzEYw9GwpZv^hJUiZR`f_Hbar~=C@*h8K&Ai;UIsJT+(M*N_t*JGiPO4ur
z)bBW>!=>oLSM=jUqK%DB#ow>jZ|*E!&T{m4$mTR(Cuir*uC7IXbFKb-y&iu_vohlI
zv$Mu&f>XUz*G6w&R`Kx>$b>_GjJQiV6<bbRGrwQsoRFYU{OruZ`}gCmo@nYuY-l(+
zS=~7Qo{d%cJ000c?2<feGP1Iso}LG<U7L2eoj-W78}CaAP`NYF{O;cB>2|fh3LYQh
zou(V@Hq$6|N$|XjpyAl}_x67L{{8vv{Cx*+-HNhO?f>}g+p`-Rldr6co$cJt=h?o;
zp-3`{LosgQV)y<>4<9<N3|e|)YxZ=OqtA~VKHPlaLcqG%-N&T!_Z*z69lj*^(+soR
zTMPzycPt=9-b8a&R@R5_-=CkN8T{kd>-DMMURxFy6(t?*5-oXqOH^K7-YoBq#@tVH
zT3cHKmwJhca2@sjpa15@MusgpH$Sy*xxc$SUr0#kiHb6KxMXQ}clYA@|Nmn4)y$Ny
z|1+`Z=_$|UdlHWK$v%F&{r)oNc0QlkX1v<k+ApW1i^#~FY2g&+;Nf|)cKf|k!N;df
znQ~<L{JK;7YJW#;%@U1^i~IR2=Jc6G0#18e)cxihxO{o?p;m5TAt5ElkE$y}SBouN
zxG-i<g`mB?y^@mBlHhqkIXOCSZ-dj1x{8X*c~;jAB0u=}_|op|DBSb&+3cFnXU(5H
z-wCczKTbL`)3{wsH_D*yPsK9-`Fhjm#v7%lr-!YN^W9zc_QSVtX|J!Xy`<@%@&Di7
z9lLi2Z_9~va&lS`u+V9q-Mv0p>rMIh?XInfbY2~{wxh4_-1O(Mpz`ksx0t;A`2&s2
zoc#RhH#enL{^K??w<vnTQTyx5!szXJJ~NG6=k<N&P@CM@DXhLJ;ULqqb924B#dJSi
z{V3h*rsg;2M3<;`#I_vCUTO1l;m7rjyPY^bGPSn0K79GIbMIc;>Tho*E|}r`<JT`C
zDXCK{gO_JqSim@O;>1guH#>TImIN+#`}6H~eoI^1v)v0g7KE*qw6C+dU-w&gZ}s<Q
zpU>ML&oF@$!8!Z>ev6)EmV4^O#l@U_eAC=7JT+kD7E8Iit8`1w%}G&Pvp$~I-|w;9
zCgJn5vlW$<7ngVno8{m0*_d?nQ)#_P)|HNCcK)J2KMMc+`Lm<yYt}xCJn-QA`I&L+
z(?2{o*wWhi@o~TXqgSuA=1w{+Yf~|y?(eUQyUX*{{N{9cEj@MpZ=!+3j~|cwZ|*A9
z_MWD5@cQ*|Hcz#*%`+c7NLc1S|J<FO#h?F7Zc0fD<WQWq`0{f9$4{TOPMs?H``g>e
z3ubtO3I}F(zJf0=0;SCJ&YYcX{y8^(j(xqHyu5tgy*(!vI=8E6q&Y4O5D*YJ(8eqM
z<IU#t5;heEo}QlIHo%|lN4v!@uLx9j>ybEkXJ_%oty`!1opj|A*E@5pS32{?2E}>v
z=DoSM*E+s0(ca#kjaTZ(rqt5~MMaYuKg!$K*jQA2P_UZY7qvC3@Yk1@HkET$6@Ho{
zqu5fR*4f#~BV#dPYxea`iHF&wOtVzxPV&|i;d*s-b$fd|``=$*mG$)c*m$KD1ZaHv
zQuw;W>esior;qhYM{duPJv-Zc`oRw~Or&_<-r9Qj+_}E(_v@tD+1dB(-TV29;VDpN
zuxT!Se(vGhx4kD*9&Nv0x47u(sY{xb4u5}rwJdnR@caAw<(Ze4-Pu*DeO~VM-sh9E
zw?=(?f8T$qR%i-?O>5lVs*Tm(^Tc!_9G3gd-BJC09k?)lKTR*z>+i3xl4dy)pwt3t
z4ro?-TwLUOb5kmJ>8mRT4<Bx}t^W38<zwU7XF++mzP^6nuUFb>XJ@tc_V&*Cksh!y
z$@Sa2yWSg<juw=cKYwtrx$@DDCq|&zhlNX)sI>9Pp8E9kG}w=`Oeg!DTr9%H`t|ko
z<yBu_eR+FZd~aO;{Q2?g_Up)sin=cMo6Et?o&5RP*&jcCoH$t*chGC;qRP+Drs&7d
zv#b3zVa^<%y;Wa}zRs8ko{L}-*NZuD{5bpGxbuz7>?^|7&MNV;yRafq`P=*Z=ilAk
zoq2JQ>$<u(x7Wq)E+{WQe)w?nT<h|nT_u^82lX9Rhppw{<xPElZmvn@r50)PJkX>>
z$DX{SU7~wF9+T#gG-^58Ex!0_R{X7Plk?{!)FlXfZf=UGQ+V{y{K(wL#}_CxH8DjN
zKdIXG)<x$2xho9}*XHzAKTv0LI`;F^!&#pbb~&GT#mZ75b=>mrXZ@qqe@{d)F*nb!
zuebXwp098z;ilW2*1IgMtb!6-wEmh3o1c59RB$b<gnvt6_W$yleSe;=v(b&)V{v<1
z?%~Uqh57mU>wdr89%EV;931@f%VqzU*VoH;i|IZ(t-t@w_pd7P$7bBzS8H8YS65M0
zwJG<u8MuBC2TkVjNFJIVUw3kvZuE=$`~Q32XP+%Jk4GwEzVdzDlvlq`cbeS&|Ji=}
zm-JWV_X}@rG>onBPP^?l^OR`fdAlz*5=>V`zRER}bFv3mey%E1RTQ89{IVv`9Ge0*
z=73Avc}q>%XWCYqiAHT(IOSMT<++nr#aFj>7p}WLA!wFJUvA8)?eX=ePEF=qA(x?D
za<XPly#AL<=bOEjCcU|_5mXSbi`{Kh`6;F7@Mc}T_<f+*wW$4N!p<+Z#Ic#})4{q(
zNAHy>PfkqKiQhM;{C@3pe)~TYrc4n5`z}0YR|#i)e0)+;5(h8uQny~I^*Qk&QPFQi
zxy~GWnqx8LW#qAw;*mQ&_#3YM`**94Z7<(kzyI&KZwXCVz2Z>RTDE4ny$|DFg#@0h
zH?Y0GfyH;ak{<tY+qrym?TWoj6Ac*En636cY&3uU{&t>=hk-3KUs9!O^Zf(*AKVVU
zlAW2xF05C0C5vP3-YzDotgY{lpO`p5$?BuS?ZWiPvNddL?{@FoRD5wwq%oh2#er*U
zqcg9r5;Zh5v@GNK|K+m(j~_oS>?}^dwl3Ctp;POV=D%E5RtC3&%GRf+L_u@1N4v#A
zy$qMU?Nwj1UR_z~yua>m#I6!eK0dznW$ip}Hg>;fo&S^jBk!61o!!}bN#C=k9Cp7P
zxzJ_4VWObF$AyA>b9c<Bocudr@nw;7FLjI)x|q0_ma{#0_OyF_xbMDuWwT~%-Yt3k
z;#^1Xe>D}mm(5<oC3Eyj<>wcVU!ObDyL@M9jqzctsz;%hd;i7Ota$y}^+TL@1MmHB
zx{S%qGru_6-m#B3wKaQT&EJDTXEZe~@OQ!10yx$f<%*j7HtdylzPI|=LP>68z0&`C
zvt?P2ZBr=rF-^P3yS*>}c~`x8jOqHagAScSsz07ik6#hBHOsB=*&G!^!-<T{Y#)w_
z$8X5Itah*Zz3tC{{YmG}om&&Jv1woJ@1X5@v2{NlvV)U=^wK3uKHRH*Z&>ui!z}O4
zi3N_$MJMJhowPvHbG6)I4{Q6$x$5tkF0PSSyg1(4-^f1aP1Ok<yB`PVMh4qI*}QV~
zk6pjmcGYibtS&AIdsk~&Yt{4KKKmTM^8N?wlfGu$Tw_$YLjG3Pk@LkICRd)${eJ)9
z=JT&Mz3SHQSaoMY73-B1vwafo{W1vSt#5Xft~)MJJi%`<-^K5X%YS4X$>`bd_jq{+
z7tj0)JfTw!&-~pYef08r&Ffje`Ifv*y}6I$0pILncNdgjzAiU)2It>G+rQRVl+^F!
zE)QEh)1pwxIPJ^=NV7@TJ!YC+<tLT6xHuJcbzv!~uHCzL*K8=B(8$aV%IVAe=6b!q
zx7X9d1JofE*m?Bn>FFzjm-CtD-FfifU~@%9#gjF%J_bP>(zMw$LRi+`@prjVrnmR9
zbhtQc=S-%nFXu|H{IpSgnyQ7?o!NVOU%p6G;+ar%{fxV=Eu-kxT|CEb*K}_FuktwO
z{ghM-?%O}PMdt|pEPh{?6k}yrdoANsmTutL-EpBN4Y%aZTo>-xSHsre?ap8DX3?wz
z2VQQ`$@kCla;nOn*%Ws2mQQZ(>1h{lTS$M?Uii;;<;vP4If=hiUo5-I^=!rEPtRuO
zZ>s-acWrI7`)sq^j^5td)Za$c-*RS|WFG1eRQ~a9_xlaow@>FU<9qV#*|C3re{bBh
z>CoZB&BE$_;Iey8-s5AvKYsnXwAh`0ntpuW;>F2*w~wa;H6AMY6`vopy=PJA`d*%Q
zalAY;^>3`&sJ-mbp?TbQc4uE$nNrBG=F<N;NqfH6iGJf^y|!Z0o+sRDLc(XJXob&y
z_V3@1-{#9J7kjB3S#~G;#r^GCYdlS*0#-<0I(J|G|F^dvzvue}HuqiWS)00SVdDOY
zGlHLA+WOq$-mBNYPux5deSCZI(n9l?t(NQ4VnL1coyF>+S|OlTt+IRHo-1$f?<{Wr
z_V)JXijPT7PEIfG?cH6oqS(UP+B)t0ytC8e>pV|S)17HoTh$k1pqd!SG4cJ<Wy?N2
zY?n8x`B89fZS-;L_j@KAmxp9~KlxVtqS1)=*2DDyi(cBw=)1G7(4F4D^;(w0(d^J$
z34d%`_FsM<{3rW-)EcX;%jU41nSRN`Fe)kXQ}5?Dp{2_ML)mlIivAIm<!|Vnx>dF8
zRz_idNOx?qUCQ-V37@vvYq%s1&rDgWS`ih!Eju-L=H4RBAhzOD*ICy66lP{SwEbWI
zO!nl!CF$PHj_y<5-EYW@Uuii1XGTC+-E_8f{O22HK1vP@yRal)W_wK2yNh?{&WZYP
z^wg;?&ERDpzFv>Nye&66?aYjiLEEYy9AM14w#IX&Q7Q)~=R&7euJiB3r#uhbcCw`X
z@6Bn+RgtN)d1T^jyAS;QJX2oe$g-YVQ^9SyoQ{r;XQFB?YQOz{IxY9rm6gK3zr9Vq
zx+?U?@88dNSj_{q{(jy&(8z3A{ETPe!i5!;l}}I9njD>%d~e2o!J<5=`)lmD%tR}z
zE5!G`dDPsMVOC>2+xD4>Q`ou0vmaki$uVrIl#jT0_`R{%`S}N?tKUz#o&J^4blrQi
zq_Fi2&-D2o-Sy|&kny`<&G~m}FZJCGbY>r$Wq)o%>e)08>r<!X%O2dxV|ia0pIUL8
z?aG&|V|Ulhe;~QHUw8lV55M*8i=Vs^G|Ru)tXaceJ=u1~_dx5g{`alZXN%wU-(k3G
zx%AR3lOrYXw55KWQ+&2>#dDv}ad$KCtYvMLw|F*V^X{33>59uf$m&GwogZ<Rb)QXC
z+q68_;y*J!egB^R{@z}ls4X3u!OQk+c)MijQp37G6}w7aIt2s>1TXhHdh6D&oq3OU
zPRR|j4Bl3eW%hJNT-Z%D-mQP{t7M*wdG_x5qv;#hYh_(4e!SaNJoDN6ZnnePnzuu?
z<wTafz2&+!>uN!HIjGZhyeGZY(p;oN$f-kog<~^YkCbWGudlD4?@Tt9{PKeL!HX4V
zKb5Xs{B2|U%P&>3E4F5su3CO;o#T4;wPI&4%kx&v6<oVm?fRGhhg!Ko5yUN~Q}Hf0
zet+FuIoqlQAzG0;3KWlai`N&jPpX-hcrW&5vS0FvKk*V_>qCCM7XNu>xA5&VnI^Ko
zg^hRbNVU3e#iJH_Ou~aN-hHiJ^^F3P8Afd`S8n#~ZTV1o<=v*KrtM<Es(;IvoBi4y
zmmUpVeM*9xZ88@xTgk}>E6$pz@l=(}JQnA2;{ONvhe!4aHy1GGy`Aqnx%`=N=IO*+
zQ@5W<yU+DgJ~?mOlil<FzpF3$;k9S?@`4{O@<wijcjQYx%s%>LareqkOyQ5)TUrcK
zP6+(^^73#Cr*OdPtLsZ-z&+b1iCVqi7A~0a^Q-#KIjgT)*@Xq1$=f|EovVvsRg9=e
z{1V;hZ4X|)^n84*_vWV5)7`vXA7W0KNitsgcIn`1tGLqIm64BYzr}f1yiA?lbuH63
zy>-&Z(|ieD_wTRW%zyFw+28Vae`=jJGtIj*1JtH~_t#!&%jSb}f>TG#p#%eioErvh
zywb<~?f)K8pI_4y^wVhHsrh?zg1vqPH`~P*He62JEfM!xLB8*Pmv89)>CxLfWBc9-
z-<q^3jj=uX?2PsARc|a+-x?z)$`GG-bJ?L*?#*?7t8Q(}?cToKJp1~(so)XQjyI~R
zst;ekKK=Feb<2tm3j6l$(=TVAw=w7P))0rA0{vco3$*-R*Znn=wLQq&{MB3kcF4+`
z(G|1xH>Q0z?8~`#sG<Coebu$!qEm8%G(>7X9AsY;zki;;{og4U7rXDDv~l^g{`mq<
z91}mgy0{1k3x7T}Ph|4WoXd;)-7jWkyNb2%<8Ax!JIU4U=CXNt89T$bX)5i_nS8IP
zHk{edwX4}^Dw7mzU}T<Q@y%tMUamh{tbBWu@2;LrzTcbTuRN8O%)M_eo$>xy_4}&}
z-?!BLS@rJZxpKL^2~R_(<$n6|CFSHK)jj|J{q~t<@^Wc@e41FRfYN-AeIIu(WjXcd
zC(GS%t0$SMT9uz}e94jhuQ*#z{RC$vmzkN_`n1F67v=;@y1v?L!|70Q!aSqx+G2w}
zlQ#Lj%dKX*kWwPOY1d=p<x_L3QlH0xhM=D;UJ!VwT~D{$p!!=*&g;FdyWgCeW3$ai
z)&J7|09{qDf6jMU#nyHo-r5o7=b`s;&ndH;(~Fj0e_em!?_`H(>~#(We>BSYKB_$v
zothgoea+b;$F^qPUGT~I_?+3N&GvExS-EpeeE;<6B};x!{l=j5t94Z&rKdjW@=vq*
zX>R)B;>(jorKUD9i&%J$9Fr<2Dm?R}YoDA^=e6B?EX<as_UW$qYh!!<lI6pP54Ysq
zowZ@ZhC6~rn>BBzTzGA;VCVijXO%r}s#%o25@BZNdvaxEuzv0BqVtOcoSqnS-aIFG
zL|EvH>goJxN2fCD?0<1|m8$Z}M=KQ6a&(*3WR^Di8+G?Bxp;D`K%TFSh3R|mb=!Z=
ztJD?JRrNZPSz~l|VrT7X<LoPCS><a^%m{3csMMJ8>Xt~~pB?l5Ik!LQUNGbSN|UHf
zr^2MxJ{E2_TYK%}%x$YL+P(JK#<e#2pX^p}W?=X~&1R#BqFmeW9abg(R|vU_ZA*xL
zzdqyVWkt4d-@{86%R8z*GgvlLT~&W=f>qWP)mKI}PR4c756|B5-(8q}T%v4R?varH
zYF9jbAFk`{xw>R>w%C+&VoKr@!3*F%#H;!%@l3mP)ct%6>z0r&8;>pCVXS5HeX7mL
zm_<ukqK$*X>K%Ihf_Fr2R-V__RUbY%OXSjB$SS#s#w$03+sIs-lpmvVO3wS_<S*BC
zRQ(Tdu}A61om=>EE@za?k?rEQKL@V%a=7NdGl20`(pJu!n`Zp;%a*wGjMpr-ukz_?
z&!elAwLgNo6D=oH&lH;9NcLm?x_{!?CHoKUF<<^TVrT9&RsU0&A~D>*CvS__TUdQs
z@5~%)qsKG%&h(xqVLVlXlmD*Yj9at$#pfq@B};$%BX??3vgxO{S(pC!g*|$n2Z>AJ
zWjBgh{JGc?-kwO5Dbwwe?aa9=)wJuIgp+mo0o6Pe+dp>6KdisySbtZ$z3eKxk4w(&
z<IlKv-<Wf6+Mz6l)zfxu{54%!qVH~!@9Cc=>n{ofKL|JZu=mXO`}@whueE>N{V~*{
zV6W-hE$TmSEN2egTEuC!Un}Bdvm2M{w{17CbDzmm-RHRc@PdnvW^9y-ay9*b@J{ut
zVo=A;-D2Z=m#trp8b_`<vj2Nbx@0pr)>C|+2DS31uYG!$J#N0$*Zcp!OB*!XS4VpF
zf7$=NRknPRZvFPe*B=W-%+;Et{B-A^nZN2*PxL4D$439k%KPLqX9q5N5)u(zV#Pf7
zmh~c|7eCj|?2TzU_k3QBY8_X*xPQXat%oe)e>Mj5)lIn)wy=Kt^_vzCg(ExlL9;$b
zv^l5pPm%NNusIp{*vg52+riySWXsq6IoQ@c_2+{9hn?%c{}WP`l6h~s<l-ZZSnW*{
z-+KKyaEWJ5Wizk3`p2p>(joa5`WauVKYEwtT(tSrCZYeTkks}^{dd;fNI9EnC+=U{
zRPj`7-J@gT!By!$J4_Vk#eCEL`2X5b$6JAmlFn+xa6Rs2TFw+TVd@Y4si0BumI@YK
zQ|Z!$X^ZtPi9V9FIp*~TRAg=!vS+$2)H~z)>3Iu#j~niZ=khz8_9*U%P^{cmq57jo
z7V>0np5qz)E%8^Ot!*361n&#}p5T(OeP>~relRoJh5hvmjEu^WdkRfIr{}tUm)V}!
zWqW*217pdX8~vH5zuRBg=2<?G_p`b|!j)g{`PY{y=iJ_Y=2pqS$Zs9e>E(hyZXEw9
z|K8u?><Px=otcw1PMK_T)vKmdWV!wigW9hQQDr|{^4m-v-s|PtB=r9&cyQ>1p^r?Q
zNmHY7nPJw&GP{qu%)e&m{M#k_*l)l8`-bORXL_iay>L5P*<JbnlpNEtJ%@Q;Z$0xS
z$nvqS)^Q^dFT2AVz1RJ>l}_$RIXvlW&b+^}uL4Ww7q#=q+}_c&c6Qpvz^{_;?H~VM
z|9@YnEMJ_@q|7CAj5p0MQ{DGdX#f9x?Uu66Mz^x8vN+=_j%*AGd*mG<;?(g+?cJFy
zwsSX*PGH^r@@4dyPm8u$$R5jm^osqqAw$$AZT+M}2|)`FHt0#pY`tk;YyR#_T^IZN
zCFy&U0@jP&KQ5nsVAcJLr7LqHTaqp<HeOYz`A=f&2a}~Q48GW&+5i85f8?Kt_170~
zySV7*`3K7uhW)PF5xOxkw)}ANmn-#q?<=g1D_*cXwzJ+gMcGeYUv15N^^?C+t}K&2
z^CLsPYaVl|>OXH4&}hC=`;MXteGTbL7yceRoqAN#WE->J3l2`HbGwDbjq{7-vr1+i
zvYb0J`nb9An#JmIrgvZ5Kia-)UmVN(UX^DzqVC+eQ}91QbE8Ad4X-<Q?)+F>z}{w}
zGwtH8=Dkd-e}xvXBpteT>0+@>_H_1(xsUurROQmoPxfS;W3)G^$Li>nf6Ng(vLv0e
z!_+Iz$M@$vJy?C=-#x*e*|VkRF3(lec~ZY4^!<TllecfG-LJ|S9k$nOYUui%#vxf-
zX4Y{U6+AgU_08YU8tr~<#eZK_Zd)JGxye;GX45ZGC0mVqzgrJoG~9daQSDdBqn9Uu
zh7eB(KC+ar;#vCITIBSjqM6r@w!T)Jyh%@g(+172Tq|zgPtp0@WL1=TQJ^aC$J~RC
zb0fdr)T_$b^e6d!zzPFjwtDx&LNmLRW-CS7yyXer7T(@+ndgn=yg+vUM~_z=xc*AN
zDyME%^2N<%8S*i1?FKhiGd#Ry|6-d>&%s`;1(i)<aoL7Fq2>CQYrh)Sb($LFe)fu*
zy8Y+*{l%~29;QZqoyDH<LShf+>(JLfOOFN>tn;|c=ltS<)^|D3P<}_v$vz>|IQ86*
z^Y56=(EJl{V`q3=%E`Y^UGLfZv@B$0xc7C{BE~D<686n&U|ex4OJUi8x|Vw{yk^Q)
zXWV(#{X5E5=8xg31H2~Mxz2}XK91SMw_wxa)Z1CFm)lHvlKo~!$YlS!I>o(Hcj-Pj
zpXYx)WNY5y9|@twY1+#S9tk=M@=SZ~e|pvRJ<icx*ZwkR=-PhtItf~yc0w>@%Gx02
zOW%qfuChCQ|LxnZl{!ydFU4=1BF?zFI^XkTNn%)HFIRxbnuF$Rb0tl)wNC4WEe~9F
zKy=?jZL{g}=bwa$>eR<(@@7taVH754|J1moH_gNP*wYpD)0L%5HcHG}wzlPV%#IAD
z($gH*=J)=7|9x3}Q=abXxK{A+M`hs5Hrtm=0}PjC8;F<wXR~*{qIrz>ht%rp(dn^q
zE{nIZxMiO{s-&I$`%4Aqf``5L<|p3SHLYo_befH;YSM9)T{`;57FhLxX1fpD*PWfF
zop$C}=^Lvu*#_&4<_5EcjarYtxp8NE?Hld@=Y>x`|F1p0^9D=Oq55ZcHLEx8J8mEx
zT%&V}YnIRSJv=#oUr9^l+&}#3VZOn_<1tFUH~0U)c2d9_6t%x5xopn5rIj>)Dd%6;
zIv%d1e}8_S;pb8^FzApsZ!54g`+4%V(Ot!quS`r<y^rVI`?JE_K62%<nz%ks&~%fF
z-$KpfCl1fn`1w`6WX0<~|Hs!v9x91{{VMVOE61wcGiH>(e)vhrFmd+HPt{!NBKE~g
zqGZ~)y*Vf%cUnJv_QiEsj&D|*CM>u$Uu>(6ao$Nm`^&E$Pi5GWX(P<gdUf}oU+(6s
zUY@&9?!dh)JS1t~jJ8wDb=d#yaTQ;mllI!@V#}K7^Ya$3l-YX!#4)Gbip_zhY`MSM
zCg-+aetLbU#ueL0d8<$M&oz_Jy;G|==j8M7olZpTq`0|%epDaTaQhoHX>KrgQtVo>
zv}A#`_F5;;U%r3L>d(RNzXhz4Ep6@Y=}lh{Kk3N_XP+1=!O-e@tKGNkW0h}pZOM$9
zWWK9cIpJd2z4aLhuh+<Jk(_*FiEevl_&?5Dq0rIk^`Fkw@B8TKZR%;M)_OC??DPb2
zrzh#WQYHykRs>#IA8&vB&E%5{+}E2Ygk%>KUa|XQ%*;RKTxCeLT1fv+r7I~HCin18
zdu6SqCvShKG<n(Sy9`U3cCqY>Tc4VByX7E@)KsnHZ*J?&7hTP2s5mFbsr>I?SkS_h
zQ2kT8ZGPWrbzILZ?Ot^0(Dhbs@uFvEB*k<hKAg#IZEwH4t2F!2p+iN7I8;+&VwtYJ
z+W795Jzwq3Pln?9GcUaSlCfpxx!_N(ZI$JV&aDcl3Mp-P@|oxJJFibCL~E{J=Jlyc
zdhC-CD<glSjWae!CS$Hqn6-M#`o(3_*RHYLGDE9>Mxq~Id}sOhDPLHV|6e+OEa}u0
zHHnD(`}MB3ze||mzcBpT<K=#HlO7ysoMTn`YH5BnI1%wJxs*{@Qdp^>k@n|I;O~~V
zE9Yh0j_T#Q^!!CoiD1l*iizFd|CTp$#J*zAU2VkbdV2oW@2_m7gns-!)5?Bp$B9;M
z@lHQ8yXkhj^&2|GPstRwJZ&?~iTlB*x!S!z=hZ#SJtz0{$$`r2S(6PNOuDVA3^GnC
zUWiXg`+x9Ev{C=|JJtdM0teQ_)-$RJok<Ibu1hp{{kYR)_Qvhu#eYw5S(VA2e0A0M
zV^MMP&qD8)D#dR!mzBLawdd+;$=K7)jofvv|67iKxUizz`uLT;xij`wNiNQ=-0;Ms
zYNndzad1`U^k=(Q@2-HCGbhIsMty&;KX<14oV#1Z_TOvV6ZTbaugGT3Ti@98zkFuv
zGR}CG@@&hm7NMgt4Z8z==^p#G+WE*eL!Hk2W1al>9#yp({-_U}5NNFB@T&5SD3@RB
zpF`(12cD5Op1r@;J|Rq2R#(bD?d8GX3M;J_58gRV-sikh=ohm%+c(93bI<x@g>hAx
zCPd85^Qq6}K788duc4|-+onHW(SE#dom)Oz-g<9w_Vzy4ow`qFfG0j%1pE(Fv@SI+
z6aIJGe@)!pR^?|V=g!7V-nhagc9~uLj|`4mrEO1Mvs*m9*A(RNXKseU@(Das)<yA(
z=vk!A;tacC9r$LemCya1{leXBk7WLeYwX?VRqr48eObk9eZexrIkMtGA2p&6d}986
zJ>yC4(UmKHq+99y&SK`g`|5DpPvH#Hj|`cD7hk-enw<51-um~g(4PG9vey^FlJ3nv
zyI~ve$90!`UY=G@G|NqYyRYL*c}Awy4*7KE=fWZ~UgGuU4690I9RGg!FDPKqe_-Zr
z$)B;OduA<U$g?Y55xAJ`srQaM;%BZ%UUvw+p)H~jlCVCY;IrSMz2yN5dVc-86zH$0
zS7TfneQsgS_47AvHnKjm2=KI2;{<QC6EA+lzG}Ds!!E_&r4tOrzZsb-G?@0UYMe2*
zCT^0-_nn8kzB@_WIo&U$>J_}NB=gkuFX>_Fx~tvSKTrF$o2zR3^68cK+WEf&R(Ghr
zRdtz{vVNER{IA*W{|h(2n7QxJJsI({a>Y{o*BY*iRnP6pJbt5A=daz-8LQs}UOB!!
zsZ`2*Q`*O4_LsBEUs>$eo4L%?D=|4W(>3?=3-4q6)8e{pHV92!&A)!PM#nBGPCreT
ze?M1GNuC562vM@0wMbO*N1wXh|HAcQ2RE%>d~-9uZ6u4=T~WU{?;q!LBfKW<aTAQ3
zF-iN){O^+8m-IH2hf93_v{&<)&VKhL34XChHgLY~G-6y8wC87HF58}9`()P5|9(}T
zS=s&K#tzHg*S{Y<dv>htusg>r6WNBTo11TxDyk(%cQxHR6K^+-)p?eXMZFy(<Lnt5
z%Y;I=n3ep<QmsAYezEwEO<JGK!MFPL2@Shv>Smic-wF)-x1o8({~xR>&)(#vor&zw
ztYPO4d#`?trFGTTBPDfGrP(|w^A}1iJKR|F@265#N#XhLPiHJC=@U@8f4XyNaU5r}
z++AnYN}m}4sWp*OC00CAxyv>wRn@(E;eP8%Kvjr`^Yk^x4EKhdYin?G`Tn!@VBEAK
zWwpJ17L(b;k5_+GOrN4sIq_58%Q@<z=i4S84*d4Ntt|EGi{7kB%H<#S?>#!x=*aIt
zX5qQ#8jfB3xbTw=+g!fI#{}0td-&GYcct27$Ns>@uFMi&*bYocK4rM~)FsPJQc)a=
zpNua1rtR3CvT>EIb{XHpH{rQ6B_^G@A<F*ZQ<Td5t1lf_R^HB(+dkLwck;%)Tf1cY
z4&VG&>fLu{_8IZ+M-i17hF7<0%EZ|;PQA|cxaKLlpXHChpy<~AV|#lSH8jPC`z2qW
zwN>V*&wR<-FA_IT)_0l|=+@`{`=a`fr8&<Q$C`bq`OBmIB{otxKz*^w$tN{2lT_||
z>^OC4t^0=pYaiEB&VP8z)>(QyR!o+Ate;&_a%NXoav0;KNlLm)XT6KMUtegg%DN_g
zwdc>ANBv%l&;B^-vH0qN<gorTufglQ;+ARo_3vzsu+f?4<mz>y^ozOA9plMdPv;!4
z>wMH}zaV`o$18KIBbV+PYfApLy1lD6`RB!FU+NDti@kN_ossNkRPc<~Y|fk`uMP79
z_Bz_Lt(os($t%Msa&*?_4__ku^rf8UhqhOgKfK*jIM@BwmCP>%zR8)rts%3%a?47U
zDJR~_D{+`KMdhUZo4XM^>g=S=|D3vXv|8g`RR6d6^{$r3#YLQ#giHplI}qq!zoML5
zYL?Z;r%!(GS-X?*S@#<2BH8Oo`-SfB&|G!ensG<T7Y6g-t1N7O%Y7=VO+Nm&%Q`zn
zhCS20v&%I2;!=Zq8U<PlSb3J*F}=Jaur#Eh*LKEZhMpx%YKs{)8m?#5FWY;6N_4ZD
z(Z^;!i=>3_{jXmdrZm{h)1R;L;ujmUUA40nXd$Olhs^P1O11aao?0M!+CycY{{9z-
z!#_70if&l?e9d-GyHtmTc^=mF|9iUg0;lIsn3Q?3YgdZ$v#*=dGaeW6yr155dBe;d
z;(^Y;s;-D6m!*2^1a63Wem*jMlaz9Mo#$uO=Qrgp9V+6q+OcBJ{2GOt$J_6GdFg*x
z$Y0u;M<(vCt$Whh41<IYX5WWK71ur=JihhG<oD<2ndaQsFrUj={nFA^ljkgOpK$ep
z6URjU9l!ZiN?*rJ3;(n{`?}tq%7^J?p;<nqYnhfSr5Y};@qIAM-Y`L-=Jo#>l?Gz&
z+NYl;ZtZ3A37oU<!Qn4gnKN!a72YR#NoB9wJSlC3qZyp7Vb{KWKkKqYMK$Sy>xPsw
z{vzwIH}fv!HjtkF`Ff$I=cN1@$~Qk8m3fx)^RcAV#em(}?r+NV9$3$n{(JAe?^>ma
zU*6wna`f)AE6jeHwJUvpS3GzM?vF?8vFQ(|*37)QPD<m{#a+|&7vHL8O86X={H*p$
z-_^&}51&4B{Vtr>QxklY|8(89DT<NhB4?`V{@3y!w%)d=^7F4NCB=^qKXu<Jx~)}K
zJZPrymY&Je`!`pfmh*WYzVTayj%w|i8CREczF%w6-&-eln0flfs<&Y^yZs&<+cPz2
zlFIxJ%YVl9vwY{w{_K1#I8LtJak+E)iJ1S#_<l2a*?5Z`4Semi_FD7I8#mhSZ=dVT
z-oP1~^M<)zeXs6bzPa!2FL%$5@e0_psAu2tf8U*VIdCX`VrKuf?uS_DuOy-Uf}b5r
z-=7ZZj{oo3^7W5Q?XpXj%*XUn4n&-`Uf2GIe|f*S@U`=kQ)-Q}F7u|<Y~f!$k2$hv
zSMjm1EFHUp?ce`BD4n2uxF=())B?*`-l{KbFJ7%Ux*(s+tgk5Qla1}|=YNkrnm>Kh
zy3qJ(Z-37|xmJszAm?JiuX8tCZ*R0W+SzTL9@hJ8?Ml^~l{pOI^(U&H+A>`GzNmZR
z6c+CvAxGyVy;@MHG0R-J=v>UCd-EMHOi;_{R9AW7J?*|uw>Eo3-No)uZ?~YMf<aH8
zfZ~dy&_488QlVC;nEmxhrL98D?sYLI<JhhF=3WVMnR{gB{cQ_#E{aY6e@`iDt=8#p
zrcs<-{->^8oA&zp`r>D2Bv*y5o~Hfnri9?3eO_;0#NHA<+g|$O>(8vVtpdNcMZneq
zNVyg+xYlr9(!jVSV{NnQ1dSZYt$WP>r@!C#c-5-^7Jd8Qr75#%yt6G@($v);{C&Y4
zX7hJbXU%M#HfvVYUzyatD|f1&%Iy>?n7!=Ty)G`TO*`*={mpC=zO=QV@%-oca<S{a
z&--k7zxe&{y8X|q_wAJb@3(jVtjhNb9^Ck7`=-S1e5Yt~M9~(3$=9zX2A^k0`kj|j
zv$y8LMm6TuTc6gunq7?D+QKg$_TWON-VNgkT#Hv<tjqA^JD)q@W#9jEbFDXS-FkMy
z_i2fs#q<9i+`He`rQNA^E7OlY^2t}daQ)@k57%mYWL^qvWMZ9Rnl08VZT{@IeEk`b
z+ch~iEBfMYoL?2X`bPC939-jPX0nwNeubMy)m?7$Jt$r2qH?=zpYz^w+4-|8*H`2{
zvlU2pOq@RX&d%-&6-V3-`K?=CI4`0s#w1{4Lf@3<79tDFQhApv%N3<PXY;Cm^0S2Z
z71zJHhO)Ltd~2UszB4p5EO~jUHFR~@#)N}SJU7{1MuJWOIREDJ#Vx-TW_hU^g*-nu
zcV*aGsodM!Qs3X(duE0qv$(kU>pM4g80%PHoA&hb?WEP$7k`d!(zBZ&|Ll*`ZmqWk
zOk6*<Ets}Zce1+c2fuE)9gKetpQ%27NmL_#-xXs|x5j3tgLhY09AFHXytqMXlJ1SC
zVTFb7T^})@PU5$C{*@tb--dPFU#;r?Sj6rwTUz(`myB(dh_v)<o|}AN=S@uWF`OP@
zpRMCKf1UT-5`{}qA0HjHEO{Z&$|ZW}_;Juun5d^^aS`_N+Zo?hn)}OfiR<~yvss#X
zdA0sA=f^jy4K<&7@-2ON>3yu3zNP=+S*}6S7w@0n)8FS^^{+RFD|q>~X+Nd?mK{4O
zac;fuqP(9?BAR!39#6E5>Gj;X?c=F?rWsLYODE_(-5L=6oJ&`%yY>Iy_w}GPkn8vV
zn>EuoJuM=@XXPFNr<7u@)}{|13N&4$0xU%jn7-6Gt~Y(c%ySME6&75sx9-b%tiEbg
z`YPnr)z$6^2?~3wz9#+n@K9swwTtW}|1UU2`Q7R)Pq$ZEwf#UW>x@MfJO1l1C`Z@#
z9bd;8QWze*&{JV)-p!BiCg0XrdT!7Bi%I7%Utj(EWsFr8--aI^M$`NH5_g>z|MLIA
z$($<Lv&VIxyInV9>D$>B|Hv@OF5Y9qa&4_P!_VJ1tU~^`DAvl}C|V~Nt#<vvn`!SQ
z$_*vI_C!|H8BQp&T&vD^cYpo-dA8M)rc61qG5NSgzygP3^=?r*VmnVx5KxM~y`xas
z(a{k!Z7XM6mGU}j2E$Z4xxKgMP5Eh{mX!49;!W$1Z~n~q;ofg>)B5q3H_gi`l6tc8
za(_(W%U&h0ZC7koe}LJV#V3!<FieivS)^+J|Bvxht<b8~jV1;L0-JB1`SbJh#}6M)
zl&tm353<dTFaK6^__}%4r?=e3@2k(g{m3u8DkyJz@)gU(|4!`-HA~h0O`iGhTGsb>
zcfY*5JA1(b1-E{=UawSH`<*EgQ5+NBva+xUh>0EBQ~CMDrKQ}3g@rfQv3>8)Pf6JM
ziN$}GOXT@gKXk1;@2oXA#>18NY)W87*qdYPvI_-j`#N=_KK)&Fl%+cK$@0VB87nHX
zcy+aU6VG1C-14_bz02n2c3aT0>^5HMqF-M!L96bzONlUkdUEpcySuwrhOd{~w{Kt0
z?QOb~Lw&xU;h%VA=6Ch}q7UD>@9Zp`xAye3#W@eY_B#qXJpWP@&~jzwk1!b#UN&ZB
z_r4|h_xI^UZ|l+D|L0OxladE${>x_Slqp9RI5ywdU9RsvO=n@u&Z3*?&wVGJ$&Q^b
zDY`GIdj6t2-Q`c7{O0ZamneJpzNz8zG;Oi4zfHEza)FyCw_R4>RZyHNZ@1&Zti{g7
z7f<cBo!|D?PgI>XUHtp9!^dvj+?4)tdVJl&^7r??yuUyH!Gj0ejadGcJ2^YQys(h@
z>@3sdJ39(9w%$4ue&omPoVcBRkHrrkTxwA|+x+Numus<NpMHIP4O$OgUS2My6Vb4G
z_3Ep+i(^2Wu2<&d<jgQm=L5Mq^YXGDIonxzcXy@s&kets*x%&1wx+PW{D|#Ko8U8Q
zT=Dit_hsm&?)y7S^ZgX1^qEX8d3MKtMJ7vrt>wsiALVjq$Jx}k3SO2=_4rqut;;>0
zV`iLp$3jd$?o5}c_KL{OX>B>}>(}e6`_EI+(CE0eHG6ZxL#HlL?P)u^X7;AtTOYS~
zR^s6{K^d7d4-Pi3EwctqbRFTgDtps0eY*JCsI5vSCQ~vmFH6jy|MmBYdC?aSWw5v!
z&NBPo;N`C-XS3^cud#IIr9}1aZ@#I?>oR8PGf$1;ezy9^j02vmDVzTtss8@%;@W6)
z6Di*Pf4@n;d-qOqm#v<j-kRv`eKU>IAH9En9<rQ$itz0&6%~~a7v1FxKRj?Odw0k4
zXqV{Ps_Lh+KqqIk1}*g}{PH5O#7g$+>hS0N_Ww>is?KSy<zt-~xbpRLt3A$@>n`P9
zu*o|v-MM?`e8<Dv3qF4^3Q=7ZptHQGLwfr`OUccb<hl!WclX_8RCpC=A?7V(xNOaR
zzi)4Ex6hw1zcu@M+VgXBCFMlsU)`P`|Lxt~(^E8qIk>n^O}qXmYVrO0|FZG%@eF=*
zEE2yug11Qh;f;=tuKV>;-K|H$aj{!(#HJL_X?L%0j()cBhlf%3fAiMZ`aA44FE%{!
z?vk13c6gQOifcLCw*SRlI*ob_=Fac&|0(z8`txHY8Ag}?YaDF~eE&H&fcccEzJI@D
zdVoc&(vv5Db)xk*-#qi<<KveX7qfSZ>!*EraWSm;_N?@&y3x}fAMbzs;)Tc7tgAoX
zZomKJ$T^42X=fKjZcYOgL#^E6j~+ajpt&ttZ3l<qk=HFNH>ZQvP{zf@`OLL4J$x<0
zw_MHk<HxEiets|J*@^TAruXw*IR5(LQa^jsmybOUsg;^#UE6y)$;E}`?(XvAIcA%a
zj&c<}KQ~u%TlLjdp*#`>4GW#yjS3zdD4Aa~d9H2MmyWu>zXavv&UJ}u8{I8katV~s
z*Dv&$X(VG^#?v5WnpN=dQ0tqGkF{jKoo6{ZrTxwPfBWCf{5@qw$+sJ?Ln5s-+FDv3
zynfx?-_O6h{QWr-N5KWYv&}%ehUVA*n<=InB_JuOsa|Hq!OMH}`0@5fN4twZJUDp9
z{GkVEihqw?b#*l;)lX1#UJ<|FPCIPPgheUBV%@DLC#zrHS)2}vs$;#<>7m~bR>!t-
zi9R~e$Q-gNgi~H#{@dHz>g?Zj4QKlF$lK5Jn`>oQ{SCa--$crH*&T7oHZiA;`iEa$
zUfz;*l`B3ze&5e$vw32-hwh2xm$O;m-Y=)3rnV{RDA%WFXJ@N^%l`iEZpphlGY=ea
zaO;!ljM6pt@YDt!GZei&??s8#G@ZyvoZ(k1=Y7%PQarLhU|o!5pS-<aQj*fvtgDYc
zJv|*XMd!tXgUvsF{R#pd#Ps&|^wcJ$hW-2P)&1u^`SSAemDS<>v(58QG5Rw{Z_7D&
z{(S$jUg^!**Vp;Ti-0q4{euS&4t#og`sMZY^8fz*TNAn2t!t8NgpSz1@B9B>T<FZ6
zc7ER3Bb~y=Pd9wFFDx!jetm82mdwj-Ql?oO>i$+){`fyBZhiWT3kz%hd_10WV?*P4
zyWeND(og)CHsaB35y*V-<mBWnSy#JmZOsO4vnsKgE228N>dOnky;WZyeSCa;P0Y?o
ze|~;W@BV97_s1gl_O_*IXJ`4$v$6br=7&GCxL(Wx@9BC|bRsA1EPnpz&6_z(zxh{z
zmVWSttO#fnR`>hx<74uxD=QCQT^+tuW9Eg4%I+V&etmkk{QlGH@%3lT90ftk|L)iQ
z?p?BENluaSuPqxhE-JmczW#gzBXh*ws?urPT@0%(E_Uyq=B(Qy@PDq~^2NTh%_Mr=
zc9p(%dw8f-Gj!Dy5w*!P>}sv#YrhC?PCE-4+l@@)_~?7P;Pf=zjoY^`_nxk2VPyr1
zfJ~Y70>Z-1@9*set?G~7p0_mda9d<qe5ynghvL1a$H)6Y<I_?m83)4SYg@xsPhAwT
z@_4`e<=y4^N=izgJvpr0VpA3|i0j8K$-TWzL@T62(l~8`vOAyVDdXqm*Vo0ajNfk;
zyv*n1r*h{BpyqM~_dK4Lpa#dT($__gkMU}DO|5(R``53aEg6AMPEINsc8gs)g=Fk%
zW-zgGJ$U^1@IvSIO|`x{V%)*Yd={pjo>ues>-95pERCb@n0dE>4rI~YxN+mc+}qn!
zR8#~+L|iT|a@CC7v_#~X*PjoE`D=bWY_IwM_dAcIk&5@ZvVZ3l9hWW&T<j*I9oC~4
zyKBOfDJ}i|{@Y@-43w0b4jgd!_v0~t*}FTP+qai*yIBs-93=%$P6*!GlIgrT?W}}(
zUd*!ClZx6MeSOQ~_t)LoRoZ=Zb$IZOf<(<z*&O`*%bnZ#7|zT#*Z00-|Dp8ly}h&V
z?XAANx4QgTkL2XHbrB~Fwg@;aY1Rr|_2G8@{=&z{ctLTyHhTLK4M|Ya`|<17B=h{Z
zLx&EjXzXjWsr<Ad{k+_tzkfe|{hE4vTW)0Ce558r>@JhbdY?5wiw2B1#l^+V^6q%N
zzP5H{;9|Du?RirdEm>??{OrR)c6o!sM=tyR|Er!^@+IG?Iq2kyz{L@}%XFRF`BHy;
zc&MVTo_?}+Z{1(3udl8?esr{Zj!osHTU)c!Z$H*M3>so!<dd`UxV<em^X4Yi<YPS(
zODpd$x^W}o*Z245r)q~s?5#3A+9f)9(UQg$D>QbMzdyG%`+COBO{vpvZU6D3;@R2R
z?ga$~&FuW=z8#MOm1Zdu+WF-#tqfLQv}jSpmW;q@QRO$H=GfKF>XWqw)tOr|FQ=r=
z-JWo9k?S1GVzt=11WPNcplvxfxBrXtS~_W4?(LwBNv@U$gh2T!IZ{VV&3D$3D=UL%
znr4fAeSLlUq9qf5etvG8c1B{E|9rpS-`*}&n0e<wBeQ1YCKb?GTuDoeCBF+eb@co>
zx-P&V=?KT4KYudr@3Vb-YwP3B=k2H8-tmY1<HwJnEj#xAet<U2c}>w!bUn%BqZYg^
zCiJ+J$JJJWLtH<8{w#cSgtPAd-*PF_tRr`KmoL@$f9U4s^pBrEA3k`n@o+o8cfc8^
zGijS!z}t`~?%la_=f>LKWm0B29rNeQ?=F8oZPA`a*KV<kI|`GVnww`@m+NiYwk=46
zvs;N5oKPYJTb&M`n5g{m^JirPg9+#6S_{j_=)_N|S5{PPJlM?s`F8&PkFVG72kl(Z
z>}qkH*tX=A6Gvie*40%<JB8IjyUcuNn?3#a_jk~gaEtnXHl|ru9F_(Zet8kd*%fJ{
zb871zP#`+lD2K0$d2?%P_bk)wN3UO>{`dFyQjPNw`)Vvf^J6zQB<jTM5U{qk*7Oon
z6OJ+J;{omKHT}ORTuah8jmI?W%7WnKekJek%@tI3OL?t-7P`A<p>z9<-R1ea?v(qd
z*hscb0}XXQ%K!2Eck!Peg}1inpa1=S|M?k)$(x=QmG;ZqKU>^yx2WdlCmH)Xnd<85
zOc~f|cMNiJa&PYKoo$$W?85f^c%v*|*2yPbLPA8=ty}l<-#x9VU5(7_msW@CX9@_3
zxmryWjcE~Z+B5O@_xHj3YAR=zC||lddFz$iTQZf8dZY_oY!z_YBiQTKd@*B+Q%8xa
za13aSwB>}#m8`8Tt`py-NPtsmhmVLDFI%&S6Ks3;9-p>_t&pfxJQ6=CF=Jw2VqjPS
z_+U3B;Q$R0jj3KfYLi8rmt;pwnFo$?MI+9au_7;h)0{!8JUAwHW|&CLe4=z|>KDj%
z;EA0c=RrsPfpT!mkJ}qGY`c~A9$w@FHFWZX+s`|`s9lFCK4GxMBj1Pzyf@rMjl0{3
zM;E%USTW8M&2g|z=O4Ex2DUA9t!j6j$qUt}c=&?S-jg{Q;5ic`!IzODFMZn<f@;^6
z6Df&-Rbp^oYM(UNGGPO_fNA;R4Yn>#o~!$4(nKQ;#g-rH@pV5{t;^mxEWTLq@DOWE
zOw5Lj8y_BQX4i?|CnGN}uQ|!%vzfT0WT)5Cq??;kSy)&eJbTs_wDQU0e*0rLZ$@^h
zfIYr-Md)g=-{0TYKfT{4W9jtt)YPROO1>*Yw9ce`*8X4i^;IaBxSo!y7yG68+ARW)
zwz9CZKYsDz$EC;X)~yo~IU4=(<?{K9u4cto{kdCyf9e8=bGLr{{@s0TnEm$|UP~9n
z?k)?G`50F6>dMQd(uKEg-n_WnUtY5hJm4Jmy>(&8rLEcFCQ@fF9|!FQzp-(#i2EK<
zRqtsk<?rsCw@ovenQ?#LUJ-Xt;l+`CWpn!Z=T+jRR<fb1!=@hU*uE?p)b`o6{Qvv6
zZ+lm-UM=DVQ5dqTB=ghr^ZZMA9JlA*4(n37x9Y%w1IxFcDJ&@gb@_t8o0SyTME?Kx
zcjM;GhtHk!Yg=$MI(&)8#O3w@J#Na3jErzK2?+|EoSYIoY<u?ZU8?YCEjJezlY)W*
zLv(bsriWFqH)o5$AuClC6@&D1GOJ_4{8wL{WL=)O@9($h)obR0s<_>r6-7lxo72y`
zt-qeWv$ngllTpFFPX@FDHZt#j(UTL70RaMBt%tO>`{$j~Z4n5|`ugfB%eHLy<xW4>
z=+BsOR)cY?sNVi>?j<^$?{^fwI@FqTH4fC1X=dYn5x2x)+MWF$j=0$5<g76Z_$&T-
z{k<oLTDhla1cD3%wSW$JMe2yX5<78R-Id$R@ZN#W;}?BAjnYql3$m?`%haEFVc!dj
zH9Oz?tpklvDH3}EAg=QRBj>GopwuET^W*e={sKFrrm*f6yplCV%fS7<^1JK3N8Z^_
zU-o{fhG0YX^tKyTt-^K%GWolLwjN4X*jbxj{QdRUq6bD@yY1h}@9lSEo$+Nx?}95Y
zgZ`yeomiZIY?8#!Z{Y$Lw@LP`Ud}GISg_T~v}r|%hQY*=uWYlLR4?3WElz)6;q%|@
zhKX!b^;)KgFK!}M9r3^I7xCZooxJT&`r<P6=TkCo+>bxIcKx-~zvrbbn_C2wJ{z{5
zRZe;yoAc>ubN2p%{Bu#0G&-`spFeqbY1FKJ)1O{a?8#D}QkM14cTPc81Z(U~s~fkp
zpBzY?a;J5}$+Ze;Z)+Bvo4N7ibpF#{_uh&BsJ%nfZMt^tBZp6~<U9O}9H*b3`-8W5
zhkd@8VDYp`Ez-A+aH~wdbnYMb<7b*SKXe&q*ZeeS`SZm@W0`C8HM94Pv30By8Xi8=
z{W6h%ew+JpxwV^fR)FU!XHI$fq|&G1<-JNDap#U%yJUqI=*E0GudXQ*FkPed(!bXF
zoOll3n?d@uoEi*wxc>gE`R%kKR#al~<azarq7|>Ga@qdRe=F<ouup6M<N5r%wjbHK
zK#Fyp#r98if9F<TEz`bpaf)uM%enfGsq=%K|2&&{M9!x3pQ8*z+wJ|yHrv{M&(>R{
z7yD^_|LW*dA48^Ue2IB0%IdX9bKdK-x2DF*sO=RrNZeJ+c=_mO34J-Ws?Tgmb^m*x
zh~J)hh_Q6%_ki7&`)>BHe4(PFGD)K9-Ok+VyW4v9I=`|LbDDaKO*GE2q9#CgYp-&|
zqmIA#3zVL{V?VIY-@x*(O<?N9Z_41-P-bQT!?YhQQ}jP)Ee&8i_T$Ht|Ld=^_ivlJ
z;YDah%#|ZLl7|<^hHrdnH7&;U)5IEc&e=73N8Zoy+p@@P=d0<5WqvaB&eoom9nJB7
zF^8P*r-l3Hwd5*R>uBaTAD(b>hsegGi;7)ycOBZhGrjQTEe2`z*sOeC9l^g<lLhh*
zuFJOVVGkD!U&HHXI%`2(g+b~I|3JCxnuiNZ0w<Ofu(t*A7Hrskw)tiJq!+gyK7G3{
zes&G>o(uE4kNkUFV<U63`9^eFCewQ9dEd@Hdbs%@Z@>g2p4=GwM)vx`nBs34P3Qbt
z4G*^5@@rLnC9<8(-{t!E=sWj()gDf?F57kZj)dX!t+%f>KY700`EH4b+`a2NOh3u-
z@y_dvetK(izK5_*`TIqurt{~?KROqF{G{z-dnvikA9knSVEcOb9^WIgde0{lzn`!-
zV|d&c>2i-bps|-}!-Cw;KeM+!-F?_C$@mL<ky!CT0nz4TM>)AWU3~uwh3>wk5jgSl
z<Bu8sZON}g<&K`!y>shj!|LtUTn8S{*7q<FIcC_;cX+Ap9l7l+$IqvI_Unvz_f5Xd
z&TjI*8xw*(I24Waxf8@cSXS<jc4$1%C7~z(=pO%T$3TJYd8UT4|7TWzHcN8o4qnZt
zahm;M;BJe+_^%K03hWPkZ%e+vz<$^L@??X?sM@*K_t!pnnCbq>MDy(9!rC3*k1EK;
zcrxh*?Bic2S|qsl0lU(+xz35<Q|5>s_N-tLORD^Fm*3{j#;HpxxHVQk2s(9WapddD
zEuYIpUM+f(5u{Rjxc>RU3ol>zF3^42+55`s;a)qncWX{-Z@;}SE%nOB=bdvtSx!uU
zw{XM6<oU}^eXl!OxcA^<;qMy-ysxyJyAx9@)#@GjhD+qoNo)RN#<M$QY-g%$wSILy
zH+<o7Mq#d;LP^u<e~iT^?Eku3W74Gg_vKtYi+DK?6#a33$5Mah?D^{scKJo+TOT`m
zSR}aAzhMyk?93?qYx47l*K~b2uUAgEb^CkB!EfvDPP?D^-ucTa<{6BKC$h%8{v~?y
zMpE#RX~(Vg6BK8jP;9O+D3;q;-m77dz&ZQq;msc-dc=3w=J4>VtvSFEA?I#m>n^<g
z%$Y5a5v3GH&*M%8D@r3eo_<{OQsNP#xqeB*sl`lsrQQ=uzGz(h&A(VDih=uNM3DQV
zFVTk$79ZNSLd<x=j(p~m$LrKC&f(WF@7O=(QgyoLT+RiW;Svn9r5G(ljRmVeX>3f-
zPr4NCAn$i*Bl8ix+YOQd`~TZTh4K~E{9@gi_00d}$(48CoL}hkS@4~nRnw)TqNiv2
zs(ikFdv3uWovu0eL}vY$oi(|;bEnGQu1wG1btkm@UdQs3KRzJg_(<1ynX<S+GSg|_
zNBTz=e3kTQ>|9%ElWDUjZFkq*Qoevh#fk0*9q+ulJ@vJ{?7=UOa;7n*ZVYt%^!2vp
z|N74io>c+QPc;5}{&z;9YWnL%D~fh9YRS9~cfY$z@RWA7;hqmcue?(4FVMZ;x*|nk
z=8xF+h8I(EjT?ey&q^-VIv~^-`2YDAiPosE=_|VK`dztMVKP;EVd$Rr<b^4V!sAa~
zdFu@704q)2Y;(i=h>J1nF}4R*AGIenyG_Vh&FXVR=SSBJ^}}1Au1bzJQ(YA~X<p2|
z{EDiD`<1qet2|9q)+k!EF+8DXyThwHAxqtOr$yV$FzQq<^o?F$b*kanX7L=&JB2If
zPv0Ty-LX5F%~_`Ef8P#~)jh?LqW$NW?3t&wKKoRC?XBe9PZT=qH{P9feD#jaCXCm^
zN(*29u*_<z-0*sp%%3f<EPr)BURCvR$Fi=u-$bYJ-7Wm8-F)2qa_1eHuT^hf-Bg=o
z<&tyLyw&=5+>hj>Q~LvV?_v&bVCms(h~gGp-Ldbk)O4xZ6Ro@-#g#UG=26*x(dEnZ
z{d@kn<=>b#^?dIA`#JwtpF4Wx|FN}q&RQok7yREPf9H;GP+GoxQ5T1zi(2aHzn`7|
z94mU4XVLByE6|^B=TiAv%q`@JjH%phroI(fyej_>9Gh|O9>W%kee<nuE~yZlq9RoP
zNF+^pMNF`o&Wy>E112yp4>Jv_5_?nHz&q_kyM~DC$<LYYQ@F}y+We01UB#ZZ_~lN0
z?^b{DH&dopZph*8{JnjmL|Mb}&E-i>t--6$Y5Hs3m>Kgg>X3bW3uE{l`?s9+8}Htd
za<q6ME_83TZ6Rku^ut)ofBz?GS}t`zlT`61eI8f+hgXlY(wWtgPZfq{1Ts8Z@#x8{
z8?%?l+du!+QmawmD*3VME!V#{H;wJ9ESn<dZCQMRSw*$s+-3Exz2XOtv3?Q%&!CWY
zom0f`=LI_+#iU1JK3eWqE~d|mJD%mm#u^oJuFE|CbZ1+7Q@)az!$QX5O?FFjZ|_?C
zQK$7L2Uqp;DaV6$-<y)BdQq&rN&0kUi6vw2Y~hc$%Tj)sEtaya`jTN;+9Gi1*BsmG
zX_NQAS@h7_GVWh;Cqq&F9hqLY&Rr3qQ=VlX$&YE?*kaGu+Vwc(^2`<S?~}qesq~f3
zdKhBJ9w^*Csp1_==PjchS8}^*Q}c_iX!zWjm2YreqNO-4J>=&-*OeVAiLcXimS#T;
zxj9{iaqWh)3t5`Ej_$2hS;3tgzedK=E~91cJDG>CCl%&@(ev6WY}UH<i2GFa`wyZt
zu11~N?sVb))#gKt$L}@XcwXlAGDzh1pFE50tGl)S2=HHD@AmpWm-P2T7E`Kxo;+lc
z@wVT8*Hdq6LH2#Q+>co|o`ij<cxN(6e($em>gAKa$QVVvxVb%kTb6$Kx|oLtmi4mi
z*!AaWu}Iw9lb07tY4r+~Ci-7F*Y9alD}DNEw1!ql<-Z+z{p#VrwN`BLTQTSDM(z*p
z&l8%rt&p1RT^W-V?hw98LhE#;+}VG9E!Q)`)6SpU5qh#cXr*KLQjuR5;tiMfNoUXI
zKe#n3aY~f>m2KgjG3BDat#}JozU{4@yh~&K+XwH?TzOEc%ym2MYR7EziA#P@5BO<%
zcuT$==k!Gvwn_H6FE(0X;+3<f#z1z4dr;r0^;f68uQ3xp-v2B4{i~LBE^$7UdkVGY
zar+)@Qqkc)x~(o%Cfmeiwy}K~>)k+0afih_z3(`e&+(pmU}Ih6m;HGvKPR?6InG+J
zE=i&2+P3;VzUMFPp2)(+`ta4Otf{%)i5nwyK26kV5xACqbycX&k0;5?#Pol>k3Vux
zHuT<{2)%jFcdq^;@@_@<^CN#AJ=U-D+#%|8@`n4;7T8(<XCHo@gwE=Bg;TjD^yBKS
zt^coj>vuQp^3l)rzn@k=Dl`e6bVlai-;!-v`qpJ{5^ByXJe#KV>AmwoX_@QccHd2J
zU!5H=O`}!m71J{DJgIdB#YcoBXWxpGQLuX_zx})Y_SD}_E8fT@RejobJ}<q|so?Xk
z%YU~?TJ+9&R@!p@#{Yi(+b@npZfk1`+?3+EIw!j5>>>fDl+=%cGxXf{{Mn((^mw&g
z{m0CS*W-6iYE%7Wb?ijRy?58|q<wt9_he!1)8hMw42|M#{)@l=@00MmeZh>hT+4q}
z%l6N>nzL`FkXdoh{Pto$QLfz)?@DjZSH0B=o+NYPnCLc1@9u$=o~IxGaL%ktIxP&I
zQ*h##7?xOgFd|R?MAUB;ThOY;76I^rBo0OJk#?X{(ZS;t0#2X<<-muggS(puX(P#<
zpc%x7jU7i^@>JEl6z?5adqjoj*qfW;pB!#^FB6%XaQC<mNK?ehpru}griz{KrsqZ7
z-6q-N_Jxg^Z`-!8?YSE{IZIw$P`tf$`XbkEp-hwL)asw>ebj_Gt8Dwq$|67AxM^+u
z_}#m_t+$tNh|m%H=^^IC@lkDC{{3@TRtDePSG!xpc~7aj-<$>M=jX}T*Tr;AWPAde
z>=JN_Fx`B!$7$gN$SyX;mWtAd5G~bB;FTv}Pk;vnK%NHo&q3Z4f*ckP8|(lD1Y8k#
zZ9hm6G?ZYsE`St40}6Wa0{E19khj6-%5y@(7J6bI=;Zk+pz$eWMO!yS=)mRo%sQDe
z2|5-c@aQUZl!#cPpyNj1ZY{)Hu-hU)M=wJC-r}kV@s^O7>#+cD&Xyn9Cr_UI^2&Mw
zGy9=UPygM%ecSN=amk9xO3!4NgDb*(e0X|inLSVIk^L!k&Shap%6_Rkw%&7h^+xQh
z5$u&Ro#Zz0>|CqTF3^nx>FMq{IXV+3P6W-fw6w5*1`2<Ce7w{Ha%A47_75KlvaaX0
ze3Z@i+CH~MX?C55W!03!|K=)f;eB&r%6fGHkt?xM?q<_w%<zb~@p<o?@bz)54hsW3
z8e3Y9>Ue4CAK^YX*V=vQsjC5Ed7c}W#i!q1@WZ9ADOBTrt@dHN-;>U}_fKV;)LU6y
z{rKIxf1mof+}-VdJ`vWO03K%QSfj6^(vnrTwP3ZJPoq@xQiYXo?sf$(yzZB#8{xlw
zLEH=<HBB$Gt2>L+O{C5i_s{fElQPTEI4TH^rHNbD#_pbW>}*uPnx8qJLUPnuD;uvV
zia*_zpZI*?YLl!_jGUaDK~r*8WM5x*zSXtW$?@~Ev+-P@gg9k`hRZag++XD-?@c?5
z=3e{g_t7e7!u|!<EI;Hu(9(0Wd=$Kwk6+Ga!lD%8)KgOqo;~~a<$r4n3jq-kl~$<j
zrja^gc2eIslhy0aSKF37n6iD3$kX|DCLvi~+h({nb?eWYTDzm{?JN=1*)MNyR=1kV
z$1v5)_3SLur5e!Cmuo)gaQoZbh+T>D@4wz@%ss~xG{rn%$31b2kT>1oq49omEEHK!
zx-l^`2Wp6{2+^`Cd7%*hNGbNvBCn;BK-o#@?~!B2f|kc6^UbUGxw%gI;*F<M7X9J=
zxr!}t!NcuWzU80jleJ!=aMNX>Q>#YQ77ew@oq2b6c{(16PB)tQ#NCr^(r=xvoagWE
z?!L0_c1OzeQ&aM^w;%ht{?2I=$JY<MUVocA<J()8U8S$JTs_rShpj!eAud*3U7g|D
zn#jYmOtUZj{hOPcD``=nU|D>MfA1TIwuQ#i4kc}jSi+;Udq=@TrmcUZAH-}kX3gfG
z+_Yrjj*jj>8<&VS|Nj1<`@FlC<Gn>KEiAvjy*>T#aC_$Ub+K)C1a9Y;2}?_NZ{A$o
zcICmTsoHPu@1GyKIxKK&R%oV3WI#xWh`4^7%gPX=`hPW`p}RXfjX^=t%4PfappuFX
z*Tl3#hYn>3U1+K)b(?&AbF;`3-3!O>J%95kjOEJrhsSRkFHwJ>Yf=7AW^L5gMM+1y
zeCAq-T3heViSE{o-gaSseLd(bunP;FmlmcN-rZH2e6WetzkY&Er?#P?AqzV@Xfc9j
zkkv&0qKUeoxx_YKy9FnH8>?#UbG14iGll=wsc#p!O){@F9XaBn=_M7lr^3)Q>q^3d
z1C3{98mmXMyEdr#&e~A_ziyg-{5f{{ngfrI_b;88qL_PY%fX8mC%$-*v1sw)&dyFn
zH8nPNcJ^6jxmv7V?w2-eY})fa{m+5)`TdK1+C?p%u2x|6WKWyi=q=4Q;c`)Pd|A%4
z*Qfq1Ua~~x*Z24S9v&Q^wHQ7#je@46gAR(3kvX$1_cmzuE-o%EXo}8(43kO9?tG6P
zKb~K0Q~vJGoxRoJTvPg%qz9&C%GZC=oZmnDP_^T2RgGoO*}J`*JH<A46tD7Hdde<l
ze_d?XC;iwx6%(~WR|&|=gHGF0(b(4%w)*Mo_50KQ{P<{+cW1`S%gen3H!*CDI?FC!
z^Wl!^&ZxCvX%D<3M3mmvP29%+=FlV4-77C-tCzhOnY3t$ajTQ#!(Yi8^zSpTU$t+J
zO=VEmr?zRj(QWJ3>w}h1{rmIN!otEM@ZT}e^ufKo)sY(#7(r_})YRO%CY>yOeeLbe
zvh_hLRdmvhJm1OV+`UFkPp?mnQFrIB0N0b(>=$0T;T33f<FB#w&TW+%+2Q(&^X~2n
znoz&s`s<$`4)gzfIz2vOQwrzP)2B~4<wx!JxBJ;*Tm5ZA)z>Vour(U4CztrB3G3<U
z9XWQ4DPUEI=DE4n>aCvXphyYR*jYbc>uOek_x$OX-cIuh*dny^6kEWiyGc=7_);!D
zIvO-#f8o<pqI>r4?d<7MF);~gdbEGR<(D(8O10u^KC*s&eLelznVCUTe7u$hU0miX
zJ;h74tgP(I%ggRvN{N9T_3s}Y?GE|0s!cOLxr^P$b>Wny3)ggdFU*?qdRwQXv%68=
zpYWgw;0d{V_wJRvywrNMTm12h7aFcV`J=YwoP2k8_e{%TwYRsnR^PH#HJYv$yDK3%
zx%p(uqT1izf+p}N9KN9BZBg<<pm$cC!#=C|laFs+EOMmlW%9eCvO?`Bla(jWNM3##
z*R$lzGg+$=joaIDlP@lEHOiTFY|^Ajpp*MR$IWG5TQl+AEfXPG*|RS%FR#39cqnOO
zz~(gHdzH^+udR>&f9?C~l9!iu?AmpTaoPrrO_O@1&4q4Qb_nUFJ&f8kPeeu7K<#Kj
zo~yf-*!h#ULQiklxbft+tvkxzO0A9Ft`@(q1~m73ILzfFU)Z{slOG-)o@tQCWLy1h
zO4A>9&`Ed6$NMTODi&PKxcBS#-F>yv*4DeX*GcP5K6&Kp>+3Vk^W{M2w=(M{2BxrX
zPCJ`c@j>EV?tY%<f2J-`nAveN<-wy}`*vu(|8TnR?2a8fQg%y$vUS|vD#Mx|1;4(&
zR_|4kjoDK%u}{`IXrU8p+WC3Cd-v`YdCGhE$_&Hg8P?@`o72vonk0OUPtL~U`@6fD
z_x4y$R`+*XeKqa(W4*`rN|&WR-t@4O{%rc5HFxrY8ItxjAyQu3eof4%_SXvi_VC%+
z**DYf^Rh8Fv-78&n4q{Oc6V4)l9g8AB9`CZ-@7jjdieC|(VLsomujqkd2Ma<l~tkL
z8#Wj?IXiEDGnaQ&=xVo!2#M?K<B#9IJ==S_-pS`%P3kA!l<0M1vMx>j*Y$SK^<wt2
z_af64r37)cHZ9^ko*2DMdtPXbMd_=MY4@K?o98`v{`~k<?QjlG&V!dP2TM6wZQi;y
z_1T%3JhE0(qPAvzyzFo98TjvD8?W?-PoEY!HnaWt^ZERfQ&Y9Q->>iP<~}`5cX9Uh
zbw7Um*ii8B(3`)Jx`!_)8F%*gFVA%F)^l}OX~rFp`F4p$<dMIZrFU#Am3V!9%>uu<
zR*^;W=DD|A`sHjTj8Zt7*?1>4=dprX636>wS4M5svj6kJd0Tw`Z&y!GPmojI-rl}B
z;b2qF&MA*>ZccxBXQy#oTpTE6MBkH%uM%@zCMYQA7&?DejQ^RlyJe?&2b@VNn)<sz
zXs!4M$J5_*B6pd1zny(CBjv`1M9^ZNsoLRQ0b7)odQX3LppjXEhfOPd-I<@CpD)!o
z&+-yfvPnvMMnptxi!I*mGt;Q0rDeg@td$WPmDK&_oG3F5=#{gr+A>Gz(5={gH8ZpJ
zZ4D6RT9C1rQ$ysFwYv7EQq2bMk5xJ|?ns|uTlM>A+OOJOwZF}JrOnmU)Y^EZ%>vd$
z7+PKkb6b2-KtQ0uZ~5Zl=jSRaDnN&qrM{l^=f~rI(22MY54RgveaR@Xirtp44ANZw
z?oP?8E1XlkTDNciUi>;NJ=KUucg?+t%I+CAp45K2!lRb|^jy${90z&Xb2E?XX39P|
z-}8Q1+O2BCyxZG)^X~34OgO;M#v?f?cVC3jX5;j8po5jOuB<rt{eJ!Yw|TEcgoT~Y
z&$G2Ge<#B&uD2xR<fKa)aRr~B`R*!tdC2<xp2LS)xeJSnCvVVjal5rCmHRF8?w~i#
z@&(#=74F{KK5KcOnCldY#^m3^k<uUk+^=2rA|ouI(}^>kUv1-$XWD+sy^|K~irVu@
z>*hA2#@_Qfatk+v8t&b!TVCgzu-4V}c#%}m>oYYwRXT5HR(D<AIkB&38`l@Fj$Jd)
zY<Krz*z#@r!XQ7^BQt-LMH}&(a7A6e5a^lu=<?jQ=W)pe)z%@$g?@gUWO=u*(NJ?%
z@oDCdI#18dn(A#FdcN&sijsxJjHTYwK}W5w3SF(HUaTf0E`Iz#BQwaa+1J)Abm<hj
znY5o@-p=Rux3{2mox<vVXTI(2tpEKp#pKh|>G7L#ZW@W{$IVGyzq!uo*r%4Ovy7j+
z-ztA~aCf8jWm63%8Jl(K&y_!%Sh;B8{BKJyFLQO}m9q?0iS?^14PY^tvGb2aPN@EO
zwa6Z^L-MnB$mKqJ$1D6@XMy)+J|5P!t5y~sJ@iO`S;}1WXVHOgR~B$=c-A7=KgWx&
zP37~2u%jE&%&zDzQ+v+6dv!rz;5AOYyT;338wT9A2wryCs&0Sk!m71ZHg$h2g16iT
zx8~QK?G#q$;N_jVU<Rk9g~f-<{`MDFhwInW)<*8DshoB%zWeB*Bb~yW92^Ocj&y>K
zqiuR*Uu(DWYLBFGn`QB{3oC=w?S8*8?r~e(;;MKl_`?eQA5T7aPT{}RRovLKbH%SK
zbEgz6DZ6~(-OS&v(<}U)PdNpM`Nf#ne$T%nBPvq-(^zI+w^t=^Gv}QcksJ2HD*K}*
zpIBDV;cZfE{O{>^{b!2~ylrh2ykpYf=4g2QIrEA`-+q2y8+o^F*;CUeOgbCYrv1rd
zncCHM^uNth+tXPW?t7*s>qH3o)@;_Bc$%%E(DiLrs_BZyy>HGnq?S(2(p8N<|9s&(
zn|Yx%J4;_r3*CM@c6ZssmoGi%T9roZuiL95IqhWW`+IYHrOl19u4w%F^0GN@{rRHb
z761R$g4W_ZJ3E_`lXGF_<z+<=?mN7_wUv{b`|*)Z;gI!lbEVDmQqC6Wv`0k<FTUup
zeZjpqKfbDnR=sn0{a{|=?Mcq*4}bY8%duX$^5NQsq}hxC7p?`LooU}QrS|*Hh@}hi
zwWGCeM0B`zuI_Cu%@($p`D^>OxiT5Er40Drc|8d<)aB(`y|Y%M@#6A%?>C-~cRB2S
z)l_1#=nkuc7i~Lb$}S0n1U9eb`l5DW<%4(foi$YlT!ibTXH{qzW=^%3<#Xo$Rgc|Q
za;2|aYny*fweZD^#^X<~PWgN(q2ANqhHKW;kfMsmpAVnSdhqkd>I2(%bnQ-`)Omiv
z;{^d48rL$I%$@fg-`vH&yF71c%+8{)t9fVBj6qG3&(F`NA8cX`Ssxec_e|%}-tYI6
zO-!cL{r&au?c1}nOtUqjwq#7Zv0c`>?84UUaM0qP4ngIBl_5n3{<E5!n_pWOd%FC7
zt^3ncQ)e0`v#pKZelF;@hHdwuEEN@%3mZQEeq&oW`_o<X^o)QSrI*th7xU}h5WVnV
zJ!7gMFRO6xpPJ`C8=eQ>ZCmv|;zyCE(&1@Gif!UNyPuiaL_YogX(7w5bN&z7938ht
zMb&W|erT9JNpiu0Y+Kn*@yRo{S$9@ztcZy{{q^z5J0{GYZcWdRZG1RSvR_H+&K=V&
zddnVNx9!xinZ#qJHS4L_=6|y!ecYPD)e28ANcg<jmByrQntRtV(!tqrb7!yMe}OA8
zAyckISX3OF^6>bghWiXQS^Y=XFSD=y{%-56vg-HuZ2$cI%gM{D3To6Pi9LDo;>4q)
z-7haK<-WQqH2LeRt23+WdDxf_xA87^@0Zhw*)ie4g9Og5%nyspE-Y{aomwTR>~>*w
zxW16Eu&NH%#A!;(${)Y2Jvw#9oGE$xWu{(}x|SWfAS`l5{UHX8m9LjBt4=kY&aYnV
z|MSmh8}3yG_YCdk&MEq^OVGs2D_gXo^}hbhg9&MucTPFpovijN^yld?PrC!pWKDHX
zuiROyIMaD~kLE$43+^*27$hb!_BLJm8Ep4}d-}CU)7=lAJbhATU$WceUpwbNo;m$q
z)rAH**0_05R+X!ER-gLtrSyMSdL-{F73JN+YZlLaQny}JHR`PT0@vuUH4zgZ-rk&k
zUQR1?Rm=MI`qJikNABJQrJFV3>*M?nA824?_VDvNSMs~y=_%2gni|lGBzAteBj?Zi
z^W{jbirA=>n3#Cv*s(`XpB|lQoUWq7H8D&+Zt9*@3(MEKc0Mx>SheQys}2>N1#F)~
znZ+j^SiFa~?pxZln5_xFQ-pmP#Dpg6thgKc#Jl!KTFR@fUH)-8rhD%87%_{z`|tWA
zXhra$rw{*n`rKXn{CDliGy77Pt$F)-{p3TFq`1!h*%5vI%G|1D@BE(43Tu3Oxwhij
z&5dr~+-GdmUa)kV^`DwQ^Q*W;ujs8)`ZUMzaNB}-)%1%}Q(pfv-pgKdX35F>DJRTk
zoJ~8{Bk6p+PZqR}Ykuvw$Yp!?_E&s15D*eN^!E1lk6*t&y|p!aX4>uPda+&Gx0}Dc
zy*>Tvs?aBAW(M=Uh<@<m#fd$YpCxQ63_#1ZpPiX0B-Z*vzfaEAD>dJK&K|a)74JM&
z-`ZBAr&Y3Q%7<;=e7xT5t~+%gSK;Cep=F<TFZ0p*`M6QiL~GSDzm|`hi`yiGkDoo?
z=CvX+ig}`%A48U?M6kU1l7Hrg*L*~{q_%Ba+LnCkYq#;NR~OWpKF+*e`RUJ;4=a^+
zOcV<jG5NTCskf%+F?HMSZCp0<XU^IDvf@;o${mw}7fWro*?gDdIzC15h4XWtu<Lm;
z#nTr$ou6Ct_ihavkCao@Jw_cRr6x9BDbT`2(DCS@tHXA^`#txu^h}dnsmsgGKK`<J
zdV2gl%Rk1Kmz_<zwPdCYr+?Mg?iq_LHe{JKNA*36`NO^Sa{TkjMNcD(4f|enFFZ5f
zY}x|<>w8<}d8+u<&N{awa`Oz+Tq%3`eV@OiSZ;l)zRY*_u`i+R7no-xS!_+^yIfya
zSUR=BtY7lX`Rk7^eU-4Vw%(j`)9BW=T<@!^LT_$JY!0nIckKA_%UiR<kM+r(UgkUd
z#pUJwH9wa$A2{Gp_w%XvvokZDPfyb|J}WD$qQkX%Q|arlKd()f1w3B)Wc8)9tG1mz
z9qc}RlGdm2Ef1CbMaBE2cT7)7`dui@S9zu8;cO${ok_XUQM;UP>{D@i7nyPGSf8Qp
zn~Yh953S*?%DZd$>A;c&uG!Ydwr*X%?%I*-uP0^3-fvsvQ0Fguckfn{bIVz#cFE1-
zx+ddwY(?P3z0J(=*Di0Uu2pLLZX$BI;heMY^G#B(me|=lq^mud$2pxTYTwIT|Lk((
z$>!2O9#7*8$!aUzruyFaw`ld{6Ejt~)BX1f7@2teGTF4SgRSJ9R9k5H<9NsBTM0?k
zYf87t&z7mwl;1M9)VlA|p3vPTGgtlsEhyZW<Z5GUdvi~v@y_Dse&7F|doAhkdeO!|
zKOT!5Tlep6>wdRQJB~m9^W*V%i|e_*Q*G)C?bW7ov)b~!+4-ew=Ikulsa0PVd+<ri
za7|k_;i=AXz2#2}R?oZdd~T6@d--HHm2*ED<@s&|?|n6S+Nwvk`#1fucrWyIR*38J
z{bzs4iyM5M{N6(LyVLI8eJA#8y*Q=d{EX9KzrMacKFc&4)Io4=<2jgM5OKG%=+hHV
zP-$~CNl`_m<<ryCr>FZ{X~pfasQ&)0c9&K_Mn=Z6JN!E}x1SIwets_b#fJp1?a@ws
zO`)64)ZM(T^<hKf>A>^z!}9vL<}{w(xV~~%rpS}>h9erM?c>6Kguj3NY{dbg-&OzZ
zo|-DyeXYtd=d}sY*63<I9ah)7?dFQ-hi=6^+1Q(O{M!7&vNFHV-#0|)_{=b9e82zy
zyqlZTgSY41<+=NO3HRxbS(b8Bwnm72lASsysA|Vzf88VIAF>0kC4D{kUL*Ja^S4vJ
zd^CLFt2JlNo8b6l-MF>iH~o82E^<MALD{DIcUtVin&GxF5tdWTARDS)t)1_4<j9f5
zo6DV+|N4BhYBHB>RnnvbC;!c?{_^s1=f%%oYV+UyY*RZt=VtvPb<4)z?E8-If3I^a
z-|EQkR#}D}`=3j@-OD}E-<Udo-@je&LL(mTpAhOV-)vod>~hI;_2c`pJly=wcXV|v
z@}F<#;^G1t``{AMNDzH~Y)j^43G+Oe`!%0^r_G)_TQc|dw%&h#e_!5TU*FW+Y<zX8
z_NC1lm+nq~y#LYT2ZdAK?^E?=b#3_Ja@yNbP{HWhr(ceO2NE8JU3tyEH+*mY6KR&C
zQ?6G$yDxQMQpR6LL5G#s0#}`7QVM<lDDvoZP<Q_2Yx{~14Q{<s8%tk@IXO8Q*=R7m
z`1+0KZw<@Tv%e#^Jbp7-`NXl8$0g1G23O==u$X3hx$VOH(?8G5xzPGacmJappI55v
z&uUG+>$ETT551>t-%z{#k;T?TzO$#NC4YS1KY!ml;lk~QB#bp?%nWvW7`ah;_qwBn
z|9`l#ZClauygK#ey<J>(p|(Hs;!lOF=Z=16@S^*@#Q%lzqINMy)<!R^`TI~w=j7!d
z6`#D`%~A;8x@=33hU8`om9JbK5oY!E6V~;fvXSmLWH|qy=R$GCNvmDv-<1|W|F~<%
zjtxamy=Ix``#n81_2!<+;<mi=huir<ZSaqej=s6Q-M?wk)Ql@D1Q#w`_~zDD?)v)r
zLx&Hm>TrDwKh`hpep63+L%FuzQGraSg#iI;PP$AgVZE^>GkDhCyo?MDQSGoLIX5@u
z+}|h5&(Ht*MxwV-UuyQ{z3sNUkFS5Wi&Mw^`-kF#VOR8?eYq3-{>{vD?cCZ1wT}|)
zX6~4@{`A`Ae=mGhnf&STqs^&`b1b)ZE5;@A$Zei^s8=f3d11}b*ow!I*O$m!-u{ww
z-~b~_q~C{+KjuF9u%<Ph&ot}*#xpVHT~nQFW^Vt-@vA*?%bza_vF+bdUu;O|*{#0$
zQCeVI&)W7?Up(q0?uqe>F3@xNy^&F6{k*da-NkguzkQ57=dh>f-(jm;7wb3th^TzE
z?|a|XEr%VO*)|qE^#ZLoKQ3QCM^U>9G!D!qsui#$Be3lKJ>Q3iTAwl>>Z+)y*iia9
zY*qMrzolMNZ){5KZgExo_h`24+U7MmikIe{GZatzTpvH}LefTq%u6buA+awnFH2gN
zt(kha{anMr7cKGK7bAMIZ<c(}Qao*bY29r3Z^zQt>t88`?5>?@)v{($Q+>LRJAdJ6
z294~%jZ2JI*ks&VYrSl-`i_2XxufwL*L#}Z%r+A&{VVdDFK)@1?eWG-*NgAEQn&Y1
z-TOt%SNIZk3cEkqIz{gMX?JIfGS9P1JpV6FOl0RdS-3L#+ww_YZw1~gnt64<%;OKo
zd|AG(J!r3MR{sA-*t+YP-kfs}t-6&kRV$Q(m-p$JnZ{SvL{7fFJ%9P>-?Mzw+<K*4
zcbC1LVVKPJ_;|nj;)^HRn@rjCmrfE?cH2<$G6+<9g7(1uGxOQh)Ggp-^Z4Ro_mX#a
zX0r3keR#co|0Ml*yVbYdM5T1tgu9roN`@Y9)Bnw~;G3LhMxE+0Zia6w=fBH(Q?z==
zj_*nz8&fy3&-q-%VD8x|_)+uno6-kw-Z6SHpR1pGsmCoqWO3$a{uN)IM4fOiul%C&
zRn`7-_p%E~2FDfzTz~Xe&37K#{*LU2E;e`D(=Dbft^c1@B73{{?kk6~Z-1A|Pp)#B
zzGkuWdxbS0!!B7CKTA0~%hYG45v#twzL-vg!{IjG`?r4I-BsGnCu_CfY8EFqH)!p8
zky1L-)m5R&Iyyc7{{DXa_U+kIQ?)hY?pV3sS&|;uvHi&2hFOKVxsulpO<Nyz`%~c5
z|KGMQ+YtIOHlN*XO5NnkVFw?4`&JVY<M;J?;g1iAT48HEHYOdtv9ox2D4T1;i|Xs0
z=8}xnQe|=+2OjJz-N=4LN_WQh!rzHrD%wH2B#$Xq?~s~o^rj~Dy_AXW>>!84igl+S
z9GaJZ@$1LKvL5?#+*bQ_R!HTZS|Yr4ulCcOUfUf%?~u(`dH1Z2jrV?2;Br6Cc~M`B
z&!2x8$Kd_xx=&QCkZqM-&cC*miTwdf$`-u%YUpz0kEDS%3k%Z&`8z$+*?;HGtLA?c
z)V4r++SYpkO9C_`tV%Szr|T_EKR>VL*URNId@}2$SMf@lJ$U*QbP3gs9firW*p4pw
zAbe()NpIxkWgAVNJkyqXWI4@e_OUIQlOz6qx#q6FyRt@Gn4LH2tNca%)_=B(R~uD)
ztu%1m_<Vid^C@iaWa17l@w^P$&%XKD`sds1p4u+b|95Ch=H(eCxv8&`)eil?adgSc
z*`1XSP2b7?n0S8LvbVn5Vq;<=w&h61?k)rEYis2e_lgivvhLiGnew#K_~)GviS=rq
zx9YBlhOGs7JMsTF*F?i8j{1y8RVAkkRv&V8U9D7a&$1jg*m0V%lDAjNH0e+ar_T%n
z#;v!k*yG;G-#>ApU(9Il@r*jVI{EbV$G&`c@6YTX-Xo~~BQ5LH^s{R`FI)UQ;c<`c
z@Qj}e+xV{9C;nTrvF6|J|2M7AZ_rq0#&du3=F{&sHFQmk*l;8+An-_A*WScV)*o*I
zJ9WPa8D4Z*eC(e0n`3g-{l}VOLoVFV-niYvMU(sX_um_}ec$kwzI%O2bmm;k^j|j5
zZJ*cr*>2-o(=>Uv`r6O`eY+WxXP7P%znsLraq(vJ<=4)qtT}q*UF&I1-oM>PXFD0h
z|LXQtPuUV<HD{0e@__Ag)J3K8p5E@174-jm^jR`nU+(>57eBk-+)~N^%)5SLczD<S
zUuJ1nH(aZztho57h24JJrvrOkLeGf1J&4X@<DYz#^{M44<2_ZCQa{A}Di1vKa0)w{
z>_21T24?9S8=lKt@B4dl>n2}~Rf3oP+J-K4V&#*yI<m}ncEr}KtG9)VtYq)*DqWm^
zf8Uz8y|eb!{#N41|6a3V)AVPz7ih`MweG)ocY2}qRqZ!V4)OKOv!1?L@9IbW>5peV
zeWDe^@a>JWcb1^WbMN<k73t+{hQdELpS-lKxQbbBp8c`XwNquDN(BX0f~JEeug`P7
z^Y*Z~k%&%v(mf`l&{bW#FFE{ZJ9<X)yREk0@ugi+uPwI}h|T=8f3{+Xqod=WpU>xm
zb`oD(8!ap@K7HN0<datJkM=GuDmkS7L#Sx!XZ89uO;IUn(Wd2mM~xrl+-lP8-t53u
zDOF}Ylf~%8ymKDc7@}ByZNEI>VGfh?8{>@@mfTM!9=@6KG-S=1Wx;ve_jRu>UmY27
zPgHPrmEne1R;$uiU!rbjn@FvT-Q6~M@@4h%yfan0w^GE+I_Ep&@>EUHe495fULb0%
z+|F+9Q`gsoufN*l*8FMd_q#nSL^ei#X}GjRnN3vhRDZhN<Gj1O0#^p9HZ?Wf+p&gQ
zTu(qk!egOR>kPA8DLp;C_j$)l#a}*@Js`Kb_1xxdmlsd6y|r%lA&2*AIa{Mv{QNBM
z^X<Dw*}*L?uCp)hu-oar<<G?}M$<0-&YT(T(9Pv5bzSe>rKnTC^Ak&7G`ZYXul3=n
z3T*CvcJkNxjhSAluT!c7j%_zhzx4P)jMwfHitlvZp5E?oFQ2pccJ(dozq7xEI?XIt
zB<=rf^TpVx#&T|EcfT#lAJ_H%o2@o=`KRm~OkH~p7jEL@<OJ=advkkx|5WYpM^B$l
z)q5AOxk*Fu<3Ty2#&u_9B`-#}irc%_@fR%#RJ;ADtoJI<>eY*?xz(R-y=<^~>#Vx3
zp=VydInpr6NGmMh(ZZ%VOG9|pDIDLvsq18K=#{mL(&p`VPWoxb>hdnutX9k^@X^l>
zud~5Uf(|PSD=QafXmgzP<~o{wb#)Ev!VK-lodE)YT}m3J@>2t9W}lgBJ^j>F?aNEO
z#l>`^zT~wkHQaMq+4xxE#qy+gPu{xR62E_ED_7)v`AoeN7t~Il-BDa(>C4;{xi!lu
zPEJn8I&bqPAtSCH$-Bb*>x-s*);hUkZF*%zow)Pts!wkZP0rjR@NQ$#83nih-=yE3
zsx+!OeDnD&6WiR^UnczJJ}#Wbxwv-zN10u#nbrBN>ce|3czzbzy=(ou81G~AUr(rc
zba+eAjsLd))UW@&SL-x2YHixBEt#MhjH<7%3=<ErTypmD7wQ%glJDr68am~4PU??K
zEWh-x7R6{U&J-4nyyLPeb?U}SSEkx+FD9&J(UZMyT-|^1+3bwA!wakQW-2}kU{ZEE
z8=|SD{;#d^qqOi2#mN)pqs2=UTc=*woc1!s+oR@*i@Uq9fWUz*nU`<uELOi?^Z9J`
zs+%5jjM%?Me>+fAXcX;l&!!Y<XVLbp*L(7w$A^Cz^jOK(r+zrJvgG9_;oZ+m6R()Z
zS|rNrum13-ZF9Q+$KMBT>2S#%vX9gb+#Z&==<<oJdw1rtC4Uipd26pWGkaLd(+P?;
z8h?0ZFEh!V`|aUVc?P5WYiBr*9k-huIMXfm;HK%Szw?;-Z&-ZM%guULJ2C0mh3ao#
za&B$W1l`N<_0`oY>*MF2IN{-`<K<CP$5HWA%<!(^$0HHSUv_bd7-{X`XmBty-(a_4
z5qr{N-6N_I9e&GxKe_R2I-7QS-mF5+RcYJyPntWs#DyVFj_Zk)@b1S;H|r{X-1zwM
z<L2&e?%&_uE)H1e^nUv%3*njG*Oo6<FiJhjm^bg&<c@W@8ZWda{Q0_QuRNnszDSwt
z*QM#cd)5lh*(clmaO2c{rQXa+seh~==Jr1LQ!6`jj?og$bAE4c`0eBQIOk4vL;Je3
zGc3Nk7iC}CmU+qk@HNjZ&kJUjcuuse?EBP{w;=6vf3#(NmU;W1sKB7lljhAkdT;&m
z^mli5-rShXE+;29MI-Rh+8a%K^(K|}wH&clTYLPr?(`m=unD_6cO*YCD=@yqbfRN_
z_LB$6?5j))Rmw$XDEGdpXuSBpE!L_hb&83m;&Sh3cNsC2fOQQXPa>PIu<Cvk*|q!l
zk6KTSlz>IoU!M%#XW2V{@`4XeU#?3=EEBw!&3}2%gVTNAuWU-a(LdvMpU0eM{*P|P
z?5~@9cX#>X5Uu<F|27GpaN7JacgFpEfv?*|5)LlhwqXC1Z5up7#FYF_PRiryRC~JZ
zDPPBu+~_<tyK`OD^FCeqb~-m+?DSoY!;_mPZQU&^FhB8i$?v?VyW721ZR0<->uEYu
z)WHj-?r{>jK3~54Vc-lc+BqljaGPQ3DUlwx$Lu@*DY%NMs@&6bQj^TOy<1;UJ$>T7
zuOS(W&nytooPKiKo+LxdBQtKMPMPS=ocNnJ$9`4v0jt`iFM)bZnvs!<f)u>fCdM;`
z#&BKz_*TnS;qY7TXz?!-xr4;F9J#z=)AXbJ*GXSq?(ZJ^vOZr;gz<@&7kjC4zFTy-
zyy11r+SjZ%=UkjVbHCZFraY;aO-s^Evhw~{GU`q^z1^^`i6?coUByn0&Q$@)+f6Q|
zob_k)w%C0r_p!)Y{`?K@E1Y)T(Th;lk?>ixcVp4Sy-D1wL`&V?Dtz!yS@U*Z$c<ys
zr$eP|R!>cQ{@~u5J%0o?|C7_M=6g2BZ`v~7%bRoN7Hqj2t+sjI3U*I73%}M?#v9fM
zMLx{?vFzdw!{cs>r<ZSWzx-z2JY5?>`^UE(x^%oe_9)M}<HB}*)80Ryeyo`JGWBRn
zL$@2l;?9@1zg)>(vn;VeSyf4~ee=}F)n6Js#9AAwMMdv@+w|zY#EoQulk6|{PBE!l
z@pylP_R~`<Bq!cJ;TxFZuwd3}s|&|xTzD3)U_R@noOUdG=a!Q(D@3lcNcv@6HNJi^
zJNd%wqsxs~Bqv=^_cs#UEO=wJ_oF-KUbCC4Z@OFN^<+zJ$JtvFi_Ut*#7<hIR6b9I
zr-tR5`F!nXE4a@{S~pJMcAv3f6^A`*#pP}N_M)q&x~fe2D!gp-@hrz}No=b>vps(Q
z^z52c`{2c)zK?d7KmT^F*eA+(l9H6=#k;Enuk2Evc5j}{?W*ZN>-i2p{G*qaZ{KD2
zCqC@DRAbF?wGAyBlP<=WWZs+q{u|4s<tz0WY}0a=r#-RzC2w5dbmQ2?q>C+i_nLnO
ziQLw|vgqlhhi~$~Dwm%2brX2K`Og7~JEr9sxxC9<j~ekrZJ8aG=Rc{@C~$Wvcc1+l
z=V>9w|9tqPx9seU_Mel~H-|r6P<prLkI1xc(aU!Ei;F>Rk-zSycFM2kOwZoIof51X
z`5e@)*t&Wqqcm?yk=TXPsXRH44jf|Toa4Q9?e5J7-bTC*e#(*6SYwmB`}0cW$me^Q
zc6D#jf9$_Dlsz-j<m--&5l+kg9NAR8Q}cx5wAhbHPY)$@?U1n8^gul9&MBv5ue&WC
z$IO(OrF2DGH~HL#mnLy_4N(_Yue@B!wW;Z<#Eo)>sl2ma><)Ha{pWeaI^KlR^IQ${
zN@cZ*mY<IQqVc=Z=kL#D*DU?~{Qi7CZ*O5;&-32b*3htO%F0d8=HJlBj4{!dU3xhy
z*<qdM<*etKrL)`CPLAxpwxnw3+sk?)xjme{-JhE0UF?xG7Lu0!eQ?*pr6(4(9AS5x
zJt5}G%{O^6XFu$pRP=5pYnRxR<h|*2UK&iDPKKqH*I2X#FRh$f;rp(|XX$k%|J~<f
z_-)$57P#Jbl98P}dBMr@$oeXkpYp!~?rgV}z0SM5>|KQ3KO^BM6BZXmvx)!VJ^f<i
z7M+a5!p=&OS-a+{g)(<a@9vg3aCDx*hwPJQE}i_R68LCR+YX_rQgNZhnI`KKcvZCK
zi<K^7J1hNFzB?o4$oYR=UddvrDwjmVj;;55@FZDV?x^gFP#?c5mCVfBB$os$f1G!r
zxa>Gv*O$+mq=mXGu4QyvzTpT@UUX3CShI$9Q?}S1!4ne#n^&B1*~a8=w?eI_;ouT>
zlZwADjvsqx$~kLO_pH6M9yo1kVO)BJRafz&A-FWVG&`op)VOl-lo?7N*Llud`HSW5
zU2U6!rqk7Wi*uhgvYnWqEL|X6`Yr3vwOgHfSKBWw-`mJztM>S%8vB>bY5W0ee)6w5
z8`$$e&pxpFq3br;_op{5eVuW0r*hC9Hzy}>1A<l5M#SWE(Sb;o%{NaqSEoJEW7@y|
z;@++6?sTP2-J+@Jp185-;hILy<3Unx7O6rngU&WG^@SbNt*zS|dB>$`l|{k3^(<z4
z^O(Aw4(#umbZOB{N9!K_+Ih3Y1ZVEGdirnb$_0yZ(~NDmODvuJGC6CC%l*{;t4^$y
zlTwVc*ME-Q+xbNH6369L0_%>MEZot3z`pw0f{bfZZ9cK*?2eh%x!Nk|%*oEOo3X;I
z^A!&@HOHJ3-mKy5Wb<o=yN>+U$!UwNHvJ2bv#GKA_##nuTFdkY)6d%^KKSf-x^bzu
zrC_@2%6_*K&rX{&KCS#avAoKuEi5s3qpQz1{XdtY|5hjH1?Po5xE>~VNp<3pMJ-2G
zUtG6Q`Rb-q9Y3OvpFi%yan5L$Nx>24WVi3rUY&Tla7nPHq@p{|6YX~nAA_{2Y`fA=
zO;uq(w!>Xa?lj93t=Gk`OHZpVPiIwqC;9MG_PhQCVlTtbH!?-K?D%XX=+iiD>(nWw
z(<ciCulg&zBFv@l-x?JWd$lux;xo*Oq!_ZBstcApm~J6-ewwc0iPQYg0zRwlaAUu`
z>662+lG3EttyMX)(X49(WvsWgOZqwsonB>`q4-wzyV13zMRyK8og($YzGzD5h0U?N
zeWmj~c;DuKk$mhhS6?qcFG1wh=G!~MwsE>H)p>EAg?DPS$)le1*k?cb6}McU&EYyb
z=jWx}%J(Ga3Lky^vRP1MRmdC5r?0@hC@b?RdhIG=sw#24OmbeUQl}QaFJV&dyXhLd
zruC!tBgwmdo@@`yCLE1aXtvRemSkW}U{OzR?6K0`K7aAlKU}A8?6YaKPihPnQ!5Xf
ztobwAS8+l--~Y_Up7Zygvzz<!?8{6odv5Z^PlV~6R)t7*?K;8jrU%dbDCOjN@!-hm
zv&@O9Klx;Lb7@Pyo?SHK^JGm{bI)L|x{!`_t}kp$dG`kE%G+AKyg$|2?@>y2^tUYb
z-KFQ|PT);>vArSqt@q-~uO`fXZ?y8gUC}w|12@%}g|_Po)y!CI>yu&PwZM$=o6Zde
zao!iJH{W+xEp@C9GoQmfbMMoNr^WT#78qxDc9|_M^fi6IBhtHPTFN(HrK5kW0?+)r
zes)6UmRIu|Yt`p<xY`B=RvGQsrE)jfUuad`<V80pMNFAc;cD}_vtjCo<t=@!lDEIO
zC|=mQQ_6qp)QFUtg_}0z<=Xh!Wo11V=<MA1;E@vZnm6~>9DVcF<Lw%wq&%q?JI@}t
z{`E@DT<&N3eh(a+CVlXdo2ealsWI;9(>pu)cWyrA@cq%tIV@i<pRUnfkrC{3tk*;P
z=cZFP_C}tb^!(f}B?a}?aEq<0R%L{G^u65Jx&Iy457DWG&pP@zm*-?JeR}Nq(}fP}
zrY;ltld}J#=BDtkGO@dw_MYo&nRo4hn*W14|JI&(Wpe22OWlhq-j2S3eL>CLC0VMI
z3ZK6ys_NlgTDDflXa2rZ&pKaxRps|z`|x)Bp%1#Nlm7*BpXI7+bngwC^Yg9xJCi?i
z@4q@?yzY9^nlNtWW4jFg%w9jk>V^L)bKL_E92c(PnCn*9o8Z0T(`2DlDjPKvKc3=v
za$bjd^UUnbp5ME_9&kE(D(h70HlvC%rtOV#x^psaxg2fy>Q=+-vPkds%gQ^aI?nYy
zIjxlZ!cA7y%KA~p23Lh%ha-x$MhiLeT18Wip1HDFj3wQ2?<wmI`Jpo2atE*T%KWr?
zoh{?3X8G3pM8qbAd3W6;7h79TIv>hb9-k|Bac*^K#+AOBxz27o?|dltaOzt3Y0J69
ztxQVamuRxu3r{hbyG^it>gyYOYYbDg9L}oE%?t|Yy5_OR?pk}|m6t!XyZPrY+|unJ
z>>r+0bGcQieOke`vmCGf33@GGBo;F}EY0hiB+t6gSu%pBXLiU`e(U=3bE(+x>D9)c
ze>?p9n{8lS6rQ{#p4Ur)*T(F&lx6$0b&H*+p9?bi%{yUDv5MXGaFL0IVgji<te5Gw
z>?!J=@ot9D@>|?{^O#zB^4By>6&63!Ai7qvy5roTZ{BG!pH6GG^_;r-;QZ#3N2WPF
zVUM@`ZLVIxbYf<-OW({3?%U?~uD>}sT46m$8u#>`ZM7WpWRG9iyx#obpSE19JcY;H
zXU^<Ss%-SwTI$&D70gv<mvJZa!a{ZXez~*fnXMlwbGmk#yHwjVrQFka<Mqz_{H%&-
zWsBP1h75_%^BAPKrj#xS+2+%saq`KO6s3HV8_T9%VZLdbl^AigTj;D?;rf(duiYG-
zu@_9cJ?G5>XBS?jf18<@Sc^_y5}C|trCnq$SpIv`?1|rW&a1p-U&3?#y>*@f<HU1~
zT1TYa{8Xy1v~3M6OpUhTes}B0ic2ZK>&uSLmTm9WXwaBY!^hYEX8rZMN*3HZx*FUZ
z`TL*mm)m0_a-vkN%sZIN%fn-jSieQ3>ZxyXzBLo=HkK4EI@e^^!k2w+3TyC;;-}fa
zu5It#WVSSL632<n)6`QwGg+lX_&#s5%h@9~{mDyB)ez65K&MBgH_ERZsCW3NrS&S^
zE9%{}*mF<#`ew*SJ17R7`xkvQsi?znv-Y`ri?*^~@oC5q@4FUZ%q|yEa8jc{<)WPX
zf_iIrqq}BBERy>wUE8*X1;?bZriAFH-u}ToH8=LVc!&Gyy*JcX-|gANpfp$7;=w|(
z59^mNOP(m+Yq0TF+z!pACZ@g8t2=JYJaFx|7}xI=D<<W{he#dxlX7P1T5hXPO`fls
z>e}zv?w@|q*80*yXaA~iJZT5pLeI?F)jDH)#PPX{vtL}?qIuSeC*|?Q+5Ug?`gR`r
z6}kUH;e}n|*H-uMI&?~td&h#>=E#(@hI4LNwXc-c-}i>?j_v-*hM9Aw*Ylrw+Mb%e
zRQhFea<#nc@`z<mPyC7h5&Yf{+@txwb?;2MdkOsktwlSH3tnIGl={16f>`O_nKs5V
zl*$e+FBABoe(9Q*>CNc(Rkb|R`Tm~N+-bg7d`?10sc5zT{YMurm2Sy-s9EZ`f2y{I
z;z!9OUjyTWN~UJ%6&xG|i#DEXl4EqgcxapM$2GMI3Rzc7cnX%ds5~jyW|Tcu?*FW;
zDI8bUE<I8s{i$QYm!P13HB9b38-7mdd?m`d?Y45>+gU3<1~A53UU;niyZde5y!q#)
z1D}emjP*MEUiFP{^5WK2sxEyp{=2iUpY&bgZz}4=vdT1I@&@b1UO~2P&qP1|KGpay
zSN4Lt)X&1(#g{gV8tuALqOhK6N*>p*@NJ)dGYW5&UZJ^iqtWYCl5fSu{%kt+@<LE=
zVkOfpCoZdLAq^Qz<84(>_s!5;wqaA`tEM{l8J0%-?HwnV8D;#N&!aJ&d&~D5ChH8H
z&zCLtJ-<EJ`_4|$D{qVDS9Y@%mHn)!dlMml^jX2H`l(aSsqA|4i2JVPA!RPR@6D@s
zSNGhXu9N(0lSCNH*807&m+q(iJye=A%TD~~f-e2dj1Q%!ZBBh0V%EX@a^5TRlX|MQ
zcTP>rnQ7hm@c0)2vlc%0;z!S3oY)_|O-N1qaMRx27oj&idc#aR-`|^i+xom==2?%f
z44pN%clG;DFW>aivpDSH#Hro35nKOi{Q6KI8Cx#4)jQ;<O&ZI-r|Ulcp7ZBRuE7Mw
z!(9a*{Fs|g&CIa05Z6}mKX2}~A!=>dOB0Vh{vmySKUm_O)|L2~|6UTvol-0oAh~u<
zZA$Gg^H8nVbHpdOGEO`;eZ94CSl(^n#H>oq6RNj_QYP81O4zpH$Bvgb`8P)?wC|N!
z?xW<(-JRgK^E{VeUyJlkJ@(u~tKVlA6{QL7_^MI6*G|doYf{2)F3`~}U*9qJwq;*m
z_wdP+FOL1nzwU6=Fu7l<{H9m+L*;zcuH#SQ>MxafxyLZ>HDlWw>650-G~=v7*Nm-E
zC99sF@0K=7)sbvw3y4x~_-f|teMSBJslPSBSKd8Lxh9l6sW|(vLex6V(+{uycm8}w
zEP~~&ci)oy7ffC1@4p$pzWB;vY2ZGWmg8r3@umJ;d`u~9?~=w%?=EreiI#X%9BFUa
zS$}!Sr;s|{Xora<6IgGrjW*W`UF8xGa3MSQ{gWq0?En91-n`j(zFJjM(ie@`H`gpL
z)?dC?&TKC)>tz;fbK>EdP4PQp>}E`J4W7rmH;h}#cTuUIPmf%0H@8yS7Q<!BpLhS`
zI61>mdE>^78dK$;MJ&n7;N*0eouK-t@u}v*89#1ocP$DRjK9^QvXJA!0}bWnvpwF4
z{6Bf(-H$&-YfhdjED;OJ&Ht@GeYVH56sM1&-%TxF`fAj9#n^8WPYGT5YNnOaq~84c
zJSVm!j+vL*^A3xDxV`?Z#>E^F@uW2^`m%lQCnvG+b1zHTSu{oa>fB_0pO*9Qo7>yj
z?bq0G9@sh6K4{MsAFi^$_tZofSyUxGghKiK*DlO_nYbp>to6V{Yk^Js&wT&)^4OWP
z&HFE#B#68Y?^~F{dHdfUhhK>;BFP2o|C#dg^YhQ@zk6oogpU;>`G<~O{;&3LURdX(
z*i*;bgv6(-^dH@EdpW1^+NT#DKEAV~xOwt#vklL@5?6`VM(Jn^w<ZNyZgcDW;mL9I
zOT!gjUB!v#Zd5s(zh<uU_@J-w;rv-kCjSf+I(^jn;R&uX)oiKePp0eE-Qkq<T<<#X
zn@1G96FT>U>Al6_fjb=K-TvE5n0a({(Z4#G`z3!G`zAYgT)APlrO5H@g4EoCau2nO
zdT)~&_ZTa>8av&IY>MJdzVPt!lu|*B<v-e|&5nB<u;}oI-|uW@`}e5!|L8d38EW}I
zUMBUf(Y_$V%GX~Ko<7dr5+Qo-n}%(&`C5;Qq4V0ZroIWXE1J_~nts#7ecSu^!;(gU
zyPRU%dbEE2IkBw;)OuG@vE!FH8K-1FjaB`pf4=JJ;E9H|>E$^KJodMK<6qv9-n(ta
zZuy65t`1>sMRIu#HtSejN{SkEOr+%-;y<YUx}JY_o9xU?(Fr<E$-B;F{J(nU*48c^
zhdaHGT`ayV+0eZ<Z|C9HvYub2PCmgkO|6?_LZ@B2-`;PLnqn3maoX9&u@bKTpL4lR
zN_#WmO^{uZ$&@wE{7+{6yR_1S-Q(^BAt525ms}I?xOz3D>QB$lxSaFYI!$!vp65kL
zYE^AAH-onP^ZmA<=#wDZ&YD?4SGP>qn5>qR6E$0>bH9tx{?9Ho+twdC?#uUL;|bXV
zMYAP}FSY+Jy)g4jgwmo5>UxI4Umutj?|l64@h4H^V`u+HmfQW>x!S8tQ%DHZ`uMS;
zph#Hq%AJXxU*~;0B6vmVvX!-9j7yb>>LkwHtJY@dcJT1cI5u_0l7}q6*wZC$70Pt|
z;Wd!uewfvhd*=nDuiCKQWLx!_XOZgej1u#5-i5T3-R;_x`qK5)uV-8Jds1u9SN*xX
zfme0s@wL&<XUf~moGr#1b~&o7?LOm!z$>w9uI)&9^ilG}(}dbN8bQW8o*v7ZbMi;(
zLYJ3Y=S2M!Ha;Z3cY(mvvtp+dE00f~r>m^?;P-R6X<t7+oIcHX=iHq5GaCa>eA0ZI
z6sdo6=^o)PE$e5UodR0tmdzEs%qQ{Gn#eV`sv4pV<&RI8e&w2<dH?^DFXq`s{ylT7
zeVXp!s?8sx=BS7*Wp{9TbXVy60jDJz59YXq&%2uP^qsTu^~JiEIwyHJ&1`?5EdBG}
zYhja>FFf1l9c1~Q<96C{cZS(I`=`=854Oi~MTN<oXo}qb;quF>9NCE0;MdoL(>RmA
zZ4%IH&wsxDv|E}_<YC+08$Nx>c751(S2L@BhwRU_^3F~rIZ}7#z1sRZaD5zud)M!W
zfk*o9x0|Fn9k&w64{BN<nJyT!uHeIy{patb7GGhV<}<@@##b)mWBL!Arf`PdYtxbb
zXK&W~YnoQv(;44(@ZG<D{_mT|X6{0pvgE1{^|Z*|IM8PM`~vesZ#Kh>YCf;0XL_nP
z8b$b?&bUx+`2Ea>4}bL3Km&7^;?KmDHmsO&8Z<DsOKwu^spx&0n!*ANulp>|?3*~*
zEVk*x`p3<7Ia*(}s*I}7U*+GH&VOG1@h|Q@wx#9$8w=i*w4Jw~FhgMK>H@|aoB4Im
z-!S`S^!d75@C9EfP0h&X9(&3!EiOH`dzmA%`@Uly7sP~r&Q(-0n`+CiQl@DInoyjq
zWNf`QwJAy=Udwy_mTSM~{0Lc79P~ncuh`nKe$RDBuDX8yl3x`R_1tE^$(J<2r9VEE
zz7ALz$kY*IT{p#gnTm6TwA+OA<#WRCJ#Q)bk}+{{VAqV~b;5PO%rt_gCWQoUaq={p
zai+uT;=c6@MB+{DX4t(Avg3U%qpNKlk{G@>^~_iOIXk{wnY3?<)aK@iv->an&{jW_
zX1plFY4y~_7jLXf^$9V_FnTcQ%`@ZIXFI$u9D5P4Gb6~Q>GGCVFXqe52K&}pPL^YT
z9(3y|Yo>&Mq;=-c#hIx(i@#qs+TI{0IxFROxq6BUZ&{U~-Kux&{Xz8^uVpk3x;j0w
zO)zb?ncZ~1Y03&$hwP+RL9#!yR<HOxb<S<~N0L>iyaI$xpPGjKb5>b>@x+6vRi8q`
zE~;%ma5SRv9-mrMnDmF$3lHhf=GolLlO7RX(g|A&uz6<WY2kor^;Nc={qlh;54x_b
zJeR*d<d!%0QT3<l>u(AaPd;7#WeLwD>40Y^oOV8ETRNrlsl?+&JBof9SuD{yI$@&C
zJ=Fr=I|svWUw?9ZMd3uN1@|@ErrbTJ`qw%z_l=&}+Z!kEW=GH5_>_ZHphMvG#>{l-
zurkT6fZbx-)6b|Z{H>y9R-Sxt$H_Cdi$oSS@Vpk~PTHB5Gxw%wd8&+Ja9z^Py>~>O
zD4bqbv9T!8|6jD-jCIDnYxrk3rQY_L#XIjT|7xcbyDDuGrxkB`Z#-}2;p)ee|5~2(
z&y29<lw>{D^X}yO<9&=5^3KTfeE7`pMP{neD^*ZG`d;9HTdIXI=axQDk5|2aCE&wh
zM&-EplLvX4qTDLYCB?U8&XmyHv|;U}qH}CrVpE0d?HWI_>+o=fOuFyeSYJ2s?d->$
z1)&%29&b!beR5<;9Dm1zbGwd2F9|TM=h~Ynx$4c*%)1M>lo;8ohFGqAvNqvMspMPk
z69Pwm{n@nH{95#P%PUJ|H^!P9ZklGj<=ayCR)=Lz?)ofFDteZ``uwV;$r=lIm;3kj
z@!SrXyKv_9WtrSt-?wKO6x&)&?s*hnp7h{Rt3<(qMfD6iw-3#-JSbA(8@l|%pSED`
zXF~f@81A$1AKZS(>QvXao?nxK+-yCPL32I*ej*1;?w?p!v7_?h5mjZ2Rf>8Cown54
z%rrgd^Zx0_JGu)}yc%?4QjVOOIXf+;Yt_uR>r<Yq$!?mn?fAFVN_-)w^a_j)9k^)y
zZ27gxEsI)Oj#MA~)sda3_UZXaKQVR2i@QxK?`{fSG-1}=>Ah1<<$k|fa?0B@pEX_X
z$f6>Ki@QyBnlKe@zhsiROzoCH{`S^SUYXpYWi78iZks0*-Ew-0g~*|l^UI80#^t~0
zeTX&>`1i$<o~n~qV!>muFV1?!Fez~PYo0o&E327ncCmP3A4^X1VY4@vHvZbec`?*)
z`nJ9bO_l?P?%b_Xdw;>Wa(iPoA5(kK+HhCJxtX2Y4<G*ED)L?CXzAb0b|?BRHY6(A
zTJ!myF{usxXKOHBe%jore&5_X-rbU%7IpT@r!$$=hiyIerg2+vGJTh`J*hTrvf*@_
zjndYIwk}4`|8%V`yq12;Nweo<`_(;n{Q?gQuh(>$!@|gZCdh3ss6AoFpmajE<;Zcp
z`P?!4bE+Lrx*E(>3|F*_+*tHvM`x4j(-n-ZQO71Yh)#DZt=a9xe_}%B+J+PTUuAwU
zG47Yjx+Oa8vg*~U$dF5!(<`hs_UkwVT)(u#&iK?Lr?_%e#YFQ<bE@7Sy}7E2Pj`;%
zvOn=2>ejKzn@xXN>t*h{&>sH%;?|4bEm(RMh3;IVAv0&vlR2-nSA`cR9Bi9hw*IVy
z-ny%Sn_H|G9L(xHd~KtU=ZPoWCF>{j?{Sq{mdPD@;ViFCqD$u48SmuMpQb9S%$T5^
z^LfFnGrz>EE0%34W7^;rVK5_SQb754rRt+%A!kdQRUfJ@>G@Tzc3xFQ<<gUHxyDlp
z6ISi-I2!5prczc}FXF@cwQBMUz6S-pn$Q>_Vv_T+<$W@@8p|uQpik>P;WJ}5%8xZz
zt?%*LDI;<JCSy_$Ph8>>U+Z>}*Rs7+&+R_)dSQR3qkdMw#T#~u{4BLEt==qI{Hk<{
zRCU>E55E3O6`O^p%AC0+QKBVovPd_2>;M0Ci;o|?^kidw(U!o#1t%v=sr;34Z0h|b
z{l!-Mv^nk@w5uNHd2y&t^+%ne+?-o(I|8OFZZ<Po9~8HS-)}?3$-Fayfq(XGwQ#IE
znqid0YQB$i!PO1fAzmEn&z8IXdGVE1@c)X9F>`#{6AP#8+}JT=zSzahE^bGJ{{DPh
zW}vIhzr;UWeOug$<JUHZPS93<8GNnr+*Mo6D+^5Dv8Kx0*wuZDS^MHGDWC4HREaGa
z=4<c7T)I;Ys=&|BotyFY%6@n2t{tZ|Z>*JaTw9&=^6kluYs+W;e{QhmxLU#5oXIZT
znw+4CG3PD5_Rg>8s6RQQlF+-k==P5uQ4ywhMcUhi9Ttlz9PARiy?^u4O(!dFrp_0h
zzo_)wZKYp#q-vPjgLb^tIA&V@Y`UARMc_K-dY`wx-#YA9T=|x7*8c6;i!GTkD$459
zb(VhV5ar4Iq567xf69qJ8DTAY3w9hkFp+ithDTvyMzQfnB%hypazSLFe?h2P7}t}$
ztwu3imTX4(Y5DW}>O9%2^eo#W)|nc&mp?doM}9}>lWd)O9>vEUwl%>f%KzJXmgHG&
z)Tj)vk-BD?e~rOVGxm*{PeEpCI#by14e5~zUYCWt=cYW;^NrRpj#|sv>&BVpu_y9O
z)D|^S{&Q{nr)|AFd6$X*o9Zu7NdZeY{QP;r<C>+A(0sw|aVn2`Je`7OPuQ}~?4Xy6
z)2Yz-vY9O`4sA)il6Mn$oQ+eSuHJKxg@5JI|4yGLhbt$PmL2;4d)o=8G$Xh8iEsbb
z?40#i%h9>*L+RSmkdK<IpT8@yui^I>Hu<Ud+P`1rzq5ST?`_K;?|N9XbdUR-t6rNI
z6mFTLl^pjmYt9bQ)7wvO4i!FkDP``gc>f=PH?B|h-G0_)W})jQjh|ZrckP%E^K1rB
z!p<z$bs=h8C!Q?H<yKYeJoUu9dxfL&zI|IS2bXG^?(mrI11j`{gyuhdKI@e3uEp7#
zJ6_M-TUqkf;8?7+g=jGM-${2vUK-8{cHTZ&KfciKoo&O#t&+macTI{`Rw(xC+A^#3
zVgl!5$0<{p)<u4BQ*d^x{^B0i7StNkSTSd>YuTLr{=d7*3Kwi{3Qo&<ZrxFT^3j*s
zJ0?xrYV7>w{x-cANt0YpK5zQ@Q1V;+S?|{ee*f9`ieoPGsn2GQQu7br@&r$d2?@<N
zmGru@U~8#TxA>VuS~bTHJew`?`E|e@+1EzbuAXVL(0*5WD){#MlT&Io-s!!xv}4nW
z;*iJNulbd%w3}l0mN(^vp`^dL<%>6qF0lCT`y(k6RG+lzV&cj9PhyRF>uQ;QZayBf
zOpIe=<P?v`hwY?~oH_dPS8?G>7Z1=7<7L-?ZkFGgg`$kzSItys9{RU%bHt53uE+m<
zzaaXjlvi|XakquGqS>u3jm-u%EBkI=Qk_)s?Cfj5!v{?{WbEb^TAvGeJF(TU@o~lu
z8wpkL;(;Fz?!D15>VLJ9ca5Zh)b$63A)EP`Ca-2Ouz&F8=_a3;s}qy_{I;BamnHvP
z)LUY~q}z77Tny}$T{&0V|F>*><|1ZQ{&T9)yqPO+eHBq$dhOz-V)vEYzl5J$xz?J~
ze_^uYN_T%frd#HUv-c`~{O~94*|{k9eR_vK>d$)kRr_LmdcsV%iuJZWpPT$HdDgpw
z6Z876%?|U<6qtk@pC%o4XLHAg^^KP7UuGD0JgMh@eEu$%_D`$*2mHAo{rtUfY6Q!a
z^_f$ZQ-m_-g?GKG(GyvFYlhQP_F&cB4XY;T*L8(m(M&Y<)w{CA{(j~C1^&VDUR75g
z6&_$%wR+vgjFV=d8U9>}h^?T>Gdthc!Mn>uy{alMR2Oo*PCv`&E%AE$+(+kD?vE%s
zb714!+dICtvaemMo)Nf_=k|N<Ie%3fR<6FbIqNX%u72$UD>lSTay_Z;<+-1s>+bCH
zJvX_cYG$j5Fd98S-CuOxZp$7&qj~c#v!B2DrUJG`<Ttm$GpptKM?Ka5>nelRsKiaw
z@nd@4|01IJa?|X^N3VJPJAY{EZm~2u!$<Rd+4yxb^Vcx+OgI|JFn8*Sl*o&qZow*_
zwDR=E+Z+A&`xIv~7}s=8?Ogrn<Lc)f`;$+Ve%gIf;o2mlhpvhbCUU;B-|%Vv+OS`y
zL6aZmFl7fxEcU2b#S#B&Q`R<Hdxv@L@%=UPMXp{JS@60^c+1se4Kb55axZ^;URm<z
z=C)IFCw#=1J*#<s_UW4g>a3NU{^cw_V%6*~x2)Z>)lKu#t23Fi-}*gP+o5rGi(Lq3
z?-ka+or@MatUPt%U!iK{rp<P0Y$kmv+6s1(dlUXUdi$*6$=H0aXNAbcV;O-rH&4xP
z^I07FpXZHrAkWeny$9lA-{h~4xW=F7m$LiqloOh+lHf)yV`ahB6C%I&hpv=cGCQ63
z&-Eh{@1H*1u`_wWb1~-l>%pH(^%}GP9}JH_VIaBogeX(O(=&>)!sljvM05?n-GLpm
z{ZH!e<1*9lPtG+uf5a%xYoBiNYw<tpw$A!+xayy;9^0KKJ=sf+NTnXs?C#LHbmBv4
zl~JYU%TTW;ZcC=tM5eyEx8~{mJ6<*+4w|CPvR`^QOg+Va7(4iUdnk5!1N+5{n0b;H
zEq9f$R?n5?v|W49ciw@Wf1@W}&1`vHtbJvP0JrDNjY?ADE17>?cANd^pQVBJjFZ=k
zgM(|QNlxee)3A17LFQ|jG?@;kzwf>N6}>$COx}BA*%5<htuafgzB;wE#2ivg5UI9K
zON&{5G-9n7TTki2)PKi){%2bCPM2R0_9d&W<;8@%a&1;C6#w(QIWM)4EBU{K=ifzQ
z(@IvH{C8^ejNA93cj>I}01a||oEEvo)n6x}Z?mg-?}4w%R(0H_Nlbb3FMT}wra~{1
zwQG90!t&Wn|Nkicb{2{#Im4heF)h_)(ITIiUC9q->81XBc>L>{y=jb_JLgYR+wdUk
zlg2WK0%cJ?!78>j-V1JjV$q2d`LbGT`LcGVcM_8O3>&@E8k-_hWcaQYpX3p27gAZM
z@li-4(<)b4*=yE~y)}l~X7i7%txDhb{reQZv({?a*Vgn%nQ~2j`MB0kUT&t}_t>Qo
zHoMGY>*d_d9gj4KzCL`|L|WvOdF}2?+yD2^DC>XPuO0c}`}!`0Ys+7+Fl|}?SM=~J
z);lL68twL9andoGG4noq(AD$(T1`CA6(J=G6<;E5U0Lt@-=3}UD_`W@++O+S3mx*e
z>tdWPW^Fa9`%^Jz&YS=ZyE`)LK;uLew<XTz>`zk+R%Bf|F;H=~!wF69iTye=HWoc<
zlH9Q2%@RQu$*Z4kD{04z7A^=XYn&{|)^~5C!1n`ZR_L<%ElGT^b_08|(XuE#i_<5R
z58bWLWx6Dl{cx4woVhG&x|cRRS$E)tm=ss%g6-d}PH*(J)KL5wggWmQ93Ykd!w$58
z<I~gApyNAco9EZHxGzwzIV?XVH%Kg6eKYeq)*StLGqRsEbZ}UE{1CNFKIXIJUASLL
z+3B7&4eUEjm{#3k{V0}`Z?jnOT%1zJDW4SW4^LRr4sl5MY&*$1x9jRs^Xn3OzsW4C
z?-z>NWL9ansQm8V-`_KDY*5@?_V&@2mzPhZ$op*qPlCyQNMFHdt8|}b)dd@aP4>JC
zmtT%#)OO=;Xx+K{tIxfK>7VkLugkE+rA>>8eHFHXQ^$erxq9+}HAhq#7CD?gmUT;f
zx?!!W&GVj15+a<P{hd0In?PG|n3$PY1})`!eSQ7^h$!(+C%@kZoVF-_jrMj)zodL(
z^`#r<`S-8if9(8vv-nrZ9mmt|{M9)k{%=yJT5+?5eI}E#`#;aiYD?{Az1~~=+^?pl
zW<}6auIg`Z4&J@{xA|Q^sPA&G!Tp<@V(_Xv0@-UibIWyhq)aI^bj=7*XukDL|9(<Y
zMNZoHdNGG5$E)~i8XMwyP2X~LzqsSlW)cu&)jN-UvE|YTpFh?I57}Ft2n)y)pKVmM
z<G|kCVOl9u3*XBtmtQg37s7CA$r7eO%?YZL-z;TWb>#Tj!=I}q?`~FKecrvk;a0Nc
zCUEcUY16e?eW!iqb2K^J-&cEcTduToJKs{!y*VmN-fPY-DJ*B4uy|p_T%(v7d*&Di
zhAFgFho6brSJcjvw{xS^F~hX4o?l+<+>&ZDncw~4f^O$&{+anVW>xU42w_rdUYoPm
zu8rMLj7LyR=85OZYi%tRUkn3ZG@f}n`(y0C$!jbl=6G3#zWgU8-RoBO;~{(1mlp?5
zP1TmPD$!W<<Kf|j;56?$XVdAQH;*rS@@abL#nakPnfHF2AvpQwnQOZ}PwO~Lb?fWl
zxbk57%~MUjRdOy8mKFDXmc{V(ykB$h{@Ft}blqa^K6!F_;e*v7@3%ja<9fCGw1ddY
zP=C*6A->Lf_R5npHfT2f)?!^8y+ZPCRPVE8jTaNu&e?C9w6<l|PDU@2v=hN@XUi*7
z|A=R+rc0VCuhRT+C!K4`>8CPwH9scX9W&poQyDx*Ge|{SG11&$>1(6=dWMFEb8ISw
zroMXHA*!#h|L@=Ld>%IDsoLSs?(8gn`DurL?u=s>Jf9oH$DF(n8C|f>z9TPgcShh+
z3*XsS68Bv_v)}z&!Q1Xtf39seT-~<y*y+RFV(K;=>#|dSO_@1E|L@Uni<jRps+J3X
z^Ue32>Z+uD22A&hj4k#&H9r>R^CoV`9PYpgi`O)*e4F**ZrA3ri8Fg&)upbqTvWDe
z{-O7;uCBhaBCt98`ntkTPdrzJtejLQr4_dN>*c&I&n4FINwI%R`Zp(qn)(_TB&^qc
zqrSp5ZENVp<WQUA6;<;xdZQF>dq+Lr!xL%5sZko1rykr8Ifqp#d2K_(ZmXBu%~z`1
z3T|8}&U`y`LGW_F^;>H_JUG@yZ(o*typKo5;zM(vV3k;!!<Fodr%g3nFKv-NxJFg$
zR?jihr!{-bP74XY{+qo>TJVs$tMR_d*cqLjU2MK=F_TimW}i)Sx!4k>z5V?2Y&A8n
z_G1s#_G<Sn4SvDYmCbVdlZ38UFL&qc*=H5HUTlqeBGr5M%kRk2<)LZ69)hk!%DmjR
ztMs)|<|P#w8JT-oKfA^CK^NfgNE)>)Sg@e~k>%CwtqU$KI5J1c^!}Rbi)vp<*)Qkm
zRqM@UI-9Xo&7@P<?{(z;Yz@n{|6Y;PzhqzBQuNddbR|vQzn|&5%HPlXCv|C3$;1g2
zySHRN@<<hD+<EF$(@oF6mz~rky&M{+M)%LV=QlgS_hlzXzRCk7dE33YDM^)@u1}UL
zh}>rb-J15}<KrzE7n?3z2zYjWzW+3x$V*4}{jIJMR^BJsv#eBmn@(iYuKPjLcFB4t
zns3TYikA5ocW(P)zl&G&jTfumPdT{m<djll`^%f2_`W(NRoiCh=q#9Z{n59cnJOjo
zj~scHrz36NDjD4uz|hFCQEoP0&FNG5O@>##Iqy!^s$St@uJG<!qFT@G7Oi3??`y{l
zG9T+leEZ|Kzf*f<++H0;#p6E`Lp9>AhuMUdo81+4+g`cr%$Fme{r+|}7MGX%s~Z@6
zn0w{ow%ll~kQELI2@2KU-X#9|@-pSr6wO%sbg#3*hBG#qyM<&G1n$d{Tq&5_D}1@s
zjs45xqPF@8Q|F)FQ=%oZHA=Vk_cv7?ou0e9%YzrWbe8O#56=ImV|FGV3sx>lTdt?M
z(N!@yA(eGv|JsVHVT(DgZhW##<-#T9o10RTzrVZt=gVb($k|O!H8nMMKk8~<Mlzmf
zQZoLc%E_5|>hJFBnb*vw9Nn!pS=q=wnz?<G$ri2Fa9@{?`?qfScD}d9JJau8zw-U7
zKTGb1b{)>XwdkSXHPcW-o8{8Kp1xY;ajx@i$J<?<6Sv-3xK-a)i^WLwJ=-hM35p$G
zf2OAAui3uH!g2@ev3b%zR<5$X*0%FTRKBFzqsZsid9TlW{UYE>&*53mcqAl`{fmDU
zknl*0i-kvZ$LZuV=a#QD*I4`5XZ4vcjxjMZ_iMk)hOdiJG%=a-=jZ4Bmw$1IXdIZR
z>@HzZpdhLpmU6I(HDpD=!_RdSp3c;~y+EdI$F~wrulZ$`%l+npE(!@*7c)~>-S5GR
z7c=Vomw(BeP;oy(SImdgaDI;Ymxsw!eDdB~FJvTMIaTrU(f(wfkAWPAX1fcx1g1PW
zF)?IiP^($)tqWVT!+(8$|3B<h-QT%IF3k?-pV{r$X}NgMPVo&ld+!+EdgE5~v~=l@
zE6x@3?wx;<C_J<L<^leq=DV9xJh@H_-IcyJ)i~N@vYzzE)4A_Ym7M~ORo2^ds2}ij
z_@d_<xBX3q%;K3Mi;kP0Fk9R<;ihap!(HER5oc;j|B6&Emenvzl$mTKe(b2}yq--)
zSH7I{EKAi}bolwQ!19z^smqwp&9QU_-4YeI*UG(5rZW4nNPJ6kGxKCM-$iL>XMyfM
z^P6k+@$dKh%TF&9dC#7{s{F^iS1(K6r=6Q~5_HI!RjHP4)E1Bbx1xXT@z|4J_2tdc
zL#n<XGxmyws+GTN@z^7}?!6Ra;PVyX`NyKQH~Y=ClCiIwbKrmjpS<0fJ(ZtN%(48;
zZhh=Y<=2yM-R6C{W4oiQ`@&D*mtui4n-?FO^xDle$pDlOdtdF^B5zvoK2YWX%ZZaS
zy!qIKPaK@3b@P_v)K|sHW{nf(v-$2@_GNNmTj2MOOShHgsR=Le+5N8m{QKi`!e+|P
zXyq0+DtQs0=0DG8b=X?F6L!!Go1(U69o<*^J7Plu<IPQ}lj8(lwzTuhU)q`-E@xA5
z;NoKUle5j=yKlT7WVYo<b^4zhfqyP+7gm3MRq@-sng0JOw8H|7PQ{;Hu}o{#joji*
zJ34u?)-4NM_2AX@&`_V;w<oTY^=&z+x~IUBi}AmdmfO`Z6}wA!rB}}od2!+c8{hm^
z(Y~WOk3Ri+U(j*qaOZ~$+uylQQ+DV3_U>-?^5yDwc6KtBMNgPzS91BfHJ)(a_VV(j
zP0}+<9@t2$y<g_s+?Da=`wp43CYGDVy59GetULM8@5-umDF@po-_V(mykke0`&prP
zpuyBdf`0J<P7@3h56f+7c=yiq(3<|vgENI<me!vCdFR@!j@>)^=iSdgx@04#!%ka;
z>0x(+{Wzw0t@c^_uQ^6?ZJ0Oc7!%WMF?N2rKM77BId5)GPyhbz?v+)c-174B+j4KK
z)mB_$WapDvkaV<5L^o=Rb30$HiIbp%j8(~tfR{pEd+HfO8~5xiHpox0mf1Y*z(v#Z
z>eXxF;=J}4UQV7nPg<?uwCeKkRME;<R;6<iI}^V1Xa_s`-cB+S)9XoJ9Jip@S9IEG
zw>f(+#nnZMynMg>i1l)r(mgYEr|ULMEuQXk=W@ptEse)F3WWpbf2d6{%h<VN6ECaL
z>l=GN`x~X5`H>fO)ktz{*40G`2b;F!-WF>pdU|T_^}LtG(a#Ml12wAG+o;rk6cW*j
z=y<fs<5-{E(K$1VZ#=j)>wa~ZvuO1ML1py~)=E+$f;#j6TZ>ltE8V|Z$RlGoH`aRk
z#@7p5duHi~=tOtvc-_ptx^;tls@IPYXY>8sll7x^?*3aYw^ib3xvHG;<Lf8q>|W$L
zb2Y<Ku4fA;<jZ~I<xD$sX6Bqf&(+G`#T+v&*S%Hp>Dh%V@iVRR#XkRUh@Zaw`IRr7
zI<}<~qCei$h~LK{X|#KJU^#Es(#SN{ny#l6>%Twv`T04hA^-E!)8xOuzJhi~$J$6Y
zDFiI_5(VvAUhX${N9AX=ef##kJhb_J@!=dZ<Dw@XZM@RQ4z+T>ytMSRd%3L2Yne{|
zQ~PRvXIwcl|4ymf#Iu+GPqsO>UfSZ4gNe`<!^-3HPP{p%Ri<-&lbM*qu?^??Qo1f&
zF5WIEAR)3u@wO>*|0nAu8~>cVZ?K?QxK-{2=dDR0b?Xj!)qR|52wLoEf3f=bg5D>0
zCNaDU`Iz$X&&?B0-!J$6bZu=k=mw})uTIUeEViq7_VRL>a@v<DVXt1H3yZI6S3Xjj
zbZL{yywC3bg5F-&%NGk4iF|#e9sbk%N~*rO8r$ScsvGBRxp-UG*Tl?w-V&#aSzk}Y
z#suuURA{&S%=aUcRJ}oG=LoC$c;vV&ahJ2N^EuupyR+)6R=1dL)pJgBA<sMeYG;3Y
zdwcQ44AA+W@pV57{rWXaoxN)UYHDjOYZCbXzmq<8mO0pd_G7UtEB}>=o#3+BF*`>~
zf_q1eqIdPq*ONP@HOS1CzxTZ5+x~hB%jIT{YyU)S%J^3;+{FOes{a4qUlF|+33++>
zLx&F8z1S%*byf(I*A{M9hEy)b<yUU}6>=5)QW#@fK2tk<-I5>gKRwlUdF7)YIel-;
zba77C-}`R7^*b`9bx~zwKuy`*^BSLJh2`b_pP!pM(<qhe_xJby*PR3vl8^PQiQPTz
z++6G6#csUQbfeX3Ywj`6HqT#H^z;<ywAKCpe!X5S+NGen!($0^m#Fr!kIC&TqZaE4
zZv1{C{QY(Q*`>b}H=KVt=gZN}LLUz}EzB{xz39s6&0SB}mFy1xR{eFY@SJ}B$sK`*
zU)<Swnf)evtJA?2PT>z<zp5%IG}Qh5wbWbr?(^*JIVLYII94(4Rhq6ByC`&ZSVeWU
zu$<hyb#MKGKwB|E>z@)56ciN|eP$Z5s;jHNT(mE>#7g$y!GoZS=FZMG58jq@v;W_W
zB_|fS>^gAw?%c=6`6eDM>GNKFJK6B1{d!>q|Gx%YJJP2)I=u>1`t~%6yWjrsZ<SAv
z_|EBbr~G`hQ{G2IZF%5gHyPWiDW=)iE-Z58mb0(Zu{|^2Ks#*Bg0i=_Ko=Xkc8k@9
zIDO=llap&{Z-4yejnC08(VxHH?_bLOTWn{=N2PuH_H_o^)Nwfl3Uv#;?BbC$+MxeL
z-=_Gq<@Oz%$6C%mKb(Elk!eAQnrp~Yr5THKk1XA4bL~+5{F{FZ)pHE5o0X?ukB`o{
zu)uNG&Gj~wp9(%aaQyV-r1Mg*sTPHgTD}<6C%q`K+V}gN^|4;*<MV8*UwoM8o^mzI
zwD{SX#49TTO)@XFWL{pTxKw|SCFq<*zqwYRYpw17{a_XspYH$chRQ~biR%KirshQK
zp0uoVkEQ8)zFo)5?R*O}<b|fqh)OWo{C!{R9M>FWXO^u!d$=r$&&~Mu>9fC^W>oIx
zu$o8BV(Z__9etcU*+Q&JA$HppSs9rbMyXtnkM}Rnyu9qpEK~2Q8TNwfVs<vIU$4*B
z-1y-`fsKug-OS|A-@mKt==AhTn_pTJX{@BIyp;Q&l#9Fj<!!mq&(6#|+{n!S;>O0u
z>g7F?BBpqhb2J_JvU0M-4egG}#o5Bu|L>@m*7)TqInP=ZpcL@-|C_s|4p+X*AOHBg
zkeyv-!GfgcZ`j_<yK%#Gw_)4eb7fY^|K6OlycpSW)Mf4>Cbp#OI!E$CewjZv>wv8V
z*zxNxqg8S7Ifi$Cf22P?{j>Lb?fyHj?e4$-{&#J@v-0yJFQe=FpSMj_EtZNj-no@`
z*XyXXcFFtJvfkWP*S4zltFD-<D;X2{H9Y=kNRjELlNk>_UE8<f-TP-QFA_gJJw1KR
z_3fFLm&sTbwSdynzS?RfW#y+0eh1C+?s)V{nZCHY+dTW)8qcZMzdt=Qb8<5~|D#v0
zyjr<LZ|tw%zvtusLmQKiuLxMkG}9<mNkyfliIscVx$~)+bCoA;23_X*?*8}138$x8
zJY6|QY5T*wYm4vc$BS`a33NN>`uB55FYl2fs{iWQyep<`;rQK@^=#RPYnE*-5qi0A
zeD3k6ZOgwqWs%*Atx>wn?0iS=-=7b<adcCPr{#=bfwf_$Cn!2w6g*&<ZI-*prBf(v
z<K5k5Zx@xmzE<-3T5sTDx7v$(%$o1I*PJ}kBWXOtJYNoU7IffZx0O}ZsybaM`U(mS
z$Ik9k4!@h3qIJ*Y=-T?VQD$3CE_@KzGI6f;V@rwelHzHZj{mE|QZ|QAipxy6vg(J3
zq?mA$T=Bu*)_1f;9$t^1w)6)7+OMy#gRX^j>zDK0o_F_2f*<3dLx(c1uao`x`FZ-!
zPfu^$zCHVBnY+9D<sF5|CQ`hdoSdMeg|e<{6+JnjXnj=r-TnRj%a^NL&F$NoeZB6}
zayQQikw?mNZL7t;M}74OUnOp-^>td)lBA9P>x%12jdzvj9i1{?ZP|~vphEuaY;*Pa
zeKlw1+s8*uRY~5rZy)Gf*Nw@?A3c4VYIghYi;K#yuC8`p8MJhUeZ3u5Ytu~QbfeT$
zB5CL5curRHO?mFIDgC@$)wegEpfhM~tG3*<5p+Ma$ip!2?k?A>ueNSH5jEGQRBdLw
zzMJufKi@p{*5sbO(wY3?>gx1WH`o36Q2{z%Wnb;@58u8mi`ba-DNS9OlarH2-foU<
z^|uX$kKJNoW6!4D&cCz6ac$JrjJvx`^X>kA2lZLAu6AjKt}@8IWs;MVbEZuG;>(gA
zS?g(MXPZBM`_^}=R_K|m-%r`ZCnlwwpEvih;z5hd^97l~;XS`tG`+Mw{kSA|Zhc<g
z_fxJbwKSL_qN6ugd`vplEA9UG*Vo9MMN9WQJn2$VVe#(WJDaL69m48<GeQGw%HPRM
z)ec{le}7*-t1E-Kxw+pQi^j0kON*YKirG<cQ0M!^4I2#Z+`YTA?yuEyzqv-cL3b)H
zvFPjTV|jPq`D0E&U1{)_kLQ<atUPe{aW=>8XGi7!zctCZVQ@7s{r0w8PA;ybi;G-A
z>2JmiiOSFW7rni`J$P-Dsa4sVj{biB%u7oQXXo<s@hx-fmHPAN^ZEMfQ+My(lgrP~
zZ{wHuySuA2a$C;K$hXR`udiQT_4QTBt1F#rqqjeL@+4*2@vb>-Vw#gKW*k}VUvHLo
z+xy}-yA_vHCN5eM@R0Fxht>YJzt!av?d$6{-*~oTheg%*cfJJ$2KVmW6VZ!_C~ML8
zSbo{C=!wUlKYwm)NMv5>J>6{e+xr`n*+J)%+`c_~mT7j~hyA;Yo^pY@k)S&L-=CjL
zd}o_oy{4h2*4D_(Ui9XM;kEVg`DW!u_V@SbbWPd$I4WX8!9N+c%F-uSE=9T6@6d4d
zO!RSM4&V0m_@iy|>V2|ivlno5`d^6HS7WK{*0Uh*?yidJ>eGB5Tkq_toIJ}kyXf^b
zUB7uYnPrZJS678rR8?(CKHgU^D#G~b>FMJWmEBLwG*%bY3VCp1qVmd3Zt3UdEKEAu
z6%iFx^yf$6X8tY(<!&X>4{@8@-dFQ4c~HLp!HnIDL{wM}UOd{dWutNBoG))~JWDov
z>v`|#p;m5_{ChIy=H_YV=FC)@Z$2mJ&%fXAD=I4uOI`%9@k*t{#h>$0JJu)Ly?wj6
zb@{s`4vkFzGwbT=>Pp_;n!3ofJ1H}BX4OA4NuD`YrCm2Sr-QDCw#-n`>AI5g{@z}b
z+>5jBd|ftqV??s1htaa`+C!}G->(SUrm#9(*FRL=L|)s>Y}!2AY9T49Q?pF7|9xiP
zSur<mefpUhhChD&+En+q>eN*2^sw9V^78(mmTc*3vFL3%iq363^HcWm%iBG<u`zk&
zyPcO*y{DOU-|G|-YmKz3{bgdBRsA4lo&L>J-={RGI2Q_<TWsGY8GGCG@$<~dOBM1#
z;eNQCKmF7cO`}yGJMZi)ZvXc7_GG<StFZNPxn<@WHjpF6CYtBRP1BF>i&{HvuG63F
zqO*%UOueRPIKH@;kaT|SlUe^Yo>T{X5~_S|vV8j0#6SFhRW$awUtbq{b9=u0<z>Ey
zPoM7o@uNcXRQ-nR>v~$@>wG4w`QF%=+<uew=;rkELF-~HZES2leET+Sfrj0z9-Xd`
z=PxfWzq00XgL|9Kll)B#+*7BDJk_qQsnicQ`<`&M>*yzw4t@byS>NyP?pjuV)46x=
z-jmbQ{kuNtOYpEA>z6<O>+9>t?Rm1I+OzI&Jp+oZ;N^a2W*WD@-}l?EjO*k46DJmV
zl*-7;a$0?0cisGA=YzX4wKYpbo@(!^PLjR8?aPt-`Q0o>`BT!<kKfvw{qpW^bI|Ps
zkB{{RP2n$jf6w;SwY8^r7C*PB{In#J?I`Fd;I+}_pw6FTV2zN`CXFE7#TPx!zyJN<
z_ix9pE#DpAPVnsd)Oz>EfBSj!V_gr({}a)P2;ll8=r`Za_w%!}D?tZ&|Npajj@KXQ
z>hJHCR)2p7YU;?>{W$nWj&s%Z*DtTHmv3xr%(=gB?i|bFQ+|qeoRX0uE6bm6+mLW_
z(a$gMZ%zyL&KCOnVUoyGZbQl5n7f7&d3jHdY&<L^^1oR~)vMsq5l$s#<)7bf=W}v%
zPhPa=areW1`+pwi=h=b+O4+^7WcPv}e|{F**w}FJ^Q&7}L^Lg$utDR<e#1}g56}F(
z?{aS4iu0F091far{>*~z=Z}7UOWDZr_{KyBqx5q+u0Q+VynTCfLn8CMdGkWn$H~UW
z$7^~``u+X=<A)CqZcILIQTa(Fbaj~NZYj_i3S6RE0UHt=x8>eWyH@Mn+TyBc7uEjl
z?Z=xB&Mqx|TUn8}TjYuH#iMQo|E9N1(Vx*)TmNPX|62bur%!{9dpp)6dH6shv!{oL
z!E5kwXI@ja0{2#xmc75%yLt2Gn^zxPSm+$GCPGkOUmtV~ebCA&%&s*>MQ0ZoXoE(w
z+;aJi!XzBF72RE`Fh3yXi0`@ed2a33dS@8cuDUsw?a!on_Vs?x&&}oJ=U-m?`&(M%
z>($}wH&uR4+p~A?%DBC|BHyuRUt5#-;6NkjAfTwNStlD6Rdl#Mu54*<4_;f9yQb*6
za`@Rop)9+`6T8!QZg^&WtY6OSe_icTh4>SjQcqjd{rM3dS?1!xvOVwaudMTDw{G2<
zdSZfN%Bhq(^L2YGO771x&Azd(*1DBjd|I=kM})|u;Ob9LI89%hC^$`=T+{nJap4sA
zbG0S9>)$-y<X;}97wxe*?d%eTv_tpr&yU=k_VE3C{i8pc;%s)k|MvFwk6*v)Ho1T2
z{8;{EmTTg)OG~}oKOEdEaz@}Z>wB>mTLXip^cTE;Qnl{aw+G+f&Q{gn`Z&|J;zL8$
z)m00FR^HfMzMchi_fE(8v&W9H?a!UPGI%3H`n2oL(qA3AKIz}FvFL7Z6t)zRa9HX!
zb%}ygjoZS73qQTEQ%nr(Sns$pWK+#cr(Y%6Ctof#?X0Va?E1v-<E9+`w=Id|_L0uy
z_NP^vigufSetr&0Cue3FOWIa#5pk(0)Y5DeFi892u_^tu*M7&Znpd(+GmEE9U9_j!
zQT%+Wy#J<~t`F-?IM~_QHM<1ndxnIF%$qkaW@l0B;dcJX>i%+?T|cx^QUW_v=3190
zJ<ebK<MZ}yHa73oSu30M-f2Fs&-Hn^XWN_muk{@RJI=p&b93_)y;v!CcXvKnD;38;
zj|dT^ohxE@n=w}Z-TBhK<F@;BZ^ubT?Dw91nOd6hGO*EC`dU(Is;aBtBjd=OMXK-K
zz3Y*)oz==Me(G!Iqt%lyUvS!U@aom67Zx83Ssgd~^~CKbV~f{BURQo~FY;XV_M|r_
z9C>6e#Fq&u+D+cE!(z{#Jsg~z4-Ygli)e*-bSZ&O_l<daZtm>I$N2&mm(}X!zVrOH
z>TrO@nS2GiSN|&4<-R_>>FDQ-o12!tk#MR}o30<ftn%}-DLRppCQJ~>$;|~{n<H>C
z_Sct}mNFs^v&-gMKDqm0^_u3m_0G$CkGp*?owqS3JNtCs+Sx0DmUg`naH>(;|K}68
zb30$^r6ryfRaIV&pzDK!grudVYih66ZhiXx-rm_SFE8I*{XMVJxWyw?Nm&_mRZsf)
zd7!?pl53Yopvxu|^BtgDx8iSHT<l(0SO~hPCHZ)tXMo7bSv@*jiY)^19Tzh|1NWe!
z_-MEI<rRU-nq3n%XtW48ad7bSFZZ6VH^WD*Pu|||+M39v8cMDcE?;osP&|^JVl*>i
zM}cDMsVNKd?(TYXcXxQxB9Fk7lt2!}mLEQc4j+DbYpeFgjT>cbDke0sa(e~3bSW!?
zuCY3zo@O)?G@!ZEd-}4xySrG}*}VfqG^at_GylTZ*VjQ>KYskk!pa)Bz=2V-%i|){
zToxZS;m^;{FVDHTsiLC7BS2(j8${dxiy&>6mU44)a<Z_pf*MttT^^=TZ60cqJ1ajw
z3tZ+SDP@x3kdUx|#dYGV6DJl4IB|RwJap*Li(6Z@t>*SUc#y!^sdRPHL|4TYfk*Kk
z9v%;t&#zmQb90kP&W#Dp?EGE<A}gT*C!g{D-rkDJ%D@c?j!w?bCubTj7jcE$jdkYk
z-QDGq76qUI1yH#D`uf^Cu;nH+<f>nMtx7vH!?9CH^~8L8`{(E9PhX&MrVrwe_W~j!
zN50>$_kVU~re(<sfuEnAPF@7PJ8ft6mluN0Z9IwZ@9jM^!?0OQH_9WhB@&djITTqE
zOr#=r6ezlOi>W%d@x<(@2y9v;0P&(@Mux`Ts;`HxU7NPI`g_opj6lsU0Z&NS+^qWb
zMRRY}*F}kk+rGTLJ)N1I&m%wtay?mby3x#tokgmx+~RI4LyA5=a=o=ZU%yrGk?a$b
zEdnP$IKO#yb@dD%wY>ZLe0_X)rt8PE9k_ULB9Ekz$HXJy58uD{zq_k+<CZNoJJ&}=
zMlQVkvY@o|>5q?(HIuB*&9OZE{eFFa-1_6^&+~_ehgTN2PS~K~a_Zsh*WR1c&TiPe
z`EZI+qS4HP;$meht66g_i-qLm^jwAFT@N;~2CfWwQL^j$GVQ5eUQ@NEzPWC%rPXDa
zd~8G6+o-xfAKh(iZ38t#J{{xw$Pad@&(^G~0jsZq4wn)ZcFxJsxp?tnL}cW}h0g4n
z6MkeBJw4^x$i#YYR$gvy@ABp8n_}~u4<2}Qv|BS^0mH<J6X)31&wF=ww>KnQG$&7<
zEd2EJ^!n^wK`W<h(Ac-@-QC^E$;qE@nN(N*PO;wf;`wv;kB^Q<?ys{2-2nLW)6*sX
z^W*M$fU2a4pX?{Refu_k_B7k-vQN*>Y9AGh4_zI$)<CLPzkb`6ElY}@pF5dn>p$Pl
zw^zy(G)M^AQFDJ^ZBgzb0U@zg4_hhItS`4p6R%ynR`UK{ESK^=tMv5rn3$Nje`|tP
zPP*FheXe!+iz_RIOJ7}C7`ZtObnPV@uha=?b#U<^9vvN>b#2YhPwA6Sy1c%&cBz3<
z<co`o*~`kx7*bA7YHecWuG^UU`fC;FQjtTg-0`#JWM$93xVTs|a1l%7=Cr4uo}T{l
z>MHkCuXm5ZsbuAq)yo%ki)cDc(}`T7p%j^MZH=VWTt02>)%tbG-n+}*TGae7aBkyK
z)Y5wO^sM&J&(EJ9>y@5kQ#r|VvfAEK$##K9k57i{DNb5>YioA0iB#a~t6$#S4CYdr
z#9Lim4VqwdaA3&2z3p$(>9^O{%irFfe>}k;!9?oiy}j1y=jJE|FZ22F^XJ~Osu+%s
zFExDBCVQSzaQSp)OJ;CQZEa^q$AOC%1G$v!ZhXtV{o?j&^;zccV*Z89b9F6#|1p{C
z>AA0w&+-kVPR`bszg_;8E#cFXlRk4SCf>MlWBU4yn>Hog+?4w1iKwETv1Fu(=DhxX
zeo)taZnN{M+h%j)`loui?u*$Oba&fJ-l|Ve&YkVt{5(bd9}gcZ`{!P>J~K(X%P%8#
zZ<%2EHa+@(MYG`gyVVR4JByx1JAQ0!Xkg&%2Hk@8=>1&#`gzxG8eM!odrnY~h>BPK
zf`1n$2X&m+=sz~=kJN0Do$m_Sl&^(_fsV-4*Vlja=+V6@UneIf?eKM=X}p@xXU#vg
zaf532@5Ng-smz@^RYZw(<E9IVxti~1R@g7T=&{(Xx1w<C6tAU+u3Zybe9@z(#%88*
zx?1$MoKGrZpj4^YV&KkMWy`?IC9)u5W73XYyMmT@2y%((c+9h@T%y|tN_hfKGmN(`
zRsG1x8N>n6;~^$;^rXoiY02&s#=VR@6J6SlFEo>Vc)l}x|DRVUUZyHeV$D3I+akbZ
zWWMl2n`cVt*5~i`Oq}!Xi){F|7!}`-cOq_|*t!%H2u>V{*EaFIY|t&dBlGy=V&&-!
z9e3@TH}9Xv-mCU9G&5lL6xX>gzfQ9h^R$l^JEbqL<+Q}9NGgg$@yy(2_P(2;Gv3KO
zy6E~hd%w|}h;vhKi#@nCRmoW@VOKf->NhtZTvODmvVSsrAMb{%8W)dBY`CIf=sM?#
z&-&&kv6uOKPdOa9wz6gI)+GmKDc-nq=k&_N%}ee%Eiw3{8o;4=CYDWIhw<;{-TD8|
zm9)+X&icfc+4JZ}k{-{SqbnauD^#<73adZ!_5!<u$oAvrkH74?tTCr+vROLA^m<O#
zDM#XYSsyw6p0Ocu^U`Y&e@+b2Tyy`k%QPF+4dISC5fzWlC;slZ(sb)$=Z@-&GfMb7
z_vE>hzYt>MVUQ?W|8%C9k!ePI>eaRgdkY_xPbHDifUi8{C%l$5bzYFghd%2G$p={?
zQcq0zu~}29wuduhTIc$@>#V-ZwD0il>URCN>q++X+3x#hFFEJ5BmmS+ygTP^q1+V?
zZ`FhgNmrU%qvrmwJ}j)X%wOT%#pm*GH)L5(`I0x&@a0#HI1`~Tt-u5adC6|AVl|<%
zzy0pfdNp5+WZ3?+)T(*+RH<^!`;c<YKV$MX---JbUs_)~c<iBG`EMVW{};cmasD>n
zT4;ZwNdKR8C;v^2nsKh6`gr9fGi|4NlFb(OAL?IBa$~&wWsdJUCqb8E4aqN0tq@~h
zuwA3!n5W~Ji^U1iOkXz6Flam^#C_m{#mD={*4=fRQ|+I#W?kV=kCrQu9U@+CVy>^I
zX^LN)DdO^L&4ZV&%GYjgO61~yaBx@i1V!^#J@L;3JZ|i0m{H+=<>s8uo10qLPm6yp
zI4Sk$WW%SArW>~YJ<p!}WX;ix+|3KmWj*=mKRJ%gmErPK<rkrR5jR%-``hWzYRq>x
zN@ZPqj>)ax+b`a$pYvn)pVE!%XSsjd9HLNnw?OZQ%+VQpwN%Z11?BvdimAKp*Z1f9
z&I|cWiI?|uUY@e-$D>w}C@YbtN8iR+hj!F#EncwUjPo|Rc~7Lmq*=pKBrj=aF8}iQ
z<JXELrfv0q#6=X3gsUreE0xwu7j05WJMOfIFWzD1I!1<Fx6?1Q$2~sy?avb35QcNH
z2bAg^bsoK2uwQhl$b>f=HuN6rFkV%{^~GF%&z$9(A~<)*w>=EJ5`X$+AOm-{?yu7i
z*FV=eFhx;ZY~w?TEma4Y4!`x5$dhEO*r#m$YQ^dsQ&;``)s<b8IP<pD;hpw|buW(1
z$`h#9XS2SY!DG1oymzW}`@c$?CH1LQZ80lkDs$(GO1?3)?Q)uFFMjOw!x;>>=P4Iz
zNxR?K)GM*Lxxzw6<KvaZix|!^TO|noFb>?bdMVe&*J?#+vMWAKIbgKb^qBSW3w9p2
znDy5@cY6JG1=n<kk56=VEqVF;wr+hYpEqZH@;v^wZ4Vl!gn`;UD&fcec2|7zD)9c3
z|0pwK?|XlnZL3>8e>oa4u|dnZ>dW(*$`W&Z$-CVPLz)gUc&Q$dk2|vW==YNb#VphK
z^%k877y7$RzH-CWhRahH-ni+Oq`*9F)^eM9clcvxK1^8?%JX>f@rX2!{+GMt`-S7;
z`9uF+OwlW!8@+gb62HFtpEcIL3dvJ*RPJ;ix0<bNxck`$lXYr0rNj^Ycwy3Cc6`FZ
zTvm}=zis{%3f%hi_{6J;o$bxl45zMdyT9tGgi&>hv$R~luO9ER8%JK2F)w>K<2c)7
zD_gC@rtkEv`-9E-uWa-BvXL!ho0pHje{uC;<4e<v-568m*-d+KTuO#3RXy)`zm5NG
z!Cu`NSzpwWmw87_)##jLv8ZPG=aki5KHcsKrRC=4$Jb7gHEg}uKDX|@$AW){GBqa%
zJTv>Cq$=?2%(iOloY3o)7U_29*Bwoc+@cuPIQ!zOgCf=QZpTJ_wutH8<&vsXv?J+t
zhDy;nlXp5c4F|7Dy`EQFc%q`;x|XeAdA|Q5rOT7+KJdTT_OJ0AgI1_T`LlJ8JwFs4
zXke0-ao1P=eJK9Ky+sCHKg{P<NAJ;|x&Q3(%O~&dF#lBjj_r<X@NMqdpS?S}SX}=w
zCzYRndUC161$%{W(*zH1w)hhMPvP6yZ2jVGYTfRN^L*c&{JXKXvPM^KO~WHw&v+x{
z^YxJj4$l=2yiwOY`82PRpUnXm5&!yE2hR4U2U>_7eRfS~-kkn}OZFdmd+>4Pt|W0Q
z2H7L}zvVZwb$pdP+AJBobNf>xt~D3t{x{BKo%iv_^Y|2XVJA?ES`xsOZu0X|!lhLk
z^RID)UWruV|L=Hbtp!7mxqbqRPn2*(WOCtaF{@@?NwGDm!8bXS>UQ@o+aSDbb-K{(
z))|XB?drwX6elEq7El)0UeC?BFuL<rv0y-wQeVf!*qv8b3qRC$Hjs#mSG{U(X)v)t
z)9z#X+xP7Ti;gVYAbg(rutIfe4#U*Rx?H}TZ$76h`R?6wB<Evg_UDjwt_Mq+XJ~gW
z=l1n<cAa*#h&ds4sUlw`ughF3w<|xZU%Yc|{-;>Dy+^`HFVQEfaZ!L~{J)I`w!6Ci
zJ+->MfI(zh=i9I^(KQ|JoyoVwG`y}J(|V+`(eRR%D~Do`l_jt4nHjudUIunnXC}|w
zJ(($m@rtXg@c;OquO2Qup2G9!pGCo&i*oIkd{5Y2>^u=6-1X*XAQPKI`D4kDO~+j>
zeeHUGYQ?qA{f2MnpPc)*LFsw)dc&jpe@x#s{qu#~Tiy2#PcC^}&$dqRw~2Y?@1El;
z_8*vAtS@v&`(^NQ_EzC(Qm#p}Coj1l8L{d>N%wK%<@@%?U0Rj)^oa0{Q{Tg1|J~?6
z`%9Ct4d)K;Y5A`wZtb2oqjYhKxVNW_*Mm1#n@d(H=>6OA&3^B_b>C}>((R7jeA@pb
zEr9*flE6jPMg11cE@$5QA1Q6V65S!<;wIL5(?CUqZ|%&I^OFvp<xR>vzF6xTqdLF1
z$hoNnaRws2bIlATBqznqH<^;Ab)m5**XTn^3jaG68<&Zu;=6A9AGG4%_EU&o)pzM1
zfx=lgj`thBa-3=RI6hH3Gv&O*$Fw(H-ZAsuN&d<<XViWpJ=Ob9=sueW0rxW>?GNtg
ze?C2F!<nhQ7V`^j><XB<+1Z8o-quL|(phs=zj>~qq8w{xVMd(l$Jq<rcq5XlzOb}D
zlHSyl(cXGUa(82Kf!=J%#cNF^zI47`w83fOmUUfA&g4v7$Po53CqN^9rQ(7OrN@-&
zIdvLZ;%hw(A{K6{Jo1_Od&+WeO{3}w)pNunV>d6}w(VE``tX#aJ7f61n7^N#9KnBS
zmS5qvSnhSQH9oiA?A&m6V%RHPP+wq@yz5H7l*}L&hTFfCJ+3daVYs-u=3>^=C7G5}
zV(hCE3;%>K+H8K%jaO`^_T*)r=9ivaZO}6JTWT61mgPB3P}%eOw6-4$=DM7hKL5Eu
z_@$e!(&Z(r7O6W9?wv1|V4uLVEo;N73!KZ3Jq_6Yx3PrPRAa-z%s(DiX6?Urw(dYd
z&eVk9dX8_$dhdMR-BJ2<U&`Ds#y5A1sTUnCSv2|GyuMq-T#LU6o#&G__#@ac@3_s~
z88>EMOnkvJr)X8cicgbLFBRxCKNl?26uSM&_T<(_<}#aC9(d3aJ1f|RyZ^|h1?-mf
zGkt%_nD5vh>Um7cQfT|;<mgNMYZ{rfWqy|_{W~A65yE=HYOi@_RDC~hLhN$>GFKgs
z4u;vcYENF<n3QzFgiWy0yZhf2yUeMRCf)h`_}2kNo9j(Sch4&NKl{4)y8gvhjqTCz
z7nTJiTspS!u*;hB6?*d39kGo}LHUY7x6)UfOqtZ(<<e!Ac!))AO_baI{G!?~Hd8FQ
zobJrMe(T|$@6uuanj&oWw)~qa?Ij{vw6JC|@7a25qq%`kiw{-&e0$B3i+4s+oR0a%
zt7q?@({k3+shD&_<EF^pz}GnwmBsJy)bH-?Kg5?>#@i;owpHX@O2h|cPUEVNe@@*u
z|J!-#^^b2SGL@2(Q`P_Rxba9In_Rm8l4A5R{yWb<O)US)^Cs^{LDm!fguu2t-1Q1`
zCr^6uU~UHY)uW!pvp@XreRA?-$M*R7NAeEni)aWP^E`huW%hQd^mNu;yVW^o+w`sY
z{VgJ`r|M?e^Y=$2drWL9J1y3R-Sbi0XCxUZ!ensg&*$bQrG}56=C5aUWw^ZfgTJ^4
zqYwYZ_`33b*Wfd&Pyc;hYG%vje`i~G)1&>d>5q3eDK%(X8EV|Ru6Ua(zes0Z@7c-9
z|0P6Rxe|CDR;`yj^5WpOfXp``4`cGa*Cqwj<UcsmwK>`<$?4yLjH+{I1uu7Y7$nR*
z+R9XHK6$zOPkyd0a53L;MeUc>#_dW5cD85#p1c%p_?6{YbIiY|9AbVi4d;DSeKmjH
z|H&m>zb?*O+-dJFr?SR9S#J%uQ@_x1skoZU>zb~Ff+`D+&B?o5j@;OHN6xIZMMBd(
zYtgx9JPoBwzggV)|JS`OcI_vD+P3A_c_Yjuy!ZX<W$5@i^VCYsB{DlCVQT^ER2VY5
zM5iv|0$-Qza*F#;=U&H)um8>Z=F9(SXLMoa53WZZ{`U%N@Bja`B>2KccfKT+l7)#k
zLf<&JO<!{RrT@J{EGjnB+W&J-yyP+8Rk6jOoG<i*-40Q{tbk{09N+!Z%FFFn`G4e>
z#`|BwrDvzlIH7mrS;6ubEV9p+&96QFI@Dq%@0}a928Hts1yvGUuQ_HH?)fe8`p(Tj
zx46frUcC*<hjuCgmH3O&4@&0Ut!G=}n9Bca#q*eqx4%EBn(s1ua7_FElegCozck!^
zVQ<C`*_X$U>g`YNDROulB+&iC`bg{^-K}EaF{6$tCo2BCF8=!b#bUd6Uw3t%kFWQg
z!dcPx?Ma~K1_8|(F`L07ix4XXz!Alv*dp*qcv7~Q>#6_8k`oW!y8XS=VCla1c5age
z_nvH-+G=z)H@YXzJtSt%n*&au8|XXEuTDQdFXhC9#w$N&eC8I{+mL=<?$_7X)3xsJ
zuKW4vB;TKz5mAwwUVU@^XQE|y`VX(0vejb2?5odzN_ZMbbozTtFc0cTaY*#)oxj0p
ziO}xy_i4YrynJ(SZ*-U9zo{!$XfQD`+1>u*KgZ(Ww%pqi=6QG0qutKUvD6G))Us!$
zQ%$PTCXE(>l@s>vwdIvI`;+r|`=(8gPE1suUn=R;@x(E&K+p-~DIu{|0T81FG@1x@
zI7k%I4^(V{^r<-%K}9iWuoav<S_E)PtAK}V;UNYNZ4Sj2=uj<sfZ~$=ym*oj$HYH2
z)!%p~pL7Wc5wV)vx7>d|Q$lujchuTxOq>uCci!4ns@*GXe(cmKE{4e`U6%RG^ziUd
zD1UdSrM1;_BE*@8Vy0?`^KFgl4O?CNHz9Oo#;q+YMLakZkBF;Y*WsFYCgseGiIY^l
z_nF>3b*js6u9asaRNtMiFD^1$TkpQUEcL{M#zn5(UWG6j)9h<JbN#-5O5YmgdwpH(
zQUfQ*V8elf2OG6QSMB)|xiH|uzS?Td9!TQpaIvlWa^T)QxdxlcPZPAm^)$O+5^{2K
zFTPe?$uhOA`ZA#j8dN{Jw{A6k_wHTB)~KkhS*ompP8=UYLD}k`TSbLM?yW6~CMF`S
zP9Kj>&o_|@)DTHIKkx5q)@zxUmz_MZOu%V|Ve82hBTvbeird%LMt}VBCFQ~b$BL?|
zS*g6BQRFKtg;l+#B%GY2nqeY!)9_3LXfs6AR;AoqTaFw%_UX+{<4q|iokBuHKvS0w
z548rVC<ZA?Mv6S*wVK<Pc)0E1<Hv`O^-62T?%Hzm+M<Gohh!`Y8XTM19zJ_^Y?<%u
zCD#{Q)+ek8iP>NGcWWlUoDIVS=?yI&N1x1cRg5eA{LEJ?eBGI)-qS5gUQCE&KPsM+
zlM}Hafw7sL|Jm*Q{i#R0L^my-p&1??zA5eOuWKdM)zxhLayqO+pv1Cr_Wb$spb`1R
z!)-Y?HZZ!oyKA1(zO^Ou@VeODGwf=uWUb3o%*?{NlG?1w-}OwLdNulT_LUWiM+L#U
zC&#T%e{o^qmseN2K}(4K{rz1(b#`P>Qxhw9(T4|)JZzxl3(p=M?G6$#zP!+xeTvu9
zczGo;u`M|_KW)tfr|KnA<?rq+%(}X2j$Q4n2M-cXP1RO+{mH&<+curZO)Yk{zd*}I
z>;8TX*X)u!IbHvLcKP$$w{Bfp>MgF>1IkPuwSWHpEi5ixT=w=>&b>V|OI}{`4E)#1
zCu{ZL_xt_EnU_>{mA&<fiJ2pE)Tg4VDk(RYS7lqcR{Xv_BCepKeu*4gbEACSkH);a
zyEayR%>sqZ^11dw6O~*!cz9IQ)Y^<vPZj+9l&aY!nQ?g;ui9kJsa~$_e6mk|K*Heo
z-s<lw;`Ul8Dk`$Dvj=ZXa@9Ol51Oy;64eHcv4R%#zrD3JXiA8l_cR^Q;>*2p{>S@d
zmnt|dDY`TzdPcAPzaP!t-rl~vHrl+6SGujWwI=-670?p+-DPhJ%FENw&9Pi@HB0kU
zx{$Q==_e;ATb94O6CU|*srU396(5y)rOp3oo!-82W97~N!QgW8|B(z6qr5v7)AZxd
zZAv}8B5La@k*CI8U0sJxpY8@tp=MrI0}VpE{yefR|GwY*dwW3_Me|CT{K$GOC@b4L
zckbQr$ZvOdm;e0ve17J|MXZz6d=DKw=vc-BDsMi%2hB=eTN|BhA_barlQd47vZzMU
zd%9lgqa&SX=G)JorW@_$XxA9MJ@4Y~@_fCxJtyAW+<f+5x4(-E%VPKbwD0fko|$7A
z{LNU=PS}!PJ!ox114B$~tY-K+8SAn)6Bhhvt*olrlzLjMjbFZR`gCzlPR^hy{Y$RD
zo@rNWrKhJSq8~SB!UO?HMfN#!=782ptqxzGcBqB(%F5vO4I2z@CW4B`BlnG^dO<hc
zndRQvQ1LOztyfC*x_X_eo}L~HJA3i>cd@6Y>;J!2+<MgBy66c<US8gh-@iB4|F3iF
zk@$E{eZ`E;MNdx^zPO+WS|u^lD0PNK;i4K#aFoCQ23o3hW#!>{w$&V5T#Kr{z6zSM
zUdMNq$(qQ`ZDzT*0=DHu*3{N6)sTPi^78VM_xI*5^PO!}@W5eP?rpbuHkC^Z-Y*PV
zDPfSnaPZ(k(0V>0Rj+~v2N)qK`iFNyf`Xo&-i{qRKuLJAWpSFOm(eBAeaJO65*7st
ze}8>-UVhm#@E+*WR#1X!bvpR*@$t-yi&{mVGN+!OH<wr1%pmWM#jWl6{_T9S6<=OV
z6mjF=>^?MSrjJ_Pub1jkTQUy*`T1GWx@?Wja<iF5#l^}N7BgO7Umv_ONHy=y4n^0>
zqfbvyUm3U8$~f&z!sBDTXXaQ=Hp#r?k!a`C%*Ja`_Xo7pN<=fLMb&$n!uMh>#l*lF
z3)TJS1+0&=EqimrF(5#|H2d0=MJmOamzSm9-BoImduxhO>ZuL2zsokS_pGV0S?u1w
z%&C>@%G&7Zkfn-R)~D;2T+EnZQ)zU2TW<2tPftTug}hw-bt!npA^zY(=k|ia!i50~
zoqqiOJz3RTOw)_&(sF<KYwP3v-{0FSX;WcPwfCORYf!VGOHB9F&(F^-tG;N&#l;yJ
z{p%FfkDF8b``g7;q1vm$*Pol9=-koUJM|F9$Hpb+7kL!=s0m+Q?w|ha%S)Yz4Gp_W
zUwZ`xZRqIW*j4`i+~)N2pp{rRH>DbPzui~=-|p}4@9HWlE&u<%ufMV`HoB}$|G~q<
z?L88PO^4h0jq~r>yt=+#-&OFDJZN=G;5@rpufuJ;A3uInG%yHgO454p<jIj^z0w~)
zd{_{&GU&~noyMX0>n~=c+}l$rq8--r=H}+&H#ZD9yW%Ho-fVn&x_<h>CRWf=!DGGB
z#jmgFf>ViWR>7ASfiW>LC+1q4?=E|5G+R)kctL;$C{cm#2e1DAjz`{3rupE6SKB^6
zIXQWXc6eXu>uV3+zd!%u<KvARH%^p&puHk&t(2moqKIykh`6};GXME`+on5leC)4Z
z{M2NN2PbGZ&6AUppKh5bS@GwG;oV)O%6fWzUteEe-jyA=c+nyW(=3tK*VZN<YT?|H
ze?RWti<Bp~wq}D?G#~Gm-<)z%=+rb_ZCAlZqDGrECT&eWKkwoaPvL88B0=kNRWxQb
zetCKM@$1*#y;7zH#l@h#U7A(ZX1TXq7CN<7e0w9gaN$BBk;x15_x)^pdwaWa!2^d`
zCYetCa<&!U-pmwn>rgpW`1n}wWTTEJ8YU)Fq)f9OJbH9!Rp@Gu>3Vm?qrc64d^=*t
z#rcoj&o4e1Kc~lRef|1f#U<L>wcm=r95B@RB%-`}@2j^nCFTk3cp3iBJdJ(lt8mrk
zsgKs*&pa&oEU5Ll>8;6MPa0mU`hRH8TK^S+iv{L?bIgnC%ZWHJ>%inW+<edal&o1d
zTQacu{a%~&_gr(!ot8<in-&_m6jeRfGK;wqur1+N_%p%7eIK27&e$x(D!czo@-q=G
zR#EM+BWKU{>c#F75EOL0x+*mIO0x9&`2Bvjx8;8P`Ze|Aqobg?w&>_+%MFt^D!F!D
z*qH3j#KiRC_ICZ5MyZP`K0XSPNc@(P5~$)`{QTU~xV=@N*{heAm)Czxf0$O9?a`gT
zC}LNMny}ftn#bGwBktz%cuzm4?At3g<z&*|dFzw<o}5{ee1B@W$-J<Yg>S#vR$huq
zp1xUNr_XcFy}vZ)r>=hLbD?s}3eD$%(|7#jju2R}u;}I*{jffvsa=~&eu`#QO>tej
zxA*;}_!*`-dg+Vr@5p=oGj-Z;8}<F0pLM@ypQ;tAq@mGqxSjv;yLadQ{{BAs@4nTS
zU;cQxe7;8PE|aIHr$7Jj@NmeA07t%8B1*oqOwP<Q?f&}ux^eBV63~*)nxC6Q+`vme
zI(mARxOR)JiP+fG(7+&Np7-bKHC69@`3yTO>P6L+{_ooI{pZX3AI&5WERss)-e-4f
zWpAg7k<qiK=g*y+$;hiEnxULrQF*$}$oT%(*Pfi|qF!5tBaPE*lV7qaDJx%;ikrX8
zlz~lmqTwy`O^Ii+Pw(BCub2B<PKnK7a(^86uH-+(VvW}q$M1V~@FVjS`D>G%q*zyM
zzPMz$IPa;5kAHeicNYICKV-X4dB5Ao(0lCZ>FL|@?s_ftnp#j=ntFd<?a8wLygWQh
zTsnnV*w}<5Bu*@FY~HYC%M@m7R;|!gB8`oWSJuVO<`UI1NIN63C2B3$*N?KTDnF^L
zTer@_%4$>oeY>LP=l+IXcVB(gpx{h_mo)!}*Xure1}EPzdu}28yoVtvuCi3*cN+hL
z-}2!bJ}mDO$v2zXJyGoo!?C|6h1zY=t(&H3+?)M(`y$J|shqxDV#!};rt3fd?et}>
zbI*J0Nv+22kDpzdX?uNfWkHQ}?W|po!WL=7Mon4h6lwnK-G1H!s{2*H$XZYL{HqnX
z>HhjVyF|~N|66x-iotBf%h!s}Gf1AkzI^AQrjM(eCoyG<SU;0p7JOG}eU7t@)q}6?
zr!y^I?VYLT>-Ev}wo7zG#D~ND_8UOA9B<7&zN7H5fmH9wwEMN6o^Z~!E??$5+pI^<
zR!Ux8zUbK*P4-f*<sn+2^?vSsG7sO}+zi^mW&iJoGnX=GiI*wpdh)2PS%zhABGS&z
zYK>a^@0WI}*MWU6wwJtImiT^2Z;SMSl4AMjVzcMB_wcRX`{mp|%P-NPlPrUO8Rwj;
zs?&Ss{kn0V$^;9gpNDTLoSS>^a6?f03;R0uov*Jg?mEPDZ`T#WwzG2gA79yIz4!N*
zWvgGZIIF!?TN`bvwKQ~L(fk94&MuF%U!Zil*7<^+rORf)_Ln9b#F^RqUG|qtz0RvU
z+`wrXC?<8ZY2p2ek(?`vE><}?3z_LX^Jw9`JooRJ(kySy<@FM}HFb?w7am<1s1&>D
zhX1P2)ouX+0*s7|S5}2~xAV)_-HknxlJxV_Q&1(}>*k!1q0uX4I*Hk8>d8r}+@Q5@
zudb}DsH(cOt2A5Arb3}rP)MxxhtQ(Mi!-mT66F@x121^ZzNRC&b}rxCdyQVc#}hC8
z-tf0T!f}7@;?U<_8DG}N#jkjJ=bf&I)DyQyp|xIKyUGs~yy4hh(%@<u!?lDlHSJaT
zyQYUXIOQIAJga#xvATP|XT!w<mp@#7V_!2<>HqJUGxvHe^pm=-9lO#X%ORD0ne|1R
zUGpp+Rep^SSQsASGuNuwYUkSvsxInlZ^m<7$#KoR9BTGY@YKcJ+b?ge=K3#kQeNi&
z+z+qrPmEvdVfNl@na|7*pFc15nQ63R$Bu#r2O8IJk`vd9S>QEQODAeehi>#X&^q)t
zZ{I#$;CJ-Zm6f1CiQJsFG~?nTornzzk-Y_=N%19C>F4Ggyn0o%*X{7XzrRn;HqYM_
zWv@DIMhTmUrIzuoU8ddDW=qbCmcL5d?y#Quf=VyXstsqCJSjTxbh`TF%PD7h93EZG
zGMrFy-b3p7nS_ZmP2PKFR@rKN3fk|PY-3}yB6PLb{b$=$9=X2qu6X)PxK-3@uUP9U
zsa_kK4;gnP`1_+$uDC6#-pRXugLc>S@XCKPE=Zm~x-;|Lw14ti3si4Z8|Ow%SJ`R1
z^2VV<O@WKuHfCH@0&V&{(kc8iNH_WMqel{UH5Q?(!<JTle#XMW60pdH({jPtmfl`o
zVKtuv_wUa)&At|}ucp#(u9Yez+OJI0i|yLI+gjPJC*j?loe|N|>bpy8>g?x8G|OEG
zSne#(@ay}kMz(;+sK!G3?ekUKHrcdFE#SZD9&xGH;5w68;>|U7mLi@TbzhczURwL#
z`R-<+a}I|>mBUsYxss8YI{k-{Z&jDD+w*;=cgikS|MiUV7vERs^<k=P{@Pcw*OwQa
zYt($0zAVqCRZMq!nNRw)g&ZQD%=4piRrr<vbInkd-o|>MHig+}|JI||*6TLEI1+rJ
zv@6YL+wY0HcUwC;I{x_mJ9xRDtekz_pUCOof4|=!yrUrT(4j*doSX+QU;f;^{qxS^
z8tLHl^J(ASJe|{9@w-#Gye}{H+Z)$2^L9;`u>0ftd7Te68>PiAZ@+1wcWjT~rmYWo
zw8Q53PG4MEXE{T3v-*Y7BhRj{yYSa!i_A@f_LbdY8#BJ;Y?IsjIq9jz+wGHXZcaB&
zKRofs#X@J5<$tFcq`#h@e@3}Y_VMq{U#CiM-#5jnm1|@5_dKiecRp`#ZGCxvzx~CF
z7r`Aa{{`;-a-d4gw))$LyXE&)PnXsd)JQL8Tbty!y)ei2UxUfgR~OIT?Y@=w@}j-X
z!7q`k86r$d&C->>q^z}_(fU^>Gq_ShTg*4%%*<Uscvm;YZkx2_boHcb&dWAsWZ6G+
z>rE){zclAt>;aR-^Y&_AFZ%I8ukP+P{=-XSbF32`rQge+EG{uCJvD7f4YR#XdWmag
zmH2IEA&!stPhRA9zq!qp;qcPfH~r;_4-dHINZjY!z9?^tUEq|7x!sf2R+#dLs|Rg;
zQ!{Umk=}|^QGN2Or!TI%+LYhY?*8WXhMtCGt3KAeMDGKY>)QXE`)cH?u_jRE^<T~6
zXJ_8r**W>q(eB5O9vxchJ>9O>VRv!+GT+&kw&%w`J2&^V{{BBs$9m$`UA&{0hwr~w
zyLX%4gd6-@oZI<Y+S-H!1sxYUwSM^caiaYvrK=N<|LxR2ckEbOx48bJH*e;=+IDt9
zuc8|_AK&`=ip*rCi;IpqF8+5xQc!rY>(d7fzm%u1?(JmkW&6s$emCRNsFU4dwpq0{
z6C)lk2wOJeSNWS86K9!bU)o#!-T!l)6wjHN#_gVy)h;e}=V#-Wo1>GruWSDo>7}!L
znYy$7NSTO-w;yr%|L2RA*0#TrI;%E*{BY;UHjnl1n%SP7u2uOriGOJn%Y*71J<BZ{
zSIC&GkIhR^_-NGT{hIw!QLrRmZqC#a-*t^APDoslV*hu`lqZAN>q<sv&-1T;W2Bv*
zX#ak8w)XNzi8*b|>NbT;axCW#@4c?1mTT{{_)e$tGyZSB=~wnz>EuR#dHTn^T=THi
z_t2aQOWt>HX1n~g&1>;nH(5t-DhD4QXe-Q<lat$jette#J6!K)vwje(WTZ&^oS>CQ
zjvZ4g&6^$dIV^xBAX~=HMnml!$60o^qu+JI|LQj-J+hgi?_PGrOLc{*rj`cN{4Z>i
zlaKcWt_soImV5iyy}i|vmPIL_c6iHYzMc}&nsV{l%TQ76uqP)bDo@dko))y!%dqlO
zN{Q9nKNrP8gPTgGmH+-&!WIv{yv%NIZx7m@wkIs;mX+%6ZGQR@QBg(}9}<|DnJo(*
zFsOP@Q_;}y;L`z*bp|oZ%gb|#YPI++e+=1Mq&0Kh$}Oc{eYtbvrW&XFndRR*b7G=0
zs0N#BU9P63<;7<-QEBB87AKC0D`Iz-fd+Y2g|1e6{k3NE+ROd6(u=gryB@p|d!c{O
z`nJ_p-L5wA+_`!C?p^czyJ+p?cg?q~cJA9DX7uRZ{Vk<kH8-!lJgOrfxBv9v>9;?0
zE!pNbSFI#%{VLtv)wK!H%jIG{--&yf<lT`dFE4*}eSLp_KmS~-(n-v|ODv2wX&gxx
zsyA<O4Nl*jaZxF4|23&6%6l(edUCjS?=>aOPyCBoS@ZJpPS)8jT^sx<|MeHG>gwt)
z5zVH!_0yYumUw^-NSeEpNB33C;;h@AUt+gTcgy?#vH9)PX=)5EZN<+jtGd#+|DGK(
z@kZ2><p-rIor5~XbLaL&woER3njtlHmAm=(8|B+dPnm2pKbpm#_VDe)Eh}w$827&U
zuscG0e`e^0?)@IS_XKC!rfAu2zNZ{w*mT#~yZF}k^ktG?r^N1@c87QK#_Zd6JkynK
zls$^RRq_AC!EMSbgUiIYQf;rX`MFkBS>JXRvRHWI<VB_PmU}?O2FJJK{s|8cxP6hj
z-^cs-;Hsx3OvgoB>vu`DtrT7v_039}kMFswe~WnT+&A~HUOSmyoD{9TTQ|q^euL1m
zDHpGKc3g3adfuUb?%cV)?Ca}h9$$R%(f=j>FQ-k3m-m=`DD&RWN#@bB_RNu!?EEHM
zuA;upr1Q}B_uFk=z3~6OM17+~l2T<}z~d($u2&v;axLmf@$0+0&7Yo{`uoK-OR3(r
z*Vor4r>9SUm8G0Jci+1A`~T1LnyPhiZM6AM0Rc_W(%O$RUH5<g9U<~Ntv~VX-Z@Xy
zJ50mxMC>oJXq6GN*p$Sy|L&=V<R4n+ZiTC@UM5f)s~cfdC)az8E$H)4H@ElQ#~*#G
zFkx;FuuPO@{vL7J?0x06V_Q;^?|<5pqkCy{dnAkN$L~|Jc=_IYq-XUW-k-H`%8hHD
zf9FqIeJ$hP(_Q;-UE>VMDwwc61U$}IrXMv+#`2;dZ>@&qI-9lZVh>(ic~dgGRQ$g2
z<P*P_C>>o@w(RvY`IS?RyX@6<7(`Rn-f0Z{DV{XPsMqXg@1Lj|N!DjSE-$%N?&ANu
z+2erX%B?5-Pp;Y8A>h<8#Wk-$@X_judhf{_PCQSrId+&sm}@!D6gS6NTcoDE3Xh)k
z;vyqwmF|Lrxl7oitFJG({l<R9#r2$gD*vx3L5JD{l9pFkuQ_^u?;&Svq3BsEzTP>R
zOlFb$QWLGMtHR`ucUg9OUQQQNog4aouD!#=XZ8;^zhIxIBkR0&)8w7G9$C7Vb}gCH
zBnF=F=y-DK%O5>PHRTomx^miP6o_4$+vt3-?R`$>B=4^iil?78NOlf?wMF94|Ee9I
zA91Y@kDkS#a^&cu-aRKz1-?EJvyyk;!EVEs*FStY`BUU-luJ$IrE8uXAM;O`c01Lm
zwRLo#nQJ26;CuLYfl9qwPG{=zzS>USDYMNFzWjMx*ViCcRYj%Yr|<W@bHaNCwO{7!
z{-5eCJY8|^iKt_f_Q~Gmd}+)4eMT$Kj4ao;U&J+U7a5s7FZ0S4{1^wa1+=0v#o=X0
z$G>8;i~oCH1~2>b_5ak}=bnV@K5|Vdc2ZRLotxWRO`5|Bc5T-FX!vmjPx(yC4-7Y_
z^FEB3tf>D#cj3c!i(BsfZj0HnE*h<L_4H=v%~)N$VbRt(O=9n-u4~}FucfnL^TxXO
z*>*eDP6qc(I~f}^YTQkq2fF{Cp7?QY?%clremjnrKbfv?0-hN9<Gi9~?beQIi|_5<
zc>mtC!|5g#s~=g^U*PY)#ameQJATncjr+FpTepY_{`a|M#R(dvo%m=)rd7Fi*lY96
z<!{$Nn9(ogTXjKYe@wN;KAFtdDlMQ@*G0=ty8YFwU8z4+ApU6gHb3q7{MT20b0kjW
z)4y%?w0iB@$z5lDPC3G?f6GepOO%3cp0e$XYrEO_ZR(%6XTDZ(=~5|o=H6WX{ZQSq
zFK@l3>(#!zU2dVhEhpjP99fsTL#cNxcXitN&c3wfw81vHy+5zL=bae-A=adEowC{g
z$3<HO!z1jgS;dq89iD8(J8{mp=tFbX*4MqTfAFr=)~D7zR*psQX}jt36nC$}C2J=O
zfF>ONOEmQ9$UCf^w)f<{G9B;mGl|ch)LG1%b?#8J;QTkuI=}wpUwK*CUG}&w@VsTu
z#Msv#e}DaP-9AjVXoB58n}y-R|H^iLEO(CBSt4hz$M5ogqxKS{aSZzuy(p*ce@;!B
z-~Q?L%opa@{JlS%a<zEotG_##OF4ILi-5eBpELL7uYdlq?K*qMx&LVWg>5;@j&#mC
zV<NeIs`TGF^^Z3--tSbtlEAQ3$k~VI<25Oh`pF-h{_QxrYJuX8ERiX?W|O1qzE((x
z^GEFe(V`Z#{*Y$zwfdO!*E*>$EU$Df&Q{y2{5z|&lRdll*ZbymR?@yzRf$}YM|bhB
znldY>*E)F11XH`UTk}N>U*6i>`sv{OMf=n|qt(IFiT@&hO#GV=*qfz#pT}zEN159!
z*-AIftKUSIw;aov?Xg_*^~(E;?5xe+F?46G`H?IpcDwyT<}*2|zOT=gi>X@fuTxpO
zEG6~%#f2%Gy=HB7@|F7cYeG#?mGC@Srxo}0ZH?33xIK+~Ha|S(_qU7(XBWi3us5Ew
zOis*a=GO_L^=B6Ke7-Mx@8gRGll4BFw@s(FHH!srIGo}(eV5Dax`RGZn>*Z&3cvom
z&hP}g{;l~Zwx(-#Z7CH3CGSV_t{>A2G{k;ZPkbq>EF-0^{_f4}FYm0K{$HPQDcCGv
z!#{>)J+jA|lCJJ3u%G1@TfgA-np&GnH`L`~xTM|IWd08iV0onf_aIj&|2I)T?u*Y~
zTr0Ct=9{~)r@3j<wCfq0-zu9u{b^{kM{mR5BPK<f^FL*znd;f}E$b=1Rc|`mX6x?_
z|8AB><#QahcRIZ#cYmP%_OJJshBYny>-marDyU3ojM=PlW}0qf*Yl$lKSlbtE-PnS
z_<wWx{foX=|H~hrpBWY8p?PA8q5@Zx#%v}gBW-5~4HlPQD`0B@0tB9%p60t%Bgi3S
zTFycR?`=9WQ%?%b+;nTxa*>IV^$ofKQW>|F{QP^-D7CYv@A_kz^9N_oo?U%^r)T`n
zJJt8iA9h^``W`1JwW)ivMRRcGsj4*QOA}T$7=BC-50mQ6Idp2DcE=N=>}%=I_By?M
zuXCwv*1~<=Gq$W_j%<8kTx~J)SpW9B!iQEB&su(aQg-6+Lam5NYup;bKiHQvTx&J5
zySj<X{OwuwKG8oZE1b@{v$^KUH+|Dv^G$uFvs3hP$D=uatlqmnTYtxYTbAt4L)&+F
zDNj0-nGzVJsa$E><@Be<Z_dUCffwG&%lDS7ml9sb@%MzV@%D8bw$;C#lQ%P}b?kl}
zS*c~Rq-(9_;r3Xi;=|uk_WnJQx~%k1A6K?`km79P1l4<%yhr9Mo#K1v*06l{+sK}?
zUVl#Rxcxk$Y3>PcF{9M#u8*@{?6(w5Q-62lxz6AF)<48feYg;rzuo@g&h(Y-x2!ss
zcu7T03DNeS+MkhV@F{h2VT<csOXc%bF9q&zk#0OayHviy^~LM7=`9v7=Ll%UHr$Bz
zFaNwCbh^`;S;cdrrhZoU+v7JOKKI{>@Ah3E#h;&(O^rGraqigc!W+{*c)ws}Sm)CA
zxWGj?wf>I3O?pAhUfCeyl+)eHc}&XB->F>6dU&8|y@Y(#Q$y~;qpS*N@9r00yGF{S
z>(pv<!|VH*m}mJv_!Ij&X}?_OW$9%{Hmv!kuHSUa%J!o0m1~|ZiGfcn4WB9}Trb*}
zu|a(Pr=yv##T^>rm1ljw=df6Ce(<_Hf3><tPM^!G4Qnqk?6ZoPJ-<5i(Sb_}PLG^t
z*pw@)J6+%O&PC=iv-s!mqzzhe&z0}3yZ-ju)p@(s>fHAC8m{@ViT7&Om1Sjb)!t|Y
zvHSO1op}>JKk#i<(2?aDZ;#6#dmwi#|F^!~pYusi4Ac65%@c5!c<k=?JYos&^*>){
zSbuO1<M}zmYw0PYcUivsD(v4Y|JxRq_W01s9Ij5~l-owra;=&~Ip*G-eJ{H>@X^*Y
zPxNA>+H>_vtN)ycFYB<_)p+so@}&Q@vXjO0l#O1o#J4^P{5|=(TkhN|*F05T8d%8i
zwVzVZpEP^6bp4-?>@P1b_qVE_wN^|=OxrA7c;=n6|7*^i+Pk(`?sNOSEb}BIyPw&|
zA9UShNqBPJcK6N%<-{lNw^<aN^Sl}KQ~qQ5<u2CrGiu7;R$Ok8v*FkKuB&=(s?&QB
z<>s?}7gKy<!VM(<=l<D{+_@=nU-4JxNf*{eo1dDh?cUBOJMY!cw9PZ^|Njv-H{U*e
zm+bYvGs~SWA1s-(r}3R=+}_s{_e*>}nH9yhO<DN=-nEyH9e9|MGylRh&f}H)G=JIs
z6wBFLd@egFdU^5bs>?Nnl4Ux-K3r7yw(z^O>_$S>n>l_Hcl-NST-xT>`TN?XZK*$2
z{?ZYvd-BueYuC?OySTGS!uoEzLyy${*ke5{-+9^Ni7e}`y}z`STU0x2N$u}%zmyeD
z_PL3<dhCfZ4qjHza`blp&i}JNbI!NXxEHnWX3CRmvieDj&lM@@^lYg9XMOPIiXHCy
zCsMp(_>Q=`3za=OQhS=?@sZBrI~y#8njft#t({e<l{|^NOY`L8TUMzGUA>L}N+d1j
zcHZB%C}XjrOm~rhRl35wqg{d*>>H9&PX}#tF>lhp$uajC==6(PvA66GPBiTk(KKvq
zn!DeFqwB+gZGODo-_k$cahSyy)35kkc+C&zTUMqyMlB0i4{L;(B-Y9a|CPIO?d0Xl
zlaCz^FG_gqVblF%!=DFp1uM3e3RUhaY3%k8`WzM-fA3fS>4LM7lf-hpSIsy%>6X=9
z|J=DQaeBQvT^%Z#S+Bn+WW82dv_V5jsq>-!!nK#v{N`-5dGK-fEh|+&&y3ec_U6v5
z2#8)T$s_+wwvad5#xZ*N=knN@|Ko32xfVTFn^>l#Z~JuXo$a@**8Z8Z)P22eQP|c}
zp}+_cPqTCHBF_fc+l$#g=#O@Jq0=QIe<<_y=iSGA);dp8-&*RFJJ)5Bz}(FmpKSiW
z*dNPa_Vw7Mm-8DhOxeru)_T9!vgc(?bxE%$^nLx}refZiJGZF!MW_GIM^jdYP0wi&
zOj~)vT-k29hnlspKHG&qcjJDPmpy)-y!B^gg({zpY~k*;!P6IATw^VG{N911SAH+F
zwe0E4oqKNSZ}(_*vuzKrX?DK<@~^W@PVUT`?X$1F&H5C}s4kQEQR4KIr`!LEf7Jc@
zpv?2ziR3f0jeeFzFF(I~`yu6e+h=?>2i*VKr5zQ}xL(Wjc(&cLwU;L?<;<OXXWB*`
zZ`-VJ-7^c1Z1a2Fo~Gve@RCJN<CV7)yG`$JRXO`!TEe_e>)ZWq<H|(6s4J&Vgez9g
zu=mTI>oQMt>$!>IdYaqLRo3bJx$+_QQG4Ff$~b1RnEd#SbADVGv_5uQSz+1<k2lYF
zcHB4ZS2YqkBl)g!o~ahs>We+Qe+qt9-FQt=-^nX;Zn1w|Q1tTp=#45juX(O5R<XQ%
zV$q4u|6HeCJg{M5U5&I1kHhLj`-_{y%9{4c`|QuWK4X8em21v%$y?97c}%qm>;9N{
z`TcD*OT8?3<^KF)Io^){E9E8Ga~FM$HjgwY(m8(e#J`)x8kQM$f2E%MeO}XQw5rsD
zE74?Tq4`Ytk2m&5-P)i3y5#!*<8E`G9%0q~9`|0TTQfZDlwFpi+U2)T?_|Apne}^`
z(Cmpdajqvz{G*pIUnP-NF54gmN-k+JUE7}>_@dcz?w#lBhPsftbj_(+QyMa77qTrn
zpP&BjCBMa$m7?kKmTt|Z?U|7l*7kb7r?*$X{!sSRXN^Vv^Eb0T@@$@#{x*H8<<!q2
zZ-0MX?D&)M{|x`N&QB@>qSYr}@f6?DVUxZwT=vU~P$7e$TicoA`X<Z#-Jd<<(tXP-
zFT-}&-VUFtm~n~Wp8OB#kLmi!hUv?WN$-Ccy@tb>KV?Cfra9l5_|wvoc^m3Kn<*KY
zKH9yp_T+5S)3u*d!(Iyr3Rb3>Y|{9o#PoP=oSH<Tv4ltQ{fhfr?}rB*EVs4KlRMn?
z=K6=3>~79{x6Z}+{=513#~SzGuyT3#!x`*5KTlokHaA+FGe+I!($jrEml<q3;I@@d
zP;g?plXADx`I(PXY&LFdDUcAZn{)P9T8Qoxx$8$h|DGGmJLi9{-La<B3ggL*-s$I@
z+alIXte&~^mz1e>?dfSNb1m$Cs^o|2p4E@9JEgC#v?2O`Qse#UGCTMDzxPv3Heqp8
z`-$#^{fCaVPX3tvh4;9Qf7RM0Q+w}Ed3xFPc&NQ#tE(VmI#=1sQ#Zuwd}hgcOrN*k
zFn6o{O{IRr>~B&X{E7c232YKs*ZTKI!71_F^J1@8-zeuzf1+PsT;U*mOphn&;ilvg
z>FC*$c<q<3zApJkI&4|Q%I`aFEm42}^i;w78f7j66&3!iHdn1&<{3ZM-ZS%XXK!M<
zoOSk*iwS<wG7ZKz_uhIq)p+kK^+`S}O8fQ{tXy5{*A%Vmcjn!*_P%d7*NWw+8LJ)0
z+L+|^B7D*Xwc}6f&WElS^A@p~{r{%l@h5$|H8Z#IRSE_ry`EFQcYWld4GD+t=Ip9g
znt1)q4C9@}=~+qAvTIz97QFw%AOFLCn(*J>LZ^PNOS*XOWrfX?E$p|K3fO&{=kWdh
zvVco}9n<Y^c6RMy`@XezQ?&4{7ZHnecGx}IJXvN%X`f2ERHR5HV~?8gyZ)ca={`JN
z2i`8=NGK}*?_Mpk`R3n!b-m^1(|x8oevIts?tXk}srSi}x9d0SeF(0qv7gR)_jF*<
zfgq(u?%mHXZt!H_7Vld)Wx0Cs_n0S-=82Z%M@$MjYLodoy6sb{w_)DrN+l=p$1h)c
ze)O+X5!z9fU^j2xyd1v%$_GX7N}F>h|M{qMtJh%l-22me4`04Kxzb&KW9cyur$^2i
zkL@R2jjnq(yY1-Dieq&QG85Y+Zf)Qc7S~&n|59kn{6E@(g6$HIug^$Y+V$}H^^Q$X
zPd_qEP@R_Y>zsuQpI*$4hJy==J6sb3T~s1}|9ZV1bhgK2b$>NGJG;oWzgtTWzo=a+
z*SOc}a`pp`B=Pl6{-0wF-h6j^jb*aVT&von>w~Qgmz1jOiL7us+H=k#T>ik3rR)(8
z;u|v^5AK+nccv*$v>?V~$FGaF-eIfCXE>!^UdXPOo`2;o`?5<9Ll;XWC?0!uB%0Ui
z!s-O6Bv0qc`)UuK9DO9uRIz`P`PO^Yv$HLjuSZ_kCwkqQ{nF}S^<!^i9B#e;%X#N@
z-}-z}o@3u%+N2mR^xmghcCo}Sa?<_A4U66TKi#W-U-<V|Y1Y+MOZjJNfa=8b+TY(&
zUte1b+BLt_OZ40O`}2LS-8pq{{`E_1qMXECob~eg<mWifTVa3TB=__qjxXLE-TSd<
z&XWf}TMVt%ceNTYp38f%L!ehw<4axjo8#@9ZEe~Xzcve88^@{hzx<Qp>7C!_J>Rw?
z;z8JK-rH}j_udf77t8G~WKfGMJUzic@t4hOzaxQ4`NvxRm!I|z)feZB-R1WzXJ(<+
z15dfljMw+Del$w)3HyDiSK9o?r_=f?V|SbBMsG`bes1oQQ&YX=jRGS?CW>o6dXd)l
z>*vN7FHA)rOMd+DA>sA4wIy$FiOS2%+x`Djye@wKyd&3d9qY|tS4>}bZ)QsWqfJNW
z3Eaz@uQubtu9w>)ZqKck7Lu}(G!jn@xx3|yob{BOiLNVt?Rxz-Qf}9r+p7huum88O
zPs+_dn{?o8!h`kN7q`!!Y}jvn<kkkpkIN@L*ZjIVeEqVLmzQGpR7_NM@4K)vSY5VB
ztkrd5z4A`kd)}vv_J34c<{Q0X!RspwZ_0ifM~BTe%e5$bBN4WGYS2<IK@pKDE!*$-
ze4I7a(rSzRY5uqaLZ$OEW%sGrJf1Dm!Jq1!ap-4jUx#syQOKg4d*MQEe=n)9SWP#{
zD?e&2Ec)*{_w7|jQ`SYZmLK@@>hQG5jN#K;Zhy1fEmd%KhUg>bjx9Gfw!Tk1*lDpN
z^YBg8SSODy(VcrlTyL4M{o7F^8xy}g>Z*xP{5E-QhCrEkq2}_7cLaocFH0}Gy?9DE
zWBBzYvODx*b~tQKJNx4Da(>W4r+0UDrktMUEAJB+A@XzkscE{PP1`%FzOFjN+i<n%
zYuK@kQ5{{o-ka3zZar~g#S!~u<q?Y)ZLnId?9Nyk%mO+B0(3rP#>GYFb7syoN=>@8
zB@=WqGw7`CHeTtXUteCT?EIT=`oeMjs`Qt)_o$_wzJ6${cf6K}&haHX<BfMNHb1Q9
zd+2<=<zjJuwc97ZuHorZt#~U{CV%&A*VNe?{_pGC>aNUCpdB;w+qu#e(N+upPEy=_
zoq77>hri6V#n*adFq&q|PFyc`bia?3RhYKl<&DuNV=r{(+UF}6g$qVme7m%!CQ~do
z_IQ7GdGcJ_Ij-#M9~o!Zt*v{1*8Q)#k=ldefB&+?)_(qb^n0VQ(6!9(jA6d>ydsij
zuUzs=IrYtrje9<yv!0?6*yP;K7q~Xcv{pm)XuxZk%7;nG$)NK#w8PdUe0y_qiq6bm
z?}V!h&vUVD{U=o1-5+6EZI)?2`|zBcgFW}fJa2V#M?8Obe%AY_Non_O?Bt3X+DeU#
zOU^_t%a?GTnNzmSmn-4S%RLp#ZO{C0-Th*7f%RGLRFA2@gMEB>Y^%Se+}@U}6SwEb
z%(kPuj~+d0m~}<Ntxv{td*0m--@hL}dQ|n~)x%jE8&=P()}OXrNHq26le+TPbB?nt
zII_+3yY64kY01ZvHYS?Q7OLZweE;Xg#v3vJ3q}9mwXe%jD?fAcjnlsg69oSJ`Ez2L
zuC%b4&jN==reBk$i@0z9Jy~4Oca{kY8(YxMqEsof9FO&JdtcnyS-fhM<F`le1NJmb
z{CF)X=gy4@|GDjb7f-YPS$J5YD7J3${MmJf=gbYQsD4<!qh?2o*uKibxX?*||BIiM
zUcT&Nt}|zuPG09sZ40q*MYE}LZT}wk+h<%{#M;Uwn)LqO-t$}S7j<`YGcq!oWMAu9
z7rPsDAl0!xS?^XZ(Mch`K53tump%HYe_K;hUgFif6ic@=X^+>|-!r}0^HZ?-!GE<A
zYOLYw<DRZwzc1<QtE)XSmXn^Ho&EZBY><v@+qCJ+5)Ze5&h@PP{OsZ)SMEiN76qL@
zCVe^aSVR0{u9VYZ4STj-W|+hCv-g5QdK>TWzNnwZf%$LB-1e0=8UD3TywDkOXHGI_
z>^I@mno|=5I4@1moGRBAx2r_+>gw?2pkuA(+y8Ghy>xE2x&G>~wMicy9Ndz7ds?5Y
z^{4Om>t8d^?E+<xDO#ai;^N|cvewHMXmE8aoe%!^=jY9h$?TRzPZret{B&|f=*fEV
z^6UGPbHaGeg|3KS&Q~gBvcPYyRLb_a#OGTW>}J%@?cXnVuE9txX=CzEk@H(`%=mpd
zebvg@SF?`Zzds+eT(0!>wY^&cUxG5?n;V9kQcgBapDu2ke$FNR>yjLG|9MOD?(XW5
zHt&0Sdb)ApBNwBT6AKP?99neZbDGgi3Hv%5=Qf^&b$@^P%rsIJ&DpB`tcOi6-gCzc
zRp)JcERx&}axT`biTGK2)x#~{`qU=F{g<}oRi3%F-$F2{dA6U(y2x3{$<GQkE-|`%
z44*MqI8i+4=)O<S=hp}AttwSgQo6CbTwm2|O2x|8kDom|_Tu7V(6LvW)6On(>y?tR
ztqPHUCGvwg^}~aMA*;iBca^>tl9KY8X_Oiw5f~v-d2yCm?x}Ngt!El0v#pKV>czJA
z?Yw&v=Px(3wwQMFe($sCi^U7i#`kvGzS_K@p4+Q(+M=6r{`+zwTW9Io%(!!LwwtYb
z(L#ovoY1+ORsXyU%3?mNCAx5-zwa{3H!*j%b|wdI6WLp@6TffHs?gP!mibDH>Bsf#
z-d#N@a{HsB-J6S_`$cWZa0Jc%Utce8o_FU$yA|t=9fitqad9%%Wj%j?f4{ugoxjJe
zRAIsu*KlX=Ny5Uyht=oT96Hu3J<~Xy&$Ua$)AV-U_NWaJ2k-v9=E_yJMlAZ#-bHI8
z)-1an-hFT1D^cqu5A5R#3k$!zyxeY@eQiU*Lnp84dUKat|NiJ`H)x4+@^L=5UMbhN
zx3+#==+~wfx5vZ5f#Kl6gEjyEe!sFZ*xhwvh_`b0q$fLe?J}zTl=AC4_&k=GcC}Tj
zZk8=RSrZ<>VxY$6%5d2>*VX6Qk#8EB78>(xtHtv3^YvnO9Jsf)dU8zw%ZYim)}No9
zO@4J{B?}A7g*}zUhYlb9deBep(c{OUqkdCQO?mkBdi?Vf6P3BT7HwD|<PmgrPi1l4
z@3-b^{_}ipZ_C{np`+!c_386@`{12Lsk6-UL2J=Z%rLCHD-Akl^Kd(V@R|t2Q&Y7;
zOTp&7ludhb)4JdR!_`%x$rl&7f@XOaI=4^Ki?tGU6A=q#@;r5~>a}iDbF<;>vmt9D
zK$Y|g4b3@c)07ny9+W*UtFQl`vRhYea_7Ilzd;dI_w%VZ=(OpKj1~8usHj<4&5GMw
zb#Z|svsKxf6WelcYsBo>z@<Ft#E-M)_iGj#c?3Os^vLP&udgR3C^D~&+NxCi?2L$Z
zm`|%`<GK0vpn2fJ!ooGt+xyJ#*BHCzgg-koGxO#q(1BAWFD^6&FZT=FS5p}i8~b(H
z{*=?x^^ae_K0Rh<(Z-seMOtBNIu0fTB)Ys?G||=5ENWZM$!lw)CC&3>WUWdrTwSJl
z=GNBi%I|l}Gw<v$)DBzoU}|`rXW{K17v1Fx-`p_laZ_IEH8tsA6KmDu_(_}7&tKY6
zn9Rl}bK=g<;+dAkYLS~#0ux<C#8h{F{_sITMTKSYMGw%yzIA_gbWJ*QGsg^c>gV$L
zbzT7h0=CuPd@d?mOB$!8Tw3DE!pbTrA>na%SE*&?rzK@A;!m#G?$Zui<8ic06f|!?
zRVy?o(M8U&eWB`2&>>RCkGF5hynJa@sCHcK*Q+5{<J46-`T0S|8^`RgoBQ?k_0QL$
z^Ow%bJwMmlT)zH~Vfeb3g|)xG-Puvtyeakcqc?BX$Ue3N6?Oae?Q?N;1%-{>j|a?O
zeMBmmq#~!Jtj@i?E$7yjPC;e23roGllai7`rldT%#w!2y)m6|+_mp#UEWiGZGM9RJ
zaWVVv@9&?#xVU(VW^kMDY_rVg6~~m6lwRD~X}mUi`?C7~|3EA8ugBNNwh9JDOu2I4
z^5w}FE(AP!^k_xQPNT%cL@h6&^zFAnYj|RJm!)1@<oe~^UF)my0TZmt^O~BPBy6ip
zVt1D<&Aq)XW><-)^=V^+>TfxF_Ut(^!%+EnpX}no$H%m~Iu0#bVwRVecV@2j^mli6
zKYsi6?4eff&?zdf*6;tf$ZxLIo7>y_Ykz<H_-^<6RT||hqqpn1_sgl~-`fLn_R8Sp
zi)QE8&V78mKX`MR?=ru+r(RuM&B?=~Qv2(RR?msx3vd6axPl_0SK1tOaCVP`A=BiO
zOBSgJ&Nj<^^tj)CS@H98U*6spS6BZVRy*ZzJOAaC!Rn75J>uZuS(0&ak=7|;&^ha%
zv&5dAnOXDq>-CbCmt23(I#IvGf5H-tqg|rO@9*se#oemV)kUwaXo@~n?e6bS|M1`-
zC{2T^$@TH}e?QhSfjVrrxApeh|C{ma>uX^F0f&;3EnJ^DUSC@aI#F_S`gzc9pIxP|
zm(}`$#<S;NTJC?oiIw}sg@w#lSA{BTXn43*dV-3oy;Wat>?~FX9j)j!RZHvC{saSw
zjQjg+L5n<ZZOxvd7duN-J8Xr)_n)uV?+3LEK#ibFOFTh~@?|WGRv0L8bu0B>Tk0)-
zZC&i?@c3HSemUEY-rmr_xCLvYw+C%Za@7i5b)=0~`ouij-CduiY)UzKs7KORBVvQX
z-s<niu3w)HS~C4>*NwMjYCba@Zf;7|jM-t(&Myx-==D~j%REEVFOx*prktGg=I-w4
z+j4J%YOS>M^L+Q!{0y1#T*J<8-owM~#;K=7Kqne*-@d);6Avh8Z@ew*le0Zle!tee
zl}q%;m&^V?A2jm^rP_kp-Jqqw6BL<mZ_oD+wY74MGl_kmwDQ=EjmeTG83K=w^)8Ov
zTcvfP7_`gi*4AwIm>3!Rzh5TvvDzs^7RCt+2{~O`6Uq4F^LhJ^-@k)aPp{mhmU(&E
z()9E5w&dIlvhDy?-<v_|Cad|b2wu+D%+7!8=uubKN;l9JpV?-)8CO?{GPChqm}@w1
z;$rvypsiV<*VaTjpPZx`Dj-<-u;kpLC0cQDaiApo`T6<J_p09)etQ!s`qaM2LI%{5
z0&RBinQhj4u$f(3!%jJTecV#VX0{$FQ!mFr7ZEYnxG6o7#+!0(8rl8%;0)T4_Uh{D
z&?(DB1O*)zyY)uwud}WG_GV$)*;!hrjD_Uo{XvIq1~2E+joK36=<;Og#l#HHt7UI*
zxn5r8`|;z)!~+eC7S^w4t*;N>9kw>=;k$R|+IXc)>K++i+nyg^_V(7%U8S$L#CM(W
zUOLg$^D3wo*;o4;bhxEy)|Cf`TDi4OC0oew9qW;F-kf%JhDoN7ot+)%T-n)Ymp!sk
z_MWDbcyCXoPQ(TW$3T}jt6rTh6}AO}%5I?5dxnLNTx9)fJG8^sU0CSMuB5CCI$>@7
z-fvPLKYpzFaFBh^$79kgY;3DM{$HG^?7kvutJdplYZtq83Tbt8>?x6o6j6NWCmCP!
zk=1*e&cXBN``7RL#kKqHx<hwVRa9EIM72JAyPdze^mSNV_1mp2`hx1}pw)7>wr0;X
z&7Q{fRsL{AGaIjjaavE)q7Ie0`zK08itN08zy7~(x0vpZy1!QT|9&h!ca#})Gm%@b
z)X^!L!E(1)e%;uZyj@>HD|XkGvL*GJc6Rf=->;v4sFm9|>xzb`i_0XQUY)Kx>fY0I
zKqaG9>8pT^Nv>u&Hzs6d7k+&eI?FWs*rTJ}H}_V5SAH+{=;6bI{Puqw7Q6NCD0w-l
zY)Spf{QLWE<e0Vb%dbo7@R+n{qU%KW8@o!i&(1PEeD$g*c$>-p_xt}Z3tb)7BVpKN
zl6lGIV)4FTue7)2-8}`0y24|^QCl(u_f~yXQc-C+Gt+prg3_W58k6R@xVwM89$&xo
zRIdMQGu@rV&;Ny=FSL-kvOa$Pp32XleQa+X`av79K-+deYYI0e9Sso>1dSU^(bv-I
zGD<xqARyrI`r6vi3GF+Io^r*-#qmg)bnx5%31AfjjR|_%_Q~7(Jv}uwa%+}oTwI*i
ziP|@1*6;WKw{z_lOFhydsMXP-61nSutINFZ?f2_uYX&d7up&@dFK*9@raR01=i6Dx
z@U`*Dt^)ad+2-@EPj-Ux_KofN^85Df`}6Dd`W-uVtk5{n#L6A9r@~NCQBk7T4HV=i
zvTb4$pPzbI@Zj-d=XEhVCG6{JR;@pNu$leiw{K~O+jvEEqe84ZI#epHcyyJ{e-~eQ
z@W7j!n{VzYRKC32-~Z}X(DnyVCX+Vza}0EubpHDeH!;^It`>!lSl;jZt*0HfW<k!)
zO)K+$zb&)gnss$i%E?JQ(q=x^EgdS-QjsDP#X)zA_{=urjgP+{QhCtZ!GXbT@kB-E
zwv2Rvi7Bq)J33Bq2XD`d{r2{@`(n4=8(T7iSM7X|`tQ$AP}@LAsA=-#y2afdpB5dy
zp!CVfQie~;G|T1No0~H&3X{6--~0RfJLrA~v%EVU%a<=NV-WN#`yw3f68A<wq`5J2
zbK1tz*I~>2=FVEQ-Bd}5NnT$5Sik)G1sX~xzgI^qcPo7|YH4i^+?eD#O+VgmXVKH3
zJKt`;Ez7#LrgQrA>F0t)Kv6ewIcQI&RoNSlf&v4}q9-1Kx34XB=kJrZKesA$b;-O%
z9V*$s*n}ree+Ie?D0sP_N!FE)8#iu<Y>ow8UhwqP)HAcq{Xv0e+pE)c!ux4~#fPt7
zRg0gU@tJ31c{Bg~zS`d_A~&mnPNpw-c&Mt`RpsVo|2V<Qo%<tr*+3hhK*uoGM(nBi
zX~ZjS25L5Obtz3M+PB<vlZKM#!IP8KR|YI(a&&Z@q91Q}bM5}KX^;1Qzqi<_m5WEp
z#3O%2hlWm9qDXTQY%M^CO?LM6bxUi1e`8^1pKP3dE@kVx`E|cE6B85H#Ow@uuG!_$
zwJ6_2(9<aE>MGak>tZM0W!2xDem?E%tE;;fxVi+X^y+kZOuD?pQ~1u^yOUinzq76W
z7O=Z4*DU{@&qAlxFV8wXx)#qrpfu?T=&;BK4<4x0TOaVAu6J=mqI2DkhwQAbf}R^q
zK#sq(GI+U0{63p*as6dUN4pq64Yk+T*H5;s{wBW9?&HUgM~)qP^y(FtO3Z~vZ{PYZ
z^_nVSkkD|?0H#0a-=Ci|O|!*(XPbde?f&!excrk76PwNN*L+^O=KPP3k3DCb>CQIG
zWdbc%|MK$k<^A>bdn1)5Ey_#@beXgz@o<}A?XQwUhYxRz&{^UOIvVjCsASxK;p>Ml
zU!I(ps0<Qp<CO-T#QpB>?voP~oA0RzdR_!OJ}M_i2Xqjim6cTcVaJ#l8RPVGH49%C
z{Q8pl@5f_)hP-=wJOcti2Wcw%&N8WLc5z7zba|4x;bso#0<4A3?MYjs^8M>26AQn+
zxp}8n;MuWW>71LJSpEI|&&{=7Zte}L(_G@VyuQAEbLwd^xsqi)N0So2zq_lS6T8fB
zZk2xmYe#42!z(L;b8c>O&7TH}=Z-xY>-T)(GS9!ar26~2CBC!8nhzHI-Z<Ml-zfLi
z7V%AT?EG>nN=lFBWxl+$RJs1&AJ777G2N&aXNACNeLCo<+Qh?c8`ICrb&2cu-OOP-
zkhlHwWq<p`?YHOuwmLJz@bR}5i!NrU<lfo>>L1>(|3A0r>8Y2#LK8*&!=1Z5Ha#e?
zm|<0_Wn1;-K-Shrj~+RJ_F-mSsaUsurfGJVl9H0a?6aWbY31wxOsx6&=_N0?fSf3<
ztEAMlV#SK%=Ps|0w+A(?ve)nZ7IR+m&ZboE+*?}?-n~25v6(G!d){65`h`+FPtMN%
zKF|4?5~ww%biUzk-tn6^XI_sx`NQS&{f=JMb9w5{x3-+mott;;XyM;9p2$nPAAVtB
z{BURLYM+@#t>50>e$6X7QRHx>s=&l03U+pKizf?9dx?8)yKV62iKLRU^4By_HsFqN
z0fEFo5Gf+&3ZjVTs8||KdGFUZ&#r&QLtUe9f2Lipo6&JQ+pmAV9;o6`npCvox>oba
z%~F|X`FBp={J31___`#MrH4PLTy_OVeo#kF?}-W5C7B~nT-b5PUfz7i!#6*zwmH`e
z{0AvvNuTQA64X_)_Wt2*#flg6%?|DjlAV+O_DI?5gL~vB{C=(W;kqS{<)?>7d}aD)
z=_x5SxQV%{9Icr>_2Ja*vMzpHN=1JidCuie*y#7K<v7pH6H*s=+&_q%`~LRq&dlA6
z%52Wp*!{yz!ERAWEnF8r-{e?N%A3EYH?{Y(HAG*n;ykE-e)ke@_H7-fIgQ)rdP~@z
z`8DCCl2Xz0-=V4k6I0Gl=nc<$qEvNPW|968=NOY~uXHP(`sy21NbRh$O*A{s`<i+F
zzYQ;ylom~N74+nc+*qT1$#U+K_m%d=#iyn6-rYEM_2lb~$w&4UdRvt9Nl#d}O5g3k
z${*4eiAOSae0uQm=BAY^z$svo(w?h5cVxYi3eLLz3X&AMyjc2W^pvBV2K(Zg!*4Hp
z`e{?`gsl>lX=l~TA1`VU{9IirXS3HW_iCt%%0xcrZjVpje%GAaI;;M@iMz@5{lA-D
zfA_g_=w<3ngMHKZe&4(-@!;FfPX5_?5W8ks=q1|E{&8M?rk;`#sNPfpiz_LCBb8Ln
zq^_4`^-7aW>ULULCieF)e|Yfm`M0J?Z_DP)ol{^WG4J%J{+K;yTE4slr^uuN3z=P_
zt2$ITLB}mCDLvY$yFYYw*q0X<o0E=q+0F;mdy8DVA3a&pY^F4+C<9c^^?**OQ1_qr
zd|S=tn`ff)_quipsfOeTPE=nqf4>r$&IT`lfcYDo3CZ;SF9{jGcIQ-qiBncNHnV}w
zohW&ENl;MG@z<A^o|Dyh!`H<m-rZFyq8a4kDkNA5N%NbEpPwuI_2uR5|Medq9libE
zZ@%4LE)}Iu5}@eNjB;=|!0ep(@KEc?>H71zRKW4FGV<=O($4Pg-RV)gN?yLWu&}vH
z86pvTYfEP3|G(eg&rI8X`{#$l{Gl@7GD+~qkH`J*Ek$BCrFd>iJ-zHu2e^oB*uMSy
z+xp!*cfJ(Z0_ya1oRB?t?%a=Gzmg0jI=Z_T2QGHg>Vit--rjcfVFBp;h^=P?1%H~`
z?+9fQtPDJK=n!a+`NoYK_5W<mzgrI~fb8w}Pv7)+sn^s+si&t&@UXq*QJR$Vf=^HB
z)1An*Vf+4mi$2yb@4vh3?TJaMr@bTlXPf6wGRwU+{nIVJ_GVS@X%8MfI<&-d@`o>9
zuJkD?oooWN({}#)dOiN-ot?(5+~R51*Tp_LGt>C$wME(2*ZIu1o11lY)x}+<+53Jz
zlfJand-+@!mq{jbH)|-JKYFxVJoDZj%eS|;gU;4CGuPTYl--q~`Jlt@vbP+3e9!Le
zEEdrU=?GmNmYEI?Do<M(K6YVsza<_Mm7biL2^#4DO%NWsqZ+*}2h?Zd6jpoi{yk_A
z>-qWl;fdh7FlZYyJKu+M*6$6{&&hx`Wj8P~$KA4QI_Wr7E0lweZ&|^^L!g5e0vEd_
zZoeHW0cminT3Jce*VkuVS>d=kY;DB;I$Jq8Iju9o1~Yv?6_W34Gf;cYYpT}KcXxNg
zG%0FmaPaf<D=8^)@bWGVT^*Li`&r~(#be&EbupE{H<W;`(Wv^mic1-kk|){BwJdI{
z`ughPa)0?{{`1c*@tk~Oj^$>q&+<Qi{Q}*9^XTzo&<x?ZIhI;opfus}Nl;w;_=Sbe
znxU&imU>SIg*9k%acgCpQObz}GYpd>HY6}AyZ0TreS7wX4F;lapyFL6^Y*@4YgMl)
z37?;x&AGd4>O9-(M~@yYI%J~`GUN8P+$$>ro6GN2wy)p+PipnmRfjrGa4)sq(QzW#
z)y?hF=5+sS>*LRFNIaZzVL@Zpr>55S_RE{o{XrK<9_bYR`0ZO*pv#li3nvzJ*jxe)
zK!mIi$h^Ev)x;#k@uTX=na1kH&(Eda*pOKB=i_mc{Cjh{#r0PifU|`DanMCgF*}VO
zAMa0pd1)!=hMq4kFRwPxk6atIm4la8HT~S2imEE3#6v8P9zPD15Uk8Ak%)462RaJ|
zG%`3vGx){r?fP$TZ$Ez0bHDcFtOegce?FYH8MKY3OI&~5qK*^B%l9)V72UqIH5)Yi
zk(ikH=HA}fe6m&=8fuAha&l|p_s_ewHX1Z8_2=*3&c41dNTQjy_QQvQn%dfz*Valm
zHa3E`yoJZthPHaL@vwnMO-)RubaZgo*w{quDp?7M&`<YozAa;7X4VW_Bf-igqF`Y$
z<JHyGs|`*DJUcUUrb(vI<>mg%^Y8Bi9Ry*Ver^RM5d76mOhD&F2#AOr(ck~)(DeAa
z&ef||b4@-Ix+0)4c6Zssr%zp11}&AaD$yt`EY#}hurZB&pfu@}i>vF!-R1ey^kPpf
zaBR-Fwq|D6B&WG{wX@`Gs|x=8sZ{fyx5oNy$BE#JCl+~VZAv)EWMgB~(b=g8iW$S!
z)6@0ky{GFf^`CDCI<aC~?rkAK!G(vwCCZ)G{r3MBq@A4w>RwtFKLZ`H5m)(CR8(zh
z#)Sop)!*JI=HA*Oq8Bqm(l~8}hSH}aB@$68tlho6ponH+WBYV7eg4I*+2M2M&W-;N
z6~P&6G5Kx%(S~ZlTU%DP{kH9ozG0TmSFN06kfP~*y#G{U)q$Wl6BBjj*)|JX&c1y9
z`saDam7LN8m)2Tnm$N6_-8f^xg4{h9GZpr=8rfXd-MOLKu-GhK!I@2X$w|Mv{ZbQd
zr#!gT$$KWIxJ4rF>0&?M6S+nSy*$Y`-{gwq-e=?A{P3L9B7WfyC&TQ1F66fC(XX3-
zV}HGUU0vObvuSF+vs&8Q+wC-zK6yw*ig+#rbucb2a^>XXOFP~tTl49py1t?Y(+rKf
zOE~kQ(~MH5$iMc#bDRA_S()<O_P*p>X5uyWH3b1I25G09&&`&${L*07wCoLQ#{Hct
zGkw@TiE8&{UKZ1_*?s?@pnJ1rZsCO0Iku-}mwovA{r+dSwN+2dR9g=4u%3QoRBE+8
zWQ|ka<i;5v64HEr&TG89EpfMf0h@k$?5{s5#{1^{T~zV$QO><RlJfHMpgaBM+tu!<
z{JhK#oRTI*#q6t@2^zfuT^rlT%npij(4P89*R^=lgqp5Cket2Y-n_i@*MBTI<J}~U
zAJ&yOY>Mva>DiSXCw#L=wAtib!&HsabKUR6X`fe@-D}9a%*XO%;DdcCX<G$EQ+K9M
zx|_v*Kq&QK-FsyhSJ#O@47PbI*8F?uqV%@xz>(c=e?M$(Y<<mlztZcAr&Em8r5zz<
z3^B#i7G4&=`|+2%?3y?8Hc#1h);qbT^L^Xo$-=T$B?;f(-34`xcN8RETNk^UzaLb}
zElJ^PZw8%Hbf}dZbohyrv-8T}<$gkuyDNBDvo9Uq&snHlz1h?zanD)7jk9+?wDXaT
zkUzfFx;XxZZGW3DQ^bw}o~1UnGj~5vN_uAU`S*I`7xy;sJUu;ak<0t}J5u($1Uy(?
zA(neS(ECuvqr1IHb^m{||Jf&Z{DHq#(4WcjmgO%6lK)5ruL&{y^G7OhTTSF1wlznX
zT;|_)tlz)SN9JySkm{`ur{@di%r%@e;fKJjAN?uS6+AmCFKx_DEbE(hYU7dX)?ZHx
z8P&y3_WAzA>9+mpg0~u*ox2~!>J(3^2#>4%8k%-?*3lV;$uF+0mF|@`SF^E+kyis%
zJD!<*vQ{pwT%t1!li8qCt253%Q`y?Lak^~9)oh>SX5qc!?-?FEI5cxgZg}^cBX>CW
z3MyX7`K}v(PQrV+U;M|{SNwP~?{@AD<UFWTw_bbx+56|$ElobLy~rT>TI}+mRw0Hv
z^%L$2ZfthjTYhPmdwNyPJ@MP?5C7EE(cZT&)Ij3?^3M~Fw&?wwcEk0uW$MNaTl~b0
zOkX-|e|I<I`3>*$2ctEf-e#GzC*qGYgJ_Z3mxVfi9n(ARpLwI3-l(nfUv19+%IEbt
zyn62<&%U(s)yyzbS^53g?c1{-JV@XY(Qp7|D;}OBdD~w%tAgsllXuU~HvjqO^Lfw#
zJZEQ{ryp+Pt*EL}ySR04V{saP&Hjq^87sMOO4m(UARl*pd4^lg#&SNv=zs53EGi#P
zs+eO|U6{W|M%}XFpVYPpZn^nCCY?_?u=c_3KkI#@{Ni?YYVN(w9B=sQyfVsk_xjxv
zEu+)sa8*n?o^!x7-*FF*q~bH3*^l1FSk^Y2we}CFxWeI9uGec=IOCF7@pBoE=P^$u
zqRe%*fB!pkk$L~7z31FTjP3OpqHfxyYo3)BsbAT?K>pa(t5fU#{@R#$n60U)Nh5d}
zPxZGq7u$VY3c@&DCankEWK#J#&B@8>1Zcx9|8~QGM}NOn@Me3TmYSVu^(%c7(?tFq
z5>L-<wQ#$`)5cy`n|kIk&&F--`{H`eb{*Ww_4W0YowX0oEVVzplkv#>UGGibni=fz
zjGfOmWsm3|rT>p=el+V%F$`Sl%g`4ZfBA{>kF=NTj<X*);(XzL-|2Zv?k{d)<=#~F
zH48Lpx!A4OqW0Goy9}k3bKJySCvBWDL*mqQ{rNmiNm2|ka}G|p*}YBoj6~1-&xuQI
z7;YrJ|5IhU|Cbu;Bj=j}vkK-dQhvlUn`f%VY;)!#ZiTOYt<^49=8O|^ZsuD5_|>Ez
zHQ~$;dbk@lck<l)^r%Dj;0D*<8<U=X4^@`jtUSrY@S?@uM6W+va~^9wn5D_>YUwJy
zKXcCZ8y$ALS@-MLwUuzTf6gt_o;ZC`UjD?}(N|xD9&5RLJ9b6P2S(-D=gvLYQT6-J
z$!>P@qFqtJT(fvjCf#2ZzFuzQ#*M!wbasGRqn4ZF)2>eG<hi|f;->pPi?YxAh2;F>
z{kmzt7vtnc<=6EOPA6J#*gQ9}`17ULi_2%2e9vB0|Kx0M-d&6Trv=^g<$Bt09yprx
zwIlOx+r7SZGx%~8j^9w2mByL=Y=^0U{GmUpiY7+ylP+{d%sKAA=*soCGj1%loBMlv
zLSntg<rJ0KyRClgd(QG^zj*rFJpP{DX#!&MhyE;m{?Z~{A=>_x_u5U#1`_wv?`=zb
zViv!kRk<PUsrTjTYwK&KybYUvB7DWgMUvG@Sw;0B+uh}o?@c+`VsS{qzQ}5R`Tq&Y
zi=SWmv-zs$CxI;;Cl;^J-0opymUTq~bk)d(h0ehbu0?isZAvOC-@pF*B3bV#tY_ZM
zm~T*dbFG+8SV6hD{mC;CI$?9(*TwE;?LI5I`Q^*S^R5c9u6s9Ql4leTKmYQerCz(N
zqFhWsBW6=M%fGX4ePNbVR{lTS+i2FA9kb_Eq@DSGKF7qq(4h9kwHbH6PV4g6w4@kw
zu;Pj2CChH5NjwqYI)1V2j5XH{o~-bBt@hO5Z@a2n%eh5fzYD_N@hRLFz8CoP(Kbz%
zdA05N_MfgqD=DpP0}UNW8hp;%%wHgBKT&?2=<oB6f(O7$&!?*dCk6CaNS(<%y?B;$
z#7(<oq4j%ay5_foYK;yROSf{PYRlkPy?6NeyUMsHP7I9r|C1pwFfii%uDcs=XFvV9
zWU+bLfilgO^Y(K*p2U84-sa+#wDUJ-XL-WPXq(;lxIsm*pl6=mn|bbs)fjbREZTzR
z%{z0q*wFB6tKqU-cC~XePP8Ab{qXs7@|#-%O-%MjH)mA$MJ3&^N>)+)sJP{${K;=0
zyji~HpWni3J>$|E6XX6k)(4N~U7fS;@{}gE<}O{Om0jl|MJA@47Zh|lTXz4`<38sp
zCFT!;ZG^8zmmf<09keND?$Mst?>By*%)j%)vFTS{PM>*i@m`O~Pc?jQ_ZX|4`fb*@
zb2X^&WzCa2mYx~4?{0MacJ1_}qm|YPslVqe-SYNIV0nVz#1MDoZly_YBB3Saj3=AK
zzaKkYIH~#h`oh1bQVWg<G4BamVIX1jFpnkpm-YAJZL0SU+!TAvu>H)HipS<P-J*%l
z8hNg0P5t>zG2{KAnV~s?o~59qaq^Dnw{};xnuT}&xfwr7^RS4C+41msh}1of{fS~+
zF&wH1r?$^%e9^9_&z_sUUhV&z!kTM~HCONUPt0lTowM%Rl%_P<CNc0RpvvFU(zo+6
z#mao84`glE66u(Kr0qg?MM#CH`w>YqNv(^j`D}~M|38)X+)O7kcz(}0-ILQaAO5?r
zPOsCu)bzodNqh{m?|L1PSJb&Tdurd@eM?I|o(Xwb)6lCr?b4oPYq&Kh>t==qyBQnY
z;8OgXvHQ2_rlsD;kA~U{KMI-ka<=`0$=e^V_&3qqd{^n?Kl<(xdn*j&(wN2dWsVD)
z&u_cB>~7qh1CpCNV)n8<a-OkYrDeU8=t5Yw{iMRjpK&O{y*}of+~V6EX*?g#AO2Ig
ziz~_{NWJ;+!-5q&N+&O`2wZGY{EWxBoiEgoVFP&W+Pir-;+|VMGXAsJD(m`p&$)=4
zb4TXHA3t%{QsUCZ*ZUl@4ki^J&GC}3-{R()Xu9)p-X_pSo-*^LtGwFU+d)Uy9Xfn?
zYjo$5BGBZA$dQ)ccNgmY;;~<M%a`+ftjL4X9dhS>xmvHd_VIMR-PMd^HP;KDSvOwp
zxEag7z*XhzrK=Z|CKc_hG%9_mQWSgRZ2#rupHDYtNPUq!QoVlnN;4%Te-BXdS(K_5
z(O9-uYlVQ4lD{WtBIC>r<--vIH`=%5-hTA_`SAu%?g)!ozT-OQKV5n8I%l?>z<edU
znoIRk?Rq!=ef+U5)%Zj9_p^`Vuge{6oL`^t=F{W<nZ0W!ChBm^`FG^@&kH{n^9x_d
zW2(QoRll?2(4vkd2G{%6EQnB*uPa=8?nLKhN$V<(-njio8VvW_T~3zReQZm**wddy
zN!gFY*)Q#Rw&VAq&?+07xYXnmk~#f8InOy~8BI3ad#q67*0(FKg@xtL^l08UF1Tfu
z^S|&P_rcSV)9vhUmUJxvl@}geN%rTD@c(`ol+W?lI!t{sr@<`En9bJ3yHir`%XjK-
zcq^u)ym-UC|0UT@w>G;(Ry<h{b*6Be%hrUN+pngqikPohY<D=({=|L5?QSpLxVT0Y
zoSxdhsy{kCIIqU4bZOj7lkZ>i_H11h=n}c6|8B|kIjJwcxR&pkw;_ao{{2($EWI;t
z@7eOp6123UV~Ng1^>X9miO(mey)UY7mOTIe>*9CPFL!RJmgAkxvZ3IigjZ#iQTch-
zH`tbeiye>m$%$Y(uVklVCBYbz7Ij#t#^yiQ{QA)M-?A-sFFTlU!mBF%fY7lEZN3tl
z3m9vo%#wwlsY_hwQRF*OaDL0{HQaxu$lX7m>h!&6ip+*7CywhLt9mby_+_W;7p<Tv
zN=Nr4e82d%L_cT>WB0fFoQJNl_c-3$wKT!-Zcu>BB(c~BN=oN_q@LIJbxNL{=^-!Z
z^b=dbR#Us$AlJLjieK{BhtJxYcPp%x+pRu*Jb7R0pWn78e`cvY(Ax7&F@AgH^Lc^W
z4`)vce0Tca6XT<sHl-Us`ZMig=+rwFubw`9eyg~0R`Y4`YrKoUZ>@Usca>EiH|vbx
zYCUoL&x)7#@3=bucJ@=rKcCD0{Q0*$CoWSZ`#^uq)961F&+pukZo2%n$<hU$+eJhs
zE^!5==KD8gIuk!0bUgm4Ui~0w&G?e(dv}<FHlNtk_nx_F^f=_eWScbRb(P$YQcukI
z;BDrw`fIA8;E&USNqT>dEqr8pvi7>s;{G$s|0c@2{oKcUae2h9i>I<@3PnAVFnV71
z=<uQMn>KxazSU&)x3EJ+MU}-r9`M-Ado=IDxyn6}MjIG6KbJYa+;QXaeeb?mdS~8S
zu!GOB8=MVn)=6$=+K~7Aq&xeH+sBQ1c1Q{_&EV~@|GHpx$JvVVjKdcbCv(Vbm%VQ_
zuYA+XR^1=HeaVln@E5Yb*|?E2@#@$8aylCem2Wwon7O)K?dIYgB416Cr<Cc<;beA~
zXe=(Xt-5x^`9i5}skfsbXm(lTgk`wvva+u!&sT`)+$#RK&VAkHya@SgcTPAq{s=H@
zQL}m0VR2Z^S@qPjvkZ{-n~nOhW1$DGnkssveamv-K5q8xe{9>On{$NcOFMm>6`T3Z
zwKVD6+{fpcwIytR^LI(jk=U9eET*9L$KOdNeEtI_(eN0COBVO}RvgU^d-mc&osNA4
zqxY4{Wu-qRZJ2Rf^nj#Hc<1iyOGkSbM^4--e|vh(>2-zjn_s;6+J3J7;{!Lp@N6%~
z&3umC9zuWm!Z}@@>^BZt_gN=xpVhQ$YgZX>NC=e@HQ7-AwN%Pq@`!uf&J`G~JfDLz
z4)<`(Y54p7(VDw&8w;=V9D<hbvnm=5&9~a?w(qQ1+R`2M=DB3>Qd@?9yxI5azCO%$
z(tJI4+FIZJ3*Pka)eL^7ThOwx_OqgvN#zN~%-^YIdXcA=O|Wo6Y&(nQt-6q1UHi*y
zy=VM_?`z^$PI&t;ekx=G+f3*N)X(qMo+!(cy!toIs3Ozc=t+weoBFT%7w*TNK3*~H
z;hoyg^(Qu7EYAFQGfTEfY@*7^2L%=~TFoAxOc1TV$>+Rou8q(UTVs}fX6BvuyRB|{
zg-rRn?zf(Oc9@QsblPUiho>Gx_loL$KO-vs+-qTZ*TjQq$1{wW%GQ(_NWMQ}y80;7
zx-=iRAI0bM^lm)Zx3r<y@6@zY|L<SF1!*S+z}kr|?d`9d)h3++&2P@rUCg<8H|J+&
zd6S3l@~Zmu^A6?sMeGyo=<HK@bA;1l^0Rq6lah@monNgnGjF-^WBaCACu`EVL7O-q
zHBJ3JtqHslt$VB4?(F^GLEeetXHC;RqqgY&oIf4BQ8LKVRb}cWRz0LSg#YtE(<wqJ
zswEG7wz*pwFowN0RbZ~{p0+G#>7^7SrfZkKX#Hi3^DeuvW5Mm<y<Kk)i7jhQTb}%P
zyZnzGwpu-+pPy}h)uX<+-Q>t$!^^HNK}w*>W}%ctC6m?mNZGV<O3(S2d%rhrYxjox
z>vcnC`8_ad*xWhk?d}Qoi(d!tKlN{`@Xvp*70%t3|Fh$wEq`QEpFow-w|U#lK(jZX
zIg3w+_Bef<HPuk!&WQxG$!il>|D>0w|8wE^pI+Et$nI<Z(q-fBDur9Rfi7;5@1pyr
zo;acLZz6kgpK<n?K$&ej*YN$_U%zk9U8gCcMo&+CO@U0*sc7yhdmF`dZ&%BMboLGI
z8=r4a;PGSie4q8;z$W<*A9W7BX%1)9zO7ca;N*dqe=K$gEp1zW)alUOIHp^?iLcL1
z{eS*mt-E{R-+Y<+^V8m_&1MnuSkmkvEF9&)q~Yu6;NaM`ph?74r=hFPQB!Hr0wq_G
z4FN7X3p9d4SX=_DxD_TY+HxUV$n56bH}C%Z^H{NHk^PdE=cRqmU!;_!eK+3xc=Pvj
zcf5~s^>02{TNTk;WwHsj7T{)Ve7(oUi<c#9_jhlPo9%u}vf6gi7fIdKJdZZ7T%2KD
zA?LPmf80FjrAwCz^00kn6&Ad^S)<5w_uFl^B@B~T`fWaOfUek!-&eEqHM7)DPL=&G
z>*YR2Y%E}9DDQpY&Z*5JWxlZN|MpGmljgnM^YL1c)mdr(&o8X@guivItAE4yY0a#Y
zdmeq2sCu3oa^h3rpODXw6w`dRUVOKClZf8E{sZl`KkK}`-)RZ+sheH;y3g@T?526M
zm3dd)u3!8&{6Ht~^Z0q(zO&7GpU<oAW0$W9h&K`pJiX||)ips&y;RiIg=J-ZudWK6
zX;W#mGb>2#v%xj#-yfg7;LP!fI=6)JyV2*9);}|XwZhy*SiPU$F?^wOJpHtIK*n2@
zUpCfkJ%?^II2H1K@O|ysS{;`de{@<%=b3rcKac1aSH8b~eujCjRPgh)tPhUrPZwtJ
zdlPeZ+XT5Orz5XVGZxxj{e0GEz4N}HwaB1#O&x;DGYpfDO*J*10BYi#*e@(7c<{tT
z@D<E6Eee%*rOi|<EiI?l?0OR!!o(0J9vXem+xGCG@29uV-y5aSda;xDhuso8XZNqm
z<?{YCtaVW0PWN3<Jnv;mc3Ah!+x`r3V!iQa4xEg?xWLWymC5^4)wS;qS*P-x?wfT}
z(_n^E)PpzQKP*38%sBu2@zeVIefCy;&A7CLGyB>aMMJ~Dcs)>MbJ7>IY;jZSY0zTG
zUMbV00}YHRX=!m+=f8ZgPgSDQ!nFU(d+vSDs*OFia=C7DKDSLl(TrWaF80iCH&Lmp
zI&QCiEv?~CsoX0Q`CZp@Th^CZ8`{EOSx0X9$MDbgzOmtAPF-QIpe--His*)P90}tO
z=)XV1q*&Cf+G5%N;(gIu*&CAGzbDO7Ke1uQx!pFp1*P-NyTx=LJ)51M_V`%un%Ldb
zmd~$~S{-`60#p@yy!+!6tz+};Msm)L4UM1$C|k2a_x*e}d-aFOEFQl;`QEEI-&_B4
zxy|L}{Qg_kczjO3_Th_k@XHw!<}cB|_w4F|`K-eBN#XM<-kF47URPzazGidlrGue1
zKR&SZwg0GC_45CWZ%<wJ-M&1<FZAx7D=#9qt?^QNx#0hmeRYu^lr@)~y!3GHj6K_x
zCjYvB^n+fx!2Lx`vG;Fv2=1IZH8b@5?mx>*+1H&@sQaJ*I#(2O67n4V{eLDca_xTQ
zt)f()Fk|N{Hg!SE%YS};2HhxGSXj8{+pTO7tts8|_jxM{f9LLy31f;gw}`P!R;cpy
z{`{z`=Z|}`65F0U8;e5A2IIJ&>8IC=Ez1N=!Laem`5f<)t$f%jUa&Lf*Cx<Z!HKP>
zUVk-8IU#Vb?sx9I+HaA&%HOZcJC=Q=jaT}`&CTjztEYZ^d|X&ueEA}f@{W|-n^L*&
z?k)!{r{DAQ*=!XRl@%J-Ikw+E``}>n&nJ`pC5%!yVt1DvHQ*6-bJ6Q~7b&vS4YY`u
zm0Rq>?(+OwTQZ&3$L%fp*>I1^Ap4q*kdP4Qh;GnT!u+;hCNwPqC;a)l|Nr~#>F0MW
zY2$<m6Ao<2yc{wmCgc0NyIV3Zw>2`ee>%u6|KT9JynwLqV#tigpS{o}cxh*59PAKO
z&bYV7a{u3Nv$-a7|M&|!Dcx~x)K(84pOnA9zFt`y9o`DAs(wBf5*Ci!oW=`UxE!~)
zYR{ifr+?Jgi9XZXnsd`A>)M)={r3M(fR4w%zFuBYQBkX><InyVjN*cfn|ABYuXx01
zmV4{SiHXWrrAkFXCwTw;{XPBOo=VWVLbKdkD>RgfF0M!obYakl+M*F(_fz$7JOA^G
z?($2EpP!SlsR(G@dB}IJ)znp?s~;8Z^zicw3ji-A`<ZW=eXYlLwprks2*bR6Khy5j
z|F6BmSorkjo=Rg-mb<f~@XS2h*@nr-1SBO_9_l#Zx;!sK@Mo~JdES!f?RhLrjG%oN
zH8p#>ie^^Y$bm+z?^VCIW#f@J@VMW;@93iYf0aJnzbNYF@-GCkgjYCrR|)7gR?s5t
z?YGw+icz~?`(5_xs?f*Z@7JdvZsYy(;-WKT`RtQVMm){!a#bC^v&{<s{3sL>5}E;8
z4qD9l_V)IEcDafM_4zfEw%@Dj&b+(~bonr37W2ueiT3q%YX0-iOwkMmoeiIRYYS+V
zlGd4e4GA8jvNsW{!q<Zq*Zp`pJwD^giiwapxuU15f|v8XzP?`F&~PH?7~P~J9DnNe
za}`fq8@qd&Rq3k_SHt5sR(woq<B@dA$XLOptTcU&T!G-k`nTWj*DtUB{_f4q&FvQ#
zyHB1mW5uFB_Dh#8HO#o6u<yqs?lwNzQ_cK#D;9N}NL@P3Rq*F^uW33bx8JWj-N49P
z@t~2t;?2h67FAzXaDCRV{rB^^ii*mFsZ)<G^PMds;tE=%Rn%^lduzg+IX)E?7NM)d
zQeRzJsddU8bZ$K8)cOsHhi91OO1<CvUGCz=i&{OPgzfQ8P*U>b?EHNv=U5g?nCHoW
zV)OH}vs!23Km4rWkv8kGDt#5OzpnP$+GzK+QCmYGli7E)6%`w&YKMb%G)S3b9O%~H
z*Kz!DVB)!tm;LQ0&X{pxmT7jx-YV1ozwiIwa5D#D*9p^t#ElUzE-q%5Hp_Vc3XG_&
zS!d?k$G0+qrhX6e+xtw{iv`U}g0?hOe0-$U(-E`ri^-ObKhHtS2;^$N1cHisP(|S5
zbH+5B`LYQ(gPUXse7|3BufOky(!+-jtt`Ql^emg6o5<Nz7%X=0Klc1{?6mI!94s3m
zbR^meBfrg`FyX+Bjmaw`HZJnx0M#LaJ1>Eztj+ILG)tLgt&mVs68HG7)78-dE@C<$
z>nF(MfVLWwsrAqzkPA*|wq?#*c1l5MXXxIlub_nkdeghNY%w`>=+K6nIWkry9je~b
z0(KUqih8;{(Z3PGBzW@drcIkHs=jEr_sg*=n3{gQaz80KIaEP#;{Jts8G<LTR(*fx
z>*8`mep>jdkj_J`+^ZeHD`JXtv+wLMY;<_ACN(iJvE=nN-=l({;F*{`_wd67m7kwU
zv=wGWDhMp_oo%Mo2M!yKlDD^}e)#Yq_<!u)s?tS^7l%rK3$lrC)EDVadviLrqGHFc
z1>lufF6VY!h}7AVeVwoSXw&`o&)4mK7Zm6MT2SEP_xj38;e!bi*m$KL6zvQV5S+-p
z#BrzVlTDt!zDpD5TBTIlPEV=NuD|Qk>BQRE)$`~4_o**Ge)_XOe8+xgci9Y(BL#n&
zPnta0b4A2Ney!>IRepbFZElIs^KCzsUd<;Ew%tzHZq6!ZU8Qi-vk#P%riVVXnHX|H
zc;;)n@K2oP*5@SFxw|~cT3U@c`V7igE)ZwBfKwKBj>`4RJbFq+uA9wkn_kaSPf+Zr
z|1RgVe75s>{-f(Jb#%lCWkrfi)DF)n=GwNdGyh^mS*di3wLtgc24UX}eemjqrlmEJ
zYa9%|GR&~{tX*pVt|!7Nx#(Gp*`bX`!;gp+Hx?U6KfG~j%Hc;=>Gr9o61AlY6W_m-
zU!$!wNqU8Jl*_t^gIA?Ll&^AfnXMSn?f3Y>4t<L+k5)$*`Dommqr`ms;JaLw#q$+a
zjGC2&#RLoEjh)WK-}rl?WKsO<Zf&&_59?x2?s=HVwQX(F%dJWMn<jpHsL-dU>+*)1
zU!aLU+NbW1^nQh>U!TU9*J&tbYW`rpXu;i*xJ-59>i7On?91<7U-nX=)8pO$hwFqo
zPKav%mHy`UhtXajY%%ZA`;v}FO5HirM4s<@emilUg!`!j|K8M=8$M{bZ?9N7TRO+j
zz`VHPUfKaJH^U^WtBM<{)PCRp-=*W!k#kk<`r+@wX}8ZEyqI73$DIG$j%O|+yhr!f
z8fxyH|M11O#|%sAOvIj7Ucagu0!~VLs}C$Hd{ei9-+j`zL<_D7?`JBuOk3A^nQ^A)
z>rLUe>VI6R?P)v9W!b+!!l}s0|5M0O#t7~(rB~mID#F&Od~N^QGWA^Z%Tj^D^RYKB
znNQeOVmEVl-P04=`N?4~oDN2FP25^2b$ps-!QA}2cO7LF1W$gw!mFzkzNTZ}`lOjx
zr(fx{_N;!a`NzNK(XLe+UGB)tlVv-6xFJO0^?GxWBzC4HzE!PTCNt{q(`3zWf3R?=
zjHIEydgipFg5eDR_4$%^-{<=`SEk}Ex525TjsH&WlK%eMq0?jCvT3e@JClC$MR!)M
z(Z4q5|2eByj7$}iw+pizdY{qH%ds+g>g4_%5@rWp9bvLqcYc2D!q2it`lr75=ggri
z*0}YZut@D9v6<F3Hy&9F)TXEQ+=%X|{T&%2$ac<s-XT%32RAyNpJe*9=K}BN4d3^?
z?&^3_22QMD4?b}AY?!U_=h^HZm)5k1bspMY$d|eL`Q)#`wd-~-)pAv@@A)6WCUAho
zNNkPbeO^;dqo2u}g1#jtsV>Si<U4!QVWIV7O@kk@K9?*%T~c21dzR3*8tYG6DyP}x
zDinXT==&GeyQu$25}Q%sR%x#t+HBP~_=4JW4wb#z<^3I$n%1SyShiB}w9AuC9)5ny
zD&I!W4Bq+YXL!NUq(XU>7g~*bQ{<X{Ul+D0`86r^$dm(z{FvO|Ex2){Oy>3V&M$Wz
z3yvo7o>nQ)_{)ELvsmA;JD;@E@8xFbhVdDF|5Nz)^NtIp_IEe;%Z9G{{BH_>=>pd$
ztH1$Ww39!r-Re8vCbK^u&p%rz%fhfYEpttT;hg31;sx{Ts~dIM|8KC^@cKG;o%jEg
zSiy;R9KP#xow&Nzr+8g<rSd-K%$EDNUByiWCwBYjb}JQ?-u@)|ySjd<RY%7k_b<xe
zrMX0|$z5k8l^*D#w>r1I-=OaJyO{4g_uZ_Y@oK8+>{T5xe@)D{bi|m|?Vlgs&Zlbn
z^@r-s`)AgtO`H)i_jOXT{>gv0?%eKcX<>nuk2|FXZyg63q$IVP^;nl2%=u*gr{8~b
zZg1ngY*KvnxmMU33BBpw>=EFS`k(Il>)wCad~bX>z23s<aPisaTt8}`6k0}X%rP`H
zG_<+^Uh?|M|L5PjS$FStP3h4Ps{Y@Ucrv!!@J)n4OZ02Agq!EPPs}!7&$pwagJV~E
zpk9E1#FDRjzfay{?l5!O`zN~=a0EQs_Wa}VJ5G0ROBFsk!YNxK2&(^f3M|XKwDGd@
znj0kzMY(Fp8=u`{dnqlo-)GiiHz|{>w&Ra49ux;Doa*rJT(bX{q<`o47bNfQ-*7`_
z=fBU>vdo(H-iw%ZuarMBGSZ3>?3*X&|Ez!i*=PP==BM&A`b?kPstCHVAwuJFlFn>?
zi7i=|=Q$`W5OxGtROd{eJYBfQJe93bHZ7s}+0JeIB*LVhM)@aQownHK!bb0P(rO-H
zGylcA?YC@uxcI*6zQ;aJF+bOz)o<5rk$yk-)OYP|A5JXxf3ZXW<VwMYPGNP!2S>v@
zUt0*fr!;2o61z6R;M9HdUj2QMhm6mEyLwU))Pg9=oo!cZWm|23+*dYDKq}UUl`-d>
z$0Mtnb1nV*o*X^s93lW}P)y0bd}n9z%TFI3AF(x<`{9O>)(Ocsi9b8)KWMw{vz;iY
z?7mV#NvY`U1s*-71B+a{cYNQ+u6h5TuoBm?)e6racI@c;(Pu1avV?P@pkT?%OOdYN
zFhBVhbZb|v?yZ1rXI(yttX|dhhc!JU@R3#B!Qk^|$~JdgZ*R*D6%iCXdG!i!ZLhbN
z*P}kQvzn#fBNlnzTh$c95bmRy_v(Wrf9d;^Z*Fc5g*f$2=I3W;ujF5tA=DeMUcZw~
z>x8{P-~+W&+yA$G<1a~jc&JqioN&F~ZAw1QmvuecVc{<3oqGTDM4!Z;2;Rc=^p@YV
zch$ExmAniR1-Bej%R$@tw_n-U`=@iS#*_<wtUv2JcI^{e8okeXvGx0;x3{)JOgwe<
z?T3el?*#oeQ+i|_w4^Fr^oi1m_-AHb_bi)kOPMKJT1K)83Qml1^>>bU4hjz5xiybD
z=eqgQh_`WCCtO9OulG2Fv2A^An(%Xyyk_L4C0*d;Ui7tK_uVVI_Zsf2G1V-st`~h`
zAF!b)nkiE9PSA6+FumX4_MnE6Qqk35^Dm|cUVpu`p|H5`rg!kFliFG*>Q|gNJNHEJ
zMsAP?oZI=p5p^~Abduil@4qw4!nK2p7HL-ZFI%}=r0edjX4!qhccQj4p00~?xwoe>
zbjs<BdwVK#?(dUr`0}MB?d+_hzrMaU<Y~UPHd<Izbg5Ej#~;?iCceR`C+0RCRQSFn
z^Zcr{nNzZVD~heIlaBG)U;9_);mg%OUbg>iX>VVxFx@jHMMY9lvZbYE&Q=>^<H_^u
z|IM6bn!U<HNvYoP?uDIGGG~>%`68fgCc7YJ_DQa;=?8gx<JHba&SUGH_3&dw$b{2N
z^6&4v@wUv!*!cV}_56E#RMgZ~fg8iummQz?;^?3G^$(Ag$tQ2L1gQ<VY<}Z)@6&V7
zbWff-738Sb{QKSR_xz{VglT`jTP`owt=9<31lJe*|5`im<>vYYf0!?CY-?GmFnwa;
z#(-r<BeuzGDe#~DjfII(^r?_`*cu7Y+5g!yFTY$8yxb4mpnkM9{lsbK%KTG2kF3rd
zcy?xSQw(Rk(dyo!wdxzIP6x`pxWRf?t>$Iv>ubFmHe9x!qol0t>ERKO2+H>Bbr;{4
zIs6))nL{UNzwA!qet4{U`KP~Kx+Smw?cgfv+?sp)Tgd7>4-byH*5zT~_V;>SVRgTx
z1KsO|?iZ@{onX>BA$mee$nEFvWgnkz_kQ`;M_%-a(5d<M_Mxl8&f4*XY0Fxb1O&Rc
z=uMEDc$RgU|9rpm_s$0{ysYE2xI4B*S87ZC8wI;N4Dl1br&rx9J9mG#Noj!Am4-76
zhtq#s)XZCZ#k1go&bea`ug>JTD`E5c>)qQsJM7;*z4((+Zl`TaSF3K#-%PgnrSW%e
zUD<QfvHP`d#pVM%GiuDwAMT$SuxrlTb05ueW}KNdckddvIdi9(M@ib>Exz{LR4Jz?
zFE{<w6rX8-?dJz>v{deVZP4r~JWD`Q@?gWi^RHR<x$G)^eQc7dw?@zsj?`0A0>P!_
zdS1}B=F^A!g|8Q~t($R4Jae_0eC^xhfa?Zb(QCKtGVES>a{1vn7f<W$V%r=Q4(!lB
zFE7GacjwXa$WZHZpL6Q(JKFy=YMLOb^3y^^%vdig_R-^<nC+L({}Hu2CC>eHngqA@
z{<6=>Y2WT||MyBg;CN$wsG5D6)Rz1g51uKReR}l#j$>X{$R=Syj!h4hY|rHIcdM;?
z?BllY?u?F(f_ZPQex7ApZ=Q4Gz`MJ<^-u0Qbm)+Q@8p&imS^YYR%hyMFq1BQb>-m0
z3mq(=KE(8csKsUcleM##I|c7-5%{ED?bmwf_v)LQ|Ep$9DoShEBz?s7;J;?Ic1B~{
z6B8LCm=jv0M2@Mbs;{ZtdEoGLOa0q*Ivf$_Z&mwDag^JCa+`6KRP4eB=UNs8vbp!q
z+;Ki)f5;TaSDcfbBL(749DBS~re^<f{uz7zukFuzJ2zxSPJ65!Ll1|xA&bObtKW9p
z`kz(Kh(C95b?Ldi#)lV#WorL6`*rTt{r}UIA6Y%JK9_P~f|^T9`{k|K)3aaQxw|Qq
zyY|<Y#G9K^j~qW9zv*w-ihxFDc0PlwD@$jd)zH=L1zi{Ns9XQiLT7f~tIA4T-9;}!
zXOHZT+OjZbcIOk#u&?`XcB-E^SE3Pj!0mPtTmD-k!=$$%MQjl}?k@NzX?xE!zwFY^
zrm273eY=iC+}Nfyd3qSTBFECK7Nd=2Wv3r5%$2jVNVuSS(eZ_%{N1!)=k6`P>vVsb
zUs7Yo9<fW?t6%Ovw<JE9oplbIq}*hK<&2qiziQtc)X04PGWzY0TDG6ciBFyQ4{Gtr
zNSI^^y50EWqxR<0<|hdgs#xEc^ndsmYGtw_QGLVR^ZMy;ZdJ|`k_Zetb<20(t%{HB
zdkWn|Ro6WhxB0t2t)+eUj_>!Xi~m;cuK!;LTA<z|X$%@c0qtR3xNzalU&i{eyQcj6
z``ftYM?o8}wA*Ag-x5aQiL1Nnp2f9Jnlx$0V$Wi)trFfPC!_!Gju%qgkhpaEV~4G;
zZU!&ix+?Yg11tI67I$-Vj@+u)v8ngx{M|3jm75K-|4v!;>dC#y!S9y+t}OV&`RpyH
zQ1B_orOkEM<T#U__B)@rvi9wQHL543?3}4{Pgv#EFM)R_zI?tqjX~haY{|TNe-E-x
z3|r+cVRPev<1;b&#bOmV`W?4MzdaK_nRP~um1;BR?oHnflS{=C3Vxq0JK*&9weFuC
zU*7+oD*WipjXQN>DSymO%Z^TYw2W{5#}AA5&aka6TW6bo`SRtL*VoHGJv|+?z`gYK
zwU1ve`#=A+fFoeB8}GC8^Zoz-`&;>H<?<EL+d=z#FSPo&h=@Jizp?Uj+AH%6pH8*I
zC*!vrtlwkKe19QB_cs>BTW-6P!m^GY68Cg;I`_g<Pb!<$XVagX|Myloi^cJr65W=t
z`M|c`n+xAOW{uk;aYrD#K;`wH->YMeblvAS&F53px_o|>uid0eiMI`fnbfi`J-l#^
zA&y~rt5SpU{NCzExl3}sZPYp0Z~lDm&GQ*rKKGxhpX8mmH0Ms}MZdc3^CMr@|Nr@V
z>b9bc?*EI*9Or8<I^a7c|3Bwj`4?vnwQ?6<{rT$iPYKH+m8D)&laBYv>csAvGCjU-
z=86?7vfu5#y(Kd^Y+a1!>uYO4hpDWM+M4wG+S*+!K&xJ2Y^=-PEGXN{CbG(U!s|z~
z;wRO%StdT?l8@tgmm$-&^X74>19kICH=ld4()RPs^x0qEJgxip=O6oW(VCTv$8<gz
z6*cx7cK?0SZ@OuB$J*yEma`dW==?u;_ylX*F`Z9_b>$O;6rbPTbn^M?nue)jon_nB
zp8Iv1p<)wvRpF<4``2Fc7KLhDFEc8?e|eg!_q3R=&sSy6s`+x!9dwZ*=!~Vv&1oOc
zS-%h1l5sKnmvzq8sFzn(3VV8bhO7u^lrqh_u(!ILPsU;a_nZz5om2nM-`iWgbLqF-
zC;o3-7q*m5&)FZzXu$s~vhof~)0yarcWW}vbLPn<d_Txg$CDLuCNASh1jo^ov*#2d
z4$j(~=aZ<tkY(n6Lk~BGoO5s1y-BvzofIiInU}q}C071WvUwCI|JDoJlO;I)m}MLu
zJaIN>3p-G2al(*CYAfH;G8Pft%;#&D@tvsbIF@JJp<S|Tcg^+nb#<qim>8d_vj3L|
ziA{gUEOhGT#>ziB9{WwU_CXtRkC&BiE_-`x^|zS0_Vsq0oSdK$c+duUN#isDaq;E)
zA0?ffok4fBy}q{gaNhRdeKnPAJQ5C7RlB;BmBR1w-xZImc*r{S;i<A89}=^!tn!Zd
z$*vLg`r`k(_A>6h@5+Mg4mouOg&q3-`{#!Zj<(?w%+1+$>|Qr{+4bm+5jr8-9ILOT
zZ9L1rUDaY`=pC<a3+dNl>7O3P?btV?w=4Cnipa)`Z`8EyYi6H(x2Wvaw9Su8A~nQY
zv_2{xvEt<tH;MQZEY^DZ#?FmP=BkOlP09Q8;hG}%|JAP*&ZGrg;4%*{4$wHWpuldj
z|77M6%{+BQyOv+(5kH=N_St92u{~s4@XuueNtc=b?0ElV??g4phhdNQ)ZM#XRu<9I
zzxlGRBFpl#3U=2H{hRA3Wmb^z=T)-bsim{$>FdOMIxX@ts!DL5Wxjjzj2SalKeLJ5
zU8W0K*8l3tN}KO@ia`U?kB)Y~=Kf>8@Z-1J`O8CBht+&MDsJ=jO0b0t->Q(6O;@u*
z8(p4sCCr$$;#=3WS+l&7*6w2AHd}Dy?ca4@E_Vh67W|n}dF@8`x`5^;zZ>P}3e4-5
zdbBY9yX^e-Pw5U*4Jp2v>F=)R1so}BT>oF#P>`)+Q~TqcSH7+NJEgXpZDY;J{wX?<
zpStW`-QJ%6eCP9dpc9us`?vh&S|x7IT+mvde7KF5N5Y^X_x3i!loJBCw&ivwN`Obe
zPt9K)yI!%H$voiIxkRmFrfR82UHWw851xA)z47+j@H@Y)YknAjMzMpJ`z@{h{_e}0
zo5IsiFJ&$l{PW>3|Cjgo<r~7|Yp4GD`g*m7Qqj#9^NwoY`SsOPWbNN6T!p+^MaAlU
z^7eH{?;f8vapJ)T2b(p+*U3aiMuJYQcs{S%&+(%YXzIfL&j)7P>TgdTw#$Q#bz2j?
z-48PNWO-cl3hT@7za`D{WI$!k-(O!r*Ty|O+#VnLJv1)1{oaFl)$bC&zPf6Xd8x&%
zSE}%S?e~qz$N5CvK?Uupkk4uC>$mbfT{>;yo=a9e;Zg0ER=4#A{a@5>Vspyt{j`NW
z`|o>d^6c8q{rv4Q@qG(_u&__w)mm76!}afd<Gr_bR_xlcPwe?D!(=w|yCuS)o&A@W
z`=@_;ax(nS-YGj<@8&&!*e;*;_t)1c8iC12e~81@0yMR>uvC40b#$V#dx#9^%p&d6
zlSJdL{o(;t>|1OmpP&49%Gw7@;-3cy{ZCA=DbzVp*xh)TQ_km`?3c~oD?Y4W95ZEV
znCj-}TYd7}o8AXB$ch)Hi7KD+ic!hAD}UwSInZp5kgzc5#F_Q`erXjyJF~Fp=_#!<
zsRD9x=RTjeKmY3L>XT>AEXldK33P~KPY(|?L?+HMo?86MaB`^hW{HhD+#7;cw>-V#
zao13gP5pgbcc+ntXyG}xH*8Ct9)6JQcdOrd;IOy%dAUEkJX##jtQ5%lQ1UrI%Pn>O
zrrq`D7rp1v|9vjPUhU9=kSmsVw^^>|?>EpW3R_^>(QW(JjxlS_oLm0RcR!xi-+y6C
zX0TPsivx%G?axe7_0G7kppmPXwWGHebY2JOs54N4xU|&!^EvDH0vs$MGJ-$-Gitbh
zhHolzTc<Zg_+D2?q0!2HtvvmnGcG6GHsWtAQar*WSgE@t|NEyWdvqRYw_Py4cx>L^
zfISVYHjZA$ZoZuRLgCAV{ygK_7V%9Vos`)l-J4v*Hx!jjZjY>!F7#WlX>y&P{JbW`
zBTt(n<#+zHxx1ixb%2(o@1>0s?q?c3{>*bIyY<qB4O6#lQJ6SU>_Eyv9dr3#xjlJO
zhtAD)eB>4BpSu{+V*Gb`<==HBnwpwl-rt|Uv-tU^FPHs4KWvv55)xV@{Ew^TFt52m
z>M0S>@jUzM{#HEhH3!WnmEW)3-leQ0J~c}0sr)Iq%#~@j?6uLY9}^<AGplQQB3>Ql
zUy}FH=wCX2HP<c8Y)+%pJhkrcES<aTTEbX!zxePxY|oVBnKSd?^y;RB_WS0)BCAz8
zM1>717IjWOv&84uxie2z$2xLN+qN-v<@B?RH@+3C>HnQ`bo=ei%WF?EW!-zVq{?pc
zLHV>xzn#rj-xZ76R`BWW`N)dK#nF3gEbN*^c=9?n1}xt*x8n@Mdi@U{E`DPuyZv(I
z-)VV!%%aml2X<^eZ|8lqOSH4IQ&Cg1^SFF{jN?Zw(C%E&@i!M1I!hWPFqmdvd-8mK
zJ!HF~o;q`W+Jv>ywLP<+Ejykgzngz1i$Ub9C+RPbF;uSK{OG&$+1bt=r@m~?n%qC7
zCM<^~qDrS)d-?N8W|otTKWr7uTG(<mCG}iMP4T{S4-Rt5@lC$VQ&%+aWyto*S9?`N
zwm-W6SY0*!<R#g<4bdywIA^T!<YQ2K#~{#fb^mY0OXrrXOBA{J?(yX02?EoNx864`
z`#t^o>!(|<$1N^-c?ooDx2Sg5heO=@pxNTff3z-dNOYd27wa`$FLuZKeZN6d<tI;`
zgp`>}`MVM})s>3PSN$1T_U4V@UH@~EZ?`n*n;w|K8t10>!7y#({5^?m2>~x`_z&MW
z_e;F_HTRr--wo7GIsLP*wO5-dCmg+Is*<I5nd~XPS^J9bT|Qa6<o)N*&zSgsH;ZLm
z=YRb2QrN4h+ln-9)}JkLWN&s<^PT1L^3u|uFPG1sVV*CSzwf6SE4SDc%{tS1Yu@Iv
zx3@Nyy^R7LF*4Vx^u?v6+|}RS1Qvp3hV*(TuWgKc?$en5t9V-VCgwYB&5noG@ju)s
zy|$%2>cwrto+2^f4FUU$%BoJkKf~AN`euS4!|clke9SG`LiSxf8M~3~<5Q=Km*)iL
zSpME<#gnv1N5Eq5e&6=ygzZ7LTGf$fXXi{1OZeBh`Tl144N;2wn&uu!Xs{FZoA9Jj
z`qT~0Z1c!zHMu2@=VuzXcZq6$0_{+){FGw%??>{fX}Z7V-(5VtD)X_VX_m-hH9eKL
z{PpFVe;j_Zzkl8?p7--}R0~S9=l50h9lUJ*?_s`sn6KKtOWJQf%*y#!XXRY|CE?ey
zE8YJ*zv=w%Nj&Ukn0LqGnD37!k^FD{ORwmD1{r$Tt+py_=Jyw2cCx08Dc>z8r_8<K
zF<&o#^Lu-nyE1(2{x%<3JUu;K+}(p0I<fxx^3pjgYn5ISsN4E~?vF0TEgETzJhQfz
z%#+hQp0}{vTb+;RUCzrTU9!*qJWy3Vy&&psz_Ahz<(%&JiKq9Rxh=ZD<!OLgDev;Q
zHLOi+zEW11K^K@R-+oz{@!=bje%$h@V!AOlvB!^{nJE|4aQLN~>*_Y6B@3I|SKVT4
zZrk3zA!TCw{BN^OvW1T=n3iaAHrwc(xBl9E{)3H<i`n?z`EV?35$#g5(3q1Nu+~wd
zR@zE4nSayC?1a-?AAI%dxvGwKiC$S5+<tFwwQ<1%2T=Fp>#M8!2j|I#<X&$3Ag}iE
z-3Pfdw~HRF+5f9tcggPy=9Tpu=U)4+_UOmtwGoTE12u{jrTyll^IxC3Ec?5bi#3l|
z$eNbNzvT;xKc*SkZpvh}zN6N<+I;b{WX|McJrCR5&T2e5k;iyGc$w9=?`i_2-mzU4
zJT}$g1|RPl^3>HE{wrpF-ZisEI&f0N>-zYgHkF?YGA<~Dt%*3;$jlD9JNj@t|MAyf
zUp1*IiTe~g$3Oh^Y<Hj6-TFg<g?B@L{_A#;cwp{%(q``&(JX<ZH&6e2vsLruv-^71
z#k*POZg_j6b<QQWrHc~o8&2*CXqGm(Ij!>P)%=g%M?G);l)iZJ;{Ph?=xsR*^Y8Bi
zT{sO|)%N@Se)-i`t2*sFXZuIpJLMf+b7S`2Lz}{9J#9I}z}>!e`SB_78&tnNe*XQP
zt?it^17|N@VOaO#L@;B~=P7w#w%eTC@_U{7S>B23j2EXiFodnK{QYUqmI>^qpP&6V
z`MU2caV2Gc{%Il~*fv<L*>U-fg>r1f`5S&wv!vU#h1x~`u!n!T@HYSR5&!o=s+ZW*
z%`Rzfl-;&``G>saUN`kEH%{~0I^C$|?BY3ALC?&rLS+sGd#ek)x2lY8oh~^yd(-oY
z`IV2J{h9u!=GLA{V`g?f6-~`f@%WmF>~a+h-LAQ8N}sNmOr7vQ#Os&5^Fr&-@0%Z;
zydHOAlkxWIL`Ok|>R9uHw78j$ACqou$rS$h@nguUkj~QA*Dh{MK7Lw2gt18?_1%M&
z>bEDDuGY7IcQ!BIxL+sz)jhF2=0Eh#7*~0G*ZJQuE%%o}?kx|YM~`Og`sMIV-YF(0
z&S0W?)IFcGuQ)^A{pMt6X8)NKqW?&%BsJJ?`umyXo;mYA^VzcPJ2_$A_JEbo+27dh
zXq;#p6==9n$+qf_^VeBdSAKh>bLT+kU2lmik1LP$z3_K>>Q}S<FNeKf&D^O6r<PQG
zDy_Tsy77a>F55C;Z}&artGa)Fe{ZM${(!ZG_ND5-^*YY4)RwJTb>OY&=h$hV7v|jD
z1lr5&Gsj}$_4xX^$K|Sh#5F-v@jIuzy}y6{ks~hi>V9Q9IXmmWuKp-%wDQ^-(X~9>
zwf50W2YzKvUU6@!VC#oDeIMP!irlA9e`#K5(r;tb!1~_ykT$>Fp;=qrB#6a4e)6*a
z`o^4N9$sgp4xPxqyfRk1I&5Y!d!K2)&ck{8yemJfd}nj@cjX`MxTllX8g9vp6+e8d
z<=y(4W2?@`A2Y9I+Q0kmnH`@47u~iw^Vr|UPrR7-JkQU@mTS2Of_l&2d+EH^>1O%)
zu-tpHP2Dp-9oBmwVH|(sH1qcVf4{V|hn-J7J~vnM@s2&kb?RzrMg<QX%HH2wtKSJ)
zIdfvF5_9xE>CC@5yf*KBlh5Ast@(KF>y{N0#P`4dQvBlG-`fX*-d+iQb3Js|+^lD2
z*Oo7N^nK^fjp^B57CeVe-#Nu>-oD{&gj!$6htKZ}bARhI<V61zUcK#>gxMQjsaScx
zeK%?%yp9K+NqwLb*OuiL_u<W>-w6%L){`Q(lyIB#*7pkDU7d6`bJ>?_zwJ@Cxzvre
zWNK`lH~;XmC9HpHp4Wzy{EcP!^67KCaYlW)>KZFamnT|_Bx78jOgj0+;i+2eO0zY_
zvQm1B-ga0XVQ@eH*`Vv~m8&Pcb|lXE{%Pv=8N4<B?C;J${{HBOt54Tizqcv9|M=kB
z>XQsj`{!>y-yq);WRhEG9_q)ofAMdRo7MZGrDd+Xy~1An&EeJ^cKtIE-@Y>%Ea(39
z)O`KrP6efK(XBeCToW!V+&DQrs_j$I>43OQ*;CrO&Xp<$-Tu{ASe0eIc=GmKqDlXc
z_xEzIO8L2&Z+*QXHt44D!5g^=Pgc7<{~arFP3=zfZV86E`!D0yuVY%OAsch_Pw36>
znzP)?KJ5*h{p#rF2L0aqm;LW0R80|`U%g}ZwXYV%e9O;fGHiRJc`%pn(-E=bU-ll`
z@bdMCN2#CRT$kMWL_=|&a713glRr`4ie(Dio-b$Ge|hEc-%ED<opa+Wv+caO%!xPo
zkN#ke>yxvw{or45^VgsEY3$RA`h%9Asf;qqH8bd+zpMV!fnTj}R@|Rvr02Z688l+C
z_1Kf<^|e11traPp;WPJb;=^+ap{uPb_pW{X<?`BkZI4fa9Gj*8ln0($aIe>F4|}s!
z&UCK_0-KNXCg1qf+EbG#lifG#&ZU2>v%Y5eXSN^I|9o%Z{SB%tlj4(ZTv__#Q)u#?
zD~&zx%d8I{>s{>LKeOjY4u|<Jo=@R9ZO`ZWeUUYtCT?Z#bmfJ9#_W3i$;)ea4^FN5
z^V6$t=4U?r+>eHz))wZRjc(uQfA{DX$M2H`raSC5E&P(1Vw85}Hj7#PY4g*QXW0I;
z-NqLF>0^(`t$K?YlTN!|Tv4T(WitD)bAHJ}kB=4a%&%WqmlnG3UhT8V0f8oa_RD^~
zeb4dCPWhi1M!{|UiA#4*-5xjJ;ZvZweg6!v-`8F>iA`L+!PCR}dCbjq^A;BK{3<I8
ze6(PC-sa7>gQD(>9-GVh;IgJ<&0OQ&IM`_OF%uoB?1Mi1lO}Pxt(aOUJEz9-=-H&z
zj&c2-rzHEgwYf`7*>Lcf@w}?C)+TG=E;mV;vz3Q_E(r=`TdD5DH+PPo!Kq`rb7g8w
zkDd(=KX|3?|NVu<VtO$r7C1Kl`1N}I`a_%Vdq>R@cTag@nXJ!Nuzcr1(azUQlfw_Z
zK303~f@S?iqrC0nw`cz5-BI{^?hcv%cCRY$W<EG0*Z17Zp!{8on(wS5Z*FcbT_`$n
z|B`C4r;n^GwOj8x#n^ERB!s(ttN)(+QbnxeM7Cn>?&<2!f4;n{*Hd$xS)w(!Vp^)f
z3v-U70{?9B+_^V{x&QqrWfLxQye_s-WtR_M@=}K|G5uQk1=-H*tTAri>ef1hHE_zO
zZ`!+H?(+PtkH1Sk>2I0JZCsh1ZBWgZ@qMc~`<JUH9aEnCY<sco(XaO6@Mu{*C0X%3
zM(c{hx9!rinDOuB^1{r0xq>^bR!T?dWd)RoY%$)FEAhURZ}A5G1KZ=Pf3U8IzU>(4
zJwdbBIC77=$Ca;&TeLEk#;EAbegAOh&bUj(ZNeKFtgls86x6U;8*0QJXcEj)G`!~Y
zsbB6_-Mhv;W?E^#B5OgzaX-GW^q>DUA#;0C{9OULoCgZU-Ti^wHgAqExV2XC$?*ko
zncBa%#aI_Rbc2@Eotk=>>t1tRn7r9WM$MCUcK1@5*rs+zeEhxe*;z%G)L)TXPafY@
z>-deGC#sAu-9d8l$<_X5-m})+mtyd-i4+LF&^1$X!F6}ZH@a^VLT0}So3FF!`4lzn
z6SGPzHeNsf%Va}aga(tv=bJm!@41~ylRrJ_k4)#Lplf-XzDUhvj&gB%k`!RRB_>7f
zvUr7t%<Dx#R|Dj%ryY#mKigvM18-h~w1X$51$c#$6R#|n+>-lXK~|)?^WxeszpSiJ
z+&ur{6E};`hUk~q?L=Ou@!P7U8*ZB|Y;C#sUizOWmU9G;h%qtxO#A=f;p92h%DoZX
zXSFOhfQAUK2MP%a?hMF^oYH-a-@<A0dt;fescP+wS-f*h`VDRt7<k<Hs%TTTc<P4$
z?po`%i4*So^+&JinK|*~hplxvhGAS5yfRO=NIc7#%pjtjy_iAmbl7jr;EQe=3H+vh
ztp4feUd{<PaHvnWt}pSzic*cO$_p|)t@BMw1Kw6?w@<ptT3cGQ`N!dpJyPa<g^Sw^
z>uwqC`F~zlF4A=753ltb4p%=s|3fz7kKTFTyDJVWg3eR;-{biEnBS>ejTzSKO|}S6
zlT5f!9=9hV<ECBDi?xwUQY1rG%buFSm)Mf~@x!-w3_rUqE0%7Yye)Ojn|m{k-sP2T
zaQqwXv?B1LU!=7K*R2lgN9wZ8X>Si#?wKQM&(=H3S~JJ+VAfLq-2rj$W!!G)$*))a
z`7>uG|NgsP&%Zt7$xBFoxJRe!mhZHc+`qJ*KDt(RcH@52Y2`uMpAu%f$oxOAYqd_>
z_xiigj&nQ1znxen`ljH2Xu+3HpN|x}Og**!u%gm+ky7W=t_hzmUY}W0^6t_7%`eRU
z^%Ultr3b1UV%YELb*`ZF?(z92<ou2&Z_*Vmx+UZvIqQk_*Z0NhJ2aC{*R9pQ(%>_B
zSMOm3frKa1<@#>>1QqR?5w804#V*b6MGq%bo_S)hp(d)~?yIJKRguak!_T@M;&uIJ
zqrrAZ#+Ua&*T2UF)|r`kGV3loUvCj#>v=wpk*VST1Fx$4*3;DMRylRfIlbuji@<`X
zmbNkHPD(}`%VHA~+_`N<YM|cADPJx-eq7>w+>h<({)wf(w!S;_=A+2lumfE4&Rll<
zc;;*Qa~9WzyT?tIWaMp2|LMLZ=W=`IWw%KJ;>WLCWiRrZwzlWZAA@A|zNh!RbU|}Q
zi}y#K6kl8=7`{+@_8qypIX|qlo3&qldGh+tp6?3rJg-ju{yhCd-tJXCw@PFhm{&jF
z^!VJiDTNs@2d#^GxY2Bt+ku6T*<z*I4gO@`UzwnBUF`D-uXis#8OiJ1beQVVEH-t~
z&bJ5McYJ&<`Q(40k?>{d4a=VIeY10+eEGNY?)L?+a-7`Hc`v?&m)$gad*jN@Ki|x?
zn{w`sN4Mm{J$s({ox0ljZ1og|2Tz~#r(gIvF>h8Qc;)CiF?GIQFTZb$Zxy)ndwb*3
zTPv5pSiic&`s$gFYy3Tz-rDG#@aqGE>*@O}ZT%kFGX$4EFaGxYj_$+nGX&=Kx0(NO
zdENEV=I^oPTk>b{2VGAqcf3Ev|0rn7_(X5jeRJmaU3IqDV>Nl9v}yR}Kheh{wtvm|
z#`^Y(=vl8F6Ys{>1fRZlrS7X-6;E++)p@O7dwv_~JvcHyZ@t3Y<!6>YnPOh~Yvr~x
zwU1)mrpKM=`<1KtI7oa8sD&6~Bwfz-@w%nuo8A+xvFn1)M?Fis`CH$}e2?DJ=EpPk
zzhW@IrqxiHqJCd`Yvq;-Ilm=32@}@}&oAHAy=NQW5x4a7{BO5B&$*hJu(_S_0e^+%
zr`uk3(ZbH#-Q$*JEv>oocHWUgr;Ic1uk8jcuuTknvR_OtG2jrJbpOm5GaZAN6L=ZQ
znuGjqOv{^Bes^*VV(vD*cV>jvDTS{sCl~5mwY<(Llk{!3vc#T4+}gh{S5^il9X@1X
zP|c^U8#;5t3QOD5sfD*|w@?1~<MEQ3@7&6E(uaEWnwhKXj-L57W7^mI_FH!z6jm=T
zd3RkP=H&7D>r)Qce!P?aZDH)$RbOQ7`+D`aC2q0g_&qB{u70iTdYj0e{`*&gFDMn|
z8u?tlqP!@4!=9h(76*NwU*Hn+_y6Km#j7W?cS--5l^39Q#Q*xs$S-f-EOJQ`P|~$%
zaGq^zvsuFdM@6|sH%vkMKuf>z&A1EaE^oi}@i)&W_d};zXYSc*T3(W^U$@di)4E>S
z%Ia7A-nxf}T0sX<t>6Fem(<T=6`*Y@9X6IFepc~cDuukCKfDuqr|owB;mMzL*XwJY
z(7wBBuCT}59e>XSJ3p<seCdOgQfCQE!SbJlu2w~%OF?6BOFSpfu&v&vw?*mGbd!?4
zm{s>xuju6~Cok{6o}RYpa?-*pEC28RU77WV^Vu7Iq2Nny69c$AH;PTF|8n7bG5?dT
zfrgjQzj~}DkaBFfdPU8&cAwo-f4^$`kig77GrFcH>OrDC+toZ?$HQvzt7Tl=<{2J)
z)H%1tpxV+?d)mg#=W=ELc1_U~o*uCOVQ+oS%B!VSKCTA@B|XdRwmjRlKm#<w60R_r
zcQ=#EH2&0@Z1(%LPZOmRHXnF5NArhKjb(aDT0@+C>W1xGngeq;{a2ssdQPMv((&4o
z!<~EUt}SmgJ>n#IVCMN_wja%c%zj5qdvaaCYLc!^-}khh#UJK9tuaVsy01F-!Mo>i
ziw=H$xT?3~1nVW+?xK@5miL{*LfLh-Zd5FBmQWPl<bGq@+}~Fk?t1P2wlKoH`h}ro
z_r|+(Zk&8qYxnv2*@ry}+XOfgHvM7epZ|pCRPK?e4(`_8nX7g>>@(W>;?w>pt(J}x
z`<F~}-RZ*fYkTRY%9ZWy%PJYU4bQzf`FD+FqM>&Bt!FDXnQ<j-%Go;KNHn*F>07|#
zD`j&mm*+V#a$9fvVcF)m_}aoNeGQ9L-|o43#j&I5(bR);4=q1jb#YU1`mesK`TWOy
z6ITEJUpaeAIs5<a^#{_U|D}AGeZWx6Pujn~I!?hdJ>Gqr&ZE|Ib6?)s$-da`&xFFq
zX_J>NR#qx<ys}ASde8l=r!%)LeDk5f{OtLUlN6_HXi`4txVlGYmDt9Hs}I(l(79xo
zam{Xy)$!&Zh8b7xoSamv7P0nArueCk0osbYYk2%V6)W4xdGahQ{#~GOVoj#xn!ugw
z85qU(mGQ4{+k2*8IPdW0pV_bOeCA1>RlZ=$Yx%gUOJy7EOzlOx+M3TgPF>d@Wb}SM
z&#si-MInyd+}qzRv2IA$*1vPP`H1KJk4{b&tL<uM>Ty4+d>|;R2wK{=vuVa2qn|90
z>9A?WGUu99-mf&ZyItkxeesk|*Nsijo61_HCS(R}FO*<m>iKi&aN+HjI~O2TfyZJl
z6zxpikX|dM`kv!3>yA@q&u%~5Z=n3tqLlk-y92|Tus!cg3fzj0ZC2gUFQ@w8l=G6q
zTuVN??hliB^~1L;VvE?4-1F`nb0od*W-eh{<KmvyzfeN9`q#_C^9OTl^4WIU9%9|G
z;#L`h{u<d+dh1X2@-cHvC^Hd_xfpY!Scmb_r8^ce>HE|7<Xze$F!QtAzb<oK-t|{q
zmv}3icb%B(fA>O>=T76Z!HysGQ&ZWGPF`Q={4FFe>_f^tzOM9j`=cE{atXUVoc&g1
z|4#v1_wz~f#kuZ(RGF(C^`q=g@sZFq9;eUE^;{7*xy4I&#?F@<OVo_FJhknaY1aCD
zby(V?Pt0#3|8V{luaYoM5eTlo<$7uQ`wL2X^E6-gGslVb*Um6&R{wBt!Y!3Ozs^P`
z=QCMw-#sz6t@U}dn$gGKXITm^Bn9Mq-Qp?8Pt7>Z;#Ozb&dD4t82>TSY{{DRl;>I@
zudJBOGiwarC_d7%@!NF!$&q)eJg+|5B)fl6Us$BR@AkaE%;}%!l)o_i;M?|PPpg-$
zeM0<!v&-iSzp%{FdF=L0Nl&luRBif0-ZzggC)P?@mFdW?4o~_pUAlA@X#d8EqXKf-
zi?di{tnw}_-Pm?)r&Ywt|7UX8w(Fg5D*tt9K^FHi{rml=1%2m=`D*q$em)+T`uNz<
zFZq)WH82)i%=jc6TyFBbzW>XUnRTX>pKtey?AXfu_f*S;^cSn|lzSUKk}_YoE%D?#
z@sByrd$&)!w9(zainogY{gjV?>fYH-+$Qv8zIC59FZbd3JKD8k-8LM2dmyp&^Go6H
zlddM4F<aR?oKK&3$b{c9#OCbmtA}msm^zk3zJ500?;bnjcQsR9nDKx9A>4oYXv|?_
z=VuRXH?{mves9OTc*Qk?zW?UCPpaKixx%7($(qrrzhp^AiGYja#AO|<EPAdlnWirb
z5)pWLiRESJsxSAP-^+R`OxQHRciSC)P7aQ3-kz0p^G|G>`T5+LPj4)Z)9?MLvDUTT
zzWj52RZ_fpkL2z36PG>n*!S|@yf)QG>YY*_U%rlsw`^OuZPS+jOKk7%o}TLW{$5Lj
z>D9HMX275Oi=5wnnN{1quK3-DpQj?eDyYre^TPk|zT%rZ9$2p`X#1BNmwavE?22Vx
z-dWoxP2u!mU$U^Q`<#W}zf))5<lfx1#)NzS|4C8I+aBzZXcM(AbXa5iZwl*TOJ2#(
zDm?eAo~7SASDJFmeyZHveaz0!j?asJ{JAKytt}`0*@QBSGQUMPS0COfQaLTY?5NP3
z@2U(u)3)51z06wAOo~U2;l|^n$MQWpPptQ2-+oR$<?y%3BEPCEly6N}d3HyT!SDUO
zL-T^w+W7vq&9aPNbZoY0!S0>2`EU0xzqg<=<lK`74_?KenEHy3-7M!>+AZ@RX(j%f
zUe;Y!nsnmj9M_3YY8qzy%(1JT)h(_+$u#@g^IdOdY}TGxyHR1SR=RTRy;D8+40`3J
z-cP=G&)ej*bbVphUA`FypFC|msH+_l<8^b{>Ys&&J*K5}EO@9Rw?qDO^11iLF=-_Q
zxo__4B%WBI7190e_`eh0eP?9*x87o(;+MTJW3GSX!@R{|(^uwLh4MC7n^hkQe>i`>
z{fnX-2Pe$7;*}2g6L)}n|FO=)W<1MIU0EOd`(Mg~h0oU|9X`<Y@2%zJ{mPSHJ!fCk
z(N{K6wc5ZmE8V*yRV4gy@(1tVHSZ_9E3mk+A(2^a@;`2YiTjt$bNzXnC(g|-|IOJC
z%kJ^yKRSBAE$-w0yjN#GsLc!cQuQp`;rI4hiQl_^+gF936iPd`^=o2Mc7D~Af3^ZY
zSR^EW%xmq;)#X{A{O@m{(MgY+n^H3`FXMfBYU*P5ez}(RcKhwA)oG^=F5~-qLtM`O
zOqR#%e(?$F-m`v)TYMLoSx|eqGrs=V;>gfBx6_i(M9;dD$=edQgJ)&ZM%9>;Le<~-
zww)-*ad?xdE~jRE@wmD1$$&YtQa1|~W?1iMDm0Ux_wM7~CN+QNm&V6FI{t6{em>P_
zs-CFCrwQt>y;F=Oetde)>%wI2|0c2gQ1+XR4{U^6c{yzB_3Zx4x)ji|dCMz<&eLJm
z7sbslExcW%6S?V8m+ZL<^Z64FAL~qwH!O;_Fu&Hn`1^W&<B#oTOIW}8%xluzVUb-`
zdu-NZ^THeJ43|6W_Z_jhUz_yu($Xn9k&~3&`wR*mIMi+gtzWWn-N&|RdGp-c$q{7-
zUB&8^WBQ-ApEZ+N+4JOE?KSNOkD^x0UA_JPt9|F$r1mE%U5Q+D0JavuR$#*IX8*2T
z8%vJLmfdOjbJS+#bM76|k~c1N&zQXN&Cj!)^=ai+{|^hy5}Dznw(rj;?>;$Oub3Da
zM@L7{VX?c*=O1>N^3Lu_@!RAF^;RGLK6(Fhi-)e*^`HE2wU2!-_$9%!rG7vAeg3x|
z-MLpI?rdaw$$j-mVU0v;_m<SfZ&hWi+df|1)SaUD_S}o_t3G{vb66(QVp^h~_3LG4
z9wn_hbfxU=)3-*gl7CKnezN#}Vbj};WoKh~&fPsTZ(G;ym6wy^uebBrwi*2Vt*?2#
zNJN+I%hHzrr?<#mm%QEoe_>97!e8;X3_3F!V%D1}m%qu-Vq-J<I>&lx%=XCL-=xl_
z@hCh$7kEWS_=tAt^w0myb5Bg#to7LHdV1dfat*<VrtRNZFMZ}UUTpoZBYK`r&V1{6
z3;*7}R2I#!-t_gc<4g4}tPE7`nzrKHTl3vPwwFJ@4860j)*7^r^uhwi9x2nV`1-$7
z4;*l)UEg^kzWnNr7`?f%b32y$O>bQAE!yCBTE)ZHLK7~}-Cir(WA#EsVL$s}t;&L$
z*RpLgWs^)VG^@E;$II6^O(<r6bm0DV21C6=kK}G}?&Wz>m0%^?ptFxn&;9S$b(tTT
zUdL8g_BMWq{uu5XzUqPN>L!DlKe2zjc09HcIo@YsP%(4Hr9)>fMTR*Q&8ob9-tYa{
zk9Wh)N>v3b&+Tp$eqUrInEU@M^X9^zha@j8^ZgxS_x{St;F^CwpTD`WvH86H|2bm1
zQ3)n?yI)2>N<ZE?f7Y?QO%uM|yJKQ_JTtUPWA64IWpzFsejoG48zMK()}6sP`)b?G
zBsQ6dwPiZfj-SbWydiPx><u$|EpoyPqUGcQ%5H3sJUc@+?2vNK4BgDi56LoS+J+Gu
zD&{=Dyy)-Ggx@J#(LDUM@8qYYZ%wTFAa{jxe)8Yh!OtVsPE6SJ=i}!2jgOD_UtaDn
z@8skJ+Jt;|w)ye1zJFPiKJi=0g)sfx?R5T_q1MCu^ZMU7i1QyWTD;IaxcP3!9wG1c
z)Qu(;ce~jldFSkAw7j%CvA;O6$ecmXY1^?6UlZB4uC5BQUM$VWZ(bwIw{L#w^wm4h
z)I^?~yL9uSng0)nShh0X%@oc#rmZtg@Alj;Rd>&L&trRcWXFeZCJsyHI!4Rx2{rg<
z|A5cecIGC9HrYZ3wp&-soFiiXq|0r&_`7kN>oG>bJ{M7u`K>mUpA7Qv*?=w}y1p*<
z=eynSU)<ksU;64wrP}H4^9$Yn$Z_so*HKWmJ5RV>vitsF@d^ofj_PN>SDx#5>RBu#
zb@op6sf=wl-ztq8E;`-XGVQoc?m?y->th~-U(b!Q{P$yLeZ;+FwG@+Cy7k5L+%`n~
znfG(szN)WU%<OzAN4rEz-rt-1^Yim$v)Nx=)s+6vsEa!lY@9o@SL1-7?ET#m>~ro5
z9^Ub`@J8cl_T&u9e|+j|*tpoMKNK}s`xZ&x-11NFa@o<Q<rB@lq&WGv7eDgSc`j!;
zUHi#~xqr>)N)!spW~HC>WPk8vI&0Xj+#XhYW!Jo;8$ZkumMN<`=gHCaF7Hd-ai^ap
zY#+*>-}UKVvt#$f>qfgbR<E}`nzQ=SQt!=$kKOkD`;`q^+qX6A>Wz)b$3I8zcq~=C
zeBIN#<xP>s@2bmUdn-S(cG?`>A}L+@eXhbbox7d7ocXsqIygYvbY~i+f^JS*?l*Ul
z_jEn3E~QWFSEdKfn`gnke@^iwmHF03%`GZb7dKwjp4(C>S@0m~{>C4AvuCnD`mt$F
zr`FELyK{>~K0HuWma&jM{Z!j;k#j6h+_Q=^Ofh$Q9{+P)y*z*Cfrm-)A}z5635Wm2
zePdQIJ>tJn_<72Ko{So!nG$i=CkL_KnOXPsd*O%guVxp{-Ddb~jq%akO;KC30zr4L
z`Oo(|Jxy0LYKz9--`~~i|NS|ixA0hx$=iGCg`$V_`E!MCTm5x5o89|<|NnV{%5DMc
z<7}OroK8&9oV;(k%ai}h=DAj`3d`v~+BR{TukFT`V;f&KK7ZG`V}^wI1Eq7E4^j?U
zNh^onme{E`tyr$jes00<gYo9`J}lon*XCBdXzkT=pS1n<$nVJ7UB;{L<#Oy|Vr)&t
zmvaBcc&5C(J>TzD&#@})@|vpk@mBWwj~|cA7r(t_`u2in!{wJL&(6$bVPyraUo*?Q
zbK*#+uqb#FecYO8PUUUi+^ra51IxnV3Lj0s)%*Cj+>S0*=9NNDQA?fFcDzlv%d4KS
z<oRp4CC44!u8P|;{l14!!^C@~XGH3K3U)v4n9s+`VEmv!wZ)0q?SbDJX4~#KHmzqy
zLAt9S{(C1k=VtGO#UHg}cWx5*-E?TuVLq*=CZFPD^>2y#>}+qjne)v4|4;tqesh%+
z6dHn-dR^R}AFrgOwB@jR(CV<YkEVvlC7zn12|7RgXt#Lq0tZG>H<x)@yw0cXpG`jL
za=+-Kr(Suczg5+3iLKwiU)fO+E>rqyW<=fB#(Vq=l5F#f+$2vV6&h7;>%Fj>ZJUTu
zklkwb_^qECPFPf(?NiQOFf(T6-IEV9A0ONkX>nn0l;qt5cjxM#a=caZE}`UOTxPjU
zq^&{X0yk6sW3fdS6gFG`&wO~SQCncj&ED;6{PLK$^B*;MwPtd|9JwuL#OG_>U-^3O
z1{U_k(=CJleDj@aW$N6{_w=&AJ?M~d5#6XMda=7Mtc$g_v9T#(P+<byZ5O^iuBELF
zbO6Y!>+9Dq>NsJ%cxAEB@A^BP8L#H-U!9YxFD2xCo%bW-r3a}`ev8dpGq1Wm_Q29;
z{`W;<`plC5pI))_fAl{4faudlZin_8R~*b=xK5>f_JQTLrRVk+J~<&+_2q@5hX=>r
zs;`gMZohZwinjiVna1fauCJE|t(uPCU$-Ui?ys4CwJJ(XQf<?gzu4P$&!^`6A@-j<
zI(I)*`!81Pl}<Y#dtJUssp0Fo*9#YL{E2rIRM2_#JwZb3>7xQ6o~};Ug-)#zyGk@c
z>lZIA@tmR?J?-l1@YM$AFMfS}{pXv_=T`(R<x=&U^5DzM%X|L+`_0t_8lv$$y;7TR
zk&K?%+RmSU4~dnV?+Pzaf3vMy$aGJI`D({juSHrRD-L{ndmD83qHWce1z~HW-rU~q
z-&)zWXz^l6lMDe>?`cPlA8((m?*Hkic>IH>PnSOGI8nSf@r8?C@HCl}HUC6kehv>e
z43c==Yy3t=^vv21vw!^fp|Yy9*g^(0i?6@`&!h(r5<vH!o||L2_2RrqJ9b!rX1X>c
z9$pc**9x=}B6=I>IEOs~uOfY(r1{N1v(oi#=Z-Drk=r-F(N1GC|Cm&u?5|bT=-)80
z@qOfs8`lnfuu3Zuu21y+e=LqC=aFxJ_@wV!BMrCf6<0(tAFKHuyNJ6BbhTF5+gpXN
zu4s0NYP&r>H5IfXeBr`{RvG{K^`>`Mets6X$b~a(ecahqp{r-u)$Rfx%W`UR=*pnh
zo5sruQl3S-#9GhExh*M>ptAVz!ROpbH&V=vHY^l3R8PM7dA9KXGw(|3?#_L&y6o4v
z?uT6`d;c(=EWh5{yU*{$mS11qeUy~1`B1kgE<8U!AGDOTQ&|1Or%#JqI)y;PgfCuX
z$nN=np^aC1MZ`v>#qRxShgvxQd^)W!q8k+gUOW3Udu#Ufv`60;TiWb3ytW~w>U$FV
z4cU+BE>CnblwQ4vWy|Y-{5!QgQufsm>&=~a7<TLsd3wa}lTEeKJ3kA>VCfH;yUSl*
z+8I14@_ot6ONK>HJifiVd-~_+=b&;Gv|zVvcP#sdc#D!30$*QWU+&z_2deh&?k*2r
z>Lm(3&gAL*%v)PDmD_lqeCX%hDP3n6x&7-CN9n!h|LaUkQ)C1lK8W-2TgQB%ELx<!
zzNT3$|Jt(p{Yxre?|gG}vvKh=pRjc?C%Z(oHRASIaEs|&VEiK?t{=C=cea^H#)XDo
zUtbr$xnX#(=CdykTP%y35?A*p{jHgoXVuy67BE_Cw{691&BYhrxy7A+bL`#Tle^nb
z36@V!Homu{uGgsM^Vbb3_vbBGpa6=#A0Hn}8YZz!)e23zyv*0Cqj9!*{-wRu<)@}<
zyWiVWX<7TrWPZ&j&Uf$LY4w5T(*M8T+$a0ukEb-B|62Dub7!C2xA*;3^T$@X$!Bgp
zsZJMNZaXb`^X%Y_DSCB3>U=h3MPFSNdUJC+f9$T3hmZU1mn9x<ld-F@aFyF#{{CL-
zj}H$;w8MCoUp~1p`S^?b`|B6&Y4CJ$Ni^(yefKqYlxhBfhnc+_j=QMcF1xnwp=;*}
z3#&EV`=8IRH+oZ1Xem2q#`RfAJA)02e@#iTWXUv;sCd10JLs^Wt=ZR?#qF&EW#yP%
zC7P}>>-pN7HG`K0Eb$QRld(M1Cu=QXoW{e*$oR`hc%q8q_cp&pZTzx6At5H+=_U&U
zx?3e|_e>7`trh-lk!RS9jfWhU&T-o${^h|_Z!<PIA3NI~z3JV;+_xrM$I5Wm&71z_
z{jrDd-uboj$!1(#C3?7xcX7_mO`v<fjZ#mk+%|J}ary9e`~69V$!=o0Q7$WkmVSAv
zs^t0JeZSgMzgvlq=ZdYpx#2q#>+JIf&!+fo%FPToKHKV;#Vx%)WrgMJhgVhGeBxqX
zeEfc@e8BQPb6%mi*;D35UFu}Go>>?B-Med#^6{^)uS=TcNQmpj?D&6UsrPiyB@q7e
zY)(GDzT>3J>1nzvBR4PGw*j;!GLC2YWzWOM@;AQknfk|f`QtR7S#yu_w;kPIlUtkr
zYQmPok7mYKJ*ur%k2YU>`1<v5Ya;<MF}I0IuAuT$)q7fqql=3W=iJRRu9z>FE&ch~
z8I}LuY<{|b1g0Bj@?E~yS(^B%e1ARr^y53r*{5HRD|+mgbLno{^XvQv-)`^CiP`Vv
z_>oc8s$_v{w-{HKl2T_+mDuFY88al}*2Z;b_pN!d);@~WbBSK=&pvDSWgD}Pr^P<_
z&~UU{{PNmpb3Qp6(CvI&;0uDzJC&4}h-!tapIXkcGw%O-u1P0iZkkU05UYG+YppTp
zDq%;VJNs&9A2{HUc7C33E0<^pc<XfKwf`UY1?}TH=VN*zZtHTcNggq`?c1}Ze{D{F
z?N;>k)Y2v!&~Yjim6adwmfv3~p`_F~Vc9v)w4mjFXP3o?2mhMd74zkTYIZB1)+uhW
zsuaoT*S@8_UiOu%t3P6M8ZYP|A{7;vqXL4SOG+BOjh?)Je|}nTcBhTW%+uEDD;4xF
zoDF_$vHOkjwY71M2b)+!C$NKVjRYN6b(r7&g`S#{Qs<1eg_AcHKR*|^EXUO-<c;sL
zNBJui&fi>HeB0vR<=uZaB^joko)+pT_wnWO`AJ)&X4+J4>QYpi)R8G+C3$+HGW+bh
zSm&Eup$1hIlNC42>-sduCFa)gkCwU-7XQ^*j~>6VDV3XBT+e6Y_f9dvz=$a>%Ju*L
ztcl+|=~QU^;TP{*f7M(KoszEeYWrfDxnISMm-#Nqy1Gj1gmO|+QpV+FyvfIU6iZ)S
z(dy{%P+DFr7TmC4fkIv6z8ROfZwuFzb#r~<iOiq#@I-&ULERaSS674m-#z~F_V)Co
zqg{df>uN=vT~s(PSaxfkSik?D*IzeZ%^Nj*E93MQHrb>aZr3X;DAH}?e|7fv`|Y8Q
za>?iC*@Es;ac<{(`2PL+rj8C3##fO(j;lge9(p4gY+?0TchS0+F&mek6f2!OA@l3Y
z&z0(|N7X@>CzZawR`dC+IVg>Bbx(@8x-NEh=i~iPKYTeBaoBIAg1<7$UC!qpuKX(b
z@xj)r{Fe`FWz+wE-}k4SpEq~={kmvS{5hHM%ysgeXVZG_?(d)_U%B>NN(*)Tr@C9W
zYl7z2itoKyHGi)Lh(6(Ta&iJ)oXXB8qY%8z2NW(si&ifTUU6-0w4kW0@BaVSmS^0K
zH~5_v5cp5g{Z4G(*^0?*>shrm{zcA};fvay2Ra!I)YzS*!0Ws`LDss)>$(4}f{HBL
zNAK>0IR0Z?ZaeL<z3K9xm3MmNmsd;up9$Kub!BDn<z>Fo;PAX;u;s$m-_Lz!f97tD
z);;><<?^OKp#dc~EbYGAHf@?6X;RPUwm8tWvT3rqf6%%Z%d+?Pe80W93EENL>gu`b
zR{fnu*VX00D~>w#$lcRAA-p3-&+kmW`@1U>L5U4i_@A18WmV8pFO$qmE&ld@Ls$g`
zJr|kq=rZ*D%@bMA_Wb#cH)$4c%|xGY`su!OiMn1nO+R+Qf*;|(uiV&ZG1t%T=M!PD
zLxayRy7RomD)Yt<-EVU@PTK#n<WA47e=i-=!@jj-zwYSXSoGwCU`<U;$Q1P*1rM22
zy{Dy|o~HZc^mKoajfbvm*4Y1^Cok{JY~#s)^d&8JE|Aq*9_;E_P<J<>XW{G{dy}<6
z8$xWWzl8+;JM#Pee*fuuv7ocKCNX~1IVFGO$dM0OU!!7*Q&jW+Uew$tH+@^p>AA}G
zo3f9mF)=aiD19vkx(R5${rm?H65bm0&oD~mN<P+;`25`5T|eAGrROB=)mOcKdR^`K
zr0KZs`v2KUuL~-#i{!nPI=Sxc?30sJPyaps{OoM^`}=Bd?yt9(lasrZC=|21tarfz
z1<)3ViAt^^0)i7$91dUj6d~5#8h?G=k-x{n&aIc8V$^*8pJhv)?73;ZujRP97(lm%
z<lox^x_Q(3H1n<P`To1h-h#a%_*LiBd@gS8#b2~VKc2bXvqsc<*^IJwxyWO!Q)6%J
z%Cu$W5~+BTy)JGq=$>>?d1PQ<aP+-4=n~Nb4UC``>fW`Weeshf#q6z`8g%ffXY3>4
zO)T4ezrEdCSR}h#d|UG2U7;e{VQ>87trHKifEt|Ja&H$Dd+(I}YMyszLBYdAI&phu
zTw3b=dV!#zr<)0nZoOvkGM}8gTjp($Ro=1d{qmH%*P@?qE6V!E{%2Fo$9Oi^j{UE$
z-@ZLNaIxFc7qv^A+xblL@5ML<x~ND7onQ1P_jtej@|;$!l(m*~cpAS>iv6vm_xACN
zGv<*N`DUT3!~WiWZ(aP1=itGEZ*Fbvma!~avg0pksIuhUotdD*;px+*%&wpa3fjcX
z&KI!rRrV6yOEQx!LpNnkzbttAtF7?Nua8|`U0eIR|Gs_AkAiD!A`id1x;psEKLJ5O
z$Akn0<MeY&l8$y+Ie<3qyC(eq_xH@Kzrud$pKg_Ag{(gXx<YdOz1Ia6pj$sc4I<s>
zZ9ay-oo;W-1>N8E=Eg=V1F+5~=VfdvCP<lP6}-5hXq0kd!kgQ>%ic!a+Lr6>;lZ(Q
z-@aRn5OwRnytz3U)G_#c-d_E6S=8DvE^)mxI|?8FE6ra6T8!1T#KLTL?;_W3P}fV!
zBx6F(?VAh1S2)kKDqUr^3p88Q@kcm%Th7C^+wVOB-7)GlRqN)~{MXmkg3eu(v8$P3
zl6fhkxf?X;al-!V$K&$C($c3}IE5#L?oLWd;^5*+`tabONzRP`=_pXxx%}&0u|k81
znR%vFsaEK!5KptWuP!cTPd`5|_1&GFw@!f4jMAt75gU_OWv$D6%qEAvzq@<-t*zO`
zFD@uDF)?jvbae^hxckokiR8-A)nZG%r!VuLZ|AvBx3I8qOWxgC>-YWg0>yZ1;Q2)-
zj7uIL<7HxIj@*_bsp>t=XVY5=g9HZa@^>jWH>KWM0kO;E>>SI<4-dC*PCqZ_qZT|V
z`oFS!A84Bm=$7AUI+0r(U0s4Cc%8d#lF!aIS1*2chDX{==j7VmhaWB|eSM8b#v%Z0
zVbJ+SCyEPRUQ+d%qH*xq+1V$j>&H)ue7}DGzgeQ%VH?WcM)9yUn=Jtcsf{*hdH?eH
zb)W{Zr&0yzzOC@M%GSBo<&%ul{c;@yCyIC-zMypeq`Q1A%Yhw*kA?p3+7+V*I=Axt
zJX=sX9%S)Vr_07&-EYnUr&g{vcXv;p9$#m9aozJBg^y?0)mlYw%Q<+~cfTfhEzOgi
zpq}FGZMn%mKRxX%`l}bWr{e#gQ-6Mb&Tj7PSW@uHNTBlEuF}`XrfP@JFwG8I6tlnV
z?X8O|gVkAG1qB(eY}WYX1hT@a^p${+kkhX(FNM!mfsV2bT<o?n`}#T+OZ6q-^rRvk
zySwb^tE;Qe%&{~MTys8u|6jL@i(Ek$Njo|^-f{r9sy->*-CI5VP%F1_)t8K{tE;{q
zy1qPcv73la#Dqp>_N>Pp9Vcp~(uFD=_w3m-!yu7KH+tKX4-XIj`1NbwhSHxupU=0f
z`(v>={rt5yLBWabFPOw9im&jRs&!>WV6&WU)t&mB+uL$wtV&+Y)GPn@=ckLiJ9EM3
zXTDZtZ%#ZpIoWXb*;^V)N}mLHoxArWY|p>1W?(Sk<YaZ@>TfyA{N{SC3|e|)YxZ^H
z$a_b-#ZOMtmHzkd-<dg<ljZCG2-erv@B8^oI+sUK@Fz1^|H|Oy&p`JVJ2taf%=KgF
zle6tQ{j^GLJL{bdiOhC(b}g-~7Z*4(8>OE+)5a^kMbX9O$wm_%-A_!dt*wFU<80mf
zWKQ1QUB2;V&iDD&MNdu`M?O0>Rr@`^<Nck*?cL(~LJ|@u7P)qBaRi^-qhk2-_I7>m
z>3UDk&9&yYeOF?2b4%uB<H%>fzP^_Kbvtcy;p=O<Qf4_Petv$wH4vg@ql94+%h_3`
zhi~8hz3}+^dwWlJiE7XL^=VzqPN$TVCx_#i^74NC__3kjq0^@H^Xqz*z`3Hc^=3}m
zvokY$Bn+AK_1Cx0yS=OQb#VN>HePADUB=zhr;A$_KYQ}vV6#ca1qW&H5jU<vi}~$-
zH0VZe6A%z+Xm4j{*tl`y9LwToEA^fq{4Bb6@7~NC8x$246`!1$87%DxE}?`rKRVhC
z>SZ=HH-G&0?byqbO_iUYE#K8s{am}Mx*C)|($3B4T)upHm%fsc|B^~E)yVFyu1Bw~
zuJ+8D8>=U-?B191=f}q<Cnh#e)eg6tdY+e;cWKhmt|k8S=XDCJPqMH7rxGgOebfn*
zk%Pgdl#OlZ>FLRz`(*dbu`0d9z01(>h~?ZRwND;RuRr~+Kq#$Ot+}5+FV8bS;NhFI
z1$r?%9Ima2+?w3oVdFk)O7x_yGpx&J#H<lmdU;8h;iMf_zx(gm-w$$enI{jrjS>=i
zN=l%i9neAJT-_iNbd)z<nUnM0Jy6mNcL-(Lb>;Y74vQR%w-3_Hj@v)q;rSyxzR3K|
z&HKEUAD_)zqCZh^`rW>RQwlt!dL;@H`u8+zKK4Gc?$@jhuI~T;8cCHY)fQZ1pO$#M
z?xl*Lcio{f!&mwBeG3a}|J<+Q_20ib=4Xil-&<EDrIYij#8jVFX<E<Im+Y8hKQsFH
zoQkxQO|So~kNKY|{Nwq7IS+rz$ZfRmKPokut#}&4n(n#>zm|M#`DZX$;*a4y3+8+G
z{(o<j+MU64ZH7rj+Q}1hUF`+771=LST0g1q!1W({{(QH;Tl#~cP|g0%p0C!ypt4_O
z=legIcQ@)uyI=Xnd?WA3{bGxEpA_FzKP#+#>FLAz>F9pF{5l`~-*#W(n(P1EKep$l
zq0NtqYc)HbpZxHlN#|xE|C#^o&1V@N%$GHPpCMyr{op|1PnN#P$2ZQu<9TG0tIL}^
z?SC)6vu@^JcF5K6=kI@ikL`Li{h^ZB@2A2?a%UXfTG?*!pj*}A|NHfy&-ZEOe`^>2
zo-jK`yra8=dH3IqZ|ifSb?rWuo|(mOep>nV1H-MNf)kIp-esPYWNBV={w#a5bb>;i
z#I1cF_TTxS>3<;n@ka%*$@Nul+Qo~WiYrUnuj;*dyW?*k^W^P<*Vi`3{L0~Y_AUN#
zyumc<hl~}o%ZnBs=dalIXA+O-W`Sy5w(~lf8`eqrwk+h5%n~^!S;k!c|I3bZy3gI-
z9Dbd+>hkQKgU@$d`7O5Zz~6%(4&_f?8k`$*eO=<2NvwMBADnuycaO=l<U2dhp3P5x
zc=egG&OL#>ISh%ryFPuI{&hqCzj@WoXV-oB;s054rmdnveCPk&mklmG`En@Lum5CW
zTlW8#ts9TZ-)7k;VZ%K!^%KvX9=Di@?mIrcbl^F=UnTkK^p2e(&Eetld9~JZ!H2K?
zNd0&?bF(VXN9&HLAExHlqBvfBf9Je)uT9OmGB3f2M{ev~B-CLu?>qbZ2f0@6FJ2hl
zu@i2tPOz=XJFi%Czcj|B@b!M(9Y2aW+DsZN($DtA7pT2ypDuJQU&3%1gOBU22^+oS
z@_6%3Z=EM!`1Shgh`8SppBI)Mu{@Asb$)7o{iA2s_BJpx$1QnjJ8}AP-=|O2SPhRH
zvD@R+R+I7~g#Y?I&X|`sr*_vqbDWkCm-X^&uWmS_^)}ntoOAbx79>~~$mqo$DtvBZ
zu9mlFf6ta?x6hxR++1?{x!;#xk}diD4N7l4<fk-ttW4&Nt>kGg*SoYg_H}<Q(}9le
z+jn04(|f-B&EW^r<<GB?^a!=%pEKw9*L|`(UVk{G?Jlxi?Co*ZBRLXhcTQ{G=qB}X
z<!?5-{*`OrpZ;Yf1lkhisl~$>@oCASRO83L-D-UT6<zY;Pv7<Sd$i8A*>=kA>2oZ~
z<{s!|juGEDuj9z#H_KCGFSXouW_Mh-W=F?SQQ`j+w~5{^NIIfDo!#%jyEfM&?;?G!
z%u42De53c!_IAO_TJLlIH(MWHeE57#V}+FBw?9vR?AOmzJI1!`%F2zp_c|tjS&}=6
zJDIKiKu3Nr*9@useYa=EckY(|w5a1&t(dv8K-mQ8%hQ-2#962_%oBb6aQbb>D~)$u
zWG%n7o2l~HHUG6TOw@~UKd-jMzGp+fl~C0-fthZ8hi-G6vnYt;IP9u(&Gh%d(vA;b
z)elY)-QU!|er-pGik(!OSf!5sot_6gx6hpwTrJF$Vt1xd|7bj8&Y#EDi`;mRU0q}P
z^0c_p<M73t(Pi8W;b-<bKH2g6@JTi~iKse*#Z|}bAKx-)(w1g$-gbZf9_jyb!M_+D
z@D#BBmOb!<aba=ijyo+TqV_7x^CJ9P9>y-R)Q|u3ayx7-z)e{fo`~NU6y>*Ps9fLE
zK4E8#!A$%8$DST6E-W^0d@W?|#`82T<@1U-#^1`hyZ!~XtG8s!2Yr?HnYHEljw=s~
zFZIfQ{jhTm=i8#Fw+*}n^*On3+v_i{Q&*aFVnuo&OTm$2-)HW8ly}_z!LEj{MLf$6
z>3T%(mS}7E#M1M&?dWxed-EP_INomC%+7Y8Qk!40ZHsc6ll-~=<;-lC9*b2memJnb
zuHaAlvy+|D<s08wcU<N>_GN!O$K$Sen{yxBm7m=G@$|^uwhd>G++i&4+{DSA=Vx=d
z;cd#FDOMA+za@S+-E{n{ZVO*z<KY+GnvV<uIc>|nPh%9YUTD2t?{>lY`xAKH?{&I!
zRa$~)=ZP;rt#?Q=eRydztKq?)ANH5jf}@z%wP@$XFqA(QVk)%!v#C1SdIw*7eWA^n
zhVoU_8-<fs{dn;r*iOGiPT|ud;Vf+>rOv*M@%!ukhOl|AQ|p)a*EAQOxz}d?4=Lx9
zVkueQxZKLkeJs5ayh>;9R-E2x`~J}G96=q|(-U{OWy)Bq6+VthG};#{wYXjWScur3
z*P%z>JnnSY6Yeu{^$1qwDE@axBKA0^{D0kY8?K4VqGWZJHhIPxvS0kX?@;7M=9Nav
z56xJrkk@7~M>Xm}ycKs|-n$vz&yQ~_74G}N)0Y37amlF{-J0%!GS9CUWoMgA3y)X!
zotmJtb@jutJkjgN)^^o@Y>lqCTHE@<)45}39ar+tT^43O&(5ukuGsUFmGR*BX@MNK
zuk}WR#~+P(EV}3Ow1c<bY6O;FN<0}m;|g!w@!K^$+P2dlg>L`ooV%-3hx=bC_v5rr
zhhOkMi`x3Z>vgaDy^ME9QjaY^puI$EK6mqM$>MoNJMQjka8Hoi)3o|==t9PQFV_{;
z9uRmZQ~Gb=wnT5f|1Y=A`5G;ezcs6M|Ni}DIf4_PpLkHTGe)afCH!fZ_~N}69bcNP
z{Qp;f)ta9V--*g3*4tKYJ!o%z%p}3S@?Rr6v%yo(?$1vO=Ut1e%&YvIyrY!qxboyS
z=|8oFcccBg`6CwpzbSqw`P%=Dxm9mw*A`UYZ2J7D)PN<yN51Xwg^rxVuLbQ^SeJh|
z^6^CapUJnb-K?MDDs3LPA;IzO4JD=X6VFA8{N(QIKK}n(c5LGFjqSF-5;VR|Ug%#s
zcaD0$ed164mUZ<KBJ;QW>(9NMd|#&bbJCZ;X4Z!qg8uDDJo13+XyK`!^;g4OT;g~@
zO`-E`{*_F##1zeRWbT(adw%tDyXxYi!WndaQHPE6#b}Pd=f$o%x~Oypp9dd!P11?k
z9z_d1cdAAjS66=gx9;wqIclMsau?rydE{NeqnGpc@IMbW&r8}Eu_d^>!zP_=`st@Z
zk}5m9dU~E*TIzjrSFf~LPQtr8J9}iU%_6s7PTDxZwz}-?6(yzfGpoc@BR_xsyf|Q?
z)5%@E&(F<0eEqul&F#T3KjCe}5!u#Tv_S*ZCL~fdiZac9PgMcWs-)!Pn_IKR^`>_#
zyY~sm$(?(4b~Ynuy}-`m=V23FT%N2n*b$>Q{glF_Eq}gT_TTgO+wCP@Q@g6$Hylh*
zP*Q5D`S}TSWWK1Ui_1K&uR2{Sj;7hyj^vmL3krU`@Hj3muI9r*cC85=9X6&`>47So
zUURKXpPrif`|xAG<&(GHuZw0C0xcq*IAvYz?y?`he%XDA0Nq|c!*FqzGPw7k8G9FU
zd;S0IN0S!D?k>~n0rw6){`~9&Ex-S}|81Fdue3Sn*b2}|+mH7C|M&ZQ`=ZlRwcTG`
zSs5w<IxkdYV#U?f;m4mAC7R7H72VR&VIw@t%KeGt&a$^s=6QE2o}Jc=+hZ}?ELSQ0
z+?<w{7Mt7pJd->B{QR73AORUA@I9sbX$t6`@$LEXY3Jtrj9KT?*V(D4rNvcW|37`-
zninrJn3$L(EQ_8bFMlT}IPr<QZMTwt=iR*YGcyc9bAWBU(nTL0I9^)nJ$=os7ezZ~
zSeNVN@BeFd@Zdqvy0M&_n=&7Fcc>&=r3XII1Pxg<Ha6-+Z|j+=9d49#q$5RKMDgFW
zCg0g+ssI1|eR6j8^hZa#KOYs37m$@*JJ-QQW#%P=Ehqjzd~tDc$-6r<i=Uq}toV?S
zb#2YeMYrm|z6$;I?d|C{UTM&n#oDN?mwxtu1Gnh&Gv8ldULHPg|KDeO-rXM$n)wZ8
z2EB`v1x=(^e}8u)$IPuy#<P`6bj#$94i#akHnE@Vj~+b&^#fVCL=L<x*_40Z&hFQX
z#UipIjG)1aTU)bV-q@)8e$QvVUTO1Ze|~<xl?d*h1@)z#o+c<Eabi{I>WHmbqVe^A
zO*1bq3r&=}u%j^f+xz?e3!PduA~&hnR((;>((-Br$D8NAJ{e1=UMbTTx3+2@ZsScp
z*u)ANU2Uyo^7r@ORsR0mxw+Omi=J}*{PZ;W(UDGzxqhPVE-IXtEW4Hb3mzZi1x>~t
zYUO@;aWQ-Dtu3A(51;*i?BCztFR!hYE`N6?@yUsaA*(_>T?GX_oj@ZK9Xobd?ECdf
zn@dEa;bcnE|9^kCWMB7l{1^{97&!adnuXre^|s{PoCJ!^Z*L++-N1d3myvra3|YCw
zKv!{q?rTXr+&0mwG)wfEX2z8jg1^7LRo2kxxU|&!^49EdC1qvE3HK9^=&Pu-tO{KX
znri=W*8Dzb5!9zACl?>GNln{4b6xE2P1)D=oSdCc&NkP7e7yfUD9$=(oQs?yqN}6R
zGtahqQ^m(5(6tXy+w*!aW`rcpo6OA42Ri&rNm*IavPdO#RY+i<i;IxtR~@e|5ozhu
z8xjv|1T0{fH*cPdMM1;&`}OgTA1%4LxRS1}3jMCHuquAP-E#l=pqwNk<{CGrm0LV$
zfdeDxMvz}$UrQRLaI6ek8k88f;MdpJo6FzFc}>-FEhsRE-BqHfq~rvNu*y0<84HJ}
zr>26ACIiiMzq_;Z$+@}Ft)41p(v0U?m#3YdXM1N?X?M-fPoOhEWcb><l$Aa)y`1Cf
z`E6bN{&V8-H3vUEJ^k|LW_8`@ZC474g$iC=P~27i-mjv<LfNfnLDbePt*(v}+zaNp
zdTz@xlWu0`Uly>?>Bp~MMGp=z-rAD6c)<)85zU|$(Df3l!}aa{d~kksZSCom!OKG=
z1SdW}b7Ot{{yME@74y%}&MtOrW;=51m{IMo5-Ib%o|`#!t4{CORL=Rh?qBO(>;8n3
z(Jef2{Qtl0+Ei>9{477=`LVk@p6d9d-9PU0>g|rB&noRdmCjF1Ss%<<&wrRX^SjO6
z?ogFQw|1xO+c`VZc-g1Bk7kFg4rAzFF1`AuVY!a6`TOS|uC0wR+~f04r>>#7`Sfzb
z^rvcNu`LUnA`L8+EO!)1757JKOt$~aJU=lt@_(Mp>9Bc!GUvU!_4M@gezV+LA8zOG
zzql<odR^RJuYv-DdGqGoy6EaMkNK;P*CtS2)Cya3qJfcF!n#by{_huI%i?D>YNt=d
zERypray3jm#If&B<$vY#ZH9Yu<<)$C^o3lV6<~E)xT)8W_omUYjsLE++S{cY1@C$5
z{>0~Pr~fvllycd`^Dl2KmlER7wQS-wKDagh`MVh#d+z2ebeehQdsyG1r_3`AUW#1J
zo_;WA_nAYS!IC%sMmsHf%U>UI#8d5Tze0Y))0tOp9X{NA-tKqL-s<l{qN1+*YJUEB
zxBGpG-D1!X=85~CLB3iQs-1Oxo$uvkzBhN3YKv-z)jT_Wc8a<<!|`K2$?;F#FaPf*
z6LXMfI`bW^V72;P?aH%EC!O$k{dYezlM$1ckJ<*;sGailJ(8KXK00@?%`ljCd6H%F
zGxurl6pW07ZZ}tID%x1C30u?uOk?)fg-_#R{_CE#&N<ALaMJnrM)?Oec9YINU-W+e
zvtRdm_gwnx%oV|taL(y4zx}ejOn$k&em_s$6P>gE?vy=ZhmF^6-@o7fe%)_f@9BD}
z2b)+ysqO0O@Yf6ipq^UgrpL$oC#(C*U0UXQx{;Y3G{v5mcRv04wwpR7?=Ck7o$LLx
z<CVvu#bMKW?wvkxaX#}k_Bh$2zXIp}$T*O4?C*Emu04h`k|*lDVK~QYq7yhnDD`x}
zmJ{kZJ4DVNbhW==u)*Lm|J{B6cYb$?zvcFy&sNfWk-2Ku)}*sLGrpaXa7ehg=*x@y
z&5Ke`1uVOv{)VgfJ)_OWKkQat{!Vk0JuQ6Yqq|=0ty3TR>>qW>x?WE#-tnw~XZ@9h
z8xQvWyS!b6<(kH{+U;|vtrxFaz-xGfk>B4WCHZ*|>z2HTNzY5oG7=ILPEXTa3|i)t
zdwW{k-l~@l5MS>BPXPN!vhheXtY82BbL<XnvHJ(Kw&Z0e-d!-?@;$@#qhEE-U7Mk`
z^RHah%c)-%zqHWpxae%R`1b@$9{oKNwp;6~O#2~sQa>&C;fI5!C0aXs&;Qa_KK^>^
zij0D0`#G;NQXSv+d;EUJUi0at`j*1Ci(fCeDzff&Ly*LY_~pXw{Q1w6uc!6>6YG8b
z%nWVE#eBVIi!M*UYAE*N=#4zzlNl^NH!FXit~&D1W}fUry_j&b54pFVP4W3HzPDA-
zedfO999O%RRlmEvuk7tD(20T0?R-#Q{+uj(^R4UKQ-w!v{?)j@LSFB8&9prG)6;Hx
zSF8@L)cMAKUVlqr#aAV+5>3ZHk+p73tF|53Sj|5D5ySRVBB!Tw^+>L~yvnri?(ZFb
zQbx}zed1%!xYm_g?I``HAiZt=k_*=lOJ3ZklD64;$J8Z02@DS$Z`rsRI6m3=L+<?L
z-P<=jpVGO|dH=#RKKVyUi-oVfDgU`@zpLi;%(}SU?Q^}SU0PP_`tQ}I1+S~a3;tbQ
z_QX^==*Bj2K~G5^t;Q4k(~ro8&tZ1H#`UG{>SRXwYllVdbg$LlooLUP;AwHnr(%D~
zlPBxdH=6MuVOnx(j^K_CYlg55%xNB?69n%(^7y)BO4{~jMe|ZAvjso)7PA~)FPK++
zw_eKl$lcS^%B0zXQ(s+QS8M;Hb>42T7k|xc4yaG)x36oR7QZcN?n=wmx0nC#=DKmi
zDw*-@t*ws2LY*Bd{=Yo+KGk>pcxw~ZF1sOM^U>uDHP`k@i5+<Ij`19C%sxr^lSN`H
zx@)gjol}e9D1LUma>xF__BCxg+Us`Bjrec+jQ_MS!;yJqj}p83E8fZ3ebnE)WI0EA
z=G8=@>1)dt>E|tR-v0E4?nn6qFOKE!9M;GNv8EgrwVt!kUj9H^ukh)8Nl&&F@Nmk`
zH;YzTIB|dcU73*emJFfQ*Hj<NN4=4h&$AT@GqbAW*p-&R_LfIU>Ex6uG1q@fUsq4R
zu==z~+3vaOho{`Tq9574E%g0|sLn4poWFm(dLomd#=b;E|LfnDL+k&x7f!Ffu*Rv}
zYU}sJpBt}7+O1KtGdy0Q{ypH{JF6-D#_`Gh4~xUfwT~WbJuLZbOK{pLouVtJ53N#t
zT{wp${p3C^li7w1zGth0HlOj}ITm&1md%uG?tMwyi_IUWuY0~&>EaqI$>aM1Kd<<G
z_;%0hnEBuS-cQV{|4{K++w42%k-7~=eG8dahCGugHvg*d;NEnnX9;Y3q<Zp~{^2*j
zA$aW9PwpS3<ubc##Wrfx#&+!rD_B?J%oW9ROVPz;lIGoa{*(4VTAv(Zx=~Y9u3b;u
zoV2xZ{{6$nLbujTykYP--SGci{zF+?6F+@E{6~L!_U5RANeBOIzkavzVh)SU4Vi>(
z#|1jo>z;|5bL`EzX#~2+Y?7*Xi9tt)4fCr=k;-#DY20=Fp6O}b^3UqOIY#_B^Z$C$
zS+UdhG2g2fUg)^VA))&tb8=UQO14!xXbJ*+KZ^^f$>rkW;xbQZ?q-coAvbRrmTOrb
zj-9bFv72FYeOT9YeczXRX8!-U{I1QeM$Y^C3fJ42GbX(jyX$>z@s&ThCg)y%-hM5$
zc7K<Srb+Ka-%s{?)!gKlxgYl}`SWn`59|FM9UhyO&2yc&{FKj!*}WekZRc@LY~1qe
zNvs!J%(BwO*DLBzy{rF|`LgKgu|Kcf*6cWtvaI5E|B0EmKKnC2TBz$T^zKxWlJiQl
zDW|@a<TqBNUQY`9`Ao&ZF%azVf4uoMLAwKOirN0?WNJn}XJ*^;>g2ko$2(YL_}CK;
zXO*h#x#yUYqGEPjd859O(Cm}HzeoR*G`_Uuak~g_MS_Du;;%W?`FnT#i=I_}qs{am
zugI#ujAu{V-dOxn!N_LL33st)9zSkNWj;NaE_2P}ka8}6>&3aoVi$Hrdfwd#o^p2i
zr~mz3QPbQT8RF}1c>BD&*tW>E`;x&1m4m7F&tL3Z`{n(G$%V>Q!HXiaug&RZ%gOn-
zPx$OV>zOB+cN>04(UFs|*ztm0icKu-*TUN$4;Gvbk=koHqqVp#2vkRL-#mk;SuCVB
zwbk`DRn(tiWQcjF$dma+?eVj7OVdv#ZIRvIcVf0;&zlP~7O8e!T6kD`ie_|MP`utz
zu9%qrvcBx+QXikU%`mE}t_fCp^!`TB+6=4Qc)^JxPS16^Y`E>%zP*V(|0?A8&F{hW
zX>ZP*+wk)(=Z!nfp2wnUc0MzXnSSU-;CtIkebElDoA_*=nJ#<1;Od&3>OTK#V#&An
zh(+vZ&5lb=Dir)trK|N#PPqPx+nZ(Ij=M+h%9ps=yPZQiaOUJ=jU~S}S<OzHnv}DB
z*^G^Wo2%;f9b|DaT>iG&Wo!JQ1+O9>%=N1I<rVDqG{)<-+SHj_4AtMV^QYc;x^zm8
z&CJcQajE~)g-@FQ5|;|g^ZK%3s|!>4_jM0y-Y$D$`npe7NlCM{O3Za$Ylb-6`z?FE
z1o>rsjX1Q8nS13_bKQM*Y&}c6CL1dVyK>FE`KGAg^&dyG9DC1&>)VYN)e7?I{_4EM
z>n&^_yNx|lxNTwYGn4P_OO*X&59sC>{^@1Nd?8XaVH#_&u(&kugo1O6W}4Wo6uS1@
zXtm$5K%MAUEe5kEYc0LQ$=RzE>tJ=~6U(HlNe7Ih<Gdt;nD0$_5jp$l{E)?OS8NS=
zeM7)LCui=)hBV$L6Bb>iNlh!#1D{B01}bzYx&EsV-K4GbP`=Dsq9}NIB6FAeo#>mQ
zb<-rT%eXx0u6z>9e|)pVZr<lFEpjFZXvwy;NB#eh_4}{#-G=^y2ew9EY~W8&Dh(4p
za%&^=xy|)^UZ*cS7H?o*tXNfZqg^VvZ*$V$Y@hqVE&uCgJ4%PUsIY?P;D7Kw<8yI2
z{^y}&Isg3o^-F&)S&UNPCvX4yWL;=ivU0kh&^sojon@yEo(v9W=&*fz<Zy~wsDoKd
zVe-$cXJv<S!}!j<JUqj8`-bDk^pB@?tC(Eso8I1NoEZ3EW^?%J3WLgvjV*D<J^Ie~
zFN~U~BOu%xtyg?V-`#pUqyKr0W#4~w_B@@vvEX$@%(t0yZtm{;yxvT8y{q#DB_++*
z^`?TAT{3TGTF-j9JIQkEZu`ZTzdU{1k{Dh6!me)4KEC5G3zq5W``o_OnqijAFniYb
zC22D!O7v`<v9(g8wfE|ej8Nr!>mp|Csk~$HvM4C+_<Eg9ZiZ(vm!FvuDf0T4oH4u1
zlFIkrBaI}Q>}5~SSa!3+GD7nBg~kxA;y!t`DJfU=tv9~idG5UF{^chR+>FWisdo1F
z{KbFn+PZw3>DK?|`j2I=BV^Q@ex#-|_E;|~IJI;3&CmQ3&s+8RIonp~{xv+c=cj7;
zao#17)5}dkk$y-bMsK=z>VzdZPftw+okH!}Ev9<0M$-Oo@5%H0pPBh@uL=9GCE*z3
zF?AP{*}U3nI%$s&zuz1o_iIi2Jt4`?yKmZ${wWVT+jW(#u1+lHn#YGPhv%HQ^k`G+
z=^2K}Y~Is!E*?}+`osfDTdu~lHvaf)xO1lNv}*l|Pt|ABbo^`X3*Y<Xq4D=~=;ng_
zt7pXQbF~F0ig-LnzFqH6F=(dXT%KI)=^j&egV|?q<d_}sImYBTML)dDXWIUHrv=J?
zL_R-1mv*()!Hqw!#dWEu;KXzX+is=vk-M5gHJ%r@yg#;PQOe|Ldv33gF|YsCas5=?
zIm6fY_uo4+!%SE*xo^Ysk~Ljg;8By4?Sk?0@&+ktxmxy@_QlNjUj6m>(};uTj0N}a
zKa-|vbm`ZL>fDxg_eTr|oE(l_kqD|}Na)?7W;5?VqN<IJQDWTD>V8WhJ1e2tk0<)~
z^{8BWb>W#Hcogi&p(m2|2fudSwPrXm$FA<uJK@vy3!LxgOJ3P%<0xDIWEOLbdHLnV
zXA{hF<Gn5_%@;^2ZB(@VCR<yvJy7}76kX?&=fws1?%Xl%cFV6j+2wjac+m;<H(b2Q
zSHH&#L~POMjoQA<W#*<S99btGEjB+`*lX!*pMGLT;hu1{4#sZ2?86-$C*J=LU&*4R
z-*Aw7&g!3e4+3oX&!zo*`RU8tEiJER`s~Qdd)#||`{&2ECZ_P6KKp6e-x(Vhv-vI-
z-)t2v+ID4~*TfwWdDaIGmx?#P<ZV0m&b{yTeu0qR?CZX)I{H(7|G~d&f9|=gBE2m2
zsO5***1A&H6wjA?ul)YeAj;3oXn8g7uP>2)`=+T&W@(?+kJTzPyBvAz+dJRpSKJzF
zthaV_aDbW(hB4)a64@7*_!T<&TN&)XJi#rsh)JOG9S6hm^8Cc}n!?WQb<SsH>{7Va
z_`SK)CsLVgaJ|cH(T@3y8y-zt&M@-~n-8O<MydzzjFr|^+VM}Vf7*Y0XKc{5D);e^
z`m(akv$+_BmVB4XdU2^cL^{+#fYnvRRYY_NM-!{7qu}lb%?(Y9rYJmRoT4Pl=(y{W
zqm%z8&u!;^D=zGMTlq3(zkbWvcT;|!$y;Zep2oMh{ae}(K7GM5=J07bDO*FAMa7gU
z%o2;8r?w$k@&5ULeM*r*H$NN>>Pyc)8~vVVK||A`j*gU|^Na3$;5nRde_!pAWy^%b
z#L6B&an+tWM^wuAQ?vTY-x>LAqQ54E-#Rw&-S>nWuM5k&#nSWV&E4@^pz+VuNRw=t
z$?djY{nqLqKdqTMWzUmI%xnH`k>`KE>g=ZQc=iAHW~mguJ0!VZzM#q0gyryw=BfKS
z_pq)mcJF4IdEVpMh0c;in|9`PSmoW_C3-kuhp6Drpevg-%=g^dnjO3{NL4Fj#epX$
zC+`$Y+fdRnhui#Sw(S}9{_82vF8r3*|B53ri94b4e!`J(=?@PQs}Fg&>DxP=YOBoI
z68>;yq5a1!W^wyJH7xEPM|S4de(QN(SN5B6pVZz7?NQfM?iA0E&HgiCTIEdd0$tCU
z_IdyRX**e&E=_FK`cwWO_)q!qLZ*3#eJYC-_AiP5|8`&UalWXnSx1j_3ctL)ef>QS
zaORvYCMS39$;rt-f4yG+^T*@<mEr5<R)?+KmAC!-iDNT+e$_Ct)IYduzP!MK;cE51
zGoN|q?76vC@%<l}!@vFSpS`m1yRf{tfBXDL5*IZ2mY+U-S$BT=t1ln_d{6M+6;fdQ
z^>ZuBw)4sFcD>d4n`D#Sls&_sQr`ES{?T?@sf)jEez%O^akINH;lG@Z`XPfCKMU;b
zBreX@J1uRvxIEWu%d69mL%&^G9vXY<xcRe?-RYTge*bzKYE$`1rS|u?rG<}=ndIG>
zp&Pv|V0T&WUO^X^bpnTX3GPaC`KNe#(+rbLp})VsFOS-q1-jGs$?56u+wM$nINp1k
zIWX|~{k+@GL6h1)tkC%P>)TyMn>_n-1?mqDzAY5W%YA%sm2ySZ&EFzw4i#5gw|^|#
zJ^6r~-;T?B^j<&w{u>lVpdA%!qqnEs*Z>+3b!Ok&aYA(2JlCB&_x+V%1Ko^zdwagR
zo?hQ9(`=)X7Xf$jzVmL^UuOJg(E^SKDzEaJl^SMBiWtWUzhim*h4Wg(@(&OHI0vwB
z@b~Y^Yku_S=jWACTeXUxpL_cI{r=~FzuylJ5E0xNRUdtIvxfQRW_JEhe?Fi8{P+8P
z<BAUn@Am)y7yUV@<3QrFo<B2eEgCgmmDf#JwR8P{?uc!<x83&F{k^ffTpx6%GH7_v
zQCDfv292VhYvcCLy0^Fb@z*MlUre*Fg#@lkR`;8e@Z-ZnpV?-;;&By?aeJ#C7FdAB
z0J9wqUnueht%24KTeBel{yv*87o0)E=6kEZuQIqUx&Qw88yk~XhO87?mboeSw%M;Q
zFBd~j!6_;&d2ylf?QQUV3t3lJ9qrcN=i%km#Z@dj!#JHUbX7>=!6sJFY5P60*3;hH
z+`QUAN$L8GwuMipmA<&3`0MNI<4vsG5+)e}bFIsdrCt8Me0A>YYiqx}zTW@$_xI1&
zqVq5AsVo+Cb$Md7Xn#-9=~Hv9&4tx`5}uuz3EE$EYisuAoSR0X&$KMd-bnb{eib=8
z%XD$->1jFl_C!KD7*Bo$?<`8K`}5HqWEtpA+~t0AZ){HQ=PGXfQDb+n^11Az$Bzql
z#^l`I)_ZSnb@KM?Z-L;-=XJ#IZ*OnztF`|4@#B?sv9leU*(T1Jv*u8Y+vfE1YIb(>
zRK2HdD0vwKGGbHe>1Bt&^DuWLV)Vp8TWLQ(J1b*fH-}49%b@O0Mbp6rU7!DNsQjE3
zwJqmlxBk8pH#R1N_V#K-Zd$^n434$rQ&Y9Y=huFd+?;;?*=ha#PrlvGPd_<HRqM?D
zk}od;=T$u7jM|ptxjJm^kDt%yXI@&;3DsD<B4niyXzcC60>>|JZcgTtwc1eqJx}ym
ze#PE6w|+U_ni`vl6DNZ9Yl8-;_PBukH#>N?S?-N(xzblxhcC~&y9;#3!jdH_qR)iV
z&dzdudTMIqz8cHZ)AiH8yttThc2;O>;vtjku(bf-*@&GxE$90E`u_gB{{BBsK|w;I
zS|JZE``cgr*)!8D_m<1&XJ<j1THf#f54x~JL_aPDGA6!r>ytAxC;Qv~owBp|`KPbf
z<AWEw@z(u#*dD!2uCufA%d4y1yWj2V1|1Au_}ER#JZ}wTZSa$8VGEsD#q{IOTv+J*
z^6F~wx3{)Fz8+t{H}%{7#~(jDo1GuDvnX{_+SyZmvep%EHXb+d4Q6i#ji`3axwtA+
zdt2UJueZ0hu8iMr_xIP=$6sDvHvekpKi_Wdj~^A+*2RJr?}0WTotUKB&BrD<arT5~
zZDKonKo=EQ6et*{oq6!{`TXTcN4w0g{!>*|1#K$xn{Rja#l^*$7Zx;1#X`EXYe840
ziO1JWJlxJ7|FLxE-8{efcE0cK>|Fn2m(S%T-qZDL{(LxmBS&fmc$jeF>o_MTCi$8V
zjGUaDT5TO2F=el$1O&m&9}pQBAp)XWK{xV&22)AkP@olJ-Nem2r#I<Lb$oJcQ`y@n
zxymPkN=ix|etyqxY)lr>3gOs(`>bsFor9~_?^~tR*>NH@F`UyS&n9mD^sle4$A7rd
z$jr{r(bpH&=;HEZ(?x?VCwNz+ot^dM#Kgz^{qAmVm$v7}i+Z?##xU|qV|SGt%=`cN
zfx}|A-Vhl`;<{S;_SV!16DAbrr_JsydwVO^$e|08#ga}<)2%)jP!bU#!Eg6t0T*Ou
zbkf4;?RlV;qA_~n;1UlqGGbKn?vCVPgAKRerq_#sO%`O}64i29=+ye*_urDYw?bJ3
z1t*^NOFz^dlh)bUxyR^@DpTG6(8&0VX9i!6zu#Ye_1nh6$;<APp0~})5u6y^kX9hL
z^9*QFz3^SRMd=6kgs0Xzo<1bYAy7GWUDLmpDMFo=Ns-@A>qAsmiMg(8xV1C?apRxL
zyi>K>qLSx+GY0qSC>3>Iu<Qm+SCcbqU33|AJM#60#ZQHw?9rB!cKq|3KWXP38QBj~
z^`^78R2~<8vddh~ea7S~OXUlmOg^f$q@yFI@KvP9&Ra?EmBMe&s;K`lq1LwT5#NzV
zd(Iw>_q!x_al)Ost0tQKaO_)ir`J^?@b2wbygJ}Eg!$ds>q<C%6e@mkL{wY;eWh?`
z<;Ip{5|aJ~N*ku@_S>p`y3_sX$w3u{NM+C3|ExhWz9+-9=WRANX?IWgBONSI{cq09
z)161-_B@ckmj6V*@v%^UcSlD|SKC6@b(<dS^8F#d#wBI<bi>cSsvn#4Eq`5leIrO$
z<96eo+NcCsyPlcm`Z5tG4{kS~yf{6U$LYkcCmc@o_FbDRbl4_jKk44#pQ^qp!fem}
z`3LHvdCZuW<o)ZN5Y)PHw%S2v@0;ciY;}~Uy?D4@ShQ3~e!JkK$kYGQ|9gK~@w}tQ
z%roAQKmMeeON;D-8`g(CZq8Y9Oh{1ht4>!;>B(YClRCpXg*9&0Ts0<LC+4VgESsi~
zK7aRyYqe(%GM?X*|Ka+C?N-~j8e1A=usQCT^WwXNdz{>vkMVaR{=JoXVLj#X{PhYQ
zom=f2_r_LFo|7UJWpUzR`@8i2U+x=x5;^xHOd@!;PUYMk>h`A>J}fp!(h{0hZhU-R
z9lJk+lh?7wM&AowOiTx@Mt4aJd=ka==L6&K-QOPnICfa#-0c(p-kfd`>pq;?RDQm(
zNBwQI*Dm%qk(-@XFf%Ae9}C!;XnuZg<K;aJKg$1eg(<yszr2wz@8ph%W7GNEc-GaJ
zdCWO>&vEgG(E83j&Qb{v&sz8!?U;2~I4rsQT;tYCu{hIya6XH%p7=Lj-Fpt_wYW+5
zs%9x?3mcwrOsO%`IWSMz?DNc(KY|&nPt4bQ-PD~VbFN2$Gr|1dk3Ai2Pvow(7u@HU
zsrcjFG-FQXG-dz3p8D)E1(Q1g%AKIn_{7#nwrN6tEB<d5U;e{y2{WV3?6isVxX%5U
zH8cOrp&g43ZcgT2lG@_%+QrSr#OS=)wxd%NRMOofw$v;?$LhSFzri8Rd|~OKL-%&@
z@$FGLAJMSb_n1|W&c=WR+?LX91sNY$YMv-;`+qn~MNrVvTYN`HOlo7YrbJ@u3F9=L
zdn%JJ?73KBeqJr}^ZSU!`xN~5^F5LN%VuZHc6-}RnR%J&X(#4Nv<TRKQT>sl^YZq$
zWD!lrc@y^*{7*|{T4xo@!CxpGd!$t`!)sgL!WmDyQ(`1ON2v4p{hfKh_UK8&8lioQ
z)f)vieXZ@(2~*kD{Q2XXxz1HV;)_0ARnM=NXOELz4@zRtg7wo(lT4vt_PqzBUAbd^
zaX#F5<4z~bVeUs7b6Ov~w->gkdU*Rn*dCQ*C*Rgr{AqgrM4#jLH-SIq(&BR6lGEHC
zWEi>mI9lv#um8=s;nx$*d;By0{J30Yx~=1cE~qOXKHJ(*`N7Wb4{n#dT+ULS^ZWKu
zmhW$W&c7yGkp3q__U__s0)i9wFD({x1q~l3VvZ;ix>NbdvgML7F6$a{)%`cVN|`ph
z=>D~NK1z2b*SUfAC5lY^4{Akp>IHQ0H-|5OY^Q2E_43J+PfpJKylRd9T~WvOKG~}m
zl$5q}9Nwhkohq=iP*p|c!$EfW2@@tPNb2Y~k-D6Gr-7EfxPU7cklzS-1XVSbkRYf4
zw+(r+Wm6xV2Q|oF8mA?&uAQB=U&<^8RMA~IpafdX{$u|vvs@|uviF{SU9kpl7n+}~
zi*&S4wtsb~<6-m5o1X(sw{(E=<G<+lJ(bfPo#U4JJkReq75r}5?fo%R-z6XLUVC%*
ze}?sYjbeE9l$4a}y`JmDEqP!3SK`wv>nZQ;zAxkHVo1En9r*8IWzYX2y}4HQ9rxdd
zAMWnx=s5H!{^QS@JJ&aI3oqAQbm;JBj<aD8mfgEQ;e*5}yBP-_@wV?={BrJ&yLqxK
zV7FFiYHDhlWL=shUvKGmr+IThOX_~rZ?}0k7ndB@zwx$j;jgz}Evmk(;A0a6Rkb$v
zCn~$Ih+53|+qP1p^uCLa`wQKz|J%+k`C<G1QE#rp(o?2CzFv?2x(?LhX?1OwYh50+
z(u<d=a#rx0br&VfK2KZgda#Z8M9@#qRUzw-&N9vZa!x={aALHz&bp|rU0J5DC#<cC
z=ekz!=gu@?c~&@=nY`~|>siL>=VU;41H0s%-7sUq1c%4V%7a!s-TkC*f4uv_$|NuI
z#Ubtb8TVg0+Wjq=!v*nJ_RkN8`(NC6n&?$N%c*ZtZz)TR(>110&F>d-e(f{pQFLwx
z-OQ`BeM9uCWp|j__@4dsR|{NsyS(MO#-eNgxgQ!H)%gFR=2&0$;eCJEfBrbUamKQh
zhhp3(O`5bphkN_&-t*5v4NiuZwl>h!N=)GPQpB{>yKk%|Ian$-MDst>jq`u`d%1v+
z;Oc4r+yBf<+2;Gw>1LB)&F`<%TiV)IX<Tp2yu9q<GGA#YC#MZM+^08fxS3P2H_mIC
z&dF<QqeE2$1$XM?{Yrds<x0pbll|SUufGL<TgbwCwEpC~o+TyASLRqt9eZ0URrKfx
zr|46wfW1|vF?!;0>&54vZ@>Tk`0KAiLPAcjudNML0L9!N^*&i^x39Nfe|-HiXUeM&
ztZ5!QqW;uddib!bM%8<Gg_vrEt~@!<wmM`=wvPWin<-kMulW0CFY|nUZZ6aW>-~Sf
zNl(?feqi!qxufr8yU%V}z4iZmuDga)J`~+u8S}QxEa%38rkG^V0w)`rKm3ZDTsnoc
z`oKwjYU%51flI4&m9%aB4!SX4JvBY>-mCrz-}?lw<QA}Vi|GV7u3NY*_x7Xjzt8`>
z|Fr1h-s*DD;v*5U|EKm;e$KeCLh|SAy{`AN_*b^BZ<te99=PZ(OVrl4N1h9-uTr?~
zu`TE3im<h}So=#~UGZ!MwYHDif4*7&&-peVf54{0DoNhv;jiNyFScd{eLFbQ=HnNJ
zxXRKNuA<&AFD^EpIN`zIHBHA8bOE!7py19md9$LQ?=g9k>bCc{;nMg2XLa0oly!6U
zk>Xzhv+eVJZ|{~eDm#;rV>Y|1NOb4jyrL&37!RxnTwL+xqB~?co=v=g#FLB1=RLS>
z>8)A(|GLHEyx`SwssFxQ&iT7Yc&hi)C#&_<AC*{zPFcMoaIu?C%#I15B^A2SW}s9q
zB6i-x-+y`NZN8j8Uqw{k&+fP@xl(Lz*uuN-Q;(b5o{%h>^0PP%bYNigF1h&HucFNC
ze4ssF;dX1%?<E`R=)CD#e>!jb<(<XpZ9I}rB_&(<Y6JyakG{U%DXgAx=ZjxoWTj@^
z`|2A$?NOhlHtTfXf5vyf>Wgi+_}Sg6=jTUn&yxl9ygfZV_xyUbI%G}6$7Q$nKY5Z8
zSO2#(?cAJ`3mltY+}x~QwmZ*TRcX<Nin%wp=gXH$n;blIea0c*wnT}xs83d#>TgQ)
zt^Ylt_49sZJ-v6+ZahES&VPA*y#2HD^Us4!xU}@NZ|%Ql`uqPJ0^Jbze*gb-D*_j1
zTw2n}S0gxa_7&dm*K-~p>pc>_)-z+~I+0AP7{`xRQ`TP-z7v+uvGtW%%Ew1X&A;wj
zwtTs9&JBY{j~;b&b|xNf<1Kl6>+7=p7nb|XG*VGf`Ebcw|KjR!{aL2j$F}9(F8Ki3
z))W`rEvCE3@9(iEnP0emW_=5J9eX@a`+i%}TKk9Rs&5&UzPd8INPhpHPu`#vf4{!I
zj@(tEslWHjrJJ{J+|in4o`3Gv)@)F(b8GhXXPxTvF8q0@q+~u*I!&VL(vy>4YWDD}
z-s)A4ei$`9b>_yOZ*3wEGctPEzexVU^}qFKr<vppgBF9s55C0luV2d-Q)F5Cc<vp^
z^yfErnZ(_ev8m_$@<zb#X0V)Gw7`RlZi-XRCnukq5|XACfA80oHLsrBb278B+rT9D
zfA0KqlT0+iE**4EeS2f$jHweoM)X>+ul1T)^CQ%1_qk}dr&4;mQusflG`-)H8~*8`
z?;oR&c5OUwyyq{kx-XJH%QXAihbzJU6KBmj^?LpOb8olbU*|9M;K#?upyitR`~OaZ
zjw!vYaTA<4`vC97%@&WoPPDnVTfQ(hJ!abijtM)?pU=MkyW?bD(3TjEQoUVn%Y2>1
zT{q26`TXm@i3XF-&4=vM8(;J9-)J%C#q$d*6@E13*s?Bs_EgqPUpjqxpo&iX+tlg<
z3i&-J-@mri@jU(G@44JMrkg&O>OXH(IlJJotlq8gQXyf^m*Q^8ucmK$?8y^%?QQp&
z&}m$EpBvRYVgK^(@^tB+e!lH|vWtqJpSx3h-nQiBrPkEb(=;{asaJn{(<5ctWm){p
zp!!?RqeqWs*j8@?53p1`jhN=TUj6rnV>1e;ytnH=aoa2YTj`F;x=k~eW`y0e5ufYi
z`zqx;M;YULSH|Os0^->d9-k>wW4DxJsAuR2Xx_8%`8=!IM61K$DJ*C2s{fZblC+4o
zE6$`ya^BBhH5b_)irenb*y8Auwxah$?(*wz{fcfFH@#f`c~3<{k?8rQamtre+8E;3
zaWjZbe!l$ksUmp`J)X60TRb`2k6t?4csedQyu|KQ`TWyYTI4g!@BjT#vrqb!@1$wd
zj$K&j47##%XYq5;EsrJd?nFL6{jW1<sh45$F`l@}r=p<igl3s!9-5{b4H^fOzEW!K
z_T*zl#{G@ltIg$?)O@eJ5gWPgOjit>JKrykl+u7DJ{(iSmV6Wr*q+95?R}l!-8A(t
zF>OoILdE%~8vp#;;(AX)diS+uhBdJQO#xfnG#<ZBVd+@B`_jyUBMYKS)fc+&{W9a>
zUX6lVFYcdm)e)F^)$LOrf3sNxPtLO^X44(BcDc?t-JoL6yx-%v#+wH-Z}I(!YnBPz
zkQQil(93U*gvnm>o(-&-J-45yS0+vPwcT=u_NCKjQZix=J0@(OAAj!tQ|~E?cIO)J
zzD-YjHC1!7vdJtX{xb<zx4gTOf2GHg^M2g-=ieiz9er|g^2_`C?LkK|-rSV><ox{j
z`BMJ_E-&-Vys|=2zV?gY*H>2;`_484Ep`G8T3RS6nNN7VDgEP@mzS^PUTFDRT{Gq7
zr?*qNyJpU5RJd&Z^MJ|D)q8k1T->zc;2lxBbEi8cHkQe)%t$|IV$QtZ^Xc8MbzNa)
zh4*Jx3#<Qr`74p@Ge29W=Bp)6i{^hl-aO+^;ft+tvg)cij&}d`dY7^{yNGiw^qe(+
zrZtnpgq`<yROW3IU*xmORjMTSjY@9&^zGuyK3lVeW?4skn9RRn#-~cH$KO)dJf2-@
zcVIzaD0{+UM@4Dxus^Z>^Bn&sskN8?D2zX7HJeXd^|tSWfDHzrtFG_UGGFsAer?p&
zE-BM2gRCnW$NJ^Zw{QxJi2R?Pzqri0-}ajXXuagL)WA(Cp7*L=ua#a`9_iwe_|dB9
z!vjaH@YP2HH=C&5>N}pcpDXxyt3=MN7mv#qB-|8uHLYBr^2+h@PVut6&$>++x#ORe
z8oW09`5_^|sGoN?Z&~iSAIv|FI`PZ#bRChB`5tREC-hAAqwmgVS1GzI{~5Xa=;>oA
zH%c!xH5~OnSx}PCz~6A%E$+b7yMC`XmGR|#-&C|%-al<p>747KGxnT6?`G~Iy0y5x
zlHKx7UE1HXM@`b(7KG>U#(WHtHj_RY)z|W;TsVHI+U@Q6``7jD2dxL*nCu?4J<oT0
z-ra~j6^5o+S1za@{U5w7=Vry{v*t7H>+L|xV0~s9otb6o4IZ!jQ@$<drc=$;SvR`3
zEzjt_zV4P~qGY4Ni92r@IktaJD78Ml?|WiP+^?VeJ{w*AnXY*2EVumk&(9zK(G%XY
zoO#dg3x{8{#@&m3rS6{d%+=_bE92=r2C>y*%Z;lRwwKp``1J6f0;{v$)|Bqd?7a;q
z_g<BK6ZQP}?R$rQSg?O9vFDz=J}z2(#dn^Y+20obdRrQ~G3ltO{kNdtU`d+_gQusb
zFL!8US`)v2p5I)nmwS$!UgOd!RP+1o_C5c8y$0RzYm#-P<Lqp6(6~EScYXfLOG~e;
ziJgDLs`*6x?~|*y#c3Z^QaYxhzoBC4^pu9+x_y?MVQKtH3dS<;{COT6U*Pt$$0Ys8
z4UVIamhW{mICzGq(y-FUiE~pz@q{Z3H)16lAE!L~RP$17wor46DA&V}L27~v&3@Sn
zxa~R3pf0e%Xnly80ps_L3hIU#x6GzkO`NcPL-R)6xf3dnZa$nRF#UJTiksWdo?pMO
zhj;$|ul?dzcO24>asQj7=Dt1drs50lld|_450&qApZu)cIQdvlcF~>4X-8W)g+a$^
zfwtTq?GgoTCiw87pw=qJvHVUU`|7Z@iz+`qGs(EnAZJ@u@a09IsGG|Zs}w#-qm&b0
zHz<leULE-2oZ%Cl>hGb^=`L$dKG+?bQhsZN1b6ZFUxm*MKjwbg^rOY`{<eir3MDO+
ze7vL_r@iGWIKj{RIQCB5fhYGYtfi+dGyEtZaw>m$4JZ4@%C~nSkEMLvWAg2J@r8X$
zn8SDUPI6okV%8Ed-Em$F+v({)PXym_TzQCZ*7WiT+f?UnbkDVkKkqr?uEKQA(A5?v
zWz#08?>;(ZEu)rbfk3=j3!7`snYq)B-EmnFdeQNyk>U|G_K2b%XLe2$TPdhkC}%(a
z%`O9-X}!f4l<ioi-(8TirdRp+!+Aw&-TS_8xLW=7wylnP;wF(bhojGLTTs8hZpO08
ztJn4>9p#GJo_F@u)zy{n_kMqIb+ve4VBoKi9Z~l~a?P~D*PUVJw>bb^X5N+<u+(en
z>MEIzjuY{_^YTu~%FUdbbw1~apj55$l-@-b3tlZby7BXO#VZQax%uXA`B$hPd(VUU
zj~M%<e+MVavi($2c>Cw870;(z{~x~o@3JqW`LFu=w}~JAKA1Dds&dYT4I8xgt*ib0
z?c<%|^B*7g+k?)>c(?n#9j|#jd)d1?p8x;-y}2=&UD>_w$-L@!pcVV;+V#I{9sGYN
z>AI_<;DNc*KK|G*dCqpiAN{6z=ac>)ef{<C|0bn^v(1mLJ}TQSX;GjcsvWk(W1<o$
zTky$RX=uc$cZq5ryB=SEc17S~(4G%oDU%1#yIn=ZT-OzB*{);v`;Bpzxc<3yvAZL7
z7O66`@$AsrJLA)P)0`U<rfP>5eZO0Nc}HQgT-}dEAz@+Yo`uuzkIUCD@tUf&=l{Ro
zKJ#p57Ct^UQQcoo^qE!qEH!@nKL*lfISM8wQ!Xv_HcmRiu_tbQm$FjP)R%LPdasDQ
zdU7voP6boKEXnO_EsGAgABwWy{qO1C7Uf00a%<(CHX3xBt5s#(QJk}Bxs8q4^ZB|f
za`sG4*P89~$7U+)^xQJh=H*r`iR*kHpPhT~^&^Kpcb@CKySsaO`uy5ypoNMhF9L3D
z&#%9^{pW*SGks#VNs9}by!gYAx;eID@sj4%X&=6xIy@mSaPjBnkh8U&JByjrBdv0^
z;(aZB&U5MqY5m!wSa@D;-ufEx&}F*I4W(zAr7PP1lRIv`e%qr@ajV}ZKE1*pS)<gV
zBgemU)%J()vML)Gq|^N$w8wf%>|Awmact?FMisMje<vONXJu1ae)99QhiZv07?@()
zipv*voQ``PySwb;A#VK*rLV*8RXpa6+M2c0KVNX-Y8Tt?=^JXcwttOl-eYLCZH-s*
ztuTAlQ#&Po1ZZ59{ht@9sop$ilcYuLgubH>UF_Ta{+2ozFY>)5a*l&{xgVQljmd#|
zr!9Er3+4)mvs%~l-z&+TK55b+Zv8zDOTDIQ1TJEEeSQ6YB}YL8|39%?bSIrTF5mxX
z>H2H&Uv4k1xmRl-U;jrhQflkok^mNi#hMdNA73=L`u`8>-IKlU$}cVm6BlIl3cT>E
zb?a`u^z*aY=THCJ#IAld+C(|`P4%AbCUb6GJ@;kLtSRe7j~}1^Zb@NxmW3_*k8h99
zOZ@#KVYc*#e$17ZvNcw3O^zn7Kcm~;C8m4o%F5u%=X1*?Y$^=CzPg%xxQ+MM2GAzw
zJ!O9B|9tIgFD1QAlV2S6i(&F@?pq4R2FF~a(yJ@LjYylT8xJqY(2%iy-!=Uu_hrd_
zj}FZY`s5~cWDf5_w)1yHy?(yvS$6aP?EZiED=g}h_RR2TUUX4BcFM$`zdAgc_lUDZ
zHSdUc+%#{6RQj_<zkQXrPH)mV)+_D)@zK$nyUX=ghp%5&|Nmc0TU*@4ug~<tSQ^$Q
z|3B>S`P}NIfBv7o&M=0YuPeen9ojj;_MOV4N#|rmHg4S6Q1GVggUU|J>XI2dGw%Nm
zJazn*;oO@iCcl34@H2y_r)R}E`#DDWTv7$Q3*&7R`;XTitop7i<-A;ddV0(R(YT&@
z?`@q5Zf8t;=V$qQj`C}<;Kx7RjjL?tR_^?M@%YZoSscstqrMz0y}xm}Yjsz9{LSvx
zSJ&6qf1Bp+JKJn&^!7X%%OV%~2qmTP39+g_viE&C6EH#Y-lwplkCDvMD_@Ew>NUq|
zU*?>`qbSyKCQG?;d%NA|+9|s7PVZlp@%`gt&D-o)HQ`Vp-><N8p{gt1{gQD?MYkm{
zUFm#ZF(GIB%sDqEozvMr*Frqt;!%g`)4#+h9!YJFoP5{E+w8?nwJBVB-6=Uw3L<Na
zb*FaRcHXJn?->*{VTFct!p4GWvpego-*4ebe1A`&(Dm%*(>wOgIQj7Gz14NX@3d77
z4JR)1o&Bh2=M1A%u8$u-hUeau^SAKle|P-SbT@;<d-@?+((F5C{yb;Zw*T_{Mt@z4
z_cu@KR8<>H<`Z9FaeBrU<9V8oe^2CB_`&@B*z~i1xWfC()Azf5*>#df(xPp*y8hDa
zoofo>x|hC*KQ}x7%#o?Kz2}#!8fG1nSX3|Ldj0yc3!8Gih4y(dMB5(ImGHe9VSPKj
z{>Y7`>KwL4t(AebGp=su$p130A(gfA+gFV?`^)oUOrtIx`_Zv*zW(B;50=ePnf&1$
z*RHxxXDWYB=;Pl}u&s=LwOZX_=l*XGvhK)UH1XY4`r7UGw%p3MTd!-x?lMU}*0V7A
zc;BxBLV`P)u5AACyf&2K%9{5}>Sr75nr7f>^&<5Omww%YFMRy+|7M&!7{ljvqp9X!
ztduuDTc@6g)9jeCs&M|qk5@wC=1e^GrP;CIK>qB%YwYEY9AV;_m+?+1H(Ba$`Vzz9
z_z5vb-@J)<m{`qF_IJjy5AwBVZqGHm^PD;BT3Mj(?xPzzGr!Dzkv8|X+v@7rKi}Qu
z<sKa2<BzrXdCI`%nv`=z_pO4F?tv4nFMaOG9!Rv$mwR!xarUyM*Vab2pG^bpzza9k
zNRv^EQ+oXBbHMikv3-9;tC<qsMjz>6%3Trv<&boZe9RZVz5jz7%j2rMp2yEVB%Qi{
z>EX-wADy+i7B^#0&spjJ-t$kLuFT3{YMo`eyf1P0k#H;X`&%k{a^A`<ye_==McUsZ
zoAu8xe>3-c0BiFk-QOE3KL^Qep0?@fcab%ZPiNk(XN{fz?&RfK_6PAltm5BkJI$Xi
zl{`)AMD_Wt>eD%Ia9?`3boYn4uZr6K5fSY52|nBWLe70N-`l!;`-h*micGZT|6MiB
zPD`r`bWqa6!|k7+PLF@3><wz~M;Uxr!g_P(<<soJcWq`qNc|t!%^;?2w)SQ!I|G9P
zgQtsQ$gjwamJZkEsMXnDStI3CGHd?U=shwg@{-gKbWN7@wltr<<><#VucLYbq*;Bt
zO=n*Icy``ouj_>`Wt5er=gdt1wdJ<eZrzWpK7ZK%%ABZ4IVxvwkrQ%|NvH0X6vLXH
zhsPP#{65Pc$Y=Lj&h6gaxP3O2_iQe#G!OSqJw5HM)Y_Zv=jC13hB<Heb9u(rN9NON
z4JZGWTjRRp|DVs?wmI|c=LVfWl0Hu;A!uQuI8%1mru*zSUIlJ+5i=;h_~~Ff$C{s=
z$}W>v-%$Pa>eBa*wFU=H+DiXSJj!=!&ri0+Rbj_omNcw2y80-oQDBp(@j*V%-SvzK
z*Jq`!xtp1Oq+@$Vlm^p$bJgP0=QJ<=uv-<e`$yKnsIaqhT({3FE%@il+PCI+pDpjL
z;5(JM@4{!=R+rU=bauozO11s#Ucez{zy7n~XQ6|qCjK<PliKvXq@i5=*SUS`GDFo9
zCfq(Kzi!^P_z%sF3vcuM&{Y;V=5l+(zb|Yz#WG)teT$g9=Z6Ku8*3?+oC^J0%Zy)p
z*u6<V<J7&aH0DaCfSXC?jjow8dpp|}tTz>ApQhh1alcu!)DDSnFJf=3=6`a$#_Z2+
z#zV%^NBnN&s#)*5-?%$i=Apwho1G1{Gndcab&%ge(K|<P!_O0+(lyV%XJ4$L`+je`
zSNb_Yo5C5wa<AX4FHHO5#=PXWt;DygJ9ifyz4>7Jj^6^E>E9pP2o~D>tKi+W?~4NW
z+qpCD8y~+Pa^P^k@cxC#{dpNpc7hXUPp%UCe`Mx8hx@LH=TGh7FnJu~*6rKrc!;a@
zLxtw#jsWSu@*InooMp-}Sux||bK`bxWr=5{4p|Iu^q948K2izTxUGA4GViHrJc$+E
z0_S3kv#ofO7Uj;_dF$YGzUV&IgNZ@9pWiu|p1z@|d7x<u_x79oGt{Q&K3`B};IO|?
zGsmZ0>8R|Kf`~PDqym`Z)t1-aZ?=BGekk+SQ%;6?uWoh}9C*jk_<&!=VxHhhuIamk
z|G#qmGp$nb%=-KFy^Bk|KS}T74*K`aHUCk~7O{;D<>3<qs@;whx1`*ebWCnubJ3q;
z&Hi<#7`M-#QrlhmY3I!!hmFM&&hsqiJMg+)XCv!|J^2QYR`XVT*kGHevUvLZ11J7W
zd-zdh;*64auby$0mohDQ*kFC~YrAb(YRl~jXL@~4N;Kt3O%G;o?l|G=W!pVHU|S8x
z_h;u%TrA&KRQPFjN!Txj_TGJKV=N3aE}lBtn15;J-W$$ew=E1=DeN`XYQz8MCULQn
z3zE0=T5Gj^irjf-opk(!XXQIOPGvH;H`hI7-n0F};f6N9cYDocIE+e*j{R&qK1p{~
zDBshkix~TE<TM<V?R9!#tUmWrq|?sLu@X=FoaT9S^DnE)xUC_g&gLs^%_-B+q-ODW
z?vqIAC0>sMe!b_LbYJj^c)w$6e8bL@S&vUYJh5vgPt7y?H5=b5KL6o(Cw8ahUFl_)
zRP5OQ9hom-`OBW6#^6zIUiC)t)Y)I2{$>9ABI0-DuP;3b->#*dsoTpZ_%eFu{udlp
zHr7q<pJme4UY!5)d}!k1?fD<7F8FW#I9anKJnq@q7?*_KoN50;=JyG=Z(MSjVOE%Q
zZ=zZHr%Qq||I$>lo3eitTvC$Lo_=VLtX*Bz@9cMfRxrP}uQj)Azjyx1LV-(pD&FkJ
zAKs7XKhW{;%GzzSS9<>vtbXRK=wI4qy23Bz?j-Sd1rId3m(?Ero?Wx1bH=$ZGUY+{
ze)ZiAeZRD1$1)!g!JUyWm))7}3F-=opSvt0mnCra{`tFe%r8A~*z|RK|J?m|V)Sl&
z?SEEcrXj`K?jN<JO-kfoQuwW};rsue_ucvY%*?|p0vFG)tNq1WxQ4|h?~YtY*RBuq
zTI9-w<mR}vTK|99RIuCj#^OZ<|N38l-EmhyuJ5yael^48OeRJf#)4m$JrB7~`FxAl
zNnJ(7pzx7P+POI^{YwNVYCG6=Pd~Y|X|DCzU+&M@;$Cg77ZW+J^L;+MU3ctE@mmwp
zHhTM(o}GEFV3FR$b8{XB{<qkYs@L&B<N5#Wz2`oBsNeZE-DzRxWtKAqi>@qDQZhFM
z9YKM9T*LH<=OXu<IW?{1!rC;Bg$Fv1C^QOmpWpW}=zE{QqtCZy{0`MJE;aq+d*<+|
zmGd{gYH1bbpLh8|&q@EKQK2iNY|ihGyY*4m{=-N1`vFfbCT`I!n)F=zzU$P?)0x3G
zkK}VtFH$U>5&6u%ColTbr?*L`(tdr7b9&z|cY4N8R~MJOtXGkHdU^sTAAe<Kxy^8{
z+~c5|A`N?PD$G@g?suMX_{1K@&3E?93CX|S)$vpQQ(=${|F>5@Pj4{WTv^4*-yi>Z
z{q$*jJp$+4*FV<FTBbgAm%ZD=HS6~CblB_vRxJ(_v$kAave@h0iPO8<*~8B##HA!A
zfmUXSh&{dEc`{XqYsu0ypZ(^Qepa(D=Z0`Ki$)tg%lG4Ta7j7FyX*XP{cCO)FL29q
zv-bojKi*~|-V)Tj=;E&RcfV@)bgVJcdV10`q{2*VdreCsZ_Ju`d#9=>2yigS-Fath
z?#@49%9NJpi>jAaPnz`M#^H0mKhvKrs(tUSDPMiB!Qt84;&XdS?k_SdsVb^`7WzXi
zHOj_m`R()XFY-(%dBox=zE)?#&-(I7vp?4h7pNC1oh){gh&BBvE%z!S(s)}(#~mdT
zo?1!Ocl;B0rmPh{HbLg)o{d+|&%S)=#KZF|N(7p(<sQy_e`<zT!O1xx$JB3EEZ)lf
z+GRqA#AM~ympwRcRPNZse8=%jfpnR|*K+;l^oo_&F1U8bm@z4uDKRtJ&6qQ-@NmJ-
ze{*I&WwE~}!hL%8{KCJxK6)O~>Pp;jI_KDwm*t<Yu87>RY5B*WTU6Z6N;p4u;+s}?
zq5onv*9DCd<K*&+O2+2i%l1hp4jA6Ct<;P?otk=o)A@?5D{@g=H+;PH!o_aZm3jJA
z{s%TS+5D~&Id5|M*cI{lPan*k|M6q9=mx)>B$w}s66Ge(4&3<e@!GU&`>i8Jc`pL&
zBDeIi@p^5@xLD?NYwxK`2g{3UF7CGZ8!tY$>))Z2a_fCno7m=@o3Zif)K9beBVC?^
zEu44s^hxmtU;kHmuyIW*GyJwNAj&mL=IdPTZ}WIxmi#rCx>rJE_gS}nHf2RK9OwO2
z>(%_-B%+g){@O~OohhM;>C10}R?p}^EqBG7UhfdNC1x7ANu&4wnr}^~mz1V{eIGMj
ziRaq7=p%oAUaow*Fp4L7^8D*=-<E_K3Lnln=iU9xFf&Fy?A}+4o40J8)~i=tJo~FN
zJ$~N#hg08Q%64>B+w)DtHom6$IFGQtSmD`O!N)hG)*Q9o+a}hZ|8VU}=N<nSs=r9n
zO>K{yW~Xm@OviM>_j%d=FE^ED+}feCd?WKopO^m_PTJmo_S0QH^-Py$%f8#~+v~d)
z<=(E6NGq<|_scYU`QE1Qs|)`;EBkyncz(blPw#hczX_Eu{qHAKkaD;DP3FaUkNm#o
zfF{dM{ZILyZ#VD9g|lyB>k4MvsmzxsJ;!Gm{b!FcL&TEbMr~`q=YIR}Nk+dmy7iLj
zeqJZuqQr;n-*Ubc^89#zsz>on(@*23#DzhMS5D<_NSr=Bl)YK&*H?|Bhd53j%iS)z
ztaATMo%<hj&vvIP8yz~ZCaCk!#+{ns$4&-mqqf|pJ%0W&KIrScRVjD$tXpzs9=@>5
z_}7~WUtZ3Y-XOZ(rH50%cfPOm%P1>tyZmD=4<(t8`z=!5l#+Lz@!8uCfg2sJoiT`>
z@hem&$IC~pI;Sk>=%W+zc@MM#=4iQvDfIkc)SYp0wcy?RaSn^`eHUtNmshWIK3=Vr
z!4c*1L~K#wi_<UWrRgbMU;4Rn(a~`3%a&3cOS3sATdC&g{55@S+4uS1DK;jtqh4(W
z3)VI7yua(`PA3W3&(2k8hiA+@ZyMuvOg{ZGLsQM33x_wJNj)OIbt3cYH~*jh5{zHP
z^z_u<k6J0OOP=x9*-N-)R=&3lIW;HxrqxGjVSaXtSf#&<!eb@QU-~Q8wA*N(OG?Wr
z26dsk^%dLxzp=inaLB--#)!3p;Z4ffeD?M7_5y$2_8YTJ&HT)C`O}nz_ZK(sO#WlG
zFZzl~GDnq<pXs`e6S|<$n2U|U7dFi7-ZS&R{S+A%oo{Mv2U8!YJZ74{x&8=qz_J!u
zCdX@PPU|!jGA10I`0=;SO!18_F48wRH~k5>PcT{#$zf@banWE3yVoqPuImrW4cH5;
zRR8ktFKls{^Lv`u#JMP~rk8*3D0P{wKXmQl<eMdjm?l~5F+R{Ym!0v$J1(K5@=2=O
z7^c0qb@Pil$k{);H<-=1?5*dYdp%z(Y&=!eesb~}{`i_<w1ey7f;x`J%kodWDfLOZ
zy`@BI>S^wmnm4*EjQ3mqvSi&6bF#+ngLl!@S0}dH*L6Ide`Nh@u}#YztGn8>K~Z?^
z*J`!tzDb){_1K%=@jw0^q9Ks{_(h)n{W)7c`e>^sTv!msvAoZLll?~V&6C@VIkw-F
zyrDd2@p6XtUb(NbZ$3MgWQyk>iEueCu_k`fu{CSOEW2l3IVFGSgq-4pC&>@QckO!m
zODN&(mGU1oTc^*>Zu{i-rQ2qnWzK8=Kb@TO^;I^!W|@Ch&-=Sr>I;iP32!Uj1({nf
zN~9I5>B~AsZWio6o9)_em8uzc<`|#0$JVL$jqlDq^V-q=>M{Rik2dogWZrPI;;ou7
zbG_47^?S)VznPZguW?T}Qt&G5_E~rN@U(k(LMyJxJzTv^{zBOg@i%X8_4zrM2W#i*
zdN%)e-rv+-`TFK}j-2%e?u0Hi2=7ZzFW`;$`8`wU-S?NqN-b)Z+k)2K^_Hosw~(Fx
za*bk!;Kb;j#K>u-wmBXZDPgN@Hw(IFeCdt$ZFl6Gf7GzuF=orbl}{AdFa0~XuU)ET
zSt^g}aTh5W-!{j)hd0*wq<&%K4*%48_{_tFY&&B%yk1gzS*GJY|GASZCm-kJ=Kg(s
z{aoX8zQu06Nx#0lG|$*~X2;)NTP?2VmHgFr@(NlXs}#JO)6!jOvty(1I}`ny*oq&l
zr*}P2_4NEPH?YO}|H)-L3#%2hv}F3NIb#AX)^FZ2W5R?3M>>UPS{5&}yXfk&j^V3L
zoZr4P0nazznDgU&^Bn85kIvjobDylMkUT@b+-~xewRS;zAOFN=YJK-TTp}d*_4t!#
zJpA^bTkPxCUiSPkf8RU*xS~Lfzps6{pPyM~d3f=aFeRn!3UfDC+%<f0<BVrWkwE#D
z=DJYR7wSq%O4s|U#MWC&8t?ZBQv@wdOZ@mNrDfKa?1w6|IazdV7z?b7)-&+3&73;n
z+$3!)o}3Gtoh?%RD=Tu&+3#;yTC`_!o~W!vlVe%#w-2{o^?0w(4lBL1bB?V0g{0SM
z`uXd=t)94k<74Z(gfrQ~H{4mCN{1YsIcaKN|F$RV*{Xue^q-v#6#kKQQET4M1Udd|
zrxrVPb(|1gp7_GM(?ny2Y}i^4^~CkA_aub-Dt>0HY<Ccwyl7&06|)YX`Y+3pSMT$7
z<X%|THdn?W=b}K>)a{0^cZY<@wNDa_y(7x`^YI*;po_=4Oyl^fcu$3uT--fl*56rP
z_fJl^<{$LtW4NUGyU@JyoTq<VHS24q7CvA&{H|`c_5B|$?~i>^Wzo0S|IOOJN)xo+
zI6U5`<>0ibFKhPdpSZT9=Rnz?hzWDsnf7eI<X*CFzee=499_eSY=(W^i(*$eUVOp*
zK5)AEr%T`0%-&}(_n|gZ`hR)3Ss(NFWjubv%3GSfJ0tR(qu&A#v2K1l|Hr$X9qs16
zx81F&>vd~}wPi1-bnPAs!@0o@TY__Id^aaO(%ZMa$;G9O`C&+&{R~CdZJW;*KU=hd
z!BgFD!V}Bu|CY$4vPT{eR;d5G?#eg6#g`|aJ|8FHs_=QIoVn*aW5G+#L6d&>@4P(s
zNT;`ro|00Lrc~R$mLvCT_RRgc`Sl{Fx3}Lsy|Fj3ChV@;B}aXKwThj~BQ?H1F~3pE
z!}WB-!|#zdWZ4onzC2!X?~cp@S7&kEqMIBoH#&G&ek2r0wRc%mu3Xm~IXQU7w3@uX
zsgGq!R=-pEvSNCA7>neMGbeNPny-{}8%$^Ptc&|+c%XKv@vlYma^5HXF@L${3!_9y
z_QRINcFR<qfApB#Nq66*d+F+)>vOcrHKQcV*tV=X*J*h9MbLhh?|g5&kLM-Ky>X=D
zowLEaj5CcJ-%q@jEs|fZd(eEbeR6sMZ*sh(-`!JBBaBO!rinhed3|xl?0Wgy_&C4k
z@2=kb_R-~<gzHQ5HKw!GYEHkAy{-B6NOjH4#e8aV)As#mS^Rz4vTa{2=cm`tR6QU1
z`1eBF7vHan$w_{eY!$aZ*6=s>=citGjj6u!{7OoT<{kZivR=qWg}HG`Pn^5_hV)dc
z_v=4AWarEAS^m0xWlp*Dl(lkB`y$wLyJzV{PB~T|R(&w_(ymP>JB91E$$#1OdlAd_
z%+26A_J}1azh>@KW?-55yLa|;v1-LrOYV5e-+#yFbl`l0=1j+niOCoKKD=@&&!MOJ
z+)hh<4W}GW_U(cR&(hAS*XHhDB${%eT>iqL4IiHguU&a1S7i1bzd30}!Mp4oiq3DZ
z{CCk^a&r>b!uQ`Eu2p5U;;(4Z+NSlT_NVKvEt#%}vAE@#r_(|<mL+P}%0D(=|8%Pu
zyTs4Ea~bdZBtM;!e)HSGG(YAg@1L#Rs9o{k<UJeHFTytS>rdprTeGF6P^VwKd_&4f
zrfHQ`2H!6q7ZCxSUGVY$QSpR#?-VYo%7h$p>y2USHe<7_GV);Pp1&=vWw)nvO~S1e
z@g|GeHw3H{$a-paW5emwtC}5ygqZitdhm3)eazvc!asYCm(AUxknl0sNan7zMfJQ}
zGb@*Eo_BYh!eO;#7qY{wG9^MZOYd3iIQ*n_SJ|{PUXRtcJ}?!ZxL!@!Y;D4&Y+H^s
zzozu@-b<*z1Z_H{b-Rfys}p6PEi~Eq*+jNKGvjq??)B`>uVQ#3c~H;4@lC_zq*SfI
zJy|iwjv9UASaU#%Z<lVt)F-b)xliu>XZ*|dyPia)QT3VRii$7NIlr@%9{NU2@IDLP
z<?+A%{@M?Z4lc>A7Jt}jH?6=?k6FR?E|X4;V}pXc{uhI)eeD;y_rBpXJn>QdwEh10
z<E-_S@z1*3rv>x<oLOt&5%FSYn%u3MN6%TV?6GD^5U88Hwf%PO<2lXWPha7WtUbhZ
zNb~+D&VALJwWrk!dDnR>f?G^a-vmxD-7mYRzV7QU*7jEW$9r7u8@>x24}QS#J|zFl
zcSYWcw{IP<96vi#b-v%38P!#<f*)H7?AUnAPF2-a-UC#qT|ejfpS8aI{H95N&sa*R
zXszze6))kHN;%1sS$g`$!;ps7<4p}G8`2b?99ZCbI6zJ#bCO)P(Tb&$z549enVT|{
zg+Dv0@261ktj3vH*u&R;eY)Y51%85)-G26NJ{x7*-|HLA)ZloZ{ZNP9gJe~+S7K(8
zRrMOBhw|o`X+>|-Y3G+;<~!T$&7GZ}ciq1yv$Mv==*#WR;>%M`&SULK5;+||jqR(m
zgn6xV+WSJG+sl?-UJ(<w;jrd`d6hzI?TvY=Y&=z#Nx4X{DSn$!e9%DU@#{A2&o?&h
z<_X%S`!sm5kZ<<0=U>ii|B(AoX)|HdyoKv`&fIU**v=Gx>G@<)bC=!KHgceY^X~7f
z&Ahm%)ou-FFX!ADU&NJ)ZdZQ%*0X<`>*`%EWS>2JTl)X{y;}B=$Zz#B_4AKD+`tlY
z#pTW{=_^~#$m}ed);0B+{_U?nI?^tCoxLRI?(eT@wkF3yi~H<eqpd~xcOJR1GK!|k
z@}>X%cR<JY^clt<m46q@yesa1UwXIw{LXh@qqF90J?FVxFYTG?-*0mK7uIykPc`Cg
z`Ca|vO*Z?=GG^_pnHf?K57*EC*2tsYyuVB+>bBR@O=4QVtj<}B#4g{~%9q-cu<^sI
zxdKslnbh=ar6LR5e<U1pxP8s8rsBxi+Em>=A2v+XJ+826%|!OiwsYA0^3SHeVdSsf
z^(*kU_=*?KK1Vm#r}SG!Ob9NyrD<a`=kD%u<FYpqpl!R&{B|c+uixiYP*Bhq6aW9h
zqaAfJUjH3A??fv0|L^@jt=-*cq2mhuUbmXF58dBAUue#E>B_#wo`vzBqLxg5|Nl2%
zwe9PlFZKslJUc1Zay_@P`U}S_bGs8Cru*o8t}@Vzaf%IBnBN<D_FdM&KLxLyQ~que
zuzCHn?wnX$maM1ydygu!b<WHGzjh2g&n8#-CU=+gvdJ~))$a%eG|rN(ogwjCCBLoa
z+=buy+8c8JK{uzqy1M%KudlCH#_Zf=x533_onZ6z*I&h2L_hNLEc^KRV?>T(>#_>p
zs2eZ5<n;B<^G{%#-PteqE%oDe%Qeh>OXkN%)z5A;+LwOb@sH^X7AcEal}~*Gj4m{o
z8~>kqMU|O%%BF)KuWUZQLeFIyukO*8Unc+f`FFG8`7K*a*!bnnoSA71x_H6g<|B(*
zFk^MfqT7v!Q;xnm_w0%!d)WVgKl(f7PE=NAo^JYrfBk%O-!iiwzmxyYK7Foi=Ev=`
z3amFBeC3~5wcyFKv$HFIKAnE@%$X;k%LY<UPx~bzF1YhpiNr6iFE3vmUtasCa@DRz
z%hUDJnI<g~8=1uR)N^z(+$?)v=_vTa-BD0M_w)9I8Pi^-Rx9#!g$JLUq&m|ek;!|S
zPU5{im7opGs@~H?)=V{t{LDP><opX4E_AawWnY-j?#l4`K5tk0^tbt@bN}W)<m@`$
zFaLaI`aDG|t6A;xbu-S+HrLkZ>d?>;i>dD4d9tMX=g;;#_w{l%N>u*T<IY^Z*CZqR
z@qZ;Att-4C%l%}h=|mm^Eh_%^^Ev1+M@~*oEg!|nvuB^4VVIn8WrbkssVRzzijB+X
zRdFp2l#f+Xy54;*au3t$YnBmP-!+NPm2>ku9azXU19XApzS?Tg9Pn(j+!q%Yv#;O#
zO-fNwQOl1}TwMJ4g@w+U7ZxzSzP=u`6;i9K;{@-5_0NNzMCypmtN&M7_VyO5Lr92-
zeBBR4z3JXZH5))XL1&xgR(!u(4%+Q<dAUD$zjI?r<LT0CPx!Rw9`1ecZBv4y-fSjc
zo~-xYncPd{%MBCPH9y`yqjF;>>*RwD*Dl()t=0Rr@L5*ugCfhaH#ZI*Xk_;E@kx1d
zV&azU>wfcDSRIOrjGWu~Qg3cb1+7PQm#+<JpY7_x)xh-GGiPJO37>m$Pq^j8(iZjH
zKC5k9^z&wFl1sSsPR`04i+2lCBDv3R-9BOJlSf~7#PrSVy(1O&<$pxl^0!Byp4y!k
zu6un|X!p@>@z3Y1-y39HPyj7fDSLl!t$(xhk-Y83MNd4WOtV~ETv#|cIc@&^cnrC-
z>iKM=m&M<Fs#~_)?d~o+diO!mbN**xt25P#m(LYkJ-5c>`ng9T{B@~kLl_U7p6J7`
zx5sst%yRzq$*<F8e}0+lzcFg<_0pTN)@3Qbzr6+RvMzgj%dp^qgWo)xmHw@-U);{$
zzqIb}uQ&Jh&hC{qH!6DK!NkNAA^}>tZU5@ZN@07QS*H(L{<YYa_{lf$PfG>6bKTFk
z*Q_P|9`wwvO|RE`6)wN@-JJ95)>nUe8y^4RMsokX38fn%bZmaV+5G0_X7=u*PJe%W
zm9($3@mn4|pYJ;8x<=5^eH%A!^qFfl_5HrzeO#hiD-1xDTm8Di$H&&hel>oXegENw
zOM&L$-Vr`qw6jzG9-IAmQTO84cF#ZjY1H|f$oTzE@p<0(`4=t(fL0uyxBq|U(o*l8
zZ@1l^VPC(`PI29)B}-J~>i-m0eS32<*x&YO8?SW6tt~4d>u`Ur3SE6{+1J&Es=6uf
z*DzMtzMH*B@z<T~qr1-^{LOCTS{Kc)C+T<WSj4sUk&cJkc)73ay1F8;xs_YosO(Jy
zXmR^=z1SN&i<jGJ{#!U_j?BHP*Sb>Xc|J?Mrh0gIJh<pCe>CC3Uk}i=F6$ju1}%-)
z{ww{|PqmG&XSWwvzjfUuXtizYMTNtj|K&gP*G}6aa`x?)9kD7k|MS;mp1!%IuC%dk
zUjFZIZ*OkPm5$w2ve3Dm545LRSl#c1@Z<bXReNjxe!cGE>bkN1f1O$Wy))7IdqIa)
zcY$_)|2WUW!m=UdV$8cWp-pe+mmixMY@%<Iaq*B_KgW!{cYQy5sr@S1^s(gZqH@s{
z*7~z!<G(Kndf<6&O^n~kNvhm`tY)801MS3~XIuT@^Jma8o;}j$e$S2LmX*G~c5zdx
zx0=t416wjL|9Cb#zoVlg0DK7Tdb#eShs@)BAN}98J?XUi+YR%rVwQBthn;M9Up4zc
z%A)dgd%4`?#i@}q<mcCG>{>Br?~Lhlk6k-&b#YUjMYq@)y^_UVii(P$t^K0fVINNG
z@4v9Xk@?x#*~|U=y)CM~X!P5B;@DgLeOdheI?y?=YQD2T>7_Mf;f@F2-Wn=vhOg6c
zZoBJzwtb)Cik!Vlfq$+rSK1gUzQ5F)to7&ijM_&G)@tR?XCALM{IvFAYgP2B@b%~R
zRDK36+yrf6ILvRKa%Dx}m36Vv^Hm(m%FO=%zW@KnkH`H#zwiHlRy@9@Fg9<Ko|4k_
zu!WZQZrU%CkFT+sCp&p}anbw!)@=_gE91_n>h`>O=52iY=yJ{-N6$!KG`ls`Y|`ZL
z3hV1Lm1p0u+GwlXzBlAW^P~4$b8k;u8@(NLx29G3J0Bk(p3KZl^SA9C*I%c;y0X&6
z)wS@)hs13;HxJFu-?wsshEkA=$c*EEW7kd94!^X~+x*m4$#c4a{Hs{GR|r?vCA{xR
zpJ9}#6uYZLbJOk0UtcmoE1;J7%sh0Ukr{Lmjh&sHxv%y5$jxoj<El6(pFDD7WAe_w
z-)>+12}+&$dt!H&UEK85mgRbN&!@BOt3<hHEa?rl-1UW5?9ANC%PsADtG|D`Q+$47
z_H{i_Dz_?s7qevh_Y)J9_k6pR4T_7v#crUD4tLo=_o8iVV4bu?C2Gqq_VSZzA$j&(
z&GVF_6=dSRnMqnN+wkYOYsl)bx65w-Q}v!^Q1HM3v`p^B#l<Uwm-|h;weQXECj!r5
zYXRQm{QG_X|EHJB=L-o7ANJPYyX5d$Pzk{r?&ik!y<|`S;+u{k@BeYSZ4T9~YOb)A
z`*yaP@%77Io#;&smzH|p-?Ft}_uZOLC)If*janwF`#<_xWhlYpn)YOE`uTZ1a<;QJ
zrJi2(7&Oo<R#RD7Y4amM{@M}sqU`nSH{5W#vg*3EjOE)moz-)?4)Q%aJJT^CVL_ME
zzgBrrLfp4+A83`HYtrl<H)XfpM_@ya*`H!&+jro~j-OZF*~;FZP_Xq)>7z52N6)R~
zzpy4TbeZ2jneL;D7R@m&e&!Qb^U-x%&do*Y(>gjl9?EKH2z6dhNVrq|{==yQ>Nh`S
zS{qH?zB+Y1^Y4`|kL)wu*H_Ore*5|)SJ(c@vu1hS-&Z@+By$qzJR6aBb3#Sj#OmMA
z&foX&Wc}CY*W~64M(vy#^+W2U#kQ$)gXJIeF4X`0rs1E~3D$RezsG?N)~NsYSzgrH
z#f8hXIPmrYcKMnKmzH`L=WqYL@<JWY(${6d+VQFC-*uH0-+9{oe!{$H@#4@4+%^Ax
zK37pw6A~9+p48Fdu`v9Jn^^tU+}mbRTi-f#w$*B0|8hU;<!Y76)7w?_>Uu4WAMbd*
zZg<E8ZW*hR4!2&ZjU_LGY}o=p`zX10DZBSw`17tz;&61}%DWtQO@7FG{i)PhZdQM2
zlPWjpf>N#`#{T1n@0Q>9UG6ux;$f?Jh=8D=rcl=xTZ2<mwT~Yw)#luopXh%_xFV)9
z>!bFBdj6x1JHFMQUiNjhsEGc<yXE(fzWxeIwJRl*l!8)3UQRq3Zz;pK>;C=jvz+QX
zz8+jTqqh0Vr|Bmj7fmqut9k0irAU){v7=q0!GU^7$NOYK2T_;4z81L9iB;6a#igrf
z_M>;Z!d6ecdB5I$xmYPEH}~~C&iZ-!!It@H$Cs>g3ES}D@+DWUuIZ~LU-o=|Z?B|j
z))Y_#Y3PXE;x(0;`Qc&3yxM1q+iEVgt+`^G_1Au4^)HEN^?T0OxAD%jtd$B|Ew{8n
z^KculvXWBMak*-rqXL2xm7XuN;Hh2cIa&Sk`v2T_emzie(z>%SuAuSEW9yriPwV0;
zH`?p(U&3wr@64;Kt3hqGj~_p_w6=n_7cSZm5du0a#pV8gfvHw|y(8Y9d&QYP*J?x4
z-TQWjTE5?39KJqI>%{yM=gz4a8BN-Lzi#&Lce_D%D+;zAU6l6u+1V{SmkTeQW#j(i
zt^d6*a}`rxe}6Ogv(EZGUu>G$`LEskDE{l)TW>!<zWADttzC*rO0LsBrrrCoVuc2%
z%Dgs>Qzpjd&9ZkK?qRhjHp<68zRW2vAT8}36g25upZ&&yhfdGV&p*GT@No!_prGcZ
zyjK&C#?Q5>oD`O8U34p^bJh30;~7TxO>V#YT4lK*!(g7Wcf_~HPN$~ppP!w-@8qki
zt0OlgG=7`VZBhD41hmre(h|=tIX8nq2QVm|T_NP~@zK%9y-AW+yVEzmUj4h!a#LQ{
z+iyGOzM1(?<me{zjfH=;PEFV6Kd^fJzEdlMm;d;1nE$G!-lF{b`&``JL5<~(LyyiL
zN!x5#fA0I1%$UyH`K2oQ+Dkq1*Potv@|@<yD_V1ppPFB<qN4KT&igZI#*@{2AAP+Z
zpMHN|?JJr6pTAy@-<*5f%&kvmWfFLA?Ac=}MjwA3>92e?ab3>OC9*d&U5zy?jrum1
zCRJPA32olFdD-LFuYb2^g6gFSip~{39<~?mzWYk%z{cd`72j^A&$Oxh<eIIfq!c6q
zTJihi%jHg+@0SuznRFgZ*(f^m?Wd|3mGyGxiw*q^OHO3h+pWFseS2GO=FLs2>-YV7
z<$GWK*4AwI%gcNvjZ!+w1O!2sz5b~-tTWBEa$W24_V(6CjlbXTUz6D{XY19>#v8FE
zLvZ56iRQWSKOQvmgAPd24qta9#VGSR_~P7&@xF7drcP9L-&FYc*z8Xm>p_Rwdrnqc
z6SH$t*40%jok4zHbfWd${{MAKN=iT8*Z;2$&fR|d?Xz>%@0XOnzxU<!b$?mN<zjlz
zo}8R~Wp#M}`hCB;($3DhYjXSVwdnkhm;LQGW?WR_VQaQ(0&V2%iUD1O_wxFB`Hvqz
zvaqp<{d{|OOQ!JK+uPGGFY}$E8yzMKxv}xzsmuQMy{DfhJv!3qT4h~aTny@EfVv)g
ztG)(-_9`D*bmHlWbLT+!LjQSR|9`RfbUoKH>-qKnD*NPYPi@J(eB}jbH}wjm6Zbb}
zUDaB&cyVWc|MHZRlU(zDPt%Y0dw*{)=p5~Z3m0Y;fI>1v^2z(Er&GgkY)<DlPCKI!
zbbIQ=iH>)7mHvFOxPQm<dDT}gfX&;W5w4`K-|yBd1!{f&`tmY3W!q~|Q!#7nr;G0L
zD<`_TxG-$?QV2hE{CNAx$?BI^g=&M2>2Yy!S$KchOyl%TNk_Rr)drgw=r{u9?oa(a
zlE!VV+~S`e_uGrT`Yo_**|MPZakk6+=le}o^S!b`Nl7VPGL-43dR)ar))+nU_j|v`
zExgyKKEGxXs9)n4=;HE(Nh)$m{Tt9Fad~%UJUu-<cyroWr|tjd*Z;G8e5^P5?yk}Z
z9Wm1t;3$pBonQA$6VxNizP3g)r1ABIh0dUDAxpid2kkD)HC@utae}o(B5GZi`n-y!
z=)9e+=J#thUs$vG_~VBY-Q^x06^{=I0Bv+->D4(EI(gD0r<4>G(1n|YkB_m|)!wgs
zE}M3K-q}s5rwydO?)7v59fY%Pr=)S(hf~_?K}R0_`SXYCP1U;_8=J*+qc-H-wbBY(
zv!YEH6bn;cKYiFPuco2V@$vC-@e^N*t>%K9R`>VU#~+XTS04oL;eT3XRs4*n^!2r;
zll|=$=HA}Mn)f%Z`mO1<{QKwDL~gzk0bYz!q^uLWYl>a%uMMTI!<?L)xXx68cHQUQ
z-6be5?;jB{L&^*6w7e;d%xn{;O*<AIUmL1$P2Jjh_K6c7$9g24udj=}>HszC=G(GY
z*Vmulka)P_K_h$Kk6)lWkl(z?+4tv@w`m{fbb?ENou16vcrypI>>p&}lqoI8A3ywh
zJ^uN--S783JALldDJ2t=DeL$DlWG8ULF)h2Bqb+bH2`hVZ*~)VI;;9|uQ}*e?wK=Z
zzPY)%ea;-2OP4Nb1TH#a8vSgJWijZAhkg6@nPgw<`SGJ-na@n8)6;aXD!{dKb8{!3
zon?CD`0>e;Cp#x3D0olP*}3fW9Mf#EyzR5!yveEi^U?j>982f8xOq~}E-p{<FP&Hv
zV=O5t3EKIXb9dL&H*em2t}Fll_q%cKtu5Kx-Wa8xD){r`<L5ZgJuSh@{c65m4WHqo
zW;y|!jP7{sE_=Hu?d+@>K5BRJzE|$&=Hz^MWo7X7l%1d*m2SOKPo6!S_kGvZtfQ~L
zPMSNn_hwEQb7x0K44Z7*wCrVWy;7hnO|42_{dst?zOu3sG!{8ar~Kj~*ZPWUo9EA$
z=ePYLkbZ7XASkT`Moei^Qc`N-l{OO)7H&S9X8nEF?YC(U54B!TnR$7+zq-0QJHzCY
zE(r;sJDU?99qIh>>(?sK5f=YI*1dkVe16@c=<RtaX=#s2tTHbx;nWUaw`bMqg57sn
zSXn1dp4>ctzWmkI;px}bL{8BT_v2MiQYvCK*`#sZ_w%!}o*o_#*6n_GC~NDctgBiw
ze8)p$kNx`k`sB%z2h%niCLCb6wmyFS0u3dlPy87v8z)5XDNszlFBkmc+~M2xFAS`O
zgm!%1c3VSEsOIXk-P!NszRB4fnae93WMi_V;B60^@?*PyGdemrN^=ASH4RFZ)hm9l
zy;GiYG+)hXkI%xW_t(sXmuH75DwqowdtD2iogZ8;&$D%PVriPQSJ}c<CtuXINA9iK
zdT0@-fIW5f>Y@~lw}-q|2(;cgWPDSrsiQ;00^9M>xKF4+k?FFrJ-AfExJA(B1?!*g
z-%NRX_{8IUB-Z{sJ8ye6U*`OO1#kCHG^v~ubT_N{Fl(Bz+ktM*W=@;P<sWZ(J$O9*
z$i^9hJJw$_@_ArcJZH`P^vORjuRE<DIk#H#XM%N2^S_(FIyyQ;M5Q8iE;)Tz8^!m0
zo9mJq8}U;u$)<1Bzdm01B<1(x@P$`OBe~D7>YNiZy)(bU;@;lRmjC{-1gX~@cxk&c
zg6YypWsb%3f6D7CmlhiBd-$-Rfb+QhJU_jsnjdcjczxgbc+>O8clWU_|2cnsKRD%z
zFzvK3(^YzVx^l<2cZc6H`G-lJYuUPFzkT-|<I10tr><$(8P)pvUZ#d5N5je}R-XO!
zwu`K$trBnXab|dC_0?d0?fIS4|1uvCXj>VhP<i@sA@e+8_5$^H$EI$WJ;$|hPQ;d;
zJ;pnyo9^HL=-wu2^9OeOq{|O+UAiCLA^l-x#LqvK;k!i_|9?MSylv^XwLfpawC(xa
zBOJ4?JJ0@~=!Pr%zW$LtZm&?E-nF9cUyXiXw4mU`LoSEaU1xqbc=>q$jvw!X(kFj1
zW8sl;_E43M%jP`u?yt=M6844uB}ZPRADa63*qii2v*pfDOm|QGd)<1<nwjqBRvP?S
zlWl49B6-E-$sZbJKdPxGbMdz?&i44FHg%H!*2Cr}y2KlNg4%UXolh0Z{`ZMFD)8A<
zu_gbqIreRzSr@&&`^@_rCwiyU{l6b!arWem^&4#OiPzhom}Na@OX*jQ?`h&O`)gc2
z<#4kdm_2v0{Fiq-M1>!mI(P2>&ZEXtDi_$rwR3uR%<R$3FVI@5`tYyRqZ?D({&zo|
zBlCwP?vuffYf67MYfAX4?UA&$5Hn^v?3^F8EHtyI|4CA{VZ)ajoHKUYxdl`R$h9*s
zzRI=Z<ez`CerN9dX}_hm-$25B+peV@9U^*Ck$Yxdt(_$_ajC`UqwGucZnPd;Z?rJ5
z-EylYtNzR-|DPuc?Z4foxWueZP?TfWU$L0m&nKAW@a%kMaHj3m6sa4m?~nOge7n`H
z!q?mU;kM@0KRlMp4|uDoC)+=GvgqfNXHDWCUVqVgYB$H@=lA&sjz6)JRy)7z$R98J
zN5?xwJsY@>x5az>%~5@DY-6gqjAfJca{+a`znb>)%yqGCL1y2$mE^h(?ab#rZvR7$
z=c(q;jN&`e>4i0qa_0OH5V_0!|JnA$bN762?RNIOT_@A5-@vKA-16h6|Aj9;`${zB
z9W`Gha&%T$Yl}rt-248!xrf6HCx4ka`y<oN)5i-xOq`gKK7-@M9ogW!>Hh+5KeC;@
z;(TXE$Du_ca?`A)GJMFr*1A9|&uIG^^<sCUb5C>%mL}OhI=DBFU+&HWUzKWqHap*>
z4vDMHmlxkZBpN6kFZ@K-YL@-sBk?Cn8!Rmtwjbv1Qz<h0<Mb@t_~V?nIXCa`J2$c3
z_S{4F`$r!d%)QU3Z@E7~W^>&kXHKW;kYgdMR1%anyp5f)_;&K0{XEa#r%m3S$m+LQ
zRDb52e;G38au%dGuW4B6qoTMrU$y*i)ww@grp@WKHfXu$*Lpr_W6n&UV{!ZmOyOKR
zGvoGH^Ut~a&*aiCK83Fix_OoPN_A=<EDjc1@ak8l{NEtCC)#qW3FDck9DDvg+A1n2
zsF_nD@k{N|-Vd{zj`G}7o>Nh7GH>(xpIdfv3;(Qq=`=Ih{n6X65~{PVEjm+we4<f`
zW!!-nC7v7nPad3NJYzSZ{GrX9pQrzSHQ16adz}645$=fkxkt_|eLnq{ZhGH0g(WwQ
zryQF8#y0bZX+_5+J@fQ7eam`le;wu0DeWERy91K0czlwRcbRvVy<qakqaH?mbMD@<
zDo?cht9yLevjC-Criy#plb)Ft>4(`kGaPN}RZ;n(;QnjB(?`za^Y+s(UuUTJn6kS;
zW>4N_*X-;k4|dLXb#dWh?$x<x(v`T4DerpzN8Oh!Y|g29l}pXtoFy2~_bRU`l(eba
z_n_=)yLB@^+kwe(Q-c=IQ~N%7q0F!IMLbguoZ%`eK5%Qn$(wy4oc9F^RNfqUT{K@l
zV0G9bmWsK{>$yvBM*jT&&nowp{`5KW4#)Us8?dxKE-qqSx%k;*Gu9-#&2!JhAD(~q
z`F6hAAK8=A-yKTO^Pl;j??A;94n3b1zDn+B7lmbFhEuv<Hu!Ho&RJ07c`4ZT<SEAJ
zl(V0&-<@Zl_VVoB$qTsZEFRS#iQQyp7j|I%jrE2C!5ja)khri#YDu2@=M$<~tNS#R
zlv;Z-Z)`|B9MX0u``(+IKjbZ4?j|`-pQo`vP__NP^nM2Jwh3N0R<TanoTWXF+1<d!
z?9hHM*{1i|{qsx3J>%1ZSEVca*g4MV`uX_!QqgM+@ikhr3)`=#E=g6n7QH&*)4OL^
zwz#IeEqzp9ZtWCxudn|2H3@#bM?ZcAP05NEUTn#6zsv4P%6+ew$D<RsNf?IbFf8Bp
zz+#rXNbN4ZB`%M<Bx;)sZu3nrdwf9t+4MgT7j0%Vkd1zBePWhH*PSVQXUr-)w)c;5
z(YIyJTlJ>YZPXU)7A&56Ov0A^<UHB-)lu3^S|O?{Dh)5y`yE?fRm$}KoliiB!OO+-
zl+Bm0s#bqHviz0ok7;HF&)P+rqOTpx;5+n${TiR_hSpyXgLS)Q(j%F&?l^8-Hq}sK
zZNzG`V>9X`=HCk4c=sOL?bz>v5jHcYJX~Cm=l{n!w}0=im-24q_4yZ98ehr2+jeVf
z_Uh#B4v)sf4^?~PrY`*O>$-5i@U|jL{;As4Pv<U3Vf9gb;AK&A^s?0IyIqR4{O`7`
zzC2gY)ZY1|!=B8UKka|8AGmq(<mqSfZ7Xv$?zP<A-}_1OkU==_(YYIb`yZBRW9WJM
z=cCH9C-;R`xJczWo1fZ$w>3C_uMgk89r`Oy-(UImz~1)CiYpRw6%QDp4V-g}9{5_f
zXuqBB``YtHZRquvL7%O~MIOFqcWrnRa))!J_!$$?>#gf!OCCm62yH&M_xN14J7Ge~
zYCQKs?o`|UU$IR|Nr@G7gW}F0MqkeA1$vd4%Q<SQOpd<OSUjtv!(-y&iLMj1r*3V!
zA|AGaPf00gNoESDB?vlN61>EU0EbKRZu&-!h$%%v^_ypH+HiZqq*I5rnjx*LFAnS5
zElMA;%<gq9pKxQ|?9;n^XJ_7f(x-HH+S=9Emt|gF_Uej~QqTg4J#p*9QztaN`S9>C
zsKtJ`oqx3iXt~$SbCDt>9NO86^Po@4kMk_%iCcetwULM8<QX$OK0iDA@%#7VZ_A25
zJ@Gtr=n$x3%*M7U@2(Z-?n0Nuz`QU?Nl6jyus4hLb8~YmtE#%X3JD7C<SUVg;##s`
zfx@jVnV;LQ=bA~c-}9-9O9j+d)-d0_e16@m0|yR#j{96;^X1LW;G=?qf;;^{de+>z
z6XWFUyz%zi=W*FrRtRQaUl&^F;^MMSxL3!k@#*W=-pBi7&;Nb%{{8vO{`PARfzLea
zdQ$zUQ~jK5SJ}>(lK1ywL7@{EA#(EiuF}`V|M$OMw_C5W6>>b<Dc=(*Muq?XerE<9
zQv(`JQ@TD=uVA9~q<QnsbqFfo*p_?SH2&Jl%gZD8*V&feuQgYj{PNa20XAk?>#`;J
z_xH^>n+7^*O+!aZ=hXd&MLR40|NUP1`K&qU<Q~)PYkTxgms-d~ti6`KPiFh=vt6Rv
zC(fNqyR)NkOaA@1UOmttw^nmZ-_Os_jS~(q{Q3L0v#)Pi&dp7(WzmlxKb~n<Yjt&X
zxVo9yG%3@p70lpv>4~Fn?(CfW@9*!;>F4EkmB07fTlF=s>uuT27|>0JtlVNB9yIeC
zB^}}D64zhHrwFn_i9y|eUdo3D2Wvi`HQ$nd|KF}T#t{Oksi}tP=VZ)sZX9@Uu(`9d
z^I+1(70qBf?#!9vr@nFH#vTd7rZ!&bOFN6xnV6XJe(h3GRTUH$cXxAROFrJWG<dn+
zD;-E2PTyJl+$iAyLsL^zM_=EwY0-IsfqI}mHE4ihs#a)6Z|~BuwNYB&!&tp8Zc9GS
zmz0!rV|%{*#fulOtPFN{{VANcefHDS(~ax@)ht@P_~-fhf6kpks-Vfqz=$b}o*wBG
z-t+6#>LW*wUR>zR9<?Q-FnHedilv)2Z=PwK&iDHI`sY`}<5LebFj~zI1dTl}{ZMQ*
zchC2G)jcwnliYfxF0Kq#|MT~6=nU;CbLRNO#K^pS`4Th<r5C$vg$CFYt(VtCZU&9O
zaq{vmt^WRQie|8z>rSt2`S<-|Vq`?M!&J=7!rDKB)=&j<_2=H+c5!{YJ)f+VOG$~z
z*6izHf$Km+*`J=Co@t&hcXwB5@{<!2r|3imfsYDv)tqtZ(k0NL)%WZF+k#S$Yqyw0
zubb#IsnaJ<CZ3yP37TMWY-R%u7}eC&K>gjMsIT8&e!q5l;9|GJ-*2~H2KnO8pO6`^
z-~9Oa*k``o+$~#7?p43H-BtE>73kj9LyPW2fNVH#_ghERdzy-s)vTbUUMn@sZ}iDp
zcXW3zj@en%BX9q2S!E8mFX+W6Bros3zwR$+>h13C^6R{dxh49KgKo1tGsm)c$M<{H
zlD1VQiHV7jJ3~)xojPmQsnh!V&%C+08FWkK+1cjTD?3>u{QUSnKRcWJ_0`oqUoLrr
zP7#S%8wNRE?dhu`3mMS(Tk@G1hIa~&%a**k5%^qHOi5Ey6Xb;{Q$+67evjQ%{(c?A
z3o){nm;0wbIM6snE41s~-QBPE9C6CIzAm=r``z+8MW=Op<n8CJSfL^63X15br*iJ?
znOXPu*TgAPT>9l~R|GCTW*XV|`HN!Rzf|*x17RB#eOJYQc#~W$q+Nf@OU=(|waL=J
zsT1ta9hrP|-q+U=yVd4~oiADbrqVrX?wc7O)n=dmw)A^$<C~cMeZ1mx-^u55{=TM{
z8a_?Ze);4FTD_S8sw^KS>_}y`?th-ZvGUB_-qlG555G*d{+T*^^0n_SiNEj0oO=^_
z?9k-U$_1W3Ojmwi|J?Z8T<ht6bFB=kzGQe!*IUb{ucTx?#V<?SrBB}8Z@yh^#@$_}
zp{qg?KRh^iT}Oi{qDlM0rl)J)L~olD8NcSylIMT!%m2${+B{wN@rwAh-R!Om$qYxE
zW8Zk#6;6zwr4hDLb!t=XK?#S54G)Db$5wjy+$rW!7hC1xHfj40_GxcEF8#H$)Kn;N
zug>Xd%LA-8X^G8|xwD~3{(0@uRmwZX-^ln0)kQVcM?BbiTe8<JYD>n!4-XGp7CvGz
z&ARen?e=?z9u}<l+zsk2{MmVNu{-GKZNE7dAVuqM_ipVCldoa4(Z0FC{BQY9&DPM<
zpPYNvfBhrvYb{&$=G%QHzkrPb2gD}}&M9M`cfWN1hZDOqgKX{hu(GiJF?@IP?G4%e
z{ww9?$Q^im<Xu6W!N$jXp3b|ma|ZuDp972ux~{K}zdJU0OM~soozIt978jU@T5q?q
zo|L&T>2Mn8kmmP_*MENACZGJYk>k~L{j#&!d7E!`^!24(Sm1bOp6%>YQ?-rr@7cU%
zQBo>$o)OyAq_3&jd9az?IO#~o>-yQE(`R$VRt5e%vHeW6#Ch4P(E==UZfulFZvUNO
zQm^*%dC}ClvfRrze<~?FHQ~UiDP2cwuFiZEcVPGAa`ydm>}M`i>)-Q(;c3zLjW2eu
zTynj>O?}o>`}&>pjMvQEK0)P=N6f~iPqs-rc2{t1PCL3$`KI$zBe!qr)jM{dSGaYR
zN3rsBP~DBWz5d6tw%xM&;B?&JS?y2mxIHzt$KCq-whPCYcDd=K9y`UdXxFu)9dpli
zf4}&qlkd6yFQrtAVuN{CR#<D~?yOtN&&~aKX8OFuv$IS=!;OdA`Lojn1Sh`sQ|@kU
zIR04C!r}+p-Tw2w>nx{K`QCcy8FgUV5gCQap_kv7<OekFx$ir3U$NojdQJcN?6)uf
zJSb5$HTwJy&DTFoz3=^ge_hh{Q|PU5uY^lIXCqF9Ezg^~=}F8I>3KJH&9GdTwI%QN
ztiHLcH>$m}Hp*M+X=<;$?AAq_UvE5;tU?|y@BLknruyNYd3ENsg{HDr_u8v}%Y-eR
z`e>e9sll|p{q9!x{Wb_1Ok3jF|7*v!e<$|o=r4JnH1WS|2#>)ZSKc`)Ya-k~vx(-P
z-~-*5QIfS0)VT6$d|JALv!`Q8>%kL;u2x&j+|fG4%vV!3;YVylNm0scwo_#*I=<#y
z*NT6wXT$!j_|#KYXJ4%;ADNR47tJz1Ui?7dr5`urzHQxYo0X#<Yo-<#R9u(a#4L63
z_ru>BiD%!ho4TqsnbBw81KX+IZnq2>g81G%bl5a;o^5sT_9^;u<^LW&yDuN07yO)0
zt1>uZ_hW@>^W0^wuciJxO?Y_I`NG!ItgDPWI!^eyw=aCEZ&}FPc;{O{@HGFQZHBYG
zKVNz6dvNBT-79^5h!{`UV$6Ab?!ySi{SO2lJh_>#va{m6-2JVh+y@g19;P?;bn9Jg
zR=)0?aZ>9_=bx`!Io8WJpPRS%MbUq=l?Qu{=N?a3<@xu-iQ<F$%3n>=_lW;=OVM?o
zy)AU6K{C_r-tteUPVDDvNas5?IqG%UsTcEZDk&8SUpleqkL7Cqm(i077*z9eJyh7v
z_}&h_n8q-flOcXHr$La`9=myecYADozI@A+yV-{3T}xUN?e9pQ`?l@L+hW7*`=%-!
zNHUu+DUtm^QCVGD_tZ~YwKl%peKN;>Ugz=3S*DX8S-dhzVlsH{pKkJ#NkAZkX??a{
z@pZXlos*yFeva6n-@Uo}=*umW6aT;UIl{#|iy`3ozF#cyI;(O-EM71$2pj^P3-By0
zNh8&6ckvVXro19|*0(xJN=3n-G3ZI^+MP>_pKpA8!|vJoy{+qdwF|@BkA<ub<W!!y
zX8!gWb1Mwj#GTo-LC<8><%#aM{geO9zJFoiR{iKnN|hfEO8wY1X~*U-GD-XA`M#cD
zEc3vQt#qBswT*Arblh)$eL(Ws)o+J4<O)~tZ+JC*?Ut-76V0rXX8w7{!EnpgtRqTt
z_m8aovz^;?-*BH#d%sM!V$!A?FMFO{<m3HObBuk>OvY!o<&!e57u@MI{di{H$qx%N
zzsTR3xgdVOB5Szft}xLY(^G#EXK(JyO~1&0`lHI!^q02;-mQ7<f2UFC%j;ktStl2l
zJg;4v<%*!Arxq<*Bw?K9^N?evA3LA;jhyH5+e@w9>@Y~)9QRN4C||p5PuLo<$D+E|
zUluf6JvBA{$eV+S<;hzk^CE16gM%yo|NZXi<8$Slf|Anpxo%=lR~bjGiP4FUT(fPz
z?5#Cnx9y|W9sT9+Z(@?P^(DiWj*dG=>aT|~fqEN|&cej~zU>R2<`-?sf2;rI^rxhQ
zoyP1sj;b2*d$~fr*#E!XS@}jYCu(cgS@lCN|2n9!r|miUY*X#ayF1RLu07qi-}LX(
zq79YDwpYyhw!m@5`dKk9E_qo}kyG}cbeei>`@`LD-KX6;eQ{52u$HLxN%i-APcF~r
z|MQnGgIo1L<rHR3y`CA6+U&p0f>hbt{;QaGoO$`*tlG$a?dgRjhOD5re5>nDCc7WM
zBv(iT@ys@D=$cqA9DDlEzq7|TO#WZtRPd<aPgTA%f7x!?=dJv)-z+v*D=Mgbi7Dsl
z-!oh7s`I(oXC4;Jsa~(1Y#*FFeZu39F-cYY-`4275!X3W{4KrxZsEbr^XI0d-aKEe
z`}a-pf>Pl;K|w*;wrTZOs(qd_&9c7UeZ978{{EjWOJ{!%Wb-g{6Z#y(d`<U8i|T=s
zU8R<P?ibgtkbUNz_Q*=DZpZu;)l4((>$R8NntIZ1+uWV^eO34VGq$<7#%5lR@|nMB
zjCNPGi!1)|hrAGW{_*dp%m=yDLr-?vT+nIy%e{Q=?0IXH&-^Q4YqQ#Fx%CJ8r(6Hz
z_e!&GtU1Or?Y+mMu(i=QHY#&|e#>vPC0pd$n!h!luZpcQ(K=gx|IOn2(zP1trJM4P
zTFts&yM3Q$C#dv1CHmR4;K2jFZ9ionKX@g%#wK7=-1%)v1?Nqs9dp0*cIvkRqvpTw
zYdZcfK7VP^8u8D&Hu}unI@MfW@9>fD%P&{n(9pWg6Joaf!#_9uf2pg3xBEqkC9ue>
z`Ta(j>0NE_ZtY(a-&A)LYWEv%IsJA+k??uDi>*O!>i>?{&AoSSmcPBy_N&}?;?-GZ
zU3+kP;o}I2Z%n#U*AsrOGl;UiFq!?bx#5$&PnVQ+94bo?zU_7G!NC)^rWEhoSiR@@
zJm%W}i>vrReYQi3Vs^&pv#<oLiIA8WJ8!vfL}8AbhaJx`FHsFqk)Es54l}KvzCqai
zMyk2+zX1Jz`U`m8ZHbjJzk68MjG19s@!ktg4eNRDG<;V1cwG5~%sY06naBE#HyJ)r
zThD&rDa)Guv+qsxtJySr#iaE2r!#at&z@4dmh$9Lyrw{MrnKss2f=$xrk<Fep5c-k
z6TCkm;L%*0Ynx54=CGZ6bEclJB(r-~go_JH$+<;;n5Ukarmm#a)N$#j<42j_H|uTl
z45p~9N_19IR$dvroX^yI*2#~vK34fpEPm&5@V=f(;s0Oq{10qBqK@qhFSxb2Aa(=a
zyZc{L;uUO97k*<Eu2b!)xclx;<kGk$pV>nnZhby~(av06;ro@%y(=T(VbhzJ%v2;(
zS^uZ^<oE4;sk}-}B5FQS=RDilkDrY7`BlaJ<3~fftTo>f<_S}$^2M>2@kg9t3o?3>
z@Kya#!h_kBr!A&ERd`)t6MyDj>%k|J=d0TKZ8CbZ-#1^=^K4XF-K`1<bp}h7^~Hg^
zV>B#^4W}5*fBPoyO+!Dc5r4kw#gNW#{(sxqvm<ZptmoI}cL`WOXCAYzlF}#b%Nr68
zx2@!1jdpWm6ISzC;5*xlH6hnVCSvyl)zy3xx4h?=_aJ&jiMIjUfysp(M>mTVnA~h!
z>XLbCI^W`wtpeZlW*uFWYQHJR)JEWP=cR*7SIU&kTED#Rc)xXBO7I^^o<v5L%Ay;x
z`)4VCJgyO@+hzSu>-@q<<H#SMjq2jWUFN61Q!}~1{=5J4%zb~{a}pP-%$dE<tT!*)
z;ohfC9iLA{R{z8|y{~B4CMu)zr%tM~ldmhBLpk-a{~F63?D-o{h47zDd{b>NEK<`o
zm-A>wfa>I{-)7ypzJjNX>B_&GH*bOtjca6P&jL5R&OP{AwO7meVA+=!fw02-4nx0i
z`{CA$W&buWwDn{6(}~b6c<GqO&(|)!`fK_7FBSzmbR%@d>-#U~9Z%ktxOdHGzPUHv
z>a8)8c3aMDEtWH_$@IazS+h?4e!t&8DQQs|o1oy%cO{M8G1YF|VY8nq=|}&39W5!f
zF-o5!-%b0P{mEjn*>CgL%wfH~{&m{Mq~L7Ooa%?^5ul+hlym8yth#8~U37F)UvB!*
zZHpsx#OC;^Cnzq-sazGl+GWGnPy5?bWbYr{bo906RjpZWsejkKHnZ3qbYn}B`9s~=
z!C~i{!~_L@GWY7766My2o}9HUS?6D(_SC$5x68LzP1^aZ*o|-YF~j5cXZU{%no<5v
z>AThX{hz<o)x~X_A$hv|abPicK;YEX_jYFS@d_gKv*j6&DrZ#2mw(7ftoJ(iK4>QM
z?!SM&yePbJa?|tscGBJUic5QE_@;fZ*kC;;#=tN#vg};-)yJ0B+1BT{C(XN@cl_Ot
z*FVxZ*ZHhIm%DOF8)yZBifr4o)t4upyH~6B^MM_MQ}d_(mumTur}%eo{&+ItY*;jV
zT-6HAb$7dG?z-A;eOe|pG%xwr_HG9I8ymCqZeLsXm?7@ntG`NfUrkZ?{6I_XoZXq+
ziH&FC3MMU&d+YY?*H53=J)K5VwI}bXx0moLu0JDeqVYKK?4DMYhwhKMM5nwLbiUs9
zuzueozwJ`~E-p{}Gg1QoDRNhoZ_0Qkv#;*GXwv@em;au8zCt#qX${N0-)Z`1t+p(4
zKK{EPfBxaFy_s+3ZRs&8U7T@ak2UX{<>zg^W<9X7v8)oQwGK&sI%}^*!KU}qf6jQ9
zadz%m>x)}UnA-hkuPBTbNp1L*TV+)BMPQfPJQck>ZqI0O<Kv;d*K(sTZ!9*wcBJ;W
zV0)AYWO0Lnx`cA3OlV_IivF`@yqOV8MHnnIk282!ZfIvd=~ngajqlwc$B-9NM~~_M
zKdjEZJ$Uuq+#3wxebb{%Pbad})F|3JMs-yED6N|6m-w*fWX=3Hhpw}TMKhQ=tFCHc
zXJT|Qd)uL2X>Dzuzi`UggAZe?Pw-{9yl;QMxJ54teCEH-t1eOPw!3-HZ|Cn<RaamB
zVC@3){s*5*Uwn<Qj@j??X!ln4&&f+}cODFx{A=Pw7qc9XloZ*f&u^yedK<@PSEhgS
z^dkw&5;OJl^Wt6<#aGN**v@S4`*`lvvZ%7^2`^5i#q9t6cKzhpI%x(X$7erR-I5~u
z%Ee7cGP2#u)cf&?BC%K7?Cbtm1TXVh=+w$}OAs6?dU^&16P(-m1m)z;O;B_OExO1n
zKWK6NT*j}vRblxjg>zSL%yiSSe{bC!AHcb1-Oe+y)e`D{leV8pK6p=f&EGuJqpQpx
z-c0OoPFJ<`Q%>XAap&0P&tJ~|zg|3P>*wv4Z8k}TJ<puPU!m|m=f`KcBk#`Cuc^HB
zZtINqYrNmyntJow-IQ(e=Z`$LIXla<P^R?n(Z%O|)eiVMS{EByThGqCyzJw*+xeeA
z9OmDga#H9m2e^aV#w%^+ldAFK&!0apE-mE-Z3MWrCG*TI({3SEF99*JvcuOjl?#<W
zi|MNC8c+3cS9rR*EqGf|XwJ#iGj@MapYyQ%!m6gmotx7)mz)eTV=;O9a@$moyI)wn
zg=!1tdK`${V_DkNw=w6XpV`vcE5ptmxR%OoIb()j!c>n_zva@Nt*LzDvoYCV>4u`8
z=cjb$INW;0`)9*<mbNXrHPt(`&byndtnrED={)_T?q5+|%&w+SbLAHoRvb{het+`R
zBdd;jM@+70xNIrrHC@|%!uj66|DKfXj@*_bDPRA`aQpo_>&<Cr7x~UMo8hAd>CZ#P
z&oyeEotx|Z|KDFuK0Y-ot64^=r$la)ezlObh)LKS`TxsS?>*AX5BDC;Vw__!M{JFX
zVd4oHt3CfD5-T>u8Su9s=5x~y>z;U}`L{>;;h%x0d6y-u3egE&@M=r#@3Kda9)Ye&
z+qiKfXuY^>tcy#Y#a*lA6F_?<-rm}(Y-To1P}!~EcJ6l2@FEiv6W5(xQo3O|)ArZB
z6?%WH=v}6P*cvef<!<ID503R*P5W`!dQI5v{EZjtx*oqg$8EKXyVye@l*^-b(&WjV
z-QADByu1urZd~%}il=OhiwhS|uZ|d_I%Mz`bOqnHx3@2EN<F=(`o^W=Hq+13Cbda^
zSo|!uJkavS@_;<AurRUW=jT%Y{`$%zXES5Q3<({vZmTWemfzNm#n1gfL$15a_3!R3
z2i@JK<;4gJ;9Vt}+~Rsq8rkI%j&uk@Z(I{>b=_(A?EHLxP`L8&ECCIN>PCmTYAQT>
z{CK8eGF#~CFjX_NX|qhTS3yQkug^dIH0jTekD#42zrMb{ywrO-Q>ESGvyX0UOa=||
zojKzJ+8@-;CmSLnC>R(qMf&}fmBFAD+n{BOcYgMyZC;nXD{j-KO*3q(&1ReBCY_j|
z2wFY(RUCBloMuJY`+L4VK0Le2-!F^ap6BA`wo2prgws#0%HxvO$L*D{tFc(Pa3N%n
z((B;1f`?AOzQ6B3o2G1KHS5EN0@0^hw>PD7UtaFN+^tvY$gyKZuh(veI=!|1^wUSp
z{B{p|&F?8_X?5wv?h=rYSaIl&vwOdss-7O-?z`u96g~#6?}NJikhZPuTt(+LgVI+a
zpyigU!`7DA-4#7?;zYu?H#c)`Zfd<AUq5$W?eA56;Gt#L<MV8*U)<kse|vlW^6KyJ
zTwGmOYD_=)v}j|>Ng*-aC>PK|V!Ij((3YtlP<(k^+*0u|DJdywL)6+YFD^PiH~!b@
zJKGF&27A`^b!T@JK3);GcUKp9-C59=op<x>{(Nw5<CS*XTlE!`^Bo->wa!Gk_sJ;g
z>G8E6cFf7q$-TWT6f~)o7?@|b+<*Qv(4w=``umr}?k?MsclXyaJAQp-<>tuEX&Xyl
zhqduaAKU-$>-t5%|KtjS!hdHBXys@9|9?l0AHTdd+FZ&cqaZkU|LLcX+U4sKK0Z2X
z^XY^#=;-qY4-!gwKoKnRbFZrRGy!4Z=94Ll-23G~tMY#4-rG^AY@BvR!N_RR#^mEc
zTeCt-S-`4yifq4q_PBignF)%{D}tBveZN<&f3!<fyXf}J88c3l->+@I{Wk5-kB?ij
zu6p(AgQqp7zdke57-Z*p`~P!_o}Loj=`d&7v|}G09<Ka!QoS?|Jix7`b84!Wk59_~
ze}7%G+YKaE#P8p?jzJu3r=XzV#QjMjoEWVllqM3f9KA39KNv{zizlT<FfcGMc)I$z
JtaD0e0suv#pJf05

-- 
GitLab


From 5489e3a299d3b241e6ca2397d792e4f571e3d654 Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Tue, 8 Jan 2019 16:45:50 -0800
Subject: [PATCH 0423/1095] [XLA] Use Iota in Cholesky rather than a std::iota
 constant.

NFC intended, just a cleanup noticed in passing.

PiperOrigin-RevId: 228424323
---
 .../compiler/xla/client/lib/cholesky.cc       | 39 +++++++++----------
 1 file changed, 18 insertions(+), 21 deletions(-)

diff --git a/tensorflow/compiler/xla/client/lib/cholesky.cc b/tensorflow/compiler/xla/client/lib/cholesky.cc
index e100d47922..414bd1494c 100644
--- a/tensorflow/compiler/xla/client/lib/cholesky.cc
+++ b/tensorflow/compiler/xla/client/lib/cholesky.cc
@@ -68,29 +68,26 @@ XlaOp CholeskyUnblocked(XlaOp a, PrecisionConfig::Precision precision) {
     auto body_fn =
         [&](XlaOp i, absl::Span<const XlaOp> loop_vars,
             XlaBuilder* body_builder) -> StatusOr<std::vector<XlaOp>> {
-      Shape col_shape;
-      Shape row_shape;
-      for (int64 d : major_dims) {
-        row_shape.add_dimensions(d);
-        col_shape.add_dimensions(d);
-      }
-      row_shape.add_dimensions(1);
-      row_shape.add_dimensions(n);
-      row_shape.set_element_type(a_shape.element_type());
-      auto mask_zeros_row = Zeros(body_builder, row_shape);
-
-      col_shape.add_dimensions(n);
-      col_shape.add_dimensions(1);
-      col_shape.set_element_type(a_shape.element_type());
-      auto mask_zeros_col = Zeros(body_builder, col_shape);
-
-      std::vector<int32> mask_vector(n);
-      std::iota(mask_vector.begin(), mask_vector.end(), 0);
-      auto mask_range = ConstantR1<int32>(body_builder, mask_vector);
+      std::vector<int64> row_shape_dims(major_dims.begin(), major_dims.end());
+      std::vector<int64> col_shape_dims(major_dims.begin(), major_dims.end());
+      row_shape_dims.push_back(1);
+      row_shape_dims.push_back(n);
+      auto mask_zeros_row =
+          Zeros(body_builder,
+                ShapeUtil::MakeShape(a_shape.element_type(), row_shape_dims));
+
+      col_shape_dims.push_back(n);
+      col_shape_dims.push_back(1);
+      auto mask_zeros_col =
+          Zeros(body_builder,
+                ShapeUtil::MakeShape(a_shape.element_type(), col_shape_dims));
+
       auto mask_range_row =
-          Broadcast(Reshape(mask_range, {0}, {1, n}), major_dims);
+          Iota(body_builder, ShapeUtil::MakeShape(S32, row_shape_dims),
+               /*iota_dimension=*/n_dims - 1);
       auto mask_range_col =
-          Broadcast(Reshape(mask_range, {0}, {n, 1}), major_dims);
+          Iota(body_builder, ShapeUtil::MakeShape(S32, col_shape_dims),
+               /*iota_dimension=*/n_dims - 2);
       auto body_a = loop_vars[0];
       auto body_l = loop_vars[1];
 
-- 
GitLab


From 2c7444faf3d23ddb959f887b5589ec53ca431e9d Mon Sep 17 00:00:00 2001
From: Michael Kuperstein <mkuper@google.com>
Date: Tue, 8 Jan 2019 16:53:15 -0800
Subject: [PATCH 0424/1095] [TF:XLA] Move some bridge implementations over to
 the new DS/DUS API

Move some bridge and xla client library op implementations over to
the new span-of-scalars API for DynamicSlice and DynamicUpdateSlice.

PiperOrigin-RevId: 228425409
---
 .../compiler/tf2xla/kernels/image_ops.cc      | 17 ++++---
 .../compiler/tf2xla/kernels/stack_ops.cc      | 12 ++---
 tensorflow/compiler/xla/client/lib/slicing.cc | 49 ++++++++++---------
 .../xla/client/lib/triangular_solve.cc        | 16 +++---
 4 files changed, 49 insertions(+), 45 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/image_ops.cc b/tensorflow/compiler/tf2xla/kernels/image_ops.cc
index 96ddd42e2a..a31b5a2cfd 100644
--- a/tensorflow/compiler/tf2xla/kernels/image_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/image_ops.cc
@@ -13,6 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#include "absl/types/span.h"
 #include "tensorflow/compiler/tf2xla/kernels/gather_op_helpers.h"
 #include "tensorflow/compiler/tf2xla/lib/util.h"
 #include "tensorflow/compiler/tf2xla/xla_helpers.h"
@@ -351,24 +352,26 @@ struct SuppressBodyFn {
     auto num_outputs_so_far = values[1];
     auto iou_mask = values[2];
     auto included_iou = values[3];
-    auto zero_r1 = xla::ConstantR1<int32>(builder, {0});
+    auto zero = xla::ConstantR0<int32>(builder, 0);
     // Determine if current elem is active using a slice.
-    auto row_idx_r1 = xla::Reshape(row_idx, {1});
-    auto active_elem = xla::DynamicSlice(included_iou, row_idx_r1, {1});
+    // TODO(b/118437727): The only reason we need an explicit vector is because
+    // some old GCCs can't deduce the right type for MakeConstSpan, and
+    // providing a single-value initializer list directly uses the wrong
+    // overload. Delete this once the deprecated overload is gone.
+    std::vector<xla::XlaOp> row_idx_vector = {row_idx};
+    auto active_elem = xla::DynamicSlice(included_iou, row_idx_vector, {1});
     active_elem = xla::Reshape(active_elem, {});
     // Increment output count iff current elem is not suppressed.
     num_outputs_so_far = xla::Select(
         active_elem, num_outputs_so_far + xla::ConstantR0<int32>(builder, 1),
         num_outputs_so_far);
     // Slice out the row_idx.
-    auto starts = xla::ConcatInDim(builder, {row_idx_r1, zero_r1}, 0);
-    auto row_iou = xla::DynamicSlice(iou_mask, starts, {1, num_boxes});
+    auto row_iou = xla::DynamicSlice(iou_mask, {row_idx, zero}, {1, num_boxes});
     // Remove the diagonal from consideration. An elem cannot suppress
     // itself.
-    auto update_starts = xla::ConcatInDim(builder, {zero_r1, row_idx_r1}, 0);
     row_iou = xla::DynamicUpdateSlice(
         row_iou, xla::ConstantR2FromArray2D<bool>(builder, {{false}}),
-        update_starts);
+        {zero, row_idx});
     // Create a suppression by inverting polarity.
     row_iou = xla::Reshape(row_iou, {num_boxes});
     auto supp_mask = xla::Not(row_iou);
diff --git a/tensorflow/compiler/tf2xla/kernels/stack_ops.cc b/tensorflow/compiler/tf2xla/kernels/stack_ops.cc
index 02d71a3942..d0c5231e84 100644
--- a/tensorflow/compiler/tf2xla/kernels/stack_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/stack_ops.cc
@@ -146,9 +146,9 @@ class StackPushOp : public XlaOpKernel {
     xla::XlaOp value = ctx->Input(1);
 
     // start_indices of the DynamicUpdateSlice are [index, 0, 0, ..., 0].
-    auto start_indices =
-        xla::Pad(xla::Reshape(index, {1}), xla::ConstantR0<int32>(b, 0),
-                 xla::MakeEdgePaddingConfig({{0, elem_shape.dims()}}));
+    std::vector<xla::XlaOp> start_indices(elem_shape.dims() + 1,
+                                          xla::ConstantR0<int32>(b, 0));
+    start_indices[0] = index;
 
     TensorShape slice_shape = elem_shape;
     slice_shape.InsertDim(0, 1LL);
@@ -202,9 +202,9 @@ class StackPopOp : public XlaOpKernel {
     OP_REQUIRES_OK(ctx, resource->SetValue(xla::Tuple(b, {ta, index})));
 
     // start_indices of the DynamicSlice are [index, 0, 0, ..., 0].
-    auto start_indices =
-        xla::Pad(xla::Reshape(index, {1}), xla::ConstantR0<int32>(b, 0),
-                 xla::MakeEdgePaddingConfig({{0, stack_shape.dims() - 1}}));
+    std::vector<xla::XlaOp> start_indices(stack_shape.dims(),
+                                          xla::ConstantR0<int32>(b, 0));
+    start_indices[0] = index;
 
     auto slice_shape = stack_shape.dim_sizes();
     slice_shape[0] = 1LL;
diff --git a/tensorflow/compiler/xla/client/lib/slicing.cc b/tensorflow/compiler/xla/client/lib/slicing.cc
index 611fffba8d..77145ba7d4 100644
--- a/tensorflow/compiler/xla/client/lib/slicing.cc
+++ b/tensorflow/compiler/xla/client/lib/slicing.cc
@@ -14,6 +14,7 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/compiler/xla/client/lib/slicing.h"
+#include "tensorflow/compiler/xla/client/xla_builder.h"
 
 namespace xla {
 
@@ -51,17 +52,17 @@ XlaOp SliceInMinorDims(XlaOp x, absl::Span<const int64> start,
 XlaOp UpdateSlice(XlaOp x, XlaOp update, absl::Span<const int64> start) {
   XlaBuilder* builder = x.builder();
   return builder->ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
-    // TODO(phawkins): make int64 work on all backends, remove the int32 cast.
-    std::vector<int32> start_as_int32(start.begin(), start.end());
-    auto start_constant = ConstantR1<int32>(builder, start_as_int32);
     TF_ASSIGN_OR_RETURN(Shape shape, builder->GetShape(x));
     const int64 n_dims = shape.rank();
-    TF_ASSIGN_OR_RETURN(Shape start_constant_shape,
-                        builder->GetShape(start_constant));
-    const int64 start_length =
-        ShapeUtil::GetDimension(start_constant_shape, -1);
-    TF_RET_CHECK(start_length == n_dims);
-    return DynamicUpdateSlice(x, update, start_constant);
+    TF_RET_CHECK(start.size() == n_dims);
+
+    // TODO(phawkins): make int64 work on all backends, remove the int32 cast.
+    std::vector<int32> start_as_int32(start.begin(), start.end());
+    std::vector<XlaOp> start_ops(start.size());
+    for (int i = 0; i < start.size(); ++i) {
+      start_ops[i] = ConstantR0(builder, start_as_int32[i]);
+    }
+    return DynamicUpdateSlice(x, update, start_ops);
   });
 }
 
@@ -90,18 +91,17 @@ std::vector<int64> ConcatVectors(absl::Span<const int64> xs,
   return output;
 }
 
-XlaOp PrependZerosInMajorDims(XlaOp x, absl::Span<const XlaOp> starts) {
+StatusOr<std::vector<XlaOp>> PrependZerosInMajorDims(
+    XlaOp x, absl::Span<const XlaOp> starts) {
   XlaBuilder* builder = x.builder();
-  return builder->ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
-    TF_ASSIGN_OR_RETURN(Shape shape, builder->GetShape(x));
-    const int64 n_dims = shape.rank();
-    auto zero = Reshape(ConstantR0<int32>(builder, 0), {1});
-    std::vector<XlaOp> padded_starts(n_dims, zero);
-    for (int i = 0; i < starts.size(); ++i) {
-      padded_starts[n_dims - starts.size() + i] = Reshape(starts[i], {1});
-    }
-    return ConcatInDim(builder, padded_starts, 0);
-  });
+  TF_ASSIGN_OR_RETURN(Shape shape, builder->GetShape(x));
+  const int64 n_dims = shape.rank();
+  auto zero = ConstantR0<int32>(builder, 0);
+  std::vector<XlaOp> padded_starts(n_dims, zero);
+  for (int i = 0; i < starts.size(); ++i) {
+    padded_starts[n_dims - starts.size() + i] = starts[i];
+  }
+  return padded_starts;
 }
 
 }  // namespace
@@ -119,7 +119,7 @@ XlaOp DynamicSliceInMinorDims(XlaOp x, absl::Span<const XlaOp> starts,
                           .subspan(
                               /*pos=*/0,
                               /*len=*/n_dims - sizes.size());
-    auto padded_starts = PrependZerosInMajorDims(x, starts);
+    TF_ASSIGN_OR_RETURN(auto padded_starts, PrependZerosInMajorDims(x, starts));
     auto padded_sizes = ConcatVectors(major_dims, sizes);
     return DynamicSlice(x, padded_starts, padded_sizes);
   });
@@ -127,8 +127,11 @@ XlaOp DynamicSliceInMinorDims(XlaOp x, absl::Span<const XlaOp> starts,
 
 XlaOp DynamicUpdateSliceInMinorDims(XlaOp x, XlaOp update,
                                     absl::Span<const XlaOp> starts) {
-  auto padded_starts = PrependZerosInMajorDims(x, starts);
-  return DynamicUpdateSlice(x, update, padded_starts);
+  XlaBuilder* builder = x.builder();
+  return builder->ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    TF_ASSIGN_OR_RETURN(auto padded_starts, PrependZerosInMajorDims(x, starts));
+    return DynamicUpdateSlice(x, update, padded_starts);
+  });
 }
 
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/client/lib/triangular_solve.cc b/tensorflow/compiler/xla/client/lib/triangular_solve.cc
index 6061e64656..c2f31742e9 100644
--- a/tensorflow/compiler/xla/client/lib/triangular_solve.cc
+++ b/tensorflow/compiler/xla/client/lib/triangular_solve.cc
@@ -165,10 +165,10 @@ XlaOp InvertDiagonalBlocks(XlaOp diag_blocks, bool lower, bool transpose_a,
     // The first or last  diagonal element should be set to 1 instead of -1
     // though, since we never update it
     auto pos_one = Reshape(One(builder, shape.element_type()), {1, 1});
-    auto start_index = (lower) ? 0 : block_size - 1;
-    auto output_block = DynamicUpdateSlice(
-        neg_identity, pos_one,
-        /*start_indices=*/ConstantR1<int>(builder, 2, start_index));
+    auto start_index = ConstantR0<int>(builder, (lower) ? 0 : block_size - 1);
+    auto output_block =
+        DynamicUpdateSlice(neg_identity, pos_one,
+                           /*start_indices=*/{start_index, start_index});
 
     // Broadcast diag([1, -1, -1, ...]) to every block
     XlaOp output = Broadcast(output_block,
@@ -211,12 +211,10 @@ XlaOp InvertDiagonalBlocks(XlaOp diag_blocks, bool lower, bool transpose_a,
       auto body_out = GetTupleElement(input_tuple, 1);
       auto body_input = GetTupleElement(input_tuple, 2);
 
-      auto zero = ConstantR1<int32>(bodyb.get(), 1, 0);
+      auto zero = ConstantR0<int32>(bodyb.get(), 0);
       auto j = (lower) ? i : ScalarLike(i, block_size - 1) - i;
-      auto start_indices =
-          ConcatInDim(bodyb.get(), {zero, Reshape(j, {1}), zero}, 0);
       auto input_row =
-          DynamicSlice(body_input, start_indices,
+          DynamicSlice(body_input, {zero, j, zero},
                        /*slice_sizes=*/{num_blocks, 1, block_size});
 
       // We want -L21 L11^{-1}
@@ -230,7 +228,7 @@ XlaOp InvertDiagonalBlocks(XlaOp diag_blocks, bool lower, bool transpose_a,
       precision_proto.add_operand_precision(precision);
       auto update = -DotGeneral(input_row, body_out, dnums, &precision_proto);
 
-      body_out = DynamicUpdateSlice(body_out, update, start_indices);
+      body_out = DynamicUpdateSlice(body_out, update, {zero, j, zero});
 
       auto next_i = i + ScalarLike(i, 1);
       Tuple(bodyb.get(), {next_i, body_out, body_input});
-- 
GitLab


From c842394896de32c4160fc0f5f94dabf06890476d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 8 Jan 2019 16:57:19 -0800
Subject: [PATCH 0425/1095] Support not fully defined batch dimension in the
 inputs to group_norm.

PiperOrigin-RevId: 228425992
---
 .../layers/python/layers/normalization.py      | 18 +++++++++++++-----
 .../layers/python/layers/normalization_test.py |  9 +++++++++
 2 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/tensorflow/contrib/layers/python/layers/normalization.py b/tensorflow/contrib/layers/python/layers/normalization.py
index 11033a2e9c..76b03ff514 100644
--- a/tensorflow/contrib/layers/python/layers/normalization.py
+++ b/tensorflow/contrib/layers/python/layers/normalization.py
@@ -186,7 +186,7 @@ def group_norm(inputs,
 
   Args:
     inputs: A Tensor with at least 2 dimensions one which is channels. All
-     shape dimensions must be fully defined.
+     shape dimensions except for batch must be fully defined.
     groups: Integer. Divide the channels into this number of groups over which
       normalization statistics are computed. This number must be commensurate
       with the number of channels in `inputs`.
@@ -249,13 +249,21 @@ def group_norm(inputs,
   """
   # TODO(shlens): Support partially defined shapes for the inputs.
   inputs = ops.convert_to_tensor(inputs)
-  original_shape = inputs.shape
 
   if inputs.shape.ndims is None:
     raise ValueError('Inputs %s has undefined rank.' % inputs.name)
   if channels_axis > (inputs.shape.ndims - 1):
     raise ValueError('Axis is out of bounds.')
 
+  # Use dynamic shape for not fully defined dimensions in the inputs.
+  dyanmic_shape = array_ops.shape(inputs)
+  input_shape_list = []
+  for i, dim in enumerate(inputs.shape):
+    if dim.value is None:
+      input_shape_list.append(dyanmic_shape[i])
+    else:
+      input_shape_list.append(dim)
+
   # Standardize the channels_axis to be positive and identify # of channels.
   if channels_axis < 0:
     channels_axis = inputs.shape.ndims + channels_axis
@@ -289,8 +297,8 @@ def group_norm(inputs,
   # Determine axes before channels. Some examples of common image formats:
   #  'NCHW': before = [N], after = [HW]
   #  'NHWC': before = [NHW], after = []
-  axes_before_channels = inputs.shape.as_list()[:channels_axis]
-  axes_after_channels = inputs.shape.as_list()[channels_axis+1:]
+  axes_before_channels = input_shape_list[:channels_axis]
+  axes_after_channels = input_shape_list[channels_axis+1:]
 
   # Manually broadcast the parameters to conform to the number of groups.
   params_shape_broadcast = ([1] * len(axes_before_channels) +
@@ -369,7 +377,7 @@ def group_norm(inputs,
     outputs = inputs * gain + offset
 
     # Collapse the groups into the channel dimension.
-    outputs = array_ops.reshape(outputs, original_shape)
+    outputs = array_ops.reshape(outputs, input_shape_list)
 
     if activation_fn is not None:
       outputs = activation_fn(outputs)
diff --git a/tensorflow/contrib/layers/python/layers/normalization_test.py b/tensorflow/contrib/layers/python/layers/normalization_test.py
index c8d3c91b10..9a85084b23 100644
--- a/tensorflow/contrib/layers/python/layers/normalization_test.py
+++ b/tensorflow/contrib/layers/python/layers/normalization_test.py
@@ -221,6 +221,15 @@ class GroupNormTest(test.TestCase):
       normalization.group_norm(inputs, channels_axis=-1,
                                reduction_axes=[-3, -2])
 
+  def testParamsShapeNotFullyDefinedBatchAxis(self):
+    height, width, groups = 3, 3, 4
+    inputs = array_ops.placeholder(dtypes.float32,
+                                   shape=(None, height, width, 2*groups))
+    output = normalization.group_norm(inputs, channels_axis=-1,
+                                      reduction_axes=[-3, -2], groups=groups)
+    self.assertListEqual([None, height, width, 2 * groups],
+                         output.shape.as_list())
+
   def testCreateOp(self):
     height, width, groups = 3, 3, 4
     images = random_ops.random_uniform((5, height, width, 2*groups), seed=1)
-- 
GitLab


From dde706e1d3561656f0a6d842a79c50cfbe1ad508 Mon Sep 17 00:00:00 2001
From: Siju <siju.samuel@huawei.com>
Date: Wed, 9 Jan 2019 06:48:03 +0530
Subject: [PATCH 0426/1095] Changed result.error_message() to result

---
 tensorflow/core/common_runtime/gpu/gpu_debug_allocator.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/common_runtime/gpu/gpu_debug_allocator.cc b/tensorflow/core/common_runtime/gpu/gpu_debug_allocator.cc
index 2879b00f55..f8c67b60c3 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_debug_allocator.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_debug_allocator.cc
@@ -46,7 +46,7 @@ bool CheckMask(se::StreamExecutor* exec, void* ptr, int64* mask) {
 
   Status result = exec->SynchronousMemcpyD2H(gpu_ptr, MASK_BYTES, tmp);
   if (!result.ok()) {
-    LOG(FATAL) << "Could not copy debug mask, " << result.error_message();
+    LOG(FATAL) << "Could not copy debug mask, " << result;
   }
 
   bool ok = true;
@@ -66,7 +66,7 @@ void InitMask(se::StreamExecutor* exec, void* ptr, int64* mask) {
   se::DeviceMemory<int64> gpu_ptr{se::DeviceMemoryBase{ptr, MASK_BYTES}};
   Status result = exec->SynchronousMemcpyH2D(mask, MASK_BYTES, &gpu_ptr);
   if (!result.ok()) {
-    LOG(FATAL) << "Could not copy debug mask, " << result.error_message();
+    LOG(FATAL) << "Could not copy debug mask, " << result;
   }
 }
 
@@ -176,7 +176,7 @@ void* GPUNanResetAllocator::AllocateRaw(size_t alignment, size_t num_bytes) {
   Status result = stream_exec_->SynchronousMemcpyH2D(&nans[0], req_size,
                                                      &nan_ptr);
   if (!result.ok()) {
-    LOG(ERROR) << "Could not initialize to NaNs, " << result.error_message();
+    LOG(ERROR) << "Could not initialize to NaNs, " << result;
   }
 
   return allocated_ptr;
@@ -192,7 +192,7 @@ void GPUNanResetAllocator::DeallocateRaw(void* ptr) {
     Status result = stream_exec_->SynchronousMemcpyH2D(&nans[0], req_size,
                                                        &nan_ptr);
     if (!result.ok()) {
-      LOG(ERROR) << "Could not initialize to NaNs, " << result.error_message();
+      LOG(ERROR) << "Could not initialize to NaNs, " << result;
     }
   }
 
-- 
GitLab


From fa8ed584884fae406e2398718bbcd6e20498c7a2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 8 Jan 2019 17:24:16 -0800
Subject: [PATCH 0427/1095] Mark certain tests in export_output_test as v1 only
 as export_output.as_signature_def does not support v2 due to its dependent
 method saved_model.build_tensor_info not working in eager. Also raise a
 runtime exception inside build_tensor_info when running in eager.

PiperOrigin-RevId: 228429679
---
 .../saved_model/model_utils/export_test.py    | 255 +++++++++---------
 tensorflow/python/saved_model/utils_impl.py   |   6 +
 tensorflow/python/saved_model/utils_test.py   |   7 +
 3 files changed, 140 insertions(+), 128 deletions(-)

diff --git a/tensorflow/python/saved_model/model_utils/export_test.py b/tensorflow/python/saved_model/model_utils/export_test.py
index 776bfff886..ef512150a2 100644
--- a/tensorflow/python/saved_model/model_utils/export_test.py
+++ b/tensorflow/python/saved_model/model_utils/export_test.py
@@ -22,7 +22,6 @@ import os
 import tempfile
 import time
 
-from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -52,110 +51,110 @@ ops.register_tensor_conversion_function(LabeledTensorMock,
 
 class ExportTest(test_util.TensorFlowTestCase):
 
+  @test_util.deprecated_graph_mode_only
   def test_build_all_signature_defs_without_receiver_alternatives(self):
-    with context.graph_mode():
-      receiver_tensor = array_ops.placeholder(dtypes.string)
-      output_1 = constant_op.constant([1.])
-      output_2 = constant_op.constant(["2"])
-      output_3 = constant_op.constant(["3"])
-      export_outputs = {
-          signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
-              export_output.RegressionOutput(value=output_1),
-          "head-2": export_output.ClassificationOutput(classes=output_2),
-          "head-3": export_output.PredictOutput(outputs={
-              "some_output_3": output_3
-          }),
-      }
-
-      signature_defs = export_utils.build_all_signature_defs(
-          receiver_tensor, export_outputs)
-
-      expected_signature_defs = {
-          "serving_default":
-              signature_def_utils.regression_signature_def(receiver_tensor,
-                                                           output_1),
-          "head-2":
-              signature_def_utils.classification_signature_def(receiver_tensor,
-                                                               output_2, None),
-          "head-3":
-              signature_def_utils.predict_signature_def({
-                  "input": receiver_tensor
-              }, {"some_output_3": output_3})
-      }
-
-      self.assertDictEqual(expected_signature_defs, signature_defs)
+    receiver_tensor = array_ops.placeholder(dtypes.string)
+    output_1 = constant_op.constant([1.])
+    output_2 = constant_op.constant(["2"])
+    output_3 = constant_op.constant(["3"])
+    export_outputs = {
+        signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
+            export_output.RegressionOutput(value=output_1),
+        "head-2": export_output.ClassificationOutput(classes=output_2),
+        "head-3": export_output.PredictOutput(outputs={
+            "some_output_3": output_3
+        }),
+    }
+
+    signature_defs = export_utils.build_all_signature_defs(
+        receiver_tensor, export_outputs)
 
+    expected_signature_defs = {
+        "serving_default":
+            signature_def_utils.regression_signature_def(receiver_tensor,
+                                                         output_1),
+        "head-2":
+            signature_def_utils.classification_signature_def(receiver_tensor,
+                                                             output_2, None),
+        "head-3":
+            signature_def_utils.predict_signature_def({
+                "input": receiver_tensor
+            }, {"some_output_3": output_3})
+    }
+
+    self.assertDictEqual(expected_signature_defs, signature_defs)
+
+  @test_util.deprecated_graph_mode_only
   def test_build_all_signature_defs_with_dict_alternatives(self):
-    with context.graph_mode():
-      receiver_tensor = array_ops.placeholder(dtypes.string)
-      receiver_tensors_alternative_1 = {
-          "foo": array_ops.placeholder(dtypes.int64),
-          "bar": array_ops.sparse_placeholder(dtypes.float32)}
-      receiver_tensors_alternatives = {"other": receiver_tensors_alternative_1}
-      output_1 = constant_op.constant([1.])
-      output_2 = constant_op.constant(["2"])
-      output_3 = constant_op.constant(["3"])
-      export_outputs = {
-          signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
-              export_output.RegressionOutput(value=output_1),
-          "head-2": export_output.ClassificationOutput(classes=output_2),
-          "head-3": export_output.PredictOutput(outputs={
-              "some_output_3": output_3
-          }),
-      }
-
-      signature_defs = export_utils.build_all_signature_defs(
-          receiver_tensor, export_outputs, receiver_tensors_alternatives)
-
-      expected_signature_defs = {
-          "serving_default":
-              signature_def_utils.regression_signature_def(
-                  receiver_tensor,
-                  output_1),
-          "head-2":
-              signature_def_utils.classification_signature_def(
-                  receiver_tensor,
-                  output_2, None),
-          "head-3":
-              signature_def_utils.predict_signature_def(
-                  {"input": receiver_tensor},
-                  {"some_output_3": output_3}),
-          "other:head-3":
-              signature_def_utils.predict_signature_def(
-                  receiver_tensors_alternative_1,
-                  {"some_output_3": output_3})
-
-          # Note that the alternatives 'other:serving_default' and
-          # 'other:head-2' are invalid, because regession and classification
-          # signatures must take a single string input.  Here we verify that
-          # these invalid signatures are not included in the export_utils.
-      }
-
-      self.assertDictEqual(expected_signature_defs, signature_defs)
+    receiver_tensor = array_ops.placeholder(dtypes.string)
+    receiver_tensors_alternative_1 = {
+        "foo": array_ops.placeholder(dtypes.int64),
+        "bar": array_ops.sparse_placeholder(dtypes.float32)}
+    receiver_tensors_alternatives = {"other": receiver_tensors_alternative_1}
+    output_1 = constant_op.constant([1.])
+    output_2 = constant_op.constant(["2"])
+    output_3 = constant_op.constant(["3"])
+    export_outputs = {
+        signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
+            export_output.RegressionOutput(value=output_1),
+        "head-2": export_output.ClassificationOutput(classes=output_2),
+        "head-3": export_output.PredictOutput(outputs={
+            "some_output_3": output_3
+        }),
+    }
 
+    signature_defs = export_utils.build_all_signature_defs(
+        receiver_tensor, export_outputs, receiver_tensors_alternatives)
+
+    expected_signature_defs = {
+        "serving_default":
+            signature_def_utils.regression_signature_def(
+                receiver_tensor,
+                output_1),
+        "head-2":
+            signature_def_utils.classification_signature_def(
+                receiver_tensor,
+                output_2, None),
+        "head-3":
+            signature_def_utils.predict_signature_def(
+                {"input": receiver_tensor},
+                {"some_output_3": output_3}),
+        "other:head-3":
+            signature_def_utils.predict_signature_def(
+                receiver_tensors_alternative_1,
+                {"some_output_3": output_3})
+
+        # Note that the alternatives 'other:serving_default' and
+        # 'other:head-2' are invalid, because regession and classification
+        # signatures must take a single string input.  Here we verify that
+        # these invalid signatures are not included in the export_utils.
+    }
+
+    self.assertDictEqual(expected_signature_defs, signature_defs)
+
+  @test_util.deprecated_graph_mode_only
   def test_build_all_signature_defs_with_single_alternatives(self):
-    with context.graph_mode():
-      receiver_tensor = array_ops.placeholder(dtypes.string)
-      receiver_tensors_alternative_1 = array_ops.placeholder(dtypes.int64)
-      receiver_tensors_alternative_2 = array_ops.sparse_placeholder(
-          dtypes.float32)
-      # Note we are passing single Tensors as values of
-      # receiver_tensors_alternatives, where normally that is a dict.
-      # In this case a dict will be created using the default receiver tensor
-      # name "input".
-      receiver_tensors_alternatives = {"other1": receiver_tensors_alternative_1,
-                                       "other2": receiver_tensors_alternative_2}
-      output_1 = constant_op.constant([1.])
-      output_2 = constant_op.constant(["2"])
-      output_3 = constant_op.constant(["3"])
-      export_outputs = {
-          signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
-              export_output.RegressionOutput(value=output_1),
-          "head-2": export_output.ClassificationOutput(classes=output_2),
-          "head-3": export_output.PredictOutput(outputs={
-              "some_output_3": output_3
-          }),
-      }
+    receiver_tensor = array_ops.placeholder(dtypes.string)
+    receiver_tensors_alternative_1 = array_ops.placeholder(dtypes.int64)
+    receiver_tensors_alternative_2 = array_ops.sparse_placeholder(
+        dtypes.float32)
+    # Note we are passing single Tensors as values of
+    # receiver_tensors_alternatives, where normally that is a dict.
+    # In this case a dict will be created using the default receiver tensor
+    # name "input".
+    receiver_tensors_alternatives = {"other1": receiver_tensors_alternative_1,
+                                     "other2": receiver_tensors_alternative_2}
+    output_1 = constant_op.constant([1.])
+    output_2 = constant_op.constant(["2"])
+    output_3 = constant_op.constant(["3"])
+    export_outputs = {
+        signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
+            export_output.RegressionOutput(value=output_1),
+        "head-2": export_output.ClassificationOutput(classes=output_2),
+        "head-3": export_output.PredictOutput(outputs={
+            "some_output_3": output_3
+        }),
+    }
 
     signature_defs = export_utils.build_all_signature_defs(
         receiver_tensor, export_outputs, receiver_tensors_alternatives)
@@ -222,35 +221,35 @@ class ExportTest(test_util.TensorFlowTestCase):
     self.assertTrue(int(time_1) < int(time_2))
     self.assertTrue(int(time_2) < int(time_3))
 
+  @test_util.deprecated_graph_mode_only
   def test_build_all_signature_defs_serving_only(self):
-    with context.graph_mode():
-      receiver_tensor = {"input": array_ops.placeholder(dtypes.string)}
-      output_1 = constant_op.constant([1.])
-      export_outputs = {
-          signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
-              export_output.PredictOutput(outputs=output_1),
-          "train": export_output.TrainOutput(loss=output_1),
-      }
-
-      signature_defs = export_utils.build_all_signature_defs(
-          receiver_tensor, export_outputs)
-
-      expected_signature_defs = {
-          "serving_default": signature_def_utils.predict_signature_def(
-              receiver_tensor, {"output": output_1})
-      }
-
-      self.assertDictEqual(expected_signature_defs, signature_defs)
-
-      signature_defs = export_utils.build_all_signature_defs(
-          receiver_tensor, export_outputs, serving_only=False)
-
-      expected_signature_defs.update({
-          "train": signature_def_utils.supervised_train_signature_def(
-              receiver_tensor, loss={"loss": output_1})
-      })
-
-      self.assertDictEqual(expected_signature_defs, signature_defs)
+    receiver_tensor = {"input": array_ops.placeholder(dtypes.string)}
+    output_1 = constant_op.constant([1.])
+    export_outputs = {
+        signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
+            export_output.PredictOutput(outputs=output_1),
+        "train": export_output.TrainOutput(loss=output_1),
+    }
+
+    signature_defs = export_utils.build_all_signature_defs(
+        receiver_tensor, export_outputs)
+
+    expected_signature_defs = {
+        "serving_default": signature_def_utils.predict_signature_def(
+            receiver_tensor, {"output": output_1})
+    }
+
+    self.assertDictEqual(expected_signature_defs, signature_defs)
+
+    signature_defs = export_utils.build_all_signature_defs(
+        receiver_tensor, export_outputs, serving_only=False)
+
+    expected_signature_defs.update({
+        "train": signature_def_utils.supervised_train_signature_def(
+            receiver_tensor, loss={"loss": output_1})
+    })
+
+    self.assertDictEqual(expected_signature_defs, signature_defs)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/saved_model/utils_impl.py b/tensorflow/python/saved_model/utils_impl.py
index 5caabe59fe..d43cfdf2b3 100644
--- a/tensorflow/python/saved_model/utils_impl.py
+++ b/tensorflow/python/saved_model/utils_impl.py
@@ -22,6 +22,7 @@ import os
 
 from tensorflow.core.framework import types_pb2
 from tensorflow.core.protobuf import meta_graph_pb2
+from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
@@ -53,7 +54,12 @@ def build_tensor_info(tensor):
 
   Returns:
     A TensorInfo protocol buffer constructed based on the supplied argument.
+
+  Raises:
+    RuntimeError: If eager execution is enabled.
   """
+  if context.executing_eagerly():
+    raise RuntimeError("build_tensor_info is not supported in Eager mode.")
   tensor_info = meta_graph_pb2.TensorInfo(
       dtype=dtypes.as_dtype(tensor.dtype).as_datatype_enum,
       tensor_shape=tensor.get_shape().as_proto())
diff --git a/tensorflow/python/saved_model/utils_test.py b/tensorflow/python/saved_model/utils_test.py
index 2afe8abfd6..1e12de91b8 100644
--- a/tensorflow/python/saved_model/utils_test.py
+++ b/tensorflow/python/saved_model/utils_test.py
@@ -19,6 +19,7 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.core.framework import types_pb2
+from tensorflow.python.eager import context
 from tensorflow.python.eager import function
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -81,6 +82,12 @@ class UtilsTest(test.TestCase):
     self.assertEqual(42, x_tensor_info.tensor_shape.dim[0].size)
     self.assertEqual(69, x_tensor_info.tensor_shape.dim[1].size)
 
+  def testBuildTensorInfoEager(self):
+    x = constant_op.constant(1, name="x")
+    with context.eager_mode(), self.assertRaisesRegexp(
+        RuntimeError, "build_tensor_info is not supported in Eager mode"):
+      utils.build_tensor_info(x)
+
   @test_util.run_v1_only("b/120545219")
   def testGetTensorFromInfoDense(self):
     expected = array_ops.placeholder(dtypes.float32, 1, name="x")
-- 
GitLab


From f9c3fea70f7304c1ee96e8d293dd57623c06fe19 Mon Sep 17 00:00:00 2001
From: Davide Libenzi <dlibenzi@google.com>
Date: Tue, 8 Jan 2019 17:24:41 -0800
Subject: [PATCH 0428/1095] The HLO input/ouput alias config can be setup by
 the user, expecting a must alias semantics, and opportunistically by the
 compiler. Since the upper layer of the code will perform aliasing actions on
 their own, based on the HLO input/output aliasing configuration, we need a
 way for the upper layer to distringuish whether a given aliasing is coming
 from the user, or the compiler. This CL adds a single field to the alias
 data, to differentiate between user and compiler/system aliases.

PiperOrigin-RevId: 228429740
---
 tensorflow/compiler/xla/client/xla_builder.cc |   5 +-
 .../compiler/xla/service/buffer_assignment.cc |   9 +-
 .../compiler/xla/service/copy_insertion.cc    |  18 +--
 .../xla/service/copy_insertion_test.cc        |  21 ++--
 tensorflow/compiler/xla/service/hlo.proto     |  14 +++
 .../xla/service/hlo_alias_analysis.cc         |   9 +-
 .../xla/service/hlo_alias_analysis_test.cc    |  24 ++--
 .../service/hlo_input_output_alias_config.cc  | 119 ++++++++++--------
 .../service/hlo_input_output_alias_config.h   |  51 ++++++--
 .../hlo_input_output_alias_config_test.cc     |  60 +++++----
 10 files changed, 209 insertions(+), 121 deletions(-)

diff --git a/tensorflow/compiler/xla/client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_builder.cc
index 328abbc7b9..7a04c0ec69 100644
--- a/tensorflow/compiler/xla/client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_builder.cc
@@ -350,8 +350,9 @@ StatusOr<XlaComputation> XlaBuilder::Build(int64 root_id) {
                              alias.param_number,
                              alias.param_index.ToString().c_str());
     }
-    TF_RETURN_IF_ERROR(config.SetUpAlias(alias.output_index, alias.param_number,
-                                         alias.param_index));
+    TF_RETURN_IF_ERROR(config.SetUpAlias(
+        alias.output_index, alias.param_number, alias.param_index,
+        HloInputOutputAliasConfig::AliasKind::kUserAlias));
   }
   *module->mutable_input_output_alias() = config.ToProto();
   return Status::OK();
diff --git a/tensorflow/compiler/xla/service/buffer_assignment.cc b/tensorflow/compiler/xla/service/buffer_assignment.cc
index c3285f516b..d07615b828 100644
--- a/tensorflow/compiler/xla/service/buffer_assignment.cc
+++ b/tensorflow/compiler/xla/service/buffer_assignment.cc
@@ -1530,15 +1530,16 @@ void BufferAssigner::BuildColocatedBufferSets(
   VLOG(4) << "Input/Output Alias Config: ";
   VLOG(4) << module->input_output_alias_config();
   module->input_output_alias_config().ForEachAlias(
-      [&](const ShapeIndex& output_index, int64 param_number,
-          const ShapeIndex& param_index) {
+      [&](const ShapeIndex& output_index,
+          const HloInputOutputAliasConfig::Alias& alias) {
         std::vector<const LogicalBuffer*> colocated_set;
         AddBufferToColocatedSet(module->entry_computation()->root_instruction(),
                                 output_index, points_to_analysis,
                                 &colocated_set);
         AddBufferToColocatedSet(
-            module->entry_computation()->parameter_instruction(param_number),
-            param_index, points_to_analysis, &colocated_set);
+            module->entry_computation()->parameter_instruction(
+                alias.parameter_number),
+            alias.parameter_index, points_to_analysis, &colocated_set);
         AddSetToColocatedBufferSets(colocated_set, colocated_buffer_sets);
       });
 
diff --git a/tensorflow/compiler/xla/service/copy_insertion.cc b/tensorflow/compiler/xla/service/copy_insertion.cc
index 2b837901f0..5e26a63ceb 100644
--- a/tensorflow/compiler/xla/service/copy_insertion.cc
+++ b/tensorflow/compiler/xla/service/copy_insertion.cc
@@ -349,11 +349,12 @@ Status AddCopiesForAliasedInputOutputs(HloModule* module) {
     ShapeTree<bool> param_indices_to_copy(param->shape());
 
     module->input_output_alias_config().ForEachAlias(
-        [&](const ShapeIndex& output_index, int64 param_number,
-            const ShapeIndex& param_index) {
-          if (param_number == param->parameter_number()) {
+        [&](const ShapeIndex& output_index,
+            const HloInputOutputAliasConfig::Alias& alias) {
+          if (alias.parameter_number == param->parameter_number()) {
             param_has_alias = true;
-            *(param_indices_to_copy.mutable_element(param_index)) = true;
+            *(param_indices_to_copy.mutable_element(alias.parameter_index)) =
+                true;
             *(output_indices_to_copy.mutable_element(output_index)) = true;
           }
         });
@@ -395,13 +396,14 @@ Status AddCopiesForAliasedInputOutputs(HloModule* module) {
 
   // Add control dependencies between the input/output copies.
   TF_RETURN_IF_ERROR(module->input_output_alias_config().ForEachAliasWithStatus(
-      [&](const ShapeIndex& output_index, int64 param_number,
-          const ShapeIndex& input_index) -> Status {
-        if (!copied_parameters[param_number]) {
+      [&](const ShapeIndex& output_index,
+          const HloInputOutputAliasConfig::Alias& alias) -> Status {
+        if (!copied_parameters[alias.parameter_number]) {
           return Status::OK();
         }
         HloInstruction* from =
-            copied_parameters[param_number]->element(input_index);
+            copied_parameters[alias.parameter_number]->element(
+                alias.parameter_index);
         HloInstruction* to = output_copy_tree.element(output_index);
 
         TF_RET_CHECK(from != nullptr);
diff --git a/tensorflow/compiler/xla/service/copy_insertion_test.cc b/tensorflow/compiler/xla/service/copy_insertion_test.cc
index e4e9d7ba05..4d4074943e 100644
--- a/tensorflow/compiler/xla/service/copy_insertion_test.cc
+++ b/tensorflow/compiler/xla/service/copy_insertion_test.cc
@@ -1376,9 +1376,11 @@ TEST_F(CopyInsertionTest, CrossingParameters) {
   builder.AddInstruction(HloInstruction::CreateTuple({gte1, gte0}));
   module->AddEntryComputation(builder.Build());
   ASSERT_IS_OK(module->input_output_alias_config().SetUpAlias(
-      /*output_index=*/{0}, /*param_number=*/0, /*param_index=*/{0}));
+      /*output_index=*/{0}, /*param_number=*/0, /*param_index=*/{0},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
   ASSERT_IS_OK(module->input_output_alias_config().SetUpAlias(
-      /*output_index=*/{1}, /*param_number=*/0, /*param_index=*/{1}));
+      /*output_index=*/{1}, /*param_number=*/0, /*param_index=*/{1},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
   InsertCopies(module.get());
 
   EXPECT_EQ(CountCopies(*module), 4);
@@ -1409,9 +1411,11 @@ TEST_F(CopyInsertionTest, ParametersAliasing) {
   builder.AddInstruction(HloInstruction::CreateTuple({gte0, gte1}));
   module->AddEntryComputation(builder.Build());
   ASSERT_IS_OK(module->input_output_alias_config().SetUpAlias(
-      /*output_index=*/{0}, /*param_number=*/0, /*param_index=*/{0}));
+      /*output_index=*/{0}, /*param_number=*/0, /*param_index=*/{0},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
   ASSERT_IS_OK(module->input_output_alias_config().SetUpAlias(
-      /*output_index=*/{1}, /*param_number=*/0, /*param_index=*/{1}));
+      /*output_index=*/{1}, /*param_number=*/0, /*param_index=*/{1},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
   InsertCopies(module.get());
 
   EXPECT_EQ(CountCopies(*module), 0);
@@ -1475,7 +1479,8 @@ TEST_F(CopyInsertionTest, ParameterWithPartialAliasing) {
   builder.AddInstruction(HloInstruction::CreateTuple({gte0, gte1}));
   module->AddEntryComputation(builder.Build());
   ASSERT_IS_OK(module->input_output_alias_config().SetUpAlias(
-      /*output_index=*/{0}, /*param_number=*/0, /*param_index=*/{0}));
+      /*output_index=*/{0}, /*param_number=*/0, /*param_index=*/{0},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
   InsertCopies(module.get());
 
   EXPECT_THAT(module->entry_computation()->root_instruction(),
@@ -1516,7 +1521,8 @@ TEST_F(CopyInsertionTest, ParameterAndParallelOpsWithPartialAliasing) {
   builder.AddInstruction(HloInstruction::CreateTuple({negate0, negate1}));
   module->AddEntryComputation(builder.Build());
   ASSERT_IS_OK(module->input_output_alias_config().SetUpAlias(
-      /*output_index=*/{0}, /*param_number=*/0, /*param_index=*/{0}));
+      /*output_index=*/{0}, /*param_number=*/0, /*param_index=*/{0},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
   InsertCopies(module.get());
 
   EXPECT_EQ(CountCopies(*module), 0);
@@ -1557,7 +1563,8 @@ TEST_F(CopyInsertionTest, ParameterAndOpsWithPartialAliasing) {
   builder.AddInstruction(HloInstruction::CreateTuple({add, negate1}));
   module->AddEntryComputation(builder.Build());
   ASSERT_IS_OK(module->input_output_alias_config().SetUpAlias(
-      /*output_index=*/{0}, /*param_number=*/0, /*param_index=*/{0}));
+      /*output_index=*/{0}, /*param_number=*/0, /*param_index=*/{0},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
   InsertCopies(module.get());
 
   EXPECT_EQ(CountCopies(*module), 0);
diff --git a/tensorflow/compiler/xla/service/hlo.proto b/tensorflow/compiler/xla/service/hlo.proto
index 9b50f1ca5b..263b42a29d 100644
--- a/tensorflow/compiler/xla/service/hlo.proto
+++ b/tensorflow/compiler/xla/service/hlo.proto
@@ -229,6 +229,18 @@ message HloScheduleProto {
 }
 
 message HloInputOutputAliasProto {
+  enum Kind {
+    // Define a UNDEFINED_ALIAS equal to zero to get around the default-0 proto3
+    // behavior and missing has_*() APIs.
+    UNDEFINED_ALIAS = 0;
+    // An alias setup by the user as must alias. A use setting USER_ALIAS is
+    // expecting the designed output to be dropped over the given input
+    // parameter number+index.
+    USER_ALIAS = 1;
+    // An alias setup by the compiler as part of its optimizations.
+    SYSTEM_ALIAS = 2;
+  }
+
   // The following proto describes a pair of aliased an input
   // (described by parameter number and a ShapeIndex of the parameter)
   // and an output (described by a ShapeIndex of the root
@@ -249,6 +261,8 @@ message HloInputOutputAliasProto {
     int64 parameter_number = 2;
     // ShapeIndex of the parameter instruction.
     repeated int64 parameter_shape_index = 3;
+    // The kind of alias to be setup.
+    Kind kind = 4;
   }
 
   repeated AliasEntryProto entries = 1;
diff --git a/tensorflow/compiler/xla/service/hlo_alias_analysis.cc b/tensorflow/compiler/xla/service/hlo_alias_analysis.cc
index 4c46ea595e..e511f1951c 100644
--- a/tensorflow/compiler/xla/service/hlo_alias_analysis.cc
+++ b/tensorflow/compiler/xla/service/hlo_alias_analysis.cc
@@ -176,13 +176,12 @@ class BufferValueMap {
       const HloValue& value, std::vector<BufferNumber>* aliased_buffers) {
     // Get parameter value from an aliased_input object.
     const auto get_parameter_value =
-        [this](const std::pair<int64, ShapeIndex>& aliased_input)
+        [this](const HloInputOutputAliasConfig::Alias& aliased_input)
         -> const HloValue& {
-      int64 param_number = aliased_input.first;
-      const ShapeIndex& param_index = aliased_input.second;
       return dataflow_.GetUniqueValueAt(
-          module_->entry_computation()->parameter_instruction(param_number),
-          param_index);
+          module_->entry_computation()->parameter_instruction(
+              aliased_input.parameter_number),
+          aliased_input.parameter_index);
     };
 
     // If the value shows up in a root instruction, alias it with parameter
diff --git a/tensorflow/compiler/xla/service/hlo_alias_analysis_test.cc b/tensorflow/compiler/xla/service/hlo_alias_analysis_test.cc
index 7e6150e941..b6dbf07959 100644
--- a/tensorflow/compiler/xla/service/hlo_alias_analysis_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_alias_analysis_test.cc
@@ -238,13 +238,16 @@ TEST_F(HloAliasAnalysisTest, ParametersWithAliasing) {
       builder.AddInstruction(HloInstruction::CreateTuple({negate0, negate1}));
   module_->AddEntryComputation(builder.Build());
   TF_ASSERT_OK(module_->input_output_alias_config().SetUpAlias(
-      /*output_index=*/{0}, /*param_number=*/0, /*param_index=*/{0}));
+      /*output_index=*/{0}, /*param_number=*/0, /*param_index=*/{0},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
   TF_ASSERT_OK(module_->input_output_alias_config().SetUpAlias(
-      /*output_index=*/{1}, /*param_number=*/0, /*param_index=*/{1}));
+      /*output_index=*/{1}, /*param_number=*/0, /*param_index=*/{1},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
 
   // Cannot alias an output twice.
   ASSERT_IS_NOT_OK(module_->input_output_alias_config().SetUpAlias(
-      /*output_index=*/{1}, /*param_number=*/0, /*param_index=*/{0}));
+      /*output_index=*/{1}, /*param_number=*/0, /*param_index=*/{0},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
 
   const HloAliasAnalysis& analysis = RunAnalysis();
 
@@ -279,13 +282,16 @@ TEST_F(HloAliasAnalysisTest, ParametersWithCrossAliasing) {
       builder.AddInstruction(HloInstruction::CreateTuple({gte0, gte1}));
   module_->AddEntryComputation(builder.Build());
   TF_ASSERT_OK(module_->input_output_alias_config().SetUpAlias(
-      /*output_index=*/{0}, /*param_number=*/0, /*param_index=*/{1}));
+      /*output_index=*/{0}, /*param_number=*/0, /*param_index=*/{1},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
   TF_ASSERT_OK(module_->input_output_alias_config().SetUpAlias(
-      /*output_index=*/{1}, /*param_number=*/0, /*param_index=*/{0}));
+      /*output_index=*/{1}, /*param_number=*/0, /*param_index=*/{0},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
 
   // Cannot alias an output twice.
   ASSERT_IS_NOT_OK(module_->input_output_alias_config().SetUpAlias(
-      /*output_index=*/{1}, /*param_number=*/0, /*param_index=*/{1}));
+      /*output_index=*/{1}, /*param_number=*/0, /*param_index=*/{1},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
 
   const HloAliasAnalysis& analysis = RunAnalysis();
 
@@ -365,9 +371,11 @@ TEST_F(HloAliasAnalysisTest, InputOutputAliasingWithWhile) {
       builder.AddInstruction(HloInstruction::CreateTuple({negate_1, negate_2}));
   module_->AddEntryComputation(builder.Build());
   TF_ASSERT_OK(module_->input_output_alias_config().SetUpAlias(
-      /*output_index=*/{0}, /*param_number=*/0, /*param_index=*/{0}));
+      /*output_index=*/{0}, /*param_number=*/0, /*param_index=*/{0},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
   TF_ASSERT_OK(module_->input_output_alias_config().SetUpAlias(
-      /*output_index=*/{1}, /*param_number=*/0, /*param_index=*/{1}));
+      /*output_index=*/{1}, /*param_number=*/0, /*param_index=*/{1},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
 
   const HloAliasAnalysis& analysis = RunAnalysis();
 
diff --git a/tensorflow/compiler/xla/service/hlo_input_output_alias_config.cc b/tensorflow/compiler/xla/service/hlo_input_output_alias_config.cc
index 70d4df5d1c..b01c00121b 100644
--- a/tensorflow/compiler/xla/service/hlo_input_output_alias_config.cc
+++ b/tensorflow/compiler/xla/service/hlo_input_output_alias_config.cc
@@ -20,28 +20,31 @@ namespace xla {
 
 bool HloInputOutputAliasConfig::OutputHasAlias(
     const ShapeIndex& output_index) const {
-  return aliased_output_indices_.count(output_index) > 0;
+  return alias_.element(output_index).has_value();
 }
 
 Status HloInputOutputAliasConfig::SetUpAlias(const ShapeIndex& output_index,
                                              int64 param_number,
-                                             const ShapeIndex& param_index) {
-  TF_RET_CHECK(!OutputHasAlias(output_index))
-      << "Output index " << output_index << " already has an alias setup";
+                                             const ShapeIndex& param_index,
+                                             AliasKind kind) {
+  TF_RET_CHECK(kind == AliasKind::kUserAlias || kind == AliasKind::kSystemAlias)
+      << kind;
   TF_RET_CHECK(ShapeUtil::IndexIsValid(alias_.shape(), output_index))
       << absl::StrCat("Tring to set up alias at ", output_index.ToString(),
                       " which is an invalid index for shape ",
                       ShapeUtil::HumanString(alias_.shape()));
+  TF_RET_CHECK(param_number >= 0) << param_number;
+  TF_RET_CHECK(!OutputHasAlias(output_index))
+      << "Output index " << output_index << " already has an alias setup";
   // Output can't be aliased with multiple parameters.
   TF_RET_CHECK(!alias_.element(output_index)) << absl::StrFormat(
       "Trying to set up output alias for param %lld at %s but failed: output "
       "index %s is already aliased with param %lld at %s",
       param_number, param_index.ToString(), output_index.ToString(),
-      alias_.element(output_index)->first,
-      alias_.element(output_index)->second.ToString());
+      alias_.element(output_index)->parameter_number,
+      alias_.element(output_index)->parameter_index.ToString());
   (*alias_.mutable_element(output_index)) =
-      std::make_pair(param_number, param_index);
-  aliased_output_indices_.insert(output_index);
+      Alias(kind, param_number, param_index);
   VLOG(4) << "Set up alias between output index " << output_index.ToString()
           << " and parameter " << param_index << " at index "
           << param_index.ToString();
@@ -51,15 +54,24 @@ Status HloInputOutputAliasConfig::SetUpAlias(const ShapeIndex& output_index,
 HloInputOutputAliasProto HloInputOutputAliasConfig::ToProto() const {
   HloInputOutputAliasProto result;
   alias_.ForEachElement(
-      [&](const ShapeIndex& index,
-          const absl::optional<std::pair<int64, ShapeIndex>>& data) {
+      [&](const ShapeIndex& index, const absl::optional<Alias>& data) {
         if (data) {
           HloInputOutputAliasProto::AliasEntryProto entry;
+          switch (data->kind) {
+            case AliasKind::kUserAlias:
+              entry.set_kind(HloInputOutputAliasProto::USER_ALIAS);
+              break;
+            case AliasKind::kSystemAlias:
+              entry.set_kind(HloInputOutputAliasProto::SYSTEM_ALIAS);
+              break;
+            default:
+              LOG(FATAL) << "Unknown alias kind " << data->kind;
+          }
           for (int64 i : index) {
             entry.add_output_shape_index(i);
           }
-          entry.set_parameter_number(data->first);
-          for (int64 i : data->second) {
+          entry.set_parameter_number(data->parameter_number);
+          for (int64 i : data->parameter_index) {
             entry.add_parameter_shape_index(i);
           }
           result.add_entries()->Swap(&entry);
@@ -75,14 +87,18 @@ StatusOr<HloInputOutputAliasConfig> HloInputOutputAliasConfig::CreateFromProto(
        proto.entries()) {
     ShapeIndex output_index(entry.output_shape_index().begin(),
                             entry.output_shape_index().end());
-
     int64 param_number = entry.parameter_number();
     ShapeIndex param_index(entry.parameter_shape_index().begin(),
                            entry.parameter_shape_index().end());
+    // Handle backward compatibility with existing protos, which only knew of
+    // system aliases.
+    AliasKind kind = AliasKind::kSystemAlias;
+    if (entry.kind() == HloInputOutputAliasProto::USER_ALIAS) {
+      kind = AliasKind::kUserAlias;
+    }
     TF_RETURN_IF_ERROR(
-        result.SetUpAlias(output_index, param_number, param_index));
+        result.SetUpAlias(output_index, param_number, param_index, kind));
   }
-
   return result;
 }
 
@@ -90,45 +106,44 @@ string HloInputOutputAliasConfig::ToString() const {
   std::vector<string> pieces;
   pieces.push_back("HloInputOutputAliasConfig");
 
-  ForEachAlias([&](const ShapeIndex& output_index, int64 param_number,
-                   const ShapeIndex& param_index) {
+  ForEachAlias([&](const ShapeIndex& output_index, const Alias& alias) {
+    const char* kind = alias.kind == AliasKind::kUserAlias ? "USER" : "SYSTEM";
     pieces.push_back(absl::StrFormat(
-        "  OutputIndex %s is aliased with parameter %lld at %s:",
-        output_index.ToString(), param_number, param_index.ToString()));
+        "  OutputIndex %s is aliased (kind=%s) with parameter %lld at %s:",
+        output_index.ToString(), kind, alias.parameter_number,
+        alias.parameter_index.ToString()));
   });
-
   return absl::StrJoin(pieces, "\n");
 }
 
-bool HloInputOutputAliasConfig::ParameterHasAlias(
+HloInputOutputAliasConfig::AliasKind
+HloInputOutputAliasConfig::ParameterAliasKind(
     int64 param_number, const ShapeIndex& param_index) const {
-  bool output = false;
+  AliasKind kind = AliasKind::kNoAlias;
   alias_.ForEachElement(
-      [&](const xla::ShapeIndex&,
-          absl::optional<std::pair<int64, ShapeIndex>> alias) {
-        if (alias && alias->first == param_number &&
-            alias->second == param_index) {
-          output = true;
+      [&](const xla::ShapeIndex&, absl::optional<Alias> alias) {
+        if (alias && alias->parameter_number == param_number &&
+            alias->parameter_index == param_index) {
+          kind = alias->kind;
         }
       });
-  return output;
+  return kind;
 }
 
 absl::optional<ShapeIndex> HloInputOutputAliasConfig::GetAliasedOutput(
     int64 param_number, const ShapeIndex& param_index) const {
   absl::optional<ShapeIndex> output;
   alias_.ForEachElement(
-      [&](const xla::ShapeIndex& output_index,
-          absl::optional<std::pair<int64, ShapeIndex>> alias) {
-        if (alias && alias->first == param_number &&
-            alias->second == param_index) {
+      [&](const xla::ShapeIndex& output_index, absl::optional<Alias> alias) {
+        if (alias && alias->parameter_number == param_number &&
+            alias->parameter_index == param_index) {
           output = output_index;
         }
       });
   return output;
 }
 
-absl::optional<std::pair<int64, ShapeIndex>>
+absl::optional<HloInputOutputAliasConfig::Alias>
 HloInputOutputAliasConfig::GetAliasedParameter(
     const ShapeIndex& output_index) const {
   CHECK(ShapeUtil::IndexIsValid(alias_.shape(), output_index));
@@ -137,10 +152,9 @@ HloInputOutputAliasConfig::GetAliasedParameter(
 
 void HloInputOutputAliasConfig::ForEachAlias(AliasFn fn) const {
   alias_.ForEachElement(
-      [&](const ShapeIndex& output_index,
-          absl::optional<std::pair<int64, ShapeIndex>> aliased) {
+      [&](const ShapeIndex& output_index, absl::optional<Alias> aliased) {
         if (aliased) {
-          fn(output_index, aliased->first, aliased->second);
+          fn(output_index, *aliased);
         }
       });
 }
@@ -148,10 +162,9 @@ void HloInputOutputAliasConfig::ForEachAlias(AliasFn fn) const {
 Status HloInputOutputAliasConfig::ForEachAliasWithStatus(
     AliasFnWithStatus fn) const {
   return alias_.ForEachElementWithStatus(
-      [&](const ShapeIndex& output_index,
-          absl::optional<std::pair<int64, ShapeIndex>> aliased) {
+      [&](const ShapeIndex& output_index, absl::optional<Alias> aliased) {
         if (aliased) {
-          TF_RETURN_IF_ERROR(fn(output_index, aliased->first, aliased->second));
+          TF_RETURN_IF_ERROR(fn(output_index, *aliased));
         }
         return Status::OK();
       });
@@ -167,20 +180,19 @@ Status HloInputOutputAliasConfig::Verify(
     param_has_seen.emplace_back(param->shape());
   }
   return ForEachAliasWithStatus([&](const ShapeIndex& output_index,
-                                    int64 param_number,
-                                    const ShapeIndex& param_index) -> Status {
+                                    const Alias& alias) -> Status {
     const HloInstruction* root = entry->root_instruction();
 
-    TF_RET_CHECK(0 <= param_number);
-    TF_RET_CHECK(entry->num_parameters() > param_number);
+    TF_RET_CHECK(0 <= alias.parameter_number);
+    TF_RET_CHECK(entry->num_parameters() > alias.parameter_number);
     const Shape& param_shape =
-        entry->parameter_instruction(param_number)->shape();
+        entry->parameter_instruction(alias.parameter_number)->shape();
     const Shape& output_shape = root->shape();
-    TF_RET_CHECK(ShapeUtil::IndexIsValid(param_shape, param_index));
+    TF_RET_CHECK(ShapeUtil::IndexIsValid(param_shape, alias.parameter_index));
     TF_RET_CHECK(ShapeUtil::IndexIsValid(output_shape, output_index));
 
     const Shape& param_subshape =
-        ShapeUtil::GetSubshape(param_shape, param_index);
+        ShapeUtil::GetSubshape(param_shape, alias.parameter_index);
     const Shape& output_subshape =
         ShapeUtil::GetSubshape(output_shape, output_index);
     TF_RET_CHECK(LayoutUtil::IsDenseArray(param_subshape));
@@ -191,19 +203,20 @@ Status HloInputOutputAliasConfig::Verify(
           "Expected aliased input %lld at index %s and output at index %s to "
           "have the same size. Input sub-shape is %s with size %lld, output "
           "sub-shape is %s with size %lld",
-          param_number, param_index.ToString(), output_index.ToString(),
+          alias.parameter_number, alias.parameter_index.ToString(),
+          output_index.ToString(),
           ShapeUtil::HumanStringWithLayout(param_subshape),
           size_func(param_subshape),
           ShapeUtil::HumanStringWithLayout(output_subshape),
           size_func(output_subshape));
     }
 
-    // Check each param_number and param_index pair only show up once. No
-    // input can be aliased with output buffers.
-    TF_RET_CHECK(param_has_seen[param_number].element(param_index) == false);
-
-    *(param_has_seen[param_number].mutable_element(param_index)) = true;
-
+    // Check each alias.parameter_number and alias.parameter_index pair only
+    // show up once. No input can be aliased with output buffers.
+    TF_RET_CHECK(param_has_seen[alias.parameter_number].element(
+                     alias.parameter_index) == false);
+    *(param_has_seen[alias.parameter_number].mutable_element(
+        alias.parameter_index)) = true;
     return Status::OK();
   });
 }
diff --git a/tensorflow/compiler/xla/service/hlo_input_output_alias_config.h b/tensorflow/compiler/xla/service/hlo_input_output_alias_config.h
index 3967743d53..b0b71dece8 100644
--- a/tensorflow/compiler/xla/service/hlo_input_output_alias_config.h
+++ b/tensorflow/compiler/xla/service/hlo_input_output_alias_config.h
@@ -32,6 +32,28 @@ class HloModule;
 // parameter index in the entry computation.
 class HloInputOutputAliasConfig {
  public:
+  // The kind of aliases which can be set. A kUserAlias is one setup at
+  // compilation time by the user, and has to be respected. A kSystemAlias one
+  // might be setup by the compiler, if it decides it is convenient to do so.
+  enum AliasKind {
+    kNoAlias,
+    kUserAlias,
+    kSystemAlias,
+  };
+
+  // Defines the alias information for a given output buffer. A given output
+  // buffer shape index can refer only to one parameter+index.
+  struct Alias {
+    Alias(AliasKind kind, int64 parameter_number, ShapeIndex parameter_index)
+        : kind(kind),
+          parameter_number(parameter_number),
+          parameter_index(std::move(parameter_index)) {}
+
+    AliasKind kind;
+    int64 parameter_number;
+    ShapeIndex parameter_index;
+  };
+
   HloInputOutputAliasConfig() = default;
 
   explicit HloInputOutputAliasConfig(Shape shape) : alias_(shape) {}
@@ -41,12 +63,19 @@ class HloInputOutputAliasConfig {
   // Sets up alias config from `output_index` to `param_index` at
   // `param_number`.
   Status SetUpAlias(const ShapeIndex& output_index, int64 param_number,
-                    const ShapeIndex& param_index);
+                    const ShapeIndex& param_index, AliasKind kind);
+
+  // Returns the kind of alias for the given parameter number and parameter
+  // index. If no alias exists, AliasKind::kNoAlias is returned.
+  AliasKind ParameterAliasKind(int64 param_number,
+                               const ShapeIndex& param_index) const;
 
   // Returns true if the given parameter is aliased with one of the output
   // buffers.
   bool ParameterHasAlias(int64 param_number,
-                         const ShapeIndex& param_index) const;
+                         const ShapeIndex& param_index) const {
+    return ParameterAliasKind(param_number, param_index) != AliasKind::kNoAlias;
+  }
 
   // Checks whether the provided output index has already been aliased.
   bool OutputHasAlias(const ShapeIndex& output_index) const;
@@ -67,19 +96,17 @@ class HloInputOutputAliasConfig {
   // Returns the number of parameter and index of the parameter buffer that the
   // given output buffer index is aliased with. A nullopt is returned if there
   // is no parameter is aliased with the specific output.
-  absl::optional<std::pair<int64, ShapeIndex>> GetAliasedParameter(
+  absl::optional<Alias> GetAliasedParameter(
       const ShapeIndex& output_index) const;
 
   using AliasFn =
-      std::function<void(const ShapeIndex& output_index, int64 param_number,
-                         const ShapeIndex& param_index)>;
+      std::function<void(const ShapeIndex& output_index, const Alias&)>;
 
   // Iterates through each aliased output and input.
   void ForEachAlias(AliasFn fn) const;
 
   using AliasFnWithStatus =
-      std::function<Status(const ShapeIndex& output_index, int64 param_number,
-                           const ShapeIndex& param_index)>;
+      std::function<Status(const ShapeIndex& output_index, const Alias&)>;
 
   // Verifies that the given config is valid for the given module.
   // Specifically, the config's input and output should be in-bound and size of
@@ -94,12 +121,10 @@ class HloInputOutputAliasConfig {
  private:
   // A ShapeTree which indicates the list of buffers that's expected to be
   // aliased. The key on this shape tree represents the output index. The value
-  // is a pair of parameter number and index into the buffer. If the value is
-  // nullopt, it means there is no parameter aliasing for this output.
-  ShapeTree<absl::optional<std::pair<int64, ShapeIndex>>> alias_;
-
-  // The indices of the output which have been aliased.
-  absl::flat_hash_set<ShapeIndex> aliased_output_indices_;
+  // is an Alias data structure which defines the input parameter coordinates.
+  // If the value is nullopt, it means there is no parameter aliasing for this
+  // output.
+  ShapeTree<absl::optional<Alias>> alias_;
 };
 
 std::ostream& operator<<(std::ostream& out,
diff --git a/tensorflow/compiler/xla/service/hlo_input_output_alias_config_test.cc b/tensorflow/compiler/xla/service/hlo_input_output_alias_config_test.cc
index aeb9b0fdc8..a46a107723 100644
--- a/tensorflow/compiler/xla/service/hlo_input_output_alias_config_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_input_output_alias_config_test.cc
@@ -45,11 +45,12 @@ class HloInputOutputAliasConfigTest : public HloTestBase {
     EXPECT_TRUE(aliased_output);
     EXPECT_EQ(aliased_output.value(), output_index);
 
-    absl::optional<std::pair<int64, ShapeIndex>> aliased_param =
+    absl::optional<HloInputOutputAliasConfig::Alias> aliased_param =
         config.GetAliasedParameter(output_index);
 
     EXPECT_TRUE(aliased_param);
-    EXPECT_EQ(aliased_param.value(), std::make_pair(param_number, param_index));
+    EXPECT_EQ(aliased_param->parameter_number, param_number);
+    EXPECT_EQ(aliased_param->parameter_index, param_index);
   }
 
   void expect_not_aliased(const ShapeIndex& output_index, int64 param_number,
@@ -60,11 +61,12 @@ class HloInputOutputAliasConfigTest : public HloTestBase {
 
     EXPECT_FALSE(aliased_output && aliased_output == output_index);
 
-    absl::optional<std::pair<int64, ShapeIndex>> aliased_param =
+    absl::optional<HloInputOutputAliasConfig::Alias> aliased_param =
         config.GetAliasedParameter(output_index);
 
-    EXPECT_FALSE(aliased_param && aliased_param->first == param_number &&
-                 aliased_param->second == param_index);
+    EXPECT_FALSE(aliased_param &&
+                 aliased_param->parameter_number == param_number &&
+                 aliased_param->parameter_index == param_index);
   }
 };
 
@@ -84,8 +86,10 @@ ENTRY main {
   HloInputOutputAliasConfig config(
       module->entry_computation()->root_instruction()->shape());
 
-  TF_ASSERT_OK(config.SetUpAlias(/*output_index=*/{0}, /*param_number=*/1,
-                                 /*param_index=*/{}));
+  TF_ASSERT_OK(config.SetUpAlias(
+      /*output_index=*/{0}, /*param_number=*/1,
+      /*param_index=*/{},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
 
   expect_aliased(/*output_index=*/{0}, /*param_number=*/1,
                  /*param_index=*/{}, config);
@@ -114,11 +118,15 @@ ENTRY main {
   HloInputOutputAliasConfig config(
       module->entry_computation()->root_instruction()->shape());
 
-  TF_ASSERT_OK(config.SetUpAlias(/*output_index=*/{0}, /*param_number=*/0,
-                                 /*param_index=*/{0}));
+  TF_ASSERT_OK(config.SetUpAlias(
+      /*output_index=*/{0}, /*param_number=*/0,
+      /*param_index=*/{0},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
 
-  TF_ASSERT_OK(config.SetUpAlias(/*output_index=*/{1}, /*param_number=*/0,
-                                 /*param_index=*/{1}));
+  TF_ASSERT_OK(config.SetUpAlias(
+      /*output_index=*/{1}, /*param_number=*/0,
+      /*param_index=*/{1},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
 
   expect_aliased(/*output_index=*/{0}, /*param_number=*/0,
                  /*param_index=*/{0}, config);
@@ -149,11 +157,15 @@ ENTRY main {
   HloInputOutputAliasConfig config(
       module->entry_computation()->root_instruction()->shape());
 
-  TF_ASSERT_OK(config.SetUpAlias(/*output_index=*/{0}, /*param_number=*/0,
-                                 /*param_index=*/{}));
+  TF_ASSERT_OK(config.SetUpAlias(
+      /*output_index=*/{0}, /*param_number=*/0,
+      /*param_index=*/{},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
 
-  TF_ASSERT_OK(config.SetUpAlias(/*output_index=*/{1}, /*param_number=*/0,
-                                 /*param_index=*/{}));
+  TF_ASSERT_OK(config.SetUpAlias(
+      /*output_index=*/{1}, /*param_number=*/0,
+      /*param_index=*/{},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
 
   ASSERT_IS_NOT_OK(config.Verify(*module, [](const Shape& shape) {
     return ShapeUtil::ByteSizeOf(shape);
@@ -176,8 +188,10 @@ ENTRY main {
   HloInputOutputAliasConfig config(
       module->entry_computation()->root_instruction()->shape());
 
-  TF_ASSERT_OK(config.SetUpAlias(/*output_index=*/{1}, /*param_number=*/0,
-                                 /*param_index=*/{}));
+  TF_ASSERT_OK(config.SetUpAlias(
+      /*output_index=*/{1}, /*param_number=*/0,
+      /*param_index=*/{},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
 
   ASSERT_IS_NOT_OK(config.Verify(*module, [](const Shape& shape) {
     return ShapeUtil::ByteSizeOf(shape);
@@ -200,11 +214,15 @@ ENTRY main {
   HloInputOutputAliasConfig config(
       module->entry_computation()->root_instruction()->shape());
 
-  TF_ASSERT_OK(config.SetUpAlias(/*output_index=*/{0}, /*param_number=*/0,
-                                 /*param_index=*/{}));
+  TF_ASSERT_OK(config.SetUpAlias(
+      /*output_index=*/{0}, /*param_number=*/0,
+      /*param_index=*/{},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
 
-  ASSERT_IS_NOT_OK(config.SetUpAlias(/*output_index=*/{0}, /*param_number=*/1,
-                                     /*param_index=*/{}));
+  ASSERT_IS_NOT_OK(config.SetUpAlias(
+      /*output_index=*/{0}, /*param_number=*/1,
+      /*param_index=*/{},
+      /*kind=*/HloInputOutputAliasConfig::AliasKind::kUserAlias));
 }
 }  // namespace
 }  // namespace xla
-- 
GitLab


From fb8a14fde29f2dc19331d73795c2ead0e8ef0e8e Mon Sep 17 00:00:00 2001
From: Yifei Feng <yifeif@google.com>
Date: Tue, 8 Jan 2019 17:33:23 -0800
Subject: [PATCH 0429/1095] Remove cuda runtime dependencies from stream
 executor.

PiperOrigin-RevId: 228430943
---
 tensorflow/stream_executor/cuda/BUILD | 16 ++--------------
 1 file changed, 2 insertions(+), 14 deletions(-)

diff --git a/tensorflow/stream_executor/cuda/BUILD b/tensorflow/stream_executor/cuda/BUILD
index 58c9480884..87c8eae416 100644
--- a/tensorflow/stream_executor/cuda/BUILD
+++ b/tensorflow/stream_executor/cuda/BUILD
@@ -146,13 +146,7 @@ cc_library(
         "//tensorflow/stream_executor/lib",
         "//tensorflow/stream_executor/platform",
         "//tensorflow/stream_executor/platform:dso_loader",
-    ] + if_static(
-        [
-            "@local_config_cuda//cuda:cublas",
-            "@local_config_cuda//cuda:cudart_static",
-        ],
-        ["@local_config_cuda//cuda:cudart"],
-    ),
+    ] + if_static(["@local_config_cuda//cuda:cublas"]),
     alwayslink = True,
 )
 
@@ -214,13 +208,7 @@ cc_library(
         "//tensorflow/stream_executor/lib",
         "//tensorflow/stream_executor/platform",
         "//tensorflow/stream_executor/platform:dso_loader",
-    ] + tf_additional_cudnn_plugin_deps() + if_static(
-        [
-            "@local_config_cuda//cuda:cudnn",
-            "@local_config_cuda//cuda:cudart_static",
-        ],
-        ["@local_config_cuda//cuda:cudart"],
-    ),
+    ] + tf_additional_cudnn_plugin_deps() + if_static(["@local_config_cuda//cuda:cudnn"]),
     alwayslink = True,
 )
 
-- 
GitLab


From 9e0e9e255fb9210ed2da50dffba156f1d5617b8b Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Wed, 9 Jan 2019 02:03:37 +0000
Subject: [PATCH 0430/1095] Update docstring for tf.train.init_from_checkpoint

This fix updates docstring for tf.train.init_from_checkpoint,
to match the actual implementation. (thanks sharvil)

This fix fixes 24756.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/training/checkpoint_utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/training/checkpoint_utils.py b/tensorflow/python/training/checkpoint_utils.py
index 74b46179e7..5e18f4b722 100644
--- a/tensorflow/python/training/checkpoint_utils.py
+++ b/tensorflow/python/training/checkpoint_utils.py
@@ -180,8 +180,8 @@ def init_from_checkpoint(ckpt_dir_or_file, assignment_map):
       (in default graph).
 
   Raises:
-    tf.errors.OpError: If missing checkpoints or tensors in checkpoints.
-    ValueError: If missing variables in current graph.
+    ValueError: If missing variables in current graph, or if missing
+      checkpoints or tensors in checkpoints.
   """
   if distribution_strategy_context.get_cross_replica_context():
     _init_from_checkpoint(None, ckpt_dir_or_file, assignment_map)
-- 
GitLab


From c8191270e47ece2bba777c271428e205d34d0827 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 8 Jan 2019 18:41:13 -0800
Subject: [PATCH 0431/1095] Remove Identity nodes where all inputs and outputs
 are on the same device, but the identity is on a different device.

PiperOrigin-RevId: 228438774
---
 .../optimizers/dependency_optimizer.cc        | 17 ++++++------
 .../optimizers/dependency_optimizer_test.cc   | 26 +++++++++++++++++++
 2 files changed, 35 insertions(+), 8 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/dependency_optimizer.cc b/tensorflow/core/grappler/optimizers/dependency_optimizer.cc
index 7fee3ae9d5..8b81cb2430 100644
--- a/tensorflow/core/grappler/optimizers/dependency_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/dependency_optimizer.cc
@@ -205,14 +205,6 @@ bool DependencyOptimizer::BypassingNodeIsBeneficial(
     num_cross_out += static_cast<int>(output_node->device() != node_dev);
   }
 
-  if ((is_identity || is_multi_input_identity_n) && num_cross_in > 0 &&
-      num_cross_out > 0) {
-    // This identity node follows a device crossing, so it might be
-    // following a _Recv node after partioning. Do not remove such nodes,
-    // unless they only have consumers on the same device as themselves.
-    return false;
-  }
-
   // Make sure we do not increase the number of device crossings.
   const int num_cross_before = num_cross_in + num_cross_out;
   int num_cross_after = 0;
@@ -225,6 +217,15 @@ bool DependencyOptimizer::BypassingNodeIsBeneficial(
   if (num_cross_after > num_cross_before) {
     return false;
   }
+
+  if ((is_identity || is_multi_input_identity_n) && num_cross_in > 0 &&
+      num_cross_out > 0 && num_cross_after > 0) {
+    // This identity node follows a device crossing, so it might be
+    // following a _Recv node after partioning. Do not remove such nodes,
+    // unless they only have consumers on the same device as themselves.
+    return false;
+  }
+
   return true;
 }
 
diff --git a/tensorflow/core/grappler/optimizers/dependency_optimizer_test.cc b/tensorflow/core/grappler/optimizers/dependency_optimizer_test.cc
index 8d70d9d5c7..5883fcb926 100644
--- a/tensorflow/core/grappler/optimizers/dependency_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/dependency_optimizer_test.cc
@@ -356,6 +356,32 @@ TEST_F(DependencyOptimizerTest, RemoveIdentityOps_DeviceBoundaries) {
   VerifyGraphsEqual(item.graph, output, __FUNCTION__);
 }
 
+TEST_F(DependencyOptimizerTest, RemoveIdentityOps_IdenticalDevices) {
+  tensorflow::Scope s = tensorflow::Scope::NewRootScope();
+  Output x = ops::RandomUniform(s.WithOpName("x").WithDevice("/CPU:0"), {1, 2},
+                                DT_FLOAT);
+  auto id_a = ops::Identity(s.WithOpName("id_a").WithDevice("/CPU:1"), x);
+  Output id =
+      ops::Identity(s.WithControlDependencies(id_a).WithDevice("/CPU:0"), id_a);
+
+  GrapplerItem item;
+  TF_CHECK_OK(s.ToGraphDef(&item.graph));
+  item.fetch.push_back("Identity");
+
+  DependencyOptimizer optimizer;
+  GraphDef output;
+  Status status = optimizer.Optimize(nullptr, item, &output);
+  TF_EXPECT_OK(status);
+
+  EXPECT_EQ(item.graph.node_size() - 1, output.node_size());
+  for (const NodeDef& node : output.node()) {
+    EXPECT_NE(node.name(), "id_a");
+    if (node.name() == "Identity") {
+      EXPECT_EQ(node.input(0), "x");
+    }
+  }
+}
+
 TEST_F(DependencyOptimizerTest, RemoveNoOps_SingleInputOrOutput) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
   Output x = ops::RandomUniform(s.WithOpName("x"), {1, 2}, DT_FLOAT);
-- 
GitLab


From ec81825aaf7e848d9f8ddffdf1e0d20aebe9172c Mon Sep 17 00:00:00 2001
From: Reed Wanderman-Milne <reedwm@google.com>
Date: Tue, 8 Jan 2019 18:50:27 -0800
Subject: [PATCH 0432/1095] Add GPU explicit padding to tf.nn.conv2d.

Benchmark results:

All benchmark results were run on a Z840 with a Titan V, with internal TensorFlow.

1. Resnet50 Eager results
The internal resnet50 Eager benchmarks were run, to ensure no regressions in Resnet50 in Eager mode that could have occurred due to the extra Python overhead this change adds. The benchmarks run are here: https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/resnet50/resnet50_test.py. Each row was run 150 times and the average was taken. Note none of these benchmarks use explicit padding. Numbers represent time, so lower is better.

Benchmark Name                                        After   Before  % diff
apply_async_gpu_batch_64_channels_first               0.0726  0.0726  -0.06%
apply_gpu_batch_64_channels_first                     0.0725  0.0725  -0.06%
apply_with_defun_gpu_batch_64_channels_first          0.0755  0.0756   0.07%
train_async_gpu_batch_16_channels_first               0.0776  0.0778   0.27%
train_async_gpu_batch_32_channels_first               0.1268  0.1271   0.23%
train_dataset_gpu_batch_16_channels_first             0.1085  0.1094   0.77%
train_dataset_gpu_batch_32_channels_first             0.1473  0.1477   0.28%
train_dataset_with_defun_gpu_batch_16_channels_first  0.0800  0.0803   0.37%
train_dataset_with_defun_gpu_batch_32_channels_first  0.1325  0.1326   0.09%
train_gpu_batch_16_channels_first                     0.0812  0.0813   0.18%
train_gpu_batch_32_channels_first                     0.1329  0.1325  -0.32%
train_with_defun_gpu_batch_16_channels_first          0.0789  0.0791   0.26%
train_with_defun_gpu_batch_32_channels_first          0.1325  0.1325  -0.02%

There is minimal impact to Eager performance.

2. tf_cnn_benchmarks
tf_cnn_benchmarks was run internally with the following flags:

--batch_size=128 --model=resnet50

It was run 60 times with and without this change. With this change, tf_cnn_benchmarks had all instances of a tf.pad followed by Conv2D replaced with an explicitly padded Conv2d. It got 330.96 images/sec with this change and 330.80 without, and the difference is likely noise. Therefore, this change does not improve tf_cnn_benchmarks performance.

3. Conv2D benchmarks

The added benchmarks to conv_ops_test.py were run with this change, each 400 times and the average was taken. They were not run without this change. The table groups the 8 benchmarks into 4 pairs, with each pair running two similar benchmarks, one with explicit padding, and one without explicit padding.

Benchmark name                Explicit  Non-explicit  % diff
explicit/manual pad forward   0.001815  0.002006      10.56%
explicit/manual pad backward  0.006261  0.006937      10.79%
eager explicit/same pad       0.039320  0.038403      -2.33%
graph explicit/same pad       0.037039  0.037034      -0.11%

The first two rows show there is theoretical performance gains to using explicit padding over a manual tf.pad followed by the convolution. On Resnet50, we were not able to achieve this performance gain in practice, as tf_cnn_benchmarks saw no improvement. On models that use larger paddings than in Resnet50, the performance gain will less negligible. The last two rows compare explicit padding padding to the equivalent same padding, to see if explicit padding adds any overhead. In Graph mode, there is no overhead, but Eager mode has some overhead with explicit padding over SAME padding.

PiperOrigin-RevId: 228439591
---
 .../tf2xla/kernels/conv_op_helpers.cc         |    9 +-
 .../api_def/base_api/api_def_Conv2D.pbtxt     |    9 +
 .../api_def_Conv2DBackpropFilter.pbtxt        |    9 +
 .../api_def_Conv2DBackpropInput.pbtxt         |    9 +
 tensorflow/core/framework/common_shape_fns.cc |   84 +-
 tensorflow/core/framework/common_shape_fns.h  |   48 +-
 .../core/framework/common_shape_fns_test.cc   |   82 +-
 tensorflow/core/framework/ops_util.cc         |    3 +
 .../large_function_graph.pbtxt                |    7 +
 .../grappler/optimizers/layout_optimizer.cc   |   23 +
 .../optimizers/layout_optimizer_test.cc       |   64 +-
 .../core/kernels/conv_grad_filter_ops.cc      |  134 +-
 .../core/kernels/conv_grad_input_ops.cc       |  146 +-
 tensorflow/core/kernels/conv_grad_ops.cc      |   51 +-
 tensorflow/core/kernels/conv_grad_ops.h       |   20 +-
 tensorflow/core/kernels/conv_grad_ops_3d.cc   |   21 +-
 tensorflow/core/kernels/conv_ops.cc           |  223 +--
 tensorflow/core/kernels/conv_ops.h            |   13 +-
 .../core/kernels/depthwise_conv_grad_op.cc    |    6 +-
 tensorflow/core/kernels/depthwise_conv_op.cc  |    3 +-
 tensorflow/core/ops/nn_ops.cc                 |   11 +-
 tensorflow/core/util/padding.cc               |   43 +
 tensorflow/core/util/padding.h                |   19 +-
 tensorflow/core/util/tensor_format.h          |   27 +-
 .../python/keras/layers/convolutional.py      |    4 +-
 tensorflow/python/keras/utils/conv_utils.py   |    4 +-
 .../python/kernel_tests/conv_ops_test.py      | 1232 +++++++++++++++--
 tensorflow/python/ops/nn_grad.py              |   15 +-
 tensorflow/python/ops/nn_ops.py               |  316 ++++-
 29 files changed, 2218 insertions(+), 417 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc b/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc
index 6b049f653c..5b4f863f74 100644
--- a/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc
+++ b/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc
@@ -212,8 +212,8 @@ Status ConvBackpropComputeDimensionsV2XlaShapes(
       XLAShapeToTensorShape(out_backprop_shape, &out_backprop_tensor_shape));
   return ConvBackpropComputeDimensionsV2(
       label, num_spatial_dims, input_tensor_shape, filter_tensor_shape,
-      out_backprop_tensor_shape, dilations, strides, padding, data_format,
-      dims);
+      out_backprop_tensor_shape, dilations, strides, padding,
+      /*explicit_paddings=*/{}, data_format, dims);
 }
 
 }  // anonymous namespace
@@ -227,6 +227,11 @@ xla::StatusOr<ConvOpAttrs> ConvOpAttrs::Create(int num_spatial_dims,
   TF_RETURN_IF_ERROR(ctx->GetAttr("dilations", &attrs.dilations));
   TF_RETURN_IF_ERROR(ctx->GetAttr("strides", &attrs.strides));
   TF_RETURN_IF_ERROR(ctx->GetAttr("padding", &attrs.padding));
+  // TODO(reedwm): Support explicit padding.
+  if (attrs.padding == EXPLICIT) {
+    return errors::Unimplemented(
+        "XLA does not yet support Conv2D with explicit padding.");
+  }
 
   string data_format;
   TF_RETURN_IF_ERROR(ctx->GetAttr("data_format", &data_format));
diff --git a/tensorflow/core/api_def/base_api/api_def_Conv2D.pbtxt b/tensorflow/core/api_def/base_api/api_def_Conv2D.pbtxt
index 070d6adb97..d0794de4ba 100644
--- a/tensorflow/core/api_def/base_api/api_def_Conv2D.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_Conv2D.pbtxt
@@ -33,6 +33,15 @@ END
     name: "padding"
     description: <<END
 The type of padding algorithm to use.
+END
+  }
+  attr {
+    name: "explicit_paddings"
+    description: <<END
+If `padding` is `"EXPLICIT"`, the list of explicit padding amounts. For the ith
+dimension, the amount of padding inserted before and after the dimension is
+`explicit_paddings[2 * i]` and `explicit_paddings[2 * i + 1]`, respectively. If
+`padding` is not `"EXPLICIT"`, `explicit_paddings` must be empty.
 END
   }
   attr {
diff --git a/tensorflow/core/api_def/base_api/api_def_Conv2DBackpropFilter.pbtxt b/tensorflow/core/api_def/base_api/api_def_Conv2DBackpropFilter.pbtxt
index ff2d9d71db..c8af9ff976 100644
--- a/tensorflow/core/api_def/base_api/api_def_Conv2DBackpropFilter.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_Conv2DBackpropFilter.pbtxt
@@ -41,6 +41,15 @@ END
     name: "padding"
     description: <<END
 The type of padding algorithm to use.
+END
+  }
+  attr {
+    name: "explicit_paddings"
+    description: <<END
+If `padding` is `"EXPLICIT"`, the list of explicit padding amounts. For the ith
+dimension, the amount of padding inserted before and after the dimension is
+`explicit_paddings[2 * i]` and `explicit_paddings[2 * i + 1]`, respectively. If
+`padding` is not `"EXPLICIT"`, `explicit_paddings` must be empty.
 END
   }
   attr {
diff --git a/tensorflow/core/api_def/base_api/api_def_Conv2DBackpropInput.pbtxt b/tensorflow/core/api_def/base_api/api_def_Conv2DBackpropInput.pbtxt
index 2de38b4263..8aaae4aab6 100644
--- a/tensorflow/core/api_def/base_api/api_def_Conv2DBackpropInput.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_Conv2DBackpropInput.pbtxt
@@ -40,6 +40,15 @@ END
     name: "padding"
     description: <<END
 The type of padding algorithm to use.
+END
+  }
+  attr {
+    name: "explicit_paddings"
+    description: <<END
+If `padding` is `"EXPLICIT"`, the list of explicit padding amounts. For the ith
+dimension, the amount of padding inserted before and after the dimension is
+`explicit_paddings[2 * i]` and `explicit_paddings[2 * i + 1]`, respectively. If
+`padding` is not `"EXPLICIT"`, `explicit_paddings` must be empty.
 END
   }
   attr {
diff --git a/tensorflow/core/framework/common_shape_fns.cc b/tensorflow/core/framework/common_shape_fns.cc
index bf2d902af4..3d3711ffc5 100644
--- a/tensorflow/core/framework/common_shape_fns.cc
+++ b/tensorflow/core/framework/common_shape_fns.cc
@@ -37,6 +37,11 @@ Status GetWindowedOutputSizeVerboseV2(int64 input_size, int64 filter_size,
       *output_size = (input_size - effective_filter_size + stride) / stride;
       *padding_before = *padding_after = 0;
       break;
+    case Padding::EXPLICIT:
+      *output_size = (input_size + *padding_before + *padding_after -
+                      effective_filter_size + stride) /
+                     stride;
+      break;
     case Padding::SAME:
       *output_size = (input_size + stride - 1) / stride;
       const int64 padding_needed =
@@ -71,6 +76,11 @@ Status GetWindowedOutputSizeVerbose(int64 input_size, int64 filter_size,
 Status GetWindowedOutputSize(int64 input_size, int64 filter_size, int64 stride,
                              Padding padding_type, int64* output_size,
                              int64* padding_size) {
+  if (padding_type == Padding::EXPLICIT) {
+    return errors::Internal(
+        "GetWindowedOutputSize does not handle EXPLICIT padding; call "
+        "GetWindowedOutputSizeVerbose instead");
+  }
   int64 padding_after_unused;
   return GetWindowedOutputSizeVerbose(input_size, filter_size, stride,
                                       padding_type, output_size, padding_size,
@@ -81,6 +91,11 @@ Status GetWindowedOutputSizeV2(int64 input_size, int64 filter_size,
                                int64 dilation_rate, int64 stride,
                                Padding padding_type, int64* output_size,
                                int64* padding_size) {
+  if (padding_type == Padding::EXPLICIT) {
+    return errors::Internal(
+        "GetWindowedOutputSizeV2 does not handle EXPLICIT padding; call "
+        "GetWindowedOutputSizeVerboseV2 instead");
+  }
   int64 padding_after_unused;
   return GetWindowedOutputSizeVerboseV2(input_size, filter_size, dilation_rate,
                                         stride, padding_type, output_size,
@@ -123,8 +138,8 @@ Status GetWindowedOutputSizeFromDimsV2(
     shape_inference::InferenceContext* c,
     shape_inference::DimensionHandle input_size,
     shape_inference::DimensionOrConstant filter_size, int64 dilation_rate,
-    int64 stride, Padding padding_type,
-    shape_inference::DimensionHandle* output_size) {
+    int64 stride, Padding padding_type, int64 padding_before,
+    int64 padding_after, shape_inference::DimensionHandle* output_size) {
   if (stride <= 0) {
     return errors::InvalidArgument("Stride must be > 0, but got ", stride);
   }
@@ -137,6 +152,11 @@ Status GetWindowedOutputSizeFromDimsV2(
   // See also the parallel implementation in GetWindowedOutputSizeVerbose.
   switch (padding_type) {
     case Padding::VALID:
+      padding_before = padding_after = 0;
+      TF_FALLTHROUGH_INTENDED;
+    case Padding::EXPLICIT:
+      TF_RETURN_IF_ERROR(
+          c->Add(input_size, padding_before + padding_after, &input_size));
       if (dilation_rate > 1) {
         DimensionHandle window_size;
         TF_RETURN_IF_ERROR(
@@ -166,9 +186,18 @@ Status GetWindowedOutputSizeFromDims(
     shape_inference::DimensionHandle input_size,
     shape_inference::DimensionOrConstant filter_size, int64 stride,
     Padding padding_type, shape_inference::DimensionHandle* output_size) {
+  if (padding_type == Padding::EXPLICIT) {
+    return errors::Internal(
+        "GetWindowedOutputSizeFromDims does not handle EXPLICIT padding; call "
+        "GetWindowedOutputSizeFromDimsV2 instead");
+  }
   return GetWindowedOutputSizeFromDimsV2(c, input_size, filter_size,
                                          /*dilation_rate=*/1, stride,
-                                         padding_type, output_size);
+                                         padding_type,
+                                         // Give dummy values of -1 to
+                                         // padding_before and padding_after,
+                                         // since explicit padding is not used.
+                                         -1, -1, output_size);
 }
 
 Status UnchangedShape(shape_inference::InferenceContext* c) {
@@ -371,7 +400,10 @@ Status ShapeFromDimensions(DimensionHandle batch_dim,
   return tensorflow::Status::OK();
 }
 
-Status Conv2DShape(shape_inference::InferenceContext* c) {
+namespace {
+
+Status Conv2DShapeImpl(shape_inference::InferenceContext* c,
+                       bool supports_explicit_padding) {
   string data_format_str, filter_format_str;
   if (!c->GetAttr("data_format", &data_format_str).ok()) {
     data_format_str = "NHWC";
@@ -464,13 +496,30 @@ Status Conv2DShape(shape_inference::InferenceContext* c) {
   Padding padding;
   TF_RETURN_IF_ERROR(c->GetAttr("padding", &padding));
 
+  std::vector<int64> explicit_paddings;
+  if (supports_explicit_padding) {
+    TF_RETURN_IF_ERROR(c->GetAttr("explicit_paddings", &explicit_paddings));
+    TF_RETURN_IF_ERROR(CheckValidPadding(padding, explicit_paddings,
+                                         /*num_dims=*/4, data_format));
+  } else {
+    DCHECK(padding != Padding::EXPLICIT);
+  }
+
   DimensionHandle output_rows, output_cols;
+  int64 pad_rows_before = -1, pad_rows_after = -1;
+  int64 pad_cols_before = -1, pad_cols_after = -1;
+  if (padding == Padding::EXPLICIT) {
+    GetExplicitPaddingForDim(explicit_paddings, data_format, 'H',
+                             &pad_rows_before, &pad_rows_after);
+    GetExplicitPaddingForDim(explicit_paddings, data_format, 'W',
+                             &pad_cols_before, &pad_cols_after);
+  }
   TF_RETURN_IF_ERROR(GetWindowedOutputSizeFromDimsV2(
       c, input_spatial_dims[0], filter_rows_dim, dilation_rows, stride_rows,
-      padding, &output_rows));
+      padding, pad_rows_before, pad_rows_after, &output_rows));
   TF_RETURN_IF_ERROR(GetWindowedOutputSizeFromDimsV2(
       c, input_spatial_dims[1], filter_cols_dim, dilation_cols, stride_cols,
-      padding, &output_cols));
+      padding, pad_cols_before, pad_cols_after, &output_cols));
 
   ShapeHandle output_shape;
   TF_RETURN_IF_ERROR(
@@ -480,6 +529,19 @@ Status Conv2DShape(shape_inference::InferenceContext* c) {
   return Status::OK();
 }
 
+}  // namespace
+
+// Shape function for Conv2D-like operations that support explicit padding.
+Status Conv2DShapeWithExplicitPadding(shape_inference::InferenceContext* c) {
+  return Conv2DShapeImpl(c, true);
+}
+
+// Shape function for Conv2D-like operations that do not support explicit
+// padding.
+Status Conv2DShape(shape_inference::InferenceContext* c) {
+  return Conv2DShapeImpl(c, false);
+}
+
 // TODO(mjanusz): Unify all conv/pooling shape functions.
 Status Conv3DShape(shape_inference::InferenceContext* c) {
   ShapeHandle input_shape;
@@ -551,13 +613,13 @@ Status Conv3DShape(shape_inference::InferenceContext* c) {
 
   TF_RETURN_IF_ERROR(GetWindowedOutputSizeFromDimsV2(
       c, in_planes_dim, filter_planes_dim, dilation_planes, stride_planes,
-      padding, &output_planes));
+      padding, -1, -1, &output_planes));
   TF_RETURN_IF_ERROR(GetWindowedOutputSizeFromDimsV2(
-      c, in_rows_dim, filter_rows_dim, dilation_rows, stride_rows, padding,
-      &output_rows));
+      c, in_rows_dim, filter_rows_dim, dilation_rows, stride_rows, padding, -1,
+      -1, &output_rows));
   TF_RETURN_IF_ERROR(GetWindowedOutputSizeFromDimsV2(
-      c, in_cols_dim, filter_cols_dim, dilation_cols, stride_cols, padding,
-      &output_cols));
+      c, in_cols_dim, filter_cols_dim, dilation_cols, stride_cols, padding, -1,
+      -1, &output_cols));
 
   ShapeHandle output_shape;
   if (data_format == "NCDHW") {
diff --git a/tensorflow/core/framework/common_shape_fns.h b/tensorflow/core/framework/common_shape_fns.h
index 362899b947..14b9688bdc 100644
--- a/tensorflow/core/framework/common_shape_fns.h
+++ b/tensorflow/core/framework/common_shape_fns.h
@@ -38,11 +38,12 @@ namespace tensorflow {
 //
 // Padding (P): the padding we apply to the input tensor along each
 // dimension. This is usually used to make sure that the spatial dimensions
-// do not shrink when we progress with convolutions. Two types of padding are
-// often used:
+// do not shrink when we progress with convolutions. This function supports two
+// types of padding.
 //   SAME: the pad value is computed so that the output will have size H/S.
 //   VALID: no padding is carried out.
-// The padded area is zero-filled.
+// If you want to use EXPLICIT padding, GetWindowedOutputSizeVerbose must be
+// called instead. Note the padded area is zero-filled.
 //
 // The output dimensions for convolution and many other operations, when given
 // all the parameters above, are as follows:
@@ -95,6 +96,9 @@ Status GetWindowedOutputSize(int64 input_size, int64 filter_size, int64 stride,
 //   When the stride is 1, the expression simplifies to
 //     H' = H-K'+1.
 //
+// If you want to use EXPLICIT padding, GetWindowedOutputSizeVerboseV2 must be
+// called instead
+//
 // TODO(b/67112639): Merge V2 versions and the original versions eventually.
 Status GetWindowedOutputSizeV2(int64 input_size, int64 filter_size,
                                int64 dilation_rate, int64 stride,
@@ -102,9 +106,12 @@ Status GetWindowedOutputSizeV2(int64 input_size, int64 filter_size,
                                int64* padding_size);
 
 // Returns the same output dimensions as in GetWindowedOutputSize, but returns
-// verbose padding dimensions (before/after). Any excess padding
-// (caused by an odd padding size value) is added to the 'padding_after'
-// dimension.
+// verbose padding dimensions (before/after), and EXPLICIT padding is supported.
+// When padding_type is EXPLICIT, *padding_before and *padding_after must
+// already point to initialized integers with the padding amounts. Otherwise,
+// *padding_before and *padding_after are set by this function, and any
+// excess padding (caused by an odd padding size value) is added to the
+// 'padding_after' dimension.
 Status GetWindowedOutputSizeVerbose(int64 input_size, int64 filter_size,
                                     int64 stride, Padding padding_type,
                                     int64* output_size, int64* padding_before,
@@ -122,7 +129,8 @@ Status GetWindowedOutputSizeVerboseV2(int64 input_size, int64 filter_size,
 // of the output tensor and padding to be applied to the input tensor at the
 // lower end of every dimension. Use for 3D convolutions, where the input data
 // is padded with zeros, as well as for 3D avg/max pooling, where the input data
-// is padded with invalid values that are not considered for pooling.
+// is padded with invalid values that are not considered for pooling. EXPLICIT
+// padding is not supported.
 Status Get3dOutputSize(const std::array<int64, 3>& input,
                        const std::array<int64, 3>& window,
                        const std::array<int64, 3>& strides,
@@ -140,21 +148,23 @@ Status Get3dOutputSizeV2(const std::array<int64, 3>& input,
 
 namespace shape_inference {
 
-// Like GetWindowedOutputSize, but deals with DimensionHandles.
+// Like GetWindowedOutputSize, but deals with DimensionHandles. Does not support
+// EXPLICIT padding.
 Status GetWindowedOutputSizeFromDims(InferenceContext* c,
                                      DimensionHandle input_size,
                                      DimensionOrConstant filter_size,
                                      int64 stride, Padding padding_type,
                                      DimensionHandle* output_size);
 
-// The V2 version computes the same outputs with arbitrary dilation_rate. For
-// detailed equations, refer to the comments for GetWindowedOutputSizeV2().
-Status GetWindowedOutputSizeFromDimsV2(InferenceContext* c,
-                                       DimensionHandle input_size,
-                                       DimensionOrConstant filter_size,
-                                       int64 dilation_rate, int64 stride,
-                                       Padding padding_type,
-                                       DimensionHandle* output_size);
+// The V2 version computes the same outputs with arbitrary dilation_rate, and
+// supports EXPLICIT padding. For detailed equations, refer to the comments
+// for GetWindowedOutputSizeV2(). The 'padding_before' and 'padding_after'
+// parameters are only used if padding_type == EXPLICIT.
+Status GetWindowedOutputSizeFromDimsV2(
+    InferenceContext* c, DimensionHandle input_size,
+    DimensionOrConstant filter_size, int64 dilation_rate, int64 stride,
+    Padding padding_type, int64 padding_before, int64 padding_after,
+    DimensionHandle* output_size);
 
 // Transfers shape of input(0) to output(0).
 Status UnchangedShape(shape_inference::InferenceContext* c);
@@ -222,7 +232,11 @@ Status BiasAddShape(shape_inference::InferenceContext* c);
 // Shape function for BiasAddGrad-like operations.
 Status BiasAddGradShape(shape_inference::InferenceContext* c);
 
-// Shape function for Conv2D-like operations.
+// Shape function for Conv2D-like operations that support explicit padding.
+Status Conv2DShapeWithExplicitPadding(shape_inference::InferenceContext* c);
+
+// Shape function for Conv2D-like operations that do not support explicit
+// padding.
 Status Conv2DShape(shape_inference::InferenceContext* c);
 
 // Shape function for Conv3D-like operations.
diff --git a/tensorflow/core/framework/common_shape_fns_test.cc b/tensorflow/core/framework/common_shape_fns_test.cc
index 7c395679d3..b94925c04e 100644
--- a/tensorflow/core/framework/common_shape_fns_test.cc
+++ b/tensorflow/core/framework/common_shape_fns_test.cc
@@ -408,12 +408,14 @@ TEST(CommonShapeFnsTest, BiasAddGradShapeTest) {
 TEST(CommonShapeFnsTest, Conv2DShapeTest) {
   ShapeInferenceTestOp op("Conv2D");
   auto set_op = [&op](const std::vector<int32>& strides, const string& padding,
-                      const string& data_format, const string& filter_format) {
+                      const string& data_format, const string& filter_format,
+                      const std::vector<int32>& explicit_paddings = {}) {
     TF_CHECK_OK(NodeDefBuilder("test", "Conv2D")
                     .Input("input", 0, DT_FLOAT)
                     .Input("filter", 0, DT_FLOAT)
                     .Attr("strides", strides)
                     .Attr("padding", padding)
+                    .Attr("explicit_paddings", explicit_paddings)
                     .Attr("data_format", data_format)
                     .Attr("filter_format", filter_format)
                     .Finalize(&op.node_def));
@@ -536,19 +538,73 @@ TEST(CommonShapeFnsTest, Conv2DShapeTest) {
   INFER_OK(op, "[1,?,4,1];[?,?,?,?]", "[d0_0,?,2,d1_3]");
   INFER_OK(op, "[1,4,?,1];[?,?,?,?]", "[d0_0,2,?,d1_3]");
   INFER_OK(op, "[1,4,4,?];[?,?,?,?]", "[d0_0,2,2,d1_3]");
+
+  // Some tests for "EXPLICIT" padding
+
+  // 4x4 input, 1x1 filter, 1x1 stride, [0, 2, 1, 4] padding
+  set_op({{1, 1, 1, 1}}, "EXPLICIT", "NHWC", "HWIO", {0, 0, 0, 2, 1, 4, 0, 0});
+  INFER_OK(op, "[1,4,4,1];[1,1,1,1]", "[d0_0,6,9,d1_3]");
+
+  // 3x3 input, 2x2 filter, 1x1 stride, [1, 0, 1, 2] padding
+  set_op({{1, 1, 1, 1}}, "EXPLICIT", "NHWC", "HWIO", {0, 0, 1, 0, 1, 2, 0, 0});
+  INFER_OK(op, "[1,3,3,1];[2,2,1,1]", "[d0_0,3,5,d1_3]");
+
+  // 4x4 input, 2x2 filter, 2x2 stride, [3, 2, 1, 0] padding
+  set_op({{1, 2, 2, 1}}, "EXPLICIT", "NHWC", "HWIO", {0, 0, 3, 2, 1, 0, 0, 0});
+  INFER_OK(op, "[1,4,4,2];[2,2,2,3]", "[d0_0,4,2,d1_3]");
+
+  // 2x2 input, 2x1 filter, 1x2 stride, [1, 1, 2, 2] padding
+  set_op({{1, 1, 2, 1}}, "EXPLICIT", "NHWC", "HWIO", {0, 0, 1, 1, 2, 2, 0, 0});
+  INFER_OK(op, "[1,2,2,1];[2,1,1,1]", "[d0_0,3,3,d1_3]");
+
+  // Unknown dims in the critical fields lead to partial inference.
+  INFER_OK(op, "[1,4,4,1];[2,1,1,1]", "[d0_0,5,4,d1_3]");
+  INFER_OK(op, "[1,?,4,1];[2,1,1,1]", "[d0_0,?,4,d1_3]");
+  INFER_OK(op, "[1,4,?,1];[2,1,1,1]", "[d0_0,5,?,d1_3]");
+  INFER_OK(op, "[1,4,4,?];[2,1,1,1]", "[d0_0,5,4,d1_3]");
+  INFER_OK(op, "[1,4,4,1];[?,1,1,1]", "[d0_0,?,4,d1_3]");
+  INFER_OK(op, "[1,4,4,1];[2,?,1,1]", "[d0_0,5,?,d1_3]");
+
+  // Explicit padding errors
+  // Negative padding
+  set_op({{1, 1, 1, 1}}, "EXPLICIT", "NHWC", "HWIO", {0, 0, 0, -1, 0, 0, 0, 0});
+  INFER_ERROR("must be nonnegative", op, "[1,2,2,1];[1,1,1,1]");
+
+  // Too little padding (7 explicit paddings instead of 8)
+  set_op({{1, 1, 1, 1}}, "EXPLICIT", "NHWC", "HWIO", {0, 0, 0, 0, 0, 0, 0});
+  INFER_ERROR("must contain 8 values", op, "[1,2,2,1];[1,1,1,1]");
+
+  // Too much padding (9 explicit paddings instead of 8)
+  set_op({{1, 1, 1, 1}}, "EXPLICIT", "NHWC", "HWIO",
+         {0, 0, 0, 0, 0, 0, 0, 0, 0});
+  INFER_ERROR("must contain 8 values", op, "[1,2,2,1];[1,1,1,1]");
+
+  // Padding in batch dimension
+  set_op({{1, 1, 1, 1}}, "EXPLICIT", "NHWC", "HWIO", {1, 0, 0, 0, 0, 0, 0, 0});
+  INFER_ERROR("batch or depth dimensions", op, "[1,2,2,1];[1,1,1,1]");
+
+  // Padding in depth dimension
+  set_op({{1, 1, 1, 1}}, "EXPLICIT", "NHWC", "HWIO", {0, 0, 0, 0, 0, 0, 1, 0});
+  INFER_ERROR("batch or depth dimensions", op, "[1,2,2,1];[1,1,1,1]");
+
+  // Padding explicit_paddings when padding is not EXPLICIT
+  set_op({{1, 1, 1, 1}}, "VALID", "NHWC", "HWIO", {0, 0, 0, 0, 0, 0, 0, 0});
+  INFER_ERROR("must be empty", op, "[1,2,2,1];[1,1,1,1]");
 }
 
 TEST(CommonShapeFnsTest, Conv2DDilatedShapeTest) {
   ShapeInferenceTestOp op("Conv2D");
   auto set_op = [&op](const std::vector<int32>& dilations,
                       const std::vector<int32>& strides, const string& padding,
-                      const string& data_format) {
+                      const string& data_format,
+                      const std::vector<int32>& explicit_paddings = {}) {
     TF_CHECK_OK(NodeDefBuilder("test", "Conv2D")
                     .Input("input", 0, DT_FLOAT)
                     .Input("filter", 0, DT_FLOAT)
                     .Attr("dilations", dilations)
                     .Attr("strides", strides)
                     .Attr("padding", padding)
+                    .Attr("explicit_paddings", explicit_paddings)
                     .Attr("data_format", data_format)
                     .Finalize(&op.node_def));
   };
@@ -628,6 +684,28 @@ TEST(CommonShapeFnsTest, Conv2DDilatedShapeTest) {
   // 4x4 input, 2x2 filter, 2x2 dilations, 1x1 stride
   set_op({{1, 2, 2, 1}}, {{1, 1, 1, 1}}, "SAME", "NHWC");
   INFER_OK(op, "[1,4,4,1];[2,2,1,1]", "[d0_0,d0_1,d0_2,d1_3]");
+
+  // Some tests for "EXPLICIT" padding
+
+  // 4x4 input, 1x1 filter, 2x1 dilations, 1x1 stride, [0, 2, 1, 4] padding
+  set_op({{1, 2, 1, 1}}, {{1, 1, 1, 1}}, "EXPLICIT", "NHWC",
+         {0, 0, 0, 2, 1, 4, 0, 0});
+  INFER_OK(op, "[1,4,4,1];[1,1,1,1]", "[d0_0,6,9,d1_3]");
+
+  // 3x3 input, 2x2 filter, 2x2 dilations, 1x1 stride, [1, 0, 1, 2] padding
+  set_op({{1, 2, 2, 1}}, {{1, 1, 1, 1}}, "EXPLICIT", "NHWC",
+         {0, 0, 1, 0, 1, 2, 0, 0});
+  INFER_OK(op, "[1,3,3,1];[2,2,1,1]", "[d0_0,2,4,d1_3]");
+
+  // 4x4 input, 2x2 filter, 1x2 dilations, 2x2 stride, [3, 2, 1, 0] padding
+  set_op({{1, 1, 2, 1}}, {{1, 2, 2, 1}}, "EXPLICIT", "NHWC",
+         {0, 0, 3, 2, 1, 0, 0, 0});
+  INFER_OK(op, "[1,4,4,1];[2,2,1,1]", "[d0_0,4,2,d1_3]");
+
+  // 4x4 input, 2x2 filter, 2x2 dilations, 1x1 stride, [1, 1, 2, 2] padding
+  set_op({{1, 2, 2, 1}}, {{1, 1, 1, 1}}, "EXPLICIT", "NHWC",
+         {0, 0, 1, 1, 2, 2, 0, 0});
+  INFER_OK(op, "[1,4,4,1];[2,2,1,1]", "[d0_0,4,6,d1_3]");
 }
 
 TEST(CommonShapeFnsTest, Conv3DShapeTest) {
diff --git a/tensorflow/core/framework/ops_util.cc b/tensorflow/core/framework/ops_util.cc
index e8cf014ca0..4e603b9598 100644
--- a/tensorflow/core/framework/ops_util.cc
+++ b/tensorflow/core/framework/ops_util.cc
@@ -30,6 +30,9 @@ Eigen::PaddingType BrainPadding2EigenPadding(Padding padding) {
       return Eigen::PADDING_VALID;
     case Padding::SAME:
       return Eigen::PADDING_SAME;
+    case Padding::EXPLICIT:
+      LOG(FATAL) << "Eigen does not have explicit padding enum "  // Crash OK
+                    "value";
   }
   return Eigen::PADDING_SAME;  // Prevent compiler warning about missing return
 }
diff --git a/tensorflow/core/grappler/costs/graph_properties_testdata/large_function_graph.pbtxt b/tensorflow/core/grappler/costs/graph_properties_testdata/large_function_graph.pbtxt
index 415c347a1d..d4e23e901a 100644
--- a/tensorflow/core/grappler/costs/graph_properties_testdata/large_function_graph.pbtxt
+++ b/tensorflow/core/grappler/costs/graph_properties_testdata/large_function_graph.pbtxt
@@ -511,6 +511,13 @@ library {
           s: "VALID"
         }
       }
+      attr {
+        key: "explicit_paddings"
+        value {
+          list {
+          }
+        }
+      }
       attr {
         key: "strides"
         value {
diff --git a/tensorflow/core/grappler/optimizers/layout_optimizer.cc b/tensorflow/core/grappler/optimizers/layout_optimizer.cc
index 8f25a1c8c1..e9b706a583 100644
--- a/tensorflow/core/grappler/optimizers/layout_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/layout_optimizer.cc
@@ -503,6 +503,7 @@ class NodeProcessor : public GraphProcessor {
       UpdateAttrKSize();
       UpdateAttrStrides();
       UpdateAttrDilations();
+      UpdateAttrExplicitPaddings();
       UpdateAttrShape();
       TF_RETURN_IF_ERROR(AddLayoutTransposeToInputs());
       TF_RETURN_IF_ERROR(AddLayoutTransposeToOutputs());
@@ -753,6 +754,28 @@ class NodeProcessor : public GraphProcessor {
     }
   }
 
+  void UpdateAttrExplicitPaddings() {
+    if (node_->attr().find("explicit_paddings") != node_->attr().end()) {
+      auto list = node_->mutable_attr()->at("explicit_paddings").mutable_list();
+      int size = list->i_size();
+      if (size == 8) {
+        int64 height_before = list->i(2);
+        int64 height_after = list->i(3);
+        int64 width_before = list->i(4);
+        int64 width_after = list->i(5);
+        list->set_i(2, 0);
+        list->set_i(3, 0);
+        list->set_i(4, height_before);
+        list->set_i(5, height_after);
+        list->set_i(6, width_before);
+        list->set_i(7, width_after);
+      } else if (size != 0) {
+        LOG(ERROR) << "Cannot handle explicit_paddings attribute of size "
+                   << size;
+      }
+    }
+  }
+
   void UpdateAttrDataFormat() {
     if (node_->attr().find("data_format") != node_->attr().end()) {
       if (node_->attr().at("data_format").s().compare("NHWC") == 0) {
diff --git a/tensorflow/core/grappler/optimizers/layout_optimizer_test.cc b/tensorflow/core/grappler/optimizers/layout_optimizer_test.cc
index 20e47c1b26..eb2a8e87dd 100644
--- a/tensorflow/core/grappler/optimizers/layout_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/layout_optimizer_test.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/core/grappler/optimizers/layout_optimizer.h"
 #include "tensorflow/cc/ops/standard_ops.h"
+#include "tensorflow/core/framework/common_shape_fns.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/framework/tensor_testutil.h"
 #include "tensorflow/core/grappler/clusters/single_machine.h"
@@ -80,8 +81,13 @@ class LayoutOptimizerTest : public GrapplerTest {
     Output filter =
         ops::Const(s->WithOpName("Filter"), Input::Initializer(filter_data));
 
+    ops::Conv2D::Attrs attrs;
+    if (padding == "EXPLICIT") {
+      attrs = attrs.ExplicitPaddings({0, 0, 1, 2, 3, 4, 0, 0});
+    }
+
     Output conv = ops::Conv2D(s->WithOpName("Conv2D").WithDevice(device), input,
-                              filter, {1, stride, stride, 1}, padding);
+                              filter, {1, stride, stride, 1}, padding, attrs);
     return conv;
   }
 
@@ -100,6 +106,28 @@ class LayoutOptimizerTest : public GrapplerTest {
     int input_depth = 3;
     int filter_count = 2;
     int stride = 1;
+    int dilation = dilated ? 2 : 1;
+    int64 padding_top = 1;
+    int64 padding_bottom = 2;
+    int64 padding_left = 3;
+    int64 padding_right = 4;
+    int64 output_height;
+    int64 output_width;
+    Padding padding_enum;
+    if (padding == "SAME") {
+      padding_enum = SAME;
+    } else if (padding == "VALID") {
+      padding_enum = VALID;
+    } else {
+      CHECK_EQ(padding, "EXPLICIT");
+      padding_enum = EXPLICIT;
+    }
+    TF_CHECK_OK(GetWindowedOutputSizeVerboseV2(
+        input_height, filter_size, dilation, stride, padding_enum,
+        &output_height, &padding_top, &padding_bottom));
+    TF_CHECK_OK(GetWindowedOutputSizeVerboseV2(
+        input_width, filter_size, dilation, stride, padding_enum, &output_width,
+        &padding_left, &padding_right));
     TensorShape input_sizes_shape({4});
     Tensor input_data(DT_INT32, input_sizes_shape);
     test::FillValues<int>(&input_data,
@@ -112,8 +140,6 @@ class LayoutOptimizerTest : public GrapplerTest {
     Output filter =
         ops::Variable(s->WithOpName("Filter"), filter_shape, DT_FLOAT);
 
-    int output_height = input_height;
-    int output_width = input_width;
     TensorShape output_shape(
         {batch_size, output_height, output_width, filter_count});
     Tensor output_data(DT_FLOAT, output_shape);
@@ -124,10 +150,21 @@ class LayoutOptimizerTest : public GrapplerTest {
     Output conv_backprop_input;
     Output input_sizes_i =
         ops::Identity(s->WithOpName("InputSizesIdentity"), input_sizes);
-    ops::Conv2DBackpropInput::Attrs attrs;
-    if (dilated) {
-      attrs = attrs.Dilations({1, 2, 2, 1});
+    std::vector<int> dilations{1, dilation, dilation, 1};
+    std::vector<int> explicit_paddings;
+    if (padding == "EXPLICIT") {
+      explicit_paddings = {0,
+                           0,
+                           static_cast<int>(padding_top),
+                           static_cast<int>(padding_bottom),
+                           static_cast<int>(padding_left),
+                           static_cast<int>(padding_right),
+                           0,
+                           0};
     }
+    auto attrs =
+        ops::Conv2DBackpropInput::Attrs().Dilations(dilations).ExplicitPaddings(
+            explicit_paddings);
     if (const_input_size) {
       conv_backprop_input = ops::Conv2DBackpropInput(
           s->WithOpName("Conv2DBackpropInput"), input_sizes, filter, output,
@@ -186,7 +223,7 @@ class LayoutOptimizerTest : public GrapplerTest {
 
 TEST_F(LayoutOptimizerTest, Conv2DBackpropInput) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
-  auto conv = SimpleConv2DBackpropInput(&s, 7, 2, "SAME");
+  auto conv = SimpleConv2DBackpropInput(&s, 7, 2, "EXPLICIT");
   Output fetch = ops::Identity(s.WithOpName("Fetch"), {conv});
   GrapplerItem item;
   TF_CHECK_OK(s.ToGraphDef(&item.graph));
@@ -306,6 +343,19 @@ TEST_F(LayoutOptimizerTest, NotEqualSizeWithValidPadding) {
   EXPECT_TRUE(node_map.GetNode("Conv2D-0-TransposeNHWCToNCHW-LayoutOptimizer"));
 }
 
+TEST_F(LayoutOptimizerTest, ExplicitPadding) {
+  tensorflow::Scope s = tensorflow::Scope::NewRootScope();
+  auto conv = SimpleConv2D(&s, 4, 2, "EXPLICIT");
+  Output fetch = ops::Identity(s.WithOpName("Fetch"), {conv});
+  GrapplerItem item;
+  TF_CHECK_OK(s.ToGraphDef(&item.graph));
+  LayoutOptimizer optimizer;
+  GraphDef output;
+  Status status = optimizer.Optimize(virtual_cluster_.get(), item, &output);
+  NodeMap node_map(&output);
+  EXPECT_TRUE(node_map.GetNode("Conv2D-0-TransposeNHWCToNCHW-LayoutOptimizer"));
+}
+
 TEST_F(LayoutOptimizerTest, Pad) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
   auto conv = SimpleConv2D(&s, 4, 2, "VALID");
diff --git a/tensorflow/core/kernels/conv_grad_filter_ops.cc b/tensorflow/core/kernels/conv_grad_filter_ops.cc
index 4e3de33e83..0df05ceb02 100644
--- a/tensorflow/core/kernels/conv_grad_filter_ops.cc
+++ b/tensorflow/core/kernels/conv_grad_filter_ops.cc
@@ -102,6 +102,7 @@ struct LaunchConv2DBackpropFilterOp<CPUDevice, T> {
                   const Tensor& out_backprop, const Tensor& input,
                   int row_dilation, int col_dilation, int row_stride,
                   int col_stride, const Padding& padding,
+                  const std::vector<int64>& explicit_paddings,
                   Tensor* filter_backprop, TensorFormat data_format) {
     const CPUDevice& d = ctx->eigen_device<CPUDevice>();
     functor::SpatialConvolutionBackwardFilter<CPUDevice, T>()(
@@ -204,6 +205,15 @@ class Conv2DFastBackpropFilterOp : public OpKernel {
                 errors::InvalidArgument(
                     "Row and column strides should be larger than 0."));
     OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_));
+    OP_REQUIRES(
+        context, padding_ != Padding::EXPLICIT,
+        errors::Unimplemented("Current CPU implementation does not support "
+                              "EXPLICIT padding yet."));
+    std::vector<int64> explicit_paddings;
+    OP_REQUIRES_OK(context,
+                   context->GetAttr("explicit_paddings", &explicit_paddings));
+    OP_REQUIRES_OK(context, CheckValidPadding(padding_, explicit_paddings,
+                                              /*num_dims=*/4, data_format_));
     OP_REQUIRES_OK(context, context->GetAttr("dilations", &dilations_));
     OP_REQUIRES(context, dilations_.size() == 4,
                 errors::InvalidArgument("Sliding window dilations field must "
@@ -282,7 +292,8 @@ class Conv2DFastBackpropFilterOp : public OpKernel {
     LaunchConv2DBackpropFilterOp<Device, T>()(
         context, false, false, out_backprop, input,
         /*row_dilation=*/1, /*col_dilation=*/1, dims.spatial_dims[0].stride,
-        dims.spatial_dims[1].stride, padding_, filter_backprop, data_format_);
+        dims.spatial_dims[1].stride, padding_, /*explicit_paddings=*/{},
+        filter_backprop, data_format_);
   }
 
  private:
@@ -319,6 +330,15 @@ class Conv2DCustomBackpropFilterOp : public OpKernel {
                 errors::InvalidArgument(
                     "Row and column strides should be larger than 0."));
     OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_));
+    OP_REQUIRES(
+        context, padding_ != Padding::EXPLICIT,
+        errors::Unimplemented("Current CPU implementation does not support "
+                              "EXPLICIT padding yet."));
+    std::vector<int64> explicit_paddings;
+    OP_REQUIRES_OK(context,
+                   context->GetAttr("explicit_paddings", &explicit_paddings));
+    OP_REQUIRES_OK(context, CheckValidPadding(padding_, explicit_paddings,
+                                              /*num_dims=*/4, data_format_));
     OP_REQUIRES_OK(context, context->GetAttr("dilations", &dilations_));
     OP_REQUIRES(context, dilations_.size() == 4,
                 errors::InvalidArgument("Sliding window dilations field must "
@@ -587,6 +607,10 @@ class Conv2DSlowBackpropFilterOp : public OpKernel {
     use_cudnn_ &= CanUseCudnn();
     cudnn_use_autotune_ = CudnnUseAutotune();
     OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_));
+    OP_REQUIRES_OK(context,
+                   context->GetAttr("explicit_paddings", &explicit_paddings_));
+    OP_REQUIRES_OK(context, CheckValidPadding(padding_, explicit_paddings_,
+                                              /*num_dims=*/4, data_format_));
   }
 
   void Compute(OpKernelContext* context) override {
@@ -626,13 +650,14 @@ class Conv2DSlowBackpropFilterOp : public OpKernel {
 
     launcher_(context, use_cudnn_, cudnn_use_autotune_, out_backprop, input,
               dilation_rows, dilation_cols, stride_rows, stride_cols, padding_,
-              filter_backprop, data_format_);
+              explicit_paddings_, filter_backprop, data_format_);
   }
 
  private:
   std::vector<int32> dilations_;
   std::vector<int32> strides_;
   Padding padding_;
+  std::vector<int64> explicit_paddings_;
   bool use_cudnn_;
   TensorFormat data_format_;
   LaunchConv2DBackpropFilterOp<Device, T> launcher_;
@@ -646,7 +671,8 @@ void LaunchConv2DBackpropFilterOp<Eigen::GpuDevice, T>::operator()(
     OpKernelContext* ctx, bool use_cudnn, bool cudnn_use_autotune,
     const Tensor& out_backprop, const Tensor& input, int row_dilation,
     int col_dilation, int row_stride, int col_stride, const Padding& padding,
-    Tensor* filter_backprop, TensorFormat data_format) {
+    const std::vector<int64>& explicit_paddings, Tensor* filter_backprop,
+    TensorFormat data_format) {
   using se::dnn::AlgorithmConfig;
   using se::dnn::AlgorithmDesc;
   using se::dnn::ProfileResult;
@@ -661,35 +687,33 @@ void LaunchConv2DBackpropFilterOp<Eigen::GpuDevice, T>::operator()(
   TensorShape filter_shape = filter_backprop->shape();
 
   ConvBackpropDimensions dims;
-  OP_REQUIRES_OK(ctx, ConvBackpropComputeDimensionsV2(
-                          "Conv2DSlowBackpropFilter", /*num_spatial_dims=*/2,
-                          input.shape(), filter_shape, out_backprop.shape(),
-                          dilations, strides, padding, data_format, &dims));
-
-  // TODO(yangzihao): The padding computations should be done in
-  // GetWindowedOutputSize() functions.
-  const int padding_rows =
-      (padding == VALID)
-          ? 0
-          : std::max<int>(0, (dims.spatial_dims[0].output_size - 1) *
-                                     dims.spatial_dims[0].stride +
-                                 (dims.spatial_dims[0].filter_size - 1) *
-                                     dims.spatial_dims[0].dilation +
-                                 1 - dims.spatial_dims[0].input_size);
-  const int padding_cols =
-      (padding == VALID)
-          ? 0
-          : std::max<int>(0, (dims.spatial_dims[1].output_size - 1) *
-                                     dims.spatial_dims[1].stride +
-                                 (dims.spatial_dims[1].filter_size - 1) *
-                                     dims.spatial_dims[1].dilation +
-                                 1 - dims.spatial_dims[1].input_size);
-
-  // TODO(zhengxq): cuDNN only supports equal padding on both sides, so only
-  // calling it when that is true. Remove this check when (if?) cuDNN starts
-  // supporting different padding.
-  bool rows_odd = (padding_rows % 2 != 0);
-  bool cols_odd = (padding_cols % 2 != 0);
+  OP_REQUIRES_OK(
+      ctx, ConvBackpropComputeDimensionsV2(
+               "Conv2DSlowBackpropFilter", /*num_spatial_dims=*/2,
+               input.shape(), filter_shape, out_backprop.shape(), dilations,
+               strides, padding, explicit_paddings, data_format, &dims));
+
+  int64 padding_top = -1, padding_bottom = -1;
+  int64 padding_left = -1, padding_right = -1;
+  if (padding == EXPLICIT) {
+    GetExplicitPaddingForDim(explicit_paddings, data_format, 'H', &padding_top,
+                             &padding_bottom);
+    GetExplicitPaddingForDim(explicit_paddings, data_format, 'W', &padding_left,
+                             &padding_right);
+  }
+  int64 expected_out_rows, expected_out_cols;
+  // The function is guaranteed to succeed because we checked the output and
+  // padding was valid earlier.
+  TF_CHECK_OK(GetWindowedOutputSizeVerboseV2(
+      dims.spatial_dims[0].input_size, dims.spatial_dims[0].filter_size,
+      row_dilation, row_stride, padding, &expected_out_rows, &padding_top,
+      &padding_bottom));
+  DCHECK_EQ(dims.spatial_dims[0].output_size, expected_out_rows);
+  TF_CHECK_OK(GetWindowedOutputSizeVerboseV2(
+      dims.spatial_dims[1].input_size, dims.spatial_dims[1].filter_size,
+      col_dilation, col_stride, padding, &expected_out_cols, &padding_left,
+      &padding_right));
+  DCHECK_EQ(dims.spatial_dims[1].output_size, expected_out_cols);
 
   auto* stream = ctx->op_device_context()->stream();
   OP_REQUIRES(ctx, stream, errors::Internal("No GPU stream available."));
@@ -711,7 +735,7 @@ void LaunchConv2DBackpropFilterOp<Eigen::GpuDevice, T>::operator()(
       dims.spatial_dims[0].filter_size == 1 &&
       dims.spatial_dims[1].filter_size == 1 && !is_grouped_convolution &&
       dims.spatial_dims[0].stride == 1 && dims.spatial_dims[1].stride == 1 &&
-      data_format == FORMAT_NHWC) {
+      data_format == FORMAT_NHWC && (padding == VALID || padding == SAME)) {
     const uint64 m = dims.in_depth;
     const uint64 k = dims.batch_size * dims.spatial_dims[0].input_size *
                      dims.spatial_dims[1].input_size;
@@ -779,31 +803,43 @@ void LaunchConv2DBackpropFilterOp<Eigen::GpuDevice, T>::operator()(
     return;
   }
 
+  const int64 common_padding_rows = std::min(padding_top, padding_bottom);
+  const int64 common_padding_cols = std::min(padding_left, padding_right);
   Tensor compatible_input;
-  if (rows_odd || cols_odd) {
-    // If a padding dimension is odd, we have one more element on the right
-    // side or the bottom side. This is unsupported in cudnn. Therefore,
-    // we pad that extra element and make it compatible.
+  if (padding_top != padding_bottom || padding_left != padding_right) {
+    // Pad the input in the same way we did during the forward pass, so that
+    // cuDNN receives the same input during the backward pass function as it did
+    // during the forward pass function.
+    const int64 padding_rows_diff = std::abs(padding_bottom - padding_top);
+    const int64 padding_cols_diff = std::abs(padding_right - padding_left);
+    const int64 new_in_rows =
+        dims.spatial_dims[0].input_size + padding_rows_diff;
+    const int64 new_in_cols =
+        dims.spatial_dims[1].input_size + padding_cols_diff;
+    const int64 input_pad_top = padding_top - common_padding_rows;
+    const int64 input_pad_bottom = padding_bottom - common_padding_rows;
+    const int64 input_pad_left = padding_left - common_padding_cols;
+    const int64 input_pad_right = padding_right - common_padding_cols;
     OP_REQUIRES_OK(
         ctx, ctx->allocate_temp(
                  DataTypeToEnum<T>::value,
-                 ShapeFromFormat(data_format, dims.batch_size,
-                                 dims.spatial_dims[0].input_size + rows_odd,
-                                 dims.spatial_dims[1].input_size + cols_odd,
-                                 dims.in_depth),
+                 ShapeFromFormat(data_format, dims.batch_size, new_in_rows,
+                                 new_in_cols, dims.in_depth),
                  &compatible_input));
 
     functor::PadInput<GPUDevice, T, int, 4>()(
         ctx->template eigen_device<GPUDevice>(), To32Bit(input.tensor<T, 4>()),
-        {{0, 0}}, {{rows_odd, cols_odd}},
+        {{static_cast<int>(input_pad_top), static_cast<int>(input_pad_left)}},
+        {{static_cast<int>(input_pad_bottom),
+          static_cast<int>(input_pad_right)}},
         To32Bit(compatible_input.tensor<T, 4>()), data_format);
   } else {
     compatible_input = input;
   }
 
-  CHECK(padding_rows >= 0 && padding_cols >= 0)
-      << "Negative row or col paddings: (" << padding_rows << ", "
-      << padding_cols << ")";
+  CHECK(common_padding_rows >= 0 && common_padding_cols >= 0)  // Crash OK
+      << "Negative row or col paddings: (" << common_padding_rows << ", "
+      << common_padding_cols << ")";
   se::dnn::BatchDescriptor input_desc;
   input_desc.set_count(dims.batch_size)
       .set_height(GetTensorDim(compatible_input, data_format, 'H'))
@@ -826,8 +862,8 @@ void LaunchConv2DBackpropFilterOp<Eigen::GpuDevice, T>::operator()(
       .set_horizontal_dilation_rate(dims.spatial_dims[1].dilation)
       .set_vertical_filter_stride(dims.spatial_dims[0].stride)
       .set_horizontal_filter_stride(dims.spatial_dims[1].stride)
-      .set_zero_padding_height(padding_rows / 2)
-      .set_zero_padding_width(padding_cols / 2)
+      .set_zero_padding_height(common_padding_rows)
+      .set_zero_padding_width(common_padding_cols)
       .set_group_count(dims.in_depth / filter_shape.dim_size(2));
 
   // NOTE(zhengxq):
@@ -922,8 +958,8 @@ void LaunchConv2DBackpropFilterOp<Eigen::GpuDevice, T>::operator()(
         dims.spatial_dims[1].dilation}},   // dilation_cols
       {{dims.spatial_dims[0].stride,       // stride_rows
         dims.spatial_dims[1].stride}},     // stride_cols
-      {{padding_rows,                      // padding_rows
-        padding_cols}},                    // padding_cols
+      {{common_padding_rows,               // padding_rows
+        common_padding_cols}},             // padding_cols
       dtype,                               // tensor datatype
       device_id,                           // device_id
   };
diff --git a/tensorflow/core/kernels/conv_grad_input_ops.cc b/tensorflow/core/kernels/conv_grad_input_ops.cc
index 9f983ed816..74b97b9864 100644
--- a/tensorflow/core/kernels/conv_grad_input_ops.cc
+++ b/tensorflow/core/kernels/conv_grad_input_ops.cc
@@ -106,8 +106,9 @@ struct LaunchConv2DBackpropInputOp<CPUDevice, T> {
   void operator()(OpKernelContext* ctx, bool use_cudnn, bool cudnn_use_autotune,
                   const Tensor& out_backprop, const Tensor& filter,
                   int row_dilation, int col_dilation, int row_stride,
-                  int col_stride, const Padding& padding, Tensor* in_backprop,
-                  TensorFormat data_format) {
+                  int col_stride, const Padding& padding,
+                  const std::vector<int64>& explicit_paddings,
+                  Tensor* in_backprop, TensorFormat data_format) {
     const CPUDevice& d = ctx->eigen_device<CPUDevice>();
     functor::SpatialConvolutionBackwardInput<CPUDevice, T>()(
         d, in_backprop->tensor<T, 4>(), filter.tensor<T, 4>(),
@@ -220,6 +221,15 @@ class Conv2DFastBackpropInputOp : public OpKernel {
                 errors::InvalidArgument(
                     "Current Eigen and libxsmm implementations do not "
                     "yet support dilation rates larger than 1."));
+    OP_REQUIRES(
+        context, padding_ != Padding::EXPLICIT,
+        errors::Unimplemented("Current CPU implementation does not support "
+                              "EXPLICIT padding yet."));
+    std::vector<int64> explicit_paddings;
+    OP_REQUIRES_OK(context,
+                   context->GetAttr("explicit_paddings", &explicit_paddings));
+    OP_REQUIRES_OK(context, CheckValidPadding(padding_, explicit_paddings,
+                                              /*num_dims=*/4, data_format_));
   }
 
   void Compute(OpKernelContext* context) override {
@@ -286,7 +296,8 @@ class Conv2DFastBackpropInputOp : public OpKernel {
     LaunchConv2DBackpropInputOp<Device, T>()(
         context, false, false, out_backprop, filter,
         /*row_dilation=*/1, /*col_dilation=*/1, dims.spatial_dims[0].stride,
-        dims.spatial_dims[1].stride, padding_, in_backprop, data_format_);
+        dims.spatial_dims[1].stride, padding_, /*explicit_paddings=*/{},
+        in_backprop, data_format_);
   }
 
  private:
@@ -336,6 +347,15 @@ class Conv2DCustomBackpropInputOp : public OpKernel {
                 errors::InvalidArgument(
                     "Current libxsmm and customized CPU implementations do "
                     "not yet support dilation rates larger than 1."));
+    OP_REQUIRES(
+        context, padding_ != Padding::EXPLICIT,
+        errors::Unimplemented("Current CPU implementation does not support "
+                              "EXPLICIT padding yet."));
+    std::vector<int64> explicit_paddings;
+    OP_REQUIRES_OK(context,
+                   context->GetAttr("explicit_paddings", &explicit_paddings));
+    OP_REQUIRES_OK(context, CheckValidPadding(padding_, explicit_paddings,
+                                              /*num_dims=*/4, data_format_));
   }
 
   void Compute(OpKernelContext* context) override {
@@ -661,6 +681,16 @@ class Conv2DSlowBackpropInputOp : public OpKernel {
     use_cudnn_ &= CanUseCudnn();
     cudnn_use_autotune_ = CudnnUseAutotune();
     OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_));
+    if (!std::is_same<Device, GPUDevice>::value) {
+      OP_REQUIRES(
+          context, padding_ != Padding::EXPLICIT,
+          errors::Unimplemented("Current CPU implementation does not support "
+                                "EXPLICIT padding yet."));
+    }
+    OP_REQUIRES_OK(context,
+                   context->GetAttr("explicit_paddings", &explicit_paddings_));
+    OP_REQUIRES_OK(context, CheckValidPadding(padding_, explicit_paddings_,
+                                              /*num_dims=*/4, data_format_));
   }
 
   void Compute(OpKernelContext* context) override {
@@ -694,13 +724,14 @@ class Conv2DSlowBackpropInputOp : public OpKernel {
 
     launcher_(context, use_cudnn_, cudnn_use_autotune_, out_backprop, filter,
               dilation_rows, dilation_cols, stride_rows, stride_cols, padding_,
-              in_backprop, data_format_);
+              explicit_paddings_, in_backprop, data_format_);
   }
 
  private:
   std::vector<int32> dilations_;
   std::vector<int32> strides_;
   Padding padding_;
+  std::vector<int64> explicit_paddings_;
   bool use_cudnn_;
   TensorFormat data_format_;
   LaunchConv2DBackpropInputOp<Device, T> launcher_;
@@ -714,7 +745,8 @@ void LaunchConv2DBackpropInputOp<GPUDevice, T>::operator()(
     OpKernelContext* ctx, bool use_cudnn, bool cudnn_use_autotune,
     const Tensor& out_backprop, const Tensor& filter, int row_dilation,
     int col_dilation, int row_stride, int col_stride, const Padding& padding,
-    Tensor* in_backprop, TensorFormat data_format) {
+    const std::vector<int64>& explicit_paddings, Tensor* in_backprop,
+    TensorFormat data_format) {
   using se::dnn::AlgorithmConfig;
   using se::dnn::AlgorithmDesc;
   using se::dnn::ProfileResult;
@@ -731,35 +763,33 @@ void LaunchConv2DBackpropInputOp<GPUDevice, T>::operator()(
 
   const TensorShape& filter_shape = filter.shape();
   ConvBackpropDimensions dims;
-  OP_REQUIRES_OK(ctx, ConvBackpropComputeDimensionsV2(
-                          "Conv2DSlowBackpropInput", /*num_spatial_dims=*/2,
-                          input_shape, filter_shape, out_backprop.shape(),
-                          dilations, strides, padding, data_format, &dims));
-
-  // TODO(yangzihao): The padding computations should be done in
-  // GetWindowedOutputSize() functions.
-  const int padding_rows =
-      (padding == VALID)
-          ? 0
-          : std::max<int>(0, (dims.spatial_dims[0].output_size - 1) *
-                                     dims.spatial_dims[0].stride +
-                                 (dims.spatial_dims[0].filter_size - 1) *
-                                     dims.spatial_dims[0].dilation +
-                                 1 - dims.spatial_dims[0].input_size);
-  const int padding_cols =
-      (padding == VALID)
-          ? 0
-          : std::max<int>(0, (dims.spatial_dims[1].output_size - 1) *
-                                     dims.spatial_dims[1].stride +
-                                 (dims.spatial_dims[1].filter_size - 1) *
-                                     dims.spatial_dims[1].dilation +
-                                 1 - dims.spatial_dims[1].input_size);
-
-  // TODO(keveman): cuDNN only supports equal padding on both sides, so only
-  // calling it when that is true. Remove this check when (if?) cuDNN starts
-  // supporting different padding.
-  bool rows_odd = (padding_rows % 2 != 0);
-  bool cols_odd = (padding_cols % 2 != 0);
+  OP_REQUIRES_OK(
+      ctx, ConvBackpropComputeDimensionsV2(
+               "Conv2DSlowBackpropInput", /*num_spatial_dims=*/2, input_shape,
+               filter_shape, out_backprop.shape(), dilations, strides, padding,
+               explicit_paddings, data_format, &dims));
+
+  int64 padding_top = -1, padding_bottom = -1;
+  int64 padding_left = -1, padding_right = -1;
+  if (padding == EXPLICIT) {
+    GetExplicitPaddingForDim(explicit_paddings, data_format, 'H', &padding_top,
+                             &padding_bottom);
+    GetExplicitPaddingForDim(explicit_paddings, data_format, 'W', &padding_left,
+                             &padding_right);
+  }
+  int64 expected_out_rows, expected_out_cols;
+  // The function is guaranteed to succeed because we checked the output and
+  // padding was valid earlier.
+  TF_CHECK_OK(GetWindowedOutputSizeVerboseV2(
+      dims.spatial_dims[0].input_size, dims.spatial_dims[0].filter_size,
+      row_dilation, row_stride, padding, &expected_out_rows, &padding_top,
+      &padding_bottom));
+  DCHECK_EQ(dims.spatial_dims[0].output_size, expected_out_rows);
+  TF_CHECK_OK(GetWindowedOutputSizeVerboseV2(
+      dims.spatial_dims[1].input_size, dims.spatial_dims[1].filter_size,
+      col_dilation, col_stride, padding, &expected_out_cols, &padding_left,
+      &padding_right));
+  DCHECK_EQ(dims.spatial_dims[1].output_size, expected_out_cols);
 
   auto* stream = ctx->op_device_context()->stream();
   OP_REQUIRES(ctx, stream, errors::Internal("No GPU stream available."));
@@ -779,7 +809,7 @@ void LaunchConv2DBackpropInputOp<GPUDevice, T>::operator()(
   if (dims.spatial_dims[0].filter_size == 1 &&
       dims.spatial_dims[1].filter_size == 1 && !is_grouped_convolution &&
       dims.spatial_dims[0].stride == 1 && dims.spatial_dims[1].stride == 1 &&
-      data_format == FORMAT_NHWC) {
+      data_format == FORMAT_NHWC && (padding == VALID || padding == SAME)) {
     // 1x1 filter, so call cublas directly.
     const uint64 m = dims.batch_size * dims.spatial_dims[0].input_size *
                      dims.spatial_dims[1].input_size;
@@ -841,22 +871,28 @@ void LaunchConv2DBackpropInputOp<GPUDevice, T>::operator()(
     return;
   }
 
+  const int64 common_padding_rows = std::min(padding_top, padding_bottom);
+  const int64 common_padding_cols = std::min(padding_left, padding_right);
   TensorShape compatible_input_shape;
-  if (rows_odd || cols_odd) {
-    // If a padding dimension is odd, we have one more element on the right
-    // side or the bottom side. This is unsupported in cudnn. Therefore,
-    // we pad that extra element and make it compatible.
+  if (padding_top != padding_bottom || padding_left != padding_right) {
+    // Pad the input in the same way we did during the forward pass, so that
+    // cuDNN receives the same input during the backward pass function as it did
+    // during the forward pass function.
+    const int64 padding_rows_diff = std::abs(padding_bottom - padding_top);
+    const int64 padding_cols_diff = std::abs(padding_right - padding_left);
+    const int64 new_in_rows =
+        dims.spatial_dims[0].input_size + padding_rows_diff;
+    const int64 new_in_cols =
+        dims.spatial_dims[1].input_size + padding_cols_diff;
     compatible_input_shape = ShapeFromFormat(
-        data_format, dims.batch_size,
-        dims.spatial_dims[0].input_size + rows_odd,
-        dims.spatial_dims[1].input_size + cols_odd, dims.in_depth);
+        data_format, dims.batch_size, new_in_rows, new_in_cols, dims.in_depth);
   } else {
     compatible_input_shape = input_shape;
   }
 
-  CHECK(padding_rows >= 0 && padding_cols >= 0)
-      << "Negative row or col paddings: (" << padding_rows << ", "
-      << padding_cols << ")";
+  CHECK(common_padding_rows >= 0 && common_padding_cols >= 0)  // Crash OK
+      << "Negative row or col paddings: (" << common_padding_rows << ", "
+      << common_padding_cols << ")";
   se::dnn::BatchDescriptor input_desc;
   input_desc.set_count(dims.batch_size)
       .set_height(GetTensorDim(compatible_input_shape, data_format, 'H'))
@@ -879,8 +915,8 @@ void LaunchConv2DBackpropInputOp<GPUDevice, T>::operator()(
       .set_horizontal_dilation_rate(dims.spatial_dims[1].dilation)
       .set_vertical_filter_stride(dims.spatial_dims[0].stride)
       .set_horizontal_filter_stride(dims.spatial_dims[1].stride)
-      .set_zero_padding_height(padding_rows / 2)
-      .set_zero_padding_width(padding_cols / 2)
+      .set_zero_padding_height(common_padding_rows)
+      .set_zero_padding_width(common_padding_cols)
       .set_group_count(dims.in_depth / filter_shape.dim_size(2));
 
   // NOTE(keveman):
@@ -971,8 +1007,8 @@ void LaunchConv2DBackpropInputOp<GPUDevice, T>::operator()(
         dims.spatial_dims[1].dilation}},   // dilation_cols
       {{dims.spatial_dims[0].stride,       // stride_rows
         dims.spatial_dims[1].stride}},     // stride_cols
-      {{padding_rows,                      // padding_rows
-        padding_cols}},                    // padding_cols
+      {{common_padding_rows,               // padding_rows
+        common_padding_cols}},             // padding_cols
       dtype,                               // tensor data type
       device_id,                           // device_id
   };
@@ -1041,7 +1077,7 @@ void LaunchConv2DBackpropInputOp<GPUDevice, T>::operator()(
     return;
   }
 
-  if (rows_odd || cols_odd) {
+  if (padding_top != padding_bottom || padding_left != padding_right) {
     Tensor in_backprop_remove_padding;
     OP_REQUIRES_OK(
         ctx, ctx->allocate_temp(
@@ -1053,12 +1089,18 @@ void LaunchConv2DBackpropInputOp<GPUDevice, T>::operator()(
                                  GetTensorDim(input_shape, data_format, 'C')),
                  &in_backprop_remove_padding));
 
-    // Remove the padding for odd rows or cols.
+    // Remove the padding that was added to the input shape above.
+    const int64 input_pad_top = padding_top - common_padding_rows;
+    const int64 input_pad_bottom = padding_bottom - common_padding_rows;
+    const int64 input_pad_left = padding_left - common_padding_cols;
+    const int64 input_pad_right = padding_right - common_padding_cols;
     functor::PadInput<GPUDevice, T, int, 4>()(
         ctx->template eigen_device<GPUDevice>(),
         To32Bit(const_cast<const Tensor&>(pre_transformed_in_backprop)
                     .tensor<T, 4>()),
-        {{0, 0}}, {{-rows_odd, -cols_odd}},
+        {{static_cast<int>(-input_pad_top), static_cast<int>(-input_pad_left)}},
+        {{static_cast<int>(-input_pad_bottom),
+          static_cast<int>(-input_pad_right)}},
         To32Bit(in_backprop_remove_padding.tensor<T, 4>()), FORMAT_NCHW);
 
     pre_transformed_in_backprop = in_backprop_remove_padding;
diff --git a/tensorflow/core/kernels/conv_grad_ops.cc b/tensorflow/core/kernels/conv_grad_ops.cc
index 507720c998..0fd7550830 100644
--- a/tensorflow/core/kernels/conv_grad_ops.cc
+++ b/tensorflow/core/kernels/conv_grad_ops.cc
@@ -52,24 +52,23 @@ int ConvBackpropDimensions::SpatialPadding(const Padding& padding,
                                        1 - input_size(dim)));
 }
 
-// The V2 version computes windowed output size with arbitrary dilation_rate,
-// while the original version only handles the cases where dilation_rates equal
-// to 1.
-Status ConvBackpropExtractAndVerifyDimensionV2(
+namespace {
+
+Status ConvBackpropExtractAndVerifyDimension(
     StringPiece label, const TensorShape& input_shape,
     const TensorShape& filter_shape, const TensorShape& output_shape,
     const gtl::ArraySlice<int32>& dilations, const std::vector<int32>& strides,
-    Padding padding, int spatial_dim, int filter_spatial_dim,
-    ConvBackpropSpatialDimension* dim) {
+    Padding padding, int64 padding_before, int64 padding_after, int spatial_dim,
+    int filter_spatial_dim, ConvBackpropSpatialDimension* dim) {
   dim->input_size = input_shape.dim_size(spatial_dim);
   dim->filter_size = filter_shape.dim_size(filter_spatial_dim);
   dim->output_size = output_shape.dim_size(spatial_dim);
   dim->stride = strides[spatial_dim];
   dim->dilation = dilations[spatial_dim];
-  int64 out_size = 0, pad_size = 0;
-  TF_RETURN_IF_ERROR(GetWindowedOutputSizeV2(dim->input_size, dim->filter_size,
-                                             dim->dilation, dim->stride,
-                                             padding, &out_size, &pad_size));
+  int64 out_size = 0;
+  TF_RETURN_IF_ERROR(GetWindowedOutputSizeVerboseV2(
+      dim->input_size, dim->filter_size, dim->dilation, dim->stride, padding,
+      &out_size, &padding_before, &padding_after));
   if (dim->output_size != out_size) {
     return errors::InvalidArgument(
         label, ": Size of out_backprop doesn't match computed: ", "actual = ",
@@ -79,10 +78,13 @@ Status ConvBackpropExtractAndVerifyDimensionV2(
         " stride: ", dim->stride, " dilation: ", dim->dilation);
   }
 
+  // TODO(reedwm): Correctly handle explicit padding here. The rest of the
+  // fields set on 'dim' are only used in XLA. TensorFlow ops do not yet support
+  // explicit padding for XLA.
   int64 effective_filter_size = (dim->filter_size - 1) * dim->dilation + 1;
   dim->expanded_output_size = (dim->output_size - 1) * dim->stride + 1;
   const auto padded_out_size = dim->input_size + effective_filter_size - 1;
-  dim->pad_before = effective_filter_size - 1 - pad_size;
+  dim->pad_before = effective_filter_size - 1 - padding_before;
   dim->pad_after =
       padded_out_size - dim->expanded_output_size - dim->pad_before;
   VLOG(2) << label << ": expanded_out = " << dim->expanded_output_size
@@ -94,22 +96,14 @@ Status ConvBackpropExtractAndVerifyDimensionV2(
   return Status::OK();
 }
 
-Status ConvBackpropExtractAndVerifyDimension(
-    StringPiece label, const TensorShape& input_shape,
-    const TensorShape& filter_shape, const TensorShape& output_shape,
-    const std::vector<int32>& strides, Padding padding, int spatial_dim,
-    int filter_spatial_dim, ConvBackpropSpatialDimension* dim) {
-  static constexpr std::array<int32, 5> one_dilations = {{1, 1, 1, 1, 1}};
-  return ConvBackpropExtractAndVerifyDimensionV2(
-      label, input_shape, filter_shape, output_shape, one_dilations, strides,
-      padding, spatial_dim, filter_spatial_dim, dim);
-}
+}  // namespace
 
 Status ConvBackpropComputeDimensionsV2(
     StringPiece label, int num_spatial_dims, const TensorShape& input_shape,
     const TensorShape& filter_shape, const TensorShape& out_backprop_shape,
     const gtl::ArraySlice<int32>& dilations, const std::vector<int32>& strides,
-    Padding padding, TensorFormat data_format, ConvBackpropDimensions* dims) {
+    Padding padding, const std::vector<int64>& explicit_paddings,
+    TensorFormat data_format, ConvBackpropDimensions* dims) {
   // The + 2 in the following line is for the batch and feature dimensions.
   const int num_dims = num_spatial_dims + 2;
   if (input_shape.dims() != num_dims) {
@@ -152,9 +146,15 @@ Status ConvBackpropComputeDimensionsV2(
   dims->spatial_dims.resize(num_spatial_dims);
   for (int i = 0; i < num_spatial_dims; ++i) {
     int image_dim = GetTensorSpatialDimIndex(num_dims, data_format, i);
-    TF_RETURN_IF_ERROR(ConvBackpropExtractAndVerifyDimensionV2(
+    int64 padding_before = -1, padding_after = -1;
+    if (padding == EXPLICIT) {
+      padding_before = explicit_paddings[2 * image_dim];
+      padding_after = explicit_paddings[2 * image_dim + 1];
+    }
+    TF_RETURN_IF_ERROR(ConvBackpropExtractAndVerifyDimension(
         label, input_shape, filter_shape, out_backprop_shape, dilations,
-        strides, padding, image_dim, i, &dims->spatial_dims[i]));
+        strides, padding, padding_before, padding_after, image_dim, i,
+        &dims->spatial_dims[i]));
   }
   return Status::OK();
 }
@@ -169,7 +169,8 @@ Status ConvBackpropComputeDimensions(StringPiece label, int num_spatial_dims,
   static constexpr std::array<int32, 5> one_dilations = {{1, 1, 1, 1, 1}};
   return ConvBackpropComputeDimensionsV2(
       label, num_spatial_dims, input_shape, filter_shape, out_backprop_shape,
-      one_dilations, strides, padding, data_format, dims);
+      one_dilations, strides, padding, /*explicit_paddings=*/{}, data_format,
+      dims);
 }
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/conv_grad_ops.h b/tensorflow/core/kernels/conv_grad_ops.h
index 9551959463..c8e8cf28c5 100644
--- a/tensorflow/core/kernels/conv_grad_ops.h
+++ b/tensorflow/core/kernels/conv_grad_ops.h
@@ -176,8 +176,9 @@ struct LaunchConv2DBackpropInputOp {
   void operator()(OpKernelContext* ctx, bool use_cudnn, bool cudnn_use_autotune,
                   const Tensor& out_backprop, const Tensor& filter,
                   int row_dilation, int col_dilation, int row_stride,
-                  int col_stride, const Padding& padding, Tensor* in_backprop,
-                  TensorFormat data_format);
+                  int col_stride, const Padding& padding,
+                  const std::vector<int64>& explicit_paddings,
+                  Tensor* in_backprop, TensorFormat data_format);
 };
 
 template <typename Device, typename T>
@@ -186,6 +187,7 @@ struct LaunchConv2DBackpropFilterOp {
                   const Tensor& out_backprop, const Tensor& input,
                   int row_dilation, int col_dilation, int row_stride,
                   int col_stride, const Padding& padding,
+                  const std::vector<int64>& explicit_paddings,
                   Tensor* filter_backprop, TensorFormat data_format);
 };
 
@@ -195,7 +197,8 @@ struct LaunchConv2DBackpropInputOp<Eigen::GpuDevice, T> {
   void operator()(OpKernelContext* ctx, bool use_cudnn, bool cudnn_use_autotune,
                   const Tensor& input, const Tensor& filter, int row_dilation,
                   int col_dilation, int row_stride, int col_stride,
-                  const Padding& padding, Tensor* output,
+                  const Padding& padding,
+                  const std::vector<int64>& explicit_paddings, Tensor* output,
                   TensorFormat data_format);
 };
 
@@ -205,6 +208,7 @@ struct LaunchConv2DBackpropFilterOp<Eigen::GpuDevice, T> {
                   const Tensor& out_backprop, const Tensor& input,
                   int row_dilation, int col_dilation, int row_stride,
                   int col_stride, const Padding& padding,
+                  const std::vector<int64>& explicit_paddings,
                   Tensor* filter_backprop, TensorFormat data_format);
 };
 #endif  // GOOGLE_CUDA
@@ -217,6 +221,8 @@ struct ConvBackpropSpatialDimension {
   int64 output_size;
   int64 stride;
   int64 dilation;
+
+  // The following fields are valid only if the padding is not EXPLICIT.
   int64 expanded_output_size;
 
   // Number of padding elements to be added before/after this dimension of
@@ -248,7 +254,7 @@ struct ConvBackpropDimensions {
 
 // Common code between implementations of Conv?DBackpropInput and
 // Conv?DBackpropFilter. Verifies that the dimensions all match, and computes
-// sizes/padding for the spatial dimensions.
+// sizes/padding for the spatial dimensions. Does not support explicit padding.
 Status ConvBackpropComputeDimensions(StringPiece label, int num_spatial_dims,
                                      const TensorShape& input_shape,
                                      const TensorShape& filter_shape,
@@ -257,13 +263,15 @@ Status ConvBackpropComputeDimensions(StringPiece label, int num_spatial_dims,
                                      Padding padding, TensorFormat data_format,
                                      ConvBackpropDimensions* dims);
 
-// The V2 version computes the same outputs with arbitrary dilation rate.
+// The V2 version computes the same outputs with arbitrary dilation rate and
+// supports explicit padding.
 // TODO(b/67112639): Merge V2 versions and the original versions eventually.
 Status ConvBackpropComputeDimensionsV2(
     StringPiece label, int num_spatial_dims, const TensorShape& input_shape,
     const TensorShape& filter_shape, const TensorShape& out_backprop_shape,
     const gtl::ArraySlice<int32>& dilations, const std::vector<int32>& strides,
-    Padding padding, TensorFormat data_format, ConvBackpropDimensions* dims);
+    Padding padding, const std::vector<int64>& explicit_paddings,
+    TensorFormat data_format, ConvBackpropDimensions* dims);
 }  // namespace tensorflow
 
 #endif  // TENSORFLOW_CORE_KERNELS_CONV_GRAD_OPS_H_
diff --git a/tensorflow/core/kernels/conv_grad_ops_3d.cc b/tensorflow/core/kernels/conv_grad_ops_3d.cc
index 562a9c8aed..ca46da6ba3 100644
--- a/tensorflow/core/kernels/conv_grad_ops_3d.cc
+++ b/tensorflow/core/kernels/conv_grad_ops_3d.cc
@@ -1152,11 +1152,11 @@ class Conv3DBackpropInputOp<GPUDevice, T> : public OpKernel {
     }
 
     ConvBackpropDimensions dims;
-    OP_REQUIRES_OK(context,
-                   ConvBackpropComputeDimensionsV2(
-                       "Conv3DBackpropInputOp", /*num_spatial_dims=*/3,
-                       input_shape, filter_shape, out_backprop_shape, dilation_,
-                       stride_, padding_, data_format_, &dims));
+    OP_REQUIRES_OK(context, ConvBackpropComputeDimensionsV2(
+                                "Conv3DBackpropInputOp", /*num_spatial_dims=*/3,
+                                input_shape, filter_shape, out_backprop_shape,
+                                dilation_, stride_, padding_,
+                                /*explicit_paddings=*/{}, data_format_, &dims));
 
     Tensor* in_backprop;
     OP_REQUIRES_OK(context,
@@ -1537,11 +1537,12 @@ class Conv3DBackpropFilterOp<GPUDevice, T> : public OpKernel {
     }
 
     ConvBackpropDimensions dims;
-    OP_REQUIRES_OK(context,
-                   ConvBackpropComputeDimensionsV2(
-                       "Conv3DBackpropFilterOp", /*num_spatial_dims=*/3,
-                       input_shape, filter_shape, out_backprop_shape, dilation_,
-                       stride_, padding_, data_format_, &dims));
+    OP_REQUIRES_OK(
+        context,
+        ConvBackpropComputeDimensionsV2(
+            "Conv3DBackpropFilterOp", /*num_spatial_dims=*/3, input_shape,
+            filter_shape, out_backprop_shape, dilation_, stride_, padding_,
+            /*explicit_paddings=*/{}, data_format_, &dims));
 
     Tensor* filter_backprop;
     OP_REQUIRES_OK(context,
diff --git a/tensorflow/core/kernels/conv_ops.cc b/tensorflow/core/kernels/conv_ops.cc
index dfba15792d..a8138fd0a7 100644
--- a/tensorflow/core/kernels/conv_ops.cc
+++ b/tensorflow/core/kernels/conv_ops.cc
@@ -122,7 +122,8 @@ struct LaunchConv2DOp<CPUDevice, T> {
   void operator()(OpKernelContext* ctx, bool use_cudnn, bool cudnn_use_autotune,
                   const Tensor& input, const Tensor& filter, int row_dilation,
                   int col_dilation, int row_stride, int col_stride,
-                  const Padding& padding, Tensor* output,
+                  const Padding& padding,
+                  const std::vector<int64>& explicit_paddings, Tensor* output,
                   TensorFormat data_format) {
     if (data_format != FORMAT_NHWC) {
       ctx->SetStatus(
@@ -130,6 +131,11 @@ struct LaunchConv2DOp<CPUDevice, T> {
                                 "NHWC tensor format for now."));
       return;
     }
+    // TODO(reedwm): Enable explicit padding on the CPU.
+    OP_REQUIRES(
+        ctx, padding != Padding::EXPLICIT,
+        errors::Unimplemented("Generic conv implementation does not support "
+                              "EXPLICIT padding yet."));
     const int64 in_depth = GetTensorDim(input, data_format, 'C');
     OP_REQUIRES(ctx, in_depth == filter.dim_size(2),
                 errors::Unimplemented("Generic conv implementation does not "
@@ -274,6 +280,10 @@ Status InitConv2DParameters(const OpKernelConstruction* context,
   TF_RETURN_IF_ERROR(context->GetAttr("dilations", &params->dilations));
   TF_RETURN_IF_ERROR(context->GetAttr("strides", &params->strides));
   TF_RETURN_IF_ERROR(context->GetAttr("padding", &params->padding));
+  if (context->HasAttr("explicit_paddings")) {
+    TF_RETURN_IF_ERROR(
+        context->GetAttr("explicit_paddings", &params->explicit_paddings));
+  }
   string data_format_string;
   TF_RETURN_IF_ERROR(context->GetAttr("data_format", &data_format_string));
   TF_REQUIRES(FormatFromString(data_format_string, &params->data_format),
@@ -313,6 +323,10 @@ Status InitConv2DParameters(const OpKernelConstruction* context,
       dilation_h > 0 && dilation_w > 0,
       errors::InvalidArgument("Dilated rates should be larger than 0."));
 
+  TF_RETURN_IF_ERROR(CheckValidPadding(params->padding,
+                                       params->explicit_paddings,
+                                       /*num_dims=*/4, data_format));
+
   return Status::OK();
 }
 
@@ -381,14 +395,22 @@ Status ComputeConv2DDimension(const Conv2DParameters& params,
   const int dilation_cols =
       GetTensorDim(params.dilations, params.data_format, 'W');
 
+  int64 pad_rows_before, pad_rows_after, pad_cols_before, pad_cols_after;
+  if (params.padding == Padding::EXPLICIT) {
+    GetExplicitPaddingForDim(params.explicit_paddings, params.data_format, 'H',
+                             &pad_rows_before, &pad_rows_after);
+    GetExplicitPaddingForDim(params.explicit_paddings, params.data_format, 'W',
+                             &pad_cols_before, &pad_cols_after);
+  }
+
   // Compute windowed output sizes for rows and columns.
-  int64 out_rows = 0, out_cols = 0, pad_rows = 0, pad_cols = 0;
-  TF_RETURN_IF_ERROR(GetWindowedOutputSizeV2(
+  int64 out_rows = 0, out_cols = 0;
+  TF_RETURN_IF_ERROR(GetWindowedOutputSizeVerboseV2(
       input_rows, filter_rows, dilation_rows, stride_rows, params.padding,
-      &out_rows, &pad_rows));
-  TF_RETURN_IF_ERROR(GetWindowedOutputSizeV2(
+      &out_rows, &pad_rows_before, &pad_rows_after));
+  TF_RETURN_IF_ERROR(GetWindowedOutputSizeVerboseV2(
       input_cols, filter_cols, dilation_cols, stride_cols, params.padding,
-      &out_cols, &pad_cols));
+      &out_cols, &pad_cols_before, &pad_cols_after));
 
   dimensions->batch = batch;
   dimensions->input_rows = input_rows;
@@ -404,8 +426,10 @@ Status ComputeConv2DDimension(const Conv2DParameters& params,
   dimensions->dilation_cols = dilation_cols;
   dimensions->out_rows = out_rows;
   dimensions->out_cols = out_cols;
-  dimensions->pad_rows = pad_rows;
-  dimensions->pad_cols = pad_cols;
+  dimensions->pad_rows_before = pad_rows_before;
+  dimensions->pad_rows_after = pad_rows_after;
+  dimensions->pad_cols_before = pad_cols_before;
+  dimensions->pad_cols_after = pad_cols_after;
 
   return Status::OK();
 }
@@ -463,33 +487,35 @@ class Conv2DOp : public BinaryOp<T> {
     }
 
 #ifdef TENSORFLOW_USE_LIBXSMM_CONVOLUTIONS
-    if (LaunchXsmmConvOp<Device, T>::Run(
+    if (params_.padding != EXPLICIT &&
+        LaunchXsmmConvOp<Device, T>::Run(
             context, input, filter, dimensions.batch, dimensions.input_rows,
             dimensions.input_cols, dimensions.in_depth, dimensions.filter_rows,
-            dimensions.filter_cols, dimensions.pad_rows, dimensions.pad_cols,
-            dimensions.out_rows, dimensions.out_cols, dimensions.out_depth,
-            dimensions.dilation_rows, dimensions.dilation_cols,
-            dimensions.stride_rows, dimensions.stride_cols, output,
-            params_.data_format)) {
+            dimensions.filter_cols, dimensions.pad_rows_before,
+            dimensions.pad_cols_before, dimensions.out_rows,
+            dimensions.out_cols, dimensions.out_depth, dimensions.dilation_rows,
+            dimensions.dilation_cols, dimensions.stride_rows,
+            dimensions.stride_cols, output, params_.data_format)) {
       return;
     }
 #endif
 
-    if (LaunchDeepConvOp<Device, T>::Run(
+    if (params_.padding != EXPLICIT &&
+        LaunchDeepConvOp<Device, T>::Run(
             context, input, filter, dimensions.batch, dimensions.input_rows,
             dimensions.input_cols, dimensions.in_depth, dimensions.filter_rows,
-            dimensions.filter_cols, dimensions.pad_rows, dimensions.pad_cols,
-            dimensions.out_rows, dimensions.out_cols, dimensions.out_depth,
-            dimensions.dilation_rows, dimensions.dilation_cols,
-            dimensions.stride_rows, dimensions.stride_cols, output,
-            params_.data_format)) {
+            dimensions.filter_cols, dimensions.pad_rows_before,
+            dimensions.pad_cols_before, dimensions.out_rows,
+            dimensions.out_cols, dimensions.out_depth, dimensions.dilation_rows,
+            dimensions.dilation_cols, dimensions.stride_rows,
+            dimensions.stride_cols, output, params_.data_format)) {
       return;
     }
 
     launcher_(context, use_cudnn_, cudnn_use_autotune_, input, filter,
               dimensions.dilation_rows, dimensions.dilation_cols,
               dimensions.stride_rows, dimensions.stride_cols, params_.padding,
-              output, params_.data_format);
+              params_.explicit_paddings, output, params_.data_format);
   }
 
  private:
@@ -551,7 +577,8 @@ void LaunchConv2DOp<GPUDevice, T>::operator()(
     OpKernelContext* ctx, bool use_cudnn, bool cudnn_use_autotune,
     const Tensor& input_param, const Tensor& filter, int row_dilation,
     int col_dilation, int row_stride, int col_stride, const Padding& padding,
-    Tensor* output, TensorFormat data_format) {
+    const std::vector<int64>& explicit_paddings, Tensor* output,
+    TensorFormat data_format) {
   using se::dnn::AlgorithmConfig;
   using se::dnn::AlgorithmDesc;
   using se::dnn::ProfileResult;
@@ -580,7 +607,8 @@ void LaunchConv2DOp<GPUDevice, T>::operator()(
   bool is_grouped_convolution = patch_depths != in_depths;
   if (patch_rows == 1 && patch_cols == 1 && !is_grouped_convolution &&
       row_dilation == 1 && col_dilation == 1 && row_stride == 1 &&
-      col_stride == 1 && data_format == FORMAT_NHWC) {
+      col_stride == 1 && data_format == FORMAT_NHWC &&
+      (padding == VALID || padding == SAME)) {
     // 1x1 filter, so call cublas directly.
     const uint64 m = in_batch * in_rows * in_cols;
     const uint64 k = patch_depths;
@@ -634,49 +662,78 @@ void LaunchConv2DOp<GPUDevice, T>::operator()(
     return;
   }
 
-  int padding_rows = 0;
-  int padding_cols = 0;
   const int64 out_batch = GetTensorDim(*output, data_format, 'N');
   const int64 out_rows = GetTensorDim(*output, data_format, 'H');
   const int64 out_cols = GetTensorDim(*output, data_format, 'W');
   const int64 out_depths = GetTensorDim(*output, data_format, 'C');
-  if (padding == SAME) {
-    // Total padding on rows and cols is
-    // Pr = (R' - 1) * S + (Kr - 1) * Dr + 1 - R
-    // Pc = (C' - 1) * S + (Kc - 1) * Dc + 1 - C
-    // where (R', C') are output dimensions, (R, C) are input dimensions, S
-    // is stride, (Dr, Dc) are dilations, (Kr, Kc) are filter dimensions.
-    // We pad Pr/2 on the left and Pr - Pr/2 on the right, Pc/2 on the top
-    // and Pc - Pc/2 on the bottom.  When Pr or Pc is odd, this means
-    // we pad more on the right and bottom than on the top and left.
-    padding_rows =
-        std::max<int>(0, (out_rows - 1) * row_stride +
-                             (patch_rows - 1) * row_dilation + 1 - in_rows);
-    padding_cols =
-        std::max<int>(0, (out_cols - 1) * col_stride +
-                             (patch_cols - 1) * col_dilation + 1 - in_cols);
-    const bool rows_odd = (padding_rows % 2 != 0);
-    const bool cols_odd = (padding_cols % 2 != 0);
-    if (rows_odd || cols_odd) {
-      Tensor transformed_input;
-      int64 new_in_rows = in_rows + rows_odd;
-      int64 new_in_cols = in_cols + cols_odd;
-      OP_REQUIRES_OK(
-          ctx,
-          ctx->allocate_temp(DataTypeToEnum<T>::value,
-                             ShapeFromFormat(data_format, in_batch, new_in_rows,
-                                             new_in_cols, in_depths),
-                             &transformed_input));
-
-      functor::PadInput<GPUDevice, T, int, 4>()(
-          ctx->eigen_device<GPUDevice>(), To32Bit(input_param.tensor<T, 4>()),
-          {{0, 0}}, {{rows_odd, cols_odd}},
-          To32Bit(transformed_input.tensor<T, 4>()), data_format);
-
-      input = transformed_input;
-      in_rows = new_in_rows;
-      in_cols = new_in_cols;
+  int64 padding_top = -1, padding_bottom = -1;
+  int64 padding_left = -1, padding_right = -1;
+  if (padding == EXPLICIT) {
+    GetExplicitPaddingForDim(explicit_paddings, data_format, 'H', &padding_top,
+                             &padding_bottom);
+    GetExplicitPaddingForDim(explicit_paddings, data_format, 'W', &padding_left,
+                             &padding_right);
+  }
+  int64 out_rows_check, out_cols_check;
+  Status status = GetWindowedOutputSizeVerboseV2(
+      in_rows, patch_rows, row_dilation, row_stride, padding, &out_rows_check,
+      &padding_top, &padding_bottom);
+  // The status is guaranteed to be OK because we checked the output and padding
+  // was valid earlier.
+  TF_CHECK_OK(status);
+  DCHECK_EQ(out_rows, out_rows_check);
+  status = GetWindowedOutputSizeVerboseV2(in_cols, patch_cols, col_dilation,
+                                          col_stride, padding, &out_cols_check,
+                                          &padding_left, &padding_right);
+  TF_CHECK_OK(status);
+  DCHECK_EQ(out_cols, out_cols_check);
+
+  const int64 common_padding_rows = std::min(padding_top, padding_bottom);
+  const int64 common_padding_cols = std::min(padding_left, padding_right);
+  if (padding_top != padding_bottom || padding_left != padding_right) {
+    // cuDNN only supports padding the same amount on the left and right sides,
+    // and on the top and bottom sides. So we manually create a new padded
+    // input tensor such that we can pass it to cuDNN.
+
+    // TODO(reedwm): In some cases, we can avoid an allocation even if the two
+    // padding sides are different. For example, if the input is 2x2, the filter
+    // is 1x1, the stride is 2, and the padding is (1, 0, 1, 0), the result is
+    // equivalent to as if the padding is (1, 1, 1, 1). Changing the padding in
+    // such a way would allow us to avoid the allocation.
+    Tensor transformed_input;
+    const int64 padding_rows_diff = std::abs(padding_bottom - padding_top);
+    const int64 padding_cols_diff = std::abs(padding_right - padding_left);
+    const int64 new_in_rows = in_rows + padding_rows_diff;
+    const int64 new_in_cols = in_cols + padding_cols_diff;
+    OP_REQUIRES_OK(ctx, ctx->allocate_temp(
+                            DataTypeToEnum<T>::value,
+                            ShapeFromFormat(data_format, in_batch, new_in_rows,
+                                            new_in_cols, in_depths),
+                            &transformed_input));
+
+    const int64 input_pad_top = padding_top - common_padding_rows;
+    const int64 input_pad_bottom = padding_bottom - common_padding_rows;
+    const int64 input_pad_left = padding_left - common_padding_cols;
+    const int64 input_pad_right = padding_right - common_padding_cols;
+    bool in_bounds =
+        FastBoundsCheck(input_pad_top, std::numeric_limits<int>::max()) &&
+        FastBoundsCheck(input_pad_bottom, std::numeric_limits<int>::max()) &&
+        FastBoundsCheck(input_pad_left, std::numeric_limits<int>::max()) &&
+        FastBoundsCheck(input_pad_right, std::numeric_limits<int>::max());
+    if (!in_bounds) {
+      ctx->SetStatus(errors::InvalidArgument("Padding is too large."));
+      return;
     }
+    functor::PadInput<GPUDevice, T, int, 4>()(
+        ctx->eigen_device<GPUDevice>(), To32Bit(input_param.tensor<T, 4>()),
+        {{static_cast<int>(input_pad_top), static_cast<int>(input_pad_left)}},
+        {{static_cast<int>(input_pad_bottom),
+          static_cast<int>(input_pad_right)}},
+        To32Bit(transformed_input.tensor<T, 4>()), data_format);
+
+    input = transformed_input;
+    in_rows = new_in_rows;
+    in_cols = new_in_cols;
   }
 
   if (data_format == FORMAT_NHWC) {
@@ -698,9 +755,9 @@ void LaunchConv2DOp<GPUDevice, T>::operator()(
     }
   }
 
-  CHECK(padding_rows >= 0 && padding_cols >= 0)
-      << "Negative row or col paddings: (" << padding_rows << ", "
-      << padding_cols << ")";
+  CHECK(common_padding_rows >= 0 && common_padding_cols >= 0)  // Crash OK
+      << "Negative row or col paddings: (" << common_padding_rows << ", "
+      << common_padding_cols << ")";
   se::dnn::BatchDescriptor input_desc;
   input_desc.set_count(in_batch)
       .set_feature_map_count(in_depths)
@@ -723,8 +780,8 @@ void LaunchConv2DOp<GPUDevice, T>::operator()(
       .set_horizontal_dilation_rate(col_dilation)
       .set_vertical_filter_stride(row_stride)
       .set_horizontal_filter_stride(col_stride)
-      .set_zero_padding_height(padding_rows / 2)
-      .set_zero_padding_width(padding_cols / 2)
+      .set_zero_padding_height(common_padding_rows)
+      .set_zero_padding_width(common_padding_cols)
       .set_group_count(in_depths / patch_depths);
 
   Tensor transformed_filter;
@@ -767,23 +824,23 @@ void LaunchConv2DOp<GPUDevice, T>::operator()(
   int device_id = stream->parent()->device_ordinal();
   DataType dtype = input.dtype();
   ConvParameters conv_parameters = {
-      in_batch,          // batch
-      in_depths,         // in_depths
-      {{in_rows,         // in_rows
-        in_cols}},       // in_cols
-      FORMAT_NCHW,       // compute_data_format
-      out_depths,        // out_depths
-      {{patch_rows,      // filter_rows
-        patch_cols,      // filter_cols
-        patch_depths}},  // filter_depths
-      {{row_dilation,    // dilation_rows
-        col_dilation}},  // dilation_cols
-      {{row_stride,      // stride_rows
-        col_stride}},    // stride_cols
-      {{padding_rows,    // padding_rows
-        padding_cols}},  // padding_cols
-      dtype,             // tensor datatype
-      device_id,         // device_id
+      in_batch,                 // batch
+      in_depths,                // in_depths
+      {{in_rows,                // in_rows
+        in_cols}},              // in_cols
+      FORMAT_NCHW,              // compute_data_format
+      out_depths,               // out_depths
+      {{patch_rows,             // filter_rows
+        patch_cols,             // filter_cols
+        patch_depths}},         // filter_depths
+      {{row_dilation,           // dilation_rows
+        col_dilation}},         // dilation_cols
+      {{row_stride,             // stride_rows
+        col_stride}},           // stride_cols
+      {{common_padding_rows,    // padding_rows
+        common_padding_cols}},  // padding_cols
+      dtype,                    // tensor datatype
+      device_id,                // device_id
   };
   AlgorithmConfig algorithm_config;
   if (cudnn_use_autotune &&
diff --git a/tensorflow/core/kernels/conv_ops.h b/tensorflow/core/kernels/conv_ops.h
index 7ccbaf4bf2..105a4b1b82 100644
--- a/tensorflow/core/kernels/conv_ops.h
+++ b/tensorflow/core/kernels/conv_ops.h
@@ -36,7 +36,8 @@ struct LaunchConv2DOp {
   void operator()(OpKernelContext* ctx, bool use_cudnn, bool cudnn_use_autotune,
                   const Tensor& input, const Tensor& filter, int row_dilation,
                   int col_dilation, int row_stride, int col_stride,
-                  const Padding& padding, Tensor* output,
+                  const Padding& padding,
+                  const std::vector<int64>& explicit_paddings, Tensor* output,
                   TensorFormat data_format);
 };
 
@@ -46,7 +47,8 @@ struct LaunchConv2DOp<Eigen::GpuDevice, T> {
   void operator()(OpKernelContext* ctx, bool use_cudnn, bool cudnn_use_autotune,
                   const Tensor& input, const Tensor& filter, int row_dilation,
                   int col_dilation, int row_stride, int col_stride,
-                  const Padding& padding, Tensor* output,
+                  const Padding& padding,
+                  const std::vector<int64>& explicit_paddings, Tensor* output,
                   TensorFormat data_format);
 };
 #endif  // GOOGLE_CUDA
@@ -72,6 +74,7 @@ struct Conv2DParameters {
   std::vector<int32> strides;
   Padding padding;
   TensorFormat data_format;
+  std::vector<int64> explicit_paddings;
 };
 
 // Convolution dimensions inferred from parameters, input and filter tensors.
@@ -94,8 +97,10 @@ struct Conv2DDimensions {
 
   int64 out_rows;
   int64 out_cols;
-  int64 pad_rows;
-  int64 pad_cols;
+  int64 pad_rows_before;
+  int64 pad_rows_after;
+  int64 pad_cols_before;
+  int64 pad_cols_after;
 };
 
 // Initializes and validates Conv2D parameters configured by OpKernel
diff --git a/tensorflow/core/kernels/depthwise_conv_grad_op.cc b/tensorflow/core/kernels/depthwise_conv_grad_op.cc
index da3bdb475e..c152f2b7e4 100644
--- a/tensorflow/core/kernels/depthwise_conv_grad_op.cc
+++ b/tensorflow/core/kernels/depthwise_conv_grad_op.cc
@@ -633,7 +633,8 @@ class DepthwiseConv2dNativeBackpropInputOp : public OpKernel {
       // conv is supported.
       launcher_(context, use_cudnn_, cudnn_use_autotune_, out_backprop,
                 reshaped_filter, /*row_dilation=*/1, /*col_dilation=*/1,
-                stride_, stride_, padding_, in_backprop, data_format_);
+                stride_, stride_, padding_, /*explicit_paddings=*/{},
+                in_backprop, data_format_);
       return;
     }
 
@@ -1115,7 +1116,8 @@ class DepthwiseConv2dNativeBackpropFilterOp : public OpKernel {
       // conv is supported.
       launcher_(context, use_cudnn_, cudnn_use_autotune_, out_backprop, input,
                 /*row_dilation=*/1, /*col_dilation=*/1, stride_, stride_,
-                padding_, &reshaped_filter, data_format_);
+                padding_, /*explicit_paddings=*/{}, &reshaped_filter,
+                data_format_);
       return;
     }
 
diff --git a/tensorflow/core/kernels/depthwise_conv_op.cc b/tensorflow/core/kernels/depthwise_conv_op.cc
index f0902fdba6..dacd3cfea8 100644
--- a/tensorflow/core/kernels/depthwise_conv_op.cc
+++ b/tensorflow/core/kernels/depthwise_conv_op.cc
@@ -404,7 +404,8 @@ class DepthwiseConv2dNativeOp : public BinaryOp<T> {
       // conv is supported.
       launcher_(context, use_cudnn_, cudnn_use_autotune_, input,
                 reshaped_filter, /*row_dilation=*/1, /*col_dilation=*/1,
-                stride_, stride_, padding_, output, data_format_);
+                stride_, stride_, padding_, /*explicit_paddings=*/{}, output,
+                data_format_);
       return;
     }
 
diff --git a/tensorflow/core/ops/nn_ops.cc b/tensorflow/core/ops/nn_ops.cc
index 0e94259f75..0f4f725937 100644
--- a/tensorflow/core/ops/nn_ops.cc
+++ b/tensorflow/core/ops/nn_ops.cc
@@ -269,10 +269,11 @@ REGISTER_OP("Conv2D")
     .Attr("T: {half, bfloat16, float, double}")
     .Attr("strides: list(int)")
     .Attr("use_cudnn_on_gpu: bool = true")
-    .Attr(GetPaddingAttrString())
+    .Attr(GetPaddingAttrStringWithExplicit())
+    .Attr(GetExplicitPaddingsAttrString())
     .Attr(GetConvnetDataFormatAttrString())
     .Attr("dilations: list(int) = [1, 1, 1, 1]")
-    .SetShapeFn(shape_inference::Conv2DShape);
+    .SetShapeFn(shape_inference::Conv2DShapeWithExplicitPadding);
 
 REGISTER_OP("Conv2DBackpropInput")
     .Input("input_sizes: int32")
@@ -282,7 +283,8 @@ REGISTER_OP("Conv2DBackpropInput")
     .Attr("T: {half, bfloat16, float, double}")
     .Attr("strides: list(int)")
     .Attr("use_cudnn_on_gpu: bool = true")
-    .Attr(GetPaddingAttrString())
+    .Attr(GetPaddingAttrStringWithExplicit())
+    .Attr(GetExplicitPaddingsAttrString())
     .Attr(GetConvnetDataFormatAttrString())
     .Attr("dilations: list(int) = [1, 1, 1, 1]")
     .SetShapeFn([](InferenceContext* c) {
@@ -304,7 +306,8 @@ REGISTER_OP("Conv2DBackpropFilter")
     .Attr("T: {half, bfloat16, float, double}")
     .Attr("strides: list(int)")
     .Attr("use_cudnn_on_gpu: bool = true")
-    .Attr(GetPaddingAttrString())
+    .Attr(GetPaddingAttrStringWithExplicit())
+    .Attr(GetExplicitPaddingsAttrString())
     .Attr(GetConvnetDataFormatAttrString())
     .Attr("dilations: list(int) = [1, 1, 1, 1]")
     .SetShapeFn([](InferenceContext* c) {
diff --git a/tensorflow/core/util/padding.cc b/tensorflow/core/util/padding.cc
index 117de5ee4b..9e7fb8489e 100644
--- a/tensorflow/core/util/padding.cc
+++ b/tensorflow/core/util/padding.cc
@@ -29,12 +29,55 @@ Status GetNodeAttr(const NodeDef& node_def, StringPiece attr_name,
     *value = SAME;
   } else if (str_value == "VALID") {
     *value = VALID;
+  } else if (str_value == "EXPLICIT") {
+    *value = EXPLICIT;
   } else {
     return errors::NotFound(str_value, " is not an allowed padding type");
   }
   return Status::OK();
 }
 
+Status CheckValidPadding(Padding padding_type,
+                         const std::vector<int64>& explicit_paddings,
+                         int num_dims, TensorFormat data_format) {
+  if (padding_type == Padding::EXPLICIT) {
+    if (explicit_paddings.size() != 2 * num_dims) {
+      return errors::InvalidArgument(
+          "explicit_paddings attribute must contain ", 2 * num_dims,
+          " values, but got: ", explicit_paddings.size());
+    }
+    for (int64 padding_value : explicit_paddings) {
+      if (padding_value < 0) {
+        return errors::InvalidArgument(
+            "All elements of explicit_paddings must be nonnegative");
+      }
+    }
+    const int32 batch_index = GetTensorBatchDimIndex(num_dims, data_format);
+    const int32 depth_index = GetTensorFeatureDimIndex(num_dims, data_format);
+    if (explicit_paddings[2 * batch_index] != 0 ||
+        explicit_paddings[2 * batch_index + 1] != 0 ||
+        explicit_paddings[2 * depth_index] != 0 ||
+        explicit_paddings[2 * depth_index + 1] != 0) {
+      return errors::InvalidArgument(
+          "Nonzero explicit padding in the batch or depth dimensions is not "
+          "supported");
+    }
+  } else if (!explicit_paddings.empty()) {
+    return errors::InvalidArgument(
+        "explicit_paddings attribute must be empty if the padding attribute is "
+        "not EXPLICIT");
+  }
+  return Status::OK();
+}
+
 string GetPaddingAttrString() { return "padding: {'SAME', 'VALID'}"; }
 
+string GetPaddingAttrStringWithExplicit() {
+  return "padding: {'SAME', 'VALID', 'EXPLICIT'}";
+}
+
+string GetExplicitPaddingsAttrString() {
+  return "explicit_paddings: list(int) = []";
+}
+
 }  // end namespace tensorflow
diff --git a/tensorflow/core/util/padding.h b/tensorflow/core/util/padding.h
index 76f9b4dd9a..a1dd1c0bd9 100644
--- a/tensorflow/core/util/padding.h
+++ b/tensorflow/core/util/padding.h
@@ -20,8 +20,10 @@ limitations under the License.
 // kernels.
 
 #include <string>
+#include <vector>
 
 #include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/util/tensor_format.h"
 
 namespace tensorflow {
 
@@ -34,16 +36,29 @@ class NodeDef;
 //   VALID: No padding is carried out.
 //   SAME: The pad value is computed so that the output will have the same
 //         dimensions as the input.
+//   EXPLICIT: The user specifies the pad values in the explicit_padding
+//             attribute.
 // The padded area is zero-filled.
 enum Padding {
-  VALID = 1,  // No padding.
-  SAME = 2,   // Input and output layers have the same size.
+  VALID = 1,     // No padding.
+  SAME = 2,      // Input and output layers have the same size.
+  EXPLICIT = 3,  // Padding is explicitly specified
 };
 
+// Returns an error if the padding attributes are invalid.
+Status CheckValidPadding(Padding padding_type,
+                         const std::vector<int64>& explicit_paddings,
+                         int num_dims, TensorFormat data_format);
+
 // Return the string containing the list of valid padding types, that can be
 // used as an Attr() in REGISTER_OP.
 string GetPaddingAttrString();
 
+// Like GetPaddingAttrString(), but also includes EXPLICIT.
+string GetPaddingAttrStringWithExplicit();
+
+string GetExplicitPaddingsAttrString();
+
 // Specialization to parse an attribute directly into a Padding enum.
 Status GetNodeAttr(const NodeDef& node_def, StringPiece attr_name,
                    Padding* value);
diff --git a/tensorflow/core/util/tensor_format.h b/tensorflow/core/util/tensor_format.h
index a296fb447e..643e14e0b5 100644
--- a/tensorflow/core/util/tensor_format.h
+++ b/tensorflow/core/util/tensor_format.h
@@ -408,18 +408,24 @@ inline int32 GetTensorDimIndex(TensorFormat format, char dimension) {
   return GetTensorDimIndex<2>(format, dimension);
 }
 
+inline int32 GetTensorDimIndex(TensorFormat format, char dimension,
+                               int num_total_dims) {
+  int32 index = (GetTensorSpatialDims(num_total_dims, format) == 3)
+                    ? GetTensorDimIndex<3>(format, dimension)
+                    : GetTensorDimIndex<2>(format, dimension);
+  CHECK(index >= 0 && index < num_total_dims)  // Crash OK.
+      << "Invalid index from the dimension: " << index << ", " << format << ", "
+      << dimension;
+  return index;
+}
+
 // Return the element from 'dimension_attributes' that corresponds to the
 // specified 'dimension' according to 'tensor_format'.
 template <typename T>
 T GetTensorDim(gtl::ArraySlice<T> dimension_attributes,
                TensorFormat tensor_format, char dimension) {
   int index =
-      (GetTensorSpatialDims(dimension_attributes.size(), tensor_format) == 3)
-          ? GetTensorDimIndex<3>(tensor_format, dimension)
-          : GetTensorDimIndex<2>(tensor_format, dimension);
-  CHECK(index >= 0 && index < dimension_attributes.size())
-      << "Invalid index from the dimension: " << index << ", " << tensor_format
-      << ", " << dimension;
+      GetTensorDimIndex(tensor_format, dimension, dimension_attributes.size());
   return dimension_attributes[index];
 }
 
@@ -476,6 +482,15 @@ inline int64 GetFilterDim(const Tensor& tensor,
   return GetFilterDim(tensor.shape(), filter_tensor_format, dimension);
 }
 
+inline void GetExplicitPaddingForDim(
+    const std::vector<int64>& explicit_paddings, TensorFormat tensor_format,
+    char dimension, int64* padding_before, int64* padding_after) {
+  int index =
+      GetTensorDimIndex(tensor_format, dimension, explicit_paddings.size() / 2);
+  *padding_before = explicit_paddings[2 * index];
+  *padding_after = explicit_paddings[2 * index + 1];
+}
+
 // Return the string that specifies the data format for convnet operations.
 string GetConvnetDataFormatAttrString();
 string GetConvnet3dDataFormatAttrString();
diff --git a/tensorflow/python/keras/layers/convolutional.py b/tensorflow/python/keras/layers/convolutional.py
index 80ec55f410..30b919cc0a 100644
--- a/tensorflow/python/keras/layers/convolutional.py
+++ b/tensorflow/python/keras/layers/convolutional.py
@@ -180,12 +180,14 @@ class Conv(Layer):
       op_padding = 'valid'
     else:
       op_padding = self.padding
+    if not isinstance(op_padding, (list, tuple)):
+      op_padding = op_padding.upper()
     self._convolution_op = nn_ops.Convolution(
         input_shape,
         filter_shape=self.kernel.get_shape(),
         dilation_rate=self.dilation_rate,
         strides=self.strides,
-        padding=op_padding.upper(),
+        padding=op_padding,
         data_format=conv_utils.convert_data_format(self.data_format,
                                                    self.rank + 2))
     self.built = True
diff --git a/tensorflow/python/keras/utils/conv_utils.py b/tensorflow/python/keras/utils/conv_utils.py
index f486e631e5..ea7427f61a 100644
--- a/tensorflow/python/keras/utils/conv_utils.py
+++ b/tensorflow/python/keras/utils/conv_utils.py
@@ -194,9 +194,11 @@ def normalize_data_format(value):
 
 
 def normalize_padding(value):
+  if isinstance(value, (list, tuple)):
+    return value
   padding = value.lower()
   if padding not in {'valid', 'same', 'causal'}:
-    raise ValueError('The `padding` argument must be one of '
+    raise ValueError('The `padding` argument must be a list/tuple or one of '
                      '"valid", "same" (or "causal", only for `Conv1D). '
                      'Received: ' + str(padding))
   return padding
diff --git a/tensorflow/python/kernel_tests/conv_ops_test.py b/tensorflow/python/kernel_tests/conv_ops_test.py
index 2f6f3bb383..7ff1a61e47 100644
--- a/tensorflow/python/kernel_tests/conv_ops_test.py
+++ b/tensorflow/python/kernel_tests/conv_ops_test.py
@@ -26,13 +26,18 @@ import numpy as np
 
 from six.moves import xrange  # pylint: disable=redefined-builtin
 from tensorflow.contrib import layers
+from tensorflow.core.protobuf import config_pb2
+from tensorflow.core.protobuf import rewriter_config_pb2
 from tensorflow.python.client import session as session_lib
+from tensorflow.python.eager import backprop
+from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import nn_impl
@@ -165,6 +170,12 @@ class Conv2DTest(test.TestCase):
       # as we will be using its gradients as reference for fp16 gradients.
       return [dtypes.float32, dtypes.float16, dtypes.float64]
 
+  def _CreateNumpyTensor(self, shape):
+    total_size = 1
+    for s in shape:
+      total_size *= s
+    return np.arange(1, total_size + 1, dtype=np.float32).reshape(shape)
+
   def _SetupValuesForDevice(self, tensor_in_sizes, filter_in_sizes, dilations,
                             strides, padding, data_format, dtype, use_gpu):
     """Verifies the output values of the convolution function.
@@ -183,26 +194,22 @@ class Conv2DTest(test.TestCase):
     Returns:
       Symbolic tensor value that can be used to execute the computation
     """
-    total_size_1 = 1
-    total_size_2 = 1
-    for s in tensor_in_sizes:
-      total_size_1 *= s
-    for s in filter_in_sizes:
-      total_size_2 *= s
-    # Initializes the input tensor with array containing incrementing
-    # numbers from 1.
-    x1 = [f * 1.0 for f in range(1, total_size_1 + 1)]
-    x2 = [f * 1.0 for f in range(1, total_size_2 + 1)]
+    x1 = self._CreateNumpyTensor(tensor_in_sizes)
+    x2 = self._CreateNumpyTensor(filter_in_sizes)
 
     with test_util.device(use_gpu):
       t1 = constant_op.constant(x1, shape=tensor_in_sizes, dtype=dtype)
       t2 = constant_op.constant(x2, shape=filter_in_sizes, dtype=dtype)
       strides = [1] + strides + [1]
       dilations = [1] + dilations + [1]
+      if isinstance(padding, (list, tuple)):
+        padding = [(0, 0)] + padding + [(0, 0)]
       if data_format == "NCHW":
         t1 = test_util.NHWCToNCHW(t1)
         strides = test_util.NHWCToNCHW(strides)
         dilations = test_util.NHWCToNCHW(dilations)
+        if isinstance(padding, (list, tuple)):
+          padding = test_util.NHWCToNCHW(padding)
       conv = nn_ops.conv2d(
           t1,
           t2,
@@ -254,17 +261,8 @@ class Conv2DTest(test.TestCase):
   def _ComputeReferenceDilatedConv(self, tensor_in_sizes, filter_in_sizes,
                                    stride, dilation, padding, data_format,
                                    use_gpu):
-    total_size_1 = 1
-    total_size_2 = 1
-    for s in tensor_in_sizes:
-      total_size_1 *= s
-    for s in filter_in_sizes:
-      total_size_2 *= s
-
-    # Initializes the input tensor with array containing incrementing
-    # numbers from 1.
-    x1 = [f * 1.0 for f in range(1, total_size_1 + 1)]
-    x2 = [f * 1.0 for f in range(1, total_size_2 + 1)]
+    x1 = self._CreateNumpyTensor(tensor_in_sizes)
+    x2 = self._CreateNumpyTensor(filter_in_sizes)
     with test_util.device(use_gpu):
       t1 = constant_op.constant(x1, shape=tensor_in_sizes)
       t2 = constant_op.constant(x2, shape=filter_in_sizes)
@@ -312,16 +310,29 @@ class Conv2DTest(test.TestCase):
       expected_values = self.evaluate(expected_results)
       computed_values = self.evaluate(computed_results)
       for e_value, c_value in zip(expected_values, computed_values):
-        tf_logging.info("expected = ", e_value)
-        tf_logging.info("actual = ", c_value)
+        tf_logging.debug("expected = %s", e_value)
+        tf_logging.debug("actual = %s", c_value)
         self.assertAllClose(
             e_value.flatten(), c_value.flatten(), atol=tolerance, rtol=1e-4)
 
-  def _VerifyValues(self, tensor_in_sizes, filter_in_sizes, strides, padding,
-                    expected):
+  def _VerifyValues(self,
+                    tensor_in_sizes,
+                    filter_in_sizes,
+                    strides,
+                    padding,
+                    expected,
+                    dilations=(1, 1),
+                    gpu_only=False,
+                    test_grappler_layout_optimizer=False,
+                    tol=1e-5,
+                    fp16_tol=1e-3):
+    if gpu_only and not test.is_gpu_available(cuda_only=True):
+      return
     tensors = []
-    dilations = [1, 1]
+    dilations = list(dilations)
     for (data_format, use_gpu) in GetTestConfigs():
+      if gpu_only and not use_gpu:
+        continue
       for dtype in self._DtypesToTest(use_gpu):
         result = self._SetupValuesForDevice(
             tensor_in_sizes,
@@ -332,19 +343,71 @@ class Conv2DTest(test.TestCase):
             data_format,
             dtype,
             use_gpu=use_gpu)
+        if test_grappler_layout_optimizer and data_format == "NHWC" and use_gpu:
+          # Grappler's layout optimizer will not optimize a fetch node, so
+          # this identity allows Grappler to optimize the Conv2D node.
+          result = array_ops.identity(result)
         tensors.append(result)
       values = self.evaluate(tensors)
       for i in range(len(tensors)):
         conv = tensors[i]
         value = values[i]
-        tf_logging.info("expected = ", expected)
-        tf_logging.info("actual = ", value)
-        tol = 1e-5
-        if value.dtype == np.float16:
-          tol = 1e-3
-        self.assertAllClose(expected, np.ravel(value), atol=tol, rtol=tol)
+        tf_logging.debug("expected = %s", expected)
+        tf_logging.debug("actual = %s", value)
+        tol_to_use = fp16_tol if value.dtype == np.float16 else tol
+        self.assertAllClose(expected, np.ravel(value), atol=tol_to_use,
+                            rtol=tol_to_use)
         self.assertShapeEqual(value, conv)
 
+  def _VerifyExplicitPaddings(self,
+                              tensor_in_sizes,
+                              filter_in_sizes,
+                              strides,
+                              padding,
+                              dilations=(1, 1),
+                              test_grappler_layout_optimizer=False,
+                              tol=1e-5,
+                              fp16_tol=1e-3):
+    """Verifies Conv2D with explicit padding generates correct values.
+
+    It does this by comparing with Conv2D without explicit padding. This
+    function assumes Conv2D without explicit padding works correctly.
+
+    Args:
+      tensor_in_sizes: Input tensor dimensions in [batch, input_rows,
+        input_cols, input_depth].
+      filter_in_sizes: Filter tensor dimensions in [kernel_rows, kernel_cols,
+        input_depth, output_depth].
+      strides: [row_stride, col_stride] for the convolution;
+      padding: Explicit padding amounts.
+      dilations: Dilation values
+      test_grappler_layout_optimizer: If True, allow the Grappler layout
+        optimizer to run, which turns NHWC Conv2Ds on the GPU to NCHW Conv2Ds.
+      tol: The absolute and relative tolerance for non-fp16 dtypes.
+      fp16_tol: The absolute and relative tolerance for fp16.
+    """
+    input_tensor = self._CreateNumpyTensor(tensor_in_sizes)
+    filter_tensor = self._CreateNumpyTensor(filter_in_sizes)
+    input_tensor = array_ops.pad(input_tensor, [(0, 0)] + padding + [(0, 0)])
+    dilations = list(dilations)
+    conv2d_result = nn_ops.conv2d(
+        input_tensor,
+        filter_tensor, [1] + list(strides) + [1],
+        "VALID",
+        dilations=[1] + dilations + [1])
+    expected = list(self.evaluate(array_ops.reshape(conv2d_result, [-1])))
+    self._VerifyValues(
+        tensor_in_sizes,
+        filter_in_sizes,
+        strides,
+        padding,
+        expected,
+        dilations,
+        gpu_only=True,
+        test_grappler_layout_optimizer=test_grappler_layout_optimizer,
+        tol=tol,
+        fp16_tol=fp16_tol)
+
   @test_util.run_in_graph_and_eager_modes
   def testConv2D1x1Filter(self):
     expected_output = [
@@ -510,6 +573,126 @@ class Conv2DTest(test.TestCase):
         dilations=[2, 2],
         padding="VALID")
 
+  @test_util.run_in_graph_and_eager_modes()
+  def testConv2D0x0Padding(self):
+    self._VerifyExplicitPaddings(
+        tensor_in_sizes=[1, 2, 3, 3],
+        filter_in_sizes=[2, 2, 3, 3],
+        strides=[1, 1],
+        padding=[[0, 0], [0, 0]])
+
+    self._VerifyExplicitPaddings(
+        tensor_in_sizes=[3, 4, 3, 2],
+        filter_in_sizes=[1, 1, 2, 1],
+        strides=[2, 2],
+        padding=[[0, 0], [0, 0]])
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testConv2D1x1Padding(self):
+    self._VerifyExplicitPaddings(
+        tensor_in_sizes=[1, 2, 3, 2],
+        filter_in_sizes=[2, 2, 2, 2],
+        strides=[1, 1],
+        padding=[[1, 1], [1, 1]])
+
+    self._VerifyExplicitPaddings(
+        tensor_in_sizes=[1, 2, 2, 1],
+        filter_in_sizes=[1, 1, 1, 2],
+        strides=[1, 1],
+        padding=[[1, 1], [1, 1]])
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testConv2D2x2Padding(self):
+    self._VerifyExplicitPaddings(
+        tensor_in_sizes=[1, 2, 1, 2],
+        filter_in_sizes=[2, 1, 2, 1],
+        strides=[1, 1],
+        padding=[[2, 2], [2, 2]])
+
+    self._VerifyExplicitPaddings(
+        tensor_in_sizes=[1, 2, 1, 2],
+        filter_in_sizes=[1, 1, 2, 1],
+        strides=[2, 1],
+        padding=[[2, 2], [2, 2]])
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testConv2DOnlyBottomPadding(self):
+    self._VerifyExplicitPaddings(
+        tensor_in_sizes=[1, 2, 3, 3],
+        filter_in_sizes=[2, 2, 3, 2],
+        strides=[1, 1],
+        padding=[[0, 3], [0, 0]], tol=2e-5)
+
+    self._VerifyExplicitPaddings(
+        tensor_in_sizes=[2, 2, 4, 3],
+        filter_in_sizes=[1, 2, 3, 2],
+        strides=[2, 2],
+        padding=[[0, 3], [0, 0]])
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testConv2DOnlyTopRightPadding(self):
+    self._VerifyExplicitPaddings(
+        tensor_in_sizes=[1, 2, 3, 3],
+        filter_in_sizes=[2, 2, 3, 2],
+        strides=[1, 1],
+        padding=[[1, 0], [0, 2]],
+        tol=5e-5)
+
+    self._VerifyExplicitPaddings(
+        tensor_in_sizes=[1, 2, 4, 2],
+        filter_in_sizes=[2, 2, 2, 2],
+        strides=[1, 3],
+        padding=[[1, 0], [0, 2]])
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testConv2DLotsPadding(self):
+    self._VerifyExplicitPaddings(
+        tensor_in_sizes=[1, 1, 1, 3],
+        filter_in_sizes=[2, 2, 3, 3],
+        strides=[1, 1],
+        padding=[[3, 4], [4, 2]])
+
+    self._VerifyExplicitPaddings(
+        tensor_in_sizes=[1, 2, 1, 1],
+        filter_in_sizes=[2, 2, 1, 3],
+        strides=[2, 1],
+        padding=[[3, 4], [4, 2]])
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testConv2DExplicitPaddingWithDilations(self):
+    self._VerifyExplicitPaddings(
+        tensor_in_sizes=[1, 3, 2, 1],
+        filter_in_sizes=[1, 2, 1, 2],
+        strides=[1, 1],
+        padding=[[1, 0], [0, 1]],
+        dilations=[2, 1])
+
+    self._VerifyExplicitPaddings(
+        tensor_in_sizes=[1, 2, 3, 2],
+        filter_in_sizes=[3, 2, 2, 1],
+        strides=[1, 1],
+        padding=[[2, 1], [1, 2]],
+        dilations=[2, 3])
+
+  def testConv2DExplicitPaddingWithLayoutOptimizer(self):
+    # Test with Grappler's layout optimizer, to ensure the layout optimizer
+    # handles explicit padding correctly.
+    self._VerifyExplicitPaddings(
+        tensor_in_sizes=[1, 3, 2, 1],
+        filter_in_sizes=[1, 2, 1, 2],
+        strides=[1, 1],
+        padding=[[1, 0], [0, 1]],
+        dilations=[2, 1],
+        test_grappler_layout_optimizer=True)
+
+    self._VerifyExplicitPaddings(
+        tensor_in_sizes=[1, 2, 3, 2],
+        filter_in_sizes=[3, 2, 2, 1],
+        strides=[1, 1],
+        padding=[[2, 1], [1, 2]],
+        dilations=[2, 3],
+        test_grappler_layout_optimizer=True)
+
   # TODO(yzhwang): this currently fails.
   # self._VerifyValues(tensor_in_sizes=[1, 8, 8, 1],
   #                   filter_in_sizes=[2, 2, 1, 1],
@@ -517,19 +700,22 @@ class Conv2DTest(test.TestCase):
   #                   expected=[72, 112, 392, 432])
 
   # Testing for backprops
-  def _RunAndVerifyBackpropInput(self, input_sizes, filter_sizes, output_sizes,
-                                 strides, padding, expected, data_format,
-                                 use_gpu, err):
-    total_output_size = 1
-    total_filter_size = 1
-    for s in output_sizes:
-      total_output_size *= s
-    for s in filter_sizes:
-      total_filter_size *= s
-    # Initializes the input tensor with array containing incrementing
-    # numbers from 1.
-    x1 = [f * 1.0 for f in range(1, total_filter_size + 1)]
-    x2 = [f * 1.0 for f in range(1, total_output_size + 1)]
+  def _RunAndVerifyBackpropInput(self,
+                                 input_sizes,
+                                 filter_sizes,
+                                 output_sizes,
+                                 strides,
+                                 padding,
+                                 expected,
+                                 data_format,
+                                 use_gpu,
+                                 err,
+                                 dilations=(1, 1)):
+    if use_gpu and not test.is_gpu_available(cuda_only=True):
+      return
+    x1 = self._CreateNumpyTensor(filter_sizes)
+    x2 = self._CreateNumpyTensor(output_sizes)
+    dilations = list(dilations)
     with test_util.device(use_gpu):
       if data_format == "NCHW":
         input_sizes = test_util.NHWCToNCHW(input_sizes)
@@ -537,18 +723,30 @@ class Conv2DTest(test.TestCase):
       t1 = constant_op.constant(x1, shape=filter_sizes)
       t2 = constant_op.constant(x2, shape=output_sizes)
       strides = [1] + strides + [1]
+      dilations = [1] + dilations + [1]
+      if isinstance(padding, (list, tuple)):
+        padding = [(0, 0)] + padding + [(0, 0)]
       if data_format == "NCHW":
         t2 = test_util.NHWCToNCHW(t2)
         strides = test_util.NHWCToNCHW(strides)
+        dilations = test_util.NHWCToNCHW(dilations)
+        if isinstance(padding, (list, tuple)):
+          padding = test_util.NHWCToNCHW((padding))
       conv = nn_ops.conv2d_backprop_input(
-          t0, t1, t2, strides=strides, padding=padding, data_format=data_format)
+          t0,
+          t1,
+          t2,
+          strides=strides,
+          padding=padding,
+          data_format=data_format,
+          dilations=dilations)
       if data_format == "NCHW":
         conv = test_util.NCHWToNHWC(conv)
       # "values" consists of two tensors for two backprops
       value = self.evaluate(conv)
       self.assertShapeEqual(value, conv)
-    tf_logging.info("expected = ", expected)
-    tf_logging.info("actual = ", value)
+    tf_logging.debug("expected = %s", expected)
+    tf_logging.debug("actual = %s", value)
     self.assertArrayNear(expected, value.flatten(), err)
 
   def _CompareBackpropInput(self, input_sizes, filter_sizes, output_sizes,
@@ -691,41 +889,51 @@ class Conv2DTest(test.TestCase):
           err=1e-5)
 
   # Testing for backprops
-  def _RunAndVerifyBackpropFilter(self, input_sizes, filter_sizes, output_sizes,
-                                  strides, padding, expected, data_format,
-                                  use_gpu):
-    total_input_size = 1
-    total_output_size = 1
-    for s in input_sizes:
-      total_input_size *= s
-    for s in output_sizes:
-      total_output_size *= s
-    # Initializes the input tensor with array containing incrementing
-    # numbers from 1.
-    x0 = [f * 1.0 for f in range(1, total_input_size + 1)]
-    x2 = [f * 1.0 for f in range(1, total_output_size + 1)]
+  def _RunAndVerifyBackpropFilter(self,
+                                  input_sizes,
+                                  filter_sizes,
+                                  output_sizes,
+                                  strides,
+                                  padding,
+                                  expected,
+                                  data_format,
+                                  use_gpu,
+                                  dilations=(1, 1),
+                                  err=1e-5):
+    x0 = self._CreateNumpyTensor(input_sizes)
+    x2 = self._CreateNumpyTensor(output_sizes)
+    dilations = list(dilations)
+    explicit_strides = [1] + strides + [1]
+    new_padding = padding
+    new_dilations = [1] + dilations + [1]
+    if isinstance(new_padding, (list, tuple)):
+      new_padding = [(0, 0)] + new_padding + [(0, 0)]
+    if data_format == "NCHW":
+      explicit_strides = test_util.NHWCToNCHW(explicit_strides)
+      new_dilations = test_util.NHWCToNCHW(new_dilations)
+      if isinstance(padding, (list, tuple)):
+        new_padding = test_util.NHWCToNCHW(new_padding)
     for dtype in self._DtypesToTest(use_gpu=use_gpu):
       with test_util.device(use_gpu):
         t0 = constant_op.constant(x0, shape=input_sizes, dtype=dtype)
         t1 = constant_op.constant(filter_sizes, shape=[len(filter_sizes)])
         t2 = constant_op.constant(x2, shape=output_sizes, dtype=dtype)
-        explicit_strides = [1] + strides + [1]
         if data_format == "NCHW":
           t0 = test_util.NHWCToNCHW(t0)
           t2 = test_util.NHWCToNCHW(t2)
-          explicit_strides = test_util.NHWCToNCHW(explicit_strides)
         conv = nn_ops.conv2d_backprop_filter(
             t0,
             t1,
             t2,
             strides=explicit_strides,
-            padding=padding,
+            padding=new_padding,
+            dilations=new_dilations,
             data_format=data_format)
         value = self.evaluate(conv)
         self.assertShapeEqual(value, conv)
-      tf_logging.info("expected = ", expected)
-      tf_logging.info("actual = ", value)
-      self.assertArrayNear(expected, value.flatten(), 1e-5)
+      tf_logging.debug("expected = %s", expected)
+      tf_logging.debug("actual = %s", value)
+      self.assertArrayNear(expected, value.flatten(), err)
 
   def _CompareBackFilter(self, input_sizes, filter_sizes, output_sizes,
                          conv_strides, padding):
@@ -866,16 +1074,8 @@ class Conv2DTest(test.TestCase):
   def _RunAndVerifyBackpropInputDilation(self, input_sizes, filter_sizes,
                                          output_sizes, strides, dilations,
                                          padding, data_format, use_gpu, err):
-    total_input_size = 1
-    total_filter_size = 1
-    for s in input_sizes:
-      total_input_size *= s
-    for s in filter_sizes:
-      total_filter_size *= s
-    # Initializes the input tensor with array containing incrementing
-    # numbers from 1.
-    x1 = [f * 1.0 for f in range(1, total_input_size + 1)]
-    x2 = [f * 1.0 for f in range(1, total_filter_size + 1)]
+    x1 = self._CreateNumpyTensor(input_sizes)
+    x2 = self._CreateNumpyTensor(filter_sizes)
     default_dilations = (dilations[0] == 1 and dilations[1] == 1)
     if default_dilations or use_gpu:
       with self.cached_session(use_gpu=use_gpu) as sess:
@@ -912,24 +1112,16 @@ class Conv2DTest(test.TestCase):
         value_2 = self.evaluate(conv_2)
         self.assertShapeEqual(value, conv)
         self.assertShapeEqual(value_2, conv_2)
-      tf_logging.info("expected = ", value_2)
-      tf_logging.info("actual = ", value)
+      tf_logging.debug("expected = %s", value_2)
+      tf_logging.debug("actual = %s", value)
       self.assertArrayNear(value_2.flatten(), value.flatten(), err)
 
   # Testing for backprops
   def _RunAndVerifyBackpropFilterDilation(self, input_sizes, filter_sizes,
                                           output_sizes, strides, dilations,
                                           padding, data_format, use_gpu, err):
-    total_input_size = 1
-    total_filter_size = 1
-    for s in input_sizes:
-      total_input_size *= s
-    for s in filter_sizes:
-      total_filter_size *= s
-    # Initializes the input tensor with array containing incrementing
-    # numbers from 1.
-    x1 = [f * 1.0 for f in range(1, total_input_size + 1)]
-    x2 = [f * 1.0 for f in range(1, total_filter_size + 1)]
+    x1 = self._CreateNumpyTensor(input_sizes)
+    x2 = self._CreateNumpyTensor(filter_sizes)
     default_dilations = (dilations[0] == 1 and dilations[1] == 1)
     if default_dilations or use_gpu:
       with self.cached_session(use_gpu=use_gpu) as sess:
@@ -965,8 +1157,8 @@ class Conv2DTest(test.TestCase):
         value_2 = self.evaluate(conv_2)
         self.assertShapeEqual(value, conv)
         self.assertShapeEqual(value_2, conv_2)
-      tf_logging.info("expected = ", value_2)
-      tf_logging.info("actual = ", value)
+      tf_logging.debug("expected = %s", value_2)
+      tf_logging.debug("actual = %s", value)
       self.assertArrayNear(value_2.flatten(), value.flatten(), err)
 
   def testConv2D2x2Depth3ValidBackpropFilterStride1x1Dilation2x1(self):
@@ -1111,20 +1303,347 @@ class Conv2DTest(test.TestCase):
             use_gpu=use_gpu,
             err=1e-5)
 
+  def _RunAndVerifyBackpropInputExplicitPadding(self,
+                                                input_sizes,
+                                                filter_sizes,
+                                                output_sizes,
+                                                strides,
+                                                padding,
+                                                data_format,
+                                                dilations=(1, 1),
+                                                err=2e-5):
+    x1 = self._CreateNumpyTensor(filter_sizes)
+    x2 = self._CreateNumpyTensor(output_sizes)
+    dilations = list(dilations)
+    padded_input_sizes = input_sizes[:]
+    padded_input_sizes[1] += padding[0][0] + padding[0][1]
+    padded_input_sizes[2] += padding[1][0] + padding[1][1]
+    c = nn_ops.conv2d_backprop_input(
+        padded_input_sizes,
+        x1,
+        x2,
+        strides=[1] + strides + [1],
+        padding="VALID",
+        dilations=[1] + dilations + [1])
+    c = c[:, padding[0][0]:(c.shape[1] - padding[0][1]), padding[1][0]:(
+        c.shape[2] - padding[1][1]), :]
+    expected = list(self.evaluate(array_ops.reshape(c, [-1])))
+    self._RunAndVerifyBackpropInput(
+        input_sizes,
+        filter_sizes,
+        output_sizes,
+        strides,
+        padding,
+        expected,
+        data_format,
+        use_gpu=True,
+        err=err,
+        dilations=dilations)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testConv2D2x2Depth1Padding0x0BackpropInput(self):
+    if not test.is_gpu_available(cuda_only=True):
+      return
+    for (data_format, use_gpu) in GetTestConfigs():
+      if use_gpu:
+        self._RunAndVerifyBackpropInputExplicitPadding(
+            input_sizes=[1, 2, 3, 1],
+            filter_sizes=[2, 2, 1, 1],
+            output_sizes=[1, 1, 2, 1],
+            strides=[1, 1],
+            padding=[[0, 0], [0, 0]],
+            data_format=data_format)
+
+        self._RunAndVerifyBackpropInputExplicitPadding(
+            input_sizes=[1, 3, 4, 2],
+            filter_sizes=[2, 2, 2, 3],
+            output_sizes=[1, 1, 2, 3],
+            strides=[2, 2],
+            padding=[[0, 0], [0, 0]],
+            data_format=data_format)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testConv2D2x2Depth1Padding1x1BackpropInput(self):
+    if not test.is_gpu_available(cuda_only=True):
+      return
+
+    for (data_format, use_gpu) in GetTestConfigs():
+      if use_gpu:
+        self._RunAndVerifyBackpropInputExplicitPadding(
+            input_sizes=[1, 2, 3, 1],
+            filter_sizes=[2, 2, 1, 2],
+            output_sizes=[1, 3, 4, 2],
+            strides=[1, 1],
+            padding=[[1, 1], [1, 1]],
+            data_format=data_format, err=1e-4)
+
+        self._RunAndVerifyBackpropInputExplicitPadding(
+            input_sizes=[1, 2, 3, 2],
+            filter_sizes=[1, 1, 2, 1],
+            output_sizes=[1, 4, 3, 1],
+            strides=[1, 2],
+            padding=[[1, 1], [1, 1]],
+            data_format=data_format)
+
+        self._RunAndVerifyBackpropInputExplicitPadding(
+            input_sizes=[1, 4, 3, 1],
+            filter_sizes=[2, 2, 1, 1],
+            output_sizes=[1, 4, 2, 1],
+            strides=[1, 2],
+            padding=[[1, 1], [1, 1]],
+            data_format=data_format,
+            dilations=[2, 2])
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testConv2D2x2Depth1Padding2x2BackpropInput(self):
+    if not test.is_gpu_available(cuda_only=True):
+      return
+
+    for (data_format, use_gpu) in GetTestConfigs():
+      if use_gpu:
+        self._RunAndVerifyBackpropInputExplicitPadding(
+            input_sizes=[2, 3, 1, 1],
+            filter_sizes=[2, 1, 1, 1],
+            output_sizes=[2, 2, 5, 1],
+            strides=[3, 1],
+            padding=[[2, 2], [2, 2]],
+            data_format=data_format)
+
+        self._RunAndVerifyBackpropInputExplicitPadding(
+            input_sizes=[1, 3, 6, 1],
+            filter_sizes=[3, 2, 1, 1],
+            output_sizes=[1, 3, 4, 1],
+            strides=[1, 2],
+            padding=[[2, 2], [2, 2]],
+            data_format=data_format,
+            dilations=[2, 3])
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testConv2D2x2Depth1Padding_1_8_4_1_BackpropInput(self):
+    if not test.is_gpu_available(cuda_only=True):
+      return
+    for (data_format, use_gpu) in GetTestConfigs():
+      if use_gpu:
+        self._RunAndVerifyBackpropInputExplicitPadding(
+            input_sizes=[1, 2, 3, 1],
+            filter_sizes=[2, 2, 1, 1],
+            output_sizes=[1, 10, 8, 1],
+            strides=[1, 1],
+            padding=[[1, 8], [4, 2]],
+            data_format=data_format, err=5e-5)
+
+        self._RunAndVerifyBackpropInputExplicitPadding(
+            input_sizes=[1, 5, 3, 1],
+            filter_sizes=[3, 2, 1, 1],
+            output_sizes=[1, 4, 8, 1],
+            strides=[3, 1],
+            padding=[[1, 8], [4, 2]],
+            data_format=data_format)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testConv2D2x2Depth1Padding_5_0_2_2_BackpropInput(self):
+    if not test.is_gpu_available(cuda_only=True):
+      return
+    for (data_format, use_gpu) in GetTestConfigs():
+      if use_gpu:
+        self._RunAndVerifyBackpropInputExplicitPadding(
+            input_sizes=[1, 3, 3, 1],
+            filter_sizes=[2, 1, 1, 1],
+            output_sizes=[1, 7, 7, 1],
+            strides=[1, 1],
+            padding=[[5, 0], [2, 2]],
+            data_format=data_format,
+            err=5e-5)
+
+        self._RunAndVerifyBackpropInputExplicitPadding(
+            input_sizes=[1, 4, 2, 1],
+            filter_sizes=[3, 3, 1, 1],
+            output_sizes=[1, 5, 2, 1],
+            strides=[1, 2],
+            padding=[[5, 0], [2, 2]],
+            data_format=data_format,
+            dilations=[2, 1])
+
+  def _RunAndVerifyBackpropFilterExplicitPadding(self,
+                                                 input_sizes,
+                                                 filter_sizes,
+                                                 output_sizes,
+                                                 strides,
+                                                 padding,
+                                                 data_format,
+                                                 dilations=(1, 1),
+                                                 err=1e-5):
+    x0 = self._CreateNumpyTensor(input_sizes)
+    x2 = self._CreateNumpyTensor(output_sizes)
+    dilations = list(dilations)
+
+    x0 = np.pad(x0, [(0, 0)] + padding + [(0, 0)], "constant")
+    c = nn_ops.conv2d_backprop_filter(
+        x0,
+        filter_sizes,
+        x2,
+        strides=[1] + strides + [1],
+        padding="VALID",
+        dilations=[1] + dilations + [1])
+    expected = list(self.evaluate(array_ops.reshape(c, [-1])))
+    self._RunAndVerifyBackpropFilter(
+        input_sizes,
+        filter_sizes,
+        output_sizes,
+        strides,
+        padding,
+        expected,
+        data_format,
+        use_gpu=True,
+        dilations=dilations,
+        err=err)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testConv2D2x2Depth1Padding0x0BackpropFilter(self):
+    if not test.is_gpu_available(cuda_only=True):
+      return
+    for (data_format, use_gpu) in GetTestConfigs():
+      if use_gpu:
+        self._RunAndVerifyBackpropFilterExplicitPadding(
+            input_sizes=[1, 2, 3, 1],
+            filter_sizes=[2, 2, 1, 1],
+            output_sizes=[1, 1, 2, 1],
+            strides=[1, 1],
+            padding=[[0, 0], [0, 0]],
+            data_format=data_format)
+
+        self._RunAndVerifyBackpropFilterExplicitPadding(
+            input_sizes=[1, 3, 4, 2],
+            filter_sizes=[2, 2, 2, 3],
+            output_sizes=[1, 1, 2, 3],
+            strides=[2, 2],
+            padding=[[0, 0], [0, 0]],
+            data_format=data_format)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testConv2D2x2Depth1Padding1x1BackpropFilter(self):
+    if not test.is_gpu_available(cuda_only=True):
+      return
+
+    for (data_format, use_gpu) in GetTestConfigs():
+      if use_gpu:
+        self._RunAndVerifyBackpropFilterExplicitPadding(
+            input_sizes=[1, 2, 3, 1],
+            filter_sizes=[2, 2, 1, 2],
+            output_sizes=[1, 3, 4, 2],
+            strides=[1, 1],
+            padding=[[1, 1], [1, 1]],
+            data_format=data_format,
+            err=5e-5)
+
+        self._RunAndVerifyBackpropFilterExplicitPadding(
+            input_sizes=[1, 2, 3, 2],
+            filter_sizes=[1, 1, 2, 1],
+            output_sizes=[1, 4, 3, 1],
+            strides=[1, 2],
+            padding=[[1, 1], [1, 1]],
+            data_format=data_format)
+
+        self._RunAndVerifyBackpropFilterExplicitPadding(
+            input_sizes=[1, 4, 3, 1],
+            filter_sizes=[2, 2, 1, 1],
+            output_sizes=[1, 4, 2, 1],
+            strides=[1, 2],
+            padding=[[1, 1], [1, 1]],
+            data_format=data_format,
+            dilations=[2, 2])
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testConv2D2x2Depth1Padding2x2BackpropFilter(self):
+    if not test.is_gpu_available(cuda_only=True):
+      return
+
+    for (data_format, use_gpu) in GetTestConfigs():
+      if use_gpu:
+        self._RunAndVerifyBackpropFilterExplicitPadding(
+            input_sizes=[2, 3, 1, 1],
+            filter_sizes=[2, 1, 1, 1],
+            output_sizes=[2, 2, 5, 1],
+            strides=[3, 1],
+            padding=[[2, 2], [2, 2]],
+            data_format=data_format)
+
+        self._RunAndVerifyBackpropFilterExplicitPadding(
+            input_sizes=[1, 3, 6, 1],
+            filter_sizes=[3, 2, 1, 1],
+            output_sizes=[1, 3, 4, 1],
+            strides=[1, 2],
+            padding=[[2, 2], [2, 2]],
+            data_format=data_format,
+            dilations=[2, 3])
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testConv2D2x2Depth1Padding_1_8_4_1_BackpropFilter(self):
+    if not test.is_gpu_available(cuda_only=True):
+      return
+    for (data_format, use_gpu) in GetTestConfigs():
+      if use_gpu:
+        self._RunAndVerifyBackpropFilterExplicitPadding(
+            input_sizes=[1, 2, 3, 1],
+            filter_sizes=[2, 2, 1, 1],
+            output_sizes=[1, 10, 8, 1],
+            strides=[1, 1],
+            padding=[[1, 8], [4, 2]],
+            data_format=data_format,
+            err=1e-4)
+
+        self._RunAndVerifyBackpropFilterExplicitPadding(
+            input_sizes=[1, 5, 3, 1],
+            filter_sizes=[3, 2, 1, 1],
+            output_sizes=[1, 4, 8, 1],
+            strides=[3, 1],
+            padding=[[1, 8], [4, 2]],
+            data_format=data_format)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testConv2D2x2Depth1Padding_5_0_2_2_BackpropFilter(self):
+    if not test.is_gpu_available(cuda_only=True):
+      return
+    for (data_format, use_gpu) in GetTestConfigs():
+      if use_gpu:
+        self._RunAndVerifyBackpropFilterExplicitPadding(
+            input_sizes=[1, 3, 3, 1],
+            filter_sizes=[2, 1, 1, 1],
+            output_sizes=[1, 7, 7, 1],
+            strides=[1, 1],
+            padding=[[5, 0], [2, 2]],
+            data_format=data_format,
+            err=1e-4)
+
+        self._RunAndVerifyBackpropFilterExplicitPadding(
+            input_sizes=[1, 4, 2, 1],
+            filter_sizes=[3, 3, 1, 1],
+            output_sizes=[1, 5, 2, 1],
+            strides=[1, 2],
+            padding=[[5, 0], [2, 2]],
+            data_format=data_format,
+            dilations=[2, 1])
+
   # Gradient checkers
   def ConstructAndTestGradient(self, batch, input_rows, input_cols, filter_rows,
                                filter_cols, in_depth, out_depth, stride_rows,
                                stride_cols, padding, test_input, data_format,
-                               use_gpu):
+                               use_gpu, max_err=0.002):
     input_shape = [batch, input_rows, input_cols, in_depth]
     filter_shape = [filter_rows, filter_cols, in_depth, out_depth]
     # TODO(yangke): re-factor the computation of output shape.
     if padding == "VALID":
       output_rows = (input_rows - filter_rows + stride_rows) // stride_rows
       output_cols = (input_cols - filter_cols + stride_cols) // stride_cols
-    else:
+    elif padding == "SAME":
       output_rows = (input_rows + stride_rows - 1) // stride_rows
       output_cols = (input_cols + stride_cols - 1) // stride_cols
+    else:
+      self.assertIsInstance(padding, (list, tuple))
+      output_rows = (input_rows + padding[1][0] + padding[1][1] - filter_rows +
+                     stride_rows) // stride_rows
+      output_cols = (input_cols + padding[2][0] + padding[2][1] - filter_cols +
+                     stride_cols) // stride_cols
     output_shape = [batch, output_rows, output_cols, out_depth]
     input_size = 1
     for x in input_shape:
@@ -1145,16 +1664,19 @@ class Conv2DTest(test.TestCase):
         filter_tensor = constant_op.constant(
             filter_data, shape=filter_shape, dtype=dtype, name="filter")
         strides = [1, stride_rows, stride_cols, 1]
+        new_padding = padding
         if data_format == "NCHW":
           new_input_tensor = test_util.NHWCToNCHW(input_tensor)
           strides = test_util.NHWCToNCHW(strides)
+          if isinstance(padding, (list, tuple)):
+            new_padding = test_util.NHWCToNCHW(padding)
         else:
           new_input_tensor = input_tensor
         conv = nn_ops.conv2d(
             new_input_tensor,
             filter_tensor,
             strides,
-            padding,
+            new_padding,
             data_format=data_format,
             name="conv")
         if data_format == "NCHW":
@@ -1178,8 +1700,8 @@ class Conv2DTest(test.TestCase):
           # since fp16 numerical gradients are too imprecise.
           err = np.fabs(jacob_t - reference_jacob_t).max()
 
-        tf_logging.info("conv_2d gradient error = ", err)
-        self.assertLess(err, 0.002)
+        tf_logging.debug("conv_2d gradient error = %s", err)
+        self.assertLess(err, max_err)
 
   def testInputGradientValidPaddingStrideOne(self):
     for (data_format, use_gpu) in GetTestConfigs():
@@ -1436,6 +1958,248 @@ class Conv2DTest(test.TestCase):
           data_format=data_format,
           use_gpu=use_gpu)
 
+  def testInputGradient1x1PaddingStrideOne(self):
+    if not test.is_gpu_available(cuda_only=True):
+      return
+    for (data_format, use_gpu) in GetTestConfigs():
+      if use_gpu:
+        self.ConstructAndTestGradient(
+            batch=2,
+            input_rows=5,
+            input_cols=4,
+            filter_rows=3,
+            filter_cols=3,
+            in_depth=2,
+            out_depth=3,
+            stride_rows=1,
+            stride_cols=1,
+            padding=[[0, 0], [1, 1], [1, 1], [0, 0]],
+            test_input=True,
+            data_format=data_format,
+            use_gpu=use_gpu,
+            max_err=0.0025)
+
+  def testFilterGradient1x1PaddingStrideOne(self):
+    if not test.is_gpu_available(cuda_only=True):
+      return
+    for (data_format, use_gpu) in GetTestConfigs():
+      if use_gpu:
+        self.ConstructAndTestGradient(
+            batch=2,
+            input_rows=5,
+            input_cols=4,
+            filter_rows=3,
+            filter_cols=3,
+            in_depth=2,
+            out_depth=3,
+            stride_rows=1,
+            stride_cols=1,
+            padding=[[0, 0], [1, 1], [1, 1], [0, 0]],
+            test_input=False,
+            data_format=data_format,
+            use_gpu=use_gpu)
+
+  def testInputGradient1x1PaddingStrideTwo(self):
+    if not test.is_gpu_available(cuda_only=True):
+      return
+    for (data_format, use_gpu) in GetTestConfigs():
+      if use_gpu:
+        self.ConstructAndTestGradient(
+            batch=2,
+            input_rows=4,
+            input_cols=5,
+            filter_rows=3,
+            filter_cols=3,
+            in_depth=2,
+            out_depth=3,
+            stride_rows=2,
+            stride_cols=2,
+            padding=[[0, 0], [1, 1], [1, 1], [0, 0]],
+            test_input=True,
+            data_format=data_format,
+            use_gpu=use_gpu)
+
+  def testFilterGradient1x1PaddingStrideTwo(self):
+    if not test.is_gpu_available(cuda_only=True):
+      return
+    for (data_format, use_gpu) in GetTestConfigs():
+      if use_gpu:
+        self.ConstructAndTestGradient(
+            batch=2,
+            input_rows=4,
+            input_cols=5,
+            filter_rows=3,
+            filter_cols=3,
+            in_depth=2,
+            out_depth=3,
+            stride_rows=2,
+            stride_cols=2,
+            padding=[[0, 0], [1, 1], [1, 1], [0, 0]],
+            test_input=False,
+            data_format=data_format,
+            use_gpu=use_gpu)
+
+  def testInputGradient2x2PaddingStrideOne(self):
+    if not test.is_gpu_available(cuda_only=True):
+      return
+    for (data_format, use_gpu) in GetTestConfigs():
+      if use_gpu:
+        self.ConstructAndTestGradient(
+            batch=2,
+            input_rows=5,
+            input_cols=4,
+            filter_rows=3,
+            filter_cols=3,
+            in_depth=2,
+            out_depth=3,
+            stride_rows=1,
+            stride_cols=1,
+            padding=[[0, 0], [2, 2], [2, 2], [0, 0]],
+            test_input=True,
+            data_format=data_format,
+            use_gpu=use_gpu)
+
+  def testFilterGradient2x2PaddingStrideOne(self):
+    if not test.is_gpu_available(cuda_only=True):
+      return
+    for (data_format, use_gpu) in GetTestConfigs():
+      if use_gpu:
+        self.ConstructAndTestGradient(
+            batch=2,
+            input_rows=5,
+            input_cols=4,
+            filter_rows=3,
+            filter_cols=3,
+            in_depth=2,
+            out_depth=3,
+            stride_rows=1,
+            stride_cols=1,
+            padding=[[0, 0], [2, 2], [2, 2], [0, 0]],
+            test_input=False,
+            data_format=data_format,
+            use_gpu=use_gpu,
+            max_err=0.003)
+
+  def testInputGradient1_2_3_4PaddingStride3x2(self):
+    if not test.is_gpu_available(cuda_only=True):
+      return
+    for (data_format, use_gpu) in GetTestConfigs():
+      if use_gpu:
+        self.ConstructAndTestGradient(
+            batch=2,
+            input_rows=8,
+            input_cols=5,
+            filter_rows=4,
+            filter_cols=2,
+            in_depth=3,
+            out_depth=2,
+            stride_rows=3,
+            stride_cols=2,
+            padding=[[0, 0], [1, 2], [3, 4], [0, 0]],
+            test_input=True,
+            data_format=data_format,
+            use_gpu=use_gpu)
+
+  def testFilterGradient1_2_3_4PaddingStride3x2(self):
+    if not test.is_gpu_available(cuda_only=True):
+      return
+    for (data_format, use_gpu) in GetTestConfigs():
+      if use_gpu:
+        self.ConstructAndTestGradient(
+            batch=2,
+            input_rows=8,
+            input_cols=5,
+            filter_rows=4,
+            filter_cols=2,
+            in_depth=3,
+            out_depth=2,
+            stride_rows=3,
+            stride_cols=2,
+            padding=[[0, 0], [1, 2], [3, 4], [0, 0]],
+            test_input=False,
+            data_format=data_format,
+            use_gpu=use_gpu)
+
+  def testInputGradient4_3_2_1PaddingStride2x1(self):
+    if not test.is_gpu_available(cuda_only=True):
+      return
+    for (data_format, use_gpu) in GetTestConfigs():
+      if use_gpu:
+        self.ConstructAndTestGradient(
+            batch=3,
+            input_rows=5,
+            input_cols=7,
+            filter_rows=3,
+            filter_cols=2,
+            in_depth=1,
+            out_depth=2,
+            stride_rows=2,
+            stride_cols=1,
+            padding=[[0, 0], [4, 3], [2, 1], [0, 0]],
+            test_input=True,
+            data_format=data_format,
+            use_gpu=use_gpu)
+
+  def testFilterGradient4_3_2_1PaddingStride2x1(self):
+    if not test.is_gpu_available(cuda_only=True):
+      return
+    for (data_format, use_gpu) in GetTestConfigs():
+      if use_gpu:
+        self.ConstructAndTestGradient(
+            batch=3,
+            input_rows=5,
+            input_cols=7,
+            filter_rows=3,
+            filter_cols=2,
+            in_depth=1,
+            out_depth=2,
+            stride_rows=2,
+            stride_cols=1,
+            padding=[[0, 0], [4, 3], [2, 1], [0, 0]],
+            test_input=False,
+            data_format=data_format,
+            use_gpu=use_gpu)
+
+  def testInputGradient0_0_0_5PaddingStride1x2(self):
+    if not test.is_gpu_available(cuda_only=True):
+      return
+    for (data_format, use_gpu) in GetTestConfigs():
+      if use_gpu:
+        self.ConstructAndTestGradient(
+            batch=2,
+            input_rows=6,
+            input_cols=7,
+            filter_rows=3,
+            filter_cols=4,
+            in_depth=3,
+            out_depth=2,
+            stride_rows=1,
+            stride_cols=2,
+            padding=[[0, 0], [0, 0], [0, 5], [0, 0]],
+            test_input=True,
+            data_format=data_format,
+            use_gpu=use_gpu)
+
+  def testFilterGradient0_0_0_5PaddingStride1x2(self):
+    if not test.is_gpu_available(cuda_only=True):
+      return
+    for (data_format, use_gpu) in GetTestConfigs():
+      if use_gpu:
+        self.ConstructAndTestGradient(
+            batch=2,
+            input_rows=6,
+            input_cols=7,
+            filter_rows=3,
+            filter_cols=4,
+            in_depth=3,
+            out_depth=2,
+            stride_rows=1,
+            stride_cols=2,
+            padding=[[0, 0], [0, 0], [0, 5], [0, 0]],
+            test_input=False,
+            data_format=data_format,
+            use_gpu=use_gpu)
+
   def testShapeFunctionEdgeCases(self):
     # All shapes unknown.
     c1 = nn_ops.conv2d(
@@ -1473,6 +2237,55 @@ class Conv2DTest(test.TestCase):
           strides=[1, 1, 1, 1],
           padding="SAME")
 
+    # Negative padding.
+    with self.assertRaises(ValueError):
+      nn_ops.conv2d(
+          array_ops.placeholder(dtypes.float32),
+          array_ops.placeholder(dtypes.float32),
+          strides=[1, 1, 1, 1],
+          padding=[[0, 0], [0, -1], [1, 2], [0, 0]])
+
+    # Nonzero padding in nonspatial dimension.
+    with self.assertRaises(ValueError):
+      nn_ops.conv2d(
+          array_ops.placeholder(dtypes.float32),
+          array_ops.placeholder(dtypes.float32),
+          strides=[1, 1, 1, 1],
+          padding=[[1, 0], [0, 0], [0, 0], [0, 0]])
+
+    # Nonzero NCHW padding in nonspatial dimension.
+    with self.assertRaises(ValueError):
+      nn_ops.conv2d(
+          array_ops.placeholder(dtypes.float32),
+          array_ops.placeholder(dtypes.float32),
+          strides=[1, 1, 1, 1],
+          padding=[[0, 0], [0, 1], [0, 0], [0, 0]],
+          data_format="NCHW")
+
+    # Wrong amount of padding
+    with self.assertRaises(ValueError):
+      nn_ops.conv2d(
+          array_ops.placeholder(dtypes.float32),
+          array_ops.placeholder(dtypes.float32),
+          strides=[1, 1, 1, 1],
+          padding=[[0, 0], [0, 0], [0, 0]])
+
+    # Only specify one padding amount per dimension
+    with self.assertRaises(ValueError):
+      nn_ops.conv2d(
+          array_ops.placeholder(dtypes.float32),
+          array_ops.placeholder(dtypes.float32),
+          strides=[1, 1, 1, 1],
+          padding=[[0], [0], [0], [0]])
+
+    # Explicit padding elements are not lists
+    with self.assertRaises(ValueError):
+      nn_ops.conv2d(
+          array_ops.placeholder(dtypes.float32),
+          array_ops.placeholder(dtypes.float32),
+          strides=[1, 1, 1, 1],
+          padding=[0, 0, 0, 0])
+
   def testOpEdgeCases(self):
     with self.cached_session() as sess:
       # Illegal strides.
@@ -1513,6 +2326,41 @@ class Conv2DTest(test.TestCase):
                 strides=[1, 1, 1, 1],
                 padding="VALID"))
 
+      # Filter larger than input + padding.
+      with self.assertRaisesRegexp(ValueError, "Negative dimension size"):
+        sess.run(
+            nn_ops.conv2d(
+                array_ops.placeholder(dtypes.float32, shape=[32, 20, 20, 3]),
+                array_ops.placeholder(dtypes.float32, shape=[24, 25, 3, 2]),
+                strides=[1, 1, 1, 1],
+                padding=[[0, 0], [2, 2], [2, 2], [0, 0]]))
+
+    if test.is_gpu_available(cuda_only=True):
+      with self.test_session(use_gpu=True):
+        # Negative padding during backprop.
+        with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
+                                     "nonnegative"):
+          sess.run(
+              nn_ops.conv2d_backprop_input([32, 20, 20, 3],
+                                           array_ops.placeholder(
+                                               dtypes.float32,
+                                               shape=[18, 18, 3, 2]),
+                                           array_ops.placeholder(
+                                               dtypes.float32,
+                                               shape=[32, 3, 2, 2]),
+                                           strides=[1, 1, 1, 1],
+                                           padding=[[0, 0], [-1, 0], [0, 0],
+                                                    [0, 0]]))
+        with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
+                                     "nonnegative"):
+          sess.run(
+              nn_ops.conv2d_backprop_filter(
+                  array_ops.placeholder(dtypes.float32, shape=[32, 20, 20, 3]),
+                  [18, 18, 3, 2],
+                  array_ops.placeholder(dtypes.float32, shape=[32, 3, 2, 2]),
+                  strides=[1, 1, 1, 1],
+                  padding=[[0, 0], [-1, 0], [0, 0], [0, 0]]))
+
 
 class DepthwiseConv2DTest(test.TestCase):
 
@@ -1546,7 +2394,7 @@ class DepthwiseConv2DTest(test.TestCase):
       conv = nn_impl.depthwise_conv2d(
           t1, t2, strides=[1, stride, stride, 1], padding=padding)
       value = self.evaluate(conv)
-    tf_logging.info("value = ", value)
+    tf_logging.debug("value = %s", value)
     self.assertArrayNear(expected, np.ravel(value), 1e-5)
     self.assertShapeEqual(value, conv)
 
@@ -1668,7 +2516,7 @@ class SeparableConv2DTest(test.TestCase):
         conv = array_ops.transpose(conv, [0, 2, 3, 1])
 
       value = self.evaluate(conv)
-    tf_logging.info("value = ", value)
+    tf_logging.debug("value = %s", value)
     self.assertArrayNear(expected, np.ravel(value), 1e-3)
     self.assertShapeEqual(value, conv)
 
@@ -1828,6 +2676,194 @@ class Conv2DBenchmark(test.Benchmark):
             name="conv_stack_iter_%d" % iter_index, wall_time=wall_time)
         tf_logging.info("conv_stack_iter_%d: %.4f" % (iter_index, wall_time))
 
+  def _bench_op(self, name, op, burn_iters, num_iters):
+    config = config_pb2.ConfigProto()
+    # Prevent Grappler from optimizing away the entire graph.
+    config.graph_options.rewrite_options.dependency_optimization = (
+        rewriter_config_pb2.RewriterConfig.OFF)
+    with session_lib.Session(config=config) as session:
+      variables.global_variables_initializer().run()
+      self.run_op_benchmark(
+          session, op, burn_iters=burn_iters, min_iters=num_iters, name=name)
+
+  def benchmarkExplicitVsManualPadding(self):
+    """Compare performance of EXPLICIT padding and calling tf.pad.
+
+    A Conv2D op with EXPLICIT padding is benchmarked, and a tf.pad with the same
+    padding followed by an equivalent Conv2D op is benchmarked.
+    """
+    if not test.is_gpu_available():
+      return
+
+    with ops.Graph().as_default():
+      burn_iters = 15
+      num_iters = 300
+      batch_size = 64
+      # The input and filter correspond to the first layer of Resnet50.
+      input = variables.Variable(  # pylint: disable=redefined-builtin
+          random_ops.random_uniform([
+              batch_size,
+              3,
+              224,
+              224
+          ]))
+      filter = variables.Variable(random_ops.random_uniform([7, 7, 3, 64]))  # pylint: disable=redefined-builtin
+      strides = [1, 1, 2, 2]
+      padding = [(0, 0), (0, 0), (3, 3), (3, 3)]
+      output_explicit_pad = nn_ops.conv2d(
+          input, filter, strides, padding=padding, data_format="NCHW")
+      input_padded = array_ops.pad(input, padding)
+      output_manual_pad = nn_ops.conv2d(
+          input_padded, filter, strides, padding="VALID", data_format="NCHW")
+      # Benchmark just the forward pass.
+      self._bench_op("explicit_pad_forward", output_explicit_pad.op, burn_iters,
+                     num_iters)
+      self._bench_op("manual_pad_forward", output_manual_pad.op, burn_iters,
+                     num_iters)
+
+      # Benchmark both the forward and backwards passes.
+      input_grad_explicit_pad, filter_grad_explicit_pad = (
+          gradients_impl.gradients(output_explicit_pad, [input, filter]))
+      self._bench_op(
+          "explicit_pad_backward",
+          control_flow_ops.group(input_grad_explicit_pad,
+                                 filter_grad_explicit_pad), burn_iters,
+          num_iters)
+      input_grad_manual_pad, filter_grad_manual_pad = gradients_impl.gradients(
+          output_manual_pad, [input, filter])
+      self._bench_op(
+          "manual_pad_backward",
+          control_flow_ops.group(input_grad_manual_pad, filter_grad_manual_pad),
+          burn_iters, num_iters)
+
+  def benchmarkExplicitVsSamePaddingGraph(self):
+    """Compare performance of EXPLICIT and SAME padding in graph mode.
+
+    A Conv2D op with SAME padding is benchmarked, and an equivalent Conv2D op
+    with explicit padding is benchmarked, where the padding is the same as in
+    the SAME case. The purpose is to ensure EXPLICIT padding is just as
+    efficient as the SAME case
+    """
+    if not test.is_gpu_available():
+      return
+
+    with ops.Graph().as_default():
+      burn_iters = 15
+      num_convs = 20
+      num_iters = 50
+      batch_size = 64
+      # The input and filter correspond to a middle layer of Resnet50.
+      input = variables.Variable(  # pylint: disable=redefined-builtin
+          random_ops.random_uniform([
+              batch_size,
+              256,
+              14,
+              14
+          ]))
+      filter = variables.Variable(random_ops.random_uniform([3, 3, 256, 256]))  # pylint: disable=redefined-builtin
+      strides = [1, 1, 1, 1]
+      padding = [(0, 0), (0, 0), (1, 1), (1, 1)]
+      output_explicit_pad = input
+      output_same_pad = input
+
+      for _ in range(num_convs):
+        output_explicit_pad = nn_ops.conv2d(
+            output_explicit_pad,
+            filter,
+            strides,
+            padding=padding,
+            data_format="NCHW")
+        output_same_pad = nn_ops.conv2d(
+            output_same_pad,
+            filter,
+            strides,
+            padding="SAME",
+            data_format="NCHW")
+      grad_explicit_pad, = gradients_impl.gradients(output_explicit_pad, filter)
+      grad_same_pad, = gradients_impl.gradients(output_same_pad, filter)
+      self._bench_op("graph_explicit_pad", grad_explicit_pad.op, burn_iters,
+                     num_iters)
+      self._bench_op("graph_same_pad", grad_same_pad.op, burn_iters, num_iters)
+
+  def benchmarkExplicitVsSamePaddingEager(self):
+    """Compare performance of EXPLICIT and SAME padding in eager mode.
+
+    A Conv2D op with SAME padding is benchmarked, and an equivalent Conv2D op
+    with explicit padding is benchmarked, where the padding is the same as in
+    the SAME case. Currently, EXPLICIT padding is slightly slower, due to the
+    fact the Python padding list must be checked and processed before the Conv2D
+    op can run.
+    """
+    # TODO(reedwm): Make EXPLICIT padding as fast as SAME padding.
+    if not test.is_gpu_available():
+      return
+
+    with context.eager_mode():
+      burn_iters = 15
+      num_convs = 20
+      num_iters = 50
+      batch_size = 64
+      # The input and filter correspond to a middle layer of Resnet50.
+      input = variables.Variable(  # pylint: disable=redefined-builtin
+          random_ops.random_uniform([
+              batch_size,
+              256,
+              14,
+              14
+          ]))
+      filter = variables.Variable(random_ops.random_uniform([3, 3, 256, 256]))  # pylint: disable=redefined-builtin
+      strides = [1, 1, 1, 1]
+      padding = [(0, 0), (0, 0), (1, 1), (1, 1)]
+      output_explicit_pad = input
+      output_same_pad = input
+      for _ in range(burn_iters):
+        output_explicit_pad = nn_ops.conv2d(
+            output_explicit_pad,
+            filter,
+            strides,
+            padding=padding,
+            data_format="NCHW")
+        output_same_pad = nn_ops.conv2d(
+            output_same_pad,
+            filter,
+            strides,
+            padding="SAME",
+            data_format="NCHW")
+
+      start = time.time()
+      for _ in range(num_iters):
+        with backprop.GradientTape() as tape:
+          for _ in range(num_convs):
+            output_explicit_pad = nn_ops.conv2d(
+                output_explicit_pad,
+                filter,
+                strides,
+                padding=padding,
+                data_format="NCHW")
+          tape.gradient(output_explicit_pad, filter)
+      end = time.time()
+      self.report_benchmark(
+          name="eager_explicit_pad",
+          wall_time=(end - start) / num_iters,
+          iters=num_iters)
+
+      start = time.time()
+      for _ in range(num_iters):
+        with backprop.GradientTape() as tape:
+          for _ in range(num_convs):
+            output_same_pad = nn_ops.conv2d(
+                output_same_pad,
+                filter,
+                strides,
+                padding="SAME",
+                data_format="NCHW")
+          tape.gradient(output_same_pad, filter)
+      end = time.time()
+      self.report_benchmark(
+          name="eager_same_pad",
+          wall_time=(end - start) / num_iters,
+          iters=num_iters)
+
 
 def GetInceptionFwdTest(input_size, filter_size, stride, padding,
                         gpu_only=False):
diff --git a/tensorflow/python/ops/nn_grad.py b/tensorflow/python/ops/nn_grad.py
index 7131e4abc4..6ca2b2aafe 100644
--- a/tensorflow/python/ops/nn_grad.py
+++ b/tensorflow/python/ops/nn_grad.py
@@ -514,29 +514,40 @@ def _SparseSoftmaxCrossEntropyWithLogitsGrad(op, grad_0, _):
 
 @ops.RegisterGradient("Conv2D")
 def _Conv2DGrad(op, grad):
+  """Gradient function for Conv2D."""
   dilations = op.get_attr("dilations")
   strides = op.get_attr("strides")
   padding = op.get_attr("padding")
+  explicit_paddings = op.get_attr("explicit_paddings")
   use_cudnn_on_gpu = op.get_attr("use_cudnn_on_gpu")
   data_format = op.get_attr("data_format")
   shape_0, shape_1 = array_ops.shape_n([op.inputs[0], op.inputs[1]])
+
+  # We call the gen_nn_ops backprop functions instead of nn_ops backprop
+  # functions for performance reasons in Eager mode. gen_nn_ops functions take a
+  # `explicit_paddings` parameter, but nn_ops functions do not. So if were were
+  # to use the nn_ops functions, we would have to convert `padding` and
+  # `explicit_paddings` into a single `padding` parameter, increasing overhead
+  # in Eager mode.
   return [
-      nn_ops.conv2d_backprop_input(
+      gen_nn_ops.conv2d_backprop_input(
           shape_0,
           op.inputs[1],
           grad,
           dilations=dilations,
           strides=strides,
           padding=padding,
+          explicit_paddings=explicit_paddings,
           use_cudnn_on_gpu=use_cudnn_on_gpu,
           data_format=data_format),
-      nn_ops.conv2d_backprop_filter(
+      gen_nn_ops.conv2d_backprop_filter(
           op.inputs[0],
           shape_1,
           grad,
           dilations=dilations,
           strides=strides,
           padding=padding,
+          explicit_paddings=explicit_paddings,
           use_cudnn_on_gpu=use_cudnn_on_gpu,
           data_format=data_format)
   ]
diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index 6f2d2c15bd..f71fcef130 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -171,7 +171,7 @@ class _NonAtrousConvolution(object):
         raise ValueError("data_format must be \"NHWC\" or \"NCHW\".")
       self.strides = strides
       self.data_format = data_format
-      self.conv_op = gen_nn_ops.conv2d
+      self.conv_op = conv2d
     elif conv_dims == 3:
       if data_format is None or data_format == "NDHWC":
         strides = [1] + list(strides) + [1]
@@ -1373,6 +1373,44 @@ def atrous_conv2d(value, filters, rate, padding, name=None):
       name=name)
 
 
+def _convert_padding(padding):
+  """Converts Python padding to C++ padding for ops which take EXPLICIT padding.
+
+  Args:
+    padding: the `padding` argument for a Python op which supports EXPLICIT
+      padding.
+
+  Returns:
+    (padding, explicit_paddings) pair, which should be passed as attributes to a
+    C++ op.
+
+  Raises:
+    ValueError: If padding is invalid.
+  """
+  explicit_paddings = []
+  if padding == "EXPLICIT":
+    # Give a better error message if EXPLICIT is passed.
+    raise ValueError('"EXPLICIT" is not a valid value for the padding '
+                     "parameter. To use explicit padding, the padding "
+                     "parameter must be a list.")
+  if isinstance(padding, (list, tuple)):
+    for i, dim_paddings in enumerate(padding):
+      if not isinstance(dim_paddings, (list, tuple)):
+        raise ValueError("When padding is a list, each element of padding must "
+                         "be a list/tuple of size 2. Element with index %d of "
+                         "padding is not a list/tuple" % i)
+      if len(dim_paddings) != 2:
+        raise ValueError("When padding is a list, each element of padding must "
+                         "be a list/tuple of size 2. Element with index %d of "
+                         "padding has size %d" % (i, len(dim_paddings)))
+      explicit_paddings.extend(dim_paddings)
+    if len(padding) != 4:
+      raise ValueError("When padding is a list, it must be of size 4. Got "
+                       "padding of size: %d" % len(padding))
+    padding = "EXPLICIT"
+  return padding, explicit_paddings
+
+
 @tf_export("nn.conv2d", v1=[])
 def conv2d_v2(input,  # pylint: disable=redefined-builtin
               filters,
@@ -1418,8 +1456,13 @@ def conv2d_v2(input,  # pylint: disable=redefined-builtin
       1-D tensor of length 4.  The stride of the sliding window for each
       dimension of `input`. The dimension order is determined by the value of
       `data_format`, see below for details.
-    padding: A `string` from: `"SAME", "VALID"`.
-      The type of padding algorithm to use.
+    padding: Either the `string `"SAME"` or `"VALID"` indicating the type of
+      padding algorithm to use, or a list indicating the explicit paddings at
+      the start and end of each dimension. When explicit padding is used and
+      data_format is `"NHWC"`, this should be in the form `[[0, 0], [pad_top,
+      pad_bottom], [pad_left, pad_right], [0, 0]]`. When explicit padding used
+      and data_format is `"NCHW"`, this should be in the form `[[0, 0], [0, 0],
+      [pad_top, pad_bottom], [pad_left, pad_right]]`.
     data_format: An optional `string` from: `"NHWC", "NCHW"`.
       Defaults to `"NHWC"`.
       Specify the data format of the input and output data. With the
@@ -1441,15 +1484,98 @@ def conv2d_v2(input,  # pylint: disable=redefined-builtin
   # pylint: enable=line-too-long
   if dilations is None:
     dilations = [1, 1, 1, 1]
+  return conv2d(input,  # pylint: disable=redefined-builtin
+                filters,
+                strides,
+                padding,
+                use_cudnn_on_gpu=True,
+                data_format=data_format,
+                dilations=dilations,
+                name=name)
+
+
+@tf_export(v1=["nn.conv2d"])
+def conv2d(  # pylint: disable=redefined-builtin,dangerous-default-value
+    input,
+    filter,
+    strides,
+    padding,
+    use_cudnn_on_gpu=True,
+    data_format="NHWC",
+    dilations=[1, 1, 1, 1],
+    name=None):
+  r"""Computes a 2-D convolution given 4-D `input` and `filter` tensors.
+
+  Given an input tensor of shape `[batch, in_height, in_width, in_channels]`
+  and a filter / kernel tensor of shape
+  `[filter_height, filter_width, in_channels, out_channels]`, this op
+  performs the following:
+
+  1. Flattens the filter to a 2-D matrix with shape
+     `[filter_height * filter_width * in_channels, output_channels]`.
+  2. Extracts image patches from the input tensor to form a *virtual*
+     tensor of shape `[batch, out_height, out_width,
+     filter_height * filter_width * in_channels]`.
+  3. For each patch, right-multiplies the filter matrix and the image patch
+     vector.
+
+  In detail, with the default NHWC format,
+
+      output[b, i, j, k] =
+          sum_{di, dj, q} input[b, strides[1] * i + di, strides[2] * j + dj, q]
+                          * filter[di, dj, q, k]
+
+  Must have `strides[0] = strides[3] = 1`.  For the most common case of the same
+  horizontal and vertices strides, `strides = [1, stride, stride, 1]`.
+
+  Args:
+    input: A `Tensor`. Must be one of the following types:
+      `half`, `bfloat16`, `float32`, `float64`.
+      A 4-D tensor. The dimension order is interpreted according to the value
+      of `data_format`, see below for details.
+    filter: A `Tensor`. Must have the same type as `input`.
+      A 4-D tensor of shape
+      `[filter_height, filter_width, in_channels, out_channels]`
+    strides: A list of `ints`.
+      1-D tensor of length 4.  The stride of the sliding window for each
+      dimension of `input`. The dimension order is determined by the value of
+      `data_format`, see below for details.
+    padding: Either the `string `"SAME"` or `"VALID"` indicating the type of
+      padding algorithm to use, or a list indicating the explicit paddings at
+      the start and end of each dimension. When explicit padding is used and
+      data_format is `"NHWC"`, this should be in the form `[[0, 0], [pad_top,
+      pad_bottom], [pad_left, pad_right], [0, 0]]`. When explicit padding used
+      and data_format is `"NCHW"`, this should be in the form `[[0, 0], [0, 0],
+      [pad_top, pad_bottom], [pad_left, pad_right]]`.
+    use_cudnn_on_gpu: An optional `bool`. Defaults to `True`.
+    data_format: An optional `string` from: `"NHWC", "NCHW"`.
+      Defaults to `"NHWC"`.
+      Specify the data format of the input and output data. With the
+      default format "NHWC", the data is stored in the order of:
+          [batch, height, width, channels].
+      Alternatively, the format could be "NCHW", the data storage order of:
+          [batch, channels, height, width].
+    dilations: An optional list of `ints`. Defaults to `[1, 1, 1, 1]`.
+      1-D tensor of length 4.  The dilation factor for each dimension of
+      `input`. If set to k > 1, there will be k-1 skipped cells between each
+      filter element on that dimension. The dimension order is determined by the
+      value of `data_format`, see above for details. Dilations in the batch and
+      depth dimensions must be 1.
+    name: A name for the operation (optional).
+
+  Returns:
+    A `Tensor`. Has the same type as `input`.
+  """
+  padding, explicit_paddings = _convert_padding(padding)
   return gen_nn_ops.conv2d(input,  # pylint: disable=redefined-builtin
-                           filters,
+                           filter,
                            strides,
                            padding,
-                           use_cudnn_on_gpu=True,
+                           use_cudnn_on_gpu=use_cudnn_on_gpu,
+                           explicit_paddings=explicit_paddings,
                            data_format=data_format,
                            dilations=dilations,
                            name=name)
-tf_export(v1=["nn.conv2d"])(gen_nn_ops.conv2d)
 
 
 @tf_export("nn.conv2d_backprop_filter", v1=[])
@@ -1478,8 +1604,13 @@ def conv2d_backprop_filter_v2(input,  # pylint: disable=redefined-builtin
       The stride of the sliding window for each dimension of the input
       of the convolution. Must be in the same order as the dimension specified
       with format.
-    padding: A `string` from: `"SAME", "VALID"`.
-      The type of padding algorithm to use.
+    padding: Either the `string `"SAME"` or `"VALID"` indicating the type of
+      padding algorithm to use, or a list indicating the explicit paddings at
+      the start and end of each dimension. When explicit padding is used and
+      data_format is `"NHWC"`, this should be in the form `[[0, 0], [pad_top,
+      pad_bottom], [pad_left, pad_right], [0, 0]]`. When explicit padding used
+      and data_format is `"NCHW"`, this should be in the form `[[0, 0], [0, 0],
+      [pad_top, pad_bottom], [pad_left, pad_right]]`.
     data_format: An optional `string` from: `"NHWC", "NCHW"`.
       Defaults to `"NHWC"`.
       Specify the data format of the input and output data. With the
@@ -1500,17 +1631,75 @@ def conv2d_backprop_filter_v2(input,  # pylint: disable=redefined-builtin
   """
   if dilations is None:
     dilations = [1, 1, 1, 1]
-  return gen_nn_ops.conv2d_backprop_filter(input,  # pylint: disable=redefined-builtin
-                                           filter_sizes,
-                                           out_backprop,
-                                           strides,
-                                           padding,
-                                           use_cudnn_on_gpu=True,
-                                           data_format=data_format,
-                                           dilations=dilations,
-                                           name=name)
-tf_export(v1=["nn.conv2d_backprop_filter"])(
-    gen_nn_ops.conv2d_backprop_filter)
+  return conv2d_backprop_filter(input,  # pylint: disable=redefined-builtin
+                                filter_sizes,
+                                out_backprop,
+                                strides,
+                                padding,
+                                use_cudnn_on_gpu=True,
+                                data_format=data_format,
+                                dilations=dilations,
+                                name=name)
+
+
+@tf_export(v1=["nn.conv2d_backprop_filter"])
+def conv2d_backprop_filter(  # pylint: disable=redefined-builtin,dangerous-default-value
+    input,
+    filter_sizes,
+    out_backprop,
+    strides,
+    padding,
+    use_cudnn_on_gpu=True,
+    data_format="NHWC",
+    dilations=[1, 1, 1, 1],
+    name=None):
+  r"""Computes the gradients of convolution with respect to the filter.
+
+  Args:
+    input: A `Tensor`. Must be one of the following types:
+      `half`, `bfloat16`, `float32`, `float64`.
+      4-D with shape `[batch, in_height, in_width, in_channels]`.
+    filter_sizes: A `Tensor` of type `int32`.
+      An integer vector representing the tensor shape of `filter`,
+      where `filter` is a 4-D
+      `[filter_height, filter_width, in_channels, out_channels]` tensor.
+    out_backprop: A `Tensor`. Must have the same type as `input`.
+      4-D with shape `[batch, out_height, out_width, out_channels]`.
+      Gradients w.r.t. the output of the convolution.
+    strides: A list of `ints`.
+      The stride of the sliding window for each dimension of the input
+      of the convolution. Must be in the same order as the dimension specified
+      with format.
+    padding: Either the `string `"SAME"` or `"VALID"` indicating the type of
+      padding algorithm to use, or a list indicating the explicit paddings at
+      the start and end of each dimension. When explicit padding is used and
+      data_format is `"NHWC"`, this should be in the form `[[0, 0], [pad_top,
+      pad_bottom], [pad_left, pad_right], [0, 0]]`. When explicit padding used
+      and data_format is `"NCHW"`, this should be in the form `[[0, 0], [0, 0],
+      [pad_top, pad_bottom], [pad_left, pad_right]]`.
+    use_cudnn_on_gpu: An optional `bool`. Defaults to `True`.
+    data_format: An optional `string` from: `"NHWC", "NCHW"`.
+      Defaults to `"NHWC"`.
+      Specify the data format of the input and output data. With the
+      default format "NHWC", the data is stored in the order of:
+          [batch, in_height, in_width, in_channels].
+      Alternatively, the format could be "NCHW", the data storage order of:
+          [batch, in_channels, in_height, in_width].
+    dilations: An optional list of `ints`. Defaults to `[1, 1, 1, 1]`.
+      1-D tensor of length 4.  The dilation factor for each dimension of
+      `input`. If set to k > 1, there will be k-1 skipped cells between each
+      filter element on that dimension. The dimension order is determined by
+      the value of `data_format`, see above for details. Dilations in the batch
+      and depth dimensions must be 1.
+    name: A name for the operation (optional).
+
+  Returns:
+    A `Tensor`. Has the same type as `input`.
+  """
+  padding, explicit_paddings = _convert_padding(padding)
+  return gen_nn_ops.conv2d_backprop_filter(
+      input, filter_sizes, out_backprop, strides, padding, use_cudnn_on_gpu,
+      explicit_paddings, data_format, dilations, name)
 
 
 @tf_export("nn.conv2d_backprop_input", v1=[])
@@ -1539,8 +1728,13 @@ def conv2d_backprop_input_v2(input_sizes,
       The stride of the sliding window for each dimension of the input
       of the convolution. Must be in the same order as the dimension specified
       with format.
-    padding: A `string` from: `"SAME", "VALID"`.
-      The type of padding algorithm to use.
+    padding: Either the `string `"SAME"` or `"VALID"` indicating the type of
+      padding algorithm to use, or a list indicating the explicit paddings at
+      the start and end of each dimension. When explicit padding is used and
+      data_format is `"NHWC"`, this should be in the form `[[0, 0], [pad_top,
+      pad_bottom], [pad_left, pad_right], [0, 0]]`. When explicit padding used
+      and data_format is `"NCHW"`, this should be in the form `[[0, 0], [0, 0],
+      [pad_top, pad_bottom], [pad_left, pad_right]]`.
     data_format: An optional `string` from: `"NHWC", "NCHW"`.
       Defaults to `"NHWC"`.
       Specify the data format of the input and output data. With the
@@ -1561,17 +1755,75 @@ def conv2d_backprop_input_v2(input_sizes,
   """
   if dilations is None:
     dilations = [1, 1, 1, 1]
-  return gen_nn_ops.conv2d_backprop_input(input_sizes,
-                                          filters,
-                                          out_backprop,
-                                          strides,
-                                          padding,
-                                          use_cudnn_on_gpu=True,
-                                          data_format=data_format,
-                                          dilations=dilations,
-                                          name=name)
-tf_export(v1=["nn.conv2d_backprop_input"])(
-    gen_nn_ops.conv2d_backprop_input)
+  return conv2d_backprop_input(input_sizes,
+                               filters,
+                               out_backprop,
+                               strides,
+                               padding,
+                               use_cudnn_on_gpu=True,
+                               data_format=data_format,
+                               dilations=dilations,
+                               name=name)
+
+
+@tf_export(v1=["nn.conv2d_backprop_input"])
+def conv2d_backprop_input(  # pylint: disable=redefined-builtin,dangerous-default-value
+    input_sizes,
+    filter,
+    out_backprop,
+    strides,
+    padding,
+    use_cudnn_on_gpu=True,
+    data_format="NHWC",
+    dilations=[1, 1, 1, 1],
+    name=None):
+  r"""Computes the gradients of convolution with respect to the input.
+
+  Args:
+    input_sizes: A `Tensor` of type `int32`.
+      An integer vector representing the shape of `input`,
+      where `input` is a 4-D `[batch, height, width, channels]` tensor.
+    filter: A `Tensor`. Must be one of the following types:
+      `half`, `bfloat16`, `float32`, `float64`.
+      4-D with shape
+      `[filter_height, filter_width, in_channels, out_channels]`.
+    out_backprop: A `Tensor`. Must have the same type as `filter`.
+      4-D with shape `[batch, out_height, out_width, out_channels]`.
+      Gradients w.r.t. the output of the convolution.
+    strides: A list of `ints`.
+      The stride of the sliding window for each dimension of the input
+      of the convolution. Must be in the same order as the dimension specified
+      with format.
+    padding: Either the `string `"SAME"` or `"VALID"` indicating the type of
+      padding algorithm to use, or a list indicating the explicit paddings at
+      the start and end of each dimension. When explicit padding is used and
+      data_format is `"NHWC"`, this should be in the form `[[0, 0], [pad_top,
+      pad_bottom], [pad_left, pad_right], [0, 0]]`. When explicit padding used
+      and data_format is `"NCHW"`, this should be in the form `[[0, 0], [0, 0],
+      [pad_top, pad_bottom], [pad_left, pad_right]]`.
+    use_cudnn_on_gpu: An optional `bool`. Defaults to `True`.
+    data_format: An optional `string` from: `"NHWC", "NCHW"`.
+      Defaults to `"NHWC"`.
+      Specify the data format of the input and output data. With the
+      default format "NHWC", the data is stored in the order of:
+          [batch, in_height, in_width, in_channels].
+      Alternatively, the format could be "NCHW", the data storage order of:
+          [batch, in_channels, in_height, in_width].
+    dilations: An optional list of `ints`. Defaults to `[1, 1, 1, 1]`.
+      1-D tensor of length 4.  The dilation factor for each dimension of
+      `input`. If set to k > 1, there will be k-1 skipped cells between each
+      filter element on that dimension. The dimension order is determined by
+      the value of `data_format`, see above for details. Dilations in the batch
+      and depth dimensions must be 1.
+    name: A name for the operation (optional).
+
+  Returns:
+    A `Tensor`. Has the same type as `filter`.
+  """
+  padding, explicit_paddings = _convert_padding(padding)
+  return gen_nn_ops.conv2d_backprop_input(
+      input_sizes, filter, out_backprop, strides, padding, use_cudnn_on_gpu,
+      explicit_paddings, data_format, dilations, name)
 
 
 @tf_export(v1=["nn.conv2d_transpose"])
-- 
GitLab


From e3b8fedc923d03fd5f388daf444b42b22dd909b8 Mon Sep 17 00:00:00 2001
From: Cong Liu <congliu@google.com>
Date: Tue, 8 Jan 2019 18:56:15 -0800
Subject: [PATCH 0433/1095] [XLA] Propagate compiler specific ShapeSizeFunction
 into TargetVerifierMetadata.

PiperOrigin-RevId: 228440043
---
 .../compiler/xla/service/hlo_verifier.h       | 33 ++++++++++++-------
 .../compiler/xla/tests/hlo_test_base.cc       |  6 ++--
 tensorflow/compiler/xla/tests/hlo_test_base.h |  8 +++--
 3 files changed, 31 insertions(+), 16 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_verifier.h b/tensorflow/compiler/xla/service/hlo_verifier.h
index a1a6aba972..479905b317 100644
--- a/tensorflow/compiler/xla/service/hlo_verifier.h
+++ b/tensorflow/compiler/xla/service/hlo_verifier.h
@@ -168,8 +168,13 @@ class ShapeVerifier : public DfsHloVisitor {
 // An interface used to encapsulate target-specific verification quirks.
 class TargetVerifierMetadata {
  public:
+  TargetVerifierMetadata(std::function<int64(const Shape&)> shape_size_function)
+      : shape_size_function_(shape_size_function) {}
+
   // Returns a target-specific shape size.
-  virtual int64 ShapeSize(const Shape& shape) const = 0;
+  int64 ShapeSize(const Shape& shape) const {
+    return shape_size_function_(shape);
+  }
 
   virtual std::unique_ptr<ShapeVerifier> GetVerifier() const = 0;
 
@@ -178,20 +183,23 @@ class TargetVerifierMetadata {
 
   TargetVerifierMetadata(const TargetVerifierMetadata&) = delete;
   TargetVerifierMetadata& operator=(const TargetVerifierMetadata&) = delete;
+
+ private:
+  // Returns a target-specific shape size.
+  std::function<int64(const Shape&)> shape_size_function_;
 };
 
 // The default implementation of TargetVerifierMetadata, used unless the target
 // needs to override it.
 class DefaultVerifierMetadata : public TargetVerifierMetadata {
  public:
-  DefaultVerifierMetadata(bool layout_sensitive, bool allow_mixed_precision)
-      : layout_sensitive_(layout_sensitive),
+  DefaultVerifierMetadata(
+      bool layout_sensitive, bool allow_mixed_precision,
+      std::function<int64(const Shape&)> shape_size_function)
+      : TargetVerifierMetadata(shape_size_function),
+        layout_sensitive_(layout_sensitive),
         allow_mixed_precision_(allow_mixed_precision) {}
 
-  int64 ShapeSize(const Shape& shape) const override {
-    return ShapeUtil::ByteSizeOf(shape);
-  }
-
   // Creates a ShapeVerifier that checks that shapes match inferred
   // expectations. This creates a new verifier every time because ShapeVerifier,
   // being a DfsHloVisitor, is stateful. We want a clean object for each run of
@@ -210,11 +218,14 @@ class DefaultVerifierMetadata : public TargetVerifierMetadata {
 // the module.
 class HloVerifier : public HloModulePass {
  public:
-  explicit HloVerifier(bool layout_sensitive, bool allow_mixed_precision,
-                       std::function<bool(const HloInstruction*)>
-                           instruction_can_change_layout_func = {})
+  explicit HloVerifier(
+      bool layout_sensitive, bool allow_mixed_precision,
+      std::function<bool(const HloInstruction*)>
+          instruction_can_change_layout_func = {},
+      std::function<int64(const Shape&)> shape_size_func =
+          [](const Shape& shape) { return ShapeUtil::ByteSizeOf(shape); })
       : target_metadata_(absl::make_unique<DefaultVerifierMetadata>(
-            layout_sensitive, allow_mixed_precision)),
+            layout_sensitive, allow_mixed_precision, shape_size_func)),
         instruction_can_change_layout_func_(
             std::move(instruction_can_change_layout_func)) {
     CHECK(instruction_can_change_layout_func_ == nullptr || layout_sensitive);
diff --git a/tensorflow/compiler/xla/tests/hlo_test_base.cc b/tensorflow/compiler/xla/tests/hlo_test_base.cc
index d57846e19b..66f72ba8d2 100644
--- a/tensorflow/compiler/xla/tests/hlo_test_base.cc
+++ b/tensorflow/compiler/xla/tests/hlo_test_base.cc
@@ -139,7 +139,8 @@ std::unique_ptr<VerifiedHloModule> HloTestBase::CreateNewVerifiedModule(
     const string& name) {
   return absl::make_unique<VerifiedHloModule>(
       name, GetModuleConfigForTest(), verifier_layout_sensitive_,
-      allow_mixed_precision_in_hlo_verifier_);
+      allow_mixed_precision_in_hlo_verifier_,
+      backend().compiler()->ShapeSizeBytesFunction());
 }
 
 StatusOr<std::unique_ptr<VerifiedHloModule>>
@@ -147,7 +148,8 @@ HloTestBase::ParseAndReturnVerifiedModule(absl::string_view hlo_text,
                                           const HloModuleConfig& config) {
   auto module = absl::make_unique<VerifiedHloModule>(
       TestName(), config, verifier_layout_sensitive_,
-      allow_mixed_precision_in_hlo_verifier_);
+      allow_mixed_precision_in_hlo_verifier_,
+      backend().compiler()->ShapeSizeBytesFunction());
   TF_RETURN_IF_ERROR(ParseHloString(hlo_text, module.get()));
   TF_RETURN_IF_ERROR(module->Verify());
   return std::move(module);
diff --git a/tensorflow/compiler/xla/tests/hlo_test_base.h b/tensorflow/compiler/xla/tests/hlo_test_base.h
index 1d1e7f4372..69a4f96288 100644
--- a/tensorflow/compiler/xla/tests/hlo_test_base.h
+++ b/tensorflow/compiler/xla/tests/hlo_test_base.h
@@ -46,10 +46,12 @@ class VerifiedHloModule : public HloModule {
  public:
   VerifiedHloModule(const string& name, const HloModuleConfig& config,
                     bool verifier_layout_sensitive,
-                    bool allow_mixed_precision_in_hlo_verifier)
+                    bool allow_mixed_precision_in_hlo_verifier,
+                    std::function<int64(const Shape&)> shape_size_function)
       : HloModule(name, config),
-        verifier_(verifier_layout_sensitive,
-                  allow_mixed_precision_in_hlo_verifier) {}
+        verifier_(
+            verifier_layout_sensitive, allow_mixed_precision_in_hlo_verifier,
+            /*instruction_can_change_layout_func=*/{}, shape_size_function) {}
 
   ~VerifiedHloModule() override { VerifyOrAddFailure("in destructor"); }
 
-- 
GitLab


From 6522d752cf3e5d26a507d9c4c517ee7633ae4c57 Mon Sep 17 00:00:00 2001
From: Tim Shen <timshen@google.com>
Date: Tue, 8 Jan 2019 19:41:27 -0800
Subject: [PATCH 0434/1095] Removes the hack that mutates an HloInstruction in
 order to auto-tune the convolution for a specific algorithm.

Redemption.

PiperOrigin-RevId: 228443963
---
 .../service/gpu/cudnn_conv_algorithm_picker.cc   | 12 +++++++-----
 .../service/gpu/cudnn_conv_algorithm_picker.h    |  3 ++-
 .../xla/service/gpu/cudnn_conv_runner.cc         | 16 ++++++++++------
 .../compiler/xla/service/gpu/cudnn_conv_runner.h | 12 ++++++++++--
 4 files changed, 29 insertions(+), 14 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/cudnn_conv_algorithm_picker.cc b/tensorflow/compiler/xla/service/gpu/cudnn_conv_algorithm_picker.cc
index 6d6780fa1c..057d0d2364 100644
--- a/tensorflow/compiler/xla/service/gpu/cudnn_conv_algorithm_picker.cc
+++ b/tensorflow/compiler/xla/service/gpu/cudnn_conv_algorithm_picker.cc
@@ -146,7 +146,8 @@ tensorflow::mutex_lock LockGpu(const se::StreamExecutor* stream_exec) {
 // trouble, but we may want to revisit this if we ever find a model where
 // caching would speed up compilation a lot.
 StatusOr<CudnnConvAlgorithmPicker::AutotuneResult>
-CudnnConvAlgorithmPicker::PickBestAlgorithm(HloCustomCallInstruction* instr) {
+CudnnConvAlgorithmPicker::PickBestAlgorithm(
+    const HloCustomCallInstruction* instr) {
   // TODO(timshen): for now only check fp16. It can be expanded to other types,
   // with some work on the HLO routines.
   const bool cross_check_enabled =
@@ -249,12 +250,13 @@ CudnnConvAlgorithmPicker::PickBestAlgorithm(HloCustomCallInstruction* instr) {
     VLOG(3) << "Trying algorithm " << AlgorithmToString(alg) << " for "
             << instr->ToString();
 
-    backend_config.set_algorithm(alg.algo_id());
-    backend_config.set_tensor_ops_enabled(alg.tensor_ops_enabled());
-    TF_RETURN_IF_ERROR(instr->set_backend_config(backend_config));
+    // Use assignment insetad of brace-list to make GCC 4.9 happy.
+    RunConvOptions options;
+    options.profile_result = &profile_result;
+    options.algo_override = alg;
     bool launch_ok =
         RunCudnnConv(instr, absl::MakeSpan(operand_buffers), result_buffer,
-                     &scratch_allocator, &stream, &profile_result)
+                     &scratch_allocator, &stream, options)
             .ok();
 
     if (launch_ok && profile_result.is_valid()) {
diff --git a/tensorflow/compiler/xla/service/gpu/cudnn_conv_algorithm_picker.h b/tensorflow/compiler/xla/service/gpu/cudnn_conv_algorithm_picker.h
index 642af787af..4991db0948 100644
--- a/tensorflow/compiler/xla/service/gpu/cudnn_conv_algorithm_picker.h
+++ b/tensorflow/compiler/xla/service/gpu/cudnn_conv_algorithm_picker.h
@@ -56,7 +56,8 @@ class CudnnConvAlgorithmPicker : public HloModulePass {
 
   StatusOr<bool> RunOnComputation(HloComputation* computation);
   StatusOr<bool> RunOnInstruction(HloInstruction* instr);
-  StatusOr<AutotuneResult> PickBestAlgorithm(HloCustomCallInstruction* instr);
+  StatusOr<AutotuneResult> PickBestAlgorithm(
+      const HloCustomCallInstruction* instr);
 
   se::StreamExecutor* stream_exec_;                   // never null
   DeviceMemoryAllocator* allocator_;                  // may be null
diff --git a/tensorflow/compiler/xla/service/gpu/cudnn_conv_runner.cc b/tensorflow/compiler/xla/service/gpu/cudnn_conv_runner.cc
index 3425e1b494..b628f27f4b 100644
--- a/tensorflow/compiler/xla/service/gpu/cudnn_conv_runner.cc
+++ b/tensorflow/compiler/xla/service/gpu/cudnn_conv_runner.cc
@@ -395,32 +395,36 @@ Status RunCudnnConv(const HloCustomCallInstruction* conv,
                     absl::Span<se::DeviceMemoryBase> operand_buffers,
                     se::DeviceMemoryBase result_buffer,
                     se::DeviceMemoryBase scratch_buf, se::Stream* stream,
-                    se::dnn::ProfileResult* profile_result) {
+                    RunConvOptions options) {
   ScratchBufAllocator scratch_allocator(scratch_buf);
   return RunCudnnConv(conv, operand_buffers, result_buffer, &scratch_allocator,
-                      stream, profile_result);
+                      stream, options);
 }
 
 Status RunCudnnConv(const HloCustomCallInstruction* conv,
                     absl::Span<se::DeviceMemoryBase> operand_buffers,
                     se::DeviceMemoryBase result_buffer,
                     se::ScratchAllocator* scratch_allocator, se::Stream* stream,
-                    se::dnn::ProfileResult* profile_result) {
+                    RunConvOptions options) {
   TF_ASSIGN_OR_RETURN(CudnnConvParams params,
                       GetCudnnConvParams(conv, operand_buffers, result_buffer));
 
+  if (options.algo_override) {
+    params.algorithm = AlgorithmConfig(*options.algo_override);
+  }
+
   PrimitiveType output_primitive_type =
       conv->shape().tuple_shapes(0).element_type();
   switch (output_primitive_type) {
     case F16:
       return RunCudnnConvImpl<Eigen::half>(params, scratch_allocator, stream,
-                                           profile_result);
+                                           options.profile_result);
     case F32:
       return RunCudnnConvImpl<float>(params, scratch_allocator, stream,
-                                     profile_result);
+                                     options.profile_result);
     case F64:
       return RunCudnnConvImpl<double>(params, scratch_allocator, stream,
-                                      profile_result);
+                                      options.profile_result);
     default:
       LOG(FATAL) << ShapeUtil::HumanString(*params.output_shape);
   }
diff --git a/tensorflow/compiler/xla/service/gpu/cudnn_conv_runner.h b/tensorflow/compiler/xla/service/gpu/cudnn_conv_runner.h
index edbc75a94a..7cc325bce3 100644
--- a/tensorflow/compiler/xla/service/gpu/cudnn_conv_runner.h
+++ b/tensorflow/compiler/xla/service/gpu/cudnn_conv_runner.h
@@ -28,6 +28,14 @@ limitations under the License.
 namespace xla {
 namespace gpu {
 
+struct RunConvOptions {
+  // Nullable output-parameter pointer for profiling results.
+  se::dnn::ProfileResult* profile_result = nullptr;
+
+  // Use this algorithm, instead the one from the instrcution.
+  absl::optional<se::dnn::AlgorithmDesc> algo_override;
+};
+
 // This file contains low-level routines for running cudnn convolutions.
 
 // Calls into cudnn to run the specified convolution.
@@ -46,13 +54,13 @@ Status RunCudnnConv(const HloCustomCallInstruction* conv,
                     absl::Span<se::DeviceMemoryBase> operand_buffers,
                     se::DeviceMemoryBase result_buffer,
                     se::DeviceMemoryBase scratch_buf, se::Stream* stream,
-                    se::dnn::ProfileResult* profile_result = nullptr);
+                    RunConvOptions = {});
 
 Status RunCudnnConv(const HloCustomCallInstruction* conv,
                     absl::Span<se::DeviceMemoryBase> operand_buffers,
                     se::DeviceMemoryBase result_buffer,
                     se::ScratchAllocator* scratch_allocator, se::Stream* stream,
-                    se::dnn::ProfileResult* profile_result = nullptr);
+                    RunConvOptions = {});
 
 }  // namespace gpu
 }  // namespace xla
-- 
GitLab


From 9a54ef2b7cd156ce9fb11d143cea61cc84aed3b3 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Tue, 8 Jan 2019 19:54:03 -0800
Subject: [PATCH 0435/1095] Automated rollback of commit
 f71e77471c140bac2f4dcb9a003139fb5c657302

PiperOrigin-RevId: 228444974
---
 tensorflow/compiler/xla/service/cpu/BUILD     |   2 +
 .../cpu/cpu_eigen_tensor_alignment_test.cc    |  38 ++++++
 .../xla/service/cpu/dot_op_emitter.cc         | 117 ++++++------------
 .../xla/service/cpu/dot_op_emitter_internal.h |  88 +++++++++++++
 4 files changed, 168 insertions(+), 77 deletions(-)
 create mode 100644 tensorflow/compiler/xla/service/cpu/dot_op_emitter_internal.h

diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD
index a197bdddc8..de62aa60aa 100644
--- a/tensorflow/compiler/xla/service/cpu/BUILD
+++ b/tensorflow/compiler/xla/service/cpu/BUILD
@@ -385,6 +385,7 @@ cc_library(
     srcs = ["dot_op_emitter.cc"],
     hdrs = [
         "dot_op_emitter.h",
+        "dot_op_emitter_internal.h",
     ],
     deps = [
         ":cpu_options",
@@ -1028,6 +1029,7 @@ tf_cc_test(
     size = "small",
     srcs = ["cpu_eigen_tensor_alignment_test.cc"],
     deps = [
+        ":dot_op_emitter",
         ":ir_emission_utils",
         ":target_machine_features_fake",
         "//tensorflow/compiler/xla:test",
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_eigen_tensor_alignment_test.cc b/tensorflow/compiler/xla/service/cpu/cpu_eigen_tensor_alignment_test.cc
index 485769a373..823bdf259c 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_eigen_tensor_alignment_test.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_eigen_tensor_alignment_test.cc
@@ -13,6 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#include "tensorflow/compiler/xla/service/cpu/dot_op_emitter_internal.h"
 #include "tensorflow/compiler/xla/service/cpu/ir_emission_utils.h"
 #include "tensorflow/compiler/xla/service/cpu/target_machine_features_fake.h"
 #include "tensorflow/compiler/xla/service/hlo_parser.h"
@@ -22,11 +23,48 @@ namespace xla {
 namespace cpu {
 namespace {
 
+using internal::DotImplementationStrategy;
+using internal::DotInfo;
+using internal::GetDotImplementationStrategy;
+
 // Test that we don't call into Eigen with tensors too small to be aligned
 // reliably.
 
 class CpuEigenTensorAlignmentTest : public ::testing::Test {};
 
+TEST_F(CpuEigenTensorAlignmentTest, EigenDotAlignment) {
+  string hlo_string = R"(
+HloModule DotOperation
+
+ENTRY DotOperation {
+  arg0 = f32[5,256] parameter(0)
+  arg1 = f32[256,1024] parameter(1)
+  ROOT dot = f32[5,1024] dot(arg0, arg1), lhs_contracting_dims={1}, rhs_contracting_dims={0}
+}
+)";
+
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
+                          ParseHloString(hlo_string));
+
+  HloInstruction* dot = module->entry_computation()->root_instruction();
+
+  TargetMachineFeaturesWithFakeAlignmentLogic target_machine_with_no_alignment(
+      [](int64 size) { return 1; });
+
+  EXPECT_EQ(GetDotImplementationStrategy(HloModuleConfig{}, DotInfo(*dot),
+                                         target_machine_with_no_alignment),
+            DotImplementationStrategy::kNaiveLlvmIr);
+
+  TargetMachineFeaturesWithFakeAlignmentLogic
+      target_machine_with_full_alignment([](int64 size) {
+        return TargetMachineFeatures::kEigenExpectedTensorAlignment;
+      });
+
+  EXPECT_NE(GetDotImplementationStrategy(HloModuleConfig{}, DotInfo(*dot),
+                                         target_machine_with_full_alignment),
+            DotImplementationStrategy::kNaiveLlvmIr);
+}
+
 TEST_F(CpuEigenTensorAlignmentTest, EigenConvAlignment) {
   string hlo_string = R"(
 HloModule ConvOperation
diff --git a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
index b018e0cd46..e8a84ebe6b 100644
--- a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
@@ -14,6 +14,7 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/compiler/xla/service/cpu/dot_op_emitter.h"
+#include "tensorflow/compiler/xla/service/cpu/dot_op_emitter_internal.h"
 
 #include <memory>
 #include <vector>
@@ -42,63 +43,17 @@ namespace xla {
 using llvm_ir::SetToFirstInsertPoint;
 
 namespace cpu {
+
+using internal::DotImplementationStrategy;
+using internal::DotInfo;
+using internal::GetDotImplementationStrategy;
+
 namespace {
 // Returns true if we should call into multi-threaded Eigen routines.
 bool ShouldUseMultiThreadedEigen(const HloModuleConfig& config) {
   return config.debug_options().xla_cpu_multi_thread_eigen();
 }
 
-// Represents a dot operation.  We use this in lieu of an `HloInstruction`
-// because we want to be able to create this for the "inner" dot operation in a
-// batch dot, for which there is no separate HLO instruction.
-struct DotInfo {
-  Shape lhs_shape;
-  Shape rhs_shape;
-  Shape result_shape;
-  DotDimensionNumbers dim_nums;
-
-  explicit DotInfo(const HloInstruction& instr) {
-    CHECK_EQ(instr.opcode(), HloOpcode::kDot);
-    lhs_shape = instr.operand(0)->shape();
-    rhs_shape = instr.operand(1)->shape();
-    result_shape = instr.shape();
-    dim_nums = instr.dot_dimension_numbers();
-  }
-};
-
-// Dictates how a dot operation is implemented.
-enum class DotImplementationStrategy {
-  // The dot operation is lowered into LLVM IR that implements a naive nested
-  // loop that computes the result one element at a time.  This is our
-  // "fallback"; we don't really want this to kick in for any non-trival dot
-  // operation.
-  kNaiveLlvmIr,
-
-  // The dot operation is lowered into LLVM IR that implements a tiled
-  // Matrix*Vector operation.  This strategy also allows fusing in a bias add
-  // into the dot.  The matrix can be row major or column major, both are
-  // supported.
-  kTiledLlvmIrGemv,
-
-  // The dot operation is lowered into LLVM IR that implemetns a tiled
-  // Matrix*Matrix operation.  No fusions are supported.  The two inputs
-  // and the output have to be row major.
-  kTiledLlvmIrGemm,
-
-  // The dot operation is lowered into a call into an Eigen routine.  No fusions
-  // are supported today.  The two inputs and the output have to be row major.
-  // However, we do allow transposing either the LHS or the RHS as part of the
-  // GEMM -- we expose this flexibility as flexibility in the contraction
-  // dimensions, but we can also see this as flexibility in the input layouts.
-  kEigen,
-};
-
-// Returns the implementation strategy for a dot with the configuration
-// `dot_info`.
-DotImplementationStrategy GetDotImplementationStrategy(
-    const HloModuleConfig& config, const DotInfo& dot_info,
-    const TargetMachineFeatures& target_machine_features);
-
 // Helper class for emitting LLVM IR to perform the dot operation.
 class DotOpEmitter {
  public:
@@ -235,8 +190,9 @@ void DotOpEmitter::EmitTiledLlvmIrGemm() {
   }
 
   int64 size_bytes = m * n * ShapeUtil::ByteSizeOfPrimitiveType(primitive_type);
-  b_->CreateMemSet(target, b_->getInt8(0), /*Size=*/size_bytes,
-                   /*Align=*/1);
+  b_->CreateMemSet(
+      target, b_->getInt8(0), size_bytes,
+      target_machine_features_.minimum_alignment_for_allocation(size_bytes));
 
   int64 max_target_vector_width =
       target_machine_features_.vector_register_num_elements(
@@ -740,34 +696,40 @@ absl::optional<int64> ProfitableToMakeDotOperandColumnMajor(
   return {};
 }
 
+namespace internal {
 namespace {
 // Return whether the given shape is rank 2.
 bool IsRank2(const Shape& shape) { return shape.rank() == 2; }
 
-bool IsSimpleLayout(const Layout& layout) {
-  return layout.tiles().empty() && layout.format() == DENSE;
-}
-
 // In a gemm operation where output = lhs * rhs, check whether the given shapes
 // are valid for the operation.
-bool AreGemmShapes(const Shape& lhs_shape, const Shape& rhs_shape,
-                   const Shape& output_shape,
-                   const TargetMachineFeatures& target_machine_features) {
-  CHECK(!lhs_shape.has_layout() || IsSimpleLayout(lhs_shape.layout()))
-      << lhs_shape.DebugString();
-  CHECK(!rhs_shape.has_layout() || IsSimpleLayout(rhs_shape.layout()))
-      << rhs_shape.DebugString();
-  CHECK(!output_shape.has_layout() || IsSimpleLayout(output_shape.layout()))
-      << output_shape.DebugString();
-
-  switch (output_shape.element_type()) {
-    case F64:
-    case F32:
-    case F16:
-      return IsRank2(lhs_shape) && IsRank2(rhs_shape) && IsRank2(output_shape);
-    default:
-      return false;
+bool AreAlignedGemmShapes(
+    const Shape& lhs_shape, const Shape& rhs_shape, const Shape& output_shape,
+    const TargetMachineFeatures& target_machine_features) {
+  // The inputs and the output must
+  // 1) be matrices with no padding, and
+  // 2) have an allowed element type.
+  PrimitiveType output_primitive_type = output_shape.element_type();
+  if (!(output_primitive_type == F64 || output_primitive_type == F32 ||
+        output_primitive_type == F16)) {
+    return false;
+  }
+
+  if (!(IsRank2(lhs_shape) && IsRank2(rhs_shape) && IsRank2(output_shape))) {
+    return false;
+  }
+
+  auto is_aligned = [&](const Shape& shape) {
+    return GetMinimumAlignmentForArray(shape, target_machine_features) >=
+           TargetMachineFeatures::kEigenExpectedTensorAlignment;
+  };
+
+  if (!is_aligned(lhs_shape) || !is_aligned(rhs_shape) ||
+      !is_aligned(output_shape)) {
+    return false;
   }
+
+  return true;
 }
 
 bool IsAlignedGemm(const DotInfo& dot_info,
@@ -777,8 +739,8 @@ bool IsAlignedGemm(const DotInfo& dot_info,
     return false;
   }
 
-  return AreGemmShapes(dot_info.lhs_shape, dot_info.rhs_shape,
-                       dot_info.result_shape, target_machine_features);
+  return AreAlignedGemmShapes(dot_info.lhs_shape, dot_info.rhs_shape,
+                              dot_info.result_shape, target_machine_features);
 }
 
 bool CanEmitTiledLlvmIrGemm(
@@ -819,6 +781,7 @@ bool CanEmitTiledLlvmIrGemm(
 
   return true;
 }
+}  // namespace
 
 DotImplementationStrategy GetDotImplementationStrategy(
     const HloModuleConfig& config, const DotInfo& dot_info,
@@ -843,7 +806,7 @@ DotImplementationStrategy GetDotImplementationStrategy(
 
   return DotImplementationStrategy::kNaiveLlvmIr;
 }
-}  // namespace
+}  // namespace internal
 
 bool DotImplementationCanHandleTranspose(
     const HloInstruction& dot_instr,
diff --git a/tensorflow/compiler/xla/service/cpu/dot_op_emitter_internal.h b/tensorflow/compiler/xla/service/cpu/dot_op_emitter_internal.h
new file mode 100644
index 0000000000..cc28918ed6
--- /dev/null
+++ b/tensorflow/compiler/xla/service/cpu/dot_op_emitter_internal.h
@@ -0,0 +1,88 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_CPU_DOT_OP_EMITTER_INTERNAL_H_
+#define TENSORFLOW_COMPILER_XLA_SERVICE_CPU_DOT_OP_EMITTER_INTERNAL_H_
+
+#include "tensorflow/compiler/xla/service/cpu/target_machine_features.h"
+#include "tensorflow/compiler/xla/service/hlo_instruction.h"
+#include "tensorflow/compiler/xla/service/hlo_module.h"
+
+// -----------------------------------------------------------------------------
+// INTERNAL HEADER.
+//
+// This file exposes internal implementation details from dot_op_emitter.cc for
+// unit tests.  Please do not depend on this!
+//
+// -----------------------------------------------------------------------------
+
+namespace xla {
+namespace cpu {
+namespace internal {
+
+// Represents a dot operation.  We use this in lieu of an `HloInstruction`
+// because we want to be able to create this for the "inner" dot operation in a
+// batch dot, for which there is no separate HLO instruction.
+struct DotInfo {
+  Shape lhs_shape;
+  Shape rhs_shape;
+  Shape result_shape;
+  DotDimensionNumbers dim_nums;
+
+  explicit DotInfo(const HloInstruction& instr) {
+    CHECK_EQ(instr.opcode(), HloOpcode::kDot);
+    lhs_shape = instr.operand(0)->shape();
+    rhs_shape = instr.operand(1)->shape();
+    result_shape = instr.shape();
+    dim_nums = instr.dot_dimension_numbers();
+  }
+};
+
+// Dictates how a dot operation is implemented.
+enum class DotImplementationStrategy {
+  // The dot operation is lowered into LLVM IR that implements a naive nested
+  // loop that computes the result one element at a time.  This is our
+  // "fallback"; we don't really want this to kick in for any non-trival dot
+  // operation.
+  kNaiveLlvmIr,
+
+  // The dot operation is lowered into LLVM IR that implements a tiled
+  // Matrix*Vector operation.  This strategy also allows fusing in a bias add
+  // into the dot.  The matrix can be row major or column major, both are
+  // supported.
+  kTiledLlvmIrGemv,
+
+  // The dot operation is lowered into LLVM IR that implemetns a tiled
+  // Matrix*Matrix operation.  No fusions are supported.  The two inputs
+  // and the output have to be row major.
+  kTiledLlvmIrGemm,
+
+  // The dot operation is lowered into a call into an Eigen routine.  No fusions
+  // are supported today.  The two inputs and the output have to be row major.
+  // However, we do allow transposing either the LHS or the RHS as part of the
+  // GEMM -- we expose this flexibility as flexibility in the contraction
+  // dimensions, but we can also see this as flexibility in the input layouts.
+  kEigen,
+};
+
+// Returns the implementation strategy for a dot with the configuration
+// `dot_info`.
+DotImplementationStrategy GetDotImplementationStrategy(
+    const HloModuleConfig& config, const DotInfo& dot_info,
+    const TargetMachineFeatures& target_machine_features);
+}  // namespace internal
+}  // namespace cpu
+}  // namespace xla
+
+#endif  // TENSORFLOW_COMPILER_XLA_SERVICE_CPU_DOT_OP_EMITTER_INTERNAL_H_
-- 
GitLab


From 8d4893d6efa971b35db9f4eb654adb1559b7186c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 8 Jan 2019 20:19:22 -0800
Subject: [PATCH 0436/1095] Update ops-related pbtxt files.

PiperOrigin-RevId: 228447178
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 254 ++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 |  27 ++
 2 files changed, 281 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 28d085b2d2..9698673dfe 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -13631,6 +13631,151 @@ op {
     }
   }
 }
+op {
+  name: "Conv2D"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "filter"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_BFLOAT16
+        type: DT_FLOAT
+        type: DT_DOUBLE
+      }
+    }
+  }
+  attr {
+    name: "strides"
+    type: "list(int)"
+  }
+  attr {
+    name: "use_cudnn_on_gpu"
+    type: "bool"
+    default_value {
+      b: true
+    }
+  }
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+        s: "EXPLICIT"
+      }
+    }
+  }
+  attr {
+    name: "explicit_paddings"
+    type: "list(int)"
+    default_value {
+      list {
+      }
+    }
+  }
+  attr {
+    name: "data_format"
+    type: "string"
+    default_value {
+      s: "NHWC"
+    }
+    allowed_values {
+      list {
+        s: "NHWC"
+        s: "NCHW"
+      }
+    }
+  }
+  attr {
+    name: "dilations"
+    type: "list(int)"
+    default_value {
+      list {
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+      }
+    }
+  }
+}
+op {
+  name: "Conv2DBackpropFilter"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "filter_sizes"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "out_backprop"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+      }
+    }
+  }
+  attr {
+    name: "strides"
+    type: "list(int)"
+  }
+  attr {
+    name: "use_cudnn_on_gpu"
+    type: "bool"
+    default_value {
+      b: true
+    }
+  }
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+      }
+    }
+  }
+  attr {
+    name: "data_format"
+    type: "string"
+    default_value {
+      s: "NHWC"
+    }
+    allowed_values {
+      list {
+        s: "NHWC"
+        s: "NCHW"
+      }
+    }
+  }
+}
 op {
   name: "Conv2DBackpropFilter"
   input_arg {
@@ -13655,6 +13800,7 @@ op {
     allowed_values {
       list {
         type: DT_HALF
+        type: DT_BFLOAT16
         type: DT_FLOAT
       }
     }
@@ -13693,6 +13839,18 @@ op {
       }
     }
   }
+  attr {
+    name: "dilations"
+    type: "list(int)"
+    default_value {
+      list {
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+      }
+    }
+  }
 }
 op {
   name: "Conv2DBackpropFilter"
@@ -13720,6 +13878,7 @@ op {
         type: DT_HALF
         type: DT_BFLOAT16
         type: DT_FLOAT
+        type: DT_DOUBLE
       }
     }
   }
@@ -13818,6 +13977,15 @@ op {
       list {
         s: "SAME"
         s: "VALID"
+        s: "EXPLICIT"
+      }
+    }
+  }
+  attr {
+    name: "explicit_paddings"
+    type: "list(int)"
+    default_value {
+      list {
       }
     }
   }
@@ -14063,6 +14231,92 @@ op {
     }
   }
 }
+op {
+  name: "Conv2DBackpropInput"
+  input_arg {
+    name: "input_sizes"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "filter"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "out_backprop"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_BFLOAT16
+        type: DT_FLOAT
+        type: DT_DOUBLE
+      }
+    }
+  }
+  attr {
+    name: "strides"
+    type: "list(int)"
+  }
+  attr {
+    name: "use_cudnn_on_gpu"
+    type: "bool"
+    default_value {
+      b: true
+    }
+  }
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+        s: "EXPLICIT"
+      }
+    }
+  }
+  attr {
+    name: "explicit_paddings"
+    type: "list(int)"
+    default_value {
+      list {
+      }
+    }
+  }
+  attr {
+    name: "data_format"
+    type: "string"
+    default_value {
+      s: "NHWC"
+    }
+    allowed_values {
+      list {
+        s: "NHWC"
+        s: "NCHW"
+      }
+    }
+  }
+  attr {
+    name: "dilations"
+    type: "list(int)"
+    default_value {
+      list {
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+      }
+    }
+  }
+}
 op {
   name: "Conv3D"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index b4767d35d2..4a1b3a6f2d 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -5876,6 +5876,15 @@ op {
       list {
         s: "SAME"
         s: "VALID"
+        s: "EXPLICIT"
+      }
+    }
+  }
+  attr {
+    name: "explicit_paddings"
+    type: "list(int)"
+    default_value {
+      list {
       }
     }
   }
@@ -5953,6 +5962,15 @@ op {
       list {
         s: "SAME"
         s: "VALID"
+        s: "EXPLICIT"
+      }
+    }
+  }
+  attr {
+    name: "explicit_paddings"
+    type: "list(int)"
+    default_value {
+      list {
       }
     }
   }
@@ -6030,6 +6048,15 @@ op {
       list {
         s: "SAME"
         s: "VALID"
+        s: "EXPLICIT"
+      }
+    }
+  }
+  attr {
+    name: "explicit_paddings"
+    type: "list(int)"
+    default_value {
+      list {
       }
     }
   }
-- 
GitLab


From 283bd7dc2914f36daf5d6604ffa25d939967bd7d Mon Sep 17 00:00:00 2001
From: Pariksheet Pinjari <pariksheet.pinjari@huawei.com>
Date: Wed, 9 Jan 2019 10:33:43 +0530
Subject: [PATCH 0437/1095] Update api_def_StaticRegexReplace.pbtxt

Documentation error fixed
---
 .../core/api_def/base_api/api_def_StaticRegexReplace.pbtxt      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/api_def/base_api/api_def_StaticRegexReplace.pbtxt b/tensorflow/core/api_def/base_api/api_def_StaticRegexReplace.pbtxt
index e382bcec81..8bb88f491a 100644
--- a/tensorflow/core/api_def/base_api/api_def_StaticRegexReplace.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_StaticRegexReplace.pbtxt
@@ -14,7 +14,7 @@ op {
   }
   attr {
     name: "rewrite"
-    description: "The rewrite to be applied to the matched expresion."
+    description: "The rewrite to be applied to the matched expression."
   }
   attr {
     name: "replace_global"
-- 
GitLab


From ae2aceda4f6630dbdaa6c9a4da6ba690f34f40de Mon Sep 17 00:00:00 2001
From: Ilango R <ilangokul@gmail.com>
Date: Wed, 9 Jan 2019 13:10:52 +0530
Subject: [PATCH 0438/1095] Fix astor url to pypi.python.org

---
 tensorflow/workspace.bzl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 5b0f6d4c84..a7f3665a31 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -285,7 +285,7 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
         system_build_file = clean_dep("//third_party/systemlibs:astor.BUILD"),
         urls = [
             "https://mirror.bazel.build/pypi.python.org/packages/99/80/f9482277c919d28bebd85813c0a70117214149a96b08981b72b63240b84c/astor-0.7.1.tar.gz",
-            "https://files.pythonhosted.org/packages/99/80/f9482277c919d28bebd85813c0a70117214149a96b08981b72b63240b84c/astor-0.7.1.tar.gz",
+            "https://pypi.python.org/packages/99/80/f9482277c919d28bebd85813c0a70117214149a96b08981b72b63240b84c/astor-0.7.1.tar.gz",
         ],
     )
 
-- 
GitLab


From 746a9750f49d98b975a4cd0ee8c0f353426ffbc0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 9 Jan 2019 01:02:25 -0800
Subject: [PATCH 0439/1095] compat: Update forward compatibility horizon to
 2019-01-09

PiperOrigin-RevId: 228471346
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 5008741d55..dfee972cab 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 8)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 9)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 1bc98108a3e835221318e019fef1642914dd5720 Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Wed, 9 Jan 2019 02:37:03 -0800
Subject: [PATCH 0440/1095] Allow adding extra parameters to a cloned
 computation.

There is currently no way to clone a computation and add additional parameters
to it, except if it is a fusion computation. In a future change this will be
necessary, so we add the possibility to add such parameters via
CloneWithReplacements().

PiperOrigin-RevId: 228483852
---
 .../compiler/xla/service/hlo_computation.cc   | 18 +++++++--
 .../compiler/xla/service/hlo_computation.h    |  4 ++
 .../xla/service/hlo_computation_test.cc       | 38 +++++++++++++++++++
 3 files changed, 56 insertions(+), 4 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_computation.cc b/tensorflow/compiler/xla/service/hlo_computation.cc
index 9e15722633..f9b64d12ae 100644
--- a/tensorflow/compiler/xla/service/hlo_computation.cc
+++ b/tensorflow/compiler/xla/service/hlo_computation.cc
@@ -846,7 +846,7 @@ std::unique_ptr<HloComputation> HloComputation::Clone(
   return CloneWithReplacements(
       /*replacements=*/std::unordered_map<const HloInstruction*,
                                           std::unique_ptr<HloInstruction>>(),
-      context, suffix);
+      /*extra_parameters=*/{}, context, suffix);
 }
 
 std::unique_ptr<HloComputation> HloComputation::CloneWithReplacementPairs(
@@ -855,7 +855,8 @@ std::unique_ptr<HloComputation> HloComputation::CloneWithReplacementPairs(
   std::unordered_map<const HloInstruction*, std::unique_ptr<HloInstruction>>
       replacements;
   replacements.emplace(std::move(r1));
-  return CloneWithReplacements(std::move(replacements), context, suffix);
+  return CloneWithReplacements(std::move(replacements), /*extra_parameters=*/{},
+                               context, suffix);
 }
 
 std::unique_ptr<HloComputation> HloComputation::CloneWithReplacementPairs(
@@ -866,7 +867,8 @@ std::unique_ptr<HloComputation> HloComputation::CloneWithReplacementPairs(
       replacements;
   replacements.emplace(std::move(r1));
   replacements.emplace(std::move(r2));
-  return CloneWithReplacements(std::move(replacements), context, suffix);
+  return CloneWithReplacements(std::move(replacements), /*extra_parameters=*/{},
+                               context, suffix);
 }
 
 std::unique_ptr<HloComputation> HloComputation::CloneWithReplacementPairs(
@@ -879,12 +881,14 @@ std::unique_ptr<HloComputation> HloComputation::CloneWithReplacementPairs(
   replacements.emplace(std::move(r1));
   replacements.emplace(std::move(r2));
   replacements.emplace(std::move(r3));
-  return CloneWithReplacements(std::move(replacements), context, suffix);
+  return CloneWithReplacements(std::move(replacements), /*extra_parameters=*/{},
+                               context, suffix);
 }
 
 std::unique_ptr<HloComputation> HloComputation::CloneWithReplacements(
     std::unordered_map<const HloInstruction*, std::unique_ptr<HloInstruction>>
         replacements,
+    absl::Span<const HloInstruction* const> extra_parameters,
     HloCloneContext* context, const string& suffix) {
   std::unique_ptr<HloCloneContext> context_ptr;
   if (context == nullptr) {
@@ -950,6 +954,12 @@ std::unique_ptr<HloComputation> HloComputation::CloneWithReplacements(
   }
 
   std::vector<std::unique_ptr<HloInstruction>> instructions;
+  // First add the extra parameters to 'instructions'.
+  for (const auto& instr : extra_parameters) {
+    CHECK_EQ(instr->opcode(), HloOpcode::kParameter)
+        << "Only parameter instructions are allowed in 'extra_parameters'";
+    instructions.emplace_back(instr->Clone());
+  }
   for (auto instr : postorder) {
     std::vector<HloInstruction*> new_operands;
     for (auto operand : instr->operands()) {
diff --git a/tensorflow/compiler/xla/service/hlo_computation.h b/tensorflow/compiler/xla/service/hlo_computation.h
index a0ccbc583f..e6a1eb89cf 100644
--- a/tensorflow/compiler/xla/service/hlo_computation.h
+++ b/tensorflow/compiler/xla/service/hlo_computation.h
@@ -323,11 +323,15 @@ class HloComputation {
   // that's not already in the computation, it's cloned and added to the new
   // computation.
   //
+  // 'extra_parameters' allows to specify additional parameters that should be
+  // added to the computation.
+  //
   // All relevant instructions are cloned, *including* unique_ptr in the
   // `replacements` map.
   std::unique_ptr<HloComputation> CloneWithReplacements(
       std::unordered_map<const HloInstruction*, std::unique_ptr<HloInstruction>>
           replacements,
+      absl::Span<const HloInstruction* const> extra_parameters = {},
       HloCloneContext* context = nullptr, const string& suffix = "clone");
 
   // Convenience overloads for CloneWithReplacements.  You want to do
diff --git a/tensorflow/compiler/xla/service/hlo_computation_test.cc b/tensorflow/compiler/xla/service/hlo_computation_test.cc
index 216d56b868..251c7bbec4 100644
--- a/tensorflow/compiler/xla/service/hlo_computation_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_computation_test.cc
@@ -15,7 +15,10 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/service/hlo_computation.h"
 
+#include <memory>
 #include <set>
+#include <unordered_map>
+#include <vector>
 
 #include "absl/container/flat_hash_set.h"
 #include "tensorflow/compiler/xla/literal.h"
@@ -492,6 +495,41 @@ TEST_F(HloComputationTest, CloneWithControlDependency) {
   EXPECT_THAT(successors, ::testing::ElementsAre(cloned_add));
 }
 
+TEST_F(HloComputationTest, CloneWithReplacements) {
+  auto builder = HloComputation::Builder(TestName());
+  Shape r0s64 = ShapeUtil::MakeShape(S64, {});
+  Shape r0s32 = ShapeUtil::MakeShape(S32, {});
+  Shape r0u32 = ShapeUtil::MakeShape(U32, {});
+  auto param0 = builder.AddInstruction(
+      HloInstruction::CreateParameter(0, r0f32_, "p.0.lhs"));
+  auto param1 = builder.AddInstruction(
+      HloInstruction::CreateParameter(1, r0f32_, "p.0.rhs"));
+  auto param2 =
+      builder.AddInstruction(HloInstruction::CreateParameter(2, r0s64, "p.1"));
+  auto lt = builder.AddInstruction(HloInstruction::CreateBinary(
+      ShapeUtil::MakeShape(PRED, {}), HloOpcode::kLt, param0, param1));
+  auto module = CreateNewVerifiedModule();
+  auto computation =
+      module->AddEntryComputation(builder.Build(/*root_instruction=*/lt));
+  std::unordered_map<const HloInstruction*, std::unique_ptr<HloInstruction>>
+      replacements;
+  replacements.emplace(param2,
+                       HloInstruction::CreateParameter(2, r0s32, "p.1"));
+  auto param3 = HloInstruction::CreateParameter(3, r0u32, "p.2");
+  std::vector<const HloInstruction*> extra_parameters{param3.get()};
+  auto clone = computation->CloneWithReplacements(std::move(replacements),
+                                                  extra_parameters);
+  ASSERT_EQ(clone->num_parameters(), 4);
+  EXPECT_TRUE(
+      ShapeUtil::Equal(clone->parameter_instruction(0)->shape(), r0f32_));
+  EXPECT_TRUE(
+      ShapeUtil::Equal(clone->parameter_instruction(1)->shape(), r0f32_));
+  EXPECT_TRUE(
+      ShapeUtil::Equal(clone->parameter_instruction(2)->shape(), r0s32));
+  EXPECT_TRUE(
+      ShapeUtil::Equal(clone->parameter_instruction(3)->shape(), r0u32));
+}
+
 TEST_F(HloComputationTest, Stringification) {
   const Shape s1 = ShapeUtil::MakeShape(F32, {5, 10});
   const Shape s2 = ShapeUtil::MakeShape(F32, {20, 10});
-- 
GitLab


From 94548dbeca41c3a94287bf9e506f876272904227 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 9 Jan 2019 02:49:25 -0800
Subject: [PATCH 0441/1095] Object based saved model: Wire __call__ attribute
 as a method in each user object class.

PiperOrigin-RevId: 228485119
---
 tensorflow/python/saved_model/load.py      | 20 +++++++++-
 tensorflow/python/saved_model/load_test.py | 43 ++++++++++++++++++++++
 2 files changed, 62 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/saved_model/load.py b/tensorflow/python/saved_model/load.py
index 1ee1b69b03..bbb1485612 100644
--- a/tensorflow/python/saved_model/load.py
+++ b/tensorflow/python/saved_model/load.py
@@ -85,11 +85,17 @@ class _Loader(object):
     raise ValueError("Can't convert node %s to tensor" % (type(obj)))
 
   def _load_all(self):
+    """Load all saved objects and wire their properties."""
     self._nodes = [self._recreate(proto) for proto in self._proto.nodes]
     # After creating the objects, construct the edges between the objects.
     for obj, object_proto in zip(self._nodes, self._proto.nodes):
       for reference in object_proto.children:
         setattr(obj, reference.local_name, self._nodes[reference.node_id])
+        # Note: if an object has an attribute `__call__` add a class method
+        # that allows `obj()` syntax to work. This is done per-instance to
+        # allow `callable` to be used to find out if an object is callable.
+        if reference.local_name == "__call__":
+          setattr(type(obj), "__call__", _call_attribute)
 
   def _restore_checkpoint(self):
     variables_path = saved_model_utils.get_variables_path(self._export_dir)
@@ -113,8 +119,16 @@ class _Loader(object):
     return factory[kind]()
 
   def _recreate_user_object(self, proto):
+    """Instantiates a SavedUserObject."""
     del proto
-    return tracking.Checkpointable()
+
+    # Note: each user object has its own class. This allows to make each one
+    # individually callable by adding a `__call__` method to the classes of
+    # the objects instances that have a `__call__` property.
+    class _UserObject(tracking.Checkpointable):
+      pass
+
+    return _UserObject()
 
   def _recreate_asset(self, proto):
     filename = os.path.join(
@@ -132,6 +146,10 @@ class _Loader(object):
     return variables.Variable(dummy_value, trainable=proto.trainable)
 
 
+def _call_attribute(instance, *args, **kwargs):
+  return instance.__call__(*args, **kwargs)
+
+
 def _load_saved_object_graph_proto(filename):
   with file_io.FileIO(filename, "rb") as f:
     contents = f.read()
diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index 35e384dbfc..f7020e4a45 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -269,6 +269,49 @@ class LoadTest(test.TestCase):
       grad = t.gradient(loss, [imported.weight, imported.bias])
       self.assertAllClose(grad, [3.5, 1.0])
 
+  def test_callable(self):
+    class M1(tracking.Checkpointable):
+
+      @def_function.function(
+          input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)])
+      def __call__(self, x):
+        return x
+
+    root = tracking.Checkpointable()
+    root.m1 = M1()
+    root.m2 = tracking.Checkpointable()
+    root.m2.__call__ = def_function.function(
+        input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)])(
+            lambda x: x*3.0)
+    imported = self.cycle(root)
+    x = constant_op.constant(1.0)
+
+    self.assertTrue(callable(imported.m1))
+    self.assertAllEqual(root.m1(x), imported.m1(x))
+
+    # Note: `root.m2` was not callable since `__call__` attribute was set
+    # into the instance and not on the class. But after a serialization cycle
+    # that starts to work.
+    self.assertTrue(callable(imported.m2))
+    self.assertAllEqual(root.m2.__call__(x), imported.m2(x))
+
+    # Verify that user objects without `__call__` attribute are not callable.
+    self.assertFalse(callable(imported))
+
+  def test_chain_callable(self):
+    func = def_function.function(
+        input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)])(
+            lambda x: x*3.0)
+    root = tracking.Checkpointable()
+    root.__call__ = tracking.Checkpointable()
+    root.__call__.__call__ = tracking.Checkpointable()
+    root.__call__.__call__.__call__ = func
+
+    imported = self.cycle(root)
+    self.assertTrue(callable(imported))
+    x = constant_op.constant(1.0)
+    self.assertAllEqual(imported(x).numpy(), 3.0)
+
 
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From 2db3c2ed12f7871322448c3265b82e21fa50e6aa Mon Sep 17 00:00:00 2001
From: Tamara Norman <tamaranorman@google.com>
Date: Wed, 9 Jan 2019 02:50:05 -0800
Subject: [PATCH 0442/1095] Alter cast conversion so returns valid data types
 and allows a positional name arg

PiperOrigin-RevId: 228485190
---
 .../tools/compatibility/tf_upgrade_v2.py      | 11 +++++++++
 .../tools/compatibility/tf_upgrade_v2_test.py | 24 ++++++++++++++++---
 2 files changed, 32 insertions(+), 3 deletions(-)

diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 2dbbe27598..940b57891a 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -1213,10 +1213,21 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
 
     # Find out the dtype to cast to from the function name
     dtype_str = name[3:]
+    # Special cases where the full dtype is not given
+    if dtype_str == "float":
+      dtype_str = "float32"
+    elif dtype_str == "double":
+      dtype_str = "float64"
     new_arg = ast.keyword(arg="dtype",
                           value=ast.Attribute(value=ast.Name(id="tf",
                                                              ctx=ast.Load()),
                                               attr=dtype_str, ctx=ast.Load()))
+    # Ensures a valid transformation when a positional name arg is given
+    if len(node.args) == 2:
+      name_arg = ast.keyword(arg="name",
+                             value=node.args[-1])
+      node.args = node.args[:-1]
+      node.keywords.append(name_arg)
 
     # Python3 ast requires the args for the Attribute, but codegen will mess up
     # the arg order if we just set them to 0.
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index 80d86d7a2b..b2bdddf15e 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -861,9 +861,27 @@ tf.print('abc')
     self.assertIn(new_text, [expected_text1, expected_text2])
 
   def testCast(self):
-    for dtype in ["int32", "int64", "float", "double",
-                  "complex64", "complex128", "bfloat16"]:
-      text = "tf.to_%s(x, name='test')" % dtype
+    for (name, dtype) in [("int32", "int32"),
+                          ("int64", "int64"),
+                          ("float", "float32"),
+                          ("double", "float64"),
+                          ("complex64", "complex64"),
+                          ("complex128", "complex128"),
+                          ("bfloat16", "bfloat16")]:
+      text = "tf.to_%s(x, name='test')" % name
+      expected_text = "tf.cast(x, name='test', dtype=tf.%s)" % dtype
+      _, unused_report, unused_errors, new_text = self._upgrade(text)
+      self.assertEqual(expected_text, new_text)
+
+  def testCastPositionalSecondArgument(self):
+    for (name, dtype) in [("int32", "int32"),
+                          ("int64", "int64"),
+                          ("float", "float32"),
+                          ("double", "float64"),
+                          ("complex64", "complex64"),
+                          ("complex128", "complex128"),
+                          ("bfloat16", "bfloat16")]:
+      text = "tf.to_%s(x, 'test')" % name
       expected_text = "tf.cast(x, name='test', dtype=tf.%s)" % dtype
       _, unused_report, unused_errors, new_text = self._upgrade(text)
       self.assertEqual(expected_text, new_text)
-- 
GitLab


From 41b0fb94457c0de1688ed595e5ff2ada068fc93f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 9 Jan 2019 03:00:27 -0800
Subject: [PATCH 0443/1095] Optimize transpose_conv.

PiperOrigin-RevId: 228486143
---
 .../internal/optimized/optimized_ops.h        | 22 ++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/tensorflow/lite/kernels/internal/optimized/optimized_ops.h b/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
index 8f09fab4ca..ac68757b06 100644
--- a/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
+++ b/tensorflow/lite/kernels/internal/optimized/optimized_ops.h
@@ -6079,7 +6079,27 @@ inline void TransposeConv(
     const float* filter_data, const RuntimeShape& output_shape,
     float* output_data, const RuntimeShape& im2col_shape, float* im2col_data) {
   gemmlowp::ScopedProfilingLabel label("TransposeConv");
-
+  // The complexity of the reference implementation is input.flat_size() *
+  // filter.flat_size() / in_channel.
+  //
+  // While the complexity of im2col->gemm
+  // implmentation is batch * output_height * output_width *
+  // (filter.flat_size() / out_channel)^2 * out_channel.
+  //
+  // so if input.flat_size() * out_channel^2 is much smaller than
+  // output.flat_size() * filter.size() * in_channel we should fall back to the
+  // reference implementation.
+  //
+  // TODO(b/122331966): optimize the intuitive implementation.
+  const int out_channel = output_shape.Dims(3);
+  const int in_channel = input_shape.Dims(3);
+  if ((input_shape.FlatSize() * out_channel * out_channel * 4) <
+      (filter_shape.FlatSize() * output_shape.FlatSize() * in_channel)) {
+    reference_ops::TransposeConv(params, input_shape, input_data, filter_shape,
+                                 filter_data, output_shape, output_data,
+                                 im2col_shape, im2col_data);
+    return;
+  }
   // Note we could use transposed weights with forward conv for unstrided
   // cases. But we are already getting good performance with this code as-is.
   TFLITE_DCHECK(im2col_data);
-- 
GitLab


From 964e3d0bd8bcfbfc8bcc1de16f32df8190cd7b75 Mon Sep 17 00:00:00 2001
From: Tamara Norman <tamaranorman@google.com>
Date: Wed, 9 Jan 2019 03:29:29 -0800
Subject: [PATCH 0444/1095] Add conversion for tf.image.resize_* methods

PiperOrigin-RevId: 228489020
---
 tensorflow/tools/compatibility/renames_v2.py  |  5 +-
 .../tools/compatibility/tf_upgrade_v2.py      | 51 +++++++++++++++++++
 .../tools/compatibility/tf_upgrade_v2_test.py | 18 +++++++
 3 files changed, 71 insertions(+), 3 deletions(-)

diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index 5ff8a86b91..e9085e94b3 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -141,8 +141,8 @@ renames = {
     'tf.diag': 'tf.linalg.tensor_diag',
     'tf.diag_part': 'tf.linalg.tensor_diag_part',
     'tf.digamma': 'tf.math.digamma',
-    'tf.dimension_at_index': 'tf.compat.v1.dimension_at_index',
-    'tf.dimension_value': 'tf.compat.v1.dimension_value',
+    'tf.dimension_at_index': 'tf.compat.dimension_at_index',
+    'tf.dimension_value': 'tf.compat.dimension_value',
     'tf.disable_eager_execution': 'tf.compat.v1.disable_eager_execution',
     'tf.disable_resource_variables': 'tf.compat.v1.disable_resource_variables',
     'tf.disable_v2_batch_normalization': 'tf.compat.v1.disable_v2_batch_normalization',
@@ -221,7 +221,6 @@ renames = {
     'tf.image.resize_area': 'tf.compat.v1.image.resize_area',
     'tf.image.resize_bicubic': 'tf.compat.v1.image.resize_bicubic',
     'tf.image.resize_bilinear': 'tf.compat.v1.image.resize_bilinear',
-    'tf.image.resize_images': 'tf.compat.v1.image.resize_images',
     'tf.image.resize_nearest_neighbor': 'tf.compat.v1.image.resize_nearest_neighbor',
     'tf.image.transpose_image': 'tf.compat.v1.image.transpose_image',
     'tf.initialize_all_tables': 'tf.compat.v1.initialize_all_tables',
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 940b57891a..e93699fdbf 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -594,6 +594,8 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "tf.compat.v1.initializers.random_normal",
         "tf.truncated_normal_initializer":
             "tf.compat.v1.initializers.truncated_normal",
+        "tf.image.resize_images":
+            "tf.image.resize",
     }
     # pylint: enable=line-too-long
 
@@ -725,6 +727,11 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.to_int64": self._cast_transformer,
         "tf.nn.softmax_cross_entropy_with_logits":
             self._softmax_cross_entropy_with_logits_transformer,
+        "tf.image.resize_area": self._image_resize_transformer,
+        "tf.image.resize_bicubic": self._image_resize_transformer,
+        "tf.image.resize_bilinear": self._image_resize_transformer,
+        "tf.image.resize_nearest_neighbor": self._image_resize_transformer,
+
     }
 
     decay_function_comment = (
@@ -1292,3 +1299,47 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     logs.append((node.lineno, node.col_offset,
                  "Added keyword argument batch_dims=-1 to tf.batch_gather."))
     return node
+
+  @staticmethod
+  def _image_resize_transformer(parent, node, full_name, name, logs, errors):
+    """Transforms image.resize_* to image.resize(..., method=*, ...)."""
+
+    resize_method = name[7:].upper()
+    new_arg = ast.keyword(arg="method",
+                          value=ast.Attribute(
+                              value=ast.Attribute(
+                                  value=ast.Attribute(
+                                      value=ast.Name(id="tf", ctx=ast.Load()),
+                                      attr="image", ctx=ast.Load()),
+                                  attr="ResizeMethod", ctx=ast.Load()),
+                              attr=resize_method, ctx=ast.Load()))
+
+    # Ensures a valid transformation when a positional name arg is given
+    if len(node.args) == 4:
+      pos_arg = ast.keyword(arg="preserve_aspect_ratio",
+                            value=node.args[-1])
+      node.args = node.args[:-1]
+      node.keywords.append(pos_arg)
+    if len(node.args) == 3:
+      pos_arg = ast.keyword(arg="align_corners",
+                            value=node.args[-1])
+      node.args = node.args[:-1]
+      node.keywords.append(pos_arg)
+
+    # Python3 ast requires the args for the Attribute, but codegen will mess up
+    # the arg order if we just set them to 0.
+    new_arg.value.lineno = node.lineno
+    new_arg.value.col_offset = node.col_offset+100
+
+    node.keywords.append(new_arg)
+    if isinstance(node.func, ast.Attribute):
+      node.func.attr = "resize"
+    else:
+      assert isinstance(node.func, ast.Name)
+      node.func.id = "resize"
+
+    logs.append((node.lineno, node.col_offset,
+                 "Changed %s call to tf.image.resize(..., "
+                 "method=tf.image.ResizeMethod.%s)." % (full_name,
+                                                        resize_method)))
+    return node
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index b2bdddf15e..c1882afd5c 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -886,6 +886,24 @@ tf.print('abc')
       _, unused_report, unused_errors, new_text = self._upgrade(text)
       self.assertEqual(expected_text, new_text)
 
+  def testImageResize(self):
+    for method in ["bilinear", "area", "bicubic", "nearest_neighbor"]:
+      text = "tf.image.resize_%s(i, s)" % method
+      expected_text = ("tf.image.resize(i, s, "
+                       "method=tf.image.ResizeMethod.%s)" % method.upper())
+      _, unused_report, unused_errors, new_text = self._upgrade(text)
+      self.assertEqual(expected_text, new_text)
+
+  def testImageResizeExtraPositionalArgs(self):
+    for method in ["bilinear", "area", "bicubic", "nearest_neighbor"]:
+      text = "tf.image.resize_%s(i, s, a, p)" % method
+      expected_text = ["tf.image.resize(i, s, ", "align_corners=a, ",
+                       "preserve_aspect_ratio=p, ",
+                       "method=tf.image.ResizeMethod.%s)" % method.upper()]
+      _, unused_report, unused_errors, new_text = self._upgrade(text)
+      for s in expected_text:
+        self.assertIn(s, new_text)
+
 
 class TestUpgradeFiles(test_util.TensorFlowTestCase):
 
-- 
GitLab


From 6e6d7aad7300294425933bd053afce55107ccd35 Mon Sep 17 00:00:00 2001
From: Tamara Norman <tamaranorman@google.com>
Date: Wed, 9 Jan 2019 04:31:58 -0800
Subject: [PATCH 0445/1095] Make tf.cond remove the strict arg and emit a
 warning to say it has been set to true.

Adds additional changes which ensure this warning is emitted if the original argument was positional.

PiperOrigin-RevId: 228495150
---
 tensorflow/tools/compatibility/ast_edits.py   |  8 +++----
 tensorflow/tools/compatibility/reorders_v2.py |  1 +
 .../tools/compatibility/tf_upgrade_v2.py      | 21 ++++++++++++-------
 .../tools/compatibility/tf_upgrade_v2_test.py |  8 +++++++
 4 files changed, 27 insertions(+), 11 deletions(-)

diff --git a/tensorflow/tools/compatibility/ast_edits.py b/tensorflow/tools/compatibility/ast_edits.py
index 859fd95314..2254c223ce 100644
--- a/tensorflow/tools/compatibility/ast_edits.py
+++ b/tensorflow/tools/compatibility/ast_edits.py
@@ -213,7 +213,7 @@ class _PastaEditVisitor(ast.NodeVisitor):
       return False
 
   def _maybe_add_call_warning(self, node, full_name, name):
-    """Print a warning when specific functions are called.
+    """Print a warning when specific functions are called with selected args.
 
     The function _print_warning_for_function matches the full name of the called
     function, e.g., tf.foo.bar(). This function matches the function name that
@@ -241,13 +241,13 @@ class _PastaEditVisitor(ast.NodeVisitor):
                                              full_name, name)
 
     used_args = [kw.arg for kw in node.keywords]
-    for arg, warning in arg_warnings.items():
-      if arg in used_args:
+    for (kwarg, arg), warning in arg_warnings.items():
+      if kwarg in used_args or len(node.args) > arg:
         warned = True
         warning_message = warning.replace("<function name>", full_name or name)
         self.add_error(node.lineno, node.col_offset,
                        "%s called with %s argument requires manual check: %s." %
-                       (full_name or name, arg, warning_message))
+                       (full_name or name, kwarg, warning_message))
 
     return warned
 
diff --git a/tensorflow/tools/compatibility/reorders_v2.py b/tensorflow/tools/compatibility/reorders_v2.py
index 5c11388516..f9b0e3f9d8 100644
--- a/tensorflow/tools/compatibility/reorders_v2.py
+++ b/tensorflow/tools/compatibility/reorders_v2.py
@@ -31,6 +31,7 @@ reorders = {
     'tf.batch_gather': ['params', 'indices', 'name'],
     'tf.batch_to_space': ['input', 'crops', 'block_size', 'name'],
     'tf.boolean_mask': ['tensor', 'mask', 'name', 'axis'],
+    'tf.cond': ['pred', 'true_fn', 'false_fn', 'strict', 'name', 'fn1', 'fn2'],
     'tf.confusion_matrix': ['labels', 'predictions', 'num_classes', 'dtype', 'name', 'weights'],
     'tf.convert_to_tensor': ['value', 'dtype', 'name', 'preferred_dtype'],
     'tf.decode_csv': ['records', 'record_defaults', 'field_delim', 'use_quote_delim', 'name', 'na_value', 'select_cols'],
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index e93699fdbf..2c56cbf36b 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -50,6 +50,11 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "*.compute_gradients": {
             "colocate_gradients_with_ops": None,
         },
+        "tf.cond": {
+            "strict": None,
+            "fn1": "true_fn",
+            "fn2": "false_fn"
+        },
         "tf.argmin": {
             "dimension": "axis",
         },
@@ -620,6 +625,7 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.argmin",
         "tf.batch_gather",
         "tf.batch_to_space",
+        "tf.cond",
         "tf.nn.space_to_batch",
         "tf.boolean_mask",
         "tf.convert_to_tensor",
@@ -847,10 +853,6 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             assert_return_type_comment,
         "tf.assert_rank":
             assert_rank_comment,
-        "tf.cond":
-            "tf.cond no longer takes 'strict'. "
-            "Now 'strict' defaults to True."
-            "fn1/fn2 arguments are replaced by true_fn/false_fn.",
         "tf.debugging.assert_equal":
             assert_return_type_comment,
         "tf.debugging.assert_greater":
@@ -1146,23 +1148,28 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
     # Warnings that are emitted only if a specific arg is found.
     self.function_arg_warnings = {
         "tf.gradients": {
-            "colocate_gradients_with_ops":
+            ("colocate_gradients_with_ops", 4):
                 "tf.gradients no longer takes "
                 "'colocate_gradients_with_ops' argument, it behaves as if it "
                 "was set to True.",
         },
         "*.minimize": {
-            "colocate_gradients_with_ops":
+            ("colocate_gradients_with_ops", 5):
                 "Optimizer.minimize no longer takes "
                 "'colocate_gradients_with_ops' argument, it behaves as if it "
                 "was set to True.",
         },
         "*.compute_gradients": {
-            "colocate_gradients_with_ops":
+            ("colocate_gradients_with_ops", 4):
                 "Optimizer.compute_gradients no "
                 "longer takes 'colocate_gradients_with_ops' argument, it "
                 "behaves as if it was set to True.",
         },
+        "tf.cond": {
+            ("strict", 3):
+                "tf.cond no longer takes 'strict' argument, it behaves as "
+                "if was set to True."
+        },
     }
 
     self.symbol_renames = {
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index c1882afd5c..0a85cb39c8 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -904,6 +904,14 @@ tf.print('abc')
       for s in expected_text:
         self.assertIn(s, new_text)
 
+  def testCond(self):
+    text = "tf.cond(a, b, c, True)"
+    expected_text = "tf.cond(pred=a, true_fn=b, false_fn=c)"
+    _, unused_report, errors, new_text = self._upgrade(text)
+    self.assertEqual(expected_text, new_text)
+    self.assertIn("tf.cond", errors[0])
+    self.assertIn("requires manual check", errors[0])
+
 
 class TestUpgradeFiles(test_util.TensorFlowTestCase):
 
-- 
GitLab


From 7e175f8f732806256a739e55e8f71e59c645c130 Mon Sep 17 00:00:00 2001
From: Jia Qingtong <wanywhn@qq.com>
Date: Wed, 9 Jan 2019 21:44:20 +0800
Subject: [PATCH 0446/1095] [typo] Fix a bazel url typo

Signed-off-by: Jia Qingtong <wanywhn@qq.com>
---
 tensorflow/workspace.bzl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 5b0f6d4c84..2ce70557fa 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -141,7 +141,7 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
         sha256 = "9de38f2d162c51599b802f7c36d9f3773980d19ac908c61638f8344d2c10e1ca",
         strip_prefix = "eigen-eigen-88fc23324517",
         urls = [
-            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/88fc23324517..tar.gz",
+            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/88fc23324517.tar.gz",
             "https://bitbucket.org/eigen/eigen/get/88fc23324517.tar.gz",
         ],
     )
-- 
GitLab


From b5fdd811998d495950c4b736879454be39acf163 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 9 Jan 2019 07:08:38 -0800
Subject: [PATCH 0447/1095] Stop using set for signatures comparison. Sets
 internal hashing mechanism makes it impossible to use any kind of structures
 (lists, etc.).

Also do some refactoring.

PiperOrigin-RevId: 228511638
---
 tensorflow/python/eager/def_function.py | 16 +++++++++------
 tensorflow/python/eager/function.py     | 27 ++++++++++++++++++-------
 2 files changed, 30 insertions(+), 13 deletions(-)

diff --git a/tensorflow/python/eager/def_function.py b/tensorflow/python/eager/def_function.py
index aa4f20df49..4c22a12c1c 100644
--- a/tensorflow/python/eager/def_function.py
+++ b/tensorflow/python/eager/def_function.py
@@ -446,10 +446,12 @@ class PolymorphicFunction(object):
   @property
   def _cached_input_signatures(self):
     """All input signatures used to call this PolymorphicFunction."""
-    seen = set()
-    # Preserves signature ordering rather than returning a set() so that we
-    # don't need to re-sort signatures later to work around Python 2's set
-    # nondeterminism.
+    seen = list()
+    # We are using a list so that:
+    #  - the returned collection is deterministic, and
+    #  - we can use a custom equality operator (is_same_structure).
+    # This is run only at serialization time on likely very small inputs so we
+    # are not concerned about O(n^2) runtime.
     # pylint: disable=protected-access
     concrete_functions = []
     if self._stateful_fn:
@@ -458,9 +460,11 @@ class PolymorphicFunction(object):
       concrete_functions.extend(self._stateless_fn._function_cache.values())
     for concrete_function in concrete_functions:
       signature = concrete_function._python_call_signature
-      if signature not in seen:
+      equal_to_signature = functools.partial(
+          function_lib.is_same_structure, signature, check_values=True)
+      if not any(equal_to_signature(s) for s in seen):
         yield signature
-        seen.add(signature)
+        seen.append(signature)
     # pylint: enable=protected-access
 
   def get_concrete_function(self, *args, **kwargs):
diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index 05fccfbcd9..67c633726f 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -74,6 +74,24 @@ CacheKey = collections.namedtuple("CacheKey", [
 ])
 
 
+def is_same_structure(structure1,
+                      structure2,
+                      check_values=False):
+  """Check two structures for equality, optionally of types and of values."""
+  try:
+    nest.assert_same_structure(structure1, structure2)
+  except (ValueError, TypeError):
+    return False
+  if check_values:
+    flattened1 = nest.flatten(structure1)
+    flattened2 = nest.flatten(structure2)
+    # First check the types to avoid AttributeErrors.
+    if any(type(f1) != type(f2) for f1, f2 in zip(flattened1, flattened2)):
+      return False
+    return flattened1 == flattened2
+  return True
+
+
 def _parse_func_attrs(attributes):
   """Convert the keyword arguments into function_def attributes.
 
@@ -919,10 +937,7 @@ class FunctionSpec(object):
     else:
       assert not kwargs
       signature_relevant_inputs = inputs[:len(self.input_signature)]
-      try:
-        nest.assert_same_structure(self.input_signature,
-                                   signature_relevant_inputs)
-      except (ValueError, TypeError):
+      if not is_same_structure(self.input_signature, signature_relevant_inputs):
         raise ValueError("Structure of Python function inputs does not match "
                          "input_signature.")
       signature_inputs_flat = nest.flatten(signature_relevant_inputs)
@@ -1049,9 +1064,7 @@ class PolymorphicFunction(object):
                          "input_signature is provided.")
       if args:
         # If args are provided, they must match the input signature.
-        try:
-          nest.assert_same_structure(self._input_signature, args)
-        except (ValueError, TypeError):
+        if not is_same_structure(self._input_signature, args):
           raise ValueError("Structure of Python function inputs does not match "
                            "input_signature.")
         flat_inputs = nest.flatten(args)
-- 
GitLab


From 433d47e39ce5e1c8c478393e7f5d2917c7df0fa7 Mon Sep 17 00:00:00 2001
From: Shashi Shekhar <shashishekhar@google.com>
Date: Wed, 9 Jan 2019 08:00:10 -0800
Subject: [PATCH 0448/1095] Add symmetric quantization support to the accuracy
 tool.

PiperOrigin-RevId: 228517966
---
 .../ilsvrc/imagenet_model_evaluator.cc        | 26 ++++------
 .../accuracy/ilsvrc/imagenet_topk_eval.cc     |  9 +++-
 .../ilsvrc/inception_preprocessing.cc         | 42 +++++++++------
 .../accuracy/ilsvrc/inception_preprocessing.h | 52 ++++++++++++++-----
 .../ilsvrc/inception_preprocessing_test.cc    | 41 ++++++++++++---
 5 files changed, 119 insertions(+), 51 deletions(-)

diff --git a/tensorflow/lite/tools/accuracy/ilsvrc/imagenet_model_evaluator.cc b/tensorflow/lite/tools/accuracy/ilsvrc/imagenet_model_evaluator.cc
index 9a74e221c1..129747fe4d 100644
--- a/tensorflow/lite/tools/accuracy/ilsvrc/imagenet_model_evaluator.cc
+++ b/tensorflow/lite/tools/accuracy/ilsvrc/imagenet_model_evaluator.cc
@@ -22,6 +22,12 @@ limitations under the License.
 
 #include "absl/memory/memory.h"
 #include "tensorflow/cc/framework/scope.h"
+#include "tensorflow/core/lib/core/blocking_counter.h"
+#include "tensorflow/core/lib/core/threadpool.h"
+#include "tensorflow/core/platform/init_main.h"
+#include "tensorflow/core/platform/mutex.h"
+#include "tensorflow/core/public/session.h"
+#include "tensorflow/core/util/command_line_flags.h"
 #include "tensorflow/lite/tools/accuracy/eval_pipeline.h"
 #include "tensorflow/lite/tools/accuracy/eval_pipeline_builder.h"
 #include "tensorflow/lite/tools/accuracy/file_reader_stage.h"
@@ -29,12 +35,6 @@ limitations under the License.
 #include "tensorflow/lite/tools/accuracy/ilsvrc/inception_preprocessing.h"
 #include "tensorflow/lite/tools/accuracy/run_tflite_model_stage.h"
 #include "tensorflow/lite/tools/accuracy/utils.h"
-#include "tensorflow/core/lib/core/blocking_counter.h"
-#include "tensorflow/core/lib/core/threadpool.h"
-#include "tensorflow/core/platform/init_main.h"
-#include "tensorflow/core/platform/mutex.h"
-#include "tensorflow/core/public/session.h"
-#include "tensorflow/core/util/command_line_flags.h"
 
 namespace {
 using tensorflow::string;
@@ -185,21 +185,17 @@ Status EvaluateModelForShard(const uint64_t shard_id,
   const TensorShape& input_shape = model_info.input_shapes[0];
   const int image_height = input_shape.dim_size(1);
   const int image_width = input_shape.dim_size(2);
-  const bool is_quantized = (model_info.input_types[0] == DT_UINT8);
 
   RunTFLiteModelStage::Params tfl_model_params;
   tfl_model_params.model_file_path = params.model_file_path;
-  if (is_quantized) {
-    tfl_model_params.input_type = {DT_UINT8};
-    tfl_model_params.output_type = {DT_UINT8};
-  } else {
-    tfl_model_params.input_type = {DT_FLOAT};
-    tfl_model_params.output_type = {DT_FLOAT};
-  }
+
+  tfl_model_params.input_type = {model_info.input_types[0]};
+  tfl_model_params.output_type = {model_info.input_types[0]};
 
   Scope root = Scope::NewRootScope();
   FileReaderStage reader;
-  InceptionPreprocessingStage inc(image_height, image_width, is_quantized);
+  InceptionPreprocessingStage inc(image_height, image_width,
+                                  model_info.input_types[0]);
   RunTFLiteModelStage tfl_model_stage(tfl_model_params);
   EvalPipelineBuilder builder;
 
diff --git a/tensorflow/lite/tools/accuracy/ilsvrc/imagenet_topk_eval.cc b/tensorflow/lite/tools/accuracy/ilsvrc/imagenet_topk_eval.cc
index 2b086cdf70..f5642d52a8 100644
--- a/tensorflow/lite/tools/accuracy/ilsvrc/imagenet_topk_eval.cc
+++ b/tensorflow/lite/tools/accuracy/ilsvrc/imagenet_topk_eval.cc
@@ -67,11 +67,18 @@ Status ImagenetTopKAccuracy::ComputeEval(
     for (size_t i = 0; i < probs.size(); i++) {
       probabilities.push_back(probs(i));
     }
-  } else {
+  } else if (output.dtype() == DT_UINT8) {
     auto probs = output.flat<uint8>();
     for (size_t i = 0; i < probs.size(); i++) {
       probabilities.push_back(probs(i));
     }
+  } else if (output.dtype() == DT_INT8) {
+    auto probs = output.flat<int8>();
+    for (size_t i = 0; i < probs.size(); i++) {
+      probabilities.push_back(probs(i));
+    }
+  } else {
+    return errors::InvalidArgument("Invalid datatype");
   }
 
   CHECK_EQ(kNumCategories, probabilities.size());
diff --git a/tensorflow/lite/tools/accuracy/ilsvrc/inception_preprocessing.cc b/tensorflow/lite/tools/accuracy/ilsvrc/inception_preprocessing.cc
index 9a889f0dd8..04b6cb7558 100644
--- a/tensorflow/lite/tools/accuracy/ilsvrc/inception_preprocessing.cc
+++ b/tensorflow/lite/tools/accuracy/ilsvrc/inception_preprocessing.cc
@@ -57,23 +57,33 @@ void InceptionPreprocessingStage::AddToGraph(const Scope& scope,
   tensorflow::Output cropped_image;
   CentralCropImage(s, decoded_jpeg, params_.cropping_fraction, &cropped_image);
   auto dims_expander = ops::ExpandDims(s, cropped_image, 0);
-  auto resized_image = ops::ResizeBilinear(
-      s, dims_expander,
-      ops::Const(s.WithOpName("size"), {image_height_, image_width_}));
-  if (is_quantized_) {
-    this->stage_output_ =
-        ops::Cast(s.WithOpName(output_name()), resized_image, DT_UINT8);
-  } else {
-    auto squeezed_image = ops::Squeeze(s, resized_image);
-    auto normalized_image =
-        ops::Div(s,
-                 ops::Sub(s, squeezed_image,
-                          {params_.input_means[0], params_.input_means[1],
-                           params_.input_means[2]}),
-                 {params_.scale});
-    this->stage_output_ =
-        ops::ExpandDims(s.WithOpName(output_name()), normalized_image, {0});
+  auto resized_image =
+      ops::ResizeBilinear(s.WithOpName("resize"), dims_expander,
+                          ops::Const(s, {image_height_, image_width_}));
+
+  ::tensorflow::Output preprocessed_image = resized_image;
+
+  if (!params_.input_means.empty()) {
+    preprocessed_image =
+        ops::Sub(s.WithOpName("sub"), preprocessed_image,
+                 {params_.input_means[0], params_.input_means[1],
+                  params_.input_means[2]});
+  }
+
+  if (std::abs(params_.scale) > 1e-7f) {
+    auto squeezed_image = ops::Squeeze(s, preprocessed_image);
+    preprocessed_image = ops::Div(s, squeezed_image, {params_.scale});
+    preprocessed_image = ops::ExpandDims(s, preprocessed_image, {0});
   }
+
+  // Cast the output from float to output datatype.
+  if (output_datatype_ != DT_FLOAT) {
+    preprocessed_image =
+        ops::Cast(s.WithOpName("cast"), preprocessed_image, output_datatype_);
+  }
+
+  this->stage_output_ =
+      ops::Identity(s.WithOpName(output_name()), preprocessed_image);
 }
 
 }  // namespace metrics
diff --git a/tensorflow/lite/tools/accuracy/ilsvrc/inception_preprocessing.h b/tensorflow/lite/tools/accuracy/ilsvrc/inception_preprocessing.h
index 4a1d3ce476..371feb3e76 100644
--- a/tensorflow/lite/tools/accuracy/ilsvrc/inception_preprocessing.h
+++ b/tensorflow/lite/tools/accuracy/ilsvrc/inception_preprocessing.h
@@ -13,14 +13,14 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_LITE_TOOLS_ACCURACY_INCEPTION_PREPROCESSING_H_
-#define TENSORFLOW_LITE_TOOLS_ACCURACY_INCEPTION_PREPROCESSING_H_
+#ifndef TENSORFLOW_LITE_TOOLS_ACCURACY_ILSVRC_INCEPTION_PREPROCESSING_H_
+#define TENSORFLOW_LITE_TOOLS_ACCURACY_ILSVRC_INCEPTION_PREPROCESSING_H_
 
 #include <utility>
 
-#include "tensorflow/lite/tools/accuracy/stage.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/lite/tools/accuracy/stage.h"
 
 namespace tensorflow {
 namespace metrics {
@@ -31,28 +31,53 @@ namespace metrics {
 // shape {1, image_height, image_width, 3}, where 3 is the number of channels.
 class InceptionPreprocessingStage : public Stage {
  public:
+  // Preprocessing params that govern scaling and normalization of channels of
+  // the image.
   struct Params {
+    // Input means are subtracted from each channel.
+    // In case of an empty vector this is skipped.
     std::vector<float> input_means;
+    // Scale is used to divide the input.
+    // A scale of 0 means divison is skipped.
     float scale;
     double cropping_fraction;
   };
 
-  static Params DefaultParams() {
-    return {.input_means = {127.5, 127.5, 127.5},
-            .scale = 127.5,
-            .cropping_fraction = 0.875};
+  // Default preprocessing for inception stage based on |output_type|
+  static Params DefaultParamsForType(DataType output_type) {
+    const float kCroppingFraction = 0.875;
+    Params params = {};
+    params.cropping_fraction = kCroppingFraction;
+    if (output_type == DT_UINT8) {
+    } else if (output_type == DT_INT8) {
+      params.input_means = {128.0, 128.0, 128.0};
+    } else {
+      // Assume floating point preprocessing.
+      params.input_means = {127.5, 127.5, 127.5};
+      params.scale = 127.5;
+    }
+    return params;
+  }
+
+  // Creates a new preprocessing stage object with provided |image_width|
+  // |image_height| as the size of output image.
+  // |output_datatype| is the datatype of output of the stage.
+  InceptionPreprocessingStage(int image_width, int image_height,
+                              DataType output_datatype)
+      : output_datatype_(output_datatype),
+        image_width_(image_width),
+        image_height_(image_height) {
+    params_ = DefaultParamsForType(output_datatype);
   }
 
   // Creates a new preprocessing stage object with provided |image_width|
   // |image_height| as the size of output image.
-  // If |is_quantized| is set to true then |params| is ignored since quantized
-  // images don't go through any preprocessing.
+  // |output_datatype| is the datatype of output of the stage.
   InceptionPreprocessingStage(int image_width, int image_height,
-                              bool is_quantized,
-                              Params params = DefaultParams())
-      : image_width_(image_width),
+                              DataType output_datatype, Params params)
+      : output_datatype_(output_datatype),
+        image_width_(image_width),
         image_height_(image_height),
-        is_quantized_(is_quantized),
         params_(std::move(params)) {}
 
   string name() const override { return "stage_inception_preprocess"; }
@@ -63,6 +88,7 @@ class InceptionPreprocessingStage : public Stage {
   void AddToGraph(const Scope& scope, const Input& input) override;
 
  private:
+  DataType output_datatype_;
   int image_width_;
   int image_height_;
   bool is_quantized_;
diff --git a/tensorflow/lite/tools/accuracy/ilsvrc/inception_preprocessing_test.cc b/tensorflow/lite/tools/accuracy/ilsvrc/inception_preprocessing_test.cc
index 5d0e01d7d1..f88847035f 100644
--- a/tensorflow/lite/tools/accuracy/ilsvrc/inception_preprocessing_test.cc
+++ b/tensorflow/lite/tools/accuracy/ilsvrc/inception_preprocessing_test.cc
@@ -17,10 +17,10 @@ limitations under the License.
 #include <string>
 
 #include <gtest/gtest.h>
-#include "tensorflow/lite/tools/accuracy/ilsvrc/inception_preprocessing.h"
 #include "tensorflow/core/platform/init_main.h"
 #include "tensorflow/core/public/session.h"
 #include "tensorflow/core/util/command_line_flags.h"
+#include "tensorflow/lite/tools/accuracy/ilsvrc/inception_preprocessing.h"
 
 namespace {
 tensorflow::string* g_test_image_file = nullptr;
@@ -48,7 +48,7 @@ Status GetContents(const string& filename, string* output) {
   }
 }
 
-TEST(InceptionPreprocessingTest, TestImagePreprocessQuantized) {
+TEST(InceptionPreprocessingTest, TestImagePreprocessUInt8Quantized) {
   ASSERT_TRUE(g_test_image_file != nullptr);
   string image_contents;
   string image_path = *g_test_image_file;
@@ -56,8 +56,8 @@ TEST(InceptionPreprocessingTest, TestImagePreprocessQuantized) {
   ASSERT_TRUE(status.ok()) << status.error_message();
   const int width = 224;
   const int height = 224;
-  const bool is_quantized = true;
-  InceptionPreprocessingStage preprocess_stage(width, height, is_quantized);
+  auto params = InceptionPreprocessingStage::DefaultParamsForType(DT_UINT8);
+  InceptionPreprocessingStage preprocess_stage(width, height, DT_UINT8, params);
   Scope scope = Scope::NewRootScope();
   preprocess_stage.AddToGraph(scope, image_contents);
   TF_CHECK_OK(scope.status());
@@ -77,6 +77,35 @@ TEST(InceptionPreprocessingTest, TestImagePreprocessQuantized) {
   EXPECT_TRUE(outputs[0].shape().IsSameSize({1, 224, 224, 3}));
 }
 
+TEST(InceptionPreprocessingTest, TestImagePreprocessInt8Quantized) {
+  ASSERT_TRUE(g_test_image_file != nullptr);
+  string image_contents;
+  string image_path = *g_test_image_file;
+  auto status = GetContents(image_path, &image_contents);
+  ASSERT_TRUE(status.ok()) << status.error_message();
+  const int width = 224;
+  const int height = 224;
+  auto params = InceptionPreprocessingStage::DefaultParamsForType(DT_INT8);
+  InceptionPreprocessingStage preprocess_stage(width, height, DT_INT8, params);
+  Scope scope = Scope::NewRootScope();
+  preprocess_stage.AddToGraph(scope, image_contents);
+  TF_CHECK_OK(scope.status());
+
+  GraphDef graph_def;
+  TF_CHECK_OK(scope.ToGraphDef(&graph_def));
+  std::unique_ptr<Session> session(NewSession(SessionOptions()));
+  TF_CHECK_OK(session->Create(graph_def));
+  std::vector<Tensor> outputs;
+  auto run_status =
+      session->Run({},                                   /*inputs*/
+                   {preprocess_stage.output_name()}, {}, /*target node names */
+                   &outputs);
+  TF_CHECK_OK(run_status);
+  EXPECT_EQ(1, outputs.size());
+  EXPECT_EQ(DT_INT8, outputs[0].dtype());
+  EXPECT_TRUE(outputs[0].shape().IsSameSize({1, 224, 224, 3}));
+}
+
 TEST(InceptionPreprocessingTest, TestImagePreprocessFloat) {
   ASSERT_TRUE(g_test_image_file != nullptr);
   string image_contents;
@@ -85,8 +114,8 @@ TEST(InceptionPreprocessingTest, TestImagePreprocessFloat) {
   ASSERT_TRUE(status.ok()) << status.error_message();
   const int width = 224;
   const int height = 224;
-  const bool is_quantized = false;
-  InceptionPreprocessingStage preprocess_stage(width, height, is_quantized);
+  auto params = InceptionPreprocessingStage::DefaultParamsForType(DT_FLOAT);
+  InceptionPreprocessingStage preprocess_stage(width, height, DT_FLOAT, params);
   Scope scope = Scope::NewRootScope();
   preprocess_stage.AddToGraph(scope, image_contents);
   TF_CHECK_OK(scope.status());
-- 
GitLab


From eed1c1413ff2c45c82142d5d4dd8b6d51a2f5636 Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Wed, 9 Jan 2019 08:11:28 -0800
Subject: [PATCH 0449/1095] Automated rollback of commit
 9f3acf3d89a4ef5f9313a6e2cb2d81e03177e8ef

PiperOrigin-RevId: 228519835
---
 tensorflow/lite/python/BUILD      |  1 +
 tensorflow/lite/python/convert.py | 13 +++++--------
 tensorflow/lite/toco/python/BUILD | 21 ++++++++++++++++++---
 3 files changed, 24 insertions(+), 11 deletions(-)

diff --git a/tensorflow/lite/python/BUILD b/tensorflow/lite/python/BUILD
index 7bfd1a8996..54b925cc05 100644
--- a/tensorflow/lite/python/BUILD
+++ b/tensorflow/lite/python/BUILD
@@ -74,6 +74,7 @@ py_test(
     data = ["@tflite_mobilenet_ssd_quant_protobuf//:tflite_graph.pb"],
     srcs_version = "PY2AND3",
     tags = [
+        "no_oss",
         "no_windows",
     ],
     deps = [
diff --git a/tensorflow/lite/python/convert.py b/tensorflow/lite/python/convert.py
index 9f315fc874..9c60399871 100644
--- a/tensorflow/lite/python/convert.py
+++ b/tensorflow/lite/python/convert.py
@@ -37,14 +37,11 @@ from tensorflow.python.util.tf_export import tf_export as _tf_export
 
 # Lazy load since some of the performance benchmark skylark rules
 # break dependencies.
-if lite_constants.EXPERIMENTAL_USE_TOCO_API_DIRECTLY:
-  _toco_python = LazyLoader(
-      "tensorflow_wrap_toco", globals(),
-      "tensorflow.lite.toco.python."
-      "tensorflow_wrap_toco")
-  del LazyLoader
-else:
-  _toco_python = None
+_toco_python = LazyLoader(
+    "tensorflow_wrap_toco", globals(),
+    "tensorflow.lite.toco.python."
+    "tensorflow_wrap_toco")
+del LazyLoader
 
 # Find the toco_from_protos binary using the resource loader if using from
 # bazel, otherwise we are in a pip where console_scripts already has
diff --git a/tensorflow/lite/toco/python/BUILD b/tensorflow/lite/toco/python/BUILD
index f3ea5b7bfe..8a6e82ec46 100644
--- a/tensorflow/lite/toco/python/BUILD
+++ b/tensorflow/lite/toco/python/BUILD
@@ -10,20 +10,35 @@ load("//tensorflow:tensorflow.bzl", "tf_py_wrap_cc")
 load("//tensorflow:tensorflow.bzl", "tf_py_test")
 load("//tensorflow:tensorflow.bzl", "py_binary")
 
+config_setting(
+    name = "tflite_convert_with_select_tf_ops",
+    define_values = {"tflite_convert_with_select_tf_ops": "true"},
+    visibility = [
+        "//tensorflow/contrib/lite:__subpackages__",
+        "//tensorflow/lite:__subpackages__",
+    ],
+)
+
 cc_library(
     name = "toco_python_api",
     srcs = ["toco_python_api.cc"],
     hdrs = ["toco_python_api.h"],
     deps = [
+        "//third_party/python_runtime:headers",
         "//tensorflow/core:lib",
-        "//tensorflow/core:ops",
         "//tensorflow/lite/toco:model_flags_proto_cc",
         "//tensorflow/lite/toco:toco_flags_proto_cc",
         "//tensorflow/lite/toco:toco_graphviz_dump_options",
         "//tensorflow/lite/toco:toco_port",
         "//tensorflow/lite/toco:toco_tooling",
-        "//third_party/python_runtime:headers",
-    ],
+    ] + select({
+        # This is required when running `tflite_convert` from `bazel`.
+        # It requires to link with TensorFlow Ops to get the op definitions.
+        ":tflite_convert_with_select_tf_ops": [
+            "//tensorflow/core:ops",
+        ],
+        "//conditions:default": [],
+    }),
 )
 
 tf_py_wrap_cc(
-- 
GitLab


From beae80428e5a7d67417dc18561b8ed74724fe4bf Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Wed, 9 Jan 2019 08:32:27 -0800
Subject: [PATCH 0450/1095] Move explicit template instantiations to namespace
 scope.

Fixes compile error on mac os on attached bug causing 1.13
build problems.

PiperOrigin-RevId: 228522576
---
 tensorflow/lite/testing/tflite_driver.cc | 121 ++++++++++++-----------
 1 file changed, 62 insertions(+), 59 deletions(-)

diff --git a/tensorflow/lite/testing/tflite_driver.cc b/tensorflow/lite/testing/tflite_driver.cc
index ffe296432a..a637dc86c0 100644
--- a/tensorflow/lite/testing/tflite_driver.cc
+++ b/tensorflow/lite/testing/tflite_driver.cc
@@ -79,32 +79,7 @@ class TfLiteDriver::Expectation {
     SetTensorData(values, &data_);
   }
 
-  template <>
-  void SetData<string>(const string& csv_values) {
-    string s = absl::HexStringToBytes(csv_values);
-    data_.raw = new char[s.size()];
-    memcpy(data_.raw, s.data(), s.size());
-  }
-
-  bool Check(bool verbose, const TfLiteTensor& tensor) {
-    switch (tensor.type) {
-      case kTfLiteFloat32:
-        return TypedCheck<float>(verbose, tensor);
-      case kTfLiteInt32:
-        return TypedCheck<int32_t>(verbose, tensor);
-      case kTfLiteInt64:
-        return TypedCheck<int64_t>(verbose, tensor);
-      case kTfLiteUInt8:
-        return TypedCheck<uint8_t>(verbose, tensor);
-      case kTfLiteBool:
-        return TypedCheck<bool>(verbose, tensor);
-      case kTfLiteString:
-        return TypedCheck<string>(verbose, tensor);
-      default:
-        fprintf(stderr, "Unsupported type %d in Check\n", tensor.type);
-        return false;
-    }
-  }
+  bool Check(bool verbose, const TfLiteTensor& tensor);
 
  private:
   template <typename T>
@@ -146,49 +121,77 @@ class TfLiteDriver::Expectation {
     return good_output;
   }
 
-  template <>
-  bool TypedCheck<string>(bool verbose, const TfLiteTensor& tensor) {
-    if (tensor.data.raw == nullptr) {
+  TfLitePtrUnion data_;
+  size_t num_elements_;
+};
+
+template <>
+void TfLiteDriver::Expectation::SetData<string>(const string& csv_values) {
+  string s = absl::HexStringToBytes(csv_values);
+  data_.raw = new char[s.size()];
+  memcpy(data_.raw, s.data(), s.size());
+}
+
+template <>
+bool TfLiteDriver::Expectation::TypedCheck<string>(bool verbose,
+                                                   const TfLiteTensor& tensor) {
+  if (tensor.data.raw == nullptr) {
+    if (verbose) {
+      std::cerr << "  got empty string" << std::endl;
+    }
+    return false;
+  }
+  int expected_num_strings = GetStringCount(data_.raw);
+  int returned_num_strings = GetStringCount(tensor.data.raw);
+  if (expected_num_strings != returned_num_strings) {
+    if (verbose) {
+      std::cerr << "  string count differ: got " << returned_num_strings
+                << ", but expected " << expected_num_strings << std::endl;
+    }
+    return false;
+  }
+  for (int i = 0; i < returned_num_strings; ++i) {
+    auto expected_ref = GetString(data_.raw, i);
+    auto returned_ref = GetString(tensor.data.raw, i);
+    if (expected_ref.len != returned_ref.len) {
       if (verbose) {
-        std::cerr << "  got empty string" << std::endl;
+        std::cerr << "  index " << i << ": got string of size "
+                  << returned_ref.len << ", but expected size "
+                  << expected_ref.len << std::endl;
       }
       return false;
     }
-    int expected_num_strings = GetStringCount(data_.raw);
-    int returned_num_strings = GetStringCount(tensor.data.raw);
-    if (expected_num_strings != returned_num_strings) {
+    if (strncmp(expected_ref.str, returned_ref.str, returned_ref.len) != 0) {
       if (verbose) {
-        std::cerr << "  string count differ: got " << returned_num_strings
-                  << ", but expected " << expected_num_strings << std::endl;
+        std::cerr << "  index " << i << ": strings are different" << std::endl;
       }
       return false;
     }
-    for (int i = 0; i < returned_num_strings; ++i) {
-      auto expected_ref = GetString(data_.raw, i);
-      auto returned_ref = GetString(tensor.data.raw, i);
-      if (expected_ref.len != returned_ref.len) {
-        if (verbose) {
-          std::cerr << "  index " << i << ": got string of size "
-                    << returned_ref.len << ", but expected size "
-                    << expected_ref.len << std::endl;
-        }
-        return false;
-      }
-      if (strncmp(expected_ref.str, returned_ref.str, returned_ref.len) != 0) {
-        if (verbose) {
-          std::cerr << "  index " << i << ": strings are different"
-                    << std::endl;
-        }
-        return false;
-      }
-    }
-
-    return true;
   }
 
-  TfLitePtrUnion data_;
-  size_t num_elements_;
-};
+  return true;
+}
+
+bool TfLiteDriver::Expectation::Check(bool verbose,
+                                      const TfLiteTensor& tensor) {
+  switch (tensor.type) {
+    case kTfLiteFloat32:
+      return TypedCheck<float>(verbose, tensor);
+    case kTfLiteInt32:
+      return TypedCheck<int32_t>(verbose, tensor);
+    case kTfLiteInt64:
+      return TypedCheck<int64_t>(verbose, tensor);
+    case kTfLiteUInt8:
+      return TypedCheck<uint8_t>(verbose, tensor);
+    case kTfLiteBool:
+      return TypedCheck<bool>(verbose, tensor);
+    case kTfLiteString:
+      return TypedCheck<string>(verbose, tensor);
+    default:
+      fprintf(stderr, "Unsupported type %d in Check\n", tensor.type);
+      return false;
+  }
+}
 
 TfLiteDriver::TfLiteDriver(bool use_nnapi, const string& delegate_name,
                            bool reference_kernel)
-- 
GitLab


From 5922eee396da936af1451c22e0433071f909eeaa Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 9 Jan 2019 09:15:17 -0800
Subject: [PATCH 0451/1095] Build file changes.

PiperOrigin-RevId: 228528921
---
 tensorflow/BUILD | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index 0b4498ef12..29d71c323a 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -385,7 +385,6 @@ package_group(
         "-//third_party/tensorflow/python/estimator",
         "//learning/deepmind/...",
         "//learning/meta_rank/...",
-        "//learning/pathways/...",  # While dataset C++ api requires internals
         "//tensorflow/...",
         "//tensorflow_estimator/contrib/...",
         "//tensorflow_fold/llgtm/...",
-- 
GitLab


From 1ee193a2563d51ee45b401f2cff91f6e480e21db Mon Sep 17 00:00:00 2001
From: Martin Wicke <wicke@google.com>
Date: Wed, 9 Jan 2019 09:33:54 -0800
Subject: [PATCH 0452/1095] Remove aux-bin from pip package, it's not needed
 any more.

PiperOrigin-RevId: 228531942
---
 tensorflow/tools/pip_package/MANIFEST.in          | 1 -
 tensorflow/tools/pip_package/build_pip_package.sh | 3 ---
 2 files changed, 4 deletions(-)

diff --git a/tensorflow/tools/pip_package/MANIFEST.in b/tensorflow/tools/pip_package/MANIFEST.in
index 272ff4735c..c304e8cf6e 100644
--- a/tensorflow/tools/pip_package/MANIFEST.in
+++ b/tensorflow/tools/pip_package/MANIFEST.in
@@ -6,7 +6,6 @@ recursive-include * *.so
 recursive-include * *.dll
 recursive-include * *.lib
 recursive-include * *.csv
-recursive-include tensorflow/aux-bin *
 recursive-include tensorflow/include/tensorflow *.h
 recursive-include tensorflow/include/Eigen *
 recursive-include tensorflow/include/external *
diff --git a/tensorflow/tools/pip_package/build_pip_package.sh b/tensorflow/tools/pip_package/build_pip_package.sh
index 439b5428b3..27815491d2 100755
--- a/tensorflow/tools/pip_package/build_pip_package.sh
+++ b/tensorflow/tools/pip_package/build_pip_package.sh
@@ -118,9 +118,6 @@ function prepare_src() {
         fi
       fi
     fi
-    mkdir "${TMPDIR}/tensorflow/aux-bin"
-    # Install toco as a binary in aux-bin.
-    cp bazel-bin/tensorflow/lite/python/tflite_convert ${TMPDIR}/tensorflow/aux-bin/
   fi
 
   # protobuf pip package doesn't ship with header files. Copy the headers
-- 
GitLab


From 2da8e8c60343d9e833f1c591c54ee2a27c240842 Mon Sep 17 00:00:00 2001
From: Tamara Norman <tamaranorman@google.com>
Date: Wed, 9 Jan 2019 10:23:54 -0800
Subject: [PATCH 0453/1095] Change so that error message received if attempting
 to do + between a python literal and tensor with incompatible dtype to come
 from the op

PiperOrigin-RevId: 228541386
---
 tensorflow/python/ops/math_ops.py      |  3 ++-
 tensorflow/python/ops/math_ops_test.py | 18 ++++++++++++++++++
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py
index 5bccf5493f..248d092538 100644
--- a/tensorflow/python/ops/math_ops.py
+++ b/tensorflow/python/ops/math_ops.py
@@ -813,7 +813,8 @@ def _OverrideBinaryOperatorHelper(func, op_name, clazz_object=ops.Tensor):
         return func(x, y, name=name)
       elif not isinstance(y, sparse_tensor.SparseTensor):
         try:
-          y = ops.convert_to_tensor(y, dtype=x.dtype.base_dtype, name="y")
+          y = ops.convert_to_tensor_v2(y, dtype_hint=x.dtype.base_dtype,
+                                       name="y")
         except TypeError:
           # If the RHS is not a tensor, it might be a tensor aware object
           # that can implement the operator with knowledge of itself
diff --git a/tensorflow/python/ops/math_ops_test.py b/tensorflow/python/ops/math_ops_test.py
index b27cf7208c..b4832e09c0 100644
--- a/tensorflow/python/ops/math_ops_test.py
+++ b/tensorflow/python/ops/math_ops_test.py
@@ -22,6 +22,7 @@ import numpy as np
 from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
@@ -664,5 +665,22 @@ class NextAfterTest(test_util.TensorFlowTestCase):
       self.assertAllEqual(math_ops.nextafter(one, two) - one, eps_const)
 
 
+class BinaryOpsTest(test_util.TensorFlowTestCase):
+
+  @test_util.run_in_graph_and_eager_modes
+  def testErrorReceivedIfDtypeMismatchFromOp(self):
+    if context.executing_eagerly():
+      error = errors_impl.InvalidArgumentError
+      error_message = (
+          r"cannot compute Add as input #0\(zero-based\) was expected to be a "
+          r"float tensor but is a int32 tensor \[Op:Add\] name: add/")
+    else:
+      error = TypeError
+      error_message = ("Input 'y' of 'Add' Op has type float32 that does not "
+                       "match type int32 of argument 'x'.")
+    with self.assertRaisesRegexp(error, error_message):
+      a = array_ops.ones([1], dtype=dtypes.int32) + 1.0
+      self.evaluate(a)
+
 if __name__ == "__main__":
   googletest.main()
-- 
GitLab


From 2c76c0a07fbf82d5bc8cfc81e688560d0b7e4537 Mon Sep 17 00:00:00 2001
From: Martin Wicke <wicke@google.com>
Date: Wed, 9 Jan 2019 10:24:39 -0800
Subject: [PATCH 0454/1095] Add dependency on google_pasta (freshly minted on
 pypi).

PiperOrigin-RevId: 228541538
---
 tensorflow/tools/pip_package/setup.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py
index 9567e0aff7..55b7046e30 100644
--- a/tensorflow/tools/pip_package/setup.py
+++ b/tensorflow/tools/pip_package/setup.py
@@ -51,6 +51,7 @@ REQUIRED_PACKAGES = [
     'absl-py >= 0.1.6',
     'astor >= 0.6.0',
     'gast >= 0.2.0',
+    'google_pasta >= 0.1.0',
     'keras_applications >= 1.0.6',
     'keras_preprocessing >= 1.0.5',
     'numpy >= 1.13.3',
-- 
GitLab


From 864fe25ff7363730ef81e402d6a10cc758df72cd Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Wed, 9 Jan 2019 10:52:22 -0800
Subject: [PATCH 0455/1095] Align Android gradle build files

Use Gradle 3.1.4 and modernize the TFLite example gradle files.

PiperOrigin-RevId: 228547238
---
 .../lite/examples/android/app/build.gradle    |  6 ++---
 tensorflow/lite/examples/android/build.gradle |  2 +-
 .../lite/g3doc/tfmobile/android_build.md      |  4 +--
 tensorflow/lite/g3doc/using_select_tf_ops.md  |  2 +-
 tensorflow/lite/java/demo/app/build.gradle    | 26 +++++++------------
 tensorflow/lite/java/demo/build.gradle        |  4 ++-
 .../gradle/wrapper/gradle-wrapper.properties  |  2 +-
 .../lite/java/ovic/demo/app/build.gradle      | 21 ++++++---------
 tensorflow/lite/java/ovic/demo/build.gradle   |  4 ++-
 .../gradle/wrapper/gradle-wrapper.properties  |  2 +-
 10 files changed, 33 insertions(+), 40 deletions(-)

diff --git a/tensorflow/lite/examples/android/app/build.gradle b/tensorflow/lite/examples/android/app/build.gradle
index e5f5c7efd1..b372afae19 100644
--- a/tensorflow/lite/examples/android/app/build.gradle
+++ b/tensorflow/lite/examples/android/app/build.gradle
@@ -2,7 +2,7 @@ apply plugin: 'com.android.application'
 
 android {
     compileSdkVersion 26
-    buildToolsVersion '26.0.2'
+    buildToolsVersion '27.0.3'
     defaultConfig {
         applicationId "org.tensorflow.lite.demo"
         minSdkVersion 15
@@ -45,6 +45,6 @@ project.ext.TMP_DIR   = project.buildDir.toString() + '/downloads'
 apply from: "download-models.gradle"
 
 dependencies {
-    compile fileTree(dir: 'libs', include: ['*.jar'])
-    compile 'org.tensorflow:tensorflow-lite:0.0.0-nightly'
+    implementation fileTree(dir: 'libs', include: ['*.jar'])
+    implementation 'org.tensorflow:tensorflow-lite:0.0.0-nightly'
 }
diff --git a/tensorflow/lite/examples/android/build.gradle b/tensorflow/lite/examples/android/build.gradle
index 7c79358e45..7c038ddd46 100644
--- a/tensorflow/lite/examples/android/build.gradle
+++ b/tensorflow/lite/examples/android/build.gradle
@@ -6,7 +6,7 @@ buildscript {
         jcenter()
     }
     dependencies {
-        classpath 'com.android.tools.build:gradle:3.0.1'
+        classpath 'com.android.tools.build:gradle:3.1.4'
 
         // NOTE: Do not place your application dependencies here; they belong
         // in the individual module build.gradle files
diff --git a/tensorflow/lite/g3doc/tfmobile/android_build.md b/tensorflow/lite/g3doc/tfmobile/android_build.md
index 2eb776d10c..f8c0243298 100644
--- a/tensorflow/lite/g3doc/tfmobile/android_build.md
+++ b/tensorflow/lite/g3doc/tfmobile/android_build.md
@@ -91,10 +91,10 @@ following lines to your Gradle build file:
         repositories {
             jcenter()
         }
-	}
+    }
 
     dependencies {
-        compile 'org.tensorflow:tensorflow-android:+'
+        implementation 'org.tensorflow:tensorflow-android:+'
     }
 
 This automatically downloads the latest stable version of TensorFlow as an AAR
diff --git a/tensorflow/lite/g3doc/using_select_tf_ops.md b/tensorflow/lite/g3doc/using_select_tf_ops.md
index aa51f58baa..269774a4b1 100644
--- a/tensorflow/lite/g3doc/using_select_tf_ops.md
+++ b/tensorflow/lite/g3doc/using_select_tf_ops.md
@@ -130,7 +130,7 @@ allprojects {
 }
 
 dependencies {
-    compile 'org.tensorflow:tensorflow-lite-with-select-tf-ops:0.1.100'
+    implementation 'org.tensorflow:tensorflow-lite-with-select-tf-ops:0.1.100'
 }
 ```
 
diff --git a/tensorflow/lite/java/demo/app/build.gradle b/tensorflow/lite/java/demo/app/build.gradle
index b8fc282cb1..8ea16a3417 100644
--- a/tensorflow/lite/java/demo/app/build.gradle
+++ b/tensorflow/lite/java/demo/app/build.gradle
@@ -2,7 +2,7 @@ apply plugin: 'com.android.application'
 
 android {
     compileSdkVersion 26
-    buildToolsVersion "26.0.1"
+    buildToolsVersion "27.0.3"
     defaultConfig {
         applicationId "android.example.com.tflitecamerademo"
         // Required by Camera2 API.
@@ -10,11 +10,6 @@ android {
         targetSdkVersion 26
         versionCode 1
         versionName "1.0"
-
-        // Remove this block.
-        jackOptions {
-            enabled true
-        }
     }
     lintOptions {
         abortOnError false
@@ -40,6 +35,7 @@ repositories {
         url 'https://google.bintray.com/tensorflow'
     }
 }
+
 allprojects {
     repositories {
         // Uncomment if you want to use a local repo.
@@ -48,20 +44,18 @@ allprojects {
     }
 }
 
-
-
 dependencies {
-    compile fileTree(dir: 'libs', include: ['*.jar'])
-    compile 'com.android.support:appcompat-v7:25.2.0'
-    compile 'com.android.support.constraint:constraint-layout:1.0.2'
-    compile 'com.android.support:design:25.2.0'
-    compile 'com.android.support:support-annotations:25.3.1'
-    compile 'com.android.support:support-v13:25.2.0'
+    implementation fileTree(dir: 'libs', include: ['*.jar'])
+    implementation 'com.android.support:appcompat-v7:25.2.0'
+    implementation 'com.android.support.constraint:constraint-layout:1.0.2'
+    implementation 'com.android.support:design:25.2.0'
+    implementation 'com.android.support:support-annotations:25.3.1'
+    implementation 'com.android.support:support-v13:25.2.0'
 
     // Build off of nightly TensorFlow Lite
-    compile 'org.tensorflow:tensorflow-lite:0.0.0-nightly'
+    implementation 'org.tensorflow:tensorflow-lite:0.0.0-nightly'
     // Use local TensorFlow library
-    // compile 'org.tensorflow:tensorflow-lite-local:0.0.0'
+    // implementation 'org.tensorflow:tensorflow-lite-local:0.0.0'
 }
 
 def targetFolder = "src/main/assets"
diff --git a/tensorflow/lite/java/demo/build.gradle b/tensorflow/lite/java/demo/build.gradle
index b78a0b86c9..a88b3fdc70 100644
--- a/tensorflow/lite/java/demo/build.gradle
+++ b/tensorflow/lite/java/demo/build.gradle
@@ -2,10 +2,11 @@
 
 buildscript {
     repositories {
+        google()
         jcenter()
     }
     dependencies {
-        classpath 'com.android.tools.build:gradle:2.3.1'
+        classpath 'com.android.tools.build:gradle:3.1.4'
 
         // NOTE: Do not place your application dependencies here; they belong
         // in the individual module build.gradle files
@@ -14,6 +15,7 @@ buildscript {
 
 allprojects {
     repositories {
+        google()
         jcenter()
     }
 }
diff --git a/tensorflow/lite/java/demo/gradle/wrapper/gradle-wrapper.properties b/tensorflow/lite/java/demo/gradle/wrapper/gradle-wrapper.properties
index fa7a38a0e4..9ff32fe2bb 100644
--- a/tensorflow/lite/java/demo/gradle/wrapper/gradle-wrapper.properties
+++ b/tensorflow/lite/java/demo/gradle/wrapper/gradle-wrapper.properties
@@ -3,4 +3,4 @@ distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-3.3-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.4-all.zip
diff --git a/tensorflow/lite/java/ovic/demo/app/build.gradle b/tensorflow/lite/java/ovic/demo/app/build.gradle
index 4f3a6cdb2f..77f568448a 100644
--- a/tensorflow/lite/java/ovic/demo/app/build.gradle
+++ b/tensorflow/lite/java/ovic/demo/app/build.gradle
@@ -2,18 +2,13 @@ apply plugin: 'com.android.application'
 
 android {
     compileSdkVersion 26
-    buildToolsVersion "26.0.1"
+    buildToolsVersion "27.0.3"
     defaultConfig {
         applicationId "android.example.com.ovicbenchmarker"
         minSdkVersion 15
         targetSdkVersion 26
         versionCode 1
         versionName "1.0"
-
-        // Remove this block.
-        jackOptions {
-            enabled true
-        }
     }
     lintOptions {
         abortOnError false
@@ -41,12 +36,12 @@ repositories {
 }
 
 dependencies {
-    compile fileTree(dir: 'libs', include: ['*.jar'])
-    compile 'com.android.support:appcompat-v7:25.2.0'
-    compile 'com.android.support.constraint:constraint-layout:1.0.2'
-    compile 'com.android.support:design:25.2.0'
-    compile 'com.android.support:support-annotations:25.3.1'
-    compile 'com.android.support:support-v13:25.2.0'
+    implementation fileTree(dir: 'libs', include: ['*.jar'])
+    implementation 'com.android.support:appcompat-v7:25.2.0'
+    implementation 'com.android.support.constraint:constraint-layout:1.0.2'
+    implementation 'com.android.support:design:25.2.0'
+    implementation 'com.android.support:support-annotations:25.3.1'
+    implementation 'com.android.support:support-v13:25.2.0'
 
-    compile 'org.tensorflow:tensorflow-lite:+'
+    implementation 'org.tensorflow:tensorflow-lite:+'
 }
diff --git a/tensorflow/lite/java/ovic/demo/build.gradle b/tensorflow/lite/java/ovic/demo/build.gradle
index b78a0b86c9..a88b3fdc70 100644
--- a/tensorflow/lite/java/ovic/demo/build.gradle
+++ b/tensorflow/lite/java/ovic/demo/build.gradle
@@ -2,10 +2,11 @@
 
 buildscript {
     repositories {
+        google()
         jcenter()
     }
     dependencies {
-        classpath 'com.android.tools.build:gradle:2.3.1'
+        classpath 'com.android.tools.build:gradle:3.1.4'
 
         // NOTE: Do not place your application dependencies here; they belong
         // in the individual module build.gradle files
@@ -14,6 +15,7 @@ buildscript {
 
 allprojects {
     repositories {
+        google()
         jcenter()
     }
 }
diff --git a/tensorflow/lite/java/ovic/demo/gradle/wrapper/gradle-wrapper.properties b/tensorflow/lite/java/ovic/demo/gradle/wrapper/gradle-wrapper.properties
index fa7a38a0e4..9ff32fe2bb 100644
--- a/tensorflow/lite/java/ovic/demo/gradle/wrapper/gradle-wrapper.properties
+++ b/tensorflow/lite/java/ovic/demo/gradle/wrapper/gradle-wrapper.properties
@@ -3,4 +3,4 @@ distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-3.3-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.4-all.zip
-- 
GitLab


From 65cfc6a54bc76912dd16c25b9e13610c4359a6d9 Mon Sep 17 00:00:00 2001
From: Blake Hechtman <blakehechtman@google.com>
Date: Wed, 9 Jan 2019 11:13:04 -0800
Subject: [PATCH 0456/1095] [XLA] Fix Dot operation semantics.

PiperOrigin-RevId: 228551710
---
 tensorflow/compiler/xla/g3doc/operation_semantics.md | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/tensorflow/compiler/xla/g3doc/operation_semantics.md b/tensorflow/compiler/xla/g3doc/operation_semantics.md
index 7377ed729b..59ba9bb658 100644
--- a/tensorflow/compiler/xla/g3doc/operation_semantics.md
+++ b/tensorflow/compiler/xla/g3doc/operation_semantics.md
@@ -871,9 +871,7 @@ DotGeneral performs the sum of products over contracting dimensions specified
 in 'dimension_numbers'.
 
 Associated contracting dimension numbers from the 'lhs' and 'rhs' do not need
-to be the same, but must be listed in the same order in both
-'lhs/rhs_contracting_dimensions' arrays and have the same dimension sizes.
-There must be exactly one contracting dimension on both 'lhs' and 'rhs'.
+to be the same and but must have the same dimension sizes.
 
 Example with contracting dimension numbers:
 
@@ -892,10 +890,8 @@ DotGeneral(lhs, rhs, dnums) -> { {6.0, 12.0},
 {15.0, 30.0} }
 ```
 
-Associated batch dimension numbers from the 'lhs' and 'rhs' must have the same
-dimension number, must be listed in the same order in both arrays, must
-have the same dimension sizes, and must be ordered before contracting and
-non-contracting/non-batch dimension numbers.
+Associated batch dimension numbers from the 'lhs' and 'rhs' must
+have the same dimension sizes.
 
 Example with batch dimension numbers (batch size 2, 2x2 matrices):
 
-- 
GitLab


From d987224e93ba153201a7c3958594396af56a255d Mon Sep 17 00:00:00 2001
From: Andiry Xu <andiryxu@google.com>
Date: Wed, 9 Jan 2019 11:24:20 -0800
Subject: [PATCH 0457/1095] Typo fix

PiperOrigin-RevId: 228554042
---
 tensorflow/core/grappler/costs/graph_properties.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/grappler/costs/graph_properties.cc b/tensorflow/core/grappler/costs/graph_properties.cc
index cfbd340f08..1ed96ca12a 100644
--- a/tensorflow/core/grappler/costs/graph_properties.cc
+++ b/tensorflow/core/grappler/costs/graph_properties.cc
@@ -1463,9 +1463,9 @@ class SymbolicShapeRefiner {
       // Due to the cost of EvaluateNode(), we run it only for certain op types
       // (white listed) and small integer tensors.
 
-      const int max_elelment_size = 17;  // Max up to 4x4 matrix or similar.
+      const int max_element_size = 17;  // Max up to 4x4 matrix or similar.
       if (AllOutputValuesKnown(c) || !AllInputValuesKnown(c) ||
-          !ShouldUpdateOutputValues(c, max_elelment_size)) {
+          !ShouldUpdateOutputValues(c, max_element_size)) {
         return Status::OK();
       }
       UpdateOutputValues(node, c).IgnoreError();  // This is optional.
-- 
GitLab


From 9e4f493abffbcb1efc7ea2b1e9bce0afff8122ce Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Wed, 9 Jan 2019 11:31:52 -0800
Subject: [PATCH 0458/1095] Reduce max_workspace_size_bytes so the GPU memory
 allocation doesn't exceed the limit.

PiperOrigin-RevId: 228555498
---
 tensorflow/contrib/tensorrt/test/quantization_mnist_test.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py b/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py
index e7d6ec4ad3..79dde2a6b5 100644
--- a/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py
+++ b/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py
@@ -144,7 +144,10 @@ class QuantizationAwareTrainingMNISTTest(test_util.TensorFlowTestCase):
           outputs=[OUTPUT_NODE_NAME],
           max_batch_size=max_batch_size,
           precision_mode='INT8',
-          max_workspace_size_bytes=4096 << 19,
+          # There is a 2GB GPU memory limit for each test, so we set
+          # max_workspace_size_bytes to 256MB to leave enough room for TF
+          # runtime to allocate GPU memory.
+          max_workspace_size_bytes=1 << 28,
           minimum_segment_size=2,
           use_calibration=False,
       )
-- 
GitLab


From 06974e29b1e9f4b4b5120acca2889bfa3bcf055d Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Wed, 9 Jan 2019 11:39:26 -0800
Subject: [PATCH 0459/1095] Internal Cleanup.

PiperOrigin-RevId: 228557086
---
 tensorflow/python/keras/optimizer_v2/BUILD    |   4 -
 tensorflow/python/keras/optimizer_v2/nadam.py |   3 +-
 .../keras/optimizer_v2/optimizer_v2_test.py   | 115 ++++++++++--------
 3 files changed, 62 insertions(+), 60 deletions(-)

diff --git a/tensorflow/python/keras/optimizer_v2/BUILD b/tensorflow/python/keras/optimizer_v2/BUILD
index 2b990c9334..da757edc74 100644
--- a/tensorflow/python/keras/optimizer_v2/BUILD
+++ b/tensorflow/python/keras/optimizer_v2/BUILD
@@ -177,11 +177,7 @@ py_test(
     srcs = ["optimizer_v2_test.py"],
     shard_count = 8,
     tags = [
-        "manual",
-        "no_gpu",
-        "no_oss",
         "no_windows",
-        "notap",
     ],
     deps = [
         ":optimizer_v2",
diff --git a/tensorflow/python/keras/optimizer_v2/nadam.py b/tensorflow/python/keras/optimizer_v2/nadam.py
index 627258f455..7141c6a079 100644
--- a/tensorflow/python/keras/optimizer_v2/nadam.py
+++ b/tensorflow/python/keras/optimizer_v2/nadam.py
@@ -84,8 +84,7 @@ class Nadam(optimizer_v2.OptimizerV2):
     """
 
     # Backwards compatiblity with keras NAdam optimizer.
-    if 'schedule_decay' in kwargs:
-      kwargs['decay'] = kwargs.pop('schedule_decay', 0.004)
+    kwargs['decay'] = kwargs.pop('schedule_decay', 0.004)
     super(Nadam, self).__init__(name, **kwargs)
     self._set_hyper('learning_rate', kwargs.get('lr', learning_rate))
     self._set_hyper('decay', self._initial_decay)
diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
index b41e4aa064..93a6a7e85e 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
@@ -18,10 +18,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import os
-import tempfile
-
-from absl.testing import parameterized
 import numpy as np
 
 from tensorflow.python import keras
@@ -33,22 +29,25 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.keras import backend
 from tensorflow.python.keras import callbacks
+from tensorflow.python.keras import keras_parameterized
 from tensorflow.python.keras import optimizers
 from tensorflow.python.keras import testing_utils
 from tensorflow.python.keras.engine import input_layer
-from tensorflow.python.keras.engine import saving
 from tensorflow.python.keras.engine import sequential
 from tensorflow.python.keras.engine import training
 from tensorflow.python.keras.layers import core
+from tensorflow.python.keras.optimizer_v2 import adadelta
+from tensorflow.python.keras.optimizer_v2 import adagrad
 from tensorflow.python.keras.optimizer_v2 import adam
+from tensorflow.python.keras.optimizer_v2 import adamax
 from tensorflow.python.keras.optimizer_v2 import gradient_descent
-from tensorflow.python.keras.optimizer_v2 import optimizer_v2
+from tensorflow.python.keras.optimizer_v2 import nadam
+from tensorflow.python.keras.optimizer_v2 import rmsprop
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import clip_ops
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variables
-from tensorflow.python.platform import gfile
 from tensorflow.python.platform import test
 from tensorflow.python.training import momentum
 from tensorflow.python.training import training_util
@@ -492,23 +491,20 @@ class OptimizerTest(test.TestCase):
     opt.iterations = global_step
     var = resource_variable_ops.ResourceVariable([1.0, 2.0],
                                                  dtype=dtypes.float32)
+    self.evaluate(variables.global_variables_initializer())
+    init_step_value = self.evaluate(global_step)
     loss = lambda: 3 * var
     opt_op = opt.minimize(loss, [var])
     self.evaluate(variables.global_variables_initializer())
-    init_step_value = self.evaluate(global_step)
     self.evaluate(opt_op)
     new_step_value = self.evaluate(global_step)
     self.assertEqual(new_step_value, init_step_value + 1)
 
 
-class OptimizersCompatibilityTest(test.TestCase, parameterized.TestCase):
+@keras_parameterized.run_with_all_model_types
+class OptimizersCompatibilityTest(keras_parameterized.TestCase):
 
-  @parameterized.named_parameters(
-      ('adadelta', 'adadelta', True), ('adagrad', 'adagrad', True),
-      ('adam', 'adam', True), ('adamax', 'adamax', True),
-      ('nadam', 'nadam', True), ('momentum', 'momentum', True),
-      ('rmsprop', 'rmsprop', True), ('sgd', 'sgd', False))
-  def testOptimizersCompatibility(self, opt_str, test_weights):
+  def _testOptimizersCompatibility(self, opt_v1, opt_v2, test_weights=True):
     np.random.seed(1331)
     with self.cached_session():
       train_samples = 20
@@ -522,43 +518,63 @@ class OptimizersCompatibilityTest(test.TestCase, parameterized.TestCase):
       y = keras.utils.to_categorical(y)
 
       num_hidden = 5
-      model = testing_utils.get_small_sequential_mlp(
+      model_v1 = testing_utils.get_small_sequential_mlp(
           num_hidden=num_hidden, num_classes=num_classes, input_dim=input_dim)
+      model_v1.compile(opt_v1, loss='categorical_crossentropy', metrics=[])
+      model_v1.fit(x, y, batch_size=5, epochs=1)
 
-      old_mode = os.environ.get('TF2_BEHAVIOR', None)
-      # Disable tf2 to create V1 optimizer.
-      disable_tf2()
-      if opt_str == 'momentum':
-        opt_v1 = optimizers.SGD(momentum=0.9)
-      else:
-        opt_v1 = optimizers.get(opt_str)
-
-      # Test compile and fit with v1 optimizer.
-      model.compile(opt_v1, loss='categorical_crossentropy', metrics=[])
-      model.fit(x, y, batch_size=5, epochs=1)
-      model_dir = tempfile.mkdtemp()
-      gfile.MakeDirs(model_dir)
-      file_name = os.path.join(model_dir, 'model.h5')
-      model.save(file_name)
-
-      enable_tf2()
-      # Test load and fit with v2 optimizer.
-      model_2 = saving.load_model(file_name)
-      opt_v2 = model_2.optimizer
-      self.assertIsInstance(opt_v2, optimizer_v2.OptimizerV2)
-      # set_weights is called inside load_model but exception is swallowed,
-      # this call checks the weights can be set correctly.
+      model_v2 = testing_utils.get_small_sequential_mlp(
+          num_hidden=num_hidden, num_classes=num_classes, input_dim=input_dim)
+      model_v2.set_weights(model_v1.get_weights())
+      model_v2.compile(opt_v2, loss='categorical_crossentropy', metrics=[])
+      model_v2._make_train_function()
       if test_weights:
         opt_v2.set_weights(opt_v1.get_weights())
 
-      hist_1 = model.fit(x, y, batch_size=5, epochs=1, shuffle=False)
-      hist_2 = model_2.fit(x, y, batch_size=5, epochs=1, shuffle=False)
-      self.assertAllClose(model.get_weights(), model_2.get_weights())
-      self.assertAllClose(model.get_weights(), model_2.get_weights())
+      hist_1 = model_v1.fit(x, y, batch_size=5, epochs=1, shuffle=False)
+      hist_2 = model_v2.fit(x, y, batch_size=5, epochs=1, shuffle=False)
+      self.assertAllClose(model_v1.get_weights(), model_v2.get_weights())
       self.assertAllClose(hist_1.history['loss'], hist_2.history['loss'])
 
-      if old_mode is not None:
-        os.environ['TF2_BEHAVIOR'] = old_mode
+  def testAdadeltaCompatibility(self):
+    opt_v1 = optimizers.Adadelta(lr=0.01)
+    opt_v2 = adadelta.Adadelta(learning_rate=0.01)
+    self._testOptimizersCompatibility(opt_v1, opt_v2)
+
+  def testAdagradCompatibility(self):
+    opt_v1 = optimizers.Adagrad(lr=0.01)
+    opt_v2 = adagrad.Adagrad(learning_rate=0.01)
+    self._testOptimizersCompatibility(opt_v1, opt_v2)
+
+  def testAdamCompatibility(self):
+    opt_v1 = optimizers.Adam()
+    opt_v2 = adam.Adam()
+    self._testOptimizersCompatibility(opt_v1, opt_v2)
+
+  def testAdamaxCompatibility(self):
+    opt_v1 = optimizers.Adamax(lr=0.01)
+    opt_v2 = adamax.Adamax(learning_rate=0.01)
+    self._testOptimizersCompatibility(opt_v1, opt_v2)
+
+  def testNadamCompatibility(self):
+    opt_v1 = optimizers.Nadam(lr=0.001)
+    opt_v2 = nadam.Nadam(learning_rate=0.001)
+    self._testOptimizersCompatibility(opt_v1, opt_v2)
+
+  def testMomentumCompatibility(self):
+    opt_v1 = optimizers.SGD(lr=0.01, momentum=0.9)
+    opt_v2 = gradient_descent.SGD(learning_rate=0.01, momentum=0.9)
+    self._testOptimizersCompatibility(opt_v1, opt_v2)
+
+  def testRMSpropCompatibility(self):
+    opt_v1 = optimizers.RMSprop()
+    opt_v2 = rmsprop.RMSprop()
+    self._testOptimizersCompatibility(opt_v1, opt_v2)
+
+  def testSGDCompatibility(self):
+    opt_v1 = optimizers.SGD(lr=0.01)
+    opt_v2 = gradient_descent.SGD(learning_rate=0.01)
+    self._testOptimizersCompatibility(opt_v1, opt_v2, False)
 
   def testNumericEquivalenceForNesterovMomentum(self):
     np.random.seed(1331)
@@ -636,15 +652,6 @@ class OptimizersCompatibilityTest(test.TestCase, parameterized.TestCase):
       self.assertAllClose(hist_k_v1.history['loss'], hist_k_v2.history['loss'])
 
 
-def disable_tf2():
-  if 'TF2_BEHAVIOR' in os.environ:
-    del os.environ['TF2_BEHAVIOR']
-
-
-def enable_tf2():
-  os.environ['TF2_BEHAVIOR'] = 'enabled'
-
-
 # Note: These tests are kept in a separate class to avoid bugs in some
 # distributions of Python that break AutoGraph which is used by tf.function.
 class OptimizerWithFunctionTest(test.TestCase):
-- 
GitLab


From f5f079ca12c376eb2b86389973661f27054136af Mon Sep 17 00:00:00 2001
From: Karim Nosir <karimnosseir@google.com>
Date: Wed, 9 Jan 2019 11:44:32 -0800
Subject: [PATCH 0460/1095] Add missing include needed for windows build. Fixes
 #24521

PiperOrigin-RevId: 228558062
---
 tensorflow/core/util/stats_calculator.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/core/util/stats_calculator.h b/tensorflow/core/util/stats_calculator.h
index e191737bb2..5005ee08a4 100644
--- a/tensorflow/core/util/stats_calculator.h
+++ b/tensorflow/core/util/stats_calculator.h
@@ -18,6 +18,7 @@ limitations under the License.
 
 #include <stdlib.h>
 
+#include <algorithm>
 #include <cmath>
 #include <limits>
 #include <map>
-- 
GitLab


From 245925078f574983593e42154091e07c311e30bd Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Wed, 9 Jan 2019 11:45:24 -0800
Subject: [PATCH 0461/1095] Remove the implementation of TPU Strategy
 initialize and finalize

PiperOrigin-RevId: 228558184
---
 .../contrib/distribute/python/tpu_strategy.py | 25 -------------------
 1 file changed, 25 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index f498f58b25..00360bf996 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -51,9 +51,6 @@ from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.util import nest
 
 
-_TPU_INITIALIZE_SYSTEM_COLLECTION = "TPU_STRATEGY_INITIALIZE"
-
-
 def initialize_tpu_system(cluster_resolver=None):
   """Initialize the TPU devices in a separate session and graph.
 
@@ -463,28 +460,6 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
     """
     initialize_tpu_system(self._tpu_cluster_resolver)
 
-  def _initialize(self):
-    if context.executing_eagerly():
-      # TODO(priyag): Add appopriate call here when eager is supported for TPUs.
-      raise NotImplementedError("Eager mode not supported in TPUStrategy.")
-    else:
-      # TODO(jhseu): We need this hack because DistributionStrategies must be
-      # pickleable for copy.deepcopy(). Remove when initialize_system goes away.
-      graph = ops.get_default_graph()
-      tpu_init = graph.get_collection(_TPU_INITIALIZE_SYSTEM_COLLECTION)
-      if tpu_init:
-        return tpu_init
-      graph.add_to_collection(_TPU_INITIALIZE_SYSTEM_COLLECTION,
-                              tpu.initialize_system())
-      return graph.get_collection(_TPU_INITIALIZE_SYSTEM_COLLECTION)
-
-  def _finalize(self):
-    if context.executing_eagerly():
-      # TODO(priyag): Add appopriate call here when eager is supported for TPUs.
-      raise NotImplementedError("Eager mode not supported in TPUStrategy.")
-    else:
-      return []
-
   def _create_variable(self, next_creator, *args, **kwargs):
     """Create a TPUMirroredVariable. See `DistributionStrategy.scope`."""
     colocate_with = kwargs.pop("colocate_with", None)
-- 
GitLab


From bebf6b3d5d11e0ae09edab0a3700e0e4000ac842 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 9 Jan 2019 11:57:48 -0800
Subject: [PATCH 0462/1095] Scope GBDT resource variables.

PiperOrigin-RevId: 228560356
---
 .../contrib/boosted_trees/python/ops/model_ops.py      | 10 +++++-----
 .../python/training/functions/gbdt_batch.py            |  5 +++--
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/tensorflow/contrib/boosted_trees/python/ops/model_ops.py b/tensorflow/contrib/boosted_trees/python/ops/model_ops.py
index fca22c71a8..c3685b54e2 100644
--- a/tensorflow/contrib/boosted_trees/python/ops/model_ops.py
+++ b/tensorflow/contrib/boosted_trees/python/ops/model_ops.py
@@ -62,8 +62,8 @@ class TreeEnsembleVariableSavable(saver.BaseSaverBuilder.SaveableObject):
         saver.BaseSaverBuilder.SaveSpec(ensemble_config, slice_spec,
                                         name + "_config"),
     ]
-    super(TreeEnsembleVariableSavable,
-          self).__init__(tree_ensemble_handle, specs, name)
+    super(TreeEnsembleVariableSavable, self).__init__(tree_ensemble_handle,
+                                                      specs, name)
     self._tree_ensemble_handle = tree_ensemble_handle
     self._create_op = create_op
 
@@ -115,7 +115,7 @@ class TreeEnsembleVariable(tracking.TrackableResource):
 
   def _gather_saveables_for_checkpoint(self):
     return {
-        "tree_ensemble_variable":
+        self.resource_handle.op.name + "/tree_ensemble_variable":
             functools.partial(
                 TreeEnsembleVariableSavable,
                 tree_ensemble_handle=self.resource_handle,
@@ -131,8 +131,8 @@ def tree_ensemble_variable(stamp_token,
 
   Args:
     stamp_token: The initial stamp token value for the ensemble resource.
-    tree_ensemble_config: A `Tensor` of type `string`.
-      Serialized proto of the tree ensemble.
+    tree_ensemble_config: A `Tensor` of type `string`. Serialized proto of the
+      tree ensemble.
     name: A name for the ensemble variable.
     container: An optional `string`. Defaults to `""`.
 
diff --git a/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py b/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py
index a5951fb737..e78ec476ab 100644
--- a/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py
+++ b/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py
@@ -566,9 +566,10 @@ class GradientBoostedDecisionTreeModel(object):
     # Determine if ensemble is colocated with the inputs.
     if self._ensemble_handle.device != input_deps[0].device:
       # Create a local ensemble and get its local stamp.
-      with ops.name_scope("local_ensemble", "TreeEnsembleVariable") as name:
+      with ops.name_scope("local_ensemble", "TreeEnsembleVariable"):
         local_ensemble_handle = (
-            gen_model_ops.decision_tree_ensemble_resource_handle_op(name=name))
+            gen_model_ops.decision_tree_ensemble_resource_handle_op(
+                self._ensemble_handle.op.name + "/local_ensemble"))
         create_op = gen_model_ops.create_tree_ensemble_variable(
             local_ensemble_handle, stamp_token=-1, tree_ensemble_config="")
         with ops.control_dependencies([create_op]):
-- 
GitLab


From 8d9872cebb57219acbfaad57e5203b3a407ac28e Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Wed, 9 Jan 2019 11:57:49 -0800
Subject: [PATCH 0463/1095] Fix typos in comments.

PiperOrigin-RevId: 228560360
---
 .../compiler/xla/service/gpu/cudnn_conv_algorithm_picker.cc     | 2 +-
 tensorflow/compiler/xla/service/gpu/cudnn_conv_runner.h         | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/cudnn_conv_algorithm_picker.cc b/tensorflow/compiler/xla/service/gpu/cudnn_conv_algorithm_picker.cc
index 057d0d2364..309b0aca64 100644
--- a/tensorflow/compiler/xla/service/gpu/cudnn_conv_algorithm_picker.cc
+++ b/tensorflow/compiler/xla/service/gpu/cudnn_conv_algorithm_picker.cc
@@ -250,7 +250,7 @@ CudnnConvAlgorithmPicker::PickBestAlgorithm(
     VLOG(3) << "Trying algorithm " << AlgorithmToString(alg) << " for "
             << instr->ToString();
 
-    // Use assignment insetad of brace-list to make GCC 4.9 happy.
+    // Use assignment instead of brace-list to make GCC 4.9 happy.
     RunConvOptions options;
     options.profile_result = &profile_result;
     options.algo_override = alg;
diff --git a/tensorflow/compiler/xla/service/gpu/cudnn_conv_runner.h b/tensorflow/compiler/xla/service/gpu/cudnn_conv_runner.h
index 7cc325bce3..25b2461ca6 100644
--- a/tensorflow/compiler/xla/service/gpu/cudnn_conv_runner.h
+++ b/tensorflow/compiler/xla/service/gpu/cudnn_conv_runner.h
@@ -32,7 +32,7 @@ struct RunConvOptions {
   // Nullable output-parameter pointer for profiling results.
   se::dnn::ProfileResult* profile_result = nullptr;
 
-  // Use this algorithm, instead the one from the instrcution.
+  // Use this algorithm, instead of the one from the instruction.
   absl::optional<se::dnn::AlgorithmDesc> algo_override;
 };
 
-- 
GitLab


From a14adaa2329fb46cb472b949ee52546c2516a21e Mon Sep 17 00:00:00 2001
From: Reed Wanderman-Milne <reedwm@google.com>
Date: Wed, 9 Jan 2019 12:00:29 -0800
Subject: [PATCH 0464/1095] Automated rollback of changelist 205730535

PiperOrigin-RevId: 228560984
---
 tensorflow/core/BUILD                         | 11 ----
 .../core/common_runtime/gpu/gpu_device.cc     | 51 +------------------
 .../core/common_runtime/gpu/gpu_device.h      | 11 ----
 .../gpu/gpu_device_kernel_check.cu.cc         | 37 --------------
 .../gpu/gpu_device_kernel_check.h             | 32 ------------
 5 files changed, 1 insertion(+), 141 deletions(-)
 delete mode 100644 tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.cu.cc
 delete mode 100644 tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.h

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 6a842a432f..e6af9211b5 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -87,7 +87,6 @@ load(
     "tf_gen_op_libs",
     "tf_generate_proto_text_sources",
     "tf_genrule_cmd_append_to_srcs",
-    "tf_gpu_kernel_library",
     "tf_opts_nortti_if_android",
     "transitive_hdrs",
 )
@@ -3163,15 +3162,6 @@ cc_library(
     ],
 )
 
-tf_gpu_kernel_library(
-    name = "gpu_device_kernel_check",
-    srcs = ["common_runtime/gpu/gpu_device_kernel_check.cu.cc"],
-    hdrs = ["common_runtime/gpu/gpu_device_kernel_check.h"],
-    deps = [
-        "//tensorflow/core:stream_executor",
-    ],
-)
-
 GPU_RUNTIME_HEADERS = [
     "common_runtime/gpu/cuda_host_allocator.h",
     "common_runtime/gpu/gpu_bfc_allocator.h",
@@ -3210,7 +3200,6 @@ tf_cuda_library(
         ":core_cpu_lib",
         ":framework",
         ":framework_internal",
-        ":gpu_device_kernel_check",
         ":gpu_id_impl",
         ":gpu_init_impl",
         ":gpu_lib",
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device.cc b/tensorflow/core/common_runtime/gpu/gpu_device.cc
index 891b6e0e2a..010fdff4e9 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_device.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_device.cc
@@ -31,7 +31,6 @@ limitations under the License.
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/common_runtime/device_factory.h"
-#include "tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.h"
 #include "tensorflow/core/common_runtime/gpu/gpu_event_mgr.h"
 #include "tensorflow/core/common_runtime/gpu/gpu_id.h"
 #include "tensorflow/core/common_runtime/gpu/gpu_id_manager.h"
@@ -389,7 +388,7 @@ Status BaseGPUDevice::Init(const SessionOptions& options) {
     }
   }
 
-  return CheckGPU();
+  return Status::OK();
 }
 
 bool BaseGPUDevice::RequiresRecordingAccessedTensors() const {
@@ -908,54 +907,6 @@ Allocator* BaseGPUDevice::GetScopedAllocator(AllocatorAttributes attr,
   return gpu_allocator_;
 }
 
-Status BaseGPUDevice::CheckGPU() {
-  se::Stream* stream = tensorflow_gpu_device_info()->stream;
-  TF_RETURN_IF_ERROR(stream->BlockHostUntilDone());
-  Tensor device_tensor(gpu_allocator_, DT_FLOAT, {});
-  if (!device_tensor.IsInitialized()) {
-    return errors::ResourceExhausted("Failed to allocate ", sizeof(float),
-                                     " bytes on the GPU for initialization "
-                                     "checks");
-  }
-  float* val_dev = device_tensor.scalar<float>().data();
-  const cudaStream_t cu_stream = *reinterpret_cast<const cudaStream_t*>(
-      stream->implementation()->GpuStreamMemberHack());
-  {
-    se::cuda::ScopedActivateExecutorContext scoped_activation{stream->parent()};
-    run_test_kernel(val_dev, cu_stream);
-    // We have to use the CUDA runtime function cudaPeekAtLastError here,
-    // because 'stream' does not provide a way to check if a kernel launch
-    // succeeds. Calling 'stream->BlockHostUntilDone()', which internally calls
-    // 'cuCtxSynchronize()', does not catch all kernel launch errors.
-    cudaError_t cuda_error = cudaPeekAtLastError();
-    if (cuda_error == cudaSuccess) {
-      cuda_error = cudaDeviceSynchronize();
-    }
-    TF_RETURN_IF_ERROR(CudaErrorToStatus(cuda_error, *stream));
-  }
-
-  float val_host = 0.;
-  stream->ThenMemcpy(&val_host, se::DeviceMemoryBase(val_dev, sizeof(float)),
-                     sizeof(float));
-  TF_RETURN_IF_ERROR(stream->BlockHostUntilDone());
-  if (val_host != 12345.) {
-    return errors::Internal(
-        "GPU kernel for initialization returned wrong value: ", val_host);
-  }
-  return Status::OK();
-}
-
-Status BaseGPUDevice::CudaErrorToStatus(cudaError_t cuda_error,
-                                        const se::Stream& stream) {
-  if (cuda_error != cudaSuccess) {
-    return errors::Internal(
-        "Failed to run GPU kernel for the initialization check. Received "
-        "error ",
-        cudaGetErrorName(cuda_error), " after running GPU kernel.");
-  }
-  return Status::OK();
-}
-
 const int BaseGPUDeviceFactory::InterconnectMap::kSameDeviceStrength = 1000;
 const int BaseGPUDeviceFactory::InterconnectMap::kStreamExecutorStrength = 1;
 
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device.h b/tensorflow/core/common_runtime/gpu/gpu_device.h
index 86622dad49..d002d02c51 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_device.h
+++ b/tensorflow/core/common_runtime/gpu/gpu_device.h
@@ -26,7 +26,6 @@ limitations under the License.
 #include <vector>
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
-#include "cuda/include/cuda_runtime_api.h"
 #include "tensorflow/core/common_runtime/device_factory.h"
 #include "tensorflow/core/common_runtime/gpu/gpu_event_mgr.h"
 #include "tensorflow/core/common_runtime/gpu/gpu_id.h"
@@ -122,12 +121,6 @@ class BaseGPUDevice : public LocalDevice {
   se::StreamExecutor* executor_;  // not owned
   std::unique_ptr<ScopedAllocatorMgr> scoped_allocator_mgr_;
 
-  // Returns a Status corresponding to a cudaError_t. The CUDA error must have
-  // been obtained from a CUDA kernel launch used to check if the GPU is
-  // initialized properly.
-  virtual Status CudaErrorToStatus(cudaError_t cuda_error,
-                                   const se::Stream& stream);
-
  private:
   struct StreamGroup {
     se::Stream* compute = nullptr;
@@ -168,10 +161,6 @@ class BaseGPUDevice : public LocalDevice {
   Status MaybeCopyTensorToGPU(const AllocatorAttributes& alloc_attrs,
                               const Tensor& from, Tensor* to,
                               StatusCallback done);
-
-  // Checks that the GPU is capable of doing work, by running a test kernel on
-  // it.
-  Status CheckGPU();
 };
 
 class BaseGPUDeviceFactory : public DeviceFactory {
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.cu.cc b/tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.cu.cc
deleted file mode 100644
index 017565195b..0000000000
--- a/tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.cu.cc
+++ /dev/null
@@ -1,37 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#if GOOGLE_CUDA
-
-#include "tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.h"
-#include "tensorflow/stream_executor/cuda/cuda_activation.h"
-
-namespace {
-__global__ void test_kernel(float* val) {
-  if (blockIdx.x == 0 && threadIdx.x == 0) {
-    (*val) = 12345.;
-  }
-}
-}  // namespace
-
-namespace tensorflow {
-
-void run_test_kernel(float* val, cudaStream_t cu_stream) {
-  test_kernel<<<1, 1, 0, cu_stream>>>(val);
-}
-
-}  // namespace tensorflow
-
-#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.h b/tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.h
deleted file mode 100644
index 064fb7a49f..0000000000
--- a/tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.h
+++ /dev/null
@@ -1,32 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_DEVICE_KERNEL_CHECK_H_
-#define TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_DEVICE_KERNEL_CHECK_H_
-
-#if GOOGLE_CUDA
-
-#include "tensorflow/core/platform/stream_executor.h"
-
-namespace tensorflow {
-
-// Runs a GPU kernel to test that it functions correctly. Sets 'val' to 12345.
-void run_test_kernel(float* val, cudaStream_t cu_stream);
-
-}  // namespace tensorflow
-
-#endif  // GOOGLE_CUDA
-
-#endif  // TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_DEVICE_KERNEL_CHECK_H_
-- 
GitLab


From 4fe7980e6b33d075a4a9fcfa40310af85f071cff Mon Sep 17 00:00:00 2001
From: Smit Hinsu <hinsu@google.com>
Date: Wed, 9 Jan 2019 12:09:12 -0800
Subject: [PATCH 0465/1095] Automated rollback of commit
 7009febfed6721cbf664b1ccfd44358392c3d5ca

PiperOrigin-RevId: 228562832
---
 tensorflow/contrib/tensorrt/BUILD | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index c17dea02f1..3d34b91a74 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -551,7 +551,6 @@ cuda_py_test(
     ],
     tags = [
         "no_cuda_on_cpu_tap",
-        "no_oss",  # TODO(b/121194394): re-enable in OSS after OOM is fixed.
         "no_pip",
         "no_tap",  # It is not able to download the mnist data.
         "no_windows",
-- 
GitLab


From 57d4745d24195699f93d1674171a03345269d2c1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 9 Jan 2019 12:10:02 -0800
Subject: [PATCH 0466/1095] Compute reshape permutation statically.

Aside from presumably being more efficient generally, this enables using the routines in this file inside of XLA. (which currently throws an error
`Input 0 to InvertPermutation operator must be a compile-time constant.`)

PiperOrigin-RevId: 228563007
---
 tensorflow/python/ops/linalg/linear_operator_util.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/ops/linalg/linear_operator_util.py b/tensorflow/python/ops/linalg/linear_operator_util.py
index 54d04e4a70..6c18943dab 100644
--- a/tensorflow/python/ops/linalg/linear_operator_util.py
+++ b/tensorflow/python/ops/linalg/linear_operator_util.py
@@ -481,9 +481,9 @@ def _reshape_for_efficiency(a,
 
   # Permutation to put the extra dims at the end.
   perm = (
-      array_ops.concat(
-          (math_ops.range(b_extra_ndims, b.shape.ndims),
-           math_ops.range(0, b_extra_ndims)), 0))
+      np.concatenate(
+          (np.arange(b_extra_ndims, b.shape.ndims),
+           np.arange(0, b_extra_ndims)), 0))
   b_extra_on_end = array_ops.transpose(b, perm=perm)
 
   # Now squash this end into one long dim.
@@ -497,7 +497,7 @@ def _reshape_for_efficiency(a,
     y_extra_shape = array_ops.concat(
         (array_ops.shape(y)[:-1], [b_main_sh[-1]], b_extra_sh), 0)
     y_extra_on_end = array_ops.reshape(y, y_extra_shape)
-    return array_ops.transpose(
-        y_extra_on_end, perm=array_ops.invert_permutation(perm))
+    inverse_perm = np.argsort(perm)
+    return array_ops.transpose(y_extra_on_end, perm=inverse_perm)
 
   return a, b_squashed_end, reshape_inv, still_need_to_transpose
-- 
GitLab


From 01d9a34786ea0da087855f88ac32d0d7085c8266 Mon Sep 17 00:00:00 2001
From: Nick Felt <nickfelt@google.com>
Date: Wed, 9 Jan 2019 12:16:00 -0800
Subject: [PATCH 0467/1095] Mark tf.summary v1 API classes and
 tf.train.summary_iterator() as v1-only

The tf.summary.FileWriter and FileWriterCache classes are only for v1 summary writing; TF 2.0 provides the v2 create_file_writer() API instead.

The tf.summary python protos are being removed as part of the overall effort to remove protos from the TF 2.0 python API. The recourse for users will be accessing these bindings from within TensorBoard - the exact package path is still TBD, but when we have one we can add a migration hint.

As for tf.train.summary_iterator(), this can be replaced by just using tf.io.record_iterator() and parsing each record with Event.FromString() using the Event proto symbol, wherever in TensorBoard it ends up.

PiperOrigin-RevId: 228564320
---
 tensorflow/python/__init__.py                 |  10 +-
 tensorflow/python/summary/summary.py          |   5 +-
 tensorflow/python/summary/summary_iterator.py |   2 +-
 tensorflow/python/summary/writer/writer.py    |   2 +-
 .../python/summary/writer/writer_cache.py     |   2 +-
 .../python/tools/api/generator/doc_srcs.py    |   1 +
 .../api/golden/v2/tensorflow.-event.pbtxt     |  74 ---------
 ...rflow.-summary-metadata.-plugin-data.pbtxt |  18 ---
 .../v2/tensorflow.-summary-metadata.pbtxt     |  40 -----
 .../v2/tensorflow.-summary.-audio.pbtxt       |  36 -----
 .../v2/tensorflow.-summary.-image.pbtxt       |  30 ----
 .../v2/tensorflow.-summary.-value.pbtxt       |  74 ---------
 .../api/golden/v2/tensorflow.-summary.pbtxt   | 144 ------------------
 .../tools/api/golden/v2/tensorflow.pbtxt      |  12 --
 .../golden/v2/tensorflow.summary.-event.pbtxt |  74 ---------
 ...ensorflow.summary.-file-writer-cache.pbtxt |  16 --
 .../v2/tensorflow.summary.-file-writer.pbtxt  |  50 ------
 ...sorflow.summary.-summary-description.pbtxt |  12 --
 .../tensorflow.summary.-summary.-audio.pbtxt  |  36 -----
 .../tensorflow.summary.-summary.-image.pbtxt  |  30 ----
 .../tensorflow.summary.-summary.-value.pbtxt  |  74 ---------
 .../v2/tensorflow.summary.-summary.pbtxt      | 144 ------------------
 ...sorflow.summary.-tagged-run-metadata.pbtxt |  18 ---
 .../api/golden/v2/tensorflow.summary.pbtxt    |  24 ---
 .../api/golden/v2/tensorflow.train.pbtxt      |   4 -
 tensorflow/tools/compatibility/renames_v2.py  |  10 ++
 26 files changed, 22 insertions(+), 920 deletions(-)
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-event.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-summary-metadata.-plugin-data.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-summary-metadata.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-summary.-audio.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-summary.-image.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-summary.-value.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.-summary.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.summary.-event.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.summary.-file-writer-cache.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.summary.-file-writer.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.summary.-summary-description.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.summary.-summary.-audio.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.summary.-summary.-image.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.summary.-summary.-value.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.summary.-summary.pbtxt
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.summary.-tagged-run-metadata.pbtxt

diff --git a/tensorflow/python/__init__.py b/tensorflow/python/__init__.py
index 207123cb14..31f9dce8e8 100644
--- a/tensorflow/python/__init__.py
+++ b/tensorflow/python/__init__.py
@@ -150,7 +150,7 @@ nn.rnn_cell = rnn_cell
 # pylint: disable=undefined-variable
 tf_export(v1=['AttrValue'])(AttrValue)
 tf_export(v1=['ConfigProto'])(ConfigProto)
-tf_export('Event', 'summary.Event')(Event)
+tf_export(v1=['Event', 'summary.Event'])(Event)
 tf_export(v1=['GPUOptions'])(GPUOptions)
 tf_export(v1=['GraphDef'])(GraphDef)
 tf_export(v1=['GraphOptions'])(GraphOptions)
@@ -163,10 +163,10 @@ tf_export(v1=['OptimizerOptions'])(OptimizerOptions)
 tf_export(v1=['RunMetadata'])(RunMetadata)
 tf_export(v1=['RunOptions'])(RunOptions)
 tf_export(v1=['SessionLog', 'summary.SessionLog'])(SessionLog)
-tf_export('Summary', 'summary.Summary')(Summary)
-tf_export('summary.SummaryDescription')(SummaryDescription)
-tf_export('SummaryMetadata')(SummaryMetadata)
-tf_export('summary.TaggedRunMetadata')(TaggedRunMetadata)
+tf_export(v1=['Summary', 'summary.Summary'])(Summary)
+tf_export(v1=['summary.SummaryDescription'])(SummaryDescription)
+tf_export(v1=['SummaryMetadata'])(SummaryMetadata)
+tf_export(v1=['summary.TaggedRunMetadata'])(TaggedRunMetadata)
 tf_export(v1=['TensorInfo'])(TensorInfo)
 # pylint: enable=undefined-variable
 
diff --git a/tensorflow/python/summary/summary.py b/tensorflow/python/summary/summary.py
index 0c13016712..a01feb3dde 100644
--- a/tensorflow/python/summary/summary.py
+++ b/tensorflow/python/summary/summary.py
@@ -13,9 +13,10 @@
 # limitations under the License.
 # ==============================================================================
 
-"""Tensor summaries for exporting information about a model.
+"""Operations for writing summary data, for use in analysis and visualization.
 
-See the [Summary](https://tensorflow.org/api_guides/python/summary) guide.
+See the [Summaries and
+TensorBoard](https://www.tensorflow.org/guide/summaries_and_tensorboard) guide.
 """
 
 from __future__ import absolute_import
diff --git a/tensorflow/python/summary/summary_iterator.py b/tensorflow/python/summary/summary_iterator.py
index 321b11ffb7..3675c235cf 100644
--- a/tensorflow/python/summary/summary_iterator.py
+++ b/tensorflow/python/summary/summary_iterator.py
@@ -24,7 +24,7 @@ from tensorflow.python.lib.io import tf_record
 from tensorflow.python.util.tf_export import tf_export
 
 
-@tf_export('train.summary_iterator')
+@tf_export(v1=['train.summary_iterator'])
 def summary_iterator(path):
   # pylint: disable=line-too-long
   """An iterator for reading `Event` protocol buffers from an event file.
diff --git a/tensorflow/python/summary/writer/writer.py b/tensorflow/python/summary/writer/writer.py
index 78217b503f..a66be4f833 100644
--- a/tensorflow/python/summary/writer/writer.py
+++ b/tensorflow/python/summary/writer/writer.py
@@ -279,7 +279,7 @@ class SummaryToEventTransformer(object):
     self.event_writer.add_event(event)
 
 
-@tf_export("summary.FileWriter")
+@tf_export(v1=["summary.FileWriter"])
 class FileWriter(SummaryToEventTransformer):
   """Writes `Summary` protocol buffers to event files.
 
diff --git a/tensorflow/python/summary/writer/writer_cache.py b/tensorflow/python/summary/writer/writer_cache.py
index 645fa28a37..c62a7ce1a3 100644
--- a/tensorflow/python/summary/writer/writer_cache.py
+++ b/tensorflow/python/summary/writer/writer_cache.py
@@ -25,7 +25,7 @@ from tensorflow.python.summary.writer.writer import FileWriter
 from tensorflow.python.util.tf_export import tf_export
 
 
-@tf_export('summary.FileWriterCache')
+@tf_export(v1=['summary.FileWriterCache'])
 class FileWriterCache(object):
   """Cache for file writers.
 
diff --git a/tensorflow/python/tools/api/generator/doc_srcs.py b/tensorflow/python/tools/api/generator/doc_srcs.py
index b567eead3d..28bf0e9d01 100644
--- a/tensorflow/python/tools/api/generator/doc_srcs.py
+++ b/tensorflow/python/tools/api/generator/doc_srcs.py
@@ -61,6 +61,7 @@ _TENSORFLOW_DOC_SOURCES = {
     'signal': DocSource(docstring_module_name='ops.signal.signal'),
     'sparse': DocSource(docstring_module_name='ops.sparse_ops'),
     'strings': DocSource(docstring_module_name='ops.string_ops'),
+    'summary': DocSource(docstring_module_name='summary.summary'),
     'sysconfig': DocSource(docstring_module_name='platform.sysconfig'),
     'test': DocSource(docstring_module_name='platform.test'),
     'train': DocSource(docstring_module_name='training.training'),
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-event.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-event.pbtxt
deleted file mode 100644
index 3b75a1735b..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.-event.pbtxt
+++ /dev/null
@@ -1,74 +0,0 @@
-path: "tensorflow.Event"
-tf_proto {
-  descriptor {
-    name: "Event"
-    field {
-      name: "wall_time"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_DOUBLE
-    }
-    field {
-      name: "step"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "file_version"
-      number: 3
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-      oneof_index: 0
-    }
-    field {
-      name: "graph_def"
-      number: 4
-      label: LABEL_OPTIONAL
-      type: TYPE_BYTES
-      oneof_index: 0
-    }
-    field {
-      name: "summary"
-      number: 5
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.Summary"
-      oneof_index: 0
-    }
-    field {
-      name: "log_message"
-      number: 6
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.LogMessage"
-      oneof_index: 0
-    }
-    field {
-      name: "session_log"
-      number: 7
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.SessionLog"
-      oneof_index: 0
-    }
-    field {
-      name: "tagged_run_metadata"
-      number: 8
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.TaggedRunMetadata"
-      oneof_index: 0
-    }
-    field {
-      name: "meta_graph_def"
-      number: 9
-      label: LABEL_OPTIONAL
-      type: TYPE_BYTES
-      oneof_index: 0
-    }
-    oneof_decl {
-      name: "what"
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-summary-metadata.-plugin-data.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-summary-metadata.-plugin-data.pbtxt
deleted file mode 100644
index a66b74b315..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.-summary-metadata.-plugin-data.pbtxt
+++ /dev/null
@@ -1,18 +0,0 @@
-path: "tensorflow.SummaryMetadata.PluginData"
-tf_proto {
-  descriptor {
-    name: "PluginData"
-    field {
-      name: "plugin_name"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-    field {
-      name: "content"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_BYTES
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-summary-metadata.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-summary-metadata.pbtxt
deleted file mode 100644
index c02575b962..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.-summary-metadata.pbtxt
+++ /dev/null
@@ -1,40 +0,0 @@
-path: "tensorflow.SummaryMetadata"
-tf_proto {
-  descriptor {
-    name: "SummaryMetadata"
-    field {
-      name: "plugin_data"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.SummaryMetadata.PluginData"
-    }
-    field {
-      name: "display_name"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-    field {
-      name: "summary_description"
-      number: 3
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-    nested_type {
-      name: "PluginData"
-      field {
-        name: "plugin_name"
-        number: 1
-        label: LABEL_OPTIONAL
-        type: TYPE_STRING
-      }
-      field {
-        name: "content"
-        number: 2
-        label: LABEL_OPTIONAL
-        type: TYPE_BYTES
-      }
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-summary.-audio.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-summary.-audio.pbtxt
deleted file mode 100644
index 94f712073e..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.-summary.-audio.pbtxt
+++ /dev/null
@@ -1,36 +0,0 @@
-path: "tensorflow.Summary.Audio"
-tf_proto {
-  descriptor {
-    name: "Audio"
-    field {
-      name: "sample_rate"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_FLOAT
-    }
-    field {
-      name: "num_channels"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "length_frames"
-      number: 3
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "encoded_audio_string"
-      number: 4
-      label: LABEL_OPTIONAL
-      type: TYPE_BYTES
-    }
-    field {
-      name: "content_type"
-      number: 5
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-summary.-image.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-summary.-image.pbtxt
deleted file mode 100644
index fc1acb483b..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.-summary.-image.pbtxt
+++ /dev/null
@@ -1,30 +0,0 @@
-path: "tensorflow.Summary.Image"
-tf_proto {
-  descriptor {
-    name: "Image"
-    field {
-      name: "height"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_INT32
-    }
-    field {
-      name: "width"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_INT32
-    }
-    field {
-      name: "colorspace"
-      number: 3
-      label: LABEL_OPTIONAL
-      type: TYPE_INT32
-    }
-    field {
-      name: "encoded_image_string"
-      number: 4
-      label: LABEL_OPTIONAL
-      type: TYPE_BYTES
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-summary.-value.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-summary.-value.pbtxt
deleted file mode 100644
index feb84b6ee9..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.-summary.-value.pbtxt
+++ /dev/null
@@ -1,74 +0,0 @@
-path: "tensorflow.Summary.Value"
-tf_proto {
-  descriptor {
-    name: "Value"
-    field {
-      name: "node_name"
-      number: 7
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-    field {
-      name: "tag"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-    field {
-      name: "metadata"
-      number: 9
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.SummaryMetadata"
-    }
-    field {
-      name: "simple_value"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_FLOAT
-      oneof_index: 0
-    }
-    field {
-      name: "obsolete_old_style_histogram"
-      number: 3
-      label: LABEL_OPTIONAL
-      type: TYPE_BYTES
-      oneof_index: 0
-    }
-    field {
-      name: "image"
-      number: 4
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.Summary.Image"
-      oneof_index: 0
-    }
-    field {
-      name: "histo"
-      number: 5
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.HistogramProto"
-      oneof_index: 0
-    }
-    field {
-      name: "audio"
-      number: 6
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.Summary.Audio"
-      oneof_index: 0
-    }
-    field {
-      name: "tensor"
-      number: 8
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.TensorProto"
-      oneof_index: 0
-    }
-    oneof_decl {
-      name: "value"
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-summary.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-summary.pbtxt
deleted file mode 100644
index b2bdff7171..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.-summary.pbtxt
+++ /dev/null
@@ -1,144 +0,0 @@
-path: "tensorflow.Summary"
-tf_proto {
-  descriptor {
-    name: "Summary"
-    field {
-      name: "value"
-      number: 1
-      label: LABEL_REPEATED
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.Summary.Value"
-    }
-    nested_type {
-      name: "Image"
-      field {
-        name: "height"
-        number: 1
-        label: LABEL_OPTIONAL
-        type: TYPE_INT32
-      }
-      field {
-        name: "width"
-        number: 2
-        label: LABEL_OPTIONAL
-        type: TYPE_INT32
-      }
-      field {
-        name: "colorspace"
-        number: 3
-        label: LABEL_OPTIONAL
-        type: TYPE_INT32
-      }
-      field {
-        name: "encoded_image_string"
-        number: 4
-        label: LABEL_OPTIONAL
-        type: TYPE_BYTES
-      }
-    }
-    nested_type {
-      name: "Audio"
-      field {
-        name: "sample_rate"
-        number: 1
-        label: LABEL_OPTIONAL
-        type: TYPE_FLOAT
-      }
-      field {
-        name: "num_channels"
-        number: 2
-        label: LABEL_OPTIONAL
-        type: TYPE_INT64
-      }
-      field {
-        name: "length_frames"
-        number: 3
-        label: LABEL_OPTIONAL
-        type: TYPE_INT64
-      }
-      field {
-        name: "encoded_audio_string"
-        number: 4
-        label: LABEL_OPTIONAL
-        type: TYPE_BYTES
-      }
-      field {
-        name: "content_type"
-        number: 5
-        label: LABEL_OPTIONAL
-        type: TYPE_STRING
-      }
-    }
-    nested_type {
-      name: "Value"
-      field {
-        name: "node_name"
-        number: 7
-        label: LABEL_OPTIONAL
-        type: TYPE_STRING
-      }
-      field {
-        name: "tag"
-        number: 1
-        label: LABEL_OPTIONAL
-        type: TYPE_STRING
-      }
-      field {
-        name: "metadata"
-        number: 9
-        label: LABEL_OPTIONAL
-        type: TYPE_MESSAGE
-        type_name: ".tensorflow.SummaryMetadata"
-      }
-      field {
-        name: "simple_value"
-        number: 2
-        label: LABEL_OPTIONAL
-        type: TYPE_FLOAT
-        oneof_index: 0
-      }
-      field {
-        name: "obsolete_old_style_histogram"
-        number: 3
-        label: LABEL_OPTIONAL
-        type: TYPE_BYTES
-        oneof_index: 0
-      }
-      field {
-        name: "image"
-        number: 4
-        label: LABEL_OPTIONAL
-        type: TYPE_MESSAGE
-        type_name: ".tensorflow.Summary.Image"
-        oneof_index: 0
-      }
-      field {
-        name: "histo"
-        number: 5
-        label: LABEL_OPTIONAL
-        type: TYPE_MESSAGE
-        type_name: ".tensorflow.HistogramProto"
-        oneof_index: 0
-      }
-      field {
-        name: "audio"
-        number: 6
-        label: LABEL_OPTIONAL
-        type: TYPE_MESSAGE
-        type_name: ".tensorflow.Summary.Audio"
-        oneof_index: 0
-      }
-      field {
-        name: "tensor"
-        number: 8
-        label: LABEL_OPTIONAL
-        type: TYPE_MESSAGE
-        type_name: ".tensorflow.TensorProto"
-        oneof_index: 0
-      }
-      oneof_decl {
-        name: "value"
-      }
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index 0cf1fe0e3c..7e5f86d7e6 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -8,10 +8,6 @@ tf_module {
     name: "DType"
     mtype: "<type \'type\'>"
   }
-  member {
-    name: "Event"
-    mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
-  }
   member {
     name: "GradientTape"
     mtype: "<type \'type\'>"
@@ -40,14 +36,6 @@ tf_module {
     name: "SparseTensor"
     mtype: "<type \'type\'>"
   }
-  member {
-    name: "Summary"
-    mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
-  }
-  member {
-    name: "SummaryMetadata"
-    mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
-  }
   member {
     name: "Tensor"
     mtype: "<type \'type\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.summary.-event.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.summary.-event.pbtxt
deleted file mode 100644
index eb99d0f533..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.summary.-event.pbtxt
+++ /dev/null
@@ -1,74 +0,0 @@
-path: "tensorflow.summary.Event"
-tf_proto {
-  descriptor {
-    name: "Event"
-    field {
-      name: "wall_time"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_DOUBLE
-    }
-    field {
-      name: "step"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "file_version"
-      number: 3
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-      oneof_index: 0
-    }
-    field {
-      name: "graph_def"
-      number: 4
-      label: LABEL_OPTIONAL
-      type: TYPE_BYTES
-      oneof_index: 0
-    }
-    field {
-      name: "summary"
-      number: 5
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.Summary"
-      oneof_index: 0
-    }
-    field {
-      name: "log_message"
-      number: 6
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.LogMessage"
-      oneof_index: 0
-    }
-    field {
-      name: "session_log"
-      number: 7
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.SessionLog"
-      oneof_index: 0
-    }
-    field {
-      name: "tagged_run_metadata"
-      number: 8
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.TaggedRunMetadata"
-      oneof_index: 0
-    }
-    field {
-      name: "meta_graph_def"
-      number: 9
-      label: LABEL_OPTIONAL
-      type: TYPE_BYTES
-      oneof_index: 0
-    }
-    oneof_decl {
-      name: "what"
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.summary.-file-writer-cache.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.summary.-file-writer-cache.pbtxt
deleted file mode 100644
index 2a5b63dcea..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.summary.-file-writer-cache.pbtxt
+++ /dev/null
@@ -1,16 +0,0 @@
-path: "tensorflow.summary.FileWriterCache"
-tf_class {
-  is_instance: "<class \'tensorflow.python.summary.writer.writer_cache.FileWriterCache\'>"
-  is_instance: "<type \'object\'>"
-  member_method {
-    name: "__init__"
-  }
-  member_method {
-    name: "clear"
-    argspec: "args=[], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get"
-    argspec: "args=[\'logdir\'], varargs=None, keywords=None, defaults=None"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.summary.-file-writer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.summary.-file-writer.pbtxt
deleted file mode 100644
index 6b65b0ace3..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.summary.-file-writer.pbtxt
+++ /dev/null
@@ -1,50 +0,0 @@
-path: "tensorflow.summary.FileWriter"
-tf_class {
-  is_instance: "<class \'tensorflow.python.summary.writer.writer.FileWriter\'>"
-  is_instance: "<class \'tensorflow.python.summary.writer.writer.SummaryToEventTransformer\'>"
-  is_instance: "<type \'object\'>"
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'logdir\', \'graph\', \'max_queue\', \'flush_secs\', \'graph_def\', \'filename_suffix\', \'session\'], varargs=None, keywords=None, defaults=[\'None\', \'10\', \'120\', \'None\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "add_event"
-    argspec: "args=[\'self\', \'event\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "add_graph"
-    argspec: "args=[\'self\', \'graph\', \'global_step\', \'graph_def\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-  member_method {
-    name: "add_meta_graph"
-    argspec: "args=[\'self\', \'meta_graph_def\', \'global_step\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "add_run_metadata"
-    argspec: "args=[\'self\', \'run_metadata\', \'tag\', \'global_step\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "add_session_log"
-    argspec: "args=[\'self\', \'session_log\', \'global_step\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "add_summary"
-    argspec: "args=[\'self\', \'summary\', \'global_step\'], varargs=None, keywords=None, defaults=[\'None\'], "
-  }
-  member_method {
-    name: "close"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "flush"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_logdir"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "reopen"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.summary.-summary-description.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.summary.-summary-description.pbtxt
deleted file mode 100644
index 4a8b59cf02..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.summary.-summary-description.pbtxt
+++ /dev/null
@@ -1,12 +0,0 @@
-path: "tensorflow.summary.SummaryDescription"
-tf_proto {
-  descriptor {
-    name: "SummaryDescription"
-    field {
-      name: "type_hint"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.summary.-summary.-audio.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.summary.-summary.-audio.pbtxt
deleted file mode 100644
index 8b271cf58f..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.summary.-summary.-audio.pbtxt
+++ /dev/null
@@ -1,36 +0,0 @@
-path: "tensorflow.summary.Summary.Audio"
-tf_proto {
-  descriptor {
-    name: "Audio"
-    field {
-      name: "sample_rate"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_FLOAT
-    }
-    field {
-      name: "num_channels"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "length_frames"
-      number: 3
-      label: LABEL_OPTIONAL
-      type: TYPE_INT64
-    }
-    field {
-      name: "encoded_audio_string"
-      number: 4
-      label: LABEL_OPTIONAL
-      type: TYPE_BYTES
-    }
-    field {
-      name: "content_type"
-      number: 5
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.summary.-summary.-image.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.summary.-summary.-image.pbtxt
deleted file mode 100644
index dbbc02dd05..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.summary.-summary.-image.pbtxt
+++ /dev/null
@@ -1,30 +0,0 @@
-path: "tensorflow.summary.Summary.Image"
-tf_proto {
-  descriptor {
-    name: "Image"
-    field {
-      name: "height"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_INT32
-    }
-    field {
-      name: "width"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_INT32
-    }
-    field {
-      name: "colorspace"
-      number: 3
-      label: LABEL_OPTIONAL
-      type: TYPE_INT32
-    }
-    field {
-      name: "encoded_image_string"
-      number: 4
-      label: LABEL_OPTIONAL
-      type: TYPE_BYTES
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.summary.-summary.-value.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.summary.-summary.-value.pbtxt
deleted file mode 100644
index 4176171cd9..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.summary.-summary.-value.pbtxt
+++ /dev/null
@@ -1,74 +0,0 @@
-path: "tensorflow.summary.Summary.Value"
-tf_proto {
-  descriptor {
-    name: "Value"
-    field {
-      name: "node_name"
-      number: 7
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-    field {
-      name: "tag"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-    field {
-      name: "metadata"
-      number: 9
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.SummaryMetadata"
-    }
-    field {
-      name: "simple_value"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_FLOAT
-      oneof_index: 0
-    }
-    field {
-      name: "obsolete_old_style_histogram"
-      number: 3
-      label: LABEL_OPTIONAL
-      type: TYPE_BYTES
-      oneof_index: 0
-    }
-    field {
-      name: "image"
-      number: 4
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.Summary.Image"
-      oneof_index: 0
-    }
-    field {
-      name: "histo"
-      number: 5
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.HistogramProto"
-      oneof_index: 0
-    }
-    field {
-      name: "audio"
-      number: 6
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.Summary.Audio"
-      oneof_index: 0
-    }
-    field {
-      name: "tensor"
-      number: 8
-      label: LABEL_OPTIONAL
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.TensorProto"
-      oneof_index: 0
-    }
-    oneof_decl {
-      name: "value"
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.summary.-summary.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.summary.-summary.pbtxt
deleted file mode 100644
index d6c5e3a87a..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.summary.-summary.pbtxt
+++ /dev/null
@@ -1,144 +0,0 @@
-path: "tensorflow.summary.Summary"
-tf_proto {
-  descriptor {
-    name: "Summary"
-    field {
-      name: "value"
-      number: 1
-      label: LABEL_REPEATED
-      type: TYPE_MESSAGE
-      type_name: ".tensorflow.Summary.Value"
-    }
-    nested_type {
-      name: "Image"
-      field {
-        name: "height"
-        number: 1
-        label: LABEL_OPTIONAL
-        type: TYPE_INT32
-      }
-      field {
-        name: "width"
-        number: 2
-        label: LABEL_OPTIONAL
-        type: TYPE_INT32
-      }
-      field {
-        name: "colorspace"
-        number: 3
-        label: LABEL_OPTIONAL
-        type: TYPE_INT32
-      }
-      field {
-        name: "encoded_image_string"
-        number: 4
-        label: LABEL_OPTIONAL
-        type: TYPE_BYTES
-      }
-    }
-    nested_type {
-      name: "Audio"
-      field {
-        name: "sample_rate"
-        number: 1
-        label: LABEL_OPTIONAL
-        type: TYPE_FLOAT
-      }
-      field {
-        name: "num_channels"
-        number: 2
-        label: LABEL_OPTIONAL
-        type: TYPE_INT64
-      }
-      field {
-        name: "length_frames"
-        number: 3
-        label: LABEL_OPTIONAL
-        type: TYPE_INT64
-      }
-      field {
-        name: "encoded_audio_string"
-        number: 4
-        label: LABEL_OPTIONAL
-        type: TYPE_BYTES
-      }
-      field {
-        name: "content_type"
-        number: 5
-        label: LABEL_OPTIONAL
-        type: TYPE_STRING
-      }
-    }
-    nested_type {
-      name: "Value"
-      field {
-        name: "node_name"
-        number: 7
-        label: LABEL_OPTIONAL
-        type: TYPE_STRING
-      }
-      field {
-        name: "tag"
-        number: 1
-        label: LABEL_OPTIONAL
-        type: TYPE_STRING
-      }
-      field {
-        name: "metadata"
-        number: 9
-        label: LABEL_OPTIONAL
-        type: TYPE_MESSAGE
-        type_name: ".tensorflow.SummaryMetadata"
-      }
-      field {
-        name: "simple_value"
-        number: 2
-        label: LABEL_OPTIONAL
-        type: TYPE_FLOAT
-        oneof_index: 0
-      }
-      field {
-        name: "obsolete_old_style_histogram"
-        number: 3
-        label: LABEL_OPTIONAL
-        type: TYPE_BYTES
-        oneof_index: 0
-      }
-      field {
-        name: "image"
-        number: 4
-        label: LABEL_OPTIONAL
-        type: TYPE_MESSAGE
-        type_name: ".tensorflow.Summary.Image"
-        oneof_index: 0
-      }
-      field {
-        name: "histo"
-        number: 5
-        label: LABEL_OPTIONAL
-        type: TYPE_MESSAGE
-        type_name: ".tensorflow.HistogramProto"
-        oneof_index: 0
-      }
-      field {
-        name: "audio"
-        number: 6
-        label: LABEL_OPTIONAL
-        type: TYPE_MESSAGE
-        type_name: ".tensorflow.Summary.Audio"
-        oneof_index: 0
-      }
-      field {
-        name: "tensor"
-        number: 8
-        label: LABEL_OPTIONAL
-        type: TYPE_MESSAGE
-        type_name: ".tensorflow.TensorProto"
-        oneof_index: 0
-      }
-      oneof_decl {
-        name: "value"
-      }
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.summary.-tagged-run-metadata.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.summary.-tagged-run-metadata.pbtxt
deleted file mode 100644
index 27c8873320..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.summary.-tagged-run-metadata.pbtxt
+++ /dev/null
@@ -1,18 +0,0 @@
-path: "tensorflow.summary.TaggedRunMetadata"
-tf_proto {
-  descriptor {
-    name: "TaggedRunMetadata"
-    field {
-      name: "tag"
-      number: 1
-      label: LABEL_OPTIONAL
-      type: TYPE_STRING
-    }
-    field {
-      name: "run_metadata"
-      number: 2
-      label: LABEL_OPTIONAL
-      type: TYPE_BYTES
-    }
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.summary.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.summary.pbtxt
index 61670bd151..c59f1b8474 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.summary.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.summary.pbtxt
@@ -1,33 +1,9 @@
 path: "tensorflow.summary"
 tf_module {
-  member {
-    name: "Event"
-    mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
-  }
-  member {
-    name: "FileWriter"
-    mtype: "<type \'type\'>"
-  }
-  member {
-    name: "FileWriterCache"
-    mtype: "<type \'type\'>"
-  }
-  member {
-    name: "Summary"
-    mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
-  }
-  member {
-    name: "SummaryDescription"
-    mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
-  }
   member {
     name: "SummaryWriter"
     mtype: "<type \'type\'>"
   }
-  member {
-    name: "TaggedRunMetadata"
-    mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
-  }
   member_method {
     name: "create_file_writer"
     argspec: "args=[\'logdir\', \'max_queue\', \'flush_millis\', \'filename_suffix\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt
index c72564e598..a3ace15ca2 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt
@@ -140,8 +140,4 @@ tf_module {
     name: "sdca_shrink_l1"
     argspec: "args=[\'weights\', \'l1\', \'l2\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "summary_iterator"
-    argspec: "args=[\'path\'], varargs=None, keywords=None, defaults=None"
-  }
 }
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index e9085e94b3..b757699f63 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -34,6 +34,7 @@ renames = {
     'tf.ConfigProto': 'tf.compat.v1.ConfigProto',
     'tf.DeviceSpec': 'tf.compat.v1.DeviceSpec',
     'tf.Dimension': 'tf.compat.v1.Dimension',
+    'tf.Event': 'tf.compat.v1.Event',
     'tf.FIFOQueue': 'tf.queue.FIFOQueue',
     'tf.FixedLenFeature': 'tf.io.FixedLenFeature',
     'tf.FixedLenSequenceFeature': 'tf.io.FixedLenSequenceFeature',
@@ -73,6 +74,8 @@ renames = {
     'tf.SparseConditionalAccumulator': 'tf.sparse.SparseConditionalAccumulator',
     'tf.SparseFeature': 'tf.io.SparseFeature',
     'tf.SparseTensorValue': 'tf.compat.v1.SparseTensorValue',
+    'tf.Summary': 'tf.compat.v1.Summary',
+    'tf.SummaryMetadata': 'tf.compat.v1.SummaryMetadata',
     'tf.TFRecordReader': 'tf.compat.v1.TFRecordReader',
     'tf.TensorInfo': 'tf.compat.v1.TensorInfo',
     'tf.TextLineReader': 'tf.compat.v1.TextLineReader',
@@ -608,8 +611,14 @@ renames = {
     'tf.string_strip': 'tf.strings.strip',
     'tf.string_to_hash_bucket_fast': 'tf.strings.to_hash_bucket_fast',
     'tf.string_to_hash_bucket_strong': 'tf.strings.to_hash_bucket_strong',
+    'tf.summary.Event': 'tf.compat.v1.summary.Event',
+    'tf.summary.FileWriter': 'tf.compat.v1.summary.FileWriter',
+    'tf.summary.FileWriterCache': 'tf.compat.v1.summary.FileWriterCache',
     'tf.summary.SessionLog': 'tf.compat.v1.summary.SessionLog',
     'tf.summary.audio': 'tf.compat.v1.summary.audio',
+    'tf.summary.Summary': 'tf.compat.v1.summary.Summary',
+    'tf.summary.SummaryDescription': 'tf.compat.v1.summary.SummaryDescription',
+    'tf.summary.TaggedRunMetadata': 'tf.compat.v1.summary.TaggedRunMetadata',
     'tf.summary.get_summary_description': 'tf.compat.v1.summary.get_summary_description',
     'tf.summary.histogram': 'tf.compat.v1.summary.histogram',
     'tf.summary.image': 'tf.compat.v1.summary.image',
@@ -713,6 +722,7 @@ renames = {
     'tf.train.slice_input_producer': 'tf.compat.v1.train.slice_input_producer',
     'tf.train.start_queue_runners': 'tf.compat.v1.train.start_queue_runners',
     'tf.train.string_input_producer': 'tf.compat.v1.train.string_input_producer',
+    'tf.train.summary_iterator': 'tf.compat.v1.train.summary_iterator',
     'tf.train.update_checkpoint_state': 'tf.compat.v1.train.update_checkpoint_state',
     'tf.train.warm_start': 'tf.compat.v1.train.warm_start',
     'tf.train.write_graph': 'tf.io.write_graph',
-- 
GitLab


From 9a5dcb4c6b7b60557e996106402c46475307c4cd Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 9 Jan 2019 12:18:30 -0800
Subject: [PATCH 0468/1095] A new Eager C API: Profiler. The profiler starts
 profiling when it is created. It stops when it is destroyed. Its output
 format is tfprof proto.

PiperOrigin-RevId: 228564754
---
 tensorflow/c/eager/BUILD                      | 32 +++++++-
 tensorflow/c/eager/c_api.cc                   |  2 +-
 tensorflow/c/eager/c_api_experimental.cc      | 23 ++++++
 tensorflow/c/eager/c_api_experimental.h       | 18 +++++
 tensorflow/c/eager/c_api_experimental_test.cc | 79 +++++++++++++++++++
 tensorflow/c/eager/c_api_internal.h           |  8 ++
 tensorflow/core/common_runtime/eager/BUILD    | 28 +++++++
 .../core/common_runtime/eager/context.cc      | 32 +++++++-
 .../core/common_runtime/eager/context.h       | 17 +++-
 .../core/common_runtime/eager/execute.cc      |  2 +-
 .../core/common_runtime/eager/profiler.cc     | 77 ++++++++++++++++++
 .../core/common_runtime/eager/profiler.h      | 63 +++++++++++++++
 tensorflow/python/pywrap_tfe.i                |  3 +
 13 files changed, 375 insertions(+), 9 deletions(-)
 create mode 100644 tensorflow/c/eager/c_api_experimental_test.cc
 create mode 100644 tensorflow/core/common_runtime/eager/profiler.cc
 create mode 100644 tensorflow/core/common_runtime/eager/profiler.h

diff --git a/tensorflow/c/eager/BUILD b/tensorflow/c/eager/BUILD
index 9f09ad1fc3..3ea1724d1e 100644
--- a/tensorflow/c/eager/BUILD
+++ b/tensorflow/c/eager/BUILD
@@ -3,11 +3,10 @@ licenses(["notice"])  # Apache 2.0
 
 load(
     "//tensorflow:tensorflow.bzl",
-    "tf_cuda_cc_test",
-    "tf_cc_test",
     "tf_copts",
-    "tfe_xla_copts",
+    "tf_cuda_cc_test",
     "tf_cuda_library",
+    "tfe_xla_copts",
 )
 
 tf_cuda_library(
@@ -29,6 +28,7 @@ tf_cuda_library(
             "//tensorflow/c:c_api_internal",
             "//tensorflow/core:core_cpu",
             "//tensorflow/core/common_runtime/eager:attr_builder",
+            "//tensorflow/core/common_runtime/eager:profiler",
             "//tensorflow/core/common_runtime/eager:context",
             "//tensorflow/core/common_runtime/eager:eager_executor",
             "//tensorflow/core/common_runtime/eager:execute",
@@ -89,6 +89,7 @@ tf_cuda_library(
         "//tensorflow/core/common_runtime/eager:eager_executor",
         "//tensorflow/core/common_runtime/eager:eager_operation",
         "//tensorflow/core/common_runtime/eager:kernel_and_device",
+        "//tensorflow/core/common_runtime/eager:profiler",
         "//tensorflow/core/common_runtime/eager:tensor_handle",
         "//tensorflow/core/distributed_runtime:remote_device",
         "//tensorflow/core/distributed_runtime:server_lib",
@@ -204,6 +205,31 @@ tf_cuda_library(
     ],
 )
 
+tf_cuda_cc_test(
+    name = "c_api_experimental_test",
+    size = "small",
+    srcs = [
+        "c_api_experimental_test.cc",
+    ],
+    extra_copts = tfe_xla_copts(),
+    tags = [
+        "guitar",
+        "multi_gpu",
+    ],
+    deps = [
+        ":c_api_experimental",
+        ":c_api_test_util",
+        "//tensorflow/c:c_test_util",
+        "//tensorflow/cc/profiler",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core/profiler:protos_all_cc",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
 cc_library(
     name = "tape",
     hdrs = ["tape.h"],
diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc
index 027d752f42..d5a391a98d 100755
--- a/tensorflow/c/eager/c_api.cc
+++ b/tensorflow/c/eager/c_api.cc
@@ -774,7 +774,7 @@ void TFE_ContextExportRunMetadata(TFE_Context* ctx, TF_Buffer* buf,
   if (!status->status.ok()) return;
   tensorflow::mutex_lock ml(*ctx->context.MetadataMu());
   status->status = MessageToBuffer(*ctx->context.RunMetadataProto(), buf);
-  ctx->context.RunMetadataProto()->Clear();
+  ctx->context.ClearRunMetadata();
 }
 
 namespace {
diff --git a/tensorflow/c/eager/c_api_experimental.cc b/tensorflow/c/eager/c_api_experimental.cc
index 3461d81b93..1ce03fb226 100644
--- a/tensorflow/c/eager/c_api_experimental.cc
+++ b/tensorflow/c/eager/c_api_experimental.cc
@@ -18,6 +18,29 @@ limitations under the License.
 #include "tensorflow/c/c_api.h"
 #include "tensorflow/c/eager/c_api_internal.h"
 
+using tensorflow::string;
+
 void TFE_OpConsumeInput(TFE_Op* op, TFE_TensorHandle* h, TF_Status* status) {
   op->operation.ConsumeInput(h->handle);
 }
+
+TFE_Profiler* TFE_NewProfiler(TFE_Context* ctx) {
+  return new TFE_Profiler(ctx);
+}
+
+void TFE_DeleteProfiler(TFE_Profiler* profiler) { delete profiler; }
+
+void TFE_ProfilerSerializeToString(TFE_Context* ctx, TFE_Profiler* profiler,
+                                   TF_Buffer* buf, TF_Status* status) {
+  TFE_ContextAsyncWait(ctx, status);
+  if (!status->status.ok()) return;
+  string content;
+  status->status = profiler->profiler->SerializeToString(&content);
+  void* data = tensorflow::port::Malloc(content.length());
+  content.copy(static_cast<char*>(data), content.length(), 0);
+  buf->data = data;
+  buf->length = content.length();
+  buf->data_deallocator = [](void* data, size_t length) {
+    tensorflow::port::Free(data);
+  };
+}
diff --git a/tensorflow/c/eager/c_api_experimental.h b/tensorflow/c/eager/c_api_experimental.h
index 4ee6c066ee..9eb80f5216 100644
--- a/tensorflow/c/eager/c_api_experimental.h
+++ b/tensorflow/c/eager/c_api_experimental.h
@@ -25,6 +25,24 @@ extern "C" {
 TF_CAPI_EXPORT extern void TFE_OpConsumeInput(TFE_Op* op, TFE_TensorHandle* h,
                                               TF_Status* status);
 
+// A profiler which will start profiling when creating the object and will stop
+// when the object is destroyed. It will profile all operations run under the
+// given TFE_Context. Multiple instance of it can be created, but at most one
+// of them will profile for each TFE_Context.
+// Thread-safety: TFE_Profiler is thread-safe.
+typedef struct TFE_Profiler TFE_Profiler;
+
+TF_CAPI_EXPORT extern TFE_Profiler* TFE_NewProfiler(TFE_Context* ctx);
+TF_CAPI_EXPORT extern void TFE_DeleteProfiler(TFE_Profiler* profiler);
+
+// The output string is a binary string of tensorflow.tfprof.ProfileProto.
+// User can write the string to file for offline analysis by tfprof command-line
+// tools or graphical user interface.
+TF_CAPI_EXPORT extern void TFE_ProfilerSerializeToString(TFE_Context* ctx,
+                                                         TFE_Profiler* profiler,
+                                                         TF_Buffer* buf,
+                                                         TF_Status* status);
+
 #ifdef __cplusplus
 } /* end extern "C" */
 #endif
diff --git a/tensorflow/c/eager/c_api_experimental_test.cc b/tensorflow/c/eager/c_api_experimental_test.cc
new file mode 100644
index 0000000000..9b4fca9d45
--- /dev/null
+++ b/tensorflow/c/eager/c_api_experimental_test.cc
@@ -0,0 +1,79 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/c/eager/c_api_experimental.h"
+
+#include <string.h>
+#include "absl/strings/match.h"
+#include "tensorflow/c/eager/c_api_test_util.h"
+#include "tensorflow/cc/profiler/profiler.h"
+#include "tensorflow/core/platform/logging.h"
+#include "tensorflow/core/platform/protobuf.h"
+#include "tensorflow/core/platform/test.h"
+#include "tensorflow/core/platform/test_benchmark.h"
+#include "tensorflow/core/profiler/tfprof_log.pb.h"
+
+using tensorflow::string;
+
+namespace {
+
+void ExecuteWithProfiling(bool async) {
+  TF_Status* status = TF_NewStatus();
+  TFE_ContextOptions* opts = TFE_NewContextOptions();
+  TFE_ContextOptionsSetAsync(opts, static_cast<unsigned char>(async));
+  TFE_Context* ctx = TFE_NewContext(opts, status);
+  TFE_Profiler* profiler = TFE_NewProfiler(ctx);
+  CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
+  TFE_DeleteContextOptions(opts);
+
+  TFE_TensorHandle* m = TestMatrixTensorHandle();
+  TFE_Op* matmul = MatMulOp(ctx, m, m);
+  TFE_TensorHandle* retvals[1] = {nullptr};
+  int num_retvals = 1;
+  TFE_Execute(matmul, &retvals[0], &num_retvals, status);
+  EXPECT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
+  TFE_DeleteOp(matmul);
+  TFE_DeleteTensorHandle(m);
+
+  ASSERT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
+  ASSERT_EQ(1, num_retvals);
+  TF_Buffer* profiler_result = TF_NewBuffer();
+  TFE_ProfilerSerializeToString(ctx, profiler, profiler_result, status);
+  TFE_DeleteProfiler(profiler);
+  ASSERT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
+  tensorflow::tfprof::ProfileProto profile_proto;
+  EXPECT_TRUE(profile_proto.ParseFromString(
+      {reinterpret_cast<const char*>(profiler_result->data),
+       profiler_result->length}));
+  TF_DeleteBuffer(profiler_result);
+
+  TF_Tensor* t = TFE_TensorHandleResolve(retvals[0], status);
+  TFE_DeleteTensorHandle(retvals[0]);
+  TFE_DeleteContext(ctx);
+  ASSERT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
+  float product[4] = {0};
+  EXPECT_EQ(sizeof(product), TF_TensorByteSize(t));
+  memcpy(&product[0], TF_TensorData(t), TF_TensorByteSize(t));
+  TF_DeleteTensor(t);
+  EXPECT_EQ(7, product[0]);
+  EXPECT_EQ(10, product[1]);
+  EXPECT_EQ(15, product[2]);
+  EXPECT_EQ(22, product[3]);
+  TF_DeleteStatus(status);
+}
+TEST(CAPI, ExecuteWithTracing) { ExecuteWithProfiling(false); }
+TEST(CAPI, ExecuteWithTracingAsync) { ExecuteWithProfiling(true); }
+
+}  // namespace
diff --git a/tensorflow/c/eager/c_api_internal.h b/tensorflow/c/eager/c_api_internal.h
index 67bc1bcd24..7b1035f631 100644
--- a/tensorflow/c/eager/c_api_internal.h
+++ b/tensorflow/c/eager/c_api_internal.h
@@ -34,6 +34,7 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/eager/eager_executor.h"
 #include "tensorflow/core/common_runtime/eager/eager_operation.h"
 #include "tensorflow/core/common_runtime/eager/kernel_and_device.h"
+#include "tensorflow/core/common_runtime/eager/profiler.h"
 #include "tensorflow/core/common_runtime/eager/tensor_handle.h"
 #include "tensorflow/core/common_runtime/function.h"
 #include "tensorflow/core/common_runtime/rendezvous_mgr.h"
@@ -100,6 +101,13 @@ struct TFE_Op {
   tensorflow::EagerOperation operation;
 };
 
+struct TFE_Profiler {
+  TFE_Profiler(TFE_Context* ctx)
+      : profiler(tensorflow::EagerProfiler::Create(&ctx->context)) {}
+
+  std::unique_ptr<tensorflow::EagerProfiler> profiler;
+};
+
 namespace tensorflow {
 // Set an AttrValue on the op. Doesn't handle the list types.
 void SetOpAttrValueScalar(TFE_Context* ctx, TFE_Op* op,
diff --git a/tensorflow/core/common_runtime/eager/BUILD b/tensorflow/core/common_runtime/eager/BUILD
index 77e3246df0..cabbddb77d 100644
--- a/tensorflow/core/common_runtime/eager/BUILD
+++ b/tensorflow/core/common_runtime/eager/BUILD
@@ -88,6 +88,34 @@ tf_cuda_library(
     ],
 )
 
+tf_cuda_library(
+    name = "profiler",
+    srcs = [
+        "profiler.cc",
+    ],
+    hdrs = [
+        "profiler.h",
+    ],
+    visibility = ["//tensorflow:internal"],
+    deps = [
+        ":context",
+        "//tensorflow/cc/profiler",
+    ] + select({
+        "//tensorflow:android": [
+            "//tensorflow/core:android_tensorflow_lib_lite",
+        ],
+        "//conditions:default": [
+            "//tensorflow/core:core_cpu_lib",
+            "//tensorflow/core:framework",
+            "//tensorflow/core:framework_internal",
+            "//tensorflow/core:lib",
+            "//tensorflow/core:lib_internal",
+            "//tensorflow/core:protos_all_cc",
+            "//tensorflow/core:session_options",
+        ],
+    }),
+)
+
 tf_cuda_library(
     name = "tensor_handle",
     srcs = [
diff --git a/tensorflow/core/common_runtime/eager/context.cc b/tensorflow/core/common_runtime/eager/context.cc
index 2212bda534..1d93d2bbe6 100644
--- a/tensorflow/core/common_runtime/eager/context.cc
+++ b/tensorflow/core/common_runtime/eager/context.cc
@@ -234,6 +234,29 @@ Status EagerContext::FindDeviceByName(const string& name, Device** result) {
   return Status::OK();
 }
 
+void EagerContext::ClearRunMetadata() {
+  if (metadata_listener_ != nullptr) {
+    metadata_listener_->BeforeClearRunMetadata();
+  }
+  run_metadata_.Clear();
+}
+
+Status EagerContext::RegisterRunMetadataListener(
+    RunMetadataListener* listener) {
+  mutex_lock l(metadata_mu_);
+  if (metadata_listener_ != nullptr) {
+    return Status(error::Code::INVALID_ARGUMENT,
+                  "Cannot run two eager profiler at the same time");
+  }
+  metadata_listener_ = listener;
+  return Status::OK();
+}
+
+void EagerContext::ClearRunMetadataListener() {
+  mutex_lock l(metadata_mu_);
+  metadata_listener_ = nullptr;
+}
+
 void EagerContext::StartStep() {
   mutex_lock ml(metadata_mu_);
   num_active_steps_++;
@@ -317,10 +340,15 @@ void EagerContext::AddKernelToCache(Fprint128 cache_key,
   gtl::InsertOrUpdate(&kernel_cache_, cache_key, kernel);
 }
 
+bool EagerContext::ShouldStoreMetadata() {
+  mutex_lock ml(metadata_mu_);
+  return should_store_metadata_.load() || metadata_listener_ != nullptr;
+}
+
 void EagerContext::SetShouldStoreMetadata(bool value) {
+  mutex_lock ml(metadata_mu_);
   should_store_metadata_.store(value);
-  if (!value) {
-    mutex_lock ml(metadata_mu_);
+  if (!value || metadata_listener_ != nullptr) {
     run_metadata_.Clear();
   }
 }
diff --git a/tensorflow/core/common_runtime/eager/context.h b/tensorflow/core/common_runtime/eager/context.h
index 5ff6b3ffbd..b83f5707da 100644
--- a/tensorflow/core/common_runtime/eager/context.h
+++ b/tensorflow/core/common_runtime/eager/context.h
@@ -29,6 +29,7 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/eager/kernel_and_device.h"
 #include "tensorflow/core/common_runtime/function.h"
 #include "tensorflow/core/common_runtime/rendezvous_mgr.h"
+#include "tensorflow/core/example/example.pb.h"
 #ifndef __ANDROID__
 #include "tensorflow/core/distributed_runtime/eager/eager_client.h"
 #include "tensorflow/core/distributed_runtime/server_lib.h"
@@ -66,6 +67,12 @@ enum ContextDevicePlacementPolicy {
   DEVICE_PLACEMENT_SILENT_FOR_INT32 = 3,
 };
 
+class RunMetadataListener {
+ public:
+  virtual ~RunMetadataListener() {}
+  virtual void BeforeClearRunMetadata() = 0;
+};
+
 class EagerContext {
  public:
   // TODO: remove this constructor once we migrate all callers to the next one.
@@ -172,10 +179,15 @@ class EagerContext {
   void ReleaseDeviceMgr() { local_device_manager_.release(); }
 
   // TODO(apassos) clean up RunMetadata storage.
-  mutex* MetadataMu() { return &metadata_mu_; }
-  bool ShouldStoreMetadata() { return should_store_metadata_.load(); }
+  mutex* MetadataMu() LOCK_RETURNED(metadata_mu_) { return &metadata_mu_; }
+  bool ShouldStoreMetadata() LOCKS_EXCLUDED(metadata_mu_);
   void SetShouldStoreMetadata(bool value);
   RunMetadata* RunMetadataProto() { return &run_metadata_; }
+  void ClearRunMetadata() EXCLUSIVE_LOCKS_REQUIRED(metadata_mu_);
+
+  Status RegisterRunMetadataListener(RunMetadataListener* listener)
+      LOCKS_EXCLUDED(metadata_mu_);
+  void ClearRunMetadataListener() LOCKS_EXCLUDED(metadata_mu_);
 
   void StartStep();
   void EndStep();
@@ -269,6 +281,7 @@ class EagerContext {
   std::atomic<bool> should_store_metadata_{false};
   mutex metadata_mu_;
   RunMetadata run_metadata_ GUARDED_BY(metadata_mu_);
+  RunMetadataListener* metadata_listener_ GUARDED_BY(metadata_mu_) = nullptr;
   GraphCollector graph_collector_;
   const bool log_device_placement_;
   // EagerExecutor for async execution.
diff --git a/tensorflow/core/common_runtime/eager/execute.cc b/tensorflow/core/common_runtime/eager/execute.cc
index 79806c3c73..9d2401929d 100644
--- a/tensorflow/core/common_runtime/eager/execute.cc
+++ b/tensorflow/core/common_runtime/eager/execute.cc
@@ -752,8 +752,8 @@ Status EagerExecute(EagerContext* ctx, Device* device,
     maybe_stats->set_all_end_rel_micros(nanos / EnvTime::kMicrosToNanos -
                                         maybe_stats->all_start_micros());
     maybe_stats->set_all_end_rel_nanos(nanos - maybe_stats->all_start_nanos());
-    mutex_lock ml(*ctx->MetadataMu());
     if (ctx->ShouldStoreMetadata()) {
+      mutex_lock ml(*ctx->MetadataMu());
       {
         GraphCollector* collector = ctx->GetGraphCollector();
         mutex_lock mll(collector->mu);
diff --git a/tensorflow/core/common_runtime/eager/profiler.cc b/tensorflow/core/common_runtime/eager/profiler.cc
new file mode 100644
index 0000000000..d670d6f344
--- /dev/null
+++ b/tensorflow/core/common_runtime/eager/profiler.cc
@@ -0,0 +1,77 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/common_runtime/eager/profiler.h"
+#include "tensorflow/cc/profiler/profiler.h"
+#include "tensorflow/core/framework/graph.pb.h"
+#include "tensorflow/core/platform/mutex.h"
+#include "tensorflow/core/protobuf/config.pb.h"
+
+namespace tensorflow {
+
+/*static*/ std::unique_ptr<EagerProfiler> EagerProfiler::Create(
+    EagerContext* const context) {
+  return absl::WrapUnique(new EagerProfiler(context));
+}
+
+void EagerProfiler::BeforeClearRunMetadata() {
+  mutex_lock l(mutex_);
+  run_metadata_.MergeFrom(*context_->RunMetadataProto());
+}
+
+Status EagerProfiler::Status() {
+  mutex_lock l(mutex_);
+  return status_;
+}
+
+Status EagerProfiler::SerializeToString(string* content) {
+  {
+    mutex_lock l(mutex_);
+    if (!status_.ok()) return status_;
+  }
+  RunMetadata metadata;
+  GetMergetRunMetadata(&metadata);
+
+  // TODO(fishx): update tfprof to use a lighter representation instead of
+  // GraphDef.
+  GraphDef graph;
+  std::unique_ptr<tfprof::Profiler> tfprof(new tfprof::Profiler(graph));
+  tfprof->AddStep(0, metadata);
+  return tfprof->SerializeToString(content);
+}
+
+EagerProfiler::EagerProfiler(EagerContext* const context) : context_(context) {
+  LOG(INFO) << "Eager Profiler started.";
+
+  status_ = context_->RegisterRunMetadataListener(this);
+  if (!status_.ok()) {
+    LOG(INFO) << "Eager Profiler failed to start. Another profiler is running.";
+    return;
+  }
+}
+
+EagerProfiler::~EagerProfiler() {
+  context_->ClearRunMetadataListener();
+  LOG(INFO) << "Eager Profiler ended with status:" << status_;
+}
+
+void EagerProfiler::GetMergetRunMetadata(RunMetadata* metadata) {
+  mutex_lock ml(*context_->MetadataMu());
+  mutex_lock l(mutex_);
+  *metadata = run_metadata_;
+  metadata->MergeFrom(*context_->RunMetadataProto());
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/eager/profiler.h b/tensorflow/core/common_runtime/eager/profiler.h
new file mode 100644
index 0000000000..cadbfdb498
--- /dev/null
+++ b/tensorflow/core/common_runtime/eager/profiler.h
@@ -0,0 +1,63 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_PROFILER_H_
+#define TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_PROFILER_H_
+
+#include "tensorflow/core/common_runtime/eager/context.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/platform/mutex.h"
+#include "tensorflow/core/protobuf/config.pb.h"
+
+namespace tensorflow {
+
+// A profiler which will start profiling when creating the object and will stop
+// when the object is destroyed. It will profile all operations run under the
+// given EagerContext.
+// Multiple instances of it can be created, but at most one of them will profile
+// for each EagerContext. Status() will return OK only for the instance that is
+// profiling.
+// Thread-safety: TFE_Profiler is thread-safe.
+class EagerProfiler : RunMetadataListener {
+ public:
+  // Creates and EagerProfiler and starts profiling.
+  static std::unique_ptr<EagerProfiler> Create(EagerContext* const context);
+
+  // Deletes an exsiting Profiler and enables starting a new one.
+  ~EagerProfiler() override;
+
+  void BeforeClearRunMetadata() override LOCKS_EXCLUDED(mutex_)
+      EXCLUSIVE_LOCKS_REQUIRED(context_->MetadataMu());
+  tensorflow::Status Status() LOCKS_EXCLUDED(mutex_);
+
+  tensorflow::Status SerializeToString(string* content) LOCKS_EXCLUDED(mutex_);
+
+ private:
+  // Constructs an instance of the class and starts profiling
+  explicit EagerProfiler(EagerContext* const context);
+
+  // Profiler is neither copyable or movable.
+  EagerProfiler(const EagerProfiler&) = delete;
+  EagerProfiler& operator=(const EagerProfiler&) = delete;
+
+  void GetMergetRunMetadata(RunMetadata* metadata) LOCKS_EXCLUDED(mutex_);
+
+  RunMetadata run_metadata_ GUARDED_BY(mutex_);
+  tensorflow::Status status_ GUARDED_BY(mutex_);
+  EagerContext* const context_;
+  mutex mutex_;
+};
+
+}  // namespace tensorflow
+#endif  // TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_PROFILER_H_
diff --git a/tensorflow/python/pywrap_tfe.i b/tensorflow/python/pywrap_tfe.i
index 733d471ca2..0620e0345c 100755
--- a/tensorflow/python/pywrap_tfe.i
+++ b/tensorflow/python/pywrap_tfe.i
@@ -32,6 +32,9 @@ limitations under the License.
 %rename("%s") TFE_ContextSetServerDef;
 %rename("%s") TFE_ContextAsyncWait;
 %rename("%s") TFE_ContextAsyncClearError;
+%rename("%s") TFE_NewProfiler;
+%rename("%s") TFE_DeleteProfiler;
+%rename("%s") TFE_ProfilerSerializeToString;
 %rename("%s") TFE_OpNameGetAttrType;
 %rename("%s") TFE_Py_InitEagerTensor;
 %rename("%s") TFE_Py_SetEagerTensorProfiler;
-- 
GitLab


From 52718ef7adf0f0a292dcd43432587c818f895ce2 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Wed, 9 Jan 2019 12:18:32 -0800
Subject: [PATCH 0469/1095] Add `axis` property to CosineProximity loss and
 metric. Refactor cosine proximity tests for better clarity.

PiperOrigin-RevId: 228564762
---
 tensorflow/python/keras/losses.py             | 26 ++++--
 tensorflow/python/keras/losses_test.py        | 93 ++++++++++++-------
 tensorflow/python/keras/metrics.py            | 12 ++-
 tensorflow/python/keras/metrics_test.py       | 58 ++++++++----
 ...rflow.keras.losses.-cosine-proximity.pbtxt |  2 +-
 .../golden/v1/tensorflow.keras.losses.pbtxt   |  4 +-
 ...flow.keras.metrics.-cosine-proximity.pbtxt |  2 +-
 .../golden/v1/tensorflow.keras.metrics.pbtxt  |  4 +-
 ...rflow.keras.losses.-cosine-proximity.pbtxt |  2 +-
 .../golden/v2/tensorflow.keras.losses.pbtxt   |  4 +-
 ...flow.keras.metrics.-cosine-proximity.pbtxt |  2 +-
 .../golden/v2/tensorflow.keras.metrics.pbtxt  |  4 +-
 12 files changed, 143 insertions(+), 70 deletions(-)

diff --git a/tensorflow/python/keras/losses.py b/tensorflow/python/keras/losses.py
index 5f9278823c..66780de0f0 100644
--- a/tensorflow/python/keras/losses.py
+++ b/tensorflow/python/keras/losses.py
@@ -821,15 +821,15 @@ def poisson(y_true, y_pred):
               'keras.metrics.cosine',
               'keras.losses.cosine_proximity',
               'keras.losses.cosine')
-def cosine_proximity(y_true, y_pred):
-  y_true = nn.l2_normalize(y_true, axis=-1)
-  y_pred = nn.l2_normalize(y_pred, axis=-1)
-  return -math_ops.reduce_sum(y_true * y_pred, axis=-1)
+def cosine_proximity(y_true, y_pred, axis=-1):
+  y_true = nn.l2_normalize(y_true, axis=axis)
+  y_pred = nn.l2_normalize(y_pred, axis=axis)
+  return -math_ops.reduce_sum(y_true * y_pred, axis=axis)
 
 
 @keras_export('keras.losses.CosineProximity')
 class CosineProximity(Loss):
-  """Computes the cosine distance between `y_true` and `y_pred`.
+  """Computes the cosine proximity between `y_true` and `y_pred`.
 
   Usage:
 
@@ -845,8 +845,22 @@ class CosineProximity(Loss):
   model = keras.models.Model(inputs, outputs)
   model.compile('sgd', loss=tf.losses.CosineProximity())
   ```
+
+  Args:
+    axis: (Optional) Defaults to -1. The dimension along which the cosine
+      proximity is computed.
+    reduction: (Optional) Type of `tf.losses.Reduction` to apply to loss.
+      Default value is `SUM_OVER_BATCH_SIZE`.
+    name: Optional name for the op.
   """
 
+  def __init__(self,
+               axis=-1,
+               reduction=losses_impl.ReductionV2.SUM_OVER_BATCH_SIZE,
+               name=None):
+    super(CosineProximity, self).__init__(reduction=reduction, name=name)
+    self.axis = axis
+
   def call(self, y_true, y_pred):
     """Calculates the cosine proximity loss.
 
@@ -859,7 +873,7 @@ class CosineProximity(Loss):
     """
     y_pred = ops.convert_to_tensor(y_pred)
     y_true = math_ops.cast(y_true, y_pred.dtype)
-    return cosine_proximity(y_true, y_pred)
+    return cosine_proximity(y_true, y_pred, axis=self.axis)
 
 
 # Aliases.
diff --git a/tensorflow/python/keras/losses_test.py b/tensorflow/python/keras/losses_test.py
index 5d6cd1be55..004c30f84d 100644
--- a/tensorflow/python/keras/losses_test.py
+++ b/tensorflow/python/keras/losses_test.py
@@ -485,59 +485,88 @@ class MeanSquaredLogarithmicErrorTest(test.TestCase):
 @test_util.run_all_in_graph_and_eager_modes
 class CosineProximityTest(test.TestCase):
 
+  def l2_norm(self, x, axis):
+    epsilon = 1e-12
+    square_sum = np.sum(np.square(x), axis=axis, keepdims=True)
+    x_inv_norm = 1 / np.sqrt(np.maximum(square_sum, epsilon))
+    return np.multiply(x, x_inv_norm)
+
+  def setup(self, axis=1):
+    self.np_y_true = np.asarray([[1, 9, 2], [-5, -2, 6]], dtype=np.float32)
+    self.np_y_pred = np.asarray([[4, 8, 12], [8, 1, 3]], dtype=np.float32)
+
+    y_true = self.l2_norm(self.np_y_true, axis)
+    y_pred = self.l2_norm(self.np_y_pred, axis)
+    self.expected_loss = -np.sum(np.multiply(y_true, y_pred), axis=(axis,))
+
+    self.y_true = constant_op.constant(self.np_y_true)
+    self.y_pred = constant_op.constant(self.np_y_pred)
+
   def test_config(self):
     cosine_obj = keras.losses.CosineProximity(
-        reduction=losses_impl.ReductionV2.SUM, name='cosine_loss')
+        axis=2, reduction=losses_impl.ReductionV2.SUM, name='cosine_loss')
     self.assertEqual(cosine_obj.name, 'cosine_loss')
     self.assertEqual(cosine_obj.reduction, losses_impl.ReductionV2.SUM)
+    self.assertEqual(cosine_obj.axis, 2)
 
   def test_unweighted(self):
+    self.setup()
     cosine_obj = keras.losses.CosineProximity()
-    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
-    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
-                                  shape=(2, 3),
-                                  dtype=dtypes.float32)
-    loss = cosine_obj(y_true, y_pred)
-    self.assertAlmostEqual(self.evaluate(loss), -0.18722, 3)
+    loss = cosine_obj(self.y_true, self.y_pred)
+    expected_loss = np.mean(self.expected_loss)
+    self.assertAlmostEqual(self.evaluate(loss), expected_loss, 3)
 
   def test_scalar_weighted(self):
+    self.setup()
     cosine_obj = keras.losses.CosineProximity()
-    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
-    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
-                                  shape=(2, 3),
-                                  dtype=dtypes.float32)
-    loss = cosine_obj(y_true, y_pred, sample_weight=2.3)
-    self.assertAlmostEqual(self.evaluate(loss), -0.43060, 3)
+    sample_weight = 2.3
+    loss = cosine_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
+    expected_loss = np.mean(self.expected_loss * sample_weight)
+    self.assertAlmostEqual(self.evaluate(loss), expected_loss, 3)
 
   def test_sample_weighted(self):
+    self.setup()
     cosine_obj = keras.losses.CosineProximity()
-    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
-    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
-                                  shape=(2, 3),
-                                  dtype=dtypes.float32)
-    sample_weight = constant_op.constant([1.2, 3.4], shape=(2, 1))
-    loss = cosine_obj(y_true, y_pred, sample_weight=sample_weight)
-    self.assertAlmostEqual(self.evaluate(loss), 0.15599, 3)
+    sample_weight = np.asarray([1.2, 3.4])
+    loss = cosine_obj(
+        self.y_true,
+        self.y_pred,
+        sample_weight=constant_op.constant(sample_weight))
+    expected_loss = np.mean(self.expected_loss * sample_weight)
+    self.assertAlmostEqual(self.evaluate(loss), expected_loss, 3)
 
   def test_timestep_weighted(self):
+    self.setup()
     cosine_obj = keras.losses.CosineProximity()
-    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3, 1))
-    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
-                                  shape=(2, 3, 1),
-                                  dtype=dtypes.float32)
-    sample_weight = constant_op.constant([3, 6, 5, 0, 4, 2], shape=(2, 3))
-    loss = cosine_obj(y_true, y_pred, sample_weight=sample_weight)
-    self.assertAlmostEqual(self.evaluate(loss), -2.0000, 3)
+    np_y_true = self.np_y_true.reshape((2, 3, 1))
+    np_y_pred = self.np_y_pred.reshape((2, 3, 1))
+    sample_weight = np.asarray([3, 6, 5, 0, 4, 2]).reshape((2, 3))
+
+    y_true = self.l2_norm(np_y_true, 2)
+    y_pred = self.l2_norm(np_y_pred, 2)
+    expected_loss = -np.sum(np.multiply(y_true, y_pred), axis=(2,))
+
+    y_true = constant_op.constant(np_y_true)
+    y_pred = constant_op.constant(np_y_pred)
+    loss = cosine_obj(
+        y_true, y_pred, sample_weight=constant_op.constant(sample_weight))
+
+    expected_loss = np.mean(expected_loss * sample_weight)
+    self.assertAlmostEqual(self.evaluate(loss), expected_loss, 3)
 
   def test_zero_weighted(self):
+    self.setup()
     cosine_obj = keras.losses.CosineProximity()
-    y_true = constant_op.constant([1, 9, 2, -5, -2, 6], shape=(2, 3))
-    y_pred = constant_op.constant([4, 8, 12, 8, 1, 3],
-                                  shape=(2, 3),
-                                  dtype=dtypes.float32)
-    loss = cosine_obj(y_true, y_pred, sample_weight=0)
+    loss = cosine_obj(self.y_true, self.y_pred, sample_weight=0)
     self.assertAlmostEqual(self.evaluate(loss), 0., 3)
 
+  def test_axis(self):
+    self.setup(axis=1)
+    cosine_obj = keras.losses.CosineProximity(axis=1)
+    loss = cosine_obj(self.y_true, self.y_pred)
+    expected_loss = np.mean(self.expected_loss)
+    self.assertAlmostEqual(self.evaluate(loss), expected_loss, 3)
+
 
 @test_util.run_all_in_graph_and_eager_modes
 class BinaryCrossentropyTest(test.TestCase):
diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index 2c2700ffa3..a13d8747d6 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -1443,8 +1443,16 @@ class CosineProximity(MeanMetricWrapper):
   ```
   """
 
-  def __init__(self, name='cosine_proximity', dtype=None):
-    super(CosineProximity, self).__init__(cosine, name, dtype=dtype)
+  def __init__(self, name='cosine_proximity', dtype=None, axis=-1):
+    """Creates a `CosineProximity` instance.
+
+    Args:
+      name: (Optional) string name of the metric instance.
+      dtype: (Optional) data type of the metric result.
+      axis: (Optional) Defaults to -1. The dimension along which the cosine
+        proximity is computed.
+    """
+    super(CosineProximity, self).__init__(cosine, name, dtype=dtype, axis=axis)
 
   @classmethod
   def from_config(cls, config):
diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py
index 4f8406f0e3..42da1dfb99 100644
--- a/tensorflow/python/keras/metrics_test.py
+++ b/tensorflow/python/keras/metrics_test.py
@@ -1049,8 +1049,26 @@ class SpecificityAtSensitivityTest(test.TestCase, parameterized.TestCase):
 @test_util.run_all_in_graph_and_eager_modes
 class CosineProximityTest(test.TestCase):
 
+  def l2_norm(self, x, axis):
+    epsilon = 1e-12
+    square_sum = np.sum(np.square(x), axis=axis, keepdims=True)
+    x_inv_norm = 1 / np.sqrt(np.maximum(square_sum, epsilon))
+    return np.multiply(x, x_inv_norm)
+
+  def setup(self, axis=1):
+    self.np_y_true = np.asarray([[1, 9, 2], [-5, -2, 6]], dtype=np.float32)
+    self.np_y_pred = np.asarray([[4, 8, 12], [8, 1, 3]], dtype=np.float32)
+
+    y_true = self.l2_norm(self.np_y_true, axis)
+    y_pred = self.l2_norm(self.np_y_pred, axis)
+    self.expected_loss = -np.sum(np.multiply(y_true, y_pred), axis=(axis,))
+
+    self.y_true = constant_op.constant(self.np_y_true)
+    self.y_pred = constant_op.constant(self.np_y_pred)
+
   def test_config(self):
-    cosine_obj = metrics.CosineProximity(name='my_cos', dtype=dtypes.int32)
+    cosine_obj = metrics.CosineProximity(
+        axis=2, name='my_cos', dtype=dtypes.int32)
     self.assertEqual(cosine_obj.name, 'my_cos')
     self.assertEqual(cosine_obj._dtype, dtypes.int32)
 
@@ -1060,29 +1078,33 @@ class CosineProximityTest(test.TestCase):
     self.assertEqual(cosine_obj2._dtype, dtypes.int32)
 
   def test_unweighted(self):
+    self.setup()
     cosine_obj = metrics.CosineProximity()
     self.evaluate(variables.variables_initializer(cosine_obj.variables))
-
-    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
-                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
-    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
-                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
-
-    update_op = cosine_obj.update_state(y_true, y_pred)
-    self.evaluate(update_op)
-    result = cosine_obj.result()
-    self.assertAllClose(-0.60723, result, atol=1e-5)
+    loss = cosine_obj(self.y_true, self.y_pred)
+    expected_loss = np.mean(self.expected_loss)
+    self.assertAlmostEqual(self.evaluate(loss), expected_loss, 3)
 
   def test_weighted(self):
+    self.setup()
     cosine_obj = metrics.CosineProximity()
     self.evaluate(variables.variables_initializer(cosine_obj.variables))
-    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
-                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
-    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
-                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
-    sample_weight = constant_op.constant((1., 1.5, 2., 2.5))
-    result = cosine_obj(y_true, y_pred, sample_weight=sample_weight)
-    self.assertAllClose(-0.59916, self.evaluate(result), atol=1e-5)
+    sample_weight = np.asarray([1.2, 3.4])
+    loss = cosine_obj(
+        self.y_true,
+        self.y_pred,
+        sample_weight=constant_op.constant(sample_weight))
+    expected_loss = np.sum(
+        self.expected_loss * sample_weight) / np.sum(sample_weight)
+    self.assertAlmostEqual(self.evaluate(loss), expected_loss, 3)
+
+  def test_axis(self):
+    self.setup(axis=1)
+    cosine_obj = metrics.CosineProximity(axis=1)
+    self.evaluate(variables.variables_initializer(cosine_obj.variables))
+    loss = cosine_obj(self.y_true, self.y_pred)
+    expected_loss = np.mean(self.expected_loss)
+    self.assertAlmostEqual(self.evaluate(loss), expected_loss, 3)
 
 
 @test_util.run_all_in_graph_and_eager_modes
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-cosine-proximity.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-cosine-proximity.pbtxt
index de35966f48..4952a76291 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-cosine-proximity.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.-cosine-proximity.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<type \'object\'>"
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], "
+    argspec: "args=[\'self\', \'axis\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'-1\', \'sum_over_batch_size\', \'None\'], "
   }
   member_method {
     name: "call"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.pbtxt
index 532d2b2f88..b71e89883e 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.pbtxt
@@ -74,11 +74,11 @@ tf_module {
   }
   member_method {
     name: "cosine"
-    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'y_true\', \'y_pred\', \'axis\'], varargs=None, keywords=None, defaults=[\'-1\'], "
   }
   member_method {
     name: "cosine_proximity"
-    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'y_true\', \'y_pred\', \'axis\'], varargs=None, keywords=None, defaults=[\'-1\'], "
   }
   member_method {
     name: "deserialize"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-cosine-proximity.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-cosine-proximity.pbtxt
index d095ed42ef..c50638740f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-cosine-proximity.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-cosine-proximity.pbtxt
@@ -89,7 +89,7 @@ tf_class {
   }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'cosine_proximity\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'dtype\', \'axis\'], varargs=None, keywords=None, defaults=[\'cosine_proximity\', \'None\', \'-1\'], "
   }
   member_method {
     name: "add_loss"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt
index 743a8ae34a..54f5b21f55 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt
@@ -122,11 +122,11 @@ tf_module {
   }
   member_method {
     name: "cosine"
-    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'y_true\', \'y_pred\', \'axis\'], varargs=None, keywords=None, defaults=[\'-1\'], "
   }
   member_method {
     name: "cosine_proximity"
-    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'y_true\', \'y_pred\', \'axis\'], varargs=None, keywords=None, defaults=[\'-1\'], "
   }
   member_method {
     name: "deserialize"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-cosine-proximity.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-cosine-proximity.pbtxt
index de35966f48..4952a76291 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-cosine-proximity.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.-cosine-proximity.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<type \'object\'>"
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'sum_over_batch_size\', \'None\'], "
+    argspec: "args=[\'self\', \'axis\', \'reduction\', \'name\'], varargs=None, keywords=None, defaults=[\'-1\', \'sum_over_batch_size\', \'None\'], "
   }
   member_method {
     name: "call"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt
index ef4fd66a7a..673593ed6c 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt
@@ -78,11 +78,11 @@ tf_module {
   }
   member_method {
     name: "cosine"
-    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'y_true\', \'y_pred\', \'axis\'], varargs=None, keywords=None, defaults=[\'-1\'], "
   }
   member_method {
     name: "cosine_proximity"
-    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'y_true\', \'y_pred\', \'axis\'], varargs=None, keywords=None, defaults=[\'-1\'], "
   }
   member_method {
     name: "deserialize"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-cosine-proximity.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-cosine-proximity.pbtxt
index d095ed42ef..c50638740f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-cosine-proximity.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-cosine-proximity.pbtxt
@@ -89,7 +89,7 @@ tf_class {
   }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'cosine_proximity\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'dtype\', \'axis\'], varargs=None, keywords=None, defaults=[\'cosine_proximity\', \'None\', \'-1\'], "
   }
   member_method {
     name: "add_loss"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt
index 743a8ae34a..54f5b21f55 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt
@@ -122,11 +122,11 @@ tf_module {
   }
   member_method {
     name: "cosine"
-    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'y_true\', \'y_pred\', \'axis\'], varargs=None, keywords=None, defaults=[\'-1\'], "
   }
   member_method {
     name: "cosine_proximity"
-    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'y_true\', \'y_pred\', \'axis\'], varargs=None, keywords=None, defaults=[\'-1\'], "
   }
   member_method {
     name: "deserialize"
-- 
GitLab


From 0d15dbe9c5df49c6a9757fedbe47f893837363db Mon Sep 17 00:00:00 2001
From: Mark Heffernan <meheff@google.com>
Date: Wed, 9 Jan 2019 12:31:46 -0800
Subject: [PATCH 0470/1095] Remove deprecated methods in ShapeUtil. No
 functional change. Replace with calls to Shape methods.

PiperOrigin-RevId: 228566953
---
 tensorflow/c/eager/c_api_debug.cc             |   4 +-
 .../compiler/xla/service/scatter_expander.cc  |   2 +-
 .../compiler/xla/service/shape_inference.cc   |   4 +-
 tensorflow/compiler/xla/shape.cc              |   4 +-
 tensorflow/compiler/xla/shape_util.cc         | 107 ++++++++----------
 tensorflow/compiler/xla/shape_util.h          |  44 +------
 tensorflow/compiler/xla/shape_util_test.cc    |   4 -
 tensorflow/compiler/xla/tests/tuple_test.cc   |   5 +-
 8 files changed, 61 insertions(+), 113 deletions(-)

diff --git a/tensorflow/c/eager/c_api_debug.cc b/tensorflow/c/eager/c_api_debug.cc
index 52b0824552..ffcd5ace0b 100644
--- a/tensorflow/c/eager/c_api_debug.cc
+++ b/tensorflow/c/eager/c_api_debug.cc
@@ -83,7 +83,7 @@ TF_CAPI_EXPORT extern TFE_TensorDebugInfo* TFE_TensorHandleTensorDebugInfo(
       }
     }
 
-    if (xla::ShapeUtil::IsTuple(padded_shape)) {
+    if (padded_shape.IsTuple()) {
       if (xla::ShapeUtil::TupleElementCount(padded_shape) != 2) {
         // Currently, the only case of XlaTensor containing a tuple shape is to
         // represent 64 bit ints, doubles, and complex numbers (we don't support
@@ -99,7 +99,7 @@ TF_CAPI_EXPORT extern TFE_TensorDebugInfo* TFE_TensorHandleTensorDebugInfo(
       xla::Shape shape0 = xla::ShapeUtil::GetTupleElementShape(padded_shape, 0);
       const xla::Shape& shape1 =
           xla::ShapeUtil::GetTupleElementShape(padded_shape, 1);
-      if (xla::ShapeUtil::IsTuple(shape0) || xla::ShapeUtil::IsTuple(shape1)) {
+      if (shape0.IsTuple() || shape1.IsTuple()) {
         status->status = tensorflow::errors::InvalidArgument(
             "XlaTensors should not contain nested tuples. Shape: ",
             padded_shape.DebugString());
diff --git a/tensorflow/compiler/xla/service/scatter_expander.cc b/tensorflow/compiler/xla/service/scatter_expander.cc
index 8b9955faf8..036c3c36f6 100644
--- a/tensorflow/compiler/xla/service/scatter_expander.cc
+++ b/tensorflow/compiler/xla/service/scatter_expander.cc
@@ -59,7 +59,7 @@ static StatusOr<HloInstruction*> CanonicalizeScatterIndices(
   TF_ASSIGN_OR_RETURN(
       HloInstruction * transposed_scatter_indices,
       TransposeIndexVectorDimToLast(scatter_indices, index_vector_dim));
-  if (ShapeUtil::Rank(scatter_indices->shape()) == index_vector_dim + 1 &&
+  if (scatter_indices->shape().rank() == index_vector_dim + 1 &&
       scatter_indices->shape().dimensions(index_vector_dim) == 1) {
     auto new_shape =
         ShapeUtil::DeleteDimension(index_vector_dim, scatter_indices->shape());
diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc
index b729dd7660..1d3f84af95 100644
--- a/tensorflow/compiler/xla/service/shape_inference.cc
+++ b/tensorflow/compiler/xla/service/shape_inference.cc
@@ -2358,7 +2358,7 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
   };
 
   // Check the shapes of computation parameters and return types.
-  if (!ShapeUtil::ShapeIs(condition.result(), PRED, {})) {
+  if (!ShapeUtil::Equal(condition.result(), ShapeUtil::MakeShape(PRED, {}))) {
     return InvalidArgument("Condition must return a boolean; got %s.",
                            shape_string());
   }
@@ -2378,7 +2378,7 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
     const Shape& predicate, const Shape& true_operand,
     const Shape& false_operand, const ProgramShape& true_computation,
     const ProgramShape& false_computation) {
-  if (!ShapeUtil::ShapeIs(predicate, PRED, {})) {
+  if (!ShapeUtil::Equal(predicate, ShapeUtil::MakeShape(PRED, {}))) {
     return InvalidArgument("Predicate must be a boolean; got %s.",
                            ShapeUtil::HumanString(predicate));
   }
diff --git a/tensorflow/compiler/xla/shape.cc b/tensorflow/compiler/xla/shape.cc
index ea4f722ef4..1a029efe85 100644
--- a/tensorflow/compiler/xla/shape.cc
+++ b/tensorflow/compiler/xla/shape.cc
@@ -80,7 +80,7 @@ string Shape::ToString(bool print_layout) const {
 }
 
 bool Shape::is_static() const {
-  if (ShapeUtil::IsTuple(*this)) {
+  if (IsTuple()) {
     for (const Shape& subshape : tuple_shapes_) {
       if (!subshape.is_static()) {
         return false;
@@ -91,7 +91,7 @@ bool Shape::is_static() const {
 }
 
 void Shape::DeleteDimension(int64 dim_to_delete) {
-  CHECK(ShapeUtil::IsArray(*this));
+  CHECK(IsArray());
   CHECK_GE(dim_to_delete, 0);
   CHECK_LT(dim_to_delete, dimensions_.size());
   dimensions_.erase(dimensions_.begin() + dim_to_delete);
diff --git a/tensorflow/compiler/xla/shape_util.cc b/tensorflow/compiler/xla/shape_util.cc
index 7f359311c3..235b065585 100644
--- a/tensorflow/compiler/xla/shape_util.cc
+++ b/tensorflow/compiler/xla/shape_util.cc
@@ -146,7 +146,7 @@ bool CompareShapes(const Shape& lhs, const Shape& rhs, bool compare_layouts,
     return false;
   }
 
-  for (int i = 0; i < ShapeUtil::Rank(lhs); ++i) {
+  for (int i = 0; i < lhs.rank(); ++i) {
     if (lhs.is_dynamic_dimension(i) != rhs.is_dynamic_dimension(i)) {
       VLOG(3)
           << "CompareShapes: lhs and rhs have different dynamic dimensions.";
@@ -208,11 +208,6 @@ StatusOr<Shape> MakeShapeWithLayoutInternal(
   return equal;
 }
 
-/* static */ int64 ShapeUtil::Rank(const Shape& shape) {
-  CHECK(shape.IsArray()) << "Non-arrays do not have a rank, shape: " << shape;
-  return shape.dimensions_size();
-}
-
 /* static */ int64 ShapeUtil::TrueRank(const Shape& shape) {
   int64 accum = 0;
   for (int64 dimension : shape.dimensions()) {
@@ -344,7 +339,7 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
 
 /* static */ void ShapeUtil::AppendMajorDimension(int bound, Shape* shape) {
   CHECK(LayoutUtil::IsDenseArray(*shape));
-  shape->mutable_layout()->add_minor_to_major(Rank(*shape));
+  shape->mutable_layout()->add_minor_to_major(shape->rank());
   shape->add_dimensions(bound);
   TF_DCHECK_OK(ValidateShape(*shape));
 }
@@ -359,7 +354,7 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
 }
 
 /* static */ bool ShapeUtil::ElementHasBitWidth(const Shape& shape, int bits) {
-  if (!IsArray(shape)) {
+  if (!shape.IsArray()) {
     return false;
   }
   return primitive_util::BitWidth(shape.element_type()) == bits;
@@ -401,26 +396,24 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
   return primitive_util::IsFloatingPointType(shape.element_type());
 }
 
-/* static */ bool ShapeUtil::IsArray(const Shape& shape) {
-  return IsArrayPrimitiveType(shape.element_type());
-}
-
 /* static */ bool ShapeUtil::IsNestedTuple(const Shape& shape) {
-  return IsTuple(shape) && absl::c_any_of(shape.tuple_shapes(), IsTuple);
+  return shape.IsTuple() &&
+         absl::c_any_of(shape.tuple_shapes(),
+                        [](const Shape& s) { return s.IsTuple(); });
 }
 
 /* static */ bool ShapeUtil::IsEmptyTuple(const Shape& shape) {
-  return IsTuple(shape) && TupleElementCount(shape) == 0;
+  return shape.IsTuple() && TupleElementCount(shape) == 0;
 }
 
 /* static */ int64 ShapeUtil::TupleElementCount(const Shape& shape) {
-  CHECK(IsTuple(shape)) << HumanString(shape);
+  CHECK(shape.IsTuple()) << HumanString(shape);
   return shape.tuple_shapes_size();
 }
 
 /* static */ const Shape& ShapeUtil::GetTupleElementShape(const Shape& shape,
                                                           int64 index) {
-  CHECK(IsTuple(shape));
+  CHECK(shape.IsTuple());
   CHECK_GT(TupleElementCount(shape), index);
   TF_DCHECK_OK(ValidateShapeWithOptionalLayout(shape.tuple_shapes(index)));
   return shape.tuple_shapes(index);
@@ -436,7 +429,7 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
 /* static */ Shape ShapeUtil::SliceTuple(const Shape& tuple, int64 start,
                                          int64 limit) {
   TF_DCHECK_OK(ValidateShapeWithOptionalLayout(tuple));
-  CHECK(IsTuple(tuple));
+  CHECK(tuple.IsTuple());
   CHECK_LE(start, TupleElementCount(tuple));
   CHECK_LE(limit, TupleElementCount(tuple));
 
@@ -453,15 +446,9 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
                                               complex_shape.element_type()));
 }
 
-/* static */ bool ShapeUtil::ShapeIs(const Shape& shape,
-                                     PrimitiveType element_type,
-                                     std::initializer_list<int64> dimensions) {
-  return Equal(shape, MakeShape(element_type, dimensions));
-}
-
 /* static */ int64 ShapeUtil::ElementsIn(const Shape& shape) {
-  DCHECK(IsArray(shape)) << ShapeUtil::HumanString(shape);
-  DCHECK_EQ(shape.dimensions_size(), Rank(shape));
+  DCHECK(shape.IsArray()) << ShapeUtil::HumanString(shape);
+  DCHECK_EQ(shape.dimensions_size(), shape.rank());
   if (shape.dimensions().size() == 1) {
     return shape.dimensions()[0];
   }
@@ -471,8 +458,8 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
 }
 
 /* static */ int64 ShapeUtil::ElementsInRecursive(const Shape& shape) {
-  CHECK(IsArray(shape) || IsTuple(shape));
-  if (IsArray(shape)) {
+  CHECK(shape.IsArray() || shape.IsTuple());
+  if (shape.IsArray()) {
     return ElementsIn(shape);
   }
   int64 count = 0;
@@ -505,7 +492,7 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
 }
 
 /* static */ string ShapeUtil::HumanString(const Shape& shape) {
-  if (IsTuple(shape)) {
+  if (shape.IsTuple()) {
     string text = "(";
     const char* prefix = "";
     for (const Shape& elem_shape : shape.tuple_shapes()) {
@@ -529,7 +516,7 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
 }
 
 /* static */ string ShapeUtil::HumanStringWithLayout(const Shape& shape) {
-  if (IsTuple(shape)) {
+  if (shape.IsTuple()) {
     string text = "(";
     const char* prefix = "";
     for (const Shape& elem_shape : shape.tuple_shapes()) {
@@ -545,7 +532,7 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
     StrAppend(&result, (i > 0) ? "," : "", shape.dimensions(i));
   }
   result += "]";
-  if (!IsScalar(shape) && IsArray(shape)) {
+  if (!IsScalar(shape) && shape.IsArray()) {
     if (LayoutUtil::HasLayout(shape)) {
       StrAppend(&result, LayoutUtil::HumanString(shape.layout()));
     }
@@ -580,8 +567,8 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
 
 /* static */ bool ShapeUtil::CompatibleIgnoringElementType(const Shape& lhs,
                                                            const Shape& rhs) {
-  if (IsArray(lhs)) {
-    return IsArray(rhs) && SameDimensions(lhs, rhs);
+  if (lhs.IsArray()) {
+    return rhs.IsArray() && SameDimensions(lhs, rhs);
   } else if (lhs.element_type() == TUPLE) {
     return rhs.element_type() == TUPLE &&
            absl::c_equal(lhs.tuple_shapes(), rhs.tuple_shapes(),
@@ -594,8 +581,8 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
 
 /* static */ bool ShapeUtil::CompatibleIgnoringFpPrecision(const Shape& lhs,
                                                            const Shape& rhs) {
-  if (IsArray(lhs)) {
-    return IsArray(rhs) && SameElementTypeIgnoringFpPrecision(lhs, rhs) &&
+  if (lhs.IsArray()) {
+    return rhs.IsArray() && SameElementTypeIgnoringFpPrecision(lhs, rhs) &&
            CompatibleIgnoringElementType(lhs, rhs);
   } else if (lhs.element_type() == TUPLE) {
     return rhs.element_type() == TUPLE &&
@@ -615,7 +602,7 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
 /* static */ int64 ShapeUtil::GetDimensionNumber(const Shape& shape,
                                                  int64 dimension_number) {
   if (dimension_number < 0) {
-    dimension_number += Rank(shape);
+    dimension_number += shape.rank();
   }
   CHECK_GE(dimension_number, 0);
   return dimension_number;
@@ -669,7 +656,7 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
   TF_DCHECK_OK(ValidateShape(shape));
   if (shape.element_type() == TUPLE) {
     return ByteSizeOfTupleIndexTable(shape, pointer_size);
-  } else if (IsArray(shape)) {
+  } else if (shape.IsArray()) {
     int64 byte_size = ByteSizeOfElements(shape);
     if (LayoutUtil::IsSparseArray(shape)) {
       byte_size += ByteSizeOfSparseIndices(shape);
@@ -755,10 +742,10 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
     return Status::OK();
   }
 
-  if (LayoutUtil::IsSparseArray(shape) && Rank(shape) == 0) {
+  if (LayoutUtil::IsSparseArray(shape) && shape.rank() == 0) {
     return InvalidArgument("sparse arrays must have rank > 0");
   }
-  for (int64 i = 0; i < Rank(shape); ++i) {
+  for (int64 i = 0; i < shape.rank(); ++i) {
     int64 dimension = shape.dimensions(i);
     if (dimension < 0) {
       return InvalidArgument(
@@ -774,7 +761,7 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
 /* static */ Status ShapeUtil::ValidateShapeSize(const Shape& shape) {
   VLOG(3) << "Validating shape size: " << ShapeUtil::HumanString(shape);
 
-  if (!IsArray(shape)) {
+  if (!shape.IsArray()) {
     return Status::OK();
   }
 
@@ -867,7 +854,7 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
                                           ShapeIndexView index) {
   const Shape* subshape = &shape;
   for (auto i : index) {
-    if (!IsTuple(*subshape) || i >= subshape->tuple_shapes_size() || i < 0) {
+    if (!subshape->IsTuple() || i >= subshape->tuple_shapes_size() || i < 0) {
       return false;
     }
     subshape = &subshape->tuple_shapes(i);
@@ -879,7 +866,7 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
                                                  ShapeIndexView index) {
   const Shape* return_shape = &shape;
   for (auto i : index) {
-    CHECK(IsTuple(*return_shape))
+    CHECK(return_shape->IsTuple())
         << "Invalid index " << index << " for shape " << shape;
     return_shape = &return_shape->tuple_shapes(i);
   }
@@ -890,7 +877,7 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
     const Shape& shape, ShapeIndexView index) {
   const Shape* return_shape = &shape;
   for (auto i : index) {
-    if (!IsTuple(*return_shape) || i < 0 ||
+    if (!return_shape->IsTuple() || i < 0 ||
         i >= return_shape->tuple_shapes_size()) {
       return InvalidArgument(
           "Shape index %s not a valid subshape index for tuple with shape %s",
@@ -905,7 +892,7 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
                                                   ShapeIndexView index) {
   Shape* return_shape = shape;
   for (auto i : index) {
-    CHECK(IsTuple(*return_shape));
+    CHECK(return_shape->IsTuple());
     return_shape = return_shape->mutable_tuple_shapes(i);
   }
   return return_shape;
@@ -913,11 +900,11 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
 
 /* static */
 bool ShapeUtil::IsLeafIndex(const Shape& shape, const ShapeIndex& index) {
-  return !IsTuple(GetSubshape(shape, index));
+  return !GetSubshape(shape, index).IsTuple();
 }
 
 /* static */ int64 ShapeUtil::GetLeafCount(const Shape& shape) {
-  if (!IsTuple(shape)) {
+  if (!shape.IsTuple()) {
     return 1;
   }
   int64 count = 0;
@@ -1081,8 +1068,8 @@ Status ForEachMutableSubshapeHelper(
 /* static */ std::tuple<bool, std::vector<int64>, std::vector<int64>>
 ShapeUtil::InsertedOrDeleted1SizedDimensions(const Shape& shape_pre,
                                              const Shape& shape_post) {
-  CHECK(IsArray(shape_pre));
-  CHECK(IsArray(shape_post));
+  CHECK(shape_pre.IsArray());
+  CHECK(shape_post.IsArray());
 
   auto nil = std::make_tuple(false, std::vector<int64>(), std::vector<int64>());
 
@@ -1129,7 +1116,7 @@ ShapeUtil::InsertedOrDeleted1SizedDimensions(const Shape& shape_pre,
     auto unmodified_dim_pair =
         i < unmodified_dims.size()
             ? unmodified_dims[i]
-            : std::make_pair(Rank(shape_pre), Rank(shape_post));
+            : std::make_pair(shape_pre.rank(), shape_post.rank());
     if (!check_modified_dims(prior_unmodified_dim_pair, unmodified_dim_pair)) {
       return nil;
     }
@@ -1141,8 +1128,8 @@ ShapeUtil::InsertedOrDeleted1SizedDimensions(const Shape& shape_pre,
 /* static */ std::vector<std::pair<int64, int64>>
 ShapeUtil::DimensionsUnmodifiedByReshape(const Shape& input_shape,
                                          const Shape& output_shape) {
-  CHECK(IsArray(input_shape));
-  CHECK(IsArray(output_shape));
+  CHECK(input_shape.IsArray());
+  CHECK(output_shape.IsArray());
 
   // Unmodified dimensions are merely common factors of rank 1.
   auto common_factors = CommonFactors(AsInt64Slice(input_shape.dimensions()),
@@ -1192,8 +1179,8 @@ ShapeUtil::DimensionsUnmodifiedByReshape(const Shape& input_shape,
 
 /* static */ bool ShapeUtil::ReshapeIsBitcast(const Shape& input_shape,
                                               const Shape& output_shape) {
-  CHECK(IsArray(input_shape));
-  CHECK(IsArray(output_shape));
+  CHECK(input_shape.IsArray());
+  CHECK(output_shape.IsArray());
   CHECK(LayoutUtil::HasLayout(input_shape));
   CHECK(LayoutUtil::HasLayout(output_shape));
 
@@ -1321,12 +1308,12 @@ ShapeUtil::DimensionsUnmodifiedByReshape(const Shape& input_shape,
     Shape output_shape_dim0_major = MakeShapeWithDescendingLayout(
         output_shape.element_type(), AsInt64Slice(output_shape.dimensions()));
 
-    for (int64 input_dim = 0; input_dim < Rank(input_shape); ++input_dim) {
+    for (int64 input_dim = 0; input_dim < input_shape.rank(); ++input_dim) {
       if (input_shape.dimensions(input_dim) <= 1) {
         continue;
       }
 
-      std::vector<int64> input_unit_index(Rank(input_shape), 0);
+      std::vector<int64> input_unit_index(input_shape.rank(), 0);
       input_unit_index[input_dim] = 1;
       int64 logical_linear_index =
           IndexUtil::MultidimensionalIndexToLinearIndex(input_shape_dim0_major,
@@ -1352,11 +1339,11 @@ ShapeUtil::DimensionsUnmodifiedByReshape(const Shape& input_shape,
 
 /* static */ absl::optional<Shape> ShapeUtil::AlignLayouts(
     const Shape& input_shape, const Shape& output_shape) {
-  CHECK(IsArray(input_shape));
-  CHECK(IsArray(output_shape));
+  CHECK(input_shape.IsArray());
+  CHECK(output_shape.IsArray());
 
-  int64 input_rank = Rank(input_shape);
-  int64 output_rank = Rank(output_shape);
+  int64 input_rank = input_shape.rank();
+  int64 output_rank = output_shape.rank();
 
   // First, calculate an alignment of the dimensions. A consecutive sequence of
   // input dimensions and output dimensions belong to the same alignment part if
@@ -1493,14 +1480,14 @@ ShapeUtil::DimensionsUnmodifiedByReshape(const Shape& input_shape,
 
 /* static */ Shape ShapeUtil::DeleteDimension(int64 dim_to_delete,
                                               Shape shape) {
-  CHECK(IsArray(shape));
+  CHECK(shape.IsArray());
   shape.DeleteDimension(dim_to_delete);
   return shape;
 }
 
 /* static */ Shape ShapeUtil::FilterDimensions(
     const std::function<bool(int64)>& p, Shape shape) {
-  CHECK(IsArray(shape));
+  CHECK(shape.IsArray());
   std::vector<int64> dims_to_delete;
   for (int64 i = shape.dimensions().size() - 1; i >= 0; --i) {
     if (!p(i)) {
diff --git a/tensorflow/compiler/xla/shape_util.h b/tensorflow/compiler/xla/shape_util.h
index c8295e85ce..e98c6e024b 100644
--- a/tensorflow/compiler/xla/shape_util.h
+++ b/tensorflow/compiler/xla/shape_util.h
@@ -185,7 +185,7 @@ class ShapeUtil {
   // may not actually be able to store this number of elements. See
   // LayoutUtil::MaxSparseElements(shape) to obtain the maximum number of
   // elements that can be stored in a sparse shape.
-  // Precondition: IsArray(shape)
+  // Precondition: shape.IsArray()
   static int64 ElementsIn(const Shape& shape);
 
   // As ElementsIn(), but recurses through tuples.
@@ -296,11 +296,6 @@ class ShapeUtil {
   // As Equal, but allow one of lhs and rhs to be F16 while the other is F32.
   static bool EqualIgnoringFpPrecision(const Shape& lhs, const Shape& rhs);
 
-  // Returns the rank (number of dimensions) of the given shape.
-  // Precondition: !IsTuple(shape)
-  ABSL_DEPRECATED("Use `Shape::rank` instead.")
-  static int64 Rank(const Shape& shape);
-
   // Returns the number of dimensions for which the dimension is not (trivially)
   // 1. e.g., f32[2x1x1] has a true rank of 1D, the other dimensions are just
   // fluff. Note that zero dimensions are included in the true rank, e.g.,
@@ -314,10 +309,10 @@ class ShapeUtil {
   // Scalar-specific
 
   static bool IsScalar(const Shape& shape) {
-    return IsArray(shape) && Rank(shape) == 0;
+    return shape.IsArray() && shape.rank() == 0;
   }
   static bool IsEffectiveScalar(const Shape& shape) {
-    return IsArray(shape) && TrueRank(shape) == 0;
+    return shape.IsArray() && TrueRank(shape) == 0;
   }
 
   // Returns whether "shape" is a scalar (array) with the given element_type.
@@ -457,31 +452,6 @@ class ShapeUtil {
   // that floating point numbers are signed.
   static bool ElementIsSigned(const Shape& shape);
 
-  // Returns whether the shape is a tuple.
-  ABSL_DEPRECATED("Use Shape::IsTuple instead.")
-  static bool IsTuple(const Shape& shape) {
-    return shape.element_type() == TUPLE;
-  }
-
-  // Returns whether the shape is an opaque value (i.e. an 'existential' typed
-  // value that is passed to CustomCall operations).
-  ABSL_DEPRECATED("Use Shape::IsOpaque instead.")
-  static bool IsOpaque(const Shape& shape) {
-    return shape.element_type() == OPAQUE;
-  }
-
-  // Returns whether the shape is an token value used for ordering
-  // side-effecting operations.
-  ABSL_DEPRECATED("Use Shape::IsToken instead.")
-  static bool IsToken(const Shape& shape) {
-    return shape.element_type() == TOKEN;
-  }
-
-  // Returns whether the shape is an array.  Note that scalars are considered
-  // arrays.
-  ABSL_DEPRECATED("Use Shape::IsArray instead.")
-  static bool IsArray(const Shape& shape);
-
   // Returns whether the given primitive type corresponds to an array shape.
   static bool IsArrayPrimitiveType(PrimitiveType primitive_type);
 
@@ -511,12 +481,6 @@ class ShapeUtil {
   // shape.
   static Shape ComplexComponentShape(const Shape& complex_shape);
 
-  // Shorthand for testing whether a shape is of a given element type and
-  // sequence of dimensions.
-  ABSL_DEPRECATED("Use Equal() instead.")
-  static bool ShapeIs(const Shape& shape, PrimitiveType element_type,
-                      std::initializer_list<int64> dimensions);
-
   // Returns true if the given shape has a subshape at the given index.
   static bool IndexIsValid(const Shape& shape, ShapeIndexView index);
 
@@ -764,7 +728,7 @@ class ShapeUtil {
     if (ShapeUtil::IsZeroElementArray(shape)) {
       return Status::OK();
     }
-    CHECK_EQ(Rank(shape), base.size());
+    CHECK_EQ(shape.rank(), base.size());
     CHECK_EQ(incr.size(), base.size());
     CHECK_EQ(count.size(), base.size());
     const int64 rank = LayoutUtil::MinorToMajor(shape).size();
diff --git a/tensorflow/compiler/xla/shape_util_test.cc b/tensorflow/compiler/xla/shape_util_test.cc
index 8e7c208193..61b4e73e06 100644
--- a/tensorflow/compiler/xla/shape_util_test.cc
+++ b/tensorflow/compiler/xla/shape_util_test.cc
@@ -538,10 +538,6 @@ TEST(ShapeUtilTest, InsertedOrDeleted1SizedDimensions) {
       ShapeUtil::InsertedOrDeleted1SizedDimensions(shape0, shape2)));
 }
 
-TEST(ShapeUtilTest, ShapeIs) {
-  EXPECT_FALSE(ShapeUtil::ShapeIs(ShapeUtil::MakeShape(PRED, {2}), PRED, {}));
-}
-
 TEST(ShapeUtilTest, ForEachIndex) {
   struct ShapeDimensionAndNumberInvocations {
     std::vector<int64> dimensions;
diff --git a/tensorflow/compiler/xla/tests/tuple_test.cc b/tensorflow/compiler/xla/tests/tuple_test.cc
index 9c586bdeb0..426d6c84ee 100644
--- a/tensorflow/compiler/xla/tests/tuple_test.cc
+++ b/tensorflow/compiler/xla/tests/tuple_test.cc
@@ -176,8 +176,9 @@ XLA_TEST_F(TupleTest, AddTupleElements) {
       {2.f, 4.f, 6.f},  // row 0
       {5.f, 7.f, 9.f},  // row 1
   });
-  ASSERT_TRUE(ShapeUtil::ShapeIs(vector_shape, F32, {3}));
-  ASSERT_TRUE(ShapeUtil::ShapeIs(matrix_shape, F32, {/*y=*/2, /*x=*/3}));
+  ASSERT_TRUE(ShapeUtil::Equal(vector_shape, ShapeUtil::MakeShape(F32, {3})));
+  ASSERT_TRUE(ShapeUtil::Equal(matrix_shape,
+                               ShapeUtil::MakeShape(F32, {/*y=*/2, /*x=*/3})));
   ComputeAndCompareR2<float>(&builder, expected, {}, error_spec_);
 }
 
-- 
GitLab


From 4053e17dbb7a1be85553e5173c2e08fb9653d33f Mon Sep 17 00:00:00 2001
From: Pete Warden <petewarden@google.com>
Date: Wed, 9 Jan 2019 12:33:36 -0800
Subject: [PATCH 0471/1095] Generate IDE projects automatically for various
 platforms

PiperOrigin-RevId: 228567285
---
 tensorflow/lite/experimental/micro/BUILD      |   4 +
 tensorflow/lite/experimental/micro/README.md  | 259 ++++++++++++++----
 .../experimental/micro/bluepill/debug_log.cc  |  27 ++
 .../lite/experimental/micro/debug_log.cc      |  41 +++
 .../lite/experimental/micro/debug_log.h       |  23 ++
 .../experimental/micro/debug_log_numbers.cc   | 185 +++++++++++++
 .../experimental/micro/debug_log_numbers.h    |  28 ++
 .../examples/micro_speech/CMSIS/Makefile.inc  |  43 +++
 .../micro/examples/micro_speech/Makefile.inc  | 183 +++++--------
 .../micro_speech/disco_f746ng/Makefile.inc    |   7 +
 .../disco_f746ng/audio_provider.cc            | 182 ++++++++++++
 .../micro_speech/disco_f746ng/timer.cc        |  24 ++
 .../examples/micro_speech/osx/Makefile.inc    |   6 +-
 .../examples/micro_speech/preprocessor.cc     |   9 +-
 .../lite/experimental/micro/mbed/debug_log.cc |  24 ++
 .../experimental/micro/micro_error_reporter.h |  22 +-
 .../micro/riscv32_mcu/debug_log.cc            | 172 ------------
 .../experimental/micro/tools/make/Makefile    | 101 ++++---
 .../micro/tools/make/helper_functions.inc     | 117 ++++++++
 .../tools/make/targets/bluepill_makefile.inc  |   5 +-
 .../tools/make/targets/mbed_makefile.inc      |   4 +
 .../micro/tools/make/targets/osx_makefile.inc |   2 +-
 .../make/templates/AUDIO_DISCO_F746NG.lib.tpl |   1 +
 .../make/templates/BSP_DISCO_F746NG.lib.tpl   |   1 +
 .../micro/tools/make/templates/Makefile.tpl   |  26 ++
 .../tools/make/templates/README_MAKE.md.tpl   |  29 ++
 .../tools/make/templates/README_MBED.md.tpl   |  48 ++++
 .../make/templates/SDRAM_DISCO_F746NG.lib.tpl |   1 +
 .../tools/make/templates/mbed-os.lib.tpl      |   1 +
 .../tools/make/templates/mbed_app.json.tpl    |   7 +
 30 files changed, 1170 insertions(+), 412 deletions(-)
 create mode 100644 tensorflow/lite/experimental/micro/bluepill/debug_log.cc
 create mode 100644 tensorflow/lite/experimental/micro/debug_log.cc
 create mode 100644 tensorflow/lite/experimental/micro/debug_log.h
 create mode 100644 tensorflow/lite/experimental/micro/debug_log_numbers.cc
 create mode 100644 tensorflow/lite/experimental/micro/debug_log_numbers.h
 create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/Makefile.inc
 create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/disco_f746ng/Makefile.inc
 create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/disco_f746ng/audio_provider.cc
 create mode 100644 tensorflow/lite/experimental/micro/examples/micro_speech/disco_f746ng/timer.cc
 create mode 100644 tensorflow/lite/experimental/micro/mbed/debug_log.cc
 create mode 100644 tensorflow/lite/experimental/micro/tools/make/helper_functions.inc
 create mode 100644 tensorflow/lite/experimental/micro/tools/make/targets/mbed_makefile.inc
 create mode 100644 tensorflow/lite/experimental/micro/tools/make/templates/AUDIO_DISCO_F746NG.lib.tpl
 create mode 100644 tensorflow/lite/experimental/micro/tools/make/templates/BSP_DISCO_F746NG.lib.tpl
 create mode 100644 tensorflow/lite/experimental/micro/tools/make/templates/Makefile.tpl
 create mode 100644 tensorflow/lite/experimental/micro/tools/make/templates/README_MAKE.md.tpl
 create mode 100644 tensorflow/lite/experimental/micro/tools/make/templates/README_MBED.md.tpl
 create mode 100644 tensorflow/lite/experimental/micro/tools/make/templates/SDRAM_DISCO_F746NG.lib.tpl
 create mode 100644 tensorflow/lite/experimental/micro/tools/make/templates/mbed-os.lib.tpl
 create mode 100644 tensorflow/lite/experimental/micro/tools/make/templates/mbed_app.json.tpl

diff --git a/tensorflow/lite/experimental/micro/BUILD b/tensorflow/lite/experimental/micro/BUILD
index e11159868e..2d00ef76f4 100644
--- a/tensorflow/lite/experimental/micro/BUILD
+++ b/tensorflow/lite/experimental/micro/BUILD
@@ -12,6 +12,8 @@ load(
 cc_library(
     name = "micro_framework",
     srcs = [
+        "debug_log.cc",
+        "debug_log_numbers.cc",
         "micro_error_reporter.cc",
         "micro_interpreter.cc",
         "micro_mutable_op_resolver.cc",
@@ -19,6 +21,8 @@ cc_library(
     ],
     hdrs = [
         "compatibility.h",
+        "debug_log.h",
+        "debug_log_numbers.h",
         "micro_error_reporter.h",
         "micro_interpreter.h",
         "micro_mutable_op_resolver.h",
diff --git a/tensorflow/lite/experimental/micro/README.md b/tensorflow/lite/experimental/micro/README.md
index 464c7b6ad7..173f91fe90 100644
--- a/tensorflow/lite/experimental/micro/README.md
+++ b/tensorflow/lite/experimental/micro/README.md
@@ -1,46 +1,142 @@
 # TensorFlow Lite for Microcontrollers
 
-This an experimental port of TensorFlow Lite aimed at micro controllers and other devices with only kilobytes of memory. It doesn't require any operating system support, any standard C or C++ libraries, or dynamic memory allocation, so it's designed to be portable even to 'bare metal' systems. The core runtime fits in 16KB on a Cortex M3, and with enough operators to run a speech keyword detection model, takes up a total of 22KB.
+This an experimental port of TensorFlow Lite aimed at micro controllers and
+other devices with only kilobytes of memory. It doesn't require any operating
+system support, any standard C or C++ libraries, or dynamic memory allocation,
+so it's designed to be portable even to 'bare metal' systems. The core runtime
+fits in 16KB on a Cortex M3, and with enough operators to run a speech keyword
+detection model, takes up a total of 22KB.
 
 The design goals are for the framework to be:
 
-- **Readable**: We want embedded software engineers to be able to understand what's required to run ML inference without having to study research papers. We've tried to keep the code base small, modular, and have reference implementations of all operations to help with this.
-
-- **Easy to modify**: We know that there are a lot of different platforms and requirements in the embedded world, and we don't expect to cover all of them in one framework. Instead, we're hoping that it can be a good starting point for developers to build on top of to meet their own needs. For example, we tried to make it easy to replace the implementations of key computational operators that are often crucial for performance, without having to touch the data flow and other runtime code. We want it to make more sense to use our workflow to handle things like model import and less-important operations, and customize the parts that matter, rather than having to reimplement everything in your own engine.
-
-- **Well-tested**: If you're modifying code, you need to know if your changes are correct. Having an easy way to test lets you develop much faster. To help there, we've written tests for all the components, and we've made sure that the tests can be run on almost any platform, with no dependencies apart from the ability to log text to a debug console somewhere. We also provide an easy way to run all the tests on-device as part of an automated test framework, and we use qemu/Renode emulation so that tests can be run even without physical devices present.
-
-- **Easy to integrate**: We want to be as open a system as possible, and use the best code available for each platform. To do that, we're going to rely on projects like [CMSIS-NN](https://www.keil.com/pack/doc/CMSIS/NN/html/index.html), [uTensor](https://github.com/uTensor/uTensor), and other vendor libraries to handle as much performance-critical code as possible. We know that there are an increasing number of options to accelerate neural networks on microcontrollers, so we're aiming to be a good host for deploying those hardware technologies too.
-
-- **Compatible**: We're using the same file schema, interpreter API, and kernel interface as regular TensorFlow Lite, so we leverage the large existing set of tools, documentation, and examples for the project. The biggest barrier to deploying ML models is getting them from a training environment into a form that's easy to run inference on, so we see reusing this rich ecosystem as being crucial to being easily usable. We also hope to integrate this experimental work back into the main codebase in the future.
+-   **Readable**: We want embedded software engineers to be able to understand
+    what's required to run ML inference without having to study research papers.
+    We've tried to keep the code base small, modular, and have reference
+    implementations of all operations to help with this.
+
+-   **Easy to modify**: We know that there are a lot of different platforms and
+    requirements in the embedded world, and we don't expect to cover all of them
+    in one framework. Instead, we're hoping that it can be a good starting point
+    for developers to build on top of to meet their own needs. For example, we
+    tried to make it easy to replace the implementations of key computational
+    operators that are often crucial for performance, without having to touch
+    the data flow and other runtime code. We want it to make more sense to use
+    our workflow to handle things like model import and less-important
+    operations, and customize the parts that matter, rather than having to
+    reimplement everything in your own engine.
+
+-   **Well-tested**: If you're modifying code, you need to know if your changes
+    are correct. Having an easy way to test lets you develop much faster. To
+    help there, we've written tests for all the components, and we've made sure
+    that the tests can be run on almost any platform, with no dependencies apart
+    from the ability to log text to a debug console somewhere. We also provide
+    an easy way to run all the tests on-device as part of an automated test
+    framework, and we use qemu/Renode emulation so that tests can be run even
+    without physical devices present.
+
+-   **Easy to integrate**: We want to be as open a system as possible, and use
+    the best code available for each platform. To do that, we're going to rely
+    on projects like
+    [CMSIS-NN](https://www.keil.com/pack/doc/CMSIS/NN/html/index.html),
+    [uTensor](https://github.com/uTensor/uTensor), and other vendor libraries to
+    handle as much performance-critical code as possible. We know that there are
+    an increasing number of options to accelerate neural networks on
+    microcontrollers, so we're aiming to be a good host for deploying those
+    hardware technologies too.
+
+-   **Compatible**: We're using the same file schema, interpreter API, and
+    kernel interface as regular TensorFlow Lite, so we leverage the large
+    existing set of tools, documentation, and examples for the project. The
+    biggest barrier to deploying ML models is getting them from a training
+    environment into a form that's easy to run inference on, so we see reusing
+    this rich ecosystem as being crucial to being easily usable. We also hope to
+    integrate this experimental work back into the main codebase in the future.
 
 To meet those goals, we've made some tradeoffs:
 
-- **Simple C++**: To help with readability, our code is written in a modern version of C++, but we generally treat it as a "better C", rather relying on more complex features such as template meta-programming. As mentioned earlier, we avoid any use of dynamic memory allocation (new/delete) or the standard C/C++ libraries, so we believe this should still be fairly portable. It does mean that some older devices with C-only toolchains won't be supported, but we're hoping that the reference operator implementations (which are simple C-like functions) can still be useful in those cases. The interfaces are also designed to be C-only, so it should be possible to integrate the resulting library with pure C projects.
-
-- **Interpreted**: Code generation is a popular pattern for embedded code, because it gives standalone code that's easy to modify and step through, but we've chosen to go with an interpreted approach. In our internal microcontroller work we've found that using an extremely stripped-down interpreter with almost no dependencies gives us a lot of the same advantages, but is easier to maintain. For example, when new updates come out for the underlying library, you can just merge your local modifications in a single step, rather than having to regenerate new code and then patch in any changes you subsequently made. The coarse granularity of the interpreted primitives means that each operation call typically takes hundreds of thousands of instruction cycles at least, so we don't see noticeable performance gains from avoiding what's essentially a single switch statement at the interpreter level to call each operation. We're still working on improving the packaging though, for example we're considering having the ability to snapshot all the source files and headers used for a particular model, being able to compile the code and data together as a library, and then access it through a minimal set of C interface calls which hide the underlying complexity.
-
-- **Flatbuffers**: We represent our models using [the standard flatbuffer schema used by the rest of TensorFlow Lite](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/schema/schema.fbs), with the difference that we always keep it in read-only program memory (typically flash) rather than relying on having a file system to read it from. This is a good fit because flatbuffer's serialized format is designed to be mapped into memory without requiring any extra memory allocations or modifications to access it. All of the functions to read model values work directly on the serialized bytes, and large sections of data like weights are directly accessible as sequential C-style arrays of their data type, with no strides or unpacking needed. We do get a lot of value from using flatbuffers, but there is a cost in complexity. The flat buffer library code is all inline [inside the main headers](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/schema/schema_generated.h), but it isn't straightforward to inspect their implementations, and the model data structures aren't easy to comprehend from the debugger. The header for the schema itself also has to be periodically updated when new information is added to the file format, though we try to handle that transparently for most developers by checking in a pre-generated version.
-
-- **Code Duplication**: Some of the code in this prototype largely duplicates the logic in other parts of the TensorFlow Lite code base, for example the operator wrappers. We've tried to keep share as much as we can between the two interpreters, but there are some assumptions built into the original runtime that make this difficult. We'll be working on modularizing the main interpreter so that we can move to an entirely shared system.
-
-This initial preview release is designed to get early feedback, and is not intended to be a final product. It only includes enough operations to run a simple keyword recognition model, and the implementations are not optimized. We're hoping this will be a good way to get feedback and collaborate to improve the framework.
-
-## Getting Started
+-   **Simple C++**: To help with readability, our code is written in a modern
+    version of C++, but we generally treat it as a "better C", rather relying on
+    more complex features such as template meta-programming. As mentioned
+    earlier, we avoid any use of dynamic memory allocation (new/delete) or the
+    standard C/C++ libraries, so we believe this should still be fairly
+    portable. It does mean that some older devices with C-only toolchains won't
+    be supported, but we're hoping that the reference operator implementations
+    (which are simple C-like functions) can still be useful in those cases. The
+    interfaces are also designed to be C-only, so it should be possible to
+    integrate the resulting library with pure C projects.
+
+-   **Interpreted**: Code generation is a popular pattern for embedded code,
+    because it gives standalone code that's easy to modify and step through, but
+    we've chosen to go with an interpreted approach. In our internal
+    microcontroller work we've found that using an extremely stripped-down
+    interpreter with almost no dependencies gives us a lot of the same
+    advantages, but is easier to maintain. For example, when new updates come
+    out for the underlying library, you can just merge your local modifications
+    in a single step, rather than having to regenerate new code and then patch
+    in any changes you subsequently made. The coarse granularity of the
+    interpreted primitives means that each operation call typically takes
+    hundreds of thousands of instruction cycles at least, so we don't see
+    noticeable performance gains from avoiding what's essentially a single
+    switch statement at the interpreter level to call each operation. We're
+    still working on improving the packaging though, for example we're
+    considering having the ability to snapshot all the source files and headers
+    used for a particular model, being able to compile the code and data
+    together as a library, and then access it through a minimal set of C
+    interface calls which hide the underlying complexity.
+
+-   **Flatbuffers**: We represent our models using
+    [the standard flatbuffer schema used by the rest of TensorFlow Lite](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/schema/schema.fbs),
+    with the difference that we always keep it in read-only program memory
+    (typically flash) rather than relying on having a file system to read it
+    from. This is a good fit because flatbuffer's serialized format is designed
+    to be mapped into memory without requiring any extra memory allocations or
+    modifications to access it. All of the functions to read model values work
+    directly on the serialized bytes, and large sections of data like weights
+    are directly accessible as sequential C-style arrays of their data type,
+    with no strides or unpacking needed. We do get a lot of value from using
+    flatbuffers, but there is a cost in complexity. The flat buffer library code
+    is all inline
+    [inside the main headers](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/schema/schema_generated.h),
+    but it isn't straightforward to inspect their implementations, and the model
+    data structures aren't easy to comprehend from the debugger. The header for
+    the schema itself also has to be periodically updated when new information
+    is added to the file format, though we try to handle that transparently for
+    most developers by checking in a pre-generated version.
+
+-   **Code Duplication**: Some of the code in this prototype largely duplicates
+    the logic in other parts of the TensorFlow Lite code base, for example the
+    operator wrappers. We've tried to keep share as much as we can between the
+    two interpreters, but there are some assumptions built into the original
+    runtime that make this difficult. We'll be working on modularizing the main
+    interpreter so that we can move to an entirely shared system.
+
+This initial preview release is designed to get early feedback, and is not
+intended to be a final product. It only includes enough operations to run a
+simple keyword recognition model, and the implementations are not optimized.
+We're hoping this will be a good way to get feedback and collaborate to improve
+the framework.
+
+## Getting Started with Make
 
 Building requires a Linux or OS X machine.
 
- - Open a terminal
- - Download the TensorFlow source with `git clone https://github.com/tensorflow/tensorflow.git`
- - Enter the source root directory by running `cd tensorflow`
- - Download the dependencies by running `tensorflow/lite/experimental/micro/tools/make/download_dependencies.sh`. This may take a few minutes
- - Build and test the library with `make -f tensorflow/lite/experimental/micro/tools/make/Makefile test`
+-   Open a terminal
+-   Download the TensorFlow source with `git clone
+    https://github.com/tensorflow/tensorflow.git`
+-   Enter the source root directory by running `cd tensorflow`
+-   Download the dependencies by running
+    `tensorflow/lite/experimental/micro/tools/make/download_dependencies.sh`.
+    This may take a few minutes
+-   Build and test the library with `make -f
+    tensorflow/lite/experimental/micro/tools/make/Makefile test`
 
 You should see a series of compilation steps, followed by `~~~ALL TESTS
 PASSED~~~` for the various tests of the code that it will run. If there's an
 error, you should get an informative message from make about what went wrong.
 
-These tests are all built as simple binaries with few dependencies, so you can run them manually. For example, here's how to run the depthwise convolution test, and its output:
+These tests are all built as simple binaries with few dependencies, so you can
+run them manually. For example, here's how to run the depthwise convolution
+test, and its output:
 
 ```
 tensorflow/lite/experimental/micro/tools/make/gen/linux_x86_64/bin/tensorflow/lite/experimental/micro/kernels/depthwise_conv_test
@@ -53,7 +149,9 @@ Testing SimpleTestReluQuantized
 ~ALL TESTS PASSED~~~
 ```
 
-Looking at the [depthwise_conv_test.cc](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/experimental/micro/kernels/depthwise_conv_test.cc) code, you'll see a sequence that looks like this:
+Looking at the
+[depthwise_conv_test.cc](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/experimental/micro/kernels/depthwise_conv_test.cc)
+code, you'll see a sequence that looks like this:
 
 ```
 ...
@@ -74,19 +172,41 @@ output, and the test harness that runs the binary during the make process knows
 that everything ran correctly. If there's an error, the lack of the expected
 string lets the harness know that the test failed.
 
-So, why are we running tests in this complicated way? So far, we've been building binaries that run locally on the Mac OS or Linux machine you're building on, but this approach becomes important when we're targeting simple micro controller devices.
+So, why are we running tests in this complicated way? So far, we've been
+building binaries that run locally on the Mac OS or Linux machine you're
+building on, but this approach becomes important when we're targeting simple
+micro controller devices.
 
 ## Building for the "Blue Pill" STM32F103
 
-The goal of this library is to enable machine learning on resource-constrained micro controllers and DSPs, and as part of that we've targeted the ["Blue Pill" STM32F103-compatible development board](https://github.com/google/stm32_bare_lib) as a cheap and popular platform. It only has 20KB of RAM and 64KB of flash, so it's a good device to ensure we can run efficiently on small chips.
-
-It's fairly easy to [buy and wire up a physical board](https://github.com/google/stm32_bare_lib#wiring-up-your-blue-pill), but even if you don't have an actual device, the [Renode project](https://renode.io/) makes it easy to run a faithful emulation on your desktop machine. You'll need [Docker](https://www.docker.com/) installed, but once you have that set up, try running the following command:
-
-`make -f tensorflow/lite/experimental/micro/tools/make/Makefile TARGET=bluepill test`
-
-You should see a similar set of outputs as you did in the previous section, with the addition of some extra Docker logging messages. These are because we're using Docker to run the Renode micro controller emulation tool, and the tests themselves are being run on a simulated STM32F103 device. The communication channels between an embedded device and the host are quite limited, so the test harness looks at the output of the debug log to see if tests have passed, just as it did in the previous section. This makes it a very flexible way to run cross-platform tests, even when a platform has no operating system facilities, as long as it can output debugging text logs.
-
-To understand what's happening here, try running the same depthwise convolution test, but through the emulated device test harness, with the following command:
+The goal of this library is to enable machine learning on resource-constrained
+micro controllers and DSPs, and as part of that we've targeted the
+["Blue Pill" STM32F103-compatible development board](https://github.com/google/stm32_bare_lib)
+as a cheap and popular platform. It only has 20KB of RAM and 64KB of flash, so
+it's a good device to ensure we can run efficiently on small chips.
+
+It's fairly easy to
+[buy and wire up a physical board](https://github.com/google/stm32_bare_lib#wiring-up-your-blue-pill),
+but even if you don't have an actual device, the
+[Renode project](https://renode.io/) makes it easy to run a faithful emulation
+on your desktop machine. You'll need [Docker](https://www.docker.com/)
+installed, but once you have that set up, try running the following command:
+
+`make -f tensorflow/lite/experimental/micro/tools/make/Makefile TARGET=bluepill
+test`
+
+You should see a similar set of outputs as you did in the previous section, with
+the addition of some extra Docker logging messages. These are because we're
+using Docker to run the Renode micro controller emulation tool, and the tests
+themselves are being run on a simulated STM32F103 device. The communication
+channels between an embedded device and the host are quite limited, so the test
+harness looks at the output of the debug log to see if tests have passed, just
+as it did in the previous section. This makes it a very flexible way to run
+cross-platform tests, even when a platform has no operating system facilities,
+as long as it can output debugging text logs.
+
+To understand what's happening here, try running the same depthwise convolution
+test, but through the emulated device test harness, with the following command:
 
 ```
 tensorflow/lite/experimental/micro/testing/test_bluepill_binary.sh \
@@ -115,7 +235,7 @@ LOGS:
 03:27:32.4834 [DEBUG] cpu.uartSemihosting: [+0.18ms host +0s virt 0s virt from start]   Testing SimpleTestReluQuantized
 03:27:32.4838 [DEBUG] cpu.uartSemihosting: [+0.4ms host +0s virt 0s virt from start]   4/4 tests passed
 03:27:32.4839 [DEBUG] cpu.uartSemihosting: [+41µs host +0s virt 0s virt from start]   ~~~ALL TESTS PASSED~~~
-03:27:32.4839 [DEBUG] cpu.uartSemihosting: [+5µs host +0s virt 0s virt from start]   
+03:27:32.4839 [DEBUG] cpu.uartSemihosting: [+5µs host +0s virt 0s virt from start]
 ...
 tensorflow/lite/experimental/micro/tools/make/gen/bluepill_cortex-m3/bin/tensorflow/lite/experimental/micro/kernels/depthwise_conv_test: PASS
 ```
@@ -128,16 +248,19 @@ than your desktop. We hope that the simplicity of this testing approach will
 help make adding support for new platforms as easy as possible.
 
 ## Building for "Hifive1" SiFive FE310 development board
-We've targeted the ["HiFive1" Arduino-compatible development board](https://www.sifive.com/boards/hifive1) as a test platform for RISC-V MCU.
 
-Similar to Blue Pill setup, you will need Docker installed. The binary can be executed on either HiFive1 board or emulated using [Renode project](https://renode.io/) on your desktop machine.
+We've targeted the
+["HiFive1" Arduino-compatible development board](https://www.sifive.com/boards/hifive1)
+as a test platform for RISC-V MCU.
+
+Similar to Blue Pill setup, you will need Docker installed. The binary can be
+executed on either HiFive1 board or emulated using
+[Renode project](https://renode.io/) on your desktop machine.
 
 The following instructions builds and transfers the source files to the Docker
-```
-docker build -t riscv_build \
--f {PATH_TO_TENSORFLOW_ROOT_DIR}/tensorflow/lite/experimental/micro/testing/Dockerfile.riscv \
-{PATH_TO_TENSORFLOW_ROOT_DIR}/tensorflow/lite/experimental/micro/testing/
-```
+`docker build -t riscv_build \ -f
+{PATH_TO_TENSORFLOW_ROOT_DIR}/tensorflow/lite/experimental/micro/testing/Dockerfile.riscv
+\ {PATH_TO_TENSORFLOW_ROOT_DIR}/tensorflow/lite/experimental/micro/testing/`
 
 You should see output that looks something like this:
 
@@ -160,18 +283,25 @@ Successfully tagged riscv_build:latest
 
 Building micro_speech_test binary
 
- - Lauch the Docker that we just created using: `docker run -it-v /tmp/copybara_out:/workspace riscv_build:latest bash`
- - Enter the source root directory by running `cd /workspace`
- - Download the dependencies by running `./tensorflow/lite/experimental/micro/tools/make/download_dependencies.sh`. This may take a few minutes.
- - Set the path to RISC-V tools: `export PATH=${PATH}:/workspace/tensorflow/lite/experimental/micro/tools/make/downloads/riscv_toolchain/bin/`
- - Build the binary: `make -f tensorflow/lite/experimental/micro/tools/make/Makefile TARGET=riscv32_mcu`
+-   Lauch the Docker that we just created using: `docker run -it-v
+    /tmp/copybara_out:/workspace riscv_build:latest bash`
+-   Enter the source root directory by running `cd /workspace`
+-   Download the dependencies by running
+    `./tensorflow/lite/experimental/micro/tools/make/download_dependencies.sh`.
+    This may take a few minutes.
+-   Set the path to RISC-V tools: `export
+    PATH=${PATH}:/workspace/tensorflow/lite/experimental/micro/tools/make/downloads/riscv_toolchain/bin/`
+-   Build the binary: `make -f
+    tensorflow/lite/experimental/micro/tools/make/Makefile TARGET=riscv32_mcu`
 
 Lauching Renode to test the binary, currently this set up is not automated.
 
- - Until https://github.com/renode/renode/pull/30 is in the Docker image, patch the change manully: `sed -E -i 's/"rv32g"/"rv32imac"/g' /opt/renode/platforms/cpus/sifive-fe310.repl`
-
+-   Until https://github.com/renode/renode/pull/30 is in the Docker image, patch
+    the change manully: `sed -E -i 's/"rv32g"/"rv32imac"/g'
+    /opt/renode/platforms/cpus/sifive-fe310.repl`
 
- - Execute the binary on Renode: `renode -P 5000 --disable-xwt -e 's @/workspace/tensorflow/lite/experimental/micro/testing/sifive_fe310.resc'`
+-   Execute the binary on Renode: `renode -P 5000 --disable-xwt -e 's
+    @/workspace/tensorflow/lite/experimental/micro/testing/sifive_fe310.resc'`
 
 You should see the following log with the magic string `~~~ALL TEST PASSED~~~`:
 
@@ -238,3 +368,24 @@ JFlashLiteExe 2. Device = AMA3B1KK-KBR 3. Interface = SWD at 1000 kHz 4. Data
 file =
 tensorflow/lite/experimental/micro/tools/make/gen/apollo3evb_cortex-m4/bin/pushbutton_cmsis_speech_test.bin
 5. Prog Addr = 0x0000C000
+
+## Generating Project Files
+
+It's not always easy or convenient to use a makefile-based build process,
+especially if you're working on a product that uses a different IDE for the rest
+of its code. To address that, it's possible to generate standalone project
+folders for various popular build systems. These projects are self-contained,
+with only the headers and source files needed by a particular binary, and
+include project files to make loading them into an IDE easy. These can be
+auto-generated for any target you can compile using the main Make system, using
+a command like this (making sure you've run `download_dependencies.sh` first):
+
+```
+make -f tensorflow/lite/experimental/micro/tools/make/Makefile TARGET=mbed TAGS="CMSIS disco_f746ng" generate_micro_speech_main_test_mbed_project
+```
+
+This will create a folder in
+`tensorflow/lite/experimental/micro/tools/make/gen/mbed_cortex-m4/prj/micro_speech_main_test/mbed`
+that contains the source and header files, some Mbed configuration files, and a
+README. You should then be able to copy this directory to another machine, and
+use it just like any other Mbed project.
diff --git a/tensorflow/lite/experimental/micro/bluepill/debug_log.cc b/tensorflow/lite/experimental/micro/bluepill/debug_log.cc
new file mode 100644
index 0000000000..4812a91849
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/bluepill/debug_log.cc
@@ -0,0 +1,27 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/experimental/micro/debug_log.h"
+
+// For Arm Cortex-M devices, calling SYS_WRITE0 will output the zero-terminated
+// string pointed to by R1 to any debug console that's attached to the system.
+extern "C" void DebugLog(const char* s) {
+  asm("mov r0, #0x04\n"  // SYS_WRITE0
+      "mov r1, %[str]\n"
+      "bkpt #0xAB\n"
+      :
+      : [ str ] "r"(s)
+      : "r0", "r1");
+}
diff --git a/tensorflow/lite/experimental/micro/debug_log.cc b/tensorflow/lite/experimental/micro/debug_log.cc
new file mode 100644
index 0000000000..3d4ca44d76
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/debug_log.cc
@@ -0,0 +1,41 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// Reference implementation of the DebugLog() function that's required for a
+// platform to support the TensorFlow Lite for Microcontrollers library. This is
+// the only function that's absolutely required to be available on a target
+// device, since it's used for communicating test results back to the host so
+// that we can verify the implementation is working correctly.
+// It's designed to be as easy as possible to supply an implementation though.
+// On platforms that have a POSIX stack or C library, it can be written as a
+// single call to `fprintf(stderr, "%s", s)` to output a string to the error
+// stream of the console, but if there's no OS or C library available, there's
+// almost always an equivalent way to write out a string to some serial
+// interface that can be used instead. For example on Arm M-series MCUs, calling
+// the `bkpt #0xAB` assembler instruction will output the string in r1 to
+// whatever debug serial connection is available. If you're running mbed, you
+// can do the same by creating `Serial pc(USBTX, USBRX)` and then calling
+// `pc.printf("%s", s)`.
+// To add an equivalent function for your own platform, create your own
+// implementation file, and place it in a subfolder with named after the OS
+// you're targeting. For example, see the Cortex M bare metal version in
+// tensorflow/lite/experimental/micro/bluepill/debug_log.cc or the mbed one on
+// tensorflow/lite/experimental/micro/mbed/debug_log.cc.
+
+#include "tensorflow/lite/experimental/micro/debug_log.h"
+
+#include <cstdio>
+
+extern "C" void DebugLog(const char* s) { fprintf(stderr, "%s", s); }
diff --git a/tensorflow/lite/experimental/micro/debug_log.h b/tensorflow/lite/experimental/micro/debug_log.h
new file mode 100644
index 0000000000..c0e395c376
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/debug_log.h
@@ -0,0 +1,23 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICRO_DEBUG_LOG_H_
+#define TENSORFLOW_LITE_EXPERIMENTAL_MICRO_DEBUG_LOG_H_
+
+// This function should be implemented by each target platform, and provide a
+// way for strings to be output to some text stream. For more information, see
+// tensorflow/lite/experimental/micro/debug_log.cc.
+extern "C" void DebugLog(const char* s);
+
+#endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICRO_DEBUG_LOG_H_
diff --git a/tensorflow/lite/experimental/micro/debug_log_numbers.cc b/tensorflow/lite/experimental/micro/debug_log_numbers.cc
new file mode 100644
index 0000000000..8e86730674
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/debug_log_numbers.cc
@@ -0,0 +1,185 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// Implements debug logging for numbers by converting them into strings and then
+// calling the main DebugLog(char*) function. These are separated into a
+// different file so that platforms can just implement the string output version
+// of DebugLog() and then get the numerical variations without requiring any
+// more code.
+
+#include "tensorflow/lite/experimental/micro/debug_log_numbers.h"
+
+#include "tensorflow/lite/experimental/micro/debug_log.h"
+
+namespace {
+
+// All input buffers to the number conversion functions must be this long.
+static const int kFastToBufferSize = 48;
+
+// Reverses a zero-terminated string in-place.
+char* ReverseStringInPlace(char* start, char* end) {
+  char* p1 = start;
+  char* p2 = end - 1;
+  while (p1 < p2) {
+    char tmp = *p1;
+    *p1++ = *p2;
+    *p2-- = tmp;
+  }
+  return start;
+}
+
+// Appends a string to a string, in-place. You need to pass in the maximum
+// string length as the second argument.
+char* StrCatStr(char* main, int main_max_length, const char* to_append) {
+  char* current = main;
+  while (*current != 0) {
+    ++current;
+  }
+  char* current_end = main + (main_max_length - 1);
+  while ((*to_append != 0) && (current < current_end)) {
+    *current = *to_append;
+    ++current;
+    ++to_append;
+  }
+  *current = 0;
+  return current;
+}
+
+// Populates the provided buffer with an ASCII representation of the number.
+char* FastUInt32ToBufferLeft(uint32_t i, char* buffer, int base) {
+  char* start = buffer;
+  do {
+    int32_t digit = i % base;
+    char character;
+    if (digit < 10) {
+      character = '0' + digit;
+    } else {
+      character = 'a' + (digit - 10);
+    }
+    *buffer++ = character;
+    i /= base;
+  } while (i > 0);
+  *buffer = 0;
+  ReverseStringInPlace(start, buffer);
+  return buffer;
+}
+
+// Populates the provided buffer with an ASCII representation of the number.
+char* FastInt32ToBufferLeft(int32_t i, char* buffer) {
+  uint32_t u = i;
+  if (i < 0) {
+    *buffer++ = '-';
+    u = -u;
+  }
+  return FastUInt32ToBufferLeft(u, buffer, 10);
+}
+
+// Converts a number to a string and appends it to another.
+char* StrCatInt32(char* main, int main_max_length, int32_t number) {
+  char number_string[kFastToBufferSize];
+  FastInt32ToBufferLeft(number, number_string);
+  return StrCatStr(main, main_max_length, number_string);
+}
+
+// Converts a number to a string and appends it to another.
+char* StrCatUInt32(char* main, int main_max_length, uint32_t number, int base) {
+  char number_string[kFastToBufferSize];
+  FastUInt32ToBufferLeft(number, number_string, base);
+  return StrCatStr(main, main_max_length, number_string);
+}
+
+// Populates the provided buffer with ASCII representation of the float number.
+// Avoids the use of any floating point instructions (since these aren't
+// supported on many microcontrollers) and as a consequence prints values with
+// power-of-two exponents.
+char* FastFloatToBufferLeft(float f, char* buffer) {
+  char* current = buffer;
+  char* current_end = buffer + (kFastToBufferSize - 1);
+  // Access the bit fields of the floating point value to avoid requiring any
+  // float instructions. These constants are derived from IEEE 754.
+  const uint32_t sign_mask = 0x80000000;
+  const uint32_t exponent_mask = 0x7f800000;
+  const int32_t exponent_shift = 23;
+  const int32_t exponent_bias = 127;
+  const uint32_t fraction_mask = 0x007fffff;
+  const uint32_t u = *reinterpret_cast<uint32_t*>(&f);
+  const int32_t exponent =
+      ((u & exponent_mask) >> exponent_shift) - exponent_bias;
+  const uint32_t fraction = (u & fraction_mask);
+  // Expect ~0x2B1B9D3 for fraction.
+  if (u & sign_mask) {
+    *current = '-';
+    current += 1;
+  }
+  *current = 0;
+  // These are special cases for infinities and not-a-numbers.
+  if (exponent == 128) {
+    if (fraction == 0) {
+      current = StrCatStr(current, (current_end - current), "Inf");
+      return current;
+    } else {
+      current = StrCatStr(current, (current_end - current), "NaN");
+      return current;
+    }
+  }
+  // 0x007fffff (8388607) represents 0.99... for the fraction, so to print the
+  // correct decimal digits we need to scale our value before passing it to the
+  // conversion function. This scale should be 10000000/8388608 = 1.1920928955.
+  // We can approximate this using multiply-adds and right-shifts using the
+  // values in this array. The 1. portion of the number string is printed out
+  // in a fixed way before the fraction, below.
+  const int32_t scale_shifts_size = 13;
+  const int8_t scale_shifts[13] = {3,  4,  8,  11, 13, 14, 17,
+                                   18, 19, 20, 21, 22, 23};
+  uint32_t scaled_fraction = fraction;
+  for (int i = 0; i < scale_shifts_size; ++i) {
+    scaled_fraction += (fraction >> scale_shifts[i]);
+  }
+  *current = '1';
+  current += 1;
+  *current = '.';
+  current += 1;
+  *current = 0;
+  current = StrCatUInt32(current, (current_end - current), scaled_fraction, 10);
+  current = StrCatStr(current, (current_end - current), "*2^");
+  current = StrCatInt32(current, (current_end - current), exponent);
+  return current;
+}
+
+}  // namespace
+
+extern "C" void DebugLogInt32(int32_t i) {
+  char number_string[kFastToBufferSize];
+  FastInt32ToBufferLeft(i, number_string);
+  DebugLog(number_string);
+}
+
+extern "C" void DebugLogUInt32(uint32_t i) {
+  char number_string[kFastToBufferSize];
+  FastUInt32ToBufferLeft(i, number_string, 10);
+  DebugLog(number_string);
+}
+
+extern "C" void DebugLogHex(uint32_t i) {
+  char number_string[kFastToBufferSize];
+  FastUInt32ToBufferLeft(i, number_string, 16);
+  DebugLog(number_string);
+}
+
+extern "C" void DebugLogFloat(float i) {
+  char number_string[kFastToBufferSize];
+  FastFloatToBufferLeft(i, number_string);
+  DebugLog(number_string);
+}
diff --git a/tensorflow/lite/experimental/micro/debug_log_numbers.h b/tensorflow/lite/experimental/micro/debug_log_numbers.h
new file mode 100644
index 0000000000..d889e75173
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/debug_log_numbers.h
@@ -0,0 +1,28 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICRO_DEBUG_LOG_NUMBERS_H_
+#define TENSORFLOW_LITE_EXPERIMENTAL_MICRO_DEBUG_LOG_NUMBERS_H_
+
+#include <cstdint>
+
+// Output numbers to the debug logging stream.
+extern "C" {
+void DebugLogInt32(int32_t i);
+void DebugLogUInt32(uint32_t i);
+void DebugLogHex(uint32_t i);
+void DebugLogFloat(float i);
+}
+
+#endif  // TENSORFLOW_LITE_EXPERIMENTAL_MICRO_DEBUG_LOG_NUMBERS_H_
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/Makefile.inc b/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/Makefile.inc
new file mode 100644
index 0000000000..3d560510ad
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/Makefile.inc
@@ -0,0 +1,43 @@
+# Settings for targets that use the CMSIS library.
+ifneq ($(filter CMSIS,$(ALL_TAGS)),)
+  INCLUDES += \
+    -isystem$(MAKEFILE_DIR)/downloads/cmsis/CMSIS/Core/Include/ \
+    -isystem$(MAKEFILE_DIR)/downloads/cmsis/CMSIS/DSP/Include/ \
+    -I$(MAKEFILE_DIR)/downloads/CMSIS_ext/
+
+  CMSIS_PREPROCESSOR_SRCS := \
+    tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/hanning.cc \
+    tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/sin_1k.cc \
+
+  CMSIS_PREPROCESSOR_HDRS := \
+    tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/hanning.h \
+    tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/sin_1k.h \
+    third_party/CMSIS_ext/arm_cmplx_mag_squared_q10p6.h
+
+  PREPROCESSOR_TEST_SRCS += $(CMSIS_PREPROCESSOR_SRCS)
+  PREPROCESSOR_TEST_HDRS += $(CMSIS_PREPROCESSOR_HDRS)
+
+  FEATURE_PROVIDER_TEST_SRCS += $(CMSIS_PREPROCESSOR_SRCS)
+  FEATURE_PROVIDER_TEST_HDRS += $(CMSIS_PREPROCESSOR_HDRS)
+
+  MICRO_SPEECH_SRCS += $(CMSIS_PREPROCESSOR_SRCS)
+  MICRO_SPEECH_HDRS += $(CMSIS_PREPROCESSOR_HDRS)
+
+  THIRD_PARTY_CC_SRCS += \
+    third_party/CMSIS_ext/arm_cmplx_mag_squared_q10p6.c \
+    third_party/cmsis/CMSIS/DSP/Source/BasicMathFunctions/arm_mult_q15.c \
+    third_party/cmsis/CMSIS/DSP/Source/TransformFunctions/arm_rfft_init_q15.c \
+    third_party/cmsis/CMSIS/DSP/Source/TransformFunctions/arm_rfft_q15.c \
+    third_party/cmsis/CMSIS/DSP/Source/TransformFunctions/arm_cfft_q15.c \
+    third_party/cmsis/CMSIS/DSP/Source/TransformFunctions/arm_cfft_radix4_q15.c \
+    third_party/cmsis/CMSIS/DSP/Source/TransformFunctions/arm_bitreversal2.S \
+    third_party/cmsis/CMSIS/DSP/Source/CommonTables/arm_const_structs.c \
+    third_party/cmsis/CMSIS/DSP/Source/CommonTables/arm_common_tables.c \
+    third_party/cmsis/CMSIS/DSP/Source/StatisticsFunctions/arm_mean_q15.c \
+    third_party/cmsis/CMSIS/DSP/Source/StatisticsFunctions/arm_max_q7.c
+
+  THIRD_PARTY_CC_HDRS += \
+    third_party/cmsis/CMSIS/DSP/Include/arm_common_tables.h \
+    third_party/cmsis/CMSIS/DSP/Include/arm_const_structs.h
+
+endif
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/Makefile.inc b/tensorflow/lite/experimental/micro/examples/micro_speech/Makefile.inc
index 6f7563c02f..49aace3d7d 100644
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/Makefile.inc
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/Makefile.inc
@@ -1,140 +1,62 @@
 
-# Tests loading and running a speech model.
 MICRO_SPEECH_TEST_SRCS := \
 tensorflow/lite/experimental/micro/examples/micro_speech/micro_speech_test.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/tiny_conv_model_data.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/no_features_data.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/yes_features_data.cc
-MICRO_SPEECH_TEST_SRCS := $(call specialize,$(MICRO_SPEECH_TEST_SRCS))
-ALL_SRCS += $(MICRO_SPEECH_TEST_SRCS)
-MICRO_SPEECH_TEST_OBJS := $(addprefix $(OBJDIR), \
-$(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(MICRO_SPEECH_TEST_SRCS))))
-MICRO_SPEECH_TEST_BINARY := $(BINDIR)micro_speech_test
-ALL_BINARIES += $(MICRO_SPEECH_TEST_BINARY)
-$(MICRO_SPEECH_TEST_BINARY): $(MICRO_SPEECH_TEST_OBJS) $(MICROLITE_LIB_PATH)
-	@mkdir -p $(dir $@)
-	$(CXX) $(CXXFLAGS) $(INCLUDES) \
-	-o $(MICRO_SPEECH_TEST_BINARY) $(MICRO_SPEECH_TEST_OBJS) \
-	$(LIBFLAGS) $(MICROLITE_LIB_PATH) $(LDFLAGS) $(MICROLITE_LIBS)
-micro_speech_test: $(MICRO_SPEECH_TEST_BINARY)
-micro_speech_test_bin: $(MICRO_SPEECH_TEST_BINARY).bin
-test_micro_speech: $(MICRO_SPEECH_TEST_BINARY)
-	$(TEST_SCRIPT) $(MICRO_SPEECH_TEST_BINARY) '~~~ALL TESTS PASSED~~~'
-
-# Source files that are used by multiple preprocessor tests.
-PREPROCESSOR_TEST_SHARED_SRCS := \
+
+MICRO_SPEECH_TEST_HDRS := \
+tensorflow/lite/experimental/micro/examples/micro_speech/tiny_conv_model_data.h \
+tensorflow/lite/experimental/micro/examples/micro_speech/no_features_data.h \
+tensorflow/lite/experimental/micro/examples/micro_speech/yes_features_data.h \
+
+PREPROCESSOR_TEST_SRCS := \
+tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor_test.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/no_30ms_sample_data.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/yes_30ms_sample_data.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/no_power_spectrum_data.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/yes_power_spectrum_data.cc
 
-# Test the float reference code for feature generation.
-PREPROCESSOR_REFERENCE_TEST_SRCS = \
-$(PREPROCESSOR_TEST_SHARED_SRCS) \
-tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.cc
-PREPROCESSOR_REFERENCE_TEST_SRCS := $(call specialize,$(PREPROCESSOR_REFERENCE_TEST_SRCS))
-ALL_SRCS += $(PREPROCESSOR_REFERENCE_TEST_SRCS)
-PREPROCESSOR_REFERENCE_TEST_OBJS := $(addprefix $(OBJDIR), \
-$(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(PREPROCESSOR_REFERENCE_TEST_SRCS))))
-PREPROCESSOR_REFERENCE_TEST_BINARY := $(BINDIR)preprocessor_reference_test
-ALL_BINARIES += $(PREPROCESSOR_REFERENCE_TEST_BINARY)
-$(PREPROCESSOR_REFERENCE_TEST_BINARY): $(PREPROCESSOR_REFERENCE_TEST_OBJS) $(MICROLITE_LIB_PATH)
-	@mkdir -p $(dir $@)
-	$(CXX) $(CXXFLAGS) $(INCLUDES) \
-	-o $(PREPROCESSOR_REFERENCE_TEST_BINARY) $(PREPROCESSOR_REFERENCE_TEST_OBJS) \
-	$(LIBFLAGS) $(MICROLITE_LIB_PATH) $(LDFLAGS) $(MICROLITE_LIBS)
-preprocessor_reference_test: $(PREPROCESSOR_REFERENCE_TEST_BINARY)
-preprocessor_reference_test_bin: $(PREPROCESSOR_REFERENCE_TEST_BINARY).bin
-test_preprocessor_reference: $(PREPROCESSOR_REFERENCE_TEST_BINARY)
-	$(TEST_SCRIPT) $(PREPROCESSOR_REFERENCE_TEST_BINARY) '~~~ALL TESTS PASSED~~~'
-
-# Test the fixed point reference code for feature generation.
-PREPROCESSOR_FIXED_TEST_SRCS = \
-$(PREPROCESSOR_TEST_SHARED_SRCS) \
-tensorflow/lite/experimental/micro/examples/micro_speech/fixed_point/preprocessor.cc
-PREPROCESSOR_FIXED_TEST_SRCS := $(call specialize,$(PREPROCESSOR_FIXED_TEST_SRCS))
-ALL_SRCS += $(PREPROCESSOR_FIXED_TEST_SRCS)
-PREPROCESSOR_FIXED_TEST_OBJS := $(addprefix $(OBJDIR), \
-$(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(PREPROCESSOR_FIXED_TEST_SRCS))))
-PREPROCESSOR_FIXED_TEST_BINARY := $(BINDIR)preprocessor_fixed_test
-ALL_BINARIES += $(PREPROCESSOR_FIXED_TEST_BINARY)
-$(PREPROCESSOR_FIXED_TEST_BINARY): $(PREPROCESSOR_FIXED_TEST_OBJS) $(MICROLITE_LIB_PATH)
-	@mkdir -p $(dir $@)
-	$(CXX) $(CXXFLAGS) $(INCLUDES) \
-	-o $(PREPROCESSOR_FIXED_TEST_BINARY) $(PREPROCESSOR_FIXED_TEST_OBJS) \
-	$(LIBFLAGS) $(MICROLITE_LIB_PATH) $(LDFLAGS) $(MICROLITE_LIBS)
-preprocessor_fixed_test: $(PREPROCESSOR_FIXED_TEST_BINARY)
-preprocessor_fixed_test_bin: $(PREPROCESSOR_FIXED_TEST_BINARY).bin
-test_preprocessor_fixed: $(PREPROCESSOR_FIXED_TEST_BINARY)
-	$(TEST_SCRIPT) $(PREPROCESSOR_FIXED_TEST_BINARY) '~~~ALL TESTS PASSED~~~'
+PREPROCESSOR_TEST_HDRS := \
+tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.h \
+tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h \
+tensorflow/lite/experimental/micro/examples/micro_speech/no_30ms_sample_data.h \
+tensorflow/lite/experimental/micro/examples/micro_speech/yes_30ms_sample_data.h \
+tensorflow/lite/experimental/micro/examples/micro_speech/no_power_spectrum_data.h \
+tensorflow/lite/experimental/micro/examples/micro_speech/yes_power_spectrum_data.h
 
-# Tests the audio provider module.
 AUDIO_PROVIDER_TEST_SRCS := \
 tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider_test.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.cc
-AUDIO_PROVIDER_TEST_SRCS := $(call specialize,$(AUDIO_PROVIDER_TEST_SRCS))
-ALL_SRCS += $(AUDIO_PROVIDER_TEST_SRCS)
-AUDIO_PROVIDER_TEST_OBJS := $(addprefix $(OBJDIR), \
-$(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(AUDIO_PROVIDER_TEST_SRCS))))
-AUDIO_PROVIDER_TEST_BINARY := $(BINDIR)audio_provider_test
-ALL_BINARIES += $(AUDIO_PROVIDER_TEST_BINARY)
-$(AUDIO_PROVIDER_TEST_BINARY): $(AUDIO_PROVIDER_TEST_OBJS) $(MICROLITE_LIB_PATH)
-	@mkdir -p $(dir $@)
-	$(CXX) $(CXXFLAGS) $(INCLUDES) \
-	-o $(AUDIO_PROVIDER_TEST_BINARY) $(AUDIO_PROVIDER_TEST_OBJS) \
-	$(LIBFLAGS) $(MICROLITE_LIB_PATH) $(LDFLAGS) $(MICROLITE_LIBS)
-audio_provider_test: $(AUDIO_PROVIDER_TEST_BINARY)
-audio_provider_test_bin: $(AUDIO_PROVIDER_TEST_BINARY).bin
-test_audio_provider: $(AUDIO_PROVIDER_TEST_BINARY)
-	$(TEST_SCRIPT) $(AUDIO_PROVIDER_TEST_BINARY) '~~~ALL TESTS PASSED~~~'
 
-# Tests the feature provider module.
+AUDIO_PROVIDER_TEST_HDRS := \
+tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h \
+tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h \
+
 FEATURE_PROVIDER_TEST_SRCS := \
 tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider_test.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.cc
-FEATURE_PROVIDER_TEST_SRCS := $(call specialize,$(FEATURE_PROVIDER_TEST_SRCS))
-ALL_SRCS += $(FEATURE_PROVIDER_TEST_SRCS)
-FEATURE_PROVIDER_TEST_OBJS := $(addprefix $(OBJDIR), \
-$(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(FEATURE_PROVIDER_TEST_SRCS))))
-FEATURE_PROVIDER_TEST_BINARY := $(BINDIR)feature_provider_test
-ALL_BINARIES += $(FEATURE_PROVIDER_TEST_BINARY)
-$(FEATURE_PROVIDER_TEST_BINARY): $(FEATURE_PROVIDER_TEST_OBJS) $(MICROLITE_LIB_PATH)
-	@mkdir -p $(dir $@)
-	$(CXX) $(CXXFLAGS) $(INCLUDES) \
-	-o $(FEATURE_PROVIDER_TEST_BINARY) $(FEATURE_PROVIDER_TEST_OBJS) \
-	$(LIBFLAGS) $(MICROLITE_LIB_PATH) $(LDFLAGS) $(MICROLITE_LIBS)
-feature_provider_test: $(FEATURE_PROVIDER_TEST_BINARY)
-feature_provider_test_bin: $(FEATURE_PROVIDER_TEST_BINARY).bin
-test_feature_provider: $(FEATURE_PROVIDER_TEST_BINARY)
-	$(TEST_SCRIPT) $(FEATURE_PROVIDER_TEST_BINARY) '~~~ALL TESTS PASSED~~~'
 
-# Tests the feature provider module.
+FEATURE_PROVIDER_TEST_HDRS := \
+tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h \
+tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h \
+tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.h \
+tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.h
+
 RECOGNIZE_COMMANDS_TEST_SRCS := \
 tensorflow/lite/experimental/micro/examples/micro_speech/recognize_commands_test.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/recognize_commands.cc
-RECOGNIZE_COMMANDS_TEST_SRCS := $(call specialize,$(RECOGNIZE_COMMANDS_TEST_SRCS))
-ALL_SRCS += $(RECOGNIZE_COMMANDS_TEST_SRCS)
-RECOGNIZE_COMMANDS_TEST_OBJS := $(addprefix $(OBJDIR), \
-$(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(RECOGNIZE_COMMANDS_TEST_SRCS))))
-RECOGNIZE_COMMANDS_TEST_BINARY := $(BINDIR)recognize_commands_test
-ALL_BINARIES += $(RECOGNIZE_COMMANDS_TEST_BINARY)
-$(RECOGNIZE_COMMANDS_TEST_BINARY): $(RECOGNIZE_COMMANDS_TEST_OBJS) $(MICROLITE_LIB_PATH)
-	@mkdir -p $(dir $@)
-	$(CXX) $(CXXFLAGS) $(INCLUDES) \
-	-o $(RECOGNIZE_COMMANDS_TEST_BINARY) $(RECOGNIZE_COMMANDS_TEST_OBJS) \
-	$(LIBFLAGS) $(MICROLITE_LIB_PATH) $(LDFLAGS) $(MICROLITE_LIBS)
-recognize_commands_test: $(RECOGNIZE_COMMANDS_TEST_BINARY)
-recognize_commands_test_bin: $(RECOGNIZE_COMMANDS_TEST_BINARY).bin
-test_recognize_commands: $(RECOGNIZE_COMMANDS_TEST_BINARY)
-	$(TEST_SCRIPT) $(RECOGNIZE_COMMANDS_TEST_BINARY) '~~~ALL TESTS PASSED~~~'
 
-# Builds a standalone speech command recognizer binary.
+RECOGNIZE_COMMANDS_TEST_HDRS := \
+tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h \
+tensorflow/lite/experimental/micro/examples/micro_speech/recognize_commands.h
+
 MICRO_SPEECH_SRCS := \
 tensorflow/lite/experimental/micro/examples/micro_speech/main.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.cc \
@@ -145,19 +67,40 @@ tensorflow/lite/experimental/micro/examples/micro_speech/no_features_data.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/yes_features_data.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/tiny_conv_model_data.cc \
 tensorflow/lite/experimental/micro/examples/micro_speech/recognize_commands.cc
-MICRO_SPEECH_SRCS := $(call specialize,$(MICRO_SPEECH_SRCS))
-ALL_SRCS += $(MICRO_SPEECH_SRCS)
-MICRO_SPEECH_OBJS := $(addprefix $(OBJDIR), \
-$(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(MICRO_SPEECH_SRCS))))
-MICRO_SPEECH_BINARY := $(BINDIR)micro_speech
-ALL_BINARIES += $(MICRO_SPEECH_BINARY)
-$(MICRO_SPEECH_BINARY): $(MICRO_SPEECH_OBJS) $(MICROLITE_LIB_PATH)
-	@mkdir -p $(dir $@)
-	$(CXX) $(CXXFLAGS) $(INCLUDES) \
-	-o $(MICRO_SPEECH_BINARY) $(MICRO_SPEECH_OBJS) \
-	$(LIBFLAGS) $(MICROLITE_LIB_PATH) $(LDFLAGS) $(MICROLITE_LIBS)
-micro_speech: $(MICRO_SPEECH_BINARY)
-micro_speech_bin: $(MICRO_SPEECH_BINARY).bin
+
+MICRO_SPEECH_HDRS := \
+tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h \
+tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h \
+tensorflow/lite/experimental/micro/examples/micro_speech/feature_provider.h \
+tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.h \
+tensorflow/lite/experimental/micro/examples/micro_speech/no_features_data.h \
+tensorflow/lite/experimental/micro/examples/micro_speech/yes_features_data.h \
+tensorflow/lite/experimental/micro/examples/micro_speech/tiny_conv_model_data.h \
+tensorflow/lite/experimental/micro/examples/micro_speech/recognize_commands.h
 
 # Find any platform-specific rules for this example.
 include $(wildcard tensorflow/lite/experimental/micro/examples/micro_speech/*/Makefile.inc)
+
+# Tests loading and running a speech model.
+$(eval $(call microlite_test,micro_speech_test,\
+$(MICRO_SPEECH_TEST_SRCS),$(MICRO_SPEECH_TEST_HDRS)))
+
+# Test the code for feature generation.
+$(eval $(call microlite_test,preprocessor_test,\
+$(PREPROCESSOR_TEST_SRCS), $(PREPROCESSOR_TEST_HDRS)))
+
+# Tests the audio provider module.
+$(eval $(call microlite_test,audio_provider_test,\
+$(AUDIO_PROVIDER_TEST_SRCS),$(AUDIO_PROVIDER_TEST_HDRS)))
+
+# Tests the feature provider module.
+$(eval $(call microlite_test,feature_provider_test,\
+$(FEATURE_PROVIDER_TEST_SRCS),$(FEATURE_PROVIDER_TEST_HDRS)))
+
+# Tests the feature provider module.
+$(eval $(call microlite_test,recognize_commands_test,\
+$(RECOGNIZE_COMMANDS_TEST_SRCS),$(RECOGNIZE_COMMANDS_TEST_HDRS)))
+
+# Builds a standalone speech command recognizer binary.
+$(eval $(call microlite_test,micro_speech,\
+$(MICRO_SPEECH_SRCS),$(MICRO_SPEECH_HDRS)))
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/disco_f746ng/Makefile.inc b/tensorflow/lite/experimental/micro/examples/micro_speech/disco_f746ng/Makefile.inc
new file mode 100644
index 0000000000..5585ed7269
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/disco_f746ng/Makefile.inc
@@ -0,0 +1,7 @@
+# Settings for the Discovery STM32F746NG board.
+ifneq ($(filter disco_f746ng,$(ALL_TAGS)),)
+  MBED_PROJECT_FILES += \
+    AUDIO_DISCO_F746NG.lib \
+    BSP_DISCO_F746NG.lib \
+    SDRAM_DISCO_F746NG.lib
+endif
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/disco_f746ng/audio_provider.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/disco_f746ng/audio_provider.cc
new file mode 100644
index 0000000000..06647d0c53
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/disco_f746ng/audio_provider.cc
@@ -0,0 +1,182 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.h"
+
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/model_settings.h"
+
+#include "AUDIO_DISCO_F746NG.h"
+#include "SDRAM_DISCO_F746NG.h"
+#include "mbed.h"  // NOLINT
+
+namespace {
+
+bool g_is_audio_initialized = false;
+constexpr int kAudioCaptureBufferSize = kAudioSampleFrequency * 0.5;
+int16_t g_audio_capture_buffer[kAudioCaptureBufferSize];
+int16_t g_audio_output_buffer[kMaxAudioSampleSize];
+int32_t g_latest_audio_timestamp = 0;
+
+// For a full example of how to access audio on the STM32F746NG board, see
+// https://os.mbed.com/teams/ST/code/DISCO-F746NG_AUDIO_demo/
+AUDIO_DISCO_F746NG g_audio_device;
+SDRAM_DISCO_F746NG g_sdram_device;
+
+typedef enum {
+  BUFFER_OFFSET_NONE = 0,
+  BUFFER_OFFSET_HALF = 1,
+  BUFFER_OFFSET_FULL = 2,
+} BUFFER_StateTypeDef;
+
+#define AUDIO_BLOCK_SIZE ((uint32_t)2048)
+#define AUDIO_BUFFER_IN SDRAM_DEVICE_ADDR /* In SDRAM */
+#define AUDIO_BUFFER_OUT \
+  (SDRAM_DEVICE_ADDR + (AUDIO_BLOCK_SIZE * 2)) /* In SDRAM */
+__IO uint32_t g_audio_rec_buffer_state = BUFFER_OFFSET_NONE;
+
+uint8_t SetSysClock_PLL_HSE_200MHz() {
+  RCC_ClkInitTypeDef RCC_ClkInitStruct;
+  RCC_OscInitTypeDef RCC_OscInitStruct;
+
+  // Enable power clock
+  __PWR_CLK_ENABLE();
+
+  // Enable HSE oscillator and activate PLL with HSE as source
+  RCC_OscInitStruct.OscillatorType = RCC_OSCILLATORTYPE_HSE;
+  RCC_OscInitStruct.HSEState = RCC_HSE_ON; /* External xtal on OSC_IN/OSC_OUT */
+
+  // Warning: this configuration is for a 25 MHz xtal clock only
+  RCC_OscInitStruct.PLL.PLLState = RCC_PLL_ON;
+  RCC_OscInitStruct.PLL.PLLSource = RCC_PLLSOURCE_HSE;
+  RCC_OscInitStruct.PLL.PLLM = 25;   // VCO input clock = 1 MHz (25 MHz / 25)
+  RCC_OscInitStruct.PLL.PLLN = 400;  // VCO output clock = 400 MHz (1 MHz * 400)
+  RCC_OscInitStruct.PLL.PLLP = RCC_PLLP_DIV2;  // PLLCLK = 200 MHz (400 MHz / 2)
+  RCC_OscInitStruct.PLL.PLLQ = 8;  // USB clock = 50 MHz (400 MHz / 8)
+
+  if (HAL_RCC_OscConfig(&RCC_OscInitStruct) != HAL_OK) {
+    return 0;  // FAIL
+  }
+
+  // Activate the OverDrive to reach the 216 MHz Frequency
+  if (HAL_PWREx_EnableOverDrive() != HAL_OK) {
+    return 0;  // FAIL
+  }
+
+  // Select PLL as system clock source and configure the HCLK, PCLK1 and PCLK2
+  // clocks dividers
+  RCC_ClkInitStruct.ClockType = (RCC_CLOCKTYPE_SYSCLK | RCC_CLOCKTYPE_HCLK |
+                                 RCC_CLOCKTYPE_PCLK1 | RCC_CLOCKTYPE_PCLK2);
+  RCC_ClkInitStruct.SYSCLKSource = RCC_SYSCLKSOURCE_PLLCLK;  // 200 MHz
+  RCC_ClkInitStruct.AHBCLKDivider = RCC_SYSCLK_DIV1;         // 200 MHz
+  RCC_ClkInitStruct.APB1CLKDivider = RCC_HCLK_DIV4;          //  50 MHz
+  RCC_ClkInitStruct.APB2CLKDivider = RCC_HCLK_DIV2;          // 100 MHz
+
+  if (HAL_RCC_ClockConfig(&RCC_ClkInitStruct, FLASH_LATENCY_7) != HAL_OK) {
+    return 0;  // FAIL
+  }
+  HAL_RCC_MCOConfig(RCC_MCO1, RCC_MCO1SOURCE_HSE, RCC_MCODIV_4);
+  return 1;  // OK
+}
+
+TfLiteStatus InitAudioRecording(tflite::ErrorReporter* error_reporter) {
+  SetSysClock_PLL_HSE_200MHz();
+
+  // Initialize SDRAM buffers.
+  memset((uint16_t*)AUDIO_BUFFER_IN, 0, AUDIO_BLOCK_SIZE * 2);
+  memset((uint16_t*)AUDIO_BUFFER_OUT, 0, AUDIO_BLOCK_SIZE * 2);
+  g_audio_rec_buffer_state = BUFFER_OFFSET_NONE;
+
+  // Start Recording.
+  g_audio_device.IN_Record((uint16_t*)AUDIO_BUFFER_IN, AUDIO_BLOCK_SIZE);
+
+  // Also play results out to headphone jack.
+  g_audio_device.OUT_SetAudioFrameSlot(CODEC_AUDIOFRAME_SLOT_02);
+  g_audio_device.OUT_Play((uint16_t*)AUDIO_BUFFER_OUT, AUDIO_BLOCK_SIZE * 2);
+
+  return kTfLiteOk;
+}
+
+void CaptureSamples(const int16_t* sample_data) {
+  const int sample_size = AUDIO_BLOCK_SIZE / (sizeof(int16_t) * 2);
+  const int32_t time_in_ms =
+      g_latest_audio_timestamp + (sample_size / (kAudioSampleFrequency / 1000));
+
+  const int32_t start_sample_offset =
+      g_latest_audio_timestamp * (kAudioSampleFrequency / 1000);
+  for (int i = 0; i < sample_size; ++i) {
+    const int capture_index =
+        (start_sample_offset + i) % kAudioCaptureBufferSize;
+    g_audio_capture_buffer[capture_index] =
+        (sample_data[(i * 2) + 0] / 2) + (sample_data[(i * 2) + 1] / 2);
+  }
+  // This is how we let the outside world know that new audio data has arrived.
+  g_latest_audio_timestamp = time_in_ms;
+}
+
+}  // namespace
+
+// These callbacks need to be linkable symbols, because they override weak
+// default versions.
+void BSP_AUDIO_IN_TransferComplete_CallBack(void) {
+  g_audio_rec_buffer_state = BUFFER_OFFSET_FULL;
+  /* Copy recorded 1st half block */
+  memcpy((uint16_t*)(AUDIO_BUFFER_OUT), (uint16_t*)(AUDIO_BUFFER_IN),
+         AUDIO_BLOCK_SIZE);
+  CaptureSamples(reinterpret_cast<int16_t*>(AUDIO_BUFFER_IN));
+  return;
+}
+
+// Another weak symbol override.
+void BSP_AUDIO_IN_HalfTransfer_CallBack(void) {
+  g_audio_rec_buffer_state = BUFFER_OFFSET_HALF;
+  /* Copy recorded 2nd half block */
+  memcpy((uint16_t*)(AUDIO_BUFFER_OUT + (AUDIO_BLOCK_SIZE)),
+         (uint16_t*)(AUDIO_BUFFER_IN + (AUDIO_BLOCK_SIZE)), AUDIO_BLOCK_SIZE);
+  CaptureSamples(
+      reinterpret_cast<int16_t*>(AUDIO_BUFFER_IN + AUDIO_BLOCK_SIZE));
+  return;
+}
+
+// Main entry point for getting audio data.
+TfLiteStatus GetAudioSamples(tflite::ErrorReporter* error_reporter,
+                             int start_ms, int duration_ms,
+                             int* audio_samples_size, int16_t** audio_samples) {
+  if (!g_is_audio_initialized) {
+    TfLiteStatus init_status = InitAudioRecording(error_reporter);
+    if (init_status != kTfLiteOk) {
+      return init_status;
+    }
+    g_is_audio_initialized = true;
+  }
+  // This should only be called when the main thread notices that the latest
+  // audio sample data timestamp has changed, so that there's new data in the
+  // capture ring buffer. The ring buffer will eventually wrap around and
+  // overwrite the data, but the assumption is that the main thread is checking
+  // often enough and the buffer is large enough that this call will be made
+  // before that happens.
+  const int start_offset = start_ms * (kAudioSampleFrequency / 1000);
+  const int duration_sample_count =
+      duration_ms * (kAudioSampleFrequency / 1000);
+  for (int i = 0; i < duration_sample_count; ++i) {
+    const int capture_index = (start_offset + i) % kAudioCaptureBufferSize;
+    g_audio_output_buffer[i] = g_audio_capture_buffer[capture_index];
+  }
+
+  *audio_samples_size = kMaxAudioSampleSize;
+  *audio_samples = g_audio_output_buffer;
+  return kTfLiteOk;
+}
+
+int32_t LatestAudioTimestamp() { return g_latest_audio_timestamp; }
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/disco_f746ng/timer.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/disco_f746ng/timer.cc
new file mode 100644
index 0000000000..a8f0fe4bd5
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/disco_f746ng/timer.cc
@@ -0,0 +1,24 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/experimental/micro/examples/micro_speech/timer.h"
+
+namespace {
+int32_t g_current_time = 0;
+}
+
+void SetTimeInMilliseconds(int32_t time) { g_current_time = time; }
+
+int32_t TimeInMilliseconds() { return g_current_time; }
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/osx/Makefile.inc b/tensorflow/lite/experimental/micro/examples/micro_speech/osx/Makefile.inc
index 89d107cfe0..8f8b33a9fa 100644
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/osx/Makefile.inc
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/osx/Makefile.inc
@@ -1,6 +1,8 @@
-#Settings for Mac OS platforms.
+# Settings for Mac OS platforms.
 ifeq ($(TARGET), osx)
-  MICROLITE_LIBS += \
+  LINKER_FLAGS := \
     -framework Foundation \
     -framework AudioToolbox
+
+  MICROLITE_LIBS += $(LINKER_FLAGS)
 endif
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.cc b/tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.cc
index 7b7db173ab..f8858aad72 100644
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.cc
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/preprocessor.cc
@@ -32,6 +32,9 @@ limitations under the License.
 
 namespace {
 
+// Needed because some platforms don't have M_PI defined.
+constexpr float kPi = 3.14159265358979323846f;
+
 // Performs a discrete Fourier transform on the real inputs. This corresponds to
 // rdft() in the FFT package at http://www.kurims.kyoto-u.ac.jp/~ooura/fft.html,
 // and to kiss_fftr() in KISSFFT at https://github.com/mborgerding/kissfft.
@@ -48,11 +51,11 @@ void CalculateDiscreteFourierTransform(float* time_series, int time_series_size,
   for (int i = 0; i < time_series_size / 2; ++i) {
     float real = 0;
     for (int j = 0; j < time_series_size; ++j) {
-      real += time_series[j] * cos(j * i * M_PI * 2 / time_series_size);
+      real += time_series[j] * cos(j * i * kPi * 2 / time_series_size);
     }
     float imaginary = 0;
     for (int j = 0; j < time_series_size; ++j) {
-      imaginary -= time_series[j] * sin(j * i * M_PI * 2 / time_series_size);
+      imaginary -= time_series[j] * sin(j * i * kPi * 2 / time_series_size);
     }
     fourier_output[(i * 2) + 0] = real;
     fourier_output[(i * 2) + 1] = imaginary;
@@ -63,7 +66,7 @@ void CalculateDiscreteFourierTransform(float* time_series, int time_series_size,
 // of the current sample window are weighted more heavily than those at the end.
 void CalculatePeriodicHann(int window_length, float* window_function) {
   for (int i = 0; i < window_length; ++i) {
-    window_function[i] = 0.5 - 0.5 * cos((2 * M_PI * i) / window_length);
+    window_function[i] = 0.5 - 0.5 * cos((2 * kPi * i) / window_length);
   }
 }
 
diff --git a/tensorflow/lite/experimental/micro/mbed/debug_log.cc b/tensorflow/lite/experimental/micro/mbed/debug_log.cc
new file mode 100644
index 0000000000..d4a4a5a842
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/mbed/debug_log.cc
@@ -0,0 +1,24 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/experimental/micro/debug_log.h"
+
+#include <mbed.h>
+
+// On mbed platforms, we set up a serial port and write to it for debug logging.
+extern "C" void DebugLog(const char* s) {
+  static Serial pc(USBTX, USBRX);
+  pc.printf("%s", s);
+}
diff --git a/tensorflow/lite/experimental/micro/micro_error_reporter.h b/tensorflow/lite/experimental/micro/micro_error_reporter.h
index 0ab853ec2a..6c18367c95 100644
--- a/tensorflow/lite/experimental/micro/micro_error_reporter.h
+++ b/tensorflow/lite/experimental/micro/micro_error_reporter.h
@@ -17,26 +17,8 @@ limitations under the License.
 
 #include "tensorflow/lite/core/api/error_reporter.h"
 #include "tensorflow/lite/experimental/micro/compatibility.h"
-
-#ifdef TF_LITE_MCU_DEBUG_LOG
-// These functions should be supplied by the micro target library
-extern "C" {
-#include <stdint.h>
-void DebugLog(const char* s);
-void DebugLogInt32(int32_t i);
-void DebugLogUInt32(uint32_t i);
-void DebugLogHex(uint32_t i);
-void DebugLogFloat(float i);
-}
-#else  // TF_LITE_MCU_DEBUG_LOG
-#include <cstdint>
-#include <cstdio>
-static void inline DebugLog(const char* s) { fprintf(stderr, "%s", s); }
-static void inline DebugLogInt32(int32_t i) { fprintf(stderr, "%d", i); }
-static void inline DebugLogUInt32(uint32_t i) { fprintf(stderr, "%d", i); }
-static void inline DebugLogHex(uint32_t i) { fprintf(stderr, "0x%8x", i); }
-static void inline DebugLogFloat(float i) { fprintf(stderr, "%f", i); }
-#endif  // TF_LITE_MCU_DEBUG_LOG
+#include "tensorflow/lite/experimental/micro/debug_log.h"
+#include "tensorflow/lite/experimental/micro/debug_log_numbers.h"
 
 namespace tflite {
 
diff --git a/tensorflow/lite/experimental/micro/riscv32_mcu/debug_log.cc b/tensorflow/lite/experimental/micro/riscv32_mcu/debug_log.cc
index f0065267bf..d1c2df866e 100644
--- a/tensorflow/lite/experimental/micro/riscv32_mcu/debug_log.cc
+++ b/tensorflow/lite/experimental/micro/riscv32_mcu/debug_log.cc
@@ -15,176 +15,4 @@ limitations under the License.
 
 #include <stdio.h>
 
-namespace {
-
-// All input buffers to the number conversion functions must be this long.
-static const int kFastToBufferSize = 48;
-
-// Reverses a zero-terminated string in-place.
-char* ReverseStringInPlace(char* start, char* end) {
-  char* p1 = start;
-  char* p2 = end - 1;
-  while (p1 < p2) {
-    char tmp = *p1;
-    *p1++ = *p2;
-    *p2-- = tmp;
-  }
-  return start;
-}
-
-// Appends a string to a string, in-place. You need to pass in the maximum
-// string length as the second argument.
-char* StrCatStr(char* main, int main_max_length, char* to_append) {
-  char* current = main;
-  while (*current != 0) {
-    ++current;
-  }
-  char* current_end = main + (main_max_length - 1);
-  while ((*to_append != 0) && (current < current_end)) {
-    *current = *to_append;
-    ++current;
-    ++to_append;
-  }
-  *current = 0;
-  return current;
-}
-
-char* StrCpy(char* main, int main_max_length, const char* source) {
-  char* current = main;
-  char* current_end = main + (main_max_length - 1);
-  while ((*source != 0) && (current < current_end)) {
-    *current = *source;
-    ++current;
-    ++source;
-  }
-  *current = 0;
-  return current;
-}
-
-// Populates the provided buffer with an ASCII representation of the number.
-char* FastUInt32ToBufferLeft(uint32_t i, char* buffer, int base) {
-  char* start = buffer;
-  do {
-    int32_t digit = i % base;
-    char character;
-    if (digit < 10) {
-      character = '0' + digit;
-    } else {
-      character = 'a' + (digit - 10);
-    }
-    *buffer++ = character;
-    i /= base;
-  } while (i > 0);
-  *buffer = 0;
-  ReverseStringInPlace(start, buffer);
-  return buffer;
-}
-
-// Populates the provided buffer with an ASCII representation of the number.
-char* FastInt32ToBufferLeft(int32_t i, char* buffer) {
-  uint32_t u = i;
-  if (i < 0) {
-    *buffer++ = '-';
-    u = -u;
-  }
-  return FastUInt32ToBufferLeft(u, buffer, 10);
-}
-
-// Converts a number to a string and appends it to another.
-char* StrCatInt32(char* main, int main_max_length, int32_t number) {
-  char number_string[kFastToBufferSize];
-  FastInt32ToBufferLeft(number, number_string);
-  return StrCatStr(main, main_max_length, number_string);
-}
-
-// Converts a number to a string and appends it to another.
-char* StrCatUInt32(char* main, int main_max_length, uint32_t number, int base) {
-  char number_string[kFastToBufferSize];
-  FastUInt32ToBufferLeft(number, number_string, base);
-  return StrCatStr(main, main_max_length, number_string);
-}
-
-// Populates the provided buffer with ASCII representation of the float number.
-// Avoids the use of any floating point instructions (since these aren't
-// supported on many microcontrollers) and as a consequence prints values with
-// power-of-two exponents.
-char* FastFloatToBufferLeft(float i, char* buffer) {
-  char* current = buffer;
-  char* current_end = buffer + (kFastToBufferSize - 1);
-  // Access the bit fields of the floating point value to avoid requiring any
-  // float instructions. These constants are derived from IEEE 754.
-  const uint32_t sign_mask = 0x80000000;
-  const uint32_t exponent_mask = 0x7f800000;
-  const int32_t exponent_shift = 23;
-  const int32_t exponent_bias = 127;
-  const uint32_t fraction_mask = 0x007fffff;
-  const uint32_t u = *(uint32_t*)(&i);
-  const int32_t exponent =
-      ((u & exponent_mask) >> exponent_shift) - exponent_bias;
-  const uint32_t fraction = (u & fraction_mask);
-  // Expect ~0x2B1B9D3 for fraction.
-  if (u & sign_mask) {
-    *current = '-';
-    current += 1;
-  }
-  *current = 0;
-  // These are special cases for infinities and not-a-numbers.
-  if (exponent == 128) {
-    if (fraction == 0) {
-      current = StrCatStr(current, (current_end - current), "Inf");
-      return current;
-    } else {
-      current = StrCatStr(current, (current_end - current), "NaN");
-      return current;
-    }
-  }
-  // 0x007fffff represents 0.99... for the fraction, so to print the correct
-  // decimal digits we need to scale our value before passing it to the
-  // conversion function. This scale should be 10000000/8388608 = 1.1920928955.
-  // We can approximate this using multipy-adds and right-shifts using the
-  // values in this array.
-  const int32_t scale_shifts_size = 13;
-  const int8_t scale_shifts[13] = {3,  4,  8,  11, 13, 14, 17,
-                                   18, 19, 20, 21, 22, 23};
-  uint32_t scaled_fraction = fraction;
-  for (int i = 0; i < scale_shifts_size; ++i) {
-    scaled_fraction += (fraction >> scale_shifts[i]);
-  }
-  *current = '1';
-  current += 1;
-  *current = '.';
-  current += 1;
-  *current = 0;
-  current = StrCatUInt32(current, (current_end - current), scaled_fraction, 10);
-  current = StrCatStr(current, (current_end - current), "*2^");
-  current = StrCatInt32(current, (current_end - current), exponent);
-  return current;
-}
-
-}  // namespace
-
 extern "C" void DebugLog(const char* s) { puts(s); }
-
-extern "C" void DebugLogInt32(int32_t i) {
-  char number_string[kFastToBufferSize];
-  FastInt32ToBufferLeft(i, number_string);
-  DebugLog(number_string);
-}
-
-extern "C" void DebugLogUInt32(uint32_t i) {
-  char number_string[kFastToBufferSize];
-  FastUInt32ToBufferLeft(i, number_string, 10);
-  DebugLog(number_string);
-}
-
-extern "C" void DebugLogHex(uint32_t i) {
-  char number_string[kFastToBufferSize];
-  FastUInt32ToBufferLeft(i, number_string, 16);
-  DebugLog(number_string);
-}
-
-extern "C" void DebugLogFloat(float i) {
-  char number_string[kFastToBufferSize];
-  FastFloatToBufferLeft(i, number_string);
-  DebugLog(number_string);
-}
diff --git a/tensorflow/lite/experimental/micro/tools/make/Makefile b/tensorflow/lite/experimental/micro/tools/make/Makefile
index f9a2dffae2..fde195118b 100644
--- a/tensorflow/lite/experimental/micro/tools/make/Makefile
+++ b/tensorflow/lite/experimental/micro/tools/make/Makefile
@@ -1,5 +1,9 @@
+
 MAKEFILE_DIR := tensorflow/lite/experimental/micro/tools/make
 
+# Pull in some convenience functions.
+include $(MAKEFILE_DIR)/helper_functions.inc
+
 # Try to figure out the host system
 HOST_OS :=
 ifeq ($(OS),Windows_NT)
@@ -21,37 +25,10 @@ HOST_ARCH := $(shell if [[ $(shell uname -m) =~ i[345678]86 ]]; then echo x86_32
 TARGET := $(HOST_OS)
 TARGET_ARCH := $(HOST_ARCH)
 
-# Look for platform or target-specific implementation files to replace reference
-# implementations with, given a tag. These are expected to occur in subfolders
-# of a directory where a reference implementation exists, and have the same
-# interface and header file. For example,
-# tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.cc
-# defines a module for supplying audio data, but since no platform or OS can be
-# presumed, it just always returns zeroes for its samples. The MacOS-specific
-# tensorflow/lite/experimental/micro/examples/micro_speech/osx/audio_provider.cc
-# has an implementation that relies on CoreAudio, and there are equivalent
-# versions for other operating systems. 
-# All lists of source files are put through this substitution process with the
-# tags of their target OS and architecture, so that implementations can be added
-# by simply placing them in the file tree, with no changes to the build files
-# needed.
-# One confusing thing about this implementation is that we're using wildcard to
-# act as a 'does file exist?' function, rather than expanding an expression.
-# Wildcard will return an empty string if given a plain file path with no actual
-# wildcards, if the file doesn't exist, so taking the first word of the list
-# between that and the reference path will pick the specialized one if it's
-# available.
-substitute_specialized_implementation = \
-  $(firstword $(wildcard $(dir $(1))$(2)/$(notdir $(1))) $(wildcard $(1)))
-substitute_specialized_implementations = \
-  $(foreach source,$(1),$(call substitute_specialized_implementation,$(source),$(2)))
-# Here we're first looking for specialized implementations in ref_dir/$(TARGET)
-# and then ref_dir/$(TARGET_ARCH), before falling back to ref_dir's
-# implementation.
-# The argument to this function should be a list of space-separated file paths,
-# with any wildcards already expanded.
-specialize = \
-  $(call substitute_specialized_implementations,$(call substitute_specialized_implementations,$(1),$(TARGET)),$(TARGET_ARCH))
+# Specify TAGS on the command line to add a particular set of specialized
+# implementations, for example TAGS="CMSIS disco_f746ng" to target a Discovery
+# STM32F746NG board, using the CMSIS library's implementations where possible.
+ALL_TAGS := $(TAGS) $(TARGET)
 
 INCLUDES := \
 -I. \
@@ -89,6 +66,9 @@ MICROLITE_TEST_SRCS := \
 $(wildcard tensorflow/lite/experimental/micro/*test.cc) \
 $(wildcard tensorflow/lite/experimental/micro/kernels/*test.cc)
 
+MICROLITE_TEST_HDRS := \
+$(wildcard tensorflow/lite/experimental/micro/testing/*.h)
+
 MICROLITE_CC_BASE_SRCS := \
 $(wildcard tensorflow/lite/experimental/micro/*.cc) \
 $(wildcard tensorflow/lite/experimental/micro/kernels/*.cc) \
@@ -101,12 +81,58 @@ tensorflow/lite/kernels/internal/quantization_util.cc
 MICROLITE_CC_SRCS := $(filter-out $(MICROLITE_TEST_SRCS), $(MICROLITE_CC_BASE_SRCS))
 MICROLITE_CC_SRCS := $(call specialize,$(MICROLITE_CC_SRCS))
 
+MICROLITE_CC_HDRS := \
+$(wildcard tensorflow/lite/experimental/micro/*.h) \
+$(wildcard tensorflow/lite/experimental/micro/kernels/*.h) \
+LICENSE \
+tensorflow/lite/c/c_api_internal.h \
+tensorflow/lite/c/builtin_op_data.h \
+tensorflow/lite/core/api/error_reporter.h \
+tensorflow/lite/core/api/flatbuffer_conversions.h \
+tensorflow/lite/core/api/op_resolver.h \
+tensorflow/lite/kernels/kernel_util.h \
+tensorflow/lite/kernels/op_macros.h \
+tensorflow/lite/kernels/padding.h \
+tensorflow/lite/kernels/internal/common.h \
+tensorflow/lite/kernels/internal/compatibility.h \
+tensorflow/lite/kernels/internal/reference/depthwiseconv_float.h \
+tensorflow/lite/kernels/internal/reference/depthwiseconv_uint8.h \
+tensorflow/lite/kernels/internal/reference/fully_connected.h \
+tensorflow/lite/kernels/internal/reference/softmax.h \
+tensorflow/lite/kernels/internal/round.h \
+tensorflow/lite/kernels/internal/tensor_ctypes.h \
+tensorflow/lite/kernels/internal/types.h \
+tensorflow/lite/kernels/internal/quantization_util.h \
+tensorflow/lite/schema/schema_generated.h \
+tensorflow/lite/version.h
+
+THIRD_PARTY_CC_HDRS := \
+third_party/gemmlowp/fixedpoint/fixedpoint.h \
+third_party/gemmlowp/fixedpoint/fixedpoint_sse.h \
+third_party/gemmlowp/internal/detect_platform.h \
+third_party/gemmlowp/LICENSE \
+third_party/flatbuffers/include/flatbuffers/base.h \
+third_party/flatbuffers/include/flatbuffers/stl_emulation.h \
+third_party/flatbuffers/include/flatbuffers/flatbuffers.h \
+third_party/flatbuffers/LICENSE.txt
+
+MAKE_PROJECT_FILES := \
+  README_MAKE.md \
+  Makefile
+
+MBED_PROJECT_FILES := \
+  README_MBED.md \
+  mbed-os.lib \
+  mbed_app.json
+
 # These target-specific makefiles should modify or replace options like
 # CXXFLAGS or LIBS to work for a specific targetted architecture. All logic
 # based on platforms or architectures should happen within these files, to
 # keep this main makefile focused on the sources and dependencies.
 include $(wildcard $(MAKEFILE_DIR)/targets/*_makefile.inc)
 
+ALL_TAGS += $(TARGET_ARCH)
+
 ALL_SRCS := \
 	$(MICROLITE_CC_SRCS) \
 	$(MICROLITE_TEST_SRCS)
@@ -116,6 +142,7 @@ GENDIR := $(MAKEFILE_DIR)/gen/$(TARGET)_$(TARGET_ARCH)/
 OBJDIR := $(GENDIR)obj/
 BINDIR := $(GENDIR)bin/
 LIBDIR := $(GENDIR)lib/
+PRJDIR := $(GENDIR)prj/
 
 MICROLITE_LIB_PATH := $(LIBDIR)$(MICROLITE_LIB_NAME)
 
@@ -142,7 +169,7 @@ $(OBJDIR)%.o: %.c
 	@mkdir -p $(dir $@)
 	$(CC) $(CCFLAGS) $(INCLUDES) -c $< -o $@
 
-  # For normal manually-created TensorFlow ASM source files.
+# For normal manually-created TensorFlow ASM source files.
 $(OBJDIR)%.o: %.S
 	@mkdir -p $(dir $@)
 	$(CC) $(CCFLAGS) $(INCLUDES) -c $< -o $@
@@ -170,13 +197,9 @@ $(BINDIR)%_test : $(OBJDIR)%_test.o $(MICROLITE_LIB_PATH)
 $(BINDIR)%.test_target: $(BINDIR)%_test
 	$(TEST_SCRIPT) $< '~~~ALL TESTS PASSED~~~'
 
-# snease: Add %.bin rule here since BINDIR is now defined
-# These are microcontroller-specific rules for converting the ELF output
-# of the linker into a binary image that can be loaded directly.
-OBJCOPY := $(TARGET_TOOLCHAIN_PREFIX)objcopy
-$(BINDIR)%.bin: $(BINDIR)%
-	@mkdir -p $(dir $@)
-	$(OBJCOPY) $< $@ -O binary
+# Generate standalone makefile projects for all of the test targets.
+$(foreach TEST_TARGET,$(MICROLITE_TEST_SRCS),\
+$(eval $(call microlite_test,$(notdir $(basename $(TEST_TARGET))),$(TEST_TARGET))))
 
 test: test_micro_speech $(MICROLITE_TEST_TARGETS)
 
diff --git a/tensorflow/lite/experimental/micro/tools/make/helper_functions.inc b/tensorflow/lite/experimental/micro/tools/make/helper_functions.inc
new file mode 100644
index 0000000000..87c002635f
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/tools/make/helper_functions.inc
@@ -0,0 +1,117 @@
+
+# Reverses a space-separated list of words.
+reverse = $(if $(1),$(call reverse,$(wordlist 2,$(words $(1)),$(1)))) $(firstword $(1))
+
+# Look for platform or target-specific implementation files to replace reference
+# implementations with, given a tag. These are expected to occur in subfolders
+# of a directory where a reference implementation exists, and have the same
+# interface and header file. For example,
+# tensorflow/lite/experimental/micro/examples/micro_speech/audio_provider.cc
+# defines a module for supplying audio data, but since no platform or OS can be
+# presumed, it just always returns zeroes for its samples. The MacOS-specific
+# tensorflow/lite/experimental/micro/examples/micro_speech/osx/audio_provider.cc
+# has an implementation that relies on CoreAudio, and there are equivalent
+# versions for other operating systems.
+# The specific implementation yielded by the first tag in the list that produces
+# a match is returned, else the reference version if none of the tags produce a
+# match.
+# All lists of source files are put through this substitution process with the
+# tags of their target OS and architecture, so that implementations can be added
+# by simply placing them in the file tree, with no changes to the build files
+# needed.
+# One confusing thing about this implementation is that we're using wildcard to
+# act as a 'does file exist?' function, rather than expanding an expression.
+# Wildcard will return an empty string if given a plain file path with no actual
+# wildcards, if the file doesn't exist, so taking the first word of the list
+# between that and the reference path will pick the specialized one if it's
+# available.
+substitute_specialized_implementation = \
+  $(firstword $(wildcard $(dir $(1))$(2)/$(notdir $(1))) $(wildcard $(1)))
+substitute_specialized_implementations = \
+  $(foreach source,$(1),$(call substitute_specialized_implementation,$(source),$(2)))
+# Here we're first looking for specialized implementations in ref_dir/$(TAG1)
+# and then ref_dir/$(TAG2), etc, before falling back to ref_dir's
+# implementation.
+# The argument to this function should be a list of space-separated file paths,
+# with any wildcards already expanded.
+define specialize_on_tags
+$(if $(2),$(call substitute_specialized_implementations,$(call specialize_on_tags,$(1),$(wordlist 2,$(words $(2)),$(2))),$(firstword $(2))),$(1))
+endef
+# The entry point that most targets should use to find implementation-specific
+# versions of their source files. The only argument is a list of file paths.
+specialize = $(call specialize_on_tags,$(1),$(strip $(call reverse,$(ALL_TAGS))))
+
+# Creates a set of rules to build a standalone makefile project for an
+# executable, including all of the source and header files required in a
+# separate folder and a simple makefile.
+# Arguments are:
+# 1 - Project type (make, mbed, etc).
+# 2 - Project file template name.
+# 3 - Name of executable.
+# 4 - List of C/C++ source files needed to build the target.
+# 5 - List of C/C++ header files needed to build the target.
+# 6 - Linker flags required.
+# 7 - C++ compilation flags needed.
+# Calling eval on the output will create a <Name>_makefile target that you
+# can invoke to create the standalone project.
+define generate_project
+$(PRJDIR)$(3)/$(1)/%: %
+	@mkdir -p $$(dir $$@)
+	cp $$< $$@
+
+$(PRJDIR)$(3)/$(1)/third_party/%: tensorflow/lite/experimental/micro/tools/make/downloads/%
+	@mkdir -p $$(dir $$@)
+	cp $$< $$@
+
+$(PRJDIR)$(3)/$(1)/%: tensorflow/lite/experimental/micro/tools/make/templates/%.tpl
+	@mkdir -p $$(dir $$@)
+	sed -E 's#\%\{SRCS\}\%#$(4)#g' $$< | \
+	sed -E 's#\%\{EXECUTABLE\}\%#$(3)#g' | \
+	sed -E 's#\%\{LINKER_FLAGS\}\%#$(6)#g' | \
+	sed -E 's#\%\{CXX_FLAGS\}\%#$(7)#g' > $$@
+
+generate_$(3)_$(1)_project: $(addprefix $(PRJDIR)$(3)/$(1)/, $(4) $(5) $(2))
+endef
+
+# Specialized version of generate_project for TF Lite Micro test targets that
+# automatically includes standard library files, so you just need to pass the
+# test name and any extra source files required.
+# Arguments are:
+# 1 - Name of test.
+# 2 - C/C++ source files implementing the test.
+# 3 - C/C++ header files needed for the test.
+# Calling eval on the output will create targets that you can invoke to
+# generate the standalone project.
+define generate_microlite_projects
+$(call generate_project,make,$(MAKE_PROJECT_FILES),$(1),$(MICROLITE_CC_SRCS) $(THIRD_PARTY_CC_SRCS) $(2),$(MICROLITE_CC_HDRS) $(THIRD_PARTY_CC_HDRS) $(MICROLITE_TEST_HDRS) $(3),$(MICROLITE_LIBS),$(CXXFLAGS))
+$(call generate_project,mbed,$(MBED_PROJECT_FILES),$(1),$(MICROLITE_CC_SRCS) $(THIRD_PARTY_CC_SRCS) $(2),$(MICROLITE_CC_HDRS) $(THIRD_PARTY_CC_HDRS) $(MICROLITE_TEST_HDRS) $(3),$(MICROLITE_LIBS),$(CXXFLAGS))
+endef
+
+
+# Handles the details of generating a binary target, including specializing
+# for the current platform, and generating project file targets.
+# Arguments are:
+# 1 - Name of test.
+# 2 - C/C++ source files implementing the test.
+# 3 - C/C++ header files needed for the test.
+# Calling eval on the output will create the targets that you need.
+define microlite_test
+$(1)_LOCAL_SRCS := $(2)
+$(1)_LOCAL_SRCS := $$(call specialize,$$($(1)_LOCAL_SRCS))
+ALL_SRCS += $$($(1)_LOCAL_SRCS)
+$(1)_LOCAL_HDRS := $(3)
+$(1)_LOCAL_OBJS := $$(addprefix $$(OBJDIR), \
+$$(patsubst %.cc,%.o,$$(patsubst %.c,%.o,$$($(1)_LOCAL_SRCS))))
+$(1)_BINARY := $$(BINDIR)$(1)
+ALL_BINARIES += $$($(1)_BINARY)
+$$($(1)_BINARY): $$($(1)_LOCAL_OBJS) $$(MICROLITE_LIB_PATH)
+	@mkdir -p $$(dir $$@)
+	$$(CXX) $$(CXXFLAGS) $$(INCLUDES) \
+	-o $$($(1)_BINARY) $$($(1)_LOCAL_OBJS) \
+	$$(LIBFLAGS) $$(MICROLITE_LIB_PATH) $$(LDFLAGS) $$(MICROLITE_LIBS)
+$(1): $$($(1)_BINARY)
+$(1)_bin: $$($(1)_BINARY).bin
+test_$(1): $$($(1)_BINARY)
+	$$(TEST_SCRIPT) $$($(1)_BINARY) '~~~ALL TESTS PASSED~~~'
+$(eval $(call generate_microlite_projects,$(1),$(call specialize,$(2)),$(3)))
+endef
diff --git a/tensorflow/lite/experimental/micro/tools/make/targets/bluepill_makefile.inc b/tensorflow/lite/experimental/micro/tools/make/targets/bluepill_makefile.inc
index 5e3105a109..b344f844bc 100644
--- a/tensorflow/lite/experimental/micro/tools/make/targets/bluepill_makefile.inc
+++ b/tensorflow/lite/experimental/micro/tools/make/targets/bluepill_makefile.inc
@@ -47,7 +47,10 @@ ifeq ($(TARGET), bluepill)
   MICROLITE_CC_SRCS += \
     $(wildcard $(MAKEFILE_DIR)/downloads/stm32_bare_lib/source/*.c) \
     $(wildcard $(MAKEFILE_DIR)/downloads/stm32_bare_lib/source/*.cc)
-    TEST_SCRIPT := tensorflow/lite/experimental/micro/testing/test_bluepill_binary.sh
+  EXCLUDED_SRCS := \
+    $(MAKEFILE_DIR)/downloads/stm32_bare_lib/source/debug_log.c
+  MICROLITE_CC_SRCS := $(filter-out $(EXCLUDED_SRCS), $(MICROLITE_CC_SRCS))
+  TEST_SCRIPT := tensorflow/lite/experimental/micro/testing/test_bluepill_binary.sh
   # These are tests that don't currently work on the blue pill.
   EXCLUDED_TESTS := \
     tensorflow/lite/experimental/micro/micro_interpreter_test.cc \
diff --git a/tensorflow/lite/experimental/micro/tools/make/targets/mbed_makefile.inc b/tensorflow/lite/experimental/micro/tools/make/targets/mbed_makefile.inc
new file mode 100644
index 0000000000..161ff34cdb
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/tools/make/targets/mbed_makefile.inc
@@ -0,0 +1,4 @@
+# Settings for mbed platforms.
+ifeq ($(TARGET), mbed)
+  TARGET_ARCH := cortex-m4
+endif
\ No newline at end of file
diff --git a/tensorflow/lite/experimental/micro/tools/make/targets/osx_makefile.inc b/tensorflow/lite/experimental/micro/tools/make/targets/osx_makefile.inc
index 0e8aad1993..3b91eeff9f 100644
--- a/tensorflow/lite/experimental/micro/tools/make/targets/osx_makefile.inc
+++ b/tensorflow/lite/experimental/micro/tools/make/targets/osx_makefile.inc
@@ -1,4 +1,4 @@
-#Settings for Mac OS platforms.
+# Settings for Mac OS platforms.
 ifeq ($(TARGET), osx)
 
   PLATFORM_FLAGS = \
diff --git a/tensorflow/lite/experimental/micro/tools/make/templates/AUDIO_DISCO_F746NG.lib.tpl b/tensorflow/lite/experimental/micro/tools/make/templates/AUDIO_DISCO_F746NG.lib.tpl
new file mode 100644
index 0000000000..11dae1ea16
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/tools/make/templates/AUDIO_DISCO_F746NG.lib.tpl
@@ -0,0 +1 @@
+https://os.mbed.com/teams/ST/code/AUDIO_DISCO_F746NG/#7046ce26b7ed
diff --git a/tensorflow/lite/experimental/micro/tools/make/templates/BSP_DISCO_F746NG.lib.tpl b/tensorflow/lite/experimental/micro/tools/make/templates/BSP_DISCO_F746NG.lib.tpl
new file mode 100644
index 0000000000..48dc131707
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/tools/make/templates/BSP_DISCO_F746NG.lib.tpl
@@ -0,0 +1 @@
+https://os.mbed.com/teams/ST/code/BSP_DISCO_F746NG/#df2ea349c37a
diff --git a/tensorflow/lite/experimental/micro/tools/make/templates/Makefile.tpl b/tensorflow/lite/experimental/micro/tools/make/templates/Makefile.tpl
new file mode 100644
index 0000000000..74d54f1ebe
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/tools/make/templates/Makefile.tpl
@@ -0,0 +1,26 @@
+SRCS := \
+%{SRCS}%
+
+OBJS := \
+$(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(SRCS)))
+
+INCLUDES := \
+-I. \
+-I./third_party/gemmlowp \
+-I./third_party/flatbuffers/include
+
+CXXFLAGS += %{CXX_FLAGS}%
+
+LDFLAGS += %{LINKER_FLAGS}%
+
+%.o: %.cc
+	$(CXX) $(CXXFLAGS) $(INCLUDES) -c $< -o $@
+
+%.o: %.c
+	$(CC) $(CCFLAGS) $(INCLUDES) -c $< -o $@
+
+%{EXECUTABLE}% : $(OBJS)
+	$(CXX) $(LDFLAGS) $(OBJS) \
+	-o $@
+
+all: %{EXECUTABLE}%
diff --git a/tensorflow/lite/experimental/micro/tools/make/templates/README_MAKE.md.tpl b/tensorflow/lite/experimental/micro/tools/make/templates/README_MAKE.md.tpl
new file mode 100644
index 0000000000..7906a3226a
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/tools/make/templates/README_MAKE.md.tpl
@@ -0,0 +1,29 @@
+# TensorFlow Lite Micro Make Project
+
+This folder has been autogenerated by TensorFlow, and contains source, header,
+and project files needed to build a single TensorFlow Lite Micro target using
+the make tool.
+
+## Usage
+
+To build this, run:
+
+```
+make
+```
+
+This should attempt to build the target locally on your platform, using the
+standard Makefile variables like CFLAGS, CC, CXX, and so on.
+
+## Project Generation
+
+See
+[tensorflow/lite/experimental/micro](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/experimental/micro)
+for details on how projects like this can be generated from the main source
+tree.
+
+## License
+
+TensorFlow's code is covered by the Apache2 License included in the repository,
+and third party dependencies are covered by their respective licenses, in the
+third_party folder of this package.
diff --git a/tensorflow/lite/experimental/micro/tools/make/templates/README_MBED.md.tpl b/tensorflow/lite/experimental/micro/tools/make/templates/README_MBED.md.tpl
new file mode 100644
index 0000000000..2682236edf
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/tools/make/templates/README_MBED.md.tpl
@@ -0,0 +1,48 @@
+# TensorFlow Lite Micro Mbed Project
+
+This folder has been autogenerated by TensorFlow, and contains source, header,
+and project files needed to build a single TensorFlow Lite Micro target using
+the Mbed command line interface.
+
+## Usage
+
+To load the dependencies this code requires, run:
+
+```
+mbed config root .
+mbed deploy
+```
+
+TensorFlow requires C++ 11, so you'll need to update your profiles to reflect
+this. Here's a short Python command that does that:
+
+```
+python -c 'import fileinput, glob;
+for filename in glob.glob("mbed-os/tools/profiles/*.json"):
+  for line in fileinput.input(filename, inplace=True):
+    print line.replace("\"-std=gnu++98\"","\"-std=c++11\", \"-fpermissive\"")'
+```
+
+With that setting updated, you should now be able to compile:
+
+```
+mbed compile -m auto -t GCC_ARM
+```
+
+If this works, it will give you a .bin file that you can flash onto the device
+you're targeting. For example, using a Discovery STM3246G board, you can deploy
+it by copying the bin to the volume mounted as a USB drive, just by dragging
+over the file.
+
+## Project Generation
+
+See
+[tensorflow/lite/experimental/micro](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/experimental/micro)
+for details on how projects like this can be generated from the main source
+tree.
+
+## License
+
+TensorFlow's code is covered by the Apache2 License included in the repository,
+and third party dependencies are covered by their respective licenses, in the
+third_party folder of this package.
diff --git a/tensorflow/lite/experimental/micro/tools/make/templates/SDRAM_DISCO_F746NG.lib.tpl b/tensorflow/lite/experimental/micro/tools/make/templates/SDRAM_DISCO_F746NG.lib.tpl
new file mode 100644
index 0000000000..e2ccd7b81b
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/tools/make/templates/SDRAM_DISCO_F746NG.lib.tpl
@@ -0,0 +1 @@
+https://os.mbed.com/teams/ST/code/SDRAM_DISCO_F746NG/#370f402a2219
diff --git a/tensorflow/lite/experimental/micro/tools/make/templates/mbed-os.lib.tpl b/tensorflow/lite/experimental/micro/tools/make/templates/mbed-os.lib.tpl
new file mode 100644
index 0000000000..69fff22f33
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/tools/make/templates/mbed-os.lib.tpl
@@ -0,0 +1 @@
+https://github.com/ARMmbed/mbed-os/#6a0a86538c0b9b2bfcc4583b1e2b7fea8f4e71e9
diff --git a/tensorflow/lite/experimental/micro/tools/make/templates/mbed_app.json.tpl b/tensorflow/lite/experimental/micro/tools/make/templates/mbed_app.json.tpl
new file mode 100644
index 0000000000..1c547369fb
--- /dev/null
+++ b/tensorflow/lite/experimental/micro/tools/make/templates/mbed_app.json.tpl
@@ -0,0 +1,7 @@
+{
+    "config": {
+	"main-stack-size": {
+            "value": 65536
+	}
+    }
+}
-- 
GitLab


From 8fefcf7b8b236baf94674ca8fcb0777fd48c2ccd Mon Sep 17 00:00:00 2001
From: Tim Shen <timshen@google.com>
Date: Wed, 9 Jan 2019 12:42:37 -0800
Subject: [PATCH 0472/1095] Roll-forward: PR #24081: Split convolution
 invocation into preparation and actual invocation

Please approve this CL. It will be submitted automatically, and its GitHub pull request will be marked as merged.

Imported from GitHub PR #24081

- split DoConvolve into:

PrepareForConvolution
DoConvolve

- split DoConvolveBackwardData into:

PrepareForConvolutionBackwardData
DoConvolveBackwardData

- split DoConvolveBackwardFilter into:

PrepareForConvolutionBackwardFilter
DoConvolveBackwardFilter

PrepareForConvolutionXXX would allocate scratch memory.
DoConolveXXX would invoke actual convolution algorithms.

Implement forward convoution, backward input convolution, backward filter
convolution on CUDA path.

Copybara import of the project:

  - fe9a9dbb8d0ef118b15beb4724f256190fd04d13 Split convolution invocation into preparation and actual ... by Wen-Heng (Jack) Chung <whchung@gmail.com>
  - baeda0d8b5da58a97d28ddce264c80c58d937699 Merge fe9a9dbb8d0ef118b15beb4724f256190fd04d13 into bbe33... by Wen-Heng (Jack) Chung <whchung@gmail.com>

PiperOrigin-RevId: 228568812
---
 tensorflow/stream_executor/cuda/cuda_dnn.cc | 471 +++++++++++++++-----
 tensorflow/stream_executor/cuda/cuda_dnn.h  | 194 +++++++-
 tensorflow/stream_executor/dnn.h            | 207 +++++++--
 tensorflow/stream_executor/stream.cc        | 172 +++++--
 4 files changed, 859 insertions(+), 185 deletions(-)

diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.cc b/tensorflow/stream_executor/cuda/cuda_dnn.cc
index a34aa9354d..0bd953fe3b 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.cc
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.cc
@@ -2799,7 +2799,7 @@ void LogCudaProto(const dnn::ConvolutionProto& conv, float profile_time_ms,
 }  // namespace
 
 template <class T>
-port::Status CudnnSupport::DoConvolveImpl(
+port::Status CudnnSupport::PrepareForConvolutionImpl(
     Stream* stream, const dnn::BatchDescriptor& input_descriptor,
     const DeviceMemory<T>& input_data,
     const dnn::FilterDescriptor& filter_descriptor,
@@ -2808,6 +2808,34 @@ port::Status CudnnSupport::DoConvolveImpl(
     const dnn::BatchDescriptor& output_descriptor, DeviceMemory<T>* output_data,
     dnn::DataType accumulator_type, ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
+    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
+  cudnnDataType_t cudnn_type = GetCudnnDataType<T>();
+  CudnnTensorDescriptor input_nd(input_descriptor, cudnn_type);
+  CudnnTensorDescriptor output_nd(output_descriptor, cudnn_type);
+  CudnnFilterDescriptor filter(filter_descriptor, cudnn_type);
+  CudnnConvolutionDescriptor conv(convolution_descriptor,
+                                  ToCudnnDataType(accumulator_type));
+
+  auto cudnn = cudnn_->GetHandle(parent_, stream);
+
+  SE_ASSIGN_OR_RETURN(*algorithm_desc,
+                      GetCudnnConvolutionForwardAlgorithm(
+                          stream, cudnn, algorithm_config, input_nd, filter,
+                          conv, output_nd, scratch_allocator, scratch_memory));
+
+  return port::Status::OK();
+}
+
+template <class T>
+port::Status CudnnSupport::DoConvolveImpl(
+    Stream* stream, const dnn::BatchDescriptor& input_descriptor,
+    const DeviceMemory<T>& input_data,
+    const dnn::FilterDescriptor& filter_descriptor,
+    const DeviceMemory<T>& filter_data,
+    const dnn::ConvolutionDescriptor& convolution_descriptor,
+    const dnn::BatchDescriptor& output_descriptor, DeviceMemory<T>* output_data,
+    dnn::DataType accumulator_type, const dnn::AlgorithmDesc& algorithm_desc,
+    DeviceMemory<uint8>* scratch_memory,
     dnn::ProfileResult* output_profile_result) {
   cudnnDataType_t cudnn_type = GetCudnnDataType<T>();
   CudnnTensorDescriptor input_nd(input_descriptor, cudnn_type);
@@ -2830,12 +2858,6 @@ port::Status CudnnSupport::DoConvolveImpl(
 
   const bool is_profiling = output_profile_result != nullptr;
 
-  DeviceMemory<uint8> scratch;
-  SE_ASSIGN_OR_RETURN(dnn::AlgorithmDesc algo_desc,
-                      GetCudnnConvolutionForwardAlgorithm(
-                          stream, cudnn, algorithm_config, input_nd, filter,
-                          conv, output_nd, scratch_allocator, &scratch));
-
   std::unique_ptr<CUDATimer, TimerDeleter> timer;
   if (is_profiling) {
     timer.reset(new CUDATimer(parent_));  // NOLINT
@@ -2851,7 +2873,7 @@ port::Status CudnnSupport::DoConvolveImpl(
   // memory. See nvbugs/2138754, b/80018418.
   if (CUDNN_VERSION < 7300) {
     SE_RETURN_IF_ERROR([&] {
-      if (algo_desc.algo_id() != CUDNN_CONVOLUTION_FWD_ALGO_FFT_TILING) {
+      if (algorithm_desc.algo_id() != CUDNN_CONVOLUTION_FWD_ALGO_FFT_TILING) {
         return port::Status::OK();
       }
       if (input_descriptor.ndims() < 3) {
@@ -2876,7 +2898,8 @@ port::Status CudnnSupport::DoConvolveImpl(
     }());
   }
 
-  if (algo_desc.algo_id() == CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED &&
+  if (algorithm_desc.algo_id() ==
+          CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED &&
       !ShouldIncludeWinogradNonfusedAlgo(input_descriptor, output_descriptor)) {
     return port::Status(port::error::FAILED_PRECONDITION,
                         "This configuration has potential integer overflow in "
@@ -2888,24 +2911,25 @@ port::Status CudnnSupport::DoConvolveImpl(
       /*alpha=*/alpha, /*srcDesc=*/input_nd.handle(),
       /*srcData=*/input_data.opaque(), /*filterDesc=*/filter.handle(),
       /*filterData=*/filter_data.opaque(), /*convDesc=*/conv.handle(),
-      /*algo=*/ToConvForwardAlgo(algo_desc), /*workSpace=*/scratch.opaque(),
-      /*workSpaceSizeInBytes=*/scratch.size(), /*beta=*/beta,
+      /*algo=*/ToConvForwardAlgo(algorithm_desc),
+      /*workSpace=*/scratch_memory->opaque(),
+      /*workSpaceSizeInBytes=*/scratch_memory->size(), /*beta=*/beta,
       /*yDesc=*/output_nd.handle(), /*y=*/output_data->opaque()));
 
   if (is_profiling) {
     if (!timer->Stop(AsCUDAStream(stream))) {
       return port::Status(port::error::INTERNAL, "Failed to stop timer");
     }
-    output_profile_result->set_algorithm(algo_desc);
+    output_profile_result->set_algorithm(algorithm_desc);
     output_profile_result->set_elapsed_time_in_ms(
         timer->GetElapsedMilliseconds());
-    output_profile_result->set_scratch_size(scratch.size());
+    output_profile_result->set_scratch_size(scratch_memory->size());
 
     LogCudaProto(
-        GenerateConvProto<T>(dnn::ConvolutionKind::FORWARD, input_descriptor,
-                             filter_descriptor, output_descriptor, algo_desc,
-                             convolution_descriptor, dalpha, dbeta,
-                             accumulator_type, dnn::ActivationMode::kNone),
+        GenerateConvProto<T>(
+            dnn::ConvolutionKind::FORWARD, input_descriptor, filter_descriptor,
+            output_descriptor, algorithm_desc, convolution_descriptor, dalpha,
+            dbeta, accumulator_type, dnn::ActivationMode::kNone),
         output_profile_result->elapsed_time_in_ms(), stream->parent());
   }
 
@@ -3310,7 +3334,7 @@ port::Status CudnnSupport::DoBatchNormalizationBackwardImpl(
   return port::Status::OK();
 }
 
-bool CudnnSupport::DoConvolve(
+bool CudnnSupport::PrepareForConvolution(
     Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
     const DeviceMemory<float>& input_data,
     const dnn::FilterDescriptor& filter_descriptor,
@@ -3319,12 +3343,70 @@ bool CudnnSupport::DoConvolve(
     const dnn::BatchDescriptor& output_descriptor,
     DeviceMemory<float>* output_data, ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
+    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
+  return IsStatusOk(PrepareForConvolutionImpl<float>(
+                        stream, batch_descriptor, input_data, filter_descriptor,
+                        filter_data, convolution_descriptor, output_descriptor,
+                        output_data, dnn::DataType::kFloat, scratch_allocator,
+                        algorithm_config, algorithm_desc, scratch_memory),
+                    /*report_error=*/true);
+}
+
+bool CudnnSupport::PrepareForConvolution(
+    Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
+    const DeviceMemory<double>& input_data,
+    const dnn::FilterDescriptor& filter_descriptor,
+    const DeviceMemory<double>& filter_data,
+    const dnn::ConvolutionDescriptor& convolution_descriptor,
+    const dnn::BatchDescriptor& output_descriptor,
+    DeviceMemory<double>* output_data, ScratchAllocator* scratch_allocator,
+    const dnn::AlgorithmConfig& algorithm_config,
+    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
+  return IsStatusOk(PrepareForConvolutionImpl<double>(
+                        stream, batch_descriptor, input_data, filter_descriptor,
+                        filter_data, convolution_descriptor, output_descriptor,
+                        output_data, dnn::DataType::kDouble, scratch_allocator,
+                        algorithm_config, algorithm_desc, scratch_memory),
+                    /*report_error=*/true);
+}
+
+bool CudnnSupport::PrepareForConvolution(
+    Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
+    const DeviceMemory<Eigen::half>& input_data,
+    const dnn::FilterDescriptor& filter_descriptor,
+    const DeviceMemory<Eigen::half>& filter_data,
+    const dnn::ConvolutionDescriptor& convolution_descriptor,
+    const dnn::BatchDescriptor& output_descriptor,
+    DeviceMemory<Eigen::half>* output_data, ScratchAllocator* scratch_allocator,
+    const dnn::AlgorithmConfig& algorithm_config,
+    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
+  dnn::DataType acc_type =
+      CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()
+          ? dnn::DataType::kFloat
+          : dnn::DataType::kHalf;
+  return IsStatusOk(
+      PrepareForConvolutionImpl<Eigen::half>(
+          stream, batch_descriptor, input_data, filter_descriptor, filter_data,
+          convolution_descriptor, output_descriptor, output_data, acc_type,
+          scratch_allocator, algorithm_config, algorithm_desc, scratch_memory),
+      /*report_error=*/true);
+}
+
+bool CudnnSupport::DoConvolve(
+    Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
+    const DeviceMemory<float>& input_data,
+    const dnn::FilterDescriptor& filter_descriptor,
+    const DeviceMemory<float>& filter_data,
+    const dnn::ConvolutionDescriptor& convolution_descriptor,
+    const dnn::BatchDescriptor& output_descriptor,
+    DeviceMemory<float>* output_data, const dnn::AlgorithmDesc& algorithm_desc,
+    DeviceMemory<uint8>* scratch_memory,
     dnn::ProfileResult* output_profile_result) {
   return IsStatusOk(
       DoConvolveImpl(stream, batch_descriptor, input_data, filter_descriptor,
                      filter_data, convolution_descriptor, output_descriptor,
-                     output_data, dnn::DataType::kFloat, scratch_allocator,
-                     algorithm_config, output_profile_result),
+                     output_data, dnn::DataType::kFloat, algorithm_desc,
+                     scratch_memory, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
@@ -3335,14 +3417,14 @@ bool CudnnSupport::DoConvolve(
     const DeviceMemory<double>& filter_data,
     const dnn::ConvolutionDescriptor& convolution_descriptor,
     const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<double>* output_data, ScratchAllocator* scratch_allocator,
-    const dnn::AlgorithmConfig& algorithm_config,
+    DeviceMemory<double>* output_data, const dnn::AlgorithmDesc& algorithm_desc,
+    DeviceMemory<uint8>* scratch_memory,
     dnn::ProfileResult* output_profile_result) {
   return IsStatusOk(
       DoConvolveImpl(stream, batch_descriptor, input_data, filter_descriptor,
                      filter_data, convolution_descriptor, output_descriptor,
-                     output_data, dnn::DataType::kDouble, scratch_allocator,
-                     algorithm_config, output_profile_result),
+                     output_data, dnn::DataType::kDouble, algorithm_desc,
+                     scratch_memory, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
@@ -3353,8 +3435,9 @@ bool CudnnSupport::DoConvolve(
     const DeviceMemory<Eigen::half>& filter_data,
     const dnn::ConvolutionDescriptor& convolution_descriptor,
     const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<Eigen::half>* output_data, ScratchAllocator* scratch_allocator,
-    const dnn::AlgorithmConfig& algorithm_config,
+    DeviceMemory<Eigen::half>* output_data,
+    const dnn::AlgorithmDesc& algorithm_desc,
+    DeviceMemory<uint8>* scratch_memory,
     dnn::ProfileResult* output_profile_result) {
   dnn::DataType acc_type =
       CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()
@@ -3363,7 +3446,7 @@ bool CudnnSupport::DoConvolve(
   return IsStatusOk(
       DoConvolveImpl(stream, batch_descriptor, input_data, filter_descriptor,
                      filter_data, convolution_descriptor, output_descriptor,
-                     output_data, acc_type, scratch_allocator, algorithm_config,
+                     output_data, acc_type, algorithm_desc, scratch_memory,
                      output_profile_result),
       /*report_error=*/!output_profile_result);
 }
@@ -3499,7 +3582,7 @@ bool CudnnSupport::DoTransformTensor(Stream* stream,
 }
 
 template <class T>
-port::Status CudnnSupport::DoConvolveBackwardDataImpl(
+port::Status CudnnSupport::PrepareForConvolutionBackwardDataImpl(
     Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
     const DeviceMemory<T>& filter_data,
     const dnn::BatchDescriptor& output_descriptor,
@@ -3509,6 +3592,36 @@ port::Status CudnnSupport::DoConvolveBackwardDataImpl(
     DeviceMemory<T>* backward_input_data, dnn::DataType accumulator_type,
     ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
+    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
+  cudnnDataType_t cudnn_type = GetCudnnDataType<T>();
+  auto cudnn = cudnn_->GetHandle(parent_, stream);
+
+  CudnnTensorDescriptor out_back_nd(output_descriptor, cudnn_type);
+  CudnnTensorDescriptor in_back_nd(input_descriptor, cudnn_type);
+  CudnnFilterDescriptor filter(filter_descriptor, cudnn_type);
+  CudnnConvolutionDescriptor conv(convolution_descriptor,
+                                  ToCudnnDataType(accumulator_type));
+
+  SE_ASSIGN_OR_RETURN(
+      *algorithm_desc,
+      GetCudnnConvolutionBackwardDataAlgorithm(
+          stream, cudnn, algorithm_config, in_back_nd, filter, conv,
+          out_back_nd, scratch_allocator, scratch_memory));
+
+  return port::Status::OK();
+}
+
+template <class T>
+port::Status CudnnSupport::DoConvolveBackwardDataImpl(
+    Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
+    const DeviceMemory<T>& filter_data,
+    const dnn::BatchDescriptor& output_descriptor,
+    DeviceMemory<T> backward_output_data,
+    const dnn::ConvolutionDescriptor& convolution_descriptor,
+    const dnn::BatchDescriptor& input_descriptor,
+    DeviceMemory<T>* backward_input_data, dnn::DataType accumulator_type,
+    const dnn::AlgorithmDesc& algorithm_desc,
+    DeviceMemory<uint8>* scratch_memory,
     dnn::ProfileResult* output_profile_result) {
   cudnnDataType_t cudnn_type = GetCudnnDataType<T>();
   // Alpha is the scaling factor for input.
@@ -3532,12 +3645,6 @@ port::Status CudnnSupport::DoConvolveBackwardDataImpl(
 
   const bool is_profiling = output_profile_result != nullptr;
 
-  DeviceMemory<uint8> scratch;
-  SE_ASSIGN_OR_RETURN(dnn::AlgorithmDesc algo_desc,
-                      GetCudnnConvolutionBackwardDataAlgorithm(
-                          stream, cudnn, algorithm_config, in_back_nd, filter,
-                          conv, out_back_nd, scratch_allocator, &scratch));
-
   std::unique_ptr<CUDATimer, TimerDeleter> timer;
   if (is_profiling) {
     timer.reset(new CUDATimer(parent_));  // NOLINT
@@ -3549,7 +3656,8 @@ port::Status CudnnSupport::DoConvolveBackwardDataImpl(
     }
   }
 
-  if (algo_desc.algo_id() == CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED &&
+  if (algorithm_desc.algo_id() ==
+          CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED &&
       !ShouldIncludeWinogradNonfusedAlgo(input_descriptor, output_descriptor)) {
     return port::Status(port::error::FAILED_PRECONDITION,
                         "This configuration has potential integer overflow in "
@@ -3559,42 +3667,42 @@ port::Status CudnnSupport::DoConvolveBackwardDataImpl(
   // Cudnn 7.1.4 has a bug if the workspace of the following convolution is not
   // zero-initialized, nvbugs/2254619.
   if (CUDNN_VERSION >= 7000 && CUDNN_VERSION < 7300 &&
-      algo_desc.algo_id() == CUDNN_CONVOLUTION_BWD_DATA_ALGO_1 &&
-      cudnn_type == CUDNN_DATA_HALF && algo_desc.tensor_ops_enabled() &&
+      algorithm_desc.algo_id() == CUDNN_CONVOLUTION_BWD_DATA_ALGO_1 &&
+      cudnn_type == CUDNN_DATA_HALF && algorithm_desc.tensor_ops_enabled() &&
       input_descriptor.layout() == dnn::DataLayout::kBatchYXDepth &&
       filter_descriptor.layout() == dnn::FilterLayout::kOutputInputYX &&
       output_descriptor.layout() == dnn::DataLayout::kBatchDepthYX &&
       (convolution_descriptor.vertical_filter_stride() > 1 ||
        convolution_descriptor.horizontal_filter_stride() > 1)) {
-    stream->ThenMemZero(&scratch, scratch.size());
+    stream->ThenMemZero(scratch_memory, scratch_memory->size());
   }
 
-  RETURN_IF_CUDNN_ERROR(
-      cudnnConvolutionBackwardData(cudnn.handle(),
-                                   /*alpha=*/alpha,
-                                   /*wDesc=*/filter.handle(),
-                                   /*w=*/filter_data.opaque(),
-                                   /*dyDesc=*/out_back_nd.handle(),
-                                   /*dy=*/backward_output_data.opaque(),
-                                   /*convDesc=*/conv.handle(),
-                                   /*algo=*/ToConvBackwardDataAlgo(algo_desc),
-                                   /*workSpace=*/scratch.opaque(),
-                                   /*workSpaceSizeInBytes=*/scratch.size(),
-                                   /*beta=*/beta,
-                                   /*dxDesc=*/in_back_nd.handle(),
-                                   /*dx=*/backward_input_data->opaque()));
+  RETURN_IF_CUDNN_ERROR(cudnnConvolutionBackwardData(
+      cudnn.handle(),
+      /*alpha=*/alpha,
+      /*wDesc=*/filter.handle(),
+      /*w=*/filter_data.opaque(),
+      /*dyDesc=*/out_back_nd.handle(),
+      /*dy=*/backward_output_data.opaque(),
+      /*convDesc=*/conv.handle(),
+      /*algo=*/ToConvBackwardDataAlgo(algorithm_desc),
+      /*workSpace=*/scratch_memory->opaque(),
+      /*workSpaceSizeInBytes=*/scratch_memory->size(),
+      /*beta=*/beta,
+      /*dxDesc=*/in_back_nd.handle(),
+      /*dx=*/backward_input_data->opaque()));
   if (is_profiling) {
     if (!timer->Stop(AsCUDAStream(stream))) {
       return port::Status(port::error::INTERNAL, "Failed to stop timer");
     }
-    output_profile_result->set_algorithm(algo_desc);
+    output_profile_result->set_algorithm(algorithm_desc);
     output_profile_result->set_elapsed_time_in_ms(
         timer->GetElapsedMilliseconds());
-    output_profile_result->set_scratch_size(scratch.size());
+    output_profile_result->set_scratch_size(scratch_memory->size());
 
     LogCudaProto(GenerateConvProto<T>(
                      dnn::ConvolutionKind::BACKWARD_DATA, input_descriptor,
-                     filter_descriptor, output_descriptor, algo_desc,
+                     filter_descriptor, output_descriptor, algorithm_desc,
                      convolution_descriptor, dalpha, dbeta, accumulator_type,
                      dnn::ActivationMode::kNone),
                  output_profile_result->elapsed_time_in_ms(), stream->parent());
@@ -3603,7 +3711,7 @@ port::Status CudnnSupport::DoConvolveBackwardDataImpl(
   return port::Status::OK();
 }
 
-bool CudnnSupport::DoConvolveBackwardData(
+bool CudnnSupport::PrepareForConvolutionBackwardData(
     Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
     const DeviceMemory<double>& filter_data,
     const dnn::BatchDescriptor& output_descriptor,
@@ -3613,17 +3721,17 @@ bool CudnnSupport::DoConvolveBackwardData(
     DeviceMemory<double>* backward_input_data,
     ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
-    dnn::ProfileResult* output_profile_result) {
+    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
   return IsStatusOk(
-      DoConvolveBackwardDataImpl(
+      PrepareForConvolutionBackwardDataImpl(
           stream, filter_descriptor, filter_data, output_descriptor,
           backward_output_data, convolution_descriptor, input_descriptor,
           backward_input_data, dnn::DataType::kDouble, scratch_allocator,
-          algorithm_config, output_profile_result),
-      /*report_error=*/!output_profile_result);
+          algorithm_config, algorithm_desc, scratch_memory),
+      /*report_error=*/true);
 }
 
-bool CudnnSupport::DoConvolveBackwardData(
+bool CudnnSupport::PrepareForConvolutionBackwardData(
     Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
     const DeviceMemory<float>& filter_data,
     const dnn::BatchDescriptor& output_descriptor,
@@ -3633,17 +3741,17 @@ bool CudnnSupport::DoConvolveBackwardData(
     DeviceMemory<float>* backward_input_data,
     ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
-    dnn::ProfileResult* output_profile_result) {
+    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
   return IsStatusOk(
-      DoConvolveBackwardDataImpl(
+      PrepareForConvolutionBackwardDataImpl(
           stream, filter_descriptor, filter_data, output_descriptor,
           backward_output_data, convolution_descriptor, input_descriptor,
           backward_input_data, dnn::DataType::kFloat, scratch_allocator,
-          algorithm_config, output_profile_result),
-      /*report_error=*/!output_profile_result);
+          algorithm_config, algorithm_desc, scratch_memory),
+      /*report_error=*/true);
 }
 
-bool CudnnSupport::DoConvolveBackwardData(
+bool CudnnSupport::PrepareForConvolutionBackwardData(
     Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
     const DeviceMemory<Eigen::half>& filter_data,
     const dnn::BatchDescriptor& output_descriptor,
@@ -3653,22 +3761,86 @@ bool CudnnSupport::DoConvolveBackwardData(
     DeviceMemory<Eigen::half>* backward_input_data,
     ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
-    dnn::ProfileResult* output_profile_result) {
+    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
   dnn::DataType acc_type =
       CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()
           ? dnn::DataType::kFloat
           : dnn::DataType::kHalf;
   return IsStatusOk(
-      DoConvolveBackwardDataImpl(
+      PrepareForConvolutionBackwardDataImpl(
           stream, filter_descriptor, filter_data, output_descriptor,
           backward_output_data, convolution_descriptor, input_descriptor,
           backward_input_data, acc_type, scratch_allocator, algorithm_config,
-          output_profile_result),
+          algorithm_desc, scratch_memory),
+      /*report_error=*/true);
+}
+
+bool CudnnSupport::DoConvolveBackwardData(
+    Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
+    const DeviceMemory<double>& filter_data,
+    const dnn::BatchDescriptor& output_descriptor,
+    DeviceMemory<double> backward_output_data,
+    const dnn::ConvolutionDescriptor& convolution_descriptor,
+    const dnn::BatchDescriptor& input_descriptor,
+    DeviceMemory<double>* backward_input_data,
+    const dnn::AlgorithmDesc& algorithm_desc,
+    DeviceMemory<uint8>* scratch_memory,
+    dnn::ProfileResult* output_profile_result) {
+  return IsStatusOk(
+      DoConvolveBackwardDataImpl(
+          stream, filter_descriptor, filter_data, output_descriptor,
+          backward_output_data, convolution_descriptor, input_descriptor,
+          backward_input_data, dnn::DataType::kDouble, algorithm_desc,
+          scratch_memory, output_profile_result),
+      /*report_error=*/!output_profile_result);
+}
+
+bool CudnnSupport::DoConvolveBackwardData(
+    Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
+    const DeviceMemory<float>& filter_data,
+    const dnn::BatchDescriptor& output_descriptor,
+    DeviceMemory<float> backward_output_data,
+    const dnn::ConvolutionDescriptor& convolution_descriptor,
+    const dnn::BatchDescriptor& input_descriptor,
+    DeviceMemory<float>* backward_input_data,
+    const dnn::AlgorithmDesc& algorithm_desc,
+    DeviceMemory<uint8>* scratch_memory,
+    dnn::ProfileResult* output_profile_result) {
+  return IsStatusOk(
+      DoConvolveBackwardDataImpl(
+          stream, filter_descriptor, filter_data, output_descriptor,
+          backward_output_data, convolution_descriptor, input_descriptor,
+          backward_input_data, dnn::DataType::kFloat, algorithm_desc,
+          scratch_memory, output_profile_result),
+      /*report_error=*/!output_profile_result);
+}
+
+bool CudnnSupport::DoConvolveBackwardData(
+    Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
+    const DeviceMemory<Eigen::half>& filter_data,
+    const dnn::BatchDescriptor& output_descriptor,
+    DeviceMemory<Eigen::half> backward_output_data,
+    const dnn::ConvolutionDescriptor& convolution_descriptor,
+    const dnn::BatchDescriptor& input_descriptor,
+    DeviceMemory<Eigen::half>* backward_input_data,
+    const dnn::AlgorithmDesc& algorithm_desc,
+    DeviceMemory<uint8>* scratch_memory,
+    dnn::ProfileResult* output_profile_result) {
+  dnn::DataType acc_type =
+      CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()
+          ? dnn::DataType::kFloat
+          : dnn::DataType::kHalf;
+  return IsStatusOk(
+      DoConvolveBackwardDataImpl(stream, filter_descriptor, filter_data,
+                                 output_descriptor, backward_output_data,
+                                 convolution_descriptor, input_descriptor,
+                                 backward_input_data, acc_type, algorithm_desc,
+                                 scratch_memory, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
 template <class T>
-port::Status CudnnSupport::DoConvolveBackwardFilterImpl(
+port::Status CudnnSupport::PrepareForConvolutionBackwardFilterImpl(
     Stream* stream, const dnn::BatchDescriptor& input_descriptor,
     const DeviceMemory<T>& input_data,
     const dnn::BatchDescriptor& output_descriptor,
@@ -3678,6 +3850,36 @@ port::Status CudnnSupport::DoConvolveBackwardFilterImpl(
     DeviceMemory<T>* backward_filter_data, dnn::DataType accumulator_type,
     ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
+    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
+  cudnnDataType_t cudnn_type = GetCudnnDataType<T>();
+  auto cudnn = cudnn_->GetHandle(parent_, stream);
+
+  CudnnTensorDescriptor out_back_nd(output_descriptor, cudnn_type);
+  CudnnTensorDescriptor input_nd(input_descriptor, cudnn_type);
+  CudnnFilterDescriptor filter(filter_descriptor, cudnn_type);
+  CudnnConvolutionDescriptor conv(convolution_descriptor,
+                                  ToCudnnDataType(accumulator_type));
+
+  SE_ASSIGN_OR_RETURN(
+      *algorithm_desc,
+      GetCudnnConvolutionBackwardFilterAlgorithm(
+          stream, cudnn, algorithm_config, input_nd, filter, conv, out_back_nd,
+          scratch_allocator, scratch_memory));
+
+  return port::Status::OK();
+}
+
+template <class T>
+port::Status CudnnSupport::DoConvolveBackwardFilterImpl(
+    Stream* stream, const dnn::BatchDescriptor& input_descriptor,
+    const DeviceMemory<T>& input_data,
+    const dnn::BatchDescriptor& output_descriptor,
+    DeviceMemory<T> backward_output_data,
+    const dnn::ConvolutionDescriptor& convolution_descriptor,
+    const dnn::FilterDescriptor& filter_descriptor,
+    DeviceMemory<T>* backward_filter_data, dnn::DataType accumulator_type,
+    const dnn::AlgorithmDesc& algorithm_desc,
+    DeviceMemory<uint8>* scratch_memory,
     dnn::ProfileResult* output_profile_result) {
   cudnnDataType_t cudnn_type = GetCudnnDataType<T>();
   // Alpha is the scaling factor for input.
@@ -3701,12 +3903,6 @@ port::Status CudnnSupport::DoConvolveBackwardFilterImpl(
 
   const bool is_profiling = output_profile_result != nullptr;
 
-  DeviceMemory<uint8> scratch;
-  SE_ASSIGN_OR_RETURN(dnn::AlgorithmDesc algo_desc,
-                      GetCudnnConvolutionBackwardFilterAlgorithm(
-                          stream, cudnn, algorithm_config, input_nd, filter,
-                          conv, out_back_nd, scratch_allocator, &scratch));
-
   std::unique_ptr<CUDATimer, TimerDeleter> timer;
   if (is_profiling) {
     timer.reset(new CUDATimer(parent_));  // NOLINT
@@ -3722,7 +3918,8 @@ port::Status CudnnSupport::DoConvolveBackwardFilterImpl(
   // results. See nvbugs/2072856
   if (CUDNN_VERSION < 7300) {
     SE_RETURN_IF_ERROR([&] {
-      if (algo_desc.algo_id() != CUDNN_CONVOLUTION_BWD_FILTER_ALGO_FFT_TILING) {
+      if (algorithm_desc.algo_id() !=
+          CUDNN_CONVOLUTION_BWD_FILTER_ALGO_FFT_TILING) {
         return port::Status::OK();
       }
       if (output_descriptor.height() > 1 && output_descriptor.width() > 1) {
@@ -3748,7 +3945,8 @@ port::Status CudnnSupport::DoConvolveBackwardFilterImpl(
     }());
   }
 
-  if (algo_desc.algo_id() == CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED &&
+  if (algorithm_desc.algo_id() ==
+          CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED &&
       !ShouldIncludeWinogradNonfusedAlgo(input_descriptor, output_descriptor)) {
     return port::Status(port::error::FAILED_PRECONDITION,
                         "This configuration has potential integer overflow in "
@@ -3764,7 +3962,7 @@ port::Status CudnnSupport::DoConvolveBackwardFilterImpl(
   //
   // See nvbugs/2379553.
   if (CUDNN_VERSION >= 7100 && CUDNN_VERSION < 7300 &&
-      algo_desc.algo_id() == CUDNN_CONVOLUTION_BWD_FILTER_ALGO_1 &&
+      algorithm_desc.algo_id() == CUDNN_CONVOLUTION_BWD_FILTER_ALGO_1 &&
       cudnn_type == CUDNN_DATA_HALF &&
       input_descriptor.layout() == dnn::DataLayout::kBatchYXDepth &&
       filter_descriptor.layout() == dnn::FilterLayout::kOutputYXInput &&
@@ -3782,9 +3980,9 @@ port::Status CudnnSupport::DoConvolveBackwardFilterImpl(
       /*diffDesc=*/out_back_nd.handle(),
       /*diffData=*/backward_output_data.opaque(),
       /*convDesc=*/conv.handle(),
-      /*algo=*/ToConvBackwardFilterAlgo(algo_desc),
-      /*workSpace=*/scratch.opaque(),
-      /*workSpaceSizeInBytes=*/scratch.size(),
+      /*algo=*/ToConvBackwardFilterAlgo(algorithm_desc),
+      /*workSpace=*/scratch_memory->opaque(),
+      /*workSpaceSizeInBytes=*/scratch_memory->size(),
       /*beta=*/beta,
       /*gradDesc=*/filter.handle(),
       /*dw=*/backward_filter_data->opaque()));
@@ -3792,14 +3990,14 @@ port::Status CudnnSupport::DoConvolveBackwardFilterImpl(
     if (!timer->Stop(AsCUDAStream(stream))) {
       return port::Status(port::error::INTERNAL, "Failed to stop timer");
     }
-    output_profile_result->set_algorithm(algo_desc);
+    output_profile_result->set_algorithm(algorithm_desc);
     output_profile_result->set_elapsed_time_in_ms(
         timer->GetElapsedMilliseconds());
-    output_profile_result->set_scratch_size(scratch.size());
+    output_profile_result->set_scratch_size(scratch_memory->size());
 
     LogCudaProto(GenerateConvProto<T>(
                      dnn::ConvolutionKind::BACKWARD_FILTER, input_descriptor,
-                     filter_descriptor, output_descriptor, algo_desc,
+                     filter_descriptor, output_descriptor, algorithm_desc,
                      convolution_descriptor, dalpha, dbeta, accumulator_type,
                      dnn::ActivationMode::kNone),
                  output_profile_result->elapsed_time_in_ms(), stream->parent());
@@ -3808,7 +4006,7 @@ port::Status CudnnSupport::DoConvolveBackwardFilterImpl(
   return port::Status::OK();
 }
 
-bool CudnnSupport::DoConvolveBackwardFilter(
+bool CudnnSupport::PrepareForConvolutionBackwardFilter(
     Stream* stream, const dnn::BatchDescriptor& input_descriptor,
     const DeviceMemory<double>& input_data,
     const dnn::BatchDescriptor& output_descriptor,
@@ -3818,18 +4016,17 @@ bool CudnnSupport::DoConvolveBackwardFilter(
     DeviceMemory<double>* backward_filter_data,
     ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
-    dnn::ProfileResult* output_profile_result) {
+    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
   return IsStatusOk(
-      DoConvolveBackwardFilterImpl(
+      PrepareForConvolutionBackwardFilterImpl(
           stream, input_descriptor, input_data, output_descriptor,
           backward_output_data, convolution_descriptor, filter_descriptor,
-          backward_filter_data, dnn::DataType::kDouble,
-
-          scratch_allocator, algorithm_config, output_profile_result),
-      /*report_error=*/!output_profile_result);
+          backward_filter_data, dnn::DataType::kDouble, scratch_allocator,
+          algorithm_config, algorithm_desc, scratch_memory),
+      /*report_error=*/true);
 }
 
-bool CudnnSupport::DoConvolveBackwardFilter(
+bool CudnnSupport::PrepareForConvolutionBackwardFilter(
     Stream* stream, const dnn::BatchDescriptor& input_descriptor,
     const DeviceMemory<float>& input_data,
     const dnn::BatchDescriptor& output_descriptor,
@@ -3839,18 +4036,17 @@ bool CudnnSupport::DoConvolveBackwardFilter(
     DeviceMemory<float>* backward_filter_data,
     ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
-    dnn::ProfileResult* output_profile_result) {
-  return IsStatusOk(DoConvolveBackwardFilterImpl(
-                        stream, input_descriptor, input_data, output_descriptor,
-                        backward_output_data, convolution_descriptor,
-                        filter_descriptor, backward_filter_data,
-
-                        dnn::DataType::kFloat, scratch_allocator,
-                        algorithm_config, output_profile_result),
-                    /*report_error=*/!output_profile_result);
+    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
+  return IsStatusOk(
+      PrepareForConvolutionBackwardFilterImpl(
+          stream, input_descriptor, input_data, output_descriptor,
+          backward_output_data, convolution_descriptor, filter_descriptor,
+          backward_filter_data, dnn::DataType::kFloat, scratch_allocator,
+          algorithm_config, algorithm_desc, scratch_memory),
+      /*report_error=*/true);
 }
 
-bool CudnnSupport::DoConvolveBackwardFilter(
+bool CudnnSupport::PrepareForConvolutionBackwardFilter(
     Stream* stream, const dnn::BatchDescriptor& input_descriptor,
     const DeviceMemory<Eigen::half>& input_data,
     const dnn::BatchDescriptor& output_descriptor,
@@ -3860,20 +4056,83 @@ bool CudnnSupport::DoConvolveBackwardFilter(
     DeviceMemory<Eigen::half>* backward_filter_data,
     ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
-    dnn::ProfileResult* output_profile_result) {
+    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
   dnn::DataType acc_type =
       CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()
           ? dnn::DataType::kFloat
           : dnn::DataType::kHalf;
   return IsStatusOk(
-      DoConvolveBackwardFilterImpl(
+      PrepareForConvolutionBackwardFilterImpl(
           stream, input_descriptor, input_data, output_descriptor,
           backward_output_data, convolution_descriptor, filter_descriptor,
           backward_filter_data, acc_type, scratch_allocator, algorithm_config,
-          output_profile_result),
+          algorithm_desc, scratch_memory),
+      /*report_error=*/true);
+}
+
+bool CudnnSupport::DoConvolveBackwardFilter(
+    Stream* stream, const dnn::BatchDescriptor& input_descriptor,
+    const DeviceMemory<double>& input_data,
+    const dnn::BatchDescriptor& output_descriptor,
+    DeviceMemory<double> backward_output_data,
+    const dnn::ConvolutionDescriptor& convolution_descriptor,
+    const dnn::FilterDescriptor& filter_descriptor,
+    DeviceMemory<double>* backward_filter_data,
+    const dnn::AlgorithmDesc& algorithm_desc,
+    DeviceMemory<uint8>* scratch_memory,
+    dnn::ProfileResult* output_profile_result) {
+  return IsStatusOk(
+      DoConvolveBackwardFilterImpl(
+          stream, input_descriptor, input_data, output_descriptor,
+          backward_output_data, convolution_descriptor, filter_descriptor,
+          backward_filter_data, dnn::DataType::kDouble, algorithm_desc,
+          scratch_memory, output_profile_result),
+      /*report_error=*/!output_profile_result);
+}
+
+bool CudnnSupport::DoConvolveBackwardFilter(
+    Stream* stream, const dnn::BatchDescriptor& input_descriptor,
+    const DeviceMemory<float>& input_data,
+    const dnn::BatchDescriptor& output_descriptor,
+    DeviceMemory<float> backward_output_data,
+    const dnn::ConvolutionDescriptor& convolution_descriptor,
+    const dnn::FilterDescriptor& filter_descriptor,
+    DeviceMemory<float>* backward_filter_data,
+    const dnn::AlgorithmDesc& algorithm_desc,
+    DeviceMemory<uint8>* scratch_memory,
+    dnn::ProfileResult* output_profile_result) {
+  return IsStatusOk(
+      DoConvolveBackwardFilterImpl(
+          stream, input_descriptor, input_data, output_descriptor,
+          backward_output_data, convolution_descriptor, filter_descriptor,
+          backward_filter_data, dnn::DataType::kFloat, algorithm_desc,
+          scratch_memory, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
+bool CudnnSupport::DoConvolveBackwardFilter(
+    Stream* stream, const dnn::BatchDescriptor& input_descriptor,
+    const DeviceMemory<Eigen::half>& input_data,
+    const dnn::BatchDescriptor& output_descriptor,
+    DeviceMemory<Eigen::half> backward_output_data,
+    const dnn::ConvolutionDescriptor& convolution_descriptor,
+    const dnn::FilterDescriptor& filter_descriptor,
+    DeviceMemory<Eigen::half>* backward_filter_data,
+    const dnn::AlgorithmDesc& algorithm_desc,
+    DeviceMemory<uint8>* scratch_memory,
+    dnn::ProfileResult* output_profile_result) {
+  dnn::DataType acc_type =
+      CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()
+          ? dnn::DataType::kFloat
+          : dnn::DataType::kHalf;
+  return IsStatusOk(DoConvolveBackwardFilterImpl(
+                        stream, input_descriptor, input_data, output_descriptor,
+                        backward_output_data, convolution_descriptor,
+                        filter_descriptor, backward_filter_data, acc_type,
+                        algorithm_desc, scratch_memory, output_profile_result),
+                    /*report_error=*/!output_profile_result);
+}
+
 template <class T>
 port::Status CudnnSupport::DoConvolveBackwardBiasImpl(
     Stream* stream, const dnn::BatchDescriptor& input_descriptor,
diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.h b/tensorflow/stream_executor/cuda/cuda_dnn.h
index 4cce3c5626..d7514981d5 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.h
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.h
@@ -258,6 +258,43 @@ class CudnnSupport : public dnn::DnnSupport {
       DeviceMemory<float>* scale_backprop,
       DeviceMemory<float>* offset_backprop) override;
 
+  bool PrepareForConvolution(
+      Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
+      const DeviceMemory<float>& input_data,
+      const dnn::FilterDescriptor& filter_descriptor,
+      const DeviceMemory<float>& filter_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<float>* output_data, ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory) override;
+
+  bool PrepareForConvolution(
+      Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
+      const DeviceMemory<double>& input_data,
+      const dnn::FilterDescriptor& filter_descriptor,
+      const DeviceMemory<double>& filter_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<double>* output_data, ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory) override;
+
+  bool PrepareForConvolution(
+      Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
+      const DeviceMemory<Eigen::half>& input_data,
+      const dnn::FilterDescriptor& filter_descriptor,
+      const DeviceMemory<Eigen::half>& filter_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<Eigen::half>* output_data,
+      ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory) override;
+
   bool DoConvolve(Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
                   const DeviceMemory<float>& input_data,
                   const dnn::FilterDescriptor& filter_descriptor,
@@ -265,8 +302,8 @@ class CudnnSupport : public dnn::DnnSupport {
                   const dnn::ConvolutionDescriptor& convolution_descriptor,
                   const dnn::BatchDescriptor& output_descriptor,
                   DeviceMemory<float>* output_data,
-                  ScratchAllocator* scratch_allocator,
-                  const dnn::AlgorithmConfig& algorithm_config,
+                  const dnn::AlgorithmDesc& algorithm_desc,
+                  DeviceMemory<uint8>* scratch_memory,
                   dnn::ProfileResult* output_profile_result) override;
 
   bool DoConvolve(Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
@@ -276,8 +313,8 @@ class CudnnSupport : public dnn::DnnSupport {
                   const dnn::ConvolutionDescriptor& convolution_descriptor,
                   const dnn::BatchDescriptor& output_descriptor,
                   DeviceMemory<double>* output_data,
-                  ScratchAllocator* scratch_allocator,
-                  const dnn::AlgorithmConfig& algorithm_config,
+                  const dnn::AlgorithmDesc& algorithm_desc,
+                  DeviceMemory<uint8>* scratch_memory,
                   dnn::ProfileResult* output_profile_result) override;
 
   bool DoConvolve(Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
@@ -287,8 +324,8 @@ class CudnnSupport : public dnn::DnnSupport {
                   const dnn::ConvolutionDescriptor& convolution_descriptor,
                   const dnn::BatchDescriptor& output_descriptor,
                   DeviceMemory<Eigen::half>* output_data,
-                  ScratchAllocator* scratch_allocator,
-                  const dnn::AlgorithmConfig& algorithm_config,
+                  const dnn::AlgorithmDesc& algorithm_desc,
+                  DeviceMemory<uint8>* scratch_memory,
                   dnn::ProfileResult* output_profile_result) override;
 
   bool DoFusedConvolve(
@@ -390,7 +427,20 @@ class CudnnSupport : public dnn::DnnSupport {
     return false;
   }
 
-  bool DoConvolveBackwardData(
+  bool PrepareForConvolutionBackwardData(
+      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
+      const DeviceMemory<float>& filter_data,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<float> backward_output_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::BatchDescriptor& input_descriptor,
+      DeviceMemory<float>* backward_input_data,
+      ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory) override;
+
+  bool PrepareForConvolutionBackwardData(
       Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
       const DeviceMemory<double>& filter_data,
       const dnn::BatchDescriptor& output_descriptor,
@@ -400,6 +450,32 @@ class CudnnSupport : public dnn::DnnSupport {
       DeviceMemory<double>* backward_input_data,
       ScratchAllocator* scratch_allocator,
       const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory) override;
+
+  bool PrepareForConvolutionBackwardData(
+      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
+      const DeviceMemory<Eigen::half>& filter_data,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<Eigen::half> backward_output_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::BatchDescriptor& input_descriptor,
+      DeviceMemory<Eigen::half>* backward_input_data,
+      ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory) override;
+
+  bool DoConvolveBackwardData(
+      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
+      const DeviceMemory<double>& filter_data,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<double> backward_output_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::BatchDescriptor& input_descriptor,
+      DeviceMemory<double>* backward_input_data,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
       dnn::ProfileResult* output_profile_result) override;
 
   bool DoConvolveBackwardData(
@@ -410,8 +486,8 @@ class CudnnSupport : public dnn::DnnSupport {
       const dnn::ConvolutionDescriptor& convolution_descriptor,
       const dnn::BatchDescriptor& input_descriptor,
       DeviceMemory<float>* backward_input_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
       dnn::ProfileResult* output_profile_result) override;
 
   bool DoConvolveBackwardData(
@@ -422,11 +498,11 @@ class CudnnSupport : public dnn::DnnSupport {
       const dnn::ConvolutionDescriptor& convolution_descriptor,
       const dnn::BatchDescriptor& input_descriptor,
       DeviceMemory<Eigen::half>* backward_input_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
       dnn::ProfileResult* output_profile_result) override;
 
-  bool DoConvolveBackwardFilter(
+  bool PrepareForConvolutionBackwardFilter(
       Stream* stream, const dnn::BatchDescriptor& input_descriptor,
       const DeviceMemory<double>& input_data,
       const dnn::BatchDescriptor& output_descriptor,
@@ -436,9 +512,10 @@ class CudnnSupport : public dnn::DnnSupport {
       DeviceMemory<double>* backward_filter_data,
       ScratchAllocator* scratch_allocator,
       const dnn::AlgorithmConfig& algorithm_config,
-      dnn::ProfileResult* output_profile_result) override;
+      dnn::AlgorithmDesc* algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory) override;
 
-  bool DoConvolveBackwardFilter(
+  bool PrepareForConvolutionBackwardFilter(
       Stream* stream, const dnn::BatchDescriptor& input_descriptor,
       const DeviceMemory<float>& input_data,
       const dnn::BatchDescriptor& output_descriptor,
@@ -448,9 +525,10 @@ class CudnnSupport : public dnn::DnnSupport {
       DeviceMemory<float>* backward_filter_data,
       ScratchAllocator* scratch_allocator,
       const dnn::AlgorithmConfig& algorithm_config,
-      dnn::ProfileResult* output_profile_result) override;
+      dnn::AlgorithmDesc* algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory) override;
 
-  bool DoConvolveBackwardFilter(
+  bool PrepareForConvolutionBackwardFilter(
       Stream* stream, const dnn::BatchDescriptor& input_descriptor,
       const DeviceMemory<Eigen::half>& input_data,
       const dnn::BatchDescriptor& output_descriptor,
@@ -460,6 +538,43 @@ class CudnnSupport : public dnn::DnnSupport {
       DeviceMemory<Eigen::half>* backward_filter_data,
       ScratchAllocator* scratch_allocator,
       const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory) override;
+
+  bool DoConvolveBackwardFilter(
+      Stream* stream, const dnn::BatchDescriptor& input_descriptor,
+      const DeviceMemory<double>& input_data,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<double> backward_output_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::FilterDescriptor& filter_descriptor,
+      DeviceMemory<double>* backward_filter_data,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
+      dnn::ProfileResult* output_profile_result) override;
+
+  bool DoConvolveBackwardFilter(
+      Stream* stream, const dnn::BatchDescriptor& input_descriptor,
+      const DeviceMemory<float>& input_data,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<float> backward_output_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::FilterDescriptor& filter_descriptor,
+      DeviceMemory<float>* backward_filter_data,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
+      dnn::ProfileResult* output_profile_result) override;
+
+  bool DoConvolveBackwardFilter(
+      Stream* stream, const dnn::BatchDescriptor& input_descriptor,
+      const DeviceMemory<Eigen::half>& input_data,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<Eigen::half> backward_output_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::FilterDescriptor& filter_descriptor,
+      DeviceMemory<Eigen::half>* backward_filter_data,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
       dnn::ProfileResult* output_profile_result) override;
 
   bool DoConvolveBackwardBias(
@@ -677,7 +792,7 @@ class CudnnSupport : public dnn::DnnSupport {
       DeviceMemory<U>* offset_backprop);
 
   template <class T>
-  port::Status DoConvolveImpl(
+  port::Status PrepareForConvolutionImpl(
       Stream* stream, const dnn::BatchDescriptor& input_descriptor,
       const DeviceMemory<T>& input_data,
       const dnn::FilterDescriptor& filter_descriptor,
@@ -687,6 +802,19 @@ class CudnnSupport : public dnn::DnnSupport {
       DeviceMemory<T>* output_data, dnn::DataType accumulator_type,
       ScratchAllocator* scratch_allocator,
       const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory);
+
+  template <class T>
+  port::Status DoConvolveImpl(
+      Stream* stream, const dnn::BatchDescriptor& input_descriptor,
+      const DeviceMemory<T>& input_data,
+      const dnn::FilterDescriptor& filter_descriptor,
+      const DeviceMemory<T>& filter_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<T>* output_data, dnn::DataType accumulator_type,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
       dnn::ProfileResult* output_profile_result);
 
   template <typename ElementType, typename BiasType, typename ScaleType>
@@ -707,7 +835,7 @@ class CudnnSupport : public dnn::DnnSupport {
       dnn::ProfileResult* output_profile_result);
 
   template <class T>
-  port::Status DoConvolveBackwardDataImpl(
+  port::Status PrepareForConvolutionBackwardDataImpl(
       Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
       const DeviceMemory<T>& filter_data,
       const dnn::BatchDescriptor& output_descriptor,
@@ -717,19 +845,45 @@ class CudnnSupport : public dnn::DnnSupport {
       DeviceMemory<T>* backward_input_data, dnn::DataType accumulator_type,
       ScratchAllocator* scratch_allocator,
       const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory);
+
+  template <class T>
+  port::Status DoConvolveBackwardDataImpl(
+      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
+      const DeviceMemory<T>& filter_data,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<T> backward_output_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::BatchDescriptor& input_descriptor,
+      DeviceMemory<T>* backward_input_data, dnn::DataType accumulator_type,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
       dnn::ProfileResult* output_profile_result);
 
   template <class T>
-  port::Status DoConvolveBackwardFilterImpl(
+  port::Status PrepareForConvolutionBackwardFilterImpl(
       Stream* stream, const dnn::BatchDescriptor& input_descriptor,
       const DeviceMemory<T>& input_data,
-      const dnn::BatchDescriptor& output_descriptor,
+      const dnn::BatchDescriptor& output_descriptor_in,
       DeviceMemory<T> backward_output_data,
       const dnn::ConvolutionDescriptor& convolution_descriptor,
       const dnn::FilterDescriptor& filter_descriptor,
       DeviceMemory<T>* backward_filter_data, dnn::DataType accumulator_type,
       ScratchAllocator* scratch_allocator,
       const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory);
+
+  template <class T>
+  port::Status DoConvolveBackwardFilterImpl(
+      Stream* stream, const dnn::BatchDescriptor& input_descriptor,
+      const DeviceMemory<T>& input_data,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<T> backward_output_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::FilterDescriptor& filter_descriptor,
+      DeviceMemory<T>* backward_filter_data, dnn::DataType accumulator_type,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
       dnn::ProfileResult* output_profile_result);
 
   template <class T>
diff --git a/tensorflow/stream_executor/dnn.h b/tensorflow/stream_executor/dnn.h
index ff7b02cf6b..f5a77d6525 100644
--- a/tensorflow/stream_executor/dnn.h
+++ b/tensorflow/stream_executor/dnn.h
@@ -738,6 +738,7 @@ class PoolingDescriptor {
 class AlgorithmDesc {
  public:
   typedef int64 Index;
+  AlgorithmDesc() : AlgorithmDesc(0, false) {}
   AlgorithmDesc(Index a, bool use_tensor_ops) {
     proto_.set_algo_id(a);
     proto_.set_math_type(use_tensor_ops ? AlgorithmProto::TENSOR_OP_MATH
@@ -1186,6 +1187,52 @@ class DnnSupport {
     return false;
   }
 
+  virtual bool PrepareForConvolution(
+      Stream* stream, const BatchDescriptor& batch_descriptor,
+      const DeviceMemory<float>& input_data,
+      const FilterDescriptor& filter_descriptor,
+      const DeviceMemory<float>& filter_data,
+      const ConvolutionDescriptor& convolution_descriptor,
+      const BatchDescriptor& output_descriptor,
+      DeviceMemory<float>* output_data, ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
+    *algorithm_desc = {};
+    *scratch_memory = {};
+    return true;
+  }
+
+  virtual bool PrepareForConvolution(
+      Stream* stream, const BatchDescriptor& batch_descriptor,
+      const DeviceMemory<double>& input_data,
+      const FilterDescriptor& filter_descriptor,
+      const DeviceMemory<double>& filter_data,
+      const ConvolutionDescriptor& convolution_descriptor,
+      const BatchDescriptor& output_descriptor,
+      DeviceMemory<double>* output_data, ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
+    *algorithm_desc = {};
+    *scratch_memory = {};
+    return true;
+  }
+
+  virtual bool PrepareForConvolution(
+      Stream* stream, const BatchDescriptor& batch_descriptor,
+      const DeviceMemory<Eigen::half>& input_data,
+      const FilterDescriptor& filter_descriptor,
+      const DeviceMemory<Eigen::half>& filter_data,
+      const ConvolutionDescriptor& convolution_descriptor,
+      const BatchDescriptor& output_descriptor,
+      DeviceMemory<Eigen::half>* output_data,
+      ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
+    *algorithm_desc = {};
+    *scratch_memory = {};
+    return true;
+  }
+
   // Enqueues a single-precision convolution operation onto the stream.
   //
   // Arguments (all borrowed):
@@ -1199,10 +1246,10 @@ class DnnSupport {
   //  output_descriptor: dimensions of the output layer.
   //  output_data: un-owned device memory region in which to place the
   //    convolution result.
-  //  scratch_allocator: un-owned, may-be-null object that may allocate scratch
-  //    space in order to speed up the convolution operation.
-  //  algorithm_config: specifies which algorithm should be used for the
+  //  algorithm_desc: specifies which algorithm should be used for the
   //    operation.
+  //  scratch: un-owned device memory for scratch space in order to speed up
+  //    the convolution operation.
   //  output_profile_result: the output profile result for this call. The
   //    profiling is only enabled when this is not nullptr.
   //
@@ -1227,8 +1274,9 @@ class DnnSupport {
       const DeviceMemory<float>& filter_data,
       const dnn::ConvolutionDescriptor& convolution_descriptor,
       const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<float>* output_data, ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
+      DeviceMemory<float>* output_data,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
       ProfileResult* output_profile_result) = 0;
 
   // Enqueues a double-precision convolution operation onto the stream.
@@ -1240,8 +1288,9 @@ class DnnSupport {
       const DeviceMemory<double>& filter_data,
       const dnn::ConvolutionDescriptor& convolution_descriptor,
       const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<double>* output_data, ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
+      DeviceMemory<double>* output_data,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
       dnn::ProfileResult* output_profile_result) = 0;
 
   // Enqueues a half-precision convolution operation onto the stream.
@@ -1254,8 +1303,8 @@ class DnnSupport {
       const dnn::ConvolutionDescriptor& convolution_descriptor,
       const dnn::BatchDescriptor& output_descriptor,
       DeviceMemory<Eigen::half>* output_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
       ProfileResult* output_profile_result) = 0;
 
   // Return a list of algorithms supported by the forward convolution pass.
@@ -1311,6 +1360,54 @@ class DnnSupport {
       const BatchDescriptor& output_descriptor,
       DeviceMemory<float>* output_data) = 0;
 
+  virtual bool PrepareForConvolutionBackwardData(
+      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
+      const DeviceMemory<float>& filter_data,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<float> backward_output_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::BatchDescriptor& input_descriptor,
+      DeviceMemory<float>* backward_input_data,
+      ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
+    *algorithm_desc = {};
+    *scratch_memory = {};
+    return true;
+  }
+
+  virtual bool PrepareForConvolutionBackwardData(
+      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
+      const DeviceMemory<double>& filter_data,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<double> backward_output_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::BatchDescriptor& input_descriptor,
+      DeviceMemory<double>* backward_input_data,
+      ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
+    *algorithm_desc = {};
+    *scratch_memory = {};
+    return true;
+  }
+
+  virtual bool PrepareForConvolutionBackwardData(
+      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
+      const DeviceMemory<Eigen::half>& filter_data,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<Eigen::half> backward_output_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::BatchDescriptor& input_descriptor,
+      DeviceMemory<Eigen::half>* backward_input_data,
+      ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
+    *algorithm_desc = {};
+    *scratch_memory = {};
+    return true;
+  }
+
   // Enqueues a single-precision backward convolution (for data) operation onto
   // the stream.
   //
@@ -1330,15 +1427,15 @@ class DnnSupport {
   //  scratch_allocator: un-owned, may-be-null object that may allocate scratch
   //    space in order to speed up the convolution operation.
   virtual bool DoConvolveBackwardData(
-      Stream* stream, const FilterDescriptor& filter_descriptor,
+      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
       const DeviceMemory<float>& filter_data,
-      const BatchDescriptor& output_descriptor,
+      const dnn::BatchDescriptor& output_descriptor,
       DeviceMemory<float> backward_output_data,
-      const ConvolutionDescriptor& convolution_descriptor,
-      const BatchDescriptor& input_descriptor,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::BatchDescriptor& input_descriptor,
       DeviceMemory<float>* backward_input_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
       ProfileResult* output_profile_result) = 0;
 
   // Return a list of algorithms supported by the backward convolution pass for
@@ -1348,28 +1445,76 @@ class DnnSupport {
       std::vector<AlgorithmDesc>* out_algorithms);
 
   virtual bool DoConvolveBackwardData(
-      Stream* stream, const FilterDescriptor& filter_descriptor,
+      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
       const DeviceMemory<double>& filter_data,
-      const BatchDescriptor& output_descriptor,
+      const dnn::BatchDescriptor& output_descriptor,
       DeviceMemory<double> backward_output_data,
-      const ConvolutionDescriptor& convolution_descriptor,
-      const BatchDescriptor& input_descriptor,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::BatchDescriptor& input_descriptor,
       DeviceMemory<double>* backward_input_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
       ProfileResult* output_profile_result) = 0;
 
   virtual bool DoConvolveBackwardData(
-      Stream* stream, const FilterDescriptor& filter_descriptor,
+      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
       const DeviceMemory<Eigen::half>& filter_data,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemory<Eigen::half> backward_output_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::BatchDescriptor& input_descriptor,
+      DeviceMemory<Eigen::half>* backward_input_data,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
+      ProfileResult* output_profile_result) = 0;
+
+  virtual bool PrepareForConvolutionBackwardFilter(
+      Stream* stream, const BatchDescriptor& input_descriptor,
+      const DeviceMemory<float>& input_data,
+      const BatchDescriptor& output_descriptor,
+      DeviceMemory<float> backward_output_data,
+      const ConvolutionDescriptor& convolution_descriptor,
+      const FilterDescriptor& filter_descriptor,
+      DeviceMemory<float>* backward_filter_data,
+      ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
+    *algorithm_desc = {};
+    *scratch_memory = {};
+    return true;
+  }
+
+  virtual bool PrepareForConvolutionBackwardFilter(
+      Stream* stream, const BatchDescriptor& input_descriptor,
+      const DeviceMemory<double>& input_data,
+      const BatchDescriptor& output_descriptor,
+      DeviceMemory<double> backward_output_data,
+      const ConvolutionDescriptor& convolution_descriptor,
+      const FilterDescriptor& filter_descriptor,
+      DeviceMemory<double>* backward_filter_data,
+      ScratchAllocator* scratch_allocator,
+      const dnn::AlgorithmConfig& algorithm_config,
+      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
+    *algorithm_desc = {};
+    *scratch_memory = {};
+    return true;
+  }
+
+  virtual bool PrepareForConvolutionBackwardFilter(
+      Stream* stream, const BatchDescriptor& input_descriptor,
+      const DeviceMemory<Eigen::half>& input_data,
       const BatchDescriptor& output_descriptor,
       DeviceMemory<Eigen::half> backward_output_data,
       const ConvolutionDescriptor& convolution_descriptor,
-      const BatchDescriptor& input_descriptor,
-      DeviceMemory<Eigen::half>* backward_input_data,
+      const FilterDescriptor& filter_descriptor,
+      DeviceMemory<Eigen::half>* backward_filter_data,
       ScratchAllocator* scratch_allocator,
       const dnn::AlgorithmConfig& algorithm_config,
-      ProfileResult* output_profile_result) = 0;
+      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
+    *algorithm_desc = {};
+    *scratch_memory = {};
+    return true;
+  }
 
   // Enqueues a single-precision backward convolution (for filter) operation
   // onto the stream.
@@ -1398,8 +1543,8 @@ class DnnSupport {
       const ConvolutionDescriptor& convolution_descriptor,
       const FilterDescriptor& filter_descriptor,
       DeviceMemory<float>* backward_filter_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
       ProfileResult* output_profile_result) = 0;
 
   // Return a list of algorithms supported by the backward convolution pass for
@@ -1416,8 +1561,8 @@ class DnnSupport {
       const ConvolutionDescriptor& convolution_descriptor,
       const FilterDescriptor& filter_descriptor,
       DeviceMemory<double>* backward_filter_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
       ProfileResult* output_profile_result) = 0;
 
   virtual bool DoConvolveBackwardFilter(
@@ -1428,8 +1573,8 @@ class DnnSupport {
       const ConvolutionDescriptor& convolution_descriptor,
       const FilterDescriptor& filter_descriptor,
       DeviceMemory<Eigen::half>* backward_filter_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
+      const dnn::AlgorithmDesc& algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory,
       ProfileResult* output_profile_result) = 0;
 
   // Enqueues a single-precision backward convolution (for bias) operation onto
diff --git a/tensorflow/stream_executor/stream.cc b/tensorflow/stream_executor/stream.cc
index 1b14e4c339..1befc18e19 100644
--- a/tensorflow/stream_executor/stream.cc
+++ b/tensorflow/stream_executor/stream.cc
@@ -549,11 +549,16 @@ Stream &Stream::ThenConvolveWithScratch(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      CheckError(dnn->DoConvolve(
+      DeviceMemory<uint8> scratch_memory;
+      dnn::AlgorithmDesc algorithm_desc;
+      CheckError(dnn->PrepareForConvolution(
           this, input_descriptor, input_data, filter_descriptor, filter_data,
           convolution_descriptor, output_descriptor, output, scratch_allocator,
-          dnn::AlgorithmConfig(),
-          /*output_profile_result=*/nullptr));
+          dnn::AlgorithmConfig(), &algorithm_desc, &scratch_memory));
+      CheckError(dnn->DoConvolve(
+          this, input_descriptor, input_data, filter_descriptor, filter_data,
+          convolution_descriptor, output_descriptor, output, algorithm_desc,
+          &scratch_memory, nullptr));
     } else {
       SetErrorAndLogNoDnnSupport();
     }
@@ -576,11 +581,16 @@ Stream &Stream::ThenConvolveWithScratch(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      CheckError(dnn->DoConvolve(
+      DeviceMemory<uint8> scratch_memory;
+      dnn::AlgorithmDesc algorithm_desc;
+      CheckError(dnn->PrepareForConvolution(
           this, input_descriptor, input_data, filter_descriptor, filter_data,
           convolution_descriptor, output_descriptor, output, scratch_allocator,
-          dnn::AlgorithmConfig(),
-          /*output_profile_result=*/nullptr));
+          dnn::AlgorithmConfig(), &algorithm_desc, &scratch_memory));
+      CheckError(dnn->DoConvolve(
+          this, input_descriptor, input_data, filter_descriptor, filter_data,
+          convolution_descriptor, output_descriptor, output, algorithm_desc,
+          &scratch_memory, nullptr));
     } else {
       SetErrorAndLogNoDnnSupport();
     }
@@ -758,10 +768,18 @@ Stream &Stream::ThenConvolveWithAlgorithm(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      auto status = dnn->DoConvolve(
+      DeviceMemory<uint8> scratch_memory;
+      dnn::AlgorithmDesc algorithm_desc;
+      auto status = dnn->PrepareForConvolution(
           this, input_descriptor, input_data, filter_descriptor, filter_data,
           convolution_descriptor, output_descriptor, output, scratch_allocator,
-          algorithm_config, output_profile_result);
+          algorithm_config, &algorithm_desc, &scratch_memory);
+      if (status) {
+        status = dnn->DoConvolve(
+            this, input_descriptor, input_data, filter_descriptor, filter_data,
+            convolution_descriptor, output_descriptor, output, algorithm_desc,
+            &scratch_memory, output_profile_result);
+      }
       if (!status && !output_profile_result) {
         SetError();
       }
@@ -789,10 +807,18 @@ Stream &Stream::ThenConvolveWithAlgorithm(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      auto status = dnn->DoConvolve(
+      DeviceMemory<uint8> scratch_memory;
+      dnn::AlgorithmDesc algorithm_desc;
+      auto status = dnn->PrepareForConvolution(
           this, input_descriptor, input_data, filter_descriptor, filter_data,
           convolution_descriptor, output_descriptor, output, scratch_allocator,
-          algorithm_config, output_profile_result);
+          algorithm_config, &algorithm_desc, &scratch_memory);
+      if (status) {
+        status = dnn->DoConvolve(
+            this, input_descriptor, input_data, filter_descriptor, filter_data,
+            convolution_descriptor, output_descriptor, output, algorithm_desc,
+            &scratch_memory, output_profile_result);
+      }
       if (!status && !output_profile_result) {
         SetError();
       }
@@ -820,10 +846,18 @@ Stream &Stream::ThenConvolveWithAlgorithm(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      auto status = dnn->DoConvolve(
+      DeviceMemory<uint8> scratch_memory;
+      dnn::AlgorithmDesc algorithm_desc;
+      auto status = dnn->PrepareForConvolution(
           this, input_descriptor, input_data, filter_descriptor, filter_data,
           convolution_descriptor, output_descriptor, output, scratch_allocator,
-          algorithm_config, output_profile_result);
+          algorithm_config, &algorithm_desc, &scratch_memory);
+      if (status) {
+        status = dnn->DoConvolve(
+            this, input_descriptor, input_data, filter_descriptor, filter_data,
+            convolution_descriptor, output_descriptor, output, algorithm_desc,
+            &scratch_memory, output_profile_result);
+      }
       if (!status && !output_profile_result) {
         SetError();
       }
@@ -969,10 +1003,17 @@ Stream &Stream::ThenConvolveBackwardDataWithScratch(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      CheckError(dnn->DoConvolveBackwardData(
+      DeviceMemory<uint8> scratch_memory;
+      dnn::AlgorithmDesc algorithm_desc;
+      CheckError(dnn->PrepareForConvolutionBackwardData(
           this, filter_descriptor, filter_data, output_descriptor,
           backward_output_data, convolution_descriptor, input_descriptor,
           backward_input_data, scratch_allocator, dnn::AlgorithmConfig(),
+          &algorithm_desc, &scratch_memory));
+      CheckError(dnn->DoConvolveBackwardData(
+          this, filter_descriptor, filter_data, output_descriptor,
+          backward_output_data, convolution_descriptor, input_descriptor,
+          backward_input_data, algorithm_desc, &scratch_memory,
           /*output_profile_result=*/nullptr));
     } else {
       SetErrorAndLogNoDnnSupport();
@@ -999,11 +1040,20 @@ Stream &Stream::ThenConvolveBackwardDataWithAlgorithm(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      auto status = dnn->DoConvolveBackwardData(
+      DeviceMemory<uint8> scratch_memory;
+      dnn::AlgorithmDesc algorithm_desc;
+      auto status = dnn->PrepareForConvolutionBackwardData(
           this, filter_descriptor, filter_data, output_descriptor,
           backward_output_data, convolution_descriptor, input_descriptor,
           backward_input_data, scratch_allocator, algorithm_config,
-          output_profile_result);
+          &algorithm_desc, &scratch_memory);
+      if (status) {
+        status = dnn->DoConvolveBackwardData(
+            this, filter_descriptor, filter_data, output_descriptor,
+            backward_output_data, convolution_descriptor, input_descriptor,
+            backward_input_data, algorithm_desc, &scratch_memory,
+            output_profile_result);
+      }
       if (!status && !output_profile_result) {
         SetError();
       }
@@ -1032,11 +1082,20 @@ Stream &Stream::ThenConvolveBackwardDataWithAlgorithm(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      auto status = dnn->DoConvolveBackwardData(
+      DeviceMemory<uint8> scratch_memory;
+      dnn::AlgorithmDesc algorithm_desc;
+      auto status = dnn->PrepareForConvolutionBackwardData(
           this, filter_descriptor, filter_data, output_descriptor,
           backward_output_data, convolution_descriptor, input_descriptor,
           backward_input_data, scratch_allocator, algorithm_config,
-          output_profile_result);
+          &algorithm_desc, &scratch_memory);
+      if (status) {
+        status = dnn->DoConvolveBackwardData(
+            this, filter_descriptor, filter_data, output_descriptor,
+            backward_output_data, convolution_descriptor, input_descriptor,
+            backward_input_data, algorithm_desc, &scratch_memory,
+            output_profile_result);
+      }
       if (!status && !output_profile_result) {
         SetError();
       }
@@ -1065,11 +1124,20 @@ Stream &Stream::ThenConvolveBackwardDataWithAlgorithm(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      auto status = dnn->DoConvolveBackwardData(
+      DeviceMemory<uint8> scratch_memory;
+      dnn::AlgorithmDesc algorithm_desc;
+      auto status = dnn->PrepareForConvolutionBackwardData(
           this, filter_descriptor, filter_data, output_descriptor,
           backward_output_data, convolution_descriptor, input_descriptor,
           backward_input_data, scratch_allocator, algorithm_config,
-          output_profile_result);
+          &algorithm_desc, &scratch_memory);
+      if (status) {
+        status = dnn->DoConvolveBackwardData(
+            this, filter_descriptor, filter_data, output_descriptor,
+            backward_output_data, convolution_descriptor, input_descriptor,
+            backward_input_data, algorithm_desc, &scratch_memory,
+            output_profile_result);
+      }
       if (!status && !output_profile_result) {
         SetError();
       }
@@ -1096,10 +1164,17 @@ Stream &Stream::ThenConvolveBackwardDataWithScratch(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      CheckError(dnn->DoConvolveBackwardData(
+      DeviceMemory<uint8> scratch_memory;
+      dnn::AlgorithmDesc algorithm_desc;
+      CheckError(dnn->PrepareForConvolutionBackwardData(
           this, filter_descriptor, filter_data, output_descriptor,
           backward_output_data, convolution_descriptor, input_descriptor,
           backward_input_data, scratch_allocator, dnn::AlgorithmConfig(),
+          &algorithm_desc, &scratch_memory));
+      CheckError(dnn->DoConvolveBackwardData(
+          this, filter_descriptor, filter_data, output_descriptor,
+          backward_output_data, convolution_descriptor, input_descriptor,
+          backward_input_data, algorithm_desc, &scratch_memory,
           /*output_profile_result=*/nullptr));
     } else {
       SetErrorAndLogNoDnnSupport();
@@ -1138,10 +1213,17 @@ Stream &Stream::ThenConvolveBackwardFilterWithScratch(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      CheckError(dnn->DoConvolveBackwardFilter(
+      DeviceMemory<uint8> scratch_memory;
+      dnn::AlgorithmDesc algorithm_desc;
+      CheckError(dnn->PrepareForConvolutionBackwardFilter(
           this, input_descriptor, input_data, output_descriptor,
           backward_output_data, convolution_descriptor, filter_descriptor,
           backward_filter_data, scratch_allocator, dnn::AlgorithmConfig(),
+          &algorithm_desc, &scratch_memory));
+      CheckError(dnn->DoConvolveBackwardFilter(
+          this, input_descriptor, input_data, output_descriptor,
+          backward_output_data, convolution_descriptor, filter_descriptor,
+          backward_filter_data, algorithm_desc, &scratch_memory,
           /*output_profile_result=*/nullptr));
     } else {
       SetErrorAndLogNoDnnSupport();
@@ -1168,11 +1250,20 @@ Stream &Stream::ThenConvolveBackwardFilterWithAlgorithm(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      auto status = dnn->DoConvolveBackwardFilter(
+      DeviceMemory<uint8> scratch_memory;
+      dnn::AlgorithmDesc algorithm_desc;
+      auto status = dnn->PrepareForConvolutionBackwardFilter(
           this, input_descriptor, input_data, output_descriptor,
           backward_output_data, convolution_descriptor, filter_descriptor,
           backward_filter_data, scratch_allocator, algorithm_config,
-          output_profile_result);
+          &algorithm_desc, &scratch_memory);
+      if (status) {
+        status = dnn->DoConvolveBackwardFilter(
+            this, input_descriptor, input_data, output_descriptor,
+            backward_output_data, convolution_descriptor, filter_descriptor,
+            backward_filter_data, algorithm_desc, &scratch_memory,
+            output_profile_result);
+      }
       if (!status && !output_profile_result) {
         SetError();
       }
@@ -1201,11 +1292,20 @@ Stream &Stream::ThenConvolveBackwardFilterWithAlgorithm(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      auto status = dnn->DoConvolveBackwardFilter(
+      DeviceMemory<uint8> scratch_memory;
+      dnn::AlgorithmDesc algorithm_desc;
+      auto status = dnn->PrepareForConvolutionBackwardFilter(
           this, input_descriptor, input_data, output_descriptor,
           backward_output_data, convolution_descriptor, filter_descriptor,
           backward_filter_data, scratch_allocator, algorithm_config,
-          output_profile_result);
+          &algorithm_desc, &scratch_memory);
+      if (status) {
+        status = dnn->DoConvolveBackwardFilter(
+            this, input_descriptor, input_data, output_descriptor,
+            backward_output_data, convolution_descriptor, filter_descriptor,
+            backward_filter_data, algorithm_desc, &scratch_memory,
+            output_profile_result);
+      }
       if (!status && !output_profile_result) {
         SetError();
       }
@@ -1232,10 +1332,17 @@ Stream &Stream::ThenConvolveBackwardFilterWithScratch(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      CheckError(dnn->DoConvolveBackwardFilter(
+      DeviceMemory<uint8> scratch_memory;
+      dnn::AlgorithmDesc algorithm_desc;
+      CheckError(dnn->PrepareForConvolutionBackwardFilter(
           this, input_descriptor, input_data, output_descriptor,
           backward_output_data, convolution_descriptor, filter_descriptor,
           backward_filter_data, scratch_allocator, dnn::AlgorithmConfig(),
+          &algorithm_desc, &scratch_memory));
+      CheckError(dnn->DoConvolveBackwardFilter(
+          this, input_descriptor, input_data, output_descriptor,
+          backward_output_data, convolution_descriptor, filter_descriptor,
+          backward_filter_data, algorithm_desc, &scratch_memory,
           /*output_profile_result=*/nullptr));
     } else {
       SetErrorAndLogNoDnnSupport();
@@ -1262,11 +1369,20 @@ Stream &Stream::ThenConvolveBackwardFilterWithAlgorithm(
 
   if (ok()) {
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      auto status = dnn->DoConvolveBackwardFilter(
+      DeviceMemory<uint8> scratch_memory;
+      dnn::AlgorithmDesc algorithm_desc;
+      auto status = dnn->PrepareForConvolutionBackwardFilter(
           this, input_descriptor, input_data, output_descriptor,
           backward_output_data, convolution_descriptor, filter_descriptor,
           backward_filter_data, scratch_allocator, algorithm_config,
-          output_profile_result);
+          &algorithm_desc, &scratch_memory);
+      if (status) {
+        status = dnn->DoConvolveBackwardFilter(
+            this, input_descriptor, input_data, output_descriptor,
+            backward_output_data, convolution_descriptor, filter_descriptor,
+            backward_filter_data, algorithm_desc, &scratch_memory,
+            output_profile_result);
+      }
       if (!status && !output_profile_result) {
         SetError();
       }
-- 
GitLab


From 04266f6c791113ec9e0098e52579a4c53e9acc0b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 9 Jan 2019 12:55:40 -0800
Subject: [PATCH 0473/1095] Fix off-by-one error in time calculation for eta
 ecm3531 platform.

PiperOrigin-RevId: 228571040
---
 .../lite/experimental/micro/tools/make/targets/ecm3531/_main.c  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/experimental/micro/tools/make/targets/ecm3531/_main.c b/tensorflow/lite/experimental/micro/tools/make/targets/ecm3531/_main.c
index 93941e119f..2764f3ba50 100644
--- a/tensorflow/lite/experimental/micro/tools/make/targets/ecm3531/_main.c
+++ b/tensorflow/lite/experimental/micro/tools/make/targets/ecm3531/_main.c
@@ -85,7 +85,7 @@ void EtaPrintExecutionTime(uint64_t time_ms) {
       time_ms = time_ms / 10;
       time_string[k1] = (char)(0x30 + c);
     }
-    for (k1 = 4; k1 > 0; k1--) {  // print out 1 char at a time
+    for (k1 = 4; k1 >= 0; k1--) {  // print out 1 char at a time
       EtaCspUartPutc(&g_sUart1, time_string[k1]);
     }
   } else {
-- 
GitLab


From 998a22ab58d8e5faa1f041afea1e322797d70329 Mon Sep 17 00:00:00 2001
From: Shivani Agrawal <shivaniagrawal@google.com>
Date: Wed, 9 Jan 2019 13:05:00 -0800
Subject: [PATCH 0474/1095] [tf.data] document update to correct the
 non-working example.

PiperOrigin-RevId: 228572720
---
 tensorflow/python/data/experimental/ops/stats_aggregator.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/data/experimental/ops/stats_aggregator.py b/tensorflow/python/data/experimental/ops/stats_aggregator.py
index d5fcc033ab..3e4c66be27 100644
--- a/tensorflow/python/data/experimental/ops/stats_aggregator.py
+++ b/tensorflow/python/data/experimental/ops/stats_aggregator.py
@@ -45,7 +45,7 @@ class StatsAggregator(object):
 
   # Apply `StatsOptions` to associate `dataset` with `aggregator`.
   options = dataset_ops.Options()
-  options.experimental_stats = tf.data.experimental.StatsOptions(aggregator)
+  options.experimental_stats.aggregator = aggregator
   dataset = dataset.with_options(options)
   ```
 
-- 
GitLab


From 02747c96978bc76427cf45d32ecb24a866520043 Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Wed, 9 Jan 2019 13:06:08 -0800
Subject: [PATCH 0475/1095] Fix tensor_utils_test

Update another test target relying on lite/kernels:test_util, which
pulls in TF and requires the tf_cc_test build rule.

A follow-up CL will remove this requirement for this and other kernel
tests.

PiperOrigin-RevId: 228572896
---
 tensorflow/lite/kernels/internal/BUILD | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/kernels/internal/BUILD b/tensorflow/lite/kernels/internal/BUILD
index 74a11a72ef..b734b2d6cc 100644
--- a/tensorflow/lite/kernels/internal/BUILD
+++ b/tensorflow/lite/kernels/internal/BUILD
@@ -1,3 +1,4 @@
+load("//tensorflow:tensorflow.bzl", "tf_cc_test")
 load("//tensorflow/lite:build_def.bzl", "tflite_copts")
 load("//tensorflow/lite:special_rules.bzl", "tflite_portable_test_suite")
 
@@ -555,10 +556,11 @@ cc_library(
     ],
 )
 
-cc_test(
+# TODO(b/122597976): Eliminate TF dependency from lite/kernels:test_util,
+# in turn eliminating the need to use tf_cc_test for any dependent tests.
+tf_cc_test(
     name = "tensor_utils_test",
     srcs = ["tensor_utils_test.cc"],
-    copts = NEON_FLAGS_IF_APPLICABLE,
     linkopts = select({
         "//tensorflow:android": [
             "-fPIE -pie",
-- 
GitLab


From 4495df6cb19802fabaae9999a3b3fdcfe88b268d Mon Sep 17 00:00:00 2001
From: Shashi Shekhar <shashishekhar@google.com>
Date: Wed, 9 Jan 2019 13:32:55 -0800
Subject: [PATCH 0476/1095] Update types in accuracy tool.

PiperOrigin-RevId: 228577610
---
 tensorflow/lite/tools/accuracy/utils.cc | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tensorflow/lite/tools/accuracy/utils.cc b/tensorflow/lite/tools/accuracy/utils.cc
index c19dc1ff7c..953892b8dd 100644
--- a/tensorflow/lite/tools/accuracy/utils.cc
+++ b/tensorflow/lite/tools/accuracy/utils.cc
@@ -38,6 +38,12 @@ DataType GetTFDataType(TfLiteType tflite_type) {
       return DT_FLOAT;
     case kTfLiteUInt8:
       return DT_UINT8;
+    case kTfLiteInt8:
+      return DT_INT8;
+    case kTfLiteInt32:
+      return DT_INT32;
+    case kTfLiteInt64:
+      return DT_INT64;
     default:
       return DT_INVALID;
   }
-- 
GitLab


From 043e72fa2cab247290e0015682e8bc818bee5a15 Mon Sep 17 00:00:00 2001
From: Grzegorz Pawelczak <grzegorzp@graphcore.ai>
Date: Wed, 9 Jan 2019 21:41:09 +0000
Subject: [PATCH 0477/1095] Fix copyright year and run clang format

---
 tensorflow/compiler/xla/service/sort_simplifier.cc      | 6 ++----
 tensorflow/compiler/xla/service/sort_simplifier_test.cc | 2 +-
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/tensorflow/compiler/xla/service/sort_simplifier.cc b/tensorflow/compiler/xla/service/sort_simplifier.cc
index ee91762253..6136dc278c 100644
--- a/tensorflow/compiler/xla/service/sort_simplifier.cc
+++ b/tensorflow/compiler/xla/service/sort_simplifier.cc
@@ -92,8 +92,7 @@ StatusOr<bool> RemoveUnusedOperandFromSort(HloInstruction* sort) {
   }
   return true;
 }
-
-} // namespace
+}  // namespace
 
 StatusOr<bool> SortSimplifier::Run(HloModule* module) {
   VLOG(2) << "HLO module before SortSimplifier:";
@@ -109,8 +108,7 @@ StatusOr<bool> SortSimplifier::Run(HloModule* module) {
   }
 
   for (HloInstruction* sort_instr : sort_instrs) {
-    TF_ASSIGN_OR_RETURN(bool result,
-                        RemoveUnusedOperandFromSort(sort_instr));
+    TF_ASSIGN_OR_RETURN(bool result, RemoveUnusedOperandFromSort(sort_instr));
     changed |= result;
   }
 
diff --git a/tensorflow/compiler/xla/service/sort_simplifier_test.cc b/tensorflow/compiler/xla/service/sort_simplifier_test.cc
index 5a81e537de..4df833bbeb 100644
--- a/tensorflow/compiler/xla/service/sort_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/sort_simplifier_test.cc
@@ -1,4 +1,4 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
-- 
GitLab


From 0dc03e940edcab6e6680990557634b335d3be4e9 Mon Sep 17 00:00:00 2001
From: Pete Warden <petewarden@google.com>
Date: Wed, 9 Jan 2019 13:45:56 -0800
Subject: [PATCH 0478/1095] Fixed target name in documentation

PiperOrigin-RevId: 228580269
---
 tensorflow/lite/experimental/micro/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/experimental/micro/README.md b/tensorflow/lite/experimental/micro/README.md
index 173f91fe90..931c8d9d31 100644
--- a/tensorflow/lite/experimental/micro/README.md
+++ b/tensorflow/lite/experimental/micro/README.md
@@ -381,7 +381,7 @@ auto-generated for any target you can compile using the main Make system, using
 a command like this (making sure you've run `download_dependencies.sh` first):
 
 ```
-make -f tensorflow/lite/experimental/micro/tools/make/Makefile TARGET=mbed TAGS="CMSIS disco_f746ng" generate_micro_speech_main_test_mbed_project
+make -f tensorflow/lite/experimental/micro/tools/make/Makefile TARGET=mbed TAGS="CMSIS disco_f746ng" generate_micro_speech_mbed_project
 ```
 
 This will create a folder in
-- 
GitLab


From 9545dccbd749a626c5451421b090ed1c70125d6a Mon Sep 17 00:00:00 2001
From: Russell Power <power@google.com>
Date: Wed, 9 Jan 2019 13:48:27 -0800
Subject: [PATCH 0479/1095] graph_partition: Fix absl::Hash related issue in
 Windows build. PiperOrigin-RevId: 228580775

---
 tensorflow/core/graph/graph_partition.cc | 35 ++++++++++++++++--------
 1 file changed, 24 insertions(+), 11 deletions(-)

diff --git a/tensorflow/core/graph/graph_partition.cc b/tensorflow/core/graph/graph_partition.cc
index be0cac50a1..00c7a5b091 100644
--- a/tensorflow/core/graph/graph_partition.cc
+++ b/tensorflow/core/graph/graph_partition.cc
@@ -67,16 +67,14 @@ struct DupRecvKey {
                       c.recv_output_on_host);
   }
 
-  friend bool operator==(const DupRecvKey& x, const DupRecvKey& y);
+  friend bool operator==(const DupRecvKey& x, const DupRecvKey& y) {
+    return (x.src_node_id == y.src_node_id) &&
+           (x.src_output_slot == y.src_output_slot) &&
+           (x.dst_graph == y.dst_graph) &&
+           (x.recv_output_on_host == y.recv_output_on_host);
+  }
 };
 
-bool operator==(const DupRecvKey& x, const DupRecvKey& y) {
-  return (x.src_node_id == y.src_node_id) &&
-         (x.src_output_slot == y.src_output_slot) &&
-         (x.dst_graph == y.dst_graph) &&
-         (x.recv_output_on_host == y.recv_output_on_host);
-}
-
 // struct used to store the recvs, so that start times can be properly updated
 struct RecvInfo {
   NodeDef* recv;
@@ -88,7 +86,22 @@ typedef absl::flat_hash_map<DupRecvKey, RecvInfo> DupRecvTable;
 
 // A map used to store memory types for the inputs/outputs of every node.
 // The key is a pair of ints consisting of a node id and input/output index.
-typedef absl::flat_hash_map<std::pair<int, int>, MemoryType> MemoryTypeMap;
+// TODO(power): migrate back to std::pair when absl::Hash is fixed for MSVC.
+struct NodePort {
+  int node_id;
+  int index;
+
+  friend bool operator==(const NodePort& x, const NodePort& y) {
+    return x.node_id == y.node_id && x.index == y.index;
+  }
+
+  template <typename H>
+  friend H AbslHashValue(H h, const NodePort& c) {
+    return H::combine(std::move(h), c.node_id, c.index);
+  }
+};
+
+typedef absl::flat_hash_map<NodePort, MemoryType> MemoryTypeMap;
 
 // We collect the following information about the graph before performing
 // graph partitioning.
@@ -552,10 +565,10 @@ Status BuildMemoryDeviceInfo(const Graph& g, GraphInfo* info) {
 
     int node_id = node->id();
     info->device_types[node_id] = DeviceType(parsed.type);
-    for (size_t i = 0; i < input_memory_types.size(); ++i) {
+    for (int i = 0; i < input_memory_types.size(); ++i) {
       info->input_types[{node_id, i}] = input_memory_types[i];
     }
-    for (size_t i = 0; i < output_memory_types.size(); ++i) {
+    for (int i = 0; i < output_memory_types.size(); ++i) {
       info->output_types[{node_id, i}] = output_memory_types[i];
     }
   }
-- 
GitLab


From 2b2c042367675fdac67a12bc9bae9689babae07d Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Wed, 9 Jan 2019 13:51:30 -0800
Subject: [PATCH 0480/1095] [Grappler] Add identity nodes to read function
 outputs in GrapplerFunctionItem instantiations

PiperOrigin-RevId: 228581341
---
 .../core/grappler/costs/graph_properties.cc   |   4 +-
 .../optimizers/constant_folding_test.cc       |   2 +-
 .../grappler/optimizers/function_optimizer.cc | 129 +++++++++---
 .../optimizers/function_optimizer_test.cc     |   2 +-
 .../optimizers/meta_optimizer_test.cc         | 111 ++++++++--
 tensorflow/core/grappler/utils/BUILD          |   1 +
 tensorflow/core/grappler/utils/functions.cc   | 196 ++++++++++++------
 tensorflow/core/grappler/utils/functions.h    |  83 ++++----
 .../core/grappler/utils/functions_test.cc     | 105 +++++++---
 .../core/grappler/utils/grappler_test.cc      |  74 ++++---
 .../core/grappler/utils/grappler_test.h       |   7 +
 11 files changed, 513 insertions(+), 201 deletions(-)

diff --git a/tensorflow/core/grappler/costs/graph_properties.cc b/tensorflow/core/grappler/costs/graph_properties.cc
index 1ed96ca12a..863a5087ae 100644
--- a/tensorflow/core/grappler/costs/graph_properties.cc
+++ b/tensorflow/core/grappler/costs/graph_properties.cc
@@ -669,7 +669,7 @@ class SymbolicShapeRefiner {
     ctx->output_tensor_protos.resize(grappler_function_item.output_size(),
                                      nullptr);
     for (auto const& out_arg : grappler_function_item.outputs()) {
-      if (out_arg.output_tensors.size() > 1) {
+      if (out_arg.output_nodes.size() > 1) {
         // TODO(jmdecker): Handle case of multiple output tensors
         return errors::Unimplemented(
             "Output arguments with multiple output tensors are not yet "
@@ -678,7 +678,7 @@ class SymbolicShapeRefiner {
 
       // It is guaranteed that output_tensors does not contain any control
       // inputs, so port_id >= 0.
-      TensorId out_tensor = ParseTensorName(out_arg.output_tensors[0]);
+      TensorId out_tensor = ParseTensorName(out_arg.output_nodes[0]);
 
       const NodeDef* retnode = gv.GetNode(out_tensor.node());
       if (retnode == nullptr) {
diff --git a/tensorflow/core/grappler/optimizers/constant_folding_test.cc b/tensorflow/core/grappler/optimizers/constant_folding_test.cc
index 192f48272f..d7cabf5a8b 100644
--- a/tensorflow/core/grappler/optimizers/constant_folding_test.cc
+++ b/tensorflow/core/grappler/optimizers/constant_folding_test.cc
@@ -1601,7 +1601,7 @@ TEST_F(ConstantFoldingTest, SplitRemoval) {
   AddNode("split_dim", "Const", {}, {}, &want);
   AddNode("s1", "Identity", {"in1", AsControlDependency("split_dim")}, {},
           &want);
-  AddNode("s2", "Split", {"in2", "split_dim"}, {}, &want);
+  AddNode("s2", "Split", {"split_dim", "in2"}, {}, &want);
   AddNode("out", "Add", {"s1", "s2"}, {}, &want);
 
   CompareGraphs(want, got);
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.cc b/tensorflow/core/grappler/optimizers/function_optimizer.cc
index d074676c3d..e9c30fee25 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer.cc
@@ -806,8 +806,17 @@ Status SpecializeFunction(const NodeDef& func_node, const FunctionDef& func,
   // update outputs for the fetch nodes, so we just skip them.
   std::vector<std::pair<int, int>> output_mapping;
   if (!signature.is_in_fetch_set) {
-    TF_RETURN_IF_ERROR(
-        RemoveUnusedOutputs(signature.active_outputs, &item, &output_mapping));
+    int num_func_outputs = 0;
+    for (const auto& out_arg : item.outputs()) {
+      num_func_outputs += out_arg.output_nodes.size();
+    }
+
+    absl::flat_hash_set<int> remove;
+    for (int i = 0; i < num_func_outputs; ++i) {
+      if (!signature.active_outputs.count(i)) remove.insert(i);
+    }
+
+    TF_RETURN_IF_ERROR(RemoveFunctionOutputs(remove, &item, &output_mapping));
   }
 
   // TODO(ezhulenev): Push down known input shapes.
@@ -962,8 +971,10 @@ NodeDef InlinedFunctionInputsNode(const NodeDef& func_node,
 
 // Create an IdentityN node to hook the function outputs to: this ensures that
 // the function body is fully evaluated before its fanout gets scheduled.
-NodeDef InlinedFunctionOutputsNode(const NodeDef& func_node,
-                                   const GrapplerFunctionItem& item) {
+NodeDef InlinedFunctionOutputsNode(
+    const NodeDef& func_node, const GrapplerFunctionItem& item,
+    const absl::flat_hash_map<absl::string_view, absl::string_view>
+        output_tensors) {
   NodeDef outputs;
   outputs.set_name(func_node.name());
   outputs.set_op("IdentityN");
@@ -972,7 +983,8 @@ NodeDef InlinedFunctionOutputsNode(const NodeDef& func_node,
       (*outputs.mutable_attr())["T"].mutable_list();
 
   for (const OutputArgExpansion& output_arg : item.outputs()) {
-    for (const string& output_tensor : output_arg.output_tensors) {
+    for (const string& output_node : output_arg.output_nodes) {
+      const absl::string_view output_tensor = output_tensors.at(output_node);
       type_list->add_type(output_arg.data_type);
       outputs.add_input(strings::StrCat(func_node.name(), "/", output_tensor));
     }
@@ -1004,29 +1016,51 @@ Status InlineDirectFunctionCall(const NodeDef& func_node,
   }
 
   // Mapping from input placeholder name to function input position.
-  int idx = 0;
-  absl::flat_hash_map<string, int> input_placeholders_idx;
+  absl::flat_hash_map<absl::string_view, int> input_placeholders_idx;
   for (const InputArgExpansion& input_arg : item.inputs()) {
     for (const string& placeholder : input_arg.placeholders) {
-      input_placeholders_idx[placeholder] = idx++;
+      const int idx = input_placeholders_idx.size();
+      input_placeholders_idx[placeholder] = idx;
+    }
+  }
+
+  // Bypass identity nodes added to the graph in place of function outputs.
+  absl::flat_hash_set<absl::string_view> output_nodes;
+  for (const OutputArgExpansion& output_arg : item.outputs()) {
+    for (const string& output_node : output_arg.output_nodes) {
+      output_nodes.insert(output_node);
     }
   }
 
+  // For each function output value we added an identity node that reads the
+  // tensor from one of the function body nodes. When we inline function into
+  // the main graph we want to bypass these nodes, so we keep a mapping from
+  // 'output node name' -> 'output tensor name'.
+  absl::flat_hash_map<absl::string_view, absl::string_view> output_tensors;
+
   // Hook inlined function inputs to IdentityN node.
   NodeDef* func_inputs = optimized_graph->add_node();
   *func_inputs = InlinedFunctionInputsNode(func_node, item);
 
   for (NodeDef& func_body_node : *item.mutable_function_body().mutable_node()) {
-    if (item.IsInputPlaceholder(func_body_node.name())) {
-      // Turn input placeholders into identity nodes.
+    const string& node_name = func_body_node.name();
+
+    // Skip output identity node, and update a mapping to the output tensor.
+    if (IsIdentity(func_body_node) && output_nodes.count(node_name)) {
+      output_tensors.emplace(node_name, func_body_node.input(0));
+      continue;
+    }
+
+    // Turn placeholders added in place of input arguments into identity nodes.
+    const auto input_placeholder_idx = input_placeholders_idx.find(node_name);
+    if (input_placeholder_idx != input_placeholders_idx.end()) {
       CHECK_EQ(0, func_body_node.input_size());
       func_body_node.set_op("Identity");
       (*func_body_node.mutable_attr())["T"] = func_body_node.attr().at("dtype");
       func_body_node.mutable_attr()->erase("dtype");
       func_body_node.mutable_attr()->erase("shape");
-      int input_idx = input_placeholders_idx[func_body_node.name()];
-      func_body_node.add_input(
-          strings::StrCat(func_inputs->name(), ":", input_idx));
+      func_body_node.add_input(strings::StrCat(func_inputs->name(), ":",
+                                               input_placeholder_idx->second));
     } else {
       // Update the input names if any.
       for (string& input : *func_body_node.mutable_input()) {
@@ -1082,9 +1116,12 @@ Status InlineDirectFunctionCall(const NodeDef& func_node,
     }
   }
 
+  DCHECK(output_tensors.size() == item.output_size())
+      << "Each function output must be mapped to an output tensor";
+
   // Hook inlined function outputs to IdentityN node.
   NodeDef* func_outputs = optimized_graph->add_node();
-  *func_outputs = InlinedFunctionOutputsNode(func_node, item);
+  *func_outputs = InlinedFunctionOutputsNode(func_node, item, output_tensors);
 
   return Status::OK();
 }
@@ -1363,11 +1400,11 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
   }
 
   // Mapping from input placeholder name to function input position.
-  int idx = 0;
   absl::flat_hash_map<absl::string_view, int> input_placeholders_idx;
   for (const InputArgExpansion& input_arg : item.inputs()) {
     for (const string& placeholder : input_arg.placeholders) {
-      input_placeholders_idx[placeholder] = idx++;
+      const int idx = input_placeholders_idx.size();
+      input_placeholders_idx[placeholder] = idx;
     }
   }
 
@@ -1378,8 +1415,11 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
   // same device as their corresponding input nodes.
 
   for (NodeDef& func_body_node : *item.graph.mutable_node()) {
-    if (item.IsInputPlaceholder(func_body_node.name())) {
-      const int input_idx = input_placeholders_idx[func_body_node.name()];
+    const auto input_placeholder_idx =
+        input_placeholders_idx.find(func_body_node.name());
+
+    if (input_placeholder_idx != input_placeholders_idx.end()) {
+      const int input_idx = input_placeholder_idx->second;
       const GraphView::OutputPort output_port =
           ctx->graph_view().GetRegularFanin({&func_node, input_idx});
 
@@ -1441,16 +1481,23 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
   // optimized graph: turn placeholders into identities, update nodes
   // connectivity, etc...
 
+  const auto inlined_node_name = [&func_node](const string& name) -> string {
+    return AddPrefixToNodeName(name, /*prefix=*/func_node.name());
+  };
+
   for (NodeDef& func_body_node : *placed_graph_def.mutable_node()) {
-    if (item.IsInputPlaceholder(func_body_node.name())) {
-      // Turn input placeholders into identity node.
+    const string& node_name = func_body_node.name();
+
+    // Turn placeholders added in place of input arguments into identity nodes.
+    const auto input_placeholder_idx = input_placeholders_idx.find(node_name);
+    if (input_placeholder_idx != input_placeholders_idx.end()) {
       DCHECK_EQ(0, func_body_node.input_size());
       func_body_node.set_op("Identity");
       (*func_body_node.mutable_attr())["T"] = func_body_node.attr().at("dtype");
       func_body_node.mutable_attr()->erase("dtype");
       func_body_node.mutable_attr()->erase("shape");
-      const int input_idx = input_placeholders_idx[func_body_node.name()];
-      func_body_node.add_input(strings::StrCat(inputs[input_idx].ToString()));
+      const int input_idx = input_placeholder_idx->second;
+      func_body_node.add_input(inputs[input_idx].ToString());
 
       // All side effects must happen before inputs can start executing.
       for (const string& hb_node : happens_before) {
@@ -1460,7 +1507,7 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
     } else {
       // Update inputs of the regular function body nodes.
       for (string& input : *func_body_node.mutable_input()) {
-        input = AddPrefixToNodeName(input, /*prefix=*/func_node.name());
+        input = inlined_node_name(input);
       }
       if (func_body_node.input_size() == 0 && !empty_inputs_hook.empty()) {
         *func_body_node.add_input() = empty_inputs_hook[0];
@@ -1494,7 +1541,7 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
   for (NodeDef& func_body_node : *placed_graph_def.mutable_node()) {
     if (!IsFreeOfSideEffect(func_body_node, &ctx->function_library())) {
       int num_fanouts = placed_graph_view.NumFanouts(
-          func_body_node, /*include_controlling_nodes=*/true);
+          func_body_node, /*include_controlled_nodes=*/true);
 
       // If the node doesn't have any outgoing edges and we do not have any
       // nodes in the `happens_after` set, we can't inline a function and
@@ -1514,11 +1561,36 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
     }
   }
 
+  // Identity nodes added to the function body in place of function outputs.
+  absl::flat_hash_set<string> output_nodes;
+  for (const OutputArgExpansion& output_arg : item.outputs()) {
+    for (const string& output_node : output_arg.output_nodes) {
+      output_nodes.insert(inlined_node_name(output_node));
+    }
+  }
+
+  // For each function output value we added an identity node that reads the
+  // tensor from one of the function body nodes. When we inline function into
+  // the main graph we want to bypass these nodes, so we keep a mapping from
+  // 'output node name' -> 'output tensor name'.
+  absl::flat_hash_map<string, string> output_tensors;
+
   // Move all the nodes to the optimized graph after successful preprocessing.
   for (NodeDef& func_body_node : *placed_graph_def.mutable_node()) {
+    const string& node_name = func_body_node.name();
+
+    // Skip output identity node, and add a mapping to the output tensor.
+    if (IsIdentity(func_body_node) && output_nodes.count(node_name)) {
+      output_tensors.emplace(node_name, func_body_node.input(0));
+      continue;
+    }
+
     optimized_graph->add_node()->Swap(&func_body_node);
   }
 
+  DCHECK(output_tensors.size() == item.output_size())
+      << "Each function output must be mapped to an output tensor";
+
   // TODO(ezhulenev): Inline nested indirect function calls.
 
   // Indirect function call is fully inlined into the optimized graph, and we do
@@ -1526,10 +1598,13 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
   // mapping from old output tensors, to the outputs of inlined nodes.
   int output_idx = 0;
   for (const OutputArgExpansion& output : item.outputs()) {
-    for (const string& output_tensor : output.output_tensors) {
+    for (const string& output_node : output.output_nodes) {
+      const string inlined_output = inlined_node_name(output_node);
+      const string& output_tensor = output_tensors.at(inlined_output);
+
       const SafeTensorId from_tensor(func_node.name(), output_idx++);
-      const SafeTensorId to_tensor = ParseTensorName(
-          AddPrefixToNodeName(output_tensor, /*prefix=*/func_node.name()));
+      const SafeTensorId to_tensor = ParseTensorName(output_tensor);
+
       ctx->AddTensorMapping(from_tensor, to_tensor);
     }
   }
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
index 79da7dfa2d..cebd002bed 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
@@ -660,7 +660,7 @@ TEST_F(FunctionOptimizerTest, InlineSymbolicGradient_IdentityFunc) {
   test::ExpectTensorEqual<float>(expected[0], optimized[0]);
 }
 
-TEST_F(FunctionOptimizerTest, InlineSymbolicGradient_NoInlineFunc) {
+TEST_F(FunctionOptimizerTest, InlineSymbolicGradientNoInlineFunc) {
   FunctionOptimizer optimizer(RewriterConfig::ON);
 
   FunctionDef func = FunctionDefHelper::Define(
diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc b/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc
index 12db5d6ca9..a061f6194a 100644
--- a/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc
@@ -231,7 +231,7 @@ TEST_F(MetaOptimizerTest, RunToggleOptimizersAndCustomGraphOptimizerTwice) {
 TEST_F(MetaOptimizerTest, OptimizeFunctionLibrary) {
   using test::function::NDef;
 
-  // Enable ony function optimization.
+  // Enable only function optimization.
   ConfigProto config_proto;
   auto& rewriter_config =
       *config_proto.mutable_graph_options()->mutable_rewrite_options();
@@ -300,7 +300,7 @@ TEST_F(MetaOptimizerTest, OptimizeFunctionLibrary) {
                                            output.library());
 
   // Specialized and optimized functions should be added to the graph.
-  EXPECT_EQ(6, optimized_flib.num_functions());
+  EXPECT_EQ(5, optimized_flib.num_functions());
 
   // Get a specialized function name.
   const auto specialized_name = [](const string& fn, const string& node,
@@ -314,25 +314,22 @@ TEST_F(MetaOptimizerTest, OptimizeFunctionLibrary) {
       specialized_name("MyQuadratic", "quadratic", "tf_graph");
 
   // MySquare should be specialized and optimized for 3 instantiations:
-  //   1. 'square' node in the main graph
-  //   2. 'square' node in the MyQuadratic specialization (not in a fetch set)
-  //   3. 'quadratic' node in the MyQuadratic specialization (is in a fetch set)
+  //   1.  'square' node in the main graph
+  //   2.  'square' node in the MyQuadratic specialization
+  //   3*. 'quadratic' node in the MyQuadratic specialization
+  //        has identical instantiation context to #2
 
   const string optimized_1 = specialized_name("MySquare", "square", "tf_graph");
   const string optimized_2 =
       specialized_name("MySquare", "square", optimized_0);
-  const string optimized_3 =
-      specialized_name("MySquare", "quadratic", optimized_0);
 
   const FunctionDef* optimized_func_0 = optimized_flib.Find(optimized_0);
   const FunctionDef* optimized_func_1 = optimized_flib.Find(optimized_1);
   const FunctionDef* optimized_func_2 = optimized_flib.Find(optimized_2);
-  const FunctionDef* optimized_func_3 = optimized_flib.Find(optimized_3);
 
   ASSERT_NE(optimized_func_0, nullptr);
   ASSERT_NE(optimized_func_1, nullptr);
   ASSERT_NE(optimized_func_2, nullptr);
-  ASSERT_NE(optimized_func_3, nullptr);
 
   // Graph should call optimized function.
   int count = 0;
@@ -351,13 +348,13 @@ TEST_F(MetaOptimizerTest, OptimizeFunctionLibrary) {
     if (node.name() == "square" && ++count) {
       EXPECT_EQ(optimized_2, node.op());
     } else if (node.name() == "quadratic" && ++count) {
-      EXPECT_EQ(optimized_3, node.op());
+      EXPECT_EQ(optimized_2, node.op());
     }
   }
   EXPECT_EQ(2, count);
 
-  const std::vector<const FunctionDef*> optimized_funcs = {
-      optimized_func_1, optimized_func_2, optimized_func_3};
+  const std::vector<const FunctionDef*> optimized_funcs = {optimized_func_1,
+                                                           optimized_func_2};
 
   // MyMul should be inlined into all optimized versions of MySquare.
   for (const FunctionDef* optimized_func : optimized_funcs) {
@@ -403,6 +400,96 @@ TEST_F(MetaOptimizerTest, OptimizeFunctionLibrary) {
   test::ExpectTensorEqual<int>(tensors_expected[1], tensors[1]);
 }
 
+TEST_F(MetaOptimizerTest, OptimizeFunctionLibraryPruneUnusedOutputs) {
+  using test::function::NDef;
+
+  ConfigProto config_proto;
+  MetaOptimizer optimizer(nullptr, config_proto);
+
+  // MyMul computes x*y three times and has three output values.
+  FunctionDef my_mul = FunctionDefHelper::Create(
+      "MyMul", {"x:T", "y:T"}, {"z0:T", "z1:T", "z2:T"}, {"T: {float, int32}"},
+      {{{"output0"}, "Mul", {"x", "y"}, {{"T", "$T"}}},
+       {{"output1"}, "Mul", {"x", "y"}, {{"T", "$T"}}},
+       {{"output2"}, "Mul", {"x", "y"}, {{"T", "$T"}}}},
+      /* Mapping between function returns and function node outputs. */
+      {{"z0", "output0:z:0"}, {"z1", "output1:z:0"}, {"z2", "output2:z:0"}});
+
+  // Call MyMyl and forward all three outputs.
+  FunctionDef my_fwd = FunctionDefHelper::Create(
+      "Fwd", {"x:T", "y:T"}, {"z0:T", "z1:T", "z2:T"}, {"T: {float, int32}"},
+      {{{"output"}, "MyMul", {"x", "y"}, {{"T", "$T"}}}},
+      /* Mapping between function returns and function node outputs. */
+      {{"z0", "output:z0:0"}, {"z1", "output:z1:0"}, {"z2", "output:z2:0"}});
+
+  // Mark both functions as `_noinline` to trigger specialization.
+  (*my_mul.mutable_attr())["_noinline"].set_b(true);
+  (*my_fwd.mutable_attr())["_noinline"].set_b(true);
+  std::vector<FunctionDef> function_library = {my_mul, my_fwd};
+
+  // Tensorflow graph:
+  //   a = Placeholder[T=float]
+  //   b = Placeholder[T=float]
+  //   fwd = Fwd(a, b)
+  //
+  // Fetch fwd:2 via Identity node.
+  GrapplerItem item;
+  item.id = "tf_graph";
+  item.fetch = {"ret"};
+  item.graph = test::function::GDef(
+      {NDef("a", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
+       NDef("b", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
+       NDef("fwd", "Fwd", {"a", "b"}, {{"T", DT_FLOAT}}, kDevice),
+       NDef("ret", "Identity", {"fwd:2"}, {{"T", DT_FLOAT}}, kDevice)},
+      function_library);
+
+  GraphDef output;
+  TF_EXPECT_OK(optimizer.Optimize(nullptr, item, &output));
+
+  FunctionLibraryDefinition optimized_flib(OpRegistry::Global(),
+                                           output.library());
+
+  // Specialized functions should be added to the graph.
+  EXPECT_EQ(3, optimized_flib.num_functions());
+
+  // Expected names of the specialized functions.
+  const string specialized_my_fwd = "Fwd_specialized_for_fwd_at_tf_graph";
+  const string specialized_my_mul =
+      absl::StrCat("MyMul_specialized_for_output_at_", specialized_my_fwd);
+
+  // Specialized MyMul should have just one output argument.
+  FunctionDef expected_my_mul = FunctionDefHelper::Create(
+      specialized_my_mul, {"x:float", "y:float"}, {"z2:float"}, {},
+      {{{"output2"}, "Mul", {"x", "y"}, {{"T", DT_FLOAT}}}},
+      /* Mapping between function returns and function node outputs. */
+      {{"z2", "output2:z:0"}});
+
+  // Specialized Fwd should also have just one output argument.
+  FunctionDef expected_my_fwd = FunctionDefHelper::Create(
+      specialized_my_fwd, {"x:float", "y:float"}, {"z2:float"}, {},
+      {{{"output"}, specialized_my_mul, {"x", "y"}, {{"T", DT_FLOAT}}}},
+      /* Mapping between function returns and function node outputs. */
+      {{"z2", "output:z2:0"}});
+
+  const FunctionDef* my_mul_spec = optimized_flib.Find(specialized_my_mul);
+  const FunctionDef* my_fwd_spec = optimized_flib.Find(specialized_my_fwd);
+
+  ASSERT_NE(my_mul_spec, nullptr);
+  ASSERT_NE(my_fwd_spec, nullptr);
+
+  CompareFunctions(expected_my_mul, *my_mul_spec);
+  CompareFunctions(expected_my_fwd, *my_fwd_spec);
+
+  item.feed.emplace_back("a", test::AsScalar<float>(2.0f));
+  item.feed.emplace_back("b", test::AsScalar<float>(4.0f));
+  auto tensors_expected = EvaluateFetchNodes(item);
+
+  GrapplerItem optimized = item.WithGraph(std::move(output));
+  auto tensors = EvaluateFetchNodes(optimized);
+
+  test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
+}
+
 TEST_F(MetaOptimizerTest, OptimizeFunctionLibraryPruneFunctionBody) {
   using test::function::NDef;
 
diff --git a/tensorflow/core/grappler/utils/BUILD b/tensorflow/core/grappler/utils/BUILD
index ff15541cdc..1fd0a02b65 100644
--- a/tensorflow/core/grappler/utils/BUILD
+++ b/tensorflow/core/grappler/utils/BUILD
@@ -182,6 +182,7 @@ cc_library(
         "@com_google_absl//absl/algorithm:container",
         "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/container:flat_hash_set",
+        "@com_google_absl//absl/container:inlined_vector",
         "@com_google_absl//absl/strings",
     ],
 )
diff --git a/tensorflow/core/grappler/utils/functions.cc b/tensorflow/core/grappler/utils/functions.cc
index 7c2180ae40..150728d030 100644
--- a/tensorflow/core/grappler/utils/functions.cc
+++ b/tensorflow/core/grappler/utils/functions.cc
@@ -328,13 +328,12 @@ GrapplerFunctionItem::GrapplerFunctionItem(
   for (const InputArgExpansion& input_arg : input_arg_expansions_) {
     for (const string& placeholder : input_arg.placeholders) {
       feed.push_back({placeholder, Tensor()});
-      input_arg_placeholders_.insert(placeholder);
     }
   }
   // Fill the fetch nodes with outputs.
   for (const OutputArgExpansion& output_arg : output_arg_expansions_) {
-    for (const string& output_tensor : output_arg.output_tensors) {
-      fetch.push_back(output_tensor);
+    for (const string& output_node : output_arg.output_nodes) {
+      fetch.push_back(output_node);
     }
   }
 
@@ -357,11 +356,6 @@ const std::size_t GrapplerFunctionItem::input_size() const {
   return input_arg_expansions_.size();
 }
 
-bool GrapplerFunctionItem::IsInputPlaceholder(const string& node_name) const {
-  return input_arg_placeholders_.find(node_name) !=
-         input_arg_placeholders_.end();
-}
-
 const std::vector<OutputArgExpansion>& GrapplerFunctionItem::outputs() const {
   return output_arg_expansions_;
 }
@@ -514,12 +508,18 @@ Status MakeGrapplerFunctionItem(const FunctionDef& func,
 
   // TODO(ezhulenev): support functions with tensor sequence inputs/outputs
 
-  // Make sure that there is no tensor sequences in outputs
+  // Make sure that there are no tensor lists in inputs or outputs.
+  for (const OpDef::ArgDef& input : signature.input_arg()) {
+    if (!input.type_list_attr().empty() || !input.number_attr().empty()) {
+      return errors::InvalidArgument(
+          "Inputs with lists of tensors are not supported. Input: ",
+          input.name());
+    }
+  }
   for (const OpDef::ArgDef& output : signature.output_arg()) {
     if (!output.type_list_attr().empty() || !output.number_attr().empty()) {
       return errors::InvalidArgument(
-          "Outputs with sequence of tensors are not supported. Unsupported "
-          "output: ",
+          "Outputs with lists of tensors are not supported. Output: ",
           output.name());
     }
   }
@@ -529,13 +529,6 @@ Status MakeGrapplerFunctionItem(const FunctionDef& func,
 
   // For each input argument create a placeholder in function body.
   for (const OpDef::ArgDef& input : signature.input_arg()) {
-    if (!input.type_list_attr().empty() || !input.number_attr().empty()) {
-      return errors::InvalidArgument(
-          "Inputs with sequence of tensors are not supported. Unsupported "
-          "input: ",
-          input.name());
-    }
-
     DataType input_data_type;
     TF_RETURN_IF_ERROR(instantiation.GetArgType(input, &input_data_type));
 
@@ -554,8 +547,25 @@ Status MakeGrapplerFunctionItem(const FunctionDef& func,
     inputs.push_back(std::move(input_expansion));
   }
 
-  // Add all function nodes to the function body
+  // Keep names of all nodes in the function body to guarantee that we do not
+  // add an identity with a duplicate name.
+  absl::flat_hash_set<absl::string_view> func_body_nodes;
+
+  // Generate unique output node name: "${out_arg_name}_output_node_${index}".
+  const auto output_node_name = [&func_body_nodes](const OpDef::ArgDef& out,
+                                                   int index) -> string {
+    string name = absl::StrCat(out.name(), "_output_node_", index);
+    int i = 1;
+    while (func_body_nodes.find(name) != func_body_nodes.end()) {
+      name = absl::StrCat(out.name(), "_output_node_", index, "_", i++);
+    }
+    return name;
+  };
+
+  // Add all function nodes to the function body.
   for (const NodeDef& func_def_node : func.node_def()) {
+    func_body_nodes.insert(func_def_node.name());
+
     NodeDef* new_node = function_body.add_node();
     *new_node = func_def_node;
 
@@ -578,8 +588,13 @@ Status MakeGrapplerFunctionItem(const FunctionDef& func,
 
   std::vector<OutputArgExpansion> outputs;
   outputs.reserve(signature.output_arg_size());
-  // Add function outputs
+
+  // For each function output argument we create an Identity node in the
+  // function body, that reads output tensor from the function body node.
   for (const OpDef::ArgDef& out : signature.output_arg()) {
+    DataType output_data_type;
+    TF_RETURN_IF_ERROR(instantiation.GetArgType(out, &output_data_type));
+
     std::vector<string> output_tensors;
     auto ret = func.ret().find(out.name());
     TF_RETURN_IF_ERROR(
@@ -589,13 +604,23 @@ Status MakeGrapplerFunctionItem(const FunctionDef& func,
             // Otherwise output must be one of the function inputs
             : connectivity.ExpandFunctionDefInput(out.name(), &output_tensors));
 
-    DataType output_data_type;
-    TF_RETURN_IF_ERROR(instantiation.GetArgType(out, &output_data_type));
+    absl::InlinedVector<string, 1> output_nodes;
+    for (int i = 0; i < output_tensors.size(); ++i) {
+      const string& output_tensor = output_tensors[i];
+
+      NodeDef* identity = function_body.add_node();
+      identity->set_name(output_node_name(out, i));
+      identity->set_op("Identity");
+      (*identity->mutable_attr())["T"].set_type(output_data_type);
+      identity->add_input(output_tensor);
+
+      output_nodes.push_back(identity->name());
+    }
 
     OutputArgExpansion output{/*output_name=*/out.name(),
                               /*data_type=*/output_data_type,
                               /*is_ref=*/out.is_ref(),
-                              /*output_tensors=*/std::move(output_tensors)};
+                              /*output_nodes=*/std::move(output_nodes)};
     outputs.push_back(std::move(output));
   }
 
@@ -663,7 +688,6 @@ Status ReplaceInputWithConst(const NodeDef& input_const, int input_index,
 
   // Delete placeholder from input expansion.
   string placeholder_name = input_arg_expansion->placeholders[placeholder_idx];
-  item->input_arg_placeholders_.erase(placeholder_name);
   input_arg_expansion->placeholders.erase(
       input_arg_expansion->placeholders.begin() + placeholder_idx);
 
@@ -687,43 +711,46 @@ Status ReplaceInputWithConst(const NodeDef& input_const, int input_index,
   return Status::OK();
 }
 
-Status RemoveUnusedOutputs(const absl::flat_hash_set<int>& active_outputs,
-                           GrapplerFunctionItem* item,
-                           std::vector<std::pair<int, int>>* output_mapping) {
+Status RemoveFunctionOutputs(const absl::flat_hash_set<int>& remove_outputs,
+                             GrapplerFunctionItem* item,
+                             std::vector<std::pair<int, int>>* output_mapping) {
   DCHECK(output_mapping->empty());
 
-  // Do some sanity checking of the active outputs positions.
-  for (int active_output : active_outputs) {
-    if (active_output < 0 || active_output >= item->output_size()) {
+  // Code below assumes that we do not support tensor list outputs and there is
+  // a 1-to-1 mapping between output tensor and output argument expansion.
+  for (const OutputArgExpansion& out_arg : item->outputs()) {
+    DCHECK(out_arg.output_nodes.size() == 1)
+        << "Output arg expansion must have single output";
+  }
+
+  // Do some sanity checking of the removed outputs positions.
+  for (int remove_output : remove_outputs) {
+    if (remove_output < 0 || remove_output >= item->output_size()) {
       return errors::InvalidArgument(
-          "Active output position is out of bound: active_output=",
-          active_output, " num_output_args=", item->output_size());
+          "Function output index is out of bound: index=", remove_output,
+          " max_output_index=", item->output_size());
     }
   }
 
-  absl::flat_hash_set<const OutputArgExpansion*> unused_output_args;
-
-  const auto is_unused_output_arg = [&](const OutputArgExpansion& output) {
-    return unused_output_args.find(&output) != unused_output_args.end();
+  absl::flat_hash_set<const OutputArgExpansion*> remove_output_args;
+  const auto is_remove_output_arg = [&](const OutputArgExpansion& output) {
+    return remove_output_args.find(&output) != remove_output_args.end();
   };
 
   for (int i = 0; i < item->output_size(); ++i) {
     const OutputArgExpansion& output = item->output(i);
-    DCHECK(output.output_tensors.size() == 1)
-        << "Output arg expansion must have single tensor";
-
-    if (active_outputs.find(i) == active_outputs.end()) {
-      VLOG(3) << "Remove unused output: output_name=" << output.output_name
-              << " output_position=" << i;
-      unused_output_args.insert(&output);
-    } else if (!unused_output_args.empty()) {
+    if (remove_outputs.find(i) != remove_outputs.end()) {
+      VLOG(3) << "Remove functions output: output_name=" << output.output_name
+              << "(index = " << i << ")";
+      remove_output_args.insert(&output);
+    } else if (!remove_output_args.empty()) {
       // Add output mapping only if output position changed.
-      output_mapping->push_back({i, i - unused_output_args.size()});
+      output_mapping->push_back({i, i - remove_output_args.size()});
     }
   }
 
   auto& o = item->output_arg_expansions_;
-  o.erase(std::remove_if(o.begin(), o.end(), is_unused_output_arg), o.end());
+  o.erase(std::remove_if(o.begin(), o.end(), is_remove_output_arg), o.end());
 
   return Status::OK();
 }
@@ -735,6 +762,55 @@ Status MakeFunctionDef(const GrapplerFunctionItem& item,
   func->mutable_signature()->set_description(item.description());
   func->mutable_signature()->set_is_stateful(item.is_stateful());
 
+  // Keep track of placeholders that were added to the graph in place of
+  // expanded function input arguments.
+  absl::flat_hash_set<absl::string_view> input_placeholders;
+  for (const InputArgExpansion& input_arg : item.inputs()) {
+    for (const string& placeholder : input_arg.placeholders) {
+      input_placeholders.insert(placeholder);
+    }
+  }
+
+  // Keep track of identity nodes that were added to the graph in place of
+  // expanded function output arguments.
+  absl::flat_hash_set<absl::string_view> output_nodes;
+  for (const OutputArgExpansion& output_arg : item.outputs()) {
+    for (const string& output_node : output_arg.output_nodes) {
+      output_nodes.insert(output_node);
+    }
+  }
+
+  // If the output identity node was not modified by any optimizer, we can
+  // bypass it and returns the function value from its input.
+  absl::flat_hash_map<absl::string_view, string> output_tensors;
+  for (const NodeDef& func_body_node : item.function_body().node()) {
+    if (!IsIdentity(func_body_node)) continue;
+
+    const string& node_name = func_body_node.name();
+    if (output_nodes.find(node_name) != output_nodes.end()) {
+      // Grappler optimizers might optimize nodes in the fanin of the output
+      // node, and forward their control dependencies. We can't express control
+      // dependencies in a function signature, so we have to keep the node.
+      if (func_body_node.input_size() == 1) {
+        VLOG(3) << "Bypass function output node: " << node_name << " -> "
+                << func_body_node.input(0);
+        output_tensors.emplace(node_name, func_body_node.input(0));
+      } else {
+        VLOG(3) << "Keep function output node: " << node_name;
+      }
+    }
+  }
+
+  // Return output tensor name (input of the output node) if it's safe to bypass
+  // output node, otherwise returns the output node name.
+  const auto output_tensor =
+      [&output_tensors](const OutputArgExpansion& output_arg) -> const string& {
+    const string& output_node = output_arg.output_nodes[0];
+    const auto is_output_tensor = output_tensors.find(output_node);
+    return is_output_tensor == output_tensors.end() ? output_node
+                                                    : is_output_tensor->second;
+  };
+
   // Build a GrapplerFunctionConnectivity from inputs and new function body.
   GrapplerFunctionConnectivity connectivity;
   TF_RETURN_IF_ERROR(
@@ -742,8 +818,8 @@ Status MakeFunctionDef(const GrapplerFunctionItem& item,
 
   // Add function input arguments.
   for (const InputArgExpansion& input_arg : item.inputs()) {
-    CHECK(input_arg.placeholders.size() == 1)  // do some sanity checking
-        << "Inputs of tensor sequences are not supported";
+    DCHECK(input_arg.placeholders.size() == 1)  // do some sanity checking
+        << "Inputs of tensor lists are not supported";
 
     OpDef::ArgDef arg_def;
     arg_def.set_name(input_arg.input_name);
@@ -754,8 +830,8 @@ Status MakeFunctionDef(const GrapplerFunctionItem& item,
 
   // Add function output arguments.
   for (const OutputArgExpansion& output_arg : item.outputs()) {
-    CHECK(output_arg.output_tensors.size() == 1)  // do some sanity checking
-        << "Outputs of tensor sequences are not supported";
+    DCHECK(output_arg.output_nodes.size() == 1)  // do some sanity checking
+        << "Outputs of tensor lists are not supported";
 
     OpDef::ArgDef arg_def;
     arg_def.set_name(output_arg.output_name);
@@ -763,11 +839,9 @@ Status MakeFunctionDef(const GrapplerFunctionItem& item,
     arg_def.set_is_ref(output_arg.is_ref);
     *func->mutable_signature()->add_output_arg() = arg_def;
 
-    string ret;
-    for (const string& output_tensor : output_arg.output_tensors) {
-      TF_RETURN_IF_ERROR(connectivity.AsFunctionDefInput(output_tensor, &ret));
-      (*func->mutable_ret())[output_arg.output_name] = ret;
-    }
+    TF_RETURN_IF_ERROR(connectivity.AsFunctionDefInput(
+        output_tensor(output_arg),
+        &(*func->mutable_ret())[output_arg.output_name]));
   }
 
   // Copy function definition specific attributes.
@@ -778,12 +852,16 @@ Status MakeFunctionDef(const GrapplerFunctionItem& item,
   }
 
   // Copy function body nodes to the FunctionDef and update input format
-  for (const NodeDef& func_body_node : item.function_body().node()) {
-    // Do not copy input placeholders
-    if (item.IsInputPlaceholder(func_body_node.name())) continue;
+  for (const NodeDef& func_node : item.function_body().node()) {
+    const string& name = func_node.name();
+
+    // Do not copy input placeholders.
+    if (IsPlaceholder(func_node) && input_placeholders.count(name)) continue;
+    // Do not copy output nodes that we bypassed.
+    if (IsIdentity(func_node) && output_tensors.count(name)) continue;
 
     NodeDef* func_def_node = func->add_node_def();
-    *func_def_node = func_body_node;
+    *func_def_node = func_node;
     TF_RETURN_IF_ERROR(connectivity.AsFunctionDefNode(func_def_node));
   }
 
diff --git a/tensorflow/core/grappler/utils/functions.h b/tensorflow/core/grappler/utils/functions.h
index ce8a3e5ac7..d5a41e7473 100644
--- a/tensorflow/core/grappler/utils/functions.h
+++ b/tensorflow/core/grappler/utils/functions.h
@@ -21,6 +21,7 @@ limitations under the License.
 
 #include "absl/container/flat_hash_map.h"
 #include "absl/container/flat_hash_set.h"
+#include "absl/container/inlined_vector.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/function.h"
 #include "tensorflow/core/framework/function.pb.h"
@@ -32,6 +33,21 @@ limitations under the License.
 namespace tensorflow {
 namespace grappler {
 
+// WARNING(ezhulenev): Currently we do not support functions with inputs or
+// outputs instantiated into multiple tensors. This can happen if the
+// input/output type is 'T*N' or 'list(type)'. This is enforced by multiple
+// checks across this file and also function_optimizer.cc. InputArgExpansion and
+// OutputArgExpansion already support lists of tensors, but that's pretty much
+// it, all other code is written with assumption that expansions are always of
+// size 1. MakeGrapplerFunctionItem will gracefully fail with Status error.
+//
+// This is a low priority feature, because in practice we don't see a lot (any
+// at all?) functions with such arguments. Tensorflow-Eager always produces
+// functions with plain input/output arguments.
+
+// TODO(ezhulenev): Support inputs and outputs of type 'T*N'.
+// TODO(ezhulenev): Support inputs and outputs of type 'list(type)'.
+
 // Depending on the function instantiation attributes, input argument to the
 // function might be a single tensor, list of tensors of the same type, or a
 // list of tensors of different types.
@@ -39,30 +55,23 @@ namespace grappler {
 // InputArgExpansion keeps track of the placeholders that were added to the
 // function body in place of function inputs and a resolved input data type.
 struct InputArgExpansion {
-  // TODO(ezhulenev): Add support for functions with tensor sequence inputs of
-  // different data types.
-  // TODO(ezhulenev): Support type parametrized inputs?
-  string input_name;                 // name of the function input argument
-  DataType data_type;                // input data type
-  bool is_ref;                       // if true, inputs are required to be refs
-  std::vector<string> placeholders;  // names of placeholder nodes in the
-                                     // function body
+  string input_name;
+  DataType data_type;
+  bool is_ref;
+  absl::InlinedVector<string, 1> placeholders;
 };
 
 // Depending on the function instantiation attributes, output argument is mapped
 // to one or more outputs of one of the function body nodes.
 //
-// OutputArgExpansion keeps mapping from a function output arg to the output
-// tensors of a function body nodes and a resolved output data type
+// OutputArgExpansion keeps track of the Identity nodes that were added to the
+// function body to forward output tensors. Adding these output nodes allows
+// nested function inlining and specialization (see function optimizer).
 struct OutputArgExpansion {
-  // TODO(ezhulenev): Add support for functions with tensor sequence outputs of
-  // different data types.
-  // TODO(ezhulenev): Support type parametrized outputs?
-  string output_name;                  // name of the function output argument
-  DataType data_type;                  // output data type
-  bool is_ref;                         // if true, outputs are refs
-  std::vector<string> output_tensors;  // names of output tensor from the
-                                       // function body nodes
+  string output_name;
+  DataType data_type;
+  bool is_ref;
+  absl::InlinedVector<string, 1> output_nodes;
 };
 
 // FunctionDef uses different connectivity encoding for the function body nodes,
@@ -87,7 +96,7 @@ class GrapplerFunctionConnectivity {
   // When expanding inputs in function def format, single input might be
   // expanded into multiple tensors. When converting back to the function def
   // format from graph def format, it's always a 1-to-1 relationship.
-  // FunctionDef built from GrapplerFunctionItem is always specialized to it's
+  // FunctionDef built from GrapplerFunctionItem is always specialized to its
   // instantiation attributes and length of input args (and node def outputs) is
   // known.
 
@@ -144,8 +153,6 @@ class GrapplerFunctionItem : public GrapplerItem {
 
   const string& description() const;
 
-  bool IsInputPlaceholder(const string& node_name) const;
-
   const std::vector<InputArgExpansion>& inputs() const;
   const InputArgExpansion& input(int i) const;
   const std::size_t input_size() const;
@@ -168,10 +175,9 @@ class GrapplerFunctionItem : public GrapplerItem {
                                          GrapplerFunctionItem*);
   friend Status ReplaceInputWithConst(const NodeDef&, int,
                                       GrapplerFunctionItem*);
-  friend Status RemoveUnusedOutputs(
-      const absl::flat_hash_set<int>& active_outputs,
-      GrapplerFunctionItem* item,
-      std::vector<std::pair<int, int>>* output_mapping);
+  friend Status RemoveFunctionOutputs(const absl::flat_hash_set<int>&,
+                                      GrapplerFunctionItem*,
+                                      std::vector<std::pair<int, int>>*);
 
   GrapplerFunctionItem(string func_name, string description,
                        AttrSlice func_attr,
@@ -187,16 +193,14 @@ class GrapplerFunctionItem : public GrapplerItem {
   std::vector<InputArgExpansion> input_arg_expansions_;
   std::vector<OutputArgExpansion> output_arg_expansions_;
 
-  std::set<string> input_arg_placeholders_;
-
   bool is_stateful_ = false;
 };
 
 // Check if function input/output types are fully defined only at instantiation
-// time (parametrized by it's instantiation node).
+// time (parametrized by its instantiation node).
 bool HasParametrizedType(const FunctionDef& func);
 
-// Check if a function body is parametrized by it's instantiation node. Function
+// Check if a function body is parametrized by its instantiation node. Function
 // body is parametrized, if it has at least one node with a 'placeholder'
 // attribute.
 bool HasParametrizedBody(const FunctionDef& func);
@@ -228,15 +232,16 @@ Status RegisterGrapplerFunctionConnectivity(
 Status ReplaceInputWithConst(const NodeDef& input_const, int input_index,
                              GrapplerFunctionItem* item);
 
-// Remove function output arguments that do not have any active outputs (output
-// tensor connected to other node inputs or in a fetch set). Active outputs uses
-// GraphDef output position encoding, and multiple active outputs could
-// potentially be connected to the same output argument (in case of tensor list
-// outputs). Add output mapping for all active outputs that changed it's output
-// position (std::pair<old position, new position>).
-Status RemoveUnusedOutputs(const absl::flat_hash_set<int>& active_outputs,
-                           GrapplerFunctionItem* item,
-                           std::vector<std::pair<int, int>>* output_mapping);
+// Removes outputs from instantiated grappler function item. Function node
+// outputs use GraphDef output index encoding, and multiple outputs might belong
+// to the same output argument expansion (in case of tensor list outputs). For
+// all active function outputs that changed its output index, this function adds
+// an output mapping (std::pair<old index, new index>).
+Status RemoveFunctionOutputs(const absl::flat_hash_set<int>& remove_outputs,
+                             GrapplerFunctionItem* item,
+                             std::vector<std::pair<int, int>>* output_mapping);
+
+// TODO(ezhulennev, b/120103818): Add RemoveFunctionInputs.
 
 // Make a GrapplerFunctionItem from the function definition and function
 // instantiation attributes (caller node attributes). Returns error if the given
@@ -251,7 +256,7 @@ Status MakeGrapplerFunctionItem(const FunctionDef& func,
 // fully defined (no type or body parametrization).
 // TODO(ezhulenev): Support parametrized functions without fully defined
 // instantiation attributes? Do we ever want to optimize parametrized function
-// without specializing it to it's instantiation attributes (at least types)?
+// without specializing it to its instantiation attributes (at least types)?
 Status MakeGrapplerFunctionItem(const FunctionDef& func,
                                 const FunctionLibraryDefinition& flib,
                                 int graph_def_version,
diff --git a/tensorflow/core/grappler/utils/functions_test.cc b/tensorflow/core/grappler/utils/functions_test.cc
index 29d6100d23..c49920c79c 100644
--- a/tensorflow/core/grappler/utils/functions_test.cc
+++ b/tensorflow/core/grappler/utils/functions_test.cc
@@ -249,15 +249,16 @@ TEST_F(FunctionsTest, FromSimpleFunctionDef) {
                                         flib, TF_GRAPH_DEF_VERSION, &item));
 
   EXPECT_EQ("XTimesTwo", item.id);
-  EXPECT_EQ(4, item.function_body().node_size());
+  EXPECT_EQ(5, item.function_body().node_size());
 
   EXPECT_EQ(1, item.input_size());
   EXPECT_EQ("x", item.input(0).input_name);
-  EXPECT_EQ(std::vector<string>{"x"}, item.input(0).placeholders);
+  ASSERT_EQ(1, item.input(0).placeholders.size());
+  EXPECT_EQ("x", item.input(0).placeholders[0]);
 
   EXPECT_EQ(1, item.output_size());
   EXPECT_EQ("y", item.output(0).output_name);
-  EXPECT_EQ("y", item.output(0).output_tensors[0]);
+  EXPECT_EQ("y_output_node_0", item.output(0).output_nodes[0]);
 
   int count = 0;
   for (const NodeDef &node : item.function_body().node()) {
@@ -279,9 +280,13 @@ TEST_F(FunctionsTest, FromSimpleFunctionDef) {
       EXPECT_EQ(2, node.input_size());
       EXPECT_EQ("x", node.input(0));
       EXPECT_EQ("scale", node.input(1));
+    } else if (node.name() == "y_output_node_0" && ++count) {
+      EXPECT_EQ("Identity", node.op());
+      ASSERT_EQ(1, node.input_size());
+      EXPECT_EQ("y", node.input(0));
     }
   }
-  EXPECT_EQ(4, count);
+  EXPECT_EQ(5, count);
 }
 
 TEST_F(FunctionsTest, FromFunctionDefWithMultiOutputNodes) {
@@ -326,7 +331,7 @@ TEST_F(FunctionsTest, FromFunctionDefWithMultiOutputNodes) {
                                         flib, TF_GRAPH_DEF_VERSION, &item));
 
   EXPECT_EQ("SubGrad", item.id);
-  EXPECT_EQ(12, item.function_body().node_size());
+  EXPECT_EQ(14, item.function_body().node_size());
 
   ASSERT_EQ(3, item.input_size());
   EXPECT_EQ("x", item.input(0).input_name);
@@ -334,8 +339,8 @@ TEST_F(FunctionsTest, FromFunctionDefWithMultiOutputNodes) {
   EXPECT_EQ("dz", item.input(2).input_name);
 
   ASSERT_EQ(2, item.output_size());
-  EXPECT_EQ("dx", item.output(0).output_tensors[0]);
-  EXPECT_EQ("dy", item.output(1).output_tensors[0]);
+  EXPECT_EQ("dx_output_node_0", item.output(0).output_nodes[0]);
+  EXPECT_EQ("dy_output_node_0", item.output(1).output_nodes[0]);
 
   int count = 0;
   for (const NodeDef &node : item.function_body().node()) {
@@ -359,9 +364,17 @@ TEST_F(FunctionsTest, FromFunctionDefWithMultiOutputNodes) {
       EXPECT_EQ(2, node.input_size());
       EXPECT_EQ("gy", node.input(0));
       EXPECT_EQ("rx:1", node.input(1));
+    } else if (node.name() == "dx_output_node_0" && ++count) {
+      EXPECT_EQ("Identity", node.op());
+      ASSERT_EQ(1, node.input_size());
+      EXPECT_EQ("dx", node.input(0));
+    } else if (node.name() == "dy_output_node_0" && ++count) {
+      EXPECT_EQ("Identity", node.op());
+      ASSERT_EQ(1, node.input_size());
+      EXPECT_EQ("dy", node.input(0));
     }
   }
-  EXPECT_EQ(6, count);
+  EXPECT_EQ(8, count);
 }
 
 TEST_F(FunctionsTest, FromFunctionDefWithNestedFuncs) {
@@ -472,7 +485,7 @@ TEST_F(FunctionsTest, FromFunctionDefWithOutputMappings) {
                                         flib, TF_GRAPH_DEF_VERSION, &item));
 
   EXPECT_EQ(1, item.output_size());
-  EXPECT_EQ("Exp", item.output(0).output_tensors[0]);
+  EXPECT_EQ("out_output_node_0", item.output(0).output_nodes[0]);
 
   int count = 0;
   for (const NodeDef &node : item.function_body().node()) {
@@ -488,9 +501,13 @@ TEST_F(FunctionsTest, FromFunctionDefWithOutputMappings) {
       EXPECT_EQ("Exp", node.op());
       EXPECT_EQ(1, node.input_size());
       EXPECT_EQ("Linear_func", node.input(0));
+    } else if (node.name() == "out_output_node_0" && ++count) {
+      EXPECT_EQ("Identity", node.op());
+      ASSERT_EQ(1, node.input_size());
+      EXPECT_EQ("Exp", node.input(0));
     }
   }
-  EXPECT_EQ(3, count);
+  EXPECT_EQ(4, count);
 }
 
 TEST_F(FunctionsTest, FromFunctionDefWithInputForwarding) {
@@ -517,27 +534,44 @@ TEST_F(FunctionsTest, FromFunctionDefWithInputForwarding) {
                                         flib, TF_GRAPH_DEF_VERSION, &item));
 
   EXPECT_EQ("ForwardInputs", item.id);
-  EXPECT_EQ(5, item.function_body().node_size());
+  EXPECT_EQ(8, item.function_body().node_size());
 
   EXPECT_EQ(3, item.output_size());
-  EXPECT_EQ("in0", item.output(0).output_tensors[0]);
-  EXPECT_EQ("arg2", item.output(1).output_tensors[0]);
-  EXPECT_EQ("arg3", item.output(2).output_tensors[0]);
+  EXPECT_EQ("out0_output_node_0", item.output(0).output_nodes[0]);
+  EXPECT_EQ("arg2_output_node_0", item.output(1).output_nodes[0]);
+  EXPECT_EQ("arg3_output_node_0", item.output(2).output_nodes[0]);
 
   int count = 0;
+
+  const auto is_arg_placeholder = [](const string &name) {
+    return name == "in0" || name == "in1" || name == "arg2" || name == "arg3" ||
+           name == "arg4";
+  };
+
   for (const NodeDef &node : item.function_body().node()) {
-    EXPECT_TRUE(node.name() == "in0" || node.name() == "in1" ||
-                node.name() == "arg2" || node.name() == "arg3" ||
-                node.name() == "arg4");
-    count++;
-    EXPECT_EQ("Placeholder", node.op());
-    if (node.name() == "arg3") {
-      EXPECT_EQ(DT_INT32, node.attr().at("dtype").type());
-    } else {
-      EXPECT_EQ(DT_FLOAT, node.attr().at("dtype").type());
+    if (is_arg_placeholder(node.name()) && node.op() == "Placeholder") {
+      count++;
+      if (node.name() == "arg3") {
+        EXPECT_EQ(DT_INT32, node.attr().at("dtype").type());
+      } else {
+        EXPECT_EQ(DT_FLOAT, node.attr().at("dtype").type());
+      }
+      continue;
+    }
+
+    EXPECT_EQ("Identity", node.op());
+    ASSERT_EQ(1, node.input_size());
+    EXPECT_TRUE(is_arg_placeholder(node.input(0)));
+
+    if (node.name() == "out0_output_node_0" && ++count) {
+      EXPECT_EQ("in0", node.input(0));
+    } else if (node.name() == "arg2_output_node_0" && ++count) {
+      EXPECT_EQ("arg2", node.input(0));
+    } else if (node.name() == "arg3_output_node_0" && ++count) {
+      EXPECT_EQ("arg3", node.input(0));
     }
   }
-  EXPECT_EQ(5, count);
+  EXPECT_EQ(8, count);
 }
 
 TEST_F(FunctionsTest, FromFunctionDefWithoutInput) {
@@ -566,16 +600,22 @@ TEST_F(FunctionsTest, FromFunctionDefWithoutInput) {
 
   EXPECT_EQ(0, item.input_size());
   EXPECT_EQ(1, item.output_size());
-  EXPECT_EQ("o", item.output(0).output_tensors[0]);
+  EXPECT_EQ("o_output_node_0", item.output(0).output_nodes[0]);
+  EXPECT_EQ(3, item.function_body().node_size());
 
-  EXPECT_EQ(2, item.function_body().node_size());
   const NodeDef &two = item.function_body().node(0);
   EXPECT_EQ("two", two.name());
   EXPECT_EQ(0, two.input_size());
+
   const NodeDef &cast = item.function_body().node(1);
   EXPECT_EQ("o", cast.name());
   EXPECT_EQ(1, cast.input_size());
   EXPECT_EQ("two", cast.input(0));
+
+  const NodeDef &retval = item.function_body().node(2);
+  EXPECT_EQ("o_output_node_0", retval.name());
+  EXPECT_EQ(1, retval.input_size());
+  EXPECT_EQ("o", retval.input(0));
 }
 
 TEST_F(FunctionsTest, FromFunctionDefWithSideEffectfulOps) {
@@ -674,7 +714,7 @@ TEST_F(FunctionsTest, ReplaceInputWithConst) {
   EXPECT_EQ(2, item.input_size());
   EXPECT_EQ(1, item.output_size());
 
-  ASSERT_EQ(3, item.function_body().node_size());
+  ASSERT_EQ(4, item.function_body().node_size());
 
   const NodeDef &input_x = item.function_body().node(0);
   const NodeDef &input_y = item.function_body().node(1);
@@ -748,8 +788,9 @@ TEST_F(FunctionsTest, SwapFunctionBodyAndMakeFunctionDef) {
       {{"z", "output:z:0"}});
 
   GraphDef id_func_body = test::function::GDef(
-      {/* pass input to output through identity */
-       NDef("output", "Identity", {"x"}, {{"T", "float"}})});
+      {/* Read and return input argument through Identity node. */
+       NDef("read_x", "Identity", {"x"}, {{"T", "float"}}),
+       NDef("z_output_node_0", "Identity", {"read_x"}, {{"T", "float"}})});
 
   protobuf::Map<string, AttrValue> func_instantiation_attr;
   func_instantiation_attr["T"].set_type(DT_FLOAT);
@@ -772,15 +813,15 @@ TEST_F(FunctionsTest, SwapFunctionBodyAndMakeFunctionDef) {
   // Check that graph body was updated.
   int count = 0;
   for (const NodeDef &node : specialized.node_def()) {
-    if (node.name() == "output" && ++count) {
+    if (node.name() == "read_x" && ++count) {
       EXPECT_EQ("Identity", node.op());
       EXPECT_EQ("x:0", node.input(0));
     }
   }
   EXPECT_EQ(1, count);
 
-  // And return tensor mapping was updated with a new output name (z->output).
-  EXPECT_EQ("output:output:0", (*specialized.mutable_ret())["z"]);
+  // And return tensor mapping was updated with a new output name (z->read_x).
+  EXPECT_EQ("read_x:output:0", (*specialized.mutable_ret())["z"]);
 }
 
 TEST_F(FunctionsTest, FunctionDefGrapplerFunctionItemRoundTrip) {
diff --git a/tensorflow/core/grappler/utils/grappler_test.cc b/tensorflow/core/grappler/utils/grappler_test.cc
index 5ef1cf444b..1b4b9f9a51 100644
--- a/tensorflow/core/grappler/utils/grappler_test.cc
+++ b/tensorflow/core/grappler/utils/grappler_test.cc
@@ -14,7 +14,9 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/core/grappler/utils/grappler_test.h"
+
 #include <memory>
+
 #include "absl/algorithm/container.h"
 #include "tensorflow/core/framework/attr_value_util.h"
 #include "tensorflow/core/grappler/utils.h"
@@ -25,6 +27,46 @@ limitations under the License.
 namespace tensorflow {
 namespace grappler {
 
+namespace {
+void CompareGraphNodes(protobuf::RepeatedPtrField<NodeDef>* want,
+                       protobuf::RepeatedPtrField<NodeDef>* got) {
+  auto comparator = [](const NodeDef& n1, const NodeDef& n2) -> bool {
+    return n1.name() < n2.name();
+  };
+
+  std::sort(want->begin(), want->end(), comparator);
+  std::sort(got->begin(), got->end(), comparator);
+
+  ASSERT_EQ(want->size(), got->size());
+
+  for (int i = 0; i < want->size(); ++i) {
+    NodeDef& want_node = (*want)[i];
+    NodeDef& got_node = (*got)[i];
+
+    EXPECT_EQ(want_node.op(), got_node.op());
+    EXPECT_EQ(want_node.name(), got_node.name());
+    EXPECT_EQ(want_node.device(), got_node.device());
+    ASSERT_EQ(want_node.input_size(), got_node.input_size());
+
+    // Order of control dependencies doesn't matter, so we sort them first.
+    const auto is_control = [](const string& input) -> bool {
+      return ParseTensorName(input).index() < 0;
+    };
+
+    auto want_inputs = want_node.mutable_input();
+    auto got_inputs = got_node.mutable_input();
+    std::sort(absl::c_find_if(*want_inputs, is_control), want_inputs->end());
+    std::sort(absl::c_find_if(*got_inputs, is_control), got_inputs->end());
+
+    for (int j = 0; j < want_node.input_size(); ++j) {
+      const TensorId want_tensor = ParseTensorName(want_node.input(j));
+      const TensorId got_tensor = ParseTensorName(got_node.input(j));
+      EXPECT_EQ(want_tensor.ToString(), got_tensor.ToString());
+    }
+  }
+}
+}  // namespace
+
 GrapplerTest::GrapplerTest() {
   // Turn off all the automatic optimizations to ensure that we run the graph
   // exactly as it is given to us. This ensures that we can compare the results
@@ -96,35 +138,11 @@ NodeDef* GrapplerTest::AddNode(
 }
 
 void GrapplerTest::CompareGraphs(GraphDef want, GraphDef got) const {
-  auto comparator = [](const NodeDef& n1, const NodeDef& n2) -> bool {
-    return n1.name() < n2.name();
-  };
-  std::sort(want.mutable_node()->begin(), want.mutable_node()->end(),
-            comparator);
-  std::sort(got.mutable_node()->begin(), got.mutable_node()->end(), comparator);
-
-  for (int i = 0; i < want.node_size(); ++i) {
-    std::sort(want.mutable_node(i)->mutable_input()->begin(),
-              want.mutable_node(i)->mutable_input()->end());
-  }
-  for (int i = 0; i < got.node_size(); ++i) {
-    std::sort(got.mutable_node(i)->mutable_input()->begin(),
-              got.mutable_node(i)->mutable_input()->end());
-  }
-
-  ASSERT_EQ(want.node_size(), got.node_size());
-  for (int i = 0; i < want.node_size(); ++i) {
-    EXPECT_EQ(want.node(i).op(), got.node(i).op());
-    EXPECT_EQ(want.node(i).name(), got.node(i).name());
-    EXPECT_EQ(want.node(i).device(), got.node(i).device());
+  CompareGraphNodes(want.mutable_node(), got.mutable_node());
+}
 
-    ASSERT_EQ(want.node(i).input_size(), got.node(i).input_size());
-    for (int j = 0; j < want.node(i).input_size(); ++j) {
-      const TensorId want_tensor = ParseTensorName(want.node(i).input(j));
-      const TensorId got_tensor = ParseTensorName(got.node(i).input(j));
-      EXPECT_EQ(want_tensor.ToString(), got_tensor.ToString());
-    }
-  }
+void GrapplerTest::CompareFunctions(FunctionDef want, FunctionDef got) const {
+  CompareGraphNodes(want.mutable_node_def(), got.mutable_node_def());
 }
 
 void GrapplerTest::CompareNodes(const NodeDef& want, const NodeDef& got) const {
diff --git a/tensorflow/core/grappler/utils/grappler_test.h b/tensorflow/core/grappler/utils/grappler_test.h
index 20fd04c1c6..26c1db3740 100644
--- a/tensorflow/core/grappler/utils/grappler_test.h
+++ b/tensorflow/core/grappler/utils/grappler_test.h
@@ -62,6 +62,13 @@ class GrapplerTest : public ::testing::Test {
   // equality, and adds all failuires to the current test.
   void CompareNodes(const NodeDef& want, const NodeDef& got) const;
 
+  // Checks if two functions are equal. Both functions must have the same set of
+  // nodes with the same inputs and attributes. Nodes can be in different order.
+  //
+  // NOTE: This function uses EXPECT/ASSERT macros to check node properties
+  // equality, and adds all failures to the current test.
+  void CompareFunctions(FunctionDef want, FunctionDef got) const;
+
   // Checks if node 'src' is directly connected to the input($position) of
   // 'dst'.
   bool IsNodesDirectlyConnected(const NodeMap& node_map, const string& src,
-- 
GitLab


From 0a88318ac73a3d30dfee4ab1541070cbf82fe05c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 9 Jan 2019 14:04:07 -0800
Subject: [PATCH 0481/1095] Add
 `tf.distribute.Strategy.experimental_make_numpy_iterator()` function.

PiperOrigin-RevId: 228584021
---
 tensorflow/contrib/distribute/python/BUILD    |   4 +
 .../python/collective_all_reduce_strategy.py  |   8 +-
 .../collective_all_reduce_strategy_test.py    |   7 ++
 .../contrib/distribute/python/keras_test.py   |  58 +++------
 .../distribute/python/mirrored_strategy.py    |  30 +++++
 .../python/mirrored_strategy_multigpu_test.py |   3 +
 .../distribute/python/one_device_strategy.py  |   9 +-
 .../python/one_device_strategy_test.py        |   4 +
 .../python/parameter_server_strategy.py       |  31 +++++
 .../python/parameter_server_strategy_test.py  |  12 ++
 .../distribute/python/strategy_test_lib.py    |  29 +++++
 .../contrib/distribute/python/tpu_strategy.py |   9 ++
 tensorflow/python/distribute/BUILD            |  32 +++++
 .../python/distribute/distribute_lib.py       |  72 +++++++++++-
 .../python/distribute/mirrored_strategy.py    |  10 ++
 tensorflow/python/distribute/numpy_dataset.py |  97 +++++++++++++++
 .../python/distribute/numpy_dataset_test.py   |  44 +++++++
 .../distribute/parameter_server_strategy.py   |  13 +-
 .../engine/distributed_training_utils.py      | 111 +-----------------
 tensorflow/python/keras/engine/training.py    |  62 +++++-----
 .../keras/engine/training_distributed.py      |   6 -
 tensorflow/python/training/optimizer.py       |  13 +-
 ...orflow.distribute.-mirrored-strategy.pbtxt |   4 +
 ...orflow.distribute.-strategy-extended.pbtxt |   4 +
 .../v1/tensorflow.distribute.-strategy.pbtxt  |   4 +
 ...orflow.distribute.-mirrored-strategy.pbtxt |   4 +
 ...orflow.distribute.-strategy-extended.pbtxt |   4 +
 .../v2/tensorflow.distribute.-strategy.pbtxt  |   4 +
 28 files changed, 490 insertions(+), 198 deletions(-)
 create mode 100644 tensorflow/python/distribute/numpy_dataset.py
 create mode 100644 tensorflow/python/distribute/numpy_dataset_test.py

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index 7ee12812af..d4758d7518 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -131,6 +131,7 @@ py_library(
         "//tensorflow/python:math_ops",
         "//tensorflow/python/distribute:distribute_lib",
         "//tensorflow/python/distribute:input_lib",
+        "//tensorflow/python/distribute:numpy_dataset",
         "//tensorflow/python/distribute:reduce_util",
         "//tensorflow/python/distribute:values",
         "//tensorflow/python/eager:context",
@@ -153,6 +154,7 @@ py_library(
         "//tensorflow/python/distribute:cross_device_utils",
         "//tensorflow/python/distribute:input_lib",
         "//tensorflow/python/distribute:multi_worker_util",
+        "//tensorflow/python/distribute:numpy_dataset",
         "//tensorflow/python/distribute:values",
         "//tensorflow/python/eager:context",
     ],
@@ -178,6 +180,7 @@ py_library(
         "//tensorflow/python/eager:backprop",
         "//tensorflow/python/eager:context",
         "//tensorflow/python/eager:test",
+        "//third_party/py/numpy",
     ],
 )
 
@@ -303,6 +306,7 @@ py_library(
         "//tensorflow/python:tensor_util",
         "//tensorflow/python:util",
         "//tensorflow/python/distribute:input_lib",
+        "//tensorflow/python/distribute:numpy_dataset",
         "//tensorflow/python/distribute:reduce_util",
         "//tensorflow/python/distribute:values",
     ],
diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
index f6361cb6e8..39756b32c5 100644
--- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
+++ b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
@@ -28,6 +28,7 @@ from tensorflow.python.distribute import device_util
 from tensorflow.python.distribute import distribute_lib
 from tensorflow.python.distribute import input_lib
 from tensorflow.python.distribute import multi_worker_util
+from tensorflow.python.distribute import numpy_dataset
 from tensorflow.python.distribute import values
 from tensorflow.python.eager import context
 from tensorflow.python.framework import ops
@@ -86,6 +87,7 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
     else:
       local_devices = ("/device:CPU:0",)
     self._worker_device = device_util.canonicalize("/device:CPU:0")
+    self._host_input_device = numpy_dataset.SingleDevice(self._worker_device)
 
     self._collective_keys = cross_device_utils.CollectiveKeys()
     self._initialize_local(local_devices)
@@ -121,6 +123,7 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
                                                 task_id)
 
     self._worker_device = "/job:%s/task:%d" % (task_type, task_id)
+    self._host_input_device = numpy_dataset.SingleDevice(self._worker_device)
     if num_gpus_per_worker:
       local_devices = tuple(
           "%s/device:GPU:%d" % (self._worker_device, i)
@@ -157,6 +160,9 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
     if colocate_with is None:
       device_map = self._device_map
       logical_device = 0  # TODO(josh11b): Get logical device from scope here.
+    elif isinstance(colocate_with, numpy_dataset.SingleDevice):
+      with ops.device(colocate_with.device):
+        return next_creator(*args, **kwargs)
     else:
       device_map = colocate_with.device_map
       logical_device = colocate_with.logical_device
@@ -347,4 +353,4 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
   # TODO(priyag): Delete this once all strategies use global batch size.
   @property
   def _global_batch_size(self):
-    return False
+    return True
diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
index 4c8c01a216..c3e9c55e96 100644
--- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
@@ -466,6 +466,13 @@ class LocalCollectiveAllReduceStrategy(
     with self.cached_session(config=config, target=target):
       self._test_all_reduce_mean_gradient_tape(distribution)
 
+  def testNumpyIterator(self):
+    num_gpus = 2
+    if context.num_gpus() < num_gpus:
+      self.skipTest('Not enough GPUs')
+    strategy, _, _ = self._get_test_object(None, None, num_gpus)
+    self._test_numpy_iterator(strategy)
+
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index 1cb5fa30a3..40916afcfa 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -429,15 +429,6 @@ class TestEstimatorDistributionStrategy(test_util.TensorFlowTestCase,
 class TestDistributionStrategyWithNumpyArrays(test.TestCase,
                                               parameterized.TestCase):
 
-  @combinations.generate(strategy_for_numpy_input_combinations())
-  def test_creating_var_with_numpy_arrays(self, distribution):
-    with self.cached_session():
-      x = np.asarray(np.random.random((64, 3)), dtype=np.float32)
-      var_x = distributed_training_utils.get_var_for_numpy(distribution, x)
-      val = self.evaluate(var_x.value())
-      # Verify that the numpy value is copied to the variable.
-      self.assertAllEqual(x, val)
-
   @combinations.generate(strategy_for_numpy_input_combinations())
   def test_calculating_input_params_no_steps_no_batch_size(self, distribution):
     # Calculate the per_replica_batch_size scaling factor for strategies
@@ -576,26 +567,26 @@ class TestDistributionStrategyWithNumpyArrays(test.TestCase,
         metrics = ['mae']
         model.compile(optimizer, loss, metrics=metrics)
 
-      inputs = np.zeros((64, 3), dtype=np.float32)
-      targets = np.zeros((64, 4), dtype=np.float32)
+        inputs = np.zeros((64, 3), dtype=np.float32)
+        targets = np.zeros((64, 4), dtype=np.float32)
 
-      # Call fit with validation data
-      model.fit(inputs, targets, epochs=1, batch_size=2, verbose=0,
-                validation_data=(inputs, targets))
+        # Call fit with validation data
+        model.fit(inputs, targets, epochs=1, batch_size=2, verbose=0,
+                  validation_data=(inputs, targets))
 
-      # TODO(anjalisridhar): We need tests for when the batch size and steps are
-      # smaller and results in a 0 batch_size and steps value.
-      model.evaluate(inputs, targets)
-      # with steps
-      model.evaluate(inputs, targets, steps=2)
-      # with batch_size
-      model.evaluate(inputs, targets, batch_size=8)
+        # TODO(anjalisridhar): We need tests for when the batch size and steps
+        # are smaller and results in a 0 batch_size and steps value.
+        model.evaluate(inputs, targets)
+        # with steps
+        model.evaluate(inputs, targets, steps=2)
+        # with batch_size
+        model.evaluate(inputs, targets, batch_size=8)
 
-      model.predict(inputs)
-      # with steps
-      model.predict(inputs, steps=2)
-      # with batch_size
-      model.predict(inputs, batch_size=8)
+        model.predict(inputs)
+        # with steps
+        model.predict(inputs, steps=2)
+        # with batch_size
+        model.predict(inputs, batch_size=8)
 
   @combinations.generate(strategy_for_numpy_input_combinations())
   def test_calling_model_with_nested_numpy_arrays(self, distribution):
@@ -1192,21 +1183,6 @@ class TestDistributionStrategyValidation(test.TestCase,
           metrics = ['mae', keras.metrics.CategoricalAccuracy()]
           model.compile(optimizer, loss, metrics=metrics)
 
-  @combinations.generate(all_strategy_combinations_minus_default())
-  def test_loop_in_scope(self, distribution):
-    with self.cached_session():
-      with self.assertRaisesRegexp(
-          RuntimeError, 'should not be run inside the tf.distribute.Strategy'):
-        with distribution.scope():
-          x = keras.layers.Input(shape=(3,), name='input')
-          y = keras.layers.Dense(4, name='dense')(x)
-          model = keras.Model(x, y)
-          optimizer = gradient_descent.GradientDescentOptimizer(0.001)
-          loss = 'mse'
-          model.compile(optimizer, loss)
-          input_array = np.zeros((3, 3), dtype=np.float32)
-          model.predict(input_array)
-
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy.py b/tensorflow/contrib/distribute/python/mirrored_strategy.py
index e3ab2bf19e..5fa36fb402 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy.py
@@ -104,6 +104,36 @@ class MirroredStrategy(distribute_lib.DistributionStrategy):
                                 auto_shard_dataset)
     super(MirroredStrategy, self).__init__(extended)
 
+  # Override to change the documentation to reflect the different handling of
+  # global vs. local batch size between core and contrib.
+  def experimental_make_numpy_iterator(  # pylint: disable=useless-super-delegation
+      self, numpy_input, batch_size, num_epochs=1, shuffle=1024, session=None):
+    """Makes an iterator for input provided via a nest of numpy arrays.
+
+    NOTE: The `batch_size` argument here has different behavior for this
+    contrib version of `MirroredStrategy`.
+
+    Args:
+      numpy_input: A nest of NumPy input arrays that will be distributed evenly
+        across all replicas.
+      batch_size: The number of entries from the array we should consume in one
+        step of the computation, across all replicas. This is the per-replica
+        batch size. The global batch size will be this times
+        `num_replicas_in_sync`.
+      num_epochs: The number of times to iterate through the examples. A value
+        of `None` means repeat forever.
+      shuffle: Size of buffer to use for shuffling the input examples.
+        Use `None` to disable shuffling.
+      session: (TensorFlow v1.x graph execution only) A session used for
+        initialization.
+
+    Returns:
+      An `tf.distribute.InputIterator` which returns inputs for each step of the
+      computation.  User should call `initialize` on the returned iterator.
+    """
+    return super(MirroredStrategy, self).experimental_make_numpy_iterator(
+        numpy_input, batch_size, num_epochs, shuffle, session)
+
 
 class MirroredExtended(CoreMirroredExtended):
   """Implementation of (contrib) MirroredStrategy."""
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
index 9821828b2d..59d711ae01 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
@@ -116,6 +116,9 @@ class MirroredTwoDeviceDistributionTest(
     self._test_input_fn_iterator(iterator, distribution.extended.worker_devices,
                                  expected_values)
 
+  def testNumpyIterator(self, distribution):
+    self._test_numpy_iterator(distribution)
+
   def testGlobalStepUpdate(self, distribution):
     self._test_global_step_update(distribution)
 
diff --git a/tensorflow/contrib/distribute/python/one_device_strategy.py b/tensorflow/contrib/distribute/python/one_device_strategy.py
index fb470f8546..34b0c31087 100644
--- a/tensorflow/contrib/distribute/python/one_device_strategy.py
+++ b/tensorflow/contrib/distribute/python/one_device_strategy.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 from tensorflow.python.distribute import device_util
 from tensorflow.python.distribute import distribute_lib
 from tensorflow.python.distribute import input_lib
+from tensorflow.python.distribute import numpy_dataset
 from tensorflow.python.distribute import values
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -50,8 +51,8 @@ class OneDeviceExtended(distribute_lib.DistributionStrategyExtended):
     super(OneDeviceExtended, self).__init__(container_strategy)
     self._device = device
     self._default_device = device
-    worker = device_util.canonicalize("/device:CPU:0")
-    worker_device_pairs = [(worker, [self._device])]
+    self._input_device = device_util.canonicalize("/device:CPU:0")
+    worker_device_pairs = [(self._input_device, [self._device])]
     device_map = values.SingleDeviceMap(device)
     self._input_workers = input_lib.InputWorkers(
         device_map, worker_device_pairs)
@@ -82,6 +83,10 @@ class OneDeviceExtended(distribute_lib.DistributionStrategyExtended):
     return input_lib.InputFunctionIterator(
         input_fn, self._input_workers, [distribute_lib.InputContext()])
 
+  def _experimental_make_numpy_dataset(self, numpy_input, session):
+    return numpy_dataset.one_host_numpy_dataset(
+        numpy_input, self._input_device, session)
+
   def _broadcast_to(self, tensor, destinations):
     del destinations
     return tensor
diff --git a/tensorflow/contrib/distribute/python/one_device_strategy_test.py b/tensorflow/contrib/distribute/python/one_device_strategy_test.py
index 2403dc8f12..f81466a6c7 100644
--- a/tensorflow/contrib/distribute/python/one_device_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/one_device_strategy_test.py
@@ -59,6 +59,10 @@ class OneDeviceStrategyTest(
     self._test_input_fn_iterator(
         iterator, d.extended.worker_devices, expected_values)
 
+  @test_util.run_in_graph_and_eager_modes
+  def testNumpyIterator(self):
+    self._test_numpy_iterator(self._get_distribution_strategy())
+
   def testAllReduceSum(self):
     self._test_all_reduce_sum(self._get_distribution_strategy())
 
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy.py b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
index bb0b8eb992..0785427c2c 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
@@ -89,6 +89,37 @@ class ParameterServerStrategy(distribute_lib.DistributionStrategy):
     super(ParameterServerStrategy, self).__init__(
         ParameterServerExtended(self, num_gpus_per_worker))
 
+  # Override to change the documentation to reflect the different handling of
+  # global vs. local batch size between core and contrib.
+  def experimental_make_numpy_iterator(  # pylint: disable=useless-super-delegation
+      self, numpy_input, batch_size, num_epochs=1, shuffle=1024, session=None):
+    """Makes an iterator for input provided via a nest of numpy arrays.
+
+    NOTE: The `batch_size` argument here has different behavior for this
+    contrib version of `ParameterServerStrategy`.
+
+    Args:
+      numpy_input: A nest of NumPy input arrays that will be distributed evenly
+        across all replicas.
+      batch_size: The number of entries from the array we should consume in one
+        step of the computation, across all replicas. This is the per-replica
+        batch size. The global batch size will be this times
+        `num_replicas_in_sync`.
+      num_epochs: The number of times to iterate through the examples. A value
+        of `None` means repeat forever.
+      shuffle: Size of buffer to use for shuffling the input examples.
+        Use `None` to disable shuffling.
+      session: (TensorFlow v1.x graph execution only) A session used for
+        initialization.
+
+    Returns:
+      An `tf.distribute.InputIterator` which returns inputs for each step of the
+      computation.  User should call `initialize` on the returned iterator.
+    """
+    return super(ParameterServerStrategy,
+                 self).experimental_make_numpy_iterator(
+                     numpy_input, batch_size, num_epochs, shuffle, session)
+
 
 class ParameterServerExtended(CoreParameterServerExtended):
   """Implementation of ParameterServerStrategy."""
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
index 7836687e7d..802809e7c7 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
@@ -893,5 +893,17 @@ class ParameterServerStrategyWithChiefTest(ParameterServerStrategyTestBase,
       strategy.extended.call_for_each_replica(f)
 
 
+class LocalParameterServerStrategyTest(strategy_test_lib.DistributionTestBase,
+                                       parameterized.TestCase):
+
+  @combinations.generate(combinations.combine(mode=['graph', 'eager'],
+                                              use_core_strategy=[True, False],
+                                              required_gpus=2))
+  def testNumpyIterator(self, use_core_strategy):
+    strategy, _, _ = create_test_objects(
+        num_gpus=2, use_core_strategy=use_core_strategy)
+    self._test_numpy_iterator(strategy)
+
+
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/contrib/distribute/python/strategy_test_lib.py b/tensorflow/contrib/distribute/python/strategy_test_lib.py
index 4fbd630cf7..7455cbd02a 100644
--- a/tensorflow/contrib/distribute/python/strategy_test_lib.py
+++ b/tensorflow/contrib/distribute/python/strategy_test_lib.py
@@ -18,6 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import numpy as np
+
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.distribute import distribution_strategy_context as ds_context
@@ -295,6 +297,33 @@ class DistributionTestBase(test.TestCase):
       global_step_values = self.evaluate(global_step_tensors)
       self.assertEqual((1,) * len(global_step_tensors), global_step_values)
 
+  def _test_numpy_iterator(self, strategy):
+    with strategy.scope(), self.cached_session() as sess:
+      x = np.asarray([[1, 2], [6, 12], [2, 4],
+                      [5, 10], [3, 6], [4, 8]])
+      y = np.asarray([5, 4, 3, 2, 1, 0])
+      batch_size = 6
+      if not strategy.extended._global_batch_size:  # pylint: disable=protected-access
+        batch_size = batch_size // strategy.num_replicas_in_sync
+      i = strategy.experimental_make_numpy_iterator(
+          (x, y), batch_size=batch_size, num_epochs=2, shuffle=None,
+          session=sess)
+      self.evaluate(i.initialize())
+
+      def run_and_concatenate(strategy, i):
+        x, y = strategy.experimental_run(lambda z: z, i)
+        x, y = self.evaluate((strategy.unwrap(x), strategy.unwrap(y)))
+        return np.concatenate(x), np.concatenate(y)
+
+      x_1, y_1 = run_and_concatenate(strategy, i)
+      self.assertAllEqual(x, x_1)
+      self.assertAllEqual(y, y_1)
+      x_2, y_2 = run_and_concatenate(strategy, i)
+      self.assertAllEqual(x, x_2)
+      self.assertAllEqual(y, y_2)
+      with self.assertRaises(errors.OutOfRangeError):
+        run_and_concatenate(strategy, i)
+
 
 class OneDeviceDistributionTestBase(test.TestCase):
   """Some tests that should work with any one-device DistributionStrategy."""
diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index 00360bf996..3f89c5869e 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -34,6 +34,7 @@ from tensorflow.python.distribute import cross_device_ops as cross_device_ops_li
 from tensorflow.python.distribute import device_util
 from tensorflow.python.distribute import distribute_lib
 from tensorflow.python.distribute import input_lib
+from tensorflow.python.distribute import numpy_dataset
 from tensorflow.python.distribute import reduce_util
 from tensorflow.python.distribute import values
 from tensorflow.python.distribute.cluster_resolver import tpu_cluster_resolver as resolver_lib
@@ -303,6 +304,11 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
         functools.partial(self._call_dataset_fn, dataset_fn),
         self._input_workers)
 
+  def _experimental_make_numpy_dataset(self, numpy_input, session):
+    return numpy_dataset.one_host_numpy_dataset(
+        numpy_input, numpy_dataset.SingleDevice(self.get_host_cpu_device(0)),
+        session)
+
   # TODO(priyag): Deal with OutOfRange errors once b/111349762 is fixed.
   # TODO(sourabhbajaj): Remove the initial_loop_values parameter when we have
   # a mechanism to infer the outputs of `fn`. Pending b/110550782.
@@ -466,6 +472,9 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
     if colocate_with is None:
       device_map = self._device_map
       logical_device = 0  # TODO(josh11b): Get logical device from scope here.
+    elif isinstance(colocate_with, numpy_dataset.SingleDevice):
+      with ops.device(colocate_with.device):
+        return next_creator(*args, **kwargs)
     else:
       device_map = colocate_with.device_map
       logical_device = colocate_with.logical_device
diff --git a/tensorflow/python/distribute/BUILD b/tensorflow/python/distribute/BUILD
index faaa61934a..a6a1c470b4 100644
--- a/tensorflow/python/distribute/BUILD
+++ b/tensorflow/python/distribute/BUILD
@@ -124,6 +124,7 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":device_util",
+        ":numpy_dataset",
         ":reduce_util",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:constant_op",
@@ -221,6 +222,7 @@ py_library(
         ":distribute_lib",
         ":input_lib",
         ":multi_worker_util",
+        ":numpy_dataset",
         ":reduce_util",
         ":shared_variable_creator",
         ":values",
@@ -249,6 +251,7 @@ py_library(
     deps = [
         ":input_lib",
         ":mirrored_strategy",
+        ":numpy_dataset",
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:framework_ops",
@@ -276,6 +279,35 @@ py_library(
     ],
 )
 
+py_library(
+    name = "numpy_dataset",
+    srcs = ["numpy_dataset.py"],
+    deps = [
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:util",
+        "//tensorflow/python:variable_scope",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow/python/eager:context",
+        "//third_party/py/numpy",
+    ],
+)
+
+py_test(
+    name = "numpy_dataset_test",
+    size = "small",
+    srcs = ["numpy_dataset_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":numpy_dataset",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:variable_scope",
+        "//tensorflow/python/eager:test",
+        "//third_party/py/numpy",
+    ],
+)
+
 py_library(
     name = "input_lib",
     srcs = ["input_lib.py"],
diff --git a/tensorflow/python/distribute/distribute_lib.py b/tensorflow/python/distribute/distribute_lib.py
index 4ad8cc00b8..31213ab647 100644
--- a/tensorflow/python/distribute/distribute_lib.py
+++ b/tensorflow/python/distribute/distribute_lib.py
@@ -26,6 +26,7 @@ import enum
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.distribute import device_util
 from tensorflow.python.distribute import distribution_strategy_context
+from tensorflow.python.distribute import numpy_dataset
 from tensorflow.python.distribute import reduce_util
 from tensorflow.python.eager import context as eager_context
 from tensorflow.python.framework import constant_op
@@ -360,7 +361,7 @@ class DistributionStrategy(object):
     return self._extended._distribute_dataset(dataset_fn)  # pylint: disable=protected-access
 
   def make_dataset_iterator(self, dataset):
-    """Makes an iterator for input provided via input_dataset.
+    """Makes an iterator for input provided via `dataset`.
 
     Data from the given dataset will be distributed evenly across all the
     compute replicas. We will assume that the input dataset is batched by the
@@ -418,6 +419,40 @@ class DistributionStrategy(object):
     return self.extended._make_input_fn_iterator(  # pylint: disable=protected-access
         input_fn, replication_mode=replication_mode)
 
+  def experimental_make_numpy_iterator(
+      self, numpy_input, batch_size, num_epochs=1, shuffle=1024, session=None):
+    """Makes an iterator for input provided via a nest of numpy arrays.
+
+    Args:
+      numpy_input: A nest of NumPy input arrays that will be distributed evenly
+        across all replicas. Note that lists of Numpy arrays are stacked,
+        as that is normal `tf.data.Dataset` behavior.
+      batch_size: The number of entries from the array we should consume in one
+        step of the computation, across all replicas. This is the global batch
+        size. It should be divisible by `num_replicas_in_sync`.
+      num_epochs: The number of times to iterate through the examples. A value
+        of `None` means repeat forever.
+      shuffle: Size of buffer to use for shuffling the input examples.
+        Use `None` to disable shuffling.
+      session: (TensorFlow v1.x graph execution only) A session used for
+        initialization.
+
+    Returns:
+      An `tf.distribute.InputIterator` which returns inputs for each step of the
+      computation.  User should call `initialize` on the returned iterator.
+    """
+    ds = self.extended.experimental_make_numpy_dataset(
+        numpy_input, session=session)
+    if shuffle:
+      ds = ds.shuffle(shuffle)
+    if num_epochs != 1:
+      ds = ds.repeat(num_epochs)
+    # We need to use the drop_remainder argument to get a known static
+    # input shape which is required for TPUs.
+    drop_remainder = self.extended.experimental_require_static_shapes
+    ds = ds.batch(batch_size, drop_remainder=drop_remainder)
+    return self.make_dataset_iterator(ds)
+
   def experimental_run(self, fn, input_iterator=None):
     """Runs ops in `fn` on each replica, with inputs from `input_iterator`.
 
@@ -1083,6 +1118,29 @@ class DistributionStrategyExtended(object):
   def _make_input_fn_iterator(self, input_fn, replication_mode):
     raise NotImplementedError("must be implemented in descendants")
 
+  def experimental_make_numpy_dataset(self, numpy_input, session=None):
+    """Makes a dataset for input provided via a numpy array.
+
+    This avoids adding `numpy_input` as a large constant in the graph,
+    and copies the data to the machine or machines that will be processing
+    the input.
+
+    Args:
+      numpy_input: A nest of NumPy input arrays that will be distributed evenly
+        across all replicas. Note that lists of Numpy arrays are stacked,
+        as that is normal `tf.data.Dataset` behavior.
+      session: (TensorFlow v1.x graph execution only) A session used for
+        initialization.
+
+    Returns:
+      A `tf.data.Dataset` representing `numpy_input`.
+    """
+    _require_cross_replica_context_extended(self)
+    return self._experimental_make_numpy_dataset(numpy_input, session=session)
+
+  def _experimental_make_numpy_dataset(self, numpy_input, session):
+    raise NotImplementedError("must be implemented in descendants")
+
   def broadcast_to(self, tensor, destinations):
     """Mirror a tensor on one device to all worker devices.
 
@@ -1660,6 +1718,18 @@ class _DefaultDistributionExtended(DistributionStrategyExtended):
                               replication_mode=InputReplicationMode.PER_WORKER):
     return input_fn(InputContext()).make_initializable_iterator()
 
+  def _experimental_make_numpy_dataset(self, numpy_input, session):
+    numpy_flat = nest.flatten(numpy_input)
+    vars_flat = tuple(
+        variable_scope.variable(array_ops.zeros(i.shape, i.dtype),
+                                trainable=False, use_resource=True)
+        for i in numpy_flat
+    )
+    for v, i in zip(vars_flat, numpy_flat):
+      numpy_dataset.init_var_from_numpy(v, i, session)
+    vars_nested = nest.pack_sequence_as(numpy_input, vars_flat)
+    return dataset_ops.Dataset.from_tensor_slices(vars_nested)
+
   def _broadcast_to(self, tensor, destinations):
     if destinations is None:
       return tensor
diff --git a/tensorflow/python/distribute/mirrored_strategy.py b/tensorflow/python/distribute/mirrored_strategy.py
index 37b493d0f7..c0a39d4b55 100644
--- a/tensorflow/python/distribute/mirrored_strategy.py
+++ b/tensorflow/python/distribute/mirrored_strategy.py
@@ -29,6 +29,7 @@ from tensorflow.python.distribute import device_util
 from tensorflow.python.distribute import distribute_lib
 from tensorflow.python.distribute import input_lib
 from tensorflow.python.distribute import multi_worker_util
+from tensorflow.python.distribute import numpy_dataset
 from tensorflow.python.distribute import reduce_util
 from tensorflow.python.distribute import shared_variable_creator
 from tensorflow.python.distribute import values
@@ -460,6 +461,7 @@ class MirroredExtended(distribute_lib.DistributionStrategyExtended):
     self._input_workers = input_lib.InputWorkers(self._device_map)
     self._inferred_cross_device_ops = cross_device_ops_lib.choose_the_best(
         devices)
+    self._host_input_device = numpy_dataset.SingleDevice("/cpu:0")
 
   def _initialize_multi_worker(self, devices):
     """Initializes the object for multi-worker training."""
@@ -488,6 +490,7 @@ class MirroredExtended(distribute_lib.DistributionStrategyExtended):
     # their ops will end up on the cpu device of its first worker, e.g.
     # "/job:worker/task:0/device:CPU:0". Note this is not used in replica mode.
     self._default_device = workers[0]
+    self._host_input_device = numpy_dataset.SingleDevice(workers[0])
 
     self._device_map = values.ReplicaDeviceMap(devices)
     self._input_workers = input_lib.InputWorkers(
@@ -501,6 +504,9 @@ class MirroredExtended(distribute_lib.DistributionStrategyExtended):
     if colocate_with is None:
       device_map = self._device_map
       logical_device = 0  # TODO(josh11b): Get logical device from scope here.
+    elif isinstance(colocate_with, numpy_dataset.SingleDevice):
+      with ops.device(colocate_with.device):
+        return next_creator(*args, **kwargs)
     else:
       device_map = colocate_with.device_map
       logical_device = colocate_with.logical_device
@@ -571,6 +577,10 @@ class MirroredExtended(distribute_lib.DistributionStrategyExtended):
     return input_lib.InputFunctionIterator(
         input_fn, self._input_workers, input_contexts)
 
+  def _experimental_make_numpy_dataset(self, numpy_input, session):
+    return numpy_dataset.one_host_numpy_dataset(
+        numpy_input, self._host_input_device, session)
+
   # TODO(priyag): Deal with OutOfRange errors once b/111349762 is fixed.
   def _experimental_run_steps_on_iterator(self, fn, iterator, iterations,
                                           initial_loop_values=None):
diff --git a/tensorflow/python/distribute/numpy_dataset.py b/tensorflow/python/distribute/numpy_dataset.py
new file mode 100644
index 0000000000..5881e4cd59
--- /dev/null
+++ b/tensorflow/python/distribute/numpy_dataset.py
@@ -0,0 +1,97 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Code for creating a dataset out of a NumPy array."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.eager import context
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import variable_scope
+from tensorflow.python.util import nest
+
+
+def init_var_from_numpy(input_var, numpy_input, session):
+  """Initialize `input_var` to `numpy_input` using `session` in graph mode."""
+  with ops.init_scope():
+    if context.executing_eagerly():
+      input_var.assign(numpy_input)
+      return
+
+    assert session is not None
+    session.run(input_var.initializer)
+
+    start_placeholder = array_ops.placeholder(dtypes.int64, ())
+    end_placeholder = array_ops.placeholder(dtypes.int64, ())
+    slice_placeholder = array_ops.placeholder(input_var.dtype)
+    assign_slice_op = input_var[start_placeholder:end_placeholder].assign(
+        slice_placeholder)
+
+    # If each batch element is > 64 MB, then we copy each batch element
+    # individually. Otherwise, the slices will be < 128 MB. There might be
+    # padding which might mean that the slices are 128 MB even if the size of
+    # the tensor allocated is less than 128 MB.  This formula gives slices with
+    # size: ceil(64 MB / byte size per batch element) bytes.  Using ceil()
+    # guarantees we get a number >= 1.
+
+    # Calculate the size of each batch element.
+    byte_size_per_batch_element = (
+        np.prod(numpy_input.shape[1:]) * input_var.dtype.size)
+
+    # Calculate number of elements we want to copy per slice.
+    batch_size_per_slice = int(
+        np.ceil((64 << 20) / byte_size_per_batch_element))
+
+    # Copy slices of the above size starting at 0, except the last slice will be
+    # smaller.
+    start = 0
+    limit = numpy_input.shape[0]
+    while start < limit:
+      end = min(start + batch_size_per_slice, limit)
+      session.run(assign_slice_op, feed_dict={
+          start_placeholder: start,
+          end_placeholder: end,
+          slice_placeholder: numpy_input[start:end]})
+      start = end
+
+
+def one_host_numpy_dataset(numpy_input, colocate_with, session):
+  """Create a dataset on `colocate_with` from `numpy_input`."""
+  def create_colocated_variable(next_creator, *args, **kwargs):
+    kwargs["colocate_with"] = colocate_with
+    return next_creator(*args, **kwargs)
+
+  numpy_flat = nest.flatten(numpy_input)
+  with variable_scope.variable_creator_scope(create_colocated_variable):
+    vars_flat = tuple(variable_scope.variable(array_ops.zeros(i.shape, i.dtype),
+                                              trainable=False)
+                      for i in numpy_flat)
+  for v, i in zip(vars_flat, numpy_flat):
+    init_var_from_numpy(v, i, session)
+  vars_nested = nest.pack_sequence_as(numpy_input, vars_flat)
+  return dataset_ops.Dataset.from_tensor_slices(vars_nested)
+
+
+class SingleDevice(object):
+  """Used with `colocate_with` to create a non-mirrored variable."""
+
+  def __init__(self, device):
+    self.device = device
diff --git a/tensorflow/python/distribute/numpy_dataset_test.py b/tensorflow/python/distribute/numpy_dataset_test.py
new file mode 100644
index 0000000000..04eae1daa2
--- /dev/null
+++ b/tensorflow/python/distribute/numpy_dataset_test.py
@@ -0,0 +1,44 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for numpy_dataset."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.python.distribute import numpy_dataset
+from tensorflow.python.eager import test
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import variable_scope
+
+
+class InitVarFromNumpyTest(test.TestCase):
+
+  @test_util.run_in_graph_and_eager_modes
+  def test_creating_var_with_numpy_arrays(self):
+    with self.cached_session() as session:
+      x = np.asarray(np.random.random((64, 3)), dtype=np.float32)
+      initial = np.zeros_like(x)
+      var_x = variable_scope.variable(initial)
+      numpy_dataset.init_var_from_numpy(var_x, x, session)
+      val = self.evaluate(var_x.value())
+      # Verify that the numpy value is copied to the variable.
+      self.assertAllEqual(x, val)
+
+
+if __name__ == '__main__':
+  test.main()
diff --git a/tensorflow/python/distribute/parameter_server_strategy.py b/tensorflow/python/distribute/parameter_server_strategy.py
index 71fbffdc0d..ac5ee6f589 100644
--- a/tensorflow/python/distribute/parameter_server_strategy.py
+++ b/tensorflow/python/distribute/parameter_server_strategy.py
@@ -26,6 +26,7 @@ from tensorflow.python.distribute import device_util
 from tensorflow.python.distribute import distribute_lib
 from tensorflow.python.distribute import input_lib
 from tensorflow.python.distribute import multi_worker_util
+from tensorflow.python.distribute import numpy_dataset
 from tensorflow.python.distribute import values
 from tensorflow.python.distribute.cluster_resolver import SimpleClusterResolver
 from tensorflow.python.distribute.cluster_resolver import TFConfigClusterResolver
@@ -137,6 +138,7 @@ class ParameterServerStrategyExtended(
     assert cluster_spec.as_dict()
 
     worker_device = "/job:%s/task:%d" % (task_type, task_id)
+    self._input_host_device = numpy_dataset.SingleDevice(worker_device)
 
     # Define compute devices which is a list of device strings and one for each
     # replica. When there are GPUs, replicate operations on these GPUs.
@@ -195,6 +197,7 @@ class ParameterServerStrategyExtended(
   def _initialize_local(self, cluster_resolver):
     """Initialize internal devices for local training."""
     worker_device = device_util.canonicalize("/device:CPU:0")
+    self._input_host_device = numpy_dataset.SingleDevice(worker_device)
     num_gpus = cluster_resolver.num_accelerators()
     # Define compute devices which is a list of device strings and one for each
     # replica. When there are GPUs, replicate operations on these GPUs.
@@ -262,6 +265,10 @@ class ParameterServerStrategyExtended(
     return input_lib.InputFunctionIterator(input_fn, self._input_workers,
                                            [input_context])
 
+  def _experimental_make_numpy_dataset(self, numpy_input, session):
+    return numpy_dataset.one_host_numpy_dataset(
+        numpy_input, self._input_host_device, session)
+
   def _broadcast_to(self, tensor, destinations):
     # This is both a fast path for Python constants, and a way to delay
     # converting Python values to a tensor until we know what type it
@@ -329,8 +336,12 @@ class ParameterServerStrategyExtended(
       var_creator = next_creator
 
     if "colocate_with" in kwargs:
+      colocate_with = kwargs["colocate_with"]
+      if isinstance(colocate_with, numpy_dataset.SingleDevice):
+        with ops.device(colocate_with.device):
+          return var_creator(*args, **kwargs)
       with ops.device(None):
-        with ops.colocate_with(kwargs["colocate_with"]):
+        with ops.colocate_with(colocate_with):
           return var_creator(*args, **kwargs)
 
     with ops.colocate_with(None, ignore_existing=True):
diff --git a/tensorflow/python/keras/engine/distributed_training_utils.py b/tensorflow/python/keras/engine/distributed_training_utils.py
index 1678d84307..b6ef33f700 100644
--- a/tensorflow/python/keras/engine/distributed_training_utils.py
+++ b/tensorflow/python/keras/engine/distributed_training_utils.py
@@ -34,25 +34,13 @@ from tensorflow.python.keras import callbacks
 from tensorflow.python.keras import metrics as metrics_module
 from tensorflow.python.keras import optimizers
 from tensorflow.python.keras.optimizer_v2 import optimizer_v2
-from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import tf_logging as logging
-from tensorflow.python.training import distribution_strategy_context
 from tensorflow.python.training.mode_keys import ModeKeys
 from tensorflow.python.util import nest
 
 
-def validate_not_in_strategy_scope():
-  """Validate fit/eval/predict are not running in DS scope."""
-  if distribution_strategy_context.has_distribution_strategy():
-    if distribution_strategy_context.in_cross_replica_context():
-      raise RuntimeError(
-          'Fit/Eval/Predict should not be run inside the tf.distribute.Strategy'
-          ' scope. Only model creation and compilation should be in '
-          'tf.distribute.Strategy scope.')
-
-
 def set_weights(distribution_strategy, dist_model, weights):
   """Sets the weights of the replicated models.
 
@@ -530,100 +518,11 @@ def get_batch_dimension(iterator):
   return dims[0] if dims else None
 
 
-def get_cpu_device(distribution_strategy):
-  """Returns the CPU device of the TPU host or the default CPU device string.
-
-  Args:
-    distribution_strategy: The DistributionStrategy used to compile the model.
-
-  Returns:
-    A device string which is the TPU host's CPU device in case of
-    TPUDistributionStrategy or the default CPU device string in all other
-    cases.
-
-  Raises:
-    NotImplementedError: We currently don't support copying numpy data to
-    multiple hosts in the case of Cloud TPU pods.
-  """
-  if is_tpu_strategy(distribution_strategy):
-    if distribution_strategy.extended.num_hosts > 1:
-      raise NotImplementedError('TPUDistributionStrategy does not '
-                                'support numpy inputs when running on Cloud'
-                                'TPU pods.')
-    return distribution_strategy.extended.get_host_cpu_device(0)
-  else:
-    # For all strategies except TPUDistributionStrategy
-    # TODO(anjalisridhar): We may need to modify this when we add support for
-    # multi-worker strategy.
-    return '/CPU:0'
-
-
-def get_var_for_numpy(distribution_strategy, x):
-  if isinstance(x, list):
-    var_x = tuple([_get_var_for_numpy(distribution_strategy, single_input)
-                   for single_input in x])
-  else:
-    var_x = _get_var_for_numpy(distribution_strategy, x)
-  return var_x
-
-
-def _get_var_for_numpy(distribution_strategy, input_array):
-  """Creates a variable and assigns the value of the numpy array to it.
-
-  Args:
-    distribution_strategy: The DistributionStrategy used to compile the model.
-    input_array: The input numpy array whose value will be assigned to the
-      variable we create.
-
-  Returns:
-    The variable to which we will copy the value of the input numpy array.
-
-  """
-  with ops.device(get_cpu_device(distribution_strategy)):
-    # Create and initialize a variable on the CPU device. This is the CPU
-    # device of the host in the case of TPUDistributionStrategy.
-    input_var = variables.VariableV1(array_ops.zeros(input_array.shape,
-                                                     input_array.dtype),
-                                     trainable=False, use_resource=True)
-  K.get_session().run(input_var.initializer)
-
-  # Create a placeholder for the numpy array input slices. We copy the value
-  # of the input numpy array to the variable in slices of size 64 MB to avoid
-  # running into memory issues or RPC message limits.
-  start_placeholder = array_ops.placeholder(dtypes.int64, ())
-  end_placeholder = array_ops.placeholder(dtypes.int64, ())
-  slice_placeholder = array_ops.placeholder(input_var.dtype)
-  assign_slice_op = input_var[start_placeholder:end_placeholder].assign(
-      slice_placeholder)
-
-  # If each batch element is > 64 MB, then we copy each batch element
-  # individually. Otherwise, the slices will be < 128 MB. There might be padding
-  # which might mean that the slices are 128 MB even if the size of the
-  # tensor allocated is less than 128 MB.
-  # This formula gives slices with size:
-  # ceil(64 MB / byte size per batch element) bytes.
-  # Using ceil() guarantees we get a number >= 1.
-
-  # Calculate the size of each batch element.
-  byte_size_per_batch_element = np.prod(input_array.shape[1:]) * \
-                                input_var.dtype.size
-
-  # Calculate number of elements we want to copy per slice.
-  batch_size_per_slice = int(np.ceil((64 << 20) / byte_size_per_batch_element))
-
-  # Copy slices of the above size starting at 0, except the last slice will be
-  # smaller.
-  start = 0
-  limit = input_array.shape[0]
-  while start < limit:
-    end = min(start + batch_size_per_slice, limit)
-    K.get_session().run(assign_slice_op, feed_dict={
-        start_placeholder: start,
-        end_placeholder: end,
-        slice_placeholder: input_array[start:end]})
-    start = end
-
-  return input_var
+def list_to_tuple(maybe_list):
+  """Datasets treat lists specially, so switch them to tuples."""
+  if isinstance(maybe_list, list):
+    return tuple(maybe_list)
+  return maybe_list
 
 
 def _get_input_from_iterator(iterator, model):
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 1eda8cf797..a65c2b6413 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -2161,53 +2161,47 @@ class Model(Network):
                        'you should specify the `{steps_name}` argument.'
                        .format(steps_name=steps_name))
 
-    first_x_value = nest.flatten(x)[0]
-    if isinstance(first_x_value, np.ndarray):
-      # We need to use the drop_remainder argument to allow for a static
-      # input shape which is required for TPUs.
-      drop_remainder = self._distribution_strategy.require_static_shapes
-      if y is not None:
-        var_x = distributed_training_utils.get_var_for_numpy(
-            self._distribution_strategy, x)
-        var_y = distributed_training_utils.get_var_for_numpy(
-            self._distribution_strategy, y)
-        if sample_weight is not None:
-          var_sample_weights = distributed_training_utils.get_var_for_numpy(
-              self._distribution_strategy, sample_weight)
-
-          x = dataset_ops.Dataset.from_tensor_slices((var_x, var_y,
-                                                      var_sample_weights))
+    if ops.executing_eagerly_outside_functions():
+      session = None
+    else:
+      session = K.get_session()
+
+    with self._distribution_strategy.scope():
+      first_x_value = nest.flatten(x)[0]
+      if isinstance(first_x_value, np.ndarray):
+        x = distributed_training_utils.list_to_tuple(x)
+        if y is not None:
+          y = distributed_training_utils.list_to_tuple(y)
+          if sample_weight is not None:
+            sample_weight = distributed_training_utils.list_to_tuple(
+                sample_weight)
+            in_tuple = (x, y, sample_weight)
+          else:
+            in_tuple = (x, y)
         else:
-          x = dataset_ops.Dataset.from_tensor_slices((var_x, var_y))
+          in_tuple = x
 
         if shuffle:
           # 1024 is a good buffer size since it is much larger than the average
           # batch size provided by the user and provides sufficient randomness.
           # One thing to keep in mind is the memory usage based on the size of
           # each sample.
-          x = x.shuffle(1024)
-        x = x.repeat()
-        x = x.batch(batch_size, drop_remainder=drop_remainder)
-        y = None
-        sample_weight = None
+          shuffle_buffer = 1024
+        else:
+          shuffle_buffer = None
+        iterator = self._distribution_strategy.experimental_make_numpy_iterator(
+            in_tuple, batch_size, num_epochs=None, shuffle=shuffle_buffer,
+            session=session)
       else:
-        # This case is for the predict call where the dataset only contains
-        # inputs and no targets, i.e. it does not return a tuple
-        var_x = distributed_training_utils.get_var_for_numpy(
-            self._distribution_strategy, x)
-        x = dataset_ops.Dataset.from_tensor_slices(var_x)
-        x = x.batch(batch_size, drop_remainder=drop_remainder)
-
-    assert isinstance(x, dataset_ops.DatasetV2)
+        assert isinstance(x, dataset_ops.DatasetV2)
+        training_utils.validate_dataset_input(x, y, sample_weight,
+                                              validation_split)
+        iterator = self._distribution_strategy.make_dataset_iterator(x)
 
-    with self._distribution_strategy.scope():
-      iterator = self._distribution_strategy.make_dataset_iterator(x)
       init_op = iterator.initialize()
       if not context.executing_eagerly():
         K.get_session().run(init_op)
 
-    training_utils.validate_dataset_input(x, y, sample_weight,
-                                          validation_split)
     return iterator
 
   def _standardize_user_data(self,
diff --git a/tensorflow/python/keras/engine/training_distributed.py b/tensorflow/python/keras/engine/training_distributed.py
index 92a46e399a..0bd79f2b47 100644
--- a/tensorflow/python/keras/engine/training_distributed.py
+++ b/tensorflow/python/keras/engine/training_distributed.py
@@ -59,8 +59,6 @@ def fit_distributed(model,
 
   first_x_value = nest.flatten(x)[0]
   if isinstance(first_x_value, np.ndarray):
-    # TODO(b/122314600): Remove the scope validate.
-    distributed_training_utils.validate_not_in_strategy_scope()
     steps_per_epoch, batch_size = (
         distributed_training_utils.get_input_params(
             model._distribution_strategy, first_x_value, steps_per_epoch,
@@ -141,8 +139,6 @@ def evaluate_distributed(model,
   distributed_training_utils.validate_inputs(x, y, model._distribution_strategy)
   first_x_value = nest.flatten(x)[0]
   if isinstance(first_x_value, np.ndarray):
-    # TODO(b/122314600): Remove the scope validate.
-    distributed_training_utils.validate_not_in_strategy_scope()
     steps, batch_size = distributed_training_utils.get_input_params(
         model._distribution_strategy, first_x_value, steps, batch_size)
   batch_size = model._validate_or_infer_batch_size(batch_size, steps, x)
@@ -179,8 +175,6 @@ def predict_distributed(model,
       x, None, model._distribution_strategy)
   first_x_value = nest.flatten(x)[0]
   if isinstance(first_x_value, np.ndarray):
-    # TODO(b/122314600): Remove the scope validate.
-    distributed_training_utils.validate_not_in_strategy_scope()
     steps, batch_size = distributed_training_utils.get_input_params(
         model._distribution_strategy, first_x_value, steps, batch_size)
   batch_size = model._validate_or_infer_batch_size(batch_size, steps, x)
diff --git a/tensorflow/python/training/optimizer.py b/tensorflow/python/training/optimizer.py
index eaa563e84a..c6cc0b6044 100644
--- a/tensorflow/python/training/optimizer.py
+++ b/tensorflow/python/training/optimizer.py
@@ -554,14 +554,15 @@ class Optimizer(
     # by most optimizers.  It relies on the subclass implementing the following
     # methods: _create_slots(), _prepare(), _apply_dense(), and _apply_sparse().
 
-    # Handle DistributionStrategy case.
-    if distribute_ctx.get_cross_replica_context():
-      raise RuntimeError("Use `_distributed_apply()` instead of "
-                         "`apply_gradients()` in a cross-replica context.")
-    # TODO(isaprykin): Get rid of `has_distribution_strategy()` check by
+    # TODO(isaprykin): Get rid of `has_strategy()` check by
     # always calling _distributed_apply(), using the default distribution
     # as needed.
-    if distribute_ctx.has_distribution_strategy():
+    if distribute_ctx.has_strategy():
+      # Handle DistributionStrategy case.
+      if distribute_ctx.in_cross_replica_context():
+        raise RuntimeError("Use `_distributed_apply()` instead of "
+                           "`apply_gradients()` in a cross-replica context.")
+
       grads_and_vars = get_filtered_grad_fn(lambda: grads_and_vars)()
       return distribute_ctx.get_replica_context().merge_call(
           self._distributed_apply, args=(grads_and_vars, global_step, name))
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-mirrored-strategy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-mirrored-strategy.pbtxt
index b06c73d126..9c29067b6d 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-mirrored-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-mirrored-strategy.pbtxt
@@ -75,6 +75,10 @@ tf_class {
     name: "experimental_initialize"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "experimental_make_numpy_iterator"
+    argspec: "args=[\'self\', \'numpy_input\', \'batch_size\', \'num_epochs\', \'shuffle\', \'session\'], varargs=None, keywords=None, defaults=[\'1\', \'1024\', \'None\'], "
+  }
   member_method {
     name: "experimental_run"
     argspec: "args=[\'self\', \'fn\', \'input_iterator\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy-extended.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy-extended.pbtxt
index 77706e5713..37b620891f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy-extended.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy-extended.pbtxt
@@ -50,6 +50,10 @@ tf_class {
     name: "colocate_vars_with"
     argspec: "args=[\'self\', \'colocate_with_variable\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "experimental_make_numpy_dataset"
+    argspec: "args=[\'self\', \'numpy_input\', \'session\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
   member_method {
     name: "experimental_run_steps_on_iterator"
     argspec: "args=[\'self\', \'fn\', \'iterator\', \'iterations\', \'initial_loop_values\'], varargs=None, keywords=None, defaults=[\'1\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt
index 9a1df55142..4aa6f1c4e1 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt
@@ -74,6 +74,10 @@ tf_class {
     name: "experimental_initialize"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "experimental_make_numpy_iterator"
+    argspec: "args=[\'self\', \'numpy_input\', \'batch_size\', \'num_epochs\', \'shuffle\', \'session\'], varargs=None, keywords=None, defaults=[\'1\', \'1024\', \'None\'], "
+  }
   member_method {
     name: "experimental_run"
     argspec: "args=[\'self\', \'fn\', \'input_iterator\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-mirrored-strategy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-mirrored-strategy.pbtxt
index b06c73d126..9c29067b6d 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-mirrored-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-mirrored-strategy.pbtxt
@@ -75,6 +75,10 @@ tf_class {
     name: "experimental_initialize"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "experimental_make_numpy_iterator"
+    argspec: "args=[\'self\', \'numpy_input\', \'batch_size\', \'num_epochs\', \'shuffle\', \'session\'], varargs=None, keywords=None, defaults=[\'1\', \'1024\', \'None\'], "
+  }
   member_method {
     name: "experimental_run"
     argspec: "args=[\'self\', \'fn\', \'input_iterator\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy-extended.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy-extended.pbtxt
index 77706e5713..37b620891f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy-extended.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy-extended.pbtxt
@@ -50,6 +50,10 @@ tf_class {
     name: "colocate_vars_with"
     argspec: "args=[\'self\', \'colocate_with_variable\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "experimental_make_numpy_dataset"
+    argspec: "args=[\'self\', \'numpy_input\', \'session\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
   member_method {
     name: "experimental_run_steps_on_iterator"
     argspec: "args=[\'self\', \'fn\', \'iterator\', \'iterations\', \'initial_loop_values\'], varargs=None, keywords=None, defaults=[\'1\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt
index 9a1df55142..4aa6f1c4e1 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt
@@ -74,6 +74,10 @@ tf_class {
     name: "experimental_initialize"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "experimental_make_numpy_iterator"
+    argspec: "args=[\'self\', \'numpy_input\', \'batch_size\', \'num_epochs\', \'shuffle\', \'session\'], varargs=None, keywords=None, defaults=[\'1\', \'1024\', \'None\'], "
+  }
   member_method {
     name: "experimental_run"
     argspec: "args=[\'self\', \'fn\', \'input_iterator\'], varargs=None, keywords=None, defaults=[\'None\'], "
-- 
GitLab


From c94be5c69b9e540c27cad8c1d3f8604606874d00 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Wed, 19 Dec 2018 16:08:48 -0800
Subject: [PATCH 0482/1095] initial commit

---
 .../contrib/tensorrt/ops/trt_engine_op.cc     |  4 +-
 .../contrib/tensorrt/test/multiple_types.py   | 94 +++++++++++++++++++
 2 files changed, 96 insertions(+), 2 deletions(-)
 create mode 100644 tensorflow/contrib/tensorrt/test/multiple_types.py

diff --git a/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc b/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc
index 92405906eb..1dce862568 100644
--- a/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc
+++ b/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc
@@ -33,8 +33,8 @@ REGISTER_OP("TRTEngineOp")
     .Attr("input_shapes: list(shape)")
     .Attr("output_shapes: list(shape)")
     .Attr("segment_funcdef_name: string")
-    .Attr("InT: list({int8,float16,float32})")
-    .Attr("OutT: list({int8,float16,float32})")
+    .Attr("InT: list({int8,float16,float32,int32})")
+    .Attr("OutT: list({int8,float16,float32,int32})")
     .Attr("static_engine: bool = true")
     .Attr("fixed_input_size: bool = true")
     .Attr("cached_engine_batches: list(int) = []")
diff --git a/tensorflow/contrib/tensorrt/test/multiple_types.py b/tensorflow/contrib/tensorrt/test/multiple_types.py
new file mode 100644
index 0000000000..35b5a8519a
--- /dev/null
+++ b/tensorflow/contrib/tensorrt/test/multiple_types.py
@@ -0,0 +1,94 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Model script to test TF-TensorRT integration."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import gen_array_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import nn
+from tensorflow.python.platform import test
+
+
+class BiasaddMatMulTest(trt_test.TfTrtIntegrationTestBase):
+
+  def _ConstOp(self, shape, dtype):
+    return constant_op.constant(np.random.randn(*shape), dtype=dtype)
+
+  def GetParams(self):
+    """Testing conversion of BiasAdd MatMul in TF-TRT conversion."""
+    # Note that tf.nn.bias_add supports up to 5 dimensions.
+    input_dims = [100, 4]
+    output_name = "output"
+    g = ops.Graph()
+    with g.as_default():
+      input1 = array_ops.placeholder(
+          dtype=dtypes.float32, shape=input_dims, name='input_float32')
+
+      b1 = self._ConstOp((4, 10), dtypes.float32)
+      x1 = math_ops.matmul(input1, b)
+      b1 = self._ConstOp((1, 10), dtypes.float32)
+      x1 = x1 + b1
+
+      input2 = array_ops.placeholder(
+          dtype=dtypes.int32, shape=input_dims, name='input_int32')
+
+      b2 = self._ConstOp((4, 10), dtypes.float32)
+      x2 = math_ops.matmul(input2, b)
+      b2 = self._ConstOp((1, 10), dtypes.float32)
+      x2 = x2 + b2
+      out = array_ops.concat([x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11],
+                             axis=-1)
+      out = array_ops.squeeze(out, name=output_name)
+    return trt_test.TfTrtIntegrationTestParams(
+        gdef=g.as_graph_def(),
+        input_names=[input_name],
+        input_dims=[input_dims],
+        output_names=[output_name],
+        expected_output_dims=[(4, 6680)])
+
+  def GetConversionParams(self, run_params):
+    """Return a ConversionParams for test."""
+    conversion_params = super(BiasaddMatMulTest,
+                              self).GetConversionParams(run_params)
+    return conversion_params._replace(
+        max_batch_size=4,
+        maximum_cached_engines=1,
+        # Disable layout optimizer, since it will convert BiasAdd with NHWC
+        # format to NCHW format under four dimentional input.
+        rewriter_config=trt_test.OptimizerDisabledRewriterConfig())
+
+  def ExpectedEnginesToBuild(self, run_params):
+    """Return the expected engines to build."""
+    return ["TRTEngineOp_0"]
+
+  def ShouldRunTest(self, run_params):
+    """Whether to run the test."""
+    # TODO(aaroey): Trt 4.0 forbids conversion for tensors with rank <3 in int8
+    # mode, which is a bug. Re-enable this when trt library is fixed.
+    return not trt_test.IsQuantizationMode(run_params.precision_mode)
+
+
+if __name__ == "__main__":
+  test.main()
-- 
GitLab


From 62d6bdef48601770d6fc0e335b5602a4a86c6d00 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Fri, 21 Dec 2018 11:45:00 -0800
Subject: [PATCH 0483/1095] rename test

---
 tensorflow/contrib/tensorrt/BUILD             |  1 +
 .../test/{multiple_types.py => int_test.py}   | 26 ++++++++++---------
 2 files changed, 15 insertions(+), 12 deletions(-)
 rename tensorflow/contrib/tensorrt/test/{multiple_types.py => int_test.py} (86%)

diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index 3d34b91a74..e58a259a18 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -493,6 +493,7 @@ cuda_py_tests(
         "test/const_broadcast_test.py",
         "test/conv2d_test.py",
         "test/identity_output_test.py",
+        "test/int_test.py",
         "test/manual_test.py",
         "test/memory_alignment_test.py",
         "test/multi_connection_neighbor_engine_test.py",
diff --git a/tensorflow/contrib/tensorrt/test/multiple_types.py b/tensorflow/contrib/tensorrt/test/int_test.py
similarity index 86%
rename from tensorflow/contrib/tensorrt/test/multiple_types.py
rename to tensorflow/contrib/tensorrt/test/int_test.py
index 35b5a8519a..a4d625a213 100644
--- a/tensorflow/contrib/tensorrt/test/multiple_types.py
+++ b/tensorflow/contrib/tensorrt/test/int_test.py
@@ -40,33 +40,35 @@ class BiasaddMatMulTest(trt_test.TfTrtIntegrationTestBase):
     """Testing conversion of BiasAdd MatMul in TF-TRT conversion."""
     # Note that tf.nn.bias_add supports up to 5 dimensions.
     input_dims = [100, 4]
-    output_name = "output"
     g = ops.Graph()
     with g.as_default():
+      # float part
       input1 = array_ops.placeholder(
           dtype=dtypes.float32, shape=input_dims, name='input_float32')
-
       b1 = self._ConstOp((4, 10), dtypes.float32)
       x1 = math_ops.matmul(input1, b)
       b1 = self._ConstOp((1, 10), dtypes.float32)
       x1 = x1 + b1
 
+      # int part
       input2 = array_ops.placeholder(
           dtype=dtypes.int32, shape=input_dims, name='input_int32')
-
-      b2 = self._ConstOp((4, 10), dtypes.float32)
+      b2 = self._ConstOp((4, 10), dtypes.int32)
       x2 = math_ops.matmul(input2, b)
-      b2 = self._ConstOp((1, 10), dtypes.float32)
+      b2 = self._ConstOp((1, 10), dtypes.int32)
       x2 = x2 + b2
-      out = array_ops.concat([x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11],
-                             axis=-1)
-      out = array_ops.squeeze(out, name=output_name)
+
+      # combine
+      #y = x1 * x2
+
+      out1 = array_ops.identity(x1, name='output_float32')
+      out2 = array_ops.identity(x2, name='output_int32')
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
-        input_names=[input_name],
-        input_dims=[input_dims],
-        output_names=[output_name],
-        expected_output_dims=[(4, 6680)])
+        input_names=['input_float32', 'input_int32'],
+        input_dims=[[100, 4], [100, 4]],
+        output_names=['output_int32', 'output_float32'],
+        expected_output_dims=[(100, 10), (100, 10)])
 
   def GetConversionParams(self, run_params):
     """Return a ConversionParams for test."""
-- 
GitLab


From dd88f64811365d313423e9b28148921ae0f92003 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Fri, 21 Dec 2018 14:17:04 -0800
Subject: [PATCH 0484/1095] fix segfault in this case. should add a check to
 GetDeviceAndAllocator

---
 .../contrib/tensorrt/convert/convert_graph.cc | 13 ++++++++
 .../contrib/tensorrt/convert/convert_nodes.cc | 30 ++++++++++++------
 .../contrib/tensorrt/segment/segment.cc       |  2 +-
 tensorflow/contrib/tensorrt/test/int_test.py  | 31 ++++++++++---------
 4 files changed, 51 insertions(+), 25 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index bf2de94e04..ffe1840c68 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -356,10 +356,18 @@ tensorflow::Status GetEngineInfo(
     if (segment_nodes.count(node_name) == 0) continue;
     auto node = *it;
     auto node_device = node->requested_device();
+    LOG(INFO) << "requested: " << node_device;
+    DeviceNameUtils::ParsedName parsed_name;
+    if (DeviceNameUtils::ParseFullName(node_device, &parsed_name) && parsed_name.type == "CPU") {
+      LOG(INFO) << "type: " << parsed_name.type;
+      LOG(INFO) << "clear!";
+      node_device.clear();
+    }
     if (!node_device.empty()) {
       segment_devices.insert(node_device);
     } else {
       if (node->has_assigned_device_name()) {
+        LOG(INFO) << "assigned: " << node->assigned_device_name();
         segment_devices.insert(node->assigned_device_name());
       } else {
         VLOG(2) << "Node " << node->name()
@@ -897,6 +905,8 @@ std::pair<int, tensorflow::Allocator*> GetDeviceAndAllocator(
       StrAppend(&msg, ". Will get the allocator from first one.");
       LOG(WARNING) << msg;
     }
+    LOG(INFO) << "here12";
+    LOG(INFO) << devices[0]->name();
     tensorflow::AllocatorAttributes alloc_attr;
     cuda_device_id = devices[0]->tensorflow_gpu_device_info()->gpu_id;
     dev_allocator = devices[0]->GetAllocator(alloc_attr);
@@ -1008,8 +1018,11 @@ tensorflow::Status ConvertAfterShapes(ConversionParams& params) {
   VLOG(1) << "Current cuda device is " << old_cuda_device;
   std::vector<Node*> engine_nodes;
   engine_nodes.resize(engine_segments.size());
+  LOG(INFO) << "here " << engine_segments.size();
   for (int i = 0; i < engine_segments.size(); ++i) {
     auto& engine = engine_segments.at(i);
+
+    LOG(INFO) << "here4 ";
     // Partition the workspace size by the average of node ratio and segment
     // graphdef size
     engine.max_workspace_size_bytes =
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 179d631a3e..9d517b3be2 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -694,6 +694,14 @@ void ReorderCKtoKC(const TRT_ShapedWeights& iweights,
           ostrides);
       break;
     }
+    case tensorflow::DataType::DT_INT32: {
+      Reorder2(
+          {k, c}, static_cast<int const*>(iweights.GetValues()),
+          istrides,
+          static_cast<int*>(const_cast<void*>(oweights->GetValues())),
+          ostrides);
+      break;
+    }
     default:
       LOG(FATAL) << "Unsupported type in reorder expected fp32 or fp16 but got "
                  << DataTypeString(iweights.type_);
@@ -1725,6 +1733,12 @@ Status BinaryTensorOpTensor(OpConverterParams* params,
         "Unsupported binary op broadcast scheme for op ", node_def.name(), ": ",
         status.error_message());
   }
+  TFAttrs attrs(node_def);
+  nvinfer1::DataType dtype = attrs.get<nvinfer1::DataType>("T");
+  if (dtype == nvinfer1::DataType::kINT32) {
+    return errors::Unimplemented(
+        "Binary op ", node_def.op(), " does not support INT32, at ", node_def.name());
+  }
   if (params->validation_only) return Status::OK();
 
   const nvinfer1::ITensor* tensor_l = nullptr;
@@ -1741,8 +1755,6 @@ Status BinaryTensorOpTensor(OpConverterParams* params,
   }
 
   // Check type consistency.
-  TFAttrs attrs(node_def);
-  nvinfer1::DataType dtype = attrs.get<nvinfer1::DataType>("T");
   TFTRT_CHECK_EQ_TYPE(tensor_l->getType(), dtype)
       << DebugString(tensor_l->getType()) << " vs " << DebugString(dtype);
   TFTRT_CHECK_EQ_TYPE(tensor_r->getType(), dtype)
@@ -3450,12 +3462,12 @@ tensorflow::Status ConvertMatMul(OpConverterParams* params) {
 
   TFAttrs attrs(node_def);
   // TODO(jie): INT32 should be converted?
-  tensorflow::DataType tf_dtype = attrs.get<tensorflow::DataType>("T");
-  if (tf_dtype != DataType::DT_FLOAT && tf_dtype != DataType::DT_HALF) {
-    return errors::Unimplemented("Data type is not supported, for node ",
-                                 node_def.name(), " got ",
-                                 DataTypeString(tf_dtype));
-  }
+  // tensorflow::DataType tf_dtype = attrs.get<tensorflow::DataType>("T");
+  // if (tf_dtype != DataType::DT_FLOAT && tf_dtype != DataType::DT_HALF) {
+  //   return errors::Unimplemented("Data type is not supported, for node ",
+  //                                node_def.name(), " got ",
+  //                                DataTypeString(tf_dtype));
+  // }
   bool transpose_a = attrs.get<bool>("transpose_a");
   bool transpose_b = attrs.get<bool>("transpose_b");
 
@@ -3844,7 +3856,7 @@ tensorflow::Status ConvertSegmentToGraphDef(
       marker_nodes.insert(node_name);
       auto seg_node = segment_def->add_node();
       tensorflow::NodeDefBuilder builder(node_name, "Identity");
-      auto status = builder.Input(connection.inside_node_name, 0, dtype)
+      auto status = builder.Input(connection.inside_node_name, connection.inside_port, dtype)
                         .Finalize(seg_node);
       VLOG(1) << "Constructing output " << node_name << " for the edge "
               << connection.inside_node_name << ":" << connection.inside_port
diff --git a/tensorflow/contrib/tensorrt/segment/segment.cc b/tensorflow/contrib/tensorrt/segment/segment.cc
index 084a96e0fa..171ae41575 100644
--- a/tensorflow/contrib/tensorrt/segment/segment.cc
+++ b/tensorflow/contrib/tensorrt/segment/segment.cc
@@ -441,7 +441,7 @@ tensorflow::Status SegmentGraph(
     } else {
       const Status status = candidate_fn(node->tf_node());
       if (!status.ok()) {
-        VLOG(1) << "Not a TF-TRT candidate, "
+        LOG(INFO) << "Not a TF-TRT candidate, "
                 << "(Op type: " << node->tf_node()->type_string() << "), "
                 << "(Op name: " << node->name() << "), "
                 << "(Reason: " << status << ")";
diff --git a/tensorflow/contrib/tensorrt/test/int_test.py b/tensorflow/contrib/tensorrt/test/int_test.py
index a4d625a213..b5d69a799f 100644
--- a/tensorflow/contrib/tensorrt/test/int_test.py
+++ b/tensorflow/contrib/tensorrt/test/int_test.py
@@ -43,39 +43,40 @@ class BiasaddMatMulTest(trt_test.TfTrtIntegrationTestBase):
     g = ops.Graph()
     with g.as_default():
       # float part
-      input1 = array_ops.placeholder(
-          dtype=dtypes.float32, shape=input_dims, name='input_float32')
-      b1 = self._ConstOp((4, 10), dtypes.float32)
-      x1 = math_ops.matmul(input1, b)
-      b1 = self._ConstOp((1, 10), dtypes.float32)
-      x1 = x1 + b1
+      # input1 = array_ops.placeholder(
+      #     dtype=dtypes.float32, shape=input_dims, name='input_float32')
+      # b1 = self._ConstOp((4, 10), dtypes.float32)
+      # x1 = math_ops.matmul(input1, b1)
+      # b1 = self._ConstOp((1, 10), dtypes.float32)
+      # x1 = x1 + b1
 
       # int part
       input2 = array_ops.placeholder(
           dtype=dtypes.int32, shape=input_dims, name='input_int32')
+      #with g.device("/GPU:0"):
       b2 = self._ConstOp((4, 10), dtypes.int32)
-      x2 = math_ops.matmul(input2, b)
-      b2 = self._ConstOp((1, 10), dtypes.int32)
-      x2 = x2 + b2
+      x2 = math_ops.matmul(input2, b2)
+      b2 = self._ConstOp((10,), dtypes.int32)
+      x2 = nn.bias_add(x2, b2)
 
       # combine
       #y = x1 * x2
 
-      out1 = array_ops.identity(x1, name='output_float32')
+      #out1 = array_ops.identity(x1, name='output_float32')
       out2 = array_ops.identity(x2, name='output_int32')
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
-        input_names=['input_float32', 'input_int32'],
-        input_dims=[[100, 4], [100, 4]],
-        output_names=['output_int32', 'output_float32'],
-        expected_output_dims=[(100, 10), (100, 10)])
+        input_names=['input_int32'],
+        input_dims=[[100, 4]],
+        output_names=['output_int32'],
+        expected_output_dims=[(100, 10)])
 
   def GetConversionParams(self, run_params):
     """Return a ConversionParams for test."""
     conversion_params = super(BiasaddMatMulTest,
                               self).GetConversionParams(run_params)
     return conversion_params._replace(
-        max_batch_size=4,
+        max_batch_size=100,
         maximum_cached_engines=1,
         # Disable layout optimizer, since it will convert BiasAdd with NHWC
         # format to NCHW format under four dimentional input.
-- 
GitLab


From d72dae711ec00e9b9b3e60b0c70fda616a21bbc1 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Fri, 21 Dec 2018 15:28:20 -0800
Subject: [PATCH 0485/1095] Exclude CPU devices via segmenter. Prevent BiasAdd
 in int32

---
 .../contrib/tensorrt/convert/convert_graph.cc | 13 +++--------
 .../contrib/tensorrt/convert/convert_nodes.cc | 22 +++++++++++--------
 .../contrib/tensorrt/segment/segment.cc       | 21 +++++++++++++++++-
 tensorflow/contrib/tensorrt/test/int_test.py  |  5 ++---
 4 files changed, 38 insertions(+), 23 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index ffe1840c68..646451d802 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -355,19 +355,14 @@ tensorflow::Status GetEngineInfo(
     const auto& node_name = (*it)->name();
     if (segment_nodes.count(node_name) == 0) continue;
     auto node = *it;
+    // TODO: check for CPU device here
+    // If device is CPU, we should've caught that in the segmenter. Fall back here.
+
     auto node_device = node->requested_device();
-    LOG(INFO) << "requested: " << node_device;
-    DeviceNameUtils::ParsedName parsed_name;
-    if (DeviceNameUtils::ParseFullName(node_device, &parsed_name) && parsed_name.type == "CPU") {
-      LOG(INFO) << "type: " << parsed_name.type;
-      LOG(INFO) << "clear!";
-      node_device.clear();
-    }
     if (!node_device.empty()) {
       segment_devices.insert(node_device);
     } else {
       if (node->has_assigned_device_name()) {
-        LOG(INFO) << "assigned: " << node->assigned_device_name();
         segment_devices.insert(node->assigned_device_name());
       } else {
         VLOG(2) << "Node " << node->name()
@@ -905,8 +900,6 @@ std::pair<int, tensorflow::Allocator*> GetDeviceAndAllocator(
       StrAppend(&msg, ". Will get the allocator from first one.");
       LOG(WARNING) << msg;
     }
-    LOG(INFO) << "here12";
-    LOG(INFO) << devices[0]->name();
     tensorflow::AllocatorAttributes alloc_attr;
     cuda_device_id = devices[0]->tensorflow_gpu_device_info()->gpu_id;
     dev_allocator = devices[0]->GetAllocator(alloc_attr);
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 9d517b3be2..0d98da08b0 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -2609,12 +2609,18 @@ tensorflow::Status ConvertBiasAdd(OpConverterParams* params) {
     return errors::InvalidArgument("Input expects tensor and weights, at ",
                                    node_def.name());
   }
+  TFAttrs attrs(node_def);
+  tensorflow::DataType tf_dtype = attrs.get<tensorflow::DataType>("T");
+  if (tf_dtype != DataType::DT_FLOAT && tf_dtype != DataType::DT_HALF) {
+    return errors::Unimplemented("Data type is not supported, for node ",
+                                 node_def.name(), " got ",
+                                 DataTypeString(tf_dtype));
+  }
   if (params->validation_only) return Status::OK();
 
   nvinfer1::ITensor* tensor =
       const_cast<nvinfer1::ITensor*>(inputs.at(0).tensor());
   const nvinfer1::Dims original_dims = tensor->getDimensions();
-  TFAttrs attrs(node_def);
   const string data_format = attrs.get<string>("data_format");
   const int channel_index =
       (data_format == "NHWC" ? original_dims.nbDims - 1 : 0);
@@ -3461,13 +3467,12 @@ tensorflow::Status ConvertMatMul(OpConverterParams* params) {
   }
 
   TFAttrs attrs(node_def);
-  // TODO(jie): INT32 should be converted?
-  // tensorflow::DataType tf_dtype = attrs.get<tensorflow::DataType>("T");
-  // if (tf_dtype != DataType::DT_FLOAT && tf_dtype != DataType::DT_HALF) {
-  //   return errors::Unimplemented("Data type is not supported, for node ",
-  //                                node_def.name(), " got ",
-  //                                DataTypeString(tf_dtype));
-  // }
+  tensorflow::DataType tf_dtype = attrs.get<tensorflow::DataType>("T");
+  if (tf_dtype != DataType::DT_FLOAT && tf_dtype != DataType::DT_HALF) {
+    return errors::Unimplemented("Data type is not supported, for node ",
+                                 node_def.name(), " got ",
+                                 DataTypeString(tf_dtype));
+  }
   bool transpose_a = attrs.get<bool>("transpose_a");
   bool transpose_b = attrs.get<bool>("transpose_b");
 
@@ -3487,7 +3492,6 @@ tensorflow::Status ConvertBatchMatMul(OpConverterParams* params) {
   const auto& node_def = params->node_def;
   TFAttrs attrs(node_def);
 
-  // TODO(jie): INT32 should be converted?
   tensorflow::DataType tf_dtype = attrs.get<tensorflow::DataType>("T");
   if (tf_dtype != tensorflow::DataType::DT_FLOAT &&
       tf_dtype != tensorflow::DataType::DT_HALF) {
diff --git a/tensorflow/contrib/tensorrt/segment/segment.cc b/tensorflow/contrib/tensorrt/segment/segment.cc
index 171ae41575..9f72f1490c 100644
--- a/tensorflow/contrib/tensorrt/segment/segment.cc
+++ b/tensorflow/contrib/tensorrt/segment/segment.cc
@@ -28,6 +28,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/types.h"
+#include "tensorflow/core/util/device_name_utils.h"
 
 namespace tensorflow {
 namespace tensorrt {
@@ -419,6 +420,11 @@ tensorflow::Status SegmentGraph(
   //    segment but are not eligible, using input/output_candidate_fn to
   //    determine the eligibilities;
   // 3. convert the segment into expected return format and return the result.
+  auto get_device_type = [](const string& device) {
+    DeviceNameUtils::ParsedName parsed_name;
+    DeviceNameUtils::ParseFullName(device, &parsed_name);
+    return parsed_name.type;
+  };
 
   // --------------------------------- Step 1 ---------------------------------
   auto graph = std::unique_ptr<SimpleGraph>(new SimpleGraph(tf_graph));
@@ -430,6 +436,11 @@ tensorflow::Status SegmentGraph(
   std::vector<UnionFind<SimpleNode*>> node_segments;
   for (int i = 0; i < graph->num_node_ids(); ++i) {
     SimpleNode* node = graph->FindNodeId(i);
+    const string requested_device =
+        get_device_type(node->tf_node()->requested_device());
+    const string assigned_device =
+        get_device_type(node->tf_node()->assigned_device_name());
+
     if (options.exclude_node_list.count(node->name()) != 0) {
       VLOG(1) << "Not a TF-TRT candidate, "
               << "(Op type: " << node->tf_node()->type_string() << "), "
@@ -438,10 +449,18 @@ tensorflow::Status SegmentGraph(
       unsupported_ops.emplace(node->tf_node()->type_string());
       num_unsupported_ops++;
       node = nullptr;
+    } else if (requested_device == "CPU" || assigned_device == "CPU") {
+      VLOG(1) << "Not a TF-TRT candidate, "
+              << "(Op type: " << node->tf_node()->type_string() << "), "
+              << "(Op name: " << node->name() << "), "
+              << "(Reason: node is assigned to CPU)";
+      unsupported_ops.emplace(node->tf_node()->type_string());
+      num_unsupported_ops++;
+      node = nullptr;
     } else {
       const Status status = candidate_fn(node->tf_node());
       if (!status.ok()) {
-        LOG(INFO) << "Not a TF-TRT candidate, "
+        VLOG(1) << "Not a TF-TRT candidate, "
                 << "(Op type: " << node->tf_node()->type_string() << "), "
                 << "(Op name: " << node->name() << "), "
                 << "(Reason: " << status << ")";
diff --git a/tensorflow/contrib/tensorrt/test/int_test.py b/tensorflow/contrib/tensorrt/test/int_test.py
index b5d69a799f..bb90c34168 100644
--- a/tensorflow/contrib/tensorrt/test/int_test.py
+++ b/tensorflow/contrib/tensorrt/test/int_test.py
@@ -53,7 +53,6 @@ class BiasaddMatMulTest(trt_test.TfTrtIntegrationTestBase):
       # int part
       input2 = array_ops.placeholder(
           dtype=dtypes.int32, shape=input_dims, name='input_int32')
-      #with g.device("/GPU:0"):
       b2 = self._ConstOp((4, 10), dtypes.int32)
       x2 = math_ops.matmul(input2, b2)
       b2 = self._ConstOp((10,), dtypes.int32)
@@ -84,13 +83,13 @@ class BiasaddMatMulTest(trt_test.TfTrtIntegrationTestBase):
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
-    return ["TRTEngineOp_0"]
+    return []
 
   def ShouldRunTest(self, run_params):
     """Whether to run the test."""
     # TODO(aaroey): Trt 4.0 forbids conversion for tensors with rank <3 in int8
     # mode, which is a bug. Re-enable this when trt library is fixed.
-    return not trt_test.IsQuantizationMode(run_params.precision_mode)
+    return not trt_test.IsQuantizationMode(run_params.precision_mode) #and not run_params.dynamic_engine
 
 
 if __name__ == "__main__":
-- 
GitLab


From 12a755183b32681281960050798bae02de4a58cd Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Fri, 21 Dec 2018 15:39:45 -0800
Subject: [PATCH 0486/1095] Clean up code

---
 .../contrib/tensorrt/convert/convert_graph.cc |  3 --
 .../contrib/tensorrt/convert/convert_nodes.cc |  8 ----
 tensorflow/contrib/tensorrt/test/int_test.py  | 43 +++++++------------
 3 files changed, 15 insertions(+), 39 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index 646451d802..fd31c356c1 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -1011,11 +1011,8 @@ tensorflow::Status ConvertAfterShapes(ConversionParams& params) {
   VLOG(1) << "Current cuda device is " << old_cuda_device;
   std::vector<Node*> engine_nodes;
   engine_nodes.resize(engine_segments.size());
-  LOG(INFO) << "here " << engine_segments.size();
   for (int i = 0; i < engine_segments.size(); ++i) {
     auto& engine = engine_segments.at(i);
-
-    LOG(INFO) << "here4 ";
     // Partition the workspace size by the average of node ratio and segment
     // graphdef size
     engine.max_workspace_size_bytes =
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 0d98da08b0..39f9e220e4 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -694,14 +694,6 @@ void ReorderCKtoKC(const TRT_ShapedWeights& iweights,
           ostrides);
       break;
     }
-    case tensorflow::DataType::DT_INT32: {
-      Reorder2(
-          {k, c}, static_cast<int const*>(iweights.GetValues()),
-          istrides,
-          static_cast<int*>(const_cast<void*>(oweights->GetValues())),
-          ostrides);
-      break;
-    }
     default:
       LOG(FATAL) << "Unsupported type in reorder expected fp32 or fp16 but got "
                  << DataTypeString(iweights.type_);
diff --git a/tensorflow/contrib/tensorrt/test/int_test.py b/tensorflow/contrib/tensorrt/test/int_test.py
index bb90c34168..b9636fa098 100644
--- a/tensorflow/contrib/tensorrt/test/int_test.py
+++ b/tensorflow/contrib/tensorrt/test/int_test.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Model script to test TF-TensorRT integration."""
+"""Test conversion of graphs involving INT32 tensors and operations."""
 
 from __future__ import absolute_import
 from __future__ import division
@@ -37,37 +37,24 @@ class BiasaddMatMulTest(trt_test.TfTrtIntegrationTestBase):
     return constant_op.constant(np.random.randn(*shape), dtype=dtype)
 
   def GetParams(self):
-    """Testing conversion of BiasAdd MatMul in TF-TRT conversion."""
-    # Note that tf.nn.bias_add supports up to 5 dimensions.
+    """Test exclusion of ops which are not supported in INT32 mode by TF-TRT"""
+    input_name = 'input'
+    output_name = 'output'
     input_dims = [100, 4]
     g = ops.Graph()
     with g.as_default():
-      # float part
-      # input1 = array_ops.placeholder(
-      #     dtype=dtypes.float32, shape=input_dims, name='input_float32')
-      # b1 = self._ConstOp((4, 10), dtypes.float32)
-      # x1 = math_ops.matmul(input1, b1)
-      # b1 = self._ConstOp((1, 10), dtypes.float32)
-      # x1 = x1 + b1
-
-      # int part
-      input2 = array_ops.placeholder(
-          dtype=dtypes.int32, shape=input_dims, name='input_int32')
-      b2 = self._ConstOp((4, 10), dtypes.int32)
-      x2 = math_ops.matmul(input2, b2)
-      b2 = self._ConstOp((10,), dtypes.int32)
-      x2 = nn.bias_add(x2, b2)
-
-      # combine
-      #y = x1 * x2
-
-      #out1 = array_ops.identity(x1, name='output_float32')
-      out2 = array_ops.identity(x2, name='output_int32')
+      x = array_ops.placeholder(
+          dtype=dtypes.int32, shape=input_dims, name=input_name)
+      b = self._ConstOp((4, 10), dtypes.int32)
+      x = math_ops.matmul(x, b)
+      b = self._ConstOp((10,), dtypes.int32)
+      x = nn.bias_add(x, b)
+      x = array_ops.identity(x, name=output_name)
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
-        input_names=['input_int32'],
-        input_dims=[[100, 4]],
-        output_names=['output_int32'],
+        input_names=[input_name],
+        input_dims=[input_dims],
+        output_names=[output_name],
         expected_output_dims=[(100, 10)])
 
   def GetConversionParams(self, run_params):
@@ -89,7 +76,7 @@ class BiasaddMatMulTest(trt_test.TfTrtIntegrationTestBase):
     """Whether to run the test."""
     # TODO(aaroey): Trt 4.0 forbids conversion for tensors with rank <3 in int8
     # mode, which is a bug. Re-enable this when trt library is fixed.
-    return not trt_test.IsQuantizationMode(run_params.precision_mode) #and not run_params.dynamic_engine
+    return not trt_test.IsQuantizationMode(run_params.precision_mode)
 
 
 if __name__ == "__main__":
-- 
GitLab


From f84227cac2abe03818cecd80ac02b86b8090e376 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Fri, 21 Dec 2018 15:41:01 -0800
Subject: [PATCH 0487/1095] Rename test

---
 tensorflow/contrib/tensorrt/BUILD                            | 2 +-
 .../contrib/tensorrt/test/{int_test.py => int32_test.py}     | 5 ++---
 2 files changed, 3 insertions(+), 4 deletions(-)
 rename tensorflow/contrib/tensorrt/test/{int_test.py => int32_test.py} (96%)

diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index e58a259a18..418e1564bf 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -493,7 +493,7 @@ cuda_py_tests(
         "test/const_broadcast_test.py",
         "test/conv2d_test.py",
         "test/identity_output_test.py",
-        "test/int_test.py",
+        "test/int32_test.py",
         "test/manual_test.py",
         "test/memory_alignment_test.py",
         "test/multi_connection_neighbor_engine_test.py",
diff --git a/tensorflow/contrib/tensorrt/test/int_test.py b/tensorflow/contrib/tensorrt/test/int32_test.py
similarity index 96%
rename from tensorflow/contrib/tensorrt/test/int_test.py
rename to tensorflow/contrib/tensorrt/test/int32_test.py
index b9636fa098..8f272efc91 100644
--- a/tensorflow/contrib/tensorrt/test/int_test.py
+++ b/tensorflow/contrib/tensorrt/test/int32_test.py
@@ -31,7 +31,7 @@ from tensorflow.python.ops import nn
 from tensorflow.python.platform import test
 
 
-class BiasaddMatMulTest(trt_test.TfTrtIntegrationTestBase):
+class ExcludeUnsupportedInt32Test(trt_test.TfTrtIntegrationTestBase):
 
   def _ConstOp(self, shape, dtype):
     return constant_op.constant(np.random.randn(*shape), dtype=dtype)
@@ -59,7 +59,7 @@ class BiasaddMatMulTest(trt_test.TfTrtIntegrationTestBase):
 
   def GetConversionParams(self, run_params):
     """Return a ConversionParams for test."""
-    conversion_params = super(BiasaddMatMulTest,
+    conversion_params = super(ExcludeUnsupportedInt32Test,
                               self).GetConversionParams(run_params)
     return conversion_params._replace(
         max_batch_size=100,
@@ -78,6 +78,5 @@ class BiasaddMatMulTest(trt_test.TfTrtIntegrationTestBase):
     # mode, which is a bug. Re-enable this when trt library is fixed.
     return not trt_test.IsQuantizationMode(run_params.precision_mode)
 
-
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From 1a73fdfa83bd50695a7d374d14a5cb3835d94d9e Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Fri, 21 Dec 2018 16:00:20 -0800
Subject: [PATCH 0488/1095] Add extra check incase segmenter does not exclude
 CPU in order to prevent segfault

---
 .../contrib/tensorrt/convert/convert_graph.cc      | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index fd31c356c1..cfde0049fc 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -355,12 +355,18 @@ tensorflow::Status GetEngineInfo(
     const auto& node_name = (*it)->name();
     if (segment_nodes.count(node_name) == 0) continue;
     auto node = *it;
-    // TODO: check for CPU device here
-    // If device is CPU, we should've caught that in the segmenter. Fall back here.
-
     auto node_device = node->requested_device();
     if (!node_device.empty()) {
-      segment_devices.insert(node_device);
+      // If device is CPU, we should've caught that in the segmenter.
+      DeviceNameUtils::ParsedName parsed_name;
+      DeviceNameUtils::ParseFullName(node_device, &parsed_name);
+      if (parsed_name.type == "CPU") {
+        LOG(WARNING) << "Node " << node->name() << " was assigned to the CPU "
+                     << "but did not get excluded by the segmenter. "
+                     << "Attempting to place on GPU.";
+      } else {
+        segment_devices.insert(node_device);
+      }
     } else {
       if (node->has_assigned_device_name()) {
         segment_devices.insert(node->assigned_device_name());
-- 
GitLab


From 399ae14f9829957e0b6eda6cc48f2a99133255e5 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Fri, 21 Dec 2018 16:01:39 -0800
Subject: [PATCH 0489/1095] Improve comment

---
 tensorflow/contrib/tensorrt/convert/convert_graph.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index cfde0049fc..d8356b31a0 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -357,7 +357,9 @@ tensorflow::Status GetEngineInfo(
     auto node = *it;
     auto node_device = node->requested_device();
     if (!node_device.empty()) {
-      // If device is CPU, we should've caught that in the segmenter.
+      // If device is CPU, we should've caught that in the segmenter. Don't add
+      // CPU to segment_device because that would cause a segfault in
+      // GetDeviceAndAllocator.
       DeviceNameUtils::ParsedName parsed_name;
       DeviceNameUtils::ParseFullName(node_device, &parsed_name);
       if (parsed_name.type == "CPU") {
-- 
GitLab


From 2434065e691a553a120aef6c16c76f29b55267a7 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Fri, 4 Jan 2019 10:40:57 -0800
Subject: [PATCH 0490/1095] Add TopK test. Specify type for topk indices
 output. Fix formatting

---
 tensorflow/contrib/tensorrt/BUILD             |  1 +
 .../contrib/tensorrt/convert/convert_nodes.cc |  8 ++-
 tensorflow/contrib/tensorrt/test/topk_test.py | 59 +++++++++++++++++++
 3 files changed, 66 insertions(+), 2 deletions(-)
 create mode 100644 tensorflow/contrib/tensorrt/test/topk_test.py

diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index 418e1564bf..b0f55fed44 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -501,6 +501,7 @@ cuda_py_tests(
         "test/quantization_test.py",
         "test/rank_two_test.py",
         "test/reshape_transpose_test.py",
+        "test/topk_test.py",
         "test/vgg_block_nchw_test.py",
         "test/vgg_block_test.py",
     ],
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 39f9e220e4..ee51c287ab 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -1728,8 +1728,9 @@ Status BinaryTensorOpTensor(OpConverterParams* params,
   TFAttrs attrs(node_def);
   nvinfer1::DataType dtype = attrs.get<nvinfer1::DataType>("T");
   if (dtype == nvinfer1::DataType::kINT32) {
-    return errors::Unimplemented(
-        "Binary op ", node_def.op(), " does not support INT32, at ", node_def.name());
+    return errors::Unimplemented("Binary op ", node_def.op(),
+                                 " does not support INT32, at ",
+                                 node_def.name());
   }
   if (params->validation_only) return Status::OK();
 
@@ -3605,6 +3606,9 @@ tensorflow::Status ConvertTopK(OpConverterParams* params) {
 
   nvinfer1::ITensor* output_value_tensor = layer->getOutput(0);
   nvinfer1::ITensor* output_indices_tensor = layer->getOutput(1);
+  // Tensor type for network output is not inferred. Indices should be INT32
+  // (default is float).
+  output_indices_tensor->setType(nvinfer1::DataType::kINT32);
   params->outputs->push_back(TRT_TensorOrWeights(output_value_tensor));
   params->outputs->push_back(TRT_TensorOrWeights(output_indices_tensor));
   return tensorflow::Status::OK();
diff --git a/tensorflow/contrib/tensorrt/test/topk_test.py b/tensorflow/contrib/tensorrt/test/topk_test.py
new file mode 100644
index 0000000000..cb539bbb93
--- /dev/null
+++ b/tensorflow/contrib/tensorrt/test/topk_test.py
@@ -0,0 +1,59 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Model script to test TF-TensorRT integration."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import nn_ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.platform import test
+
+
+class TopKTest(trt_test.TfTrtIntegrationTestBase):
+
+  def GetParams(self):
+    """Testing Top-K in TF-TRT conversion."""
+    dtype = dtypes.float32
+    input_name = "input"
+    input_dims = [100, 100]
+    k = 5
+    g = ops.Graph()
+    with g.as_default():
+      x = array_ops.placeholder(dtype=dtype, shape=input_dims, name=input_name)
+      values, indices = nn_ops.top_k(x, k)
+      values = array_ops.identity(values, name='output_values')
+      indices = array_ops.identity(indices, name='output_indices')
+    return trt_test.TfTrtIntegrationTestParams(
+        gdef=g.as_graph_def(),
+        input_names=[input_name],
+        input_dims=[input_dims],
+        output_names=['output_values', 'output_indices'],
+        expected_output_dims=[(100, k), (100, k)])
+
+  def ExpectedEnginesToBuild(self, run_params):
+    """Return the expected engines to build."""
+    return ["TRTEngineOp_0"]
+
+
+if __name__ == "__main__":
+  test.main()
-- 
GitLab


From 4d0a420c4b4d1fbe3e666bd377de2a40401177d2 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Fri, 4 Jan 2019 10:48:57 -0800
Subject: [PATCH 0491/1095] Fix clang-format and pylint

---
 tensorflow/contrib/tensorrt/convert/convert_nodes.cc | 6 ++++--
 tensorflow/contrib/tensorrt/test/topk_test.py        | 3 ---
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index ee51c287ab..9c0d79b810 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -3856,8 +3856,10 @@ tensorflow::Status ConvertSegmentToGraphDef(
       marker_nodes.insert(node_name);
       auto seg_node = segment_def->add_node();
       tensorflow::NodeDefBuilder builder(node_name, "Identity");
-      auto status = builder.Input(connection.inside_node_name, connection.inside_port, dtype)
-                        .Finalize(seg_node);
+      auto status =
+          builder
+              .Input(connection.inside_node_name, connection.inside_port, dtype)
+              .Finalize(seg_node);
       VLOG(1) << "Constructing output " << node_name << " for the edge "
               << connection.inside_node_name << ":" << connection.inside_port
               << " -> " << connection.outside_node_name << ":"
diff --git a/tensorflow/contrib/tensorrt/test/topk_test.py b/tensorflow/contrib/tensorrt/test/topk_test.py
index cb539bbb93..2edf7cb587 100644
--- a/tensorflow/contrib/tensorrt/test/topk_test.py
+++ b/tensorflow/contrib/tensorrt/test/topk_test.py
@@ -18,10 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import numpy as np
-
 from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
-from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import nn_ops
-- 
GitLab


From 0107c9155260f9aeb32d7660f7a9e8a8a5126850 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Fri, 4 Jan 2019 14:18:37 -0800
Subject: [PATCH 0492/1095] Remove segmenter rule against CPU ops.

---
 tensorflow/contrib/tensorrt/convert/convert_graph.cc |  2 ++
 tensorflow/contrib/tensorrt/segment/segment.cc       | 12 ------------
 tensorflow/contrib/tensorrt/test/int32_test.py       |  7 ++++---
 3 files changed, 6 insertions(+), 15 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index d8356b31a0..1ed6bbe5a3 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -371,6 +371,8 @@ tensorflow::Status GetEngineInfo(
       }
     } else {
       if (node->has_assigned_device_name()) {
+        // It appears that nodes will not have assigned devices at this point in
+        // execution.
         segment_devices.insert(node->assigned_device_name());
       } else {
         VLOG(2) << "Node " << node->name()
diff --git a/tensorflow/contrib/tensorrt/segment/segment.cc b/tensorflow/contrib/tensorrt/segment/segment.cc
index 9f72f1490c..6f2a7846d7 100644
--- a/tensorflow/contrib/tensorrt/segment/segment.cc
+++ b/tensorflow/contrib/tensorrt/segment/segment.cc
@@ -436,10 +436,6 @@ tensorflow::Status SegmentGraph(
   std::vector<UnionFind<SimpleNode*>> node_segments;
   for (int i = 0; i < graph->num_node_ids(); ++i) {
     SimpleNode* node = graph->FindNodeId(i);
-    const string requested_device =
-        get_device_type(node->tf_node()->requested_device());
-    const string assigned_device =
-        get_device_type(node->tf_node()->assigned_device_name());
 
     if (options.exclude_node_list.count(node->name()) != 0) {
       VLOG(1) << "Not a TF-TRT candidate, "
@@ -449,14 +445,6 @@ tensorflow::Status SegmentGraph(
       unsupported_ops.emplace(node->tf_node()->type_string());
       num_unsupported_ops++;
       node = nullptr;
-    } else if (requested_device == "CPU" || assigned_device == "CPU") {
-      VLOG(1) << "Not a TF-TRT candidate, "
-              << "(Op type: " << node->tf_node()->type_string() << "), "
-              << "(Op name: " << node->name() << "), "
-              << "(Reason: node is assigned to CPU)";
-      unsupported_ops.emplace(node->tf_node()->type_string());
-      num_unsupported_ops++;
-      node = nullptr;
     } else {
       const Status status = candidate_fn(node->tf_node());
       if (!status.ok()) {
diff --git a/tensorflow/contrib/tensorrt/test/int32_test.py b/tensorflow/contrib/tensorrt/test/int32_test.py
index 8f272efc91..50af816d8e 100644
--- a/tensorflow/contrib/tensorrt/test/int32_test.py
+++ b/tensorflow/contrib/tensorrt/test/int32_test.py
@@ -41,13 +41,14 @@ class ExcludeUnsupportedInt32Test(trt_test.TfTrtIntegrationTestBase):
     input_name = 'input'
     output_name = 'output'
     input_dims = [100, 4]
+    dtype = dtypes.int32
     g = ops.Graph()
     with g.as_default():
       x = array_ops.placeholder(
-          dtype=dtypes.int32, shape=input_dims, name=input_name)
-      b = self._ConstOp((4, 10), dtypes.int32)
+          dtype=dtype, shape=input_dims, name=input_name)
+      b = self._ConstOp((4, 10), dtype)
       x = math_ops.matmul(x, b)
-      b = self._ConstOp((10,), dtypes.int32)
+      b = self._ConstOp((10,), dtype)
       x = nn.bias_add(x, b)
       x = array_ops.identity(x, name=output_name)
     return trt_test.TfTrtIntegrationTestParams(
-- 
GitLab


From a5037695df5cc9eee60ea31fc1136c92786b57b1 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Fri, 4 Jan 2019 14:25:17 -0800
Subject: [PATCH 0493/1095] Update comment on fix to prevent segfault

---
 tensorflow/contrib/tensorrt/convert/convert_graph.cc | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index 1ed6bbe5a3..df4bc4777c 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -357,15 +357,15 @@ tensorflow::Status GetEngineInfo(
     auto node = *it;
     auto node_device = node->requested_device();
     if (!node_device.empty()) {
-      // If device is CPU, we should've caught that in the segmenter. Don't add
-      // CPU to segment_device because that would cause a segfault in
-      // GetDeviceAndAllocator.
+      // If device is CPU, treat as if no device was assigned. Don't add CPU to
+      // segment_device because that would cause a segfault in
+      // GetDeviceAndAllocator. This is because GetDeviceAndAllocator assumes
+      // any already set device is a GPU.
       DeviceNameUtils::ParsedName parsed_name;
       DeviceNameUtils::ParseFullName(node_device, &parsed_name);
       if (parsed_name.type == "CPU") {
-        LOG(WARNING) << "Node " << node->name() << " was assigned to the CPU "
-                     << "but did not get excluded by the segmenter. "
-                     << "Attempting to place on GPU.";
+        VLOG(1) << "Node " << node->name() << " was assigned to the CPU. "
+                << "Attempting to place on GPU.";
       } else {
         segment_devices.insert(node_device);
       }
-- 
GitLab


From 0fb8e5f9c0149c4cfaf8ec41b1293a947bc91895 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 9 Jan 2019 14:52:30 -0800
Subject: [PATCH 0494/1095] This change makes errors more informative in case
 of functions. We now report two additional things: 1. The line number/file
 name of the culprit node inside the function. 2. The function calling stack

For eg, consider the following code:

import tensorflow as tf
tf.enable_eager_execution()

@tf.contrib.eager.defun
def fn3(x):
  return x + 2

@tf.contrib.eager.defun
def fn2(x):
  tf.assert_equal(fn3(x), 3)
  return 2

@tf.contrib.eager.defun
def fn(x):
  return fn2(x)

@tf.contrib.eager.defun
def main(argv):
  x = fn(2)
  print(x)

if __name__ == '__main__':
  tf.app.run()

In this, the error message would contain the following:
InvalidArgumentError: assertion failed: [] [Condition x == y did not hold element-wise:] [x (PartitionedCall:0) = ] [4] [y (assert_equal/y:0) = ] [3]
	 [[node assert_equal/Assert/Assert (defined at filename:line_num) ]]
	 [[StatefulPartitionedCall]]
	 [[StatefulPartitionedCall]] [Op:StatefulPartitionedCall]

Function calling stack:
main -> fn -> fn2

PiperOrigin-RevId: 228593653
---
 .../core/kernels/partitioned_function_ops.cc  |  9 ++-
 tensorflow/core/lib/core/errors.h             |  4 ++
 tensorflow/python/eager/function.py           | 61 ++++++++++++++++---
 tensorflow/python/eager/function_test.py      | 24 ++++++++
 .../python/framework/error_interpolation.py   |  8 ++-
 5 files changed, 92 insertions(+), 14 deletions(-)

diff --git a/tensorflow/core/kernels/partitioned_function_ops.cc b/tensorflow/core/kernels/partitioned_function_ops.cc
index e493dca188..cadb83d8cf 100644
--- a/tensorflow/core/kernels/partitioned_function_ops.cc
+++ b/tensorflow/core/kernels/partitioned_function_ops.cc
@@ -20,6 +20,7 @@ limitations under the License.
 #include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/framework/types.pb.h"
 #include "tensorflow/core/grappler/optimizers/meta_optimizer.h"
+#include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/protobuf/config.pb.h"
 #include "tensorflow/core/protobuf/rewriter_config.pb.h"
 #include "tensorflow/core/util/ptr_util.h"
@@ -213,10 +214,14 @@ class PartitionedCallOp : public AsyncOpKernel {
     run_opts.rendezvous = rendez;
 
     std::vector<Tensor>* rets = new std::vector<Tensor>;
+    const string& func_name = func_.name();
     lib->Run(run_opts, handle, inputs, rets,
-             [rets, rendez, done, ctx](const Status& status) {
+             [rets, rendez, done, ctx, func_name](const Status& status) {
                if (!status.ok()) {
-                 ctx->SetStatus(status);
+                 const string function_and_msg =
+                     strings::StrCat(errors::FormatFunctionForError(func_name),
+                                     " ", status.error_message());
+                 ctx->SetStatus(Status(status.code(), function_and_msg));
                } else {
                  for (int i = 0; i < rets->size(); ++i) {
                    ctx->set_output(i, (*rets)[i]);
diff --git a/tensorflow/core/lib/core/errors.h b/tensorflow/core/lib/core/errors.h
index d5cbe6c616..4815f7c2cc 100644
--- a/tensorflow/core/lib/core/errors.h
+++ b/tensorflow/core/lib/core/errors.h
@@ -150,6 +150,10 @@ string FormatColocationNodeForError(const T& names) {
       });
 }
 
+inline string FormatFunctionForError(const string& name) {
+  return strings::StrCat("{{function_node ", name, "}}");
+}
+
 // The CanonicalCode() for non-errors.
 using ::tensorflow::error::OK;
 
diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index 67c633726f..83cd140158 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -41,6 +41,8 @@ from tensorflow.python.framework import c_api_util
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import device as pydev
 from tensorflow.python.framework import dtypes as dtypes_module
+from tensorflow.python.framework import error_interpolation
+from tensorflow.python.framework import errors
 from tensorflow.python.framework import func_graph as func_graph_module
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_spec
@@ -131,6 +133,46 @@ def _parse_func_attrs(attributes):
   return attrs
 
 
+class _InterpolateFunctionError(object):
+  """Context Manager that interpolates the exception from 'top_level_func'."""
+
+  def __init__(self, top_level_func):
+    self._func = top_level_func
+
+  def __enter__(self):
+    pass
+
+  def __exit__(self, typ, exc, tb):
+    if not exc or not isinstance(exc, errors.OpError):
+      return False
+    message = compat.as_text(exc.message)
+    _, tags = error_interpolation.parse_message(message)
+    g = None
+    func_stack = []
+    # pylint: disable=protected-access
+    for t in tags:
+      if t.type == "function_node":
+        if t.name == compat.as_str(self._func.name):
+          g = self._func._graph
+        elif g:
+          next_func = g._get_function(t.name)
+          if next_func is not None and isinstance(next_func,
+                                                  _EagerDefinedFunction):
+            g = next_func._graph
+        if g:
+          func_stack.append(g.name)
+        else:
+          func_stack.append("<unknown>")
+    # pylint: enable=protected-access
+    if g:
+      message = error_interpolation.interpolate(message, g)
+      message += "\n\nFunction call stack:\n"
+      message += " -> ".join(func_stack)
+      message += "\n"
+      exc._message = message  # pylint: disable=protected-access
+    return False
+
+
 def _forward_name(n):
   """The name of a generated forward defun named n."""
   return "__forward_%s_%s" % (n, ops.uid())
@@ -279,13 +321,14 @@ class _EagerDefinedFunction(object):
             "Arguments and signature arguments do not match: %s %s " %
             (len(args), len(list(self.signature.input_arg))))
       function_call_options = ctx.get_function_call_options()
-      outputs = functional_ops.partitioned_call(
-          args=args,
-          f=self,
-          tout=self._output_types,
-          executing_eagerly=executing_eagerly,
-          config=function_call_options.config_proto_serialized,
-          executor_type=function_call_options.executor_type)
+      with _InterpolateFunctionError(self):
+        outputs = functional_ops.partitioned_call(
+            args=args,
+            f=self,
+            tout=self._output_types,
+            executing_eagerly=executing_eagerly,
+            config=function_call_options.config_proto_serialized,
+            executor_type=function_call_options.executor_type)
 
     if executing_eagerly:
       return outputs
@@ -401,8 +444,8 @@ class Function(object):
     """
     return self._call_flat(
         (t for t in nest.flatten((args, kwargs))
-         if isinstance(
-             t, (ops.Tensor, resource_variable_ops.ResourceVariable))))
+         if isinstance(t, (ops.Tensor,
+                           resource_variable_ops.ResourceVariable))))
 
   def _call_flat(self, args):
     """Executes the wrapped function.
diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py
index 55a9cc4e92..1966b259bf 100644
--- a/tensorflow/python/eager/function_test.py
+++ b/tensorflow/python/eager/function_test.py
@@ -45,6 +45,7 @@ from tensorflow.python.framework import test_util
 from tensorflow.python.keras.engine import training as keras_training
 from tensorflow.python.layers import convolutional
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import check_ops
 from tensorflow.python.ops import clip_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import init_ops
@@ -2063,6 +2064,29 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
     # function itself is not involved in a reference cycle.
     self.assertIs(None, weak_fn())
 
+  def testFunctionStackInErrorMessage(self):
+
+    @def_function.function()
+    def fn3(x):
+      return x + 2
+
+    @def_function.function()
+    def fn2(x):
+      check_ops.assert_equal(fn3(x), 3)
+      return 2
+
+    @def_function.function()
+    def fn(x):
+      return fn2(x)
+
+    try:
+      fn(2)
+      self.assertFail()
+    except errors.InvalidArgumentError as e:
+      self.assertIn('fn -> fn2', e.message)
+      self.assertIn('node assert_equal/Assert/Assert (defined at', e.message)
+      self.assertNotIn('fn3', e.message)
+
 
 if __name__ == '__main__':
   ops.enable_eager_execution(
diff --git a/tensorflow/python/framework/error_interpolation.py b/tensorflow/python/framework/error_interpolation.py
index 5e1bed8e0e..af83b70a46 100644
--- a/tensorflow/python/framework/error_interpolation.py
+++ b/tensorflow/python/framework/error_interpolation.py
@@ -31,7 +31,7 @@ import six
 
 from tensorflow.python.util import tf_stack
 
-_NAME_REGEX = r"[A-Za-z0-9.][A-Za-z0-9_.\-/]*?"
+_NAME_REGEX = r"[A-Za-z0-9_.][A-Za-z0-9_.\-/]*?"
 _TAG_REGEX = r"{{{{({name}) ({name})}}}}".format(name=_NAME_REGEX)
 _INTERPOLATION_REGEX = r"^(.*?)({tag})".format(tag=_TAG_REGEX)
 _INTERPOLATION_PATTERN = re.compile(_INTERPOLATION_REGEX, re.DOTALL)
@@ -45,7 +45,7 @@ _BAD_FILE_SUBSTRINGS = [
 ]
 
 
-def _parse_message(message):
+def parse_message(message):
   """Parses the message.
 
   Splits the message into separators and tags. Tags are named tuples
@@ -376,7 +376,7 @@ def interpolate(error_message, graph):
   Returns:
     The string with tags of the form {{type name}} interpolated.
   """
-  seps, tags = _parse_message(error_message)
+  seps, tags = parse_message(error_message)
   subs = []
   end_msg = collections.defaultdict(list)
   tagged_ops = []
@@ -404,6 +404,8 @@ def interpolate(error_message, graph):
         msg = "node %s%s placed on device %s " % (
             ops[0].name, field_dict["defined_at"], field_dict["devices"])
         end_msg["colocations"].append(field_dict["devs_and_colocs"])
+    if tag.type == "function_node":
+      msg = ""
     subs.append(msg)
 
   if "source_nodes" in end_msg:
-- 
GitLab


From 00a3be761b620761a71effb793e2440096f1eebd Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 9 Jan 2019 14:55:31 -0800
Subject: [PATCH 0495/1095] Add LinearOperator.inverse method.

PiperOrigin-RevId: 228594130
---
 .../linalg/linear_operator_algebra_test.py    |  48 ++++++
 .../linalg/linear_operator_block_diag_test.py |  38 +++--
 .../linalg/linear_operator_diag_test.py       |   9 +-
 .../linalg/linear_operator_identity_test.py   |  24 ++-
 .../linalg/linear_operator_kronecker_test.py  |  37 +++--
 .../linalg/linear_operator_zeros_test.py      |   3 +-
 .../ops/linalg/inverse_registrations.py       | 108 +++++++++++++
 tensorflow/python/ops/linalg/linalg.py        |   1 +
 .../python/ops/linalg/linear_operator.py      |  25 +++
 .../ops/linalg/linear_operator_algebra.py     |  71 +++++++++
 .../ops/linalg/linear_operator_test_util.py   |  25 ++-
 ...w.linalg.-linear-operator-block-diag.pbtxt |   4 +
 ...ow.linalg.-linear-operator-circulant.pbtxt |   4 +
 ...linalg.-linear-operator-circulant2-d.pbtxt |   4 +
 ...linalg.-linear-operator-circulant3-d.pbtxt |   4 +
 ....linalg.-linear-operator-composition.pbtxt |   4 +
 ...sorflow.linalg.-linear-operator-diag.pbtxt |   4 +
 ....linalg.-linear-operator-full-matrix.pbtxt |   4 +
 ...low.linalg.-linear-operator-identity.pbtxt |   4 +
 ...ow.linalg.-linear-operator-inversion.pbtxt | 142 ++++++++++++++++++
 ...ow.linalg.-linear-operator-kronecker.pbtxt |   4 +
 ...alg.-linear-operator-low-rank-update.pbtxt |   4 +
 ...lg.-linear-operator-lower-triangular.pbtxt |   4 +
 ...alg.-linear-operator-scaled-identity.pbtxt |   4 +
 ...orflow.linalg.-linear-operator-zeros.pbtxt |   4 +
 .../tensorflow.linalg.-linear-operator.pbtxt  |   4 +
 .../api/golden/v1/tensorflow.linalg.pbtxt     |   4 +
 ...w.linalg.-linear-operator-block-diag.pbtxt |   4 +
 ...ow.linalg.-linear-operator-circulant.pbtxt |   4 +
 ...linalg.-linear-operator-circulant2-d.pbtxt |   4 +
 ...linalg.-linear-operator-circulant3-d.pbtxt |   4 +
 ....linalg.-linear-operator-composition.pbtxt |   4 +
 ...sorflow.linalg.-linear-operator-diag.pbtxt |   4 +
 ....linalg.-linear-operator-full-matrix.pbtxt |   4 +
 ...low.linalg.-linear-operator-identity.pbtxt |   4 +
 ...ow.linalg.-linear-operator-inversion.pbtxt | 142 ++++++++++++++++++
 ...ow.linalg.-linear-operator-kronecker.pbtxt |   4 +
 ...alg.-linear-operator-low-rank-update.pbtxt |   4 +
 ...lg.-linear-operator-lower-triangular.pbtxt |   4 +
 ...alg.-linear-operator-scaled-identity.pbtxt |   4 +
 ...orflow.linalg.-linear-operator-zeros.pbtxt |   4 +
 .../tensorflow.linalg.-linear-operator.pbtxt  |   4 +
 .../api/golden/v2/tensorflow.linalg.pbtxt     |   4 +
 43 files changed, 761 insertions(+), 32 deletions(-)
 create mode 100644 tensorflow/python/ops/linalg/inverse_registrations.py
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-inversion.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-inversion.pbtxt

diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_algebra_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_algebra_test.py
index 8e296c026c..ec78a3ffe0 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_algebra_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_algebra_test.py
@@ -30,6 +30,8 @@ _CHOLESKY_DECOMPS = linear_operator_algebra._CHOLESKY_DECOMPS
 _MATMUL = linear_operator_algebra._MATMUL
 _registered_cholesky = linear_operator_algebra._registered_cholesky
 _registered_matmul = linear_operator_algebra._registered_matmul
+_INVERSES = linear_operator_algebra._INVERSES
+_registered_inverse = linear_operator_algebra._registered_inverse
 # pylint: enable=protected-access
 
 
@@ -129,5 +131,51 @@ class MatmulTest(test.TestCase):
       self.assertEqual(v, _registered_matmul(k[0], k[1]))
 
 
+class InverseTest(test.TestCase):
+
+  def testRegistration(self):
+
+    class CustomLinOp(linear_operator.LinearOperator):
+
+      def _matmul(self, a):
+        pass
+
+      def _shape(self):
+        return tensor_shape.TensorShape([1, 1])
+
+      def _shape_tensor(self):
+        pass
+
+    # Register Inverse to a lambda that spits out the name parameter
+    @linear_operator_algebra.RegisterInverse(CustomLinOp)
+    def _inverse(a):  # pylint: disable=unused-argument,unused-variable
+      return "OK"
+
+    with self.assertRaisesRegexp(ValueError, "singular"):
+      CustomLinOp(dtype=None, is_non_singular=False).inverse()
+
+    self.assertEqual("OK", CustomLinOp(
+        dtype=None, is_non_singular=True).inverse())
+
+  def testRegistrationFailures(self):
+
+    class CustomLinOp(linear_operator.LinearOperator):
+      pass
+
+    with self.assertRaisesRegexp(TypeError, "must be callable"):
+      linear_operator_algebra.RegisterInverse(CustomLinOp)("blah")
+
+    # First registration is OK
+    linear_operator_algebra.RegisterInverse(CustomLinOp)(lambda a: None)
+
+    # Second registration fails
+    with self.assertRaisesRegexp(ValueError, "has already been registered"):
+      linear_operator_algebra.RegisterInverse(CustomLinOp)(lambda a: None)
+
+  def testExactRegistrationsAllMatch(self):
+    for (k, v) in _INVERSES.items():
+      self.assertEqual(v, _registered_inverse(k[0]))
+
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_block_diag_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_block_diag_test.py
index f0cc5d709f..96e6e3c04c 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_block_diag_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_block_diag_test.py
@@ -155,20 +155,38 @@ class SquareLinearOperatorBlockDiagTest(
         is_self_adjoint=True,
     )
     cholesky_factor = operator.cholesky()
-    self.assertTrue(isinstance(
+    self.assertIsInstance(
         cholesky_factor,
-        block_diag.LinearOperatorBlockDiag))
+        block_diag.LinearOperatorBlockDiag)
     self.assertEqual(2, len(cholesky_factor.operators))
-    self.assertTrue(
-        isinstance(
-            cholesky_factor.operators[0],
-            lower_triangular.LinearOperatorLowerTriangular)
+    self.assertIsInstance(
+        cholesky_factor.operators[0],
+        lower_triangular.LinearOperatorLowerTriangular)
+    self.assertIsInstance(
+        cholesky_factor.operators[1],
+        lower_triangular.LinearOperatorLowerTriangular
     )
-    self.assertTrue(
-        isinstance(
-            cholesky_factor.operators[1],
-            lower_triangular.LinearOperatorLowerTriangular)
+
+  def test_block_diag_inverse_type(self):
+    matrix = [[1., 0.], [0., 1.]]
+    operator = block_diag.LinearOperatorBlockDiag(
+        [
+            linalg.LinearOperatorFullMatrix(
+                matrix,
+                is_non_singular=True,
+            ),
+            linalg.LinearOperatorFullMatrix(
+                matrix,
+                is_non_singular=True,
+            ),
+        ],
+        is_non_singular=True,
     )
+    inverse = operator.inverse()
+    self.assertIsInstance(
+        inverse,
+        block_diag.LinearOperatorBlockDiag)
+    self.assertEqual(2, len(inverse.operators))
 
   def test_is_non_singular_auto_set(self):
     # Matrix with two positive eigenvalues, 11 and 8.
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_diag_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_diag_test.py
index dcbc0dd7c9..4d7a31be87 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_diag_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_diag_test.py
@@ -194,9 +194,12 @@ class LinearOperatorDiagTest(
         is_positive_definite=True,
         is_self_adjoint=True,
     )
-    self.assertTrue(isinstance(
-        operator.cholesky(),
-        linalg.LinearOperatorDiag))
+    self.assertIsInstance(operator.cholesky(), linalg.LinearOperatorDiag)
+
+  def test_diag_inverse_type(self):
+    diag = [1., 3., 5., 8.]
+    operator = linalg.LinearOperatorDiag(diag, is_non_singular=True)
+    self.assertIsInstance(operator.inverse(), linalg.LinearOperatorDiag)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py
index 2da5e712d7..14b5228bca 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py
@@ -265,9 +265,14 @@ class LinearOperatorIdentityTest(
         is_positive_definite=True,
         is_self_adjoint=True,
     )
-    self.assertTrue(isinstance(
-        operator.cholesky(),
-        linalg_lib.LinearOperatorIdentity))
+    self.assertIsInstance(
+        operator.cholesky(), linalg_lib.LinearOperatorIdentity)
+
+  def test_identity_inverse_type(self):
+    operator = linalg_lib.LinearOperatorIdentity(
+        num_rows=2, is_non_singular=True)
+    self.assertIsInstance(
+        operator.inverse(), linalg_lib.LinearOperatorIdentity)
 
 
 class LinearOperatorScaledIdentityTest(
@@ -491,10 +496,19 @@ class LinearOperatorScaledIdentityTest(
         is_positive_definite=True,
         is_self_adjoint=True,
     )
-    self.assertTrue(isinstance(
+    self.assertIsInstance(
         operator.cholesky(),
-        linalg_lib.LinearOperatorScaledIdentity))
+        linalg_lib.LinearOperatorScaledIdentity)
 
+  def test_scaled_identity_inverse_type(self):
+    operator = linalg_lib.LinearOperatorScaledIdentity(
+        num_rows=2,
+        multiplier=3.,
+        is_non_singular=True,
+    )
+    self.assertIsInstance(
+        operator.inverse(),
+        linalg_lib.LinearOperatorScaledIdentity)
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_kronecker_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_kronecker_test.py
index 513b246803..54ccc0c5f6 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_kronecker_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_kronecker_test.py
@@ -100,7 +100,7 @@ class SquareLinearOperatorKroneckerTest(
 
   @property
   def _tests_to_skip(self):
-    return ["det", "solve", "solve_with_broadcast"]
+    return ["det", "inverse", "solve", "solve_with_broadcast"]
 
   def _operator_and_matrix(
       self, build_info, dtype, use_placeholder,
@@ -211,20 +211,33 @@ class SquareLinearOperatorKroneckerTest(
         is_self_adjoint=True,
     )
     cholesky_factor = operator.cholesky()
-    self.assertTrue(isinstance(
+    self.assertIsInstance(
         cholesky_factor,
-        kronecker.LinearOperatorKronecker))
+        kronecker.LinearOperatorKronecker)
     self.assertEqual(2, len(cholesky_factor.operators))
-    self.assertTrue(
-        isinstance(
-            cholesky_factor.operators[0],
-            lower_triangular.LinearOperatorLowerTriangular)
-    )
-    self.assertTrue(
-        isinstance(
-            cholesky_factor.operators[1],
-            lower_triangular.LinearOperatorLowerTriangular)
+    self.assertIsInstance(
+        cholesky_factor.operators[0],
+        lower_triangular.LinearOperatorLowerTriangular)
+    self.assertIsInstance(
+        cholesky_factor.operators[1],
+        lower_triangular.LinearOperatorLowerTriangular)
+
+  def test_kronecker_inverse_type(self):
+    matrix = [[1., 0.], [0., 1.]]
+    operator = kronecker.LinearOperatorKronecker(
+        [
+            linalg.LinearOperatorFullMatrix(
+                matrix, is_non_singular=True),
+            linalg.LinearOperatorFullMatrix(
+                matrix, is_non_singular=True),
+        ],
+        is_non_singular=True,
     )
+    inverse = operator.inverse()
+    self.assertIsInstance(
+        inverse,
+        kronecker.LinearOperatorKronecker)
+    self.assertEqual(2, len(inverse.operators))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_zeros_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_zeros_test.py
index eb0b8ef127..10651d3c8a 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_zeros_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_zeros_test.py
@@ -36,7 +36,8 @@ class LinearOperatorZerosTest(
 
   @property
   def _tests_to_skip(self):
-    return ["cholesky", "log_abs_det", "solve", "solve_with_broadcast"]
+    return [
+        "cholesky", "log_abs_det", "inverse", "solve", "solve_with_broadcast"]
 
   @property
   def _operator_build_infos(self):
diff --git a/tensorflow/python/ops/linalg/inverse_registrations.py b/tensorflow/python/ops/linalg/inverse_registrations.py
new file mode 100644
index 0000000000..b89bdd240e
--- /dev/null
+++ b/tensorflow/python/ops/linalg/inverse_registrations.py
@@ -0,0 +1,108 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Registrations for LinearOperator.inverse."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.ops.linalg import linear_operator
+from tensorflow.python.ops.linalg import linear_operator_algebra
+from tensorflow.python.ops.linalg import linear_operator_block_diag
+from tensorflow.python.ops.linalg import linear_operator_circulant
+from tensorflow.python.ops.linalg import linear_operator_diag
+from tensorflow.python.ops.linalg import linear_operator_identity
+from tensorflow.python.ops.linalg import linear_operator_inversion
+from tensorflow.python.ops.linalg import linear_operator_kronecker
+
+
+# By default, return LinearOperatorInversion which switched the .matmul
+# and .solve methods.
+@linear_operator_algebra.RegisterInverse(linear_operator.LinearOperator)
+def _inverse_linear_operator(linop):
+  return linear_operator_inversion.LinearOperatorInversion(
+      linop,
+      is_non_singular=linop.is_non_singular,
+      is_self_adjoint=linop.is_self_adjoint,
+      is_positive_definite=linop.is_positive_definite,
+      is_square=linop.is_square)
+
+
+@linear_operator_algebra.RegisterInverse(
+    linear_operator_diag.LinearOperatorDiag)
+def _inverse_diag(diag_operator):
+  return linear_operator_diag.LinearOperatorDiag(
+      1. / diag_operator.diag,
+      is_non_singular=diag_operator.is_non_singular,
+      is_self_adjoint=diag_operator.is_self_adjoint,
+      is_positive_definite=diag_operator.is_positive_definite,
+      is_square=True)
+
+
+@linear_operator_algebra.RegisterInverse(
+    linear_operator_identity.LinearOperatorIdentity)
+def _inverse_identity(identity_operator):
+  return identity_operator
+
+
+@linear_operator_algebra.RegisterInverse(
+    linear_operator_identity.LinearOperatorScaledIdentity)
+def _inverse_scaled_identity(identity_operator):
+  return linear_operator_identity.LinearOperatorScaledIdentity(
+      num_rows=identity_operator._num_rows,  # pylint: disable=protected-access
+      multiplier=1. / identity_operator.multiplier,
+      is_non_singular=identity_operator.is_non_singular,
+      is_self_adjoint=True,
+      is_positive_definite=identity_operator.is_positive_definite,
+      is_square=True)
+
+
+@linear_operator_algebra.RegisterInverse(
+    linear_operator_block_diag.LinearOperatorBlockDiag)
+def _inverse_block_diag(block_diag_operator):
+    # We take the inverse of each block on the diagonal.
+  return linear_operator_block_diag.LinearOperatorBlockDiag(
+      operators=[
+          operator.inverse() for operator in block_diag_operator.operators],
+      is_non_singular=block_diag_operator.is_non_singular,
+      is_self_adjoint=block_diag_operator.is_self_adjoint,
+      is_positive_definite=block_diag_operator.is_positive_definite,
+      is_square=True)
+
+
+@linear_operator_algebra.RegisterInverse(
+    linear_operator_kronecker.LinearOperatorKronecker)
+def _inverse_kronecker(kronecker_operator):
+    # Inverse decomposition of a Kronecker product is the Kronecker product
+    # of inverse decompositions.
+  return linear_operator_kronecker.LinearOperatorKronecker(
+      operators=[
+          operator.inverse() for operator in kronecker_operator.operators],
+      is_non_singular=kronecker_operator.is_non_singular,
+      is_self_adjoint=kronecker_operator.is_self_adjoint,
+      is_positive_definite=kronecker_operator.is_positive_definite,
+      is_square=True)
+
+
+@linear_operator_algebra.RegisterInverse(
+    linear_operator_circulant.LinearOperatorCirculant)
+def _inverse_circulant(circulant_operator):
+  # Inverting the spectrum is sufficient to get the inverse.
+  return linear_operator_circulant.LinearOperatorCirculant(
+      spectrum=1. / circulant_operator.spectrum,
+      is_non_singular=circulant_operator.is_non_singular,
+      is_self_adjoint=circulant_operator.is_self_adjoint,
+      is_positive_definite=circulant_operator.is_positive_definite,
+      is_square=True)
diff --git a/tensorflow/python/ops/linalg/linalg.py b/tensorflow/python/ops/linalg/linalg.py
index ac4fd4ebc6..eebe741337 100644
--- a/tensorflow/python/ops/linalg/linalg.py
+++ b/tensorflow/python/ops/linalg/linalg.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 # go/tf-wildcard-import
 # pylint: disable=wildcard-import,unused-import
 from tensorflow.python.ops.linalg import cholesky_registrations as _cholesky_registrations
+from tensorflow.python.ops.linalg import inverse_registrations as _inverse_registrations
 from tensorflow.python.ops.linalg import linear_operator_algebra as _linear_operator_algebra
 from tensorflow.python.ops.linalg import matmul_registrations as _matmul_registrations
 from tensorflow.python.ops.linalg.linalg_impl import *
diff --git a/tensorflow/python/ops/linalg/linear_operator.py b/tensorflow/python/ops/linalg/linear_operator.py
index 6be81f4b34..4c99e86dc5 100644
--- a/tensorflow/python/ops/linalg/linear_operator.py
+++ b/tensorflow/python/ops/linalg/linear_operator.py
@@ -847,6 +847,31 @@ class LinearOperator(object):
 
       return self._solvevec(rhs, adjoint=adjoint)
 
+  def inverse(self, name="inverse"):
+    """Returns the Inverse of this `LinearOperator`.
+
+    Given `A` representing this `LinearOperator`, return a `LinearOperator`
+    representing `A^-1`.
+
+    Args:
+      name: A name scope to use for ops added by this method.
+
+    Returns:
+      `LinearOperator` representing inverse of this matrix.
+
+    Raises:
+      ValueError: When the `LinearOperator` is not hinted to be `non_singular`.
+    """
+    if self.is_square is False:  # pylint: disable=g-bool-id-comparison
+      raise ValueError("Cannot take the Inverse: This operator represents "
+                       "a non square matrix.")
+    if self.is_non_singular is False:  # pylint: disable=g-bool-id-comparison
+      raise ValueError("Cannot take the Inverse: This operator represents "
+                       "a singular matrix.")
+
+    with self._name_scope(name):
+      return linear_operator_algebra.inverse(self)
+
   def cholesky(self, name="cholesky"):
     """Returns a Cholesky factor as a `LinearOperator`.
 
diff --git a/tensorflow/python/ops/linalg/linear_operator_algebra.py b/tensorflow/python/ops/linalg/linear_operator_algebra.py
index 7b99066e4c..c1513fdb38 100644
--- a/tensorflow/python/ops/linalg/linear_operator_algebra.py
+++ b/tensorflow/python/ops/linalg/linear_operator_algebra.py
@@ -27,6 +27,7 @@ from tensorflow.python.util import tf_inspect
 
 _CHOLESKY_DECOMPS = {}
 _MATMUL = {}
+_INVERSES = {}
 
 
 def _registered_function(type_list, registry):
@@ -55,6 +56,11 @@ def _registered_matmul(type_a, type_b):
   return _registered_function([type_a, type_b], _MATMUL)
 
 
+def _registered_inverse(type_a):
+  """Get the Cholesky function registered for class a."""
+  return _registered_function([type_a], _INVERSES)
+
+
 def cholesky(lin_op_a, name=None):
   """Get the Cholesky factor associated to lin_op_a.
 
@@ -103,6 +109,29 @@ def matmul(lin_op_a, lin_op_b, name=None):
     return matmul_fn(lin_op_a, lin_op_b)
 
 
+def inverse(lin_op_a, name=None):
+  """Get the Inverse associated to lin_op_a.
+
+  Args:
+    lin_op_a: The LinearOperator to decompose.
+    name: Name to use for this operation.
+
+  Returns:
+    A LinearOperator that represents the inverse of `lin_op_a`.
+
+  Raises:
+    NotImplementedError: If no Inverse method is defined for the LinearOperator
+      type of `lin_op_a`.
+  """
+  inverse_fn = _registered_inverse(type(lin_op_a))
+  if inverse_fn is None:
+    raise ValueError("No inverse registered for {}".format(
+        type(lin_op_a)))
+
+  with ops.name_scope(name, "Inverse"):
+    return inverse_fn(lin_op_a)
+
+
 class RegisterCholesky(object):
   """Decorator to register a Cholesky implementation function.
 
@@ -189,3 +218,45 @@ class RegisterMatmul(object):
           self._key[1].__name__))
     _MATMUL[self._key] = matmul_fn
     return matmul_fn
+
+
+class RegisterInverse(object):
+  """Decorator to register an Inverse implementation function.
+
+  Usage:
+
+  @linear_operator_algebra.RegisterInverse(lin_op.LinearOperatorIdentity)
+  def _inverse_identity(lin_op_a):
+    # Return the identity matrix.
+  """
+
+  def __init__(self, lin_op_cls_a):
+    """Initialize the LinearOperator registrar.
+
+    Args:
+      lin_op_cls_a: the class of the LinearOperator to decompose.
+    """
+    self._key = (lin_op_cls_a,)
+
+  def __call__(self, inverse_fn):
+    """Perform the Inverse registration.
+
+    Args:
+      inverse_fn: The function to use for the Inverse.
+
+    Returns:
+      inverse_fn
+
+    Raises:
+      TypeError: if inverse_fn is not a callable.
+      ValueError: if a Inverse function has already been registered for
+        the given argument classes.
+    """
+    if not callable(inverse_fn):
+      raise TypeError(
+          "inverse_fn must be callable, received: {}".format(inverse_fn))
+    if self._key in _INVERSES:
+      raise ValueError("Inverse({}) has already been registered to: {}".format(
+          self._key[0].__name__, _INVERSES[self._key]))
+    _INVERSES[self._key] = inverse_fn
+    return inverse_fn
diff --git a/tensorflow/python/ops/linalg/linear_operator_test_util.py b/tensorflow/python/ops/linalg/linear_operator_test_util.py
index e50f572b5f..a957c84dc1 100644
--- a/tensorflow/python/ops/linalg/linear_operator_test_util.py
+++ b/tensorflow/python/ops/linalg/linear_operator_test_util.py
@@ -336,6 +336,22 @@ class LinearOperatorDerivedClassTest(test.TestCase):
     self._skip_if_tests_to_skip_contains("solve_with_broadcast")
     self._test_solve(with_batch=False)
 
+  def _test_inverse(self):
+    for use_placeholder in self._use_placeholder_options:
+      for build_info in self._operator_build_infos:
+        for dtype in self._dtypes_to_test:
+          with self.session(graph=ops.Graph()) as sess:
+            sess.graph.seed = random_seed.DEFAULT_GRAPH_SEED
+            operator, mat = self._operator_and_matrix(
+                build_info, dtype, use_placeholder=use_placeholder)
+            op_inverse_v, mat_inverse_v = sess.run([
+                operator.inverse().to_dense(), linalg.inv(mat)])
+            self.assertAC(op_inverse_v, mat_inverse_v)
+
+  def test_inverse(self):
+    self._skip_if_tests_to_skip_contains("inverse")
+    self._test_inverse()
+
   def test_trace(self):
     self._skip_if_tests_to_skip_contains("trace")
     for use_placeholder in self._use_placeholder_options:
@@ -463,7 +479,14 @@ class NonSquareLinearOperatorDerivedClassTest(LinearOperatorDerivedClassTest):
   @property
   def _tests_to_skip(self):
     """List of test names to skip."""
-    return ["cholesky", "solve", "solve_with_broadcast", "det", "log_abs_det"]
+    return [
+        "cholesky",
+        "inverse",
+        "solve",
+        "solve_with_broadcast",
+        "det",
+        "log_abs_det"
+    ]
 
   @property
   def _operator_build_infos(self):
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-block-diag.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-block-diag.pbtxt
index 773c74e64d..c7a50969b5 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-block-diag.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-block-diag.pbtxt
@@ -95,6 +95,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant.pbtxt
index 533544d21f..3900c752c8 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant.pbtxt
@@ -116,6 +116,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant2-d.pbtxt
index e3926eb6d4..7b876099af 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant2-d.pbtxt
@@ -116,6 +116,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant3-d.pbtxt
index ba209df782..5bddba8e79 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-circulant3-d.pbtxt
@@ -116,6 +116,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-composition.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-composition.pbtxt
index 081fb0e08b..62ba8bb59e 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-composition.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-composition.pbtxt
@@ -95,6 +95,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-diag.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-diag.pbtxt
index 2014a04301..0803feeabd 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-diag.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-diag.pbtxt
@@ -95,6 +95,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-full-matrix.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-full-matrix.pbtxt
index 9a87ae9687..6def32864b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-full-matrix.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-full-matrix.pbtxt
@@ -91,6 +91,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-identity.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-identity.pbtxt
index 33afb835ce..dbf1ac82d3 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-identity.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-identity.pbtxt
@@ -92,6 +92,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-inversion.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-inversion.pbtxt
new file mode 100644
index 0000000000..6a3fe4dd66
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-inversion.pbtxt
@@ -0,0 +1,142 @@
+path: "tensorflow.linalg.LinearOperatorInversion"
+tf_class {
+  is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator_inversion.LinearOperatorInversion\'>"
+  is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator.LinearOperator\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "batch_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "domain_dimension"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "graph_parents"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_non_singular"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_positive_definite"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_self_adjoint"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_square"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "operator"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "range_dimension"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "tensor_rank"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'operator\', \'is_non_singular\', \'is_self_adjoint\', \'is_positive_definite\', \'is_square\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_to_tensor"
+    argspec: "args=[\'self\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'add_to_tensor\'], "
+  }
+  member_method {
+    name: "assert_non_singular"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_non_singular\'], "
+  }
+  member_method {
+    name: "assert_positive_definite"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_positive_definite\'], "
+  }
+  member_method {
+    name: "assert_self_adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_self_adjoint\'], "
+  }
+  member_method {
+    name: "batch_shape_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], "
+  }
+  member_method {
+    name: "cholesky"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], "
+  }
+  member_method {
+    name: "determinant"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], "
+  }
+  member_method {
+    name: "diag_part"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'diag_part\'], "
+  }
+  member_method {
+    name: "domain_dimension_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
+  }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
+  member_method {
+    name: "log_abs_determinant"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
+  }
+  member_method {
+    name: "matmul"
+    argspec: "args=[\'self\', \'x\', \'adjoint\', \'adjoint_arg\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'matmul\'], "
+  }
+  member_method {
+    name: "matvec"
+    argspec: "args=[\'self\', \'x\', \'adjoint\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'matvec\'], "
+  }
+  member_method {
+    name: "range_dimension_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'range_dimension_tensor\'], "
+  }
+  member_method {
+    name: "shape_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'shape_tensor\'], "
+  }
+  member_method {
+    name: "solve"
+    argspec: "args=[\'self\', \'rhs\', \'adjoint\', \'adjoint_arg\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'solve\'], "
+  }
+  member_method {
+    name: "solvevec"
+    argspec: "args=[\'self\', \'rhs\', \'adjoint\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'solve\'], "
+  }
+  member_method {
+    name: "tensor_rank_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'tensor_rank_tensor\'], "
+  }
+  member_method {
+    name: "to_dense"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'to_dense\'], "
+  }
+  member_method {
+    name: "trace"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'trace\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-kronecker.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-kronecker.pbtxt
index a9078c8ab5..85d902b977 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-kronecker.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-kronecker.pbtxt
@@ -95,6 +95,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-low-rank-update.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-low-rank-update.pbtxt
index 4cfa3bb30d..638d82a599 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-low-rank-update.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-low-rank-update.pbtxt
@@ -115,6 +115,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-lower-triangular.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-lower-triangular.pbtxt
index a87649133f..ab1b04bd3c 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-lower-triangular.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-lower-triangular.pbtxt
@@ -91,6 +91,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-scaled-identity.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-scaled-identity.pbtxt
index 3265646784..961969aac5 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-scaled-identity.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-scaled-identity.pbtxt
@@ -96,6 +96,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-zeros.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-zeros.pbtxt
index 49d8890c89..e76738a964 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-zeros.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator-zeros.pbtxt
@@ -91,6 +91,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator.pbtxt
index c89dc067b3..b35cd69da4 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.-linear-operator.pbtxt
@@ -90,6 +90,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.linalg.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.linalg.pbtxt
index 9f7b422fab..5e49b75c31 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.linalg.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.linalg.pbtxt
@@ -36,6 +36,10 @@ tf_module {
     name: "LinearOperatorIdentity"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "LinearOperatorInversion"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "LinearOperatorKronecker"
     mtype: "<type \'type\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-block-diag.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-block-diag.pbtxt
index 773c74e64d..c7a50969b5 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-block-diag.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-block-diag.pbtxt
@@ -95,6 +95,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant.pbtxt
index 533544d21f..3900c752c8 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant.pbtxt
@@ -116,6 +116,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant2-d.pbtxt
index e3926eb6d4..7b876099af 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant2-d.pbtxt
@@ -116,6 +116,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant3-d.pbtxt
index ba209df782..5bddba8e79 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-circulant3-d.pbtxt
@@ -116,6 +116,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-composition.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-composition.pbtxt
index 081fb0e08b..62ba8bb59e 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-composition.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-composition.pbtxt
@@ -95,6 +95,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-diag.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-diag.pbtxt
index 2014a04301..0803feeabd 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-diag.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-diag.pbtxt
@@ -95,6 +95,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-full-matrix.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-full-matrix.pbtxt
index 9a87ae9687..6def32864b 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-full-matrix.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-full-matrix.pbtxt
@@ -91,6 +91,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-identity.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-identity.pbtxt
index 33afb835ce..dbf1ac82d3 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-identity.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-identity.pbtxt
@@ -92,6 +92,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-inversion.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-inversion.pbtxt
new file mode 100644
index 0000000000..6a3fe4dd66
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-inversion.pbtxt
@@ -0,0 +1,142 @@
+path: "tensorflow.linalg.LinearOperatorInversion"
+tf_class {
+  is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator_inversion.LinearOperatorInversion\'>"
+  is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator.LinearOperator\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "batch_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "domain_dimension"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "graph_parents"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_non_singular"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_positive_definite"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_self_adjoint"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_square"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "operator"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "range_dimension"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "tensor_rank"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'operator\', \'is_non_singular\', \'is_self_adjoint\', \'is_positive_definite\', \'is_square\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_to_tensor"
+    argspec: "args=[\'self\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'add_to_tensor\'], "
+  }
+  member_method {
+    name: "assert_non_singular"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_non_singular\'], "
+  }
+  member_method {
+    name: "assert_positive_definite"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_positive_definite\'], "
+  }
+  member_method {
+    name: "assert_self_adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_self_adjoint\'], "
+  }
+  member_method {
+    name: "batch_shape_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], "
+  }
+  member_method {
+    name: "cholesky"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'cholesky\'], "
+  }
+  member_method {
+    name: "determinant"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], "
+  }
+  member_method {
+    name: "diag_part"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'diag_part\'], "
+  }
+  member_method {
+    name: "domain_dimension_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
+  }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
+  member_method {
+    name: "log_abs_determinant"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
+  }
+  member_method {
+    name: "matmul"
+    argspec: "args=[\'self\', \'x\', \'adjoint\', \'adjoint_arg\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'matmul\'], "
+  }
+  member_method {
+    name: "matvec"
+    argspec: "args=[\'self\', \'x\', \'adjoint\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'matvec\'], "
+  }
+  member_method {
+    name: "range_dimension_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'range_dimension_tensor\'], "
+  }
+  member_method {
+    name: "shape_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'shape_tensor\'], "
+  }
+  member_method {
+    name: "solve"
+    argspec: "args=[\'self\', \'rhs\', \'adjoint\', \'adjoint_arg\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'solve\'], "
+  }
+  member_method {
+    name: "solvevec"
+    argspec: "args=[\'self\', \'rhs\', \'adjoint\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'solve\'], "
+  }
+  member_method {
+    name: "tensor_rank_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'tensor_rank_tensor\'], "
+  }
+  member_method {
+    name: "to_dense"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'to_dense\'], "
+  }
+  member_method {
+    name: "trace"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'trace\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-kronecker.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-kronecker.pbtxt
index a9078c8ab5..85d902b977 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-kronecker.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-kronecker.pbtxt
@@ -95,6 +95,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-low-rank-update.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-low-rank-update.pbtxt
index 4cfa3bb30d..638d82a599 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-low-rank-update.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-low-rank-update.pbtxt
@@ -115,6 +115,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-lower-triangular.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-lower-triangular.pbtxt
index a87649133f..ab1b04bd3c 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-lower-triangular.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-lower-triangular.pbtxt
@@ -91,6 +91,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-scaled-identity.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-scaled-identity.pbtxt
index 3265646784..961969aac5 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-scaled-identity.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-scaled-identity.pbtxt
@@ -96,6 +96,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-zeros.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-zeros.pbtxt
index 49d8890c89..e76738a964 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-zeros.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator-zeros.pbtxt
@@ -91,6 +91,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator.pbtxt
index c89dc067b3..b35cd69da4 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.-linear-operator.pbtxt
@@ -90,6 +90,10 @@ tf_class {
     name: "domain_dimension_tensor"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
   }
+  member_method {
+    name: "inverse"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'inverse\'], "
+  }
   member_method {
     name: "log_abs_determinant"
     argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.linalg.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.linalg.pbtxt
index 3e1e2e3d54..f9119cdd5f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.linalg.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.linalg.pbtxt
@@ -36,6 +36,10 @@ tf_module {
     name: "LinearOperatorIdentity"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "LinearOperatorInversion"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "LinearOperatorKronecker"
     mtype: "<type \'type\'>"
-- 
GitLab


From 4b2b456ea4353864478e608a587ea9b5f7eb696c Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Wed, 9 Jan 2019 15:00:20 -0800
Subject: [PATCH 0496/1095] Remove a deprecation warning from
 tf.saved_model.save

PiperOrigin-RevId: 228594973
---
 tensorflow/python/saved_model/save.py       | 2 +-
 tensorflow/python/saved_model/utils_impl.py | 5 +++++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/saved_model/save.py b/tensorflow/python/saved_model/save.py
index d79b5fe3a6..9db6d03ed0 100644
--- a/tensorflow/python/saved_model/save.py
+++ b/tensorflow/python/saved_model/save.py
@@ -222,7 +222,7 @@ def _normalize_outputs(outputs, function_name, signature_key):
 
 
 def _tensor_dict_to_tensorinfo(tensor_dict):
-  return {key: utils_impl.build_tensor_info(value)
+  return {key: utils_impl.build_tensor_info_internal(value)
           for key, value in tensor_dict.items()}
 
 
diff --git a/tensorflow/python/saved_model/utils_impl.py b/tensorflow/python/saved_model/utils_impl.py
index d43cfdf2b3..a82007fd54 100644
--- a/tensorflow/python/saved_model/utils_impl.py
+++ b/tensorflow/python/saved_model/utils_impl.py
@@ -60,6 +60,11 @@ def build_tensor_info(tensor):
   """
   if context.executing_eagerly():
     raise RuntimeError("build_tensor_info is not supported in Eager mode.")
+  return build_tensor_info_internal(tensor)
+
+
+def build_tensor_info_internal(tensor):
+  """Utility function to build TensorInfo proto from a Tensor."""
   tensor_info = meta_graph_pb2.TensorInfo(
       dtype=dtypes.as_dtype(tensor.dtype).as_datatype_enum,
       tensor_shape=tensor.get_shape().as_proto())
-- 
GitLab


From 00a074f82ebaebb436830b13978b3c24e514869d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 9 Jan 2019 15:04:45 -0800
Subject: [PATCH 0497/1095] Precalculate eager op's cache key, so this somewhat
 expensive operation doesn't have to be repeated for each inference.

PiperOrigin-RevId: 228595957
---
 .../core/common_runtime/eager/attr_builder.cc       | 13 ++++++++++++-
 tensorflow/core/common_runtime/eager/attr_builder.h | 11 +++++++++--
 .../core/common_runtime/eager/attr_builder_test.cc  | 13 +++++++++++++
 tensorflow/lite/delegates/flex/kernel.cc            |  6 ++++++
 4 files changed, 40 insertions(+), 3 deletions(-)

diff --git a/tensorflow/core/common_runtime/eager/attr_builder.cc b/tensorflow/core/common_runtime/eager/attr_builder.cc
index a750f8cbba..689b04274f 100644
--- a/tensorflow/core/common_runtime/eager/attr_builder.cc
+++ b/tensorflow/core/common_runtime/eager/attr_builder.cc
@@ -125,6 +125,7 @@ Status AttrTypeMapForOp(const char* op_name, const AttrTypeMap** out,
   template <>                                                                \
   AttrBuilder& AttrBuilder::Set(StringPiece attr_name, value_type&& value) { \
     value_field.push_back(std::make_pair(string(attr_name), value));         \
+    cached_cache_key_ = absl::nullopt;                                       \
     return *this;                                                            \
   }
 
@@ -231,7 +232,17 @@ inline tensorflow::Fprint128 CacheKeyHelper(StringPiece s, uint64 b) {
 
 }  // namespace
 
-tensorflow::Fprint128 AttrBuilder::CacheKey(const string& device) const {
+tensorflow::Fprint128 AttrBuilder::CacheKey(const string& device) {
+  if (!cached_cache_key_ || device != device_for_cached_cache_key_) {
+    cached_cache_key_ = BuildCacheKeyForDevice(device);
+    device_for_cached_cache_key_ = device;
+  }
+
+  return *cached_cache_key_;
+}
+
+tensorflow::Fprint128 AttrBuilder::BuildCacheKeyForDevice(
+    const string& device) const {
   tensorflow::Fprint128 f = tensorflow::Fingerprint128(op_name_);
   f = tensorflow::FingerprintCat128(f, tensorflow::Fingerprint128(device));
   if (node_def_ != nullptr) {
diff --git a/tensorflow/core/common_runtime/eager/attr_builder.h b/tensorflow/core/common_runtime/eager/attr_builder.h
index 5e0172dfd3..aa64b5f59b 100644
--- a/tensorflow/core/common_runtime/eager/attr_builder.h
+++ b/tensorflow/core/common_runtime/eager/attr_builder.h
@@ -28,6 +28,7 @@ limitations under the License.
 #include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/gtl/inlined_vector.h"
+#include "tensorflow/core/lib/gtl/optional.h"
 #include "tensorflow/core/platform/fingerprint.h"
 #include "tensorflow/core/util/tensor_slice_reader_cache.h"
 
@@ -74,7 +75,7 @@ Status AttrTypeByName(const AttrTypeMap& m, const string& attr_name,
 // AttrBuilder a;
 // a.NumInputs(2);
 // a.Set("T", TF_FLOAT);
-// uint64 cache_key = a.CacheKey("cpu:0");
+// tensorflow::Fprint128 cache_key = a.CacheKey("cpu:0");
 // const NodeDef& n = a.BuildNodeDef();
 //
 // Note that all calls to Set and NumInputs should happen before calling
@@ -100,10 +101,11 @@ class AttrBuilder {
   AttrBuilder& Set(StringPiece attr_name, T&& value) {
     MayBeInitializeNodeDef();
     SetInAttrValueMap(node_def_->mutable_attr(), string(attr_name), value);
+    cached_cache_key_ = absl::nullopt;
     return *this;
   }
 
-  tensorflow::Fprint128 CacheKey(const string& device) const;
+  tensorflow::Fprint128 CacheKey(const string& device);
 
   void FillAttrValueMap(AttrValueMap* m) const { FillAttrValueMap(m, true); }
   const NodeDef& BuildNodeDef();
@@ -112,6 +114,8 @@ class AttrBuilder {
   template <class T>
   using AttrVec = tensorflow::gtl::InlinedVector<std::pair<string, T>, 2>;
 
+  tensorflow::Fprint128 BuildCacheKeyForDevice(const string& device) const;
+
   void MayBeInitializeNodeDef();
   // Fill `m` with the attr-value pairs set via AttrBuilder::Set() so far, as
   // well as any default attr-value pairs from the associated op_def, if there
@@ -148,6 +152,9 @@ class AttrBuilder {
   int num_inputs_;
   std::unique_ptr<NodeDef> node_def_;
   bool node_def_finalized_;
+
+  absl::optional<tensorflow::Fprint128> cached_cache_key_;
+  string device_for_cached_cache_key_;
 };  // namespace tensorflow
 
 template <>
diff --git a/tensorflow/core/common_runtime/eager/attr_builder_test.cc b/tensorflow/core/common_runtime/eager/attr_builder_test.cc
index 220cc6f5ce..8245660bfc 100644
--- a/tensorflow/core/common_runtime/eager/attr_builder_test.cc
+++ b/tensorflow/core/common_runtime/eager/attr_builder_test.cc
@@ -67,5 +67,18 @@ TEST(AttrTypeMap, Lookup) {
   EXPECT_NE(is_list, 0);
 }
 
+TEST(AttrTypeMap, CacheKey) {
+  AttrBuilder a("op_name");
+  a.NumInputs(2);
+  a.Set("T", TF_FLOAT);
+  tensorflow::Fprint128 cache_key = a.CacheKey("cpu:0");
+
+  ASSERT_FALSE(cache_key == a.CacheKey("cpu:1"));
+  ASSERT_TRUE(cache_key == a.CacheKey("cpu:0"));
+
+  a.Set("x", 1.0);
+  ASSERT_FALSE(cache_key == a.CacheKey("cpu:0"));
+}
+
 }  // namespace
 }  // namespace tensorflow
diff --git a/tensorflow/lite/delegates/flex/kernel.cc b/tensorflow/lite/delegates/flex/kernel.cc
index d2853cf9f8..2e0fc22ad6 100644
--- a/tensorflow/lite/delegates/flex/kernel.cc
+++ b/tensorflow/lite/delegates/flex/kernel.cc
@@ -245,6 +245,12 @@ class OpNode {
       op_->MutableAttrs()->Set(attr.first, attr.second);
     }
 
+    // Precalculating a cache key saves about 10% of inference time for very
+    // small models.
+    tensorflow::Device* device = op_->Device();
+    op_->MutableAttrs()->CacheKey(device == nullptr ? "unspecified"
+                                                    : device->name());
+
     return tensorflow::Status::OK();
   }
 
-- 
GitLab


From d7cde5bd2cf5d0d7a69773422716590b79c7bed2 Mon Sep 17 00:00:00 2001
From: Yuefeng Zhou <yuefengz@google.com>
Date: Wed, 9 Jan 2019 15:18:43 -0800
Subject: [PATCH 0498/1095] Local collective all reduce with eager mode: wraps
 invocations to collective allreduce into a defun; Only broadcast send/recv
 initial values on the first replica of each worker.

PiperOrigin-RevId: 228598348
---
 .../python/collective_all_reduce_strategy.py  | 71 ++++++++++++-------
 .../collective_all_reduce_strategy_test.py    | 26 ++++---
 .../python/distribute/cross_device_ops.py     | 15 +---
 .../python/distribute/cross_device_utils.py   | 28 +++++---
 4 files changed, 85 insertions(+), 55 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
index 39756b32c5..a3538dfc50 100644
--- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
+++ b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
@@ -31,6 +31,7 @@ from tensorflow.python.distribute import multi_worker_util
 from tensorflow.python.distribute import numpy_dataset
 from tensorflow.python.distribute import values
 from tensorflow.python.eager import context
+from tensorflow.python.eager import tape
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import collective_ops
@@ -91,6 +92,7 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
 
     self._collective_keys = cross_device_utils.CollectiveKeys()
     self._initialize_local(local_devices)
+    # TODO(yuefengz): remove num_gpus_per_worker from CollectiveAllReduce.
     self._cross_device_ops = cross_device_ops_lib.CollectiveAllReduce(
         num_workers=self._num_workers,
         num_gpus_per_worker=num_gpus_per_worker,
@@ -166,16 +168,17 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
     else:
       device_map = colocate_with.device_map
       logical_device = colocate_with.logical_device
-    group_size = device_map.num_replicas_in_graph * self._num_workers
-    group_key = self._collective_keys.get_group_key(self.worker_devices)
 
     def _real_mirrored_creator(devices, *args, **kwargs):
       """Creates one MirroredVariable on the current worker."""
-      value_list = []
       unique_var_name = ops.get_default_graph().unique_name(
           kwargs["name"], mark_as_used=False).rstrip("/")
+      # pylint: disable=protected-access
       collective_instance_key = self._collective_keys.get_instance_key(
           key_id=unique_var_name)
+      # Only the first device participles in the broadcast of initial values.
+      group_key = self._collective_keys.get_group_key([devices[0]])
+      group_size = self._num_workers
       if "initial_value" not in kwargs:
         raise ValueError("Initial value must be specified.")
       initial_value = kwargs["initial_value"]
@@ -184,9 +187,33 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
       else:
         initial_value_fn = lambda: initial_value
 
+      value_list = []
       for i, d in enumerate(devices):
-        with ops.device(d):
-          if i > 0:
+        with ops.init_scope(), ops.device(d):
+          if i == 0:
+            # The initial value fn makes sure variables all initialized to
+            # same values. The first device of the chief worker will send their
+            # variable values to other workers.
+            def _overridden_initial_value_fn(device=d, index=i):  # pylint: disable=g-missing-docstring
+              with ops.device(device):
+                initial_value = initial_value_fn()
+                assert not callable(initial_value)
+                initial_value = ops.convert_to_tensor(initial_value)
+
+                assert index == 0, index
+                if self._num_workers > 1:
+                  if self._is_chief:
+                    bcast_send = collective_ops.broadcast_send(
+                        initial_value, initial_value.shape, initial_value.dtype,
+                        group_size, group_key, collective_instance_key)
+                    with ops.control_dependencies([bcast_send]):
+                      return array_ops.identity(initial_value)
+                  else:
+                    return collective_ops.broadcast_recv(
+                        initial_value.shape, initial_value.dtype, group_size,
+                        group_key, collective_instance_key)
+                return initial_value
+          else:
             # Give replicas meaningful distinct names:
             var0name = value_list[0].name.split(":")[0]
             # We append a / to variable names created on replicas with id > 0 to
@@ -194,30 +221,22 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
             # name as the absolute name of the variable.
             kwargs["name"] = "%s/replica_%d/" % (var0name, i)
 
-          # The initial value fn makes sure variables all initialized to
-          # same values. The first device of the chief worker will send their
-          # variable values to other devices and other workers.
-          def _overridden_initial_value_fn(device=d, index=i):  # pylint: disable=g-missing-docstring
-            with ops.device(device):
-              initial_value = initial_value_fn()
-              assert not callable(initial_value)
-              initial_value = ops.convert_to_tensor(initial_value)
-
-              if self._is_chief and index == 0:
-                bcast_send = collective_ops.broadcast_send(
-                    initial_value, initial_value.shape, initial_value.dtype,
-                    group_size, group_key, collective_instance_key)
-                with ops.control_dependencies([bcast_send]):
-                  return array_ops.identity(initial_value)
-              else:
-                return collective_ops.broadcast_recv(
-                    initial_value.shape, initial_value.dtype, group_size,
-                    group_key, collective_instance_key)
+            # Variables on non-first replica get initial values from the
+            # variables created on the first device of each worker.
+            def _overridden_initial_value_fn(device=d, index=i):
+              assert index > 0
+              with ops.device(device):
+                if context.executing_eagerly():
+                  return array_ops.identity(value_list[0].value())
+                else:
+                  return array_ops.identity(value_list[0].initial_value)
 
           kwargs["initial_value"] = _overridden_initial_value_fn
-
           with context.context().device_policy(context.DEVICE_PLACEMENT_SILENT):
-            v = next_creator(*args, **kwargs)
+            # Don't record operations (e.g. other variable reads) during
+            # variable creation.
+            with tape.stop_recording():
+              v = next_creator(*args, **kwargs)
 
           if i == 0:
             actual_var_name = v.name.split(":")[0]
diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
index c3e9c55e96..682da44f5d 100644
--- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
@@ -192,6 +192,7 @@ class CollectiveAllReduceStrategyTestBase(
       image = random_ops.random_uniform([2, 28, 28])
       label = random_ops.random_uniform([2, 1], maxval=10, dtype=dtypes.int32)
       logits = model(image, training=True)
+      # TODO(yuefengz): make loss a callable for eager mode.
       loss = losses.sparse_softmax_cross_entropy(labels=label, logits=logits)
       optimizer = adam.AdamOptimizer(learning_rate=1e-4)
       train_op = optimizer.minimize(loss,
@@ -403,24 +404,31 @@ class LocalCollectiveAllReduceStrategy(
     strategy_test_lib.TwoDeviceDistributionTestBase,
     parameterized.TestCase):
 
-  def testMinimizeLossGraph(self, num_gpus=2):
+  @combinations.generate(
+      combinations.combine(
+          mode=['graph', 'eager'], num_gpus=[2, 4], required_gpus=2))
+  def testMinimizeLoss(self, num_gpus):
     # Collective ops doesn't support strategy with one device.
     if context.num_gpus() < num_gpus:
       self.skipTest('Not enough GPUs')
-    self._test_minimize_loss_graph(None, None, num_gpus)
+    if context.executing_eagerly():
+      strategy, _, _ = self._get_test_object(None, None, num_gpus)
+      self._test_minimize_loss_eager(strategy)
+    else:
+      self._test_minimize_loss_graph(None, None, num_gpus)
 
-  def testComplexModel(self, num_gpus=2):
-    # Collective ops doesn't support strategy with one device.
+  @combinations.generate(
+      combinations.combine(mode=['graph'], num_gpus=[2, 4], required_gpus=2))
+  def testComplexModel(self, num_gpus):
     if context.num_gpus() < num_gpus:
       self.skipTest('Not enough GPUs')
     self._test_complex_model(None, None, num_gpus)
 
+  @combinations.generate(
+      combinations.combine(mode=['graph', 'eager'], required_gpus=2))
   def testMakeInputFnIterator(self, num_gpus=2):
-    # Collective ops doesn't support strategy with one device.
-    if context.num_gpus() < num_gpus:
-      self.skipTest('Not enough GPUs')
-    dataset_fn = lambda: dataset_ops.Dataset.range(10)
-    expected_values = [[i, i+1] for i in range(0, 10, 2)]
+    dataset_fn = lambda: dataset_ops.Dataset.range(5 * num_gpus)
+    expected_values = [range(i, i + num_gpus) for i in range(0, 10, num_gpus)]
 
     input_fn = self._input_fn_to_test_input_context(
         dataset_fn,
diff --git a/tensorflow/python/distribute/cross_device_ops.py b/tensorflow/python/distribute/cross_device_ops.py
index 9575301d97..9729302c6d 100644
--- a/tensorflow/python/distribute/cross_device_ops.py
+++ b/tensorflow/python/distribute/cross_device_ops.py
@@ -323,6 +323,9 @@ class ReductionToOneDeviceCrossDeviceOps(CrossDeviceOps):
     assert check_destinations(destinations)
     devices = get_devices_from(destinations)
     reduce_to_device = self.reduce_to_device or devices[0]
+    logging.log_first_n(
+        logging.INFO,
+        "Reduce to %s then broadcast to %r." % (reduce_to_device, devices), 10)
     reduced = _simple_reduce(per_replica_value, reduce_to_device,
                              self.accumulation_fn, reduce_op)
     return self.broadcast(reduced, destinations)
@@ -839,9 +842,6 @@ class CollectiveAllReduce(CrossDeviceOps):
     if cross_device_utils.contains_indexed_slices(per_replica_value):
       raise ValueError(
           "`IndexSlices` is not supported for Collective All-Reduce.")
-    if context.executing_eagerly():
-      raise ValueError(
-          "Eager execution is not supported for Collective All-Reduce")
 
     all_reduced = self._batch_all_reduce(reduce_op, [per_replica_value])[0]
     device_map, logical_device = get_device_map_from(destinations)
@@ -865,9 +865,6 @@ class CollectiveAllReduce(CrossDeviceOps):
     if cross_device_utils.contains_indexed_slices(value_destination_pairs):
       raise ValueError(
           "`IndexSlices` is not supported for Collective All-Reduce.")
-    if context.executing_eagerly():
-      raise ValueError(
-          "Eager execution is not supported for Collective All-Reduce")
 
     all_devices_match = _all_devices_match(value_destination_pairs)
     if all_devices_match:
@@ -886,9 +883,6 @@ class CollectiveAllReduce(CrossDeviceOps):
 
   def _batch_all_reduce(self, reduce_op, per_replica_values):
     """All-reduce across all workers in a batch."""
-    if context.executing_eagerly():
-      raise ValueError(
-          "Eager execution with collective ops is not supported yet.")
 
     logging.log_first_n(
         logging.INFO, "Collective All-reduce invoked with batches size = %d, "
@@ -949,12 +943,9 @@ def _has_dgx1_like_links(gpu_links):
 
 def _choose_all_reduce_algorithm(device_links):
   if _has_dgx1_like_links(device_links):
-    logging.info("Configured hierarchical_copy with num_packs=%d",
-                 len(device_links))
     return AllReduceCrossDeviceOps(
         "hierarchical_copy", num_packs=len(device_links))
   else:
-    logging.info("Configured nccl all-reduce.")
     return AllReduceCrossDeviceOps("nccl", num_packs=1)
 
 
diff --git a/tensorflow/python/distribute/cross_device_utils.py b/tensorflow/python/distribute/cross_device_utils.py
index 5b4b3a6f97..e8066dd467 100644
--- a/tensorflow/python/distribute/cross_device_utils.py
+++ b/tensorflow/python/distribute/cross_device_utils.py
@@ -23,6 +23,8 @@ import threading
 
 from tensorflow.python.distribute import all_reduce
 from tensorflow.python.distribute import values as value_lib
+from tensorflow.python.eager import context
+from tensorflow.python.eager import def_function
 from tensorflow.python.framework import device as pydev
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -353,15 +355,25 @@ def build_collective_reduce(input_tensors,
   num_devices = len(devices)
   group_key = collective_keys.get_group_key(devices)
   instance_key = collective_keys.get_instance_key()
-  out_tensors = []
   subdiv_offsets = [0]  # TODO(tucker): maybe support non-default subdiv spec
-  for d in range(num_devices):
-    with ops.device(devices[d]):
-      reduce_op = collective_ops.all_reduce(
-          input_tensors[d], group_size, group_key, instance_key, reduction_op,
-          unary_op, subdiv_offsets)
-      out_tensors.append(reduce_op)
-  return out_tensors
+
+  def collective_all_reduce():
+    """Call collective allreduce."""
+    assert not context.executing_eagerly()
+    out_tensors = []
+    for d in range(num_devices):
+      with ops.device(devices[d]):
+        reduce_op = collective_ops.all_reduce(
+            input_tensors[d], group_size, group_key, instance_key, reduction_op,
+            unary_op, subdiv_offsets)
+        out_tensors.append(reduce_op)
+    return out_tensors
+
+  if context.executing_eagerly():
+    # Collective ops will block unless they are executed concurrently such as in
+    # a graph or a defun.
+    collective_all_reduce = def_function.function(collective_all_reduce)
+  return collective_all_reduce()
 
 
 def sum_grad_and_var_all_reduce(grad_and_vars,
-- 
GitLab


From 615182adb3d01fd8357e574bd8920c0995c6bbb8 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 9 Jan 2019 15:42:02 -0800
Subject: [PATCH 0499/1095] Fix issue with Subclassed models and generator
 input.

PiperOrigin-RevId: 228602224
---
 tensorflow/python/keras/callbacks.py          |  9 +---
 tensorflow/python/keras/callbacks_test.py     | 42 -------------------
 .../python/keras/engine/training_generator.py |  4 --
 .../keras/engine/training_generator_test.py   | 21 ++++------
 4 files changed, 8 insertions(+), 68 deletions(-)

diff --git a/tensorflow/python/keras/callbacks.py b/tensorflow/python/keras/callbacks.py
index 589cd992d6..b9074669eb 100644
--- a/tensorflow/python/keras/callbacks.py
+++ b/tensorflow/python/keras/callbacks.py
@@ -123,14 +123,6 @@ def configure_callbacks(callbacks,
       'metrics': callback_metrics,
   }
   callback_list.set_params(callback_params)
-
-  if (do_validation and not model._distribution_strategy and
-      not model.run_eagerly):
-    # Need to create the eval_function before start of the first epoch
-    # because TensorBoard callback on_epoch_begin adds summary to the
-    # list of fetches of the eval_function
-    callback_model._make_eval_function()
-
   callback_list.model.stop_training = False
   return callback_list
 # pylint: enable=protected-access
@@ -1373,6 +1365,7 @@ class TensorBoard(Callback):
       self._epoch = epoch
       # pylint: disable=protected-access
       # add the histogram summary op if it should run this epoch
+      self.model._make_eval_function()
       if self.merged not in self.model._eval_function.fetches:
         self.model._eval_function.fetches.append(self.merged)
         self.model._eval_function.fetch_callbacks[
diff --git a/tensorflow/python/keras/callbacks_test.py b/tensorflow/python/keras/callbacks_test.py
index ef469c5e4f..d5ffdf789a 100644
--- a/tensorflow/python/keras/callbacks_test.py
+++ b/tensorflow/python/keras/callbacks_test.py
@@ -31,7 +31,6 @@ import numpy as np
 
 from tensorflow.core.framework import summary_pb2
 from tensorflow.python import keras
-from tensorflow.python.framework import ops
 from tensorflow.python.framework import random_seed
 from tensorflow.python.framework import test_util
 from tensorflow.python.keras import keras_parameterized
@@ -1397,47 +1396,6 @@ class KerasCallbacksTest(test.TestCase):
             callbacks=cbks,
             epochs=1)
 
-  @test_util.run_deprecated_v1
-  def test_fit_generator_with_callback(self):
-
-    class TestCallback(keras.callbacks.Callback):
-
-      def set_model(self, model):
-        # Check the model operations for the optimizer operations that
-        # the _make_train_function adds under a named scope for the
-        # optimizer. This ensurs the full model is populated before the
-        # set_model callback is called.
-        optimizer_name_scope = 'training/' + model.optimizer.__class__.__name__
-        graph_def = ops.get_default_graph().as_graph_def()
-        for node in graph_def.node:
-          if node.name.startswith(optimizer_name_scope):
-            return
-        raise RuntimeError('The optimizer operations are not present in the '
-                           'model graph when the Callback.set_model function '
-                           'is called')
-    np.random.seed(1337)
-
-    def generator():
-      x = np.random.randn(10, 100).astype(np.float32)
-      y = np.random.randn(10, 10).astype(np.float32)
-      while True:
-        yield x, y
-
-    with self.cached_session():
-      model = testing_utils.get_small_sequential_mlp(
-          num_hidden=10, num_classes=10, input_dim=100)
-      model.compile(
-          loss='categorical_crossentropy',
-          optimizer='sgd',
-          metrics=['accuracy'])
-      model.fit_generator(
-          generator(),
-          steps_per_epoch=2,
-          epochs=1,
-          validation_data=generator(),
-          validation_steps=2,
-          callbacks=[TestCallback()],
-          verbose=0)
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/keras/engine/training_generator.py b/tensorflow/python/keras/engine/training_generator.py
index d1699b2827..07de934259 100644
--- a/tensorflow/python/keras/engine/training_generator.py
+++ b/tensorflow/python/keras/engine/training_generator.py
@@ -430,12 +430,8 @@ def _make_enqueued_generator(generator,
 def _make_execution_function(model, mode, class_weight=None):
   """Makes function to run one step of model execution."""
   if mode == ModeKeys.TRAIN:
-    if not context.executing_eagerly():
-      model._make_fit_function()
     f = functools.partial(model.train_on_batch, class_weight=class_weight)
   elif mode == ModeKeys.TEST:
-    if not context.executing_eagerly():
-      model._make_eval_function()
     f = model.test_on_batch
   else:
     # Match signature of other modes to allow
diff --git a/tensorflow/python/keras/engine/training_generator_test.py b/tensorflow/python/keras/engine/training_generator_test.py
index 90c45dfcb7..6b754c18b3 100644
--- a/tensorflow/python/keras/engine/training_generator_test.py
+++ b/tensorflow/python/keras/engine/training_generator_test.py
@@ -66,8 +66,7 @@ class TestGeneratorMethods(keras_parameterized.TestCase):
   @unittest.skipIf(
       os.name == 'nt',
       'use_multiprocessing=True does not work on windows properly.')
-  # TODO(b/120940700): Bug with subclassed model inputs.
-  @keras_parameterized.run_with_all_model_types(exclude_models='subclass')
+  @keras_parameterized.run_with_all_model_types
   @keras_parameterized.run_all_keras_modes
   def test_fit_generator_method(self):
     model = testing_utils.get_small_mlp(
@@ -107,8 +106,7 @@ class TestGeneratorMethods(keras_parameterized.TestCase):
   @unittest.skipIf(
       os.name == 'nt',
       'use_multiprocessing=True does not work on windows properly.')
-  # TODO(b/120940700): Bug with subclassed model inputs.
-  @keras_parameterized.run_with_all_model_types(exclude_models='subclass')
+  @keras_parameterized.run_with_all_model_types
   @keras_parameterized.run_all_keras_modes
   def test_evaluate_generator_method(self):
     model = testing_utils.get_small_mlp(
@@ -173,8 +171,7 @@ class TestGeneratorMethods(keras_parameterized.TestCase):
                             max_queue_size=10,
                             workers=0)
 
-  # TODO(b/120940700): Bug with subclassed model inputs.
-  @keras_parameterized.run_with_all_model_types(exclude_models='subclass')
+  @keras_parameterized.run_with_all_model_types
   @keras_parameterized.run_all_keras_modes
   def test_generator_methods_with_sample_weights(self):
     model = testing_utils.get_small_mlp(
@@ -208,8 +205,7 @@ class TestGeneratorMethods(keras_parameterized.TestCase):
                              max_queue_size=10,
                              use_multiprocessing=False)
 
-  # TODO(b/120940700): Bug with subclassed model inputs.
-  @keras_parameterized.run_with_all_model_types(exclude_models='subclass')
+  @keras_parameterized.run_with_all_model_types
   @keras_parameterized.run_all_keras_modes
   def test_generator_methods_invalid_use_case(self):
 
@@ -249,8 +245,7 @@ class TestGeneratorMethods(keras_parameterized.TestCase):
                                max_queue_size=10,
                                use_multiprocessing=False)
 
-  # TODO(b/120940700): Bug with subclassed model inputs.
-  @keras_parameterized.run_with_all_model_types(exclude_models='subclass')
+  @keras_parameterized.run_with_all_model_types
   @keras_parameterized.run_all_keras_modes
   def test_generator_input_to_fit_eval_predict(self):
     val_data = np.ones([10, 10], np.float32), np.ones([10, 1], np.float32)
@@ -275,8 +270,7 @@ class TestGeneratorMethods(keras_parameterized.TestCase):
 
 class TestGeneratorMethodsWithSequences(keras_parameterized.TestCase):
 
-  # TODO(b/120940700): Bug with subclassed model inputs.
-  @keras_parameterized.run_with_all_model_types(exclude_models='subclass')
+  @keras_parameterized.run_with_all_model_types
   @keras_parameterized.run_all_keras_modes
   def test_training_with_sequences(self):
 
@@ -307,8 +301,7 @@ class TestGeneratorMethodsWithSequences(keras_parameterized.TestCase):
                         workers=0,
                         use_multiprocessing=False)
 
-  # TODO(b/120940700): Bug with subclassed model inputs.
-  @keras_parameterized.run_with_all_model_types(exclude_models='subclass')
+  @keras_parameterized.run_with_all_model_types
   @keras_parameterized.run_all_keras_modes
   def test_sequence_input_to_fit_eval_predict(self):
     val_data = np.ones([10, 10], np.float32), np.ones([10, 1], np.float32)
-- 
GitLab


From ac40a68cd5fac2227fb6c4086b2eb01a7dc726c4 Mon Sep 17 00:00:00 2001
From: Samantha Andow <samdow@fb.com>
Date: Wed, 26 Dec 2018 08:58:44 -0800
Subject: [PATCH 0500/1095] Fix iterable bug

Co-authored-by: melissagrueter <melissagrueter@fb.com>
Co-authored-by: irenedea <irenedea@fb.com>
---
 tensorflow/java/src/gen/cc/op_specs.cc | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tensorflow/java/src/gen/cc/op_specs.cc b/tensorflow/java/src/gen/cc/op_specs.cc
index 4f5a491d25..4024efedef 100644
--- a/tensorflow/java/src/gen/cc/op_specs.cc
+++ b/tensorflow/java/src/gen/cc/op_specs.cc
@@ -91,11 +91,6 @@ class TypeResolver {
 
 Type TypeResolver::TypeOf(const OpDef_ArgDef& arg_def, bool* iterable_out) {
   *iterable_out = false;
-  if (!arg_def.number_attr().empty()) {
-    // when number_attr is set, argument has to be a list of tensors
-    *iterable_out = true;
-    visited_attrs_.insert(std::make_pair(arg_def.number_attr(), Type::Int()));
-  }
   Type type = Type::Wildcard();
   if (arg_def.type() != DataType::DT_INVALID) {
     type = Type::ForDataType(arg_def.type());
@@ -122,6 +117,11 @@ Type TypeResolver::TypeOf(const OpDef_ArgDef& arg_def, bool* iterable_out) {
     LOG(FATAL) << "Cannot resolve data type of argument \"" << arg_def.name()
                << "\" in operation \"" << op_def_.name() << "\"";
   }
+  if (!arg_def.number_attr().empty()) {
+    // when number_attr is set, argument has to be a list of tensors
+    *iterable_out = true;
+    visited_attrs_.insert(std::make_pair(arg_def.number_attr(), Type::Int()));
+  }
   return type;
 }
 
-- 
GitLab


From fe13148bf6afcb85d71c05391d168302d8d4bf2d Mon Sep 17 00:00:00 2001
From: Frank Chen <frankchn@google.com>
Date: Wed, 9 Jan 2019 16:24:30 -0800
Subject: [PATCH 0501/1095] Change TPU Cluster Resolver doc string to clarify
 current / future features

PiperOrigin-RevId: 228610010
---
 .../cluster_resolver/tpu_cluster_resolver.py          | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py b/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py
index 529a443412..72b9990701 100644
--- a/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py
+++ b/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py
@@ -192,11 +192,12 @@ class TPUClusterResolver(ClusterResolver):
     for the IP addresses and ports of each Cloud TPU listed.
 
     Args:
-      tpu: Either a string, or a list of strings corresponding to the TPUs to
-        use. If the single string is the empty string, the string 'local', or a
-        string that begins with 'grpc://' or '/bns', then it is assumed to not
-        correspond with a Cloud TPU and will instead be passed as the session
-        master and no ClusterSpec propagation will be done.
+      tpu: A string corresponding to the TPU to use. If the string is the empty
+        string, the string 'local', or a string that begins with 'grpc://' or
+        '/bns', then it is assumed to not correspond with a Cloud TPU and will
+        instead be passed as the session master and no ClusterSpec propagation
+        will be done. In the future, this may also support a list of strings
+        when multiple Cloud TPUs are used.
       zone: Zone where the TPUs are located. If omitted or empty, we will assume
         that the zone of the TPU is the same as the zone of the GCE VM, which we
         will try to discover from the GCE metadata service.
-- 
GitLab


From 4bb6da8e92da8d75456ee00b264e4c02877c5d7d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 9 Jan 2019 16:35:23 -0800
Subject: [PATCH 0502/1095] Enable TPUEstimator to ignore non predict mode
 exporters rather than just hard failing.

PiperOrigin-RevId: 228611812
---
 tensorflow/contrib/tpu/python/tpu/tpu_estimator.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 2317300041..2df5b3e4eb 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -2186,7 +2186,9 @@ class TPUEstimator(estimator_lib.Estimator):
       eval_on_tpu: If False, evaluation runs on CPU or GPU. In this case, the
         model_fn must return `EstimatorSpec` when called with `mode` as `EVAL`.
       export_to_tpu: If True, `export_savedmodel()` exports a metagraph for
-        serving on TPU besides the one on CPU.
+        serving on TPU besides the one on CPU. Note that unsupported export
+        modes such as EVAL will be ignored. For those modes, only a CPU model
+        will be exported. Currently, export_to_tpu only supports PREDICT.
       warm_start_from: Optional string filepath to a checkpoint or SavedModel to
         warm-start from, or a `tf.estimator.WarmStartSettings` object to fully
         configure warm-starting.  If the string filepath is provided instead of
@@ -2277,10 +2279,9 @@ class TPUEstimator(estimator_lib.Estimator):
                                export_tags=None,
                                check_variables=True):
     if self._export_to_tpu and mode != model_fn_lib.ModeKeys.PREDICT:
-      raise NotImplementedError(
-          'TPUEstimator only handles mode PREDICT for exporting '
-          'when `export_to_tpu` is `True`; '
-          'got {}.'.format(mode))
+      logging.warning('TPUEstimator only handles mode PREDICT for exporting '
+                      'when `export_to_tpu` is `True`; Mode {} will be ignored '
+                      'for TPU.'.format(mode))
 
     (super(TPUEstimator, self)._add_meta_graph_for_mode(
         builder,
@@ -2291,7 +2292,7 @@ class TPUEstimator(estimator_lib.Estimator):
         export_tags=export_tags,
         check_variables=check_variables))
 
-    if self._export_to_tpu:
+    if self._export_to_tpu and mode == model_fn_lib.ModeKeys.PREDICT:
       input_receiver_fn_map = {
           _REWRITE_FOR_INFERENCE_MODE: input_receiver_fn_map[mode]
       }
-- 
GitLab


From 1a768c9dc3895b3188b19d36a630f0e8ba5c85a0 Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Wed, 9 Jan 2019 16:46:17 -0800
Subject: [PATCH 0503/1095] Populate entry parameter's dynamic dimension when
 using XlaBuilder::SetDynamicBinding

Since now that we have is_dynamic_dimension field, we can set the field when calling SetDynamicBinding. ShapeInference will soon start populating this field based on the parameter shape.

PiperOrigin-RevId: 228613510
---
 tensorflow/compiler/xla/client/xla_builder.cc | 23 +++++++++++++++++++
 .../compiler/xla/client/xla_builder_test.cc   | 20 ++++++++++++++++
 2 files changed, 43 insertions(+)

diff --git a/tensorflow/compiler/xla/client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_builder.cc
index 7a04c0ec69..59e156eb72 100644
--- a/tensorflow/compiler/xla/client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_builder.cc
@@ -247,6 +247,29 @@ Status XlaBuilder::SetDynamicBinding(int64 dynamic_size_param_num,
                                      int64 target_param_num,
                                      ShapeIndex target_param_index,
                                      int64 target_dim_num) {
+  bool param_exists = false;
+  for (HloInstructionProto& instr : instructions_) {
+    if (instr.opcode() == HloOpcodeString(HloOpcode::kParameter) &&
+        instr.parameter_number() == target_param_num) {
+      param_exists = true;
+      Shape param_shape(instr.shape());
+      Shape* param_shape_ptr = &param_shape;
+      for (int64 index : target_param_index) {
+        param_shape_ptr = param_shape_ptr->mutable_tuple_shapes(index);
+      }
+      param_shape_ptr->set_dynamic_dimension(target_dim_num,
+                                             /*is_dynamic=*/true);
+      *instr.mutable_shape() = param_shape.ToProto();
+    }
+  }
+
+  if (!param_exists) {
+    return InvalidArgument(
+        "Asked to mark parameter %lld as dynamic sized parameter, but the "
+        "doesn't exists",
+        target_param_num);
+  }
+
   TF_RETURN_IF_ERROR(dynamic_parameter_binding_.Bind(
       DynamicParameterBinding::DynamicParameter{dynamic_size_param_num,
                                                 dynamic_size_param_index},
diff --git a/tensorflow/compiler/xla/client/xla_builder_test.cc b/tensorflow/compiler/xla/client/xla_builder_test.cc
index ba929a1200..abc11b4732 100644
--- a/tensorflow/compiler/xla/client/xla_builder_test.cc
+++ b/tensorflow/compiler/xla/client/xla_builder_test.cc
@@ -446,6 +446,26 @@ TEST_F(XlaBuilderTest, ProtoMatches) {
   EXPECT_EQ(c0_string, c1_string);
 }
 
+TEST_F(XlaBuilderTest, DynamicParameter) {
+  std::vector<XlaComputation> computations;
+  XlaBuilder b("builder");
+  Shape tuple_param_shape = ShapeUtil::MakeTupleShape(
+      {ShapeUtil::MakeShape(F32, {5}), ShapeUtil::MakeShape(F32, {6})});
+  auto p0 = Parameter(&b, 0, tuple_param_shape, "p0");
+  Parameter(&b, 1, ShapeUtil::MakeShape(U32, {}), "p1");
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/1,
+                                   /*dynamic_size_param_index=*/{},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{1},
+                                   /*target_dim_num=*/0));
+  TF_ASSERT_OK_AND_ASSIGN(auto module, BuildHloModule(&b, /*root=*/p0));
+  const Shape& param_shape = module->entry_computation()
+                                 ->parameter_instruction(0)
+                                 ->shape()
+                                 .tuple_shapes(1);
+  EXPECT_TRUE(param_shape.is_dynamic_dimension(0));
+}
+
 TEST_F(XlaBuilderTest, AfterAllWithNonTokenOperands) {
   XlaBuilder b(TestName());
   AfterAll(&b, {CreateToken(&b), ConstantR0<float>(&b, 1.0)});
-- 
GitLab


From 8e55a2f3fe768451b0e1ac8ae657f5ecfcdd0123 Mon Sep 17 00:00:00 2001
From: Dimitris Vardoulakis <dimvar@google.com>
Date: Wed, 9 Jan 2019 16:55:33 -0800
Subject: [PATCH 0504/1095] [TF/XLA] Fix bug in AR/CRS combiner. Don't recur to
 the arguments when comparing cross-module AllReduces for equality.

PiperOrigin-RevId: 228615009
---
 tensorflow/compiler/xla/service/ar_crs_combiner.cc   | 12 +++++++++---
 .../compiler/xla/service/ar_crs_combiner_test.cc     |  5 +++--
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/tensorflow/compiler/xla/service/ar_crs_combiner.cc b/tensorflow/compiler/xla/service/ar_crs_combiner.cc
index 4a227d3b5c..35a32f537b 100644
--- a/tensorflow/compiler/xla/service/ar_crs_combiner.cc
+++ b/tensorflow/compiler/xla/service/ar_crs_combiner.cc
@@ -190,6 +190,15 @@ bool ArCrsCombiner::InstructionsComputeSameValue(
   if (opcode1 != i2->opcode() || operands1.size() != i2->operands().size()) {
     return false;
   }
+  auto eq_computations = [](const HloComputation* a, const HloComputation* b) {
+    return *a == *b;
+  };
+  if (i1->IsCrossModuleAllReduce()) {
+    return i1->Identical(*i2,
+                         /*eq_operands=*/std::equal_to<const HloInstruction*>(),
+                         eq_computations,
+                         /*layout_sensitive=*/false);
+  }
   visited_pairs->emplace(min_uid, max_uid);
   for (int i = 0; i < operands1.size(); ++i) {
     auto operand1 = operands1[i];
@@ -215,9 +224,6 @@ bool ArCrsCombiner::InstructionsComputeSameValue(
   // InstructionsComputeSameValue earlier.
   auto eq_instructions = [](const HloInstruction* i1,
                             const HloInstruction* i2) -> bool { return true; };
-  auto eq_computations = [](const HloComputation* a, const HloComputation* b) {
-    return *a == *b;
-  };
   return i1->Identical(*i2, eq_instructions, eq_computations,
                        /*layout_sensitive=*/false);
 }
diff --git a/tensorflow/compiler/xla/service/ar_crs_combiner_test.cc b/tensorflow/compiler/xla/service/ar_crs_combiner_test.cc
index caa57296f4..b12b63b2dd 100644
--- a/tensorflow/compiler/xla/service/ar_crs_combiner_test.cc
+++ b/tensorflow/compiler/xla/service/ar_crs_combiner_test.cc
@@ -360,6 +360,7 @@ HloModule foobar
 
 ENTRY %entrycomp (p: bf16[]) -> (f32[], f32[]) {
   %p = bf16[] parameter(0)
+  %constant.bf16 = bf16[] constant(1)
 
   %all-reduce.ar.1 = bf16[]
       all-reduce(%p),
@@ -377,7 +378,7 @@ ENTRY %entrycomp (p: bf16[]) -> (f32[], f32[]) {
       sharding={maximal device=0}
 
   %all-reduce.ar.2 = bf16[]
-      all-reduce(%p),
+      all-reduce(%constant.bf16),
       replica_groups={{0},{1}},
       all_reduce_id=1,
       to_apply=%sum.bf16,
@@ -407,7 +408,7 @@ ENTRY %entrycomp (p: bf16[]) -> (f32[], f32[]) {
   EXPECT_TRUE(changed);
   EXPECT_THAT(module->entry_computation()->root_instruction(),
               op::Tuple(op::AllReduce(op::Convert(op::Parameter())),
-                        op::AllReduce(op::Convert(op::Parameter()))));
+                        op::AllReduce(op::Convert(op::Constant()))));
   auto crs_after =
       module->entry_computation()->root_instruction()->operands()[0];
   auto replica_groups_after = crs_after->replica_groups();
-- 
GitLab


From 7f9c7afd8829d20df25ee04f1999074290d7a45a Mon Sep 17 00:00:00 2001
From: Andiry Xu <andiryxu@google.com>
Date: Wed, 9 Jan 2019 16:58:27 -0800
Subject: [PATCH 0505/1095] Fix AllOutputValuesKnown logic.

PiperOrigin-RevId: 228615404
---
 .../core/grappler/costs/graph_properties.cc     | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/tensorflow/core/grappler/costs/graph_properties.cc b/tensorflow/core/grappler/costs/graph_properties.cc
index 863a5087ae..e869f7830c 100644
--- a/tensorflow/core/grappler/costs/graph_properties.cc
+++ b/tensorflow/core/grappler/costs/graph_properties.cc
@@ -1087,15 +1087,20 @@ class SymbolicShapeRefiner {
         c->output_tensor_protos.size() < ic->num_outputs()) {
       return false;
     } else {
+      // Checks if we can get output value via either output_tensor_proto or
+      // output_tensors_as_shapes.
       for (int i = 0; i < ic->num_outputs(); i++) {
-        if (c->output_tensor_protos.size() <= i ||
-            c->output_tensor_protos[i] == nullptr) {
-          return false;
+        if (c->output_tensor_protos.size() > i &&
+            c->output_tensor_protos[i] != nullptr) {
+          continue;
         }
-        if (c->output_tensors_as_shapes.size() <= i ||
-            !ic->FullyDefined(c->output_tensors_as_shapes[i])) {
-          return false;
+        if (c->output_tensors_as_shapes.size() > i &&
+            ic->FullyDefined(c->output_tensors_as_shapes[i])) {
+          continue;
         }
+
+        // Unknown for output[i].
+        return false;
       }
     }
     return true;
-- 
GitLab


From c2aa5390828b39863f9baaa2e5baad34788c7dec Mon Sep 17 00:00:00 2001
From: Jian Li <jianlijianli@google.com>
Date: Wed, 9 Jan 2019 17:11:46 -0800
Subject: [PATCH 0506/1095] Update basic rnn test case.

PiperOrigin-RevId: 228617690
---
 tensorflow/lite/kernels/basic_rnn_test.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/kernels/basic_rnn_test.cc b/tensorflow/lite/kernels/basic_rnn_test.cc
index c71849fff3..9eb20444a6 100644
--- a/tensorflow/lite/kernels/basic_rnn_test.cc
+++ b/tensorflow/lite/kernels/basic_rnn_test.cc
@@ -240,7 +240,7 @@ class HybridRNNOpModel : public RNNOpModel {
 
   TensorType tensor_type_;
 
-  void SetWeights(int weights_idx, std::vector<float> f) {
+  void SetWeights(int weights_idx, const std::vector<float>& f) {
     if (tensor_type_ == TensorType_UINT8) {
       SymmetricQuantizeAndPopulate(weights_idx, f);
     } else {
-- 
GitLab


From 12f810aa76194935959cc0d414f7de6eeaa234b9 Mon Sep 17 00:00:00 2001
From: Bixia Zheng <bixia@google.com>
Date: Wed, 9 Jan 2019 17:14:32 -0800
Subject: [PATCH 0507/1095] [XLA:GPU] Enhance unrolling heuristics for column
 reduction.

Previously, we enable unrolling only when the reduce operands are of small data
types. This change adds a simple analysis to count the number of tensors that
can be vectorized and can't be vectorized in order to decide whether unrolling
is beneficial for the kernel.

Add test cases.

PiperOrigin-RevId: 228618121
---
 .../xla/service/gpu/ir_emitter_unnested.cc    | 106 ++++++++++++--
 .../xla/service/gpu/ir_emitter_unnested.h     |   8 +-
 .../gpu/tests/gpu_kernel_tiling_test.cc       | 129 +++++++++++++++++-
 3 files changed, 228 insertions(+), 15 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
index 2bc4912155..34ddeb1d41 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
@@ -3403,11 +3403,101 @@ std::tuple<int64, int64, int64> GetReductionToVectorDimensions(
   return std::make_tuple(num_reduced_major, num_kept, num_reduced_minor);
 }
 
+// Returns true if all the transitive users of hlo before hitting users in
+// use_chain_endings are elementwise operations.
+bool AreUsersElementwise(const HloInstruction* hlo,
+                         const ConstHloInstructionSet& use_chain_endings) {
+  return absl::c_all_of(hlo->users(), [&](const HloInstruction* user) {
+    return use_chain_endings.count(user) ||
+           (user->IsElementwise() &&
+            AreUsersElementwise(user, use_chain_endings));
+  });
+}
+
+// Returns the number of fusion inputs that have the same dimension as the
+// given shape, and involve in only elementwise operations.
+int64 NumInputsInvolveInOnlyElementwiseOps(
+    const HloInstruction* unnested_hlo, const Shape& op_shape,
+    const ConstHloInstructionSet& use_chain_endings) {
+  return absl::c_count_if(
+      unnested_hlo->fused_parameters(), [&](const HloInstruction* parameter) {
+        const Shape& parameter_shape = parameter->shape();
+        return ShapeUtil::SameDimensions(op_shape, parameter_shape) &&
+               AreUsersElementwise(parameter, use_chain_endings);
+      });
+}
+
+// Returns the number of fusion inputs that have more elements than the given
+// shape.
+int64 NumInputsWithMoreElementsThan(const HloInstruction* unnested_hlo,
+                                    const Shape& shape) {
+  int64 num_elements = ShapeUtil::ElementsIn(shape);
+  return absl::c_count_if(
+      unnested_hlo->fused_parameters(), [&](const HloInstruction* parameter) {
+        return ShapeUtil::ElementsIn(parameter->shape()) > num_elements;
+      });
+}
+
+// The benefit of unrolling a kInput fusion that is a column reduction comes
+// from the vectorization of non-reduction fusion outputs and fusion inputs.
+// On the other hand, unrolling can also introduce factors that can cause
+// the kernel to run slower. This routine uses a simple heuristic to estimate
+// the benefit as well as the overhead of unrolling in order to decide whether
+// unrolling is beneficial for the given kInput fusion.
+bool IsUnrollingColumnReductionBeneficial(const HloInstruction* unnested_hlo,
+                                          const Shape& input_shape,
+                                          int64 num_kept) {
+  // TODO(b/122468062): Need further investigate to see whether we can
+  // remove the constraint on IsPowerOfTwo.
+  if (!IsPowerOfTwo(static_cast<uint64>(num_kept))) {
+    return false;
+  }
+
+  if (unnested_hlo->opcode() == HloOpcode::kReduce) {
+    return true;
+  }
+
+  CHECK_EQ(unnested_hlo->opcode(), HloOpcode::kFusion);
+  int64 can_be_vectorized = 0;
+  int64 cannot_be_vectorized = 0;
+  const HloInstruction* fused_root = unnested_hlo->fused_expression_root();
+  ConstHloInstructionSet use_chain_endings;
+  if (fused_root->opcode() == HloOpcode::kReduce) {
+    use_chain_endings.insert(fused_root);
+    // Atomic.add of the reduction result can't be vectorized.
+    cannot_be_vectorized++;
+  } else {
+    CHECK_EQ(fused_root->opcode(), HloOpcode::kTuple);
+    for (const HloInstruction* instr : fused_root->operands()) {
+      if (instr->opcode() == HloOpcode::kReduce) {
+        // Atomic.add of the reduction result can't be vectorized.
+        cannot_be_vectorized++;
+      } else {
+        // Write of the non-reduction result can be vectorized.
+        can_be_vectorized++;
+      }
+      use_chain_endings.insert(instr);
+    }
+  }
+  // Fusion inputs that have the same dimension as the reduce input and
+  // only involve in elementwise operations can be vectorized.
+  can_be_vectorized += NumInputsInvolveInOnlyElementwiseOps(
+      unnested_hlo, input_shape, use_chain_endings);
+  // Fusion inputs with more elements than the reduce op input must participate
+  // in non-elementwise operations and we assume that they are not vectorizable
+  // for the purpose of estimating the benefit of unrolling. If the kernel is
+  // unrolled even with such an assumption,  and the accesses to those inputs
+  // turn out to be vectorizable, the compiler will still vectorize them.
+  cannot_be_vectorized +=
+      NumInputsWithMoreElementsThan(unnested_hlo, input_shape);
+  return can_be_vectorized >= cannot_be_vectorized;
+}
+
 }  // namespace
 
 std::tuple<KernelMappingScheme, bool>
 IrEmitterUnnested::ComputeMappingSchemeAndReductionKind(
-    const HloInstruction* first_reduce) {
+    const HloInstruction* unnested_hlo, const HloInstruction* first_reduce) {
   int64 depth = 1;
   int64 height = 1;
   int64 width = 1;
@@ -3437,15 +3527,6 @@ IrEmitterUnnested::ComputeMappingSchemeAndReductionKind(
     height = num_reduced_major;
     width = num_kept;
     is_row_reduction = false;
-    // Assume unrolling is beneficial only when we can vectorize the loads
-    // of small data types.
-    auto is_unrolling_beneficial = [&] {
-      // TODO(b/122468062): Need further investigate to see whether we can
-      // remove the constraint on IsPowerOfTwo.
-      return IsPowerOfTwo(static_cast<uint64>(num_kept)) &&
-             primitive_util::BitWidth(
-                 first_reduce->operand(0)->shape().element_type()) <= 16;
-    };
     // Column reduction without transpose doesn't require communication among
     // threads processing elements in the same tile. The current implementation
     // only support the use of one hardware thread block to process one block of
@@ -3454,7 +3535,8 @@ IrEmitterUnnested::ComputeMappingSchemeAndReductionKind(
     // num_threads_x and tile_size_x to allow a bigger hardware thread block.
     int64 hw_threads_per_block_limit =
         ThreadsPerBlockLimit(ir_emitter_context_->device_description());
-    if (is_unrolling_beneficial()) {
+    if (IsUnrollingColumnReductionBeneficial(unnested_hlo, input_shape,
+                                             num_kept)) {
       tile_size_x = std::min(2 * hw_threads_per_block_limit, num_kept);
       num_threads_x = tile_size_x / 2;
       dilated_x = false;
@@ -3539,7 +3621,7 @@ Status IrEmitterUnnested::EmitReductionToVector(HloInstruction* unnested_hlo) {
   bool is_row_reduction;
   llvm_ir::KernelMappingScheme mapping_scheme;
   std::tie(mapping_scheme, is_row_reduction) =
-      ComputeMappingSchemeAndReductionKind(first_reduce);
+      ComputeMappingSchemeAndReductionKind(unnested_hlo, first_reduce);
   ReductionCodegenInfo reduction_info(&mapping_scheme, is_row_reduction);
   KernelCodeGenerator kernel_generator(
       /*tile_element_generator=*/
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h
index eb0ea90e14..21b842bb2c 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h
@@ -217,9 +217,13 @@ class IrEmitterUnnested : public IrEmitter {
   Status EmitReductionToVector(HloInstruction* unnested_hlo);
 
   // Computes the KernelMappingScheme for the reduce HLO and indicates whether
-  // the reduction is a row reduction.
+  // the reduction is a row reduction. For an un-fused reduce op, unnested_hlo
+  // and first_reduce are the same instruction. For a kInput fusion,
+  // unnested_hlo is the fusion instruction while first_reduce is the first
+  // reduce op.
   std::tuple<llvm_ir::KernelMappingScheme, bool>
-  ComputeMappingSchemeAndReductionKind(const HloInstruction* first_reduce);
+  ComputeMappingSchemeAndReductionKind(const HloInstruction* unnested_hlo,
+                                       const HloInstruction* first_reduce);
 
   // Emits code for an in-place scatter, modifying `thunk`s launch dimensions in
   // the process. `scatter` may be fused, scatter indices are taken from
diff --git a/tensorflow/compiler/xla/service/gpu/tests/gpu_kernel_tiling_test.cc b/tensorflow/compiler/xla/service/gpu/tests/gpu_kernel_tiling_test.cc
index 8004ebdc8e..869724db60 100644
--- a/tensorflow/compiler/xla/service/gpu/tests/gpu_kernel_tiling_test.cc
+++ b/tensorflow/compiler/xla/service/gpu/tests/gpu_kernel_tiling_test.cc
@@ -270,7 +270,7 @@ TEST_F(GpuKernelTilingTest, TransposedInputWithoutUnsafeUseTiled) {
         kind=kLoop, calls=fused_computation
     })";
 
-  // Check that a call to llvm.nvvm.barrier0 is not generated.
+  // Check that a call to llvm.nvvm.barrier0 is generated.
   auto hlo_module =
       ParseHloString(kHloString, ConfigWithoutLayoutAssignment()).ValueOrDie();
   CompileAndVerifyIr(std::move(hlo_module),
@@ -284,6 +284,133 @@ TEST_F(GpuKernelTilingTest, TransposedInputWithoutUnsafeUseTiled) {
   EXPECT_TRUE(RunAndCompareNoHloPasses(kHloString, ErrorSpec{0.0}));
 }
 
+TEST_F(GpuKernelTilingTest, ColumnReductionWithPowerOf2OutputElementsUnrolled) {
+  const char *const kHloString = R"(
+  HloModule column_reduce_powerof2
+
+  reduction {
+    x = f32[] parameter(0)
+    y = f32[] parameter(1)
+    ROOT add = f32[] add(x, y)
+  }
+
+  ENTRY kernel_entry {
+    constant0 = f32[] constant(0)
+    arg1 = f16[1024,512]{1,0} parameter(0)
+    arg1_conv = f32[1024,512]{1,0} convert(arg1)
+    ROOT reduce = f32[512]{0} reduce(arg1_conv, constant0), dimensions={0}, to_apply=reduction
+  })";
+
+  // Check that two calls to llvm.nvvm.atomic are generated.
+  auto hlo_module =
+      ParseHloString(kHloString, ConfigWithoutLayoutAssignment()).ValueOrDie();
+  CompileAndVerifyIr(std::move(hlo_module),
+                     R"(
+; CHECK-LABEL: define void @fusion
+; CHECK: call float @llvm.nvvm.atomic.load.add.f32.p0f32
+; CHECK: call float @llvm.nvvm.atomic.load.add.f32.p0f32
+; CHECK-NOT: call float @llvm.nvvm.atomic.load.add.f32.p0f32
+; CHECK: }
+)",
+                     /*match_optimized_ir=*/true);
+  // Check that the kernel runs correctly.
+  EXPECT_TRUE(RunAndCompareNoHloPasses(kHloString, ErrorSpec{1.0e-5, 1.0e-5}));
+}
+
+TEST_F(GpuKernelTilingTest,
+       ColumnReductionWithInputLargerThenReduceInputNotUnrolled) {
+  const char *const kHloString = R"(
+  HloModule larger_than_reduce_input_parameter
+
+  reduction22 {
+    x = f32[] parameter(0)
+    y = f32[] parameter(1)
+    ROOT add = f32[] add(x, y)
+  }
+
+  fused_computation {
+    constant0 = f32[] constant(0)
+    arg.1 = f16[1024,512]{1,0} parameter(0)
+    arg.2 = f16[1027,513]{1,0} parameter(1)
+    arg1.conv = f32[1024,512]{1,0} convert(arg.1)
+    arg2.conv = f32[1027,513]{1,0} convert(arg.2)
+    slice2 = f32[1024,512]{1,0} slice(arg2.conv), slice={[2:1026], [1:513]}
+    add2 = f32[1024,512]{1,0} add(arg1.conv, slice2)
+    ROOT reduce = f32[512]{0} reduce(add2, constant0), dimensions={0},
+      to_apply=reduction22
+  }
+
+  ENTRY kernel_entry {
+    arg1 = f16[1024,512]{1,0} parameter(0)
+    arg2 = f16[1027,513]{1,0} parameter(1)
+    ROOT fusion = f32[512]{0} fusion(arg1, arg2), kind=kInput,
+      calls=fused_computation
+  })";
+
+  // Check that one call to llvm.nvvm.atomic is generated.
+  auto hlo_module =
+      ParseHloString(kHloString, ConfigWithoutLayoutAssignment()).ValueOrDie();
+  CompileAndVerifyIr(std::move(hlo_module),
+                     R"(
+; CHECK-LABEL: define void @fusion
+; CHECK: call float @llvm.nvvm.atomic.load.add.f32.p0f32
+; CHECK-NOT: call float @llvm.nvvm.atomic.load.add.f32.p0f32
+; CHECK: }
+)",
+                     /*match_optimized_ir=*/true);
+  // Check that the kernel runs correctly.
+  EXPECT_TRUE(RunAndCompareNoHloPasses(kHloString, ErrorSpec{1.0e-5, 1.0e-5}));
+}
+
+TEST_F(GpuKernelTilingTest, ColumnReductionMOFUnrolled) {
+  const char *const kHloString = R"(
+  HloModule column_reduce_powerof2_mof
+
+  reduction22 {
+    x = f32[] parameter(0)
+    y = f32[] parameter(1)
+    ROOT add = f32[] add(x, y)
+  }
+
+  fused_computation {
+    constant0 = f32[] constant(0)
+    arg.1 = f16[1024,512]{1,0} parameter(0)
+    arg.2 = f16[1024,512]{1,0} parameter(1)
+    arg1.conv = f32[1024,512]{1,0} convert(arg.1)
+    arg2.conv = f32[1024,512]{1,0} convert(arg.2)
+    reduce1 = f32[512]{0} reduce(arg1.conv, constant0), dimensions={0},
+      to_apply=reduction22
+    reduce2 = f32[512]{0} reduce(arg2.conv, constant0), dimensions={0},
+      to_apply=reduction22
+    add = f32[1024,512]{1,0} add(arg1.conv, arg2.conv)
+    ROOT tuple = (f32[512]{0}, f32[512]{0}, f32[1024,512]{1,0})
+      tuple(reduce1, reduce2, add)
+  }
+
+  ENTRY kernel_entry {
+    arg1 = f16[1024,512]{1,0} parameter(0)
+    arg2 = f16[1024,512]{1,0} parameter(1)
+    ROOT fusion = (f32[512]{0}, f32[512]{0}, f32[1024,512]{1,0})
+      fusion(arg1, arg2), kind=kInput, calls=fused_computation
+  })";
+
+  // Check that four calls to llvm.nvvm.atomic are generated.
+  auto hlo_module =
+      ParseHloString(kHloString, ConfigWithoutLayoutAssignment()).ValueOrDie();
+  CompileAndVerifyIr(std::move(hlo_module),
+                     R"(
+; CHECK-LABEL: define void @fusion
+; CHECK: call float @llvm.nvvm.atomic.load.add.f32.p0f32
+; CHECK: call float @llvm.nvvm.atomic.load.add.f32.p0f32
+; CHECK: call float @llvm.nvvm.atomic.load.add.f32.p0f32
+; CHECK: call float @llvm.nvvm.atomic.load.add.f32.p0f32
+; CHECK-NOT: call float @llvm.nvvm.atomic.load.add.f32.p0f32
+; CHECK: }
+)",
+                     /*match_optimized_ir=*/true);
+  // Check that the kernel runs correctly.
+  EXPECT_TRUE(RunAndCompareNoHloPasses(kHloString, ErrorSpec{1.0e-5, 1.0e-5}));
+}
 }  // namespace
 }  // namespace gpu
 }  // namespace xla
-- 
GitLab


From 84bf8020f3a99dc49116417a6430c7a51e349e14 Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Wed, 9 Jan 2019 17:25:33 -0800
Subject: [PATCH 0508/1095] Set device on Identity ops representing inlined
 function inputs/outputs.

This fixes a strange multi-GPU performance issue where the identities from inlined cond_v2 branch functions (part of the control flow lowering pass) were being placed alone on a GPU, creating unnecessary device traffic. Ideally the placer wouldn't be this stupid or grappler would remove the useless identities, but this fix is easier for now.

This also adds a simple benchmark testing nested function calls, which currently are inlined. This benchmark doesn't expose the original problem (all of the ops, including the identities added by inlining, are placed on a single device regardless), but I've included it in case it catches any future regressions. I also increased the number of iterations, as I found this made the results more stable.

PiperOrigin-RevId: 228619628
---
 tensorflow/core/common_runtime/function.cc |  4 ++++
 tensorflow/python/eager/benchmarks_test.py | 25 +++++++++++++++++++++-
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/common_runtime/function.cc b/tensorflow/core/common_runtime/function.cc
index 24a0eee0df..48f32df275 100644
--- a/tensorflow/core/common_runtime/function.cc
+++ b/tensorflow/core/common_runtime/function.cc
@@ -104,6 +104,10 @@ static Node* AddIdentity(Graph* g, Endpoint input) {
   NodeDef ndef;
   ndef.set_name(g->NewName(kNodeLabel));
   ndef.set_op("Identity");
+  // NOTE(skyewm): we explicitly set the device here to address a multi-GPU
+  // performance issue where this Identity would be placed alone on a GPU,
+  // causing unnecessary device traffic. See b/122483225 for details.
+  ndef.set_device(input.node->def().device());
   ndef.add_input(input.name());
   AddNodeAttr("T", BaseType(input.dtype()), &ndef);
   Status s;
diff --git a/tensorflow/python/eager/benchmarks_test.py b/tensorflow/python/eager/benchmarks_test.py
index 9191c8e689..62c4a12cbf 100644
--- a/tensorflow/python/eager/benchmarks_test.py
+++ b/tensorflow/python/eager/benchmarks_test.py
@@ -140,7 +140,7 @@ class MicroBenchmarks(test.Benchmark):
     self._m_2_by_2 = random_ops.random_uniform((2, 2))
     self._m_100_by_784 = random_ops.random_uniform((100, 784))
     self._num_iters_2_by_2 = 30000
-    self._num_iters_100_by_784 = 1000
+    self._num_iters_100_by_784 = 30000
 
   def _run(self, func, num_iters, execution_mode=None):
     # call func to maybe warm up the GPU
@@ -370,6 +370,19 @@ class MicroBenchmarks(test.Benchmark):
     func = lambda: f(m, m, transpose_b=transpose_b)
     self._run(func, num_iters, execution_mode=execution_mode)
 
+  def _benchmark_nested_defun_matmul(self, m, transpose_b, num_iters):
+    inner = function.defun(math_ops.matmul)
+
+    @function.defun
+    def outer(a, b, c, transpose_b):
+      return math_ops.matmul(inner(a, b, transpose_b=transpose_b), c)
+
+    func = lambda: outer(m, m, m, transpose_b=transpose_b)
+    # Warmup before benchmark
+    for _ in range(1000):
+      func()
+    self._run(func, num_iters)
+
   def _benchmark_defun_matmul_forward_backward(self,
                                                m,
                                                transpose_b,
@@ -525,6 +538,11 @@ class MicroBenchmarks(test.Benchmark):
           num_iters=self._num_iters_2_by_2,
           execution_mode=context.ASYNC)
 
+  def benchmark_nested_defun_matmul_2_by_2(self):
+    m = self._m_2_by_2.cpu()
+    self._benchmark_nested_defun_matmul(
+        m, transpose_b=False, num_iters=self._num_iters_2_by_2)
+
   # Benchmarks for AA.T, A of dimension 100 by 784.
   def benchmark_np_matmul_100_by_784(self):
     self._benchmark_np_matmul(
@@ -614,6 +632,11 @@ class MicroBenchmarks(test.Benchmark):
       self._benchmark_defun_matmul(
           m, transpose_b=True, num_iters=self._num_iters_100_by_784)
 
+  def benchmark_nested_defun_matmul_100_by_784(self):
+    m = self._m_100_by_784.gpu()
+    self._benchmark_nested_defun_matmul(
+        m, transpose_b=True, num_iters=self._num_iters_100_by_784)
+
   def benchmark_defun_without_signature(self):
 
     def func(t1, t2, t3, t4, t5, t6, t7, t8):
-- 
GitLab


From 0d53e22c582605d4f7f2a26b179a637a2d469206 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 9 Jan 2019 17:30:56 -0800
Subject: [PATCH 0509/1095] Refactor pfor tests into multiple files.

PiperOrigin-RevId: 228620325
---
 tensorflow/python/ops/parallel_for/BUILD      |  47 +-
 .../python/ops/parallel_for/array_test.py     | 274 +++++++
 .../ops/parallel_for/control_flow_ops_test.py | 693 ++----------------
 .../python/ops/parallel_for/gradients_test.py |   2 +
 .../python/ops/parallel_for/math_test.py      | 405 ++++++++++
 .../python/ops/parallel_for/test_util.py      |  59 ++
 6 files changed, 830 insertions(+), 650 deletions(-)
 create mode 100644 tensorflow/python/ops/parallel_for/array_test.py
 create mode 100644 tensorflow/python/ops/parallel_for/math_test.py
 create mode 100644 tensorflow/python/ops/parallel_for/test_util.py

diff --git a/tensorflow/python/ops/parallel_for/BUILD b/tensorflow/python/ops/parallel_for/BUILD
index 07fc9433a2..1028963b1a 100644
--- a/tensorflow/python/ops/parallel_for/BUILD
+++ b/tensorflow/python/ops/parallel_for/BUILD
@@ -15,11 +15,13 @@ py_library(
         "control_flow_ops.py",
         "gradients.py",
         "pfor.py",
+        "test_util.py",
     ],
     srcs_version = "PY2AND3",
     deps = [
         ":control_flow_ops",
         ":gradients",
+        ":test_util",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:check_ops",
         "//tensorflow/python:constant_op",
@@ -83,12 +85,25 @@ py_library(
     ],
 )
 
+py_library(
+    name = "test_util",
+    srcs = ["test_util.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":pfor_lib",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:util",
+        "//tensorflow/python:variables",
+    ],
+)
+
 cuda_py_test(
     name = "control_flow_ops_test",
-    size = "large",
     srcs = ["control_flow_ops_test.py"],
     additional_deps = [
         ":control_flow_ops",
+        ":test_util",
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:gradients",
@@ -101,6 +116,34 @@ cuda_py_test(
     ],
 )
 
+cuda_py_test(
+    name = "array_test",
+    srcs = ["array_test.py"],
+    additional_deps = [
+        ":control_flow_ops",
+        ":test_util",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:random_ops",
+        "//tensorflow/python:util",
+        "//tensorflow/python/eager:backprop",
+    ],
+)
+
+cuda_py_test(
+    name = "math_test",
+    srcs = ["math_test.py"],
+    additional_deps = [
+        ":control_flow_ops",
+        ":test_util",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:random_ops",
+        "//tensorflow/python:util",
+    ],
+    tags = ["optonly"],  # Too slow in non-opt mode
+)
+
 py_library(
     name = "gradients",
     srcs = ["gradients.py"],
@@ -115,7 +158,6 @@ py_library(
 
 cuda_py_test(
     name = "gradients_test",
-    size = "large",
     srcs = ["gradients_test.py"],
     additional_deps = [
         ":control_flow_ops",
@@ -128,4 +170,5 @@ cuda_py_test(
         "//tensorflow/python:random_ops",
         "//tensorflow/python/ops/losses",
     ],
+    tags = ["optonly"],  # Too slow in non-opt mode
 )
diff --git a/tensorflow/python/ops/parallel_for/array_test.py b/tensorflow/python/ops/parallel_for/array_test.py
new file mode 100644
index 0000000000..7f0c0f5b99
--- /dev/null
+++ b/tensorflow/python/ops/parallel_for/array_test.py
@@ -0,0 +1,274 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for vectorization of array kernels."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.eager import backprop
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import nn
+from tensorflow.python.ops import random_ops
+from tensorflow.python.ops import tensor_array_grad  # pylint: disable=unused-import
+from tensorflow.python.ops.parallel_for import control_flow_ops as pfor_control_flow_ops
+from tensorflow.python.ops.parallel_for.test_util import PForTestCase
+from tensorflow.python.platform import test
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class ArrayTest(PForTestCase):
+
+  def test_gather(self):
+    x = random_ops.random_uniform([3, 3, 3])
+
+    def loop_fn(i):
+      outputs = []
+      x_i = array_ops.gather(x, i)
+      for y in [x, x_i]:
+        axes = [0, 2, -1] if y == x else [0]
+        for axis in axes:
+          outputs.append(array_ops.gather(y, 2, axis=axis))
+          outputs.append(array_ops.gather(y, i, axis=axis))
+          outputs.append(array_ops.gather(y, [i], axis=axis))
+          outputs.append(array_ops.gather(y, [i, 2], axis=axis))
+          outputs.append(array_ops.gather(y, [[2, i], [i, 1]], axis=axis))
+      return outputs
+
+    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 20)
+
+  def test_shape(self):
+    x = random_ops.random_uniform([3, 2, 3])
+
+    def loop_fn(i):
+      x_i = array_ops.gather(x, i)
+      return array_ops.shape(x_i), array_ops.shape(x_i, out_type=dtypes.int64)
+
+    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.int32, dtypes.int64])
+
+  def test_size(self):
+    x = random_ops.random_uniform([3, 2, 3])
+
+    def loop_fn(i):
+      x_i = array_ops.gather(x, i)
+      return array_ops.size(x_i), array_ops.size(x_i, out_type=dtypes.int64)
+
+    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.int32, dtypes.int64])
+
+  def test_rank(self):
+    x = random_ops.random_uniform([3, 2, 3])
+
+    def loop_fn(i):
+      x_i = array_ops.gather(x, i)
+      return array_ops.rank(x_i)
+
+    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.int32])
+
+  def test_shape_n(self):
+    x = random_ops.random_uniform([3, 2, 3])
+    y = random_ops.random_uniform([3])
+
+    def loop_fn(i):
+      x_i = array_ops.gather(x, i)
+      y_i = array_ops.gather(y, i)
+      return array_ops.shape_n([x_i, x, y, y_i]), array_ops.shape_n(
+          [x_i, x, y, y_i], out_type=dtypes.int64)
+
+    self._test_loop_fn(
+        loop_fn, 3, loop_fn_dtypes=[dtypes.int32] * 4 + [dtypes.int64] * 4)
+
+  def test_reshape(self):
+    x = random_ops.random_uniform([3, 2, 3])
+
+    def loop_fn(i):
+      x1 = array_ops.gather(x, i)
+      return array_ops.reshape(x1, [-1]), array_ops.reshape(x1, [1, 3, 1, -1])
+
+    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 2)
+
+  def test_expand_dims(self):
+    x = random_ops.random_uniform([3, 2, 3])
+
+    def loop_fn(i):
+      x1 = array_ops.gather(x, i)
+      return array_ops.expand_dims(
+          x1, axis=-1), array_ops.expand_dims(
+              x1, axis=1)
+
+    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 2)
+
+  def test_slice(self):
+    x = random_ops.random_uniform([3, 2, 3])
+
+    def loop_fn(i):
+      x1 = array_ops.gather(x, i)
+      return array_ops.slice(x1, begin=(0, 1), size=(2, 1))
+
+    self._test_loop_fn(loop_fn, 3)
+
+  def test_tile(self):
+    x = random_ops.random_uniform([3, 2, 3])
+
+    def loop_fn(i):
+      x1 = array_ops.gather(x, i)
+      return array_ops.tile(x1, [2, 1])
+
+    self._test_loop_fn(loop_fn, 3)
+
+  def test_tile_loop_dependent(self):
+    x = random_ops.random_uniform([3, 2, 3])
+
+    def loop_fn(i):
+      x1 = array_ops.gather(x, i)
+      return array_ops.tile(x1, [i, 1])
+
+    with self.assertRaisesRegexp(ValueError, "expected to be loop invariant"):
+      pfor_control_flow_ops.pfor(loop_fn, 2)
+
+  def test_pack(self):
+    x = random_ops.random_uniform([3, 2, 3])
+    y = random_ops.random_uniform([2, 3])
+
+    def loop_fn(i):
+      x1 = array_ops.gather(x, i)
+      return array_ops.stack([x1, y], axis=-1)
+
+    self._test_loop_fn(loop_fn, 1)
+
+  def test_unpack(self):
+    x = random_ops.random_uniform([3, 2, 3, 4])
+
+    def loop_fn(i):
+      x_i = array_ops.gather(x, i)
+      return array_ops.unstack(
+          x_i, 4, axis=-1), array_ops.unstack(
+              x_i, 3, axis=1)
+
+    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 7)
+
+  def test_pad(self):
+    x = random_ops.random_uniform([3, 2, 3])
+    padding = constant_op.constant([[1, 2], [3, 4]])
+
+    def loop_fn(i):
+      x1 = array_ops.gather(x, i)
+      return array_ops.pad(x1, padding, mode="CONSTANT")
+
+    self._test_loop_fn(loop_fn, 3)
+
+  def test_split(self):
+    x = random_ops.random_uniform([3, 2, 3])
+
+    def loop_fn(i):
+      x1 = array_ops.gather(x, i)
+      return array_ops.split(x1, 2, axis=0), array_ops.split(x1, 3, axis=-1)
+
+    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 5)
+
+  def test_split_v(self):
+    x = random_ops.random_uniform([3, 6, 3])
+
+    def loop_fn(i):
+      x1 = array_ops.gather(x, i)
+      return (array_ops.split(x1, [2, 1, 3], axis=0),
+              array_ops.split(x1, [3], axis=-1))
+
+    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 4)
+
+  def test_transpose(self):
+    x = random_ops.random_uniform([3, 2, 3, 4])
+
+    def loop_fn(i):
+      x1 = array_ops.gather(x, i)
+      return array_ops.transpose(x1, [2, 1, 0])
+
+    self._test_loop_fn(loop_fn, 3)
+
+  def test_zeros_like(self):
+    x = random_ops.random_uniform([3, 2, 3])
+
+    def loop_fn(i):
+      x1 = array_ops.gather(x, i)
+      z = array_ops.zeros_like(x1),
+      return z, z + x1
+
+    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 2)
+
+  def test_concat_v2(self):
+    x = random_ops.random_uniform([3, 2, 3])
+    y = random_ops.random_uniform([2, 3])
+
+    def loop_fn(i):
+      x1 = array_ops.gather(x, i)
+      return array_ops.concat(
+          [x1, x1, y], axis=0), array_ops.concat(
+              [x1, x1, y], axis=-1)
+
+    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 2)
+
+  def test_unary_cwise_ops(self):
+    for op in [array_ops.identity, array_ops.stop_gradient]:
+      with backprop.GradientTape(persistent=True) as g:
+        x = random_ops.random_uniform([3, 5])
+        g.watch(x)
+
+      # pylint: disable=cell-var-from-loop
+      def loop_fn(i):
+        with g:
+          x1 = array_ops.gather(x, i)
+          y = op(x1) + x1
+          loss = nn.l2_loss(y)
+        return op(x), y, g.gradient(loss, x1)
+
+      # pylint: enable=cell-var-from-loop
+
+      self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 3)
+
+  def test_identity_n(self):
+    x = random_ops.random_uniform([3, 4])
+
+    def loop_fn(i):
+      return array_ops.identity_n([x, array_ops.gather(x, i)])
+
+    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 2)
+
+  def test_matrix_diag_part(self):
+    x = random_ops.random_uniform([3, 4, 2])
+
+    def loop_fn(i):
+      return array_ops.matrix_diag_part(array_ops.gather(x, i))
+
+    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32])
+
+  def test_strided_slice(self):
+    with backprop.GradientTape(persistent=True) as g:
+      x = random_ops.random_uniform([3, 3, 4, 4, 2, 2, 2])
+      g.watch(x)
+
+    def loop_fn(i):
+      with g:
+        x_i = array_ops.gather(x, i)
+        y = x_i[:2, ::2, 1::3, ..., array_ops.newaxis, 1]
+        loss = nn.l2_loss(y)
+      return y, g.gradient(loss, x_i)
+
+    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 2)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/ops/parallel_for/control_flow_ops_test.py b/tensorflow/python/ops/parallel_for/control_flow_ops_test.py
index 933bddd8cc..1a8f639d43 100644
--- a/tensorflow/python/ops/parallel_for/control_flow_ops_test.py
+++ b/tensorflow/python/ops/parallel_for/control_flow_ops_test.py
@@ -34,7 +34,6 @@ from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import bitwise_ops
-from tensorflow.python.ops import clip_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import data_flow_ops
 from tensorflow.python.ops import functional_ops
@@ -50,40 +49,13 @@ from tensorflow.python.ops import tensor_array_grad  # pylint: disable=unused-im
 from tensorflow.python.ops import tensor_array_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.ops.parallel_for import control_flow_ops as pfor_control_flow_ops
+from tensorflow.python.ops.parallel_for.test_util import PForTestCase
 from tensorflow.python.platform import test
 from tensorflow.python.util import nest
 
 
 @test_util.run_all_in_graph_and_eager_modes
-class PForTest(test.TestCase):
-
-  def _run_targets(self, targets1, targets2=None, run_init=True):
-    targets1 = nest.flatten(targets1)
-    targets2 = ([] if targets2 is None else nest.flatten(targets2))
-    assert len(targets1) == len(targets2) or not targets2
-    if run_init:
-      init = variables.global_variables_initializer()
-      self.evaluate(init)
-    return self.evaluate(targets1 + targets2)
-
-  def run_and_assert_equal(self, targets1, targets2):
-    outputs = self._run_targets(targets1, targets2)
-    outputs = nest.flatten(outputs)  # flatten SparseTensorValues
-    n = len(outputs) // 2
-    for i in range(n):
-      if outputs[i + n].dtype != np.object:
-        self.assertAllClose(outputs[i + n], outputs[i], rtol=1e-4, atol=1e-5)
-      else:
-        self.assertAllEqual(outputs[i + n], outputs[i])
-
-  def _test_loop_fn(self, loop_fn, iters,
-                    loop_fn_dtypes=dtypes.float32,
-                    parallel_iterations=None):
-    t1 = pfor_control_flow_ops.pfor(loop_fn, iters=iters,
-                                    parallel_iterations=parallel_iterations)
-    t2 = pfor_control_flow_ops.for_loop(loop_fn, loop_fn_dtypes, iters=iters,
-                                        parallel_iterations=parallel_iterations)
-    self.run_and_assert_equal(t1, t2)
+class PForTest(PForTestCase):
 
   def test_op_conversion_fallback_to_while_loop(self):
     # Note that we used top_k op for this test. If a converter gets defined for
@@ -129,246 +101,7 @@ class PForTest(test.TestCase):
 
 
 @test_util.run_all_in_graph_and_eager_modes
-class ArrayTest(PForTest):
-
-  def test_gather(self):
-    x = random_ops.random_uniform([3, 3, 3])
-
-    def loop_fn(i):
-      outputs = []
-      x_i = array_ops.gather(x, i)
-      for y in [x, x_i]:
-        axes = [0, 2, -1] if y == x else [0]
-        for axis in axes:
-          outputs.append(array_ops.gather(y, 2, axis=axis))
-          outputs.append(array_ops.gather(y, i, axis=axis))
-          outputs.append(array_ops.gather(y, [i], axis=axis))
-          outputs.append(array_ops.gather(y, [i, 2], axis=axis))
-          outputs.append(array_ops.gather(y, [[2, i], [i, 1]], axis=axis))
-      return outputs
-
-    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 20)
-
-  def test_shape(self):
-    x = random_ops.random_uniform([3, 2, 3])
-
-    def loop_fn(i):
-      x_i = array_ops.gather(x, i)
-      return array_ops.shape(x_i), array_ops.shape(x_i, out_type=dtypes.int64)
-
-    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.int32, dtypes.int64])
-
-  def test_size(self):
-    x = random_ops.random_uniform([3, 2, 3])
-
-    def loop_fn(i):
-      x_i = array_ops.gather(x, i)
-      return array_ops.size(x_i), array_ops.size(x_i, out_type=dtypes.int64)
-
-    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.int32, dtypes.int64])
-
-  def test_rank(self):
-    x = random_ops.random_uniform([3, 2, 3])
-
-    def loop_fn(i):
-      x_i = array_ops.gather(x, i)
-      return array_ops.rank(x_i)
-
-    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.int32])
-
-  def test_shape_n(self):
-    x = random_ops.random_uniform([3, 2, 3])
-    y = random_ops.random_uniform([3])
-
-    def loop_fn(i):
-      x_i = array_ops.gather(x, i)
-      y_i = array_ops.gather(y, i)
-      return array_ops.shape_n([x_i, x, y, y_i]), array_ops.shape_n(
-          [x_i, x, y, y_i], out_type=dtypes.int64)
-
-    self._test_loop_fn(
-        loop_fn, 3, loop_fn_dtypes=[dtypes.int32] * 4 + [dtypes.int64] * 4)
-
-  def test_reshape(self):
-    x = random_ops.random_uniform([3, 2, 3])
-
-    def loop_fn(i):
-      x1 = array_ops.gather(x, i)
-      return array_ops.reshape(x1, [-1]), array_ops.reshape(x1, [1, 3, 1, -1])
-
-    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 2)
-
-  def test_expand_dims(self):
-    x = random_ops.random_uniform([3, 2, 3])
-
-    def loop_fn(i):
-      x1 = array_ops.gather(x, i)
-      return array_ops.expand_dims(
-          x1, axis=-1), array_ops.expand_dims(
-              x1, axis=1)
-
-    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 2)
-
-  def test_slice(self):
-    x = random_ops.random_uniform([3, 2, 3])
-
-    def loop_fn(i):
-      x1 = array_ops.gather(x, i)
-      return array_ops.slice(x1, begin=(0, 1), size=(2, 1))
-
-    self._test_loop_fn(loop_fn, 3)
-
-  def test_tile(self):
-    x = random_ops.random_uniform([3, 2, 3])
-
-    def loop_fn(i):
-      x1 = array_ops.gather(x, i)
-      return array_ops.tile(x1, [2, 1])
-
-    self._test_loop_fn(loop_fn, 3)
-
-  def test_tile_loop_dependent(self):
-    x = random_ops.random_uniform([3, 2, 3])
-
-    def loop_fn(i):
-      x1 = array_ops.gather(x, i)
-      return array_ops.tile(x1, [i, 1])
-
-    with self.assertRaisesRegexp(ValueError, "expected to be loop invariant"):
-      pfor_control_flow_ops.pfor(loop_fn, 2)
-
-  def test_pack(self):
-    x = random_ops.random_uniform([3, 2, 3])
-    y = random_ops.random_uniform([2, 3])
-
-    def loop_fn(i):
-      x1 = array_ops.gather(x, i)
-      return array_ops.stack([x1, y], axis=-1)
-
-    self._test_loop_fn(loop_fn, 1)
-
-  def test_unpack(self):
-    x = random_ops.random_uniform([3, 2, 3, 4])
-
-    def loop_fn(i):
-      x_i = array_ops.gather(x, i)
-      return array_ops.unstack(
-          x_i, 4, axis=-1), array_ops.unstack(
-              x_i, 3, axis=1)
-
-    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 7)
-
-  def test_pad(self):
-    x = random_ops.random_uniform([3, 2, 3])
-    padding = constant_op.constant([[1, 2], [3, 4]])
-
-    def loop_fn(i):
-      x1 = array_ops.gather(x, i)
-      return array_ops.pad(x1, padding, mode="CONSTANT")
-
-    self._test_loop_fn(loop_fn, 3)
-
-  def test_split(self):
-    x = random_ops.random_uniform([3, 2, 3])
-
-    def loop_fn(i):
-      x1 = array_ops.gather(x, i)
-      return array_ops.split(x1, 2, axis=0), array_ops.split(x1, 3, axis=-1)
-
-    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 5)
-
-  def test_split_v(self):
-    x = random_ops.random_uniform([3, 6, 3])
-
-    def loop_fn(i):
-      x1 = array_ops.gather(x, i)
-      return (array_ops.split(x1, [2, 1, 3], axis=0),
-              array_ops.split(x1, [3], axis=-1))
-
-    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 4)
-
-  def test_transpose(self):
-    x = random_ops.random_uniform([3, 2, 3, 4])
-
-    def loop_fn(i):
-      x1 = array_ops.gather(x, i)
-      return array_ops.transpose(x1, [2, 1, 0])
-
-    self._test_loop_fn(loop_fn, 3)
-
-  def test_zeros_like(self):
-    x = random_ops.random_uniform([3, 2, 3])
-
-    def loop_fn(i):
-      x1 = array_ops.gather(x, i)
-      z = array_ops.zeros_like(x1),
-      return z, z + x1
-
-    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 2)
-
-  def test_concat_v2(self):
-    x = random_ops.random_uniform([3, 2, 3])
-    y = random_ops.random_uniform([2, 3])
-
-    def loop_fn(i):
-      x1 = array_ops.gather(x, i)
-      return array_ops.concat(
-          [x1, x1, y], axis=0), array_ops.concat(
-              [x1, x1, y], axis=-1)
-
-    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 2)
-
-  def test_unary_cwise_ops(self):
-    for op in [array_ops.identity, array_ops.stop_gradient]:
-      with backprop.GradientTape(persistent=True) as g:
-        x = random_ops.random_uniform([3, 5])
-        g.watch(x)
-
-      # pylint: disable=cell-var-from-loop
-      def loop_fn(i):
-        with g:
-          x1 = array_ops.gather(x, i)
-          y = op(x1) + x1
-          loss = nn.l2_loss(y)
-        return op(x), y, g.gradient(loss, x1)
-
-      # pylint: enable=cell-var-from-loop
-
-      self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 3)
-
-  def test_identity_n(self):
-    x = random_ops.random_uniform([3, 4])
-
-    def loop_fn(i):
-      return array_ops.identity_n([x, array_ops.gather(x, i)])
-
-    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 2)
-
-  def test_matrix_diag_part(self):
-    x = random_ops.random_uniform([3, 4, 2])
-
-    def loop_fn(i):
-      return array_ops.matrix_diag_part(array_ops.gather(x, i))
-
-    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32])
-
-  def test_strided_slice(self):
-    with backprop.GradientTape(persistent=True) as g:
-      x = random_ops.random_uniform([3, 3, 4, 4, 2, 2, 2])
-      g.watch(x)
-
-    def loop_fn(i):
-      with g:
-        x_i = array_ops.gather(x, i)
-        y = x_i[:2, ::2, 1::3, ..., array_ops.newaxis, 1]
-        loss = nn.l2_loss(y)
-      return y, g.gradient(loss, x_i)
-
-    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 2)
-
-
-@test_util.run_all_in_graph_and_eager_modes
-class BitwiseTest(PForTest):
+class BitwiseTest(PForTestCase):
 
   def test_unary_cwise(self):
     for op in [bitwise_ops.invert]:
@@ -408,376 +141,7 @@ class BitwiseTest(PForTest):
 
 
 @test_util.run_all_in_graph_and_eager_modes
-class MathTest(PForTest):
-
-  def test_unary_cwise_ops(self):
-    complex_ops = [
-        math_ops.angle,
-        math_ops.imag,
-        math_ops.complex_abs,
-        math_ops.real,
-        math_ops.conj,
-    ]
-    real_ops = [
-        lambda x: math_ops.acosh(1 + math_ops.square(x)),
-        math_ops.abs,
-        math_ops.acos,
-        math_ops.asin,
-        math_ops.asinh,
-        math_ops.atan,
-        math_ops.atanh,
-        math_ops.bessel_i0e,
-        math_ops.bessel_i1e,
-        math_ops.cos,
-        math_ops.cosh,
-        math_ops.digamma,
-        math_ops.erf,
-        math_ops.erfc,
-        math_ops.exp,
-        math_ops.expm1,
-        math_ops.inv,
-        math_ops.is_finite,
-        math_ops.is_inf,
-        math_ops.lgamma,
-        math_ops.log,
-        math_ops.log1p,
-        math_ops.neg,
-        math_ops.negative,
-        math_ops.reciprocal,
-        math_ops.rint,
-        math_ops.round,
-        math_ops.rsqrt,
-        math_ops.sigmoid,
-        math_ops.sign,
-        math_ops.sin,
-        math_ops.sinh,
-        math_ops.sqrt,
-        math_ops.square,
-        math_ops.tan,
-        math_ops.tanh,
-        math_ops.tanh,
-        nn.elu,
-        nn.relu,
-        nn.relu6,
-        nn.selu,
-        nn.softplus,
-        nn.softsign,
-    ]
-    for op in complex_ops + real_ops:
-      with backprop.GradientTape(persistent=True) as g:
-        x = random_ops.random_uniform([3, 5])
-        g.watch(x)
-        if op in complex_ops:
-          y = random_ops.random_uniform([3, 5])
-          g.watch(y)
-          x = math_ops.complex(x, y)
-
-      # pylint: disable=cell-var-from-loop
-      output_dtypes = []
-      def loop_fn(i):
-        with g:
-          x1 = array_ops.gather(x, i)
-          y1 = op(x1)
-          outputs = [op(x), y1]
-          if y1.dtype == dtypes.float32:
-            loss = math_ops.reduce_sum(y1 * y1)
-          else:
-            loss = None
-        if loss is not None:
-          grad = g.gradient(loss, x1)
-          if grad is not None:
-            outputs.append(grad)
-        del output_dtypes[:]
-        output_dtypes.extend([t.dtype for t in outputs])
-        return outputs
-
-      # pylint: enable=cell-var-from-loop
-
-      self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=output_dtypes)
-
-  def test_unary_cwise_no_grad(self):
-    for op in [math_ops.ceil,
-               math_ops.floor,
-               math_ops.logical_not]:
-      x = random_ops.random_uniform([3, 5])
-      if op == math_ops.logical_not:
-        x = x > 0
-
-      # pylint: disable=cell-var-from-loop
-      def loop_fn(i):
-        return op(array_ops.gather(x, i))
-
-      # pylint: enable=cell-var-from-loop
-
-      self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=x.dtype)
-
-  def test_binary_cwise_ops(self):
-    logical_ops = [
-        math_ops.logical_and,
-        math_ops.logical_or,
-        math_ops.logical_xor
-    ]
-
-    # Wrapper functions restricting the range of inputs of zeta and polygamma.
-    def safe_polygamma(x, y):
-      return math_ops.polygamma(
-          math_ops.round(clip_ops.clip_by_value(y, 1, 10)),
-          x * x + 1)
-
-    def safe_zeta(x, y):
-      return math_ops.zeta(x * x + 1, y * y)
-
-    float_ops = [
-        math_ops.add,
-        math_ops.add_v2,
-        math_ops.atan2,
-        math_ops.complex,
-        math_ops.div,
-        math_ops.divide,
-        math_ops.div_no_nan,
-        math_ops.equal,
-        math_ops.floor_div,
-        math_ops.floor_mod,
-        math_ops.greater,
-        math_ops.greater_equal,
-        math_ops.igamma,
-        math_ops.igammac,
-        math_ops.igamma_grad_a,
-        math_ops.less,
-        math_ops.less_equal,
-        math_ops.maximum,
-        math_ops.minimum,
-        math_ops.mod,
-        math_ops.multiply,
-        math_ops.not_equal,
-        math_ops.pow,
-        math_ops.squared_difference,
-        math_ops.subtract,
-        math_ops.truncate_mod,
-        safe_polygamma,
-        safe_zeta,
-    ]
-    for op in logical_ops + float_ops:
-      x = random_ops.random_uniform([7, 3, 5])
-      y = random_ops.random_uniform([3, 5])
-      if op in logical_ops:
-        x = x > 0
-        y = y > 0
-
-      output_dtypes = []
-      # pylint: disable=cell-var-from-loop
-      def loop_fn(i):
-        x1 = array_ops.gather(x, i)
-        y1 = array_ops.gather(y, i)
-        outputs = [op(x, y), op(x1, y), op(x, y1), op(x1, y1), op(x1, x1)]
-        del output_dtypes[:]
-        output_dtypes.extend([t.dtype for t in outputs])
-        return outputs
-      # pylint: enable=cell-var-from-loop
-
-      self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=output_dtypes)
-
-  def test_approximate_equal(self):
-    x = random_ops.random_uniform([3, 5])
-    y = random_ops.random_uniform([3, 5])
-
-    def loop_fn(i):
-      x1 = array_ops.gather(x, i)
-      y1 = array_ops.gather(y, i)
-      return math_ops.approximate_equal(x1, y1)
-
-    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.bool])
-
-  def test_addn(self):
-    x = random_ops.random_uniform([2, 3, 5])
-    y = random_ops.random_uniform([3, 5])
-    z = random_ops.random_uniform([3, 5])
-
-    def loop_fn(i):
-      x1 = array_ops.gather(x, i)
-      return math_ops.add_n([x1, y, z])
-
-    self._test_loop_fn(loop_fn, 2)
-
-  def test_matmul(self):
-    for tr_a in (True, False):
-      for tr_b in (True, False):
-        for stack_a in (True, False):
-          for stack_b in (True, False):
-            shape_a = (5, 3) if tr_a else (3, 5)
-            if stack_a:
-              shape_a = (2,) + shape_a
-            shape_b = (7, 5) if tr_b else (5, 7)
-            if stack_b:
-              shape_b = (2,) + shape_b
-
-            x = random_ops.random_uniform(shape_a)
-            y = random_ops.random_uniform(shape_b)
-
-            # pylint: disable=cell-var-from-loop
-            def loop_fn(i):
-              a = array_ops.gather(x, i) if stack_a else x
-              b = array_ops.gather(y, i) if stack_b else y
-              return math_ops.matmul(a, b, transpose_a=tr_a, transpose_b=tr_b)
-
-            # pylint: enable=cell-var-from-loop
-
-            self._test_loop_fn(loop_fn, 2)
-
-  def test_batch_matmul(self):
-    for tr_a in (True, False):
-      for tr_b in (True, False):
-        for stack_a in (True, False):
-          for stack_b in (True, False):
-            shape_a = (4, 5, 3) if tr_a else (4, 3, 5)
-            if stack_a:
-              shape_a = (2,) + shape_a
-            shape_b = (4, 7, 5) if tr_b else (4, 5, 7)
-            if stack_b:
-              shape_b = (2,) + shape_b
-
-            x = random_ops.random_uniform(shape_a)
-            y = random_ops.random_uniform(shape_b)
-
-            # pylint: disable=cell-var-from-loop
-            def loop_fn(i):
-              a = array_ops.gather(x, i) if stack_a else x
-              b = array_ops.gather(y, i) if stack_b else y
-              return math_ops.matmul(a, b, transpose_a=tr_a, transpose_b=tr_b)
-
-            # pylint: enable=cell-var-from-loop
-
-            self._test_loop_fn(loop_fn, 2)
-
-  def test_reduction(self):
-    x = random_ops.random_uniform([2, 3, 4, 5])
-    for op in [
-        math_ops.reduce_sum, math_ops.reduce_prod, math_ops.reduce_max,
-        math_ops.reduce_min
-    ]:
-      for axis in ([1], None, [0, 2]):
-        for keepdims in (True, False):
-
-          # pylint: disable=cell-var-from-loop
-          def loop_fn(i):
-            a = array_ops.gather(x, i)
-            return op(a, axis=axis, keepdims=keepdims)
-
-          # pylint: enable=cell-var-from-loop
-
-          self._test_loop_fn(loop_fn, 2)
-
-  def test_cum_sum(self):
-    x = random_ops.random_uniform([2, 3, 4, 5])
-    for axis in (1, -2):
-      for exclusive in (True, False):
-        for reverse in (True, False):
-
-          # pylint: disable=cell-var-from-loop
-          def loop_fn(i):
-            a = array_ops.gather(x, i)
-            return math_ops.cumsum(
-                a, axis=axis, exclusive=exclusive, reverse=reverse)
-
-          # pylint: enable=cell-var-from-loop
-
-          self._test_loop_fn(loop_fn, 2)
-
-  def test_cum_prod(self):
-    x = random_ops.random_uniform([2, 3, 4, 5])
-    for axis in (1, -2):
-      for exclusive in (True, False):
-        for reverse in (True, False):
-
-          # pylint: disable=cell-var-from-loop
-          def loop_fn(i):
-            a = array_ops.gather(x, i)
-            return math_ops.cumprod(
-                a, axis=axis, exclusive=exclusive, reverse=reverse)
-
-          # pylint: enable=cell-var-from-loop
-
-          self._test_loop_fn(loop_fn, 2)
-
-  def test_bias_add(self):
-    x_shape = [2, 3, 4, 5, 6]
-    x = random_ops.random_uniform(x_shape)
-    for data_format in ("NCHW", "NHWC"):
-      with backprop.GradientTape(persistent=True) as g:
-        bias_dim = 2 if data_format == "NCHW" else -1
-        bias_shape = x_shape[bias_dim]
-        bias = random_ops.random_uniform([bias_shape])
-        g.watch(bias)
-
-      # pylint: disable=cell-var-from-loop
-      def loop_fn(i):
-        with g:
-          a = array_ops.gather(x, i)
-          y = nn.bias_add(a, bias, data_format=data_format)
-          loss = math_ops.reduce_sum(y * y)
-        return y, g.gradient(loss, bias)
-      # pylint: enable=cell-var-from-loop
-
-      self._test_loop_fn(
-          loop_fn, 2, loop_fn_dtypes=[dtypes.float32, dtypes.float32])
-
-  def test_unsorted_segment_sum(self):
-    t = random_ops.random_uniform([3, 3, 2])
-    segment_ids = constant_op.constant([[0, 0, 2], [0, 1, 2], [2, 2, 2]])
-    num_segments = 3
-
-    def loop_fn(i):
-      data = array_ops.gather(t, i)
-      data_0 = array_ops.gather(t, 0)
-      seg_ids = array_ops.gather(segment_ids, i)
-      return (math_ops.unsorted_segment_sum(data, seg_ids, num_segments),
-              math_ops.unsorted_segment_sum(data_0, seg_ids, num_segments))
-
-    self._test_loop_fn(loop_fn, 3, [dtypes.float32] * 2)
-
-  def test_cast(self):
-    x = constant_op.constant([[1], [2]])
-    y = constant_op.constant([[1.0], [2.0]])
-
-    def loop_fn(i):
-      return (math_ops.cast(array_ops.gather(x, i), dtypes.float32),
-              math_ops.cast(array_ops.gather(y, i), dtypes.int32))
-
-    self._test_loop_fn(
-        loop_fn, 2, loop_fn_dtypes=[dtypes.float32, dtypes.int32])
-
-  def test_tanh_axpy(self):
-    a = constant_op.constant(3.)
-    x = random_ops.random_uniform([4, 5])
-    y = random_ops.random_uniform([6, 5])
-    n = x.shape[0]
-
-    def loop_fn(i):
-      return math_ops.tanh(a * array_ops.gather(x, i) + array_ops.gather(y, i))
-
-    self._test_loop_fn(loop_fn, n)
-
-  def test_select(self):
-    cond = constant_op.constant([True, False])
-    a = random_ops.random_uniform([2, 3, 5])
-    b = random_ops.random_uniform([2, 3, 5])
-    for cond_shape in [2], [2, 3], [2, 3, 5]:
-      cond = random_ops.random_uniform(cond_shape) > 0.5
-
-      # pylint: disable=cell-var-from-loop
-      def loop_fn(i):
-        a_i = array_ops.gather(a, i)
-        b_i = array_ops.gather(b, i)
-        cond_i = array_ops.gather(cond, i)
-        return array_ops.where(cond_i, a_i, b_i)
-
-      # pylint: enable=cell-var-from-loop
-
-      self._test_loop_fn(loop_fn, 2)
-
-
-@test_util.run_all_in_graph_and_eager_modes
-class NNTest(PForTest):
+class NNTest(PForTestCase):
 
   def test_conv2d(self):
     x = random_ops.random_uniform([3, 2, 12, 12, 3])
@@ -956,7 +320,7 @@ class NNTest(PForTest):
     self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 2)
 
 
-class RandomTest(PForTest):
+class RandomTest(PForTestCase):
 
   # The random values generated in the two implementations are not guaranteed to
   # match. So we only check the returned shapes.
@@ -1009,8 +373,9 @@ class RandomTest(PForTest):
     self._test_loop_fn(loop_fn, 5)
 
 
-class LoggingTest(PForTest):
+class LoggingTest(PForTestCase):
 
+  @test_util.run_v1_only("b/122612051")
   def test_print(self):
     x = random_ops.random_uniform([3, 5])
 
@@ -1031,8 +396,9 @@ class LoggingTest(PForTest):
       sess.run(pfor_control_flow_ops.pfor(loop_fn, 3))
 
 
-class TensorArrayTest(PForTest):
+class TensorArrayTest(PForTestCase):
 
+  @test_util.run_v1_only("b/122612051")
   def test_create_outside_and_read(self):
 
     ta = tensor_array_ops.TensorArray(
@@ -1043,6 +409,7 @@ class TensorArrayTest(PForTest):
 
     self._test_loop_fn(loop_fn, 2, [dtypes.int32] * 2)
 
+  @test_util.run_v1_only("b/122612051")
   def test_create_outside_and_gather(self):
 
     ta = tensor_array_ops.TensorArray(
@@ -1053,6 +420,7 @@ class TensorArrayTest(PForTest):
 
     self._test_loop_fn(loop_fn, 2, [dtypes.int32] * 2)
 
+  @test_util.run_v1_only("b/122612051")
   def test_create_outside_and_write_and_scatter(self):
 
     t = tensor_array_ops.TensorArray(dtypes.int32, 10, clear_after_read=False)
@@ -1074,6 +442,7 @@ class TensorArrayTest(PForTest):
     output2 = self._run_targets(out2)
     self.assertAllClose(output2, output1)
 
+  @test_util.run_v1_only("b/122612051")
   def test_create_inside_and_write(self):
 
     def loop_fn(i):
@@ -1085,6 +454,7 @@ class TensorArrayTest(PForTest):
 
     self._test_loop_fn(loop_fn, 3, [dtypes.int32] * 2)
 
+  @test_util.run_v1_only("b/122612051")
   def test_create_inside_and_scatter(self):
 
     def loop_fn(i):
@@ -1097,6 +467,7 @@ class TensorArrayTest(PForTest):
 
     self._test_loop_fn(loop_fn, 3, [dtypes.int32] * 2)
 
+  @test_util.run_v1_only("b/122612051")
   def test_create_inside_and_read(self):
 
     def loop_fn(i):
@@ -1109,6 +480,7 @@ class TensorArrayTest(PForTest):
 
     self._test_loop_fn(loop_fn, 2, [dtypes.int32] * 3)
 
+  @test_util.run_v1_only("b/122612051")
   def test_create_inside_and_gather(self):
 
     def loop_fn(i):
@@ -1121,6 +493,7 @@ class TensorArrayTest(PForTest):
 
     self._test_loop_fn(loop_fn, 2, [dtypes.int32] * 3)
 
+  @test_util.run_v1_only("b/122612051")
   def test_grad(self):
     x = random_ops.random_uniform([3, 2])
     ta = tensor_array_ops.TensorArray(
@@ -1140,8 +513,9 @@ class TensorArrayTest(PForTest):
       self.assertAllClose(actual_grad, computed_grad)
 
 
-class StackTest(PForTest):
+class StackTest(PForTestCase):
 
+  @test_util.run_v1_only("b/122612051")
   def test_stack_inside_loop_invariant(self):
 
     def loop_fn(_):
@@ -1157,6 +531,7 @@ class StackTest(PForTest):
 
     self._test_loop_fn(loop_fn, 2, [dtypes.int32] * 2)
 
+  @test_util.run_v1_only("b/122612051")
   def test_stack_inside_push_loop_dependent(self):
 
     def loop_fn(i):
@@ -1172,6 +547,7 @@ class StackTest(PForTest):
 
     self._test_loop_fn(loop_fn, 2, [dtypes.int32] * 2)
 
+  @test_util.run_v1_only("b/122612051")
   def test_stack_outside_pop(self):
     s = data_flow_ops.stack_v2(max_size=4, elem_type=dtypes.int32)
     op = data_flow_ops.stack_push_v2(s, 5)
@@ -1195,6 +571,7 @@ class StackTest(PForTest):
     self.assertAllEqual([6, 6], v2)
     self.assertAllEqual(5, v3)
 
+  @test_util.run_v1_only("b/122612051")
   def test_stack_outside_push(self):
     s = data_flow_ops.stack_v2(max_size=4, elem_type=dtypes.int32)
 
@@ -1207,7 +584,7 @@ class StackTest(PForTest):
 
 # TODO(agarwal): test nested while_loops. This currently requires converting a
 # tf.cond.
-class ControlFlowTest(PForTest):
+class ControlFlowTest(PForTestCase):
 
   def test_while_outside_loop(self):
 
@@ -1218,6 +595,7 @@ class ControlFlowTest(PForTest):
 
     self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.int32])
 
+  @test_util.run_v1_only("b/122612051")
   def test_invariant_while(self):
 
     def loop_fn(_):
@@ -1225,6 +603,7 @@ class ControlFlowTest(PForTest):
 
     self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.int32])
 
+  @test_util.run_v1_only("b/122612051")
   def test_invariant_while_with_control_dependency(self):
 
     def loop_fn(i):
@@ -1234,6 +613,7 @@ class ControlFlowTest(PForTest):
 
     self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.int32])
 
+  @test_util.run_v1_only("b/122612051")
   def test_while_with_stateful_ops(self):
 
     def loop_fn(_):
@@ -1243,6 +623,7 @@ class ControlFlowTest(PForTest):
 
     self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.int32])
 
+  @test_util.run_v1_only("b/122612051")
   def test_while_unstacked_condition(self):
 
     def loop_fn(i):
@@ -1251,6 +632,7 @@ class ControlFlowTest(PForTest):
 
     self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.int32, dtypes.int32])
 
+  @test_util.run_v1_only("b/122612051")
   def test_while(self):
     x = random_ops.random_uniform([3, 5])
     lengths = constant_op.constant([4, 0, 2])
@@ -1266,6 +648,7 @@ class ControlFlowTest(PForTest):
 
     self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32])
 
+  @test_util.run_v1_only("b/122612051")
   def test_while_jacobian(self):
     x = random_ops.random_uniform([1, 3])
     y = random_ops.random_uniform([3, 3])
@@ -1293,6 +676,7 @@ class ControlFlowTest(PForTest):
       out, expected = sess.run([out, expected_output])
       self.assertAllClose(expected, out)
 
+  @test_util.run_v1_only("b/122612051")
   def test_tensor_array_as_loop_variable(self):
 
     def loop_fn(i):
@@ -1308,6 +692,7 @@ class ControlFlowTest(PForTest):
 
     self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.int32])
 
+  @test_util.run_v1_only("b/122612051")
   def test_read_tensor_array_partitioned_indices(self):
     # Note that tensor array values are pfor loop dependent, and the while loop
     # termination condition is also dependent on pfor iteration.
@@ -1325,6 +710,7 @@ class ControlFlowTest(PForTest):
 
     self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.int32])
 
+  @test_util.run_v1_only("b/122612051")
   def test_external_while_loop_grad(self):
     # Here we test that external while_loops that are extended from inside pfor
     # (due to gradient calls) are not actually converted. If the below was
@@ -1350,6 +736,7 @@ class ControlFlowTest(PForTest):
       self.assertAllEqual([1, 1, 1],
                           sess.run(pfor_control_flow_ops.pfor(loop_fn, 3)))
 
+  @test_util.run_v1_only("b/122612051")
   def test_tensor_array_grad(self):
     inp = constant_op.constant(np.random.rand(3, 4, 2), dtype=dtypes.float32)
     ta = tensor_array_ops.TensorArray(dtypes.float32, size=3)
@@ -1447,13 +834,15 @@ def create_dynamic_lstm(cell_fn, batch_size, state_size, max_steps):
   return pfor_output, tf_output
 
 
-class RNNTest(PForTest):
+class RNNTest(PForTestCase):
 
+  @test_util.run_v1_only("b/122612051")
   def test_dynamic_rnn(self):
     pfor_outputs, tf_outputs = create_dynamic_lstm(rnn_cell.BasicRNNCell,
                                                    3, 5, 7)
     self.run_and_assert_equal(pfor_outputs, tf_outputs)
 
+  @test_util.run_v1_only("b/122612051")
   def test_dynamic_lstm(self):
     pfor_outputs, tf_outputs = create_dynamic_lstm(rnn_cell.BasicLSTMCell,
                                                    3, 5, 7)
@@ -1576,8 +965,9 @@ class Benchmarks(test.Benchmark):
       self._run(tf_outputs, 100, name="tf_rnn")
 
 
-class SparseTest(PForTest):
+class SparseTest(PForTestCase):
 
+  @test_util.run_v1_only("b/122612051")
   def test_var_loop_len(self):
     num_iters = array_ops.placeholder(dtypes.int32)
 
@@ -1589,6 +979,7 @@ class SparseTest(PForTest):
     with self.cached_session() as sess:
       sess.run(pfor, feed_dict={num_iters: 3})
 
+  @test_util.run_v1_only("b/122612051")
   def test_sparse_result_none_stacked(self):
     num_iters = 10
 
@@ -1605,6 +996,7 @@ class SparseTest(PForTest):
     manual = sparse_tensor.SparseTensor(indices, values, dense_shapes)
     self.run_and_assert_equal(pfor, manual)
 
+  @test_util.run_v1_only("b/122612051")
   def test_sparse_result_all_stacked(self):
     num_iters = 10
 
@@ -1620,6 +1012,7 @@ class SparseTest(PForTest):
                                         (num_iters, num_iters))
     self.run_and_assert_equal(pfor, manual)
 
+  @test_util.run_v1_only("b/122612051")
   def test_sparse_result_indices_stacked(self):
     num_iters = 10
 
@@ -1634,6 +1027,7 @@ class SparseTest(PForTest):
                                         [1] * num_iters, (num_iters, num_iters))
     self.run_and_assert_equal(pfor, manual)
 
+  @test_util.run_v1_only("b/122612051")
   def test_sparse_result_values_stacked(self):
     num_iters = 10
 
@@ -1648,6 +1042,7 @@ class SparseTest(PForTest):
                                         (num_iters, num_iters))
     self.run_and_assert_equal(pfor, manual)
 
+  @test_util.run_v1_only("b/122612051")
   def test_sparse_result_shapes_stacked(self):
     num_iters = 10
 
@@ -1661,6 +1056,7 @@ class SparseTest(PForTest):
                                         [1] * num_iters, (num_iters, num_iters))
     self.run_and_assert_equal(pfor, manual)
 
+  @test_util.run_v1_only("b/122612051")
   def test_sparse_result_shapes_stacked_2D(self):
     num_iters = 10
 
@@ -1677,7 +1073,7 @@ class SparseTest(PForTest):
     self.run_and_assert_equal(pfor, manual)
 
 
-class ParsingTest(PForTest):
+class ParsingTest(PForTestCase):
 
   def test_decode_csv(self):
     csv_tensor = constant_op.constant([["1:2:3"], ["::"], ["7:8:9"]])
@@ -1689,6 +1085,7 @@ class ParsingTest(PForTest):
 
     self._test_loop_fn(loop_fn, iters=3, loop_fn_dtypes=[dtypes.int32] * 3)
 
+  @test_util.run_v1_only("b/122612051")
   def test_parse_single_example(self):
 
     def _int64_feature(*values):
diff --git a/tensorflow/python/ops/parallel_for/gradients_test.py b/tensorflow/python/ops/parallel_for/gradients_test.py
index 4342833e3e..133e790992 100644
--- a/tensorflow/python/ops/parallel_for/gradients_test.py
+++ b/tensorflow/python/ops/parallel_for/gradients_test.py
@@ -29,6 +29,7 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.keras.engine import training as keras_training
 from tensorflow.python.layers import layers as tf_layers
 from tensorflow.python.ops import array_ops
@@ -338,6 +339,7 @@ def create_fc_per_eg_jacobians(batch_size, activation_size, num_layers):
   return jacobians, per_eg_jacobians_pfor, per_eg_jacobians_while
 
 
+@test_util.run_v1_only("b/122612051")
 class GradientsTest(test.TestCase):
 
   def run_and_assert_equal(self, targets1, targets2, atol=1e-4, rtol=1e-4):
diff --git a/tensorflow/python/ops/parallel_for/math_test.py b/tensorflow/python/ops/parallel_for/math_test.py
new file mode 100644
index 0000000000..db88f4fe03
--- /dev/null
+++ b/tensorflow/python/ops/parallel_for/math_test.py
@@ -0,0 +1,405 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for vectorization of math kernels."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.eager import backprop
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import clip_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import nn
+from tensorflow.python.ops import random_ops
+from tensorflow.python.ops import tensor_array_grad  # pylint: disable=unused-import
+from tensorflow.python.ops.parallel_for.test_util import PForTestCase
+from tensorflow.python.platform import test
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class MathTest(PForTestCase):
+
+  def test_unary_cwise_ops(self):
+    complex_ops = [
+        math_ops.angle,
+        math_ops.imag,
+        math_ops.complex_abs,
+        math_ops.real,
+        math_ops.conj,
+    ]
+    real_ops = [
+        lambda x: math_ops.acosh(1 + math_ops.square(x)),
+        math_ops.abs,
+        math_ops.acos,
+        math_ops.asin,
+        math_ops.asinh,
+        math_ops.atan,
+        math_ops.atanh,
+        math_ops.bessel_i0e,
+        math_ops.bessel_i1e,
+        math_ops.cos,
+        math_ops.cosh,
+        math_ops.digamma,
+        math_ops.erf,
+        math_ops.erfc,
+        math_ops.exp,
+        math_ops.expm1,
+        math_ops.inv,
+        math_ops.is_finite,
+        math_ops.is_inf,
+        math_ops.lgamma,
+        math_ops.log,
+        math_ops.log1p,
+        math_ops.neg,
+        math_ops.negative,
+        math_ops.reciprocal,
+        math_ops.rint,
+        math_ops.round,
+        math_ops.rsqrt,
+        math_ops.sigmoid,
+        math_ops.sign,
+        math_ops.sin,
+        math_ops.sinh,
+        math_ops.sqrt,
+        math_ops.square,
+        math_ops.tan,
+        math_ops.tanh,
+        math_ops.tanh,
+        nn.elu,
+        nn.relu,
+        nn.relu6,
+        nn.selu,
+        nn.softplus,
+        nn.softsign,
+    ]
+    for op in complex_ops + real_ops:
+      with backprop.GradientTape(persistent=True) as g:
+        x = random_ops.random_uniform([3, 5])
+        g.watch(x)
+        if op in complex_ops:
+          y = random_ops.random_uniform([3, 5])
+          g.watch(y)
+          x = math_ops.complex(x, y)
+
+      # pylint: disable=cell-var-from-loop
+      output_dtypes = []
+      def loop_fn(i):
+        with g:
+          x1 = array_ops.gather(x, i)
+          y1 = op(x1)
+          outputs = [op(x), y1]
+          if y1.dtype == dtypes.float32:
+            loss = math_ops.reduce_sum(y1 * y1)
+          else:
+            loss = None
+        if loss is not None:
+          grad = g.gradient(loss, x1)
+          if grad is not None:
+            outputs.append(grad)
+        del output_dtypes[:]
+        output_dtypes.extend([t.dtype for t in outputs])
+        return outputs
+
+      # pylint: enable=cell-var-from-loop
+
+      self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=output_dtypes)
+
+  def test_unary_cwise_no_grad(self):
+    for op in [math_ops.ceil,
+               math_ops.floor,
+               math_ops.logical_not]:
+      x = random_ops.random_uniform([3, 5])
+      if op == math_ops.logical_not:
+        x = x > 0
+
+      # pylint: disable=cell-var-from-loop
+      def loop_fn(i):
+        return op(array_ops.gather(x, i))
+
+      # pylint: enable=cell-var-from-loop
+
+      self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=x.dtype)
+
+  def test_binary_cwise_ops(self):
+    logical_ops = [
+        math_ops.logical_and,
+        math_ops.logical_or,
+        math_ops.logical_xor
+    ]
+
+    # Wrapper functions restricting the range of inputs of zeta and polygamma.
+    def safe_polygamma(x, y):
+      return math_ops.polygamma(
+          math_ops.round(clip_ops.clip_by_value(y, 1, 10)),
+          x * x + 1)
+
+    def safe_zeta(x, y):
+      return math_ops.zeta(x * x + 1, y * y)
+
+    float_ops = [
+        math_ops.add,
+        math_ops.add_v2,
+        math_ops.atan2,
+        math_ops.complex,
+        math_ops.div,
+        math_ops.divide,
+        math_ops.div_no_nan,
+        math_ops.equal,
+        math_ops.floor_div,
+        math_ops.floor_mod,
+        math_ops.greater,
+        math_ops.greater_equal,
+        math_ops.igamma,
+        math_ops.igammac,
+        math_ops.igamma_grad_a,
+        math_ops.less,
+        math_ops.less_equal,
+        math_ops.maximum,
+        math_ops.minimum,
+        math_ops.mod,
+        math_ops.multiply,
+        math_ops.not_equal,
+        math_ops.pow,
+        math_ops.squared_difference,
+        math_ops.subtract,
+        math_ops.truncate_mod,
+        safe_polygamma,
+        safe_zeta,
+    ]
+    for op in logical_ops + float_ops:
+      x = random_ops.random_uniform([7, 3, 5])
+      y = random_ops.random_uniform([3, 5])
+      if op in logical_ops:
+        x = x > 0
+        y = y > 0
+
+      output_dtypes = []
+      # pylint: disable=cell-var-from-loop
+      def loop_fn(i):
+        x1 = array_ops.gather(x, i)
+        y1 = array_ops.gather(y, i)
+        outputs = [op(x, y), op(x1, y), op(x, y1), op(x1, y1), op(x1, x1)]
+        del output_dtypes[:]
+        output_dtypes.extend([t.dtype for t in outputs])
+        return outputs
+      # pylint: enable=cell-var-from-loop
+
+      self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=output_dtypes)
+
+  def test_approximate_equal(self):
+    x = random_ops.random_uniform([3, 5])
+    y = random_ops.random_uniform([3, 5])
+
+    def loop_fn(i):
+      x1 = array_ops.gather(x, i)
+      y1 = array_ops.gather(y, i)
+      return math_ops.approximate_equal(x1, y1)
+
+    self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.bool])
+
+  def test_addn(self):
+    x = random_ops.random_uniform([2, 3, 5])
+    y = random_ops.random_uniform([3, 5])
+    z = random_ops.random_uniform([3, 5])
+
+    def loop_fn(i):
+      x1 = array_ops.gather(x, i)
+      return math_ops.add_n([x1, y, z])
+
+    self._test_loop_fn(loop_fn, 2)
+
+  def test_matmul(self):
+    for tr_a in (True, False):
+      for tr_b in (True, False):
+        for stack_a in (True, False):
+          for stack_b in (True, False):
+            shape_a = (5, 3) if tr_a else (3, 5)
+            if stack_a:
+              shape_a = (2,) + shape_a
+            shape_b = (7, 5) if tr_b else (5, 7)
+            if stack_b:
+              shape_b = (2,) + shape_b
+
+            x = random_ops.random_uniform(shape_a)
+            y = random_ops.random_uniform(shape_b)
+
+            # pylint: disable=cell-var-from-loop
+            def loop_fn(i):
+              a = array_ops.gather(x, i) if stack_a else x
+              b = array_ops.gather(y, i) if stack_b else y
+              return math_ops.matmul(a, b, transpose_a=tr_a, transpose_b=tr_b)
+
+            # pylint: enable=cell-var-from-loop
+
+            self._test_loop_fn(loop_fn, 2)
+
+  def test_batch_matmul(self):
+    for tr_a in (True, False):
+      for tr_b in (True, False):
+        for stack_a in (True, False):
+          for stack_b in (True, False):
+            shape_a = (4, 5, 3) if tr_a else (4, 3, 5)
+            if stack_a:
+              shape_a = (2,) + shape_a
+            shape_b = (4, 7, 5) if tr_b else (4, 5, 7)
+            if stack_b:
+              shape_b = (2,) + shape_b
+
+            x = random_ops.random_uniform(shape_a)
+            y = random_ops.random_uniform(shape_b)
+
+            # pylint: disable=cell-var-from-loop
+            def loop_fn(i):
+              a = array_ops.gather(x, i) if stack_a else x
+              b = array_ops.gather(y, i) if stack_b else y
+              return math_ops.matmul(a, b, transpose_a=tr_a, transpose_b=tr_b)
+
+            # pylint: enable=cell-var-from-loop
+
+            self._test_loop_fn(loop_fn, 2)
+
+  def test_reduction(self):
+    x = random_ops.random_uniform([2, 3, 4, 5])
+    for op in [
+        math_ops.reduce_sum, math_ops.reduce_prod, math_ops.reduce_max,
+        math_ops.reduce_min
+    ]:
+      for axis in ([1], None, [0, 2]):
+        for keepdims in (True, False):
+
+          # pylint: disable=cell-var-from-loop
+          def loop_fn(i):
+            a = array_ops.gather(x, i)
+            return op(a, axis=axis, keepdims=keepdims)
+
+          # pylint: enable=cell-var-from-loop
+
+          self._test_loop_fn(loop_fn, 2)
+
+  def test_cum_sum(self):
+    x = random_ops.random_uniform([2, 3, 4, 5])
+    for axis in (1, -2):
+      for exclusive in (True, False):
+        for reverse in (True, False):
+
+          # pylint: disable=cell-var-from-loop
+          def loop_fn(i):
+            a = array_ops.gather(x, i)
+            return math_ops.cumsum(
+                a, axis=axis, exclusive=exclusive, reverse=reverse)
+
+          # pylint: enable=cell-var-from-loop
+
+          self._test_loop_fn(loop_fn, 2)
+
+  def test_cum_prod(self):
+    x = random_ops.random_uniform([2, 3, 4, 5])
+    for axis in (1, -2):
+      for exclusive in (True, False):
+        for reverse in (True, False):
+
+          # pylint: disable=cell-var-from-loop
+          def loop_fn(i):
+            a = array_ops.gather(x, i)
+            return math_ops.cumprod(
+                a, axis=axis, exclusive=exclusive, reverse=reverse)
+
+          # pylint: enable=cell-var-from-loop
+
+          self._test_loop_fn(loop_fn, 2)
+
+  def test_bias_add(self):
+    x_shape = [2, 3, 4, 5, 6]
+    x = random_ops.random_uniform(x_shape)
+    for data_format in ("NCHW", "NHWC"):
+      with backprop.GradientTape(persistent=True) as g:
+        bias_dim = 2 if data_format == "NCHW" else -1
+        bias_shape = x_shape[bias_dim]
+        bias = random_ops.random_uniform([bias_shape])
+        g.watch(bias)
+
+      # pylint: disable=cell-var-from-loop
+      def loop_fn(i):
+        with g:
+          a = array_ops.gather(x, i)
+          y = nn.bias_add(a, bias, data_format=data_format)
+          loss = math_ops.reduce_sum(y * y)
+        return y, g.gradient(loss, bias)
+      # pylint: enable=cell-var-from-loop
+
+      self._test_loop_fn(
+          loop_fn, 2, loop_fn_dtypes=[dtypes.float32, dtypes.float32])
+
+  def test_unsorted_segment_sum(self):
+    t = random_ops.random_uniform([3, 3, 2])
+    segment_ids = constant_op.constant([[0, 0, 2], [0, 1, 2], [2, 2, 2]])
+    num_segments = 3
+
+    def loop_fn(i):
+      data = array_ops.gather(t, i)
+      data_0 = array_ops.gather(t, 0)
+      seg_ids = array_ops.gather(segment_ids, i)
+      return (math_ops.unsorted_segment_sum(data, seg_ids, num_segments),
+              math_ops.unsorted_segment_sum(data_0, seg_ids, num_segments))
+
+    self._test_loop_fn(loop_fn, 3, [dtypes.float32] * 2)
+
+  def test_cast(self):
+    x = constant_op.constant([[1], [2]])
+    y = constant_op.constant([[1.0], [2.0]])
+
+    def loop_fn(i):
+      return (math_ops.cast(array_ops.gather(x, i), dtypes.float32),
+              math_ops.cast(array_ops.gather(y, i), dtypes.int32))
+
+    self._test_loop_fn(
+        loop_fn, 2, loop_fn_dtypes=[dtypes.float32, dtypes.int32])
+
+  def test_tanh_axpy(self):
+    a = constant_op.constant(3.)
+    x = random_ops.random_uniform([4, 5])
+    y = random_ops.random_uniform([6, 5])
+    n = x.shape[0]
+
+    def loop_fn(i):
+      return math_ops.tanh(a * array_ops.gather(x, i) + array_ops.gather(y, i))
+
+    self._test_loop_fn(loop_fn, n)
+
+  def test_select(self):
+    cond = constant_op.constant([True, False])
+    a = random_ops.random_uniform([2, 3, 5])
+    b = random_ops.random_uniform([2, 3, 5])
+    for cond_shape in [2], [2, 3], [2, 3, 5]:
+      cond = random_ops.random_uniform(cond_shape) > 0.5
+
+      # pylint: disable=cell-var-from-loop
+      def loop_fn(i):
+        a_i = array_ops.gather(a, i)
+        b_i = array_ops.gather(b, i)
+        cond_i = array_ops.gather(cond, i)
+        return array_ops.where(cond_i, a_i, b_i)
+
+      # pylint: enable=cell-var-from-loop
+
+      self._test_loop_fn(loop_fn, 2)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/ops/parallel_for/test_util.py b/tensorflow/python/ops/parallel_for/test_util.py
new file mode 100644
index 0000000000..7b4ef2239e
--- /dev/null
+++ b/tensorflow/python/ops/parallel_for/test_util.py
@@ -0,0 +1,59 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test utility."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.python.framework import dtypes
+from tensorflow.python.ops import variables
+from tensorflow.python.ops.parallel_for import control_flow_ops as pfor_control_flow_ops
+from tensorflow.python.platform import test
+from tensorflow.python.util import nest
+
+
+class PForTestCase(test.TestCase):
+  """Base class for test cases."""
+
+  def _run_targets(self, targets1, targets2=None, run_init=True):
+    targets1 = nest.flatten(targets1)
+    targets2 = ([] if targets2 is None else nest.flatten(targets2))
+    assert len(targets1) == len(targets2) or not targets2
+    if run_init:
+      init = variables.global_variables_initializer()
+      self.evaluate(init)
+    return self.evaluate(targets1 + targets2)
+
+  def run_and_assert_equal(self, targets1, targets2):
+    outputs = self._run_targets(targets1, targets2)
+    outputs = nest.flatten(outputs)  # flatten SparseTensorValues
+    n = len(outputs) // 2
+    for i in range(n):
+      if outputs[i + n].dtype != np.object:
+        self.assertAllClose(outputs[i + n], outputs[i], rtol=1e-4, atol=1e-5)
+      else:
+        self.assertAllEqual(outputs[i + n], outputs[i])
+
+  def _test_loop_fn(self, loop_fn, iters,
+                    loop_fn_dtypes=dtypes.float32,
+                    parallel_iterations=None):
+    t1 = pfor_control_flow_ops.pfor(loop_fn, iters=iters,
+                                    parallel_iterations=parallel_iterations)
+    t2 = pfor_control_flow_ops.for_loop(loop_fn, loop_fn_dtypes, iters=iters,
+                                        parallel_iterations=parallel_iterations)
+    self.run_and_assert_equal(t1, t2)
-- 
GitLab


From d852dec831a4a4c30c3018187e52e25010bf9711 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 9 Jan 2019 17:42:18 -0800
Subject: [PATCH 0510/1095] Use %zu format specifier for size_t printf
 argument.

This fixes -Wformat error in 32-bit Android build.

PiperOrigin-RevId: 228621967
---
 tensorflow/core/lib/core/status.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/lib/core/status.cc b/tensorflow/core/lib/core/status.cc
index 3076c09337..0b63f66f6d 100644
--- a/tensorflow/core/lib/core/status.cc
+++ b/tensorflow/core/lib/core/status.cc
@@ -200,7 +200,7 @@ Status StatusGroup::as_status() const {
          const std::pair<error::Code, int>& b) { return a.second < b.second; });
 
   fmt.push_back(
-      strings::Printf("Combined status information from %lu operations:\n",
+      strings::Printf("Combined status information from %zu operations:\n",
                       num_ok_ + children_.size()));
 
   for (const auto& p : count_vec) {
-- 
GitLab


From 0b1b49fd5a39a5834d897dc61068e68f0649e313 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 9 Jan 2019 17:45:02 -0800
Subject: [PATCH 0511/1095] Go: Update generated wrapper functions for
 TensorFlow ops. PiperOrigin-RevId: 228622263

---
 tensorflow/go/op/wrappers.go | 52 ++++++++++++++++++++++++++++++------
 1 file changed, 44 insertions(+), 8 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index 208c15d75a..6f6fb793a0 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -7477,6 +7477,19 @@ func Conv2DBackpropFilterUseCudnnOnGpu(value bool) Conv2DBackpropFilterAttr {
 	}
 }
 
+// Conv2DBackpropFilterExplicitPaddings sets the optional explicit_paddings attribute to value.
+//
+// value: If `padding` is `"EXPLICIT"`, the list of explicit padding amounts. For the ith
+// dimension, the amount of padding inserted before and after the dimension is
+// `explicit_paddings[2 * i]` and `explicit_paddings[2 * i + 1]`, respectively. If
+// `padding` is not `"EXPLICIT"`, `explicit_paddings` must be empty.
+// If not specified, defaults to <>
+func Conv2DBackpropFilterExplicitPaddings(value []int64) Conv2DBackpropFilterAttr {
+	return func(m optionalAttr) {
+		m["explicit_paddings"] = value
+	}
+}
+
 // Conv2DBackpropFilterDataFormat sets the optional data_format attribute to value.
 //
 // value: Specify the data format of the input and output data. With the
@@ -8176,7 +8189,7 @@ func RegexReplaceReplaceGlobal(value bool) RegexReplaceAttr {
 // Arguments:
 //	input: The text to be processed.
 //	pattern: The regular expression to match the input.
-//	rewrite: The rewrite to be applied to the matched expresion.
+//	rewrite: The rewrite to be applied to the matched expression.
 //
 // Returns The text after applying pattern and rewrite.
 func RegexReplace(scope *Scope, input tf.Output, pattern tf.Output, rewrite tf.Output, optional ...RegexReplaceAttr) (output tf.Output) {
@@ -11605,15 +11618,12 @@ func NthElement(scope *Scope, input tf.Output, n tf.Output, optional ...NthEleme
 //
 // Arguments:
 //
-//	segment_ids: A tensor whose shape is a prefix of `data.shape`.END
-//   }
-//   out_arg {
-//     name: "output"
-//     description: <<END
-// Has same shape as data, except for the first `segment_ids.rank`
+//	segment_ids: A tensor whose shape is a prefix of `data.shape`.
+//
+//
+// Returns Has same shape as data, except for the first `segment_ids.rank`
 // dimensions, which are replaced with a single dimension which has size
 // `num_segments`.
-//
 func UnsortedSegmentMax(scope *Scope, data tf.Output, segment_ids tf.Output, num_segments tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
@@ -33501,6 +33511,19 @@ func Conv2DBackpropInputUseCudnnOnGpu(value bool) Conv2DBackpropInputAttr {
 	}
 }
 
+// Conv2DBackpropInputExplicitPaddings sets the optional explicit_paddings attribute to value.
+//
+// value: If `padding` is `"EXPLICIT"`, the list of explicit padding amounts. For the ith
+// dimension, the amount of padding inserted before and after the dimension is
+// `explicit_paddings[2 * i]` and `explicit_paddings[2 * i + 1]`, respectively. If
+// `padding` is not `"EXPLICIT"`, `explicit_paddings` must be empty.
+// If not specified, defaults to <>
+func Conv2DBackpropInputExplicitPaddings(value []int64) Conv2DBackpropInputAttr {
+	return func(m optionalAttr) {
+		m["explicit_paddings"] = value
+	}
+}
+
 // Conv2DBackpropInputDataFormat sets the optional data_format attribute to value.
 //
 // value: Specify the data format of the input and output data. With the
@@ -35308,6 +35331,19 @@ func Conv2DUseCudnnOnGpu(value bool) Conv2DAttr {
 	}
 }
 
+// Conv2DExplicitPaddings sets the optional explicit_paddings attribute to value.
+//
+// value: If `padding` is `"EXPLICIT"`, the list of explicit padding amounts. For the ith
+// dimension, the amount of padding inserted before and after the dimension is
+// `explicit_paddings[2 * i]` and `explicit_paddings[2 * i + 1]`, respectively. If
+// `padding` is not `"EXPLICIT"`, `explicit_paddings` must be empty.
+// If not specified, defaults to <>
+func Conv2DExplicitPaddings(value []int64) Conv2DAttr {
+	return func(m optionalAttr) {
+		m["explicit_paddings"] = value
+	}
+}
+
 // Conv2DDataFormat sets the optional data_format attribute to value.
 //
 // value: Specify the data format of the input and output data. With the
-- 
GitLab


From 767a1fe74600fadee070fbaa88cbb5f5917170bd Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Wed, 9 Jan 2019 18:31:55 -0800
Subject: [PATCH 0512/1095] Delete unused
 ServiceExecutableRunOptions::xla_intra_op_thread_pool; NFC

PiperOrigin-RevId: 228627868
---
 tensorflow/compiler/xla/client/local_client.cc     |  5 ++---
 tensorflow/compiler/xla/service/hlo_runner.cc      |  4 +---
 tensorflow/compiler/xla/service/service.cc         |  4 +---
 .../xla/service/service_executable_run_options.h   | 14 +++-----------
 .../compiler/xla/tests/xla_hlo_profile_test.cc     |  5 ++---
 5 files changed, 9 insertions(+), 23 deletions(-)

diff --git a/tensorflow/compiler/xla/client/local_client.cc b/tensorflow/compiler/xla/client/local_client.cc
index 049cd15738..48b5f94538 100644
--- a/tensorflow/compiler/xla/client/local_client.cc
+++ b/tensorflow/compiler/xla/client/local_client.cc
@@ -164,9 +164,8 @@ StatusOr<ScopedShapedBuffer> LocalExecutable::Run(
   //    ExecutableRunOptions.eigen_intra_op_thread_pool.
   // *) The thread pool used for XLA CPU ops is from
   //    backend_->eigen_intra_op_thread_pool().
-  ServiceExecutableRunOptions service_options(
-      run_options, backend_->StreamBorrower(),
-      backend_->eigen_intra_op_thread_pool());
+  ServiceExecutableRunOptions service_options(run_options,
+                                              backend_->StreamBorrower());
 
   if (executable_->dumping_snapshot()) {
     return ExecuteAndDump(&service_options, arguments);
diff --git a/tensorflow/compiler/xla/service/hlo_runner.cc b/tensorflow/compiler/xla/service/hlo_runner.cc
index 5a9b820a9d..d7d66ae1c4 100644
--- a/tensorflow/compiler/xla/service/hlo_runner.cc
+++ b/tensorflow/compiler/xla/service/hlo_runner.cc
@@ -383,9 +383,7 @@ ServiceExecutableRunOptions HloRunner::GetServiceRunOptionsForDevice(
   if (device_assignment != nullptr) {
     run_options.set_device_assignment(device_assignment);
   }
-  return ServiceExecutableRunOptions(
-      run_options, backend().StreamBorrower(),
-      /*xla_intra_op_thread_pool=*/backend().eigen_intra_op_thread_pool());
+  return ServiceExecutableRunOptions(run_options, backend().StreamBorrower());
 }
 
 Backend& HloRunner::backend() {
diff --git a/tensorflow/compiler/xla/service/service.cc b/tensorflow/compiler/xla/service/service.cc
index 4a8d272261..2732d498d8 100644
--- a/tensorflow/compiler/xla/service/service.cc
+++ b/tensorflow/compiler/xla/service/service.cc
@@ -553,9 +553,7 @@ StatusOr<GlobalDataHandle> Service::ExecuteAndRegisterResult(
     options.set_intra_op_thread_pool(
         backend->eigen_intra_op_thread_pool_device());
     options.set_device_assignment(&device_assignment);
-    run_options.emplace_back(
-        options, backend->StreamBorrower(),
-        /*xla_intra_op_thread_pool=*/backend->eigen_intra_op_thread_pool());
+    run_options.emplace_back(options, backend->StreamBorrower());
   }
 
   if (options_.number_of_replicas() == 1) {
diff --git a/tensorflow/compiler/xla/service/service_executable_run_options.h b/tensorflow/compiler/xla/service/service_executable_run_options.h
index dbfed628bf..6bee671056 100644
--- a/tensorflow/compiler/xla/service/service_executable_run_options.h
+++ b/tensorflow/compiler/xla/service/service_executable_run_options.h
@@ -32,12 +32,10 @@ class ServiceExecutableRunOptions {
   ServiceExecutableRunOptions()
       : ServiceExecutableRunOptions(ExecutableRunOptions()) {}
 
-  explicit ServiceExecutableRunOptions(
-      ExecutableRunOptions run_options, StreamBorrower borrow_stream = nullptr,
-      tensorflow::thread::ThreadPool* xla_intra_op_thread_pool = nullptr)
+  explicit ServiceExecutableRunOptions(ExecutableRunOptions run_options,
+                                       StreamBorrower borrow_stream = nullptr)
       : run_options_(std::move(run_options)),
-        borrow_stream_(std::move(borrow_stream)),
-        xla_intra_op_thread_pool_(xla_intra_op_thread_pool) {}
+        borrow_stream_(std::move(borrow_stream)) {}
 
   // Returns reference or pointer to `ExecutableRunOptions` member.
   const ExecutableRunOptions& run_options() const { return run_options_; }
@@ -56,15 +54,9 @@ class ServiceExecutableRunOptions {
                : Status(tensorflow::error::UNIMPLEMENTED, "No stream cache");
   }
 
-  // Returns reference to thread pool for execution of XLA ops on CPU backend.
-  tensorflow::thread::ThreadPool* xla_intra_op_thread_pool() const {
-    return xla_intra_op_thread_pool_;
-  }
-
  private:
   ExecutableRunOptions run_options_;
   StreamBorrower borrow_stream_;
-  tensorflow::thread::ThreadPool* xla_intra_op_thread_pool_;
 };
 
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc b/tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc
index 1538f2afba..c7337e8caa 100644
--- a/tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc
+++ b/tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc
@@ -174,9 +174,8 @@ void ExecuteAndFetchProfile(string* profile_output, LocalClient* client,
   exec_run_options.set_allocator(backend->memory_allocator());
   exec_run_options.set_intra_op_thread_pool(
       backend->eigen_intra_op_thread_pool_device());
-  ServiceExecutableRunOptions run_options(
-      exec_run_options, /*borrow_stream=*/nullptr,
-      backend->eigen_intra_op_thread_pool());
+  ServiceExecutableRunOptions run_options(exec_run_options,
+                                          /*borrow_stream=*/nullptr);
   std::vector<const ShapedBuffer*> args = {&lhs_arg, &rhs_arg};
   TF_ASSERT_OK_AND_ASSIGN(
       auto execution_result,
-- 
GitLab


From 07b238033996f813381f1e1a0410c877c13c6c94 Mon Sep 17 00:00:00 2001
From: Aurelien Geron <aurelien.geron@gmail.com>
Date: Thu, 10 Jan 2019 12:41:40 +0800
Subject: [PATCH 0513/1095] Rename Function to ConcreteFunction

---
 .../python/mirrored_strategy_multigpu_test.py |  2 +-
 tensorflow/python/eager/def_function.py       |  9 ++--
 tensorflow/python/eager/function.py           | 50 ++++++++++---------
 tensorflow/python/eager/wrap_function.py      |  2 +-
 tensorflow/python/keras/backend.py            |  2 +-
 .../saved_model/function_deserialization.py   | 12 ++---
 tensorflow/python/saved_model/save.py         |  2 +-
 7 files changed, 42 insertions(+), 37 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
index 59d711ae01..2925b8c88c 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
@@ -1266,7 +1266,7 @@ class MirroredStrategyDefunTest(test.TestCase):
                             self.evaluate(device_result))
 
       for defun in defuns:
-        # PolymorphicFunctions are specialized to the current device stack, so
+        # `PolymorphicFunction`s are specialized to the current device stack, so
         # call_for_each has one trace per device. To check that the expected set
         # of variables was accessed on each trace, we first retrieve each
         # device-specific graph function.
diff --git a/tensorflow/python/eager/def_function.py b/tensorflow/python/eager/def_function.py
index 4c22a12c1c..b3f361b92d 100644
--- a/tensorflow/python/eager/def_function.py
+++ b/tensorflow/python/eager/def_function.py
@@ -408,7 +408,8 @@ class PolymorphicFunction(object):
     return self._function_spec
 
   def get_initialization_function(self, *args, **kwargs):
-    """Returns a `Function` object which initializes this function's variables.
+    """Returns a `ConcreteFunction` object which initializes this function's
+    variables.
 
     Requires that this function hasn't been accessed yet through either calling
     it or calling get_concrete_function. Fails if we cannot build an initializer
@@ -420,7 +421,8 @@ class PolymorphicFunction(object):
       **kwargs: keyword arguments to the python callable.
 
     Returns:
-      A `Function` object which initializes the variables of this function.
+      A `ConcreteFunction` object which initializes the variables of this
+      function.
 
     Raises:
       RuntimeError: if called after the variables have been initialized.
@@ -468,7 +470,8 @@ class PolymorphicFunction(object):
     # pylint: enable=protected-access
 
   def get_concrete_function(self, *args, **kwargs):
-    """Returns a `Function` object specialized to inputs and execution context.
+    """Returns a `ConcreteFunction` object specialized to inputs and execution
+    context.
 
     If this `PolymorphicFunction` was created with an `input_signature`, `args`
     and `kwargs` may be omitted. With an input signature there is only one
diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index 83cd140158..3bd7df9502 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -271,8 +271,8 @@ class _EagerDefinedFunction(object):
   def call(self, ctx, args):
     """Calls this function with `args` as inputs.
 
-    Function execution respects device annotations only if the function won't
-    be compiled with xla.
+    `ConcreteFunction` execution respects device annotations only if the
+    function won't be compiled with xla.
 
     Args:
       ctx: a Context object
@@ -340,15 +340,15 @@ class _EagerDefinedFunction(object):
       return outputs
 
 
-class Function(object):
+class ConcreteFunction(object):
   """Callable object encapsulating a function definition and its gradient.
 
-  `Function` is a callable that encapsulates a function definition and
+  `ConcreteFunction` is a callable that encapsulates a function definition and
   is differentiable under `tf.GradientTape` objects.
   """
 
   def __init__(self, func_graph, attrs=None, signature=None):
-    """Initialize a Function.
+    """Initialize a `ConcreteFunction`.
 
     Args:
       func_graph: An instance of FuncGraph: the function body to wrap.
@@ -384,8 +384,8 @@ class Function(object):
       *args: Tensors or Variables. Positional arguments are only accepted when
         they correspond one-to-one with arguments of the traced Python function.
       **kwargs: Tensors or Variables specified by name. When
-        `get_concrete_function` was called to create this `Function`, each
-        Tensor input was given a name, defaulting to the name of the Python
+        `get_concrete_function` was called to create this `ConcreteFunction`,
+        each Tensor input was given a name, defaulting to the name of the Python
         function's argument but possibly overridden by the `name=` argument to
         `tf.TensorSpec`. These names become the argument names for the concrete
         function.
@@ -394,7 +394,7 @@ class Function(object):
       The result of applying the TF function on the given Tensors.
 
     Raises:
-      AssertionError: If this `Function` was not created through
+      AssertionError: If this `ConcreteFunction` was not created through
         `get_concrete_function`.
       ValueError: If arguments contains anything other than Tensors or
         Variables.
@@ -478,7 +478,7 @@ class Function(object):
         tensor_inputs.append(
             ops.convert_to_tensor(arg, self._signature[i].dtype))
       else:
-        raise ValueError("All inputs to `Function`s must be Tensors; "
+        raise ValueError("All inputs to `ConcreteFunction`s must be Tensors; "
                          "on invocation of %s, the %d-th input (%s) was not a "
                          "Tensor." % (self._func_graph.name, i, str(arg)))
     args = tensor_inputs + self._captured_inputs
@@ -504,7 +504,8 @@ class Function(object):
     return self._build_call_outputs(outputs)
 
   def _register_gradient(self, name):
-    """Registers the gradient for the current Function under the given name.
+    """Registers the gradient for the current `ConcreteFunction` under the given
+    name.
 
     The gradient rewrites an inference call op to a forward call op, but does
     not modify a pre-existing forward call op. It then computes the gradient
@@ -545,7 +546,7 @@ class Function(object):
 
   @property
   def name(self):
-    """Function name."""
+    """`ConcreteFunction` name."""
     return self._inference_function.name
 
   @property
@@ -672,7 +673,7 @@ class Function(object):
         grad for grad in func_graph_module.flatten(gradients_wrt_inputs)
         if grad is not None)
     backwards_graph.structured_outputs = gradients_wrt_inputs
-    self._backward_graph_function = Function(
+    self._backward_graph_function = ConcreteFunction(
         backwards_graph, attrs=backward_function_attr)
 
     forward_function_attr = _parse_func_attrs({
@@ -1087,14 +1088,15 @@ class PolymorphicFunction(object):
     graph_function = self._get_concrete_function_internal_garbage_collected(
         *args, **kwargs)
     # We're returning this concrete function to someone, and they may keep a
-    # reference to the FuncGraph without keeping a reference to the Function
-    # object. So we won't clean up the reference cycles manually and instead
-    # will leave them to Python's garbage collector.
+    # reference to the FuncGraph without keeping a reference to the
+    # ConcreteFunction object. So we won't clean up the reference cycles
+    # manually and instead will leave them to Python's garbage collector.
     graph_function._garbage_collector.release()  # pylint: disable=protected-access
     return graph_function
 
   def get_concrete_function(self, *args, **kwargs):
-    """Returns a `Function` object specialized to inputs and execution context.
+    """Returns a `ConcreteFunction` object specialized to inputs and execution
+    context.
 
     Args:
       *args: inputs to specialize on.
@@ -1292,7 +1294,7 @@ class PolymorphicFunction(object):
             self._function_spec.arg_names[:arglen]
             + [self._function_spec.vararg_name] *
             (arglen - len(self._function_spec.arg_names)))
-        graph_function = Function(
+        graph_function = ConcreteFunction(
             func_graph_module.func_graph_from_py_func(
                 self._name,
                 self._python_function,
@@ -1311,10 +1313,10 @@ class PolymorphicFunction(object):
         # Save information about non-Tensor arguments with the concrete
         # function. Used to serialize PolymorphicFunctions.
         graph_function._python_call_signature = python_call_signature
-        # Tell the Function to clean up its graph once it goes out of
-        # scope. Function does not do this in its constructor since it gets used
-        # in some places (like Keras) where the FuncGraph lives longer than the
-        # Function.
+        # Tell the ConcreteFunction to clean up its graph once it goes out of
+        # scope. ConcreteFunction does not do this in its constructor since it
+        # gets used in some places (like Keras) where the FuncGraph lives longer
+        # than the ConcreteFunction.
         graph_function._garbage_collector = _FunctionGarbageCollector(
             graph_function.graph)
         # pylint: enable=protected-access
@@ -1335,7 +1337,7 @@ def register(func, *args, **kwargs):
     **kwargs: input keyword arguments for the Python function.
 
   Returns:
-    a `Function` object specialized to inputs and execution context.
+    a `ConcreteFunction` object specialized to inputs and execution context.
 
   Raises:
     ValueError: When the input function is not a defun wrapped python function.
@@ -1706,7 +1708,7 @@ def defun_with_attributes(func=None,
       whitelisted attribute name is allowed. Unwhitelisted attribute name or
       unsupported value will result into ValueError. `func_name` is also one of
       the whitelisted argument which is a python string, and sets the name for
-      this `Function` in the graph.
+      this `ConcreteFunction` in the graph.
     autograph: same as defun()'s autograph.
 
   Returns:
@@ -1828,7 +1830,7 @@ class _PolymorphicFunctionGarbageCollector(object):
 
 
 class _FunctionGarbageCollector(object):
-  """Cleans up reference cycles when a Function goes out of scope."""
+  """Cleans up reference cycles when a `ConcreteFunction` goes out of scope."""
 
   def __init__(self, func_graph):
     self._func_graph = func_graph
diff --git a/tensorflow/python/eager/wrap_function.py b/tensorflow/python/eager/wrap_function.py
index 6f978feb3c..61b4876830 100644
--- a/tensorflow/python/eager/wrap_function.py
+++ b/tensorflow/python/eager/wrap_function.py
@@ -60,7 +60,7 @@ class VariableHolder(object):
 
 
 # TODO(allenl): make this checkpointable
-class WrappedFunction(function.Function):
+class WrappedFunction(function.ConcreteFunction):
   """Wraps a tf V1 piece of code in a function."""
 
   def __init__(self, fn_graph, variable_holder, attrs=None, signature=None):
diff --git a/tensorflow/python/keras/backend.py b/tensorflow/python/keras/backend.py
index 6693243a68..2e53ee4ec9 100644
--- a/tensorflow/python/keras/backend.py
+++ b/tensorflow/python/keras/backend.py
@@ -3138,7 +3138,7 @@ class EagerExecutionFunction(object):
     # the relevant subgraph?
     graph.inputs = self.inputs + list(graph.captures.values())
     graph.outputs = self.outputs
-    graph_fn = eager_function.Function(graph)
+    graph_fn = eager_function.ConcreteFunction(graph)
     graph_fn._num_positional_args = len(self.inputs)
     graph_fn._arg_keywords = []
     self._graph_fn = graph_fn
diff --git a/tensorflow/python/saved_model/function_deserialization.py b/tensorflow/python/saved_model/function_deserialization.py
index dc839cf61c..bc10c81fbe 100644
--- a/tensorflow/python/saved_model/function_deserialization.py
+++ b/tensorflow/python/saved_model/function_deserialization.py
@@ -60,7 +60,7 @@ def recreate_polymorphic_function(
 
   Args:
     saved_polymorphic_function: SavedPolymorphicFunction proto.
-    functions: map from function name to Function.
+    functions: map from function name to `ConcreteFunction`.
 
   Returns:
     A PolymorphicFunction.
@@ -123,8 +123,8 @@ def load_function_def_library(library):
     library: FunctionDefLibrary proto message.
 
   Returns:
-    Map of original function names in the library to instances of `Function`
-    without captured inputs.
+    Map of original function names in the library to instances of
+    `ConcreteFunction` without captured inputs.
 
   Raises:
     ValueError: if functions dependencies have a cycle.
@@ -142,7 +142,7 @@ def load_function_def_library(library):
       copy = _fix_fdef(fdef, name_mapping)
 
       func_graph = function_def_lib.function_def_to_graph(copy)
-      func = function_lib.Function(func_graph)
+      func = function_lib.ConcreteFunction(func_graph)
       func.add_to_graph(import_graph)
 
       name_mapping[fdef.signature.name] = func.name
@@ -209,8 +209,8 @@ def _list_function_deps(fdef):
 
 def _clean_function_name(name):
   """Vanity function to keep the function names comprehensible."""
-  # Note: each time a function is wrapped into `function_lib.Function` its
-  # name becomes "__inference_<orig>_xyz".
+  # Note: each time a function is wrapped into `function_lib.ConcreteFunction`
+  # its name becomes "__inference_<orig>_xyz".
   match = re.search(r"^__inference_(.*)_\d+$", name)
   if match:
     return match.group(1)
diff --git a/tensorflow/python/saved_model/save.py b/tensorflow/python/saved_model/save.py
index 9db6d03ed0..89fa845b75 100644
--- a/tensorflow/python/saved_model/save.py
+++ b/tensorflow/python/saved_model/save.py
@@ -169,7 +169,7 @@ def _canonicalize_signatures(signatures):
              "converted to concrete functions using "
              "`f.get_concrete_function(...)`.").format(signature_function))
       signature_function = signature_function.get_concrete_function()
-    elif not isinstance(signature_function, defun.Function):
+    elif not isinstance(signature_function, defun.ConcreteFunction):
       raise ValueError(
           ("Expected a TensorFlow function to generate a signature for, but "
            "got {}. Python functions may be decorated with "
-- 
GitLab


From 10b4e7755c9c3c94cc9adc2d0bd0ffe975d718d7 Mon Sep 17 00:00:00 2001
From: Aurelien Geron <aurelien.geron@gmail.com>
Date: Thu, 10 Jan 2019 14:08:58 +0800
Subject: [PATCH 0514/1095] Rename PolymorphicFunction to Function

---
 .../python/mirrored_strategy_multigpu_test.py |  2 +-
 tensorflow/python/eager/def_function.py       | 36 +++++------
 tensorflow/python/eager/function.py           | 61 +++++++++----------
 tensorflow/python/eager/function_test.py      |  8 +--
 .../python/keras/engine/training_utils.py     |  2 +-
 .../saved_model/function_deserialization.py   | 23 ++++---
 .../saved_model/function_serialization.py     | 29 +++++----
 tensorflow/python/saved_model/load.py         | 21 ++++---
 .../saved_model/nested_structure_coder.py     |  8 +--
 tensorflow/python/saved_model/save.py         | 43 +++++++------
 .../python/training/checkpointable/util.py    |  3 +-
 11 files changed, 116 insertions(+), 120 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
index 2925b8c88c..b3b4fa85d3 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
@@ -1266,7 +1266,7 @@ class MirroredStrategyDefunTest(test.TestCase):
                             self.evaluate(device_result))
 
       for defun in defuns:
-        # `PolymorphicFunction`s are specialized to the current device stack, so
+        # `Function`s are specialized to the current device stack, so
         # call_for_each has one trace per device. To check that the expected set
         # of variables was accessed on each trace, we first retrieve each
         # device-specific graph function.
diff --git a/tensorflow/python/eager/def_function.py b/tensorflow/python/eager/def_function.py
index b3f361b92d..6471841780 100644
--- a/tensorflow/python/eager/def_function.py
+++ b/tensorflow/python/eager/def_function.py
@@ -202,13 +202,13 @@ class UnliftedInitializerVariable(resource_variable_ops.ResourceVariable):
     self._cached_shape_as_list = None
 
 
-class PolymorphicFunction(object):
+class Function(object):
   """Wrapper class for the graph functions defined for a Python function.
 
   See the documentation for `tf.function` for more information on the semantics
   of defined functions.
 
-  PolymorphicFunction is thread-compatible.
+  `Function` is thread-compatible.
   """
 
   def __init__(self,
@@ -217,7 +217,7 @@ class PolymorphicFunction(object):
                input_signature=None,
                autograph=True,
                experimental_autograph_options=None):
-    """Initializes a polymorphic function.
+    """Initializes a `Function`.
 
     Args:
       python_function: the function to be wrapped.
@@ -280,10 +280,10 @@ class PolymorphicFunction(object):
   def _initialize(self, args, kwds, add_initializers_to=None):
     """Initializes, on the first call.
 
-    Creates two polymorphic functions, one that will allow creation of variables
+    Creates two `Function`s, one that will allow creation of variables
     and one that won't.
 
-    Additionally runs a trace for the polymorphic function that allows creation
+    Additionally runs a trace for the `Function` that allows creation
     of variables.
 
     Args:
@@ -447,7 +447,7 @@ class PolymorphicFunction(object):
 
   @property
   def _cached_input_signatures(self):
-    """All input signatures used to call this PolymorphicFunction."""
+    """All input signatures used to call this `Function`."""
     seen = list()
     # We are using a list so that:
     #  - the returned collection is deterministic, and
@@ -473,15 +473,15 @@ class PolymorphicFunction(object):
     """Returns a `ConcreteFunction` object specialized to inputs and execution
     context.
 
-    If this `PolymorphicFunction` was created with an `input_signature`, `args`
-    and `kwargs` may be omitted. With an input signature there is only one
-    concrete function associated with this `PolymorphicFunction`.
+    If this `Function` was created with an `input_signature`, `args` and
+    `kwargs` may be omitted. With an input signature there is only one
+    concrete function associated with this `Function`.
 
     If there is no fixed `input_signature` associated with this
-    `PolymorphicFunction`, positional and keyword arguments to
-    `get_concrete_function` follow the same rules as input signature
-    specification, with `tf.TensorSpec` objects describing `tf.Tensor`s which
-    will be passed to the concrete function.
+    `Function`, positional and keyword arguments to `get_concrete_function`
+    follow the same rules as input signature specification, with `tf.TensorSpec`
+    objects describing `tf.Tensor`s which will be passed to the concrete
+    function.
 
     Each `tf.Tensor` argument to the concrete function must have a unique name,
     either because it is the only one associated with a named argument of the
@@ -566,8 +566,8 @@ class PolymorphicFunction(object):
   def __get__(self, instance, owner):
     """Makes it possible to defun instance methods."""
     del owner
-    # `instance` here is the instance that this `PolymorphicFunction` was
-    # accessed through; e.g., for
+    # `instance` here is the instance that this `Function` was accessed through
+    # e.g., for
     #
     #   class Foo(object):
     #
@@ -576,10 +576,10 @@ class PolymorphicFunction(object):
     #       ...
     #
     #   foo = Foo()
-    #   foo.bar()  # `foo.bar` is a `PolymorphicFunction` instance
+    #   foo.bar()  # `foo.bar` is a `Function` instance
     #
     # then `instance` will be `foo` (and `owner` will be `Foo`).  We create a
-    # new instance of PolymorphicFunction here to allow different instances each
+    # new instance of `Function` here to allow different instances each
     # to create variables once, thereby allowing methods to be decorated with
     # tf.function. Keeps a cache to avoid retracing the function every time the
     # descriptor is accessed.
@@ -835,7 +835,7 @@ def function(func=None,
       name = "function"
     return tf_decorator.make_decorator(
         inner_function,
-        PolymorphicFunction(
+        Function(
             inner_function,
             name,
             input_signature=input_signature,
diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index 3bd7df9502..26ad52f59d 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -997,16 +997,16 @@ class FunctionSpec(object):
       return inputs, {}
 
 
-class PolymorphicFunction(object):
+class Function(object):
   """Wrapper class for the graph functions defined for a Python function.
 
   See the documentation for `defun` for more information on the semantics of
   defined functions.
 
-  PolymorphicFunction class is thread-compatible meaning that minimal
-  usage of defuns (defining and calling) is thread-safe, but if users call other
-  methods or invoke the base `python_function` themselves, external
-  synchronization is necessary.
+  `Function` class is thread-compatible meaning that minimal usage of defuns
+  (defining and calling) is thread-safe, but if users call other methods or
+  invoke the base `python_function` themselves, external synchronization is
+  necessary.
   """
 
   def __init__(self,
@@ -1015,7 +1015,7 @@ class PolymorphicFunction(object):
                input_signature=None,
                attributes=None,
                autograph=True):
-    """Initializes a polymorphic function.
+    """Initializes a `Function`.
 
     Args:
       python_function: the function to be wrapped.
@@ -1042,14 +1042,14 @@ class PolymorphicFunction(object):
     self._name = name
     self._autograph = autograph
     self._function_cache = collections.OrderedDict()
-    self._garbage_collector = _PolymorphicFunctionGarbageCollector(
+    self._garbage_collector = _FunctionGarbageCollector(
         self._function_cache)
     self._function_attributes = attributes or {}
 
     self._lock = threading.Lock()
     # _descriptor_cache is a of instance of a class to an instance-specific
-    # PolymorphicFunction, used to make sure defun-decorated methods create
-    # different functions for each instance.
+    # `Function`, used to make sure defun-decorated methods create different
+    # functions for each instance.
     self._descriptor_cache = weakref.WeakKeyDictionary()
 
   def __call__(self, *args, **kwargs):
@@ -1169,8 +1169,8 @@ class PolymorphicFunction(object):
   def __get__(self, instance, owner):
     """Makes it possible to defun instance methods."""
     del owner
-    # `instance` here is the instance that this `PolymorphicFunction` was
-    # accessed through; e.g., for
+    # `instance` here is the instance that this `Function` was accessed through
+    # e.g., for
     #
     #   class Foo(object):
     #
@@ -1179,26 +1179,25 @@ class PolymorphicFunction(object):
     #       ...
     #
     #   foo = Foo()
-    #   foo.bar()  # `foo.bar` is a `PolymorphicFunction` instance
+    #   foo.bar()  # `foo.bar` is a `Function` instance
     #
     # then `instance` will be `foo` (and `owner` will be `Foo`).  We create a
-    # new instance of PolymorphicFunction here to allow different instances each
+    # new instance of `Function` here to allow different instances each
     # to create variables once, thereby allowing methods to be decorated with
     # defun. Keeps a cache to avoid retracing the function every time the
     # descriptor is accessed.
     if instance not in self._descriptor_cache:
       if instance is None:
         return self
-      # If there is no instance-specific polymorphic func in the cache,
-      # we construct an instance-specific polymorphic function
-      # that uses a weak reference to the instance (so that the instance will
-      # be correctly gc'd).
+      # If there is no instance-specific `Function` in the cache, we construct
+      # an instance-specific `Function` that uses a weak reference to the
+      # instance (so that the instance will be correctly gc'd).
 
       # And finally add the wrapped function to the description cache
       self._descriptor_cache[instance] = class_method_to_instance_method(
           self, instance)
 
-    # Return the cached polymorphic function for the instance
+    # Return the cached `Function` for the instance
     return self._descriptor_cache[instance]
 
   def _cache_key(self, args, kwargs):
@@ -1256,8 +1255,8 @@ class PolymorphicFunction(object):
   def _maybe_define_function(self, args, kwargs):
     """Gets a function for these inputs, defining it if necessary.
 
-    `args` and `kwargs` can be None if this `PolymorphicFunction` was created
-    with an `input_signature`.
+    `args` and `kwargs` can be None if this `Function` was created with an
+    `input_signature`.
 
     Args:
       args: The varargs for the Python function.
@@ -1311,13 +1310,13 @@ class PolymorphicFunction(object):
               _encode_arg_for_serialization(arg) for arg in args)
         # pylint: disable=protected-access
         # Save information about non-Tensor arguments with the concrete
-        # function. Used to serialize PolymorphicFunctions.
+        # function. Used to serialize `Function`s.
         graph_function._python_call_signature = python_call_signature
         # Tell the ConcreteFunction to clean up its graph once it goes out of
         # scope. ConcreteFunction does not do this in its constructor since it
         # gets used in some places (like Keras) where the FuncGraph lives longer
         # than the ConcreteFunction.
-        graph_function._garbage_collector = _FunctionGarbageCollector(
+        graph_function._garbage_collector = _ConcreteFunctionGarbageCollector(
             graph_function.graph)
         # pylint: enable=protected-access
         self._function_cache[cache_key] = graph_function
@@ -1325,14 +1324,14 @@ class PolymorphicFunction(object):
 
 
 def register(func, *args, **kwargs):
-  """Register a specialization of a PolymorphicFunction into the graph.
+  """Register a specialization of a `Function` into the graph.
 
   This won't actually call the function with the inputs, and only put the
   function definition into graph. Register function with different input param
   will result into multiple version of functions registered in graph.
 
   Args:
-    func: the PolymorphicFunction instance that generated by a @defun
+    func: the `Function` instance that generated by a @defun
     *args: input arguments for the Python function.
     **kwargs: input keyword arguments for the Python function.
 
@@ -1342,7 +1341,7 @@ def register(func, *args, **kwargs):
   Raises:
     ValueError: When the input function is not a defun wrapped python function.
   """
-  if not isinstance(func, PolymorphicFunction):
+  if not isinstance(func, Function):
     raise ValueError("Only defun function is allowed to be registered. "
                      "Got type: %s" % type(func))
   concrete_func = func.get_concrete_function(*args, **kwargs)
@@ -1729,7 +1728,7 @@ def defun_with_attributes(func=None,
       name = "function"
     return tf_decorator.make_decorator(
         function,
-        PolymorphicFunction(
+        Function(
             function,
             name,
             input_signature=input_signature,
@@ -1761,7 +1760,7 @@ class _WeakrefSelf(object):
 
 
 def class_method_to_instance_method(original_function, instance):
-  """Constructs a new PolymorphicFunction with `self` bound."""
+  """Constructs a new `Function` with `self` bound."""
   weak_instance = weakref.ref(instance)
 
   # Note: while we could bind to a weakref proxy instead, that causes the
@@ -1769,8 +1768,8 @@ def class_method_to_instance_method(original_function, instance):
   bound_method = types_lib.MethodType(original_function.python_function,
                                       _WeakrefSelf(weak_instance))
 
-  # original_function is expected to be of one of the two PolymorphicFunction
-  # types (defined either in function.py or def_function.py).
+  # original_function is expected to be of one of the two `Function` types
+  # (defined either in function.py or def_function.py).
   assert hasattr(original_function, "_name")
   assert hasattr(original_function, "_autograph")
   assert hasattr(original_function, "_input_signature")
@@ -1812,7 +1811,7 @@ def class_method_to_instance_method(original_function, instance):
   return wrapped_instance_func
 
 
-class _PolymorphicFunctionGarbageCollector(object):
+class _FunctionGarbageCollector(object):
   """Cleans up cycles when a defun goes out of scope."""
 
   def __init__(self, cache):
@@ -1829,7 +1828,7 @@ class _PolymorphicFunctionGarbageCollector(object):
       pass
 
 
-class _FunctionGarbageCollector(object):
+class _ConcreteFunctionGarbageCollector(object):
   """Cleans up reference cycles when a `ConcreteFunction` goes out of scope."""
 
   def __init__(self, func_graph):
diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py
index 1966b259bf..3bb4f2452c 100644
--- a/tensorflow/python/eager/function_test.py
+++ b/tensorflow/python/eager/function_test.py
@@ -966,8 +966,8 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
     self.assertAllEqual([[3.0]], self.evaluate(y))
 
     # Break the reference cycle between the MiniModel and the defun:
-    # MiniModel --(through its `call` method)--> PolymorphicFunction
-    # PolymorphicFunction --(instancemethod on MiniModel)--> MiniModel
+    # `MiniModel` --(through its `call` method)--> `Function`
+    # `Function` --(instancemethod on `MiniModel`)--> `MiniModel`
     del model.call
 
   # Note: The ConfigProto below unfortunately only configures graph
@@ -1535,7 +1535,7 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
           t = constant_op.constant([[1.0, 2.0], [3.0, 4.0]])
           add(t, t)
 
-  def testRegisterPolymorphicFunction(self):
+  def testRegisterFunction(self):
     @function.defun
     def add(x, y):
       return math_ops.add(x, y)
@@ -1818,7 +1818,7 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
     self.assertAllEqual(instance_call_one, instance_call_two)
     self.assertAllEqual(instance_call_one, class_call)
 
-  def testDecoratedMethodUniquePolymorphicFuncPerInstance(self):
+  def testDecoratedMethodUniqueFunctionPerInstance(self):
     m = DefunnedMiniModel()
     n = DefunnedMiniModel()
 
diff --git a/tensorflow/python/keras/engine/training_utils.py b/tensorflow/python/keras/engine/training_utils.py
index 949f1e400d..6072d09428 100644
--- a/tensorflow/python/keras/engine/training_utils.py
+++ b/tensorflow/python/keras/engine/training_utils.py
@@ -1402,7 +1402,7 @@ def trace_model_call(model, input_signature=None):
     ValueError: if input signature cannot be inferred from the model.
   """
   if input_signature is None:
-    if isinstance(model.call, def_function.PolymorphicFunction):
+    if isinstance(model.call, def_function.Function):
       input_signature = model.call.input_signature
 
   if input_signature is None:
diff --git a/tensorflow/python/saved_model/function_deserialization.py b/tensorflow/python/saved_model/function_deserialization.py
index bc10c81fbe..f1c43cb7a4 100644
--- a/tensorflow/python/saved_model/function_deserialization.py
+++ b/tensorflow/python/saved_model/function_deserialization.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tools for deserializing PolymorphicFunctions."""
+"""Tools for deserializing `Function`s."""
 
 from __future__ import absolute_import
 from __future__ import division
@@ -54,25 +54,24 @@ def _inputs_compatible(args, stored_inputs):
   return True
 
 
-def recreate_polymorphic_function(
-    saved_polymorphic_function, functions):
-  """Creates a PolymorphicFunction from a SavedPolymorphicFunction.
+def recreate_function(saved_function, concrete_functions):
+  """Creates a `Function` from a `SavedPolymorphicFunction`.
 
   Args:
-    saved_polymorphic_function: SavedPolymorphicFunction proto.
-    functions: map from function name to `ConcreteFunction`.
+    saved_function: `SavedPolymorphicFunction` proto.
+    concrete_functions: map from function name to `ConcreteFunction`.
 
   Returns:
-    A PolymorphicFunction.
+    A `Function`.
   """
-  # TODO(andresp): Construct a PolymorphicFunction with the cache populated
-  # instead of creating a new PolymorphicFunction backed by a Python layer to
+  # TODO(andresp): Construct a `Function` with the cache populated
+  # instead of creating a new `Function` backed by a Python layer to
   # glue things together. Current approach is nesting functions deeper for each
   # serialization cycle.
 
   coder = nested_structure_coder.StructureCoder()
   function_spec_tuple = coder.decode_proto(
-      saved_polymorphic_function.function_spec_tuple)
+      saved_function.function_spec_tuple)
   function_spec = function_lib.FunctionSpec.from_tuple(function_spec_tuple)
 
   # TODO(mdan): We may enable autograph once exceptions are supported.
@@ -81,8 +80,8 @@ def recreate_polymorphic_function(
     """Calls a restored function."""
     # TODO(allenl): Functions saved with input_signatures should revive with
     # input_signatures.
-    for monomorphic_function in saved_polymorphic_function.monomorphic_function:
-      function_obj = functions[monomorphic_function.concrete_function]
+    for monomorphic_function in saved_function.monomorphic_function:
+      function_obj = concrete_functions[monomorphic_function.concrete_function]
       canonicalized_original_inputs = coder.decode_proto(
           monomorphic_function.canonicalized_input)
 
diff --git a/tensorflow/python/saved_model/function_serialization.py b/tensorflow/python/saved_model/function_serialization.py
index 014a4ef04c..b645a7c964 100644
--- a/tensorflow/python/saved_model/function_serialization.py
+++ b/tensorflow/python/saved_model/function_serialization.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tools for serializing PolymorphicFunctions."""
+"""Tools for serializing `Function`s."""
 
 from __future__ import absolute_import
 from __future__ import division
@@ -24,15 +24,14 @@ from tensorflow.python.saved_model import nested_structure_coder
 from tensorflow.python.saved_model import saved_object_graph_pb2
 
 
-def serialize_polymorphic_function(polymorphic_function, node_ids):
-  """Build a SavedPolymorphicProto."""
+def serialize_function(function, node_ids):
+  """Build a SavedPolymorphicFunction proto."""
   coder = nested_structure_coder.StructureCoder()
   proto = saved_object_graph_pb2.SavedPolymorphicFunction()
 
   proto.function_spec_tuple.CopyFrom(
-      coder.encode_structure(polymorphic_function.function_spec.as_tuple()))  # pylint: disable=protected-access
-  for signature, concrete_function in list_all_concrete_functions(
-      polymorphic_function):
+      coder.encode_structure(function.function_spec.as_tuple()))  # pylint: disable=protected-access
+  for signature, concrete_function in list_all_concrete_functions(function):
     bound_inputs = []
     try:
       for capture in concrete_function.captured_inputs:
@@ -52,23 +51,23 @@ def serialize_polymorphic_function(polymorphic_function, node_ids):
   return proto
 
 
-def list_all_concrete_functions(polymorphic_function):
-  """Given a polymorphic function, returns all of its concrete functions.
+def list_all_concrete_functions(function):
+  """Given a `Function`, returns all of its concrete functions.
 
   Args:
-    polymorphic_function: Instance of `PolymorphicFunction`.
+    function: Instance of `Function`.
 
   Returns:
-    A list of tuples in the form (signature, concrete_function), where concrete
-    function is an instance of `Function`.
+    A list of tuples in the form (signature, concrete_function), where
+    `concrete_function` is an instance of `ConcreteFunction`.
   """
-  input_signature = polymorphic_function._input_signature  # pylint: disable=protected-access
+  input_signature = function._input_signature  # pylint: disable=protected-access
   if input_signature is not None:
-    polymorphic_function.get_concrete_function()
+    function.get_concrete_function()
   concrete_functions = []
-  for signature in polymorphic_function._cached_input_signatures:  # pylint: disable=protected-access
+  for signature in function._cached_input_signatures:  # pylint: disable=protected-access
     if any(isinstance(arg, defun_lib.UnknownArgument) for arg in signature):
       continue
-    concrete_function = polymorphic_function.get_concrete_function(*signature)
+    concrete_function = function.get_concrete_function(*signature)
     concrete_functions.append((signature, concrete_function))
   return concrete_functions
diff --git a/tensorflow/python/saved_model/load.py b/tensorflow/python/saved_model/load.py
index bbb1485612..ae71a228ea 100644
--- a/tensorflow/python/saved_model/load.py
+++ b/tensorflow/python/saved_model/load.py
@@ -42,15 +42,16 @@ class _Loader(object):
     self._asset_file_def = meta_graph.asset_file_def
     self._proto = object_graph_proto
     self._export_dir = export_dir
-    self._functions = function_deserialization.load_function_def_library(
-        meta_graph.graph_def.library)
+    self._concrete_functions = (
+        function_deserialization.load_function_def_library(
+            meta_graph.graph_def.library))
     self._load_all()
     self._bind_function_captures()
     self._restore_checkpoint()
 
   def _bind_function_captures(self):
     """Setup captured tensors in restored concrete functions."""
-    seen_functions = set()
+    seen_concrete_functions = set()
     for object_proto in self._proto.nodes:
       if object_proto.WhichOneof("kind") == "function":
         for monomorphic_function in object_proto.function.monomorphic_function:
@@ -63,18 +64,18 @@ class _Loader(object):
               for node_id in monomorphic_function.bound_inputs
               if self._proto.nodes[node_id].WhichOneof("kind") == "variable"
           ]
-          if name in seen_functions:
-            if self._functions[name]._captured_inputs != bound_inputs:  # pylint: disable=protected-access
+          if name in seen_concrete_functions:
+            if self._concrete_functions[name]._captured_inputs != bound_inputs:  # pylint: disable=protected-access
               raise NotImplementedError(
                   "Function %s is used more than once with different "
                   "captured inputs." % name)
           else:
-            seen_functions.add(name)
+            seen_concrete_functions.add(name)
             # TODO(andresp): This is only injecting the captured inputs into the
             # concrete function, note that we did not modify the FuncGraph
             # itself.
-            self._functions[name]._captured_inputs = bound_inputs  # pylint: disable=protected-access
-            self._functions[name]._func_graph.variables = bound_variables  # pylint: disable=protected-access
+            self._concrete_functions[name]._captured_inputs = bound_inputs  # pylint: disable=protected-access
+            self._concrete_functions[name]._func_graph.variables = bound_variables  # pylint: disable=protected-access
 
   def _get_tensor_from_node(self, node_id):
     obj = self._nodes[node_id]
@@ -137,8 +138,8 @@ class _Loader(object):
     return tracking.TrackableAsset(filename)
 
   def _recreate_function(self, proto):
-    return function_deserialization.recreate_polymorphic_function(
-        proto, self._functions)
+    return function_deserialization.recreate_function(
+        proto, self._concrete_functions)
 
   def _recreate_variable(self, proto):
     # TODO(andresp): Can we use the checkpointed value as initializer?
diff --git a/tensorflow/python/saved_model/nested_structure_coder.py b/tensorflow/python/saved_model/nested_structure_coder.py
index 410ebda5c1..31ee239f13 100644
--- a/tensorflow/python/saved_model/nested_structure_coder.py
+++ b/tensorflow/python/saved_model/nested_structure_coder.py
@@ -14,14 +14,14 @@
 # ==============================================================================
 """Module that encodes (decodes) nested structures into (from) protos.
 
-The intended use is to serialize everything needed to restore a
-PolymorphicFunction that was saved into a SavedModel. This may include concrete
-function inputs and outputs, signatures, function specs, etc.
+The intended use is to serialize everything needed to restore a `Function` that
+was saved into a SavedModel. This may include concrete function inputs and
+outputs, signatures, function specs, etc.
 
 Example use:
 coder = nested_structure_coder.StructureCoder()
 # Encode into proto.
-signature_proto = coder.encode_structure(polymorphic_function.input_signature)
+signature_proto = coder.encode_structure(function.input_signature)
 # Decode into a Python object.
 restored_signature = coder.decode_proto(signature_proto)
 """
diff --git a/tensorflow/python/saved_model/save.py b/tensorflow/python/saved_model/save.py
index 89fa845b75..31f6aad4cf 100644
--- a/tensorflow/python/saved_model/save.py
+++ b/tensorflow/python/saved_model/save.py
@@ -68,17 +68,17 @@ class _SaveableView(object):
     self.nodes = checkpointable_objects
     self.node_ids = node_ids
     self.slot_variables = slot_variables
-    self.polymorphic_functions = util.ObjectIdentityDictionary()
+    self.functions = util.ObjectIdentityDictionary()
 
-    # Also add polymorphic functions as nodes.
+    # Also add `Function`s as nodes.
     for obj in self.nodes:
-      self.polymorphic_functions[obj] = self._list_polymorphic_functions(obj)
-      for function in self.polymorphic_functions[obj].values():
+      self.functions[obj] = self._list_functions(obj)
+      for function in self.functions[obj].values():
         if function not in self.node_ids:
           self.node_ids[function] = len(self.nodes)
           self.nodes.append(function)
         # Force listing the concrete functions for the side effects:
-        #  - populate the cache for polymorphic functions that have an
+        #  - populate the cache for `Function`s that have an
         #  input_signature and have not been called.
         #  - force side effects of creation of concrete functions, e.g. create
         #  variables on first run.
@@ -94,20 +94,20 @@ class _SaveableView(object):
       assert self.node_ids[node] == node_id
       object_proto = proto.nodes.add()
       object_proto.slot_variables.extend(self.slot_variables.get(node, ()))
-      if isinstance(node, def_function.PolymorphicFunction):
+      if isinstance(node, def_function.Function):
         continue
       for child in node._checkpoint_dependencies:  # pylint: disable=protected-access
         child_proto = object_proto.children.add()
         child_proto.node_id = self.node_ids[child.ref]
         child_proto.local_name = child.name
-      for local_name, ref_function in self.polymorphic_functions[node].items():
+      for local_name, ref_function in self.functions[node].items():
         child_proto = object_proto.children.add()
         child_proto.node_id = self.node_ids[ref_function]
         child_proto.local_name = local_name
 
-  def _list_polymorphic_functions(self, checkpointable_object):
-    """Return a dict of polymorphic functions of a checkpointable."""
-    polymorphic_functions = dict()
+  def _list_functions(self, checkpointable_object):
+    """Return a dict of `Function`s of a checkpointable."""
+    functions = dict()
     for attribute_name in dir(checkpointable_object):
       try:
         attribute_value = getattr(checkpointable_object, attribute_name, None)
@@ -115,16 +115,16 @@ class _SaveableView(object):
         # We really don't want to throw an exception just because some object's
         # attribute accessor is broken.
         attribute_value = None
-      if isinstance(attribute_value, def_function.PolymorphicFunction):
-        polymorphic_functions[attribute_name] = attribute_value
-    return polymorphic_functions
+      if isinstance(attribute_value, def_function.Function):
+        functions[attribute_name] = attribute_value
+    return functions
 
 
 def _find_function_to_export(saveable_view):
   """Iterate over `root`'s attributes, finding traced functions."""
   exported_function = None
   previous_attribute_name = None
-  functions = saveable_view.polymorphic_functions[saveable_view.root]
+  functions = saveable_view.functions[saveable_view.root]
   for name, value in sorted(functions.items()):
     if exported_function is not None:
       raise ValueError(
@@ -158,8 +158,7 @@ def _canonicalize_signatures(signatures):
         signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: signatures}
   concrete_signatures = {}
   for serving_key, signature_function in signatures.items():
-    if isinstance(signature_function, (defun.PolymorphicFunction,
-                                       def_function.PolymorphicFunction)):
+    if isinstance(signature_function, (defun.Function, def_function.Function)):
       input_signature = signature_function._input_signature  # pylint: disable=protected-access
       if input_signature is None:
         raise ValueError(
@@ -547,12 +546,12 @@ def _fill_meta_graph_def(meta_graph_def, saveable_view, signature_functions,
   # the exported graph (thus the `to_graph` argument).
   saver = object_saver.freeze(object_map=object_map, to_graph=exported_graph)
 
-  # We must instantiate and list all concrete functions of polymorphic functions
-  # while in eager mode so they end up added to the graph and can later be used
-  # by the object based saved model.
+  # We must instantiate and list all concrete functions of `Function`s while in
+  # eager mode so they end up added to the graph and can later be used by the
+  # object based saved model.
   concrete_functions = []
   for obj in accessible_objects:
-    for function in saveable_view.polymorphic_functions[obj].values():
+    for function in saveable_view.functions[obj].values():
       concrete_functions.extend(
           function_serialization.list_all_concrete_functions(function))
 
@@ -613,9 +612,9 @@ def _write_object_proto(obj, proto, asset_file_def_index, node_ids):
     proto.variable.trainable = obj.trainable
     proto.variable.dtype = obj.dtype.as_datatype_enum
     proto.variable.shape.CopyFrom(obj.shape.as_proto())
-  elif isinstance(obj, def_function.PolymorphicFunction):
+  elif isinstance(obj, def_function.Function):
     proto.function.CopyFrom(
-        function_serialization.serialize_polymorphic_function(obj, node_ids))
+        function_serialization.serialize_function(obj, node_ids))
   else:
     proto.user_object.SetInParent()
 
diff --git a/tensorflow/python/training/checkpointable/util.py b/tensorflow/python/training/checkpointable/util.py
index a45263f5c6..4a80b25a98 100644
--- a/tensorflow/python/training/checkpointable/util.py
+++ b/tensorflow/python/training/checkpointable/util.py
@@ -1748,8 +1748,7 @@ class Checkpoint(tracking.Checkpointable):
     """
     super(Checkpoint, self).__init__()
     for k, v in sorted(kwargs.items(), key=lambda item: item[0]):
-      if not isinstance(v, (base.CheckpointableBase,
-                            def_function.PolymorphicFunction)):
+      if not isinstance(v, (base.CheckpointableBase, def_function.Function)):
         raise ValueError(
             ("`Checkpoint` was expecting a checkpointable object (an object "
              "derived from `CheckpointableBase`), got %s. If you believe this "
-- 
GitLab


From ebb3429856441149e41388dfbea59496f8dbf17b Mon Sep 17 00:00:00 2001
From: Penporn Koanantakool <penporn@google.com>
Date: Wed, 9 Jan 2019 22:30:09 -0800
Subject: [PATCH 0515/1095] Remove a TODO about moving choose() out of
 TensorContraction.h since it is now in
 unsupported/Eigen/CXX11/scr/Tensor/TensorMeta.h.

PiperOrigin-RevId: 228648231
---
 tensorflow/core/kernels/eigen_spatial_convolutions.h | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tensorflow/core/kernels/eigen_spatial_convolutions.h b/tensorflow/core/kernels/eigen_spatial_convolutions.h
index 86d8c98ee6..8b19813940 100644
--- a/tensorflow/core/kernels/eigen_spatial_convolutions.h
+++ b/tensorflow/core/kernels/eigen_spatial_convolutions.h
@@ -1683,8 +1683,6 @@ EIGEN_DEVICE_FUNC
     kernel_dims[0] = kernelChannels * kernelRows * kernelCols;
     kernel_dims[1] = kernelFilters;
   }
-  // TODO(yangke): choose() is defined in TensorContraction.h -- consider
-  // moving it to somewhere more "common".
   return choose(
       Cond<internal::traits<Input>::Layout == ColMajor>(),
       kernel.reshape(kernel_dims)
-- 
GitLab


From 6d3cbd2bcc165f8de064643719991db9753c8dca Mon Sep 17 00:00:00 2001
From: Smit Hinsu <hinsu@google.com>
Date: Wed, 9 Jan 2019 23:57:12 -0800
Subject: [PATCH 0516/1095] Update TensorRT SegmentGraph function to return
 node pointers

This is in preparation to share the graph segmentation code between XLA and
TensorRT.

TESTED with existing unit tests

PiperOrigin-RevId: 228654815
---
 .../contrib/tensorrt/convert/convert_graph.cc | 51 ++++++++++---------
 .../contrib/tensorrt/convert/convert_nodes.cc | 15 +++---
 .../contrib/tensorrt/convert/convert_nodes.h  |  3 +-
 .../contrib/tensorrt/segment/segment.cc       | 17 +++----
 tensorflow/contrib/tensorrt/segment/segment.h |  8 +--
 .../contrib/tensorrt/segment/segment_test.cc  |  5 +-
 6 files changed, 53 insertions(+), 46 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index bf2de94e04..eef647473a 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -334,13 +334,12 @@ struct EdgePtrCompare {
 tensorflow::Status GetEngineInfo(
     const tensorflow::Graph* g,
     const tensorflow::grappler::GraphProperties& graph_properties,
-    const std::set<string>& segment_nodes,
+    const std::set<const Node*>& segment_nodes,
     const std::unordered_map<string, tensorflow::Node*>& node_map,
     const std::vector<tensorflow::Node*>& reverse_topo_order,
     EngineInfo* info) {
-  std::vector<int> subgraph_node_ids;  // Topologically sorted node ids.
-  std::set<string> subgraph_node_names = segment_nodes;
-  std::set<int> added_const_node_ids;  // Used to prevent double insertion.
+  std::vector<const Node*> subgraph_nodes;  // Topologically sorted nodes.
+  std::set<const Node*> added_const_nodes;  // Used to prevent double insertion.
   std::set<string> segment_devices;
 
   // Map from src_node_name+port to the unique port numbers of the TRT op, where
@@ -352,9 +351,8 @@ tensorflow::Status GetEngineInfo(
   std::unordered_map<string, int> input_to_engine_port, output_to_engine_port;
   for (auto it = reverse_topo_order.rbegin(); it != reverse_topo_order.rend();
        ++it) {
-    const auto& node_name = (*it)->name();
-    if (segment_nodes.count(node_name) == 0) continue;
-    auto node = *it;
+    const Node* node = *it;
+    if (segment_nodes.count(node) == 0) continue;
     auto node_device = node->requested_device();
     if (!node_device.empty()) {
       segment_devices.insert(node_device);
@@ -366,8 +364,11 @@ tensorflow::Status GetEngineInfo(
                 << " neither have requested device nor assigned device";
       }
     }
+    subgraph_nodes.push_back(node);
+
     const int node_id = node->id();
-    subgraph_node_ids.push_back(node_id);
+    const string& node_name = node->name();
+
     // Create input connections. Sort edges first to make determnistic since
     // in_edges is a set of pointers.
     std::vector<const tensorflow::Edge*> in_edges(node->in_edges().begin(),
@@ -375,7 +376,7 @@ tensorflow::Status GetEngineInfo(
     std::sort(in_edges.begin(), in_edges.end(), EdgePtrCompare());
     for (const auto edge : in_edges) {
       auto input_node = edge->src();
-      if (input_node->IsSource() || segment_nodes.count(input_node->name())) {
+      if (input_node->IsSource() || segment_nodes.count(input_node)) {
         continue;
       }
       if (edge->IsControlEdge()) {
@@ -392,12 +393,11 @@ tensorflow::Status GetEngineInfo(
         //
         // Note that the segmenter already ensure that the constant data input
         // is valid and suppported by the engine.
-        if (!added_const_node_ids.insert(input_node->id()).second) {
+        if (!added_const_nodes.insert(input_node).second) {
           // Already added before.
           continue;
         }
         VLOG(1) << "Adding const node " << input_node->name();
-        QCHECK(subgraph_node_names.insert(input_node->name()).second);
         // Since we already add (duplicate) the const input node to the segment
         // graphdef, it's now not a data dependency any more, but to make the
         // dependency correct we still add a control dependency.
@@ -428,7 +428,7 @@ tensorflow::Status GetEngineInfo(
     std::sort(out_edges.begin(), out_edges.end(), EdgePtrCompare());
     for (const auto edge : out_edges) {
       auto output_node = edge->dst();
-      if (output_node->IsSink() || segment_nodes.count(output_node->name())) {
+      if (output_node->IsSink() || segment_nodes.count(output_node)) {
         continue;
       }
       if (edge->IsControlEdge()) {
@@ -456,12 +456,11 @@ tensorflow::Status GetEngineInfo(
   }  // For each segment node in topological order.
 
   // Construct the const nodes first.
-  subgraph_node_ids.insert(subgraph_node_ids.begin(),
-                           added_const_node_ids.begin(),
-                           added_const_node_ids.end());
+  subgraph_nodes.insert(subgraph_nodes.begin(), added_const_nodes.begin(),
+                        added_const_nodes.end());
   TF_RETURN_IF_ERROR(ConvertSegmentToGraphDef(
-      g, graph_properties, subgraph_node_names, subgraph_node_ids,
-      &info->connections, &info->segment_graph_def, &info->engine_name));
+      g, graph_properties, subgraph_nodes, &info->connections,
+      &info->segment_graph_def, &info->engine_name));
   // TODO(sami): This should not happen once segmenter is updated.
   if (segment_devices.size() == 1) {
     info->device = *segment_devices.begin();
@@ -1033,27 +1032,31 @@ tensorflow::Status ConvertAfterShapes(ConversionParams& params) {
     cudaSetDevice(cuda_device_id);
     auto status = CreateTRTNode(engine_segments, i, params.max_batch_size,
                                 &graph, alloc.get(), &engine_nodes);
-    // If status is ok, we successfully added the node to the graph and can
-    // remove segment ops. Otherwise graph is not modified.
+
     string msg = StrCat("TensorRT node ", engine.engine_name,
                         " added for segment ", i, " consisting of ",
                         converted_segments.at(i).first.size(), " nodes");
     if (status.ok()) {
       LOG(INFO) << msg << " succeeded.";
-      for (auto node_name : converted_segments.at(i).first) {
-        graph.RemoveNode(node_map.at(node_name));
-      }
     } else {
       // Graph is not modified.
       LOG(WARNING) << msg << " failed: " << status << ". Fallback to TF...";
     }
     if (VLOG_IS_ON(1)) {
       msg = "Segment consists of nodes: ";
-      for (const string& node_name : converted_segments.at(i).first) {
-        StrAppend(&msg, node_name, ", ");
+      for (const Node* node : converted_segments.at(i).first) {
+        StrAppend(&msg, node->name(), ", ");
       }
       VLOG(1) << msg;
     }
+
+    // If status is ok, we successfully added the node to the graph and can
+    // remove segment ops. Otherwise graph is not modified.
+    if (status.ok()) {
+      for (const Node* node : converted_segments.at(i).first) {
+        graph.RemoveNode(const_cast<Node*>(node));
+      }
+    }
   }
   cudaSetDevice(old_cuda_device);
   graph.ToGraphDef(params.output_graph_def);
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 179d631a3e..7b0c4b446d 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -3780,8 +3780,7 @@ tensorflow::Status ConvertGraphDefToEngine(
 tensorflow::Status ConvertSegmentToGraphDef(
     const tensorflow::Graph* graph,
     const tensorflow::grappler::GraphProperties& graph_properties,
-    const std::set<string>& subgraph_node_names,
-    const std::vector<int>& subgraph_node_ids,  // In topological order
+    const std::vector<const Node*>& subgraph_nodes,  // In topological order
     std::vector<EngineConnection>* connections,
     tensorflow::GraphDef* segment_def, string* common_scope) {
   std::set<string> marker_nodes;
@@ -3855,11 +3854,10 @@ tensorflow::Status ConvertSegmentToGraphDef(
 
   std::unordered_map<int, int> old_to_new_id_map;
   // Copy internal nodes to new graphdef
-  string local_scope = graph->FindNodeId(*subgraph_node_ids.begin())->name();
-  for (const auto node_id : subgraph_node_ids) {
-    const auto node = graph->FindNodeId(node_id);
+  string local_scope = subgraph_nodes.front()->name();
+  for (const Node* node : subgraph_nodes) {
     local_scope = GetCommonNameScope(local_scope, node->name());
-    old_to_new_id_map[node_id] = segment_def->node_size();
+    old_to_new_id_map[node->id()] = segment_def->node_size();
     auto snode = segment_def->add_node();
     snode->CopyFrom(node->def());
     VLOG(2) << "Copying " << snode->name() << " to subgraph";
@@ -3877,6 +3875,11 @@ tensorflow::Status ConvertSegmentToGraphDef(
             << placeholder_name;
     snode->set_input(connection.inside_port, placeholder_name);
   }
+  std::set<string> subgraph_node_names;
+  for (const Node* node : subgraph_nodes) {
+    subgraph_node_names.insert(node->name());
+  }
+
   // Remove control inputs that are not inside the segment.
   for (int i = 0; i < segment_def->node_size(); ++i) {
     auto snode = segment_def->mutable_node(i);
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.h b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
index 54e19b7395..8f2271ee3f 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.h
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
@@ -128,8 +128,7 @@ struct EngineInfo {
 tensorflow::Status ConvertSegmentToGraphDef(
     const tensorflow::Graph* graph,
     const tensorflow::grappler::GraphProperties& graph_properties,
-    const std::set<string>& subgraph_node_names,
-    const std::vector<int>& subgraph_node_ids,
+    const std::vector<const Node*>& subgraph_nodes,
     std::vector<EngineConnection>* connections,
     tensorflow::GraphDef* segment_def, string* common_scope);
 
diff --git a/tensorflow/contrib/tensorrt/segment/segment.cc b/tensorflow/contrib/tensorrt/segment/segment.cc
index 084a96e0fa..ecaffa3023 100644
--- a/tensorflow/contrib/tensorrt/segment/segment.cc
+++ b/tensorflow/contrib/tensorrt/segment/segment.cc
@@ -673,10 +673,11 @@ tensorflow::Status SegmentGraph(
   // --------------------------------- Step 3 ---------------------------------
   // Convert the segments into the expected return format
   for (const auto& itr : sg_map) {
-    const std::set<const tensorflow::Node*, NodePtrCompare>& segment_nodes =
-        itr.second;
+    const string& segment_root = itr.first;
+    // Return format does not require set comparator.
+    std::set<const Node*> segment_nodes(itr.second.begin(), itr.second.end());
     if (VLOG_IS_ON(1)) {
-      string s = "parent=" + itr.first + ":";
+      string s = "parent=" + segment_root + ":";
       for (auto node : segment_nodes) s += " " + node->name();
       VLOG(1) << "Segment " << segments->size() << ": " << s;
     }
@@ -689,12 +690,10 @@ tensorflow::Status SegmentGraph(
     }
 
     // TODO(sami): Make segmenter placement aware once trtscopes are in place
-    std::set<string> segment_node_names;
-    for (auto node : itr.second) segment_node_names.insert(node->name());
-    const auto& dev_itr = device_maps.find(itr.first);
+    const auto& dev_itr = device_maps.find(segment_root);
     if (dev_itr == device_maps.end() || dev_itr->second.empty()) {
       VLOG(1) << "No device assigned to segment " << segments->size();
-      segments->emplace_back(std::make_pair(segment_node_names, string()));
+      segments->emplace_back(std::make_pair(segment_nodes, string()));
     } else if (dev_itr->second.size() > 1) {
       string s("Segment ");
       StrAppend(&s, segments->size(), " has multiple devices attached: ");
@@ -703,10 +702,10 @@ tensorflow::Status SegmentGraph(
       }
       LOG(WARNING) << s << " choosing " << *(dev_itr->second.begin());
       segments->emplace_back(
-          std::make_pair(segment_node_names, *(dev_itr->second.begin())));
+          std::make_pair(segment_nodes, *(dev_itr->second.begin())));
     } else {
       segments->emplace_back(
-          std::make_pair(segment_node_names, *(dev_itr->second.begin())));
+          std::make_pair(segment_nodes, *(dev_itr->second.begin())));
     }
   }
   if (VLOG_IS_ON(1)) {
diff --git a/tensorflow/contrib/tensorrt/segment/segment.h b/tensorflow/contrib/tensorrt/segment/segment.h
index b9693aad1b..6cc92cdb5d 100644
--- a/tensorflow/contrib/tensorrt/segment/segment.h
+++ b/tensorflow/contrib/tensorrt/segment/segment.h
@@ -29,10 +29,10 @@ namespace tensorflow {
 namespace tensorrt {
 namespace segment {
 
-// Vector of segments, each entry contains a set of node names and a device name
-// in the segment.
-// TODO(aaroey): use node pointer instead of node name.
-using SegmentNodesVector = std::vector<std::pair<std::set<string>, string>>;
+// Vector of segments, each entry contains a set of node pointers and a device
+// name in the segment.
+using SegmentNodesVector =
+    std::vector<std::pair<std::set<const Node*>, string>>;
 
 struct SegmentOptions {
   // Segment must contain at least this many nodes.
diff --git a/tensorflow/contrib/tensorrt/segment/segment_test.cc b/tensorflow/contrib/tensorrt/segment/segment_test.cc
index 4805ef9c61..4ac02327ae 100644
--- a/tensorflow/contrib/tensorrt/segment/segment_test.cc
+++ b/tensorflow/contrib/tensorrt/segment/segment_test.cc
@@ -75,7 +75,10 @@ class SegmentTest : public ::testing::Test {
                        const std::vector<std::set<string>>& expected_segments) {
     EXPECT_EQ(expected_segments.size(), segments.size());
     for (int i = 0; i < segments.size(); ++i) {
-      const auto& segment_node_names = segments[i].first;
+      std::set<string> segment_node_names;
+      for (const Node* node : segments[i].first) {
+        segment_node_names.insert(node->name());
+      }
       const auto& expected = expected_segments[i];
       for (const auto& name : expected) {
         EXPECT_TRUE(segment_node_names.count(name))
-- 
GitLab


From 1e104bb923494b667a0ac8c28dac0d8fe516239c Mon Sep 17 00:00:00 2001
From: Grzegorz Pawelczak <grzegorzp@graphcore.ai>
Date: Thu, 10 Jan 2019 08:36:05 +0000
Subject: [PATCH 0517/1095] Fix nit in build file

---
 tensorflow/compiler/xla/service/BUILD | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index 7be180c624..4b691c7340 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -3484,9 +3484,9 @@ tf_cc_test(
     srcs = ["sort_simplifier_test.cc"],
     deps = [
         ":hlo_matchers",
-        ":sort_simplifier",
         ":pattern_matcher",
         ":pattern_matcher_gmock",
+        ":sort_simplifier",
         "//tensorflow/compiler/xla:test",
         "//tensorflow/compiler/xla/service:hlo_parser",
         "//tensorflow/compiler/xla/tests:hlo_test_base",
-- 
GitLab


From 439d4ec9568a259f665efd878791530a6f5ca8f2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 10 Jan 2019 01:02:17 -0800
Subject: [PATCH 0518/1095] compat: Update forward compatibility horizon to
 2019-01-10

PiperOrigin-RevId: 228661930
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index dfee972cab..638bd445a6 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 9)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 10)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 2f7e56a65104aeca0a7fcc98a7b10fb2cff9b1a3 Mon Sep 17 00:00:00 2001
From: Tamara Norman <tamaranorman@google.com>
Date: Thu, 10 Jan 2019 02:46:36 -0800
Subject: [PATCH 0519/1095] Remove partitioned_info as argument to initializer
 in Keras make_variable to work with the v2 initializers

PiperOrigin-RevId: 228675005
---
 tensorflow/python/keras/engine/base_layer_utils.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/keras/engine/base_layer_utils.py b/tensorflow/python/keras/engine/base_layer_utils.py
index 60d0cf3391..f1b1f42c22 100644
--- a/tensorflow/python/keras/engine/base_layer_utils.py
+++ b/tensorflow/python/keras/engine/base_layer_utils.py
@@ -25,6 +25,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.keras import backend
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import init_ops
+from tensorflow.python.ops import init_ops_v2
 from tensorflow.python.ops import variables as tf_variables
 from tensorflow.python.util import nest
 
@@ -55,7 +56,6 @@ def make_variable(name,
                   shape=None,
                   dtype=dtypes.float32,
                   initializer=None,
-                  partition_info=None,
                   trainable=None,
                   caching_device=None,
                   validate_shape=True,
@@ -76,14 +76,12 @@ def make_variable(name,
   rid of this temporary solution.
 
   TODO(fchollet): remove this method when no longer needed.
-  TODO(fchollet): handle `partitioner` argument.
 
   Arguments:
     name: Variable name.
     shape: Variable shape.
     dtype: The type of the variable. Defaults to `self.dtype` or `float32`.
     initializer: Initializer instance (callable).
-    partition_info: Not handled at this time.
     trainable: Whether the variable should be part of the layer's
       "trainable_variables" (e.g. variables, biases)
       or "non_trainable_variables" (e.g. BatchNorm mean, stddev).
@@ -123,8 +121,9 @@ def make_variable(name,
       # Instantiate initializer if provided initializer is a type object.
       if isinstance(initializer, type(init_ops.Initializer)):
         initializer = initializer(dtype=dtype)
-      init_val = lambda: initializer(  # pylint: disable=g-long-lambda
-          shape, dtype=dtype, partition_info=partition_info)
+      elif isinstance(initializer, type(init_ops_v2.Initializer)):
+        initializer = initializer()
+      init_val = lambda: initializer(shape, dtype=dtype)
       variable_dtype = dtype.base_dtype
   if use_resource is None:
     use_resource = True
-- 
GitLab


From ef088e54734e0412bc47e4f33d1df7c0672fdd87 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 10 Jan 2019 04:21:44 -0800
Subject: [PATCH 0520/1095] Add a test for soft matching of inputs.

PiperOrigin-RevId: 228686125
---
 tensorflow/python/saved_model/load_test.py | 31 ++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index f7020e4a45..8b34414d25 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -29,6 +29,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import tensor_spec
 from tensorflow.python.lib.io import file_io
 from tensorflow.python.ops import variables
+from tensorflow.python.saved_model import function_serialization
 from tensorflow.python.saved_model import load
 from tensorflow.python.saved_model import save
 from tensorflow.python.training.checkpointable import tracking
@@ -312,6 +313,36 @@ class LoadTest(test.TestCase):
     x = constant_op.constant(1.0)
     self.assertAllEqual(imported(x).numpy(), 3.0)
 
+  def test_soft_matching(self):
+
+    @def_function.function(
+        input_signature=[tensor_spec.TensorSpec([None], dtypes.int32)])
+    def func(x):
+      return 2 * x
+
+    root = tracking.Checkpointable()
+    root.f = func
+
+    self.assertAllEqual([2], root.f(constant_op.constant([1])).numpy())
+    self.assertAllEqual([2, 4], root.f(constant_op.constant([1, 2])).numpy())
+
+    self.assertEqual(
+        1, len(function_serialization.list_all_concrete_functions(root.f)))
+
+    imported = self.cycle(root)
+
+    with self.assertRaises(AssertionError):
+      # We cannot call the function with a constant of shape ().
+      self.assertEqual(7, imported.f(constant_op.constant(2)).numpy())
+
+    # TODO(vbardiovsky): When classes are revived with input_signatures, we
+    # should also check that the calls below are not generating any more
+    # concrete functions.
+    self.assertAllEqual([2, 4, 6, 8],
+                        imported.f(constant_op.constant([1, 2, 3, 4])).numpy())
+    self.assertAllEqual([2, 4, 6],
+                        imported.f(constant_op.constant([1, 2, 3])).numpy())
+
 
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From 2af977ba922fe498e30c1fcc30942709f75d2e78 Mon Sep 17 00:00:00 2001
From: Nutti <nutti.metro@gmail.com>
Date: Thu, 10 Jan 2019 23:07:08 +0900
Subject: [PATCH 0521/1095] Fix: typo in SavedModelBundle.java

---
 .../java/src/main/java/org/tensorflow/SavedModelBundle.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/java/src/main/java/org/tensorflow/SavedModelBundle.java b/tensorflow/java/src/main/java/org/tensorflow/SavedModelBundle.java
index 49594e6b47..e653373f85 100644
--- a/tensorflow/java/src/main/java/org/tensorflow/SavedModelBundle.java
+++ b/tensorflow/java/src/main/java/org/tensorflow/SavedModelBundle.java
@@ -84,7 +84,7 @@ public class SavedModelBundle implements AutoCloseable {
    * <p>This method is a shorthand for:
    *
    * <pre>{@code
-   * SavedModelBundler.loader().withTags(tags).load();
+   * SavedModelBundle.loader().withTags(tags).load();
    * }</pre>
    *
    * @param exportDir the directory path containing a saved model.
-- 
GitLab


From 9ad0810fd55096fc86a58300c5a2710b2f3b5175 Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Thu, 10 Jan 2019 08:15:42 -0800
Subject: [PATCH 0522/1095] [XLA:Python] Fix build error in opensource due to
 xla::int64/protobuf_int64 differences.

PiperOrigin-RevId: 228713538
---
 tensorflow/compiler/xla/python/local_computation_builder.i | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/python/local_computation_builder.i b/tensorflow/compiler/xla/python/local_computation_builder.i
index cc96d84724..66ecee5c4d 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.i
+++ b/tensorflow/compiler/xla/python/local_computation_builder.i
@@ -171,7 +171,7 @@ bool HandleStringAttribute(PyObject* o,
 
 bool HandleRepeatedInt64Attribute(
     PyObject* o, const char* attr_name,
-    tensorflow::protobuf::RepeatedField<int64>* field) {
+    tensorflow::protobuf::RepeatedField<tensorflow::protobuf_int64>* field) {
   PyObject* seq = PyObject_GetAttrString(o, attr_name);
   if (!seq) {
     return false;
-- 
GitLab


From 4e462bc347ddadeb939e4861f77fa349c083fe64 Mon Sep 17 00:00:00 2001
From: Karim Nosir <karimnosseir@google.com>
Date: Thu, 10 Jan 2019 09:31:51 -0800
Subject: [PATCH 0523/1095] Update unit-test to enable for OSS

PiperOrigin-RevId: 228724452
---
 tensorflow/lite/models/smartreply/BUILD           |  1 -
 .../lite/models/smartreply/predictor_test.cc      | 15 +++++++++------
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/tensorflow/lite/models/smartreply/BUILD b/tensorflow/lite/models/smartreply/BUILD
index a3f4e61cb5..fccbf79eb9 100644
--- a/tensorflow/lite/models/smartreply/BUILD
+++ b/tensorflow/lite/models/smartreply/BUILD
@@ -58,7 +58,6 @@ tf_cc_test(
         "//tensorflow/lite/models:testdata/smartreply_samples.tsv",
         "@tflite_smartreply//:smartreply.tflite",
     ],
-    tags = ["no_oss"],
     deps = [
         ":predictor_lib",
         "//tensorflow/core:test",
diff --git a/tensorflow/lite/models/smartreply/predictor_test.cc b/tensorflow/lite/models/smartreply/predictor_test.cc
index 9bdd7b537a..9ce8ce9a77 100644
--- a/tensorflow/lite/models/smartreply/predictor_test.cc
+++ b/tensorflow/lite/models/smartreply/predictor_test.cc
@@ -31,12 +31,16 @@ namespace custom {
 namespace smartreply {
 namespace {
 
-const char kModelName[] = "smartreply_ondevice_model.bin";
 const char kSamples[] = "smartreply_samples.tsv";
 
-string TestDataPath() {
+string GetModelFilePath() {
+  return "third_party/tensorflow/lite/models/testdata/"
+         "smartreply_ondevice_model.bin";
+}
+
+string GetSamplesFilePath() {
   return string(absl::StrCat(tensorflow::testing::TensorFlowSrcRoot(), "/",
-                             "lite/models/testdata/"));
+                             "lite/models/testdata/", kSamples));
 }
 
 MATCHER_P(IncludeAnyResponesIn, expected_response, "contains the response") {
@@ -57,8 +61,7 @@ class PredictorTest : public ::testing::Test {
   ~PredictorTest() override {}
 
   void SetUp() override {
-    model_ = tflite::FlatBufferModel::BuildFromFile(
-        absl::StrCat(TestDataPath(), "/", kModelName).c_str());
+    model_ = tflite::FlatBufferModel::BuildFromFile(GetModelFilePath().c_str());
     ASSERT_NE(model_.get(), nullptr);
   }
 
@@ -123,7 +126,7 @@ TEST_F(PredictorTest, BatchTest) {
   int total_triggers = 0;
 
   string line;
-  std::ifstream fin(absl::StrCat(TestDataPath(), "/", kSamples));
+  std::ifstream fin(GetSamplesFilePath());
   while (std::getline(fin, line)) {
     const std::vector<string> fields = absl::StrSplit(line, '\t');
     if (fields.empty()) {
-- 
GitLab


From 73b6d9262ea6d1578374ecb63560ad0fcad13e35 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 10 Jan 2019 09:55:14 -0800
Subject: [PATCH 0524/1095] Split Keras TPU correctness tests by models.

PiperOrigin-RevId: 228727689
---
 tensorflow/contrib/distribute/python/BUILD    |  30 +-
 ...test.py => keras_correctness_test_base.py} | 261 ++++--------------
 .../python/keras_dnn_correctness_test.py      | 163 +++++++++++
 .../keras_image_model_correctness_test.py     |  92 ++++++
 4 files changed, 337 insertions(+), 209 deletions(-)
 rename tensorflow/contrib/distribute/python/{keras_correctness_test.py => keras_correctness_test_base.py} (58%)
 create mode 100644 tensorflow/contrib/distribute/python/keras_dnn_correctness_test.py
 create mode 100644 tensorflow/contrib/distribute/python/keras_image_model_correctness_test.py

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index d4758d7518..4c25f49f16 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -657,7 +657,11 @@ cuda_py_test(
 py_library(
     name = "keras_correctness_test_lib",
     testonly = 1,
-    srcs = ["keras_correctness_test.py"],
+    srcs = [
+        "keras_correctness_test_base.py",
+        "keras_dnn_correctness_test.py",
+        "keras_image_model_correctness_test.py",
+    ],
     deps = [
         ":combinations",
         "//tensorflow/contrib/distribute/python:mirrored_strategy",
@@ -673,8 +677,9 @@ py_library(
 )
 
 cuda_py_test(
-    name = "keras_correctness_test",
-    srcs = ["keras_correctness_test.py"],
+    name = "keras_dnn_correctness_test",
+    size = "medium",
+    srcs = ["keras_dnn_correctness_test.py"],
     additional_deps = [
         ":keras_correctness_test_lib",
     ],
@@ -690,6 +695,25 @@ cuda_py_test(
     ],
 )
 
+cuda_py_test(
+    name = "keras_image_model_correctness_test",
+    size = "medium",
+    srcs = ["keras_image_model_correctness_test.py"],
+    additional_deps = [
+        ":keras_correctness_test_lib",
+    ],
+    # Shard count is set to an odd number to distribute tasks across
+    # shards more evenly.
+    shard_count = 31,
+    tags = [
+        "multi_and_single_gpu",
+        "no_oss",  # TODO(b/117919883): Fix python error.
+        "no_pip",
+        "no_windows_gpu",
+        "notsan",
+    ],
+)
+
 py_library(
     name = "metrics_v1_test_lib",
     testonly = 1,
diff --git a/tensorflow/contrib/distribute/python/keras_correctness_test.py b/tensorflow/contrib/distribute/python/keras_correctness_test_base.py
similarity index 58%
rename from tensorflow/contrib/distribute/python/keras_correctness_test.py
rename to tensorflow/contrib/distribute/python/keras_correctness_test_base.py
index 3abdee2c0e..e13984b26d 100644
--- a/tensorflow/contrib/distribute/python/keras_correctness_test.py
+++ b/tensorflow/contrib/distribute/python/keras_correctness_test_base.py
@@ -30,8 +30,6 @@ from tensorflow.python.eager import context
 from tensorflow.python.eager import test
 from tensorflow.python.framework import random_seed
 from tensorflow.python.keras.engine import distributed_training_utils
-from tensorflow.python.keras.optimizer_v2 import gradient_descent as gradient_descent_keras
-from tensorflow.python.training import gradient_descent
 
 _RANDOM_SEED = 1337
 _EVAL_STEPS = 20
@@ -62,24 +60,13 @@ def all_strategy_combinations_with_graph_mode():
   return combinations.combine(distribution=all_strategies, mode=['graph'])
 
 
-def strategy_and_input_combinations():
-  def cnn_model_with_batch_norm(**kwargs):
-    return _create_cnn_model(with_batch_norm=True, **kwargs)
-
+def all_strategy_and_input_config_combinations():
   return (
       combinations.times(
           combinations.combine(distribution=all_strategies),
           combinations.combine(mode=['graph', 'eager'],
                                use_numpy=[True, False],
-                               use_validation_data=[True, False]),
-          combinations.combine(model_with_data=[
-              ModelWithData('dnn', _create_dnn_model, _dnn_training_data),
-              ModelWithData('cnn', _create_cnn_model, _cnn_training_data),
-              ModelWithData('cnn_batch_norm',
-                            cnn_model_with_batch_norm,
-                            _cnn_training_data,
-                            with_batch_norm=True),
-          ])))
+                               use_validation_data=[True, False])))
 
 
 class MaybeDistributionScope(object):
@@ -100,97 +87,6 @@ class MaybeDistributionScope(object):
       self._scope = None
 
 
-class ModelWithData(object):
-  """An object giving a good name in combinations.
-
-  The model_fn must take two arguments: initial_weights and distribution.
-  """
-
-  def __init__(self, name, model_fn, data_fn, with_batch_norm=False):
-    self.name = name
-    self.model_fn = model_fn
-    self.data_fn = data_fn
-    self.with_batch_norm = with_batch_norm
-
-  def __repr__(self):
-    return self.name
-
-
-def _dnn_training_data():
-  # TODO(xiejw): Change this back to 10000, once we support final partial
-  # batch.
-  num_samples = 9984
-  x_train = np.random.rand(num_samples, 1)
-  y_train = 3 * x_train
-  x_train = x_train.astype('float32')
-  y_train = y_train.astype('float32')
-  x_predict = [[1.], [2.], [3.], [4.]]
-  return x_train, y_train, x_predict
-
-
-def _create_dnn_model(initial_weights=None, distribution=None):
-  with MaybeDistributionScope(distribution):
-    # We add few non-linear layers to make it non-trivial.
-    model = keras.Sequential()
-    model.add(keras.layers.Dense(10, activation='relu', input_shape=(1,)))
-    model.add(keras.layers.Dense(10, activation='relu'))
-    model.add(keras.layers.Dense(10, activation='relu'))
-    model.add(keras.layers.Dense(1))
-
-    if initial_weights:
-      model.set_weights(initial_weights)
-
-    model.compile(
-        loss=keras.losses.mean_squared_error,
-        optimizer=gradient_descent_keras.SGD(0.5),
-        metrics=['mse'])
-    return model
-
-
-def _cnn_training_data(count=_GLOBAL_BATCH_SIZE * _EVAL_STEPS,
-                       shape=(28, 28, 3), num_classes=10):
-  centers = np.random.randn(num_classes, *shape)
-
-  features = []
-  labels = []
-  for _ in range(count):
-    label = np.random.randint(0, num_classes, size=1)[0]
-    offset = np.random.normal(loc=0, scale=0.1, size=np.prod(shape))
-    offset = offset.reshape(shape)
-    labels.append(label)
-    features.append(centers[label] + offset)
-
-  x_train = np.asarray(features, dtype=np.float32)
-  y_train = np.asarray(labels, dtype=np.float32).reshape((count, 1))
-  x_predict = x_train
-  return x_train, y_train, x_predict
-
-
-def _create_cnn_model(initial_weights=None, distribution=None,
-                      with_batch_norm=False):
-  with MaybeDistributionScope(distribution):
-    image = keras.layers.Input(shape=(28, 28, 3), name='image')
-    c1 = keras.layers.Conv2D(
-        name='conv1', filters=16, kernel_size=(3, 3), strides=(4, 4))(
-            image)
-    if with_batch_norm:
-      c1 = keras.layers.BatchNormalization(name='bn1')(c1)
-    c1 = keras.layers.MaxPooling2D(pool_size=(2, 2))(c1)
-    logits = keras.layers.Dense(
-        10, activation='softmax', name='pred')(
-            keras.layers.Flatten()(c1))
-    model = keras.Model(inputs=[image], outputs=[logits])
-
-    if initial_weights:
-      model.set_weights(initial_weights)
-
-    model.compile(
-        optimizer=gradient_descent.GradientDescentOptimizer(learning_rate=0.1),
-        loss='sparse_categorical_crossentropy',
-        metrics=['sparse_categorical_accuracy'])
-  return model
-
-
 def batch_wrapper(dataset, batch_size, distribution, repeat=None):
   if repeat:
     dataset = dataset.repeat(repeat)
@@ -350,6 +246,11 @@ def compare_results(results_with_ds, results_without_ds, distribution,
         msg='Fail to assert {}.'.format(key))
 
 
+def should_skip_tpu_with_eager(distribution):
+  return (context.executing_eagerly() and
+          isinstance(distribution, tpu_strategy.TPUStrategy))
+
+
 class LearningRateBatchScheduler(keras.callbacks.Callback):
 
   def __init__(self, update_freq=None):
@@ -364,107 +265,60 @@ class LearningRateBatchScheduler(keras.callbacks.Callback):
     keras.backend.set_value(self.model.optimizer.lr, lr)
 
 
-class TestDistributionStrategyCorrectness(test.TestCase,
-                                          parameterized.TestCase):
+class TestDistributionStrategyCorrectnessBase(test.TestCase,
+                                              parameterized.TestCase):
+  """Model agnostic testing infra to test correctness of Keras models."""
+
+  def set_up_test_config(self, use_numpy=False,
+                         use_validation_data=False,
+                         with_batch_norm=False):
+    self.use_numpy = use_numpy
+    self.use_validation_data = use_validation_data
+    self.with_batch_norm = with_batch_norm
 
-  def _should_skip_tpu_with_eager(self, distribution):
-    return (context.executing_eagerly() and
-            isinstance(distribution, tpu_strategy.TPUStrategy))
+    keras.backend.set_image_data_format('channels_last')
+    np.random.seed(_RANDOM_SEED)
+    random_seed.set_random_seed(_RANDOM_SEED)
 
-  @combinations.generate(all_strategy_combinations_with_eager_and_graph_modes())
-  def test_metric_correctness(self, distribution):
-    if self._should_skip_tpu_with_eager(distribution):
-      self.skipTest('TPUStrategy does not support eager mode now.')
+  def get_data(self):
+    num_samples = 10000
+    x_train = np.random.randint(0, 2, num_samples)
+    x_train = np.reshape(x_train, (num_samples, 1))
+    y_train = x_train
+    return (x_train.astype('float32'), y_train.astype('float32'), None)
 
-    with self.cached_session():
-      keras.backend.set_image_data_format('channels_last')
-      num_samples = 10000
-
-      x_train = np.random.randint(0, 2, num_samples)
-      x_train = np.reshape(x_train, (num_samples, 1))
-      y_train = x_train
-      x_train = x_train.astype('float32')
-      y_train = y_train.astype('float32')
-
-      # Create identity model.
-      with distribution.scope():
-        model = keras.Sequential()
-        model.add(
-            keras.layers.Dense(1, input_shape=(1,), kernel_initializer='ones'))
-        model.compile(
-            loss=keras.losses.mean_squared_error,
-            optimizer=gradient_descent.GradientDescentOptimizer(0.5),
-            metrics=[keras.metrics.BinaryAccuracy()])
-
-      batch_size = 64
-      batch_size = get_batch_size(batch_size, distribution)
-      train_dataset = dataset_ops.Dataset.from_tensor_slices((x_train, y_train))
-      train_dataset = batch_wrapper(train_dataset, batch_size, distribution)
-
-      history = model.fit(x=train_dataset, epochs=2, steps_per_epoch=10)
-      self.assertEqual(history.history['binary_accuracy'], [1.0, 1.0])
-
-  @combinations.generate(all_strategy_combinations_with_eager_and_graph_modes())
-  def test_eval_metrics_correctness(self, distribution):
-    if self._should_skip_tpu_with_eager(distribution):
-      self.skipTest('TPUStrategy does not support eager mode now.')
+  def get_model(self, distribution=None):
+    raise NotImplementedError
 
-    with self.cached_session():
-      with distribution.scope():
-        model = keras.Sequential()
-        model.add(
-            keras.layers.Dense(
-                3, activation='relu', input_dim=4, kernel_initializer='ones'))
-        model.add(
-            keras.layers.Dense(
-                1, activation='sigmoid', kernel_initializer='ones'))
-        model.compile(
-            loss='mae',
-            metrics=['accuracy', keras.metrics.BinaryAccuracy()],
-            optimizer=gradient_descent.GradientDescentOptimizer(0.001))
-
-      # verify correctness of stateful and stateless metrics.
-      x = np.ones((100, 4)).astype('float32')
-      y = np.ones((100, 1)).astype('float32')
-      dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).repeat()
-      dataset = batch_wrapper(dataset, 4, distribution)
-      outs = model.evaluate(dataset, steps=10)
-      self.assertEqual(outs[1], 1.)
-      self.assertEqual(outs[2], 1.)
-
-      y = np.zeros((100, 1)).astype('float32')
-      dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).repeat()
-      dataset = batch_wrapper(dataset, 4, distribution)
-      outs = model.evaluate(dataset, steps=10)
-      self.assertEqual(outs[1], 0.)
-      self.assertEqual(outs[2], 0.)
-
-  @combinations.generate(strategy_and_input_combinations())
-  def test_correctness(self, distribution, use_numpy, use_validation_data,
-                       model_with_data):
-    if self._should_skip_tpu_with_eager(distribution):
+  def skip_unsupported_test_configuration(self, distribution):
+    if should_skip_tpu_with_eager(distribution):
       self.skipTest('TPUStrategy does not support eager mode now.')
 
-    if context.executing_eagerly() and use_numpy:
+    if context.executing_eagerly() and self.use_numpy:
       self.skipTest('Numpy as inputs is not supported with strategy in eager.')
 
-    if context.executing_eagerly() and use_validation_data:
-      self.skipTest('TODO')
+    if context.executing_eagerly() and self.use_validation_data:
+      self.skipTest('TODO(hongjunchoi): Add test logic for using validation '
+                    'data for eager execution.')
+    return
 
+  def run_correctness_test(self,
+                           distribution,
+                           use_numpy,
+                           use_validation_data,
+                           with_batch_norm=False):
     with self.cached_session():
-      keras.backend.set_image_data_format('channels_last')
-      np.random.seed(_RANDOM_SEED)
-      random_seed.set_random_seed(_RANDOM_SEED)
+      self.set_up_test_config(use_numpy, use_validation_data, with_batch_norm)
+      self.skip_unsupported_test_configuration(distribution)
 
-      model_fn, data_fn = model_with_data.model_fn, model_with_data.data_fn
       # Train, eval, and predict datasets are created with the same input numpy
       # arrays.
-      x_train, y_train, x_predict = data_fn()
+      x_train, y_train, x_predict = self.get_data()
 
       # The model is built once and the initial weights are saved.
       # This is used to initialize the model for both the distribution and
       # non-distribution run.
-      model = model_fn()
+      model = self.get_model()
       initial_weights = model.get_weights()
 
       def input_fn(dist):
@@ -472,15 +326,15 @@ class TestDistributionStrategyCorrectness(test.TestCase,
             use_numpy, use_validation_data, dist, x_train, y_train, x_predict)
 
       results_with_ds = fit_eval_and_predict(
-          initial_weights, input_fn=input_fn, model_fn=model_fn,
+          initial_weights, input_fn=input_fn, model_fn=self.get_model,
           distribution=distribution)
       results_without_ds = fit_eval_and_predict(
-          initial_weights, input_fn=input_fn, model_fn=model_fn,
+          initial_weights, input_fn=input_fn, model_fn=self.get_model,
           distribution=None)
 
       # First, special case, for multi-replica distributed training, batch norm
       # is not aggregated globally. So it is expected to have different weights.
-      if (model_with_data.with_batch_norm and
+      if (self.with_batch_norm and
           distribution.num_replicas_in_sync > 1):
         with self.assertRaises(AssertionError):
           compare_results(results_with_ds, results_without_ds, distribution,
@@ -489,21 +343,16 @@ class TestDistributionStrategyCorrectness(test.TestCase,
         compare_results(results_with_ds, results_without_ds, distribution,
                         testcase=self)
 
-  @combinations.generate(all_strategy_combinations_with_graph_mode())
-  def test_dynamic_lr(self, distribution):
-
+  def run_dynamic_lr_test(self, distribution):
     with self.cached_session():
+      self.set_up_test_config()
+      self.skip_unsupported_test_configuration(distribution)
 
-      keras.backend.set_image_data_format('channels_last')
-      np.random.seed(_RANDOM_SEED)
-      random_seed.set_random_seed(_RANDOM_SEED)
-
-      x_train, y_train, _ = _dnn_training_data()
-
-      model = _create_dnn_model()
+      x_train, y_train, _ = self.get_data()
+      model = self.get_model()
       initial_weights = model.get_weights()
-
       update_freq = None
+
       if (isinstance(distribution, tpu_strategy.TPUStrategy) and
           distribution.extended.steps_per_run > 1):
         # For TPUStrategy with steps_per_run > 1, the callback is not invoked
@@ -530,10 +379,10 @@ class TestDistributionStrategyCorrectness(test.TestCase,
         return training_inputs, eval_inputs, predict_inputs
 
       results_with_ds = fit_eval_and_predict(
-          initial_weights, input_fn=input_fn, model_fn=_create_dnn_model,
+          initial_weights, input_fn=input_fn, model_fn=self.get_model,
           distribution=distribution)
       results_without_ds = fit_eval_and_predict(
-          initial_weights, input_fn=input_fn, model_fn=_create_dnn_model,
+          initial_weights, input_fn=input_fn, model_fn=self.get_model,
           distribution=None)
       compare_results(results_with_ds, results_without_ds, distribution,
                       testcase=self)
diff --git a/tensorflow/contrib/distribute/python/keras_dnn_correctness_test.py b/tensorflow/contrib/distribute/python/keras_dnn_correctness_test.py
new file mode 100644
index 0000000000..f21b1c6752
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/keras_dnn_correctness_test.py
@@ -0,0 +1,163 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Correctness tests for tf.keras DNN model using DistributionStrategy."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.distribute.python import combinations
+from tensorflow.contrib.distribute.python import keras_correctness_test_base
+from tensorflow.python import keras
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.eager import test
+from tensorflow.python.keras.optimizer_v2 import gradient_descent as gradient_descent_keras
+from tensorflow.python.training import gradient_descent
+
+
+class TestDistributionStrategyDnnCorrectness(
+    keras_correctness_test_base.TestDistributionStrategyCorrectnessBase):
+
+  def get_model(self, initial_weights=None, distribution=None):
+    with keras_correctness_test_base.MaybeDistributionScope(distribution):
+      # We add few non-linear layers to make it non-trivial.
+      model = keras.Sequential()
+      model.add(keras.layers.Dense(10, activation='relu', input_shape=(1,)))
+      model.add(keras.layers.Dense(10, activation='relu'))
+      model.add(keras.layers.Dense(10, activation='relu'))
+      model.add(keras.layers.Dense(1))
+
+      if initial_weights:
+        model.set_weights(initial_weights)
+
+      model.compile(
+          loss=keras.losses.mean_squared_error,
+          optimizer=gradient_descent_keras.SGD(0.5),
+          metrics=['mse'])
+      return model
+
+  def get_data(self):
+    # TODO(xiejw): Change this back to 10000, once we support final partial
+    # batch.
+    num_samples = 9984
+    x_train = np.random.rand(num_samples, 1)
+    y_train = 3 * x_train
+    x_train = x_train.astype('float32')
+    y_train = y_train.astype('float32')
+    x_predict = [[1.], [2.], [3.], [4.]]
+    return x_train, y_train, x_predict
+
+  @combinations.generate(keras_correctness_test_base.
+                         all_strategy_and_input_config_combinations())
+  def test_dnn_correctness(self, distribution, use_numpy, use_validation_data):
+    self.run_correctness_test(distribution, use_numpy, use_validation_data)
+
+  @combinations.generate(keras_correctness_test_base.
+                         all_strategy_combinations_with_graph_mode())
+  def test_dnn_with_dynamic_learning_rate(self, distribution):
+    self.run_dynamic_lr_test(distribution)
+
+
+class TestDistributionStrategyDnnMetricCorrectness(
+    keras_correctness_test_base.TestDistributionStrategyCorrectnessBase):
+
+  def get_model(self, distribution=None):
+    with distribution.scope():
+      model = keras.Sequential()
+      model.add(keras.layers.Dense(1,
+                                   input_shape=(1,),
+                                   kernel_initializer='ones'))
+      model.compile(
+          loss=keras.losses.mean_squared_error,
+          optimizer=gradient_descent.GradientDescentOptimizer(0.5),
+          metrics=[keras.metrics.BinaryAccuracy()])
+    return model
+
+  def run_metric_correctness_test(self, distribution):
+    with self.cached_session():
+      self.set_up_test_config()
+      self.skip_unsupported_test_configuration(distribution)
+
+      x_train, y_train, _ = self.get_data()
+      model = self.get_model(distribution=distribution)
+
+      batch_size = 64
+      batch_size = (keras_correctness_test_base.
+                    get_batch_size(batch_size, distribution))
+      train_dataset = dataset_ops.Dataset.from_tensor_slices((x_train, y_train))
+      train_dataset = (keras_correctness_test_base.
+                       batch_wrapper(train_dataset, batch_size, distribution))
+
+      history = model.fit(x=train_dataset, epochs=2, steps_per_epoch=10)
+      self.assertEqual(history.history['binary_accuracy'], [1.0, 1.0])
+
+  @combinations.generate(keras_correctness_test_base.
+                         all_strategy_combinations_with_eager_and_graph_modes())
+  def test_simple_dnn_metric_correctness(self, distribution):
+    self.run_metric_correctness_test(distribution)
+
+
+class TestDistributionStrategyDnnMetricEvalCorrectness(
+    keras_correctness_test_base.TestDistributionStrategyCorrectnessBase):
+
+  def get_model(self, distribution=None):
+    with distribution.scope():
+      model = keras.Sequential()
+      model.add(
+          keras.layers.Dense(
+              3, activation='relu', input_dim=4, kernel_initializer='ones'))
+      model.add(
+          keras.layers.Dense(
+              1, activation='sigmoid', kernel_initializer='ones'))
+      model.compile(
+          loss='mae',
+          metrics=['accuracy', keras.metrics.BinaryAccuracy()],
+          optimizer=gradient_descent.GradientDescentOptimizer(0.001))
+    return model
+
+  def run_eval_metrics_correctness_test(self, distribution):
+    with self.cached_session():
+      self.set_up_test_config()
+      self.skip_unsupported_test_configuration(distribution)
+
+      model = self.get_model(distribution=distribution)
+
+      # verify correctness of stateful and stateless metrics.
+      x = np.ones((100, 4)).astype('float32')
+      y = np.ones((100, 1)).astype('float32')
+      dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).repeat()
+      dataset = (keras_correctness_test_base.
+                 batch_wrapper(dataset, 4, distribution))
+      outs = model.evaluate(dataset, steps=10)
+      self.assertEqual(outs[1], 1.)
+      self.assertEqual(outs[2], 1.)
+
+      y = np.zeros((100, 1)).astype('float32')
+      dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).repeat()
+      dataset = (keras_correctness_test_base.
+                 batch_wrapper(dataset, 4, distribution))
+      outs = model.evaluate(dataset, steps=10)
+      self.assertEqual(outs[1], 0.)
+      self.assertEqual(outs[2], 0.)
+
+  @combinations.generate(keras_correctness_test_base.
+                         all_strategy_combinations_with_eager_and_graph_modes())
+  def test_identity_model_metric_eval_correctness(self, distribution):
+    self.run_eval_metrics_correctness_test(distribution)
+
+
+if __name__ == '__main__':
+  test.main()
diff --git a/tensorflow/contrib/distribute/python/keras_image_model_correctness_test.py b/tensorflow/contrib/distribute/python/keras_image_model_correctness_test.py
new file mode 100644
index 0000000000..f625664372
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/keras_image_model_correctness_test.py
@@ -0,0 +1,92 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Correctness tests for tf.keras CNN models using DistributionStrategy."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.distribute.python import combinations
+from tensorflow.contrib.distribute.python import keras_correctness_test_base
+from tensorflow.python import keras
+from tensorflow.python.eager import test
+from tensorflow.python.training import gradient_descent
+
+
+class DistributionStrategyCnnCorrectnessTest(
+    keras_correctness_test_base.TestDistributionStrategyCorrectnessBase):
+
+  def get_model(self, initial_weights=None, distribution=None):
+    with keras_correctness_test_base.MaybeDistributionScope(distribution):
+      image = keras.layers.Input(shape=(28, 28, 3), name='image')
+      c1 = keras.layers.Conv2D(
+          name='conv1', filters=16, kernel_size=(3, 3), strides=(4, 4))(
+              image)
+      if self.with_batch_norm:
+        c1 = keras.layers.BatchNormalization(name='bn1')(c1)
+      c1 = keras.layers.MaxPooling2D(pool_size=(2, 2))(c1)
+      logits = keras.layers.Dense(
+          10, activation='softmax', name='pred')(
+              keras.layers.Flatten()(c1))
+      model = keras.Model(inputs=[image], outputs=[logits])
+
+      if initial_weights:
+        model.set_weights(initial_weights)
+
+      model.compile(
+          optimizer=gradient_descent.GradientDescentOptimizer(
+              learning_rate=0.1),
+          loss='sparse_categorical_crossentropy',
+          metrics=['sparse_categorical_accuracy'])
+
+    return model
+
+  def get_data(self,
+               count=keras_correctness_test_base._GLOBAL_BATCH_SIZE
+               * keras_correctness_test_base._EVAL_STEPS,
+               shape=(28, 28, 3),
+               num_classes=10):
+    centers = np.random.randn(num_classes, *shape)
+
+    features = []
+    labels = []
+    for _ in range(count):
+      label = np.random.randint(0, num_classes, size=1)[0]
+      offset = np.random.normal(loc=0, scale=0.1, size=np.prod(shape))
+      offset = offset.reshape(shape)
+      labels.append(label)
+      features.append(centers[label] + offset)
+
+    x_train = np.asarray(features, dtype=np.float32)
+    y_train = np.asarray(labels, dtype=np.float32).reshape((count, 1))
+    x_predict = x_train
+    return x_train, y_train, x_predict
+
+  @combinations.generate(keras_correctness_test_base.
+                         all_strategy_and_input_config_combinations())
+  def test_cnn_correctness(self, distribution, use_numpy, use_validation_data):
+    self.run_correctness_test(distribution, use_numpy, use_validation_data)
+
+  @combinations.generate(keras_correctness_test_base.
+                         all_strategy_and_input_config_combinations())
+  def test_cnn_with_batch_norm_correctness(self, distribution, use_numpy,
+                                           use_validation_data):
+    self.run_correctness_test(distribution, use_numpy, use_validation_data,
+                              with_batch_norm=True)
+
+
+if __name__ == '__main__':
+  test.main()
-- 
GitLab


From 1210b6506dbde52a5bf5c33d56f3c52dfbd505e7 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 10 Jan 2019 10:28:19 -0800
Subject: [PATCH 0525/1095] fix links to renamed markdown documentation

PiperOrigin-RevId: 228733282
---
 tensorflow/lite/g3doc/apis.md | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/tensorflow/lite/g3doc/apis.md b/tensorflow/lite/g3doc/apis.md
index b15159ce41..1a05142bc4 100644
--- a/tensorflow/lite/g3doc/apis.md
+++ b/tensorflow/lite/g3doc/apis.md
@@ -1,4 +1,3 @@
-
 # TensorFlow Lite APIs
 
 TensorFlow Lite provides programming APIs in C++ and Java, and in both cases
@@ -8,8 +7,7 @@ no surprise that the APIs try to avoid unnecessary copies at the expense of
 convenience.  Similarly, consistency with TensorFlow APIs was not an explicit
 goal and some variance is to be expected.
 
-There is also a Python API for TensorFlow Lite described
-[here](../toco/g3doc/python_api.md#interpreter).
+There is also a [Python API for TensorFlow Lite](g3doc/convert/python_api.md).
 
 ## C++
 
-- 
GitLab


From 3c4b35297e4b44baaf4cc7ed5b099c586ffa049b Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Thu, 10 Jan 2019 10:44:31 -0800
Subject: [PATCH 0526/1095] Re-apply "Don't check for alignment before calling
 into a tiled dot (Eigen or LLVM)" with fix

PiperOrigin-RevId: 228736280
---
 tensorflow/compiler/xla/service/cpu/BUILD     |   2 -
 .../cpu/cpu_eigen_tensor_alignment_test.cc    |  38 ------
 .../xla/service/cpu/dot_op_emitter.cc         | 117 ++++++++++++------
 3 files changed, 77 insertions(+), 80 deletions(-)

diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD
index de62aa60aa..a197bdddc8 100644
--- a/tensorflow/compiler/xla/service/cpu/BUILD
+++ b/tensorflow/compiler/xla/service/cpu/BUILD
@@ -385,7 +385,6 @@ cc_library(
     srcs = ["dot_op_emitter.cc"],
     hdrs = [
         "dot_op_emitter.h",
-        "dot_op_emitter_internal.h",
     ],
     deps = [
         ":cpu_options",
@@ -1029,7 +1028,6 @@ tf_cc_test(
     size = "small",
     srcs = ["cpu_eigen_tensor_alignment_test.cc"],
     deps = [
-        ":dot_op_emitter",
         ":ir_emission_utils",
         ":target_machine_features_fake",
         "//tensorflow/compiler/xla:test",
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_eigen_tensor_alignment_test.cc b/tensorflow/compiler/xla/service/cpu/cpu_eigen_tensor_alignment_test.cc
index 823bdf259c..485769a373 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_eigen_tensor_alignment_test.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_eigen_tensor_alignment_test.cc
@@ -13,7 +13,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/compiler/xla/service/cpu/dot_op_emitter_internal.h"
 #include "tensorflow/compiler/xla/service/cpu/ir_emission_utils.h"
 #include "tensorflow/compiler/xla/service/cpu/target_machine_features_fake.h"
 #include "tensorflow/compiler/xla/service/hlo_parser.h"
@@ -23,48 +22,11 @@ namespace xla {
 namespace cpu {
 namespace {
 
-using internal::DotImplementationStrategy;
-using internal::DotInfo;
-using internal::GetDotImplementationStrategy;
-
 // Test that we don't call into Eigen with tensors too small to be aligned
 // reliably.
 
 class CpuEigenTensorAlignmentTest : public ::testing::Test {};
 
-TEST_F(CpuEigenTensorAlignmentTest, EigenDotAlignment) {
-  string hlo_string = R"(
-HloModule DotOperation
-
-ENTRY DotOperation {
-  arg0 = f32[5,256] parameter(0)
-  arg1 = f32[256,1024] parameter(1)
-  ROOT dot = f32[5,1024] dot(arg0, arg1), lhs_contracting_dims={1}, rhs_contracting_dims={0}
-}
-)";
-
-  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
-                          ParseHloString(hlo_string));
-
-  HloInstruction* dot = module->entry_computation()->root_instruction();
-
-  TargetMachineFeaturesWithFakeAlignmentLogic target_machine_with_no_alignment(
-      [](int64 size) { return 1; });
-
-  EXPECT_EQ(GetDotImplementationStrategy(HloModuleConfig{}, DotInfo(*dot),
-                                         target_machine_with_no_alignment),
-            DotImplementationStrategy::kNaiveLlvmIr);
-
-  TargetMachineFeaturesWithFakeAlignmentLogic
-      target_machine_with_full_alignment([](int64 size) {
-        return TargetMachineFeatures::kEigenExpectedTensorAlignment;
-      });
-
-  EXPECT_NE(GetDotImplementationStrategy(HloModuleConfig{}, DotInfo(*dot),
-                                         target_machine_with_full_alignment),
-            DotImplementationStrategy::kNaiveLlvmIr);
-}
-
 TEST_F(CpuEigenTensorAlignmentTest, EigenConvAlignment) {
   string hlo_string = R"(
 HloModule ConvOperation
diff --git a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
index e8a84ebe6b..b018e0cd46 100644
--- a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
@@ -14,7 +14,6 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/compiler/xla/service/cpu/dot_op_emitter.h"
-#include "tensorflow/compiler/xla/service/cpu/dot_op_emitter_internal.h"
 
 #include <memory>
 #include <vector>
@@ -43,17 +42,63 @@ namespace xla {
 using llvm_ir::SetToFirstInsertPoint;
 
 namespace cpu {
-
-using internal::DotImplementationStrategy;
-using internal::DotInfo;
-using internal::GetDotImplementationStrategy;
-
 namespace {
 // Returns true if we should call into multi-threaded Eigen routines.
 bool ShouldUseMultiThreadedEigen(const HloModuleConfig& config) {
   return config.debug_options().xla_cpu_multi_thread_eigen();
 }
 
+// Represents a dot operation.  We use this in lieu of an `HloInstruction`
+// because we want to be able to create this for the "inner" dot operation in a
+// batch dot, for which there is no separate HLO instruction.
+struct DotInfo {
+  Shape lhs_shape;
+  Shape rhs_shape;
+  Shape result_shape;
+  DotDimensionNumbers dim_nums;
+
+  explicit DotInfo(const HloInstruction& instr) {
+    CHECK_EQ(instr.opcode(), HloOpcode::kDot);
+    lhs_shape = instr.operand(0)->shape();
+    rhs_shape = instr.operand(1)->shape();
+    result_shape = instr.shape();
+    dim_nums = instr.dot_dimension_numbers();
+  }
+};
+
+// Dictates how a dot operation is implemented.
+enum class DotImplementationStrategy {
+  // The dot operation is lowered into LLVM IR that implements a naive nested
+  // loop that computes the result one element at a time.  This is our
+  // "fallback"; we don't really want this to kick in for any non-trival dot
+  // operation.
+  kNaiveLlvmIr,
+
+  // The dot operation is lowered into LLVM IR that implements a tiled
+  // Matrix*Vector operation.  This strategy also allows fusing in a bias add
+  // into the dot.  The matrix can be row major or column major, both are
+  // supported.
+  kTiledLlvmIrGemv,
+
+  // The dot operation is lowered into LLVM IR that implemetns a tiled
+  // Matrix*Matrix operation.  No fusions are supported.  The two inputs
+  // and the output have to be row major.
+  kTiledLlvmIrGemm,
+
+  // The dot operation is lowered into a call into an Eigen routine.  No fusions
+  // are supported today.  The two inputs and the output have to be row major.
+  // However, we do allow transposing either the LHS or the RHS as part of the
+  // GEMM -- we expose this flexibility as flexibility in the contraction
+  // dimensions, but we can also see this as flexibility in the input layouts.
+  kEigen,
+};
+
+// Returns the implementation strategy for a dot with the configuration
+// `dot_info`.
+DotImplementationStrategy GetDotImplementationStrategy(
+    const HloModuleConfig& config, const DotInfo& dot_info,
+    const TargetMachineFeatures& target_machine_features);
+
 // Helper class for emitting LLVM IR to perform the dot operation.
 class DotOpEmitter {
  public:
@@ -190,9 +235,8 @@ void DotOpEmitter::EmitTiledLlvmIrGemm() {
   }
 
   int64 size_bytes = m * n * ShapeUtil::ByteSizeOfPrimitiveType(primitive_type);
-  b_->CreateMemSet(
-      target, b_->getInt8(0), size_bytes,
-      target_machine_features_.minimum_alignment_for_allocation(size_bytes));
+  b_->CreateMemSet(target, b_->getInt8(0), /*Size=*/size_bytes,
+                   /*Align=*/1);
 
   int64 max_target_vector_width =
       target_machine_features_.vector_register_num_elements(
@@ -696,40 +740,34 @@ absl::optional<int64> ProfitableToMakeDotOperandColumnMajor(
   return {};
 }
 
-namespace internal {
 namespace {
 // Return whether the given shape is rank 2.
 bool IsRank2(const Shape& shape) { return shape.rank() == 2; }
 
+bool IsSimpleLayout(const Layout& layout) {
+  return layout.tiles().empty() && layout.format() == DENSE;
+}
+
 // In a gemm operation where output = lhs * rhs, check whether the given shapes
 // are valid for the operation.
-bool AreAlignedGemmShapes(
-    const Shape& lhs_shape, const Shape& rhs_shape, const Shape& output_shape,
-    const TargetMachineFeatures& target_machine_features) {
-  // The inputs and the output must
-  // 1) be matrices with no padding, and
-  // 2) have an allowed element type.
-  PrimitiveType output_primitive_type = output_shape.element_type();
-  if (!(output_primitive_type == F64 || output_primitive_type == F32 ||
-        output_primitive_type == F16)) {
-    return false;
-  }
-
-  if (!(IsRank2(lhs_shape) && IsRank2(rhs_shape) && IsRank2(output_shape))) {
-    return false;
-  }
-
-  auto is_aligned = [&](const Shape& shape) {
-    return GetMinimumAlignmentForArray(shape, target_machine_features) >=
-           TargetMachineFeatures::kEigenExpectedTensorAlignment;
-  };
-
-  if (!is_aligned(lhs_shape) || !is_aligned(rhs_shape) ||
-      !is_aligned(output_shape)) {
-    return false;
+bool AreGemmShapes(const Shape& lhs_shape, const Shape& rhs_shape,
+                   const Shape& output_shape,
+                   const TargetMachineFeatures& target_machine_features) {
+  CHECK(!lhs_shape.has_layout() || IsSimpleLayout(lhs_shape.layout()))
+      << lhs_shape.DebugString();
+  CHECK(!rhs_shape.has_layout() || IsSimpleLayout(rhs_shape.layout()))
+      << rhs_shape.DebugString();
+  CHECK(!output_shape.has_layout() || IsSimpleLayout(output_shape.layout()))
+      << output_shape.DebugString();
+
+  switch (output_shape.element_type()) {
+    case F64:
+    case F32:
+    case F16:
+      return IsRank2(lhs_shape) && IsRank2(rhs_shape) && IsRank2(output_shape);
+    default:
+      return false;
   }
-
-  return true;
 }
 
 bool IsAlignedGemm(const DotInfo& dot_info,
@@ -739,8 +777,8 @@ bool IsAlignedGemm(const DotInfo& dot_info,
     return false;
   }
 
-  return AreAlignedGemmShapes(dot_info.lhs_shape, dot_info.rhs_shape,
-                              dot_info.result_shape, target_machine_features);
+  return AreGemmShapes(dot_info.lhs_shape, dot_info.rhs_shape,
+                       dot_info.result_shape, target_machine_features);
 }
 
 bool CanEmitTiledLlvmIrGemm(
@@ -781,7 +819,6 @@ bool CanEmitTiledLlvmIrGemm(
 
   return true;
 }
-}  // namespace
 
 DotImplementationStrategy GetDotImplementationStrategy(
     const HloModuleConfig& config, const DotInfo& dot_info,
@@ -806,7 +843,7 @@ DotImplementationStrategy GetDotImplementationStrategy(
 
   return DotImplementationStrategy::kNaiveLlvmIr;
 }
-}  // namespace internal
+}  // namespace
 
 bool DotImplementationCanHandleTranspose(
     const HloInstruction& dot_instr,
-- 
GitLab


From eae721281d6c5bdd1e84a17de56c1f2cb911fd76 Mon Sep 17 00:00:00 2001
From: Michael Kuperstein <mkuper@google.com>
Date: Thu, 10 Jan 2019 11:07:54 -0800
Subject: [PATCH 0527/1095] [XLA] Move over some uses of CreateDS / CreateDUS
 to new API

Move over some uses of the R1-index API to use the span-of-scalars one.

PiperOrigin-RevId: 228740894
---
 .../xla/service/algebraic_simplifier_test.cc  | 34 ++++++++++-----
 .../cpu/cpu_instruction_fusion_test.cc        | 43 ++++++++++++-------
 2 files changed, 51 insertions(+), 26 deletions(-)

diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
index e39321f1f3..916b953cfc 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
@@ -3774,12 +3774,16 @@ TEST_F(AlgebraicSimplifierTest, TrivialDynamicSlice) {
   HloComputation::Builder builder(TestName());
 
   Shape shape = ShapeUtil::MakeShape(F32, {10, 100, 1000});
+  std::vector<HloInstruction*> params;
+  for (int i = 0; i < 3; ++i) {
+    params.push_back(builder.AddInstruction(HloInstruction::CreateParameter(
+        i + 1, ShapeUtil::MakeShape(U32, {}), "slice_indices")));
+  }
   builder.AddInstruction(HloInstruction::CreateDynamicSlice(
       shape,
       builder.AddInstruction(
           HloInstruction::CreateParameter(0, shape, "slice_from")),
-      builder.AddInstruction(HloInstruction::CreateParameter(
-          1, ShapeUtil::MakeShape(U32, {3}), "slice_indices")),
+      params,
       /*slice_sizes=*/{10, 100, 1000}));
 
   auto computation = m->AddEntryComputation(builder.Build());
@@ -3798,28 +3802,35 @@ TEST_F(AlgebraicSimplifierTest, TrivialDynamicUpdateSlice) {
   Shape full_shape = ShapeUtil::MakeShape(F32, {10, 100, 1000});
   Shape slice_shape = ShapeUtil::MakeShape(F32, {10, 1, 1000});
 
+  std::vector<HloInstruction*> slice_indices, update_indices;
+  for (int i = 0; i < 3; ++i) {
+    slice_indices.push_back(
+        builder.AddInstruction(HloInstruction::CreateParameter(
+            i + 1, ShapeUtil::MakeShape(U32, {}), "slice_indices")));
+    update_indices.push_back(
+        builder.AddInstruction(HloInstruction::CreateParameter(
+            i + 5, ShapeUtil::MakeShape(U32, {}), "update_indices")));
+  }
   HloInstruction* slice =
       builder.AddInstruction(HloInstruction::CreateDynamicSlice(
           slice_shape,
           builder.AddInstruction(
               HloInstruction::CreateParameter(0, full_shape, "slice_from")),
-          builder.AddInstruction(HloInstruction::CreateParameter(
-              1, ShapeUtil::MakeShape(U32, {3}), "slice_indices")),
+          slice_indices,
           /*slice_sizes=*/{10, 1, 1000}));
 
   builder.AddInstruction(HloInstruction::CreateDynamicUpdateSlice(
       slice_shape,
       builder.AddInstruction(
-          HloInstruction::CreateParameter(2, slice_shape, "to_update")),
-      slice,
-      builder.AddInstruction(HloInstruction::CreateParameter(
-          3, ShapeUtil::MakeShape(U32, {3}), "update_indices"))));
+          HloInstruction::CreateParameter(4, slice_shape, "to_update")),
+      slice, update_indices));
 
   auto computation = m->AddEntryComputation(builder.Build());
   AlgebraicSimplifier simplifier(default_options_);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
   EXPECT_THAT(computation->root_instruction(),
-              GmockMatch(m::DynamicSlice(m::Parameter(), m::Parameter())));
+              GmockMatch(m::DynamicSlice(m::Parameter(), m::Parameter(),
+                                         m::Parameter(), m::Parameter())));
 }
 
 // Test that two consecutive broadcasts can be merged to one.
@@ -4480,9 +4491,10 @@ TEST_F(AlgebraicSimplifierTest, DynamicUpdateSliceZeroUpdate) {
   HloInstruction* const update = builder.AddInstruction(
       HloInstruction::CreateParameter(1, update_shape, "update"));
   HloInstruction* const start_indices = builder.AddInstruction(
-      HloInstruction::CreateConstant(LiteralUtil::CreateR1<int>({0})));
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0<int>({})));
   builder.AddInstruction(HloInstruction::CreateDynamicUpdateSlice(
-      dslice_shape, operand, update, start_indices));
+      dslice_shape, operand, update,
+      std::initializer_list<HloInstruction*>({start_indices})));
   const HloComputation* const computation =
       m->AddEntryComputation(builder.Build());
 
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion_test.cc b/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion_test.cc
index 527df0bd1c..46c1d4c38e 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion_test.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion_test.cc
@@ -332,7 +332,7 @@ TEST_F(OpcodeFusionTest, Exponential_Reshape_Negate) {
 TEST_F(OpcodeFusionTest, Broadcast_Reshape_DynamicSlice_Tanh) {
   HloComputation::Builder builder(TestName());
   Shape param_shape = ShapeUtil::MakeShape(F32, {8});
-  Shape starts_shape = ShapeUtil::MakeShape(F32, {2});
+  Shape starts_shape = ShapeUtil::MakeShape(F32, {});
   Shape broadcast_shape = ShapeUtil::MakeShape(F32, {1, 8, 8});
   Shape reshape_shape = ShapeUtil::MakeShape(F32, {8, 8});
   Shape dynamic_slice_shape = ShapeUtil::MakeShape(F32, {4, 4});
@@ -340,13 +340,15 @@ TEST_F(OpcodeFusionTest, Broadcast_Reshape_DynamicSlice_Tanh) {
       HloInstruction::CreateParameter(0, param_shape, "param"));
   HloInstruction* param1 = builder.AddInstruction(
       HloInstruction::CreateParameter(1, starts_shape, "starts"));
+  HloInstruction* param2 = builder.AddInstruction(
+      HloInstruction::CreateParameter(2, starts_shape, "starts"));
   HloInstruction* broadcast2 = builder.AddInstruction(
       HloInstruction::CreateBroadcast(broadcast_shape, param0, {1}));
   HloInstruction* reshape3 = builder.AddInstruction(
       HloInstruction::CreateReshape(reshape_shape, broadcast2));
   HloInstruction* dynamic_slice4 =
       builder.AddInstruction(HloInstruction::CreateDynamicSlice(
-          dynamic_slice_shape, reshape3, param1, {4, 4}));
+          dynamic_slice_shape, reshape3, {param1, param2}, {4, 4}));
   builder.AddInstruction(HloInstruction::CreateUnary(
       dynamic_slice_shape, HloOpcode::kTanh, dynamic_slice4));
 
@@ -356,7 +358,8 @@ TEST_F(OpcodeFusionTest, Broadcast_Reshape_DynamicSlice_Tanh) {
   RunFusionAndCheckOpcodesWereFused(
       module.get(),
       {HloOpcode::kTanh, HloOpcode::kDynamicSlice, HloOpcode::kReshape,
-       HloOpcode::kBroadcast, HloOpcode::kParameter, HloOpcode::kParameter});
+       HloOpcode::kBroadcast, HloOpcode::kParameter, HloOpcode::kParameter,
+       HloOpcode::kParameter});
 }
 
 TEST_F(OpcodeFusionTest, Broadcast_Negate) {
@@ -381,14 +384,16 @@ TEST_F(OpcodeFusionTest, Broadcast_Negate) {
 TEST_F(OpcodeFusionTest, DynamicSlice_Negate) {
   HloComputation::Builder builder(TestName());
   Shape param_shape = ShapeUtil::MakeShape(F32, {4});
-  Shape slice_shape = ShapeUtil::MakeShape(F32, {1});
+  Shape slice_shape = ShapeUtil::MakeShape(F32, {});
   Shape result_shape = ShapeUtil::MakeShape(F32, {2});
   HloInstruction* param0 = builder.AddInstruction(
       HloInstruction::CreateParameter(0, param_shape, "param"));
   HloInstruction* param1 = builder.AddInstruction(
       HloInstruction::CreateParameter(1, slice_shape, "starts"));
-  HloInstruction* dynamic_slice2 = builder.AddInstruction(
-      HloInstruction::CreateDynamicSlice(result_shape, param0, param1, {2}));
+  HloInstruction* dynamic_slice2 =
+      builder.AddInstruction(HloInstruction::CreateDynamicSlice(
+          result_shape, param0,
+          std::initializer_list<HloInstruction*>({param1}), {2}));
   builder.AddInstruction(HloInstruction::CreateUnary(
       result_shape, HloOpcode::kNegate, dynamic_slice2));
 
@@ -548,28 +553,36 @@ TEST_F(OpcodeFusionTest, DynamicSliceWithDynamicUpdateSlice) {
   Shape full_shape = ShapeUtil::MakeShape(F32, {10, 100, 1000});
   Shape slice_shape = ShapeUtil::MakeShape(F32, {10, 1, 1000});
 
+  std::vector<HloInstruction*> slice_indices, update_indices;
+  for (int i = 0; i < 3; ++i) {
+    slice_indices.push_back(
+        builder.AddInstruction(HloInstruction::CreateParameter(
+            1 + i, ShapeUtil::MakeShape(U32, {}), "slice_indices")));
+    update_indices.push_back(
+        builder.AddInstruction(HloInstruction::CreateParameter(
+            5 + i, ShapeUtil::MakeShape(U32, {}), "update_indices")));
+  }
   HloInstruction* slice =
       builder.AddInstruction(HloInstruction::CreateDynamicSlice(
           slice_shape,
           builder.AddInstruction(
               HloInstruction::CreateParameter(0, full_shape, "slice_from")),
-          builder.AddInstruction(HloInstruction::CreateParameter(
-              1, ShapeUtil::MakeShape(U32, {3}), "slice_indices")),
+          slice_indices,
           /*slice_sizes=*/{10, 1, 1000}));
 
   builder.AddInstruction(HloInstruction::CreateDynamicUpdateSlice(
       full_shape,
       builder.AddInstruction(
-          HloInstruction::CreateParameter(2, full_shape, "to_update")),
-      slice,
-      builder.AddInstruction(HloInstruction::CreateParameter(
-          3, ShapeUtil::MakeShape(U32, {3}), "update_indices"))));
+          HloInstruction::CreateParameter(4, full_shape, "to_update")),
+      slice, update_indices));
 
   module->AddEntryComputation(builder.Build());
   RunFusionAndCheckOpcodesWereFused(
-      module.get(), {HloOpcode::kDynamicSlice, HloOpcode::kDynamicUpdateSlice,
-                     HloOpcode::kParameter, HloOpcode::kParameter,
-                     HloOpcode::kParameter, HloOpcode::kParameter});
+      module.get(),
+      {HloOpcode::kDynamicSlice, HloOpcode::kDynamicUpdateSlice,
+       HloOpcode::kParameter, HloOpcode::kParameter, HloOpcode::kParameter,
+       HloOpcode::kParameter, HloOpcode::kParameter, HloOpcode::kParameter,
+       HloOpcode::kParameter, HloOpcode::kParameter});
 }
 
 TEST_F(OpcodeFusionTest, MessOfFusibleNodes) {
-- 
GitLab


From 8ccb11412614e7d9b702d9a56b6f876af8779f72 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 10 Jan 2019 11:24:15 -0800
Subject: [PATCH 0528/1095] Allows variable targets in gradient tapes. This
 allows using the native python `max` method on lists of variables inside of
 gradient tapes.

PiperOrigin-RevId: 228743966
---
 tensorflow/python/eager/backprop.py      |  9 +++++----
 tensorflow/python/eager/backprop_test.py | 16 ++++++++++++----
 2 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py
index 7d99f6e95a..42db726a7b 100644
--- a/tensorflow/python/eager/backprop.py
+++ b/tensorflow/python/eager/backprop.py
@@ -930,11 +930,12 @@ class GradientTape(object):
                             "gradient in order to compute higher order "
                             "derrivatives.", 1)
 
-    flat_targets = nest.flatten(target)
-    for t in flat_targets:
+    flat_targets = []
+    for t in nest.flatten(target):
       if resource_variable_ops.is_resource_variable(t):
-        raise ValueError("GradientTape.gradient is not supported for variable "
-                         "targets.")
+        with self:
+          t = ops.convert_to_tensor(t)
+      flat_targets.append(t)
 
     flat_sources = nest.flatten(sources)
     flat_sources = [_handle_or_self(x) for x in flat_sources]
diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py
index 22ae6f74cb..5f4fda8897 100644
--- a/tensorflow/python/eager/backprop_test.py
+++ b/tensorflow/python/eager/backprop_test.py
@@ -354,6 +354,16 @@ class BackpropTest(test.TestCase):
       loss += v * v
     self.assertAllEqual(t.gradient(loss, v), 2.0)
 
+  def testPythonMax(self):
+    x = [resource_variable_ops.ResourceVariable(2.),
+         resource_variable_ops.ResourceVariable(3.),
+         resource_variable_ops.ResourceVariable(5.)]
+    with backprop.GradientTape() as t:
+      f = max(x)
+    grad = t.gradient(f, x)
+    self.assertAllEqual(self.evaluate(f), 5.)
+    self.assertAllEqual(self.evaluate(grad), [None, None, 1.0])
+
   def testAutomaticWatchedVariables(self):
     with backprop.GradientTape() as t:
       self.assertEqual(0, len(t.watched_variables()))
@@ -674,10 +684,8 @@ class BackpropTest(test.TestCase):
     with backprop.GradientTape() as g:
       x = variables.Variable([3.0])
       y = variables.Variable([2.0])
-    with self.assertRaisesRegexp(
-        ValueError,
-        'GradientTape.gradient is not supported for variable targets.'):
-      g.gradient(x, y)
+    grad = g.gradient(x, y)
+    self.assertAllEqual(grad, None)
 
   @test_util.run_in_graph_and_eager_modes
   @test_util.run_v1_only('b/120545219')
-- 
GitLab


From 32a8626373897a05195dfb3d895385a7584037cc Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Thu, 10 Jan 2019 11:26:10 -0800
Subject: [PATCH 0529/1095] [Java]: Correct some comments.

PiperOrigin-RevId: 228744342
---
 .../org/tensorflow/processor/OperatorProcessor.java  |  2 +-
 .../java/org/tensorflow/op/annotation/Operator.java  | 12 ++++++------
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/tensorflow/java/src/gen/java/org/tensorflow/processor/OperatorProcessor.java b/tensorflow/java/src/gen/java/org/tensorflow/processor/OperatorProcessor.java
index 1b7bcdab35..df1426ad75 100644
--- a/tensorflow/java/src/gen/java/org/tensorflow/processor/OperatorProcessor.java
+++ b/tensorflow/java/src/gen/java/org/tensorflow/processor/OperatorProcessor.java
@@ -340,7 +340,7 @@ public final class OperatorProcessor extends AbstractProcessor {
                     + "{@link $T @Operator} is exposed\n"
                     + "by this API or one of its subgroup.\n<p>Example usage:\n<pre>{@code\n"
                     + "try (Graph g = new Graph()) {\n"
-                    + "  Ops ops = new Ops(g);\n"
+                    + "  Ops ops = Ops.create(g);\n"
                     + "  // Operations are typed classes with convenience\n"
                     + "  // builders in Ops.\n"
                     + "  Constant three = ops.constant(3);\n"
diff --git a/tensorflow/java/src/main/java/org/tensorflow/op/annotation/Operator.java b/tensorflow/java/src/main/java/org/tensorflow/op/annotation/Operator.java
index 3782240edb..38f466c574 100644
--- a/tensorflow/java/src/main/java/org/tensorflow/op/annotation/Operator.java
+++ b/tensorflow/java/src/main/java/org/tensorflow/op/annotation/Operator.java
@@ -25,11 +25,11 @@ import java.lang.annotation.Target;
  * Annotation used by classes to make TensorFlow operations conveniently accessible via {@code
  * org.tensorflow.op.Ops}.
  *
- * <p>An annotation processor (TODO: not yet implemented) builds the {@code Ops} class by
- * aggregating all classes annotated as {@code @Operator}s. Each annotated class <b>must</b> have at
- * least one public static factory method named {@code create} that accepts a {@link
- * org.tensorflow.op.Scope} as its first argument. The processor then adds a convenience method in
- * the {@code Ops} class. For example:
+ * <p>An annotation processor ({@code org.tensorflow.processor.OperatorProcessor}) builds the
+ * {@code Ops} class by aggregating all classes annotated as {@code @Operator}s. Each annotated
+ * class <b>must</b> have at least one public static factory method named {@code create} that
+ * accepts a {@link org.tensorflow.op.Scope} as its first argument. The processor then adds a
+ * convenience method in the {@code Ops} class. For example:
  *
  * <pre>{@code
  * @Operator
@@ -45,7 +45,7 @@ import java.lang.annotation.Target;
  * <pre>{@code
  * import org.tensorflow.op.Ops;
  * ...
- * Ops ops = new Ops(graph);
+ * Ops ops = Ops.create(graph);
  * ...
  * ops.myOp(operand);
  * // and has exactly the same effect as calling
-- 
GitLab


From 519efc7913ef2c6e5271c90727c6a1dc43817efb Mon Sep 17 00:00:00 2001
From: Michael Kuperstein <mkuper@google.com>
Date: Thu, 10 Jan 2019 11:27:18 -0800
Subject: [PATCH 0530/1095] [XLA] Move over more uses of CreateDS / CreateDUS
 to new API

Move over more uses of the R1-index API to use the span-of-scalars one.

PiperOrigin-RevId: 228744562
---
 .../xla/service/hlo_dataflow_analysis_test.cc | 47 +++++++++++--------
 .../xla/service/hlo_evaluator_test.cc         | 28 ++++++-----
 2 files changed, 45 insertions(+), 30 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_dataflow_analysis_test.cc b/tensorflow/compiler/xla/service/hlo_dataflow_analysis_test.cc
index 1d165ac35f..888886865b 100644
--- a/tensorflow/compiler/xla/service/hlo_dataflow_analysis_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_dataflow_analysis_test.cc
@@ -1970,12 +1970,13 @@ TEST_F(DoesNotUseOperandBufferTest, FusedDynamicUpdateSlice) {
 
   // Create a DynamicUpdateSlice instruction of tuple element 1.
   auto starts = builder.AddInstruction(
-      HloInstruction::CreateConstant(LiteralUtil::CreateR1<int32>({2})));
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0<int32>(2)));
   auto update = builder.AddInstruction(HloInstruction::CreateConstant(
       LiteralUtil::CreateR1<float>({2.f, 2.f, 2.f})));
   auto dynamic_update_slice =
       builder.AddInstruction(HloInstruction::CreateDynamicUpdateSlice(
-          data_shape, gte1, update, starts));
+          data_shape, gte1, update,
+          std::initializer_list<HloInstruction*>({starts})));
   builder.AddInstruction(
       HloInstruction::CreateTuple({gte0, dynamic_update_slice}));
 
@@ -2012,12 +2013,13 @@ TEST_F(DoesNotUseOperandBufferTest, IndirectUses) {
 
   // Create a DynamicUpdateSlice instruction of tuple element 1.
   auto starts = builder.AddInstruction(
-      HloInstruction::CreateConstant(LiteralUtil::CreateR1<int32>({2})));
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0<int32>(2)));
   auto update = builder.AddInstruction(HloInstruction::CreateConstant(
       LiteralUtil::CreateR1<float>({2.f, 2.f, 2.f})));
   auto dynamic_update_slice =
       builder.AddInstruction(HloInstruction::CreateDynamicUpdateSlice(
-          data_shape, gte1, update, starts));
+          data_shape, gte1, update,
+          std::initializer_list<HloInstruction*>({starts})));
   builder.AddInstruction(
       HloInstruction::CreateTuple({gte0, dynamic_update_slice}));
 
@@ -2150,17 +2152,17 @@ TEST_F(CanShareOperandBufferWithUserTest,
 
   auto param = builder.AddInstruction(
       HloInstruction::CreateParameter(0, data_shape, "param0"));
-  auto index = builder.AddInstruction(
-      HloInstruction::CreateConstant(LiteralUtil::CreateR1<int64>({0, 0})));
-  auto ds = builder.AddInstruction(
-      HloInstruction::CreateDynamicSlice(slice_shape, param, index, {1, 2, 2}));
+  auto zero = builder.AddInstruction(
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0<int64>(0)));
+  auto ds = builder.AddInstruction(HloInstruction::CreateDynamicSlice(
+      slice_shape, param, {zero, zero}, {1, 2, 2}));
 
-  auto dus = builder.AddInstruction(
-      HloInstruction::CreateDynamicUpdateSlice(data_shape, param, ds, index));
+  auto dus = builder.AddInstruction(HloInstruction::CreateDynamicUpdateSlice(
+      data_shape, param, ds, {zero, zero}));
 
   BuildModule(builder.Build());
   auto fusion = computation_->CreateFusionInstruction(
-      {dus, ds, index}, HloInstruction::FusionKind::kLoop);
+      {dus, ds, zero}, HloInstruction::FusionKind::kLoop);
   RunAnalysis();
 
   EXPECT_TRUE(
@@ -2219,12 +2221,13 @@ TEST_F(CanShareOperandBufferWithUserTest, FusedDynamicUpdateSlice) {
 
   // Create a DynamicUpdateSlice instruction of tuple element 1.
   auto starts = builder.AddInstruction(
-      HloInstruction::CreateConstant(LiteralUtil::CreateR1<int32>({2})));
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0<int32>(2)));
   auto update = builder.AddInstruction(HloInstruction::CreateConstant(
       LiteralUtil::CreateR1<float>({2.f, 2.f, 2.f})));
   auto dynamic_update_slice =
       builder.AddInstruction(HloInstruction::CreateDynamicUpdateSlice(
-          data_shape, gte1, update, starts));
+          data_shape, gte1, update,
+          std::initializer_list<HloInstruction*>({starts})));
   builder.AddInstruction(
       HloInstruction::CreateTuple({gte0, dynamic_update_slice}));
 
@@ -2259,12 +2262,13 @@ TEST_F(CanShareOperandBufferWithUserTest,
 
   // Create a DynamicUpdateSlice instruction of tuple element 1.
   auto starts = builder.AddInstruction(
-      HloInstruction::CreateConstant(LiteralUtil::CreateR1<int32>({2})));
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0<int32>(2)));
   auto update = builder.AddInstruction(HloInstruction::CreateConstant(
       LiteralUtil::CreateR1<float>({2.f, 2.f, 2.f})));
   auto dynamic_update_slice =
       builder.AddInstruction(HloInstruction::CreateDynamicUpdateSlice(
-          data_shape_bf16, convert1, update, starts));
+          data_shape_bf16, convert1, update,
+          std::initializer_list<HloInstruction*>({starts})));
 
   auto convert2 = builder.AddInstruction(
       HloInstruction::CreateConvert(data_shape, dynamic_update_slice));
@@ -2290,10 +2294,13 @@ TEST_F(CanShareOperandBufferWithUserTest, DynamicUpdateSliceCanShare) {
       HloInstruction::CreateParameter(0, data_shape, "data"));
   auto update = builder.AddInstruction(
       HloInstruction::CreateParameter(1, update_shape, "update"));
-  auto starts = builder.AddInstruction(
-      HloInstruction::CreateParameter(2, starts_shape, "starts"));
+  auto start0 = builder.AddInstruction(
+      HloInstruction::CreateParameter(2, starts_shape, "start0"));
+  auto start1 = builder.AddInstruction(
+      HloInstruction::CreateParameter(3, starts_shape, "start1"));
+
   auto dus = builder.AddInstruction(HloInstruction::CreateDynamicUpdateSlice(
-      data_shape, data, update, starts));
+      data_shape, data, update, {start0, start1}));
 
   BuildModuleAndRunAnalysis(builder.Build());
 
@@ -2304,7 +2311,9 @@ TEST_F(CanShareOperandBufferWithUserTest, DynamicUpdateSliceCanShare) {
   EXPECT_FALSE(
       dataflow_analysis_->CanShareOperandBufferWithUser(update, {}, dus, {}));
   EXPECT_FALSE(
-      dataflow_analysis_->CanShareOperandBufferWithUser(starts, {}, dus, {}));
+      dataflow_analysis_->CanShareOperandBufferWithUser(start0, {}, dus, {}));
+  EXPECT_FALSE(
+      dataflow_analysis_->CanShareOperandBufferWithUser(start1, {}, dus, {}));
 }
 
 TEST_F(CanShareOperandBufferWithUserTest, ScatterCanShare) {
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_test.cc b/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
index 319f0db97a..674df0016a 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
@@ -1739,12 +1739,14 @@ TEST_P(HloEvaluatorBf16Test, DynamicSlice) {
   HloInstruction* operand = b.AddInstruction(
       HloInstruction::CreateConstant(std::move(operand_literal)));
 
-  auto start_indices = b.AddInstruction(
-      HloInstruction::CreateConstant(LiteralUtil::CreateR1<int32>({0, 1})));
+  auto zero = b.AddInstruction(
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0<int32>(0)));
+  auto one = b.AddInstruction(
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0<int32>(1)));
 
   Shape shape = ShapeUtil::MakeShape(F32, {2, 3});
-  b.AddInstruction(HloInstruction::CreateDynamicSlice(shape, operand,
-                                                      start_indices, {2, 3}));
+  b.AddInstruction(
+      HloInstruction::CreateDynamicSlice(shape, operand, {zero, one}, {2, 3}));
   m_->AddEntryComputation(b.Build());
 
   Literal result = Evaluate();
@@ -1775,12 +1777,14 @@ TEST_P(HloEvaluatorBf16Test, DynamicSliceModSlice) {
   HloInstruction* operand = b.AddInstruction(
       HloInstruction::CreateConstant(std::move(operand_literal)));
 
-  auto start_indices = b.AddInstruction(
-      HloInstruction::CreateConstant(LiteralUtil::CreateR1<int32>({2, 1})));
+  auto two = b.AddInstruction(
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0<int32>(2)));
+  auto one = b.AddInstruction(
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0<int32>(1)));
 
   Shape shape = ShapeUtil::MakeShape(F32, {2, 3});
-  b.AddInstruction(HloInstruction::CreateDynamicSlice(shape, operand,
-                                                      start_indices, {2, 3}));
+  b.AddInstruction(
+      HloInstruction::CreateDynamicSlice(shape, operand, {two, one}, {2, 3}));
   m_->AddEntryComputation(b.Build());
 
   Literal result = Evaluate();
@@ -1809,15 +1813,17 @@ TEST_P(HloEvaluatorBf16Test, DynamicSliceUpdate) {
   HloInstruction* operand = b.AddInstruction(
       HloInstruction::CreateConstant(std::move(operand_literal)));
 
-  auto start_indices = b.AddInstruction(
-      HloInstruction::CreateConstant(LiteralUtil::CreateR1<int64>({0, 1})));
+  auto zero = b.AddInstruction(
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0<int32>(0)));
+  auto one = b.AddInstruction(
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0<int32>(1)));
 
   auto update = b.AddInstruction(HloInstruction::CreateConstant(
       LiteralUtil::CreateR2<double>({{-2.0, -3.0}, {-6.0, -7.0}})));
 
   Shape shape = ShapeUtil::MakeShape(F64, {2, 3});
   b.AddInstruction(HloInstruction::CreateDynamicUpdateSlice(
-      shape, operand, update, start_indices));
+      shape, operand, update, {zero, one}));
   m_->AddEntryComputation(b.Build());
 
   Literal result = Evaluate();
-- 
GitLab


From 4644d4e5f26cf72905d1ce40194a9733c4f8252f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 10 Jan 2019 11:29:29 -0800
Subject: [PATCH 0531/1095] [convergence_tools]: Tracing the subset of the ops
 that are in the execution path.

PiperOrigin-RevId: 228744967
---
 .../contrib/tpu/python/tpu/tensor_tracer.py   | 53 +++++++++++++++++--
 .../contrib/tpu/python/tpu/tpu_estimator.py   |  3 +-
 2 files changed, 52 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tensor_tracer.py b/tensorflow/contrib/tpu/python/tpu/tensor_tracer.py
index c7e29860e5..bf492e78a1 100644
--- a/tensorflow/contrib/tpu/python/tpu/tensor_tracer.py
+++ b/tensorflow/contrib/tpu/python/tpu/tensor_tracer.py
@@ -59,6 +59,7 @@ _REASON_SCALAR_GET_TRACED = 'traced-scalar'
 _REASON_TENSOR_GET_TRACED = 'traced-tensor'
 _REASON_USER_INCLUDED = 'traced-user-included'
 _REASON_USER_EXCLUDED = 'not-traced-user-excluded'
+_REASON_NOT_EXECUTED = 'not-traced-not-in-exec-path'
 _REASON_NON_NUMERIC_TENSOR = 'not-traced-non-numeric-tensor'
 _MARKER_SECTION_BEGIN = '!!!!!!! section-begin:'
 _MARKER_SECTION_END = '!!!!!!! section-end:'
@@ -864,7 +865,8 @@ class TensorTracer(object):
     raise RuntimeError('Tensor trace fun for %s is not yet implemented'
                        %self._trace_mode)
 
-  def _skip_op(self, op_id, op, user_included, user_excluded):
+  def _skip_op(self, op_id, op, user_included, user_excluded,
+               in_exec_path=True):
     """Returns True if we should not trace Op."""
 
     if user_included:
@@ -875,6 +877,10 @@ class TensorTracer(object):
       self._instrument_records[op.name] = TensorTracer.reason(
           op_id, _REASON_USER_EXCLUDED)
       return True
+    if not in_exec_path:
+      self._instrument_records[op.name] = TensorTracer.reason(
+          op_id, _REASON_NOT_EXECUTED)
+      return True
     if not self._inside_op_range(op_id):
       self._instrument_records[op.name] = TensorTracer.reason(
           op_id, _REASON_OUTSIDE_OP_RANGE)
@@ -946,6 +952,40 @@ class TensorTracer(object):
           op_id, _REASON_TENSOR_GET_TRACED)
       return False
 
+  def _filter_execution_path_operations(self, operations, fetches):
+    """Returns the set of ops in the execution path to compute given fetches."""
+    # If no fetch provided, then return all operations.
+    if fetches is None:
+      return set(operations)
+    # Convert to list, if a single element is provided.
+    if not isinstance(fetches, (list, tuple)):
+      fetches = [fetches]
+    # If a tensor is given as fetch, convert it to op.
+    op_fetches = []
+    for fetch in fetches:
+      if isinstance(fetch, ops.Operation):
+        op_fetches.append(fetch)
+      elif isinstance(fetch, ops.Tensor):
+        op_fetches.append(fetch.op)
+      else:
+        raise RuntimeError('Given fetch:%s is neither a tensor nor an op.'
+                           %fetch)
+
+    execution_path_operations = set(op_fetches)
+    traverse_stack = list(op_fetches)
+    while True:
+      if not traverse_stack:
+        break
+      head_op = traverse_stack.pop()
+      input_ops = [tensor_input.op for tensor_input in head_op.inputs]
+      input_ops.extend(head_op.control_inputs)
+
+      for input_op in input_ops:
+        if input_op not in execution_path_operations:
+          execution_path_operations.add(input_op)
+          traverse_stack.append(input_op)
+    return execution_path_operations
+
   def _pre_tracing(self, graph):
     """Work needs to be done prior to TPU or CPU tracing."""
 
@@ -987,13 +1027,15 @@ class TensorTracer(object):
                                   _TENSOR_TRACER_CHECKPOINT))
     return checkpoint_operations
 
-  def trace_tpu(self, graph, result_tensor, num_replicas=None):
+  def trace_tpu(self, graph, result_tensor, num_replicas=None, fetches=None):
     """Traces the tensors generated by TPU Ops in a TF graph.
 
     Args:
       graph: the graph of Ops executed on the TPU.
       result_tensor: a result tensor of evaluating the graph.
       num_replicas: number of replicas used on the TPU.
+      fetches: the list of fetches given to session.run, used to determine the
+      ops in execution path. If None, the whole graph will be traced.
 
     Returns:
       A tuple (result_tensor_copy, tracing_ops), where:
@@ -1022,6 +1064,10 @@ class TensorTracer(object):
     result_tensor_copy = self._add_replica_id_to_graph(num_replicas,
                                                        result_tensor)
     (operations, succeed, sorted_or_cycle) = self._pre_tracing(graph)
+    # Filter out the operations that won't be executed.
+    # if fetches=None, then ops_in_exec_path = set(operations)
+    ops_in_exec_path = self._filter_execution_path_operations(operations,
+                                                              fetches)
     tracing_ops = []
     checkpoint_operations = self._get_checkpoints(graph)
 
@@ -1030,7 +1076,8 @@ class TensorTracer(object):
         continue
       user_included = self._is_user_included_op(op)
       user_excluded = self._is_user_excluded_op(op)
-      if self._skip_op(op_id, op, user_included, user_excluded):
+      in_exec_path = op in ops_in_exec_path
+      if self._skip_op(op_id, op, user_included, user_excluded, in_exec_path):
         continue
       for i in range(len(op.outputs)):
         out_tensor = op.outputs[i]
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 2df5b3e4eb..075ecbc52a 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -1412,7 +1412,8 @@ class _ModelFnWrapper(object):
       if tensor_tracer.TensorTracer.is_enabled():
         tt = tensor_tracer.TensorTracer()
         loss, tracing_ops = tt.trace_tpu(ops.get_default_graph(), loss,
-                                         self._ctx.num_replicas)
+                                         self._ctx.num_replicas,
+                                         fetches=[loss, train_op])
 
       # We must run train_op to update the variables prior to running the
       # outfeed.
-- 
GitLab


From 53bd138262df6e141c0e196a262f4ae760c2cf5f Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Thu, 10 Jan 2019 11:34:00 -0800
Subject: [PATCH 0532/1095] [Java]: Add place to put unittests for generated Op
 wrappers.

PiperOrigin-RevId: 228745935
---
 tensorflow/java/BUILD                         | 13 ++++++
 .../op/core/GeneratedOperationsTest.java      | 43 +++++++++++++++++++
 2 files changed, 56 insertions(+)
 create mode 100644 tensorflow/java/src/test/java/org/tensorflow/op/core/GeneratedOperationsTest.java

diff --git a/tensorflow/java/BUILD b/tensorflow/java/BUILD
index 10808e162e..af5503f2ad 100644
--- a/tensorflow/java/BUILD
+++ b/tensorflow/java/BUILD
@@ -295,6 +295,19 @@ tf_java_test(
     ],
 )
 
+tf_java_test(
+    name = "GeneratedOperationsTest",
+    size = "small",
+    srcs = ["src/test/java/org/tensorflow/op/core/GeneratedOperationsTest.java"],
+    javacopts = JAVACOPTS,
+    test_class = "org.tensorflow.op.core.GeneratedOperationsTest",
+    deps = [
+        ":tensorflow",
+        ":testutil",
+        "@junit",
+    ],
+)
+
 tf_java_test(
     name = "GradientsTest",
     size = "small",
diff --git a/tensorflow/java/src/test/java/org/tensorflow/op/core/GeneratedOperationsTest.java b/tensorflow/java/src/test/java/org/tensorflow/op/core/GeneratedOperationsTest.java
new file mode 100644
index 0000000000..42d126c3c4
--- /dev/null
+++ b/tensorflow/java/src/test/java/org/tensorflow/op/core/GeneratedOperationsTest.java
@@ -0,0 +1,43 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+package org.tensorflow.op.core;
+
+import static org.junit.Assert.assertEquals;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.tensorflow.Graph;
+import org.tensorflow.Operand;
+import org.tensorflow.Session;
+import org.tensorflow.Tensor;
+import org.tensorflow.op.Ops;
+
+@RunWith(JUnit4.class)
+public final class GeneratedOperationsTest {
+
+  @Test
+  public void tensorInputTensorOutput() {
+    try (Graph g = new Graph();
+        Session sess = new Session(g)) {
+      Ops ops = Ops.create(g);
+      Operand<Integer> x = ops.math().add(ops.constant(1), ops.constant(2));
+      try (Tensor<Integer> result = sess.runner().fetch(x).run().get(0).expect(Integer.class)) {
+        assertEquals(3, result.intValue());
+      }
+    }
+  }
+}
-- 
GitLab


From 1095ed8fe6808a70edb66d7970e9462db791e690 Mon Sep 17 00:00:00 2001
From: James Ring <sjr@google.com>
Date: Thu, 10 Jan 2019 11:48:55 -0800
Subject: [PATCH 0533/1095] Add TF_StepId and TF_ExpectedOutputDataType

PiperOrigin-RevId: 228748943
---
 tensorflow/c/kernels.cc      | 9 +++++++++
 tensorflow/c/kernels.h       | 8 ++++++++
 tensorflow/c/kernels_test.cc | 6 ++++++
 3 files changed, 23 insertions(+)

diff --git a/tensorflow/c/kernels.cc b/tensorflow/c/kernels.cc
index 2a4eaecb6c..673aed558a 100644
--- a/tensorflow/c/kernels.cc
+++ b/tensorflow/c/kernels.cc
@@ -158,3 +158,12 @@ void TF_SetOutput(TF_OpKernelContext* ctx, int i, const TF_Tensor* tensor,
     cc_ctx->set_output(i, cc_tensor);
   }
 }
+
+TF_DataType TF_ExpectedOutputDataType(TF_OpKernelContext* ctx, int i) {
+  auto* cc_ctx = reinterpret_cast<::tensorflow::OpKernelContext*>(ctx);
+  return static_cast<TF_DataType>(cc_ctx->expected_output_dtype(i));
+}
+
+int64_t TF_StepId(TF_OpKernelContext* ctx) {
+  return reinterpret_cast<::tensorflow::OpKernelContext*>(ctx)->step_id();
+}
diff --git a/tensorflow/c/kernels.h b/tensorflow/c/kernels.h
index cefc30bcdf..721a4aca0b 100644
--- a/tensorflow/c/kernels.h
+++ b/tensorflow/c/kernels.h
@@ -111,6 +111,14 @@ TF_CAPI_EXPORT extern void TF_SetOutput(TF_OpKernelContext* ctx, int i,
                                         const TF_Tensor* tensor,
                                         TF_Status* status);
 
+// Returns the expected output data type of the ith output. If i < 0 or
+// i >= TF_NumOutputs(ctx), the program aborts.
+TF_CAPI_EXPORT extern TF_DataType TF_ExpectedOutputDataType(
+    TF_OpKernelContext* ctx, int i);
+
+// Returns the step ID of the given context.
+TF_CAPI_EXPORT extern int64_t TF_StepId(TF_OpKernelContext* ctx);
+
 #ifdef __cplusplus
 } /* end extern "C" */
 #endif
diff --git a/tensorflow/c/kernels_test.cc b/tensorflow/c/kernels_test.cc
index e659ee3c3d..fdeb04c84c 100644
--- a/tensorflow/c/kernels_test.cc
+++ b/tensorflow/c/kernels_test.cc
@@ -41,6 +41,9 @@ static void* MyCreateFunc(TF_OpKernelConstruction* ctx) {
 static void MyComputeFunc(void* kernel, TF_OpKernelContext* ctx) {
   struct MyCustomKernel* s = static_cast<struct MyCustomKernel*>(kernel);
   s->compute_called = true;
+  if (ctx != nullptr) {
+    EXPECT_EQ(43, TF_StepId(ctx));
+  }
 }
 
 static void MyDeleteFunc(void* kernel) {
@@ -155,6 +158,8 @@ TEST(TestKernel, TestInputAndOutputCount) {
     TF_SetOutput(ctx, 24, input, s);
     EXPECT_EQ(TF_OUT_OF_RANGE, TF_GetCode(s));
 
+    EXPECT_EQ(TF_UINT8, TF_ExpectedOutputDataType(ctx, 0));
+
     TF_DeleteStatus(s);
     if (input != nullptr) {
       TF_DeleteTensor(input);
@@ -175,6 +180,7 @@ TEST(TestKernel, TestInputAndOutputCount) {
     OpKernelContext::Params p;
     DummyDevice dummy_device(nullptr, false);
     p.device = &dummy_device;
+    p.step_id = 43;
 
     Tensor t(tensorflow::uint8(123));
 
-- 
GitLab


From d5977dfe93b95f6902cb9df28525a1dc50e4704d Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Thu, 10 Jan 2019 11:53:59 -0800
Subject: [PATCH 0534/1095] Internal Cleanup.

PiperOrigin-RevId: 228749766
---
 tensorflow/python/keras/BUILD | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 9f8da126bf..fb0161a677 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -482,13 +482,14 @@ cuda_py_test(
 
 tf_py_test(
     name = "pooling_test",
-    size = "large",
+    size = "medium",
     srcs = ["layers/pooling_test.py"],
     additional_deps = [
         ":keras",
         "@absl_py//absl/testing:parameterized",
         "//tensorflow/python:client_testlib",
     ],
+    shard_count = 8,
 )
 
 tf_py_test(
-- 
GitLab


From 84572bff5cf02b1722ab763982a92be44e7719ef Mon Sep 17 00:00:00 2001
From: Yuefeng Zhou <yuefengz@google.com>
Date: Thu, 10 Jan 2019 12:18:36 -0800
Subject: [PATCH 0535/1095] Fix collective_all_reduce_strategy test failure.

PiperOrigin-RevId: 228754463
---
 .../distribute/python/collective_all_reduce_strategy_test.py   | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
index 682da44f5d..62ff4b178e 100644
--- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
@@ -426,7 +426,8 @@ class LocalCollectiveAllReduceStrategy(
 
   @combinations.generate(
       combinations.combine(mode=['graph', 'eager'], required_gpus=2))
-  def testMakeInputFnIterator(self, num_gpus=2):
+  def testMakeInputFnIterator(self):
+    num_gpus = 2
     dataset_fn = lambda: dataset_ops.Dataset.range(5 * num_gpus)
     expected_values = [range(i, i + num_gpus) for i in range(0, 10, num_gpus)]
 
-- 
GitLab


From 02630688f2fa239ce1d1551caa8a2f3559c248e3 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 10 Jan 2019 12:25:01 -0800
Subject: [PATCH 0536/1095] Automated rollback of commit
 73b6d9262ea6d1578374ecb63560ad0fcad13e35

PiperOrigin-RevId: 228755594
---
 tensorflow/contrib/distribute/python/BUILD    |  30 +-
 ...test_base.py => keras_correctness_test.py} | 261 ++++++++++++++----
 .../python/keras_dnn_correctness_test.py      | 163 -----------
 .../keras_image_model_correctness_test.py     |  92 ------
 4 files changed, 209 insertions(+), 337 deletions(-)
 rename tensorflow/contrib/distribute/python/{keras_correctness_test_base.py => keras_correctness_test.py} (58%)
 delete mode 100644 tensorflow/contrib/distribute/python/keras_dnn_correctness_test.py
 delete mode 100644 tensorflow/contrib/distribute/python/keras_image_model_correctness_test.py

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index 4c25f49f16..d4758d7518 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -657,11 +657,7 @@ cuda_py_test(
 py_library(
     name = "keras_correctness_test_lib",
     testonly = 1,
-    srcs = [
-        "keras_correctness_test_base.py",
-        "keras_dnn_correctness_test.py",
-        "keras_image_model_correctness_test.py",
-    ],
+    srcs = ["keras_correctness_test.py"],
     deps = [
         ":combinations",
         "//tensorflow/contrib/distribute/python:mirrored_strategy",
@@ -677,9 +673,8 @@ py_library(
 )
 
 cuda_py_test(
-    name = "keras_dnn_correctness_test",
-    size = "medium",
-    srcs = ["keras_dnn_correctness_test.py"],
+    name = "keras_correctness_test",
+    srcs = ["keras_correctness_test.py"],
     additional_deps = [
         ":keras_correctness_test_lib",
     ],
@@ -695,25 +690,6 @@ cuda_py_test(
     ],
 )
 
-cuda_py_test(
-    name = "keras_image_model_correctness_test",
-    size = "medium",
-    srcs = ["keras_image_model_correctness_test.py"],
-    additional_deps = [
-        ":keras_correctness_test_lib",
-    ],
-    # Shard count is set to an odd number to distribute tasks across
-    # shards more evenly.
-    shard_count = 31,
-    tags = [
-        "multi_and_single_gpu",
-        "no_oss",  # TODO(b/117919883): Fix python error.
-        "no_pip",
-        "no_windows_gpu",
-        "notsan",
-    ],
-)
-
 py_library(
     name = "metrics_v1_test_lib",
     testonly = 1,
diff --git a/tensorflow/contrib/distribute/python/keras_correctness_test_base.py b/tensorflow/contrib/distribute/python/keras_correctness_test.py
similarity index 58%
rename from tensorflow/contrib/distribute/python/keras_correctness_test_base.py
rename to tensorflow/contrib/distribute/python/keras_correctness_test.py
index e13984b26d..3abdee2c0e 100644
--- a/tensorflow/contrib/distribute/python/keras_correctness_test_base.py
+++ b/tensorflow/contrib/distribute/python/keras_correctness_test.py
@@ -30,6 +30,8 @@ from tensorflow.python.eager import context
 from tensorflow.python.eager import test
 from tensorflow.python.framework import random_seed
 from tensorflow.python.keras.engine import distributed_training_utils
+from tensorflow.python.keras.optimizer_v2 import gradient_descent as gradient_descent_keras
+from tensorflow.python.training import gradient_descent
 
 _RANDOM_SEED = 1337
 _EVAL_STEPS = 20
@@ -60,13 +62,24 @@ def all_strategy_combinations_with_graph_mode():
   return combinations.combine(distribution=all_strategies, mode=['graph'])
 
 
-def all_strategy_and_input_config_combinations():
+def strategy_and_input_combinations():
+  def cnn_model_with_batch_norm(**kwargs):
+    return _create_cnn_model(with_batch_norm=True, **kwargs)
+
   return (
       combinations.times(
           combinations.combine(distribution=all_strategies),
           combinations.combine(mode=['graph', 'eager'],
                                use_numpy=[True, False],
-                               use_validation_data=[True, False])))
+                               use_validation_data=[True, False]),
+          combinations.combine(model_with_data=[
+              ModelWithData('dnn', _create_dnn_model, _dnn_training_data),
+              ModelWithData('cnn', _create_cnn_model, _cnn_training_data),
+              ModelWithData('cnn_batch_norm',
+                            cnn_model_with_batch_norm,
+                            _cnn_training_data,
+                            with_batch_norm=True),
+          ])))
 
 
 class MaybeDistributionScope(object):
@@ -87,6 +100,97 @@ class MaybeDistributionScope(object):
       self._scope = None
 
 
+class ModelWithData(object):
+  """An object giving a good name in combinations.
+
+  The model_fn must take two arguments: initial_weights and distribution.
+  """
+
+  def __init__(self, name, model_fn, data_fn, with_batch_norm=False):
+    self.name = name
+    self.model_fn = model_fn
+    self.data_fn = data_fn
+    self.with_batch_norm = with_batch_norm
+
+  def __repr__(self):
+    return self.name
+
+
+def _dnn_training_data():
+  # TODO(xiejw): Change this back to 10000, once we support final partial
+  # batch.
+  num_samples = 9984
+  x_train = np.random.rand(num_samples, 1)
+  y_train = 3 * x_train
+  x_train = x_train.astype('float32')
+  y_train = y_train.astype('float32')
+  x_predict = [[1.], [2.], [3.], [4.]]
+  return x_train, y_train, x_predict
+
+
+def _create_dnn_model(initial_weights=None, distribution=None):
+  with MaybeDistributionScope(distribution):
+    # We add few non-linear layers to make it non-trivial.
+    model = keras.Sequential()
+    model.add(keras.layers.Dense(10, activation='relu', input_shape=(1,)))
+    model.add(keras.layers.Dense(10, activation='relu'))
+    model.add(keras.layers.Dense(10, activation='relu'))
+    model.add(keras.layers.Dense(1))
+
+    if initial_weights:
+      model.set_weights(initial_weights)
+
+    model.compile(
+        loss=keras.losses.mean_squared_error,
+        optimizer=gradient_descent_keras.SGD(0.5),
+        metrics=['mse'])
+    return model
+
+
+def _cnn_training_data(count=_GLOBAL_BATCH_SIZE * _EVAL_STEPS,
+                       shape=(28, 28, 3), num_classes=10):
+  centers = np.random.randn(num_classes, *shape)
+
+  features = []
+  labels = []
+  for _ in range(count):
+    label = np.random.randint(0, num_classes, size=1)[0]
+    offset = np.random.normal(loc=0, scale=0.1, size=np.prod(shape))
+    offset = offset.reshape(shape)
+    labels.append(label)
+    features.append(centers[label] + offset)
+
+  x_train = np.asarray(features, dtype=np.float32)
+  y_train = np.asarray(labels, dtype=np.float32).reshape((count, 1))
+  x_predict = x_train
+  return x_train, y_train, x_predict
+
+
+def _create_cnn_model(initial_weights=None, distribution=None,
+                      with_batch_norm=False):
+  with MaybeDistributionScope(distribution):
+    image = keras.layers.Input(shape=(28, 28, 3), name='image')
+    c1 = keras.layers.Conv2D(
+        name='conv1', filters=16, kernel_size=(3, 3), strides=(4, 4))(
+            image)
+    if with_batch_norm:
+      c1 = keras.layers.BatchNormalization(name='bn1')(c1)
+    c1 = keras.layers.MaxPooling2D(pool_size=(2, 2))(c1)
+    logits = keras.layers.Dense(
+        10, activation='softmax', name='pred')(
+            keras.layers.Flatten()(c1))
+    model = keras.Model(inputs=[image], outputs=[logits])
+
+    if initial_weights:
+      model.set_weights(initial_weights)
+
+    model.compile(
+        optimizer=gradient_descent.GradientDescentOptimizer(learning_rate=0.1),
+        loss='sparse_categorical_crossentropy',
+        metrics=['sparse_categorical_accuracy'])
+  return model
+
+
 def batch_wrapper(dataset, batch_size, distribution, repeat=None):
   if repeat:
     dataset = dataset.repeat(repeat)
@@ -246,11 +350,6 @@ def compare_results(results_with_ds, results_without_ds, distribution,
         msg='Fail to assert {}.'.format(key))
 
 
-def should_skip_tpu_with_eager(distribution):
-  return (context.executing_eagerly() and
-          isinstance(distribution, tpu_strategy.TPUStrategy))
-
-
 class LearningRateBatchScheduler(keras.callbacks.Callback):
 
   def __init__(self, update_freq=None):
@@ -265,60 +364,107 @@ class LearningRateBatchScheduler(keras.callbacks.Callback):
     keras.backend.set_value(self.model.optimizer.lr, lr)
 
 
-class TestDistributionStrategyCorrectnessBase(test.TestCase,
-                                              parameterized.TestCase):
-  """Model agnostic testing infra to test correctness of Keras models."""
-
-  def set_up_test_config(self, use_numpy=False,
-                         use_validation_data=False,
-                         with_batch_norm=False):
-    self.use_numpy = use_numpy
-    self.use_validation_data = use_validation_data
-    self.with_batch_norm = with_batch_norm
+class TestDistributionStrategyCorrectness(test.TestCase,
+                                          parameterized.TestCase):
 
-    keras.backend.set_image_data_format('channels_last')
-    np.random.seed(_RANDOM_SEED)
-    random_seed.set_random_seed(_RANDOM_SEED)
+  def _should_skip_tpu_with_eager(self, distribution):
+    return (context.executing_eagerly() and
+            isinstance(distribution, tpu_strategy.TPUStrategy))
 
-  def get_data(self):
-    num_samples = 10000
-    x_train = np.random.randint(0, 2, num_samples)
-    x_train = np.reshape(x_train, (num_samples, 1))
-    y_train = x_train
-    return (x_train.astype('float32'), y_train.astype('float32'), None)
+  @combinations.generate(all_strategy_combinations_with_eager_and_graph_modes())
+  def test_metric_correctness(self, distribution):
+    if self._should_skip_tpu_with_eager(distribution):
+      self.skipTest('TPUStrategy does not support eager mode now.')
 
-  def get_model(self, distribution=None):
-    raise NotImplementedError
+    with self.cached_session():
+      keras.backend.set_image_data_format('channels_last')
+      num_samples = 10000
+
+      x_train = np.random.randint(0, 2, num_samples)
+      x_train = np.reshape(x_train, (num_samples, 1))
+      y_train = x_train
+      x_train = x_train.astype('float32')
+      y_train = y_train.astype('float32')
+
+      # Create identity model.
+      with distribution.scope():
+        model = keras.Sequential()
+        model.add(
+            keras.layers.Dense(1, input_shape=(1,), kernel_initializer='ones'))
+        model.compile(
+            loss=keras.losses.mean_squared_error,
+            optimizer=gradient_descent.GradientDescentOptimizer(0.5),
+            metrics=[keras.metrics.BinaryAccuracy()])
+
+      batch_size = 64
+      batch_size = get_batch_size(batch_size, distribution)
+      train_dataset = dataset_ops.Dataset.from_tensor_slices((x_train, y_train))
+      train_dataset = batch_wrapper(train_dataset, batch_size, distribution)
+
+      history = model.fit(x=train_dataset, epochs=2, steps_per_epoch=10)
+      self.assertEqual(history.history['binary_accuracy'], [1.0, 1.0])
+
+  @combinations.generate(all_strategy_combinations_with_eager_and_graph_modes())
+  def test_eval_metrics_correctness(self, distribution):
+    if self._should_skip_tpu_with_eager(distribution):
+      self.skipTest('TPUStrategy does not support eager mode now.')
 
-  def skip_unsupported_test_configuration(self, distribution):
-    if should_skip_tpu_with_eager(distribution):
+    with self.cached_session():
+      with distribution.scope():
+        model = keras.Sequential()
+        model.add(
+            keras.layers.Dense(
+                3, activation='relu', input_dim=4, kernel_initializer='ones'))
+        model.add(
+            keras.layers.Dense(
+                1, activation='sigmoid', kernel_initializer='ones'))
+        model.compile(
+            loss='mae',
+            metrics=['accuracy', keras.metrics.BinaryAccuracy()],
+            optimizer=gradient_descent.GradientDescentOptimizer(0.001))
+
+      # verify correctness of stateful and stateless metrics.
+      x = np.ones((100, 4)).astype('float32')
+      y = np.ones((100, 1)).astype('float32')
+      dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).repeat()
+      dataset = batch_wrapper(dataset, 4, distribution)
+      outs = model.evaluate(dataset, steps=10)
+      self.assertEqual(outs[1], 1.)
+      self.assertEqual(outs[2], 1.)
+
+      y = np.zeros((100, 1)).astype('float32')
+      dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).repeat()
+      dataset = batch_wrapper(dataset, 4, distribution)
+      outs = model.evaluate(dataset, steps=10)
+      self.assertEqual(outs[1], 0.)
+      self.assertEqual(outs[2], 0.)
+
+  @combinations.generate(strategy_and_input_combinations())
+  def test_correctness(self, distribution, use_numpy, use_validation_data,
+                       model_with_data):
+    if self._should_skip_tpu_with_eager(distribution):
       self.skipTest('TPUStrategy does not support eager mode now.')
 
-    if context.executing_eagerly() and self.use_numpy:
+    if context.executing_eagerly() and use_numpy:
       self.skipTest('Numpy as inputs is not supported with strategy in eager.')
 
-    if context.executing_eagerly() and self.use_validation_data:
-      self.skipTest('TODO(hongjunchoi): Add test logic for using validation '
-                    'data for eager execution.')
-    return
+    if context.executing_eagerly() and use_validation_data:
+      self.skipTest('TODO')
 
-  def run_correctness_test(self,
-                           distribution,
-                           use_numpy,
-                           use_validation_data,
-                           with_batch_norm=False):
     with self.cached_session():
-      self.set_up_test_config(use_numpy, use_validation_data, with_batch_norm)
-      self.skip_unsupported_test_configuration(distribution)
+      keras.backend.set_image_data_format('channels_last')
+      np.random.seed(_RANDOM_SEED)
+      random_seed.set_random_seed(_RANDOM_SEED)
 
+      model_fn, data_fn = model_with_data.model_fn, model_with_data.data_fn
       # Train, eval, and predict datasets are created with the same input numpy
       # arrays.
-      x_train, y_train, x_predict = self.get_data()
+      x_train, y_train, x_predict = data_fn()
 
       # The model is built once and the initial weights are saved.
       # This is used to initialize the model for both the distribution and
       # non-distribution run.
-      model = self.get_model()
+      model = model_fn()
       initial_weights = model.get_weights()
 
       def input_fn(dist):
@@ -326,15 +472,15 @@ class TestDistributionStrategyCorrectnessBase(test.TestCase,
             use_numpy, use_validation_data, dist, x_train, y_train, x_predict)
 
       results_with_ds = fit_eval_and_predict(
-          initial_weights, input_fn=input_fn, model_fn=self.get_model,
+          initial_weights, input_fn=input_fn, model_fn=model_fn,
           distribution=distribution)
       results_without_ds = fit_eval_and_predict(
-          initial_weights, input_fn=input_fn, model_fn=self.get_model,
+          initial_weights, input_fn=input_fn, model_fn=model_fn,
           distribution=None)
 
       # First, special case, for multi-replica distributed training, batch norm
       # is not aggregated globally. So it is expected to have different weights.
-      if (self.with_batch_norm and
+      if (model_with_data.with_batch_norm and
           distribution.num_replicas_in_sync > 1):
         with self.assertRaises(AssertionError):
           compare_results(results_with_ds, results_without_ds, distribution,
@@ -343,16 +489,21 @@ class TestDistributionStrategyCorrectnessBase(test.TestCase,
         compare_results(results_with_ds, results_without_ds, distribution,
                         testcase=self)
 
-  def run_dynamic_lr_test(self, distribution):
+  @combinations.generate(all_strategy_combinations_with_graph_mode())
+  def test_dynamic_lr(self, distribution):
+
     with self.cached_session():
-      self.set_up_test_config()
-      self.skip_unsupported_test_configuration(distribution)
 
-      x_train, y_train, _ = self.get_data()
-      model = self.get_model()
+      keras.backend.set_image_data_format('channels_last')
+      np.random.seed(_RANDOM_SEED)
+      random_seed.set_random_seed(_RANDOM_SEED)
+
+      x_train, y_train, _ = _dnn_training_data()
+
+      model = _create_dnn_model()
       initial_weights = model.get_weights()
-      update_freq = None
 
+      update_freq = None
       if (isinstance(distribution, tpu_strategy.TPUStrategy) and
           distribution.extended.steps_per_run > 1):
         # For TPUStrategy with steps_per_run > 1, the callback is not invoked
@@ -379,10 +530,10 @@ class TestDistributionStrategyCorrectnessBase(test.TestCase,
         return training_inputs, eval_inputs, predict_inputs
 
       results_with_ds = fit_eval_and_predict(
-          initial_weights, input_fn=input_fn, model_fn=self.get_model,
+          initial_weights, input_fn=input_fn, model_fn=_create_dnn_model,
           distribution=distribution)
       results_without_ds = fit_eval_and_predict(
-          initial_weights, input_fn=input_fn, model_fn=self.get_model,
+          initial_weights, input_fn=input_fn, model_fn=_create_dnn_model,
           distribution=None)
       compare_results(results_with_ds, results_without_ds, distribution,
                       testcase=self)
diff --git a/tensorflow/contrib/distribute/python/keras_dnn_correctness_test.py b/tensorflow/contrib/distribute/python/keras_dnn_correctness_test.py
deleted file mode 100644
index f21b1c6752..0000000000
--- a/tensorflow/contrib/distribute/python/keras_dnn_correctness_test.py
+++ /dev/null
@@ -1,163 +0,0 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Correctness tests for tf.keras DNN model using DistributionStrategy."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-
-from tensorflow.contrib.distribute.python import combinations
-from tensorflow.contrib.distribute.python import keras_correctness_test_base
-from tensorflow.python import keras
-from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.eager import test
-from tensorflow.python.keras.optimizer_v2 import gradient_descent as gradient_descent_keras
-from tensorflow.python.training import gradient_descent
-
-
-class TestDistributionStrategyDnnCorrectness(
-    keras_correctness_test_base.TestDistributionStrategyCorrectnessBase):
-
-  def get_model(self, initial_weights=None, distribution=None):
-    with keras_correctness_test_base.MaybeDistributionScope(distribution):
-      # We add few non-linear layers to make it non-trivial.
-      model = keras.Sequential()
-      model.add(keras.layers.Dense(10, activation='relu', input_shape=(1,)))
-      model.add(keras.layers.Dense(10, activation='relu'))
-      model.add(keras.layers.Dense(10, activation='relu'))
-      model.add(keras.layers.Dense(1))
-
-      if initial_weights:
-        model.set_weights(initial_weights)
-
-      model.compile(
-          loss=keras.losses.mean_squared_error,
-          optimizer=gradient_descent_keras.SGD(0.5),
-          metrics=['mse'])
-      return model
-
-  def get_data(self):
-    # TODO(xiejw): Change this back to 10000, once we support final partial
-    # batch.
-    num_samples = 9984
-    x_train = np.random.rand(num_samples, 1)
-    y_train = 3 * x_train
-    x_train = x_train.astype('float32')
-    y_train = y_train.astype('float32')
-    x_predict = [[1.], [2.], [3.], [4.]]
-    return x_train, y_train, x_predict
-
-  @combinations.generate(keras_correctness_test_base.
-                         all_strategy_and_input_config_combinations())
-  def test_dnn_correctness(self, distribution, use_numpy, use_validation_data):
-    self.run_correctness_test(distribution, use_numpy, use_validation_data)
-
-  @combinations.generate(keras_correctness_test_base.
-                         all_strategy_combinations_with_graph_mode())
-  def test_dnn_with_dynamic_learning_rate(self, distribution):
-    self.run_dynamic_lr_test(distribution)
-
-
-class TestDistributionStrategyDnnMetricCorrectness(
-    keras_correctness_test_base.TestDistributionStrategyCorrectnessBase):
-
-  def get_model(self, distribution=None):
-    with distribution.scope():
-      model = keras.Sequential()
-      model.add(keras.layers.Dense(1,
-                                   input_shape=(1,),
-                                   kernel_initializer='ones'))
-      model.compile(
-          loss=keras.losses.mean_squared_error,
-          optimizer=gradient_descent.GradientDescentOptimizer(0.5),
-          metrics=[keras.metrics.BinaryAccuracy()])
-    return model
-
-  def run_metric_correctness_test(self, distribution):
-    with self.cached_session():
-      self.set_up_test_config()
-      self.skip_unsupported_test_configuration(distribution)
-
-      x_train, y_train, _ = self.get_data()
-      model = self.get_model(distribution=distribution)
-
-      batch_size = 64
-      batch_size = (keras_correctness_test_base.
-                    get_batch_size(batch_size, distribution))
-      train_dataset = dataset_ops.Dataset.from_tensor_slices((x_train, y_train))
-      train_dataset = (keras_correctness_test_base.
-                       batch_wrapper(train_dataset, batch_size, distribution))
-
-      history = model.fit(x=train_dataset, epochs=2, steps_per_epoch=10)
-      self.assertEqual(history.history['binary_accuracy'], [1.0, 1.0])
-
-  @combinations.generate(keras_correctness_test_base.
-                         all_strategy_combinations_with_eager_and_graph_modes())
-  def test_simple_dnn_metric_correctness(self, distribution):
-    self.run_metric_correctness_test(distribution)
-
-
-class TestDistributionStrategyDnnMetricEvalCorrectness(
-    keras_correctness_test_base.TestDistributionStrategyCorrectnessBase):
-
-  def get_model(self, distribution=None):
-    with distribution.scope():
-      model = keras.Sequential()
-      model.add(
-          keras.layers.Dense(
-              3, activation='relu', input_dim=4, kernel_initializer='ones'))
-      model.add(
-          keras.layers.Dense(
-              1, activation='sigmoid', kernel_initializer='ones'))
-      model.compile(
-          loss='mae',
-          metrics=['accuracy', keras.metrics.BinaryAccuracy()],
-          optimizer=gradient_descent.GradientDescentOptimizer(0.001))
-    return model
-
-  def run_eval_metrics_correctness_test(self, distribution):
-    with self.cached_session():
-      self.set_up_test_config()
-      self.skip_unsupported_test_configuration(distribution)
-
-      model = self.get_model(distribution=distribution)
-
-      # verify correctness of stateful and stateless metrics.
-      x = np.ones((100, 4)).astype('float32')
-      y = np.ones((100, 1)).astype('float32')
-      dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).repeat()
-      dataset = (keras_correctness_test_base.
-                 batch_wrapper(dataset, 4, distribution))
-      outs = model.evaluate(dataset, steps=10)
-      self.assertEqual(outs[1], 1.)
-      self.assertEqual(outs[2], 1.)
-
-      y = np.zeros((100, 1)).astype('float32')
-      dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).repeat()
-      dataset = (keras_correctness_test_base.
-                 batch_wrapper(dataset, 4, distribution))
-      outs = model.evaluate(dataset, steps=10)
-      self.assertEqual(outs[1], 0.)
-      self.assertEqual(outs[2], 0.)
-
-  @combinations.generate(keras_correctness_test_base.
-                         all_strategy_combinations_with_eager_and_graph_modes())
-  def test_identity_model_metric_eval_correctness(self, distribution):
-    self.run_eval_metrics_correctness_test(distribution)
-
-
-if __name__ == '__main__':
-  test.main()
diff --git a/tensorflow/contrib/distribute/python/keras_image_model_correctness_test.py b/tensorflow/contrib/distribute/python/keras_image_model_correctness_test.py
deleted file mode 100644
index f625664372..0000000000
--- a/tensorflow/contrib/distribute/python/keras_image_model_correctness_test.py
+++ /dev/null
@@ -1,92 +0,0 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Correctness tests for tf.keras CNN models using DistributionStrategy."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-
-from tensorflow.contrib.distribute.python import combinations
-from tensorflow.contrib.distribute.python import keras_correctness_test_base
-from tensorflow.python import keras
-from tensorflow.python.eager import test
-from tensorflow.python.training import gradient_descent
-
-
-class DistributionStrategyCnnCorrectnessTest(
-    keras_correctness_test_base.TestDistributionStrategyCorrectnessBase):
-
-  def get_model(self, initial_weights=None, distribution=None):
-    with keras_correctness_test_base.MaybeDistributionScope(distribution):
-      image = keras.layers.Input(shape=(28, 28, 3), name='image')
-      c1 = keras.layers.Conv2D(
-          name='conv1', filters=16, kernel_size=(3, 3), strides=(4, 4))(
-              image)
-      if self.with_batch_norm:
-        c1 = keras.layers.BatchNormalization(name='bn1')(c1)
-      c1 = keras.layers.MaxPooling2D(pool_size=(2, 2))(c1)
-      logits = keras.layers.Dense(
-          10, activation='softmax', name='pred')(
-              keras.layers.Flatten()(c1))
-      model = keras.Model(inputs=[image], outputs=[logits])
-
-      if initial_weights:
-        model.set_weights(initial_weights)
-
-      model.compile(
-          optimizer=gradient_descent.GradientDescentOptimizer(
-              learning_rate=0.1),
-          loss='sparse_categorical_crossentropy',
-          metrics=['sparse_categorical_accuracy'])
-
-    return model
-
-  def get_data(self,
-               count=keras_correctness_test_base._GLOBAL_BATCH_SIZE
-               * keras_correctness_test_base._EVAL_STEPS,
-               shape=(28, 28, 3),
-               num_classes=10):
-    centers = np.random.randn(num_classes, *shape)
-
-    features = []
-    labels = []
-    for _ in range(count):
-      label = np.random.randint(0, num_classes, size=1)[0]
-      offset = np.random.normal(loc=0, scale=0.1, size=np.prod(shape))
-      offset = offset.reshape(shape)
-      labels.append(label)
-      features.append(centers[label] + offset)
-
-    x_train = np.asarray(features, dtype=np.float32)
-    y_train = np.asarray(labels, dtype=np.float32).reshape((count, 1))
-    x_predict = x_train
-    return x_train, y_train, x_predict
-
-  @combinations.generate(keras_correctness_test_base.
-                         all_strategy_and_input_config_combinations())
-  def test_cnn_correctness(self, distribution, use_numpy, use_validation_data):
-    self.run_correctness_test(distribution, use_numpy, use_validation_data)
-
-  @combinations.generate(keras_correctness_test_base.
-                         all_strategy_and_input_config_combinations())
-  def test_cnn_with_batch_norm_correctness(self, distribution, use_numpy,
-                                           use_validation_data):
-    self.run_correctness_test(distribution, use_numpy, use_validation_data,
-                              with_batch_norm=True)
-
-
-if __name__ == '__main__':
-  test.main()
-- 
GitLab


From a76a7e8545c6f0255d7607ca817fa92ed7b08349 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 10 Jan 2019 12:29:21 -0800
Subject: [PATCH 0537/1095] Use CUB-based row reduction for sparse_xent_op. The
 Eigen implementation seems to trigger a bug on Volta GPUs.

PiperOrigin-RevId: 228756250
---
 tensorflow/core/kernels/BUILD                 |  5 ++-
 tensorflow/core/kernels/sparse_xent_op.cc     |  9 +++--
 tensorflow/core/kernels/sparse_xent_op.h      | 29 ++++++++++++++--
 .../core/kernels/sparse_xent_op_gpu.cu.cc     | 34 +++++++++++++++++--
 4 files changed, 65 insertions(+), 12 deletions(-)

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 5794ba520e..1a20e8628e 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -4487,7 +4487,10 @@ tf_kernel_library(
     deps = SPARSE_DEPS + [
         ":bounds_check",
         "//third_party/eigen3",
-    ],
+    ] + if_cuda([
+        ":reduction_ops",
+        "@cub_archive//:cub",
+    ]),
 )
 
 tf_kernel_library(
diff --git a/tensorflow/core/kernels/sparse_xent_op.cc b/tensorflow/core/kernels/sparse_xent_op.cc
index f84ffd5323..37d4d0661c 100644
--- a/tensorflow/core/kernels/sparse_xent_op.cc
+++ b/tensorflow/core/kernels/sparse_xent_op.cc
@@ -90,9 +90,8 @@ class SparseSoftmaxXentWithLogitsOp : public OpKernel {
             context, CheckInvalidLabelIndex<Index>(labels, logits.dim_size(1)));
       }
       functor::SparseXentFunctor<Device, T, Index> functor;
-      functor(context->eigen_device<Device>(), logits.matrix<T>(),
-              labels.vec<Index>(), scratch.vec<T>(), loss_out->vec<T>(),
-              back_out->matrix<T>());
+      functor(context, logits.matrix<T>(), labels.vec<Index>(),
+              scratch.vec<T>(), loss_out->vec<T>(), back_out->matrix<T>());
     }
   }
 };
@@ -102,11 +101,11 @@ class SparseSoftmaxXentWithLogitsOp : public OpKernel {
 namespace functor {
 template <typename T, typename Index>
 struct SparseXentFunctor<CPUDevice, T, Index> {
-  void operator()(const CPUDevice& d, typename TTypes<T>::ConstMatrix logits,
+  void operator()(OpKernelContext* ctx, typename TTypes<T>::ConstMatrix logits,
                   typename TTypes<Index>::ConstVec labels,
                   typename TTypes<T>::Vec scratch, typename TTypes<T>::Vec loss,
                   typename TTypes<T>::Matrix backprop) {
-    SparseXentEigenImpl<CPUDevice, T, Index>::Compute(d, logits, labels,
+    SparseXentEigenImpl<CPUDevice, T, Index>::Compute(ctx, logits, labels,
                                                       scratch, loss, backprop);
   }
 };
diff --git a/tensorflow/core/kernels/sparse_xent_op.h b/tensorflow/core/kernels/sparse_xent_op.h
index 6ba7931ab5..5e462424ed 100644
--- a/tensorflow/core/kernels/sparse_xent_op.h
+++ b/tensorflow/core/kernels/sparse_xent_op.h
@@ -18,6 +18,7 @@ limitations under the License.
 // Functor definition for SparseXentOp, must be compilable by nvcc.
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/platform/macros.h"
@@ -128,6 +129,26 @@ class SparseXentGradGenerator {
 
 namespace functor {
 
+template <typename Device, typename T>
+struct RowMaxReduction {
+  // Computes the maximum across the rows of logits
+  //
+  // logits: batch_size, num_classes.
+  // maximum: temporary tensor, dims: batch_size, 1
+  static inline void Compute(OpKernelContext* ctx,
+                             typename TTypes<T>::ConstMatrix logits,
+                             typename TTypes<T>::Vec maximum) {
+#if !defined(EIGEN_HAS_INDEX_LIST)
+    Eigen::array<int, 1> along_row;
+    along_row[0] = 1;
+#else
+    Eigen::IndexList<Eigen::type2index<1> > along_row;
+#endif
+    Device d = ctx->eigen_device<Device>();
+    To32Bit(maximum).device(d) = To32Bit(logits).maximum(along_row);
+  }
+};
+
 // Functor used by SparseXentOp to do the computations.
 template <typename Device, typename T, typename Index>
 struct SparseXentFunctor {
@@ -138,7 +159,7 @@ struct SparseXentFunctor {
   // scratch: temporary tensor, dims: batch_size, 1
   // loss: output tensor for the loss, dims: batch_size.
   // backprop: output tensor for the backprop, dims: batch_size, num_classes.
-  void operator()(const Device& d, typename TTypes<T>::ConstMatrix logits,
+  void operator()(OpKernelContext* ctx, typename TTypes<T>::ConstMatrix logits,
                   typename TTypes<Index>::ConstVec labels,
                   typename TTypes<T>::Vec scratch, typename TTypes<T>::Vec loss,
                   typename TTypes<T>::Matrix backprop);
@@ -149,7 +170,8 @@ struct SparseXentFunctor {
 // specializations for both device types.
 template <typename Device, typename T, typename Index>
 struct SparseXentEigenImpl {
-  static void Compute(const Device& d, typename TTypes<T>::ConstMatrix logits,
+  static void Compute(OpKernelContext* ctx,
+                      typename TTypes<T>::ConstMatrix logits,
                       typename TTypes<Index>::ConstVec labels,
                       typename TTypes<T>::Vec scratch,
                       typename TTypes<T>::Vec loss,
@@ -188,8 +210,9 @@ struct SparseXentEigenImpl {
 #endif
 
     // scratch = max_logits along classes.
-    To32Bit(scratch).device(d) = To32Bit(logits).maximum(along_class);
+    RowMaxReduction<Device, T>::Compute(ctx, logits, scratch);
 
+    Device d = ctx->eigen_device<Device>();
     // backprop = logits - max_logits.
     To32Bit(backprop).device(d) =
         To32Bit(logits) -
diff --git a/tensorflow/core/kernels/sparse_xent_op_gpu.cu.cc b/tensorflow/core/kernels/sparse_xent_op_gpu.cu.cc
index d053966028..5fe15352c3 100644
--- a/tensorflow/core/kernels/sparse_xent_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/sparse_xent_op_gpu.cu.cc
@@ -20,22 +20,50 @@ limitations under the License.
 #include "tensorflow/core/kernels/sparse_xent_op.h"
 
 #include "tensorflow/core/framework/tensor_types.h"
+#include "tensorflow/core/kernels/reduction_gpu_kernels.cu.h"
+#include "tensorflow/core/kernels/reduction_ops_common.h"
 #include "tensorflow/core/platform/types.h"
 
 namespace tensorflow {
 
 typedef Eigen::GpuDevice GPUDevice;
 
+namespace functor {
+
+// Partial specialization for a GPUDevice, that uses the CUB implementation
+// from reduction_gpu_kernels.cu.h.
+template <typename T>
+struct RowMaxReduction<GPUDevice, T> {
+  // Computes the maximum across the rows of logits
+  //
+  // logits: batch_size, num_classes.
+  // maximum: temporary tensor, dims: batch_size, 1
+  static inline void Compute(OpKernelContext* ctx,
+                             typename TTypes<T>::ConstMatrix logits,
+                             typename TTypes<T>::Vec maximum) {
+    const int kBatchDim = 0;
+    const int kClassDim = 1;
+    const int rows = logits.dimension(kBatchDim);
+    const int cols = logits.dimension(kClassDim);
+
+    typedef const Eigen::array<TTypes<float>::Tensor::Index, 1>& ReductionAxes;
+    Constants<GPUDevice> constants;
+    cub::Max op;
+    functor::ReduceImpl<T, cub::Max, T*, const T*, ReductionAxes>(
+        ctx, maximum.data(), logits.data(), 2, rows, cols, 1, 1, constants.kOne,
+        op);
+  }
+};
+
 // Partial specialization for a GPUDevice, that uses the Eigen implementation
 // from XentEigenImpl.
-namespace functor {
 template <typename T, typename Index>
 struct SparseXentFunctor<GPUDevice, T, Index> {
-  void operator()(const GPUDevice& d, typename TTypes<T>::ConstMatrix logits,
+  void operator()(OpKernelContext* ctx, typename TTypes<T>::ConstMatrix logits,
                   typename TTypes<Index>::ConstVec labels,
                   typename TTypes<T>::Vec scratch, typename TTypes<T>::Vec loss,
                   typename TTypes<T>::Matrix backprop) {
-    SparseXentEigenImpl<GPUDevice, T, Index>::Compute(d, logits, labels,
+    SparseXentEigenImpl<GPUDevice, T, Index>::Compute(ctx, logits, labels,
                                                       scratch, loss, backprop);
   }
 };
-- 
GitLab


From 0b3c3c55e177b35d38ba33170ebe2baa3f5badff Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Thu, 10 Jan 2019 12:30:40 -0800
Subject: [PATCH 0538/1095] Be resilient towards graphs without source/sink
 connectivity.

PiperOrigin-RevId: 228756459
---
 tensorflow/compiler/jit/mark_for_compilation_pass.cc | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass.cc b/tensorflow/compiler/jit/mark_for_compilation_pass.cc
index 6618e3a58a..50afec020d 100644
--- a/tensorflow/compiler/jit/mark_for_compilation_pass.cc
+++ b/tensorflow/compiler/jit/mark_for_compilation_pass.cc
@@ -677,6 +677,11 @@ Status MarkForCompilationPass::Run(
   VLOG(1) << "flags->tf_xla_auto_jit = " << flags->tf_xla_auto_jit;
   const FunctionLibraryDefinition* fld = options.flib_def;
 
+  // Deadness analysis expects a graph with source and sink edges properly
+  // connected but sometimes the incoming graph does not follow this invariant.
+  // So fix up the source and sink edges before calling into deadness analysis.
+  FixupSourceAndSinkEdges(options.graph->get());
+
   std::unique_ptr<DeadnessAnalysis> deadness;
   {
     XLA_SCOPED_LOGGING_TIMER_LEVEL("DeadnessAnalysis", 1);
-- 
GitLab


From 9c45814aac694a6454d312225121dd4c4cd33de9 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 10 Jan 2019 12:45:38 -0800
Subject: [PATCH 0539/1095] Update documentation of `tf.distribute.Strategy
 methods` to clarify and/or correct whether they expect global or per-replica
 batching.

PiperOrigin-RevId: 228758901
---
 .../python/collective_all_reduce_strategy.py  |  8 ++++
 .../distribute/python/mirrored_strategy.py    | 26 +++++++++++
 .../distribute/python/one_device_strategy.py  |  1 +
 .../python/parameter_server_strategy.py       | 26 +++++++++++
 .../contrib/distribute/python/tpu_strategy.py |  9 +++-
 .../python/distribute/distribute_lib.py       | 44 ++++++++++++-------
 .../python/distribute/mirrored_strategy.py    |  8 ++++
 .../distribute/parameter_server_strategy.py   |  8 ++++
 8 files changed, 113 insertions(+), 17 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
index a3538dfc50..eee0754325 100644
--- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
+++ b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
@@ -372,4 +372,12 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
   # TODO(priyag): Delete this once all strategies use global batch size.
   @property
   def _global_batch_size(self):
+    """`make_dataset_iterator` and `make_numpy_iterator` use global batch size.
+
+    `distribute_dataset` and `make_input_fn_iterator` assume per-replica
+    batching.
+
+    Returns:
+      Boolean.
+    """
     return True
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy.py b/tensorflow/contrib/distribute/python/mirrored_strategy.py
index 5fa36fb402..2e23a51ee5 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy.py
@@ -104,6 +104,31 @@ class MirroredStrategy(distribute_lib.DistributionStrategy):
                                 auto_shard_dataset)
     super(MirroredStrategy, self).__init__(extended)
 
+  # Override to change the documentation to reflect the different handling of
+  # global vs. local batch size between core and contrib.
+  def make_dataset_iterator(self, dataset):  # pylint: disable=useless-super-delegation
+    """Makes an iterator for input provided via `dataset`.
+
+    NOTE: The batch size of the `dataset` argument is treated differently for
+    this contrib version of `MirroredStrategy`.
+
+    Data from the given dataset will be distributed evenly across all the
+    compute replicas. We will assume that the input dataset is batched by the
+    per-replica batch size.
+
+    The user could also use `make_input_fn_iterator` if they want to
+    customize which input is fed to which replica/worker etc.
+
+    Args:
+      dataset: `tf.data.Dataset` that will be distributed evenly across all
+        replicas.
+
+    Returns:
+      An `tf.distribute.InputIterator` which returns inputs for each step of the
+      computation.  User should call `initialize` on the returned iterator.
+    """
+    return super(MirroredStrategy, self).make_dataset_iterator(dataset)
+
   # Override to change the documentation to reflect the different handling of
   # global vs. local batch size between core and contrib.
   def experimental_make_numpy_iterator(  # pylint: disable=useless-super-delegation
@@ -180,4 +205,5 @@ class MirroredExtended(CoreMirroredExtended):
   # TODO(priyag): Delete this once all strategies use global batch size.
   @property
   def _global_batch_size(self):
+    """The contrib version of Mirrored strategy uses per-replica batch size."""
     return False
diff --git a/tensorflow/contrib/distribute/python/one_device_strategy.py b/tensorflow/contrib/distribute/python/one_device_strategy.py
index 34b0c31087..836cb7cc41 100644
--- a/tensorflow/contrib/distribute/python/one_device_strategy.py
+++ b/tensorflow/contrib/distribute/python/one_device_strategy.py
@@ -199,6 +199,7 @@ class OneDeviceExtended(distribute_lib.DistributionStrategyExtended):
   # TODO(priyag): Delete this once all strategies use global batch size.
   @property
   def _global_batch_size(self):
+    """Global and per-replica batching are equivalent for OneDeviceStrategy."""
     return True
 
 
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy.py b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
index 0785427c2c..0cefef7545 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
@@ -89,6 +89,31 @@ class ParameterServerStrategy(distribute_lib.DistributionStrategy):
     super(ParameterServerStrategy, self).__init__(
         ParameterServerExtended(self, num_gpus_per_worker))
 
+  # Override to change the documentation to reflect the different handling of
+  # global vs. local batch size between core and contrib.
+  def make_dataset_iterator(self, dataset):  # pylint: disable=useless-super-delegation
+    """Makes an iterator for input provided via `dataset`.
+
+    NOTE: The batch size of the `dataset` argument is treated differently for
+    this contrib version of `ParameterServerStrategy`.
+
+    Data from the given dataset will be distributed evenly across all the
+    compute replicas. We will assume that the input dataset is batched by the
+    per-replica batch size.
+
+    The user could also use `make_input_fn_iterator` if they want to
+    customize which input is fed to which replica/worker etc.
+
+    Args:
+      dataset: `tf.data.Dataset` that will be distributed evenly across all
+        replicas.
+
+    Returns:
+      An `tf.distribute.InputIterator` which returns inputs for each step of the
+      computation.  User should call `initialize` on the returned iterator.
+    """
+    return super(ParameterServerStrategy, self).make_dataset_iterator(dataset)
+
   # Override to change the documentation to reflect the different handling of
   # global vs. local batch size between core and contrib.
   def experimental_make_numpy_iterator(  # pylint: disable=useless-super-delegation
@@ -143,4 +168,5 @@ class ParameterServerExtended(CoreParameterServerExtended):
   # TODO(priyag): Delete this once all strategies use global batch size.
   @property
   def _global_batch_size(self):
+    """The contrib version of PS strategy uses per-replica batch size."""
     return False
diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index 3f89c5869e..518c704b89 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -295,7 +295,6 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
 
   def _make_dataset_iterator(self, dataset):
     """Make iterators for each of the TPU hosts."""
-
     return input_lib.DatasetIterator(dataset, self._input_workers,
                                      self._num_replicas_in_sync)
 
@@ -667,6 +666,14 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
   # TODO(priyag): Delete this once all strategies use global batch size.
   @property
   def _global_batch_size(self):
+    """`make_dataset_iterator` and `make_numpy_iterator` use global batch size.
+
+    `distribute_dataset` and `make_input_fn_iterator` assume per-replica
+    batching.
+
+    Returns:
+      Boolean.
+    """
     return True
 
 
diff --git a/tensorflow/python/distribute/distribute_lib.py b/tensorflow/python/distribute/distribute_lib.py
index 31213ab647..5fe77e5478 100644
--- a/tensorflow/python/distribute/distribute_lib.py
+++ b/tensorflow/python/distribute/distribute_lib.py
@@ -210,12 +210,14 @@ class _SameScopeAgainContext(object):
 # TODO(yuefengz): add more replication modes.
 @tf_export("distribute.InputReplicationMode")
 class InputReplicationMode(enum.Enum):
-  """Replication mode for input function."""
+  """Replication mode for input function.
 
-  # The input function will be called on each worker independently, creating as
-  # many input pipelines as number of workers. Replicas will dequeue from the
-  # local Dataset on their worker. Distribution Strategy doesn't manage any
-  # state sharing between such separate input pipelines.
+  * `PER_WORKER`: The input function will be called on each worker
+    independently, creating as many input pipelines as number of workers.
+    Replicas will dequeue from the local Dataset on their worker.
+    `tf.distribute.Strategy` doesn't manage any state sharing between such
+    separate input pipelines.
+  """
   PER_WORKER = "PER_WORKER"
 
 
@@ -353,7 +355,8 @@ class DistributionStrategy(object):
     ```
 
     Args:
-      dataset_fn: A function that returns a `tf.data.Dataset`.
+      dataset_fn: A function that returns a `tf.data.Dataset` with per-replica
+        batching.
 
     Returns:
       A `PerReplicaDataset` that will produce data for each replica.
@@ -390,28 +393,36 @@ class DistributionStrategy(object):
     """Returns an iterator split across replicas created from an input function.
 
     The `input_fn` should take an `tf.distribute.InputContext` object where
-    information about input sharding can be accessed:
+    information about batching and input sharding can be accessed:
 
     ```
     def input_fn(input_context):
-      d = tf.data.Dataset.from_tensors([[1.]]).repeat()
+      batch_size = input_context.get_per_replica_batch_size(global_batch_size)
+      d = tf.data.Dataset.from_tensors([[1.]]).repeat().batch(batch_size)
       return d.shard(input_context.num_input_pipelines,
                      input_context.input_pipeline_id)
     with strategy.scope():
-      iterator = strategy.make_input_fn_iterator(
-          input_fn)
-      replica_results = strategy.extended.call_for_each_replica(
-          replica_fn, iterator.get_next())
+      iterator = strategy.make_input_fn_iterator(input_fn)
+      replica_results = strategy.experimental_run(replica_fn, iterator)
     ```
 
+    The `tf.data.Dataset` returned by `input_fn` should have a per-replica
+    batch size, which may be computed using
+    `input_context.get_per_replica_batch_size`.
+
     Args:
-      input_fn: A function that returns a `tf.data.Dataset`. This function is
-        expected to take an `tf.distribute.InputContext` object.
+      input_fn: A function taking a `tf.distribute.InputContext` object and
+        returning a `tf.data.Dataset`.
       replication_mode: an enum value of `tf.distribute.InputReplicationMode`.
-        Only `PER_WORKER` is supported currently.
+        Only `PER_WORKER` is supported currently, which means there will be
+        a single call to `input_fn` per worker. Replicas will dequeue from the
+        local `tf.data.Dataset` on their worker.
 
     Returns:
-      An iterator object that can be initialized and fetched next element.
+      An iterator object that should first be `.initialize()`-ed. It may then
+      either be passed to `strategy.experimental_run()` or you can
+      `iterator.get_next()` to get the next value to pass to
+      `strategy.extended.call_for_each_replica()`.
     """
     if replication_mode != InputReplicationMode.PER_WORKER:
       raise ValueError(
@@ -1813,6 +1824,7 @@ class _DefaultDistributionExtended(DistributionStrategyExtended):
   # TODO(priyag): Delete this once all strategies use global batch size.
   @property
   def _global_batch_size(self):
+    """Global and per-replica batching are equivalent for this strategy."""
     return True
 
 
diff --git a/tensorflow/python/distribute/mirrored_strategy.py b/tensorflow/python/distribute/mirrored_strategy.py
index c0a39d4b55..1ed7eef1ed 100644
--- a/tensorflow/python/distribute/mirrored_strategy.py
+++ b/tensorflow/python/distribute/mirrored_strategy.py
@@ -782,6 +782,14 @@ class MirroredExtended(distribute_lib.DistributionStrategyExtended):
   # TODO(priyag): Delete this once all strategies use global batch size.
   @property
   def _global_batch_size(self):
+    """`make_dataset_iterator` and `make_numpy_iterator` use global batch size.
+
+    `distribute_dataset` and `make_input_fn_iterator` assume per-replica
+    batching.
+
+    Returns:
+      Boolean.
+    """
     return True
 
 
diff --git a/tensorflow/python/distribute/parameter_server_strategy.py b/tensorflow/python/distribute/parameter_server_strategy.py
index ac5ee6f589..9e865082f1 100644
--- a/tensorflow/python/distribute/parameter_server_strategy.py
+++ b/tensorflow/python/distribute/parameter_server_strategy.py
@@ -536,4 +536,12 @@ class ParameterServerStrategyExtended(
   # TODO(priyag): Delete this once all strategies use global batch size.
   @property
   def _global_batch_size(self):
+    """`make_dataset_iterator` and `make_numpy_iterator` use global batch size.
+
+    `distribute_dataset` and `make_input_fn_iterator` assume per-replica
+    batching.
+
+    Returns:
+      Boolean.
+    """
     return True
-- 
GitLab


From a23420e75ddf38b92c93fe80e0d83ea7b70b11b1 Mon Sep 17 00:00:00 2001
From: Karim Nosir <karimnosseir@google.com>
Date: Thu, 10 Jan 2019 12:47:37 -0800
Subject: [PATCH 0540/1095] Enable test in OSS

PiperOrigin-RevId: 228759239
---
 tensorflow/lite/kernels/BUILD | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tensorflow/lite/kernels/BUILD b/tensorflow/lite/kernels/BUILD
index 50deac9bb6..7a4b6b8644 100644
--- a/tensorflow/lite/kernels/BUILD
+++ b/tensorflow/lite/kernels/BUILD
@@ -585,9 +585,6 @@ tf_cc_test(
     name = "bidirectional_sequence_rnn_test",
     size = "small",
     srcs = ["bidirectional_sequence_rnn_test.cc"],
-    tags = [
-        "tflite_not_portable",
-    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/lite:framework",
-- 
GitLab


From b1876c7c25cb1d40c79a77c21f1ae05deca36cc4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 10 Jan 2019 12:52:05 -0800
Subject: [PATCH 0541/1095] Migrate to new names: * get_distribution_strategy
 -> get_strategy * has_distribution_strategy -> has_strategy

PiperOrigin-RevId: 228759939
---
 tensorflow/contrib/distribute/__init__.py             | 2 ++
 tensorflow/contrib/optimizer_v2/optimizer_v2.py       | 3 +--
 tensorflow/python/distribute/input_lib.py             | 2 +-
 tensorflow/python/distribute/values.py                | 4 ++--
 tensorflow/python/eager/tape.py                       | 4 ++--
 tensorflow/python/framework/test_util.py              | 2 +-
 tensorflow/python/keras/engine/training.py            | 4 ++--
 tensorflow/python/keras/layers/normalization.py       | 9 +++++----
 tensorflow/python/keras/optimizer_v2/optimizer_v2.py  | 8 +++-----
 tensorflow/python/keras/optimizers.py                 | 2 +-
 tensorflow/python/training/optimizer.py               | 5 ++---
 tensorflow/python/training/session_manager.py         | 2 +-
 tensorflow/python/training/slot_creator.py            | 6 ++----
 tensorflow/python/training/sync_replicas_optimizer.py | 3 +--
 14 files changed, 26 insertions(+), 30 deletions(-)

diff --git a/tensorflow/contrib/distribute/__init__.py b/tensorflow/contrib/distribute/__init__.py
index 4c3f9b8f02..1bcc453a7e 100644
--- a/tensorflow/contrib/distribute/__init__.py
+++ b/tensorflow/contrib/distribute/__init__.py
@@ -64,7 +64,9 @@ _allowed_symbols = [
     'get_distribution_strategy',
     'get_loss_reduction',
     'get_replica_context',
+    'get_strategy',
     'has_distribution_strategy',
+    'has_strategy',
     'in_cross_replica_context',
     'require_replica_context',
     'run_standard_tensorflow_server',
diff --git a/tensorflow/contrib/optimizer_v2/optimizer_v2.py b/tensorflow/contrib/optimizer_v2/optimizer_v2.py
index 7fb23abc38..1323ed014c 100644
--- a/tensorflow/contrib/optimizer_v2/optimizer_v2.py
+++ b/tensorflow/contrib/optimizer_v2/optimizer_v2.py
@@ -843,8 +843,7 @@ class OptimizerV2(optimizer_v1.Optimizer):
       scale_loss_by_num_replicas = (
           distribute_lib.get_loss_reduction() == ds_reduce_util.ReduceOp.MEAN)
     if scale_loss_by_num_replicas:
-      num_replicas = \
-        distribute_ctx.get_distribution_strategy().num_replicas_in_sync
+      num_replicas = distribute_ctx.get_strategy().num_replicas_in_sync
       if num_replicas > 1:
         loss_value *= 1. / num_replicas
     return loss_value
diff --git a/tensorflow/python/distribute/input_lib.py b/tensorflow/python/distribute/input_lib.py
index 022595146d..c64eea1604 100644
--- a/tensorflow/python/distribute/input_lib.py
+++ b/tensorflow/python/distribute/input_lib.py
@@ -689,7 +689,7 @@ class MultiStepContext(object):
       if reduce_op is None:
         self._last_step_outputs[name] = output
       else:
-        distribution = distribution_strategy_context.get_distribution_strategy()
+        distribution = distribution_strategy_context.get_strategy()
         self._last_step_outputs[name] = distribution.reduce(reduce_op, output)
     else:
       assert reduce_op is not None
diff --git a/tensorflow/python/distribute/values.py b/tensorflow/python/distribute/values.py
index a9dcabdab6..2a57d5cfc7 100644
--- a/tensorflow/python/distribute/values.py
+++ b/tensorflow/python/distribute/values.py
@@ -411,11 +411,11 @@ def _assign_on_device(device, variable, tensor):
 
 
 def _assert_strategy(strategy):
-  if not distribution_strategy_context.has_distribution_strategy():
+  if not distribution_strategy_context.has_strategy():
     raise RuntimeError(
         'Need to be inside "with strategy.scope()" for %s' %
         (strategy,))
-  current_strategy = distribution_strategy_context.get_distribution_strategy()
+  current_strategy = distribution_strategy_context.get_strategy()
   if current_strategy is not strategy:
     raise RuntimeError(
         "Mixing different tf.distribute.Strategy objects: %s is not %s" %
diff --git a/tensorflow/python/eager/tape.py b/tensorflow/python/eager/tape.py
index e501b403a3..56b68b9eea 100644
--- a/tensorflow/python/eager/tape.py
+++ b/tensorflow/python/eager/tape.py
@@ -61,7 +61,7 @@ def watch(tape, tensor):
 
 def watch_variable(tape, variable):
   """Marks this variable to be watched by the given tape."""
-  strategy = distribution_strategy_context.get_distribution_strategy()
+  strategy = distribution_strategy_context.get_strategy()
   if distribution_strategy_context.get_replica_context():
     variables = [strategy.extended.value_container(variable)]
   else:
@@ -76,7 +76,7 @@ def variable_accessed(variable):
   Args:
     variable: variable to be watched.
   """
-  strategy = distribution_strategy_context.get_distribution_strategy()
+  strategy = distribution_strategy_context.get_strategy()
   if distribution_strategy_context.get_replica_context():
     variables = [strategy.extended.value_container(variable)]
   else:
diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py
index 21d21cc7f4..6d01d3bf54 100644
--- a/tensorflow/python/framework/test_util.py
+++ b/tensorflow/python/framework/test_util.py
@@ -735,7 +735,7 @@ def assert_no_garbage_created(f):
     """Sets DEBUG_SAVEALL, runs the test, and checks for new garbage."""
     # Force-load `distribution_strategy_context` to prevent GC at
     # test time when using eager. Remove once b/117329403 is resolved.
-    tape.distribution_strategy_context.get_distribution_strategy()
+    tape.distribution_strategy_context.get_strategy()
 
     gc.disable()
     previous_debug_flags = gc.get_debug()
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index a65c2b6413..0b1743af38 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -216,14 +216,14 @@ class Model(Network):
       self._distribution_strategy = distribute
       self._compile_distribution = True
     else:
-      if distribution_strategy_context.has_distribution_strategy():
+      if distribution_strategy_context.has_strategy():
         # When the user builds the model in the DS scope and cross replica
         # context we want distribution strategy to be set but when building the
         # replica copies of the models internally we should not be compiling
         # with distribution strategy and use the default compilation path.
         if distribution_strategy_context.in_cross_replica_context():
           self._distribution_strategy = (
-              distribution_strategy_context.get_distribution_strategy())
+              distribution_strategy_context.get_strategy())
 
     # Validate that arguments passed by the user to `compile` are supported by
     # DistributionStrategy.
diff --git a/tensorflow/python/keras/layers/normalization.py b/tensorflow/python/keras/layers/normalization.py
index 98bc10881b..c174c8ddd6 100644
--- a/tensorflow/python/keras/layers/normalization.py
+++ b/tensorflow/python/keras/layers/normalization.py
@@ -417,8 +417,8 @@ class BatchNormalizationV2(Layer):
       # since TPUStrategy does not implement replica local variables.
       # Remove this hack once we support TPULocalVariables.
       is_tpu_strategy = False
-      if distribution_strategy_context.has_distribution_strategy():
-        distribute = distribution_strategy_context.get_distribution_strategy()
+      if distribution_strategy_context.has_strategy():
+        distribute = distribution_strategy_context.get_strategy()
         if distribute.__class__.__name__ == 'TPUStrategy':
           is_tpu_strategy = True
 
@@ -474,7 +474,7 @@ class BatchNormalizationV2(Layer):
       momentum = ops.convert_to_tensor(self.momentum)
     if training_value or training_value is None:
       if distribution_strategy_context.in_cross_replica_context():
-        strategy = distribution_strategy_context.get_distribution_strategy()
+        strategy = distribution_strategy_context.get_strategy()
         mean_update = strategy.extended.update(
             self.moving_mean, self._assign_moving_average,
             (mean, self.momentum))
@@ -666,7 +666,8 @@ class BatchNormalizationV2(Layer):
         scale, offset = _compose_transforms(r, d, scale, offset)
 
       if distribution_strategy_context.in_cross_replica_context():
-        strategy = distribution_strategy_context.get_distribution_strategy()
+        strategy = distribution_strategy_context.get_strategy()
+
         def _do_update(var, value):
           """Compute the updates for mean and variance."""
           if in_eager_mode and not self.trainable:
diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
index 98f87a41ae..894af66f5d 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
@@ -290,8 +290,7 @@ class OptimizerV2(checkpointable.CheckpointableBase):
   @staticmethod
   def _scale_loss(loss_value):
     if distribute_lib.get_loss_reduction() == ds_reduce_util.ReduceOp.MEAN:
-      num_replicas = \
-        distribute_ctx.get_distribution_strategy().num_replicas_in_sync
+      num_replicas = distribute_ctx.get_strategy().num_replicas_in_sync
       if num_replicas > 1:
         loss_value *= (1. / num_replicas)
     return loss_value
@@ -349,7 +348,7 @@ class OptimizerV2(checkpointable.CheckpointableBase):
     """
     grads_and_vars = _filter_grads(grads_and_vars)
     var_list = [v for (_, v) in grads_and_vars]
-    if distribute_ctx.has_distribution_strategy():
+    if distribute_ctx.has_strategy():
       reduced_grads = merge_grads(grads_and_vars)
       grads_and_vars = zip(reduced_grads, var_list)
 
@@ -900,8 +899,7 @@ def _var_key(var):
   """
 
   # pylint: disable=protected-access
-  if distribute_ctx.has_distribution_strategy() and hasattr(
-      var, "_primary_var"):
+  if distribute_ctx.has_strategy() and hasattr(var, "_primary_var"):
     var = var._primary_var
   if hasattr(var, "op"):
     return var._shared_name
diff --git a/tensorflow/python/keras/optimizers.py b/tensorflow/python/keras/optimizers.py
index 12168e5574..82fb555b57 100644
--- a/tensorflow/python/keras/optimizers.py
+++ b/tensorflow/python/keras/optimizers.py
@@ -740,7 +740,7 @@ class TFOptimizer(Optimizer, checkpointable.CheckpointableBase):
     return self.optimizer.compute_gradients(loss, params)
 
   def get_updates(self, loss, params):
-    if distribution_strategy_context.has_distribution_strategy():
+    if distribution_strategy_context.has_strategy():
       self.updates = []
 
       if not params:
diff --git a/tensorflow/python/training/optimizer.py b/tensorflow/python/training/optimizer.py
index c6cc0b6044..8076ed31bf 100644
--- a/tensorflow/python/training/optimizer.py
+++ b/tensorflow/python/training/optimizer.py
@@ -521,8 +521,7 @@ class Optimizer(
   @staticmethod
   def _scale_loss(loss_value):
     if distribute_lib.get_loss_reduction() == ds_reduce_util.ReduceOp.MEAN:
-      num_replicas = \
-        distribute_ctx.get_distribution_strategy().num_replicas_in_sync
+      num_replicas = distribute_ctx.get_strategy().num_replicas_in_sync
       if num_replicas > 1:
         loss_value *= (1. / num_replicas)
     return loss_value
@@ -816,7 +815,7 @@ class Optimizer(
     v = self._non_slot_dict.get(key, None)
     if v is None:
       self._maybe_initialize_checkpointable()
-      distribution_strategy = distribute_ctx.get_distribution_strategy()
+      distribution_strategy = distribute_ctx.get_strategy()
       with distribution_strategy.colocate_vars_with(colocate_with):
         if eager:
           restored_initial_value = self._preload_simple_restoration(
diff --git a/tensorflow/python/training/session_manager.py b/tensorflow/python/training/session_manager.py
index cd22e7e7de..7bd0891e35 100644
--- a/tensorflow/python/training/session_manager.py
+++ b/tensorflow/python/training/session_manager.py
@@ -186,7 +186,7 @@ class SessionManager(object):
     # This is required to so that we initialize the TPU device before
     # restoring from checkpoint since we'll be placing variables on the device
     # and TPUInitialize wipes out the memory of the device.
-    strategy = distribution_strategy_context.get_distribution_strategy()
+    strategy = distribution_strategy_context.get_strategy()
     if strategy and hasattr(strategy.extended,
                             "_experimental_initialize_system"):
       strategy.extended._experimental_initialize_system()  # pylint: disable=protected-access
diff --git a/tensorflow/python/training/slot_creator.py b/tensorflow/python/training/slot_creator.py
index bc1137e200..abe1253b00 100644
--- a/tensorflow/python/training/slot_creator.py
+++ b/tensorflow/python/training/slot_creator.py
@@ -121,8 +121,7 @@ def create_slot(primary, val, name, colocate_with_primary=True):
     prefix = primary.op.name
   with variable_scope.variable_scope(None, prefix + "/" + name):
     if colocate_with_primary:
-      distribution_strategy = (
-          distribution_strategy_context.get_distribution_strategy())
+      distribution_strategy = distribution_strategy_context.get_strategy()
       with distribution_strategy.colocate_vars_with(primary):
         return _create_slot_var(primary, val, "", validate_shape, None, None)
     else:
@@ -159,8 +158,7 @@ def create_slot_with_initializer(primary, initializer, shape, dtype, name,
     prefix = primary.op.name
   with variable_scope.variable_scope(None, prefix + "/" + name):
     if colocate_with_primary:
-      distribution_strategy = (
-          distribution_strategy_context.get_distribution_strategy())
+      distribution_strategy = distribution_strategy_context.get_strategy()
       with distribution_strategy.colocate_vars_with(primary):
         return _create_slot_var(primary, initializer, "", validate_shape, shape,
                                 dtype)
diff --git a/tensorflow/python/training/sync_replicas_optimizer.py b/tensorflow/python/training/sync_replicas_optimizer.py
index cd4590db7f..0079ecc98b 100644
--- a/tensorflow/python/training/sync_replicas_optimizer.py
+++ b/tensorflow/python/training/sync_replicas_optimizer.py
@@ -260,8 +260,7 @@ class SyncReplicasOptimizer(optimizer.Optimizer):
     # local_anchor op will be placed on this worker task by default.
     local_anchor = control_flow_ops.no_op()
     # Colocating local_step variable prevents it being placed on the PS.
-    distribution_strategy = (
-        distribution_strategy_context.get_distribution_strategy())
+    distribution_strategy = distribution_strategy_context.get_strategy()
     with distribution_strategy.colocate_vars_with(local_anchor):
       self._local_step = variable_scope.variable(
           initial_value=0,
-- 
GitLab


From 0418e820833f88bc3545644d0aed81f51022a359 Mon Sep 17 00:00:00 2001
From: Shashi Shekhar <shashishekhar@google.com>
Date: Thu, 10 Jan 2019 13:09:58 -0800
Subject: [PATCH 0542/1095] TfLite calibration library.

TfLite calibration library to capture activation ranges of tensors.

PiperOrigin-RevId: 228762955
---
 .../lite/tools/optimize/calibration_common.h  |  61 ++++
 .../lite/tools/optimize/calibration_logger.h  |  85 +++++
 .../lite/tools/optimize/calibration_reader.cc |  55 +++
 .../lite/tools/optimize/calibration_reader.h  |  55 +++
 tensorflow/lite/tools/optimize/calibrator.cc  | 345 ++++++++++++++++++
 tensorflow/lite/tools/optimize/calibrator.h   |  64 ++++
 .../lite/tools/optimize/calibrator_test.cc    | 189 ++++++++++
 .../tools/optimize/logging_op_resolver.cc     |  61 ++++
 .../lite/tools/optimize/logging_op_resolver.h |  59 +++
 .../optimize/logging_op_resolver_test.cc      | 141 +++++++
 .../lite/tools/optimize/node_info_delegate.cc |  66 ++++
 .../lite/tools/optimize/node_info_delegate.h  |  67 ++++
 .../tools/optimize/node_info_delegate_test.cc | 152 ++++++++
 13 files changed, 1400 insertions(+)
 create mode 100644 tensorflow/lite/tools/optimize/calibration_common.h
 create mode 100644 tensorflow/lite/tools/optimize/calibration_logger.h
 create mode 100644 tensorflow/lite/tools/optimize/calibration_reader.cc
 create mode 100644 tensorflow/lite/tools/optimize/calibration_reader.h
 create mode 100644 tensorflow/lite/tools/optimize/calibrator.cc
 create mode 100644 tensorflow/lite/tools/optimize/calibrator.h
 create mode 100644 tensorflow/lite/tools/optimize/calibrator_test.cc
 create mode 100644 tensorflow/lite/tools/optimize/logging_op_resolver.cc
 create mode 100644 tensorflow/lite/tools/optimize/logging_op_resolver.h
 create mode 100644 tensorflow/lite/tools/optimize/logging_op_resolver_test.cc
 create mode 100644 tensorflow/lite/tools/optimize/node_info_delegate.cc
 create mode 100644 tensorflow/lite/tools/optimize/node_info_delegate.h
 create mode 100644 tensorflow/lite/tools/optimize/node_info_delegate_test.cc

diff --git a/tensorflow/lite/tools/optimize/calibration_common.h b/tensorflow/lite/tools/optimize/calibration_common.h
new file mode 100644
index 0000000000..1ff2d3f18a
--- /dev/null
+++ b/tensorflow/lite/tools/optimize/calibration_common.h
@@ -0,0 +1,61 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_TOOLS_OPTIMIZE_CALIBRATION_COMMON_H_
+#define TENSORFLOW_LITE_TOOLS_OPTIMIZE_CALIBRATION_COMMON_H_
+
+#include <unordered_map>
+#include <unordered_set>
+
+#include "tensorflow/lite/mutable_op_resolver.h"
+
+namespace tflite {
+namespace optimize {
+namespace calibration {
+using BuiltinOperatorKey = std::pair<BuiltinOperator, int>;
+
+using BuiltinOpsSet = std::unordered_set<
+    BuiltinOperatorKey,
+    op_resolver_hasher::OperatorKeyHasher<BuiltinOperatorKey>>;
+
+template <typename T>
+class BuiltinOpsMap
+    : public std::unordered_map<
+          BuiltinOperatorKey, T,
+          op_resolver_hasher::OperatorKeyHasher<BuiltinOperatorKey>> {};
+
+// An alias for |TfLiteRegistration.invoke|.
+using KernelEvalFuncPtr = TfLiteStatus (*)(TfLiteContext*, TfLiteNode*);
+
+enum class OperatorTensorType { kNone, kInput, kOutput, kIntermediate };
+
+// Information about an operator in the TfLite graph.
+struct OperatorInfo {
+  int node_index;
+  std::string name;
+  BuiltinOperator builtin_op_code;
+  bool is_custom_op;
+  std::vector<int> inputs;
+  std::vector<int> outputs;
+  // Inputs that need to be logged.
+  std::vector<int> loggable_inputs;
+  // Outputs that need to be logged.
+  std::vector<int> loggable_outputs;
+  const TfLiteRegistration* registration;
+};
+
+}  // namespace calibration
+}  // namespace optimize
+}  // namespace tflite
+#endif  // TENSORFLOW_LITE_TOOLS_OPTIMIZE_CALIBRATION_COMMON_H_
diff --git a/tensorflow/lite/tools/optimize/calibration_logger.h b/tensorflow/lite/tools/optimize/calibration_logger.h
new file mode 100644
index 0000000000..8fd380423a
--- /dev/null
+++ b/tensorflow/lite/tools/optimize/calibration_logger.h
@@ -0,0 +1,85 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_TOOLS_OPTIMIZE_CALIBRATION_LOGGER_H_
+#define TENSORFLOW_LITE_TOOLS_OPTIMIZE_CALIBRATION_LOGGER_H_
+
+#include <unordered_map>
+
+#include "tensorflow/lite/c/c_api_internal.h"
+
+namespace tflite {
+namespace optimize {
+namespace calibration {
+class MinMax {
+ public:
+  void Update(const float* values, size_t tensor_size) {
+    // TODO(shashishekhar): Really slow implementation, optimize
+    if (tensor_size <= 0) return;
+
+    if (!has_values_) {
+      min_ = max_ = values[0];
+      has_values_ = true;
+      return;
+    }
+
+    // We are only logging absolute min/max here.
+    // TODO(shashishekhar): Make it possible to use weighted/moving average.
+    for (size_t i = 0; i < tensor_size; i++) {
+      float val = values[i];
+      if (min_ > val) {
+        min_ = val;
+      } else if (max_ < val) {
+        max_ = val;
+      }
+    }
+  }
+
+  bool HasValues() const { return has_values_; }
+
+  TfLiteStatus Get(float* min_val, float* max_val) const {
+    if (!has_values_) return kTfLiteError;
+    *min_val = min_;
+    *max_val = max_;
+    return kTfLiteOk;
+  }
+
+ private:
+  bool has_values_;
+  float min_, max_;
+};
+
+// Captures min max values for tensors.
+class Logger {
+ public:
+  // Log the value for tensor at |tensor_index| which has |tensor_values|
+  void LogTensorValue(int tensor_index, const float* tensor_values,
+                      size_t tensor_size) {
+    tensor_id_to_stats_map_[tensor_index].Update(tensor_values, tensor_size);
+  }
+
+  // Returns a map from tensor_index -> observed min max values.
+  const std::unordered_map<int, MinMax>& GetCalibrationValues() const {
+    return tensor_id_to_stats_map_;
+  }
+
+ private:
+  std::unordered_map<int, MinMax> tensor_id_to_stats_map_;
+};
+
+}  // namespace calibration
+}  // namespace optimize
+}  // namespace tflite
+
+#endif  // TENSORFLOW_LITE_TOOLS_OPTIMIZE_CALIBRATION_LOGGER_H_
diff --git a/tensorflow/lite/tools/optimize/calibration_reader.cc b/tensorflow/lite/tools/optimize/calibration_reader.cc
new file mode 100644
index 0000000000..b01a62bd6c
--- /dev/null
+++ b/tensorflow/lite/tools/optimize/calibration_reader.cc
@@ -0,0 +1,55 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/tools/optimize/calibration_reader.h"
+
+#include "absl/memory/memory.h"
+
+namespace tflite {
+namespace optimize {
+namespace calibration {
+TfLiteStatus CalibrationReader::GetTensorStatsAsMap(
+    std::unordered_map<int, CalibrationStats>* tensor_id_to_stats_map) const {
+  tensor_id_to_stats_map->clear();
+  for (const auto& tensorid_stat : logger_->GetCalibrationValues()) {
+    auto minmax = tensorid_stat.second;
+    CalibrationReader::CalibrationStats stats;
+    TF_LITE_ENSURE_STATUS(minmax.Get(&stats.min, &stats.max));
+    tensor_id_to_stats_map->insert({tensorid_stat.first, stats});
+  }
+
+  return kTfLiteOk;
+}
+
+TfLiteStatus CalibrationReader::AddCalibrationToModel(ModelT* model) const {
+  if (!model || model->subgraphs.empty()) {
+    return kTfLiteError;
+  }
+  const auto& subgraph = model->subgraphs[0];
+  for (const auto& tensorid_stat : logger_->GetCalibrationValues()) {
+    auto minmax = tensorid_stat.second;
+    float min, max;
+    TF_LITE_ENSURE_STATUS(minmax.Get(&min, &max));
+    auto quant_params = absl::make_unique<tflite::QuantizationParametersT>();
+    quant_params->min.push_back(min);
+    quant_params->max.push_back(max);
+    subgraph->tensors[tensorid_stat.first]->quantization =
+        std::move(quant_params);
+  }
+
+  return kTfLiteOk;
+}
+}  // namespace calibration
+}  // namespace optimize
+}  // namespace tflite
diff --git a/tensorflow/lite/tools/optimize/calibration_reader.h b/tensorflow/lite/tools/optimize/calibration_reader.h
new file mode 100644
index 0000000000..af0da1bb38
--- /dev/null
+++ b/tensorflow/lite/tools/optimize/calibration_reader.h
@@ -0,0 +1,55 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_TOOLS_OPTIMIZE_CALIBRATION_READER_H_
+#define TENSORFLOW_LITE_TOOLS_OPTIMIZE_CALIBRATION_READER_H_
+
+#include <unordered_map>
+
+#include "tensorflow/lite/context.h"
+#include "tensorflow/lite/model.h"
+#include "tensorflow/lite/tools/optimize/calibration_logger.h"
+
+namespace tflite {
+namespace optimize {
+namespace calibration {
+// Warning: This is not a public API and subject to change.
+//
+// Reads calibrator data collected by running the interpreter through
+// a calibration set.
+class CalibrationReader {
+ public:
+  struct CalibrationStats {
+    float min;
+    float max;
+  };
+  explicit CalibrationReader(const Logger* logger) : logger_(logger) {}
+
+  // Gets a map from tensor index to recorded calibration values.
+  virtual TfLiteStatus GetTensorStatsAsMap(
+      std::unordered_map<int, CalibrationStats>* tensor_id_to_stats_map) const;
+
+  // Annotates the tensors in the given model with statistics captured during
+  // calibration.
+  virtual TfLiteStatus AddCalibrationToModel(ModelT* model) const;
+
+  virtual ~CalibrationReader() {}
+
+ private:
+  const Logger* logger_;
+};
+}  // namespace calibration
+}  // namespace optimize
+}  // namespace tflite
+#endif  // TENSORFLOW_LITE_TOOLS_OPTIMIZE_CALIBRATION_READER_H_
diff --git a/tensorflow/lite/tools/optimize/calibrator.cc b/tensorflow/lite/tools/optimize/calibrator.cc
new file mode 100644
index 0000000000..0e817f9346
--- /dev/null
+++ b/tensorflow/lite/tools/optimize/calibrator.cc
@@ -0,0 +1,345 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/tools/optimize/calibrator.h"
+
+#include <fstream>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
+
+#include "absl/memory/memory.h"
+#include "tensorflow/lite/core/api/error_reporter.h"
+#include "tensorflow/lite/core/api/op_resolver.h"
+#include "tensorflow/lite/interpreter.h"
+#include "tensorflow/lite/kernels/register.h"
+#include "tensorflow/lite/model.h"
+#include "tensorflow/lite/op_resolver.h"
+#include "tensorflow/lite/schema/schema_generated.h"
+#include "tensorflow/lite/string_util.h"
+#include "tensorflow/lite/tools/optimize/calibration_common.h"
+#include "tensorflow/lite/tools/optimize/calibration_logger.h"
+#include "tensorflow/lite/tools/optimize/calibration_reader.h"
+#include "tensorflow/lite/tools/optimize/logging_op_resolver.h"
+#include "tensorflow/lite/tools/optimize/node_info_delegate.h"
+
+namespace tflite {
+namespace optimize {
+namespace calibration {
+
+namespace {
+
+// Calibrator is used to hold information that can be accessed during kernel
+// invocations.
+// TfLite kernel invocations are C functions and cannot look at the global
+// structure of the graph. Calibrator allows the kernel invoke functions to
+// access the global structure of graph and know which node is currently being
+// executed. This also allows us to write a simple kernel invoke wrapper
+// (see LoggingEval) that can work for most builtin ops.
+class Calibrator {
+ public:
+  Calibrator(const std::unordered_map<const TfLiteNode*, OperatorInfo>&
+                 node_ptr_opinfo_map,
+             std::unique_ptr<LoggingOpResolver> logging_op_resolver)
+      : node_ptr_opinfo_map_(node_ptr_opinfo_map),
+        logging_op_resolver_(std::move(logging_op_resolver)) {
+    logger_ = absl::make_unique<Logger>();
+  }
+
+  // Returns the wrapped kernel invoke function |TfLiteRegistration.invoke|.
+  KernelEvalFuncPtr GetKernelInvoke(const TfLiteNode* node) const;
+
+  // Gets the instance of logger associated with the current context.
+  Logger* GetLogger() const { return logger_.get(); }
+
+  // Gets the operator information about the given TfLiteNode.
+  const OperatorInfo& GetOpInfo(const TfLiteNode* node) const {
+    return node_ptr_opinfo_map_.at(node);
+  }
+
+ private:
+  std::unordered_map<const TfLiteNode*, OperatorInfo> node_ptr_opinfo_map_;
+  std::unique_ptr<LoggingOpResolver> logging_op_resolver_;
+  const std::unordered_map<int, OperatorInfo> index_opinfo_;
+  std::unique_ptr<Logger> logger_;
+};
+
+KernelEvalFuncPtr Calibrator::GetKernelInvoke(const TfLiteNode* node) const {
+  auto op_info = node_ptr_opinfo_map_.at(node);
+  return logging_op_resolver_->GetWrappedKernelInvoke(op_info.builtin_op_code,
+                                                      1);
+}
+
+// A registry of |Calibrator| objects per |TfLiteContext|.
+// This global registry is needed to access |Calibrator| objects in the kernel
+// invoke functions i.e. |TfLiteRegistration.invoke|.
+// Kernel invoke functions are C functions that have limited access to
+// |TfLiteContext|. Kernel invoke functions don't have access to global state of
+// graph. That means during a kernel invocation, the function cannot know which
+// node it was invoked for. E.g. in case of a model with |Conv| op at two
+// locations, there is no easy way for the Conv.invoke function to disambiguate
+// the calls.
+//
+// For calibration we solve this problem by creating a map of calibrators
+// per |TfLiteContext|. This map is |GlobalCalibrationRegistry|.
+//
+// This registry is then accessed using a global getter function:
+// |GetCalibratorRegistry|.
+// E.g.
+// TfLiteStatus SomeKernelInvokeFn(TfLiteContext* context, TfLiteNode* node) {
+//   .... code ....
+//   auto registry = GetCalibratorRegistry();
+//   auto calibrator = registry->GetCalibrator(context);
+//   ..... code ....
+//  }
+//
+// This way the kernel invoke functions can get the access to the Calibrator
+// object associated with the |TfLiteContext|.
+class GlobalCalibratorRegistry {
+ public:
+  // Get the |Calibrator| associated with given context, returns null if no
+  // calibrator is associated with the given context.
+  Calibrator* GetCalibrator(const TfLiteContext* context) const {
+    if (calibrator_registry_.find(context) == calibrator_registry_.cend()) {
+      return nullptr;
+    }
+    return calibrator_registry_.at(context).get();
+  }
+
+  // Removes the association between calibrator and context.
+  // Note: This deletes the calibrator as well.
+  void RemoveCalibrator(const TfLiteContext* context) {
+    calibrator_registry_.erase(context);
+  }
+
+  // Creates an instance of |Calibrator|.
+  // Registry owns the |Calibrator| object which can be deleted by calling
+  // |RemoveCalibrator|.
+  TfLiteStatus CreateCalibrator(
+      const TfLiteContext* context,
+      const std::unordered_map<const TfLiteNode*, OperatorInfo>& node_to_opinfo,
+      std::unique_ptr<LoggingOpResolver> logging_op_resolver,
+      Calibrator** calibrator_ptr, ErrorReporter* reporter) {
+    if (calibrator_registry_.find(context) != calibrator_registry_.cend()) {
+      reporter->Report(
+          "Failed to create calibrator, context already registered.");
+      return kTfLiteError;
+    }
+    std::unique_ptr<Calibrator> calibrator = absl::make_unique<Calibrator>(
+        node_to_opinfo, std::move(logging_op_resolver));
+    calibrator_registry_[context] = std::move(calibrator);
+    *calibrator_ptr = calibrator_registry_.at(context).get();
+    return kTfLiteOk;
+  }
+
+ private:
+  std::unordered_map<const TfLiteContext*, std::unique_ptr<Calibrator>>
+      calibrator_registry_;
+};
+
+GlobalCalibratorRegistry* GetCalibratorRegistry() {
+  static GlobalCalibratorRegistry* registry = new GlobalCalibratorRegistry();
+  return registry;
+}
+
+// A wrapper implementation for |TfLiteRegistration.invoke| that logs inputs,
+// invokes the wrapped implementation and then logs the outputs.
+TfLiteStatus LoggingEval(TfLiteContext* context, TfLiteNode* node) {
+  Calibrator* calibrator = GetCalibratorRegistry()->GetCalibrator(context);
+
+  if (!calibrator) {
+    context->ReportError(context, "No calibrator found for context.");
+    return kTfLiteError;
+  }
+
+  auto kernel_invoke = calibrator->GetKernelInvoke(node);
+  auto logger = calibrator->GetLogger();
+  auto op_info = calibrator->GetOpInfo(node);
+
+  for (int i : op_info.loggable_inputs) {
+    auto tensor = context->tensors[i];
+    logger->LogTensorValue(i, tensor.data.f, tensor.bytes / sizeof(float));
+  }
+
+  auto status = kernel_invoke(context, node);
+  // TODO(shashishekhar): An intermediate tensor in graph will get logged twice
+  // once as an input and second time as output. This doesn't change the min max
+  // values but is inefficient.
+  // Using moving average will also break this.
+
+  for (int i : op_info.loggable_outputs) {
+    auto tensor = context->tensors[i];
+    logger->LogTensorValue(i, tensor.data.f, tensor.bytes / sizeof(float));
+  }
+
+  return status;
+}
+
+// Returns the loggable tensors. Not all inputs and outputs need to be logged.
+// For example, const weight tensors which have buffers associated with them
+// don't need to be logged.
+std::vector<int> GetLoggableTensorIndices(
+    const std::vector<int>& tensor_indices,
+    const flatbuffers::Vector<flatbuffers::Offset<Tensor>>* tensors,
+    const flatbuffers::Vector<flatbuffers::Offset<Buffer>>* tensor_buffers) {
+  std::vector<int> loggable;
+  for (auto tensor_index : tensor_indices) {
+    auto tensor = tensors->Get(tensor_index);
+    auto buffer_index = tensor->buffer();
+    bool has_no_buffer =
+        buffer_index == 0 || (tensor_buffers->Get(buffer_index) == nullptr);
+    if (has_no_buffer && tensor->type() == tflite::TensorType_FLOAT32) {
+      loggable.push_back(tensor_index);
+    }
+  }
+  return loggable;
+}
+
+// Creates a mapping between the static model graph and the runtime TfLiteNode*
+// nodes in the graph for the given context.
+// This is done by querying the TfLiteContext for node and registrations using
+// the |NodeInfoDelegateObserver|.
+TfLiteStatus GetNodeOpInfoMapAndContext(
+    const std::unordered_map<int, OperatorInfo>& node_to_opinfo,
+    tflite::Interpreter* const interpreter,
+    std::unordered_map<const TfLiteNode*, OperatorInfo>* node_ptr_opinfo_map,
+    const TfLiteContext** context
+
+) {
+  NodeInfoDelegateObserver delegate_observer(node_to_opinfo,
+                                             node_ptr_opinfo_map);
+  NodeInfoDelegateParams delegate_params;
+  delegate_params.delegate_observer = &delegate_observer;
+  TfLiteDelegate logging_delegate = CreateNodeInfoDelegate(&delegate_params);
+
+  auto modify_status = interpreter->ModifyGraphWithDelegate(&logging_delegate);
+  if (modify_status != kTfLiteOk) {
+    return kTfLiteError;
+  }
+  *context = delegate_observer.GetContext();
+  return kTfLiteOk;
+}
+
+string GetOpName(const tflite::OperatorCode& opcode) {
+  if (opcode.custom_code() != nullptr) {
+    return opcode.custom_code()->str();
+  }
+  return tflite::EnumNamesBuiltinOperator()[opcode.builtin_code()];
+}
+
+// A |CalibrationReader| that owns the Calibrator.
+class Reader : public CalibrationReader {
+ public:
+  Reader(const TfLiteContext* context, const Logger* logger)
+      : CalibrationReader(logger), context_(context) {}
+
+  ~Reader() override { GetCalibratorRegistry()->RemoveCalibrator(context_); }
+
+ private:
+  const TfLiteContext* context_;
+};
+
+}  // namespace
+
+TfLiteStatus BuildLoggingInterpreter(
+    const FlatBufferModel& model, const OpResolver& op_resolver,
+    std::unique_ptr<Interpreter>* interpreter,
+    std::unique_ptr<CalibrationReader>* calibration_reader) {
+  auto tflite_model = model.GetModel();
+  auto subgraphs = tflite_model->subgraphs();
+  auto tensor_buffers = tflite_model->buffers();
+
+  if (subgraphs->size() != 1) {
+    model.error_reporter()->Report(
+        "Only models with a single subgraph are supported, model had %d "
+        "subgraphs",
+        subgraphs->size());
+    return kTfLiteError;
+  }
+
+  // Populate the node index to operator info map.
+  // We want to collect this information so we can use it during runtime to
+  // log details of which inputs and outputs.
+  // At runtime TFLite kernel invoke functions can only look into their
+  // own node in the graph (TFLiteNode*) and some limited context information.
+  auto primary_subgraph = subgraphs->Get(0);
+  auto operator_codes = tflite_model->operator_codes();
+  auto operators = primary_subgraph->operators();
+  auto tensors = primary_subgraph->tensors();
+  std::unordered_map<int, OperatorInfo> node_to_opinfo;
+  BuiltinOpsSet op_and_versions;
+
+  for (size_t i = 0; i < operators->size(); i++) {
+    OperatorInfo op_info;
+    op_info.node_index = i;
+    auto op = operators->Get(i);
+    auto operator_code = operator_codes->Get(op->opcode_index());
+    op_info.builtin_op_code = operator_code->builtin_code();
+    op_info.name = GetOpName(*operator_code);
+    op_info.is_custom_op = operator_code->custom_code() != nullptr;
+
+    auto op_inputs = op->inputs();
+    auto op_outputs = op->outputs();
+    op_info.inputs = std::vector<int>(op_inputs->begin(), op_inputs->end());
+    op_info.outputs = std::vector<int>(op_outputs->begin(), op_outputs->end());
+    op_info.loggable_inputs =
+        GetLoggableTensorIndices(op_info.inputs, tensors, tensor_buffers);
+    op_info.loggable_outputs =
+        GetLoggableTensorIndices(op_info.outputs, tensors, tensor_buffers);
+    if (!op_info.is_custom_op) {
+      op_info.registration = op_resolver.FindOp(operator_code->builtin_code(),
+                                                operator_code->version());
+    } else {
+      op_info.registration =
+          op_resolver.FindOp(op_info.name.c_str(), operator_code->version());
+    }
+    node_to_opinfo[i] = op_info;
+    op_and_versions.insert({op_info.builtin_op_code, operator_code->version()});
+  }
+
+  // Prepare the logging op resolver to use |LoggingEval| for kernel
+  // invocations.
+  auto logging_op_resolver = absl::make_unique<LoggingOpResolver>(
+      op_and_versions, op_resolver, LoggingEval);
+  tflite::InterpreterBuilder(model, *logging_op_resolver)(interpreter);
+
+  if (!(*interpreter)) {
+    model.error_reporter()->Report("Failed to construct interpreter");
+    return kTfLiteError;
+  }
+
+  // Compute the mapping between runtime and static graph structure, i.e.
+  // (TfLiteContext, TfLiteNode) -> OperatorInfo
+  std::unordered_map<const TfLiteNode*, OperatorInfo> node_ptr_opinfo_map;
+  const TfLiteContext* context = nullptr;
+  GetNodeOpInfoMapAndContext(node_to_opinfo, interpreter->get(),
+                             &node_ptr_opinfo_map, &context);
+
+  Calibrator* calibrator = nullptr;
+  // Register a calibrator object for the context. This can be accessed
+  // during invocations by the logging kernels.
+  TF_LITE_ENSURE_STATUS(GetCalibratorRegistry()->CreateCalibrator(
+      context, node_ptr_opinfo_map, std::move(logging_op_resolver), &calibrator,
+      model.error_reporter()));
+  *calibration_reader = std::unique_ptr<CalibrationReader>(
+      new Reader(context, calibrator->GetLogger()));
+
+  return kTfLiteOk;
+}
+
+}  // namespace calibration
+}  // namespace optimize
+}  // namespace tflite
diff --git a/tensorflow/lite/tools/optimize/calibrator.h b/tensorflow/lite/tools/optimize/calibrator.h
new file mode 100644
index 0000000000..ab3cb27eb7
--- /dev/null
+++ b/tensorflow/lite/tools/optimize/calibrator.h
@@ -0,0 +1,64 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_TOOLS_OPTIMIZE_CALIBRATOR_H_
+#define TENSORFLOW_LITE_TOOLS_OPTIMIZE_CALIBRATOR_H_
+
+#include <unordered_map>
+
+#include "flatbuffers/flatbuffers.h"  // TF:flatbuffers
+#include "tensorflow/lite/core/api/op_resolver.h"
+#include "tensorflow/lite/model.h"
+#include "tensorflow/lite/tools/optimize/calibration_reader.h"
+
+namespace tflite {
+namespace optimize {
+namespace calibration {
+
+// Warning: This is not a public API and subject to change.
+
+// Builds a interpreter that logs the calibration data in memory.
+// The calibration data can be recovered using |calibration_reader|.
+//
+// Sample usage:
+// std::unique_ptr<Interpreter> interpreter;
+// std::unique_ptr<CalibrationReader> calibration_reader;
+// BuiltinOpResolver resolver = ...
+// FlatBufferModel model = ..
+//
+// BuildLoggingInterpreter(model, resolver, &interpreter,
+//  &calibration_reader);
+//
+//
+// * Allocate tensors...
+// * Call interpreter->invoke on calibration dataset.
+//
+// Calibration data can be read either directly by calling
+// std::unordered_map<int,  CalibrationStats>> tensor_index_to_stats;
+// calibration_reader->GetTensorStatsAsMap(&tensor_index_to_stats);
+//
+// or adding calibration data to model itself.
+// ModelT * original_floating_point_model = ...
+// calibration_reader->AddCalibrationToModel(original_floating_point_model);
+//
+TfLiteStatus BuildLoggingInterpreter(
+    const FlatBufferModel& model, const OpResolver& op_resolver,
+    std::unique_ptr<Interpreter>* interpreter,
+    std::unique_ptr<CalibrationReader>* calibration_reader);
+
+}  // namespace calibration
+}  // namespace optimize
+}  // namespace tflite
+
+#endif  // TENSORFLOW_LITE_TOOLS_OPTIMIZE_CALIBRATOR_H_
diff --git a/tensorflow/lite/tools/optimize/calibrator_test.cc b/tensorflow/lite/tools/optimize/calibrator_test.cc
new file mode 100644
index 0000000000..bbbcc70fae
--- /dev/null
+++ b/tensorflow/lite/tools/optimize/calibrator_test.cc
@@ -0,0 +1,189 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <cstring>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+#include "tensorflow/lite/kernels/register.h"
+#include "tensorflow/lite/model.h"
+#include "tensorflow/lite/tools/optimize/calibrator.h"
+
+namespace tflite {
+namespace optimize {
+namespace calibration {
+namespace {
+
+TEST(CalibratorTest, CalibrationStatsAreCollected) {
+  auto model = FlatBufferModel::BuildFromFile(
+      "third_party/tensorflow/lite/testdata/multi_add.bin");
+  ASSERT_TRUE(model);
+  std::unique_ptr<Interpreter> interpreter;
+  std::unique_ptr<CalibrationReader> reader;
+  auto status = BuildLoggingInterpreter(
+      *model, ops::builtin::BuiltinOpResolver{}, &interpreter, &reader);
+  EXPECT_EQ(kTfLiteOk, status);
+
+  ASSERT_TRUE(interpreter);
+  ASSERT_TRUE(reader);
+  std::unordered_map<int, CalibrationReader::CalibrationStats> stats;
+  status = reader->GetTensorStatsAsMap(&stats);
+  EXPECT_EQ(kTfLiteOk, status);
+  EXPECT_TRUE(stats.empty());
+
+  status = interpreter->AllocateTensors();
+  ASSERT_EQ(kTfLiteOk, status);
+  // Model does the following:
+  // 0        1       2        3
+  // |        |__ ____|        |
+  // |           |             |
+  // |          Add(tensor:4)  |
+  // |____ ______|______ ______|
+  //      |             |
+  //      Add          Add
+  //      |             |
+  //    Output:5      Output:6
+
+  const size_t tensor_size = 1 * 8 * 8 * 3;
+
+  std::vector<float> ones(tensor_size, 1.0f);
+  // Fill input tensor i with i+1, i.e. input[0] = 1.0f, input[1] = 2.0f,
+  // input[2] = 3.0f
+
+  for (size_t i = 0; i < interpreter->inputs().size(); i++) {
+    int input_tensor_idx = interpreter->inputs()[i];
+    TfLiteTensor* tensor = interpreter->tensor(input_tensor_idx);
+    ASSERT_EQ(tensor->bytes, tensor_size * sizeof(float));
+    for (size_t j = 0; j < tensor_size; j++) {
+      tensor->data.f[j] = i + 1;
+    }
+  }
+  status = interpreter->Invoke();
+  ASSERT_EQ(kTfLiteOk, status);
+  const float eps = 1e-6f;
+  // Verify that tensor 5: is 6
+  // Verify that tensor 6: is 9
+  TfLiteTensor* tensor = interpreter->tensor(interpreter->outputs()[0]);
+  for (size_t i = 0; i < tensor_size; i++) {
+    EXPECT_NEAR(tensor->data.f[i], 6.0f, eps);
+  }
+  tensor = interpreter->tensor(interpreter->outputs()[1]);
+  for (size_t i = 0; i < tensor_size; i++) {
+    EXPECT_NEAR(tensor->data.f[i], 9.0f, eps);
+  }
+
+  // Verify that min max of tensors.
+  status = reader->GetTensorStatsAsMap(&stats);
+  EXPECT_EQ(kTfLiteOk, status);
+  EXPECT_EQ(7, stats.size());
+  // Check inputs
+  for (int tensor_idx = 0; tensor_idx < 4; tensor_idx++) {
+    EXPECT_NEAR(stats.at(tensor_idx).min, tensor_idx + 1, eps);
+    EXPECT_NEAR(stats.at(tensor_idx).max, tensor_idx + 1, eps);
+  }
+  // Check tensor 4 max.
+  EXPECT_NEAR(stats.at(4).min, 5, eps);
+  EXPECT_NEAR(stats.at(4).max, 5, eps);
+
+  // Check outputs
+  EXPECT_NEAR(stats.at(5).min, 6, eps);
+  EXPECT_NEAR(stats.at(5).max, 6, eps);
+
+  EXPECT_NEAR(stats.at(6).min, 9, eps);
+  EXPECT_NEAR(stats.at(6).max, 9, eps);
+}
+
+TEST(CalibratorTest, MultipleInvokes) {
+  auto model = FlatBufferModel::BuildFromFile(
+      "third_party/tensorflow/lite/testdata/multi_add.bin");
+  ASSERT_TRUE(model);
+  std::unique_ptr<Interpreter> interpreter;
+  std::unique_ptr<CalibrationReader> reader;
+  auto status = BuildLoggingInterpreter(
+      *model, ops::builtin::BuiltinOpResolver{}, &interpreter, &reader);
+  EXPECT_EQ(kTfLiteOk, status);
+
+  ASSERT_TRUE(interpreter);
+  ASSERT_TRUE(reader);
+  status = interpreter->AllocateTensors();
+
+  EXPECT_EQ(kTfLiteOk, status);
+  const size_t tensor_size = 1 * 8 * 8 * 3;
+  // Fill input tensor i with i+1, i.e. input[0] = 1.0f, input[1] = 2.0f,
+  // input[2] = 3.0f
+
+  for (size_t i = 0; i < interpreter->inputs().size(); i++) {
+    int input_tensor_idx = interpreter->inputs()[i];
+    TfLiteTensor* tensor = interpreter->tensor(input_tensor_idx);
+    ASSERT_EQ(tensor->bytes, tensor_size * sizeof(float));
+    for (size_t j = 0; j < tensor_size; j++) {
+      tensor->data.f[j] = i + 1;
+    }
+  }
+  status = interpreter->Invoke();
+  ASSERT_EQ(kTfLiteOk, status);
+  const float eps = 1e-6f;
+  // Verify that min max of tensors.
+  std::unordered_map<int, CalibrationReader::CalibrationStats> stats;
+  status = reader->GetTensorStatsAsMap(&stats);
+  EXPECT_EQ(kTfLiteOk, status);
+  EXPECT_EQ(7, stats.size());
+  const float expected_values[7] = {
+      1.0f,  // input 0
+      2.0f,  // input 1
+      3.0f,  // input 2
+      4.0f,  // input 3
+      5.0f,  // Add(1, 2)
+      6.0f,  // Output 5: Add(0, Add(1,2))
+      9.0f,  // Output 6: Add(Add(1,2), 3)
+  };
+  for (int tensor_idx = 0; tensor_idx < 7; tensor_idx++) {
+    EXPECT_NEAR(stats.at(tensor_idx).min, expected_values[tensor_idx], eps);
+    EXPECT_NEAR(stats.at(tensor_idx).max, expected_values[tensor_idx], eps);
+  }
+  // Set input[0][0] = 1.5 and input[0][1] = 0.5 this should change the values
+  // only for input[0] and tensor 4 and ouputs 5, 6.
+  TfLiteTensor* input0 = interpreter->tensor(0);
+  input0->data.f[0] = 1.5f;
+  input0->data.f[1] = 0.5f;
+  status = interpreter->Invoke();
+  ASSERT_EQ(kTfLiteOk, status);
+  status = reader->GetTensorStatsAsMap(&stats);
+  EXPECT_EQ(kTfLiteOk, status);
+  EXPECT_EQ(7, stats.size());
+  EXPECT_NEAR(stats.at(0).min, 0.5f, eps);
+  EXPECT_NEAR(stats.at(0).max, 1.5f, eps);
+
+  for (int tensor_idx = 1; tensor_idx < 5; tensor_idx++) {
+    EXPECT_NEAR(stats.at(tensor_idx).min, expected_values[tensor_idx], eps);
+    EXPECT_NEAR(stats.at(tensor_idx).max, expected_values[tensor_idx], eps);
+  }
+
+  EXPECT_NEAR(stats.at(5).min, 5.5f, eps);
+  EXPECT_NEAR(stats.at(5).max, 6.5f, eps);
+
+  EXPECT_NEAR(stats.at(6).min, 9.0f, eps);
+  EXPECT_NEAR(stats.at(6).max, 9.0f, eps);
+}
+
+}  // namespace
+}  // namespace calibration
+}  // namespace optimize
+}  // namespace tflite
+
+int main(int argc, char** argv) {
+  // On Linux, add: FLAGS_logtostderr = true;
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/tensorflow/lite/tools/optimize/logging_op_resolver.cc b/tensorflow/lite/tools/optimize/logging_op_resolver.cc
new file mode 100644
index 0000000000..7633ebb8dd
--- /dev/null
+++ b/tensorflow/lite/tools/optimize/logging_op_resolver.cc
@@ -0,0 +1,61 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/tools/optimize/logging_op_resolver.h"
+
+#include "absl/memory/memory.h"
+
+namespace tflite {
+namespace optimize {
+namespace calibration {
+
+LoggingOpResolver::LoggingOpResolver(const BuiltinOpsSet& ops_to_replace,
+                                     const OpResolver& base_resolver,
+                                     KernelEvalFuncPtr logging_eval_fn) {
+  for (const auto& op_and_version : ops_to_replace) {
+    const TfLiteRegistration* base_registration =
+        base_resolver.FindOp(op_and_version.first, op_and_version.second);
+    BuiltinOperatorKey key = op_and_version;
+    builtin_op_evalfn_map_[key] = base_registration->invoke;
+    std::unique_ptr<TfLiteRegistration> logging_registation =
+        absl::make_unique<TfLiteRegistration>(*base_registration);
+    logging_registation->invoke = logging_eval_fn;
+    builtin_op_registration_map_[key] = std::move(logging_registation);
+  }
+}
+
+const TfLiteRegistration* LoggingOpResolver::FindOp(BuiltinOperator op,
+                                                    int version) const {
+  BuiltinOperatorKey key = {op, version};
+  if (builtin_op_registration_map_.find(key) !=
+      builtin_op_registration_map_.end()) {
+    return builtin_op_registration_map_.at(key).get();
+  }
+
+  return nullptr;
+}
+
+KernelEvalFuncPtr LoggingOpResolver::GetWrappedKernelInvoke(BuiltinOperator op,
+                                                            int version) const {
+  return builtin_op_evalfn_map_.at({op, version});
+}
+
+const TfLiteRegistration* LoggingOpResolver::FindOp(const char* op,
+                                                    int version) const {
+  // TODO(b/121374947): Support custom ops as well.
+  return nullptr;
+}
+}  // namespace calibration
+}  // namespace optimize
+}  // namespace tflite
diff --git a/tensorflow/lite/tools/optimize/logging_op_resolver.h b/tensorflow/lite/tools/optimize/logging_op_resolver.h
new file mode 100644
index 0000000000..58a3a0fe3c
--- /dev/null
+++ b/tensorflow/lite/tools/optimize/logging_op_resolver.h
@@ -0,0 +1,59 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_TOOLS_OPTIMIZE_LOGGING_OP_RESOLVER_H_
+#define TENSORFLOW_LITE_TOOLS_OPTIMIZE_LOGGING_OP_RESOLVER_H_
+
+#include <set>
+#include <unordered_map>
+
+#include "tensorflow/lite/core/api/op_resolver.h"
+#include "tensorflow/lite/mutable_op_resolver.h"
+#include "tensorflow/lite/op_resolver.h"
+#include "tensorflow/lite/tools/optimize/calibration_common.h"
+
+namespace tflite {
+namespace optimize {
+namespace calibration {
+// A resolver that replaces the kernel invocations with a wrapper
+// eval function.
+class LoggingOpResolver : public OpResolver {
+ public:
+  // Creates an instance of |LoggingOpResolver|.
+  // All |TfLiteRegistration.invoke| functions are replaced by
+  // |logging_eval_fn|.
+  // TODO(shashishekhar): This interface needs to change for custom ops and
+  // BuiltinOps that need special logging implementations.
+  LoggingOpResolver(const BuiltinOpsSet& ops_to_replace,
+                    const OpResolver& base_resolver,
+                    KernelEvalFuncPtr logging_eval_fn);
+
+  const TfLiteRegistration* FindOp(BuiltinOperator op,
+                                   int version) const override;
+
+  KernelEvalFuncPtr GetWrappedKernelInvoke(BuiltinOperator op,
+                                           int version) const;
+  const TfLiteRegistration* FindOp(const char* op, int version) const override;
+
+ private:
+  BuiltinOpsMap<std::unique_ptr<TfLiteRegistration>>
+      builtin_op_registration_map_;
+  BuiltinOpsMap<KernelEvalFuncPtr> builtin_op_evalfn_map_;
+};
+
+}  // namespace calibration
+}  // namespace optimize
+}  // namespace tflite
+
+#endif  // TENSORFLOW_LITE_TOOLS_OPTIMIZE_LOGGING_OP_RESOLVER_H_
diff --git a/tensorflow/lite/tools/optimize/logging_op_resolver_test.cc b/tensorflow/lite/tools/optimize/logging_op_resolver_test.cc
new file mode 100644
index 0000000000..18c29abec6
--- /dev/null
+++ b/tensorflow/lite/tools/optimize/logging_op_resolver_test.cc
@@ -0,0 +1,141 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/tools/optimize/logging_op_resolver.h"
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+#include "tensorflow/lite/mutable_op_resolver.h"
+
+namespace tflite {
+namespace optimize {
+namespace calibration {
+namespace {
+
+TfLiteStatus ConvPrepare(TfLiteContext* context, TfLiteNode* node) {
+  return kTfLiteOk;
+}
+
+TfLiteStatus ConvEval(TfLiteContext* context, TfLiteNode* node) {
+  return kTfLiteOk;
+}
+
+TfLiteStatus AddPrepare(TfLiteContext* context, TfLiteNode* node) {
+  return kTfLiteOk;
+}
+
+TfLiteStatus AddEval(TfLiteContext* context, TfLiteNode* node) {
+  return kTfLiteOk;
+}
+
+TfLiteStatus WrappingInvoke(TfLiteContext* context, TfLiteNode* node) {
+  return kTfLiteOk;
+}
+
+TEST(LoggingOpResolverTest, KernelInvokesAreReplaced) {
+  MutableOpResolver base_resolver;
+  TfLiteRegistration conv_registration = {
+      .prepare = ConvPrepare,
+      .invoke = ConvEval,
+  };
+  base_resolver.AddBuiltin(BuiltinOperator_CONV_2D, &conv_registration);
+
+  TfLiteRegistration add_registration = {
+      .prepare = AddPrepare,
+      .invoke = AddEval,
+  };
+  base_resolver.AddBuiltin(BuiltinOperator_ADD, &add_registration);
+  BuiltinOpsSet ops_to_replace = {
+      {BuiltinOperator_CONV_2D, /*version*/ 1},
+      {BuiltinOperator_ADD, /*version*/ 1},
+  };
+
+  LoggingOpResolver resolver(ops_to_replace, base_resolver, WrappingInvoke);
+
+  auto reg = resolver.FindOp(BuiltinOperator_CONV_2D, 1);
+
+  EXPECT_EQ(reg->builtin_code, BuiltinOperator_CONV_2D);
+  EXPECT_TRUE(reg->prepare == ConvPrepare);
+  EXPECT_TRUE(reg->invoke == WrappingInvoke);
+
+  reg = resolver.FindOp(BuiltinOperator_ADD, 1);
+
+  EXPECT_EQ(reg->builtin_code, BuiltinOperator_ADD);
+  EXPECT_TRUE(reg->prepare == AddPrepare);
+  EXPECT_TRUE(reg->invoke == WrappingInvoke);
+}
+
+TEST(LoggingOpResolverTest, OriginalKernelInvokesAreRetained) {
+  MutableOpResolver base_resolver;
+  TfLiteRegistration conv_registration = {
+      .prepare = ConvPrepare,
+      .invoke = ConvEval,
+  };
+  base_resolver.AddBuiltin(BuiltinOperator_CONV_2D, &conv_registration);
+
+  TfLiteRegistration add_registration = {
+      .prepare = AddPrepare,
+      .invoke = AddEval,
+  };
+  base_resolver.AddBuiltin(BuiltinOperator_ADD, &add_registration);
+  BuiltinOpsSet ops_to_replace = {
+      {BuiltinOperator_CONV_2D, /*version*/ 1},
+      {BuiltinOperator_ADD, /*version*/ 1},
+  };
+
+  LoggingOpResolver resolver(ops_to_replace, base_resolver, WrappingInvoke);
+  auto kernel_invoke =
+      resolver.GetWrappedKernelInvoke(BuiltinOperator_CONV_2D, 1);
+  EXPECT_TRUE(kernel_invoke == ConvEval);
+  kernel_invoke = resolver.GetWrappedKernelInvoke(BuiltinOperator_ADD, 1);
+  EXPECT_TRUE(kernel_invoke == AddEval);
+}
+
+TEST(LoggingOpResolverTest, OnlyOpsInReplacementSetAreReplaces) {
+  MutableOpResolver base_resolver;
+  TfLiteRegistration conv_registration = {
+      .prepare = ConvPrepare,
+      .invoke = ConvEval,
+  };
+  base_resolver.AddBuiltin(BuiltinOperator_CONV_2D, &conv_registration);
+
+  TfLiteRegistration add_registration = {
+      .prepare = AddPrepare,
+      .invoke = AddEval,
+  };
+  base_resolver.AddBuiltin(BuiltinOperator_ADD, &add_registration);
+  // Only replace conv2d
+  BuiltinOpsSet ops_to_replace = {
+      {BuiltinOperator_CONV_2D, /*version*/ 1},
+  };
+
+  LoggingOpResolver resolver(ops_to_replace, base_resolver, WrappingInvoke);
+  auto reg = resolver.FindOp(BuiltinOperator_CONV_2D, 1);
+  EXPECT_EQ(reg->builtin_code, BuiltinOperator_CONV_2D);
+  EXPECT_TRUE(reg->prepare == ConvPrepare);
+  EXPECT_TRUE(reg->invoke == WrappingInvoke);
+
+  reg = resolver.FindOp(BuiltinOperator_ADD, 1);
+  EXPECT_EQ(nullptr, reg);
+}
+
+}  // namespace
+}  // namespace calibration
+}  // namespace optimize
+}  // namespace tflite
+
+int main(int argc, char** argv) {
+  // On Linux, add: FLAGS_logtostderr = true;
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/tensorflow/lite/tools/optimize/node_info_delegate.cc b/tensorflow/lite/tools/optimize/node_info_delegate.cc
new file mode 100644
index 0000000000..ccaa69373f
--- /dev/null
+++ b/tensorflow/lite/tools/optimize/node_info_delegate.cc
@@ -0,0 +1,66 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/tools/optimize/node_info_delegate.h"
+
+namespace tflite {
+namespace optimize {
+namespace calibration {
+
+namespace {
+// The prepare function for delegate that forwards the prepare call to the
+// delegate observer in node info delegate params.
+// The function simply calls a delegate observer OnDelegatePrepareMethod.
+TfLiteStatus NodeInfoDelegatePrepare(TfLiteContext* context,
+                                     TfLiteDelegate* delegate) {
+  if (delegate == nullptr) return TfLiteStatus::kTfLiteError;
+
+  NodeInfoDelegateParams* params =
+      reinterpret_cast<NodeInfoDelegateParams*>(delegate->data_);
+  return params->delegate_observer->OnDelegatePrepareCalled(context);
+}
+}  // namespace
+
+TfLiteDelegate CreateNodeInfoDelegate(NodeInfoDelegateParams* params) {
+  return {.data_ = params,
+          .Prepare = NodeInfoDelegatePrepare,
+          .CopyFromBufferHandle = nullptr,
+          .CopyToBufferHandle = nullptr,
+          .FreeBufferHandle = nullptr};
+}
+
+TfLiteStatus NodeInfoDelegateObserver::OnDelegatePrepareCalled(
+    TfLiteContext* context) {
+  context_ = context;
+  const size_t num_nodes = node_index_opinfo_map_.size();
+  for (size_t node_index = 0; node_index < num_nodes; node_index++) {
+    TfLiteNode* node = nullptr;
+    TfLiteRegistration* reg = nullptr;
+    TF_LITE_ENSURE_STATUS(
+        context->GetNodeAndRegistration(context, node_index, &node, &reg));
+    auto op_info = node_index_opinfo_map_.at(node_index);
+    op_info.registration = reg;
+    node_ptr_opinfo_map_->insert({node, op_info});
+  }
+
+  if (node_ptr_opinfo_map_->size() != node_index_opinfo_map_.size()) {
+    // Something wrong.
+    return kTfLiteError;
+  }
+  return kTfLiteOk;
+}
+
+}  // namespace calibration
+}  // namespace optimize
+}  // namespace tflite
diff --git a/tensorflow/lite/tools/optimize/node_info_delegate.h b/tensorflow/lite/tools/optimize/node_info_delegate.h
new file mode 100644
index 0000000000..8ee2ce1978
--- /dev/null
+++ b/tensorflow/lite/tools/optimize/node_info_delegate.h
@@ -0,0 +1,67 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_TOOLS_OPTIMIZE_NODE_INFO_DELEGATE_H_
+#define TENSORFLOW_LITE_TOOLS_OPTIMIZE_NODE_INFO_DELEGATE_H_
+
+#include <unordered_map>
+
+#include "tensorflow/lite/context.h"
+#include "tensorflow/lite/tools/optimize/calibration_common.h"
+
+namespace tflite {
+namespace optimize {
+namespace calibration {
+
+// An interface for delegate observer that can listen to TfLiteDelegate::Prepare
+// calls.
+class DelegateObserver {
+ public:
+  virtual TfLiteStatus OnDelegatePrepareCalled(TfLiteContext* context) = 0;
+  virtual ~DelegateObserver() {}
+};
+
+// The parameters for the node info delegate.
+struct NodeInfoDelegateParams {
+  DelegateObserver* delegate_observer;
+};
+
+// Creates a delegate with the given |params|.
+TfLiteDelegate CreateNodeInfoDelegate(NodeInfoDelegateParams* params);
+
+// A delegate observer that can construct the map from TfLiteNode* ->
+// OperatorInfo.
+class NodeInfoDelegateObserver : public DelegateObserver {
+ public:
+  NodeInfoDelegateObserver(
+      const std::unordered_map<int, OperatorInfo>& node_index_to_op,
+      std::unordered_map<const TfLiteNode*, OperatorInfo>* node_ptr_opinfo_map)
+      : node_index_opinfo_map_(node_index_to_op),
+        node_ptr_opinfo_map_(node_ptr_opinfo_map) {}
+
+  TfLiteStatus OnDelegatePrepareCalled(TfLiteContext* context) override;
+
+  // Returns the context that was used to called the prepare method.
+  const TfLiteContext* GetContext() const { return context_; }
+
+ private:
+  const TfLiteContext* context_ = nullptr;
+  const std::unordered_map<int, OperatorInfo>& node_index_opinfo_map_;
+  std::unordered_map<const TfLiteNode*, OperatorInfo>* node_ptr_opinfo_map_;
+};
+
+}  // namespace calibration
+}  // namespace optimize
+}  // namespace tflite
+#endif  // TENSORFLOW_LITE_TOOLS_OPTIMIZE_NODE_INFO_DELEGATE_H_
diff --git a/tensorflow/lite/tools/optimize/node_info_delegate_test.cc b/tensorflow/lite/tools/optimize/node_info_delegate_test.cc
new file mode 100644
index 0000000000..e762d5c014
--- /dev/null
+++ b/tensorflow/lite/tools/optimize/node_info_delegate_test.cc
@@ -0,0 +1,152 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <unordered_map>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+#include "tensorflow/lite/kernels/register.h"
+#include "tensorflow/lite/model.h"
+#include "tensorflow/lite/tools/optimize/node_info_delegate.h"
+
+namespace tflite {
+namespace optimize {
+namespace calibration {
+namespace {
+
+class TestDelegateObserver : public DelegateObserver {
+ public:
+  explicit TestDelegateObserver(TfLiteStatus status_to_return)
+      : status_to_return_(status_to_return) {}
+
+  TfLiteStatus OnDelegatePrepareCalled(TfLiteContext* context) override {
+    num_times_called_++;
+    return status_to_return_;
+  }
+  int num_times_called() { return num_times_called_; }
+
+ private:
+  int num_times_called_ = 0;
+  TfLiteStatus status_to_return_;
+};
+
+TEST(NodeInfoDelegateTest, DelegateObserverIsCalled) {
+  TestDelegateObserver observer(kTfLiteOk);
+  NodeInfoDelegateParams params;
+  params.delegate_observer = &observer;
+  auto model = FlatBufferModel::BuildFromFile(
+      "third_party/tensorflow/lite/testdata/multi_add.bin");
+  ASSERT_TRUE(model);
+  std::unique_ptr<Interpreter> interpreter;
+  ASSERT_EQ(InterpreterBuilder(*model,
+                               ops::builtin::BuiltinOpResolver{})(&interpreter),
+            kTfLiteOk);
+  ASSERT_TRUE(interpreter);
+  EXPECT_EQ(0, observer.num_times_called());
+  TfLiteDelegate delegate = CreateNodeInfoDelegate(&params);
+
+  auto status = interpreter->ModifyGraphWithDelegate(&delegate);
+  EXPECT_EQ(kTfLiteOk, status);
+  EXPECT_EQ(1, observer.num_times_called());
+}
+
+TEST(NodeInfoDelegateTest, ObserverErrorCausesModifyGraphFailure) {
+  // Observer returns error
+  TestDelegateObserver observer(kTfLiteError);
+  NodeInfoDelegateParams params;
+  params.delegate_observer = &observer;
+  auto model = FlatBufferModel::BuildFromFile(
+      "third_party/tensorflow/lite/testdata/multi_add.bin");
+  ASSERT_TRUE(model);
+  std::unique_ptr<Interpreter> interpreter;
+  ASSERT_EQ(InterpreterBuilder(*model,
+                               ops::builtin::BuiltinOpResolver{})(&interpreter),
+            kTfLiteOk);
+  ASSERT_TRUE(interpreter);
+  TfLiteDelegate delegate = CreateNodeInfoDelegate(&params);
+
+  auto status = interpreter->ModifyGraphWithDelegate(&delegate);
+  EXPECT_EQ(kTfLiteError, status);
+}
+
+TEST(NodeInfoDelegateTest, NodeInfoDelegateObserver) {
+  auto model = FlatBufferModel::BuildFromFile(
+      "third_party/tensorflow/lite/testdata/multi_add.bin");
+  ASSERT_TRUE(model);
+
+  std::unordered_map<int, OperatorInfo> index_to_opinfo;
+  auto primary_subgraph = model->GetModel()->subgraphs()->Get(0);
+  auto operators = primary_subgraph->operators();
+  auto subgraph_tensors = primary_subgraph->tensors();
+  for (size_t i = 0; i < operators->size(); i++) {
+    OperatorInfo info;
+    auto op_inputs = operators->Get(i)->inputs();
+    auto op_outputs = operators->Get(i)->outputs();
+    info.inputs = std::vector<int>(op_inputs->begin(), op_inputs->end());
+    info.outputs = std::vector<int>(op_outputs->begin(), op_outputs->end());
+    index_to_opinfo[i] = info;
+  }
+
+  std::unordered_map<const TfLiteNode*, OperatorInfo> node_to_opinfo;
+  NodeInfoDelegateObserver observer(index_to_opinfo, &node_to_opinfo);
+  NodeInfoDelegateParams params;
+  params.delegate_observer = &observer;
+  std::unique_ptr<Interpreter> interpreter;
+  ASSERT_EQ(InterpreterBuilder(*model,
+                               ops::builtin::BuiltinOpResolver{})(&interpreter),
+            kTfLiteOk);
+  ASSERT_TRUE(interpreter);
+
+  TfLiteDelegate delegate = CreateNodeInfoDelegate(&params);
+
+  auto status = interpreter->ModifyGraphWithDelegate(&delegate);
+  EXPECT_EQ(kTfLiteOk, status);
+  EXPECT_EQ(index_to_opinfo.size(), node_to_opinfo.size());
+  EXPECT_EQ(interpreter->nodes_size(), node_to_opinfo.size());
+
+  for (const auto& node_and_opinfo : node_to_opinfo) {
+    const TfLiteNode* tflite_node = node_and_opinfo.first;
+    const OperatorInfo& info = node_and_opinfo.second;
+    ASSERT_EQ(tflite_node->inputs->size, info.inputs.size());
+    ASSERT_EQ(tflite_node->outputs->size, info.outputs.size());
+
+    for (size_t input_index = 0; input_index < info.inputs.size();
+         input_index++) {
+      const TfLiteTensor* tflite_tensor =
+          interpreter->tensor(tflite_node->inputs->data[input_index]);
+      EXPECT_EQ(tflite_tensor->name,
+                subgraph_tensors->Get(info.inputs[input_index])->name()->str());
+    }
+
+    for (size_t output_index = 0; output_index < info.outputs.size();
+         output_index++) {
+      const TfLiteTensor* tflite_tensor =
+          interpreter->tensor(tflite_node->outputs->data[output_index]);
+      EXPECT_EQ(
+          tflite_tensor->name,
+          subgraph_tensors->Get(info.outputs[output_index])->name()->str());
+    }
+  }
+}
+
+}  // namespace
+}  // namespace calibration
+}  // namespace optimize
+}  // namespace tflite
+
+int main(int argc, char** argv) {
+  // On Linux, add: FLAGS_logtostderr = true;
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
-- 
GitLab


From adfde389863219e3f2c90e566ba620d48fd44978 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Thu, 10 Jan 2019 13:18:59 -0800
Subject: [PATCH 0543/1095] Needed this change to compile after rebasing

---
 tensorflow/contrib/tensorrt/resources/trt_resources.h | 2 +-
 tensorflow/contrib/tensorrt/segment/segment.cc        | 7 -------
 2 files changed, 1 insertion(+), 8 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/resources/trt_resources.h b/tensorflow/contrib/tensorrt/resources/trt_resources.h
index 8d877b392f..aac9e5c7bd 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_resources.h
+++ b/tensorflow/contrib/tensorrt/resources/trt_resources.h
@@ -48,7 +48,7 @@ class TRTCalibrationResource : public tensorflow::ResourceBase {
     allocator_.reset();
   }
 
-  string DebugString() const override {
+  string DebugString() override {
     std::stringstream oss;
     using std::dec;
     using std::endl;
diff --git a/tensorflow/contrib/tensorrt/segment/segment.cc b/tensorflow/contrib/tensorrt/segment/segment.cc
index 6f2a7846d7..084a96e0fa 100644
--- a/tensorflow/contrib/tensorrt/segment/segment.cc
+++ b/tensorflow/contrib/tensorrt/segment/segment.cc
@@ -28,7 +28,6 @@ limitations under the License.
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/types.h"
-#include "tensorflow/core/util/device_name_utils.h"
 
 namespace tensorflow {
 namespace tensorrt {
@@ -420,11 +419,6 @@ tensorflow::Status SegmentGraph(
   //    segment but are not eligible, using input/output_candidate_fn to
   //    determine the eligibilities;
   // 3. convert the segment into expected return format and return the result.
-  auto get_device_type = [](const string& device) {
-    DeviceNameUtils::ParsedName parsed_name;
-    DeviceNameUtils::ParseFullName(device, &parsed_name);
-    return parsed_name.type;
-  };
 
   // --------------------------------- Step 1 ---------------------------------
   auto graph = std::unique_ptr<SimpleGraph>(new SimpleGraph(tf_graph));
@@ -436,7 +430,6 @@ tensorflow::Status SegmentGraph(
   std::vector<UnionFind<SimpleNode*>> node_segments;
   for (int i = 0; i < graph->num_node_ids(); ++i) {
     SimpleNode* node = graph->FindNodeId(i);
-
     if (options.exclude_node_list.count(node->name()) != 0) {
       VLOG(1) << "Not a TF-TRT candidate, "
               << "(Op type: " << node->tf_node()->type_string() << "), "
-- 
GitLab


From 1271cb81425692e81a905cb0e7265c5b9c512093 Mon Sep 17 00:00:00 2001
From: Andy Ly <lyandy@google.com>
Date: Thu, 10 Jan 2019 13:10:22 -0800
Subject: [PATCH 0544/1095] [Grappler] Fix updating ports when removing regular
 fanins. Cleaned up some mutation logic and separated out adding and removing
 of regular and controlling fanins.

PiperOrigin-RevId: 228763029
---
 .../core/grappler/mutable_graph_view.cc       | 310 ++++++++------
 tensorflow/core/grappler/mutable_graph_view.h |  78 ++--
 .../core/grappler/mutable_graph_view_test.cc  | 396 +++++++++++-------
 3 files changed, 470 insertions(+), 314 deletions(-)

diff --git a/tensorflow/core/grappler/mutable_graph_view.cc b/tensorflow/core/grappler/mutable_graph_view.cc
index 5df8bd8113..a8a90d1a57 100644
--- a/tensorflow/core/grappler/mutable_graph_view.cc
+++ b/tensorflow/core/grappler/mutable_graph_view.cc
@@ -39,6 +39,22 @@ bool IsTensorIdPortValid(const TensorId& tensor_id) {
   return tensor_id.index() >= Graph::kControlSlot;
 }
 
+bool IsTensorIdRegular(const TensorId& tensor_id) {
+  return tensor_id.index() > Graph::kControlSlot;
+}
+
+bool IsTensorIdControlling(const TensorId& tensor_id) {
+  return tensor_id.index() == Graph::kControlSlot;
+}
+
+bool IsOutputPortRegular(const MutableGraphView::OutputPort& port) {
+  return port.port_id > Graph::kControlSlot;
+}
+
+bool IsOutputPortControlling(const MutableGraphView::OutputPort& port) {
+  return port.port_id == Graph::kControlSlot;
+}
+
 // Determines if node is an Identity where it's first regular input is a Switch
 // node.
 bool IsIdentityConsumingSwitch(const MutableGraphView& graph,
@@ -46,7 +62,10 @@ bool IsIdentityConsumingSwitch(const MutableGraphView& graph,
   if ((IsIdentity(node) || IsIdentityNSingleInput(node)) &&
       node.input_size() > 0) {
     TensorId tensor_id = ParseTensorName(node.input(0));
-    if (tensor_id.index() == Graph::kControlSlot) return false;
+    if (IsTensorIdControlling(tensor_id)) {
+      return false;
+    }
+
     NodeDef* input_node = graph.GetNode(tensor_id.node());
     return IsSwitch(*input_node);
   }
@@ -83,7 +102,7 @@ void MutableGraphView::AddAndDedupFanouts(NodeDef* node) {
   while (pos <= last_pos) {
     TensorId tensor_id = ParseTensorName(node->input(pos));
     absl::string_view input_node_name = tensor_id.node();
-    bool is_control_input = IsControlInput(tensor_id);
+    bool is_control_input = IsTensorIdControlling(tensor_id);
     bool can_dedup_control_with_regular_input =
         CanDedupControlWithRegularInput(*this, input_node_name);
     bool can_dedup_control =
@@ -246,15 +265,13 @@ bool MutableGraphView::UpdateFanoutsInternal(NodeDef* from_node,
   auto control_fanouts =
       GetFanout(GraphView::OutputPort(from_node, Graph::kControlSlot));
 
-  const string from_control_input = absl::StrCat("^", from_node->name());
-  const string to_control_input = absl::StrCat("^", to_node->name());
-
   for (const InputPort& control_port : control_fanouts) {
     // Node can't be control dependency of itself.
     if (control_port.node == to_node) continue;
 
     NodeDef* node = control_port.node;
     modified |= RemoveControllingFaninInternal(node, from_node);
+    // TODO(lyandy): Handle Switch control dependencies.
     modified |= AddFaninInternal(node, {to_node, Graph::kControlSlot});
   }
 
@@ -276,7 +293,7 @@ bool MutableGraphView::AddFaninInternal(NodeDef* node,
                                         const OutputPort& fanin) {
   int num_non_controlling_fanins =
       NumFanins(*node, /*include_controlling_nodes=*/false);
-  bool input_is_control = fanin.port_id == Graph::kControlSlot;
+  bool input_is_control = IsOutputPortControlling(fanin);
   bool can_dedup_control_with_regular_input =
       CanDedupControlWithRegularInput(*this, *fanin.node);
   // Don't add duplicate control dependencies.
@@ -296,12 +313,12 @@ bool MutableGraphView::AddFaninInternal(NodeDef* node,
       input_is_control ? Graph::kControlSlot : num_non_controlling_fanins;
 
   node->add_input(TensorIdToString({fanin.node->name(), fanin.port_id}));
-  if (fanin.port_id > Graph::kControlSlot) {
-    int node_input_size = node->input_size() - 1;
+  if (IsOutputPortRegular(fanin)) {
+    int last_node_input = node->input_size() - 1;
     // If there are control dependencies in node, move newly inserted fanin to
     // be before such control dependencies.
-    if (num_non_controlling_fanins < node_input_size) {
-      node->mutable_input()->SwapElements(node_input_size,
+    if (num_non_controlling_fanins < last_node_input) {
+      node->mutable_input()->SwapElements(last_node_input,
                                           num_non_controlling_fanins);
     }
   }
@@ -327,9 +344,9 @@ bool MutableGraphView::AddFaninInternal(NodeDef* node, const TensorId& fanin) {
   return AddFaninInternal(node, {fanin_node, fanin.index()});
 }
 
-bool MutableGraphView::AddFanin(absl::string_view node_name,
-                                const TensorId& fanin) {
-  if (!IsTensorIdPortValid(fanin)) {
+bool MutableGraphView::AddRegularFanin(absl::string_view node_name,
+                                       const TensorId& fanin) {
+  if (!IsTensorIdRegular(fanin)) {
     return false;
   }
   NodeDef* node = GetNode(node_name);
@@ -339,60 +356,153 @@ bool MutableGraphView::AddFanin(absl::string_view node_name,
   return AddFaninInternal(node, fanin);
 }
 
-bool MutableGraphView::RemoveFanins(NodeDef* node,
-                                    absl::Span<const TensorId> fanins) {
-  bool modified = false;
-  auto mutable_inputs = node->mutable_input();
-  int curr_pos = 0;
-  int num_inputs = node->input_size();
-  for (int i = 0; i < num_inputs; ++i) {
-    TensorId tensor_id = ParseTensorName(node->input(i));
-    bool remove_fanin =
-        std::find(fanins.begin(), fanins.end(), tensor_id) != fanins.end();
-    bool update_fanin = !remove_fanin && modified;
-    if (remove_fanin || update_fanin) {
-      OutputPort fanin(nodes()[tensor_id.node()], tensor_id.index());
-
-      InputPort input;
-      input.node = node;
-      input.port_id =
-          tensor_id.index() == Graph::kControlSlot ? Graph::kControlSlot : i;
-
-      auto& fanouts_set = fanouts()[fanin];
-      if (remove_fanin) {
-        fanouts_set.erase(input);
-      } else {
-        // Shift inputs to be retained.
-        if (tensor_id.index() > Graph::kControlSlot) {
-          fanouts_set.erase(input);
-          fanouts_set.insert(InputPort(node, i));
+bool MutableGraphView::AddControllingFanin(absl::string_view node_name,
+                                           const TensorId& fanin) {
+  NodeDef* node = GetNode(node_name);
+  if (node == nullptr) {
+    return false;
+  }
+  NodeDef* fanin_node = GetNode(fanin.node());
+  if (fanin_node == nullptr) {
+    return false;
+  }
+
+  if (!IsSwitch(*fanin_node)) {
+    return AddFaninInternal(node, {fanin_node, Graph::kControlSlot});
+  } else {
+    if (IsTensorIdControlling(fanin)) {
+      // Cannot add a Switch node control dependency.
+      return false;
+    }
+    // We can't anchor control dependencies directly on the switch node: unlike
+    // other nodes only one of the outputs of the switch node will be generated
+    // when the switch node is executed, and we need to make sure the control
+    // dependency is only triggered when the corresponding output is triggered.
+    // We start by looking for an identity node connected to the output of the
+    // switch node, and use it to anchor the control dependency.
+    auto fanouts = GetFanouts(*fanin_node, /*include_controlled_nodes=*/false);
+    for (auto fanout : fanouts) {
+      if (IsIdentity(*fanout.node) || IsIdentityNSingleInput(*fanout.node)) {
+        if (ParseTensorName(fanout.node->input(0)) == fanin) {
+          return AddFaninInternal(node, {fanout.node, Graph::kControlSlot});
         }
-        mutable_inputs->SwapElements(i, curr_pos++);
       }
-      UpdateMaxRegularOutputPortForRemovedFanin(fanin, fanouts_set);
+    }
+    // We haven't found an existing node where we can anchor the control
+    // dependency: add a new identity node.
+    string ctrl_dep_name = AddPrefixToNodeName(
+        absl::StrCat(fanin.node(), "_", fanin.index()), kMutableGraphViewCtrl);
 
+    // Reuse a previously created node, if possible.
+    NodeDef* ctrl_dep_node = GetNode(ctrl_dep_name);
+    if (ctrl_dep_node == nullptr) {
+      NodeDef new_node;
+      new_node.set_name(ctrl_dep_name);
+      new_node.set_op("Identity");
+      new_node.set_device(fanin_node->device());
+      (*new_node.mutable_attr())["T"].set_type(
+          fanin_node->attr().at("T").type());
+      new_node.add_input(TensorIdToString(fanin));
+      ctrl_dep_node = AddNode(std::move(new_node));
+    }
+    return AddFaninInternal(node, {ctrl_dep_node, Graph::kControlSlot});
+  }
+}
+
+bool MutableGraphView::RemoveRegularFaninInternal(NodeDef* node,
+                                                  const TensorId& fanin) {
+  auto remove_input = [this, node](const TensorId& tensor_id, int port,
+                                   bool update_max_port) {
+    OutputPort fanin_port(nodes()[tensor_id.node()], tensor_id.index());
+    InputPort input(node, port);
+
+    absl::flat_hash_set<InputPort>* fanouts_set = &fanouts()[fanin_port];
+    fanouts_set->erase(input);
+    if (update_max_port) {
+      UpdateMaxRegularOutputPortForRemovedFanin(fanin_port, *fanouts_set);
+    }
+    return fanouts_set;
+  };
+
+  auto mutable_inputs = node->mutable_input();
+  bool modified = false;
+  const int num_inputs = node->input_size();
+  int i;
+  int curr_pos = 0;
+  for (i = 0; i < num_inputs; ++i) {
+    TensorId tensor_id = ParseTensorName(node->input(i));
+    if (IsTensorIdControlling(tensor_id)) {
+      break;
+    }
+    if (tensor_id == fanin) {
+      remove_input(tensor_id, i, /*update_max_port=*/true);
       modified = true;
+    } else if (modified) {
+      // Regular inputs will need to have their ports updated.
+      auto fanouts_set = remove_input(tensor_id, i, /*update_max_port=*/false);
+      fanouts_set->insert({node, curr_pos});
+      // Shift inputs to be retained.
+      mutable_inputs->SwapElements(i, curr_pos++);
     } else {
       // Skip inputs to be retained until first modification.
       curr_pos++;
     }
   }
-  if (modified) {
-    mutable_inputs->DeleteSubrange(curr_pos, num_inputs - curr_pos);
+
+  if (modified && curr_pos < i) {
+    // Remove fanins from node inputs.
+    mutable_inputs->DeleteSubrange(curr_pos, i - curr_pos);
   }
+
   return modified;
 }
 
-bool MutableGraphView::RemoveFanin(absl::string_view node_name,
-                                   const TensorId& fanin) {
-  if (!IsTensorIdPortValid(fanin)) {
+bool MutableGraphView::RemoveRegularFanin(absl::string_view node_name,
+                                          const TensorId& fanin) {
+  if (!IsTensorIdRegular(fanin)) {
     return false;
   }
   NodeDef* node = GetNode(node_name);
   if (node == nullptr) {
     return false;
   }
-  return RemoveFanins(node, {fanin});
+  return RemoveRegularFaninInternal(node, fanin);
+}
+
+bool MutableGraphView::RemoveControllingFaninInternal(NodeDef* node,
+                                                      NodeDef* fanin_node) {
+  for (int i = node->input_size() - 1; i >= 0; --i) {
+    TensorId tensor_id = ParseTensorName(node->input(i));
+    if (tensor_id.index() > Graph::kControlSlot) {
+      break;
+    }
+    if (tensor_id.node() == fanin_node->name()) {
+      fanouts()[{fanin_node, Graph::kControlSlot}].erase(
+          {node, Graph::kControlSlot});
+      node->mutable_input()->SwapElements(i, node->input_size() - 1);
+      node->mutable_input()->RemoveLast();
+      return true;
+    }
+  }
+  return false;
+}
+
+bool MutableGraphView::RemoveControllingFaninInternal(
+    NodeDef* node, absl::string_view fanin_node_name) {
+  NodeDef* fanin = GetNode(fanin_node_name);
+  if (fanin == nullptr) {
+    return false;
+  }
+  return RemoveControllingFaninInternal(node, fanin);
+}
+
+bool MutableGraphView::RemoveControllingFanin(
+    absl::string_view node_name, absl::string_view fanin_node_name) {
+  NodeDef* node = GetNode(node_name);
+  if (node == nullptr) {
+    return false;
+  }
+  return RemoveControllingFaninInternal(node, fanin_node_name);
 }
 
 bool MutableGraphView::RemoveAllFanins(absl::string_view node_name,
@@ -432,10 +542,18 @@ bool MutableGraphView::UpdateFanin(absl::string_view node_name,
 
   // When replacing a non control dependency fanin with a control dependency, or
   // vice versa, remove and add, so ports can be updated properly in fanout(s).
-  if (from_fanin.index() == Graph::kControlSlot ||
-      to_fanin.index() == Graph::kControlSlot) {
-    bool modified = RemoveFanins(node, {from_fanin});
-    modified |= AddFaninInternal(node, to_fanin);
+  bool from_fanin_is_control = IsTensorIdControlling(from_fanin);
+  if (from_fanin_is_control || IsTensorIdControlling(to_fanin)) {
+    bool modified = false;
+    if (from_fanin_is_control) {
+      modified |= RemoveControllingFaninInternal(node, from_fanin.node());
+    } else {
+      modified |= RemoveRegularFaninInternal(node, from_fanin);
+    }
+    if (modified) {
+      AddFaninInternal(node, to_fanin);
+    }
+
     return modified;
   }
 
@@ -456,7 +574,7 @@ bool MutableGraphView::UpdateFanin(absl::string_view node_name,
       InputPort old_input;
       old_input.node = node;
       old_input.port_id =
-          from_fanin.index() == Graph::kControlSlot ? Graph::kControlSlot : i;
+          IsTensorIdControlling(from_fanin) ? Graph::kControlSlot : i;
       if (from_fanin_port_fanouts == nullptr) {
         OutputPort from_fanin_port(from_fanin_node, from_fanin.index());
         from_fanin_port_fanouts = &fanouts()[from_fanin_port];
@@ -466,7 +584,7 @@ bool MutableGraphView::UpdateFanin(absl::string_view node_name,
       InputPort new_input;
       new_input.node = node;
       new_input.port_id =
-          to_fanin.index() == Graph::kControlSlot ? Graph::kControlSlot : i;
+          IsTensorIdControlling(to_fanin) ? Graph::kControlSlot : i;
       if (to_fanin_port_fanouts == nullptr) {
         OutputPort to_fanin_port(to_fanin_node, to_fanin.index());
         to_fanin_port_fanouts = &fanouts()[to_fanin_port];
@@ -493,75 +611,6 @@ bool MutableGraphView::UpdateFanin(absl::string_view node_name,
   return modified;
 }
 
-bool MutableGraphView::AddControllingFanin(absl::string_view node_name,
-                                           const TensorId& fanin) {
-  NodeDef* node = GetNode(node_name);
-  if (node == nullptr) {
-    return false;
-  }
-  NodeDef* fanin_node = GetNode(fanin.node());
-  if (fanin_node == nullptr) {
-    return false;
-  }
-  if (fanin.index() == Graph::kControlSlot) {
-    return AddFaninInternal(node, {fanin_node, Graph::kControlSlot});
-  }
-
-  if (!IsSwitch(*fanin_node)) {
-    return AddFaninInternal(node, {fanin_node, Graph::kControlSlot});
-  } else {
-    // We can't anchor control dependencies directly on the switch node: unlike
-    // other nodes only one of the outputs of the switch node will be generated
-    // when the switch node is executed, and we need to make sure the control
-    // dependency is only triggered when the corresponding output is triggered.
-    // We start by looking for an identity node connected to the output of the
-    // switch node, and use it to anchor the control dependency.
-    auto fanouts = GetFanouts(*fanin_node, /*include_controlled_nodes=*/false);
-    for (auto fanout : fanouts) {
-      if (IsIdentity(*fanout.node) || IsIdentityNSingleInput(*fanout.node)) {
-        if (ParseTensorName(fanout.node->input(0)) == fanin) {
-          return AddFaninInternal(node, {fanout.node, Graph::kControlSlot});
-        }
-      }
-    }
-    // We haven't found an existing node where we can anchor the control
-    // dependency: add a new identity node.
-    string ctrl_dep_name = AddPrefixToNodeName(
-        absl::StrCat(fanin.node(), "_", fanin.index()), kMutableGraphViewCtrl);
-
-    NodeDef* ctrl_dep_node = GetNode(ctrl_dep_name);
-    if (ctrl_dep_node == nullptr) {
-      NodeDef new_node;
-      new_node.set_name(ctrl_dep_name);
-      new_node.set_op("Identity");
-      new_node.set_device(fanin_node->device());
-      (*new_node.mutable_attr())["T"].set_type(
-          fanin_node->attr().at("T").type());
-      new_node.add_input(TensorIdToString(fanin));
-      ctrl_dep_node = AddNode(std::move(new_node));
-    }
-    return AddFaninInternal(node, {ctrl_dep_node, Graph::kControlSlot});
-  }
-}
-
-bool MutableGraphView::RemoveControllingFaninInternal(NodeDef* node,
-                                                      NodeDef* fanin_node) {
-  for (int i = node->input_size() - 1; i >= 0; --i) {
-    TensorId tensor_id = ParseTensorName(node->input(i));
-    if (tensor_id.index() > Graph::kControlSlot) {
-      break;
-    }
-    if (tensor_id.node() == fanin_node->name()) {
-      fanouts()[{fanin_node, Graph::kControlSlot}].erase(
-          {node, Graph::kControlSlot});
-      node->mutable_input()->SwapElements(i, node->input_size() - 1);
-      node->mutable_input()->RemoveLast();
-      return true;
-    }
-  }
-  return false;
-}
-
 void MutableGraphView::DeleteNodes(const std::set<string>& nodes_to_delete) {
   for (const string& node_name_to_delete : nodes_to_delete)
     RemoveFaninsInternal(nodes().at(node_name_to_delete),
@@ -575,21 +624,18 @@ void MutableGraphView::RemoveFaninsInternal(NodeDef* deleted_node,
                                             bool keep_controlling_fanins) {
   for (int i = 0; i < deleted_node->input_size(); ++i) {
     TensorId tensor_id = ParseTensorName(deleted_node->input(i));
-    if (keep_controlling_fanins && tensor_id.index() < 0) {
+    if (keep_controlling_fanins && IsTensorIdControlling(tensor_id)) {
       break;
     }
     OutputPort fanin(nodes()[tensor_id.node()], tensor_id.index());
 
     InputPort input;
     input.node = deleted_node;
-    if (tensor_id.index() < 0)
-      input.port_id = Graph::kControlSlot;
-    else
-      input.port_id = i;
-
-    auto& fanouts_set = fanouts()[fanin];
-    fanouts_set.erase(input);
-    UpdateMaxRegularOutputPortForRemovedFanin(fanin, fanouts_set);
+    input.port_id = IsTensorIdControlling(tensor_id) ? Graph::kControlSlot : i;
+
+    absl::flat_hash_set<InputPort>* fanouts_set = &fanouts()[fanin];
+    fanouts_set->erase(input);
+    UpdateMaxRegularOutputPortForRemovedFanin(fanin, *fanouts_set);
   }
 }
 
diff --git a/tensorflow/core/grappler/mutable_graph_view.h b/tensorflow/core/grappler/mutable_graph_view.h
index 93f0958d5e..f07254068d 100644
--- a/tensorflow/core/grappler/mutable_graph_view.h
+++ b/tensorflow/core/grappler/mutable_graph_view.h
@@ -78,39 +78,17 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   // This will return true iff the nodes are modified.
   bool UpdateFanouts(absl::string_view from_node, absl::string_view to_node);
 
-  // Adds fanin to node `node_name`. If the node or fanin do not exist in the
-  // graph, nothing will be modified in the graph. If fanin is a control
-  // dependency, existing control dependencies will be checked first before
-  // adding. Otherwise fanin will be added after existing non control dependency
-  // inputs.
-  //
-  // This will return true iff the node is modified. If a control dependency
-  // already exists, the node will not be modified.
-  bool AddFanin(absl::string_view node_name, const TensorId& fanin);
-
-  // Removes fanin from node `node_name`. If the node or fanin do not exist in
-  // the graph, nothing will be modified in the graph. If there are multiple
-  // inputs that match the fanin, all of them will be removed.
-  //
-  // This will return true iff the node is modified. If no inputs match the
-  // fanin, the node will not be modified.
-  bool RemoveFanin(absl::string_view node_name, const TensorId& fanin);
-
-  // Removes all fanins from node `node_name`. Control dependencies will be
-  // retained if keep_controlling_fanins is true.
-  //
-  // This will return true iff the node is modified.
-  bool RemoveAllFanins(absl::string_view node_name,
-                       bool keep_controlling_fanins);
-
-  // Replaces all fanins `from_fanin` with `to_fanin` in node `node_name`. If
-  // the fanins or node do not exist, nothing will be modified in the graph.
+  // Adds regular fanin `fanin` to node `node_name`. If the node or fanin do not
+  // exist in the graph, nothing will be modified in the graph. Otherwise fanin
+  // will be added after existing non control dependency fanins. Control
+  // dependencies will be deduped. To add control dependencies, use
+  // AddControllingFanin.
   //
   // This will return true iff the node is modified.
-  bool UpdateFanin(absl::string_view node_name, const TensorId& from_fanin,
-                   const TensorId& to_fanin);
+  bool AddRegularFanin(absl::string_view node_name, const TensorId& fanin);
 
-  // Adds a control dependency to the target node named `node_name`.
+  // Adds control dependency `fanin` to the target node named `node_name`. To
+  // add regular fanins, use AddRegularFanin.
   //
   // Case 1: If the fanin is not a Switch node, the control dependency is simply
   // added to the target node:
@@ -130,6 +108,37 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   // This will return true iff the node is modified.
   bool AddControllingFanin(absl::string_view node_name, const TensorId& fanin);
 
+  // Removes regular fanin `fanin` from node `node_name`. If the node or fanin
+  // do not exist in the graph, nothing will be modified in the graph. If there
+  // are multiple inputs that match the fanin, all of them will be removed. To
+  // remove controlling fanins, use RemoveControllingFanin.
+  //
+  // This will return true iff the node is modified.
+  bool RemoveRegularFanin(absl::string_view node_name, const TensorId& fanin);
+
+  // Removes control dependency `fanin_node_name` from the target node named
+  // `node_name`. If the node or fanin do not exist in the graph, nothing will
+  // be modified in the graph. To remove regular fanins, use RemoveRegualrFanin.
+  //
+  // This will return true iff the node is modified.
+  bool RemoveControllingFanin(absl::string_view node_name,
+                              absl::string_view fanin_node_name);
+
+  // Removes all fanins from node `node_name`. Control dependencies will be
+  // retained if keep_controlling_fanins is true.
+  //
+  // This will return true iff the node is modified.
+  bool RemoveAllFanins(absl::string_view node_name,
+                       bool keep_controlling_fanins);
+
+  // Replaces all fanins `from_fanin` with `to_fanin` in node `node_name`. If
+  // the fanins or node do not exist, nothing will be modified in the graph.
+  // Control dependencies will be deduped.
+  //
+  // This will return true iff the node is modified.
+  bool UpdateFanin(absl::string_view node_name, const TensorId& from_fanin,
+                   const TensorId& to_fanin);
+
   // Deletes nodes from the graph.
   void DeleteNodes(const std::set<string>& nodes_to_delete);
 
@@ -189,7 +198,14 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   // Removes any fanin in node that matches to a fanin in fanins.
   //
   // This will return true iff the node is modified.
-  bool RemoveFanins(NodeDef* node, absl::Span<const TensorId> fanins);
+  bool RemoveRegularFaninInternal(NodeDef* node, const TensorId& fanin);
+
+  // Removes controlling fanin `fanin_node_name` from node if such controlling
+  // fanin exists.
+  //
+  // This will return true iff the node is modified.
+  bool RemoveControllingFaninInternal(NodeDef* node,
+                                      absl::string_view fanin_node_name);
 
   // Removes controlling fanin `fanin_node` from node if such controlling fanin
   // exists.
diff --git a/tensorflow/core/grappler/mutable_graph_view_test.cc b/tensorflow/core/grappler/mutable_graph_view_test.cc
index 526ca85ec0..2048c67e3e 100644
--- a/tensorflow/core/grappler/mutable_graph_view_test.cc
+++ b/tensorflow/core/grappler/mutable_graph_view_test.cc
@@ -186,10 +186,10 @@ GraphDef SimpleMutateFaninGraph() {
 void CompareNodeInputs(const MutableGraphView& graph, const NodeDef* expected,
                        NodeDef* actual) {
   ASSERT_EQ(actual->input_size(), expected->input_size());
-  int port;
   for (int i = 0; i < actual->input_size(); ++i) {
     EXPECT_EQ(actual->input(i), expected->input(i));
     TensorId tensor_id = ParseTensorName(expected->input(i));
+    int port;
     if (tensor_id.index() == Graph::kControlSlot) {
       port = Graph::kControlSlot;
     } else {
@@ -203,8 +203,9 @@ void CompareNodeInputs(const MutableGraphView& graph, const NodeDef* expected,
   }
 }
 
-void TestAddFanin(absl::string_view node_name, const TensorId& fanin_to_add,
-                  bool modified, const NodeDef* expected_node) {
+void TestAddRegularFanin(absl::string_view node_name,
+                         const TensorId& fanin_to_add, bool modified,
+                         const NodeDef* expected_node) {
   GraphDef graph_def = SimpleMutateFaninGraph();
 
   MutableGraphView graph(&graph_def);
@@ -216,70 +217,71 @@ void TestAddFanin(absl::string_view node_name, const TensorId& fanin_to_add,
     EXPECT_NE(node, nullptr);
   }
 
-  EXPECT_EQ(modified, graph.AddFanin(node_name, fanin_to_add));
+  EXPECT_EQ(modified, graph.AddRegularFanin(node_name, fanin_to_add));
   if (expected_node != nullptr) {
     CompareNodeInputs(graph, expected_node, node);
   }
 }
 
-TEST(MutableGraphViewTest, AddFanin) {
+TEST(MutableGraphViewTest, AddRegularFanin) {
   NodeDef expected_node;
   // Add input to node with 1 input 0 controls.
   expected_node = NDef("", "", {"a", "b:1"});
-  TestAddFanin("foo_1", {"b", 1}, /*modified=*/true, &expected_node);
+  TestAddRegularFanin("foo_1", {"b", 1}, /*modified=*/true, &expected_node);
   // Add input to node with multiple inputs and 0 controls.
   expected_node = NDef("", "", {"b", "a:1", "a:1", "b:2"});
-  TestAddFanin("foo_3", {"b", 2}, /*modified=*/true, &expected_node);
+  TestAddRegularFanin("foo_3", {"b", 2}, /*modified=*/true, &expected_node);
   // Add input to node with 1 input multiple controls.
   expected_node = NDef("", "", {"b", "a", "^c"});
-  TestAddFanin("foo_2", {"a", 0}, /*modified=*/true, &expected_node);
+  TestAddRegularFanin("foo_2", {"a", 0}, /*modified=*/true, &expected_node);
   // Add input to node with multiple inputs and controls.
   expected_node = NDef("", "", {"a", "b:2", "b:2", "a:1", "^d", "^c"});
-  TestAddFanin("foo_4", {"a", 1}, /*modified=*/true, &expected_node);
+  TestAddRegularFanin("foo_4", {"a", 1}, /*modified=*/true, &expected_node);
   // Add input to node with 0 inputs 0 controls.
   expected_node = NDef("", "", {"a:1"});
-  TestAddFanin("foo_5", {"a", 1}, /*modified=*/true, &expected_node);
+  TestAddRegularFanin("foo_5", {"a", 1}, /*modified=*/true, &expected_node);
   // Add input to node with 0 inputs multiple controls.
   expected_node = NDef("", "", {"c:1", "^b", "^a"});
-  TestAddFanin("foo_6", {"c", 1}, /*modified=*/true, &expected_node);
+  TestAddRegularFanin("foo_6", {"c", 1}, /*modified=*/true, &expected_node);
 
   // Add control to node with 1 input 0 controls.
-  expected_node = NDef("", "", {"a", "^b"});
-  TestAddFanin("foo_1", {"b", Graph::kControlSlot}, /*modified=*/true,
-               &expected_node);
+  expected_node = NDef("", "", {"a"});
+  TestAddRegularFanin("foo_1", {"b", Graph::kControlSlot}, /*modified=*/false,
+                      &expected_node);
   // Add control to node with multiple inputs and 0 controls.
-  expected_node = NDef("", "", {"b", "a:1", "a:1", "^c"});
-  TestAddFanin("foo_3", {"c", Graph::kControlSlot}, /*modified=*/true,
-               &expected_node);
+  expected_node = NDef("", "", {"b", "a:1", "a:1"});
+  TestAddRegularFanin("foo_3", {"c", Graph::kControlSlot}, /*modified=*/false,
+                      &expected_node);
   // Add control to node with 1 input multiple controls.
-  expected_node = NDef("", "", {"b", "^a", "^c", "^d"});
-  TestAddFanin("foo_2", {"d", Graph::kControlSlot}, /*modified=*/true,
-               &expected_node);
+  expected_node = NDef("", "", {"b", "^a", "^c"});
+  TestAddRegularFanin("foo_2", {"d", Graph::kControlSlot}, /*modified=*/false,
+                      &expected_node);
   // Add control to node with multiple input multiple controls.
   expected_node = NDef("", "", {"a", "b:2", "b:2", "^c", "^d"});
-  TestAddFanin("foo_4", {"a", Graph::kControlSlot}, /*modified=*/false,
-               &expected_node);
+  TestAddRegularFanin("foo_4", {"a", Graph::kControlSlot},
+                      /*modified=*/false, &expected_node);
   // Add control to node with 0 inputs 0 controls.
-  expected_node = NDef("", "", {"^a"});
-  TestAddFanin("foo_5", {"a", Graph::kControlSlot}, /*modified=*/true,
-               &expected_node);
+  expected_node = NDef("", "", {});
+  TestAddRegularFanin("foo_5", {"a", Graph::kControlSlot}, /*modified=*/false,
+                      &expected_node);
   // Add control to node with 0 inputs multiple controls.
-  expected_node = NDef("", "", {"^a", "^b", "^c"});
-  TestAddFanin("foo_6", {"c", Graph::kControlSlot}, /*modified=*/true,
-               &expected_node);
+  expected_node = NDef("", "", {"^a", "^b"});
+  TestAddRegularFanin("foo_6", {"c", Graph::kControlSlot}, /*modified=*/false,
+                      &expected_node);
   // Add control to node with control that already exists.
   expected_node = NDef("", "", {"b", "^a", "^c"});
-  TestAddFanin("foo_2", {"a", Graph::kControlSlot}, /*modified=*/false,
-               &expected_node);
+  TestAddRegularFanin("foo_2", {"a", Graph::kControlSlot},
+                      /*modified=*/false, &expected_node);
 
   // Add fanin to node where node is missing.
-  TestAddFanin("foo_missing", {"a", 0}, /*modified=*/false, nullptr);
+  TestAddRegularFanin("foo_missing", {"a", 0}, /*modified=*/false, nullptr);
   // Add fanin to node where fanin is missing.
   expected_node = NDef("", "", {"a"});
-  TestAddFanin("foo_1", {"bar_missing", 0}, /*modified=*/false, &expected_node);
+  TestAddRegularFanin("foo_1", {"bar_missing", 0}, /*modified=*/false,
+                      &expected_node);
   // Add fanin to node where node and fanin are missing.
-  TestAddFanin("foo_missing", {"bar_missing", 0}, /*modified=*/false,
-               /*expected_node=*/nullptr);
+  TestAddRegularFanin("foo_missing", {"bar_missing", 0}, /*modified=*/false,
+                      /*expected_node=*/nullptr);
 }
 
 void CheckFanout(const MutableGraphView& graph, const TensorId& fanin,
@@ -292,9 +294,9 @@ void CheckFanout(const MutableGraphView& graph, const TensorId& fanin,
   }
 }
 
-void TestRemoveFanin(absl::string_view node_name,
-                     const TensorId& fanin_to_remove, bool modified,
-                     const NodeDef* expected_node) {
+void TestRemoveRegularFanin(absl::string_view node_name,
+                            const TensorId& fanin_to_remove, bool modified,
+                            const NodeDef* expected_node) {
   GraphDef graph_def = SimpleMutateFaninGraph();
 
   MutableGraphView graph(&graph_def);
@@ -306,7 +308,7 @@ void TestRemoveFanin(absl::string_view node_name,
     EXPECT_NE(nullptr, node);
   }
 
-  EXPECT_EQ(modified, graph.RemoveFanin(node_name, fanin_to_remove));
+  EXPECT_EQ(modified, graph.RemoveRegularFanin(node_name, fanin_to_remove));
   if (expected_node != nullptr) {
     CompareNodeInputs(graph, expected_node, node);
     if (modified) {
@@ -315,63 +317,68 @@ void TestRemoveFanin(absl::string_view node_name,
   }
 }
 
-TEST(MutableGraphViewTest, RemoveFanin) {
+TEST(MutableGraphViewTest, RemoveRegularFanin) {
   NodeDef expected_node;
   // Remove input from node with 1 input 0 controls.
   expected_node = NDef("", "", {});
-  TestRemoveFanin("foo_1", {"a", 0}, /*modified=*/true, &expected_node);
+  TestRemoveRegularFanin("foo_1", {"a", 0}, /*modified=*/true, &expected_node);
   // Remove input from node with multiple inputs and 0 controls.
   expected_node = NDef("", "", {"b"});
-  TestRemoveFanin("foo_3", {"a", 1}, /*modified=*/true, &expected_node);
+  TestRemoveRegularFanin("foo_3", {"a", 1}, /*modified=*/true, &expected_node);
   // Remove input from node with 1 input multiple controls.
   expected_node = NDef("", "", {"^a", "^c"});
-  TestRemoveFanin("foo_2", {"b", 0}, /*modified=*/true, &expected_node);
+  TestRemoveRegularFanin("foo_2", {"b", 0}, /*modified=*/true, &expected_node);
   // Remove input from node with multiple inputs and controls.
   expected_node = NDef("", "", {"a", "^c", "^d"});
-  TestRemoveFanin("foo_4", {"b", 2}, /*modified=*/true, &expected_node);
+  TestRemoveRegularFanin("foo_4", {"b", 2}, /*modified=*/true, &expected_node);
+  // Remove input from node with multiple inputs and controls, and results in
+  // shifting of ports.
+  expected_node = NDef("", "", {"b:2", "b:2", "^c", "^d"});
+  TestRemoveRegularFanin("foo_4", {"a", 0}, /*modified=*/true, &expected_node);
 
   // Remove control from node with 1 input multiple controls.
-  expected_node = NDef("", "", {"b", "^c"});
-  TestRemoveFanin("foo_2", {"a", Graph::kControlSlot}, /*modified=*/true,
-                  &expected_node);
+  expected_node = NDef("", "", {"b", "^a", "^c"});
+  TestRemoveRegularFanin("foo_2", {"a", Graph::kControlSlot},
+                         /*modified=*/false, &expected_node);
   // Remove control from node with multiple input multiple controls.
-  expected_node = NDef("", "", {"a", "b:2", "b:2", "^c"});
-  TestRemoveFanin("foo_4", {"d", Graph::kControlSlot}, /*modified=*/true,
-                  &expected_node);
+  expected_node = NDef("", "", {"a", "b:2", "b:2", "^c", "^d"});
+  TestRemoveRegularFanin("foo_4", {"d", Graph::kControlSlot},
+                         /*modified=*/false, &expected_node);
   // Remove control from node with 0 inputs multiple controls.
-  expected_node = NDef("", "", {"^b"});
-  TestRemoveFanin("foo_6", {"a", Graph::kControlSlot}, /*modified=*/true,
-                  &expected_node);
+  expected_node = NDef("", "", {"^a", "^b"});
+  TestRemoveRegularFanin("foo_6", {"a", Graph::kControlSlot},
+                         /*modified=*/false, &expected_node);
 
   // Remove input from node with 0 inputs 0 controls.
   expected_node = NDef("", "", {});
-  TestRemoveFanin("foo_5", {"a", 1}, /*modified=*/false, &expected_node);
+  TestRemoveRegularFanin("foo_5", {"a", 1}, /*modified=*/false, &expected_node);
   // Remove input from node with 0 inputs multiple controls.
   expected_node = NDef("", "", {"^a", "^b"});
-  TestRemoveFanin("foo_6", {"a", 1}, /*modified=*/false, &expected_node);
+  TestRemoveRegularFanin("foo_6", {"a", 1}, /*modified=*/false, &expected_node);
+
   // Remove control from node with 1 input 0 controls.
   expected_node = NDef("", "", {"a"});
-  TestRemoveFanin("foo_1", {"b", Graph::kControlSlot}, /*modified=*/false,
-                  &expected_node);
+  TestRemoveRegularFanin("foo_1", {"b", Graph::kControlSlot},
+                         /*modified=*/false, &expected_node);
   // Remove control from node with multiple inputs and 0 controls.
   expected_node = NDef("", "", {"b", "a:1", "a:1"});
-  TestRemoveFanin("foo_3", {"c", Graph::kControlSlot}, /*modified=*/false,
-                  &expected_node);
+  TestRemoveRegularFanin("foo_3", {"c", Graph::kControlSlot},
+                         /*modified=*/false, &expected_node);
   // Remove control from node with 0 inputs 0 controls.
   expected_node = NDef("", "", {});
-  TestRemoveFanin("foo_5", {"a", Graph::kControlSlot}, /*modified=*/false,
-                  &expected_node);
+  TestRemoveRegularFanin("foo_5", {"a", Graph::kControlSlot},
+                         /*modified=*/false, &expected_node);
 
   // Remove fanin from node where node is missing.
-  TestRemoveFanin("foo_missing", {"a", 0}, /*modified=*/false,
-                  /*expected_node=*/nullptr);
+  TestRemoveRegularFanin("foo_missing", {"a", 0}, /*modified=*/false,
+                         /*expected_node=*/nullptr);
   // Remove fanin from node where fanin is missing.
   expected_node = NDef("", "", {"a"});
-  TestRemoveFanin("foo_1", {"bar_missing", 0}, /*modified=*/false,
-                  &expected_node);
+  TestRemoveRegularFanin("foo_1", {"bar_missing", 0}, /*modified=*/false,
+                         &expected_node);
   // Remove fanin from node where node and fanin are missing.
-  TestRemoveFanin("foo_missing", {"bar_missing", 0}, /*modified=*/false,
-                  /*expected_node=*/nullptr);
+  TestRemoveRegularFanin("foo_missing", {"bar_missing", 0}, /*modified=*/false,
+                         /*expected_node=*/nullptr);
 }
 
 void TestRemoveAllFanins(absl::string_view node_name,
@@ -540,56 +547,59 @@ TEST(MutableGraphViewTest, DedupControllingFaninsOnGraphInit) {
   MutableGraphView graph(&graph_def);
 
   EXPECT_EQ(graph.graph()->node_size(), 11);
+  NodeDef expected;
   NodeDef* a = graph.GetNode("a");
   ASSERT_NE(a, nullptr);
-  ASSERT_EQ(a->input_size(), 0);
+  expected = NDef("", "", {});
+  CompareNodeInputs(graph, &expected, a);
+
   NodeDef* b = graph.GetNode("b");
   ASSERT_NE(b, nullptr);
-  ASSERT_EQ(b->input_size(), 0);
+  CompareNodeInputs(graph, &expected, b);
+
   NodeDef* c = graph.GetNode("c");
   ASSERT_NE(c, nullptr);
-  ASSERT_EQ(c->input_size(), 0);
+  CompareNodeInputs(graph, &expected, c);
+
   NodeDef* d = graph.GetNode("d");
   ASSERT_NE(d, nullptr);
-  ASSERT_EQ(d->input_size(), 1);
-  EXPECT_EQ(d->input(0), "c:1");
+  expected = NDef("", "", {"c:1"});
+  CompareNodeInputs(graph, &expected, d);
+
   NodeDef* foo_1 = graph.GetNode("foo_1");
   ASSERT_NE(foo_1, nullptr);
-  ASSERT_EQ(foo_1->input_size(), 2);
-  EXPECT_EQ(foo_1->input(0), "a");
-  EXPECT_EQ(foo_1->input(1), "b:1");
+  expected = NDef("", "", {"a", "b:1"});
+  CompareNodeInputs(graph, &expected, foo_1);
+
   NodeDef* foo_2 = graph.GetNode("foo_2");
   ASSERT_NE(foo_2, nullptr);
-  ASSERT_EQ(foo_2->input_size(), 2);
-  EXPECT_EQ(foo_2->input(0), "a");
-  EXPECT_EQ(foo_2->input(1), "^b");
+  expected = NDef("", "", {"a", "^b"});
+  CompareNodeInputs(graph, &expected, foo_2);
+
   NodeDef* foo_3 = graph.GetNode("foo_3");
   ASSERT_NE(foo_3, nullptr);
-  ASSERT_EQ(foo_3->input_size(), 2);
-  EXPECT_EQ(foo_3->input(0), "a");
-  EXPECT_EQ(foo_3->input(1), "b:1");
+  expected = NDef("", "", {"a", "b:1"});
+  CompareNodeInputs(graph, &expected, foo_3);
+
   NodeDef* foo_4 = graph.GetNode("foo_4");
   ASSERT_NE(foo_4, nullptr);
-  ASSERT_EQ(foo_4->input_size(), 2);
-  EXPECT_EQ(foo_4->input(0), "a:2");
-  EXPECT_EQ(foo_4->input(1), "b:1");
+  expected = NDef("", "", {"a:2", "b:1"});
+  CompareNodeInputs(graph, &expected, foo_4);
+
   NodeDef* foo_5 = graph.GetNode("foo_5");
   ASSERT_NE(foo_5, nullptr);
-  ASSERT_EQ(foo_5->input_size(), 2);
-  EXPECT_EQ(foo_5->input(0), "a:2");
-  EXPECT_EQ(foo_5->input(1), "b:1");
+  expected = NDef("", "", {"a:2", "b:1"});
+  CompareNodeInputs(graph, &expected, foo_5);
+
   NodeDef* foo_6 = graph.GetNode("foo_6");
   ASSERT_NE(foo_6, nullptr);
-  ASSERT_EQ(foo_6->input_size(), 2);
-  EXPECT_EQ(foo_6->input(0), "d");
-  EXPECT_EQ(foo_6->input(1), "^d");
+  expected = NDef("", "", {"d", "^d"});
+  CompareNodeInputs(graph, &expected, foo_6);
+
   NodeDef* foo_7 = graph.GetNode("foo_7");
   ASSERT_NE(foo_7, nullptr);
-  ASSERT_EQ(foo_7->input_size(), 4);
-  EXPECT_EQ(foo_7->input(0), "a:3");
-  EXPECT_EQ(foo_7->input(1), "b:2");
-  EXPECT_EQ(foo_7->input(2), "d");
-  EXPECT_EQ(foo_7->input(3), "^d");
+  expected = NDef("", "", {"a:3", "b:2", "d", "^d"});
+  CompareNodeInputs(graph, &expected, foo_7);
 }
 
 TEST(MutableGraphViewTest, DedupControllingFaninsOnAddFanin) {
@@ -601,17 +611,18 @@ TEST(MutableGraphViewTest, DedupControllingFaninsOnAddFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.AddFanin("b", {"a", 2}));
+  EXPECT_TRUE(graph.AddRegularFanin("b", {"a", 2}));
+  NodeDef expected;
   NodeDef* b = graph.GetNode("b");
   ASSERT_NE(b, nullptr);
-  ASSERT_EQ(b->input_size(), 1);
-  EXPECT_EQ(b->input(0), "a:2");
+  expected = NDef("", "", {"a:2"});
+  CompareNodeInputs(graph, &expected, b);
 
-  EXPECT_FALSE(graph.AddFanin("c", {"a", Graph::kControlSlot}));
+  EXPECT_FALSE(graph.AddControllingFanin("c", {"a", Graph::kControlSlot}));
   NodeDef* c = graph.GetNode("c");
   ASSERT_NE(c, nullptr);
-  ASSERT_EQ(c->input_size(), 1);
-  EXPECT_EQ(c->input(0), "a:1");
+  expected = NDef("", "", {"a:1"});
+  CompareNodeInputs(graph, &expected, c);
 }
 
 TEST(MutableGraphViewTest, NoDedupControlFlowControllingFaninsOnAddFanin) {
@@ -622,32 +633,30 @@ TEST(MutableGraphViewTest, NoDedupControlFlowControllingFaninsOnAddFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.AddFanin("c", {"b", 2}));
+  EXPECT_TRUE(graph.AddRegularFanin("c", {"b", 2}));
+  NodeDef expected;
   NodeDef* c = graph.GetNode("c");
   ASSERT_NE(c, nullptr);
-  ASSERT_EQ(c->input_size(), 1);
-  EXPECT_EQ(c->input(0), "b:2");
-  EXPECT_TRUE(graph.AddFanin("c", {"b", Graph::kControlSlot}));
-  ASSERT_EQ(c->input_size(), 2);
-  EXPECT_EQ(c->input(0), "b:2");
-  EXPECT_EQ(c->input(1), "^b");
-  EXPECT_FALSE(graph.AddFanin("c", {"b", Graph::kControlSlot}));
-  ASSERT_EQ(c->input_size(), 2);
-  EXPECT_EQ(c->input(0), "b:2");
-  EXPECT_EQ(c->input(1), "^b");
-
-  EXPECT_TRUE(graph.AddFanin("d", {"b", Graph::kControlSlot}));
+  expected = NDef("", "", {"b:2"});
+  CompareNodeInputs(graph, &expected, c);
+  EXPECT_TRUE(graph.AddControllingFanin("c", {"b", Graph::kControlSlot}));
+  expected = NDef("", "", {"b:2", "^b"});
+  CompareNodeInputs(graph, &expected, c);
+  EXPECT_FALSE(graph.AddControllingFanin("c", {"b", Graph::kControlSlot}));
+  expected = NDef("", "", {"b:2", "^b"});
+  CompareNodeInputs(graph, &expected, c);
+
+  EXPECT_TRUE(graph.AddControllingFanin("d", {"b", Graph::kControlSlot}));
   NodeDef* d = graph.GetNode("d");
   ASSERT_NE(d, nullptr);
-  ASSERT_EQ(d->input_size(), 1);
-  EXPECT_EQ(d->input(0), "^b");
-  EXPECT_FALSE(graph.AddFanin("d", {"b", Graph::kControlSlot}));
-  ASSERT_EQ(d->input_size(), 1);
-  EXPECT_EQ(d->input(0), "^b");
-  EXPECT_TRUE(graph.AddFanin("d", {"b", 3}));
-  ASSERT_EQ(d->input_size(), 2);
-  EXPECT_EQ(d->input(0), "b:3");
-  EXPECT_EQ(d->input(1), "^b");
+  expected = NDef("", "", {"^b"});
+  CompareNodeInputs(graph, &expected, d);
+  EXPECT_FALSE(graph.AddControllingFanin("d", {"b", Graph::kControlSlot}));
+  expected = NDef("", "", {"^b"});
+  CompareNodeInputs(graph, &expected, d);
+  EXPECT_TRUE(graph.AddRegularFanin("d", {"b", 3}));
+  expected = NDef("", "", {"b:3", "^b"});
+  CompareNodeInputs(graph, &expected, d);
 }
 
 TEST(MutableGraphViewTest, DedupControllingFaninsOnUpdateFanin) {
@@ -662,8 +671,8 @@ TEST(MutableGraphViewTest, DedupControllingFaninsOnUpdateFanin) {
   EXPECT_TRUE(graph.UpdateFanin("c", {"a", 1}, {"b", 2}));
   NodeDef* c = graph.GetNode("c");
   ASSERT_NE(c, nullptr);
-  ASSERT_EQ(c->input_size(), 1);
-  EXPECT_EQ(c->input(0), "b:2");
+  NodeDef expected = NDef("", "", {"b:2"});
+  CompareNodeInputs(graph, &expected, c);
 }
 
 TEST(MutableGraphViewTest, NoDedupControlFlowControllingFaninsOnUpdateFanin) {
@@ -677,23 +686,22 @@ TEST(MutableGraphViewTest, NoDedupControlFlowControllingFaninsOnUpdateFanin) {
 
   EXPECT_TRUE(graph.UpdateFanin("d", {"b", Graph::kControlSlot},
                                 {"c", Graph::kControlSlot}));
+  NodeDef expected;
   NodeDef* d = graph.GetNode("d");
   ASSERT_NE(d, nullptr);
-  ASSERT_EQ(d->input_size(), 2);
-  EXPECT_EQ(d->input(0), "c");
-  EXPECT_EQ(d->input(1), "^c");
+  expected = NDef("", "", {"c", "^c"});
+  CompareNodeInputs(graph, &expected, d);
 
   EXPECT_TRUE(graph.UpdateFanin("e", {"b", 0}, {"c", 3}));
   NodeDef* e = graph.GetNode("e");
   ASSERT_NE(e, nullptr);
-  ASSERT_EQ(e->input_size(), 2);
-  EXPECT_EQ(e->input(0), "c:3");
-  EXPECT_EQ(e->input(1), "^c");
+  expected = NDef("", "", {"c:3", "^c"});
+  CompareNodeInputs(graph, &expected, e);
 
   EXPECT_TRUE(graph.UpdateFanin("e", {"c", 3}, {"c", Graph::kControlSlot}));
   ASSERT_NE(e, nullptr);
-  ASSERT_EQ(e->input_size(), 1);
-  EXPECT_EQ(e->input(0), "^c");
+  expected = NDef("", "", {"^c"});
+  CompareNodeInputs(graph, &expected, e);
 }
 
 TEST(MutableGraphViewTest, UpdateMaxRegularOutputPortOnAddFanin) {
@@ -705,14 +713,17 @@ TEST(MutableGraphViewTest, UpdateMaxRegularOutputPortOnAddFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.AddFanin("c", {"a", 3}));
+  EXPECT_TRUE(graph.AddRegularFanin("c", {"a", 3}));
   NodeDef* a = graph.GetNode("a");
   ASSERT_NE(a, nullptr);
+
   auto fanouts = graph.GetFanouts(*a, /*include_controlled_nodes=*/true);
   EXPECT_EQ(fanouts.size(), 2);
+
   NodeDef* b = graph.GetNode("b");
   ASSERT_NE(b, nullptr);
   EXPECT_EQ(fanouts.count(MutableGraphView::InputPort(b, 0)), 1);
+
   NodeDef* c = graph.GetNode("c");
   ASSERT_NE(c, nullptr);
   EXPECT_EQ(fanouts.count(MutableGraphView::InputPort(c, 0)), 1);
@@ -727,11 +738,13 @@ TEST(MutableGraphViewTest, UpdateMaxRegularOutputPortOnRemoveFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.RemoveFanin("c", {"a", 2}));
+  EXPECT_TRUE(graph.RemoveRegularFanin("c", {"a", 2}));
   NodeDef* a = graph.GetNode("a");
   ASSERT_NE(a, nullptr);
+
   auto fanouts = graph.GetFanouts(*a, /*include_controlled_nodes=*/true);
   EXPECT_EQ(fanouts.size(), 1);
+
   NodeDef* b = graph.GetNode("b");
   ASSERT_NE(b, nullptr);
   EXPECT_EQ(fanouts.count(MutableGraphView::InputPort(b, 0)), 1);
@@ -746,11 +759,13 @@ TEST(MutableGraphViewTest, KeepMaxRegularOutputPortOnRemoveFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.RemoveFanin("b", {"a", 1}));
+  EXPECT_TRUE(graph.RemoveRegularFanin("b", {"a", 1}));
   NodeDef* a = graph.GetNode("a");
   ASSERT_NE(a, nullptr);
+
   auto fanouts = graph.GetFanouts(*a, /*include_controlled_nodes=*/true);
   EXPECT_EQ(fanouts.size(), 1);
+
   NodeDef* c = graph.GetNode("c");
   ASSERT_NE(c, nullptr);
   EXPECT_EQ(fanouts.count(MutableGraphView::InputPort(c, 0)), 1);
@@ -768,13 +783,17 @@ TEST(MutableGraphViewTest, UpdateMaxRegularOutputPortOnUpdateFanin) {
   EXPECT_TRUE(graph.UpdateFanin("c", {"a", 2}, {"b", 3}));
   NodeDef* a = graph.GetNode("a");
   ASSERT_NE(a, nullptr);
+
   auto a_fanouts = graph.GetFanouts(*a, /*include_controlled_nodes=*/true);
   EXPECT_EQ(a_fanouts.size(), 1);
+
   NodeDef* b = graph.GetNode("b");
   ASSERT_NE(b, nullptr);
   EXPECT_EQ(a_fanouts.count(MutableGraphView::InputPort(b, 0)), 1);
+
   auto b_fanouts = graph.GetFanouts(*b, /*include_controlled_nodes=*/true);
   EXPECT_EQ(b_fanouts.size(), 1);
+
   NodeDef* c = graph.GetNode("c");
   ASSERT_NE(c, nullptr);
   EXPECT_EQ(b_fanouts.count(MutableGraphView::InputPort(c, 0)), 1);
@@ -795,12 +814,15 @@ TEST(MutableGraphViewTest, AddControllingFaninMissing) {
   EXPECT_FALSE(graph.AddControllingFanin("c", {"d", Graph::kControlSlot}));
 
   ASSERT_EQ(graph.graph()->node_size(), 2);
+  NodeDef expected;
   NodeDef* a = graph.GetNode("a");
   ASSERT_NE(a, nullptr);
-  ASSERT_EQ(a->input_size(), 0);
+  expected = NDef("", "", {});
+  CompareNodeInputs(graph, &expected, a);
+
   NodeDef* b = graph.GetNode("b");
   ASSERT_NE(b, nullptr);
-  ASSERT_EQ(b->input_size(), 0);
+  CompareNodeInputs(graph, &expected, b);
 }
 
 TEST(MutableGraphViewTest, AddControllingFaninExistingControl) {
@@ -814,13 +836,16 @@ TEST(MutableGraphViewTest, AddControllingFaninExistingControl) {
   EXPECT_FALSE(graph.AddControllingFanin("a", {"b", Graph::kControlSlot}));
 
   ASSERT_EQ(graph.graph()->node_size(), 2);
+  NodeDef expected;
   NodeDef* a = graph.GetNode("a");
   ASSERT_NE(a, nullptr);
-  ASSERT_EQ(a->input_size(), 1);
-  EXPECT_EQ(a->input(0), "^b");
+  expected = NDef("", "", {"^b"});
+  CompareNodeInputs(graph, &expected, a);
+
   NodeDef* b = graph.GetNode("b");
   ASSERT_NE(b, nullptr);
-  ASSERT_EQ(b->input_size(), 0);
+  expected = NDef("", "", {});
+  CompareNodeInputs(graph, &expected, b);
 }
 
 TEST(MutableGraphViewTest, AddControllingFaninNotSwitch) {
@@ -834,13 +859,16 @@ TEST(MutableGraphViewTest, AddControllingFaninNotSwitch) {
   EXPECT_FALSE(graph.AddControllingFanin("a", {"b", 2}));
 
   ASSERT_EQ(graph.graph()->node_size(), 2);
+  NodeDef expected;
   NodeDef* a = graph.GetNode("a");
   ASSERT_NE(a, nullptr);
-  ASSERT_EQ(a->input_size(), 1);
-  EXPECT_EQ(a->input(0), "^b");
+  expected = NDef("", "", {"^b"});
+  CompareNodeInputs(graph, &expected, a);
+
   NodeDef* b = graph.GetNode("b");
   ASSERT_NE(b, nullptr);
-  ASSERT_EQ(b->input_size(), 0);
+  expected = NDef("", "", {});
+  CompareNodeInputs(graph, &expected, b);
 }
 
 TEST(MutableGraphViewTest, AddControllingFaninSwitchWithIdentity) {
@@ -857,8 +885,8 @@ TEST(MutableGraphViewTest, AddControllingFaninSwitchWithIdentity) {
   ASSERT_EQ(graph.graph()->node_size(), 3);
   NodeDef* a = graph.GetNode("a");
   ASSERT_NE(a, nullptr);
-  ASSERT_EQ(a->input_size(), 1);
-  EXPECT_EQ(a->input(0), "^identity");
+  NodeDef expected = NDef("", "", {"^identity"});
+  CompareNodeInputs(graph, &expected, a);
 }
 
 TEST(MutableGraphViewTest, AddControllingFaninSwitchWithNoExistingIdentity) {
@@ -874,14 +902,16 @@ TEST(MutableGraphViewTest, AddControllingFaninSwitchWithNoExistingIdentity) {
   EXPECT_FALSE(graph.AddControllingFanin("a", {"switch", 0}));
 
   ASSERT_EQ(graph.graph()->node_size(), 3);
+  NodeDef expected;
   NodeDef* a = graph.GetNode("a");
   ASSERT_NE(a, nullptr);
-  ASSERT_EQ(a->input_size(), 1);
-  EXPECT_EQ(a->input(0), "^ConstantFoldingCtrl/switch_0");
+  expected = NDef("", "", {"^ConstantFoldingCtrl/switch_0"});
+  CompareNodeInputs(graph, &expected, a);
+
   NodeDef* identity = graph.GetNode("ConstantFoldingCtrl/switch_0");
   ASSERT_NE(identity, nullptr);
-  ASSERT_EQ(identity->input_size(), 1);
-  EXPECT_EQ(identity->input(0), "switch");
+  expected = NDef("", "", {"switch"});
+  CompareNodeInputs(graph, &expected, identity);
   EXPECT_EQ(identity->op(), "Identity");
   EXPECT_EQ(identity->device(), kDevice);
   ASSERT_TRUE(identity->attr().count("T"));
@@ -902,8 +932,72 @@ TEST(MutableGraphViewTest, AddControllingFaninSwitchWithExistingAddedIdentity) {
   ASSERT_EQ(graph.graph()->node_size(), 3);
   NodeDef* a = graph.GetNode("a");
   ASSERT_NE(a, nullptr);
-  ASSERT_EQ(a->input_size(), 1);
-  EXPECT_EQ(a->input(0), "^ConstantFoldingCtrl/switch_0");
+  NodeDef expected = NDef("", "", {"^ConstantFoldingCtrl/switch_0"});
+  CompareNodeInputs(graph, &expected, a);
+}
+
+TEST(MutableGraphViewTest, RemoveControllingFaninMissing) {
+  // Actual node.op() is not important in this test.
+  GraphDef graph_def = test::function::GDef(
+      {
+          NDef("a", "NotImportant", {}, {}),
+          NDef("b", "NotImportant", {}, {}),
+          NDef("c", "NotImportant", {}, {}),
+          NDef("d", "NotImportant", {"^a", "^b"}),
+      },
+      /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  EXPECT_FALSE(graph.RemoveControllingFanin("d", "c"));
+
+  ASSERT_EQ(graph.graph()->node_size(), 4);
+  NodeDef* d = graph.GetNode("d");
+  ASSERT_NE(d, nullptr);
+  NodeDef expected = NDef("", "", {"^a", "^b"});
+  CompareNodeInputs(graph, &expected, d);
+}
+
+TEST(MutableGraphViewTest, RemoveControllingFaninExisting) {
+  // Actual node.op() is not important in this test.
+  GraphDef graph_def = test::function::GDef(
+      {
+          NDef("a", "NotImportant", {}, {}),
+          NDef("b", "NotImportant", {}, {}),
+          NDef("c", "NotImportant", {}, {}),
+          NDef("d", "NotImportant", {"^a", "^b", "^c"}),
+      },
+      /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  EXPECT_TRUE(graph.RemoveControllingFanin("d", "a"));
+  EXPECT_FALSE(graph.RemoveControllingFanin("d", "a"));
+
+  ASSERT_EQ(graph.graph()->node_size(), 4);
+  NodeDef* d = graph.GetNode("d");
+  ASSERT_NE(d, nullptr);
+  NodeDef expected = NDef("", "", {"^c", "^b"});
+  CompareNodeInputs(graph, &expected, d);
+}
+
+TEST(MutableGraphViewTest, RemoveControllingFaninOnRegularFanin) {
+  // Actual node.op() is not important in this test.
+  GraphDef graph_def = test::function::GDef(
+      {NDef("a", "NotImportant", {}, {}), NDef("b", "NotImportant", {"a"}),
+       NDef("c", "NotImportant", {"a", "b", "^c"})},
+      /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  EXPECT_FALSE(graph.RemoveControllingFanin("c", "a"));
+  EXPECT_FALSE(graph.RemoveControllingFanin("c", "b"));
+
+  ASSERT_EQ(graph.graph()->node_size(), 3);
+  NodeDef* c = graph.GetNode("c");
+  ASSERT_NE(c, nullptr);
+  NodeDef expected = NDef("", "", {"a", "b", "^c"});
+  CompareNodeInputs(graph, &expected, c);
 }
 
 TEST(MutableGraphViewTest, DeleteNodes) {
-- 
GitLab


From 80e29253740b251d7632f613ceb414205248ce93 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Thu, 10 Jan 2019 14:06:26 -0800
Subject: [PATCH 0545/1095] [TF:XLA] Bump open source llvm revision to r350820

PiperOrigin-RevId: 228773540
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index a7f3665a31..06caa200e4 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -500,11 +500,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "llvm",
         build_file = clean_dep("//third_party/llvm:llvm.autogenerated.BUILD"),
-        sha256 = "83a4f199742f3d6892994dd6dc46d6a53019aedaa28590b460c120f3dfc7bc47",
-        strip_prefix = "llvm-671f057a2cf137914be6c786daf8000469adebab",
+        sha256 = "b85f0571c5d2997142cd379eda3def286b26671fabb8453c5d56fee6e0698f52",
+        strip_prefix = "llvm-437f3bdb2355b0f453a4b4824e4b74ed6b6a73a5",
         urls = [
-            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/671f057a2cf137914be6c786daf8000469adebab.tar.gz",
-            "https://github.com/llvm-mirror/llvm/archive/671f057a2cf137914be6c786daf8000469adebab.tar.gz",
+            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/437f3bdb2355b0f453a4b4824e4b74ed6b6a73a5.tar.gz",
+            "https://github.com/llvm-mirror/llvm/archive/437f3bdb2355b0f453a4b4824e4b74ed6b6a73a5.tar.gz",
         ],
     )
 
-- 
GitLab


From 462fccab21bc9a6ad0377a0c3998b47291eea279 Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Thu, 10 Jan 2019 14:38:16 -0800
Subject: [PATCH 0546/1095] [TF:XLA] Fix failure in sort_ops_test when built in
 -UNDEBUG mode.

The current shape function fails to set a shape for the second output tensor.

PiperOrigin-RevId: 228779527
---
 tensorflow/compiler/tf2xla/ops/xla_ops.cc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/tf2xla/ops/xla_ops.cc b/tensorflow/compiler/tf2xla/ops/xla_ops.cc
index ab77984684..af641131ed 100644
--- a/tensorflow/compiler/tf2xla/ops/xla_ops.cc
+++ b/tensorflow/compiler/tf2xla/ops/xla_ops.cc
@@ -369,7 +369,11 @@ REGISTER_OP("XlaKeyValueSort")
     .Output("sorted_values: V")
     .Attr("K: realnumbertype")
     .Attr("V: type")
-    .SetShapeFn(shape_inference::UnchangedShape)
+    .SetShapeFn([](shape_inference::InferenceContext* c) {
+      c->set_output(0, c->input(0));
+      c->set_output(1, c->input(1));
+      return Status::OK();
+    })
     .Doc(R"doc(
 Wraps the XLA Sort operator, documented at
  https://www.tensorflow.org/performance/xla/operation_semantics#sort
-- 
GitLab


From 12c9bb638e8e428bccca0ee8f76c26776f75d5b0 Mon Sep 17 00:00:00 2001
From: Saurabh Deoras <sdeoras@gmail.com>
Date: Thu, 10 Jan 2019 14:49:47 -0800
Subject: [PATCH 0547/1095] use smaller slice length for 32-bit arch

Signed-off-by: Saurabh Deoras <sdeoras@gmail.com>
---
 tensorflow/go/attrs.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/go/attrs.go b/tensorflow/go/attrs.go
index f86c5737bc..ed1a1f0b54 100644
--- a/tensorflow/go/attrs.go
+++ b/tensorflow/go/attrs.go
@@ -170,7 +170,8 @@ func listAttribute(op *Operation, cname *C.char, meta C.TF_AttrMetadata) (interf
 			}
 			// A []C.int64_t slice backed by C memory.
 			// See: https://github.com/golang/go/wiki/cgo#turning-c-arrays-into-go-slices
-			slice := (*[1 << 30]C.int64_t)(unsafe.Pointer(dim))[:numDim:numDim]
+			// Using [1<<27] instead of [1<<30] so it works on 32-bit architecture
+			slice := (*[1 << 27]C.int64_t)(unsafe.Pointer(dim))[:numDim:numDim]
 			list[i] = makeCShape(slice)
 		}
 		return list, nil
-- 
GitLab


From f4fd5aae1d19607497047f5d5ae2d9de3aec4112 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 10 Jan 2019 14:45:30 -0800
Subject: [PATCH 0548/1095] Introduced eager profiler python api.

PiperOrigin-RevId: 228780846
---
 tensorflow/python/BUILD                  |  2 +
 tensorflow/python/eager/BUILD            | 24 +++++++++
 tensorflow/python/eager/profiler.py      | 69 ++++++++++++++++++++++++
 tensorflow/python/eager/profiler_test.py | 44 +++++++++++++++
 tensorflow/python/pywrap_tfe.i           |  2 +
 5 files changed, 141 insertions(+)
 create mode 100644 tensorflow/python/eager/profiler.py
 create mode 100644 tensorflow/python/eager/profiler_test.py

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 933ccf1c7b..c5a8994b80 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -174,6 +174,7 @@ py_library(
         "//tensorflow/python/distribute",
         "//tensorflow/python/distribute:estimator_training",
         "//tensorflow/python/eager:def_function",
+        "//tensorflow/python/eager:profiler",
         "//tensorflow/python/eager:remote",
         "//tensorflow/python/ops/distributions",
         "//tensorflow/python/ops/linalg",
@@ -4117,6 +4118,7 @@ tf_py_wrap_cc(
         "//tensorflow/c:python_api",
         "//tensorflow/c:tf_status_helper",
         "//tensorflow/c/eager:c_api",
+        "//tensorflow/c/eager:c_api_experimental",
         "//tensorflow/core/distributed_runtime/rpc:grpc_rpc_factory_registration",
         "//tensorflow/core/distributed_runtime/rpc:grpc_server_lib",
         "//tensorflow/core/distributed_runtime/rpc:grpc_session",
diff --git a/tensorflow/python/eager/BUILD b/tensorflow/python/eager/BUILD
index 1f1cb22d58..3a35734051 100644
--- a/tensorflow/python/eager/BUILD
+++ b/tensorflow/python/eager/BUILD
@@ -25,6 +25,7 @@ cc_library(
         "//tensorflow/c:c_api",
         "//tensorflow/c:c_api_internal",
         "//tensorflow/c/eager:c_api",
+        "//tensorflow/c/eager:c_api_experimental",
         "//tensorflow/c/eager:c_api_internal",
         "//tensorflow/c/eager:tape",
         "//tensorflow/core:framework",
@@ -55,6 +56,7 @@ py_library(
         ":execute",
         ":function",
         ":graph_only_ops",
+        ":profiler",
         ":tape",
         ":test",
         ":wrap_function",
@@ -89,6 +91,28 @@ py_library(
     ],
 )
 
+py_library(
+    name = "profiler",
+    srcs = ["profiler.py"],
+    srcs_version = "PY2AND3",
+    visibility = ["//tensorflow:internal"],
+    deps = [
+        ":context",
+        "//tensorflow/python:pywrap_tensorflow",
+    ],
+)
+
+cuda_py_test(
+    name = "profiler_test",
+    srcs = ["profiler_test.py"],
+    additional_deps = [
+        ":profiler",
+        ":test",
+        "//tensorflow/core/profiler:protos_all_py",
+        "//tensorflow/python:constant_op",
+    ],
+)
+
 py_library(
     name = "tape",
     srcs = ["tape.py"],
diff --git a/tensorflow/python/eager/profiler.py b/tensorflow/python/eager/profiler.py
new file mode 100644
index 0000000000..016bca3e3c
--- /dev/null
+++ b/tensorflow/python/eager/profiler.py
@@ -0,0 +1,69 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Profiler for eager mode."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import threading
+
+from tensorflow.python import pywrap_tensorflow
+from tensorflow.python.eager import context
+from tensorflow.python.framework import c_api_util
+
+_profiler = None
+_profiler_lock = threading.Lock()
+
+
+def start():
+  """Start profiling.
+
+  Only one active profiling session is allowed.
+
+  Raises:
+    AssertionError: If another profiling session is running.
+  """
+  global _profiler
+  if _profiler is not None:
+    raise AssertionError('Another profiler is running.')
+  with _profiler_lock:
+    _profiler = pywrap_tensorflow.TFE_NewProfiler(context.context()._handle)  # pylint: disable=protected-access
+
+
+def stop():
+  """Stop current profiling session and return its result.
+
+  Returns:
+    A binary string of tensorflow.tfprof.ProfileProto. User can write the string
+    to file for offline analysis by tfprof command-line tools or graphical user
+    interface.
+
+  Raises:
+    AssertionError: If there is no active profiling session.
+  """
+  global _profiler
+  if _profiler is None:
+    raise AssertionError('Cannot stop profiling. No profiler is running.')
+  with c_api_util.tf_buffer() as buffer_:
+    pywrap_tensorflow.TFE_ProfilerSerializeToString(
+        context.context()._handle,  # pylint: disable=protected-access
+        _profiler,
+        buffer_)
+    result = pywrap_tensorflow.TF_GetBuffer(buffer_)
+  with _profiler_lock:
+    pywrap_tensorflow.TFE_DeleteProfiler(_profiler)
+    _profiler = None
+  return result
diff --git a/tensorflow/python/eager/profiler_test.py b/tensorflow/python/eager/profiler_test.py
new file mode 100644
index 0000000000..b3fe2efabe
--- /dev/null
+++ b/tensorflow/python/eager/profiler_test.py
@@ -0,0 +1,44 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for eager profiler."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.core.profiler import tfprof_log_pb2
+from tensorflow.python.eager import profiler
+from tensorflow.python.eager import test
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import test_util
+
+
+class ProfilerTest(test_util.TensorFlowTestCase):
+
+  def test_profile(self):
+    profiler.start()
+    three = constant_op.constant(3)
+    five = constant_op.constant(5)
+    product = three * five
+    self.assertAllEqual(15, product)
+    profile_result = profiler.stop()
+    profile_pb = tfprof_log_pb2.ProfileProto()
+    profile_pb.ParseFromString(profile_result)
+    profile_pb_str = '%s' % profile_pb
+    self.assertTrue('Mul' in profile_pb_str)
+
+
+if __name__ == '__main__':
+  test.main()
diff --git a/tensorflow/python/pywrap_tfe.i b/tensorflow/python/pywrap_tfe.i
index 0620e0345c..d0ba244853 100755
--- a/tensorflow/python/pywrap_tfe.i
+++ b/tensorflow/python/pywrap_tfe.i
@@ -77,6 +77,7 @@ limitations under the License.
 %{
 #include "tensorflow/python/eager/pywrap_tfe.h"
 #include "tensorflow/c/c_api_experimental.h"
+#include "tensorflow/c/eager/c_api_experimental.h"
 %}
 
 %typemap(in) (const void* proto) {
@@ -233,6 +234,7 @@ limitations under the License.
 
 %include "tensorflow/python/eager/pywrap_tfe.h"
 %include "tensorflow/c/c_api_experimental.h"
+%include "tensorflow/c/eager/c_api_experimental.h"
 
 // Clear all typemaps.
 %typemap(out) TF_DataType;
-- 
GitLab


From 4df903f8fff67700567abb68ae46c4cddab67cee Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Thu, 10 Jan 2019 14:51:46 -0800
Subject: [PATCH 0549/1095] Splitting metrics_test file - moving confusion
 matrix related metric tests into a separate file.

PiperOrigin-RevId: 228781953
---
 tensorflow/python/keras/BUILD                 |  15 +-
 .../keras/metrics_confusion_matrix_test.py    | 745 ++++++++++++++++++
 tensorflow/python/keras/metrics_test.py       | 713 -----------------
 3 files changed, 759 insertions(+), 714 deletions(-)
 create mode 100644 tensorflow/python/keras/metrics_confusion_matrix_test.py

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index fb0161a677..536acca298 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -395,7 +395,7 @@ tf_py_test(
 
 tf_py_test(
     name = "metrics_functional_test",
-    size = "medium",
+    size = "small",
     srcs = ["metrics_functional_test.py"],
     additional_deps = [
         ":keras",
@@ -417,6 +417,19 @@ tf_py_test(
     shard_count = 4,
 )
 
+tf_py_test(
+    name = "metrics_confusion_matrix_test",
+    size = "medium",
+    srcs = ["metrics_confusion_matrix_test.py"],
+    additional_deps = [
+        ":keras",
+        "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
+    ],
+    shard_count = 4,
+)
+
 tf_py_test(
     name = "applications_test",
     size = "enormous",
diff --git a/tensorflow/python/keras/metrics_confusion_matrix_test.py b/tensorflow/python/keras/metrics_confusion_matrix_test.py
new file mode 100644
index 0000000000..37d2880088
--- /dev/null
+++ b/tensorflow/python/keras/metrics_confusion_matrix_test.py
@@ -0,0 +1,745 @@
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for Keras metrics functions."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from absl.testing import parameterized
+import numpy as np
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
+from tensorflow.python.keras import metrics
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import random_ops
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import test
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class FalsePositivesTest(test.TestCase):
+
+  def test_config(self):
+    fp_obj = metrics.FalsePositives(name='my_fp', thresholds=[0.4, 0.9])
+    self.assertEqual(fp_obj.name, 'my_fp')
+    self.assertEqual(len(fp_obj.variables), 1)
+    self.assertEqual(fp_obj.thresholds, [0.4, 0.9])
+
+    # Check save and restore config
+    fp_obj2 = metrics.FalsePositives.from_config(fp_obj.get_config())
+    self.assertEqual(fp_obj2.name, 'my_fp')
+    self.assertEqual(len(fp_obj2.variables), 1)
+    self.assertEqual(fp_obj2.thresholds, [0.4, 0.9])
+
+  def test_unweighted(self):
+    fp_obj = metrics.FalsePositives()
+    self.evaluate(variables.variables_initializer(fp_obj.variables))
+
+    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
+                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
+    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
+                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
+
+    update_op = fp_obj.update_state(y_true, y_pred)
+    self.evaluate(update_op)
+    result = fp_obj.result()
+    self.assertAllClose(7., result)
+
+  def test_weighted(self):
+    fp_obj = metrics.FalsePositives()
+    self.evaluate(variables.variables_initializer(fp_obj.variables))
+    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
+                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
+    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
+                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
+    sample_weight = constant_op.constant((1., 1.5, 2., 2.5))
+    result = fp_obj(y_true, y_pred, sample_weight=sample_weight)
+    self.assertAllClose(14., self.evaluate(result))
+
+  def test_unweighted_with_thresholds(self):
+    fp_obj = metrics.FalsePositives(thresholds=[0.15, 0.5, 0.85])
+    self.evaluate(variables.variables_initializer(fp_obj.variables))
+
+    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),
+                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))
+    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),
+                                   (1, 1, 1, 1)))
+
+    update_op = fp_obj.update_state(y_true, y_pred)
+    self.evaluate(update_op)
+    result = fp_obj.result()
+    self.assertAllClose([7., 4., 2.], result)
+
+  def test_weighted_with_thresholds(self):
+    fp_obj = metrics.FalsePositives(thresholds=[0.15, 0.5, 0.85])
+    self.evaluate(variables.variables_initializer(fp_obj.variables))
+
+    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),
+                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))
+    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),
+                                   (1, 1, 1, 1)))
+    sample_weight = ((1.0, 2.0, 3.0, 5.0), (7.0, 11.0, 13.0, 17.0),
+                     (19.0, 23.0, 29.0, 31.0), (5.0, 15.0, 10.0, 0))
+
+    result = fp_obj(y_true, y_pred, sample_weight=sample_weight)
+    self.assertAllClose([125., 42., 12.], self.evaluate(result))
+
+  def test_threshold_limit(self):
+    with self.assertRaisesRegexp(
+        ValueError,
+        r'Threshold values must be in \[0, 1\]. Invalid values: \[-1, 2\]'):
+      metrics.FalsePositives(thresholds=[-1, 0.5, 2])
+
+    with self.assertRaisesRegexp(
+        ValueError,
+        r'Threshold values must be in \[0, 1\]. Invalid values: \[None\]'):
+      metrics.FalsePositives(thresholds=[None])
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class FalseNegativesTest(test.TestCase):
+
+  def test_config(self):
+    fn_obj = metrics.FalseNegatives(name='my_fn', thresholds=[0.4, 0.9])
+    self.assertEqual(fn_obj.name, 'my_fn')
+    self.assertEqual(len(fn_obj.variables), 1)
+    self.assertEqual(fn_obj.thresholds, [0.4, 0.9])
+
+    # Check save and restore config
+    fn_obj2 = metrics.FalseNegatives.from_config(fn_obj.get_config())
+    self.assertEqual(fn_obj2.name, 'my_fn')
+    self.assertEqual(len(fn_obj2.variables), 1)
+    self.assertEqual(fn_obj2.thresholds, [0.4, 0.9])
+
+  def test_unweighted(self):
+    fn_obj = metrics.FalseNegatives()
+    self.evaluate(variables.variables_initializer(fn_obj.variables))
+
+    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
+                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
+    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
+                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
+
+    update_op = fn_obj.update_state(y_true, y_pred)
+    self.evaluate(update_op)
+    result = fn_obj.result()
+    self.assertAllClose(3., result)
+
+  def test_weighted(self):
+    fn_obj = metrics.FalseNegatives()
+    self.evaluate(variables.variables_initializer(fn_obj.variables))
+    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
+                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
+    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
+                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
+    sample_weight = constant_op.constant((1., 1.5, 2., 2.5))
+    result = fn_obj(y_true, y_pred, sample_weight=sample_weight)
+    self.assertAllClose(5., self.evaluate(result))
+
+  def test_unweighted_with_thresholds(self):
+    fn_obj = metrics.FalseNegatives(thresholds=[0.15, 0.5, 0.85])
+    self.evaluate(variables.variables_initializer(fn_obj.variables))
+
+    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),
+                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))
+    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),
+                                   (1, 1, 1, 1)))
+
+    update_op = fn_obj.update_state(y_true, y_pred)
+    self.evaluate(update_op)
+    result = fn_obj.result()
+    self.assertAllClose([1., 4., 6.], result)
+
+  def test_weighted_with_thresholds(self):
+    fn_obj = metrics.FalseNegatives(thresholds=[0.15, 0.5, 0.85])
+    self.evaluate(variables.variables_initializer(fn_obj.variables))
+
+    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),
+                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))
+    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),
+                                   (1, 1, 1, 1)))
+    sample_weight = ((3.0,), (5.0,), (7.0,), (4.0,))
+
+    result = fn_obj(y_true, y_pred, sample_weight=sample_weight)
+    self.assertAllClose([4., 16., 23.], self.evaluate(result))
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class TrueNegativesTest(test.TestCase):
+
+  def test_config(self):
+    tn_obj = metrics.TrueNegatives(name='my_tn', thresholds=[0.4, 0.9])
+    self.assertEqual(tn_obj.name, 'my_tn')
+    self.assertEqual(len(tn_obj.variables), 1)
+    self.assertEqual(tn_obj.thresholds, [0.4, 0.9])
+
+    # Check save and restore config
+    tn_obj2 = metrics.TrueNegatives.from_config(tn_obj.get_config())
+    self.assertEqual(tn_obj2.name, 'my_tn')
+    self.assertEqual(len(tn_obj2.variables), 1)
+    self.assertEqual(tn_obj2.thresholds, [0.4, 0.9])
+
+  def test_unweighted(self):
+    tn_obj = metrics.TrueNegatives()
+    self.evaluate(variables.variables_initializer(tn_obj.variables))
+
+    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
+                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
+    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
+                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
+
+    update_op = tn_obj.update_state(y_true, y_pred)
+    self.evaluate(update_op)
+    result = tn_obj.result()
+    self.assertAllClose(3., result)
+
+  def test_weighted(self):
+    tn_obj = metrics.TrueNegatives()
+    self.evaluate(variables.variables_initializer(tn_obj.variables))
+    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
+                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
+    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
+                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
+    sample_weight = constant_op.constant((1., 1.5, 2., 2.5))
+    result = tn_obj(y_true, y_pred, sample_weight=sample_weight)
+    self.assertAllClose(4., self.evaluate(result))
+
+  def test_unweighted_with_thresholds(self):
+    tn_obj = metrics.TrueNegatives(thresholds=[0.15, 0.5, 0.85])
+    self.evaluate(variables.variables_initializer(tn_obj.variables))
+
+    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),
+                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))
+    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),
+                                   (1, 1, 1, 1)))
+
+    update_op = tn_obj.update_state(y_true, y_pred)
+    self.evaluate(update_op)
+    result = tn_obj.result()
+    self.assertAllClose([2., 5., 7.], result)
+
+  def test_weighted_with_thresholds(self):
+    tn_obj = metrics.TrueNegatives(thresholds=[0.15, 0.5, 0.85])
+    self.evaluate(variables.variables_initializer(tn_obj.variables))
+
+    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),
+                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))
+    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),
+                                   (1, 1, 1, 1)))
+    sample_weight = ((0.0, 2.0, 3.0, 5.0),)
+
+    result = tn_obj(y_true, y_pred, sample_weight=sample_weight)
+    self.assertAllClose([5., 15., 23.], self.evaluate(result))
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class TruePositivesTest(test.TestCase):
+
+  def test_config(self):
+    tp_obj = metrics.TruePositives(name='my_tp', thresholds=[0.4, 0.9])
+    self.assertEqual(tp_obj.name, 'my_tp')
+    self.assertEqual(len(tp_obj.variables), 1)
+    self.assertEqual(tp_obj.thresholds, [0.4, 0.9])
+
+    # Check save and restore config
+    tp_obj2 = metrics.TruePositives.from_config(tp_obj.get_config())
+    self.assertEqual(tp_obj2.name, 'my_tp')
+    self.assertEqual(len(tp_obj2.variables), 1)
+    self.assertEqual(tp_obj2.thresholds, [0.4, 0.9])
+
+  def test_unweighted(self):
+    tp_obj = metrics.TruePositives()
+    self.evaluate(variables.variables_initializer(tp_obj.variables))
+
+    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
+                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
+    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
+                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
+
+    update_op = tp_obj.update_state(y_true, y_pred)
+    self.evaluate(update_op)
+    result = tp_obj.result()
+    self.assertAllClose(7., result)
+
+  def test_weighted(self):
+    tp_obj = metrics.TruePositives()
+    self.evaluate(variables.variables_initializer(tp_obj.variables))
+    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
+                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
+    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
+                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
+    sample_weight = constant_op.constant((1., 1.5, 2., 2.5))
+    result = tp_obj(y_true, y_pred, sample_weight=sample_weight)
+    self.assertAllClose(12., self.evaluate(result))
+
+  def test_unweighted_with_thresholds(self):
+    tp_obj = metrics.TruePositives(thresholds=[0.15, 0.5, 0.85])
+    self.evaluate(variables.variables_initializer(tp_obj.variables))
+
+    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),
+                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))
+    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),
+                                   (1, 1, 1, 1)))
+
+    update_op = tp_obj.update_state(y_true, y_pred)
+    self.evaluate(update_op)
+    result = tp_obj.result()
+    self.assertAllClose([6., 3., 1.], result)
+
+  def test_weighted_with_thresholds(self):
+    tp_obj = metrics.TruePositives(thresholds=[0.15, 0.5, 0.85])
+    self.evaluate(variables.variables_initializer(tp_obj.variables))
+
+    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),
+                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))
+    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),
+                                   (1, 1, 1, 1)))
+
+    result = tp_obj(y_true, y_pred, sample_weight=37.)
+    self.assertAllClose([222., 111., 37.], self.evaluate(result))
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class PrecisionTest(test.TestCase):
+
+  def test_config(self):
+    p_obj = metrics.Precision(name='my_precision', thresholds=[0.4, 0.9])
+    self.assertEqual(p_obj.name, 'my_precision')
+    self.assertEqual(len(p_obj.variables), 2)
+    self.assertEqual([v.name for v in p_obj.variables],
+                     ['true_positives:0', 'false_positives:0'])
+    self.assertEqual(p_obj.thresholds, [0.4, 0.9])
+
+    # Check save and restore config
+    p_obj2 = metrics.Precision.from_config(p_obj.get_config())
+    self.assertEqual(p_obj2.name, 'my_precision')
+    self.assertEqual(len(p_obj2.variables), 2)
+    self.assertEqual(p_obj2.thresholds, [0.4, 0.9])
+
+  def test_value_is_idempotent(self):
+    p_obj = metrics.Precision(thresholds=[0.3, 0.72])
+    y_pred = random_ops.random_uniform(shape=(10, 3))
+    y_true = random_ops.random_uniform(shape=(10, 3))
+    update_op = p_obj.update_state(y_true, y_pred)
+    self.evaluate(variables.variables_initializer(p_obj.variables))
+
+    # Run several updates.
+    for _ in range(10):
+      self.evaluate(update_op)
+
+    # Then verify idempotency.
+    initial_precision = self.evaluate(p_obj.result())
+    for _ in range(10):
+      self.assertArrayNear(initial_precision, self.evaluate(p_obj.result()),
+                           1e-3)
+
+  def test_unweighted(self):
+    p_obj = metrics.Precision()
+    y_pred = constant_op.constant([1, 0, 1, 0], shape=(1, 4))
+    y_true = constant_op.constant([0, 1, 1, 0], shape=(1, 4))
+    self.evaluate(variables.variables_initializer(p_obj.variables))
+    result = p_obj(y_true, y_pred)
+    self.assertAlmostEqual(0.5, self.evaluate(result))
+
+  def test_unweighted_all_incorrect(self):
+    p_obj = metrics.Precision(thresholds=[0.5])
+    inputs = np.random.randint(0, 2, size=(100, 1))
+    y_pred = constant_op.constant(inputs)
+    y_true = constant_op.constant(1 - inputs)
+    self.evaluate(variables.variables_initializer(p_obj.variables))
+    result = p_obj(y_true, y_pred)
+    self.assertAlmostEqual(0, self.evaluate(result))
+
+  def test_weighted(self):
+    p_obj = metrics.Precision()
+    y_pred = constant_op.constant([[1, 0, 1, 0], [1, 0, 1, 0]])
+    y_true = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]])
+    self.evaluate(variables.variables_initializer(p_obj.variables))
+    result = p_obj(
+        y_true,
+        y_pred,
+        sample_weight=constant_op.constant([[1, 2, 3, 4], [4, 3, 2, 1]]))
+    weighted_tp = 3.0 + 4.0
+    weighted_positives = (1.0 + 3.0) + (4.0 + 2.0)
+    expected_precision = weighted_tp / weighted_positives
+    self.assertAlmostEqual(expected_precision, self.evaluate(result))
+
+  def test_div_by_zero(self):
+    p_obj = metrics.Precision()
+    y_pred = constant_op.constant([0, 0, 0, 0])
+    y_true = constant_op.constant([0, 0, 0, 0])
+    self.evaluate(variables.variables_initializer(p_obj.variables))
+    result = p_obj(y_true, y_pred)
+    self.assertEqual(0, self.evaluate(result))
+
+  def test_unweighted_with_threshold(self):
+    p_obj = metrics.Precision(thresholds=[0.5, 0.7])
+    y_pred = constant_op.constant([1, 0, 0.6, 0], shape=(1, 4))
+    y_true = constant_op.constant([0, 1, 1, 0], shape=(1, 4))
+    self.evaluate(variables.variables_initializer(p_obj.variables))
+    result = p_obj(y_true, y_pred)
+    self.assertArrayNear([0.5, 0.], self.evaluate(result), 0)
+
+  def test_weighted_with_threshold(self):
+    p_obj = metrics.Precision(thresholds=[0.5, 1.])
+    y_true = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2))
+    y_pred = constant_op.constant([[1, 0], [0.6, 0]],
+                                  shape=(2, 2),
+                                  dtype=dtypes.float32)
+    weights = constant_op.constant([[4, 0], [3, 1]],
+                                   shape=(2, 2),
+                                   dtype=dtypes.float32)
+    self.evaluate(variables.variables_initializer(p_obj.variables))
+    result = p_obj(y_true, y_pred, sample_weight=weights)
+    weighted_tp = 0 + 3.
+    weighted_positives = (0 + 3.) + (4. + 0.)
+    expected_precision = weighted_tp / weighted_positives
+    self.assertArrayNear([expected_precision, 0], self.evaluate(result), 1e-3)
+
+  def test_multiple_updates(self):
+    p_obj = metrics.Precision(thresholds=[0.5, 1.])
+    y_true = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2))
+    y_pred = constant_op.constant([[1, 0], [0.6, 0]],
+                                  shape=(2, 2),
+                                  dtype=dtypes.float32)
+    weights = constant_op.constant([[4, 0], [3, 1]],
+                                   shape=(2, 2),
+                                   dtype=dtypes.float32)
+    self.evaluate(variables.variables_initializer(p_obj.variables))
+    update_op = p_obj.update_state(y_true, y_pred, sample_weight=weights)
+    for _ in range(2):
+      self.evaluate(update_op)
+
+    weighted_tp = (0 + 3.) + (0 + 3.)
+    weighted_positives = ((0 + 3.) + (4. + 0.)) + ((0 + 3.) + (4. + 0.))
+    expected_precision = weighted_tp / weighted_positives
+    self.assertArrayNear([expected_precision, 0], self.evaluate(p_obj.result()),
+                         1e-3)
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class RecallTest(test.TestCase):
+
+  def test_config(self):
+    r_obj = metrics.Recall(name='my_recall', thresholds=[0.4, 0.9])
+    self.assertEqual(r_obj.name, 'my_recall')
+    self.assertEqual(len(r_obj.variables), 2)
+    self.assertEqual([v.name for v in r_obj.variables],
+                     ['true_positives:0', 'false_negatives:0'])
+    self.assertEqual(r_obj.thresholds, [0.4, 0.9])
+
+    # Check save and restore config
+    r_obj2 = metrics.Recall.from_config(r_obj.get_config())
+    self.assertEqual(r_obj2.name, 'my_recall')
+    self.assertEqual(len(r_obj2.variables), 2)
+    self.assertEqual(r_obj2.thresholds, [0.4, 0.9])
+
+  def test_value_is_idempotent(self):
+    r_obj = metrics.Recall(thresholds=[0.3, 0.72])
+    y_pred = random_ops.random_uniform(shape=(10, 3))
+    y_true = random_ops.random_uniform(shape=(10, 3))
+    update_op = r_obj.update_state(y_true, y_pred)
+    self.evaluate(variables.variables_initializer(r_obj.variables))
+
+    # Run several updates.
+    for _ in range(10):
+      self.evaluate(update_op)
+
+    # Then verify idempotency.
+    initial_recall = self.evaluate(r_obj.result())
+    for _ in range(10):
+      self.assertArrayNear(initial_recall, self.evaluate(r_obj.result()), 1e-3)
+
+  def test_unweighted(self):
+    r_obj = metrics.Recall()
+    y_pred = constant_op.constant([1, 0, 1, 0], shape=(1, 4))
+    y_true = constant_op.constant([0, 1, 1, 0], shape=(1, 4))
+    self.evaluate(variables.variables_initializer(r_obj.variables))
+    result = r_obj(y_true, y_pred)
+    self.assertAlmostEqual(0.5, self.evaluate(result))
+
+  def test_unweighted_all_incorrect(self):
+    r_obj = metrics.Recall(thresholds=[0.5])
+    inputs = np.random.randint(0, 2, size=(100, 1))
+    y_pred = constant_op.constant(inputs)
+    y_true = constant_op.constant(1 - inputs)
+    self.evaluate(variables.variables_initializer(r_obj.variables))
+    result = r_obj(y_true, y_pred)
+    self.assertAlmostEqual(0, self.evaluate(result))
+
+  def test_weighted(self):
+    r_obj = metrics.Recall()
+    y_pred = constant_op.constant([[1, 0, 1, 0], [0, 1, 0, 1]])
+    y_true = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]])
+    self.evaluate(variables.variables_initializer(r_obj.variables))
+    result = r_obj(
+        y_true,
+        y_pred,
+        sample_weight=constant_op.constant([[1, 2, 3, 4], [4, 3, 2, 1]]))
+    weighted_tp = 3.0 + 1.0
+    weighted_t = (2.0 + 3.0) + (4.0 + 1.0)
+    expected_recall = weighted_tp / weighted_t
+    self.assertAlmostEqual(expected_recall, self.evaluate(result))
+
+  def test_div_by_zero(self):
+    r_obj = metrics.Recall()
+    y_pred = constant_op.constant([0, 0, 0, 0])
+    y_true = constant_op.constant([0, 0, 0, 0])
+    self.evaluate(variables.variables_initializer(r_obj.variables))
+    result = r_obj(y_true, y_pred)
+    self.assertEqual(0, self.evaluate(result))
+
+  def test_unweighted_with_threshold(self):
+    r_obj = metrics.Recall(thresholds=[0.5, 0.7])
+    y_pred = constant_op.constant([1, 0, 0.6, 0], shape=(1, 4))
+    y_true = constant_op.constant([0, 1, 1, 0], shape=(1, 4))
+    self.evaluate(variables.variables_initializer(r_obj.variables))
+    result = r_obj(y_true, y_pred)
+    self.assertArrayNear([0.5, 0.], self.evaluate(result), 0)
+
+  def test_weighted_with_threshold(self):
+    r_obj = metrics.Recall(thresholds=[0.5, 1.])
+    y_true = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2))
+    y_pred = constant_op.constant([[1, 0], [0.6, 0]],
+                                  shape=(2, 2),
+                                  dtype=dtypes.float32)
+    weights = constant_op.constant([[1, 4], [3, 2]],
+                                   shape=(2, 2),
+                                   dtype=dtypes.float32)
+    self.evaluate(variables.variables_initializer(r_obj.variables))
+    result = r_obj(y_true, y_pred, sample_weight=weights)
+    weighted_tp = 0 + 3.
+    weighted_positives = (0 + 3.) + (4. + 0.)
+    expected_recall = weighted_tp / weighted_positives
+    self.assertArrayNear([expected_recall, 0], self.evaluate(result), 1e-3)
+
+  def test_multiple_updates(self):
+    r_obj = metrics.Recall(thresholds=[0.5, 1.])
+    y_true = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2))
+    y_pred = constant_op.constant([[1, 0], [0.6, 0]],
+                                  shape=(2, 2),
+                                  dtype=dtypes.float32)
+    weights = constant_op.constant([[1, 4], [3, 2]],
+                                   shape=(2, 2),
+                                   dtype=dtypes.float32)
+    self.evaluate(variables.variables_initializer(r_obj.variables))
+    update_op = r_obj.update_state(y_true, y_pred, sample_weight=weights)
+    for _ in range(2):
+      self.evaluate(update_op)
+
+    weighted_tp = (0 + 3.) + (0 + 3.)
+    weighted_positives = ((0 + 3.) + (4. + 0.)) + ((0 + 3.) + (4. + 0.))
+    expected_recall = weighted_tp / weighted_positives
+    self.assertArrayNear([expected_recall, 0], self.evaluate(r_obj.result()),
+                         1e-3)
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class SensitivityAtSpecificityTest(test.TestCase, parameterized.TestCase):
+
+  def test_config(self):
+    s_obj = metrics.SensitivityAtSpecificity(
+        0.4, num_thresholds=100, name='sensitivity_at_specificity_1')
+    self.assertEqual(s_obj.name, 'sensitivity_at_specificity_1')
+    self.assertLen(s_obj.variables, 4)
+    self.assertEqual(s_obj.specificity, 0.4)
+    self.assertEqual(s_obj.num_thresholds, 100)
+
+    # Check save and restore config
+    s_obj2 = metrics.SensitivityAtSpecificity.from_config(s_obj.get_config())
+    self.assertEqual(s_obj2.name, 'sensitivity_at_specificity_1')
+    self.assertLen(s_obj2.variables, 4)
+    self.assertEqual(s_obj2.specificity, 0.4)
+    self.assertEqual(s_obj2.num_thresholds, 100)
+
+  def test_value_is_idempotent(self):
+    s_obj = metrics.SensitivityAtSpecificity(0.7)
+    y_pred = random_ops.random_uniform((10, 3),
+                                       maxval=1,
+                                       dtype=dtypes.float32,
+                                       seed=1)
+    y_true = random_ops.random_uniform((10, 3),
+                                       maxval=2,
+                                       dtype=dtypes.int64,
+                                       seed=1)
+    update_op = s_obj.update_state(y_true, y_pred)
+    self.evaluate(variables.variables_initializer(s_obj.variables))
+
+    # Run several updates.
+    for _ in range(10):
+      self.evaluate(update_op)
+
+    # Then verify idempotency.
+    initial_sensitivity = self.evaluate(s_obj.result())
+    for _ in range(10):
+      self.assertAlmostEqual(initial_sensitivity, self.evaluate(s_obj.result()),
+                             1e-3)
+
+  def test_unweighted_all_correct(self):
+    s_obj = metrics.SensitivityAtSpecificity(0.7)
+    inputs = np.random.randint(0, 2, size=(100, 1))
+    y_pred = constant_op.constant(inputs, dtype=dtypes.float32)
+    y_true = constant_op.constant(inputs)
+    self.evaluate(variables.variables_initializer(s_obj.variables))
+    result = s_obj(y_true, y_pred)
+    self.assertAlmostEqual(1, self.evaluate(result))
+
+  def test_unweighted_high_specificity(self):
+    s_obj = metrics.SensitivityAtSpecificity(0.8)
+    pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.1, 0.45, 0.5, 0.8, 0.9]
+    label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]
+
+    y_pred = constant_op.constant(pred_values, dtype=dtypes.float32)
+    y_true = constant_op.constant(label_values)
+    self.evaluate(variables.variables_initializer(s_obj.variables))
+    result = s_obj(y_true, y_pred)
+    self.assertAlmostEqual(0.8, self.evaluate(result))
+
+  def test_unweighted_low_specificity(self):
+    s_obj = metrics.SensitivityAtSpecificity(0.4)
+    pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.01, 0.02, 0.25, 0.26, 0.26]
+    label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]
+
+    y_pred = constant_op.constant(pred_values, dtype=dtypes.float32)
+    y_true = constant_op.constant(label_values)
+    self.evaluate(variables.variables_initializer(s_obj.variables))
+    result = s_obj(y_true, y_pred)
+    self.assertAlmostEqual(0.6, self.evaluate(result))
+
+  @parameterized.parameters([dtypes.bool, dtypes.int32, dtypes.float32])
+  def test_weighted(self, label_dtype):
+    s_obj = metrics.SensitivityAtSpecificity(0.4)
+    pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.01, 0.02, 0.25, 0.26, 0.26]
+    label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]
+    weight_values = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+
+    y_pred = constant_op.constant(pred_values, dtype=dtypes.float32)
+    y_true = math_ops.cast(label_values, dtype=label_dtype)
+    weights = constant_op.constant(weight_values)
+    self.evaluate(variables.variables_initializer(s_obj.variables))
+    result = s_obj(y_true, y_pred, sample_weight=weights)
+    self.assertAlmostEqual(0.675, self.evaluate(result))
+
+  def test_invalid_specificity(self):
+    with self.assertRaisesRegexp(
+        ValueError, r'`specificity` must be in the range \[0, 1\].'):
+      metrics.SensitivityAtSpecificity(-1)
+
+  def test_invalid_num_thresholds(self):
+    with self.assertRaisesRegexp(ValueError, '`num_thresholds` must be > 0.'):
+      metrics.SensitivityAtSpecificity(0.4, num_thresholds=-1)
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class SpecificityAtSensitivityTest(test.TestCase, parameterized.TestCase):
+
+  def test_config(self):
+    s_obj = metrics.SpecificityAtSensitivity(
+        0.4, num_thresholds=100, name='specificity_at_sensitivity_1')
+    self.assertEqual(s_obj.name, 'specificity_at_sensitivity_1')
+    self.assertLen(s_obj.variables, 4)
+    self.assertEqual(s_obj.sensitivity, 0.4)
+    self.assertEqual(s_obj.num_thresholds, 100)
+
+    # Check save and restore config
+    s_obj2 = metrics.SpecificityAtSensitivity.from_config(s_obj.get_config())
+    self.assertEqual(s_obj2.name, 'specificity_at_sensitivity_1')
+    self.assertLen(s_obj2.variables, 4)
+    self.assertEqual(s_obj2.sensitivity, 0.4)
+    self.assertEqual(s_obj2.num_thresholds, 100)
+
+  def test_value_is_idempotent(self):
+    s_obj = metrics.SpecificityAtSensitivity(0.7)
+    y_pred = random_ops.random_uniform((10, 3),
+                                       maxval=1,
+                                       dtype=dtypes.float32,
+                                       seed=1)
+    y_true = random_ops.random_uniform((10, 3),
+                                       maxval=2,
+                                       dtype=dtypes.int64,
+                                       seed=1)
+    update_op = s_obj.update_state(y_true, y_pred)
+    self.evaluate(variables.variables_initializer(s_obj.variables))
+
+    # Run several updates.
+    for _ in range(10):
+      self.evaluate(update_op)
+
+    # Then verify idempotency.
+    initial_specificity = self.evaluate(s_obj.result())
+    for _ in range(10):
+      self.assertAlmostEqual(initial_specificity, self.evaluate(s_obj.result()),
+                             1e-3)
+
+  def test_unweighted_all_correct(self):
+    s_obj = metrics.SpecificityAtSensitivity(0.7)
+    inputs = np.random.randint(0, 2, size=(100, 1))
+    y_pred = constant_op.constant(inputs, dtype=dtypes.float32)
+    y_true = constant_op.constant(inputs)
+    self.evaluate(variables.variables_initializer(s_obj.variables))
+    result = s_obj(y_true, y_pred)
+    self.assertAlmostEqual(1, self.evaluate(result))
+
+  def test_unweighted_high_sensitivity(self):
+    s_obj = metrics.SpecificityAtSensitivity(0.8)
+    pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.1, 0.45, 0.5, 0.8, 0.9]
+    label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]
+
+    y_pred = constant_op.constant(pred_values, dtype=dtypes.float32)
+    y_true = constant_op.constant(label_values)
+    self.evaluate(variables.variables_initializer(s_obj.variables))
+    result = s_obj(y_true, y_pred)
+    self.assertAlmostEqual(0.4, self.evaluate(result))
+
+  def test_unweighted_low_sensitivity(self):
+    s_obj = metrics.SpecificityAtSensitivity(0.4)
+    pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.01, 0.02, 0.25, 0.26, 0.26]
+    label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]
+
+    y_pred = constant_op.constant(pred_values, dtype=dtypes.float32)
+    y_true = constant_op.constant(label_values)
+    self.evaluate(variables.variables_initializer(s_obj.variables))
+    result = s_obj(y_true, y_pred)
+    self.assertAlmostEqual(0.6, self.evaluate(result))
+
+  @parameterized.parameters([dtypes.bool, dtypes.int32, dtypes.float32])
+  def test_weighted(self, label_dtype):
+    s_obj = metrics.SpecificityAtSensitivity(0.4)
+    pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.01, 0.02, 0.25, 0.26, 0.26]
+    label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]
+    weight_values = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+
+    y_pred = constant_op.constant(pred_values, dtype=dtypes.float32)
+    y_true = math_ops.cast(label_values, dtype=label_dtype)
+    weights = constant_op.constant(weight_values)
+    self.evaluate(variables.variables_initializer(s_obj.variables))
+    result = s_obj(y_true, y_pred, sample_weight=weights)
+    self.assertAlmostEqual(0.4, self.evaluate(result))
+
+  def test_invalid_sensitivity(self):
+    with self.assertRaisesRegexp(
+        ValueError, r'`sensitivity` must be in the range \[0, 1\].'):
+      metrics.SpecificityAtSensitivity(-1)
+
+  def test_invalid_num_thresholds(self):
+    with self.assertRaisesRegexp(ValueError, '`num_thresholds` must be > 0.'):
+      metrics.SpecificityAtSensitivity(0.4, num_thresholds=-1)
+
+
+if __name__ == '__main__':
+  test.main()
diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py
index 42da1dfb99..f902838c11 100644
--- a/tensorflow/python/keras/metrics_test.py
+++ b/tensorflow/python/keras/metrics_test.py
@@ -20,7 +20,6 @@ from __future__ import print_function
 
 import math
 import os
-from absl.testing import parameterized
 import numpy as np
 
 from tensorflow.python.eager import context
@@ -33,8 +32,6 @@ from tensorflow.python.keras import layers
 from tensorflow.python.keras import metrics
 from tensorflow.python.keras import testing_utils
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import math_ops
-from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
 from tensorflow.python.training.checkpointable import util as checkpointable_utils
@@ -336,716 +333,6 @@ class KerasAccuracyTest(test.TestCase):
       self.assertAlmostEqual(result, 0.71, 2)  # 2.5/2.7
 
 
-@test_util.run_all_in_graph_and_eager_modes
-class FalsePositivesTest(test.TestCase):
-
-  def test_config(self):
-    fp_obj = metrics.FalsePositives(name='my_fp', thresholds=[0.4, 0.9])
-    self.assertEqual(fp_obj.name, 'my_fp')
-    self.assertEqual(len(fp_obj.variables), 1)
-    self.assertEqual(fp_obj.thresholds, [0.4, 0.9])
-
-    # Check save and restore config
-    fp_obj2 = metrics.FalsePositives.from_config(fp_obj.get_config())
-    self.assertEqual(fp_obj2.name, 'my_fp')
-    self.assertEqual(len(fp_obj2.variables), 1)
-    self.assertEqual(fp_obj2.thresholds, [0.4, 0.9])
-
-  def test_unweighted(self):
-    fp_obj = metrics.FalsePositives()
-    self.evaluate(variables.variables_initializer(fp_obj.variables))
-
-    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
-                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
-    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
-                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
-
-    update_op = fp_obj.update_state(y_true, y_pred)
-    self.evaluate(update_op)
-    result = fp_obj.result()
-    self.assertAllClose(7., result)
-
-  def test_weighted(self):
-    fp_obj = metrics.FalsePositives()
-    self.evaluate(variables.variables_initializer(fp_obj.variables))
-    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
-                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
-    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
-                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
-    sample_weight = constant_op.constant((1., 1.5, 2., 2.5))
-    result = fp_obj(y_true, y_pred, sample_weight=sample_weight)
-    self.assertAllClose(14., self.evaluate(result))
-
-  def test_unweighted_with_thresholds(self):
-    fp_obj = metrics.FalsePositives(thresholds=[0.15, 0.5, 0.85])
-    self.evaluate(variables.variables_initializer(fp_obj.variables))
-
-    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),
-                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))
-    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),
-                                   (1, 1, 1, 1)))
-
-    update_op = fp_obj.update_state(y_true, y_pred)
-    self.evaluate(update_op)
-    result = fp_obj.result()
-    self.assertAllClose([7., 4., 2.], result)
-
-  def test_weighted_with_thresholds(self):
-    fp_obj = metrics.FalsePositives(thresholds=[0.15, 0.5, 0.85])
-    self.evaluate(variables.variables_initializer(fp_obj.variables))
-
-    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),
-                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))
-    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),
-                                   (1, 1, 1, 1)))
-    sample_weight = ((1.0, 2.0, 3.0, 5.0), (7.0, 11.0, 13.0, 17.0),
-                     (19.0, 23.0, 29.0, 31.0), (5.0, 15.0, 10.0, 0))
-
-    result = fp_obj(y_true, y_pred, sample_weight=sample_weight)
-    self.assertAllClose([125., 42., 12.], self.evaluate(result))
-
-  def test_threshold_limit(self):
-    with self.assertRaisesRegexp(
-        ValueError,
-        r'Threshold values must be in \[0, 1\]. Invalid values: \[-1, 2\]'):
-      metrics.FalsePositives(thresholds=[-1, 0.5, 2])
-
-    with self.assertRaisesRegexp(
-        ValueError,
-        r'Threshold values must be in \[0, 1\]. Invalid values: \[None\]'):
-      metrics.FalsePositives(thresholds=[None])
-
-
-@test_util.run_all_in_graph_and_eager_modes
-class FalseNegativesTest(test.TestCase):
-
-  def test_config(self):
-    fn_obj = metrics.FalseNegatives(name='my_fn', thresholds=[0.4, 0.9])
-    self.assertEqual(fn_obj.name, 'my_fn')
-    self.assertEqual(len(fn_obj.variables), 1)
-    self.assertEqual(fn_obj.thresholds, [0.4, 0.9])
-
-    # Check save and restore config
-    fn_obj2 = metrics.FalseNegatives.from_config(fn_obj.get_config())
-    self.assertEqual(fn_obj2.name, 'my_fn')
-    self.assertEqual(len(fn_obj2.variables), 1)
-    self.assertEqual(fn_obj2.thresholds, [0.4, 0.9])
-
-  def test_unweighted(self):
-    fn_obj = metrics.FalseNegatives()
-    self.evaluate(variables.variables_initializer(fn_obj.variables))
-
-    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
-                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
-    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
-                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
-
-    update_op = fn_obj.update_state(y_true, y_pred)
-    self.evaluate(update_op)
-    result = fn_obj.result()
-    self.assertAllClose(3., result)
-
-  def test_weighted(self):
-    fn_obj = metrics.FalseNegatives()
-    self.evaluate(variables.variables_initializer(fn_obj.variables))
-    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
-                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
-    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
-                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
-    sample_weight = constant_op.constant((1., 1.5, 2., 2.5))
-    result = fn_obj(y_true, y_pred, sample_weight=sample_weight)
-    self.assertAllClose(5., self.evaluate(result))
-
-  def test_unweighted_with_thresholds(self):
-    fn_obj = metrics.FalseNegatives(thresholds=[0.15, 0.5, 0.85])
-    self.evaluate(variables.variables_initializer(fn_obj.variables))
-
-    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),
-                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))
-    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),
-                                   (1, 1, 1, 1)))
-
-    update_op = fn_obj.update_state(y_true, y_pred)
-    self.evaluate(update_op)
-    result = fn_obj.result()
-    self.assertAllClose([1., 4., 6.], result)
-
-  def test_weighted_with_thresholds(self):
-    fn_obj = metrics.FalseNegatives(thresholds=[0.15, 0.5, 0.85])
-    self.evaluate(variables.variables_initializer(fn_obj.variables))
-
-    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),
-                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))
-    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),
-                                   (1, 1, 1, 1)))
-    sample_weight = ((3.0,), (5.0,), (7.0,), (4.0,))
-
-    result = fn_obj(y_true, y_pred, sample_weight=sample_weight)
-    self.assertAllClose([4., 16., 23.], self.evaluate(result))
-
-
-@test_util.run_all_in_graph_and_eager_modes
-class TrueNegativesTest(test.TestCase):
-
-  def test_config(self):
-    tn_obj = metrics.TrueNegatives(name='my_tn', thresholds=[0.4, 0.9])
-    self.assertEqual(tn_obj.name, 'my_tn')
-    self.assertEqual(len(tn_obj.variables), 1)
-    self.assertEqual(tn_obj.thresholds, [0.4, 0.9])
-
-    # Check save and restore config
-    tn_obj2 = metrics.TrueNegatives.from_config(tn_obj.get_config())
-    self.assertEqual(tn_obj2.name, 'my_tn')
-    self.assertEqual(len(tn_obj2.variables), 1)
-    self.assertEqual(tn_obj2.thresholds, [0.4, 0.9])
-
-  def test_unweighted(self):
-    tn_obj = metrics.TrueNegatives()
-    self.evaluate(variables.variables_initializer(tn_obj.variables))
-
-    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
-                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
-    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
-                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
-
-    update_op = tn_obj.update_state(y_true, y_pred)
-    self.evaluate(update_op)
-    result = tn_obj.result()
-    self.assertAllClose(3., result)
-
-  def test_weighted(self):
-    tn_obj = metrics.TrueNegatives()
-    self.evaluate(variables.variables_initializer(tn_obj.variables))
-    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
-                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
-    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
-                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
-    sample_weight = constant_op.constant((1., 1.5, 2., 2.5))
-    result = tn_obj(y_true, y_pred, sample_weight=sample_weight)
-    self.assertAllClose(4., self.evaluate(result))
-
-  def test_unweighted_with_thresholds(self):
-    tn_obj = metrics.TrueNegatives(thresholds=[0.15, 0.5, 0.85])
-    self.evaluate(variables.variables_initializer(tn_obj.variables))
-
-    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),
-                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))
-    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),
-                                   (1, 1, 1, 1)))
-
-    update_op = tn_obj.update_state(y_true, y_pred)
-    self.evaluate(update_op)
-    result = tn_obj.result()
-    self.assertAllClose([2., 5., 7.], result)
-
-  def test_weighted_with_thresholds(self):
-    tn_obj = metrics.TrueNegatives(thresholds=[0.15, 0.5, 0.85])
-    self.evaluate(variables.variables_initializer(tn_obj.variables))
-
-    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),
-                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))
-    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),
-                                   (1, 1, 1, 1)))
-    sample_weight = ((0.0, 2.0, 3.0, 5.0),)
-
-    result = tn_obj(y_true, y_pred, sample_weight=sample_weight)
-    self.assertAllClose([5., 15., 23.], self.evaluate(result))
-
-
-@test_util.run_all_in_graph_and_eager_modes
-class TruePositivesTest(test.TestCase):
-
-  def test_config(self):
-    tp_obj = metrics.TruePositives(name='my_tp', thresholds=[0.4, 0.9])
-    self.assertEqual(tp_obj.name, 'my_tp')
-    self.assertEqual(len(tp_obj.variables), 1)
-    self.assertEqual(tp_obj.thresholds, [0.4, 0.9])
-
-    # Check save and restore config
-    tp_obj2 = metrics.TruePositives.from_config(tp_obj.get_config())
-    self.assertEqual(tp_obj2.name, 'my_tp')
-    self.assertEqual(len(tp_obj2.variables), 1)
-    self.assertEqual(tp_obj2.thresholds, [0.4, 0.9])
-
-  def test_unweighted(self):
-    tp_obj = metrics.TruePositives()
-    self.evaluate(variables.variables_initializer(tp_obj.variables))
-
-    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
-                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
-    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
-                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
-
-    update_op = tp_obj.update_state(y_true, y_pred)
-    self.evaluate(update_op)
-    result = tp_obj.result()
-    self.assertAllClose(7., result)
-
-  def test_weighted(self):
-    tp_obj = metrics.TruePositives()
-    self.evaluate(variables.variables_initializer(tp_obj.variables))
-    y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
-                                   (1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
-    y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
-                                   (0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
-    sample_weight = constant_op.constant((1., 1.5, 2., 2.5))
-    result = tp_obj(y_true, y_pred, sample_weight=sample_weight)
-    self.assertAllClose(12., self.evaluate(result))
-
-  def test_unweighted_with_thresholds(self):
-    tp_obj = metrics.TruePositives(thresholds=[0.15, 0.5, 0.85])
-    self.evaluate(variables.variables_initializer(tp_obj.variables))
-
-    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),
-                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))
-    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),
-                                   (1, 1, 1, 1)))
-
-    update_op = tp_obj.update_state(y_true, y_pred)
-    self.evaluate(update_op)
-    result = tp_obj.result()
-    self.assertAllClose([6., 3., 1.], result)
-
-  def test_weighted_with_thresholds(self):
-    tp_obj = metrics.TruePositives(thresholds=[0.15, 0.5, 0.85])
-    self.evaluate(variables.variables_initializer(tp_obj.variables))
-
-    y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),
-                                   (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))
-    y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),
-                                   (1, 1, 1, 1)))
-
-    result = tp_obj(y_true, y_pred, sample_weight=37.)
-    self.assertAllClose([222., 111., 37.], self.evaluate(result))
-
-
-@test_util.run_all_in_graph_and_eager_modes
-class PrecisionTest(test.TestCase):
-
-  def test_config(self):
-    p_obj = metrics.Precision(name='my_precision', thresholds=[0.4, 0.9])
-    self.assertEqual(p_obj.name, 'my_precision')
-    self.assertEqual(len(p_obj.variables), 2)
-    self.assertEqual([v.name for v in p_obj.variables],
-                     ['true_positives:0', 'false_positives:0'])
-    self.assertEqual(p_obj.thresholds, [0.4, 0.9])
-
-    # Check save and restore config
-    p_obj2 = metrics.Precision.from_config(p_obj.get_config())
-    self.assertEqual(p_obj2.name, 'my_precision')
-    self.assertEqual(len(p_obj2.variables), 2)
-    self.assertEqual(p_obj2.thresholds, [0.4, 0.9])
-
-  def test_value_is_idempotent(self):
-    p_obj = metrics.Precision(thresholds=[0.3, 0.72])
-    y_pred = random_ops.random_uniform(shape=(10, 3))
-    y_true = random_ops.random_uniform(shape=(10, 3))
-    update_op = p_obj.update_state(y_true, y_pred)
-    self.evaluate(variables.variables_initializer(p_obj.variables))
-
-    # Run several updates.
-    for _ in range(10):
-      self.evaluate(update_op)
-
-    # Then verify idempotency.
-    initial_precision = self.evaluate(p_obj.result())
-    for _ in range(10):
-      self.assertArrayNear(initial_precision, self.evaluate(p_obj.result()),
-                           1e-3)
-
-  def test_unweighted(self):
-    p_obj = metrics.Precision()
-    y_pred = constant_op.constant([1, 0, 1, 0], shape=(1, 4))
-    y_true = constant_op.constant([0, 1, 1, 0], shape=(1, 4))
-    self.evaluate(variables.variables_initializer(p_obj.variables))
-    result = p_obj(y_true, y_pred)
-    self.assertAlmostEqual(0.5, self.evaluate(result))
-
-  def test_unweighted_all_incorrect(self):
-    p_obj = metrics.Precision(thresholds=[0.5])
-    inputs = np.random.randint(0, 2, size=(100, 1))
-    y_pred = constant_op.constant(inputs)
-    y_true = constant_op.constant(1 - inputs)
-    self.evaluate(variables.variables_initializer(p_obj.variables))
-    result = p_obj(y_true, y_pred)
-    self.assertAlmostEqual(0, self.evaluate(result))
-
-  def test_weighted(self):
-    p_obj = metrics.Precision()
-    y_pred = constant_op.constant([[1, 0, 1, 0], [1, 0, 1, 0]])
-    y_true = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]])
-    self.evaluate(variables.variables_initializer(p_obj.variables))
-    result = p_obj(
-        y_true,
-        y_pred,
-        sample_weight=constant_op.constant([[1, 2, 3, 4], [4, 3, 2, 1]]))
-    weighted_tp = 3.0 + 4.0
-    weighted_positives = (1.0 + 3.0) + (4.0 + 2.0)
-    expected_precision = weighted_tp / weighted_positives
-    self.assertAlmostEqual(expected_precision, self.evaluate(result))
-
-  def test_div_by_zero(self):
-    p_obj = metrics.Precision()
-    y_pred = constant_op.constant([0, 0, 0, 0])
-    y_true = constant_op.constant([0, 0, 0, 0])
-    self.evaluate(variables.variables_initializer(p_obj.variables))
-    result = p_obj(y_true, y_pred)
-    self.assertEqual(0, self.evaluate(result))
-
-  def test_unweighted_with_threshold(self):
-    p_obj = metrics.Precision(thresholds=[0.5, 0.7])
-    y_pred = constant_op.constant([1, 0, 0.6, 0], shape=(1, 4))
-    y_true = constant_op.constant([0, 1, 1, 0], shape=(1, 4))
-    self.evaluate(variables.variables_initializer(p_obj.variables))
-    result = p_obj(y_true, y_pred)
-    self.assertArrayNear([0.5, 0.], self.evaluate(result), 0)
-
-  def test_weighted_with_threshold(self):
-    p_obj = metrics.Precision(thresholds=[0.5, 1.])
-    y_true = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2))
-    y_pred = constant_op.constant([[1, 0], [0.6, 0]],
-                                  shape=(2, 2),
-                                  dtype=dtypes.float32)
-    weights = constant_op.constant([[4, 0], [3, 1]],
-                                   shape=(2, 2),
-                                   dtype=dtypes.float32)
-    self.evaluate(variables.variables_initializer(p_obj.variables))
-    result = p_obj(y_true, y_pred, sample_weight=weights)
-    weighted_tp = 0 + 3.
-    weighted_positives = (0 + 3.) + (4. + 0.)
-    expected_precision = weighted_tp / weighted_positives
-    self.assertArrayNear([expected_precision, 0], self.evaluate(result), 1e-3)
-
-  def test_multiple_updates(self):
-    p_obj = metrics.Precision(thresholds=[0.5, 1.])
-    y_true = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2))
-    y_pred = constant_op.constant([[1, 0], [0.6, 0]],
-                                  shape=(2, 2),
-                                  dtype=dtypes.float32)
-    weights = constant_op.constant([[4, 0], [3, 1]],
-                                   shape=(2, 2),
-                                   dtype=dtypes.float32)
-    self.evaluate(variables.variables_initializer(p_obj.variables))
-    update_op = p_obj.update_state(y_true, y_pred, sample_weight=weights)
-    for _ in range(2):
-      self.evaluate(update_op)
-
-    weighted_tp = (0 + 3.) + (0 + 3.)
-    weighted_positives = ((0 + 3.) + (4. + 0.)) + ((0 + 3.) + (4. + 0.))
-    expected_precision = weighted_tp / weighted_positives
-    self.assertArrayNear([expected_precision, 0], self.evaluate(p_obj.result()),
-                         1e-3)
-
-
-@test_util.run_all_in_graph_and_eager_modes
-class RecallTest(test.TestCase):
-
-  def test_config(self):
-    r_obj = metrics.Recall(name='my_recall', thresholds=[0.4, 0.9])
-    self.assertEqual(r_obj.name, 'my_recall')
-    self.assertEqual(len(r_obj.variables), 2)
-    self.assertEqual([v.name for v in r_obj.variables],
-                     ['true_positives:0', 'false_negatives:0'])
-    self.assertEqual(r_obj.thresholds, [0.4, 0.9])
-
-    # Check save and restore config
-    r_obj2 = metrics.Recall.from_config(r_obj.get_config())
-    self.assertEqual(r_obj2.name, 'my_recall')
-    self.assertEqual(len(r_obj2.variables), 2)
-    self.assertEqual(r_obj2.thresholds, [0.4, 0.9])
-
-  def test_value_is_idempotent(self):
-    r_obj = metrics.Recall(thresholds=[0.3, 0.72])
-    y_pred = random_ops.random_uniform(shape=(10, 3))
-    y_true = random_ops.random_uniform(shape=(10, 3))
-    update_op = r_obj.update_state(y_true, y_pred)
-    self.evaluate(variables.variables_initializer(r_obj.variables))
-
-    # Run several updates.
-    for _ in range(10):
-      self.evaluate(update_op)
-
-    # Then verify idempotency.
-    initial_recall = self.evaluate(r_obj.result())
-    for _ in range(10):
-      self.assertArrayNear(initial_recall, self.evaluate(r_obj.result()), 1e-3)
-
-  def test_unweighted(self):
-    r_obj = metrics.Recall()
-    y_pred = constant_op.constant([1, 0, 1, 0], shape=(1, 4))
-    y_true = constant_op.constant([0, 1, 1, 0], shape=(1, 4))
-    self.evaluate(variables.variables_initializer(r_obj.variables))
-    result = r_obj(y_true, y_pred)
-    self.assertAlmostEqual(0.5, self.evaluate(result))
-
-  def test_unweighted_all_incorrect(self):
-    r_obj = metrics.Recall(thresholds=[0.5])
-    inputs = np.random.randint(0, 2, size=(100, 1))
-    y_pred = constant_op.constant(inputs)
-    y_true = constant_op.constant(1 - inputs)
-    self.evaluate(variables.variables_initializer(r_obj.variables))
-    result = r_obj(y_true, y_pred)
-    self.assertAlmostEqual(0, self.evaluate(result))
-
-  def test_weighted(self):
-    r_obj = metrics.Recall()
-    y_pred = constant_op.constant([[1, 0, 1, 0], [0, 1, 0, 1]])
-    y_true = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]])
-    self.evaluate(variables.variables_initializer(r_obj.variables))
-    result = r_obj(
-        y_true,
-        y_pred,
-        sample_weight=constant_op.constant([[1, 2, 3, 4], [4, 3, 2, 1]]))
-    weighted_tp = 3.0 + 1.0
-    weighted_t = (2.0 + 3.0) + (4.0 + 1.0)
-    expected_recall = weighted_tp / weighted_t
-    self.assertAlmostEqual(expected_recall, self.evaluate(result))
-
-  def test_div_by_zero(self):
-    r_obj = metrics.Recall()
-    y_pred = constant_op.constant([0, 0, 0, 0])
-    y_true = constant_op.constant([0, 0, 0, 0])
-    self.evaluate(variables.variables_initializer(r_obj.variables))
-    result = r_obj(y_true, y_pred)
-    self.assertEqual(0, self.evaluate(result))
-
-  def test_unweighted_with_threshold(self):
-    r_obj = metrics.Recall(thresholds=[0.5, 0.7])
-    y_pred = constant_op.constant([1, 0, 0.6, 0], shape=(1, 4))
-    y_true = constant_op.constant([0, 1, 1, 0], shape=(1, 4))
-    self.evaluate(variables.variables_initializer(r_obj.variables))
-    result = r_obj(y_true, y_pred)
-    self.assertArrayNear([0.5, 0.], self.evaluate(result), 0)
-
-  def test_weighted_with_threshold(self):
-    r_obj = metrics.Recall(thresholds=[0.5, 1.])
-    y_true = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2))
-    y_pred = constant_op.constant([[1, 0], [0.6, 0]],
-                                  shape=(2, 2),
-                                  dtype=dtypes.float32)
-    weights = constant_op.constant([[1, 4], [3, 2]],
-                                   shape=(2, 2),
-                                   dtype=dtypes.float32)
-    self.evaluate(variables.variables_initializer(r_obj.variables))
-    result = r_obj(y_true, y_pred, sample_weight=weights)
-    weighted_tp = 0 + 3.
-    weighted_positives = (0 + 3.) + (4. + 0.)
-    expected_recall = weighted_tp / weighted_positives
-    self.assertArrayNear([expected_recall, 0], self.evaluate(result), 1e-3)
-
-  def test_multiple_updates(self):
-    r_obj = metrics.Recall(thresholds=[0.5, 1.])
-    y_true = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2))
-    y_pred = constant_op.constant([[1, 0], [0.6, 0]],
-                                  shape=(2, 2),
-                                  dtype=dtypes.float32)
-    weights = constant_op.constant([[1, 4], [3, 2]],
-                                   shape=(2, 2),
-                                   dtype=dtypes.float32)
-    self.evaluate(variables.variables_initializer(r_obj.variables))
-    update_op = r_obj.update_state(y_true, y_pred, sample_weight=weights)
-    for _ in range(2):
-      self.evaluate(update_op)
-
-    weighted_tp = (0 + 3.) + (0 + 3.)
-    weighted_positives = ((0 + 3.) + (4. + 0.)) + ((0 + 3.) + (4. + 0.))
-    expected_recall = weighted_tp / weighted_positives
-    self.assertArrayNear([expected_recall, 0], self.evaluate(r_obj.result()),
-                         1e-3)
-
-
-@test_util.run_all_in_graph_and_eager_modes
-class SensitivityAtSpecificityTest(test.TestCase, parameterized.TestCase):
-
-  def test_config(self):
-    s_obj = metrics.SensitivityAtSpecificity(
-        0.4, num_thresholds=100, name='sensitivity_at_specificity_1')
-    self.assertEqual(s_obj.name, 'sensitivity_at_specificity_1')
-    self.assertLen(s_obj.variables, 4)
-    self.assertEqual(s_obj.specificity, 0.4)
-    self.assertEqual(s_obj.num_thresholds, 100)
-
-    # Check save and restore config
-    s_obj2 = metrics.SensitivityAtSpecificity.from_config(s_obj.get_config())
-    self.assertEqual(s_obj2.name, 'sensitivity_at_specificity_1')
-    self.assertLen(s_obj2.variables, 4)
-    self.assertEqual(s_obj2.specificity, 0.4)
-    self.assertEqual(s_obj2.num_thresholds, 100)
-
-  def test_value_is_idempotent(self):
-    s_obj = metrics.SensitivityAtSpecificity(0.7)
-    y_pred = random_ops.random_uniform((10, 3),
-                                       maxval=1,
-                                       dtype=dtypes.float32,
-                                       seed=1)
-    y_true = random_ops.random_uniform((10, 3),
-                                       maxval=2,
-                                       dtype=dtypes.int64,
-                                       seed=1)
-    update_op = s_obj.update_state(y_true, y_pred)
-    self.evaluate(variables.variables_initializer(s_obj.variables))
-
-    # Run several updates.
-    for _ in range(10):
-      self.evaluate(update_op)
-
-    # Then verify idempotency.
-    initial_sensitivity = self.evaluate(s_obj.result())
-    for _ in range(10):
-      self.assertAlmostEqual(initial_sensitivity, self.evaluate(s_obj.result()),
-                             1e-3)
-
-  def test_unweighted_all_correct(self):
-    s_obj = metrics.SensitivityAtSpecificity(0.7)
-    inputs = np.random.randint(0, 2, size=(100, 1))
-    y_pred = constant_op.constant(inputs, dtype=dtypes.float32)
-    y_true = constant_op.constant(inputs)
-    self.evaluate(variables.variables_initializer(s_obj.variables))
-    result = s_obj(y_true, y_pred)
-    self.assertAlmostEqual(1, self.evaluate(result))
-
-  def test_unweighted_high_specificity(self):
-    s_obj = metrics.SensitivityAtSpecificity(0.8)
-    pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.1, 0.45, 0.5, 0.8, 0.9]
-    label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]
-
-    y_pred = constant_op.constant(pred_values, dtype=dtypes.float32)
-    y_true = constant_op.constant(label_values)
-    self.evaluate(variables.variables_initializer(s_obj.variables))
-    result = s_obj(y_true, y_pred)
-    self.assertAlmostEqual(0.8, self.evaluate(result))
-
-  def test_unweighted_low_specificity(self):
-    s_obj = metrics.SensitivityAtSpecificity(0.4)
-    pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.01, 0.02, 0.25, 0.26, 0.26]
-    label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]
-
-    y_pred = constant_op.constant(pred_values, dtype=dtypes.float32)
-    y_true = constant_op.constant(label_values)
-    self.evaluate(variables.variables_initializer(s_obj.variables))
-    result = s_obj(y_true, y_pred)
-    self.assertAlmostEqual(0.6, self.evaluate(result))
-
-  @parameterized.parameters([dtypes.bool, dtypes.int32, dtypes.float32])
-  def test_weighted(self, label_dtype):
-    s_obj = metrics.SensitivityAtSpecificity(0.4)
-    pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.01, 0.02, 0.25, 0.26, 0.26]
-    label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]
-    weight_values = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
-
-    y_pred = constant_op.constant(pred_values, dtype=dtypes.float32)
-    y_true = math_ops.cast(label_values, dtype=label_dtype)
-    weights = constant_op.constant(weight_values)
-    self.evaluate(variables.variables_initializer(s_obj.variables))
-    result = s_obj(y_true, y_pred, sample_weight=weights)
-    self.assertAlmostEqual(0.675, self.evaluate(result))
-
-  def test_invalid_specificity(self):
-    with self.assertRaisesRegexp(
-        ValueError, r'`specificity` must be in the range \[0, 1\].'):
-      metrics.SensitivityAtSpecificity(-1)
-
-  def test_invalid_num_thresholds(self):
-    with self.assertRaisesRegexp(ValueError, '`num_thresholds` must be > 0.'):
-      metrics.SensitivityAtSpecificity(0.4, num_thresholds=-1)
-
-
-@test_util.run_all_in_graph_and_eager_modes
-class SpecificityAtSensitivityTest(test.TestCase, parameterized.TestCase):
-
-  def test_config(self):
-    s_obj = metrics.SpecificityAtSensitivity(
-        0.4, num_thresholds=100, name='specificity_at_sensitivity_1')
-    self.assertEqual(s_obj.name, 'specificity_at_sensitivity_1')
-    self.assertLen(s_obj.variables, 4)
-    self.assertEqual(s_obj.sensitivity, 0.4)
-    self.assertEqual(s_obj.num_thresholds, 100)
-
-    # Check save and restore config
-    s_obj2 = metrics.SpecificityAtSensitivity.from_config(s_obj.get_config())
-    self.assertEqual(s_obj2.name, 'specificity_at_sensitivity_1')
-    self.assertLen(s_obj2.variables, 4)
-    self.assertEqual(s_obj2.sensitivity, 0.4)
-    self.assertEqual(s_obj2.num_thresholds, 100)
-
-  def test_value_is_idempotent(self):
-    s_obj = metrics.SpecificityAtSensitivity(0.7)
-    y_pred = random_ops.random_uniform((10, 3),
-                                       maxval=1,
-                                       dtype=dtypes.float32,
-                                       seed=1)
-    y_true = random_ops.random_uniform((10, 3),
-                                       maxval=2,
-                                       dtype=dtypes.int64,
-                                       seed=1)
-    update_op = s_obj.update_state(y_true, y_pred)
-    self.evaluate(variables.variables_initializer(s_obj.variables))
-
-    # Run several updates.
-    for _ in range(10):
-      self.evaluate(update_op)
-
-    # Then verify idempotency.
-    initial_specificity = self.evaluate(s_obj.result())
-    for _ in range(10):
-      self.assertAlmostEqual(initial_specificity, self.evaluate(s_obj.result()),
-                             1e-3)
-
-  def test_unweighted_all_correct(self):
-    s_obj = metrics.SpecificityAtSensitivity(0.7)
-    inputs = np.random.randint(0, 2, size=(100, 1))
-    y_pred = constant_op.constant(inputs, dtype=dtypes.float32)
-    y_true = constant_op.constant(inputs)
-    self.evaluate(variables.variables_initializer(s_obj.variables))
-    result = s_obj(y_true, y_pred)
-    self.assertAlmostEqual(1, self.evaluate(result))
-
-  def test_unweighted_high_sensitivity(self):
-    s_obj = metrics.SpecificityAtSensitivity(0.8)
-    pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.1, 0.45, 0.5, 0.8, 0.9]
-    label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]
-
-    y_pred = constant_op.constant(pred_values, dtype=dtypes.float32)
-    y_true = constant_op.constant(label_values)
-    self.evaluate(variables.variables_initializer(s_obj.variables))
-    result = s_obj(y_true, y_pred)
-    self.assertAlmostEqual(0.4, self.evaluate(result))
-
-  def test_unweighted_low_sensitivity(self):
-    s_obj = metrics.SpecificityAtSensitivity(0.4)
-    pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.01, 0.02, 0.25, 0.26, 0.26]
-    label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]
-
-    y_pred = constant_op.constant(pred_values, dtype=dtypes.float32)
-    y_true = constant_op.constant(label_values)
-    self.evaluate(variables.variables_initializer(s_obj.variables))
-    result = s_obj(y_true, y_pred)
-    self.assertAlmostEqual(0.6, self.evaluate(result))
-
-  @parameterized.parameters([dtypes.bool, dtypes.int32, dtypes.float32])
-  def test_weighted(self, label_dtype):
-    s_obj = metrics.SpecificityAtSensitivity(0.4)
-    pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.01, 0.02, 0.25, 0.26, 0.26]
-    label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]
-    weight_values = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
-
-    y_pred = constant_op.constant(pred_values, dtype=dtypes.float32)
-    y_true = math_ops.cast(label_values, dtype=label_dtype)
-    weights = constant_op.constant(weight_values)
-    self.evaluate(variables.variables_initializer(s_obj.variables))
-    result = s_obj(y_true, y_pred, sample_weight=weights)
-    self.assertAlmostEqual(0.4, self.evaluate(result))
-
-  def test_invalid_sensitivity(self):
-    with self.assertRaisesRegexp(
-        ValueError, r'`sensitivity` must be in the range \[0, 1\].'):
-      metrics.SpecificityAtSensitivity(-1)
-
-  def test_invalid_num_thresholds(self):
-    with self.assertRaisesRegexp(ValueError, '`num_thresholds` must be > 0.'):
-      metrics.SpecificityAtSensitivity(0.4, num_thresholds=-1)
-
-
 @test_util.run_all_in_graph_and_eager_modes
 class CosineProximityTest(test.TestCase):
 
-- 
GitLab


From 7e3077a22530e253ab31417abb18de5bb60779ef Mon Sep 17 00:00:00 2001
From: Karim Nosir <karimnosseir@google.com>
Date: Thu, 10 Jan 2019 14:55:53 -0800
Subject: [PATCH 0550/1095] Add the data file to the data section in BUILD.

PiperOrigin-RevId: 228782621
---
 tensorflow/lite/java/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/lite/java/BUILD b/tensorflow/lite/java/BUILD
index adf7bc9087..a9d4a422c5 100644
--- a/tensorflow/lite/java/BUILD
+++ b/tensorflow/lite/java/BUILD
@@ -121,6 +121,7 @@ java_test(
         "src/testdata/int64.bin",
         "src/testdata/invalid_model.bin",
         "src/testdata/quantized.bin",
+        "src/testdata/string.bin",
         "src/testdata/uint8.bin",
         "src/testdata/with_custom_op.lite",
     ],
-- 
GitLab


From d08fa433af627143e23624e9aaadf32373f4db0e Mon Sep 17 00:00:00 2001
From: Thomas O'Malley <omalleyt@google.com>
Date: Thu, 10 Jan 2019 15:27:19 -0800
Subject: [PATCH 0551/1095] Fix error message when Subclassing Network without
 calling super's __init__ first.

PiperOrigin-RevId: 228788216
---
 tensorflow/python/keras/engine/network.py       |  8 +++++---
 tensorflow/python/keras/engine/topology_test.py | 10 ++++++++++
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/keras/engine/network.py b/tensorflow/python/keras/engine/network.py
index 8e130aef40..6c3691a30c 100644
--- a/tensorflow/python/keras/engine/network.py
+++ b/tensorflow/python/keras/engine/network.py
@@ -335,9 +335,11 @@ class Network(base_layer.Layer):
     if not getattr(self, '_setattr_tracking', True):
       super(Network, self).__setattr__(name, value)
       return
-    if (isinstance(value, (base_layer.Layer,
-                           data_structures.CheckpointableDataStructure))
-        or checkpointable_layer_utils.has_weights(value)):
+
+    if all(
+        isinstance(v, (base_layer.Layer,
+                       data_structures.CheckpointableDataStructure)) or
+        checkpointable_layer_utils.has_weights(v) for v in nest.flatten(value)):
       try:
         self._is_graph_network
       except AttributeError:
diff --git a/tensorflow/python/keras/engine/topology_test.py b/tensorflow/python/keras/engine/topology_test.py
index e9aeae75ce..6c2835aece 100644
--- a/tensorflow/python/keras/engine/topology_test.py
+++ b/tensorflow/python/keras/engine/topology_test.py
@@ -920,6 +920,16 @@ class TopologyConstructionTest(keras_parameterized.TestCase):
       yaml_str = model.to_yaml()
       keras.models.model_from_yaml(yaml_str)
 
+  def test_subclassed_error_if_init_not_called(self):
+
+    class MyNetwork(network_lib.Network):
+
+      def __init__(self):
+        self._foo = [keras.layers.Dense(10), keras.layers.Dense(10)]
+
+    with self.assertRaisesRegexp(RuntimeError, 'forgot to call'):
+      MyNetwork()
+
 
 class DeferredModeTest(test.TestCase):
 
-- 
GitLab


From d6623c9341cda24851827e5069fec810860f1616 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 10 Jan 2019 16:08:49 -0800
Subject: [PATCH 0552/1095] A context manager api for eager profiler.

PiperOrigin-RevId: 228796096
---
 tensorflow/python/eager/profiler.py | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/tensorflow/python/eager/profiler.py b/tensorflow/python/eager/profiler.py
index 016bca3e3c..f1d4373085 100644
--- a/tensorflow/python/eager/profiler.py
+++ b/tensorflow/python/eager/profiler.py
@@ -23,6 +23,7 @@ import threading
 from tensorflow.python import pywrap_tensorflow
 from tensorflow.python.eager import context
 from tensorflow.python.framework import c_api_util
+from tensorflow.python.platform import gfile
 
 _profiler = None
 _profiler_lock = threading.Lock()
@@ -67,3 +68,26 @@ def stop():
     pywrap_tensorflow.TFE_DeleteProfiler(_profiler)
     _profiler = None
   return result
+
+
+class Profiler(object):
+  """Context-manager eager profiler api.
+
+  Example usage:
+  ```python
+  with Profiler("/path/to/save/result"):
+    # do some work
+  ```
+  """
+
+  def __init__(self, filename):
+    self._filename = filename
+
+  def __enter__(self):
+    start()
+
+  def __exit__(self, typ, value, tb):
+    result = stop()
+    with gfile.Open(self._filename, 'wb') as f:
+      f.write(result)
+
-- 
GitLab


From 0b3ef83939b9deb2a32c5afae254555f395e0935 Mon Sep 17 00:00:00 2001
From: Andy Ly <lyandy@google.com>
Date: Thu, 10 Jan 2019 16:15:12 -0800
Subject: [PATCH 0553/1095] [Grappler] Update UpdateFanouts and UpdateFanin to
 skip mutations when to_node is a Switch that is being added as a control
 dependency.

PiperOrigin-RevId: 228797109
---
 .../core/grappler/mutable_graph_view.cc       | 115 ++++++-----
 tensorflow/core/grappler/mutable_graph_view.h |  11 +-
 .../core/grappler/mutable_graph_view_test.cc  | 181 ++++++++++++++----
 3 files changed, 209 insertions(+), 98 deletions(-)

diff --git a/tensorflow/core/grappler/mutable_graph_view.cc b/tensorflow/core/grappler/mutable_graph_view.cc
index a8a90d1a57..5acdb02dee 100644
--- a/tensorflow/core/grappler/mutable_graph_view.cc
+++ b/tensorflow/core/grappler/mutable_graph_view.cc
@@ -185,6 +185,9 @@ NodeDef* MutableGraphView::AddNode(NodeDef&& node) {
 
 bool MutableGraphView::UpdateFanouts(absl::string_view from_node,
                                      absl::string_view to_node) {
+  if (from_node == to_node) {
+    return false;
+  }
   NodeDef* from_node_ptr = GetNode(from_node);
   NodeDef* to_node_ptr = GetNode(to_node);
   if (from_node_ptr && to_node_ptr) {
@@ -218,6 +221,35 @@ bool MutableGraphView::UpdateFanoutsInternal(NodeDef* from_node,
     fanouts()[output_port].erase(input_port);
   };
 
+  bool modified = false;
+
+  // For the control fanouts we do not know the input index in a NodeDef,
+  // so we have to traverse all control inputs.
+
+  auto control_fanouts =
+      GetFanout(GraphView::OutputPort(from_node, Graph::kControlSlot));
+
+  bool to_node_is_switch = IsSwitch(*to_node);
+  for (const InputPort& control_port : control_fanouts) {
+    // Node can't be control dependency of itself.
+    if (control_port.node == to_node) continue;
+
+    // Can't add Switch node as a control dependency.
+    if (to_node_is_switch) {
+      // This will only be printed to the log if we are trying to add a Switch
+      // as a control dependency, which if allowed will make the graph invalid.
+      LOG(WARNING) << absl::Substitute(
+          "Can't update fanouts from '$0' to '$1', to node is being added as a "
+          "control dependency when it is a Switch.",
+          from_node, to_node);
+      return false;
+    }
+
+    NodeDef* node = control_port.node;
+    modified |= RemoveControllingFaninInternal(node, from_node);
+    modified |= AddFaninInternal(node, {to_node, Graph::kControlSlot});
+  }
+
   // First we update regular fanouts. For the regular fanouts
   // `input_port:port_id` is the input index in NodeDef.
 
@@ -228,7 +260,6 @@ bool MutableGraphView::UpdateFanoutsInternal(NodeDef* from_node,
   // input to some other node.
   int keep_max_regular_output_port = -1;
 
-  bool modified = false;
   for (const Edge& edge : regular_edges) {
     const OutputPort output_port = edge.src;
     const InputPort input_port = edge.dst;
@@ -259,22 +290,6 @@ bool MutableGraphView::UpdateFanoutsInternal(NodeDef* from_node,
     modified = true;
   }
 
-  // For the control fanouts we do not know the input index in a NodeDef,
-  // so we have to traverse all control inputs.
-
-  auto control_fanouts =
-      GetFanout(GraphView::OutputPort(from_node, Graph::kControlSlot));
-
-  for (const InputPort& control_port : control_fanouts) {
-    // Node can't be control dependency of itself.
-    if (control_port.node == to_node) continue;
-
-    NodeDef* node = control_port.node;
-    modified |= RemoveControllingFaninInternal(node, from_node);
-    // TODO(lyandy): Handle Switch control dependencies.
-    modified |= AddFaninInternal(node, {to_node, Graph::kControlSlot});
-  }
-
   // Because we update all regular fanouts of `from_node`, we can just copy
   // the value `num_regular_outputs`.
   max_regular_output_port()[to_node] = max_regular_output_port()[from_node];
@@ -371,7 +386,7 @@ bool MutableGraphView::AddControllingFanin(absl::string_view node_name,
     return AddFaninInternal(node, {fanin_node, Graph::kControlSlot});
   } else {
     if (IsTensorIdControlling(fanin)) {
-      // Cannot add a Switch node control dependency.
+      // Can't add a Switch node control dependency.
       return false;
     }
     // We can't anchor control dependencies directly on the switch node: unlike
@@ -410,11 +425,10 @@ bool MutableGraphView::AddControllingFanin(absl::string_view node_name,
 }
 
 bool MutableGraphView::RemoveRegularFaninInternal(NodeDef* node,
-                                                  const TensorId& fanin) {
-  auto remove_input = [this, node](const TensorId& tensor_id, int port,
-                                   bool update_max_port) {
-    OutputPort fanin_port(nodes()[tensor_id.node()], tensor_id.index());
-    InputPort input(node, port);
+                                                  const OutputPort& fanin) {
+  auto remove_input = [this, node](const OutputPort& fanin_port,
+                                   int node_input_port, bool update_max_port) {
+    InputPort input(node, node_input_port);
 
     absl::flat_hash_set<InputPort>* fanouts_set = &fanouts()[fanin_port];
     fanouts_set->erase(input);
@@ -434,12 +448,14 @@ bool MutableGraphView::RemoveRegularFaninInternal(NodeDef* node,
     if (IsTensorIdControlling(tensor_id)) {
       break;
     }
-    if (tensor_id == fanin) {
-      remove_input(tensor_id, i, /*update_max_port=*/true);
+    if (tensor_id.node() == fanin.node->name() &&
+        tensor_id.index() == fanin.port_id) {
+      remove_input(fanin, i, /*update_max_port=*/true);
       modified = true;
     } else if (modified) {
       // Regular inputs will need to have their ports updated.
-      auto fanouts_set = remove_input(tensor_id, i, /*update_max_port=*/false);
+      OutputPort fanin_port(nodes()[tensor_id.node()], tensor_id.index());
+      auto fanouts_set = remove_input(fanin_port, i, /*update_max_port=*/false);
       fanouts_set->insert({node, curr_pos});
       // Shift inputs to be retained.
       mutable_inputs->SwapElements(i, curr_pos++);
@@ -466,7 +482,11 @@ bool MutableGraphView::RemoveRegularFanin(absl::string_view node_name,
   if (node == nullptr) {
     return false;
   }
-  return RemoveRegularFaninInternal(node, fanin);
+  NodeDef* fanin_node = GetNode(fanin.node());
+  if (fanin_node == nullptr) {
+    return false;
+  }
+  return RemoveRegularFaninInternal(node, {fanin_node, fanin.index()});
 }
 
 bool MutableGraphView::RemoveControllingFaninInternal(NodeDef* node,
@@ -487,22 +507,17 @@ bool MutableGraphView::RemoveControllingFaninInternal(NodeDef* node,
   return false;
 }
 
-bool MutableGraphView::RemoveControllingFaninInternal(
-    NodeDef* node, absl::string_view fanin_node_name) {
-  NodeDef* fanin = GetNode(fanin_node_name);
-  if (fanin == nullptr) {
-    return false;
-  }
-  return RemoveControllingFaninInternal(node, fanin);
-}
-
 bool MutableGraphView::RemoveControllingFanin(
     absl::string_view node_name, absl::string_view fanin_node_name) {
   NodeDef* node = GetNode(node_name);
   if (node == nullptr) {
     return false;
   }
-  return RemoveControllingFaninInternal(node, fanin_node_name);
+  NodeDef* fanin = GetNode(fanin_node_name);
+  if (fanin == nullptr) {
+    return false;
+  }
+  return RemoveControllingFaninInternal(node, fanin);
 }
 
 bool MutableGraphView::RemoveAllFanins(absl::string_view node_name,
@@ -540,30 +555,36 @@ bool MutableGraphView::UpdateFanin(absl::string_view node_name,
     return false;
   }
 
+  NodeDef* from_fanin_node = GetNode(from_fanin.node());
+  NodeDef* to_fanin_node = GetNode(to_fanin.node());
+  if (from_fanin_node == nullptr || to_fanin_node == nullptr) {
+    return false;
+  }
+
   // When replacing a non control dependency fanin with a control dependency, or
   // vice versa, remove and add, so ports can be updated properly in fanout(s).
   bool from_fanin_is_control = IsTensorIdControlling(from_fanin);
-  if (from_fanin_is_control || IsTensorIdControlling(to_fanin)) {
+  bool to_fanin_is_control = IsTensorIdControlling(to_fanin);
+  if (to_fanin_is_control && IsSwitch(*to_fanin_node)) {
+    // Can't add Switch node as a control dependency.
+    return false;
+  }
+  if (from_fanin_is_control || to_fanin_is_control) {
     bool modified = false;
     if (from_fanin_is_control) {
-      modified |= RemoveControllingFaninInternal(node, from_fanin.node());
+      modified |= RemoveControllingFaninInternal(node, from_fanin_node);
     } else {
-      modified |= RemoveRegularFaninInternal(node, from_fanin);
+      modified |= RemoveRegularFaninInternal(
+          node, {from_fanin_node, from_fanin.index()});
     }
     if (modified) {
-      AddFaninInternal(node, to_fanin);
+      AddFaninInternal(node, {to_fanin_node, to_fanin.index()});
     }
 
     return modified;
   }
 
   // In place mutation, requires no shifting of ports.
-  NodeDef* from_fanin_node = GetNode(from_fanin.node());
-  NodeDef* to_fanin_node = GetNode(to_fanin.node());
-  if (from_fanin_node == nullptr || to_fanin_node == nullptr) {
-    return false;
-  }
-
   string to_fanin_string = TensorIdToString(to_fanin);
   int num_inputs = node->input_size();
   bool modified = false;
diff --git a/tensorflow/core/grappler/mutable_graph_view.h b/tensorflow/core/grappler/mutable_graph_view.h
index f07254068d..0a441d9115 100644
--- a/tensorflow/core/grappler/mutable_graph_view.h
+++ b/tensorflow/core/grappler/mutable_graph_view.h
@@ -195,17 +195,10 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   // already exists, the node will not be modified.
   bool AddFaninInternal(NodeDef* node, const TensorId& fanin);
 
-  // Removes any fanin in node that matches to a fanin in fanins.
+  // Removes all instances of regular fanin `fanin` from node `node`.
   //
   // This will return true iff the node is modified.
-  bool RemoveRegularFaninInternal(NodeDef* node, const TensorId& fanin);
-
-  // Removes controlling fanin `fanin_node_name` from node if such controlling
-  // fanin exists.
-  //
-  // This will return true iff the node is modified.
-  bool RemoveControllingFaninInternal(NodeDef* node,
-                                      absl::string_view fanin_node_name);
+  bool RemoveRegularFaninInternal(NodeDef* node, const OutputPort& fanin);
 
   // Removes controlling fanin `fanin_node` from node if such controlling fanin
   // exists.
diff --git a/tensorflow/core/grappler/mutable_graph_view_test.cc b/tensorflow/core/grappler/mutable_graph_view_test.cc
index 2048c67e3e..b44e1a360d 100644
--- a/tensorflow/core/grappler/mutable_graph_view_test.cc
+++ b/tensorflow/core/grappler/mutable_graph_view_test.cc
@@ -168,21 +168,6 @@ TEST(MutableGraphViewTest, AddAndUpdateFanoutsWithoutSelfLoops) {
   EXPECT_EQ(new_bar_fanouts.count(MutableGraphView::InputPort(foo_2, -1)), 1);
 }
 
-GraphDef SimpleMutateFaninGraph() {
-  // Actual node.op() is not important in this test.
-  GraphDef graph_def = test::function::GDef(
-      {NDef("a", "NotImportant", {}, {}), NDef("b", "NotImportant", {}, {}),
-       NDef("c", "NotImportant", {}, {}), NDef("d", "NotImportant", {}, {}),
-       NDef("foo_1", "NotImportant", {"a"}),
-       NDef("foo_2", "NotImportant", {"b", "^a", "^c"}),
-       NDef("foo_3", "NotImportant", {"b", "a:1", "a:1"}),
-       NDef("foo_4", "NotImportant", {"a", "b:2", "b:2", "^c", "^d"}),
-       NDef("foo_5", "NotImportant", {}),
-       NDef("foo_6", "NotImportant", {"^a", "^b"})},
-      /*funcs=*/{});
-  return graph_def;
-}
-
 void CompareNodeInputs(const MutableGraphView& graph, const NodeDef* expected,
                        NodeDef* actual) {
   ASSERT_EQ(actual->input_size(), expected->input_size());
@@ -203,6 +188,94 @@ void CompareNodeInputs(const MutableGraphView& graph, const NodeDef* expected,
   }
 }
 
+TEST(MutableGraphViewTest, UpdateFanoutsToSwitchWithControlFromSwitch) {
+  GraphDef graph_def = test::function::GDef(
+      {NDef("a", "NotImportant", {}, {}), NDef("b", "Switch", {}, {}),
+       NDef("c", "NotImportant", {}, {}), NDef("d", "NotImportant", {}, {}),
+       NDef("e", "NotImportant", {"c", "b", "^a", "^d"})},
+      /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  EXPECT_FALSE(graph.UpdateFanouts("a", "b"));
+  EXPECT_FALSE(graph.UpdateFanouts("d", "b"));
+
+  EXPECT_EQ(graph.graph()->node_size(), 5);
+
+  NodeDef expected = NDef("", "", {}, {});
+  NodeDef* a = graph.GetNode("a");
+  ASSERT_NE(a, nullptr);
+  CompareNodeInputs(graph, &expected, a);
+
+  NodeDef* b = graph.GetNode("b");
+  ASSERT_NE(b, nullptr);
+  CompareNodeInputs(graph, &expected, b);
+
+  NodeDef* c = graph.GetNode("c");
+  ASSERT_NE(c, nullptr);
+  CompareNodeInputs(graph, &expected, c);
+
+  NodeDef* d = graph.GetNode("d");
+  ASSERT_NE(d, nullptr);
+  CompareNodeInputs(graph, &expected, d);
+
+  NodeDef* e = graph.GetNode("e");
+  ASSERT_NE(e, nullptr);
+  expected = NDef("", "", {"c", "b", "^a", "^d"});
+  CompareNodeInputs(graph, &expected, e);
+}
+
+TEST(MutableGraphViewTest, UpdateFanoutsToSwitchWithNoControlFromSwitch) {
+  GraphDef graph_def = test::function::GDef(
+      {NDef("a", "NotImportant", {}, {}), NDef("b", "Switch", {}, {}),
+       NDef("c", "NotImportant", {}, {}), NDef("d", "NotImportant", {}, {}),
+       NDef("e", "NotImportant", {"c", "b", "^a", "^d"})},
+      /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  EXPECT_TRUE(graph.UpdateFanouts("c", "b"));
+
+  EXPECT_EQ(graph.graph()->node_size(), 5);
+
+  NodeDef expected = NDef("", "", {}, {});
+  NodeDef* a = graph.GetNode("a");
+  ASSERT_NE(a, nullptr);
+  CompareNodeInputs(graph, &expected, a);
+
+  NodeDef* b = graph.GetNode("b");
+  ASSERT_NE(b, nullptr);
+  CompareNodeInputs(graph, &expected, b);
+
+  NodeDef* c = graph.GetNode("c");
+  ASSERT_NE(c, nullptr);
+  CompareNodeInputs(graph, &expected, c);
+
+  NodeDef* d = graph.GetNode("d");
+  ASSERT_NE(d, nullptr);
+  CompareNodeInputs(graph, &expected, d);
+
+  NodeDef* e = graph.GetNode("e");
+  ASSERT_NE(e, nullptr);
+  expected = NDef("", "", {"b", "b", "^a", "^d"});
+  CompareNodeInputs(graph, &expected, e);
+}
+
+GraphDef SimpleMutateFaninGraph() {
+  // Actual node.op() is not important in this test.
+  GraphDef graph_def = test::function::GDef(
+      {NDef("a", "NotImportant", {}, {}), NDef("b", "NotImportant", {}, {}),
+       NDef("c", "NotImportant", {}, {}), NDef("d", "NotImportant", {}, {}),
+       NDef("foo_1", "NotImportant", {"a"}),
+       NDef("foo_2", "NotImportant", {"b", "^a", "^c"}),
+       NDef("foo_3", "NotImportant", {"b", "a:1", "a:1"}),
+       NDef("foo_4", "NotImportant", {"a", "b:2", "b:2", "^c", "^d"}),
+       NDef("foo_5", "NotImportant", {}),
+       NDef("foo_6", "NotImportant", {"^a", "^b"})},
+      /*funcs=*/{});
+  return graph_def;
+}
+
 void TestAddRegularFanin(absl::string_view node_name,
                          const TensorId& fanin_to_add, bool modified,
                          const NodeDef* expected_node) {
@@ -525,23 +598,53 @@ TEST(MutableGraphViewTest, UpdateFanin) {
                   /*modified=*/false, /*expected_node=*/nullptr);
 }
 
+void TestUpdateFaninFromFaninToNodeAsSwitchControl(const TensorId& fanin) {
+  string tensor_id_str = TensorIdToString(fanin);
+  GraphDef graph_def = test::function::GDef(
+      {NDef("a", "NotImportant", {}, {}), NDef("b", "Switch", {}, {}),
+       NDef("c", "NotImportant", {tensor_id_str})},
+      /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  EXPECT_FALSE(graph.UpdateFanin("c", fanin, {"b", Graph::kControlSlot}));
+
+  EXPECT_EQ(graph.graph()->node_size(), 3);
+
+  NodeDef expected;
+  NodeDef* a = graph.GetNode("a");
+  ASSERT_NE(a, nullptr);
+  expected = NDef("", "", {});
+  CompareNodeInputs(graph, &expected, a);
+
+  NodeDef* b = graph.GetNode("b");
+  ASSERT_NE(b, nullptr);
+  CompareNodeInputs(graph, &expected, b);
+
+  NodeDef* c = graph.GetNode("c");
+  ASSERT_NE(c, nullptr);
+  expected = NDef("", "", {tensor_id_str});
+  CompareNodeInputs(graph, &expected, c);
+}
+
+TEST(MutableGraphViewTest, UpdateFaninToNodeAsSwitchControl) {
+  TestUpdateFaninFromFaninToNodeAsSwitchControl({"a", 0});
+  TestUpdateFaninFromFaninToNodeAsSwitchControl({"a", 1});
+  TestUpdateFaninFromFaninToNodeAsSwitchControl({"a", Graph::kControlSlot});
+}
+
 TEST(MutableGraphViewTest, DedupControllingFaninsOnGraphInit) {
-  // Actual node.op() is not important in this test.
   GraphDef graph_def = test::function::GDef(
-      {
-          NDef("a", "NotImportant", {}, {}),
-          NDef("b", "NotImportant", {}, {}),
-          NDef("c", "Switch", {}, {}),
-          NDef("d", "Identity", {"c:1"}),
-          NDef("foo_1", "IdentityN", {"a", "b:1", "^b"}),
-          NDef("foo_2", "IdentityN", {"a", "^b", "^b"}),
-          NDef("foo_3", "IdentityN", {"a", "b:1", "^b", "^b"}),
-          NDef("foo_4", "IdentityN", {"a:2", "b:1", "^b", "^b", "^a", "^a"}),
-          NDef("foo_5", "NotImportant", {"a:2", "b:1", "^b", "^b", "^a", "^a"}),
-          NDef("foo_6", "Identity", {"d", "^d"}),
-          NDef("foo_7", "NotImportant",
-               {"a:3", "b:2", "d", "^d", "^d", "^a", "^b", "^a", "^b"}),
-      },
+      {NDef("a", "NotImportant", {}, {}), NDef("b", "NotImportant", {}, {}),
+       NDef("c", "Switch", {}, {}), NDef("d", "Identity", {"c:1"}),
+       NDef("foo_1", "IdentityN", {"a", "b:1", "^b"}),
+       NDef("foo_2", "IdentityN", {"a", "^b", "^b"}),
+       NDef("foo_3", "IdentityN", {"a", "b:1", "^b", "^b"}),
+       NDef("foo_4", "IdentityN", {"a:2", "b:1", "^b", "^b", "^a", "^a"}),
+       NDef("foo_5", "NotImportant", {"a:2", "b:1", "^b", "^b", "^a", "^a"}),
+       NDef("foo_6", "Identity", {"d", "^d"}),
+       NDef("foo_7", "NotImportant",
+            {"a:3", "b:2", "d", "^d", "^d", "^a", "^b", "^a", "^b"})},
       /*funcs=*/{});
 
   MutableGraphView graph(&graph_def);
@@ -939,12 +1042,9 @@ TEST(MutableGraphViewTest, AddControllingFaninSwitchWithExistingAddedIdentity) {
 TEST(MutableGraphViewTest, RemoveControllingFaninMissing) {
   // Actual node.op() is not important in this test.
   GraphDef graph_def = test::function::GDef(
-      {
-          NDef("a", "NotImportant", {}, {}),
-          NDef("b", "NotImportant", {}, {}),
-          NDef("c", "NotImportant", {}, {}),
-          NDef("d", "NotImportant", {"^a", "^b"}),
-      },
+      {NDef("a", "NotImportant", {}, {}), NDef("b", "NotImportant", {}, {}),
+       NDef("c", "NotImportant", {}, {}),
+       NDef("d", "NotImportant", {"^a", "^b"})},
       /*funcs=*/{});
 
   MutableGraphView graph(&graph_def);
@@ -961,12 +1061,9 @@ TEST(MutableGraphViewTest, RemoveControllingFaninMissing) {
 TEST(MutableGraphViewTest, RemoveControllingFaninExisting) {
   // Actual node.op() is not important in this test.
   GraphDef graph_def = test::function::GDef(
-      {
-          NDef("a", "NotImportant", {}, {}),
-          NDef("b", "NotImportant", {}, {}),
-          NDef("c", "NotImportant", {}, {}),
-          NDef("d", "NotImportant", {"^a", "^b", "^c"}),
-      },
+      {NDef("a", "NotImportant", {}, {}), NDef("b", "NotImportant", {}, {}),
+       NDef("c", "NotImportant", {}, {}),
+       NDef("d", "NotImportant", {"^a", "^b", "^c"})},
       /*funcs=*/{});
 
   MutableGraphView graph(&graph_def);
-- 
GitLab


From f497b36f3250a4e26051be3f3616e7ad233f3cc8 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Thu, 10 Jan 2019 16:17:39 -0800
Subject: [PATCH 0554/1095] Remove the experimental initialize and finalize
 API.

PiperOrigin-RevId: 228797485
---
 .../python/distribute/distribute_lib.py       | 42 -------------------
 ...orflow.distribute.-mirrored-strategy.pbtxt | 16 -------
 .../v1/tensorflow.distribute.-strategy.pbtxt  | 16 -------
 ...orflow.distribute.-mirrored-strategy.pbtxt | 16 -------
 .../v2/tensorflow.distribute.-strategy.pbtxt  | 16 -------
 5 files changed, 106 deletions(-)

diff --git a/tensorflow/python/distribute/distribute_lib.py b/tensorflow/python/distribute/distribute_lib.py
index 5fe77e5478..66112dceda 100644
--- a/tensorflow/python/distribute/distribute_lib.py
+++ b/tensorflow/python/distribute/distribute_lib.py
@@ -505,42 +505,6 @@ class DistributionStrategy(object):
     """DEPRECATED: use extended.broadcast_to() instead."""
     return self._extended.broadcast_to(tensor, destinations)
 
-  @doc_controls.do_not_generate_docs  # Use experimental_initialize() instead.
-  def initialize(self):
-    """DEPRECATED: Use `experimental_initialize()` instead."""
-    return self._extended._initialize()  # pylint: disable=protected-access
-
-  def experimental_initialize(self):
-    """Any initialization to be done before running any computations.
-
-    In eager mode, it executes any initialization as a side effect.
-    In graph mode, it creates the initialization ops and returns them.
-
-    For example, TPU initialize_system ops.
-
-    Returns:
-      A list of ops to execute.
-    """
-    return self._extended._initialize()  # pylint: disable=protected-access
-
-  @doc_controls.do_not_generate_docs  # Use experimental_finalize() instead.
-  def finalize(self):
-    """DEPRECATED: Use `experimental_finalize()` instead."""
-    return self._extended._finalize()  # pylint: disable=protected-access
-
-  def experimental_finalize(self):
-    """Any final actions to be done at the end of all computations.
-
-    In eager mode, it executes any finalize actions as a side effect.
-    In graph mode, it creates the finalize ops and returns them.
-
-    For example, TPU shutdown ops.
-
-    Returns:
-      A list of ops to execute.
-    """
-    return self._extended._finalize()  # pylint: disable=protected-access
-
   @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
   def run_steps_on_dataset(self, fn, iterator, iterations=1,
                            initial_loop_values=None):
@@ -1171,12 +1135,6 @@ class DistributionStrategyExtended(object):
   def _broadcast_to(self, tensor, destinations):
     raise NotImplementedError("must be implemented in descendants")
 
-  def _initialize(self):
-    return []
-
-  def _finalize(self):
-    return []
-
   def experimental_run_steps_on_iterator(self, fn, iterator, iterations=1,
                                          initial_loop_values=None):
     """Run `fn` with input from `iterator` for `iterations` times.
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-mirrored-strategy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-mirrored-strategy.pbtxt
index 9c29067b6d..863ee9c210 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-mirrored-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-mirrored-strategy.pbtxt
@@ -67,14 +67,6 @@ tf_class {
     name: "distribute_dataset"
     argspec: "args=[\'self\', \'dataset_fn\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "experimental_finalize"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "experimental_initialize"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "experimental_make_numpy_iterator"
     argspec: "args=[\'self\', \'numpy_input\', \'batch_size\', \'num_epochs\', \'shuffle\', \'session\'], varargs=None, keywords=None, defaults=[\'1\', \'1024\', \'None\'], "
@@ -83,18 +75,10 @@ tf_class {
     name: "experimental_run"
     argspec: "args=[\'self\', \'fn\', \'input_iterator\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "finalize"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "group"
     argspec: "args=[\'self\', \'value\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "initialize"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "make_dataset_iterator"
     argspec: "args=[\'self\', \'dataset\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt
index 4aa6f1c4e1..1c06f7873e 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt
@@ -66,14 +66,6 @@ tf_class {
     name: "distribute_dataset"
     argspec: "args=[\'self\', \'dataset_fn\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "experimental_finalize"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "experimental_initialize"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "experimental_make_numpy_iterator"
     argspec: "args=[\'self\', \'numpy_input\', \'batch_size\', \'num_epochs\', \'shuffle\', \'session\'], varargs=None, keywords=None, defaults=[\'1\', \'1024\', \'None\'], "
@@ -82,18 +74,10 @@ tf_class {
     name: "experimental_run"
     argspec: "args=[\'self\', \'fn\', \'input_iterator\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "finalize"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "group"
     argspec: "args=[\'self\', \'value\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "initialize"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "make_dataset_iterator"
     argspec: "args=[\'self\', \'dataset\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-mirrored-strategy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-mirrored-strategy.pbtxt
index 9c29067b6d..863ee9c210 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-mirrored-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-mirrored-strategy.pbtxt
@@ -67,14 +67,6 @@ tf_class {
     name: "distribute_dataset"
     argspec: "args=[\'self\', \'dataset_fn\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "experimental_finalize"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "experimental_initialize"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "experimental_make_numpy_iterator"
     argspec: "args=[\'self\', \'numpy_input\', \'batch_size\', \'num_epochs\', \'shuffle\', \'session\'], varargs=None, keywords=None, defaults=[\'1\', \'1024\', \'None\'], "
@@ -83,18 +75,10 @@ tf_class {
     name: "experimental_run"
     argspec: "args=[\'self\', \'fn\', \'input_iterator\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "finalize"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "group"
     argspec: "args=[\'self\', \'value\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "initialize"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "make_dataset_iterator"
     argspec: "args=[\'self\', \'dataset\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt
index 4aa6f1c4e1..1c06f7873e 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt
@@ -66,14 +66,6 @@ tf_class {
     name: "distribute_dataset"
     argspec: "args=[\'self\', \'dataset_fn\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "experimental_finalize"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "experimental_initialize"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "experimental_make_numpy_iterator"
     argspec: "args=[\'self\', \'numpy_input\', \'batch_size\', \'num_epochs\', \'shuffle\', \'session\'], varargs=None, keywords=None, defaults=[\'1\', \'1024\', \'None\'], "
@@ -82,18 +74,10 @@ tf_class {
     name: "experimental_run"
     argspec: "args=[\'self\', \'fn\', \'input_iterator\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "finalize"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "group"
     argspec: "args=[\'self\', \'value\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "initialize"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "make_dataset_iterator"
     argspec: "args=[\'self\', \'dataset\'], varargs=None, keywords=None, defaults=None"
-- 
GitLab


From a7443e6ad57bd60693cf58a9dad3f462fdf5bc3d Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Thu, 10 Jan 2019 16:20:26 -0800
Subject: [PATCH 0555/1095] Allow scalar shapes in the Python converter API

Scalars are represented as empty lists in the shape. As the
converter and runtime now support scalars, allow such empty
(scalar) shape values in the Python converter API. Also
add a test to validate conversion using string types.

PiperOrigin-RevId: 228797882
---
 tensorflow/lite/python/BUILD        |  1 +
 tensorflow/lite/python/lite.py      |  5 ++-
 tensorflow/lite/python/lite_test.py | 65 ++++++++++++++++++++++++++---
 3 files changed, 63 insertions(+), 8 deletions(-)

diff --git a/tensorflow/lite/python/BUILD b/tensorflow/lite/python/BUILD
index 54b925cc05..4949d7c92e 100644
--- a/tensorflow/lite/python/BUILD
+++ b/tensorflow/lite/python/BUILD
@@ -72,6 +72,7 @@ py_test(
     name = "lite_test",
     srcs = ["lite_test.py"],
     data = ["@tflite_mobilenet_ssd_quant_protobuf//:tflite_graph.pb"],
+    shard_count = 4,
     srcs_version = "PY2AND3",
     tags = [
         "no_oss",
diff --git a/tensorflow/lite/python/lite.py b/tensorflow/lite/python/lite.py
index 5ffa8c426b..3b0aa02b7c 100644
--- a/tensorflow/lite/python/lite.py
+++ b/tensorflow/lite/python/lite.py
@@ -435,15 +435,16 @@ class TFLiteConverter(object):
     if self._has_valid_tensors():
       for tensor in self._input_tensors:
         shape = tensor.get_shape()
-        if not shape or not shape.as_list():
+        if not shape:
           raise ValueError("Provide an input shape for input array "
                            "'{0}'.".format(_tensor_name(tensor)))
+        # Note that shape_list might be empty for scalar shapes.
         shape_list = shape.as_list()
         if None in shape_list[1:]:
           raise ValueError(
               "None is only supported in the 1st dimension. Tensor '{0}' has "
               "invalid shape '{1}'.".format(_tensor_name(tensor), shape_list))
-        elif shape_list[0] is None:
+        elif shape_list and shape_list[0] is None:
           self._set_batch_size(batch_size=1)
 
     # Get quantization stats. Ensures there is one stat per name if the stats
diff --git a/tensorflow/lite/python/lite_test.py b/tensorflow/lite/python/lite_test.py
index 1f9c768b44..83fd56bf1d 100644
--- a/tensorflow/lite/python/lite_test.py
+++ b/tensorflow/lite/python/lite_test.py
@@ -113,6 +113,35 @@ class FromSessionTest(test_util.TensorFlowTestCase):
     self.assertTrue(([1, 16, 16, 3] == output_details[0]['shape']).all())
     self.assertEqual((0., 0.), output_details[0]['quantization'])
 
+  def testString(self):
+    in_tensor = array_ops.placeholder(shape=[4], dtype=dtypes.string)
+    out_tensor = array_ops.reshape(in_tensor, shape=[2, 2])
+    sess = session.Session()
+
+    # Convert model and ensure model is not None.
+    converter = lite.TFLiteConverter.from_session(sess, [in_tensor],
+                                                  [out_tensor])
+    tflite_model = converter.convert()
+    self.assertTrue(tflite_model)
+
+    # Check values from converted model.
+    interpreter = Interpreter(model_content=tflite_model)
+    interpreter.allocate_tensors()
+
+    input_details = interpreter.get_input_details()
+    self.assertEqual(1, len(input_details))
+    self.assertEqual('Placeholder', input_details[0]['name'])
+    self.assertEqual(np.object_, input_details[0]['dtype'])
+    self.assertTrue(([4] == input_details[0]['shape']).all())
+
+    output_details = interpreter.get_output_details()
+    self.assertEqual(1, len(output_details))
+    self.assertEqual('Reshape', output_details[0]['name'])
+    self.assertEqual(np.object_, output_details[0]['dtype'])
+    self.assertTrue(([2, 2] == output_details[0]['shape']).all())
+    # TODO(b/122659643): Test setting/getting string data via the python
+    # interpreter API after support has been added.
+
   def testQuantization(self):
     in_tensor_1 = array_ops.placeholder(
         shape=[1, 16, 16, 3], dtype=dtypes.float32, name='inputA')
@@ -223,18 +252,42 @@ class FromSessionTest(test_util.TensorFlowTestCase):
     self.assertEqual('Provide an input shape for input array \'Placeholder\'.',
                      str(error.exception))
 
-  def testSizeEmptyInvalid(self):
+  def testScalarValid(self):
+    # Construct a graph using a scalar (empty shape) input.
     in_tensor = array_ops.placeholder(dtype=dtypes.float32, shape=[])
     out_tensor = in_tensor + in_tensor
     sess = session.Session()
 
-    # Test empty shape.
+    # Test conversion with the scalar input shape.
     converter = lite.TFLiteConverter.from_session(sess, [in_tensor],
                                                   [out_tensor])
-    with self.assertRaises(ValueError) as error:
-      converter.convert()
-    self.assertEqual('Provide an input shape for input array \'Placeholder\'.',
-                     str(error.exception))
+    tflite_model = converter.convert()
+    self.assertTrue(tflite_model)
+
+    # Check values from converted model.
+    interpreter = Interpreter(model_content=tflite_model)
+    interpreter.allocate_tensors()
+
+    input_details = interpreter.get_input_details()
+    self.assertEqual(1, len(input_details))
+    self.assertEqual('Placeholder', input_details[0]['name'])
+    self.assertEqual(np.float32, input_details[0]['dtype'])
+    self.assertTrue(([] == input_details[0]['shape']).all())
+
+    output_details = interpreter.get_output_details()
+    self.assertEqual(1, len(output_details))
+    self.assertEqual('add', output_details[0]['name'])
+    self.assertEqual(np.float32, output_details[0]['dtype'])
+    self.assertTrue(([] == input_details[0]['shape']).all())
+
+    # Validate inference using the scalar inputs/outputs.
+    test_input = np.array(4.0, dtype=np.float32)
+    expected_output = np.array(8.0, dtype=np.float32)
+    interpreter.set_tensor(input_details[0]['index'], test_input)
+    interpreter.invoke()
+
+    output_data = interpreter.get_tensor(output_details[0]['index'])
+    self.assertTrue((expected_output == output_data).all())
 
   def testSizeInvalid(self):
     in_tensor = array_ops.placeholder(
-- 
GitLab


From 63a6b272919a1621b6ecc57b4960047e2c61d654 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 10 Jan 2019 16:22:20 -0800
Subject: [PATCH 0556/1095] Internal change

PiperOrigin-RevId: 228798154
---
 .../feature_column/feature_column_v2.py       | 18 ++----
 .../feature_column/feature_column_v2_test.py  | 56 ++++++++++++++++++-
 2 files changed, 60 insertions(+), 14 deletions(-)

diff --git a/tensorflow/python/feature_column/feature_column_v2.py b/tensorflow/python/feature_column/feature_column_v2.py
index d4e3fc5d2d..73e3001fc0 100644
--- a/tensorflow/python/feature_column/feature_column_v2.py
+++ b/tensorflow/python/feature_column/feature_column_v2.py
@@ -2780,12 +2780,8 @@ class NumericColumn(
     """See 'FeatureColumn` base class."""
     _check_config_keys(config, cls._fields)
     kwargs = config.copy()
-    # TODO(b/118820158): Simplify if deserialize_keras_object supports None.
-    if config['normalizer_fn']:
-      kwargs['normalizer_fn'] = utils.deserialize_keras_object(
-          config['normalizer_fn'], custom_objects=custom_objects)
-    else:
-      kwargs['normalizer_fn'] = None
+    kwargs['normalizer_fn'] = utils.deserialize_keras_object(
+        config['normalizer_fn'], custom_objects=custom_objects)
     kwargs['dtype'] = dtypes.as_dtype(config['dtype'])
     return cls(**kwargs)
 
@@ -3168,12 +3164,8 @@ class EmbeddingColumn(
     kwargs = config.copy()
     kwargs['categorical_column'] = deserialize_feature_column(
         config['categorical_column'], custom_objects, columns_by_name)
-    # TODO(b/118820158): Simplify if deserialize_keras_object supports None.
-    if config['initializer']:
-      kwargs['initializer'] = utils.deserialize_keras_object(
-          config['initializer'], custom_objects=custom_objects)
-    else:
-      kwargs['initializer'] = None
+    kwargs['initializer'] = utils.deserialize_keras_object(
+        config['initializer'], custom_objects=custom_objects)
     return cls(**kwargs)
 
 
@@ -4681,7 +4673,7 @@ def deserialize_feature_column(config,
           IdentityCategoricalColumn, IndicatorColumn, NumericColumn,
           SequenceCategoricalColumn, SequenceDenseColumn, SharedEmbeddingColumn,
           VocabularyFileCategoricalColumn, VocabularyListCategoricalColumn,
-          WeightedCategoricalColumn
+          WeightedCategoricalColumn, init_ops.TruncatedNormal
       ]
   }
   if columns_by_name is None:
diff --git a/tensorflow/python/feature_column/feature_column_v2_test.py b/tensorflow/python/feature_column/feature_column_v2_test.py
index aeaa8df8b5..137d5e0a8c 100644
--- a/tensorflow/python/feature_column/feature_column_v2_test.py
+++ b/tensorflow/python/feature_column/feature_column_v2_test.py
@@ -40,6 +40,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import lookup_ops
 from tensorflow.python.ops import parsing_ops
 from tensorflow.python.ops import partitioned_variables
@@ -7147,7 +7148,60 @@ class EmbeddingColumnTest(test.TestCase):
                           self.evaluate(predictions))
 
   @test_util.run_deprecated_v1
-  def test_serialization(self):
+  def test_serialization_with_default_initializer(self):
+
+    # Build columns.
+    categorical_column = fc.categorical_column_with_identity(
+        key='aaa', num_buckets=3)
+    embedding_column = fc.embedding_column(categorical_column, dimension=2)
+
+    self.assertEqual([categorical_column], embedding_column.parents)
+
+    config = embedding_column._get_config()
+    self.assertEqual({
+        'categorical_column': {
+            'class_name': 'IdentityCategoricalColumn',
+            'config': {
+                'number_buckets': 3,
+                'key': 'aaa',
+                'default_value': None
+            }
+        },
+        'ckpt_to_load_from': None,
+        'combiner': 'mean',
+        'dimension': 2,
+        'initializer': {
+            'class_name': 'TruncatedNormal',
+            'config': {
+                'dtype': 'float32',
+                'stddev': 0.7071067811865475,
+                'seed': None,
+                'mean': 0.0
+            }
+        },
+        'max_norm': None,
+        'tensor_name_in_ckpt': None,
+        'trainable': True
+    }, config)
+
+    custom_objects = {'TruncatedNormal': init_ops.TruncatedNormal}
+    new_embedding_column = fc.EmbeddingColumn._from_config(
+        config, custom_objects=custom_objects)
+    self.assertEqual(embedding_column._get_config(),
+                     new_embedding_column._get_config())
+    self.assertIsNot(categorical_column,
+                     new_embedding_column.categorical_column)
+
+    new_embedding_column = fc.EmbeddingColumn._from_config(
+        config,
+        custom_objects=custom_objects,
+        columns_by_name={categorical_column.name: categorical_column})
+    self.assertEqual(embedding_column._get_config(),
+                     new_embedding_column._get_config())
+    self.assertIs(categorical_column, new_embedding_column.categorical_column)
+
+  @test_util.run_deprecated_v1
+  def test_serialization_with_custom_initializer(self):
 
     def _initializer(shape, dtype, partition_info):
       del shape, dtype, partition_info
-- 
GitLab


From d2490b91e3f3d7771a58986a2d1446a32053eb19 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 10 Jan 2019 16:27:21 -0800
Subject: [PATCH 0557/1095] Use Cub-based tensor reduction in multinomial op.
 The Eigen implementation seems to trigger a bug on Volta GPUs.

PiperOrigin-RevId: 228798930
---
 tensorflow/core/kernels/BUILD                    |  7 +++++--
 tensorflow/core/kernels/multinomial_op_gpu.cu.cc | 14 ++++++++++----
 2 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 1a20e8628e..0045c5dd19 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -4982,11 +4982,14 @@ tf_kernel_library(
         ":random_op",
         ":random_ops",
         ":stateless_random_ops",
+        "//third_party/eigen3",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//third_party/eigen3",
-    ],
+    ] + if_cuda([
+        ":reduction_ops",
+        "@cub_archive//:cub",
+    ]),
 )
 
 tf_cuda_cc_test(
diff --git a/tensorflow/core/kernels/multinomial_op_gpu.cu.cc b/tensorflow/core/kernels/multinomial_op_gpu.cu.cc
index 5cc5877cce..62e38694c8 100644
--- a/tensorflow/core/kernels/multinomial_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/multinomial_op_gpu.cu.cc
@@ -22,9 +22,10 @@ limitations under the License.
 #include <assert.h>
 #include <stdio.h>
 
-#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/kernels/random_op.h"
+#include "tensorflow/core/kernels/reduction_gpu_kernels.cu.h"
+#include "tensorflow/core/kernels/reduction_ops_common.h"
 #include "tensorflow/core/lib/random/philox_random.h"
 #include "tensorflow/core/lib/random/random_distributions.h"
 #include "tensorflow/core/util/cuda_kernel_helper.h"
@@ -67,7 +68,6 @@ struct MultinomialFunctor<GPUDevice, T, OutputType> {
                                                  noises.size(), Dist());
 
 #if defined(EIGEN_HAS_INDEX_LIST)
-    Eigen::IndexList<Eigen::type2index<2>> kTwo;
     Eigen::IndexList<int, int, int> bsc;
     bsc.set(0, batch_size);
     bsc.set(1, num_samples);
@@ -80,7 +80,6 @@ struct MultinomialFunctor<GPUDevice, T, OutputType> {
     Eigen::IndexList<Eigen::type2index<1>, int, Eigen::type2index<1>> oso;
     oso.set(1, num_samples);
 #else
-    Eigen::array<int, 1> kTwo{2};
     Eigen::array<int, 3> bsc{batch_size, num_samples, num_classes};
     Eigen::array<int, 3> boc{batch_size, 1, num_classes};
     Eigen::array<int, 3> oso{1, num_samples, 1};
@@ -98,7 +97,14 @@ struct MultinomialFunctor<GPUDevice, T, OutputType> {
         ((-((To32Bit(noises) + 2e-30f).log())).log());
 
     // Max-reduce along classes for each (batch, sample).
-    To32Bit(maxima).device(d) = To32Bit(scores).reshape(bsc).maximum(kTwo);
+    typedef const Eigen::array<TTypes<float>::Tensor::Index, 1>& ReductionAxes;
+    Constants<GPUDevice> constants;
+    cub::Max op;
+    functor::ReduceImpl<float, cub::Max, float*, const float*, ReductionAxes>(
+        /*ctx=*/ctx, /*out=*/maxima.data(), /*in=*/scores.data(), /*in_rank=*/2,
+        /*in_dim0=*/batch_size * num_samples,
+        /*in_dim1=*/num_classes, /*in_dim2=*/1, /*out_rank=*/1,
+        /*reduction_axes=*/constants.kOne, /*Op=*/op);
 
     // Necessary for atomicMax() inside the kernel.
     output.device(d) = output.constant(0LL);
-- 
GitLab


From 40874676a61cc1823355016953a6ac6222f8b404 Mon Sep 17 00:00:00 2001
From: Yuefeng Zhou <yuefengz@google.com>
Date: Thu, 10 Jan 2019 16:43:49 -0800
Subject: [PATCH 0558/1095] Make nccl work in eager mode: wrap nccl ops in a
 defun; remove control dependencies on NcclAllReduce

PiperOrigin-RevId: 228801431
---
 tensorflow/python/BUILD                       |  2 ++
 .../python/framework/auto_control_deps.py     |  1 +
 tensorflow/python/ops/nccl_ops.py             | 34 ++++++++++++-------
 3 files changed, 25 insertions(+), 12 deletions(-)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index c5a8994b80..82c542f51d 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -5869,6 +5869,8 @@ py_library(
     deps = [
         ":framework_for_generated_wrappers",
         ":nccl_ops_gen",
+        "//tensorflow/python/eager:context",
+        "//tensorflow/python/eager:def_function",
     ],
 )
 
diff --git a/tensorflow/python/framework/auto_control_deps.py b/tensorflow/python/framework/auto_control_deps.py
index a7d61417bf..da76a84e55 100644
--- a/tensorflow/python/framework/auto_control_deps.py
+++ b/tensorflow/python/framework/auto_control_deps.py
@@ -35,6 +35,7 @@ ASYNC_STATEFUL_OPS = [
     "CollectiveReduce",
     "CollectiveBcastSend",
     "CollectiveBcastRecv",
+    "NcclAllReduce",
 ]
 
 
diff --git a/tensorflow/python/ops/nccl_ops.py b/tensorflow/python/ops/nccl_ops.py
index 6259ce0f94..6c8685cf63 100644
--- a/tensorflow/python/ops/nccl_ops.py
+++ b/tensorflow/python/ops/nccl_ops.py
@@ -19,6 +19,8 @@ from __future__ import print_function
 
 import threading
 
+from tensorflow.python.eager import context
+from tensorflow.python.eager import def_function
 from tensorflow.python.framework import device
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import gen_nccl_ops
@@ -211,19 +213,27 @@ def _apply_all_reduce(reduction, tensors):
     raise ValueError('Must pass >0 tensors to all reduce operations')
 
   shared_name = _get_shared_name()
-  res = []
 
-  for t in tensors:
-    _check_device(t)
-    with ops.device(t.device):
-      res.append(
-          gen_nccl_ops.nccl_all_reduce(
-              input=t,
-              reduction=reduction,
-              num_devices=len(tensors),
-              shared_name=shared_name))
-
-  return res
+  def _all_reduce():
+    """Call nccl allreduce."""
+    res = []
+    for t in tensors:
+      _check_device(t)
+      with ops.device(t.device):
+        res.append(
+            gen_nccl_ops.nccl_all_reduce(
+                input=t,
+                reduction=reduction,
+                num_devices=len(tensors),
+                shared_name=shared_name))
+    return res
+
+  if context.executing_eagerly():
+    # Nccl ops will block unless they are executed concurrently such as in a
+    # graph or a defun.
+    return def_function.function(_all_reduce)()
+  else:
+    return _all_reduce()
 
 
 def _apply_reduce(reduction, tensors):
-- 
GitLab


From e2760cb89f08c3c31ad0322f5b2d8532d0c11b7d Mon Sep 17 00:00:00 2001
From: Ayush Dubey <ayushd@google.com>
Date: Thu, 10 Jan 2019 16:44:20 -0800
Subject: [PATCH 0559/1095] Enable a soft ordering for collectives which adds
 dependencies as node attrs.

Before this change, OrderCollectives would add control edges between collective
ops to make their execution sequential.  This change introduces a technique of
encoding the ordering information as a node attribute.  In future changes, the
collective executor will parse these changes and enforce a fine-grained
ordering between collective ops.

Additionally, this change also
* restricts ordering to CollectiveReduce nodes,
* and fixes a bug so that this ordering is invoked from both DirectSession and
MasterSession.

PiperOrigin-RevId: 228801501
---
 tensorflow/core/BUILD                         |   1 +
 .../common_runtime/build_graph_options.cc     |  13 ++
 .../core/common_runtime/build_graph_options.h |   6 +
 .../core/common_runtime/direct_session.cc     |  11 +-
 .../common_runtime/graph_execution_state.cc   |   7 +
 .../distributed_runtime/master_session.cc     |  45 ++---
 tensorflow/core/graph/collective_order.cc     | 161 +++++++++++++++---
 tensorflow/core/graph/collective_order.h      |  16 +-
 .../core/graph/collective_order_test.cc       |  71 +++++++-
 tensorflow/core/ops/collective_ops.cc         |   1 +
 10 files changed, 268 insertions(+), 64 deletions(-)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index e6af9211b5..690fa4a19e 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -2819,6 +2819,7 @@ tf_cuda_library(
         ":protos_all_cc",
         "//third_party/eigen3",
         "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/container:flat_hash_set",
     ],
 )
 
diff --git a/tensorflow/core/common_runtime/build_graph_options.cc b/tensorflow/core/common_runtime/build_graph_options.cc
index 00f7a8e645..b095fcfa3b 100644
--- a/tensorflow/core/common_runtime/build_graph_options.cc
+++ b/tensorflow/core/common_runtime/build_graph_options.cc
@@ -35,6 +35,19 @@ string BuildGraphOptions::DebugString() const {
   if (collective_graph_key != kNoCollectiveGraphKey) {
     strings::StrAppend(&rv, "\ncollective_graph_key: ", collective_graph_key);
   }
+  string collective_order_str;
+  switch (collective_order) {
+    case GraphCollectiveOrder::kNone:
+      collective_order_str = "none";
+      break;
+    case GraphCollectiveOrder::kEdges:
+      collective_order_str = "edges";
+      break;
+    case GraphCollectiveOrder::kAttrs:
+      collective_order_str = "attrs";
+      break;
+  }
+  strings::StrAppend(&rv, "\ncollective_order: ", collective_order_str);
   return rv;
 }
 
diff --git a/tensorflow/core/common_runtime/build_graph_options.h b/tensorflow/core/common_runtime/build_graph_options.h
index 3d0f242ea5..24b71cc741 100644
--- a/tensorflow/core/common_runtime/build_graph_options.h
+++ b/tensorflow/core/common_runtime/build_graph_options.h
@@ -18,6 +18,7 @@ limitations under the License.
 
 #include <vector>
 
+#include "tensorflow/core/graph/collective_order.h"
 #include "tensorflow/core/platform/types.h"
 #include "tensorflow/core/protobuf/config.pb.h"
 
@@ -34,6 +35,11 @@ struct BuildGraphOptions {
   static const int64 kNoCollectiveGraphKey = 0;
   int64 collective_graph_key = kNoCollectiveGraphKey;
 
+  // If not `kNone`, order all CollectiveReduce operations statically and
+  // deterministically.  If `kEdges`, encode dependencies as explicit control
+  // edges, if `kAttrs` encode as attribute on collective op.
+  GraphCollectiveOrder collective_order = GraphCollectiveOrder::kNone;
+
   string DebugString() const;
 };
 
diff --git a/tensorflow/core/common_runtime/direct_session.cc b/tensorflow/core/common_runtime/direct_session.cc
index 36f1a92aab..8a1c648f61 100644
--- a/tensorflow/core/common_runtime/direct_session.cc
+++ b/tensorflow/core/common_runtime/direct_session.cc
@@ -45,7 +45,6 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/versions.pb.h"
 #include "tensorflow/core/graph/algorithm.h"
-#include "tensorflow/core/graph/collective_order.h"
 #include "tensorflow/core/graph/graph.h"
 #include "tensorflow/core/graph/graph_constructor.h"
 #include "tensorflow/core/graph/graph_partition.h"
@@ -1192,6 +1191,10 @@ Status DirectSession::CreateExecutors(
   options.use_function_convention = !run_state_args->is_partial_run;
   options.collective_graph_key =
       callable_options.run_options().experimental().collective_graph_key();
+  if (options_.config.experimental()
+          .collective_deterministic_sequential_execution()) {
+    options.collective_order = GraphCollectiveOrder::kEdges;
+  }
 
   std::unique_ptr<FunctionInfo> func_info(new FunctionInfo);
   std::unique_ptr<ExecutorsAndKeys> ek(new ExecutorsAndKeys);
@@ -1523,12 +1526,6 @@ Status DirectSession::CreateGraphs(
     CopyGraph(*execution_state->full_graph(), run_state_args->graph.get());
   }
 
-  // Make collective execution order deterministic if needed.
-  if (options_.config.experimental()
-          .collective_deterministic_sequential_execution()) {
-    TF_RETURN_IF_ERROR(OrderCollectives(&client_graph->graph));
-  }
-
   // Partition the graph across devices.
   PartitionOptions popts;
   popts.node_to_loc = [](const Node* node) {
diff --git a/tensorflow/core/common_runtime/graph_execution_state.cc b/tensorflow/core/common_runtime/graph_execution_state.cc
index 9ecbc34f5f..8875883f62 100644
--- a/tensorflow/core/common_runtime/graph_execution_state.cc
+++ b/tensorflow/core/common_runtime/graph_execution_state.cc
@@ -32,6 +32,7 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor.pb.h"
 #include "tensorflow/core/framework/versions.pb.h"
 #include "tensorflow/core/graph/algorithm.h"
+#include "tensorflow/core/graph/collective_order.h"
 #include "tensorflow/core/graph/graph.h"
 #include "tensorflow/core/graph/graph_constructor.h"
 #include "tensorflow/core/graph/subgraph.h"
@@ -819,6 +820,12 @@ Status GraphExecutionState::BuildGraph(const BuildGraphOptions& options,
     }
   }
 
+  // Make collective execution order deterministic if needed.
+  if (options.collective_order != GraphCollectiveOrder::kNone) {
+    TF_RETURN_IF_ERROR(
+        OrderCollectives(optimized_graph.get(), options.collective_order));
+  }
+
   // Copy the extracted graph in order to make its node ids dense,
   // since the local CostModel used to record its stats is sized by
   // the largest node id.
diff --git a/tensorflow/core/distributed_runtime/master_session.cc b/tensorflow/core/distributed_runtime/master_session.cc
index 2bb4375831..48b72fb948 100644
--- a/tensorflow/core/distributed_runtime/master_session.cc
+++ b/tensorflow/core/distributed_runtime/master_session.cc
@@ -292,8 +292,8 @@ class MasterSession::ReffedClientGraph : public core::RefCounted {
     if (tot >= 0.1 * 1048576.0) {
       bytes = strings::Printf("[%.1fMB] ", tot / 1048576.0);
     }
-    return strings::StrCat(bytes, stats.node_name(), " = ",
-                           details.type_string, details.detail_text);
+    return strings::StrCat(bytes, stats.node_name(), " = ", details.type_string,
+                           details.detail_text);
   }
 
   // Send/Recv nodes that are the result of client-added
@@ -1081,17 +1081,18 @@ void CopyAndSortStrings(size_t size,
 }  // namespace
 
 void BuildBuildGraphOptions(const RunStepRequestWrapper& req,
+                            const ConfigProto& config,
                             BuildGraphOptions* opts) {
   CallableOptions* callable_opts = &opts->callable_options;
-  CopyAndSortStrings(req.num_feeds(),
-                     [&req](size_t i) { return req.feed_name(i); },
-                     callable_opts->mutable_feed());
-  CopyAndSortStrings(req.num_fetches(),
-                     [&req](size_t i) { return req.fetch_name(i); },
-                     callable_opts->mutable_fetch());
-  CopyAndSortStrings(req.num_targets(),
-                     [&req](size_t i) { return req.target_name(i); },
-                     callable_opts->mutable_target());
+  CopyAndSortStrings(
+      req.num_feeds(), [&req](size_t i) { return req.feed_name(i); },
+      callable_opts->mutable_feed());
+  CopyAndSortStrings(
+      req.num_fetches(), [&req](size_t i) { return req.fetch_name(i); },
+      callable_opts->mutable_fetch());
+  CopyAndSortStrings(
+      req.num_targets(), [&req](size_t i) { return req.target_name(i); },
+      callable_opts->mutable_target());
 
   if (!req.options().debug_options().debug_tensor_watch_opts().empty()) {
     *callable_opts->mutable_run_options()->mutable_debug_options() =
@@ -1100,19 +1101,23 @@ void BuildBuildGraphOptions(const RunStepRequestWrapper& req,
 
   opts->collective_graph_key =
       req.options().experimental().collective_graph_key();
+  if (config.experimental().collective_deterministic_sequential_execution()) {
+    opts->collective_order = GraphCollectiveOrder::kEdges;
+  }
 }
 
 void BuildBuildGraphOptions(const PartialRunSetupRequest& req,
                             BuildGraphOptions* opts) {
   CallableOptions* callable_opts = &opts->callable_options;
-  CopyAndSortStrings(req.feed_size(), [&req](size_t i) { return req.feed(i); },
-                     callable_opts->mutable_feed());
-  CopyAndSortStrings(req.fetch_size(),
-                     [&req](size_t i) { return req.fetch(i); },
-                     callable_opts->mutable_fetch());
-  CopyAndSortStrings(req.target_size(),
-                     [&req](size_t i) { return req.target(i); },
-                     callable_opts->mutable_target());
+  CopyAndSortStrings(
+      req.feed_size(), [&req](size_t i) { return req.feed(i); },
+      callable_opts->mutable_feed());
+  CopyAndSortStrings(
+      req.fetch_size(), [&req](size_t i) { return req.fetch(i); },
+      callable_opts->mutable_fetch());
+  CopyAndSortStrings(
+      req.target_size(), [&req](size_t i) { return req.target(i); },
+      callable_opts->mutable_target());
 
   // TODO(cais): Add TFDBG support to partial runs.
 }
@@ -1852,7 +1857,7 @@ Status MasterSession::DoRunWithLocalExecution(
 
   // Prepare.
   BuildGraphOptions bgopts;
-  BuildBuildGraphOptions(req, &bgopts);
+  BuildBuildGraphOptions(req, session_opts_.config, &bgopts);
   ReffedClientGraph* rcg = nullptr;
   int64 count;
   TF_RETURN_IF_ERROR(StartStep(bgopts, false, &rcg, &count));
diff --git a/tensorflow/core/graph/collective_order.cc b/tensorflow/core/graph/collective_order.cc
index bd8fd767ee..bbba9264ce 100644
--- a/tensorflow/core/graph/collective_order.cc
+++ b/tensorflow/core/graph/collective_order.cc
@@ -14,55 +14,70 @@ limitations under the License.
 ==============================================================================*/
 #include "tensorflow/core/graph/collective_order.h"
 
+#include "absl/container/flat_hash_map.h"
+#include "absl/container/flat_hash_set.h"
 #include "tensorflow/core/graph/algorithm.h"
 
 namespace tensorflow {
+namespace {
 
-Status OrderCollectives(Graph* graph) {
-  // `instance_keys[i]` corresponds to `collective_nodes[i]`
-  std::vector<Node*> collective_nodes;
-  std::vector<int32> instance_keys;
-  // node -> set of collectives on which node depends.
-  std::unordered_map<Node*, std::unordered_set<int32>> node_dependencies;
+// Find all CollectiveReduce nodes and the existing data dependencies between
+// them.
+Status DiscoverDataDependencies(
+    const Graph* graph, std::vector<Node*>* collective_nodes,
+    std::vector<int32>* instance_keys,
+    absl::flat_hash_map<Node*, absl::flat_hash_set<int32>>* data_dependencies) {
   Status s;
-
   // Algorithm: do Reverse DFS starting at sink.  `node_leave` is called when
-  // all parents of `node` have been visited.  At that point, the collectives
-  // on which this node depends on are up to date.  For this node's children,
-  // add all these collectives.  Also, if this node is collective, add as a
-  // dependency for the children.
-  auto node_leave = [&collective_nodes, &instance_keys, &node_dependencies,
+  // all parents of `node` have been visited.  At that point,
+  // `data_dependencies[node]` is a list containing `instance_key` of every
+  // `CollectiveReduce` on which `node` has a data dependency.
+  // For this node's children, add all these instance keys.  Also, if this node
+  // is collective, add as a dependency for the children.
+  auto node_leave = [collective_nodes, instance_keys, data_dependencies,
                      &s](Node* node) {
     int32 instance_key;
-    if (node->IsCollective()) {
+    bool enter_node =
+        node->IsCollective() && node->type_string() == "CollectiveReduce";
+    if (enter_node) {
       Status get_attr_status =
           GetNodeAttr(node->attrs(), "instance_key", &instance_key);
       s.Update(get_attr_status);
-      collective_nodes.push_back(node);
-      instance_keys.push_back(instance_key);
+      collective_nodes->push_back(node);
+      instance_keys->push_back(instance_key);
       VLOG(2) << "collective node " << node->DebugString();
     }
-    const auto& node_deps = node_dependencies[node];
+    const auto& node_deps = (*data_dependencies)[node];
     for (const Edge* out_edge : node->out_edges()) {
-      auto& child_deps = node_dependencies[out_edge->dst()];
+      auto& child_deps = (*data_dependencies)[out_edge->dst()];
       child_deps.insert(node_deps.begin(), node_deps.end());
-      if (node->IsCollective() && s.ok()) {
+      if (enter_node && s.ok()) {
         child_deps.insert(instance_key);
       }
     }
   };
   ReverseDFS(*graph, nullptr, node_leave);
-  if (!s.ok()) return s;
+  return s;
+}
 
-  // For all pairs of collective nodes n1 and n2 on the same device, if n1 does
-  // not depend on n2 and n2 does not depend on n1, then they are potentially
-  // concurrent.  Add an arbitrary, deterministic control edge between them.
+// Given a list of `collective_nodes` and `data_dependencies` between the
+// collective nodes, create control dependencies between concurrent collectives
+// and store in `dependency_edges`.
+// If there exists an edge a -> b then `dependency_edges[a]` contains `b`
+Status CreateControlDependencies(
+    const std::vector<Node*>& collective_nodes,
+    const std::vector<int32>& instance_keys,
+    absl::flat_hash_map<Node*, absl::flat_hash_set<int32>>* data_dependencies,
+    absl::flat_hash_map<Node*, absl::flat_hash_set<Node*>>* dependency_edges) {
+  // If there exists some path a -> ... -> b then `all_paths[a]` contains `b`
+  absl::flat_hash_map<Node*, absl::flat_hash_set<Node*>> all_paths;
   for (int i = 0; i < collective_nodes.size() - 1; i++) {
-    if (!collective_nodes[i]->IsCollective()) {
+    if (!collective_nodes[i]->IsCollective() ||
+        collective_nodes[i]->type_string() != "CollectiveReduce") {
       return errors::Internal("Unexpected node ",
                               collective_nodes[i]->DebugString());
     }
-    const auto& deps_i = node_dependencies[collective_nodes[i]];
+    const auto& deps_i = (*data_dependencies)[collective_nodes[i]];
     for (int j = i + 1; j < collective_nodes.size(); j++) {
       if (collective_nodes[i]->requested_device() !=
           collective_nodes[j]->requested_device()) {
@@ -74,17 +89,48 @@ Status OrderCollectives(Graph* graph) {
                                 " on 2 nodes with the same device ",
                                 collective_nodes[i]->requested_device());
       }
-      const auto& deps_j = node_dependencies[collective_nodes[j]];
+      const auto& deps_j = (*data_dependencies)[collective_nodes[j]];
       if (deps_i.find(instance_keys[j]) == deps_i.end() &&
           deps_j.find(instance_keys[i]) == deps_j.end()) {
         int src_idx = instance_keys[i] < instance_keys[j] ? i : j;
         int dst_idx = instance_keys[i] < instance_keys[j] ? j : i;
         Node* src_node = collective_nodes[src_idx];
         Node* dst_node = collective_nodes[dst_idx];
-        VLOG(1) << "Adding control edge from node " << src_node->name()
+        VLOG(1) << "Adding control dependency from node " << src_node->name()
                 << " instance " << instance_keys[src_idx] << " to node "
                 << dst_node->name() << " instance " << instance_keys[dst_idx];
-        graph->AddControlEdge(src_node, dst_node);
+        (*dependency_edges)[src_node].insert(dst_node);
+        auto& src_paths = all_paths[src_node];
+        src_paths.insert(dst_node);
+        for (Node* downstream_node : all_paths[dst_node]) {
+          src_paths.insert(downstream_node);
+        }
+      }
+    }
+  }
+
+  // Prune dependency edges so that if there are edges a -> b, b -> c, and a ->
+  // c, then remove a -> c.  This dependency would be handled naturally during
+  // op scheduling.
+  for (int i = 0; i < collective_nodes.size(); ++i) {
+    Node* node = collective_nodes[i];
+    auto& neighbor_set = (*dependency_edges)[node];
+    std::vector<Node*> neighbor_list(neighbor_set.begin(), neighbor_set.end());
+    // For all n1, n2 in `neighbor_list` if there is a path from n1 -> n2 then
+    // eliminate n2 from `neighbor_set` and `neighbor_list`.  We remove from
+    // `neighbor_list` by replacing with a `nullptr`, hence the `nullptr` checks
+    // below.
+    for (int j = 0; j < neighbor_list.size(); ++j) {
+      Node* n1 = neighbor_list[j];
+      if (n1 == nullptr) continue;
+      auto& n1_paths = all_paths[n1];
+      for (int k = 0; k < neighbor_list.size(); ++k) {
+        Node* n2 = neighbor_list[k];
+        if (j == k || n2 == nullptr) continue;
+        if (n1_paths.find(n2) != n1_paths.end()) {
+          neighbor_set.erase(n2);
+          neighbor_list[k] = nullptr;
+        }
       }
     }
   }
@@ -92,4 +138,65 @@ Status OrderCollectives(Graph* graph) {
   return Status::OK();
 }
 
+// Insert control dependencies defined by `dependency_edges` in `graph`.  If
+// `order_type` is `kEdges`, insert explicit control edges, else if `order_type`
+// is `kAttrs`, encode depdencies as an attribute on collective node.
+Status InsertControlDependencies(
+    Graph* graph, GraphCollectiveOrder order_type,
+    const absl::flat_hash_map<Node*, absl::flat_hash_set<Node*>>&
+        dependency_edges) {
+  if (order_type == GraphCollectiveOrder::kEdges) {
+    for (const auto& pair : dependency_edges) {
+      Node* src_node = pair.first;
+      for (Node* dst_node : pair.second) {
+        graph->AddControlEdge(src_node, dst_node);
+      }
+    }
+  } else if (order_type == GraphCollectiveOrder::kAttrs) {
+    // `wait_for` is the inverse of `dependency_edges`, i.e. `wait_for[node]`
+    // contains the list of instance keys for which `node` must wait.
+    absl::flat_hash_map<Node*, absl::flat_hash_set<int32>> wait_for;
+    for (const auto& pair : dependency_edges) {
+      int32 src_instance;
+      TF_RETURN_IF_ERROR(
+          GetNodeAttr(pair.first->attrs(), "instance_key", &src_instance));
+      for (Node* dst_node : pair.second) {
+        wait_for[dst_node].insert(src_instance);
+      }
+    }
+    for (const auto& pair : wait_for) {
+      std::vector<int32> wait_for_list(pair.second.begin(), pair.second.end());
+      pair.first->ClearAttr("wait_for");
+      pair.first->AddAttr("wait_for", wait_for_list);
+    }
+  } else {
+    return errors::Internal("Unexpected GraphCollectiveOrder type ",
+                            static_cast<int>(order_type));
+  }
+  return Status::OK();
+}
+
+}  // namespace
+
+Status OrderCollectives(Graph* graph, GraphCollectiveOrder order_type) {
+  // `instance_keys[i]` corresponds to `collective_nodes[i]`
+  std::vector<Node*> collective_nodes;
+  std::vector<int32> instance_keys;
+  // node -> set of collectives on which node depends.
+  absl::flat_hash_map<Node*, absl::flat_hash_set<int32>> data_dependencies;
+  TF_RETURN_IF_ERROR(DiscoverDataDependencies(
+      graph, &collective_nodes, &instance_keys, &data_dependencies));
+
+  if (collective_nodes.empty()) return Status::OK();
+
+  absl::flat_hash_map<Node*, absl::flat_hash_set<Node*>> dependency_edges;
+  // For all pairs of collective nodes n1 and n2 on the same device, if n1 does
+  // not depend on n2 and n2 does not depend on n1, then they are potentially
+  // concurrent.  Create an arbitrary, deterministic ordering between them.
+  TF_RETURN_IF_ERROR(CreateControlDependencies(
+      collective_nodes, instance_keys, &data_dependencies, &dependency_edges));
+
+  return InsertControlDependencies(graph, order_type, dependency_edges);
+}
+
 }  // namespace tensorflow
diff --git a/tensorflow/core/graph/collective_order.h b/tensorflow/core/graph/collective_order.h
index 66210a2ab4..67a1427a96 100644
--- a/tensorflow/core/graph/collective_order.h
+++ b/tensorflow/core/graph/collective_order.h
@@ -19,11 +19,17 @@ limitations under the License.
 
 namespace tensorflow {
 
-// Introduces control edges between potentially concurrent CollectiveOps to make
-// their execution order deterministic. This may be used to execute collectives
-// in the same order across all workers in a distributed execution, if all
-// workers are executing the same graph.
-Status OrderCollectives(Graph* graph);
+enum class GraphCollectiveOrder { kNone, kEdges, kAttrs };
+
+// Introduces a deterministic execution order between potentially concurrent
+// CollectiveOps.  This may be used to execute collectives in the same order
+// across all workers in a distributed execution, if all workers are executing
+// the same graph.
+// If `order_type` is `kEdges`, introduce the ordering in the form of explicit
+// control edges between collective graph nodes.  If `order_type` is `kAttrs`,
+// add an attribute to the node which may be used by collective executor to
+// ensure the required ordering.
+Status OrderCollectives(Graph* graph, GraphCollectiveOrder order_type);
 
 }  // namespace tensorflow
 
diff --git a/tensorflow/core/graph/collective_order_test.cc b/tensorflow/core/graph/collective_order_test.cc
index b0ced7b007..241c98b549 100644
--- a/tensorflow/core/graph/collective_order_test.cc
+++ b/tensorflow/core/graph/collective_order_test.cc
@@ -59,6 +59,23 @@ void VerifyGraph(const Graph& graph,
               UnorderedElementsAreArray(expected_collective_control_edges));
 }
 
+// Verifies that the `wait_for` attribute on collective nodes matches
+// `wait_for_map`.
+void VerifyAttrs(
+    const Graph& graph,
+    const std::unordered_map<string, std::vector<int32>> wait_for_map) {
+  for (const Node* node : graph.nodes()) {
+    if (node->IsCollective() ||
+        wait_for_map.find(node->name()) == wait_for_map.end()) {
+      continue;
+    }
+    std::vector<int32> wait_for_actual;
+    TF_EXPECT_OK(GetNodeAttr(node->attrs(), "wait_for", &wait_for_actual));
+    auto wait_for_expected = wait_for_map.at(node->name());
+    EXPECT_THAT(wait_for_actual, UnorderedElementsAreArray(wait_for_expected));
+  }
+}
+
 Node* CollectiveReduceNode(GraphDefBuilder* builder, Node* input,
                            const string& name, const string& device,
                            int instance_key) {
@@ -123,11 +140,17 @@ std::unique_ptr<Graph> InitGraph() {
 // added after calling `OrderCollectives`: c2_0 -> c3_0 and c2_1 -> c3_1.
 TEST(CollectiveOrderTest, SimpleOrder) {
   std::unique_ptr<Graph> graph = InitGraph();
-  TF_EXPECT_OK(OrderCollectives(graph.get()));
+  TF_EXPECT_OK(OrderCollectives(graph.get(), GraphCollectiveOrder::kEdges));
   VerifyGraph(*graph, {"c1_0", "c1_1", "c2_0", "c2_1", "c3_0", "c3_1"},
               {{"c2_0", "c3_0"}, {"c2_1", "c3_1"}});
 }
 
+TEST(CollectiveOrderTest, SimpleOrderAttr) {
+  std::unique_ptr<Graph> graph = InitGraph();
+  TF_EXPECT_OK(OrderCollectives(graph.get(), GraphCollectiveOrder::kAttrs));
+  VerifyAttrs(*graph, {{"c3_0", {2}}, {"c3_1", {2}}});
+}
+
 // Initialize the following graph:
 //
 //         a
@@ -162,12 +185,50 @@ std::unique_ptr<Graph> InitGraph2() {
 }
 
 // Tests that in the graph created by `InitGraph2`, we add the following control
-// edges after calling `OrderCollectives`: c2 -> c3, c3 -> c4, and c2 -> c4.
+// edges after calling `OrderCollectives`: c2 -> c3, c3 -> c4.  c2->c4 is
+// pruned because it follows from the other two edges.
 TEST(CollectiveOrderTest, SimpleOrder2) {
   std::unique_ptr<Graph> graph = InitGraph2();
-  TF_EXPECT_OK(OrderCollectives(graph.get()));
-  VerifyGraph(*graph, {"c1", "c2", "c3", "c4"},
-              {{"c2", "c3"}, {"c3", "c4"}, {"c2", "c4"}});
+  TF_EXPECT_OK(OrderCollectives(graph.get(), GraphCollectiveOrder::kEdges));
+  VerifyGraph(*graph, {"c1", "c2", "c3", "c4"}, {{"c2", "c3"}, {"c3", "c4"}});
+}
+
+// Initialize the following graph:
+//
+//         w   x   y   z
+//         |   |   |   |
+//         c1  c2  c3  c4
+//
+std::unique_ptr<Graph> InitGraphForPruning() {
+  GraphDefBuilder builder(GraphDefBuilder::kFailImmediately);
+  const string dev0 = "/job:localhost/replica:0/task:0/device:CPU:0";
+  Node* w = ops::SourceOp("TestParams",
+                          builder.opts().WithName("w").WithDevice(dev0));
+  Node* x = ops::SourceOp("TestParams",
+                          builder.opts().WithName("x").WithDevice(dev0));
+  Node* y = ops::SourceOp("TestParams",
+                          builder.opts().WithName("y").WithDevice(dev0));
+  Node* z = ops::SourceOp("TestParams",
+                          builder.opts().WithName("z").WithDevice(dev0));
+  CollectiveReduceNode(&builder, w, "c1", dev0, 1);
+  CollectiveReduceNode(&builder, x, "c2", dev0, 2);
+  CollectiveReduceNode(&builder, y, "c3", dev0, 3);
+  CollectiveReduceNode(&builder, z, "c4", dev0, 4);
+
+  std::unique_ptr<Graph> graph = absl::make_unique<Graph>(OpRegistry::Global());
+  Status s = GraphDefBuilderToGraph(builder, graph.get());
+  if (!s.ok()) {
+    LOG(FATAL) << "Error building graph " << s;
+  }
+  return graph;
+}
+
+// Tests that in the graph created by `InitGraphForPruning`, we only add c1 ->
+// c2, c2 -> c3, c3 -> c4, and other edges are pruned away.
+TEST(CollectiveOrderTest, Pruning) {
+  std::unique_ptr<Graph> graph = InitGraphForPruning();
+  TF_EXPECT_OK(OrderCollectives(graph.get(), GraphCollectiveOrder::kAttrs));
+  VerifyAttrs(*graph, {{"c4", {3}}, {"c3", {2}}, {"c2", {1}}});
 }
 
 }  // namespace
diff --git a/tensorflow/core/ops/collective_ops.cc b/tensorflow/core/ops/collective_ops.cc
index d6157a69df..e45a8a9b36 100644
--- a/tensorflow/core/ops/collective_ops.cc
+++ b/tensorflow/core/ops/collective_ops.cc
@@ -28,6 +28,7 @@ REGISTER_OP("CollectiveReduce")
     .Attr("merge_op: {'Min', 'Max', 'Mul', 'Add'}")
     .Attr("final_op: {'Id', 'Div'}")
     .Attr("subdiv_offsets: list(int)")
+    .Attr("wait_for: list(int) = []")
     .SetIsStateful()
     .SetShapeFn(shape_inference::UnchangedShape);
 
-- 
GitLab


From 0364aef0ba5385b540632d692fcb2843ef382047 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 10 Jan 2019 16:54:28 -0800
Subject: [PATCH 0560/1095] Add registration for (A^-1)^-1 = A.

PiperOrigin-RevId: 228803130
---
 .../python/ops/linalg/inverse_registrations.py       | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/ops/linalg/inverse_registrations.py b/tensorflow/python/ops/linalg/inverse_registrations.py
index b89bdd240e..12d1e7554c 100644
--- a/tensorflow/python/ops/linalg/inverse_registrations.py
+++ b/tensorflow/python/ops/linalg/inverse_registrations.py
@@ -40,6 +40,12 @@ def _inverse_linear_operator(linop):
       is_square=linop.is_square)
 
 
+@linear_operator_algebra.RegisterInverse(
+    linear_operator_inversion.LinearOperatorInversion)
+def _inverse_inverse_linear_operator(linop_inversion):
+  return linop_inversion.operator
+
+
 @linear_operator_algebra.RegisterInverse(
     linear_operator_diag.LinearOperatorDiag)
 def _inverse_diag(diag_operator):
@@ -72,7 +78,7 @@ def _inverse_scaled_identity(identity_operator):
 @linear_operator_algebra.RegisterInverse(
     linear_operator_block_diag.LinearOperatorBlockDiag)
 def _inverse_block_diag(block_diag_operator):
-    # We take the inverse of each block on the diagonal.
+  # We take the inverse of each block on the diagonal.
   return linear_operator_block_diag.LinearOperatorBlockDiag(
       operators=[
           operator.inverse() for operator in block_diag_operator.operators],
@@ -85,8 +91,8 @@ def _inverse_block_diag(block_diag_operator):
 @linear_operator_algebra.RegisterInverse(
     linear_operator_kronecker.LinearOperatorKronecker)
 def _inverse_kronecker(kronecker_operator):
-    # Inverse decomposition of a Kronecker product is the Kronecker product
-    # of inverse decompositions.
+  # Inverse decomposition of a Kronecker product is the Kronecker product
+  # of inverse decompositions.
   return linear_operator_kronecker.LinearOperatorKronecker(
       operators=[
           operator.inverse() for operator in kronecker_operator.operators],
-- 
GitLab


From 5269e3db18693866373a8c774936bbb7515fa027 Mon Sep 17 00:00:00 2001
From: Katherine Wu <kathywu@google.com>
Date: Thu, 10 Jan 2019 17:04:19 -0800
Subject: [PATCH 0561/1095] Move Keras saving and loading functions into
 keras/saving/, and resolve circular dependencies with the SavedModel code.

* saving.py has been split by format (hdf5 and model config)
* saving.py will be deleted after dependencies to keras.engine.saving are all fixed.

PiperOrigin-RevId: 228804552
---
 tensorflow/contrib/saved_model/BUILD          |  29 -
 .../python/saved_model/keras_saved_model.py   | 397 +-------
 .../saved_model/keras_saved_model_test.py     | 538 ----------
 tensorflow/python/keras/BUILD                 |  64 +-
 tensorflow/python/keras/__init__.py           |   2 -
 tensorflow/python/keras/engine/network.py     |   8 +-
 tensorflow/python/keras/engine/saving.py      | 945 +-----------------
 tensorflow/python/keras/engine/training.py    |   3 +-
 .../python/keras/engine/training_utils.py     |  56 --
 .../keras/engine/training_utils_test.py       | 179 ----
 .../keras/layers/cudnn_recurrent_test.py      |   2 +-
 tensorflow/python/keras/models.py             |  13 +-
 tensorflow/python/keras/saving/BUILD          |  72 --
 tensorflow/python/keras/saving/__init__.py    |  16 +-
 tensorflow/python/keras/saving/hdf5_format.py | 895 +++++++++++++++++
 .../hdf5_format_test.py}                      |  20 +-
 .../python/keras/saving/model_config.py       |  96 ++
 tensorflow/python/keras/saving/saved_model.py |  12 +-
 .../python/keras/saving/saving_utils.py       |  60 +-
 .../python/keras/saving/saving_utils_test.py  | 209 ++++
 tensorflow/python/saved_model/BUILD           |   2 +-
 21 files changed, 1369 insertions(+), 2249 deletions(-)
 delete mode 100644 tensorflow/contrib/saved_model/python/saved_model/keras_saved_model_test.py
 delete mode 100644 tensorflow/python/keras/saving/BUILD
 create mode 100644 tensorflow/python/keras/saving/hdf5_format.py
 rename tensorflow/python/keras/{engine/saving_test.py => saving/hdf5_format_test.py} (98%)
 create mode 100644 tensorflow/python/keras/saving/model_config.py
 create mode 100644 tensorflow/python/keras/saving/saving_utils_test.py

diff --git a/tensorflow/contrib/saved_model/BUILD b/tensorflow/contrib/saved_model/BUILD
index 269443b2c6..f0242a3b40 100644
--- a/tensorflow/contrib/saved_model/BUILD
+++ b/tensorflow/contrib/saved_model/BUILD
@@ -84,35 +84,6 @@ py_library(
     srcs_version = "PY2AND3",
     visibility = ["//visibility:public"],
     deps = [
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:lib",
-        "//tensorflow/python:metrics",
-        "//tensorflow/python:platform",
-        "//tensorflow/python:saver",
-        "//tensorflow/python:util",
-        "//tensorflow/python/estimator:estimator_py",
-        "//tensorflow/python/keras:engine",
-        "//tensorflow/python/saved_model",
-    ],
-)
-
-py_test(
-    name = "keras_saved_model_test",
-    size = "medium",
-    srcs = ["python/saved_model/keras_saved_model_test.py"],
-    srcs_version = "PY2AND3",
-    tags = [
-        "no_oss",  # TODO(b/119349471): Re-enable
-        "no_windows",
-    ],
-    deps = [
-        ":keras_saved_model",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:training",
-        "//tensorflow/python/estimator:estimator_py",
         "//tensorflow/python/keras",
-        "//third_party/py/numpy",
-        "@absl_py//absl/testing:parameterized",
     ],
 )
diff --git a/tensorflow/contrib/saved_model/python/saved_model/keras_saved_model.py b/tensorflow/contrib/saved_model/python/saved_model/keras_saved_model.py
index 2a4b6eae36..0392ed9eee 100644
--- a/tensorflow/contrib/saved_model/python/saved_model/keras_saved_model.py
+++ b/tensorflow/contrib/saved_model/python/saved_model/keras_saved_model.py
@@ -18,398 +18,9 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import os
-import six
+from tensorflow.python.keras import saving
 
-from tensorflow.python.client import session
-from tensorflow.python.framework import ops
-from tensorflow.python.keras import backend as K
-from tensorflow.python.keras import models as models_lib
-from tensorflow.python.keras import optimizers
-from tensorflow.python.keras.engine import sequential
-from tensorflow.python.keras.engine import training_utils
-from tensorflow.python.keras.metrics import Metric
-from tensorflow.python.keras.models import model_from_json
-from tensorflow.python.lib.io import file_io
-from tensorflow.python.ops import variables
-from tensorflow.python.platform import tf_logging as logging
-from tensorflow.python.saved_model import builder as saved_model_builder
-from tensorflow.python.saved_model import constants
-from tensorflow.python.saved_model import save as save_lib
-from tensorflow.python.saved_model import utils_impl as saved_model_utils
-from tensorflow.python.training import saver as saver_lib
-from tensorflow.python.training.checkpointable import util as checkpointable_utils
-from tensorflow.python.util import compat
-from tensorflow.python.util import nest
-from tensorflow_estimator.python.estimator import keras as estimator_keras_util
-from tensorflow_estimator.python.estimator import model_fn as model_fn_lib
-from tensorflow_estimator.python.estimator.export import export as export_helpers
 
-
-def save_keras_model(
-    model, saved_model_path, custom_objects=None, as_text=None,
-    input_signature=None, serving_only=False):
-  """Saves a `tf.keras.Model` into Tensorflow SavedModel format.
-
-  `save_model` generates new files/folders under the `saved_model_path` folder:
-  1) a checkpoint containing the model weights.
-  2) a saved_model.pb file containing the model's MetaGraphs. The prediction
-     graph is always exported. The evaluaton and training graphs are exported
-     if the following conditions are met:
-     - Evaluation: model loss is defined.
-     - Training: model is compiled with an optimizer defined under `tf.train`.
-       This is because `tf.keras.optimizers.Optimizer` instances cannot be
-       saved to checkpoints.
-  3) Model's json configuration, if model.get_config() has been implemented.
-     This file can be used to reload the model using
-     tf.keras.models.model_from_json(). Note that if any custom objects were
-     used, they should be passed to the `custom_object` argument when loading
-     the model.
-
-  Model limitations:
-  - Sequential and functional models can always be saved.
-  - Subclassed models can only be saved when `serving_only=True`. This is due to
-    the current implementation copying the model in order to export the training
-    and evaluation graphs. Because the topology of subclassed models cannot be
-    determined, the subclassed models cannot be cloned. Subclassed models will
-    be entirely exportable in the future.
-
-  Note that each mode is exported in separate graphs, so different modes do not
-  share variables. To use the train graph with evaluation or prediction graphs,
-  create a new checkpoint if variable values have been updated.
-
-  Example:
-
-  ```python
-  import tensorflow as tf
-
-  # Create a tf.keras model.
-  model = tf.keras.Sequential()
-  model.add(tf.keras.layers.Dense(1, input_shape=[10]))
-  model.summary()
-
-  # Save the tf.keras model in the SavedModel format.
-  saved_to_path = tf.contrib.saved_model.save_keras_model(
-        model, '/tmp/my_simple_tf_keras_saved_model')
-
-  # Load the saved keras model back.
-  model_prime = tf.contrib.saved_model.load_keras_model(saved_to_path)
-  model_prime.summary()
-  ```
-
-  Args:
-    model: A `tf.keras.Model` to be saved. If the model is subclassed, the flag
-      `serving_only` must be set to True.
-    saved_model_path: a string specifying the path to the SavedModel directory.
-      The SavedModel will be saved to a timestamped folder created within this
-      directory.
-    custom_objects: Optional dictionary mapping string names to custom classes
-      or functions (e.g. custom loss functions).
-    as_text: whether to write the `SavedModel` proto in text format. Currently
-      unavailable in serving-only mode.
-    input_signature: A possibly nested sequence of `tf.TensorSpec` objects, used
-      to specify the expected model inputs. `input_signature`'s nested structure
-      should match the expected nested structure of the inputs to the model. If
-      this is not set, this function will attempt to infer the input shapes and
-      dtypes from the model. Note that if the model is subclassed, the tensor
-      inputs to the call function should be nested in the first argument (this
-      is a general requirement for using subclassed models with Keras functions
-      .fit(), .predict(), etc.).
-    serving_only: Export only the outputs produced from calling the model in
-      predict mode. The losses, optimizer, and other training configurations are
-      not saved. If the SavedModel will only be used for serving (rather than
-      retraining), or if the model is subclassed, this can be set to True.
-
-  Returns:
-    String path to the SavedModel folder, a subdirectory of `saved_model_path`.
-
-  Raises:
-    NotImplementedError: If the model is a subclassed model, and serving_only is
-      False.
-    ValueError: If the input signature cannot be inferred from the model.
-  """
-  export_dir = export_helpers.get_timestamped_export_dir(saved_model_path)
-
-  if serving_only:
-    save_lib.save(
-        model, export_dir,
-        signatures=training_utils.trace_model_call(model, input_signature))
-  else:
-    _save_v1_format(model, export_dir, custom_objects, as_text, input_signature)
-
-  try:
-    _export_model_json(model, export_dir)
-  except NotImplementedError:
-    logging.warning('Skipped saving model JSON, subclassed model does not have '
-                    'get_config() defined.')
-
-  return export_dir
-
-
-def _export_model_json(model, saved_model_path):
-  """Saves model configuration as a json string under assets folder."""
-  model_json = model.to_json()
-  model_json_filepath = os.path.join(
-      saved_model_utils.get_or_create_assets_dir(saved_model_path),
-      compat.as_text(constants.SAVED_MODEL_FILENAME_JSON))
-  file_io.write_string_to_file(model_json_filepath, model_json)
-
-
-def _export_model_variables(model, saved_model_path):
-  """Saves model weights in checkpoint format under variables folder."""
-  saved_model_utils.get_or_create_variables_dir(saved_model_path)
-  checkpoint_prefix = saved_model_utils.get_variables_path(saved_model_path)
-  model.save_weights(checkpoint_prefix, save_format='tf', overwrite=True)
-  return checkpoint_prefix
-
-
-def _save_v1_format(model, path, custom_objects, as_text, input_signature):
-  """Exports model to v1 SavedModel format."""
-  if not model._is_graph_network:
-    if isinstance(model, sequential.Sequential):
-      # If input shape is not directly set in the model, the exported model
-      # will infer the expected shapes of the input from the model.
-      if not model.built and input_signature is None:
-        raise ValueError(
-            'Sequential model\'s input shape is unknown. Please build the '
-            'model, or use the input_signature argument to specify the '
-            'model inputs.')
-    else:
-      raise NotImplementedError(
-          'Subclassed models can only be exported for serving. Please set '
-          'argument serving_only=True.')
-
-  builder = saved_model_builder._SavedModelBuilder(path)
-
-  # Manually save variables to export them in an object-based checkpoint. This
-  # skips the `builder.add_meta_graph_and_variables()` step, which saves a
-  # named-based checkpoint.
-  # TODO(b/113134168): Add fn to Builder to save with object-based saver.
-  # TODO(b/113178242): This should only export the model json structure. Only
-  # one save is needed once the weights can be copied from the model to clone.
-  checkpoint_path = _export_model_variables(model, path)
-
-  # Export each mode. Use ModeKeys enums defined for `Estimator` to ensure that
-  # Keras models and `Estimator`s are exported with the same format.
-  # Every time a mode is exported, the code checks to see if new variables have
-  # been created (e.g. optimizer slot variables). If that is the case, the
-  # checkpoint is re-saved to include the new variables.
-  export_args = {'builder': builder,
-                 'model': model,
-                 'custom_objects': custom_objects,
-                 'checkpoint_path': checkpoint_path,
-                 'input_signature': input_signature}
-
-  has_saved_vars = False
-  if model.optimizer:
-    # TODO(kathywu): Verify this works with v2 optimizer.
-    if isinstance(model.optimizer, optimizers.TFOptimizer):
-      _export_mode(model_fn_lib.ModeKeys.TRAIN, has_saved_vars, **export_args)
-      has_saved_vars = True
-      _export_mode(model_fn_lib.ModeKeys.EVAL, has_saved_vars, **export_args)
-    else:
-      logging.warning(
-          'Model was compiled with an optimizer, but the optimizer is not from '
-          '`tf.train` (e.g. `tf.train.AdagradOptimizer`). Only the serving '
-          'graph was exported. The train and evaluate graphs were not added to '
-          'the SavedModel.')
-  _export_mode(model_fn_lib.ModeKeys.PREDICT, has_saved_vars, **export_args)
-
-  builder.save(as_text)
-
-
-def _get_var_list(model):
-  """Returns list of all checkpointed saveable objects in the model."""
-  return checkpointable_utils.named_saveables(model)
-
-
-def create_placeholder(spec):
-  return K.placeholder(shape=spec.shape, dtype=spec.dtype, name=spec.name)
-
-
-def _export_mode(
-    mode, has_saved_vars, builder, model, custom_objects, checkpoint_path,
-    input_signature):
-  """Exports a model, and optionally saves new vars from the clone model.
-
-  Args:
-    mode: A `tf.estimator.ModeKeys` string.
-    has_saved_vars: A `boolean` indicating whether the SavedModel has already
-      exported variables.
-    builder: A `SavedModelBuilder` object.
-    model: A `tf.keras.Model` object.
-    custom_objects: A dictionary mapping string names to custom classes
-      or functions.
-    checkpoint_path: String path to checkpoint.
-    input_signature: Nested TensorSpec containing the expected inputs. Can be
-      `None`, in which case the signature will be inferred from the model.
-
-  Raises:
-    ValueError: If the train/eval mode is being exported, but the model does
-      not have an optimizer.
-  """
-  compile_clone = (mode != model_fn_lib.ModeKeys.PREDICT)
-  if compile_clone and not model.optimizer:
-    raise ValueError(
-        'Model does not have an optimizer. Cannot export mode %s' % mode)
-
-  model_graph = ops.get_default_graph()
-  with ops.Graph().as_default() as g:
-
-    K.set_learning_phase(mode == model_fn_lib.ModeKeys.TRAIN)
-
-    if input_signature is None:
-      input_tensors = None
-    else:
-      input_tensors = nest.map_structure(create_placeholder, input_signature)
-
-    # Clone the model into blank graph. This will create placeholders for inputs
-    # and targets.
-    clone = models_lib.clone_and_build_model(
-        model, input_tensors=input_tensors, custom_objects=custom_objects,
-        compile_clone=compile_clone)
-
-    # Make sure that iterations variable is added to the global step collection,
-    # to ensure that, when the SavedModel graph is loaded, the iterations
-    # variable is returned by `tf.train.get_global_step()`. This is required for
-    # compatibility with the SavedModelEstimator.
-    if compile_clone:
-      g.add_to_collection(ops.GraphKeys.GLOBAL_STEP, clone.optimizer.iterations)
-
-    # Extract update and train ops from train/test/predict functions.
-    train_op = None
-    if mode == model_fn_lib.ModeKeys.TRAIN:
-      clone._make_train_function()
-      train_op = clone.train_function.updates_op
-    elif mode == model_fn_lib.ModeKeys.EVAL:
-      clone._make_test_function()
-    else:
-      clone._make_predict_function()
-    g.get_collection_ref(ops.GraphKeys.UPDATE_OPS).extend(clone.state_updates)
-
-    clone_var_list = checkpointable_utils.named_saveables(clone)
-
-    with session.Session().as_default():
-      if has_saved_vars:
-        # Confirm all variables in the clone have an entry in the checkpoint.
-        status = clone.load_weights(checkpoint_path)
-        status.assert_existing_objects_matched()
-      else:
-        # Confirm that variables between the clone and model match up exactly,
-        # not counting optimizer objects. Optimizer objects are ignored because
-        # if the model has not trained, the slot variables will not have been
-        # created yet.
-        # TODO(b/113179535): Replace with checkpointable equivalence.
-        _assert_same_non_optimizer_objects(model, model_graph, clone, g)
-
-        # TODO(b/113178242): Use value transfer for checkpointable objects.
-        clone.load_weights(checkpoint_path)
-
-        # Add graph and variables to SavedModel.
-        # TODO(b/113134168): Switch to add_meta_graph_and_variables.
-        clone.save_weights(checkpoint_path, save_format='tf', overwrite=True)
-        builder._has_saved_variables = True
-
-    # Add graph to the SavedModel builder.
-    builder.add_meta_graph(
-        model_fn_lib.EXPORT_TAG_MAP[mode],
-        signature_def_map=_create_signature_def_map(clone, mode),
-        saver=saver_lib.Saver(clone_var_list),
-        init_op=variables.local_variables_initializer(),
-        train_op=train_op)
-    return None
-
-
-def _create_signature_def_map(model, mode):
-  """Creates a SignatureDef map from a Keras model."""
-  inputs_dict = {name: x for name, x in zip(model.input_names, model.inputs)}
-  if model.optimizer:
-    targets_dict = {x.name.split(':')[0]: x
-                    for x in model.targets if x is not None}
-    inputs_dict.update(targets_dict)
-  outputs_dict = {name: x
-                  for name, x in zip(model.output_names, model.outputs)}
-  metrics = estimator_keras_util._convert_keras_metrics_to_estimator(model)
-
-  # Add metric variables to the `LOCAL_VARIABLES` collection. Metric variables
-  # are by default not added to any collections. We are doing this here, so
-  # that metric variables get initialized.
-  local_vars = set(ops.get_collection(ops.GraphKeys.LOCAL_VARIABLES))
-  vars_to_add = set()
-  if metrics is not None:
-    for key, value in six.iteritems(metrics):
-      if isinstance(value, Metric):
-        vars_to_add.update(value.variables)
-        # Convert Metric instances to (value_tensor, update_op) tuple.
-        metrics[key] = (value.result(), value.updates[0])
-  # Remove variables that are in the local variables collection already.
-  vars_to_add = vars_to_add.difference(local_vars)
-  for v in vars_to_add:
-    ops.add_to_collection(ops.GraphKeys.LOCAL_VARIABLES, v)
-
-  export_outputs = model_fn_lib.export_outputs_for_mode(
-      mode,
-      predictions=outputs_dict,
-      loss=model.total_loss if model.optimizer else None,
-      metrics=metrics)
-  return export_helpers.build_all_signature_defs(
-      inputs_dict,
-      export_outputs=export_outputs,
-      serving_only=(mode == model_fn_lib.ModeKeys.PREDICT))
-
-
-def _assert_same_non_optimizer_objects(model, model_graph, clone, clone_graph):  # pylint: disable=unused-argument
-  """Asserts model and clone contain the same checkpointable objects."""
-
-  # TODO(fchollet, kathywu): make sure this works in eager mode.
-  return True
-
-
-def load_keras_model(saved_model_path):
-  """Loads a keras.Model from SavedModel.
-
-  load_model reinstantiates model state by:
-  1) loading model topology from json (this will eventually come
-     from metagraph).
-  2) loading model weights from checkpoint.
-
-  Example:
-
-  ```python
-  import tensorflow as tf
-
-  # Create a tf.keras model.
-  model = tf.keras.Sequential()
-  model.add(tf.keras.layers.Dense(1, input_shape=[10]))
-  model.summary()
-
-  # Save the tf.keras model in the SavedModel format.
-  saved_to_path = tf.contrib.saved_model.save_keras_model(
-        model, '/tmp/my_simple_tf_keras_saved_model')
-
-  # Load the saved keras model back.
-  model_prime = tf.contrib.saved_model.load_keras_model(saved_to_path)
-  model_prime.summary()
-  ```
-
-  Args:
-    saved_model_path: a string specifying the path to an existing SavedModel.
-
-  Returns:
-    a keras.Model instance.
-  """
-  # restore model topology from json string
-  model_json_filepath = os.path.join(
-      compat.as_bytes(saved_model_path),
-      compat.as_bytes(constants.ASSETS_DIRECTORY),
-      compat.as_bytes(constants.SAVED_MODEL_FILENAME_JSON))
-  model_json = file_io.read_file_to_string(model_json_filepath)
-  model = model_from_json(model_json)
-
-  # restore model weights
-  checkpoint_prefix = os.path.join(
-      compat.as_text(saved_model_path),
-      compat.as_text(constants.VARIABLES_DIRECTORY),
-      compat.as_text(constants.VARIABLES_FILENAME))
-  model.load_weights(checkpoint_prefix)
-  return model
+# TODO(kathywu): Remove all contrib callers, switch to tf.keras.
+save_keras_model = saving.export
+load_keras_model = saving.load_from_saved_model
diff --git a/tensorflow/contrib/saved_model/python/saved_model/keras_saved_model_test.py b/tensorflow/contrib/saved_model/python/saved_model/keras_saved_model_test.py
deleted file mode 100644
index fbf8138493..0000000000
--- a/tensorflow/contrib/saved_model/python/saved_model/keras_saved_model_test.py
+++ /dev/null
@@ -1,538 +0,0 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-# pylint: disable=protected-access
-"""Tests for saving/loading function for keras Model."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os
-import shutil
-
-from absl.testing import parameterized
-import numpy as np
-
-from tensorflow.contrib.saved_model.python.saved_model import keras_saved_model
-from tensorflow.python import keras
-from tensorflow.python.client import session
-from tensorflow.python.eager import context
-from tensorflow.python.estimator import model_fn as model_fn_lib
-from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import ops
-from tensorflow.python.framework import tensor_spec
-from tensorflow.python.framework import test_util
-from tensorflow.python.keras.engine import training
-from tensorflow.python.keras.utils import tf_utils
-from tensorflow.python.ops import array_ops
-from tensorflow.python.platform import test
-from tensorflow.python.saved_model import loader_impl
-from tensorflow.python.saved_model import signature_constants
-from tensorflow.python.training import training as training_module
-
-
-class TestModelSavingandLoading(test.TestCase):
-
-  def _save_model_dir(self, dirname='saved_model'):
-    temp_dir = self.get_temp_dir()
-    self.addCleanup(shutil.rmtree, temp_dir, ignore_errors=True)
-    return os.path.join(temp_dir, dirname)
-
-  def test_saving_sequential_model(self):
-    with self.cached_session():
-      model = keras.models.Sequential()
-      model.add(keras.layers.Dense(2, input_shape=(3,)))
-      model.add(keras.layers.RepeatVector(3))
-      model.add(keras.layers.TimeDistributed(keras.layers.Dense(3)))
-      model.compile(
-          loss=keras.losses.MSE,
-          optimizer=keras.optimizers.RMSprop(lr=0.0001),
-          metrics=[keras.metrics.categorical_accuracy],
-          sample_weight_mode='temporal')
-      x = np.random.random((1, 3))
-      y = np.random.random((1, 3, 3))
-      model.train_on_batch(x, y)
-
-      ref_y = model.predict(x)
-
-      temp_saved_model = self._save_model_dir()
-      output_path = keras_saved_model.save_keras_model(model, temp_saved_model)
-
-      loaded_model = keras_saved_model.load_keras_model(output_path)
-      y = loaded_model.predict(x)
-      self.assertAllClose(ref_y, y, atol=1e-05)
-
-  @test_util.run_in_graph_and_eager_modes
-  def test_saving_sequential_model_without_compile(self):
-    with self.cached_session():
-      model = keras.models.Sequential()
-      model.add(keras.layers.Dense(2, input_shape=(3,)))
-      model.add(keras.layers.RepeatVector(3))
-      model.add(keras.layers.TimeDistributed(keras.layers.Dense(3)))
-
-      x = np.random.random((1, 3))
-      ref_y = model.predict(x)
-
-      temp_saved_model = self._save_model_dir()
-      output_path = keras_saved_model.save_keras_model(model, temp_saved_model)
-      loaded_model = keras_saved_model.load_keras_model(output_path)
-
-      y = loaded_model.predict(x)
-      self.assertAllClose(ref_y, y, atol=1e-05)
-
-  def test_saving_functional_model(self):
-    with self.cached_session():
-      inputs = keras.layers.Input(shape=(3,))
-      x = keras.layers.Dense(2)(inputs)
-      output = keras.layers.Dense(3)(x)
-
-      model = keras.models.Model(inputs, output)
-      model.compile(
-          loss=keras.losses.MSE,
-          optimizer=keras.optimizers.RMSprop(lr=0.0001),
-          metrics=[keras.metrics.categorical_accuracy])
-      x = np.random.random((1, 3))
-      y = np.random.random((1, 3))
-      model.train_on_batch(x, y)
-
-      ref_y = model.predict(x)
-
-      temp_saved_model = self._save_model_dir()
-      output_path = keras_saved_model.save_keras_model(model, temp_saved_model)
-      loaded_model = keras_saved_model.load_keras_model(output_path)
-
-      y = loaded_model.predict(x)
-      self.assertAllClose(ref_y, y, atol=1e-05)
-
-  @test_util.run_in_graph_and_eager_modes
-  def test_saving_functional_model_without_compile(self):
-    with self.cached_session():
-      inputs = keras.layers.Input(shape=(3,))
-      x = keras.layers.Dense(2)(inputs)
-      output = keras.layers.Dense(3)(x)
-
-      model = keras.models.Model(inputs, output)
-
-      x = np.random.random((1, 3))
-      y = np.random.random((1, 3))
-
-      ref_y = model.predict(x)
-
-      temp_saved_model = self._save_model_dir()
-      output_path = keras_saved_model.save_keras_model(model, temp_saved_model)
-      loaded_model = keras_saved_model.load_keras_model(output_path)
-
-      y = loaded_model.predict(x)
-      self.assertAllClose(ref_y, y, atol=1e-05)
-
-  @test_util.run_in_graph_and_eager_modes
-  def test_saving_with_tf_optimizer(self):
-    with self.cached_session():
-      model = keras.models.Sequential()
-      model.add(keras.layers.Dense(2, input_shape=(3,)))
-      model.add(keras.layers.Dense(3))
-      model.compile(
-          loss='mse',
-          optimizer=training_module.RMSPropOptimizer(0.1),
-          metrics=['acc'])
-
-      x = np.random.random((1, 3))
-      y = np.random.random((1, 3))
-      model.train_on_batch(x, y)
-      ref_y = model.predict(x)
-
-      temp_saved_model = self._save_model_dir()
-      output_path = keras_saved_model.save_keras_model(model, temp_saved_model)
-      loaded_model = keras_saved_model.load_keras_model(output_path)
-      loaded_model.compile(
-          loss='mse',
-          optimizer=training_module.RMSPropOptimizer(0.1),
-          metrics=['acc'])
-      y = loaded_model.predict(x)
-      self.assertAllClose(ref_y, y, atol=1e-05)
-
-      # test that new updates are the same with both models
-      x = np.random.random((1, 3))
-      y = np.random.random((1, 3))
-
-      ref_loss = model.train_on_batch(x, y)
-      loss = loaded_model.train_on_batch(x, y)
-      self.assertAllClose(ref_loss, loss, atol=1e-05)
-
-      ref_y = model.predict(x)
-      y = loaded_model.predict(x)
-      self.assertAllClose(ref_y, y, atol=1e-05)
-
-      # test saving/loading again
-      temp_saved_model2 = self._save_model_dir('saved_model_2')
-      output_path2 = keras_saved_model.save_keras_model(
-          loaded_model, temp_saved_model2)
-      loaded_model = keras_saved_model.load_keras_model(output_path2)
-      y = loaded_model.predict(x)
-      self.assertAllClose(ref_y, y, atol=1e-05)
-
-  def test_saving_subclassed_model_raise_error(self):
-    # For now, saving subclassed model should raise an error. It should be
-    # avoided later with loading from SavedModel.pb.
-
-    class SubclassedModel(training.Model):
-
-      def __init__(self):
-        super(SubclassedModel, self).__init__()
-        self.layer1 = keras.layers.Dense(3)
-        self.layer2 = keras.layers.Dense(1)
-
-      def call(self, inp):
-        return self.layer2(self.layer1(inp))
-
-    model = SubclassedModel()
-
-    temp_saved_model = self._save_model_dir()
-    with self.assertRaises(NotImplementedError):
-      keras_saved_model.save_keras_model(model, temp_saved_model)
-
-
-class LayerWithLearningPhase(keras.engine.base_layer.Layer):
-
-  def call(self, x):
-    phase = keras.backend.learning_phase()
-    output = tf_utils.smart_cond(
-        phase, lambda: x * 0, lambda: array_ops.identity(x))
-    if not context.executing_eagerly():
-      output._uses_learning_phase = True  # pylint: disable=protected-access
-    return output
-
-  def compute_output_shape(self, input_shape):
-    return input_shape
-
-
-def functional_model(uses_learning_phase=True):
-  inputs = keras.layers.Input(shape=(3,))
-  x = keras.layers.Dense(2)(inputs)
-  x = keras.layers.Dense(3)(x)
-  if uses_learning_phase:
-    x = LayerWithLearningPhase()(x)
-  return keras.models.Model(inputs, x)
-
-
-def sequential_model(uses_learning_phase=True):
-  model = keras.models.Sequential()
-  model.add(keras.layers.Dense(2, input_shape=(3,)))
-  model.add(keras.layers.Dense(3))
-  if uses_learning_phase:
-    model.add(LayerWithLearningPhase())
-  return model
-
-
-def sequential_model_without_input_shape(uses_learning_phase=True):
-  model = keras.models.Sequential()
-  model.add(keras.layers.Dense(2))
-  model.add(keras.layers.Dense(3))
-  if uses_learning_phase:
-    model.add(LayerWithLearningPhase())
-  return model
-
-
-class Subclassed(keras.models.Model):
-
-  def __init__(self):
-    super(Subclassed, self).__init__()
-    self.dense1 = keras.layers.Dense(2)
-    self.dense2 = keras.layers.Dense(3)
-
-  def call(self, inputs):
-    x = self.dense1(inputs)
-    x = self.dense2(x)
-    return x
-
-
-def subclassed_model():
-  return Subclassed()
-
-
-def load_model(sess, path, mode):
-  tags = model_fn_lib.EXPORT_TAG_MAP[mode]
-  if mode == model_fn_lib.ModeKeys.PREDICT:
-    sig_def_key = signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY
-  else:
-    sig_def_key = mode
-
-  meta_graph_def = loader_impl.load(sess, tags, path)
-  inputs = {
-      k: sess.graph.get_tensor_by_name(v.name)
-      for k, v in meta_graph_def.signature_def[sig_def_key].inputs.items()}
-  outputs = {
-      k: sess.graph.get_tensor_by_name(v.name)
-      for k, v in meta_graph_def.signature_def[sig_def_key].outputs.items()}
-  return inputs, outputs, meta_graph_def
-
-
-@test_util.run_all_in_graph_and_eager_modes
-class TestModelSavedModelExport(test.TestCase, parameterized.TestCase):
-
-  def _save_model_dir(self, dirname='saved_model'):
-    temp_dir = self.get_temp_dir()
-    self.addCleanup(shutil.rmtree, temp_dir, ignore_errors=True)
-    return os.path.join(temp_dir, dirname)
-
-  @parameterized.parameters(
-      {
-          'model_builder': functional_model,
-          'uses_learning_phase': True,
-          'optimizer': training_module.AdadeltaOptimizer(),
-          'train_before_export': True},
-      {
-          'model_builder': functional_model,
-          'uses_learning_phase': True,
-          'optimizer': training_module.AdadeltaOptimizer(),
-          'train_before_export': False},
-      {
-          'model_builder': functional_model,
-          'uses_learning_phase': False,
-          'optimizer': None,
-          'train_before_export': False},
-      {
-          'model_builder': sequential_model,
-          'uses_learning_phase': True,
-          'optimizer': training_module.AdadeltaOptimizer(),
-          'train_before_export': True},
-      {
-          'model_builder': sequential_model,
-          'uses_learning_phase': True,
-          'optimizer': training_module.AdadeltaOptimizer(),
-          'train_before_export': False},
-      {
-          'model_builder': sequential_model,
-          'uses_learning_phase': False,
-          'optimizer': None,
-          'train_before_export': False},
-      {
-          'model_builder': sequential_model_without_input_shape,
-          'uses_learning_phase': True,
-          'optimizer': training_module.AdadeltaOptimizer(),
-          'train_before_export': False})
-  def testSaveAndLoadSavedModelExport(
-      self, model_builder, uses_learning_phase, optimizer, train_before_export):
-    saved_model_path = self._save_model_dir()
-    with self.session(graph=ops.Graph()):
-      np.random.seed(130)
-      input_arr = np.random.random((1, 3))
-      target_arr = np.random.random((1, 3))
-
-      model = model_builder(uses_learning_phase)
-      if optimizer is not None:
-        model.compile(
-            loss='mse',
-            optimizer=optimizer,
-            metrics=['mae'])
-        if train_before_export:
-          model.train_on_batch(input_arr, target_arr)
-
-        ref_loss, ref_mae = model.evaluate(input_arr, target_arr)
-
-      ref_predict = model.predict(input_arr)
-
-      # Export SavedModel
-      output_path = keras_saved_model.save_keras_model(model, saved_model_path)
-
-    input_name = model.input_names[0]
-    output_name = model.output_names[0]
-    target_name = output_name + '_target'
-
-    # Load predict graph, and test predictions
-    with session.Session(graph=ops.Graph()) as sess:
-      inputs, outputs, _ = load_model(sess, output_path,
-                                      model_fn_lib.ModeKeys.PREDICT)
-
-      predictions = sess.run(outputs[output_name],
-                             {inputs[input_name]: input_arr})
-      self.assertAllClose(ref_predict, predictions, atol=1e-05)
-
-    if optimizer:
-      # Load eval graph, and test predictions, loss and metric values
-      with session.Session(graph=ops.Graph()) as sess:
-        inputs, outputs, _ = load_model(sess, output_path,
-                                        model_fn_lib.ModeKeys.EVAL)
-
-        # First obtain the loss and predictions, and run the metric update op by
-        # feeding in the inputs and targets.
-        loss, predictions, _ = sess.run(
-            (outputs['loss'], outputs['predictions/' + output_name],
-             outputs['metrics/mean_absolute_error/update_op']), {
-                 inputs[input_name]: input_arr,
-                 inputs[target_name]: target_arr
-             })
-
-        # The metric value should be run after the update op, to ensure that it
-        # reflects the correct value.
-        metric_value = sess.run(outputs['metrics/mean_absolute_error/value'])
-
-        self.assertEqual(int(train_before_export),
-                         sess.run(training_module.get_global_step()))
-        self.assertAllClose(ref_loss, loss, atol=1e-05)
-        self.assertAllClose(ref_mae, metric_value, atol=1e-05)
-        self.assertAllClose(ref_predict, predictions, atol=1e-05)
-
-      # Load train graph, and check for the train op, and prediction values
-      with session.Session(graph=ops.Graph()) as sess:
-        inputs, outputs, meta_graph_def = load_model(
-            sess, output_path, model_fn_lib.ModeKeys.TRAIN)
-        self.assertEqual(int(train_before_export),
-                         sess.run(training_module.get_global_step()))
-        self.assertIn('loss', outputs)
-        self.assertIn('metrics/mean_absolute_error/update_op', outputs)
-        self.assertIn('metrics/mean_absolute_error/value', outputs)
-        self.assertIn('predictions/' + output_name, outputs)
-
-        # Train for a step
-        train_op = loader_impl.get_train_op(meta_graph_def)
-        train_outputs, _ = sess.run(
-            [outputs, train_op], {inputs[input_name]: input_arr,
-                                  inputs[target_name]: target_arr})
-        self.assertEqual(int(train_before_export) + 1,
-                         sess.run(training_module.get_global_step()))
-
-        if uses_learning_phase:
-          self.assertAllClose(
-              [[0, 0, 0]], train_outputs['predictions/' + output_name],
-              atol=1e-05)
-        else:
-          self.assertNotAllClose(
-              [[0, 0, 0]], train_outputs['predictions/' + output_name],
-              atol=1e-05)
-
-  def testSaveAndLoadSavedModelWithCustomObject(self):
-    saved_model_path = self._save_model_dir()
-    with session.Session(graph=ops.Graph()) as sess:
-      def relu6(x):
-        return keras.backend.relu(x, max_value=6)
-      inputs = keras.layers.Input(shape=(1,))
-      outputs = keras.layers.Activation(relu6)(inputs)
-      model = keras.models.Model(inputs, outputs)
-      output_path = keras_saved_model.save_keras_model(
-          model, saved_model_path, custom_objects={'relu6': relu6})
-    with session.Session(graph=ops.Graph()) as sess:
-      inputs, outputs, _ = load_model(sess, output_path,
-                                      model_fn_lib.ModeKeys.PREDICT)
-      input_name = model.input_names[0]
-      output_name = model.output_names[0]
-      predictions = sess.run(
-          outputs[output_name], {inputs[input_name]: [[7], [-3], [4]]})
-      self.assertAllEqual([[6], [0], [4]], predictions)
-
-  def testAssertModelCloneSameObjectsIgnoreOptimizer(self):
-    input_arr = np.random.random((1, 3))
-    target_arr = np.random.random((1, 3))
-
-    model_graph = ops.Graph()
-    clone_graph = ops.Graph()
-
-    # Create two models with the same layers but different optimizers.
-    with session.Session(graph=model_graph):
-      inputs = keras.layers.Input(shape=(3,))
-      x = keras.layers.Dense(2)(inputs)
-      x = keras.layers.Dense(3)(x)
-      model = keras.models.Model(inputs, x)
-
-      model.compile(loss='mse', optimizer=training_module.AdadeltaOptimizer())
-      model.train_on_batch(input_arr, target_arr)
-
-    with session.Session(graph=clone_graph):
-      inputs = keras.layers.Input(shape=(3,))
-      x = keras.layers.Dense(2)(inputs)
-      x = keras.layers.Dense(3)(x)
-      clone = keras.models.Model(inputs, x)
-      clone.compile(loss='mse', optimizer=keras.optimizers.RMSprop(lr=0.0001))
-      clone.train_on_batch(input_arr, target_arr)
-
-    keras_saved_model._assert_same_non_optimizer_objects(
-        model, model_graph, clone, clone_graph)
-
-  def testAssertModelCloneSameObjectsThrowError(self):
-    input_arr = np.random.random((1, 3))
-    target_arr = np.random.random((1, 3))
-
-    model_graph = ops.Graph()
-    clone_graph = ops.Graph()
-
-    # Create two models with the same layers but different optimizers.
-    with session.Session(graph=model_graph):
-      inputs = keras.layers.Input(shape=(3,))
-      x = keras.layers.Dense(2)(inputs)
-      x = keras.layers.Dense(3)(x)
-      model = keras.models.Model(inputs, x)
-
-      model.compile(loss='mse', optimizer=training_module.AdadeltaOptimizer())
-      model.train_on_batch(input_arr, target_arr)
-
-    with session.Session(graph=clone_graph):
-      inputs = keras.layers.Input(shape=(3,))
-      x = keras.layers.Dense(2)(inputs)
-      x = keras.layers.Dense(4)(x)
-      x = keras.layers.Dense(3)(x)
-      clone = keras.models.Model(inputs, x)
-      clone.compile(loss='mse', optimizer=keras.optimizers.RMSprop(lr=0.0001))
-      clone.train_on_batch(input_arr, target_arr)
-
-  def testSaveSequentialModelWithoutInputShapes(self):
-    model = sequential_model_without_input_shape(True)
-    # A Sequential model that hasn't been built should raise an error.
-    with self.assertRaisesRegexp(ValueError, 'Please build the model'):
-      keras_saved_model.save_keras_model(model, '')
-
-    saved_model_path = self._save_model_dir()
-    output_path = keras_saved_model.save_keras_model(
-        model, saved_model_path,
-        input_signature=tensor_spec.TensorSpec(shape=(10, 11, 12, 13, 14),
-                                               dtype=dtypes.float32,
-                                               name='spec_input'))
-
-    with session.Session(graph=ops.Graph()) as sess:
-      inputs, outputs, _ = load_model(sess, output_path,
-                                      model_fn_lib.ModeKeys.PREDICT)
-      self.assertEqual(5, inputs[next(iter(inputs.keys()))].shape.ndims)
-      self.assertEqual(5, outputs[next(iter(outputs.keys()))].shape.ndims)
-      self.assertEqual(3, outputs[next(iter(outputs.keys()))].shape[-1])
-
-  @test_util.run_v2_only
-  @parameterized.parameters(
-      {
-          'model_builder': sequential_model_without_input_shape,
-          'input_signature': [tensor_spec.TensorSpec(shape=[None, 3],
-                                                     dtype=dtypes.float32)]},
-      {
-          'model_builder': subclassed_model,
-          'input_signature': [tensor_spec.TensorSpec(shape=[None, 3],
-                                                     dtype=dtypes.float32)]})
-  def testServingOnly(self, model_builder, input_signature):
-    saved_model_path = self._save_model_dir()
-    input_arr = np.random.random((5, 3)).astype(np.float32)
-    model = model_builder()
-    ref_predict = model.predict(input_arr)
-
-    output_path = keras_saved_model.save_keras_model(
-        model, saved_model_path, serving_only=True,
-        input_signature=input_signature)
-
-    # Load predict graph, and test predictions
-    with session.Session(graph=ops.Graph()) as sess:
-      inputs, outputs, _ = load_model(sess, output_path,
-                                      model_fn_lib.ModeKeys.PREDICT)
-      predictions = sess.run(outputs[next(iter(outputs.keys()))],
-                             {inputs[next(iter(inputs.keys()))]: input_arr})
-      self.assertAllClose(ref_predict, predictions, atol=1e-05)
-
-
-if __name__ == '__main__':
-  test.main()
diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 536acca298..5f44a1b4e8 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -61,13 +61,11 @@ py_library(
         ":engine",
         ":layers",
         ":pil_for_keras",
-        "@keras_applications_archive//:keras_applications",
+        ":saving",
         "//tensorflow/python:training",
         "//tensorflow/python/keras/optimizer_v2",
-        # TODO(kathywu): move saving into engine after resolving circular
-        # dependencies between Keras and SavedModel
-        "//tensorflow/python/keras/saving",
         "//tensorflow/python/saved_model",
+        "@keras_applications_archive//:keras_applications",
     ],
 )
 
@@ -152,6 +150,7 @@ py_library(
         ":losses",
         ":optimizers",
         ":regularizers",
+        ":saving",
         "//tensorflow/python/data",
         "//tensorflow/python/distribute:reduce_util",
         "//tensorflow/python/training/checkpointable:data_structures",
@@ -160,6 +159,28 @@ py_library(
     ],
 )
 
+py_library(
+    name = "saving",
+    srcs = [
+        "saving/__init__.py",
+        "saving/hdf5_format.py",
+        "saving/model_config.py",
+        "saving/saved_model.py",
+        "saving/saving_utils.py",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":backend",
+        ":engine_utils",
+        ":optimizers",
+        "//tensorflow/python:lib",
+        "//tensorflow/python:mode_keys",
+        "//tensorflow/python:saver",
+        "//tensorflow/python/saved_model",
+        "//tensorflow/python/saved_model/model_utils",
+    ],
+)
+
 py_library(
     name = "activations",
     srcs = [
@@ -248,6 +269,7 @@ py_library(
 py_library(
     name = "engine_utils",
     srcs = [
+        "utils/conv_utils.py",
         "utils/data_utils.py",
         "utils/io_utils.py",
         "utils/losses_utils.py",
@@ -276,7 +298,6 @@ py_library(
         "layers/recurrent.py",
         "layers/serialization.py",
         "layers/wrappers.py",
-        "utils/conv_utils.py",
         "utils/generic_utils.py",
         "utils/layer_utils.py",
         "utils/tf_utils.py",
@@ -1001,9 +1022,9 @@ tf_py_test(
 )
 
 tf_py_test(
-    name = "saving_test",
+    name = "hdf5_format_test",
     size = "medium",
-    srcs = ["engine/saving_test.py"],
+    srcs = ["saving/hdf5_format_test.py"],
     additional_deps = [
         ":keras",
         "@absl_py//absl/testing:parameterized",
@@ -1064,3 +1085,32 @@ tf_py_test(
     ],
     tags = ["notsan"],
 )
+
+tf_py_test(
+    name = "saved_model_test",
+    size = "medium",
+    srcs = ["saving/saved_model_test.py"],
+    additional_deps = [
+        ":keras",
+        "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
+    ],
+    tags = [
+        "no_oss",  # TODO(b/119349471): Re-enable
+        "no_windows",
+    ],
+)
+
+tf_py_test(
+    name = "saving_utils_test",
+    size = "medium",
+    srcs = ["saving/saving_utils_test.py"],
+    additional_deps = [
+        ":keras",
+        "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
+    ],
+    tags = ["notsan"],
+)
diff --git a/tensorflow/python/keras/__init__.py b/tensorflow/python/keras/__init__.py
index e59744f64d..f024b9b59a 100644
--- a/tensorflow/python/keras/__init__.py
+++ b/tensorflow/python/keras/__init__.py
@@ -42,8 +42,6 @@ from tensorflow.python.keras import wrappers
 from tensorflow.python.keras.layers import Input
 from tensorflow.python.keras.models import Model
 from tensorflow.python.keras.models import Sequential
-from tensorflow.python.keras.saving.saved_model import export
-from tensorflow.python.keras.saving.saved_model import load_from_saved_model
 
 from tensorflow.python.util.tf_export import keras_export
 
diff --git a/tensorflow/python/keras/engine/network.py b/tensorflow/python/keras/engine/network.py
index 6c3691a30c..991950993f 100644
--- a/tensorflow/python/keras/engine/network.py
+++ b/tensorflow/python/keras/engine/network.py
@@ -37,8 +37,8 @@ from tensorflow.python.framework import tensor_shape
 from tensorflow.python.keras import backend
 from tensorflow.python.keras.engine import base_layer
 from tensorflow.python.keras.engine import base_layer_utils
-from tensorflow.python.keras.engine import saving
 from tensorflow.python.keras.engine import training_utils
+from tensorflow.python.keras.saving import hdf5_format
 from tensorflow.python.keras.utils import generic_utils
 from tensorflow.python.keras.utils import layer_utils
 from tensorflow.python.keras.utils import tf_utils
@@ -1366,7 +1366,7 @@ class Network(base_layer.Layer):
         return
     if save_format == 'h5':
       with h5py.File(filepath, 'w') as f:
-        saving.save_weights_to_hdf5_group(f, self.layers)
+        hdf5_format.save_weights_to_hdf5_group(f, self.layers)
     else:
       if context.executing_eagerly():
         session = None
@@ -1466,9 +1466,9 @@ class Network(base_layer.Layer):
       if 'layer_names' not in f.attrs and 'model_weights' in f:
         f = f['model_weights']
       if by_name:
-        saving.load_weights_from_hdf5_group_by_name(f, self.layers)
+        hdf5_format.load_weights_from_hdf5_group_by_name(f, self.layers)
       else:
-        saving.load_weights_from_hdf5_group(f, self.layers)
+        hdf5_format.load_weights_from_hdf5_group(f, self.layers)
 
   def _updated_config(self):
     """Util shared between different serialization methods.
diff --git a/tensorflow/python/keras/engine/saving.py b/tensorflow/python/keras/engine/saving.py
index 0604a389a9..b4da86d984 100644
--- a/tensorflow/python/keras/engine/saving.py
+++ b/tensorflow/python/keras/engine/saving.py
@@ -14,950 +14,11 @@
 # ==============================================================================
 # pylint: disable=protected-access
 """Model saving utilities.
+
+Everything has been moved to keras/saving/. This file will be deleted soon.
 """
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import json
-import os
-
-import numpy as np
-from six.moves import zip  # pylint: disable=redefined-builtin
-
-from tensorflow.python.framework import ops
-from tensorflow.python.keras import backend as K
-from tensorflow.python.keras import optimizers
-from tensorflow.python.keras.utils import conv_utils
-from tensorflow.python.keras.utils.io_utils import ask_to_proceed_with_overwrite
-from tensorflow.python.platform import tf_logging as logging
-from tensorflow.python.util import serialization
-from tensorflow.python.util.tf_export import keras_export
-
-# pylint: disable=g-import-not-at-top
-try:
-  import h5py
-  HDF5_OBJECT_HEADER_LIMIT = 64512
-except ImportError:
-  h5py = None
-
-try:
-  import yaml
-except ImportError:
-  yaml = None
-# pylint: enable=g-import-not-at-top
-
-
-@keras_export('keras.models.save_model')
-def save_model(model, filepath, overwrite=True, include_optimizer=True):
-  """Saves a model to a HDF5 file.
-
-  The saved model contains:
-      - the model's configuration (topology)
-      - the model's weights
-      - the model's optimizer's state (if any)
-
-  Thus the saved model can be reinstantiated in
-  the exact same state, without any of the code
-  used for model definition or training.
-
-  Arguments:
-      model: Keras model instance to be saved.
-      filepath: One of the following:
-          - String, path where to save the model
-          - `h5py.File` object where to save the model
-      overwrite: Whether we should overwrite any existing
-          model at the target location, or instead
-          ask the user with a manual prompt.
-      include_optimizer: If True, save optimizer's state together.
-
-  Raises:
-      ImportError: if h5py is not available.
-  """
-
-  if h5py is None:
-    raise ImportError('`save_model` requires h5py.')
-
-  from tensorflow.python.keras import __version__ as keras_version  # pylint: disable=g-import-not-at-top
-
-  # TODO(psv) Add warning when we save models that contain non-serializable
-  # entities like metrics added using `add_metric` and losses added using
-  # `add_loss.`
-
-  if not isinstance(filepath, h5py.File):
-    # If file exists and should not be overwritten.
-    if not overwrite and os.path.isfile(filepath):
-      proceed = ask_to_proceed_with_overwrite(filepath)
-      if not proceed:
-        return
-
-    f = h5py.File(filepath, mode='w')
-    opened_new_file = True
-  else:
-    f = filepath
-    opened_new_file = False
-
-  try:
-    f.attrs['keras_version'] = str(keras_version).encode('utf8')
-    f.attrs['backend'] = K.backend().encode('utf8')
-    f.attrs['model_config'] = json.dumps(
-        {
-            'class_name': model.__class__.__name__,
-            'config': model.get_config()
-        },
-        default=serialization.get_json_type).encode('utf8')
-
-    model_weights_group = f.create_group('model_weights')
-    model_layers = model.layers
-    save_weights_to_hdf5_group(model_weights_group, model_layers)
-
-    if include_optimizer and model.optimizer:
-      if isinstance(model.optimizer, optimizers.TFOptimizer):
-        logging.warning(
-            'TensorFlow optimizers do not '
-            'make it possible to access '
-            'optimizer attributes or optimizer state '
-            'after instantiation. '
-            'As a result, we cannot save the optimizer '
-            'as part of the model save file.'
-            'You will have to compile your model again after loading it. '
-            'Prefer using a Keras optimizer instead '
-            '(see keras.io/optimizers).')
-      else:
-        f.attrs['training_config'] = json.dumps(
-            {
-                'optimizer_config': {
-                    'class_name': model.optimizer.__class__.__name__,
-                    'config': model.optimizer.get_config()
-                },
-                'loss': model.loss,
-                'metrics': model._compile_metrics,
-                'weighted_metrics': model._compile_weighted_metrics,
-                'sample_weight_mode': model.sample_weight_mode,
-                'loss_weights': model.loss_weights,
-            },
-            default=serialization.get_json_type).encode('utf8')
-
-        # Save optimizer weights.
-        symbolic_weights = getattr(model.optimizer, 'weights')
-        if symbolic_weights:
-          optimizer_weights_group = f.create_group('optimizer_weights')
-          weight_values = K.batch_get_value(symbolic_weights)
-          weight_names = []
-          for w, val in zip(symbolic_weights, weight_values):
-            name = str(w.name)
-            weight_names.append(name.encode('utf8'))
-          optimizer_weights_group.attrs['weight_names'] = weight_names
-          for name, val in zip(weight_names, weight_values):
-            param_dset = optimizer_weights_group.create_dataset(
-                name, val.shape, dtype=val.dtype)
-            if not val.shape:
-              # scalar
-              param_dset[()] = val
-            else:
-              param_dset[:] = val
-    f.flush()
-  finally:
-    if opened_new_file:
-      f.close()
-
-
-@keras_export('keras.models.load_model')
-def load_model(filepath, custom_objects=None, compile=True):  # pylint: disable=redefined-builtin
-  """Loads a model saved via `save_model`.
-
-  Arguments:
-      filepath: One of the following:
-          - String, path to the saved model
-          - `h5py.File` object from which to load the model
-      custom_objects: Optional dictionary mapping names
-          (strings) to custom classes or functions to be
-          considered during deserialization.
-      compile: Boolean, whether to compile the model
-          after loading.
-
-  Returns:
-      A Keras model instance. If an optimizer was found
-      as part of the saved model, the model is already
-      compiled. Otherwise, the model is uncompiled and
-      a warning will be displayed. When `compile` is set
-      to False, the compilation is omitted without any
-      warning.
-
-  Raises:
-      ImportError: if h5py is not available.
-      ValueError: In case of an invalid savefile.
-  """
-  if h5py is None:
-    raise ImportError('`load_model` requires h5py.')
-
-  if not custom_objects:
-    custom_objects = {}
-
-  def convert_custom_objects(obj):
-    """Handles custom object lookup.
-
-    Arguments:
-        obj: object, dict, or list.
-
-    Returns:
-        The same structure, where occurrences
-            of a custom object name have been replaced
-            with the custom object.
-    """
-    if isinstance(obj, list):
-      deserialized = []
-      for value in obj:
-        deserialized.append(convert_custom_objects(value))
-      return deserialized
-    if isinstance(obj, dict):
-      deserialized = {}
-      for key, value in obj.items():
-        deserialized[key] = convert_custom_objects(value)
-      return deserialized
-    if obj in custom_objects:
-      return custom_objects[obj]
-    return obj
-
-  opened_new_file = not isinstance(filepath, h5py.File)
-  if opened_new_file:
-    f = h5py.File(filepath, mode='r')
-  else:
-    f = filepath
-
-  model = None
-  try:
-    # instantiate model
-    model_config = f.attrs.get('model_config')
-    if model_config is None:
-      raise ValueError('No model found in config file.')
-    model_config = json.loads(model_config.decode('utf-8'))
-    model = model_from_config(model_config, custom_objects=custom_objects)
-
-    # set weights
-    load_weights_from_hdf5_group(f['model_weights'], model.layers)
-
-    if compile:
-      # instantiate optimizer
-      training_config = f.attrs.get('training_config')
-      if training_config is None:
-        logging.warning('No training configuration found in save file: '
-                        'the model was *not* compiled. Compile it manually.')
-        return model
-      training_config = json.loads(training_config.decode('utf-8'))
-      optimizer_config = training_config['optimizer_config']
-      optimizer = optimizers.deserialize(
-          optimizer_config, custom_objects=custom_objects)
-
-      # Recover loss functions and metrics.
-      loss = convert_custom_objects(training_config['loss'])
-      metrics = convert_custom_objects(training_config['metrics'])
-      weighted_metrics = convert_custom_objects(
-          training_config.get('weighted_metrics', None))
-      sample_weight_mode = training_config['sample_weight_mode']
-      loss_weights = training_config['loss_weights']
-
-      # Compile model.
-      model.compile(
-          optimizer=optimizer,
-          loss=loss,
-          metrics=metrics,
-          weighted_metrics=weighted_metrics,
-          loss_weights=loss_weights,
-          sample_weight_mode=sample_weight_mode)
-
-      # Set optimizer weights.
-      if 'optimizer_weights' in f:
-        # Build train function (to get weight updates).
-        # Models that aren't graph networks must wait until they are called
-        # with data to _make_train_function() and so can't load optimizer
-        # weights.
-        if model._is_graph_network:  # pylint: disable=protected-access
-          model._make_train_function()
-          optimizer_weights_group = f['optimizer_weights']
-          optimizer_weight_names = [
-              n.decode('utf8')
-              for n in optimizer_weights_group.attrs['weight_names']
-          ]
-          optimizer_weight_values = [
-              optimizer_weights_group[n] for n in optimizer_weight_names
-          ]
-          try:
-            model.optimizer.set_weights(optimizer_weight_values)
-          except ValueError:
-            logging.warning('Error in loading the saved optimizer '
-                            'state. As a result, your model is '
-                            'starting with a freshly initialized '
-                            'optimizer.')
-        else:
-          logging.warning('Sequential models without an `input_shape` '
-                          'passed to the first layer cannot reload their '
-                          'optimizer state. As a result, your model is'
-                          'starting with a freshly initialized optimizer.')
-
-  finally:
-    if opened_new_file:
-      f.close()
-  return model
-
-
-@keras_export('keras.models.model_from_config')
-def model_from_config(config, custom_objects=None):
-  """Instantiates a Keras model from its config.
-
-  Arguments:
-      config: Configuration dictionary.
-      custom_objects: Optional dictionary mapping names
-          (strings) to custom classes or functions to be
-          considered during deserialization.
-
-  Returns:
-      A Keras model instance (uncompiled).
-
-  Raises:
-      TypeError: if `config` is not a dictionary.
-  """
-  if isinstance(config, list):
-    raise TypeError('`model_from_config` expects a dictionary, not a list. '
-                    'Maybe you meant to use '
-                    '`Sequential.from_config(config)`?')
-  from tensorflow.python.keras.layers import deserialize  # pylint: disable=g-import-not-at-top
-  return deserialize(config, custom_objects=custom_objects)
-
-
-@keras_export('keras.models.model_from_yaml')
-def model_from_yaml(yaml_string, custom_objects=None):
-  """Parses a yaml model configuration file and returns a model instance.
-
-  Arguments:
-      yaml_string: YAML string encoding a model configuration.
-      custom_objects: Optional dictionary mapping names
-          (strings) to custom classes or functions to be
-          considered during deserialization.
-
-  Returns:
-      A Keras model instance (uncompiled).
-
-  Raises:
-      ImportError: if yaml module is not found.
-  """
-  if yaml is None:
-    raise ImportError('Requires yaml module installed (`pip install pyyaml`).')
-  config = yaml.load(yaml_string)
-  from tensorflow.python.keras.layers import deserialize  # pylint: disable=g-import-not-at-top
-  return deserialize(config, custom_objects=custom_objects)
-
-
-@keras_export('keras.models.model_from_json')
-def model_from_json(json_string, custom_objects=None):
-  """Parses a JSON model configuration file and returns a model instance.
-
-  Arguments:
-      json_string: JSON string encoding a model configuration.
-      custom_objects: Optional dictionary mapping names
-          (strings) to custom classes or functions to be
-          considered during deserialization.
-
-  Returns:
-      A Keras model instance (uncompiled).
-  """
-  config = json.loads(json_string)
-  from tensorflow.python.keras.layers import deserialize  # pylint: disable=g-import-not-at-top
-  return deserialize(config, custom_objects=custom_objects)
-
-
-def preprocess_weights_for_loading(layer,
-                                   weights,
-                                   original_keras_version=None,
-                                   original_backend=None):
-  """Preprocess layer weights between different Keras formats.
-
-  Converts layers weights from Keras 1 format to Keras 2 and also weights of
-  CuDNN layers in Keras 2.
-
-  Arguments:
-      layer: Layer instance.
-      weights: List of weights values (Numpy arrays).
-      original_keras_version: Keras version for the weights, as a string.
-      original_backend: Keras backend the weights were trained with,
-          as a string.
-
-  Returns:
-      A list of weights values (Numpy arrays).
-  """
-  def convert_nested_bidirectional(weights):
-    """Converts layers nested in `Bidirectional` wrapper.
-
-    This function uses `preprocess_weights_for_loading()` for converting
-    layers.
-
-    Arguments:
-        weights: List of weights values (Numpy arrays).
-
-    Returns:
-        A list of weights values (Numpy arrays).
-    """
-    num_weights_per_layer = len(weights) // 2
-    forward_weights = preprocess_weights_for_loading(
-        layer.forward_layer, weights[:num_weights_per_layer],
-        original_keras_version, original_backend)
-    backward_weights = preprocess_weights_for_loading(
-        layer.backward_layer, weights[num_weights_per_layer:],
-        original_keras_version, original_backend)
-    return forward_weights + backward_weights
-
-  def convert_nested_time_distributed(weights):
-    """Converts layers nested in `TimeDistributed` wrapper.
-
-    This function uses `preprocess_weights_for_loading()` for converting nested
-    layers.
-
-    Arguments:
-        weights: List of weights values (Numpy arrays).
-
-    Returns:
-        A list of weights values (Numpy arrays).
-    """
-    return preprocess_weights_for_loading(
-        layer.layer, weights, original_keras_version, original_backend)
-
-  def convert_nested_model(weights):
-    """Converts layers nested in `Model` or `Sequential`.
-
-    This function uses `preprocess_weights_for_loading()` for converting nested
-    layers.
-
-    Arguments:
-        weights: List of weights values (Numpy arrays).
-
-    Returns:
-        A list of weights values (Numpy arrays).
-    """
-    new_weights = []
-    # trainable weights
-    for sublayer in layer.layers:
-      num_weights = len(sublayer.trainable_weights)
-      if num_weights > 0:
-        new_weights.extend(preprocess_weights_for_loading(
-            layer=sublayer,
-            weights=weights[:num_weights],
-            original_keras_version=original_keras_version,
-            original_backend=original_backend))
-        weights = weights[num_weights:]
-
-    # non-trainable weights
-    for sublayer in layer.layers:
-      num_weights = len([l for l in sublayer.weights
-                         if l not in sublayer.trainable_weights])
-      if num_weights > 0:
-        new_weights.extend(preprocess_weights_for_loading(
-            layer=sublayer,
-            weights=weights[:num_weights],
-            original_keras_version=original_keras_version,
-            original_backend=original_backend))
-        weights = weights[num_weights:]
-    return new_weights
-
-  # Convert layers nested in Bidirectional/Model/Sequential.
-  # Both transformation should be ran for both Keras 1->2 conversion
-  # and for conversion of CuDNN layers.
-  if layer.__class__.__name__ == 'Bidirectional':
-    weights = convert_nested_bidirectional(weights)
-  if layer.__class__.__name__ == 'TimeDistributed':
-    weights = convert_nested_time_distributed(weights)
-  elif layer.__class__.__name__ in ['Model', 'Sequential']:
-    weights = convert_nested_model(weights)
-
-  if original_keras_version == '1':
-    if layer.__class__.__name__ == 'TimeDistributed':
-      weights = preprocess_weights_for_loading(
-          layer.layer, weights, original_keras_version, original_backend)
-
-    if layer.__class__.__name__ == 'Conv1D':
-      shape = weights[0].shape
-      # Handle Keras 1.1 format
-      if shape[:2] != (layer.kernel_size[0], 1) or shape[3] != layer.filters:
-        # Legacy shape:
-        # (filters, input_dim, filter_length, 1)
-        assert shape[0] == layer.filters and shape[2:] == (layer.kernel_size[0],
-                                                           1)
-        weights[0] = np.transpose(weights[0], (2, 3, 1, 0))
-      weights[0] = weights[0][:, 0, :, :]
-
-    if layer.__class__.__name__ == 'Conv2D':
-      if layer.data_format == 'channels_first':
-        # old: (filters, stack_size, kernel_rows, kernel_cols)
-        # new: (kernel_rows, kernel_cols, stack_size, filters)
-        weights[0] = np.transpose(weights[0], (2, 3, 1, 0))
-
-    if layer.__class__.__name__ == 'Conv2DTranspose':
-      if layer.data_format == 'channels_last':
-        # old: (kernel_rows, kernel_cols, stack_size, filters)
-        # new: (kernel_rows, kernel_cols, filters, stack_size)
-        weights[0] = np.transpose(weights[0], (0, 1, 3, 2))
-      if layer.data_format == 'channels_first':
-        # old: (filters, stack_size, kernel_rows, kernel_cols)
-        # new: (kernel_rows, kernel_cols, filters, stack_size)
-        weights[0] = np.transpose(weights[0], (2, 3, 0, 1))
-
-    if layer.__class__.__name__ == 'Conv3D':
-      if layer.data_format == 'channels_first':
-        # old: (filters, stack_size, ...)
-        # new: (..., stack_size, filters)
-        weights[0] = np.transpose(weights[0], (2, 3, 4, 1, 0))
-
-    if layer.__class__.__name__ == 'GRU':
-      if len(weights) == 9:
-        kernel = np.concatenate([weights[0], weights[3], weights[6]], axis=-1)
-        recurrent_kernel = np.concatenate(
-            [weights[1], weights[4], weights[7]], axis=-1)
-        bias = np.concatenate([weights[2], weights[5], weights[8]], axis=-1)
-        weights = [kernel, recurrent_kernel, bias]
-
-    if layer.__class__.__name__ == 'LSTM':
-      if len(weights) == 12:
-        # old: i, c, f, o
-        # new: i, f, c, o
-        kernel = np.concatenate(
-            [weights[0], weights[6], weights[3], weights[9]], axis=-1)
-        recurrent_kernel = np.concatenate(
-            [weights[1], weights[7], weights[4], weights[10]], axis=-1)
-        bias = np.concatenate(
-            [weights[2], weights[8], weights[5], weights[11]], axis=-1)
-        weights = [kernel, recurrent_kernel, bias]
-
-    if layer.__class__.__name__ == 'ConvLSTM2D':
-      if len(weights) == 12:
-        kernel = np.concatenate(
-            [weights[0], weights[6], weights[3], weights[9]], axis=-1)
-        recurrent_kernel = np.concatenate(
-            [weights[1], weights[7], weights[4], weights[10]], axis=-1)
-        bias = np.concatenate(
-            [weights[2], weights[8], weights[5], weights[11]], axis=-1)
-        if layer.data_format == 'channels_first':
-          # old: (filters, stack_size, kernel_rows, kernel_cols)
-          # new: (kernel_rows, kernel_cols, stack_size, filters)
-          kernel = np.transpose(kernel, (2, 3, 1, 0))
-          recurrent_kernel = np.transpose(recurrent_kernel, (2, 3, 1, 0))
-        weights = [kernel, recurrent_kernel, bias]
-
-  conv_layers = ['Conv1D', 'Conv2D', 'Conv3D', 'Conv2DTranspose', 'ConvLSTM2D']
-  if layer.__class__.__name__ in conv_layers:
-    if original_backend == 'theano':
-      weights[0] = conv_utils.convert_kernel(weights[0])
-      if layer.__class__.__name__ == 'ConvLSTM2D':
-        weights[1] = conv_utils.convert_kernel(weights[1])
-    if K.int_shape(layer.weights[0]) != weights[0].shape:
-      weights[0] = np.transpose(weights[0], (3, 2, 0, 1))
-      if layer.__class__.__name__ == 'ConvLSTM2D':
-        weights[1] = np.transpose(weights[1], (3, 2, 0, 1))
-
-  # convert CuDNN layers
-  return _convert_rnn_weights(layer, weights)
-
-
-def _convert_rnn_weights(layer, weights):
-  """Converts weights for RNN layers between native and CuDNN format.
-
-  Input kernels for each gate are transposed and converted between Fortran
-  and C layout, recurrent kernels are transposed. For LSTM biases are summed/
-  split in half, for GRU biases are reshaped.
-
-  Weights can be converted in both directions between `LSTM` and`CuDNNSLTM`
-  and between `CuDNNGRU` and `GRU(reset_after=True)`. Default `GRU` is not
-  compatible with `CuDNNGRU`.
-
-  For missing biases in `LSTM`/`GRU` (`use_bias=False`) no conversion is made.
-
-  Arguments:
-      layer: Target layer instance.
-      weights: List of source weights values (input kernels, recurrent
-          kernels, [biases]) (Numpy arrays).
-
-  Returns:
-      A list of converted weights values (Numpy arrays).
-
-  Raises:
-      ValueError: for incompatible GRU layer/weights or incompatible biases
-  """
-
-  def transform_kernels(kernels, func, n_gates):
-    """Transforms kernel for each gate separately using given function.
-
-    Arguments:
-        kernels: Stacked array of kernels for individual gates.
-        func: Function applied to kernel of each gate.
-        n_gates: Number of gates (4 for LSTM, 3 for GRU).
-
-    Returns:
-        Stacked array of transformed kernels.
-    """
-    return np.hstack([func(k) for k in np.hsplit(kernels, n_gates)])
-
-  def transpose_input(from_cudnn):
-    """Makes a function that transforms input kernels from/to CuDNN format.
-
-    It keeps the shape, but changes between the layout (Fortran/C). Eg.:
-
-    ```
-    Keras                 CuDNN
-    [[0, 1, 2],  <--->  [[0, 2, 4],
-     [3, 4, 5]]          [1, 3, 5]]
-    ```
-
-    It can be passed to `transform_kernels()`.
-
-    Arguments:
-        from_cudnn: `True` if source weights are in CuDNN format, `False`
-            if they're in plain Keras format.
-
-    Returns:
-        Function that converts input kernel to the other format.
-    """
-    order = 'F' if from_cudnn else 'C'
-
-    def transform(kernel):
-      return kernel.T.reshape(kernel.shape, order=order)
-
-    return transform
-
-  target_class = layer.__class__.__name__
-
-  # convert the weights between CuDNNLSTM and LSTM
-  if target_class in ['LSTM', 'CuDNNLSTM'] and len(weights) == 3:
-    # determine if we're loading a CuDNNLSTM layer
-    # from the number of bias weights:
-    # CuDNNLSTM has (units * 8) weights; while LSTM has (units * 4)
-    # if there's no bias weight in the file, skip this conversion
-    units = weights[1].shape[0]
-    bias_shape = weights[2].shape
-    n_gates = 4
-
-    if bias_shape == (2 * units * n_gates,):
-      source = 'CuDNNLSTM'
-    elif bias_shape == (units * n_gates,):
-      source = 'LSTM'
-    else:
-      raise ValueError('Invalid bias shape: ' + str(bias_shape))
-
-    def convert_lstm_weights(weights, from_cudnn=True):
-      """Converts the weights between CuDNNLSTM and LSTM.
-
-      Arguments:
-        weights: Original weights.
-        from_cudnn: Indicates whether original weights are from CuDNN layer.
-
-      Returns:
-        Updated weights compatible with LSTM.
-      """
-
-      # Transpose (and reshape) input and recurrent kernels
-      kernels = transform_kernels(weights[0], transpose_input(from_cudnn),
-                                  n_gates)
-      recurrent_kernels = transform_kernels(weights[1], lambda k: k.T, n_gates)
-      if from_cudnn:
-        # merge input and recurrent biases into a single set
-        biases = np.sum(np.split(weights[2], 2, axis=0), axis=0)
-      else:
-        # Split single set of biases evenly to two sets. The way of
-        # splitting doesn't matter as long as the two sets sum is kept.
-        biases = np.tile(0.5 * weights[2], 2)
-      return [kernels, recurrent_kernels, biases]
-
-    if source != target_class:
-      weights = convert_lstm_weights(weights, from_cudnn=source == 'CuDNNLSTM')
-
-  # convert the weights between CuDNNGRU and GRU(reset_after=True)
-  if target_class in ['GRU', 'CuDNNGRU'] and len(weights) == 3:
-    # We can determine the source of the weights from the shape of the bias.
-    # If there is no bias we skip the conversion since
-    # CuDNNGRU always has biases.
-
-    units = weights[1].shape[0]
-    bias_shape = weights[2].shape
-    n_gates = 3
-
-    def convert_gru_weights(weights, from_cudnn=True):
-      """Converts the weights between CuDNNGRU and GRU.
-
-      Arguments:
-        weights: Original weights.
-        from_cudnn: Indicates whether original weights are from CuDNN layer.
-
-      Returns:
-        Updated weights compatible with GRU.
-      """
-
-      kernels = transform_kernels(weights[0], transpose_input(from_cudnn),
-                                  n_gates)
-      recurrent_kernels = transform_kernels(weights[1], lambda k: k.T, n_gates)
-      biases = np.array(weights[2]).reshape((2, -1) if from_cudnn else -1)
-      return [kernels, recurrent_kernels, biases]
-
-    if bias_shape == (2 * units * n_gates,):
-      source = 'CuDNNGRU'
-    elif bias_shape == (2, units * n_gates):
-      source = 'GRU(reset_after=True)'
-    elif bias_shape == (units * n_gates,):
-      source = 'GRU(reset_after=False)'
-    else:
-      raise ValueError('Invalid bias shape: ' + str(bias_shape))
-
-    if target_class == 'CuDNNGRU':
-      target = 'CuDNNGRU'
-    elif layer.reset_after:
-      target = 'GRU(reset_after=True)'
-    else:
-      target = 'GRU(reset_after=False)'
-
-    # only convert between different types
-    if source != target:
-      types = (source, target)
-      if 'GRU(reset_after=False)' in types:
-        raise ValueError('%s is not compatible with %s' % types)
-      if source == 'CuDNNGRU':
-        weights = convert_gru_weights(weights, from_cudnn=True)
-      elif source == 'GRU(reset_after=True)':
-        weights = convert_gru_weights(weights, from_cudnn=False)
-
-  return weights
-
-
-def save_weights_to_hdf5_group(f, layers):
-  """Saves the weights of a list of layers to a HDF5 group.
-
-  Arguments:
-      f: HDF5 group.
-      layers: List of layer instances.
-  """
-  from tensorflow.python.keras import __version__ as keras_version  # pylint: disable=g-import-not-at-top
-
-  save_attributes_to_hdf5_group(
-      f, 'layer_names', [layer.name.encode('utf8') for layer in layers])
-  f.attrs['backend'] = K.backend().encode('utf8')
-  f.attrs['keras_version'] = str(keras_version).encode('utf8')
-
-  # On TPUs, modifying the graph between session.runs() triggers some expensive
-  # recompilation overhead. To avoid this, we build up the full set of tensors
-  # to save before fetching weights, thus only modifying the graph once.
-  layer_weights_dict = {}
-  for layer in layers:
-    layer_weights_dict[layer.name] = [ops.convert_to_tensor(w)
-                                      for w in layer.weights]
-
-  for layer in layers:
-    g = f.create_group(layer.name)
-    symbolic_weights = layer_weights_dict[layer.name]
-    weight_values = K.batch_get_value(symbolic_weights)
-    weight_names = []
-    for i, (w, val) in enumerate(zip(symbolic_weights, weight_values)):
-      if hasattr(w, 'name') and w.name:
-        name = str(w.name)
-      else:
-        name = 'param_' + str(i)
-      weight_names.append(name.encode('utf8'))
-    save_attributes_to_hdf5_group(g, 'weight_names', weight_names)
-    for name, val in zip(weight_names, weight_values):
-      param_dset = g.create_dataset(name, val.shape, dtype=val.dtype)
-      if not val.shape:
-        # scalar
-        param_dset[()] = val
-      else:
-        param_dset[:] = val
-
-
-def load_weights_from_hdf5_group(f, layers):
-  """Implements topological (order-based) weight loading.
-
-  Arguments:
-      f: A pointer to a HDF5 group.
-      layers: a list of target layers.
-
-  Raises:
-      ValueError: in case of mismatch between provided layers
-          and weights file.
-  """
-  if 'keras_version' in f.attrs:
-    original_keras_version = f.attrs['keras_version'].decode('utf8')
-  else:
-    original_keras_version = '1'
-  if 'backend' in f.attrs:
-    original_backend = f.attrs['backend'].decode('utf8')
-  else:
-    original_backend = None
-
-  filtered_layers = []
-  for layer in layers:
-    weights = layer.weights
-    if weights:
-      filtered_layers.append(layer)
-
-  layer_names = load_attributes_from_hdf5_group(f, 'layer_names')
-  filtered_layer_names = []
-  for name in layer_names:
-    g = f[name]
-    weight_names = load_attributes_from_hdf5_group(g, 'weight_names')
-    if weight_names:
-      filtered_layer_names.append(name)
-  layer_names = filtered_layer_names
-  if len(layer_names) != len(filtered_layers):
-    raise ValueError('You are trying to load a weight file '
-                     'containing ' + str(len(layer_names)) +
-                     ' layers into a model with ' + str(len(filtered_layers)) +
-                     ' layers.')
-
-  # We batch weight value assignments in a single backend call
-  # which provides a speedup in TensorFlow.
-  weight_value_tuples = []
-  for k, name in enumerate(layer_names):
-    g = f[name]
-    weight_names = load_attributes_from_hdf5_group(g, 'weight_names')
-    weight_values = [np.asarray(g[weight_name]) for weight_name in weight_names]
-    layer = filtered_layers[k]
-    symbolic_weights = layer.weights
-    weight_values = preprocess_weights_for_loading(
-        layer, weight_values, original_keras_version, original_backend)
-    if len(weight_values) != len(symbolic_weights):
-      raise ValueError('Layer #' + str(k) + ' (named "' + layer.name +
-                       '" in the current model) was found to '
-                       'correspond to layer ' + name + ' in the save file. '
-                       'However the new layer ' + layer.name + ' expects ' +
-                       str(len(symbolic_weights)) +
-                       ' weights, but the saved weights have ' +
-                       str(len(weight_values)) + ' elements.')
-    weight_value_tuples += zip(symbolic_weights, weight_values)
-  K.batch_set_value(weight_value_tuples)
-
-
-def load_weights_from_hdf5_group_by_name(f, layers):
-  """Implements name-based weight loading.
-
-  (instead of topological weight loading).
-
-  Layers that have no matching name are skipped.
-
-  Arguments:
-      f: A pointer to a HDF5 group.
-      layers: a list of target layers.
-
-  Raises:
-      ValueError: in case of mismatch between provided layers
-          and weights file.
-  """
-  if 'keras_version' in f.attrs:
-    original_keras_version = f.attrs['keras_version'].decode('utf8')
-  else:
-    original_keras_version = '1'
-  if 'backend' in f.attrs:
-    original_backend = f.attrs['backend'].decode('utf8')
-  else:
-    original_backend = None
-
-  # New file format.
-  layer_names = load_attributes_from_hdf5_group(f, 'layer_names')
-
-  # Reverse index of layer name to list of layers with name.
-  index = {}
-  for layer in layers:
-    if layer.name:
-      index.setdefault(layer.name, []).append(layer)
-
-  # We batch weight value assignments in a single backend call
-  # which provides a speedup in TensorFlow.
-  weight_value_tuples = []
-  for k, name in enumerate(layer_names):
-    g = f[name]
-    weight_names = load_attributes_from_hdf5_group(g, 'weight_names')
-    weight_values = [np.asarray(g[weight_name]) for weight_name in weight_names]
-
-    for layer in index.get(name, []):
-      symbolic_weights = layer.weights
-      weight_values = preprocess_weights_for_loading(
-          layer, weight_values, original_keras_version, original_backend)
-      if len(weight_values) != len(symbolic_weights):
-        raise ValueError('Layer #' + str(k) + ' (named "' + layer.name +
-                         '") expects ' + str(len(symbolic_weights)) +
-                         ' weight(s), but the saved weights' + ' have ' +
-                         str(len(weight_values)) + ' element(s).')
-      # Set values.
-      for i in range(len(weight_values)):
-        if K.int_shape(symbolic_weights[i]) != weight_values[i].shape:
-          raise ValueError('Layer #' + str(k) +' (named "' + layer.name +
-                           '"), weight ' + str(symbolic_weights[i]) +
-                           ' has shape {}'.format(K.int_shape(
-                               symbolic_weights[i])) +
-                           ', but the saved weight has shape ' +
-                           str(weight_values[i].shape) + '.')
-
-        else:
-          weight_value_tuples.append((symbolic_weights[i], weight_values[i]))
-  K.batch_set_value(weight_value_tuples)
-
-
-def save_attributes_to_hdf5_group(group, name, data):
-  """Saves attributes (data) of the specified name into the HDF5 group.
-
-  This method deals with an inherent problem of HDF5 file which is not
-  able to store data larger than HDF5_OBJECT_HEADER_LIMIT bytes.
-
-  Arguments:
-      group: A pointer to a HDF5 group.
-      name: A name of the attributes to save.
-      data: Attributes data to store.
-
-  Raises:
-    RuntimeError: If any single attribute is too large to be saved.
-  """
-  # Check that no item in `data` is larger than `HDF5_OBJECT_HEADER_LIMIT`
-  # because in that case even chunking the array would not make the saving
-  # possible.
-  bad_attributes = [x for x in data if len(x) > HDF5_OBJECT_HEADER_LIMIT]
-
-  # Expecting this to never be true.
-  if bad_attributes:
-    raise RuntimeError('The following attributes cannot be saved to HDF5 '
-                       'file because they are larger than %d bytes: %s' %
-                       (HDF5_OBJECT_HEADER_LIMIT,
-                        ', '.join([x for x in bad_attributes])))
-
-  data_npy = np.asarray(data)
-
-  num_chunks = 1
-  chunked_data = np.array_split(data_npy, num_chunks)
-
-  # This will never loop forever thanks to the test above.
-  while any(x.nbytes > HDF5_OBJECT_HEADER_LIMIT for x in chunked_data):
-    num_chunks += 1
-    chunked_data = np.array_split(data_npy, num_chunks)
-
-  if num_chunks > 1:
-    for chunk_id, chunk_data in enumerate(chunked_data):
-      group.attrs['%s%d' % (name, chunk_id)] = chunk_data
-  else:
-    group.attrs[name] = data
-
-
-def load_attributes_from_hdf5_group(group, name):
-  """Loads attributes of the specified name from the HDF5 group.
-
-  This method deals with an inherent problem
-  of HDF5 file which is not able to store
-  data larger than HDF5_OBJECT_HEADER_LIMIT bytes.
-
-  Arguments:
-      group: A pointer to a HDF5 group.
-      name: A name of the attributes to load.
-
-  Returns:
-      data: Attributes data.
-  """
-  if name in group.attrs:
-    data = [n.decode('utf8') for n in group.attrs[name]]
-  else:
-    data = []
-    chunk_id = 0
-    while '%s%d' % (name, chunk_id) in group.attrs:
-      data.extend(
-          [n.decode('utf8') for n in group.attrs['%s%d' % (name, chunk_id)]])
-      chunk_id += 1
-  return data
+from tensorflow.python.keras.saving import *  # pylint: disable=wildcard-import
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 0b1743af38..5614c93842 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -40,6 +40,7 @@ from tensorflow.python.keras.engine import training_generator
 from tensorflow.python.keras.engine import training_utils
 from tensorflow.python.keras.engine.network import Network
 from tensorflow.python.keras.optimizer_v2 import optimizer_v2
+from tensorflow.python.keras.saving import saving_utils
 from tensorflow.python.keras.utils import data_utils
 from tensorflow.python.keras.utils.generic_utils import slice_arrays
 from tensorflow.python.keras.utils.losses_utils import squeeze_or_expand_dimensions
@@ -1702,7 +1703,7 @@ class Model(Network):
 
   @property
   def _default_save_signature(self):
-    return training_utils.trace_model_call(self)
+    return saving_utils.trace_model_call(self)
 
   def _set_sample_weight_attributes(self, sample_weight_mode,
                                     skip_target_weighing_indices):
diff --git a/tensorflow/python/keras/engine/training_utils.py b/tensorflow/python/keras/engine/training_utils.py
index 949f1e400d..a4ee6c1ba5 100644
--- a/tensorflow/python/keras/engine/training_utils.py
+++ b/tensorflow/python/keras/engine/training_utils.py
@@ -29,12 +29,10 @@ from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import iterator_ops
 from tensorflow.python.data.ops import readers
 from tensorflow.python.eager import context
-from tensorflow.python.eager import def_function
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
-from tensorflow.python.framework import tensor_spec
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.keras import backend as K
 from tensorflow.python.keras import callbacks as cbks
@@ -1387,60 +1385,6 @@ def generic_output_names(outputs_list):
   return ['output_%d' % (i + 1) for i in range(len(outputs_list))]
 
 
-def trace_model_call(model, input_signature=None):
-  """Trace the model call to create a tf.function for exporting a Keras model.
-
-  Args:
-    model: A Keras model.
-    input_signature: optional, a list of tf.TensorSpec objects specifying the
-      inputs to the model.
-
-  Returns:
-    A tf.function wrapping the model's call function with input signatures set.
-
-  Raises:
-    ValueError: if input signature cannot be inferred from the model.
-  """
-  if input_signature is None:
-    if isinstance(model.call, def_function.PolymorphicFunction):
-      input_signature = model.call.input_signature
-
-  if input_signature is None:
-    try:
-      inputs = model.inputs
-      input_names = model.input_names
-    except AttributeError:
-      raise ValueError(
-          'Model {} cannot be saved because the input shapes have not been '
-          'set. Usually, input shapes are automatically determined from calling'
-          ' .fit() or .predict(). To manually set the shapes, call '
-          'model._set_inputs(inputs).'.format(model))
-    input_specs = []
-    for input_tensor, input_name in zip(inputs, input_names):
-      input_specs.append(tensor_spec.TensorSpec(
-          shape=input_tensor.shape, dtype=input_tensor.dtype,
-          name=input_name))
-    # The input signature of the call function is a list with one element, since
-    # all tensor inputs must be passed in as the first argument.
-    input_signature = [input_specs] if len(input_specs) > 1 else input_specs
-
-  # TODO(mdan): Should the model's call be autographed by default?
-  @def_function.function(input_signature=input_signature, autograph=False)
-  def _wrapped_model(*args):
-    """A concrete tf.function that wraps the model's call function."""
-    # When given a single input, Keras models will call the model on the tensor
-    # rather than a list consisting of the single tensor.
-    inputs = args[0] if len(input_signature) == 1 else list(args)
-    outputs_list = nest.flatten(model(inputs=inputs))
-    try:
-      output_names = model.output_names
-    except AttributeError:
-      output_names = generic_output_names(outputs_list)
-    return {name: output for name, output in zip(output_names, outputs_list)}
-
-  return _wrapped_model
-
-
 def set_run_eagerly_for_dict_structure(model, x):
   """Set model.run_eagerly to true if x is dict structure.
 
diff --git a/tensorflow/python/keras/engine/training_utils_test.py b/tensorflow/python/keras/engine/training_utils_test.py
index d0b2567b55..d8d8c58610 100644
--- a/tensorflow/python/keras/engine/training_utils_test.py
+++ b/tensorflow/python/keras/engine/training_utils_test.py
@@ -18,33 +18,18 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import os
-
 from absl.testing import parameterized
 import numpy as np
 
 
-from tensorflow.python import keras
-from tensorflow.python.client import session as session_lib
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import readers
 from tensorflow.python.eager import context
-from tensorflow.python.eager import def_function
-from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import ops
-from tensorflow.python.framework import tensor_spec
 from tensorflow.python.framework import tensor_util
-from tensorflow.python.keras import backend as K
 from tensorflow.python.keras import keras_parameterized
-from tensorflow.python.keras import testing_utils
 from tensorflow.python.keras.engine import training_utils
 from tensorflow.python.keras.utils import tf_utils
-from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
-from tensorflow.python.saved_model import loader
-from tensorflow.python.saved_model import save as save_lib
-from tensorflow.python.saved_model import signature_constants
-from tensorflow.python.saved_model import tag_constants
 
 
 class ModelInputsTest(test.TestCase):
@@ -105,170 +90,6 @@ class ModelInputsTest(test.TestCase):
       self.assertTrue(tf_utils.is_symbolic_tensor(vals['b']))
 
 
-class TraceModelCallTest(keras_parameterized.TestCase):
-
-  def _assert_all_close(self, expected, actual):
-    if not context.executing_eagerly():
-      with self.cached_session() as sess:
-        K._initialize_variables(sess)
-        self.assertAllClose(expected, actual)
-    else:
-      self.assertAllClose(expected, actual)
-
-  @keras_parameterized.run_with_all_model_types
-  @keras_parameterized.run_all_keras_modes
-  def test_trace_model_outputs(self):
-    input_dim = 5 if testing_utils.get_model_type() == 'functional' else None
-    model = testing_utils.get_small_mlp(10, 3, input_dim)
-    inputs = array_ops.ones((8, 5))
-
-    if input_dim is None:
-      with self.assertRaisesRegexp(ValueError,
-                                   'input shapes have not been set'):
-        training_utils.trace_model_call(model)
-      model._set_inputs(inputs)
-
-    fn = training_utils.trace_model_call(model)
-    signature_outputs = fn(inputs)
-    expected_outputs = {model.output_names[0]: model(inputs)}
-
-    self._assert_all_close(expected_outputs, signature_outputs)
-
-  @keras_parameterized.run_with_all_model_types
-  @keras_parameterized.run_all_keras_modes
-  def test_trace_model_outputs_after_fitting(self):
-    input_dim = 5 if testing_utils.get_model_type() == 'functional' else None
-    model = testing_utils.get_small_mlp(10, 3, input_dim)
-    model.compile(optimizer='sgd', loss='mse')
-    model.fit(x=np.random.random((8, 5)),
-              y=np.random.random((8, 3)), epochs=2)
-
-    inputs = array_ops.ones((8, 5))
-
-    fn = training_utils.trace_model_call(model)
-    signature_outputs = fn(inputs)
-    expected_outputs = {model.output_names[0]: model(inputs)}
-
-    self._assert_all_close(expected_outputs, signature_outputs)
-
-  @keras_parameterized.run_with_all_model_types(exclude_models='sequential')
-  @keras_parameterized.run_all_keras_modes
-  def test_trace_multi_io_model_outputs(self):
-    input_dim = 5
-    num_classes = 3
-    num_classes_b = 4
-    input_a = keras.layers.Input(shape=(input_dim,), name='input_a')
-    input_b = keras.layers.Input(shape=(input_dim,), name='input_b')
-
-    dense = keras.layers.Dense(num_classes, name='dense')
-    dense2 = keras.layers.Dense(num_classes_b, name='dense2')
-    dropout = keras.layers.Dropout(0.5, name='dropout')
-    branch_a = [input_a, dense]
-    branch_b = [input_b, dense, dense2, dropout]
-
-    model = testing_utils.get_multi_io_model(branch_a, branch_b)
-
-    input_a_np = np.random.random((10, input_dim)).astype(np.float32)
-    input_b_np = np.random.random((10, input_dim)).astype(np.float32)
-
-    if testing_utils.get_model_type() == 'subclass':
-      with self.assertRaisesRegexp(ValueError,
-                                   'input shapes have not been set'):
-        training_utils.trace_model_call(model)
-
-    model.compile(optimizer='sgd', loss='mse')
-    model.fit(x=[np.random.random((8, input_dim)).astype(np.float32),
-                 np.random.random((8, input_dim)).astype(np.float32)],
-              y=[np.random.random((8, num_classes)).astype(np.float32),
-                 np.random.random((8, num_classes_b)).astype(np.float32)],
-              epochs=2)
-
-    fn = training_utils.trace_model_call(model)
-    signature_outputs = fn([input_a_np, input_b_np])
-    outputs = model([input_a_np, input_b_np])
-    expected_outputs = {model.output_names[0]: outputs[0],
-                        model.output_names[1]: outputs[1]}
-
-    self._assert_all_close(expected_outputs, signature_outputs)
-
-  @keras_parameterized.run_all_keras_modes
-  def test_specify_input_signature(self):
-    model = testing_utils.get_small_sequential_mlp(10, 3, None)
-    inputs = array_ops.ones((8, 5))
-
-    with self.assertRaisesRegexp(ValueError, 'input shapes have not been set'):
-      training_utils.trace_model_call(model)
-
-    fn = training_utils.trace_model_call(
-        model, [tensor_spec.TensorSpec(shape=[None, 5], dtype=dtypes.float32)])
-    signature_outputs = fn(inputs)
-    expected_outputs = {model.output_names[0]: model(inputs)}
-    self._assert_all_close(expected_outputs, signature_outputs)
-
-  @keras_parameterized.run_all_keras_modes
-  def test_subclassed_model_with_input_signature(self):
-
-    class Model(keras.Model):
-
-      def __init__(self):
-        super(Model, self).__init__()
-        self.dense = keras.layers.Dense(3, name='dense')
-
-      @def_function.function(
-          input_signature=[[tensor_spec.TensorSpec([None, 5], dtypes.float32),
-                            tensor_spec.TensorSpec([None], dtypes.float32)]],)
-      def call(self, inputs, *args):
-        x, y = inputs
-        return self.dense(x) + y
-
-    model = Model()
-    fn = training_utils.trace_model_call(model)
-    x = array_ops.ones((8, 5), dtype=dtypes.float32)
-    y = array_ops.ones((3,), dtype=dtypes.float32)
-    expected_outputs = {'output_1': model([x, y])}
-    signature_outputs = fn([x, y])
-    self._assert_all_close(expected_outputs, signature_outputs)
-
-
-def _import_and_infer(save_dir, inputs):
-  """Import a SavedModel into a TF 1.x-style graph and run `signature_key`."""
-  graph = ops.Graph()
-  with graph.as_default(), session_lib.Session() as session:
-    model = loader.load(session, [tag_constants.SERVING], save_dir)
-    signature = model.signature_def[
-        signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY]
-    assert set(inputs.keys()) == set(signature.inputs.keys())
-    feed_dict = {}
-    for arg_name in inputs.keys():
-      feed_dict[graph.get_tensor_by_name(signature.inputs[arg_name].name)] = (
-          inputs[arg_name])
-    output_dict = {}
-    for output_name, output_tensor_info in signature.outputs.items():
-      output_dict[output_name] = graph.get_tensor_by_name(
-          output_tensor_info.name)
-    return session.run(output_dict, feed_dict=feed_dict)
-
-
-class ModelSaveTest(keras_parameterized.TestCase):
-
-  @keras_parameterized.run_with_all_model_types
-  @keras_parameterized.run_all_keras_modes(always_skip_v1=True)
-  def test_model_save(self):
-    input_dim = 5
-    model = testing_utils.get_small_mlp(10, 3, input_dim)
-    inputs = array_ops.ones((8, 5))
-
-    if testing_utils.get_model_type() == 'subclass':
-      model._set_inputs(inputs)
-
-    save_dir = os.path.join(self.get_temp_dir(), 'saved_model')
-    save_lib.save(model, save_dir)
-
-    self.assertAllClose(
-        {model.output_names[0]: model.predict_on_batch(inputs)},
-        _import_and_infer(save_dir, {model.input_names[0]: np.ones((8, 5))}))
-
-
 class DatasetUtilsTest(test.TestCase, parameterized.TestCase):
 
   @parameterized.named_parameters(
diff --git a/tensorflow/python/keras/layers/cudnn_recurrent_test.py b/tensorflow/python/keras/layers/cudnn_recurrent_test.py
index 36f2d2fa38..1e305baefd 100644
--- a/tensorflow/python/keras/layers/cudnn_recurrent_test.py
+++ b/tensorflow/python/keras/layers/cudnn_recurrent_test.py
@@ -467,7 +467,7 @@ class CuDNNV1OnlyTest(keras_parameterized.TestCase):
 
     def assert_not_compatible(src, dest, message):
       with self.assertRaises(ValueError) as ex:
-        keras.engine.saving.preprocess_weights_for_loading(
+        keras.saving.preprocess_weights_for_loading(
             dest,
             get_layer_weights(src))
       self.assertIn(message, str(ex.exception))
diff --git a/tensorflow/python/keras/models.py b/tensorflow/python/keras/models.py
index 851de6e71f..9bc5aa2be5 100644
--- a/tensorflow/python/keras/models.py
+++ b/tensorflow/python/keras/models.py
@@ -22,13 +22,14 @@ from __future__ import print_function
 from tensorflow.python.keras import backend as K
 from tensorflow.python.keras import metrics as metrics_module
 from tensorflow.python.keras import optimizers
-from tensorflow.python.keras.engine import saving
 from tensorflow.python.keras.engine import sequential
 from tensorflow.python.keras.engine import training
 from tensorflow.python.keras.engine.base_layer import Layer
 from tensorflow.python.keras.engine.input_layer import Input
 from tensorflow.python.keras.engine.input_layer import InputLayer
 from tensorflow.python.keras.engine.network import Network
+from tensorflow.python.keras.saving import hdf5_format
+from tensorflow.python.keras.saving import model_config
 from tensorflow.python.keras.utils import generic_utils
 from tensorflow.python.keras.utils.generic_utils import CustomObjectScope
 from tensorflow.python.util import nest
@@ -38,11 +39,11 @@ from tensorflow.python.util.tf_export import keras_export
 # API entries importable from `keras.models`:
 Model = training.Model  # pylint: disable=invalid-name
 Sequential = sequential.Sequential  # pylint: disable=invalid-name
-save_model = saving.save_model
-load_model = saving.load_model
-model_from_config = saving.model_from_config
-model_from_yaml = saving.model_from_yaml
-model_from_json = saving.model_from_json
+save_model = hdf5_format.save_model
+load_model = hdf5_format.load_model
+model_from_config = model_config.model_from_config
+model_from_yaml = model_config.model_from_yaml
+model_from_json = model_config.model_from_json
 
 
 def _clone_layer(layer):
diff --git a/tensorflow/python/keras/saving/BUILD b/tensorflow/python/keras/saving/BUILD
deleted file mode 100644
index a78d58e876..0000000000
--- a/tensorflow/python/keras/saving/BUILD
+++ /dev/null
@@ -1,72 +0,0 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-# Description:
-#   Keras saving and loading libraries.
-load("//tensorflow:tensorflow.bzl", "py_test")
-
-package(default_visibility = ["//tensorflow:__subpackages__"])
-
-licenses(["notice"])  # Apache 2.0
-
-exports_files(["LICENSE"])
-
-py_library(
-    name = "saving",
-    srcs = ["__init__.py"],
-    deps = [":saved_model"],
-)
-
-py_library(
-    name = "saved_model",
-    srcs = [
-        "saved_model.py",
-        "saving_utils.py",
-    ],
-    srcs_version = "PY2AND3",
-    visibility = ["//visibility:public"],
-    deps = [
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:lib",
-        "//tensorflow/python:metrics",
-        "//tensorflow/python:mode_keys",
-        "//tensorflow/python:platform",
-        "//tensorflow/python:saver",
-        "//tensorflow/python:util",
-        "//tensorflow/python/keras:engine",
-        "//tensorflow/python/saved_model",
-        "//tensorflow/python/saved_model/model_utils",
-    ],
-)
-
-py_test(
-    name = "saved_model_test",
-    size = "medium",
-    srcs = ["saved_model_test.py"],
-    srcs_version = "PY2AND3",
-    tags = [
-        "no_oss",  # TODO(b/119349471): Re-enable
-        "no_windows",
-    ],
-    deps = [
-        ":saved_model",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:mode_keys",
-        "//tensorflow/python/keras",
-        "//third_party/py/numpy",
-        "@absl_py//absl/testing:parameterized",
-    ],
-)
diff --git a/tensorflow/python/keras/saving/__init__.py b/tensorflow/python/keras/saving/__init__.py
index 8ff9f3b74e..bb4db68124 100644
--- a/tensorflow/python/keras/saving/__init__.py
+++ b/tensorflow/python/keras/saving/__init__.py
@@ -12,10 +12,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Utils for saving a Keras Model or Estimator to the SavedModel format."""
+"""Utils for saving and loading Keras Models."""
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.keras.saving.hdf5_format import load_attributes_from_hdf5_group
+from tensorflow.python.keras.saving.hdf5_format import load_model
+from tensorflow.python.keras.saving.hdf5_format import load_weights_from_hdf5_group
+from tensorflow.python.keras.saving.hdf5_format import load_weights_from_hdf5_group_by_name
+from tensorflow.python.keras.saving.hdf5_format import preprocess_weights_for_loading
+from tensorflow.python.keras.saving.hdf5_format import save_attributes_to_hdf5_group
+from tensorflow.python.keras.saving.hdf5_format import save_model
+from tensorflow.python.keras.saving.hdf5_format import save_weights_to_hdf5_group
+from tensorflow.python.keras.saving.model_config import model_from_config
+from tensorflow.python.keras.saving.model_config import model_from_json
+from tensorflow.python.keras.saving.model_config import model_from_yaml
 from tensorflow.python.keras.saving.saved_model import export
 from tensorflow.python.keras.saving.saved_model import load_from_saved_model
+from tensorflow.python.keras.saving.saving_utils import trace_model_call
+
+
diff --git a/tensorflow/python/keras/saving/hdf5_format.py b/tensorflow/python/keras/saving/hdf5_format.py
new file mode 100644
index 0000000000..cb925c30a6
--- /dev/null
+++ b/tensorflow/python/keras/saving/hdf5_format.py
@@ -0,0 +1,895 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+# pylint: disable=protected-access
+"""Functions for saving and loading a Keras Model from HDF5 format.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import json
+import os
+
+import numpy as np
+from six.moves import zip  # pylint: disable=redefined-builtin
+
+from tensorflow.python.framework import ops
+from tensorflow.python.keras import backend as K
+from tensorflow.python.keras import optimizers
+from tensorflow.python.keras.saving import model_config as model_config_lib
+from tensorflow.python.keras.utils import conv_utils
+from tensorflow.python.keras.utils.io_utils import ask_to_proceed_with_overwrite
+from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.util import serialization
+from tensorflow.python.util.tf_export import keras_export
+
+# pylint: disable=g-import-not-at-top
+try:
+  import h5py
+  HDF5_OBJECT_HEADER_LIMIT = 64512
+except ImportError:
+  h5py = None
+# pylint: enable=g-import-not-at-top
+
+
+@keras_export('keras.models.save_model')
+def save_model(model, filepath, overwrite=True, include_optimizer=True):
+  """Saves a model to a HDF5 file.
+
+  The saved model contains:
+      - the model's configuration (topology)
+      - the model's weights
+      - the model's optimizer's state (if any)
+
+  Thus the saved model can be reinstantiated in
+  the exact same state, without any of the code
+  used for model definition or training.
+
+  Arguments:
+      model: Keras model instance to be saved.
+      filepath: One of the following:
+          - String, path where to save the model
+          - `h5py.File` object where to save the model
+      overwrite: Whether we should overwrite any existing
+          model at the target location, or instead
+          ask the user with a manual prompt.
+      include_optimizer: If True, save optimizer's state together.
+
+  Raises:
+      ImportError: if h5py is not available.
+  """
+
+  if h5py is None:
+    raise ImportError('`save_model` requires h5py.')
+
+  from tensorflow.python.keras import __version__ as keras_version  # pylint: disable=g-import-not-at-top
+
+  # TODO(psv) Add warning when we save models that contain non-serializable
+  # entities like metrics added using `add_metric` and losses added using
+  # `add_loss.`
+
+  if not isinstance(filepath, h5py.File):
+    # If file exists and should not be overwritten.
+    if not overwrite and os.path.isfile(filepath):
+      proceed = ask_to_proceed_with_overwrite(filepath)
+      if not proceed:
+        return
+
+    f = h5py.File(filepath, mode='w')
+    opened_new_file = True
+  else:
+    f = filepath
+    opened_new_file = False
+
+  try:
+    f.attrs['keras_version'] = str(keras_version).encode('utf8')
+    f.attrs['backend'] = K.backend().encode('utf8')
+    f.attrs['model_config'] = json.dumps(
+        {
+            'class_name': model.__class__.__name__,
+            'config': model.get_config()
+        },
+        default=serialization.get_json_type).encode('utf8')
+
+    model_weights_group = f.create_group('model_weights')
+    model_layers = model.layers
+    save_weights_to_hdf5_group(model_weights_group, model_layers)
+
+    if include_optimizer and model.optimizer:
+      if isinstance(model.optimizer, optimizers.TFOptimizer):
+        logging.warning(
+            'TensorFlow optimizers do not '
+            'make it possible to access '
+            'optimizer attributes or optimizer state '
+            'after instantiation. '
+            'As a result, we cannot save the optimizer '
+            'as part of the model save file.'
+            'You will have to compile your model again after loading it. '
+            'Prefer using a Keras optimizer instead '
+            '(see keras.io/optimizers).')
+      else:
+        f.attrs['training_config'] = json.dumps(
+            {
+                'optimizer_config': {
+                    'class_name': model.optimizer.__class__.__name__,
+                    'config': model.optimizer.get_config()
+                },
+                'loss': model.loss,
+                'metrics': model._compile_metrics,
+                'weighted_metrics': model._compile_weighted_metrics,
+                'sample_weight_mode': model.sample_weight_mode,
+                'loss_weights': model.loss_weights,
+            },
+            default=serialization.get_json_type).encode('utf8')
+
+        # Save optimizer weights.
+        symbolic_weights = getattr(model.optimizer, 'weights')
+        if symbolic_weights:
+          optimizer_weights_group = f.create_group('optimizer_weights')
+          weight_values = K.batch_get_value(symbolic_weights)
+          weight_names = []
+          for w, val in zip(symbolic_weights, weight_values):
+            name = str(w.name)
+            weight_names.append(name.encode('utf8'))
+          optimizer_weights_group.attrs['weight_names'] = weight_names
+          for name, val in zip(weight_names, weight_values):
+            param_dset = optimizer_weights_group.create_dataset(
+                name, val.shape, dtype=val.dtype)
+            if not val.shape:
+              # scalar
+              param_dset[()] = val
+            else:
+              param_dset[:] = val
+    f.flush()
+  finally:
+    if opened_new_file:
+      f.close()
+
+
+@keras_export('keras.models.load_model')
+def load_model(filepath, custom_objects=None, compile=True):  # pylint: disable=redefined-builtin
+  """Loads a model saved via `save_model`.
+
+  Arguments:
+      filepath: One of the following:
+          - String, path to the saved model
+          - `h5py.File` object from which to load the model
+      custom_objects: Optional dictionary mapping names
+          (strings) to custom classes or functions to be
+          considered during deserialization.
+      compile: Boolean, whether to compile the model
+          after loading.
+
+  Returns:
+      A Keras model instance. If an optimizer was found
+      as part of the saved model, the model is already
+      compiled. Otherwise, the model is uncompiled and
+      a warning will be displayed. When `compile` is set
+      to False, the compilation is omitted without any
+      warning.
+
+  Raises:
+      ImportError: if h5py is not available.
+      ValueError: In case of an invalid savefile.
+  """
+  if h5py is None:
+    raise ImportError('`load_model` requires h5py.')
+
+  if not custom_objects:
+    custom_objects = {}
+
+  def convert_custom_objects(obj):
+    """Handles custom object lookup.
+
+    Arguments:
+        obj: object, dict, or list.
+
+    Returns:
+        The same structure, where occurrences
+            of a custom object name have been replaced
+            with the custom object.
+    """
+    if isinstance(obj, list):
+      deserialized = []
+      for value in obj:
+        deserialized.append(convert_custom_objects(value))
+      return deserialized
+    if isinstance(obj, dict):
+      deserialized = {}
+      for key, value in obj.items():
+        deserialized[key] = convert_custom_objects(value)
+      return deserialized
+    if obj in custom_objects:
+      return custom_objects[obj]
+    return obj
+
+  opened_new_file = not isinstance(filepath, h5py.File)
+  if opened_new_file:
+    f = h5py.File(filepath, mode='r')
+  else:
+    f = filepath
+
+  model = None
+  try:
+    # instantiate model
+    model_config = f.attrs.get('model_config')
+    if model_config is None:
+      raise ValueError('No model found in config file.')
+    model_config = json.loads(model_config.decode('utf-8'))
+    model = model_config_lib.model_from_config(model_config,
+                                               custom_objects=custom_objects)
+
+    # set weights
+    load_weights_from_hdf5_group(f['model_weights'], model.layers)
+
+    if compile:
+      # instantiate optimizer
+      training_config = f.attrs.get('training_config')
+      if training_config is None:
+        logging.warning('No training configuration found in save file: '
+                        'the model was *not* compiled. Compile it manually.')
+        return model
+      training_config = json.loads(training_config.decode('utf-8'))
+      optimizer_config = training_config['optimizer_config']
+      optimizer = optimizers.deserialize(
+          optimizer_config, custom_objects=custom_objects)
+
+      # Recover loss functions and metrics.
+      loss = convert_custom_objects(training_config['loss'])
+      metrics = convert_custom_objects(training_config['metrics'])
+      weighted_metrics = convert_custom_objects(
+          training_config.get('weighted_metrics', None))
+      sample_weight_mode = training_config['sample_weight_mode']
+      loss_weights = training_config['loss_weights']
+
+      # Compile model.
+      model.compile(
+          optimizer=optimizer,
+          loss=loss,
+          metrics=metrics,
+          weighted_metrics=weighted_metrics,
+          loss_weights=loss_weights,
+          sample_weight_mode=sample_weight_mode)
+
+      # Set optimizer weights.
+      if 'optimizer_weights' in f:
+        # Build train function (to get weight updates).
+        # Models that aren't graph networks must wait until they are called
+        # with data to _make_train_function() and so can't load optimizer
+        # weights.
+        if model._is_graph_network:  # pylint: disable=protected-access
+          model._make_train_function()
+          optimizer_weights_group = f['optimizer_weights']
+          optimizer_weight_names = [
+              n.decode('utf8')
+              for n in optimizer_weights_group.attrs['weight_names']
+          ]
+          optimizer_weight_values = [
+              optimizer_weights_group[n] for n in optimizer_weight_names
+          ]
+          try:
+            model.optimizer.set_weights(optimizer_weight_values)
+          except ValueError:
+            logging.warning('Error in loading the saved optimizer '
+                            'state. As a result, your model is '
+                            'starting with a freshly initialized '
+                            'optimizer.')
+        else:
+          logging.warning('Sequential models without an `input_shape` '
+                          'passed to the first layer cannot reload their '
+                          'optimizer state. As a result, your model is'
+                          'starting with a freshly initialized optimizer.')
+
+  finally:
+    if opened_new_file:
+      f.close()
+  return model
+
+
+def preprocess_weights_for_loading(layer,
+                                   weights,
+                                   original_keras_version=None,
+                                   original_backend=None):
+  """Preprocess layer weights between different Keras formats.
+
+  Converts layers weights from Keras 1 format to Keras 2 and also weights of
+  CuDNN layers in Keras 2.
+
+  Arguments:
+      layer: Layer instance.
+      weights: List of weights values (Numpy arrays).
+      original_keras_version: Keras version for the weights, as a string.
+      original_backend: Keras backend the weights were trained with,
+          as a string.
+
+  Returns:
+      A list of weights values (Numpy arrays).
+  """
+  def convert_nested_bidirectional(weights):
+    """Converts layers nested in `Bidirectional` wrapper.
+
+    This function uses `preprocess_weights_for_loading()` for converting
+    layers.
+
+    Arguments:
+        weights: List of weights values (Numpy arrays).
+
+    Returns:
+        A list of weights values (Numpy arrays).
+    """
+    num_weights_per_layer = len(weights) // 2
+    forward_weights = preprocess_weights_for_loading(
+        layer.forward_layer, weights[:num_weights_per_layer],
+        original_keras_version, original_backend)
+    backward_weights = preprocess_weights_for_loading(
+        layer.backward_layer, weights[num_weights_per_layer:],
+        original_keras_version, original_backend)
+    return forward_weights + backward_weights
+
+  def convert_nested_time_distributed(weights):
+    """Converts layers nested in `TimeDistributed` wrapper.
+
+    This function uses `preprocess_weights_for_loading()` for converting nested
+    layers.
+
+    Arguments:
+        weights: List of weights values (Numpy arrays).
+
+    Returns:
+        A list of weights values (Numpy arrays).
+    """
+    return preprocess_weights_for_loading(
+        layer.layer, weights, original_keras_version, original_backend)
+
+  def convert_nested_model(weights):
+    """Converts layers nested in `Model` or `Sequential`.
+
+    This function uses `preprocess_weights_for_loading()` for converting nested
+    layers.
+
+    Arguments:
+        weights: List of weights values (Numpy arrays).
+
+    Returns:
+        A list of weights values (Numpy arrays).
+    """
+    new_weights = []
+    # trainable weights
+    for sublayer in layer.layers:
+      num_weights = len(sublayer.trainable_weights)
+      if num_weights > 0:
+        new_weights.extend(preprocess_weights_for_loading(
+            layer=sublayer,
+            weights=weights[:num_weights],
+            original_keras_version=original_keras_version,
+            original_backend=original_backend))
+        weights = weights[num_weights:]
+
+    # non-trainable weights
+    for sublayer in layer.layers:
+      num_weights = len([l for l in sublayer.weights
+                         if l not in sublayer.trainable_weights])
+      if num_weights > 0:
+        new_weights.extend(preprocess_weights_for_loading(
+            layer=sublayer,
+            weights=weights[:num_weights],
+            original_keras_version=original_keras_version,
+            original_backend=original_backend))
+        weights = weights[num_weights:]
+    return new_weights
+
+  # Convert layers nested in Bidirectional/Model/Sequential.
+  # Both transformation should be ran for both Keras 1->2 conversion
+  # and for conversion of CuDNN layers.
+  if layer.__class__.__name__ == 'Bidirectional':
+    weights = convert_nested_bidirectional(weights)
+  if layer.__class__.__name__ == 'TimeDistributed':
+    weights = convert_nested_time_distributed(weights)
+  elif layer.__class__.__name__ in ['Model', 'Sequential']:
+    weights = convert_nested_model(weights)
+
+  if original_keras_version == '1':
+    if layer.__class__.__name__ == 'TimeDistributed':
+      weights = preprocess_weights_for_loading(
+          layer.layer, weights, original_keras_version, original_backend)
+
+    if layer.__class__.__name__ == 'Conv1D':
+      shape = weights[0].shape
+      # Handle Keras 1.1 format
+      if shape[:2] != (layer.kernel_size[0], 1) or shape[3] != layer.filters:
+        # Legacy shape:
+        # (filters, input_dim, filter_length, 1)
+        assert shape[0] == layer.filters and shape[2:] == (layer.kernel_size[0],
+                                                           1)
+        weights[0] = np.transpose(weights[0], (2, 3, 1, 0))
+      weights[0] = weights[0][:, 0, :, :]
+
+    if layer.__class__.__name__ == 'Conv2D':
+      if layer.data_format == 'channels_first':
+        # old: (filters, stack_size, kernel_rows, kernel_cols)
+        # new: (kernel_rows, kernel_cols, stack_size, filters)
+        weights[0] = np.transpose(weights[0], (2, 3, 1, 0))
+
+    if layer.__class__.__name__ == 'Conv2DTranspose':
+      if layer.data_format == 'channels_last':
+        # old: (kernel_rows, kernel_cols, stack_size, filters)
+        # new: (kernel_rows, kernel_cols, filters, stack_size)
+        weights[0] = np.transpose(weights[0], (0, 1, 3, 2))
+      if layer.data_format == 'channels_first':
+        # old: (filters, stack_size, kernel_rows, kernel_cols)
+        # new: (kernel_rows, kernel_cols, filters, stack_size)
+        weights[0] = np.transpose(weights[0], (2, 3, 0, 1))
+
+    if layer.__class__.__name__ == 'Conv3D':
+      if layer.data_format == 'channels_first':
+        # old: (filters, stack_size, ...)
+        # new: (..., stack_size, filters)
+        weights[0] = np.transpose(weights[0], (2, 3, 4, 1, 0))
+
+    if layer.__class__.__name__ == 'GRU':
+      if len(weights) == 9:
+        kernel = np.concatenate([weights[0], weights[3], weights[6]], axis=-1)
+        recurrent_kernel = np.concatenate(
+            [weights[1], weights[4], weights[7]], axis=-1)
+        bias = np.concatenate([weights[2], weights[5], weights[8]], axis=-1)
+        weights = [kernel, recurrent_kernel, bias]
+
+    if layer.__class__.__name__ == 'LSTM':
+      if len(weights) == 12:
+        # old: i, c, f, o
+        # new: i, f, c, o
+        kernel = np.concatenate(
+            [weights[0], weights[6], weights[3], weights[9]], axis=-1)
+        recurrent_kernel = np.concatenate(
+            [weights[1], weights[7], weights[4], weights[10]], axis=-1)
+        bias = np.concatenate(
+            [weights[2], weights[8], weights[5], weights[11]], axis=-1)
+        weights = [kernel, recurrent_kernel, bias]
+
+    if layer.__class__.__name__ == 'ConvLSTM2D':
+      if len(weights) == 12:
+        kernel = np.concatenate(
+            [weights[0], weights[6], weights[3], weights[9]], axis=-1)
+        recurrent_kernel = np.concatenate(
+            [weights[1], weights[7], weights[4], weights[10]], axis=-1)
+        bias = np.concatenate(
+            [weights[2], weights[8], weights[5], weights[11]], axis=-1)
+        if layer.data_format == 'channels_first':
+          # old: (filters, stack_size, kernel_rows, kernel_cols)
+          # new: (kernel_rows, kernel_cols, stack_size, filters)
+          kernel = np.transpose(kernel, (2, 3, 1, 0))
+          recurrent_kernel = np.transpose(recurrent_kernel, (2, 3, 1, 0))
+        weights = [kernel, recurrent_kernel, bias]
+
+  conv_layers = ['Conv1D', 'Conv2D', 'Conv3D', 'Conv2DTranspose', 'ConvLSTM2D']
+  if layer.__class__.__name__ in conv_layers:
+    if original_backend == 'theano':
+      weights[0] = conv_utils.convert_kernel(weights[0])
+      if layer.__class__.__name__ == 'ConvLSTM2D':
+        weights[1] = conv_utils.convert_kernel(weights[1])
+    if K.int_shape(layer.weights[0]) != weights[0].shape:
+      weights[0] = np.transpose(weights[0], (3, 2, 0, 1))
+      if layer.__class__.__name__ == 'ConvLSTM2D':
+        weights[1] = np.transpose(weights[1], (3, 2, 0, 1))
+
+  # convert CuDNN layers
+  return _convert_rnn_weights(layer, weights)
+
+
+def _convert_rnn_weights(layer, weights):
+  """Converts weights for RNN layers between native and CuDNN format.
+
+  Input kernels for each gate are transposed and converted between Fortran
+  and C layout, recurrent kernels are transposed. For LSTM biases are summed/
+  split in half, for GRU biases are reshaped.
+
+  Weights can be converted in both directions between `LSTM` and`CuDNNSLTM`
+  and between `CuDNNGRU` and `GRU(reset_after=True)`. Default `GRU` is not
+  compatible with `CuDNNGRU`.
+
+  For missing biases in `LSTM`/`GRU` (`use_bias=False`) no conversion is made.
+
+  Arguments:
+      layer: Target layer instance.
+      weights: List of source weights values (input kernels, recurrent
+          kernels, [biases]) (Numpy arrays).
+
+  Returns:
+      A list of converted weights values (Numpy arrays).
+
+  Raises:
+      ValueError: for incompatible GRU layer/weights or incompatible biases
+  """
+
+  def transform_kernels(kernels, func, n_gates):
+    """Transforms kernel for each gate separately using given function.
+
+    Arguments:
+        kernels: Stacked array of kernels for individual gates.
+        func: Function applied to kernel of each gate.
+        n_gates: Number of gates (4 for LSTM, 3 for GRU).
+
+    Returns:
+        Stacked array of transformed kernels.
+    """
+    return np.hstack([func(k) for k in np.hsplit(kernels, n_gates)])
+
+  def transpose_input(from_cudnn):
+    """Makes a function that transforms input kernels from/to CuDNN format.
+
+    It keeps the shape, but changes between the layout (Fortran/C). Eg.:
+
+    ```
+    Keras                 CuDNN
+    [[0, 1, 2],  <--->  [[0, 2, 4],
+     [3, 4, 5]]          [1, 3, 5]]
+    ```
+
+    It can be passed to `transform_kernels()`.
+
+    Arguments:
+        from_cudnn: `True` if source weights are in CuDNN format, `False`
+            if they're in plain Keras format.
+
+    Returns:
+        Function that converts input kernel to the other format.
+    """
+    order = 'F' if from_cudnn else 'C'
+
+    def transform(kernel):
+      return kernel.T.reshape(kernel.shape, order=order)
+
+    return transform
+
+  target_class = layer.__class__.__name__
+
+  # convert the weights between CuDNNLSTM and LSTM
+  if target_class in ['LSTM', 'CuDNNLSTM'] and len(weights) == 3:
+    # determine if we're loading a CuDNNLSTM layer
+    # from the number of bias weights:
+    # CuDNNLSTM has (units * 8) weights; while LSTM has (units * 4)
+    # if there's no bias weight in the file, skip this conversion
+    units = weights[1].shape[0]
+    bias_shape = weights[2].shape
+    n_gates = 4
+
+    if bias_shape == (2 * units * n_gates,):
+      source = 'CuDNNLSTM'
+    elif bias_shape == (units * n_gates,):
+      source = 'LSTM'
+    else:
+      raise ValueError('Invalid bias shape: ' + str(bias_shape))
+
+    def convert_lstm_weights(weights, from_cudnn=True):
+      """Converts the weights between CuDNNLSTM and LSTM.
+
+      Arguments:
+        weights: Original weights.
+        from_cudnn: Indicates whether original weights are from CuDNN layer.
+
+      Returns:
+        Updated weights compatible with LSTM.
+      """
+
+      # Transpose (and reshape) input and recurrent kernels
+      kernels = transform_kernels(weights[0], transpose_input(from_cudnn),
+                                  n_gates)
+      recurrent_kernels = transform_kernels(weights[1], lambda k: k.T, n_gates)
+      if from_cudnn:
+        # merge input and recurrent biases into a single set
+        biases = np.sum(np.split(weights[2], 2, axis=0), axis=0)
+      else:
+        # Split single set of biases evenly to two sets. The way of
+        # splitting doesn't matter as long as the two sets sum is kept.
+        biases = np.tile(0.5 * weights[2], 2)
+      return [kernels, recurrent_kernels, biases]
+
+    if source != target_class:
+      weights = convert_lstm_weights(weights, from_cudnn=source == 'CuDNNLSTM')
+
+  # convert the weights between CuDNNGRU and GRU(reset_after=True)
+  if target_class in ['GRU', 'CuDNNGRU'] and len(weights) == 3:
+    # We can determine the source of the weights from the shape of the bias.
+    # If there is no bias we skip the conversion since
+    # CuDNNGRU always has biases.
+
+    units = weights[1].shape[0]
+    bias_shape = weights[2].shape
+    n_gates = 3
+
+    def convert_gru_weights(weights, from_cudnn=True):
+      """Converts the weights between CuDNNGRU and GRU.
+
+      Arguments:
+        weights: Original weights.
+        from_cudnn: Indicates whether original weights are from CuDNN layer.
+
+      Returns:
+        Updated weights compatible with GRU.
+      """
+
+      kernels = transform_kernels(weights[0], transpose_input(from_cudnn),
+                                  n_gates)
+      recurrent_kernels = transform_kernels(weights[1], lambda k: k.T, n_gates)
+      biases = np.array(weights[2]).reshape((2, -1) if from_cudnn else -1)
+      return [kernels, recurrent_kernels, biases]
+
+    if bias_shape == (2 * units * n_gates,):
+      source = 'CuDNNGRU'
+    elif bias_shape == (2, units * n_gates):
+      source = 'GRU(reset_after=True)'
+    elif bias_shape == (units * n_gates,):
+      source = 'GRU(reset_after=False)'
+    else:
+      raise ValueError('Invalid bias shape: ' + str(bias_shape))
+
+    if target_class == 'CuDNNGRU':
+      target = 'CuDNNGRU'
+    elif layer.reset_after:
+      target = 'GRU(reset_after=True)'
+    else:
+      target = 'GRU(reset_after=False)'
+
+    # only convert between different types
+    if source != target:
+      types = (source, target)
+      if 'GRU(reset_after=False)' in types:
+        raise ValueError('%s is not compatible with %s' % types)
+      if source == 'CuDNNGRU':
+        weights = convert_gru_weights(weights, from_cudnn=True)
+      elif source == 'GRU(reset_after=True)':
+        weights = convert_gru_weights(weights, from_cudnn=False)
+
+  return weights
+
+
+def save_weights_to_hdf5_group(f, layers):
+  """Saves the weights of a list of layers to a HDF5 group.
+
+  Arguments:
+      f: HDF5 group.
+      layers: List of layer instances.
+  """
+  from tensorflow.python.keras import __version__ as keras_version  # pylint: disable=g-import-not-at-top
+
+  save_attributes_to_hdf5_group(
+      f, 'layer_names', [layer.name.encode('utf8') for layer in layers])
+  f.attrs['backend'] = K.backend().encode('utf8')
+  f.attrs['keras_version'] = str(keras_version).encode('utf8')
+
+  # On TPUs, modifying the graph between session.runs() triggers some expensive
+  # recompilation overhead. To avoid this, we build up the full set of tensors
+  # to save before fetching weights, thus only modifying the graph once.
+  layer_weights_dict = {}
+  for layer in layers:
+    layer_weights_dict[layer.name] = [ops.convert_to_tensor(w)
+                                      for w in layer.weights]
+
+  for layer in layers:
+    g = f.create_group(layer.name)
+    symbolic_weights = layer_weights_dict[layer.name]
+    weight_values = K.batch_get_value(symbolic_weights)
+    weight_names = []
+    for i, (w, val) in enumerate(zip(symbolic_weights, weight_values)):
+      if hasattr(w, 'name') and w.name:
+        name = str(w.name)
+      else:
+        name = 'param_' + str(i)
+      weight_names.append(name.encode('utf8'))
+    save_attributes_to_hdf5_group(g, 'weight_names', weight_names)
+    for name, val in zip(weight_names, weight_values):
+      param_dset = g.create_dataset(name, val.shape, dtype=val.dtype)
+      if not val.shape:
+        # scalar
+        param_dset[()] = val
+      else:
+        param_dset[:] = val
+
+
+def load_weights_from_hdf5_group(f, layers):
+  """Implements topological (order-based) weight loading.
+
+  Arguments:
+      f: A pointer to a HDF5 group.
+      layers: a list of target layers.
+
+  Raises:
+      ValueError: in case of mismatch between provided layers
+          and weights file.
+  """
+  if 'keras_version' in f.attrs:
+    original_keras_version = f.attrs['keras_version'].decode('utf8')
+  else:
+    original_keras_version = '1'
+  if 'backend' in f.attrs:
+    original_backend = f.attrs['backend'].decode('utf8')
+  else:
+    original_backend = None
+
+  filtered_layers = []
+  for layer in layers:
+    weights = layer.weights
+    if weights:
+      filtered_layers.append(layer)
+
+  layer_names = load_attributes_from_hdf5_group(f, 'layer_names')
+  filtered_layer_names = []
+  for name in layer_names:
+    g = f[name]
+    weight_names = load_attributes_from_hdf5_group(g, 'weight_names')
+    if weight_names:
+      filtered_layer_names.append(name)
+  layer_names = filtered_layer_names
+  if len(layer_names) != len(filtered_layers):
+    raise ValueError('You are trying to load a weight file '
+                     'containing ' + str(len(layer_names)) +
+                     ' layers into a model with ' + str(len(filtered_layers)) +
+                     ' layers.')
+
+  # We batch weight value assignments in a single backend call
+  # which provides a speedup in TensorFlow.
+  weight_value_tuples = []
+  for k, name in enumerate(layer_names):
+    g = f[name]
+    weight_names = load_attributes_from_hdf5_group(g, 'weight_names')
+    weight_values = [np.asarray(g[weight_name]) for weight_name in weight_names]
+    layer = filtered_layers[k]
+    symbolic_weights = layer.weights
+    weight_values = preprocess_weights_for_loading(
+        layer, weight_values, original_keras_version, original_backend)
+    if len(weight_values) != len(symbolic_weights):
+      raise ValueError('Layer #' + str(k) + ' (named "' + layer.name +
+                       '" in the current model) was found to '
+                       'correspond to layer ' + name + ' in the save file. '
+                       'However the new layer ' + layer.name + ' expects ' +
+                       str(len(symbolic_weights)) +
+                       ' weights, but the saved weights have ' +
+                       str(len(weight_values)) + ' elements.')
+    weight_value_tuples += zip(symbolic_weights, weight_values)
+  K.batch_set_value(weight_value_tuples)
+
+
+def load_weights_from_hdf5_group_by_name(f, layers):
+  """Implements name-based weight loading.
+
+  (instead of topological weight loading).
+
+  Layers that have no matching name are skipped.
+
+  Arguments:
+      f: A pointer to a HDF5 group.
+      layers: a list of target layers.
+
+  Raises:
+      ValueError: in case of mismatch between provided layers
+          and weights file.
+  """
+  if 'keras_version' in f.attrs:
+    original_keras_version = f.attrs['keras_version'].decode('utf8')
+  else:
+    original_keras_version = '1'
+  if 'backend' in f.attrs:
+    original_backend = f.attrs['backend'].decode('utf8')
+  else:
+    original_backend = None
+
+  # New file format.
+  layer_names = load_attributes_from_hdf5_group(f, 'layer_names')
+
+  # Reverse index of layer name to list of layers with name.
+  index = {}
+  for layer in layers:
+    if layer.name:
+      index.setdefault(layer.name, []).append(layer)
+
+  # We batch weight value assignments in a single backend call
+  # which provides a speedup in TensorFlow.
+  weight_value_tuples = []
+  for k, name in enumerate(layer_names):
+    g = f[name]
+    weight_names = load_attributes_from_hdf5_group(g, 'weight_names')
+    weight_values = [np.asarray(g[weight_name]) for weight_name in weight_names]
+
+    for layer in index.get(name, []):
+      symbolic_weights = layer.weights
+      weight_values = preprocess_weights_for_loading(
+          layer, weight_values, original_keras_version, original_backend)
+      if len(weight_values) != len(symbolic_weights):
+        raise ValueError('Layer #' + str(k) + ' (named "' + layer.name +
+                         '") expects ' + str(len(symbolic_weights)) +
+                         ' weight(s), but the saved weights' + ' have ' +
+                         str(len(weight_values)) + ' element(s).')
+      # Set values.
+      for i in range(len(weight_values)):
+        if K.int_shape(symbolic_weights[i]) != weight_values[i].shape:
+          raise ValueError('Layer #' + str(k) +' (named "' + layer.name +
+                           '"), weight ' + str(symbolic_weights[i]) +
+                           ' has shape {}'.format(K.int_shape(
+                               symbolic_weights[i])) +
+                           ', but the saved weight has shape ' +
+                           str(weight_values[i].shape) + '.')
+
+        else:
+          weight_value_tuples.append((symbolic_weights[i], weight_values[i]))
+  K.batch_set_value(weight_value_tuples)
+
+
+def save_attributes_to_hdf5_group(group, name, data):
+  """Saves attributes (data) of the specified name into the HDF5 group.
+
+  This method deals with an inherent problem of HDF5 file which is not
+  able to store data larger than HDF5_OBJECT_HEADER_LIMIT bytes.
+
+  Arguments:
+      group: A pointer to a HDF5 group.
+      name: A name of the attributes to save.
+      data: Attributes data to store.
+
+  Raises:
+    RuntimeError: If any single attribute is too large to be saved.
+  """
+  # Check that no item in `data` is larger than `HDF5_OBJECT_HEADER_LIMIT`
+  # because in that case even chunking the array would not make the saving
+  # possible.
+  bad_attributes = [x for x in data if len(x) > HDF5_OBJECT_HEADER_LIMIT]
+
+  # Expecting this to never be true.
+  if bad_attributes:
+    raise RuntimeError('The following attributes cannot be saved to HDF5 '
+                       'file because they are larger than %d bytes: %s' %
+                       (HDF5_OBJECT_HEADER_LIMIT,
+                        ', '.join([x for x in bad_attributes])))
+
+  data_npy = np.asarray(data)
+
+  num_chunks = 1
+  chunked_data = np.array_split(data_npy, num_chunks)
+
+  # This will never loop forever thanks to the test above.
+  while any(x.nbytes > HDF5_OBJECT_HEADER_LIMIT for x in chunked_data):
+    num_chunks += 1
+    chunked_data = np.array_split(data_npy, num_chunks)
+
+  if num_chunks > 1:
+    for chunk_id, chunk_data in enumerate(chunked_data):
+      group.attrs['%s%d' % (name, chunk_id)] = chunk_data
+  else:
+    group.attrs[name] = data
+
+
+def load_attributes_from_hdf5_group(group, name):
+  """Loads attributes of the specified name from the HDF5 group.
+
+  This method deals with an inherent problem
+  of HDF5 file which is not able to store
+  data larger than HDF5_OBJECT_HEADER_LIMIT bytes.
+
+  Arguments:
+      group: A pointer to a HDF5 group.
+      name: A name of the attributes to load.
+
+  Returns:
+      data: Attributes data.
+  """
+  if name in group.attrs:
+    data = [n.decode('utf8') for n in group.attrs[name]]
+  else:
+    data = []
+    chunk_id = 0
+    while '%s%d' % (name, chunk_id) in group.attrs:
+      data.extend(
+          [n.decode('utf8') for n in group.attrs['%s%d' % (name, chunk_id)]])
+      chunk_id += 1
+  return data
diff --git a/tensorflow/python/keras/engine/saving_test.py b/tensorflow/python/keras/saving/hdf5_format_test.py
similarity index 98%
rename from tensorflow/python/keras/engine/saving_test.py
rename to tensorflow/python/keras/saving/hdf5_format_test.py
index 92fac6f242..3b496e70c1 100644
--- a/tensorflow/python/keras/engine/saving_test.py
+++ b/tensorflow/python/keras/saving/hdf5_format_test.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #,============================================================================
-"""Tests for model saving."""
+"""Tests for model saving in the HDF5 format."""
 
 from __future__ import absolute_import
 from __future__ import division
@@ -31,8 +31,8 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.keras import optimizers
-from tensorflow.python.keras.engine import saving
 from tensorflow.python.keras.engine import training
+from tensorflow.python.keras.saving import hdf5_format
 from tensorflow.python.lib.io import file_io
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import random_ops
@@ -174,17 +174,17 @@ class TestWeightSavingAndLoading(test.TestCase, parameterized.TestCase):
     ]
     for layer, weights, input_shape in cases:
       layer.build(input_shape)
-      _ = keras.engine.saving.preprocess_weights_for_loading(
+      _ = hdf5_format.preprocess_weights_for_loading(
           layer, weights, original_keras_version='1')
 
     model = keras.models.Sequential([keras.layers.Dense(2, input_dim=2)])
-    _ = keras.engine.saving.preprocess_weights_for_loading(
+    _ = hdf5_format.preprocess_weights_for_loading(
         model, model.weights, original_keras_version='1')
 
     x = keras.Input((2,))
     y = keras.layers.Dense(2)(x)
     model = keras.models.Model(x, y)
-    _ = keras.engine.saving.preprocess_weights_for_loading(
+    _ = hdf5_format.preprocess_weights_for_loading(
         model, model.weights, original_keras_version='1')
 
   @parameterized.named_parameters(
@@ -215,7 +215,7 @@ class TestWeightSavingAndLoading(test.TestCase, parameterized.TestCase):
       layer = layer_class(**layer_args)
       layer.build(input_shape=layer_args.get('input_shape'))
       weights1 = layer.get_weights()
-      weights2 = keras.engine.saving.preprocess_weights_for_loading(
+      weights2 = hdf5_format.preprocess_weights_for_loading(
           layer, weights1)
       _ = [
           self.assertAllClose(x, y, rtol=1e-05)
@@ -274,7 +274,7 @@ class TestWeightSavingAndLoading(test.TestCase, parameterized.TestCase):
                         metrics=[keras.metrics.categorical_accuracy])
 
       f_ref_model = h5py.File(h5_path, 'w')
-      saving.save_weights_to_hdf5_group(f_ref_model, ref_model.layers)
+      hdf5_format.save_weights_to_hdf5_group(f_ref_model, ref_model.layers)
 
       f_model = h5py.File(h5_path, 'r')
       model = keras.models.Sequential()
@@ -288,7 +288,7 @@ class TestWeightSavingAndLoading(test.TestCase, parameterized.TestCase):
                                  r'Layer #0 \(named \"d1\"\) expects 1 '
                                  r'weight\(s\), but the saved weights have 2 '
                                  r'element\(s\)\.'):
-      saving.load_weights_from_hdf5_group_by_name(f_model, model.layers)
+      hdf5_format.load_weights_from_hdf5_group_by_name(f_model, model.layers)
 
   @test_util.run_deprecated_v1
   def test_sequential_weight_loading_group_name_with_incorrect_shape(self):
@@ -312,7 +312,7 @@ class TestWeightSavingAndLoading(test.TestCase, parameterized.TestCase):
                         metrics=[keras.metrics.categorical_accuracy])
 
       f_ref_model = h5py.File(h5_path, 'w')
-      saving.save_weights_to_hdf5_group(f_ref_model, ref_model.layers)
+      hdf5_format.save_weights_to_hdf5_group(f_ref_model, ref_model.layers)
 
       f_model = h5py.File(h5_path, 'r')
       model = keras.models.Sequential()
@@ -328,7 +328,7 @@ class TestWeightSavingAndLoading(test.TestCase, parameterized.TestCase):
                                    r'shape=\(3, 10\) dtype=float32> has '
                                    r'shape \(3, 10\), but the saved weight has '
                                    r'shape \(3, 5\)\.'):
-        saving.load_weights_from_hdf5_group_by_name(f_model, model.layers)
+        hdf5_format.load_weights_from_hdf5_group_by_name(f_model, model.layers)
 
 
 class TestWholeModelSaving(test.TestCase):
diff --git a/tensorflow/python/keras/saving/model_config.py b/tensorflow/python/keras/saving/model_config.py
new file mode 100644
index 0000000000..7f59ecd7df
--- /dev/null
+++ b/tensorflow/python/keras/saving/model_config.py
@@ -0,0 +1,96 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+# pylint: disable=protected-access
+"""Functions that save the model's config into different formats.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import json
+
+from tensorflow.python.util.tf_export import keras_export
+
+# pylint: disable=g-import-not-at-top
+try:
+  import yaml
+except ImportError:
+  yaml = None
+# pylint: enable=g-import-not-at-top
+
+
+@keras_export('keras.models.model_from_config')
+def model_from_config(config, custom_objects=None):
+  """Instantiates a Keras model from its config.
+
+  Arguments:
+      config: Configuration dictionary.
+      custom_objects: Optional dictionary mapping names
+          (strings) to custom classes or functions to be
+          considered during deserialization.
+
+  Returns:
+      A Keras model instance (uncompiled).
+
+  Raises:
+      TypeError: if `config` is not a dictionary.
+  """
+  if isinstance(config, list):
+    raise TypeError('`model_from_config` expects a dictionary, not a list. '
+                    'Maybe you meant to use '
+                    '`Sequential.from_config(config)`?')
+  from tensorflow.python.keras.layers import deserialize  # pylint: disable=g-import-not-at-top
+  return deserialize(config, custom_objects=custom_objects)
+
+
+@keras_export('keras.models.model_from_yaml')
+def model_from_yaml(yaml_string, custom_objects=None):
+  """Parses a yaml model configuration file and returns a model instance.
+
+  Arguments:
+      yaml_string: YAML string encoding a model configuration.
+      custom_objects: Optional dictionary mapping names
+          (strings) to custom classes or functions to be
+          considered during deserialization.
+
+  Returns:
+      A Keras model instance (uncompiled).
+
+  Raises:
+      ImportError: if yaml module is not found.
+  """
+  if yaml is None:
+    raise ImportError('Requires yaml module installed (`pip install pyyaml`).')
+  config = yaml.load(yaml_string)
+  from tensorflow.python.keras.layers import deserialize  # pylint: disable=g-import-not-at-top
+  return deserialize(config, custom_objects=custom_objects)
+
+
+@keras_export('keras.models.model_from_json')
+def model_from_json(json_string, custom_objects=None):
+  """Parses a JSON model configuration file and returns a model instance.
+
+  Arguments:
+      json_string: JSON string encoding a model configuration.
+      custom_objects: Optional dictionary mapping names
+          (strings) to custom classes or functions to be
+          considered during deserialization.
+
+  Returns:
+      A Keras model instance (uncompiled).
+  """
+  config = json.loads(json_string)
+  from tensorflow.python.keras.layers import deserialize  # pylint: disable=g-import-not-at-top
+  return deserialize(config, custom_objects=custom_objects)
diff --git a/tensorflow/python/keras/saving/saved_model.py b/tensorflow/python/keras/saving/saved_model.py
index 5de6d2657f..06e02acf32 100644
--- a/tensorflow/python/keras/saving/saved_model.py
+++ b/tensorflow/python/keras/saving/saved_model.py
@@ -24,12 +24,8 @@ import six
 from tensorflow.python.client import session
 from tensorflow.python.framework import ops
 from tensorflow.python.keras import backend as K
-from tensorflow.python.keras import models as models_lib
 from tensorflow.python.keras import optimizers
-from tensorflow.python.keras.engine import sequential
-from tensorflow.python.keras.engine import training_utils
-from tensorflow.python.keras.metrics import Metric
-from tensorflow.python.keras.models import model_from_json
+from tensorflow.python.keras.saving import model_from_json
 from tensorflow.python.keras.saving import saving_utils
 from tensorflow.python.lib.io import file_io
 from tensorflow.python.ops import variables
@@ -135,7 +131,7 @@ def export(
   if serving_only:
     save_lib.save(
         model, export_dir,
-        signatures=training_utils.trace_model_call(model, input_signature))
+        signatures=saving_utils.trace_model_call(model, input_signature))
   else:
     _save_v1_format(model, export_dir, custom_objects, as_text, input_signature)
 
@@ -167,6 +163,8 @@ def _export_model_variables(model, saved_model_path):
 
 def _save_v1_format(model, path, custom_objects, as_text, input_signature):
   """Exports model to v1 SavedModel format."""
+  from tensorflow.python.keras.engine import sequential  # pylint: disable=g-import-not-at-top
+
   if not model._is_graph_network:
     if isinstance(model, sequential.Sequential):
       # If input shape is not directly set in the model, the exported model
@@ -250,6 +248,7 @@ def _export_mode(
     ValueError: If the train/eval mode is being exported, but the model does
       not have an optimizer.
   """
+  from tensorflow.python.keras import models as models_lib  # pylint: disable=g-import-not-at-top
   compile_clone = (mode != mode_keys.ModeKeys.PREDICT)
   if compile_clone and not model.optimizer:
     raise ValueError(
@@ -339,6 +338,7 @@ def _create_signature_def_map(model, mode):
   local_vars = set(ops.get_collection(ops.GraphKeys.LOCAL_VARIABLES))
   vars_to_add = set()
   if metrics is not None:
+    from tensorflow.python.keras.metrics import Metric  # pylint: disable=g-import-not-at-top
     for key, value in six.iteritems(metrics):
       if isinstance(value, Metric):
         vars_to_add.update(value.variables)
diff --git a/tensorflow/python/keras/saving/saving_utils.py b/tensorflow/python/keras/saving/saving_utils.py
index ddd98aa581..0b643453c9 100644
--- a/tensorflow/python/keras/saving/saving_utils.py
+++ b/tensorflow/python/keras/saving/saving_utils.py
@@ -18,7 +18,9 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.python.keras import metrics
+from tensorflow.python.eager import def_function
+from tensorflow.python.framework import tensor_spec
+from tensorflow.python.util import nest
 
 
 def extract_model_metrics(model):
@@ -33,6 +35,7 @@ def extract_model_metrics(model):
     Dictionary mapping metric names to tuples of (value, update) ops. May return
     `None` if the model does not contain any metrics.
   """
+  from tensorflow.python.keras import metrics  # pylint: disable=g-import-not-at-top
   if not getattr(model, '_compile_metrics', None):
     return None
 
@@ -43,3 +46,58 @@ def extract_model_metrics(model):
     m(model._compile_metrics_tensors[metric_name])
     eval_metric_ops[metric_name] = m
   return eval_metric_ops
+
+
+def trace_model_call(model, input_signature=None):
+  """Trace the model call to create a tf.function for exporting a Keras model.
+
+  Args:
+    model: A Keras model.
+    input_signature: optional, a list of tf.TensorSpec objects specifying the
+      inputs to the model.
+
+  Returns:
+    A tf.function wrapping the model's call function with input signatures set.
+
+  Raises:
+    ValueError: if input signature cannot be inferred from the model.
+  """
+  if input_signature is None:
+    if isinstance(model.call, def_function.PolymorphicFunction):
+      input_signature = model.call.input_signature
+
+  if input_signature is None:
+    try:
+      inputs = model.inputs
+      input_names = model.input_names
+    except AttributeError:
+      raise ValueError(
+          'Model {} cannot be saved because the input shapes have not been '
+          'set. Usually, input shapes are automatically determined from calling'
+          ' .fit() or .predict(). To manually set the shapes, call '
+          'model._set_inputs(inputs).'.format(model))
+    input_specs = []
+    for input_tensor, input_name in zip(inputs, input_names):
+      input_specs.append(tensor_spec.TensorSpec(
+          shape=input_tensor.shape, dtype=input_tensor.dtype,
+          name=input_name))
+    # The input signature of the call function is a list with one element, since
+    # all tensor inputs must be passed in as the first argument.
+    input_signature = [input_specs] if len(input_specs) > 1 else input_specs
+
+  # TODO(mdan): Should the model's call be autographed by default?
+  @def_function.function(input_signature=input_signature, autograph=False)
+  def _wrapped_model(*args):
+    """A concrete tf.function that wraps the model's call function."""
+    # When given a single input, Keras models will call the model on the tensor
+    # rather than a list consisting of the single tensor.
+    inputs = args[0] if len(input_signature) == 1 else list(args)
+    outputs_list = nest.flatten(model(inputs=inputs))
+    try:
+      output_names = model.output_names
+    except AttributeError:
+      from tensorflow.python.keras.engine import training_utils  # pylint: disable=g-import-not-at-top
+      output_names = training_utils.generic_output_names(outputs_list)
+    return {name: output for name, output in zip(output_names, outputs_list)}
+
+  return _wrapped_model
diff --git a/tensorflow/python/keras/saving/saving_utils_test.py b/tensorflow/python/keras/saving/saving_utils_test.py
new file mode 100644
index 0000000000..ae267e283c
--- /dev/null
+++ b/tensorflow/python/keras/saving/saving_utils_test.py
@@ -0,0 +1,209 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for saving utility functions."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+
+import numpy as np
+
+
+from tensorflow.python import keras
+from tensorflow.python.client import session as session_lib
+from tensorflow.python.eager import context
+from tensorflow.python.eager import def_function
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_spec
+from tensorflow.python.keras import backend as K
+from tensorflow.python.keras import keras_parameterized
+from tensorflow.python.keras import testing_utils
+from tensorflow.python.keras.saving import saving_utils
+from tensorflow.python.ops import array_ops
+from tensorflow.python.platform import test
+from tensorflow.python.saved_model import loader
+from tensorflow.python.saved_model import save as save_lib
+from tensorflow.python.saved_model import signature_constants
+from tensorflow.python.saved_model import tag_constants
+
+
+class TraceModelCallTest(keras_parameterized.TestCase):
+
+  def _assert_all_close(self, expected, actual):
+    if not context.executing_eagerly():
+      with self.cached_session() as sess:
+        K._initialize_variables(sess)
+        self.assertAllClose(expected, actual)
+    else:
+      self.assertAllClose(expected, actual)
+
+  @keras_parameterized.run_with_all_model_types
+  @keras_parameterized.run_all_keras_modes
+  def test_trace_model_outputs(self):
+    input_dim = 5 if testing_utils.get_model_type() == 'functional' else None
+    model = testing_utils.get_small_mlp(10, 3, input_dim)
+    inputs = array_ops.ones((8, 5))
+
+    if input_dim is None:
+      with self.assertRaisesRegexp(ValueError,
+                                   'input shapes have not been set'):
+        saving_utils.trace_model_call(model)
+      model._set_inputs(inputs)
+
+    fn = saving_utils.trace_model_call(model)
+    signature_outputs = fn(inputs)
+    expected_outputs = {model.output_names[0]: model(inputs)}
+
+    self._assert_all_close(expected_outputs, signature_outputs)
+
+  @keras_parameterized.run_with_all_model_types
+  @keras_parameterized.run_all_keras_modes
+  def test_trace_model_outputs_after_fitting(self):
+    input_dim = 5 if testing_utils.get_model_type() == 'functional' else None
+    model = testing_utils.get_small_mlp(10, 3, input_dim)
+    model.compile(optimizer='sgd', loss='mse')
+    model.fit(x=np.random.random((8, 5)),
+              y=np.random.random((8, 3)), epochs=2)
+
+    inputs = array_ops.ones((8, 5))
+
+    fn = saving_utils.trace_model_call(model)
+    signature_outputs = fn(inputs)
+    expected_outputs = {model.output_names[0]: model(inputs)}
+
+    self._assert_all_close(expected_outputs, signature_outputs)
+
+  @keras_parameterized.run_with_all_model_types(exclude_models='sequential')
+  @keras_parameterized.run_all_keras_modes
+  def test_trace_multi_io_model_outputs(self):
+    input_dim = 5
+    num_classes = 3
+    num_classes_b = 4
+    input_a = keras.layers.Input(shape=(input_dim,), name='input_a')
+    input_b = keras.layers.Input(shape=(input_dim,), name='input_b')
+
+    dense = keras.layers.Dense(num_classes, name='dense')
+    dense2 = keras.layers.Dense(num_classes_b, name='dense2')
+    dropout = keras.layers.Dropout(0.5, name='dropout')
+    branch_a = [input_a, dense]
+    branch_b = [input_b, dense, dense2, dropout]
+
+    model = testing_utils.get_multi_io_model(branch_a, branch_b)
+
+    input_a_np = np.random.random((10, input_dim)).astype(np.float32)
+    input_b_np = np.random.random((10, input_dim)).astype(np.float32)
+
+    if testing_utils.get_model_type() == 'subclass':
+      with self.assertRaisesRegexp(ValueError,
+                                   'input shapes have not been set'):
+        saving_utils.trace_model_call(model)
+
+    model.compile(optimizer='sgd', loss='mse')
+    model.fit(x=[np.random.random((8, input_dim)).astype(np.float32),
+                 np.random.random((8, input_dim)).astype(np.float32)],
+              y=[np.random.random((8, num_classes)).astype(np.float32),
+                 np.random.random((8, num_classes_b)).astype(np.float32)],
+              epochs=2)
+
+    fn = saving_utils.trace_model_call(model)
+    signature_outputs = fn([input_a_np, input_b_np])
+    outputs = model([input_a_np, input_b_np])
+    expected_outputs = {model.output_names[0]: outputs[0],
+                        model.output_names[1]: outputs[1]}
+
+    self._assert_all_close(expected_outputs, signature_outputs)
+
+  @keras_parameterized.run_all_keras_modes
+  def test_specify_input_signature(self):
+    model = testing_utils.get_small_sequential_mlp(10, 3, None)
+    inputs = array_ops.ones((8, 5))
+
+    with self.assertRaisesRegexp(ValueError, 'input shapes have not been set'):
+      saving_utils.trace_model_call(model)
+
+    fn = saving_utils.trace_model_call(
+        model, [tensor_spec.TensorSpec(shape=[None, 5], dtype=dtypes.float32)])
+    signature_outputs = fn(inputs)
+    expected_outputs = {model.output_names[0]: model(inputs)}
+    self._assert_all_close(expected_outputs, signature_outputs)
+
+  @keras_parameterized.run_all_keras_modes
+  def test_subclassed_model_with_input_signature(self):
+
+    class Model(keras.Model):
+
+      def __init__(self):
+        super(Model, self).__init__()
+        self.dense = keras.layers.Dense(3, name='dense')
+
+      @def_function.function(
+          input_signature=[[tensor_spec.TensorSpec([None, 5], dtypes.float32),
+                            tensor_spec.TensorSpec([None], dtypes.float32)]],)
+      def call(self, inputs, *args):
+        x, y = inputs
+        return self.dense(x) + y
+
+    model = Model()
+    fn = saving_utils.trace_model_call(model)
+    x = array_ops.ones((8, 5), dtype=dtypes.float32)
+    y = array_ops.ones((3,), dtype=dtypes.float32)
+    expected_outputs = {'output_1': model([x, y])}
+    signature_outputs = fn([x, y])
+    self._assert_all_close(expected_outputs, signature_outputs)
+
+
+def _import_and_infer(save_dir, inputs):
+  """Import a SavedModel into a TF 1.x-style graph and run `signature_key`."""
+  graph = ops.Graph()
+  with graph.as_default(), session_lib.Session() as session:
+    model = loader.load(session, [tag_constants.SERVING], save_dir)
+    signature = model.signature_def[
+        signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY]
+    assert set(inputs.keys()) == set(signature.inputs.keys())
+    feed_dict = {}
+    for arg_name in inputs.keys():
+      feed_dict[graph.get_tensor_by_name(signature.inputs[arg_name].name)] = (
+          inputs[arg_name])
+    output_dict = {}
+    for output_name, output_tensor_info in signature.outputs.items():
+      output_dict[output_name] = graph.get_tensor_by_name(
+          output_tensor_info.name)
+    return session.run(output_dict, feed_dict=feed_dict)
+
+
+class ModelSaveTest(keras_parameterized.TestCase):
+
+  @keras_parameterized.run_with_all_model_types
+  @keras_parameterized.run_all_keras_modes(always_skip_v1=True)
+  def test_model_save(self):
+    input_dim = 5
+    model = testing_utils.get_small_mlp(10, 3, input_dim)
+    inputs = array_ops.ones((8, 5))
+
+    if testing_utils.get_model_type() == 'subclass':
+      model._set_inputs(inputs)
+
+    save_dir = os.path.join(self.get_temp_dir(), 'saved_model')
+    save_lib.save(model, save_dir)
+
+    self.assertAllClose(
+        {model.output_names[0]: model.predict_on_batch(inputs)},
+        _import_and_infer(save_dir, {model.input_names[0]: np.ones((8, 5))}))
+
+if __name__ == '__main__':
+  test.main()
diff --git a/tensorflow/python/saved_model/BUILD b/tensorflow/python/saved_model/BUILD
index 8da6ff5142..8a54888582 100644
--- a/tensorflow/python/saved_model/BUILD
+++ b/tensorflow/python/saved_model/BUILD
@@ -71,7 +71,7 @@ py_library(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:lib",
         "//tensorflow/python:platform",
-        "//tensorflow/python:training",
+        "//tensorflow/python:saver",
         "//tensorflow/python:util",
         "//tensorflow/python:variables",
     ],
-- 
GitLab


From bcf94f9cf2a4cfe8d2b7a76a1fe1be553a591edd Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Thu, 10 Jan 2019 17:04:59 -0800
Subject: [PATCH 0562/1095] Minor refactor of
 _WhileBodyGradFuncGraph._capture_helper.

It was getting kind of long. This also makes minor stylistic changes to the moved code.

PiperOrigin-RevId: 228804655
---
 tensorflow/python/ops/while_v2.py | 61 +++++++++++++++++++------------
 1 file changed, 37 insertions(+), 24 deletions(-)

diff --git a/tensorflow/python/ops/while_v2.py b/tensorflow/python/ops/while_v2.py
index f5a51bb1bc..cefadfb098 100644
--- a/tensorflow/python/ops/while_v2.py
+++ b/tensorflow/python/ops/while_v2.py
@@ -724,31 +724,9 @@ class _WhileBodyGradFuncGraph(util.WhileBodyFuncGraph):
     if captured_tensor is not None:
       return captured_tensor
 
+    # Resource tensors are not accumulated and handled specially.
     if tensor.dtype == dtypes.resource:
-      # Resource-type tensors are not accumulated.
-      # If a resource tensor exists in the loop body it must either be a loop
-      # input or an output of a nested While op inside the loop body which
-      # had captured the external resource.
-      if tensor in self._forward_graph.inputs:
-        index = self._forward_graph.inputs.index(tensor)
-      elif tensor.op.type == "While":
-        # Captured resources occur at the same index in the lists of inputs and
-        # outputs of a while op. So we lookup the input of `tensor.op` at the
-        # same index as the index of `tensor` in the `tensor.op.outputs`.
-        index = self._forward_graph.inputs.index(
-            tensor.op.inputs[tensor.value_index])
-      else:
-        raise ValueError(
-            "Taking gradient of a while loop which creates"
-            " a resource in its body is not supported: %s" % str(tensor))
-      # This must be a loop invariant.
-      assert self._forward_graph.inputs[index] == self._forward_graph.outputs[
-          index], "Resource tensors must be loop invariants %s." % str(
-              self._forward_graph._while.inputs[index])
-      tensor_in_outer_graph = self._forward_graph._while.inputs[index]
-      self._indirect_captures[tensor] = self.capture(
-          tensor_in_outer_graph, whitelisted=True)
-      return self._indirect_captures[tensor]
+      return self._resource_capture_helper(tensor)
 
     # Create or find an existing accumulator output for `tensor` in the forward
     # graph, and fetch from this accumulator in the gradient graph to get the
@@ -789,6 +767,41 @@ class _WhileBodyGradFuncGraph(util.WhileBodyFuncGraph):
     self.popped_tensor_lists[captured_accumulator] = new_tensor_list
     return captured_tensor
 
+  def _resource_capture_helper(self, tensor):
+    """Returns the captured resource tensor.
+
+    Resource-type tensors are not accumulated. If a resource tensor exists in
+    the loop body it must either be a loop input or an output of a nested While
+    op inside the loop body which had captured the external resource.
+
+    Args:
+      tensor: the external resource Tensor to be captured.
+
+    Returns:
+      Tensor in this graph.
+    """
+    assert tensor.dtype == dtypes.resource
+    if tensor in self._forward_graph.inputs:
+      index = self._forward_graph.inputs.index(tensor)
+    elif tensor.op.type == "While":
+      # Captured resources occur at the same index in the lists of inputs and
+      # outputs of a while op. So we lookup the input of `tensor.op` at the
+      # same index as the index of `tensor` in the `tensor.op.outputs`.
+      index = self._forward_graph.inputs.index(
+          tensor.op.inputs[tensor.value_index])
+    else:
+      raise ValueError(
+          "Taking gradient of a while loop which creates "
+          "a resource in its body is not supported: %s" % tensor)
+    # This must be a loop invariant.
+    assert self._forward_graph.inputs[index] == self._forward_graph.outputs[
+        index], ("Resource tensors must be loop invariants %s." %
+                 self._forward_graph._while.inputs[index])
+    tensor_in_outer_graph = self._forward_graph._while.inputs[index]
+    self._indirect_captures[tensor] = self.capture(
+        tensor_in_outer_graph, whitelisted=True)
+    return self._indirect_captures[tensor]
+
 
 def _check_shapes_compat(output_tensors, shape_invariants, input_tensors):
   for (t, shape, input_t) in zip(output_tensors, shape_invariants,
-- 
GitLab


From 3fc2b09b601e9d2c3e1bdd80d299c31b3adb0c0d Mon Sep 17 00:00:00 2001
From: Tim Shen <timshen@google.com>
Date: Thu, 10 Jan 2019 17:19:40 -0800
Subject: [PATCH 0563/1095] Centralize the logic that infers accumulator type
 from element type.

PiperOrigin-RevId: 228806506
---
 tensorflow/stream_executor/cuda/cuda_dnn.cc | 132 ++++++++++----------
 1 file changed, 68 insertions(+), 64 deletions(-)

diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.cc b/tensorflow/stream_executor/cuda/cuda_dnn.cc
index 0bd953fe3b..60648075c0 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.cc
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.cc
@@ -2710,6 +2710,23 @@ cudnnDataType_t GetRnnComputeType(dnn::DataType data_type) {
   }
 }
 
+dnn::DataType GetConvAccumulatorType(dnn::DataType data_type) {
+  switch (data_type) {
+    case dnn::DataType::kFloat:
+    case dnn::DataType::kDouble:
+      return data_type;
+    case dnn::DataType::kHalf:
+      return CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()
+                 ? dnn::DataType::kFloat
+                 : dnn::DataType::kHalf;
+    case dnn::DataType::kInt8:
+    case dnn::DataType::kInt32:
+      return dnn::DataType::kInt32;
+    default:
+      LOG(FATAL) << "Invalid DNN data type: " << static_cast<int>(data_type);
+  }
+}
+
 // Determines whether we can safely perform a winograd non-fused convolution for
 // the given input and output shapes.  This works around b/68264959, an integer
 // overflow in cuDNNv5 and cuDNNv6.
@@ -3405,8 +3422,8 @@ bool CudnnSupport::DoConvolve(
   return IsStatusOk(
       DoConvolveImpl(stream, batch_descriptor, input_data, filter_descriptor,
                      filter_data, convolution_descriptor, output_descriptor,
-                     output_data, dnn::DataType::kFloat, algorithm_desc,
-                     scratch_memory, output_profile_result),
+                     output_data, GetConvAccumulatorType(dnn::DataType::kFloat),
+                     algorithm_desc, scratch_memory, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
@@ -3423,8 +3440,9 @@ bool CudnnSupport::DoConvolve(
   return IsStatusOk(
       DoConvolveImpl(stream, batch_descriptor, input_data, filter_descriptor,
                      filter_data, convolution_descriptor, output_descriptor,
-                     output_data, dnn::DataType::kDouble, algorithm_desc,
-                     scratch_memory, output_profile_result),
+                     output_data,
+                     GetConvAccumulatorType(dnn::DataType::kDouble),
+                     algorithm_desc, scratch_memory, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
@@ -3439,15 +3457,11 @@ bool CudnnSupport::DoConvolve(
     const dnn::AlgorithmDesc& algorithm_desc,
     DeviceMemory<uint8>* scratch_memory,
     dnn::ProfileResult* output_profile_result) {
-  dnn::DataType acc_type =
-      CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()
-          ? dnn::DataType::kFloat
-          : dnn::DataType::kHalf;
   return IsStatusOk(
       DoConvolveImpl(stream, batch_descriptor, input_data, filter_descriptor,
                      filter_data, convolution_descriptor, output_descriptor,
-                     output_data, acc_type, algorithm_desc, scratch_memory,
-                     output_profile_result),
+                     output_data, GetConvAccumulatorType(dnn::DataType::kHalf),
+                     algorithm_desc, scratch_memory, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
@@ -3465,13 +3479,13 @@ bool CudnnSupport::DoFusedConvolve(
     const dnn::AlgorithmConfig& algorithm_config,
     dnn::ProfileResult* output_profile_result) {
   return IsStatusOk(
-      DoFusedConvolveImpl(stream, conv_input_descriptor, conv_input_data,
-                          conv_input_scale, filter_descriptor, filter_data,
-                          convolution_descriptor, side_input_data,
-                          side_input_scale, bias_descriptor, biases,
-                          activation_mode, output_descriptor, output_data,
-                          dnn::DataType::kDouble, scratch_allocator,
-                          algorithm_config, output_profile_result),
+      DoFusedConvolveImpl(
+          stream, conv_input_descriptor, conv_input_data, conv_input_scale,
+          filter_descriptor, filter_data, convolution_descriptor,
+          side_input_data, side_input_scale, bias_descriptor, biases,
+          activation_mode, output_descriptor, output_data,
+          GetConvAccumulatorType(dnn::DataType::kDouble), scratch_allocator,
+          algorithm_config, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
@@ -3489,13 +3503,13 @@ bool CudnnSupport::DoFusedConvolve(
     const dnn::AlgorithmConfig& algorithm_config,
     dnn::ProfileResult* output_profile_result) {
   return IsStatusOk(
-      DoFusedConvolveImpl(stream, conv_input_descriptor, conv_input_data,
-                          conv_input_scale, filter_descriptor, filter_data,
-                          convolution_descriptor, side_input_data,
-                          side_input_scale, bias_descriptor, biases,
-                          activation_mode, output_descriptor, output_data,
-                          dnn::DataType::kFloat, scratch_allocator,
-                          algorithm_config, output_profile_result),
+      DoFusedConvolveImpl(
+          stream, conv_input_descriptor, conv_input_data, conv_input_scale,
+          filter_descriptor, filter_data, convolution_descriptor,
+          side_input_data, side_input_scale, bias_descriptor, biases,
+          activation_mode, output_descriptor, output_data,
+          GetConvAccumulatorType(dnn::DataType::kFloat), scratch_allocator,
+          algorithm_config, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
@@ -3513,17 +3527,14 @@ bool CudnnSupport::DoFusedConvolve(
     DeviceMemory<Eigen::half>* output_data, ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
     dnn::ProfileResult* output_profile_result) {
-  dnn::DataType acc_type =
-      CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()
-          ? dnn::DataType::kFloat
-          : dnn::DataType::kHalf;
   return IsStatusOk(
       DoFusedConvolveImpl(
           stream, conv_input_descriptor, conv_input_data, conv_input_scale,
           filter_descriptor, filter_data, convolution_descriptor,
           side_input_data, side_input_scale, bias_descriptor, biases,
-          activation_mode, output_descriptor, output_data, acc_type,
-          scratch_allocator, algorithm_config, output_profile_result),
+          activation_mode, output_descriptor, output_data,
+          GetConvAccumulatorType(dnn::DataType::kHalf), scratch_allocator,
+          algorithm_config, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
@@ -3549,13 +3560,13 @@ bool CudnnSupport::DoFusedConvolve(
     return false;
   }
   return IsStatusOk(
-      DoFusedConvolveImpl(stream, conv_input_descriptor, conv_input_data,
-                          conv_input_scale, filter_descriptor, filter_data,
-                          convolution_descriptor, side_input_data,
-                          side_input_scale, bias_descriptor, biases,
-                          activation_mode, output_descriptor, output_data,
-                          dnn::DataType::kInt32, scratch_allocator,
-                          algorithm_config, output_profile_result),
+      DoFusedConvolveImpl(
+          stream, conv_input_descriptor, conv_input_data, conv_input_scale,
+          filter_descriptor, filter_data, convolution_descriptor,
+          side_input_data, side_input_scale, bias_descriptor, biases,
+          activation_mode, output_descriptor, output_data,
+          GetConvAccumulatorType(dnn::DataType::kInt8), scratch_allocator,
+          algorithm_config, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
@@ -3790,8 +3801,8 @@ bool CudnnSupport::DoConvolveBackwardData(
       DoConvolveBackwardDataImpl(
           stream, filter_descriptor, filter_data, output_descriptor,
           backward_output_data, convolution_descriptor, input_descriptor,
-          backward_input_data, dnn::DataType::kDouble, algorithm_desc,
-          scratch_memory, output_profile_result),
+          backward_input_data, GetConvAccumulatorType(dnn::DataType::kDouble),
+          algorithm_desc, scratch_memory, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
@@ -3810,8 +3821,8 @@ bool CudnnSupport::DoConvolveBackwardData(
       DoConvolveBackwardDataImpl(
           stream, filter_descriptor, filter_data, output_descriptor,
           backward_output_data, convolution_descriptor, input_descriptor,
-          backward_input_data, dnn::DataType::kFloat, algorithm_desc,
-          scratch_memory, output_profile_result),
+          backward_input_data, GetConvAccumulatorType(dnn::DataType::kFloat),
+          algorithm_desc, scratch_memory, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
@@ -3826,16 +3837,12 @@ bool CudnnSupport::DoConvolveBackwardData(
     const dnn::AlgorithmDesc& algorithm_desc,
     DeviceMemory<uint8>* scratch_memory,
     dnn::ProfileResult* output_profile_result) {
-  dnn::DataType acc_type =
-      CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()
-          ? dnn::DataType::kFloat
-          : dnn::DataType::kHalf;
   return IsStatusOk(
-      DoConvolveBackwardDataImpl(stream, filter_descriptor, filter_data,
-                                 output_descriptor, backward_output_data,
-                                 convolution_descriptor, input_descriptor,
-                                 backward_input_data, acc_type, algorithm_desc,
-                                 scratch_memory, output_profile_result),
+      DoConvolveBackwardDataImpl(
+          stream, filter_descriptor, filter_data, output_descriptor,
+          backward_output_data, convolution_descriptor, input_descriptor,
+          backward_input_data, GetConvAccumulatorType(dnn::DataType::kHalf),
+          algorithm_desc, scratch_memory, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
@@ -4085,8 +4092,8 @@ bool CudnnSupport::DoConvolveBackwardFilter(
       DoConvolveBackwardFilterImpl(
           stream, input_descriptor, input_data, output_descriptor,
           backward_output_data, convolution_descriptor, filter_descriptor,
-          backward_filter_data, dnn::DataType::kDouble, algorithm_desc,
-          scratch_memory, output_profile_result),
+          backward_filter_data, GetConvAccumulatorType(dnn::DataType::kDouble),
+          algorithm_desc, scratch_memory, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
@@ -4105,8 +4112,8 @@ bool CudnnSupport::DoConvolveBackwardFilter(
       DoConvolveBackwardFilterImpl(
           stream, input_descriptor, input_data, output_descriptor,
           backward_output_data, convolution_descriptor, filter_descriptor,
-          backward_filter_data, dnn::DataType::kFloat, algorithm_desc,
-          scratch_memory, output_profile_result),
+          backward_filter_data, GetConvAccumulatorType(dnn::DataType::kFloat),
+          algorithm_desc, scratch_memory, output_profile_result),
       /*report_error=*/!output_profile_result);
 }
 
@@ -4121,16 +4128,13 @@ bool CudnnSupport::DoConvolveBackwardFilter(
     const dnn::AlgorithmDesc& algorithm_desc,
     DeviceMemory<uint8>* scratch_memory,
     dnn::ProfileResult* output_profile_result) {
-  dnn::DataType acc_type =
-      CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()
-          ? dnn::DataType::kFloat
-          : dnn::DataType::kHalf;
-  return IsStatusOk(DoConvolveBackwardFilterImpl(
-                        stream, input_descriptor, input_data, output_descriptor,
-                        backward_output_data, convolution_descriptor,
-                        filter_descriptor, backward_filter_data, acc_type,
-                        algorithm_desc, scratch_memory, output_profile_result),
-                    /*report_error=*/!output_profile_result);
+  return IsStatusOk(
+      DoConvolveBackwardFilterImpl(
+          stream, input_descriptor, input_data, output_descriptor,
+          backward_output_data, convolution_descriptor, filter_descriptor,
+          backward_filter_data, GetConvAccumulatorType(dnn::DataType::kHalf),
+          algorithm_desc, scratch_memory, output_profile_result),
+      /*report_error=*/!output_profile_result);
 }
 
 template <class T>
-- 
GitLab


From 578bd3a2765f32112ef69bf355b4e1221b1c723b Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Thu, 10 Jan 2019 17:33:58 -0800
Subject: [PATCH 0564/1095] Move clustering ops to core.

PiperOrigin-RevId: 228808275
---
 tensorflow/contrib/factorization/BUILD        | 24 +----
 .../contrib/factorization/kernels/BUILD       | 28 ------
 .../factorization/ops/clustering_ops.cc       | 91 -------------------
 .../python/ops/clustering_ops.py              | 15 +--
 tensorflow/core/BUILD                         |  3 +
 .../api_def_KMC2ChainInitialization.pbtxt     | 30 ++++++
 ...api_def_KmeansPlusPlusInitialization.pbtxt | 44 +++++++++
 .../base_api/api_def_NearestNeighbors.pbtxt   | 43 +++++++++
 .../api_def_KMC2ChainInitialization.pbtxt     |  4 +
 ...api_def_KmeansPlusPlusInitialization.pbtxt |  4 +
 .../python_api/api_def_NearestNeighbors.pbtxt |  4 +
 tensorflow/core/kernels/BUILD                 | 27 ++++++
 .../kernels/clustering_ops.cc                 | 10 +-
 .../kernels/clustering_ops_test.cc            |  0
 tensorflow/core/ops/clustering_ops.cc         | 43 +++++++++
 tensorflow/python/BUILD                       |  8 ++
 16 files changed, 221 insertions(+), 157 deletions(-)
 delete mode 100644 tensorflow/contrib/factorization/ops/clustering_ops.cc
 create mode 100644 tensorflow/core/api_def/base_api/api_def_KMC2ChainInitialization.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_KmeansPlusPlusInitialization.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_NearestNeighbors.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_KMC2ChainInitialization.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_KmeansPlusPlusInitialization.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_NearestNeighbors.pbtxt
 rename tensorflow/{contrib/factorization => core}/kernels/clustering_ops.cc (99%)
 rename tensorflow/{contrib/factorization => core}/kernels/clustering_ops_test.cc (100%)
 create mode 100644 tensorflow/core/ops/clustering_ops.cc

diff --git a/tensorflow/contrib/factorization/BUILD b/tensorflow/contrib/factorization/BUILD
index e344d7a23b..cb86efb8da 100644
--- a/tensorflow/contrib/factorization/BUILD
+++ b/tensorflow/contrib/factorization/BUILD
@@ -28,7 +28,6 @@ tf_custom_op_py_library(
         "python/ops/wals.py",
     ],
     dso = [
-        ":python/ops/_clustering_ops.so",
         ":python/ops/_factorization_ops.so",
     ],
     kernels = [
@@ -38,12 +37,12 @@ tf_custom_op_py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":factorization_ops_test_utils_py",
-        ":gen_clustering_ops",
         ":gen_factorization_ops",
         "//tensorflow/contrib/framework:framework_py",
         "//tensorflow/contrib/util:util_py",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:check_ops",
+        "//tensorflow/python:clustering_ops_gen",
         "//tensorflow/python:control_flow_ops",
         "//tensorflow/python:data_flow_ops",
         "//tensorflow/python:embedding_ops",
@@ -77,17 +76,6 @@ py_library(
     ],
 )
 
-# Ops
-tf_custom_op_library(
-    name = "python/ops/_clustering_ops.so",
-    srcs = [
-        "ops/clustering_ops.cc",
-    ],
-    deps = [
-        "//tensorflow/contrib/factorization/kernels:clustering_ops",
-    ],
-)
-
 tf_custom_op_library(
     name = "python/ops/_factorization_ops.so",
     srcs = [
@@ -100,26 +88,16 @@ tf_custom_op_library(
 )
 
 tf_gen_op_libs([
-    "clustering_ops",
     "factorization_ops",
 ])
 
 cc_library(
     name = "all_ops",
     deps = [
-        ":clustering_ops_op_lib",
         ":factorization_ops_op_lib",
     ],
 )
 
-tf_gen_op_wrapper_py(
-    name = "gen_clustering_ops",
-    out = "python/ops/gen_clustering_ops.py",
-    deps = [
-        ":clustering_ops_op_lib",
-    ],
-)
-
 tf_gen_op_wrapper_py(
     name = "gen_factorization_ops",
     out = "python/ops/gen_factorization_ops.py",
diff --git a/tensorflow/contrib/factorization/kernels/BUILD b/tensorflow/contrib/factorization/kernels/BUILD
index ea8b9a17a2..23d7e088d0 100644
--- a/tensorflow/contrib/factorization/kernels/BUILD
+++ b/tensorflow/contrib/factorization/kernels/BUILD
@@ -11,7 +11,6 @@ load("//tensorflow:tensorflow.bzl", "tf_cc_test")
 cc_library(
     name = "all_kernels",
     deps = [
-        ":clustering_ops",
         ":masked_matmul_ops",
         ":wals_solver_ops",
         "@protobuf_archive//:protobuf_headers",
@@ -29,17 +28,6 @@ cc_library(
     alwayslink = 1,
 )
 
-cc_library(
-    name = "clustering_ops",
-    srcs = ["clustering_ops.cc"],
-    deps = [
-        "//tensorflow/core:framework_headers_lib",
-        "//third_party/eigen3",
-        "@protobuf_archive//:protobuf_headers",
-    ],
-    alwayslink = 1,
-)
-
 cc_library(
     name = "masked_matmul_ops",
     srcs = ["masked_matmul_ops.cc"],
@@ -51,19 +39,3 @@ cc_library(
     ],
     alwayslink = 1,
 )
-
-tf_cc_test(
-    name = "clustering_ops_test",
-    srcs = ["clustering_ops_test.cc"],
-    deps = [
-        ":clustering_ops",
-        "//tensorflow/contrib/factorization:clustering_ops_op_lib",
-        "//tensorflow/core:core_cpu",
-        "//tensorflow/core:framework",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:protos_all_cc",
-        "//tensorflow/core:test",
-        "//tensorflow/core:test_main",
-        "//tensorflow/core:testlib",
-    ],
-)
diff --git a/tensorflow/contrib/factorization/ops/clustering_ops.cc b/tensorflow/contrib/factorization/ops/clustering_ops.cc
deleted file mode 100644
index 2686702c1d..0000000000
--- a/tensorflow/contrib/factorization/ops/clustering_ops.cc
+++ /dev/null
@@ -1,91 +0,0 @@
-// Copyright 2016 The TensorFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License"); you may not
-// use this file except in compliance with the License.  You may obtain a copy
-// of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
-// License for the specific language governing permissions and limitations under
-// the License.
-// ==============================================================================
-
-#include "tensorflow/core/framework/common_shape_fns.h"
-#include "tensorflow/core/framework/op.h"
-
-namespace tensorflow {
-
-REGISTER_OP("KmeansPlusPlusInitialization")
-    .Input("points: float32")
-    .Input("num_to_sample: int64")
-    .Input("seed: int64")
-    .Input("num_retries_per_sample: int64")
-    .Output("samples: float32")
-    .SetShapeFn(shape_inference::UnknownShape)
-    .Doc(R"(
-Selects num_to_sample rows of input using the KMeans++ criterion.
-
-Rows of points are assumed to be input points. One row is selected at random.
-Subsequent rows are sampled with probability proportional to the squared L2
-distance from the nearest row selected thus far till num_to_sample rows have
-been sampled.
-
-points: Matrix of shape (n, d). Rows are assumed to be input points.
-num_to_sample: Scalar. The number of rows to sample. This value must not be
-  larger than n.
-seed: Scalar. Seed for initializing the random number generator.
-num_retries_per_sample: Scalar. For each row that is sampled, this parameter
-  specifies the number of additional points to draw from the current
-  distribution before selecting the best. If a negative value is specified, a
-  heuristic is used to sample O(log(num_to_sample)) additional points.
-samples: Matrix of shape (num_to_sample, d). The sampled rows.
-)");
-
-REGISTER_OP("KMC2ChainInitialization")
-    .Input("distances: float32")
-    .Input("seed: int64")
-    .Output("index: int64")
-    .SetShapeFn(shape_inference::ScalarShape)
-    .Doc(R"(
-Returns the index of a data point that should be added to the seed set.
-
-Entries in distances are assumed to be squared distances of candidate points to
-the already sampled centers in the seed set. The op constructs one Markov chain
-of the k-MC^2 algorithm and returns the index of one candidate point to be added
-as an additional cluster center.
-
-distances: Vector with squared distances to the closest previously sampled
-  cluster center for each candidate point.
-seed: Scalar. Seed for initializing the random number generator.
-index: Scalar with the index of the sampled point.
-)");
-
-REGISTER_OP("NearestNeighbors")
-    .Input("points: float32")
-    .Input("centers: float32")
-    .Input("k: int64")
-    .Output("nearest_center_indices: int64")
-    .Output("nearest_center_distances: float32")
-    .SetShapeFn(shape_inference::UnknownShape)
-    .Doc(R"(
-Selects the k nearest centers for each point.
-
-Rows of points are assumed to be input points. Rows of centers are assumed to be
-the list of candidate centers. For each point, the k centers that have least L2
-distance to it are computed.
-
-points: Matrix of shape (n, d). Rows are assumed to be input points.
-centers: Matrix of shape (m, d). Rows are assumed to be centers.
-k: Scalar. Number of nearest centers to return for each point. If k is larger
-  than m, then only m centers are returned.
-nearest_center_indices: Matrix of shape (n, min(m, k)). Each row contains the
-  indices of the centers closest to the corresponding point, ordered by
-  increasing distance.
-nearest_center_distances: Matrix of shape (n, min(m, k)). Each row contains the
-  squared L2 distance to the corresponding center in nearest_center_indices.
-)");
-
-}  // namespace tensorflow
diff --git a/tensorflow/contrib/factorization/python/ops/clustering_ops.py b/tensorflow/contrib/factorization/python/ops/clustering_ops.py
index 84e80791f4..d48b89cbac 100644
--- a/tensorflow/contrib/factorization/python/ops/clustering_ops.py
+++ b/tensorflow/contrib/factorization/python/ops/clustering_ops.py
@@ -18,28 +18,23 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.factorization.python.ops import gen_clustering_ops
-# go/tf-wildcard-import
-# pylint: disable=wildcard-import
-from tensorflow.contrib.factorization.python.ops.gen_clustering_ops import *
-# pylint: enable=wildcard-import
-from tensorflow.contrib.util import loader
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import check_ops
 from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import gen_clustering_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn_impl
 from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops.embedding_ops import embedding_lookup
-from tensorflow.python.platform import resource_loader
-
-_clustering_ops = loader.load_op_library(
-    resource_loader.get_path_to_datafile('_clustering_ops.so'))
+# go/tf-wildcard-import
+# pylint: disable=wildcard-import
+from tensorflow.python.ops.gen_clustering_ops import *
+# pylint: enable=wildcard-import
 
 # Euclidean distance between vectors U and V is defined as \\(||U - V||_F\\)
 # which is the square root of the sum of the absolute squares of the elements
diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 690fa4a19e..20f5815423 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -1074,6 +1074,7 @@ tf_gen_op_libs(
         "tensor_forest_ops",
         "candidate_sampling_ops",
         "checkpoint_ops",
+        "clustering_ops",
         "collective_ops",
         "control_flow_ops",
         "ctc_ops",
@@ -1228,6 +1229,7 @@ cc_library(
         ":tensor_forest_ops_op_lib",
         ":candidate_sampling_ops_op_lib",
         ":checkpoint_ops_op_lib",
+        ":clustering_ops_op_lib",
         ":collective_ops_op_lib",
         ":control_flow_ops_op_lib",
         ":ctc_ops_op_lib",
@@ -1382,6 +1384,7 @@ cc_library(
         "//tensorflow/core/kernels:tensor_forest_ops",
         "//tensorflow/core/kernels:candidate_sampler_ops",
         "//tensorflow/core/kernels:checkpoint_ops",
+        "//tensorflow/core/kernels:clustering_ops",
         "//tensorflow/core/kernels:collective_ops",
         "//tensorflow/core/kernels:control_flow_ops",
         "//tensorflow/core/kernels:ctc_ops",
diff --git a/tensorflow/core/api_def/base_api/api_def_KMC2ChainInitialization.pbtxt b/tensorflow/core/api_def/base_api/api_def_KMC2ChainInitialization.pbtxt
new file mode 100644
index 0000000000..c6ff4b9e2d
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_KMC2ChainInitialization.pbtxt
@@ -0,0 +1,30 @@
+op {
+  graph_op_name: "KMC2ChainInitialization"
+  visibility: HIDDEN
+  in_arg {
+    name: "distances"
+    description: <<END
+Vector with squared distances to the closest previously sampled cluster center
+for each candidate point.
+END
+  }
+  in_arg {
+    name: "seed"
+    description: <<END
+Scalar. Seed for initializing the random number generator.
+END
+  }
+  out_arg {
+    name: "index"
+    description: <<END
+Scalar with the index of the sampled point.
+END
+  }
+  summary: "Returns the index of a data point that should be added to the seed set."
+  description: <<END
+Entries in distances are assumed to be squared distances of candidate points to
+the already sampled centers in the seed set. The op constructs one Markov chain
+of the k-MC^2 algorithm and returns the index of one candidate point to be added
+as an additional cluster center.
+END
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_KmeansPlusPlusInitialization.pbtxt b/tensorflow/core/api_def/base_api/api_def_KmeansPlusPlusInitialization.pbtxt
new file mode 100644
index 0000000000..461820486b
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_KmeansPlusPlusInitialization.pbtxt
@@ -0,0 +1,44 @@
+op {
+  graph_op_name: "KmeansPlusPlusInitialization"
+  visibility: HIDDEN
+  in_arg {
+    name: "points"
+    description: <<END
+Matrix of shape (n, d). Rows are assumed to be input points.
+END
+  }
+  in_arg {
+    name: "num_to_sample"
+    description: <<END
+Scalar. The number of rows to sample. This value must not be larger than n.
+END
+  }
+  in_arg {
+    name: "seed"
+    description: <<END
+Scalar. Seed for initializing the random number generator.
+END
+  }
+  in_arg {
+    name: "num_retries_per_sample"
+    description: <<END
+Scalar. For each row that is sampled, this parameter
+specifies the number of additional points to draw from the current
+distribution before selecting the best. If a negative value is specified, a
+heuristic is used to sample O(log(num_to_sample)) additional points.
+END
+  }
+  out_arg {
+    name: "samples"
+    description: <<END
+Matrix of shape (num_to_sample, d). The sampled rows.
+END
+  }
+  summary: "Selects num_to_sample rows of input using the KMeans++ criterion."
+  description: <<END
+Rows of points are assumed to be input points. One row is selected at random.
+Subsequent rows are sampled with probability proportional to the squared L2
+distance from the nearest row selected thus far till num_to_sample rows have
+been sampled.
+END
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_NearestNeighbors.pbtxt b/tensorflow/core/api_def/base_api/api_def_NearestNeighbors.pbtxt
new file mode 100644
index 0000000000..2bdf68fb9f
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_NearestNeighbors.pbtxt
@@ -0,0 +1,43 @@
+op {
+  graph_op_name: "NearestNeighbors"
+  visibility: HIDDEN
+  in_arg {
+    name: "points"
+    description: <<END
+Matrix of shape (n, d). Rows are assumed to be input points.
+END
+  }
+  in_arg {
+    name: "centers"
+    description: <<END
+Matrix of shape (m, d). Rows are assumed to be centers.
+END
+  }
+  in_arg {
+    name: "k"
+    description: <<END
+Number of nearest centers to return for each point. If k is larger than m, then
+only m centers are returned.
+END
+  }
+  out_arg {
+    name: "nearest_center_indices"
+    description: <<END
+Matrix of shape (n, min(m, k)). Each row contains the indices of the centers
+closest to the corresponding point, ordered by increasing distance.
+END
+  }
+  out_arg {
+    name: "nearest_center_distances"
+    description: <<END
+Matrix of shape (n, min(m, k)). Each row contains the squared L2 distance to the
+corresponding center in nearest_center_indices.
+END
+  }
+  summary: "Selects the k nearest centers for each point."
+  description: <<END
+Rows of points are assumed to be input points. Rows of centers are assumed to be
+the list of candidate centers. For each point, the k centers that have least L2
+distance to it are computed.
+END
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_KMC2ChainInitialization.pbtxt b/tensorflow/core/api_def/python_api/api_def_KMC2ChainInitialization.pbtxt
new file mode 100644
index 0000000000..3e8ec113f8
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_KMC2ChainInitialization.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "KMC2ChainInitialization"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_KmeansPlusPlusInitialization.pbtxt b/tensorflow/core/api_def/python_api/api_def_KmeansPlusPlusInitialization.pbtxt
new file mode 100644
index 0000000000..83aa3a3a69
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_KmeansPlusPlusInitialization.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "KmeansPlusPlusInitialization"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_NearestNeighbors.pbtxt b/tensorflow/core/api_def/python_api/api_def_NearestNeighbors.pbtxt
new file mode 100644
index 0000000000..ad05bb3486
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_NearestNeighbors.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "NearestNeighbors"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 0045c5dd19..4db0505a8f 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -152,6 +152,33 @@ tf_kernel_library(
     ],
 )
 
+tf_kernel_library(
+    name = "clustering_ops",
+    prefix = "clustering_ops",
+    deps = [
+        "//tensorflow/core:clustering_ops_op_lib",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:framework_headers_lib",
+        "//tensorflow/core:lib",
+    ],
+)
+
+tf_cc_test(
+    name = "clustering_ops_test",
+    srcs = ["clustering_ops_test.cc"],
+    deps = [
+        ":clustering_ops",
+        "//tensorflow/core:clustering_ops_op_lib",
+        "//tensorflow/core:core_cpu",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
+    ],
+)
+
 tf_kernel_library(
     name = "collective_ops",
     prefix = "collective_ops",
diff --git a/tensorflow/contrib/factorization/kernels/clustering_ops.cc b/tensorflow/core/kernels/clustering_ops.cc
similarity index 99%
rename from tensorflow/contrib/factorization/kernels/clustering_ops.cc
rename to tensorflow/core/kernels/clustering_ops.cc
index 025534d540..7e1a1fdcd2 100644
--- a/tensorflow/contrib/factorization/kernels/clustering_ops.cc
+++ b/tensorflow/core/kernels/clustering_ops.cc
@@ -392,7 +392,7 @@ class NearestNeighborsOp : public OpKernel {
       for (; start < limit; ++start) {
         const int64 start_row = num_points * start / num_units;
         const int64 limit_row = num_points * (start + 1) / num_units;
-        CHECK_LE(limit_row, num_points);
+        DCHECK_LE(limit_row, num_points);
         const int64 num_rows = limit_row - start_row;
         auto points_shard = points.middleRows(start_row, num_rows);
         const Eigen::VectorXf points_half_squared_norm =
@@ -430,7 +430,7 @@ class NearestNeighborsOp : public OpKernel {
       const Eigen::Ref<const Eigen::VectorXf>& centers_half_squared_norm,
       const Eigen::Ref<MatrixXi64RowMajor>& nearest_center_indices,
       const Eigen::Ref<MatrixXfRowMajor>& nearest_center_distances) {
-    CHECK_LE(k, centers.rows());
+    DCHECK_LE(k, centers.rows());
     if (centers.rows() <= kNearestNeighborsCentersMaxBlockSize) {
       FindKNearestCentersOneBlock(k, points, points_half_squared_norm, centers,
                                   centers_half_squared_norm,
@@ -451,7 +451,7 @@ class NearestNeighborsOp : public OpKernel {
       const Eigen::Ref<const Eigen::VectorXf>& centers_half_squared_norm,
       Eigen::Ref<MatrixXi64RowMajor> nearest_center_indices,
       Eigen::Ref<MatrixXfRowMajor> nearest_center_distances) {
-    CHECK_LE(k, centers.rows());
+    DCHECK_LE(k, centers.rows());
     const int64 num_points = points.rows();
     const MatrixXfRowMajor inner_product = points * centers.transpose();
     // Find nearest neighbors.
@@ -500,8 +500,8 @@ class NearestNeighborsOp : public OpKernel {
       Eigen::Ref<MatrixXfRowMajor> nearest_center_distances) {
     const int64 num_points = points.rows();
     const int64 num_centers = centers.rows();
-    CHECK_LE(k, num_centers);
-    CHECK_GT(num_centers, kNearestNeighborsCentersMaxBlockSize);
+    DCHECK_LE(k, num_centers);
+    DCHECK_GT(num_centers, kNearestNeighborsCentersMaxBlockSize);
     // Store nearest neighbors with first block of centers directly into the
     // output matrices.
     int64 out_k = std::min(k, kNearestNeighborsCentersMaxBlockSize);
diff --git a/tensorflow/contrib/factorization/kernels/clustering_ops_test.cc b/tensorflow/core/kernels/clustering_ops_test.cc
similarity index 100%
rename from tensorflow/contrib/factorization/kernels/clustering_ops_test.cc
rename to tensorflow/core/kernels/clustering_ops_test.cc
diff --git a/tensorflow/core/ops/clustering_ops.cc b/tensorflow/core/ops/clustering_ops.cc
new file mode 100644
index 0000000000..189f00730b
--- /dev/null
+++ b/tensorflow/core/ops/clustering_ops.cc
@@ -0,0 +1,43 @@
+// Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License.  You may obtain a copy
+// of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+// License for the specific language governing permissions and limitations under
+// the License.
+// ==============================================================================
+
+#include "tensorflow/core/framework/common_shape_fns.h"
+#include "tensorflow/core/framework/op.h"
+
+namespace tensorflow {
+
+REGISTER_OP("KmeansPlusPlusInitialization")
+    .Input("points: float32")
+    .Input("num_to_sample: int64")
+    .Input("seed: int64")
+    .Input("num_retries_per_sample: int64")
+    .Output("samples: float32")
+    .SetShapeFn(shape_inference::UnknownShape);
+
+REGISTER_OP("KMC2ChainInitialization")
+    .Input("distances: float32")
+    .Input("seed: int64")
+    .Output("index: int64")
+    .SetShapeFn(shape_inference::ScalarShape);
+
+REGISTER_OP("NearestNeighbors")
+    .Input("points: float32")
+    .Input("centers: float32")
+    .Input("k: int64")
+    .Output("nearest_center_indices: int64")
+    .Output("nearest_center_distances: float32")
+    .SetShapeFn(shape_inference::UnknownShape);
+
+}  // namespace tensorflow
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 82c542f51d..659f0735cb 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -1680,6 +1680,14 @@ tf_gen_op_wrapper_private_py(
     ],
 )
 
+tf_gen_op_wrapper_private_py(
+    name = "clustering_ops_gen",
+    visibility = ["//tensorflow:internal"],
+    deps = [
+        "//tensorflow/core:clustering_ops_op_lib",
+    ],
+)
+
 tf_gen_op_wrapper_private_py(
     name = "collective_ops_gen",
     visibility = ["//tensorflow:internal"],
-- 
GitLab


From bde6a28b2c8df473aa674aa6b9ee3ba4aa43db2f Mon Sep 17 00:00:00 2001
From: Samantha Andow <samdow@fb.com>
Date: Thu, 10 Jan 2019 18:04:53 -0800
Subject: [PATCH 0565/1095] Added unit test for InputList fix

@Co-authored-by: irenedea<irenedea@fb.com>
@Co-authored-by: melissagrueter<melissagrueter@fb.com>
---
 .../op/core/GeneratedOperationsTest.java        | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/tensorflow/java/src/test/java/org/tensorflow/op/core/GeneratedOperationsTest.java b/tensorflow/java/src/test/java/org/tensorflow/op/core/GeneratedOperationsTest.java
index 42d126c3c4..49c4ff639e 100644
--- a/tensorflow/java/src/test/java/org/tensorflow/op/core/GeneratedOperationsTest.java
+++ b/tensorflow/java/src/test/java/org/tensorflow/op/core/GeneratedOperationsTest.java
@@ -17,6 +17,7 @@ package org.tensorflow.op.core;
 
 import static org.junit.Assert.assertEquals;
 
+import java.util.ArrayList;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
@@ -40,4 +41,20 @@ public final class GeneratedOperationsTest {
       }
     }
   }
+
+  @Test
+  public void testListInputTensorOutput() {
+    try (Graph g = new Graph();
+        Session sess = new Session(g)) {
+      Ops ops = Ops.create(g);
+      ArrayList<Operand<Integer>> inputs = new ArrayList<Operand<Integer>>();
+      inputs.add(ops.constant(1));
+      inputs.add(ops.constant(2));
+      inputs.add(ops.constant(3));
+      Operand<Integer> x = ops.math().addN(inputs);
+      try (Tensor<Integer> result = sess.runner().fetch(x).run().get(0).expect(Integer.class)) {
+        assertEquals(6, result.intValue());
+      }
+    }
+  }
 }
-- 
GitLab


From b0571c92bd5893ab2ceadd00be2b928821b9d490 Mon Sep 17 00:00:00 2001
From: Davide Libenzi <dlibenzi@google.com>
Date: Thu, 10 Jan 2019 18:01:40 -0800
Subject: [PATCH 0566/1095] Automated rollback of commit
 55fda7fb66f1774ac5271e523259faa7d2a44d0c

PiperOrigin-RevId: 228811469
---
 .../xla/service/cpu/cpu_executable.cc         |  29 +++-
 .../xla/service/gpu/gpu_executable.cc         |  26 +++-
 tensorflow/compiler/xrt/kernels/BUILD         |   1 +
 .../compiler/xrt/kernels/xrt_execute_op.cc    |  20 +++
 tensorflow/compiler/xrt/tests/raw_api_test.cc | 127 +++++++++++++++++-
 tensorflow/compiler/xrt/xrt_state.cc          |  43 +++++-
 tensorflow/compiler/xrt/xrt_state.h           |  23 +++-
 7 files changed, 252 insertions(+), 17 deletions(-)

diff --git a/tensorflow/compiler/xla/service/cpu/cpu_executable.cc b/tensorflow/compiler/xla/service/cpu/cpu_executable.cc
index 412c2715b9..23d0af3423 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_executable.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_executable.cc
@@ -213,6 +213,8 @@ StatusOr<ScopedShapedBuffer> CpuExecutable::CreateResultShapedBuffer(
       /*on_host_shape=*/result_shape(),
       /*on_device_shape=*/result_shape(), run_options->allocator(),
       stream->parent()->device_ordinal());
+  const HloInputOutputAliasConfig& input_output_alias =
+      module().input_output_alias_config();
 
   // Move OwningDeviceMemory values which contain the array(s) of the result
   // into the respective location in ScopedShapedBuffer which is returned to the
@@ -232,12 +234,31 @@ StatusOr<ScopedShapedBuffer> CpuExecutable::CreateResultShapedBuffer(
         TF_ASSIGN_OR_RETURN(
             const BufferAllocation::Slice slice,
             this->assignment_->GetUniqueSlice(src, buffer_source->index()));
-        CHECK(!slice.allocation()->is_entry_computation_parameter());
-
         const BufferAllocation::Index buffer_index = slice.index();
         OwningDeviceMemory& buffer = buffers[buffer_index];
-        CHECK(!buffer.is_null() || buffer.size() == 0);
-        *device_memory = buffer.Forget();
+        if (!slice.allocation()->is_entry_computation_parameter()) {
+          // If the buffer coming out of the result is from a parameter, the
+          // owning buffer will be null, and that means the caller aliased some
+          // parameter buffer to an output one (via the
+          // HloInputOutputAliasConfig API). If that is the case, the caller
+          // will receive a partially complete scoped shaped buffer, which they
+          // will have to fill up on return. Unfortunately the interface to the
+          // execute APIs are ShapedBuffer pointer based, which assumes caller
+          // ownership, and hence a buffer coming from there cannot be part of
+          // the new ScopedShapedBuffer we create for the result (which assumes
+          // ownership).
+          *device_memory = buffer.Forget();
+        } else {
+          auto output_alias = input_output_alias.GetAliasedOutput(
+              slice.allocation()->parameter_number(),
+              slice.allocation()->param_shape_index());
+          CHECK(output_alias)
+              << "Ouput buffer is coming from parameter "
+              << slice.allocation()->parameter_number() << " at index "
+              << slice.allocation()->param_shape_index()
+              << ", but no alias exists";
+          CHECK_EQ(*output_alias, index);
+        }
         return Status::OK();
       }));
   return std::move(result_buffer);
diff --git a/tensorflow/compiler/xla/service/gpu/gpu_executable.cc b/tensorflow/compiler/xla/service/gpu/gpu_executable.cc
index 128cecd765..434060ad89 100644
--- a/tensorflow/compiler/xla/service/gpu/gpu_executable.cc
+++ b/tensorflow/compiler/xla/service/gpu/gpu_executable.cc
@@ -310,12 +310,34 @@ StatusOr<ScopedShapedBuffer> GpuExecutable::ExecuteOnStream(
         TF_ASSIGN_OR_RETURN(
             const BufferAllocation::Slice slice,
             this->assignment_->GetUniqueSlice(src_hlo, sources[0]->index()));
-        CHECK(!slice.allocation()->is_entry_computation_parameter());
 
         se::DeviceMemoryBase src_base =
             buffer_allocations->GetDeviceAddress(slice.index());
         CHECK(!src_base.is_null() || src_base.size() == 0);
-        *device_memory = src_base;
+        if (!slice.allocation()->is_entry_computation_parameter()) {
+          // If the buffer coming out of the result is from a parameter, it
+          // means the caller aliased some parameter buffer to an output one
+          // (via the HloInputOutputAliasConfig API). If that is the case, the
+          // caller will receive a partially complete scoped shaped buffer,
+          // which they will have to fill up on return.
+          // Unfortunately the interface to the execute APIs are ShapedBuffer
+          // pointer based, which assumes caller ownership, and hence a buffer
+          // coming from there cannot be part of the new ScopedShapedBuffer we
+          // create for the result (which assumes ownership).
+          *device_memory = src_base;
+        } else {
+          const HloInputOutputAliasConfig& input_output_alias =
+              module().input_output_alias_config();
+          auto output_alias = input_output_alias.GetAliasedOutput(
+              slice.allocation()->parameter_number(),
+              slice.allocation()->param_shape_index());
+          CHECK(output_alias)
+              << "Ouput buffer is coming from parameter "
+              << slice.allocation()->parameter_number() << " at index "
+              << slice.allocation()->param_shape_index()
+              << ", but no alias exists";
+          CHECK_EQ(*output_alias, index);
+        }
         buffers_in_result.insert(src_base);
         return Status::OK();
       }));
diff --git a/tensorflow/compiler/xrt/kernels/BUILD b/tensorflow/compiler/xrt/kernels/BUILD
index 78d093ec4a..d38f41ab43 100644
--- a/tensorflow/compiler/xrt/kernels/BUILD
+++ b/tensorflow/compiler/xrt/kernels/BUILD
@@ -55,6 +55,7 @@ cc_library(
         "//tensorflow/compiler/xla/client:xla_computation",
         "//tensorflow/compiler/xla/service:compiler",
         "//tensorflow/compiler/xla/service:computation_placer",
+        "//tensorflow/compiler/xla/service:hlo",
         "//tensorflow/compiler/xrt:xrt_proto",
         "//tensorflow/compiler/xrt:xrt_utils",
         "//tensorflow/core:core_cpu_internal",
diff --git a/tensorflow/compiler/xrt/kernels/xrt_execute_op.cc b/tensorflow/compiler/xrt/kernels/xrt_execute_op.cc
index 7f0ac123a5..116c193cab 100644
--- a/tensorflow/compiler/xrt/kernels/xrt_execute_op.cc
+++ b/tensorflow/compiler/xrt/kernels/xrt_execute_op.cc
@@ -19,6 +19,7 @@ limitations under the License.
 
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/service/computation_placer.h"
+#include "tensorflow/compiler/xla/service/hlo_input_output_alias_config.h"
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/status_macros.h"
 #include "tensorflow/compiler/xla/statusor.h"
@@ -228,6 +229,25 @@ Status XRTExecuteOp::DoWork(OpKernelContext* context) {
   TF_RETURN_IF_ERROR(XRTTupleAllocation::CreateFromBuffer(
       shaped_buffer, device_ref.backend(), device_ref.device_ordinal(),
       &output_tuple));
+
+  // The ScopedShapedBuffer returned by the executable Run() API, in case of
+  // input/output buffer aliasing, might have holes in it, which need to be
+  // filled using the proper input tuples buffers which are the source of
+  // aliasing.
+  const xla::HloInputOutputAliasConfig& input_output_alias =
+      executable->executable()->module().input_output_alias_config();
+  auto alias_function =
+      [&](const xla::ShapeIndex& output_index,
+          const xla::HloInputOutputAliasConfig::Alias& alias) -> Status {
+    TF_RET_CHECK(alias.parameter_number < input_tuples.size());
+    return alias.kind == xla::HloInputOutputAliasConfig::AliasKind::kUserAlias
+               ? output_tuple->AliasBufferFrom(
+                     *input_tuples[alias.parameter_number],
+                     alias.parameter_index, output_index)
+               : Status::OK();
+  };
+  TF_RETURN_IF_ERROR(input_output_alias.ForEachAliasWithStatus(alias_function));
+
   if (config_proto.return_exploded_tuple() &&
       output_tuple->on_device_shape().IsTuple()) {
     int64 tuple_element_count =
diff --git a/tensorflow/compiler/xrt/tests/raw_api_test.cc b/tensorflow/compiler/xrt/tests/raw_api_test.cc
index be0c4b9392..8e70cc3959 100644
--- a/tensorflow/compiler/xrt/tests/raw_api_test.cc
+++ b/tensorflow/compiler/xrt/tests/raw_api_test.cc
@@ -96,14 +96,21 @@ xla::LiteralProto FloatMatrix(
   return array.ToProto();
 }
 
+xla::Literal ReadOutputLiteral(const std::vector<Tensor>& outputs, size_t idx) {
+  xla::LiteralProto response;
+  CHECK(response.ParseFromString(outputs[idx].scalar<string>()()));
+  return xla::Literal::CreateFromProto(response).ValueOrDie();
+}
+
 bool CompareLiteralProtos(const xla::LiteralProto& a,
                           const xla::LiteralProto& b) {
   auto l_a = xla::Literal::CreateFromProto(a).ValueOrDie();
   auto l_b = xla::Literal::CreateFromProto(b).ValueOrDie();
   bool equal = l_a == l_b;
   if (!equal) {
-    LOG(INFO) << "LiteralProtos don't match: " << a.DebugString()
-              << " != " << b.DebugString();
+    LOG(INFO) << "LiteralProtos don't match:\n"
+              << a.DebugString() << "\n!=\n"
+              << b.DebugString();
   }
   return equal;
 }
@@ -113,8 +120,19 @@ bool CompareLiteralToLiteralProto(const xla::Literal& a,
   auto l_b = xla::Literal::CreateFromProto(b).ValueOrDie();
   bool equal = a == l_b;
   if (!equal) {
-    LOG(INFO) << "Literal and LiteralProto don't match "
-              << a.ToProto().DebugString() << " != " << b.DebugString();
+    LOG(INFO) << "Literal and LiteralProto don't match:\n"
+              << a.ToProto().DebugString() << "\n!=\n"
+              << b.DebugString();
+  }
+  return equal;
+}
+
+bool CompareLiterals(const xla::Literal& a, const xla::Literal& b) {
+  bool equal = a == b;
+  if (!equal) {
+    LOG(INFO) << "Literals don't match:\n"
+              << a.ToProto().DebugString() << "\n!=\n"
+              << b.ToProto().DebugString();
   }
   return equal;
 }
@@ -939,6 +957,107 @@ TEST(RawApiTest, LeakCompilationReference) {
   TF_EXPECT_OK(session.Run({c_handle.handle}, &outputs));
 }
 
+TEST(RawApiTest, CompileAndExecuteWithReusedBuffers) {
+  xla::Shape element_shape = xla::ShapeUtil::MakeShape(xla::F32, {2});
+  xla::Shape shape =
+      xla::ShapeUtil::MakeTupleShape({element_shape, element_shape});
+  xla::Shape return_shape = xla::ShapeUtil::MakeTupleShape(
+      {element_shape, element_shape, element_shape, element_shape});
+  xla::XlaBuilder builder("ReuseBuffer");
+  auto param = xla::Parameter(&builder, 0, shape, "param");
+  auto p0 = xla::GetTupleElement(param, 0);
+  auto p1 = xla::GetTupleElement(param, 1);
+  auto add = xla::Add(p0, p1);
+  auto sub = xla::Sub(p0, p1);
+  xla::Tuple(&builder, {add, sub, p0, p1});
+
+  // Flip the tuple literals in the input handle.
+  builder.SetUpAlias({1}, 0, {0});
+  builder.SetUpAlias({0}, 0, {1});
+
+  auto computation = builder.Build().ValueOrDie();
+
+  auto literal0 = xla::LiteralUtil::CreateR1<float>({1.0f, 2.0f});
+  auto literal1 = xla::LiteralUtil::CreateR1<float>({5.0f, 9.0f});
+  auto literal = xla::LiteralUtil::MakeTuple({&literal0, &literal1});
+
+  xrt::XLAAllocation param_alloc;
+  *param_alloc.mutable_value() = literal.ToProto();
+
+  xrt::XLAComputation c;
+  auto config = c.mutable_config();
+  auto shapes = config->mutable_program_shape();
+  *shapes->add_parameters() = shape.ToProto();
+  *shapes->mutable_result() = return_shape.ToProto();
+  StoreComputationSnapshot(computation, c.mutable_hlo_snapshot());
+
+  xrt::XRTExecutionConfig e;
+  e.set_release_input_handles(false);
+  e.set_release_compilation_handle(true);
+
+  Scope root = Scope::NewRootScope().WithDevice(DeviceFromFlag());
+  ClientSession session(root);
+  auto e_config =
+      ops::Const(root.WithDevice("/device:CPU:0"), e.SerializeAsString());
+  auto c_data =
+      ops::Const(root.WithDevice("/device:CPU:0"), c.SerializeAsString());
+  auto c_handle = ops::XRTCompile(root, c_data);
+  auto param_value = ops::Const(root.WithDevice("/device:CPU:0"),
+                                param_alloc.SerializeAsString());
+  auto param_handle = ops::XRTAllocate(root, param_value);
+  TF_ASSERT_OK(root.status());
+
+  std::vector<Tensor> outputs;
+  TF_EXPECT_OK(session.Run({param_handle}, &outputs));
+
+  int64 alloc_handle = outputs[0].scalar<int64>()();
+
+  // Note that we release the result handle immediately, but since we aliased
+  // the output buffers onto the input allocation ones (held in alloc_handle),
+  // we can fetch the result from there.
+  auto result =
+      ops::XRTExecute(root, c_handle.handle, e_config, {Input(alloc_handle)});
+  auto read_back = ops::XRTReadLiteral(root, result);
+  auto release = ops::XRTReleaseAllocationHandle(
+      root.WithControlDependencies(read_back), result);
+  TF_ASSERT_OK(root.status());
+
+  outputs.clear();
+  TF_EXPECT_OK(session.Run(tensorflow::ClientSession::FeedType(), {read_back},
+                           {release}, &outputs));
+
+  xla::Literal exec_literal = ReadOutputLiteral(outputs, 0);
+  auto exec_literal_parts = exec_literal.DecomposeTuple();
+  ASSERT_EQ(exec_literal_parts.size(), 4);
+
+  EXPECT_TRUE(CompareLiterals(exec_literal_parts[2], literal0));
+  EXPECT_TRUE(CompareLiterals(exec_literal_parts[3], literal1));
+
+  // Now we read back the original input handle values, which at this point
+  // should contain the result of the XLA computation.
+  auto read_handle = ops::XRTReadLiteral(root, Input(alloc_handle));
+  TF_ASSERT_OK(root.status());
+  auto release_handle = ops::XRTReleaseAllocationHandle(
+      root.WithControlDependencies(read_handle), Input(alloc_handle));
+  TF_ASSERT_OK(root.status());
+
+  outputs.clear();
+  TF_EXPECT_OK(session.Run(tensorflow::ClientSession::FeedType(), {read_handle},
+                           {release_handle}, &outputs));
+
+  xla::Literal return_literal = ReadOutputLiteral(outputs, 0);
+
+  auto expected_literal0 = xla::LiteralUtil::CreateR1<float>({6.0f, 11.0f});
+  auto expected_literal1 = xla::LiteralUtil::CreateR1<float>({-4.0f, -7.0f});
+  // The first element of the computation returned tuple would be the add
+  // (expected_literal0), but since we flipped the buffers, the sub
+  // (expected_literal1) should come first.
+  auto expected_literal =
+      xla::LiteralUtil::MakeTuple({&expected_literal1, &expected_literal0});
+
+  EXPECT_TRUE(CompareLiterals(return_literal, expected_literal));
+}
+
 TEST(RawApiTest, CompileAndExecuteWithS64Argument) {
   xrt::XLAAllocation p0;
   *p0.mutable_value() = xla::LiteralUtil::CreateR0<int64>(11031965).ToProto();
diff --git a/tensorflow/compiler/xrt/xrt_state.cc b/tensorflow/compiler/xrt/xrt_state.cc
index 343460ff10..13c275aaca 100644
--- a/tensorflow/compiler/xrt/xrt_state.cc
+++ b/tensorflow/compiler/xrt/xrt_state.cc
@@ -133,7 +133,8 @@ Status AllocateScopedShapedBuffer(
 XRTBufferAllocation::XRTBufferAllocation(const se::DeviceMemoryBase& allocation,
                                          int device_ordinal,
                                          xla::DeviceMemoryAllocator* allocator)
-    : allocation_(allocation),
+    : size_(allocation.size()),
+      allocation_(allocation),
       device_ordinal_(device_ordinal),
       allocator_(allocator) {
   if (VLOG_IS_ON(2)) {
@@ -223,8 +224,19 @@ Status XRTTupleAllocation::ToLiteral(xla::Backend* backend, int device_ordinal,
                                      xla::Literal* literal) {
   auto transfer_manager = backend->transfer_manager();
   TF_ASSIGN_OR_RETURN(auto stream, backend->BorrowStream(device_ordinal));
+
+  // Validate the allocation buffers as if nulls gets to
+  // TransferLiteralFromDevice() a CHECK is issued.
+  xla::ShapedBuffer shaped_buffer = ToShapedBuffer();
+  for (auto& index_buffer : shaped_buffer.buffers()) {
+    if (index_buffer.second.is_null()) {
+      return errors::InvalidArgument("Literal buffer at index ",
+                                     index_buffer.first.ToString(),
+                                     " has been released");
+    }
+  }
   TF_ASSIGN_OR_RETURN(*literal, transfer_manager->TransferLiteralFromDevice(
-                                    stream.get(), ToShapedBuffer()));
+                                    stream.get(), shaped_buffer));
   return Status::OK();
 }
 
@@ -505,11 +517,34 @@ xla::ShapedBuffer XRTTupleAllocation::ToShapedBuffer() {
   return shaped_buffer;
 }
 
+Status XRTTupleAllocation::AliasBufferFrom(const XRTTupleAllocation& source,
+                                           const xla::ShapeIndex& source_index,
+                                           const xla::ShapeIndex& dest_index) {
+  XRTBufferAllocation* source_buffer = source.buffers_.element(source_index);
+  XRTBufferAllocation* dest_buffer = buffers_.element(dest_index);
+  // We allow the destination size being zero, because there are cases where we
+  // are coming in later filling in null/uninitialized device buffers.
+  // In all other cases, the size of the new buffer must match.
+  if (source_buffer->size() != dest_buffer->size() &&
+      dest_buffer->size() != 0) {
+    return errors::InvalidArgument(
+        "Source buffer at index ", source_index.ToString(),
+        " does not match the size of destination buffer at index ",
+        dest_index.ToString(), ": ", source_buffer->size(), " vs ",
+        dest_buffer->size());
+  }
+  *buffers_.mutable_element(dest_index) = source_buffer;
+  source_buffer->Ref();
+  dest_buffer->Unref();
+  return Status::OK();
+}
+
 xla::ShapeTree<xla::MaybeOwningDeviceMemory>
-XRTTupleAllocation::ToDeviceMemoryTree(bool release) {
+XRTTupleAllocation::ToDeviceMemoryTree(
+    const std::function<bool(const xla::ShapeIndex&)>& release_checker) {
   xla::ShapeTree<xla::MaybeOwningDeviceMemory> shaped_tree(on_device_shape());
   for (const auto& buffer : buffers_) {
-    if (!release) {
+    if (!release_checker(buffer.first)) {
       *shaped_tree.mutable_element(buffer.first) = buffer.second->allocation();
     } else {
       *shaped_tree.mutable_element(buffer.first) = xla::OwningDeviceMemory(
diff --git a/tensorflow/compiler/xrt/xrt_state.h b/tensorflow/compiler/xrt/xrt_state.h
index 4aac37737e..1b32b6a20f 100644
--- a/tensorflow/compiler/xrt/xrt_state.h
+++ b/tensorflow/compiler/xrt/xrt_state.h
@@ -18,6 +18,7 @@ limitations under the License.
 #ifndef TENSORFLOW_COMPILER_XRT_XRT_STATE_H_
 #define TENSORFLOW_COMPILER_XRT_XRT_STATE_H_
 
+#include <functional>
 #include <memory>
 #include <string>
 #include <vector>
@@ -58,7 +59,14 @@ class XRTBufferAllocation : public core::RefCounted {
   // freed when the reference count drops to zero.
   void DiscardAllocation();
 
+  // Returns the expected size of the allocation. Since DiscardAllocation() will
+  // set allocation_ to {null,0}, and since later we might want to replace the
+  // discarded buffer with a new one, we need to be able to verify the size
+  // compatibility.
+  uint64 size() const { return size_; }
+
  private:
+  uint64 size_ = 0;
   se::DeviceMemoryBase allocation_;
   int device_ordinal_;
   xla::DeviceMemoryAllocator* allocator_;
@@ -168,9 +176,18 @@ class XRTTupleAllocation : public ResourceBase {
   // the same shape as on_host_shape.
   xla::ShapedBuffer ToShapedBuffer();
 
-  // Returns the device memory tree of this allocation. If 'release' is set, the
-  // ownership of the device memory is transferred to the result.
-  xla::ShapeTree<xla::MaybeOwningDeviceMemory> ToDeviceMemoryTree(bool release);
+  // Aliases the source buffer at source_index into the current tuple allocation
+  // dest_index.
+  Status AliasBufferFrom(const XRTTupleAllocation& source,
+                         const xla::ShapeIndex& source_index,
+                         const xla::ShapeIndex& dest_index);
+
+  // Returns the device memory tree of this allocation. If the release_checker
+  // function returns true for a given index, the ownership of the device memory
+  // at that index is transferred to the result. Every attempt to read the value
+  // at that index will fail.
+  xla::ShapeTree<xla::MaybeOwningDeviceMemory> ToDeviceMemoryTree(
+      const std::function<bool(const xla::ShapeIndex&)>& release_checker);
 
   string DebugString() const override { return "XLA allocation handle"; }
 
-- 
GitLab


From bdd76e2f04b17512d5c64a294975e7feb1231fab Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 10 Jan 2019 18:22:45 -0800
Subject: [PATCH 0567/1095] Update ops-related pbtxt files.

PiperOrigin-RevId: 228813872
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 132 ++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 |  69 +++++++++
 2 files changed, 201 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 9698673dfe..15a4a10189 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -12758,6 +12758,77 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "CollectiveReduce"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "data"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_HALF
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  attr {
+    name: "group_size"
+    type: "int"
+  }
+  attr {
+    name: "group_key"
+    type: "int"
+  }
+  attr {
+    name: "instance_key"
+    type: "int"
+  }
+  attr {
+    name: "merge_op"
+    type: "string"
+    allowed_values {
+      list {
+        s: "Min"
+        s: "Max"
+        s: "Mul"
+        s: "Add"
+      }
+    }
+  }
+  attr {
+    name: "final_op"
+    type: "string"
+    allowed_values {
+      list {
+        s: "Id"
+        s: "Div"
+      }
+    }
+  }
+  attr {
+    name: "subdiv_offsets"
+    type: "list(int)"
+  }
+  attr {
+    name: "wait_for"
+    type: "list(int)"
+    default_value {
+      list {
+      }
+    }
+  }
+  is_stateful: true
+}
 op {
   name: "CompareAndBitpack"
   input_arg {
@@ -30125,6 +30196,44 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "KMC2ChainInitialization"
+  input_arg {
+    name: "distances"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "seed"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "index"
+    type: DT_INT64
+  }
+}
+op {
+  name: "KmeansPlusPlusInitialization"
+  input_arg {
+    name: "points"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "num_to_sample"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "seed"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "num_retries_per_sample"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "samples"
+    type: DT_FLOAT
+  }
+}
 op {
   name: "L2Loss"
   input_arg {
@@ -38646,6 +38755,29 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "NearestNeighbors"
+  input_arg {
+    name: "points"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "centers"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "k"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "nearest_center_indices"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "nearest_center_distances"
+    type: DT_FLOAT
+  }
+}
 op {
   name: "Neg"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 4a1b3a6f2d..9949c00aa4 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -5417,6 +5417,14 @@ op {
     name: "subdiv_offsets"
     type: "list(int)"
   }
+  attr {
+    name: "wait_for"
+    type: "list(int)"
+    default_value {
+      list {
+      }
+    }
+  }
   is_stateful: true
 }
 op {
@@ -15070,6 +15078,44 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "KMC2ChainInitialization"
+  input_arg {
+    name: "distances"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "seed"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "index"
+    type: DT_INT64
+  }
+}
+op {
+  name: "KmeansPlusPlusInitialization"
+  input_arg {
+    name: "points"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "num_to_sample"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "seed"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "num_retries_per_sample"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "samples"
+    type: DT_FLOAT
+  }
+}
 op {
   name: "L2Loss"
   input_arg {
@@ -18960,6 +19006,29 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "NearestNeighbors"
+  input_arg {
+    name: "points"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "centers"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "k"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "nearest_center_indices"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "nearest_center_distances"
+    type: DT_FLOAT
+  }
+}
 op {
   name: "Neg"
   input_arg {
-- 
GitLab


From a4b7a2575485aa4918597d1974b299b886f4a1cd Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 10 Jan 2019 18:47:26 -0800
Subject: [PATCH 0568/1095] Go: Update generated wrapper functions for
 TensorFlow ops. PiperOrigin-RevId: 228816108

---
 tensorflow/go/op/wrappers.go | 103 ++++++++++++++++++++++++++++++++++-
 1 file changed, 102 insertions(+), 1 deletion(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index 6f6fb793a0..f8a5b5e7e1 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -4526,12 +4526,26 @@ func CollectiveBcastSend(scope *Scope, input tf.Output, group_size int64, group_
 	return op.Output(0)
 }
 
+// CollectiveReduceAttr is an optional argument to CollectiveReduce.
+type CollectiveReduceAttr func(optionalAttr)
+
+// CollectiveReduceWaitFor sets the optional wait_for attribute to value.
+// If not specified, defaults to <>
+func CollectiveReduceWaitFor(value []int64) CollectiveReduceAttr {
+	return func(m optionalAttr) {
+		m["wait_for"] = value
+	}
+}
+
 // Mutually reduces multiple tensors of identical type and shape.
-func CollectiveReduce(scope *Scope, input tf.Output, group_size int64, group_key int64, instance_key int64, merge_op string, final_op string, subdiv_offsets []int64) (data tf.Output) {
+func CollectiveReduce(scope *Scope, input tf.Output, group_size int64, group_key int64, instance_key int64, merge_op string, final_op string, subdiv_offsets []int64, optional ...CollectiveReduceAttr) (data tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	attrs := map[string]interface{}{"group_size": group_size, "group_key": group_key, "instance_key": instance_key, "merge_op": merge_op, "final_op": final_op, "subdiv_offsets": subdiv_offsets}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
 		Type: "CollectiveReduce",
 		Input: []tf.Input{
@@ -6846,6 +6860,35 @@ func Div(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	return op.Output(0)
 }
 
+// Selects the k nearest centers for each point.
+//
+// Rows of points are assumed to be input points. Rows of centers are assumed to be
+// the list of candidate centers. For each point, the k centers that have least L2
+// distance to it are computed.
+//
+// Arguments:
+//	points: Matrix of shape (n, d). Rows are assumed to be input points.
+//	centers: Matrix of shape (m, d). Rows are assumed to be centers.
+//	k: Number of nearest centers to return for each point. If k is larger than m, then
+// only m centers are returned.
+//
+// Returns Matrix of shape (n, min(m, k)). Each row contains the indices of the centers
+// closest to the corresponding point, ordered by increasing distance.Matrix of shape (n, min(m, k)). Each row contains the squared L2 distance to the
+// corresponding center in nearest_center_indices.
+func NearestNeighbors(scope *Scope, points tf.Output, centers tf.Output, k tf.Output) (nearest_center_indices tf.Output, nearest_center_distances tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "NearestNeighbors",
+		Input: []tf.Input{
+			points, centers, k,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1)
+}
+
 // Returns x * y element-wise.
 //
 // *NOTE*: `Multiply` supports broadcasting. More about broadcasting
@@ -7649,6 +7692,33 @@ func TensorSliceDataset(scope *Scope, components []tf.Output, output_shapes []tf
 	return op.Output(0)
 }
 
+// Returns the index of a data point that should be added to the seed set.
+//
+// Entries in distances are assumed to be squared distances of candidate points to
+// the already sampled centers in the seed set. The op constructs one Markov chain
+// of the k-MC^2 algorithm and returns the index of one candidate point to be added
+// as an additional cluster center.
+//
+// Arguments:
+//	distances: Vector with squared distances to the closest previously sampled cluster center
+// for each candidate point.
+//	seed: Scalar. Seed for initializing the random number generator.
+//
+// Returns Scalar with the index of the sampled point.
+func KMC2ChainInitialization(scope *Scope, distances tf.Output, seed tf.Output) (index tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "KMC2ChainInitialization",
+		Input: []tf.Input{
+			distances, seed,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Computes hyperbolic sine of x element-wise.
 func Sinh(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
@@ -10408,6 +10478,37 @@ func BiasAddV1(scope *Scope, value tf.Output, bias tf.Output) (output tf.Output)
 	return op.Output(0)
 }
 
+// Selects num_to_sample rows of input using the KMeans++ criterion.
+//
+// Rows of points are assumed to be input points. One row is selected at random.
+// Subsequent rows are sampled with probability proportional to the squared L2
+// distance from the nearest row selected thus far till num_to_sample rows have
+// been sampled.
+//
+// Arguments:
+//	points: Matrix of shape (n, d). Rows are assumed to be input points.
+//	num_to_sample: Scalar. The number of rows to sample. This value must not be larger than n.
+//	seed: Scalar. Seed for initializing the random number generator.
+//	num_retries_per_sample: Scalar. For each row that is sampled, this parameter
+// specifies the number of additional points to draw from the current
+// distribution before selecting the best. If a negative value is specified, a
+// heuristic is used to sample O(log(num_to_sample)) additional points.
+//
+// Returns Matrix of shape (num_to_sample, d). The sampled rows.
+func KmeansPlusPlusInitialization(scope *Scope, points tf.Output, num_to_sample tf.Output, seed tf.Output, num_retries_per_sample tf.Output) (samples tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "KmeansPlusPlusInitialization",
+		Input: []tf.Input{
+			points, num_to_sample, seed, num_retries_per_sample,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Shuffle dimensions of x according to a permutation.
 //
 // The output `y` has the same rank as `x`. The shapes of `x` and `y` satisfy:
-- 
GitLab


From aeddcb55fadc0ad9085d71d736ba49c6b69620bf Mon Sep 17 00:00:00 2001
From: Francois Chollet <fchollet@google.com>
Date: Thu, 10 Jan 2019 18:53:55 -0800
Subject: [PATCH 0569/1095] Make `steps_per_epoch`, `steps`, `validation_steps`
 optional for non-infinite datasets.

(Note: if your dataset is infinite but that fact is unknowable (cardinality=UNKNOWN), and you forget the `steps` argument,
your model will train forever, and ever, and ever)

PiperOrigin-RevId: 228816626
---
 tensorflow/python/keras/callbacks.py          |  15 +-
 .../engine/distributed_training_utils.py      |  11 +-
 tensorflow/python/keras/engine/training.py    |  44 +--
 .../python/keras/engine/training_arrays.py    | 256 ++++++++++--------
 .../keras/engine/training_dataset_test.py     |  44 ++-
 .../keras/engine/training_distributed.py      |  48 ++--
 .../keras/engine/training_eager_test.py       |   3 +-
 .../python/keras/engine/training_generator.py |  70 ++++-
 .../python/keras/engine/training_utils.py     |  63 ++++-
 9 files changed, 368 insertions(+), 186 deletions(-)

diff --git a/tensorflow/python/keras/callbacks.py b/tensorflow/python/keras/callbacks.py
index b9074669eb..e8985c2306 100644
--- a/tensorflow/python/keras/callbacks.py
+++ b/tensorflow/python/keras/callbacks.py
@@ -688,15 +688,14 @@ class ProgbarLogger(Callback):
     if self.verbose:
       if self.epochs > 1:
         print('Epoch %d/%d' % (epoch + 1, self.epochs))
-      self.progbar = Progbar(
-          target=self.target,
-          verbose=self.verbose,
-          stateful_metrics=self.stateful_metrics,
-          unit_name='step' if self.use_steps else 'sample')
+    self.progbar = Progbar(
+        target=self.target,
+        verbose=self.verbose,
+        stateful_metrics=self.stateful_metrics,
+        unit_name='step' if self.use_steps else 'sample')
 
   def on_batch_begin(self, batch, logs=None):
-    if self.seen < self.target:
-      self.log_values = []
+    self.log_values = []
 
   def on_batch_end(self, batch, logs=None):
     logs = logs or {}
@@ -715,7 +714,7 @@ class ProgbarLogger(Callback):
 
     # Skip progbar update for the last batch;
     # will be handled by on_epoch_end.
-    if self.verbose and self.seen < self.target:
+    if self.verbose and (self.target is None or self.seen < self.target):
       self.progbar.update(self.seen, self.log_values)
 
   def on_epoch_end(self, epoch, logs=None):
diff --git a/tensorflow/python/keras/engine/distributed_training_utils.py b/tensorflow/python/keras/engine/distributed_training_utils.py
index b6ef33f700..0185e87cb0 100644
--- a/tensorflow/python/keras/engine/distributed_training_utils.py
+++ b/tensorflow/python/keras/engine/distributed_training_utils.py
@@ -525,6 +525,15 @@ def list_to_tuple(maybe_list):
   return maybe_list
 
 
+def get_iterator(dataset, distribution_strategy):
+  with distribution_strategy.scope():
+    iterator = distribution_strategy.make_dataset_iterator(dataset)
+    init_op = iterator.initialize()
+    if not context.executing_eagerly():
+      K.get_session().run(init_op)
+  return iterator
+
+
 def _get_input_from_iterator(iterator, model):
   """Get elements from the iterator and verify the input shape and type."""
   next_element = iterator.get_next()
@@ -541,8 +550,6 @@ def _get_input_from_iterator(iterator, model):
     x, y, sample_weights = next_element
 
   # Validate that all the elements in x and y are of the same type and shape.
-  # We can then pass the first element of x and y to `_standardize_weights`
-  # below and be confident of the output.
   validate_distributed_dataset_inputs(
       model._distribution_strategy, x, y, sample_weights)
   return x, y, sample_weights
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 5614c93842..3e6c13232c 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -896,7 +896,8 @@ class Model(Network):
           validation_steps=validation_steps,
           workers=0,
           shuffle=shuffle,
-          initial_epoch=initial_epoch)
+          initial_epoch=initial_epoch,
+          steps_name='steps_per_epoch')
     else:
       return training_arrays.fit_loop(
           self,
@@ -913,7 +914,8 @@ class Model(Network):
           shuffle=shuffle,
           initial_epoch=initial_epoch,
           steps_per_epoch=steps_per_epoch,
-          validation_steps=validation_steps)
+          validation_steps=validation_steps,
+          steps_name='steps_per_epoch')
 
   def evaluate(self,
                x=None,
@@ -1500,7 +1502,8 @@ class Model(Network):
         workers=workers,
         use_multiprocessing=use_multiprocessing,
         shuffle=shuffle,
-        initial_epoch=initial_epoch)
+        initial_epoch=initial_epoch,
+        steps_name='steps_per_epoch')
 
   def evaluate_generator(self,
                          generator,
@@ -2136,7 +2139,7 @@ class Model(Network):
       shuffle: Boolean whether to shuffle the training data before each epoch.
 
     Returns:
-      Iterator for reading the dataset `x`.
+      Dataset instance.
 
     Raises:
       ValueError: In case of invalid user-provided data.
@@ -2167,7 +2170,8 @@ class Model(Network):
     else:
       session = K.get_session()
 
-    with self._distribution_strategy.scope():
+    strategy = self._distribution_strategy
+    with strategy.scope():
       first_x_value = nest.flatten(x)[0]
       if isinstance(first_x_value, np.ndarray):
         x = distributed_training_utils.list_to_tuple(x)
@@ -2190,20 +2194,20 @@ class Model(Network):
           shuffle_buffer = 1024
         else:
           shuffle_buffer = None
-        iterator = self._distribution_strategy.experimental_make_numpy_iterator(
-            in_tuple, batch_size, num_epochs=None, shuffle=shuffle_buffer,
-            session=session)
+        ds = strategy.extended.experimental_make_numpy_dataset(in_tuple,
+                                                               session=session)
+        if shuffle_buffer:
+          ds = ds.shuffle(shuffle_buffer)
+        ds = ds.repeat()
+        # We need to use the drop_remainder argument to get a known static
+        # input shape which is required for TPUs.
+        drop_remainder = strategy.extended.experimental_require_static_shapes
+        x = ds.batch(batch_size, drop_remainder=drop_remainder)
       else:
         assert isinstance(x, dataset_ops.DatasetV2)
         training_utils.validate_dataset_input(x, y, sample_weight,
                                               validation_split)
-        iterator = self._distribution_strategy.make_dataset_iterator(x)
-
-      init_op = iterator.initialize()
-      if not context.executing_eagerly():
-        K.get_session().run(init_op)
-
-    return iterator
+    return x
 
   def _standardize_user_data(self,
                              x,
@@ -2276,7 +2280,7 @@ class Model(Network):
       ValueError: In case of invalid user-provided data.
       RuntimeError: If the model was never compiled.
     """
-    if isinstance(x, (dataset_ops.DatasetV2, dataset_ops.DatasetV1)):
+    if isinstance(x, (dataset_ops.DatasetV1, dataset_ops.DatasetV2)):
       # Graph mode dataset. We'll pass the dataset as-is (unless
       # `extract_tensors_from_dataset` is True, in which case we extract
       # the tensors from the dataset and we output them.
@@ -2313,7 +2317,7 @@ class Model(Network):
       # If input data is a dataset iterator in graph mode or if it is an eager
       # iterator and only one batch of samples is required, we fetch the data
       # tensors from the iterator and then standardize them.
-      if isinstance(x, (dataset_ops.DatasetV2, dataset_ops.DatasetV1)):
+      if isinstance(x, (dataset_ops.DatasetV1, dataset_ops.DatasetV2)):
         x_input, y_input, _ = training_utils.extract_tensors_from_dataset(x)
       else:
         x_input = x
@@ -2433,7 +2437,7 @@ class Model(Network):
       feed_input_shapes = self._feed_input_shapes
 
     # Standardize the inputs.
-    if not isinstance(x, (dataset_ops.DatasetV2, dataset_ops.DatasetV1)):
+    if not isinstance(x, (dataset_ops.DatasetV1, dataset_ops.DatasetV2)):
       # TODO(fchollet): run static checks with dataset output shape(s).
       x = training_utils.standardize_input_data(
           x,
@@ -2514,8 +2518,8 @@ class Model(Network):
                          str(x[0].shape[0]) + ' samples')
 
     # If dictionary inputs were provided, we return a dictionary as well.
-    if dict_inputs and not isinstance(x, (dataset_ops.DatasetV2,
-                                          dataset_ops.DatasetV1)):
+    if dict_inputs and not isinstance(x, (dataset_ops.DatasetV1,
+                                          dataset_ops.DatasetV2)):
       x = dict(zip(feed_input_names, x))
     return x, y, sample_weights
 
diff --git a/tensorflow/python/keras/engine/training_arrays.py b/tensorflow/python/keras/engine/training_arrays.py
index ab7d455bfa..a356f8b328 100644
--- a/tensorflow/python/keras/engine/training_arrays.py
+++ b/tensorflow/python/keras/engine/training_arrays.py
@@ -41,90 +41,6 @@ except ImportError:
   issparse = None
 
 
-def _get_model_feed(model, mode):
-  if mode == ModeKeys.PREDICT:
-    feed = model._feed_inputs
-  else:
-    feed = (
-        model._feed_inputs + model._feed_targets + model._feed_sample_weights)
-  return feed
-
-
-def _validate_arguments(steps_per_epoch, validation_steps, kwargs):
-  for k in kwargs:
-    if k != 'steps':
-      raise ValueError('Invalid argument passed: {}'.format(k))
-
-  # Validate inputs when in training mode.
-  if validation_steps and steps_per_epoch is None:
-    raise ValueError('Can only use `validation_steps` '
-                     'when doing step-wise '
-                     'training, i.e. `steps_per_epoch` '
-                     'must be set.')
-
-
-def _print_train_info(inputs, val_inputs, steps_per_epoch, verbose):
-  if (val_inputs and steps_per_epoch is None and verbose and inputs and
-      hasattr(inputs[0], 'shape') and hasattr(val_inputs[0], 'shape')):
-    print('Train on %d samples, validate on %d samples' %
-          (inputs[0].shape[0], val_inputs[0].shape[0]))
-
-
-def _get_num_samples_or_steps(ins, batch_size, steps_per_epoch):
-  """Returns total number of samples (when training in batch mode) or steps."""
-  if steps_per_epoch:
-    return steps_per_epoch
-  return training_utils.check_num_samples(ins, batch_size, steps_per_epoch,
-                                          'steps_per_epoch')
-
-
-def _prepare_feed_values(model, inputs, targets, sample_weights, mode):
-  """Prepare feed values to the model execution function.
-
-  Arguments:
-    model: Model to prepare feed values for.
-    inputs: List or dict of model inputs.
-    targets: Optional list of model targets.
-    sample_weights: Optional list of sample weight arrays.
-    mode: One of ModeKeys.TRAIN/ModeKeys.TEST/ModeKeys.PREDICT.
-
-  Returns:
-    Feed values for the model in the given mode.
-  """
-  if model._distribution_strategy:
-    def get_distributed_inputs():
-      return distributed_training_utils._prepare_feed_values(
-          model, inputs, targets, sample_weights, mode)
-
-    # In the eager case, we want to call the input method per step, so return
-    # a lambda from here that can be called. Note that this is applicable only
-    # in Distribution Strategy case as it follows the same code path for both
-    # eager and graph modes.
-    # TODO(priyag,omalleyt): Either we should move the training DS with
-    # EagerIterator to use training_generator code path, or figure out how to
-    # set a symbolic Iterator out of a Dataset when in eager mode.
-    if context.executing_eagerly():
-      return get_distributed_inputs
-    else:
-      return get_distributed_inputs()
-
-  inputs = training_utils.ModelInputs(inputs).as_list()
-  targets = targets or []
-  sample_weights = sample_weights or []
-  ins = inputs + targets + sample_weights
-  if mode == ModeKeys.TRAIN and not isinstance(K.symbolic_learning_phase(),
-                                               int):
-    ins += [True]
-  return ins
-
-
-def _make_execution_function(model, mode):
-  """Makes function to run one step of model execution."""
-  if model._distribution_strategy:
-    return distributed_training_utils._make_execution_function(model, mode)
-  return model._make_execution_function(mode)
-
-
 def model_iteration(model,
                     inputs,
                     targets=None,
@@ -142,6 +58,7 @@ def model_iteration(model,
                     validation_steps=None,
                     mode=ModeKeys.TRAIN,
                     validation_in_fit=False,
+                    steps_name='steps',
                     **kwargs):
   """Loop function for arrays of data with modes TRAIN/TEST/PREDICT.
 
@@ -173,6 +90,9 @@ def model_iteration(model,
         weights when using a tf.distribute.Strategy. The input is deprecated as
         it is not required if the user creates a distributed model under the
         distribution strategy scope rather than passing it to compile.
+      steps_name: The string name of the steps argument, either `steps`,
+        `validation_steps`, or `steps_per_epoch`. Only used for error message
+        formatting.
       **kwargs: Additional arguments for backwards compatibility.
 
   Returns:
@@ -185,23 +105,25 @@ def model_iteration(model,
   """
   # Backwards compatibility.
   if 'steps' in kwargs:
-    steps_per_epoch = kwargs['steps']
+    steps_per_epoch = kwargs.pop('steps')
+  if kwargs:
+    raise TypeError('Unknown arguments: %s' % (kwargs,))
+
+  # In case we were passed a dataset, we extract symbolic tensors from it.
+  reset_dataset_after_each_epoch = False
+  original_dataset = None
+  is_dataset = isinstance(inputs,
+                          (dataset_ops.DatasetV1, dataset_ops.DatasetV2))
+  # TODO(fchollet): consider moving `steps_per_epoch` inference to
+  # _standardize_user_data and set reset_dataset_after_each_epoch as an
+  # attribute on the dataset instance.
+  if is_dataset:
+    original_dataset = inputs
+    if steps_per_epoch is None:
+      reset_dataset_after_each_epoch = True
+      steps_per_epoch = training_utils.infer_steps_for_dataset(
+          inputs, steps_per_epoch, epochs=epochs, steps_name=steps_name)
 
-  # In case we are passed datasets, we extract symbolic tensors from them.
-  if isinstance(inputs, (dataset_ops.DatasetV2, dataset_ops.DatasetV1)):
-    inputs, targets, sample_weights = model._standardize_user_data(
-        inputs,
-        steps_name='steps_per_epoch',
-        steps=steps_per_epoch,
-        extract_tensors_from_dataset=True)
-  if isinstance(val_inputs, (dataset_ops.DatasetV2, dataset_ops.DatasetV1)):
-    val_inputs, val_targets, val_sample_weights = model._standardize_user_data(
-        val_inputs,
-        steps_name='validation_steps',
-        steps=validation_steps,
-        extract_tensors_from_dataset=True)
-
-  _validate_arguments(steps_per_epoch, validation_steps, kwargs)
   if mode == ModeKeys.TRAIN:
     _print_train_info(inputs, val_inputs, steps_per_epoch, verbose)
 
@@ -212,7 +134,7 @@ def model_iteration(model,
 
   # Get step function and loop type.
   f = _make_execution_function(model, mode)
-  use_steps = steps_per_epoch is not None
+  use_steps = is_dataset or steps_per_epoch is not None
   do_validation = val_inputs is not None
 
   # Convert Eager Tensors to NumPy arrays to support batching/shuffling.
@@ -221,8 +143,11 @@ def model_iteration(model,
 
   # Prepare input data.
   ins = _prepare_feed_values(model, inputs, targets, sample_weights, mode)
-  num_samples_or_steps = _get_num_samples_or_steps(ins, batch_size,
-                                                   steps_per_epoch)
+  if not is_dataset:
+    num_samples_or_steps = _get_num_samples_or_steps(ins, batch_size,
+                                                     steps_per_epoch)
+  else:
+    num_samples_or_steps = steps_per_epoch
 
   # Configure callbacks.
   count_mode = 'steps' if use_steps else 'samples'
@@ -278,7 +203,15 @@ def model_iteration(model,
 
     if use_steps:
       # Step-wise loop.
-      for step in range(steps_per_epoch):
+      if steps_per_epoch is None:
+        # Loop over dataset until `OutOfRangeError` is raised.
+        target_steps = np.inf
+      else:
+        # Loop over dataset for the specified number of steps.
+        target_steps = steps_per_epoch
+
+      step = 0
+      while step < target_steps:
         batch_logs = {'batch': step, 'size': 1}
         callbacks._call_batch_hook(mode, 'begin', step, batch_logs)
         progbar.on_batch_begin(step, batch_logs)
@@ -289,13 +222,26 @@ def model_iteration(model,
           actual_inputs = ins() if callable(ins) else ins
           batch_outs = f(actual_inputs)
         except errors.OutOfRangeError:
-          logging.warning('Your dataset iterator ran out of data; '
-                          'interrupting training. Make sure that your dataset '
-                          'can generate at least `steps_per_epoch * epochs` '
-                          'batches (in this case, %d batches). You may need to'
-                          'use the repeat() function when building your '
-                          'dataset.' % steps_per_epoch * epochs)
+          if original_dataset is None:
+            # We ran out of batches while the user passed an iterator (legacy).
+            logging.warning(
+                'Your dataset iterator ran out of data; '
+                'interrupting training. Make sure that your iterator '
+                'can generate at least `%s * epochs` '
+                'batches (in this case, %d batches). You may need to'
+                'use the repeat() function when building your '
+                'dataset.' % (steps_name, steps_per_epoch * epochs))
+            callbacks.model.stop_training = True
+          else:
+            # The dataset passed by the user ran out of batches.
+            # Now we know the cardinality of the dataset.
+            if step > 0:
+              steps_per_epoch = step
+              aggregator.num_samples_or_steps = steps_per_epoch
+              progbar.params['steps'] = steps_per_epoch
+              progbar.progbar.target = steps_per_epoch
           break
+
         if not isinstance(batch_outs, list):
           batch_outs = [batch_outs]
 
@@ -312,6 +258,7 @@ def model_iteration(model,
         batch_logs = cbks.make_logs(model, batch_logs, batch_outs, mode)
         callbacks._call_batch_hook(mode, 'end', step, batch_logs)
         progbar.on_batch_end(step, batch_logs)
+        step += 1
 
         if callbacks.model.stop_training:
           break
@@ -385,7 +332,8 @@ def model_iteration(model,
           callbacks=callbacks,
           verbose=0,
           mode=ModeKeys.TEST,
-          validation_in_fit=True)
+          validation_in_fit=True,
+          steps_name='validation_steps')
       if not isinstance(val_results, list):
         val_results = [val_results]
       epoch_logs = cbks.make_logs(
@@ -394,7 +342,11 @@ def model_iteration(model,
     if mode == ModeKeys.TRAIN:
       # Epochs only apply to `fit`.
       callbacks.on_epoch_end(epoch, epoch_logs)
-      progbar.on_epoch_end(epoch, epoch_logs)
+    progbar.on_epoch_end(epoch, epoch_logs)
+
+    # Recreate dataset iterator for the next epoch.
+    if reset_dataset_after_each_epoch and epoch < epochs - 1:
+      ins = _prepare_feed_values(model, original_dataset, None, None, mode)
 
   callbacks._call_end_hook(mode)
 
@@ -410,6 +362,86 @@ def model_iteration(model,
   return results
 
 
+def _get_model_feed(model, mode):
+  if mode == ModeKeys.PREDICT:
+    feed = model._feed_inputs
+  else:
+    feed = (
+        model._feed_inputs + model._feed_targets + model._feed_sample_weights)
+  return feed
+
+
+def _print_train_info(inputs, val_inputs, steps_per_epoch, verbose):
+  if (val_inputs and steps_per_epoch is None and verbose and inputs and
+      hasattr(inputs[0], 'shape') and hasattr(val_inputs[0], 'shape')):
+    print('Train on %d samples, validate on %d samples' %
+          (inputs[0].shape[0], val_inputs[0].shape[0]))
+
+
+def _get_num_samples_or_steps(ins, batch_size, steps_per_epoch):
+  """Returns total number of samples (when training in batch mode) or steps."""
+  if steps_per_epoch:
+    return steps_per_epoch
+  return training_utils.check_num_samples(ins, batch_size, steps_per_epoch,
+                                          'steps_per_epoch')
+
+
+def _prepare_feed_values(model, inputs, targets, sample_weights, mode):
+  """Prepare feed values to the model execution function.
+
+  Arguments:
+    model: Model to prepare feed values for.
+    inputs: List or dict of model inputs.
+    targets: Optional list of model targets.
+    sample_weights: Optional list of sample weight arrays.
+    mode: One of ModeKeys.TRAIN/ModeKeys.TEST/ModeKeys.PREDICT.
+
+  Returns:
+    Feed values for the model in the given mode.
+  """
+  if model._distribution_strategy:
+    if isinstance(inputs, (dataset_ops.DatasetV1, dataset_ops.DatasetV2)):
+      inputs = distributed_training_utils.get_iterator(
+          inputs, model._distribution_strategy)
+
+    def get_distributed_inputs():
+      return distributed_training_utils._prepare_feed_values(
+          model, inputs, targets, sample_weights, mode)
+
+    # In the eager case, we want to call the input method per step, so return
+    # a lambda from here that can be called. Note that this is applicable only
+    # in Distribution Strategy case as it follows the same code path for both
+    # eager and graph modes.
+    # TODO(priyag,omalleyt): Either we should move the training DS with
+    # EagerIterator to use training_generator code path, or figure out how to
+    # set a symbolic Iterator out of a Dataset when in eager mode.
+    if context.executing_eagerly():
+      return get_distributed_inputs
+    else:
+      return get_distributed_inputs()
+
+  if isinstance(inputs, (dataset_ops.DatasetV1, dataset_ops.DatasetV2)):
+    inputs, targets, sample_weights = model._standardize_user_data(
+        inputs,
+        extract_tensors_from_dataset=True)
+
+  inputs = training_utils.ModelInputs(inputs).as_list()
+  targets = targets or []
+  sample_weights = sample_weights or []
+  ins = inputs + targets + sample_weights
+  if mode == ModeKeys.TRAIN and not isinstance(K.symbolic_learning_phase(),
+                                               int):
+    ins += [True]  # Add learning phase value.
+  return ins
+
+
+def _make_execution_function(model, mode):
+  """Makes function to run one step of model execution."""
+  if model._distribution_strategy:
+    return distributed_training_utils._make_execution_function(model, mode)
+  return model._make_execution_function(mode)
+
+
 # For backwards compatibility for internal users of these loops.
 fit_loop = functools.partial(model_iteration, mode=ModeKeys.TRAIN)
 test_loop = functools.partial(
diff --git a/tensorflow/python/keras/engine/training_dataset_test.py b/tensorflow/python/keras/engine/training_dataset_test.py
index 40d42c13f0..0ce01ec606 100644
--- a/tensorflow/python/keras/engine/training_dataset_test.py
+++ b/tensorflow/python/keras/engine/training_dataset_test.py
@@ -23,6 +23,7 @@ import logging
 import numpy as np
 
 from tensorflow.python import keras
+from tensorflow.python.data.experimental.ops import cardinality
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.eager import context
 from tensorflow.python.framework import test_util as tf_test_util
@@ -164,7 +165,7 @@ class TestTrainingWithDataset(keras_parameterized.TestCase):
     inputs = np.zeros((10, 3), np.float32)
     targets = np.zeros((10, 4), np.float32)
     dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
-    dataset = dataset.repeat(100)
+    dataset = dataset.repeat()  # Infinite dataset.
     dataset = dataset.batch(10)
 
     model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=1)
@@ -215,6 +216,7 @@ class TestTrainingWithDataset(keras_parameterized.TestCase):
       model.fit(dataset, dataset,
                 epochs=1, steps_per_epoch=2, verbose=0)
 
+    # With an infinite dataset, `steps_per_epoch`/`steps` argument is required.
     with self.assertRaisesRegexp(
         ValueError, 'the `steps_per_epoch` argument'):
       model.fit(dataset, epochs=1, verbose=0)
@@ -322,6 +324,46 @@ class TestTrainingWithDataset(keras_parameterized.TestCase):
                                    r'expected (.*?) to have shape \(3,\)'):
         model.train_on_batch(dataset)
 
+  @keras_parameterized.run_with_all_model_types
+  @keras_parameterized.run_all_keras_modes
+  def test_finite_dataset_known_cardinality_no_steps_arg(self):
+    model = testing_utils.get_small_mlp(1, 4, input_dim=3)
+    optimizer = RMSPropOptimizer(learning_rate=0.001)
+    model.compile(optimizer, 'mse',
+                  run_eagerly=testing_utils.should_run_eagerly())
+
+    inputs = np.zeros((100, 3), dtype=np.float32)
+    targets = np.random.randint(0, 4, size=100, dtype=np.int32)
+    dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
+    dataset = dataset.batch(10)
+
+    history = model.fit(dataset, epochs=2, verbose=1)
+    self.assertEqual(len(history.history['loss']), 2)
+    model.evaluate(dataset)
+    out = model.predict(dataset)
+    self.assertEqual(out.shape[0], 100)
+
+  @keras_parameterized.run_with_all_model_types
+  @keras_parameterized.run_all_keras_modes
+  def test_finite_dataset_unknown_cardinality_no_steps_arg(self):
+    model = testing_utils.get_small_mlp(1, 4, input_dim=3)
+    optimizer = RMSPropOptimizer(learning_rate=0.001)
+    model.compile(optimizer, 'mse',
+                  run_eagerly=testing_utils.should_run_eagerly())
+
+    inputs = np.zeros((100, 3), dtype=np.float32)
+    targets = np.random.randint(0, 4, size=100, dtype=np.int32)
+    dataset = dataset_ops.Dataset.from_tensor_slices((inputs, targets))
+    dataset = dataset.filter(lambda x, y: True).batch(10)
+    self.assertEqual(keras.backend.get_value(cardinality.cardinality(dataset)),
+                     cardinality.UNKNOWN)
+
+    history = model.fit(dataset, epochs=2, verbose=1)
+    self.assertEqual(len(history.history['loss']), 2)
+    model.evaluate(dataset)
+    out = model.predict(dataset)
+    self.assertEqual(out.shape[0], 100)
+
 
 class TestMetricsWithDatasetIterators(keras_parameterized.TestCase):
 
diff --git a/tensorflow/python/keras/engine/training_distributed.py b/tensorflow/python/keras/engine/training_distributed.py
index 0bd79f2b47..58a8637a84 100644
--- a/tensorflow/python/keras/engine/training_distributed.py
+++ b/tensorflow/python/keras/engine/training_distributed.py
@@ -65,7 +65,7 @@ def fit_distributed(model,
             batch_size, is_training=True))
   batch_size = model._validate_or_infer_batch_size(
       batch_size, steps_per_epoch, x)
-  iterator = model._distribution_standardize_user_data(
+  dataset = model._distribution_standardize_user_data(
       x, y,
       sample_weight=sample_weight,
       class_weight=class_weight,
@@ -76,7 +76,7 @@ def fit_distributed(model,
       validation_split=validation_split,
       shuffle=shuffle)
 
-  val_iterator = None
+  val_dataset = None
   if validation_data:
     val_x, val_y, val_sample_weights = model._unpack_validation_data(
         validation_data)
@@ -87,7 +87,7 @@ def fit_distributed(model,
       validation_steps, _ = distributed_training_utils.get_input_params(
           model._distribution_strategy, first_valx_value, validation_steps,
           batch_size)
-    val_iterator = model._distribution_standardize_user_data(
+    val_dataset = model._distribution_standardize_user_data(
         val_x, val_y,
         sample_weight=val_sample_weights,
         class_weight=None,
@@ -104,23 +104,23 @@ def fit_distributed(model,
   if distributed_training_utils.is_tpu_strategy(model._distribution_strategy):
     return experimental_tpu_fit_loop(
         model,
-        iterator,
+        dataset,
         epochs=epochs,
         verbose=verbose,
         callbacks=callbacks,
-        val_iterator=val_iterator,
+        val_dataset=val_dataset,
         initial_epoch=initial_epoch,
         steps_per_epoch=steps_per_epoch,
         validation_steps=validation_steps)
   else:
     return training_arrays.fit_loop(
         model,
-        iterator,
+        dataset,
         batch_size=batch_size,
         epochs=epochs,
         verbose=verbose,
         callbacks=callbacks,
-        val_inputs=val_iterator,
+        val_inputs=val_dataset,
         shuffle=shuffle,
         initial_epoch=initial_epoch,
         steps_per_epoch=steps_per_epoch,
@@ -142,7 +142,7 @@ def evaluate_distributed(model,
     steps, batch_size = distributed_training_utils.get_input_params(
         model._distribution_strategy, first_x_value, steps, batch_size)
   batch_size = model._validate_or_infer_batch_size(batch_size, steps, x)
-  iterator = model._distribution_standardize_user_data(
+  dataset = model._distribution_standardize_user_data(
       x, y,
       sample_weight=sample_weight,
       batch_size=batch_size,
@@ -153,11 +153,11 @@ def evaluate_distributed(model,
   if distributed_training_utils.is_tpu_strategy(model._distribution_strategy):
     # TODO(fchollet): why aren't callbacks supported here?
     return experimental_tpu_test_loop(
-        model, iterator=iterator, verbose=verbose, steps=steps)
+        model, dataset, verbose=verbose, steps=steps)
   else:
     return training_arrays.test_loop(
         model,
-        inputs=iterator,
+        inputs=dataset,
         batch_size=batch_size,
         verbose=verbose,
         steps=steps,
@@ -178,7 +178,7 @@ def predict_distributed(model,
     steps, batch_size = distributed_training_utils.get_input_params(
         model._distribution_strategy, first_x_value, steps, batch_size)
   batch_size = model._validate_or_infer_batch_size(batch_size, steps, x)
-  iterator = model._distribution_standardize_user_data(
+  dataset = model._distribution_standardize_user_data(
       x,
       batch_size=batch_size,
       check_steps=True,
@@ -187,11 +187,11 @@ def predict_distributed(model,
   if distributed_training_utils.is_tpu_strategy(model._distribution_strategy):
     # TODO(fchollet): why aren't callbacks supported here?
     return experimental_tpu_predict_loop(
-        model, iterator, verbose=verbose, steps=steps)
+        model, dataset, verbose=verbose, steps=steps)
   else:
     return training_arrays.predict_loop(
         model,
-        iterator,
+        dataset,
         batch_size=batch_size,
         verbose=verbose,
         steps=steps,
@@ -199,19 +199,19 @@ def predict_distributed(model,
 
 
 def experimental_tpu_fit_loop(model,
-                              iterator,
+                              dataset,
                               epochs=100,
                               verbose=1,
                               callbacks=None,
                               initial_epoch=0,
                               steps_per_epoch=None,
-                              val_iterator=None,
+                              val_dataset=None,
                               validation_steps=None):
   """Fit loop for training with TPU DistributionStrategy.
 
   Arguments:
       model: Keras Model instance.
-      iterator: Iterator that returns inputs and targets
+      dataset: Dataset that returns inputs and targets
       epochs: Number of times to iterate over the data
       verbose: Integer, Verbosity mode, 0, 1 or 2
       callbacks: List of callbacks to be called during training
@@ -220,7 +220,7 @@ def experimental_tpu_fit_loop(model,
       steps_per_epoch: Total number of steps (batches of samples)
           before declaring one epoch finished and starting the
           next epoch. Ignored with the default value of `None`.
-      val_iterator: Iterator for validation data.
+      val_dataset: Dataset for validation data.
       validation_steps: Number of steps to run validation for
           (only if doing validation from data tensors).
           Ignored with the default value of `None`.
@@ -231,7 +231,9 @@ def experimental_tpu_fit_loop(model,
   Raises:
       ValueError: in case of invalid arguments.
   """
+  # TODO(fchollet): add support for `steps_per_epoch=None` in TPU loops.
   current_strategy = model._distribution_strategy
+  iterator = distributed_training_utils.get_iterator(dataset, current_strategy)
   scope = current_strategy.scope()
   scope.__enter__()
 
@@ -373,7 +375,7 @@ def experimental_tpu_fit_loop(model,
 
       val_outs = experimental_tpu_test_loop(  # pylint: disable=undefined-variable
           model,
-          val_iterator,
+          val_dataset,
           steps=validation_steps,
           verbose=verbose)
       if not isinstance(val_outs, list):
@@ -397,14 +399,14 @@ def experimental_tpu_fit_loop(model,
 
 
 def experimental_tpu_test_loop(model,
-                               iterator,
+                               dataset,
                                verbose=0,
                                steps=None):
   """Test loop for evaluating with TPU DistributionStrategy.
 
   Arguments:
       model: Keras Model instance.
-      iterator: Iterator for input data.
+      dataset: Dataset for input data.
       verbose: Integer, Verbosity mode 0 or 1.
       steps: Total number of steps (batches of samples)
           before declaring predictions finished.
@@ -417,6 +419,7 @@ def experimental_tpu_test_loop(model,
       the display labels for the outputs.
   """
   current_strategy = model._distribution_strategy
+  iterator = distributed_training_utils.get_iterator(dataset, current_strategy)
   scope = current_strategy.scope()
   scope.__enter__()
 
@@ -520,12 +523,12 @@ def experimental_tpu_test_loop(model,
   return outs
 
 
-def experimental_tpu_predict_loop(model, iterator, verbose=0, steps=None):
+def experimental_tpu_predict_loop(model, dataset, verbose=0, steps=None):
   """Predict loop for predicting with TPU DistributionStrategy.
 
   Arguments:
       model: Keras Model instance.
-      iterator: Iterator for input data.
+      dataset: Dataset for input data.
       verbose: Integer, Verbosity mode 0 or 1.
       steps: Total number of steps (batches of samples)
           before declaring `_predict_loop` finished.
@@ -537,6 +540,7 @@ def experimental_tpu_predict_loop(model, iterator, verbose=0, steps=None):
       (if the model has multiple outputs).
   """
   current_strategy = model._distribution_strategy
+  iterator = distributed_training_utils.get_iterator(dataset, current_strategy)
   scope = current_strategy.scope()
   scope.__enter__()
 
diff --git a/tensorflow/python/keras/engine/training_eager_test.py b/tensorflow/python/keras/engine/training_eager_test.py
index a6e2c24ec2..84f1fa0efc 100644
--- a/tensorflow/python/keras/engine/training_eager_test.py
+++ b/tensorflow/python/keras/engine/training_eager_test.py
@@ -156,7 +156,7 @@ class TrainingTest(keras_parameterized.TestCase):
     dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).repeat(10).batch(5)
     iterator = dataset_ops.make_one_shot_iterator(dataset)
     validation_dataset = dataset_ops.Dataset.from_tensor_slices(
-        (x, y)).repeat(10).batch(5)
+        (x, y)).repeat().batch(5)  # Infinite dataset.
     validation_iterator = dataset_ops.make_one_shot_iterator(validation_dataset)
 
     with self.assertRaisesRegexp(
@@ -169,6 +169,7 @@ class TrainingTest(keras_parameterized.TestCase):
           ValueError, r'provide either `batch_size` or `validation_steps`'):
         model.fit(iterator, steps_per_epoch=2, epochs=1, verbose=0,
                   validation_data=(x, y))
+    # Step argument is required for infinite datasets.
     with self.assertRaisesRegexp(ValueError,
                                  'specify the `validation_steps` argument.'):
       model.fit(iterator, steps_per_epoch=2, epochs=1, verbose=0,
diff --git a/tensorflow/python/keras/engine/training_generator.py b/tensorflow/python/keras/engine/training_generator.py
index 07de934259..8a5e24856c 100644
--- a/tensorflow/python/keras/engine/training_generator.py
+++ b/tensorflow/python/keras/engine/training_generator.py
@@ -54,6 +54,7 @@ def model_iteration(model,
                     initial_epoch=0,
                     mode=ModeKeys.TRAIN,
                     batch_size=None,
+                    steps_name='steps',
                     **kwargs):
   """Loop function for arrays of data with modes TRAIN/TEST/PREDICT.
 
@@ -93,6 +94,9 @@ def model_iteration(model,
       mode: One of ModeKeys.TRAIN/ModeKeys.TEST/ModeKeys.PREDICT.
       batch_size: Integer batch size or None if unknown. Will only be used if
         `data` is in NumPy/Tensor format.
+      steps_name: The string name of the steps argument, either `steps`,
+        `validation_steps`, or `steps_per_epoch`. Only used for error message
+        formatting.
       **kwargs: Additional arguments for backwards compatibility. `steps` is
         accepted as an alias for `steps_per_epoch`.
 
@@ -107,6 +111,18 @@ def model_iteration(model,
   if 'steps' in kwargs:
     steps_per_epoch = kwargs['steps']
 
+  # Determine the number of steps per epoch and whether we should reset the
+  # dataset at the end of each epoch.
+  reset_dataset_after_each_epoch = False
+  original_dataset = None
+  is_dataset = isinstance(data, (dataset_ops.DatasetV2, dataset_ops.DatasetV1))
+  if is_dataset:
+    original_dataset = data
+    if steps_per_epoch is None:
+      reset_dataset_after_each_epoch = True
+      steps_per_epoch = training_utils.infer_steps_for_dataset(
+          data, steps_per_epoch, epochs=epochs, steps_name=steps_name)
+
   # Convert to a format that supports `next(generator)`.
   generator, steps_per_epoch = convert_to_generator_like(
       data,
@@ -118,7 +134,7 @@ def model_iteration(model,
   do_validation = validation_data is not None
   should_set_learning_phase = context.executing_eagerly() and model.run_eagerly
   is_sequence = isinstance(generator, data_utils.Sequence)
-  _validate_arguments(is_sequence, use_multiprocessing, workers,
+  _validate_arguments(is_sequence, is_dataset, use_multiprocessing, workers,
                       steps_per_epoch, validation_data, validation_steps, mode,
                       kwargs)
 
@@ -175,10 +191,36 @@ def model_iteration(model,
       callbacks.on_epoch_begin(epoch, epoch_logs)
     progbar.on_epoch_begin(epoch, epoch_logs)
 
-    for step in range(steps_per_epoch):
+    if steps_per_epoch is None:
+      # Loop over dataset until `OutOfRangeError` is raised.
+      target_steps = np.inf
+    else:
+      # Loop over dataset for the specified number of steps.
+      target_steps = steps_per_epoch
+
+    step = 0
+    while step < target_steps:
       batch_data = _get_next_batch(output_generator, mode)
       if batch_data is None:
-        callbacks.model.stop_training = True
+        if not is_dataset:
+          # We ran out of batches while the user passed an iterator (legacy).
+          logging.warning(
+              'Your dataset iterator ran out of data; '
+              'interrupting training. Make sure that your iterator '
+              'can generate at least `%s * epochs` '
+              'batches (in this case, %d batches). You may need to'
+              'use the repeat() function when building your '
+              'dataset.' % (steps_name, steps_per_epoch * epochs))
+          callbacks.model.stop_training = True
+        else:
+          # The dataset passed by the user ran out of batches.
+          # Now we know the cardinality of the dataset.
+          # assert steps_per_epoch is None
+          if step > 0:
+            steps_per_epoch = step
+            aggregator.num_samples_or_steps = steps_per_epoch
+            progbar.params['steps'] = steps_per_epoch
+            progbar.progbar.target = steps_per_epoch
         break
 
       # `batch_size` used for validation data if validation
@@ -203,6 +245,7 @@ def model_iteration(model,
       batch_logs = cbks.make_logs(model, batch_logs, batch_outs, mode)
       callbacks._call_batch_hook(mode, 'end', step, batch_logs)
       progbar.on_batch_end(step, batch_logs)
+      step += 1
 
       if callbacks.model.stop_training:
         break
@@ -226,7 +269,8 @@ def model_iteration(model,
           max_queue_size=max_queue_size,
           callbacks=callbacks,
           verbose=0,
-          mode=ModeKeys.TEST)
+          mode=ModeKeys.TEST,
+          steps_name='validation_steps')
 
       if not isinstance(val_results, list):
         val_results = [val_results]
@@ -236,7 +280,11 @@ def model_iteration(model,
     if mode == ModeKeys.TRAIN:
       # Epochs only apply to `fit`.
       callbacks.on_epoch_end(epoch, epoch_logs)
-      progbar.on_epoch_end(epoch, epoch_logs)
+    progbar.on_epoch_end(epoch, epoch_logs)
+
+    # Recreate dataset iterator for the next epoch.
+    if reset_dataset_after_each_epoch and epoch < epochs - 1:
+      generator = dataset_ops.make_one_shot_iterator(original_dataset)
 
   callbacks._call_end_hook(mode)
 
@@ -263,9 +311,7 @@ def _get_next_batch(output_generator, mode):
   """Retrieves the next batch of input data."""
   try:
     generator_output = next(output_generator)
-  except (errors.OutOfRangeError, StopIteration):
-    # Returning `None` will trigger looping to stop.
-    logging.warning('Your dataset iterator ran out of data.')
+  except (StopIteration, errors.OutOfRangeError):
     return None
   if not isinstance(generator_output, tuple):
     if mode == ModeKeys.PREDICT:
@@ -283,7 +329,7 @@ def _get_next_batch(output_generator, mode):
   return generator_output
 
 
-def _validate_arguments(is_sequence, use_multiprocessing, workers,
+def _validate_arguments(is_sequence, is_dataset, use_multiprocessing, workers,
                         steps_per_epoch, validation_data, validation_steps,
                         mode, kwargs):
   """Raises errors if arguments are invalid.
@@ -291,6 +337,7 @@ def _validate_arguments(is_sequence, use_multiprocessing, workers,
   Arguments:
     is_sequence: Boolean, whether data is a `keras.utils.data_utils.Sequence`
       instance.
+    is_dataset: Boolean, whether data is a dataset instance.
     use_multiprocessing: Boolean. If `True`, use process-based threading. If
       unspecified, `use_multiprocessing` will default to `False`. Note that
       because this implementation relies on multiprocessing, you should not pass
@@ -322,15 +369,14 @@ def _validate_arguments(is_sequence, use_multiprocessing, workers,
                     ' Please consider using the `keras.utils.Sequence`'
                     ' class.'))
 
-  if steps_per_epoch is None:
+  if steps_per_epoch is None and not is_dataset:
     arg_name = 'steps_per_epoch' if mode == ModeKeys.TRAIN else 'steps'
     raise ValueError('Please specify the number of steps via the '
                      '`{}` argument.'.format(arg_name))
 
   val_gen = (
       data_utils.is_generator_or_sequence(validation_data) or
-      isinstance(validation_data, iterator_ops.EagerIterator) or
-      isinstance(validation_data, dataset_ops.DatasetV2))
+      isinstance(validation_data, iterator_ops.EagerIterator))
   if (val_gen and not isinstance(validation_data, data_utils.Sequence) and
       not validation_steps):
     raise ValueError('Please specify the `validation_steps` argument.')
diff --git a/tensorflow/python/keras/engine/training_utils.py b/tensorflow/python/keras/engine/training_utils.py
index a4ee6c1ba5..0c3beb4681 100644
--- a/tensorflow/python/keras/engine/training_utils.py
+++ b/tensorflow/python/keras/engine/training_utils.py
@@ -25,6 +25,7 @@ import copy
 import numpy as np
 import six
 
+from tensorflow.python.data.experimental.ops import cardinality
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import iterator_ops
 from tensorflow.python.data.ops import readers
@@ -106,6 +107,8 @@ class MetricsAggregator(Aggregator):
     self.results[1:] = batch_outs[1:]
 
   def finalize(self):
+    if not self.results:
+      raise ValueError('Empty training data.')
     self.results[0] /= self.num_samples_or_steps
 
 
@@ -952,13 +955,12 @@ def check_steps_argument(input_data, steps, steps_name):
         but not provided.
   """
   # TODO(fchollet): allow datasets with steps=None if cardinality is known.
-  is_x_dataset = isinstance(input_data, (iterator_ops.Iterator,
-                                         iterator_ops.EagerIterator,
-                                         dataset_ops.DatasetV2))
-  if (input_data is None or is_x_dataset or has_symbolic_tensors(input_data) or
+  is_x_iterator = isinstance(input_data, (iterator_ops.Iterator,
+                                          iterator_ops.EagerIterator))
+  if (input_data is None or is_x_iterator or has_symbolic_tensors(input_data) or
       (isinstance(input_data, list) and not input_data)):
     if steps is None:
-      input_type_str = 'a Dataset' if is_x_dataset else 'data tensors'
+      input_type_str = 'a Dataset iterator' if is_x_iterator else 'data tensors'
       raise ValueError('When using {input_type} as input to a model, you should'
                        ' specify the `{steps_name}` argument.'.format(
                            input_type=input_type_str, steps_name=steps_name))
@@ -1208,8 +1210,6 @@ def is_dataset_or_iterator(data):
 def extract_tensors_from_dataset(dataset):
   """Extract a tuple of tensors `inputs, targets, sample_weight` from a dataset.
 
-  Works only for graph mode.
-
   Arguments:
     dataset: Dataset instance.
 
@@ -1217,7 +1217,9 @@ def extract_tensors_from_dataset(dataset):
     Tuple of tensors `x, y, weights`. `y` and `weights` entry may be None.
   """
   iterator = dataset_ops.make_initializable_iterator(dataset)
-  K.get_session().run(iterator.initializer)
+  init_op = iterator.initializer
+  if not context.executing_eagerly():
+    K.get_session().run(init_op)
   inputs, targets, sample_weight = unpack_iterator_input(iterator)
   return inputs, targets, sample_weight
 
@@ -1256,6 +1258,51 @@ def unpack_iterator_input(iterator):
   return x, y, weights
 
 
+def infer_steps_for_dataset(dataset, steps, epochs=1, steps_name='steps'):
+  """Infers steps_per_epoch needed to loop through a dataset.
+
+  Arguments:
+      dataset: Input data of type tf.data.Dataset.
+      steps: Number of steps to draw from the dataset (may be None if unknown).
+      epochs: Number of times to iterate over the dataset.
+      steps_name: The string name of the steps argument, either `steps`,
+        `validation_steps`, or `steps_per_epoch`. Only used for error message
+        formatting.
+
+  Returns:
+    Integer or `None`. Inferred number of steps to loop through the dataset.
+    `None` is returned if the size of the dataset is unknown and `steps` was
+    not specified.
+
+  Raises:
+    ValueError: In case of invalid argument values.
+  """
+  assert isinstance(dataset, dataset_ops.DatasetV2)
+  size = K.get_value(cardinality.cardinality(dataset))
+  if size == cardinality.INFINITE and steps is None:
+    raise ValueError('When passing an infinitely repeating dataset, you '
+                     'must specify the `%s` argument.' % (steps_name,))
+  if size != cardinality.UNKNOWN:
+    if steps is not None and steps * epochs > size:
+      if epochs > 1:
+        raise ValueError('The dataset you passed contains %s batches, but you '
+                         'passed `epochs=%s` and `%s=%s`, which is a total of '
+                         '%s steps. We cannot draw that many steps from this '
+                         'dataset. We suggest to set `%s=%s`.' %
+                         (size, epochs, steps_name, steps, steps * epochs,
+                          steps_name, size // epochs))
+      else:
+        raise ValueError('The dataset you passed contains %s batches, but you '
+                         'passed `%s=%s`. We cannot draw that many steps from '
+                         'this dataset. We suggest to set `%s=%s`.' %
+                         (size, steps_name, steps, steps_name, size))
+  if steps is None:
+    if size >= 0:
+      return size
+    return None
+  return steps
+
+
 class ModelInputs(object):
   """Encapsulates model inputs.
 
-- 
GitLab


From 7305fd64837a2137cd1ace2c548f390a348f2f67 Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Thu, 10 Jan 2019 18:58:10 -0800
Subject: [PATCH 0570/1095] Put all calibration resources into resource
 manager. This fixes a race condition triggered in int8 mode: during
 calibration, when two sess.run()s fetch two different sets of ops and both of
 them trigger the computation of the TRTEngineOp, two different TRTEngineOp
 will be created, and dev_tensors_ will only be allocated once by the first op
 by the device's resource manager via LookupOrCreate(). As a result, the
 second TRTEngineOp will have a empty dev_tensors_ vector which will result in
 std::out_of_range error when the op tries to access it later.

We should consider marking TRTEngineOp as stateful, the problem is it uses TF
functions which has to be stateless, but we can use function library as key to
cache them.

PiperOrigin-RevId: 228816949
---
 .../contrib/tensorrt/kernels/trt_engine_op.cc     | 15 +++++++++------
 .../contrib/tensorrt/kernels/trt_engine_op.h      |  6 ------
 tensorflow/contrib/tensorrt/ops/trt_engine_op.cc  |  3 +++
 .../contrib/tensorrt/resources/trt_resources.h    |  8 ++++++++
 4 files changed, 20 insertions(+), 12 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
index bad568644b..1f6409341c 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
+++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
@@ -175,11 +175,13 @@ void TRTEngineOp::ExecuteNativeSegment(OpKernelContext* ctx,
   lib->Run(opts, native_func_, inputs, outputs,
            [this, ctx, outputs, helper](const tensorflow::Status& s) {
              tensorflow::core::ScopedUnref sc(helper);
-             VLOG(1) << "Native Segment completed";
              if (!s.ok()) {
+               LOG(ERROR) << "Failed to execute native segment " << this->name()
+                          << ": " << s;
                ctx->SetStatus(s);
                return;
              }
+             VLOG(1) << "Native Segment completed";
              for (size_t t = 0; t < outputs->size(); ++t) {
                ctx->set_output(t, outputs->at(t));
              }
@@ -219,7 +221,8 @@ void TRTEngineOp::ExecuteCalibration(OpKernelContext* ctx,
       return;
     }
     // Check the allocated buffer is sufficient for input
-    const auto device_tensor = dev_tensors_.at(i).AccessTensor(ctx);
+    const auto device_tensor =
+        calib_res->device_tensors_.at(i).AccessTensor(ctx);
     CHECK_EQ(t.TotalBytes(), device_tensor->TotalBytes());
     input_data.emplace(StrCat(kInputPHName, i), data_address);
   }
@@ -536,7 +539,7 @@ tensorflow::Status TRTEngineOp::AllocateCalibrationResources(
   const int batch_size = ctx->input(0).dim_size(0);
   const int num_inputs = ctx->num_inputs();
   std::vector<tensorflow::PartialTensorShape> shapes;
-  dev_tensors_.resize(num_inputs);
+  cres->device_tensors_.resize(num_inputs);
   VLOG(1) << " Constructing calibrator";
   for (int i = 0; i < num_inputs; i++) {
     // allocate workspace on device for inputs
@@ -544,19 +547,19 @@ tensorflow::Status TRTEngineOp::AllocateCalibrationResources(
     shapes.emplace_back(t.shape());
     Tensor* device_tensor;
     TF_RETURN_IF_ERROR(ctx->allocate_persistent(
-        t.dtype(), t.shape(), &dev_tensors_.at(i), &device_tensor));
+        t.dtype(), t.shape(), &cres->device_tensors_.at(i), &device_tensor));
     CHECK_EQ(t.TotalBytes(), device_tensor->TotalBytes());
     void* device_address = GetTensorAddress(device_tensor);
     if (device_address == nullptr) {
       return tensorflow::errors::InvalidArgument(
           "Unsupported data type encountered in input ", i);
     }
-    device_buffers_.emplace(
+    cres->device_buffers_.emplace(
         StrCat(kInputPHName, i),
         std::pair<void*, size_t>(device_address, device_tensor->TotalBytes()));
   }
   cres->calibrator_.reset(
-      new TRTInt8Calibrator(device_buffers_, batch_size, name()));
+      new TRTInt8Calibrator(cres->device_buffers_, batch_size, name()));
   const string label(name());
   auto segment_graph = &segment_graph_;
   const int platform_gpu_id =
diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
index b545f497f3..ceb7483642 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
+++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
@@ -98,12 +98,6 @@ class TRTEngineOp : public AsyncOpKernel {
   // GraphDef representation of the segment.
   GraphDef segment_graph_;
 
-  // Lookup table for temporary staging areas of input tensors for calibration.
-  std::unordered_map<string, std::pair<void*, size_t>> device_buffers_;
-
-  // Temporary staging areas for calibration inputs.
-  std::vector<PersistentTensor> dev_tensors_;
-
   // Engine Precision mode.
   int precision_mode_;
 
diff --git a/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc b/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc
index 92405906eb..0f0688a280 100644
--- a/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc
+++ b/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc
@@ -28,6 +28,9 @@ namespace shape_inference {
 extern Status TRTEngineOpShapeInference(InferenceContext* c);
 }
 
+// TODO(laigd): consider making this op stateful. The only problem is it uses TF
+// function which has to be stateless, but we can use function library as the
+// key to cache the instantiated functions for different executor subgraphs.
 REGISTER_OP("TRTEngineOp")
     .Attr("serialized_segment: string")
     .Attr("input_shapes: list(shape)")
diff --git a/tensorflow/contrib/tensorrt/resources/trt_resources.h b/tensorflow/contrib/tensorrt/resources/trt_resources.h
index 8d877b392f..11930f3671 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_resources.h
+++ b/tensorflow/contrib/tensorrt/resources/trt_resources.h
@@ -20,12 +20,14 @@ limitations under the License.
 #include <sstream>
 #include <string>
 #include <thread>
+#include <unordered_map>
 #include <vector>
 
 #include "tensorflow/contrib/tensorrt/convert/utils.h"
 #include "tensorflow/contrib/tensorrt/log/trt_logger.h"
 #include "tensorflow/contrib/tensorrt/resources/trt_allocator.h"
 #include "tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.h"
+#include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/resource_mgr.h"
 
 #if GOOGLE_CUDA
@@ -62,6 +64,12 @@ class TRTCalibrationResource : public tensorflow::ResourceBase {
     return oss.str();
   }
 
+  // Lookup table for temporary staging areas of input tensors for calibration.
+  std::unordered_map<string, std::pair<void*, size_t>> device_buffers_;
+
+  // Temporary staging areas for calibration inputs.
+  std::vector<PersistentTensor> device_tensors_;
+
   std::unique_ptr<TRTInt8Calibrator> calibrator_;
   TrtUniquePtrType<nvinfer1::IBuilder> builder_;
   TrtUniquePtrType<nvinfer1::ICudaEngine> engine_;
-- 
GitLab


From ed7a8b4dc39abf4b3a82d0f8edda267caf2e5451 Mon Sep 17 00:00:00 2001
From: Francois Chollet <fchollet@google.com>
Date: Thu, 10 Jan 2019 19:05:20 -0800
Subject: [PATCH 0571/1095] Add logging of samples/steps count in each line in
 verbose=2 mode for fit method.

PiperOrigin-RevId: 228817708
---
 tensorflow/python/keras/utils/generic_utils.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/keras/utils/generic_utils.py b/tensorflow/python/keras/utils/generic_utils.py
index 454854618c..8b5cdadd45 100644
--- a/tensorflow/python/keras/utils/generic_utils.py
+++ b/tensorflow/python/keras/utils/generic_utils.py
@@ -391,9 +391,8 @@ class Progbar(object):
         sys.stdout.write('\n')
 
       if self.target is not None:
-        numdigits = int(np.floor(np.log10(self.target))) + 1
-        barstr = '%%%dd/%d [' % (numdigits, self.target)
-        bar = barstr % current
+        numdigits = int(np.log10(self.target)) + 1
+        bar = ('%' + str(numdigits) + 'd/%d') % (current, self.target)
         prog = float(current) / self.target
         prog_width = int(self.width * prog)
         if prog_width > 0:
@@ -456,7 +455,10 @@ class Progbar(object):
       sys.stdout.flush()
 
     elif self.verbose == 2:
-      if self.target is None or current >= self.target:
+      if self.target is not None and current >= self.target:
+        numdigits = int(np.log10(self.target)) + 1
+        count = ('%' + str(numdigits) + 'd/%d') % (current, self.target)
+        info = count + info
         for k in self._values_order:
           info += ' - %s:' % k
           avg = np.mean(self._values[k][0] / max(1, self._values[k][1]))
-- 
GitLab


From 03fb5daa9cc0c84c52ab800843dcef22580f64c2 Mon Sep 17 00:00:00 2001
From: Tim Shen <timshen@google.com>
Date: Thu, 10 Jan 2019 19:11:40 -0800
Subject: [PATCH 0572/1095] Simplify PrepareForConvolution: * Change the
 virtual functions overloaded on DeviceMemory<T> to a single function
 DeviceMemoryBase. * Create a non-virtual, template wrapper that takes
 DeviceMemory<T>.

PiperOrigin-RevId: 228818212
---
 tensorflow/stream_executor/cuda/cuda_dnn.cc | 325 ++++----------------
 tensorflow/stream_executor/cuda/cuda_dnn.h  | 167 +---------
 tensorflow/stream_executor/dnn.h            | 172 ++---------
 tensorflow/stream_executor/stream.cc        | 173 ++++++-----
 tensorflow/stream_executor/stream.h         |  10 +
 5 files changed, 216 insertions(+), 631 deletions(-)

diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.cc b/tensorflow/stream_executor/cuda/cuda_dnn.cc
index 60648075c0..7bae6c207e 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.cc
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.cc
@@ -995,9 +995,11 @@ cudnnDataType_t ToCudnnDataType(
     dnn::DataLayout data_layout = dnn::DataLayout::kBatchDepthYX) {
   switch (data_type) {
     case dnn::DataType::kFloat:
+      return CUDNN_DATA_FLOAT;
     case dnn::DataType::kDouble:
+      return CUDNN_DATA_DOUBLE;
     case dnn::DataType::kHalf:
-      return static_cast<cudnnDataType_t>(data_type);
+      return CUDNN_DATA_HALF;
     case dnn::DataType::kInt8:
       return data_layout == dnn::DataLayout::kBatchDepthYX4 ? CUDNN_DATA_INT8x4
                                                             : CUDNN_DATA_INT8;
@@ -1008,6 +1010,15 @@ cudnnDataType_t ToCudnnDataType(
   }
 }
 
+cudnnDataType_t ToCudnnDataType(dnn::DataType data_type,
+                                dnn::FilterLayout filter_layout) {
+  if (data_type == dnn::DataType::kInt8 &&
+      filter_layout == dnn::FilterLayout::kOutputInputYX4) {
+    return CUDNN_DATA_INT8x4;
+  }
+  return ToCudnnDataType(data_type);
+}
+
 template <typename T>
 cudnnDataType_t GetCudnnDataType(
     dnn::DataLayout data_layout = dnn::DataLayout::kBatchDepthYX) {
@@ -2815,30 +2826,60 @@ void LogCudaProto(const dnn::ConvolutionProto& conv, float profile_time_ms,
 
 }  // namespace
 
-template <class T>
-port::Status CudnnSupport::PrepareForConvolutionImpl(
-    Stream* stream, const dnn::BatchDescriptor& input_descriptor,
-    const DeviceMemory<T>& input_data,
+port::Status CudnnSupport::DoPrepareForConvolution(
+    dnn::ConvolutionKind kind, dnn::DataType element_type, Stream* stream,
+    const dnn::BatchDescriptor& input_descriptor, DeviceMemoryBase input_data,
     const dnn::FilterDescriptor& filter_descriptor,
-    const DeviceMemory<T>& filter_data,
+    DeviceMemoryBase filter_data, const dnn::BatchDescriptor& output_descriptor,
+    DeviceMemoryBase output_data,
     const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::BatchDescriptor& output_descriptor, DeviceMemory<T>* output_data,
-    dnn::DataType accumulator_type, ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
-    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-  cudnnDataType_t cudnn_type = GetCudnnDataType<T>();
-  CudnnTensorDescriptor input_nd(input_descriptor, cudnn_type);
-  CudnnTensorDescriptor output_nd(output_descriptor, cudnn_type);
-  CudnnFilterDescriptor filter(filter_descriptor, cudnn_type);
-  CudnnConvolutionDescriptor conv(convolution_descriptor,
-                                  ToCudnnDataType(accumulator_type));
+    ScratchAllocator* scratch_allocator, dnn::AlgorithmDesc* algorithm_desc,
+    DeviceMemory<uint8>* scratch_memory) {
+  CudnnTensorDescriptor input_nd(
+      input_descriptor,
+      ToCudnnDataType(element_type, input_descriptor.layout()));
+  CudnnFilterDescriptor filter_nd(
+      filter_descriptor,
+      ToCudnnDataType(element_type, filter_descriptor.layout()));
+  CudnnTensorDescriptor output_nd(
+      output_descriptor,
+      ToCudnnDataType(element_type, output_descriptor.layout()));
+  CudnnConvolutionDescriptor conv(
+      convolution_descriptor,
+      ToCudnnDataType(GetConvAccumulatorType(element_type)));
 
   auto cudnn = cudnn_->GetHandle(parent_, stream);
 
-  SE_ASSIGN_OR_RETURN(*algorithm_desc,
-                      GetCudnnConvolutionForwardAlgorithm(
-                          stream, cudnn, algorithm_config, input_nd, filter,
-                          conv, output_nd, scratch_allocator, scratch_memory));
+  switch (kind) {
+    case dnn::ConvolutionKind::FORWARD: {
+      SE_ASSIGN_OR_RETURN(
+          *algorithm_desc,
+          GetCudnnConvolutionForwardAlgorithm(
+              stream, cudnn, algorithm_config, input_nd, filter_nd, conv,
+              output_nd, scratch_allocator, scratch_memory));
+      break;
+    }
+    case dnn::ConvolutionKind::BACKWARD_DATA: {
+      SE_ASSIGN_OR_RETURN(
+          *algorithm_desc,
+          GetCudnnConvolutionBackwardDataAlgorithm(
+              stream, cudnn, algorithm_config, input_nd, filter_nd, conv,
+              output_nd, scratch_allocator, scratch_memory));
+      break;
+    }
+    case dnn::ConvolutionKind::BACKWARD_FILTER: {
+      SE_ASSIGN_OR_RETURN(
+          *algorithm_desc,
+          GetCudnnConvolutionBackwardFilterAlgorithm(
+              stream, cudnn, algorithm_config, input_nd, filter_nd, conv,
+              output_nd, scratch_allocator, scratch_memory));
+      break;
+    }
+    default:
+      return port::InternalError(
+          absl::StrCat("Unexpected convolution kind ", static_cast<int>(kind)));
+  }
 
   return port::Status::OK();
 }
@@ -3351,64 +3392,6 @@ port::Status CudnnSupport::DoBatchNormalizationBackwardImpl(
   return port::Status::OK();
 }
 
-bool CudnnSupport::PrepareForConvolution(
-    Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
-    const DeviceMemory<float>& input_data,
-    const dnn::FilterDescriptor& filter_descriptor,
-    const DeviceMemory<float>& filter_data,
-    const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<float>* output_data, ScratchAllocator* scratch_allocator,
-    const dnn::AlgorithmConfig& algorithm_config,
-    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-  return IsStatusOk(PrepareForConvolutionImpl<float>(
-                        stream, batch_descriptor, input_data, filter_descriptor,
-                        filter_data, convolution_descriptor, output_descriptor,
-                        output_data, dnn::DataType::kFloat, scratch_allocator,
-                        algorithm_config, algorithm_desc, scratch_memory),
-                    /*report_error=*/true);
-}
-
-bool CudnnSupport::PrepareForConvolution(
-    Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
-    const DeviceMemory<double>& input_data,
-    const dnn::FilterDescriptor& filter_descriptor,
-    const DeviceMemory<double>& filter_data,
-    const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<double>* output_data, ScratchAllocator* scratch_allocator,
-    const dnn::AlgorithmConfig& algorithm_config,
-    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-  return IsStatusOk(PrepareForConvolutionImpl<double>(
-                        stream, batch_descriptor, input_data, filter_descriptor,
-                        filter_data, convolution_descriptor, output_descriptor,
-                        output_data, dnn::DataType::kDouble, scratch_allocator,
-                        algorithm_config, algorithm_desc, scratch_memory),
-                    /*report_error=*/true);
-}
-
-bool CudnnSupport::PrepareForConvolution(
-    Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
-    const DeviceMemory<Eigen::half>& input_data,
-    const dnn::FilterDescriptor& filter_descriptor,
-    const DeviceMemory<Eigen::half>& filter_data,
-    const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<Eigen::half>* output_data, ScratchAllocator* scratch_allocator,
-    const dnn::AlgorithmConfig& algorithm_config,
-    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-  dnn::DataType acc_type =
-      CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()
-          ? dnn::DataType::kFloat
-          : dnn::DataType::kHalf;
-  return IsStatusOk(
-      PrepareForConvolutionImpl<Eigen::half>(
-          stream, batch_descriptor, input_data, filter_descriptor, filter_data,
-          convolution_descriptor, output_descriptor, output_data, acc_type,
-          scratch_allocator, algorithm_config, algorithm_desc, scratch_memory),
-      /*report_error=*/true);
-}
-
 bool CudnnSupport::DoConvolve(
     Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
     const DeviceMemory<float>& input_data,
@@ -3592,36 +3575,6 @@ bool CudnnSupport::DoTransformTensor(Stream* stream,
   return IsStatusOk(status, /*report_error=*/true);
 }
 
-template <class T>
-port::Status CudnnSupport::PrepareForConvolutionBackwardDataImpl(
-    Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
-    const DeviceMemory<T>& filter_data,
-    const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<T> backward_output_data,
-    const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::BatchDescriptor& input_descriptor,
-    DeviceMemory<T>* backward_input_data, dnn::DataType accumulator_type,
-    ScratchAllocator* scratch_allocator,
-    const dnn::AlgorithmConfig& algorithm_config,
-    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-  cudnnDataType_t cudnn_type = GetCudnnDataType<T>();
-  auto cudnn = cudnn_->GetHandle(parent_, stream);
-
-  CudnnTensorDescriptor out_back_nd(output_descriptor, cudnn_type);
-  CudnnTensorDescriptor in_back_nd(input_descriptor, cudnn_type);
-  CudnnFilterDescriptor filter(filter_descriptor, cudnn_type);
-  CudnnConvolutionDescriptor conv(convolution_descriptor,
-                                  ToCudnnDataType(accumulator_type));
-
-  SE_ASSIGN_OR_RETURN(
-      *algorithm_desc,
-      GetCudnnConvolutionBackwardDataAlgorithm(
-          stream, cudnn, algorithm_config, in_back_nd, filter, conv,
-          out_back_nd, scratch_allocator, scratch_memory));
-
-  return port::Status::OK();
-}
-
 template <class T>
 port::Status CudnnSupport::DoConvolveBackwardDataImpl(
     Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
@@ -3722,70 +3675,6 @@ port::Status CudnnSupport::DoConvolveBackwardDataImpl(
   return port::Status::OK();
 }
 
-bool CudnnSupport::PrepareForConvolutionBackwardData(
-    Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
-    const DeviceMemory<double>& filter_data,
-    const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<double> backward_output_data,
-    const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::BatchDescriptor& input_descriptor,
-    DeviceMemory<double>* backward_input_data,
-    ScratchAllocator* scratch_allocator,
-    const dnn::AlgorithmConfig& algorithm_config,
-    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-  return IsStatusOk(
-      PrepareForConvolutionBackwardDataImpl(
-          stream, filter_descriptor, filter_data, output_descriptor,
-          backward_output_data, convolution_descriptor, input_descriptor,
-          backward_input_data, dnn::DataType::kDouble, scratch_allocator,
-          algorithm_config, algorithm_desc, scratch_memory),
-      /*report_error=*/true);
-}
-
-bool CudnnSupport::PrepareForConvolutionBackwardData(
-    Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
-    const DeviceMemory<float>& filter_data,
-    const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<float> backward_output_data,
-    const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::BatchDescriptor& input_descriptor,
-    DeviceMemory<float>* backward_input_data,
-    ScratchAllocator* scratch_allocator,
-    const dnn::AlgorithmConfig& algorithm_config,
-    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-  return IsStatusOk(
-      PrepareForConvolutionBackwardDataImpl(
-          stream, filter_descriptor, filter_data, output_descriptor,
-          backward_output_data, convolution_descriptor, input_descriptor,
-          backward_input_data, dnn::DataType::kFloat, scratch_allocator,
-          algorithm_config, algorithm_desc, scratch_memory),
-      /*report_error=*/true);
-}
-
-bool CudnnSupport::PrepareForConvolutionBackwardData(
-    Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
-    const DeviceMemory<Eigen::half>& filter_data,
-    const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<Eigen::half> backward_output_data,
-    const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::BatchDescriptor& input_descriptor,
-    DeviceMemory<Eigen::half>* backward_input_data,
-    ScratchAllocator* scratch_allocator,
-    const dnn::AlgorithmConfig& algorithm_config,
-    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-  dnn::DataType acc_type =
-      CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()
-          ? dnn::DataType::kFloat
-          : dnn::DataType::kHalf;
-  return IsStatusOk(
-      PrepareForConvolutionBackwardDataImpl(
-          stream, filter_descriptor, filter_data, output_descriptor,
-          backward_output_data, convolution_descriptor, input_descriptor,
-          backward_input_data, acc_type, scratch_allocator, algorithm_config,
-          algorithm_desc, scratch_memory),
-      /*report_error=*/true);
-}
-
 bool CudnnSupport::DoConvolveBackwardData(
     Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
     const DeviceMemory<double>& filter_data,
@@ -3846,36 +3735,6 @@ bool CudnnSupport::DoConvolveBackwardData(
       /*report_error=*/!output_profile_result);
 }
 
-template <class T>
-port::Status CudnnSupport::PrepareForConvolutionBackwardFilterImpl(
-    Stream* stream, const dnn::BatchDescriptor& input_descriptor,
-    const DeviceMemory<T>& input_data,
-    const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<T> backward_output_data,
-    const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::FilterDescriptor& filter_descriptor,
-    DeviceMemory<T>* backward_filter_data, dnn::DataType accumulator_type,
-    ScratchAllocator* scratch_allocator,
-    const dnn::AlgorithmConfig& algorithm_config,
-    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-  cudnnDataType_t cudnn_type = GetCudnnDataType<T>();
-  auto cudnn = cudnn_->GetHandle(parent_, stream);
-
-  CudnnTensorDescriptor out_back_nd(output_descriptor, cudnn_type);
-  CudnnTensorDescriptor input_nd(input_descriptor, cudnn_type);
-  CudnnFilterDescriptor filter(filter_descriptor, cudnn_type);
-  CudnnConvolutionDescriptor conv(convolution_descriptor,
-                                  ToCudnnDataType(accumulator_type));
-
-  SE_ASSIGN_OR_RETURN(
-      *algorithm_desc,
-      GetCudnnConvolutionBackwardFilterAlgorithm(
-          stream, cudnn, algorithm_config, input_nd, filter, conv, out_back_nd,
-          scratch_allocator, scratch_memory));
-
-  return port::Status::OK();
-}
-
 template <class T>
 port::Status CudnnSupport::DoConvolveBackwardFilterImpl(
     Stream* stream, const dnn::BatchDescriptor& input_descriptor,
@@ -4013,70 +3872,6 @@ port::Status CudnnSupport::DoConvolveBackwardFilterImpl(
   return port::Status::OK();
 }
 
-bool CudnnSupport::PrepareForConvolutionBackwardFilter(
-    Stream* stream, const dnn::BatchDescriptor& input_descriptor,
-    const DeviceMemory<double>& input_data,
-    const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<double> backward_output_data,
-    const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::FilterDescriptor& filter_descriptor,
-    DeviceMemory<double>* backward_filter_data,
-    ScratchAllocator* scratch_allocator,
-    const dnn::AlgorithmConfig& algorithm_config,
-    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-  return IsStatusOk(
-      PrepareForConvolutionBackwardFilterImpl(
-          stream, input_descriptor, input_data, output_descriptor,
-          backward_output_data, convolution_descriptor, filter_descriptor,
-          backward_filter_data, dnn::DataType::kDouble, scratch_allocator,
-          algorithm_config, algorithm_desc, scratch_memory),
-      /*report_error=*/true);
-}
-
-bool CudnnSupport::PrepareForConvolutionBackwardFilter(
-    Stream* stream, const dnn::BatchDescriptor& input_descriptor,
-    const DeviceMemory<float>& input_data,
-    const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<float> backward_output_data,
-    const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::FilterDescriptor& filter_descriptor,
-    DeviceMemory<float>* backward_filter_data,
-    ScratchAllocator* scratch_allocator,
-    const dnn::AlgorithmConfig& algorithm_config,
-    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-  return IsStatusOk(
-      PrepareForConvolutionBackwardFilterImpl(
-          stream, input_descriptor, input_data, output_descriptor,
-          backward_output_data, convolution_descriptor, filter_descriptor,
-          backward_filter_data, dnn::DataType::kFloat, scratch_allocator,
-          algorithm_config, algorithm_desc, scratch_memory),
-      /*report_error=*/true);
-}
-
-bool CudnnSupport::PrepareForConvolutionBackwardFilter(
-    Stream* stream, const dnn::BatchDescriptor& input_descriptor,
-    const DeviceMemory<Eigen::half>& input_data,
-    const dnn::BatchDescriptor& output_descriptor,
-    DeviceMemory<Eigen::half> backward_output_data,
-    const dnn::ConvolutionDescriptor& convolution_descriptor,
-    const dnn::FilterDescriptor& filter_descriptor,
-    DeviceMemory<Eigen::half>* backward_filter_data,
-    ScratchAllocator* scratch_allocator,
-    const dnn::AlgorithmConfig& algorithm_config,
-    dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-  dnn::DataType acc_type =
-      CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()
-          ? dnn::DataType::kFloat
-          : dnn::DataType::kHalf;
-  return IsStatusOk(
-      PrepareForConvolutionBackwardFilterImpl(
-          stream, input_descriptor, input_data, output_descriptor,
-          backward_output_data, convolution_descriptor, filter_descriptor,
-          backward_filter_data, acc_type, scratch_allocator, algorithm_config,
-          algorithm_desc, scratch_memory),
-      /*report_error=*/true);
-}
-
 bool CudnnSupport::DoConvolveBackwardFilter(
     Stream* stream, const dnn::BatchDescriptor& input_descriptor,
     const DeviceMemory<double>& input_data,
diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.h b/tensorflow/stream_executor/cuda/cuda_dnn.h
index d7514981d5..383a99e118 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.h
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.h
@@ -258,43 +258,6 @@ class CudnnSupport : public dnn::DnnSupport {
       DeviceMemory<float>* scale_backprop,
       DeviceMemory<float>* offset_backprop) override;
 
-  bool PrepareForConvolution(
-      Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
-      const DeviceMemory<float>& input_data,
-      const dnn::FilterDescriptor& filter_descriptor,
-      const DeviceMemory<float>& filter_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<float>* output_data, ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory) override;
-
-  bool PrepareForConvolution(
-      Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
-      const DeviceMemory<double>& input_data,
-      const dnn::FilterDescriptor& filter_descriptor,
-      const DeviceMemory<double>& filter_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<double>* output_data, ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory) override;
-
-  bool PrepareForConvolution(
-      Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
-      const DeviceMemory<Eigen::half>& input_data,
-      const dnn::FilterDescriptor& filter_descriptor,
-      const DeviceMemory<Eigen::half>& filter_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<Eigen::half>* output_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory) override;
-
   bool DoConvolve(Stream* stream, const dnn::BatchDescriptor& batch_descriptor,
                   const DeviceMemory<float>& input_data,
                   const dnn::FilterDescriptor& filter_descriptor,
@@ -427,45 +390,6 @@ class CudnnSupport : public dnn::DnnSupport {
     return false;
   }
 
-  bool PrepareForConvolutionBackwardData(
-      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
-      const DeviceMemory<float>& filter_data,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<float> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& input_descriptor,
-      DeviceMemory<float>* backward_input_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory) override;
-
-  bool PrepareForConvolutionBackwardData(
-      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
-      const DeviceMemory<double>& filter_data,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<double> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& input_descriptor,
-      DeviceMemory<double>* backward_input_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory) override;
-
-  bool PrepareForConvolutionBackwardData(
-      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
-      const DeviceMemory<Eigen::half>& filter_data,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<Eigen::half> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& input_descriptor,
-      DeviceMemory<Eigen::half>* backward_input_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory) override;
-
   bool DoConvolveBackwardData(
       Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
       const DeviceMemory<double>& filter_data,
@@ -502,45 +426,6 @@ class CudnnSupport : public dnn::DnnSupport {
       DeviceMemory<uint8>* scratch_memory,
       dnn::ProfileResult* output_profile_result) override;
 
-  bool PrepareForConvolutionBackwardFilter(
-      Stream* stream, const dnn::BatchDescriptor& input_descriptor,
-      const DeviceMemory<double>& input_data,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<double> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::FilterDescriptor& filter_descriptor,
-      DeviceMemory<double>* backward_filter_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory) override;
-
-  bool PrepareForConvolutionBackwardFilter(
-      Stream* stream, const dnn::BatchDescriptor& input_descriptor,
-      const DeviceMemory<float>& input_data,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<float> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::FilterDescriptor& filter_descriptor,
-      DeviceMemory<float>* backward_filter_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory) override;
-
-  bool PrepareForConvolutionBackwardFilter(
-      Stream* stream, const dnn::BatchDescriptor& input_descriptor,
-      const DeviceMemory<Eigen::half>& input_data,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<Eigen::half> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::FilterDescriptor& filter_descriptor,
-      DeviceMemory<Eigen::half>* backward_filter_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc,
-      DeviceMemory<uint8>* scratch_memory) override;
-
   bool DoConvolveBackwardFilter(
       Stream* stream, const dnn::BatchDescriptor& input_descriptor,
       const DeviceMemory<double>& input_data,
@@ -791,19 +676,6 @@ class CudnnSupport : public dnn::DnnSupport {
       DeviceMemory<T>* x_backprop, DeviceMemory<U>* scale_backprop,
       DeviceMemory<U>* offset_backprop);
 
-  template <class T>
-  port::Status PrepareForConvolutionImpl(
-      Stream* stream, const dnn::BatchDescriptor& input_descriptor,
-      const DeviceMemory<T>& input_data,
-      const dnn::FilterDescriptor& filter_descriptor,
-      const DeviceMemory<T>& filter_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<T>* output_data, dnn::DataType accumulator_type,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory);
-
   template <class T>
   port::Status DoConvolveImpl(
       Stream* stream, const dnn::BatchDescriptor& input_descriptor,
@@ -834,19 +706,6 @@ class CudnnSupport : public dnn::DnnSupport {
       const dnn::AlgorithmConfig& algorithm_config,
       dnn::ProfileResult* output_profile_result);
 
-  template <class T>
-  port::Status PrepareForConvolutionBackwardDataImpl(
-      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
-      const DeviceMemory<T>& filter_data,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<T> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& input_descriptor,
-      DeviceMemory<T>* backward_input_data, dnn::DataType accumulator_type,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory);
-
   template <class T>
   port::Status DoConvolveBackwardDataImpl(
       Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
@@ -860,19 +719,6 @@ class CudnnSupport : public dnn::DnnSupport {
       DeviceMemory<uint8>* scratch_memory,
       dnn::ProfileResult* output_profile_result);
 
-  template <class T>
-  port::Status PrepareForConvolutionBackwardFilterImpl(
-      Stream* stream, const dnn::BatchDescriptor& input_descriptor,
-      const DeviceMemory<T>& input_data,
-      const dnn::BatchDescriptor& output_descriptor_in,
-      DeviceMemory<T> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::FilterDescriptor& filter_descriptor,
-      DeviceMemory<T>* backward_filter_data, dnn::DataType accumulator_type,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory);
-
   template <class T>
   port::Status DoConvolveBackwardFilterImpl(
       Stream* stream, const dnn::BatchDescriptor& input_descriptor,
@@ -938,6 +784,19 @@ class CudnnSupport : public dnn::DnnSupport {
       ScratchAllocator* workspace_allocator,
       dnn::ProfileResult* output_profile_result);
 
+ private:
+  port::Status DoPrepareForConvolution(
+      dnn::ConvolutionKind kind, dnn::DataType element_type, Stream* stream,
+      const dnn::BatchDescriptor& input_descriptor, DeviceMemoryBase input_data,
+      const dnn::FilterDescriptor& filter_descriptor,
+      DeviceMemoryBase filter_data,
+      const dnn::BatchDescriptor& output_descriptor,
+      DeviceMemoryBase output_data,
+      const dnn::ConvolutionDescriptor& convolution_descriptor,
+      const dnn::AlgorithmConfig& algorithm_config,
+      ScratchAllocator* scratch_allocator, dnn::AlgorithmDesc* algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory) override;
+
   SE_DISALLOW_COPY_AND_ASSIGN(CudnnSupport);
 };
 
diff --git a/tensorflow/stream_executor/dnn.h b/tensorflow/stream_executor/dnn.h
index f5a77d6525..f5eef29109 100644
--- a/tensorflow/stream_executor/dnn.h
+++ b/tensorflow/stream_executor/dnn.h
@@ -942,11 +942,7 @@ class VersionInfo {
 //   burden.
 // * Poor error handling: the API should return Status objects.
 //
-// Things worth trying:
-// * Move functions that are not actually common back to the backends. Then,
-//   callers may use dynamic_cast to access specific backends. This may not be
-//   that hard, as many of the callers are Stream::ThenXxx functions.
-// * Change all the returned bools to Status.
+// PrepareForConvolution is an example for how new APIs should be written.
 class DnnSupport {
  public:
   DnnSupport() {}
@@ -1187,50 +1183,24 @@ class DnnSupport {
     return false;
   }
 
-  virtual bool PrepareForConvolution(
-      Stream* stream, const BatchDescriptor& batch_descriptor,
-      const DeviceMemory<float>& input_data,
-      const FilterDescriptor& filter_descriptor,
-      const DeviceMemory<float>& filter_data,
-      const ConvolutionDescriptor& convolution_descriptor,
-      const BatchDescriptor& output_descriptor,
-      DeviceMemory<float>* output_data, ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-    *algorithm_desc = {};
-    *scratch_memory = {};
-    return true;
-  }
-
-  virtual bool PrepareForConvolution(
-      Stream* stream, const BatchDescriptor& batch_descriptor,
-      const DeviceMemory<double>& input_data,
+  template <typename ElementType>
+  port::Status PrepareForConvolution(
+      ConvolutionKind kind, Stream* stream,
+      const BatchDescriptor& batch_descriptor,
+      DeviceMemory<ElementType> input_data,
       const FilterDescriptor& filter_descriptor,
-      const DeviceMemory<double>& filter_data,
-      const ConvolutionDescriptor& convolution_descriptor,
+      DeviceMemory<ElementType> filter_data,
       const BatchDescriptor& output_descriptor,
-      DeviceMemory<double>* output_data, ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-    *algorithm_desc = {};
-    *scratch_memory = {};
-    return true;
-  }
-
-  virtual bool PrepareForConvolution(
-      Stream* stream, const BatchDescriptor& batch_descriptor,
-      const DeviceMemory<Eigen::half>& input_data,
-      const FilterDescriptor& filter_descriptor,
-      const DeviceMemory<Eigen::half>& filter_data,
+      DeviceMemory<ElementType> output_data,
       const ConvolutionDescriptor& convolution_descriptor,
-      const BatchDescriptor& output_descriptor,
-      DeviceMemory<Eigen::half>* output_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-    *algorithm_desc = {};
-    *scratch_memory = {};
-    return true;
+      const AlgorithmConfig& algorithm_config,
+      ScratchAllocator* scratch_allocator, AlgorithmDesc* algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory) {
+    return DoPrepareForConvolution(
+        kind, ToDataType<ElementType>::value, stream, batch_descriptor,
+        input_data, filter_descriptor, filter_data, output_descriptor,
+        output_data, convolution_descriptor, algorithm_config,
+        scratch_allocator, algorithm_desc, scratch_memory);
   }
 
   // Enqueues a single-precision convolution operation onto the stream.
@@ -1360,54 +1330,6 @@ class DnnSupport {
       const BatchDescriptor& output_descriptor,
       DeviceMemory<float>* output_data) = 0;
 
-  virtual bool PrepareForConvolutionBackwardData(
-      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
-      const DeviceMemory<float>& filter_data,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<float> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& input_descriptor,
-      DeviceMemory<float>* backward_input_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-    *algorithm_desc = {};
-    *scratch_memory = {};
-    return true;
-  }
-
-  virtual bool PrepareForConvolutionBackwardData(
-      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
-      const DeviceMemory<double>& filter_data,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<double> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& input_descriptor,
-      DeviceMemory<double>* backward_input_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-    *algorithm_desc = {};
-    *scratch_memory = {};
-    return true;
-  }
-
-  virtual bool PrepareForConvolutionBackwardData(
-      Stream* stream, const dnn::FilterDescriptor& filter_descriptor,
-      const DeviceMemory<Eigen::half>& filter_data,
-      const dnn::BatchDescriptor& output_descriptor,
-      DeviceMemory<Eigen::half> backward_output_data,
-      const dnn::ConvolutionDescriptor& convolution_descriptor,
-      const dnn::BatchDescriptor& input_descriptor,
-      DeviceMemory<Eigen::half>* backward_input_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-    *algorithm_desc = {};
-    *scratch_memory = {};
-    return true;
-  }
-
   // Enqueues a single-precision backward convolution (for data) operation onto
   // the stream.
   //
@@ -1468,54 +1390,6 @@ class DnnSupport {
       DeviceMemory<uint8>* scratch_memory,
       ProfileResult* output_profile_result) = 0;
 
-  virtual bool PrepareForConvolutionBackwardFilter(
-      Stream* stream, const BatchDescriptor& input_descriptor,
-      const DeviceMemory<float>& input_data,
-      const BatchDescriptor& output_descriptor,
-      DeviceMemory<float> backward_output_data,
-      const ConvolutionDescriptor& convolution_descriptor,
-      const FilterDescriptor& filter_descriptor,
-      DeviceMemory<float>* backward_filter_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-    *algorithm_desc = {};
-    *scratch_memory = {};
-    return true;
-  }
-
-  virtual bool PrepareForConvolutionBackwardFilter(
-      Stream* stream, const BatchDescriptor& input_descriptor,
-      const DeviceMemory<double>& input_data,
-      const BatchDescriptor& output_descriptor,
-      DeviceMemory<double> backward_output_data,
-      const ConvolutionDescriptor& convolution_descriptor,
-      const FilterDescriptor& filter_descriptor,
-      DeviceMemory<double>* backward_filter_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-    *algorithm_desc = {};
-    *scratch_memory = {};
-    return true;
-  }
-
-  virtual bool PrepareForConvolutionBackwardFilter(
-      Stream* stream, const BatchDescriptor& input_descriptor,
-      const DeviceMemory<Eigen::half>& input_data,
-      const BatchDescriptor& output_descriptor,
-      DeviceMemory<Eigen::half> backward_output_data,
-      const ConvolutionDescriptor& convolution_descriptor,
-      const FilterDescriptor& filter_descriptor,
-      DeviceMemory<Eigen::half>* backward_filter_data,
-      ScratchAllocator* scratch_allocator,
-      const dnn::AlgorithmConfig& algorithm_config,
-      dnn::AlgorithmDesc* algorithm_desc, DeviceMemory<uint8>* scratch_memory) {
-    *algorithm_desc = {};
-    *scratch_memory = {};
-    return true;
-  }
-
   // Enqueues a single-precision backward convolution (for filter) operation
   // onto the stream.
   //
@@ -2516,6 +2390,20 @@ class DnnSupport {
   }
 
  private:
+  virtual port::Status DoPrepareForConvolution(
+      ConvolutionKind kind, DataType element_type, Stream* stream,
+      const BatchDescriptor& batch_descriptor, DeviceMemoryBase input_data,
+      const FilterDescriptor& filter_descriptor, DeviceMemoryBase filter_data,
+      const BatchDescriptor& output_descriptor, DeviceMemoryBase output_data,
+      const ConvolutionDescriptor& convolution_descriptor,
+      const AlgorithmConfig& algorithm_config,
+      ScratchAllocator* scratch_allocator, AlgorithmDesc* algorithm_desc,
+      DeviceMemory<uint8>* scratch_memory) {
+    *algorithm_desc = {};
+    *scratch_memory = {};
+    return port::Status::OK();
+  }
+
   SE_DISALLOW_COPY_AND_ASSIGN(DnnSupport);
 };
 
diff --git a/tensorflow/stream_executor/stream.cc b/tensorflow/stream_executor/stream.cc
index 1befc18e19..9937861d88 100644
--- a/tensorflow/stream_executor/stream.cc
+++ b/tensorflow/stream_executor/stream.cc
@@ -551,10 +551,11 @@ Stream &Stream::ThenConvolveWithScratch(
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
       DeviceMemory<uint8> scratch_memory;
       dnn::AlgorithmDesc algorithm_desc;
-      CheckError(dnn->PrepareForConvolution(
-          this, input_descriptor, input_data, filter_descriptor, filter_data,
-          convolution_descriptor, output_descriptor, output, scratch_allocator,
-          dnn::AlgorithmConfig(), &algorithm_desc, &scratch_memory));
+      CheckStatus(dnn->PrepareForConvolution(
+          dnn::ConvolutionKind::FORWARD, this, input_descriptor, input_data,
+          filter_descriptor, filter_data, output_descriptor, *output,
+          convolution_descriptor, dnn::AlgorithmConfig(), scratch_allocator,
+          &algorithm_desc, &scratch_memory));
       CheckError(dnn->DoConvolve(
           this, input_descriptor, input_data, filter_descriptor, filter_data,
           convolution_descriptor, output_descriptor, output, algorithm_desc,
@@ -583,10 +584,11 @@ Stream &Stream::ThenConvolveWithScratch(
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
       DeviceMemory<uint8> scratch_memory;
       dnn::AlgorithmDesc algorithm_desc;
-      CheckError(dnn->PrepareForConvolution(
-          this, input_descriptor, input_data, filter_descriptor, filter_data,
-          convolution_descriptor, output_descriptor, output, scratch_allocator,
-          dnn::AlgorithmConfig(), &algorithm_desc, &scratch_memory));
+      CheckStatus(dnn->PrepareForConvolution(
+          dnn::ConvolutionKind::FORWARD, this, input_descriptor, input_data,
+          filter_descriptor, filter_data, output_descriptor, *output,
+          convolution_descriptor, dnn::AlgorithmConfig(), scratch_allocator,
+          &algorithm_desc, &scratch_memory));
       CheckError(dnn->DoConvolve(
           this, input_descriptor, input_data, filter_descriptor, filter_data,
           convolution_descriptor, output_descriptor, output, algorithm_desc,
@@ -770,10 +772,13 @@ Stream &Stream::ThenConvolveWithAlgorithm(
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
       DeviceMemory<uint8> scratch_memory;
       dnn::AlgorithmDesc algorithm_desc;
-      auto status = dnn->PrepareForConvolution(
-          this, input_descriptor, input_data, filter_descriptor, filter_data,
-          convolution_descriptor, output_descriptor, output, scratch_allocator,
-          algorithm_config, &algorithm_desc, &scratch_memory);
+      auto status =
+          dnn->PrepareForConvolution(
+                 dnn::ConvolutionKind::FORWARD, this, input_descriptor,
+                 input_data, filter_descriptor, filter_data, output_descriptor,
+                 *output, convolution_descriptor, algorithm_config,
+                 scratch_allocator, &algorithm_desc, &scratch_memory)
+              .ok();
       if (status) {
         status = dnn->DoConvolve(
             this, input_descriptor, input_data, filter_descriptor, filter_data,
@@ -809,10 +814,13 @@ Stream &Stream::ThenConvolveWithAlgorithm(
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
       DeviceMemory<uint8> scratch_memory;
       dnn::AlgorithmDesc algorithm_desc;
-      auto status = dnn->PrepareForConvolution(
-          this, input_descriptor, input_data, filter_descriptor, filter_data,
-          convolution_descriptor, output_descriptor, output, scratch_allocator,
-          algorithm_config, &algorithm_desc, &scratch_memory);
+      auto status =
+          dnn->PrepareForConvolution(
+                 dnn::ConvolutionKind::FORWARD, this, input_descriptor,
+                 input_data, filter_descriptor, filter_data, output_descriptor,
+                 *output, convolution_descriptor, algorithm_config,
+                 scratch_allocator, &algorithm_desc, &scratch_memory)
+              .ok();
       if (status) {
         status = dnn->DoConvolve(
             this, input_descriptor, input_data, filter_descriptor, filter_data,
@@ -848,10 +856,13 @@ Stream &Stream::ThenConvolveWithAlgorithm(
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
       DeviceMemory<uint8> scratch_memory;
       dnn::AlgorithmDesc algorithm_desc;
-      auto status = dnn->PrepareForConvolution(
-          this, input_descriptor, input_data, filter_descriptor, filter_data,
-          convolution_descriptor, output_descriptor, output, scratch_allocator,
-          algorithm_config, &algorithm_desc, &scratch_memory);
+      auto status =
+          dnn->PrepareForConvolution(
+                 dnn::ConvolutionKind::FORWARD, this, input_descriptor,
+                 input_data, filter_descriptor, filter_data, output_descriptor,
+                 *output, convolution_descriptor, algorithm_config,
+                 scratch_allocator, &algorithm_desc, &scratch_memory)
+              .ok();
       if (status) {
         status = dnn->DoConvolve(
             this, input_descriptor, input_data, filter_descriptor, filter_data,
@@ -1005,11 +1016,12 @@ Stream &Stream::ThenConvolveBackwardDataWithScratch(
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
       DeviceMemory<uint8> scratch_memory;
       dnn::AlgorithmDesc algorithm_desc;
-      CheckError(dnn->PrepareForConvolutionBackwardData(
-          this, filter_descriptor, filter_data, output_descriptor,
-          backward_output_data, convolution_descriptor, input_descriptor,
-          backward_input_data, scratch_allocator, dnn::AlgorithmConfig(),
-          &algorithm_desc, &scratch_memory));
+      CheckStatus(dnn->PrepareForConvolution(
+          dnn::ConvolutionKind::BACKWARD_DATA, this, input_descriptor,
+          *backward_input_data, filter_descriptor, filter_data,
+          output_descriptor, backward_output_data, convolution_descriptor,
+          dnn::AlgorithmConfig(), scratch_allocator, &algorithm_desc,
+          &scratch_memory));
       CheckError(dnn->DoConvolveBackwardData(
           this, filter_descriptor, filter_data, output_descriptor,
           backward_output_data, convolution_descriptor, input_descriptor,
@@ -1042,11 +1054,14 @@ Stream &Stream::ThenConvolveBackwardDataWithAlgorithm(
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
       DeviceMemory<uint8> scratch_memory;
       dnn::AlgorithmDesc algorithm_desc;
-      auto status = dnn->PrepareForConvolutionBackwardData(
-          this, filter_descriptor, filter_data, output_descriptor,
-          backward_output_data, convolution_descriptor, input_descriptor,
-          backward_input_data, scratch_allocator, algorithm_config,
-          &algorithm_desc, &scratch_memory);
+      auto status =
+          dnn->PrepareForConvolution(
+                 dnn::ConvolutionKind::BACKWARD_DATA, this, input_descriptor,
+                 *backward_input_data, filter_descriptor, filter_data,
+                 output_descriptor, backward_output_data,
+                 convolution_descriptor, algorithm_config, scratch_allocator,
+                 &algorithm_desc, &scratch_memory)
+              .ok();
       if (status) {
         status = dnn->DoConvolveBackwardData(
             this, filter_descriptor, filter_data, output_descriptor,
@@ -1084,11 +1099,14 @@ Stream &Stream::ThenConvolveBackwardDataWithAlgorithm(
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
       DeviceMemory<uint8> scratch_memory;
       dnn::AlgorithmDesc algorithm_desc;
-      auto status = dnn->PrepareForConvolutionBackwardData(
-          this, filter_descriptor, filter_data, output_descriptor,
-          backward_output_data, convolution_descriptor, input_descriptor,
-          backward_input_data, scratch_allocator, algorithm_config,
-          &algorithm_desc, &scratch_memory);
+      auto status =
+          dnn->PrepareForConvolution(
+                 dnn::ConvolutionKind::BACKWARD_DATA, this, input_descriptor,
+                 *backward_input_data, filter_descriptor, filter_data,
+                 output_descriptor, backward_output_data,
+                 convolution_descriptor, algorithm_config, scratch_allocator,
+                 &algorithm_desc, &scratch_memory)
+              .ok();
       if (status) {
         status = dnn->DoConvolveBackwardData(
             this, filter_descriptor, filter_data, output_descriptor,
@@ -1126,11 +1144,14 @@ Stream &Stream::ThenConvolveBackwardDataWithAlgorithm(
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
       DeviceMemory<uint8> scratch_memory;
       dnn::AlgorithmDesc algorithm_desc;
-      auto status = dnn->PrepareForConvolutionBackwardData(
-          this, filter_descriptor, filter_data, output_descriptor,
-          backward_output_data, convolution_descriptor, input_descriptor,
-          backward_input_data, scratch_allocator, algorithm_config,
-          &algorithm_desc, &scratch_memory);
+      auto status =
+          dnn->PrepareForConvolution(
+                 dnn::ConvolutionKind::BACKWARD_DATA, this, input_descriptor,
+                 *backward_input_data, filter_descriptor, filter_data,
+                 output_descriptor, backward_output_data,
+                 convolution_descriptor, algorithm_config, scratch_allocator,
+                 &algorithm_desc, &scratch_memory)
+              .ok();
       if (status) {
         status = dnn->DoConvolveBackwardData(
             this, filter_descriptor, filter_data, output_descriptor,
@@ -1166,11 +1187,12 @@ Stream &Stream::ThenConvolveBackwardDataWithScratch(
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
       DeviceMemory<uint8> scratch_memory;
       dnn::AlgorithmDesc algorithm_desc;
-      CheckError(dnn->PrepareForConvolutionBackwardData(
-          this, filter_descriptor, filter_data, output_descriptor,
-          backward_output_data, convolution_descriptor, input_descriptor,
-          backward_input_data, scratch_allocator, dnn::AlgorithmConfig(),
-          &algorithm_desc, &scratch_memory));
+      CheckStatus(dnn->PrepareForConvolution(
+          dnn::ConvolutionKind::BACKWARD_DATA, this, input_descriptor,
+          *backward_input_data, filter_descriptor, filter_data,
+          output_descriptor, backward_output_data, convolution_descriptor,
+          dnn::AlgorithmConfig(), scratch_allocator, &algorithm_desc,
+          &scratch_memory));
       CheckError(dnn->DoConvolveBackwardData(
           this, filter_descriptor, filter_data, output_descriptor,
           backward_output_data, convolution_descriptor, input_descriptor,
@@ -1215,11 +1237,12 @@ Stream &Stream::ThenConvolveBackwardFilterWithScratch(
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
       DeviceMemory<uint8> scratch_memory;
       dnn::AlgorithmDesc algorithm_desc;
-      CheckError(dnn->PrepareForConvolutionBackwardFilter(
-          this, input_descriptor, input_data, output_descriptor,
-          backward_output_data, convolution_descriptor, filter_descriptor,
-          backward_filter_data, scratch_allocator, dnn::AlgorithmConfig(),
-          &algorithm_desc, &scratch_memory));
+      CheckStatus(dnn->PrepareForConvolution(
+          dnn::ConvolutionKind::BACKWARD_FILTER, this, input_descriptor,
+          input_data, filter_descriptor, *backward_filter_data,
+          output_descriptor, backward_output_data, convolution_descriptor,
+          dnn::AlgorithmConfig(), scratch_allocator, &algorithm_desc,
+          &scratch_memory));
       CheckError(dnn->DoConvolveBackwardFilter(
           this, input_descriptor, input_data, output_descriptor,
           backward_output_data, convolution_descriptor, filter_descriptor,
@@ -1252,11 +1275,14 @@ Stream &Stream::ThenConvolveBackwardFilterWithAlgorithm(
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
       DeviceMemory<uint8> scratch_memory;
       dnn::AlgorithmDesc algorithm_desc;
-      auto status = dnn->PrepareForConvolutionBackwardFilter(
-          this, input_descriptor, input_data, output_descriptor,
-          backward_output_data, convolution_descriptor, filter_descriptor,
-          backward_filter_data, scratch_allocator, algorithm_config,
-          &algorithm_desc, &scratch_memory);
+      auto status =
+          dnn->PrepareForConvolution(
+                 dnn::ConvolutionKind::BACKWARD_FILTER, this, input_descriptor,
+                 input_data, filter_descriptor, *backward_filter_data,
+                 output_descriptor, backward_output_data,
+                 convolution_descriptor, algorithm_config, scratch_allocator,
+                 &algorithm_desc, &scratch_memory)
+              .ok();
       if (status) {
         status = dnn->DoConvolveBackwardFilter(
             this, input_descriptor, input_data, output_descriptor,
@@ -1294,11 +1320,14 @@ Stream &Stream::ThenConvolveBackwardFilterWithAlgorithm(
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
       DeviceMemory<uint8> scratch_memory;
       dnn::AlgorithmDesc algorithm_desc;
-      auto status = dnn->PrepareForConvolutionBackwardFilter(
-          this, input_descriptor, input_data, output_descriptor,
-          backward_output_data, convolution_descriptor, filter_descriptor,
-          backward_filter_data, scratch_allocator, algorithm_config,
-          &algorithm_desc, &scratch_memory);
+      auto status =
+          dnn->PrepareForConvolution(
+                 dnn::ConvolutionKind::BACKWARD_FILTER, this, input_descriptor,
+                 input_data, filter_descriptor, *backward_filter_data,
+                 output_descriptor, backward_output_data,
+                 convolution_descriptor, algorithm_config, scratch_allocator,
+                 &algorithm_desc, &scratch_memory)
+              .ok();
       if (status) {
         status = dnn->DoConvolveBackwardFilter(
             this, input_descriptor, input_data, output_descriptor,
@@ -1334,11 +1363,12 @@ Stream &Stream::ThenConvolveBackwardFilterWithScratch(
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
       DeviceMemory<uint8> scratch_memory;
       dnn::AlgorithmDesc algorithm_desc;
-      CheckError(dnn->PrepareForConvolutionBackwardFilter(
-          this, input_descriptor, input_data, output_descriptor,
-          backward_output_data, convolution_descriptor, filter_descriptor,
-          backward_filter_data, scratch_allocator, dnn::AlgorithmConfig(),
-          &algorithm_desc, &scratch_memory));
+      CheckStatus(dnn->PrepareForConvolution(
+          dnn::ConvolutionKind::BACKWARD_FILTER, this, input_descriptor,
+          input_data, filter_descriptor, *backward_filter_data,
+          output_descriptor, backward_output_data, convolution_descriptor,
+          dnn::AlgorithmConfig(), scratch_allocator, &algorithm_desc,
+          &scratch_memory));
       CheckError(dnn->DoConvolveBackwardFilter(
           this, input_descriptor, input_data, output_descriptor,
           backward_output_data, convolution_descriptor, filter_descriptor,
@@ -1371,11 +1401,14 @@ Stream &Stream::ThenConvolveBackwardFilterWithAlgorithm(
     if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
       DeviceMemory<uint8> scratch_memory;
       dnn::AlgorithmDesc algorithm_desc;
-      auto status = dnn->PrepareForConvolutionBackwardFilter(
-          this, input_descriptor, input_data, output_descriptor,
-          backward_output_data, convolution_descriptor, filter_descriptor,
-          backward_filter_data, scratch_allocator, algorithm_config,
-          &algorithm_desc, &scratch_memory);
+      auto status =
+          dnn->PrepareForConvolution(
+                 dnn::ConvolutionKind::BACKWARD_FILTER, this, input_descriptor,
+                 input_data, filter_descriptor, *backward_filter_data,
+                 output_descriptor, backward_output_data,
+                 convolution_descriptor, algorithm_config, scratch_allocator,
+                 &algorithm_desc, &scratch_memory)
+              .ok();
       if (status) {
         status = dnn->DoConvolveBackwardFilter(
             this, input_descriptor, input_data, output_descriptor,
diff --git a/tensorflow/stream_executor/stream.h b/tensorflow/stream_executor/stream.h
index d5e2fdf58d..85d65c1061 100644
--- a/tensorflow/stream_executor/stream.h
+++ b/tensorflow/stream_executor/stream.h
@@ -2099,6 +2099,16 @@ class Stream {
     ok_ = false;
   }
 
+  // Checks the status and logs the error message, if any.
+  void CheckStatus(port::Status status) LOCKS_EXCLUDED(mu_) {
+    if (status.ok()) {
+      return;
+    }
+    LOG(ERROR) << status;
+    mutex_lock lock(mu_);
+    ok_ = false;
+  }
+
   void SetError() { CheckError(false /* = operation_retcode */); }
 
   void SetErrorAndLogNoDnnSupport() {
-- 
GitLab


From 96b579c4d2f592b8183b34abe6cb155e7796809c Mon Sep 17 00:00:00 2001
From: Aurelien Geron <aurelien.geron@gmail.com>
Date: Fri, 11 Jan 2019 11:53:21 +0800
Subject: [PATCH 0573/1095] Fix issue in function_test.py: Function to
 ConcreteFunction

---
 tensorflow/python/eager/function_test.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py
index 3bb4f2452c..73692e3efc 100644
--- a/tensorflow/python/eager/function_test.py
+++ b/tensorflow/python/eager/function_test.py
@@ -1729,8 +1729,8 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
       return x
 
     graph_function = foo.get_concrete_function(constant_op.constant(1.0))
-    with self.assertRaisesRegexp(ValueError, 'All inputs to `Function`s must '
-                                 'be Tensors;.*'):
+    with self.assertRaisesRegexp(
+        ValueError, 'All inputs to `ConcreteFunction`s must be Tensors;.*'):
       graph_function('Not a Tensor.')
 
   def testSwapImplementationWithGrapplerPlugin(self):
-- 
GitLab


From 764109a35299f55f7da5cbf6d3d8fc01a06668b6 Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Thu, 10 Jan 2019 20:35:37 -0800
Subject: [PATCH 0574/1095] Reenable test cases that are broken by TRT 4.0
 since TF is now built against TRT 5.0.

PiperOrigin-RevId: 228824151
---
 .../tensorrt/test/batch_matmul_test.py        | 22 -------------------
 .../tensorrt/test/biasadd_matmul_test.py      |  6 -----
 .../tensorrt/test/identity_output_test.py     |  6 -----
 .../contrib/tensorrt/test/rank_two_test.py    |  6 -----
 4 files changed, 40 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/test/batch_matmul_test.py b/tensorflow/contrib/tensorrt/test/batch_matmul_test.py
index f42308ecb7..384b84cb02 100644
--- a/tensorflow/contrib/tensorrt/test/batch_matmul_test.py
+++ b/tensorflow/contrib/tensorrt/test/batch_matmul_test.py
@@ -86,28 +86,6 @@ class BatchMatMulTest(trt_test.TfTrtIntegrationTestBase):
     """Return the expected engines to run."""
     return ["TRTEngineOp_1"]
 
-  def ShouldRunTest(self, run_params):
-    """Whether to run the test."""
-    # TODO(aaroey): Trt library will fail like:
-    #
-    # ../builder/cudnnBuilder2.cpp:685:
-    # virtual std::vector<nvinfer1::query::Ports<
-    #     nvinfer1::query::TensorRequirements>>
-    # nvinfer1::builder::Node::getSupportedFormats(
-    #     const nvinfer1::query::Ports<nvinfer1::query::AbstractTensor>&,
-    #     const nvinfer1::cudnn::HardwareContext&,
-    #     nvinfer1::builder::Format::Type,
-    #     const nvinfer1::builder::FormatTypeHack&) const:
-    # Assertion `sf' failed.
-    #
-    # To reproduce, run:
-    # bazel test -c opt --copt=-mavx \
-    #   --test_arg=BatchMatMulTest.testTfTrt_ToolConversion_INT8_DynamicEngine \
-    #   tensorflow/contrib/tensorrt:batch_matmul_test
-    #
-    # Investigate and fix it.
-    return not trt_test.IsQuantizationMode(run_params.precision_mode)
-
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py b/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py
index 053b38ff1c..49f5e4cc9f 100644
--- a/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py
+++ b/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py
@@ -130,12 +130,6 @@ class BiasaddMatMulTest(trt_test.TfTrtIntegrationTestBase):
     """Return the expected engines to build."""
     return ["TRTEngineOp_0"]
 
-  def ShouldRunTest(self, run_params):
-    """Whether to run the test."""
-    # TODO(aaroey): Trt 4.0 forbids conversion for tensors with rank <3 in int8
-    # mode, which is a bug. Re-enable this when trt library is fixed.
-    return not trt_test.IsQuantizationMode(run_params.precision_mode)
-
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/tensorrt/test/identity_output_test.py b/tensorflow/contrib/tensorrt/test/identity_output_test.py
index d1c6dd5204..196ccdbe9d 100644
--- a/tensorflow/contrib/tensorrt/test/identity_output_test.py
+++ b/tensorflow/contrib/tensorrt/test/identity_output_test.py
@@ -69,12 +69,6 @@ class IdentityTest(trt_test.TfTrtIntegrationTestBase):
     """Return the expected engines to build."""
     return ['TRTEngineOp_0']
 
-  def ShouldRunTest(self, run_params):
-    """Whether to run the test."""
-    # TODO(aaroey): Trt 4.0 forbids conversion for tensors with rank <3 in int8
-    # mode, which is a bug. Re-enable this when trt library is fixed.
-    return not trt_test.IsQuantizationMode(run_params.precision_mode)
-
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/contrib/tensorrt/test/rank_two_test.py b/tensorflow/contrib/tensorrt/test/rank_two_test.py
index 563232fc12..dd3b72d212 100644
--- a/tensorflow/contrib/tensorrt/test/rank_two_test.py
+++ b/tensorflow/contrib/tensorrt/test/rank_two_test.py
@@ -80,12 +80,6 @@ class RankTwoTest(trt_test.TfTrtIntegrationTestBase):
         ],
     }
 
-  def ShouldRunTest(self, run_params):
-    """Whether to run the test."""
-    # TODO(aaroey): Trt 4.0 forbids conversion for tensors with rank <3 in int8
-    # mode, which is a bug. Re-enable this when trt library is fixed.
-    return not trt_test.IsQuantizationMode(run_params.precision_mode)
-
 
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From 9d1679d8f3fbc76f2b2a74cce7e8014f8d4543bc Mon Sep 17 00:00:00 2001
From: Aurelien Geron <aurelien.geron@gmail.com>
Date: Fri, 11 Jan 2019 13:13:49 +0800
Subject: [PATCH 0575/1095] Rename `SavedPolymorphicFunction` to `SavedFunction
 and `MonomorphicFunction` to `ConcreteFunction` and `concrete_function`
 attribute (in `ConcreteFunction` proto) to `name`

---
 .../python/saved_model/function_deserialization.py     | 10 +++++-----
 .../python/saved_model/function_serialization.py       |  8 ++++----
 tensorflow/python/saved_model/load.py                  |  8 ++++----
 tensorflow/python/saved_model/saved_object_graph.proto | 10 +++++-----
 4 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/tensorflow/python/saved_model/function_deserialization.py b/tensorflow/python/saved_model/function_deserialization.py
index f1c43cb7a4..41694fb749 100644
--- a/tensorflow/python/saved_model/function_deserialization.py
+++ b/tensorflow/python/saved_model/function_deserialization.py
@@ -55,10 +55,10 @@ def _inputs_compatible(args, stored_inputs):
 
 
 def recreate_function(saved_function, concrete_functions):
-  """Creates a `Function` from a `SavedPolymorphicFunction`.
+  """Creates a `Function` from a `SavedFunction`.
 
   Args:
-    saved_function: `SavedPolymorphicFunction` proto.
+    saved_function: `SavedFunction` proto.
     concrete_functions: map from function name to `ConcreteFunction`.
 
   Returns:
@@ -80,10 +80,10 @@ def recreate_function(saved_function, concrete_functions):
     """Calls a restored function."""
     # TODO(allenl): Functions saved with input_signatures should revive with
     # input_signatures.
-    for monomorphic_function in saved_function.monomorphic_function:
-      function_obj = concrete_functions[monomorphic_function.concrete_function]
+    for concrete_function in saved_function.concrete_function:
+      function_obj = concrete_functions[concrete_function.name]
       canonicalized_original_inputs = coder.decode_proto(
-          monomorphic_function.canonicalized_input)
+          concrete_function.canonicalized_input)
 
       try:
         can_args, can_kwargs = function_spec.canonicalize_function_inputs(
diff --git a/tensorflow/python/saved_model/function_serialization.py b/tensorflow/python/saved_model/function_serialization.py
index b645a7c964..2868ca6f66 100644
--- a/tensorflow/python/saved_model/function_serialization.py
+++ b/tensorflow/python/saved_model/function_serialization.py
@@ -25,9 +25,9 @@ from tensorflow.python.saved_model import saved_object_graph_pb2
 
 
 def serialize_function(function, node_ids):
-  """Build a SavedPolymorphicFunction proto."""
+  """Build a SavedFunction proto."""
   coder = nested_structure_coder.StructureCoder()
-  proto = saved_object_graph_pb2.SavedPolymorphicFunction()
+  proto = saved_object_graph_pb2.SavedFunction()
 
   proto.function_spec_tuple.CopyFrom(
       coder.encode_structure(function.function_spec.as_tuple()))  # pylint: disable=protected-access
@@ -43,8 +43,8 @@ def serialize_function(function, node_ids):
           "captures tensor %s which is unsupported or not reachable from root.",
           concrete_function.name, capture)
       continue
-    function_proto = proto.monomorphic_function.add()
-    function_proto.concrete_function = concrete_function.name
+    function_proto = proto.concrete_function.add()
+    function_proto.name = concrete_function.name
     function_proto.canonicalized_input.CopyFrom(
         coder.encode_structure(signature))
     function_proto.bound_inputs.extend(bound_inputs)
diff --git a/tensorflow/python/saved_model/load.py b/tensorflow/python/saved_model/load.py
index ae71a228ea..8c4f1c9f74 100644
--- a/tensorflow/python/saved_model/load.py
+++ b/tensorflow/python/saved_model/load.py
@@ -54,14 +54,14 @@ class _Loader(object):
     seen_concrete_functions = set()
     for object_proto in self._proto.nodes:
       if object_proto.WhichOneof("kind") == "function":
-        for monomorphic_function in object_proto.function.monomorphic_function:
-          name = monomorphic_function.concrete_function
+        for concrete_function in object_proto.function.concrete_function:
+          name = concrete_function.name
           bound_inputs = [
               self._get_tensor_from_node(node_id)
-              for node_id in monomorphic_function.bound_inputs]
+              for node_id in concrete_function.bound_inputs]
           bound_variables = [
               self._nodes[node_id]
-              for node_id in monomorphic_function.bound_inputs
+              for node_id in concrete_function.bound_inputs
               if self._proto.nodes[node_id].WhichOneof("kind") == "variable"
           ]
           if name in seen_concrete_functions:
diff --git a/tensorflow/python/saved_model/saved_object_graph.proto b/tensorflow/python/saved_model/saved_object_graph.proto
index 1e2514b7f7..6b44a75e3f 100644
--- a/tensorflow/python/saved_model/saved_object_graph.proto
+++ b/tensorflow/python/saved_model/saved_object_graph.proto
@@ -51,7 +51,7 @@ message SavedObject {
   oneof kind {
     SavedUserObject user_object = 4;
     SavedAsset asset = 5;
-    SavedPolymorphicFunction function = 6;
+    SavedFunction function = 6;
     SavedVariable variable = 7;
   }
 }
@@ -78,16 +78,16 @@ message SavedAsset {
 }
 
 // A function with multiple signatures, possibly with non-Tensor arguments.
-message SavedPolymorphicFunction {
-  repeated SavedMonomorphicFunction monomorphic_function = 1;
+message SavedFunction {
+  repeated SavedConcreteFunction concrete_function = 1;
   // Tuple representing a `FunctionSpec`.
   // TODO(vbardiovsky): Make this a proto.
   StructuredValue function_spec_tuple = 2;
 }
 
-message SavedMonomorphicFunction {
+message SavedConcreteFunction {
   // A reference to a TensorFlow function in the MetaGraph's FunctionDefLibrary
-  string concrete_function = 1;
+  string name = 1;
 
   // Bound inputs to the function. The SavedObjects identified by the node ids
   // given here are appended as extra inputs to the caller-supplied inputs.
-- 
GitLab


From 489f1dc4573eabbff59c149031984ea8319a7bf8 Mon Sep 17 00:00:00 2001
From: Penporn Koanantakool <penporn@google.com>
Date: Thu, 10 Jan 2019 22:07:22 -0800
Subject: [PATCH 0576/1095] Fix typo.

PiperOrigin-RevId: 228831583
---
 configure.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configure.py b/configure.py
index bb59063f79..adc9ef9cac 100644
--- a/configure.py
+++ b/configure.py
@@ -1694,7 +1694,7 @@ def main():
   config_info_line('noaws', 'Disable AWS S3 filesystem support.')
   config_info_line('nogcp', 'Disable GCP support.')
   config_info_line('nohdfs', 'Disable HDFS support.')
-  config_info_line('noignite', 'Disable Apacha Ignite support.')
+  config_info_line('noignite', 'Disable Apache Ignite support.')
   config_info_line('nokafka', 'Disable Apache Kafka support.')
   config_info_line('nonccl', 'Disable NVIDIA NCCL support.')
 
-- 
GitLab


From 30afb286a47474dc68b61e572028e3b96b43af6a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 10 Jan 2019 22:23:11 -0800
Subject: [PATCH 0577/1095] Split off SeparableConv and ConvTranspose tests and
 parameterize many test so they can be properly sharded.

PiperOrigin-RevId: 228832550
---
 tensorflow/python/keras/BUILD                 |  30 +-
 .../python/keras/layers/convolutional_test.py | 547 ++++--------------
 .../layers/convolutional_transpose_test.py    | 209 +++++++
 .../layers/separable_convolutional_test.py    | 167 ++++++
 4 files changed, 502 insertions(+), 451 deletions(-)
 create mode 100644 tensorflow/python/keras/layers/convolutional_transpose_test.py
 create mode 100644 tensorflow/python/keras/layers/separable_convolutional_test.py

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 5f44a1b4e8..ef8cb23a09 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -487,9 +487,9 @@ tf_py_test(
     shard_count = 2,
 )
 
-tf_py_test(
+cuda_py_test(
     name = "convolutional_test",
-    size = "large",
+    size = "medium",
     srcs = ["layers/convolutional_test.py"],
     additional_deps = [
         ":keras",
@@ -497,7 +497,19 @@ tf_py_test(
         "//third_party/py/numpy",
         "//tensorflow/python:client_testlib",
     ],
-    shard_count = 11,
+    shard_count = 8,
+)
+
+cuda_py_test(
+    name = "convolutional_transpose_test",
+    size = "medium",
+    srcs = ["layers/convolutional_transpose_test.py"],
+    additional_deps = [
+        ":keras",
+        "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
+    ],
 )
 
 cuda_py_test(
@@ -659,6 +671,18 @@ tf_py_test(
     shard_count = 4,
 )
 
+cuda_py_test(
+    name = "separable_convolutional_test",
+    size = "medium",
+    srcs = ["layers/separable_convolutional_test.py"],
+    additional_deps = [
+        ":keras",
+        "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
+    ],
+)
+
 cuda_py_test(
     name = "unified_lstm_test",
     size = "medium",
diff --git a/tensorflow/python/keras/layers/convolutional_test.py b/tensorflow/python/keras/layers/convolutional_test.py
index eedf66ff67..9140ce426e 100644
--- a/tensorflow/python/keras/layers/convolutional_test.py
+++ b/tensorflow/python/keras/layers/convolutional_test.py
@@ -18,8 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import copy
-
+from absl.testing import parameterized
 import numpy as np
 
 from tensorflow.python import keras
@@ -30,39 +29,32 @@ from tensorflow.python.platform import test
 
 
 @keras_parameterized.run_all_keras_modes
-class Convolution1DTest(keras_parameterized.TestCase):
+class Conv1DTest(keras_parameterized.TestCase):
 
-  def _run_test(self, kwargs, arg, values):
+  def _run_test(self, kwargs):
     num_samples = 2
     stack_size = 3
     length = 7
 
-    test_kwargs = copy.copy(kwargs)
-    for value in values:
-      test_kwargs[arg] = value
-      with self.cached_session(use_gpu=True):
-        testing_utils.layer_test(
-            keras.layers.Conv1D,
-            kwargs=test_kwargs,
-            input_shape=(num_samples, length, stack_size))
-
-  def test_conv1d(self):
-    kwargs = {
-        'filters': 2,
-        'kernel_size': 3,
-    }
-
-    self._run_test(kwargs, 'padding', ['valid', 'same', 'causal'])
-    self._run_test(kwargs, 'strides', [2])
-    self._run_test(kwargs, 'dilation_rate', [2])
-
-    kwargs = {
-        'filters': 2,
-        'kernel_size': 3,
-        'padding': 'same',
-    }
-    self._run_test(kwargs, 'dilation_rate', [2])
-    self._run_test(kwargs, 'dilation_rate', [3])
+    with self.cached_session(use_gpu=True):
+      testing_utils.layer_test(
+          keras.layers.Conv1D,
+          kwargs=kwargs,
+          input_shape=(num_samples, length, stack_size))
+
+  @parameterized.named_parameters(
+      ('padding_valid', {'padding': 'valid'}),
+      ('padding_same', {'padding': 'same'}),
+      ('padding_same_dilation_2', {'padding': 'same', 'dilation_rate': 2}),
+      ('padding_same_dilation_3', {'padding': 'same', 'dilation_rate': 3}),
+      ('padding_causal', {'padding': 'causal'}),
+      ('strides', {'strides': 2}),
+      ('dilation_rate', {'dilation_rate': 2}),
+  )
+  def test_conv1d(self, kwargs):
+    kwargs['filters'] = 2
+    kwargs['kernel_size'] = 3
+    self._run_test(kwargs)
 
   def test_conv1d_regularizers(self):
     kwargs = {
@@ -74,7 +66,7 @@ class Convolution1DTest(keras_parameterized.TestCase):
         'activity_regularizer': 'l2',
         'strides': 1
     }
-    with self.session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       layer = keras.layers.Conv1D(**kwargs)
       layer.build((None, 5, 2))
       self.assertEqual(len(layer.losses), 2)
@@ -93,7 +85,7 @@ class Convolution1DTest(keras_parameterized.TestCase):
         'bias_constraint': b_constraint,
         'strides': 1
     }
-    with self.session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       layer = keras.layers.Conv1D(**kwargs)
       layer.build((None, 5, 2))
       self.assertEqual(layer.kernel.constraint, k_constraint)
@@ -103,41 +95,33 @@ class Convolution1DTest(keras_parameterized.TestCase):
 @keras_parameterized.run_all_keras_modes
 class Conv2DTest(keras_parameterized.TestCase):
 
-  def _run_test(self, kwargs, arg, values):
+  def _run_test(self, kwargs):
     num_samples = 2
     stack_size = 3
     num_row = 7
     num_col = 6
 
-    test_kwargs = copy.copy(kwargs)
-    for value in values:
-      test_kwargs[arg] = value
-      with self.cached_session(use_gpu=True):
-        testing_utils.layer_test(
-            keras.layers.Conv2D,
-            kwargs=test_kwargs,
-            input_shape=(num_samples, num_row, num_col, stack_size))
-
-  def test_conv2d(self):
-    kwargs = {
-        'filters': 2,
-        'kernel_size': (3, 3),
-    }
-
-    self._run_test(kwargs, 'padding', ['valid', 'same'])
-    self._run_test(kwargs, 'strides', [(2, 2)])
-    if test.is_gpu_available(cuda_only=True):
+    with self.cached_session(use_gpu=True):
+      testing_utils.layer_test(
+          keras.layers.Conv2D,
+          kwargs=kwargs,
+          input_shape=(num_samples, num_row, num_col, stack_size))
+
+  @parameterized.named_parameters(
+      ('padding_valid', {'padding': 'valid'}),
+      ('padding_same', {'padding': 'same'}),
+      ('padding_same_dilation_2', {'padding': 'same', 'dilation_rate': 2}),
+      ('strides', {'strides': (2, 2)}),
+      ('dilation_rate', {'dilation_rate': (2, 2)}),
       # Only runs on GPU with CUDA, channels_first is not supported on CPU.
       # TODO(b/62340061): Support channels_first on CPU.
-      self._run_test(kwargs, 'data_format', ['channels_first'])
-    self._run_test(kwargs, 'dilation_rate', [(2, 2)])
-
-    kwargs = {
-        'filters': 2,
-        'kernel_size': 3,
-        'padding': 'same',
-    }
-    self._run_test(kwargs, 'dilation_rate', [2])
+      ('data_format', {'data_format': 'channels_first'}),
+  )
+  def test_conv2d(self, kwargs):
+    kwargs['filters'] = 2
+    kwargs['kernel_size'] = (3, 3)
+    if 'data_format' not in kwargs or test.is_gpu_available(cuda_only=True):
+      self._run_test(kwargs)
 
   def test_conv2d_regularizers(self):
     kwargs = {
@@ -149,7 +133,7 @@ class Conv2DTest(keras_parameterized.TestCase):
         'activity_regularizer': 'l2',
         'strides': 1
     }
-    with self.session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       layer = keras.layers.Conv2D(**kwargs)
       layer.build((None, 5, 5, 2))
       self.assertEqual(len(layer.losses), 2)
@@ -168,381 +152,43 @@ class Conv2DTest(keras_parameterized.TestCase):
         'bias_constraint': b_constraint,
         'strides': 1
     }
-    with self.session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       layer = keras.layers.Conv2D(**kwargs)
       layer.build((None, 5, 5, 2))
       self.assertEqual(layer.kernel.constraint, k_constraint)
       self.assertEqual(layer.bias.constraint, b_constraint)
 
 
-@keras_parameterized.run_all_keras_modes
-class Conv2DTransposeTest(keras_parameterized.TestCase):
-
-  def _run_test(self, kwargs, arg, values):
-    num_samples = 2
-    stack_size = 3
-    num_row = 7
-    num_col = 6
-
-    test_kwargs = copy.copy(kwargs)
-    for value in values:
-      test_kwargs[arg] = value
-      with self.cached_session(use_gpu=True):
-        testing_utils.layer_test(
-            keras.layers.Conv2DTranspose,
-            kwargs=test_kwargs,
-            input_shape=(num_samples, num_row, num_col, stack_size))
-
-  def test_conv2dtranspose(self):
-    kwargs = {
-        'filters': 2,
-        'kernel_size': (3, 3),
-    }
-
-    self._run_test(kwargs, 'padding', ['valid', 'same'])
-    self._run_test(kwargs, 'strides', [(2, 2)])
-    if test.is_gpu_available(cuda_only=True):
-      self._run_test(kwargs, 'data_format', ['channels_first'])
-
-    kwargs['strides'] = (2, 2)
-    self._run_test(kwargs, 'output_padding', [(1, 1)])
-
-  def test_conv2dtranspose_regularizers(self):
-    kwargs = {
-        'filters': 3,
-        'kernel_size': 3,
-        'padding': 'valid',
-        'kernel_regularizer': 'l2',
-        'bias_regularizer': 'l2',
-        'activity_regularizer': 'l2',
-        'strides': 1
-    }
-    with self.session(use_gpu=True):
-      layer = keras.layers.Conv2DTranspose(**kwargs)
-      layer.build((None, 5, 5, 2))
-      self.assertEqual(len(layer.losses), 2)
-      layer(keras.backend.variable(np.ones((1, 5, 5, 2))))
-      self.assertEqual(len(layer.losses), 3)
-
-  def test_conv2dtranspose_constraints(self):
-    k_constraint = lambda x: x
-    b_constraint = lambda x: x
-
-    kwargs = {
-        'filters': 3,
-        'kernel_size': 3,
-        'padding': 'valid',
-        'kernel_constraint': k_constraint,
-        'bias_constraint': b_constraint,
-        'strides': 1
-    }
-    with self.session(use_gpu=True):
-      layer = keras.layers.Conv2DTranspose(**kwargs)
-      layer.build((None, 5, 5, 2))
-      self.assertEqual(layer.kernel.constraint, k_constraint)
-      self.assertEqual(layer.bias.constraint, b_constraint)
-
-  def test_conv2d_transpose_dilation(self):
-    testing_utils.layer_test(keras.layers.Conv2DTranspose,
-                             kwargs={'filters': 2,
-                                     'kernel_size': 3,
-                                     'padding': 'same',
-                                     'data_format': 'channels_last',
-                                     'dilation_rate': (2, 2)},
-                             input_shape=(2, 5, 6, 3))
-
-    input_data = np.arange(48).reshape((1, 4, 4, 3)).astype(np.float32)
-    expected_output = np.float32([[192, 228, 192, 228],
-                                  [336, 372, 336, 372],
-                                  [192, 228, 192, 228],
-                                  [336, 372, 336, 372]]).reshape((1, 4, 4, 1))
-    testing_utils.layer_test(keras.layers.Conv2DTranspose,
-                             input_data=input_data,
-                             kwargs={'filters': 1,
-                                     'kernel_size': 3,
-                                     'padding': 'same',
-                                     'data_format': 'channels_last',
-                                     'dilation_rate': (2, 2),
-                                     'kernel_initializer': 'ones'},
-                             expected_output=expected_output)
-
-
-@keras_parameterized.run_all_keras_modes
-class Conv3DTransposeTest(keras_parameterized.TestCase):
-
-  def _run_test(self, kwargs, arg, values):
-    num_samples = 2
-    stack_size = 3
-    num_row = 7
-    num_col = 6
-    depth = 5
-
-    test_kwargs = copy.copy(kwargs)
-    for value in values:
-      test_kwargs[arg] = value
-      with self.cached_session(use_gpu=True):
-        testing_utils.layer_test(
-            keras.layers.Conv3DTranspose,
-            kwargs=test_kwargs,
-            input_shape=(num_samples, depth, num_row, num_col, stack_size))
-
-  def test_conv3dtranspose(self):
-    kwargs = {
-        'filters': 2,
-        'kernel_size': (3, 3, 3),
-    }
-
-    self._run_test(kwargs, 'padding', ['valid', 'same'])
-    self._run_test(kwargs, 'strides', [(2, 2, 2)])
-    if test.is_gpu_available(cuda_only=True):
-      self._run_test(kwargs, 'data_format', ['channels_first'])
-
-    kwargs['strides'] = (2, 2, 2)
-    self._run_test(kwargs, 'output_padding', [(1, 1, 1)])
-
-  def test_conv3dtranspose_regularizers(self):
-    kwargs = {
-        'filters': 3,
-        'kernel_size': 3,
-        'padding': 'valid',
-        'kernel_regularizer': 'l2',
-        'bias_regularizer': 'l2',
-        'activity_regularizer': 'l2',
-        'strides': 1
-    }
-    with self.session(use_gpu=True):
-      layer = keras.layers.Conv3DTranspose(**kwargs)
-      layer.build((None, 5, 5, 5, 2))
-      self.assertEqual(len(layer.losses), 2)
-      layer(keras.backend.variable(np.ones((1, 5, 5, 5, 2))))
-      self.assertEqual(len(layer.losses), 3)
-
-  def test_conv3dtranspose_constraints(self):
-    k_constraint = lambda x: x
-    b_constraint = lambda x: x
-
-    kwargs = {
-        'filters': 3,
-        'kernel_size': 3,
-        'padding': 'valid',
-        'kernel_constraint': k_constraint,
-        'bias_constraint': b_constraint,
-        'strides': 1
-    }
-    with self.session(use_gpu=True):
-      layer = keras.layers.Conv3DTranspose(**kwargs)
-      layer.build((None, 5, 5, 5, 2))
-      self.assertEqual(layer.kernel.constraint, k_constraint)
-      self.assertEqual(layer.bias.constraint, b_constraint)
-
-  def test_conv3dtranspose_dynamic_shape(self):
-    input_data = np.random.random((1, 3, 3, 3, 3)).astype(np.float32)
-    with self.session(use_gpu=True):
-      # Won't raise error here.
-      testing_utils.layer_test(
-          keras.layers.Conv3DTranspose,
-          kwargs={
-              'data_format': 'channels_last',
-              'filters': 3,
-              'kernel_size': 3
-          },
-          input_shape=(None, None, None, None, 3),
-          input_data=input_data)
-      if test.is_gpu_available(cuda_only=True):
-        testing_utils.layer_test(
-            keras.layers.Conv3DTranspose,
-            kwargs={
-                'data_format': 'channels_first',
-                'filters': 3,
-                'kernel_size': 3
-            },
-            input_shape=(None, 3, None, None, None),
-            input_data=input_data)
-
-
-@keras_parameterized.run_all_keras_modes
-class SeparableConv1DTest(keras_parameterized.TestCase):
-
-  def _run_test(self, kwargs, arg, values):
-    num_samples = 2
-    stack_size = 3
-    length = 7
-
-    test_kwargs = copy.copy(kwargs)
-    for value in values:
-      test_kwargs[arg] = value
-      with self.cached_session(use_gpu=True):
-        testing_utils.layer_test(
-            keras.layers.SeparableConv1D,
-            kwargs=test_kwargs,
-            input_shape=(num_samples, length, stack_size))
-
-  def test_separable_conv1d(self):
-    kwargs = {
-        'filters': 2,
-        'kernel_size': 3,
-    }
-
-    self._run_test(kwargs, 'padding', ['valid', 'same', 'causal'])
-    self._run_test(kwargs, 'strides', [2])
-    self._run_test(kwargs, 'dilation_rate', [2])
-    self._run_test(kwargs, 'depth_multiplier', [2])
-
-    kwargs = {
-        'filters': 2,
-        'kernel_size': 3,
-        'padding': 'same',
-    }
-    self._run_test(kwargs, 'dilation_rate', [2])
-
-  def test_separable_conv1d_regularizers(self):
-    kwargs = {
-        'filters': 3,
-        'kernel_size': 3,
-        'padding': 'valid',
-        'depthwise_regularizer': 'l2',
-        'pointwise_regularizer': 'l2',
-        'bias_regularizer': 'l2',
-        'activity_regularizer': 'l2',
-        'strides': 1
-    }
-    with self.session(use_gpu=True):
-      layer = keras.layers.SeparableConv1D(**kwargs)
-      layer.build((None, 5, 2))
-      self.assertEqual(len(layer.losses), 3)
-      layer(keras.backend.variable(np.ones((1, 5, 2))))
-      self.assertEqual(len(layer.losses), 4)
-
-  def test_separable_conv1d_constraints(self):
-    d_constraint = lambda x: x
-    p_constraint = lambda x: x
-    b_constraint = lambda x: x
-
-    kwargs = {
-        'filters': 3,
-        'kernel_size': 3,
-        'padding': 'valid',
-        'pointwise_constraint': p_constraint,
-        'depthwise_constraint': d_constraint,
-        'bias_constraint': b_constraint,
-        'strides': 1
-    }
-    with self.session(use_gpu=True):
-      layer = keras.layers.SeparableConv1D(**kwargs)
-      layer.build((None, 5, 2))
-      self.assertEqual(layer.depthwise_kernel.constraint, d_constraint)
-      self.assertEqual(layer.pointwise_kernel.constraint, p_constraint)
-      self.assertEqual(layer.bias.constraint, b_constraint)
-
-
-@keras_parameterized.run_all_keras_modes
-class SeparableConv2DTest(keras_parameterized.TestCase):
-
-  def _run_test(self, kwargs, arg, values):
-    num_samples = 2
-    stack_size = 3
-    num_row = 7
-    num_col = 6
-
-    test_kwargs = copy.copy(kwargs)
-    for value in values:
-      test_kwargs[arg] = value
-      with self.cached_session(use_gpu=True):
-        testing_utils.layer_test(
-            keras.layers.SeparableConv2D,
-            kwargs=test_kwargs,
-            input_shape=(num_samples, num_row, num_col, stack_size))
-
-  def test_separable_conv2d(self):
-    kwargs = {
-        'filters': 2,
-        'kernel_size': 3,
-    }
-
-    self._run_test(kwargs, 'padding', ['valid', 'same'])
-    self._run_test(kwargs, 'strides', [2])
-    if test.is_gpu_available(cuda_only=True):
-      self._run_test(kwargs, 'data_format', ['channels_first'])
-    self._run_test(kwargs, 'dilation_rate', [2])
-    self._run_test(kwargs, 'depth_multiplier', [2])
-
-    kwargs = {
-        'filters': 2,
-        'kernel_size': 3,
-        'padding': 'same',
-    }
-    self._run_test(kwargs, 'dilation_rate', [2])
-
-  def test_separable_conv2d_regularizers(self):
-    kwargs = {
-        'filters': 3,
-        'kernel_size': 3,
-        'padding': 'valid',
-        'depthwise_regularizer': 'l2',
-        'pointwise_regularizer': 'l2',
-        'bias_regularizer': 'l2',
-        'activity_regularizer': 'l2',
-        'strides': 1
-    }
-    with self.session(use_gpu=True):
-      layer = keras.layers.SeparableConv2D(**kwargs)
-      layer.build((None, 5, 5, 2))
-      self.assertEqual(len(layer.losses), 3)
-      layer(keras.backend.variable(np.ones((1, 5, 5, 2))))
-      self.assertEqual(len(layer.losses), 4)
-
-  def test_separable_conv2d_constraints(self):
-    d_constraint = lambda x: x
-    p_constraint = lambda x: x
-    b_constraint = lambda x: x
-
-    kwargs = {
-        'filters': 3,
-        'kernel_size': 3,
-        'padding': 'valid',
-        'pointwise_constraint': p_constraint,
-        'depthwise_constraint': d_constraint,
-        'bias_constraint': b_constraint,
-        'strides': 1
-    }
-    with self.session(use_gpu=True):
-      layer = keras.layers.SeparableConv2D(**kwargs)
-      layer.build((None, 5, 5, 2))
-      self.assertEqual(layer.depthwise_kernel.constraint, d_constraint)
-      self.assertEqual(layer.pointwise_kernel.constraint, p_constraint)
-      self.assertEqual(layer.bias.constraint, b_constraint)
-
-
 @keras_parameterized.run_all_keras_modes
 class Conv3DTest(keras_parameterized.TestCase):
 
-  def _run_test(self, kwargs, arg, values):
+  def _run_test(self, kwargs):
     num_samples = 2
     stack_size = 3
     num_row = 7
     num_col = 6
     depth = 5
 
-    test_kwargs = copy.copy(kwargs)
-    for value in values:
-      test_kwargs[arg] = value
-      with self.cached_session(use_gpu=True):
-        testing_utils.layer_test(
-            keras.layers.Conv3D,
-            kwargs=test_kwargs,
-            input_shape=(num_samples, depth, num_row, num_col, stack_size))
-
-  def test_conv3d(self):
-    kwargs = {
-        'filters': 2,
-        'kernel_size': (3, 3, 3),
-    }
-
-    self._run_test(kwargs, 'padding', ['valid', 'same'])
-    self._run_test(kwargs, 'strides', [(2, 2, 2)])
-    self._run_test(kwargs, 'dilation_rate', [(2, 2, 2)])
-    if test.is_gpu_available(cuda_only=True):
-      self._run_test(kwargs, 'data_format', ['channels_first'])
+    with self.cached_session(use_gpu=True):
+      testing_utils.layer_test(
+          keras.layers.Conv3D,
+          kwargs=kwargs,
+          input_shape=(num_samples, depth, num_row, num_col, stack_size))
+
+  @parameterized.named_parameters(
+      ('padding_valid', {'padding': 'valid'}),
+      ('padding_same', {'padding': 'same'}),
+      ('strides', {'strides': (2, 2, 2)}),
+      ('dilation_rate', {'dilation_rate': (2, 2, 2)}),
+      # Only runs on GPU with CUDA, channels_first is not supported on CPU.
+      # TODO(b/62340061): Support channels_first on CPU.
+      ('data_format', {'data_format': 'channels_first'}),
+  )
+  def test_conv3d(self, kwargs):
+    kwargs['filters'] = 2
+    kwargs['kernel_size'] = (3, 3, 3)
+    if 'data_format' not in kwargs or test.is_gpu_available(cuda_only=True):
+      self._run_test(kwargs)
 
   def test_conv3d_regularizers(self):
     kwargs = {
@@ -554,7 +200,7 @@ class Conv3DTest(keras_parameterized.TestCase):
         'activity_regularizer': 'l2',
         'strides': 1
     }
-    with self.session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       layer = keras.layers.Conv3D(**kwargs)
       layer.build((None, 5, 5, 5, 2))
       self.assertEqual(len(layer.losses), 2)
@@ -574,7 +220,7 @@ class Conv3DTest(keras_parameterized.TestCase):
         'bias_constraint': b_constraint,
         'strides': 1
     }
-    with self.session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       layer = keras.layers.Conv3D(**kwargs)
       layer.build((None, 5, 5, 5, 2))
       self.assertEqual(layer.kernel.constraint, k_constraint)
@@ -582,7 +228,7 @@ class Conv3DTest(keras_parameterized.TestCase):
 
   def test_conv3d_dynamic_shape(self):
     input_data = np.random.random((1, 3, 3, 3, 3)).astype(np.float32)
-    with self.session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       # Won't raise error here.
       testing_utils.layer_test(
           keras.layers.Conv3D,
@@ -615,7 +261,7 @@ class ZeroPaddingTest(keras_parameterized.TestCase):
     shape = (num_samples, num_steps, input_dim)
     inputs = np.ones(shape)
 
-    with self.session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       # basic test
       testing_utils.layer_test(
           keras.layers.ZeroPadding1D,
@@ -746,7 +392,7 @@ class ZeroPaddingTest(keras_parameterized.TestCase):
     inputs = np.ones((num_samples, input_len_dim1, input_len_dim2,
                       input_len_dim3, stack_size))
 
-    with self.session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       # basic test
       testing_utils.layer_test(
           keras.layers.ZeroPadding3D,
@@ -778,7 +424,7 @@ class ZeroPaddingTest(keras_parameterized.TestCase):
 class UpSamplingTest(keras_parameterized.TestCase):
 
   def test_upsampling_1d(self):
-    with self.session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       testing_utils.layer_test(
           keras.layers.UpSampling1D, kwargs={'size': 2}, input_shape=(3, 5, 4))
 
@@ -932,7 +578,7 @@ class CroppingTest(keras_parameterized.TestCase):
     input_len_dim1 = 2
     inputs = np.random.rand(num_samples, time_length, input_len_dim1)
 
-    with self.session(use_gpu=True):
+    with self.cached_session(use_gpu=True):
       testing_utils.layer_test(
           keras.layers.Cropping1D,
           kwargs={'cropping': (2, 2)},
@@ -1068,30 +714,34 @@ class CroppingTest(keras_parameterized.TestCase):
 @keras_parameterized.run_all_keras_modes
 class DepthwiseConv2DTest(keras_parameterized.TestCase):
 
-  def _run_test(self, kwargs, arg, values):
+  def _run_test(self, kwargs):
     num_samples = 2
     stack_size = 3
     num_row = 7
     num_col = 6
 
-    test_kwargs = copy.copy(kwargs)
-    for value in values:
-      test_kwargs[arg] = value
-      with self.cached_session(use_gpu=True):
-        testing_utils.layer_test(
-            keras.layers.DepthwiseConv2D,
-            kwargs=test_kwargs,
-            input_shape=(num_samples, num_row, num_col, stack_size))
-
-  def test_depthwise_conv2d(self):
-    kwargs = {'kernel_size': (3, 3)}
-
-    self._run_test(kwargs, 'padding', ['valid', 'same'])
-    self._run_test(kwargs, 'strides', [(2, 2)])
-    if test.is_gpu_available(cuda_only=True):
-      self._run_test(kwargs, 'data_format', ['channels_first'])
-    self._run_test(kwargs, 'depth_multiplier', [1, 2])
-
+    with self.cached_session(use_gpu=True):
+      testing_utils.layer_test(
+          keras.layers.DepthwiseConv2D,
+          kwargs=kwargs,
+          input_shape=(num_samples, num_row, num_col, stack_size))
+
+  @parameterized.named_parameters(
+      ('padding_valid', {'padding': 'valid'}),
+      ('padding_same', {'padding': 'same'}),
+      ('strides', {'strides': (2, 2)}),
+      # Only runs on GPU with CUDA, channels_first is not supported on CPU.
+      # TODO(b/62340061): Support channels_first on CPU.
+      ('data_format', {'data_format': 'channels_first'}),
+      ('depth_multiplier_1', {'depth_multiplier': 1}),
+      ('depth_multiplier_2', {'depth_multiplier': 2}),
+  )
+  def test_depthwise_conv2d(self, kwargs):
+    kwargs['kernel_size'] = (3, 3)
+    if 'data_format' not in kwargs or test.is_gpu_available(cuda_only=True):
+      self._run_test(kwargs)
+
+  def test_depthwise_conv2d_full(self):
     kwargs = {
         'kernel_size': 3,
         'padding': 'valid',
@@ -1103,8 +753,9 @@ class DepthwiseConv2DTest(keras_parameterized.TestCase):
         'depthwise_constraint': 'unit_norm',
         'use_bias': True,
         'strides': (2, 2),
+        'depth_multiplier': 1,
     }
-    self._run_test(kwargs, 'depth_multiplier', [1])
+    self._run_test(kwargs)
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/keras/layers/convolutional_transpose_test.py b/tensorflow/python/keras/layers/convolutional_transpose_test.py
new file mode 100644
index 0000000000..dd73d22d51
--- /dev/null
+++ b/tensorflow/python/keras/layers/convolutional_transpose_test.py
@@ -0,0 +1,209 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for convolutional transpose layers."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from absl.testing import parameterized
+import numpy as np
+
+from tensorflow.python import keras
+from tensorflow.python.keras import keras_parameterized
+from tensorflow.python.keras import testing_utils
+from tensorflow.python.platform import test
+
+
+@keras_parameterized.run_all_keras_modes
+class Conv2DTransposeTest(keras_parameterized.TestCase):
+
+  def _run_test(self, kwargs):
+    num_samples = 2
+    stack_size = 3
+    num_row = 7
+    num_col = 6
+
+    with self.cached_session(use_gpu=True):
+      testing_utils.layer_test(
+          keras.layers.Conv2DTranspose,
+          kwargs=kwargs,
+          input_shape=(num_samples, num_row, num_col, stack_size))
+
+  @parameterized.named_parameters(
+      ('padding_valid', {'padding': 'valid'}),
+      ('padding_same', {'padding': 'same'}),
+      ('strides', {'strides': (2, 2)}),
+      # Only runs on GPU with CUDA, channels_first is not supported on CPU.
+      # TODO(b/62340061): Support channels_first on CPU.
+      ('data_format', {'data_format': 'channels_first'}),
+      ('strides_output_padding', {'strides': (2, 2), 'output_padding': (1, 1)}),
+  )
+  def test_conv2d_transpose(self, kwargs):
+    kwargs['filters'] = 2
+    kwargs['kernel_size'] = (3, 3)
+    if 'data_format' not in kwargs or test.is_gpu_available(cuda_only=True):
+      self._run_test(kwargs)
+
+  def test_conv2d_transpose_regularizers(self):
+    kwargs = {
+        'filters': 3,
+        'kernel_size': 3,
+        'padding': 'valid',
+        'kernel_regularizer': 'l2',
+        'bias_regularizer': 'l2',
+        'activity_regularizer': 'l2',
+        'strides': 1
+    }
+    with self.cached_session(use_gpu=True):
+      layer = keras.layers.Conv2DTranspose(**kwargs)
+      layer.build((None, 5, 5, 2))
+      self.assertEqual(len(layer.losses), 2)
+      layer(keras.backend.variable(np.ones((1, 5, 5, 2))))
+      self.assertEqual(len(layer.losses), 3)
+
+  def test_conv2d_transpose_constraints(self):
+    k_constraint = lambda x: x
+    b_constraint = lambda x: x
+
+    kwargs = {
+        'filters': 3,
+        'kernel_size': 3,
+        'padding': 'valid',
+        'kernel_constraint': k_constraint,
+        'bias_constraint': b_constraint,
+        'strides': 1
+    }
+    with self.cached_session(use_gpu=True):
+      layer = keras.layers.Conv2DTranspose(**kwargs)
+      layer.build((None, 5, 5, 2))
+      self.assertEqual(layer.kernel.constraint, k_constraint)
+      self.assertEqual(layer.bias.constraint, b_constraint)
+
+  def test_conv2d_transpose_dilation(self):
+    testing_utils.layer_test(keras.layers.Conv2DTranspose,
+                             kwargs={'filters': 2,
+                                     'kernel_size': 3,
+                                     'padding': 'same',
+                                     'data_format': 'channels_last',
+                                     'dilation_rate': (2, 2)},
+                             input_shape=(2, 5, 6, 3))
+
+    input_data = np.arange(48).reshape((1, 4, 4, 3)).astype(np.float32)
+    expected_output = np.float32([[192, 228, 192, 228],
+                                  [336, 372, 336, 372],
+                                  [192, 228, 192, 228],
+                                  [336, 372, 336, 372]]).reshape((1, 4, 4, 1))
+    testing_utils.layer_test(keras.layers.Conv2DTranspose,
+                             input_data=input_data,
+                             kwargs={'filters': 1,
+                                     'kernel_size': 3,
+                                     'padding': 'same',
+                                     'data_format': 'channels_last',
+                                     'dilation_rate': (2, 2),
+                                     'kernel_initializer': 'ones'},
+                             expected_output=expected_output)
+
+
+@keras_parameterized.run_all_keras_modes
+class Conv3DTransposeTest(keras_parameterized.TestCase):
+
+  def _run_test(self, kwargs):
+    num_samples = 2
+    stack_size = 3
+    num_row = 7
+    num_col = 6
+    depth = 5
+
+    with self.cached_session(use_gpu=True):
+      testing_utils.layer_test(
+          keras.layers.Conv3DTranspose,
+          kwargs=kwargs,
+          input_shape=(num_samples, depth, num_row, num_col, stack_size))
+
+  @parameterized.named_parameters(
+      ('padding_valid', {'padding': 'valid'}),
+      ('padding_same', {'padding': 'same'}),
+      ('strides', {'strides': (2, 2, 2)}),
+      # Only runs on GPU with CUDA, channels_first is not supported on CPU.
+      # TODO(b/62340061): Support channels_first on CPU.
+      ('data_format', {'data_format': 'channels_first'}),
+      ('strides_output_padding', {'strides': (2, 2, 2),
+                                  'output_padding': (1, 1, 1)}),
+  )
+  def test_conv3d_transpose(self, kwargs):
+    kwargs['filters'] = 2
+    kwargs['kernel_size'] = (3, 3, 3)
+    if 'data_format' not in kwargs or test.is_gpu_available(cuda_only=True):
+      self._run_test(kwargs)
+
+  def test_conv3d_transpose_regularizers(self):
+    kwargs = {
+        'filters': 3,
+        'kernel_size': 3,
+        'padding': 'valid',
+        'kernel_regularizer': 'l2',
+        'bias_regularizer': 'l2',
+        'activity_regularizer': 'l2',
+        'strides': 1
+    }
+    with self.cached_session(use_gpu=True):
+      layer = keras.layers.Conv3DTranspose(**kwargs)
+      layer.build((None, 5, 5, 5, 2))
+      self.assertEqual(len(layer.losses), 2)
+      layer(keras.backend.variable(np.ones((1, 5, 5, 5, 2))))
+      self.assertEqual(len(layer.losses), 3)
+
+  def test_conv3d_transpose_constraints(self):
+    k_constraint = lambda x: x
+    b_constraint = lambda x: x
+
+    kwargs = {
+        'filters': 3,
+        'kernel_size': 3,
+        'padding': 'valid',
+        'kernel_constraint': k_constraint,
+        'bias_constraint': b_constraint,
+        'strides': 1
+    }
+    with self.cached_session(use_gpu=True):
+      layer = keras.layers.Conv3DTranspose(**kwargs)
+      layer.build((None, 5, 5, 5, 2))
+      self.assertEqual(layer.kernel.constraint, k_constraint)
+      self.assertEqual(layer.bias.constraint, b_constraint)
+
+  def test_conv3d_transpose_dynamic_shape(self):
+    input_data = np.random.random((1, 3, 3, 3, 3)).astype(np.float32)
+    with self.cached_session(use_gpu=True):
+      # Won't raise error here.
+      testing_utils.layer_test(
+          keras.layers.Conv3DTranspose,
+          kwargs={
+              'data_format': 'channels_last',
+              'filters': 3,
+              'kernel_size': 3
+          },
+          input_shape=(None, None, None, None, 3),
+          input_data=input_data)
+      if test.is_gpu_available(cuda_only=True):
+        testing_utils.layer_test(
+            keras.layers.Conv3DTranspose,
+            kwargs={
+                'data_format': 'channels_first',
+                'filters': 3,
+                'kernel_size': 3
+            },
+            input_shape=(None, 3, None, None, None),
+            input_data=input_data)
diff --git a/tensorflow/python/keras/layers/separable_convolutional_test.py b/tensorflow/python/keras/layers/separable_convolutional_test.py
new file mode 100644
index 0000000000..8234bfe704
--- /dev/null
+++ b/tensorflow/python/keras/layers/separable_convolutional_test.py
@@ -0,0 +1,167 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for separable convolutional layers."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from absl.testing import parameterized
+import numpy as np
+
+from tensorflow.python import keras
+from tensorflow.python.keras import keras_parameterized
+from tensorflow.python.keras import testing_utils
+from tensorflow.python.platform import test
+
+
+@keras_parameterized.run_all_keras_modes
+class SeparableConv1DTest(keras_parameterized.TestCase):
+
+  def _run_test(self, kwargs):
+    num_samples = 2
+    stack_size = 3
+    length = 7
+
+    with self.cached_session(use_gpu=True):
+      testing_utils.layer_test(
+          keras.layers.SeparableConv1D,
+          kwargs=kwargs,
+          input_shape=(num_samples, length, stack_size))
+
+  @parameterized.named_parameters(
+      ('padding_valid', {'padding': 'valid'}),
+      ('padding_same', {'padding': 'same'}),
+      ('padding_same_dilation_2', {'padding': 'same', 'dilation_rate': 2}),
+      ('padding_causal', {'padding': 'causal'}),
+      ('strides', {'strides': 2}),
+      ('dilation_rate', {'dilation_rate': 2}),
+      ('depth_multiplier', {'depth_multiplier': 2}),
+  )
+  def test_separable_conv1d(self, kwargs):
+    kwargs['filters'] = 2
+    kwargs['kernel_size'] = 3
+    self._run_test(kwargs)
+
+  def test_separable_conv1d_regularizers(self):
+    kwargs = {
+        'filters': 3,
+        'kernel_size': 3,
+        'padding': 'valid',
+        'depthwise_regularizer': 'l2',
+        'pointwise_regularizer': 'l2',
+        'bias_regularizer': 'l2',
+        'activity_regularizer': 'l2',
+        'strides': 1
+    }
+    with self.cached_session(use_gpu=True):
+      layer = keras.layers.SeparableConv1D(**kwargs)
+      layer.build((None, 5, 2))
+      self.assertEqual(len(layer.losses), 3)
+      layer(keras.backend.variable(np.ones((1, 5, 2))))
+      self.assertEqual(len(layer.losses), 4)
+
+  def test_separable_conv1d_constraints(self):
+    d_constraint = lambda x: x
+    p_constraint = lambda x: x
+    b_constraint = lambda x: x
+
+    kwargs = {
+        'filters': 3,
+        'kernel_size': 3,
+        'padding': 'valid',
+        'pointwise_constraint': p_constraint,
+        'depthwise_constraint': d_constraint,
+        'bias_constraint': b_constraint,
+        'strides': 1
+    }
+    with self.cached_session(use_gpu=True):
+      layer = keras.layers.SeparableConv1D(**kwargs)
+      layer.build((None, 5, 2))
+      self.assertEqual(layer.depthwise_kernel.constraint, d_constraint)
+      self.assertEqual(layer.pointwise_kernel.constraint, p_constraint)
+      self.assertEqual(layer.bias.constraint, b_constraint)
+
+
+@keras_parameterized.run_all_keras_modes
+class SeparableConv2DTest(keras_parameterized.TestCase):
+
+  def _run_test(self, kwargs):
+    num_samples = 2
+    stack_size = 3
+    num_row = 7
+    num_col = 6
+
+    with self.cached_session(use_gpu=True):
+      testing_utils.layer_test(
+          keras.layers.SeparableConv2D,
+          kwargs=kwargs,
+          input_shape=(num_samples, num_row, num_col, stack_size))
+
+  @parameterized.named_parameters(
+      ('padding_valid', {'padding': 'valid'}),
+      ('padding_same', {'padding': 'same'}),
+      ('padding_same_dilation_2', {'padding': 'same', 'dilation_rate': 2}),
+      ('strides', {'strides': 2}),
+      # Only runs on GPU with CUDA, channels_first is not supported on CPU.
+      # TODO(b/62340061): Support channels_first on CPU.
+      ('data_format', {'data_format': 'channels_first'}),
+      ('dilation_rate', {'dilation_rate': 2}),
+      ('depth_multiplier', {'depth_multiplier': 2}),
+  )
+  def test_separable_conv2d(self, kwargs):
+    kwargs['filters'] = 2
+    kwargs['kernel_size'] = 3
+    if 'data_format' not in kwargs or test.is_gpu_available(cuda_only=True):
+      self._run_test(kwargs)
+
+  def test_separable_conv2d_regularizers(self):
+    kwargs = {
+        'filters': 3,
+        'kernel_size': 3,
+        'padding': 'valid',
+        'depthwise_regularizer': 'l2',
+        'pointwise_regularizer': 'l2',
+        'bias_regularizer': 'l2',
+        'activity_regularizer': 'l2',
+        'strides': 1
+    }
+    with self.cached_session(use_gpu=True):
+      layer = keras.layers.SeparableConv2D(**kwargs)
+      layer.build((None, 5, 5, 2))
+      self.assertEqual(len(layer.losses), 3)
+      layer(keras.backend.variable(np.ones((1, 5, 5, 2))))
+      self.assertEqual(len(layer.losses), 4)
+
+  def test_separable_conv2d_constraints(self):
+    d_constraint = lambda x: x
+    p_constraint = lambda x: x
+    b_constraint = lambda x: x
+
+    kwargs = {
+        'filters': 3,
+        'kernel_size': 3,
+        'padding': 'valid',
+        'pointwise_constraint': p_constraint,
+        'depthwise_constraint': d_constraint,
+        'bias_constraint': b_constraint,
+        'strides': 1
+    }
+    with self.cached_session(use_gpu=True):
+      layer = keras.layers.SeparableConv2D(**kwargs)
+      layer.build((None, 5, 5, 2))
+      self.assertEqual(layer.depthwise_kernel.constraint, d_constraint)
+      self.assertEqual(layer.pointwise_kernel.constraint, p_constraint)
+      self.assertEqual(layer.bias.constraint, b_constraint)
-- 
GitLab


From 8fcb0f87ffe41b072ed8ed893254d293e3a53b93 Mon Sep 17 00:00:00 2001
From: Smit Hinsu <hinsu@google.com>
Date: Thu, 10 Jan 2019 23:17:26 -0800
Subject: [PATCH 0578/1095] Automated rollback of commit
 68e869eb40ff0acd03515336bf31eecbabf97adc

PiperOrigin-RevId: 228836014
---
 tensorflow/contrib/tensorrt/BUILD | 20 +-------------------
 1 file changed, 1 insertion(+), 19 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index 3d34b91a74..003a7e0b0a 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -500,6 +500,7 @@ cuda_py_tests(
         "test/quantization_test.py",
         "test/rank_two_test.py",
         "test/reshape_transpose_test.py",
+        "test/unary_test.py",
         "test/vgg_block_nchw_test.py",
         "test/vgg_block_test.py",
     ],
@@ -515,25 +516,6 @@ cuda_py_tests(
     ],
 )
 
-cuda_py_tests(
-    name = "tf_trt_integration_test_no_oss",
-    srcs = [
-        "test/unary_test.py",
-    ],
-    additional_deps = [
-        ":tf_trt_integration_test_base",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework_test_lib",
-    ],
-    tags = [
-        "no_cuda_on_cpu_tap",
-        "no_oss",  # TODO(b/117274186): re-enable in OSS after crash fixed
-        "no_pip",  # TODO(b/117274186): re-enable in OSS after crash fixed
-        "no_windows",
-        "nomac",
-    ],
-)
-
 cuda_py_test(
     name = "quantization_mnist_test",
     srcs = ["test/quantization_mnist_test.py"],
-- 
GitLab


From cfe3f839c852bb590f7b349376a2c3bfeb0a031f Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Fri, 11 Jan 2019 00:04:54 -0800
Subject: [PATCH 0579/1095] Update Eigen to commit ad3bcd81cc49.

PiperOrigin-RevId: 228839247
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 06caa200e4..ca4feb691e 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -138,11 +138,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "eigen_archive",
         build_file = clean_dep("//third_party:eigen.BUILD"),
-        sha256 = "9de38f2d162c51599b802f7c36d9f3773980d19ac908c61638f8344d2c10e1ca",
-        strip_prefix = "eigen-eigen-88fc23324517",
+        sha256 = "df54d805573871d03aaef6d27d45439c18ff6e8db215a3b922b2daefd6914147",
+        strip_prefix = "eigen-eigen-ad3bcd81cc49",
         urls = [
-            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/88fc23324517..tar.gz",
-            "https://bitbucket.org/eigen/eigen/get/88fc23324517.tar.gz",
+            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/ad3bcd81cc49..tar.gz",
+            "https://bitbucket.org/eigen/eigen/get/ad3bcd81cc49.tar.gz",
         ],
     )
 
-- 
GitLab


From ac345af940a5bfc2d205aa14001450b10ee01c16 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 11 Jan 2019 01:02:23 -0800
Subject: [PATCH 0580/1095] compat: Update forward compatibility horizon to
 2019-01-11

PiperOrigin-RevId: 228844118
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 638bd445a6..4028302362 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 10)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 11)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From bac29e41394d029e94659b6d1df833237056b764 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 11 Jan 2019 01:04:23 -0800
Subject: [PATCH 0581/1095] Added simple nested structure serialization on
 input.

PiperOrigin-RevId: 228844725
---
 tensorflow/python/eager/function.py           | 30 +++++++++-------
 .../saved_model/function_deserialization.py   | 35 +++++++++++--------
 .../saved_model/function_serialization.py     |  5 ++-
 tensorflow/python/saved_model/load_test.py    | 33 +++++++++++++++++
 4 files changed, 74 insertions(+), 29 deletions(-)

diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index 83cd140158..9a604620fa 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -814,17 +814,22 @@ class UnknownArgument(object):
   pass
 
 
-def _encode_arg_for_serialization(arg):
-  """A representation for this argument, for serializing signatures."""
-  if isinstance(arg, ops.Tensor):
-    return tensor_spec.TensorSpec(arg.shape, arg.dtype)
-  if isinstance(arg, int):
-    return arg
-  if isinstance(arg, float):
-    return arg
-  if isinstance(arg, bool):
-    return arg
-  return UnknownArgument()
+def _convert_inputs_to_signature(inputs):
+  """Convert a list of potentially nested inputs to a signature."""
+
+  def encode_arg(arg, name=None):
+    """A representation for this argument, for converting into signatures."""
+    if isinstance(arg, ops.Tensor):
+      return tensor_spec.TensorSpec(arg.shape, arg.dtype, name)
+    if isinstance(arg, (int, float, bool)):
+      return arg
+    return UnknownArgument()
+
+  # We are using the flattened paths to name the TensorSpecs. We need an
+  # explicit name for them downstream.
+  flattened_with_paths = nest.flatten_with_joined_string_paths(inputs)
+  mapped = [encode_arg(arg, path) for path, arg in flattened_with_paths]
+  return nest.pack_sequence_as(inputs, mapped)
 
 
 pywrap_tensorflow.RegisterType("Tensor", ops.Tensor)
@@ -1305,8 +1310,7 @@ class PolymorphicFunction(object):
         if self._input_signature:
           python_call_signature = self._input_signature
         else:
-          python_call_signature = tuple(
-              _encode_arg_for_serialization(arg) for arg in args)
+          python_call_signature = tuple(_convert_inputs_to_signature(args))
         # pylint: disable=protected-access
         # Save information about non-Tensor arguments with the concrete
         # function. Used to serialize PolymorphicFunctions.
diff --git a/tensorflow/python/saved_model/function_deserialization.py b/tensorflow/python/saved_model/function_deserialization.py
index dc839cf61c..3cf27fd5a4 100644
--- a/tensorflow/python/saved_model/function_deserialization.py
+++ b/tensorflow/python/saved_model/function_deserialization.py
@@ -38,19 +38,25 @@ def _is_tensor(t):
 
 def _inputs_compatible(args, stored_inputs):
   """Checks whether function arguments are compatible with parameters."""
-  # TODO(vbardiovsky): The compatibility check should be about the signature,
-  # not the flattened version of it.
   if len(args) != len(stored_inputs):
     return False
-  for a, b in zip(args, stored_inputs):
-    if _is_tensor(a):
-      if not isinstance(b, tensor_spec.TensorSpec):
-        return False
-      if a.dtype != b.dtype or not b.shape.is_compatible_with(a.shape):
-        return False
-    else:
-      if a != b:
-        return False
+
+  for arg, stored_input in zip(args, stored_inputs):
+    if not function_lib.is_same_structure(arg, stored_input):
+      return False
+
+    flattened_arg = nest.flatten(arg)
+    flattened_stored_input = nest.flatten(stored_input)
+
+    for a, b in zip(flattened_arg, flattened_stored_input):
+      if _is_tensor(a):
+        if not isinstance(b, tensor_spec.TensorSpec):
+          return False
+        if a.dtype != b.dtype or not b.shape.is_compatible_with(a.shape):
+          return False
+      else:
+        if a != b:
+          return False
   return True
 
 
@@ -98,11 +104,10 @@ def recreate_polymorphic_function(
       except ValueError:
         continue
 
-      canonicalized_inputs = nest.flatten(can_args)
-
-      if _inputs_compatible(canonicalized_inputs,
+      if _inputs_compatible(can_args,
                             canonicalized_original_inputs):
-        filtered_inputs = [t for t in canonicalized_inputs if _is_tensor(t)]
+        flattened_inputs = nest.flatten(can_args)
+        filtered_inputs = [t for t in flattened_inputs if _is_tensor(t)]
         flattened_outputs = function_obj._call_flat(filtered_inputs)  # pylint: disable=protected-access
         # TODO(vbardiovsky): Rebuild output structure.
         single_output, = flattened_outputs
diff --git a/tensorflow/python/saved_model/function_serialization.py b/tensorflow/python/saved_model/function_serialization.py
index 014a4ef04c..c826ffa02c 100644
--- a/tensorflow/python/saved_model/function_serialization.py
+++ b/tensorflow/python/saved_model/function_serialization.py
@@ -22,6 +22,7 @@ from tensorflow.python.eager import function as defun_lib
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.saved_model import nested_structure_coder
 from tensorflow.python.saved_model import saved_object_graph_pb2
+from tensorflow.python.util import nest
 
 
 def serialize_polymorphic_function(polymorphic_function, node_ids):
@@ -67,7 +68,9 @@ def list_all_concrete_functions(polymorphic_function):
     polymorphic_function.get_concrete_function()
   concrete_functions = []
   for signature in polymorphic_function._cached_input_signatures:  # pylint: disable=protected-access
-    if any(isinstance(arg, defun_lib.UnknownArgument) for arg in signature):
+    flattened = nest.flatten(signature)
+    if any(isinstance(arg, defun_lib.UnknownArgument) for arg in flattened):
+      logging.info("Unsupported signature for serialization: %s.", signature)
       continue
     concrete_function = polymorphic_function.get_concrete_function(*signature)
     concrete_functions.append((signature, concrete_function))
diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index 8b34414d25..32bb841556 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -185,6 +185,39 @@ class LoadTest(test.TestCase):
     self.assertEqual(4, imported.f(constant_op.constant(2), True).numpy())
     self.assertEqual(7, imported.f(constant_op.constant(2)).numpy())
 
+  def test_structured_inputs(self):
+
+    def func(x, training=True):
+      # x is a nested structure, we care about one particular tensor.
+      _, (a, b) = x
+      if training:
+        return 2 * a["a"] + b
+      else:
+        return 7
+
+    root = tracking.Checkpointable()
+    root.f = def_function.function(func)
+
+    x = constant_op.constant(10)
+    y = constant_op.constant(11)
+
+    input1 = [6, ({"a": x}, y)]
+    input2 = [7, ({"a": x}, y)]  # Not compatible with input1 signature.
+    input3 = [6, ({"a": y}, x)]  # Compatible with input1 signature.
+
+    # Note: by only calling f(input1) before serialization, only inputs with
+    # matching signature will be valid on the loaded model.
+    self.assertEqual(31, root.f(input1).numpy())
+
+    imported = self.cycle(root)
+
+    with self.assertRaisesRegexp(AssertionError,
+                                 "Could not find matching function to call.*"):
+      imported.f(input2)
+
+    self.assertEqual(31, imported.f(input1).numpy())
+    self.assertEqual(32, imported.f(input3).numpy())
+
   def test_positional_arguments(self):
     def func(x, training=False, abc=7.1, defg=7.7):
       del abc
-- 
GitLab


From c2ab6dc7a19cb121d22f36e03107315aeb5e3513 Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Fri, 11 Jan 2019 01:31:06 -0800
Subject: [PATCH 0582/1095] Add default comparators which can be used for
 sorting.

For sorting, we will allow soon to specify comparator computations.
This adds default comparators which implement the current comparison
behavior. For floating point values special logic is used to create a total
order.

PiperOrigin-RevId: 228847705
---
 tensorflow/compiler/xla/client/lib/BUILD      |  33 ++++
 .../compiler/xla/client/lib/comparators.cc    | 159 ++++++++++++++++++
 .../compiler/xla/client/lib/comparators.h     |  47 ++++++
 .../xla/client/lib/comparators_test.cc        | 150 +++++++++++++++++
 tensorflow/compiler/xla/literal_util.cc       |   7 +-
 5 files changed, 393 insertions(+), 3 deletions(-)
 create mode 100644 tensorflow/compiler/xla/client/lib/comparators.cc
 create mode 100644 tensorflow/compiler/xla/client/lib/comparators.h
 create mode 100644 tensorflow/compiler/xla/client/lib/comparators_test.cc

diff --git a/tensorflow/compiler/xla/client/lib/BUILD b/tensorflow/compiler/xla/client/lib/BUILD
index df1ee330f1..27f9c61848 100644
--- a/tensorflow/compiler/xla/client/lib/BUILD
+++ b/tensorflow/compiler/xla/client/lib/BUILD
@@ -91,6 +91,39 @@ xla_test(
     ],
 )
 
+cc_library(
+    name = "comparators",
+    srcs = ["comparators.cc"],
+    hdrs = ["comparators.h"],
+    deps = [
+        ":constants",
+        "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla:types",
+        "//tensorflow/compiler/xla:xla_data_proto",
+        "//tensorflow/compiler/xla/client:xla_builder",
+        "//tensorflow/compiler/xla/client:xla_computation",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/types:span",
+    ],
+)
+
+xla_test(
+    name = "comparators_test",
+    srcs = ["comparators_test.cc"],
+    deps = [
+        ":comparators",
+        ":constants",
+        "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla:test",
+        "//tensorflow/compiler/xla:types",
+        "//tensorflow/compiler/xla:xla_data_proto",
+        "//tensorflow/compiler/xla/client:xla_builder",
+        "//tensorflow/compiler/xla/tests:client_library_test_base",
+        "//tensorflow/compiler/xla/tests:xla_internal_test_main",
+        "@com_google_absl//absl/container:inlined_vector",
+    ],
+)
+
 cc_library(
     name = "constants",
     srcs = ["constants.cc"],
diff --git a/tensorflow/compiler/xla/client/lib/comparators.cc b/tensorflow/compiler/xla/client/lib/comparators.cc
new file mode 100644
index 0000000000..c620c9841a
--- /dev/null
+++ b/tensorflow/compiler/xla/client/lib/comparators.cc
@@ -0,0 +1,159 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/client/lib/comparators.h"
+
+#include <limits>
+#include <string>
+#include <vector>
+
+#include "absl/strings/str_cat.h"
+#include "absl/types/span.h"
+#include "tensorflow/compiler/xla/client/lib/constants.h"
+#include "tensorflow/compiler/xla/client/xla_builder.h"
+#include "tensorflow/compiler/xla/client/xla_computation.h"
+#include "tensorflow/compiler/xla/primitive_util.h"
+#include "tensorflow/compiler/xla/shape_util.h"
+#include "tensorflow/compiler/xla/types.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
+
+namespace xla {
+namespace {
+
+using XlaOpGenerator = XlaOp (*)(const XlaOp&, const XlaOp&,
+                                 absl::Span<const int64>);
+
+XlaOp BitcastConvertFloatingPointToIntegral(const XlaOp& value,
+                                            int64 bit_width) {
+  PrimitiveType signed_type;
+  PrimitiveType unsigned_type;
+  XlaOp max_value;
+  switch (bit_width) {
+    case 16:
+      max_value =
+          ConstantR0(value.builder(),
+                     static_cast<uint16>(std::numeric_limits<int16>::max()));
+      signed_type = S16;
+      unsigned_type = U16;
+      break;
+    case 32:
+      max_value =
+          ConstantR0(value.builder(),
+                     static_cast<uint32>(std::numeric_limits<int32>::max()));
+      signed_type = S32;
+      unsigned_type = U32;
+      break;
+    case 64:
+      max_value =
+          ConstantR0(value.builder(),
+                     static_cast<uint64>(std::numeric_limits<int64>::max()));
+      signed_type = S64;
+      unsigned_type = U64;
+      break;
+    default:
+      return value.builder()->ReportError(
+          InvalidArgument("Invalid bit width %lld for Comparator floating "
+                          "point parameter.",
+                          bit_width));
+  }
+  // Switch from a floating point value to a integer value in such a way that
+  // when using the integer value to compare, we get the same result for normal
+  // values, and -Nan is treated as the smallest value, and Nan is treated as
+  // the largest value.
+  // If f is a float, and
+  // x = bit_cast<int32>(f);
+  // y = x < 0 ? numeric_limits<int32>::max() - x : x;
+  // then y is ordered as an int32 such that finite values have the obvious
+  // order, -0 is ordered before 0, and -NaN and NaN appear at the beginning
+  // and end of the ordering.
+  // Note that in order to avoid -x to overflow, we calculate
+  // numeric_limits<int32>::max() - x as unsigned, and then convert back to
+  // signed.
+  auto signed_value = BitcastConvertType(value, signed_type);
+  auto unsigned_value = BitcastConvertType(value, unsigned_type);
+  auto flipped_value =
+      BitcastConvertType(Sub(max_value, unsigned_value), signed_type);
+  auto is_negative = Lt(signed_value, Zero(value.builder(), signed_type));
+  return Select(is_negative, flipped_value, signed_value);
+}
+
+XlaComputation CreateScalarComparisonComputation(
+    const string& name, const std::vector<PrimitiveType>& operand_types,
+    XlaBuilder* builder, XlaOpGenerator generator) {
+  // Create a default computation where we compare only the first two
+  // parameters of type 'operand_types[0]'.
+  auto b = builder->CreateSubBuilder(name);
+  if (operand_types.empty()) {
+    b->ReportError(InvalidArgument("operand_types should not be empty"));
+    return b->BuildAndNoteError();
+  }
+
+  int64 parameter_count = 0;
+  XlaOp first_lhs_param;
+  XlaOp first_rhs_param;
+
+  // For each type in 'operand_types' we create two parameters of this type. The
+  // idea is that this computation can be used by n-ary Sort, and potentially
+  // should support comparing also the other operands of sort. In this default
+  // computation, however, we will not actually use any parameters except the
+  // first two.
+  for (auto operand_type : operand_types) {
+    auto scalar_shape = ShapeUtil::MakeShape(operand_type, {});
+    auto lhs_param = Parameter(b.get(), parameter_count * 2, scalar_shape,
+                               absl::StrCat("p.", parameter_count, ".lhs"));
+    auto rhs_param = Parameter(b.get(), parameter_count * 2 + 1, scalar_shape,
+                               absl::StrCat("p.", parameter_count, ".rhs"));
+    if (parameter_count == 0) {
+      first_lhs_param = lhs_param;
+      first_rhs_param = rhs_param;
+    }
+    ++parameter_count;
+  }
+  if (primitive_util::IsFloatingPointType(operand_types[0])) {
+    PrimitiveType compare_type = operand_types[0];
+    // Special-case handling for BF16. We currently do not support direct
+    // comparisons with BF16, so we convert to F32 and then use the F32
+    // comparison logic.
+    if (compare_type == BF16) {
+      compare_type = F32;
+      first_lhs_param = ConvertElementType(first_lhs_param, F32);
+      first_rhs_param = ConvertElementType(first_rhs_param, F32);
+    }
+    int64 bit_width = primitive_util::BitWidth(compare_type);
+    first_lhs_param =
+        BitcastConvertFloatingPointToIntegral(first_lhs_param, bit_width);
+    first_rhs_param =
+        BitcastConvertFloatingPointToIntegral(first_rhs_param, bit_width);
+  }
+  generator(first_lhs_param, first_rhs_param, {});
+  return b->BuildAndNoteError();
+}
+}  // namespace
+
+// Creates a scalar less-than computation and returns it.
+XlaComputation CreateScalarLtComputation(
+    const std::vector<PrimitiveType>& operand_types, XlaBuilder* builder) {
+  return CreateScalarComparisonComputation("compare-less-than", operand_types,
+                                           builder, Lt);
+}
+
+// Creates a scalar greater-than computation and returns it.
+XlaComputation CreateScalarGtComputation(
+    const std::vector<PrimitiveType>& operand_types, XlaBuilder* builder) {
+  return CreateScalarComparisonComputation("compare-greater-than",
+                                           operand_types, builder, Gt);
+}
+
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/client/lib/comparators.h b/tensorflow/compiler/xla/client/lib/comparators.h
new file mode 100644
index 0000000000..cbcfc227dd
--- /dev/null
+++ b/tensorflow/compiler/xla/client/lib/comparators.h
@@ -0,0 +1,47 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_COMPILER_XLA_CLIENT_LIB_COMPARATORS_H_
+#define TENSORFLOW_COMPILER_XLA_CLIENT_LIB_COMPARATORS_H_
+
+#include <vector>
+
+#include "tensorflow/compiler/xla/client/xla_builder.h"
+#include "tensorflow/compiler/xla/client/xla_computation.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
+
+namespace xla {
+
+// Creates a scalar less-than computation and returns it. The created
+// computation has 2 * 'operand_types.size()' many parameters, where parameters
+// 2 * i and 2 * i + 1 are a scalar with primitive type 'operand_types[i]'. The
+// computation compares the first two parameters. For floating point types, a
+// total order is created where
+// -NaN < -infinity < ... < -0 < 0 < ... < infinity < NaN
+XlaComputation CreateScalarLtComputation(
+    const std::vector<PrimitiveType>& operand_types, XlaBuilder* builder);
+
+// Creates a scalar greater-than computation and returns it. The created
+// computation has 2 * 'operand_types.size()' many parameters, where parameters
+// 2 * i and 2 * i + 1 are a scalar with primitive type 'operand_types[i]'. The
+// computation compares the first two parameters. For floating point types, a
+// total order is created where
+// NaN > infinity > ... > 0 > -0 > ... > -infinity > -NaN
+XlaComputation CreateScalarGtComputation(
+    const std::vector<PrimitiveType>& operand_types, XlaBuilder* builder);
+
+}  // namespace xla
+
+#endif  // TENSORFLOW_COMPILER_XLA_CLIENT_LIB_COMPARATORS_H_
diff --git a/tensorflow/compiler/xla/client/lib/comparators_test.cc b/tensorflow/compiler/xla/client/lib/comparators_test.cc
new file mode 100644
index 0000000000..efec984eb2
--- /dev/null
+++ b/tensorflow/compiler/xla/client/lib/comparators_test.cc
@@ -0,0 +1,150 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/client/lib/comparators.h"
+
+#include <limits>
+#include <vector>
+
+#include "absl/container/inlined_vector.h"
+#include "tensorflow/compiler/xla/client/lib/constants.h"
+#include "tensorflow/compiler/xla/client/xla_builder.h"
+#include "tensorflow/compiler/xla/primitive_util.h"
+#include "tensorflow/compiler/xla/test.h"
+#include "tensorflow/compiler/xla/tests/client_library_test_base.h"
+#include "tensorflow/compiler/xla/tests/test_macros.h"
+#include "tensorflow/compiler/xla/types.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
+
+namespace xla {
+namespace {
+
+class ComparatorsTest : public ClientLibraryTestBase {
+ public:
+  ComparatorsTest() : builder_(TestName()) {}
+  XlaBuilder* builder() { return &builder_; }
+
+ private:
+  XlaBuilder builder_;
+};
+
+template <
+    PrimitiveType type,
+    typename T = typename primitive_util::PrimitiveTypeToNative<type>::type>
+void BuildComparatorAndComparisons(ComparatorsTest* test,
+                                   bool compare_less_than,
+                                   absl::InlinedVector<bool, 10>* expected) {
+  auto compare = compare_less_than
+                     ? CreateScalarLtComputation({type}, test->builder())
+                     : CreateScalarGtComputation({type}, test->builder());
+
+  auto negative_nan = ConstantR0<T>(
+      test->builder(), -T(std::numeric_limits<float>::quiet_NaN()));
+  auto positive_nan = ConstantR0<T>(test->builder(),
+                                    T(std::numeric_limits<float>::quiet_NaN()));
+  auto negative_zero = ConstantR0<T>(test->builder(), T(-0.));
+  auto positive_zero = ConstantR0<T>(test->builder(), T(0.));
+  auto negative_infinity = MinValue(test->builder(), type);
+  auto positive_infinity = MaxValue(test->builder(), type);
+
+  // List the values in the expected sorting order from smallest to largest.
+  std::vector<XlaOp> all_constants{negative_nan,      negative_infinity,
+                                   negative_zero,     positive_zero,
+                                   positive_infinity, positive_nan};
+
+  // Do pairwise comparisons.
+  std::vector<XlaOp> all_comparisons;
+  for (const XlaOp& lhs_constant : all_constants) {
+    for (const XlaOp& rhs_constant : all_constants) {
+      all_comparisons.push_back(Broadcast(
+          Call(test->builder(), compare, {lhs_constant, rhs_constant}), {1}));
+    }
+  }
+
+  // Concantenate the comparison results.
+  ConcatInDim(test->builder(), all_comparisons, 0);
+
+  // If we use less-than comparisons, we expect the comparison to result in true
+  // if the lhs value to be compared appears earlier in 'all_constants' than the
+  // rhs value. Likewise, if we use greater-than comparisons, we expect the
+  // comparison to return true if the rhs value appears earlier in
+  // 'all_constants' than the lhs value.
+  expected->clear();
+  for (int i = 0; i < all_constants.size(); ++i) {
+    for (int j = 0; j < all_constants.size(); ++j) {
+      expected->push_back(compare_less_than ? i < j : i > j);
+    }
+  }
+}
+
+XLA_TEST_F(ComparatorsTest, CompareLtBF16) {
+  absl::InlinedVector<bool, 10> expected;
+  BuildComparatorAndComparisons<BF16>(this, /*compare_less_than=*/true,
+                                      &expected);
+  ComputeAndCompareR1<bool>(builder(), expected, {});
+}
+
+XLA_TEST_F(ComparatorsTest, CompareGtBF16) {
+  absl::InlinedVector<bool, 10> expected;
+  BuildComparatorAndComparisons<BF16>(this, /*compare_less_than=*/false,
+                                      &expected);
+  ComputeAndCompareR1<bool>(builder(), expected, {});
+}
+
+// The interpreter doesn't support S16, and F16 results in comparisons with S16.
+XLA_TEST_F(ComparatorsTest, DISABLED_ON_INTERPRETER(CompareLtF16)) {
+  absl::InlinedVector<bool, 10> expected;
+  BuildComparatorAndComparisons<F16>(this, /*compare_less_than=*/true,
+                                     &expected);
+  ComputeAndCompareR1<bool>(builder(), expected, {});
+}
+
+XLA_TEST_F(ComparatorsTest, DISABLED_ON_INTERPRETER(CompareGtF16)) {
+  absl::InlinedVector<bool, 10> expected;
+  BuildComparatorAndComparisons<F16>(this, /*compare_less_than=*/false,
+                                     &expected);
+  ComputeAndCompareR1<bool>(builder(), expected, {});
+}
+
+XLA_TEST_F(ComparatorsTest, CompareLtF32) {
+  absl::InlinedVector<bool, 10> expected;
+  BuildComparatorAndComparisons<F32>(this, /*compare_less_than=*/true,
+                                     &expected);
+  ComputeAndCompareR1<bool>(builder(), expected, {});
+}
+
+XLA_TEST_F(ComparatorsTest, CompareGtF32) {
+  absl::InlinedVector<bool, 10> expected;
+  BuildComparatorAndComparisons<F32>(this, /*compare_less_than=*/false,
+                                     &expected);
+  ComputeAndCompareR1<bool>(builder(), expected, {});
+}
+
+XLA_TEST_F(ComparatorsTest, CompareLtF64) {
+  absl::InlinedVector<bool, 10> expected;
+  BuildComparatorAndComparisons<F64>(this, /*compare_less_than=*/true,
+                                     &expected);
+  ComputeAndCompareR1<bool>(builder(), expected, {});
+}
+
+XLA_TEST_F(ComparatorsTest, CompareGtF64) {
+  absl::InlinedVector<bool, 10> expected;
+  BuildComparatorAndComparisons<F64>(this, /*compare_less_than=*/false,
+                                     &expected);
+  ComputeAndCompareR1<bool>(builder(), expected, {});
+}
+
+}  // namespace
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/literal_util.cc b/tensorflow/compiler/xla/literal_util.cc
index 67eab1bff8..5881e43e96 100644
--- a/tensorflow/compiler/xla/literal_util.cc
+++ b/tensorflow/compiler/xla/literal_util.cc
@@ -106,12 +106,16 @@ Literal ConvertType(LiteralSlice literal) {
   switch (primitive_type) {
     case U8:
       return LiteralUtil::CreateR0<uint8>(0);
+    case U16:
+      return LiteralUtil::CreateR0<uint16>(0);
     case U32:
       return LiteralUtil::CreateR0<uint32>(0);
     case U64:
       return LiteralUtil::CreateR0<uint64>(0);
     case S8:
       return LiteralUtil::CreateR0<int8>(0);
+    case S16:
+      return LiteralUtil::CreateR0<int16>(0);
     case S32:
       return LiteralUtil::CreateR0<int32>(0);
     case S64:
@@ -128,9 +132,6 @@ Literal ConvertType(LiteralSlice literal) {
       return LiteralUtil::CreateR0<complex64>(0);
     case PRED:
       return LiteralUtil::CreateR0<bool>(false);
-    case S16:
-    case U16:
-      LOG(FATAL) << "u16/s16 literals not yet implemented";
     case TUPLE:
       LOG(FATAL) << "tuple element type cannot take on value of 0";
     case OPAQUE:
-- 
GitLab


From bb87d0021f12bc2e10d4cbb12e669012b2897d41 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 11 Jan 2019 02:22:26 -0800
Subject: [PATCH 0583/1095] Update ConvLSTMCell to accept shape as list or
 tuple

PiperOrigin-RevId: 228852641
---
 tensorflow/contrib/rnn/python/ops/rnn_cell.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/rnn/python/ops/rnn_cell.py b/tensorflow/contrib/rnn/python/ops/rnn_cell.py
index 8a1c09f171..0702b2a723 100644
--- a/tensorflow/contrib/rnn/python/ops/rnn_cell.py
+++ b/tensorflow/contrib/rnn/python/ops/rnn_cell.py
@@ -2071,7 +2071,7 @@ class ConvLSTMCell(rnn_cell_impl.RNNCell):
       conv_ndims: Convolution dimensionality (1, 2 or 3).
       input_shape: Shape of the input as int tuple, excluding the batch size.
       output_channels: int, number of output channels of the conv LSTM.
-      kernel_shape: Shape of kernel as in tuple (of size 1,2 or 3).
+      kernel_shape: Shape of kernel as an int tuple (of size 1, 2 or 3).
       use_bias: (bool) Use bias in convolutions.
       skip_connection: If set to `True`, concatenate the input to the
         output of the conv LSTM. Default: `False`.
@@ -2092,7 +2092,7 @@ class ConvLSTMCell(rnn_cell_impl.RNNCell):
     self._conv_ndims = conv_ndims
     self._input_shape = input_shape
     self._output_channels = output_channels
-    self._kernel_shape = kernel_shape
+    self._kernel_shape = list(kernel_shape)
     self._use_bias = use_bias
     self._forget_bias = forget_bias
     self._skip_connection = skip_connection
@@ -2172,7 +2172,7 @@ def _conv(args, filter_size, num_features, bias, bias_start=0.0):
   Args:
     args: a Tensor or a list of Tensors of dimension 3D, 4D or 5D,
     batch x n, Tensors.
-    filter_size: int tuple of filter height and width.
+    filter_size: int tuple of filter shape (of size 1, 2 or 3).
     num_features: int, number of features.
     bias: Whether to use biases in the convolution layer.
     bias_start: starting value to initialize the bias; 0 by default.
-- 
GitLab


From cdbaf19b7bb6b6ccfc1f4123f5121448683cbfa3 Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Fri, 11 Jan 2019 03:03:43 -0800
Subject: [PATCH 0584/1095] [Go]: Allow code in the "master" branch to work
 against the latest C library (1.12)

Fixes #23257 for now.
A more principled way of ensuring that we don't end up in this situation
as the Go API is updated to use new C API features remains to be investigated.

PiperOrigin-RevId: 228856302
---
 tensorflow/go/tensor_handle.go | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/tensorflow/go/tensor_handle.go b/tensorflow/go/tensor_handle.go
index 3b06773dd1..09192eccef 100644
--- a/tensorflow/go/tensor_handle.go
+++ b/tensorflow/go/tensor_handle.go
@@ -123,13 +123,22 @@ func (th *TensorHandle) DeviceName() (string, error) {
 // BackingDeviceName returns the name of the device in whose memory the tensor
 // handle resides. This function will block till the operation that produces
 // `h` has completed.
+//
+// WARNING: The implementation currently returns the same as DeviceName().
+// After TensoFlow 1.13's C library is released, this implementation will
+// be updated to return what the documentation says!
 func (th *TensorHandle) BackingDeviceName() (string, error) {
+	// TODO(ashankar): Restore after TensorFlow 1.13 is released.
+	// See https://github.com/tensorflow/tensorflow/issues/23257#issuecomment-433751410
+	return th.DeviceName()
+	/*
 	status := newStatus()
 	name := C.TFE_TensorHandleBackingDeviceName(th.c, status.c)
 	if err := status.Err(); err != nil {
 		return "", err
 	}
 	return C.GoString(name), nil
+	*/
 }
 
 // ToTensor returns the Tensor referenced by th. It may block if this tensor is
-- 
GitLab


From 24bfc9b93c6b51dbdd1a1a4d271f49ee13bf0ceb Mon Sep 17 00:00:00 2001
From: Dayananda V <dayanandasiet@gmail.com>
Date: Mon, 7 Jan 2019 11:23:07 +0530
Subject: [PATCH 0585/1095] Fix several DeprecationWarning: invlid escape
 sequence

---
 tensorflow/contrib/learn/python/learn/utils/gc_test.py | 4 ++--
 tensorflow/contrib/session_bundle/exporter.py          | 4 ++--
 tensorflow/contrib/session_bundle/gc_test.py           | 2 +-
 tensorflow/python/kernel_tests/scatter_nd_ops_test.py  | 8 ++++----
 4 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/tensorflow/contrib/learn/python/learn/utils/gc_test.py b/tensorflow/contrib/learn/python/learn/utils/gc_test.py
index e7d091e18a..af93e517f5 100644
--- a/tensorflow/contrib/learn/python/learn/utils/gc_test.py
+++ b/tensorflow/contrib/learn/python/learn/utils/gc_test.py
@@ -36,10 +36,10 @@ def _create_parser(base_dir):
     # Modify the path object for RegEx match for Windows Paths
     if os.name == "nt":
       match = re.match(
-          "^" + compat.as_str_any(base_dir).replace("\\", "/") + "/(\\d+)$",
+          r"^" + compat.as_str_any(base_dir).replace("\\", "/") + r"/(\d+)$",
           compat.as_str_any(path.path).replace("\\", "/"))
     else:
-      match = re.match("^" + compat.as_str_any(base_dir) + "/(\\d+)$",
+      match = re.match(r"^" + compat.as_str_any(base_dir) + r"/(\d+)$",
                        compat.as_str_any(path.path))
     if not match:
       return None
diff --git a/tensorflow/contrib/session_bundle/exporter.py b/tensorflow/contrib/session_bundle/exporter.py
index 08983337fc..f3efd292cf 100644
--- a/tensorflow/contrib/session_bundle/exporter.py
+++ b/tensorflow/contrib/session_bundle/exporter.py
@@ -304,10 +304,10 @@ class Exporter(object):
       def parser(path):
         if os.name == "nt":
           match = re.match(
-              "^" + export_dir_base.replace("\\", "/") + "/(\\d{8})$",
+              r"^" + export_dir_base.replace("\\", "/") + r"/(\d{8})$",
               path.path.replace("\\", "/"))
         else:
-          match = re.match("^" + export_dir_base + "/(\\d{8})$", path.path)
+          match = re.match(r"^" + export_dir_base + r"/(\d{8})$", path.path)
         if not match:
           return None
         return path._replace(export_version=int(match.group(1)))
diff --git a/tensorflow/contrib/session_bundle/gc_test.py b/tensorflow/contrib/session_bundle/gc_test.py
index 8faf3ef3d4..02725bb1cb 100644
--- a/tensorflow/contrib/session_bundle/gc_test.py
+++ b/tensorflow/contrib/session_bundle/gc_test.py
@@ -104,7 +104,7 @@ class GcTest(test_util.TensorFlowTestCase):
 
     # create a simple parser that pulls the export_version from the directory.
     def parser(path):
-      match = re.match("^" + base_dir + "/(\\d+)$", path.path)
+      match = re.match(r"^" + base_dir + r"/(\d+)$", path.path)
       if not match:
         return None
       return path._replace(export_version=int(match.group(1)))
diff --git a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
index e1cd9e7622..7ae2df6b01 100644
--- a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
+++ b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
@@ -325,7 +325,7 @@ class StatefulScatterNdTest(test.TestCase):
     shape = np.array([2, 2, 2])
     ref = variables.Variable(array_ops.zeros(shape, dtypes.int32))
     with self.assertRaisesWithPredicateMatch(
-        ValueError, "The outer \\d+ dimensions of indices\\.shape="):
+        ValueError, r"The outer \d+ dimensions of indices\.shape="):
       state_ops.scatter_nd_update(ref, indices, updates)
 
   @test_util.run_deprecated_v1
@@ -335,7 +335,7 @@ class StatefulScatterNdTest(test.TestCase):
     shape = np.array([2, 2, 2])
     ref = variables.Variable(array_ops.zeros(shape, dtypes.int32))
     with self.assertRaisesWithPredicateMatch(
-        ValueError, "The inner \\d+ dimensions of input\\.shape="):
+        ValueError, r"The inner \d+ dimensions of input\.shape="):
       state_ops.scatter_nd_update(ref, indices, updates)
 
   @test_util.run_deprecated_v1
@@ -539,7 +539,7 @@ class ScatterNdTest(test.TestCase):
     updates = array_ops.zeros([2, 2, 2], dtypes.int32)
     shape = np.array([2, 2, 2])
     with self.assertRaisesWithPredicateMatch(
-        ValueError, "The outer \\d+ dimensions of indices\\.shape="):
+        ValueError, r"The outer \d+ dimensions of indices\.shape="):
       self.scatter_nd(indices, updates, shape)
 
   @test_util.run_deprecated_v1
@@ -548,7 +548,7 @@ class ScatterNdTest(test.TestCase):
     updates = array_ops.zeros([2, 2], dtypes.int32)
     shape = np.array([2, 2, 2])
     with self.assertRaisesWithPredicateMatch(
-        ValueError, "The inner \\d+ dimensions of (input|output)\\.shape="):
+        ValueError, r"The inner \d+ dimensions of (input|output)\.shape="):
       self.scatter_nd(indices, updates, shape)
 
   @test_util.run_deprecated_v1
-- 
GitLab


From b86e0ff748dd66f7450bf6859cc881ff5f1c34ba Mon Sep 17 00:00:00 2001
From: Vojtech Bardiovsky <vbardiovsky@google.com>
Date: Fri, 11 Jan 2019 05:39:19 -0800
Subject: [PATCH 0586/1095] Added nested structure reconstruction on output.

PiperOrigin-RevId: 228868264
---
 tensorflow/python/eager/function.py           | 25 +++++++++----
 .../saved_model/function_deserialization.py   |  8 ++---
 .../saved_model/function_serialization.py     |  6 +++-
 tensorflow/python/saved_model/load.py         | 35 ++++++++++++++-----
 tensorflow/python/saved_model/load_test.py    | 30 ++++++++++++++++
 .../saved_model/saved_object_graph.proto      |  6 +++-
 6 files changed, 87 insertions(+), 23 deletions(-)

diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index 9a604620fa..e21ad4e2ee 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -560,9 +560,14 @@ class Function(object):
 
   @property
   def outputs(self):
-    """Returns tensors in `self.graph` corresponding to return values."""
+    """Returns tensors in `self.graph` corresponding to returned tensors."""
     return self._func_graph.outputs
 
+  @property
+  def structured_outputs(self):
+    """Returns outputs in `self.graph` as returned by the original function."""
+    return self._func_graph.structured_outputs
+
   @property
   def captured_inputs(self):
     """Returns external Tensors captured by this function.
@@ -814,8 +819,16 @@ class UnknownArgument(object):
   pass
 
 
-def _convert_inputs_to_signature(inputs):
-  """Convert a list of potentially nested inputs to a signature."""
+def convert_structure_to_signature(structure):
+  """Convert a potentially nested structure to a signature.
+
+  Args:
+    structure: Structure to convert.
+
+  Returns:
+    Identical structure that has TensorSpec objects instead of Tensors and
+    UknownArgument instead of any unsupported types.
+  """
 
   def encode_arg(arg, name=None):
     """A representation for this argument, for converting into signatures."""
@@ -827,9 +840,9 @@ def _convert_inputs_to_signature(inputs):
 
   # We are using the flattened paths to name the TensorSpecs. We need an
   # explicit name for them downstream.
-  flattened_with_paths = nest.flatten_with_joined_string_paths(inputs)
+  flattened_with_paths = nest.flatten_with_joined_string_paths(structure)
   mapped = [encode_arg(arg, path) for path, arg in flattened_with_paths]
-  return nest.pack_sequence_as(inputs, mapped)
+  return nest.pack_sequence_as(structure, mapped)
 
 
 pywrap_tensorflow.RegisterType("Tensor", ops.Tensor)
@@ -1310,7 +1323,7 @@ class PolymorphicFunction(object):
         if self._input_signature:
           python_call_signature = self._input_signature
         else:
-          python_call_signature = tuple(_convert_inputs_to_signature(args))
+          python_call_signature = tuple(convert_structure_to_signature(args))
         # pylint: disable=protected-access
         # Save information about non-Tensor arguments with the concrete
         # function. Used to serialize PolymorphicFunctions.
diff --git a/tensorflow/python/saved_model/function_deserialization.py b/tensorflow/python/saved_model/function_deserialization.py
index 3cf27fd5a4..ec6cefb606 100644
--- a/tensorflow/python/saved_model/function_deserialization.py
+++ b/tensorflow/python/saved_model/function_deserialization.py
@@ -90,8 +90,7 @@ def recreate_polymorphic_function(
     for monomorphic_function in saved_polymorphic_function.monomorphic_function:
       function_obj = functions[monomorphic_function.concrete_function]
       canonicalized_original_inputs = coder.decode_proto(
-          monomorphic_function.canonicalized_input)
-
+          monomorphic_function.canonicalized_input_signature)
       try:
         can_args, can_kwargs = function_spec.canonicalize_function_inputs(
             *args, **kwargs)
@@ -108,10 +107,7 @@ def recreate_polymorphic_function(
                             canonicalized_original_inputs):
         flattened_inputs = nest.flatten(can_args)
         filtered_inputs = [t for t in flattened_inputs if _is_tensor(t)]
-        flattened_outputs = function_obj._call_flat(filtered_inputs)  # pylint: disable=protected-access
-        # TODO(vbardiovsky): Rebuild output structure.
-        single_output, = flattened_outputs
-        return single_output
+        return function_obj._call_flat(filtered_inputs)  # pylint: disable=protected-access
 
     raise AssertionError(
         "Could not find matching function to call for arguments: %s" % (args,))
diff --git a/tensorflow/python/saved_model/function_serialization.py b/tensorflow/python/saved_model/function_serialization.py
index c826ffa02c..267852ef4a 100644
--- a/tensorflow/python/saved_model/function_serialization.py
+++ b/tensorflow/python/saved_model/function_serialization.py
@@ -47,8 +47,12 @@ def serialize_polymorphic_function(polymorphic_function, node_ids):
       continue
     function_proto = proto.monomorphic_function.add()
     function_proto.concrete_function = concrete_function.name
-    function_proto.canonicalized_input.CopyFrom(
+    function_proto.canonicalized_input_signature.CopyFrom(
         coder.encode_structure(signature))
+    structured_outputs = defun_lib.convert_structure_to_signature(
+        concrete_function.structured_outputs)
+    function_proto.output_signature.CopyFrom(
+        coder.encode_structure(structured_outputs))
     function_proto.bound_inputs.extend(bound_inputs)
   return proto
 
diff --git a/tensorflow/python/saved_model/load.py b/tensorflow/python/saved_model/load.py
index bbb1485612..d184133fe9 100644
--- a/tensorflow/python/saved_model/load.py
+++ b/tensorflow/python/saved_model/load.py
@@ -27,6 +27,7 @@ from tensorflow.python.ops import variables
 from tensorflow.python.saved_model import constants
 from tensorflow.python.saved_model import function_deserialization
 from tensorflow.python.saved_model import loader_impl
+from tensorflow.python.saved_model import nested_structure_coder
 from tensorflow.python.saved_model import saved_object_graph_pb2
 from tensorflow.python.saved_model import utils_impl as saved_model_utils
 from tensorflow.python.training.checkpointable import tracking
@@ -45,12 +46,13 @@ class _Loader(object):
     self._functions = function_deserialization.load_function_def_library(
         meta_graph.graph_def.library)
     self._load_all()
-    self._bind_function_captures()
+    self._setup_concrete_functions()
     self._restore_checkpoint()
 
-  def _bind_function_captures(self):
-    """Setup captured tensors in restored concrete functions."""
+  def _setup_concrete_functions(self):
+    """Setup captured tensors and output structure in restored functions."""
     seen_functions = set()
+    coder = nested_structure_coder.StructureCoder()
     for object_proto in self._proto.nodes:
       if object_proto.WhichOneof("kind") == "function":
         for monomorphic_function in object_proto.function.monomorphic_function:
@@ -64,17 +66,32 @@ class _Loader(object):
               if self._proto.nodes[node_id].WhichOneof("kind") == "variable"
           ]
           if name in seen_functions:
-            if self._functions[name]._captured_inputs != bound_inputs:  # pylint: disable=protected-access
-              raise NotImplementedError(
-                  "Function %s is used more than once with different "
-                  "captured inputs." % name)
+            raise RuntimeError(
+                "Monomorphic function with a duplicate name: %s." % name)
           else:
             seen_functions.add(name)
             # TODO(andresp): This is only injecting the captured inputs into the
             # concrete function, note that we did not modify the FuncGraph
             # itself.
-            self._functions[name]._captured_inputs = bound_inputs  # pylint: disable=protected-access
-            self._functions[name]._func_graph.variables = bound_variables  # pylint: disable=protected-access
+            function = self._functions[name]
+            function._captured_inputs = bound_inputs  # pylint: disable=protected-access
+            function._func_graph.variables = bound_variables  # pylint: disable=protected-access
+            # By setting the structured_outputs directly, we can rely on this
+            # function_lib.Function object to perform the output repacking
+            # logic. The only limitation of that logic is that it only works
+            # with output that is convertible to Tensors and the conversion
+            # always happens. For example tf.TensorShape([2, 3]) will be
+            # converted to Tensor representing [2, 3].
+            original_outputs = coder.decode_proto(
+                monomorphic_function.output_signature)
+            # The original_outputs here had Tensors converted to TensorSpecs, so
+            # the restored function's structured_outputs field will not be
+            # exactly the same. Fortunately the repacking logic cares only about
+            # the structure.
+            # TODO(vbardiovsky): Should we just replicate the structures, with
+            # Nones instead of real objects? Decide when we start solving
+            # idempotency.
+            function._func_graph.structured_outputs = original_outputs  # pylint: disable=protected-access
 
   def _get_tensor_from_node(self, node_id):
     obj = self._nodes[node_id]
diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index 32bb841556..defd269012 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import collections
 import os
 import tempfile
 
@@ -218,6 +219,35 @@ class LoadTest(test.TestCase):
     self.assertEqual(31, imported.f(input1).numpy())
     self.assertEqual(32, imported.f(input3).numpy())
 
+  def test_structured_output(self):
+
+    # Use fields with non-alphabetical order
+    named_tuple_type = collections.namedtuple("NamedTupleHello", ["b", "a"])
+
+    def func(input1, input2):
+      named_tuple = named_tuple_type(a=input1 + input2, b=input1 * input2)
+      return [named_tuple, input2, {"x": 0.5}]
+
+    root = tracking.Checkpointable()
+    root.f = def_function.function(func)
+
+    result = root.f(constant_op.constant(2), constant_op.constant(3))
+
+    self.assertEqual(5, result[0].a.numpy())
+    self.assertEqual(6, result[0].b.numpy())
+    self.assertEqual(["b", "a"], list(result[0]._asdict().keys()))
+    self.assertEqual(3, result[1].numpy())
+    self.assertEqual(0.5, result[2]["x"].numpy())
+
+    imported = self.cycle(root)
+
+    result = imported.f(constant_op.constant(2), constant_op.constant(5))
+    self.assertEqual(7, result[0].a.numpy())
+    self.assertEqual(10, result[0].b.numpy())
+    self.assertEqual(["b", "a"], list(result[0]._asdict().keys()))
+    self.assertEqual(5, result[1].numpy())
+    self.assertEqual(0.5, result[2]["x"].numpy())
+
   def test_positional_arguments(self):
     def func(x, training=False, abc=7.1, defg=7.7):
       del abc
diff --git a/tensorflow/python/saved_model/saved_object_graph.proto b/tensorflow/python/saved_model/saved_object_graph.proto
index 1e2514b7f7..a870eee383 100644
--- a/tensorflow/python/saved_model/saved_object_graph.proto
+++ b/tensorflow/python/saved_model/saved_object_graph.proto
@@ -96,7 +96,11 @@ message SavedMonomorphicFunction {
   repeated int32 bound_inputs = 2;
   // Input in canonicalized form that was received to create this concrete
   // function.
-  StructuredValue canonicalized_input = 3;
+  StructuredValue canonicalized_input_signature = 3;
+  // Output that was the return value of this function after replacing all
+  // Tensors with TensorSpecs. This can be an arbitrary nested function and will
+  // be used to reconstruct the full structure from pure tensors.
+  StructuredValue output_signature = 4;
 }
 
 // Represents a Variable that is initialized by loading the contents from the
-- 
GitLab


From b044b5977e6c0425d4eab66d7f9daa11e161e3ee Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Fri, 11 Jan 2019 06:01:22 -0800
Subject: [PATCH 0587/1095] Support S16 and U16 in the HLO evaluator

This allows to enable ComparatorsTest.CompareLtF16 and ComparatorsTest.CompareGtF16 tests.
While at it, also enable other evaluator tests which are already passing.

PiperOrigin-RevId: 228869860
---
 .../xla/client/lib/comparators_test.cc        |  5 ++--
 tensorflow/compiler/xla/literal.cc            |  4 +++
 tensorflow/compiler/xla/literal_test.cc       | 24 +++++++++++++++--
 tensorflow/compiler/xla/service/BUILD         |  2 ++
 .../compiler/xla/service/hlo_evaluator.cc     | 26 +++++++++----------
 .../hlo_evaluator_typed_visitor_int16.cc      | 22 ++++++++++++++++
 .../hlo_evaluator_typed_visitor_uint16.cc     | 22 ++++++++++++++++
 tensorflow/compiler/xla/tests/tuple_test.cc   |  3 +--
 tensorflow/compiler/xla/tests/while_test.cc   |  4 +--
 9 files changed, 89 insertions(+), 23 deletions(-)
 create mode 100644 tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor_int16.cc
 create mode 100644 tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor_uint16.cc

diff --git a/tensorflow/compiler/xla/client/lib/comparators_test.cc b/tensorflow/compiler/xla/client/lib/comparators_test.cc
index efec984eb2..598956803b 100644
--- a/tensorflow/compiler/xla/client/lib/comparators_test.cc
+++ b/tensorflow/compiler/xla/client/lib/comparators_test.cc
@@ -103,15 +103,14 @@ XLA_TEST_F(ComparatorsTest, CompareGtBF16) {
   ComputeAndCompareR1<bool>(builder(), expected, {});
 }
 
-// The interpreter doesn't support S16, and F16 results in comparisons with S16.
-XLA_TEST_F(ComparatorsTest, DISABLED_ON_INTERPRETER(CompareLtF16)) {
+XLA_TEST_F(ComparatorsTest, CompareLtF16) {
   absl::InlinedVector<bool, 10> expected;
   BuildComparatorAndComparisons<F16>(this, /*compare_less_than=*/true,
                                      &expected);
   ComputeAndCompareR1<bool>(builder(), expected, {});
 }
 
-XLA_TEST_F(ComparatorsTest, DISABLED_ON_INTERPRETER(CompareGtF16)) {
+XLA_TEST_F(ComparatorsTest, CompareGtF16) {
   absl::InlinedVector<bool, 10> expected;
   BuildComparatorAndComparisons<F16>(this, /*compare_less_than=*/false,
                                      &expected);
diff --git a/tensorflow/compiler/xla/literal.cc b/tensorflow/compiler/xla/literal.cc
index 6c44189930..81874fa405 100644
--- a/tensorflow/compiler/xla/literal.cc
+++ b/tensorflow/compiler/xla/literal.cc
@@ -1319,9 +1319,11 @@ StatusOr<Literal> ConvertIfDestTypeMatches(const LiteralBase& src_literal,
                                                            bitcast);
     CONVERT_IF_TYPES_MATCH(PRED)
     CONVERT_IF_TYPES_MATCH(S8)
+    CONVERT_IF_TYPES_MATCH(S16)
     CONVERT_IF_TYPES_MATCH(S32)
     CONVERT_IF_TYPES_MATCH(S64)
     CONVERT_IF_TYPES_MATCH(U8)
+    CONVERT_IF_TYPES_MATCH(U16)
     CONVERT_IF_TYPES_MATCH(U32)
     CONVERT_IF_TYPES_MATCH(U64)
     CONVERT_IF_TYPES_MATCH(F16)
@@ -1357,9 +1359,11 @@ StatusOr<Literal> ConvertSwitch(const LiteralBase& literal,
                                             bitcast);
     CONVERT_IF_DEST_TYPE_MATCHES(PRED)
     CONVERT_IF_DEST_TYPE_MATCHES(S8)
+    CONVERT_IF_DEST_TYPE_MATCHES(S16)
     CONVERT_IF_DEST_TYPE_MATCHES(S32)
     CONVERT_IF_DEST_TYPE_MATCHES(S64)
     CONVERT_IF_DEST_TYPE_MATCHES(U8)
+    CONVERT_IF_DEST_TYPE_MATCHES(U16)
     CONVERT_IF_DEST_TYPE_MATCHES(U32)
     CONVERT_IF_DEST_TYPE_MATCHES(U64)
     CONVERT_IF_DEST_TYPE_MATCHES(F16)
diff --git a/tensorflow/compiler/xla/literal_test.cc b/tensorflow/compiler/xla/literal_test.cc
index 0a71cfd5ab..a4088c6d11 100644
--- a/tensorflow/compiler/xla/literal_test.cc
+++ b/tensorflow/compiler/xla/literal_test.cc
@@ -1237,11 +1237,21 @@ TEST_F(LiteralUtilTest, ConvertIfTypesMatch) {
     {{0, 19, 0, 21}, {22, 0, 24, 0}},
     {{26, 0, 28, 0}, {0, 31, 0, 33}},
   }}, layout_r4_dim0major_);
+  auto s16 = LiteralUtil::CreateR4WithLayout<int16>({{
+    {{10, 0, 12, 0}, {0, 15, 0, 17}},
+    {{0, 19, 0, 21}, {22, 0, 24, 0}},
+    {{26, 0, 28, 0}, {0, 31, 0, 33}},
+  }}, layout_r4_dim0major_);
   auto s32 = LiteralUtil::CreateR4WithLayout<int32>({{
     {{10, 0, 12, 0}, {0, 15, 0, 17}},
     {{0, 19, 0, 21}, {22, 0, 24, 0}},
     {{26, 0, 28, 0}, {0, 31, 0, 33}},
   }}, layout_r4_dim0major_);
+  auto u16 = LiteralUtil::CreateR4WithLayout<uint16>({{
+    {{10, 0, 12, 0}, {0, 15, 0, 17}},
+    {{0, 19, 0, 21}, {22, 0, 24, 0}},
+    {{26, 0, 28, 0}, {0, 31, 0, 33}},
+  }}, layout_r4_dim0major_);
   auto u32 = LiteralUtil::CreateR4WithLayout<uint32>({{
     {{10, 0, 12, 0}, {0, 15, 0, 17}},
     {{0, 19, 0, 21}, {22, 0, 24, 0}},
@@ -1301,6 +1311,12 @@ TEST_F(LiteralUtilTest, ConvertIfTypesMatch) {
   // clang-format on
   Literal conv;
 
+  conv = s8.Convert(U16).ConsumeValueOrDie();
+  EXPECT_EQ(conv, u16);
+
+  conv = s8.Convert(S16).ConsumeValueOrDie();
+  EXPECT_EQ(conv, s16);
+
   conv = s8.Convert(U32).ConsumeValueOrDie();
   EXPECT_EQ(conv, u32);
 
@@ -1352,10 +1368,14 @@ TEST_F(LiteralUtilTest, ConvertIfTypesMatch) {
   conv = f16.Convert(C64).ConsumeValueOrDie();
   EXPECT_EQ(conv, c64);
 
+  conv = s32.Convert(S16).ConsumeValueOrDie();
+  EXPECT_EQ(conv, s16);
+
+  conv = s32.Convert(U16).ConsumeValueOrDie();
+  EXPECT_EQ(conv, u16);
+
   EXPECT_EQ(s32.Convert(TUPLE).status().code(),
             tensorflow::error::UNIMPLEMENTED);
-  EXPECT_EQ(s32.Convert(S16).status().code(), tensorflow::error::UNIMPLEMENTED);
-  EXPECT_EQ(s32.Convert(U16).status().code(), tensorflow::error::UNIMPLEMENTED);
   EXPECT_EQ(c64.Convert(F32).status().code(), tensorflow::error::UNIMPLEMENTED);
   EXPECT_EQ(c64.Convert(S32).status().code(), tensorflow::error::UNIMPLEMENTED);
 }
diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index 728adb67a0..ff14c500e3 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -227,9 +227,11 @@ cc_library(
         "hlo_evaluator_typed_visitor_double.cc",
         "hlo_evaluator_typed_visitor_float.cc",
         "hlo_evaluator_typed_visitor_half.cc",
+        "hlo_evaluator_typed_visitor_int16.cc",
         "hlo_evaluator_typed_visitor_int32.cc",
         "hlo_evaluator_typed_visitor_int64.cc",
         "hlo_evaluator_typed_visitor_int8.cc",
+        "hlo_evaluator_typed_visitor_uint16.cc",
         "hlo_evaluator_typed_visitor_uint32.cc",
         "hlo_evaluator_typed_visitor_uint64.cc",
         "hlo_evaluator_typed_visitor_uint8.cc",
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc
index b8d95fb244..407615f842 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc
@@ -150,22 +150,14 @@ HloEvaluator::HloEvaluator(int64 max_loop_iterations)
   typed_visitors_[U8] =
       absl::make_unique<HloEvaluatorTypedVisitor<uint8>>(this);
   typed_visitors_[U16] =
-      absl::make_unique<FunctionVisitor>([](HloInstruction*) {
-        return Unimplemented(
-            "HloEvaluator::HloEvaluatorTypedVisitor: unhandled primitive type: "
-            "U16.");
-      });
+      absl::make_unique<HloEvaluatorTypedVisitor<uint16>>(this);
   typed_visitors_[U32] =
       absl::make_unique<HloEvaluatorTypedVisitor<uint32>>(this);
   typed_visitors_[U64] =
       absl::make_unique<HloEvaluatorTypedVisitor<uint64>>(this);
   typed_visitors_[S8] = absl::make_unique<HloEvaluatorTypedVisitor<int8>>(this);
   typed_visitors_[S16] =
-      absl::make_unique<FunctionVisitor>([](HloInstruction*) {
-        return Unimplemented(
-            "HloEvaluator::HloEvaluatorTypedVisitor: unhandled primitive type: "
-            "S16.");
-      });
+      absl::make_unique<HloEvaluatorTypedVisitor<int16>>(this);
   typed_visitors_[S32] =
       absl::make_unique<HloEvaluatorTypedVisitor<int32>>(this);
   typed_visitors_[S64] =
@@ -595,8 +587,11 @@ Status HloEvaluator::HandleCompare(HloInstruction* compare) {
           evaluated_[compare],
           Compare<uint8>(compare->shape(), opcode, lhs_literal, rhs_literal));
     } break;
-    case U16:
-      return Unimplemented("unhandled primitive type: U16.");
+    case U16: {
+      TF_ASSIGN_OR_RETURN(
+          evaluated_[compare],
+          Compare<uint16>(compare->shape(), opcode, lhs_literal, rhs_literal));
+    } break;
     case U32: {
       TF_ASSIGN_OR_RETURN(
           evaluated_[compare],
@@ -612,8 +607,11 @@ Status HloEvaluator::HandleCompare(HloInstruction* compare) {
           evaluated_[compare],
           Compare<int8>(compare->shape(), opcode, lhs_literal, rhs_literal));
     } break;
-    case S16:
-      return Unimplemented("unhandled primitive type: S16.");
+    case S16: {
+      TF_ASSIGN_OR_RETURN(
+          evaluated_[compare],
+          Compare<int16>(compare->shape(), opcode, lhs_literal, rhs_literal));
+    } break;
     case S32: {
       TF_ASSIGN_OR_RETURN(
           evaluated_[compare],
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor_int16.cc b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor_int16.cc
new file mode 100644
index 0000000000..e54285a157
--- /dev/null
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor_int16.cc
@@ -0,0 +1,22 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h"
+
+#include "tensorflow/compiler/xla/service/hlo_evaluator.h"
+
+namespace xla {
+template class HloEvaluatorTypedVisitor<int16>;
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor_uint16.cc b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor_uint16.cc
new file mode 100644
index 0000000000..cc708952d2
--- /dev/null
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor_uint16.cc
@@ -0,0 +1,22 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h"
+
+#include "tensorflow/compiler/xla/service/hlo_evaluator.h"
+
+namespace xla {
+template class HloEvaluatorTypedVisitor<uint16>;
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/tests/tuple_test.cc b/tensorflow/compiler/xla/tests/tuple_test.cc
index 426d6c84ee..0def0577e0 100644
--- a/tensorflow/compiler/xla/tests/tuple_test.cc
+++ b/tensorflow/compiler/xla/tests/tuple_test.cc
@@ -513,8 +513,7 @@ XLA_TEST_F(TupleTest, ComplexTuples) {
 
 class TupleHloTest : public HloTestBase {};
 
-// Disabled on the interpreter because bitcast doesn't exist on the interpreter.
-XLA_TEST_F(TupleHloTest, DISABLED_ON_INTERPRETER(BitcastAfterGTE)) {
+XLA_TEST_F(TupleHloTest, BitcastAfterGTE) {
   const char* testcase = R"(
     HloModule m, is_scheduled=true
 
diff --git a/tensorflow/compiler/xla/tests/while_test.cc b/tensorflow/compiler/xla/tests/while_test.cc
index 6d5f276e82..b1566dc8ad 100644
--- a/tensorflow/compiler/xla/tests/while_test.cc
+++ b/tensorflow/compiler/xla/tests/while_test.cc
@@ -901,7 +901,7 @@ XLA_TEST_F(WhileTest, WhileWithDynamicUpdateSlice) {
 // Per backend the values generated can be different as the different backends
 // use different random number generators.
 // TODO(b/32240857): Extend test to verify outputs.
-XLA_TEST_F(WhileTest, DISABLED_ON_INTERPRETER(WhileWithPrngScalarResult)) {
+XLA_TEST_F(WhileTest, WhileWithPrngScalarResult) {
   auto v6s32 = ShapeUtil::MakeShape(S32, {6});
 
   // Create a computation for the condition: repeat for count iterations.
@@ -1146,7 +1146,7 @@ XLA_TEST_F(WhileTest, NestedWhileWithScalarResult) {
 // while (f(result).get<0>()) {
 //   result = result + 1;
 // }
-XLA_TEST_F(WhileTest, DISABLED_ON_INTERPRETER(WhileWithCallInsideCondition)) {
+XLA_TEST_F(WhileTest, WhileWithCallInsideCondition) {
   auto result_shape = ShapeUtil::MakeShape(S32, {});
 
   // Create a computation for the condition: repeat for 5 iterations.
-- 
GitLab


From 277636caf62cddf094806129239c6ae6788c636a Mon Sep 17 00:00:00 2001
From: Sergei Lebedev <slebedev@google.com>
Date: Fri, 11 Jan 2019 09:28:34 -0800
Subject: [PATCH 0588/1095] Corrected ResourceVariable constructor
 documentation wrt validate_shape

PiperOrigin-RevId: 228894072
---
 tensorflow/python/ops/resource_variable_ops.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index 2d46e21074..1f2ff9f197 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -219,7 +219,7 @@ class ResourceVariable(variables.VariableV1):
                initial_value=None,
                trainable=True,
                collections=None,
-               validate_shape=True,
+               validate_shape=True,  # pylint: disable=unused-argument
                caching_device=None,
                name=None,
                dtype=None,
@@ -231,8 +231,7 @@ class ResourceVariable(variables.VariableV1):
 
     Args:
       initial_value: A `Tensor`, or Python object convertible to a `Tensor`,
-        which is the initial value for the Variable. The initial value must have
-        a shape specified unless `validate_shape` is set to False. Can also be a
+        which is the initial value for the Variable. Can also be a
         callable with no argument that returns the initial value when called.
         (Note that initializer functions from init_ops.py must first be bound
          to a shape before being used here.)
@@ -292,7 +291,6 @@ class ResourceVariable(variables.VariableV1):
           initial_value=initial_value,
           trainable=trainable,
           collections=collections,
-          validate_shape=validate_shape,
           caching_device=caching_device,
           name=name,
           dtype=dtype,
@@ -307,12 +305,10 @@ class ResourceVariable(variables.VariableV1):
       return "<tf.Variable '%s' shape=%s dtype=%s>" % (
           self.name, self.get_shape(), self.dtype.name)
 
-  # pylint: disable=unused-argument
   def _init_from_args(self,
                       initial_value=None,
                       trainable=True,
                       collections=None,
-                      validate_shape=True,
                       caching_device=None,
                       name=None,
                       dtype=None,
-- 
GitLab


From 196715b526a695678891ec11d13b7290b3f6861a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 11 Jan 2019 09:32:51 -0800
Subject: [PATCH 0589/1095] Remove unnecessary dependency on :pooling_ops from
 NN_DEPS (used by :bias_op and others.)

PiperOrigin-RevId: 228894672
---
 tensorflow/core/kernels/BUILD                  | 1 -
 tensorflow/core/kernels/fused_batch_norm_op.cc | 1 -
 2 files changed, 2 deletions(-)

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 4db0505a8f..0792b398f1 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -3820,7 +3820,6 @@ NN_DEPS = [
     ":conv_2d",
     ":eigen_contraction_kernel",
     ":ops_util",
-    ":pooling_ops",
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
diff --git a/tensorflow/core/kernels/fused_batch_norm_op.cc b/tensorflow/core/kernels/fused_batch_norm_op.cc
index dbd3bb05db..48b339508b 100644
--- a/tensorflow/core/kernels/fused_batch_norm_op.cc
+++ b/tensorflow/core/kernels/fused_batch_norm_op.cc
@@ -18,7 +18,6 @@ limitations under the License.
 #if GOOGLE_CUDA
 #define EIGEN_USE_GPU
 #include "tensorflow/core/kernels/conv_2d.h"
-#include "tensorflow/core/kernels/conv_ops_gpu.h"
 #include "tensorflow/core/util/stream_executor_util.h"
 #endif
 
-- 
GitLab


From 93a4a83b08f0290926bca774013f4667e4634bf5 Mon Sep 17 00:00:00 2001
From: Dheeraj Rajaram Reddy <dheeraj98reddy@gmail.com>
Date: Fri, 11 Jan 2019 23:33:02 +0530
Subject: [PATCH 0590/1095] Add take_api golden API to pbtxt

---
 .../tools/api/golden/v2/tensorflow.data.experimental.pbtxt    | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt
index a262c0f799..abc98a74b6 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt
@@ -180,6 +180,10 @@ tf_module {
     name: "shuffle_and_repeat"
     argspec: "args=[\'buffer_size\', \'count\', \'seed\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
+  member_method {
+    name: "take_while"
+    argspec: "args=[\'predicate\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "unbatch"
     argspec: "args=[], varargs=None, keywords=None, defaults=None"
-- 
GitLab


From 8d0896bc0020c68faaf56b4240db6da144b584d4 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Fri, 11 Jan 2019 10:07:16 -0800
Subject: [PATCH 0591/1095] Make TopK test more explicit

---
 tensorflow/contrib/tensorrt/test/topk_test.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/test/topk_test.py b/tensorflow/contrib/tensorrt/test/topk_test.py
index 2edf7cb587..efbfb26898 100644
--- a/tensorflow/contrib/tensorrt/test/topk_test.py
+++ b/tensorflow/contrib/tensorrt/test/topk_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import constant_op
 from tensorflow.python.ops import nn_ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
@@ -37,7 +38,8 @@ class TopKTest(trt_test.TfTrtIntegrationTestBase):
     g = ops.Graph()
     with g.as_default():
       x = array_ops.placeholder(dtype=dtype, shape=input_dims, name=input_name)
-      values, indices = nn_ops.top_k(x, k)
+      k_tensor = constant_op.constant(k, dtype=tf.int32, name="Const")
+      values, indices = nn_ops.top_k(x, k_tensor, name="TopK")
       values = array_ops.identity(values, name='output_values')
       indices = array_ops.identity(indices, name='output_indices')
     return trt_test.TfTrtIntegrationTestParams(
@@ -49,7 +51,7 @@ class TopKTest(trt_test.TfTrtIntegrationTestBase):
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
-    return ["TRTEngineOp_0"]
+    return {"TRTEngineOp_0" : ["Const", "TopK"]}
 
 
 if __name__ == "__main__":
-- 
GitLab


From 1546c90bfea6e6e072eb889f4986a8db78c10ab4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 11 Jan 2019 10:03:13 -0800
Subject: [PATCH 0592/1095] Add bitwise ops to ragged_dispatch

PiperOrigin-RevId: 228899228
---
 tensorflow/python/ops/ragged/ragged_dispatch.py      | 7 +++++++
 tensorflow/python/ops/ragged/ragged_dispatch_test.py | 7 +++++++
 2 files changed, 14 insertions(+)

diff --git a/tensorflow/python/ops/ragged/ragged_dispatch.py b/tensorflow/python/ops/ragged/ragged_dispatch.py
index 52c4142346..970d88663a 100644
--- a/tensorflow/python/ops/ragged/ragged_dispatch.py
+++ b/tensorflow/python/ops/ragged/ragged_dispatch.py
@@ -25,6 +25,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import clip_ops
+from tensorflow.python.ops import gen_bitwise_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import parsing_ops
 from tensorflow.python.ops import string_ops
@@ -282,6 +283,7 @@ _UNARY_ELEMENTWISE_OPS = [
     array_ops.zeros_like,
     array_ops.zeros_like_v2,
     clip_ops.clip_by_value,
+    gen_bitwise_ops.invert,
     math_ops.abs,
     math_ops.acos,
     math_ops.acosh,
@@ -348,6 +350,11 @@ _UNARY_LIST_ELEMENTWISE_OPS = [
 ]
 
 _BINARY_ELEMENTWISE_OPS = [
+    gen_bitwise_ops.bitwise_and,
+    gen_bitwise_ops.bitwise_or,
+    gen_bitwise_ops.bitwise_xor,
+    gen_bitwise_ops.left_shift,
+    gen_bitwise_ops.right_shift,
     math_ops.add,
     math_ops.atan2,
     math_ops.complex,
diff --git a/tensorflow/python/ops/ragged/ragged_dispatch_test.py b/tensorflow/python/ops/ragged/ragged_dispatch_test.py
index 0c546accc3..1aa612633e 100644
--- a/tensorflow/python/ops/ragged/ragged_dispatch_test.py
+++ b/tensorflow/python/ops/ragged/ragged_dispatch_test.py
@@ -29,6 +29,7 @@ from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import clip_ops
+from tensorflow.python.ops import gen_bitwise_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import parsing_ops
 from tensorflow.python.ops import string_ops
@@ -121,9 +122,15 @@ BINARY_BOOL_OPS = [
     math_ops.logical_xor,
 ]
 UNARY_INT_OPS = [
+    gen_bitwise_ops.invert,
     string_ops.unicode_script,
 ]
 BINARY_INT_OPS = [
+    gen_bitwise_ops.bitwise_and,
+    gen_bitwise_ops.bitwise_or,
+    gen_bitwise_ops.bitwise_xor,
+    gen_bitwise_ops.left_shift,
+    gen_bitwise_ops.right_shift,
     math_ops.truncatediv,
     math_ops.truncatemod,
 ]
-- 
GitLab


From 78032512e66dd8a829dcfce338c4fb50e3b9ed49 Mon Sep 17 00:00:00 2001
From: Sergei Lebedev <slebedev@google.com>
Date: Fri, 11 Jan 2019 10:18:57 -0800
Subject: [PATCH 0593/1095] Lifted *_slice_info methods to Variable

That way they are easier to delete once PartitionInfo is gone.

PiperOrigin-RevId: 228901926
---
 .../python/ops/resource_variable_ops.py       | 11 ----------
 tensorflow/python/ops/variables.py            | 22 +++++++++----------
 2 files changed, 11 insertions(+), 22 deletions(-)

diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index 1f2ff9f197..2681deb78c 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -730,17 +730,6 @@ class ResourceVariable(variables.VariableV1):
     return gen_state_ops.resource_count_up_to(self.handle, limit=limit,
                                               T=self.dtype)
 
-  def _set_save_slice_info(self, save_slice_info):
-    """Sets the slice info for this `ResourceVariable`.
-
-    Args:
-      save_slice_info: A `Variable.SaveSliceInfo` object.
-    """
-    self._save_slice_info = save_slice_info
-
-  def _get_save_slice_info(self):
-    return self._save_slice_info
-
   def _read_variable_op(self):
     if self.trainable:
       tape.variable_accessed(self)
diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py
index f9fc72a6da..d765936399 100644
--- a/tensorflow/python/ops/variables.py
+++ b/tensorflow/python/ops/variables.py
@@ -1049,6 +1049,17 @@ class Variable(six.with_metaclass(VariableMetaclass,
     return RefVariable(variable_def=variable_def,
                        import_scope=import_scope)
 
+  def _set_save_slice_info(self, save_slice_info):
+    """Sets the slice info for this `Variable`.
+
+    Args:
+      save_slice_info: A `Variable.SaveSliceInfo` object.
+    """
+    self._save_slice_info = save_slice_info
+
+  def _get_save_slice_info(self):
+    return self._save_slice_info
+
   class SaveSliceInfo(object):
     """Information on how to save this Variable as a slice.
 
@@ -2295,17 +2306,6 @@ class RefVariable(VariableV1):
         " if you want a new python Tensor object.", 1)
     return self ** other
 
-  def _set_save_slice_info(self, save_slice_info):
-    """Sets the slice info for this `Variable`.
-
-    Args:
-      save_slice_info: A `Variable.SaveSliceInfo` object.
-    """
-    self._save_slice_info = save_slice_info
-
-  def _get_save_slice_info(self):
-    return self._save_slice_info
-
 
 def _try_guard_against_uninitialized_dependencies(name, initial_value):
   """Attempt to guard against dependencies on uninitialized variables.
-- 
GitLab


From a4f5558529a56d72bdcec9c0c8d0a3fb2a91fb64 Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Fri, 11 Jan 2019 10:33:58 -0800
Subject: [PATCH 0594/1095] Fix typos in deprecation messages for convolutional
 layers for TF 2.0 API.

PiperOrigin-RevId: 228904460
---
 tensorflow/python/layers/convolutional.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/tensorflow/python/layers/convolutional.py b/tensorflow/python/layers/convolutional.py
index 5d4805e245..03344c844d 100644
--- a/tensorflow/python/layers/convolutional.py
+++ b/tensorflow/python/layers/convolutional.py
@@ -117,7 +117,7 @@ class Conv1D(keras_layers.Conv1D, base.Layer):
 
 @deprecation.deprecated(
     date=None,
-    instructions='Use keras.layers.conv1d instead.')
+    instructions='Use tf.keras.layers.Conv1D instead.')
 @tf_export(v1=['layers.conv1d'])
 def conv1d(inputs,
            filters,
@@ -316,7 +316,7 @@ class Conv2D(keras_layers.Conv2D, base.Layer):
 
 @deprecation.deprecated(
     date=None,
-    instructions='Use keras.layers.conv2d instead.')
+    instructions='Use tf.keras.layers.Conv2D instead.')
 @tf_export(v1=['layers.conv2d'])
 def conv2d(inputs,
            filters,
@@ -523,7 +523,7 @@ class Conv3D(keras_layers.Conv3D, base.Layer):
 
 @deprecation.deprecated(
     date=None,
-    instructions='Use keras.layers.conv3d instead.')
+    instructions='Use tf.keras.layers.Conv3D instead.')
 @tf_export(v1=['layers.conv3d'])
 def conv3d(inputs,
            filters,
@@ -853,7 +853,7 @@ class SeparableConv2D(keras_layers.SeparableConv2D, base.Layer):
 
 @deprecation.deprecated(
     date=None,
-    instructions='Use keras.layers.separable_conv1d instead.')
+    instructions='Use tf.keras.layers.SeparableConv1D instead.')
 @tf_export(v1=['layers.separable_conv1d'])
 def separable_conv1d(inputs,
                      filters,
@@ -973,7 +973,7 @@ def separable_conv1d(inputs,
 
 @deprecation.deprecated(
     date=None,
-    instructions='Use keras.layers.separable_conv2d instead.')
+    instructions='Use tf.keras.layers.SeparableConv2D instead.')
 @tf_export(v1=['layers.separable_conv2d'])
 def separable_conv2d(inputs,
                      filters,
@@ -1183,7 +1183,7 @@ class Conv2DTranspose(keras_layers.Conv2DTranspose, base.Layer):
 
 @deprecation.deprecated(
     date=None,
-    instructions='Use keras.layers.conv2d_transpose instead.')
+    instructions='Use tf.keras.layers.Conv2DTranspose instead.')
 @tf_export(v1=['layers.conv2d_transpose'])
 def conv2d_transpose(inputs,
                      filters,
@@ -1363,7 +1363,7 @@ class Conv3DTranspose(keras_layers.Conv3DTranspose, base.Layer):
 
 @deprecation.deprecated(
     date=None,
-    instructions='Use keras.layers.conv3d_transpose instead.')
+    instructions='Use tf.keras.layers.Conv3DTranspose instead.')
 @tf_export(v1=['layers.conv3d_transpose'])
 def conv3d_transpose(inputs,
                      filters,
-- 
GitLab


From 1dea159cd3cbde8763544ecc961ef11c9d748e50 Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Fri, 11 Jan 2019 10:34:40 -0800
Subject: [PATCH 0595/1095] [tf.data] Adding a benchmark for parallel
 interleave and using it to improve the performance of the v2 implementation.

PiperOrigin-RevId: 228904615
---
 .../data/parallel_interleave_dataset_op.cc    |   8 +-
 .../python/data/experimental/benchmarks/BUILD |  16 +++
 .../parallel_interleave_benchmark.py          | 103 ++++++++++++++++++
 3 files changed, 124 insertions(+), 3 deletions(-)
 create mode 100644 tensorflow/python/data/experimental/benchmarks/parallel_interleave_benchmark.py

diff --git a/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc b/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc
index dc2663d1e0..336fa52b49 100644
--- a/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc
+++ b/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc
@@ -28,6 +28,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/threadpool.h"
 #include "tensorflow/core/lib/gtl/cleanup.h"
 #include "tensorflow/core/lib/random/random.h"
+#include "tensorflow/core/platform/cpu_info.h"
 #include "tensorflow/core/util/ptr_util.h"
 
 namespace tensorflow {
@@ -202,7 +203,7 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
             thread_pool_(new thread::ThreadPool(
                 Env::Default(), ThreadOptions(),
                 "data_parallel_interleave_worker_pool",
-                dataset()->cycle_length_ /* num_threads */,
+                port::NumSchedulableCPUs() /* num_threads */,
                 false /* low_latency_hint */)) {
         std::vector<string> components =
             str_util::Split(params.prefix, "::", str_util::SkipEmpty());
@@ -262,7 +263,7 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
         return model::MakeAsyncInterleaveManyNode(
             std::move(args),
             {model::MakeParameter("parallelism", num_parallel_calls_, /*min=*/1,
-                                  /*max=*/dataset()->cycle_length_)});
+                                  /*max=*/port::NumSchedulableCPUs())});
       }
 
       Status SaveInternal(IteratorStateWriter* writer) override {
@@ -574,8 +575,9 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
         RecordStart(ctx.get());
         auto cleanup = gtl::MakeCleanup([this, ctx] { RecordStop(ctx.get()); });
         auto busy = [this]() EXCLUSIVE_LOCKS_REQUIRED(*mu_) -> bool {
+          // TODO(jsimsa): Autotune the buffer size.
           return num_calls_ >= num_parallel_calls_->value ||
-                 future_elements_.size() >= dataset()->cycle_length_;
+                 future_elements_.size() >= 2 * dataset()->cycle_length_;
         };
         while (true) {
           mutex_lock l(*mu_);
diff --git a/tensorflow/python/data/experimental/benchmarks/BUILD b/tensorflow/python/data/experimental/benchmarks/BUILD
index 651dfd6857..2443f7edad 100644
--- a/tensorflow/python/data/experimental/benchmarks/BUILD
+++ b/tensorflow/python/data/experimental/benchmarks/BUILD
@@ -124,6 +124,22 @@ py_test(
     ],
 )
 
+py_test(
+    name = "parallel_interleave_benchmark",
+    srcs = ["parallel_interleave_benchmark.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:session",
+        "//tensorflow/python/data/experimental/ops:interleave_ops",
+        "//tensorflow/python/data/experimental/ops:optimization",
+        "//tensorflow/python/data/experimental/ops:sleep",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//third_party/py/numpy",
+    ],
+)
+
 py_test(
     name = "rejection_resample_benchmark",
     srcs = ["rejection_resample_benchmark.py"],
diff --git a/tensorflow/python/data/experimental/benchmarks/parallel_interleave_benchmark.py b/tensorflow/python/data/experimental/benchmarks/parallel_interleave_benchmark.py
new file mode 100644
index 0000000000..f5108e5ff4
--- /dev/null
+++ b/tensorflow/python/data/experimental/benchmarks/parallel_interleave_benchmark.py
@@ -0,0 +1,103 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Benchmarks for `tf.data.experimental.parallel_interleave()`."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import time
+
+import numpy as np
+
+from tensorflow.python.client import session
+from tensorflow.python.data.experimental.ops import interleave_ops
+from tensorflow.python.data.experimental.ops import optimization
+from tensorflow.python.data.experimental.ops import sleep
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.framework import ops
+from tensorflow.python.platform import test
+
+
+def _make_fake_dataset_fn():
+  """Returns a dataset that emulates a remote storage data source.
+
+  Returns a dataset factory which creates a dataset with 100 elements that
+  emulates the performance characteristic of a file-based dataset stored in a
+  remote storage. In particular, the first element will take an order of
+  magnitude longer to produce than the remaining elements (1s vs. 1ms).
+  """
+
+  def fake_dataset_fn(unused):
+    del unused
+
+    def make_dataset(time_us, num_elements):
+      return dataset_ops.Dataset.range(num_elements).apply(sleep.sleep(time_us))
+
+    return make_dataset(1000 * 1000, 0).concatenate(make_dataset(1000,
+                                                                 100)).take(100)
+
+  return fake_dataset_fn
+
+
+class ParallelInterleaveBenchmark(test.Benchmark):
+  """Benchmarks for `tf.data.experimental.parallel_interleave()`."""
+
+  def _benchmark(self, dataset_fn, iters, num_elements):
+    with ops.Graph().as_default():
+      iterator = dataset_ops.make_one_shot_iterator(dataset_fn())
+      next_element = iterator.get_next()
+      with session.Session() as sess:
+        deltas = []
+        for _ in range(iters):
+          start = time.time()
+          for _ in range(num_elements):
+            sess.run(next_element.op)
+          end = time.time()
+          deltas.append(end - start)
+
+    mean_wall_time = np.mean(deltas) / num_elements
+    self.report_benchmark(iters=iters, wall_time=mean_wall_time)
+
+  def benchmark_sequential_interleave(self):
+
+    def dataset_fn():
+      return dataset_ops.Dataset.range(1).repeat().interleave(
+          _make_fake_dataset_fn(), cycle_length=10)
+
+    self._benchmark(dataset_fn=dataset_fn, iters=10, num_elements=100)
+
+  def benchmark_parallel_interleave_v1(self):
+    """Benchmark for parallel interleave that does not support autotuning."""
+
+    def dataset_fn():
+      return dataset_ops.Dataset.range(1).repeat().apply(
+          interleave_ops.parallel_interleave(
+              _make_fake_dataset_fn(), cycle_length=10))
+
+    self._benchmark(dataset_fn=dataset_fn, iters=100, num_elements=1000)
+
+  def benchmark_parallel_interleave_v2(self):
+    """Benchmark for parallel interleave that supports autotuning."""
+
+    def dataset_fn():
+      return dataset_ops.Dataset.range(1).repeat().interleave(
+          _make_fake_dataset_fn(),
+          cycle_length=10, num_parallel_calls=optimization.AUTOTUNE)
+
+    self._benchmark(dataset_fn=dataset_fn, iters=100, num_elements=1000)
+
+
+if __name__ == "__main__":
+  test.main()
-- 
GitLab


From 104c9dac9cf69f2b3f409d9b874dc82795c5569d Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Fri, 11 Jan 2019 10:40:40 -0800
Subject: [PATCH 0596/1095] replace keras.optimizers with keras v2 optimizer in
 1.0.

PiperOrigin-RevId: 228905734
---
 tensorflow/python/keras/BUILD                 |   2 +-
 .../feature_columns_integration_test.py       |  13 +-
 .../python/keras/engine/sequential_test.py    |  41 ++--
 .../python/keras/engine/topology_test.py      |   8 +-
 .../keras/engine/training_dataset_test.py     |  26 ++-
 .../python/keras/engine/training_gpu_test.py  |   3 +-
 .../keras/layers/cudnn_recurrent_test.py      |  13 +-
 tensorflow/python/keras/layers/gru_test.py    |  16 +-
 tensorflow/python/keras/layers/lstm_test.py   |  26 +--
 .../python/keras/layers/pooling_test.py       |   3 +-
 .../python/keras/layers/recurrent_test.py     | 204 ++++++++++--------
 .../python/keras/layers/simplernn_test.py     |   6 +-
 .../python/keras/layers/wrappers_test.py      |   7 +-
 tensorflow/python/keras/metrics_test.py       |   3 +-
 .../python/keras/model_subclassing_test.py    |  81 ++++---
 .../python/keras/optimizer_v2/adadelta.py     |   2 +-
 .../python/keras/optimizer_v2/adagrad.py      |   2 +-
 tensorflow/python/keras/optimizer_v2/adam.py  |   2 +-
 .../python/keras/optimizer_v2/adamax.py       |   2 +-
 tensorflow/python/keras/optimizer_v2/ftrl.py  |   2 +-
 .../keras/optimizer_v2/gradient_descent.py    |   2 +-
 tensorflow/python/keras/optimizer_v2/nadam.py |   2 +-
 .../python/keras/optimizer_v2/optimizer_v2.py |   2 +-
 .../python/keras/optimizer_v2/rmsprop.py      |   2 +-
 tensorflow/python/keras/optimizers.py         |  39 +---
 tensorflow/python/keras/optimizers_test.py    |   3 +-
 tensorflow/python/keras/testing_utils.py      |  10 +-
 ...ensorflow.keras.optimizers.-adadelta.pbtxt |  45 +++-
 ...tensorflow.keras.optimizers.-adagrad.pbtxt |  45 +++-
 .../tensorflow.keras.optimizers.-adam.pbtxt   |  45 +++-
 .../tensorflow.keras.optimizers.-adamax.pbtxt |  45 +++-
 .../tensorflow.keras.optimizers.-nadam.pbtxt  |  45 +++-
 ...nsorflow.keras.optimizers.-optimizer.pbtxt |  43 +++-
 ...nsorflow.keras.optimizers.-r-m-sprop.pbtxt |  45 +++-
 .../tensorflow.keras.optimizers.-s-g-d.pbtxt  |  45 +++-
 35 files changed, 596 insertions(+), 284 deletions(-)

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index ef8cb23a09..0ff2aaffcc 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -398,7 +398,7 @@ tf_py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:training",
     ],
-    shard_count = 4,
+    shard_count = 8,
     tags = ["notsan"],
 )
 
diff --git a/tensorflow/python/keras/engine/feature_columns_integration_test.py b/tensorflow/python/keras/engine/feature_columns_integration_test.py
index b3f8cfe725..3bed40b084 100644
--- a/tensorflow/python/keras/engine/feature_columns_integration_test.py
+++ b/tensorflow/python/keras/engine/feature_columns_integration_test.py
@@ -27,7 +27,6 @@ from tensorflow.python.keras import keras_parameterized
 from tensorflow.python.keras import metrics as metrics_module
 from tensorflow.python.keras import testing_utils
 from tensorflow.python.platform import test
-from tensorflow.python.training import rmsprop
 
 
 class TestDNNModel(keras.models.Model):
@@ -57,7 +56,7 @@ class FeatureColumnsIntegrationTest(keras_parameterized.TestCase):
         keras.layers.Dense(20, activation='softmax')
     ])
     model.compile(
-        optimizer=rmsprop.RMSPropOptimizer(1e-3),
+        optimizer='rmsprop',
         loss='categorical_crossentropy',
         metrics=['accuracy'],
         run_eagerly=testing_utils.should_run_eagerly())
@@ -79,7 +78,7 @@ class FeatureColumnsIntegrationTest(keras_parameterized.TestCase):
         keras.layers.Dense(20, activation='softmax')
     ])
     model.compile(
-        optimizer=rmsprop.RMSPropOptimizer(1e-3),
+        optimizer='rmsprop',
         loss='categorical_crossentropy',
         metrics=['accuracy'],
         run_eagerly=testing_utils.should_run_eagerly())
@@ -103,7 +102,7 @@ class FeatureColumnsIntegrationTest(keras_parameterized.TestCase):
     dnn_model = TestDNNModel([col_a, col_b], 20)
 
     dnn_model.compile(
-        optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
+        optimizer='rmsprop',
         loss='categorical_crossentropy',
         metrics=['accuracy'],
         run_eagerly=testing_utils.should_run_eagerly())
@@ -124,7 +123,7 @@ class FeatureColumnsIntegrationTest(keras_parameterized.TestCase):
     dnn_model = TestDNNModel([col_a, col_b], 20)
 
     dnn_model.compile(
-        optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
+        optimizer='rmsprop',
         loss='categorical_crossentropy',
         metrics=['accuracy'],
         run_eagerly=testing_utils.should_run_eagerly())
@@ -155,7 +154,7 @@ class FeatureColumnsIntegrationTest(keras_parameterized.TestCase):
 
     model = keras.models.Model([feature_layer], [output])
 
-    optimizer = rmsprop.RMSPropOptimizer(learning_rate=0.001)
+    optimizer = 'rmsprop'
     loss = 'mse'
     loss_weights = [1., 0.5]
     model.compile(
@@ -184,7 +183,7 @@ class FeatureColumnsIntegrationTest(keras_parameterized.TestCase):
 
     model = keras.models.Model([fc1, fc2], [output])
 
-    optimizer = rmsprop.RMSPropOptimizer(learning_rate=0.001)
+    optimizer = 'rmsprop'
     loss = 'mse'
     loss_weights = [1., 0.5]
     model.compile(
diff --git a/tensorflow/python/keras/engine/sequential_test.py b/tensorflow/python/keras/engine/sequential_test.py
index 30a41e39b7..8e5d6fe93a 100644
--- a/tensorflow/python/keras/engine/sequential_test.py
+++ b/tensorflow/python/keras/engine/sequential_test.py
@@ -30,7 +30,6 @@ from tensorflow.python.keras import keras_parameterized
 from tensorflow.python.keras import testing_utils
 from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
-from tensorflow.python.training import rmsprop
 
 
 class TestSequential(keras_parameterized.TestCase):
@@ -57,16 +56,20 @@ class TestSequential(keras_parameterized.TestCase):
 
     model = testing_utils.get_small_sequential_mlp(
         num_hidden, num_classes, input_dim)
-    model.compile(loss='mse', optimizer=rmsprop.RMSPropOptimizer(1e-3),
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        loss='mse',
+        optimizer='rmsprop',
+        run_eagerly=testing_utils.should_run_eagerly())
     x = np.random.random((batch_size, input_dim))
     y = np.random.random((batch_size, num_classes))
     model.fit(x, y, epochs=1)
     model.pop()
     self.assertEqual(len(model.layers), 1)
     self.assertEqual(model.output_shape, (None, num_hidden))
-    model.compile(loss='mse', optimizer=rmsprop.RMSPropOptimizer(1e-3),
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        loss='mse',
+        optimizer='rmsprop',
+        run_eagerly=testing_utils.should_run_eagerly())
     y = np.random.random((batch_size, num_hidden))
     model.fit(x, y, epochs=1)
 
@@ -92,7 +95,7 @@ class TestSequential(keras_parameterized.TestCase):
     model = testing_utils.get_small_sequential_mlp(num_hidden, num_classes)
     model.compile(
         loss='mse',
-        optimizer=rmsprop.RMSPropOptimizer(1e-3),
+        optimizer='rmsprop',
         metrics=[keras.metrics.CategoricalAccuracy()],
         run_eagerly=testing_utils.should_run_eagerly())
     self.assertEqual(len(model.layers), 2)
@@ -117,7 +120,7 @@ class TestSequential(keras_parameterized.TestCase):
     model = testing_utils.get_small_sequential_mlp(num_hidden, num_classes)
     model.compile(
         loss='mse',
-        optimizer=rmsprop.RMSPropOptimizer(1e-3),
+        optimizer='rmsprop',
         metrics=[keras.metrics.CategoricalAccuracy()],
         run_eagerly=testing_utils.should_run_eagerly())
     self.assertEqual(len(model.layers), 2)
@@ -148,7 +151,7 @@ class TestSequential(keras_parameterized.TestCase):
         else:
           model = testing_utils.get_small_sequential_mlp(10, 4, input_dim=3)
         model.compile(
-            optimizer=rmsprop.RMSPropOptimizer(1e-3),
+            optimizer='rmsprop',
             loss='categorical_crossentropy',
             metrics=['accuracy'])
         return model
@@ -264,7 +267,7 @@ class TestSequential(keras_parameterized.TestCase):
     model = testing_utils.get_small_sequential_mlp(num_hidden, num_classes)
     model.compile(
         loss='mse',
-        optimizer=rmsprop.RMSPropOptimizer(1e-3),
+        optimizer='rmsprop',
         metrics=[keras.metrics.CategoricalAccuracy()],
         run_eagerly=testing_utils.should_run_eagerly())
     self.assertFalse(model.built)
@@ -310,8 +313,10 @@ class TestSequential(keras_parameterized.TestCase):
     inner_model = testing_utils.get_small_sequential_mlp(4, 5)
     model.add(inner_model)
 
-    model.compile(loss='mse', optimizer=rmsprop.RMSPropOptimizer(1e-3),
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        loss='mse',
+        optimizer='rmsprop',
+        run_eagerly=testing_utils.should_run_eagerly())
     x = np.random.random((2, 6))
     y = np.random.random((2, 5))
     model.fit(x, y, epochs=1)
@@ -352,8 +357,10 @@ class TestSequentialEagerIntegration(keras_parameterized.TestCase):
     model.add(keras.layers.Dense(4, activation='relu'))
     model.add(keras.layers.Dense(5, activation='softmax'))
 
-    model.compile(loss='mse', optimizer=rmsprop.RMSPropOptimizer(1e-3),
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        loss='mse',
+        optimizer='rmsprop',
+        run_eagerly=testing_utils.should_run_eagerly())
 
     x = np.random.random((2, 6))
     y = np.random.random((2, 5))
@@ -363,8 +370,10 @@ class TestSequentialEagerIntegration(keras_parameterized.TestCase):
   def test_build_before_fit(self):
     # Fix for b/112433577
     model = testing_utils.get_small_sequential_mlp(4, 5)
-    model.compile(loss='mse', optimizer=rmsprop.RMSPropOptimizer(1e-3),
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        loss='mse',
+        optimizer='rmsprop',
+        run_eagerly=testing_utils.should_run_eagerly())
 
     model.build((None, 6))
 
@@ -378,7 +387,7 @@ class TestSequentialEagerIntegration(keras_parameterized.TestCase):
     model = testing_utils.get_small_sequential_mlp(
         num_hidden=10, num_classes=num_classes)
     model.compile(
-        rmsprop.RMSPropOptimizer(learning_rate=0.001),
+        'rmsprop',
         metrics=['acc'],
         weighted_metrics=['mae'],
         loss='categorical_crossentropy',
diff --git a/tensorflow/python/keras/engine/topology_test.py b/tensorflow/python/keras/engine/topology_test.py
index 6c2835aece..08d21193a0 100644
--- a/tensorflow/python/keras/engine/topology_test.py
+++ b/tensorflow/python/keras/engine/topology_test.py
@@ -30,12 +30,10 @@ from tensorflow.python.keras import keras_parameterized
 from tensorflow.python.keras import testing_utils
 from tensorflow.python.keras.engine import input_layer as input_layer_lib
 from tensorflow.python.keras.engine import network as network_lib
-from tensorflow.python.keras.optimizer_v2 import gradient_descent
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import state_ops
 from tensorflow.python.platform import test
-from tensorflow.python.training import rmsprop
 
 try:
   import yaml  # pylint:disable=g-import-not-at-top
@@ -860,7 +858,7 @@ class TopologyConstructionTest(keras_parameterized.TestCase):
     x = np.ones((100, 2))
     y = np.ones((100, 2))
     model.compile(
-        optimizer=gradient_descent.SGD(),
+        optimizer='sgd',
         loss='mse',
         run_eagerly=testing_utils.should_run_eagerly())
     loss = model.train_on_batch(x, y)
@@ -909,7 +907,7 @@ class TopologyConstructionTest(keras_parameterized.TestCase):
     model.add(keras.layers.Dense(3))
     model.compile(
         loss='mse',
-        optimizer=gradient_descent.SGD(),
+        optimizer='sgd',
         metrics=['acc'],
         run_eagerly=testing_utils.should_run_eagerly())
 
@@ -1161,7 +1159,7 @@ class DefaultShapeInferenceBehaviorTest(keras_parameterized.TestCase):
     x = keras.layers.wrappers.TimeDistributed(s)(x)
     model = keras.Model(inputs=inputs, outputs=x)
     model.compile(
-        optimizer=rmsprop.RMSPropOptimizer(1e-3),
+        optimizer='rmsprop',
         loss='mse',
         run_eagerly=testing_utils.should_run_eagerly())
 
diff --git a/tensorflow/python/keras/engine/training_dataset_test.py b/tensorflow/python/keras/engine/training_dataset_test.py
index 0ce01ec606..1c67100f20 100644
--- a/tensorflow/python/keras/engine/training_dataset_test.py
+++ b/tensorflow/python/keras/engine/training_dataset_test.py
@@ -33,7 +33,6 @@ from tensorflow.python.keras import testing_utils
 from tensorflow.python.ops.losses import losses_impl
 from tensorflow.python.platform import test
 from tensorflow.python.platform import tf_logging as logging
-from tensorflow.python.training.rmsprop import RMSPropOptimizer
 
 
 class TestTrainingWithDatasetIterators(keras_parameterized.TestCase):
@@ -42,7 +41,7 @@ class TestTrainingWithDatasetIterators(keras_parameterized.TestCase):
   @keras_parameterized.run_all_keras_modes
   def test_training_and_eval_methods_on_iterators_single_io(self):
     model = testing_utils.get_small_mlp(1, 4, input_dim=3)
-    optimizer = RMSPropOptimizer(learning_rate=0.001)
+    optimizer = 'rmsprop'
     loss = 'mse'
     metrics = ['mae', metrics_module.CategoricalAccuracy()]
     model.compile(optimizer, loss, metrics=metrics,
@@ -103,7 +102,7 @@ class TestTrainingWithDatasetIterators(keras_parameterized.TestCase):
   @keras_parameterized.run_all_keras_modes
   def test_iterators_running_out_of_data(self):
     model = testing_utils.get_small_mlp(1, 4, input_dim=3)
-    optimizer = RMSPropOptimizer(learning_rate=0.001)
+    optimizer = 'rmsprop'
     loss = 'mse'
     metrics = ['mae']
     model.compile(optimizer, loss, metrics=metrics,
@@ -134,7 +133,7 @@ class TestTrainingWithDataset(keras_parameterized.TestCase):
       self.skipTest('b/120673224')
 
     model = testing_utils.get_small_mlp(1, 4, input_dim=3)
-    optimizer = RMSPropOptimizer(learning_rate=0.001)
+    optimizer = 'rmsprop'
     loss = 'mse'
     metrics = ['mae']
     model.compile(optimizer, loss, metrics=metrics,
@@ -156,7 +155,7 @@ class TestTrainingWithDataset(keras_parameterized.TestCase):
   @keras_parameterized.run_all_keras_modes
   def test_training_and_eval_methods_on_dataset(self):
     model = testing_utils.get_small_mlp(1, 4, input_dim=3)
-    optimizer = RMSPropOptimizer(learning_rate=0.001)
+    optimizer = 'rmsprop'
     loss = 'mse'
     metrics = ['mae', metrics_module.CategoricalAccuracy()]
     model.compile(optimizer, loss, metrics=metrics,
@@ -231,7 +230,7 @@ class TestTrainingWithDataset(keras_parameterized.TestCase):
   @keras_parameterized.run_all_keras_modes
   def test_dataset_with_sample_weights(self):
     model = testing_utils.get_small_mlp(1, 4, input_dim=3)
-    optimizer = RMSPropOptimizer(learning_rate=0.001)
+    optimizer = 'rmsprop'
     loss = 'mse'
     metrics = ['mae', metrics_module.CategoricalAccuracy()]
     model.compile(optimizer, loss, metrics=metrics,
@@ -253,7 +252,7 @@ class TestTrainingWithDataset(keras_parameterized.TestCase):
   @keras_parameterized.run_all_keras_modes
   def test_dataset_with_sparse_labels(self):
     model = testing_utils.get_small_mlp(1, 4, input_dim=3)
-    optimizer = RMSPropOptimizer(learning_rate=0.001)
+    optimizer = 'rmsprop'
     for loss in ['sparse_categorical_crossentropy',
                  losses_impl.sparse_softmax_cross_entropy]:
       model.compile(optimizer, loss,
@@ -279,9 +278,8 @@ class TestTrainingWithDataset(keras_parameterized.TestCase):
         return keras.backend.sum(inputs) + self.w * 0
 
     model = keras.Sequential([SumLayer(input_shape=(2,))])
-    model.compile(RMSPropOptimizer(learning_rate=0.001),
-                  loss='mae',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        'rmsprop', loss='mae', run_eagerly=testing_utils.should_run_eagerly())
 
     inputs = np.zeros((40, 2), dtype=np.float32)
     inputs[10:20, :] = 2
@@ -299,7 +297,7 @@ class TestTrainingWithDataset(keras_parameterized.TestCase):
   def test_dataset_input_shape_validation(self):
     with self.cached_session():
       model = testing_utils.get_small_functional_mlp(1, 4, input_dim=3)
-      model.compile(optimizer=RMSPropOptimizer(learning_rate=0.001), loss='mse')
+      model.compile(optimizer='rmsprop', loss='mse')
 
       # User forgets to batch the dataset
       inputs = np.zeros((10, 3))
@@ -328,7 +326,7 @@ class TestTrainingWithDataset(keras_parameterized.TestCase):
   @keras_parameterized.run_all_keras_modes
   def test_finite_dataset_known_cardinality_no_steps_arg(self):
     model = testing_utils.get_small_mlp(1, 4, input_dim=3)
-    optimizer = RMSPropOptimizer(learning_rate=0.001)
+    optimizer = 'rmsprop'
     model.compile(optimizer, 'mse',
                   run_eagerly=testing_utils.should_run_eagerly())
 
@@ -347,7 +345,7 @@ class TestTrainingWithDataset(keras_parameterized.TestCase):
   @keras_parameterized.run_all_keras_modes
   def test_finite_dataset_unknown_cardinality_no_steps_arg(self):
     model = testing_utils.get_small_mlp(1, 4, input_dim=3)
-    optimizer = RMSPropOptimizer(learning_rate=0.001)
+    optimizer = 'rmsprop'
     model.compile(optimizer, 'mse',
                   run_eagerly=testing_utils.should_run_eagerly())
 
@@ -381,7 +379,7 @@ class TestMetricsWithDatasetIterators(keras_parameterized.TestCase):
     model.compile(
         loss='binary_crossentropy',
         metrics=['accuracy', metrics_module.BinaryAccuracy()],
-        optimizer=RMSPropOptimizer(learning_rate=0.001),
+        optimizer='rmsprop',
         run_eagerly=testing_utils.should_run_eagerly())
 
     np.random.seed(123)
diff --git a/tensorflow/python/keras/engine/training_gpu_test.py b/tensorflow/python/keras/engine/training_gpu_test.py
index 45dcfe4399..ddc947339d 100644
--- a/tensorflow/python/keras/engine/training_gpu_test.py
+++ b/tensorflow/python/keras/engine/training_gpu_test.py
@@ -25,7 +25,6 @@ from tensorflow.python.framework import test_util
 from tensorflow.python.keras import backend as K
 from tensorflow.python.keras.layers.convolutional import Conv2D
 from tensorflow.python.platform import test
-from tensorflow.python.training import rmsprop
 
 
 class TrainingGPUTest(test.TestCase):
@@ -65,7 +64,7 @@ class TrainingGPUTest(test.TestCase):
                            bias_initializer='ones')(input_tensor)
       simple_model = keras.models.Model(inputs=input_tensor,
                                         outputs=predictions)
-      simple_model.compile(optimizer=rmsprop.RMSPropOptimizer(1e-3), loss=loss)
+      simple_model.compile(optimizer='rmsprop', loss=loss)
       return simple_model
 
     if test.is_gpu_available(cuda_only=True):
diff --git a/tensorflow/python/keras/layers/cudnn_recurrent_test.py b/tensorflow/python/keras/layers/cudnn_recurrent_test.py
index 1e305baefd..c7d8d82ee2 100644
--- a/tensorflow/python/keras/layers/cudnn_recurrent_test.py
+++ b/tensorflow/python/keras/layers/cudnn_recurrent_test.py
@@ -31,7 +31,6 @@ from tensorflow.python.keras.optimizer_v2.rmsprop import RMSprop
 from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
 from tensorflow.python.training import gradient_descent
-from tensorflow.python.training.rmsprop import RMSPropOptimizer
 
 
 @keras_parameterized.run_all_keras_modes
@@ -408,8 +407,7 @@ class CuDNNV1OnlyTest(keras_parameterized.TestCase):
     model.add(
         keras.layers.Bidirectional(
             rnn(output_dim), merge_mode=mode, input_shape=(None, dim)))
-    model.compile(
-        loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001))
+    model.compile(loss='mse', optimizer='rmsprop')
     model.fit(x, y, epochs=1, batch_size=1)
 
     # test config
@@ -425,8 +423,7 @@ class CuDNNV1OnlyTest(keras_parameterized.TestCase):
             merge_mode=mode,
             input_shape=(None, dim)))
     model.add(keras.layers.Bidirectional(rnn(output_dim), merge_mode=mode))
-    model.compile(
-        loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001))
+    model.compile(loss='mse', optimizer=R'rmsprop')
     model.fit(x, y, epochs=1, batch_size=1)
 
     # test with functional API
@@ -435,8 +432,7 @@ class CuDNNV1OnlyTest(keras_parameterized.TestCase):
         rnn(output_dim), merge_mode=mode)(
             inputs)
     model = keras.Model(inputs, outputs)
-    model.compile(
-        loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001))
+    model.compile(loss='mse', optimizer=R'rmsprop')
     model.fit(x, y, epochs=1, batch_size=1)
 
     # Bidirectional and stateful
@@ -445,8 +441,7 @@ class CuDNNV1OnlyTest(keras_parameterized.TestCase):
         rnn(output_dim, stateful=True), merge_mode=mode)(
             inputs)
     model = keras.Model(inputs, outputs)
-    model.compile(
-        loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001))
+    model.compile(loss='mse', optimizer='rmsprop')
     model.fit(x, y, epochs=1, batch_size=1)
 
   @test_util.run_gpu_only
diff --git a/tensorflow/python/keras/layers/gru_test.py b/tensorflow/python/keras/layers/gru_test.py
index e2b65de661..0e9dde8eae 100644
--- a/tensorflow/python/keras/layers/gru_test.py
+++ b/tensorflow/python/keras/layers/gru_test.py
@@ -26,7 +26,6 @@ from tensorflow.python.framework import test_util as tf_test_util
 from tensorflow.python.keras import keras_parameterized
 from tensorflow.python.keras import testing_utils
 from tensorflow.python.platform import test
-from tensorflow.python.training.rmsprop import RMSPropOptimizer
 
 
 @keras_parameterized.run_all_keras_modes
@@ -51,8 +50,8 @@ class GRULayerTest(keras_parameterized.TestCase):
     layer = keras.layers.GRU(units, input_shape=(None, embedding_dim))
     model = keras.models.Sequential()
     model.add(layer)
-    model.compile(RMSPropOptimizer(0.01), 'mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        'rmsprop', 'mse', run_eagerly=testing_utils.should_run_eagerly())
     x = np.random.random((num_samples, timesteps, embedding_dim))
     y = np.random.random((num_samples, units))
     model.train_on_batch(x, y)
@@ -99,8 +98,8 @@ class GRULayerTest(keras_parameterized.TestCase):
                                  reset_after=True)
     output = gru_layer(inputs)
     gru_model = keras.models.Model(inputs, output)
-    gru_model.compile(RMSPropOptimizer(0.01), 'mse',
-                      run_eagerly=testing_utils.should_run_eagerly())
+    gru_model.compile(
+        'rmsprop', 'mse', run_eagerly=testing_utils.should_run_eagerly())
     gru_model.fit(x_train, y_train)
     gru_model.predict(x_train)
 
@@ -112,9 +111,10 @@ class GRULayerTest(keras_parameterized.TestCase):
     model = keras.models.Sequential()
     model.add(keras.layers.Masking(input_shape=(3, 4)))
     model.add(layer_class(units=5, return_sequences=True, unroll=False))
-    model.compile(loss='categorical_crossentropy',
-                  optimizer=RMSPropOptimizer(0.01),
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        loss='categorical_crossentropy',
+        optimizer='rmsprop',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.fit(inputs, targets, epochs=1, batch_size=2, verbose=1)
 
   def test_statefulness_GRU(self):
diff --git a/tensorflow/python/keras/layers/lstm_test.py b/tensorflow/python/keras/layers/lstm_test.py
index 38c0177e39..38e165653e 100644
--- a/tensorflow/python/keras/layers/lstm_test.py
+++ b/tensorflow/python/keras/layers/lstm_test.py
@@ -28,7 +28,6 @@ from tensorflow.python.keras import testing_utils
 from tensorflow.python.platform import test
 from tensorflow.python.training import adam
 from tensorflow.python.training import gradient_descent
-from tensorflow.python.training.rmsprop import RMSPropOptimizer
 
 
 @keras_parameterized.run_all_keras_modes
@@ -68,8 +67,8 @@ class LSTMLayerTest(keras_parameterized.TestCase):
     layer = keras.layers.LSTM(units, input_shape=(None, embedding_dim))
     model = keras.models.Sequential()
     model.add(layer)
-    model.compile(RMSPropOptimizer(0.001), 'mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        'rmsprop', 'mse', run_eagerly=testing_utils.should_run_eagerly())
 
     x = np.random.random((num_samples, timesteps, embedding_dim))
     y = np.random.random((num_samples, units))
@@ -126,9 +125,10 @@ class LSTMLayerTest(keras_parameterized.TestCase):
     model = keras.models.Sequential()
     model.add(keras.layers.Masking(input_shape=(3, 4)))
     model.add(layer_class(units=5, return_sequences=True, unroll=False))
-    model.compile(loss='categorical_crossentropy',
-                  optimizer=RMSPropOptimizer(0.01),
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        loss='categorical_crossentropy',
+        optimizer='rmsprop',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.fit(inputs, targets, epochs=1, batch_size=2, verbose=1)
 
   def test_masking_with_stacking_LSTM(self):
@@ -139,9 +139,10 @@ class LSTMLayerTest(keras_parameterized.TestCase):
     model.add(keras.layers.Masking(input_shape=(3, 4)))
     lstm_cells = [keras.layers.LSTMCell(10), keras.layers.LSTMCell(5)]
     model.add(keras.layers.RNN(lstm_cells, return_sequences=True, unroll=False))
-    model.compile(loss='categorical_crossentropy',
-                  optimizer=RMSPropOptimizer(0.01),
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        loss='categorical_crossentropy',
+        optimizer='rmsprop',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.fit(inputs, targets, epochs=1, batch_size=2, verbose=1)
 
   def test_from_config_LSTM(self):
@@ -246,9 +247,10 @@ class LSTMLayerTest(keras_parameterized.TestCase):
     output = keras.layers.LSTM(units)(inputs, initial_state=initial_state)
 
     model = keras.models.Model([inputs] + initial_state, output)
-    model.compile(loss='categorical_crossentropy',
-                  optimizer=RMSPropOptimizer(0.01),
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        loss='categorical_crossentropy',
+        optimizer='rmsprop',
+        run_eagerly=testing_utils.should_run_eagerly())
 
     inputs = np.random.random((num_samples, timesteps, embedding_dim))
     initial_state = [np.random.random((num_samples, units))
diff --git a/tensorflow/python/keras/layers/pooling_test.py b/tensorflow/python/keras/layers/pooling_test.py
index 936e73ecf9..67df4d7a25 100644
--- a/tensorflow/python/keras/layers/pooling_test.py
+++ b/tensorflow/python/keras/layers/pooling_test.py
@@ -25,7 +25,6 @@ from tensorflow.python.eager import context
 from tensorflow.python.framework import test_util as tf_test_util
 from tensorflow.python.keras import testing_utils
 from tensorflow.python.platform import test
-from tensorflow.python.training import rmsprop
 
 
 class GlobalPoolingTest(test.TestCase):
@@ -48,7 +47,7 @@ class GlobalPoolingTest(test.TestCase):
     model = keras.Sequential()
     model.add(keras.layers.Masking(mask_value=0., input_shape=(3, 4)))
     model.add(keras.layers.GlobalAveragePooling1D())
-    model.compile(loss='mae', optimizer=rmsprop.RMSPropOptimizer(0.001))
+    model.compile(loss='mae', optimizer='rmsprop')
 
     model_input = np.random.random((2, 3, 4))
     model_input[0, 1:, :] = 0
diff --git a/tensorflow/python/keras/layers/recurrent_test.py b/tensorflow/python/keras/layers/recurrent_test.py
index e9bf788740..ddea2f4eae 100644
--- a/tensorflow/python/keras/layers/recurrent_test.py
+++ b/tensorflow/python/keras/layers/recurrent_test.py
@@ -40,7 +40,6 @@ from tensorflow.python.ops import special_math_ops
 from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variables as variables_lib
 from tensorflow.python.platform import test
-from tensorflow.python.training import rmsprop
 from tensorflow.python.training.checkpointable import util as checkpointable_util
 from tensorflow.python.util import nest
 
@@ -73,9 +72,10 @@ class RNNTest(keras_parameterized.TestCase):
     layer = keras.layers.RNN(cell)
     y = layer(x)
     model = keras.models.Model(x, y)
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(np.zeros((6, 5, 5)), np.zeros((6, 32)))
 
     # Test stacking.
@@ -85,9 +85,10 @@ class RNNTest(keras_parameterized.TestCase):
     layer = keras.layers.RNN(cells)
     y = layer(x)
     model = keras.models.Model(x, y)
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(np.zeros((6, 5, 5)), np.zeros((6, 32)))
 
   def test_minimal_rnn_cell_non_layer_multiple_states(self):
@@ -114,9 +115,10 @@ class RNNTest(keras_parameterized.TestCase):
     layer = keras.layers.RNN(cell)
     y = layer(x)
     model = keras.models.Model(x, y)
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(np.zeros((6, 5, 5)), np.zeros((6, 32)))
 
     # Test stacking.
@@ -128,9 +130,10 @@ class RNNTest(keras_parameterized.TestCase):
     self.assertEqual(layer.cell.output_size, 32)
     y = layer(x)
     model = keras.models.Model(x, y)
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(np.zeros((6, 5, 5)), np.zeros((6, 32)))
 
   def test_minimal_rnn_cell_layer(self):
@@ -169,9 +172,10 @@ class RNNTest(keras_parameterized.TestCase):
     layer = keras.layers.RNN(cell)
     y = layer(x)
     model = keras.models.Model(x, y)
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(np.zeros((6, 5, 5)), np.zeros((6, 32)))
 
     # Test basic case serialization.
@@ -194,9 +198,10 @@ class RNNTest(keras_parameterized.TestCase):
     layer = keras.layers.RNN(cells)
     y = layer(x)
     model = keras.models.Model(x, y)
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(np.zeros((6, 5, 5)), np.zeros((6, 32)))
 
     # Test stacked RNN serialization.
@@ -234,9 +239,10 @@ class RNNTest(keras_parameterized.TestCase):
     y = keras.layers.Lambda(lambda t: array_ops.transpose(t, [1, 0, 2]))(y)
 
     model = keras.models.Model(x, y)
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(
         np.zeros((batch, time_step, embedding_dim)),
         np.zeros((batch, time_step, units)))
@@ -253,9 +259,10 @@ class RNNTest(keras_parameterized.TestCase):
 
     y = keras.layers.Lambda(lambda t: array_ops.transpose(t, [1, 0, 2]))(y)
     model = keras.models.Model(x, y)
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(
         np.zeros((batch, time_step, embedding_dim)),
         np.zeros((batch, time_step, cell_units[-1])))
@@ -269,9 +276,10 @@ class RNNTest(keras_parameterized.TestCase):
         units, time_major=True, return_sequences=True)(mask)
     y = keras.layers.Lambda(lambda t: array_ops.transpose(t, [1, 0, 2]))(rnn)
     model = keras.models.Model(x, y)
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(
         np.zeros((batch, time_step, embedding_dim)),
         np.zeros((batch, time_step, units)))
@@ -282,9 +290,10 @@ class RNNTest(keras_parameterized.TestCase):
     y = rnn_1(x)
 
     model = keras.models.Model(x, y)
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(
         np.zeros((batch, time_step, embedding_dim)),
         np.zeros((batch, time_step, units)))
@@ -357,9 +366,10 @@ class RNNTest(keras_parameterized.TestCase):
     y = layer(x, constants=c)
 
     model = keras.models.Model([x, c], y)
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(
         [np.zeros((6, 5, 5)), np.zeros((6, 3))],
         np.zeros((6, 32))
@@ -396,9 +406,10 @@ class RNNTest(keras_parameterized.TestCase):
     layer = keras.layers.recurrent.RNN(cells)
     y = layer(x, constants=c)
     model = keras.models.Model([x, c], y)
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(
         [np.zeros((6, 5, 5)), np.zeros((6, 3))],
         np.zeros((6, 32))
@@ -411,9 +422,10 @@ class RNNTest(keras_parameterized.TestCase):
     layer = keras.layers.recurrent.RNN(cells)
     y = layer(x, constants=c)
     model = keras.models.Model([x, c], y)
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(
         [np.zeros((6, 5, 5)), np.zeros((6, 3))],
         np.zeros((6, 32))
@@ -484,9 +496,10 @@ class RNNTest(keras_parameterized.TestCase):
     layer = keras.layers.RNN(cell)
     y = layer(x, initial_state=s, constants=c)
     model = keras.models.Model([x, s, c], y)
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(
         [np.zeros((6, 5, 5)), np.zeros((6, 32)), np.zeros((6, 3))],
         np.zeros((6, 32))
@@ -615,9 +628,10 @@ class RNNTest(keras_parameterized.TestCase):
       layer = keras.layers.RNN(cell)
       y = layer(x)
       model = keras.models.Model(x, y)
-      model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                    loss='mse',
-                    run_eagerly=testing_utils.should_run_eagerly())
+      model.compile(
+          optimizer='rmsprop',
+          loss='mse',
+          run_eagerly=testing_utils.should_run_eagerly())
 
       # Test basic case serialization.
       x_np = np.random.random((6, 5, 5))
@@ -638,9 +652,10 @@ class RNNTest(keras_parameterized.TestCase):
       layer = keras.layers.RNN(cells)
       y = layer(x)
       model = keras.models.Model(x, y)
-      model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                    loss='mse',
-                    run_eagerly=testing_utils.should_run_eagerly())
+      model.compile(
+          optimizer='rmsprop',
+          loss='mse',
+          run_eagerly=testing_utils.should_run_eagerly())
 
       # Test stacked RNN serialization.
       x_np = np.random.random((6, 5, 5))
@@ -706,9 +721,10 @@ class RNNTest(keras_parameterized.TestCase):
     y = np.random.random((2, 2))
     model = keras.models.Sequential()
     model.add(rnn(2))
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.fit(x, y, epochs=1, batch_size=1)
 
     # check whether the model variables are present in the
@@ -740,9 +756,10 @@ class RNNTest(keras_parameterized.TestCase):
                        [None, unit_a, unit_b])
 
     model = keras.models.Model(x, y)
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(
         np.zeros((batch, time_step, input_a, input_b)),
         np.zeros((batch, unit_a, unit_b)))
@@ -757,9 +774,10 @@ class RNNTest(keras_parameterized.TestCase):
     layer = keras.layers.RNN(cells)
     y = layer(x)
     model = keras.models.Model(x, y)
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(
         np.zeros((batch, time_step, input_a, input_b)),
         np.zeros((batch, unit_a * 4, unit_b * 4)))
@@ -781,9 +799,10 @@ class RNNTest(keras_parameterized.TestCase):
     y = layer(x, initial_state=s)
 
     model = keras.models.Model([x, s], y)
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch([
         np.zeros((batch, time_step, input_a, input_b)),
         np.zeros((batch, unit_a, unit_b))
@@ -819,9 +838,10 @@ class RNNTest(keras_parameterized.TestCase):
                        [None, state_size])
 
     model = keras.models.Model(x, y)
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(
         np.zeros((batch, time_step, input_size)),
         np.zeros((batch, input_size)))
@@ -875,9 +895,10 @@ class RNNTest(keras_parameterized.TestCase):
     self.assertEqual(outputs[1].shape.as_list(), [None, o2, o3])
 
     model = keras.models.Model((input_1, input_2), outputs)
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(
         [np.zeros((batch, t, i1)), np.zeros((batch, t, i2, i3))],
         [np.zeros((batch, o1)), np.zeros((batch, o2, o3))])
@@ -897,9 +918,10 @@ class RNNTest(keras_parameterized.TestCase):
     self.assertEqual(outputs[1].shape.as_list(), [None, o2, o3])
 
     model = keras.models.Model([input_1, input_2], outputs)
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(
         [np.zeros((batch, t, i1)),
          np.zeros((batch, t, i2, i3))],
@@ -926,9 +948,10 @@ class RNNTest(keras_parameterized.TestCase):
     self.assertEqual(s2.shape.as_list(), [None, o2, o3])
 
     model = keras.models.Model([input_1, input_2], [output1, output2])
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(
         [np.zeros((batch, t, i1)),
          np.zeros((batch, t, i2, i3))],
@@ -951,9 +974,10 @@ class RNNTest(keras_parameterized.TestCase):
     self.assertEqual(s2.shape.as_list(), [None, o2, o3])
 
     model = keras.models.Model([input_1, input_2], [output1, output2])
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(
         [np.zeros((batch, t, i1)),
          np.zeros((batch, t, i2, i3))],
@@ -985,9 +1009,10 @@ class RNNTest(keras_parameterized.TestCase):
 
     model = keras.models.Model([input_1, input_2, init_s1, init_s2],
                                [output1, output2])
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(
         [np.zeros((batch, t, i1)),
          np.zeros((batch, t, i2, i3)),
@@ -1017,9 +1042,10 @@ class RNNTest(keras_parameterized.TestCase):
 
     model = keras.models.Model([input_1, input_2, init_s1, init_s2],
                                [output1, output2])
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch(
         [np.zeros((batch, t, i1)),
          np.zeros((batch, t, i2, i3)),
@@ -1093,9 +1119,10 @@ class RNNTest(keras_parameterized.TestCase):
     y, s = keras.layers.RNN(
         Cell(), return_state=True)(x_masked, initial_state=s_0)
     model = keras.models.Model([x, s_0], [y, s])
-    model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                  loss='mse',
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        optimizer='rmsprop',
+        loss='mse',
+        run_eagerly=testing_utils.should_run_eagerly())
 
     # last time step masked
     x_np = np.array([[[1.], [2.], [0.]]])
@@ -1118,9 +1145,10 @@ class RNNTest(keras_parameterized.TestCase):
       masked_input = mask(x)
       y = layer(masked_input)
       model = keras.models.Model(x, y)
-      model.compile(optimizer=rmsprop.RMSPropOptimizer(learning_rate=0.001),
-                    loss='mse',
-                    run_eagerly=testing_utils.should_run_eagerly())
+      model.compile(
+          optimizer='rmsprop',
+          loss='mse',
+          run_eagerly=testing_utils.should_run_eagerly())
 
       np_x = np.ones((6, 5, 5))
       result_1 = model.predict(np_x)
diff --git a/tensorflow/python/keras/layers/simplernn_test.py b/tensorflow/python/keras/layers/simplernn_test.py
index 0e599dade9..0ee074d19c 100644
--- a/tensorflow/python/keras/layers/simplernn_test.py
+++ b/tensorflow/python/keras/layers/simplernn_test.py
@@ -26,7 +26,6 @@ from tensorflow.python.keras import keras_parameterized
 from tensorflow.python.keras import testing_utils
 from tensorflow.python.platform import test
 from tensorflow.python.training import gradient_descent
-from tensorflow.python.training.rmsprop import RMSPropOptimizer
 
 
 @keras_parameterized.run_all_keras_modes
@@ -51,7 +50,7 @@ class SimpleRNNLayerTest(keras_parameterized.TestCase):
     layer = keras.layers.SimpleRNN(units, input_shape=(None, embedding_dim))
     model = keras.models.Sequential()
     model.add(layer)
-    model.compile(RMSPropOptimizer(0.01), 'mse')
+    model.compile('rmsprop', 'mse')
     x = np.random.random((num_samples, timesteps, embedding_dim))
     y = np.random.random((num_samples, units))
     model.train_on_batch(x, y)
@@ -107,8 +106,7 @@ class SimpleRNNLayerTest(keras_parameterized.TestCase):
     model = keras.models.Sequential()
     model.add(keras.layers.Masking(input_shape=(3, 4)))
     model.add(layer_class(units=5, return_sequences=True, unroll=False))
-    model.compile(loss='categorical_crossentropy',
-                  optimizer=RMSPropOptimizer(0.01))
+    model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
     model.fit(inputs, targets, epochs=1, batch_size=2, verbose=1)
 
   def test_from_config_SimpleRNN(self):
diff --git a/tensorflow/python/keras/layers/wrappers_test.py b/tensorflow/python/keras/layers/wrappers_test.py
index 46d5487b2c..00e5276d6a 100644
--- a/tensorflow/python/keras/layers/wrappers_test.py
+++ b/tensorflow/python/keras/layers/wrappers_test.py
@@ -27,7 +27,6 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import test_util as tf_test_util
 from tensorflow.python.platform import test
 from tensorflow.python.training.checkpointable import util as checkpointable_util
-from tensorflow.python.training.rmsprop import RMSPropOptimizer
 
 
 class _RNNCellWithConstants(keras.layers.Layer):
@@ -77,7 +76,7 @@ class TimeDistributedTest(test.TestCase):
     model.add(
         keras.layers.TimeDistributed(
             keras.layers.Dense(2), input_shape=(3, 4)))
-    model.compile(optimizer=RMSPropOptimizer(0.01), loss='mse')
+    model.compile(optimizer='rmsprop', loss='mse')
     model.fit(
         np.random.random((10, 3, 4)),
         np.random.random((10, 3, 2)),
@@ -98,7 +97,7 @@ class TimeDistributedTest(test.TestCase):
     model.add(
         keras.layers.TimeDistributed(
             keras.layers.Dense(2), input_shape=(3, 4), batch_size=10))
-    model.compile(optimizer=RMSPropOptimizer(0.01), loss='mse')
+    model.compile(optimizer='rmsprop', loss='mse')
     model.fit(
         np.random.random((10, 3, 4)),
         np.random.random((10, 3, 2)),
@@ -277,7 +276,7 @@ class BidirectionalTest(test.TestCase):
         model.add(
             keras.layers.Bidirectional(
                 rnn(output_dim), merge_mode=mode, input_shape=(timesteps, dim)))
-        model.compile(optimizer=RMSPropOptimizer(0.01), loss='mse')
+        model.compile(optimizer='rmsprop', loss='mse')
         model.fit(x, y, epochs=1, batch_size=1)
 
         # check whether the model variables are present in the
diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py
index f902838c11..6cf61d33d0 100644
--- a/tensorflow/python/keras/metrics_test.py
+++ b/tensorflow/python/keras/metrics_test.py
@@ -35,7 +35,6 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
 from tensorflow.python.training.checkpointable import util as checkpointable_utils
-from tensorflow.python.training.rmsprop import RMSPropOptimizer
 
 
 @test_util.run_all_in_graph_and_eager_modes
@@ -977,7 +976,7 @@ def _get_model(compile_metrics):
   model.compile(
       loss='mae',
       metrics=compile_metrics,
-      optimizer=RMSPropOptimizer(learning_rate=0.001),
+      optimizer='rmsprop',
       run_eagerly=testing_utils.should_run_eagerly())
   return model
 
diff --git a/tensorflow/python/keras/model_subclassing_test.py b/tensorflow/python/keras/model_subclassing_test.py
index cf64e00d20..6d8ff9d847 100644
--- a/tensorflow/python/keras/model_subclassing_test.py
+++ b/tensorflow/python/keras/model_subclassing_test.py
@@ -36,7 +36,6 @@ from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.platform import test
 from tensorflow.python.training.checkpointable import data_structures
-from tensorflow.python.training.rmsprop import RMSPropOptimizer
 
 try:
   import h5py  # pylint:disable=g-import-not-at-top
@@ -572,7 +571,7 @@ class ModelSubclassCompiledTest(keras_parameterized.TestCase):
                             use_bn=True)
     model.compile(
         loss='mse',
-        optimizer=RMSPropOptimizer(learning_rate=0.001),
+        optimizer='rmsprop',
         metrics=['acc', keras.metrics.CategoricalAccuracy()],
         run_eagerly=testing_utils.should_run_eagerly())
 
@@ -590,10 +589,11 @@ class ModelSubclassCompiledTest(keras_parameterized.TestCase):
     model = MultiIOTestModel(num_classes=num_classes,
                              use_dp=True,
                              use_bn=True)
-    model.compile(loss='mse',
-                  optimizer=RMSPropOptimizer(learning_rate=0.001),
-                  metrics=['acc'],
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        loss='mse',
+        optimizer='rmsprop',
+        metrics=['acc'],
+        run_eagerly=testing_utils.should_run_eagerly())
 
     x1 = np.ones((num_samples, input_dim))
     x2 = np.ones((num_samples, input_dim))
@@ -611,7 +611,8 @@ class ModelSubclassCompiledTest(keras_parameterized.TestCase):
     with self.cached_session():
       model = SimpleTestModel(num_classes=num_classes, use_dp=True, use_bn=True)
       model.compile(
-          loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001),
+          loss='mse',
+          optimizer='rmsprop',
           run_eagerly=testing_utils.should_run_eagerly())
 
       x = np.ones((num_samples, input_dim), dtype=np.float32)
@@ -643,7 +644,8 @@ class ModelSubclassCompiledTest(keras_parameterized.TestCase):
     self.assertEqual(len(model.weights), 0)
 
     model.compile(
-        loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001),
+        loss='mse',
+        optimizer='rmsprop',
         run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch([x1, x2], [y1, y2])
 
@@ -675,7 +677,8 @@ class ModelSubclassCompiledTest(keras_parameterized.TestCase):
 
     model = BNNet()
     model.compile(
-        loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001),
+        loss='mse',
+        optimizer='rmsprop',
         run_eagerly=testing_utils.should_run_eagerly())
     y_ref = model.predict(x)
 
@@ -707,7 +710,8 @@ class ModelSubclassCompiledTest(keras_parameterized.TestCase):
     y = model.predict(x)
     self.assertEqual(np.sum(y), np.sum(x))
     model.compile(
-        loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001),
+        loss='mse',
+        optimizer='rmsprop',
         run_eagerly=testing_utils.should_run_eagerly())
     loss = model.train_on_batch(x, y)
     self.assertGreater(loss, 0.1)
@@ -727,7 +731,8 @@ class ModelSubclassCompiledTest(keras_parameterized.TestCase):
 
     model = MultiIOTestModel(num_classes=num_classes, use_bn=True)
     model.compile(
-        loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001),
+        loss='mse',
+        optimizer='rmsprop',
         run_eagerly=testing_utils.should_run_eagerly())
     model.fit([x1, x2], [y1, y2], epochs=2, batch_size=32, verbose=0)
     model.fit({'input_1': x1, 'input_2': x2},
@@ -738,7 +743,8 @@ class ModelSubclassCompiledTest(keras_parameterized.TestCase):
 
     model = MultiIOTestModel(num_classes=num_classes, use_bn=True)
     model.compile(
-        loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001),
+        loss='mse',
+        optimizer='rmsprop',
         run_eagerly=testing_utils.should_run_eagerly())
     model.train_on_batch([x1, x2], [y1, y2])
     model.train_on_batch({'input_1': x1, 'input_2': x2},
@@ -758,7 +764,8 @@ class ModelSubclassCompiledTest(keras_parameterized.TestCase):
 
     model = MultiIOTestModel(num_classes=num_classes, use_bn=True)
     model.compile(
-        loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001),
+        loss='mse',
+        optimizer='rmsprop',
         run_eagerly=testing_utils.should_run_eagerly())
     model.evaluate([x1, x2], [y1, y2])
     model.test_on_batch([x1, x2], [y1, y2])
@@ -782,7 +789,8 @@ class ModelSubclassCompiledTest(keras_parameterized.TestCase):
 
     model = MultiIOTestModel(num_classes=num_classes, use_bn=True)
     model.compile(
-        loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001),
+        loss='mse',
+        optimizer='rmsprop',
         run_eagerly=testing_utils.should_run_eagerly())
     model.fit([x1, x2], [y1, y2], epochs=2, batch_size=32, verbose=0)
     y_ref_1, y_ref_2 = model.predict([x1, x2])
@@ -818,10 +826,11 @@ class ModelSubclassCompiledTest(keras_parameterized.TestCase):
     input_dim = 50
 
     model = NestedTestModel1(num_classes=num_classes)
-    model.compile(loss='mse',
-                  optimizer=RMSPropOptimizer(learning_rate=0.001),
-                  metrics=['acc'],
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        loss='mse',
+        optimizer='rmsprop',
+        metrics=['acc'],
+        run_eagerly=testing_utils.should_run_eagerly())
 
     x = np.ones((num_samples, input_dim))
     y = np.zeros((num_samples, num_classes))
@@ -841,10 +850,11 @@ class ModelSubclassCompiledTest(keras_parameterized.TestCase):
     input_dim = 50
 
     model = NestedTestModel2(num_classes=num_classes)
-    model.compile(loss='mse',
-                  optimizer=RMSPropOptimizer(learning_rate=0.001),
-                  metrics=['acc'],
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        loss='mse',
+        optimizer='rmsprop',
+        metrics=['acc'],
+        run_eagerly=testing_utils.should_run_eagerly())
 
     x = np.ones((num_samples, input_dim))
     y = np.zeros((num_samples, num_classes))
@@ -864,10 +874,11 @@ class ModelSubclassCompiledTest(keras_parameterized.TestCase):
     input_dim = 50
 
     model = get_nested_model_3(input_dim=input_dim, num_classes=num_classes)
-    model.compile(loss='mse',
-                  optimizer=RMSPropOptimizer(learning_rate=0.001),
-                  metrics=['acc'],
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        loss='mse',
+        optimizer='rmsprop',
+        metrics=['acc'],
+        run_eagerly=testing_utils.should_run_eagerly())
 
     x = np.ones((num_samples, input_dim))
     y = np.zeros((num_samples, num_classes))
@@ -898,10 +909,11 @@ class ModelSubclassCompiledTest(keras_parameterized.TestCase):
         return self.bn(x)
 
     model = keras.Sequential([Inner()])
-    model.compile(loss='mse',
-                  optimizer=RMSPropOptimizer(learning_rate=0.001),
-                  metrics=['acc'],
-                  run_eagerly=testing_utils.should_run_eagerly())
+    model.compile(
+        loss='mse',
+        optimizer='rmsprop',
+        metrics=['acc'],
+        run_eagerly=testing_utils.should_run_eagerly())
 
     x = np.ones((num_samples, input_dim))
     y = np.zeros((num_samples, num_classes))
@@ -938,7 +950,8 @@ class ModelSubclassCompiledTest(keras_parameterized.TestCase):
     y = model.predict(x)
     self.assertEqual(np.sum(y), np.sum(x))
     model.compile(
-        loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001),
+        loss='mse',
+        optimizer='rmsprop',
         run_eagerly=testing_utils.should_run_eagerly())
     loss = model.train_on_batch(x, y)
     self.assertGreater(loss, 0.1)
@@ -956,7 +969,7 @@ class GraphSpecificModelSubclassingTests(test.TestCase):
       model = SimpleTestModel(num_classes=num_classes,
                               use_dp=True,
                               use_bn=True)
-      model.compile(loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001))
+      model.compile(loss='mse', optimizer='rmsprop')
 
       x = array_ops.ones((num_samples, input_dim))
       y = array_ops.zeros((num_samples, num_classes))
@@ -974,7 +987,7 @@ class GraphSpecificModelSubclassingTests(test.TestCase):
       model = MultiIOTestModel(num_classes=num_classes,
                                use_dp=True,
                                use_bn=True)
-      model.compile(loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001))
+      model.compile(loss='mse', optimizer='rmsprop')
 
       x1 = array_ops.ones((num_samples, input_dim))
       x2 = array_ops.ones((num_samples, input_dim))
@@ -1062,7 +1075,7 @@ class GraphSpecificModelSubclassingTests(test.TestCase):
       model = MultiIOTestModel(num_classes=num_classes,
                                use_dp=True,
                                use_bn=True)
-      model.compile(loss='mse', optimizer=RMSPropOptimizer(learning_rate=0.001))
+      model.compile(loss='mse', optimizer='rmsprop')
 
       x1 = np.ones((num_samples, input_dim))
       x2 = np.ones((num_samples, input_dim))
diff --git a/tensorflow/python/keras/optimizer_v2/adadelta.py b/tensorflow/python/keras/optimizer_v2/adadelta.py
index cb9b7d1015..b6cba263bc 100644
--- a/tensorflow/python/keras/optimizer_v2/adadelta.py
+++ b/tensorflow/python/keras/optimizer_v2/adadelta.py
@@ -25,7 +25,7 @@ from tensorflow.python.training import training_ops
 from tensorflow.python.util.tf_export import keras_export
 
 
-@keras_export('keras.optimizers.Adadelta', v1=[])
+@keras_export('keras.optimizers.Adadelta')
 class Adadelta(optimizer_v2.OptimizerV2):
   r"""Optimizer that implements the Adadelta algorithm.
 
diff --git a/tensorflow/python/keras/optimizer_v2/adagrad.py b/tensorflow/python/keras/optimizer_v2/adagrad.py
index 7e4a153678..01c722521f 100644
--- a/tensorflow/python/keras/optimizer_v2/adagrad.py
+++ b/tensorflow/python/keras/optimizer_v2/adagrad.py
@@ -30,7 +30,7 @@ from tensorflow.python.ops import state_ops
 from tensorflow.python.util.tf_export import keras_export
 
 
-@keras_export('keras.optimizers.Adagrad', v1=[])
+@keras_export('keras.optimizers.Adagrad')
 class Adagrad(optimizer_v2.OptimizerV2):
   r"""Optimizer that implements the Adagrad algorithm.
 
diff --git a/tensorflow/python/keras/optimizer_v2/adam.py b/tensorflow/python/keras/optimizer_v2/adam.py
index a05bc7e96c..263004c9df 100644
--- a/tensorflow/python/keras/optimizer_v2/adam.py
+++ b/tensorflow/python/keras/optimizer_v2/adam.py
@@ -26,7 +26,7 @@ from tensorflow.python.training import training_ops
 from tensorflow.python.util.tf_export import keras_export
 
 
-@keras_export('keras.optimizers.Adam', v1=[])
+@keras_export('keras.optimizers.Adam')
 class Adam(optimizer_v2.OptimizerV2):
   """Optimizer that implements the Adam algorithm.
 
diff --git a/tensorflow/python/keras/optimizer_v2/adamax.py b/tensorflow/python/keras/optimizer_v2/adamax.py
index 4a4660d328..9a1b63f6c6 100644
--- a/tensorflow/python/keras/optimizer_v2/adamax.py
+++ b/tensorflow/python/keras/optimizer_v2/adamax.py
@@ -27,7 +27,7 @@ from tensorflow.python.training import training_ops
 from tensorflow.python.util.tf_export import keras_export
 
 
-@keras_export('keras.optimizers.Adamax', v1=[])
+@keras_export('keras.optimizers.Adamax')
 class Adamax(optimizer_v2.OptimizerV2):
   """Optimizer that implements the Adamax algorithm.
 
diff --git a/tensorflow/python/keras/optimizer_v2/ftrl.py b/tensorflow/python/keras/optimizer_v2/ftrl.py
index 15515616b2..a86fd8d89d 100644
--- a/tensorflow/python/keras/optimizer_v2/ftrl.py
+++ b/tensorflow/python/keras/optimizer_v2/ftrl.py
@@ -24,7 +24,7 @@ from tensorflow.python.training import training_ops
 from tensorflow.python.util.tf_export import keras_export
 
 
-@keras_export('keras.optimizers.Ftrl', v1=[])
+@keras_export('keras.optimizers.Ftrl')
 class Ftrl(optimizer_v2.OptimizerV2):
   """Optimizer that implements the FTRL algorithm.
 
diff --git a/tensorflow/python/keras/optimizer_v2/gradient_descent.py b/tensorflow/python/keras/optimizer_v2/gradient_descent.py
index adb4ae2bf8..c444f969f6 100644
--- a/tensorflow/python/keras/optimizer_v2/gradient_descent.py
+++ b/tensorflow/python/keras/optimizer_v2/gradient_descent.py
@@ -24,7 +24,7 @@ from tensorflow.python.training import training_ops
 from tensorflow.python.util.tf_export import keras_export
 
 
-@keras_export("keras.optimizers.SGD", v1=[])
+@keras_export("keras.optimizers.SGD")
 class SGD(optimizer_v2.OptimizerV2):
   """Stochastic gradient descent and momentum optimizer.
 
diff --git a/tensorflow/python/keras/optimizer_v2/nadam.py b/tensorflow/python/keras/optimizer_v2/nadam.py
index 7141c6a079..4c94b1358a 100644
--- a/tensorflow/python/keras/optimizer_v2/nadam.py
+++ b/tensorflow/python/keras/optimizer_v2/nadam.py
@@ -26,7 +26,7 @@ from tensorflow.python.ops import state_ops
 from tensorflow.python.util.tf_export import keras_export
 
 
-@keras_export('keras.optimizers.Nadam', v1=[])
+@keras_export('keras.optimizers.Nadam')
 class Nadam(optimizer_v2.OptimizerV2):
   r"""Optimizer that implements the NAdam algorithm.
 
diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
index 894af66f5d..91dab9338d 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
@@ -69,7 +69,7 @@ def _deduplicate_indexed_slices(values, indices):
 
 
 @six.add_metaclass(abc.ABCMeta)
-@keras_export("keras.optimizers.Optimizer", v1=[])
+@keras_export("keras.optimizers.Optimizer")
 class OptimizerV2(checkpointable.CheckpointableBase):
   """Updated base class for optimizers.
 
diff --git a/tensorflow/python/keras/optimizer_v2/rmsprop.py b/tensorflow/python/keras/optimizer_v2/rmsprop.py
index 83b81cbe5d..d459d9811e 100644
--- a/tensorflow/python/keras/optimizer_v2/rmsprop.py
+++ b/tensorflow/python/keras/optimizer_v2/rmsprop.py
@@ -29,7 +29,7 @@ from tensorflow.python.training import training_ops
 from tensorflow.python.util.tf_export import keras_export
 
 
-@keras_export("keras.optimizers.RMSprop", v1=[])
+@keras_export("keras.optimizers.RMSprop")
 class RMSprop(optimizer_v2.OptimizerV2):
   r"""Optimizer that implements the RMSprop algorithm.
 
diff --git a/tensorflow/python/keras/optimizers.py b/tensorflow/python/keras/optimizers.py
index 82fb555b57..8c3bf110d4 100644
--- a/tensorflow/python/keras/optimizers.py
+++ b/tensorflow/python/keras/optimizers.py
@@ -22,7 +22,6 @@ from __future__ import print_function
 import six
 from six.moves import zip  # pylint: disable=redefined-builtin
 
-from tensorflow.python import tf2
 from tensorflow.python.distribute import distribution_strategy_context
 from tensorflow.python.framework import ops
 from tensorflow.python.keras import backend as K
@@ -45,7 +44,6 @@ from tensorflow.python.training.checkpointable import base as checkpointable
 from tensorflow.python.util.tf_export import keras_export
 
 
-@keras_export(v1=['keras.optimizers.Optimizer'])
 class Optimizer(object):
   """Abstract optimizer base class.
 
@@ -159,7 +157,6 @@ class Optimizer(object):
     return cls(**config)
 
 
-@keras_export(v1=['keras.optimizers.SGD'])
 class SGD(Optimizer):
   """Stochastic gradient descent optimizer.
 
@@ -224,7 +221,6 @@ class SGD(Optimizer):
     return dict(list(base_config.items()) + list(config.items()))
 
 
-@keras_export(v1=['keras.optimizers.RMSprop'])
 class RMSprop(Optimizer):
   """RMSProp optimizer.
 
@@ -291,7 +287,6 @@ class RMSprop(Optimizer):
     return dict(list(base_config.items()) + list(config.items()))
 
 
-@keras_export(v1=['keras.optimizers.Adagrad'])
 class Adagrad(Optimizer):
   """Adagrad optimizer.
 
@@ -358,7 +353,6 @@ class Adagrad(Optimizer):
     return dict(list(base_config.items()) + list(config.items()))
 
 
-@keras_export(v1=['keras.optimizers.Adadelta'])
 class Adadelta(Optimizer):
   """Adadelta optimizer.
 
@@ -442,7 +436,6 @@ class Adadelta(Optimizer):
     return dict(list(base_config.items()) + list(config.items()))
 
 
-@keras_export(v1=['keras.optimizers.Adam'])
 class Adam(Optimizer):
   """Adam optimizer.
 
@@ -539,7 +532,6 @@ class Adam(Optimizer):
     return dict(list(base_config.items()) + list(config.items()))
 
 
-@keras_export(v1=['keras.optimizers.Adamax'])
 class Adamax(Optimizer):
   """Adamax optimizer from Adam paper's Section 7.
 
@@ -623,7 +615,6 @@ class Adamax(Optimizer):
     return dict(list(base_config.items()) + list(config.items()))
 
 
-@keras_export(v1=['keras.optimizers.Nadam'])
 class Nadam(Optimizer):
   """Nesterov Adam optimizer.
 
@@ -808,27 +799,15 @@ def deserialize(config, custom_objects=None):
   Returns:
       A Keras Optimizer instance.
   """
-  if tf2.enabled():
-    all_classes = {
-        'adadelta': adadelta_v2.Adadelta,
-        'adagrad': adagrad_v2.Adagrad,
-        'adam': adam_v2.Adam,
-        'adamax': adamax_v2.Adamax,
-        'nadam': nadam_v2.Nadam,
-        'rmsprop': rmsprop_v2.RMSprop,
-        'sgd': gradient_descent_v2.SGD
-    }
-  else:
-    all_classes = {
-        'adadelta': Adadelta,
-        'adagrad': Adagrad,
-        'adam': Adam,
-        'adamax': Adamax,
-        'nadam': Nadam,
-        'rmsprop': RMSprop,
-        'sgd': SGD,
-        'tfoptimizer': TFOptimizer
-    }
+  all_classes = {
+      'adadelta': adadelta_v2.Adadelta,
+      'adagrad': adagrad_v2.Adagrad,
+      'adam': adam_v2.Adam,
+      'adamax': adamax_v2.Adamax,
+      'nadam': nadam_v2.Nadam,
+      'rmsprop': rmsprop_v2.RMSprop,
+      'sgd': gradient_descent_v2.SGD
+  }
 
   # Make deserialization case-insensitive for built-in optimizers.
   if config['class_name'].lower() in all_classes:
diff --git a/tensorflow/python/keras/optimizers_test.py b/tensorflow/python/keras/optimizers_test.py
index de24bf0536..606e711483 100644
--- a/tensorflow/python/keras/optimizers_test.py
+++ b/tensorflow/python/keras/optimizers_test.py
@@ -62,7 +62,7 @@ class KerasOptimizersTest(test.TestCase):
     np.testing.assert_equal(
         keras.backend.get_value(model.optimizer.iterations),
         126)  # 63 steps per epoch
-    assert history.history['acc'][-1] >= target
+    self.assertGreaterEqual(history.history['acc'][-1], target)
     config = keras.optimizers.serialize(optimizer)
     optim = keras.optimizers.deserialize(config)
     new_config = keras.optimizers.serialize(optim)
@@ -190,7 +190,6 @@ class KerasOptimizersTest(test.TestCase):
     self.assertIs(graph_weak(), None)
     self.assertIs(optimizer_weak(), None)
 
-  @test_util.run_in_graph_and_eager_modes
   def test_tf_optimizer_iterations(self):
     with self.cached_session():
       optimizer = keras.optimizers.TFOptimizer(AdamOptimizer(0.01))
diff --git a/tensorflow/python/keras/testing_utils.py b/tensorflow/python/keras/testing_utils.py
index 6448f87d25..eff0f39b6d 100644
--- a/tensorflow/python/keras/testing_utils.py
+++ b/tensorflow/python/keras/testing_utils.py
@@ -32,7 +32,6 @@ from tensorflow.python.keras.optimizer_v2 import adamax as adamax_v2
 from tensorflow.python.keras.optimizer_v2 import gradient_descent as gradient_descent_v2
 from tensorflow.python.keras.optimizer_v2 import nadam as nadam_v2
 from tensorflow.python.keras.optimizer_v2 import rmsprop as rmsprop_v2
-from tensorflow.python.training.rmsprop import RMSPropOptimizer
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util import tf_inspect
 
@@ -169,10 +168,13 @@ def layer_test(layer_cls, kwargs=None, input_shape=None, input_dtype=None,
   # See b/120160788 for more details. This should be mitigated after 2.0.
   model = keras.models.Model(x, layer(x))
   if _thread_local_data.run_eagerly is not None:
-    model.compile(RMSPropOptimizer(0.01), 'mse', weighted_metrics=['acc'],
-                  run_eagerly=should_run_eagerly())
+    model.compile(
+        'rmsprop',
+        'mse',
+        weighted_metrics=['acc'],
+        run_eagerly=should_run_eagerly())
   else:
-    model.compile(RMSPropOptimizer(0.01), 'mse', weighted_metrics=['acc'])
+    model.compile('rmsprop', 'mse', weighted_metrics=['acc'])
   model.train_on_batch(input_data, actual_output)
 
   # test as first layer in Sequential API
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adadelta.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adadelta.pbtxt
index b9ce154bdd..5426269793 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adadelta.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adadelta.pbtxt
@@ -1,15 +1,36 @@
 path: "tensorflow.keras.optimizers.Adadelta"
 tf_class {
-  is_instance: "<class \'tensorflow.python.keras.optimizers.Adadelta\'>"
-  is_instance: "<class \'tensorflow.python.keras.optimizers.Optimizer\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.adadelta.Adadelta\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "iterations"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'lr\', \'rho\', \'epsilon\', \'decay\'], varargs=None, keywords=kwargs, defaults=[\'1.0\', \'0.95\', \'None\', \'0.0\'], "
+    argspec: "args=[\'self\', \'learning_rate\', \'rho\', \'epsilon\', \'name\'], varargs=None, keywords=kwargs, defaults=[\'0.001\', \'0.95\', \'1e-07\', \'Adadelta\'], "
+  }
+  member_method {
+    name: "add_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\', \'initializer\'], varargs=None, keywords=None, defaults=[\'zeros\'], "
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'trainable\', \'synchronization\', \'aggregation\'], varargs=None, keywords=None, defaults=[\'None\', \'zeros\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+  }
+  member_method {
+    name: "apply_gradients"
+    argspec: "args=[\'self\', \'grads_and_vars\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "from_config"
-    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'cls\', \'config\', \'custom_objects\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "get_config"
@@ -19,6 +40,14 @@ tf_class {
     name: "get_gradients"
     argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "get_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_slot_names"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "get_updates"
     argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
@@ -27,8 +56,16 @@ tf_class {
     name: "get_weights"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "minimize"
+    argspec: "args=[\'self\', \'loss\', \'var_list\', \'grad_loss\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "set_weights"
     argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "variables"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
 }
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adagrad.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adagrad.pbtxt
index d0dc9e37a3..c39fe6ba4f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adagrad.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adagrad.pbtxt
@@ -1,15 +1,36 @@
 path: "tensorflow.keras.optimizers.Adagrad"
 tf_class {
-  is_instance: "<class \'tensorflow.python.keras.optimizers.Adagrad\'>"
-  is_instance: "<class \'tensorflow.python.keras.optimizers.Optimizer\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.adagrad.Adagrad\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "iterations"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'lr\', \'epsilon\', \'decay\'], varargs=None, keywords=kwargs, defaults=[\'0.01\', \'None\', \'0.0\'], "
+    argspec: "args=[\'self\', \'learning_rate\', \'initial_accumulator_value\', \'epsilon\', \'name\'], varargs=None, keywords=kwargs, defaults=[\'0.001\', \'0.1\', \'1e-07\', \'Adagrad\'], "
+  }
+  member_method {
+    name: "add_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\', \'initializer\'], varargs=None, keywords=None, defaults=[\'zeros\'], "
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'trainable\', \'synchronization\', \'aggregation\'], varargs=None, keywords=None, defaults=[\'None\', \'zeros\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+  }
+  member_method {
+    name: "apply_gradients"
+    argspec: "args=[\'self\', \'grads_and_vars\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "from_config"
-    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'cls\', \'config\', \'custom_objects\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "get_config"
@@ -19,6 +40,14 @@ tf_class {
     name: "get_gradients"
     argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "get_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_slot_names"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "get_updates"
     argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
@@ -27,8 +56,16 @@ tf_class {
     name: "get_weights"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "minimize"
+    argspec: "args=[\'self\', \'loss\', \'var_list\', \'grad_loss\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "set_weights"
     argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "variables"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
 }
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adam.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adam.pbtxt
index 06815fa99a..05d46d380b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adam.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adam.pbtxt
@@ -1,15 +1,36 @@
 path: "tensorflow.keras.optimizers.Adam"
 tf_class {
-  is_instance: "<class \'tensorflow.python.keras.optimizers.Adam\'>"
-  is_instance: "<class \'tensorflow.python.keras.optimizers.Optimizer\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.adam.Adam\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "iterations"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'lr\', \'beta_1\', \'beta_2\', \'epsilon\', \'decay\', \'amsgrad\'], varargs=None, keywords=kwargs, defaults=[\'0.001\', \'0.9\', \'0.999\', \'None\', \'0.0\', \'False\'], "
+    argspec: "args=[\'self\', \'learning_rate\', \'beta_1\', \'beta_2\', \'epsilon\', \'amsgrad\', \'name\'], varargs=None, keywords=kwargs, defaults=[\'0.001\', \'0.9\', \'0.999\', \'1e-07\', \'False\', \'Adam\'], "
+  }
+  member_method {
+    name: "add_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\', \'initializer\'], varargs=None, keywords=None, defaults=[\'zeros\'], "
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'trainable\', \'synchronization\', \'aggregation\'], varargs=None, keywords=None, defaults=[\'None\', \'zeros\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+  }
+  member_method {
+    name: "apply_gradients"
+    argspec: "args=[\'self\', \'grads_and_vars\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "from_config"
-    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'cls\', \'config\', \'custom_objects\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "get_config"
@@ -19,6 +40,14 @@ tf_class {
     name: "get_gradients"
     argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "get_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_slot_names"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "get_updates"
     argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
@@ -27,8 +56,16 @@ tf_class {
     name: "get_weights"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "minimize"
+    argspec: "args=[\'self\', \'loss\', \'var_list\', \'grad_loss\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "set_weights"
     argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "variables"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
 }
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adamax.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adamax.pbtxt
index 47b55fdb44..88fd96c71d 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adamax.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adamax.pbtxt
@@ -1,15 +1,36 @@
 path: "tensorflow.keras.optimizers.Adamax"
 tf_class {
-  is_instance: "<class \'tensorflow.python.keras.optimizers.Adamax\'>"
-  is_instance: "<class \'tensorflow.python.keras.optimizers.Optimizer\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.adamax.Adamax\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "iterations"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'lr\', \'beta_1\', \'beta_2\', \'epsilon\', \'decay\'], varargs=None, keywords=kwargs, defaults=[\'0.002\', \'0.9\', \'0.999\', \'None\', \'0.0\'], "
+    argspec: "args=[\'self\', \'learning_rate\', \'beta_1\', \'beta_2\', \'epsilon\', \'name\'], varargs=None, keywords=kwargs, defaults=[\'0.001\', \'0.9\', \'0.999\', \'1e-07\', \'Adamax\'], "
+  }
+  member_method {
+    name: "add_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\', \'initializer\'], varargs=None, keywords=None, defaults=[\'zeros\'], "
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'trainable\', \'synchronization\', \'aggregation\'], varargs=None, keywords=None, defaults=[\'None\', \'zeros\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+  }
+  member_method {
+    name: "apply_gradients"
+    argspec: "args=[\'self\', \'grads_and_vars\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "from_config"
-    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'cls\', \'config\', \'custom_objects\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "get_config"
@@ -19,6 +40,14 @@ tf_class {
     name: "get_gradients"
     argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "get_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_slot_names"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "get_updates"
     argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
@@ -27,8 +56,16 @@ tf_class {
     name: "get_weights"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "minimize"
+    argspec: "args=[\'self\', \'loss\', \'var_list\', \'grad_loss\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "set_weights"
     argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "variables"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
 }
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-nadam.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-nadam.pbtxt
index 8c63a7dda9..7b824e7f0b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-nadam.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-nadam.pbtxt
@@ -1,15 +1,36 @@
 path: "tensorflow.keras.optimizers.Nadam"
 tf_class {
-  is_instance: "<class \'tensorflow.python.keras.optimizers.Nadam\'>"
-  is_instance: "<class \'tensorflow.python.keras.optimizers.Optimizer\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.nadam.Nadam\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "iterations"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'lr\', \'beta_1\', \'beta_2\', \'epsilon\', \'schedule_decay\'], varargs=None, keywords=kwargs, defaults=[\'0.002\', \'0.9\', \'0.999\', \'None\', \'0.004\'], "
+    argspec: "args=[\'self\', \'learning_rate\', \'beta_1\', \'beta_2\', \'epsilon\', \'name\'], varargs=None, keywords=kwargs, defaults=[\'0.001\', \'0.9\', \'0.999\', \'1e-07\', \'Nadam\'], "
+  }
+  member_method {
+    name: "add_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\', \'initializer\'], varargs=None, keywords=None, defaults=[\'zeros\'], "
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'trainable\', \'synchronization\', \'aggregation\'], varargs=None, keywords=None, defaults=[\'None\', \'zeros\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+  }
+  member_method {
+    name: "apply_gradients"
+    argspec: "args=[\'self\', \'grads_and_vars\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "from_config"
-    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'cls\', \'config\', \'custom_objects\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "get_config"
@@ -19,6 +40,14 @@ tf_class {
     name: "get_gradients"
     argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "get_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_slot_names"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "get_updates"
     argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
@@ -27,8 +56,16 @@ tf_class {
     name: "get_weights"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "minimize"
+    argspec: "args=[\'self\', \'loss\', \'var_list\', \'grad_loss\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "set_weights"
     argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "variables"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
 }
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-optimizer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-optimizer.pbtxt
index 53d64dae93..58b7f27491 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-optimizer.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-optimizer.pbtxt
@@ -1,14 +1,35 @@
 path: "tensorflow.keras.optimizers.Optimizer"
 tf_class {
-  is_instance: "<class \'tensorflow.python.keras.optimizers.Optimizer\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "iterations"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\'], varargs=None, keywords=kwargs, defaults=None"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\', \'initializer\'], varargs=None, keywords=None, defaults=[\'zeros\'], "
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'trainable\', \'synchronization\', \'aggregation\'], varargs=None, keywords=None, defaults=[\'None\', \'zeros\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+  }
+  member_method {
+    name: "apply_gradients"
+    argspec: "args=[\'self\', \'grads_and_vars\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "from_config"
-    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'cls\', \'config\', \'custom_objects\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "get_config"
@@ -18,6 +39,14 @@ tf_class {
     name: "get_gradients"
     argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "get_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_slot_names"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "get_updates"
     argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
@@ -26,8 +55,16 @@ tf_class {
     name: "get_weights"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "minimize"
+    argspec: "args=[\'self\', \'loss\', \'var_list\', \'grad_loss\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "set_weights"
     argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "variables"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
 }
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-r-m-sprop.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-r-m-sprop.pbtxt
index a1e9b8cceb..8de796edde 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-r-m-sprop.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-r-m-sprop.pbtxt
@@ -1,15 +1,36 @@
 path: "tensorflow.keras.optimizers.RMSprop"
 tf_class {
-  is_instance: "<class \'tensorflow.python.keras.optimizers.RMSprop\'>"
-  is_instance: "<class \'tensorflow.python.keras.optimizers.Optimizer\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.rmsprop.RMSprop\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "iterations"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'lr\', \'rho\', \'epsilon\', \'decay\'], varargs=None, keywords=kwargs, defaults=[\'0.001\', \'0.9\', \'None\', \'0.0\'], "
+    argspec: "args=[\'self\', \'learning_rate\', \'rho\', \'momentum\', \'epsilon\', \'centered\', \'name\'], varargs=None, keywords=kwargs, defaults=[\'0.001\', \'0.9\', \'0.0\', \'1e-07\', \'False\', \'RMSprop\'], "
+  }
+  member_method {
+    name: "add_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\', \'initializer\'], varargs=None, keywords=None, defaults=[\'zeros\'], "
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'trainable\', \'synchronization\', \'aggregation\'], varargs=None, keywords=None, defaults=[\'None\', \'zeros\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+  }
+  member_method {
+    name: "apply_gradients"
+    argspec: "args=[\'self\', \'grads_and_vars\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "from_config"
-    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'cls\', \'config\', \'custom_objects\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "get_config"
@@ -19,6 +40,14 @@ tf_class {
     name: "get_gradients"
     argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "get_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_slot_names"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "get_updates"
     argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
@@ -27,8 +56,16 @@ tf_class {
     name: "get_weights"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "minimize"
+    argspec: "args=[\'self\', \'loss\', \'var_list\', \'grad_loss\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "set_weights"
     argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "variables"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
 }
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-s-g-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-s-g-d.pbtxt
index a67fefb1ba..393eeb3d6c 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-s-g-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-s-g-d.pbtxt
@@ -1,15 +1,36 @@
 path: "tensorflow.keras.optimizers.SGD"
 tf_class {
-  is_instance: "<class \'tensorflow.python.keras.optimizers.SGD\'>"
-  is_instance: "<class \'tensorflow.python.keras.optimizers.Optimizer\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.gradient_descent.SGD\'>"
+  is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "iterations"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'lr\', \'momentum\', \'decay\', \'nesterov\'], varargs=None, keywords=kwargs, defaults=[\'0.01\', \'0.0\', \'0.0\', \'False\'], "
+    argspec: "args=[\'self\', \'learning_rate\', \'momentum\', \'nesterov\', \'name\'], varargs=None, keywords=kwargs, defaults=[\'0.001\', \'0.0\', \'False\', \'SGD\'], "
+  }
+  member_method {
+    name: "add_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\', \'initializer\'], varargs=None, keywords=None, defaults=[\'zeros\'], "
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'trainable\', \'synchronization\', \'aggregation\'], varargs=None, keywords=None, defaults=[\'None\', \'zeros\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+  }
+  member_method {
+    name: "apply_gradients"
+    argspec: "args=[\'self\', \'grads_and_vars\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "from_config"
-    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'cls\', \'config\', \'custom_objects\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
   member_method {
     name: "get_config"
@@ -19,6 +40,14 @@ tf_class {
     name: "get_gradients"
     argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "get_slot"
+    argspec: "args=[\'self\', \'var\', \'slot_name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_slot_names"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "get_updates"
     argspec: "args=[\'self\', \'loss\', \'params\'], varargs=None, keywords=None, defaults=None"
@@ -27,8 +56,16 @@ tf_class {
     name: "get_weights"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "minimize"
+    argspec: "args=[\'self\', \'loss\', \'var_list\', \'grad_loss\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
   member_method {
     name: "set_weights"
     argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "variables"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
 }
-- 
GitLab


From 99ed07a8e162e2c30b591ee6f7dcc8e5fb57838d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 11 Jan 2019 10:56:15 -0800
Subject: [PATCH 0597/1095] Switch callers from deprecated
 tf.distribute.Strategy.colocate_vars_with() to its replacement
 tf.distribute.Strategy.extended.colocate_vars_with().

PiperOrigin-RevId: 228908656
---
 .../distribute/python/parameter_server_strategy.py        | 6 +++---
 .../distribute/python/parameter_server_strategy_test.py   | 4 ++--
 tensorflow/python/distribute/parameter_server_strategy.py | 6 +++---
 tensorflow/python/keras/layers/normalization.py           | 8 ++++----
 tensorflow/python/training/optimizer.py                   | 2 +-
 tensorflow/python/training/slot_creator.py                | 4 ++--
 tensorflow/python/training/sync_replicas_optimizer.py     | 2 +-
 7 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy.py b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
index 0cefef7545..0da269204f 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy.py
@@ -70,9 +70,9 @@ class ParameterServerStrategy(distribute_lib.DistributionStrategy):
   variables.
 
   3) It is also not recommended to open a colocation scope (i.e. calling
-  `tf.colocate_with`) under the strategy's scope. For colocating variables,
-  use `distribution.colocate_vars_with` instead. Colocation of ops will possibly
-  create conflicts of device assignment.
+  `tf.colocate_with`) under the strategy's scope. For colocating variables, use
+  `strategy.extended.colocate_vars_with` instead. Colocation of ops will
+  possibly create conflicts of device assignment.
   """
 
   def __init__(self, num_gpus_per_worker=0):
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
index 802809e7c7..2d4f8a9be5 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
@@ -190,7 +190,7 @@ class ParameterServerStrategyTestBase(
                          '/job:worker/replica:0/task:0/%s' % last_part_device)
 
         # The colocate_vars_with can override the distribution's device.
-        with d.colocate_vars_with(x):
+        with d.extended.colocate_vars_with(x):
           y = variable_scope.get_variable(
               'y', initializer=20.0,
               aggregation=variable_scope.VariableAggregation.SUM)
@@ -344,7 +344,7 @@ class ParameterServerStrategyTestBase(
         self.assertEqual(e.device, device_util.canonicalize('/device:GPU:2'))
 
         # The colocate_vars_with can override the distribution's device.
-        with d.colocate_vars_with(x):
+        with d.extended.colocate_vars_with(x):
           y = variable_scope.get_variable(
               'y', initializer=20.0,
               aggregation=variable_scope.VariableAggregation.SUM)
diff --git a/tensorflow/python/distribute/parameter_server_strategy.py b/tensorflow/python/distribute/parameter_server_strategy.py
index 9e865082f1..8093f8ccb0 100644
--- a/tensorflow/python/distribute/parameter_server_strategy.py
+++ b/tensorflow/python/distribute/parameter_server_strategy.py
@@ -78,9 +78,9 @@ class ParameterServerStrategy(distribute_lib.DistributionStrategy):
   variables.
 
   2) It is also not recommended to open a colocation scope (i.e. calling
-  `tf.colocate_with`) under the strategy's scope. For colocating variables,
-  use `distribution.colocate_vars_with` instead. Colocation of ops will possibly
-  create conflicts of device assignment.
+  `tf.colocate_with`) under the strategy's scope. For colocating variables, use
+  `strategy.extended.colocate_vars_with` instead. Colocation of ops will
+  possibly create conflicts of device assignment.
   """
 
   def __init__(self):
diff --git a/tensorflow/python/keras/layers/normalization.py b/tensorflow/python/keras/layers/normalization.py
index c174c8ddd6..9167a747cd 100644
--- a/tensorflow/python/keras/layers/normalization.py
+++ b/tensorflow/python/keras/layers/normalization.py
@@ -392,16 +392,16 @@ class BatchNormalizationV2(Layer):
               aggregation=tf_variables.VariableAggregation.MEAN)
           return var
 
-        with distribution_strategy_context.get_distribution_strategy(
-        ).colocate_vars_with(self.moving_mean):
+        with distribution_strategy_context.get_strategy(
+        ).extended.colocate_vars_with(self.moving_mean):
           self.renorm_mean = _renorm_variable('renorm_mean', param_shape)
           self.renorm_mean_weight = _renorm_variable('renorm_mean_weight', ())
         # We initialize renorm_stddev to 0, and maintain the (0-initialized)
         # renorm_stddev_weight. This allows us to (1) mix the average
         # stddev with the minibatch stddev early in training, and (2) compute
         # the unbiased average stddev by dividing renorm_stddev by the weight.
-        with distribution_strategy_context.get_distribution_strategy(
-        ).colocate_vars_with(self.moving_variance):
+        with distribution_strategy_context.get_strategy(
+        ).extended.colocate_vars_with(self.moving_variance):
           self.renorm_stddev = _renorm_variable('renorm_stddev', param_shape)
           self.renorm_stddev_weight = _renorm_variable('renorm_stddev_weight',
                                                        ())
diff --git a/tensorflow/python/training/optimizer.py b/tensorflow/python/training/optimizer.py
index 8076ed31bf..a591dc9fec 100644
--- a/tensorflow/python/training/optimizer.py
+++ b/tensorflow/python/training/optimizer.py
@@ -816,7 +816,7 @@ class Optimizer(
     if v is None:
       self._maybe_initialize_checkpointable()
       distribution_strategy = distribute_ctx.get_strategy()
-      with distribution_strategy.colocate_vars_with(colocate_with):
+      with distribution_strategy.extended.colocate_vars_with(colocate_with):
         if eager:
           restored_initial_value = self._preload_simple_restoration(
               name=name, shape=None)
diff --git a/tensorflow/python/training/slot_creator.py b/tensorflow/python/training/slot_creator.py
index abe1253b00..0868cfdea8 100644
--- a/tensorflow/python/training/slot_creator.py
+++ b/tensorflow/python/training/slot_creator.py
@@ -122,7 +122,7 @@ def create_slot(primary, val, name, colocate_with_primary=True):
   with variable_scope.variable_scope(None, prefix + "/" + name):
     if colocate_with_primary:
       distribution_strategy = distribution_strategy_context.get_strategy()
-      with distribution_strategy.colocate_vars_with(primary):
+      with distribution_strategy.extended.colocate_vars_with(primary):
         return _create_slot_var(primary, val, "", validate_shape, None, None)
     else:
       return _create_slot_var(primary, val, "", validate_shape, None, None)
@@ -159,7 +159,7 @@ def create_slot_with_initializer(primary, initializer, shape, dtype, name,
   with variable_scope.variable_scope(None, prefix + "/" + name):
     if colocate_with_primary:
       distribution_strategy = distribution_strategy_context.get_strategy()
-      with distribution_strategy.colocate_vars_with(primary):
+      with distribution_strategy.extended.colocate_vars_with(primary):
         return _create_slot_var(primary, initializer, "", validate_shape, shape,
                                 dtype)
     else:
diff --git a/tensorflow/python/training/sync_replicas_optimizer.py b/tensorflow/python/training/sync_replicas_optimizer.py
index 0079ecc98b..21e9a99e7c 100644
--- a/tensorflow/python/training/sync_replicas_optimizer.py
+++ b/tensorflow/python/training/sync_replicas_optimizer.py
@@ -261,7 +261,7 @@ class SyncReplicasOptimizer(optimizer.Optimizer):
     local_anchor = control_flow_ops.no_op()
     # Colocating local_step variable prevents it being placed on the PS.
     distribution_strategy = distribution_strategy_context.get_strategy()
-    with distribution_strategy.colocate_vars_with(local_anchor):
+    with distribution_strategy.extended.colocate_vars_with(local_anchor):
       self._local_step = variable_scope.variable(
           initial_value=0,
           trainable=False,
-- 
GitLab


From 1d1aeb64b91effd4329965cc7954a3e21820c88b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 11 Jan 2019 11:16:53 -0800
Subject: [PATCH 0598/1095] Copy remote tensors to be local in ResolveTensor if
 they are Remote.

PiperOrigin-RevId: 228912733
---
 tensorflow/c/eager/c_api.cc                   | 30 ++++++++++++++-----
 tensorflow/c/eager/c_api_test.cc              |  7 +----
 .../core/common_runtime/eager/tensor_handle.h |  4 +--
 3 files changed, 26 insertions(+), 15 deletions(-)

diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc
index d5a391a98d..f0408677d4 100755
--- a/tensorflow/c/eager/c_api.cc
+++ b/tensorflow/c/eager/c_api.cc
@@ -443,15 +443,15 @@ TF_Tensor* TFE_TensorHandleResolve(TFE_TensorHandle* h, TF_Status* status) {
     return nullptr;
   }
   // TODO(agarwal): move this implementation inside TFE_TensorHandle.
-  tensorflow::Device* d = nullptr;
-  tensorflow::Device* op_device = nullptr;
   const tensorflow::Tensor* t = nullptr;
-  status->status = h->handle->TensorAndDevice(&t, &d, &op_device);
-  if (!status->status.ok()) return nullptr;
   tensorflow::TensorHandle* h_cpu = nullptr;
-  if (!IsCPU(d)) {
-    status->status = h->handle->CopyToDevice(
-        h->handle->Context(), h->handle->Context()->HostCPU(), &h_cpu);
+  tensorflow::Device* d = nullptr;
+  tensorflow::Device* op_device = nullptr;
+
+  if (h->handle->IsRemote()) {
+    status->status = EagerCopyToDevice(
+        h->handle, h->handle->Context(),
+        h->handle->Context()->HostCPU()->name().c_str(), &h_cpu);
     if (!status->status.ok()) {
       return nullptr;
     }
@@ -460,6 +460,22 @@ TF_Tensor* TFE_TensorHandleResolve(TFE_TensorHandle* h, TF_Status* status) {
       h_cpu->Unref();
       return nullptr;
     }
+  } else {
+    status->status = h->handle->TensorAndDevice(&t, &d, &op_device);
+    if (!status->status.ok()) return nullptr;
+
+    if (!IsCPU(d)) {
+      status->status = h->handle->CopyToDevice(
+          h->handle->Context(), h->handle->Context()->HostCPU(), &h_cpu);
+      if (!status->status.ok()) {
+        return nullptr;
+      }
+      status->status = h_cpu->TensorAndDevice(&t, &d, &op_device);
+      if (!status->status.ok()) {
+        h_cpu->Unref();
+        return nullptr;
+      }
+    }
   }
   TF_Tensor* retval = tensorflow::TF_TensorFromTensor(*t, status);
   if (h_cpu != nullptr) {
diff --git a/tensorflow/c/eager/c_api_test.cc b/tensorflow/c/eager/c_api_test.cc
index 6b39b79ee8..3d1ca4fb4b 100644
--- a/tensorflow/c/eager/c_api_test.cc
+++ b/tensorflow/c/eager/c_api_test.cc
@@ -175,13 +175,8 @@ void TestRemoteExecute(bool async) {
   TFE_Execute(matmul, &retvals[0], &num_retvals, status);
   EXPECT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
 
-  auto* retval_task0 = TFE_TensorHandleCopyToDevice(
-      retvals[0], ctx, "/job:localhost/replica:0/task:0/device:CPU:0", status);
-  ASSERT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
-
-  TF_Tensor* t = TFE_TensorHandleResolve(retval_task0, status);
+  TF_Tensor* t = TFE_TensorHandleResolve(retvals[0], status);
   ASSERT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
-  TFE_DeleteTensorHandle(retval_task0);
   float product[4] = {0};
   EXPECT_EQ(sizeof(product), TF_TensorByteSize(t));
   memcpy(&product[0], TF_TensorData(t), TF_TensorByteSize(t));
diff --git a/tensorflow/core/common_runtime/eager/tensor_handle.h b/tensorflow/core/common_runtime/eager/tensor_handle.h
index 1c81087f1f..3eae3c7cc3 100644
--- a/tensorflow/core/common_runtime/eager/tensor_handle.h
+++ b/tensorflow/core/common_runtime/eager/tensor_handle.h
@@ -147,6 +147,8 @@ class TensorHandle : public core::RefCounted {
            (ctx_ == nullptr || ctx_->HostCPU() == device_);
   }
 
+  bool IsRemote();
+
  private:
   // If the contents of the Tensor pointed to by this handle is yet to be
   // computed by a EagerNode, this function will block till that compuatation is
@@ -156,8 +158,6 @@ class TensorHandle : public core::RefCounted {
 
   bool IsReady();
 
-  bool IsRemote();
-
   // Id for the EagerNode that will compute the value pointed to by this handle.
   // If the value is 0, the handle is already ready, but not vice-versa.
   const uint64 node_id_;
-- 
GitLab


From 3c4a3e36f4c1c998fb31e2feca09b793f2b4a550 Mon Sep 17 00:00:00 2001
From: Thomas O'Malley <omalleyt@google.com>
Date: Fri, 11 Jan 2019 11:45:42 -0800
Subject: [PATCH 0599/1095] Whitelists TensorFlow Operations for use with the
 Keras Functional API.

Raw TensorFlow functions can now be used in conjunction with the Keras
Functional API during model creation. This obviates the need for users to create Lambda layers in most cases when using the Functional API. Like Lambda layers, TensorFlow functions that result in Variable creation or assign ops are not supported.

PiperOrigin-RevId: 228918545
---
 tensorflow/python/keras/BUILD                 |  11 ++
 tensorflow/python/keras/engine/base_layer.py  |  87 +++++++++++-
 .../python/keras/engine/base_layer_utils.py   |  98 +++++++++++++
 tensorflow/python/keras/engine/network.py     |  62 ++++++++-
 .../python/keras/layers/serialization.py      |   1 +
 .../keras/layers/tensorflow_op_layer_test.py  | 131 ++++++++++++++++++
 6 files changed, 388 insertions(+), 2 deletions(-)
 create mode 100644 tensorflow/python/keras/layers/tensorflow_op_layer_test.py

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 0ff2aaffcc..719a4a04c4 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -474,6 +474,17 @@ tf_py_test(
     ],
 )
 
+tf_py_test(
+    name = "tensorflow_op_layer_test",
+    size = "medium",
+    srcs = ["layers/tensorflow_op_layer_test.py"],
+    additional_deps = [
+        ":keras",
+        "@absl_py//absl/testing:parameterized",
+        "//tensorflow/python:client_testlib",
+    ],
+)
+
 tf_py_test(
     name = "convolutional_recurrent_test",
     size = "large",
diff --git a/tensorflow/python/keras/engine/base_layer.py b/tensorflow/python/keras/engine/base_layer.py
index f7c51f7993..47900e7533 100644
--- a/tensorflow/python/keras/engine/base_layer.py
+++ b/tensorflow/python/keras/engine/base_layer.py
@@ -25,7 +25,9 @@ import itertools
 import numpy as np
 from six.moves import zip  # pylint: disable=redefined-builtin
 
+from tensorflow.core.framework import node_def_pb2
 from tensorflow.python.eager import context
+from tensorflow.python.eager import function
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import func_graph
 from tensorflow.python.framework import ops
@@ -509,7 +511,6 @@ class Layer(checkpointable.CheckpointableBase):
       ValueError: if the layer's `call` method returns None (an invalid value).
     """
     input_list = nest.flatten(inputs)
-
     if context.executing_eagerly():
       # Accept NumPy inputs by converting to Tensors when executing eagerly.
       if all(isinstance(x, (np.ndarray, float, int)) for x in input_list):
@@ -522,6 +523,13 @@ class Layer(checkpointable.CheckpointableBase):
     # models using the functional API).
     build_graph = tf_utils.are_all_symbolic_tensors(input_list)
 
+    if build_graph:
+      # Only create Keras history if at least one tensor originates from a
+      # `keras.Input`. Otherwise this Layer may be being used outside the Keras
+      # framework.
+      if base_layer_utils.uses_keras_input_layers(inputs):
+        base_layer_utils.create_keras_history(inputs)
+
     # Handle Keras mask propagation from previous layer to current layer.
     previous_mask = None
     if build_graph and (not hasattr(self, '_compute_previous_mask') or
@@ -1748,6 +1756,83 @@ class Node(object):
     }
 
 
+class TensorFlowOpLayer(Layer):
+  """Wraps a TensorFlow Operation in a Layer.
+
+  This class is used internally by the Functional API. When a user
+  uses a raw TensorFlow Operation on symbolic tensors originating
+  from an `Input` Layer, the resultant operation will be wrapped
+  with this Layer object in order to make the operation compatible
+  with the Keras API.
+
+  This Layer will create a new, identical operation (except for inputs
+  and outputs) every time it is called. If `run_eagerly` is `True`,
+  the op creation and calculation will happen inside an Eager function.
+
+  Instances of this Layer are created when `autolambda` is called, which
+  is whenever a Layer's `__call__` encounters symbolic inputs that do
+  not have Keras metadata, or when a Network's `__init__` encounters
+  outputs that do not have Keras metadata.
+
+  Attributes:
+    node_def: String, the serialized NodeDef of the Op this layer will wrap.
+    constants: Dict of NumPy arrays, the values of any Tensors needed for this
+      Operation that do not originate from a Keras `Input` Layer. Since all
+      placeholders must come from Keras `Input` Layers, these Tensors must be
+      treated as constant in the Functional API.
+    name: String, the name of the Layer.
+    trainable: Bool, whether this Layer is trainable. Currently Variables are
+      not supported, and so this parameter has no effect.
+    dtype: The default dtype of this Layer. Inherited from `Layer` and has no
+      effect on this class, however is used in `get_config`.
+  """
+
+  def __init__(self,
+               node_def,
+               constants=None,
+               name=None,
+               trainable=True,
+               dtype=None):
+    super(TensorFlowOpLayer, self).__init__(
+        name=name, trainable=trainable, dtype=dtype)
+    self.node_def = node_def_pb2.NodeDef.FromString(node_def)
+    self.constants = constants or {}
+
+  def call(self, inputs):
+    if context.executing_eagerly():
+      return self._defun_call(inputs)
+    return self._make_op(inputs)
+
+  def _make_op(self, inputs):
+    inputs = nest.flatten(inputs)
+    graph = inputs[0].graph
+    with graph.as_default():
+      for index, constant in self.constants.items():
+        constant = ops.convert_to_tensor(constant)
+        inputs.insert(index, constant)
+
+      self.node_def.name = graph.unique_name(self.node_def.name)
+      c_op = ops._create_c_op(graph, self.node_def, inputs, control_inputs=[])
+      op = graph._create_op_from_tf_operation(c_op)
+
+      if len(op.outputs) == 1:
+        return op.outputs[0]
+      return op.outputs
+
+  @function.defun
+  def _defun_call(self, inputs):
+    """Wraps the op creation method in an Eager function for `run_eagerly`."""
+    return self._make_op(inputs)
+
+  def get_config(self):
+    config = super(TensorFlowOpLayer, self).get_config()
+    config.update({
+        'node_def': self.node_def.SerializeToString(),
+        'constants': self.constants
+    })
+    return config
+
+
 def default(method):
   """Decorates a method to detect overrides in subclasses."""
   method._is_default = True
diff --git a/tensorflow/python/keras/engine/base_layer_utils.py b/tensorflow/python/keras/engine/base_layer_utils.py
index f1b1f42c22..0d0257a1dd 100644
--- a/tensorflow/python/keras/engine/base_layer_utils.py
+++ b/tensorflow/python/keras/engine/base_layer_utils.py
@@ -229,3 +229,101 @@ def have_all_keras_metadata(tensors):
 
 def generate_placeholders_from_shape(shape):
   return array_ops.placeholder(shape=shape, dtype=backend.floatx())
+
+
+def create_keras_history(tensors):
+  """Wraps TensorFlow Operations for compatibility with the Functional API.
+
+  This method checks to see if a Tensor in `tensors` is missing Keras metadata
+  and has its origin in a Keras `Input` Layer. If so, this method will replace
+  the raw TensorFlow Operations that created this tensor with
+  `TensorFlowOpLayer`
+  instances that create identical operations.
+
+  Any Tensors not originating from a Keras `Input` Layer will be treated as
+  constants when constructing `TensorFlowOpLayer` instances.
+
+  Arguments:
+    tensors: A structure of Tensors, some of which come from raw TensorFlow
+      operations and need to have Keras metadata assigned to them.
+  """
+
+  _create_keras_history_helper(tensors, set())
+
+
+def _create_keras_history_helper(tensors, processed_ops=None):
+  """Helper method for `create_keras_history`.
+
+  Arguments:
+    tensors: A structure of Tensors for which to create Keras metadata.
+    processed_ops: TensorFlow operations that have already been wrapped in
+      `TensorFlowOpLayer` instances.
+
+  Returns:
+    The updated set of TensorFlow Operations that have been wrapped
+    in `TensorFlowOpLayer` instances.
+  """
+  # Import of `base_layer` needed in order to create `TensorFlowOpLayer`.
+  # Cannot be imported at top because of circular dependencies.
+  # TODO(omalleyt): Resolve circular dependency.
+  from tensorflow.python.keras.engine import base_layer  # pylint: disable=g-import-not-at-top
+  tensor_list = nest.flatten(tensors)
+  for tensor in tensor_list:
+    if getattr(tensor, '_keras_history', None) is not None:
+      continue
+    op = tensor.op  # The Op that created this Tensor.
+    if op not in processed_ops:
+      # Recursively set `_keras_history`.
+      op_inputs = list(op.inputs)
+      constants = {}
+      layer_inputs = []
+      for i, op_input in enumerate(op_inputs):
+        if uses_keras_input_layers(op_input):
+          layer_inputs.append(op_input)
+        else:
+          # Treat any value not originating from a `keras.Input` as
+          # a constant (Variables currently have `Placeholder` op type
+          # when originating from an eager context
+          # so can't be supported.
+          constants[i] = backend.function([], [op_input])([])
+      processed_ops = _create_keras_history_helper(layer_inputs, processed_ops)
+      name = op.name
+      node_def = op.node_def.SerializeToString()
+      op_layer = base_layer.TensorFlowOpLayer(
+          node_def, constants=constants, name=name)
+      op_layer._add_inbound_node(  # pylint: disable=protected-access
+          layer_inputs, op.outputs)
+      processed_ops.update([op])
+  return processed_ops
+
+
+def uses_keras_input_layers(tensors):
+  """Checks if at least one Tensor in `tensors` originates from a Keras `Input`.
+
+  If so, the Functional API is being used.
+
+  Arguments:
+    tensors: An arbitrary nested structure of Tensors.
+
+  Returns:
+    Bool, whether at least one Tensor originates from a Keras `Input`.
+  """
+  checked_tensors = set()
+  input_tensors = nest.flatten(tensors)
+
+  while input_tensors:
+    if any(
+        getattr(tensor, '_keras_history', None) is not None
+        for tensor in input_tensors):
+      return True
+    checked_tensors.update(input_tensors)
+    new_input_tensors = set()
+    for tensor in input_tensors:
+      try:
+        new_input_tensors.update(tensor.op.inputs)
+      except AttributeError:
+        # In case `tensor` is a Variable created in an Eager
+        # context
+        pass
+    input_tensors = list(new_input_tensors - checked_tensors)
+  return False
diff --git a/tensorflow/python/keras/engine/network.py b/tensorflow/python/keras/engine/network.py
index 991950993f..cca9874e8b 100644
--- a/tensorflow/python/keras/engine/network.py
+++ b/tensorflow/python/keras/engine/network.py
@@ -69,8 +69,64 @@ except ImportError:
 class Network(base_layer.Layer):
   """A `Network` is a composition of layers.
 
-  It is the topological form of a "model". A `Model`
+  `Network` is the topological form of a "model". A `Model`
   is simply a `Network` with added training routines.
+
+  Two types of `Networks` exist: Graph Networks and Subclass Networks. Graph
+  networks are used in the Keras Functional and Sequential APIs. Subclassed
+  networks are used when a user subclasses the `Model` class. In general,
+  more Keras features are supported with Graph Networks than with Subclassed
+  Networks, specifically:
+
+  - Model cloning (`keras.models.clone`)
+  - Serialization (`model.get_config()/from_config`, `model.to_json()/to_yaml()`
+  - Whole-model saving (`model.save()`)
+
+  A Graph Network can be instantiated by passing two arguments to `__init__`.
+  The first argument is the `keras.Input` Tensors that represent the inputs
+  to the Network. The second argument specifies the output Tensors that
+  represent the outputs of this Network. Both arguments can be a nested
+  structure of Tensors.
+
+  Example:
+
+  ```
+  inputs = {'x1': keras.Input(shape=(10,)), 'x2': keras.Input(shape=(1,))}
+  t = keras.layers.Dense(1, activation='relu')(inputs['x1'])
+  outputs = keras.layers.Add()([t, inputs['x2'])
+  network = Network(inputs, outputs)
+  ```
+
+  A Graph Network constructed using the Functional API can also include raw
+  TensorFlow functions, with the exception of functions that create Variables
+  or assign ops.
+
+  Example:
+
+  ```
+  inputs = keras.Input(shape=(10,))
+  x = keras.layers.Dense(1)(inputs)
+  outputs = tf.nn.relu(x)
+  network = Network(inputs, outputs)
+  ```
+
+  Subclassed Networks can be instantiated via `name` and (optional) `dynamic`
+  keyword arguments. Subclassed Networks keep track of their Layers, and their
+  `call` method can be overridden. Subclassed Networks are typically created
+  indirectly, by subclassing the `Model` class.
+
+  Example:
+
+  ```
+  class MyModel(keras.Model):
+    def __init__(self):
+      super(MyModel, self).__init__(name='my_model', dynamic=False)
+
+      self.layer1 = keras.layers.Dense(10, activation='relu')
+
+    def call(self, inputs):
+      return self.layer1(inputs)
+  ```
   """
 
   def __init__(self, *args, **kwargs):  # pylint: disable=super-init-not-called
@@ -174,6 +230,10 @@ class Network(base_layer.Layer):
     self._nested_inputs = inputs
     self.inputs = nest.flatten(inputs)
     self.outputs = nest.flatten(outputs)
+
+    if any(not hasattr(tensor, '_keras_history') for tensor in self.outputs):
+      base_layer_utils.create_keras_history(self._nested_outputs)
+
     self._validate_graph_inputs_and_outputs()
 
     self._base_init(name=name)
diff --git a/tensorflow/python/keras/layers/serialization.py b/tensorflow/python/keras/layers/serialization.py
index cef078a28c..e4c508b0f3 100644
--- a/tensorflow/python/keras/layers/serialization.py
+++ b/tensorflow/python/keras/layers/serialization.py
@@ -20,6 +20,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.keras.engine.base_layer import TensorFlowOpLayer
 from tensorflow.python.keras.engine.input_layer import Input
 from tensorflow.python.keras.engine.input_layer import InputLayer
 from tensorflow.python.keras.layers.advanced_activations import *
diff --git a/tensorflow/python/keras/layers/tensorflow_op_layer_test.py b/tensorflow/python/keras/layers/tensorflow_op_layer_test.py
new file mode 100644
index 0000000000..3268bf8b64
--- /dev/null
+++ b/tensorflow/python/keras/layers/tensorflow_op_layer_test.py
@@ -0,0 +1,131 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test for allowing TF ops to work with Keras Functional API."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from absl.testing import parameterized
+import numpy as np
+
+from tensorflow.python import keras
+from tensorflow.python.framework import ops
+from tensorflow.python.keras import keras_parameterized
+from tensorflow.python.keras import testing_utils
+from tensorflow.python.keras.optimizer_v2 import adam
+from tensorflow.python.ops import gen_nn_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.platform import test
+from tensorflow.python.util import nest
+
+
+def _single_op_at_end():
+  inputs = keras.Input(shape=(10,))
+  x = keras.layers.Dense(10)(inputs)
+  outputs = gen_nn_ops.relu(x, name='hey')
+  return inputs, outputs
+
+
+def _multiple_ops_at_end():
+  inputs = keras.Input(shape=(10,))
+  inputs = keras.Input(shape=(10,))
+  x = keras.layers.Dense(10)(inputs)
+  x = gen_nn_ops.relu(x, name='hey')
+  outputs = gen_nn_ops.relu(x, name='hey2')
+  return inputs, outputs
+
+
+def _single_op_in_middle():
+  inputs = keras.Input(shape=(10,))
+  x = keras.layers.Dense(10)(inputs)
+  x = gen_nn_ops.relu(x, name='hey')
+  outputs = keras.layers.Dense(10)(x)
+  return inputs, outputs
+
+
+def _multiple_ops_in_middle():
+  inputs = keras.Input(shape=(10,))
+  x = keras.layers.Dense(10)(inputs)
+  x = gen_nn_ops.relu(x, name='hey')
+  x = gen_nn_ops.relu(x, name='hey2')
+  outputs = keras.layers.Dense(10)(x)
+  return inputs, outputs
+
+
+def _single_standalone_branch():
+  inputs = keras.Input(shape=(10,))
+  x = keras.layers.Dense(10)(inputs)
+  outputs = x * 2
+  return inputs, outputs
+
+
+def _single_op_with_attrs():
+  inputs = keras.Input(shape=(10,))
+  x = math_ops.reduce_mean(inputs, axis=1, keepdims=True)
+  outputs = keras.layers.Dense(10)(x)
+  return inputs, outputs
+
+
+@keras_parameterized.run_all_keras_modes
+class AutoLambdaTest(keras_parameterized.TestCase):
+
+  @parameterized.named_parameters(
+      ('single_op_at_end', _single_op_at_end),
+      ('multiple_ops_at_end', _multiple_ops_at_end),
+      ('single_op_in_middle', _single_op_in_middle),
+      ('multiple_ops_in_middle', _multiple_ops_in_middle),
+      ('single_standalone_branch', _single_standalone_branch),
+      ('single_op_with_attrs', _single_op_with_attrs))
+  def test_autolambda(self, model_fn):
+    inputs, outputs = model_fn()
+    model = keras.Model(inputs, outputs)
+    model.compile(
+        adam.Adam(0.001), 'mse', run_eagerly=testing_utils.should_run_eagerly())
+
+    np_inputs = nest.map_structure(lambda x: np.ones((10, 10), 'float32'),
+                                   inputs)
+    np_outputs = nest.map_structure(lambda x: np.ones((10, 10), 'float32'),
+                                    outputs)
+    model.fit(np_inputs, np_outputs, batch_size=2)
+
+  def test_numerical_correctness_simple(self):
+    x = ops.convert_to_tensor([[-1., 0., -2., 1.]])
+    inputs = keras.Input(shape=(4,))
+    outputs = gen_nn_ops.relu(inputs)
+    model = keras.Model(inputs, outputs)
+    y = self.evaluate(model(x))
+    self.assertAllClose(y, [[0., 0., 0., 1.]])
+
+  def test_numerical_correctness_with_attrs(self):
+    x = ops.convert_to_tensor([[1.5, 1.5], [2.5, 3.5]])
+    inputs = keras.Input(shape=(10,))
+    outputs = math_ops.reduce_mean(inputs, axis=1)
+    model = keras.Model(inputs, outputs)
+    y = self.evaluate(model(x))
+    self.assertAllClose(y, [1.5, 3.])
+
+  def test_serialization(self):
+    x = ops.convert_to_tensor([-1., 0., -2., 1.])
+    inputs = keras.Input(shape=(4,))
+    outputs = gen_nn_ops.relu(inputs)
+    model1 = keras.Model(inputs, outputs)
+    y1 = self.evaluate(model1(x))
+    model2 = model1.from_config(model1.get_config())
+    y2 = self.evaluate(model2(x))
+    self.assertAllClose(y1, y2)
+
+
+if __name__ == '__main__':
+  test.main()
-- 
GitLab


From 80b129e2f6af86bf097861e586b2440ef18b6557 Mon Sep 17 00:00:00 2001
From: James Ring <sjr@google.com>
Date: Fri, 11 Jan 2019 11:49:42 -0800
Subject: [PATCH 0600/1095] Make c_api depend on model_loader_lite instead of
 model_loader

In the near future, some kernels will be implemented against the C API,
and will therefore depend on //tensorflow/c:c_api. c_api currently
depends on saved_models/loader, which depends on all of tensorflow,
which depends on all kernels. This will be a circular dependency when C
kernel implementations are introduced.

PiperOrigin-RevId: 228919312
---
 tensorflow/c/BUILD                              | 2 +-
 tensorflow/core/distributed_runtime/eager/BUILD | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/c/BUILD b/tensorflow/c/BUILD
index 9d267e9e59..4e2fe34d28 100644
--- a/tensorflow/c/BUILD
+++ b/tensorflow/c/BUILD
@@ -83,7 +83,7 @@ tf_cuda_library(
         ],
         "//conditions:default": [
             ":c_api_internal",
-            "//tensorflow/cc/saved_model:loader",
+            "//tensorflow/cc/saved_model:loader_lite",
             "//tensorflow/cc:gradients",
             "//tensorflow/cc:ops",
             "//tensorflow/cc:grad_ops",
diff --git a/tensorflow/core/distributed_runtime/eager/BUILD b/tensorflow/core/distributed_runtime/eager/BUILD
index 55b2657e74..6f08943e2d 100644
--- a/tensorflow/core/distributed_runtime/eager/BUILD
+++ b/tensorflow/core/distributed_runtime/eager/BUILD
@@ -84,6 +84,7 @@ tf_cc_test(
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:tensorflow",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core/common_runtime/eager:tensor_handle",
-- 
GitLab


From 2273a9c4e0d0ec3bd03be25887953d6abf8d09ad Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Fri, 11 Jan 2019 11:50:57 -0800
Subject: [PATCH 0601/1095] Fix pip issues with lazy loader and binary count

PiperOrigin-RevId: 228919530
---
 tensorflow/lite/python/interpreter.py      | 40 ++++++++++++++--------
 tensorflow/lite/tools/pip_package/setup.py | 12 ++++++-
 2 files changed, 37 insertions(+), 15 deletions(-)

diff --git a/tensorflow/lite/python/interpreter.py b/tensorflow/lite/python/interpreter.py
index a6183d13b5..a1325f0b1f 100644
--- a/tensorflow/lite/python/interpreter.py
+++ b/tensorflow/lite/python/interpreter.py
@@ -19,20 +19,32 @@ from __future__ import print_function
 
 import sys
 import numpy as np
-from tensorflow.python.util.lazy_loader import LazyLoader
-from tensorflow.python.util.tf_export import tf_export as _tf_export
-
-# Lazy load since some of the performance benchmark skylark rules
-# break dependencies. Must use double quotes to match code internal rewrite
-# rule.
-# pylint: disable=g-inconsistent-quotes
-_interpreter_wrapper = LazyLoader(
-    "_interpreter_wrapper", globals(),
-    "tensorflow.lite.python.interpreter_wrapper."
-    "tensorflow_wrap_interpreter_wrapper")
-# pylint: enable=g-inconsistent-quotes
-
-del LazyLoader
+
+# pylint: disable=g-import-not-at-top
+try:
+  from tensorflow.python.util.lazy_loader import LazyLoader
+  from tensorflow.python.util.tf_export import tf_export as _tf_export
+
+  # Lazy load since some of the performance benchmark skylark rules
+  # break dependencies. Must use double quotes to match code internal rewrite
+  # rule.
+  # pylint: disable=g-inconsistent-quotes
+  _interpreter_wrapper = LazyLoader(
+      "_interpreter_wrapper", globals(),
+      "tensorflow.lite.python.interpreter_wrapper."
+      "tensorflow_wrap_interpreter_wrapper")
+  # pylint: enable=g-inconsistent-quotes
+
+  del LazyLoader
+except ImportError:
+  # When full Tensorflow Python PIP is not available do not use lazy load
+  # and instead uf the tflite_runtime path.
+  from tflite_runtime.lite.python import interpreter_wrapper as _interpreter_wrapper
+
+  def tf_export_dummy(*x, **kwargs):
+    del x, kwargs
+    return lambda x: x
+  _tf_export = tf_export_dummy
 
 
 @_tf_export('lite.Interpreter')
diff --git a/tensorflow/lite/tools/pip_package/setup.py b/tensorflow/lite/tools/pip_package/setup.py
index 82f9ac5b8f..c5141c1753 100644
--- a/tensorflow/lite/tools/pip_package/setup.py
+++ b/tensorflow/lite/tools/pip_package/setup.py
@@ -57,12 +57,22 @@ RELATIVE_MAKEFILE_PATH = os.path.join(RELATIVE_MAKE_DIR, 'Makefile')
 DOWNLOAD_SCRIPT_PATH = os.path.join(MAKE_DIR, 'download_dependencies.sh')
 
 
+# Check physical memory and if we are on a reasonable non small SOC machine
+# with more than 4GB, use all the CPUs, otherwisxe only 1.
+def get_build_cpus():
+  physical_bytes = os.sysconf('SC_PAGESIZE') * os.sysconf('SC_PHYS_PAGES')
+  if physical_bytes < (1<<30) * 4:
+    return 1
+  else:
+    return multiprocessing.cpu_count()
+
+
 def make_args(target='', quiet=True):
   """Construct make command line."""
   args = (['make', 'SHELL=/bin/bash', '-C', TENSORFLOW_DIR]
           + MAKE_CROSS_OPTIONS +
           ['-f', RELATIVE_MAKEFILE_PATH, '-j',
-           str(multiprocessing.cpu_count())])
+           str(get_build_cpus())])
   if quiet:
     args.append('--quiet')
   if target:
-- 
GitLab


From fa5a0f85cd542c8fc2ec2b96813aafcf8231dd04 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Fri, 11 Jan 2019 11:53:15 -0800
Subject: [PATCH 0602/1095] Add some infrastructure for type registrations, use
 it to revive dictionaries

Lots of boilerplate for handling version numbers. This should make it reasonably easy to maintain backwards compatibility by appending new wrapper versions over time.

The main reason I want this is so we can eventually save a .signatures attribute without special casing, which requires dictionaries be revived as such. But hopefully the same infrastructure will work for Keras Layers. Those will need their own proto format, presumably another oneof in SavedUserObject.

PiperOrigin-RevId: 228919938
---
 tensorflow/python/saved_model/BUILD           |  23 +++
 tensorflow/python/saved_model/load.py         |  36 ++--
 tensorflow/python/saved_model/load_test.py    |  10 ++
 .../python/saved_model/revived_types.py       | 167 ++++++++++++++++++
 .../python/saved_model/revived_types_test.py  | 110 ++++++++++++
 tensorflow/python/saved_model/save.py         |  11 +-
 .../saved_model/saved_object_graph.proto      |   8 +-
 .../python/training/checkpointable/BUILD      |   1 +
 .../checkpointable/data_structures.py         |  12 ++
 9 files changed, 363 insertions(+), 15 deletions(-)
 create mode 100644 tensorflow/python/saved_model/revived_types.py
 create mode 100644 tensorflow/python/saved_model/revived_types_test.py

diff --git a/tensorflow/python/saved_model/BUILD b/tensorflow/python/saved_model/BUILD
index 8a54888582..ebbefd7cde 100644
--- a/tensorflow/python/saved_model/BUILD
+++ b/tensorflow/python/saved_model/BUILD
@@ -271,6 +271,7 @@ py_library(
         ":builder",
         ":constants",
         ":function_serialization",
+        ":revived_types",
         ":saved_object_graph_py",
         ":signature_constants",
         ":signature_def_utils",
@@ -320,6 +321,7 @@ py_library(
         ":constants",
         ":function_deserialization",
         ":loader",
+        ":revived_types",
         ":saved_object_graph_py",
         ":utils",
         "//tensorflow/python:function",
@@ -345,6 +347,27 @@ tf_py_test(
     ],
 )
 
+py_library(
+    name = "revived_types",
+    srcs = [
+        "revived_types.py",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":saved_object_graph_py",
+    ],
+)
+
+tf_py_test(
+    name = "revived_types_test",
+    srcs = ["revived_types_test.py"],
+    additional_deps = [
+        ":revived_types",
+        ":saved_object_graph_py",
+        "//tensorflow/python:client_testlib",
+    ],
+)
+
 py_library(
     name = "function_serialization",
     srcs = [
diff --git a/tensorflow/python/saved_model/load.py b/tensorflow/python/saved_model/load.py
index d184133fe9..93a081b62b 100644
--- a/tensorflow/python/saved_model/load.py
+++ b/tensorflow/python/saved_model/load.py
@@ -28,6 +28,7 @@ from tensorflow.python.saved_model import constants
 from tensorflow.python.saved_model import function_deserialization
 from tensorflow.python.saved_model import loader_impl
 from tensorflow.python.saved_model import nested_structure_coder
+from tensorflow.python.saved_model import revived_types
 from tensorflow.python.saved_model import saved_object_graph_pb2
 from tensorflow.python.saved_model import utils_impl as saved_model_utils
 from tensorflow.python.training.checkpointable import tracking
@@ -103,11 +104,17 @@ class _Loader(object):
 
   def _load_all(self):
     """Load all saved objects and wire their properties."""
-    self._nodes = [self._recreate(proto) for proto in self._proto.nodes]
+    self._nodes = []
+    node_setters = []
+    for proto in self._proto.nodes:
+      node, setter = self._recreate(proto)
+      self._nodes.append(node)
+      node_setters.append(setter)
     # After creating the objects, construct the edges between the objects.
-    for obj, object_proto in zip(self._nodes, self._proto.nodes):
+    for obj, object_proto, setter in zip(self._nodes, self._proto.nodes,
+                                         node_setters):
       for reference in object_proto.children:
-        setattr(obj, reference.local_name, self._nodes[reference.node_id])
+        setter(obj, reference.local_name, self._nodes[reference.node_id])
         # Note: if an object has an attribute `__call__` add a class method
         # that allows `obj()` syntax to work. This is done per-instance to
         # allow `callable` to be used to find out if an object is callable.
@@ -137,30 +144,32 @@ class _Loader(object):
 
   def _recreate_user_object(self, proto):
     """Instantiates a SavedUserObject."""
-    del proto
+    looked_up = revived_types.deserialize(proto)
+    if looked_up is None:
+      # Note: each user object has its own class. This allows to make each one
+      # individually callable by adding a `__call__` method to the classes of
+      # the objects instances that have a `__call__` property.
 
-    # Note: each user object has its own class. This allows to make each one
-    # individually callable by adding a `__call__` method to the classes of
-    # the objects instances that have a `__call__` property.
-    class _UserObject(tracking.Checkpointable):
-      pass
+      class _UserObject(tracking.Checkpointable):
+        pass
 
-    return _UserObject()
+      return _UserObject(), setattr
+    return looked_up
 
   def _recreate_asset(self, proto):
     filename = os.path.join(
         saved_model_utils.get_assets_dir(self._export_dir),
         self._asset_file_def[proto.asset_file_def_index].filename)
-    return tracking.TrackableAsset(filename)
+    return tracking.TrackableAsset(filename), setattr
 
   def _recreate_function(self, proto):
     return function_deserialization.recreate_polymorphic_function(
-        proto, self._functions)
+        proto, self._functions), setattr
 
   def _recreate_variable(self, proto):
     # TODO(andresp): Can we use the checkpointed value as initializer?
     dummy_value = init_ops.Zeros(dtype=proto.dtype)(shape=proto.shape)
-    return variables.Variable(dummy_value, trainable=proto.trainable)
+    return variables.Variable(dummy_value, trainable=proto.trainable), setattr
 
 
 def _call_attribute(instance, *args, **kwargs):
@@ -191,3 +200,4 @@ def load(export_dir):
         "Currently only SavedModels exported with `tf.saved_model.save` may be "
         "imported. Other SavedModels may eventually be supported via load().")
   return root
+
diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index defd269012..1d6d1fe88c 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -406,6 +406,16 @@ class LoadTest(test.TestCase):
     self.assertAllEqual([2, 4, 6],
                         imported.f(constant_op.constant([1, 2, 3])).numpy())
 
+  def test_dict(self):
+    root = tracking.Checkpointable()
+    root.variables = dict(a=variables.Variable(1.))
+    root.variables["b"] = variables.Variable(2.)
+    root.variables["c"] = 1
+    imported = self.cycle(root)
+    self.assertEqual(1., imported.variables["a"].numpy())
+    self.assertEqual(2., imported.variables["b"].numpy())
+    self.assertEqual(set(["a", "b"]), set(imported.variables.keys()))
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/saved_model/revived_types.py b/tensorflow/python/saved_model/revived_types.py
new file mode 100644
index 0000000000..ae063202bb
--- /dev/null
+++ b/tensorflow/python/saved_model/revived_types.py
@@ -0,0 +1,167 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Handles types registrations for tf.saved_model.load."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.core.framework import versions_pb2
+from tensorflow.python.saved_model import saved_object_graph_pb2
+
+
+class VersionedTypeRegistration(object):
+  """Holds information about one version of a revived type."""
+
+  def __init__(self, object_factory, version, min_producer_version,
+               min_consumer_version, bad_consumers=None, setter=setattr):
+    """Identify a revived type version.
+
+    Args:
+      object_factory: A callable which takes a SavedUserObject proto and returns
+        a checkpointable object. Dependencies are added later via `setter`.
+      version: An integer, the producer version of this wrapper type. When
+        making incompatible changes to a wrapper, add a new
+        `VersionedTypeRegistration` with an incremented `version`. The most
+        recent version will be saved, and all registrations with a matching
+        identifier will be searched for the highest compatible version to use
+        when loading.
+      min_producer_version: The minimum producer version number required to use
+        this `VersionedTypeRegistration` when loading a proto.
+      min_consumer_version: `VersionedTypeRegistration`s with a version number
+        less than `min_consumer_version` will not be used to load a proto saved
+        with this object. `min_consumer_version` should be set to the lowest
+        version number which can successfully load protos saved by this
+        object. If no matching registration is available on load, the object
+        will be revived with a generic checkpointable type.
+
+        `min_consumer_version` and `bad_consumers` are a blunt tool, and using
+        them will generally break forward compatibility: previous versions of
+        TensorFlow will revive newly saved objects as opaque checkpointable
+        objects rather than wrapped objects. When updating wrappers, prefer
+        saving new information but preserving compatibility with previous
+        wrapper versions. They are, however, useful for ensuring that
+        previously-released buggy wrapper versions degrade gracefully rather
+        than throwing exceptions when presented with newly-saved SavedModels.
+      bad_consumers: A list of consumer versions which are incompatible (in
+        addition to any version less than `min_consumer_version`).
+      setter: A callable with the same signature as `setattr` to use when adding
+        dependencies to generated objects.
+    """
+    self.setter = setter
+    self.identifier = None  # Set after registration
+    self._object_factory = object_factory
+    self.version = version
+    self._min_consumer_version = min_consumer_version
+    self._min_producer_version = min_producer_version
+    if bad_consumers is None:
+      bad_consumers = []
+    self._bad_consumers = bad_consumers
+
+  def to_proto(self):
+    """Create a SavedUserObject proto."""
+    # For now wrappers just use dependencies to save their state, so the
+    # SavedUserObject doesn't depend on the object being saved.
+    # TODO(allenl): Add a wrapper which uses its own proto.
+    return saved_object_graph_pb2.SavedUserObject(
+        identifier=self.identifier,
+        version=versions_pb2.VersionDef(
+            producer=self.version,
+            min_consumer=self._min_consumer_version,
+            bad_consumers=self._bad_consumers))
+
+  def from_proto(self, proto):
+    """Recreate a checkpointable object from a SavedUserObject proto."""
+    return self._object_factory(proto)
+
+  def should_load(self, proto):
+    """Checks if this object should load the SavedUserObject `proto`."""
+    if proto.identifier != self.identifier:
+      return False
+    if self.version < proto.version.min_consumer:
+      return False
+    if proto.version.producer < self._min_producer_version:
+      return False
+    for bad_version in proto.version.bad_consumers:
+      if self.version == bad_version:
+        return False
+    return True
+
+
+# string identifier -> (predicate, [VersionedTypeRegistration])
+_REVIVED_TYPE_REGISTRY = {}
+_TYPE_IDENTIFIERS = []
+
+
+def register_revived_type(identifier, predicate, versions):
+  """Register a type for revived objects.
+
+  Args:
+    identifier: A unique string identifying this class of objects.
+    predicate: A Boolean predicate for this registration. Takes a
+      checkpointable object as an argument. If True, `type_registration` may be
+      used to save and restore the object.
+    versions: A list of `VersionedTypeRegistration` objects.
+  """
+  # Keep registrations in order of version. We always use the highest matching
+  # version (respecting the min consumer version and bad consumers).
+  versions.sort(key=lambda reg: reg.version, reverse=True)
+  if not versions:
+    raise AssertionError("Need at least one version of a registered type.")
+  version_numbers = set()
+  for registration in versions:
+    # Copy over the identifier for use in generating protos
+    registration.identifier = identifier
+    if registration.version in version_numbers:
+      raise AssertionError(
+          "Got multiple registrations with version {} for type {}".format(
+              registration.version, identifier))
+    version_numbers.add(registration.version)
+  if identifier in _REVIVED_TYPE_REGISTRY:
+    raise AssertionError(
+        "Duplicate registrations for type {}".format(identifier))
+
+  _REVIVED_TYPE_REGISTRY[identifier] = (predicate, versions)
+  _TYPE_IDENTIFIERS.append(identifier)
+
+
+def serialize(obj):
+  """Create a SavedUserObject from a checkpointable object."""
+  for identifier in _TYPE_IDENTIFIERS:
+    predicate, versions = _REVIVED_TYPE_REGISTRY[identifier]
+    if predicate(obj):
+      # Always uses the most recent version to serialize.
+      return versions[0].to_proto()
+  return None
+
+
+def deserialize(proto):
+  """Create a checkpointable object from a SavedUserObject proto.
+
+  Args:
+    proto: A SavedUserObject to deserialize.
+
+  Returns:
+    A tuple of (checkpointable, assignment_fn) where assignment_fn has the same
+    signature as setattr and should be used to add dependencies to
+    `checkpointable` when they are available.
+  """
+  _, type_registrations = _REVIVED_TYPE_REGISTRY.get(
+      proto.identifier, (None, None))
+  if type_registrations is not None:
+    for type_registration in type_registrations:
+      if type_registration.should_load(proto):
+        return (type_registration.from_proto(proto), type_registration.setter)
+  return None
diff --git a/tensorflow/python/saved_model/revived_types_test.py b/tensorflow/python/saved_model/revived_types_test.py
new file mode 100644
index 0000000000..98e10cfec0
--- /dev/null
+++ b/tensorflow/python/saved_model/revived_types_test.py
@@ -0,0 +1,110 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for revived type matching."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.core.framework import versions_pb2
+from tensorflow.python.platform import test
+from tensorflow.python.saved_model import revived_types
+from tensorflow.python.saved_model import saved_object_graph_pb2
+from tensorflow.python.training.checkpointable import tracking
+
+
+class CustomTestClass(tracking.Checkpointable):
+
+  def __init__(self, version):
+    self.version = version
+
+
+revived_types.register_revived_type(
+    "test_type",
+    lambda obj: isinstance(obj, CustomTestClass),
+    versions=[
+        revived_types.VersionedTypeRegistration(
+            object_factory=lambda _: CustomTestClass(1),
+            version=1, min_producer_version=1,
+            min_consumer_version=1),
+        revived_types.VersionedTypeRegistration(
+            object_factory=lambda _: CustomTestClass(2),
+            version=2, min_producer_version=2, min_consumer_version=1),
+        revived_types.VersionedTypeRegistration(
+            object_factory=lambda _: CustomTestClass(3),
+            version=3, min_producer_version=3, min_consumer_version=2),
+        revived_types.VersionedTypeRegistration(
+            object_factory=lambda _: CustomTestClass(4),
+            version=4, min_producer_version=4, min_consumer_version=2,
+            bad_consumers=[3]),
+    ]
+)
+
+
+class RegistrationMatchingTest(test.TestCase):
+
+  def test_save_typecheck(self):
+    self.assertIs(revived_types.serialize(tracking.Checkpointable()), None)
+
+  def test_load_identifier_not_found(self):
+    nothing_matches = revived_types.deserialize(
+        saved_object_graph_pb2.SavedUserObject(
+            identifier="_unregistered_type",
+            version=versions_pb2.VersionDef(
+                producer=1,
+                min_consumer=1,
+                bad_consumers=[])))
+    self.assertIs(nothing_matches, None)
+
+  def test_most_recent_version_saved(self):
+    serialized = revived_types.serialize(CustomTestClass(None))
+    self.assertEqual([3], serialized.version.bad_consumers)
+    deserialized, _ = revived_types.deserialize(serialized)
+    self.assertIsInstance(deserialized, CustomTestClass)
+    self.assertEqual(4, deserialized.version)
+
+  def test_min_consumer_version(self):
+    nothing_matches = revived_types.deserialize(
+        saved_object_graph_pb2.SavedUserObject(
+            identifier="test_type",
+            version=versions_pb2.VersionDef(
+                producer=5,
+                min_consumer=5,
+                bad_consumers=[])))
+    self.assertIs(nothing_matches, None)
+
+  def test_bad_versions(self):
+    deserialized, _ = revived_types.deserialize(
+        saved_object_graph_pb2.SavedUserObject(
+            identifier="test_type",
+            version=versions_pb2.VersionDef(
+                producer=5,
+                min_consumer=1,
+                bad_consumers=[4, 3])))
+    self.assertEqual(2, deserialized.version)
+
+  def test_min_producer_version(self):
+    deserialized, _ = revived_types.deserialize(
+        saved_object_graph_pb2.SavedUserObject(
+            identifier="test_type",
+            version=versions_pb2.VersionDef(
+                producer=3,
+                min_consumer=0,
+                bad_consumers=[])))
+    self.assertEqual(3, deserialized.version)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/saved_model/save.py b/tensorflow/python/saved_model/save.py
index 9db6d03ed0..0f98766553 100644
--- a/tensorflow/python/saved_model/save.py
+++ b/tensorflow/python/saved_model/save.py
@@ -22,6 +22,7 @@ import collections
 import functools
 import os
 
+from tensorflow.core.framework import versions_pb2
 from tensorflow.core.protobuf import meta_graph_pb2
 from tensorflow.core.protobuf import saved_model_pb2
 from tensorflow.python.eager import context
@@ -38,6 +39,7 @@ from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.saved_model import builder_impl
 from tensorflow.python.saved_model import constants
 from tensorflow.python.saved_model import function_serialization
+from tensorflow.python.saved_model import revived_types
 from tensorflow.python.saved_model import saved_object_graph_pb2
 from tensorflow.python.saved_model import signature_constants
 from tensorflow.python.saved_model import signature_def_utils
@@ -617,7 +619,14 @@ def _write_object_proto(obj, proto, asset_file_def_index, node_ids):
     proto.function.CopyFrom(
         function_serialization.serialize_polymorphic_function(obj, node_ids))
   else:
-    proto.user_object.SetInParent()
+    registered_type_proto = revived_types.serialize(obj)
+    if registered_type_proto is None:
+      # Fallback for types with no matching registration
+      registered_type_proto = saved_object_graph_pb2.SavedUserObject(
+          identifier="_generic_user_object",
+          version=versions_pb2.VersionDef(
+              producer=1, min_consumer=1, bad_consumers=[]))
+    proto.user_object.CopyFrom(registered_type_proto)
 
 
 @tf_export("saved_model.save", v1=["saved_model.experimental.save"])
diff --git a/tensorflow/python/saved_model/saved_object_graph.proto b/tensorflow/python/saved_model/saved_object_graph.proto
index a870eee383..a322726e1e 100644
--- a/tensorflow/python/saved_model/saved_object_graph.proto
+++ b/tensorflow/python/saved_model/saved_object_graph.proto
@@ -3,6 +3,7 @@ syntax = "proto3";
 import "tensorflow/core/protobuf/checkpointable_object_graph.proto";
 import "tensorflow/core/framework/tensor_shape.proto";
 import "tensorflow/core/framework/types.proto";
+import "tensorflow/core/framework/versions.proto";
 import "tensorflow/python/saved_model/struct.proto";
 
 option cc_enable_arenas = true;
@@ -62,7 +63,12 @@ message SavedObject {
 //
 // This object cannot be evaluated as a tensor, and therefore cannot be bound
 // to an input of a function.
-message SavedUserObject {}
+message SavedUserObject {
+  // Corresponds to a registration of the type to use in the loading program.
+  string identifier = 1;
+  // Version information from the producer of this SavedUserObject.
+  VersionDef version = 2;
+}
 
 // A SavedAsset represents a file in a SavedModel.
 //
diff --git a/tensorflow/python/training/checkpointable/BUILD b/tensorflow/python/training/checkpointable/BUILD
index 855dc4fb68..a39462732f 100644
--- a/tensorflow/python/training/checkpointable/BUILD
+++ b/tensorflow/python/training/checkpointable/BUILD
@@ -74,6 +74,7 @@ py_library(
     deps = [
         ":base",
         ":layer_utils",
+        "//tensorflow/python/saved_model:revived_types",
     ],
 )
 
diff --git a/tensorflow/python/training/checkpointable/data_structures.py b/tensorflow/python/training/checkpointable/data_structures.py
index 817552f326..c46af68155 100644
--- a/tensorflow/python/training/checkpointable/data_structures.py
+++ b/tensorflow/python/training/checkpointable/data_structures.py
@@ -19,10 +19,12 @@ from __future__ import print_function
 
 import collections
 import copy
+import operator
 
 import six
 
 from tensorflow.python.ops import variables
+from tensorflow.python.saved_model import revived_types
 from tensorflow.python.training.checkpointable import base
 from tensorflow.python.training.checkpointable import layer_utils
 
@@ -727,3 +729,13 @@ class _DictWrapper(Mapping, collections.MutableMapping):
   def update(self, *args, **kwargs):
     for key, value in dict(*args, **kwargs).items():
       self[key] = value
+
+revived_types.register_revived_type(
+    "checkpointable_dict_wrapper",
+    lambda obj: isinstance(obj, _DictWrapper),
+    versions=[revived_types.VersionedTypeRegistration(
+        object_factory=lambda _: _DictWrapper({}),
+        version=1,
+        min_producer_version=1,
+        min_consumer_version=1,
+        setter=operator.setitem)])
-- 
GitLab


From d2d8584845253fe30e1a43394f281d045b23492c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 11 Jan 2019 11:58:08 -0800
Subject: [PATCH 0603/1095] Automated rollback of commit
 196715b526a695678891ec11d13b7290b3f6861a

PiperOrigin-RevId: 228920631
---
 tensorflow/core/kernels/BUILD                  | 1 +
 tensorflow/core/kernels/fused_batch_norm_op.cc | 1 +
 2 files changed, 2 insertions(+)

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 0792b398f1..4db0505a8f 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -3820,6 +3820,7 @@ NN_DEPS = [
     ":conv_2d",
     ":eigen_contraction_kernel",
     ":ops_util",
+    ":pooling_ops",
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
diff --git a/tensorflow/core/kernels/fused_batch_norm_op.cc b/tensorflow/core/kernels/fused_batch_norm_op.cc
index 48b339508b..dbd3bb05db 100644
--- a/tensorflow/core/kernels/fused_batch_norm_op.cc
+++ b/tensorflow/core/kernels/fused_batch_norm_op.cc
@@ -18,6 +18,7 @@ limitations under the License.
 #if GOOGLE_CUDA
 #define EIGEN_USE_GPU
 #include "tensorflow/core/kernels/conv_2d.h"
+#include "tensorflow/core/kernels/conv_ops_gpu.h"
 #include "tensorflow/core/util/stream_executor_util.h"
 #endif
 
-- 
GitLab


From 3567f27bf3fe437def0588c92ebe8ef982e74e47 Mon Sep 17 00:00:00 2001
From: Saurabh Saxena <srbs@google.com>
Date: Fri, 11 Jan 2019 12:03:13 -0800
Subject: [PATCH 0604/1095] Support parallel_iterations in while_v2.

PiperOrigin-RevId: 228921645
---
 .../core/common_runtime/lower_while_op.cc     | 28 +++++++++++++------
 .../common_runtime/lower_while_op_test.cc     |  2 ++
 tensorflow/core/ops/functional_ops.cc         |  1 +
 .../kernel_tests/control_flow_ops_py_test.py  |  2 --
 tensorflow/python/ops/control_flow_ops.py     |  1 +
 tensorflow/python/ops/while_v2.py             |  4 +++
 6 files changed, 28 insertions(+), 10 deletions(-)

diff --git a/tensorflow/core/common_runtime/lower_while_op.cc b/tensorflow/core/common_runtime/lower_while_op.cc
index 8b68c31a72..7552838ca1 100644
--- a/tensorflow/core/common_runtime/lower_while_op.cc
+++ b/tensorflow/core/common_runtime/lower_while_op.cc
@@ -53,9 +53,10 @@ using NodeOut = NodeBuilder::NodeOut;
 class LowerWhileHelper {
  public:
   static Status Run(Node* while_op, const string& cond_fn_name,
-                    const string& body_fn_name, Graph* graph,
-                    const FunctionLibraryDefinition& flib) {
-    LowerWhileHelper helper(while_op, cond_fn_name, body_fn_name, graph, flib);
+                    const string& body_fn_name, int parallel_iterations,
+                    Graph* graph, const FunctionLibraryDefinition& flib) {
+    LowerWhileHelper helper(while_op, cond_fn_name, body_fn_name,
+                            parallel_iterations, graph, flib);
     return helper.RunInternal();
   }
 
@@ -64,8 +65,8 @@ class LowerWhileHelper {
   // and body functions named `cond_fn_name` and `body_fn_name` respectively in
   // the given graph.
   LowerWhileHelper(Node* while_op, const string& cond_fn_name,
-                   const string& body_fn_name, Graph* graph,
-                   const FunctionLibraryDefinition& flib);
+                   const string& body_fn_name, int parallel_iterations,
+                   Graph* graph, const FunctionLibraryDefinition& flib);
 
   Status RunInternal();
 
@@ -132,6 +133,8 @@ class LowerWhileHelper {
   const FunctionLibraryDefinition& flib_;
   // Name of the `while_op_`.
   string name_;
+  // Max number of parallel_iterations for the while loop.
+  const int parallel_iterations_;
 
   NodeDebugInfo debug_info_;
   NodeBuilder cond_call_builder_;
@@ -147,12 +150,14 @@ class LowerWhileHelper {
 };
 
 LowerWhileHelper::LowerWhileHelper(Node* while_op, const string& cond_fn_name,
-                                   const string& body_fn_name, Graph* graph,
+                                   const string& body_fn_name,
+                                   int parallel_iterations, Graph* graph,
                                    const FunctionLibraryDefinition& flib)
     : while_op_(while_op),
       graph_(graph),
       flib_(flib),
       name_(while_op->name()),
+      parallel_iterations_(parallel_iterations),
       debug_info_(*while_op_),
       cond_call_builder_(NewName("cond"), cond_fn_name, graph->op_registry(),
                          &debug_info_),
@@ -194,6 +199,7 @@ Status LowerWhileHelper::CreateEnterNodes() {
                                    graph_->op_registry(), &debug_info_)
                            .Input(NodeOut(edge->src(), edge->src_output()))
                            .Attr("frame_name", name_)
+                           .Attr("parallel_iterations", parallel_iterations_)
                            .Finalize(graph_, &enter_node));
     enter_nodes_[edge->dst_input()] = enter_node;
   }
@@ -392,9 +398,15 @@ Status RewriteWhileNode(Node* n, Graph* g,
   if (body_attr == nullptr) {
     return errors::InvalidArgument("While body function missing");
   }
+  const AttrValue* parallel_iterations_attr =
+      n->attrs().Find("parallel_iterations");
+  if (parallel_iterations_attr == nullptr) {
+    return errors::InvalidArgument("parallel_iterations attr missing");
+  }
 
-  TF_RETURN_IF_ERROR(LowerWhileHelper::Run(n, cond_attr->func().name(),
-                                           body_attr->func().name(), g, flib));
+  TF_RETURN_IF_ERROR(LowerWhileHelper::Run(
+      n, cond_attr->func().name(), body_attr->func().name(),
+      parallel_iterations_attr->i(), g, flib));
   g->RemoveNode(n);
 
   return Status::OK();
diff --git a/tensorflow/core/common_runtime/lower_while_op_test.cc b/tensorflow/core/common_runtime/lower_while_op_test.cc
index 24fd4ed5bb..fcb10bc75d 100644
--- a/tensorflow/core/common_runtime/lower_while_op_test.cc
+++ b/tensorflow/core/common_runtime/lower_while_op_test.cc
@@ -66,6 +66,7 @@ TEST(LowerWhileOpTest, Simple) {
                    .Attr("T", {DT_INT32})
                    .Attr("cond", cond_func)
                    .Attr("body", body_func)
+                   .Attr("parallel_iterations", 100)
                    .Attr(LowerIfWhilePass::kLowerUsingSwitchMergeAttr, true)
                    .Finalize(root.graph(), &while_node));
   TF_ASSERT_OK(root.DoShapeInference(while_node));
@@ -97,6 +98,7 @@ TEST(LowerWhileOpTest, Simple) {
   for (const auto* op : graph->op_nodes()) {
     if (op->IsEnter()) {
       ++enter_count;
+      ASSERT_EQ(op->attrs().Find("parallel_iterations")->i(), 100);
     }
     if (op->IsExit()) {
       ++exit_count;
diff --git a/tensorflow/core/ops/functional_ops.cc b/tensorflow/core/ops/functional_ops.cc
index 5e0bdd888c..be440ed728 100644
--- a/tensorflow/core/ops/functional_ops.cc
+++ b/tensorflow/core/ops/functional_ops.cc
@@ -170,6 +170,7 @@ REGISTER_OP("While")
     .Attr("cond: func")
     .Attr("body: func")
     .Attr("output_shapes: list(shape) = []")
+    .Attr("parallel_iterations: int = 10")
     .SetIsStateful()
     .SetShapeFn([](shape_inference::InferenceContext* c) {
       std::vector<PartialTensorShape> output_shapes;
diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
index 2949038728..25977510fb 100644
--- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
@@ -1065,7 +1065,6 @@ class ControlFlowTest(test.TestCase):
       self.assertEqual(printed.contents(), "D\nD\n")
 
   # Microbenchmark: 256,000 iterations/s.
-  @test_util.disable_control_flow_v2("b/116630618 (Times out)")
   def testWhile_1(self):
     with self.cached_session():
       n = constant_op.constant(0)
@@ -2245,7 +2244,6 @@ class ControlFlowTest(test.TestCase):
       self.assertAllClose(216.0, grad_a_val)
       self.assertAllClose(81.0, grad_v_val)
 
-  @test_util.disable_control_flow_v2("b/116630618 (parallel_iters: times out)")
   @test_util.run_deprecated_v1
   def testWhileGrad_Mul(self):
     self._testWhileGrad_Mul(use_gpu=False, p_iters=1)
diff --git a/tensorflow/python/ops/control_flow_ops.py b/tensorflow/python/ops/control_flow_ops.py
index 2b481bded5..3b3bfbbfc2 100644
--- a/tensorflow/python/ops/control_flow_ops.py
+++ b/tensorflow/python/ops/control_flow_ops.py
@@ -3492,6 +3492,7 @@ def while_loop(cond,
         body,
         loop_vars,
         shape_invariants=shape_invariants,
+        parallel_iterations=parallel_iterations,
         maximum_iterations=maximum_iterations,
         name=name,
         return_same_structure=return_same_structure)
diff --git a/tensorflow/python/ops/while_v2.py b/tensorflow/python/ops/while_v2.py
index cefadfb098..50011e34b1 100644
--- a/tensorflow/python/ops/while_v2.py
+++ b/tensorflow/python/ops/while_v2.py
@@ -57,6 +57,7 @@ def while_loop(cond,
                body,
                loop_vars,
                shape_invariants=None,
+               parallel_iterations=10,
                maximum_iterations=None,
                name=None,
                return_same_structure=True):
@@ -215,6 +216,7 @@ def while_loop(cond,
         util.create_new_tf_function(cond_graph),
         util.create_new_tf_function(body_graph),
         output_shapes=[t.shape for t in body_graph.outputs],
+        parallel_iterations=parallel_iterations,
         name=scope)
 
     _copy_handle_data(body_graph.outputs, outputs)
@@ -257,6 +259,7 @@ def _WhileGrad(op, *grads):  # pylint: disable=invalid-name
 
   maximum_iterations = op.get_attr(
       "_maximum_iterations") if _is_in_xla_context() else None
+  parallel_iterations = op.get_attr("parallel_iterations")
   assert not _is_in_xla_context() or maximum_iterations is not None
   maximum_iterations = _validate_and_convert_to_tensor(maximum_iterations)
 
@@ -325,6 +328,7 @@ def _WhileGrad(op, *grads):  # pylint: disable=invalid-name
       util.create_new_tf_function(cond_grad_graph),
       util.create_new_tf_function(body_grad_graph),
       output_shapes=[t.shape for t in body_grad_graph.outputs],
+      parallel_iterations=parallel_iterations,
       name="%s_grad" % while_op.name)
 
   _copy_handle_data(body_grad_graph.outputs, outputs)
-- 
GitLab


From 36d26bdbe45a2539de079aaf94a6c1c72bae5342 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Fri, 11 Jan 2019 12:09:44 -0800
Subject: [PATCH 0605/1095] [TF:XLA] Bump open source llvm revision to r350887

PiperOrigin-RevId: 228922762
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index ca4feb691e..bea57b8ed7 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -500,11 +500,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "llvm",
         build_file = clean_dep("//third_party/llvm:llvm.autogenerated.BUILD"),
-        sha256 = "b85f0571c5d2997142cd379eda3def286b26671fabb8453c5d56fee6e0698f52",
-        strip_prefix = "llvm-437f3bdb2355b0f453a4b4824e4b74ed6b6a73a5",
+        sha256 = "1b5e02d72413ae0de88a4457ff549434ab9c7b4f583f628eb6f29a9d199e280f",
+        strip_prefix = "llvm-ec5ab87c7698f6ede8905f833db67cd16449e360",
         urls = [
-            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/437f3bdb2355b0f453a4b4824e4b74ed6b6a73a5.tar.gz",
-            "https://github.com/llvm-mirror/llvm/archive/437f3bdb2355b0f453a4b4824e4b74ed6b6a73a5.tar.gz",
+            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/ec5ab87c7698f6ede8905f833db67cd16449e360.tar.gz",
+            "https://github.com/llvm-mirror/llvm/archive/ec5ab87c7698f6ede8905f833db67cd16449e360.tar.gz",
         ],
     )
 
-- 
GitLab


From 2193a41cdde2a1e9ea870e5c8df4dc84f0aabf92 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 11 Jan 2019 12:18:27 -0800
Subject: [PATCH 0606/1095] Update ops-related pbtxt files.

PiperOrigin-RevId: 228924111
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 40 +++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 |  7 ++++
 2 files changed, 47 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 15a4a10189..8705492c66 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -80322,6 +80322,46 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "While"
+  input_arg {
+    name: "input"
+    type_list_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_list_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "cond"
+    type: "func"
+  }
+  attr {
+    name: "body"
+    type: "func"
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    default_value {
+      list {
+      }
+    }
+  }
+  attr {
+    name: "parallel_iterations"
+    type: "int"
+    default_value {
+      i: 10
+    }
+  }
+  is_stateful: true
+}
 op {
   name: "WholeFileReader"
   output_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 9949c00aa4..012c03a834 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -38987,6 +38987,13 @@ op {
       }
     }
   }
+  attr {
+    name: "parallel_iterations"
+    type: "int"
+    default_value {
+      i: 10
+    }
+  }
   is_stateful: true
 }
 op {
-- 
GitLab


From 5a4bf97a3167ef6a725e689ef5b6cb1a7bedfa73 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Fri, 11 Jan 2019 12:31:33 -0800
Subject: [PATCH 0607/1095] Improve GDB interaction with the XLA CPU JIT

 - Register JITted code with GDB
 - Emit unwind information for JITed functions

With these changes GDB can unwind through JITted frames.

PiperOrigin-RevId: 228925859
---
 .../xla/service/cpu/simple_orc_jit.cc         | 29 ++++++++++++++++++-
 .../compiler/xla/service/cpu/simple_orc_jit.h | 15 ++++++++++
 .../compiler/xla/service/llvm_ir/llvm_util.cc |  4 +++
 3 files changed, 47 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
index 296f39a485..12ab6360c5 100644
--- a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
+++ b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
@@ -116,13 +116,26 @@ SimpleOrcJIT::SimpleOrcJIT(const llvm::TargetOptions& target_options,
                 orc_jit_memory_mapper::GetInstance());
             result.Resolver = symbol_resolver_;
             return result;
+          },
+          /*NotifyLoaded=*/
+          llvm::orc::LegacyRTDyldObjectLinkingLayer::NotifyLoadedFtor(),
+          /*NotifyFinalized=*/
+          [this](VModuleKeyT, const llvm::object::ObjectFile& object,
+                 const llvm::RuntimeDyld::LoadedObjectInfo& object_info) {
+            this->NotifyObjectFinalized(object, object_info);
+          },
+          /*NotifyFreed=*/
+          [this](VModuleKeyT, const llvm::object::ObjectFile& object) {
+            this->NotifyObjectFreed(object);
           }),
       compile_layer_(object_layer_,
                      CompilerFunctor(target_machine_.get(), &disassembler_,
                                      opt_level, optimize_for_size,
                                      enable_fast_math, disable_expensive_passes,
                                      std::move(pre_optimization_hook),
-                                     std::move(post_optimization_hook))) {
+                                     std::move(post_optimization_hook))),
+      gdb_jit_event_listener_(
+          llvm::JITEventListener::createGDBRegistrationListener()) {
   VLOG(1) << "CPU target: " << target_machine_->getTargetCPU().str()
           << " features: " << target_machine_->getTargetFeatureString().str();
 }
@@ -147,6 +160,20 @@ llvm::JITSymbol SimpleOrcJIT::ResolveRuntimeSymbol(const std::string& name) {
   return symbol_info;
 }
 
+void SimpleOrcJIT::NotifyObjectFinalized(
+    const llvm::object::ObjectFile& object,
+    const llvm::RuntimeDyld::LoadedObjectInfo& object_info) {
+  uint64_t key = static_cast<uint64_t>(
+      reinterpret_cast<uintptr_t>(object.getData().data()));
+  gdb_jit_event_listener_->notifyObjectLoaded(key, object, object_info);
+}
+
+void SimpleOrcJIT::NotifyObjectFreed(const llvm::object::ObjectFile& object) {
+  uint64_t key = static_cast<uint64_t>(
+      reinterpret_cast<uintptr_t>(object.getData().data()));
+  gdb_jit_event_listener_->notifyFreeingObject(key);
+}
+
 SimpleOrcJIT::VModuleKeyT SimpleOrcJIT::AddModule(
     std::unique_ptr<llvm::Module> module) {
   auto key = execution_session_.allocateVModule();
diff --git a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.h b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.h
index 78406ba143..3307c2f93d 100644
--- a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.h
+++ b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.h
@@ -21,6 +21,7 @@ limitations under the License.
 #include <vector>
 
 #include "llvm/ADT/Triple.h"
+#include "llvm/ExecutionEngine/JITEventListener.h"
 #include "llvm/ExecutionEngine/Orc/Core.h"
 #include "llvm/ExecutionEngine/Orc/IRCompileLayer.h"
 #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
@@ -99,6 +100,11 @@ class SimpleOrcJIT {
  private:
   llvm::JITSymbol ResolveRuntimeSymbol(const std::string& name);
 
+  void NotifyObjectFinalized(
+      const llvm::object::ObjectFile& object,
+      const llvm::RuntimeDyld::LoadedObjectInfo& object_info);
+  void NotifyObjectFreed(const llvm::object::ObjectFile& object);
+
   std::vector<VModuleKeyT> module_keys_;
   std::unique_ptr<llvm::TargetMachine> target_machine_;
   const Disassembler disassembler_;
@@ -107,6 +113,15 @@ class SimpleOrcJIT {
   std::shared_ptr<llvm::orc::SymbolResolver> symbol_resolver_;
   ObjLayerT object_layer_;
   CompileLayerT compile_layer_;
+
+  // Non owning pointer to a JIT event listener that registers the JIT events
+  // with an attached GDB.
+  //
+  // Note: we get a pointer to this event listener using
+  // `createGDBRegistrationListener` which makes it look like we're supposed to
+  // free this, but the function is poorly named and really just returns a
+  // pointer to a static object.
+  llvm::JITEventListener* gdb_jit_event_listener_;
 };
 
 }  // namespace cpu
diff --git a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc
index 75c4245f5f..83af5d2543 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc
@@ -621,6 +621,10 @@ llvm::Function* CreateFunction(llvm::FunctionType* function_type,
   function->setCallingConv(llvm::CallingConv::C);
   function->addFnAttr("no-frame-pointer-elim", "false");
 
+  // Generate unwind information so that GDB can crawl through the stack frames
+  // created by the JIT compiled code.
+  function->setHasUWTable();
+
   if (enable_fast_math) {
     function->addFnAttr("unsafe-fp-math", "true");
     function->addFnAttr("no-infs-fp-math", "true");
-- 
GitLab


From 25abe10651f6f0a4177d28874bcd11567708e02f Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Fri, 11 Jan 2019 12:34:47 -0800
Subject: [PATCH 0608/1095] [XLA:CPU] Implement batch dot

For now we lower a batch dot to N non-batch dot operations.  In the future we
may consider a more direct lowering.  This current CL is still better than what
we had because:

 - We don't blow up compile time by unrolling the batch dot
 - We avoid the slice and concat in the cases where they wouldn't get optimized away

After this CL DotDecomposer can be simplified, but I'll wait a bit and do that
in a separate CL to make it easier to roll this CL back if necessary.

PiperOrigin-RevId: 228926335
---
 tensorflow/compiler/xla/service/cpu/BUILD     |   2 +
 .../compiler/xla/service/cpu/cpu_compiler.cc  |   2 +-
 .../xla/service/cpu/dot_op_emitter.cc         | 179 +++++++++++++++++-
 .../compiler/xla/service/cpu/ir_emitter.cc    |   4 -
 .../compiler/xla/service/llvm_ir/ir_array.cc  |   4 +-
 .../compiler/xla/service/llvm_ir/ir_array.h   |  12 +-
 6 files changed, 185 insertions(+), 18 deletions(-)

diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD
index a197bdddc8..3f501fb64e 100644
--- a/tensorflow/compiler/xla/service/cpu/BUILD
+++ b/tensorflow/compiler/xla/service/cpu/BUILD
@@ -94,6 +94,7 @@ cc_library(
         ":target_machine_features",
         "@com_google_absl//absl/types:span",
         "//tensorflow/compiler/tf2xla:cpu_function_runtime",
+        "//tensorflow/compiler/xla/service:hlo_casting_utils",
         "//tensorflow/compiler/xla/service:map_inliner",
         "//tensorflow/compiler/xla/service:hlo_get_dimension_size_rewriter",
         "//tensorflow/compiler/xla/service:scatter_expander",
@@ -399,6 +400,7 @@ cc_library(
         "//tensorflow/compiler/xla:util",
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/service:hlo",
+        "//tensorflow/compiler/xla/service:hlo_casting_utils",
         "//tensorflow/compiler/xla/service:hlo_module_config",
         "//tensorflow/compiler/xla/service/llvm_ir:ir_array",
         "//tensorflow/compiler/xla/service/llvm_ir:kernel_support_library",
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
index a6d92ce10e..23c568236b 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
@@ -258,7 +258,7 @@ Status CpuCompiler::RunHloPassesThroughLayoutAssn(
   // pass.
   pipeline.AddPass<CallInliner>();
   pipeline.AddPass<BatchDotSimplification>();
-  pipeline.AddPass<DotDecomposer>();
+  pipeline.AddPass<DotDecomposer>(/*decompose_batch_dot=*/false);
   auto cost_model = [](HloInstruction* conv) {
     // We need a cost model for CPUs. Currently, do nothing.
     return false;
diff --git a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
index b018e0cd46..dba28aa51a 100644
--- a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
@@ -28,6 +28,8 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/cpu/target_machine_features.h"
 #include "tensorflow/compiler/xla/service/cpu/tiled_dot_emitter.h"
 #include "tensorflow/compiler/xla/service/cpu/vector_support_library.h"
+#include "tensorflow/compiler/xla/service/hlo_casting_utils.h"
+#include "tensorflow/compiler/xla/service/hlo_instructions.h"
 #include "tensorflow/compiler/xla/service/hlo_module.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/kernel_support_library.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/llvm_util.h"
@@ -57,6 +59,8 @@ struct DotInfo {
   Shape result_shape;
   DotDimensionNumbers dim_nums;
 
+  DotInfo() = default;
+
   explicit DotInfo(const HloInstruction& instr) {
     CHECK_EQ(instr.opcode(), HloOpcode::kDot);
     lhs_shape = instr.operand(0)->shape();
@@ -843,6 +847,163 @@ DotImplementationStrategy GetDotImplementationStrategy(
 
   return DotImplementationStrategy::kNaiveLlvmIr;
 }
+
+Status EmitNonBatchDotOperation(
+    DotInfo dot_info, string hlo_name, const llvm_ir::IrArray& target_array,
+    const llvm_ir::IrArray& lhs_array, const llvm_ir::IrArray& rhs_array,
+    const llvm_ir::IrArray* addend_array,
+    llvm::Value* executable_run_options_value, llvm::IRBuilder<>* b,
+    const HloModuleConfig& hlo_module_config,
+    const TargetMachineFeatures& target_machine_features) {
+  PrimitiveType type = target_array.GetShape().element_type();
+  TF_RET_CHECK(F16 == type || F32 == type || F64 == type || C64 == type);
+  DotOpEmitter dot_emitter(std::move(dot_info), std::move(hlo_name),
+                           target_array, lhs_array, rhs_array, addend_array,
+                           executable_run_options_value, b, hlo_module_config,
+                           target_machine_features);
+  return dot_emitter.Emit();
+}
+
+Shape DropFirstDim(const Shape& shape) {
+  absl::Span<int64 const> array_shape_dims(shape.dimensions());
+  array_shape_dims.remove_prefix(1);
+  return ShapeUtil::MakeShapeWithDescendingLayout(shape.element_type(),
+                                                  array_shape_dims);
+}
+
+Shape CollapseFirstNDims(const Shape& shape, int64 n) {
+  absl::Span<int64 const> input_shape_dims(shape.dimensions());
+  int64 prefix_dim =
+      std::accumulate(input_shape_dims.begin(), input_shape_dims.begin() + n,
+                      1ll, std::multiplies<int64>());
+  DimensionVector result_dims;
+  result_dims.push_back(prefix_dim);
+  std::copy(input_shape_dims.begin() + n, input_shape_dims.end(),
+            std::back_inserter(result_dims));
+  return ShapeUtil::MakeShapeWithDescendingLayout(shape.element_type(),
+                                                  result_dims);
+}
+
+llvm_ir::IrArray CollapseFirstNDims(llvm::IRBuilder<>* b,
+                                    const llvm_ir::IrArray& array, int64 n) {
+  llvm::Module* module = b->GetInsertBlock()->getParent()->getParent();
+  const Shape& shape = array.GetShape();
+  CHECK(shape.has_layout() &&
+        LayoutUtil::IsMonotonicWithDim0Major(shape.layout()));
+  CHECK_GE(shape.dimensions_size(), n);
+  Shape new_shape = CollapseFirstNDims(shape, n);
+  llvm::Value* new_value = b->CreateBitCast(
+      array.GetBasePointer(),
+      llvm_ir::ShapeToIrType(new_shape, module)->getPointerTo());
+  return llvm_ir::IrArray(new_value, std::move(new_shape));
+}
+
+Status ValidateDotDimensionNumbers(const DotDimensionNumbers& dim_numbers) {
+  // Checks some invariants that do not hold in general, but DotDecomposer
+  // should have established for us.  This is just a debugging aid.
+  TF_RET_CHECK(dim_numbers.lhs_contracting_dimensions_size() == 1);
+  std::vector<int64> batch_dim_numbers(dim_numbers.lhs_batch_dimensions_size());
+  absl::c_iota(batch_dim_numbers, 0);
+  TF_RET_CHECK(
+      absl::c_equal(batch_dim_numbers, dim_numbers.lhs_batch_dimensions()));
+  TF_RET_CHECK(
+      absl::c_equal(batch_dim_numbers, dim_numbers.rhs_batch_dimensions()));
+  return Status::OK();
+}
+
+// Slice out the inner array at batch index `batch_index` from `outer_array`.
+llvm_ir::IrArray SliceOutInnerArray(llvm_ir::IrArray outer_array,
+                                    llvm::Value* batch_index,
+                                    llvm::IRBuilder<>* b) {
+  llvm::Module* module = b->GetInsertBlock()->getParent()->getParent();
+
+  Shape inner_shape = DropFirstDim(outer_array.GetShape());
+  llvm_ir::IrArray::Index slice_index(b->getInt64Ty());
+  slice_index.push_back(batch_index);
+  slice_index.InsertAt(
+      /*index=*/1, outer_array.GetShape().dimensions_size() - 1,
+      b->getInt64(0));
+  llvm::Value* slice_ptr = outer_array.EmitArrayElementAddress(slice_index, b);
+  llvm::Type* slice_ptr_type =
+      llvm_ir::ShapeToIrType(inner_shape, module)->getPointerTo();
+  return llvm_ir::IrArray(b->CreateBitCast(slice_ptr, slice_ptr_type),
+                          std::move(inner_shape));
+}
+
+Status EmitBatchDotOperation(
+    const HloInstruction& dot, const llvm_ir::IrArray& target_array,
+    const llvm_ir::IrArray& lhs_array, const llvm_ir::IrArray& rhs_array,
+    llvm::Value* executable_run_options_value, llvm::IRBuilder<>* b,
+    const HloModuleConfig& hlo_module_config,
+    const TargetMachineFeatures& target_machine_features) {
+  TF_RETURN_IF_ERROR(ValidateDotDimensionNumbers(dot.dot_dimension_numbers()));
+
+  // Lower a batch dot into a sequence of non-batch dot operations.
+
+  int64 num_batch_dims =
+      dot.dot_dimension_numbers().lhs_batch_dimensions_size();
+
+  // First reshape the inputs to make sure we only have one batch dimension.
+  // This is a no-op bitcast because the operands have to be in row-major layout
+  // (enforced in CpuLayoutAssignment), and the batch dimensions are the leading
+  // dimensions (established by DotDecomposer and checked by
+  // ValidateDotDimensionNumbers above).
+  llvm_ir::IrArray lhs_array_reshaped =
+      CollapseFirstNDims(b, lhs_array, num_batch_dims);
+  llvm_ir::IrArray rhs_array_reshaped =
+      CollapseFirstNDims(b, rhs_array, num_batch_dims);
+  llvm_ir::IrArray target_array_reshaped =
+      CollapseFirstNDims(b, target_array, num_batch_dims);
+
+  int64 batch_count = lhs_array_reshaped.GetShape().dimensions(0);
+
+  KernelSupportLibrary ksl(b);
+
+  return ksl.ForWithStatus(
+      "bdot", /*start=*/0, /*end=*/batch_count, /*step=*/1,
+      [&](llvm::Value* indvar) {
+        DotDimensionNumbers adjusted_dim_numbers = dot.dot_dimension_numbers();
+        adjusted_dim_numbers.clear_lhs_batch_dimensions();
+        adjusted_dim_numbers.clear_rhs_batch_dimensions();
+
+        // Create a DotInfo representing the "inner" non-batch dot operation.
+        DotInfo dot_info;
+        dot_info.lhs_shape = DropFirstDim(lhs_array_reshaped.GetShape());
+        dot_info.rhs_shape = DropFirstDim(rhs_array_reshaped.GetShape());
+        dot_info.result_shape = DropFirstDim(target_array_reshaped.GetShape());
+        dot_info.dim_nums = dot.dot_dimension_numbers();
+        dot_info.dim_nums.clear_lhs_batch_dimensions();
+        dot_info.dim_nums.clear_rhs_batch_dimensions();
+
+        dot_info.dim_nums.set_lhs_contracting_dimensions(
+            0,
+            dot_info.dim_nums.lhs_contracting_dimensions(0) - num_batch_dims);
+        dot_info.dim_nums.set_rhs_contracting_dimensions(
+            0,
+            dot_info.dim_nums.rhs_contracting_dimensions(0) - num_batch_dims);
+
+        llvm_ir::IrArray lhs_slice =
+            SliceOutInnerArray(lhs_array_reshaped, /*batch_index=*/indvar, b);
+        llvm_ir::IrArray rhs_slice =
+            SliceOutInnerArray(rhs_array_reshaped, /*batch_index=*/indvar, b);
+        llvm_ir::IrArray target_slice = SliceOutInnerArray(
+            target_array_reshaped, /*batch_index=*/indvar, b);
+
+        // Emit the inner non-batch dot operation.
+        return EmitNonBatchDotOperation(
+            dot_info, dot.name(), target_slice, lhs_slice, rhs_slice, nullptr,
+            executable_run_options_value, b, hlo_module_config,
+            target_machine_features);
+      });
+}
+
+bool IsBatchDot(const HloInstruction& instr) {
+  if (auto* dot_instr = DynCast<HloDotInstruction>(&instr)) {
+    return dot_instr->dot_dimension_numbers().lhs_batch_dimensions_size() > 0;
+  }
+
+  return false;
+}
 }  // namespace
 
 bool DotImplementationCanHandleTranspose(
@@ -884,13 +1045,17 @@ Status EmitDotOperation(const HloInstruction& dot,
   // enclosing computation.
   CHECK(dot.parent()->root_instruction()->outer_dimension_partitions().empty());
 
-  PrimitiveType type = target_array.GetShape().element_type();
-  TF_RET_CHECK(F16 == type || F32 == type || F64 == type || C64 == type);
-  DotOpEmitter dot_emitter(DotInfo(dot), dot.name(), target_array, lhs_array,
-                           rhs_array, addend_array,
-                           executable_run_options_value, b, hlo_module_config,
-                           target_machine_features);
-  return dot_emitter.Emit();
+  if (IsBatchDot(dot)) {
+    TF_RET_CHECK(addend_array == nullptr);
+    return EmitBatchDotOperation(dot, target_array, lhs_array, rhs_array,
+                                 executable_run_options_value, b,
+                                 hlo_module_config, target_machine_features);
+  }
+
+  return EmitNonBatchDotOperation(DotInfo(dot), dot.name(), target_array,
+                                  lhs_array, rhs_array, addend_array,
+                                  executable_run_options_value, b,
+                                  hlo_module_config, target_machine_features);
 }
 }  // namespace cpu
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
index 91e3693354..b27a232557 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
@@ -944,10 +944,6 @@ Status IrEmitter::HandleDot(HloInstruction* dot) {
       /*instruction=*/*dot, /*operands=*/{lhs, rhs},
       /*supported_types=*/{F16, F32, F64, C64}));
   const DotDimensionNumbers& dnums = dot->dot_dimension_numbers();
-  if (dnums.lhs_batch_dimensions_size() > 0 ||
-      dnums.rhs_batch_dimensions_size() > 0) {
-    return Unimplemented("Dot with batch dimensions not implemented.");
-  }
 
   if (dnums.lhs_contracting_dimensions_size() != 1) {
     // This is disallowed by ShapeInference today.
diff --git a/tensorflow/compiler/xla/service/llvm_ir/ir_array.cc b/tensorflow/compiler/xla/service/llvm_ir/ir_array.cc
index f658761dff..8ee07ae833 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/ir_array.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/ir_array.cc
@@ -104,8 +104,8 @@ IrArray::Index::Index(absl::Span<llvm::Value* const> multidim,
   CHECK(LayoutUtil::HasLayout(shape));
 }
 
-IrArray::IrArray(llvm::Value* base_ptr, const Shape& shape)
-    : base_ptr_(base_ptr), shape_(&shape) {
+IrArray::IrArray(llvm::Value* base_ptr, Shape shape)
+    : base_ptr_(base_ptr), shape_(std::move(shape)) {
   TF_CHECK_OK(ShapeUtil::ValidateShape(shape));
   CHECK(base_ptr_->getType()->isPointerTy());
   int depth = 0;
diff --git a/tensorflow/compiler/xla/service/llvm_ir/ir_array.h b/tensorflow/compiler/xla/service/llvm_ir/ir_array.h
index a483f7051f..b706ebd311 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/ir_array.h
+++ b/tensorflow/compiler/xla/service/llvm_ir/ir_array.h
@@ -130,6 +130,11 @@ class IrArray {
       CHECK_LE(index, size());
       mutable_multidim().insert(mutable_multidim().begin() + index, value);
     }
+    void InsertAt(int64 index, int64 count, llvm::Value* value) {
+      CHECK_LE(index, size());
+      mutable_multidim().insert(mutable_multidim().begin() + index, count,
+                                value);
+    }
 
     using iterator = std::vector<llvm::Value*>::iterator;
     using const_iterator = std::vector<llvm::Value*>::const_iterator;
@@ -222,11 +227,11 @@ class IrArray {
   };
 
   // Default constructor. Constructs an IrArray in a null status.
-  IrArray() : base_ptr_(nullptr), shape_(nullptr) {}
+  IrArray() : base_ptr_(nullptr) {}
 
   // Construct an IrArray with the given base pointer and shape. base_ptr is a
   // pointer type pointing to the first element(lowest address) of the array.
-  IrArray(llvm::Value* base_ptr, const Shape& shape);
+  IrArray(llvm::Value* base_ptr, Shape shape);
 
   // Default implementations of copying and moving.
   IrArray(IrArray&& other) = default;
@@ -238,7 +243,6 @@ class IrArray {
   llvm::Type* GetElementLlvmType() const { return element_type_; }
 
   const Shape& GetShape() const {
-    CHECK(shape_ != nullptr);
     return *shape_;
   }
 
@@ -333,7 +337,7 @@ class IrArray {
   llvm::Type* element_type_;
 
   // Shape of the XLA array.
-  const Shape* shape_;
+  absl::optional<Shape> shape_;
 
   // The list of key/value pairs used when attaching metadata to emitted
   // loads/stores for this array.  They keys are the metadata kinds and the
-- 
GitLab


From a7a85ad829bc2401d4df4fc5400a99f5069ec8ba Mon Sep 17 00:00:00 2001
From: Scott Zhu <scottzhu@google.com>
Date: Fri, 11 Jan 2019 12:48:10 -0800
Subject: [PATCH 0609/1095] Internal test cleanup.

PiperOrigin-RevId: 228928263
---
 tensorflow/python/keras/BUILD              |  4 ++--
 tensorflow/python/keras/layers/gru_test.py | 15 ++++++++-------
 2 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 719a4a04c4..d36aaaf632 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -640,7 +640,7 @@ tf_py_test(
 
 tf_py_test(
     name = "gru_test",
-    size = "large",
+    size = "medium",
     srcs = ["layers/gru_test.py"],
     additional_deps = [
         ":keras",
@@ -648,7 +648,7 @@ tf_py_test(
         "//third_party/py/numpy",
         "//tensorflow/python:client_testlib",
     ],
-    shard_count = 2,
+    shard_count = 4,
     tags = ["notsan"],  # http://b/62136390
 )
 
diff --git a/tensorflow/python/keras/layers/gru_test.py b/tensorflow/python/keras/layers/gru_test.py
index 0e9dde8eae..91183a4d73 100644
--- a/tensorflow/python/keras/layers/gru_test.py
+++ b/tensorflow/python/keras/layers/gru_test.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from absl.testing import parameterized
 import numpy as np
 
 from tensorflow.python import keras
@@ -68,17 +69,17 @@ class GRULayerTest(keras_parameterized.TestCase):
                 'recurrent_dropout': 0.1},
         input_shape=(num_samples, timesteps, embedding_dim))
 
-  def test_implementation_mode_GRU(self):
+  @parameterized.parameters([0, 1, 2])
+  def test_implementation_mode_GRU(self, implementation_mode):
     num_samples = 2
     timesteps = 3
     embedding_dim = 4
     units = 2
-    for mode in [0, 1, 2]:
-      testing_utils.layer_test(
-          keras.layers.GRU,
-          kwargs={'units': units,
-                  'implementation': mode},
-          input_shape=(num_samples, timesteps, embedding_dim))
+    testing_utils.layer_test(
+        keras.layers.GRU,
+        kwargs={'units': units,
+                'implementation': implementation_mode},
+        input_shape=(num_samples, timesteps, embedding_dim))
 
   def test_reset_after_GRU(self):
     num_samples = 2
-- 
GitLab


From 2cecb70a63372d1ff510e7708a6e581bbe167efd Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 11 Jan 2019 12:58:15 -0800
Subject: [PATCH 0610/1095] Adapt SRUCell to keras RNN layer.

PiperOrigin-RevId: 228930242
---
 .../python/kernel_tests/core_rnn_cell_test.py | 29 +++++++++++++++++++
 tensorflow/contrib/rnn/python/ops/rnn_cell.py |  8 +++--
 2 files changed, 34 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py b/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py
index a0d013c618..7bad4a60a1 100644
--- a/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py
+++ b/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py
@@ -210,6 +210,35 @@ class RNNCellTest(test.TestCase):
         # Smoke test
         self.assertAllClose(res[0], [[0.509682, 0.509682]])
 
+  def testSRUCellKerasRNN(self):
+    """Tests that SRUCell works with keras RNN layer."""
+    cell = contrib_rnn_cell.SRUCell(10)
+    seq_input = ops.convert_to_tensor(
+        np.random.rand(2, 3, 5), name="seq_input", dtype=dtypes.float32)
+    rnn_layer = keras_layers.RNN(cell=cell)
+    rnn_outputs_keras = rnn_layer(seq_input)
+    with self.cached_session() as sess:
+      sess.run([variables_lib.global_variables_initializer()])
+      self.assertEqual(sess.run(rnn_outputs_keras).shape, (2, 10))
+
+  def testSRUCellBiasType(self):
+    """Tests that the bias' dtype is properly set."""
+    cell = contrib_rnn_cell.SRUCell(10)
+    cell.build((2, 3, 5))
+    self.assertEqual(cell._bias.dtype, dtypes.float32_ref)
+
+    cell = contrib_rnn_cell.SRUCell(10, dtype=dtypes.int32)
+    cell.build((2, 3, 5))
+    self.assertEqual(cell._bias.dtype, dtypes.int32_ref)
+
+    cell_input = ops.convert_to_tensor(
+        np.random.rand(2, 5), name="cell_input", dtype=dtypes.float16)
+    cell_state = ops.convert_to_tensor(
+        np.random.rand(2, 10), name="cell_state", dtype=dtypes.float16)
+    cell = contrib_rnn_cell.SRUCell(10)
+    cell(cell_input, [cell_state])
+    self.assertEqual(cell._bias.dtype, dtypes.float16_ref)
+
   def testSRUCellWithDiffSize(self):
     with self.cached_session() as sess:
       with variable_scope.variable_scope(
diff --git a/tensorflow/contrib/rnn/python/ops/rnn_cell.py b/tensorflow/contrib/rnn/python/ops/rnn_cell.py
index 0702b2a723..a2179cae0c 100644
--- a/tensorflow/contrib/rnn/python/ops/rnn_cell.py
+++ b/tensorflow/contrib/rnn/python/ops/rnn_cell.py
@@ -2744,10 +2744,12 @@ class SRUCell(rnn_cell_impl.LayerRNNCell):
     name: (optional) String, the name of the layer. Layers with the same name
       will share weights, but to avoid mistakes we require reuse=True in such
       cases.
+    **kwargs: Additional keyword arguments.
   """
 
-  def __init__(self, num_units, activation=None, reuse=None, name=None):
-    super(SRUCell, self).__init__(_reuse=reuse, name=name)
+  def __init__(self, num_units, activation=None, reuse=None, name=None,
+               **kwargs):
+    super(SRUCell, self).__init__(_reuse=reuse, name=name, **kwargs)
     self._num_units = num_units
     self._activation = activation or math_ops.tanh
 
@@ -2777,7 +2779,7 @@ class SRUCell(rnn_cell_impl.LayerRNNCell):
     self._bias = self.add_variable(
         rnn_cell_impl._BIAS_VARIABLE_NAME,  # pylint: disable=protected-access
         shape=[2 * self._num_units],
-        initializer=init_ops.constant_initializer(0.0, dtype=self.dtype))
+        initializer=init_ops.zeros_initializer)
 
     self._built = True
 
-- 
GitLab


From 937ff1b4cf1d954806e075d66198429a6d2312be Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Fri, 11 Jan 2019 13:05:56 -0800
Subject: [PATCH 0611/1095] [XLA] Add complex128 support.

Minimally tested at the moment (tested via the TF tests).

PiperOrigin-RevId: 228931680
---
 .../jit/mark_for_compilation_pass_test.cc     | 10 +-
 tensorflow/compiler/jit/xla_cpu_device.cc     |  4 +-
 .../compiler/jit/xla_interpreter_device.cc    |  4 +-
 tensorflow/compiler/tests/binary_ops_test.py  |  2 +-
 tensorflow/compiler/tests/build_defs.bzl      |  2 +-
 tensorflow/compiler/tests/unary_ops_test.py   |  2 +-
 tensorflow/compiler/tf2xla/kernels/cast_op.cc |  4 +-
 .../compiler/tf2xla/kernels/const_op.cc       | 11 +++
 .../compiler/tf2xla/kernels/matmul_op.cc      |  4 +-
 tensorflow/compiler/tf2xla/lib/util.cc        |  6 ++
 tensorflow/compiler/tf2xla/type_util.cc       |  3 +
 tensorflow/compiler/tf2xla/xla_op_registry.h  |  9 +-
 .../compiler/xla/client/lib/constants.h       |  2 +
 tensorflow/compiler/xla/client/lib/math.cc    |  4 +-
 tensorflow/compiler/xla/literal.cc            | 98 +++++++++++++++----
 tensorflow/compiler/xla/literal_comparison.cc | 58 +++++++++--
 tensorflow/compiler/xla/literal_test.cc       | 56 ++++++++++-
 tensorflow/compiler/xla/literal_util.cc       | 14 +++
 tensorflow/compiler/xla/primitive_util.cc     |  7 +-
 tensorflow/compiler/xla/primitive_util.h      | 10 ++
 .../compiler/xla/python/numpy_bridge.cc       | 11 +++
 tensorflow/compiler/xla/python/xla_client.py  |  1 +
 tensorflow/compiler/xla/service/BUILD         |  2 +
 .../xla/service/algebraic_simplifier.cc       |  3 +
 .../xla/service/cpu/dot_op_emitter.cc         |  3 +-
 .../compiler/xla/service/cpu/ir_emitter.cc    | 12 ++-
 .../compiler/xla/service/gpu/gemm_thunk.cc    |  8 ++
 .../xla/service/gpu/ir_emission_utils.cc      |  3 +-
 .../compiler/xla/service/hlo_evaluator.cc     | 97 ++++++++++++++++--
 .../xla/service/hlo_evaluator_typed_visitor.h | 15 ++-
 .../hlo_evaluator_typed_visitor_complex128.cc | 22 +++++
 .../compiler/xla/service/hlo_parser_test.cc   | 11 +++
 .../compiler/xla/service/llvm_ir/llvm_util.cc | 11 ++-
 .../compiler/xla/service/shape_inference.cc   |  4 +-
 .../xla/service/shape_inference_test.cc       |  8 +-
 tensorflow/compiler/xla/shape_util.cc         |  3 +
 .../xla/tests/client_library_test_base.h      | 15 ++-
 tensorflow/compiler/xla/types.h               |  1 +
 tensorflow/compiler/xla/xla_data.proto        |  6 +-
 39 files changed, 462 insertions(+), 84 deletions(-)
 create mode 100644 tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor_complex128.cc

diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc b/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc
index bf2c5508ea..c2b6250f73 100644
--- a/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc
+++ b/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc
@@ -151,7 +151,7 @@ TEST(XlaCompilationTest, CompilableCycles) {
   EXPECT_EQ(clusters["A"], clusters["C"]);
 }
 
-TEST(XlaCompilationTest, Complex128Unsupported) {
+TEST(XlaCompilationTest, StringUnsupported) {
   std::unique_ptr<Graph> graph(new Graph(OpRegistry::Global()));
   GraphDef graphdef;
   {
@@ -159,10 +159,10 @@ TEST(XlaCompilationTest, Complex128Unsupported) {
     Node* a = ops::SourceOp(
         "Const", builder.opts()
                      .WithName("A")
-                     .WithAttr("dtype", DT_COMPLEX128)
-                     .WithAttr("value", Tensor(DT_COMPLEX128, TensorShape())));
-    Node* b = ops::UnaryOp("Neg", a, builder.opts().WithName("B"));
-    ops::BinaryOp("MatMul", a, b, builder.opts().WithName("C"));
+                     .WithAttr("dtype", DT_STRING)
+                     .WithAttr("value", Tensor(DT_STRING, TensorShape())));
+    Node* b = ops::UnaryOp("EncodeBase64", a, builder.opts().WithName("B"));
+    ops::BinaryOp("StringSplit", a, b, builder.opts().WithName("C"));
     TF_EXPECT_OK(GraphDefBuilderToGraph(builder, graph.get()));
   }
 
diff --git a/tensorflow/compiler/jit/xla_cpu_device.cc b/tensorflow/compiler/jit/xla_cpu_device.cc
index e9770647e7..94dc61d55f 100644
--- a/tensorflow/compiler/jit/xla_cpu_device.cc
+++ b/tensorflow/compiler/jit/xla_cpu_device.cc
@@ -83,9 +83,9 @@ REGISTER_LOCAL_DEVICE_FACTORY(DEVICE_XLA_CPU, XlaCpuDeviceFactory);
 
 // Kernel registrations
 
-constexpr std::array<DataType, 12> kAllXlaCpuTypes = {
+constexpr std::array<DataType, 13> kAllXlaCpuTypes = {
     {DT_UINT8, DT_QUINT8, DT_INT8, DT_QINT8, DT_INT32, DT_QINT32, DT_INT64,
-     DT_HALF, DT_FLOAT, DT_DOUBLE, DT_COMPLEX64, DT_BOOL}};
+     DT_HALF, DT_FLOAT, DT_DOUBLE, DT_COMPLEX64, DT_COMPLEX128, DT_BOOL}};
 
 REGISTER_XLA_LAUNCH_KERNEL(DEVICE_XLA_CPU, XlaLocalLaunchOp, kAllXlaCpuTypes);
 REGISTER_XLA_COMPILE_KERNEL(DEVICE_XLA_CPU, XlaCompileOp, kAllXlaCpuTypes);
diff --git a/tensorflow/compiler/jit/xla_interpreter_device.cc b/tensorflow/compiler/jit/xla_interpreter_device.cc
index 4007309ed1..e1a5824061 100644
--- a/tensorflow/compiler/jit/xla_interpreter_device.cc
+++ b/tensorflow/compiler/jit/xla_interpreter_device.cc
@@ -26,9 +26,9 @@ namespace tensorflow {
 const char* const DEVICE_XLA_INTERPRETER = "XLA_INTERPRETER";
 const char* const DEVICE_INTERPRETER_XLA_JIT = "XLA_INTERPRETER_JIT";
 
-constexpr std::array<DataType, 9> kExecAllTypes = {
+constexpr std::array<DataType, 10> kExecAllTypes = {
     {DT_INT8, DT_INT32, DT_INT64, DT_HALF, DT_FLOAT, DT_DOUBLE, DT_COMPLEX64,
-     DT_BOOL, DT_BFLOAT16}};
+     DT_COMPLEX128, DT_BOOL, DT_BFLOAT16}};
 
 class XlaInterpreterDeviceFactory : public DeviceFactory {
  public:
diff --git a/tensorflow/compiler/tests/binary_ops_test.py b/tensorflow/compiler/tests/binary_ops_test.py
index 9a5423c1b2..a3651b4b0d 100644
--- a/tensorflow/compiler/tests/binary_ops_test.py
+++ b/tensorflow/compiler/tests/binary_ops_test.py
@@ -400,7 +400,7 @@ class BinaryOpsTest(xla_test.XLATestCase):
 
   def testComplexOps(self):
     for dtype in self.complex_types:
-      ctypes = {np.complex64: np.float32}
+      ctypes = {np.complex64: np.float32, np.complex128: np.float64}
       self._testBinary(
           math_ops.complex,
           np.array([[[[-1, 2], [2, 0]]]], dtype=ctypes[dtype]),
diff --git a/tensorflow/compiler/tests/build_defs.bzl b/tensorflow/compiler/tests/build_defs.bzl
index 447a7de2cb..be9766c4ef 100644
--- a/tensorflow/compiler/tests/build_defs.bzl
+++ b/tensorflow/compiler/tests/build_defs.bzl
@@ -64,7 +64,7 @@ def tf_xla_py_test(
         if backend == "cpu":
             backend_args += [
                 "--test_device=XLA_CPU",
-                "--types=DT_HALF,DT_FLOAT,DT_DOUBLE,DT_UINT8,DT_QUINT8,DT_INT8,DT_QINT8,DT_INT32,DT_QINT32,DT_INT64,DT_BOOL,DT_COMPLEX64",
+                "--types=DT_HALF,DT_FLOAT,DT_DOUBLE,DT_UINT8,DT_QUINT8,DT_INT8,DT_QINT8,DT_INT32,DT_QINT32,DT_INT64,DT_BOOL,DT_COMPLEX64,DT_COMPLEX128",
             ]
         elif backend == "gpu":
             backend_args += [
diff --git a/tensorflow/compiler/tests/unary_ops_test.py b/tensorflow/compiler/tests/unary_ops_test.py
index 95c9e7ffd4..3c2875ba47 100644
--- a/tensorflow/compiler/tests/unary_ops_test.py
+++ b/tensorflow/compiler/tests/unary_ops_test.py
@@ -647,7 +647,7 @@ class UnaryOpsTest(xla_test.XLATestCase):
           np.array([1, 2j, 2 - 3j, 4 + 5j], dtype=dtype),
           expected=np.tan(np.array([1, 2j, 2 - 3j, 4 + 5j], dtype=dtype)))
 
-      ctypes = {np.complex64: np.float32}
+      ctypes = {np.complex64: np.float32, np.complex128: np.float64}
       self._assertOpOutputMatchesExpected(
           math_ops.abs,
           np.array([[3 - 4j, -1j, np.inf]], dtype=dtype),
diff --git a/tensorflow/compiler/tf2xla/kernels/cast_op.cc b/tensorflow/compiler/tf2xla/kernels/cast_op.cc
index 8cc2479dd5..ce8131eeb4 100644
--- a/tensorflow/compiler/tf2xla/kernels/cast_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/cast_op.cc
@@ -79,8 +79,8 @@ class BitcastOp : public XlaOpKernel {
     if (src_dtype_ == dst_dtype_) {
       output = input;
     } else {
-      // The only complex type in XLA is C64, so error out if the bitcast has a
-      // complex source or destination type and the bitcast is not trivial.
+      // Error out if the bitcast has a complex source or destination type and
+      // the bitcast is not trivial.
       OP_REQUIRES(ctx,
                   !xla::primitive_util::IsComplexType(src_type_) &&
                       !xla::primitive_util::IsComplexType(dst_type_),
diff --git a/tensorflow/compiler/tf2xla/kernels/const_op.cc b/tensorflow/compiler/tf2xla/kernels/const_op.cc
index dff8af8002..ff6c54e47c 100644
--- a/tensorflow/compiler/tf2xla/kernels/const_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/const_op.cc
@@ -83,6 +83,17 @@ class ConstOp : public XlaOpKernel {
             return;
           }
           break;
+        case DT_COMPLEX128:
+          if (proto_.scomplex_val_size() == 2) {
+            ctx->SetOutput(
+                0,
+                xla::Broadcast(xla::ConstantR0<xla::complex128>(
+                                   b, xla::complex128(proto_.dcomplex_val(0),
+                                                      proto_.dcomplex_val(1))),
+                               shape.dim_sizes()));
+            return;
+          }
+          break;
         case DT_INT32:
           if (proto_.int_val_size() == 1) {
             ctx->SetOutput(
diff --git a/tensorflow/compiler/tf2xla/kernels/matmul_op.cc b/tensorflow/compiler/tf2xla/kernels/matmul_op.cc
index 6440770c29..f36e002525 100644
--- a/tensorflow/compiler/tf2xla/kernels/matmul_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/matmul_op.cc
@@ -24,8 +24,8 @@ limitations under the License.
 namespace tensorflow {
 namespace {
 
-constexpr std::array<DataType, 5> kMatmulTypes = {
-    {DT_HALF, DT_BFLOAT16, DT_FLOAT, DT_DOUBLE, DT_COMPLEX64}};
+constexpr std::array<DataType, 6> kMatmulTypes = {
+    {DT_HALF, DT_BFLOAT16, DT_FLOAT, DT_DOUBLE, DT_COMPLEX64, DT_COMPLEX128}};
 
 class MatMulOp : public XlaOpKernel {
  public:
diff --git a/tensorflow/compiler/tf2xla/lib/util.cc b/tensorflow/compiler/tf2xla/lib/util.cc
index c0bd172d17..06eda41611 100644
--- a/tensorflow/compiler/tf2xla/lib/util.cc
+++ b/tensorflow/compiler/tf2xla/lib/util.cc
@@ -54,6 +54,9 @@ xla::XlaOp FloatLiteral(xla::XlaBuilder* builder, xla::PrimitiveType type,
     case xla::C64:
       return xla::ConstantR0<xla::complex64>(builder, value);
       break;
+    case xla::C128:
+      return xla::ConstantR0<xla::complex128>(builder, value);
+      break;
     default:
       LOG(FATAL) << "unhandled element type " << type;
   }
@@ -90,6 +93,9 @@ xla::XlaOp IntegerLiteral(xla::XlaBuilder* builder, xla::PrimitiveType type,
     case xla::C64:
       literal = xla::LiteralUtil::CreateR0<complex64>(value);
       break;
+    case xla::C128:
+      literal = xla::LiteralUtil::CreateR0<complex128>(value);
+      break;
     case xla::PRED:
       LOG(FATAL) << "pred element type is not integral";
     case xla::S16:
diff --git a/tensorflow/compiler/tf2xla/type_util.cc b/tensorflow/compiler/tf2xla/type_util.cc
index d00b137662..732f957d73 100644
--- a/tensorflow/compiler/tf2xla/type_util.cc
+++ b/tensorflow/compiler/tf2xla/type_util.cc
@@ -69,6 +69,9 @@ Status DataTypeToPrimitiveType(DataType data_type, xla::PrimitiveType* type) {
     case tensorflow::DT_COMPLEX64:
       *type = xla::C64;
       return Status::OK();
+    case tensorflow::DT_COMPLEX128:
+      *type = xla::C128;
+      return Status::OK();
     default:
       return errors::InvalidArgument(
           "Unsupported type in DataTypeToPrimitiveType ",
diff --git a/tensorflow/compiler/tf2xla/xla_op_registry.h b/tensorflow/compiler/tf2xla/xla_op_registry.h
index 0bdd4a1085..ce3b6b298c 100644
--- a/tensorflow/compiler/tf2xla/xla_op_registry.h
+++ b/tensorflow/compiler/tf2xla/xla_op_registry.h
@@ -47,13 +47,14 @@ extern const char* const DEVICE_XLA_GPU;
 
 constexpr std::array<DataType, 4> kFloatTypes = {
     {DT_HALF, DT_FLOAT, DT_DOUBLE, DT_BFLOAT16}};
-constexpr std::array<DataType, 11> kNumericTypes = {
+constexpr std::array<DataType, 12> kNumericTypes = {
     {DT_UINT8, DT_UINT32, DT_UINT64, DT_INT8, DT_INT32, DT_INT64, DT_HALF,
-     DT_FLOAT, DT_DOUBLE, DT_COMPLEX64, DT_BFLOAT16}};
+     DT_FLOAT, DT_DOUBLE, DT_COMPLEX64, DT_COMPLEX128, DT_BFLOAT16}};
 
-constexpr std::array<DataType, 14> kCpuAllTypes = {
+constexpr std::array<DataType, 15> kCpuAllTypes = {
     {DT_UINT8, DT_QUINT8, DT_UINT32, DT_UINT64, DT_INT8, DT_QINT8, DT_INT32,
-     DT_QINT32, DT_INT64, DT_HALF, DT_FLOAT, DT_DOUBLE, DT_COMPLEX64, DT_BOOL}};
+     DT_QINT32, DT_INT64, DT_HALF, DT_FLOAT, DT_DOUBLE, DT_COMPLEX64,
+     DT_COMPLEX128, DT_BOOL}};
 
 constexpr std::array<DataType, 15> kGpuAllTypes = {
     {DT_UINT8, DT_QUINT8, DT_UINT32, DT_UINT64, DT_INT8, DT_QINT8, DT_INT32,
diff --git a/tensorflow/compiler/xla/client/lib/constants.h b/tensorflow/compiler/xla/client/lib/constants.h
index 81624614c1..a38282e8db 100644
--- a/tensorflow/compiler/xla/client/lib/constants.h
+++ b/tensorflow/compiler/xla/client/lib/constants.h
@@ -56,6 +56,8 @@ XlaOp ConstantR0WithType(XlaBuilder* builder, PrimitiveType type, T value) {
       return ConstantR0<double>(builder, static_cast<double>(value));
     case C64:
       return ConstantR0<complex64>(builder, static_cast<complex64>(value));
+    case C128:
+      return ConstantR0<complex128>(builder, static_cast<complex128>(value));
     case U8:
       return ConstantR0<uint8>(builder, static_cast<uint8>(value));
     case U32:
diff --git a/tensorflow/compiler/xla/client/lib/math.cc b/tensorflow/compiler/xla/client/lib/math.cc
index 36fdda39b4..3d0e3a2b93 100644
--- a/tensorflow/compiler/xla/client/lib/math.cc
+++ b/tensorflow/compiler/xla/client/lib/math.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/client/lib/math.h"
 
 #include "tensorflow/compiler/xla/client/lib/constants.h"
+#include "tensorflow/compiler/xla/primitive_util.h"
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/status_macros.h"
 
@@ -323,7 +324,8 @@ XlaOp MaybeConjugate(XlaOp x, bool conjugate) {
   XlaBuilder* builder = x.builder();
   return builder->ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
     TF_ASSIGN_OR_RETURN(Shape shape, builder->GetShape(x));
-    auto perform_conj = shape.element_type() == C64 && conjugate;
+    auto perform_conj =
+        primitive_util::IsComplexType(shape.element_type()) && conjugate;
     return perform_conj ? Conj(x) : x;
   });
 }
diff --git a/tensorflow/compiler/xla/literal.cc b/tensorflow/compiler/xla/literal.cc
index 81874fa405..8600e8752c 100644
--- a/tensorflow/compiler/xla/literal.cc
+++ b/tensorflow/compiler/xla/literal.cc
@@ -29,10 +29,12 @@ limitations under the License.
 #include "absl/strings/str_join.h"
 #include "absl/types/span.h"
 #include "tensorflow/compiler/xla/index_util.h"
+#include "tensorflow/compiler/xla/primitive_util.h"
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/status_macros.h"
 #include "tensorflow/compiler/xla/types.h"
 #include "tensorflow/compiler/xla/util.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/hash/hash.h"
 #include "tensorflow/core/platform/logging.h"
@@ -411,6 +413,7 @@ Status LiteralBase::Piece::CopyFrom(const LiteralBase::Piece& src) {
       COPY_ELEMENTS(F32, float);
       COPY_ELEMENTS(F64, double);
       COPY_ELEMENTS(C64, complex64);
+      COPY_ELEMENTS(C128, complex128);
       COPY_ELEMENTS(PRED, bool);
 #undef COPY_ELEMENTS
       default:
@@ -548,6 +551,9 @@ Status MutableLiteralBase::CopySliceFrom(const LiteralSlice& src_literal,
     case C64:
       return CopySliceFromInternal<complex64>(src_literal, src_base, dest_base,
                                               copy_size);
+    case C128:
+      return CopySliceFromInternal<complex128>(src_literal, src_base, dest_base,
+                                               copy_size);
     case PRED:
       return CopySliceFromInternal<bool>(src_literal, src_base, dest_base,
                                          copy_size);
@@ -766,6 +772,8 @@ Literal LiteralBase::Slice(absl::Span<const int64> start_indices,
       return SliceInternal<double>(result_shape, start_indices);
     case C64:
       return SliceInternal<complex64>(result_shape, start_indices);
+    case C128:
+      return SliceInternal<complex128>(result_shape, start_indices);
     default:
       LOG(FATAL) << "not yet implemented: "
                  << PrimitiveType_Name(result_shape.element_type());
@@ -814,6 +822,10 @@ string LiteralBase::GetAsString(absl::Span<const int64> multi_index,
       complex64 c = Get<complex64>(multi_index, shape_index);
       return StrCat("(", c.real(), ", ", c.imag(), ")");
     }
+    case C128: {
+      complex128 c = Get<complex128>(multi_index, shape_index);
+      return StrCat("(", c.real(), ", ", c.imag(), ")");
+    }
     default:
       LOG(FATAL) << PrimitiveType_Name(subshape.element_type());
   }
@@ -868,6 +880,11 @@ string LiteralBase::GetSparseElementAsString(
           GetSparseElement<complex64>(sparse_element_number, shape_index);
       return StrCat("(", c.real(), ", ", c.imag(), ")");
     }
+    case C128: {
+      complex128 c =
+          GetSparseElement<complex128>(sparse_element_number, shape_index);
+      return StrCat("(", c.real(), ", ", c.imag(), ")");
+    }
     default:
       LOG(FATAL) << "Invalid element type for sparse arrays: "
                  << PrimitiveType_Name(subshape.element_type());
@@ -996,6 +1013,9 @@ void LiteralBase::Piece::SortSparseElements() {
     case C64:
       SortSparseElementsInternal<complex64>();
       break;
+    case C128:
+      SortSparseElementsInternal<complex128>();
+      break;
     case F16:
       SortSparseElementsInternal<half>();
       break;
@@ -1230,7 +1250,24 @@ Literal ConvertBetweenNativeTypesWithConverter(const LiteralBase& src_literal,
 }
 
 template <typename NativeSrcT, typename NativeDestT>
-Literal ConvertBetweenNativeTypes(const LiteralBase& src_literal) {
+typename std::enable_if<(std::is_same<NativeSrcT, Eigen::half>::value) &&
+                            (std::is_same<NativeDestT, complex64>::value ||
+                             std::is_same<NativeDestT, complex128>::value),
+                        Literal>::type
+ConvertBetweenNativeTypes(const LiteralBase& src_literal) {
+  auto converter = [](NativeSrcT src) {
+    return NativeDestT(static_cast<typename NativeDestT::value_type>(src));
+  };
+  return ConvertBetweenNativeTypesWithConverter<NativeSrcT, NativeDestT>(
+      src_literal, converter);
+}
+
+template <typename NativeSrcT, typename NativeDestT>
+typename std::enable_if<(!std::is_same<NativeSrcT, Eigen::half>::value) ||
+                            (!std::is_same<NativeDestT, complex64>::value &&
+                             !std::is_same<NativeDestT, complex128>::value),
+                        Literal>::type
+ConvertBetweenNativeTypes(const LiteralBase& src_literal) {
   auto converter = [](NativeSrcT src) { return static_cast<NativeDestT>(src); };
   return ConvertBetweenNativeTypesWithConverter<NativeSrcT, NativeDestT>(
       src_literal, converter);
@@ -1274,22 +1311,6 @@ BitcastBetweenNativeTypes(const LiteralBase& src_literal) {
   LOG(FATAL) << "Invalid bitcast between types of different sizes.";
 }
 
-template <PrimitiveType primitive_src_type>
-Literal ConvertToC64(const LiteralBase& src_literal) {
-  CHECK(src_literal.shape().IsArray());
-  Literal result_literal(
-      ShapeUtil::ChangeElementType(src_literal.shape(), C64));
-  using NativeSrcT =
-      typename primitive_util::PrimitiveTypeToNative<primitive_src_type>::type;
-  absl::Span<const NativeSrcT> src_data = src_literal.data<NativeSrcT>();
-  absl::Span<complex64> dest_data = result_literal.data<complex64>();
-  int64 num_elements = src_literal.element_count();
-  for (int64 i = 0; i < num_elements; ++i) {
-    dest_data[i] = complex64(static_cast<float>(src_data[i]), 0);
-  }
-  return result_literal;
-}
-
 template <PrimitiveType primitive_src_type, PrimitiveType primitive_dest_type>
 Literal ConvertIfTypesMatch(const LiteralBase& src_literal, bool bitcast) {
   CHECK_EQ(primitive_src_type, src_literal.shape().element_type());
@@ -1332,10 +1353,15 @@ StatusOr<Literal> ConvertIfDestTypeMatches(const LiteralBase& src_literal,
     CONVERT_IF_TYPES_MATCH(BF16)
 #undef CONVERT_IF_TYPES_MATCH
     case C64:
-      if (!bitcast) {
-        return ConvertToC64<primitive_src_type>(src_literal);
+      if (bitcast) {
+        break;
       }
-      break;
+      return ConvertIfTypesMatch<primitive_src_type, C64>(src_literal, false);
+    case C128:
+      if (bitcast) {
+        break;
+      }
+      return ConvertIfTypesMatch<primitive_src_type, C128>(src_literal, false);
     // Other types are not yet supported.
     default:
       break;
@@ -1485,6 +1511,8 @@ bool LiteralBase::Piece::EqualElements(const LiteralBase::Piece& other) const {
       return EqualElementsInternal<bfloat16>(other, &multi_index);
     case C64:
       return EqualElementsInternal<complex64>(other, &multi_index);
+    case C128:
+      return EqualElementsInternal<complex128>(other, &multi_index);
     default:
       LOG(FATAL) << "Unimplemented: LiteralBase::Piece::EqualElements for type "
                  << PrimitiveType_Name(subshape().element_type());
@@ -1628,6 +1656,9 @@ bool LiteralBase::IsAllComplex(complex64 value) const {
     case C64:
       return AllElementsEqualValue<complex64>(root_piece().data<complex64>(),
                                               value);
+    case C128:
+      return AllElementsEqualValue<complex128>(root_piece().data<complex128>(),
+                                               value);
     default:
       return false;
   }
@@ -1707,6 +1738,11 @@ bool LiteralBase::IsAllFirst() const {
               auto data = piece.data<uint64>();
               return AllElementsEqualValue<uint64>(data, data[0]);
             }
+
+            case C128: {
+              auto data = piece.data<complex128>();
+              return AllElementsEqualValue<complex128>(data, data[0]);
+            }
             default:
               return false;
           }
@@ -1756,6 +1792,8 @@ bool LiteralBase::IsR1Iota() const {
         return Get<bfloat16>({idx}) == static_cast<bfloat16>(idx);
       case C64:
         return Get<complex64>({idx}) == complex64(idx, 0.0f);
+      case C128:
+        return Get<complex128>({idx}) == complex128(idx, 0.0f);
       case PRED:
         return Get<bool>({idx}) == idx;
       // token, opaque, tuple, etc. are all not iota.
@@ -1799,6 +1837,8 @@ bool LiteralBase::IsZero(absl::Span<const int64> indices) const {
       return Get<double>(indices) == 0.0;
     case C64:
       return Get<complex64>(indices) == complex64(0.0f, 0.0f);
+    case C128:
+      return Get<complex128>(indices) == complex128(0.0f, 0.0f);
     case F16:
       return Get<half>(indices) == static_cast<half>(0.0f);
     case BF16:
@@ -1886,6 +1926,12 @@ void LiteralBase::Piece::WriteToProto(LiteralProto* proto) const {
         proto->add_c64s(value.imag());
       }
       break;
+    case C128:
+      for (complex128 value : data<complex128>()) {
+        proto->add_c128s(value.real());
+        proto->add_c128s(value.imag());
+      }
+      break;
     case TUPLE:
     case TOKEN:
       // Nothing to do but assign the shape which is done above.
@@ -2018,7 +2064,17 @@ Status LiteralBase::Piece::CopyFromProto(const LiteralProto& proto) {
       for (int64 i = 0; i < complex_data.size(); ++i) {
         complex_data[i] = complex64{proto.c64s(i * 2), proto.c64s(i * 2 + 1)};
       }
-    } break;
+      break;
+    }
+    case C128: {
+      auto complex_data = data<complex128>();
+      TF_RET_CHECK(proto.c128s_size() == complex_data.size() * 2);
+      for (int64 i = 0; i < complex_data.size(); ++i) {
+        complex_data[i] =
+            complex128{proto.c128s(i * 2), proto.c128s(i * 2 + 1)};
+      }
+      break;
+    }
     case TUPLE:
       return InvalidArgument("Should not be called on tuple shapes: %s",
                              ShapeUtil::HumanString(subshape()));
diff --git a/tensorflow/compiler/xla/literal_comparison.cc b/tensorflow/compiler/xla/literal_comparison.cc
index 258bc966b1..69efa06d39 100644
--- a/tensorflow/compiler/xla/literal_comparison.cc
+++ b/tensorflow/compiler/xla/literal_comparison.cc
@@ -90,6 +90,12 @@ bool CompareEqual<complex64>(complex64 lhs, complex64 rhs,
   return CompareEqual<float>(lhs.real(), rhs.real(), multi_index) &&
          CompareEqual<float>(lhs.imag(), rhs.imag(), multi_index);
 }
+template <>
+bool CompareEqual<complex128>(complex128 lhs, complex128 rhs,
+                              absl::Span<const int64> multi_index) {
+  return CompareEqual<double>(lhs.real(), rhs.real(), multi_index) &&
+         CompareEqual<double>(lhs.imag(), rhs.imag(), multi_index);
+}
 
 template <typename NativeT, typename UnsignedT>
 Status MakeBitwiseErrorStatus(NativeT lhs, NativeT rhs,
@@ -143,6 +149,14 @@ Status MakeErrorStatus(complex64 lhs, complex64 rhs,
   }
   return MakeErrorStatus(lhs.imag(), rhs.imag(), multi_index);
 }
+template <>
+Status MakeErrorStatus(complex128 lhs, complex128 rhs,
+                       absl::Span<const int64> multi_index) {
+  if (!CompareEqual<double>(lhs.real(), rhs.real(), multi_index)) {
+    return MakeErrorStatus(lhs.real(), rhs.real(), multi_index);
+  }
+  return MakeErrorStatus(lhs.imag(), rhs.imag(), multi_index);
+}
 
 // A recursive function which iterates through every index of expected and
 // actual literal and compares their values elementwise. Returns true if all
@@ -197,13 +211,6 @@ bool NanMismatch(NativeT expected, NativeT actual, bool relaxed_nans) {
   }
 }
 
-template <>
-bool NanMismatch<complex64>(complex64 expected, complex64 actual,
-                            bool relaxed_nans) {
-  return NanMismatch<float>(expected.real(), actual.real(), relaxed_nans) ||
-         NanMismatch<float>(expected.imag(), actual.imag(), relaxed_nans);
-}
-
 template <>
 bool NanMismatch<half>(half expected, half actual, bool relaxed_nans) {
   return NanMismatch<float>(static_cast<float>(expected),
@@ -232,6 +239,11 @@ string FpValueToString<complex64>(complex64 value) {
   return absl::StrFormat("%8.4g + %8.4fi", value.real(), value.imag());
 }
 
+template <>
+string FpValueToString<complex128>(complex128 value) {
+  return absl::StrFormat("%8.4g + %8.4fi", value.real(), value.imag());
+}
+
 // Returns the absolute value of the given floating point value. This function
 // is used instead of std::abs directly in order to allow type-dependent
 // implementations for NearComparator.
@@ -434,7 +446,7 @@ class NearComparator {
     mismatches_.data<bool>()[linear_index] = true;
   }
 
-  // For complex64 types, we compare real and imaginary parts individually.
+  // For complex types, we compare real and imaginary parts individually.
   void CompareValues(complex64 expected, complex64 actual, int64 linear_index) {
     bool mismatch = false;
     CompareValues<float>(expected.real(), actual.real(), linear_index);
@@ -457,6 +469,29 @@ class NearComparator {
     mismatches_.data<bool>()[linear_index] = mismatch;
   }
 
+  void CompareValues(complex128 expected, complex128 actual,
+                     int64 linear_index) {
+    bool mismatch = false;
+    CompareValues<double>(expected.real(), actual.real(), linear_index);
+    if (mismatches_.data<bool>()[linear_index] == true) {
+      mismatch = true;
+      // Delay the mismatch count increase for real part, instead increase
+      // mismatch by 1 for the entire complex number.
+      num_mismatches_--;
+    }
+    CompareValues<double>(expected.imag(), actual.imag(), linear_index);
+    if (mismatches_.data<bool>()[linear_index] == true) {
+      mismatch = true;
+      // Delay the mismatch count increase for imag part, instead increase
+      // mismatch by 1 for the entire complex number.
+      num_mismatches_--;
+    }
+    if (mismatch == true) {
+      num_mismatches_++;
+    }
+    mismatches_.data<bool>()[linear_index] = mismatch;
+  }
+
   // Compares the two literals elementwise.
   void CompareLiterals() {
     // Fast path optimization for the case were layouts match.
@@ -665,6 +700,9 @@ Status EqualHelper(const LiteralSlice& expected, const LiteralSlice& actual) {
     case C64:
       result = Equal<complex64>(expected, actual, index, 0);
       break;
+    case C128:
+      result = Equal<complex128>(expected, actual, index, 0);
+      break;
     case TUPLE: {
       for (int i = 0; i < ShapeUtil::TupleElementCount(expected.shape()); ++i) {
         result.Update(EqualHelper(LiteralSlice(expected, {i}),
@@ -749,6 +787,10 @@ Status NearHelper(const LiteralSlice& expected, const LiteralSlice& actual,
         return NearComparator<complex64>::Compare(
             expected, actual, error, detailed_message, miscompare_callback);
         break;
+      case C128:
+        return NearComparator<complex128>::Compare(
+            expected, actual, error, detailed_message, miscompare_callback);
+        break;
       default:
         LOG(FATAL) << "Unsupported primitive type in near comparator: "
                    << PrimitiveType_Name(expected.shape().element_type())
diff --git a/tensorflow/compiler/xla/literal_test.cc b/tensorflow/compiler/xla/literal_test.cc
index a4088c6d11..e67bb5c32f 100644
--- a/tensorflow/compiler/xla/literal_test.cc
+++ b/tensorflow/compiler/xla/literal_test.cc
@@ -118,6 +118,9 @@ TEST_F(LiteralUtilTest, LiteralScalarToString) {
   auto c64_lit = LiteralUtil::CreateR0<complex64>({3.14f, 2.78f});
   EXPECT_EQ("c64[] (3.14, 2.78)", c64_lit.ToString());
 
+  auto c128_lit = LiteralUtil::CreateR0<complex128>({3.14f, 2.78f});
+  EXPECT_EQ("c128[] (3.14, 2.78)", c128_lit.ToString());
+
   auto bf16_lit = LiteralUtil::CreateR0<bfloat16>(static_cast<bfloat16>(0.5f));
   EXPECT_EQ("bf16[] 0.5", bf16_lit.ToString());
 
@@ -469,6 +472,21 @@ TEST_F(LiteralUtilTest, C64Equality) {
   EXPECT_NE(vector, vector_reversed);
 }
 
+TEST_F(LiteralUtilTest, C128Equality) {
+  // Test equality with tuples.
+  auto vector = LiteralUtil::CreateR1<complex128>({{1.0, 2.0}, {3.0, 4.0}});
+
+  // Tuple with the same elements. One element is shared with the original
+  // tuple, the other is a clone of the element in the original tuple.
+  auto vector_clone =
+      LiteralUtil::CreateR1<complex128>({{1.0, 2.0}, {3.0, 4.0}});
+  EXPECT_EQ(vector, vector_clone);
+
+  auto vector_reversed =
+      LiteralUtil::CreateR1<complex128>({{3.0, 4.0}, {1.0, 2.0}});
+  EXPECT_NE(vector, vector_reversed);
+}
+
 TEST_F(LiteralUtilTest, IsAllTuple) {
   auto element1 = LiteralUtil::CreateR0<float>(0.0);
   auto element2 = LiteralUtil::CreateR2<float>({{0.0, 0.0}, {0.0, 0.0}});
@@ -836,6 +854,13 @@ TEST_F(LiteralUtilTest, PopulateR1C64) {
   EXPECT_EQ(output, expected);
 }
 
+TEST_F(LiteralUtilTest, PopulateR1C128) {
+  Literal output(ShapeUtil::MakeShape(C128, {1}));
+  output.PopulateR1<complex128>({{77, 88}});
+  auto expected = LiteralUtil::CreateR1<complex128>({{77, 88}});
+  EXPECT_EQ(output, expected);
+}
+
 TEST_F(LiteralUtilTest, PopulateR2C64) {
   Literal output(ShapeUtil::MakeShape(C64, {2, 2}));
   output.PopulateR2<complex64>({{{7, 8}, {9, 10}}, {{1, 2}, {3, 4}}});
@@ -897,6 +922,14 @@ TEST_F(LiteralUtilTest, PopulateWithValueR2C64) {
   EXPECT_EQ(output, expected);
 }
 
+TEST_F(LiteralUtilTest, PopulateWithValueR2C128) {
+  Literal output(ShapeUtil::MakeShape(C128, {2, 2}));
+  output.PopulateWithValue<complex128>({4, 2});
+  auto expected =
+      LiteralUtil::CreateR2<complex128>({{{4, 2}, {4, 2}}, {{4, 2}, {4, 2}}});
+  EXPECT_EQ(output, expected);
+}
+
 TEST_F(LiteralUtilTest, PopulateWithValueR0F16) {
   Literal output(ShapeUtil::MakeShape(F16, {}));
   half h(0.25f);
@@ -1308,7 +1341,11 @@ TEST_F(LiteralUtilTest, ConvertIfTypesMatch) {
     {{0.0f, 19.0f, 0.0f, 21.0f}, {22.0f, 0.0f, 24.0f, 0.0f}},
     {{26.0f, 0.0f, 28.0f, 0.0f}, {0.0f, 31.0f, 0.0f, 33.0f}},
   }}, layout_r4_dim0major_);
-  // clang-format on
+  auto c128 = LiteralUtil::CreateR4WithLayout<complex128>({{
+    {{10.0, 0.0, 12.0, 0.0}, {0.0, 15.0, 0.0, 17.0}},
+    {{0.0, 19.0, 0.0, 21.0}, {22.0, 0.0, 24.0, 0.0}},
+    {{26.0, 0.0, 28.0, 0.0}, {0.0, 31.0, 0.0, 33.0}},
+  }}, layout_r4_dim0major_);  // clang-format on
   Literal conv;
 
   conv = s8.Convert(U16).ConsumeValueOrDie();
@@ -1374,10 +1411,20 @@ TEST_F(LiteralUtilTest, ConvertIfTypesMatch) {
   conv = s32.Convert(U16).ConsumeValueOrDie();
   EXPECT_EQ(conv, u16);
 
+  conv = s32.Convert(C128).ConsumeValueOrDie();
+  EXPECT_EQ(conv, c128);
+
+  conv = f16.Convert(C128).ConsumeValueOrDie();
+  EXPECT_EQ(conv, c128);
+
   EXPECT_EQ(s32.Convert(TUPLE).status().code(),
             tensorflow::error::UNIMPLEMENTED);
   EXPECT_EQ(c64.Convert(F32).status().code(), tensorflow::error::UNIMPLEMENTED);
   EXPECT_EQ(c64.Convert(S32).status().code(), tensorflow::error::UNIMPLEMENTED);
+  EXPECT_EQ(c128.Convert(F32).status().code(),
+            tensorflow::error::UNIMPLEMENTED);
+  EXPECT_EQ(c128.Convert(S32).status().code(),
+            tensorflow::error::UNIMPLEMENTED);
 }
 
 TEST_F(LiteralUtilTest, BitcastConvert) {
@@ -1739,7 +1786,8 @@ TEST_F(LiteralUtilTest, CreateFromShapeZeroInitialized) {
 
   Literal tuple = Literal::CreateFromShape(ShapeUtil::MakeTupleShape(
       {ShapeUtil::MakeShape(F64, {}), ShapeUtil::MakeShape(PRED, {2}),
-       ShapeUtil::MakeShape(U64, {2, 1}), ShapeUtil::MakeShape(C64, {})}));
+       ShapeUtil::MakeShape(U64, {2, 1}), ShapeUtil::MakeShape(C64, {}),
+       ShapeUtil::MakeShape(C128, {})}));
 
   EXPECT_EQ(tuple.Get<double>({}, {0}), 0.0);
   EXPECT_EQ(tuple.Get<bool>({0}, {1}), false);
@@ -1747,6 +1795,7 @@ TEST_F(LiteralUtilTest, CreateFromShapeZeroInitialized) {
   EXPECT_EQ(tuple.Get<uint64>({0, 0}, {2}), 0);
   EXPECT_EQ(tuple.Get<uint64>({1, 0}, {2}), 0);
   EXPECT_EQ(tuple.Get<complex64>({}, {3}), complex64(0.0f, 0.0f));
+  EXPECT_EQ(tuple.Get<complex128>({}, {4}), complex128(0.0, 0.0));
 }
 
 TEST_F(LiteralUtilTest, ProtoRoundTrip) {
@@ -1756,6 +1805,8 @@ TEST_F(LiteralUtilTest, ProtoRoundTrip) {
   auto vector_int8 = LiteralUtil::CreateR1<int8>({-128, 0, 2, 4, 7, 56, 127});
   auto vector_uint8 = LiteralUtil::CreateR1<uint8>({128, 0, 2, 56, 127, 255});
   auto vector_c64 = LiteralUtil::CreateR1<complex64>({{1.0, 2.0}, {3.0, 4.0}});
+  auto vector_c128 =
+      LiteralUtil::CreateR1<complex128>({{1.0, 2.0}, {3.0, 4.0}});
   auto vector_bfloat16 = LiteralUtil::CreateR1<bfloat16>(
       {bfloat16{-1.0}, bfloat16{2.0}, bfloat16{-3.0}});
   auto vector_half =
@@ -1776,6 +1827,7 @@ TEST_F(LiteralUtilTest, ProtoRoundTrip) {
   EXPECT_EQ(vector_int8, to_from_proto(vector_int8));
   EXPECT_EQ(vector_uint8, to_from_proto(vector_uint8));
   EXPECT_EQ(vector_c64, to_from_proto(vector_c64));
+  EXPECT_EQ(vector_c128, to_from_proto(vector_c128));
   EXPECT_EQ(vector_bfloat16, to_from_proto(vector_bfloat16));
   EXPECT_EQ(matrix_pred, to_from_proto(matrix_pred));
   EXPECT_EQ(tuple, to_from_proto(tuple));
diff --git a/tensorflow/compiler/xla/literal_util.cc b/tensorflow/compiler/xla/literal_util.cc
index 5881e43e96..26b029c8d0 100644
--- a/tensorflow/compiler/xla/literal_util.cc
+++ b/tensorflow/compiler/xla/literal_util.cc
@@ -130,6 +130,8 @@ Literal ConvertType(LiteralSlice literal) {
       return LiteralUtil::CreateR0<double>(0);
     case C64:
       return LiteralUtil::CreateR0<complex64>(0);
+    case C128:
+      return LiteralUtil::CreateR0<complex128>(0);
     case PRED:
       return LiteralUtil::CreateR0<bool>(false);
     case TUPLE:
@@ -165,6 +167,8 @@ Literal ConvertType(LiteralSlice literal) {
       return LiteralUtil::CreateR0<double>(1);
     case C64:
       return LiteralUtil::CreateR0<complex64>(1);
+    case C128:
+      return LiteralUtil::CreateR0<complex128>(1);
     case PRED:
       return LiteralUtil::CreateR0<bool>(true);
     case S16:
@@ -201,6 +205,8 @@ Literal ConvertType(LiteralSlice literal) {
           -std::numeric_limits<double>::infinity());
     case C64:
       LOG(FATAL) << "C64 element type has no minimum value";
+    case C128:
+      LOG(FATAL) << "C128 element type has no minimum value";
     case PRED:
       return LiteralUtil::CreateR0<bool>(false);
     case S16:
@@ -345,6 +351,10 @@ Literal ConvertType(LiteralSlice literal) {
         new_literal.Set<complex64>(to_multi_index,
                                    literal.Get<complex64>(from_multi_index));
         break;
+      case C128:
+        new_literal.Set<complex128>(to_multi_index,
+                                    literal.Get<complex128>(from_multi_index));
+        break;
       default:
         LOG(FATAL) << "Unhandled primitive element type: "
                    << PrimitiveType_Name(literal.shape().element_type());
@@ -393,6 +403,10 @@ Literal ConvertType(LiteralSlice literal) {
       return LiteralUtil::CreateR0<int64>(literal.GetFirstElement<int64>());
     case U64:
       return LiteralUtil::CreateR0<uint64>(literal.GetFirstElement<uint64>());
+
+    case C128:
+      return LiteralUtil::CreateR0<complex128>(
+          literal.GetFirstElement<complex128>());
     default:
       LOG(FATAL) << "Unhandled primitive type "
                  << literal.shape().element_type();
diff --git a/tensorflow/compiler/xla/primitive_util.cc b/tensorflow/compiler/xla/primitive_util.cc
index 00ad01fc40..04b40efbed 100644
--- a/tensorflow/compiler/xla/primitive_util.cc
+++ b/tensorflow/compiler/xla/primitive_util.cc
@@ -27,7 +27,7 @@ bool IsFloatingPointType(PrimitiveType type) {
   return type == F16 || type == F32 || type == F64 || type == BF16;
 }
 
-bool IsComplexType(PrimitiveType type) { return type == C64; }
+bool IsComplexType(PrimitiveType type) { return type == C64 || type == C128; }
 
 bool IsSignedIntegralType(PrimitiveType type) {
   return type == S8 || type == S16 || type == S32 || type == S64;
@@ -67,6 +67,9 @@ int BitWidth(PrimitiveType type) {
     case C64:
       return 64;
 
+    case C128:
+      return 128;
+
     case TUPLE:
       LOG(FATAL) << "TUPLE is an invalid type for BitWidth";
 
@@ -82,6 +85,8 @@ PrimitiveType ComplexComponentType(PrimitiveType complex_type) {
   switch (complex_type) {
     case C64:
       return F32;
+    case C128:
+      return F64;
     default:
       LOG(FATAL) << "Primitive type is not complex: "
                  << PrimitiveType_Name(complex_type);
diff --git a/tensorflow/compiler/xla/primitive_util.h b/tensorflow/compiler/xla/primitive_util.h
index 70603b6fed..739a7c409c 100644
--- a/tensorflow/compiler/xla/primitive_util.h
+++ b/tensorflow/compiler/xla/primitive_util.h
@@ -126,6 +126,11 @@ inline PrimitiveType NativeToPrimitiveType<complex64>() {
   return C64;
 }
 
+template <>
+inline PrimitiveType NativeToPrimitiveType<complex128>() {
+  return C128;
+}
+
 bool IsFloatingPointType(PrimitiveType type);
 
 bool IsComplexType(PrimitiveType type);
@@ -225,6 +230,11 @@ struct PrimitiveTypeToNative<C64> {
   using type = complex64;
 };
 
+template <>
+struct PrimitiveTypeToNative<C128> {
+  using type = complex128;
+};
+
 // Returns the lower-case name of the given primitive type.
 const string& LowercasePrimitiveTypeName(PrimitiveType s);
 
diff --git a/tensorflow/compiler/xla/python/numpy_bridge.cc b/tensorflow/compiler/xla/python/numpy_bridge.cc
index 9c90550825..52c5c621f7 100644
--- a/tensorflow/compiler/xla/python/numpy_bridge.cc
+++ b/tensorflow/compiler/xla/python/numpy_bridge.cc
@@ -54,6 +54,8 @@ int PrimitiveTypeToNumpyType(PrimitiveType primitive_type) {
       return NPY_FLOAT64;
     case C64:
       return NPY_COMPLEX64;
+    case C128:
+      return NPY_COMPLEX128;
     case TUPLE:
       return NPY_OBJECT;
     default:
@@ -89,6 +91,8 @@ PrimitiveType NumpyTypeToPrimitiveType(int np_type) {
       return F64;
     case NPY_COMPLEX64:
       return C64;
+    case NPY_COMPLEX128:
+      return C128;
     case NPY_OBJECT:
       return TUPLE;
     default:
@@ -111,6 +115,7 @@ bool NumpyTypeIsValid(int np_type) {
     case NPY_FLOAT32:
     case NPY_FLOAT64:
     case NPY_COMPLEX64:
+    case NPY_COMPLEX128:
     case NPY_OBJECT:
       return true;
     default:
@@ -430,6 +435,9 @@ Status CopyNumpyArrayToLiteral(int np_type, PyArrayObject* py_array,
     case NPY_COMPLEX64:
       CopyNumpyArrayToLiteral<complex64>(py_array, literal);
       break;
+    case NPY_COMPLEX128:
+      CopyNumpyArrayToLiteral<complex128>(py_array, literal);
+      break;
     default:
       return InvalidArgument(
           "No XLA literal container for Numpy type number: %d", np_type);
@@ -470,6 +478,9 @@ void CopyLiteralToNumpyArray(int np_type, const LiteralSlice& literal,
     case NPY_COMPLEX64:
       CopyLiteralToNumpyArray<complex64>(literal, py_array);
       break;
+    case NPY_COMPLEX128:
+      CopyLiteralToNumpyArray<complex128>(literal, py_array);
+      break;
     default:
       LOG(FATAL) << "No XLA literal container for Numpy type" << np_type;
   }
diff --git a/tensorflow/compiler/xla/python/xla_client.py b/tensorflow/compiler/xla/python/xla_client.py
index 4e71121c09..1684cb20e6 100644
--- a/tensorflow/compiler/xla/python/xla_client.py
+++ b/tensorflow/compiler/xla/python/xla_client.py
@@ -199,6 +199,7 @@ XLA_ELEMENT_TYPE_TO_DTYPE = {
     xla_data_pb2.F32: np.dtype('float32'),
     xla_data_pb2.F64: np.dtype('float64'),
     xla_data_pb2.C64: np.dtype('complex64'),
+    xla_data_pb2.C128: np.dtype('complex128'),
     xla_data_pb2.TUPLE: np.dtype(np.object),
 }
 
diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index ff14c500e3..b324a810ce 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -223,6 +223,7 @@ cc_library(
         "hlo_evaluator_typed_visitor.h",
         "hlo_evaluator_typed_visitor_bfloat16.cc",
         "hlo_evaluator_typed_visitor_bool.cc",
+        "hlo_evaluator_typed_visitor_complex128.cc",
         "hlo_evaluator_typed_visitor_complex64.cc",
         "hlo_evaluator_typed_visitor_double.cc",
         "hlo_evaluator_typed_visitor_float.cc",
@@ -259,6 +260,7 @@ cc_library(
         "@com_google_absl//absl/container:inlined_vector",
         "@com_google_absl//absl/container:node_hash_map",
         "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/meta:type_traits",
         "@com_google_absl//absl/strings",
         "@com_google_absl//absl/types:optional",
         "@com_google_absl//absl/types:span",
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
index 5ac746c9f3..4d6042b0d4 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
@@ -877,6 +877,9 @@ Status AlgebraicSimplifierVisitor::HandleDivide(HloInstruction* divide) {
       case C64:
         TF_RETURN_IF_ERROR(InvertConstant<complex64>(*b, &new_literal));
         break;
+      case C128:
+        TF_RETURN_IF_ERROR(InvertConstant<complex128>(*b, &new_literal));
+        break;
       default:
         return Status::OK();
     }
diff --git a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
index dba28aa51a..48510181bd 100644
--- a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
@@ -856,7 +856,8 @@ Status EmitNonBatchDotOperation(
     const HloModuleConfig& hlo_module_config,
     const TargetMachineFeatures& target_machine_features) {
   PrimitiveType type = target_array.GetShape().element_type();
-  TF_RET_CHECK(F16 == type || F32 == type || F64 == type || C64 == type);
+  TF_RET_CHECK(F16 == type || F32 == type || F64 == type || C64 == type ||
+               C128 == type);
   DotOpEmitter dot_emitter(std::move(dot_info), std::move(hlo_name),
                            target_array, lhs_array, rhs_array, addend_array,
                            executable_run_options_value, b, hlo_module_config,
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
index b27a232557..f8a997045a 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
@@ -239,10 +239,12 @@ Status IrEmitter::HandleCopy(HloInstruction* copy) {
 int IrEmitter::MinimumAlignmentForPrimitiveType(PrimitiveType primitive_type) {
   int64 byte_size = ShapeUtil::ByteSizeOfPrimitiveType(primitive_type);
   DCHECK_GE(byte_size, 0);
-  // Largest scalar is a complex64 so we don't need to worry about the
+  // Largest scalar is a complex128 so we don't need to worry about the
   // int64->int truncation here.
-  DCHECK_LE(byte_size, 8);
-  return byte_size;
+  DCHECK_LE(byte_size, 16);
+
+  // Allocations may be 8-byte aligned if part of a small block.
+  return std::min(8LL, byte_size);
 }
 
 int64 IrEmitter::ByteSizeOf(const Shape& shape) const {
@@ -942,7 +944,7 @@ Status IrEmitter::HandleDot(HloInstruction* dot) {
   auto rhs = dot->operand(1);
   TF_RETURN_IF_ERROR(ElementTypesSameAndSupported(
       /*instruction=*/*dot, /*operands=*/{lhs, rhs},
-      /*supported_types=*/{F16, F32, F64, C64}));
+      /*supported_types=*/{F16, F32, F64, C64, C128}));
   const DotDimensionNumbers& dnums = dot->dot_dimension_numbers();
 
   if (dnums.lhs_contracting_dimensions_size() != 1) {
@@ -1114,7 +1116,7 @@ Status IrEmitter::HandleConvolution(HloInstruction* convolution) {
   auto rhs = convolution->operand(1);
   TF_RETURN_IF_ERROR(ElementTypesSameAndSupported(
       /*instruction=*/*convolution, /*operands=*/{lhs, rhs},
-      /*supported_types=*/{F16, F32, C64}));
+      /*supported_types=*/{F16, F32, C64, C128}));
 
   // TODO(tonywy): Add PotentiallyImplementedAsMKLCovolution to support
   // different data layouts.
diff --git a/tensorflow/compiler/xla/service/gpu/gemm_thunk.cc b/tensorflow/compiler/xla/service/gpu/gemm_thunk.cc
index b8fbe7d2bc..86c9bc6a34 100644
--- a/tensorflow/compiler/xla/service/gpu/gemm_thunk.cc
+++ b/tensorflow/compiler/xla/service/gpu/gemm_thunk.cc
@@ -206,6 +206,8 @@ auto GetGemmFn(PrimitiveType type) -> decltype(&DoGemm<float>) {
       return &DoGemm<double>;
     case C64:
       return &DoGemm<std::complex<float>>;
+    case C128:
+      return &DoGemm<std::complex<double>>;
     default:
       LOG(FATAL) << "Unsupported type.";
   }
@@ -221,6 +223,8 @@ auto GetGemmWithAlgorithmFn(PrimitiveType type)
       return &DoGemmWithAlgorithm<double>;
     case C64:
       return &DoGemmWithAlgorithm<std::complex<float>>;
+    case C128:
+      return &DoGemmWithAlgorithm<std::complex<double>>;
     default:
       LOG(FATAL) << "Unsupported type.";
   }
@@ -235,6 +239,8 @@ auto GetGemmAutotuneFn(PrimitiveType type) -> decltype(&DoGemmAutotune<float>) {
       return &DoGemmAutotune<double>;
     case C64:
       return &DoGemmAutotune<std::complex<float>>;
+    case C128:
+      return &DoGemmAutotune<std::complex<double>>;
     default:
       LOG(FATAL) << "Unsupported type.";
   }
@@ -255,6 +261,8 @@ se::blas::ComputationType GetBlasComputationType(PrimitiveType type) {
       return se::blas::ComputationType::kF64;
     case C64:
       return se::blas::ComputationType::kComplexF32;
+    case C128:
+      return se::blas::ComputationType::kComplexF64;
     default:
       LOG(FATAL) << "Unsupported type.";
   }
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emission_utils.cc b/tensorflow/compiler/xla/service/gpu/ir_emission_utils.cc
index caccb18899..82bdd677d9 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emission_utils.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emission_utils.cc
@@ -54,7 +54,8 @@ bool AreValidGemmShapes(const Shape& lhs_shape, const Shape& rhs_shape,
   PrimitiveType output_primitive_type = output_shape.element_type();
   bool type_is_allowed =
       (output_primitive_type == F16 || output_primitive_type == F32 ||
-       output_primitive_type == F64 || output_primitive_type == C64);
+       output_primitive_type == F64 || output_primitive_type == C64 ||
+       output_primitive_type == C128);
   return type_is_allowed && IsRank2(lhs_shape, batch_dimensions_size) &&
          IsRank2(rhs_shape, batch_dimensions_size) &&
          IsRank2(output_shape, batch_dimensions_size) &&
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc
index 407615f842..927e751d38 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc
@@ -136,6 +136,37 @@ StatusOr<Literal> Compare<complex64>(const Shape& shape, HloOpcode opcode,
   return std::move(result);
 }
 
+template <>
+StatusOr<Literal> Compare<complex128>(const Shape& shape, HloOpcode opcode,
+                                      LiteralSlice lhs_literal,
+                                      LiteralSlice rhs_literal) {
+  std::function<bool(complex128, complex128)> compare_op;
+  switch (opcode) {
+    case HloOpcode::kEq:
+      compare_op = [](complex128 lhs_el, complex128 rhs_el) {
+        return lhs_el == rhs_el;
+      };
+      break;
+    case HloOpcode::kNe:
+      compare_op = [](complex128 lhs_el, complex128 rhs_el) {
+        return lhs_el != rhs_el;
+      };
+      break;
+    default:
+      LOG(FATAL) << "unhandled HLO opcode for conversion to Comparison: "
+                 << HloOpcodeString(opcode);
+  }
+
+  Literal result(shape);
+  TF_RETURN_IF_ERROR(
+      result.Populate<bool>([&](absl::Span<const int64> multi_index) {
+        return compare_op(lhs_literal.Get<complex128>(multi_index),
+                          rhs_literal.Get<complex128>(multi_index));
+      }));
+
+  return std::move(result);
+}
+
 }  // namespace
 
 // Note that unsupported types by the typed visitor does not necessarily imply
@@ -170,6 +201,8 @@ HloEvaluator::HloEvaluator(int64 max_loop_iterations)
       absl::make_unique<HloEvaluatorTypedVisitor<double>>(this);
   typed_visitors_[C64] =
       absl::make_unique<HloEvaluatorTypedVisitor<complex64>>(this);
+  typed_visitors_[C128] =
+      absl::make_unique<HloEvaluatorTypedVisitor<complex128>>(this);
 
   // Most of the evaluator computations we use don't support BF16 (e.g.,
   // std::ceil, std::tanh). To make evaluator work with BF16, we set all
@@ -500,6 +533,13 @@ Status HloEvaluator::HandleReal(HloInstruction* real) {
       TF_ASSIGN_OR_RETURN(evaluated_[real], std::move(result_or));
       break;
     }
+    case C128: {
+      auto result_or = ElementWiseUnaryOpImpl<float, complex128>(
+          real, [](complex128 elem_operand) { return std::real(elem_operand); },
+          GetEvaluatedLiteralFor(operand));
+      TF_ASSIGN_OR_RETURN(evaluated_[real], std::move(result_or));
+      break;
+    }
     case F16: {
       auto result_or = ElementWiseUnaryOpImpl<Eigen::half, Eigen::half>(
           real, [](Eigen::half elem_operand) { return elem_operand; },
@@ -530,11 +570,29 @@ Status HloEvaluator::HandleReal(HloInstruction* real) {
 }
 
 Status HloEvaluator::HandleImag(HloInstruction* imag) {
-  auto result_or = ElementWiseUnaryOpImpl<float, complex64>(
-      imag, [](complex64 elem_operand) { return std::imag(elem_operand); },
-      GetEvaluatedLiteralFor(imag->operand(0)));
+  auto operand = imag->operand(0);
+  switch (operand->shape().element_type()) {
+    case C64: {
+      auto result_or = ElementWiseUnaryOpImpl<float, complex64>(
+          imag, [](complex64 elem_operand) { return std::imag(elem_operand); },
+          GetEvaluatedLiteralFor(imag->operand(0)));
+
+      TF_ASSIGN_OR_RETURN(evaluated_[imag], std::move(result_or));
+      break;
+    }
+    case C128: {
+      auto result_or = ElementWiseUnaryOpImpl<double, complex128>(
+          imag, [](complex128 elem_operand) { return std::imag(elem_operand); },
+          GetEvaluatedLiteralFor(imag->operand(0)));
+
+      TF_ASSIGN_OR_RETURN(evaluated_[imag], std::move(result_or));
+      break;
+    }
+    default:
+      LOG(FATAL) << "HandleImag: unknown/unhandled primitive type: "
+                 << PrimitiveType_Name(operand->shape().element_type());
+  }
 
-  TF_ASSIGN_OR_RETURN(evaluated_[imag], std::move(result_or));
   return Status::OK();
 }
 
@@ -544,11 +602,27 @@ Status HloEvaluator::HandleComplex(HloInstruction* complex) {
   TF_RET_CHECK(ShapeUtil::Compatible(real.shape(), imag.shape()));
 
   Literal result(complex->shape());
-  TF_RETURN_IF_ERROR(
-      result.Populate<complex64>([&](absl::Span<const int64> multi_index) {
-        return std::complex<float>(real.Get<float>(multi_index),
-                                   imag.Get<float>(multi_index));
-      }));
+  switch (complex->shape().element_type()) {
+    case C64: {
+      TF_RETURN_IF_ERROR(
+          result.Populate<complex64>([&](absl::Span<const int64> multi_index) {
+            return std::complex<float>(real.Get<float>(multi_index),
+                                       imag.Get<float>(multi_index));
+          }));
+      break;
+    }
+    case C128: {
+      TF_RETURN_IF_ERROR(
+          result.Populate<complex128>([&](absl::Span<const int64> multi_index) {
+            return std::complex<float>(real.Get<double>(multi_index),
+                                       imag.Get<double>(multi_index));
+          }));
+      break;
+    }
+    default:
+      LOG(FATAL) << "HandleComplex: unknown/unhandled primitive type: "
+                 << PrimitiveType_Name(complex->shape().element_type());
+  }
 
   evaluated_[complex] = std::move(result);
   return Status::OK();
@@ -647,6 +721,11 @@ Status HloEvaluator::HandleCompare(HloInstruction* compare) {
                           Compare<complex64>(compare->shape(), opcode,
                                              lhs_literal, rhs_literal));
     } break;
+    case C128: {
+      TF_ASSIGN_OR_RETURN(evaluated_[compare],
+                          Compare<complex128>(compare->shape(), opcode,
+                                              lhs_literal, rhs_literal));
+    } break;
     default:
       LOG(FATAL) << "HandleCompare: unknown primitive type: "
                  << PrimitiveType_Name(lhs->shape().element_type());
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
index 698b177310..f3f400e46f 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
@@ -22,6 +22,7 @@ limitations under the License.
 #include "absl/base/casts.h"
 #include "absl/container/inlined_vector.h"
 #include "absl/memory/memory.h"
+#include "absl/meta/type_traits.h"
 #include "absl/types/optional.h"
 #include "tensorflow/compiler/xla/array2d.h"
 #include "tensorflow/compiler/xla/literal_util.h"
@@ -39,9 +40,8 @@ namespace xla {
 // Anyway this is relatively safe as-is because hlo_evaluator_typed_visitor.h is
 // a "private" header that's not exposed outside of hlo_evaluator.cc.
 template <typename T>
-using is_complex_t = std::is_same<T, complex64>;
-template <typename T>
-using is_complex64_t = std::is_same<T, complex64>;
+using is_complex_t =
+    absl::disjunction<std::is_same<T, complex64>, std::is_same<T, complex128>>;
 
 // It's UB to use std::sort with std::less<float>, because of NaNs. Define
 // "safe" less functions which are actually strict weak orders. -NaN and NaN
@@ -212,7 +212,7 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
 
   template <
       typename NativeT,
-      typename std::enable_if<is_complex64_t<NativeT>::value>::type* = nullptr>
+      typename std::enable_if<is_complex_t<NativeT>::value>::type* = nullptr>
   Status HandleAbs(HloInstruction* abs) {
     const Literal& operand_literal =
         parent_->GetEvaluatedLiteralFor(abs->operand(0));
@@ -231,6 +231,8 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
     // specifying the ElementwiseT explicitly as C64 is needed below.
     if (abs->operand(0)->shape().element_type() == C64) {
       return HandleAbs<complex64>(abs);
+    } else if (abs->operand(0)->shape().element_type() == C128) {
+      return HandleAbs<complex128>(abs);
     }
     return HandleAbs<ElementwiseT>(abs);
   }
@@ -1616,6 +1618,10 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
         TF_ASSIGN_OR_RETURN(parent_->evaluated_[map], MapImpl<complex64>(map));
         break;
       }
+      case C128: {
+        TF_ASSIGN_OR_RETURN(parent_->evaluated_[map], MapImpl<complex128>(map));
+        break;
+      }
       default:
         LOG(FATAL) << "HandleMap: unhandled primitive type for "
                       "input operand: "
@@ -3040,6 +3046,7 @@ extern template class HloEvaluatorTypedVisitor<Eigen::half, float>;
 extern template class HloEvaluatorTypedVisitor<float>;
 extern template class HloEvaluatorTypedVisitor<double>;
 extern template class HloEvaluatorTypedVisitor<complex64>;
+extern template class HloEvaluatorTypedVisitor<complex128>;
 extern template class HloEvaluatorTypedVisitor<bfloat16, float>;
 
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor_complex128.cc b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor_complex128.cc
new file mode 100644
index 0000000000..1f48140ee4
--- /dev/null
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor_complex128.cc
@@ -0,0 +1,22 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h"
+
+#include "tensorflow/compiler/xla/service/hlo_evaluator.h"
+
+namespace xla {
+template class HloEvaluatorTypedVisitor<complex128>;
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/hlo_parser_test.cc b/tensorflow/compiler/xla/service/hlo_parser_test.cc
index 76b8a5bc11..74ef8a2fba 100644
--- a/tensorflow/compiler/xla/service/hlo_parser_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_parser_test.cc
@@ -551,6 +551,17 @@ ENTRY %Transpose.v2 () -> s32[1,2,3] {
   ROOT %transpose = s32[1,2,3]{2,1,0} transpose(s32[1,2,3]{2,1,0} %constant), dimensions={0,1,2}
 }
 
+)"
+},
+{
+"TransposeC128",
+R"(HloModule TransposeC128_module
+
+ENTRY %Transpose.v3 (input: c128[1,2,3]) -> c128[1,2,3] {
+  %input = c128[1,2,3]{2,1,0} parameter(0)
+  ROOT %transpose = c128[1,2,3]{2,1,0} transpose(c128[1,2,3]{2,1,0} %input), dimensions={0,1,2}
+}
+
 )"
 },
 // Dynamic slice
diff --git a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc
index 83af5d2543..807296329c 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc
@@ -188,7 +188,16 @@ llvm::Type* PrimitiveTypeToIrType(PrimitiveType element_type,
       }
       return cplx_t;
     }
-    // A Tuple contains an array of pointers. Use i8*.
+    case C128: {
+      auto cplx_t = module->getTypeByName("complex128");
+      if (cplx_t == nullptr) {
+        return llvm::StructType::create(
+            {llvm::Type::getDoubleTy(module->getContext()),
+             llvm::Type::getDoubleTy(module->getContext())},
+            "complex128", /*isPacked=*/true);
+      }
+      return cplx_t;
+    }  // A Tuple contains an array of pointers. Use i8*.
     case TUPLE:
     // An Opaque is like a void*, use i8*.
     case OPAQUE:
diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc
index 1d3f84af95..c2e014ad42 100644
--- a/tensorflow/compiler/xla/service/shape_inference.cc
+++ b/tensorflow/compiler/xla/service/shape_inference.cc
@@ -906,6 +906,8 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
                                                         broadcast_dimensions));
       if (lhs.element_type() == F32 && rhs.element_type() == F32) {
         return ShapeUtil::ChangeElementType(shape, C64);
+      } else if (lhs.element_type() == F64 && rhs.element_type() == F64) {
+        return ShapeUtil::ChangeElementType(shape, C128);
       } else {
         return Unimplemented("Complex component type is not implemented.");
       }
@@ -1733,7 +1735,7 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
     case FFT:
     case IFFT:
       if (in.element_type() != C64) {
-        return InvalidArgument("%s requires C64 input type, found %s.",
+        return InvalidArgument("%s requires complex input type, found %s.",
                                FftType_Name(fft_type),
                                PrimitiveType_Name(in.element_type()));
       }
diff --git a/tensorflow/compiler/xla/service/shape_inference_test.cc b/tensorflow/compiler/xla/service/shape_inference_test.cc
index bb00bfd2ee..26120a06b8 100644
--- a/tensorflow/compiler/xla/service/shape_inference_test.cc
+++ b/tensorflow/compiler/xla/service/shape_inference_test.cc
@@ -35,6 +35,7 @@ class ShapeInferenceTest : public ::testing::Test {
  protected:
   // Some handy scalar shapes.
   const Shape s32_ = ShapeUtil::MakeShape(S32, {});
+  const Shape f16_ = ShapeUtil::MakeShape(F16, {});
   const Shape f32_ = ShapeUtil::MakeShape(F32, {});
   const Shape f64_ = ShapeUtil::MakeShape(F64, {});
   const Shape pred_ = ShapeUtil::MakeShape(PRED, {});
@@ -260,8 +261,8 @@ TEST_F(ShapeInferenceTest, Complex) {
   ASSERT_FALSE(complex_shape(pred_, pred_, {}).ok());
   // Component types must match.
   ASSERT_FALSE(complex_shape(f32_, f64_, {}).ok());
-  // Only F32->C64 supported.
-  ASSERT_FALSE(complex_shape(f64_, f64_, {}).ok());
+  // Only F32->C64 and F64->C128 supported.
+  ASSERT_FALSE(complex_shape(f16_, f16_, {}).ok());
   // Validate correct uses.
   Shape c64_32 = ShapeUtil::MakeShape(C64, {32});
   TF_ASSERT_OK_AND_ASSIGN(Shape result, complex_shape(f32_, f32_, {}));
@@ -285,6 +286,9 @@ TEST_F(ShapeInferenceTest, Complex) {
   ASSERT_TRUE(ShapeUtil::Equal(result, c64_32_64));
   TF_ASSERT_OK_AND_ASSIGN(result, complex_shape(matrix_32_64_, f32_, {}));
   ASSERT_TRUE(ShapeUtil::Equal(result, c64_32_64));
+
+  TF_ASSERT_OK_AND_ASSIGN(result, complex_shape(f64_, f64_, {}));
+  ASSERT_TRUE(ShapeUtil::Equal(result, ShapeUtil::MakeShape(C128, {})));
 }
 
 TEST_F(ShapeInferenceTest, VariadicOpTuplify) {
diff --git a/tensorflow/compiler/xla/shape_util.cc b/tensorflow/compiler/xla/shape_util.cc
index 235b065585..9821790cb3 100644
--- a/tensorflow/compiler/xla/shape_util.cc
+++ b/tensorflow/compiler/xla/shape_util.cc
@@ -378,6 +378,7 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
     case U32:
     case U64:
     case C64:
+    case C128:
     case TUPLE:
     case OPAQUE:
     case TOKEN:
@@ -639,6 +640,8 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
       return sizeof(double);
     case C64:
       return sizeof(complex64);
+    case C128:
+      return sizeof(complex128);
     case TOKEN:
       // Tokens require no space.
       return 0;
diff --git a/tensorflow/compiler/xla/tests/client_library_test_base.h b/tensorflow/compiler/xla/tests/client_library_test_base.h
index 65a23dd883..3f65ed7fce 100644
--- a/tensorflow/compiler/xla/tests/client_library_test_base.h
+++ b/tensorflow/compiler/xla/tests/client_library_test_base.h
@@ -431,7 +431,8 @@ void ClientLibraryTestBase::ComputeAndCompareR0(
                     std::is_same<NativeT, double>::value ||
                     std::is_same<NativeT, bfloat16>::value ||
                     std::is_same<NativeT, half>::value ||
-                    std::is_same<NativeT, complex64>::value,
+                    std::is_same<NativeT, complex64>::value ||
+                    std::is_same<NativeT, complex128>::value,
                 "Float or complex type required when specifying an ErrorSpec");
   Literal expected_literal = LiteralUtil::CreateR0<NativeT>(expected);
   ClientLibraryTestBase::ComputeAndCompareLiteral(builder, expected_literal,
@@ -455,7 +456,8 @@ void ClientLibraryTestBase::ComputeAndCompareR1(
                     std::is_same<NativeT, double>::value ||
                     std::is_same<NativeT, bfloat16>::value ||
                     std::is_same<NativeT, half>::value ||
-                    std::is_same<NativeT, complex64>::value,
+                    std::is_same<NativeT, complex64>::value ||
+                    std::is_same<NativeT, complex128>::value,
                 "Float or complex type required when specifying an ErrorSpec");
   Literal expected_literal = LiteralUtil::CreateR1<NativeT>(expected);
   ClientLibraryTestBase::ComputeAndCompareLiteral(builder, expected_literal,
@@ -480,7 +482,8 @@ void ClientLibraryTestBase::ComputeAndCompareR2(
                     std::is_same<NativeT, double>::value ||
                     std::is_same<NativeT, bfloat16>::value ||
                     std::is_same<NativeT, half>::value ||
-                    std::is_same<NativeT, complex64>::value,
+                    std::is_same<NativeT, complex64>::value ||
+                    std::is_same<NativeT, complex128>::value,
                 "Float or complex type required when specifying an ErrorSpec");
   Literal expected_literal =
       LiteralUtil::CreateR2FromArray2D<NativeT>(expected);
@@ -506,7 +509,8 @@ void ClientLibraryTestBase::ComputeAndCompareR3(
                     std::is_same<NativeT, double>::value ||
                     std::is_same<NativeT, bfloat16>::value ||
                     std::is_same<NativeT, half>::value ||
-                    std::is_same<NativeT, complex64>::value,
+                    std::is_same<NativeT, complex64>::value ||
+                    std::is_same<NativeT, complex128>::value,
                 "Float or complex type required when specifying an ErrorSpec");
   Literal expected_literal =
       LiteralUtil::CreateR3FromArray3D<NativeT>(expected);
@@ -532,7 +536,8 @@ void ClientLibraryTestBase::ComputeAndCompareR4(
                     std::is_same<NativeT, double>::value ||
                     std::is_same<NativeT, bfloat16>::value ||
                     std::is_same<NativeT, half>::value ||
-                    std::is_same<NativeT, complex64>::value,
+                    std::is_same<NativeT, complex64>::value ||
+                    std::is_same<NativeT, complex128>::value,
                 "Float or complex type required when specifying an ErrorSpec");
   Literal expected_literal =
       LiteralUtil::CreateR4FromArray4D<NativeT>(expected);
diff --git a/tensorflow/compiler/xla/types.h b/tensorflow/compiler/xla/types.h
index b645acb700..daf678f690 100644
--- a/tensorflow/compiler/xla/types.h
+++ b/tensorflow/compiler/xla/types.h
@@ -41,6 +41,7 @@ using ::tensorflow::uint32;
 using ::tensorflow::uint64;
 
 using complex64 = std::complex<float>;
+using complex128 = std::complex<double>;
 
 using ::Eigen::half;
 
diff --git a/tensorflow/compiler/xla/xla_data.proto b/tensorflow/compiler/xla/xla_data.proto
index d029669cbe..a64e2f5df5 100644
--- a/tensorflow/compiler/xla/xla_data.proto
+++ b/tensorflow/compiler/xla/xla_data.proto
@@ -56,6 +56,7 @@ enum PrimitiveType {
 
   // Complex values of fixed width.
   C64 = 15;  // Paired F32 (real, imag), as in std::complex<float>.
+  C128 = 18;  // Paired F64 (real, imag), as in std::complex<double>.
 
   // A tuple is a polymorphic sequence; e.g. a shape that holds different
   // sub-shapes. They are used for things like returning multiple values from a
@@ -75,7 +76,7 @@ enum PrimitiveType {
   // primitive type will have empty dimensions and tuple_shapes fields.
   TOKEN = 17;
 
-  // Next = 18
+  // Next = 19
 }
 
 // Describes the padding configuration for Pad operation. The padding amount on
@@ -367,6 +368,7 @@ message LiteralProto {
   repeated float f32s = 8;
   repeated double f64s = 9;
   repeated float c64s = 12;  // Stored as interleaved real, imag floats.
+  repeated double c128s = 18;  // Stored as interleaved real, imag doubles.
   repeated LiteralProto tuple_literals = 10;
   // The F16s, BF16s, U16s and S16s are encoded in little endian byte order
   bytes f16s = 11;
@@ -374,7 +376,7 @@ message LiteralProto {
   bytes u16s = 16;
   bytes s16s = 17;
   repeated int64 sparse_indices = 14;
-  // Next = 18
+  // Next = 19
 }
 
 message WindowDimension {
-- 
GitLab


From 5bc8f987097b9cccc63c3aa310595f719edd1344 Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Fri, 11 Jan 2019 13:18:50 -0800
Subject: [PATCH 0612/1095] [XLA:Python] Remove unused variable to silence
 warning.

PiperOrigin-RevId: 228933374
---
 tensorflow/compiler/xla/python/local_computation_builder.i | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tensorflow/compiler/xla/python/local_computation_builder.i b/tensorflow/compiler/xla/python/local_computation_builder.i
index 66ecee5c4d..39bff3f571 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.i
+++ b/tensorflow/compiler/xla/python/local_computation_builder.i
@@ -794,8 +794,6 @@ tensorflow::ImportNumpy();
   }
   dimension_numbers.set_kernel_input_feature_dimension(value);
 
-  PyObject* o;
-
   if (!HandleRepeatedInt64Attribute(
         $input, "input_spatial_dimensions",
         dimension_numbers.mutable_input_spatial_dimensions())) {
-- 
GitLab


From 655d3387a362b5c4b1070fa847824acb3b6ae644 Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Fri, 11 Jan 2019 13:24:09 -0800
Subject: [PATCH 0613/1095] Print disk usage after building stuff.

PiperOrigin-RevId: 228934277
---
 tensorflow/tools/ci_build/ci_parameterized_build.sh | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/tensorflow/tools/ci_build/ci_parameterized_build.sh b/tensorflow/tools/ci_build/ci_parameterized_build.sh
index 41b9f241d5..0631806dc5 100755
--- a/tensorflow/tools/ci_build/ci_parameterized_build.sh
+++ b/tensorflow/tools/ci_build/ci_parameterized_build.sh
@@ -613,6 +613,13 @@ if [[ "${DO_DOCKER}" == "1" ]]; then
   fi
 fi
 
+# Set a disk usage trap.
+function finish {
+    echo "Finished script... disk usage report in ${TMP_DIR}"
+    du -k -d 2 ${TMP_DIR} | sort -n -r
+}
+trap finish EXIT
+
 chmod +x ${TMP_SCRIPT}
 
 # Map TF_BUILD container types to containers we actually have.
-- 
GitLab


From a1515c0edfd45bccf942564dad1d302ca4d0d491 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 11 Jan 2019 13:29:56 -0800
Subject: [PATCH 0614/1095] Automated rollback of commit
 73b6d9262ea6d1578374ecb63560ad0fcad13e35

PiperOrigin-RevId: 228935205
---
 tensorflow/contrib/distribute/python/BUILD    |  30 +-
 ...test.py => keras_correctness_test_base.py} | 266 ++++--------------
 .../python/keras_dnn_correctness_test.py      | 163 +++++++++++
 .../keras_image_model_correctness_test.py     |  92 ++++++
 4 files changed, 342 insertions(+), 209 deletions(-)
 rename tensorflow/contrib/distribute/python/{keras_correctness_test.py => keras_correctness_test_base.py} (58%)
 create mode 100644 tensorflow/contrib/distribute/python/keras_dnn_correctness_test.py
 create mode 100644 tensorflow/contrib/distribute/python/keras_image_model_correctness_test.py

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index d4758d7518..4c25f49f16 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -657,7 +657,11 @@ cuda_py_test(
 py_library(
     name = "keras_correctness_test_lib",
     testonly = 1,
-    srcs = ["keras_correctness_test.py"],
+    srcs = [
+        "keras_correctness_test_base.py",
+        "keras_dnn_correctness_test.py",
+        "keras_image_model_correctness_test.py",
+    ],
     deps = [
         ":combinations",
         "//tensorflow/contrib/distribute/python:mirrored_strategy",
@@ -673,8 +677,9 @@ py_library(
 )
 
 cuda_py_test(
-    name = "keras_correctness_test",
-    srcs = ["keras_correctness_test.py"],
+    name = "keras_dnn_correctness_test",
+    size = "medium",
+    srcs = ["keras_dnn_correctness_test.py"],
     additional_deps = [
         ":keras_correctness_test_lib",
     ],
@@ -690,6 +695,25 @@ cuda_py_test(
     ],
 )
 
+cuda_py_test(
+    name = "keras_image_model_correctness_test",
+    size = "medium",
+    srcs = ["keras_image_model_correctness_test.py"],
+    additional_deps = [
+        ":keras_correctness_test_lib",
+    ],
+    # Shard count is set to an odd number to distribute tasks across
+    # shards more evenly.
+    shard_count = 31,
+    tags = [
+        "multi_and_single_gpu",
+        "no_oss",  # TODO(b/117919883): Fix python error.
+        "no_pip",
+        "no_windows_gpu",
+        "notsan",
+    ],
+)
+
 py_library(
     name = "metrics_v1_test_lib",
     testonly = 1,
diff --git a/tensorflow/contrib/distribute/python/keras_correctness_test.py b/tensorflow/contrib/distribute/python/keras_correctness_test_base.py
similarity index 58%
rename from tensorflow/contrib/distribute/python/keras_correctness_test.py
rename to tensorflow/contrib/distribute/python/keras_correctness_test_base.py
index 3abdee2c0e..9f3cab49a8 100644
--- a/tensorflow/contrib/distribute/python/keras_correctness_test.py
+++ b/tensorflow/contrib/distribute/python/keras_correctness_test_base.py
@@ -30,8 +30,6 @@ from tensorflow.python.eager import context
 from tensorflow.python.eager import test
 from tensorflow.python.framework import random_seed
 from tensorflow.python.keras.engine import distributed_training_utils
-from tensorflow.python.keras.optimizer_v2 import gradient_descent as gradient_descent_keras
-from tensorflow.python.training import gradient_descent
 
 _RANDOM_SEED = 1337
 _EVAL_STEPS = 20
@@ -62,24 +60,13 @@ def all_strategy_combinations_with_graph_mode():
   return combinations.combine(distribution=all_strategies, mode=['graph'])
 
 
-def strategy_and_input_combinations():
-  def cnn_model_with_batch_norm(**kwargs):
-    return _create_cnn_model(with_batch_norm=True, **kwargs)
-
+def all_strategy_and_input_config_combinations():
   return (
       combinations.times(
           combinations.combine(distribution=all_strategies),
           combinations.combine(mode=['graph', 'eager'],
                                use_numpy=[True, False],
-                               use_validation_data=[True, False]),
-          combinations.combine(model_with_data=[
-              ModelWithData('dnn', _create_dnn_model, _dnn_training_data),
-              ModelWithData('cnn', _create_cnn_model, _cnn_training_data),
-              ModelWithData('cnn_batch_norm',
-                            cnn_model_with_batch_norm,
-                            _cnn_training_data,
-                            with_batch_norm=True),
-          ])))
+                               use_validation_data=[True, False])))
 
 
 class MaybeDistributionScope(object):
@@ -100,97 +87,6 @@ class MaybeDistributionScope(object):
       self._scope = None
 
 
-class ModelWithData(object):
-  """An object giving a good name in combinations.
-
-  The model_fn must take two arguments: initial_weights and distribution.
-  """
-
-  def __init__(self, name, model_fn, data_fn, with_batch_norm=False):
-    self.name = name
-    self.model_fn = model_fn
-    self.data_fn = data_fn
-    self.with_batch_norm = with_batch_norm
-
-  def __repr__(self):
-    return self.name
-
-
-def _dnn_training_data():
-  # TODO(xiejw): Change this back to 10000, once we support final partial
-  # batch.
-  num_samples = 9984
-  x_train = np.random.rand(num_samples, 1)
-  y_train = 3 * x_train
-  x_train = x_train.astype('float32')
-  y_train = y_train.astype('float32')
-  x_predict = [[1.], [2.], [3.], [4.]]
-  return x_train, y_train, x_predict
-
-
-def _create_dnn_model(initial_weights=None, distribution=None):
-  with MaybeDistributionScope(distribution):
-    # We add few non-linear layers to make it non-trivial.
-    model = keras.Sequential()
-    model.add(keras.layers.Dense(10, activation='relu', input_shape=(1,)))
-    model.add(keras.layers.Dense(10, activation='relu'))
-    model.add(keras.layers.Dense(10, activation='relu'))
-    model.add(keras.layers.Dense(1))
-
-    if initial_weights:
-      model.set_weights(initial_weights)
-
-    model.compile(
-        loss=keras.losses.mean_squared_error,
-        optimizer=gradient_descent_keras.SGD(0.5),
-        metrics=['mse'])
-    return model
-
-
-def _cnn_training_data(count=_GLOBAL_BATCH_SIZE * _EVAL_STEPS,
-                       shape=(28, 28, 3), num_classes=10):
-  centers = np.random.randn(num_classes, *shape)
-
-  features = []
-  labels = []
-  for _ in range(count):
-    label = np.random.randint(0, num_classes, size=1)[0]
-    offset = np.random.normal(loc=0, scale=0.1, size=np.prod(shape))
-    offset = offset.reshape(shape)
-    labels.append(label)
-    features.append(centers[label] + offset)
-
-  x_train = np.asarray(features, dtype=np.float32)
-  y_train = np.asarray(labels, dtype=np.float32).reshape((count, 1))
-  x_predict = x_train
-  return x_train, y_train, x_predict
-
-
-def _create_cnn_model(initial_weights=None, distribution=None,
-                      with_batch_norm=False):
-  with MaybeDistributionScope(distribution):
-    image = keras.layers.Input(shape=(28, 28, 3), name='image')
-    c1 = keras.layers.Conv2D(
-        name='conv1', filters=16, kernel_size=(3, 3), strides=(4, 4))(
-            image)
-    if with_batch_norm:
-      c1 = keras.layers.BatchNormalization(name='bn1')(c1)
-    c1 = keras.layers.MaxPooling2D(pool_size=(2, 2))(c1)
-    logits = keras.layers.Dense(
-        10, activation='softmax', name='pred')(
-            keras.layers.Flatten()(c1))
-    model = keras.Model(inputs=[image], outputs=[logits])
-
-    if initial_weights:
-      model.set_weights(initial_weights)
-
-    model.compile(
-        optimizer=gradient_descent.GradientDescentOptimizer(learning_rate=0.1),
-        loss='sparse_categorical_crossentropy',
-        metrics=['sparse_categorical_accuracy'])
-  return model
-
-
 def batch_wrapper(dataset, batch_size, distribution, repeat=None):
   if repeat:
     dataset = dataset.repeat(repeat)
@@ -289,6 +185,7 @@ def get_correctness_test_inputs(use_numpy, use_validation_data,
 
 def fit_eval_and_predict(
     initial_weights, input_fn, model_fn, distribution=None):
+  """Generates results for fit/predict/evaluate for given model."""
   model = model_fn(initial_weights=initial_weights, distribution=distribution)
   training_inputs, eval_inputs, predict_inputs = input_fn(distribution)
 
@@ -317,6 +214,8 @@ def fit_eval_and_predict(
 
 def compare_results(results_with_ds, results_without_ds, distribution,
                     testcase):
+  """Compares results of model compiled with/without distribution strategy."""
+
   default_tolerance = 1e-5
   tol_table = {}
 
@@ -350,7 +249,13 @@ def compare_results(results_with_ds, results_without_ds, distribution,
         msg='Fail to assert {}.'.format(key))
 
 
+def should_skip_tpu_with_eager(distribution):
+  return (context.executing_eagerly() and
+          isinstance(distribution, tpu_strategy.TPUStrategy))
+
+
 class LearningRateBatchScheduler(keras.callbacks.Callback):
+  """Scheduler that dynamically sets the learning rate of model."""
 
   def __init__(self, update_freq=None):
     self._update_freq = update_freq
@@ -364,107 +269,60 @@ class LearningRateBatchScheduler(keras.callbacks.Callback):
     keras.backend.set_value(self.model.optimizer.lr, lr)
 
 
-class TestDistributionStrategyCorrectness(test.TestCase,
-                                          parameterized.TestCase):
+class TestDistributionStrategyCorrectnessBase(test.TestCase,
+                                              parameterized.TestCase):
+  """Model agnostic testing infra to test correctness of Keras models."""
 
-  def _should_skip_tpu_with_eager(self, distribution):
-    return (context.executing_eagerly() and
-            isinstance(distribution, tpu_strategy.TPUStrategy))
+  def set_up_test_config(self, use_numpy=False,
+                         use_validation_data=False,
+                         with_batch_norm=False):
+    self.use_numpy = use_numpy
+    self.use_validation_data = use_validation_data
+    self.with_batch_norm = with_batch_norm
 
-  @combinations.generate(all_strategy_combinations_with_eager_and_graph_modes())
-  def test_metric_correctness(self, distribution):
-    if self._should_skip_tpu_with_eager(distribution):
-      self.skipTest('TPUStrategy does not support eager mode now.')
+    keras.backend.set_image_data_format('channels_last')
+    np.random.seed(_RANDOM_SEED)
+    random_seed.set_random_seed(_RANDOM_SEED)
 
-    with self.cached_session():
-      keras.backend.set_image_data_format('channels_last')
-      num_samples = 10000
-
-      x_train = np.random.randint(0, 2, num_samples)
-      x_train = np.reshape(x_train, (num_samples, 1))
-      y_train = x_train
-      x_train = x_train.astype('float32')
-      y_train = y_train.astype('float32')
-
-      # Create identity model.
-      with distribution.scope():
-        model = keras.Sequential()
-        model.add(
-            keras.layers.Dense(1, input_shape=(1,), kernel_initializer='ones'))
-        model.compile(
-            loss=keras.losses.mean_squared_error,
-            optimizer=gradient_descent.GradientDescentOptimizer(0.5),
-            metrics=[keras.metrics.BinaryAccuracy()])
-
-      batch_size = 64
-      batch_size = get_batch_size(batch_size, distribution)
-      train_dataset = dataset_ops.Dataset.from_tensor_slices((x_train, y_train))
-      train_dataset = batch_wrapper(train_dataset, batch_size, distribution)
-
-      history = model.fit(x=train_dataset, epochs=2, steps_per_epoch=10)
-      self.assertEqual(history.history['binary_accuracy'], [1.0, 1.0])
-
-  @combinations.generate(all_strategy_combinations_with_eager_and_graph_modes())
-  def test_eval_metrics_correctness(self, distribution):
-    if self._should_skip_tpu_with_eager(distribution):
-      self.skipTest('TPUStrategy does not support eager mode now.')
+  def get_data(self):
+    num_samples = 10000
+    x_train = np.random.randint(0, 2, num_samples)
+    x_train = np.reshape(x_train, (num_samples, 1))
+    y_train = x_train
+    return (x_train.astype('float32'), y_train.astype('float32'), None)
 
-    with self.cached_session():
-      with distribution.scope():
-        model = keras.Sequential()
-        model.add(
-            keras.layers.Dense(
-                3, activation='relu', input_dim=4, kernel_initializer='ones'))
-        model.add(
-            keras.layers.Dense(
-                1, activation='sigmoid', kernel_initializer='ones'))
-        model.compile(
-            loss='mae',
-            metrics=['accuracy', keras.metrics.BinaryAccuracy()],
-            optimizer=gradient_descent.GradientDescentOptimizer(0.001))
-
-      # verify correctness of stateful and stateless metrics.
-      x = np.ones((100, 4)).astype('float32')
-      y = np.ones((100, 1)).astype('float32')
-      dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).repeat()
-      dataset = batch_wrapper(dataset, 4, distribution)
-      outs = model.evaluate(dataset, steps=10)
-      self.assertEqual(outs[1], 1.)
-      self.assertEqual(outs[2], 1.)
-
-      y = np.zeros((100, 1)).astype('float32')
-      dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).repeat()
-      dataset = batch_wrapper(dataset, 4, distribution)
-      outs = model.evaluate(dataset, steps=10)
-      self.assertEqual(outs[1], 0.)
-      self.assertEqual(outs[2], 0.)
-
-  @combinations.generate(strategy_and_input_combinations())
-  def test_correctness(self, distribution, use_numpy, use_validation_data,
-                       model_with_data):
-    if self._should_skip_tpu_with_eager(distribution):
+  def get_model(self, distribution=None):
+    raise NotImplementedError
+
+  def skip_unsupported_test_configuration(self, distribution):
+    if should_skip_tpu_with_eager(distribution):
       self.skipTest('TPUStrategy does not support eager mode now.')
 
-    if context.executing_eagerly() and use_numpy:
+    if context.executing_eagerly() and self.use_numpy:
       self.skipTest('Numpy as inputs is not supported with strategy in eager.')
 
-    if context.executing_eagerly() and use_validation_data:
-      self.skipTest('TODO')
+    if context.executing_eagerly() and self.use_validation_data:
+      self.skipTest('TODO(hongjunchoi): Add test logic for using validation '
+                    'data for eager execution.')
+    return
 
+  def run_correctness_test(self,
+                           distribution,
+                           use_numpy,
+                           use_validation_data,
+                           with_batch_norm=False):
     with self.cached_session():
-      keras.backend.set_image_data_format('channels_last')
-      np.random.seed(_RANDOM_SEED)
-      random_seed.set_random_seed(_RANDOM_SEED)
+      self.set_up_test_config(use_numpy, use_validation_data, with_batch_norm)
+      self.skip_unsupported_test_configuration(distribution)
 
-      model_fn, data_fn = model_with_data.model_fn, model_with_data.data_fn
       # Train, eval, and predict datasets are created with the same input numpy
       # arrays.
-      x_train, y_train, x_predict = data_fn()
+      x_train, y_train, x_predict = self.get_data()
 
       # The model is built once and the initial weights are saved.
       # This is used to initialize the model for both the distribution and
       # non-distribution run.
-      model = model_fn()
+      model = self.get_model()
       initial_weights = model.get_weights()
 
       def input_fn(dist):
@@ -472,15 +330,15 @@ class TestDistributionStrategyCorrectness(test.TestCase,
             use_numpy, use_validation_data, dist, x_train, y_train, x_predict)
 
       results_with_ds = fit_eval_and_predict(
-          initial_weights, input_fn=input_fn, model_fn=model_fn,
+          initial_weights, input_fn=input_fn, model_fn=self.get_model,
           distribution=distribution)
       results_without_ds = fit_eval_and_predict(
-          initial_weights, input_fn=input_fn, model_fn=model_fn,
+          initial_weights, input_fn=input_fn, model_fn=self.get_model,
           distribution=None)
 
       # First, special case, for multi-replica distributed training, batch norm
       # is not aggregated globally. So it is expected to have different weights.
-      if (model_with_data.with_batch_norm and
+      if (self.with_batch_norm and
           distribution.num_replicas_in_sync > 1):
         with self.assertRaises(AssertionError):
           compare_results(results_with_ds, results_without_ds, distribution,
@@ -489,21 +347,16 @@ class TestDistributionStrategyCorrectness(test.TestCase,
         compare_results(results_with_ds, results_without_ds, distribution,
                         testcase=self)
 
-  @combinations.generate(all_strategy_combinations_with_graph_mode())
-  def test_dynamic_lr(self, distribution):
-
+  def run_dynamic_lr_test(self, distribution):
     with self.cached_session():
+      self.set_up_test_config()
+      self.skip_unsupported_test_configuration(distribution)
 
-      keras.backend.set_image_data_format('channels_last')
-      np.random.seed(_RANDOM_SEED)
-      random_seed.set_random_seed(_RANDOM_SEED)
-
-      x_train, y_train, _ = _dnn_training_data()
-
-      model = _create_dnn_model()
+      x_train, y_train, _ = self.get_data()
+      model = self.get_model()
       initial_weights = model.get_weights()
-
       update_freq = None
+
       if (isinstance(distribution, tpu_strategy.TPUStrategy) and
           distribution.extended.steps_per_run > 1):
         # For TPUStrategy with steps_per_run > 1, the callback is not invoked
@@ -512,6 +365,7 @@ class TestDistributionStrategyCorrectness(test.TestCase,
         update_freq = distribution.extended.steps_per_run
 
       def input_fn(dist):
+        """Generates training test given test configuration."""
         training_epochs = 2
         global_batch_size = 64
         batch_size = get_batch_size(global_batch_size, dist)
@@ -530,10 +384,10 @@ class TestDistributionStrategyCorrectness(test.TestCase,
         return training_inputs, eval_inputs, predict_inputs
 
       results_with_ds = fit_eval_and_predict(
-          initial_weights, input_fn=input_fn, model_fn=_create_dnn_model,
+          initial_weights, input_fn=input_fn, model_fn=self.get_model,
           distribution=distribution)
       results_without_ds = fit_eval_and_predict(
-          initial_weights, input_fn=input_fn, model_fn=_create_dnn_model,
+          initial_weights, input_fn=input_fn, model_fn=self.get_model,
           distribution=None)
       compare_results(results_with_ds, results_without_ds, distribution,
                       testcase=self)
diff --git a/tensorflow/contrib/distribute/python/keras_dnn_correctness_test.py b/tensorflow/contrib/distribute/python/keras_dnn_correctness_test.py
new file mode 100644
index 0000000000..f21b1c6752
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/keras_dnn_correctness_test.py
@@ -0,0 +1,163 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Correctness tests for tf.keras DNN model using DistributionStrategy."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.distribute.python import combinations
+from tensorflow.contrib.distribute.python import keras_correctness_test_base
+from tensorflow.python import keras
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.eager import test
+from tensorflow.python.keras.optimizer_v2 import gradient_descent as gradient_descent_keras
+from tensorflow.python.training import gradient_descent
+
+
+class TestDistributionStrategyDnnCorrectness(
+    keras_correctness_test_base.TestDistributionStrategyCorrectnessBase):
+
+  def get_model(self, initial_weights=None, distribution=None):
+    with keras_correctness_test_base.MaybeDistributionScope(distribution):
+      # We add few non-linear layers to make it non-trivial.
+      model = keras.Sequential()
+      model.add(keras.layers.Dense(10, activation='relu', input_shape=(1,)))
+      model.add(keras.layers.Dense(10, activation='relu'))
+      model.add(keras.layers.Dense(10, activation='relu'))
+      model.add(keras.layers.Dense(1))
+
+      if initial_weights:
+        model.set_weights(initial_weights)
+
+      model.compile(
+          loss=keras.losses.mean_squared_error,
+          optimizer=gradient_descent_keras.SGD(0.5),
+          metrics=['mse'])
+      return model
+
+  def get_data(self):
+    # TODO(xiejw): Change this back to 10000, once we support final partial
+    # batch.
+    num_samples = 9984
+    x_train = np.random.rand(num_samples, 1)
+    y_train = 3 * x_train
+    x_train = x_train.astype('float32')
+    y_train = y_train.astype('float32')
+    x_predict = [[1.], [2.], [3.], [4.]]
+    return x_train, y_train, x_predict
+
+  @combinations.generate(keras_correctness_test_base.
+                         all_strategy_and_input_config_combinations())
+  def test_dnn_correctness(self, distribution, use_numpy, use_validation_data):
+    self.run_correctness_test(distribution, use_numpy, use_validation_data)
+
+  @combinations.generate(keras_correctness_test_base.
+                         all_strategy_combinations_with_graph_mode())
+  def test_dnn_with_dynamic_learning_rate(self, distribution):
+    self.run_dynamic_lr_test(distribution)
+
+
+class TestDistributionStrategyDnnMetricCorrectness(
+    keras_correctness_test_base.TestDistributionStrategyCorrectnessBase):
+
+  def get_model(self, distribution=None):
+    with distribution.scope():
+      model = keras.Sequential()
+      model.add(keras.layers.Dense(1,
+                                   input_shape=(1,),
+                                   kernel_initializer='ones'))
+      model.compile(
+          loss=keras.losses.mean_squared_error,
+          optimizer=gradient_descent.GradientDescentOptimizer(0.5),
+          metrics=[keras.metrics.BinaryAccuracy()])
+    return model
+
+  def run_metric_correctness_test(self, distribution):
+    with self.cached_session():
+      self.set_up_test_config()
+      self.skip_unsupported_test_configuration(distribution)
+
+      x_train, y_train, _ = self.get_data()
+      model = self.get_model(distribution=distribution)
+
+      batch_size = 64
+      batch_size = (keras_correctness_test_base.
+                    get_batch_size(batch_size, distribution))
+      train_dataset = dataset_ops.Dataset.from_tensor_slices((x_train, y_train))
+      train_dataset = (keras_correctness_test_base.
+                       batch_wrapper(train_dataset, batch_size, distribution))
+
+      history = model.fit(x=train_dataset, epochs=2, steps_per_epoch=10)
+      self.assertEqual(history.history['binary_accuracy'], [1.0, 1.0])
+
+  @combinations.generate(keras_correctness_test_base.
+                         all_strategy_combinations_with_eager_and_graph_modes())
+  def test_simple_dnn_metric_correctness(self, distribution):
+    self.run_metric_correctness_test(distribution)
+
+
+class TestDistributionStrategyDnnMetricEvalCorrectness(
+    keras_correctness_test_base.TestDistributionStrategyCorrectnessBase):
+
+  def get_model(self, distribution=None):
+    with distribution.scope():
+      model = keras.Sequential()
+      model.add(
+          keras.layers.Dense(
+              3, activation='relu', input_dim=4, kernel_initializer='ones'))
+      model.add(
+          keras.layers.Dense(
+              1, activation='sigmoid', kernel_initializer='ones'))
+      model.compile(
+          loss='mae',
+          metrics=['accuracy', keras.metrics.BinaryAccuracy()],
+          optimizer=gradient_descent.GradientDescentOptimizer(0.001))
+    return model
+
+  def run_eval_metrics_correctness_test(self, distribution):
+    with self.cached_session():
+      self.set_up_test_config()
+      self.skip_unsupported_test_configuration(distribution)
+
+      model = self.get_model(distribution=distribution)
+
+      # verify correctness of stateful and stateless metrics.
+      x = np.ones((100, 4)).astype('float32')
+      y = np.ones((100, 1)).astype('float32')
+      dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).repeat()
+      dataset = (keras_correctness_test_base.
+                 batch_wrapper(dataset, 4, distribution))
+      outs = model.evaluate(dataset, steps=10)
+      self.assertEqual(outs[1], 1.)
+      self.assertEqual(outs[2], 1.)
+
+      y = np.zeros((100, 1)).astype('float32')
+      dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).repeat()
+      dataset = (keras_correctness_test_base.
+                 batch_wrapper(dataset, 4, distribution))
+      outs = model.evaluate(dataset, steps=10)
+      self.assertEqual(outs[1], 0.)
+      self.assertEqual(outs[2], 0.)
+
+  @combinations.generate(keras_correctness_test_base.
+                         all_strategy_combinations_with_eager_and_graph_modes())
+  def test_identity_model_metric_eval_correctness(self, distribution):
+    self.run_eval_metrics_correctness_test(distribution)
+
+
+if __name__ == '__main__':
+  test.main()
diff --git a/tensorflow/contrib/distribute/python/keras_image_model_correctness_test.py b/tensorflow/contrib/distribute/python/keras_image_model_correctness_test.py
new file mode 100644
index 0000000000..f625664372
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/keras_image_model_correctness_test.py
@@ -0,0 +1,92 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Correctness tests for tf.keras CNN models using DistributionStrategy."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.distribute.python import combinations
+from tensorflow.contrib.distribute.python import keras_correctness_test_base
+from tensorflow.python import keras
+from tensorflow.python.eager import test
+from tensorflow.python.training import gradient_descent
+
+
+class DistributionStrategyCnnCorrectnessTest(
+    keras_correctness_test_base.TestDistributionStrategyCorrectnessBase):
+
+  def get_model(self, initial_weights=None, distribution=None):
+    with keras_correctness_test_base.MaybeDistributionScope(distribution):
+      image = keras.layers.Input(shape=(28, 28, 3), name='image')
+      c1 = keras.layers.Conv2D(
+          name='conv1', filters=16, kernel_size=(3, 3), strides=(4, 4))(
+              image)
+      if self.with_batch_norm:
+        c1 = keras.layers.BatchNormalization(name='bn1')(c1)
+      c1 = keras.layers.MaxPooling2D(pool_size=(2, 2))(c1)
+      logits = keras.layers.Dense(
+          10, activation='softmax', name='pred')(
+              keras.layers.Flatten()(c1))
+      model = keras.Model(inputs=[image], outputs=[logits])
+
+      if initial_weights:
+        model.set_weights(initial_weights)
+
+      model.compile(
+          optimizer=gradient_descent.GradientDescentOptimizer(
+              learning_rate=0.1),
+          loss='sparse_categorical_crossentropy',
+          metrics=['sparse_categorical_accuracy'])
+
+    return model
+
+  def get_data(self,
+               count=keras_correctness_test_base._GLOBAL_BATCH_SIZE
+               * keras_correctness_test_base._EVAL_STEPS,
+               shape=(28, 28, 3),
+               num_classes=10):
+    centers = np.random.randn(num_classes, *shape)
+
+    features = []
+    labels = []
+    for _ in range(count):
+      label = np.random.randint(0, num_classes, size=1)[0]
+      offset = np.random.normal(loc=0, scale=0.1, size=np.prod(shape))
+      offset = offset.reshape(shape)
+      labels.append(label)
+      features.append(centers[label] + offset)
+
+    x_train = np.asarray(features, dtype=np.float32)
+    y_train = np.asarray(labels, dtype=np.float32).reshape((count, 1))
+    x_predict = x_train
+    return x_train, y_train, x_predict
+
+  @combinations.generate(keras_correctness_test_base.
+                         all_strategy_and_input_config_combinations())
+  def test_cnn_correctness(self, distribution, use_numpy, use_validation_data):
+    self.run_correctness_test(distribution, use_numpy, use_validation_data)
+
+  @combinations.generate(keras_correctness_test_base.
+                         all_strategy_and_input_config_combinations())
+  def test_cnn_with_batch_norm_correctness(self, distribution, use_numpy,
+                                           use_validation_data):
+    self.run_correctness_test(distribution, use_numpy, use_validation_data,
+                              with_batch_norm=True)
+
+
+if __name__ == '__main__':
+  test.main()
-- 
GitLab


From b6e1f16f12ad9408fb8c983bc98bd62234ed8e39 Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Fri, 11 Jan 2019 13:47:13 -0800
Subject: [PATCH 0615/1095] Constant-fold GetDimensionSize ops in XLABuilder.

This CL implements constant-folding for GetDimensionSize ops if there is no dynamic dimension.

- If the operand of GetDimensionSize is constant dimension, consider GetDimensionSize as constant.

- Replace GetDimensionSize with a constant instruction in the constant subgraph for constant folding.

PiperOrigin-RevId: 228938348
---
 tensorflow/compiler/xla/client/xla_builder.cc | 61 ++++++++++++++++---
 .../xla/tests/compute_constant_test.cc        | 19 +++---
 2 files changed, 62 insertions(+), 18 deletions(-)

diff --git a/tensorflow/compiler/xla/client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_builder.cc
index 59e156eb72..c98f5b1140 100644
--- a/tensorflow/compiler/xla/client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_builder.cc
@@ -211,11 +211,21 @@ void XlaBuilder::IsConstantVisitor(const int64 op_handle,
       }
       // TODO(b/32495713): We aren't checking the called computations.
       break;
+    case HloOpcode::kGetDimensionSize: {
+      int64 dimension_number = instr.dimensions(0);
+      const HloInstructionProto& operand =
+          *(LookUpInstructionByHandle(instr.operand_ids(0)).ValueOrDie());
+      Shape operand_shape(operand.shape());
+      if (operand_shape.is_dynamic_dimension(dimension_number)) {
+        *is_constant = false;
+      }
+      break;
+    }
 
     // Non functional ops.
     case HloOpcode::kRng:
     case HloOpcode::kAllReduce:
-      // TODO(b/33009255): Implmement constant folding for cross replica sum.
+      // TODO(b/33009255): Implement constant folding for cross replica sum.
     case HloOpcode::kInfeed:
     case HloOpcode::kOutfeed:
     case HloOpcode::kCall:
@@ -2541,21 +2551,56 @@ StatusOr<XlaComputation> XlaBuilder::BuildConstantSubGraph(
     worklist.pop();
     TF_ASSIGN_OR_RETURN(const HloInstructionProto* instr_proto,
                         LookUpInstructionByHandle(handle));
-    for (int64 id : instr_proto->operand_ids()) {
-      if (related_ops.insert(id).second) {
-        worklist.push(id);
+
+    if (instr_proto->opcode() ==
+        HloOpcodeString(HloOpcode::kGetDimensionSize)) {
+      // At this point, BuildConstantSubGraph should never encounter a
+      // GetDimensionSize with a dynamic dimension. IsConstant check would have
+      // failed at the beginning of this function.
+      //
+      // Replace GetDimensionSize with a Constant representing the static bound
+      // of the shape.
+      int64 dimension = instr_proto->dimensions(0);
+      int64 operand_handle = instr_proto->operand_ids(0);
+      TF_ASSIGN_OR_RETURN(const HloInstructionProto* operand_proto,
+                          LookUpInstructionByHandle(operand_handle));
+
+      TF_RET_CHECK(!operand_proto->shape().is_dynamic_dimension(dimension));
+      auto constant_dimension_size =
+          static_cast<uint32>(operand_proto->shape().dimensions(dimension));
+
+      Literal literal = LiteralUtil::CreateR0(constant_dimension_size);
+
+      HloInstructionProto const_instr;
+      *const_instr.mutable_shape() = literal.shape().ToProto();
+      *const_instr.mutable_literal() = literal.ToProto();
+      *const_instr.mutable_opcode() = HloOpcodeString(HloOpcode::kConstant);
+      *const_instr.mutable_name() = HloOpcodeString(HloOpcode::kConstant);
+      const_instr.set_id(handle);
+      *entry.add_instructions() =
+          const_instr;  // Add to the result constant graph.
+    } else {
+      for (int64 id : instr_proto->operand_ids()) {
+        if (related_ops.insert(id).second) {
+          worklist.push(id);
+        }
+      }
+      for (int64 called_id : instr_proto->called_computation_ids()) {
+        related_calls.insert(called_id);
       }
-    }
-    for (int64 called_id : instr_proto->called_computation_ids()) {
-      related_calls.insert(called_id);
     }
   }
 
   // Add related ops to the computation.
   for (int64 id : related_ops) {
-    auto* instr = entry.add_instructions();
     TF_ASSIGN_OR_RETURN(const HloInstructionProto* instr_src,
                         LookUpInstructionByHandle(id));
+
+    if (instr_src->opcode() == HloOpcodeString(HloOpcode::kGetDimensionSize)) {
+      continue;
+    }
+    auto* instr = entry.add_instructions();
+
     *instr = *instr_src;
     // Ensures that the instruction names are unique among the graph.
     const string& new_name =
diff --git a/tensorflow/compiler/xla/tests/compute_constant_test.cc b/tensorflow/compiler/xla/tests/compute_constant_test.cc
index 3b0414a604..960f496977 100644
--- a/tensorflow/compiler/xla/tests/compute_constant_test.cc
+++ b/tensorflow/compiler/xla/tests/compute_constant_test.cc
@@ -151,19 +151,18 @@ TEST_F(ComputeConstantTest, DirectParamMissing) {
   }
 }
 
-TEST_F(ComputeConstantTest, IndirectParamMissing) {
+TEST_F(ComputeConstantTest, GetDimensionSize) {
   for (ClientType client_type : client_types) {
     Client* client = ClientOrDie(platform_, client_type);
     XlaBuilder b(TestName());
-    auto computation =
-        Add(ConstantR0<float>(&b, 1.0f),
-            Parameter(&b, 0, ShapeUtil::MakeShape(F32, {}), "param"));
-    EXPECT_FALSE(IsConstant(computation, &b));
-
-    auto value = ComputeConstantScalar<float>(client, computation, &b);
-    EXPECT_TRUE(
-        absl::StrContains(value.status().ToString(), "depends on a parameter"))
-        << value.status();
+    auto add =
+        Add(ConstantR1<float>(&b, {1.0f}), ConstantR1<float>(&b, {1.0f}));
+    auto get_dimension_size = GetDimensionSize(add, 0);
+    EXPECT_TRUE(IsConstant(get_dimension_size, &b));
+
+    TF_ASSERT_OK_AND_ASSIGN(auto value, ComputeConstantScalar<uint32>(
+                                            client, get_dimension_size, &b));
+    EXPECT_EQ(value, 1);
   }
 }
 
-- 
GitLab


From b96c6716234797aadaec84d1952c309c76cbba8a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 11 Jan 2019 13:53:33 -0800
Subject: [PATCH 0616/1095] Auto-detect number of hosts, cores, etc in
 mid-level TPU embedding API.

PiperOrigin-RevId: 228939416
---
 tensorflow/contrib/tpu/BUILD                  |  3 +-
 .../contrib/tpu/python/tpu/tpu_embedding.py   | 44 +++++++------------
 2 files changed, 18 insertions(+), 29 deletions(-)

diff --git a/tensorflow/contrib/tpu/BUILD b/tensorflow/contrib/tpu/BUILD
index b9c225b44a..0fe9d0cdfc 100644
--- a/tensorflow/contrib/tpu/BUILD
+++ b/tensorflow/contrib/tpu/BUILD
@@ -453,7 +453,8 @@ py_library(
     srcs = ["python/tpu/tpu_embedding.py"],
     srcs_version = "PY2AND3",
     deps = [
-        "//tensorflow/contrib/tpu:tpu_ops",
+        ":tpu_lib",
+        ":tpu_ops",
         "//tensorflow/contrib/tpu/proto:tpu_embedding_configuration_proto_py",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:framework_for_generated_wrappers",
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_embedding.py b/tensorflow/contrib/tpu/python/tpu/tpu_embedding.py
index ccba8a46c7..1a909a3ac6 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_embedding.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_embedding.py
@@ -28,6 +28,7 @@ from tensorflow.contrib.framework.python.framework import experimental
 from tensorflow.contrib.tpu.ops import gen_tpu_ops
 from tensorflow.contrib.tpu.proto import tpu_embedding_configuration_pb2 as elc
 from tensorflow.contrib.tpu.python.ops import tpu_ops
+from tensorflow.contrib.tpu.python.tpu import tpu_system_metadata as tpu_system_metadata_lib
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
@@ -43,19 +44,6 @@ from tensorflow.python.ops import variables
 TRAINING = elc.TPUEmbeddingConfiguration.TRAINING
 INFERENCE = elc.TPUEmbeddingConfiguration.INFERENCE
 
-# TODO(shizhiw): A better interface is to make `num_hosts` and
-# `num_cores_per_host` optional parameters for `TPUEmbedding`
-# constructor. Usually they can be automatically detected, but
-# user can also specify them for debugging (b/112112496).
-# Auto-detection can be done with `tpu_system_metadata.py`.
-_MASTER_JOB = 'tpu_worker'
-_HOST_PATTERN = '/job:tpu_worker/task:{}/device:CPU:0'
-_NUM_CORES_PER_HOST = 8
-
-_TEST_MASTER_JOB = None
-_TEST_HOST = '/replica:0/task:0/device:CPU:0'
-_TEST_NUM_CORES_PER_HOST = 2
-
 
 class TableConfig(
     collections.namedtuple(
@@ -301,10 +289,9 @@ class TPUEmbedding(object):
                table_to_config_dict,
                feature_to_table_dict,
                batch_size,
-               num_hosts,
                mode,
-               optimization_parameters=None,
-               tpu_embedding_test=False):
+               master,
+               optimization_parameters=None):
     """API for using TPU for embedding lookups.
 
     Args:
@@ -315,12 +302,11 @@ class TPUEmbedding(object):
         to string of table name. Feature refers to ids to lookup in embedding
         table, e.g. `sp_ids` argument to `tf.nn.embedding_lookup_sparse()`.
       batch_size: An `int` representing the global batch size.
-      num_hosts: An `int` representing the number of TPU hosts.
       mode: `TRAINING` or `INFERENCE`.
+      master: A `string` representing the TensorFlow master to use.
       optimization_parameters: `AdagradParameters`, `AdamParameters`,
         `Stochasticgradientdescentparameters`. Must be set in training and must
         be `None` in inference.
-      tpu_embedding_test: A `bool`. Only used for testing.
 
     Raises:
       ValueError: if any input is invalid.
@@ -337,15 +323,17 @@ class TPUEmbedding(object):
 
     self._batch_size = batch_size
 
-    if tpu_embedding_test:
-      self._num_hosts = 1
-      self._hosts = [_TEST_HOST]
-      self._num_cores_per_host = _TEST_NUM_CORES_PER_HOST
-    else:
-      self._num_hosts = num_hosts
-      self._hosts = [_HOST_PATTERN.format(i) for i in range(self._num_hosts)]
-      self._num_cores_per_host = _NUM_CORES_PER_HOST
-    self._num_cores = self._num_cores_per_host * self._num_hosts
+    self._master = master
+    self._tpu_system_metadata = (
+        tpu_system_metadata_lib._query_tpu_system_metadata(self._master))  # pylint: disable=protected-access
+    if self._tpu_system_metadata.num_cores == 0:
+      raise ValueError('TPUEmbedding needs TPUs, but master {} does not have '
+                       'TPUs.'.format(self._master))
+    self._num_hosts = self._tpu_system_metadata.num_hosts
+    self._hosts = [device.name for device in self._tpu_system_metadata.devices
+                   if 'device:CPU:' in device.name]
+    self._num_cores_per_host = self._tpu_system_metadata.num_of_cores_per_host
+    self._num_cores = self._tpu_system_metadata.num_cores
 
     _validate_batch_size(self._batch_size, self._num_cores)
     self._batch_size_per_core = self._batch_size // self._num_cores
@@ -389,7 +377,7 @@ class TPUEmbedding(object):
     Returns:
       A list of device names for CPU hosts.
     """
-    return self._hosts
+    return copy.copy(self._hosts)
 
   # TODO(shizhiw): change to num_tensor_cores_per_host to be more explicit and
   # to be consistent with `tpu_embedding_configuration.proto`.
-- 
GitLab


From 7ae5dd13e3c0c92db1f866bdbe63590a431ff3f4 Mon Sep 17 00:00:00 2001
From: Penporn Koanantakool <penporn@google.com>
Date: Fri, 11 Jan 2019 13:54:29 -0800
Subject: [PATCH 0617/1095] Fix --config=mkl performance regression: Prevent
 grappler from generating a _FusedConv2D node that doesn't have a
 corresponding TF-MKL node. For example, MKL graph rewriter doesn't support
 _FusedConv2D with BatchNorm yet so any Conv2D that gets fused with BatchNorm
 will miss using MKL-DNN's convolutions.

PiperOrigin-RevId: 228939593
---
 tensorflow/core/grappler/optimizers/BUILD       | 2 +-
 tensorflow/core/grappler/optimizers/remapper.cc | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index 7e29cee86a..a67d6ac5ea 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -683,7 +683,7 @@ tf_cc_test(
     ],
 )
 
-cc_library(
+tf_kernel_library(
     name = "remapper",
     srcs = ["remapper.cc"],
     hdrs = [
diff --git a/tensorflow/core/grappler/optimizers/remapper.cc b/tensorflow/core/grappler/optimizers/remapper.cc
index 0869e3b49b..193772fcda 100644
--- a/tensorflow/core/grappler/optimizers/remapper.cc
+++ b/tensorflow/core/grappler/optimizers/remapper.cc
@@ -817,6 +817,9 @@ Status Remapper::Optimize(Cluster* /*cluster*/, const GrapplerItem& item,
       continue;
     }
 
+// TODO(penporn):
+// Remove this once TF-MKL supports _FusedConv2D with these operations.
+#ifndef INTEL_MKL
     // Remap Conv2D+Squeeze+BiasAdd into the _FusedConv2D+Squeeze.
     if (FindConv2DWithSqueezeAndBias(ctx, &node,
                                      &conv2d_with_squeeze_and_bias)) {
@@ -839,6 +842,7 @@ Status Remapper::Optimize(Cluster* /*cluster*/, const GrapplerItem& item,
                          &invalidated_nodes);
       continue;
     }
+#endif  // !INTEL_MKL
 
     // Infer properties lazily in case they are not needed.
     if (!ctx.inferred_graph_properties && IsFusedBatchNormCandidate(node)) {
-- 
GitLab


From 03f9fc0b5e65b5979720bdbd4c466c319afd0597 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 11 Jan 2019 14:08:45 -0800
Subject: [PATCH 0618/1095] Add an explicit dependency on :pooling_ops to
 tensorflow/core:all_kernels_impl.

PiperOrigin-RevId: 228942375
---
 tensorflow/core/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 20f5815423..ee9ceeae89 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -1409,6 +1409,7 @@ cc_library(
         "//tensorflow/core/kernels:parameterized_truncated_normal_op",
         "//tensorflow/core/kernels:parsing",
         "//tensorflow/core/kernels:partitioned_function_ops",
+        "//tensorflow/core/kernels:pooling_ops",
         "//tensorflow/core/kernels:ragged_ops",
         "//tensorflow/core/kernels:random_ops",
         "//tensorflow/core/kernels:random_poisson_op",
-- 
GitLab


From d745e4f0a96dd782f8a2e3ca0f7fd846901d9764 Mon Sep 17 00:00:00 2001
From: Katherine Wu <kathywu@google.com>
Date: Fri, 11 Jan 2019 14:22:50 -0800
Subject: [PATCH 0619/1095] Add recall and precision at K to metrics.

This change adds the options to set top_k and class_id to the Recall and Precision metric classes. The top_k is computed for dense labels, which differs from tf.metrics.recall_at_k, which expects sparse labels.

PiperOrigin-RevId: 228944786
---
 tensorflow/python/keras/metrics.py            | 152 +++++++++++----
 .../keras/metrics_confusion_matrix_test.py    | 178 +++++++++++++++++-
 .../python/keras/utils/metrics_utils.py       |  51 ++++-
 .../tensorflow.keras.metrics.-precision.pbtxt |   2 +-
 .../v1/tensorflow.keras.metrics.-recall.pbtxt |   2 +-
 .../tensorflow.keras.metrics.-precision.pbtxt |   2 +-
 .../v2/tensorflow.keras.metrics.-recall.pbtxt |   2 +-
 7 files changed, 340 insertions(+), 49 deletions(-)

diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index a13d8747d6..c04487f175 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -52,6 +52,7 @@ from tensorflow.python.keras.utils.generic_utils import to_list
 from tensorflow.python.keras.utils.losses_utils import squeeze_or_expand_dimensions
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import confusion_matrix
+from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn
@@ -792,8 +793,11 @@ class _ConfusionMatrixConditionCount(Metric):
       Update op.
     """
     return metrics_utils.update_confusion_matrix_variables(
-        {self._confusion_matrix_cond: self.accumulator}, y_true, y_pred,
-        self.thresholds, sample_weight)
+        {self._confusion_matrix_cond: self.accumulator},
+        y_true,
+        y_pred,
+        thresholds=self.thresholds,
+        sample_weight=sample_weight)
 
   def result(self):
     if len(self.thresholds) == 1:
@@ -1029,6 +1033,15 @@ class Precision(Metric):
   If `sample_weight` is `None`, weights default to 1.
   Use `sample_weight` of 0 to mask values.
 
+  If `top_k` is set, we'll calculate precision as how often on average a class
+  among the top-k classes with the highest predicted values of a batch entry is
+  correct and can be found in the label for that entry.
+
+  If `class_id` is specified, we calculate precision by considering only the
+  entries in the batch for which `class_id` is above the threshold and/or in the
+  top-k highest predictions, and computing the fraction of them for which
+  `class_id` is indeed a correct label.
+
   Usage:
 
   ```python
@@ -1045,22 +1058,37 @@ class Precision(Metric):
   ```
   """
 
-  def __init__(self, thresholds=None, name=None, dtype=None):
+  def __init__(self,
+               thresholds=None,
+               top_k=None,
+               class_id=None,
+               name=None,
+               dtype=None):
     """Creates a `Precision` instance.
 
     Args:
-      thresholds: (Optional) Defaults to 0.5. A float value or a python
-        list/tuple of float threshold values in [0, 1]. A threshold is compared
-        with prediction values to determine the truth value of predictions
-        (i.e., above the threshold is `true`, below is `false`). One metric
-        value is generated for each threshold value.
+      thresholds: (Optional) A float value or a python list/tuple of float
+        threshold values in [0, 1]. A threshold is compared with prediction
+        values to determine the truth value of predictions (i.e., above the
+        threshold is `true`, below is `false`). One metric value is generated
+        for each threshold value. If neither thresholds nor top_k are set, the
+        default is to calculate precision with `thresholds=0.5`.
+      top_k: (Optional) Unset by default. An int value specifying the top-k
+        predictions to consider when calculating precision.
+      class_id: (Optional) Integer class ID for which we want binary metrics.
+        This must be in the half-open interval `[0, num_classes)`, where
+        `num_classes` is the last dimension of predictions.
       name: (Optional) string name of the metric instance.
       dtype: (Optional) data type of the metric result.
     """
     super(Precision, self).__init__(name=name, dtype=dtype)
     self.init_thresholds = thresholds
+    self.top_k = top_k
+    self.class_id = class_id
+
+    default_threshold = 0.5 if top_k is None else metrics_utils.NEG_INF
     self.thresholds = metrics_utils.parse_init_thresholds(
-        thresholds, default_threshold=0.5)
+        thresholds, default_threshold=default_threshold)
     self.tp = self.add_weight(
         'true_positives',
         shape=(len(self.thresholds),),
@@ -1074,8 +1102,9 @@ class Precision(Metric):
     """Accumulates true positive and false positive statistics.
 
     Args:
-      y_true: The ground truth values.
-      y_pred: The predicted values.
+      y_true: The ground truth values, with the same dimensions as `y_pred`.
+        Will be cast to `bool`.
+      y_pred: The predicted values. Each element must be in the range `[0, 1]`.
       sample_weight: Optional weighting of each example. Defaults to 1. Can be a
         `Tensor` whose rank is either 0, or the same rank as `y_true`, and must
         be broadcastable to `y_true`.
@@ -1083,10 +1112,17 @@ class Precision(Metric):
     Returns:
       Update op.
     """
-    return metrics_utils.update_confusion_matrix_variables({
-        metrics_utils.ConfusionMatrix.TRUE_POSITIVES: self.tp,
-        metrics_utils.ConfusionMatrix.FALSE_POSITIVES: self.fp
-    }, y_true, y_pred, self.thresholds, sample_weight)
+    return metrics_utils.update_confusion_matrix_variables(
+        {
+            metrics_utils.ConfusionMatrix.TRUE_POSITIVES: self.tp,
+            metrics_utils.ConfusionMatrix.FALSE_POSITIVES: self.fp
+        },
+        y_true,
+        y_pred,
+        thresholds=self.thresholds,
+        top_k=self.top_k,
+        class_id=self.class_id,
+        sample_weight=sample_weight)
 
   def result(self):
     result = math_ops.div_no_nan(self.tp, self.tp + self.fp)
@@ -1098,7 +1134,11 @@ class Precision(Metric):
       K.set_value(v, np.zeros((num_thresholds,)))
 
   def get_config(self):
-    config = {'thresholds': self.init_thresholds}
+    config = {
+        'thresholds': self.init_thresholds,
+        'top_k': self.top_k,
+        'class_id': self.class_id
+    }
     base_config = super(Precision, self).get_config()
     return dict(list(base_config.items()) + list(config.items()))
 
@@ -1119,6 +1159,14 @@ class Recall(Metric):
   If `sample_weight` is `None`, weights default to 1.
   Use `sample_weight` of 0 to mask values.
 
+  If `top_k` is set, recall will be computed as how often on average a class
+  among the labels of a batch entry is in the top-k predictions.
+
+  If `class_id` is specified, we calculate recall by considering only the
+  entries in the batch for which `class_id` is in the label, and computing the
+  fraction of them for which `class_id` is above the threshold and/or in the
+  top-k predictions.
+
   Usage:
 
   ```python
@@ -1135,22 +1183,37 @@ class Recall(Metric):
   ```
   """
 
-  def __init__(self, thresholds=None, name=None, dtype=None):
+  def __init__(self,
+               thresholds=None,
+               top_k=None,
+               class_id=None,
+               name=None,
+               dtype=None):
     """Creates a `Recall` instance.
 
     Args:
-      thresholds: (Optional) Defaults to 0.5. A float value or a python
-        list/tuple of float threshold values in [0, 1]. A threshold is compared
-        with prediction values to determine the truth value of predictions
-        (i.e., above the threshold is `true`, below is `false`). One metric
-        value is generated for each threshold value.
+      thresholds: (Optional) A float value or a python list/tuple of float
+        threshold values in [0, 1]. A threshold is compared with prediction
+        values to determine the truth value of predictions (i.e., above the
+        threshold is `true`, below is `false`). One metric value is generated
+        for each threshold value. If neither thresholds nor top_k are set, the
+        default is to calculate recall with `thresholds=0.5`.
+      top_k: (Optional) Unset by default. An int value specifying the top-k
+        predictions to consider when calculating recall.
+      class_id: (Optional) Integer class ID for which we want binary metrics.
+        This must be in the half-open interval `[0, num_classes)`, where
+        `num_classes` is the last dimension of predictions.
       name: (Optional) string name of the metric instance.
       dtype: (Optional) data type of the metric result.
     """
     super(Recall, self).__init__(name=name, dtype=dtype)
     self.init_thresholds = thresholds
+    self.top_k = top_k
+    self.class_id = class_id
+
+    default_threshold = 0.5 if top_k is None else metrics_utils.NEG_INF
     self.thresholds = metrics_utils.parse_init_thresholds(
-        thresholds, default_threshold=0.5)
+        thresholds, default_threshold=default_threshold)
     self.tp = self.add_weight(
         'true_positives',
         shape=(len(self.thresholds),),
@@ -1164,8 +1227,9 @@ class Recall(Metric):
     """Accumulates true positive and false negative statistics.
 
     Args:
-      y_true: The ground truth values.
-      y_pred: The predicted values.
+      y_true: The ground truth values, with the same dimensions as `y_pred`.
+        Will be cast to `bool`.
+      y_pred: The predicted values. Each element must be in the range `[0, 1]`.
       sample_weight: Optional weighting of each example. Defaults to 1. Can be a
         `Tensor` whose rank is either 0, or the same rank as `y_true`, and must
         be broadcastable to `y_true`.
@@ -1173,10 +1237,17 @@ class Recall(Metric):
     Returns:
       Update op.
     """
-    return metrics_utils.update_confusion_matrix_variables({
-        metrics_utils.ConfusionMatrix.TRUE_POSITIVES: self.tp,
-        metrics_utils.ConfusionMatrix.FALSE_NEGATIVES: self.fn
-    }, y_true, y_pred, self.thresholds, sample_weight)
+    return metrics_utils.update_confusion_matrix_variables(
+        {
+            metrics_utils.ConfusionMatrix.TRUE_POSITIVES: self.tp,
+            metrics_utils.ConfusionMatrix.FALSE_NEGATIVES: self.fn
+        },
+        y_true,
+        y_pred,
+        thresholds=self.thresholds,
+        top_k=self.top_k,
+        class_id=self.class_id,
+        sample_weight=sample_weight)
 
   def result(self):
     result = math_ops.div_no_nan(self.tp, self.tp + self.fn)
@@ -1188,7 +1259,11 @@ class Recall(Metric):
       K.set_value(v, np.zeros((num_thresholds,)))
 
   def get_config(self):
-    config = {'thresholds': self.init_thresholds}
+    config = {
+        'thresholds': self.init_thresholds,
+        'top_k': self.top_k,
+        'class_id': self.class_id
+    }
     base_config = super(Recall, self).get_config()
     return dict(list(base_config.items()) + list(config.items()))
 
@@ -1244,12 +1319,17 @@ class SensitivitySpecificityBase(Metric):
     Returns:
       Update op.
     """
-    return metrics_utils.update_confusion_matrix_variables({
-        metrics_utils.ConfusionMatrix.TRUE_POSITIVES: self.tp,
-        metrics_utils.ConfusionMatrix.TRUE_NEGATIVES: self.tn,
-        metrics_utils.ConfusionMatrix.FALSE_POSITIVES: self.fp,
-        metrics_utils.ConfusionMatrix.FALSE_NEGATIVES: self.fn,
-    }, y_true, y_pred, self.thresholds, sample_weight)
+    return metrics_utils.update_confusion_matrix_variables(
+        {
+            metrics_utils.ConfusionMatrix.TRUE_POSITIVES: self.tp,
+            metrics_utils.ConfusionMatrix.TRUE_NEGATIVES: self.tn,
+            metrics_utils.ConfusionMatrix.FALSE_POSITIVES: self.fp,
+            metrics_utils.ConfusionMatrix.FALSE_NEGATIVES: self.fn,
+        },
+        y_true,
+        y_pred,
+        thresholds=self.thresholds,
+        sample_weight=sample_weight)
 
   def reset_states(self):
     num_thresholds = len(self.thresholds)
diff --git a/tensorflow/python/keras/metrics_confusion_matrix_test.py b/tensorflow/python/keras/metrics_confusion_matrix_test.py
index 37d2880088..6db034b721 100644
--- a/tensorflow/python/keras/metrics_confusion_matrix_test.py
+++ b/tensorflow/python/keras/metrics_confusion_matrix_test.py
@@ -318,18 +318,23 @@ class TruePositivesTest(test.TestCase):
 class PrecisionTest(test.TestCase):
 
   def test_config(self):
-    p_obj = metrics.Precision(name='my_precision', thresholds=[0.4, 0.9])
+    p_obj = metrics.Precision(
+        name='my_precision', thresholds=[0.4, 0.9], top_k=15, class_id=12)
     self.assertEqual(p_obj.name, 'my_precision')
     self.assertEqual(len(p_obj.variables), 2)
     self.assertEqual([v.name for v in p_obj.variables],
                      ['true_positives:0', 'false_positives:0'])
     self.assertEqual(p_obj.thresholds, [0.4, 0.9])
+    self.assertEqual(p_obj.top_k, 15)
+    self.assertEqual(p_obj.class_id, 12)
 
     # Check save and restore config
     p_obj2 = metrics.Precision.from_config(p_obj.get_config())
     self.assertEqual(p_obj2.name, 'my_precision')
     self.assertEqual(len(p_obj2.variables), 2)
     self.assertEqual(p_obj2.thresholds, [0.4, 0.9])
+    self.assertEqual(p_obj2.top_k, 15)
+    self.assertEqual(p_obj2.class_id, 12)
 
   def test_value_is_idempotent(self):
     p_obj = metrics.Precision(thresholds=[0.3, 0.72])
@@ -431,23 +436,110 @@ class PrecisionTest(test.TestCase):
     self.assertArrayNear([expected_precision, 0], self.evaluate(p_obj.result()),
                          1e-3)
 
+  def test_unweighted_top_k(self):
+    p_obj = metrics.Precision(top_k=3)
+    y_pred = constant_op.constant([0.2, 0.1, 0.5, 0, 0.2], shape=(1, 5))
+    y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))
+    self.evaluate(variables.variables_initializer(p_obj.variables))
+    result = p_obj(y_true, y_pred)
+    self.assertAlmostEqual(1. / 3, self.evaluate(result))
+
+  def test_weighted_top_k(self):
+    p_obj = metrics.Precision(top_k=3)
+    y_pred1 = constant_op.constant([0.2, 0.1, 0.4, 0, 0.2], shape=(1, 5))
+    y_true1 = constant_op.constant([0, 1, 1, 0, 1], shape=(1, 5))
+    self.evaluate(variables.variables_initializer(p_obj.variables))
+    self.evaluate(
+        p_obj(
+            y_true1,
+            y_pred1,
+            sample_weight=constant_op.constant([[1, 4, 2, 3, 5]])))
+
+    y_pred2 = constant_op.constant([0.2, 0.6, 0.4, 0.2, 0.2], shape=(1, 5))
+    y_true2 = constant_op.constant([1, 0, 1, 1, 1], shape=(1, 5))
+    result = p_obj(y_true2, y_pred2, sample_weight=constant_op.constant(3))
+
+    tp = (2 + 5) + (3 + 3)
+    predicted_positives = (1 + 2 + 5) + (3 + 3 + 3)
+    expected_precision = tp / predicted_positives
+    self.assertAlmostEqual(expected_precision, self.evaluate(result))
+
+  def test_unweighted_class_id(self):
+    p_obj = metrics.Precision(class_id=2)
+    self.evaluate(variables.variables_initializer(p_obj.variables))
+
+    y_pred = constant_op.constant([0.2, 0.1, 0.6, 0, 0.2], shape=(1, 5))
+    y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))
+    result = p_obj(y_true, y_pred)
+    self.assertAlmostEqual(1, self.evaluate(result))
+    self.assertAlmostEqual(1, self.evaluate(p_obj.tp))
+    self.assertAlmostEqual(0, self.evaluate(p_obj.fp))
+
+    y_pred = constant_op.constant([0.2, 0.1, 0, 0, 0.2], shape=(1, 5))
+    y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))
+    result = p_obj(y_true, y_pred)
+    self.assertAlmostEqual(1, self.evaluate(result))
+    self.assertAlmostEqual(1, self.evaluate(p_obj.tp))
+    self.assertAlmostEqual(0, self.evaluate(p_obj.fp))
+
+    y_pred = constant_op.constant([0.2, 0.1, 0.6, 0, 0.2], shape=(1, 5))
+    y_true = constant_op.constant([0, 1, 0, 0, 0], shape=(1, 5))
+    result = p_obj(y_true, y_pred)
+    self.assertAlmostEqual(0.5, self.evaluate(result))
+    self.assertAlmostEqual(1, self.evaluate(p_obj.tp))
+    self.assertAlmostEqual(1, self.evaluate(p_obj.fp))
+
+  def test_unweighted_top_k_and_class_id(self):
+    p_obj = metrics.Precision(class_id=2, top_k=2)
+    self.evaluate(variables.variables_initializer(p_obj.variables))
+
+    y_pred = constant_op.constant([0.2, 0.6, 0.3, 0, 0.2], shape=(1, 5))
+    y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))
+    result = p_obj(y_true, y_pred)
+    self.assertAlmostEqual(1, self.evaluate(result))
+    self.assertAlmostEqual(1, self.evaluate(p_obj.tp))
+    self.assertAlmostEqual(0, self.evaluate(p_obj.fp))
+
+    y_pred = constant_op.constant([1, 1, 0.9, 1, 1], shape=(1, 5))
+    y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))
+    result = p_obj(y_true, y_pred)
+    self.assertAlmostEqual(1, self.evaluate(result))
+    self.assertAlmostEqual(1, self.evaluate(p_obj.tp))
+    self.assertAlmostEqual(0, self.evaluate(p_obj.fp))
+
+  def test_unweighted_top_k_and_threshold(self):
+    p_obj = metrics.Precision(thresholds=.7, top_k=2)
+    self.evaluate(variables.variables_initializer(p_obj.variables))
+
+    y_pred = constant_op.constant([0.2, 0.8, 0.6, 0, 0.2], shape=(1, 5))
+    y_true = constant_op.constant([0, 1, 1, 0, 1], shape=(1, 5))
+    result = p_obj(y_true, y_pred)
+    self.assertAlmostEqual(1, self.evaluate(result))
+    self.assertAlmostEqual(1, self.evaluate(p_obj.tp))
+    self.assertAlmostEqual(0, self.evaluate(p_obj.fp))
+
 
 @test_util.run_all_in_graph_and_eager_modes
 class RecallTest(test.TestCase):
 
   def test_config(self):
-    r_obj = metrics.Recall(name='my_recall', thresholds=[0.4, 0.9])
+    r_obj = metrics.Recall(
+        name='my_recall', thresholds=[0.4, 0.9], top_k=15, class_id=12)
     self.assertEqual(r_obj.name, 'my_recall')
     self.assertEqual(len(r_obj.variables), 2)
     self.assertEqual([v.name for v in r_obj.variables],
                      ['true_positives:0', 'false_negatives:0'])
     self.assertEqual(r_obj.thresholds, [0.4, 0.9])
+    self.assertEqual(r_obj.top_k, 15)
+    self.assertEqual(r_obj.class_id, 12)
 
     # Check save and restore config
     r_obj2 = metrics.Recall.from_config(r_obj.get_config())
     self.assertEqual(r_obj2.name, 'my_recall')
     self.assertEqual(len(r_obj2.variables), 2)
     self.assertEqual(r_obj2.thresholds, [0.4, 0.9])
+    self.assertEqual(r_obj2.top_k, 15)
+    self.assertEqual(r_obj2.class_id, 12)
 
   def test_value_is_idempotent(self):
     r_obj = metrics.Recall(thresholds=[0.3, 0.72])
@@ -548,6 +640,88 @@ class RecallTest(test.TestCase):
     self.assertArrayNear([expected_recall, 0], self.evaluate(r_obj.result()),
                          1e-3)
 
+  def test_unweighted_top_k(self):
+    r_obj = metrics.Recall(top_k=3)
+    y_pred = constant_op.constant([0.2, 0.1, 0.5, 0, 0.2], shape=(1, 5))
+    y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))
+    self.evaluate(variables.variables_initializer(r_obj.variables))
+    result = r_obj(y_true, y_pred)
+    self.assertAlmostEqual(0.5, self.evaluate(result))
+
+  def test_weighted_top_k(self):
+    r_obj = metrics.Recall(top_k=3)
+    y_pred1 = constant_op.constant([0.2, 0.1, 0.4, 0, 0.2], shape=(1, 5))
+    y_true1 = constant_op.constant([0, 1, 1, 0, 1], shape=(1, 5))
+    self.evaluate(variables.variables_initializer(r_obj.variables))
+    self.evaluate(
+        r_obj(
+            y_true1,
+            y_pred1,
+            sample_weight=constant_op.constant([[1, 4, 2, 3, 5]])))
+
+    y_pred2 = constant_op.constant([0.2, 0.6, 0.4, 0.2, 0.2], shape=(1, 5))
+    y_true2 = constant_op.constant([1, 0, 1, 1, 1], shape=(1, 5))
+    result = r_obj(y_true2, y_pred2, sample_weight=constant_op.constant(3))
+
+    tp = (2 + 5) + (3 + 3)
+    positives = (4 + 2 + 5) + (3 + 3 + 3 + 3)
+    expected_recall = tp / positives
+    self.assertAlmostEqual(expected_recall, self.evaluate(result))
+
+  def test_unweighted_class_id(self):
+    r_obj = metrics.Recall(class_id=2)
+    self.evaluate(variables.variables_initializer(r_obj.variables))
+
+    y_pred = constant_op.constant([0.2, 0.1, 0.6, 0, 0.2], shape=(1, 5))
+    y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))
+    result = r_obj(y_true, y_pred)
+    self.assertAlmostEqual(1, self.evaluate(result))
+    self.assertAlmostEqual(1, self.evaluate(r_obj.tp))
+    self.assertAlmostEqual(0, self.evaluate(r_obj.fn))
+
+    y_pred = constant_op.constant([0.2, 0.1, 0, 0, 0.2], shape=(1, 5))
+    y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))
+    result = r_obj(y_true, y_pred)
+    self.assertAlmostEqual(0.5, self.evaluate(result))
+    self.assertAlmostEqual(1, self.evaluate(r_obj.tp))
+    self.assertAlmostEqual(1, self.evaluate(r_obj.fn))
+
+    y_pred = constant_op.constant([0.2, 0.1, 0.6, 0, 0.2], shape=(1, 5))
+    y_true = constant_op.constant([0, 1, 0, 0, 0], shape=(1, 5))
+    result = r_obj(y_true, y_pred)
+    self.assertAlmostEqual(0.5, self.evaluate(result))
+    self.assertAlmostEqual(1, self.evaluate(r_obj.tp))
+    self.assertAlmostEqual(1, self.evaluate(r_obj.fn))
+
+  def test_unweighted_top_k_and_class_id(self):
+    r_obj = metrics.Recall(class_id=2, top_k=2)
+    self.evaluate(variables.variables_initializer(r_obj.variables))
+
+    y_pred = constant_op.constant([0.2, 0.6, 0.3, 0, 0.2], shape=(1, 5))
+    y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))
+    result = r_obj(y_true, y_pred)
+    self.assertAlmostEqual(1, self.evaluate(result))
+    self.assertAlmostEqual(1, self.evaluate(r_obj.tp))
+    self.assertAlmostEqual(0, self.evaluate(r_obj.fn))
+
+    y_pred = constant_op.constant([1, 1, 0.9, 1, 1], shape=(1, 5))
+    y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))
+    result = r_obj(y_true, y_pred)
+    self.assertAlmostEqual(0.5, self.evaluate(result))
+    self.assertAlmostEqual(1, self.evaluate(r_obj.tp))
+    self.assertAlmostEqual(1, self.evaluate(r_obj.fn))
+
+  def test_unweighted_top_k_and_threshold(self):
+    r_obj = metrics.Recall(thresholds=.7, top_k=2)
+    self.evaluate(variables.variables_initializer(r_obj.variables))
+
+    y_pred = constant_op.constant([0.2, 0.8, 0.6, 0, 0.2], shape=(1, 5))
+    y_true = constant_op.constant([1, 1, 1, 0, 1], shape=(1, 5))
+    result = r_obj(y_true, y_pred)
+    self.assertAlmostEqual(0.25, self.evaluate(result))
+    self.assertAlmostEqual(1, self.evaluate(r_obj.tp))
+    self.assertAlmostEqual(3, self.evaluate(r_obj.fn))
+
 
 @test_util.run_all_in_graph_and_eager_modes
 class SensitivityAtSpecificityTest(test.TestCase, parameterized.TestCase):
diff --git a/tensorflow/python/keras/utils/metrics_utils.py b/tensorflow/python/keras/utils/metrics_utils.py
index 6125b95b6e..7db128e8ab 100644
--- a/tensorflow/python/keras/utils/metrics_utils.py
+++ b/tensorflow/python/keras/utils/metrics_utils.py
@@ -33,11 +33,15 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import check_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import nn_ops
 from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import weights_broadcast_ops
 from tensorflow.python.util import tf_decorator
 
 
+NEG_INF = -1e10
+
+
 def update_state_wrapper(update_state_fn):
   """Decorator to wrap metric `update_state()` with `add_update()`.
 
@@ -120,16 +124,18 @@ def weakmethod(method):
 
 
 def assert_thresholds_range(thresholds):
-  invalid_thresholds = [t for t in thresholds if t is None or t < 0 or t > 1]
-  if invalid_thresholds:
-    raise ValueError(
-        'Threshold values must be in [0, 1]. Invalid values: {}'.format(
-            invalid_thresholds))
+  if thresholds is not None:
+    invalid_thresholds = [t for t in thresholds if t is None or t < 0 or t > 1]
+    if invalid_thresholds:
+      raise ValueError(
+          'Threshold values must be in [0, 1]. Invalid values: {}'.format(
+              invalid_thresholds))
 
 
 def parse_init_thresholds(thresholds, default_threshold=0.5):
+  if thresholds is not None:
+    assert_thresholds_range(to_list(thresholds))
   thresholds = to_list(default_threshold if thresholds is None else thresholds)
-  assert_thresholds_range(thresholds)
   return thresholds
 
 
@@ -144,6 +150,8 @@ def update_confusion_matrix_variables(variables_to_update,
                                       y_true,
                                       y_pred,
                                       thresholds,
+                                      top_k=None,
+                                      class_id=None,
                                       sample_weight=None):
   """Returns op to update the given confusion matrix variables.
 
@@ -170,7 +178,11 @@ def update_confusion_matrix_variables(variables_to_update,
     y_pred: A floating point `Tensor` of arbitrary shape and whose values are in
       the range `[0, 1]`.
     thresholds: A float value or a python list or tuple of float thresholds in
-      `[0, 1]`.
+      `[0, 1]`, or NEG_INF (used when top_k is set).
+    top_k: Optional int, indicates that the positive labels should be limited to
+      the top k predictions.
+    class_id: Optional int, limits the prediction and labels to the class
+      specified by this argument.
     sample_weight: Optional `Tensor` whose rank is either 0, or the same rank as
       `y_true`, and must be broadcastable to `y_true` (i.e., all dimensions must
       be either `1`, or the same as the corresponding `y_true` dimension).
@@ -219,6 +231,12 @@ def update_confusion_matrix_variables(variables_to_update,
         math_ops.cast(y_pred, dtype=dtypes.float32),
         math_ops.cast(y_true, dtype=dtypes.bool), sample_weight)
 
+  if top_k is not None:
+    y_pred = _filter_top_k(y_pred, top_k)
+  if class_id is not None:
+    y_true = y_true[..., class_id]
+    y_pred = y_pred[..., class_id]
+
   thresholds = to_list(thresholds)
   num_thresholds = len(thresholds)
   num_predictions = array_ops.size(y_pred)
@@ -282,3 +300,22 @@ def update_confusion_matrix_variables(variables_to_update,
           weighted_assign_add(label, pred, weights_tiled,
                               variables_to_update[matrix_cond]))
   return control_flow_ops.group(update_ops)
+
+
+def _filter_top_k(x, k):
+  """Filters top-k values in the last dim of x and set the rest to NEG_INF.
+
+  Used for computing top-k prediction values in dense labels (which has the same
+  shape as predictions) for recall and precision top-k metrics.
+
+  Args:
+    x: tensor with any dimensions.
+    k: the number of values to keep.
+
+  Returns:
+    tensor with same shape and dtype as x.
+  """
+  _, top_k_idx = nn_ops.top_k(x, k, sorted=False)
+  top_k_mask = math_ops.reduce_sum(
+      array_ops.one_hot(top_k_idx, x.shape[-1], axis=-1), axis=-2)
+  return x * top_k_mask + NEG_INF * (1 - top_k_mask)
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-precision.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-precision.pbtxt
index e81ecfe3f6..794da447d7 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-precision.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-precision.pbtxt
@@ -87,7 +87,7 @@ tf_class {
   }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+    argspec: "args=[\'self\', \'thresholds\', \'top_k\', \'class_id\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_loss"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-recall.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-recall.pbtxt
index f8470b94d7..cae610b9ea 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-recall.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-recall.pbtxt
@@ -87,7 +87,7 @@ tf_class {
   }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+    argspec: "args=[\'self\', \'thresholds\', \'top_k\', \'class_id\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_loss"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-precision.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-precision.pbtxt
index e81ecfe3f6..794da447d7 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-precision.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-precision.pbtxt
@@ -87,7 +87,7 @@ tf_class {
   }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+    argspec: "args=[\'self\', \'thresholds\', \'top_k\', \'class_id\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_loss"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-recall.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-recall.pbtxt
index f8470b94d7..cae610b9ea 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-recall.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-recall.pbtxt
@@ -87,7 +87,7 @@ tf_class {
   }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'thresholds\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
+    argspec: "args=[\'self\', \'thresholds\', \'top_k\', \'class_id\', \'name\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_loss"
-- 
GitLab


From eacf1972e653069683e739ba401bb55a8cd58ce5 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 11 Jan 2019 14:24:48 -0800
Subject: [PATCH 0620/1095] Start device tracer when running eager profiler.

PiperOrigin-RevId: 228945120
---
 tensorflow/c/eager/BUILD                      | 18 +++--
 tensorflow/c/eager/c_api_experimental_test.cc | 27 +++++++-
 tensorflow/core/common_runtime/eager/BUILD    |  1 +
 .../core/common_runtime/eager/profiler.cc     | 68 ++++++++++++++-----
 .../core/common_runtime/eager/profiler.h      | 10 +--
 5 files changed, 98 insertions(+), 26 deletions(-)

diff --git a/tensorflow/c/eager/BUILD b/tensorflow/c/eager/BUILD
index 3ea1724d1e..710850b5f2 100644
--- a/tensorflow/c/eager/BUILD
+++ b/tensorflow/c/eager/BUILD
@@ -8,6 +8,15 @@ load(
     "tf_cuda_library",
     "tfe_xla_copts",
 )
+load(
+    "//tensorflow/core:platform/default/build_config.bzl",
+    "tf_additional_device_tracer_test_flags",
+    "tf_kernel_tests_linkstatic",
+)
+load(
+    "//tensorflow/core:platform/default/build_config_root.bzl",
+    "tf_cuda_tests_tags",
+)
 
 tf_cuda_library(
     name = "c_api",
@@ -211,11 +220,10 @@ tf_cuda_cc_test(
     srcs = [
         "c_api_experimental_test.cc",
     ],
-    extra_copts = tfe_xla_copts(),
-    tags = [
-        "guitar",
-        "multi_gpu",
-    ],
+    args =
+        ["--heap_check=local"] + tf_additional_device_tracer_test_flags(),
+    linkstatic = tf_kernel_tests_linkstatic(),
+    tags = tf_cuda_tests_tags() + ["nomac"],
     deps = [
         ":c_api_experimental",
         ":c_api_test_util",
diff --git a/tensorflow/c/eager/c_api_experimental_test.cc b/tensorflow/c/eager/c_api_experimental_test.cc
index 9b4fca9d45..c3d29bd2e0 100644
--- a/tensorflow/c/eager/c_api_experimental_test.cc
+++ b/tensorflow/c/eager/c_api_experimental_test.cc
@@ -16,9 +16,9 @@ limitations under the License.
 #include "tensorflow/c/eager/c_api_experimental.h"
 
 #include <string.h>
-#include "absl/strings/match.h"
 #include "tensorflow/c/eager/c_api_test_util.h"
 #include "tensorflow/cc/profiler/profiler.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/protobuf.h"
 #include "tensorflow/core/platform/test.h"
@@ -27,8 +27,15 @@ limitations under the License.
 
 using tensorflow::string;
 
+namespace tensorflow {
 namespace {
 
+static bool HasSubstr(absl::string_view base, absl::string_view substr) {
+  bool ok = str_util::StrContains(base, substr);
+  EXPECT_TRUE(ok) << base << ", expected substring " << substr;
+  return ok;
+}
+
 void ExecuteWithProfiling(bool async) {
   TF_Status* status = TF_NewStatus();
   TFE_ContextOptions* opts = TFE_NewContextOptions();
@@ -42,6 +49,16 @@ void ExecuteWithProfiling(bool async) {
   TFE_Op* matmul = MatMulOp(ctx, m, m);
   TFE_TensorHandle* retvals[1] = {nullptr};
   int num_retvals = 1;
+
+  // Run op on GPU if it is present.
+  string gpu_device_name;
+  if (GetDeviceName(ctx, &gpu_device_name, "GPU")) {
+    TFE_OpSetDevice(matmul, gpu_device_name.c_str(), status);
+    ASSERT_TRUE(TF_GetCode(status) == TF_OK) << TF_Message(status);
+    const char* device_name = TFE_OpGetDevice(matmul, status);
+    ASSERT_TRUE(strstr(device_name, "GPU:0") != nullptr);
+  }
+
   TFE_Execute(matmul, &retvals[0], &num_retvals, status);
   EXPECT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
   TFE_DeleteOp(matmul);
@@ -57,6 +74,13 @@ void ExecuteWithProfiling(bool async) {
   EXPECT_TRUE(profile_proto.ParseFromString(
       {reinterpret_cast<const char*>(profiler_result->data),
        profiler_result->length}));
+  string profile_proto_str = profile_proto.DebugString();
+  if (!gpu_device_name.empty()) {
+    EXPECT_TRUE(HasSubstr(profile_proto_str, "gpu:0"));
+    // device name with "stream:all" is collected by Device Tracer.
+    EXPECT_TRUE(HasSubstr(profile_proto_str, "stream:all"));
+  }
+  EXPECT_TRUE(HasSubstr(profile_proto_str, "cpu:0"));
   TF_DeleteBuffer(profiler_result);
 
   TF_Tensor* t = TFE_TensorHandleResolve(retvals[0], status);
@@ -77,3 +101,4 @@ TEST(CAPI, ExecuteWithTracing) { ExecuteWithProfiling(false); }
 TEST(CAPI, ExecuteWithTracingAsync) { ExecuteWithProfiling(true); }
 
 }  // namespace
+}  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/eager/BUILD b/tensorflow/core/common_runtime/eager/BUILD
index cabbddb77d..2697aea783 100644
--- a/tensorflow/core/common_runtime/eager/BUILD
+++ b/tensorflow/core/common_runtime/eager/BUILD
@@ -112,6 +112,7 @@ tf_cuda_library(
             "//tensorflow/core:lib_internal",
             "//tensorflow/core:protos_all_cc",
             "//tensorflow/core:session_options",
+            "//tensorflow/core:device_tracer",
         ],
     }),
 )
diff --git a/tensorflow/core/common_runtime/eager/profiler.cc b/tensorflow/core/common_runtime/eager/profiler.cc
index d670d6f344..46449d0771 100644
--- a/tensorflow/core/common_runtime/eager/profiler.cc
+++ b/tensorflow/core/common_runtime/eager/profiler.cc
@@ -15,7 +15,11 @@ limitations under the License.
 
 #include "tensorflow/core/common_runtime/eager/profiler.h"
 #include "tensorflow/cc/profiler/profiler.h"
+#include "tensorflow/core/common_runtime/eager/context.h"
+#include "tensorflow/core/common_runtime/step_stats_collector.h"
 #include "tensorflow/core/framework/graph.pb.h"
+#include "tensorflow/core/platform/device_tracer.h"
+#include "tensorflow/core/platform/env.h"
 #include "tensorflow/core/platform/mutex.h"
 #include "tensorflow/core/protobuf/config.pb.h"
 
@@ -37,18 +41,28 @@ Status EagerProfiler::Status() {
 }
 
 Status EagerProfiler::SerializeToString(string* content) {
-  {
-    mutex_lock l(mutex_);
-    if (!status_.ok()) return status_;
+  mutex_lock l(mutex_);
+  if (!status_.ok()) return status_;
+  Stop();
+
+  // Get profiling data from device tracer
+  if (device_tracer_ != nullptr) {
+    std::unique_ptr<StepStatsCollector> step_stats_collector(
+        new StepStatsCollector(run_metadata_.mutable_step_stats()));
+    tensorflow::Status s = device_tracer_->Collect(step_stats_collector.get());
+    if (!s.ok()) {
+      device_tracer_.reset(nullptr);
+      LOG(WARNING) << "Failed to collect data from device tracer. "
+                   << s.error_message();
+    }
+    step_stats_collector->Finalize();
   }
-  RunMetadata metadata;
-  GetMergetRunMetadata(&metadata);
 
   // TODO(fishx): update tfprof to use a lighter representation instead of
   // GraphDef.
   GraphDef graph;
   std::unique_ptr<tfprof::Profiler> tfprof(new tfprof::Profiler(graph));
-  tfprof->AddStep(0, metadata);
+  tfprof->AddStep(0, run_metadata_);
   return tfprof->SerializeToString(content);
 }
 
@@ -57,21 +71,43 @@ EagerProfiler::EagerProfiler(EagerContext* const context) : context_(context) {
 
   status_ = context_->RegisterRunMetadataListener(this);
   if (!status_.ok()) {
-    LOG(INFO) << "Eager Profiler failed to start. Another profiler is running.";
+    context_ = nullptr;
+    LOG(WARNING)
+        << "Eager Profiler failed to start. Another profiler is running.";
     return;
   }
-}
 
-EagerProfiler::~EagerProfiler() {
-  context_->ClearRunMetadataListener();
-  LOG(INFO) << "Eager Profiler ended with status:" << status_;
+  // TODO(fishx): Allow user disable device tracer.
+  device_tracer_ = CreateDeviceTracer();
+  if (!device_tracer_) {
+    LOG(WARNING) << "Continue profiling without device tracer. "
+                 << "Failed to create device tracer.";
+    return;
+  }
+  class Status s = device_tracer_->Start();
+  if (!s.ok()) {
+    device_tracer_.reset(nullptr);
+    LOG(WARNING) << "Continue profiling without device tracer. "
+                 << s.error_message();
+  }
 }
 
-void EagerProfiler::GetMergetRunMetadata(RunMetadata* metadata) {
-  mutex_lock ml(*context_->MetadataMu());
-  mutex_lock l(mutex_);
-  *metadata = run_metadata_;
-  metadata->MergeFrom(*context_->RunMetadataProto());
+EagerProfiler::~EagerProfiler() { Stop(); }
+
+void EagerProfiler::Stop() {
+  if (context_ != nullptr) {
+    context_->ClearRunMetadataListener();
+    run_metadata_.MergeFrom(*context_->RunMetadataProto());
+    context_ = nullptr;
+    if (device_tracer_ != nullptr) {
+      tensorflow::Status s = device_tracer_->Stop();
+      if (!s.ok()) {
+        device_tracer_.reset(nullptr);
+        LOG(WARNING) << "Failed to stop device tracer. " << s.error_message();
+      }
+    }
+    LOG(INFO) << "Eager Profiler ended with status:" << status_;
+  }
 }
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/eager/profiler.h b/tensorflow/core/common_runtime/eager/profiler.h
index cadbfdb498..62df869e94 100644
--- a/tensorflow/core/common_runtime/eager/profiler.h
+++ b/tensorflow/core/common_runtime/eager/profiler.h
@@ -17,14 +17,15 @@ limitations under the License.
 
 #include "tensorflow/core/common_runtime/eager/context.h"
 #include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/platform/device_tracer.h"
 #include "tensorflow/core/platform/mutex.h"
 #include "tensorflow/core/protobuf/config.pb.h"
 
 namespace tensorflow {
 
 // A profiler which will start profiling when creating the object and will stop
-// when the object is destroyed. It will profile all operations run under the
-// given EagerContext.
+// when either the object is destroyed or SerializedToString is called. It will
+// profile all operations run under the given EagerContext.
 // Multiple instances of it can be created, but at most one of them will profile
 // for each EagerContext. Status() will return OK only for the instance that is
 // profiling.
@@ -51,11 +52,12 @@ class EagerProfiler : RunMetadataListener {
   EagerProfiler(const EagerProfiler&) = delete;
   EagerProfiler& operator=(const EagerProfiler&) = delete;
 
-  void GetMergetRunMetadata(RunMetadata* metadata) LOCKS_EXCLUDED(mutex_);
+  void Stop() EXCLUSIVE_LOCKS_REQUIRED(mutex_);
 
   RunMetadata run_metadata_ GUARDED_BY(mutex_);
   tensorflow::Status status_ GUARDED_BY(mutex_);
-  EagerContext* const context_;
+  std::unique_ptr<DeviceTracer> device_tracer_ GUARDED_BY(mutex_);
+  EagerContext* context_ GUARDED_BY(mutex_);
   mutex mutex_;
 };
 
-- 
GitLab


From e51fa30400b3a469b03111b25344bc47bdff96bd Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Fri, 11 Jan 2019 14:38:07 -0800
Subject: [PATCH 0621/1095] Add a mutex for each (ICudaEngine,
 IExecutionContext) pair since some of their operations are not thread safe.

This fixed one of the issues mentioned in
#23853

PiperOrigin-RevId: 228947504
---
 tensorflow/contrib/tensorrt/BUILD             |  1 +
 .../contrib/tensorrt/kernels/trt_engine_op.cc | 71 +++++++++----------
 .../contrib/tensorrt/kernels/trt_engine_op.h  | 27 ++++---
 3 files changed, 55 insertions(+), 44 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index 003a7e0b0a..73bb1fe0c5 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -97,6 +97,7 @@ cc_library(
         ":trt_plugins",
         ":trt_resources",
         ":utils",
+        "@com_google_absl//absl/memory",
         "//tensorflow/core:gpu_headers_lib",
         "//tensorflow/core:lib_proto_parsing",
         "//tensorflow/core:stream_executor_headers_lib",
diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
index 1f6409341c..ae5e64e823 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
+++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
@@ -16,6 +16,7 @@ limitations under the License.
 
 #include <algorithm>
 
+#include "absl/memory/memory.h"
 #include "tensorflow/contrib/tensorrt/convert/convert_nodes.h"
 #include "tensorflow/contrib/tensorrt/convert/utils.h"
 #include "tensorflow/contrib/tensorrt/log/trt_logger.h"
@@ -284,16 +285,14 @@ void TRTEngineOp::ComputeAsync(OpKernelContext* ctx,
   }
 
   const int num_batch = ctx->input(0).shape().dim_size(0);
-  auto& engine_ctx_pair = GetEngine(smallest_engine, ctx);
-  auto& trt_engine_ptr = engine_ctx_pair.first;
-  if (!trt_engine_ptr) {
+  EngineContext* engine_context = GetEngine(smallest_engine, ctx);
+  if (!engine_context->cuda_engine) {
     LOG(WARNING) << "Engine retrieval for batch size " << num_batch
                  << " failed. Running native segment for " << name();
     ExecuteNativeSegment(ctx, helper);
     return;
   }
-  const bool retry = ExecuteTrtEngine(ctx, num_batch, trt_engine_ptr.get(),
-                                      engine_ctx_pair.second.get());
+  const bool retry = ExecuteTrtEngine(ctx, num_batch, engine_context);
   if (retry) {
     LOG(WARNING) << "Failed to execute engine, "
                  << "retrying with native segment for " << name();
@@ -302,18 +301,17 @@ void TRTEngineOp::ComputeAsync(OpKernelContext* ctx,
   }
 }
 
-bool TRTEngineOp::ExecuteTrtEngine(
-    OpKernelContext* ctx, const int num_batch,
-    nvinfer1::ICudaEngine* trt_engine_ptr,
-    nvinfer1::IExecutionContext* trt_execution_context_ptr) {
+bool TRTEngineOp::ExecuteTrtEngine(OpKernelContext* ctx, const int num_batch,
+                                   EngineContext* engine_context) {
+  auto& cuda_engine = engine_context->cuda_engine;
+
   VLOG(1) << "Executing TRT engine: " << name();
   const bool kRetry = true;
   const int num_binding = ctx->num_inputs() + ctx->num_outputs();
   std::vector<void*> buffers(num_binding);
   for (int i = 0; i < ctx->num_inputs(); i++) {
     const string input_name = StrCat(kInputPHName, i);
-    const int binding_index =
-        trt_engine_ptr->getBindingIndex(input_name.c_str());
+    const int binding_index = cuda_engine->getBindingIndex(input_name.c_str());
     if (binding_index == -1) {
       LOG(ERROR) << "Input node not found, at " << input_name;
       return kRetry;
@@ -326,7 +324,7 @@ bool TRTEngineOp::ExecuteTrtEngine(
                  << " vs " << input_shape.dim_size(0);
       return kRetry;
     }
-    auto dtype = trt_engine_ptr->getBindingDataType(binding_index);
+    auto dtype = cuda_engine->getBindingDataType(binding_index);
     switch (dtype) {
       case nvinfer1::DataType::kFLOAT:
         buffers[binding_index] = (void*)(input_tensor.flat<float>().data());
@@ -349,13 +347,12 @@ bool TRTEngineOp::ExecuteTrtEngine(
   for (int i = 0; i < ctx->num_outputs(); i++) {
     // Create an output tensor
     const string output_name = StrCat(kOutputPHName, i);
-    const int binding_index =
-        trt_engine_ptr->getBindingIndex(output_name.c_str());
+    const int binding_index = cuda_engine->getBindingIndex(output_name.c_str());
     Tensor* output_tensor = nullptr;
 
     TensorShape output_shape;
     if (binding_index != -1) {
-      auto dims = trt_engine_ptr->getBindingDimensions(binding_index);
+      auto dims = cuda_engine->getBindingDimensions(binding_index);
       std::vector<int> trt_shape(dims.nbDims + 1);
       trt_shape[0] = num_batch;
       for (int j = 0; j < dims.nbDims; j++) trt_shape[j + 1] = dims.d[j];
@@ -377,7 +374,7 @@ bool TRTEngineOp::ExecuteTrtEngine(
       // TODO(aaroey): ideally we should retry, fix this.
       return !kRetry;
     }
-    auto dtype = trt_engine_ptr->getBindingDataType(binding_index);
+    auto dtype = cuda_engine->getBindingDataType(binding_index);
     switch (dtype) {
       case nvinfer1::DataType::kFLOAT:
         buffers[binding_index] =
@@ -405,9 +402,12 @@ bool TRTEngineOp::ExecuteTrtEngine(
                                                 ->implementation()
                                                 ->GpuStreamMemberHack()));
 
+  // nvinfer1::IExecutionContext::enqueue is not thread safe and we need a mutex
+  // for it.
+  tensorflow::mutex_lock lock(engine_context->mu);
   // TODO(jie): trt enqueue does not return error
-  auto ret = trt_execution_context_ptr->enqueue(num_batch, &buffers[0], *stream,
-                                                nullptr);
+  auto ret = engine_context->execution_context->enqueue(num_batch, &buffers[0],
+                                                        *stream, nullptr);
   if (!ret) {
     LOG(WARNING) << "Failed to enqueue batch for TRT engine: " << name();
     return kRetry;
@@ -421,8 +421,8 @@ TRTEngineOp::~TRTEngineOp() {
   // We need to manually destroy the engine and execution context before
   // the allocator is destructed.
   for (auto& eng : engine_map_) {
-    eng.second.first.reset();
-    eng.second.second.reset();
+    eng.second->cuda_engine.reset();
+    eng.second->execution_context.reset();
   }
   allocator_.reset();
 }
@@ -440,11 +440,9 @@ nvinfer1::IGpuAllocator* TRTEngineOp::GetAllocator(OpKernelContext* ctx) {
   return allocator_.get();
 }
 
-TRTEngineOp::EngineCtxPair& TRTEngineOp::GetEngine(int batch_size,
+TRTEngineOp::EngineContext* TRTEngineOp::GetEngine(int batch_size,
                                                    OpKernelContext* ctx) {
-  static EngineCtxPair null_pair = {
-      TrtUniquePtrType<nvinfer1::ICudaEngine>(nullptr),
-      TrtUniquePtrType<nvinfer1::IExecutionContext>(nullptr)};
+  static EngineContext empty_context;
   // TODO(sami): This method needs to be re-written to use resource manager and
   // with LRU mechanism option.
   tensorflow::mutex_lock lock(engine_mutex_);
@@ -452,14 +450,14 @@ TRTEngineOp::EngineCtxPair& TRTEngineOp::GetEngine(int batch_size,
   if (static_engine_) {
     if (engine_map_.size()) {
       if (engine_map_.begin()->first >= batch_size) {
-        return engine_map_.begin()->second;
+        return engine_map_.begin()->second.get();
       }
-      return null_pair;
+      return &empty_context;
     }
     TrtUniquePtrType<IRuntime> infer(nvinfer1::createInferRuntime(logger));
     auto allocator = GetAllocator(ctx);
     if (allocator == nullptr) {
-      return null_pair;
+      return &empty_context;
     }
     infer->setGpuAllocator(allocator);
     TrtUniquePtrType<nvinfer1::ICudaEngine> static_engine(
@@ -468,16 +466,16 @@ TRTEngineOp::EngineCtxPair& TRTEngineOp::GetEngine(int batch_size,
                                      PluginFactoryTensorRT::GetInstance()));
     auto raw_static_engine = static_engine.get();
     const auto max_batch_size = raw_static_engine->getMaxBatchSize();
-    engine_map_[max_batch_size] = {
+    engine_map_[max_batch_size] = absl::make_unique<EngineContext>(
         std::move(static_engine),
         TrtUniquePtrType<nvinfer1::IExecutionContext>(
-            raw_static_engine->createExecutionContext())};
+            raw_static_engine->createExecutionContext()));
     // Runtime is safe to delete after engine creation
     serialized_segment_.clear();
     if (max_batch_size < batch_size) {
-      return null_pair;
+      return &empty_context;
     }
-    return engine_map_.at(max_batch_size);
+    return engine_map_.at(max_batch_size).get();
   }  // static_engine_
 
   // Handle the dynamic engine case.
@@ -487,7 +485,7 @@ TRTEngineOp::EngineCtxPair& TRTEngineOp::GetEngine(int batch_size,
     nvinfer1::IGpuAllocator* allocator = nullptr;
     allocator = GetAllocator(ctx);
     if (allocator == nullptr) {
-      return null_pair;
+      return &empty_context;
     }
     std::vector<tensorflow::PartialTensorShape> shapes;
     for (int i = 0; i < ctx->num_inputs(); ++i) {
@@ -508,18 +506,19 @@ TRTEngineOp::EngineCtxPair& TRTEngineOp::GetEngine(int batch_size,
         // This means it fail to build the engine even when the network is built
         // successfully, probably due to internal issues. In this case we don't
         // retry in the future.
-        engine_map_[batch_size] = {nullptr, nullptr};
+        engine_map_[batch_size] = absl::make_unique<EngineContext>();
       }
       LOG(WARNING) << "Engine creation for batch size " << batch_size
                    << " failed " << status;
-      return null_pair;
+      return &empty_context;
     }
     VLOG(1) << "Conversion is done";
     TrtUniquePtrType<nvinfer1::IExecutionContext> exec_context(
         engine->createExecutionContext());
-    engine_map_[batch_size] = {std::move(engine), std::move(exec_context)};
+    engine_map_[batch_size] = absl::make_unique<EngineContext>(
+        std::move(engine), std::move(exec_context));
   }
-  return engine_map_.at(batch_size);
+  return engine_map_.at(batch_size).get();
 }
 
 tensorflow::Status TRTEngineOp::AllocateCalibrationResources(
diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
index ceb7483642..38ea224063 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
+++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
@@ -27,6 +27,7 @@ limitations under the License.
 #include "tensorflow/core/framework/op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/platform/mutex.h"
+#include "tensorflow/core/platform/thread_annotations.h"
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
@@ -51,6 +52,21 @@ class TRTEngineOp : public AsyncOpKernel {
   ~TRTEngineOp();
 
  private:
+  // TODO(samikama): context should go to a resource manager!
+  struct EngineContext {
+    EngineContext() {}  // Creates an empty context.
+    EngineContext(
+        TrtUniquePtrType<nvinfer1::ICudaEngine>&& input_cuda_engine,
+        TrtUniquePtrType<nvinfer1::IExecutionContext>&& input_execution_context)
+        : cuda_engine(std::move(input_cuda_engine)),
+          execution_context(std::move(input_execution_context)) {}
+
+    mutex mu;
+    TrtUniquePtrType<nvinfer1::ICudaEngine> cuda_engine;
+    TrtUniquePtrType<nvinfer1::IExecutionContext> execution_context
+        GUARDED_BY(mu);
+  };
+
   // Execute calibration
   void ExecuteCalibration(OpKernelContext* ctx, AsyncHelper* helper);
 
@@ -63,18 +79,13 @@ class TRTEngineOp : public AsyncOpKernel {
   // Execute the tensorrt engine. Returns whether we need to retry by running
   // the native segment.
   bool ExecuteTrtEngine(OpKernelContext* ctx, const int num_batch,
-                        nvinfer1::ICudaEngine* trt_engine_ptr,
-                        nvinfer1::IExecutionContext* trt_execution_context_ptr);
+                        EngineContext* engine_context);
 
   // Allocate necessary resources for calibration
   Status AllocateCalibrationResources(OpKernelContext* ctx,
                                       TRTCalibrationResource** cr);
 
-  // TODO(samikama): context should go to a resource manager!
-  typedef std::pair<TrtUniquePtrType<nvinfer1::ICudaEngine>,
-                    TrtUniquePtrType<nvinfer1::IExecutionContext>>
-      EngineCtxPair;
-  EngineCtxPair& GetEngine(int batch_size, OpKernelContext* ctx);
+  EngineContext* GetEngine(int batch_size, OpKernelContext* ctx);
 
   // Return engine batch closest to input batch.
   int GetEngineBatch(OpKernelContext* ctx);
@@ -82,7 +93,7 @@ class TRTEngineOp : public AsyncOpKernel {
   nvinfer1::IGpuAllocator* GetAllocator(OpKernelContext* ctx);
 
   // map to keep engines and their execution context for given batch size.
-  std::unordered_map<int, EngineCtxPair> engine_map_;
+  std::unordered_map<int, std::unique_ptr<EngineContext>> engine_map_;
   std::vector<string> input_nodes_;
   std::vector<string> output_nodes_;
 
-- 
GitLab


From 67c18c4f8be9544dac4f65e932327872228b8cec Mon Sep 17 00:00:00 2001
From: Francois Chollet <fchollet@google.com>
Date: Fri, 11 Jan 2019 14:50:39 -0800
Subject: [PATCH 0622/1095] Increase coverage of Keras integration tests across
 model types and runtime modes.

PiperOrigin-RevId: 228949803
---
 tensorflow/python/keras/BUILD               |   3 +-
 tensorflow/python/keras/integration_test.py | 446 +++++++-------------
 2 files changed, 164 insertions(+), 285 deletions(-)

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index d36aaaf632..d1ec5f0ac3 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -333,9 +333,8 @@ tf_py_test(
         "@absl_py//absl/testing:parameterized",
         "//third_party/py/numpy",
         "//tensorflow/python:client_testlib",
-        "//tensorflow/python:layers",
-        "//tensorflow/python:nn",
     ],
+    shard_count = 12,
     tags = ["notsan"],
 )
 
diff --git a/tensorflow/python/keras/integration_test.py b/tensorflow/python/keras/integration_test.py
index fbe3508f07..febee44cd4 100644
--- a/tensorflow/python/keras/integration_test.py
+++ b/tensorflow/python/keras/integration_test.py
@@ -22,299 +22,179 @@ import numpy as np
 
 from tensorflow.python import keras
 from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import test_util
+from tensorflow.python.keras import keras_parameterized
 from tensorflow.python.keras import testing_utils
-from tensorflow.python.layers import core as tf_core_layers
-from tensorflow.python.ops import nn
 from tensorflow.python.ops import rnn_cell
 from tensorflow.python.platform import test
 
 
-class KerasIntegrationTest(test.TestCase):
+@keras_parameterized.run_with_all_model_types
+@keras_parameterized.run_all_keras_modes
+class VectorClassificationIntegrationTest(keras_parameterized.TestCase):
+
+  def test_vector_classification(self):
+    np.random.seed(1337)
+    (x_train, y_train), _ = testing_utils.get_test_data(
+        train_samples=150,
+        test_samples=0,
+        input_shape=(10,),
+        num_classes=2)
+    y_train = keras.utils.to_categorical(y_train)
+
+    model = testing_utils.get_model_from_layers(
+        [keras.layers.Dense(16, activation='relu'),
+         keras.layers.Dropout(0.1),
+         keras.layers.Dense(y_train.shape[-1], activation='softmax')],
+        input_shape=x_train.shape[1:])
+    model.compile(loss='categorical_crossentropy',
+                  optimizer=keras.optimizer_v2.adam.Adam(0.01),
+                  metrics=['accuracy'],
+                  run_eagerly=testing_utils.should_run_eagerly())
+    history = model.fit(x_train, y_train, epochs=10, batch_size=16,
+                        validation_data=(x_train, y_train),
+                        verbose=2)
+    self.assertGreater(history.history['val_acc'][-1], 0.7)
+    _, val_acc = model.evaluate(x_train, y_train)
+    self.assertAlmostEqual(history.history['val_acc'][-1], val_acc)
+    predictions = model.predict(x_train)
+    self.assertEqual(predictions.shape, (x_train.shape[0], 2))
 
-  def test_version(self):
-    self.assertTrue(keras.__version__.endswith('-tf'))
-
-  def test_vector_classification_sequential(self):
-    with self.cached_session():
-      np.random.seed(1337)
-      (x_train, y_train), _ = testing_utils.get_test_data(
-          train_samples=100,
-          test_samples=0,
-          input_shape=(10,),
-          num_classes=2)
-      y_train = keras.utils.to_categorical(y_train)
-
-      model = keras.models.Sequential([
-          keras.layers.Dense(16,
-                             activation='relu',
-                             input_shape=x_train.shape[1:]),
-          keras.layers.Dropout(0.1),
-          keras.layers.Dense(y_train.shape[-1], activation='softmax')
-      ])
-      model.compile(loss='categorical_crossentropy',
-                    optimizer=keras.optimizers.Adam(lr=0.1),
-                    metrics=['accuracy'])
-      history = model.fit(x_train, y_train, epochs=10, batch_size=16,
-                          validation_data=(x_train, y_train),
-                          verbose=2)
-      self.assertGreater(history.history['val_acc'][-1], 0.7)
-
-  @test_util.run_deprecated_v1
-  def test_vector_classification_functional(self):
-    with self.cached_session():
-      np.random.seed(1337)
-      (x_train, y_train), _ = testing_utils.get_test_data(
-          train_samples=100,
-          test_samples=0,
-          input_shape=(20,),
-          num_classes=2)
-      y_train = keras.utils.to_categorical(y_train)
-
-      inputs = keras.layers.Input(shape=x_train.shape[1:])
-      x = keras.layers.Dense(16, activation='relu')(inputs)
-      x = keras.layers.Dropout(0.1)(x)
-      outputs = keras.layers.Dense(y_train.shape[-1], activation='softmax')(x)
-
-      model = keras.models.Model(inputs, outputs)
-      model.compile(loss='categorical_crossentropy',
-                    optimizer=keras.optimizers.Adam(lr=0.1),
-                    metrics=['accuracy'])
-      history = model.fit(x_train, y_train, epochs=10, batch_size=16,
-                          validation_data=(x_train, y_train),
-                          verbose=2)
-      self.assertGreater(history.history['val_acc'][-1], 0.7)
-
-  @test_util.run_deprecated_v1
-  def test_temporal_classification_sequential(self):
-    with self.cached_session():
-      np.random.seed(1337)
-      (x_train, y_train), _ = testing_utils.get_test_data(
-          train_samples=100,
-          test_samples=0,
-          input_shape=(4, 10),
-          num_classes=2)
-      y_train = keras.utils.to_categorical(y_train)
-
-      model = keras.models.Sequential()
-      model.add(keras.layers.LSTM(5, return_sequences=True,
-                                  input_shape=x_train.shape[1:]))
-      model.add(keras.layers.GRU(y_train.shape[-1], activation='softmax'))
-      model.compile(loss='categorical_crossentropy',
-                    optimizer=keras.optimizers.Adam(lr=0.1),
-                    metrics=['accuracy'])
-      history = model.fit(x_train, y_train, epochs=15, batch_size=16,
-                          validation_data=(x_train, y_train),
-                          verbose=2)
-      self.assertGreater(history.history['val_acc'][-1], 0.7)
-
-  @test_util.run_deprecated_v1
-  def test_temporal_classification_sequential_tf_rnn(self):
-    with self.cached_session():
-      np.random.seed(1337)
-      (x_train, y_train), _ = testing_utils.get_test_data(
-          train_samples=100,
-          test_samples=0,
-          input_shape=(4, 10),
-          num_classes=2)
-      y_train = keras.utils.to_categorical(y_train)
-
-      model = keras.models.Sequential()
-      model.add(keras.layers.RNN(rnn_cell.LSTMCell(5), return_sequences=True,
-                                 input_shape=x_train.shape[1:]))
-      model.add(keras.layers.RNN(rnn_cell.GRUCell(y_train.shape[-1],
-                                                  activation='softmax',
-                                                  dtype=dtypes.float32)))
-      model.compile(loss='categorical_crossentropy',
-                    optimizer=keras.optimizers.Adam(lr=0.1),
-                    metrics=['accuracy'])
-      history = model.fit(x_train, y_train, epochs=15, batch_size=16,
-                          validation_data=(x_train, y_train),
-                          verbose=2)
-      self.assertGreater(history.history['val_acc'][-1], 0.7)
-
-  def test_image_classification_sequential(self):
-    with self.cached_session():
-      np.random.seed(1337)
-      (x_train, y_train), _ = testing_utils.get_test_data(
-          train_samples=100,
-          test_samples=0,
-          input_shape=(12, 12, 3),
-          num_classes=2)
-      y_train = keras.utils.to_categorical(y_train)
-
-      model = keras.models.Sequential()
-      model.add(keras.layers.Conv2D(
-          4, 3,
-          padding='same',
-          activation='relu',
-          input_shape=x_train.shape[1:]))
-      model.add(keras.layers.Conv2D(
-          8, 3,
-          padding='same',
-          activation='relu'))
-      model.add(keras.layers.Conv2D(
-          16, 3,
-          padding='same',
-          activation='relu'))
-      model.add(keras.layers.Flatten())
-      model.add(keras.layers.Dense(y_train.shape[-1], activation='softmax'))
-      model.compile(loss='categorical_crossentropy',
-                    optimizer=keras.optimizers.SGD(lr=0.01, momentum=0.8),
-                    metrics=['accuracy'])
-      history = model.fit(x_train, y_train, epochs=10, batch_size=16,
-                          validation_data=(x_train, y_train),
-                          verbose=2)
-      self.assertGreater(history.history['val_acc'][-1], 0.7)
-
-  def test_video_classification_functional(self):
-    with self.cached_session():
-      np.random.seed(1337)
-      (x_train, y_train), _ = testing_utils.get_test_data(
-          train_samples=100,
-          test_samples=0,
-          input_shape=(4, 8, 8, 3),
-          num_classes=3)
-      y_train = keras.utils.to_categorical(y_train)
-
-      inputs = keras.layers.Input(shape=x_train.shape[1:])
-      x = keras.layers.TimeDistributed(
-          keras.layers.Conv2D(4, 3, activation='relu'))(inputs)
-      x = keras.layers.BatchNormalization()(x)
-      x = keras.layers.TimeDistributed(keras.layers.GlobalMaxPooling2D())(x)
-      x = keras.layers.Conv1D(8, 3, activation='relu')(x)
-      x = keras.layers.Flatten()(x)
-      outputs = keras.layers.Dense(y_train.shape[-1], activation='softmax')(x)
-
-      model = keras.models.Model(inputs, outputs)
-      model.compile(loss='categorical_crossentropy',
-                    optimizer=keras.optimizers.SGD(lr=0.01, momentum=0.8),
-                    metrics=['accuracy'])
-      history = model.fit(x_train, y_train, epochs=10, batch_size=16,
-                          validation_data=(x_train, y_train),
-                          verbose=2)
-      self.assertGreater(history.history['val_acc'][-1], 0.7)
-
-  def test_vector_classification_shared_sequential(self):
+  def test_vector_classification_shared_model(self):
     # Test that Sequential models that feature internal updates
     # and internal losses can be shared.
-    with self.cached_session():
-      np.random.seed(1337)
-      (x_train, y_train), _ = testing_utils.get_test_data(
-          train_samples=100,
-          test_samples=0,
-          input_shape=(10,),
-          num_classes=2)
-      y_train = keras.utils.to_categorical(y_train)
-
-      base_model = keras.models.Sequential([
-          keras.layers.Dense(16,
-                             activation='relu',
-                             kernel_regularizer=keras.regularizers.l2(1e-5),
-                             bias_regularizer=keras.regularizers.l2(1e-5),
-                             input_shape=x_train.shape[1:]),
-          keras.layers.BatchNormalization(),
-      ])
-      x = keras.layers.Input(x_train.shape[1:])
-      y = base_model(x)
-      y = keras.layers.Dense(y_train.shape[-1], activation='softmax')(y)
-      model = keras.models.Model(x, y)
-      model.compile(loss='categorical_crossentropy',
-                    optimizer=keras.optimizers.Adam(lr=0.1),
-                    metrics=['accuracy'])
+    np.random.seed(1337)
+    (x_train, y_train), _ = testing_utils.get_test_data(
+        train_samples=150,
+        test_samples=0,
+        input_shape=(10,),
+        num_classes=2)
+    y_train = keras.utils.to_categorical(y_train)
+
+    base_model = testing_utils.get_model_from_layers(
+        [keras.layers.Dense(16,
+                            activation='relu',
+                            kernel_regularizer=keras.regularizers.l2(1e-5),
+                            bias_regularizer=keras.regularizers.l2(1e-5)),
+         keras.layers.BatchNormalization()],
+        input_shape=x_train.shape[1:])
+    x = keras.layers.Input(x_train.shape[1:])
+    y = base_model(x)
+    y = keras.layers.Dense(y_train.shape[-1], activation='softmax')(y)
+    model = keras.models.Model(x, y)
+    model.compile(loss='categorical_crossentropy',
+                  optimizer=keras.optimizer_v2.adam.Adam(0.01),
+                  metrics=['accuracy'],
+                  run_eagerly=testing_utils.should_run_eagerly())
+    if not testing_utils.should_run_eagerly():
       self.assertEqual(len(model.losses), 2)
       self.assertEqual(len(model.updates), 2)
-      history = model.fit(x_train, y_train, epochs=10, batch_size=16,
-                          validation_data=(x_train, y_train),
-                          verbose=2)
-      self.assertGreater(history.history['val_acc'][-1], 0.7)
-
-  def test_vector_classification_shared_model(self):
-    # Test that functional models that feature internal updates
-    # and internal losses can be shared.
-    with self.cached_session():
-      np.random.seed(1337)
-      (x_train, y_train), _ = testing_utils.get_test_data(
-          train_samples=100,
-          test_samples=0,
-          input_shape=(10,),
-          num_classes=2)
-      y_train = keras.utils.to_categorical(y_train)
-
-      inputs = keras.layers.Input(x_train.shape[1:])
-      x = keras.layers.Dense(16,
-                             activation='relu',
-                             kernel_regularizer=keras.regularizers.l2(1e-5),
-                             bias_regularizer=keras.regularizers.l2(1e-5),
-                             input_shape=x_train.shape[1:])(inputs)
-      x = keras.layers.BatchNormalization()(x)
-      base_model = keras.models.Model(inputs, x)
-
-      x = keras.layers.Input(x_train.shape[1:])
-      y = base_model(x)
-      y = keras.layers.Dense(y_train.shape[-1], activation='softmax')(y)
-      model = keras.models.Model(x, y)
-      model.compile(loss='categorical_crossentropy',
-                    optimizer=keras.optimizers.Adam(lr=0.1),
-                    metrics=['accuracy'])
-      history = model.fit(x_train, y_train, epochs=10, batch_size=16,
-                          validation_data=(x_train, y_train),
-                          verbose=2)
-      self.assertGreater(history.history['val_acc'][-1], 0.7)
-
-  def test_embedding_with_clipnorm(self):
-    with self.cached_session():
-      model = keras.models.Sequential()
-      model.add(keras.layers.Embedding(input_dim=1, output_dim=1))
-      model.compile(optimizer=keras.optimizers.SGD(clipnorm=0.1), loss='mse')
-      model.fit(np.array([[0]]), np.array([[[0.5]]]), epochs=1)
-
-  def test_using_tf_layers_in_keras_sequential_model(self):
-    with self.cached_session():
-      np.random.seed(1337)
-      (x_train, y_train), _ = testing_utils.get_test_data(
-          train_samples=100,
-          test_samples=0,
-          input_shape=(10,),
-          num_classes=2)
-
-      model = keras.models.Sequential()
-      model.add(tf_core_layers.Dense(32, activation=nn.relu, input_shape=(10,)))
-      model.add(tf_core_layers.Dense(2, activation=nn.softmax))
-      model.summary()
-
-      y_train = keras.utils.to_categorical(y_train)
-      model.compile(loss='categorical_crossentropy',
-                    optimizer=keras.optimizers.Adam(lr=0.1),
-                    metrics=['accuracy'])
-      history = model.fit(x_train, y_train, epochs=10, batch_size=16,
-                          validation_data=(x_train, y_train),
-                          verbose=0)
-      self.assertGreater(history.history['val_acc'][-1], 0.7)
-
-  def test_using_tf_layers_in_keras_functional_model(self):
-    with self.cached_session():
-      np.random.seed(1337)
-      (x_train, y_train), _ = testing_utils.get_test_data(
-          train_samples=100,
-          test_samples=0,
-          input_shape=(10,),
-          num_classes=2)
-      y_train = keras.utils.to_categorical(y_train)
-
-      inputs = keras.Input(shape=(10,))
-      x = tf_core_layers.Dense(32, activation=nn.relu)(inputs)
-      outputs = tf_core_layers.Dense(2, activation=nn.softmax)(x)
-      model = keras.Model(inputs, outputs)
-      model.summary()
-
-      model.compile(loss='categorical_crossentropy',
-                    optimizer=keras.optimizers.Adam(lr=0.1),
-                    metrics=['accuracy'])
-      history = model.fit(x_train, y_train, epochs=10, batch_size=16,
-                          validation_data=(x_train, y_train),
-                          verbose=0)
-      self.assertGreater(history.history['val_acc'][-1], 0.7)
+    history = model.fit(x_train, y_train, epochs=10, batch_size=16,
+                        validation_data=(x_train, y_train),
+                        verbose=2)
+    self.assertGreater(history.history['val_acc'][-1], 0.7)
+    _, val_acc = model.evaluate(x_train, y_train)
+    self.assertAlmostEqual(history.history['val_acc'][-1], val_acc)
+    predictions = model.predict(x_train)
+    self.assertEqual(predictions.shape, (x_train.shape[0], 2))
+
+
+# See b/122473407
+@keras_parameterized.run_all_keras_modes(always_skip_v1=True)
+class TimeseriesClassificationIntegrationTest(keras_parameterized.TestCase):
+
+  @keras_parameterized.run_with_all_model_types
+  def test_timeseries_classification(self):
+    np.random.seed(1337)
+    (x_train, y_train), _ = testing_utils.get_test_data(
+        train_samples=250,
+        test_samples=0,
+        input_shape=(4, 10),
+        num_classes=2)
+    y_train = keras.utils.to_categorical(y_train)
+
+    layers = [
+        keras.layers.LSTM(5, return_sequences=True),
+        keras.layers.GRU(y_train.shape[-1], activation='softmax')
+    ]
+    model = testing_utils.get_model_from_layers(
+        layers, input_shape=x_train.shape[1:])
+    model.compile(loss='categorical_crossentropy',
+                  optimizer=keras.optimizer_v2.adam.Adam(0.01),
+                  metrics=['accuracy'],
+                  run_eagerly=testing_utils.should_run_eagerly())
+    history = model.fit(x_train, y_train, epochs=15, batch_size=16,
+                        validation_data=(x_train, y_train),
+                        verbose=2)
+    self.assertGreater(history.history['val_acc'][-1], 0.7)
+    _, val_acc = model.evaluate(x_train, y_train)
+    self.assertAlmostEqual(history.history['val_acc'][-1], val_acc)
+    predictions = model.predict(x_train)
+    self.assertEqual(predictions.shape, (250, 2))
+
+  def test_timeseries_classification_sequential_tf_rnn(self):
+    np.random.seed(1337)
+    (x_train, y_train), _ = testing_utils.get_test_data(
+        train_samples=150,
+        test_samples=0,
+        input_shape=(4, 10),
+        num_classes=2)
+    y_train = keras.utils.to_categorical(y_train)
+
+    model = keras.models.Sequential()
+    model.add(keras.layers.RNN(rnn_cell.LSTMCell(5), return_sequences=True,
+                               input_shape=x_train.shape[1:]))
+    model.add(keras.layers.RNN(rnn_cell.GRUCell(y_train.shape[-1],
+                                                activation='softmax',
+                                                dtype=dtypes.float32)))
+    model.compile(loss='categorical_crossentropy',
+                  optimizer=keras.optimizer_v2.adam.Adam(0.01),
+                  metrics=['accuracy'],
+                  run_eagerly=testing_utils.should_run_eagerly())
+    history = model.fit(x_train, y_train, epochs=15, batch_size=16,
+                        validation_data=(x_train, y_train),
+                        verbose=2)
+    self.assertGreater(history.history['val_acc'][-1], 0.7)
+    _, val_acc = model.evaluate(x_train, y_train)
+    self.assertAlmostEqual(history.history['val_acc'][-1], val_acc)
+    predictions = model.predict(x_train)
+    self.assertEqual(predictions.shape, (150, 2))
+
+
+@keras_parameterized.run_with_all_model_types
+@keras_parameterized.run_all_keras_modes
+class ImageClassificationIntegrationTest(keras_parameterized.TestCase):
+
+  def test_image_classification(self):
+    np.random.seed(1337)
+    (x_train, y_train), _ = testing_utils.get_test_data(
+        train_samples=250,
+        test_samples=0,
+        input_shape=(10, 10, 3),
+        num_classes=2)
+    y_train = keras.utils.to_categorical(y_train)
+
+    layers = [
+        keras.layers.Conv2D(4, 3, padding='same', activation='relu'),
+        keras.layers.Conv2D(8, 3, padding='same'),
+        keras.layers.BatchNormalization(),
+        keras.layers.Conv2D(8, 3, padding='same'),
+        keras.layers.Flatten(),
+        keras.layers.Dense(y_train.shape[-1], activation='softmax')
+    ]
+    model = testing_utils.get_model_from_layers(
+        layers, input_shape=x_train.shape[1:])
+    model.compile(loss='categorical_crossentropy',
+                  optimizer=keras.optimizer_v2.adam.Adam(0.01),
+                  metrics=['accuracy'],
+                  run_eagerly=testing_utils.should_run_eagerly())
+    history = model.fit(x_train, y_train, epochs=10, batch_size=16,
+                        validation_data=(x_train, y_train),
+                        verbose=2)
+    self.assertGreater(history.history['val_acc'][-1], 0.7)
+    _, val_acc = model.evaluate(x_train, y_train)
+    self.assertAlmostEqual(history.history['val_acc'][-1], val_acc)
+    predictions = model.predict(x_train)
+    self.assertEqual(predictions.shape, (250, 2))
 
 
 if __name__ == '__main__':
-- 
GitLab


From 892ea4dcc74be16fbfb0e31cedb3a41b807bb6b1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 11 Jan 2019 15:00:37 -0800
Subject: [PATCH 0623/1095] Add annotation to eager mode.

With these annotations, when user turns on tracing, it will display op name like "BiasAddGrad" instead of kernel name "void tensorflow::BiasGradNCHW_SharedAtomics<float>(float const*, float*, int, int, int, int)".

PiperOrigin-RevId: 228951394
---
 .../core/common_runtime/eager/kernel_and_device.cc   | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/common_runtime/eager/kernel_and_device.cc b/tensorflow/core/common_runtime/eager/kernel_and_device.cc
index 354f96c440..4096763b11 100644
--- a/tensorflow/core/common_runtime/eager/kernel_and_device.cc
+++ b/tensorflow/core/common_runtime/eager/kernel_and_device.cc
@@ -28,6 +28,7 @@ limitations under the License.
 #include "tensorflow/core/lib/gtl/stl_util.h"
 #include "tensorflow/core/platform/fingerprint.h"
 #include "tensorflow/core/platform/mutex.h"
+#include "tensorflow/core/platform/tracing.h"
 #include "tensorflow/core/public/version.h"
 #include "tensorflow/core/util/tensor_slice_reader_cache.h"
 
@@ -129,7 +130,16 @@ Status KernelAndDevice::Run(ScopedStepContainer* step_container,
     device_->ComputeAsync(async, &context, [&done]() { done.Notify(); });
     done.WaitForNotification();
   } else {
-    device_->Compute(kernel_.get(), &context);
+    const string& op_name = kernel_->name();
+    // If tracing if off, the overheads of ScopedAnnotation and ScopedActivity
+    // are negligible.
+    if (device_->TraceUsingAnnotations()) {
+      tracing::ScopedAnnotation activity(op_name, kernel_->type_string());
+      device_->Compute(kernel_.get(), &context);
+    } else {
+      tracing::ScopedActivity activity(op_name, kernel_->type_string());
+      device_->Compute(kernel_.get(), &context);
+    }
   }
   if (!context.status().ok()) return context.status();
 
-- 
GitLab


From e25754186b415cb668632a938d7c24ed2d925bb8 Mon Sep 17 00:00:00 2001
From: Michael Kuperstein <mkuper@google.com>
Date: Fri, 11 Jan 2019 15:07:17 -0800
Subject: [PATCH 0624/1095] [XLA] Move more XlaBuilder users over to the new
 DS/DUS API

Move more tests and bridge op implementations over to the new
span-of-scalars API for DynamicSlice and DynamicUpdateSlice.

Also fix a bug in DynamicIndexSplitter to handle the case of an
already-empty index span.

PiperOrigin-RevId: 228952769
---
 .../compiler/tf2xla/kernels/random_ops.cc     | 19 ++++++---
 .../tf2xla/kernels/tensor_array_ops.cc        | 23 ++++++-----
 .../tf2xla/kernels/tensor_list_ops.cc         | 13 +++---
 .../xla/service/dynamic_index_splitter.cc     | 22 +++++-----
 .../xla/service/hlo_cost_analysis_test.cc     |  5 ++-
 .../compiler/xla/tests/dot_operation_test.cc  | 40 +++++++++++--------
 6 files changed, 70 insertions(+), 52 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/random_ops.cc b/tensorflow/compiler/tf2xla/kernels/random_ops.cc
index 2d92056e4f..01b047f732 100644
--- a/tensorflow/compiler/tf2xla/kernels/random_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/random_ops.cc
@@ -160,17 +160,24 @@ class RandomShuffleOp : public XlaOpKernel {
         -> xla::StatusOr<std::vector<xla::XlaOp>> {
       auto swaps = loop_vars[0];
       auto indices = loop_vars[1];
-      i = xla::Reshape(i, {1});
+      // TODO(b/118437727): The absl::Span nonsense is only necessary because
+      // the deprecated overload creates ambiguity for the single-element span
+      // case. Remove it once the deprecated overload is gone.
       // temp = indices[i]
-      auto temp = xla::DynamicSlice(indices, i, {1});
+      auto temp =
+          xla::DynamicSlice(indices, absl::Span<const xla::XlaOp>({i}), {1});
       // swap_index = swaps[i]
-      auto swap_index = xla::DynamicSlice(swaps, i, {1});
+      auto swap_index = xla::Reshape(
+          xla::DynamicSlice(swaps, absl::Span<const xla::XlaOp>({i}), {1}), {});
       // swap_value = indices[swaps[i]]
-      auto swap_value = xla::DynamicSlice(indices, swap_index, {1});
+      auto swap_value = xla::DynamicSlice(
+          indices, absl::Span<const xla::XlaOp>({swap_index}), {1});
       // indices[i] = indices[swaps[i]]
-      indices = xla::DynamicUpdateSlice(indices, swap_value, i);
+      indices = xla::DynamicUpdateSlice(indices, swap_value,
+                                        absl::Span<const xla::XlaOp>({i}));
       // indices[swaps[i]] = temp
-      indices = xla::DynamicUpdateSlice(indices, temp, swap_index);
+      indices = xla::DynamicUpdateSlice(
+          indices, temp, absl::Span<const xla::XlaOp>({swap_index}));
       return std::vector<xla::XlaOp>{swaps, indices};
     };
     // for i in range(n):
diff --git a/tensorflow/compiler/tf2xla/kernels/tensor_array_ops.cc b/tensorflow/compiler/tf2xla/kernels/tensor_array_ops.cc
index 939d7e1951..22e64789a3 100644
--- a/tensorflow/compiler/tf2xla/kernels/tensor_array_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/tensor_array_ops.cc
@@ -123,7 +123,8 @@ Status GetTensorArrayShape(const XlaResource* resource,
 xla::XlaOp DynamicAddSlice(xla::XlaBuilder* builder, const xla::XlaOp& operand,
                            const xla::XlaOp& update,
                            absl::Span<const int64> update_dims,
-                           const xla::XlaOp& start_indices, DataType dtype) {
+                           absl::Span<const xla::XlaOp> start_indices,
+                           DataType dtype) {
   xla::XlaOp current = xla::DynamicSlice(operand, start_indices, update_dims);
   xla::XlaOp sum =
       dtype == DT_BOOL ? xla::Or(current, update) : xla::Add(current, update);
@@ -212,9 +213,9 @@ class TensorArrayWriteOp : public XlaOpKernel {
     xla::XlaOp flow = ctx->Input(3);
 
     // start_indices of the DynamicUpdateSlice are [index, 0, 0, ..., 0].
-    auto start_indices =
-        xla::Pad(xla::Reshape(index, {1}), xla::ConstantR0<int32>(b, 0),
-                 xla::MakeEdgePaddingConfig({{0, elem_shape.dims()}}));
+    std::vector<xla::XlaOp> start_indices(elem_shape.dims() + 1,
+                                          xla::ConstantR0<int32>(b, 0));
+    start_indices[0] = index;
 
     TensorShape slice_shape = elem_shape;
     slice_shape.InsertDim(0, 1LL);
@@ -263,9 +264,9 @@ class TensorArrayReadOp : public XlaOpKernel {
     xla::XlaOp index = ctx->Input(1);
 
     // start_indices of the DynamicSlice are [index, 0, 0, ..., 0].
-    auto start_indices =
-        xla::Pad(xla::Reshape(index, {1}), xla::ConstantR0<int32>(b, 0),
-                 xla::MakeEdgePaddingConfig({{0, ta_shape.dims() - 1}}));
+    std::vector<xla::XlaOp> start_indices(ta_shape.dims(),
+                                          xla::ConstantR0<int32>(b, 0));
+    start_indices[0] = index;
 
     auto slice_shape = ta_shape.dim_sizes();
     slice_shape[0] = 1LL;
@@ -419,10 +420,10 @@ class TensorArrayScatterOp : public XlaOpKernel {
         auto slice = xla::Slice(value, value_starts, value_ends, value_strides);
 
         // start_indices of the DynamicUpdateSlice are [index, 0, 0, ..., 0].
-        auto index = xla::Slice(indices, {i}, {i + 1}, {1});
-        auto start_indices =
-            xla::Pad(xla::Reshape(index, {1}), xla::ConstantR0<int32>(b, 0),
-                     xla::MakeEdgePaddingConfig({{0, elem_shape.dims()}}));
+        auto index = xla::Reshape(xla::Slice(indices, {i}, {i + 1}, {1}), {});
+        std::vector<xla::XlaOp> start_indices(elem_shape.dims() + 1,
+                                              xla::ConstantR0<int32>(b, 0));
+        start_indices[0] = index;
         ta = DynamicAddSlice(b, ta, slice, slice_dims, start_indices, dtype_);
       }
     }
diff --git a/tensorflow/compiler/tf2xla/kernels/tensor_list_ops.cc b/tensorflow/compiler/tf2xla/kernels/tensor_list_ops.cc
index 747f591a1b..ba3d7eb00d 100644
--- a/tensorflow/compiler/tf2xla/kernels/tensor_list_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/tensor_list_ops.cc
@@ -155,9 +155,9 @@ class TensorListPushBackOp : public XlaOpKernel {
     xla::XlaOp value = ctx->Input(1);
 
     // start_indices of the DynamicUpdateSlice are [index, 0, 0, ..., 0].
-    auto start_indices =
-        xla::Pad(xla::Reshape(index, {1}), xla::ConstantR0<int32>(b, 0),
-                 xla::MakeEdgePaddingConfig({{0, elem_shape.dims()}}));
+    std::vector<xla::XlaOp> start_indices(elem_shape.dims() + 1,
+                                          xla::ConstantR0<int32>(b, 0));
+    start_indices[0] = index;
 
     TensorShape slice_shape = elem_shape;
     slice_shape.InsertDim(0, 1LL);
@@ -197,10 +197,9 @@ class TensorListPopBackOp : public XlaOpKernel {
     index = index - xla::ConstantR0<int32>(b, 1);
 
     // start_indices of the DynamicSlice are [index, 0, 0, ..., 0].
-    auto start_indices =
-        xla::Pad(xla::Reshape(index, {1}), xla::ConstantR0<int32>(b, 0),
-                 xla::MakeEdgePaddingConfig({{0, shape.dims() - 1}}));
-
+    std::vector<xla::XlaOp> start_indices(shape.dims(),
+                                          xla::ConstantR0<int32>(b, 0));
+    start_indices[0] = index;
     auto slice_shape = shape.dim_sizes();
     slice_shape[0] = 1LL;
 
diff --git a/tensorflow/compiler/xla/service/dynamic_index_splitter.cc b/tensorflow/compiler/xla/service/dynamic_index_splitter.cc
index ede4a5afdb..e34adfd2d2 100644
--- a/tensorflow/compiler/xla/service/dynamic_index_splitter.cc
+++ b/tensorflow/compiler/xla/service/dynamic_index_splitter.cc
@@ -45,17 +45,10 @@ StatusOr<bool> DynamicIndexSplitter::Run(HloModule* module) {
       }
       auto parent = dynamic_op->parent();
       bool is_update = dynamic_op->opcode() == HloOpcode::kDynamicUpdateSlice;
-      int64 index_operand_number = Cast<HloDynamicIndexInstruction>(dynamic_op)
-                                       ->first_index_operand_number();
-      auto index_operand = dynamic_op->mutable_operand(index_operand_number);
-      if (ShapeUtil::IsScalar(index_operand->shape())) {
-        // This DS/DUS already uses scalar indices.
-        continue;
-      }
-      TF_RET_CHECK(index_operand->shape().rank() == 1);
-      int64 num_indices = index_operand->shape().dimensions(0);
+      int64 num_indices = dynamic_op->operand(0)->shape().rank();
+
       if (num_indices == 0) {
-        // If the operand dimension is 0, directly replace R0 DS/DUS with the
+        // If the operand rank is 0, directly replace R0 DS/DUS with the
         // operand (for DS) or update (for DUS).
         if (is_update) {
           TF_CHECK_OK(parent->ReplaceInstruction(
@@ -67,6 +60,15 @@ StatusOr<bool> DynamicIndexSplitter::Run(HloModule* module) {
         changed = true;
         continue;
       }
+
+      int64 index_operand_number = Cast<HloDynamicIndexInstruction>(dynamic_op)
+                                       ->first_index_operand_number();
+      auto index_operand = dynamic_op->mutable_operand(index_operand_number);
+      if (ShapeUtil::IsScalar(index_operand->shape())) {
+        // This DS/DUS already uses scalar indices.
+        continue;
+      }
+      TF_RET_CHECK(index_operand->shape().rank() == 1);
       auto index_element_type = index_operand->shape().element_type();
       std::vector<HloInstruction*> index_array;
       for (int64 dim = 0; dim < num_indices; ++dim) {
diff --git a/tensorflow/compiler/xla/service/hlo_cost_analysis_test.cc b/tensorflow/compiler/xla/service/hlo_cost_analysis_test.cc
index ff32faf298..82dd57d3c6 100644
--- a/tensorflow/compiler/xla/service/hlo_cost_analysis_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_cost_analysis_test.cc
@@ -529,7 +529,8 @@ TEST_F(HloCostAnalysisTest, DynamicSlice) {
   // Test the analysis on a slice.
   XlaBuilder builder("dynamic-slice");
   auto x = Parameter(&builder, 0, ShapeUtil::MakeShape(F32, {2}), "x");
-  DynamicSlice(x, ConstantR1<int32>(&builder, {1}), {1});
+  DynamicSlice(x, absl::Span<const XlaOp>({ConstantR0<int32>(&builder, 1)}),
+               {1});
   auto hlo_module = BuildHloGraph(&builder);
 
   // Run HLO cost analysis.
@@ -545,7 +546,7 @@ TEST_F(HloCostAnalysisTest, DynamicUpdateSlice) {
   XlaBuilder builder("dynamic-update-slice");
   auto x = Parameter(&builder, 0, ShapeUtil::MakeShape(F32, {2}), "x");
   DynamicUpdateSlice(x, ConstantR1<float>(&builder, {1.0}),
-                     ConstantR1<int32>(&builder, {1}));
+                     absl::Span<const XlaOp>({ConstantR0<int32>(&builder, 1)}));
   auto hlo_module = BuildHloGraph(&builder);
 
   // Run HLO cost analysis.
diff --git a/tensorflow/compiler/xla/tests/dot_operation_test.cc b/tensorflow/compiler/xla/tests/dot_operation_test.cc
index 2e02968ac5..33d816292c 100644
--- a/tensorflow/compiler/xla/tests/dot_operation_test.cc
+++ b/tensorflow/compiler/xla/tests/dot_operation_test.cc
@@ -919,8 +919,9 @@ XLA_TEST_F(DotOperationTest, DotOfGatherOptimizationWithConstRHSClassicMM) {
   XlaBuilder builder(TestName());
   auto lhs_constant = ConstantR2FromArray2D(&builder, *constant_lhs_array);
   auto rhs_constant = ConstantR2FromArray2D(&builder, *constant_rhs_array);
-  auto start_constant = ConstantR1<int32>(&builder, {1, 0});
-  auto dynamic_slice = DynamicSlice(lhs_constant, start_constant, {1, 6});
+  auto one = ConstantR0<int32>(&builder, 1);
+  auto zero = ConstantR0<int32>(&builder, 0);
+  auto dynamic_slice = DynamicSlice(lhs_constant, {one, zero}, {1, 6});
 
   DotDimensionNumbers dot_dnums;
   dot_dnums.add_lhs_contracting_dimensions(1);
@@ -946,8 +947,9 @@ XLA_TEST_F(DotOperationTest, DotOfGatherOptimizationWithConstLHSClassicMM) {
   XlaBuilder builder(TestName());
   auto lhs_constant = ConstantR2FromArray2D(&builder, *constant_lhs_array);
   auto rhs_constant = ConstantR2FromArray2D(&builder, *constant_rhs_array);
-  auto start_constant = ConstantR1<int32>(&builder, {0, 1});
-  auto dynamic_slice = DynamicSlice(rhs_constant, start_constant, {6, 1});
+  auto zero = ConstantR0<int32>(&builder, 0);
+  auto one = ConstantR0<int32>(&builder, 1);
+  auto dynamic_slice = DynamicSlice(rhs_constant, {zero, one}, {6, 1});
 
   DotDimensionNumbers dot_dnums;
   dot_dnums.add_lhs_contracting_dimensions(1);
@@ -975,8 +977,9 @@ XLA_TEST_F(DotOperationTest,
   XlaBuilder builder(TestName());
   auto lhs_constant = ConstantR2FromArray2D(&builder, *constant_lhs_array);
   auto rhs_constant = ConstantR2FromArray2D(&builder, *constant_rhs_array);
-  auto start_constant = ConstantR1<int32>(&builder, {0, 1});
-  auto dynamic_slice = DynamicSlice(lhs_constant, start_constant, {6, 1});
+  auto zero = ConstantR0<int32>(&builder, 0);
+  auto one = ConstantR0<int32>(&builder, 1);
+  auto dynamic_slice = DynamicSlice(lhs_constant, {zero, one}, {6, 1});
 
   DotDimensionNumbers dot_dnums;
   dot_dnums.add_lhs_contracting_dimensions(0);
@@ -1002,8 +1005,9 @@ XLA_TEST_F(DotOperationTest, DotOfGatherOptimizationWithConstLHSReverseMM) {
   XlaBuilder builder(TestName());
   auto lhs_constant = ConstantR2FromArray2D(&builder, *constant_lhs_array);
   auto rhs_constant = ConstantR2FromArray2D(&builder, *constant_rhs_array);
-  auto start_constant = ConstantR1<int32>(&builder, {1, 0});
-  auto dynamic_slice = DynamicSlice(rhs_constant, start_constant, {1, 6});
+  auto zero = ConstantR0<int32>(&builder, 0);
+  auto one = ConstantR0<int32>(&builder, 1);
+  auto dynamic_slice = DynamicSlice(rhs_constant, {one, zero}, {1, 6});
 
   DotDimensionNumbers dot_dnums;
   dot_dnums.add_lhs_contracting_dimensions(0);
@@ -1034,8 +1038,9 @@ XLA_TEST_F(DotOperationTest, DotOfGatherOptimizationWithConstRHSRows) {
   XlaBuilder builder(TestName());
   auto lhs_constant = ConstantR2FromArray2D(&builder, *constant_lhs_array);
   auto rhs_constant = ConstantR2FromArray2D(&builder, *constant_rhs_array);
-  auto start_constant = ConstantR1<int32>(&builder, {0, 1});
-  auto dynamic_slice = DynamicSlice(lhs_constant, start_constant, {6, 1});
+  auto zero = ConstantR0<int32>(&builder, 0);
+  auto one = ConstantR0<int32>(&builder, 1);
+  auto dynamic_slice = DynamicSlice(lhs_constant, {zero, one}, {6, 1});
 
   DotDimensionNumbers dot_dnums;
   dot_dnums.add_lhs_contracting_dimensions(0);
@@ -1066,8 +1071,9 @@ XLA_TEST_F(DotOperationTest, DotOfGatherOptimizationWithConstLHSRows) {
   XlaBuilder builder(TestName());
   auto lhs_constant = ConstantR2FromArray2D(&builder, *constant_lhs_array);
   auto rhs_constant = ConstantR2FromArray2D(&builder, *constant_rhs_array);
-  auto start_constant = ConstantR1<int32>(&builder, {0, 1});
-  auto dynamic_slice = DynamicSlice(rhs_constant, start_constant, {6, 1});
+  auto zero = ConstantR0<int32>(&builder, 0);
+  auto one = ConstantR0<int32>(&builder, 1);
+  auto dynamic_slice = DynamicSlice(rhs_constant, {zero, one}, {6, 1});
 
   DotDimensionNumbers dot_dnums;
   dot_dnums.add_lhs_contracting_dimensions(0);
@@ -1090,8 +1096,9 @@ XLA_TEST_F(DotOperationTest, DotOfGatherOptimizationWithConstRHSCols) {
   XlaBuilder builder(TestName());
   auto lhs_constant = ConstantR2FromArray2D(&builder, *constant_lhs_array);
   auto rhs_constant = ConstantR2FromArray2D(&builder, *constant_rhs_array);
-  auto start_constant = ConstantR1<int32>(&builder, {1, 0});
-  auto dynamic_slice = DynamicSlice(lhs_constant, start_constant, {1, 6});
+  auto zero = ConstantR0<int32>(&builder, 0);
+  auto one = ConstantR0<int32>(&builder, 1);
+  auto dynamic_slice = DynamicSlice(lhs_constant, {one, zero}, {1, 6});
 
   DotDimensionNumbers dot_dnums;
   dot_dnums.add_lhs_contracting_dimensions(1);
@@ -1114,8 +1121,9 @@ XLA_TEST_F(DotOperationTest, DotOfGatherOptimizationWithConstLHSCols) {
   XlaBuilder builder(TestName());
   auto lhs_constant = ConstantR2FromArray2D(&builder, *constant_lhs_array);
   auto rhs_constant = ConstantR2FromArray2D(&builder, *constant_rhs_array);
-  auto start_constant = ConstantR1<int32>(&builder, {1, 0});
-  auto dynamic_slice = DynamicSlice(rhs_constant, start_constant, {1, 6});
+  auto zero = ConstantR0<int32>(&builder, 0);
+  auto one = ConstantR0<int32>(&builder, 1);
+  auto dynamic_slice = DynamicSlice(rhs_constant, {one, zero}, {1, 6});
 
   DotDimensionNumbers dot_dnums;
   dot_dnums.add_lhs_contracting_dimensions(1);
-- 
GitLab


From 0186c2d8d3735c2ea58d00f331eaa70b5ecacc42 Mon Sep 17 00:00:00 2001
From: Davide Libenzi <dlibenzi@google.com>
Date: Fri, 11 Jan 2019 15:12:56 -0800
Subject: [PATCH 0625/1095] Implement an XRT allocation op that takes a Tensor
 (with optional layout) as input.

PiperOrigin-RevId: 228953639
---
 tensorflow/compiler/tf2xla/shape_util.cc      |  93 ++++++++++++-
 tensorflow/compiler/tf2xla/shape_util.h       |  15 ++-
 tensorflow/compiler/xrt/kernels/BUILD         |   7 +-
 .../compiler/xrt/kernels/xrt_state_ops.cc     |  11 ++
 .../compiler/xrt/kernels/xrt_state_ops.h      | 108 +++++++++++++++
 tensorflow/compiler/xrt/ops/xrt_state_ops.cc  |  33 ++++-
 tensorflow/compiler/xrt/tests/BUILD           |   1 +
 tensorflow/compiler/xrt/tests/raw_api_test.cc | 124 ++++++++++++++++++
 tensorflow/compiler/xrt/xrt_state.cc          |   2 +-
 tensorflow/compiler/xrt/xrt_state.h           |   2 +-
 10 files changed, 383 insertions(+), 13 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/shape_util.cc b/tensorflow/compiler/tf2xla/shape_util.cc
index cb04a75d95..8997b2f5c6 100644
--- a/tensorflow/compiler/tf2xla/shape_util.cc
+++ b/tensorflow/compiler/tf2xla/shape_util.cc
@@ -44,6 +44,43 @@ Status PopulateInfeedLayoutVector(const xla::Shape& shape,
   return Status::OK();
 }
 
+// Populate the output layout unless the minor_to_major array contains all -1
+// value, in which case the layout is considered missing and the API returns
+// false.
+xla::StatusOr<bool> MakeLayout(absl::Span<const int64> minor_to_major,
+                               xla::Layout* layout) {
+  if (std::all_of(minor_to_major.begin(), minor_to_major.end(),
+                  [](int64 dim) { return dim == -1; })) {
+    return false;
+  }
+  std::vector<bool> dim_present(minor_to_major.size(), false);
+  for (auto dim : minor_to_major) {
+    if (dim < 0 || dim >= minor_to_major.size()) {
+      return errors::InvalidArgument("Layout dimension out of range: dim=", dim,
+                                     " rank=", minor_to_major.size());
+    }
+    if (dim_present[dim]) {
+      return errors::InvalidArgument("Repeated layout dimension: dim=", dim);
+    }
+    dim_present[dim] = true;
+  }
+  *layout = xla::LayoutUtil::MakeLayout(minor_to_major);
+  return true;
+}
+
+Status AssignLayout(
+    absl::Span<const int64> minor_to_major,
+    const std::function<xla::Layout(const xla::Shape&)>& layout_func,
+    xla::Shape* shape) {
+  xla::Layout layout;
+  TF_ASSIGN_OR_RETURN(bool has_layout, MakeLayout(minor_to_major, &layout));
+  if (!has_layout && layout_func) {
+    layout = layout_func(*shape);
+  }
+  *shape->mutable_layout() = layout;
+  return Status::OK();
+}
+
 }  // namespace
 
 // Convert an XLA Shape into the equivalent TensorFlow shape.
@@ -84,10 +121,64 @@ xla::Shape TensorShapeToXLAShape(xla::PrimitiveType type,
   return xla::ShapeUtil::MakeShapeWithLayout(type, dimensions, layout);
 }
 
-xla::StatusOr<std::vector<int>> GetInfeedLayoutVector(const xla::Shape& shape) {
+xla::StatusOr<std::vector<int>> GetShapeLayoutVector(const xla::Shape& shape) {
   std::vector<int> layouts;
   TF_RETURN_IF_ERROR(PopulateInfeedLayoutVector(shape, &layouts));
   return layouts;
 }
 
+Status GetShapeWithLayout(
+    const xla::Shape& input_shape, absl::Span<const int64> minor_to_major,
+    const std::function<xla::Layout(const xla::Shape&)>& layout_func,
+    xla::Shape* output_shape) {
+  if (input_shape.IsTuple()) {
+    int64 tuple_elements = xla::ShapeUtil::TupleElementCount(input_shape);
+    std::vector<xla::Shape> shapes;
+    shapes.reserve(tuple_elements);
+    size_t position = 0;
+    for (int64 i = 0; i < tuple_elements; ++i) {
+      const xla::Shape& shape =
+          xla::ShapeUtil::GetTupleElementShape(input_shape, i);
+      if (shape.IsTuple()) {
+        return errors::InvalidArgument(
+            "Nested tuples not supported: ",
+            xla::ShapeUtil::HumanString(input_shape));
+      }
+      int64 rank = shape.rank();
+      if (position + rank > minor_to_major.size()) {
+        return errors::InvalidArgument(
+            "Not enough layout attribute elements: position=", position,
+            " rank=", rank, " elements=", minor_to_major.size());
+      }
+      shapes.push_back(shape);
+      TF_RETURN_IF_ERROR(AssignLayout(
+          absl::Span<const int64>(minor_to_major).subspan(position, rank),
+          layout_func, &shapes.back()));
+      position += rank;
+
+      VLOG(4) << "Shape[" << i
+              << "] = " << xla::ShapeUtil::HumanStringWithLayout(shapes.back());
+    }
+    if (position != minor_to_major.size()) {
+      return errors::InvalidArgument(
+          "Too many elements passed in the layout attribute: position=",
+          position, " size=", minor_to_major.size());
+    }
+    *output_shape = xla::ShapeUtil::MakeTupleShape(shapes);
+  } else {
+    int64 rank = input_shape.rank();
+    if (rank != minor_to_major.size()) {
+      return errors::InvalidArgument(
+          "Wrong number of layout attribute elements: rank=", rank,
+          " elements=", minor_to_major.size());
+    }
+    *output_shape = input_shape;
+    TF_RETURN_IF_ERROR(AssignLayout(minor_to_major, layout_func, output_shape));
+
+    VLOG(4) << "Shape[] = "
+            << xla::ShapeUtil::HumanStringWithLayout(*output_shape);
+  }
+  return Status::OK();
+}
+
 }  // namespace tensorflow
diff --git a/tensorflow/compiler/tf2xla/shape_util.h b/tensorflow/compiler/tf2xla/shape_util.h
index cf52bf46e7..e775c4462c 100644
--- a/tensorflow/compiler/tf2xla/shape_util.h
+++ b/tensorflow/compiler/tf2xla/shape_util.h
@@ -45,12 +45,23 @@ xla::Shape TensorShapeToXLAShape(xla::PrimitiveType type,
                                  const TensorShape& tensor_shape);
 
 // Given an XLA shape with layouts, builds a layout vector in the form able to
-// be fed to an InfeedEnqueue/InfeedEnqueueTuple ops.
+// be fed to ops like InfeedEnqueue/InfeedEnqueueTuple/XRTAllocateV2/....
 // THe returned vector is a linearized sequence of the minor-to-major values of
 // the layouts held within the input shape.
 // In case the input shape is a tuple, the minor-to-major values will be in the
 // order of the tuple elements within the tuple shape.
-xla::StatusOr<std::vector<int>> GetInfeedLayoutVector(const xla::Shape& shape);
+// If a shape (or a subshape of a tuple shape) has missing layout, a rank long
+// sequence of -1 values will be emittted.
+xla::StatusOr<std::vector<int>> GetShapeLayoutVector(const xla::Shape& shape);
+
+// Given the input shape and a linearized sequence of the minor-to-major values
+// of the layouts, create the output shape by rewriting the input shape layouts.
+// If a layout is missing (has -1 values) for a matching tuple subshape, the
+// layout_func will be called, if not nullptr.
+Status GetShapeWithLayout(
+    const xla::Shape& input_shape, absl::Span<const int64> minor_to_major,
+    const std::function<xla::Layout(const xla::Shape&)>& layout_func,
+    xla::Shape* output_shape);
 
 }  // namespace tensorflow
 
diff --git a/tensorflow/compiler/xrt/kernels/BUILD b/tensorflow/compiler/xrt/kernels/BUILD
index d38f41ab43..dc02fd272f 100644
--- a/tensorflow/compiler/xrt/kernels/BUILD
+++ b/tensorflow/compiler/xrt/kernels/BUILD
@@ -11,20 +11,15 @@ cc_library(
     name = "xrt_state_ops",
     hdrs = ["xrt_state_ops.h"],
     deps = [
+        "//tensorflow/compiler/tf2xla:common",
         "//tensorflow/compiler/tf2xla:xla_compiler",
-        "//tensorflow/compiler/xla:debug_options_flags",
         "//tensorflow/compiler/xla:literal",
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:status_macros",
         "//tensorflow/compiler/xla:statusor",
         "//tensorflow/compiler/xla:xla_data_proto",
-        "//tensorflow/compiler/xla/client:client_library",
-        "//tensorflow/compiler/xla/client:compile_only_client",
         "//tensorflow/compiler/xla/client:local_client",
-        "//tensorflow/compiler/xla/client:xla_computation",
-        "//tensorflow/compiler/xla/service:compiler",
         "//tensorflow/compiler/xla/service:computation_placer",
-        "//tensorflow/compiler/xla/service:hlo_proto",
         "//tensorflow/compiler/xrt:xrt_proto",
         "//tensorflow/compiler/xrt:xrt_utils",
         "//tensorflow/core:core_cpu_internal",
diff --git a/tensorflow/compiler/xrt/kernels/xrt_state_ops.cc b/tensorflow/compiler/xrt/kernels/xrt_state_ops.cc
index 1a5bfac337..6a7f106525 100644
--- a/tensorflow/compiler/xrt/kernels/xrt_state_ops.cc
+++ b/tensorflow/compiler/xrt/kernels/xrt_state_ops.cc
@@ -37,6 +37,17 @@ REGISTER_KERNEL_BUILDER(Name("XRTAllocate")
                             .HostMemory("handle"),
                         XRTAllocateOp<XRTGenericDeviceAccessor>);
 
+REGISTER_KERNEL_BUILDER(Name("XRTAllocateFromTensor")
+                            .Device(DEVICE_XLA_GPU)
+                            .HostMemory("inputs")
+                            .HostMemory("handle"),
+                        XRTAllocateFromTensorOp<XRTGenericDeviceAccessor>);
+REGISTER_KERNEL_BUILDER(Name("XRTAllocateFromTensor")
+                            .Device(DEVICE_XLA_CPU)
+                            .HostMemory("inputs")
+                            .HostMemory("handle"),
+                        XRTAllocateFromTensorOp<XRTGenericDeviceAccessor>);
+
 REGISTER_KERNEL_BUILDER(Name("XRTSubTuple")
                             .Device(DEVICE_XLA_GPU)
                             .HostMemory("base_handle")
diff --git a/tensorflow/compiler/xrt/kernels/xrt_state_ops.h b/tensorflow/compiler/xrt/kernels/xrt_state_ops.h
index c8def16bbc..e2c223b3db 100644
--- a/tensorflow/compiler/xrt/kernels/xrt_state_ops.h
+++ b/tensorflow/compiler/xrt/kernels/xrt_state_ops.h
@@ -19,10 +19,14 @@ limitations under the License.
 #ifndef TENSORFLOW_COMPILER_XRT_KERNELS_XRT_STATE_OPS_H_
 #define TENSORFLOW_COMPILER_XRT_KERNELS_XRT_STATE_OPS_H_
 
+#include <functional>
 #include <memory>
 #include <string>
 
+#include "tensorflow/compiler/tf2xla/literal_util.h"
+#include "tensorflow/compiler/tf2xla/shape_util.h"
 #include "tensorflow/compiler/xla/client/local_client.h"
+#include "tensorflow/compiler/xla/layout_util.h"
 #include "tensorflow/compiler/xla/literal.h"
 #include "tensorflow/compiler/xla/status_macros.h"
 #include "tensorflow/compiler/xla/statusor.h"
@@ -30,6 +34,7 @@ limitations under the License.
 #include "tensorflow/compiler/xrt/xrt.pb.h"
 #include "tensorflow/compiler/xrt/xrt_device.h"
 #include "tensorflow/compiler/xrt/xrt_state.h"
+#include "tensorflow/core/common_runtime/dma_helper.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/resource_mgr.h"
 #include "tensorflow/core/framework/tensor.h"
@@ -200,6 +205,109 @@ class XRTAllocateOp : public OpKernel {
   }
 };
 
+// Op that allocates memory for a tensor (with optional layout) and transfers it
+// to the device, returning an allocation handle.
+template <class DeviceAccessor>
+class XRTAllocateFromTensorOp : public OpKernel {
+ public:
+  explicit XRTAllocateFromTensorOp(OpKernelConstruction* ctx) : OpKernel(ctx) {
+    bool make_tuple = false;
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("shapes", &tf_shapes_));
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("dtypes", &dtypes_));
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("make_tuple", &make_tuple));
+    if (ctx->HasAttr("layouts")) {
+      OP_REQUIRES_OK(ctx, ctx->GetAttr("layouts", &minor_to_major_));
+    }
+    OP_REQUIRES(
+        ctx, tf_shapes_.size() == dtypes_.size(),
+        errors::InvalidArgument("shapes and dtypes must be the same length"));
+    std::vector<xla::Shape> xla_shapes;
+    for (int i = 0; i < tf_shapes_.size(); i++) {
+      xla::Shape xla_shape;
+      OP_REQUIRES_OK(
+          ctx, TensorShapeToXLAShape(dtypes_[i], tf_shapes_[i], &xla_shape));
+      xla_shapes.push_back(xla_shape);
+    }
+    if (xla_shapes.size() > 1 || make_tuple) {
+      shape_ = xla::ShapeUtil::MakeTupleShape(xla_shapes);
+    } else {
+      shape_.Swap(&xla_shapes.front());
+    }
+    if (!minor_to_major_.empty()) {
+      xla::Shape shape_with_layouts;
+      OP_REQUIRES_OK(ctx, GetShapeWithLayout(shape_, minor_to_major_,
+                                             /*layout_func=*/nullptr,
+                                             &shape_with_layouts));
+      shape_.Swap(&shape_with_layouts);
+    }
+  }
+
+  ~XRTAllocateFromTensorOp() override = default;
+  XRTAllocateFromTensorOp(const XRTAllocateFromTensorOp&) = delete;
+  XRTAllocateFromTensorOp& operator=(const XRTAllocateFromTensorOp&) = delete;
+
+  void Compute(OpKernelContext* ctx) override {
+    VLOG(1) << "XRTAllocateFromTensorOp::Compute";
+
+    OpInputList values;
+    OP_REQUIRES_OK(ctx, ctx->input_list("inputs", &values));
+    OP_REQUIRES(ctx, values.size() == tf_shapes_.size(),
+                errors::InvalidArgument(
+                    "Wrong number of inputs to XRTAllocateFromTensor: ",
+                    values.size(), " vs. ", tf_shapes_.size()));
+
+    std::vector<const char*> tensors_data;
+    for (size_t i = 0; i < values.size(); ++i) {
+      const Tensor& input_tensor = values[i];
+      OP_REQUIRES(ctx, input_tensor.dtype() == dtypes_[i],
+                  errors::InvalidArgument(
+                      "Input tensor type and input dtype do not match"));
+      // We allow the requested on-device shape to differ from the shape of the
+      // input tensor, as long as they have the same number of elements.
+      OP_REQUIRES(
+          ctx,
+          input_tensor.shape().num_elements() == tf_shapes_[i].num_elements(),
+          errors::InvalidArgument(
+              "Input tensor must have the number of elements specified "
+              "in the matching input shape: ",
+              input_tensor.shape().num_elements(), " vs. ",
+              tf_shapes_[i].num_elements(), " at index ", i));
+      tensors_data.push_back(
+          static_cast<const char*>(DMAHelper::base(&input_tensor)));
+    }
+    // Use the buffer straight out of the input tensors to create the literal.
+    xla::BorrowingLiteral literal =
+        shape_.IsTuple() ? xla::BorrowingLiteral(tensors_data, shape_)
+                         : xla::BorrowingLiteral(tensors_data.front(), shape_);
+    ResourceMgr* rm;
+    OP_REQUIRES_OK(ctx, DeviceAccessor::GetResourceManager(ctx, &rm));
+
+    // We are guaranteed that the underlying device object won't be deleted out
+    // from under us, while the ScopedRef is live.
+    class DeviceAccessor::ScopedRef device_ref;
+    OP_REQUIRES_OK(ctx, DeviceAccessor::InitScopedRef(ctx, &device_ref));
+
+    XRTTupleAllocation* allocation;
+    OP_REQUIRES_OK(ctx, XRTTupleAllocation::CreateAndTransfer(
+                            literal, device_ref.backend(),
+                            device_ref.device_ordinal(), &allocation));
+
+    // Intern takes ownership of our reference to allocation.
+    int64 key;
+    OP_REQUIRES_OK(ctx, allocation->Intern(rm, &key));
+
+    Tensor output(DT_INT64, TensorShape({}));
+    output.scalar<int64>()() = key;
+    ctx->set_output(0, output);
+  }
+
+ private:
+  std::vector<TensorShape> tf_shapes_;
+  DataTypeVector dtypes_;
+  std::vector<int64> minor_to_major_;
+  xla::Shape shape_;
+};
+
 // Op that takes a tuple handle input and returns a handle to a sub-tuple of the
 // input.
 template <bool discard_, class DeviceAccessor>
diff --git a/tensorflow/compiler/xrt/ops/xrt_state_ops.cc b/tensorflow/compiler/xrt/ops/xrt_state_ops.cc
index db58f0797d..2e743fec49 100644
--- a/tensorflow/compiler/xrt/ops/xrt_state_ops.cc
+++ b/tensorflow/compiler/xrt/ops/xrt_state_ops.cc
@@ -26,12 +26,41 @@ REGISTER_OP("XRTAllocate")
     .SetShapeFn(tensorflow::shape_inference::ScalarShape)
     .Doc(
         R"(
-Reads a literal proto and transfers it to TPU device memory.
+Reads a literal proto and transfers it to device memory.
 
-'allocation' is a serialized xrt::TPUAllocation proto.
+'allocation' is a serialized xrt::XLAAllocation proto.
 'handle' is an id that can be used in other ops to refer to the allocation.
 )");
 
+REGISTER_OP("XRTAllocateFromTensor")
+    .Input("inputs: dtypes")
+    .Output("handle: int64")
+    .Attr("dtypes: list(type)")
+    .Attr("shapes: list(shape)")
+    .Attr("layouts: list(int) = []")
+    .Attr("make_tuple: bool = false")
+    .SetShapeFn(tensorflow::shape_inference::ScalarShape)
+    .Doc(
+        R"(
+Reads a list of tensors with optional layouts, and transfers it to device
+memory.
+
+inputs: The tensors holding the input data.
+shapes: The shapes which the tensors should have on device. The i-th shape
+corresponds to the i-th input. The shapes, together with the (optional)
+layouts, helps creating the fully qualified shape of the data on the device.
+The shapes can differ from the corresponding input one, as long as the total
+number of elements matches. In other words, it is possible to feed an input
+tensor with shape {8} and have a corresponding shape {2,2,2}.
+layouts: A vector holding the requested layout in minor-to-major sequence.
+If empty, the default layout wil be used.
+For a tuple, the layouts vector holds a linearized minor-to-major numbers
+for all the tuple leaves, in the order they appear within the tuple.
+The elements within the layouts sequence corresponding to a given tuple
+subshape can be set to -1, to leave such subshape to the default shape.
+handle: An id that can be used in other ops to refer to the allocation.
+)");
+
 REGISTER_OP("XRTSubTuple")
     .Input("base_handle: int64")
     .Input("shape_index: int32")
diff --git a/tensorflow/compiler/xrt/tests/BUILD b/tensorflow/compiler/xrt/tests/BUILD
index be44a3474a..3a19327e5b 100644
--- a/tensorflow/compiler/xrt/tests/BUILD
+++ b/tensorflow/compiler/xrt/tests/BUILD
@@ -24,6 +24,7 @@ cc_library(
         "//tensorflow/cc:client_session",
         "//tensorflow/cc:ops",
         "//tensorflow/cc:scope",
+        "//tensorflow/compiler/tf2xla:common",
         "//tensorflow/compiler/tf2xla:xla_compiler",
         "//tensorflow/compiler/xla:literal",
         "//tensorflow/compiler/xla:literal_util",
diff --git a/tensorflow/compiler/xrt/tests/raw_api_test.cc b/tensorflow/compiler/xrt/tests/raw_api_test.cc
index 8e70cc3959..1111f82405 100644
--- a/tensorflow/compiler/xrt/tests/raw_api_test.cc
+++ b/tensorflow/compiler/xrt/tests/raw_api_test.cc
@@ -22,6 +22,8 @@ limitations under the License.
 #include "tensorflow/cc/framework/ops.h"
 #include "tensorflow/cc/framework/scope.h"
 #include "tensorflow/cc/ops/standard_ops.h"
+#include "tensorflow/compiler/tf2xla/literal_util.h"
+#include "tensorflow/compiler/tf2xla/shape_util.h"
 #include "tensorflow/compiler/xla/client/client_library.h"
 #include "tensorflow/compiler/xla/client/local_client.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
@@ -53,6 +55,14 @@ string DeviceFromFlag() {
   return absl::StrCat("/device:", xla_test_device, ":0");
 }
 
+std::vector<int> GetAttrLayout(absl::Span<const int64> minor_to_mayor) {
+  std::vector<int> layout;
+  for (auto dim : minor_to_mayor) {
+    layout.push_back(static_cast<int>(dim));
+  }
+  return layout;
+}
+
 xla::LiteralProto TwoElementTuple() {
   auto array = xla::LiteralUtil::CreateR1<float>({1.0f, 3.0f});
   auto matrix = xla::LiteralUtil::CreateR2({{4, 5}, {6, 7}});
@@ -233,6 +243,120 @@ xla::ProgramShape XlaCompiledProgramShape(
       ->ComputeProgramShape();
 }
 
+TEST(RawApiTest, AllocFromTensor) {
+  xla::Literal literal =
+      xla::LiteralUtil::CreateR2<float>({{4.0f, 5.0f}, {6.0f, 7.0f}});
+  Tensor tensor;
+  TF_ASSERT_OK(LiteralToHostTensor(literal, DT_FLOAT, &tensor));
+
+  Scope root = Scope::NewRootScope().WithDevice(DeviceFromFlag());
+  std::vector<int> layout =
+      GetAttrLayout(literal.shape().layout().minor_to_major());
+  ops::XRTAllocateFromTensor::Attrs alloc_attrs =
+      ops::XRTAllocateFromTensor::Layouts(layout);
+  auto handle =
+      ops::XRTAllocateFromTensor(root, {tensor}, {tensor.shape()}, alloc_attrs);
+  auto read_back = ops::XRTReadLiteralAndRelease(root, handle);
+  TF_ASSERT_OK(root.status());
+
+  ClientSession session(root);
+  std::vector<Tensor> outputs;
+  TF_EXPECT_OK(session.Run({read_back}, &outputs));
+  EXPECT_EQ(outputs.size(), 1);
+
+  xla::LiteralProto response;
+  EXPECT_TRUE(response.ParseFromString(outputs[0].scalar<string>()()));
+  EXPECT_TRUE(CompareLiteralToLiteralProto(literal, response));
+}
+
+TEST(RawApiTest, AllocFromTensorTuple) {
+  xla::Literal literal0 =
+      xla::LiteralUtil::CreateR2<float>({{4.0f, 5.0f}, {6.0f, 7.0f}});
+  xla::Literal literal1 =
+      xla::LiteralUtil::CreateR2<float>({{14.0f, -5.0f}, {16.0f, 17.0f}});
+  xla::Literal literal = xla::LiteralUtil::MakeTuple({&literal0, &literal1});
+  Tensor tensor0;
+  TF_ASSERT_OK(LiteralToHostTensor(literal0, DT_FLOAT, &tensor0));
+  Tensor tensor1;
+  TF_ASSERT_OK(LiteralToHostTensor(literal1, DT_FLOAT, &tensor1));
+
+  Scope root = Scope::NewRootScope().WithDevice(DeviceFromFlag());
+  std::vector<int> layout = GetShapeLayoutVector(literal.shape()).ValueOrDie();
+  ops::XRTAllocateFromTensor::Attrs alloc_attrs =
+      ops::XRTAllocateFromTensor::Layouts(layout);
+  auto handle = ops::XRTAllocateFromTensor(root, {tensor0, tensor1},
+                                           {tensor0.shape(), tensor1.shape()},
+                                           alloc_attrs);
+  auto read_back = ops::XRTReadLiteralAndRelease(root, handle);
+  TF_ASSERT_OK(root.status());
+
+  ClientSession session(root);
+  std::vector<Tensor> outputs;
+  TF_EXPECT_OK(session.Run({read_back}, &outputs));
+  EXPECT_EQ(outputs.size(), 1);
+
+  xla::LiteralProto response;
+  EXPECT_TRUE(response.ParseFromString(outputs[0].scalar<string>()()));
+  EXPECT_TRUE(CompareLiteralToLiteralProto(literal, response));
+}
+
+TEST(RawApiTest, AllocFromTensorTupleSingle) {
+  xla::Literal literal0 =
+      xla::LiteralUtil::CreateR2<float>({{4.0f, 5.0f}, {6.0f, 7.0f}});
+  xla::Literal literal = xla::LiteralUtil::MakeTuple({&literal0});
+  Tensor tensor0;
+  TF_ASSERT_OK(LiteralToHostTensor(literal0, DT_FLOAT, &tensor0));
+
+  Scope root = Scope::NewRootScope().WithDevice(DeviceFromFlag());
+  std::vector<int> layout = GetShapeLayoutVector(literal.shape()).ValueOrDie();
+  ops::XRTAllocateFromTensor::Attrs alloc_attrs =
+      ops::XRTAllocateFromTensor::Layouts(layout).MakeTuple(true);
+  auto handle = ops::XRTAllocateFromTensor(root, {tensor0}, {tensor0.shape()},
+                                           alloc_attrs);
+  auto read_back = ops::XRTReadLiteralAndRelease(root, handle);
+  TF_ASSERT_OK(root.status());
+
+  ClientSession session(root);
+  std::vector<Tensor> outputs;
+  TF_EXPECT_OK(session.Run({read_back}, &outputs));
+  EXPECT_EQ(outputs.size(), 1);
+
+  xla::LiteralProto response;
+  EXPECT_TRUE(response.ParseFromString(outputs[0].scalar<string>()()));
+  EXPECT_TRUE(CompareLiteralToLiteralProto(literal, response));
+}
+
+TEST(RawApiTest, AllocFromTensorRelayout) {
+  xla::Literal literal =
+      xla::LiteralUtil::CreateR2<float>({{4.0f, 5.0f}, {6.0f, 7.0f}});
+  Tensor tensor;
+  TF_ASSERT_OK(LiteralToHostTensor(literal, DT_FLOAT, &tensor));
+
+  Scope root = Scope::NewRootScope().WithDevice(DeviceFromFlag());
+  // Use inverse array layout with the tensor data above.
+  std::vector<int> layout({0, 1});
+  ops::XRTAllocateFromTensor::Attrs alloc_attrs =
+      ops::XRTAllocateFromTensor::Layouts(layout);
+  auto handle =
+      ops::XRTAllocateFromTensor(root, {tensor}, {tensor.shape()}, alloc_attrs);
+  auto read_back = ops::XRTReadLiteralAndRelease(root, handle);
+  TF_ASSERT_OK(root.status());
+
+  ClientSession session(root);
+  std::vector<Tensor> outputs;
+  TF_EXPECT_OK(session.Run({read_back}, &outputs));
+  EXPECT_EQ(outputs.size(), 1);
+
+  xla::LiteralProto response;
+  EXPECT_TRUE(response.ParseFromString(outputs[0].scalar<string>()()));
+  // We have sent literal's data (in array layout) with a attribute layout
+  // {0,1}, so the expected literal read from device needs to be changed
+  // accordingly.
+  xla::Literal expected_literal =
+      xla::LiteralUtil::CreateR2<float>({{4.0f, 6.0f}, {5.0f, 7.0f}});
+  EXPECT_TRUE(CompareLiteralToLiteralProto(expected_literal, response));
+}
+
 TEST(RawApiTest, AllocAndRewrite) {
   xrt::XLAAllocation alloc;
   *alloc.mutable_value() =
diff --git a/tensorflow/compiler/xrt/xrt_state.cc b/tensorflow/compiler/xrt/xrt_state.cc
index 13c275aaca..1e2a9584f8 100644
--- a/tensorflow/compiler/xrt/xrt_state.cc
+++ b/tensorflow/compiler/xrt/xrt_state.cc
@@ -182,7 +182,7 @@ XRTTupleAllocation::~XRTTupleAllocation() {
 }
 
 /*static*/ Status XRTTupleAllocation::CreateAndTransfer(
-    const xla::Literal& literal, xla::Backend* backend, int device_ordinal,
+    const xla::LiteralBase& literal, xla::Backend* backend, int device_ordinal,
     XRTTupleAllocation** allocation) {
   auto transfer_manager = backend->transfer_manager();
   auto allocator = backend->memory_allocator();
diff --git a/tensorflow/compiler/xrt/xrt_state.h b/tensorflow/compiler/xrt/xrt_state.h
index 1b32b6a20f..ddf2656e6f 100644
--- a/tensorflow/compiler/xrt/xrt_state.h
+++ b/tensorflow/compiler/xrt/xrt_state.h
@@ -88,7 +88,7 @@ class XRTTupleAllocation : public ResourceBase {
   // Allocates new device memory buffers sufficient to store literal, transfers
   // literal to that memory, and returns a XRTTupleAllocation handle to the
   // allocated buffers.
-  static Status CreateAndTransfer(const xla::Literal& literal,
+  static Status CreateAndTransfer(const xla::LiteralBase& literal,
                                   xla::Backend* backend, int device_ordinal,
                                   XRTTupleAllocation** allocation);
 
-- 
GitLab


From 8ec7b777318a4f93dfb2f34bc6a0e89e48bc4ea3 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 11 Jan 2019 15:27:00 -0800
Subject: [PATCH 0626/1095] Enable quantization support for ReduceMax and
 ReduceMin

PiperOrigin-RevId: 228955777
---
 .../lite/toco/graph_transformations/hardcode_min_max.cc       | 4 +++-
 tensorflow/lite/toco/graph_transformations/quantize.cc        | 3 ++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/toco/graph_transformations/hardcode_min_max.cc b/tensorflow/lite/toco/graph_transformations/hardcode_min_max.cc
index 2e41767095..6882a19801 100644
--- a/tensorflow/lite/toco/graph_transformations/hardcode_min_max.cc
+++ b/tensorflow/lite/toco/graph_transformations/hardcode_min_max.cc
@@ -440,6 +440,8 @@ bool HardcodeMinMaxForLstmCell(Model* model, Operator* op) {
     case OperatorType::kGather:
     case OperatorType::kTranspose:
     case OperatorType::kMean:
+    case OperatorType::kReduceMax:
+    case OperatorType::kReduceMin:
       changed = HardcodeMinMaxFromFirstInput(model, op);
       break;
     case OperatorType::kSum:
@@ -448,7 +450,7 @@ bool HardcodeMinMaxForLstmCell(Model* model, Operator* op) {
       // in special circumstances like when computing expected value using
       // reduce_sum the input range and the output range matches. Hence the
       // below code would act as a fallback. If a fake_quant node is observed in
-      // the output that takes precendence over the hard coding logic below.
+      // the output that takes precedence over the hard coding logic below.
       changed = HardcodeMinMaxFromFirstInput(model, op);
       if (changed) {
         LOG(WARNING) << "Using the input range for output in reduce_sum op."
diff --git a/tensorflow/lite/toco/graph_transformations/quantize.cc b/tensorflow/lite/toco/graph_transformations/quantize.cc
index 2fa80f2eda..ee65f92e00 100644
--- a/tensorflow/lite/toco/graph_transformations/quantize.cc
+++ b/tensorflow/lite/toco/graph_transformations/quantize.cc
@@ -66,7 +66,8 @@ bool SupportsQuantization(const Operator& op) {
          type == OperatorType::kPack || type == OperatorType::kTopK_V2 ||
          type == OperatorType::kRandomUniform ||
          type == OperatorType::kResizeNearestNeighbor ||
-         type == OperatorType::kPRelu;
+         type == OperatorType::kPRelu || type == OperatorType::kReduceMax ||
+         type == OperatorType::kReduceMin;
 }
 
 // The quantized op allows output arrays of type float using
-- 
GitLab


From 175962fc170ba3f3dc0fc85261500361ffeeca77 Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Fri, 11 Jan 2019 15:54:46 -0800
Subject: [PATCH 0627/1095] Debug disk usage of parameterized build.

PiperOrigin-RevId: 228960346
---
 tensorflow/tools/ci_build/ci_parameterized_build.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tensorflow/tools/ci_build/ci_parameterized_build.sh b/tensorflow/tools/ci_build/ci_parameterized_build.sh
index 0631806dc5..62c1e014d5 100755
--- a/tensorflow/tools/ci_build/ci_parameterized_build.sh
+++ b/tensorflow/tools/ci_build/ci_parameterized_build.sh
@@ -614,11 +614,11 @@ if [[ "${DO_DOCKER}" == "1" ]]; then
 fi
 
 # Set a disk usage trap.
-function finish {
+function debug_disk_usage {
     echo "Finished script... disk usage report in ${TMP_DIR}"
     du -k -d 2 ${TMP_DIR} | sort -n -r
 }
-trap finish EXIT
+# trap debug_disk_usage EXIT
 
 chmod +x ${TMP_SCRIPT}
 
@@ -655,6 +655,8 @@ echo ""
 echo "Parameterized build ends with ${RESULT} at: $(date) "\
 "(Elapsed time: $((END_TIME - START_TIME)) s)"
 
+# Dump disk usage
+debug_disk_usage
 
 # Clean up temporary directory if it exists
 if [[ ! -z "${TMP_DIR}" ]]; then
-- 
GitLab


From 3f594e1b481ee16fe3a3aa08c87dc53158250de7 Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Fri, 11 Jan 2019 16:01:26 -0800
Subject: [PATCH 0628/1095] [tf.data] Replacing usage of
 `std::unique_ptr<Foo>(new Foo(...))` with `absl::make_unique<Foo>(...)` when
 possible and wrapping it `absl::WrapUnique<Foo>` otherwise.

PiperOrigin-RevId: 228961343
---
 .../core/kernels/data/batch_dataset_op.cc     |  4 +-
 .../core/kernels/data/cache_dataset_ops.cc    | 35 ++++++-----
 .../core/kernels/data/captured_function.cc    | 15 ++---
 .../kernels/data/concatenate_dataset_op.cc    |  4 +-
 .../experimental/assert_next_dataset_op.cc    |  4 +-
 .../data/experimental/csv_dataset_op.cc       |  6 +-
 .../dense_to_sparse_batch_dataset_op.cc       |  4 +-
 .../directed_interleave_dataset_op.cc         |  4 +-
 .../group_by_reducer_dataset_op.cc            |  4 +-
 .../group_by_window_dataset_op.cc             |  4 +-
 .../experimental/ignore_errors_dataset_op.cc  |  4 +-
 .../data/experimental/indexed_dataset_op.cc   |  7 +--
 .../data/experimental/lmdb_dataset_op.cc      |  5 +-
 .../experimental/map_and_batch_dataset_op.cc  |  3 +-
 .../experimental/matching_files_dataset_op.cc |  4 +-
 .../non_serializable_dataset_op.cc            |  4 +-
 .../numa_map_and_batch_dataset_op.cc          |  7 +--
 .../parallel_interleave_dataset_op.cc         |  5 +-
 .../experimental/parse_example_dataset_op.cc  |  4 +-
 .../data/experimental/random_dataset_op.cc    |  4 +-
 .../data/experimental/scan_dataset_op.cc      |  4 +-
 .../set_stats_aggregator_dataset_op.cc        |  5 +-
 .../data/experimental/sleep_dataset_op.cc     |  3 +-
 .../experimental/sliding_window_dataset_op.cc |  5 +-
 .../data/experimental/sql_dataset_op.cc       |  4 +-
 .../data/experimental/stats_aggregator_ops.cc |  4 +-
 .../data/experimental/stats_dataset_ops.cc    |  9 ++-
 .../experimental/threadpool_dataset_op.cc     | 17 +++--
 .../data/experimental/to_tf_record_op.cc      | 13 ++--
 .../data/experimental/unbatch_dataset_op.cc   |  4 +-
 .../data/experimental/unique_dataset_op.cc    |  4 +-
 .../data/filter_by_component_dataset_op.cc    |  4 +-
 .../core/kernels/data/filter_dataset_op.cc    |  3 +-
 .../core/kernels/data/flat_map_dataset_op.cc  |  4 +-
 .../core/kernels/data/generator_dataset_op.cc |  8 +--
 .../kernels/data/interleave_dataset_op.cc     |  4 +-
 tensorflow/core/kernels/data/iterator_ops.cc  | 27 ++++----
 .../core/kernels/data/map_dataset_op.cc       |  3 +-
 tensorflow/core/kernels/data/map_defun_op.cc  |  2 +-
 .../kernels/data/multi_device_iterator_ops.cc | 21 +++----
 tensorflow/core/kernels/data/optional_ops.h   |  7 ++-
 .../kernels/data/padded_batch_dataset_op.cc   |  4 +-
 .../data/parallel_interleave_dataset_op.cc    | 14 ++---
 .../kernels/data/parallel_map_dataset_op.cc   |  5 +-
 .../kernels/data/parallel_map_iterator.cc     |  3 +-
 .../core/kernels/data/prefetch_dataset_op.cc  |  9 +--
 .../core/kernels/data/range_dataset_op.cc     |  4 +-
 .../core/kernels/data/reader_dataset_ops.cc   | 63 ++++++++++---------
 .../core/kernels/data/repeat_dataset_op.cc    | 12 ++--
 .../core/kernels/data/shuffle_dataset_op.cc   | 35 ++++++-----
 .../kernels/data/single_threaded_executor.cc  |  4 +-
 .../data/single_threaded_executor_test.cc     |  8 +--
 .../core/kernels/data/skip_dataset_op.cc      |  8 +--
 .../data/sparse_tensor_slice_dataset_op.cc    |  4 +-
 .../core/kernels/data/take_dataset_op.cc      |  8 +--
 .../core/kernels/data/tensor_dataset_op.cc    |  4 +-
 .../kernels/data/tensor_slice_dataset_op.cc   |  4 +-
 .../core/kernels/data/window_dataset.cc       |  4 +-
 .../core/kernels/data/window_dataset_op.cc    |  4 +-
 .../core/kernels/data/zip_dataset_op.cc       |  4 +-
 60 files changed, 246 insertions(+), 241 deletions(-)

diff --git a/tensorflow/core/kernels/data/batch_dataset_op.cc b/tensorflow/core/kernels/data/batch_dataset_op.cc
index 1f8d2bdbae..f9ce0d9642 100644
--- a/tensorflow/core/kernels/data/batch_dataset_op.cc
+++ b/tensorflow/core/kernels/data/batch_dataset_op.cc
@@ -79,8 +79,8 @@ class BatchDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(new Iterator(
-          Iterator::Params{this, strings::StrCat(prefix, "::Batch")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::Batch")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/cache_dataset_ops.cc b/tensorflow/core/kernels/data/cache_dataset_ops.cc
index 535f49cff8..343157de6f 100644
--- a/tensorflow/core/kernels/data/cache_dataset_ops.cc
+++ b/tensorflow/core/kernels/data/cache_dataset_ops.cc
@@ -69,8 +69,8 @@ class CacheDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new FileIterator({this, strings::StrCat(prefix, "::FileCache")}));
+      return absl::make_unique<FileIterator>(
+          FileIterator::Params{this, strings::StrCat(prefix, "::FileCache")});
     }
 
     const DataTypeVector& output_dtypes() const override {
@@ -325,7 +325,7 @@ class CacheDatasetOp : public UnaryDatasetOpKernel {
           }
           filename_ = strings::StrCat(dataset()->filename_, "_", shard_id_);
           lockfile_ = strings::StrCat(filename_, ".lockfile");
-          writer_.reset(new BundleWriter(dataset()->env_, filename_));
+          writer_ = absl::make_unique<BundleWriter>(dataset()->env_, filename_);
           return Status::OK();
         }
 
@@ -385,7 +385,8 @@ class CacheDatasetOp : public UnaryDatasetOpKernel {
             // conditions are not met since BundleWriter's constructor creates
             // new temp files which can delete the temp files created by a
             // BundleWriter in another Session.
-            writer_.reset(new BundleWriter(dataset()->env_, filename_));
+            writer_ =
+                absl::make_unique<BundleWriter>(dataset()->env_, filename_);
             lockfile_created_ = true;
             return Status::OK();
           }
@@ -537,12 +538,14 @@ class CacheDatasetOp : public UnaryDatasetOpKernel {
         // `FileReaderIterator` and seek to the `cur_index`.
         switch (mode_) {
           case Mode::read:
-            iterator_.reset(new FileReaderIterator(
-                {dataset(), strings::StrCat(prefix(), "Impl")}));
+            iterator_ = absl::make_unique<FileReaderIterator>(
+                FileReaderIterator::Params{dataset(),
+                                           strings::StrCat(prefix(), "Impl")});
             break;
           case Mode::write:
-            iterator_.reset(new FileWriterIterator(
-                {dataset(), strings::StrCat(prefix(), "Impl")}));
+            iterator_ = absl::make_unique<FileWriterIterator>(
+                FileWriterIterator::Params{dataset(),
+                                           strings::StrCat(prefix(), "Impl")});
         }
       }
 
@@ -573,8 +576,8 @@ class CacheDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new MemoryIterator({this, strings::StrCat(prefix, "::MemoryCache")}));
+      return absl::make_unique<MemoryIterator>(MemoryIterator::Params{
+          this, strings::StrCat(prefix, "::MemoryCache")});
     }
 
     const DataTypeVector& output_dtypes() const override {
@@ -933,12 +936,16 @@ class CacheDatasetOp : public UnaryDatasetOpKernel {
       void InitializeIterator() EXCLUSIVE_LOCKS_REQUIRED(mu_) {
         switch (mode_) {
           case Mode::read:
-            iterator_.reset(new MemoryReaderIterator(
-                {dataset(), strings::StrCat(prefix(), "Impl")}, cache_));
+            iterator_ = absl::make_unique<MemoryReaderIterator>(
+                MemoryReaderIterator::Params{dataset(),
+                                             strings::StrCat(prefix(), "Impl")},
+                cache_);
             break;
           case Mode::write:
-            iterator_.reset(new MemoryWriterIterator(
-                {dataset(), strings::StrCat(prefix(), "Impl")}, cache_));
+            iterator_ = absl::make_unique<MemoryWriterIterator>(
+                MemoryWriterIterator::Params{dataset(),
+                                             strings::StrCat(prefix(), "Impl")},
+                cache_);
         }
       }
 
diff --git a/tensorflow/core/kernels/data/captured_function.cc b/tensorflow/core/kernels/data/captured_function.cc
index 973b6b0604..99b745b4c4 100644
--- a/tensorflow/core/kernels/data/captured_function.cc
+++ b/tensorflow/core/kernels/data/captured_function.cc
@@ -25,7 +25,6 @@ limitations under the License.
 #include "tensorflow/core/lib/random/random.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/notification.h"
-#include "tensorflow/core/util/ptr_util.h"
 
 namespace tensorflow {
 namespace data {
@@ -114,8 +113,8 @@ Status CapturedFunction::Create(
   OpInputList inputs;
   TF_RETURN_IF_ERROR(ctx->input_list(argument, &inputs));
   std::vector<Tensor> arguments(inputs.begin(), inputs.end());
-  *out_function = WrapUnique(new CapturedFunction(func, std::move(arguments),
-                                                  use_inter_op_parallelism));
+  *out_function = absl::WrapUnique(new CapturedFunction(
+      func, std::move(arguments), use_inter_op_parallelism));
   return Status::OK();
 }
 
@@ -144,8 +143,10 @@ Status CapturedFunction::Instantiate(
     ret_types.push_back(ret_type);
   }
 
-  instantiated_captured_function->reset(new InstantiatedCapturedFunction(
-      lib, f_handle, std::move(ret_types), *ctx->runner(), this));
+  *instantiated_captured_function =
+      absl::WrapUnique<InstantiatedCapturedFunction>(
+          new InstantiatedCapturedFunction(lib, f_handle, std::move(ret_types),
+                                           *ctx->runner(), this));
   return Status::OK();
 }
 
@@ -422,11 +423,11 @@ void InstantiatedCapturedFunction::RunAsync(
   // (such as queue kernels) that depend on the non-nullness of
   // `OpKernelContext::cancellation_manager()`, but additional effort
   // will be required to plumb it through the `IteratorContext`.
-  CancellationManager* c_mgr = new CancellationManager;
+  CancellationManager* c_mgr = new CancellationManager();
   f_opts.cancellation_manager = c_mgr;
   std::shared_ptr<SimpleStepStatsCollector> stats_collector;
   if (ctx->model() || ctx->stats_aggregator()) {
-    stats_collector = MakeUnique<SimpleStepStatsCollector>();
+    stats_collector = absl::make_unique<SimpleStepStatsCollector>();
   }
   f_opts.stats_collector = stats_collector.get();
 
diff --git a/tensorflow/core/kernels/data/concatenate_dataset_op.cc b/tensorflow/core/kernels/data/concatenate_dataset_op.cc
index 066b2c9aef..1d7c3a65d5 100644
--- a/tensorflow/core/kernels/data/concatenate_dataset_op.cc
+++ b/tensorflow/core/kernels/data/concatenate_dataset_op.cc
@@ -63,8 +63,8 @@ class ConcatenateDatasetOp : public BinaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::Concatenate")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::Concatenate")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/experimental/assert_next_dataset_op.cc b/tensorflow/core/kernels/data/experimental/assert_next_dataset_op.cc
index 3e87f484b9..eb54713360 100644
--- a/tensorflow/core/kernels/data/experimental/assert_next_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/assert_next_dataset_op.cc
@@ -61,8 +61,8 @@ class AssertNextDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::Assert")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::AssertNext")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/experimental/csv_dataset_op.cc b/tensorflow/core/kernels/data/experimental/csv_dataset_op.cc
index f6f58fc430..4435c2a131 100644
--- a/tensorflow/core/kernels/data/experimental/csv_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/csv_dataset_op.cc
@@ -12,8 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-
-// See docs in ../ops/parsing_ops.cc.
 #include "tensorflow/core/framework/common_shape_fns.h"
 #include "tensorflow/core/framework/dataset.h"
 #include "tensorflow/core/framework/op.h"
@@ -159,8 +157,8 @@ class CSVDatasetOp : public DatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::CSV")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::CSV")});
     }
 
     const DataTypeVector& output_dtypes() const override { return out_type_; }
diff --git a/tensorflow/core/kernels/data/experimental/dense_to_sparse_batch_dataset_op.cc b/tensorflow/core/kernels/data/experimental/dense_to_sparse_batch_dataset_op.cc
index 97e64dd744..31f081a727 100644
--- a/tensorflow/core/kernels/data/experimental/dense_to_sparse_batch_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/dense_to_sparse_batch_dataset_op.cc
@@ -96,8 +96,8 @@ class DenseToSparseBatchDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(new Iterator(
-          {this, strings::StrCat(prefix, "::DenseToSparseBatch")}));
+      return absl::make_unique<Iterator>(typename Iterator::Params{
+          this, strings::StrCat(prefix, "::DenseToSparseBatch")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/experimental/directed_interleave_dataset_op.cc b/tensorflow/core/kernels/data/experimental/directed_interleave_dataset_op.cc
index d8bb696167..f55718a006 100644
--- a/tensorflow/core/kernels/data/experimental/directed_interleave_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/directed_interleave_dataset_op.cc
@@ -93,8 +93,8 @@ class DirectedInterleaveDatasetOp : public DatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(new Iterator(
-          {this, strings::StrCat(prefix, "::DirectedInterleave")}));
+      return absl::make_unique<Iterator>(Iterator::Params{
+          this, strings::StrCat(prefix, "::DirectedInterleave")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/experimental/group_by_reducer_dataset_op.cc b/tensorflow/core/kernels/data/experimental/group_by_reducer_dataset_op.cc
index 5f0c01be4b..56159593a9 100644
--- a/tensorflow/core/kernels/data/experimental/group_by_reducer_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/group_by_reducer_dataset_op.cc
@@ -90,8 +90,8 @@ class GroupByReducerDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::GroupByReducer")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::GroupByReducer")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/experimental/group_by_window_dataset_op.cc b/tensorflow/core/kernels/data/experimental/group_by_window_dataset_op.cc
index 11491e00db..49122807b2 100644
--- a/tensorflow/core/kernels/data/experimental/group_by_window_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/group_by_window_dataset_op.cc
@@ -89,8 +89,8 @@ class GroupByWindowDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::GroupByWindow")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::GroupByWindow")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/experimental/ignore_errors_dataset_op.cc b/tensorflow/core/kernels/data/experimental/ignore_errors_dataset_op.cc
index d445d9c809..5e07bdb32e 100644
--- a/tensorflow/core/kernels/data/experimental/ignore_errors_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/ignore_errors_dataset_op.cc
@@ -45,8 +45,8 @@ class IgnoreErrorsDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::IgnoreErrors")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::IgnoreErrors")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/experimental/indexed_dataset_op.cc b/tensorflow/core/kernels/data/experimental/indexed_dataset_op.cc
index 83eeed7892..e75e6e4b80 100644
--- a/tensorflow/core/kernels/data/experimental/indexed_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/indexed_dataset_op.cc
@@ -12,7 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-
 #include "tensorflow/core/framework/dataset.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/resource_mgr.h"
@@ -424,7 +423,7 @@ class IdentityIndexedDatasetOp : public IndexedDatasetOpKernel {
 
     Status MaterializeDataset(
         std::shared_ptr<MaterializedIndexedDataset>* materialized) override {
-      materialized->reset(new Materialized(this));
+      (*materialized) = std::make_shared<Materialized>(this);
       return Status::OK();
     }
 
@@ -441,8 +440,8 @@ class IdentityIndexedDatasetOp : public IndexedDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(new Iterator(
-          {this, strings::StrCat(prefix, "::IdentityIndexedDataset")}));
+      return absl::make_unique<Iterator>(Iterator::Params{
+          this, strings::StrCat(prefix, "::IdentityIndexedDataset")});
     }
 
     string DebugString() const override {
diff --git a/tensorflow/core/kernels/data/experimental/lmdb_dataset_op.cc b/tensorflow/core/kernels/data/experimental/lmdb_dataset_op.cc
index 6248eb775e..cf900f1336 100644
--- a/tensorflow/core/kernels/data/experimental/lmdb_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/lmdb_dataset_op.cc
@@ -12,7 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-
 #include <sys/stat.h>
 
 #include "tensorflow/core/framework/dataset.h"
@@ -52,8 +51,8 @@ class LMDBDatasetOp : public DatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::LMDB")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::LMDB")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc b/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc
index a6348e4647..5d8e7cfabe 100644
--- a/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc
@@ -32,7 +32,6 @@ limitations under the License.
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/cpu_info.h"
 #include "tensorflow/core/platform/tracing.h"
-#include "tensorflow/core/util/ptr_util.h"
 
 namespace tensorflow {
 namespace data {
@@ -163,7 +162,7 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return MakeUnique<Iterator>(
+      return absl::make_unique<Iterator>(
           Iterator::Params{this, strings::StrCat(prefix, "::MapAndBatch")},
           map_func_);
     }
diff --git a/tensorflow/core/kernels/data/experimental/matching_files_dataset_op.cc b/tensorflow/core/kernels/data/experimental/matching_files_dataset_op.cc
index aa27a13416..381b9691d1 100644
--- a/tensorflow/core/kernels/data/experimental/matching_files_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/matching_files_dataset_op.cc
@@ -61,8 +61,8 @@ class MatchingFilesDatasetOp : public DatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::MatchingFiles")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::MatchingFiles")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/experimental/non_serializable_dataset_op.cc b/tensorflow/core/kernels/data/experimental/non_serializable_dataset_op.cc
index 61811ea14e..9ca8e33b94 100644
--- a/tensorflow/core/kernels/data/experimental/non_serializable_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/non_serializable_dataset_op.cc
@@ -53,8 +53,8 @@ class NonSerializableDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::NonSerializable")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::NonSerializable")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/experimental/numa_map_and_batch_dataset_op.cc b/tensorflow/core/kernels/data/experimental/numa_map_and_batch_dataset_op.cc
index 2b1aec358c..643b6460e8 100644
--- a/tensorflow/core/kernels/data/experimental/numa_map_and_batch_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/numa_map_and_batch_dataset_op.cc
@@ -32,7 +32,6 @@ limitations under the License.
 #include "tensorflow/core/platform/cpu_info.h"
 #include "tensorflow/core/platform/numa.h"
 #include "tensorflow/core/platform/tracing.h"
-#include "tensorflow/core/util/ptr_util.h"
 
 namespace tensorflow {
 namespace data {
@@ -121,8 +120,8 @@ class NumaMapAndBatchDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::NumaMapAndBatch")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::NumaMapAndBatch")});
     }
 
     const DataTypeVector& output_dtypes() const override {
@@ -321,7 +320,7 @@ class NumaMapAndBatchDatasetOp : public UnaryDatasetOpKernel {
         }
         workers_.resize(num_workers);
         for (size_t i = 0; i < num_workers; ++i) {
-          workers_[i] = MakeUnique<NumaWorkerBlock>(this);
+          workers_[i] = absl::make_unique<NumaWorkerBlock>(this);
           TF_RETURN_IF_ERROR(
               workers_[i]->manager.Restore(ctx, reader, this, i));
         }
diff --git a/tensorflow/core/kernels/data/experimental/parallel_interleave_dataset_op.cc b/tensorflow/core/kernels/data/experimental/parallel_interleave_dataset_op.cc
index 1c19119d88..f6d522078d 100644
--- a/tensorflow/core/kernels/data/experimental/parallel_interleave_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/parallel_interleave_dataset_op.cc
@@ -26,7 +26,6 @@ limitations under the License.
 #include "tensorflow/core/lib/core/threadpool.h"
 #include "tensorflow/core/lib/gtl/cleanup.h"
 #include "tensorflow/core/lib/random/random.h"
-#include "tensorflow/core/util/ptr_util.h"
 
 namespace tensorflow {
 namespace data {
@@ -113,8 +112,8 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(new Iterator(
-          {this, strings::StrCat(prefix, "::ParallelInterleave")}));
+      return absl::make_unique<Iterator>(Iterator::Params{
+          this, strings::StrCat(prefix, "::ParallelInterleave")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/experimental/parse_example_dataset_op.cc b/tensorflow/core/kernels/data/experimental/parse_example_dataset_op.cc
index ea99a8b32c..0057405734 100644
--- a/tensorflow/core/kernels/data/experimental/parse_example_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/parse_example_dataset_op.cc
@@ -183,8 +183,8 @@ class ParseExampleDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      std::unique_ptr<ParallelMapFunctor> parse_example_functor(
-          new ParseExampleFunctor(this));
+      std::unique_ptr<ParallelMapFunctor> parse_example_functor =
+          absl::make_unique<ParseExampleFunctor>(this);
       return NewParallelMapIterator(
           {this, strings::StrCat(prefix, "::ParseExample")}, input_,
           std::move(parse_example_functor), num_parallel_calls_, sloppy_,
diff --git a/tensorflow/core/kernels/data/experimental/random_dataset_op.cc b/tensorflow/core/kernels/data/experimental/random_dataset_op.cc
index 6d85cd5c45..114bb6a856 100644
--- a/tensorflow/core/kernels/data/experimental/random_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/random_dataset_op.cc
@@ -56,8 +56,8 @@ class RandomDatasetOp : public DatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::Random")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::Random")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/experimental/scan_dataset_op.cc b/tensorflow/core/kernels/data/experimental/scan_dataset_op.cc
index 87263b5606..55e22c1cac 100644
--- a/tensorflow/core/kernels/data/experimental/scan_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/scan_dataset_op.cc
@@ -84,8 +84,8 @@ class ScanDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::Scan")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::Scan")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/experimental/set_stats_aggregator_dataset_op.cc b/tensorflow/core/kernels/data/experimental/set_stats_aggregator_dataset_op.cc
index fe128005fa..67bb1e160b 100644
--- a/tensorflow/core/kernels/data/experimental/set_stats_aggregator_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/set_stats_aggregator_dataset_op.cc
@@ -13,6 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 #include <memory>
+
 #include "tensorflow/core/framework/dataset.h"
 #include "tensorflow/core/framework/partial_tensor_shape.h"
 #include "tensorflow/core/framework/stats_aggregator.h"
@@ -114,8 +115,8 @@ class SetStatsAggregatorDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(new Iterator(
-          {this, strings::StrCat(prefix, "::SetStatsAggregator")}));
+      return absl::make_unique<Iterator>(Iterator::Params{
+          this, strings::StrCat(prefix, "::SetStatsAggregator")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/experimental/sleep_dataset_op.cc b/tensorflow/core/kernels/data/experimental/sleep_dataset_op.cc
index d2fb8ac4f3..9d63690622 100644
--- a/tensorflow/core/kernels/data/experimental/sleep_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/sleep_dataset_op.cc
@@ -15,7 +15,6 @@ limitations under the License.
 #include "tensorflow/core/framework/dataset.h"
 #include "tensorflow/core/framework/partial_tensor_shape.h"
 #include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/util/ptr_util.h"
 
 namespace tensorflow {
 namespace data {
@@ -55,7 +54,7 @@ class SleepDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return MakeUnique<Iterator>(
+      return absl::make_unique<Iterator>(
           Iterator::Params{this, strings::StrCat(prefix, "::Sleep")});
     }
 
diff --git a/tensorflow/core/kernels/data/experimental/sliding_window_dataset_op.cc b/tensorflow/core/kernels/data/experimental/sliding_window_dataset_op.cc
index 1ce4fbd313..c5851eaf86 100644
--- a/tensorflow/core/kernels/data/experimental/sliding_window_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/sliding_window_dataset_op.cc
@@ -12,7 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-
 #include <deque>
 #include <vector>
 
@@ -86,8 +85,8 @@ class SlidingWindowDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(new Iterator(
-          Iterator::Params{this, strings::StrCat(prefix, "::Slide")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::Slide")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/experimental/sql_dataset_op.cc b/tensorflow/core/kernels/data/experimental/sql_dataset_op.cc
index c16d8ed02c..84f6fba36d 100644
--- a/tensorflow/core/kernels/data/experimental/sql_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/sql_dataset_op.cc
@@ -91,8 +91,8 @@ class SqlDatasetOp : public DatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::Sql")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::Sql")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/experimental/stats_aggregator_ops.cc b/tensorflow/core/kernels/data/experimental/stats_aggregator_ops.cc
index 894465e181..1d1b788b6c 100644
--- a/tensorflow/core/kernels/data/experimental/stats_aggregator_ops.cc
+++ b/tensorflow/core/kernels/data/experimental/stats_aggregator_ops.cc
@@ -108,8 +108,8 @@ class StatsAggregatorHandleOp
  private:
   Status CreateResource(StatsAggregatorResource** ret) override
       EXCLUSIVE_LOCKS_REQUIRED(mu_) {
-    *ret = new StatsAggregatorResource(
-        std::unique_ptr<StatsAggregator>(new StatsAggregatorImpl));
+    *ret =
+        new StatsAggregatorResource(absl::make_unique<StatsAggregatorImpl>());
     return Status::OK();
   }
 
diff --git a/tensorflow/core/kernels/data/experimental/stats_dataset_ops.cc b/tensorflow/core/kernels/data/experimental/stats_dataset_ops.cc
index 1961f25df8..bf96be4eb0 100644
--- a/tensorflow/core/kernels/data/experimental/stats_dataset_ops.cc
+++ b/tensorflow/core/kernels/data/experimental/stats_dataset_ops.cc
@@ -12,7 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-
 #include "tensorflow/core/example/example.pb.h"
 #include "tensorflow/core/example/feature.pb.h"
 #include "tensorflow/core/framework/dataset.h"
@@ -63,8 +62,8 @@ class LatencyStatsDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::LatencyStats")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::LatencyStats")});
     }
 
     const DataTypeVector& output_dtypes() const override {
@@ -173,8 +172,8 @@ class BytesProducedStatsDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(new Iterator(
-          {this, strings::StrCat(prefix, "::BytesProducedStats")}));
+      return absl::make_unique<Iterator>(Iterator::Params{
+          this, strings::StrCat(prefix, "::BytesProducedStats")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/experimental/threadpool_dataset_op.cc b/tensorflow/core/kernels/data/experimental/threadpool_dataset_op.cc
index fab3cab7da..7a16cda0f3 100644
--- a/tensorflow/core/kernels/data/experimental/threadpool_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/threadpool_dataset_op.cc
@@ -12,13 +12,12 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-
 #include <memory>
+
 #include "tensorflow/core/framework/dataset.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/resource_mgr.h"
 #include "tensorflow/core/lib/core/threadpool.h"
-#include "tensorflow/core/util/ptr_util.h"
 #include "tensorflow/core/util/work_sharder.h"
 
 namespace tensorflow {
@@ -154,8 +153,8 @@ class ThreadPoolDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::ThreadPool")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::ThreadPool")});
     }
 
     const DataTypeVector& output_dtypes() const override {
@@ -261,8 +260,8 @@ class MaxIntraOpParallelismDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(new Iterator(
-          {this, strings::StrCat(prefix, "::MaxIntraOpParallelism")}));
+      return absl::make_unique<Iterator>(Iterator::Params{
+          this, strings::StrCat(prefix, "::MaxIntraOpParallelism")});
     }
 
     const DataTypeVector& output_dtypes() const override {
@@ -362,7 +361,7 @@ class PrivateThreadPoolDatasetOp : public UnaryDatasetOpKernel {
         : DatasetBase(DatasetContext(ctx)),
           input_(input),
           num_threads_(num_threads) {
-      thread_pool_ = MakeUnique<thread::ThreadPool>(
+      thread_pool_ = absl::make_unique<thread::ThreadPool>(
           ctx->env(), ThreadOptions{}, "data_private_threadpool", num_threads,
           /*low_latency_hint=*/false);
       input_->Ref();
@@ -372,8 +371,8 @@ class PrivateThreadPoolDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::PrivateThreadPool")}));
+      return absl::make_unique<Iterator>(Iterator::Params{
+          this, strings::StrCat(prefix, "::PrivateThreadPool")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/experimental/to_tf_record_op.cc b/tensorflow/core/kernels/data/experimental/to_tf_record_op.cc
index 7728baf150..6cf6198432 100644
--- a/tensorflow/core/kernels/data/experimental/to_tf_record_op.cc
+++ b/tensorflow/core/kernels/data/experimental/to_tf_record_op.cc
@@ -12,7 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-
 #include "tensorflow/core/framework/dataset.h"
 #include "tensorflow/core/framework/function_handle_cache.h"
 #include "tensorflow/core/framework/op_kernel.h"
@@ -59,18 +58,18 @@ class ToTFRecordOp : public AsyncOpKernel {
       std::unique_ptr<WritableFile> file;
       OP_REQUIRES_OK_ASYNC(ctx, ctx->env()->NewWritableFile(filename, &file),
                            done);
-      std::unique_ptr<io::RecordWriter> writer;
-      writer.reset(new io::RecordWriter(
-          file.get(), io::RecordWriterOptions::CreateRecordWriterOptions(
-                          compression_type)));
+      std::unique_ptr<io::RecordWriter> writer =
+          absl::make_unique<io::RecordWriter>(
+              file.get(), io::RecordWriterOptions::CreateRecordWriterOptions(
+                              compression_type));
 
       DatasetBase* dataset;
       OP_REQUIRES_OK_ASYNC(
           ctx, GetDatasetFromVariantTensor(ctx->input(0), &dataset), done);
       std::unique_ptr<IteratorBase> iterator;
       IteratorContext::Params params(ctx);
-      std::unique_ptr<FunctionHandleCache> function_handle_cache(
-          new FunctionHandleCache(params.lib));
+      std::unique_ptr<FunctionHandleCache> function_handle_cache =
+          absl::make_unique<FunctionHandleCache>(params.lib);
       params.function_handle_cache = function_handle_cache.get();
       IteratorContext iter_ctx(std::move(params));
 
diff --git a/tensorflow/core/kernels/data/experimental/unbatch_dataset_op.cc b/tensorflow/core/kernels/data/experimental/unbatch_dataset_op.cc
index 2626ec3ed7..cb26fd3e43 100644
--- a/tensorflow/core/kernels/data/experimental/unbatch_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/unbatch_dataset_op.cc
@@ -58,8 +58,8 @@ class UnbatchDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::Unbatch")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::Unbatch")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/experimental/unique_dataset_op.cc b/tensorflow/core/kernels/data/experimental/unique_dataset_op.cc
index 23dd9ff612..57865c45fc 100644
--- a/tensorflow/core/kernels/data/experimental/unique_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/unique_dataset_op.cc
@@ -58,8 +58,8 @@ class UniqueDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::Unique")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::Unique")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/filter_by_component_dataset_op.cc b/tensorflow/core/kernels/data/filter_by_component_dataset_op.cc
index 784f987286..3b9b319ea9 100644
--- a/tensorflow/core/kernels/data/filter_by_component_dataset_op.cc
+++ b/tensorflow/core/kernels/data/filter_by_component_dataset_op.cc
@@ -64,8 +64,8 @@ class FilterByLastComponentDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<Iterator>(new Iterator(
-          {this, strings::StrCat(prefix, "::FilterByLastComponent")}));
+      return absl::make_unique<Iterator>(Iterator::Params{
+          this, strings::StrCat(prefix, "::FilterByLastComponent")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/filter_dataset_op.cc b/tensorflow/core/kernels/data/filter_dataset_op.cc
index 30b2fc5db8..483d42c809 100644
--- a/tensorflow/core/kernels/data/filter_dataset_op.cc
+++ b/tensorflow/core/kernels/data/filter_dataset_op.cc
@@ -22,7 +22,6 @@ limitations under the License.
 #include "tensorflow/core/lib/gtl/cleanup.h"
 #include "tensorflow/core/lib/random/random.h"
 #include "tensorflow/core/lib/strings/str_util.h"
-#include "tensorflow/core/util/ptr_util.h"
 
 namespace tensorflow {
 namespace data {
@@ -109,7 +108,7 @@ class FilterDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return MakeUnique<Iterator>(
+      return absl::make_unique<Iterator>(
           Iterator::Params{this, strings::StrCat(prefix, "::Filter")},
           filter_pred_);
     }
diff --git a/tensorflow/core/kernels/data/flat_map_dataset_op.cc b/tensorflow/core/kernels/data/flat_map_dataset_op.cc
index efa76ab34b..3f01ac5569 100644
--- a/tensorflow/core/kernels/data/flat_map_dataset_op.cc
+++ b/tensorflow/core/kernels/data/flat_map_dataset_op.cc
@@ -67,8 +67,8 @@ class FlatMapDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::FlatMap")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::FlatMap")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/generator_dataset_op.cc b/tensorflow/core/kernels/data/generator_dataset_op.cc
index 48697ec6c8..5dff2be39d 100644
--- a/tensorflow/core/kernels/data/generator_dataset_op.cc
+++ b/tensorflow/core/kernels/data/generator_dataset_op.cc
@@ -12,11 +12,11 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
+#include "tensorflow/core/kernels/data/generator_dataset_op.h"
+
 #include <iterator>
 #include <vector>
 
-#include "tensorflow/core/kernels/data/generator_dataset_op.h"
-
 #include "tensorflow/core/framework/partial_tensor_shape.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/kernels/data/captured_function.h"
@@ -44,8 +44,8 @@ class GeneratorDatasetOp::Dataset : public DatasetBase {
 
   std::unique_ptr<IteratorBase> MakeIteratorInternal(
       const string& prefix) const override {
-    return std::unique_ptr<IteratorBase>(
-        new Iterator({this, strings::StrCat(prefix, "::Generator")}));
+    return absl::make_unique<Iterator>(
+        Iterator::Params{this, strings::StrCat(prefix, "::Generator")});
   }
 
   const DataTypeVector& output_dtypes() const override { return output_types_; }
diff --git a/tensorflow/core/kernels/data/interleave_dataset_op.cc b/tensorflow/core/kernels/data/interleave_dataset_op.cc
index 1a5e6edb5b..69310bcff2 100644
--- a/tensorflow/core/kernels/data/interleave_dataset_op.cc
+++ b/tensorflow/core/kernels/data/interleave_dataset_op.cc
@@ -89,8 +89,8 @@ class InterleaveDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::Interleave")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::Interleave")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/iterator_ops.cc b/tensorflow/core/kernels/data/iterator_ops.cc
index 81e26d35c0..a8c3f7f77f 100644
--- a/tensorflow/core/kernels/data/iterator_ops.cc
+++ b/tensorflow/core/kernels/data/iterator_ops.cc
@@ -38,7 +38,6 @@ limitations under the License.
 #include "tensorflow/core/platform/env.h"
 #include "tensorflow/core/platform/mutex.h"
 #include "tensorflow/core/public/session_options.h"
-#include "tensorflow/core/util/ptr_util.h"
 
 namespace tensorflow {
 namespace data {
@@ -137,7 +136,7 @@ class IteratorResource : public ResourceBase {
     std::unique_ptr<ProcessFunctionLibraryRuntime> pflr(nullptr);
     TF_RETURN_IF_ERROR(ctx->function_library()->Clone(&flib_def, &pflr, &lib));
     TF_RETURN_IF_ERROR(flib_def->AddLibrary(graph_def.library()));
-    std::unique_ptr<State> new_state = MakeUnique<State>(
+    std::unique_ptr<State> new_state = absl::make_unique<State>(
         std::move(flib_def), std::move(pflr), lib, nullptr /* iterator */);
 
     TF_RETURN_IF_ERROR(
@@ -211,7 +210,7 @@ class IteratorResource : public ResourceBase {
     }
 
     new_state->function_handle_cache =
-        MakeUnique<FunctionHandleCache>(new_state->lib);
+        absl::make_unique<FunctionHandleCache>(new_state->lib);
     // Create new iterator.
     std::unique_ptr<IteratorBase> iterator;
     IteratorContext::Params params(ctx);
@@ -247,7 +246,7 @@ class IteratorResource : public ResourceBase {
         : flib_def(flib_def),
           pflr(pflr),
           lib(lib),
-          function_handle_cache(MakeUnique<FunctionHandleCache>(lib)),
+          function_handle_cache(absl::make_unique<FunctionHandleCache>(lib)),
           iterator(std::move(iterator)) {}
 
     State(std::shared_ptr<FunctionLibraryDefinition> flib_def,
@@ -434,7 +433,7 @@ class IteratorStateVariant {
     SerializationContext::Params params;
     params.flib_def = ctx->function_library()->GetFunctionLibraryDefinition();
     SerializationContext serialization_ctx(params);
-    data_ = MakeUnique<VariantTensorData>();
+    data_ = absl::make_unique<VariantTensorData>();
     data_->set_type_name(TypeName());
     VariantTensorDataWriter writer(data_.get());
     TF_RETURN_IF_ERROR(iterator_resource->Save(&serialization_ctx, &writer));
@@ -448,10 +447,10 @@ class IteratorStateVariant {
       return false;
     }
     std::unique_ptr<VariantTensorData> tensor_data =
-        MakeUnique<VariantTensorData>();
+        absl::make_unique<VariantTensorData>();
     std::swap(*tensor_data, data);
     std::unique_ptr<VariantTensorDataReader> reader =
-        MakeUnique<VariantTensorDataReader>(tensor_data.get());
+        absl::make_unique<VariantTensorDataReader>(tensor_data.get());
     status_ = reader->status();
     if (!status_.ok()) {
       return false;
@@ -585,9 +584,9 @@ FunctionLibraryRuntime* IteratorHandleOp::CreatePrivateFLR(
   *device_mgr = absl::make_unique<DeviceMgr>(RenamedDevice::NewRenamedDevice(
       ctx->device()->name(), down_cast<Device*>(ctx->device()),
       false /* owns_underlying */, false /* isolate_session_state */));
-  *flib_def = MakeUnique<FunctionLibraryDefinition>(
+  *flib_def = absl::make_unique<FunctionLibraryDefinition>(
       *ctx->function_library()->GetFunctionLibraryDefinition());
-  *pflr = MakeUnique<ProcessFunctionLibraryRuntime>(
+  *pflr = absl::make_unique<ProcessFunctionLibraryRuntime>(
       device_mgr->get(), ctx->env(), graph_def_version_, flib_def->get(),
       OptimizerOptions{} /* TODO(mrry): OptimizerOptions? */,
       nullptr /* TODO(mrry): ClusterFLR */);
@@ -679,9 +678,10 @@ class ToSingleElementOp : public AsyncOpKernel {
       std::unique_ptr<IteratorBase> iterator;
       IteratorContext::Params params(ctx);
       std::unique_ptr<FunctionHandleCache> function_handle_cache =
-          MakeUnique<FunctionHandleCache>(params.lib);
+          absl::make_unique<FunctionHandleCache>(params.lib);
       params.function_handle_cache = function_handle_cache.get();
-      std::unique_ptr<ResourceMgr> resource_mgr = MakeUnique<ResourceMgr>();
+      std::unique_ptr<ResourceMgr> resource_mgr =
+          absl::make_unique<ResourceMgr>();
       params.resource_mgr = resource_mgr.get();
       IteratorContext iter_ctx(std::move(params));
 
@@ -769,9 +769,10 @@ class ReduceDatasetOp : public AsyncOpKernel {
 
       IteratorContext::Params params(ctx);
       std::unique_ptr<FunctionHandleCache> function_handle_cache =
-          MakeUnique<FunctionHandleCache>(params.lib);
+          absl::make_unique<FunctionHandleCache>(params.lib);
       params.function_handle_cache = function_handle_cache.get();
-      std::unique_ptr<ResourceMgr> resource_mgr = MakeUnique<ResourceMgr>();
+      std::unique_ptr<ResourceMgr> resource_mgr =
+          absl::make_unique<ResourceMgr>();
       params.resource_mgr = resource_mgr.get();
       IteratorContext iter_ctx(std::move(params));
       std::unique_ptr<InstantiatedCapturedFunction> instantiated_captured_func;
diff --git a/tensorflow/core/kernels/data/map_dataset_op.cc b/tensorflow/core/kernels/data/map_dataset_op.cc
index 02c0199a0c..95f4c1c891 100644
--- a/tensorflow/core/kernels/data/map_dataset_op.cc
+++ b/tensorflow/core/kernels/data/map_dataset_op.cc
@@ -19,7 +19,6 @@ limitations under the License.
 #include "tensorflow/core/kernels/data/captured_function.h"
 #include "tensorflow/core/kernels/data/dataset_utils.h"
 #include "tensorflow/core/lib/random/random.h"
-#include "tensorflow/core/util/ptr_util.h"
 
 namespace tensorflow {
 namespace data {
@@ -120,7 +119,7 @@ class MapDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return MakeUnique<Iterator>(
+      return absl::make_unique<Iterator>(
           Iterator::Params{this, strings::StrCat(prefix, "::Map")}, map_func_);
     }
 
diff --git a/tensorflow/core/kernels/data/map_defun_op.cc b/tensorflow/core/kernels/data/map_defun_op.cc
index 705b0393de..8122048702 100644
--- a/tensorflow/core/kernels/data/map_defun_op.cc
+++ b/tensorflow/core/kernels/data/map_defun_op.cc
@@ -92,7 +92,7 @@ class MapDefunOp : public AsyncOpKernel {
       // We use a different cancellation manager each time the function is run
       // to avoid the race condition between a function run error and other
       // functions being cancelled as a result.
-      CancellationManager* c_mgr = new CancellationManager;
+      CancellationManager* c_mgr = new CancellationManager();
       CancellationToken token = parent_mgr->get_cancellation_token();
       const bool success = parent_mgr->RegisterCallback(
           token, [c_mgr]() { c_mgr->StartCancel(); });
diff --git a/tensorflow/core/kernels/data/multi_device_iterator_ops.cc b/tensorflow/core/kernels/data/multi_device_iterator_ops.cc
index 05528e0ee0..eb08d2b28a 100644
--- a/tensorflow/core/kernels/data/multi_device_iterator_ops.cc
+++ b/tensorflow/core/kernels/data/multi_device_iterator_ops.cc
@@ -81,9 +81,8 @@ class MultiDeviceIterator : public ResourceBase {
     ++incarnation_id_;
     *incarnation_id = incarnation_id_;
 
-    multi_device_buffer_.reset(
-        new MultiDeviceBuffer(devices_.size(), max_buffer_size, incarnation_id_,
-                              std::move(iterator)));
+    multi_device_buffer_ = absl::make_unique<MultiDeviceBuffer>(
+        devices_.size(), max_buffer_size, incarnation_id_, std::move(iterator));
     return Status::OK();
   }
 
@@ -217,10 +216,11 @@ class MultiDeviceIterator : public ResourceBase {
     void EnsureBackgroundThreadStarted(IteratorContext* ctx)
         EXCLUSIVE_LOCKS_REQUIRED(mu_) {
       if (!background_thread_) {
-        background_thread_.reset(ctx->env()->StartThread(
+        auto ctx_copy = std::make_shared<IteratorContext>(*ctx);
+        background_thread_ = absl::WrapUnique<Thread>(ctx->env()->StartThread(
             {}, "tf_data_multi_device_iterator",
             std::bind(&MultiDeviceIterator::MultiDeviceBuffer::BackgroundThread,
-                      this, new IteratorContext(*ctx))));
+                      this, std::move(ctx_copy))));
       }
     }
 
@@ -258,12 +258,11 @@ class MultiDeviceIterator : public ResourceBase {
       }
     }
 
-    void BackgroundThread(IteratorContext* ctx) {
+    void BackgroundThread(std::shared_ptr<IteratorContext> ctx) {
       {
         mutex_lock l(mu_);
         background_thread_started_ = true;
       }
-      std::unique_ptr<IteratorContext> cleanup(ctx);
       int shard_to_fetch = 0;
       while (true) {
         HostBufferElement elem;
@@ -284,8 +283,8 @@ class MultiDeviceIterator : public ResourceBase {
           }
         }
 
-        elem.status =
-            host_iterator_->GetNext(ctx, &elem.value, &elem.end_of_sequence);
+        elem.status = host_iterator_->GetNext(ctx.get(), &elem.value,
+                                              &elem.end_of_sequence);
 
         if (elem.status.ok() && elem.end_of_sequence) {
           end_of_iterator = true;
@@ -397,8 +396,8 @@ class MultiDeviceIteratorHandleOp : public OpKernel {
         std::unique_ptr<ProcessFunctionLibraryRuntime> pflr(nullptr);
         OP_REQUIRES_OK(context, context->function_library()->Clone(
                                     &flib_def, &pflr, &lib));
-        std::unique_ptr<FunctionHandleCache> function_handle_cache(
-            new FunctionHandleCache(lib));
+        std::unique_ptr<FunctionHandleCache> function_handle_cache =
+            absl::make_unique<FunctionHandleCache>(lib);
         ResourceMgr* mgr = context->resource_manager();
         OP_REQUIRES_OK(context, cinfo_.Init(mgr, def()));
 
diff --git a/tensorflow/core/kernels/data/optional_ops.h b/tensorflow/core/kernels/data/optional_ops.h
index ef14e84311..7089a423d7 100644
--- a/tensorflow/core/kernels/data/optional_ops.h
+++ b/tensorflow/core/kernels/data/optional_ops.h
@@ -44,8 +44,9 @@ class OptionalVariant {
 
   // Create an `OptionalVariant` with the actual value given by the tuple of
   // tensors in `values`.
-  explicit OptionalVariant(std::vector<Tensor> values)
-      : values_(new std::vector<Tensor>(std::move(values))) {}
+  explicit OptionalVariant(std::vector<Tensor> values) {
+    values_ = std::make_shared<std::vector<Tensor>>(std::move(values));
+  }
 
   OptionalVariant(const OptionalVariant& other) : values_(other.values_) {}
 
@@ -79,7 +80,7 @@ class OptionalVariant {
       return false;
     }
     if (has_value) {
-      values_.reset(new std::vector<Tensor>(data.tensors()));
+      values_ = std::make_shared<std::vector<Tensor>>(data.tensors());
     } else {
       values_.reset();
     }
diff --git a/tensorflow/core/kernels/data/padded_batch_dataset_op.cc b/tensorflow/core/kernels/data/padded_batch_dataset_op.cc
index 0fff4c5370..41ea36263c 100644
--- a/tensorflow/core/kernels/data/padded_batch_dataset_op.cc
+++ b/tensorflow/core/kernels/data/padded_batch_dataset_op.cc
@@ -135,8 +135,8 @@ class PaddedBatchDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::PaddedBatch")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::PaddedBatch")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc b/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc
index 336fa52b49..add123f34a 100644
--- a/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc
+++ b/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc
@@ -29,7 +29,6 @@ limitations under the License.
 #include "tensorflow/core/lib/gtl/cleanup.h"
 #include "tensorflow/core/lib/random/random.h"
 #include "tensorflow/core/platform/cpu_info.h"
-#include "tensorflow/core/util/ptr_util.h"
 
 namespace tensorflow {
 namespace data {
@@ -120,7 +119,7 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return MakeUnique<ParallelInterleaveIterator>(
+      return absl::make_unique<ParallelInterleaveIterator>(
           ParallelInterleaveIterator::Params{
               this, strings::StrCat(prefix, "::ParallelInterleaveV2")},
           sloppy_);
@@ -200,7 +199,7 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
                 params.dataset->num_parallel_calls_, mu_, cond_var_)),
             sloppy_(sloppy),
             current_elements_(params.dataset->cycle_length_),
-            thread_pool_(new thread::ThreadPool(
+            thread_pool_(absl::make_unique<thread::ThreadPool>(
                 Env::Default(), ThreadOptions(),
                 "data_parallel_interleave_worker_pool",
                 port::NumSchedulableCPUs() /* num_threads */,
@@ -513,15 +512,16 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
         if (!current_elements_manager_) {
           auto new_ctx = std::make_shared<IteratorContext>(*ctx);
           current_elements_manager_ =
-              WrapUnique<Thread>(ctx->env()->StartThread(
+              absl::WrapUnique<Thread>(ctx->env()->StartThread(
                   {}, "tf_data_parallel_interleave_current",
                   [this, new_ctx]() { CurrentElementsManager(new_ctx); }));
         }
         if (!future_elements_manager_) {
           auto new_ctx = std::make_shared<IteratorContext>(*ctx);
-          future_elements_manager_ = WrapUnique<Thread>(ctx->env()->StartThread(
-              {}, "tf_data_parallel_interleave_future",
-              [this, new_ctx]() { FutureElementsManager(new_ctx); }));
+          future_elements_manager_ =
+              absl::WrapUnique<Thread>(ctx->env()->StartThread(
+                  {}, "tf_data_parallel_interleave_future",
+                  [this, new_ctx]() { FutureElementsManager(new_ctx); }));
         }
       }
 
diff --git a/tensorflow/core/kernels/data/parallel_map_dataset_op.cc b/tensorflow/core/kernels/data/parallel_map_dataset_op.cc
index c0002c86d8..e2c9057cb2 100644
--- a/tensorflow/core/kernels/data/parallel_map_dataset_op.cc
+++ b/tensorflow/core/kernels/data/parallel_map_dataset_op.cc
@@ -102,9 +102,10 @@ class ParallelMapDatasetOp : public UnaryDatasetOpKernel {
         const string& prefix) const override {
       std::unique_ptr<ParallelMapFunctor> parallel_map_functor(nullptr);
       if (indices_.empty()) {
-        parallel_map_functor.reset(new ParallelMapDatasetFunctor(this));
+        parallel_map_functor =
+            absl::make_unique<ParallelMapDatasetFunctor>(this);
       } else {
-        parallel_map_functor.reset(new ShortCircuitFunctor(this));
+        parallel_map_functor = absl::make_unique<ShortCircuitFunctor>(this);
       }
       return NewParallelMapIterator(
           {this, strings::StrCat(prefix, "::ParallelMap")}, input_,
diff --git a/tensorflow/core/kernels/data/parallel_map_iterator.cc b/tensorflow/core/kernels/data/parallel_map_iterator.cc
index 9a4ce981d0..be91de12fe 100644
--- a/tensorflow/core/kernels/data/parallel_map_iterator.cc
+++ b/tensorflow/core/kernels/data/parallel_map_iterator.cc
@@ -24,7 +24,6 @@ limitations under the License.
 #include "tensorflow/core/framework/stats_aggregator.h"
 #include "tensorflow/core/lib/gtl/cleanup.h"
 #include "tensorflow/core/platform/cpu_info.h"
-#include "tensorflow/core/util/ptr_util.h"
 
 namespace tensorflow {
 namespace data {
@@ -410,7 +409,7 @@ std::unique_ptr<IteratorBase> NewParallelMapIterator(
     const DatasetBase* input_dataset,
     std::unique_ptr<ParallelMapFunctor> parallel_map_functor,
     int32 num_parallel_calls, bool sloppy, bool preserve_cardinality) {
-  return MakeUnique<ParallelMapIterator>(
+  return absl::make_unique<ParallelMapIterator>(
       params, input_dataset,
       ParallelMapIterator::Params{std::move(parallel_map_functor),
                                   num_parallel_calls, sloppy,
diff --git a/tensorflow/core/kernels/data/prefetch_dataset_op.cc b/tensorflow/core/kernels/data/prefetch_dataset_op.cc
index 08d6de4bf9..1f2dd6ca95 100644
--- a/tensorflow/core/kernels/data/prefetch_dataset_op.cc
+++ b/tensorflow/core/kernels/data/prefetch_dataset_op.cc
@@ -42,8 +42,8 @@ class PrefetchDatasetOp::Dataset : public DatasetBase {
 
   std::unique_ptr<IteratorBase> MakeIteratorInternal(
       const string& prefix) const override {
-    return std::unique_ptr<IteratorBase>(
-        new Iterator({this, strings::StrCat(prefix, "::Prefetch")}));
+    return absl::make_unique<Iterator>(
+        Iterator::Params{this, strings::StrCat(prefix, "::Prefetch")});
   }
 
   const DataTypeVector& output_dtypes() const override {
@@ -266,8 +266,9 @@ class PrefetchDatasetOp::Dataset : public DatasetBase {
     Status EnsurePrefetchThreadStarted(IteratorContext* ctx)
         EXCLUSIVE_LOCKS_REQUIRED(mu_) {
       if (!prefetch_thread_) {
-        std::shared_ptr<IteratorContext> new_ctx(new IteratorContext(*ctx));
-        prefetch_thread_.reset(ctx->env()->StartThread(
+        std::shared_ptr<IteratorContext> new_ctx =
+            std::make_shared<IteratorContext>(*ctx);
+        prefetch_thread_ = absl::WrapUnique<Thread>(ctx->env()->StartThread(
             {}, "tf_data_prefetch",
             [this, new_ctx]() { PrefetchThread(new_ctx); }));
       }
diff --git a/tensorflow/core/kernels/data/range_dataset_op.cc b/tensorflow/core/kernels/data/range_dataset_op.cc
index 580702f741..aa14d27d5c 100644
--- a/tensorflow/core/kernels/data/range_dataset_op.cc
+++ b/tensorflow/core/kernels/data/range_dataset_op.cc
@@ -53,8 +53,8 @@ class RangeDatasetOp : public DatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::Range")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::Range")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/reader_dataset_ops.cc b/tensorflow/core/kernels/data/reader_dataset_ops.cc
index 971fd2a436..789f9c859a 100644
--- a/tensorflow/core/kernels/data/reader_dataset_ops.cc
+++ b/tensorflow/core/kernels/data/reader_dataset_ops.cc
@@ -91,8 +91,8 @@ class TextLineDatasetOp : public DatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::TextLine")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::TextLine")});
     }
 
     const DataTypeVector& output_dtypes() const override {
@@ -221,20 +221,20 @@ class TextLineDatasetOp : public DatasetOpKernel {
         // Actually move on to next file.
         TF_RETURN_IF_ERROR(env->NewRandomAccessFile(
             dataset()->filenames_[current_file_index_], &file_));
-        input_stream_.reset(
-            new io::RandomAccessInputStream(file_.get(), false));
+        input_stream_ =
+            absl::make_unique<io::RandomAccessInputStream>(file_.get(), false);
 
         if (dataset()->use_compression_) {
-          zlib_input_stream_.reset(new io::ZlibInputStream(
+          zlib_input_stream_ = absl::make_unique<io::ZlibInputStream>(
               input_stream_.get(), dataset()->options_.input_buffer_size,
-              dataset()->options_.input_buffer_size, dataset()->options_));
-          buffered_input_stream_.reset(new io::BufferedInputStream(
+              dataset()->options_.input_buffer_size, dataset()->options_);
+          buffered_input_stream_ = absl::make_unique<io::BufferedInputStream>(
               zlib_input_stream_.get(), dataset()->options_.input_buffer_size,
-              false));
+              false);
         } else {
-          buffered_input_stream_.reset(new io::BufferedInputStream(
+          buffered_input_stream_ = absl::make_unique<io::BufferedInputStream>(
               input_stream_.get(), dataset()->options_.input_buffer_size,
-              false));
+              false);
         }
         return Status::OK();
       }
@@ -344,11 +344,12 @@ class FixedLengthRecordDatasetOp : public DatasetOpKernel {
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
       if (compression_type_.empty()) {
-        return std::unique_ptr<IteratorBase>(new UncompressedIterator(
-            {this, strings::StrCat(prefix, "::FixedLengthRecord")}));
+        return absl::make_unique<UncompressedIterator>(
+            UncompressedIterator::Params{
+                this, strings::StrCat(prefix, "::FixedLengthRecord")});
       } else {
-        return std::unique_ptr<IteratorBase>(new CompressedIterator(
-            {this, strings::StrCat(prefix, "::FixedLengthRecord")}));
+        return absl::make_unique<CompressedIterator>(CompressedIterator::Params{
+            this, strings::StrCat(prefix, "::FixedLengthRecord")});
       }
     }
 
@@ -452,8 +453,8 @@ class FixedLengthRecordDatasetOp : public DatasetOpKernel {
           }
           TF_RETURN_IF_ERROR(ctx->env()->NewRandomAccessFile(
               dataset()->filenames_[current_file_index_], &file_));
-          input_buffer_.reset(
-              new io::InputBuffer(file_.get(), dataset()->buffer_size_));
+          input_buffer_ = absl::make_unique<io::InputBuffer>(
+              file_.get(), dataset()->buffer_size_);
           TF_RETURN_IF_ERROR(
               input_buffer_->SkipNBytes(dataset()->header_bytes_));
         } while (true);
@@ -495,8 +496,8 @@ class FixedLengthRecordDatasetOp : public DatasetOpKernel {
           file_pos_limit_ = file_size - dataset()->footer_bytes_;
           TF_RETURN_IF_ERROR(ctx->env()->NewRandomAccessFile(
               dataset()->filenames_[current_file_index_], &file_));
-          input_buffer_.reset(
-              new io::InputBuffer(file_.get(), dataset()->buffer_size_));
+          input_buffer_ = absl::make_unique<io::InputBuffer>(
+              file_.get(), dataset()->buffer_size_);
           TF_RETURN_IF_ERROR(input_buffer_->Seek(current_pos));
         }
 
@@ -612,13 +613,14 @@ class FixedLengthRecordDatasetOp : public DatasetOpKernel {
                 dataset()->compression_type_ == "ZLIB"
                     ? io::ZlibCompressionOptions::DEFAULT()
                     : io::ZlibCompressionOptions::GZIP();
-            file_stream_.reset(new io::RandomAccessInputStream(file_.get()));
-            buffered_input_stream_.reset(new io::ZlibInputStream(
+            file_stream_ =
+                absl::make_unique<io::RandomAccessInputStream>(file_.get());
+            buffered_input_stream_ = absl::make_unique<io::ZlibInputStream>(
                 file_stream_.get(), dataset()->buffer_size_,
-                dataset()->buffer_size_, zlib_options));
+                dataset()->buffer_size_, zlib_options);
           } else {
-            buffered_input_stream_.reset(new io::BufferedInputStream(
-                file_.get(), dataset()->buffer_size_));
+            buffered_input_stream_ = absl::make_unique<io::BufferedInputStream>(
+                file_.get(), dataset()->buffer_size_);
           }
           TF_RETURN_IF_ERROR(
               buffered_input_stream_->SkipNBytes(dataset()->header_bytes_));
@@ -672,10 +674,11 @@ class FixedLengthRecordDatasetOp : public DatasetOpKernel {
               dataset()->compression_type_ == "ZLIB"
                   ? io::ZlibCompressionOptions::DEFAULT()
                   : io::ZlibCompressionOptions::GZIP();
-          file_stream_.reset(new io::RandomAccessInputStream(file_.get()));
-          buffered_input_stream_.reset(new io::ZlibInputStream(
+          file_stream_ =
+              absl::make_unique<io::RandomAccessInputStream>(file_.get());
+          buffered_input_stream_ = absl::make_unique<io::ZlibInputStream>(
               file_stream_.get(), dataset()->buffer_size_,
-              dataset()->buffer_size_, zlib_options));
+              dataset()->buffer_size_, zlib_options);
           lookahead_cache_.clear();
           TF_RETURN_IF_ERROR(buffered_input_stream_->SkipNBytes(
               current_pos - dataset()->footer_bytes_));
@@ -763,8 +766,8 @@ class TFRecordDatasetOp : public DatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::TFRecord")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::TFRecord")});
     }
 
     const DataTypeVector& output_dtypes() const override {
@@ -885,8 +888,8 @@ class TFRecordDatasetOp : public DatasetOpKernel {
         const string& next_filename =
             dataset()->filenames_[current_file_index_];
         TF_RETURN_IF_ERROR(env->NewRandomAccessFile(next_filename, &file_));
-        reader_.reset(
-            new io::SequentialRecordReader(file_.get(), dataset()->options_));
+        reader_ = absl::make_unique<io::SequentialRecordReader>(
+            file_.get(), dataset()->options_);
         return Status::OK();
       }
 
diff --git a/tensorflow/core/kernels/data/repeat_dataset_op.cc b/tensorflow/core/kernels/data/repeat_dataset_op.cc
index 8100f2695b..ef507ffdd1 100644
--- a/tensorflow/core/kernels/data/repeat_dataset_op.cc
+++ b/tensorflow/core/kernels/data/repeat_dataset_op.cc
@@ -51,14 +51,14 @@ class RepeatDatasetOp : public UnaryDatasetOpKernel {
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
       if (count_ < 0) {
-        return std::unique_ptr<IteratorBase>(new ForeverIterator(
-            {this, strings::StrCat(prefix, "::ForeverRepeat")}));
+        return absl::make_unique<ForeverIterator>(ForeverIterator::Params{
+            this, strings::StrCat(prefix, "::ForeverRepeat")});
       } else if (count_ == 0) {
-        return std::unique_ptr<IteratorBase>(new EmptyIterator(
-            {this, strings::StrCat(prefix, "::EmptyRepeat")}));
+        return absl::make_unique<EmptyIterator>(EmptyIterator::Params{
+            this, strings::StrCat(prefix, "::EmptyRepeat")});
       } else {
-        return std::unique_ptr<IteratorBase>(new FiniteIterator(
-            {this, strings::StrCat(prefix, "::FiniteRepeat")}));
+        return absl::make_unique<FiniteIterator>(FiniteIterator::Params{
+            this, strings::StrCat(prefix, "::FiniteRepeat")});
       }
     }
 
diff --git a/tensorflow/core/kernels/data/shuffle_dataset_op.cc b/tensorflow/core/kernels/data/shuffle_dataset_op.cc
index 4c380c1fa2..e0c435718a 100644
--- a/tensorflow/core/kernels/data/shuffle_dataset_op.cc
+++ b/tensorflow/core/kernels/data/shuffle_dataset_op.cc
@@ -23,7 +23,6 @@ limitations under the License.
 #include "tensorflow/core/lib/random/philox_random.h"
 #include "tensorflow/core/lib/random/random.h"
 #include "tensorflow/core/lib/random/random_distributions.h"
-#include "tensorflow/core/util/ptr_util.h"
 
 namespace tensorflow {
 namespace data {
@@ -80,8 +79,9 @@ class ShuffleDatasetOpBase : public UnaryDatasetOpKernel {
             num_elements_(0),
             parent_generator_(seed, seed2),
             generator_(&parent_generator_) {
-        buffer_.reset(new std::vector<Tensor>[params.dataset->buffer_size_]);
-        slices_.push_back(MakeUnique<Slice>(0, 0));
+        buffer_ = absl::make_unique<std::vector<Tensor>[]>(
+            params.dataset->buffer_size_);
+        slices_.push_back(absl::make_unique<Slice>(0, 0));
       }
 
       Status GetNextInternal(IteratorContext* ctx,
@@ -124,7 +124,7 @@ class ShuffleDatasetOpBase : public UnaryDatasetOpKernel {
             }
             epoch_++;
             int64 n = slices_.back()->end;
-            slices_.push_back(MakeUnique<Slice>(n, n));
+            slices_.push_back(absl::make_unique<Slice>(n, n));
             TF_RETURN_IF_ERROR(this->dataset()->input_->MakeIterator(
                 ctx, this->prefix(), &input_impl_));
           }
@@ -273,7 +273,8 @@ class ShuffleDatasetOpBase : public UnaryDatasetOpKernel {
               reader->ReadScalar(this->full_name("slices_size"), &temp));
           slices_size = static_cast<size_t>(temp);
         }
-        buffer_.reset(new std::vector<Tensor>[this->dataset()->buffer_size_]);
+        buffer_ = absl::make_unique<std::vector<Tensor>[]>(
+            this->dataset()->buffer_size_);
         for (size_t i = 0; i < slices_size; ++i) {
           int64 start;
           TF_RETURN_IF_ERROR(reader->ReadScalar(
@@ -281,7 +282,7 @@ class ShuffleDatasetOpBase : public UnaryDatasetOpKernel {
           int64 end;
           TF_RETURN_IF_ERROR(reader->ReadScalar(
               this->full_name(strings::StrCat("slices_end_", i)), &end));
-          slices_.push_back(MakeUnique<Slice>(start, end));
+          slices_.push_back(absl::make_unique<Slice>(start, end));
           for (size_t j = start; j < end; ++j) {
             size_t index = j % this->dataset()->buffer_size_;
             int64 list_size;
@@ -399,8 +400,9 @@ class ShuffleDatasetOp : public ShuffleDatasetOpBase {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(new Iterator(
-          {this, strings::StrCat(prefix, "::Shuffle")}, seed_, seed2_));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::Shuffle")}, seed_,
+          seed2_);
     }
 
    protected:
@@ -578,9 +580,11 @@ class ShuffleDatasetOp : public ShuffleDatasetOpBase {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new ShuffleDatasetBase::Iterator<ShuffleDatasetBase>(
-              {this, strings::StrCat(prefix, "::Shuffle")}, seed_, seed2_));
+      return absl::make_unique<
+          ShuffleDatasetBase::Iterator<ShuffleDatasetBase>>(
+          ShuffleDatasetBase::Iterator<ShuffleDatasetBase>::Params{
+              this, strings::StrCat(prefix, "::Shuffle")},
+          seed_, seed2_);
     }
 
    protected:
@@ -663,10 +667,11 @@ class ShuffleAndRepeatDatasetOp : public ShuffleDatasetOpBase {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new ShuffleDatasetBase::Iterator<ShuffleDatasetBase>(
-              {this, strings::StrCat(prefix, "::ShuffleAndRepeat")}, seed_,
-              seed2_));
+      return absl::make_unique<
+          ShuffleDatasetBase::Iterator<ShuffleDatasetBase>>(
+          ShuffleDatasetBase::Iterator<ShuffleDatasetBase>::Params{
+              this, strings::StrCat(prefix, "::ShuffleAndRepeat")},
+          seed_, seed2_);
     }
 
    protected:
diff --git a/tensorflow/core/kernels/data/single_threaded_executor.cc b/tensorflow/core/kernels/data/single_threaded_executor.cc
index 89e3881037..aab4bfe640 100644
--- a/tensorflow/core/kernels/data/single_threaded_executor.cc
+++ b/tensorflow/core/kernels/data/single_threaded_executor.cc
@@ -376,8 +376,8 @@ static SingleThreadedExecutorRegistrar registrar;
 Status NewSingleThreadedExecutor(const LocalExecutorParams& params,
                                  std::unique_ptr<const Graph> graph,
                                  Executor** executor) {
-  std::unique_ptr<SingleThreadedExecutorImpl> impl(
-      new SingleThreadedExecutorImpl(params));
+  std::unique_ptr<SingleThreadedExecutorImpl> impl =
+      absl::make_unique<SingleThreadedExecutorImpl>(params);
   TF_RETURN_IF_ERROR(impl->Initialize(*graph));
   *executor = impl.release();
   return Status::OK();
diff --git a/tensorflow/core/kernels/data/single_threaded_executor_test.cc b/tensorflow/core/kernels/data/single_threaded_executor_test.cc
index 7bb51fb8b5..df669e53d3 100644
--- a/tensorflow/core/kernels/data/single_threaded_executor_test.cc
+++ b/tensorflow/core/kernels/data/single_threaded_executor_test.cc
@@ -139,7 +139,7 @@ Rendezvous::ParsedKey Key(const string& sender, const uint64 incarnation,
 
 TEST_F(ExecutorTest, SimpleAdd) {
   // c = a + b
-  std::unique_ptr<Graph> g(new Graph(OpRegistry::Global()));
+  std::unique_ptr<Graph> g = absl::make_unique<Graph>(OpRegistry::Global());
   auto in0 = test::graph::Arg(g.get(), 0, DT_FLOAT);
   auto in1 = test::graph::Arg(g.get(), 0, DT_FLOAT);
   auto tmp = test::graph::Add(g.get(), in0, in1);
@@ -163,7 +163,7 @@ TEST_F(ExecutorTest, SelfAdd) {
   //
   // b <- v10
   // All nodes are executed by one thread.
-  std::unique_ptr<Graph> g(new Graph(OpRegistry::Global()));
+  std::unique_ptr<Graph> g = absl::make_unique<Graph>(OpRegistry::Global());
   auto v = test::graph::Arg(g.get(), 0, DT_FLOAT);
   const int N = 10;
   for (int i = 1; i <= N; ++i) {
@@ -219,7 +219,7 @@ void BuildTree(int N, Graph* g) {
 }
 
 TEST_F(ExecutorTest, RandomTree) {
-  std::unique_ptr<Graph> g(new Graph(OpRegistry::Global()));
+  std::unique_ptr<Graph> g = absl::make_unique<Graph>(OpRegistry::Global());
   BuildTree(4096, g.get());
   Create(std::move(g));
   FunctionCallFrame call_frame({DT_FLOAT}, {DT_FLOAT});
@@ -231,7 +231,7 @@ TEST_F(ExecutorTest, RandomTree) {
 }
 
 TEST_F(ExecutorTest, OpError) {
-  std::unique_ptr<Graph> g(new Graph(OpRegistry::Global()));
+  std::unique_ptr<Graph> g = absl::make_unique<Graph>(OpRegistry::Global());
   auto zero = test::graph::Constant(g.get(), V(0.0));
   auto inf = test::graph::Unary(g.get(), "Reciprocal", zero);
   auto check = test::graph::CheckNumerics(g.get(), inf, "message");
diff --git a/tensorflow/core/kernels/data/skip_dataset_op.cc b/tensorflow/core/kernels/data/skip_dataset_op.cc
index e321066a71..5b85a10edf 100644
--- a/tensorflow/core/kernels/data/skip_dataset_op.cc
+++ b/tensorflow/core/kernels/data/skip_dataset_op.cc
@@ -50,11 +50,11 @@ class SkipDatasetOp : public UnaryDatasetOpKernel {
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
       if (count_ < 0) {
-        return std::unique_ptr<IteratorBase>(
-            new EmptyIterator({this, strings::StrCat(prefix, "::EmptySkip")}));
+        return absl::make_unique<EmptyIterator>(EmptyIterator::Params{
+            this, strings::StrCat(prefix, "::EmptySkip")});
       } else {
-        return std::unique_ptr<IteratorBase>(new FiniteIterator(
-            {this, strings::StrCat(prefix, "::FiniteSkip")}));
+        return absl::make_unique<FiniteIterator>(FiniteIterator::Params{
+            this, strings::StrCat(prefix, "::FiniteSkip")});
       }
     }
 
diff --git a/tensorflow/core/kernels/data/sparse_tensor_slice_dataset_op.cc b/tensorflow/core/kernels/data/sparse_tensor_slice_dataset_op.cc
index be105f8170..d8d7cd204d 100644
--- a/tensorflow/core/kernels/data/sparse_tensor_slice_dataset_op.cc
+++ b/tensorflow/core/kernels/data/sparse_tensor_slice_dataset_op.cc
@@ -41,8 +41,8 @@ class Dataset : public DatasetBase {
 
   std::unique_ptr<IteratorBase> MakeIteratorInternal(
       const string& prefix) const override {
-    return std::unique_ptr<IteratorBase>(
-        new Iterator({this, strings::StrCat(prefix, "::SparseTensorSlice")}));
+    return absl::make_unique<Iterator>(typename Iterator::Params{
+        this, strings::StrCat(prefix, "::SparseTensorSlice")});
   }
 
   const DataTypeVector& output_dtypes() const override { return dtypes_; }
diff --git a/tensorflow/core/kernels/data/take_dataset_op.cc b/tensorflow/core/kernels/data/take_dataset_op.cc
index 0a3d586953..0dd0c0c80d 100644
--- a/tensorflow/core/kernels/data/take_dataset_op.cc
+++ b/tensorflow/core/kernels/data/take_dataset_op.cc
@@ -50,11 +50,11 @@ class TakeDatasetOp : public UnaryDatasetOpKernel {
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
       if (count_ == 0) {
-        return std::unique_ptr<IteratorBase>(
-            new EmptyIterator({this, strings::StrCat(prefix, "::EmptyTake")}));
+        return absl::make_unique<EmptyIterator>(EmptyIterator::Params{
+            this, strings::StrCat(prefix, "::EmptyTake")});
       } else {
-        return std::unique_ptr<IteratorBase>(new FiniteIterator(
-            {this, strings::StrCat(prefix, "::FiniteTake")}));
+        return absl::make_unique<FiniteIterator>(FiniteIterator::Params{
+            this, strings::StrCat(prefix, "::FiniteTake")});
       }
     }
 
diff --git a/tensorflow/core/kernels/data/tensor_dataset_op.cc b/tensorflow/core/kernels/data/tensor_dataset_op.cc
index 98c23f23b2..a44dbd0d4d 100644
--- a/tensorflow/core/kernels/data/tensor_dataset_op.cc
+++ b/tensorflow/core/kernels/data/tensor_dataset_op.cc
@@ -50,8 +50,8 @@ class TensorDatasetOp : public DatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::FromTensor")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::FromTensor")});
     }
 
     const DataTypeVector& output_dtypes() const override { return dtypes_; }
diff --git a/tensorflow/core/kernels/data/tensor_slice_dataset_op.cc b/tensorflow/core/kernels/data/tensor_slice_dataset_op.cc
index 4ba2bde718..54dcd7eb7d 100644
--- a/tensorflow/core/kernels/data/tensor_slice_dataset_op.cc
+++ b/tensorflow/core/kernels/data/tensor_slice_dataset_op.cc
@@ -71,8 +71,8 @@ class TensorSliceDatasetOp : public DatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::TensorSlice")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::TensorSlice")});
     }
 
     const DataTypeVector& output_dtypes() const override { return dtypes_; }
diff --git a/tensorflow/core/kernels/data/window_dataset.cc b/tensorflow/core/kernels/data/window_dataset.cc
index c295631550..dc27702f1e 100644
--- a/tensorflow/core/kernels/data/window_dataset.cc
+++ b/tensorflow/core/kernels/data/window_dataset.cc
@@ -31,8 +31,8 @@ class WindowDataset : public DatasetBase {
 
   std::unique_ptr<IteratorBase> MakeIteratorInternal(
       const string& prefix) const override {
-    return std::unique_ptr<IteratorBase>(
-        new Iterator({this, strings::StrCat(prefix, "::Window")}));
+    return absl::make_unique<Iterator>(
+        Iterator::Params{this, strings::StrCat(prefix, "::Window")});
   }
 
   const DataTypeVector& output_dtypes() const override { return output_types_; }
diff --git a/tensorflow/core/kernels/data/window_dataset_op.cc b/tensorflow/core/kernels/data/window_dataset_op.cc
index ae13ae5da8..0b24c11891 100644
--- a/tensorflow/core/kernels/data/window_dataset_op.cc
+++ b/tensorflow/core/kernels/data/window_dataset_op.cc
@@ -78,8 +78,8 @@ class WindowDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(new Iterator(
-          Iterator::Params{this, strings::StrCat(prefix, "::Window")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::Window")});
     }
 
     const DataTypeVector& output_dtypes() const override {
diff --git a/tensorflow/core/kernels/data/zip_dataset_op.cc b/tensorflow/core/kernels/data/zip_dataset_op.cc
index 1760e63a9e..cdc2969fc2 100644
--- a/tensorflow/core/kernels/data/zip_dataset_op.cc
+++ b/tensorflow/core/kernels/data/zip_dataset_op.cc
@@ -62,8 +62,8 @@ class ZipDatasetOp : public DatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::Zip")}));
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::Zip")});
     }
 
     const DataTypeVector& output_dtypes() const override {
-- 
GitLab


From 5704143a6790225f7f08533fa0225136095688b1 Mon Sep 17 00:00:00 2001
From: Nick Felt <nickfelt@google.com>
Date: Fri, 11 Jan 2019 16:03:25 -0800
Subject: [PATCH 0629/1095] Check syntax when parsing ApiDef textprotos

This change ensures that syntax errors in ApiDef textprotos (e.g. mistyping a field name, which I did) fail the overall API generation rule instead of having parsing fail silently and produce an empty result for that single file.

PiperOrigin-RevId: 228961785
---
 tensorflow/core/BUILD                         |   1 +
 tensorflow/core/framework/op_gen_lib.cc       |  12 +-
 tensorflow/core/framework/op_gen_lib_test.cc  |  15 ++-
 tensorflow/core/platform/default/protobuf.h   |   1 +
 tensorflow/core/util/proto/BUILD              |  15 +++
 tensorflow/core/util/proto/proto_utils.cc     |  51 +++++++-
 tensorflow/core/util/proto/proto_utils.h      |  31 +++++
 .../core/util/proto/proto_utils_test.cc       | 112 ++++++++++++++++++
 tensorflow/js/ops/ts_op_gen_test.cc           |  15 +--
 9 files changed, 237 insertions(+), 16 deletions(-)
 create mode 100644 tensorflow/core/util/proto/proto_utils_test.cc

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index ee9ceeae89..b655fdcbff 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -1017,6 +1017,7 @@ cc_library(
         ":lib",
         ":lib_internal",
         ":protos_all_cc",
+        "//tensorflow/core/util/proto:proto_utils",
     ],
 )
 
diff --git a/tensorflow/core/framework/op_gen_lib.cc b/tensorflow/core/framework/op_gen_lib.cc
index 505ab54775..92a7038a40 100644
--- a/tensorflow/core/framework/op_gen_lib.cc
+++ b/tensorflow/core/framework/op_gen_lib.cc
@@ -23,6 +23,7 @@ limitations under the License.
 #include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/protobuf.h"
+#include "tensorflow/core/util/proto/proto_utils.h"
 
 namespace tensorflow {
 
@@ -488,14 +489,21 @@ Status ApiDefMap::LoadFile(Env* env, const string& filename) {
   if (filename.empty()) return Status::OK();
   string contents;
   TF_RETURN_IF_ERROR(ReadFileToString(env, filename, &contents));
-  TF_RETURN_IF_ERROR(LoadApiDef(contents));
+  Status status = LoadApiDef(contents);
+  if (!status.ok()) {
+    // Return failed status annotated with filename to aid in debugging.
+    return Status(status.code(),
+                  strings::StrCat("Error parsing ApiDef file ", filename, ": ",
+                                  status.error_message()));
+  }
   return Status::OK();
 }
 
 Status ApiDefMap::LoadApiDef(const string& api_def_file_contents) {
   const string contents = PBTxtFromMultiline(api_def_file_contents);
   ApiDefs api_defs;
-  protobuf::TextFormat::ParseFromString(contents, &api_defs);
+  TF_RETURN_IF_ERROR(
+      proto_utils::ParseTextFormatFromString(contents, &api_defs));
   for (const auto& api_def : api_defs.op()) {
     // Check if the op definition is loaded. If op definition is not
     // loaded, then we just skip this ApiDef.
diff --git a/tensorflow/core/framework/op_gen_lib_test.cc b/tensorflow/core/framework/op_gen_lib_test.cc
index e0e77c7449..6b43d7dc68 100644
--- a/tensorflow/core/framework/op_gen_lib_test.cc
+++ b/tensorflow/core/framework/op_gen_lib_test.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #include "tensorflow/core/framework/op_gen_lib.h"
 
 #include "tensorflow/core/framework/op_def.pb.h"
+#include "tensorflow/core/lib/core/error_codes.pb.h"
 #include "tensorflow/core/platform/test.h"
 
 namespace tensorflow {
@@ -39,7 +40,7 @@ constexpr char kTestOpList[] = R"(op {
     version: 123
     explanation: "foo"
   }
-)";
+})";
 
 constexpr char kTestApiDef[] = R"(op {
   graph_op_name: "testop"
@@ -455,6 +456,18 @@ op {
   ASSERT_EQ(tensorflow::error::FAILED_PRECONDITION, status.code());
 }
 
+TEST(OpGenLibTest, ApiDefInvalidSyntax) {
+  const string api_def = R"pb(
+    op { bad_op_name: "testop" }
+  )pb";
+
+  OpList op_list;
+  ApiDefMap api_map(op_list);
+  // Loading with invalid syntax (e.g. unrecognized field name) should fail.
+  auto status = api_map.LoadApiDef(api_def);
+  ASSERT_EQ(tensorflow::error::INVALID_ARGUMENT, status.code());
+}
+
 TEST(OpGenLibTest, ApiDefUpdateDocs) {
   const string op_list1 = R"(op {
   name: "testop"
diff --git a/tensorflow/core/platform/default/protobuf.h b/tensorflow/core/platform/default/protobuf.h
index 2708d6ebda..aeef2d9b88 100644
--- a/tensorflow/core/platform/default/protobuf.h
+++ b/tensorflow/core/platform/default/protobuf.h
@@ -23,6 +23,7 @@ limitations under the License.
 #include "google/protobuf/descriptor.h"
 #include "google/protobuf/descriptor.pb.h"
 #include "google/protobuf/dynamic_message.h"
+#include "google/protobuf/io/tokenizer.h"
 #include "google/protobuf/text_format.h"
 #include "google/protobuf/util/json_util.h"
 #include "google/protobuf/util/type_resolver_util.h"
diff --git a/tensorflow/core/util/proto/BUILD b/tensorflow/core/util/proto/BUILD
index 7e549c7764..b990f0a749 100644
--- a/tensorflow/core/util/proto/BUILD
+++ b/tensorflow/core/util/proto/BUILD
@@ -68,5 +68,20 @@ cc_library(
     deps = [
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
+        "//tensorflow/core:platform_base",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+tf_cc_test(
+    name = "proto_utils_test",
+    srcs = ["proto_utils_test.cc"],
+    deps = [
+        ":proto_utils",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
+        "@com_google_googletest//:gtest_main",
     ],
 )
diff --git a/tensorflow/core/util/proto/proto_utils.cc b/tensorflow/core/util/proto/proto_utils.cc
index 201f05a129..f106414139 100644
--- a/tensorflow/core/util/proto/proto_utils.cc
+++ b/tensorflow/core/util/proto/proto_utils.cc
@@ -13,11 +13,14 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#include "tensorflow/core/util/proto/proto_utils.h"
+
+#include "absl/strings/string_view.h"
+#include "absl/strings/substitute.h"
 #include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/protobuf.h"
 
-#include "tensorflow/core/util/proto/proto_utils.h"
-
 namespace tensorflow {
 namespace proto_utils {
 
@@ -66,5 +69,49 @@ bool IsCompatibleType(FieldDescriptor::Type field_type, DataType dtype) {
   }
 }
 
+Status ParseTextFormatFromString(absl::string_view input,
+                                 protobuf::Message* output) {
+  DCHECK(output != nullptr) << "output must be non NULL";
+  // When checks are disabled, instead log the error and return an error status.
+  if (output == nullptr) {
+    LOG(ERROR) << "output must be non NULL";
+    return Status(error::INVALID_ARGUMENT, "output must be non NULL");
+  }
+  string err;
+  StringErrorCollector err_collector(&err, /*one-indexing=*/true);
+  protobuf::TextFormat::Parser parser;
+  parser.RecordErrorsTo(&err_collector);
+  if (!parser.ParseFromString(string(input), output)) {
+    return Status(error::INVALID_ARGUMENT, err);
+  }
+  return Status::OK();
+}
+
+StringErrorCollector::StringErrorCollector(string* error_text)
+    : StringErrorCollector(error_text, false) {}
+
+StringErrorCollector::StringErrorCollector(string* error_text,
+                                           bool one_indexing)
+    : error_text_(error_text), index_offset_(one_indexing ? 1 : 0) {
+  DCHECK(error_text_ != nullptr) << "error_text must be non NULL";
+  // When checks are disabled, just log and then ignore added errors/warnings.
+  if (error_text_ == nullptr) {
+    LOG(ERROR) << "error_text must be non NULL";
+  }
+}
+
+void StringErrorCollector::AddError(int line, int column,
+                                    const string& message) {
+  if (error_text_ != nullptr) {
+    absl::SubstituteAndAppend(error_text_, "$0($1): $2\n", line + index_offset_,
+                              column + index_offset_, message);
+  }
+}
+
+void StringErrorCollector::AddWarning(int line, int column,
+                                      const string& message) {
+  AddError(line, column, message);
+}
+
 }  // namespace proto_utils
 }  // namespace tensorflow
diff --git a/tensorflow/core/util/proto/proto_utils.h b/tensorflow/core/util/proto/proto_utils.h
index d5e0b9006c..9451e317a1 100644
--- a/tensorflow/core/util/proto/proto_utils.h
+++ b/tensorflow/core/util/proto/proto_utils.h
@@ -16,7 +16,9 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_UTIL_PROTO_PROTO_UTILS_H_
 #define TENSORFLOW_CORE_UTIL_PROTO_PROTO_UTILS_H_
 
+#include "absl/strings/string_view.h"
 #include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/protobuf.h"
 
 namespace tensorflow {
@@ -27,6 +29,35 @@ using tensorflow::protobuf::FieldDescriptor;
 // Returns true if the proto field type can be converted to the tensor dtype.
 bool IsCompatibleType(FieldDescriptor::Type field_type, DataType dtype);
 
+// Parses a text-formatted protobuf from a string into the given Message* output
+// and returns status OK if valid, or INVALID_ARGUMENT with an accompanying
+// parser error message if the text format is invalid.
+Status ParseTextFormatFromString(absl::string_view input,
+                                 protobuf::Message* output);
+
+class StringErrorCollector : public protobuf::io::ErrorCollector {
+ public:
+  // String error_text is unowned and must remain valid during the use of
+  // StringErrorCollector.
+  explicit StringErrorCollector(string* error_text);
+  // If one_indexing is set to true, all line and column numbers will be
+  // increased by one for cases when provided indices are 0-indexed and
+  // 1-indexed error messages are desired
+  StringErrorCollector(string* error_text, bool one_indexing);
+  StringErrorCollector(const StringErrorCollector&) = delete;
+  StringErrorCollector& operator=(const StringErrorCollector&) = delete;
+
+  // Implementation of protobuf::io::ErrorCollector::AddError.
+  void AddError(int line, int column, const string& message) override;
+
+  // Implementation of protobuf::io::ErrorCollector::AddWarning.
+  void AddWarning(int line, int column, const string& message) override;
+
+ private:
+  string* const error_text_;
+  const int index_offset_;
+};
+
 }  // namespace proto_utils
 }  // namespace tensorflow
 
diff --git a/tensorflow/core/util/proto/proto_utils_test.cc b/tensorflow/core/util/proto/proto_utils_test.cc
new file mode 100644
index 0000000000..7f6312a718
--- /dev/null
+++ b/tensorflow/core/util/proto/proto_utils_test.cc
@@ -0,0 +1,112 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/util/proto/proto_utils.h"
+
+#include <gmock/gmock.h>
+#include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/platform/protobuf.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace tensorflow {
+
+using proto_utils::ParseTextFormatFromString;
+using proto_utils::StringErrorCollector;
+using ::testing::ContainsRegex;
+
+TEST(ParseTextFormatFromStringTest, Success) {
+  protobuf::DescriptorProto output;
+  TF_ASSERT_OK(ParseTextFormatFromString("name: \"foo\"", &output));
+  EXPECT_EQ(output.name(), "foo");
+}
+
+TEST(ParseTextFormatFromStringTest, ErrorOnInvalidSyntax) {
+  protobuf::DescriptorProto output;
+  Status status = ParseTextFormatFromString("name: foo", &output);
+  EXPECT_EQ(status.code(), error::INVALID_ARGUMENT);
+  EXPECT_THAT(status.error_message(), ContainsRegex("foo"));
+  EXPECT_FALSE(output.has_name());
+}
+
+TEST(ParseTextFormatFromStringTest, ErrorOnUnknownFieldName) {
+  protobuf::DescriptorProto output;
+  Status status = ParseTextFormatFromString("badname: \"foo\"", &output);
+  EXPECT_EQ(status.code(), error::INVALID_ARGUMENT);
+  EXPECT_THAT(status.error_message(), ContainsRegex("badname"));
+  EXPECT_FALSE(output.has_name());
+}
+
+TEST(ParseTextFormatFromStringTest, DiesOnNullOutputPointer) {
+#ifndef NDEBUG
+  ASSERT_DEATH(ParseTextFormatFromString("foo", nullptr).IgnoreError(),
+               "output.*non NULL");
+#else
+  // Under NDEBUG we don't die but should still return an error status.
+  Status status = ParseTextFormatFromString("foo", nullptr);
+  EXPECT_EQ(status.code(), error::INVALID_ARGUMENT);
+  EXPECT_THAT(status.error_message(), ContainsRegex("output.*non NULL"));
+#endif
+}
+
+TEST(StringErrorCollectorTest, AppendsError) {
+  string err;
+  StringErrorCollector collector(&err);
+  collector.AddError(1, 2, "foo");
+  EXPECT_EQ("1(2): foo\n", err);
+}
+
+TEST(StringErrorCollectorTest, AppendsWarning) {
+  string err;
+  StringErrorCollector collector(&err);
+  collector.AddWarning(1, 2, "foo");
+  EXPECT_EQ("1(2): foo\n", err);
+}
+
+TEST(StringErrorCollectorTest, AppendsMultipleError) {
+  string err;
+  StringErrorCollector collector(&err);
+  collector.AddError(1, 2, "foo");
+  collector.AddError(3, 4, "bar");
+  EXPECT_EQ("1(2): foo\n3(4): bar\n", err);
+}
+
+TEST(StringErrorCollectorTest, AppendsMultipleWarning) {
+  string err;
+  StringErrorCollector collector(&err);
+  collector.AddWarning(1, 2, "foo");
+  collector.AddWarning(3, 4, "bar");
+  EXPECT_EQ("1(2): foo\n3(4): bar\n", err);
+}
+
+TEST(StringErrorCollectorTest, OffsetWorks) {
+  string err;
+  StringErrorCollector collector(&err, true);
+  collector.AddError(1, 2, "foo");
+  collector.AddWarning(3, 4, "bar");
+  EXPECT_EQ("2(3): foo\n4(5): bar\n", err);
+}
+
+TEST(StringErrorCollectorTest, DiesOnNullErrorText) {
+#ifndef NDEBUG
+  ASSERT_DEATH(StringErrorCollector(nullptr), "error_text.*non NULL");
+#else
+  // Under NDEBUG we don't die and instead AddError/AddWarning just do nothing.
+  StringErrorCollector collector(nullptr);
+  collector.AddError(1, 2, "foo");
+  collector.AddWarning(3, 4, "bar");
+#endif
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/js/ops/ts_op_gen_test.cc b/tensorflow/js/ops/ts_op_gen_test.cc
index 03241689b5..1c51dd030f 100644
--- a/tensorflow/js/ops/ts_op_gen_test.cc
+++ b/tensorflow/js/ops/ts_op_gen_test.cc
@@ -112,22 +112,15 @@ import {createTensorsTypeOpAttr, nodeBackend} from './op_utils';
 }
 
 TEST(TsOpGenTest, InputSingleAndList) {
-  const string api_def = R"(
-op {
-  name: "Foo"
-  input_arg {
-    name: "images"
-    type_attr: "T"
-    number_attr: "N"
-  }
-}
-)";
+  const string api_def = R"pb(
+    op { graph_op_name: "Foo" arg_order: "dim" arg_order: "images" }
+  )pb";
 
   string ts_file_text;
   GenerateTsOpFileText("", api_def, &ts_file_text);
 
   const string expected = R"(
-export function Foo(images: tfc.Tensor[], dim: tfc.Tensor): tfc.Tensor {
+export function Foo(dim: tfc.Tensor, images: tfc.Tensor[]): tfc.Tensor {
 )";
   ExpectContainsStr(ts_file_text, expected);
 }
-- 
GitLab


From 5c89fd3016286e8c3f71b29174a7d6efde90d611 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 11 Jan 2019 16:07:23 -0800
Subject: [PATCH 0630/1095] Automated rollback of commit
 196715b526a695678891ec11d13b7290b3f6861a

PiperOrigin-RevId: 228962444
---
 tensorflow/core/kernels/BUILD                  | 1 -
 tensorflow/core/kernels/fused_batch_norm_op.cc | 1 -
 2 files changed, 2 deletions(-)

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 4db0505a8f..0792b398f1 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -3820,7 +3820,6 @@ NN_DEPS = [
     ":conv_2d",
     ":eigen_contraction_kernel",
     ":ops_util",
-    ":pooling_ops",
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
     "//tensorflow/core:lib_internal",
diff --git a/tensorflow/core/kernels/fused_batch_norm_op.cc b/tensorflow/core/kernels/fused_batch_norm_op.cc
index dbd3bb05db..48b339508b 100644
--- a/tensorflow/core/kernels/fused_batch_norm_op.cc
+++ b/tensorflow/core/kernels/fused_batch_norm_op.cc
@@ -18,7 +18,6 @@ limitations under the License.
 #if GOOGLE_CUDA
 #define EIGEN_USE_GPU
 #include "tensorflow/core/kernels/conv_2d.h"
-#include "tensorflow/core/kernels/conv_ops_gpu.h"
 #include "tensorflow/core/util/stream_executor_util.h"
 #endif
 
-- 
GitLab


From 2f2f865d9857f8d2c84d7a2554e41563947ac6cc Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 11 Jan 2019 16:10:53 -0800
Subject: [PATCH 0631/1095] Test to show errors in conv2d_transpose

PiperOrigin-RevId: 228962978
---
 tensorflow/lite/build_def.bzl                 |  1 +
 tensorflow/lite/testing/generate_examples.py  | 42 +++++++++++++++++++
 .../testing/generated_examples_zip_test.cc    |  2 +
 3 files changed, 45 insertions(+)

diff --git a/tensorflow/lite/build_def.bzl b/tensorflow/lite/build_def.bzl
index 343ec60f2a..03b217c120 100644
--- a/tensorflow/lite/build_def.bzl
+++ b/tensorflow/lite/build_def.bzl
@@ -232,6 +232,7 @@ def generated_test_models():
         "constant",
         "control_dep",
         "conv",
+        "conv2d_transpose",
         "conv_with_shared_weights",
         "conv_to_depthwiseconv_with_shared_weights",
         "depthwiseconv",
diff --git a/tensorflow/lite/testing/generate_examples.py b/tensorflow/lite/testing/generate_examples.py
index 87e7d7eb02..d2c275f787 100644
--- a/tensorflow/lite/testing/generate_examples.py
+++ b/tensorflow/lite/testing/generate_examples.py
@@ -3222,6 +3222,48 @@ def make_slice_tests(zip_path):
   make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
 
 
+def make_conv2d_transpose_tests(zip_path):
+  """Make a set of tests to do transpose_conv."""
+
+  test_parameters = [{
+      "input_shape": [[1, 50, 54, 3]],
+      "filter_shape": [[1, 1, 8, 3], [1, 2, 8, 3], [1, 3, 8, 3], [1, 4, 8, 3]],
+      "output_shape": [[1, 100, 108, 8]],
+  }]
+
+  def build_graph(parameters):
+    """Build a transpose_conv graph given `parameters`."""
+    input_tensor = tf.placeholder(
+        dtype=tf.float32, name="input", shape=parameters["input_shape"])
+
+    filter_tensor = tf.placeholder(
+        dtype=tf.float32, name="filter", shape=parameters["filter_shape"])
+
+    out = tf.nn.conv2d_transpose(
+        input_tensor,
+        filter_tensor,
+        output_shape=parameters["output_shape"],
+        padding="SAME",
+        strides=(1, 2, 2, 1))
+
+    input_tensors = [input_tensor, filter_tensor]
+    return input_tensors, [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    values = [
+        create_tensor_data(np.float32, parameters["input_shape"]),
+        create_tensor_data(np.float32, parameters["filter_shape"])
+    ]
+    return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
+
+  make_zip_of_tests(
+      zip_path,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      expected_tf_success=4)
+
+
 # Since compute output_shape is fairly complicated for
 # tf.nn.conv2d_backprop_input input_sizes argument, so we here first perform a
 # "conv2d" operation to get the output, then we use the output to feed in
diff --git a/tensorflow/lite/testing/generated_examples_zip_test.cc b/tensorflow/lite/testing/generated_examples_zip_test.cc
index a9a31ad088..bade6590a4 100644
--- a/tensorflow/lite/testing/generated_examples_zip_test.cc
+++ b/tensorflow/lite/testing/generated_examples_zip_test.cc
@@ -105,6 +105,8 @@ std::map<string, string> kBrokenTests = {
 
     // Strided Slice chooses the wrong dimension.
     {R"(^\/strided_slice_buggy)", "119786029"},
+
+    {R"(^\/conv2d_transpose_filter_shape=\[1,4,8,3\])", "122735908"},
 };
 
 // Allows test data to be unarchived into a temporary directory and makes
-- 
GitLab


From b523734dd86bb943c490a5c25c358014922a797e Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Fri, 11 Jan 2019 16:18:21 -0800
Subject: [PATCH 0632/1095] [XLA] Don't re-seed RNG on every call to
 HloEvaluator::Evaluate.

Some tests -- in particular, tests that do reductions and so invoke a small
subcomputation many times -- invoke HloEvaluator::Evaluate() many times.
Previously, each such invocation was creating an std::random_device.

Creating std::random_device is very expensive -- it touches /dev/urandom and so
on.  This was creating a lot of overhead (25%+) in these tests.  So instead, we
do it only once, and do a simple atomic op to generate new seeds.

PiperOrigin-RevId: 228963974
---
 tensorflow/compiler/xla/service/hlo_evaluator.cc | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc
index 927e751d38..bcfa96d17f 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc
@@ -258,14 +258,18 @@ StatusOr<Literal> HloEvaluator::Evaluate(
   for (const auto& literal_ptr : arg_literals) {
     arg_literals_.push_back(&*literal_ptr);
   }
+
+  // Re-seed RNG, either from the configuration's seed or a monotonic
+  // per-evaluator seed (which prevents two evaluators from returning the same
+  // random sequence).
   if (computation.parent()->config().seed()) {
     seed_ = computation.parent()->config().seed();
   } else {
-    std::random_device rd;
-    seed_ = rd();
+    // Start global_seed at a (true) random value.
+    static std::atomic<uint64> global_seed{std::random_device()()};
+    seed_ = global_seed.fetch_add(1);
   }
-
-  engine_ = std::minstd_rand0(seed_);
+  engine_.seed(seed_);
 
   TF_RETURN_IF_ERROR(computation.Accept(this));
   return GetEvaluatedLiteralFor(computation.root_instruction()).Clone();
-- 
GitLab


From 3bf87a030a7d05a347b541047dc9308edaafb7d7 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 11 Jan 2019 16:46:46 -0800
Subject: [PATCH 0633/1095] move eager profiler to core/profiler/lib.

PiperOrigin-RevId: 228967775
---
 tensorflow/c/eager/BUILD                      |  4 +-
 tensorflow/c/eager/c_api_internal.h           |  2 +-
 tensorflow/core/common_runtime/eager/BUILD    | 29 -------------
 tensorflow/core/profiler/lib/BUILD            | 42 +++++++++++++++++++
 .../lib/eager_profiler.cc}                    |  2 +-
 .../lib/eager_profiler.h}                     |  0
 6 files changed, 46 insertions(+), 33 deletions(-)
 create mode 100644 tensorflow/core/profiler/lib/BUILD
 rename tensorflow/core/{common_runtime/eager/profiler.cc => profiler/lib/eager_profiler.cc} (98%)
 rename tensorflow/core/{common_runtime/eager/profiler.h => profiler/lib/eager_profiler.h} (100%)

diff --git a/tensorflow/c/eager/BUILD b/tensorflow/c/eager/BUILD
index 710850b5f2..51665fb9db 100644
--- a/tensorflow/c/eager/BUILD
+++ b/tensorflow/c/eager/BUILD
@@ -37,7 +37,6 @@ tf_cuda_library(
             "//tensorflow/c:c_api_internal",
             "//tensorflow/core:core_cpu",
             "//tensorflow/core/common_runtime/eager:attr_builder",
-            "//tensorflow/core/common_runtime/eager:profiler",
             "//tensorflow/core/common_runtime/eager:context",
             "//tensorflow/core/common_runtime/eager:eager_executor",
             "//tensorflow/core/common_runtime/eager:execute",
@@ -71,6 +70,7 @@ tf_cuda_library(
         "//tensorflow/core/distributed_runtime:remote_device",
         "//tensorflow/core/distributed_runtime:server_lib",
         "//tensorflow/core/distributed_runtime:worker_env",
+        "//tensorflow/core/profiler/lib:eager_profiler",
         "//tensorflow/core:gpu_runtime",
     ],
 )
@@ -98,7 +98,6 @@ tf_cuda_library(
         "//tensorflow/core/common_runtime/eager:eager_executor",
         "//tensorflow/core/common_runtime/eager:eager_operation",
         "//tensorflow/core/common_runtime/eager:kernel_and_device",
-        "//tensorflow/core/common_runtime/eager:profiler",
         "//tensorflow/core/common_runtime/eager:tensor_handle",
         "//tensorflow/core/distributed_runtime:remote_device",
         "//tensorflow/core/distributed_runtime:server_lib",
@@ -111,6 +110,7 @@ tf_cuda_library(
         "//tensorflow/core/distributed_runtime/rpc:grpc_worker_service",
         "//tensorflow/core/distributed_runtime/rpc:rpc_rendezvous_mgr",
         "//tensorflow/core/distributed_runtime/rpc/eager:grpc_eager_client",
+        "//tensorflow/core/profiler/lib:eager_profiler",
     ],
 )
 
diff --git a/tensorflow/c/eager/c_api_internal.h b/tensorflow/c/eager/c_api_internal.h
index 7b1035f631..d330aa4290 100644
--- a/tensorflow/c/eager/c_api_internal.h
+++ b/tensorflow/c/eager/c_api_internal.h
@@ -34,7 +34,6 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/eager/eager_executor.h"
 #include "tensorflow/core/common_runtime/eager/eager_operation.h"
 #include "tensorflow/core/common_runtime/eager/kernel_and_device.h"
-#include "tensorflow/core/common_runtime/eager/profiler.h"
 #include "tensorflow/core/common_runtime/eager/tensor_handle.h"
 #include "tensorflow/core/common_runtime/function.h"
 #include "tensorflow/core/common_runtime/rendezvous_mgr.h"
@@ -53,6 +52,7 @@ limitations under the License.
 #include "tensorflow/core/lib/gtl/stl_util.h"
 #include "tensorflow/core/platform/mutex.h"
 #include "tensorflow/core/platform/thread_annotations.h"
+#include "tensorflow/core/profiler/lib/eager_profiler.h"
 #include "tensorflow/core/public/version.h"
 
 struct TFE_ContextOptions {
diff --git a/tensorflow/core/common_runtime/eager/BUILD b/tensorflow/core/common_runtime/eager/BUILD
index 2697aea783..77e3246df0 100644
--- a/tensorflow/core/common_runtime/eager/BUILD
+++ b/tensorflow/core/common_runtime/eager/BUILD
@@ -88,35 +88,6 @@ tf_cuda_library(
     ],
 )
 
-tf_cuda_library(
-    name = "profiler",
-    srcs = [
-        "profiler.cc",
-    ],
-    hdrs = [
-        "profiler.h",
-    ],
-    visibility = ["//tensorflow:internal"],
-    deps = [
-        ":context",
-        "//tensorflow/cc/profiler",
-    ] + select({
-        "//tensorflow:android": [
-            "//tensorflow/core:android_tensorflow_lib_lite",
-        ],
-        "//conditions:default": [
-            "//tensorflow/core:core_cpu_lib",
-            "//tensorflow/core:framework",
-            "//tensorflow/core:framework_internal",
-            "//tensorflow/core:lib",
-            "//tensorflow/core:lib_internal",
-            "//tensorflow/core:protos_all_cc",
-            "//tensorflow/core:session_options",
-            "//tensorflow/core:device_tracer",
-        ],
-    }),
-)
-
 tf_cuda_library(
     name = "tensor_handle",
     srcs = [
diff --git a/tensorflow/core/profiler/lib/BUILD b/tensorflow/core/profiler/lib/BUILD
new file mode 100644
index 0000000000..3d46606cbd
--- /dev/null
+++ b/tensorflow/core/profiler/lib/BUILD
@@ -0,0 +1,42 @@
+package(
+    default_visibility = [
+        "//tensorflow:internal",
+        "//tensorflow_models:__subpackages__",
+    ],
+)
+
+licenses(["notice"])  # Apache 2.0
+
+load(
+    "//tensorflow:tensorflow.bzl",
+    "tf_cuda_library",
+)
+
+tf_cuda_library(
+    name = "eager_profiler",
+    srcs = [
+        "eager_profiler.cc",
+    ],
+    hdrs = [
+        "eager_profiler.h",
+    ],
+    visibility = ["//tensorflow:internal"],
+    deps = [
+        "//tensorflow/core/common_runtime/eager:context",
+        "//tensorflow/cc/profiler",
+    ] + select({
+        "//tensorflow:android": [
+            "//tensorflow/core:android_tensorflow_lib_lite",
+        ],
+        "//conditions:default": [
+            "//tensorflow/core:core_cpu_lib",
+            "//tensorflow/core:framework",
+            "//tensorflow/core:framework_internal",
+            "//tensorflow/core:lib",
+            "//tensorflow/core:lib_internal",
+            "//tensorflow/core:protos_all_cc",
+            "//tensorflow/core:session_options",
+            "//tensorflow/core:device_tracer",
+        ],
+    }),
+)
diff --git a/tensorflow/core/common_runtime/eager/profiler.cc b/tensorflow/core/profiler/lib/eager_profiler.cc
similarity index 98%
rename from tensorflow/core/common_runtime/eager/profiler.cc
rename to tensorflow/core/profiler/lib/eager_profiler.cc
index 46449d0771..7702febbc3 100644
--- a/tensorflow/core/common_runtime/eager/profiler.cc
+++ b/tensorflow/core/profiler/lib/eager_profiler.cc
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/core/common_runtime/eager/profiler.h"
+#include "tensorflow/core/profiler/lib/eager_profiler.h"
 #include "tensorflow/cc/profiler/profiler.h"
 #include "tensorflow/core/common_runtime/eager/context.h"
 #include "tensorflow/core/common_runtime/step_stats_collector.h"
diff --git a/tensorflow/core/common_runtime/eager/profiler.h b/tensorflow/core/profiler/lib/eager_profiler.h
similarity index 100%
rename from tensorflow/core/common_runtime/eager/profiler.h
rename to tensorflow/core/profiler/lib/eager_profiler.h
-- 
GitLab


From 5846ad65a0cd37586f248fea2fb2c2015bb1221a Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Fri, 11 Jan 2019 17:24:57 -0800
Subject: [PATCH 0634/1095] Add outfeed support to replay_computation.

Without this, computations that outfeed will hang.

PiperOrigin-RevId: 228972323
---
 .../compiler/xla/service/gpu/xfeed_queue.h    |  13 ++
 tensorflow/compiler/xla/tools/BUILD           |   1 +
 .../compiler/xla/tools/replay_computation.cc  | 180 +++++++++++++-----
 3 files changed, 142 insertions(+), 52 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/xfeed_queue.h b/tensorflow/compiler/xla/service/gpu/xfeed_queue.h
index dd46ff433b..167c038420 100644
--- a/tensorflow/compiler/xla/service/gpu/xfeed_queue.h
+++ b/tensorflow/compiler/xla/service/gpu/xfeed_queue.h
@@ -47,6 +47,10 @@ class XfeedQueue {
   // Blocks until the queue is non-empty, then returns the buffer at the head of
   // the queue.
   BufferType BlockingGetNextDestination() {
+    for (const auto& callback : before_get_next_dest_callbacks_) {
+      callback();
+    }
+
     bool became_empty;
     BufferType current_buffer;
     {
@@ -69,6 +73,10 @@ class XfeedQueue {
   void RegisterOnEmptyCallback(std::function<void()> callback) {
     on_empty_callbacks_.push_back(std::move(callback));
   }
+  void RegisterBeforeGetNextDestinationCallback(
+      std::function<void()> callback) {
+    before_get_next_dest_callbacks_.push_back(std::move(callback));
+  }
 
  private:
   tensorflow::mutex mu_;
@@ -82,6 +90,11 @@ class XfeedQueue {
   // List of callbacks which will be called when 'enqueued_buffers_' becomes
   // empty.
   std::vector<std::function<void()>> on_empty_callbacks_;
+
+  // List of callbacks which will be called before BlockingGetNextDestination()
+  // is called. This lets you e.g. call EnqueueDestination() for each call to
+  // BlockingGetNextDestination().
+  std::vector<std::function<void()>> before_get_next_dest_callbacks_;
 };
 
 }  // namespace gpu
diff --git a/tensorflow/compiler/xla/tools/BUILD b/tensorflow/compiler/xla/tools/BUILD
index 99b32c19a5..a7ef32fd35 100644
--- a/tensorflow/compiler/xla/tools/BUILD
+++ b/tensorflow/compiler/xla/tools/BUILD
@@ -95,6 +95,7 @@ cc_library(
         "//tensorflow/compiler/xla/service:hlo_parser",
         "//tensorflow/compiler/xla/service:hlo_proto",
         "//tensorflow/compiler/xla/service/gpu:infeed_manager",
+        "//tensorflow/compiler/xla/service/gpu:outfeed_manager",
         "//tensorflow/compiler/xla/tests:test_utils",
         "//tensorflow/core:framework_internal",
         "//tensorflow/core:lib",
diff --git a/tensorflow/compiler/xla/tools/replay_computation.cc b/tensorflow/compiler/xla/tools/replay_computation.cc
index 75023f6edf..c01a47b510 100644
--- a/tensorflow/compiler/xla/tools/replay_computation.cc
+++ b/tensorflow/compiler/xla/tools/replay_computation.cc
@@ -51,6 +51,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/execution_options_util.h"
 #include "tensorflow/compiler/xla/literal.h"
 #include "tensorflow/compiler/xla/service/gpu/infeed_manager.h"
+#include "tensorflow/compiler/xla/service/gpu/outfeed_manager.h"
 #include "tensorflow/compiler/xla/service/hlo.pb.h"
 #include "tensorflow/compiler/xla/service/hlo_parser.h"
 #include "tensorflow/compiler/xla/shape_util.h"
@@ -73,7 +74,17 @@ namespace {
 // fields.
 struct Options {
   string fake_infeed_shape;
-  bool generate_fake_infeed = false;
+  string fake_outfeed_shape;
+
+  // generate_fake_infeed == true is a safe default: If the model has 0 or 1
+  // infeeds, then it will work like normal.  If the model has more than one
+  // infeed, it will be an error, but that wouldn't have worked anyway if you
+  // hadn't passed generate_fake_infeed.
+  //
+  // Same for generate_fake_outfeed.
+  bool generate_fake_infeed = true;
+  bool generate_fake_outfeed = true;
+
   bool use_fake_data = false;
   bool print_result = true;
   int num_runs = 1;
@@ -96,6 +107,83 @@ std::unique_ptr<LocalExecutable> CompileExecutable(const HloSnapshot& module,
       .ValueOrDie();
 }
 
+absl::optional<Shape> GetXfeedShape(bool is_infeed,
+                                    const HloModuleProto& module,
+                                    const Options& opts) {
+  std::vector<HloInstructionProto> xfeed_instrs;
+  for (const auto& comp : module.computations()) {
+    for (const auto& instruction : comp.instructions()) {
+      if (instruction.opcode() == HloOpcodeString(is_infeed
+                                                      ? HloOpcode::kInfeed
+                                                      : HloOpcode::kOutfeed)) {
+        xfeed_instrs.push_back(instruction);
+      }
+    }
+  }
+
+  auto log_xfeed_instrs = [&] {
+    for (const auto& infeed : xfeed_instrs) {
+      LOG(ERROR) << "  " << ShapeUtil::HumanString(Shape(infeed.shape())) << " "
+                 << infeed.name();
+    }
+  };
+
+  auto find_instruction_from_id_or_die = [&](int64 id) {
+    for (const auto& comp : module.computations()) {
+      for (const auto& instruction : comp.instructions()) {
+        if (instruction.id() == id) {
+          return instruction;
+        }
+      }
+    }
+    LOG(FATAL) << "No instruction with id " << id;
+  };
+
+  absl::optional<Shape> xfeed_shape;
+  string xfeed_name = is_infeed ? "infeed" : "outfeed";
+  string fake_xfeed_shape =
+      is_infeed ? opts.fake_infeed_shape : opts.fake_outfeed_shape;
+  bool generate_fake_xfeed =
+      is_infeed ? opts.generate_fake_infeed : opts.generate_fake_outfeed;
+  if (!fake_xfeed_shape.empty()) {
+    xfeed_shape = std::move(ParseShape(fake_xfeed_shape)).ValueOrDie();
+  } else if (generate_fake_xfeed) {
+    CHECK_LT(xfeed_instrs.size(), 2)
+        << "--generate_fake_" << xfeed_name
+        << " only works if the model has 0 or 1 " << xfeed_name << " ops.";
+    if (xfeed_instrs.empty()) {
+      LOG(INFO) << "Not generating fake " << xfeed_name
+                << " shape; model has no " << xfeed_name << "s.";
+    } else if (xfeed_instrs.size() == 1) {
+      // kInfeed instructions should have a shape (buffer, token).  kOutfeed
+      // instructions should have operand 0 of shape `buffer`. We want to xfeed
+      // just `buffer`.
+      xfeed_shape = is_infeed
+                        ? Shape(xfeed_instrs.front().shape()).tuple_shapes(0)
+                        : Shape(find_instruction_from_id_or_die(
+                                    xfeed_instrs.front().operand_ids(0))
+                                    .shape());
+      LOG(INFO) << "Generating fake " << xfeed_name << " with inferred shape: "
+                << ShapeUtil::HumanString(*xfeed_shape);
+    } else {
+      LOG(ERROR) << "--generate_fake_" << xfeed_name
+                 << " only works if the model has 0 or 1 " << xfeed_name
+                 << " ops, but this model has " << xfeed_instrs.size()
+                 << " of them:";
+      log_xfeed_instrs();
+      LOG(FATAL) << "Can't run model with --generate_fake_infeed.";
+    }
+  } else if (!xfeed_instrs.empty()) {
+    LOG(ERROR) << "Model contains " << xfeed_instrs.size() << " " << xfeed_name
+               << " instruction(s), but neither --generate_fake_" << xfeed_name
+               << " nor --fake_" << xfeed_name
+               << "_shape was specified.  Execution will likely hang.";
+    log_xfeed_instrs();
+  }
+
+  return xfeed_shape;
+}
+
 // Invokes the given computation passing arbitrary data for every (unbound)
 // parameter if use_fake_data, Otherwise use recorded data if available.
 //
@@ -142,56 +230,37 @@ StatusOr<Literal> ReplayComputation(const HloSnapshot& module,
     }
   }
 
-  bool provide_infeed = false;
-  Shape infeed_shape;
-  if (!opts.fake_infeed_shape.empty()) {
-    StatusOr<Shape> shape_status = ParseShape(opts.fake_infeed_shape);
-    TF_CHECK_OK(shape_status.status());
-    infeed_shape = std::move(shape_status).ValueOrDie();
-    provide_infeed = true;
-  } else if (opts.generate_fake_infeed) {
-    for (const auto& comp : computation.proto().computations()) {
-      for (const auto& instruction : comp.instructions()) {
-        if (instruction.opcode() == HloOpcodeString(HloOpcode::kInfeed)) {
-          CHECK(!provide_infeed)
-              << "--generate_fake_infeed only works if the model has 0 or 1 "
-                 "infeed ops, but this one has >= 2.";
-          provide_infeed = true;
-          // The kInfeed instruction should have a shape (buffer, token).  We
-          // want to infeed just `buffer`.
-          infeed_shape = Shape(instruction.shape()).tuple_shapes(0);
-          LOG(INFO) << "Generating fake infeed shape for inferred shape: "
-                    << ShapeUtil::HumanString(infeed_shape);
-        }
-      }
-    }
+  if (absl::optional<Shape> infeed_shape = GetXfeedShape(
+          /*is_infeed=*/true, computation.proto(), opts)) {
+    auto infeed_data = std::make_shared<Literal>(
+        std::move(MakeFakeLiteral(*infeed_shape)).ValueOrDie());
+    xla::gpu::GetOrCreateInfeedManager()
+        ->RegisterBeforeGetNextDestinationCallback([infeed_data, client] {
+          TF_CHECK_OK(client->TransferToInfeed(*infeed_data));
+        });
   }
-  // We only instantiate the thread pool if the user has requested that a
-  // concurrent infeed occur via the fake_infeed_shape, or when
-  // --generate_fake_infeed is passed and there exists an infeed operation in
-  // the HloSnapshot.
-  absl::optional<tensorflow::thread::ThreadPool> pool;
-  Literal data;
-  if (provide_infeed) {
-    data = std::move(MakeFakeLiteral(infeed_shape)).ValueOrDie();
-  }
-  auto transfer_infeed = [&data, client]() {
-    TF_CHECK_OK(client->TransferToInfeed(data));
-  };
-  if (provide_infeed) {
-    pool.emplace(tensorflow::Env::Default(), "infeed",
-                 /*num_threads=*/1);
-    pool->Schedule([transfer_infeed]() {
-      // There may be several infeed buffers needed, however we don't know how
-      // many. If we proactively transfer too many infeed buffers, we may run
-      // out of memory. If we transfer too few infeed buffers, the program will
-      // hang. Therefore, we register a callback that is called when the infeed
-      // becomes empty, and in this callback we will transfer another fake
-      // infeed.
-      auto infeed_manager = xla::gpu::GetOrCreateInfeedManager();
-      infeed_manager->RegisterOnEmptyCallback(transfer_infeed);
-      transfer_infeed();
-    });
+
+  absl::optional<tensorflow::thread::ThreadPool> outfeed_thread_pool;
+  if (absl::optional<Shape> outfeed_shape = GetXfeedShape(
+          /*is_infeed=*/false, computation.proto(), opts)) {
+    // For each an outfeed that runs, enqueue a task that will consume it.  We
+    // need a thread pool because the act of running an outfeed blocks on there
+    // being a destination available, and the act of making a destination
+    // available blocks on there being outfeed data available.
+    outfeed_thread_pool.emplace(tensorflow::Env::Default(), "infeed",
+                                /*num_threads=*/1);
+    auto consume_outfeed = [client, outfeed_shape] {
+      TF_CHECK_OK(
+          client->TransferFromOutfeedLocal(*outfeed_shape, /*device_ordinal=*/0)
+              .status());
+      VLOG(1) << "Received outfeed data of shape "
+              << ShapeUtil::HumanStringWithLayout(*outfeed_shape);
+    };
+    xla::gpu::GetOrCreateOutfeedManager()
+        ->RegisterBeforeGetNextDestinationCallback(
+            [consume_outfeed, &outfeed_thread_pool] {
+              outfeed_thread_pool->Schedule(consume_outfeed);
+            });
   }
 
   // Do not attempt to run the executable if num_runs is less than 1.
@@ -306,8 +375,10 @@ int RealMain(absl::Span<char* const> args, const Options& opts) {
 
   for (int64 i = 0; i < executables.size(); ++i) {
     LocalExecutable* executable = executables[i].get();
+    LOG(ERROR) << "Running iteration " << i;
     StatusOr<Literal> result_status =
         ReplayComputation(snapshots[i], executable, client, opts);
+    LOG(ERROR) << "iteration complete.";
     if (!result_status.ok()) {
       fprintf(stderr, "%s: error: %s\n", args[i],
               result_status.status().ToString().c_str());
@@ -352,9 +423,14 @@ int main(int argc, char** argv) {
                        "Number of times to run each computation"),
       tensorflow::Flag("fake_infeed_shape", &opts.fake_infeed_shape,
                        "Shape of fake data to construct for (infinite) infeed"),
+      tensorflow::Flag("fake_outfeed_shape", &opts.fake_outfeed_shape,
+                       "Shape of fake data to outfeed from computation"),
       tensorflow::Flag("generate_fake_infeed", &opts.generate_fake_infeed,
-                       "Whether a fake infeed shape should be generated "
-                       "derived from the computation"),
+                       "Whether a fake infeed shape should be derived "
+                       "from the computation"),
+      tensorflow::Flag("generate_fake_outfeed", &opts.generate_fake_outfeed,
+                       "Whether a fake outfeed shape should be derived "
+                       "from the computation"),
   };
   xla::string usage = tensorflow::Flags::Usage(argv[0], flag_list);
   bool parse_ok = tensorflow::Flags::Parse(&argc, argv, flag_list);
-- 
GitLab


From 0eca4f6281579e4db607429c9dd97c9146008d65 Mon Sep 17 00:00:00 2001
From: Peter Ma <pcma@google.com>
Date: Fri, 11 Jan 2019 17:31:01 -0800
Subject: [PATCH 0635/1095] Readability improvement on graph_properties: - Use
 'ic' for InferenceContext*, and 'ctx' for NodeContext* to be consistent
 within graph_properties (there are places where 'c' is used for either one in
 short functions). - The inner loop for iterating over input nodes in
 UpdateNode is redundant, and thus removed. - UpdateMergeNode is renamed to
 UpdateMerge in consistence with other Update functions; - Some other minor
 changes.

PiperOrigin-RevId: 228972916
---
 tensorflow/core/grappler/costs/BUILD          |   2 -
 .../core/grappler/costs/graph_properties.cc   | 298 ++++++++----------
 .../core/grappler/costs/graph_properties.h    |   4 +-
 3 files changed, 142 insertions(+), 162 deletions(-)

diff --git a/tensorflow/core/grappler/costs/BUILD b/tensorflow/core/grappler/costs/BUILD
index 92294379b5..35ca93d934 100644
--- a/tensorflow/core/grappler/costs/BUILD
+++ b/tensorflow/core/grappler/costs/BUILD
@@ -41,8 +41,6 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = [
         ":utils",
-        "@com_google_absl//absl/memory",
-        "@com_google_absl//absl/strings",
         "//tensorflow/core/grappler/utils:functions",
         "//tensorflow/core/grappler/utils:topological_sort",
         "//tensorflow/core/grappler:mutable_graph_view",
diff --git a/tensorflow/core/grappler/costs/graph_properties.cc b/tensorflow/core/grappler/costs/graph_properties.cc
index e869f7830c..9f5966c4af 100644
--- a/tensorflow/core/grappler/costs/graph_properties.cc
+++ b/tensorflow/core/grappler/costs/graph_properties.cc
@@ -15,12 +15,6 @@ limitations under the License.
 
 #include "tensorflow/core/grappler/costs/graph_properties.h"
 
-#include <limits>
-#include <list>
-#include <queue>
-#include <unordered_map>
-#include <unordered_set>
-#include "absl/memory/memory.h"
 #include "tensorflow/core/framework/common_shape_fns.h"
 #include "tensorflow/core/framework/function.pb.h"
 #include "tensorflow/core/framework/node_def_util.h"
@@ -44,7 +38,6 @@ limitations under the License.
 
 namespace tensorflow {
 namespace grappler {
-using TensorVector = gtl::InlinedVector<TensorValue, 4>;
 
 namespace {
 
@@ -52,6 +45,7 @@ using shape_inference::DimensionHandle;
 using shape_inference::InferenceContext;
 using shape_inference::ShapeAndType;
 using shape_inference::ShapeHandle;
+using TensorVector = gtl::InlinedVector<TensorValue, 4>;
 
 template <typename Handle>
 struct HashHandle {
@@ -416,6 +410,7 @@ NodeDef MakeConstNodeDefFromShape(InferenceContext* ic,
   return MakeConstNodeDefFromTensorProto(
       ic, MakeTensorProtoFromShape(ic, shape, tensor_as_shape, dtype), dtype);
 }
+
 }  // namespace
 
 // Queue of nodes to process. Nodes can be enqueued in any order, but will be
@@ -607,8 +602,8 @@ class SymbolicShapeRefiner {
                                           " was not found in the graph.");
       }
 
-      InferenceContext* input_inference_context = GetContext(input_node);
-      if (input_inference_context == nullptr) {
+      InferenceContext* input_ic = GetContext(input_node);
+      if (input_ic == nullptr) {
         return errors::FailedPrecondition(
             "Inference context has not been created for ", input_tensor.node());
       }
@@ -616,8 +611,8 @@ class SymbolicShapeRefiner {
       int output_port_num = input_tensor.index();
       AttrValue attr_output_shape;
       TensorShapeProto proto;
-      const auto& handle = input_inference_context->output(output_port_num);
-      input_inference_context->ShapeHandleToProto(handle, &proto);
+      const auto& handle = input_ic->output(output_port_num);
+      input_ic->ShapeHandleToProto(handle, &proto);
       // There may be dim.size < -1 in SymbolicShapeRefiner. Change those to -1.
       for (int i = 0; i < proto.dim_size(); i++) {
         if (proto.dim(i).size() < -1) {
@@ -715,145 +710,132 @@ class SymbolicShapeRefiner {
     return Status::OK();
   }
 
+  // Prepares input shapes/values/handles, then runs shape inference, and
+  // finally sets output shapes/values/handles.
   Status UpdateNode(const NodeDef* node, bool* refined) {
-    NodeContext* node_context = GetNodeContext(node);
-    if (node_context == nullptr) {
+    NodeContext* ctx = GetNodeContext(node);
+    if (ctx == nullptr) {
       TF_RETURN_IF_ERROR(AddNode(node));
-      node_context = CHECK_NOTNULL(GetNodeContext(node));
+      ctx = CHECK_NOTNULL(GetNodeContext(node));
       *refined = true;
     }
 
     // Check if the shapes of the nodes in the fan-in of this node have changed,
     // and if they have, update the node input shapes.
-    InferenceContext* inference_context = node_context->inference_context.get();
-    std::vector<Tensor> const_values(inference_context->num_inputs());
-    std::vector<const Tensor*> input_tensors(inference_context->num_inputs(),
-                                             nullptr);
-    std::vector<ShapeHandle> input_tensors_as_shapes(
-        inference_context->num_inputs());
-    node_context->input_tensor_protos.resize(inference_context->num_inputs(),
-                                             nullptr);
-
-    for (int dst_input = 0; dst_input < inference_context->num_inputs();
-         ++dst_input) {
-      GraphView::InputPort port(node, dst_input);
-      for (const GraphView::OutputPort fanin : graph_.GetFanin(port)) {
-        int src_output = fanin.port_id;
-        const NodeDef* input = fanin.node;
-        NodeContext* c = GetNodeContext(input);
-        if (c == nullptr) {
-          return errors::FailedPrecondition(
-              "Input ", dst_input, " ('", input->name(), "') for '",
-              node->name(),
-              "' was not previously added to SymbolicShapeRefiner.");
-        }
+    InferenceContext* ic = ctx->inference_context.get();
+    std::vector<Tensor> const_values(ic->num_inputs());
+    std::vector<const Tensor*> input_tensors(ic->num_inputs(), nullptr);
+    std::vector<ShapeHandle> input_tensors_as_shapes(ic->num_inputs());
+    ctx->input_tensor_protos.resize(ic->num_inputs(), nullptr);
+
+    for (int dst_input = 0; dst_input < ic->num_inputs(); ++dst_input) {
+      const GraphView::InputPort port(node, dst_input);
+      const GraphView::OutputPort fanin = graph_.GetRegularFanin(port);
+      int src_output = fanin.port_id;
+      const NodeDef* src = fanin.node;
+      NodeContext* src_ctx = GetNodeContext(src);
+      InferenceContext* src_ic = src_ctx->inference_context.get();
+      if (src_ctx == nullptr) {
+        return errors::FailedPrecondition(
+            "Input ", dst_input, " ('", src->name(), "') for '", node->name(),
+            "' was not previously added to SymbolicShapeRefiner.");
+      }
 
-        if (src_output >= c->inference_context->num_outputs())
-          return errors::OutOfRange("src_output = ", src_output,
-                                    ", but num_outputs is only ",
-                                    c->inference_context->num_outputs());
+      if (src_output >= src_ic->num_outputs()) {
+        return errors::OutOfRange("src_output = ", src_output,
+                                  ", but num_outputs is only ",
+                                  src_ic->num_outputs());
+      }
 
-        // Propagate input node's NodeContext info to the current node's
-        // NodeContext:
-        // output_tensor_protos to input_tensor_protos and input_tensors, and
-        // output_tensors_as_shapes to input_tensors_as_shapes.
+      // Propagate input node's NodeContext info to the current node's
+      // NodeContext:
+      // output_tensor_protos to input_tensor_protos and input_tensors, and
+      // output_tensors_as_shapes to input_tensors_as_shapes.
 
-        if (c->output_tensors_as_shapes.size() > src_output) {
-          input_tensors_as_shapes[dst_input] =
-              c->output_tensors_as_shapes[src_output];
-        }
+      if (src_ctx->output_tensors_as_shapes.size() > src_output) {
+        input_tensors_as_shapes[dst_input] =
+            src_ctx->output_tensors_as_shapes[src_output];
+      }
 
-        if (c->output_tensor_protos.size() > src_output) {
-          auto* tensor_proto = c->output_tensor_protos[src_output];
-          if (tensor_proto != nullptr &&
-              const_values[dst_input].FromProto(*tensor_proto)) {
-            input_tensors[dst_input] = &const_values[dst_input];
-            node_context->input_tensor_protos[dst_input] = tensor_proto;
-
-            if (!inference_context->FullyDefined(
-                    input_tensors_as_shapes[dst_input])) {
-              // Shape from a Const is not fully defined when the Const has
-              // value -1 (e.g., Reshape(x, Const(-1)) to reshape an arbitrary
-              // tensor x to a vector).
-              // It's possible that the same Const with -1 is used in many
-              // places, but that doesn't mean the resultant shapes are
-              // identical. e.g., x1 = Reshape(x, c) and y1 = Reshape(y, c),
-              // where c is -1. In this case, shape inference yields both x1 and
-              // y1 as rank 1, size unknown, but still the shapes of x1 and y1
-              // can be different. (even if we use different Const(-1) for x1
-              // and x2, graph optimzier may merge them to single Const through
-              // duplicate removal.)
-              // If we reuse output_tensors_as_shapes to input_tensors_as_shapes
-              // by copying ShapeHandle, they share the same Shape object, and
-              // SymbolicShapeManager, later in InferStatically(), assigns the
-              // same symbolic dim value (unique value < -1); in the above
-              // Reshape example, the shapes of x1 and y1 become, for example,
-              // [-278] and graph optimizer may yield incorrect output 'cause it
-              // assumes x1 and y1 have the same shape.
-              // To prevent this, we re-create a ShapeHandle from the Const
-              // tensor, instead of reusing output_tensors_as_shapes (so that
-              // ShapeHandles of the const fanouts have the same values,
-              // but different Shape objects -- SymbolicShapeManager assigns
-              // different symbol id to each fanout shape).
-              // TODO(dyoon): clean up the way values are propagated.
-              MaybeTensorValueToShape(inference_context,
-                                      const_values[dst_input],
-                                      &input_tensors_as_shapes[dst_input]);
-            }
+      if (src_ctx->output_tensor_protos.size() > src_output) {
+        auto* tensor_proto = src_ctx->output_tensor_protos[src_output];
+        if (tensor_proto != nullptr &&
+            const_values[dst_input].FromProto(*tensor_proto)) {
+          input_tensors[dst_input] = &const_values[dst_input];
+          ctx->input_tensor_protos[dst_input] = tensor_proto;
+
+          if (!ic->FullyDefined(input_tensors_as_shapes[dst_input])) {
+            // Shape from a Const is not fully defined when the Const has
+            // value -1 (e.g., Reshape(x, Const(-1)) to reshape an arbitrary
+            // tensor x to a vector).
+            // It's possible that the same Const with -1 is used in many
+            // places, but that doesn't mean the resultant shapes are
+            // identical. e.g., x1 = Reshape(x, c) and y1 = Reshape(y, c),
+            // where c is -1. In this case, shape inference yields both x1 and
+            // y1 as rank 1, size unknown, but still the shapes of x1 and y1
+            // can be different. (even if we use different Const(-1) for x1
+            // and x2, graph optimzier may merge them to single Const through
+            // duplicate removal.)
+            // If we reuse output_tensors_as_shapes to input_tensors_as_shapes
+            // by copying ShapeHandle, they share the same Shape object, and
+            // SymbolicShapeManager, later in InferStatically(), assigns the
+            // same symbolic dim value (unique value < -1); in the above
+            // Reshape example, the shapes of x1 and y1 become, for example,
+            // [-278] and graph optimizer may yield incorrect output 'cause it
+            // assumes x1 and y1 have the same shape.
+            // To prevent this, we re-create a ShapeHandle from the Const
+            // tensor, instead of reusing output_tensors_as_shapes (so that
+            // ShapeHandles of the const fanouts have the same values,
+            // but different Shape objects -- SymbolicShapeManager assigns
+            // different symbol id to each fanout shape).
+            // TODO(dyoon): clean up the way values are propagated.
+            MaybeTensorValueToShape(ic, const_values[dst_input],
+                                    &input_tensors_as_shapes[dst_input]);
           }
         }
+      }
 
-        DCHECK_GE(dst_input, 0);
-        // NOTE: we check only shape is refined; we do not (yet) check whether
-        // tensor value is refined.
-        if (!*refined && !inference_context->input(dst_input).SameHandle(
-                             c->inference_context->output(src_output))) {
-          *refined = true;
-        }
-        inference_context->SetInput(dst_input,
-                                    c->inference_context->output(src_output));
-
-        if (!*refined &&
-            inference_context->requested_input_tensor_as_partial_shape(
-                dst_input)) {
-          // The input value may have changed. Since we have no way to know if
-          // that's indeed the case, err on the safe side.
-          *refined = true;
-        }
+      // NOTE: we check only shape is refined; we do not (yet) check whether
+      // tensor value is refined.
+      if (!*refined &&
+          !ic->input(dst_input).SameHandle(src_ic->output(src_output))) {
+        *refined = true;
+      }
+      ic->SetInput(dst_input, src_ic->output(src_output));
 
-        // Also propagate handle shape and dtype of edges which are carrying
-        // resource handles.
-        if (node_context->input_types[dst_input] == DT_RESOURCE) {
-          auto* outputs =
-              c->inference_context->output_handle_shapes_and_types(src_output);
-          if (!outputs) continue;
-          auto* inputs =
-              inference_context->input_handle_shapes_and_types(dst_input);
-
-          if (!inputs || !EquivalentShapesAndTypes(*outputs, *inputs)) {
-            *refined = true;
-          }
-          inference_context->set_input_handle_shapes_and_types(dst_input,
-                                                               *outputs);
-        }
+      if (!*refined && ic->requested_input_tensor_as_partial_shape(dst_input)) {
+        // The input value may have changed. Since we have no way to know if
+        // that's indeed the case, err on the safe side.
+        *refined = true;
+      }
+
+      // Also propagate handle shape and dtype of edges which are carrying
+      // resource handles.
+      if (ctx->input_types[dst_input] == DT_RESOURCE) {
+        auto* outputs = src_ic->output_handle_shapes_and_types(src_output);
+        if (!outputs) continue;
+        auto* inputs = ic->input_handle_shapes_and_types(dst_input);
+
+        if (!inputs || !EquivalentShapesAndTypes(*outputs, *inputs))
+          *refined = true;
+        ic->set_input_handle_shapes_and_types(dst_input, *outputs);
       }
     }
 
     // Make sure we schedule the fanout of resources (which have no input)
     // whenever the resources are updated.
-    *refined |= inference_context->num_inputs() == 0;
+    *refined |= ic->num_inputs() == 0;
 
     if (!*refined) {
       // No input shape has changed, we're done.
       return Status::OK();
     }
 
-    node_context->inference_context->set_input_tensors(input_tensors);
-    node_context->inference_context->set_input_tensors_as_shapes(
-        input_tensors_as_shapes);
+    ic->set_input_tensors(input_tensors);
+    ic->set_input_tensors_as_shapes(input_tensors_as_shapes);
 
     // Properly handle function nodes.
-    if (node_context->op_data && node_context->op_data->is_function_op) {
+    if (ctx->op_data && ctx->op_data->is_function_op) {
       // TODO(jmdecker): Detect if the input shapes have changed for this
       // function. Note that when we hit a function call node, refined will be
       // true, as the updates to the call node will have changed, even if it's
@@ -870,7 +852,7 @@ class SymbolicShapeRefiner {
     }
 
     // Update the shapes of the outputs.
-    return InferShapes(*node, node_context);
+    return InferShapes(*node, ctx);
   }
 
   Status SetUnknownShape(const NodeDef* node, int output_port) {
@@ -1681,45 +1663,45 @@ Status GraphProperties::RelaxEnqueueShapesAndMergeTypes(
 
 // Compute the output shape of the merge node as the union of the available
 // input shapes.
-Status GraphProperties::UpdateMergeNode(SymbolicShapeRefiner* shape_refiner,
-                                        const NodeDef* node,
-                                        bool* new_shapes) const {
-  InferenceContext* c = shape_refiner->GetContext(node);
-  if (!c) {
+Status GraphProperties::UpdateMerge(SymbolicShapeRefiner* shape_refiner,
+                                    const NodeDef* node,
+                                    bool* new_shapes) const {
+  InferenceContext* ic = shape_refiner->GetContext(node);
+  if (!ic) {
     // Now we can run shape inference
     TF_RETURN_IF_ERROR(shape_refiner->AddNode(node));
-    c = CHECK_NOTNULL(shape_refiner->GetContext(node));
+    ic = CHECK_NOTNULL(shape_refiner->GetContext(node));
     *new_shapes = true;
 
     // Infer the shape of the second output once and for all since it never
     // changes.
-    ShapeHandle out1 = c->Scalar();
-    c->set_output(1, out1);
+    ShapeHandle out1 = ic->Scalar();
+    ic->set_output(1, out1);
   }
 
   ShapeHandle out;
   bool out_initialized = false;
-  for (const GraphView::Edge fanin :
-       shape_refiner->graph().GetFaninEdges(*node, false)) {
-    InferenceContext* in = shape_refiner->GetContext(fanin.src.node);
-    if (!in) {
+  for (const GraphView::Edge fanin : shape_refiner->graph().GetFaninEdges(
+           *node, /*include_controlling_edges=*/false)) {
+    InferenceContext* src_ic = shape_refiner->GetContext(fanin.src.node);
+    if (!src_ic) {
       // Handling a loop for the first time, the back edge won't have any shape
       // info.
       continue;
     }
-    ShapeHandle input = in->output(fanin.src.port_id);
+    ShapeHandle input = src_ic->output(fanin.src.port_id);
     CHECK_EQ(fanin.dst.node, node);
-    c->SetInput(fanin.dst.port_id, input);
+    ic->SetInput(fanin.dst.port_id, input);
     if (!out_initialized) {
       out_initialized = true;
       out = input;
-      continue;
+    } else {
+      out = shape_refiner->OutputAsUnion(node, 0, input, out);
     }
-    out = shape_refiner->OutputAsUnion(node, 0, input, out);
   }
 
-  if (*new_shapes || !shape_refiner->EquivalentShapes(out, c->output(0))) {
-    c->set_output(0, out);
+  if (*new_shapes || !shape_refiner->EquivalentShapes(out, ic->output(0))) {
+    ic->set_output(0, out);
     *new_shapes = true;
   }
 
@@ -1729,26 +1711,26 @@ Status GraphProperties::UpdateMergeNode(SymbolicShapeRefiner* shape_refiner,
 // Manually propagate the input shape for Enter nodes.
 Status GraphProperties::UpdateEnter(SymbolicShapeRefiner* shape_refiner,
                                     const NodeDef* node, bool* new_shapes) {
-  auto enter_ctx = shape_refiner->GetContext(node);
-  if (!enter_ctx) {
+  InferenceContext* ic = shape_refiner->GetContext(node);
+  if (!ic) {
     TF_RETURN_IF_ERROR(shape_refiner->UpdateNode(node, new_shapes));
-    enter_ctx = shape_refiner->GetContext(node);
+    ic = shape_refiner->GetContext(node);
   }
 
-  GraphView::InputPort inp(node, 0);
-  GraphView::OutputPort fanin = shape_refiner->graph().GetRegularFanin(inp);
+  GraphView::InputPort port(node, 0);
+  GraphView::OutputPort fanin = shape_refiner->graph().GetRegularFanin(port);
 
-  InferenceContext* in = shape_refiner->GetContext(fanin.node);
-  ShapeHandle input = in->output(fanin.port_id);
-  if (!enter_ctx->output(0).SameHandle(input)) {
-    enter_ctx->SetInput(0, input);
-    enter_ctx->set_output(0, input);
+  InferenceContext* src_ic = shape_refiner->GetContext(fanin.node);
+  ShapeHandle input = src_ic->output(fanin.port_id);
+  if (!ic->output(0).SameHandle(input)) {
+    ic->SetInput(0, input);
+    ic->set_output(0, input);
     *new_shapes = true;
   }
-  auto* outputs = in->output_handle_shapes_and_types(fanin.port_id);
+  auto* outputs = src_ic->output_handle_shapes_and_types(fanin.port_id);
   if (outputs) {
-    enter_ctx->set_input_handle_shapes_and_types(0, *outputs);
-    enter_ctx->set_output_handle_shapes_and_types(0, *outputs);
+    ic->set_input_handle_shapes_and_types(0, *outputs);
+    ic->set_output_handle_shapes_and_types(0, *outputs);
     *new_shapes = true;
   }
   return Status::OK();
@@ -1764,7 +1746,7 @@ Status GraphProperties::UpdateShapes(
     TF_RETURN_IF_ERROR(UpdateEnter(shape_refiner, n, new_shapes));
   } else if (IsMerge(*n)) {
     // Properly handle merge nodes.
-    TF_RETURN_IF_ERROR(UpdateMergeNode(shape_refiner, n, new_shapes));
+    TF_RETURN_IF_ERROR(UpdateMerge(shape_refiner, n, new_shapes));
   } else if (IsEnqueue(*n)) {
     // Make sure the shapes of enqueued tensors are propagated to the queue
     // itself.
@@ -1811,8 +1793,8 @@ Status GraphProperties::PropagateShapes(
       TF_RETURN_IF_ERROR(
           UpdateShapes(shape_refiner, resource_handles, n, &updated));
       if (updated) {
-        for (const GraphView::InputPort& fanout :
-             shape_refiner->graph().GetFanouts(*n, false)) {
+        for (const auto& fanout : shape_refiner->graph().GetFanouts(
+                 *n, /*include_controlled_nodes=*/false)) {
           new_shapes->push(fanout.node);
         }
         // Make sure the corresponding queue nodes are (re)processed.
@@ -1837,7 +1819,7 @@ Status GraphProperties::PropagateShapes(
 Status GraphProperties::UpdateQueue(const NodeDef* queue_node,
                                     SymbolicShapeRefiner* shape_refiner,
                                     bool* new_shapes) {
-  auto ctx = shape_refiner->GetNodeContext(queue_node);
+  auto* ctx = shape_refiner->GetNodeContext(queue_node);
   if (!ctx) {
     TF_RETURN_IF_ERROR(shape_refiner->AddNode(queue_node));
     ctx = CHECK_NOTNULL(shape_refiner->GetNodeContext(queue_node));
diff --git a/tensorflow/core/grappler/costs/graph_properties.h b/tensorflow/core/grappler/costs/graph_properties.h
index 0635222fe9..3fcad6eb1b 100644
--- a/tensorflow/core/grappler/costs/graph_properties.h
+++ b/tensorflow/core/grappler/costs/graph_properties.h
@@ -111,8 +111,8 @@ class GraphProperties {
 
   // Update the output shapes of a Merge node, and enqueue its fanout in
   // new_shapes if needed.
-  Status UpdateMergeNode(SymbolicShapeRefiner* shape_refiner,
-                         const NodeDef* node, bool* new_shapes) const;
+  Status UpdateMerge(SymbolicShapeRefiner* shape_refiner, const NodeDef* node,
+                     bool* new_shapes) const;
   // Process the Enter node, and enqueue its fanout in new_shapes if needed.
   static Status UpdateEnter(SymbolicShapeRefiner* shape_refiner,
                             const NodeDef* node, bool* new_shapes);
-- 
GitLab


From 28093291532750e8f1529938f1a44d3b6eceb3b1 Mon Sep 17 00:00:00 2001
From: Aurelien Geron <aurelien.geron@gmail.com>
Date: Sat, 12 Jan 2019 09:40:46 +0800
Subject: [PATCH 0636/1095] Fix typo, renamed _setup_concrete_functions()

---
 tensorflow/python/saved_model/load.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/saved_model/load.py b/tensorflow/python/saved_model/load.py
index b86e2f0aec..b30d392099 100644
--- a/tensorflow/python/saved_model/load.py
+++ b/tensorflow/python/saved_model/load.py
@@ -51,7 +51,7 @@ class _Loader(object):
     self._setup_concrete_functions()
     self._restore_checkpoint()
 
-  def _bind_function_captures(self):
+  def _setup_concrete_functions(self):
     """Setup captured tensors and output structure in restored concrete
     functions."""
     seen_concrete_functions = set()
-- 
GitLab


From c6bb5cd0447a0af2764c195fb14d218df8ae6471 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 11 Jan 2019 17:40:46 -0800
Subject: [PATCH 0637/1095] Fix Windows build by undefining unwanted ERROR
 macro.

PiperOrigin-RevId: 228974009
---
 tensorflow/stream_executor/cuda/cuda_gpu_executor.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc
index 4874d096ad..5b33d97594 100644
--- a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc
+++ b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc
@@ -53,6 +53,10 @@ limitations under the License.
 #include "tensorflow/stream_executor/stream_executor_pimpl.h"
 #include "tensorflow/stream_executor/timer.h"
 
+// LOG(ERROR) uses a const named ERROR, so a macro with the same name is
+// always unwanted. This happens on Windows that defines such a macro.
+#undef ERROR
+
 #ifdef PLATFORMS_GPUS_CUDA_DYNAMIC_LIBCUDA_DYNAMIC_LIBCUDA_H_
 #error \
     "No driver calls in this file, wrap driver functionality in cuda_driver.cc."
-- 
GitLab


From f6d74c3653a8a2c3c1bdcbe09be04b97b3f952cb Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Fri, 11 Jan 2019 17:43:18 -0800
Subject: [PATCH 0638/1095] [Resubmit][TF2XLA] Use dynamic dimension in SizeOp.

This is needed if the number of elements in a tensor can be dynamic.

Resubmit after xla_builder support constant-folding get_dynamic_size.

PiperOrigin-RevId: 228974238
---
 tensorflow/compiler/tf2xla/kernels/shape_op.cc | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/shape_op.cc b/tensorflow/compiler/tf2xla/kernels/shape_op.cc
index 12830816ec..85b0367f73 100644
--- a/tensorflow/compiler/tf2xla/kernels/shape_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/shape_op.cc
@@ -20,6 +20,7 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_helpers.h"
 #include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
+#include "tensorflow/compiler/xla/client/lib/constants.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/core/framework/kernel_def_builder.h"
 #include "tensorflow/core/framework/tensor_shape.h"
@@ -91,14 +92,20 @@ class SizeOp : public XlaOpKernel {
 
   void Compile(XlaOpKernelContext* ctx) override {
     const TensorShape input_shape = ctx->InputShape(0);
-    const int64 size = input_shape.num_elements();
-    OP_REQUIRES(ctx, FastBoundsCheck(size, std::numeric_limits<int32>::max()),
+    OP_REQUIRES(ctx,
+                FastBoundsCheck(input_shape.num_elements(),
+                                std::numeric_limits<int32>::max()),
                 errors::InvalidArgument("Size does not work for tensors > "
                                         "int32 max."));
     Tensor size_constant(DT_INT32, TensorShape({}));
-    size_constant.scalar<int32>()() = static_cast<int32>(size);
-
-    ctx->SetConstantOutput(0, size_constant);
+    const int rank = input_shape.dims();
+    xla::XlaBuilder* builder = ctx->builder();
+    auto size = xla::One(builder, xla::U32);
+    for (int64 i = 0; i < rank; ++i) {
+      size = xla::Mul(size, xla::GetDimensionSize(ctx->Input(0), i));
+    }
+    size = xla::ConvertElementType(size, xla::S32);
+    ctx->SetOutput(0, size);
   }
 };
 
-- 
GitLab


From 706cce3f2395806757fc9f3ffeb07c132f13c993 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Fri, 11 Jan 2019 17:49:28 -0800
Subject: [PATCH 0639/1095] [XLA] Speed up reduce_window_test by using iota
 rather than generating random numbers.

1.3x faster on some of the large testcases.

PiperOrigin-RevId: 228974823
---
 tensorflow/compiler/xla/array.h               | 13 ++++++--
 .../compiler/xla/tests/reduce_window_test.cc  | 31 +++++++++++--------
 2 files changed, 29 insertions(+), 15 deletions(-)

diff --git a/tensorflow/compiler/xla/array.h b/tensorflow/compiler/xla/array.h
index 58cc157585..529e7f77ce 100644
--- a/tensorflow/compiler/xla/array.h
+++ b/tensorflow/compiler/xla/array.h
@@ -272,6 +272,15 @@ class Array {
     std::iota(&values_[0], &values_[0] + num_elements(), value);
   }
 
+  // Fills the array with a repeating sequence:
+  //   [value, value + 1, ..., value + length - 1, value, ... ]
+  void FillRepeatedIota(const T& value, int64 length) {
+    for (int64 i = 0; i < num_elements(); i += length) {
+      std::iota(&values_[i], &values_[std::min(i + length, num_elements())],
+                value);
+    }
+  }
+
   // Fills the array with the sequence i*multiplier for i=0,1,...
   void FillWithMultiples(const T& multiplier) {
     for (int64 i = 0; i < num_elements(); ++i) {
@@ -280,11 +289,11 @@ class Array {
   }
 
   // Fills the array with random normal variables with the specified mean.
-  void FillRandom(const T& value, const double mean = 0.0,
+  void FillRandom(const T& stddev, const double mean = 0.0,
                   const int seed = 12345) {
     std::mt19937 g(seed);
     std::normal_distribution<double> distribution(mean,
-                                                  static_cast<double>(value));
+                                                  static_cast<double>(stddev));
     for (int64 i = 0; i < num_elements(); ++i) {
       values_[i] = static_cast<T>(distribution(g));
     }
diff --git a/tensorflow/compiler/xla/tests/reduce_window_test.cc b/tensorflow/compiler/xla/tests/reduce_window_test.cc
index 22fe4a2670..16c67d94c7 100644
--- a/tensorflow/compiler/xla/tests/reduce_window_test.cc
+++ b/tensorflow/compiler/xla/tests/reduce_window_test.cc
@@ -607,7 +607,10 @@ class R4ReduceWindowTest : public ReduceWindowTestBase,
 
     Array4D<float> input(param.base_bounds[0], param.base_bounds[1],
                          param.base_bounds[2], param.base_bounds[3]);
-    input.FillRandom(0.1f, 0.1f);
+    // Choose a prime iota length so that each window sees a unique set of
+    // values. (Technically, the requirement is that the iota length is
+    // relatively prime to all of the dimensions involved in the reduce-window.)
+    input.FillRepeatedIota(0, 137);
     Literal input_literal = LiteralUtil::CreateR4FromArray4DWithLayout(
         input, LayoutUtil::MakeLayout(param.layout));
     XlaOp parameter;
@@ -623,9 +626,9 @@ class R4ReduceWindowTest : public ReduceWindowTestBase,
         CreateConstantFromLiteral(LiteralUtil::CreateR0(kInitValue), &b);
     CHECK(param.reducer == kAdd || param.reducer == kMax);
     auto reducer = param.reducer;
-    if (use_bfloat16() && Product(param.window_bounds) > 128) {
-      // To avoid numerical issues, force the reducer to be kMax for large bf16
-      // windows.
+    if (use_bfloat16()) {
+      // To avoid numerical issues, force the reducer to be kMax for bf16
+      // inputs.
       reducer = kMax;
     }
 
@@ -949,16 +952,16 @@ struct R3ReduceWindowTestData {
      /*padding=*/Padding::kValid, /*reducer=*/Reducer::kAdd},
     {/*base_bounds=*/{95, 202, 251}, /*window_bounds=*/{95, 202, 251},
      /*strides=*/{1, 1, 1}, /*layout=*/{2, 1, 0},
-     /*padding=*/Padding::kValid, /*reducer=*/Reducer::kAdd},
+     /*padding=*/Padding::kValid, /*reducer=*/Reducer::kMax},
     {/*base_bounds=*/{999, 57, 3}, /*window_bounds=*/{999, 57, 3},
      /*strides=*/{1, 1, 1}, /*layout=*/{2, 1, 0},
      /*padding=*/Padding::kValid, /*reducer=*/Reducer::kAdd},
     {/*base_bounds=*/{178, 302, 64}, /*window_bounds=*/{178, 302, 64},
      /*strides=*/{1, 1, 1}, /*layout=*/{2, 1, 0},
-     /*padding=*/Padding::kValid, /*reducer=*/Reducer::kAdd},
+     /*padding=*/Padding::kValid, /*reducer=*/Reducer::kMax},
     {/*base_bounds=*/{63, 261, 257}, /*window_bounds=*/{63, 261, 257},
      /*strides=*/{1, 1, 1}, /*layout=*/{2, 1, 0},
-     /*padding=*/Padding::kValid, /*reducer=*/Reducer::kAdd},
+     /*padding=*/Padding::kValid, /*reducer=*/Reducer::kMax},
     {/*base_bounds=*/{10003, 10, 5}, /*window_bounds=*/{9999, 7, 3},
      /*strides=*/{1, 1, 1}, /*layout=*/{2, 1, 0},
      /*padding=*/Padding::kValid, /*reducer=*/Reducer::kAdd},
@@ -1001,17 +1004,19 @@ TEST_P(R3ReduceWindowTest, DoIt) {
   const float kInitValue = 0.0f;
   Array3D<float> input(param.base_bounds[0], param.base_bounds[1],
                        param.base_bounds[2]);
-  input.FillRandom(0.1f, 0.1f);
+  // Choose a prime iota length so that each window sees a unique set of values.
+  // (Technically, the requirement is that the iota length is relatively prime
+  // to all of the dimensions involved in the reduce-window.)
+  input.FillRepeatedIota(0, 137);
   Literal input_literal = LiteralUtil::CreateR3FromArray3DWithLayout(
       input, LayoutUtil::MakeLayout(param.layout));
   auto reducer = param.reducer;
   if (use_bfloat16()) {
     input_literal = LiteralUtil::ConvertF32ToBF16(input_literal);
-    if (Product(param.window_bounds) > 128) {
-      // To avoid numerical issues, force the reducer to be kMax for large bf16
-      // windows.
-      reducer = kMax;
-    }
+
+    // To avoid numerical issues, force the reducer to be kMax for bf16
+    // inputs.
+    reducer = kMax;
   }
 
   XlaOp parameter = Parameter(&b, 0, input_literal.shape(), "input");
-- 
GitLab


From 32f4097f8ac09039c010e59200f4f0e056990fcc Mon Sep 17 00:00:00 2001
From: Peter Ma <pcma@google.com>
Date: Fri, 11 Jan 2019 18:08:53 -0800
Subject: [PATCH 0640/1095] Modify UpdateMerge() so that it passes
 input/output_handle_shapes_and_types correctly.

PiperOrigin-RevId: 228976515
---
 .../core/grappler/costs/graph_properties.cc   |   9 +-
 .../grappler/costs/graph_properties_test.cc   |  38 +++
 .../while_loop_var_handle_op.pbtxt            | 291 ++++++++++++++++++
 3 files changed, 337 insertions(+), 1 deletion(-)
 create mode 100644 tensorflow/core/grappler/costs/graph_properties_testdata/while_loop_var_handle_op.pbtxt

diff --git a/tensorflow/core/grappler/costs/graph_properties.cc b/tensorflow/core/grappler/costs/graph_properties.cc
index 9f5966c4af..59696f39f8 100644
--- a/tensorflow/core/grappler/costs/graph_properties.cc
+++ b/tensorflow/core/grappler/costs/graph_properties.cc
@@ -1680,6 +1680,7 @@ Status GraphProperties::UpdateMerge(SymbolicShapeRefiner* shape_refiner,
   }
 
   ShapeHandle out;
+  const std::vector<ShapeAndType>* out_handle = nullptr;
   bool out_initialized = false;
   for (const GraphView::Edge fanin : shape_refiner->graph().GetFaninEdges(
            *node, /*include_controlling_edges=*/false)) {
@@ -1690,18 +1691,24 @@ Status GraphProperties::UpdateMerge(SymbolicShapeRefiner* shape_refiner,
       continue;
     }
     ShapeHandle input = src_ic->output(fanin.src.port_id);
-    CHECK_EQ(fanin.dst.node, node);
     ic->SetInput(fanin.dst.port_id, input);
+    auto* input_handle =
+        src_ic->output_handle_shapes_and_types(fanin.src.port_id);
+    if (input_handle)
+      ic->set_input_handle_shapes_and_types(fanin.dst.port_id, *input_handle);
     if (!out_initialized) {
       out_initialized = true;
       out = input;
+      out_handle = input_handle;
     } else {
+      // Note here only out, not out_handle, is modified.
       out = shape_refiner->OutputAsUnion(node, 0, input, out);
     }
   }
 
   if (*new_shapes || !shape_refiner->EquivalentShapes(out, ic->output(0))) {
     ic->set_output(0, out);
+    if (out_handle) ic->set_output_handle_shapes_and_types(0, *out_handle);
     *new_shapes = true;
   }
 
diff --git a/tensorflow/core/grappler/costs/graph_properties_test.cc b/tensorflow/core/grappler/costs/graph_properties_test.cc
index c7f53b2cde..0a7697a213 100644
--- a/tensorflow/core/grappler/costs/graph_properties_test.cc
+++ b/tensorflow/core/grappler/costs/graph_properties_test.cc
@@ -341,6 +341,44 @@ TEST_F(GraphPropertiesTest, VarHandles) {
   EXPECT_EQ(7, prop.shape().dim(1).size());
 }
 
+TEST_F(GraphPropertiesTest, WhileLoopWithVarHandleOpInput) {
+  // Test graph is first generated in python using:
+  /*
+    i0 = tf.constant(0)
+    v = tf.get_variable(initializer=i0, name='loop_var', use_resource=True)
+    def cond(i, x):
+      return i < 3
+    def body(i, x):
+      return i + 1, x + x
+    v, y = tf.while_loop(cond, body, loop_vars=[v, tf.constant(1)])
+  */
+  // and then modified by hand such that the ReadVariableOp is inside the loop
+  // body instead of outside the while loop (which is the case when constructed
+  // using the python API), such that we have the following pattern: VarHandleOp
+  // -> Enter -> Switch -> ReadVariableOp -> other parts of loop body. Note
+  // DT_RESOURCE is passed all the way until ReadVariableOp.
+  GrapplerItem item;
+  string filename = io::JoinPath(testing::TensorFlowSrcRoot(), kTestDataPath,
+                                 "while_loop_var_handle_op.pbtxt");
+  TF_CHECK_OK(ReadGraphDefFromFile(filename, &item.graph));
+  GraphProperties properties(item);
+  TF_CHECK_OK(properties.InferStatically(false));
+
+  std::vector<string> resource_nodes{
+      "loop_var",       "while/Enter",         "while/Merge", "while/Switch",
+      "while/Identity", "while/NextIteration", "while/Exit"};
+  for (const string& node : resource_nodes) {
+    const auto props = properties.GetOutputProperties(node);
+    EXPECT_GE(props.size(), 1);  // Merge has 2 outputs.
+    EXPECT_EQ("resource: []", PropToString(props[0]));
+  }
+
+  // After ReadVariableOp, the shape should be recovered.
+  const auto props = properties.GetOutputProperties("while/ReadVariableOp");
+  EXPECT_EQ(1, props.size());
+  EXPECT_EQ("int32: []", PropToString(props[0]));
+}
+
 TEST_F(GraphPropertiesTest, QueueWithOnlyDequeue_NoShapeAttr) {
   tensorflow::Scope root = tensorflow::Scope::NewRootScope();
   auto q1 = ops::FIFOQueue(root.WithOpName("Queue1"), {DataType::DT_FLOAT});
diff --git a/tensorflow/core/grappler/costs/graph_properties_testdata/while_loop_var_handle_op.pbtxt b/tensorflow/core/grappler/costs/graph_properties_testdata/while_loop_var_handle_op.pbtxt
new file mode 100644
index 0000000000..048b8a23dc
--- /dev/null
+++ b/tensorflow/core/grappler/costs/graph_properties_testdata/while_loop_var_handle_op.pbtxt
@@ -0,0 +1,291 @@
+node {
+  name: "Const"
+  op: "Const"
+  attr {
+    key: "dtype"
+    value { type: DT_INT32 }
+  }
+  attr {
+    key: "value"
+    value {
+      tensor {
+        dtype: DT_INT32
+        tensor_shape {}
+        int_val: 0
+      }
+    }
+  }
+}
+node {
+  name: "loop_var"
+  op: "VarHandleOp"
+  attr {
+    key: "_class"
+    value { list { s: "loc:@loop_var" } }
+  }
+  attr {
+    key: "container"
+    value { s: "" }
+  }
+  attr {
+    key: "dtype"
+    value { type: DT_INT32 }
+  }
+  attr {
+    key: "shape"
+    value { shape {} }
+  }
+  attr {
+    key: "shared_name"
+    value { s: "loop_var" }
+  }
+}
+node {
+  name: "Const_1"
+  op: "Const"
+  attr {
+    key: "dtype"
+    value { type: DT_INT32 }
+  }
+  attr {
+    key: "value"
+    value {
+      tensor {
+        dtype: DT_INT32
+        tensor_shape {}
+        int_val: 1
+      }
+    }
+  }
+}
+node {
+  name: "while/Enter"
+  op: "Enter"
+  input: "loop_var"
+  attr {
+    key: "T"
+    value { type: DT_RESOURCE }
+  }
+  attr {
+    key: "frame_name"
+    value { s: "while/while_context" }
+  }
+  attr {
+    key: "is_constant"
+    value { b: false }
+  }
+  attr {
+    key: "parallel_iterations"
+    value { i: 10 }
+  }
+}
+node {
+  name: "while/Enter_1"
+  op: "Enter"
+  input: "Const_1"
+  attr {
+    key: "T"
+    value { type: DT_INT32 }
+  }
+  attr {
+    key: "frame_name"
+    value { s: "while/while_context" }
+  }
+  attr {
+    key: "is_constant"
+    value { b: false }
+  }
+  attr {
+    key: "parallel_iterations"
+    value { i: 10 }
+  }
+}
+node {
+  name: "while/Merge"
+  op: "Merge"
+  input: "while/Enter"
+  input: "while/NextIteration"
+  attr {
+    key: "N"
+    value { i: 2 }
+  }
+  attr {
+    key: "T"
+    value { type: DT_RESOURCE }
+  }
+}
+node {
+  name: "while/Merge_1"
+  op: "Merge"
+  input: "while/Enter_1"
+  input: "while/NextIteration_1"
+  attr {
+    key: "N"
+    value { i: 2 }
+  }
+  attr {
+    key: "T"
+    value { type: DT_INT32 }
+  }
+}
+node {
+  name: "while/Less/y"
+  op: "Const"
+  input: "^while/Merge"
+  attr {
+    key: "dtype"
+    value { type: DT_INT32 }
+  }
+  attr {
+    key: "value"
+    value {
+      tensor {
+        dtype: DT_INT32
+        tensor_shape {}
+        int_val: 3
+      }
+    }
+  }
+}
+node {
+  name: "while/Less"
+  op: "Less"
+  input: "while/Merge"
+  input: "while/Less/y"
+  attr {
+    key: "T"
+    value { type: DT_INT32 }
+  }
+}
+node { name: "while/LoopCond" op: "LoopCond" input: "while/Less" }
+node {
+  name: "while/Switch"
+  op: "Switch"
+  input: "while/Merge"
+  input: "while/LoopCond"
+  attr {
+    key: "T"
+    value { type: DT_RESOURCE }
+  }
+  attr {
+    key: "_class"
+    value { list { s: "loc:@while/Merge" } }
+  }
+}
+node {
+  name: "while/Switch_1"
+  op: "Switch"
+  input: "while/Merge_1"
+  input: "while/LoopCond"
+  attr {
+    key: "T"
+    value { type: DT_INT32 }
+  }
+  attr {
+    key: "_class"
+    value { list { s: "loc:@while/Merge_1" } }
+  }
+}
+node {
+  name: "while/Identity"
+  op: "Identity"
+  input: "while/Switch:1"
+  attr {
+    key: "T"
+    value { type: DT_RESOURCE }
+  }
+}
+node {
+  name: "while/ReadVariableOp"
+  op: "ReadVariableOp"
+  input: "while/Identity"
+  attr {
+    key: "dtype"
+    value { type: DT_INT32 }
+  }
+}
+node {
+  name: "while/Identity_1"
+  op: "Identity"
+  input: "while/Switch_1:1"
+  attr {
+    key: "T"
+    value { type: DT_INT32 }
+  }
+}
+node {
+  name: "while/add/y"
+  op: "Const"
+  input: "^while/ReadVariableOp"
+  attr {
+    key: "dtype"
+    value { type: DT_INT32 }
+  }
+  attr {
+    key: "value"
+    value {
+      tensor {
+        dtype: DT_INT32
+        tensor_shape {}
+        int_val: 1
+      }
+    }
+  }
+}
+node {
+  name: "while/add"
+  op: "Add"
+  input: "while/ReadVariableOp"
+  input: "while/add/y"
+  attr {
+    key: "T"
+    value { type: DT_INT32 }
+  }
+}
+node {
+  name: "while/add_1"
+  op: "Add"
+  input: "while/Identity_1"
+  input: "while/Identity_1"
+  attr {
+    key: "T"
+    value { type: DT_INT32 }
+  }
+}
+node {
+  name: "while/NextIteration"
+  op: "NextIteration"
+  input: "while/Identity"
+  attr {
+    key: "T"
+    value { type: DT_RESOURCE }
+  }
+}
+node {
+  name: "while/NextIteration_1"
+  op: "NextIteration"
+  input: "while/add_1"
+  attr {
+    key: "T"
+    value { type: DT_INT32 }
+  }
+}
+node {
+  name: "while/Exit"
+  op: "Exit"
+  input: "while/Switch"
+  attr {
+    key: "T"
+    value { type: DT_RESOURCE }
+  }
+}
+node {
+  name: "while/Exit_1"
+  op: "Exit"
+  input: "while/Switch_1"
+  attr {
+    key: "T"
+    value { type: DT_INT32 }
+  }
+}
+versions { producer: 27 }
-- 
GitLab


From aca318aaee0944c8305feb8c57ed888c718973ed Mon Sep 17 00:00:00 2001
From: Igor Ganichev <iga@google.com>
Date: Fri, 11 Jan 2019 18:09:15 -0800
Subject: [PATCH 0641/1095] Call multi-device functions natively from eager
 runtime

PiperOrigin-RevId: 228976541
---
 tensorflow/compiler/tests/eager_test.py       |  12 +-
 tensorflow/core/common_runtime/eager/BUILD    |   5 +
 .../core/common_runtime/eager/attr_builder.cc |   2 +-
 .../common_runtime/eager/attr_builder_test.cc |   4 +-
 .../core/common_runtime/eager/context.h       |  12 +-
 .../eager/copy_to_device_node.h               |   2 +-
 .../core/common_runtime/eager/execute.cc      | 340 ++++++++++++------
 .../core/common_runtime/eager/execute.h       |  12 +-
 .../core/common_runtime/eager/execute_node.h  |   2 +-
 .../common_runtime/eager/kernel_and_device.cc | 243 ++++++++++---
 .../common_runtime/eager/kernel_and_device.h  | 180 ++++++++--
 .../eager/kernel_and_device_test.cc           |  38 +-
 .../common_runtime/eager/tensor_handle.cc     |  67 ++++
 .../core/common_runtime/eager/tensor_handle.h |  55 +--
 tensorflow/python/eager/function.py           |  68 ++--
 tensorflow/python/eager/function_test.py      | 265 ++++++++++++++
 tensorflow/python/eager/pywrap_tfe_test.py    |   5 +
 tensorflow/python/ops/functional_ops.py       |  18 +-
 tensorflow/python/util/function_utils.py      |  14 +
 19 files changed, 1044 insertions(+), 300 deletions(-)

diff --git a/tensorflow/compiler/tests/eager_test.py b/tensorflow/compiler/tests/eager_test.py
index 2af32b537b..c36c12a0e3 100644
--- a/tensorflow/compiler/tests/eager_test.py
+++ b/tensorflow/compiler/tests/eager_test.py
@@ -463,7 +463,7 @@ class EagerFunctionTest(xla_test.XLATestCase):
       def f(x, y):
         return x[0::2, y:, ...]
 
-      x = array_ops.ones([2, 3, 4])
+      x = array_ops.ones([2, 3, 4], dtype=dtypes.float32)
       y = array_ops.ones([], dtype=dtypes.int32)
       with backprop.GradientTape() as tape:
         tape.watch(x)
@@ -479,15 +479,15 @@ class EagerFunctionTest(xla_test.XLATestCase):
 
       @function.defun
       def times_two(x):
-        return 2 * x
+        return 2. * x
 
       @function.defun
       def two_x_plus_1(x):
-        return times_two(x) + 1
+        return times_two(x) + 1.
 
-      x = constant_op.constant([2, 3, 4])
+      x = constant_op.constant([2., 3., 4.])
       y = two_x_plus_1(x)
-      self.assertAllEqual([5, 7, 9], y.numpy())
+      self.assertAllEqual([5., 7., 9.], y.numpy())
 
   def testNestedDefunWithVariable(self):
     with self.test_scope():
@@ -506,7 +506,7 @@ class EagerFunctionTest(xla_test.XLATestCase):
       x = constant_op.constant(3.0)
       y = f(x)
 
-    self.assertEqual(75, y.numpy())
+    self.assertEqual(75.0, y.numpy())
 
   def testNestedDefunInGradientTape(self):
     with self.test_scope():
diff --git a/tensorflow/core/common_runtime/eager/BUILD b/tensorflow/core/common_runtime/eager/BUILD
index 77e3246df0..06cbe8049f 100644
--- a/tensorflow/core/common_runtime/eager/BUILD
+++ b/tensorflow/core/common_runtime/eager/BUILD
@@ -165,6 +165,7 @@ tf_cuda_library(
             "//tensorflow/core:lib",
             "//tensorflow/core:lib_internal",
             "//tensorflow/core:protos_all_cc",
+            "//tensorflow/core/grappler/optimizers:meta_optimizer",
         ],
     }),
 )
@@ -181,10 +182,14 @@ tf_cc_test(
         "//tensorflow/cc:scope",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:core_cpu_internal",
+        "//tensorflow/core:direct_session",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
+        "//tensorflow/core/kernels:constant_op",
+        "//tensorflow/core/kernels:matmul_op",
         "@com_google_absl//absl/memory",
     ],
 )
diff --git a/tensorflow/core/common_runtime/eager/attr_builder.cc b/tensorflow/core/common_runtime/eager/attr_builder.cc
index 689b04274f..77be4c951e 100644
--- a/tensorflow/core/common_runtime/eager/attr_builder.cc
+++ b/tensorflow/core/common_runtime/eager/attr_builder.cc
@@ -42,7 +42,7 @@ const uint32 kIsList = 1U << 31;
 AttrTypeMap* DefaultFunctionAttrTypeMap() {
   AttrTypeMap* map = new AttrTypeMap();
   (*map)["executor_type"] = TF_ATTR_STRING;
-  (*map)["config"] = TF_ATTR_STRING;
+  (*map)["config_proto"] = TF_ATTR_STRING;
   return map;
 }
 
diff --git a/tensorflow/core/common_runtime/eager/attr_builder_test.cc b/tensorflow/core/common_runtime/eager/attr_builder_test.cc
index 8245660bfc..31c998a670 100644
--- a/tensorflow/core/common_runtime/eager/attr_builder_test.cc
+++ b/tensorflow/core/common_runtime/eager/attr_builder_test.cc
@@ -41,8 +41,10 @@ TEST(AttrTypeMap, Lookup) {
   Status s = AttrTypeMapForOp("SomeFunctionName", &m, &is_function);
   EXPECT_TRUE(s.ok());
   EXPECT_TRUE(is_function);
+  ASSERT_NE(m->end(), m->find("executor_type"));
   EXPECT_EQ(TF_ATTR_STRING, m->find("executor_type")->second);
-  EXPECT_EQ(TF_ATTR_STRING, m->find("config")->second);
+  ASSERT_NE(m->end(), m->find("config_proto"));
+  EXPECT_EQ(TF_ATTR_STRING, m->find("config_proto")->second);
 
   is_function = true;
   s = AttrTypeMapForOp("MatMul", &m, &is_function);
diff --git a/tensorflow/core/common_runtime/eager/context.h b/tensorflow/core/common_runtime/eager/context.h
index b83f5707da..d8807cf839 100644
--- a/tensorflow/core/common_runtime/eager/context.h
+++ b/tensorflow/core/common_runtime/eager/context.h
@@ -93,6 +93,8 @@ class EagerContext {
     return pflr_->GetFLR(d->name());
   }
 
+  ProcessFunctionLibraryRuntime* pflr() const { return pflr_.get(); }
+
   // True if running in asynchronous mode.
   bool Async() const;
 
@@ -138,7 +140,7 @@ class EagerContext {
 
   Status FindDeviceByName(const string& name, Device** result);
 
-  Device* HostCPU() { return devices_[0]; }
+  Device* HostCPU() const { return devices_[0]; }
 
   GraphCollector* GetGraphCollector() { return &graph_collector_; }
 
@@ -152,10 +154,10 @@ class EagerContext {
 
   void AddKernelToCache(Fprint128 cache_key, KernelAndDevice* kernel);
 
-  bool LogDevicePlacement() { return log_device_placement_; }
-  bool LogMemory() { return log_memory_; }
+  bool LogDevicePlacement() const { return log_device_placement_; }
+  bool LogMemory() const { return log_memory_; }
 
-  Rendezvous* GetRendezvous() { return rendezvous_; }
+  Rendezvous* GetRendezvous() const { return rendezvous_; }
   CollectiveExecutorMgrInterface* collective_executor_mgr() {
     return (collective_executor_mgr_ != nullptr)
                ? collective_executor_mgr_.get()
@@ -171,7 +173,7 @@ class EagerContext {
     return (local_device_manager_ != nullptr) ? local_device_manager_.get()
                                               : local_unowned_device_manager_;
   }
-  const tensorflow::DeviceMgr* remote_device_mgr() {
+  const tensorflow::DeviceMgr* remote_device_mgr() const {
     return remote_device_manager_.get();
   }
 
diff --git a/tensorflow/core/common_runtime/eager/copy_to_device_node.h b/tensorflow/core/common_runtime/eager/copy_to_device_node.h
index 5bc3bb689e..a807e7f68d 100644
--- a/tensorflow/core/common_runtime/eager/copy_to_device_node.h
+++ b/tensorflow/core/common_runtime/eager/copy_to_device_node.h
@@ -30,7 +30,7 @@ class CopyToDeviceNode : public EagerNode {
         src_(src),
         dstd_(dstd),
         ctx_(ctx),
-        dst_(new TensorHandle(id, dstd_, dstd_, src->dtype, ctx)) {
+        dst_(new TensorHandle(id, dstd_, dstd_, nullptr, src->dtype, ctx)) {
     src_->Ref();
     dst_->Ref();
   }
diff --git a/tensorflow/core/common_runtime/eager/execute.cc b/tensorflow/core/common_runtime/eager/execute.cc
index 9d2401929d..9df1511bc7 100644
--- a/tensorflow/core/common_runtime/eager/execute.cc
+++ b/tensorflow/core/common_runtime/eager/execute.cc
@@ -24,6 +24,9 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/eager/execute_node.h"
 #include "tensorflow/core/common_runtime/eager/kernel_and_device.h"
 #include "tensorflow/core/common_runtime/eager/tensor_handle.h"
+#include "tensorflow/core/framework/function.h"
+#include "tensorflow/core/framework/node_def_util.h"
+#include "tensorflow/core/framework/types.pb.h"
 #include "tensorflow/core/lib/core/errors.h"
 #ifndef __ANDROID__
 #include "tensorflow/core/distributed_runtime/eager/eager_client.h"
@@ -81,18 +84,20 @@ int StepStatsDeviceIndex(StepStats* step_stats, EagerContext* ctx,
 // tensor handle.
 //
 // The passed in *handle will be Unreffed if it is replaced.
-Status MaybeCopyInputToExpectedDevice(EagerOperation* op, int i,
-                                      const Device* expected_device,
+//
+// `op_device` is passed in explicitly because `op->device()` might be unset
+// and we might have selected some specific device to run this op on.
+Status MaybeCopyInputToExpectedDevice(EagerOperation* op,
+                                      const Device* op_device, int i,
+                                      const Device* expected_input_device,
                                       RunMetadata* run_metadata,
                                       TensorHandle** handle) {
   EagerContext* ctx = op->EagerContext();
   Device* handle_device = (*handle)->device();
   const Device* actual_device =
       handle_device == nullptr ? ctx->HostCPU() : handle_device;
-  const Device* op_device =
-      op->Device() == nullptr ? ctx->HostCPU() : op->Device();
 
-  if (expected_device != actual_device) {
+  if (expected_input_device != actual_device) {
     switch (ctx->GetDevicePlacementPolicy()) {
       case DEVICE_PLACEMENT_SILENT_FOR_INT32:
         // TODO(xpan): See if we could bubble python related error up
@@ -108,7 +113,7 @@ Status MaybeCopyInputToExpectedDevice(EagerOperation* op, int i,
             "Tensors on conflicting devices:"
             " cannot compute ",
             op->Name(), " as input #", i, " was expected to be on ",
-            expected_device->name(), " but is actually on ",
+            expected_input_device->name(), " but is actually on ",
             actual_device->name(), " (operation running on ", op_device->name(),
             ")",
             " Tensors can be copied explicitly using .gpu() or .cpu() "
@@ -119,9 +124,10 @@ Status MaybeCopyInputToExpectedDevice(EagerOperation* op, int i,
             " may slow down your model");
       case DEVICE_PLACEMENT_WARN:
         LOG(WARNING) << "before computing " << op->Name() << " input #" << i
-                     << " was expected to be on " << expected_device->name()
-                     << " but is actually on " << actual_device->name()
-                     << " (operation running on " << op_device->name()
+                     << " was expected to be on "
+                     << expected_input_device->name() << " but is actually on "
+                     << actual_device->name() << " (operation running on "
+                     << op_device->name()
                      << "). This triggers a copy which can be a performance "
                         "bottleneck.";
         break;
@@ -133,7 +139,7 @@ Status MaybeCopyInputToExpectedDevice(EagerOperation* op, int i,
     auto pre_time_nanos = Env::Default()->NowNanos();
     TensorHandle* result_handle = nullptr;
     Status status = EagerCopyToDevice(
-        *handle, ctx, expected_device->name().c_str(), &result_handle);
+        *handle, ctx, expected_input_device->name().c_str(), &result_handle);
     if (run_metadata != nullptr) {
       auto* step_stats = run_metadata->mutable_step_stats();
       MaybeInitializeStepStats(step_stats, ctx);
@@ -155,10 +161,10 @@ Status MaybeCopyInputToExpectedDevice(EagerOperation* op, int i,
     }
     if (!status.ok()) {
       if (result_handle != nullptr) result_handle->Unref();
-      return errors::Internal("Failed copying input tensor from ",
-                              actual_device->name(), " to ",
-                              expected_device->name(), " in order to run ",
-                              op->Name(), ": ", status.error_message());
+      return errors::Internal(
+          "Failed copying input tensor from ", actual_device->name(), " to ",
+          expected_input_device->name(), " in order to run ", op->Name(), ": ",
+          status.error_message());
     }
 
     (*handle)->Unref();
@@ -168,19 +174,18 @@ Status MaybeCopyInputToExpectedDevice(EagerOperation* op, int i,
 }
 
 Status ValidateInputTypeAndPlacement(EagerContext* ctx, Device* op_device,
-                                     EagerOperation* op, const OpKernel* kernel,
+                                     EagerOperation* op,
+                                     const KernelAndDevice* kernel,
                                      RunMetadata* run_metadata) {
-  Device* host_device = ctx->HostCPU();
-  const MemoryTypeVector& memtypes = kernel->input_memory_types();
-  if (memtypes.size() != op->Inputs().size()) {
-    return errors::InvalidArgument("expected ", memtypes.size(),
+  if (kernel->num_inputs() != op->Inputs().size()) {
+    return errors::InvalidArgument("expected ", kernel->num_inputs(),
                                    " inputs, got ", op->Inputs().size());
   }
   for (int i = 0; i < op->Inputs().size(); ++i) {
-    const Device* expected_device =
-        memtypes[i] == HOST_MEMORY ? host_device : op_device;
+    const Device* expected_device = kernel->InputDevice(i);
     TF_RETURN_IF_ERROR(MaybeCopyInputToExpectedDevice(
-        op, i, expected_device, run_metadata, &((*op->MutableInputs())[i])));
+        op, op_device, i, expected_device, run_metadata,
+        &((*op->MutableInputs())[i])));
     tensorflow::TensorHandle* handle = op->Inputs()[i];
     if (handle->dtype != kernel->input_type(i)) {
       return errors::InvalidArgument(
@@ -246,6 +251,89 @@ bool OnSameTask(EagerContext* ctx, Device* first, Device* second) {
          first->parsed_name().task == second->parsed_name().task;
 }
 
+inline tensorflow::Fprint128 FingerprintCat128(const tensorflow::Fprint128& a,
+                                               const tensorflow::Fprint128& b) {
+  return {tensorflow::FingerprintCat64(a.low64, b.low64),
+          tensorflow::FingerprintCat64(a.high64, b.high64)};
+}
+
+Status FindDeviceFromName(const EagerContext* ctx, const char* device_name,
+                          Device** device) {
+  *device = ctx->HostCPU();
+  if (device_name == nullptr || strlen(device_name) == 0) {
+    return Status::OK();
+  }
+
+  auto status = ctx->local_device_mgr()->LookupDevice(device_name, device);
+  if (status.ok()) {
+    return status;
+  }
+
+  if (ctx->remote_device_mgr() != nullptr) {
+    return ctx->remote_device_mgr()->LookupDevice(device_name, device);
+  }
+
+  return status;
+}
+
+bool IsMultiDevice(const FunctionDef* fdef, const string& op_device) {
+  if (fdef == nullptr) {
+    // Primitive op.
+    return false;
+  }
+
+  // Run all functions as multi-device.
+  return true;
+
+  // We can eliminate some overhead by running simple functions using regular
+  // CallOp kernel. However, it is tricky to figure out which functions should
+  // be run using CallOp. Also, currently CallOp runs neither optimization
+  // passes (needed for TPU/XLA) nor grappler.
+  // Here are some cases where a function should be run in multi-device mode:
+  //  - Function takes at least two resources on different devices.
+  //  - Function takes a resource on deviceA and a body op explicitly placed
+  //  on deviceB.
+  //  - Function has a colocation constraint.
+  //  - Function has an explicit device annotation (which might not be using
+  //    full canonical device name) different from op_device. Note that false
+  //    positives are ok.
+  //  - Function has a node or a (node) attribute that can potentially make
+  //    the function multi-device after a rewrite pass (e.g. various XLA/TPU
+  //    special nodes and attributes)
+}
+
+Status AddInputDevicesToCacheKey(const EagerContext* ctx,
+                                 const EagerOperation* op,
+                                 std::vector<Device*>* input_dev_ptrs,
+                                 Fprint128* cache_key) {
+  input_dev_ptrs->reserve(op->Inputs().size());
+  Device* cpu_device = ctx->HostCPU();
+  for (TensorHandle* tensor_handle : op->Inputs()) {
+    string device_name;
+    if (tensor_handle->dtype == DT_RESOURCE) {
+      // Use the resource's actual device because it is the device that will
+      // influence partitioning the multi-device function.
+      const Tensor* tensor;
+      TF_RETURN_IF_ERROR(tensor_handle->Tensor(&tensor));
+      const ResourceHandle& handle = tensor->flat<ResourceHandle>()(0);
+      device_name = handle.device();
+
+      Device* input_device;
+      TF_RETURN_IF_ERROR(
+          FindDeviceFromName(ctx, device_name.c_str(), &input_device));
+      input_dev_ptrs->push_back(input_device);
+    } else if (MTypeFromDType(tensor_handle->dtype) == HOST_MEMORY) {
+      input_dev_ptrs->push_back(cpu_device);
+    } else {
+      Device* device = tensor_handle->device();
+      device_name = device != nullptr ? device->name() : cpu_device->name();
+      input_dev_ptrs->push_back(device == nullptr ? cpu_device : device);
+    }
+    *cache_key = FingerprintCat128(*cache_key, Fingerprint128(device_name));
+  }
+  return Status::OK();
+}
+
 Status EagerLocalExecute(EagerOperation* op,
                          gtl::InlinedVector<TensorHandle*, 2>* retvals,
                          int* num_retvals) {
@@ -254,18 +342,34 @@ Status EagerLocalExecute(EagerOperation* op,
   if (!status.ok()) return status;
   Device* device = op->Device();
 
-  Fprint128 cache_key = op->MutableAttrs()->CacheKey(
-      device == nullptr ? "unspecified" : device->name());
+  const string& maybe_unspecified_device_name =
+      device == nullptr ? "unspecified" : device->name();
+  Fprint128 cache_key =
+      op->MutableAttrs()->CacheKey(maybe_unspecified_device_name);
+
+  bool is_multi_device_function = IsMultiDevice(
+      ctx->FindFunctionDef(op->Name()), maybe_unspecified_device_name);
+
+  std::vector<Device*> input_dev_ptrs;
+  if (is_multi_device_function) {
+    TF_RETURN_IF_ERROR(
+        AddInputDevicesToCacheKey(ctx, op, &input_dev_ptrs, &cache_key));
+  }
+
   KernelAndDevice* kernel = ctx->GetCachedKernel(cache_key);
   if (kernel == nullptr) {
+    VLOG(2) << "Creating new kernel for " << op->Name() << " on device "
+            << maybe_unspecified_device_name;
     // If we are running a function on explicitly requested TPU,
     // compile it with XLA.
     // Note that it is not ideal, but currently ok, to set this
     // attribute after computing the kernel cache key above.
+    bool compile_with_xla = false;
     if (op->is_function() && device != nullptr &&
         (device->device_type() == "TPU" || device->device_type() == "XLA_GPU" ||
          device->device_type() == "XLA_CPU")) {
       op->MutableAttrs()->Set(kXlaCompileAttr, true);
+      compile_with_xla = true;
     }
 
     const NodeDef& ndef = op->MutableAttrs()->BuildNodeDef();
@@ -273,7 +377,6 @@ Status EagerLocalExecute(EagerOperation* op,
       status = SelectDevice(ndef, ctx, &device);
       if (!status.ok()) return status;
     }
-    CHECK(device != nullptr);
     if (ctx->LogDevicePlacement()) {
       LOG(INFO) << "Executing op " << ndef.op() << " in device "
                 << device->name();
@@ -285,9 +388,35 @@ Status EagerLocalExecute(EagerOperation* op,
           "Unable to find a FunctionLibraryRuntime corresponding to device ",
           device->name());
     }
-    kernel = new KernelAndDevice(ctx->GetRendezvous(), ctx->LogMemory(),
-                                 ctx->GetCollectiveExecutorHandle());
-    status = KernelAndDevice::Init(ndef, flr, ctx->runner(), kernel);
+    GraphCollector* graph_collector = nullptr;
+    if (ctx->ShouldStoreMetadata()) {
+      graph_collector = ctx->GetGraphCollector();
+    }
+    // Treat the function as multi_device only when we are not compiling
+    // it wholly with XLA. When compiling wholly with XLA, flr->CreateKernel
+    // will create an XlaLaunchOp kernel to compile and run the function.
+    if (is_multi_device_function && !compile_with_xla) {
+      // Multi-device functions don't use the rendezvous from eager context.
+      // If we use that rendezvous, multiple concurrent calls to the same
+      // function will likely result in collisions. However, this also means
+      // that we don't support legitimate sending/receiving across function
+      // boundary.
+      VLOG(2) << "Running " << ndef.op() << " using multi-device function. "
+              << "compile_with_xla=" << compile_with_xla
+              << ". Full node_def=" << ndef.DebugString();
+      kernel = new KernelAndDeviceFunc(
+          flr, ctx->pflr(), std::move(input_dev_ptrs), ctx->runner(),
+          ctx->GetCollectiveExecutorHandle(), ctx->HostCPU());
+    } else {
+      VLOG(2) << "Running " << ndef.op() << " using op kernel. "
+              << "compile_with_xla=" << compile_with_xla
+              << ". Full node_def=" << ndef.DebugString();
+      kernel = new KernelAndDeviceOp(
+          ctx->GetRendezvous(), ctx->LogMemory(), flr, ctx->runner(),
+          ctx->GetCollectiveExecutorHandle(), ctx->HostCPU());
+    }
+
+    status = kernel->Init(ndef, graph_collector);
     if (!status.ok()) {
       delete kernel;
       return status;
@@ -309,7 +438,7 @@ Status EagerLocalExecute(EagerOperation* op,
     device = kernel->device();
   }
   status = ValidateInputTypeAndPlacement(
-      ctx, device, op, kernel->kernel(),
+      ctx, device, op, kernel,
       ctx->ShouldStoreMetadata() ? ctx->RunMetadataProto() : nullptr);
   if (!status.ok()) return status;
   std::unique_ptr<NodeExecStats> maybe_stats;
@@ -333,12 +462,15 @@ Status EagerLocalExecute(EagerOperation* op,
   if (ctx->Async()) {
     // Note that for async mode, execution order will make sure that all
     // input handles are ready before executing them.
-    // TODO(agarwal): Consider executing "cheap" kernels inline for performance.
+    // TODO(agarwal): Consider executing "cheap" kernels inline for
+    // performance.
     tensorflow::uint64 id = ctx->NextId();
     for (int i = 0; i < *num_retvals; ++i) {
-      (*retvals)[i] = new TensorHandle(id, /* d= */ kernel->OutputDevice(i),
-                                       /* op_device= */ kernel->device(),
-                                       output_dtypes[i], ctx);
+      (*retvals)[i] = new TensorHandle(
+          id, /* d= */ kernel->OutputDevice(i),
+          /* op_device= */ kernel->device(),
+          /* resource_device= */ kernel->OutputResourceDevice(i),
+          output_dtypes[i], ctx);
     }
     EagerNode* node = new ExecuteNode(
         id, ctx, op->Device(), op->Inputs(), kernel, maybe_stats.release(),
@@ -347,9 +479,9 @@ Status EagerLocalExecute(EagerOperation* op,
   } else {
     // Execute checks if retvals[i] is nullptr or not to figure if it needs to
     // allocate it.
-    status = EagerExecute(ctx, op->Device(), op->Inputs(), kernel,
-                          maybe_stats.get(), maybe_step_stats, graph_collector,
-                          retvals->data(), *num_retvals);
+    status = EagerKernelExecute(ctx, op->Device(), op->Inputs(), kernel,
+                                maybe_stats.get(), maybe_step_stats,
+                                graph_collector, retvals->data(), *num_retvals);
   }
 
   return status;
@@ -361,9 +493,9 @@ std::function<void()> GetRemoteTensorDestructor(
     uint64 op_id, int output_num) {
   return [ctx, eager_client, context_id, op_id, output_num]() {
     if (!ctx->HasActiveRemoteContext(context_id)) {
-      // This means that this tensor was pointing to a remote device, which has
-      // been changed out from under us. Simply return since there is nothing we
-      // can do.
+      // This means that this tensor was pointing to a remote device, which
+      // has been changed out from under us. Simply return since there is
+      // nothing we can do.
       return tensorflow::Status::OK();
     }
 
@@ -423,10 +555,10 @@ Status EagerRemoteSendTensor(EagerContext* ctx, TensorHandle* h,
 
   Device* tensor_handle_device = h->device();
 
-  // AsProtoTensorContent doesn't work when the tensor is on the GPU, hence copy
-  // it to the CPU before copying it out.
-  // TODO(nareshmodi): this is currently slow, but can be fixed by making tensor
-  // handles aware of more than one device.
+  // AsProtoTensorContent doesn't work when the tensor is on the GPU, hence
+  // copy it to the CPU before copying it out.
+  // TODO(nareshmodi): this is currently slow, but can be fixed by making
+  // tensor handles aware of more than one device.
   TensorHandle* actual_handle;
   if (tensor_handle_device != nullptr &&
       tensor_handle_device->device_type() != "CPU") {
@@ -458,7 +590,8 @@ Status EagerRemoteSendTensor(EagerContext* ctx, TensorHandle* h,
 
   *result = new TensorHandle(id, /*output_num=*/0, /*remote_shape_node_id=*/0,
                              tensor->dtype(), std::move(destructor),
-                             recv_device, recv_device, ctx);
+                             /*d=*/recv_device, /*op_device=*/recv_device,
+                             /*resource_device=*/nullptr, ctx);
   (*result)->SetRemoteShape(MakeUnique<TensorShape>(tensor->shape()));
 
   actual_handle->Unref();
@@ -497,7 +630,7 @@ Status EagerRemoteExecute(EagerOperation* op, TensorHandle** retvals,
       // TODO(b/110044833): It's possible the same tensor gets copied to the
       // remote device repeatedly.
       TF_RETURN_IF_ERROR(MaybeCopyInputToExpectedDevice(
-          op, i, op->Device(), /* run_metadata= */ nullptr,
+          op, op->Device(), i, op->Device(), /* run_metadata= */ nullptr,
           &(*op->MutableInputs())[i]));
     }
 
@@ -537,14 +670,25 @@ Status EagerRemoteExecute(EagerOperation* op, TensorHandle** retvals,
 
   const tensorflow::uint64 id = remote_op->id();
   for (int i = 0; i < *num_retvals; i++) {
-    // TODO(nareshmodi): Change the callback to instead add the decref to a list
-    // of pending decrefs that we can send as a batch with the next execute.
+    // TODO(nareshmodi): Change the callback to instead add the decref to a
+    // list of pending decrefs that we can send as a batch with the next
+    // execute.
     std::function<void()> destructor =
         GetRemoteTensorDestructor(ctx, eager_client, context_id, id, i);
 
-    retvals[i] = new TensorHandle(remote_op->id(), i, remote_node_id,
-                                  output_dtypes[i], std::move(destructor),
-                                  op_device, op_device, op->EagerContext());
+    // The device_ and resource_device_ or this TensorHandle are not correct.
+    // It is pretty hard to make it correct because for multi-device functions,
+    // we don't know the output device until the function is instantiated.
+    // Luckily, we don't need to know the correct remote device here. We just
+    // need to know that it is remote. If we need to copy this tensor to this
+    // process, the remote end will know the correct device of this handle.
+    retvals[i] = new TensorHandle(
+        remote_op->id(), i, remote_node_id, output_dtypes[i],
+        std::move(destructor),
+        /*d=*/op_device, /*op_device=*/op_device,
+        /*resource_device=*/output_dtypes[i] == DT_RESOURCE ? op_device
+                                                            : nullptr,
+        op->EagerContext());
   }
 
   if (is_async) {
@@ -619,43 +763,56 @@ bool IsPinnableOp(const string& op_type) {
 // "TF_EAGER_ENABLE_SMALL_TENSOR_CPU_PINNING" to "0" or "false".
 Status MaybeUpdateOpDevice(EagerOperation* op) {
   EagerContext* ctx = op->EagerContext();
-  bool device_set_for_resource_variable = false;
   bool all_inputs_eligible_for_cpu_pinning =
       ctx->PinSmallOpsToCPU() && IsPinnableOp(op->Name());
-
+  Device* op_device = op->Device() == nullptr ? ctx->HostCPU() : op->Device();
   for (int i = 0; i < op->Inputs().size(); ++i) {
-    Device* input_op_device = op->Inputs()[i]->op_device();
-    VLOG(2) << "for op " << op->Name() << " input " << i << " "
-            << DataTypeString(op->Inputs()[i]->dtype) << " "
-            << (input_op_device == nullptr ? "cpu" : input_op_device->name())
-            << " " << (op->Device() == nullptr ? "cpu" : op->Device()->name());
-    if (op->Inputs()[i]->dtype == DT_RESOURCE &&
-        (input_op_device != op->Device() || input_op_device == nullptr)) {
-      Device* d = input_op_device == nullptr ? ctx->HostCPU() : input_op_device;
-      VLOG(1) << "Changing device of operation " << op->Name() << " to "
-              << d->name() << " because input #" << i
-              << " is a resource in this device.";
-      op->SetDevice(d);
-
-      device_set_for_resource_variable = true;
+    TensorHandle* tensor_handle = op->Inputs()[i];
+    if (tensor_handle->dtype == DT_RESOURCE) {
+      Device* resource_device = tensor_handle->resource_device();
+      VLOG(2) << "for op " << op->Name() << " input " << i << " "
+              << DataTypeString(tensor_handle->dtype)
+              << " input device = " << resource_device->name()
+              << ", op device = " << op_device->name();
+      // We check for `op->Device() == nullptr` because it can be later
+      // interpreted as unspecified device and a different device can
+      // be selected based on device priority. If any input to an op
+      // is a resource we must pin it to prevent different device selection.
+      // TODO(iga): null device can mean "unspecified" or "CPU". Clean this up.
+      if (resource_device != op_device || op->Device() == nullptr) {
+        VLOG(1) << (resource_device != op_device ? "Changing " : "Setting ")
+                << "device of operation " << op->Name() << " to "
+                << resource_device->name() << " because input #" << i
+                << " is a resource in this device.";
+        op->SetDevice(resource_device);
+      }
       all_inputs_eligible_for_cpu_pinning = false;
+      // No point in looking at other inputs. If there are other resources,
+      // they must have the same device and we already declared the op to be
+      // ineligible for CPU pinning.
+      break;
     } else if (all_inputs_eligible_for_cpu_pinning) {
-      TensorHandle* handle = op->Inputs()[i];
+      Device* input_device = tensor_handle->device();
+      input_device = input_device == nullptr ? ctx->HostCPU() : input_device;
+      VLOG(2) << "for op " << op->Name() << " input " << i << " "
+              << DataTypeString(tensor_handle->dtype)
+              << " input device = " << input_device->name()
+              << ", op device = " << op_device->name();
 
       // Input is on CPU.
-      if (input_op_device != nullptr && input_op_device != ctx->HostCPU()) {
+      if (input_device != ctx->HostCPU()) {
         all_inputs_eligible_for_cpu_pinning = false;
         continue;
       }
 
-      if (handle->dtype != DataType::DT_INT32 &&
-          handle->dtype != DataType::DT_INT64) {
+      if (tensor_handle->dtype != DataType::DT_INT32 &&
+          tensor_handle->dtype != DataType::DT_INT64) {
         all_inputs_eligible_for_cpu_pinning = false;
         continue;
       }
 
       int64 num_elements;
-      TF_RETURN_IF_ERROR(handle->NumElements(&num_elements));
+      TF_RETURN_IF_ERROR(tensor_handle->NumElements(&num_elements));
       if (num_elements > 64) {
         all_inputs_eligible_for_cpu_pinning = false;
       }
@@ -697,12 +854,12 @@ Status EagerExecute(EagerOperation* op,
   return EagerRemoteExecute(op, retvals->data(), num_retvals);
 }
 
-Status EagerExecute(EagerContext* ctx, Device* device,
-                    const gtl::InlinedVector<TensorHandle*, 4>& op_inputs,
-                    KernelAndDevice* kernel, NodeExecStats* maybe_stats,
-                    StepStats* maybe_step_stats,
-                    GraphCollector* graph_collector, TensorHandle** retvals,
-                    int num_retvals) {
+Status EagerKernelExecute(EagerContext* ctx, Device* device,
+                          const gtl::InlinedVector<TensorHandle*, 4>& op_inputs,
+                          KernelAndDevice* kernel, NodeExecStats* maybe_stats,
+                          StepStats* maybe_step_stats,
+                          GraphCollector* graph_collector,
+                          TensorHandle** retvals, int num_retvals) {
   if (device == nullptr) {
     // TODO(apassos) debug how the assignment below might return a different
     // device from the one requested above.
@@ -710,8 +867,6 @@ Status EagerExecute(EagerContext* ctx, Device* device,
   }
 
   std::vector<Tensor> outputs(1);
-  const MemoryTypeVector* output_memory_types = nullptr;
-  output_memory_types = &kernel->kernel()->output_memory_types();
 
   // If there are multiple references to a TensorHandle in 'op_inputs' we must
   // increment the reference count of the corresponding Tensor or risk it being
@@ -790,8 +945,8 @@ Status EagerExecute(EagerContext* ctx, Device* device,
                            /* op_device= */ device, ctx);
     } else {
       // In the async case, the retval is not a nullptr, and its device is
-      // already set since all TensorHandles always have their device set during
-      // construction.
+      // already set since all TensorHandles always have their device set
+      // during construction.
       DCHECK_EQ(device, retvals[i]->op_device());
       DCHECK_EQ(kernel->OutputDevice(i), retvals[i]->device());
 
@@ -822,25 +977,6 @@ Status LocalEagerCopyToDevice(TensorHandle* h, EagerContext* ctx, Device* dstd,
   }
 }
 
-Status FindDeviceFromName(EagerContext* ctx, const char* device_name,
-                          Device** device) {
-  *device = ctx->HostCPU();
-  if (device_name == nullptr || strlen(device_name) == 0) {
-    return Status::OK();
-  }
-
-  auto status = ctx->local_device_mgr()->LookupDevice(device_name, device);
-  if (status.ok()) {
-    return status;
-  }
-
-  if (ctx->remote_device_mgr() != nullptr) {
-    return ctx->remote_device_mgr()->LookupDevice(device_name, device);
-  }
-
-  return status;
-}
-
 Status ExecuteSend(EagerContext* ctx, tensorflow::Device* device,
                    TensorHandle* h, StringPiece wire_id,
                    const string& recv_device) {
@@ -902,8 +1038,8 @@ Status ExecuteRecv(EagerContext* ctx, tensorflow::Device* device,
   return Status::OK();
 }
 
-// This gets a unique wire ID. We add a random identifier so that if the worker
-// has other clients that it is servicing, we don't have any collision.
+// This gets a unique wire ID. We add a random identifier so that if the
+// worker has other clients that it is servicing, we don't have any collision.
 string GetUniqueWireID() {
   static tensorflow::uint64 random_seed = random::New64();
   static tensorflow::mutex wireid_mutex(tensorflow::LINKER_INITIALIZED);
diff --git a/tensorflow/core/common_runtime/eager/execute.h b/tensorflow/core/common_runtime/eager/execute.h
index 6143a52d4b..80d381cecd 100644
--- a/tensorflow/core/common_runtime/eager/execute.h
+++ b/tensorflow/core/common_runtime/eager/execute.h
@@ -43,12 +43,12 @@ Status EagerExecute(
 
 // Low-level utility to execute the kernel specified by kernel on device device,
 // with the inputs op_inputs, in the context ctx.
-Status EagerExecute(EagerContext* ctx, Device* device,
-                    const gtl::InlinedVector<TensorHandle*, 4>& op_inputs,
-                    KernelAndDevice* kernel, NodeExecStats* maybe_stats,
-                    StepStats* maybe_step_stats,
-                    GraphCollector* graph_collector, TensorHandle** retvals,
-                    int num_retvals);
+Status EagerKernelExecute(EagerContext* ctx, Device* device,
+                          const gtl::InlinedVector<TensorHandle*, 4>& op_inputs,
+                          KernelAndDevice* kernel, NodeExecStats* maybe_stats,
+                          StepStats* maybe_step_stats,
+                          GraphCollector* graph_collector,
+                          TensorHandle** retvals, int num_retvals);
 
 // Low-level utility to copy a tensor handle from one device to another.
 Status EagerCopyToDevice(TensorHandle* h, EagerContext* ctx,
diff --git a/tensorflow/core/common_runtime/eager/execute_node.h b/tensorflow/core/common_runtime/eager/execute_node.h
index a99d509dd6..4459e3221b 100644
--- a/tensorflow/core/common_runtime/eager/execute_node.h
+++ b/tensorflow/core/common_runtime/eager/execute_node.h
@@ -64,7 +64,7 @@ class ExecuteNode : public EagerNode {
   }
 
   tensorflow::Status Run() override {
-    const Status status = EagerExecute(
+    const Status status = EagerKernelExecute(
         ctx_, op_device_, inputs_, kernel_, maybe_stats_.get(),
         maybe_step_stats_, graph_collector_, retvals_.begin(), retvals_.size());
     if (status.ok()) {
diff --git a/tensorflow/core/common_runtime/eager/kernel_and_device.cc b/tensorflow/core/common_runtime/eager/kernel_and_device.cc
index 4096763b11..943b183d63 100644
--- a/tensorflow/core/common_runtime/eager/kernel_and_device.cc
+++ b/tensorflow/core/common_runtime/eager/kernel_and_device.cc
@@ -17,12 +17,17 @@ limitations under the License.
 
 #include "tensorflow/core/common_runtime/device_factory.h"
 #include "tensorflow/core/common_runtime/eager/attr_builder.h"
+#include "tensorflow/core/common_runtime/process_function_library_runtime.h"
 #include "tensorflow/core/common_runtime/rendezvous_mgr.h"
 #include "tensorflow/core/common_runtime/step_stats_collector.h"
 #include "tensorflow/core/framework/allocator.h"
+#include "tensorflow/core/framework/function.h"
 #include "tensorflow/core/framework/node_def.pb.h"
+#include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/resource_mgr.h"
 #include "tensorflow/core/framework/step_stats.pb.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/framework/types.pb.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/gtl/map_util.h"
 #include "tensorflow/core/lib/gtl/stl_util.h"
@@ -31,31 +36,77 @@ limitations under the License.
 #include "tensorflow/core/platform/tracing.h"
 #include "tensorflow/core/public/version.h"
 #include "tensorflow/core/util/tensor_slice_reader_cache.h"
+#ifndef __ANDROID__
+#include "tensorflow/core/grappler/optimizers/meta_optimizer.h"
+#endif
 
 namespace tensorflow {
 
-// static
-Status KernelAndDevice::Init(const NodeDef& ndef, FunctionLibraryRuntime* flr,
-                             std::function<void(std::function<void()>)>* runner,
-                             KernelAndDevice* out) {
+KernelAndDeviceFunc::~KernelAndDeviceFunc() {
+  if (handle_ != kInvalidHandle) {
+    Status status = pflr_->ReleaseHandle(handle_);
+    if (!status.ok()) {
+      LOG(INFO) << "Ignoring error status when releasing multi-device function "
+                   "handle "
+                << status.ToString();
+    }
+  }
+}
+
+Status KernelAndDeviceOp::Init(const NodeDef& ndef,
+                               GraphCollector* graph_collector) {
   OpKernel* k = nullptr;
-  TF_RETURN_IF_ERROR(flr->CreateKernel(ndef, &k));
-  out->device_ = flr->device();
-  out->kernel_.reset(k);
-  out->flr_ = flr;
-  out->runner_ = runner;
-  out->default_runner_ = [](std::function<void()> f) { f(); };
-
-  // Update output_dtypes_.
+  TF_RETURN_IF_ERROR(flr_->CreateKernel(ndef, &k));
+  kernel_.reset(k);
+  return Status::OK();
+}
+
+Status KernelAndDeviceFunc::Init(const NodeDef& ndef,
+                                 GraphCollector* graph_collector) {
   const OpDef* op_def = nullptr;
   const FunctionDef* function_def =
-      flr->GetFunctionLibraryDefinition()->Find(ndef.op());
+      flr_->GetFunctionLibraryDefinition()->Find(ndef.op());
   if (function_def != nullptr) {
     op_def = &(function_def->signature());
   } else {
     TF_RETURN_IF_ERROR(OpDefForOp(ndef.op().c_str(), &op_def));
   }
-  return OutputTypesForNode(ndef, *op_def, &out->output_dtypes_);
+  TF_RETURN_IF_ERROR(
+      InOutTypesForNode(ndef, *op_def, &input_dtypes_, &output_dtypes_));
+
+  FunctionLibraryRuntime::InstantiateOptions options;
+  options.target = device_->name();
+  options.is_multi_device_function = true;
+  for (const Device* device : input_devices_) {
+    options.input_devices.push_back(device->name());
+  }
+
+  const auto& it = ndef.attr().find("executor_type");
+  if (it != ndef.attr().end()) {
+    options.executor_type = it->second.s();
+  }
+#ifndef __ANDROID__
+  // Android tf library does not include grappler.
+  const auto& config_it = ndef.attr().find("config_proto");
+  if (it != ndef.attr().end()) {
+    ConfigProto config_proto;
+    if (!config_proto.ParseFromString(config_it->second.s())) {
+      return errors::InvalidArgument(
+          "Failed to parse config_proto attribute as tensorflow::ConfigProto "
+          "proto.");
+    }
+    options.optimize_graph_fn =
+        std::bind(grappler::OptimizeGraph, std::placeholders::_1,
+                  std::placeholders::_2, std::placeholders::_3,
+                  std::placeholders::_4, config_proto, std::placeholders::_5);
+  }
+#endif
+  options.graph_collector = graph_collector;
+
+  TF_RETURN_IF_ERROR(
+      pflr_->Instantiate(ndef.op(), AttrSlice(ndef), options, &handle_));
+  return pflr_->GetOutputDevices(handle_, &output_devices_);
+  return Status::OK();
 }
 
 Status KernelAndDevice::Run(const gtl::InlinedVector<TensorValue, 4>& inputs,
@@ -69,11 +120,39 @@ Status KernelAndDevice::Run(const gtl::InlinedVector<TensorValue, 4>& inputs,
                    graph_collector);
 }
 
-Status KernelAndDevice::Run(ScopedStepContainer* step_container,
-                            const gtl::InlinedVector<TensorValue, 4>& inputs,
-                            std::vector<Tensor>* outputs, NodeExecStats* stats,
-                            StepStats* step_stats,
-                            GraphCollector* graph_collector) {
+namespace {
+void UpdateStats(OpKernelContext* context,
+                 StepStatsCollector* step_stats_collector,
+                 NodeExecStats* stats) {
+  for (const auto& allocator_pair : context->ConsumeWrappedAllocators()) {
+    AllocatorMemoryUsed* memory = stats->add_memory();
+    memory->set_allocator_name(allocator_pair.first->Name());
+    auto sizes = allocator_pair.second->GetSizes();
+    memory->set_total_bytes(std::get<0>(sizes));
+    memory->set_peak_bytes(std::get<1>(sizes));
+    memory->set_live_bytes(std::get<2>(sizes));
+
+    AllocatorStats allocator_stats;
+    allocator_pair.first->GetStats(&allocator_stats);
+    memory->set_allocator_bytes_in_use(allocator_stats.bytes_in_use);
+    allocator_pair.second->GetRecordsAndUnRef();
+  }
+  auto* ms = stats->mutable_memory_stats();
+  ms->set_temp_memory_size(context->temp_memory_allocated());
+  for (const auto& alloc_id : context->persistent_alloc_ids()) {
+    ms->mutable_persistent_tensor_alloc_ids()->Add(alloc_id);
+  }
+
+  ms->set_persistent_memory_size(context->persistent_memory_allocated());
+  step_stats_collector->Finalize();
+}
+}  // anonymous namespace
+
+Status KernelAndDeviceOp::Run(ScopedStepContainer* step_container,
+                              const gtl::InlinedVector<TensorValue, 4>& inputs,
+                              std::vector<Tensor>* outputs,
+                              NodeExecStats* stats, StepStats* step_stats,
+                              GraphCollector* graph_collector) {
   std::vector<AllocatorAttributes> out_attrs(kernel_->num_outputs());
   for (size_t i = 0; i < out_attrs.size(); ++i) {
     out_attrs[i].set_on_host(kernel_->output_memory_types()[i] ==
@@ -108,11 +187,7 @@ Status KernelAndDevice::Run(ScopedStepContainer* step_container,
     params.stats_collector = step_stats_collector.get();
     params.graph_collector = graph_collector;
   }
-  if (runner_ == nullptr) {
-    params.runner = &default_runner_;
-  } else {
-    params.runner = runner_;
-  }
+  params.runner = runner_ != nullptr ? runner_ : &default_runner_;
 
   params.step_container = step_container;
   params.collective_executor =
@@ -148,37 +223,111 @@ Status KernelAndDevice::Run(ScopedStepContainer* step_container,
     outputs->push_back(Tensor(*context.mutable_output(i)));
   }
   if (stats != nullptr) {
-    for (const auto& allocator_pair : context.ConsumeWrappedAllocators()) {
-      AllocatorMemoryUsed* memory = stats->add_memory();
-      memory->set_allocator_name(allocator_pair.first->Name());
-      auto sizes = allocator_pair.second->GetSizes();
-      memory->set_total_bytes(std::get<0>(sizes));
-      memory->set_peak_bytes(std::get<1>(sizes));
-      memory->set_live_bytes(std::get<2>(sizes));
-
-      AllocatorStats allocator_stats;
-      allocator_pair.first->GetStats(&allocator_stats);
-      memory->set_allocator_bytes_in_use(allocator_stats.bytes_in_use);
-      allocator_pair.second->GetRecordsAndUnRef();
-    }
-    auto* ms = stats->mutable_memory_stats();
-    ms->set_temp_memory_size(context.temp_memory_allocated());
-    for (const auto& alloc_id : context.persistent_alloc_ids()) {
-      ms->mutable_persistent_tensor_alloc_ids()->Add(alloc_id);
-    }
+    UpdateStats(&context, step_stats_collector.get(), stats);
+  }
+  return Status::OK();
+}
+
+Status KernelAndDeviceFunc::Run(
+    ScopedStepContainer* step_container,
+    const gtl::InlinedVector<TensorValue, 4>& inputs,
+    std::vector<Tensor>* outputs, NodeExecStats* stats, StepStats* step_stats,
+    GraphCollector* graph_collector) {
+  FunctionLibraryRuntime::Options opts;
+  // We don't pass rendezvous from eager context because we can get tensor
+  // name collisions in send/recv ops when running multiple instances
+  // of the same multi-device function concurrently. Instead, we ask the
+  // function library runtime to create a new for this call. We could have
+  // created one here but it requires more state to be kept in
+  // KernelAndDeviceFunc.
+  opts.rendezvous = nullptr;
+  opts.create_rendezvous = true;
+  opts.cancellation_manager = &cm_;
+  // eager runtime does not yet support collective ops.
+  opts.collective_executor = nullptr;
+  opts.allow_dead_tensors = true;
+  opts.step_container = step_container;
+  opts.collective_executor =
+      collective_executor_ ? collective_executor_->get() : nullptr;
 
-    ms->set_persistent_memory_size(context.persistent_memory_allocated());
+  std::unique_ptr<StepStatsCollector> step_stats_collector;
+  if (stats != nullptr) {
+    step_stats_collector.reset(new StepStatsCollector(step_stats));
+  }
+  opts.stats_collector = step_stats_collector.get();
+  opts.runner = (runner_ == nullptr) ? &default_runner_ : runner_;
+
+  Notification done;
+  Status status;
+  outputs->clear();
+  std::vector<Tensor> input_vector;
+  input_vector.reserve(inputs.size());
+  for (const TensorValue& tensor_value : inputs) {
+    input_vector.push_back(*tensor_value.tensor);
+  }
+
+  flr_->Run(opts, handle_, input_vector, outputs,
+            [&status, &done](const Status& s) {
+              status = s;
+              done.Notify();
+            });
+  done.WaitForNotification();
+
+  if (step_stats_collector != nullptr) {
     step_stats_collector->Finalize();
   }
-  return Status::OK();
+  return status;
+}
+
+tensorflow::Device* KernelAndDeviceOp::OutputDevice(int idx) const {
+  if (kernel_->output_memory_types()[idx] == HOST_MEMORY) {
+    return nullptr;
+  }
+  return device_;
 }
 
-tensorflow::Device* KernelAndDevice::OutputDevice(int idx) const {
-  if (device_ != nullptr &&
-      kernel_->output_memory_types()[idx] == HOST_MEMORY) {
+tensorflow::Device* KernelAndDeviceFunc::OutputDevice(int idx) const {
+  if (output_dtypes_[idx] == DT_RESOURCE) {
     return nullptr;
   }
+  return output_devices_[idx];
+}
+
+tensorflow::Device* KernelAndDeviceOp::OutputResourceDevice(int idx) const {
+  if (kernel_->output_type(idx) == DT_RESOURCE) {
+    return device_;
+  }
+  return nullptr;
+}
+
+tensorflow::Device* KernelAndDeviceFunc::OutputResourceDevice(int idx) const {
+  if (output_dtypes_[idx] == DT_RESOURCE) {
+    return output_devices_[idx];
+  }
+  return nullptr;
+}
+
+DataType KernelAndDeviceOp::input_type(int i) const {
+  return kernel_->input_type(i);
+}
+
+DataType KernelAndDeviceFunc::input_type(int i) const {
+  return input_dtypes_[i];
+}
+
+Device* KernelAndDeviceOp::InputDevice(int i) const {
+  if (kernel_->input_memory_types()[i] == HOST_MEMORY) {
+    return host_cpu_device_;
+  }
   return device_;
 }
 
+Device* KernelAndDeviceFunc::InputDevice(int i) const {
+  if (input_dtypes_[i] == DT_RESOURCE) {
+    return host_cpu_device_;
+  } else {
+    return input_devices_[i];
+  }
+}
+
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/eager/kernel_and_device.h b/tensorflow/core/common_runtime/eager/kernel_and_device.h
index f7a5b76518..c4ea99f53e 100644
--- a/tensorflow/core/common_runtime/eager/kernel_and_device.h
+++ b/tensorflow/core/common_runtime/eager/kernel_and_device.h
@@ -38,8 +38,13 @@ namespace tensorflow {
 // include the proto header
 class NodeExecStats;
 class StepStats;
+class ProcessFunctionLibraryRuntime;
+class FunctionLibraryRuntime;
 
-// KernelAndDevice encapsulates an instantiated kernel and the device it is on.
+// KernelAndDevice encapsulates the logic needed to run a computation eagerly.
+// The computation can be a single instantiated kernel (implemented by
+// KernelAndDeviceOp below) or a multi-device function (implemented by
+// KernelAndDeviceFunc below).
 //
 // Also see:
 // https://www.tensorflow.org/code/tensorflow/core/common_runtime/kernel_benchmark_testlib.h
@@ -47,60 +52,171 @@ class StepStats;
 // https://www.tensorflow.org/code/tensorflow/core/kernels/ops_testutil.h
 class KernelAndDevice {
  public:
-  // Populates 'out' with a kernel appropriate for 'ndef'.
+  // Populates this with a kernel appropriate for 'ndef'.
   //
   // The provided FunctionLibraryRuntime MUST outlive all calls to
   // Run() on the returned KernelAndDevice.
-  static Status Init(const NodeDef& ndef, FunctionLibraryRuntime* flr,
-                     std::function<void(std::function<void()>)>* runner,
-                     KernelAndDevice* out);
-
-  KernelAndDevice(tensorflow::Rendezvous* rendez, bool log_memory)
-      : KernelAndDevice(rendez, log_memory, nullptr) {}
+  virtual Status Init(const NodeDef& ndef, GraphCollector* graph_collector) = 0;
 
+  // Non-multi-device functions are run using regular CallOp and look like
+  // primitive operations from KernelAndDevice perspective.
   KernelAndDevice(
-      tensorflow::Rendezvous* rendez, bool log_memory,
-      std::unique_ptr<CollectiveExecutor::Handle> collective_executor)
-      : device_(nullptr),
-        flr_(nullptr),
-        rendez_(rendez),
-        log_memory_(log_memory),
+      FunctionLibraryRuntime* flr,
+      std::function<void(std::function<void()>)>* runner,
+      std::unique_ptr<CollectiveExecutor::Handle> collective_executor,
+      Device* host_cpu_device)
+      : device_(flr->device()),
+        host_cpu_device_(host_cpu_device),
+        flr_(flr),
+        runner_(runner),
+        default_runner_([](std::function<void()> f) { f(); }),
         collective_executor_(std::move(collective_executor)) {}
 
+  virtual ~KernelAndDevice() {}
+
   // TODO(ashankar): Handle list-valued inputs.
   Status Run(const gtl::InlinedVector<TensorValue, 4>& inputs,
              std::vector<Tensor>* outputs, NodeExecStats* stats,
              StepStats* step_stats, GraphCollector* graph_collector);
 
-  Status Run(ScopedStepContainer* step_container,
-             const gtl::InlinedVector<TensorValue, 4>& inputs,
-             std::vector<Tensor>* outputs, NodeExecStats* stats,
-             StepStats* step_stats, GraphCollector* graph_collector);
-
-  Device* OutputDevice(int idx) const;
-
-  const OpKernel* kernel() const { return kernel_.get(); }
-
+  virtual Status Run(ScopedStepContainer* step_container,
+                     const gtl::InlinedVector<TensorValue, 4>& inputs,
+                     std::vector<Tensor>* outputs, NodeExecStats* stats,
+                     StepStats* step_stats,
+                     GraphCollector* graph_collector) = 0;
+
+  virtual Device* InputDevice(int i) const = 0;
+  virtual Device* OutputDevice(int idx) const = 0;
+  // If idx'th output is a resource, returns the device backing the resource.
+  // Else, returns nullptr.
+  virtual Device* OutputResourceDevice(int idx) const = 0;
+
+  // Returns nullptr for functions.
+  virtual const OpKernel* kernel() const = 0;
+
+  // Returns the device on which this kernel will run. In the case of
+  // multi-device functions, this is the default device that is passed to the
+  // placer but actual computation can happen on a different set of devices.
+  // Also, outputs can be produced on devices different from what this method
+  // returns.
   Device* device() const { return device_; }
 
-  const DataTypeVector& output_dtypes() { return output_dtypes_; }
+  virtual const DataTypeVector& output_dtypes() const = 0;
 
- private:
+  virtual DataType input_type(int i) const = 0;
+  virtual int num_inputs() const = 0;
+  virtual int num_outputs() const = 0;
+
+ protected:
   // TODO(apassos) Consider a shared cancellation manager. Note that this
   // cancellation manager is not useful to actually cancel anything, and is
   // provided here only for the few kernels which can't handle one being
   // missing.
   CancellationManager cm_;
+  Device* const device_;           // non-null
+  Device* const host_cpu_device_;  // non-null
+  FunctionLibraryRuntime* const flr_;
+  std::function<void(std::function<void()>)>* const runner_;
+  std::function<void(std::function<void()>)> default_runner_;
+  const std::unique_ptr<CollectiveExecutor::Handle> collective_executor_;
+};
+
+// Represents an op kernel and the device it will be run on.
+class KernelAndDeviceOp final : public KernelAndDevice {
+ public:
+  KernelAndDeviceOp(
+      tensorflow::Rendezvous* rendez, bool log_memory,
+      FunctionLibraryRuntime* flr,
+      std::function<void(std::function<void()>)>* runner,
+      std::unique_ptr<CollectiveExecutor::Handle> collective_executor,
+      Device* host_cpu_device)
+      : KernelAndDevice(flr, runner, std::move(collective_executor),
+                        host_cpu_device),
+        rendez_(rendez),
+        log_memory_(log_memory) {}
+
+  Status Init(const NodeDef& ndef, GraphCollector* graph_collector) override;
+
+  using KernelAndDevice::Run;
+
+  Status Run(ScopedStepContainer* step_container,
+             const gtl::InlinedVector<TensorValue, 4>& inputs,
+             std::vector<Tensor>* outputs, NodeExecStats* stats,
+             StepStats* step_stats, GraphCollector* graph_collector) override;
+
+  const OpKernel* kernel() const override { return kernel_.get(); }
+
+  Device* InputDevice(int i) const override;
+  Device* OutputDevice(int idx) const override;
+  Device* OutputResourceDevice(int idx) const override;
+
+  DataType input_type(int i) const override;
+  const DataTypeVector& output_dtypes() const {
+    return kernel_->output_types();
+  }
+  int num_inputs() const override { return kernel_->num_inputs(); }
+  int num_outputs() const override { return kernel_->num_outputs(); }
+
+ private:
   std::unique_ptr<OpKernel> kernel_;
-  Device* device_;
-  FunctionLibraryRuntime* flr_;
+  Rendezvous* const rendez_;
   checkpoint::TensorSliceReaderCacheWrapper slice_reader_cache_;
-  Rendezvous* rendez_;
-  DataTypeVector output_dtypes_;
-  std::function<void(std::function<void()>)>* runner_;
-  std::function<void(std::function<void()>)> default_runner_;
   const bool log_memory_;
-  const std::unique_ptr<CollectiveExecutor::Handle> collective_executor_;
+};
+
+// Represents a multi-device function. Functions can also be run using
+// various function-calling kernels including CallOp and PartitionedCallOp.
+// In such cases, KernelAndDeviceOp is used.
+class KernelAndDeviceFunc final : public KernelAndDevice {
+ public:
+  KernelAndDeviceFunc(
+      FunctionLibraryRuntime* flr, ProcessFunctionLibraryRuntime* pflr,
+      std::vector<Device*> input_devices,
+      std::function<void(std::function<void()>)>* runner,
+      std::unique_ptr<CollectiveExecutor::Handle> collective_executor,
+      Device* host_cpu_device)
+      : KernelAndDevice(flr, runner, std::move(collective_executor),
+                        host_cpu_device),
+        pflr_(pflr),
+        handle_(kInvalidHandle),
+        input_devices_(std::move(input_devices)) {}
+
+  virtual ~KernelAndDeviceFunc();
+
+  Status Init(const NodeDef& ndef, GraphCollector* graph_collector) override;
+
+  using KernelAndDevice::Run;
+
+  Status Run(ScopedStepContainer* step_container,
+             const gtl::InlinedVector<TensorValue, 4>& inputs,
+             std::vector<Tensor>* outputs, NodeExecStats* stats,
+             StepStats* step_stats, GraphCollector* graph_collector) override;
+
+  const OpKernel* kernel() const override { return nullptr; }
+
+  Device* InputDevice(int i) const override;
+  Device* OutputDevice(int idx) const override;
+  Device* OutputResourceDevice(int idx) const override;
+
+  DataType input_type(int i) const override;
+  const DataTypeVector& output_dtypes() const override {
+    return output_dtypes_;
+  }
+  int num_inputs() const override { return input_dtypes_.size(); }
+  int num_outputs() const override { return output_dtypes_.size(); }
+
+ private:
+  ProcessFunctionLibraryRuntime* const pflr_;
+  FunctionLibraryRuntime::Handle handle_;
+  // CPU devices are null. Resource handles' devices are actual backing
+  // devices.
+  std::vector<Device*> output_devices_;
+  // CPU devices are not null. Resource handles' devices are actual backing
+  // devices.
+  std::vector<Device*> input_devices_;
+
+  DataTypeVector input_dtypes_;
+  DataTypeVector output_dtypes_;
 };
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/eager/kernel_and_device_test.cc b/tensorflow/core/common_runtime/eager/kernel_and_device_test.cc
index 8d6db96779..703f3eb9b7 100644
--- a/tensorflow/core/common_runtime/eager/kernel_and_device_test.cc
+++ b/tensorflow/core/common_runtime/eager/kernel_and_device_test.cc
@@ -27,10 +27,13 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/device_mgr.h"
 #include "tensorflow/core/common_runtime/eager/attr_builder.h"
 #include "tensorflow/core/common_runtime/function.h"
+#include "tensorflow/core/common_runtime/process_function_library_runtime.h"
 #include "tensorflow/core/platform/env.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/core/platform/test_benchmark.h"
+#include "tensorflow/core/protobuf/config.pb.h"
 #include "tensorflow/core/public/version.h"
+#include "tensorflow/core/util/ptr_util.h"
 
 namespace tensorflow {
 namespace {
@@ -41,20 +44,27 @@ class TestEnv {
     std::vector<std::unique_ptr<Device>> devices;
     devices.push_back(
         DeviceFactory::NewDevice("CPU", {}, "/job:a/replica:0/task:0"));
+    cpu_device_ = devices.back().get();
     device_mgr_ = absl::make_unique<DeviceMgr>(std::move(devices));
-    flib_runtime_ = NewFunctionLibraryRuntime(
-        device_mgr_.get(), Env::Default(), device_mgr_->ListDevices()[0],
-        TF_GRAPH_DEF_VERSION, &flib_def_, nullptr, {}, nullptr);
-  }
+    OptimizerOptions opts;
+    pflr_ = tensorflow::MakeUnique<ProcessFunctionLibraryRuntime>(
+        device_mgr_.get(), Env::Default(), TF_GRAPH_DEF_VERSION, &flib_def_,
+        opts, /*default_thread_pool=*/nullptr, /*cluster_flr=*/nullptr);
 
-  FunctionLibraryRuntime* function_library_runtime() const {
-    return flib_runtime_.get();
+    flr_ = pflr_->GetFLR("/job:a/replica:0/task:0/device:CPU:0");
+    CHECK(flr_ != nullptr);
   }
 
+  FunctionLibraryRuntime* function_library_runtime() const { return flr_; }
+  ProcessFunctionLibraryRuntime* pflr() const { return pflr_.get(); }
+  Device* cpu_device() { return cpu_device_; }
+
  private:
   FunctionLibraryDefinition flib_def_;
   std::unique_ptr<DeviceMgr> device_mgr_;
-  std::unique_ptr<FunctionLibraryRuntime> flib_runtime_;
+  FunctionLibraryRuntime* flr_;
+  std::unique_ptr<ProcessFunctionLibraryRuntime> pflr_;
+  Device* cpu_device_;
 };
 
 void BM_CreateGraph(int iters) {
@@ -106,11 +116,11 @@ void BM_KernelAndDeviceInit(int iters) {
                    .NumInputs(2)
                    .BuildNodeDef());
   TestEnv env;
-  KernelAndDevice k(nullptr, false);
+  KernelAndDeviceOp k(nullptr, false, env.function_library_runtime(), nullptr,
+                      nullptr, env.cpu_device());
   tensorflow::testing::StartTiming();
   for (int i = 0; i < iters; ++i) {
-    TF_CHECK_OK(KernelAndDevice::Init(ndef, env.function_library_runtime(),
-                                      nullptr, &k));
+    TF_CHECK_OK(k.Init(ndef, nullptr));
   }
 }
 BENCHMARK(BM_KernelAndDeviceInit);
@@ -129,12 +139,12 @@ void BM_KernelAndDeviceRun(int iters) {
                    .NumInputs(inputs.size())
                    .BuildNodeDef());
   TestEnv env;
-  KernelAndDevice kernel(nullptr, false);
-  TF_CHECK_OK(KernelAndDevice::Init(ndef, env.function_library_runtime(),
-                                    nullptr, &kernel));
+  KernelAndDeviceOp k(nullptr, false, env.function_library_runtime(), nullptr,
+                      nullptr, env.cpu_device());
+  TF_CHECK_OK(k.Init(ndef, nullptr));
   tensorflow::testing::StartTiming();
   for (int i = 0; i < iters; ++i) {
-    TF_CHECK_OK(kernel.Run(inputs, &outputs, nullptr, nullptr, nullptr));
+    TF_CHECK_OK(k.Run(inputs, &outputs, nullptr, nullptr, nullptr));
   }
 }
 BENCHMARK(BM_KernelAndDeviceRun);
diff --git a/tensorflow/core/common_runtime/eager/tensor_handle.cc b/tensorflow/core/common_runtime/eager/tensor_handle.cc
index 47a856a36e..eaca6d7b32 100644
--- a/tensorflow/core/common_runtime/eager/tensor_handle.cc
+++ b/tensorflow/core/common_runtime/eager/tensor_handle.cc
@@ -31,6 +31,7 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/rendezvous_mgr.h"
 #include "tensorflow/core/framework/rendezvous.h"
 #include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/types.pb.h"
 #include "tensorflow/core/lib/core/stringpiece.h"
 #include "tensorflow/core/lib/gtl/inlined_vector.h"
 #include "tensorflow/core/lib/gtl/map_util.h"
@@ -43,6 +44,61 @@ limitations under the License.
 
 namespace tensorflow {
 
+TensorHandle::TensorHandle(const class Tensor& t, Device* d, Device* op_device,
+                           EagerContext* ctx)
+    : dtype(t.dtype()),
+      node_id_(0),
+      tensor_(t),
+      device_(d),
+      op_device_(op_device),
+      resource_device_(GetResourceDevice(t, ctx)),
+      remote_op_id_(-1),
+      remote_output_num_(-1),
+      remote_shape_node_id_(-1),
+      ctx_(ctx),
+      is_ready_(true) {}
+
+TensorHandle::TensorHandle(uint64 node_id, Device* d, Device* op_device,
+                           Device* resource_device, DataType dtype,
+                           EagerContext* ctx)
+    : dtype(dtype),
+      node_id_(node_id),
+      tensor_(dtype),
+      device_(d),
+      op_device_(op_device),
+      resource_device_(resource_device),
+      remote_op_id_(-1),
+      remote_output_num_(-1),
+      remote_shape_node_id_(-1),
+      ctx_(ctx),
+      is_ready_(ctx == nullptr) {
+  DCHECK_GT(node_id_, 0);
+  DCHECK(dtype == DT_RESOURCE ? resource_device_ != nullptr
+                              : resource_device_ == nullptr);
+}
+
+TensorHandle::TensorHandle(int64 op_id, int32 output_num,
+                           uint64 remote_shape_node_id, DataType dtype,
+                           std::function<void()> call_on_destroy, Device* d,
+                           Device* op_device, Device* resource_device,
+                           EagerContext* ctx)
+    : dtype(dtype),
+      node_id_(0),
+      device_(d),
+      op_device_(op_device),
+      resource_device_(resource_device),
+      remote_op_id_(op_id),
+      remote_output_num_(output_num),
+      remote_shape_node_id_(remote_shape_node_id),
+      call_on_destroy_(std::move(call_on_destroy)),
+      ctx_(ctx),
+      is_ready_(true) {
+  DCHECK(IsRemote()) << "Op ID and output num should be >= 0. Op ID: " << op_id
+                     << ", Output num: " << output_num;
+  DCHECK(dtype == DT_RESOURCE ? resource_device_ != nullptr
+                              : resource_device_ == nullptr);
+}
+
 bool TensorHandle::IsReady() {
   if (node_id_ == 0) return true;
   mutex_lock l(ctx_mutex_);
@@ -245,4 +301,15 @@ Status TensorHandle::CopyToDevice(EagerContext* ctx, tensorflow::Device* dstd,
   return status;
 }
 
+Device* GetResourceDevice(const Tensor& t, EagerContext* ctx) {
+  if (t.dtype() != DT_RESOURCE) {
+    return nullptr;
+  }
+  const ResourceHandle& resource_handle = t.flat<ResourceHandle>()(0);
+  const auto& map = *ctx->device_map();
+  auto it = map.find(resource_handle.device());
+  DCHECK(it != map.end());
+  return it->second;
+}
+
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/eager/tensor_handle.h b/tensorflow/core/common_runtime/eager/tensor_handle.h
index 3eae3c7cc3..a4727d6f76 100644
--- a/tensorflow/core/common_runtime/eager/tensor_handle.h
+++ b/tensorflow/core/common_runtime/eager/tensor_handle.h
@@ -48,50 +48,15 @@ namespace tensorflow {
 // (unrelated to python TensorHandle).
 class TensorHandle : public core::RefCounted {
  public:
-  TensorHandle(const Tensor& t, Device* d, Device* op_device, EagerContext* ctx)
-      : dtype(t.dtype()),
-        node_id_(0),
-        tensor_(t),
-        device_(d),
-        op_device_(op_device),
-        remote_op_id_(-1),
-        remote_output_num_(-1),
-        remote_shape_node_id_(-1),
-        ctx_(ctx),
-        is_ready_(true) {}
-
-  TensorHandle(uint64 node_id, Device* d, Device* op_device, DataType dtype,
-               EagerContext* ctx)
-      : dtype(dtype),
-        node_id_(node_id),
-        tensor_(dtype),
-        device_(d),
-        op_device_(op_device),
-        remote_op_id_(-1),
-        remote_output_num_(-1),
-        remote_shape_node_id_(-1),
-        ctx_(ctx),
-        is_ready_(ctx == nullptr) {
-    DCHECK_GT(node_id_, 0);
-  }
+  TensorHandle(const Tensor& t, Device* d, Device* op_device,
+               EagerContext* ctx);
+  TensorHandle(uint64 node_id, Device* d, Device* op_device,
+               Device* resource_device, DataType dtype, EagerContext* ctx);
 
   // Remote tensor handle constructor.
   TensorHandle(int64 op_id, int32 output_num, uint64 remote_shape_node_id,
                DataType dtype, std::function<void()> call_on_destroy, Device* d,
-               Device* op_device, EagerContext* ctx)
-      : dtype(dtype),
-        node_id_(0),
-        device_(d),
-        op_device_(op_device),
-        remote_op_id_(op_id),
-        remote_output_num_(output_num),
-        remote_shape_node_id_(remote_shape_node_id),
-        call_on_destroy_(std::move(call_on_destroy)),
-        ctx_(ctx),
-        is_ready_(true) {
-    DCHECK(IsRemote()) << "Op ID and output num should be >= 0. Op ID: "
-                       << op_id << ", Output num: " << output_num;
-  }
+               Device* op_device, Device* resource_device, EagerContext* ctx);
 
   ~TensorHandle() override {
     if (call_on_destroy_) {
@@ -104,8 +69,8 @@ class TensorHandle : public core::RefCounted {
   Status TensorValue(tensorflow::TensorValue* t);
 
   tensorflow::Device* device() const { return device_; }
-
   tensorflow::Device* op_device() const { return op_device_; }
+  tensorflow::Device* resource_device() const { return resource_device_; }
 
   Status TensorAndDevice(const tensorflow::Tensor** tensor,
                          tensorflow::Device** device,
@@ -178,6 +143,10 @@ class TensorHandle : public core::RefCounted {
   // device_ for constant tensors.
   tensorflow::Device* const op_device_;
 
+  // If the tensor dtype is DT_RESOURCE, resource_device_ holds the device
+  // backing the resource. Else resource_device_ is nullptr.
+  tensorflow::Device* const resource_device_;
+
   // IDs required when this class is representing a remote tensor handle.
   const int64 remote_op_id_;
   const int32 remote_output_num_;
@@ -198,6 +167,10 @@ class TensorHandle : public core::RefCounted {
   bool is_ready_ GUARDED_BY(ctx_mutex_);
 };
 
+// If tensor's dtype is DT_RESOURCE, returns the device backing the resource.
+// Else, returns nullptr.
+Device* GetResourceDevice(const Tensor& t, EagerContext* ctx);
+
 }  // namespace tensorflow
 
 #endif  // TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_TENSOR_HANDLE_H_
diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index e21ad4e2ee..bbf8630a2e 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -52,6 +52,7 @@ from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.util import compat
+from tensorflow.python.util import function_utils
 from tensorflow.python.util import memory
 from tensorflow.python.util import nest
 from tensorflow.python.util import tf_decorator
@@ -192,7 +193,7 @@ def _inference_name(n):
 # so it doesn't have the definition-generating logic and is just a container for
 # an already-defined function.
 class _EagerDefinedFunction(object):
-  """Callable with the interface of `framework.function._DefinedFunction.`
+  """Callable with the interface of `framework.function._DefinedFunction`.
 
   `_EagerDefinedFunction` encapsulates a function definition and its properties,
   and it provides a method for calling the encapsulated function. Some Ops
@@ -284,51 +285,60 @@ class _EagerDefinedFunction(object):
     Raises:
       ValueError: if the number of arguments is incorrect.
     """
+    if len(args) != len(self.signature.input_arg):
+      raise ValueError(
+          "Arguments and signature arguments do not match: %s %s " %
+          (len(args), len(list(self.signature.input_arg))))
+
+    function_call_options = ctx.get_function_call_options()
+    if function_call_options.config_proto_serialized is None:
+      config = function_utils.get_disabled_rewriter_config()
+    else:
+      config = function_call_options.config_proto_serialized
+    executor_type = function_call_options.executor_type or ""
 
     executing_eagerly = ctx.executing_eagerly()
-
-    if self._graph._xla_compile:  # pylint: disable=protected-access
-      # XLA compilation relies upon a custom kernel creator to run functions.
-      signature = self.signature
-      if executing_eagerly:
+    if executing_eagerly:
+      with _InterpolateFunctionError(self):
         outputs = execute.execute(
-            str(signature.name),
+            str(self.signature.name),
             num_outputs=self._num_outputs,
             inputs=args,
-            attrs=None,
+            attrs=("executor_type", executor_type,
+                   "config_proto", config),
             ctx=ctx)
+      # Replace empty list with None
+      outputs = outputs or None
+    elif self._graph._xla_compile:  # pylint: disable=protected-access
+      g = ops.get_default_graph()
+      self.add_to_graph(g)
+      signature = self.signature
+      op = g.create_op(
+          signature.name,
+          [ops.internal_convert_to_tensor(x, ctx=ctx) for x in args],
+          tuple(dtypes_module.DType(x.type) for x in signature.output_arg),
+          op_def=signature,
+          name="FunctionCall",
+          compute_shapes=False)
+      outputs = op.outputs
+      if not outputs:
+        return op
+      if isinstance(outputs, (ops.Tensor, type(None))):
+        outputs = [outputs]
       else:
-        g = ops.get_default_graph()
-        self.add_to_graph(g)
-        op = g.create_op(
-            signature.name,
-            [ops.internal_convert_to_tensor(x, ctx=ctx) for x in args],
-            tuple(dtypes_module.DType(x.type) for x in signature.output_arg),
-            op_def=signature,
-            name="FunctionCall",
-            compute_shapes=False)
-        outputs = op.outputs
-        if not outputs:
-          return op
-        outputs = [outputs] if isinstance(
-            outputs, (ops.Tensor, type(None))) else list(outputs)
+        outputs = list(outputs)
     else:
       # TODO(akshayka): Either remove this if the FunctionLibraryRuntime
       # creates `PartitionedCallOp` kernels by default, or remove the previous
       # branch if a TPU kernel is registered for `PartitionedCall`.
-      if len(args) != len(self.signature.input_arg):
-        raise ValueError(
-            "Arguments and signature arguments do not match: %s %s " %
-            (len(args), len(list(self.signature.input_arg))))
-      function_call_options = ctx.get_function_call_options()
       with _InterpolateFunctionError(self):
         outputs = functional_ops.partitioned_call(
             args=args,
             f=self,
             tout=self._output_types,
             executing_eagerly=executing_eagerly,
-            config=function_call_options.config_proto_serialized,
-            executor_type=function_call_options.executor_type)
+            config=config,
+            executor_type=executor_type)
 
     if executing_eagerly:
       return outputs
diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py
index 1966b259bf..68c581cbf8 100644
--- a/tensorflow/python/eager/function_test.py
+++ b/tensorflow/python/eager/function_test.py
@@ -19,6 +19,7 @@ from __future__ import print_function
 
 import collections
 import functools
+import itertools
 from multiprocessing.pool import ThreadPool
 import sys
 import weakref
@@ -48,6 +49,7 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import check_ops
 from tensorflow.python.ops import clip_ops
 from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import gen_resource_variable_ops
 from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import list_ops
 from tensorflow.python.ops import math_ops
@@ -2065,6 +2067,10 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
     self.assertIs(None, weak_fn())
 
   def testFunctionStackInErrorMessage(self):
+    if context.executing_eagerly():
+      # TODO(b/122736651): Remove this skipTest once fixed.
+      self.skipTest('Error interpolation is not working when function is '
+                    'invoked without PartitionedCallOp.')
 
     @def_function.function()
     def fn3(x):
@@ -2088,6 +2094,265 @@ class FunctionTest(test.TestCase, parameterized.TestCase):
       self.assertNotIn('fn3', e.message)
 
 
+class MultiDeviceTest(test.TestCase, parameterized.TestCase):
+
+  def testMultiDeviceOutput(self):
+    """Tests that functions can produce outputs on multiple devices."""
+    if not context.context().num_gpus():
+      self.skipTest('No GPUs found.')
+
+    @function.defun
+    def func(a, b, transpose_a):
+      with ops.device('/device:CPU:0'):
+        m1 = math_ops.matmul(a, b, transpose_a=transpose_a)
+      with ops.device('/device:GPU:0'):
+        m2 = math_ops.matmul(a, b, transpose_a=transpose_a)
+      return m1, m2
+
+    t = constant_op.constant([[1.0, 2.0], [3.0, 4.0]])
+    m1, m2 = func(t, t, transpose_a=True)
+    self.assertAllEqual(m1.numpy(), [[10, 14], [14, 20]])
+    self.assertRegexpMatches(m1.backing_device, 'CPU')
+    self.assertAllEqual(m2.numpy(), [[10, 14], [14, 20]])
+    self.assertRegexpMatches(m2.backing_device, 'GPU')
+
+  def testEmptyBody(self):
+    if not context.context().num_gpus():
+      self.skipTest('No GPUs found.')
+
+    @function.defun
+    def func(a, b):
+      return b, a
+
+    with ops.device('/device:CPU:0'):
+      a = constant_op.constant(3.0)
+    with ops.device('/device:GPU:0'):
+      b = constant_op.constant(5.0)
+
+    m1, m2 = func(a, b)
+    self.assertAllEqual(m1.numpy(), 5.0)
+    self.assertRegexpMatches(m1.backing_device, 'GPU')
+    self.assertAllEqual(m2.numpy(), 3.0)
+    self.assertRegexpMatches(m2.backing_device, 'CPU')
+
+  def testMultiDeviceInt32(self):
+    """Tests that multi-device functions can take and output INT32s.
+
+    When an INT32 device tensor is fed into a function, it is copied to CPU
+    by the eager runtime. The function sees all INT32 inputs on CPU.
+
+    We set allocator attribute 'on_host' for INT32 outputs. They can be
+    partitioned into the GPU component function, but will be allocated on
+    CPU nevertheless.
+
+    There is experimental support for `ints_on_device` in
+    FunctionLibraryRuntime now. We can try that.
+
+    """
+    if not context.context().num_gpus():
+      self.skipTest('No GPUs found.')
+
+    with ops.device('/device:CPU:0'):
+      int_cpu = constant_op.constant(3, dtype=dtypes.int32)
+      resource = resource_variable_ops.ResourceVariable(5, dtype=dtypes.int32)
+    with ops.device('/device:GPU:0'):
+      int_gpu = constant_op.constant(7, dtype=dtypes.int32)
+
+    @function.defun
+    def func(int_cpu, resource, int_gpu):
+      with ops.device('/device:CPU:0'):
+        m1 = int_cpu * resource + int_gpu
+      with ops.device('/device:GPU:0'):
+        # This computation will happen on GPU but m2 will be copied to CPU.
+        m2 = int_gpu * resource + int_cpu + 1
+      return m1, m2
+
+    m1, m2 = func(int_cpu, resource, int_gpu)
+    self.assertAllEqual(m1.numpy(), 22)
+    self.assertRegexpMatches(m1.backing_device, 'CPU')
+    self.assertAllEqual(m2.numpy(), 39)
+    self.assertRegexpMatches(m2.backing_device, 'CPU')
+
+    # flip arguments
+    m1, m2 = func(int_gpu, resource, int_cpu)
+    self.assertAllEqual(m1.numpy(), 38)
+    self.assertRegexpMatches(m1.backing_device, 'CPU')
+    self.assertAllEqual(m2.numpy(), 23)
+    self.assertRegexpMatches(m2.backing_device, 'CPU')
+
+  def testMultiDeviceColocateWith(self):
+    """Tests that function's outputs respect colocation constraints."""
+    if not context.context().num_gpus():
+      self.skipTest('No GPUs found.')
+
+    @function.defun
+    def func(a, b):
+      with ops.colocate_with(a):
+        ra = 2 * a
+      with ops.colocate_with(b):
+        rb = 3 * b
+      return ra, rb
+
+    devices = ['/device:CPU:0', '/device:GPU:0']
+    for dev1, dev2 in itertools.product(devices, devices):
+      with ops.device(dev1):
+        a = constant_op.constant(1.0)
+      with ops.device(dev2):
+        b = constant_op.constant(10.0)
+
+      ra, rb = func(a, b)
+      self.assertEqual(ra.numpy(), 2.0)
+      self.assertRegexpMatches(ra.backing_device, dev1)
+      self.assertEqual(rb.numpy(), 30.0)
+      self.assertRegexpMatches(rb.backing_device, dev2)
+
+  def testMultiDeviceResources(self):
+    if not context.context().num_gpus():
+      self.skipTest('No GPUs found.')
+
+    with ops.device('/device:CPU:0'):
+      c1 = resource_variable_ops.ResourceVariable(2.0)
+      c2 = resource_variable_ops.ResourceVariable(7.0)
+    with ops.device('/device:GPU:0'):
+      g1 = resource_variable_ops.ResourceVariable(3.0)
+      g2 = resource_variable_ops.ResourceVariable(5.0)
+
+    @function.defun
+    def func(resource1, resource2):
+      with ops.device('/device:CPU:0'):
+        result1 = resource1 * g2
+      with ops.device('/device:GPU:0'):
+        result2 = resource2 * c2
+      return result1, result2
+
+    r1, r2 = func(c1, g1)
+    self.assertEqual(r1.numpy(), 10.0)
+    self.assertRegexpMatches(r1.backing_device, 'CPU')
+    self.assertEqual(r2.numpy(), 21.0)
+    self.assertRegexpMatches(r2.backing_device, 'GPU')
+
+    # Call with flipped inputs. Check that we look at resource's
+    # device and reinstantiates the function when inputs' devices change.
+    r1, r2 = func(g1, c1)
+    self.assertEqual(r1.numpy(), 15.0)
+    self.assertRegexpMatches(r1.backing_device, 'CPU')
+    self.assertEqual(r2.numpy(), 14.0)
+    self.assertRegexpMatches(r2.backing_device, 'GPU')
+
+  def testOutputResources(self):
+    if not context.context().num_gpus():
+      self.skipTest('No GPUs found.')
+
+    with ops.device('/device:CPU:0'):
+      c1 = resource_variable_ops.ResourceVariable(2.0)
+    with ops.device('/device:GPU:0'):
+      g1 = resource_variable_ops.ResourceVariable(3.0)
+
+    @function.defun
+    def func(resource1, resource2):
+      with ops.device('/device:CPU:0'):
+        result1 = resource1 * 5
+      with ops.device('/device:GPU:0'):
+        result2 = resource2 * 7
+      return result1, resource1.handle, result2, resource2.handle
+
+    r1, res1, r2, res2 = func(c1, g1)
+    self.assertEqual(r1.numpy(), 10.0)
+    self.assertRegexpMatches(r1.backing_device, 'CPU')
+    self.assertEqual(r2.numpy(), 21.0)
+    self.assertRegexpMatches(r2.backing_device, 'GPU')
+
+    def check_handle(handle, expected_value):
+      self.assertRegexpMatches(handle.backing_device, 'CPU')
+      tensor = gen_resource_variable_ops.read_variable_op(
+          handle, dtypes.float32)
+      self.assertEqual(tensor.numpy(), expected_value)
+
+    # Check that handles returned from functions are on CPU and an op using
+    # the resource handle is correctly placed on the device backing the
+    # resource.
+    check_handle(res1, 2.0)
+    check_handle(res2, 3.0)
+
+    # Call with flipped inputs to make sure the same the function is
+    # reinstantiated and eager runtime does not mess up the device assignment
+    # for ops consuming handles returned from defuns.
+    r1, res1, r2, res2 = func(g1, c1)
+    self.assertEqual(r1.numpy(), 15.0)
+    self.assertRegexpMatches(r1.backing_device, 'CPU')
+    self.assertEqual(r2.numpy(), 14.0)
+    self.assertRegexpMatches(r2.backing_device, 'GPU')
+    check_handle(res1, 3.0)
+    check_handle(res2, 2.0)
+
+  def testComplexInputOutputDevicePattern(self):
+    """Tests input/output mapping logic in partitioning."""
+    if not context.context().num_gpus():
+      self.skipTest('No GPUs found.')
+
+    with ops.device('/device:CPU:0'):
+      rc0 = resource_variable_ops.ResourceVariable(2.0)
+      rc1 = resource_variable_ops.ResourceVariable(3.0)
+      cc0 = constant_op.constant(5.0)
+      cc1 = constant_op.constant(7.0)
+    with ops.device('/device:GPU:0'):
+      rg0 = resource_variable_ops.ResourceVariable(11.0)
+      rg1 = resource_variable_ops.ResourceVariable(13.0)
+      cg0 = constant_op.constant(17.0)
+      cg1 = constant_op.constant(19.0)
+
+    # Make sure tensors are on expected devices.
+    for tensor in [cc0, cc1]:
+      self.assertRegexpMatches(tensor.backing_device, 'CPU:0')
+    for tensor in [cg0, cg1]:
+      self.assertRegexpMatches(tensor.backing_device, 'GPU:0')
+
+    @function.defun
+    def func(rc0, cc0, cg0, rc1, cg1, rg0, rg1, cc1):
+      with ops.device('/device:CPU:0'):
+        m1 = rc0 * cg0
+      with ops.device('/device:GPU:0'):
+        m2 = rg0 * cc0
+
+      with ops.device('/device:CPU:0'):
+        r1 = 1000.0 * m2 + rc1 * cg1
+      with ops.device('/device:GPU:0'):
+        r2 = 1000.0 * m1 + rg1 * cc1
+
+      return r1, r2, m2, m1
+
+    r1, r2, m2, m1 = func(rc0, cc0, cg0, rc1, cg1, rg0, rg1, cc1)
+    self.assertRegexpMatches(m1.backing_device, 'CPU')
+    self.assertRegexpMatches(r1.backing_device, 'CPU')
+    self.assertRegexpMatches(m2.backing_device, 'GPU')
+    self.assertRegexpMatches(r2.backing_device, 'GPU')
+    self.assertEqual(m1.numpy(), 34.0)
+    self.assertEqual(r1.numpy(), 55000.0 + 3.0 * 19.0)
+    self.assertEqual(m2.numpy(), 55.0)
+    self.assertEqual(r2.numpy(), 34000.0 + 13.0 * 7.0)
+
+  def testArgumentPrunning(self):
+    """Tests functions taking unnecessary arguments."""
+    if not context.context().num_gpus():
+      self.skipTest('No GPUs found.')
+
+    with ops.device('/device:CPU:0'):
+      c1 = constant_op.constant(5.0)
+      c2 = constant_op.constant(7.0)
+
+    with ops.device('/device:GPU:0'):
+      g1 = constant_op.constant(11.0)
+      g2 = constant_op.constant(13.0)
+      g3 = constant_op.constant(17.0)
+
+    @function.defun
+    def func(g1, g2, c1, g3, c2):  # pylint: disable=unused-argument
+      # arguments g1 and g2 are unused and can be pruned by grappler.
+      return c1 * g3 * c2
+
+    result = func(g1, g2, c1, g3, c2)
+    self.assertEqual(result.numpy(), 5.0 * 7.0 * 17.0)
+
 if __name__ == '__main__':
   ops.enable_eager_execution(
       config=config_pb2.ConfigProto(device_count={'CPU': 4}))
diff --git a/tensorflow/python/eager/pywrap_tfe_test.py b/tensorflow/python/eager/pywrap_tfe_test.py
index 669fa08488..2abd4678af 100644
--- a/tensorflow/python/eager/pywrap_tfe_test.py
+++ b/tensorflow/python/eager/pywrap_tfe_test.py
@@ -254,6 +254,11 @@ class Tests(test.TestCase):
         "Value for attr 'num_split' of -1 must be at least minimum 1"):
       array_ops.split(value=[1, 2, 3], num_or_size_splits=-1)
 
+    with self.assertRaisesRegexp(
+        Exception,
+        "Value for attr 'num_split' of 0 must be at least minimum 1"):
+      array_ops.split(value=[1, 2, 3], num_or_size_splits=0)
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/ops/functional_ops.py b/tensorflow/python/ops/functional_ops.py
index df4be1d65a..b240d1e465 100644
--- a/tensorflow/python/ops/functional_ops.py
+++ b/tensorflow/python/ops/functional_ops.py
@@ -19,7 +19,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.core.protobuf import config_pb2
 from tensorflow.core.framework import attr_value_pb2
 from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
@@ -40,6 +39,7 @@ from tensorflow.python.ops.gen_functional_ops import remote_call
 from tensorflow.python.ops.gen_functional_ops import symbolic_gradient
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.util import compat
+from tensorflow.python.util import function_utils
 from tensorflow.python.util import nest
 from tensorflow.python.util.tf_export import tf_export
 
@@ -1024,17 +1024,6 @@ def For(start,
   return ret
 # pylint: enable=invalid-name,protected-access
 
-_rewriter_config_optimizer_disabled = None
-
-def _get_disabled_rewriter_config():
-  global _rewriter_config_optimizer_disabled
-  if _rewriter_config_optimizer_disabled is None:
-    config = config_pb2.ConfigProto()
-    rewriter_config = config.graph_options.rewrite_options
-    rewriter_config.disable_meta_optimizer = True
-    _rewriter_config_optimizer_disabled = config.SerializeToString()
-  return _rewriter_config_optimizer_disabled
-
 
 def partitioned_call(args, f, tout=None, executing_eagerly=None, config=None,
                      executor_type=None):
@@ -1071,7 +1060,7 @@ def partitioned_call(args, f, tout=None, executing_eagerly=None, config=None,
     executing_eagerly = context.executing_eagerly()
 
   if config is None:
-    config = _get_disabled_rewriter_config()
+    config = function_utils.get_disabled_rewriter_config()
 
   if executor_type is None:
     executor_type = ""
@@ -1103,7 +1092,8 @@ def partitioned_call(args, f, tout=None, executing_eagerly=None, config=None,
   # When running in graph mode, the graph and function graphs are optimized
   # (i.e. run through grappler) per the session options, so we can disable any
   # eager-specific rewriting.
-  config_proto = attr_value_pb2.AttrValue(s=_get_disabled_rewriter_config())
+  config_proto = attr_value_pb2.AttrValue(
+      s=function_utils.get_disabled_rewriter_config())
 
   graph = ops.get_default_graph()
   f.add_to_graph(graph)
diff --git a/tensorflow/python/util/function_utils.py b/tensorflow/python/util/function_utils.py
index a56dfbff8e..84e45bec6f 100644
--- a/tensorflow/python/util/function_utils.py
+++ b/tensorflow/python/util/function_utils.py
@@ -22,6 +22,7 @@ import functools
 
 import six
 
+from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.util import tf_decorator
 from tensorflow.python.util import tf_inspect
 
@@ -113,3 +114,16 @@ def get_func_code(func):
       return None
   else:
     raise ValueError('Argument must be callable')
+
+
+_rewriter_config_optimizer_disabled = None
+
+
+def get_disabled_rewriter_config():
+  global _rewriter_config_optimizer_disabled
+  if _rewriter_config_optimizer_disabled is None:
+    config = config_pb2.ConfigProto()
+    rewriter_config = config.graph_options.rewrite_options
+    rewriter_config.disable_meta_optimizer = True
+    _rewriter_config_optimizer_disabled = config.SerializeToString()
+  return _rewriter_config_optimizer_disabled
-- 
GitLab


From ef32bf6e22f6c9ec701d32b8625e87047aa08d15 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Fri, 11 Jan 2019 18:11:21 -0800
Subject: [PATCH 0642/1095] [XLA] Don't copy parameters in HloEvaluator.

Previously, every kParameter instruction resulted in a copy of the parameter's
literal.  This is unnecessary.

Speeds up one testcase in reduce_window_test_gpu from 17s to 13s.

PiperOrigin-RevId: 228976711
---
 .../compiler/xla/service/hlo_evaluator.cc      |  6 +++++-
 .../compiler/xla/service/hlo_evaluator.h       | 18 ++++++++++++++++--
 2 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc
index bcfa96d17f..7589c99202 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc
@@ -416,15 +416,19 @@ Status HloEvaluator::HandleGetDimensionSize(
 }
 
 Status HloEvaluator::HandleParameter(HloInstruction* parameter) {
+  // Nothing to do other than sanity checks. Parameters' values are stored in
+  // arg_literals_.
   CHECK_LT(parameter->parameter_number(), arg_literals_.size());
+
+#ifndef NDEBUG
   const Literal* input_literal = arg_literals_[parameter->parameter_number()];
   VLOG(2) << "Parameter evaluated to: " << input_literal->ToString();
   DCHECK(ShapeUtil::Equal(parameter->shape(), input_literal->shape()))
       << "parameter shape is: " << ShapeUtil::HumanString(parameter->shape())
       << ", but input literal shape is: "
       << ShapeUtil::HumanString(input_literal->shape());
+#endif
 
-  evaluated_[parameter] = input_literal->Clone();
   return Status::OK();
 }
 
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.h b/tensorflow/compiler/xla/service/hlo_evaluator.h
index 604b861913..ccb8af4fb0 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.h
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.h
@@ -243,13 +243,21 @@ class HloEvaluator : public DfsHloVisitorWithDefault {
   };
 
   // Returns the already-evaluated literal result for the instruction.
+  //
   // A Constant instruction is considered evaluated and its literal will be
   // returned directly without looking up the cache.
+  //
+  // Similarly, a Parameter instruction is considered evaluated and its literal
+  // is looked up in arg_literals.
+  //
   // Crash with log if the given instruction has not been evaluated previously.
   const Literal& GetEvaluatedLiteralFor(const HloInstruction* hlo) {
     if (hlo->IsConstant()) {
       return hlo->literal();
     }
+    if (hlo->opcode() == HloOpcode::kParameter) {
+      return *arg_literals_.at(hlo->parameter_number());
+    }
     auto it = evaluated_.find(hlo);
     CHECK(it != evaluated_.end())
         << "could not find evaluated value for: " << hlo->ToString();
@@ -257,12 +265,18 @@ class HloEvaluator : public DfsHloVisitorWithDefault {
   }
 
   // Tracks the HLO instruction and its evaluated literal result.
+  //
+  // Parameters and constants aren't stored here, see implementation of
+  // GetEvaluatedLiteralFor.
+  //
   // TODO(b/35950897): have better memory management here to free instructions
   // that are no longer a parent for any other subsequent instruction in
   // post-orderring.
+  //
   // Must be cleared for each evaluation.
-  // Storing Literal in place require the container to have pointer stability so
-  // we cannot use flat_hash_map any more.
+  //
+  // Storing Literal in place requires the container to have pointer stability
+  // so we cannot use flat_hash_map any more.
   absl::node_hash_map<const HloInstruction*, Literal> evaluated_;
 
   // Use fast path that uses eigen in the evaluator.
-- 
GitLab


From ff2cd836a36281f9fc01a9a04c2a4ea71ab42235 Mon Sep 17 00:00:00 2001
From: Ian Langmore <langmore@google.com>
Date: Fri, 11 Jan 2019 18:17:15 -0800
Subject: [PATCH 0643/1095] Auto-set `is_self_adjoint` hint for
 LinearOperatorScaledIdentity.

PiperOrigin-RevId: 228977102
---
 .../linalg/linear_operator_identity_test.py         |  3 ++-
 .../python/ops/linalg/linear_operator_identity.py   | 13 ++++++++++---
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py
index 14b5228bca..ea9ee99a58 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py
@@ -463,7 +463,7 @@ class LinearOperatorScaledIdentityTest(
         is_positive_definite=False, is_non_singular=True)
     self.assertFalse(operator.is_positive_definite)
     self.assertTrue(operator.is_non_singular)
-    self.assertTrue(operator.is_self_adjoint is None)
+    self.assertTrue(operator.is_self_adjoint)  # Auto-set due to real multiplier
 
   def test_identity_matmul(self):
     operator1 = linalg_lib.LinearOperatorIdentity(num_rows=2)
@@ -510,5 +510,6 @@ class LinearOperatorScaledIdentityTest(
         operator.inverse(),
         linalg_lib.LinearOperatorScaledIdentity)
 
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/ops/linalg/linear_operator_identity.py b/tensorflow/python/ops/linalg/linear_operator_identity.py
index 32b222cb2a..694557e50a 100644
--- a/tensorflow/python/ops/linalg/linear_operator_identity.py
+++ b/tensorflow/python/ops/linalg/linear_operator_identity.py
@@ -588,12 +588,19 @@ class LinearOperatorScaledIdentity(BaseLinearOperatorIdentity):
     """
     self._assert_proper_shapes = assert_proper_shapes
 
-    if not is_square:
-      raise ValueError("A ScaledIdentity operator is always square.")
-
     with ops.name_scope(name, values=[multiplier, num_rows]):
       self._multiplier = ops.convert_to_tensor(multiplier, name="multiplier")
 
+      # Check and auto-set hints.
+      if not self._multiplier.dtype.is_complex:
+        if is_self_adjoint is False:  # pylint: disable=g-bool-id-comparison
+          raise ValueError("A real diagonal operator is always self adjoint.")
+        else:
+          is_self_adjoint = True
+
+      if not is_square:
+        raise ValueError("A ScaledIdentity operator is always square.")
+
       super(LinearOperatorScaledIdentity, self).__init__(
           dtype=self._multiplier.dtype,
           is_non_singular=is_non_singular,
-- 
GitLab


From a79cdcf41968cdadce2c65ec4933a9cf5998910b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 11 Jan 2019 18:18:03 -0800
Subject: [PATCH 0644/1095] [tf.keras] Add warning message when non-shuffled
 dataset input is passed when `shuffle` parameter of fit() is set to True .

PiperOrigin-RevId: 228977169
---
 tensorflow/python/keras/engine/training.py    | 23 ++++++++++++++-----
 .../python/keras/engine/training_utils.py     | 21 +++++++++++++++++
 .../keras/engine/training_utils_test.py       | 14 +++++++++++
 3 files changed, 52 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 3e6c13232c..598766db74 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -824,6 +824,10 @@ class Model(Network):
       # Make sure that y, sample_weights, validation_split are not passed.
       training_utils.validate_dataset_input(x, y, sample_weight,
                                             validation_split)
+      if (isinstance(x, (dataset_ops.DatasetV1, dataset_ops.DatasetV2))
+          and shuffle):
+        training_utils.verify_dataset_shuffled(x)
+
       return self.fit_generator(
           x,
           steps_per_epoch=steps_per_epoch,
@@ -2155,15 +2159,19 @@ class Model(Network):
       raise NotImplementedError('`sample_weight` is currently not supported '
                                 'when using TPUStrategy.')
 
-    # Validates `steps` argument right at the beginning since we use it to
-    # construct the dataset object.
+    # Validates `steps` and `shuffle` arguments right at the beginning
+    # since we use it to construct the dataset object.
     # TODO(anjalisridhar): Remove this check once we refactor the
     # _standardize_user_data code path. This check is already present elsewhere
     # in the codebase.
-    if check_steps and isinstance(x, dataset_ops.DatasetV2) and steps is None:
-      raise ValueError('When using Datasets as input, '
-                       'you should specify the `{steps_name}` argument.'
-                       .format(steps_name=steps_name))
+    if isinstance(x, dataset_ops.DatasetV2):
+      if shuffle:
+        training_utils.verify_dataset_shuffled(x)
+
+      if check_steps and steps is None:
+        raise ValueError('When using Datasets as input, '
+                         'you should specify the `{steps_name}` argument.'
+                         .format(steps_name=steps_name))
 
     if ops.executing_eagerly_outside_functions():
       session = None
@@ -2286,6 +2294,9 @@ class Model(Network):
       # the tensors from the dataset and we output them.
       training_utils.validate_dataset_input(x, y, sample_weight,
                                             validation_split)
+      if shuffle:
+        training_utils.verify_dataset_shuffled(x)
+
       is_dataset = True
       if extract_tensors_from_dataset:
         # We do this for `train_on_batch`/etc.
diff --git a/tensorflow/python/keras/engine/training_utils.py b/tensorflow/python/keras/engine/training_utils.py
index 0c3beb4681..f7cbf4101a 100644
--- a/tensorflow/python/keras/engine/training_utils.py
+++ b/tensorflow/python/keras/engine/training_utils.py
@@ -45,6 +45,7 @@ from tensorflow.python.keras.utils.losses_utils import squeeze_or_expand_dimensi
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import weights_broadcast_ops
+from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.util import nest
 
 
@@ -1200,6 +1201,26 @@ def assert_not_shuffled(dataset):
     raise ValueError('Could not assert that dataset is not shuffled.')
 
 
+def verify_dataset_shuffled(x):
+  """Verifies that the dataset is shuffled.
+
+  Args:
+    x: Dataset passed as an input to the model.
+
+  Raises:
+    ValueError: if the dataset is not already shuffled.
+  """
+  assert isinstance(x, dataset_ops.DatasetV2)
+  try:
+    assert_not_shuffled(x)
+  except ValueError:
+    # Dataset may or may not be shuffled.
+    return
+  else:
+    logging.warning('Expected a shuffled dataset but input dataset `x` is '
+                    'not shuffled. Please invoke `shuffle()` on input dataset.')
+
+
 def is_dataset_or_iterator(data):
   return isinstance(data, (dataset_ops.DatasetV1,
                            dataset_ops.DatasetV2,
diff --git a/tensorflow/python/keras/engine/training_utils_test.py b/tensorflow/python/keras/engine/training_utils_test.py
index d8d8c58610..ec44252c9b 100644
--- a/tensorflow/python/keras/engine/training_utils_test.py
+++ b/tensorflow/python/keras/engine/training_utils_test.py
@@ -30,6 +30,7 @@ from tensorflow.python.keras import keras_parameterized
 from tensorflow.python.keras.engine import training_utils
 from tensorflow.python.keras.utils import tf_utils
 from tensorflow.python.platform import test
+from tensorflow.python.platform import tf_logging as logging
 
 
 class ModelInputsTest(test.TestCase):
@@ -187,6 +188,19 @@ class DatasetUtilsTest(test.TestCase, parameterized.TestCase):
       with self.assertRaises(expected_error):
         training_utils.assert_not_shuffled(dataset_fn())
 
+  def test_verify_dataset_shuffled(self):
+    dataset = dataset_ops.Dataset.range(5)
+    training_utils.assert_not_shuffled(dataset)
+
+    with test.mock.patch.object(logging, 'warning') as mock_log:
+      training_utils.verify_dataset_shuffled(dataset)
+      self.assertRegexpMatches(
+          str(mock_log.call_args),
+          'input dataset `x` is not shuffled.')
+
+    shuffled_dataset = dataset.shuffle(10)
+    training_utils.verify_dataset_shuffled(shuffled_dataset)
+
 
 class StandardizeWeightsTest(keras_parameterized.TestCase):
 
-- 
GitLab


From ba7f78b9b0535f38e3e12dcb0b095fc7a6e7bca5 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Fri, 11 Jan 2019 18:28:05 -0800
Subject: [PATCH 0645/1095] Adds v2 AUC metric.

PiperOrigin-RevId: 228978078
---
 tensorflow/python/keras/metrics.py            | 307 ++++++++++++++++--
 .../keras/metrics_confusion_matrix_test.py    | 249 ++++++++++++--
 tensorflow/python/keras/metrics_test.py       |  65 ++--
 .../python/keras/utils/metrics_utils.py       |  11 +
 4 files changed, 556 insertions(+), 76 deletions(-)

diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index c04487f175..ec09ad9ba8 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -1089,11 +1089,11 @@ class Precision(Metric):
     default_threshold = 0.5 if top_k is None else metrics_utils.NEG_INF
     self.thresholds = metrics_utils.parse_init_thresholds(
         thresholds, default_threshold=default_threshold)
-    self.tp = self.add_weight(
+    self.true_positives = self.add_weight(
         'true_positives',
         shape=(len(self.thresholds),),
         initializer=init_ops.zeros_initializer)
-    self.fp = self.add_weight(
+    self.false_positives = self.add_weight(
         'false_positives',
         shape=(len(self.thresholds),),
         initializer=init_ops.zeros_initializer)
@@ -1114,8 +1114,8 @@ class Precision(Metric):
     """
     return metrics_utils.update_confusion_matrix_variables(
         {
-            metrics_utils.ConfusionMatrix.TRUE_POSITIVES: self.tp,
-            metrics_utils.ConfusionMatrix.FALSE_POSITIVES: self.fp
+            metrics_utils.ConfusionMatrix.TRUE_POSITIVES: self.true_positives,
+            metrics_utils.ConfusionMatrix.FALSE_POSITIVES: self.false_positives
         },
         y_true,
         y_pred,
@@ -1125,7 +1125,8 @@ class Precision(Metric):
         sample_weight=sample_weight)
 
   def result(self):
-    result = math_ops.div_no_nan(self.tp, self.tp + self.fp)
+    result = math_ops.div_no_nan(self.true_positives,
+                                 self.true_positives + self.false_positives)
     return result[0] if len(self.thresholds) == 1 else result
 
   def reset_states(self):
@@ -1214,11 +1215,11 @@ class Recall(Metric):
     default_threshold = 0.5 if top_k is None else metrics_utils.NEG_INF
     self.thresholds = metrics_utils.parse_init_thresholds(
         thresholds, default_threshold=default_threshold)
-    self.tp = self.add_weight(
+    self.true_positives = self.add_weight(
         'true_positives',
         shape=(len(self.thresholds),),
         initializer=init_ops.zeros_initializer)
-    self.fn = self.add_weight(
+    self.false_negatives = self.add_weight(
         'false_negatives',
         shape=(len(self.thresholds),),
         initializer=init_ops.zeros_initializer)
@@ -1239,8 +1240,8 @@ class Recall(Metric):
     """
     return metrics_utils.update_confusion_matrix_variables(
         {
-            metrics_utils.ConfusionMatrix.TRUE_POSITIVES: self.tp,
-            metrics_utils.ConfusionMatrix.FALSE_NEGATIVES: self.fn
+            metrics_utils.ConfusionMatrix.TRUE_POSITIVES: self.true_positives,
+            metrics_utils.ConfusionMatrix.FALSE_NEGATIVES: self.false_negatives
         },
         y_true,
         y_pred,
@@ -1250,7 +1251,8 @@ class Recall(Metric):
         sample_weight=sample_weight)
 
   def result(self):
-    result = math_ops.div_no_nan(self.tp, self.tp + self.fn)
+    result = math_ops.div_no_nan(self.true_positives,
+                                 self.true_positives + self.false_negatives)
     return result[0] if len(self.thresholds) == 1 else result
 
   def reset_states(self):
@@ -1281,19 +1283,19 @@ class SensitivitySpecificityBase(Metric):
     if num_thresholds <= 0:
       raise ValueError('`num_thresholds` must be > 0.')
     self.value = value
-    self.tp = self.add_weight(
+    self.true_positives = self.add_weight(
         'true_positives',
         shape=(num_thresholds,),
         initializer=init_ops.zeros_initializer)
-    self.tn = self.add_weight(
+    self.true_negatives = self.add_weight(
         'true_negatives',
         shape=(num_thresholds,),
         initializer=init_ops.zeros_initializer)
-    self.fp = self.add_weight(
+    self.false_positives = self.add_weight(
         'false_positives',
         shape=(num_thresholds,),
         initializer=init_ops.zeros_initializer)
-    self.fn = self.add_weight(
+    self.false_negatives = self.add_weight(
         'false_negatives',
         shape=(num_thresholds,),
         initializer=init_ops.zeros_initializer)
@@ -1321,10 +1323,10 @@ class SensitivitySpecificityBase(Metric):
     """
     return metrics_utils.update_confusion_matrix_variables(
         {
-            metrics_utils.ConfusionMatrix.TRUE_POSITIVES: self.tp,
-            metrics_utils.ConfusionMatrix.TRUE_NEGATIVES: self.tn,
-            metrics_utils.ConfusionMatrix.FALSE_POSITIVES: self.fp,
-            metrics_utils.ConfusionMatrix.FALSE_NEGATIVES: self.fn,
+            metrics_utils.ConfusionMatrix.TRUE_POSITIVES: self.true_positives,
+            metrics_utils.ConfusionMatrix.TRUE_NEGATIVES: self.true_negatives,
+            metrics_utils.ConfusionMatrix.FALSE_POSITIVES: self.false_positives,
+            metrics_utils.ConfusionMatrix.FALSE_NEGATIVES: self.false_negatives,
         },
         y_true,
         y_pred,
@@ -1395,7 +1397,8 @@ class SensitivityAtSpecificity(SensitivitySpecificityBase):
 
   def result(self):
     # Calculate specificities at all the thresholds.
-    specificities = math_ops.div_no_nan(self.tn, self.tn + self.fp)
+    specificities = math_ops.div_no_nan(
+        self.true_negatives, self.true_negatives + self.false_positives)
 
     # Find the index of the threshold where the specificity is closest to the
     # given specificity.
@@ -1404,8 +1407,9 @@ class SensitivityAtSpecificity(SensitivitySpecificityBase):
     min_index = math_ops.cast(min_index, dtypes.int32)
 
     # Compute sensitivity at that index.
-    return math_ops.div_no_nan(self.tp[min_index],
-                               self.tp[min_index] + self.fn[min_index])
+    return math_ops.div_no_nan(
+        self.true_positives[min_index],
+        self.true_positives[min_index] + self.false_negatives[min_index])
 
   def get_config(self):
     config = {
@@ -1474,7 +1478,8 @@ class SpecificityAtSensitivity(SensitivitySpecificityBase):
 
   def result(self):
     # Calculate sensitivities at all the thresholds.
-    sensitivities = math_ops.div_no_nan(self.tp, self.tp + self.fn)
+    sensitivities = math_ops.div_no_nan(
+        self.true_positives, self.true_positives + self.false_negatives)
 
     # Find the index of the threshold where the sensitivity is closest to the
     # given specificity.
@@ -1483,8 +1488,9 @@ class SpecificityAtSensitivity(SensitivitySpecificityBase):
     min_index = math_ops.cast(min_index, dtypes.int32)
 
     # Compute specificity at that index.
-    return math_ops.div_no_nan(self.tn[min_index],
-                               self.tn[min_index] + self.fp[min_index])
+    return math_ops.div_no_nan(
+        self.true_negatives[min_index],
+        self.true_negatives[min_index] + self.false_positives[min_index])
 
   def get_config(self):
     config = {
@@ -1495,6 +1501,259 @@ class SpecificityAtSensitivity(SensitivitySpecificityBase):
     return dict(list(base_config.items()) + list(config.items()))
 
 
+class AUC(Metric):
+  """Computes the approximate AUC (Area under the curve) via a Riemann sum.
+
+  This metric creates four local variables, `true_positives`, `true_negatives`,
+  `false_positives` and `false_negatives` that are used to compute the AUC.
+  To discretize the AUC curve, a linearly spaced set of thresholds is used to
+  compute pairs of recall and precision values. The area under the ROC-curve is
+  therefore computed using the height of the recall values by the false positive
+  rate, while the area under the PR-curve is the computed using the height of
+  the precision values by the recall.
+
+  This value is ultimately returned as `auc`, an idempotent operation that
+  computes the area under a discretized curve of precision versus recall values
+  (computed using the aforementioned variables). The `num_thresholds` variable
+  controls the degree of discretization with larger numbers of thresholds more
+  closely approximating the true AUC. The quality of the approximation may vary
+  dramatically depending on `num_thresholds`.
+
+  For best results, `predictions` should be distributed approximately uniformly
+  in the range [0, 1] and not peaked around 0 or 1. The quality of the AUC
+  approximation may be poor if this is not the case. Setting `summation_method`
+  to 'minoring' or 'majoring' can help quantify the error in the approximation
+  by providing lower or upper bound estimate of the AUC.
+
+  If `sample_weight` is `None`, weights default to 1.
+  Use `sample_weight` of 0 to mask values.
+
+  Usage:
+
+  ```python
+  m = tf.keras.metrics.AUC(num_thresholds=3)
+  m.update_state([0, 0, 1, 1], [0, 0.5, 0.3, 0.9])
+
+  # threshold values are [0 - 1e-7, 0.5, 1 + 1e-7]
+  # tp = [2, 1, 0], fp = [2, 0, 0], fn = [0, 1, 2], tn = [0, 2, 2]
+  # recall = [1, 0.5, 0], fp_rate = [1, 0, 0]
+  # auc = ((((1+0.5)/2)*(1-0))+ (((0.5+0)/2)*(0-0))) = 0.75
+
+  print('Final result: ', m.result().numpy())  # Final result: 0.75
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile('sgd', loss='mse', metrics=[tf.keras.metrics.AUC()])
+  ```
+  """
+
+  def __init__(self,
+               num_thresholds=200,
+               curve=metrics_utils.AUCCurve.ROC,
+               summation_method=metrics_utils.AUCSummationMethod.INTERPOLATION,
+               name=None,
+               dtype=None):
+    """Creates an `AUC` instance.
+
+    Args:
+      num_thresholds: (Optional) Defaults to 200. The number of thresholds to
+        use when discretizing the roc curve. Values must be > 1.
+      curve: (Optional) Specifies the name of the curve to be computed, 'ROC'
+        [default] or 'PR' for the Precision-Recall-curve.
+      summation_method: (Optional) Specifies the Riemann summation method used
+        (https://en.wikipedia.org/wiki/Riemann_sum): 'interpolation' [default],
+          applies mid-point summation scheme for `ROC`. For PR-AUC, interpolates
+          (true/false) positives but not the ratio that is precision (see Davis
+          & Goadrich 2006 for details); 'minoring' that applies left summation
+          for increasing intervals and right summation for decreasing intervals;
+          'majoring' that does the opposite.
+      name: (Optional) string name of the metric instance.
+      dtype: (Optional) data type of the metric result.
+    """
+    # Validate configurations.
+    if num_thresholds <= 1:
+      raise ValueError('`num_thresholds` must be > 1.')
+    if curve not in list(metrics_utils.AUCCurve):
+      raise ValueError('Invalid curve: "{}". Valid options are: "{}"'.format(
+          curve, list(metrics_utils.AUCCurve)))
+    if summation_method not in list(metrics_utils.AUCSummationMethod):
+      raise ValueError(
+          'Invalid summation method: "{}". Valid options are: "{}"'.format(
+              summation_method, list(metrics_utils.AUCSummationMethod)))
+
+    # Update properties.
+    self.num_thresholds = num_thresholds
+    self.curve = curve
+    self.summation_method = summation_method
+    super(AUC, self).__init__(name=name, dtype=dtype)
+
+    # Create metric variables
+    self.true_positives = self.add_weight(
+        'true_positives',
+        shape=(num_thresholds,),
+        initializer=init_ops.zeros_initializer)
+    self.true_negatives = self.add_weight(
+        'true_negatives',
+        shape=(num_thresholds,),
+        initializer=init_ops.zeros_initializer)
+    self.false_positives = self.add_weight(
+        'false_positives',
+        shape=(num_thresholds,),
+        initializer=init_ops.zeros_initializer)
+    self.false_negatives = self.add_weight(
+        'false_negatives',
+        shape=(num_thresholds,),
+        initializer=init_ops.zeros_initializer)
+
+    # Compute `num_thresholds` thresholds in [0, 1]
+    thresholds = [
+        (i + 1) * 1.0 / (num_thresholds - 1) for i in range(num_thresholds - 2)
+    ]
+    self.thresholds = [0.0 - K.epsilon()] + thresholds + [1.0 + K.epsilon()]
+    # epsilon - to account for floating point imprecisions.
+
+  def update_state(self, y_true, y_pred, sample_weight=None):
+    """Accumulates confusion matrix statistics.
+
+    Args:
+      y_true: The ground truth values.
+      y_pred: The predicted values.
+      sample_weight: Optional weighting of each example. Defaults to 1. Can be a
+        `Tensor` whose rank is either 0, or the same rank as `y_true`, and must
+        be broadcastable to `y_true`.
+
+    Returns:
+      Update op.
+    """
+    return metrics_utils.update_confusion_matrix_variables({
+        metrics_utils.ConfusionMatrix.TRUE_POSITIVES: self.true_positives,
+        metrics_utils.ConfusionMatrix.TRUE_NEGATIVES: self.true_negatives,
+        metrics_utils.ConfusionMatrix.FALSE_POSITIVES: self.false_positives,
+        metrics_utils.ConfusionMatrix.FALSE_NEGATIVES: self.false_negatives,
+    }, y_true, y_pred, self.thresholds, sample_weight=sample_weight)
+
+  def interpolate_pr_auc(self):
+    """Interpolation formula inspired by section 4 of Davis & Goadrich 2006.
+
+    https://www.biostat.wisc.edu/~page/rocpr.pdf
+
+    Note here we derive & use a closed formula not present in the paper
+    as follows:
+
+      Precision = TP / (TP + FP) = TP / P
+
+    Modeling all of TP (true positive), FP (false positive) and their sum
+    P = TP + FP (predicted positive) as varying linearly within each interval
+    [A, B] between successive thresholds, we get
+
+      Precision slope = dTP / dP
+                      = (TP_B - TP_A) / (P_B - P_A)
+                      = (TP - TP_A) / (P - P_A)
+      Precision = (TP_A + slope * (P - P_A)) / P
+
+    The area within the interval is (slope / total_pos_weight) times
+
+      int_A^B{Precision.dP} = int_A^B{(TP_A + slope * (P - P_A)) * dP / P}
+      int_A^B{Precision.dP} = int_A^B{slope * dP + intercept * dP / P}
+
+    where intercept = TP_A - slope * P_A = TP_B - slope * P_B, resulting in
+
+      int_A^B{Precision.dP} = TP_B - TP_A + intercept * log(P_B / P_A)
+
+    Bringing back the factor (slope / total_pos_weight) we'd put aside, we get
+
+      slope * [dTP + intercept *  log(P_B / P_A)] / total_pos_weight
+
+    where dTP == TP_B - TP_A.
+
+    Note that when P_A == 0 the above calculation simplifies into
+
+      int_A^B{Precision.dTP} = int_A^B{slope * dTP} = slope * (TP_B - TP_A)
+
+    which is really equivalent to imputing constant precision throughout the
+    first bucket having >0 true positives.
+
+    Returns:
+      pr_auc: an approximation of the area under the P-R curve.
+    """
+    dtp = self.true_positives[:self.num_thresholds -
+                              1] - self.true_positives[1:]
+    p = self.true_positives + self.false_positives
+    dp = p[:self.num_thresholds - 1] - p[1:]
+
+    prec_slope = math_ops.div_no_nan(
+        dtp, math_ops.maximum(dp, 0), name='prec_slope')
+    intercept = self.true_positives[1:] - math_ops.multiply(prec_slope, p[1:])
+
+    safe_p_ratio = array_ops.where(
+        math_ops.logical_and(p[:self.num_thresholds - 1] > 0, p[1:] > 0),
+        math_ops.div_no_nan(
+            p[:self.num_thresholds - 1],
+            math_ops.maximum(p[1:], 0),
+            name='recall_relative_ratio'),
+        array_ops.ones_like(p[1:]))
+
+    return math_ops.reduce_sum(
+        math_ops.div_no_nan(
+            prec_slope * (dtp + intercept * math_ops.log(safe_p_ratio)),
+            math_ops.maximum(self.true_positives[1:] + self.false_negatives[1:],
+                             0),
+            name='pr_auc_increment'),
+        name='interpolate_pr_auc')
+
+  def result(self):
+    if (self.curve == metrics_utils.AUCCurve.PR and
+        self.summation_method == metrics_utils.AUCSummationMethod.INTERPOLATION
+       ):
+      # This use case is different and is handled separately.
+      return self.interpolate_pr_auc()
+
+    # Set `x` and `y` values for the curves based on `curve` config.
+    recall = math_ops.div_no_nan(self.true_positives,
+                                 self.true_positives + self.false_negatives)
+    if self.curve == metrics_utils.AUCCurve.ROC:
+      fp_rate = math_ops.div_no_nan(self.false_positives,
+                                    self.false_positives + self.true_negatives)
+      x = fp_rate
+      y = recall
+    else:  # curve == 'PR'.
+      precision = math_ops.div_no_nan(
+          self.true_positives, self.true_positives + self.false_positives)
+      x = recall
+      y = precision
+
+    # Find the rectangle heights based on `summation_method`.
+    if self.summation_method == metrics_utils.AUCSummationMethod.INTERPOLATION:
+      # Note: the case ('PR', 'interpolation') has been handled above.
+      heights = (y[:self.num_thresholds - 1] + y[1:]) / 2.
+    elif self.summation_method == metrics_utils.AUCSummationMethod.MINORING:
+      heights = math_ops.minimum(y[:self.num_thresholds - 1], y[1:])
+    else:  # self.summation_method = metrics_utils.AUCSummationMethod.MAJORING:
+      heights = math_ops.maximum(y[:self.num_thresholds - 1], y[1:])
+
+    # Sum up the areas of all the rectangles.
+    return math_ops.reduce_sum(
+        math_ops.multiply(x[:self.num_thresholds - 1] - x[1:], heights),
+        name=self.name)
+
+  def reset_states(self):
+    num_thresholds = len(self.thresholds)
+    for v in self.variables:
+      K.set_value(v, np.zeros((num_thresholds,)))
+
+  def get_config(self):
+    config = {
+        'num_thresholds': self.num_thresholds,
+        'curve': self.curve,
+        'summation_method': self.summation_method,
+    }
+    base_config = super(AUC, self).get_config()
+    return dict(list(base_config.items()) + list(config.items()))
+
+
 @keras_export('keras.metrics.CosineProximity')
 class CosineProximity(MeanMetricWrapper):
   """Computes the cosine distance between the labels and predictions.
diff --git a/tensorflow/python/keras/metrics_confusion_matrix_test.py b/tensorflow/python/keras/metrics_confusion_matrix_test.py
index 6db034b721..6a0fcfed07 100644
--- a/tensorflow/python/keras/metrics_confusion_matrix_test.py
+++ b/tensorflow/python/keras/metrics_confusion_matrix_test.py
@@ -25,6 +25,7 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import test_util
 from tensorflow.python.keras import metrics
+from tensorflow.python.keras.utils import metrics_utils
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import variables
@@ -472,22 +473,22 @@ class PrecisionTest(test.TestCase):
     y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))
     result = p_obj(y_true, y_pred)
     self.assertAlmostEqual(1, self.evaluate(result))
-    self.assertAlmostEqual(1, self.evaluate(p_obj.tp))
-    self.assertAlmostEqual(0, self.evaluate(p_obj.fp))
+    self.assertAlmostEqual(1, self.evaluate(p_obj.true_positives))
+    self.assertAlmostEqual(0, self.evaluate(p_obj.false_positives))
 
     y_pred = constant_op.constant([0.2, 0.1, 0, 0, 0.2], shape=(1, 5))
     y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))
     result = p_obj(y_true, y_pred)
     self.assertAlmostEqual(1, self.evaluate(result))
-    self.assertAlmostEqual(1, self.evaluate(p_obj.tp))
-    self.assertAlmostEqual(0, self.evaluate(p_obj.fp))
+    self.assertAlmostEqual(1, self.evaluate(p_obj.true_positives))
+    self.assertAlmostEqual(0, self.evaluate(p_obj.false_positives))
 
     y_pred = constant_op.constant([0.2, 0.1, 0.6, 0, 0.2], shape=(1, 5))
     y_true = constant_op.constant([0, 1, 0, 0, 0], shape=(1, 5))
     result = p_obj(y_true, y_pred)
     self.assertAlmostEqual(0.5, self.evaluate(result))
-    self.assertAlmostEqual(1, self.evaluate(p_obj.tp))
-    self.assertAlmostEqual(1, self.evaluate(p_obj.fp))
+    self.assertAlmostEqual(1, self.evaluate(p_obj.true_positives))
+    self.assertAlmostEqual(1, self.evaluate(p_obj.false_positives))
 
   def test_unweighted_top_k_and_class_id(self):
     p_obj = metrics.Precision(class_id=2, top_k=2)
@@ -497,15 +498,15 @@ class PrecisionTest(test.TestCase):
     y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))
     result = p_obj(y_true, y_pred)
     self.assertAlmostEqual(1, self.evaluate(result))
-    self.assertAlmostEqual(1, self.evaluate(p_obj.tp))
-    self.assertAlmostEqual(0, self.evaluate(p_obj.fp))
+    self.assertAlmostEqual(1, self.evaluate(p_obj.true_positives))
+    self.assertAlmostEqual(0, self.evaluate(p_obj.false_positives))
 
     y_pred = constant_op.constant([1, 1, 0.9, 1, 1], shape=(1, 5))
     y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))
     result = p_obj(y_true, y_pred)
     self.assertAlmostEqual(1, self.evaluate(result))
-    self.assertAlmostEqual(1, self.evaluate(p_obj.tp))
-    self.assertAlmostEqual(0, self.evaluate(p_obj.fp))
+    self.assertAlmostEqual(1, self.evaluate(p_obj.true_positives))
+    self.assertAlmostEqual(0, self.evaluate(p_obj.false_positives))
 
   def test_unweighted_top_k_and_threshold(self):
     p_obj = metrics.Precision(thresholds=.7, top_k=2)
@@ -515,8 +516,8 @@ class PrecisionTest(test.TestCase):
     y_true = constant_op.constant([0, 1, 1, 0, 1], shape=(1, 5))
     result = p_obj(y_true, y_pred)
     self.assertAlmostEqual(1, self.evaluate(result))
-    self.assertAlmostEqual(1, self.evaluate(p_obj.tp))
-    self.assertAlmostEqual(0, self.evaluate(p_obj.fp))
+    self.assertAlmostEqual(1, self.evaluate(p_obj.true_positives))
+    self.assertAlmostEqual(0, self.evaluate(p_obj.false_positives))
 
 
 @test_util.run_all_in_graph_and_eager_modes
@@ -676,22 +677,22 @@ class RecallTest(test.TestCase):
     y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))
     result = r_obj(y_true, y_pred)
     self.assertAlmostEqual(1, self.evaluate(result))
-    self.assertAlmostEqual(1, self.evaluate(r_obj.tp))
-    self.assertAlmostEqual(0, self.evaluate(r_obj.fn))
+    self.assertAlmostEqual(1, self.evaluate(r_obj.true_positives))
+    self.assertAlmostEqual(0, self.evaluate(r_obj.false_negatives))
 
     y_pred = constant_op.constant([0.2, 0.1, 0, 0, 0.2], shape=(1, 5))
     y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))
     result = r_obj(y_true, y_pred)
     self.assertAlmostEqual(0.5, self.evaluate(result))
-    self.assertAlmostEqual(1, self.evaluate(r_obj.tp))
-    self.assertAlmostEqual(1, self.evaluate(r_obj.fn))
+    self.assertAlmostEqual(1, self.evaluate(r_obj.true_positives))
+    self.assertAlmostEqual(1, self.evaluate(r_obj.false_negatives))
 
     y_pred = constant_op.constant([0.2, 0.1, 0.6, 0, 0.2], shape=(1, 5))
     y_true = constant_op.constant([0, 1, 0, 0, 0], shape=(1, 5))
     result = r_obj(y_true, y_pred)
     self.assertAlmostEqual(0.5, self.evaluate(result))
-    self.assertAlmostEqual(1, self.evaluate(r_obj.tp))
-    self.assertAlmostEqual(1, self.evaluate(r_obj.fn))
+    self.assertAlmostEqual(1, self.evaluate(r_obj.true_positives))
+    self.assertAlmostEqual(1, self.evaluate(r_obj.false_negatives))
 
   def test_unweighted_top_k_and_class_id(self):
     r_obj = metrics.Recall(class_id=2, top_k=2)
@@ -701,15 +702,15 @@ class RecallTest(test.TestCase):
     y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))
     result = r_obj(y_true, y_pred)
     self.assertAlmostEqual(1, self.evaluate(result))
-    self.assertAlmostEqual(1, self.evaluate(r_obj.tp))
-    self.assertAlmostEqual(0, self.evaluate(r_obj.fn))
+    self.assertAlmostEqual(1, self.evaluate(r_obj.true_positives))
+    self.assertAlmostEqual(0, self.evaluate(r_obj.false_negatives))
 
     y_pred = constant_op.constant([1, 1, 0.9, 1, 1], shape=(1, 5))
     y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))
     result = r_obj(y_true, y_pred)
     self.assertAlmostEqual(0.5, self.evaluate(result))
-    self.assertAlmostEqual(1, self.evaluate(r_obj.tp))
-    self.assertAlmostEqual(1, self.evaluate(r_obj.fn))
+    self.assertAlmostEqual(1, self.evaluate(r_obj.true_positives))
+    self.assertAlmostEqual(1, self.evaluate(r_obj.false_negatives))
 
   def test_unweighted_top_k_and_threshold(self):
     r_obj = metrics.Recall(thresholds=.7, top_k=2)
@@ -719,8 +720,8 @@ class RecallTest(test.TestCase):
     y_true = constant_op.constant([1, 1, 1, 0, 1], shape=(1, 5))
     result = r_obj(y_true, y_pred)
     self.assertAlmostEqual(0.25, self.evaluate(result))
-    self.assertAlmostEqual(1, self.evaluate(r_obj.tp))
-    self.assertAlmostEqual(3, self.evaluate(r_obj.fn))
+    self.assertAlmostEqual(1, self.evaluate(r_obj.true_positives))
+    self.assertAlmostEqual(3, self.evaluate(r_obj.false_negatives))
 
 
 @test_util.run_all_in_graph_and_eager_modes
@@ -915,5 +916,205 @@ class SpecificityAtSensitivityTest(test.TestCase, parameterized.TestCase):
       metrics.SpecificityAtSensitivity(0.4, num_thresholds=-1)
 
 
+@test_util.run_all_in_graph_and_eager_modes
+class AUCTest(test.TestCase):
+
+  def setup(self):
+    self.num_thresholds = 3
+    self.y_pred = constant_op.constant([0, 0.5, 0.3, 0.9], dtype=dtypes.float32)
+    self.y_true = constant_op.constant([0, 0, 1, 1])
+    self.sample_weight = [1, 2, 3, 4]
+
+    # threshold values are [0 - 1e-7, 0.5, 1 + 1e-7]
+    # y_pred when threshold = 0 - 1e-7  : [1, 1, 1, 1]
+    # y_pred when threshold = 0.5       : [0, 0, 0, 1]
+    # y_pred when threshold = 1 + 1e-7  : [0, 0, 0, 0]
+
+    # without sample_weight:
+    # tp = np.sum([[0, 0, 1, 1], [0, 0, 0, 1], [0, 0, 0, 0]], axis=1)
+    # fp = np.sum([[1, 1, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]], axis=1)
+    # fn = np.sum([[0, 0, 0, 0], [0, 0, 1, 0], [0, 0, 1, 1]], axis=1)
+    # tn = np.sum([[0, 0, 0, 0], [1, 1, 0, 0], [1, 1, 0, 0]], axis=1)
+
+    # tp = [2, 1, 0], fp = [2, 0, 0], fn = [0, 1, 2], tn = [0, 2, 2]
+
+    # with sample_weight:
+    # tp = np.sum([[0, 0, 3, 4], [0, 0, 0, 4], [0, 0, 0, 0]], axis=1)
+    # fp = np.sum([[1, 2, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]], axis=1)
+    # fn = np.sum([[0, 0, 0, 0], [0, 0, 3, 0], [0, 0, 3, 4]], axis=1)
+    # tn = np.sum([[0, 0, 0, 0], [1, 2, 0, 0], [1, 2, 0, 0]], axis=1)
+
+    # tp = [7, 4, 0], fp = [3, 0, 0], fn = [0, 3, 7], tn = [0, 3, 3]
+
+  def test_config(self):
+    auc_obj = metrics.AUC(
+        num_thresholds=100,
+        curve=metrics_utils.AUCCurve.PR,
+        summation_method=metrics_utils.AUCSummationMethod.MAJORING,
+        name='auc_1')
+    self.assertEqual(auc_obj.name, 'auc_1')
+    self.assertEqual(len(auc_obj.variables), 4)
+    self.assertEqual(auc_obj.num_thresholds, 100)
+    self.assertEqual(auc_obj.curve, metrics_utils.AUCCurve.PR)
+    self.assertEqual(auc_obj.summation_method,
+                     metrics_utils.AUCSummationMethod.MAJORING)
+
+    # Check save and restore config
+    auc_obj2 = metrics.AUC.from_config(auc_obj.get_config())
+    self.assertEqual(auc_obj2.name, 'auc_1')
+    self.assertEqual(len(auc_obj2.variables), 4)
+    self.assertEqual(auc_obj2.num_thresholds, 100)
+    self.assertEqual(auc_obj2.curve, metrics_utils.AUCCurve.PR)
+    self.assertEqual(auc_obj2.summation_method,
+                     metrics_utils.AUCSummationMethod.MAJORING)
+
+  def test_value_is_idempotent(self):
+    self.setup()
+    auc_obj = metrics.AUC(num_thresholds=3)
+    self.evaluate(variables.variables_initializer(auc_obj.variables))
+
+    # Run several updates.
+    update_op = auc_obj.update_state(self.y_true, self.y_pred)
+    for _ in range(10):
+      self.evaluate(update_op)
+
+    # Then verify idempotency.
+    initial_auc = self.evaluate(auc_obj.result())
+    for _ in range(10):
+      self.assertAllClose(initial_auc, self.evaluate(auc_obj.result()), 1e-3)
+
+  def test_unweighted_all_correct(self):
+    self.setup()
+    auc_obj = metrics.AUC()
+    self.evaluate(variables.variables_initializer(auc_obj.variables))
+    result = auc_obj(self.y_true, self.y_true)
+    self.assertEqual(self.evaluate(result), 1)
+
+  def test_unweighted(self):
+    self.setup()
+    auc_obj = metrics.AUC(num_thresholds=self.num_thresholds)
+    self.evaluate(variables.variables_initializer(auc_obj.variables))
+    result = auc_obj(self.y_true, self.y_pred)
+
+    # tp = [2, 1, 0], fp = [2, 0, 0], fn = [0, 1, 2], tn = [0, 2, 2]
+    # recall = [2/2, 1/(1+1), 0] = [1, 0.5, 0]
+    # fp_rate = [2/2, 0, 0] = [1, 0, 0]
+    # heights = [(1 + 0.5)/2, (0.5 + 0)/2] = [0.75, 0.25]
+    # widths = [(1 - 0), (0 - 0)] = [1, 0]
+    expected_result = (0.75 * 1 + 0.25 * 0)
+    self.assertAllClose(self.evaluate(result), expected_result, 1e-3)
+
+  def test_weighted_roc_interpolation(self):
+    self.setup()
+    auc_obj = metrics.AUC(num_thresholds=self.num_thresholds)
+    self.evaluate(variables.variables_initializer(auc_obj.variables))
+    result = auc_obj(self.y_true, self.y_pred, sample_weight=self.sample_weight)
+
+    # tp = [7, 4, 0], fp = [3, 0, 0], fn = [0, 3, 7], tn = [0, 3, 3]
+    # recall = [7/7, 4/(4+3), 0] = [1, 0.571, 0]
+    # fp_rate = [3/3, 0, 0] = [1, 0, 0]
+    # heights = [(1 + 0.571)/2, (0.571 + 0)/2] = [0.7855, 0.2855]
+    # widths = [(1 - 0), (0 - 0)] = [1, 0]
+    expected_result = (0.7855 * 1 + 0.2855 * 0)
+    self.assertAllClose(self.evaluate(result), expected_result, 1e-3)
+
+  def test_weighted_roc_majoring(self):
+    self.setup()
+    auc_obj = metrics.AUC(
+        num_thresholds=self.num_thresholds,
+        summation_method=metrics_utils.AUCSummationMethod.MAJORING)
+    self.evaluate(variables.variables_initializer(auc_obj.variables))
+    result = auc_obj(self.y_true, self.y_pred, sample_weight=self.sample_weight)
+
+    # tp = [7, 4, 0], fp = [3, 0, 0], fn = [0, 3, 7], tn = [0, 3, 3]
+    # recall = [7/7, 4/(4+3), 0] = [1, 0.571, 0]
+    # fp_rate = [3/3, 0, 0] = [1, 0, 0]
+    # heights = [max(1, 0.571), max(0.571, 0)] = [1, 0.571]
+    # widths = [(1 - 0), (0 - 0)] = [1, 0]
+    expected_result = (1 * 1 + 0.571 * 0)
+    self.assertAllClose(self.evaluate(result), expected_result, 1e-3)
+
+  def test_weighted_roc_minoring(self):
+    self.setup()
+    auc_obj = metrics.AUC(
+        num_thresholds=self.num_thresholds,
+        summation_method=metrics_utils.AUCSummationMethod.MINORING)
+    self.evaluate(variables.variables_initializer(auc_obj.variables))
+    result = auc_obj(self.y_true, self.y_pred, sample_weight=self.sample_weight)
+
+    # tp = [7, 4, 0], fp = [3, 0, 0], fn = [0, 3, 7], tn = [0, 3, 3]
+    # recall = [7/7, 4/(4+3), 0] = [1, 0.571, 0]
+    # fp_rate = [3/3, 0, 0] = [1, 0, 0]
+    # heights = [min(1, 0.571), min(0.571, 0)] = [0.571, 0]
+    # widths = [(1 - 0), (0 - 0)] = [1, 0]
+    expected_result = (0.571 * 1 + 0 * 0)
+    self.assertAllClose(self.evaluate(result), expected_result, 1e-3)
+
+  def test_weighted_pr_majoring(self):
+    self.setup()
+    auc_obj = metrics.AUC(
+        num_thresholds=self.num_thresholds,
+        curve=metrics_utils.AUCCurve.PR,
+        summation_method=metrics_utils.AUCSummationMethod.MAJORING)
+    self.evaluate(variables.variables_initializer(auc_obj.variables))
+    result = auc_obj(self.y_true, self.y_pred, sample_weight=self.sample_weight)
+
+    # tp = [7, 4, 0], fp = [3, 0, 0], fn = [0, 3, 7], tn = [0, 3, 3]
+    # precision = [7/(7+3), 4/4, 0] = [0.7, 1, 0]
+    # recall = [7/7, 4/(4+3), 0] = [1, 0.571, 0]
+    # heights = [max(0.7, 1), max(1, 0)] = [1, 1]
+    # widths = [(1 - 0.571), (0.571 - 0)] = [0.429, 0.571]
+    expected_result = (1 * 0.429 + 1 * 0.571)
+    self.assertAllClose(self.evaluate(result), expected_result, 1e-3)
+
+  def test_weighted_pr_minoring(self):
+    self.setup()
+    auc_obj = metrics.AUC(
+        num_thresholds=self.num_thresholds,
+        curve=metrics_utils.AUCCurve.PR,
+        summation_method=metrics_utils.AUCSummationMethod.MINORING)
+    self.evaluate(variables.variables_initializer(auc_obj.variables))
+    result = auc_obj(self.y_true, self.y_pred, sample_weight=self.sample_weight)
+
+    # tp = [7, 4, 0], fp = [3, 0, 0], fn = [0, 3, 7], tn = [0, 3, 3]
+    # precision = [7/(7+3), 4/4, 0] = [0.7, 1, 0]
+    # recall = [7/7, 4/(4+3), 0] = [1, 0.571, 0]
+    # heights = [min(0.7, 1), min(1, 0)] = [0.7, 0]
+    # widths = [(1 - 0.571), (0.571 - 0)] = [0.429, 0.571]
+    expected_result = (0.7 * 0.429 + 0 * 0.571)
+    self.assertAllClose(self.evaluate(result), expected_result, 1e-3)
+
+  def test_weighted_pr_interpolation(self):
+    self.setup()
+    auc_obj = metrics.AUC(
+        num_thresholds=self.num_thresholds,
+        curve=metrics_utils.AUCCurve.PR,
+        summation_method=metrics_utils.AUCSummationMethod.INTERPOLATION)
+    self.evaluate(variables.variables_initializer(auc_obj.variables))
+    result = auc_obj(self.y_true, self.y_pred, sample_weight=self.sample_weight)
+
+    # auc = (slope / Total Pos) * [dTP - intercept * log(Pb/Pa)]
+
+    # tp = [7, 4, 0], fp = [3, 0, 0], fn = [0, 3, 7], tn = [0, 3, 3]
+    # P = tp + fp = [10, 4, 0]
+    # dTP = [7-4, 4-0] = [3, 4]
+    # dP = [10-4, 4-0] = [6, 4]
+    # slope = dTP/dP = [0.5, 1]
+    # intercept = (TPa+(slope*Pa) = [(4 - 0.5*4), (0 - 1*0)] = [2, 0]
+    # (Pb/Pa) = (Pb/Pa) if Pb > 0 AND Pa > 0 else 1 = [10/4, 4/0] = [2.5, 1]
+    # auc * TotalPos = [(0.5 * (3 + 2 * log(2.5))), (1 * (4 + 0))]
+    #                = [2.416, 4]
+    # auc = [2.416, 4]/(tp[1:]+fn[1:])
+    expected_result = (2.416/7 + 4/7)
+    self.assertAllClose(self.evaluate(result), expected_result, 1e-3)
+
+  def test_invalid_num_thresholds(self):
+    with self.assertRaisesRegexp(ValueError, '`num_thresholds` must be > 1.'):
+      metrics.AUC(num_thresholds=-1)
+
+    with self.assertRaisesRegexp(ValueError, '`num_thresholds` must be > 1.'):
+      metrics.AUC(num_thresholds=1)
+
+
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py
index 6cf61d33d0..d28a615c02 100644
--- a/tensorflow/python/keras/metrics_test.py
+++ b/tensorflow/python/keras/metrics_test.py
@@ -1031,11 +1031,11 @@ class ResetStatesTest(keras_parameterized.TestCase):
     x = np.concatenate((np.ones((50, 4)), np.ones((50, 4))))
     y = np.concatenate((np.ones((50, 1)), np.zeros((50, 1))))
     model.evaluate(x, y)
-    self.assertEqual(self.evaluate(p_obj.tp), 50.)
-    self.assertEqual(self.evaluate(p_obj.fp), 50.)
+    self.assertEqual(self.evaluate(p_obj.true_positives), 50.)
+    self.assertEqual(self.evaluate(p_obj.false_positives), 50.)
     model.evaluate(x, y)
-    self.assertEqual(self.evaluate(p_obj.tp), 50.)
-    self.assertEqual(self.evaluate(p_obj.fp), 50.)
+    self.assertEqual(self.evaluate(p_obj.true_positives), 50.)
+    self.assertEqual(self.evaluate(p_obj.false_positives), 50.)
 
   def test_reset_states_recall(self):
     r_obj = metrics.Recall()
@@ -1043,11 +1043,11 @@ class ResetStatesTest(keras_parameterized.TestCase):
     x = np.concatenate((np.ones((50, 4)), np.zeros((50, 4))))
     y = np.concatenate((np.ones((50, 1)), np.ones((50, 1))))
     model.evaluate(x, y)
-    self.assertEqual(self.evaluate(r_obj.tp), 50.)
-    self.assertEqual(self.evaluate(r_obj.fn), 50.)
+    self.assertEqual(self.evaluate(r_obj.true_positives), 50.)
+    self.assertEqual(self.evaluate(r_obj.false_negatives), 50.)
     model.evaluate(x, y)
-    self.assertEqual(self.evaluate(r_obj.tp), 50.)
-    self.assertEqual(self.evaluate(r_obj.fn), 50.)
+    self.assertEqual(self.evaluate(r_obj.true_positives), 50.)
+    self.assertEqual(self.evaluate(r_obj.false_negatives), 50.)
 
   def test_reset_states_sensitivity_at_specificity(self):
     s_obj = metrics.SensitivityAtSpecificity(0.5, num_thresholds=1)
@@ -1056,16 +1056,13 @@ class ResetStatesTest(keras_parameterized.TestCase):
                         np.ones((25, 4))))
     y = np.concatenate((np.ones((25, 1)), np.zeros((25, 1)), np.ones((25, 1)),
                         np.zeros((25, 1))))
-    model.evaluate(x, y)
-    self.assertEqual(self.evaluate(s_obj.tp), 25.)
-    self.assertEqual(self.evaluate(s_obj.fp), 25.)
-    self.assertEqual(self.evaluate(s_obj.fn), 25.)
-    self.assertEqual(self.evaluate(s_obj.tn), 25.)
-    model.evaluate(x, y)
-    self.assertEqual(self.evaluate(s_obj.tp), 25.)
-    self.assertEqual(self.evaluate(s_obj.fp), 25.)
-    self.assertEqual(self.evaluate(s_obj.fn), 25.)
-    self.assertEqual(self.evaluate(s_obj.tn), 25.)
+
+    for _ in range(2):
+      model.evaluate(x, y)
+      self.assertEqual(self.evaluate(s_obj.true_positives), 25.)
+      self.assertEqual(self.evaluate(s_obj.false_positives), 25.)
+      self.assertEqual(self.evaluate(s_obj.false_negatives), 25.)
+      self.assertEqual(self.evaluate(s_obj.true_negatives), 25.)
 
   def test_reset_states_specificity_at_sensitivity(self):
     s_obj = metrics.SpecificityAtSensitivity(0.5, num_thresholds=1)
@@ -1074,16 +1071,28 @@ class ResetStatesTest(keras_parameterized.TestCase):
                         np.ones((25, 4))))
     y = np.concatenate((np.ones((25, 1)), np.zeros((25, 1)), np.ones((25, 1)),
                         np.zeros((25, 1))))
-    model.evaluate(x, y)
-    self.assertEqual(self.evaluate(s_obj.tp), 25.)
-    self.assertEqual(self.evaluate(s_obj.fp), 25.)
-    self.assertEqual(self.evaluate(s_obj.fn), 25.)
-    self.assertEqual(self.evaluate(s_obj.tn), 25.)
-    model.evaluate(x, y)
-    self.assertEqual(self.evaluate(s_obj.tp), 25.)
-    self.assertEqual(self.evaluate(s_obj.fp), 25.)
-    self.assertEqual(self.evaluate(s_obj.fn), 25.)
-    self.assertEqual(self.evaluate(s_obj.tn), 25.)
+
+    for _ in range(2):
+      model.evaluate(x, y)
+      self.assertEqual(self.evaluate(s_obj.true_positives), 25.)
+      self.assertEqual(self.evaluate(s_obj.false_positives), 25.)
+      self.assertEqual(self.evaluate(s_obj.false_negatives), 25.)
+      self.assertEqual(self.evaluate(s_obj.true_negatives), 25.)
+
+  def test_reset_states_auc(self):
+    auc_obj = metrics.AUC(num_thresholds=3)
+    model = _get_model([auc_obj])
+    x = np.concatenate((np.ones((25, 4)), np.zeros((25, 4)), np.zeros((25, 4)),
+                        np.ones((25, 4))))
+    y = np.concatenate((np.ones((25, 1)), np.zeros((25, 1)), np.ones((25, 1)),
+                        np.zeros((25, 1))))
+
+    for _ in range(2):
+      model.evaluate(x, y)
+      self.assertEqual(self.evaluate(auc_obj.true_positives[1]), 25.)
+      self.assertEqual(self.evaluate(auc_obj.false_positives[1]), 25.)
+      self.assertEqual(self.evaluate(auc_obj.false_negatives[1]), 25.)
+      self.assertEqual(self.evaluate(auc_obj.true_negatives[1]), 25.)
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/keras/utils/metrics_utils.py b/tensorflow/python/keras/utils/metrics_utils.py
index 7db128e8ab..108d6ee642 100644
--- a/tensorflow/python/keras/utils/metrics_utils.py
+++ b/tensorflow/python/keras/utils/metrics_utils.py
@@ -146,6 +146,17 @@ class ConfusionMatrix(Enum):
   FALSE_NEGATIVES = 'fn'
 
 
+class AUCCurve(Enum):
+  ROC = 'ROC'
+  PR = 'PR'
+
+
+class AUCSummationMethod(Enum):
+  INTERPOLATION = 'interpolation'
+  MAJORING = 'majoring'
+  MINORING = 'minoring'
+
+
 def update_confusion_matrix_variables(variables_to_update,
                                       y_true,
                                       y_pred,
-- 
GitLab


From bf87f8347ce24735d9d22173e289d7860a40498f Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Fri, 11 Jan 2019 19:03:45 -0800
Subject: [PATCH 0646/1095] Increase the tolerance value for
 quantization_mnist_test.

PiperOrigin-RevId: 228980452
---
 tensorflow/contrib/tensorrt/test/quantization_mnist_test.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py b/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py
index 79dde2a6b5..966476dc3e 100644
--- a/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py
+++ b/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py
@@ -274,7 +274,7 @@ class QuantizationAwareTrainingMNISTTest(test_util.TensorFlowTestCase):
         num_epochs=None,
         model_dir=model_dir)['accuracy']
     logging.info('accuracy_tf_native: %f', accuracy_tf_native)
-    self.assertAllClose(accuracy_tf_native, 0.9662)
+    self.assertAllClose(0.9662, accuracy_tf_native, rtol=1e-3, atol=1e-3)
 
     if trt_convert.get_linked_tensorrt_version()[0] < 5:
       return
@@ -286,7 +286,7 @@ class QuantizationAwareTrainingMNISTTest(test_util.TensorFlowTestCase):
         num_epochs=None,
         model_dir=model_dir)['accuracy']
     logging.info('accuracy_tf_trt: %f', accuracy_tf_trt)
-    self.assertAllClose(accuracy_tf_trt, 0.9677)
+    self.assertAllClose(0.9675, accuracy_tf_trt, rtol=1e-3, atol=1e-3)
 
 
 if __name__ == '__main__':
-- 
GitLab


From 0a958b9d697df62df0206e5e76665ee72e95aed9 Mon Sep 17 00:00:00 2001
From: Nick Felt <nickfelt@google.com>
Date: Fri, 11 Jan 2019 19:11:17 -0800
Subject: [PATCH 0647/1095] fix logic adding generated API directory to
 tensorflow.__path__

---
 tensorflow/api_template_v1.__init__.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/tensorflow/api_template_v1.__init__.py b/tensorflow/api_template_v1.__init__.py
index b293d0f153..eeca8f0d56 100644
--- a/tensorflow/api_template_v1.__init__.py
+++ b/tensorflow/api_template_v1.__init__.py
@@ -62,12 +62,15 @@ if '__all__' in vars():
   vars()['__all__'].append('contrib')
 
 from tensorflow.python.platform import flags  # pylint: disable=g-import-not-at-top
-from tensorflow.python.platform import app  # pylint: disable=g-import-not-at-top
-app.flags = flags
+# The 'app' module will be imported as part of the placeholder section above.
+app.flags = flags  # pylint: disable=undefined-variable
+
+# Also use 'app' module (choice is arbitrary) to derive the API directory below.
+_API_MODULE = app  # pylint: disable=undefined-variable
 
 # Make sure directory containing top level submodules is in
 # the __path__ so that "from tensorflow.foo import bar" works.
-_tf_api_dir = _os.path.dirname(_os.path.dirname(app.__file__))  # pylint: disable=undefined-variable
+_tf_api_dir = _os.path.dirname(_os.path.dirname(_API_MODULE.__file__))  # pylint: disable=undefined-variable
 if not hasattr(_current_module, '__path__'):
   __path__ = [_tf_api_dir]
 elif _tf_api_dir not in __path__:
-- 
GitLab


From 66e36030a8fce773c883ae1857fd067564138b27 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 11 Jan 2019 21:10:22 -0800
Subject: [PATCH 0648/1095] Switch callers away from deprecated methods of
 tf.distribute.Strategy.

PiperOrigin-RevId: 228987556
---
 .../python/checkpoint_utils_test.py           |  2 +-
 .../collective_all_reduce_strategy_test.py    |  8 ++--
 .../python/keras_optimizer_v2_test.py         |  3 +-
 .../distribute/python/metrics_v1_test.py      |  6 +--
 .../distribute/python/minimize_loss_test.py   | 28 ++++++-----
 .../distribute/python/moving_averages_test.py |  6 +--
 .../distribute/python/optimizer_v2_test.py    |  7 +--
 .../python/parameter_server_strategy_test.py  | 14 +++---
 .../contrib/distribute/python/step_fn.py      |  4 +-
 .../distribute/python/strategy_test_lib.py    | 48 ++++++++++---------
 tensorflow/python/distribute/values.py        | 25 ++++++----
 .../engine/distributed_training_utils.py      |  2 +-
 .../python/keras/optimizer_v2/optimizer_v2.py |  4 +-
 tensorflow/python/training/moving_averages.py |  4 +-
 14 files changed, 88 insertions(+), 73 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/checkpoint_utils_test.py b/tensorflow/contrib/distribute/python/checkpoint_utils_test.py
index 31bd0e996a..3ef8b9574a 100644
--- a/tensorflow/contrib/distribute/python/checkpoint_utils_test.py
+++ b/tensorflow/contrib/distribute/python/checkpoint_utils_test.py
@@ -71,7 +71,7 @@ class CheckpointUtilsWithDistributionStrategyTest(
 
     with ops.Graph().as_default() as g, distribution.scope():
       if in_replica_mode:
-        distribution.call_for_each_replica(init_and_verify, args=[g])
+        distribution.extended.call_for_each_replica(init_and_verify, args=[g])
       else:
         init_and_verify(g)
 
diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
index 62ff4b178e..9b6236fd9f 100644
--- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy_test.py
@@ -123,7 +123,7 @@ class CollectiveAllReduceStrategyTestBase(
       def step():
         """Perform one optimization step."""
         # Run forward & backward to get gradients, variables list.
-        g_v = d.call_for_each_replica(grad_fn, args=[one])
+        g_v = d.extended.call_for_each_replica(grad_fn, args=[one])
         # Update the variables using the gradients and the update() function.
         before_list = []
         after_list = []
@@ -135,7 +135,7 @@ class CollectiveAllReduceStrategyTestBase(
             g = d.extended.reduce_to(
                 reduce_util.ReduceOp.SUM, g, destinations=v)
             with ops.control_dependencies(
-                d.update(v, update, g, grouped=False)):
+                d.extended.update(v, update, args=(g,), group=False)):
               after_list.append(d.extended.read_var(v))
         return before_list, after_list
 
@@ -203,7 +203,7 @@ class CollectiveAllReduceStrategyTestBase(
          self.cached_session(config=config,
                              target=master_target) as sess:
       with d.scope():
-        train_op = d.call_for_each_replica(model_fn)
+        train_op = d.extended.call_for_each_replica(model_fn)
         train_op = d.group(d.unwrap(train_op))
 
       sess.run(variables.global_variables_initializer())
@@ -226,7 +226,7 @@ class CollectiveAllReduceStrategyTestBase(
                 1.0, 10.0, dtype=dtypes.float32))
         return array_ops.identity(x)
 
-      x = distribution.call_for_each_replica(model_fn)
+      x = distribution.extended.call_for_each_replica(model_fn)
       reduced_x = distribution.reduce(reduce_util.ReduceOp.MEAN, x)
       x = distribution.unwrap(x)[0]
 
diff --git a/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py b/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py
index cce93b3c10..5349794334 100644
--- a/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py
+++ b/tensorflow/contrib/distribute/python/keras_optimizer_v2_test.py
@@ -65,7 +65,8 @@ class MirroredStrategyOptimizerV2Test(test.TestCase, parameterized.TestCase):
 
     devices = ['/device:GPU:0', '/device:CPU:0']
     with distribution.scope():
-      (var, m, v, op, counter) = distribution.call_for_each_replica(create_fn)
+      (var, m, v, op,
+       counter) = distribution.extended.call_for_each_replica(create_fn)
       self.evaluate(variables.global_variables_initializer())
       var_val = [2.0, 2.0, 2.0]
       self.assertAllClose(
diff --git a/tensorflow/contrib/distribute/python/metrics_v1_test.py b/tensorflow/contrib/distribute/python/metrics_v1_test.py
index 7472f6dde4..96006276f6 100644
--- a/tensorflow/contrib/distribute/python/metrics_v1_test.py
+++ b/tensorflow/contrib/distribute/python/metrics_v1_test.py
@@ -99,12 +99,12 @@ class MetricsV1Test(test.TestCase, parameterized.TestCase):
           dataset_fn).make_initializable_iterator()
       if isinstance(distribution, tpu_strategy.TPUStrategy):
         def step_fn(ctx, inputs):
-          value, update = distribution.call_for_each_replica(
+          value, update = distribution.extended.call_for_each_replica(
               metric_fn, args=(inputs,))
           ctx.set_non_tensor_output(name="value", output=value)
           return distribution.group(update)
 
-        ctx = distribution.run_steps_on_dataset(
+        ctx = distribution.extended.experimental_run_steps_on_iterator(
             step_fn, iterator, iterations=distribution.extended.steps_per_run)
         update = ctx.run_op
         value = ctx.non_tensor_outputs["value"]
@@ -114,7 +114,7 @@ class MetricsV1Test(test.TestCase, parameterized.TestCase):
             distribution.num_replicas_in_sync *
             distribution.extended.steps_per_run)
       else:
-        value, update = distribution.call_for_each_replica(
+        value, update = distribution.extended.call_for_each_replica(
             metric_fn, args=(iterator.get_next(),))
         update = distribution.group(update)
         # TODO(josh11b): Once we switch to using a global batch size for input,
diff --git a/tensorflow/contrib/distribute/python/minimize_loss_test.py b/tensorflow/contrib/distribute/python/minimize_loss_test.py
index b0e24a53f6..37e6b418a0 100644
--- a/tensorflow/contrib/distribute/python/minimize_loss_test.py
+++ b/tensorflow/contrib/distribute/python/minimize_loss_test.py
@@ -67,12 +67,13 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
       def step_fn(ctx, inputs):
         del ctx  # Unused
         return distribution.group(
-            distribution.call_for_each_replica(model_fn, args=(inputs,)))
+            distribution.extended.call_for_each_replica(
+                model_fn, args=(inputs,)))
 
       iterator = self._get_iterator(distribution.distribute_dataset(dataset_fn))
 
       def run_step():
-        return distribution.run_steps_on_dataset(
+        return distribution.extended.experimental_run_steps_on_iterator(
             step_fn, iterator, iterations=2).run_op
 
       if not context.executing_eagerly():
@@ -105,7 +106,7 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
 
       def run_step():
         return distribution.group(
-            distribution.call_for_each_replica(
+            distribution.extended.call_for_each_replica(
                 model_fn, args=(iterator.get_next(),)))
 
       if not context.executing_eagerly():
@@ -157,12 +158,13 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
       def step_fn(ctx, inputs):
         del ctx  # Unused
         return distribution.group(
-            distribution.call_for_each_replica(model_fn, args=(inputs,)))
+            distribution.extended.call_for_each_replica(
+                model_fn, args=(inputs,)))
 
       iterator = self._get_iterator(distribution.distribute_dataset(dataset_fn))
 
       def run_step():
-        return distribution.run_steps_on_dataset(
+        return distribution.extended.experimental_run_steps_on_iterator(
             step_fn, iterator, iterations=1).run_op
 
       if not context.executing_eagerly():
@@ -189,7 +191,7 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
 
       self.assertEqual(
           get_expected_variables(optimizer_fn,
-                                 len(distribution.parameter_devices)),
+                                 len(distribution.extended.parameter_devices)),
           set(created_variables))
 
   @combinations.generate(
@@ -222,7 +224,8 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
       def step_fn(ctx, inputs):
         del ctx  # Unused
         fetches = distribution.unwrap(
-            distribution.call_for_each_replica(model_fn, args=(inputs,)))
+            distribution.extended.call_for_each_replica(
+                model_fn, args=(inputs,)))
         if update_ops_in_cross_replica_mode:
           fetches += tuple(ops.get_collection(ops.GraphKeys.UPDATE_OPS))
         return control_flow_ops.group(fetches)
@@ -230,7 +233,7 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
       iterator = self._get_iterator(distribution.distribute_dataset(dataset_fn))
 
       def run_step():
-        return distribution.run_steps_on_dataset(
+        return distribution.extended.experimental_run_steps_on_iterator(
             step_fn, iterator, iterations=1).run_op
 
       if not context.executing_eagerly():
@@ -316,12 +319,13 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
       def step_fn(ctx, inputs):
         del ctx  # Unused
         return distribution.group(
-            distribution.call_for_each_replica(model_fn, args=(inputs,)))
+            distribution.extended.call_for_each_replica(
+                model_fn, args=(inputs,)))
 
       iterator = self._get_iterator(distribution.distribute_dataset(dataset_fn))
 
       def run_step():
-        return distribution.run_steps_on_dataset(
+        return distribution.extended.experimental_run_steps_on_iterator(
             step_fn, iterator, iterations=1).run_op
 
       if not context.executing_eagerly():
@@ -398,7 +402,7 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
         return (train_op, loss)
 
       def step_fn(output_context, inputs):
-        (train_op, loss) = distribution.call_for_each_replica(
+        (train_op, loss) = distribution.extended.call_for_each_replica(
             model_fn, args=(output_context, inputs))
         output_context.set_last_step_output(
             name="cross_replica_loss_reduced",
@@ -425,7 +429,7 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
             "cross_replica_loss_not_reduced":
             distribution.unwrap(distribution.broadcast(initial_loss()))
         }
-        ctx = distribution.run_steps_on_dataset(
+        ctx = distribution.extended.experimental_run_steps_on_iterator(
             step_fn, iterator, iterations=2,
             initial_loop_values=initial_loop_values)
 
diff --git a/tensorflow/contrib/distribute/python/moving_averages_test.py b/tensorflow/contrib/distribute/python/moving_averages_test.py
index 8f13e9153e..c4622cdd2a 100644
--- a/tensorflow/contrib/distribute/python/moving_averages_test.py
+++ b/tensorflow/contrib/distribute/python/moving_averages_test.py
@@ -53,7 +53,7 @@ class AssignMovingAveragesTest(test.TestCase, parameterized.TestCase):
       return var, assign
 
     with distribution.scope(), self.cached_session() as sess:
-      var, assign = distribution.call_for_each_replica(replica_fn)
+      var, assign = distribution.extended.call_for_each_replica(replica_fn)
       variables.global_variables_initializer().run()
       self.assertAllClose([10.0, 11.0], var.eval())
       sess.run(distribution.unwrap(assign))
@@ -79,7 +79,7 @@ class AssignMovingAveragesTest(test.TestCase, parameterized.TestCase):
       return var, assign.op
 
     with distribution.scope(), self.cached_session() as sess:
-      var, assign_op = distribution.call_for_each_replica(replica_fn)
+      var, assign_op = distribution.extended.call_for_each_replica(replica_fn)
       variables.global_variables_initializer().run()
       self.assertAllClose([0.0, 0.0], var.eval())
       sess.run(distribution.unwrap(assign_op))
@@ -152,7 +152,7 @@ class AssignMovingAveragesTest(test.TestCase, parameterized.TestCase):
       return var, assign
 
     with distribution.scope(), self.cached_session() as sess:
-      var, assign = distribution.call_for_each_replica(replica_fn)
+      var, assign = distribution.extended.call_for_each_replica(replica_fn)
       variables.global_variables_initializer().run()
       self.assertAllClose([10.0, 11.0], var.eval())
       sess.run(distribution.unwrap(assign))
diff --git a/tensorflow/contrib/distribute/python/optimizer_v2_test.py b/tensorflow/contrib/distribute/python/optimizer_v2_test.py
index fa4705af7c..a832ef61f9 100644
--- a/tensorflow/contrib/distribute/python/optimizer_v2_test.py
+++ b/tensorflow/contrib/distribute/python/optimizer_v2_test.py
@@ -49,9 +49,10 @@ class MinimizeLossOptimizerV2Test(test.TestCase, parameterized.TestCase):
         iterator = ds.make_initializable_iterator()
 
       def run_step():
-        return control_flow_ops.group(distribution.unwrap(
-            distribution.call_for_each_replica(
-                model_fn, args=(iterator.get_next(),))))
+        return control_flow_ops.group(
+            distribution.unwrap(
+                distribution.extended.call_for_each_replica(
+                    model_fn, args=(iterator.get_next(),))))
 
       if not context.executing_eagerly():
         with self.cached_session() as sess:
diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
index 2d4f8a9be5..c7760a8431 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
@@ -236,7 +236,7 @@ class ParameterServerStrategyTestBase(
         self.assertIn('/job:ps/', h.device)
         return y_add, z_add, f
 
-      y, z, f = d.call_for_each_replica(model_fn)
+      y, z, f = d.extended.call_for_each_replica(model_fn)
       self.assertNotEqual(y, None)
       self.assertNotEqual(z, None)
       self.assertNotEqual(f, None)
@@ -252,7 +252,7 @@ class ParameterServerStrategyTestBase(
       self, task_type, task_id, num_gpus, use_core_strategy=False):
     d, _, sess_config = self._get_test_objects(
         task_type, task_id, num_gpus, use_core_strategy=use_core_strategy)
-    num_shards = len(d.parameter_devices)
+    num_shards = len(d.extended.parameter_devices)
     partitioner = partitioned_variables.fixed_size_partitioner(num_shards)
     with ops.Graph().as_default(), \
          self.cached_session(target=self._default_target,
@@ -286,7 +286,7 @@ class ParameterServerStrategyTestBase(
 
         return x_add
 
-      x = d.call_for_each_replica(model_fn)
+      x = d.extended.call_for_each_replica(model_fn)
 
       if context.num_gpus() >= 1:
         variables.global_variables_initializer().run()
@@ -387,7 +387,7 @@ class ParameterServerStrategyTestBase(
             device_util.canonicalize(h.device))
         return y_add, z_add, f
 
-      y, z, f = d.call_for_each_replica(model_fn)
+      y, z, f = d.extended.call_for_each_replica(model_fn)
       self.assertNotEqual(y, None)
       self.assertNotEqual(z, None)
       self.assertNotEqual(f, None)
@@ -438,7 +438,7 @@ class ParameterServerStrategyTestBase(
         train_op = control_flow_ops.group(x_add, y_add, z_add)
         return x, y, z, train_op
 
-      x, y, z, train_op = d.call_for_each_replica(model_fn)
+      x, y, z, train_op = d.extended.call_for_each_replica(model_fn)
       train_op = d.group(train_op)
 
       if context.num_gpus() < d.extended._num_gpus_per_worker:
@@ -519,7 +519,7 @@ class ParameterServerStrategyTestBase(
       def step():
         """Perform one optimization step."""
         # Run forward & backward to get gradients, variables list.
-        g_v = d.call_for_each_replica(grad_fn, args=(one,))
+        g_v = d.extended.call_for_each_replica(grad_fn, args=(one,))
         # Update the variables using the gradients and the update() function.
         before_list = []
         after_list = []
@@ -531,7 +531,7 @@ class ParameterServerStrategyTestBase(
             g = d.extended.reduce_to(
                 reduce_util.ReduceOp.SUM, g, destinations=v)
             with ops.control_dependencies(
-                d.update(v, update, g, grouped=False)):
+                d.extended.update(v, update, args=(g,), group=False)):
               after_list.append(d.extended.read_var(v))
         return before_list, after_list
 
diff --git a/tensorflow/contrib/distribute/python/step_fn.py b/tensorflow/contrib/distribute/python/step_fn.py
index faeb96bcb7..24b95f748b 100644
--- a/tensorflow/contrib/distribute/python/step_fn.py
+++ b/tensorflow/contrib/distribute/python/step_fn.py
@@ -99,7 +99,7 @@ class StandardSingleLossStep(StandardInputStep):
         gradients_fn = backprop.implicit_grad(self._loss_fn)
         gradients_fn = optimizer_lib.get_filtered_grad_fn(gradients_fn)
 
-        grads_and_vars = self.distribution.call_for_each_replica(
+        grads_and_vars = self.distribution.extended.call_for_each_replica(
             gradients_fn, args=(ctx, inputs))
         # If threads use layers, then we need to run the first step
         # sequentially, so that layers.build() is not executed in parallel.
@@ -109,6 +109,6 @@ class StandardSingleLossStep(StandardInputStep):
             self.distribution, grads_and_vars)
 
       # TODO(priyag): Return the outputs, context, etc as well.
-      ctx = self.distribution.run_steps_on_dataset(
+      ctx = self.distribution.extended.experimental_run_steps_on_iterator(
           step_fn, self._iterator, self._iterations_per_step)
       return ctx.run_op
diff --git a/tensorflow/contrib/distribute/python/strategy_test_lib.py b/tensorflow/contrib/distribute/python/strategy_test_lib.py
index 7455cbd02a..2e2ee92b6e 100644
--- a/tensorflow/contrib/distribute/python/strategy_test_lib.py
+++ b/tensorflow/contrib/distribute/python/strategy_test_lib.py
@@ -45,25 +45,26 @@ class _TestException(Exception):
   pass
 
 
-# May be the argument to either distribution.call_for_each_replica() or
+# May be the argument to either distribution.extended.call_for_each_replica() or
 # get_replica_context().merge_call()
 def _raise_exception_fn(_=None):
   raise _TestException()
 
 
-# Must be the argument to a distribution.call_for_each_replica() call, calls a
-# get_replica_context().merge_call() that raises an exception.
+# Must be the argument to a distribution.extended.call_for_each_replica() call,
+# calls a get_replica_context().merge_call() that raises an exception.
 def _merge_raises_fn():
   ds_context.get_replica_context().merge_call(_raise_exception_fn)
 
 
 # Must be the argument to a get_replica_context().merge_call() call, calls
-# dist.call_for_each_replica() with a function that raises an exception.
+# dist.extended.call_for_each_replica() with a function that raises an
+# exception.
 def _call_raises_fn(dist):
-  dist.call_for_each_replica(_raise_exception_fn)
+  dist.extended.call_for_each_replica(_raise_exception_fn)
 
 
-# Must be the argument to a distribution.call_for_each_replica() call,
+# Must be the argument to a distribution.extended.call_for_each_replica() call,
 # calls a get_replica_context().merge_call() that calls a
 # call_for_each_replica() that raises an exception.
 def _merge_call_raises_fn():
@@ -71,15 +72,16 @@ def _merge_call_raises_fn():
 
 
 # Must be the argument to a get_replica_context().merge_call() call, calls
-# dist.call_for_each_replica() with a function that calls a
+# dist.extended.call_for_each_replica() with a function that calls a
 # get_replica_context().merge_call() that raises an exception.
 def _call_merge_raises_fn(dist):
-  dist.call_for_each_replica(_merge_raises_fn)
+  dist.extended.call_for_each_replica(_merge_raises_fn)
 
 
-# Must be the argument to a distribution.call_for_each_replica() call, calls a
-# get_replica_context().merge_call() that calls a call_for_each_replica() that
-# calls a get_replica_context().merge_call() that raises an exception.
+# Must be the argument to a distribution.extended.call_for_each_replica() call,
+# calls a get_replica_context().merge_call() that calls a
+# call_for_each_replica() that calls a get_replica_context().merge_call() that
+# raises an exception.
 def _merge_call_merge_raises_fn():
   ds_context.get_replica_context().merge_call(_call_merge_raises_fn)
 
@@ -110,7 +112,7 @@ class DistributionTestBase(test.TestCase):
       def step():
         """Perform one optimization step."""
         # Run forward & backward to get gradients, variables list.
-        g_v = d.call_for_each_replica(grad_fn, args=(one,))
+        g_v = d.extended.call_for_each_replica(grad_fn, args=(one,))
 
         # Update the variables using the gradients and the update() function.
         before_list = []
@@ -122,8 +124,8 @@ class DistributionTestBase(test.TestCase):
           with ops.control_dependencies([fetched]):
             g = d.extended.reduce_to(
                 reduce_util.ReduceOp.SUM, g, destinations=v)
-            with ops.control_dependencies(d.update(
-                v, update, g, grouped=False)):
+            with ops.control_dependencies(d.extended.update(
+                v, update, args=(g,), group=False)):
               after_list.append(d.extended.read_var(v))
         return before_list, after_list
 
@@ -166,7 +168,7 @@ class DistributionTestBase(test.TestCase):
       def step():
         """Perform one optimization step."""
         # Run forward & backward to get gradients, variables list.
-        g_v = d.call_for_each_replica(grad_fn, args=(one,))
+        g_v = d.extended.call_for_each_replica(grad_fn, args=(one,))
 
         # Update the variables using the gradients and the update() function.
         before_list = []
@@ -177,8 +179,8 @@ class DistributionTestBase(test.TestCase):
           with ops.control_dependencies([fetched]):
             g = d.extended.reduce_to(
                 reduce_util.ReduceOp.SUM, g, destinations=v)
-            with ops.control_dependencies(d.update(
-                v, update, g, grouped=False)):
+            with ops.control_dependencies(d.extended.update(
+                v, update, args=(g,), group=False)):
               after_list.append(d.extended.read_var(v))
         return before_list, after_list
 
@@ -206,20 +208,20 @@ class DistributionTestBase(test.TestCase):
         self.assertFalse(expected_devices[replica_id])
         expected_devices[replica_id] = True
 
-      d.call_for_each_replica(mark_devices_fn)
+      d.extended.call_for_each_replica(mark_devices_fn)
       self.assertAllEqual(expected_devices,
                           [True] * len(d.extended.worker_devices))
 
   def _test_call_and_merge_exceptions(self, dist):
     with dist.scope():
       with self.assertRaises(_TestException):
-        dist.call_for_each_replica(_raise_exception_fn)
+        dist.extended.call_for_each_replica(_raise_exception_fn)
       with self.assertRaises(_TestException):
-        dist.call_for_each_replica(_merge_raises_fn)
+        dist.extended.call_for_each_replica(_merge_raises_fn)
       with self.assertRaises(_TestException):
-        dist.call_for_each_replica(_merge_call_raises_fn)
+        dist.extended.call_for_each_replica(_merge_call_raises_fn)
       with self.assertRaises(_TestException):
-        dist.call_for_each_replica(_merge_call_merge_raises_fn)
+        dist.extended.call_for_each_replica(_merge_call_merge_raises_fn)
 
   def _input_fn_to_test_input_context(self,
                                       dataset_fn,
@@ -291,7 +293,7 @@ class DistributionTestBase(test.TestCase):
         value = global_step.read_value()
         return train_op, value
 
-      train_ops, value = strategy.call_for_each_replica(model_fn)
+      train_ops, value = strategy.extended.call_for_each_replica(model_fn)
       self.evaluate(strategy.group(train_ops))
       global_step_tensors = strategy.unwrap(value)
       global_step_values = self.evaluate(global_step_tensors)
diff --git a/tensorflow/python/distribute/values.py b/tensorflow/python/distribute/values.py
index 2a57d5cfc7..ec0ea4bc2d 100644
--- a/tensorflow/python/distribute/values.py
+++ b/tensorflow/python/distribute/values.py
@@ -653,8 +653,9 @@ class MirroredVariable(DistributedVariable, Mirrored,
         return f(v, *args, **kwargs)
 
       # We are calling assign on the mirrored variable in cross replica context,
-      # use `strategy.update()` to update the variable.
-      return self._distribute_strategy.update(self, f, *args, **kwargs)
+      # use `strategy.extended.update()` to update the variable.
+      return self._distribute_strategy.extended.update(
+          self, f, args=args, kwargs=kwargs)
     else:
       _assert_replica_context(self._distribute_strategy)
       # We are calling an assign function on the mirrored variable in replica
@@ -669,7 +670,8 @@ class MirroredVariable(DistributedVariable, Mirrored,
 
       def merge_fn(strategy, value, *other_args, **other_kwargs):
         v = _apply_aggregation(strategy, value, self._aggregation, self)
-        return strategy.update(self, f, v, *other_args, **other_kwargs)
+        return strategy.extended.update(
+            self, f, args=(v,) + other_args, kwargs=other_kwargs)
 
       return distribution_strategy_context.get_replica_context().merge_call(
           merge_fn, args=args, kwargs=kwargs)
@@ -721,7 +723,7 @@ class MirroredVariable(DistributedVariable, Mirrored,
 # allowing instances of the class to be used as tensors.
 def _tensor_conversion_mirrored(var, dtype=None, name=None, as_ref=False):
   # Try to avoid assignments to and other mutations of MirroredVariable
-  # state except through a DistributionStrategy.update() call.
+  # state except through a DistributionStrategy.extended.update() call.
   assert not as_ref
   return ops.internal_convert_to_tensor(
       var.get(), dtype=dtype, name=name, as_ref=as_ref)
@@ -914,7 +916,8 @@ class TPUMirroredVariable(checkpointable.CheckpointableBase):
     f = kwargs.pop("f")
     if distribution_strategy_context.in_cross_replica_context():
       if _enclosing_tpu_context() is not None:
-        return self._distribute_strategy.update(self, f, *args, **kwargs)
+        return self._distribute_strategy.extended.update(
+            self, f, args=args, kwargs=kwargs)
 
       update_device = distribute_lib.get_update_device()
       # We are calling update on the mirrored variable in cross replica context.
@@ -924,7 +927,8 @@ class TPUMirroredVariable(checkpointable.CheckpointableBase):
         v = self._get(device=update_device)
         return f(v, *args, **kwargs)
 
-      return self._distribute_strategy.update(self, f, *args, **kwargs)
+      return self._distribute_strategy.extended.update(
+          self, f, args=args, kwargs=kwargs)
     else:
       _assert_replica_context(self._distribute_strategy)
       # We are calling an assign function on the mirrored variable in replica
@@ -939,7 +943,8 @@ class TPUMirroredVariable(checkpointable.CheckpointableBase):
 
       def merge_fn(strategy, value, *other_args, **other_kwargs):
         v = _apply_aggregation(strategy, value, self._aggregation, self)
-        return strategy.update(self, f, v, *other_args, **other_kwargs)
+        return strategy.extended.update(
+            self, f, args=(v,) + other_args, kwargs=other_kwargs)
 
       return distribution_strategy_context.get_replica_context().merge_call(
           merge_fn, args=args, kwargs=kwargs)
@@ -1459,7 +1464,8 @@ class AggregatingVariable(checkpointable.CheckpointableBase):
 
       # We are calling an assign function in cross replica context, wrap it in
       # an update call.
-      return self._distribute_strategy.update(self, f, *args, **kwargs)
+      return self._distribute_strategy.extended.update(
+          self, f, args=args, kwargs=kwargs)
     else:
       replica_context = distribution_strategy_context.get_replica_context()
       assert replica_context
@@ -1473,7 +1479,8 @@ class AggregatingVariable(checkpointable.CheckpointableBase):
 
       def merge_fn(strategy, value, *other_args, **other_kwargs):
         v = _apply_aggregation(strategy, value, self._aggregation, self)
-        return strategy.update(self, f, v, *other_args, **other_kwargs)
+        return strategy.extended.update(
+            self, f, args=(v,) + other_args, kwargs=other_kwargs)
 
       return replica_context.merge_call(merge_fn, args=args, kwargs=kwargs)
 
diff --git a/tensorflow/python/keras/engine/distributed_training_utils.py b/tensorflow/python/keras/engine/distributed_training_utils.py
index 0185e87cb0..27ad6a5c0b 100644
--- a/tensorflow/python/keras/engine/distributed_training_utils.py
+++ b/tensorflow/python/keras/engine/distributed_training_utils.py
@@ -816,7 +816,7 @@ def _make_eager_execution_function(model, mode):
   with K.get_graph().as_default(), strategy.scope():
     # Create train ops on each of the devices when we call
     # `_per_device_fit_function`.
-    (grouped_inputs, grouped_outputs) = strategy.call_for_each_replica(
+    (grouped_inputs, grouped_outputs) = strategy.extended.call_for_each_replica(
         _per_device_function, args=(model._distributed_model,))
 
     # Unwrap all the per device values returned from `call_for_each_replica`.
diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
index 91dab9338d..0415c97a0b 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
@@ -876,8 +876,8 @@ def merge_grads(grads_and_vars):
   """Merge gradients from different replicas."""
 
   def merge_grad_fn(strategy, grads_and_vars):
-    reduced_grads = strategy.batch_reduce(ds_reduce_util.ReduceOp.SUM,
-                                          grads_and_vars)
+    reduced_grads = strategy.extended.batch_reduce_to(
+        ds_reduce_util.ReduceOp.SUM, grads_and_vars)
     return reduced_grads
 
   return distribute_ctx.get_replica_context().merge_call(
diff --git a/tensorflow/python/training/moving_averages.py b/tensorflow/python/training/moving_averages.py
index 4b267dfb98..6efcab28c5 100644
--- a/tensorflow/python/training/moving_averages.py
+++ b/tensorflow/python/training/moving_averages.py
@@ -98,12 +98,12 @@ def assign_moving_average(variable, value, decay, zero_debias=True, name=None):
       def merge_fn(strategy, v, value):
         value = strategy.extended.reduce_to(
             ds_reduce_util.ReduceOp.MEAN, value, v)
-        return strategy.update(v, update_fn, value)
+        return strategy.extended.update(v, update_fn, args=(value,))
 
       return replica_context.merge_call(merge_fn, args=(variable, value))
     else:
       strategy = distribution_strategy_context.get_cross_replica_context()
-      return strategy.update(variable, update_fn, value)
+      return strategy.extended.update(variable, update_fn, args=(value,))
 
 
 def weighted_moving_average(value,
-- 
GitLab


From 7d707dcce6f7755790a23271bedc1c737b643dc7 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sat, 12 Jan 2019 01:02:32 -0800
Subject: [PATCH 0649/1095] compat: Update forward compatibility horizon to
 2019-01-12

PiperOrigin-RevId: 229000037
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 4028302362..1f4840ff38 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 11)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 12)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 3a10998daf52ea922aa7ff895af3012afe956866 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Sat, 12 Jan 2019 15:56:44 -0800
Subject: [PATCH 0650/1095] [XLA] Don't cache Executables resulting from
 Client::Execute().

Previously, Client::Execute() would cache the Executables it created forever.
This is surprising, and it causes long-running tests (the primary user of the
plain Client API) to consume a lot of resources.

Change the Client API implementation so that we don't cache these Executables
at all.

PiperOrigin-RevId: 229038268
---
 tensorflow/compiler/xla/client/client.cc    | 86 ++++++++++-----------
 tensorflow/compiler/xla/client/client.h     | 10 +++
 tensorflow/compiler/xla/rpc/grpc_service.cc | 16 ++++
 tensorflow/compiler/xla/rpc/grpc_service.h  |  7 ++
 tensorflow/compiler/xla/service/service.cc  | 51 ++++++++----
 tensorflow/compiler/xla/service/service.h   |  5 +-
 6 files changed, 114 insertions(+), 61 deletions(-)

diff --git a/tensorflow/compiler/xla/client/client.cc b/tensorflow/compiler/xla/client/client.cc
index 43127cae1e..4f020bcec2 100644
--- a/tensorflow/compiler/xla/client/client.cc
+++ b/tensorflow/compiler/xla/client/client.cc
@@ -278,53 +278,51 @@ StatusOr<std::unique_ptr<GlobalData>> Client::Execute(
     const XlaComputation& computation, absl::Span<GlobalData* const> arguments,
     const ExecutionOptions* execution_options,
     ExecutionProfile* execution_profile) {
-  if (execution_options != nullptr &&
-      execution_options->device_handles_size() > 1) {
-    std::vector<XlaComputationInstance> computation_instances = {
-        XlaComputationInstance{
-            computation,
-            std::vector<GlobalData*>(arguments.begin(), arguments.end()),
-            *execution_options, execution_profile}};
-    TF_ASSIGN_OR_RETURN(auto results, ExecuteParallel(computation_instances));
-    // The result selection is a bit hacky, but better than assuming it is
-    // device 0.
-    //
-    // TODO(b/118493728): Allow Execute to return one result per computation.
-    for (int64 i = 0; i < results.size(); i++) {
-      TF_ASSIGN_OR_RETURN(const Shape& shape, GetShape(*results[i]));
-      if (!ShapeUtil::IsEmptyTuple(shape)) {
-        VLOG(3) << "Fetching result from device " << i << ": "
-                << ShapeUtil::HumanString(shape);
-        return std::move(results[i]);
-      }
+  // Create an ExecutionOptions if necessary, or set its DeviceHandles.
+  absl::optional<ExecutionOptions> options_storage;
+  if (!execution_options || execution_options->device_handles().empty()) {
+    if (execution_options) {
+      options_storage.emplace(*execution_options);
+    } else {
+      options_storage.emplace(CreateDefaultExecutionOptions());
     }
-    TF_RET_CHECK(!results.empty());
-    VLOG(1) << "Defaulting to device 0 result";
-    return std::move(results[0]);
-  }
-
-  // The argument shapes affect how the computation is compiled.
-  std::vector<Shape> arg_shapes(arguments.size());
-  for (int i = 0; i < arguments.size(); i++) {
-    TF_ASSIGN_OR_RETURN(arg_shapes[i], GetShape(*arguments[i]));
-  }
-
-  TF_ASSIGN_OR_RETURN(auto handle,
-                      Compile(computation, arg_shapes, execution_options));
-
-  TF_ASSIGN_OR_RETURN(auto result,
-                      Execute(handle, arguments, execution_profile));
-
-  if (execution_profile != nullptr) {
-    if (VLOG_IS_ON(1)) {
-      TF_ASSIGN_OR_RETURN(
-          auto execution_stats,
-          ExecutionStatsAsString(computation, *execution_profile));
-      VLOG(1) << execution_stats;
+    execution_options = &*options_storage;
+
+    TF_ASSIGN_OR_RETURN(auto device_handles,
+                        GetDeviceHandles(/*device_count=*/1));
+    TF_RET_CHECK(!device_handles.empty());
+    *options_storage->add_device_handles() = std::move(device_handles[0]);
+  }
+
+  std::vector<XlaComputationInstance> computation_instances = {
+      XlaComputationInstance{
+          computation,
+          std::vector<GlobalData*>(arguments.begin(), arguments.end()),
+          *execution_options, execution_profile}};
+
+  // Instead of invoking Compile() and Execute(), invoke
+  // Service::ExecuteParallel() to execute our one computation.  Compile()
+  // caches the executable forever, which isn't what we want.
+  VLOG(1) << "Making ExecuteParallel request: "
+          << execution_options->DebugString();
+  TF_ASSIGN_OR_RETURN(auto results, ExecuteParallel(computation_instances));
+  VLOG(1) << "ExecuteParallel request done.";
+
+  // The result selection is a bit hacky, but better than assuming it is
+  // device 0.
+  //
+  // TODO(b/118493728): Allow Execute to return one result per computation.
+  for (int64 i = 0; i < results.size(); i++) {
+    TF_ASSIGN_OR_RETURN(const Shape& shape, GetShape(*results[i]));
+    if (!ShapeUtil::IsEmptyTuple(shape)) {
+      VLOG(3) << "Fetching result from device " << i << ": "
+              << ShapeUtil::HumanString(shape);
+      return std::move(results[i]);
     }
   }
-
-  return std::move(result);
+  TF_RET_CHECK(!results.empty());
+  VLOG(1) << "Defaulting to device 0 result";
+  return std::move(results[0]);
 }
 
 StatusOr<std::vector<std::unique_ptr<GlobalData>>> Client::ExecuteParallel(
diff --git a/tensorflow/compiler/xla/client/client.h b/tensorflow/compiler/xla/client/client.h
index d0ac4703c6..eff8713ac3 100644
--- a/tensorflow/compiler/xla/client/client.h
+++ b/tensorflow/compiler/xla/client/client.h
@@ -52,6 +52,12 @@ class Client {
   //   need to live beyond this call.)
   // * If execution_options.device_handles should be empty. If you need
   //   non-empty device handles, call 'Execute' instead.
+  //
+  // TODO(b/122731460): This call caches the resulting Executable in the Service
+  // *forever*.  If you're only going to run the computation once, you may want
+  // to call the Execute(const XlaComputation&) overload.  If you're going to
+  // run the computation more than once but you want control over when the
+  // Executable is unloaded, use the LocalClient API.
   StatusOr<ExecutionHandle> Compile(
       const XlaComputation& computation,
       absl::Span<const Shape> argument_shapes,
@@ -76,6 +82,10 @@ class Client {
   //   device is chosen by the service.
   // * If execution_profile is not nullptr then the pointed-to ExecutionProfile
   //   will be filled with profile data from the execution.
+  //
+  // TODO(b/122731460): The given computation is compiled and then thrown away
+  // immediately after it's run.  If you want control over how long the
+  // resulting Executable lives, use the LocalClient API.
   StatusOr<std::unique_ptr<GlobalData>> Execute(
       const XlaComputation& computation,
       absl::Span<GlobalData* const> arguments,
diff --git a/tensorflow/compiler/xla/rpc/grpc_service.cc b/tensorflow/compiler/xla/rpc/grpc_service.cc
index d8123a6de2..22b4218fbd 100644
--- a/tensorflow/compiler/xla/rpc/grpc_service.cc
+++ b/tensorflow/compiler/xla/rpc/grpc_service.cc
@@ -47,6 +47,14 @@ namespace xla {
   });
 }
 
+::grpc::Status GRPCService::GetDeviceHandles(::grpc::ServerContext* context,
+                                             const GetDeviceHandlesRequest* arg,
+                                             GetDeviceHandlesResponse* result) {
+  return DelegateRPC([this, arg, result]() {
+    return service_->GetDeviceHandles(arg, result);
+  });
+}
+
 ::grpc::Status GRPCService::Compile(::grpc::ServerContext* /*context*/,
                                     const CompileRequest* arg,
                                     CompileResponse* result) {
@@ -61,6 +69,14 @@ namespace xla {
       [this, arg, result]() { return service_->Execute(arg, result); });
 }
 
+::grpc::Status GRPCService::ExecuteGraphParallel(
+    ::grpc::ServerContext* /*context*/, const ExecuteGraphParallelRequest* arg,
+    ExecuteParallelResponse* result) {
+  return DelegateRPC([this, arg, result]() {
+    return service_->ExecuteGraphParallel(arg, result);
+  });
+}
+
 ::grpc::Status GRPCService::WaitForExecution(::grpc::ServerContext* context,
                                              const WaitForExecutionRequest* arg,
                                              WaitForExecutionResponse* result) {
diff --git a/tensorflow/compiler/xla/rpc/grpc_service.h b/tensorflow/compiler/xla/rpc/grpc_service.h
index 3e586b288a..b546704f73 100644
--- a/tensorflow/compiler/xla/rpc/grpc_service.h
+++ b/tensorflow/compiler/xla/rpc/grpc_service.h
@@ -39,6 +39,10 @@ class GRPCService : public grpc::XlaService::Service {
                                   const DeconstructTupleRequest* arg,
                                   DeconstructTupleResponse* result) override;
 
+  ::grpc::Status GetDeviceHandles(::grpc::ServerContext* context,
+                                  const GetDeviceHandlesRequest* arg,
+                                  GetDeviceHandlesResponse* result) override;
+
   ::grpc::Status Compile(::grpc::ServerContext* context,
                          const CompileRequest* arg,
                          CompileResponse* result) override;
@@ -46,6 +50,9 @@ class GRPCService : public grpc::XlaService::Service {
   ::grpc::Status Execute(::grpc::ServerContext* context,
                          const ExecuteRequest* arg,
                          ExecuteResponse* result) override;
+  ::grpc::Status ExecuteGraphParallel(::grpc::ServerContext* context,
+                                      const ExecuteGraphParallelRequest* arg,
+                                      ExecuteParallelResponse* result) override;
 
   ::grpc::Status WaitForExecution(::grpc::ServerContext* context,
                                   const WaitForExecutionRequest* arg,
diff --git a/tensorflow/compiler/xla/service/service.cc b/tensorflow/compiler/xla/service/service.cc
index 2732d498d8..32707de700 100644
--- a/tensorflow/compiler/xla/service/service.cc
+++ b/tensorflow/compiler/xla/service/service.cc
@@ -524,13 +524,13 @@ Service::ExecuteParallelAndRegisterResult(
 
 StatusOr<GlobalDataHandle> Service::ExecuteAndRegisterResult(
     Executable* executable,
-    const absl::Span<const std::vector<const ShapedBuffer*>> arguments,
-    Backend* backend, const string& result_tag, ExecutionProfile* profile) {
+    absl::Span<const std::vector<const ShapedBuffer*>> arguments,
+    Backend* backend, const DeviceHandle& device_handle,
+    const string& result_tag, ExecutionProfile* profile) {
   // Set up streams.
   std::vector<StreamPool::Ptr> streams;
 
-  TF_ASSIGN_OR_RETURN(auto replicas,
-                      Replicas(*backend, SingleComputationDeviceHandle()));
+  TF_ASSIGN_OR_RETURN(auto replicas, Replicas(*backend, device_handle));
   TF_RET_CHECK(!replicas.empty());
   for (se::StreamExecutor* executor : replicas) {
     TF_ASSIGN_OR_RETURN(StreamPool::Ptr stream,
@@ -538,10 +538,11 @@ StatusOr<GlobalDataHandle> Service::ExecuteAndRegisterResult(
     streams.push_back(std::move(stream));
   }
 
-  TF_ASSIGN_OR_RETURN(DeviceAssignment device_assignment,
-                      backend->computation_placer()->AssignDevices(
-                          options_.number_of_replicas(),
-                          /*computation_count=*/1));
+  DeviceAssignment device_assignment(options_.number_of_replicas(),
+                                     /*computation_count=*/1);
+  for (int64 replica = 0; replica < replicas.size(); ++replica) {
+    device_assignment(replica, 0) = replicas[replica]->device_ordinal();
+  }
 
   // Set up run options.
   std::vector<ServiceExecutableRunOptions> run_options;
@@ -710,14 +711,33 @@ Status Service::ExecuteGraphParallel(const ExecuteGraphParallelRequest* arg,
     }
   }
 
-  // Execute the generated executables in parallel and return the device
-  // handles for each computation's output.
+  // If we have multiple executables to run, execute them all in parallel.  But
+  // if we only have one executable, execute it using the vanilla, non-parallel
+  // call.
+  //
+  // We do this because the Client API uses ExecuteGraphParallel when it wants
+  // to compile and run one computation without caching the executable, but not
+  // all backends support the async StreamExecutor API required by
+  // ExecuteParallelAndRegisterResult.
+  //
+  // TODO(b/122731460): Consolidate Execute{,Parallel}AndRegisterResult; they do
+  // basically the same thing.
   ExecutionProfile profile;
-  TF_ASSIGN_OR_RETURN(
-      std::vector<GlobalDataHandle> outputs,
-      ExecuteParallelAndRegisterResult(executable_ptrs, all_arguments,
-                                       execute_backend_.get(), device_handles,
-                                       computation_names, &profile));
+  std::vector<GlobalDataHandle> outputs;
+  if (executable_ptrs.size() == 1) {
+    TF_ASSIGN_OR_RETURN(
+        auto output,
+        ExecuteAndRegisterResult(executable_ptrs[0], all_arguments[0],
+                                 execute_backend_.get(), device_handles[0],
+                                 computation_names[0], &profile));
+    outputs.push_back(std::move(output));
+  } else {
+    TF_ASSIGN_OR_RETURN(
+        outputs, ExecuteParallelAndRegisterResult(
+                     executable_ptrs, all_arguments, execute_backend_.get(),
+                     device_handles, computation_names, &profile));
+  }
+
   for (const GlobalDataHandle& output : outputs) {
     ExecuteResponse response;
     *response.mutable_output() = output;
@@ -903,6 +923,7 @@ Status Service::Execute(const ExecuteRequest* arg, ExecuteResponse* result) {
       *result->mutable_output(),
       ExecuteAndRegisterResult(executable.get(), replicated_arguments,
                                execute_backend_.get(),
+                               SingleComputationDeviceHandle(),
                                "result of " + executable->module().name(),
                                result->mutable_profile()));
 
diff --git a/tensorflow/compiler/xla/service/service.h b/tensorflow/compiler/xla/service/service.h
index abd3ee5a05..6e31bea728 100644
--- a/tensorflow/compiler/xla/service/service.h
+++ b/tensorflow/compiler/xla/service/service.h
@@ -250,8 +250,9 @@ class Service : public ServiceInterface {
   // ExecutionProfile object which will be filled in with profile data.
   StatusOr<GlobalDataHandle> ExecuteAndRegisterResult(
       Executable* executable,
-      const absl::Span<const std::vector<const ShapedBuffer*>> arguments,
-      Backend* backend, const string& result_tag, ExecutionProfile* profile);
+      absl::Span<const std::vector<const ShapedBuffer*>> arguments,
+      Backend* backend, const DeviceHandle& device_handle,
+      const string& result_tag, ExecutionProfile* profile);
 
   // Runs the given executables with the given arguments and register the result
   // from each executable in the allocation tracker. The handles of the result
-- 
GitLab


From 8b6251446c730269bc698db83acd0787f44f3645 Mon Sep 17 00:00:00 2001
From: Martin Wicke <wicke@google.com>
Date: Sat, 12 Jan 2019 17:07:42 -0800
Subject: [PATCH 0651/1095] Add a test for paren handling.

PiperOrigin-RevId: 229041154
---
 tensorflow/tools/compatibility/ast_edits.py      |  5 ++++-
 tensorflow/tools/compatibility/ast_edits_test.py | 14 ++++++++++++++
 .../tools/compatibility/tf_upgrade_v2_test.py    | 16 ++++++++++++++++
 3 files changed, 34 insertions(+), 1 deletion(-)

diff --git a/tensorflow/tools/compatibility/ast_edits.py b/tensorflow/tools/compatibility/ast_edits.py
index 2254c223ce..940be4c23c 100644
--- a/tensorflow/tools/compatibility/ast_edits.py
+++ b/tensorflow/tools/compatibility/ast_edits.py
@@ -289,8 +289,11 @@ class _PastaEditVisitor(ast.NodeVisitor):
       reordered = function_reorders[full_name]
       new_keywords = []
       for idx, arg in enumerate(node.args):
+        if sys.version_info[:2] >= (3, 5) and isinstance(arg, ast.Starred):
+          continue  # Can't move Starred to keywords
         keyword_arg = reordered[idx]
-        new_keywords.append(ast.keyword(arg=keyword_arg, value=arg))
+        keyword = ast.keyword(arg=keyword_arg, value=arg)
+        new_keywords.append(keyword)
 
       if new_keywords:
         self.add_log(node.lineno, node.col_offset,
diff --git a/tensorflow/tools/compatibility/ast_edits_test.py b/tensorflow/tools/compatibility/ast_edits_test.py
index 4accd8fe29..ff8b8d6982 100644
--- a/tensorflow/tools/compatibility/ast_edits_test.py
+++ b/tensorflow/tools/compatibility/ast_edits_test.py
@@ -190,6 +190,20 @@ class TestAstEdits(test_util.TensorFlowTestCase):
     _, new_text = self._upgrade(RenameKeywordSpec(), text)
     self.assertEqual(new_text, text)
 
+  def testKeywordReorderWithParens(self):
+    """Test that we get the expected result if there are parens around args."""
+    text = "f((a), ( ( b ) ))\n"
+    acceptable_outputs = [
+        # No change is a valid output
+        text,
+        # Also cases where all arguments are fully specified are allowed
+        "f(a=(a), b=( ( b ) ))\n",
+        # Making the parens canonical is ok
+        "f(a=(a), b=((b)))\n",
+    ]
+    _, new_text = self._upgrade(ReorderKeywordSpec(), text)
+    self.assertIn(new_text, acceptable_outputs)
+
   def testKeywordReorder(self):
     """Test that we get the expected result if kw2 is now before kw1."""
     text = "f(a, b, kw1=c, kw2=d)\n"
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index 0a85cb39c8..0dde5045aa 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -912,6 +912,22 @@ tf.print('abc')
     self.assertIn("tf.cond", errors[0])
     self.assertIn("requires manual check", errors[0])
 
+  def testParens(self):
+    text = """
+def _log_prob(self, x):
+  return tf.reduce_logsumexp(
+      (self.mixture_distribution.logits + self.distribution.log_prob(
+          x[..., tf.newaxis])),
+          axis=-1)"""
+    expected_text = """
+def _log_prob(self, x):
+  return tf.reduce_logsumexp(
+      input_tensor=(self.mixture_distribution.logits + self.distribution.log_prob(
+          x[..., tf.newaxis])),
+          axis=-1)"""
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(expected_text, new_text)
+
 
 class TestUpgradeFiles(test_util.TensorFlowTestCase):
 
-- 
GitLab


From 4eb7eaa1814d5f0822adf5f61d327f68f0b70982 Mon Sep 17 00:00:00 2001
From: Dheeraj Rajaram Reddy <dheeraj98reddy@gmail.com>
Date: Fri, 11 Jan 2019 23:33:02 +0530
Subject: [PATCH 0652/1095] Add take_while golden API to pbtxt

---
 .../tools/api/golden/v2/tensorflow.data.experimental.pbtxt    | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt
index a262c0f799..abc98a74b6 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.pbtxt
@@ -180,6 +180,10 @@ tf_module {
     name: "shuffle_and_repeat"
     argspec: "args=[\'buffer_size\', \'count\', \'seed\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
+  member_method {
+    name: "take_while"
+    argspec: "args=[\'predicate\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "unbatch"
     argspec: "args=[], varargs=None, keywords=None, defaults=None"
-- 
GitLab


From 898738bce44c0cfdaa36b1dfc865239eb48fa999 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sun, 13 Jan 2019 01:02:14 -0800
Subject: [PATCH 0653/1095] compat: Update forward compatibility horizon to
 2019-01-13

PiperOrigin-RevId: 229060750
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 1f4840ff38..f977d0d580 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 12)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 13)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 3e6d902b6e8180fbfe9de0bb2e5069068cb5cbc5 Mon Sep 17 00:00:00 2001
From: Kashif Rasul <kashif.rasul@gmail.com>
Date: Sun, 13 Jan 2019 13:35:32 +0100
Subject: [PATCH 0654/1095] removed deprecated cuda driver API calls

---
 .../stream_executor/cuda/cuda_driver.cc       | 34 +++++++++----------
 tensorflow/stream_executor/cuda/cuda_driver.h | 10 ------
 .../stream_executor/cuda/cuda_gpu_executor.cc | 22 +++++++-----
 3 files changed, 31 insertions(+), 35 deletions(-)

diff --git a/tensorflow/stream_executor/cuda/cuda_driver.cc b/tensorflow/stream_executor/cuda/cuda_driver.cc
index ca7a717bdb..3b6e1bef66 100644
--- a/tensorflow/stream_executor/cuda/cuda_driver.cc
+++ b/tensorflow/stream_executor/cuda/cuda_driver.cc
@@ -1290,15 +1290,26 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                                                            CUdevice device) {
   *cc_major = 0;
   *cc_minor = 0;
-  CUresult result = cuDeviceComputeCapability(cc_major, cc_minor, device);
-  if (result == CUDA_SUCCESS) {
-    return port::Status::OK();
+
+  CUresult res = 
+      cuDeviceGetAttribute(cc_major, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, device);
+  if (res != CUDA_SUCCESS) {
+    return port::Status(
+      port::error::INTERNAL,
+      port::Printf("failed to get compute capability major for device: %s; %d",
+                   ToString(res).c_str(), device));
   }
 
-  return port::Status(
+  res = 
+      cuDeviceGetAttribute(cc_minor, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, device);
+  if (res != CUDA_SUCCESS) {
+    return port::Status(
       port::error::INTERNAL,
-      port::Printf("failed to get compute capability for device: %s; %d",
-                   ToString(result).c_str(), device));
+      port::Printf("failed to get compute capability minor for device: %s; %d",
+                   ToString(res).c_str(), device));
+  }
+
+  return port::Status::OK();
 }
 
 // Helper function that turns the integer output of cuDeviceGetAttribute to type
@@ -1398,17 +1409,6 @@ static port::StatusOr<T> GetSimpleAttribute(CUdevice device,
   return true;
 }
 
-/* static */ bool CUDADriver::GetDeviceProperties(CUdevprop *device_properties,
-                                                  int device_ordinal) {
-  CUresult res = cuDeviceGetProperties(device_properties, device_ordinal);
-  if (res != CUDA_SUCCESS) {
-    LOG(ERROR) << "failed to query device properties: " << ToString(res);
-    return false;
-  }
-
-  return true;
-}
-
 /* static */ port::StatusOr<int> CUDADriver::GetDeviceAttribute(
     CUdevice_attribute attribute, CUdevice device) {
   int val;
diff --git a/tensorflow/stream_executor/cuda/cuda_driver.h b/tensorflow/stream_executor/cuda/cuda_driver.h
index 447422739d..825455d448 100644
--- a/tensorflow/stream_executor/cuda/cuda_driver.h
+++ b/tensorflow/stream_executor/cuda/cuda_driver.h
@@ -407,16 +407,6 @@ class CUDADriver {
   // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE.html#group__CUDA__DEVICE_1g9c3e1414f0ad901d3278a4d6645fc266
   static bool GetGridLimits(int *x, int *y, int *z, CUdevice device);
 
-  // Returns a grab-bag of device properties in a caller-owned device_properties
-  // structure for device_ordinal via cuDeviceGetProperties.
-  //
-  // This call is deprecated in the NVIDIA driver API; its replacement is
-  // GetDeviceAttribute
-  //
-  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE__DEPRECATED.html#group__CUDA__DEVICE__DEPRECATED_1g65a5b4e25186bd257df80b98c98cffe6
-  static bool GetDeviceProperties(CUdevprop *device_properties,
-                                  int device_ordinal);
-
   // Gets a specific integer-valued property about the given device.
   //
   // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE.html#group__CUDA__DEVICE_1g9c3e1414f0ad901d3278a4d6645fc266
diff --git a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc
index 5b33d97594..7705e7e8f2 100644
--- a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc
+++ b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc
@@ -1043,18 +1043,24 @@ DeviceDescription *CUDAExecutor::PopulateDeviceDescription() const {
     builder.set_numa_node(numa_node);
   }
 
-  CUdevprop prop;
-  if (CUDADriver::GetDeviceProperties(&prop, device_ordinal_)) {
-    builder.set_threads_per_block_limit(prop.maxThreadsPerBlock);
+  {
+    builder.set_threads_per_block_limit(
+      CUDADriver::GetDeviceAttribute(
+          CU_DEVICE_ATTRIBUTE_MAX_THREADS_PER_BLOCK, device_)
+          .ValueOrDie());
 
     ThreadDim thread_dim_limit;
-    thread_dim_limit.x = prop.maxThreadsDim[0];
-    thread_dim_limit.y = prop.maxThreadsDim[1];
-    thread_dim_limit.z = prop.maxThreadsDim[2];
+    thread_dim_limit.x = CUDADriver::GetDeviceAttribute(
+        CU_DEVICE_ATTRIBUTE_MAX_BLOCK_DIM_X, device_).ValueOrDie();
+    thread_dim_limit.y = CUDADriver::GetDeviceAttribute(
+        CU_DEVICE_ATTRIBUTE_MAX_BLOCK_DIM_Y, device_).ValueOrDie();
+    thread_dim_limit.z = CUDADriver::GetDeviceAttribute(
+        CU_DEVICE_ATTRIBUTE_MAX_BLOCK_DIM_Z, device_).ValueOrDie();
     builder.set_thread_dim_limit(thread_dim_limit);
 
-    float clock_rate_ghz = static_cast<float>(prop.clockRate) / 1e6;
-    builder.set_clock_rate_ghz(clock_rate_ghz);
+    int clockRate = CUDADriver::GetDeviceAttribute(
+        CU_DEVICE_ATTRIBUTE_CLOCK_RATE, device_).ValueOrDie();
+    builder.set_clock_rate_ghz(static_cast<float>(clockRate) / 1e6);
   }
 
   {
-- 
GitLab


From 9a9e7655f1a4121342ffd2ee51dad7a908c40f36 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sun, 13 Jan 2019 23:41:33 +0000
Subject: [PATCH 0655/1095] Fix warnings in feature_column.py

This fix fixes warnings in feature_column.py where
`math_ops.to_int64` has been deprecated and replaced with
`math_ops.cast`.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 .../python/feature_column/feature_column.py   | 22 +++++++++----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/tensorflow/python/feature_column/feature_column.py b/tensorflow/python/feature_column/feature_column.py
index a858d92608..f474d8a4fe 100644
--- a/tensorflow/python/feature_column/feature_column.py
+++ b/tensorflow/python/feature_column/feature_column.py
@@ -2361,7 +2361,7 @@ class _BucketizedColumn(_DenseColumn, _CategoricalColumn,
     del trainable
     input_tensor = inputs.get(self)
     return array_ops.one_hot(
-        indices=math_ops.to_int64(input_tensor),
+        indices=math_ops.cast(input_tensor, dtypes.int64),
         depth=len(self.boundaries) + 1,
         on_value=1.,
         off_value=0.)
@@ -2391,9 +2391,9 @@ class _BucketizedColumn(_DenseColumn, _CategoricalColumn,
         array_ops.reshape(input_tensor, (-1,)) +
         (len(self.boundaries) + 1) * i2)
 
-    indices = math_ops.to_int64(array_ops.transpose(array_ops.stack((i1, i2))))
-    dense_shape = math_ops.to_int64(array_ops.stack(
-        [batch_size, source_dimension]))
+    indices = math_ops.cast(array_ops.transpose(array_ops.stack((i1, i2))), dtypes.int64)
+    dense_shape = math_ops.cast(array_ops.stack(
+        [batch_size, source_dimension]), dtypes.int64)
     sparse_tensor = sparse_tensor_lib.SparseTensor(
         indices=indices,
         values=bucket_indices,
@@ -2829,7 +2829,7 @@ class _VocabularyFileCategoricalColumn(
     if input_tensor.dtype.is_integer:
       # `index_table_from_file` requires 64-bit integer keys.
       key_dtype = dtypes.int64
-      input_tensor = math_ops.to_int64(input_tensor)
+      input_tensor = math_ops.cast(input_tensor, dtypes.int64)
 
     return lookup_ops.index_table_from_file(
         vocabulary_file=self.vocabulary_file,
@@ -2881,7 +2881,7 @@ class _VocabularyListCategoricalColumn(
     if input_tensor.dtype.is_integer:
       # `index_table_from_tensor` requires 64-bit integer keys.
       key_dtype = dtypes.int64
-      input_tensor = math_ops.to_int64(input_tensor)
+      input_tensor = math_ops.cast(input_tensor, dtypes.int64)
 
     return lookup_ops.index_table_from_tensor(
         vocabulary_list=tuple(self.vocabulary_list),
@@ -2924,9 +2924,9 @@ class _IdentityCategoricalColumn(
           'Invalid input, not integer. key: {} dtype: {}'.format(
               self.key, input_tensor.dtype))
 
-    values = math_ops.to_int64(input_tensor.values, name='values')
-    num_buckets = math_ops.to_int64(self.num_buckets, name='num_buckets')
-    zero = math_ops.to_int64(0, name='zero')
+    values = math_ops.cast(input_tensor.values, dtypes.int64, name='values')
+    num_buckets = math_ops.cast(self.num_buckets, dtypes.int64, name='num_buckets')
+    zero = math_ops.cast(0, dtypes.int64, name='zero')
     if self.default_value is None:
       # Fail if values are out-of-range.
       assert_less = check_ops.assert_less(
@@ -2944,7 +2944,7 @@ class _IdentityCategoricalColumn(
               values < zero, values >= num_buckets, name='out_of_range'),
           array_ops.fill(
               dims=array_ops.shape(values),
-              value=math_ops.to_int64(self.default_value),
+              value=math_ops.cast(self.default_value, dtypes.int64),
               name='default_values'),
           values)
 
@@ -3256,7 +3256,7 @@ def _sequence_length_from_sparse_tensor(sp_tensor, num_elements=1):
     # Example: orig tensor [[1, 2], [3]], col_ids = (0, 1, 1),
     # row_ids = (0, 0, 1), seq_length = [2, 1]. If num_elements = 2,
     # these will get grouped, and the final seq_length is [1, 1]
-    seq_length = math_ops.to_int64(math_ops.ceil(seq_length / num_elements))
+    seq_length = math_ops.cast(math_ops.ceil(seq_length / num_elements), dtypes.int64)
 
     # If the last n rows do not have ids, seq_length will have shape
     # [batch_size - n]. Pad the remaining values with zeros.
-- 
GitLab


From fdaf63b38b511126d66aa9b3da254df8c85e47e2 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sun, 13 Jan 2019 23:45:42 +0000
Subject: [PATCH 0656/1095] Line too long (line-too-long) fix

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/feature_column/feature_column.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/feature_column/feature_column.py b/tensorflow/python/feature_column/feature_column.py
index f474d8a4fe..84db3d3ac2 100644
--- a/tensorflow/python/feature_column/feature_column.py
+++ b/tensorflow/python/feature_column/feature_column.py
@@ -2391,7 +2391,8 @@ class _BucketizedColumn(_DenseColumn, _CategoricalColumn,
         array_ops.reshape(input_tensor, (-1,)) +
         (len(self.boundaries) + 1) * i2)
 
-    indices = math_ops.cast(array_ops.transpose(array_ops.stack((i1, i2))), dtypes.int64)
+    indices = math_ops.cast(
+        array_ops.transpose(array_ops.stack((i1, i2))), dtypes.int64)
     dense_shape = math_ops.cast(array_ops.stack(
         [batch_size, source_dimension]), dtypes.int64)
     sparse_tensor = sparse_tensor_lib.SparseTensor(
@@ -2925,7 +2926,8 @@ class _IdentityCategoricalColumn(
               self.key, input_tensor.dtype))
 
     values = math_ops.cast(input_tensor.values, dtypes.int64, name='values')
-    num_buckets = math_ops.cast(self.num_buckets, dtypes.int64, name='num_buckets')
+    num_buckets = math_ops.cast(
+        self.num_buckets, dtypes.int64, name='num_buckets')
     zero = math_ops.cast(0, dtypes.int64, name='zero')
     if self.default_value is None:
       # Fail if values are out-of-range.
@@ -3256,7 +3258,8 @@ def _sequence_length_from_sparse_tensor(sp_tensor, num_elements=1):
     # Example: orig tensor [[1, 2], [3]], col_ids = (0, 1, 1),
     # row_ids = (0, 0, 1), seq_length = [2, 1]. If num_elements = 2,
     # these will get grouped, and the final seq_length is [1, 1]
-    seq_length = math_ops.cast(math_ops.ceil(seq_length / num_elements), dtypes.int64)
+    seq_length = math_ops.cast(
+        math_ops.ceil(seq_length / num_elements), dtypes.int64)
 
     # If the last n rows do not have ids, seq_length will have shape
     # [batch_size - n]. Pad the remaining values with zeros.
-- 
GitLab


From ac4d3682939dac303082097d25ebda805409acde Mon Sep 17 00:00:00 2001
From: Suharsh Sivakumar <suharshs@google.com>
Date: Sun, 13 Jan 2019 18:02:02 -0800
Subject: [PATCH 0657/1095] Add quantized_dimension field to quantization
 params.

PiperOrigin-RevId: 229109989
---
 tensorflow/lite/schema/schema.fbs         | 15 ++++++++++--
 tensorflow/lite/schema/schema_generated.h | 29 ++++++++++++++++++-----
 2 files changed, 36 insertions(+), 8 deletions(-)

diff --git a/tensorflow/lite/schema/schema.fbs b/tensorflow/lite/schema/schema.fbs
index aca926a3b9..13f47d2cfc 100644
--- a/tensorflow/lite/schema/schema.fbs
+++ b/tensorflow/lite/schema/schema.fbs
@@ -64,9 +64,20 @@ table QuantizationParameters {
   scale:[float];  // For dequantizing the tensor's values.
   zero_point:[long];
 
-  // If this is not none, the quantization parameters above are ignored and the
-  // value of the QuantizationDetails union below should be used.
+  // If this is not none, the other quantization parameters (i.e. min, max,
+  // scale, zero_point fields above) are ignored and the value of the
+  // QuantizationDetails union should be used.
   details:QuantizationDetails;
+
+  // Specifies the dimension of the Tensor's shape that the scales and
+  // zero_points correspond to. For example, a tensor t, with dims=[4, 3, 2, 1]
+  // with quantization params:
+  //   scale=[1.0, 2.0, 3.0], zero_point=[1, 2, 3], quantization_dimension=1
+  // will be quantized across the second dimension of t.
+  //   t[:, 0, :, :] will have scale[0]=1.0, zero_point[0]=1
+  //   t[:, 1, :, :] will have scale[1]=2.0, zero_point[0]=2
+  //   t[:, 2, :, :] will have scale[2]=3.0, zero_point[0]=3
+  quantized_dimension:int;
 }
 
 table Tensor {
diff --git a/tensorflow/lite/schema/schema_generated.h b/tensorflow/lite/schema/schema_generated.h
index 1d153cc717..a1174e79f7 100755
--- a/tensorflow/lite/schema/schema_generated.h
+++ b/tensorflow/lite/schema/schema_generated.h
@@ -2335,7 +2335,9 @@ struct QuantizationParametersT : public flatbuffers::NativeTable {
   std::vector<float> scale;
   std::vector<int64_t> zero_point;
   QuantizationDetailsUnion details;
-  QuantizationParametersT() {
+  int32_t quantized_dimension;
+  QuantizationParametersT()
+      : quantized_dimension(0) {
   }
 };
 
@@ -2347,7 +2349,8 @@ struct QuantizationParameters FLATBUFFERS_FINAL_CLASS : private flatbuffers::Tab
     VT_SCALE = 8,
     VT_ZERO_POINT = 10,
     VT_DETAILS_TYPE = 12,
-    VT_DETAILS = 14
+    VT_DETAILS = 14,
+    VT_QUANTIZED_DIMENSION = 16
   };
   const flatbuffers::Vector<float> *min() const {
     return GetPointer<const flatbuffers::Vector<float> *>(VT_MIN);
@@ -2371,6 +2374,9 @@ struct QuantizationParameters FLATBUFFERS_FINAL_CLASS : private flatbuffers::Tab
   const CustomQuantization *details_as_CustomQuantization() const {
     return details_type() == QuantizationDetails_CustomQuantization ? static_cast<const CustomQuantization *>(details()) : nullptr;
   }
+  int32_t quantized_dimension() const {
+    return GetField<int32_t>(VT_QUANTIZED_DIMENSION, 0);
+  }
   bool Verify(flatbuffers::Verifier &verifier) const {
     return VerifyTableStart(verifier) &&
            VerifyOffset(verifier, VT_MIN) &&
@@ -2384,6 +2390,7 @@ struct QuantizationParameters FLATBUFFERS_FINAL_CLASS : private flatbuffers::Tab
            VerifyField<uint8_t>(verifier, VT_DETAILS_TYPE) &&
            VerifyOffset(verifier, VT_DETAILS) &&
            VerifyQuantizationDetails(verifier, details(), details_type()) &&
+           VerifyField<int32_t>(verifier, VT_QUANTIZED_DIMENSION) &&
            verifier.EndTable();
   }
   QuantizationParametersT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const;
@@ -2416,6 +2423,9 @@ struct QuantizationParametersBuilder {
   void add_details(flatbuffers::Offset<void> details) {
     fbb_.AddOffset(QuantizationParameters::VT_DETAILS, details);
   }
+  void add_quantized_dimension(int32_t quantized_dimension) {
+    fbb_.AddElement<int32_t>(QuantizationParameters::VT_QUANTIZED_DIMENSION, quantized_dimension, 0);
+  }
   explicit QuantizationParametersBuilder(flatbuffers::FlatBufferBuilder &_fbb)
         : fbb_(_fbb) {
     start_ = fbb_.StartTable();
@@ -2435,8 +2445,10 @@ inline flatbuffers::Offset<QuantizationParameters> CreateQuantizationParameters(
     flatbuffers::Offset<flatbuffers::Vector<float>> scale = 0,
     flatbuffers::Offset<flatbuffers::Vector<int64_t>> zero_point = 0,
     QuantizationDetails details_type = QuantizationDetails_NONE,
-    flatbuffers::Offset<void> details = 0) {
+    flatbuffers::Offset<void> details = 0,
+    int32_t quantized_dimension = 0) {
   QuantizationParametersBuilder builder_(_fbb);
+  builder_.add_quantized_dimension(quantized_dimension);
   builder_.add_details(details);
   builder_.add_zero_point(zero_point);
   builder_.add_scale(scale);
@@ -2453,7 +2465,8 @@ inline flatbuffers::Offset<QuantizationParameters> CreateQuantizationParametersD
     const std::vector<float> *scale = nullptr,
     const std::vector<int64_t> *zero_point = nullptr,
     QuantizationDetails details_type = QuantizationDetails_NONE,
-    flatbuffers::Offset<void> details = 0) {
+    flatbuffers::Offset<void> details = 0,
+    int32_t quantized_dimension = 0) {
   return tflite::CreateQuantizationParameters(
       _fbb,
       min ? _fbb.CreateVector<float>(*min) : 0,
@@ -2461,7 +2474,8 @@ inline flatbuffers::Offset<QuantizationParameters> CreateQuantizationParametersD
       scale ? _fbb.CreateVector<float>(*scale) : 0,
       zero_point ? _fbb.CreateVector<int64_t>(*zero_point) : 0,
       details_type,
-      details);
+      details,
+      quantized_dimension);
 }
 
 flatbuffers::Offset<QuantizationParameters> CreateQuantizationParameters(flatbuffers::FlatBufferBuilder &_fbb, const QuantizationParametersT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
@@ -8263,6 +8277,7 @@ inline void QuantizationParameters::UnPackTo(QuantizationParametersT *_o, const
   { auto _e = zero_point(); if (_e) { _o->zero_point.resize(_e->size()); for (flatbuffers::uoffset_t _i = 0; _i < _e->size(); _i++) { _o->zero_point[_i] = _e->Get(_i); } } };
   { auto _e = details_type(); _o->details.type = _e; };
   { auto _e = details(); if (_e) _o->details.value = QuantizationDetailsUnion::UnPack(_e, details_type(), _resolver); };
+  { auto _e = quantized_dimension(); _o->quantized_dimension = _e; };
 }
 
 inline flatbuffers::Offset<QuantizationParameters> QuantizationParameters::Pack(flatbuffers::FlatBufferBuilder &_fbb, const QuantizationParametersT* _o, const flatbuffers::rehasher_function_t *_rehasher) {
@@ -8279,6 +8294,7 @@ inline flatbuffers::Offset<QuantizationParameters> CreateQuantizationParameters(
   auto _zero_point = _o->zero_point.size() ? _fbb.CreateVector(_o->zero_point) : 0;
   auto _details_type = _o->details.type;
   auto _details = _o->details.Pack(_fbb);
+  auto _quantized_dimension = _o->quantized_dimension;
   return tflite::CreateQuantizationParameters(
       _fbb,
       _min,
@@ -8286,7 +8302,8 @@ inline flatbuffers::Offset<QuantizationParameters> CreateQuantizationParameters(
       _scale,
       _zero_point,
       _details_type,
-      _details);
+      _details,
+      _quantized_dimension);
 }
 
 inline TensorT *Tensor::UnPack(const flatbuffers::resolver_function_t *_resolver) const {
-- 
GitLab


From 5b4c85b9d94484e7ba1f29344f0d2672228ed014 Mon Sep 17 00:00:00 2001
From: Jonas Rauber <jonasrauber@users.noreply.github.com>
Date: Mon, 14 Jan 2019 09:33:47 +0100
Subject: [PATCH 0658/1095] fixed typo

---
 tensorflow/python/eager/backprop.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py
index 42db726a7b..effbc03ae3 100644
--- a/tensorflow/python/eager/backprop.py
+++ b/tensorflow/python/eager/backprop.py
@@ -465,7 +465,7 @@ def val_and_grad_function(f, params=None):
 
 
 def make_vjp(f, params=None, persistent=True):
-  """Returns a function that computes f and is vjp w.r.t. params.
+  """Returns a function that computes f and its vjp w.r.t. params.
 
   The term "vjp" here is an abbreviation for vector-jacobian product.
 
-- 
GitLab


From 2a50782fb68f0b7c9fc4926c7c9e6b05c05abb1c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 14 Jan 2019 00:47:08 -0800
Subject: [PATCH 0659/1095] internal change

PiperOrigin-RevId: 229136654
---
 tensorflow/lite/experimental/examples/lstm/tflite_lstm.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/experimental/examples/lstm/tflite_lstm.py b/tensorflow/lite/experimental/examples/lstm/tflite_lstm.py
index 2fe8ebf9e9..461345060b 100644
--- a/tensorflow/lite/experimental/examples/lstm/tflite_lstm.py
+++ b/tensorflow/lite/experimental/examples/lstm/tflite_lstm.py
@@ -184,7 +184,7 @@ class TFLiteLSTMCell(rnn_cell_impl.LayerRNNCell):
       var = self.add_variable(
           name, shape=shape, initializer=initializer, partitioner=partitioner)
       return self._tflite_wrapper.add_input(
-          var, name="name", index_override=index)
+          var, name=name, index_override=index)
 
     weight_initializer = self._initializer
     if self.dtype is None:
-- 
GitLab


From cfbf81335a3de3c31411bad33ff43ffcfda20817 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 14 Jan 2019 01:02:53 -0800
Subject: [PATCH 0660/1095] compat: Update forward compatibility horizon to
 2019-01-14

PiperOrigin-RevId: 229138454
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index f977d0d580..c9f2397fa5 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 13)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 14)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 6dd6ad9fd7353d945227fd87a93ef1f11eccc75e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 14 Jan 2019 01:32:22 -0800
Subject: [PATCH 0661/1095] Set up remote GPU testing.

Currently, we set the tag "local" for GPU tests in order to be able to execute
remote CPU and local GPU tests within the same bazel invocation.

This change introduces the possibility to set REMOTE_GPU_TESTING to enable
GPU tests to also run remotely; given that tags cannot use starlark's select,
we use an autoconfig rule that defines a function returning the tags we want:
"local" by default and "remote-gpu" if REMOTE_GPU_TESTING is set.

The platform is set via exec_compatible_with constraints, so we select on the
"remote-gpu" tag to add a constraint that is only fulfilled by GPU-enabled
platforms.

PiperOrigin-RevId: 229141861
---
 tensorflow/compiler/tests/build_defs.bzl      |  5 ++-
 .../platform/default/build_config_root.bzl    | 12 +++++-
 tensorflow/opensource_only.files              |  4 ++
 tensorflow/tensorflow.bzl                     |  8 +++-
 tensorflow/workspace.bzl                      |  2 +
 third_party/toolchains/BUILD                  | 35 +++++++++++++++
 third_party/toolchains/remote/BUILD           |  0
 third_party/toolchains/remote/BUILD.tpl       |  0
 third_party/toolchains/remote/configure.bzl   | 43 +++++++++++++++++++
 .../toolchains/remote/execution.bzl.tpl       |  2 +
 10 files changed, 107 insertions(+), 4 deletions(-)
 create mode 100644 third_party/toolchains/remote/BUILD
 create mode 100644 third_party/toolchains/remote/BUILD.tpl
 create mode 100644 third_party/toolchains/remote/configure.bzl
 create mode 100644 third_party/toolchains/remote/execution.bzl.tpl

diff --git a/tensorflow/compiler/tests/build_defs.bzl b/tensorflow/compiler/tests/build_defs.bzl
index be9766c4ef..ed580f95b6 100644
--- a/tensorflow/compiler/tests/build_defs.bzl
+++ b/tensorflow/compiler/tests/build_defs.bzl
@@ -5,6 +5,7 @@ load("//tensorflow/compiler/tests:plugin.bzl", "plugins")
 load(
     "//tensorflow/core:platform/default/build_config_root.bzl",
     "tf_cuda_tests_tags",
+    "tf_exec_compatible_with",
 )
 
 def all_backends():
@@ -84,6 +85,7 @@ def tf_xla_py_test(
         else:
             fail("Unknown backend {}".format(backend))
 
+        test_tags = tags + backend_tags
         native.py_test(
             name = test_name,
             srcs = srcs,
@@ -92,7 +94,8 @@ def tf_xla_py_test(
             main = "{}.py".format(name) if main == None else main,
             data = data + backend_data,
             deps = deps + backend_deps,
-            tags = tags + backend_tags,
+            tags = test_tags,
+            exec_compatible_with = tf_exec_compatible_with({"tags": test_tags}),
             **kwargs
         )
         test_names.append(test_name)
diff --git a/tensorflow/core/platform/default/build_config_root.bzl b/tensorflow/core/platform/default/build_config_root.bzl
index 37475feebe..ab05b25d68 100644
--- a/tensorflow/core/platform/default/build_config_root.bzl
+++ b/tensorflow/core/platform/default/build_config_root.bzl
@@ -2,11 +2,19 @@
 # The functions in this file might be referred by tensorflow.bzl. They have to
 # be separate to avoid cyclic references.
 
+load("@local_config_remote_execution//:remote_execution.bzl", "gpu_test_tags")
+
 def tf_cuda_tests_tags():
-    return ["requires-gpu", "local", "gpu"]
+    return ["requires-gpu", "gpu"] + gpu_test_tags()
 
 def tf_sycl_tests_tags():
-    return ["requires-gpu", "local", "gpu"]
+    return ["requires-gpu", "gpu"] + gpu_test_tags()
+
+def tf_exec_compatible_with(kwargs):
+    if ("tags" in kwargs and kwargs["tags"] != None and
+        "remote-gpu" in kwargs["tags"]):
+        return ["@org_tensorflow//third_party/toolchains:gpu_test"]
+    return []
 
 def tf_additional_plugin_deps():
     return select({
diff --git a/tensorflow/opensource_only.files b/tensorflow/opensource_only.files
index bade64dcf8..03b5e5a73c 100644
--- a/tensorflow/opensource_only.files
+++ b/tensorflow/opensource_only.files
@@ -71,6 +71,10 @@ tensorflow/third_party/toolchains/cpus/arm/CROSSTOOL.tpl
 tensorflow/third_party/toolchains/cpus/arm/BUILD
 tensorflow/third_party/toolchains/cpus/py3/BUILD
 tensorflow/third_party/toolchains/cpus/py/BUILD
+tensorflow/third_party/toolchains/remote/configure.bzl
+tensorflow/third_party/toolchains/remote/BUILD.tpl
+tensorflow/third_party/toolchains/remote/BUILD
+tensorflow/third_party/toolchains/remote/execution.bzl.tpl
 tensorflow/third_party/toolchains/BUILD
 tensorflow/third_party/gpus/BUILD
 tensorflow/third_party/gpus/crosstool/clang/bin/crosstool_wrapper_driver_rocm.tpl
diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl
index 1024b686eb..b7da5bf913 100644
--- a/tensorflow/tensorflow.bzl
+++ b/tensorflow/tensorflow.bzl
@@ -9,6 +9,7 @@ load(
     "tf_additional_grpc_deps_py",
     "tf_additional_xla_deps_py",
     "tf_cuda_tests_tags",
+    "tf_exec_compatible_with",
     "tf_sycl_tests_tags",
 )
 load(
@@ -785,6 +786,7 @@ def tf_cc_test(
             ],
         ),
         data = data + tf_binary_dynamic_kernel_dsos(kernels),
+        exec_compatible_with = tf_exec_compatible_with(kwargs),
         # Nested select() statements seem not to be supported when passed to
         # linkstatic, and we already have a cuda select() passed in to this
         # function.
@@ -897,6 +899,7 @@ def tf_cuda_only_cc_test(
         args = [],
         kernels = [],
         linkopts = []):
+    tags = tags + tf_cuda_tests_tags()
     native.cc_test(
         name = "%s%s" % (name, "_gpu"),
         srcs = srcs + tf_binary_additional_srcs(),
@@ -919,7 +922,8 @@ def tf_cuda_only_cc_test(
             clean_dep("//tensorflow:darwin"): 1,
             "//conditions:default": 0,
         }),
-        tags = tags + tf_cuda_tests_tags(),
+        tags = tags,
+        exec_compatible_with = tf_exec_compatible_with({"tags": tags}),
     )
 
 register_extension_info(
@@ -983,6 +987,7 @@ def tf_cc_test_mkl(
             }) + _rpath_linkopts(src_to_test_name(src)),
             deps = deps + tf_binary_dynamic_kernel_deps(kernels) + mkl_deps(),
             data = data + tf_binary_dynamic_kernel_dsos(kernels),
+            exec_compatible_with = tf_exec_compatible_with({"tags": tags}),
             linkstatic = linkstatic,
             tags = tags,
             size = size,
@@ -1752,6 +1757,7 @@ def py_test(deps = [], data = [], kernels = [], **kwargs):
             "//conditions:default": [],
             clean_dep("//tensorflow:no_tensorflow_py_deps"): ["//tensorflow/tools/pip_package:win_pip_package_marker"],
         }) + tf_binary_dynamic_kernel_dsos(kernels),
+        exec_compatible_with = tf_exec_compatible_with(kwargs),
         **kwargs
     )
 
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index bea57b8ed7..46c1261be2 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -10,6 +10,7 @@ load("//third_party/py:python_configure.bzl", "python_configure")
 
 load("//third_party/sycl:sycl_configure.bzl", "sycl_configure")
 load("//third_party/systemlibs:syslibs_configure.bzl", "syslibs_configure")
+load("//third_party/toolchains/remote:configure.bzl", "remote_execution_configure")
 load("//third_party/toolchains/clang6:repo.bzl", "clang6_configure")
 load("//third_party/toolchains/cpus/arm:arm_compiler_configure.bzl", "arm_compiler_configure")
 load("//third_party:repo.bzl", "tf_http_archive")
@@ -64,6 +65,7 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     syslibs_configure(name = "local_config_syslibs")
     python_configure(name = "local_config_python")
     rocm_configure(name = "local_config_rocm")
+    remote_execution_configure(name = "local_config_remote_execution")
 
     initialize_third_party()
 
diff --git a/third_party/toolchains/BUILD b/third_party/toolchains/BUILD
index 6ed6e5c367..509b541400 100644
--- a/third_party/toolchains/BUILD
+++ b/third_party/toolchains/BUILD
@@ -4,6 +4,17 @@ package(default_visibility = ["//visibility:public"])
 
 load("//third_party/toolchains/preconfig/generate:containers.bzl", "container_digests")
 
+# Constraint used for platforms below so we can force certain rules to be executed
+# on specific platforms.
+constraint_setting(name = "custom_platforms")
+
+# Constraint for platforms that allow GPU testing (i.e. have a GPU available).
+# This is used in exec_compatible_with of rules that need GPU access.
+constraint_value(
+    name = "gpu_test",
+    constraint_setting = ":custom_platforms",
+)
+
 # TODO(b/122347293): This is the RBE config based on the CPU configuration / image provided
 # in the asci-toolchain setup. Delete this once we switched CPU remote builds to the
 # new platform below.
@@ -68,3 +79,27 @@ platform(
             value:"docker://gcr.io/tensorflow-testing/nosla-cuda10.0-cudnn7-ubuntu14.04@%s"
         }""" % container_digests["cuda10.0-cudnn7-ubuntu14.04"],
 )
+
+# The above platform with GPU support.
+platform(
+    name = "rbe_cuda10.0-cudnn7-ubuntu14.04-gpu",
+    constraint_values = [
+        "@bazel_tools//platforms:x86_64",
+        "@bazel_tools//platforms:linux",
+        ":gpu_test",
+    ],
+    remote_execution_properties = """
+        properties: {
+            name: "container-image"
+            value: "docker://gcr.io/tensorflow-testing/nosla-cuda10.0-cudnn7-ubuntu14.04@%s"
+        }
+        properties: {
+            name: "dockerRuntime"
+            value: "nvidia"
+        }
+        properties: {
+            name: "Pool"
+            value: "gpu-pool"
+        }
+        """ % container_digests["cuda10.0-cudnn7-ubuntu14.04"],
+)
diff --git a/third_party/toolchains/remote/BUILD b/third_party/toolchains/remote/BUILD
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/third_party/toolchains/remote/BUILD.tpl b/third_party/toolchains/remote/BUILD.tpl
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/third_party/toolchains/remote/configure.bzl b/third_party/toolchains/remote/configure.bzl
new file mode 100644
index 0000000000..cc5b984264
--- /dev/null
+++ b/third_party/toolchains/remote/configure.bzl
@@ -0,0 +1,43 @@
+"""Repository rule for remote GPU autoconfiguration.
+
+This rule creates the starlark file
+//third_party/toolchains/remote:execution.bzl
+providing the function `gpu_test_tags`.
+
+`gpu_test_tags` will return:
+
+  * `local`: if `REMOTE_GPU_TESTING` is false, allowing CPU tests to run
+    remotely and GPU tests to run locally in the same bazel invocation.
+  * `remote-gpu`: if `REMOTE_GPU_TESTING` is true; this allows rules to
+    set an execution requirement that enables a GPU-enabled remote platform.
+"""
+
+_REMOTE_GPU_TESTING = "REMOTE_GPU_TESTING"
+
+def _flag_enabled(repository_ctx, flag_name):
+    if flag_name not in repository_ctx.os.environ:
+        return False
+    return repository_ctx.os.environ[flag_name].strip() == "1"
+
+def _remote_execution_configure(repository_ctx):
+    # If we do not support remote gpu test execution, mark them as local, so we
+    # can combine remote builds with local gpu tests.
+    gpu_test_tags = "\"local\""
+    if _flag_enabled(repository_ctx, _REMOTE_GPU_TESTING):
+        gpu_test_tags = "\"remote-gpu\""
+    repository_ctx.template(
+        "remote_execution.bzl",
+        Label("//third_party/toolchains/remote:execution.bzl.tpl"),
+        {
+            "%{gpu_test_tags}": gpu_test_tags,
+        },
+    )
+    repository_ctx.template(
+        "BUILD",
+        Label("//third_party/toolchains/remote:BUILD.tpl"),
+    )
+
+remote_execution_configure = repository_rule(
+    implementation = _remote_execution_configure,
+    environ = [_REMOTE_GPU_TESTING],
+)
diff --git a/third_party/toolchains/remote/execution.bzl.tpl b/third_party/toolchains/remote/execution.bzl.tpl
new file mode 100644
index 0000000000..18858cc0dc
--- /dev/null
+++ b/third_party/toolchains/remote/execution.bzl.tpl
@@ -0,0 +1,2 @@
+def gpu_test_tags():
+    return [%{gpu_test_tags}]
-- 
GitLab


From 4ac9bda9d0259ac42fc60ab6ad0d2520d3dde88a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 14 Jan 2019 02:36:35 -0800
Subject: [PATCH 0662/1095] [TF:XLA] Enable XLA through autojit for all
 tensorflow/python/kernel_test/ tests.

Some test methods are disabled, but all tests now have a new "_xla" version of the test for XLA:GPU testing. This will run 2 different tests. One with XLA and one without.

PiperOrigin-RevId: 229149574
---
 tensorflow/python/BUILD                       |   8 +
 tensorflow/python/framework/function_test.py  |   1 +
 .../python/framework/is_xla_test_true.py      |  29 +++
 tensorflow/python/framework/test_util.py      | 166 ++++++++++++------
 tensorflow/python/kernel_tests/BUILD          | 130 ++++++++++++++
 .../python/kernel_tests/atrous_conv2d_test.py |   1 +
 .../python/kernel_tests/benchmark_test.py     |   2 +
 .../python/kernel_tests/check_ops_test.py     |   4 +
 .../python/kernel_tests/cholesky_op_test.py   |   1 +
 .../python/kernel_tests/concat_op_test.py     |   5 +
 .../kernel_tests/control_flow_ops_py_test.py  |   2 +
 .../kernel_tests/depthtospace_op_test.py      |   1 +
 .../kernel_tests/depthwise_conv_op_test.py    |   2 +
 .../python/kernel_tests/diag_op_test.py       |   2 +
 .../python/kernel_tests/distributions/BUILD   |  17 ++
 .../kernel_tests/functional_ops_test.py       |   4 +
 .../python/kernel_tests/gather_nd_op_test.py  |   3 +
 .../python/kernel_tests/gather_op_test.py     |   2 +
 tensorflow/python/kernel_tests/linalg/BUILD   |  16 ++
 tensorflow/python/kernel_tests/pool_test.py   |   1 +
 .../python/kernel_tests/pooling_ops_test.py   |  20 ++-
 tensorflow/python/kernel_tests/random/BUILD   |   8 +
 .../kernel_tests/random/random_ops_test.py    |   1 +
 .../python/kernel_tests/relu_op_test.py       |   1 +
 .../resource_variable_ops_test.py             |   1 +
 .../python/kernel_tests/scan_ops_test.py      |   2 +
 .../kernel_tests/scatter_nd_ops_test.py       |   1 +
 tensorflow/python/kernel_tests/signal/BUILD   |   8 +
 .../kernel_tests/signal/fft_ops_test.py       |   1 +
 .../kernel_tests/spacetodepth_op_test.py      |   1 +
 .../python/kernel_tests/split_op_test.py      |   1 +
 tensorflow/python/ops/bitwise_ops_test.py     |   3 +
 tensorflow/python/ops/nn_batchnorm_test.py    |   1 +
 .../python/ops/nn_fused_batchnorm_test.py     |   1 +
 tensorflow/python/ops/nn_test.py              |   2 +
 tensorflow/python/ops/parallel_for/BUILD      |   2 +
 .../ops/parallel_for/control_flow_ops_test.py |   1 +
 .../python/ops/parallel_for/gradients_test.py |   1 +
 .../python/ops/special_math_ops_test.py       |   1 +
 tensorflow/tensorflow.bzl                     |  23 ++-
 40 files changed, 422 insertions(+), 55 deletions(-)
 create mode 100644 tensorflow/python/framework/is_xla_test_true.py

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 659f0735cb..2c6672c81e 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -1103,6 +1103,14 @@ py_library(
     ],
 )
 
+# Including this as a dependency will result in tests using
+# :framework_test_lib to use XLA.
+py_library(
+    name = "is_xla_test_true",
+    srcs = ["framework/is_xla_test_true.py"],
+    srcs_version = "PY2AND3",
+)
+
 py_library(
     name = "distributed_framework_test_lib",
     srcs_version = "PY2AND3",
diff --git a/tensorflow/python/framework/function_test.py b/tensorflow/python/framework/function_test.py
index 7543376bcf..3d5a5fe797 100644
--- a/tensorflow/python/framework/function_test.py
+++ b/tensorflow/python/framework/function_test.py
@@ -284,6 +284,7 @@ class FunctionTest(test.TestCase):
         out, = sess.run(dlogits, {logits: x, labels: y})
       self.assertAllClose(out, np.exp(prob - y))
 
+  @test_util.disable_xla("This test never passed for XLA")
   def testCustomGradientError(self):
     dtype = dtypes.float32
 
diff --git a/tensorflow/python/framework/is_xla_test_true.py b/tensorflow/python/framework/is_xla_test_true.py
new file mode 100644
index 0000000000..7ae1e68b36
--- /dev/null
+++ b/tensorflow/python/framework/is_xla_test_true.py
@@ -0,0 +1,29 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Including this as a dependency will result in Tensorflow tests using XLA.
+
+This function is defined by default in test_util.py to False. The test_util then
+attempts to import this module. If this file is made available through the BUILD
+rule, then this function is overridden and will instead cause Tensorflow graphs
+to be compiled with XLA.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+def is_xla_enabled():
+  """Returns true to state XLA should be enabled for Tensorflow tests."""
+  return True
diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py
index 6d01d3bf54..491249089e 100644
--- a/tensorflow/python/framework/test_util.py
+++ b/tensorflow/python/framework/test_util.py
@@ -82,6 +82,19 @@ from tensorflow.python.util.protobuf import compare
 from tensorflow.python.util.tf_export import tf_export
 
 
+# If the above import is made available through the BUILD rule, then this
+# function is overridden and will instead return True and cause Tensorflow
+# graphs to be compiled with XLA.
+def is_xla_enabled():
+  return False
+
+
+try:
+  from tensorflow.python.framework.is_xla_test_true import is_xla_enabled  # pylint: disable=g-import-not-at-top
+except:
+  pass
+
+
 @tf_export("test.gpu_device_name")
 def gpu_device_name():
   """Returns the name of a GPU device if available or the empty string."""
@@ -97,6 +110,7 @@ def assert_ops_in_graph(expected_ops, graph):
   Args:
     expected_ops: `dict<string, string>` of op name to op type.
     graph: Graph to check.
+
   Returns:
     `dict<string, node>` of node name to node.
 
@@ -149,7 +163,7 @@ def assert_equal_graph_def_v1(actual, expected, checkpoint_v2=False):
     actual: The `GraphDef` we have.
     expected: The `GraphDef` we expected.
     checkpoint_v2: boolean determining whether to ignore randomized attribute
-        values that appear in V2 checkpoints.
+      values that appear in V2 checkpoints.
 
   Raises:
     AssertionError: If the `GraphDef`s do not match.
@@ -360,7 +374,8 @@ def skip_if(condition):
 
   Args:
     condition: Either an expression that can be used in "if not condition"
-               statement, or a callable whose result should be a boolean.
+      statement, or a callable whose result should be a boolean.
+
   Returns:
     The wrapped function
   """
@@ -483,9 +498,11 @@ def disable_control_flow_v2(unused_msg):
   Returns:
     The wrapped function with _disable_control_flow_v2 attr set to True.
   """
+
   def wrapper(func):
     func._disable_control_flow_v2 = True
     return func
+
   return wrapper
 
 
@@ -568,6 +585,7 @@ def assert_no_new_tensors(f):
 
   Args:
     f: The test case to run.
+
   Returns:
     The decorated test case.
   """
@@ -727,6 +745,7 @@ def assert_no_garbage_created(f):
 
   Args:
     f: The function to decorate.
+
   Returns:
     The decorated function.
   """
@@ -799,8 +818,8 @@ def _combine_named_parameters(**kwargs):
   can be computed using `times()`.
 
   Args:
-    **kwargs: keyword arguments of form `option=[possibilities, ...]`
-         or `option=the_only_possibility`.
+    **kwargs: keyword arguments of form `option=[possibilities, ...]` or
+      `option=the_only_possibility`.
 
   Returns:
     a list of dictionaries for each combination. Keys in the dictionaries are
@@ -838,8 +857,8 @@ def generate_combinations_with_testcase_name(**kwargs):
   parameterized tests.
 
   Args:
-    **kwargs: keyword arguments of form `option=[possibilities, ...]`
-         or `option=the_only_possibility`.
+    **kwargs: keyword arguments of form `option=[possibilities, ...]` or
+      `option=the_only_possibility`.
 
   Returns:
     a list of dictionaries for each combination. Keys in the dictionaries are
@@ -866,11 +885,12 @@ def generate_combinations_with_testcase_name(**kwargs):
 def run_all_in_graph_and_eager_modes(cls):
   """Execute all test methods in the given class with and without eager."""
   base_decorator = run_in_graph_and_eager_modes
-  for name, value in cls.__dict__.copy().items():
-    if (callable(value) and
-        name.startswith(unittest.TestLoader.testMethodPrefix) and
-        not (name.startswith("testSkipEager")
-             or name.startswith("test_skip_eager"))):
+  for name in dir(cls):
+    value = getattr(cls, name)
+    if callable(value) and name.startswith(
+        unittest.TestLoader.testMethodPrefix) and not (
+            name.startswith("testSkipEager") or
+            name.startswith("test_skip_eager") or name == "test_session"):
       setattr(cls, name, base_decorator(value))
   return cls
 
@@ -917,8 +937,8 @@ def run_in_graph_and_eager_modes(func=None,
     func: function to be annotated. If `func` is None, this method returns a
       decorator the can be applied to a function. If `func` is not None this
       returns the decorator applied to `func`.
-    config: An optional config_pb2.ConfigProto to use to configure the
-      session when executing graphs.
+    config: An optional config_pb2.ConfigProto to use to configure the session
+      when executing graphs.
     use_gpu: If True, attempt to run as many operations as possible on GPU.
     reset_test: If True, tearDown and SetUp the test case between the two
       executions of the test (once with and once without eager execution).
@@ -932,6 +952,7 @@ def run_in_graph_and_eager_modes(func=None,
       collected elsewhere in the unit test file will not work). Additionally,
       checks that nothing still has a reference to Tensors that the test
       allocated.
+
   Returns:
     Returns a decorator that will run the decorated test method twice:
     once by constructing and executing a graph in a session and once with
@@ -992,9 +1013,10 @@ def py_func_if_in_function(f):
     if not ops.get_default_graph()._building_function:
       return f(*args, **kwds)
 
-    tensor_args, tensor_indices = zip(
-        *[(x, i) for i, x in enumerate(args)
-          if isinstance(x, (ops.Tensor, variables.Variable))])
+    tensor_args, tensor_indices = zip(*[(x, i)
+                                        for i, x in enumerate(args)
+                                        if isinstance(x, (ops.Tensor,
+                                                          variables.Variable))])
 
     def inner_f(*inner_tensor_args):
       my_args = list(args)
@@ -1056,6 +1078,7 @@ def deprecated_graph_mode_only(func=None):
     func: function to be annotated. If `func` is None, this method returns a
       decorator the can be applied to a function. If `func` is not None this
       returns the decorator applied to `func`.
+
   Returns:
     Returns a decorator that will run the decorated test method in graph mode.
   """
@@ -1390,8 +1413,7 @@ class FakeEagerSession(object):
 
 
 class ErrorLoggingSession(session.Session):
-  """Wrapper around a Session that logs errors in run().
-  """
+  """Wrapper around a Session that logs errors in run()."""
 
   def run(self, *args, **kwargs):
     try:
@@ -1405,13 +1427,56 @@ class ErrorLoggingSession(session.Session):
       raise
 
 
+# The description is just for documentation purposes.
+def disable_xla(description):
+
+  def disable_xla_impl(func):
+    """Execute the test method only if xla is not enabled."""
+
+    def decorator(func):
+
+      def decorated(self, *args, **kwargs):
+        if is_xla_enabled():
+          return
+        else:
+          return func(self, *args, **kwargs)
+
+      return decorated
+
+    if func is not None:
+      return decorator(func)
+
+    return decorator
+
+  return disable_xla_impl
+
+
+# The description is just for documentation purposes.
+def disable_all_xla(description):
+
+  def disable_all_impl(cls):
+    """Execute all test methods in this class only if xla is not enabled."""
+    base_decorator = disable_xla
+    for name in dir(cls):
+      value = getattr(cls, name)
+      if callable(value) and name.startswith(
+          "test") and not name == "test_session":
+        setattr(cls, name, base_decorator(value))
+    return cls
+
+  return disable_all_impl
+
+
 @tf_export("test.TestCase")
 class TensorFlowTestCase(googletest.TestCase):
-  """Base class for tests that need to test TensorFlow.
-  """
+  """Base class for tests that need to test TensorFlow."""
 
   def __init__(self, methodName="runTest"):  # pylint: disable=invalid-name
     super(TensorFlowTestCase, self).__init__(methodName)
+    if is_xla_enabled():
+      os.putenv(
+          "TF_XLA_FLAGS", "--tf_xla_auto_jit=2 --tf_xla_min_cluster_size=1 "
+          "--tf_xla_enable_lazy_compilation=false")
     self._threads = []
     self._tempdir = None
     self._cached_session = None
@@ -1489,9 +1554,9 @@ class TensorFlowTestCase(googletest.TestCase):
     ```
 
     Args:
-      stream: The stream whose writes should be captured. This
-        stream must have a file descriptor, support writing via using that
-        file descriptor, and must have a `.flush()` method.
+      stream: The stream whose writes should be captured. This stream must have
+        a file descriptor, support writing via using that file descriptor, and
+        must have a `.flush()` method.
 
     Yields:
       A `CapturedWrites` object that contains all writes to the specified stream
@@ -1840,7 +1905,6 @@ class TensorFlowTestCase(googletest.TestCase):
     self._threads.append(ret)
     return ret
 
-
   # pylint: enable=invalid-name
   @py_func_if_in_function
   def assertNear(self, f1, f2, err, msg=None):
@@ -1857,9 +1921,8 @@ class TensorFlowTestCase(googletest.TestCase):
     """
     # f1 == f2 is needed here as we might have: f1, f2 = inf, inf
     self.assertTrue(
-        f1 == f2 or math.fabs(f1 - f2) <= err,
-        "%f != %f +/- %f%s" % (f1, f2, err, " (%s)" % msg
-                               if msg is not None else ""))
+        f1 == f2 or math.fabs(f1 - f2) <= err, "%f != %f +/- %f%s" %
+        (f1, f2, err, " (%s)" % msg if msg is not None else ""))
 
   @py_func_if_in_function
   def assertArrayNear(self, farray1, farray2, err, msg=None):
@@ -2028,11 +2091,11 @@ class TensorFlowTestCase(googletest.TestCase):
 
     Args:
       a: The expected numpy `ndarray`, or anything that can be converted into a
-         numpy `ndarray` (including Tensor), or any arbitrarily nested of
-         structure of these.
+        numpy `ndarray` (including Tensor), or any arbitrarily nested of
+        structure of these.
       b: The actual numpy `ndarray`, or anything that can be converted into a
-         numpy `ndarray` (including Tensor), or any arbitrarily nested of
-         structure of these.
+        numpy `ndarray` (including Tensor), or any arbitrarily nested of
+        structure of these.
       rtol: relative tolerance.
       atol: absolute tolerance.
       msg: Optional message to report on failure.
@@ -2160,8 +2223,8 @@ class TensorFlowTestCase(googletest.TestCase):
     """Assert element values are all greater than a target value.
 
     Args:
-      a: The numpy `ndarray`, or anything that can be converted into a
-         numpy `ndarray` (including Tensor).
+      a: The numpy `ndarray`, or anything that can be converted into a numpy
+        `ndarray` (including Tensor).
       comparison_target: The target value of comparison.
     """
     a = self._GetNdArray(a)
@@ -2172,8 +2235,8 @@ class TensorFlowTestCase(googletest.TestCase):
     """Assert element values are all less than a target value.
 
     Args:
-      a: The numpy `ndarray`, or anything that can be converted into a
-         numpy `ndarray` (including Tensor).
+      a: The numpy `ndarray`, or anything that can be converted into a numpy
+        `ndarray` (including Tensor).
       comparison_target: The target value of comparison.
     """
     a = self._GetNdArray(a)
@@ -2184,8 +2247,8 @@ class TensorFlowTestCase(googletest.TestCase):
     """Assert element values are all greater than or equal to a target value.
 
     Args:
-      a: The numpy `ndarray`, or anything that can be converted into a
-         numpy `ndarray` (including Tensor).
+      a: The numpy `ndarray`, or anything that can be converted into a numpy
+        `ndarray` (including Tensor).
       comparison_target: The target value of comparison.
     """
     a = self._GetNdArray(a)
@@ -2196,8 +2259,8 @@ class TensorFlowTestCase(googletest.TestCase):
     """Assert element values are all less than or equal to a target value.
 
     Args:
-      a: The numpy `ndarray`, or anything that can be converted into a
-         numpy `ndarray` (including Tensor).
+      a: The numpy `ndarray`, or anything that can be converted into a numpy
+        `ndarray` (including Tensor).
       comparison_target: The target value of comparison.
     """
     a = self._GetNdArray(a)
@@ -2245,7 +2308,7 @@ class TensorFlowTestCase(googletest.TestCase):
 
     Args:
       target: The numpy `ndarray`, or anything that can be converted into a
-         numpy `ndarray` (including Tensor).
+        numpy `ndarray` (including Tensor).
       lower_bound: lower bound of the range
       upper_bound: upper bound of the range
       open_lower_bound: (`bool`) whether the lower bound is open (i.e., > rather
@@ -2279,8 +2342,8 @@ class TensorFlowTestCase(googletest.TestCase):
                  str(upper_bound) + (")" if open_upper_bound else "]"))
 
     violations = (
-        np.less_equal(target, lower_bound)
-        if open_lower_bound else np.less(target, lower_bound))
+        np.less_equal(target, lower_bound) if open_lower_bound else np.less(
+            target, lower_bound))
     violations = np.logical_or(
         violations,
         np.greater_equal(target, upper_bound)
@@ -2299,7 +2362,7 @@ class TensorFlowTestCase(googletest.TestCase):
 
     Args:
       target: The numpy `ndarray`, or anything that can be converted into a
-         numpy `ndarray` (including Tensor).
+        numpy `ndarray` (including Tensor).
       expected_set: (`list`, `tuple` or `set`) The closed set that the elements
         of the value of `target` are expected to fall into.
 
@@ -2321,7 +2384,7 @@ class TensorFlowTestCase(googletest.TestCase):
 
     Args:
       target: The numpy `ndarray`, or anything that can be converted into a
-         numpy `ndarray` (including Tensor).
+        numpy `ndarray` (including Tensor).
       expected_dtype: Expected data type.
     """
     target = self._GetNdArray(target)
@@ -2342,9 +2405,9 @@ class TensorFlowTestCase(googletest.TestCase):
     Args:
       exception_type: The expected type of exception that should be raised.
       expected_err_re_or_predicate: If this is callable, it should be a function
-        of one argument that inspects the passed-in exception and
-        returns True (success) or False (please fail the test). Otherwise, the
-        error message is expected to match this regular expression partially.
+        of one argument that inspects the passed-in exception and returns True
+        (success) or False (please fail the test). Otherwise, the error message
+        is expected to match this regular expression partially.
 
     Returns:
       A context manager to surround code that is expected to raise an
@@ -2445,6 +2508,7 @@ class TensorFlowTestCase(googletest.TestCase):
 
   def _create_session(self, graph, config, force_gpu):
     """See session() for details."""
+
     def prepare_config(config):
       """Returns a config for sessions.
 
@@ -2547,10 +2611,10 @@ def create_local_cluster(num_workers,
   Args:
     num_workers: Number of worker servers to start.
     num_ps: Number of PS servers to start.
-    protocol: Communication protocol.  Allowed values are documented in
-      the documentation of `tf.train.Server`.
-    worker_config: (optional) ConfigProto to initialize workers. Can be used
-      to instantiate multiple devices etc.
+    protocol: Communication protocol.  Allowed values are documented in the
+      documentation of `tf.train.Server`.
+    worker_config: (optional) ConfigProto to initialize workers. Can be used to
+      instantiate multiple devices etc.
     ps_config: (optional) ConfigProto to initialize PS servers.
 
   Returns:
diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index 276342cbbc..d9ea42374d 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -131,6 +131,7 @@ cuda_py_test(
         "//tensorflow/python:client_testlib",
     ],
     grpc_enabled = True,
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -145,6 +146,7 @@ cuda_py_test(
         "//tensorflow/python:platform_benchmark",
     ],
     tags = ["no_windows"],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -161,6 +163,7 @@ cuda_py_test(
         "//tensorflow/python:platform",
         "//tensorflow/python:platform_benchmark",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 tf_py_test(
@@ -226,6 +229,7 @@ cuda_py_test(
     ],
     shard_count = 5,
     tags = ["no_windows_gpu"],
+    xla_enable_strict_auto_jit = True,
 )
 
 tf_py_test(
@@ -282,6 +286,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:gradients",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 tf_py_test(
@@ -387,6 +392,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:linalg_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 tf_py_test(
@@ -645,6 +651,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
     ],
     tags = ["optonly"],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -659,6 +666,7 @@ cuda_py_test(
         "//tensorflow/python:linalg_ops",
         "//tensorflow/python:math_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -671,6 +679,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:linalg_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -684,6 +693,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:linalg_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -695,6 +705,7 @@ cuda_py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:linalg_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -713,6 +724,7 @@ cuda_py_test(
         "//tensorflow/python:platform",
         "//tensorflow/python:random_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 tf_py_test(
@@ -795,6 +807,7 @@ cuda_py_test(
         "//tensorflow/python:resource_variable_ops",
         "//tensorflow/python:variables",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 tf_py_test(
@@ -853,6 +866,7 @@ cuda_py_test(
         "//tensorflow/python:resource_variable_ops",
     ],
     tags = ["noasan"],  # http://b/32635055
+    xla_enable_strict_auto_jit = True,
 )
 
 tf_py_test(
@@ -1159,6 +1173,7 @@ cuda_py_test(
         "//tensorflow/python:nn_grad",
         "//tensorflow/python:nn_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1174,6 +1189,7 @@ cuda_py_test(
         "//tensorflow/python:nn_grad",
         "//tensorflow/python:nn_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 tf_py_test(
@@ -1290,6 +1306,7 @@ cuda_py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_for_generated_wrappers",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1311,6 +1328,7 @@ cuda_py_test(
         "noguitar",
         "notap",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1326,6 +1344,7 @@ cuda_py_test(
         "//tensorflow/python:variables",
     ],
     tags = ["notsan"],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1342,6 +1361,7 @@ cuda_py_test(
     ],
     shard_count = 2,
     tags = ["no_windows_gpu"],
+    xla_enable_strict_auto_jit = True,
 )
 
 tf_py_test(
@@ -1374,6 +1394,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:math_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1385,6 +1406,7 @@ cuda_py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:math_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1413,6 +1435,7 @@ cuda_py_test(
         "noasan",  # times out
         "optonly",  # times out
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1425,6 +1448,7 @@ cuda_py_test(
         "//tensorflow/python:client",
         "//tensorflow/python:client_testlib",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1441,6 +1465,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
     ],
     shard_count = 10,
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1455,6 +1480,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
     ],
     shard_count = 20,
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1468,6 +1494,7 @@ cuda_py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_for_generated_wrappers",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1482,6 +1509,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:platform",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1497,6 +1525,7 @@ cuda_py_test(
         "//tensorflow/python:nn_grad",
         "//tensorflow/python:nn_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1509,6 +1538,7 @@ cuda_py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_for_generated_wrappers",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1527,6 +1557,7 @@ cuda_py_test(
         "//tensorflow/python:framework",
         "//tensorflow/python:framework_for_generated_wrappers",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1542,6 +1573,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:util",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1560,6 +1592,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:util",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1600,6 +1633,7 @@ cuda_py_test(
     ],
     shard_count = 16,
     tags = ["no_gpu"],  # TODO(b/117928656)
+    xla_enable_strict_auto_jit = True,
 )
 
 tf_py_test(
@@ -1640,6 +1674,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:nn_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1654,6 +1689,7 @@ cuda_py_test(
         "//tensorflow/python:nn_grad",
         "//tensorflow/python:nn_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1668,6 +1704,7 @@ cuda_py_test(
         "//tensorflow/python:nn_grad",
         "//tensorflow/python:nn_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1679,6 +1716,7 @@ cuda_py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:math_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1692,6 +1730,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:platform",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1707,6 +1746,7 @@ cuda_py_test(
         "//tensorflow/python:state_ops",
         "//tensorflow/python:variables",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1721,6 +1761,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
     ],
     tags = ["no_windows_gpu"],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1733,6 +1774,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
     ],
     tags = ["manual"],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1748,6 +1790,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:gradients",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1762,6 +1805,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:gradients",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1774,6 +1818,7 @@ cuda_py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_for_generated_wrappers",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1786,6 +1831,7 @@ cuda_py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_for_generated_wrappers",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1812,6 +1858,7 @@ cuda_py_test(
     grpc_enabled = True,
     shard_count = 2,
     tags = ["no_windows"],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1827,6 +1874,7 @@ cuda_py_test(
         "//tensorflow/python:gradients",
         "//tensorflow/python:variables",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1841,6 +1889,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:gradients",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1854,6 +1903,7 @@ cuda_py_test(
         "//tensorflow/python:gradients",
         "//tensorflow/python:math_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1881,6 +1931,7 @@ cuda_py_test(
         "noasan",
         "notap",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1898,6 +1949,7 @@ cuda_py_test(
         "//tensorflow/python/ops/linalg",
     ],
     tags = ["no_windows_gpu"],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1913,6 +1965,7 @@ cuda_py_test(
         "//tensorflow/python:nn",
         "//tensorflow/python:nn_grad",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1929,6 +1982,7 @@ cuda_py_test(
         "//tensorflow/python:platform",
         "//tensorflow/python/ops/linalg",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1942,6 +1996,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
     ],
     tags = ["no_windows_gpu"],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1959,6 +2014,7 @@ cuda_py_test(
         "//tensorflow/python:variables",
     ],
     shard_count = 20,
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1972,6 +2028,7 @@ cuda_py_test(
         "//tensorflow/python:nn_grad",
         "//tensorflow/python:nn_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1987,6 +2044,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:numerics",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2000,6 +2058,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
     ],
     tags = ["no_windows_gpu"],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2014,6 +2073,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:variables",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2026,6 +2086,7 @@ cuda_py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_for_generated_wrappers",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2040,6 +2101,7 @@ cuda_py_test(
         "//tensorflow/python:errors",
         "//tensorflow/python:framework_for_generated_wrappers",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2058,6 +2120,7 @@ cuda_py_test(
         "//tensorflow/python/eager:function",
     ],
     tags = ["no_windows"],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2071,6 +2134,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:string_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2086,6 +2150,7 @@ cuda_py_test(
     ],
     shard_count = 4,
     tags = ["no_windows_gpu"],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2106,6 +2171,7 @@ cuda_py_test(
         "noguitar",
         "notap",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2124,6 +2190,7 @@ cuda_py_test(
         "//tensorflow/python/eager:backprop",
         "//tensorflow/python:tf2",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2136,6 +2203,7 @@ cuda_py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_for_generated_wrappers",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2148,6 +2216,7 @@ cuda_py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_for_generated_wrappers",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2160,6 +2229,7 @@ cuda_py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_for_generated_wrappers",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2178,6 +2248,7 @@ cuda_py_test(
         "//tensorflow/python:random_ops",
         "//tensorflow/python:sparse_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2191,6 +2262,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:math_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2203,6 +2275,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:session_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2219,6 +2292,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:gradients",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2233,6 +2307,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:nn_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2246,6 +2321,7 @@ cuda_py_test(
         "//tensorflow/python:nn_grad",
         "//tensorflow/python:nn_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2259,6 +2335,7 @@ cuda_py_test(
         "//tensorflow/python:nn_grad",
         "//tensorflow/python:nn_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2274,6 +2351,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:math_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2288,6 +2366,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
     ],
     tags = ["no_windows_gpu"],
+    xla_enable_strict_auto_jit = True,
 )
 
 tf_py_test(
@@ -2332,6 +2411,7 @@ cuda_py_test(
         "//tensorflow/python:sparse_grad",
         "//tensorflow/python:sparse_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2356,6 +2436,7 @@ cuda_py_test(
         "//tensorflow/python:random_ops",
         "//tensorflow/python:variables",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2370,6 +2451,7 @@ cuda_py_test(
         "//tensorflow/python:gradients",
         "//tensorflow/python:math_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2385,6 +2467,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:math_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2397,6 +2480,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:string_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2409,6 +2493,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:parsing_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2422,6 +2507,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:summary",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2437,6 +2523,7 @@ cuda_py_test(
         "//tensorflow/python:nn_grad",
         "//tensorflow/python:summary",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2467,6 +2554,7 @@ cuda_py_test(
         "//tensorflow/python/eager:context",
     ],
     flaky = 1,  # create_local_cluster sometimes times out.
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2479,6 +2567,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
     ],
     tags = ["no_windows_gpu"],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2497,6 +2586,7 @@ cuda_py_test(
         "no_oss",
         "optonly",  # times out
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2509,6 +2599,7 @@ cuda_py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_for_generated_wrappers",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2526,6 +2617,7 @@ cuda_py_test(
         "//tensorflow/python:state_ops_gen",
         "//tensorflow/python:variables",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2542,6 +2634,7 @@ cuda_py_test(
         "//tensorflow/python:nn_ops",
         "//tensorflow/python:nn_ops_gen",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2553,6 +2646,7 @@ cuda_py_test(
         "//tensorflow/python:errors",
         "//tensorflow/python:framework_for_generated_wrappers",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2572,6 +2666,7 @@ cuda_py_test(
     tags = [
         "no_gpu",  #  Flaky: b/80127739
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2587,6 +2682,7 @@ cuda_py_test(
         "//tensorflow/python:nn_ops",
     ],
     tags = ["manual"],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2600,6 +2696,7 @@ cuda_py_test(
         "//tensorflow/python:nn_grad",
         "//tensorflow/python:nn_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2618,6 +2715,7 @@ cuda_py_test(
     tags = [
         "optonly",  # flaky timeouts unless optimized
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2631,6 +2729,7 @@ cuda_py_test(
         "//tensorflow/python:nn_grad",
         "//tensorflow/python:nn_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2659,6 +2758,7 @@ cuda_py_test(
         "no_oss",
         "optonly",  # times out
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2676,6 +2776,7 @@ cuda_py_test(
     ],
     # TODO(b/118842098): Re-enable this test in Kokoro.
     tags = ["no_oss"],
+    xla_enable_strict_auto_jit = True,
 )
 
 tf_py_test(
@@ -2704,6 +2805,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
     ],
     tags = ["manual"],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2717,6 +2819,7 @@ cuda_py_test(
         "//tensorflow/python:nn_grad",
         "//tensorflow/python:nn_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2735,6 +2838,7 @@ cuda_py_test(
         "//tensorflow/python:nn_ops_gen",
     ],
     shard_count = 4,
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2764,6 +2868,7 @@ cuda_py_test(
         "//tensorflow/python/eager:context",
     ],
     shard_count = 10,
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2779,6 +2884,7 @@ cuda_py_test(
     ],
     shard_count = 2,
     tags = ["optonly"],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2793,6 +2899,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:gradients",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2809,6 +2916,7 @@ cuda_py_test(
     tags = [
         "no_oss",  # Requires 4GB+ RAM
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2822,6 +2930,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
     ],
     tags = ["no_windows"],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2845,6 +2954,7 @@ cuda_py_test(
         "noasan",
         "optonly",  # b/77589990
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2864,6 +2974,7 @@ cuda_py_test(
         "//tensorflow/python:platform",
         "//tensorflow/python:sparse_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 # TODO(gpapan): Revisit the gradient of extract_image_patches_op to resolve
@@ -2880,6 +2991,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
     ],
     tags = ["notap"],  # http://b/31080670
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2894,6 +3006,7 @@ cuda_py_test(
         "//tensorflow/python:util",
         "//tensorflow/python:data_flow_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2908,6 +3021,7 @@ cuda_py_test(
         "//tensorflow/python:util",
         "//tensorflow/python:data_flow_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2925,6 +3039,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:variables",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2940,6 +3055,7 @@ cuda_py_test(
         "nomsan",
         "notsan",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2953,6 +3069,7 @@ cuda_py_test(
         "//tensorflow/python:nn_ops",
     ],
     shard_count = 30,
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2973,6 +3090,7 @@ cuda_py_test(
         "//tensorflow/python:variables",
     ],
     shard_count = 50,
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2993,6 +3111,7 @@ cuda_py_test(
         "//tensorflow/python:variables",
     ],
     shard_count = 50,
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -3013,6 +3132,7 @@ cuda_py_test(
         "//tensorflow/python:variables",
     ],
     shard_count = 50,
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -3038,6 +3158,7 @@ cuda_py_test(
         "//tensorflow/python:variables",
     ],
     shard_count = 20,
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -3055,6 +3176,7 @@ cuda_py_test(
         "//tensorflow/python/ops/linalg",
     ],
     shard_count = 20,
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -3068,6 +3190,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
     ],
     shard_count = 20,
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -3085,6 +3208,7 @@ cuda_py_test(
     data = ["//tensorflow/python/kernel_tests/testdata:self_adjoint_eig_op_test_files"],
     shard_count = 20,
     tags = ["no_windows"],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -3100,6 +3224,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
     ],
     shard_count = 20,
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -3119,6 +3244,7 @@ cuda_py_test(
         "no_oss",  # b/117185141.
         "nomsan",  # TODO(b/117236102): Re-enable in msan build.
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -3138,6 +3264,7 @@ cuda_py_test(
         "no_windows_gpu",
         "nomsan",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -3153,6 +3280,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
     ],
     shard_count = 20,
+    xla_enable_strict_auto_jit = True,
 )
 
 sycl_py_test(
@@ -3399,6 +3527,7 @@ cuda_py_test(
         "//tensorflow/python:while_v2",
     ],
     grpc_enabled = True,
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -3424,4 +3553,5 @@ cuda_py_test(
         "//tensorflow/python:while_v2",
     ],
     grpc_enabled = True,
+    xla_enable_strict_auto_jit = True,
 )
diff --git a/tensorflow/python/kernel_tests/atrous_conv2d_test.py b/tensorflow/python/kernel_tests/atrous_conv2d_test.py
index a13e325835..5d7b2dd30f 100644
--- a/tensorflow/python/kernel_tests/atrous_conv2d_test.py
+++ b/tensorflow/python/kernel_tests/atrous_conv2d_test.py
@@ -139,6 +139,7 @@ class AtrousConv2DTest(test.TestCase):
                   y1.eval(), self.evaluate(y2), rtol=1e-2, atol=1e-2)
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA")  # larger error range
   def testGradient(self):
     with self.session(use_gpu=True):
       # Input: [batch, height, width, input_depth]
diff --git a/tensorflow/python/kernel_tests/benchmark_test.py b/tensorflow/python/kernel_tests/benchmark_test.py
index bffa5e6e8f..a91f96cf95 100644
--- a/tensorflow/python/kernel_tests/benchmark_test.py
+++ b/tensorflow/python/kernel_tests/benchmark_test.py
@@ -26,6 +26,7 @@ import numpy as np
 from tensorflow.core.util import test_log_pb2
 from tensorflow.python.client import session
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import benchmark
 from tensorflow.python.platform import gfile
@@ -125,6 +126,7 @@ class BenchmarkTest(test.TestCase):
     self.assertFalse(_ran_somebenchmark_2[0])
     self.assertFalse(_ran_somebenchmark_but_shouldnt[0])
 
+  @test_util.disable_xla("This test never passed for XLA")
   def testReportingBenchmark(self):
     tempdir = test.get_temp_dir()
     try:
diff --git a/tensorflow/python/kernel_tests/check_ops_test.py b/tensorflow/python/kernel_tests/check_ops_test.py
index 95bac85027..d5f3696a9d 100644
--- a/tensorflow/python/kernel_tests/check_ops_test.py
+++ b/tensorflow/python/kernel_tests/check_ops_test.py
@@ -889,6 +889,8 @@ class EnsureShapeTest(test.TestCase):
 
   # Dynamic shape check
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA"
+                        )  # Dynamic shapes not supported now with XLA
   def testEnsuresDynamicShape_RaisesError(self):
     placeholder = array_ops.placeholder(dtypes.int32)
     derived = math_ops.divide(placeholder, 3, name="MyDivide")
@@ -902,6 +904,8 @@ class EnsureShapeTest(test.TestCase):
         sess.run(derived, feed_dict={placeholder: feed_val})
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA"
+                        )  # Dynamic shapes not supported now with XLA
   def testEnsuresDynamicShape_RaisesErrorDimUnknown(self):
     placeholder = array_ops.placeholder(dtypes.int32)
     derived = placeholder / 3
diff --git a/tensorflow/python/kernel_tests/cholesky_op_test.py b/tensorflow/python/kernel_tests/cholesky_op_test.py
index a08cfe960d..abb71a672c 100644
--- a/tensorflow/python/kernel_tests/cholesky_op_test.py
+++ b/tensorflow/python/kernel_tests/cholesky_op_test.py
@@ -163,6 +163,7 @@ class CholeskyOpTest(test.TestCase):
     with self.assertRaises(ValueError):
       linalg_ops.cholesky(tensor3)
 
+  @test_util.disable_xla("This test never passed for XLA")  # all nan on XLA
   def testNotInvertibleCPU(self):
     # The input should be invertible.
     with self.session(use_gpu=True):
diff --git a/tensorflow/python/kernel_tests/concat_op_test.py b/tensorflow/python/kernel_tests/concat_op_test.py
index 474760a93f..a968b06127 100644
--- a/tensorflow/python/kernel_tests/concat_op_test.py
+++ b/tensorflow/python/kernel_tests/concat_op_test.py
@@ -33,6 +33,7 @@ from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
 
 
+@test_util.disable_all_xla("This test never passed for XLA")
 class ConcatOpTest(test.TestCase):
 
   @test_util.run_deprecated_v1
@@ -640,6 +641,8 @@ class ConcatOpTest(test.TestCase):
         output = self.evaluate(c)
         self.assertAllEqual([[1, 2, 3, 7, 8, 9], [4, 5, 6, 10, 11, 12]], output)
 
+
+@test_util.disable_all_xla("This test never passed for XLA")
 class ConcatOffsetTest(test.TestCase):
 
   def testBasic(self):
@@ -683,6 +686,8 @@ class ConcatOffsetTest(test.TestCase):
       self.evaluate(off)
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla(
+      "This test never passed for XLA")  # Different error message on XLA
   def testSizeMismatch(self):
     cdim = constant_op.constant(1, dtypes.int32)
     s0 = constant_op.constant([2, 3, 5], dtypes.int32)
diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
index 25977510fb..870dd8285e 100644
--- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
@@ -2380,6 +2380,8 @@ class ControlFlowTest(test.TestCase):
       self.assertEqual(i_val, 3)
       self.assertAllClose(x_val, 1.0)
 
+  @test_util.disable_xla("This test never passed for XLA"
+                        )  # Resource variable issue for ControlFlowV2
   @test_util.run_gpu_only
   def testGpuResourceAccess(self):
     with ops.device(test.gpu_device_name()):
diff --git a/tensorflow/python/kernel_tests/depthtospace_op_test.py b/tensorflow/python/kernel_tests/depthtospace_op_test.py
index 96c9b5258e..97d3645b61 100644
--- a/tensorflow/python/kernel_tests/depthtospace_op_test.py
+++ b/tensorflow/python/kernel_tests/depthtospace_op_test.py
@@ -295,6 +295,7 @@ class DepthToSpaceTest(test.TestCase):
       actual_vals, expected_vals = self.evaluate([actual, expected])
       self.assertTrue(np.array_equal(actual_vals, expected_vals))
 
+  @test_util.disable_xla("This test never passed for XLA")
   def testAgainstTranspose(self):
     self.compareToTranspose(3, 2, 3, 1, 2, "NHWC", False)
     self.compareToTranspose(3, 2, 3, 2, 2, "NHWC", False)
diff --git a/tensorflow/python/kernel_tests/depthwise_conv_op_test.py b/tensorflow/python/kernel_tests/depthwise_conv_op_test.py
index 5b1a47fb03..6f6d17b4eb 100644
--- a/tensorflow/python/kernel_tests/depthwise_conv_op_test.py
+++ b/tensorflow/python/kernel_tests/depthwise_conv_op_test.py
@@ -481,6 +481,7 @@ class DepthwiseConv2DTest(test.TestCase):
             data_format="NCHW")
 
   @test_util.run_v1_only("b/120545219")
+  @test_util.disable_xla("This test never passed for XLA")
   def testDepthwiseConv2DFilterGrad(self):
     for index, (input_size, filter_size, output_size, stride,
                 padding) in enumerate(CheckGradConfigsToTest()):
@@ -612,6 +613,7 @@ class DepthwiseConv2DTest(test.TestCase):
     cpu_value = _GetVal(use_gpu=False)
     self.assertAllClose(cpu_value, gpu_value, rtol=1e-4, atol=1e-4)
 
+  @test_util.disable_xla("This test never passed for XLA")
   def testDepthwiseConv2DFilterGradCompare(self):
     for index, (input_size, filter_size, output_size, stride,
                 padding) in enumerate(ConfigsToTest()):
diff --git a/tensorflow/python/kernel_tests/diag_op_test.py b/tensorflow/python/kernel_tests/diag_op_test.py
index ed2a9e8e47..b8139918c5 100644
--- a/tensorflow/python/kernel_tests/diag_op_test.py
+++ b/tensorflow/python/kernel_tests/diag_op_test.py
@@ -65,6 +65,7 @@ class MatrixDiagTest(test.TestCase):
       array_ops.matrix_diag(0)
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA")
   def testInvalidShapeAtEval(self):
     with self.session(use_gpu=True):
       v = array_ops.placeholder(dtype=dtypes_lib.float32)
@@ -269,6 +270,7 @@ class MatrixDiagPartTest(test.TestCase):
       array_ops.matrix_diag_part(0)
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA")
   def testInvalidShapeAtEval(self):
     with self.session(use_gpu=True):
       v = array_ops.placeholder(dtype=dtypes_lib.float32)
diff --git a/tensorflow/python/kernel_tests/distributions/BUILD b/tensorflow/python/kernel_tests/distributions/BUILD
index a83622ac9a..7a10ed9852 100644
--- a/tensorflow/python/kernel_tests/distributions/BUILD
+++ b/tensorflow/python/kernel_tests/distributions/BUILD
@@ -23,6 +23,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:platform_test",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -42,6 +43,7 @@ cuda_py_test(
         "//tensorflow/python:platform_test",
     ],
     shard_count = 3,
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -54,6 +56,7 @@ cuda_py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:platform_test",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -71,6 +74,7 @@ cuda_py_test(
         "//tensorflow/python:nn_ops",
         "//tensorflow/python:platform_test",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -86,6 +90,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:platform_test",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -105,6 +110,7 @@ cuda_py_test(
         "//tensorflow/python:platform_test",
         "//tensorflow/python:random_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -119,6 +125,7 @@ cuda_py_test(
         "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:platform_test",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -139,6 +146,7 @@ cuda_py_test(
         "noguitar",  # b/110489471
         "notap",  # b/110489471
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -154,6 +162,7 @@ cuda_py_test(
         "//tensorflow/python:nn_ops",
         "//tensorflow/python:platform_test",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -169,6 +178,7 @@ cuda_py_test(
         "//tensorflow/python:nn_ops",
         "//tensorflow/python:platform_test",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -184,6 +194,7 @@ cuda_py_test(
         "//tensorflow/python:nn_ops",
         "//tensorflow/python:platform_test",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -200,6 +211,7 @@ cuda_py_test(
         "//tensorflow/python:platform_test",
     ],
     tags = ["manual"],  # b/69001419
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -224,6 +236,7 @@ cuda_py_test(
         # disable to avoid false positives from scipy.
         "nomsan",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -240,6 +253,7 @@ cuda_py_test(
         "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:math_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -258,6 +272,7 @@ cuda_py_test(
         "//tensorflow/python:platform_test",
         "//tensorflow/python:variables",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -274,6 +289,7 @@ cuda_py_test(
         "//tensorflow/python:platform_test",
         "//tensorflow/python:variables",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -291,4 +307,5 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:platform_test",
     ],
+    xla_enable_strict_auto_jit = True,
 )
diff --git a/tensorflow/python/kernel_tests/functional_ops_test.py b/tensorflow/python/kernel_tests/functional_ops_test.py
index 8988305bde..92f4e7b39e 100644
--- a/tensorflow/python/kernel_tests/functional_ops_test.py
+++ b/tensorflow/python/kernel_tests/functional_ops_test.py
@@ -58,6 +58,7 @@ def simple_scoped_fn(a, x):
 
 
 @test_util.with_control_flow_v2
+@test_util.disable_all_xla("This test never passed for XLA")
 class FunctionalOpsTest(test.TestCase):
 
   @test_util.run_in_graph_and_eager_modes
@@ -826,6 +827,8 @@ class FunctionalOpsTest(test.TestCase):
     self.assertAllEqual(Run(100., True), 5050.)
 
   @test_util.run_v1_only("b/120545219")
+  @test_util.disable_xla(
+      "This test never passed for XLA")  # Different error message
   def testWhileError(self):
     for use_gpu in (True, False):
       with ops.Graph().as_default() as g:
@@ -1102,6 +1105,7 @@ class FunctionalOpsTest(test.TestCase):
 
 # TODO(akshayka): Replace `function.Defun` with tf.contrib.eager.defun` in the
 # below test cases.
+@test_util.disable_all_xla("This test never passed for XLA")
 class PartitionedCallTest(test.TestCase):
 
   @test_util.run_deprecated_v1
diff --git a/tensorflow/python/kernel_tests/gather_nd_op_test.py b/tensorflow/python/kernel_tests/gather_nd_op_test.py
index 320ffc9674..76ae2fcb72 100644
--- a/tensorflow/python/kernel_tests/gather_nd_op_test.py
+++ b/tensorflow/python/kernel_tests/gather_nd_op_test.py
@@ -34,6 +34,7 @@ from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
 
 
+@test_util.disable_all_xla("This test never passed for XLA")
 class GatherNdTest(test.TestCase):
 
   def _testSimpleDtype(self, dtype):
@@ -56,6 +57,7 @@ class GatherNdTest(test.TestCase):
     self._testSimpleDtype("|S")  # byte strings in python2 + 3
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA")
   def testEmptyIndicesAndParamsOKButJustEmptyParamsFails(self):
     with self.session(use_gpu=True):
       params = np.ones((3, 3), dtype=np.float32)
@@ -358,6 +360,7 @@ class GatherNdTest(test.TestCase):
       self.assertAllEqual(expected_grads, ops.convert_to_tensor(grads).eval())
 
 
+@test_util.disable_all_xla("This test never passed for XLA")
 class GatherNdOpBenchmark(test.Benchmark):
 
   def benchmark_gather_nd_op(self):
diff --git a/tensorflow/python/kernel_tests/gather_op_test.py b/tensorflow/python/kernel_tests/gather_op_test.py
index 1d6f7955aa..3b8a9a38de 100644
--- a/tensorflow/python/kernel_tests/gather_op_test.py
+++ b/tensorflow/python/kernel_tests/gather_op_test.py
@@ -213,6 +213,8 @@ class GatherTest(test.TestCase, parameterized.TestCase):
         array_ops.gather(params, [[7]], axis=1).eval()
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla(
+      "This test never passed for XLA")  # Different error message.
   def testBadAxis(self):
     with self.session(use_gpu=True):
       params = [0, 1, 2]
diff --git a/tensorflow/python/kernel_tests/linalg/BUILD b/tensorflow/python/kernel_tests/linalg/BUILD
index ba9e64979a..c1aa99cc37 100644
--- a/tensorflow/python/kernel_tests/linalg/BUILD
+++ b/tensorflow/python/kernel_tests/linalg/BUILD
@@ -22,6 +22,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:platform_test",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -38,6 +39,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:platform_test",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -60,6 +62,7 @@ cuda_py_test(
         "noasan",  # times out, b/63678675
         "optonly",  # times out
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -76,6 +79,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:platform_test",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -98,6 +102,7 @@ cuda_py_test(
         "noasan",
         "optonly",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -120,6 +125,7 @@ cuda_py_test(
         "noasan",  # times out, b/63678675
         "optonly",  # times out
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -144,6 +150,7 @@ cuda_py_test(
         "noasan",  # times out, b/63678675
         "optonly",  # times out, b/79171797
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -166,6 +173,7 @@ cuda_py_test(
         "noasan",
         "optonly",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -187,6 +195,7 @@ cuda_py_test(
         "noasan",
         "optonly",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -209,6 +218,7 @@ cuda_py_test(
         "noasan",  # times out, b/63678675
         "optonly",  # times out
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -229,6 +239,7 @@ cuda_py_test(
         "noasan",
         "optonly",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -251,6 +262,7 @@ cuda_py_test(
         "noasan",
         "optonly",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -270,6 +282,7 @@ cuda_py_test(
         "noasan",
         "optonly",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -290,6 +303,7 @@ cuda_py_test(
         "noasan",  # times out
         "optonly",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -310,6 +324,7 @@ cuda_py_test(
         "noasan",
         "optonly",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -328,4 +343,5 @@ cuda_py_test(
     ],
     shard_count = 5,
     tags = ["optonly"],  # Test is flaky without optimization.
+    xla_enable_strict_auto_jit = True,
 )
diff --git a/tensorflow/python/kernel_tests/pool_test.py b/tensorflow/python/kernel_tests/pool_test.py
index 78e786f01c..367c94dd1e 100644
--- a/tensorflow/python/kernel_tests/pool_test.py
+++ b/tensorflow/python/kernel_tests/pool_test.py
@@ -303,6 +303,7 @@ class PoolingTest(test.TestCase):
     self.assertLess(err, err_tolerance)
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA")  # Much larger error
   def testGradient1D(self):
     with self.session(use_gpu=test.is_gpu_available()):
       for padding in ["SAME", "VALID"]:
diff --git a/tensorflow/python/kernel_tests/pooling_ops_test.py b/tensorflow/python/kernel_tests/pooling_ops_test.py
index c33b59bb99..27a7190434 100644
--- a/tensorflow/python/kernel_tests/pooling_ops_test.py
+++ b/tensorflow/python/kernel_tests/pooling_ops_test.py
@@ -730,6 +730,7 @@ class PoolingTest(test.TestCase):
         t = nn_ops.max_pool(
             t, ksize=ksize, strides=strides, padding="SAME").eval()
 
+  @test_util.disable_xla("This test never passed for XLA")
   def testDepthwiseMaxPoolInvalidConfigs(self):
     self._testDepthwiseMaxPoolInvalidConfig(
         [1, 2, 2, 4], [1, 2, 2, 2], [1, 1, 1, 2],
@@ -1174,6 +1175,7 @@ class PoolingTest(test.TestCase):
           use_gpu=use_gpu)
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA")
   def testMaxPoolGrad(self):
     for (data_format, use_gpu) in GetTestConfigs():
       self._testMaxPoolGradValidPadding1_1(data_format, use_gpu)
@@ -1210,6 +1212,7 @@ class PoolingTest(test.TestCase):
                      [1, window_rows, window_cols, 1],
                      [1, row_stride, col_stride, 1], padding)
 
+  @test_util.disable_xla("This test never passed for XLA")
   def _testMaxPoolGradDirect(self, input_data, output_backprop,
                              expected_input_backprop, input_sizes, output_sizes,
                              window_rows, window_cols, row_stride, col_stride,
@@ -1625,6 +1628,7 @@ class PoolingTest(test.TestCase):
           use_gpu=use_gpu)
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA")
   def testMaxPoolGradGrad(self):
     for (data_format, use_gpu) in GetTestConfigs():
       self._testMaxPoolGradGradValidPadding1_1(data_format, use_gpu)
@@ -1659,6 +1663,7 @@ class PoolingTest(test.TestCase):
         [1, row_stride, col_stride, 1], padding)
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA")
   def testAvgPoolGrad(self):
     for (data_format, use_gpu) in GetTestConfigs():
       self._testAvgPoolGradValidPadding1_1(data_format, use_gpu)
@@ -1818,6 +1823,7 @@ class PoolingTest(test.TestCase):
             padding="SAME")
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA")
   def testOpEdgeCases(self):
     with self.session(use_gpu=test.is_gpu_available()) as sess:
       pool_funcs = [nn_ops.max_pool, nn_ops.avg_pool]
@@ -1893,9 +1899,17 @@ if __name__ == "__main__":
        padding_) in GetShrunkInceptionMaxPoolShapes():
     setattr(PoolingTest, "testMaxPoolFwd_" + name_,
             GetMaxPoolFwdTest(input_size_, filter_size_, stride_, padding_))
-    setattr(PoolingTest, "testMaxPoolGrad_" + name_,
-            GetMaxPoolGradTest(input_size_, filter_size_, output_size_, stride_,
-                               padding_))
+    if name_ == "maxpool5":
+      setattr(
+          PoolingTest, "testMaxPoolGrad_" + name_,
+          test_util.disable_xla("maxpool5 fails while all others pass")(
+              GetMaxPoolGradTest(input_size_, filter_size_, output_size_,
+                                 stride_, padding_)))
+    else:
+      setattr(
+          PoolingTest, "testMaxPoolGrad_" + name_,
+          GetMaxPoolGradTest(input_size_, filter_size_, output_size_, stride_,
+                             padding_))
     setattr(PoolingTest, "testMaxPoolGradGrad_" + name_,
             GetMaxPoolGradGradTest(input_size_, filter_size_, output_size_,
                                    stride_, padding_))
diff --git a/tensorflow/python/kernel_tests/random/BUILD b/tensorflow/python/kernel_tests/random/BUILD
index dd81306db0..445aaf9dc4 100644
--- a/tensorflow/python/kernel_tests/random/BUILD
+++ b/tensorflow/python/kernel_tests/random/BUILD
@@ -45,6 +45,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:random_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -64,6 +65,7 @@ cuda_py_test(
         "//tensorflow/python:random_ops",
     ],
     shard_count = 3,
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -75,6 +77,7 @@ cuda_py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:random_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -88,6 +91,7 @@ cuda_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:random_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -103,6 +107,7 @@ cuda_py_test(
         "//tensorflow/python:random_ops",
         "//tensorflow/python:stateless_random_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -120,6 +125,7 @@ cuda_py_test(
     ],
     shard_count = 4,
     tags = ["nozapfhahn"],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -137,6 +143,7 @@ cuda_py_test(
         "//tensorflow/python:random_grad",
         "//tensorflow/python:random_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -152,4 +159,5 @@ cuda_py_test(
         "//tensorflow/python:platform",
         "//tensorflow/python:random_ops",
     ],
+    xla_enable_strict_auto_jit = True,
 )
diff --git a/tensorflow/python/kernel_tests/random/random_ops_test.py b/tensorflow/python/kernel_tests/random/random_ops_test.py
index 1384c3f446..68672a04bb 100644
--- a/tensorflow/python/kernel_tests/random/random_ops_test.py
+++ b/tensorflow/python/kernel_tests/random/random_ops_test.py
@@ -257,6 +257,7 @@ class TruncatedNormalTest(test.TestCase):
       self.assertAllEqual(rnd1, rnd2)
 
 
+@test_util.disable_all_xla("This never passed on XLA")
 class RandomUniformTest(RandomOpTestCommon):
 
   def _Sampler(self, num, minv, maxv, dtype, use_gpu, seed=None):
diff --git a/tensorflow/python/kernel_tests/relu_op_test.py b/tensorflow/python/kernel_tests/relu_op_test.py
index d4ba1ad77d..3b8924904c 100644
--- a/tensorflow/python/kernel_tests/relu_op_test.py
+++ b/tensorflow/python/kernel_tests/relu_op_test.py
@@ -86,6 +86,7 @@ class ReluTest(test.TestCase):
     self.assertAllClose(np_relu, tf_relu)
     self.assertShapeEqual(np_relu, tf_relu)
 
+  @test_util.disable_xla("This test never passed for XLA")
   def testReluInt8x4BadShape(self):
     if not test.is_gpu_available(cuda_only=True):
       self.skipTest("No GPU available")
diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
index 2594055119..ec93038f1b 100644
--- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py
+++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
@@ -741,6 +741,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
 
   @test_util.run_in_graph_and_eager_modes
   @test_util.run_v1_only("b/120545219")
+  @test_util.disable_xla("This test never passed for XLA")
   def testDestroyResource(self):
     v = resource_variable_ops.ResourceVariable(3.0, name="var0")
     self.evaluate(variables.global_variables_initializer())
diff --git a/tensorflow/python/kernel_tests/scan_ops_test.py b/tensorflow/python/kernel_tests/scan_ops_test.py
index 33e491fee1..4e15894fb4 100644
--- a/tensorflow/python/kernel_tests/scan_ops_test.py
+++ b/tensorflow/python/kernel_tests/scan_ops_test.py
@@ -70,6 +70,7 @@ def handle_options(func, x, axis, exclusive, reverse):
   return x
 
 
+@test_util.disable_all_xla("This test never passed for XLA")
 class CumsumTest(test.TestCase):
 
   valid_dtypes = [
@@ -193,6 +194,7 @@ class CumsumTest(test.TestCase):
           self._compareGradient([5, 10], axis, exclusive, reverse)
 
 
+@test_util.disable_all_xla("This test never passed for XLA")
 class CumprodTest(test.TestCase):
 
   valid_dtypes = [
diff --git a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
index e1cd9e7622..c443c23b16 100644
--- a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
+++ b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
@@ -295,6 +295,7 @@ class StatefulScatterNdTest(test.TestCase):
                                     updates).get_shape().as_list(), shape)
 
   @test_util.run_v1_only("b/120545219")
+  @test_util.disable_xla("This test never passed for XLA")
   def testResVarInvalidOutputShape(self):
     res = variables.Variable(
         initial_value=lambda: array_ops.zeros(shape=[], dtype=dtypes.float32),
diff --git a/tensorflow/python/kernel_tests/signal/BUILD b/tensorflow/python/kernel_tests/signal/BUILD
index 8f4e31abe3..554bf38029 100644
--- a/tensorflow/python/kernel_tests/signal/BUILD
+++ b/tensorflow/python/kernel_tests/signal/BUILD
@@ -29,6 +29,7 @@ cuda_py_tests(
         "//tensorflow/python:spectral_ops_test_util",
         "//tensorflow/python/ops/signal",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_tests(
@@ -45,6 +46,7 @@ cuda_py_tests(
     ],
     shard_count = 4,
     tags = ["optonly"],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_tests(
@@ -56,6 +58,7 @@ cuda_py_tests(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python/ops/signal",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_tests(
@@ -70,6 +73,7 @@ cuda_py_tests(
         "//tensorflow/python/ops/signal",
         "//tensorflow/python:spectral_ops_test_util",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_tests(
@@ -87,6 +91,7 @@ cuda_py_tests(
         "//tensorflow/python/ops/signal",
         "//tensorflow/python:platform_test",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_tests(
@@ -104,6 +109,7 @@ cuda_py_tests(
         "//tensorflow/python/ops/signal",
         "//tensorflow/python:platform_test",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_tests(
@@ -125,6 +131,7 @@ cuda_py_tests(
         "//tensorflow/python/ops/signal",
     ],
     tags = ["nomac"],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_tests(
@@ -140,4 +147,5 @@ cuda_py_tests(
         "//tensorflow/python/ops/signal",
         "//tensorflow/python:platform_test",
     ],
+    xla_enable_strict_auto_jit = True,
 )
diff --git a/tensorflow/python/kernel_tests/signal/fft_ops_test.py b/tensorflow/python/kernel_tests/signal/fft_ops_test.py
index 5b1053428c..d98a568491 100644
--- a/tensorflow/python/kernel_tests/signal/fft_ops_test.py
+++ b/tensorflow/python/kernel_tests/signal/fft_ops_test.py
@@ -465,6 +465,7 @@ class RFFTOpsTest(BaseFFTOpsTest):
                 gen_complex(complex_dims), rank, (size,) * rank)
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA")
   def testError(self):
     with spectral_ops_test_util.fft_kernel_label_map():
       for rank in VALID_FFT_RANKS:
diff --git a/tensorflow/python/kernel_tests/spacetodepth_op_test.py b/tensorflow/python/kernel_tests/spacetodepth_op_test.py
index e96bc09f36..7f3c381fa1 100644
--- a/tensorflow/python/kernel_tests/spacetodepth_op_test.py
+++ b/tensorflow/python/kernel_tests/spacetodepth_op_test.py
@@ -285,6 +285,7 @@ class SpaceToDepthTest(test.TestCase):
       actual_vals, expected_vals = self.evaluate([actual, expected])
       self.assertTrue(np.array_equal(actual_vals, expected_vals))
 
+  @test_util.disable_xla("This test never passed for XLA")
   def testAgainstTranspose(self):
     self.compareToTranspose(3, 2, 3, 1, 2, "NHWC", False)
     self.compareToTranspose(1, 2, 3, 2, 2, "NHWC", False)
diff --git a/tensorflow/python/kernel_tests/split_op_test.py b/tensorflow/python/kernel_tests/split_op_test.py
index 517db3450f..80004db833 100644
--- a/tensorflow/python/kernel_tests/split_op_test.py
+++ b/tensorflow/python/kernel_tests/split_op_test.py
@@ -373,6 +373,7 @@ class SplitOpTest(test.TestCase):
     assert s1.shape.as_list() == [1]
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA")
   def testNonexistentDimTensor(self):
     x = array_ops.placeholder(dtypes.int32)
     values = np.zeros([5, 30])
diff --git a/tensorflow/python/ops/bitwise_ops_test.py b/tensorflow/python/ops/bitwise_ops_test.py
index d154b6759b..c182874c7f 100644
--- a/tensorflow/python/ops/bitwise_ops_test.py
+++ b/tensorflow/python/ops/bitwise_ops_test.py
@@ -35,6 +35,7 @@ class BitwiseOpTest(test_util.TensorFlowTestCase):
     super(BitwiseOpTest, self).__init__(method_name)
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA")
   def testBinaryOps(self):
     dtype_list = [dtypes.int8, dtypes.int16, dtypes.int32, dtypes.int64,
                   dtypes.uint8, dtypes.uint16, dtypes.uint32, dtypes.uint64]
@@ -72,6 +73,7 @@ class BitwiseOpTest(test_util.TensorFlowTestCase):
         self.assertAllEqual(truth, popcnt_result)
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA")
   def testInvertOp(self):
     dtype_list = [dtypes.int8, dtypes.int16, dtypes.int32, dtypes.int64,
                   dtypes.uint8, dtypes.uint16, dtypes.uint32, dtypes.uint64]
@@ -97,6 +99,7 @@ class BitwiseOpTest(test_util.TensorFlowTestCase):
           self.assertAllEqual(inverted, expected)
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA")
   def testShiftsWithPositiveLHS(self):
     dtype_list = [np.int8, np.int16, np.int32, np.int64,
                   np.uint8, np.uint16, np.uint32, np.uint64]
diff --git a/tensorflow/python/ops/nn_batchnorm_test.py b/tensorflow/python/ops/nn_batchnorm_test.py
index e978f1d326..fedf8e44c3 100644
--- a/tensorflow/python/ops/nn_batchnorm_test.py
+++ b/tensorflow/python/ops/nn_batchnorm_test.py
@@ -206,6 +206,7 @@ class BatchNormalizationTest(test.TestCase):
                                   2)
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA")
   def testBatchNormGradImpl(self):
     x_shape = [7, 5, 4, 6]
     param_shape = [6]
diff --git a/tensorflow/python/ops/nn_fused_batchnorm_test.py b/tensorflow/python/ops/nn_fused_batchnorm_test.py
index 4bc33ff8bd..4e5cff5c16 100644
--- a/tensorflow/python/ops/nn_fused_batchnorm_test.py
+++ b/tensorflow/python/ops/nn_fused_batchnorm_test.py
@@ -523,6 +523,7 @@ class BatchNormalizationTest(test.TestCase):
             data_format='NHWC',
             is_training=is_training)
 
+  @test_util.disable_xla('This test never passed for XLA')
   def testBatchNormGradShape5(self):
     for is_training in [True, False]:
       x_shape = [0, 7, 11, 4]
diff --git a/tensorflow/python/ops/nn_test.py b/tensorflow/python/ops/nn_test.py
index 82fab74183..74561349ed 100644
--- a/tensorflow/python/ops/nn_test.py
+++ b/tensorflow/python/ops/nn_test.py
@@ -41,6 +41,7 @@ from tensorflow.python.ops.nn_impl import _compute_sampled_logits
 from tensorflow.python.platform import test as test_lib
 
 
+@test_util.disable_all_xla("This test never passed for XLA")
 class ZeroFractionTest(test_lib.TestCase):
 
   def _ZeroFraction(self, x):
@@ -1017,6 +1018,7 @@ class LeakyReluTest(test_lib.TestCase):
 class SwishTest(test_lib.TestCase):
 
   @test_util.run_deprecated_v1
+  @test_util.disable_xla("This test never passed for XLA")
   def testValues(self):
     np_values = np.array(
         [np.linspace(-10.0, 0.0, 100),
diff --git a/tensorflow/python/ops/parallel_for/BUILD b/tensorflow/python/ops/parallel_for/BUILD
index 1028963b1a..f54ca558f4 100644
--- a/tensorflow/python/ops/parallel_for/BUILD
+++ b/tensorflow/python/ops/parallel_for/BUILD
@@ -114,6 +114,7 @@ cuda_py_test(
         "//tensorflow/python:random_ops",
         "//tensorflow/python:util",
     ],
+    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -171,4 +172,5 @@ cuda_py_test(
         "//tensorflow/python/ops/losses",
     ],
     tags = ["optonly"],  # Too slow in non-opt mode
+    xla_enable_strict_auto_jit = True,
 )
diff --git a/tensorflow/python/ops/parallel_for/control_flow_ops_test.py b/tensorflow/python/ops/parallel_for/control_flow_ops_test.py
index 1a8f639d43..8a5830e28f 100644
--- a/tensorflow/python/ops/parallel_for/control_flow_ops_test.py
+++ b/tensorflow/python/ops/parallel_for/control_flow_ops_test.py
@@ -251,6 +251,7 @@ class NNTest(PForTestCase):
 
     self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 3)
 
+  @test_util.disable_xla("This test never passed for XLA")
   def test_fused_batch_norm(self):
     data_formats = ["NHWC"]
     if test.is_gpu_available():
diff --git a/tensorflow/python/ops/parallel_for/gradients_test.py b/tensorflow/python/ops/parallel_for/gradients_test.py
index 133e790992..66110f12d9 100644
--- a/tensorflow/python/ops/parallel_for/gradients_test.py
+++ b/tensorflow/python/ops/parallel_for/gradients_test.py
@@ -482,6 +482,7 @@ class GradientsTest(test.TestCase):
     pfor_jacobian, while_jacobian = create_lstm_batch_jacobian(8, 4, 2)
     self.run_and_assert_equal(pfor_jacobian, while_jacobian)
 
+  @test_util.disable_xla("This test never passed for XLA")
   def test_dynamic_lstm_batch_jacobian(self):
     pfor_jacobian, while_gradients = create_dynamic_lstm_batch_jacobian(8, 4, 3)
     with session.Session() as sess:
diff --git a/tensorflow/python/ops/special_math_ops_test.py b/tensorflow/python/ops/special_math_ops_test.py
index 94aaebed95..0224e7ebb8 100644
--- a/tensorflow/python/ops/special_math_ops_test.py
+++ b/tensorflow/python/ops/special_math_ops_test.py
@@ -119,6 +119,7 @@ class LBetaTest(test.TestCase):
           special_math_ops.lbeta(x).get_shape())
 
   @test_util.run_in_graph_and_eager_modes
+  @test_util.disable_xla('This test never passed for XLA')
   def test_length_1_last_dimension_results_in_one(self):
     # If there is only one coefficient, the formula still works, and we get one
     # as the answer, always.
diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl
index b7da5bf913..37d52cdb39 100644
--- a/tensorflow/tensorflow.bzl
+++ b/tensorflow/tensorflow.bzl
@@ -1802,8 +1802,17 @@ def tf_py_test(
         additional_visibility = [],
         kernels = [],
         flaky = 0,
+        xla_enable_strict_auto_jit = False,
         xla_enabled = False,
         grpc_enabled = False):
+    """Create one or more python tests with extra tensorflow dependencies."""
+    xla_test_true_list = []
+
+    # xla_enable_strict_auto_jit is used to run Tensorflow unit tests with all XLA compilable
+    # kernels compiled with XLA.
+    if xla_enable_strict_auto_jit:
+        xla_enabled = True
+        xla_test_true_list += ["//tensorflow/python:is_xla_test_true"]
     if xla_enabled:
         additional_deps = additional_deps + tf_additional_xla_deps_py()
     if grpc_enabled:
@@ -1824,7 +1833,7 @@ def tf_py_test(
         deps = [
             clean_dep("//tensorflow/python:extra_py_tests_deps"),
             clean_dep("//tensorflow/python:gradient_checker"),
-        ] + additional_deps,
+        ] + additional_deps + xla_test_true_list,
     )
 
 register_extension_info(
@@ -1844,8 +1853,12 @@ def cuda_py_test(
         kernels = [],
         tags = [],
         flaky = 0,
+        xla_enable_strict_auto_jit = False,
         xla_enabled = False,
         grpc_enabled = False):
+    # TODO(b/122522101): Don't ignore xla_enable_strict_auto_jit and enable additional
+    # XLA tests once enough compute resources are available.
+    _ignored = [xla_enable_strict_auto_jit]
     if main == None:
         main = name + ".py"
     for config in ["cpu", "gpu"]:
@@ -1868,6 +1881,7 @@ def cuda_py_test(
             shard_count = shard_count,
             tags = test_tags,
             xla_enabled = xla_enabled,
+            xla_enable_strict_auto_jit = False,
         )
 
 register_extension_info(
@@ -1921,6 +1935,7 @@ def py_tests(
         tags = [],
         shard_count = 1,
         prefix = "",
+        xla_enable_strict_auto_jit = False,
         xla_enabled = False,
         grpc_enabled = False):
     for src in srcs:
@@ -1939,6 +1954,7 @@ def py_tests(
             shard_count = shard_count,
             tags = tags,
             xla_enabled = xla_enabled,
+            xla_enable_strict_auto_jit = xla_enable_strict_auto_jit,
         )
 
 def cuda_py_tests(
@@ -1951,8 +1967,12 @@ def cuda_py_tests(
         shard_count = 1,
         tags = [],
         prefix = "",
+        xla_enable_strict_auto_jit = False,
         xla_enabled = False,
         grpc_enabled = False):
+    # TODO(b/122522101): Don't ignore xla_enable_strict_auto_jit and enable additional
+    # XLA tests once enough compute resources are available.
+    _ignored = [xla_enable_strict_auto_jit]
     test_tags = tags + tf_cuda_tests_tags()
     py_tests(
         name = name,
@@ -1966,6 +1986,7 @@ def cuda_py_tests(
         shard_count = shard_count,
         tags = test_tags,
         xla_enabled = xla_enabled,
+        xla_enable_strict_auto_jit = False,
     )
 
 # Creates a genrule named <name> for running tools/proto_text's generator to
-- 
GitLab


From 267263629fd8d612719f61991eb2d2e3cd3b2b01 Mon Sep 17 00:00:00 2001
From: Tamara Norman <tamaranorman@google.com>
Date: Mon, 14 Jan 2019 02:42:51 -0800
Subject: [PATCH 0663/1095] Change so that tf.random_poisson is renamed to the
 v2 API instead of to compat

PiperOrigin-RevId: 229150058
---
 tensorflow/tools/compatibility/renames_v2.py         | 1 -
 tensorflow/tools/compatibility/tf_upgrade_v2.py      | 2 ++
 tensorflow/tools/compatibility/tf_upgrade_v2_test.py | 9 +++++++++
 3 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index b757699f63..f143a260b9 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -478,7 +478,6 @@ renames = {
     'tf.random_crop': 'tf.image.random_crop',
     'tf.random_gamma': 'tf.random.gamma',
     'tf.random_normal': 'tf.random.normal',
-    'tf.random_poisson': 'tf.compat.v1.random_poisson',
     'tf.random_shuffle': 'tf.random.shuffle',
     'tf.random_uniform': 'tf.random.uniform',
     'tf.read_file': 'tf.io.read_file',
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 2c56cbf36b..994d7f9b7e 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -601,6 +601,8 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "tf.compat.v1.initializers.truncated_normal",
         "tf.image.resize_images":
             "tf.image.resize",
+        "tf.random_poisson":
+            "tf.random.poisson",
     }
     # pylint: enable=line-too-long
 
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index 0dde5045aa..caf63416d8 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -539,6 +539,15 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
         )
     self.assertEqual(new_text, expected_text)
 
+  def testRandomPoissonConversion(self):
+    text1 = "tf.random_poisson(lam, shape, dtype)"
+    text2 = "tf.random.poisson(lam, shape, dtype)"
+    expected_text = "tf.random.poisson(lam=lam, shape=shape, dtype=dtype)"
+    _, unused_report, unused_errors, new_text1 = self._upgrade(text1)
+    self.assertEqual(new_text1, expected_text)
+    _, unused_report, unused_errors, new_text2 = self._upgrade(text2)
+    self.assertEqual(new_text2, expected_text)
+
   def testConvolutionOpUpdate(self):
     text = (
         "tf.nn.convolution(input, filter, padding, strides, dilation_rate, "
-- 
GitLab


From 74a6cca5d867d37e79ec9d780f2c57b926f07a80 Mon Sep 17 00:00:00 2001
From: Sergei Lebedev <slebedev@google.com>
Date: Mon, 14 Jan 2019 03:04:27 -0800
Subject: [PATCH 0664/1095] Removed a linear scan in dtypes.as_dtype

PiperOrigin-RevId: 229152423
---
 tensorflow/python/framework/dtypes.py | 98 +++++++++++++--------------
 1 file changed, 47 insertions(+), 51 deletions(-)

diff --git a/tensorflow/python/framework/dtypes.py b/tensorflow/python/framework/dtypes.py
index 9a4fe4e93b..9386cca379 100644
--- a/tensorflow/python/framework/dtypes.py
+++ b/tensorflow/python/framework/dtypes.py
@@ -535,29 +535,31 @@ _np_qint32 = np.dtype([("qint32", np.int32, 1)])
 np_resource = np.dtype([("resource", np.ubyte, 1)])
 
 # Standard mappings between types_pb2.DataType values and numpy.dtypes.
-_NP_TO_TF = frozenset([
-    (np.float16, float16),
-    (np.float32, float32),
-    (np.float64, float64),
-    (np.int32, int32),
-    (np.int64, int64),
-    (np.uint8, uint8),
-    (np.uint16, uint16),
-    (np.uint32, uint32),
-    (np.uint64, uint64),
-    (np.int16, int16),
-    (np.int8, int8),
-    (np.complex64, complex64),
-    (np.complex128, complex128),
-    (np.object_, string),
-    (np.bool_, bool),
-    (_np_qint8, qint8),
-    (_np_quint8, quint8),
-    (_np_qint16, qint16),
-    (_np_quint16, quint16),
-    (_np_qint32, qint32),
-    (_np_bfloat16, bfloat16),
-])
+_NP_TO_TF = {
+    np.float16: float16,
+    np.float32: float32,
+    np.float64: float64,
+    np.int32: int32,
+    np.int64: int64,
+    np.uint8: uint8,
+    np.uint16: uint16,
+    np.uint32: uint32,
+    np.uint64: uint64,
+    np.int16: int16,
+    np.int8: int8,
+    np.complex64: complex64,
+    np.complex128: complex128,
+    np.object_: string,
+    np.string_: string,
+    np.unicode_: string,
+    np.bool_: bool,
+    _np_qint8: qint8,
+    _np_quint8: quint8,
+    _np_qint16: qint16,
+    _np_quint16: quint16,
+    _np_qint32: qint32,
+    _np_bfloat16: bfloat16,
+}
 _TF_TO_NP = {
     types_pb2.DT_HALF:
         np.float16,
@@ -664,6 +666,20 @@ _PYTHON_TO_TF = {
     builtins.object: string
 }
 
+_ANY_TO_TF = {}
+_ANY_TO_TF.update(_INTERN_TABLE)
+_ANY_TO_TF.update(_STRING_TO_TF)
+_ANY_TO_TF.update(_PYTHON_TO_TF)
+_ANY_TO_TF.update(_NP_TO_TF)
+
+# Ensure no collisions.
+assert len(_ANY_TO_TF) == sum(len(d) for d in [
+    _INTERN_TABLE,
+    _STRING_TO_TF,
+    _PYTHON_TO_TF,
+    _NP_TO_TF
+])
+
 
 @tf_export("dtypes.as_dtype", "as_dtype")
 def as_dtype(type_value):
@@ -684,36 +700,16 @@ def as_dtype(type_value):
   if isinstance(type_value, DType):
     return type_value
 
-  try:
-    return _INTERN_TABLE[type_value]
-  except KeyError:
-    pass
-
-  try:
-    return _STRING_TO_TF[type_value]
-  except KeyError:
-    pass
+  if isinstance(type_value, np.dtype):
+    try:
+      return _NP_TO_TF[type_value.type]
+    except KeyError:
+      pass
 
   try:
-    return _PYTHON_TO_TF[type_value]
+    return _ANY_TO_TF[type_value]
   except KeyError:
     pass
 
-  if isinstance(type_value, np.dtype):
-    # The numpy dtype for strings is variable length. We can not compare
-    # dtype with a single constant (np.string does not exist) to decide
-    # dtype is a "string" type. We need to compare the dtype.type to be
-    # sure it's a string type.
-    if type_value.type == np.string_ or type_value.type == np.unicode_:
-      return string
-
-  if isinstance(type_value, (type, np.dtype)):
-    for key, val in _NP_TO_TF:
-      try:
-        if key == type_value:
-          return val
-      except TypeError as e:
-        raise TypeError("Cannot convert {} to a dtype. {}".format(
-            type_value, e))
-
-  raise TypeError("Cannot convert value %r to a TensorFlow DType." % type_value)
+  raise TypeError(
+      "Cannot convert value %r to a TensorFlow DType." % type_value)
-- 
GitLab


From 8f593c48c84a2d52d2ba8becf2eaef20250325a0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 14 Jan 2019 03:07:27 -0800
Subject: [PATCH 0665/1095] Correctly read tensor values.

PiperOrigin-RevId: 229152770
---
 tensorflow/python/framework/tensor_util.py    | 95 ++++++-------------
 .../python/framework/tensor_util_test.py      | 30 +++---
 2 files changed, 43 insertions(+), 82 deletions(-)

diff --git a/tensorflow/python/framework/tensor_util.py b/tensorflow/python/framework/tensor_util.py
index 51f71616a1..ca8b067935 100644
--- a/tensorflow/python/framework/tensor_util.py
+++ b/tensorflow/python/framework/tensor_util.py
@@ -598,88 +598,53 @@ def MakeNdarray(tensor):
   dtype = tensor_dtype.as_numpy_dtype
 
   if tensor.tensor_content:
-    return (np.frombuffer(tensor.tensor_content, dtype=dtype).copy()
-            .reshape(shape))
-  elif tensor_dtype == dtypes.float16 or tensor_dtype == dtypes.bfloat16:
+    return (np.frombuffer(tensor.tensor_content,
+                          dtype=dtype).copy().reshape(shape))
+
+  if tensor_dtype == dtypes.string:
+    # np.pad throws on these arrays of type np.object.
+    values = list(tensor.string_val)
+    padding = num_elements - len(values)
+    if padding > 0:
+      last = values[-1] if values else ""
+      values.extend([last] * padding)
+    return np.array(values, dtype=dtype).reshape(shape)
+
+  if tensor_dtype == dtypes.float16 or tensor_dtype == dtypes.bfloat16:
     # the half_val field of the TensorProto stores the binary representation
     # of the fp16: we need to reinterpret this as a proper float16
-    if len(tensor.half_val) == 1:
-      tmp = np.array(tensor.half_val[0], dtype=np.uint16)
-      tmp.dtype = tensor_dtype.as_numpy_dtype
-      return np.repeat(tmp, num_elements).reshape(shape)
-    else:
-      tmp = np.fromiter(tensor.half_val, dtype=np.uint16)
-      tmp.dtype = tensor_dtype.as_numpy_dtype
-      return tmp.reshape(shape)
+    values = np.fromiter(tensor.half_val, dtype=np.uint16)
+    values.dtype = tensor_dtype.as_numpy_dtype
   elif tensor_dtype == dtypes.float32:
-    if len(tensor.float_val) == 1:
-      return np.repeat(
-          np.array(tensor.float_val[0], dtype=dtype),
-          num_elements).reshape(shape)
-    else:
-      return np.fromiter(tensor.float_val, dtype=dtype).reshape(shape)
+    values = np.fromiter(tensor.float_val, dtype=dtype)
   elif tensor_dtype == dtypes.float64:
-    if len(tensor.double_val) == 1:
-      return np.repeat(
-          np.array(tensor.double_val[0], dtype=dtype),
-          num_elements).reshape(shape)
-    else:
-      return np.fromiter(tensor.double_val, dtype=dtype).reshape(shape)
+    values = np.fromiter(tensor.double_val, dtype=dtype)
   elif tensor_dtype in [
       dtypes.int32, dtypes.uint8, dtypes.uint16, dtypes.int16, dtypes.int8,
       dtypes.qint32, dtypes.quint8, dtypes.qint8, dtypes.qint16, dtypes.quint16
   ]:
-    if len(tensor.int_val) == 1:
-      return np.repeat(np.array(tensor.int_val[0], dtype=dtype),
-                       num_elements).reshape(shape)
-    else:
-      return np.fromiter(tensor.int_val, dtype=dtype).reshape(shape)
+    values = np.fromiter(tensor.int_val, dtype=dtype)
   elif tensor_dtype == dtypes.int64:
-    if len(tensor.int64_val) == 1:
-      return np.repeat(
-          np.array(tensor.int64_val[0], dtype=dtype),
-          num_elements).reshape(shape)
-    else:
-      return np.fromiter(tensor.int64_val, dtype=dtype).reshape(shape)
-  elif tensor_dtype == dtypes.string:
-    if len(tensor.string_val) == 1:
-      return np.repeat(
-          np.array(tensor.string_val[0], dtype=dtype),
-          num_elements).reshape(shape)
-    else:
-      return np.array(
-          [x for x in tensor.string_val], dtype=dtype).reshape(shape)
+    values = np.fromiter(tensor.int64_val, dtype=dtype)
   elif tensor_dtype == dtypes.complex64:
     it = iter(tensor.scomplex_val)
-    if len(tensor.scomplex_val) == 2:
-      return np.repeat(
-          np.array(
-              complex(tensor.scomplex_val[0], tensor.scomplex_val[1]),
-              dtype=dtype), num_elements).reshape(shape)
-    else:
-      return np.array(
-          [complex(x[0], x[1]) for x in zip(it, it)],
-          dtype=dtype).reshape(shape)
+    values = np.array([complex(x[0], x[1]) for x in zip(it, it)], dtype=dtype)
   elif tensor_dtype == dtypes.complex128:
     it = iter(tensor.dcomplex_val)
-    if len(tensor.dcomplex_val) == 2:
-      return np.repeat(
-          np.array(
-              complex(tensor.dcomplex_val[0], tensor.dcomplex_val[1]),
-              dtype=dtype), num_elements).reshape(shape)
-    else:
-      return np.array(
-          [complex(x[0], x[1]) for x in zip(it, it)],
-          dtype=dtype).reshape(shape)
+    values = np.array([complex(x[0], x[1]) for x in zip(it, it)], dtype=dtype)
   elif tensor_dtype == dtypes.bool:
-    if len(tensor.bool_val) == 1:
-      return np.repeat(np.array(tensor.bool_val[0], dtype=dtype),
-                       num_elements).reshape(shape)
-    else:
-      return np.fromiter(tensor.bool_val, dtype=dtype).reshape(shape)
+    values = np.fromiter(tensor.bool_val, dtype=dtype)
   else:
     raise TypeError("Unsupported tensor type: %s" % tensor.dtype)
 
+  if values.size == 0:
+    return np.zeros(shape, dtype)
+
+  if values.size != num_elements:
+    values = np.pad(values, (0, num_elements - values.size), "edge")
+
+  return values.reshape(shape)
+
 
 def ShapeEquals(tensor_proto, shape):
   """Returns True if "tensor_proto" has the given "shape".
diff --git a/tensorflow/python/framework/tensor_util_test.py b/tensorflow/python/framework/tensor_util_test.py
index 0033754618..cdacdfaaad 100644
--- a/tensorflow/python/framework/tensor_util_test.py
+++ b/tensorflow/python/framework/tensor_util_test.py
@@ -336,23 +336,16 @@ class TensorUtilTest(test.TestCase):
       self.assertAllClose(np.array([10, 20, 30], dtype=nptype), a)
 
   def testIntTypesWithImplicitRepeat(self):
-    for dtype, nptype in [(dtypes.int64, np.int64),
-                          (dtypes.int32, np.int32),
-                          (dtypes.uint8, np.uint8),
-                          (dtypes.uint16, np.uint16),
-                          (dtypes.int16, np.int16),
-                          (dtypes.int8, np.int8)]:
+    for dtype, nptype in [(dtypes.int64, np.int64), (dtypes.int32, np.int32),
+                          (dtypes.uint8, np.uint8), (dtypes.uint16, np.uint16),
+                          (dtypes.int16, np.int16), (dtypes.int8, np.int8)]:
       self.assertAllEqual(
-          np.array(
-              [[10, 10, 10, 10],
-               [10, 10, 10, 10],
-               [10, 10, 10, 10]],
-              dtype=nptype),
+          np.array([[10, 11, 12, 12], [12, 12, 12, 12], [12, 12, 12, 12]],
+                   dtype=nptype),
           tensor_util.MakeNdarray(
-              tensor_util.make_tensor_proto(
-                  [10],
-                  shape=[3, 4],
-                  dtype=dtype)))
+              tensor_util.make_tensor_proto([10, 11, 12],
+                                            shape=[3, 4],
+                                            dtype=dtype)))
 
   def testIntMixedWithDimension(self):
     # Github issue: 11974
@@ -500,9 +493,12 @@ class TensorUtilTest(test.TestCase):
     self.assertEquals([b"foo"], a)
 
   def testStringWithImplicitRepeat(self):
-    t = tensor_util.make_tensor_proto("f", shape=[3, 4])
+    t = tensor_util.make_tensor_proto(["f", "g"], shape=[3, 4])
     a = tensor_util.MakeNdarray(t)
-    self.assertAllEqual(np.array([[b"f"] * 4] * 3, dtype=np.object), a)
+    self.assertAllEqual(
+        np.array([[b"f", b"g", b"g", b"g"], [b"g", b"g", b"g", b"g"],
+                  [b"g", b"g", b"g", b"g"]],
+                 dtype=np.object), a)
 
   def testStringN(self):
     t = tensor_util.make_tensor_proto([b"foo", b"bar", b"baz"], shape=[1, 3])
-- 
GitLab


From ac2f7471b9f6ba0316d865b65c11f86ee939e68a Mon Sep 17 00:00:00 2001
From: Tamara Norman <tamaranorman@google.com>
Date: Mon, 14 Jan 2019 03:22:00 -0800
Subject: [PATCH 0666/1095] Fix assert_* conversions, ensure emit warnings and
 change to compat.v1

PiperOrigin-RevId: 229154096
---
 .../tools/compatibility/tf_upgrade_v2.py      | 76 +++++++++++++++++++
 .../tools/compatibility/tf_upgrade_v2_test.py | 39 +++++++++-
 2 files changed, 112 insertions(+), 3 deletions(-)

diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 994d7f9b7e..fd71103cc6 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -603,6 +603,48 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "tf.image.resize",
         "tf.random_poisson":
             "tf.random.poisson",
+        "tf.debugging.assert_greater":
+            "tf.compat.v1.debugging.assert_greater",
+        "tf.debugging.assert_greater_equal":
+            "tf.compat.v1.debugging.assert_greater_equal",
+        "tf.debugging.assert_integer":
+            "tf.compat.v1.debugging.assert_integer",
+        "tf.debugging.assert_less":
+            "tf.compat.v1.debugging.assert_less",
+        "tf.debugging.assert_less_equal":
+            "tf.compat.v1.debugging.assert_less_equal",
+        "tf.debugging.assert_near":
+            "tf.compat.v1.debugging.assert_near",
+        "tf.debugging.assert_negative":
+            "tf.compat.v1.debugging.assert_negative",
+        "tf.debugging.assert_non_negative":
+            "tf.compat.v1.debugging.assert_non_negative",
+        "tf.debugging.assert_non_positive":
+            "tf.compat.v1.debugging.assert_non_positive",
+        "tf.debugging.assert_none_equal":
+            "tf.compat.v1.debugging.assert_none_equal",
+        "tf.debugging.assert_type":
+            "tf.compat.v1.debugging.assert_type",
+        "tf.debugging.assert_positive":
+            "tf.compat.v1.debugging.assert_positive",
+        "tf.debugging.assert_equal":
+            "tf.compat.v1.debugging.assert_equal",
+        "tf.debugging.assert_scalar":
+            "tf.compat.v1.debugging.assert_scalar",
+        "tf.assert_equal":
+            "tf.compat.v1.assert_equal",
+        "tf.assert_less":
+            "tf.compat.v1.assert_less",
+        "tf.assert_greater":
+            "tf.compat.v1.assert_greater",
+        "tf.debugging.assert_rank":
+            "tf.compat.v1.debugging.assert_rank",
+        "tf.debugging.assert_rank_at_least":
+            "tf.compat.v1.debugging.assert_rank_at_least",
+        "tf.debugging.assert_rank_in":
+            "tf.compat.v1.debugging.assert_rank_in",
+        "tf.assert_rank":
+            "tf.compat.v1.assert_rank",
     }
     # pylint: enable=line-too-long
 
@@ -851,10 +893,40 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             assert_return_type_comment,
         "tf.assert_equal":
             assert_return_type_comment,
+        "tf.assert_none_equal":
+            assert_return_type_comment,
+        "tf.assert_less":
+            assert_return_type_comment,
+        "tf.assert_negative":
+            assert_return_type_comment,
+        "tf.assert_positive":
+            assert_return_type_comment,
+        "tf.assert_non_negative":
+            assert_return_type_comment,
+        "tf.assert_non_positive":
+            assert_return_type_comment,
+        "tf.assert_near":
+            assert_return_type_comment,
         "tf.assert_less":
             assert_return_type_comment,
+        "tf.assert_less_equal":
+            assert_return_type_comment,
+        "tf.assert_greater":
+            assert_return_type_comment,
+        "tf.assert_greater_equal":
+            assert_return_type_comment,
+        "tf.assert_integer":
+            assert_return_type_comment,
+        "tf.assert_type":
+            assert_return_type_comment,
+        "tf.assert_scalar":
+            assert_return_type_comment,
         "tf.assert_rank":
             assert_rank_comment,
+        "tf.assert_rank_at_least":
+            assert_rank_comment,
+        "tf.assert_rank_in":
+            assert_rank_comment,
         "tf.debugging.assert_equal":
             assert_return_type_comment,
         "tf.debugging.assert_greater":
@@ -879,6 +951,10 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             assert_return_type_comment,
         "tf.debugging.assert_positive":
             assert_return_type_comment,
+        "tf.debugging.assert_type":
+            assert_return_type_comment,
+        "tf.debugging.assert_scalar":
+            assert_return_type_comment,
         "tf.debugging.assert_rank":
             assert_rank_comment,
         "tf.debugging.assert_rank_at_least":
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index caf63416d8..0adee6a566 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -114,12 +114,12 @@ class TestUpgrade(test_util.TensorFlowTestCase):
     self.assertTrue(report.find("Failed to parse") != -1)
 
   def testReport(self):
-    text = "tf.assert_near(a)\n"
+    text = "tf.angle(a)\n"
     _, report, unused_errors, unused_new_text = self._upgrade(text)
     # This is not a complete test, but it is a sanity test that a report
     # is generating information.
-    self.assertTrue(report.find("Renamed function `tf.assert_near` to "
-                                "`tf.debugging.assert_near`"))
+    self.assertTrue(report.find("Renamed function `tf.angle` to "
+                                "`tf.math.angle`"))
 
   def testRename(self):
     text = "tf.conj(a)\n"
@@ -937,6 +937,39 @@ def _log_prob(self, x):
     _, unused_report, unused_errors, new_text = self._upgrade(text)
     self.assertEqual(expected_text, new_text)
 
+  def testAssertStatements(self):
+    for name in ["assert_greater", "assert_equal", "assert_none_equal",
+                 "assert_less", "assert_negative", "assert_positive",
+                 "assert_non_negative", "assert_non_positive", "assert_near",
+                 "assert_less", "assert_less_equal", "assert_greater",
+                 "assert_greater_equal", "assert_integer", "assert_type",
+                 "assert_scalar"]:
+      text = "tf.%s(a)" % name
+      expected_text = "tf.compat.v1.%s(a)" % name
+      _, unused_report, errors, new_text = self._upgrade(text)
+      self.assertEqual(expected_text, new_text)
+      self.assertIn("assert_* functions", errors[0])
+
+      text = "tf.debugging.%s(a)" % name
+      expected_text = "tf.compat.v1.debugging.%s(a)" % name
+      _, unused_report, errors, new_text = self._upgrade(text)
+      self.assertEqual(expected_text, new_text)
+      self.assertIn("assert_* functions", errors[0])
+
+  def testAssertRankStatements(self):
+    for name in ["assert_rank", "assert_rank_at_least", "assert_rank_in"]:
+      text = "tf.%s(a)" % name
+      expected_text = "tf.compat.v1.%s(a)" % name
+      _, unused_report, errors, new_text = self._upgrade(text)
+      self.assertEqual(expected_text, new_text)
+      self.assertIn("assert_rank_* functions", errors[0])
+
+      text = "tf.debugging.%s(a)" % name
+      expected_text = "tf.compat.v1.debugging.%s(a)" % name
+      _, unused_report, errors, new_text = self._upgrade(text)
+      self.assertEqual(expected_text, new_text)
+      self.assertIn("assert_rank_* functions", errors[0])
+
 
 class TestUpgradeFiles(test_util.TensorFlowTestCase):
 
-- 
GitLab


From 3ed46c325f70e2f1521b850b43d2b174101d9472 Mon Sep 17 00:00:00 2001
From: Tamara Norman <tamaranorman@google.com>
Date: Mon, 14 Jan 2019 04:06:24 -0800
Subject: [PATCH 0667/1095] Add conversion for count_nonzero outside of package

PiperOrigin-RevId: 229158255
---
 tensorflow/python/ops/math_ops.py                  |  2 ++
 tensorflow/tools/compatibility/renames_v2.py       |  1 -
 tensorflow/tools/compatibility/tf_upgrade_v2.py    |  7 +++++++
 .../tools/compatibility/tf_upgrade_v2_test.py      | 14 +++++++++++++-
 4 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py
index 248d092538..2ae29d0eb9 100644
--- a/tensorflow/python/ops/math_ops.py
+++ b/tensorflow/python/ops/math_ops.py
@@ -1339,6 +1339,8 @@ def reduce_sum(input_tensor, axis=None, keepdims=False, name=None):
 @tf_export(v1=["math.count_nonzero", "count_nonzero"])
 @deprecation.deprecated_args(
     None, "keep_dims is deprecated, use keepdims instead", "keep_dims")
+@deprecation.deprecated_args(
+    None, "reduction_indices is deprecated, use axis instead", "axis")
 def count_nonzero(input_tensor,
                   axis=None,
                   keepdims=None,
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index f143a260b9..fb2303e9dd 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -121,7 +121,6 @@ renames = {
     'tf.container': 'tf.compat.v1.container',
     'tf.convert_to_tensor_or_indexed_slices': 'tf.compat.v1.convert_to_tensor_or_indexed_slices',
     'tf.convert_to_tensor_or_sparse_tensor': 'tf.compat.v1.convert_to_tensor_or_sparse_tensor',
-    'tf.count_nonzero': 'tf.compat.v1.count_nonzero',
     'tf.count_up_to': 'tf.compat.v1.count_up_to',
     'tf.create_partitioned_variables': 'tf.compat.v1.create_partitioned_variables',
     'tf.cross': 'tf.linalg.cross',
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index fd71103cc6..3f9032bb33 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -119,6 +119,11 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
         "tf.load_file_system_library": {
             "library_filename": "library_location",
         },
+        "tf.count_nonzero": {
+            "input_tensor": "input",
+            "keep_dims": "keepdims",
+            "reduction_indices": "axis",
+        },
         "tf.math.count_nonzero": {
             "input_tensor": "input",
             "keep_dims": "keepdims",
@@ -511,6 +516,8 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             "tf.sort",
         "tf.contrib.framework.argsort":
             "tf.argsort",
+        "tf.count_nonzero":
+            "tf.math.count_nonzero",
         "tf.manip.batch_to_space_nd":
             "tf.batch_to_space",
         "tf.quantize_v2":
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index 0adee6a566..23446e30b1 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -506,7 +506,7 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
         "tf.nn.dropout(x, 1 - (1 - func(3 + 4.)), name=\"foo\")\n",
     )
 
-  def testCountNonZeroChanges(self):
+  def testMathCountNonZeroChanges(self):
     text = (
         "tf.math.count_nonzero(input_tensor=input, dtype=dtype, name=name, "
         "reduction_indices=axis, keep_dims=keepdims)\n"
@@ -518,6 +518,18 @@ bazel-bin/tensorflow/tools/compatibility/update/generate_v2_reorders_map
         )
     self.assertEqual(new_text, expected_text)
 
+  def testCountNonZeroChanges(self):
+    text = (
+        "tf.count_nonzero(input_tensor=input, dtype=dtype, name=name, "
+        "reduction_indices=axis, keep_dims=keepdims)\n"
+        )
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    expected_text = (
+        "tf.math.count_nonzero(input=input, dtype=dtype, name=name, "
+        "axis=axis, keepdims=keepdims)\n"
+        )
+    self.assertEqual(new_text, expected_text)
+
   def testRandomMultinomialToRandomCategorical(self):
     text = (
         "tf.random.multinomial(logits, samples, seed, name, output_dtype)\n"
-- 
GitLab


From ad683f866b465fa753731be283515bb0a67ad078 Mon Sep 17 00:00:00 2001
From: Vojtech Bardiovsky <vbardiovsky@google.com>
Date: Mon, 14 Jan 2019 06:51:47 -0800
Subject: [PATCH 0668/1095] Serialize FunctionSpec as proto instead of a tuple.

PiperOrigin-RevId: 229174621
---
 tensorflow/python/eager/function.py           | 51 ++++++++++++-------
 .../saved_model/function_deserialization.py   | 16 ++++--
 .../saved_model/function_serialization.py     | 19 ++++++-
 .../saved_model/saved_object_graph.proto      | 21 ++++++--
 4 files changed, 82 insertions(+), 25 deletions(-)

diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index bbf8630a2e..6429022df3 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -866,14 +866,6 @@ def _deterministic_dict_values(dictionary):
 class FunctionSpec(object):
   """Specification of how to bind arguments to a function."""
 
-  def as_tuple(self):
-    return (self._fullargspec, self._is_method, self._args_to_prepend,
-            self._kwargs_to_include, self.input_signature)
-
-  @staticmethod
-  def from_tuple(spec_tuple):
-    return FunctionSpec(*spec_tuple)
-
   @staticmethod
   def from_function_and_signature(python_function, input_signature):
     """Create a FunctionSpec instance given a python function and signature."""
@@ -920,7 +912,7 @@ class FunctionSpec(object):
     }
     self._default_values_start_index = offset
     if input_signature is None:
-      self.input_signature = None
+      self._input_signature = None
     else:
       if fullargspec.varkw is not None or fullargspec.kwonlyargs:
         raise ValueError("Cannot define a TensorFlow function from a Python "
@@ -931,8 +923,32 @@ class FunctionSpec(object):
         raise TypeError("input_signature must be either a tuple or a "
                         "list, received " + str(type(input_signature)))
 
-      self.input_signature = tuple(input_signature)
-      self.flat_input_signature = tuple(nest.flatten(input_signature))
+      self._input_signature = tuple(input_signature)
+      self._flat_input_signature = tuple(nest.flatten(input_signature))
+
+  @property
+  def fullargspec(self):
+    return self._fullargspec
+
+  @property
+  def is_method(self):
+    return self._is_method
+
+  @property
+  def args_to_prepend(self):
+    return self._args_to_prepend
+
+  @property
+  def kwargs_to_include(self):
+    return self._kwargs_to_include
+
+  @property
+  def input_signature(self):
+    return self._input_signature
+
+  @property
+  def flat_input_signature(self):
+    return self._flat_input_signature
 
   def canonicalize_function_inputs(self, *args, **kwargs):
     """Canonicalizes `args` and `kwargs`.
@@ -980,7 +996,7 @@ class FunctionSpec(object):
         if index is not None:
           arg_indices_to_values[index] = value
           consumed_args.append(arg)
-        elif self.input_signature is not None:
+        elif self._input_signature is not None:
           raise ValueError("Cannot define a TensorFlow function from a Python "
                            "function with keyword arguments when "
                            "input_signature is provided.")
@@ -1003,12 +1019,13 @@ class FunctionSpec(object):
     if need_packing:
       inputs = nest.pack_sequence_as(
           structure=inputs, flat_sequence=flat_inputs)
-    if self.input_signature is None:
+    if self._input_signature is None:
       return inputs, kwargs
     else:
       assert not kwargs
-      signature_relevant_inputs = inputs[:len(self.input_signature)]
-      if not is_same_structure(self.input_signature, signature_relevant_inputs):
+      signature_relevant_inputs = inputs[:len(self._input_signature)]
+      if not is_same_structure(self._input_signature,
+                               signature_relevant_inputs):
         raise ValueError("Structure of Python function inputs does not match "
                          "input_signature.")
       signature_inputs_flat = nest.flatten(signature_relevant_inputs)
@@ -1017,10 +1034,10 @@ class FunctionSpec(object):
         raise ValueError("When input_signature is provided, all inputs to "
                          "the Python function must be Tensors.")
       if any(not spec.is_compatible_with(other) for spec, other in zip(
-          self.flat_input_signature, signature_inputs_flat)):
+          self._flat_input_signature, signature_inputs_flat)):
         raise ValueError("Python inputs incompatible with input_signature: "
                          "inputs (%s), input_signature (%s)" %
-                         (str(inputs), str(self.input_signature)))
+                         (str(inputs), str(self._input_signature)))
       return inputs, {}
 
 
diff --git a/tensorflow/python/saved_model/function_deserialization.py b/tensorflow/python/saved_model/function_deserialization.py
index ec6cefb606..e19a46ac19 100644
--- a/tensorflow/python/saved_model/function_deserialization.py
+++ b/tensorflow/python/saved_model/function_deserialization.py
@@ -60,6 +60,17 @@ def _inputs_compatible(args, stored_inputs):
   return True
 
 
+def _deserialize_function_spec(function_spec_proto, coder):
+  """Deserialize a FunctionSpec object from its proto representation."""
+  fullargspec = coder.decode_proto(function_spec_proto.fullargspec)
+  is_method = function_spec_proto.is_method
+  args_to_prepend = coder.decode_proto(function_spec_proto.args_to_prepend)
+  kwargs_to_include = coder.decode_proto(function_spec_proto.kwargs_to_include)
+  input_signature = coder.decode_proto(function_spec_proto.input_signature)
+  return function_lib.FunctionSpec(fullargspec, is_method, args_to_prepend,
+                                   kwargs_to_include, input_signature)
+
+
 def recreate_polymorphic_function(
     saved_polymorphic_function, functions):
   """Creates a PolymorphicFunction from a SavedPolymorphicFunction.
@@ -77,9 +88,8 @@ def recreate_polymorphic_function(
   # serialization cycle.
 
   coder = nested_structure_coder.StructureCoder()
-  function_spec_tuple = coder.decode_proto(
-      saved_polymorphic_function.function_spec_tuple)
-  function_spec = function_lib.FunctionSpec.from_tuple(function_spec_tuple)
+  function_spec = _deserialize_function_spec(
+      saved_polymorphic_function.function_spec, coder)
 
   # TODO(mdan): We may enable autograph once exceptions are supported.
   @def_function.function(autograph=False)
diff --git a/tensorflow/python/saved_model/function_serialization.py b/tensorflow/python/saved_model/function_serialization.py
index 267852ef4a..d2c5b33a5f 100644
--- a/tensorflow/python/saved_model/function_serialization.py
+++ b/tensorflow/python/saved_model/function_serialization.py
@@ -25,13 +25,28 @@ from tensorflow.python.saved_model import saved_object_graph_pb2
 from tensorflow.python.util import nest
 
 
+def _serialize_function_spec(function_spec, coder):
+  """Serialize a FunctionSpec object into its proto representation."""
+  proto = saved_object_graph_pb2.FunctionSpec()
+  proto.fullargspec.CopyFrom(coder.encode_structure(function_spec.fullargspec))
+  proto.is_method = function_spec.is_method
+  proto.args_to_prepend.CopyFrom(
+      coder.encode_structure(function_spec.args_to_prepend))
+  proto.kwargs_to_include.CopyFrom(
+      coder.encode_structure(function_spec.kwargs_to_include))
+  proto.input_signature.CopyFrom(
+      coder.encode_structure(function_spec.input_signature))
+  return proto
+
+
 def serialize_polymorphic_function(polymorphic_function, node_ids):
   """Build a SavedPolymorphicProto."""
   coder = nested_structure_coder.StructureCoder()
   proto = saved_object_graph_pb2.SavedPolymorphicFunction()
 
-  proto.function_spec_tuple.CopyFrom(
-      coder.encode_structure(polymorphic_function.function_spec.as_tuple()))  # pylint: disable=protected-access
+  function_spec_proto = _serialize_function_spec(
+      polymorphic_function.function_spec, coder)
+  proto.function_spec.CopyFrom(function_spec_proto)
   for signature, concrete_function in list_all_concrete_functions(
       polymorphic_function):
     bound_inputs = []
diff --git a/tensorflow/python/saved_model/saved_object_graph.proto b/tensorflow/python/saved_model/saved_object_graph.proto
index a322726e1e..f48d2d2b7e 100644
--- a/tensorflow/python/saved_model/saved_object_graph.proto
+++ b/tensorflow/python/saved_model/saved_object_graph.proto
@@ -86,9 +86,7 @@ message SavedAsset {
 // A function with multiple signatures, possibly with non-Tensor arguments.
 message SavedPolymorphicFunction {
   repeated SavedMonomorphicFunction monomorphic_function = 1;
-  // Tuple representing a `FunctionSpec`.
-  // TODO(vbardiovsky): Make this a proto.
-  StructuredValue function_spec_tuple = 2;
+  FunctionSpec function_spec = 2;
 }
 
 message SavedMonomorphicFunction {
@@ -118,3 +116,20 @@ message SavedVariable {
 
   // TODO(andresp): Add save_slice_info_def?
 }
+
+// Represents FunctionSpec used in PolymorphicFunction. This represents a
+// function that has been wrapped as a PolymorphicFunction.
+message FunctionSpec {
+  // Full arg spec from inspect.getfullargspec().
+  StructuredValue fullargspec = 1;
+  // Whether this represents a class method.
+  bool is_method = 2;
+  // Which arguments to always prepend, in case the original function is based
+  // on a functools.partial.
+  StructuredValue args_to_prepend = 3;
+  // Which kwargs to always include, in case the original function is based on a
+  // functools.partial.
+  StructuredValue kwargs_to_include = 4;
+  // The input signature, if specified.
+  StructuredValue input_signature = 5;
+}
-- 
GitLab


From 471b523d9323ebed3f466b7e737e6536f45f4aee Mon Sep 17 00:00:00 2001
From: James Keeling <jtkeeling@google.com>
Date: Mon, 14 Jan 2019 07:28:18 -0800
Subject: [PATCH 0669/1095] Split cwise igammas GPU code into multiple files

This file was a bottleneck during compilation, often taking many minutes to compile. In local testing this change reduces the wall-clock build time for the affected kernels from 141s to 87s.

PiperOrigin-RevId: 229179013
---
 .../kernels/cwise_op_gpu_igammas_double.cu.cc | 29 +++++++++++++++++++
 ...cu.cc => cwise_op_gpu_igammas_float.cu.cc} |  6 ++--
 2 files changed, 32 insertions(+), 3 deletions(-)
 create mode 100644 tensorflow/core/kernels/cwise_op_gpu_igammas_double.cu.cc
 rename tensorflow/core/kernels/{cwise_op_gpu_igammas.cu.cc => cwise_op_gpu_igammas_float.cu.cc} (88%)

diff --git a/tensorflow/core/kernels/cwise_op_gpu_igammas_double.cu.cc b/tensorflow/core/kernels/cwise_op_gpu_igammas_double.cu.cc
new file mode 100644
index 0000000000..2bcc7aa885
--- /dev/null
+++ b/tensorflow/core/kernels/cwise_op_gpu_igammas_double.cu.cc
@@ -0,0 +1,29 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+
+#include "tensorflow/core/kernels/cwise_ops_gpu_common.cu.h"
+#include "tensorflow/core/kernels/cwise_ops_gpu_gradients.cu.h"
+
+namespace tensorflow {
+namespace functor {
+DEFINE_BINARY1(igamma, double);
+DEFINE_BINARY1(igamma_grad_a, double);
+DEFINE_BINARY1(igammac, double);
+}  // namespace functor
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/cwise_op_gpu_igammas.cu.cc b/tensorflow/core/kernels/cwise_op_gpu_igammas_float.cu.cc
similarity index 88%
rename from tensorflow/core/kernels/cwise_op_gpu_igammas.cu.cc
rename to tensorflow/core/kernels/cwise_op_gpu_igammas_float.cu.cc
index 508a47deda..e6412216e9 100644
--- a/tensorflow/core/kernels/cwise_op_gpu_igammas.cu.cc
+++ b/tensorflow/core/kernels/cwise_op_gpu_igammas_float.cu.cc
@@ -20,9 +20,9 @@ limitations under the License.
 
 namespace tensorflow {
 namespace functor {
-DEFINE_BINARY2(igamma, float, double);
-DEFINE_BINARY2(igamma_grad_a, float, double);
-DEFINE_BINARY2(igammac, float, double);
+DEFINE_BINARY1(igamma, float);
+DEFINE_BINARY1(igamma_grad_a, float);
+DEFINE_BINARY1(igammac, float);
 }  // namespace functor
 }  // namespace tensorflow
 
-- 
GitLab


From 6dae7e3ebbfd83b9f1d318ba5f2f2baeae17b9f5 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 14 Jan 2019 07:40:31 -0800
Subject: [PATCH 0670/1095] Adapt exec_compatible_with for remote testing in
 add_an_op test.

PiperOrigin-RevId: 229180723
---
 tensorflow/examples/adding_an_op/BUILD | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/examples/adding_an_op/BUILD b/tensorflow/examples/adding_an_op/BUILD
index 2b39b3683f..a4d6f204cd 100644
--- a/tensorflow/examples/adding_an_op/BUILD
+++ b/tensorflow/examples/adding_an_op/BUILD
@@ -10,6 +10,7 @@ licenses(["notice"])  # Apache 2.0
 load("//tensorflow:tensorflow.bzl", "tf_custom_op_library")
 load("//tensorflow:tensorflow.bzl", "tf_cuda_tests_tags")
 load("//tensorflow:tensorflow.bzl", "tf_cc_binary")
+load("//tensorflow:tensorflow.bzl", "tf_exec_compatible_with")
 
 exports_files(["LICENSE"])
 
@@ -118,6 +119,7 @@ py_test(
     name = "cuda_op_test",
     size = "small",
     srcs = ["cuda_op_test.py"],
+    exec_compatible_with = tf_exec_compatible_with({"tags": tf_cuda_tests_tags()}),
     srcs_version = "PY2AND3",
     tags = tf_cuda_tests_tags() + ["notap"],
     deps = [
-- 
GitLab


From 22e458382d3001a0cda4e594decf175f2387475e Mon Sep 17 00:00:00 2001
From: Jian Li <jianlijianli@google.com>
Date: Mon, 14 Jan 2019 08:12:13 -0800
Subject: [PATCH 0671/1095] Create quantize multiplier for array.

PiperOrigin-RevId: 229185162
---
 .../kernels/internal/quantization_util.cc     |  9 ++++
 .../lite/kernels/internal/quantization_util.h | 11 +++++
 .../internal/quantization_util_test.cc        | 47 +++++++++++++++++++
 3 files changed, 67 insertions(+)

diff --git a/tensorflow/lite/kernels/internal/quantization_util.cc b/tensorflow/lite/kernels/internal/quantization_util.cc
index 0279d2a922..71eef71372 100644
--- a/tensorflow/lite/kernels/internal/quantization_util.cc
+++ b/tensorflow/lite/kernels/internal/quantization_util.cc
@@ -366,4 +366,13 @@ bool CheckedLog2(const float x, int* log2_result) {
   return std::abs(x_log2_fracpart) < 1e-3;
 }
 
+void QuantizeMultiplierArray(const double* effective_scales, size_t size,
+                             int32_t* effective_scale_significand,
+                             int* effective_shift) {
+  for (size_t i = 0; i < size; ++i) {
+    QuantizeMultiplier(effective_scales[i], &effective_scale_significand[i],
+                       &effective_shift[i]);
+  }
+}
+
 }  // namespace tflite
diff --git a/tensorflow/lite/kernels/internal/quantization_util.h b/tensorflow/lite/kernels/internal/quantization_util.h
index bf313f39cd..5d67c0d027 100644
--- a/tensorflow/lite/kernels/internal/quantization_util.h
+++ b/tensorflow/lite/kernels/internal/quantization_util.h
@@ -275,6 +275,17 @@ void FakeQuantizeArray(const float nudged_scale, const float nudged_min,
 // returns false.
 bool CheckedLog2(const float x, int* log2_result);
 
+// Decomposes an array of double multipliers into a Q0.31 int32 representation
+// of its significand, and shift representation of its exponent.
+//
+// Handles an arbitrary multiplier. The 'shift' output-value is
+// basically the 'floating-point exponent' of the multiplier:
+// Negative for a right-shift (when the multiplier is <1), positive for a
+// left-shift (when the multiplier is >1)
+void QuantizeMultiplierArray(const double* effective_scales, size_t size,
+                             int32_t* effective_scale_significand,
+                             int* effective_shift);
+
 }  // namespace tflite
 
 #endif  // TENSORFLOW_LITE_KERNELS_INTERNAL_QUANTIZATION_UTIL_H_
diff --git a/tensorflow/lite/kernels/internal/quantization_util_test.cc b/tensorflow/lite/kernels/internal/quantization_util_test.cc
index 2f8f771379..ca4ff370ad 100644
--- a/tensorflow/lite/kernels/internal/quantization_util_test.cc
+++ b/tensorflow/lite/kernels/internal/quantization_util_test.cc
@@ -20,6 +20,7 @@ limitations under the License.
 namespace tflite {
 namespace {
 
+using ::testing::ElementsAreArray;
 using ::testing::Pair;
 
 template <class FloatIn, class IntOut>
@@ -406,6 +407,52 @@ TEST(QuantizationUtilTest, CalculateInputRadius) {
   EXPECT_EQ(CalculateInputRadius(4, 2), 503316480);
 }
 
+TEST(QuantizationUtilTest, QuantizeMultiplierArray) {
+  const std::vector<double> weights = {-4,    -2,   -1,  -0.5, -0.25, -0.125, 0,
+                                       0.125, 0.25, 0.5, 1,    2,     4};
+  const int size = weights.size();
+  std::vector<int32> effective_scale_significand(size);
+  std::vector<int> effective_scale_shift(size);
+  QuantizeMultiplierArray(weights.data(), size,
+                          effective_scale_significand.data(),
+                          effective_scale_shift.data());
+  const std::vector<int32> expected_effective_scale_significand = {
+      -1073741824,  // float scale = -4
+      -1073741824,  // float scale = -2
+      -1073741824,  // float scale = -1
+      -1073741824,  // float scale = -0.5
+      -1073741824,  // float scale = -0.25
+      -1073741824,  // float scale = -0.125
+      0,            // float scale = 0
+      1073741824,   // float scale = 0.125
+      1073741824,   // float scale = 0.25
+      1073741824,   // float scale = 0.5
+      1073741824,   // float scale = 1
+      1073741824,   // float scale = 2
+      1073741824,   // float scale = 4
+  };
+
+  const std::vector<int> expected_effective_scale_shift = {
+      3,   // float scale = -4
+      2,   // float scale = -2
+      1,   // float scale = -1
+      0,   // float scale = -0.5
+      -1,  // float scale = -0.25
+      -2,  // float scale = -0.125
+      0,   // float scale = 0
+      -2,  // float scale = 0.125
+      -1,  // float scale = 0.25
+      0,   // float scale = 0.5
+      1,   // float scale = 1
+      2,   // float scale = 2
+      3,   // float scale = 4
+  };
+  EXPECT_THAT(effective_scale_significand,
+              ElementsAreArray(expected_effective_scale_significand));
+  EXPECT_THAT(effective_scale_shift,
+              ElementsAreArray(expected_effective_scale_shift));
+}
+
 }  // namespace
 }  // namespace tflite
 
-- 
GitLab


From dbb3f15afb2ee8d6af7c4a4adc9843314ecfa1cc Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 14 Jan 2019 09:13:28 -0800
Subject: [PATCH 0672/1095] Add tests for batch_group_count.

PiperOrigin-RevId: 229194140
---
 tensorflow/compiler/xla/service/BUILD         |   4 +-
 .../service/convolution_group_converter.cc    |  14 +-
 ...cc => convolution_group_converter_test.cc} |  27 ++++
 .../xla/service/hlo_evaluator_typed_visitor.h |  33 +++-
 .../compiler/xla/service/hlo_instructions.cc  |   4 +
 .../compiler/xla/service/shape_inference.cc   |  18 +++
 tensorflow/compiler/xla/tests/BUILD           |  20 +++
 .../conv_depthwise_backprop_filter_test.cc    | 150 ++++++++++++++++++
 8 files changed, 256 insertions(+), 14 deletions(-)
 rename tensorflow/compiler/xla/service/{convolution_feature_group_converter_test.cc => convolution_group_converter_test.cc} (75%)
 create mode 100644 tensorflow/compiler/xla/tests/conv_depthwise_backprop_filter_test.cc

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index b324a810ce..48defc011b 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -1731,9 +1731,9 @@ cc_library(
 )
 
 tf_cc_test(
-    name = "convolution_feature_group_converter_test",
+    name = "convolution_group_converter_test",
     size = "small",
-    srcs = ["convolution_feature_group_converter_test.cc"],
+    srcs = ["convolution_group_converter_test.cc"],
     deps = [
         ":convolution_group_converter",
         ":hlo",
diff --git a/tensorflow/compiler/xla/service/convolution_group_converter.cc b/tensorflow/compiler/xla/service/convolution_group_converter.cc
index 7a24faec17..b3ed27d9a8 100644
--- a/tensorflow/compiler/xla/service/convolution_group_converter.cc
+++ b/tensorflow/compiler/xla/service/convolution_group_converter.cc
@@ -207,7 +207,8 @@ Status ConvolutionVisitor::HandleBatchGroupCount(HloInstruction* convolution) {
     return Status::OK();
   }
 
-  VLOG(2) << "Dealing with batch_group_count " << batch_group_count << "\n";
+  VLOG(2) << "Dealing with batch_group_count " << batch_group_count
+          << " for convolution " << convolution->ToString() << "\n";
 
   auto add = [&](std::unique_ptr<HloInstruction> inst) {
     return computation_->AddInstruction(std::move(inst));
@@ -332,18 +333,19 @@ Status ConvolutionVisitor::HandleBatchGroupCount(HloInstruction* convolution) {
       return computation_->parent()->AddEmbeddedComputation(b.Build(scalar_op));
     };
 
+    auto reduce_window_shape = new_convolution->shape();
+    reduce_window_shape.set_dimensions(output_batch_dimension, 1);
+
     // Ensure that data input to reduce window is of type F32.
     if (primitive_util::BitWidth(new_filter->shape().element_type()) <
         primitive_util::BitWidth(F32)) {
+      reduce_window_shape.set_element_type(F32);
       Shape convert_shape = new_filter->shape();
       convert_shape.set_element_type(F32);
       new_filter =
-          add(HloInstruction::CreateBitcastConvert(convert_shape, new_filter));
+          add(HloInstruction::CreateConvert(convert_shape, new_filter));
     }
 
-    auto reduce_window_shape = new_convolution->shape();
-    reduce_window_shape.set_dimensions(output_batch_dimension, 1);
-
     // Create the reduce window.
     Window window;
     for (int64 i = 0; i < new_convolution->shape().dimensions_size(); ++i) {
@@ -369,7 +371,7 @@ Status ConvolutionVisitor::HandleBatchGroupCount(HloInstruction* convolution) {
 
     // Convert reduced data back to the original data type.
     auto reduce_window_converted =
-        HloInstruction::CreateBitcastConvert(convert_back_shape, reduce_window);
+        HloInstruction::CreateConvert(convert_back_shape, reduce_window);
 
     TF_RETURN_IF_ERROR(computation_->ReplaceWithNewInstruction(
         convolution, std::move(reduce_window_converted)));
diff --git a/tensorflow/compiler/xla/service/convolution_feature_group_converter_test.cc b/tensorflow/compiler/xla/service/convolution_group_converter_test.cc
similarity index 75%
rename from tensorflow/compiler/xla/service/convolution_feature_group_converter_test.cc
rename to tensorflow/compiler/xla/service/convolution_group_converter_test.cc
index d58f157242..585b81a5db 100644
--- a/tensorflow/compiler/xla/service/convolution_feature_group_converter_test.cc
+++ b/tensorflow/compiler/xla/service/convolution_group_converter_test.cc
@@ -94,5 +94,32 @@ ENTRY %Convolve1D1Window_0.v3 (input: f32[1,2,4], filter: f32[1,2,2]) -> f32[1,2
   EXPECT_EQ(root->operand(1)->feature_group_count(), 1);
 }
 
+TEST_F(ConvolutionGroupConverterTest,
+       ConvertBatchGroupCountEqualToInputBatchDim) {
+  string hlo_string = R"(HloModule Convolve1D1Window_0_module
+
+ENTRY %Convolve1D1Window_0.v3 (input: f32[16,19,19,512]{3,2,1,0}, filter: f32[16,19,19,512]{3,2,1,0}) -> f32[3,3,512,1]{3,2,1,0} {
+  %input = f32[16,19,19,512]{3,2,1,0} parameter(0)
+  %filter = f32[16,19,19,512]{3,2,1,0} parameter(1)
+  ROOT %convolution = f32[3,3,512,1]{3,2,1,0} convolution(f32[16,19,19,512]{3,2,1,0} %input, f32[16,19,19,512]{3,2,1,0} %filter), window={size=19x19 pad=1_1x1_1}, dim_labels=f01b_i01o->01fb, batch_group_count=512
+  })";
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
+                          ParseHloString(hlo_string));
+
+  auto computation = module->entry_computation();
+  HloInstruction* root = computation->root_instruction();
+  EXPECT_EQ(root->opcode(), HloOpcode::kConvolution);
+  auto cost_model = [](HloInstruction* conv) { return true; };
+  ConvolutionGroupConverter converter(cost_model, /*convert_batch_groups_only=*/
+                                      true);
+  ASSERT_TRUE(converter.Run(module.get()).ValueOrDie());
+  root = computation->root_instruction();
+  // Make sure the convolution is converted to one with batch_group_count = 1.
+  EXPECT_EQ(root->operand(0)->opcode(), HloOpcode::kConvolution);
+  EXPECT_EQ(root->operand(0)->batch_group_count(), 1);
+  // Verify that the convolution is replaced by a reshape.
+  EXPECT_EQ(root->opcode(), HloOpcode::kReshape);
+}
+
 }  // namespace
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
index f3f400e46f..5245c563ec 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
@@ -1042,15 +1042,13 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
     auto lhs_literal_data = lhs_literal.data<ReturnT>();
     auto rhs_literal_data = rhs_literal.data<ReturnT>();
 
-    int64 feature_group_count = conv->feature_group_count();
-    int64 batch_group_count = conv->batch_group_count();
+    const int64 feature_group_count = conv->feature_group_count();
+    const int64 batch_group_count = conv->batch_group_count();
 
-    // The batch count > 1 case is unimplemented in the HLO evaluator so far.
-    TF_RET_CHECK(batch_group_count == 1);
     auto func = [&window_shape, &dnums, &lhs_shape, &rhs_shape, &window,
                  &lhs_dim_multipliers, &rhs_dim_multipliers, lhs_literal_data,
-                 rhs_literal_data,
-                 feature_group_count](const absl::Span<const int64> out_index) {
+                 rhs_literal_data, feature_group_count,
+                 batch_group_count](const absl::Span<const int64> out_index) {
       // Dimension number applicable for input (lhs).
       const int64 input_batch_dim = dnums.input_batch_dimension();
       const int64 input_z_dim = dnums.input_feature_dimension();
@@ -1063,6 +1061,12 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
 
       const int64 input_z_size =
           ShapeUtil::GetDimension(lhs_shape, input_z_dim);
+
+      const int64 input_batch_size =
+          ShapeUtil::GetDimension(lhs_shape, input_batch_dim);
+
+      const int64 batch_group_size = input_batch_size / batch_group_count;
+
       // The size of an input feature group.
       const int64 input_feature_group_size = input_z_size / feature_group_count;
 
@@ -1078,11 +1082,15 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
       const int64 feature_group_index =
           out_index[output_z_dim] / output_feature_group_size;
 
+      const int64 batch_group_index = out_index[output_z_dim];
+
       ElementwiseT result_val = static_cast<ElementwiseT>(0);
       DimensionVector rhs_spatial_index(dnums.kernel_spatial_dimensions_size(),
                                         0);
 
       // Convolve input feature with kernel.
+      // The mechanism indexes into the correct LHS (input) and RHS (kernel)
+      // locations and accumulates multiplications for a given output index.
       do {
         // Find corresponding spatial dimension index for input (lhs).
         int64 lhs_linear_spatial_index = 0;
@@ -1135,11 +1143,24 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
               feature_group_index * input_feature_group_size + rhs_iz;
 
           int64 lhs_linear_index = lhs_linear_spatial_index;
+
           lhs_linear_index += out_index[output_batch_dim] *
                               lhs_dim_multipliers[input_batch_dim];
+
+          // We are scraping only the diagonal elements in the resultant
+          // convolution output when batch_group_count is greater than 1,
+          // where 1 is the default. No scraping is done in that case.
+          // This approach works out automatically for 'groups' in batches
+          // with group_size > 1, because we already descend down the batch
+          // dimension for the 'output_batch_dim' above.
+          lhs_linear_index +=
+              ((batch_group_index * batch_group_size) % input_batch_size) *
+              lhs_dim_multipliers[input_batch_dim];
+
           lhs_linear_index += iz * lhs_dim_multipliers[input_z_dim];
 
           int64 rhs_linear_index = rhs_linear_spatial_index;
+
           rhs_linear_index += out_index[output_z_dim] *
                               rhs_dim_multipliers[kernel_output_z_dim];
           rhs_linear_index += rhs_iz * rhs_dim_multipliers[kernel_input_z_dim];
diff --git a/tensorflow/compiler/xla/service/hlo_instructions.cc b/tensorflow/compiler/xla/service/hlo_instructions.cc
index c24bf41ff8..3a0d71dd88 100644
--- a/tensorflow/compiler/xla/service/hlo_instructions.cc
+++ b/tensorflow/compiler/xla/service/hlo_instructions.cc
@@ -1702,6 +1702,10 @@ std::vector<string> HloConvolutionInstruction::ExtraAttributesToStringImpl(
     extra.push_back(StrCat("feature_group_count=", feature_group_count_));
   }
 
+  if (batch_group_count_ != 1) {
+    extra.push_back(StrCat("batch_group_count=", batch_group_count_));
+  }
+
   string precision_config_string = PrecisionConfigToString(precision_config_);
   if (!precision_config_string.empty()) {
     extra.push_back(precision_config_string);
diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc
index c2e014ad42..fad7afefaf 100644
--- a/tensorflow/compiler/xla/service/shape_inference.cc
+++ b/tensorflow/compiler/xla/service/shape_inference.cc
@@ -1539,6 +1539,13 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
         batch_group_count);
   }
 
+  if (batch_group_count > 1 && feature_group_count > 1) {
+    return InvalidArgument(
+        "both batch_group_count %d and feature_group_count %d cannot be "
+        "greater than 1",
+        batch_group_count, feature_group_count);
+  }
+
   if (!ShapeUtil::SameElementTypeIgnoringFpPrecision(lhs, rhs)) {
     return InvalidArgument(
         "Convolution with different element types: %s and %s.",
@@ -1652,6 +1659,17 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
   const int64 kernel_output_features =
       rhs.dimensions(dnums.kernel_output_feature_dimension());
 
+  if (batch_group_count > 1 && input_batch % kernel_output_features != 0) {
+    return InvalidArgument(
+        "Expected output feature dimension (value %d) to be divisible by "
+        "input_batch (value %d) for batch group count %d; "
+        "got <conv>(%s, %s)\n"
+        "Dimension numbers: {%s}.",
+        kernel_output_features, input_batch, batch_group_count,
+        ShapeUtil::HumanString(lhs), ShapeUtil::HumanString(rhs),
+        dnums.DebugString());
+  }
+
   if (input_features % feature_group_count != 0 ||
       input_features / feature_group_count != kernel_input_features) {
     return InvalidArgument(
diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index 0fd0fc108a..4e7480c9af 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -313,6 +313,26 @@ xla_test(
     ],
 )
 
+xla_test(
+    name = "conv_depthwise_backprop_filter_test",
+    timeout = "long",
+    srcs = ["conv_depthwise_backprop_filter_test.cc"],
+    shard_count = 1,
+    deps = [
+        "//tensorflow/compiler/xla:execution_options_util",
+        "//tensorflow/compiler/xla:status_macros",
+        "//tensorflow/compiler/xla:test",
+        "//tensorflow/compiler/xla/client:xla_computation",
+        "//tensorflow/compiler/xla/service:bfloat16_normalization",
+        "//tensorflow/compiler/xla/service:despecializer",
+        "//tensorflow/compiler/xla/service:hlo_parser",
+        "//tensorflow/compiler/xla/tests:client_library_test_base",
+        "//tensorflow/compiler/xla/tests:hlo_test_base",
+        "//tensorflow/compiler/xla/tests:xla_internal_test_main",
+        "@com_google_absl//absl/types:optional",
+    ],
+)
+
 xla_test(
     name = "grouped_convolution_test",
     timeout = "long",
diff --git a/tensorflow/compiler/xla/tests/conv_depthwise_backprop_filter_test.cc b/tensorflow/compiler/xla/tests/conv_depthwise_backprop_filter_test.cc
new file mode 100644
index 0000000000..90c1971403
--- /dev/null
+++ b/tensorflow/compiler/xla/tests/conv_depthwise_backprop_filter_test.cc
@@ -0,0 +1,150 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "absl/types/optional.h"
+#include "tensorflow/compiler/xla/client/xla_computation.h"
+#include "tensorflow/compiler/xla/execution_options_util.h"
+#include "tensorflow/compiler/xla/service/bfloat16_normalization.h"
+#include "tensorflow/compiler/xla/service/despecializer.h"
+#include "tensorflow/compiler/xla/service/hlo_parser.h"
+#include "tensorflow/compiler/xla/status_macros.h"
+#include "tensorflow/compiler/xla/test.h"
+#include "tensorflow/compiler/xla/tests/client_library_test_base.h"
+#include "tensorflow/compiler/xla/tests/hlo_test_base.h"
+#include "tensorflow/compiler/xla/tests/test_macros.h"
+
+namespace xla {
+namespace {
+
+string GetFloatDataType(bool use_bfloat16) {
+  return use_bfloat16 ? "bf16" : "f32";
+}
+
+struct DepthwiseConvolution2DSpec {
+  int64 output_batch, window;
+  std::vector<int64> activation_dims;
+  std::vector<int64> activation_layout;
+  std::vector<int64> kernel_dims;
+  std::vector<int64> kernel_layout;
+  std::vector<int64> output_dims;
+  std::vector<int64> output_layout;
+};
+
+class DepthwiseConvolution2DTest
+    : public HloTestBase,
+      public ::testing::WithParamInterface<
+          ::testing::tuple<DepthwiseConvolution2DSpec, bool>> {};
+
+static std::vector<DepthwiseConvolution2DSpec> GetConv2DTestCases() {
+  std::vector<DepthwiseConvolution2DSpec> config_set;
+  std::vector<std::vector<int64>> config_options = {
+      {16, 5, 5, 2}, {64, 4, 4, 16}, {2, 5, 5, 256}};
+
+  for (auto option : config_options) {
+    int64 feature = option[3];
+    int64 activation_size = option[1];
+    int64 kernel_size = option[2];
+    int64 batch = option[0];
+
+    std::vector<int64> kernel_layout = {3, 2, 1, 0};
+    DepthwiseConvolution2DSpec config;
+    config.output_batch = feature;
+    config.window = kernel_size;
+
+    config.activation_dims = {batch, activation_size, activation_size, feature};
+    config.activation_layout = {0, 3, 2, 1};
+
+    config.kernel_dims = {batch, kernel_size, kernel_size, feature};
+    config.kernel_layout = {0, 2, 3, 1};
+
+    config.output_dims = {3, 3, feature, 1};
+
+    // Try this layout for all kernel shapes.
+    config.output_layout = {3, 2, 0, 1};
+    config_set.push_back(config);
+  }
+
+  return config_set;
+}
+
+string DepthwiseConvolution2DTestDataToString(
+    const ::testing::TestParamInfo<
+        ::testing::tuple<DepthwiseConvolution2DSpec, bool>>& data) {
+  const auto& spec = ::testing::get<0>(data.param);
+  const string data_type = GetFloatDataType(::testing::get<1>(data.param));
+  string str = absl::StrCat(
+      "activation_dims_", absl::StrJoin(spec.activation_dims, "x"),
+      "_activation_layout_", absl::StrJoin(spec.activation_layout, "_"),
+      "_kernel_dims_", absl::StrJoin(spec.kernel_dims, "x"), "_kernel_layout_",
+      absl::StrJoin(spec.kernel_layout, "_"), "_output_dims_",
+      absl::StrJoin(spec.output_dims, "x"), "_output_layout_",
+      absl::StrJoin(spec.output_layout, "_"), data_type);
+
+  // Test names are not allowed to contain the '-' character.
+  absl::c_replace(str, '-', 'n');
+  return str;
+}
+
+string BuildHloTextDepthwiseConvolution2D(
+    const DepthwiseConvolution2DSpec& spec, bool use_bfloat16) {
+  const string data_type = GetFloatDataType(use_bfloat16);
+  return absl::StrFormat(
+      R"(
+    HloModule TensorFlowDepthwiseConv
+
+    ENTRY main {
+      activation = %s[%s]{%s} parameter(0)
+      kernel = %s[%s]{%s} parameter(1)
+      ROOT conv = %s[%s]{%s} convolution(%s[%s]{%s} activation, %s[%s]{%s} kernel),
+          window={size=%dx%d pad=1_1x1_1}, dim_labels=f01b_i01o->01fb,
+          batch_group_count=%d
+    }
+    )",
+      data_type, absl::StrJoin(spec.activation_dims, ","),
+      absl::StrJoin(spec.activation_layout, ","), data_type,
+      absl::StrJoin(spec.kernel_dims, ","),
+      absl::StrJoin(spec.kernel_layout, ","), data_type,
+      absl::StrJoin(spec.output_dims, ","),
+      absl::StrJoin(spec.output_layout, ","), data_type,
+      absl::StrJoin(spec.activation_dims, ","),
+      absl::StrJoin(spec.activation_layout, ","), data_type,
+      absl::StrJoin(spec.kernel_dims, ","),
+      absl::StrJoin(spec.kernel_layout, ","), spec.window, spec.window,
+      spec.output_batch);
+}
+
+XLA_TEST_P(DepthwiseConvolution2DTest, DoIt) {
+  const DepthwiseConvolution2DSpec& spec = ::testing::get<0>(GetParam());
+  bool use_bfloat16 = ::testing::get<1>(GetParam());
+  const string hlo_text =
+      BuildHloTextDepthwiseConvolution2D(spec, use_bfloat16);
+
+  EXPECT_TRUE(RunAndCompare(hlo_text, ErrorSpec{0.01, 0.01},
+                            [](HloModule* module) -> Status {
+                              BFloat16MixedPrecisionRemoval remover;
+                              TF_RETURN_IF_ERROR(remover.Run(module).status());
+                              Despecializer despecializer;
+                              return despecializer.Run(module).status();
+                            }));
+}
+
+INSTANTIATE_TEST_CASE_P(
+    DepthwiseConvolution2DTestWithRandomIndices, DepthwiseConvolution2DTest,
+    ::testing::Combine(::testing::ValuesIn(GetConv2DTestCases()),
+                       ::testing::Bool()),
+    DepthwiseConvolution2DTestDataToString);
+
+}  // namespace
+}  // namespace xla
-- 
GitLab


From b9598f45d4de3baaa745436891907f989446d6b6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 14 Jan 2019 09:30:31 -0800
Subject: [PATCH 0673/1095] Rollback change in Eager mode decorator.

This change causes the Eager mode to be run with all methods in the class, but not all methods in its super classes.

PiperOrigin-RevId: 229196652
---
 tensorflow/python/framework/test_util.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py
index 491249089e..22cd50321f 100644
--- a/tensorflow/python/framework/test_util.py
+++ b/tensorflow/python/framework/test_util.py
@@ -885,8 +885,7 @@ def generate_combinations_with_testcase_name(**kwargs):
 def run_all_in_graph_and_eager_modes(cls):
   """Execute all test methods in the given class with and without eager."""
   base_decorator = run_in_graph_and_eager_modes
-  for name in dir(cls):
-    value = getattr(cls, name)
+  for name, value in cls.__dict__.copy().items():
     if callable(value) and name.startswith(
         unittest.TestLoader.testMethodPrefix) and not (
             name.startswith("testSkipEager") or
-- 
GitLab


From caf00d258648be6ad19c98830a63431e31209045 Mon Sep 17 00:00:00 2001
From: Vojtech Bardiovsky <vbardiovsky@google.com>
Date: Mon, 14 Jan 2019 09:33:38 -0800
Subject: [PATCH 0674/1095] Refactoring: move list_all_concrete_functions into
 PolymorphicFunction.

This will enable overriding this function in a "RestoredPolymorphicFunction", which will be helpful for repeated saving/restoring.

PiperOrigin-RevId: 229197202
---
 tensorflow/python/eager/def_function.py       | 23 ++++++++++++++
 .../saved_model/function_serialization.py     | 30 ++-----------------
 tensorflow/python/saved_model/load_test.py    |  5 ++--
 tensorflow/python/saved_model/save.py         |  4 +--
 4 files changed, 30 insertions(+), 32 deletions(-)

diff --git a/tensorflow/python/eager/def_function.py b/tensorflow/python/eager/def_function.py
index 4c56358e24..bd70b9dad0 100644
--- a/tensorflow/python/eager/def_function.py
+++ b/tensorflow/python/eager/def_function.py
@@ -30,7 +30,9 @@ from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import variable_scope
+from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.training.checkpointable import base as checkpointable
+from tensorflow.python.util import nest
 from tensorflow.python.util import tf_decorator
 from tensorflow.python.util.tf_export import tf_export
 
@@ -443,6 +445,27 @@ class PolymorphicFunction(object):
 
     return initialize_variables.get_concrete_function()
 
+  def _list_all_concrete_functions_for_serialization(self):
+    """Returns all of the concrete functions.
+
+    Returns:
+      A list of tuples in the form (signature, concrete_function), where
+      concrete function is an instance of `Function`.
+    """
+    input_signature = self._input_signature
+    if input_signature is not None:
+      self.get_concrete_function()
+    concrete_functions = []
+    for signature in self._cached_input_signatures:
+      flattened = nest.flatten(signature)
+      if any(
+          isinstance(arg, function_lib.UnknownArgument) for arg in flattened):
+        logging.info("Unsupported signature for serialization: %s.", signature)
+        continue
+      concrete_function = self.get_concrete_function(*signature)
+      concrete_functions.append((signature, concrete_function))
+    return concrete_functions
+
   @property
   def _cached_input_signatures(self):
     """All input signatures used to call this PolymorphicFunction."""
diff --git a/tensorflow/python/saved_model/function_serialization.py b/tensorflow/python/saved_model/function_serialization.py
index d2c5b33a5f..e5d3cc4bd2 100644
--- a/tensorflow/python/saved_model/function_serialization.py
+++ b/tensorflow/python/saved_model/function_serialization.py
@@ -22,7 +22,6 @@ from tensorflow.python.eager import function as defun_lib
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.saved_model import nested_structure_coder
 from tensorflow.python.saved_model import saved_object_graph_pb2
-from tensorflow.python.util import nest
 
 
 def _serialize_function_spec(function_spec, coder):
@@ -47,8 +46,9 @@ def serialize_polymorphic_function(polymorphic_function, node_ids):
   function_spec_proto = _serialize_function_spec(
       polymorphic_function.function_spec, coder)
   proto.function_spec.CopyFrom(function_spec_proto)
-  for signature, concrete_function in list_all_concrete_functions(
-      polymorphic_function):
+  all_concrete_functions = \
+      polymorphic_function._list_all_concrete_functions_for_serialization()  # pylint: disable=protected-access
+  for signature, concrete_function in all_concrete_functions:
     bound_inputs = []
     try:
       for capture in concrete_function.captured_inputs:
@@ -70,27 +70,3 @@ def serialize_polymorphic_function(polymorphic_function, node_ids):
         coder.encode_structure(structured_outputs))
     function_proto.bound_inputs.extend(bound_inputs)
   return proto
-
-
-def list_all_concrete_functions(polymorphic_function):
-  """Given a polymorphic function, returns all of its concrete functions.
-
-  Args:
-    polymorphic_function: Instance of `PolymorphicFunction`.
-
-  Returns:
-    A list of tuples in the form (signature, concrete_function), where concrete
-    function is an instance of `Function`.
-  """
-  input_signature = polymorphic_function._input_signature  # pylint: disable=protected-access
-  if input_signature is not None:
-    polymorphic_function.get_concrete_function()
-  concrete_functions = []
-  for signature in polymorphic_function._cached_input_signatures:  # pylint: disable=protected-access
-    flattened = nest.flatten(signature)
-    if any(isinstance(arg, defun_lib.UnknownArgument) for arg in flattened):
-      logging.info("Unsupported signature for serialization: %s.", signature)
-      continue
-    concrete_function = polymorphic_function.get_concrete_function(*signature)
-    concrete_functions.append((signature, concrete_function))
-  return concrete_functions
diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index 1d6d1fe88c..e94cda533e 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -30,7 +30,6 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import tensor_spec
 from tensorflow.python.lib.io import file_io
 from tensorflow.python.ops import variables
-from tensorflow.python.saved_model import function_serialization
 from tensorflow.python.saved_model import load
 from tensorflow.python.saved_model import save
 from tensorflow.python.training.checkpointable import tracking
@@ -389,8 +388,8 @@ class LoadTest(test.TestCase):
     self.assertAllEqual([2], root.f(constant_op.constant([1])).numpy())
     self.assertAllEqual([2, 4], root.f(constant_op.constant([1, 2])).numpy())
 
-    self.assertEqual(
-        1, len(function_serialization.list_all_concrete_functions(root.f)))
+    concrete_functions = root.f._list_all_concrete_functions_for_serialization()
+    self.assertEqual(1, len(concrete_functions))  # pylint: disable=protected-access
 
     imported = self.cycle(root)
 
diff --git a/tensorflow/python/saved_model/save.py b/tensorflow/python/saved_model/save.py
index 0f98766553..207dc2cb74 100644
--- a/tensorflow/python/saved_model/save.py
+++ b/tensorflow/python/saved_model/save.py
@@ -84,7 +84,7 @@ class _SaveableView(object):
         #  input_signature and have not been called.
         #  - force side effects of creation of concrete functions, e.g. create
         #  variables on first run.
-        function_serialization.list_all_concrete_functions(function)
+        function._list_all_concrete_functions_for_serialization()  # pylint: disable=protected-access
 
   @property
   def root(self):
@@ -556,7 +556,7 @@ def _fill_meta_graph_def(meta_graph_def, saveable_view, signature_functions,
   for obj in accessible_objects:
     for function in saveable_view.polymorphic_functions[obj].values():
       concrete_functions.extend(
-          function_serialization.list_all_concrete_functions(function))
+          function._list_all_concrete_functions_for_serialization())  # pylint: disable=protected-access
 
   with exported_graph.as_default():
     signatures = _generate_signatures(signature_functions, resource_map)
-- 
GitLab


From a1a8cf6038812c3f5f81d9ddda11553bd2dcc149 Mon Sep 17 00:00:00 2001
From: Karim Nosir <karimnosseir@google.com>
Date: Mon, 14 Jan 2019 09:42:00 -0800
Subject: [PATCH 0675/1095] Fix file path for test

PiperOrigin-RevId: 229198411
---
 tensorflow/lite/models/smartreply/predictor_test.cc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tensorflow/lite/models/smartreply/predictor_test.cc b/tensorflow/lite/models/smartreply/predictor_test.cc
index 9ce8ce9a77..f4a9453b42 100644
--- a/tensorflow/lite/models/smartreply/predictor_test.cc
+++ b/tensorflow/lite/models/smartreply/predictor_test.cc
@@ -34,8 +34,7 @@ namespace {
 const char kSamples[] = "smartreply_samples.tsv";
 
 string GetModelFilePath() {
-  return "third_party/tensorflow/lite/models/testdata/"
-         "smartreply_ondevice_model.bin";
+  return "external/tflite_smartreply/smartreply.tflite";  // NOLINT
 }
 
 string GetSamplesFilePath() {
-- 
GitLab


From 558974db23ee0ea8845ba4b0b7fa94f70408ac1b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 14 Jan 2019 09:48:57 -0800
Subject: [PATCH 0676/1095] Increase timeout for for a python kernel_test.

There have been several timeouts with //third_party/tensorflow/python/kernel_tests:bias_op_test_xla

PiperOrigin-RevId: 229199505
---
 tensorflow/python/kernel_tests/BUILD | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index d9ea42374d..7a514875d6 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -1514,7 +1514,7 @@ cuda_py_test(
 
 cuda_py_test(
     name = "bias_op_test",
-    size = "small",
+    size = "medium",
     srcs = ["bias_op_test.py"],
     additional_deps = [
         "//third_party/py/numpy",
-- 
GitLab


From 72678f0315963cbc19c371f90f3605149fece063 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 14 Jan 2019 10:29:25 -0800
Subject: [PATCH 0677/1095] Remove unused methods. Those are obsolete now,
 after the recent refactoring efforts.

PiperOrigin-RevId: 229207631
---
 tensorflow/lite/delegates/flex/buffer_map.cc  | 14 -----
 tensorflow/lite/delegates/flex/buffer_map.h   |  6 ---
 .../lite/delegates/flex/buffer_map_test.cc    | 51 -------------------
 3 files changed, 71 deletions(-)

diff --git a/tensorflow/lite/delegates/flex/buffer_map.cc b/tensorflow/lite/delegates/flex/buffer_map.cc
index 9408efaaa1..0d0c953636 100644
--- a/tensorflow/lite/delegates/flex/buffer_map.cc
+++ b/tensorflow/lite/delegates/flex/buffer_map.cc
@@ -173,19 +173,5 @@ void BufferMap::SetFromTensorFlow(int tensor_index, tensorflow::Tensor tensor) {
   owned_by_tf_.insert(tensor_index);
 }
 
-void BufferMap::RemoveTensor(int tensor_index) {
-  id_to_tensor_.erase(tensor_index);
-}
-
-void BufferMap::RemoveTensorsNotInSet(const std::set<int>& keep) {
-  for (auto it = id_to_tensor_.begin(); it != id_to_tensor_.end();) {
-    if (keep.count(it->first) == 0 && IsTensorFlowTensor(it->first)) {
-      it = id_to_tensor_.erase(it);
-    } else {
-      ++it;
-    }
-  }
-}
-
 }  // namespace flex
 }  // namespace tflite
diff --git a/tensorflow/lite/delegates/flex/buffer_map.h b/tensorflow/lite/delegates/flex/buffer_map.h
index d4724a011d..45bc571f92 100644
--- a/tensorflow/lite/delegates/flex/buffer_map.h
+++ b/tensorflow/lite/delegates/flex/buffer_map.h
@@ -52,12 +52,6 @@ class BufferMap {
   // shallow copy.
   void SetFromTensorFlow(int tensor_index, tensorflow::Tensor tensor);
 
-  // Removes any association of a tensor and the given 'tensor_index'.
-  void RemoveTensor(int tensor_index);
-
-  // Removes all *TF* tensors that are not contained in the given 'keep' set.
-  void RemoveTensorsNotInSet(const std::set<int>& keep);
-
   // Same as above but creates a new tensorflow::Tensor with a copy of the
   // given TfLiteTensor's data.
   void SetFromTfLite(int tensor_index, const TfLiteTensor* tensor);
diff --git a/tensorflow/lite/delegates/flex/buffer_map_test.cc b/tensorflow/lite/delegates/flex/buffer_map_test.cc
index 7b4acbd69d..8edeb280eb 100644
--- a/tensorflow/lite/delegates/flex/buffer_map_test.cc
+++ b/tensorflow/lite/delegates/flex/buffer_map_test.cc
@@ -222,57 +222,6 @@ TEST(BufferMapTest, TensorFlowOverwritesTfLite) {
               ElementsAre(0, 0, 0, 0.123f, 0, 0));
 }
 
-TEST(BufferMapTest, RemoveTensorFlowTensor) {
-  tensorflow::Tensor t1 =
-      MakeTensor<float>({1, 2, 1, 3}, {0, 0, 0, 0.123f, 0, 0});
-  BufferMap buffer_map;
-  buffer_map.SetFromTensorFlow(0, t1);
-  EXPECT_TRUE(buffer_map.HasTensor(0));
-  buffer_map.RemoveTensor(0);
-  EXPECT_FALSE(buffer_map.HasTensor(0));
-}
-
-TEST(BufferMapTest, RemoveTfLiteTensor) {
-  UniqueTfLiteTensor t2 =
-      MakeLiteTensor<int>({1, 2, 4}, {0, 0, 0, 3, 0, 0, 1, 2});
-  BufferMap buffer_map;
-  buffer_map.SetFromTfLite(0, t2.get());
-  EXPECT_TRUE(buffer_map.HasTensor(0));
-  buffer_map.RemoveTensor(0);
-  EXPECT_FALSE(buffer_map.HasTensor(0));
-}
-
-TEST(BufferMapTest, RemoveTensorsInSet) {
-  tensorflow::Tensor t1 =
-      MakeTensor<float>({1, 2, 1, 3}, {0, 0, 0, 0.123f, 0, 0});
-  UniqueTfLiteTensor t2 =
-      MakeLiteTensor<int>({1, 2, 4}, {0, 0, 0, 3, 0, 0, 1, 2});
-  BufferMap buffer_map;
-
-  auto list_existing_tensors = [&buffer_map]() {
-    std::vector<int> result;
-    for (int i : {0, 1, 2, 3}) {
-      if (buffer_map.HasTensor(i)) {
-        result.push_back(i);
-      }
-    }
-    return result;
-  };
-
-  buffer_map.SetFromTensorFlow(0, t1);
-  buffer_map.SetFromTfLite(1, t2.get());
-  buffer_map.SetFromTfLite(2, t2.get());
-  buffer_map.SetFromTensorFlow(3, t1);
-
-  EXPECT_THAT(list_existing_tensors(), ElementsAre(0, 1, 2, 3));
-
-  buffer_map.RemoveTensorsNotInSet({0, 3});
-  EXPECT_THAT(list_existing_tensors(), ElementsAre(0, 1, 2, 3));
-
-  buffer_map.RemoveTensorsNotInSet({});
-  EXPECT_THAT(list_existing_tensors(), ElementsAre(1, 2));
-}
-
 TEST(BufferMapTest, Forwardable) {
   BufferMap buffer_map;
   EXPECT_FALSE(buffer_map.IsForwardable(0));
-- 
GitLab


From 4f2400fd319f7f7d12d90b08cecd6df9116524c6 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Mon, 14 Jan 2019 18:35:31 +0000
Subject: [PATCH 0678/1095] Update kinesis dataset to match v2 API

Since DatasetSource now requires varient_tensor to be passed into constructor,
the kinesis data will need to be updated to fix the incompatibility issue.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/contrib/kinesis/python/ops/kinesis_dataset_ops.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/kinesis/python/ops/kinesis_dataset_ops.py b/tensorflow/contrib/kinesis/python/ops/kinesis_dataset_ops.py
index 2b1d478a9b..9479afb180 100644
--- a/tensorflow/contrib/kinesis/python/ops/kinesis_dataset_ops.py
+++ b/tensorflow/contrib/kinesis/python/ops/kinesis_dataset_ops.py
@@ -71,7 +71,6 @@ class KinesisDataset(dataset_ops.DatasetSource):
       interval: The interval for the Kinesis Client to wait before
         it tries to get records again (in millisecond).
     """
-    super(KinesisDataset, self).__init__()
     self._stream = ops.convert_to_tensor(
         stream, dtype=dtypes.string, name="stream")
     self._shard = ops.convert_to_tensor(
@@ -80,6 +79,7 @@ class KinesisDataset(dataset_ops.DatasetSource):
         read_indefinitely, dtype=dtypes.bool, name="read_indefinitely")
     self._interval = ops.convert_to_tensor(
         interval, dtype=dtypes.int64, name="interval")
+    super(KinesisDataset, self).__init__(self._as_variant_tensor())
 
   def _as_variant_tensor(self):
     return gen_dataset_ops.kinesis_dataset(
-- 
GitLab


From 959494c31ed3a0eb84af9057eaa34c2ed4c0799c Mon Sep 17 00:00:00 2001
From: Guangda Lai <31743510+aaroey@users.noreply.github.com>
Date: Mon, 14 Jan 2019 11:15:54 -0800
Subject: [PATCH 0679/1095] fix CI build failures

---
 tensorflow/contrib/tensorrt/test/int32_test.py | 7 +++----
 tensorflow/contrib/tensorrt/test/topk_test.py  | 8 ++++----
 2 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/test/int32_test.py b/tensorflow/contrib/tensorrt/test/int32_test.py
index 50af816d8e..181229c8a7 100644
--- a/tensorflow/contrib/tensorrt/test/int32_test.py
+++ b/tensorflow/contrib/tensorrt/test/int32_test.py
@@ -25,7 +25,6 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import gen_array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn
 from tensorflow.python.platform import test
@@ -44,8 +43,7 @@ class ExcludeUnsupportedInt32Test(trt_test.TfTrtIntegrationTestBase):
     dtype = dtypes.int32
     g = ops.Graph()
     with g.as_default():
-      x = array_ops.placeholder(
-          dtype=dtype, shape=input_dims, name=input_name)
+      x = array_ops.placeholder(dtype=dtype, shape=input_dims, name=input_name)
       b = self._ConstOp((4, 10), dtype)
       x = math_ops.matmul(x, b)
       b = self._ConstOp((10,), dtype)
@@ -79,5 +77,6 @@ class ExcludeUnsupportedInt32Test(trt_test.TfTrtIntegrationTestBase):
     # mode, which is a bug. Re-enable this when trt library is fixed.
     return not trt_test.IsQuantizationMode(run_params.precision_mode)
 
-if __name__ == "__main__":
+
+if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/contrib/tensorrt/test/topk_test.py b/tensorflow/contrib/tensorrt/test/topk_test.py
index efbfb26898..263ea0cfa1 100644
--- a/tensorflow/contrib/tensorrt/test/topk_test.py
+++ b/tensorflow/contrib/tensorrt/test/topk_test.py
@@ -40,18 +40,18 @@ class TopKTest(trt_test.TfTrtIntegrationTestBase):
       x = array_ops.placeholder(dtype=dtype, shape=input_dims, name=input_name)
       k_tensor = constant_op.constant(k, dtype=tf.int32, name="Const")
       values, indices = nn_ops.top_k(x, k_tensor, name="TopK")
-      values = array_ops.identity(values, name='output_values')
-      indices = array_ops.identity(indices, name='output_indices')
+      values = array_ops.identity(values, name="output_values")
+      indices = array_ops.identity(indices, name="output_indices")
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
         input_dims=[input_dims],
-        output_names=['output_values', 'output_indices'],
+        output_names=["output_values", "output_indices"],
         expected_output_dims=[(100, k), (100, k)])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
-    return {"TRTEngineOp_0" : ["Const", "TopK"]}
+    return {"TRTEngineOp_0": ["Const", "TopK"]}
 
 
 if __name__ == "__main__":
-- 
GitLab


From 5bb89f16124ebbcff38a7bd6ec404cfa54ec9b91 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 14 Jan 2019 11:03:56 -0800
Subject: [PATCH 0680/1095] Cleaning up copying system files to execroot. Copy
 and use nvcc from execroot.

PiperOrigin-RevId: 229214994
---
 third_party/gpus/cuda/BUILD.tpl             |   2 +-
 third_party/gpus/cuda/BUILD.windows.tpl     |   2 +-
 third_party/gpus/cuda_configure.bzl         | 206 ++++++++------------
 third_party/gpus/rocm/BUILD.tpl             |   2 +-
 third_party/gpus/rocm_configure.bzl         | 131 ++++---------
 third_party/tensorrt/BUILD.tpl              |   2 +-
 third_party/tensorrt/tensorrt_configure.bzl |  45 +++--
 7 files changed, 157 insertions(+), 233 deletions(-)

diff --git a/third_party/gpus/cuda/BUILD.tpl b/third_party/gpus/cuda/BUILD.tpl
index f6b497f813..1921ef7c1f 100644
--- a/third_party/gpus/cuda/BUILD.tpl
+++ b/third_party/gpus/cuda/BUILD.tpl
@@ -202,4 +202,4 @@ cc_library(
     visibility = ["//visibility:public"],
 )
 
-%{cuda_include_genrules}
+%{copy_rules}
diff --git a/third_party/gpus/cuda/BUILD.windows.tpl b/third_party/gpus/cuda/BUILD.windows.tpl
index 325d18b9cb..3ed4fd415c 100644
--- a/third_party/gpus/cuda/BUILD.windows.tpl
+++ b/third_party/gpus/cuda/BUILD.windows.tpl
@@ -161,4 +161,4 @@ cc_library(
     visibility = ["//visibility:public"],
 )
 
-%{cuda_include_genrules}
+%{copy_rules}
diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl
index 8de4fe58e5..af8876ba17 100644
--- a/third_party/gpus/cuda_configure.bzl
+++ b/third_party/gpus/cuda_configure.bzl
@@ -1058,7 +1058,7 @@ def _create_dummy_repository(repository_ctx):
               _lib_name("curand", cpu_value),
           "%{cupti_lib}":
               _lib_name("cupti", cpu_value),
-          "%{cuda_include_genrules}":
+          "%{copy_rules}":
               "",
           "%{cuda_headers}":
               "",
@@ -1148,65 +1148,37 @@ def _norm_path(path):
     path = path[:-1]
   return path
 
-
-def symlink_genrule_for_dir(
-        repository_ctx,
-        src_dir,
-        dest_dir,
-        genrule_name,
-        src_files = [],
-        dest_files = []):
-  """Returns a genrule to symlink(or copy if on Windows) a set of files.
-
-    If src_dir is passed, files will be read from the given directory; otherwise
-    we assume files are in src_files and dest_files
-    """
-  if src_dir != None:
-    src_dir = _norm_path(src_dir)
-    dest_dir = _norm_path(dest_dir)
-    files = "\n".join(sorted(_read_dir(repository_ctx, src_dir).splitlines()))
-
-    # Create a list with the src_dir stripped to use for outputs.
-    dest_files = files.replace(src_dir, "").splitlines()
-    src_files = files.splitlines()
-  command = []
-  if not _is_windows(repository_ctx):
-    # We clear folders that might have been generated previously to avoid
-    # undesired inclusions
-    command.append('if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi')
-    command.append('if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi')
-    command.append('if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi')
-    command.append('if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi')
-  outs = []
-  for i in range(len(dest_files)):
-    if dest_files[i] != "":
-      # If we have only one file to link we do not want to use the dest_dir, as
-      # $(@D) will include the full path to the file.
-      dest = "$(@D)/" + dest_dir + dest_files[i] if len(
-          dest_files) != 1 else "$(@D)/" + dest_files[i]
-
-      # Copy the headers to create a sandboxable setup.
-      cmd = "cp -f"
-      command.append(cmd + ' "%s" "%s"' % (src_files[i], dest))
-      outs.append('        "' + dest_dir + dest_files[i] + '",')
-  genrule = _genrule(
-      src_dir,
-      genrule_name,
-      " && ".join(command),
-      "\n".join(outs),
-  )
-  return genrule
-
-
-def _genrule(src_dir, genrule_name, command, outs):
-  """Returns a string with a genrule.
-
-    Genrule executes the given command and produces the given outputs.
-    """
-  return (
-      "genrule(\n" + '    name = "' + genrule_name + '",\n' + "    outs = [\n" +
-      outs + "\n    ],\n" + '    cmd = """\n' + command + '\n   """,\n' + ")\n")
-
+def make_copy_files_rule(repository_ctx, name, srcs, outs):
+  """Returns a rule to copy a set of files."""
+  cmds = []
+  # Copy files.
+  for src, out in zip(srcs, outs):
+    cmds.append('cp -f "%s" $(location %s)' % (src, out))
+  outs = [('        "%s",' % out) for out in outs]
+  return """genrule(
+    name = "%s",
+    outs = [
+%s
+    ],
+    cmd = \"""%s \""",
+)""" % (name, "\n".join(outs), " && ".join(cmds))
+
+def make_copy_dir_rule(repository_ctx, name, src_dir, out_dir):
+  """Returns a rule to recursively copy a directory."""
+  src_dir = _norm_path(src_dir)
+  out_dir = _norm_path(out_dir)
+  outs = _read_dir(repository_ctx, src_dir)
+  outs = [('        "%s",' % out.replace(src_dir, out_dir)) for out in outs]
+  # '@D' already contains the relative path for a single file, see
+  # http://docs.bazel.build/versions/master/be/make-variables.html#predefined_genrule_variables
+  out_dir = "$(@D)/%s" % out_dir if len(outs) > 1 else "$(@D)"
+  return """genrule(
+    name = "%s",
+    outs = [
+%s
+    ],
+    cmd = \"""cp -rf "%s/." "%s/" \""",
+)""" % (name, "\n".join(outs), src_dir, out_dir)
 
 def _read_dir(repository_ctx, src_dir):
   """Returns a string with all files in a directory.
@@ -1233,7 +1205,7 @@ def _read_dir(repository_ctx, src_dir):
         empty_stdout_fine=True,
     )
     result = find_result.stdout
-  return result
+  return sorted(result.splitlines())
 
 
 def _flag_enabled(repository_ctx, flag_name):
@@ -1272,69 +1244,59 @@ def _create_local_cuda_repository(repository_ctx):
   cupti_header_dir = _find_cupti_header_dir(repository_ctx, cuda_config)
   nvvm_libdevice_dir = _find_nvvm_libdevice_dir(repository_ctx, cuda_config)
 
-  # Set up symbolic links for the cuda toolkit by creating genrules to do
-  # symlinking. We create one genrule for each directory we want to track under
-  # cuda_toolkit_path
-  cuda_toolkit_path = cuda_config.cuda_toolkit_path
-  genrules = [
-      symlink_genrule_for_dir(
+  # Create genrule to copy files from the installed CUDA toolkit into execroot.
+  copy_rules = [
+      make_copy_dir_rule(
           repository_ctx,
-          cuda_include_path,
-          "cuda/include",
-          "cuda-include",
-      )
-  ]
-  genrules.append(
-      symlink_genrule_for_dir(
+          name = "cuda-include",
+          src_dir = cuda_include_path,
+          out_dir = "cuda/include",
+      ),
+      make_copy_dir_rule(
           repository_ctx,
-          nvvm_libdevice_dir,
-          "cuda/nvvm/libdevice",
-          "cuda-nvvm",
-      ))
-  genrules.append(
-      symlink_genrule_for_dir(
+          name = "cuda-nvvm",
+          src_dir = nvvm_libdevice_dir,
+          out_dir = "cuda/nvvm/libdevice",
+      ),
+      make_copy_dir_rule(
           repository_ctx,
-          cupti_header_dir,
-          "cuda/extras/CUPTI/include",
-          "cuda-extras",
-      ))
+          name = "cuda-extras",
+          src_dir = cupti_header_dir,
+          out_dir = "cuda/extras/CUPTI/include",
+      ),
+  ]
 
   cuda_libs = _find_libs(repository_ctx, cuda_config)
-  cuda_lib_src = []
-  cuda_lib_dest = []
+  cuda_lib_srcs = []
+  cuda_lib_outs = []
   for lib in cuda_libs.values():
-    cuda_lib_src.append(lib.path)
-    cuda_lib_dest.append("cuda/lib/" + lib.file_name)
-  genrules.append(
-      symlink_genrule_for_dir(
-          repository_ctx,
-          None,
-          "",
-          "cuda-lib",
-          cuda_lib_src,
-          cuda_lib_dest,
-      ))
+    cuda_lib_srcs.append(lib.path)
+    cuda_lib_outs.append("cuda/lib/" + lib.file_name)
+  copy_rules.append(make_copy_files_rule(
+      repository_ctx,
+      name = "cuda-lib",
+      srcs = cuda_lib_srcs,
+      outs = cuda_lib_outs,
+  ))
+
+  copy_rules.append(make_copy_dir_rule(
+      repository_ctx,
+      name = "cuda-bin",
+      src_dir = cuda_config.cuda_toolkit_path + "/bin",
+      out_dir = "cuda/bin"
+  ))
 
-  # Set up the symbolic links for cudnn if cndnn was not installed to
-  # CUDA_TOOLKIT_PATH.
-  included_files = _read_dir(repository_ctx, cuda_include_path).replace(
-      cuda_include_path,
-      "",
-  ).splitlines()
-  if "/cudnn.h" not in included_files:
-    genrules.append(
-        symlink_genrule_for_dir(
-            repository_ctx,
-            None,
-            "cuda/include/",
-            "cudnn-include",
-            [cudnn_header_dir + "/cudnn.h"],
-            ["cudnn.h"],
-        ))
+  # Copy cudnn.h if cuDNN was not installed to CUDA_TOOLKIT_PATH.
+  included_files = _read_dir(repository_ctx, cuda_include_path)
+  if not any([file.endswith("cudnn.h") for file in included_files]):
+    copy_rules.append(make_copy_files_rule(
+        repository_ctx,
+        name = "cudnn-include",
+        srcs = [cudnn_header_dir + "/cudnn.h"],
+        outs = ["cuda/include/cudnn.h"],
+    ))
   else:
-    genrules.append(
-        "filegroup(\n" + '    name = "cudnn-include",\n' + "    srcs = [],\n" +
-        ")\n",)
+    copy_rules.append("filegroup(name = 'cudnn-include')\n")
 
   # Set up BUILD file for cuda/
   _tpl(
@@ -1374,8 +1336,8 @@ def _create_local_cuda_repository(repository_ctx):
               cuda_libs["curand"].file_name,
           "%{cupti_lib}":
               cuda_libs["cupti"].file_name,
-          "%{cuda_include_genrules}":
-              "\n".join(genrules),
+          "%{copy_rules}":
+              "\n".join(copy_rules),
           "%{cuda_headers}": ('":cuda-include",\n' + '        ":cudnn-include",'
                              ),
       },
@@ -1447,11 +1409,9 @@ def _create_local_cuda_repository(repository_ctx):
     # bazel's header check failing.
     cuda_defines["%{extra_no_canonical_prefixes_flags}"] = (
         "flag: \"-fno-canonical-system-headers\"")
-    nvcc_path = str(
-        repository_ctx.path("%s/bin/nvcc%s" % (
-            cuda_config.cuda_toolkit_path,
-            ".exe" if _is_windows(repository_ctx) else "",
-        )))
+    nvcc_path = "cuda/bin/nvcc"
+    if _is_windows(repository_ctx):
+      nvcc_path += ".exe"
     _tpl(
         repository_ctx,
         "crosstool:BUILD",
diff --git a/third_party/gpus/rocm/BUILD.tpl b/third_party/gpus/rocm/BUILD.tpl
index 8258bb3589..655acaec97 100644
--- a/third_party/gpus/rocm/BUILD.tpl
+++ b/third_party/gpus/rocm/BUILD.tpl
@@ -96,4 +96,4 @@ cc_library(
     ],
 )
 
-%{rocm_include_genrules}
+%{copy_rules}
diff --git a/third_party/gpus/rocm_configure.bzl b/third_party/gpus/rocm_configure.bzl
index 6df6799bd7..a57ec9f7a6 100644
--- a/third_party/gpus/rocm_configure.bzl
+++ b/third_party/gpus/rocm_configure.bzl
@@ -14,6 +14,12 @@
     `gfx803,gfx900`.
 """
 
+load(
+    ":cuda_configure.bzl",
+    "make_copy_dir_rule",
+    "make_copy_files_rule",
+)
+
 _GCC_HOST_COMPILER_PATH = "GCC_HOST_COMPILER_PATH"
 _ROCM_TOOLKIT_PATH = "ROCM_TOOLKIT_PATH"
 _TF_ROCM_VERSION = "TF_ROCM_VERSION"
@@ -445,7 +451,7 @@ def _create_dummy_repository(repository_ctx):
             "%{miopen_lib}": _lib_name("miopen", cpu_value),
             "%{rocfft_lib}": _lib_name("rocfft", cpu_value),
             "%{hiprand_lib}": _lib_name("hiprand", cpu_value),
-            "%{rocm_include_genrules}": "",
+            "%{copy_rules}": "",
             "%{rocm_headers}": "",
         },
     )
@@ -510,51 +516,6 @@ def _norm_path(path):
         path = path[:-1]
     return path
 
-def _symlink_genrule_for_dir(
-        repository_ctx,
-        src_dir,
-        dest_dir,
-        genrule_name,
-        src_files = [],
-        dest_files = []):
-    """Returns a genrule to symlink(or copy if on Windows) a set of files.
-
-    If src_dir is passed, files will be read from the given directory; otherwise
-    we assume files are in src_files and dest_files
-    """
-    if src_dir != None:
-        src_dir = _norm_path(src_dir)
-        dest_dir = _norm_path(dest_dir)
-        files = _read_dir(repository_ctx, src_dir)
-
-        # Create a list with the src_dir stripped to use for outputs.
-        dest_files = files.replace(src_dir, "").splitlines()
-        src_files = files.splitlines()
-    command = []
-
-    # We clear folders that might have been generated previously to avoid
-    # undesired inclusions
-    command.append('if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi')
-    command.append('if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi')
-    outs = []
-    for i in range(len(dest_files)):
-        if dest_files[i] != "":
-            # If we have only one file to link we do not want to use the dest_dir, as
-            # $(@D) will include the full path to the file.
-            dest = "$(@D)/" + dest_dir + dest_files[i] if len(dest_files) != 1 else "$(@D)/" + dest_files[i]
-
-            # On Windows, symlink is not supported, so we just copy all the files.
-            cmd = "ln -s"
-            command.append(cmd + ' "%s" "%s"' % (src_files[i], dest))
-            outs.append('        "' + dest_dir + dest_files[i] + '",')
-    genrule = _genrule(
-        src_dir,
-        genrule_name,
-        " && ".join(command),
-        "\n".join(outs),
-    )
-    return genrule
-
 def _genrule(src_dir, genrule_name, command, outs):
     """Returns a string with a genrule.
 
@@ -601,55 +562,49 @@ def _create_local_rocm_repository(repository_ctx):
     """Creates the repository containing files set up to build with ROCm."""
     rocm_config = _get_rocm_config(repository_ctx)
 
-    # Set up symbolic links for the rocm toolkit by creating genrules to do
-    # symlinking. We create one genrule for each directory we want to track under
+    # Copy header and library files to execroot.
     # rocm_toolkit_path
     rocm_toolkit_path = rocm_config.rocm_toolkit_path
-    rocm_include_path = rocm_toolkit_path + "/include"
-    genrules = [_symlink_genrule_for_dir(
-        repository_ctx,
-        rocm_include_path,
-        "rocm/include",
-        "rocm-include",
-    )]
-    genrules.append(_symlink_genrule_for_dir(
-        repository_ctx,
-        rocm_toolkit_path + "/rocfft/include",
-        "rocm/include/rocfft",
-        "rocfft-include",
-    ))
-    genrules.append(_symlink_genrule_for_dir(
-        repository_ctx,
-        rocm_toolkit_path + "/rocblas/include",
-        "rocm/include/rocblas",
-        "rocblas-include",
-    ))
-    genrules.append(_symlink_genrule_for_dir(
-        repository_ctx,
-        rocm_toolkit_path + "/miopen/include",
-        "rocm/include/miopen",
-        "miopen-include",
-    ))
+    copy_rules = [
+        make_copy_dir_rule(
+            repository_ctx,
+            name = "rocm-include",
+            src_dir = rocm_toolkit_path + "/include",
+            out_dir = "rocm/include",
+        ),
+        make_copy_dir_rule(
+            repository_ctx,
+            name = "rocfft-include",
+            src_dir = rocm_toolkit_path + "/rocfft/include",
+            out_dir = "rocm/include/rocfft",
+        ),
+        make_copy_dir_rule(
+            repository_ctx,
+            name = "rocm-blas",
+            src_dir = rocm_toolkit_path + "/rocblas/include",
+            out_dir = "rocm/include/rocblas",
+        ),
+        make_copy_dir_rule(
+            repository_ctx,
+            name = "rocm-miopen",
+            src_dir = rocm_toolkit_path + "/miopen/include",
+            out_dir = "rocm/include/miopen",
+        ),
+    ]
 
     rocm_libs = _find_libs(repository_ctx, rocm_config)
-    rocm_lib_src = []
-    rocm_lib_dest = []
+    rocm_lib_srcs = []
+    rocm_lib_outs = []
     for lib in rocm_libs.values():
-        rocm_lib_src.append(lib.path)
-        rocm_lib_dest.append("rocm/lib/" + lib.file_name)
-    genrules.append(_symlink_genrule_for_dir(
+        rocm_lib_srcs.append(lib.path)
+        rocm_lib_outs.append("rocm/lib/" + lib.file_name)
+    copy_rules.append(make_copy_files_rule(
         repository_ctx,
-        None,
-        "",
-        "rocm-lib",
-        rocm_lib_src,
-        rocm_lib_dest,
+        name = "rocm-lib",
+        srcs = rocm_lib_srcs,
+        outs = rocm_lib_outs,
     ))
 
-    included_files = _read_dir(repository_ctx, rocm_include_path).replace(
-        rocm_include_path,
-        "",
-    ).splitlines()
 
     # Set up BUILD file for rocm/
     _tpl(
@@ -672,7 +627,7 @@ def _create_local_rocm_repository(repository_ctx):
             "%{rocfft_lib}": rocm_libs["rocfft"].file_name,
             "%{hiprand_lib}": rocm_libs["hiprand"].file_name,
             "%{miopen_lib}": rocm_libs["miopen"].file_name,
-            "%{rocm_include_genrules}": "\n".join(genrules),
+            "%{copy_rules}": "\n".join(copy_rules),
             "%{rocm_headers}": ('":rocm-include",\n' +
                                 '":rocfft-include",\n' +
                                 '":rocblas-include",\n' +
diff --git a/third_party/tensorrt/BUILD.tpl b/third_party/tensorrt/BUILD.tpl
index a2c30b8b94..17ab2fa119 100644
--- a/third_party/tensorrt/BUILD.tpl
+++ b/third_party/tensorrt/BUILD.tpl
@@ -29,5 +29,5 @@ cc_library(
 )
 
 
-%{tensorrt_genrules}
+%{copy_rules}
 
diff --git a/third_party/tensorrt/tensorrt_configure.bzl b/third_party/tensorrt/tensorrt_configure.bzl
index 77ee6622d1..d1e116d9dd 100644
--- a/third_party/tensorrt/tensorrt_configure.bzl
+++ b/third_party/tensorrt/tensorrt_configure.bzl
@@ -13,7 +13,8 @@ load(
     "get_cpu_value",
     "find_cuda_define",
     "matches_version",
-    "symlink_genrule_for_dir",
+    "make_copy_dir_rule",
+    "make_copy_files_rule",
 )
 
 _TENSORRT_INSTALL_PATH = "TENSORRT_INSTALL_PATH"
@@ -177,35 +178,43 @@ def _tensorrt_configure_impl(repository_ctx):
     auto_configure_fail(
         "Cannot find TensorRT install path %s." % trt_install_path)
 
-  # Set up the symbolic links for the library files.
+  # Copy the library files.
   trt_lib_version = _trt_lib_version(repository_ctx, trt_install_path)
   trt_libs = _find_trt_libs(repository_ctx, trt_install_path, trt_lib_version)
-  trt_lib_src = []
-  trt_lib_dest = []
+  trt_lib_srcs = []
+  trt_lib_outs = []
   for lib in trt_libs.values():
-    trt_lib_src.append(lib.src_file_path)
-    trt_lib_dest.append(lib.dst_file_name)
-  genrules = [
-      symlink_genrule_for_dir(repository_ctx, None, "tensorrt/lib/",
-                              "tensorrt_lib", trt_lib_src, trt_lib_dest)
-  ]
-
-  # Set up the symbolic links for the header files.
+    trt_lib_srcs.append(lib.src_file_path)
+    trt_lib_outs.append("tensorrt/lib/" + lib.dst_file_name)
+  copy_rules = [make_copy_files_rule(
+      repository_ctx,
+      name = "tensorrt_lib",
+      srcs = trt_lib_srcs,
+      outs = trt_lib_outs,
+  )]
+
+  # Copy the header files header files.
   trt_header_dir = _find_trt_header_dir(repository_ctx, trt_install_path)
-  src_files = [
+  trt_header_srcs = [
       "%s/%s" % (trt_header_dir, header) for header in _TF_TENSORRT_HEADERS
   ]
-  dest_files = _TF_TENSORRT_HEADERS
-  genrules.append(
-      symlink_genrule_for_dir(repository_ctx, None, "tensorrt/include/",
-                              "tensorrt_include", src_files, dest_files))
+  trt_header_outs = [
+      "tensorrt/include/" + header for header in _TF_TENSORRT_HEADERS
+  ]
+  copy_rules.append(
+      make_copy_files_rule(
+          repository_ctx,
+          name = "tensorrt_include",
+          srcs = trt_header_srcs,
+          outs = trt_header_outs,
+  ))
 
   # Set up config file.
   _tpl(repository_ctx, "build_defs.bzl", {"%{tensorrt_is_configured}": "True"})
 
   # Set up BUILD file.
   substitutions = {
-      "%{tensorrt_genrules}": "\n".join(genrules),
+      "%{copy_rules}": "\n".join(copy_rules),
       "%{tensorrt_headers}": '":tensorrt_include"',
   }
   for lib in _TF_TENSORRT_LIBS:
-- 
GitLab


From adf8dac492adb33d4c71a4271f6f003a156eae22 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 14 Jan 2019 11:18:16 -0800
Subject: [PATCH 0681/1095] Implicitly initialize platforms in
 PlatformUtil::GetSupportedPlatforms.

PiperOrigin-RevId: 229218011
---
 tensorflow/compiler/xla/service/platform_util.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/compiler/xla/service/platform_util.cc b/tensorflow/compiler/xla/service/platform_util.cc
index 0491f641fc..886a054562 100644
--- a/tensorflow/compiler/xla/service/platform_util.cc
+++ b/tensorflow/compiler/xla/service/platform_util.cc
@@ -70,6 +70,9 @@ PlatformUtil::GetSupportedPlatforms() {
   for (se::Platform* platform : all_platforms) {
     auto compiler_status = Compiler::GetForPlatform(platform);
     if (compiler_status.ok()) {
+      if (!platform->Initialized()) {
+        TF_RETURN_IF_ERROR(platform->Initialize({}));
+      }
       platforms.push_back(platform);
     } else {
       LOG(INFO) << "platform " << platform->Name() << " present but no "
-- 
GitLab


From 2d58c2502f8971bc9af65c4f8270512f7c53fd2a Mon Sep 17 00:00:00 2001
From: Thomas O'Malley <omalleyt@google.com>
Date: Mon, 14 Jan 2019 11:19:51 -0800
Subject: [PATCH 0682/1095] Modify flaky topology_test

PiperOrigin-RevId: 229218301
---
 tensorflow/python/keras/engine/topology_test.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/keras/engine/topology_test.py b/tensorflow/python/keras/engine/topology_test.py
index 08d21193a0..9022dc9481 100644
--- a/tensorflow/python/keras/engine/topology_test.py
+++ b/tensorflow/python/keras/engine/topology_test.py
@@ -1228,8 +1228,12 @@ class NestedNetworkTest(test.TestCase):
     result = self.evaluate(result_tensor)
     self.assertAllEqual(result, [[2.]])
 
-    output_shape = network.compute_output_shape({'x1': (10, 1), 'x2': (10, 1)})
-    self.assertListEqual(output_shape.as_list(), [10, 1])
+    # TODO(b/122726584): Investigate why concrete batch is flaky in some builds.
+    output_shape = network.compute_output_shape({
+        'x1': (None, 1),
+        'x2': (None, 1)
+    })
+    self.assertListEqual(output_shape.as_list(), [None, 1])
 
   def test_nested_outputs_network(self):
     inputs = keras.Input(shape=(1,))
-- 
GitLab


From f96ef427440f7fe15cb2b493a8de25202fa96cc5 Mon Sep 17 00:00:00 2001
From: Yu-Cheng Ling <ycling@google.com>
Date: Mon, 14 Jan 2019 11:20:04 -0800
Subject: [PATCH 0683/1095] Toco FuseBinaryIntoFollowingAffine bugfix

A binary op can't be fused into a following affine op if
the output is consumed by multiple ops.

PiperOrigin-RevId: 229218339
---
 .../fuse_binary_into_following_affine.cc      |  10 +-
 .../toco/graph_transformations/tests/BUILD    |  12 ++
 .../fuse_binary_into_following_affine_test.cc | 148 ++++++++++++++++++
 3 files changed, 167 insertions(+), 3 deletions(-)
 create mode 100644 tensorflow/lite/toco/graph_transformations/tests/fuse_binary_into_following_affine_test.cc

diff --git a/tensorflow/lite/toco/graph_transformations/fuse_binary_into_following_affine.cc b/tensorflow/lite/toco/graph_transformations/fuse_binary_into_following_affine.cc
index 436b639253..9ea8d8fa5b 100644
--- a/tensorflow/lite/toco/graph_transformations/fuse_binary_into_following_affine.cc
+++ b/tensorflow/lite/toco/graph_transformations/fuse_binary_into_following_affine.cc
@@ -218,6 +218,12 @@ void FuseMulOrDivParamsIntoFollowingAffine(Model* model, Operator* following_op,
     return ::tensorflow::Status::OK();
   }
 
+  if (CountOpsWithInput(*model, binary_op->outputs[0]) != 1) {
+    AddMessageF("Not fusing %s because it's consumed by multiple ops",
+                LogName(*binary_op));
+    return ::tensorflow::Status::OK();
+  }
+
   Operator* following_op = GetOpWithInput(*model, binary_op->outputs[0]);
 
   if (!following_op) {
@@ -287,9 +293,7 @@ void FuseMulOrDivParamsIntoFollowingAffine(Model* model, Operator* following_op,
   AddMessageF("Fusing %s into the following %s", LogName(*binary_op),
               LogName(*following_op));
 
-  if (CountOpsWithInput(*model, binary_op->outputs[0]) == 1) {
-    model->EraseArray(binary_op->outputs[0]);
-  }
+  model->EraseArray(binary_op->outputs[0]);
 
   following_op->inputs[0] = binary_op->inputs[index_of_variable_input];
   const auto& old_constant_param_name =
diff --git a/tensorflow/lite/toco/graph_transformations/tests/BUILD b/tensorflow/lite/toco/graph_transformations/tests/BUILD
index bbbedbe3a9..03d331226d 100644
--- a/tensorflow/lite/toco/graph_transformations/tests/BUILD
+++ b/tensorflow/lite/toco/graph_transformations/tests/BUILD
@@ -40,3 +40,15 @@ tf_cc_test(
         "@com_google_googletest//:gtest_main",
     ],
 )
+
+tf_cc_test(
+    name = "fuse_binary_into_following_affine_test",
+    srcs = ["fuse_binary_into_following_affine_test.cc"],
+    deps = [
+        "//tensorflow/lite/toco:graph_transformations",
+        "//tensorflow/lite/toco:model",
+        "//tensorflow/lite/toco:tooling_util",
+        "@com_google_absl//absl/memory",
+        "@com_google_googletest//:gtest_main",
+    ],
+)
diff --git a/tensorflow/lite/toco/graph_transformations/tests/fuse_binary_into_following_affine_test.cc b/tensorflow/lite/toco/graph_transformations/tests/fuse_binary_into_following_affine_test.cc
new file mode 100644
index 0000000000..2cba6824cf
--- /dev/null
+++ b/tensorflow/lite/toco/graph_transformations/tests/fuse_binary_into_following_affine_test.cc
@@ -0,0 +1,148 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+#include "tensorflow/lite/toco/graph_transformations/graph_transformations.h"
+#include "tensorflow/lite/toco/model.h"
+#include "tensorflow/lite/toco/tooling_util.h"
+
+namespace toco {
+
+namespace {
+// A gmock matcher that check that elements of a float vector match to a given
+// tolerance.
+std::vector<testing::Matcher<float>> ArrayFloatNear(
+    const std::vector<float>& values, float max_abs_error = 1e-5) {
+  std::vector<testing::Matcher<float>> matchers;
+  matchers.reserve(values.size());
+  for (const float& v : values) {
+    matchers.emplace_back(testing::FloatNear(v, max_abs_error));
+  }
+  return matchers;
+}
+}  // namespace
+
+class FuseBinaryIntoFollowingAffineTest : public ::testing::Test {
+ protected:
+  FuseBinaryIntoFollowingAffineTest() {}
+
+  void SetUp() override { model_.reset(new Model); }
+
+  void CreateArray(const string& name, const std::vector<int>& shape) {
+    Array& array = model_->GetOrCreateArray(name);
+    array.data_type = ArrayDataType::kFloat;
+    Shape* array_shape = array.mutable_shape();
+    *(array_shape->mutable_dims()) = shape;
+  }
+
+  void CreateConstantArray(const string& name, const std::vector<int>& shape,
+                           const std::vector<float>& data) {
+    CreateArray(name, shape);
+    Array& array = model_->GetOrCreateArray(name);
+    auto& array_buffer = array.GetMutableBuffer<ArrayDataType::kFloat>();
+    int bufsize = 1;
+    for (int dim : shape) {
+      bufsize *= dim;
+    }
+    array_buffer.data.resize(bufsize);
+    float* buf_ptr = array_buffer.data.data();
+    for (int i = 0; i < bufsize; ++i) {
+      buf_ptr[i] = data[i];
+    }
+  }
+
+  std::unique_ptr<Model> model_;
+};
+
+TEST_F(FuseBinaryIntoFollowingAffineTest, FuseMulIntoFullyConnected) {
+  // Creating a model.
+  {
+    CreateArray("Input", {2, 2});
+    CreateConstantArray("MulInput2", {1}, {2.0});
+    CreateArray("MulOutput", {2, 2});
+    CreateConstantArray("FCWeight", {2, 2}, {1.0, 2.0, 3.0, 4.0});
+    CreateConstantArray("FCBias", {1}, {1.0});
+    CreateArray("Output", {2, 2});
+
+    auto* mul_op = new MulOperator;
+    mul_op->inputs = {"Input", "MulInput2"};
+    mul_op->outputs = {"MulOutput"};
+    model_->operators.push_back(std::unique_ptr<Operator>(mul_op));
+
+    auto* fc_op = new FullyConnectedOperator;
+    fc_op->inputs = {"MulOutput", "FCWeight", "FCBias"};
+    fc_op->outputs = {"Output"};
+    model_->operators.push_back(std::unique_ptr<Operator>(fc_op));
+  }
+  toco::FuseBinaryIntoFollowingAffine transformation;
+  bool modified;
+  ASSERT_TRUE(transformation.Run(model_.get(), /*op_index=*/0, &modified).ok());
+  EXPECT_TRUE(modified);
+
+  // `Mul` should be fused into `FullyConnected`. Only 1 op is left.
+  ASSERT_EQ(model_->operators.size(), 1);
+  const auto& op = model_->operators[0];
+  ASSERT_EQ(op->type, OperatorType::kFullyConnected);
+  ASSERT_EQ(op->inputs.size(), 3);
+
+  auto& weights_array = model_->GetArray(op->inputs[1]);
+  EXPECT_THAT(weights_array.GetBuffer<toco::ArrayDataType::kFloat>().data,
+              ElementsAreArray(ArrayFloatNear({2.0, 4.0, 6.0, 8.0})));
+
+  auto& bias_array = model_->GetArray(op->inputs[2]);
+  EXPECT_THAT(bias_array.GetBuffer<toco::ArrayDataType::kFloat>().data,
+              ElementsAreArray(ArrayFloatNear({1.0})));
+}
+
+// This is a regression test of b/121287325. Toco crashes before the fix.
+TEST_F(FuseBinaryIntoFollowingAffineTest, DoNotFuseWithMultipleConsumers) {
+  // Creating a model.
+  {
+    CreateArray("Input", {2, 2});
+    CreateConstantArray("MulInput2", {1}, {2.0});
+    CreateArray("MulOutput", {2, 2});
+    CreateConstantArray("FCWeight", {2, 2}, {1.0, 2.0, 3.0, 4.0});
+    CreateConstantArray("FCBias", {1}, {1.0});
+    CreateArray("Output", {2, 2});
+    CreateArray("AnotherOutput", {2, 2});
+
+    auto* mul_op = new MulOperator;
+    mul_op->inputs = {"Input", "MulInput2"};
+    mul_op->outputs = {"MulOutput"};
+    model_->operators.push_back(std::unique_ptr<Operator>(mul_op));
+
+    auto* fc_op = new FullyConnectedOperator;
+    fc_op->inputs = {"MulOutput", "FCWeight", "FCBias"};
+    fc_op->outputs = {"Output"};
+    model_->operators.push_back(std::unique_ptr<Operator>(fc_op));
+
+    auto identity_op = new TensorFlowIdentityOperator;
+    identity_op->inputs = {"MulOutput"};
+    identity_op->outputs = {"AnotherOutput"};
+    model_->operators.push_back(std::unique_ptr<Operator>(identity_op));
+  }
+
+  toco::FuseBinaryIntoFollowingAffine transformation;
+  bool modified;
+  ASSERT_TRUE(transformation.Run(model_.get(), /*op_index=*/0, &modified).ok());
+  // Do not modify the graph if the binary operator has another output.
+  EXPECT_FALSE(modified);
+  EXPECT_EQ(model_->operators.size(), 3);
+}
+
+}  // namespace toco
-- 
GitLab


From a05eec6f5a6f6efcd9847719e32d2e34197457d8 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Mon, 14 Jan 2019 11:22:15 -0800
Subject: [PATCH 0684/1095] Better error message when using test session in
 eager mode.

PiperOrigin-RevId: 229218872
---
 tensorflow/python/framework/test_util.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py
index 22cd50321f..e2b906e37c 100644
--- a/tensorflow/python/framework/test_util.py
+++ b/tensorflow/python/framework/test_util.py
@@ -1466,6 +1466,18 @@ def disable_all_xla(description):
   return disable_all_impl
 
 
+class EagerSessionWarner(object):
+
+  def __getattr__(self, attr):
+    raise AttributeError(
+        "Trying to access properties or call methods on the result of "
+        "self.session(), self.cached_session(), etc while eager execution "
+        "is enabled. If you're porting this test case to TF 2.0, either "
+        "adapt the test to work with eager execution or insert a call to "
+        "tf.disable_eager_execution() in the main() function of this test "
+        "file.")
+
+
 @tf_export("test.TestCase")
 class TensorFlowTestCase(googletest.TestCase):
   """Base class for tests that need to test TensorFlow."""
@@ -1714,7 +1726,7 @@ class TensorFlowTestCase(googletest.TestCase):
       the graph building and execution code in a test case.
     """
     if context.executing_eagerly():
-      yield None
+      yield EagerSessionWarner()
     else:
       with self._create_session(graph, config, force_gpu) as sess:
         with self._constrain_devices_and_set_default(sess, use_gpu, force_gpu):
-- 
GitLab


From 4c6b506694ef03f1ff527d6330b10afc056287eb Mon Sep 17 00:00:00 2001
From: Michael Kuperstein <mkuper@google.com>
Date: Mon, 14 Jan 2019 11:24:27 -0800
Subject: [PATCH 0685/1095] [XLA] Move more XlaBuilder users over to the new
 DS/DUS API

Move more tests and bridge op implementations over to the new
span-of-scalars API for DynamicSlice and DynamicUpdateSlice.

PiperOrigin-RevId: 229219316
---
 .../tf2xla/kernels/strided_slice_op.cc        |  17 +--
 .../compiler/xla/tests/dynamic_ops_test.cc    | 142 +++++++++++-------
 tensorflow/compiler/xla/tests/while_test.cc   |   6 +-
 3 files changed, 100 insertions(+), 65 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/strided_slice_op.cc b/tensorflow/compiler/tf2xla/kernels/strided_slice_op.cc
index 10d990b321..2273b59246 100644
--- a/tensorflow/compiler/tf2xla/kernels/strided_slice_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/strided_slice_op.cc
@@ -288,19 +288,21 @@ class StridedSliceAssignOp : public XlaOpKernel {
     xla::XlaOp rhs = ctx->Input(4);
 
     absl::InlinedVector<int64, 4> dimensions_to_reverse;
-    absl::InlinedVector<int64, 4> slice_begin, slice_dims;
+    absl::InlinedVector<xla::XlaOp, 4> slice_begin;
+    absl::InlinedVector<int64, 4> slice_dims;
     for (int i = 0; i < begin.size(); ++i) {
       // TODO(phawkins): implement strides != 1
       OP_REQUIRES(
           ctx, strides[i] == 1 || strides[i] == -1,
           errors::Unimplemented("Strides != 1 or -1 are not yet implemented"));
       if (strides[i] > 0) {
-        slice_begin.push_back(begin[i]);
+        slice_begin.push_back(xla::ConstantR0<int64>(ctx->builder(), begin[i]));
         slice_dims.push_back(end[i] - begin[i]);
       } else {
         // Negative stride: swap begin and end, add 1 because the interval
         // is semi-open, and mark the dimension to be reversed.
-        slice_begin.push_back(end[i] + 1);
+        slice_begin.push_back(
+            xla::ConstantR0<int64>(ctx->builder(), end[i] + 1));
         slice_dims.push_back(begin[i] - end[i]);
         dimensions_to_reverse.push_back(i);
       }
@@ -311,14 +313,7 @@ class StridedSliceAssignOp : public XlaOpKernel {
     }
     rhs = xla::Reshape(rhs, slice_dims);
 
-    if (lhs_shape.dims() == 0) {
-      // TODO(b/38323843): DynamicUpdateSlice crashes on rank 0 inputs. Fix
-      // and remove this workaround.
-      lhs = rhs;
-    } else {
-      lhs = xla::DynamicUpdateSlice(
-          lhs, rhs, xla::ConstantR1<int64>(ctx->builder(), slice_begin));
-    }
+    lhs = xla::DynamicUpdateSlice(lhs, rhs, slice_begin);
 
     OP_REQUIRES_OK(ctx, ctx->AssignVariable(0, dtype_, lhs));
   }
diff --git a/tensorflow/compiler/xla/tests/dynamic_ops_test.cc b/tensorflow/compiler/xla/tests/dynamic_ops_test.cc
index 7501c6d957..82e2db3614 100644
--- a/tensorflow/compiler/xla/tests/dynamic_ops_test.cc
+++ b/tensorflow/compiler/xla/tests/dynamic_ops_test.cc
@@ -135,11 +135,11 @@ class DynamicSliceTest : public ClientLibraryTestBase {
     XlaBuilder builder(TestName());
     // Initialize and transfer dynamic slice start indices parameter.
     XlaOp starts;
-    std::unique_ptr<GlobalData> start_data = CreateR1Parameter<IndexT>(
-        slice_starts, 0, "slice_starts", &builder, &starts);
+    std::unique_ptr<GlobalData> start_data = CreateR0Parameter<IndexT>(
+        slice_starts[0], 0, "slice_starts", &builder, &starts);
     // Build dynamic slice computation.
     auto input = ConstantLiteral(&builder, input_values);
-    DynamicSlice(input, starts, slice_sizes);
+    DynamicSlice(input, absl::Span<const XlaOp>({starts}), slice_sizes);
     // Run computation and compare against expected values.
     ComputeAndCompareLiteral(&builder, expected_values, {start_data.get()});
   }
@@ -160,14 +160,23 @@ class DynamicSliceTest : public ClientLibraryTestBase {
 
     XlaBuilder builder(TestName());
     // Initialize and transfer dynamic slice start indices parameter.
-    XlaOp starts;
-    std::unique_ptr<GlobalData> start_data = CreateR1Parameter<IndexT>(
-        slice_starts, 0, "slice_starts", &builder, &starts);
+    std::vector<XlaOp> starts(2);
+    std::vector<std::unique_ptr<GlobalData>> start_data(2);
+    for (int i = 0; i < 2; ++i) {
+      start_data[i] = CreateR0Parameter<IndexT>(
+          slice_starts[i], i, "slice_starts", &builder, &starts[i]);
+    }
+
     // Build dynamic slice computation.
     auto input = ConstantLiteral(&builder, input_values);
     DynamicSlice(input, starts, slice_sizes);
     // Run computation and compare against expected values.
-    ComputeAndCompareLiteral(&builder, expected_values, {start_data.get()});
+    std::vector<GlobalData*> argument_ptrs;
+    absl::c_transform(start_data, std::back_inserter(argument_ptrs),
+                      [](const std::unique_ptr<GlobalData>& argument) {
+                        return argument.get();
+                      });
+    ComputeAndCompareLiteral(&builder, expected_values, argument_ptrs);
   }
 
   template <typename IndexT, typename DataT>
@@ -186,14 +195,22 @@ class DynamicSliceTest : public ClientLibraryTestBase {
 
     XlaBuilder builder(TestName());
     // Initialize and transfer dynamic slice start indices parameter.
-    XlaOp starts;
-    std::unique_ptr<GlobalData> start_data = CreateR1Parameter<IndexT>(
-        slice_starts, 0, "slice_starts", &builder, &starts);
+    std::vector<XlaOp> starts(3);
+    std::vector<std::unique_ptr<GlobalData>> start_data(3);
+    for (int i = 0; i < 3; ++i) {
+      start_data[i] = CreateR0Parameter<IndexT>(
+          slice_starts[i], i, "slice_starts", &builder, &starts[i]);
+    }
     // Build dynamic slice computation.
     auto input = ConstantLiteral(&builder, input_values);
     DynamicSlice(input, starts, slice_sizes);
     // Run computation and compare against expected values.
-    ComputeAndCompareLiteral(&builder, expected_values, {start_data.get()});
+    std::vector<GlobalData*> argument_ptrs;
+    absl::c_transform(start_data, std::back_inserter(argument_ptrs),
+                      [](const std::unique_ptr<GlobalData>& argument) {
+                        return argument.get();
+                      });
+    ComputeAndCompareLiteral(&builder, expected_values, argument_ptrs);
   }
 };
 
@@ -372,16 +389,12 @@ class DynamicUpdateSliceTest : public ClientLibraryTestBase {
                       .ValueOrDie());
 
     XlaBuilder builder(TestName());
-    // Initialize and transfer dynamic slice start indices parameter.
-    XlaOp starts;
-    std::unique_ptr<GlobalData> start_data = CreateR1Parameter<IndexT>(
-        slice_starts, 0, "slice_starts", &builder, &starts);
     // Build dynamic slice computation.
     auto input = ConstantLiteral(&builder, input_value);
     auto update = ConstantLiteral(&builder, update_value);
-    DynamicUpdateSlice(input, update, starts);
+    DynamicUpdateSlice(input, update, absl::Span<const XlaOp>({}));
     // Run computation and compare against expected values.
-    ComputeAndCompareLiteral(&builder, expected_value, {start_data.get()});
+    ComputeAndCompareLiteral(&builder, expected_value, {});
   }
 
   template <typename IndexT, typename DataT>
@@ -405,12 +418,12 @@ class DynamicUpdateSliceTest : public ClientLibraryTestBase {
     XlaBuilder builder(TestName());
     // Initialize and transfer dynamic slice start indices parameter.
     XlaOp starts;
-    std::unique_ptr<GlobalData> start_data = CreateR1Parameter<IndexT>(
-        slice_starts, 0, "slice_starts", &builder, &starts);
+    std::unique_ptr<GlobalData> start_data = CreateR0Parameter<IndexT>(
+        slice_starts[0], 0, "slice_starts", &builder, &starts);
     // Build dynamic slice computation.
     auto input = ConstantLiteral(&builder, input_values);
     auto update = ConstantLiteral(&builder, update_values);
-    DynamicUpdateSlice(input, update, starts);
+    DynamicUpdateSlice(input, update, absl::Span<const XlaOp>({starts}));
     // Run computation and compare against expected values.
     ComputeAndCompareLiteral(&builder, expected_values, {start_data.get()});
   }
@@ -435,15 +448,23 @@ class DynamicUpdateSliceTest : public ClientLibraryTestBase {
 
     XlaBuilder builder(TestName());
     // Initialize and transfer dynamic slice start indices parameter.
-    XlaOp starts;
-    std::unique_ptr<GlobalData> start_data = CreateR1Parameter<IndexT>(
-        slice_starts, 0, "slice_starts", &builder, &starts);
+    std::vector<XlaOp> starts(2);
+    std::vector<std::unique_ptr<GlobalData>> start_data(2);
+    for (int i = 0; i < 2; ++i) {
+      start_data[i] = CreateR0Parameter<IndexT>(
+          slice_starts[i], i, "slice_starts", &builder, &starts[i]);
+    }
     // Build dynamic slice computation.
     auto input = ConstantLiteral(&builder, input_values);
     auto update = ConstantLiteral(&builder, update_values);
     DynamicUpdateSlice(input, update, starts);
     // Run computation and compare against expected values.
-    ComputeAndCompareLiteral(&builder, expected_values, {start_data.get()});
+    std::vector<GlobalData*> argument_ptrs;
+    absl::c_transform(start_data, std::back_inserter(argument_ptrs),
+                      [](const std::unique_ptr<GlobalData>& argument) {
+                        return argument.get();
+                      });
+    ComputeAndCompareLiteral(&builder, expected_values, argument_ptrs);
   }
 
   template <typename IndexT, typename DataT>
@@ -466,15 +487,24 @@ class DynamicUpdateSliceTest : public ClientLibraryTestBase {
 
     XlaBuilder builder(TestName());
     // Initialize and transfer dynamic slice start indices parameter.
-    XlaOp starts;
-    std::unique_ptr<GlobalData> start_data = CreateR1Parameter<IndexT>(
-        slice_starts, 0, "slice_starts", &builder, &starts);
+    std::vector<XlaOp> starts(3);
+    std::vector<std::unique_ptr<GlobalData>> start_data(3);
+    for (int i = 0; i < 3; ++i) {
+      start_data[i] = CreateR0Parameter<IndexT>(
+          slice_starts[i], i, "slice_starts", &builder, &starts[i]);
+    }
+
     // Build dynamic slice computation.
     auto input = ConstantLiteral(&builder, input_values);
     auto update = ConstantLiteral(&builder, update_values);
     DynamicUpdateSlice(input, update, starts);
     // Run computation and compare against expected values.
-    ComputeAndCompareLiteral(&builder, expected_values, {start_data.get()});
+    std::vector<GlobalData*> argument_ptrs;
+    absl::c_transform(start_data, std::back_inserter(argument_ptrs),
+                      [](const std::unique_ptr<GlobalData>& argument) {
+                        return argument.get();
+                      });
+    ComputeAndCompareLiteral(&builder, expected_values, argument_ptrs);
   }
 
   template <class T>
@@ -518,8 +548,9 @@ class DynamicUpdateSliceTest : public ClientLibraryTestBase {
     XlaOp update;
     std::unique_ptr<GlobalData> update_data = CreateR3Parameter<T>(
         update_values, 1, "update_values", &builder, &update);
-    auto starts = ConstantR1<int32>(&builder, {index, 0, 0});
-    DynamicUpdateSlice(input, update, starts);
+    auto constant_index = ConstantR0<int32>(&builder, index);
+    auto zero = ConstantR0<int32>(&builder, 0);
+    DynamicUpdateSlice(input, update, {constant_index, zero, zero});
 
     // Run computation and compare against expected values.
     ComputeAndCompareR3<T>(&builder, expected_values,
@@ -720,46 +751,55 @@ void BM_DynamicSlice(int num_iters) {
         {{13, 14, 15, 16}, {17, 18, 19, 20}, {21, 22, 23, 24}}}});
   auto input = ConstantLiteral(&builder, input_literal);
 
+  auto stream =
+      client->mutable_backend()->BorrowStream(device_ordinal).ValueOrDie();
+
   // Create dynamic slice start indices as a parameter: shape [4]
-  auto start_indices_shape = ShapeUtil::MakeShape(S32, {4});
-  auto start_indices =
-      Parameter(&builder, 0, start_indices_shape, "start_indices");
+  auto start_indices_shape = ShapeUtil::MakeShape(S32, {});
+  std::vector<XlaOp> start_indices(4);
+  std::vector<ScopedShapedBuffer> shaped_buffers;
+  std::vector<const Shape*> host_shapes(4);
+  for (int i = 0; i < 4; ++i) {
+    start_indices[i] =
+        Parameter(&builder, i, start_indices_shape, "start_indices");
+    auto start_index_literal = LiteralUtil::CreateR0<int32>(i + 1);
+    // Initialize and transfer parameter buffer.
+    shaped_buffers.emplace_back(
+        client->backend()
+            .transfer_manager()
+            ->AllocateScopedShapedBuffer(start_indices_shape, &allocator,
+                                         /*device_ordinal=*/0)
+            .ConsumeValueOrDie());
+    host_shapes[i] = &shaped_buffers[i].on_host_shape();
+    ASSERT_IS_OK(transfer_manager->TransferLiteralToDevice(
+        stream.get(), start_index_literal, shaped_buffers[i]));
+  }
+
   // Add DynamicSlice op to the computatation.
   DynamicSlice(input, start_indices, {1, 1, 1, 1});
   auto computation = builder.Build().ConsumeValueOrDie();
 
-  // Initialize and transfer parameter buffer.
-  auto buffer = client->backend()
-                    .transfer_manager()
-                    ->AllocateScopedShapedBuffer(
-                        start_indices_shape, &allocator, /*device_ordinal=*/0)
-                    .ConsumeValueOrDie();
-
-  auto start_indices_literal = LiteralUtil::CreateR1<int32>({0, 1, 2, 3});
-  auto stream =
-      client->mutable_backend()->BorrowStream(device_ordinal).ValueOrDie();
-  ASSERT_IS_OK(transfer_manager->TransferLiteralToDevice(
-      stream.get(), start_indices_literal, buffer));
-
   std::unique_ptr<LocalExecutable> executable =
-      client
-          ->Compile(computation, {&buffer.on_host_shape()},
-                    ExecutableBuildOptions())
+      client->Compile(computation, host_shapes, ExecutableBuildOptions())
           .ConsumeValueOrDie();
 
   // Run some warm-up executions.
   ExecutableRunOptions options;
   options.set_allocator(&allocator);
   const int kWarmups = 2;
+  std::vector<const ShapedBuffer*> shaped_buffer_ptrs;
+  absl::c_transform(shaped_buffers, std::back_inserter(shaped_buffer_ptrs),
+                    [](const ScopedShapedBuffer& buffer) { return &buffer; });
+
   for (int i = 0; i < kWarmups; ++i) {
-    auto result = executable->Run({&buffer}, options);
+    auto result = executable->Run(shaped_buffer_ptrs, options);
     ASSERT_TRUE(result.ok());
   }
 
   // Run benchmark.
   tensorflow::testing::StartTiming();
   for (int i = 0; i < num_iters; ++i) {
-    auto result = executable->Run({&buffer}, options);
+    auto result = executable->Run(shaped_buffer_ptrs, options);
     ASSERT_TRUE(result.ok());
   }
 }
diff --git a/tensorflow/compiler/xla/tests/while_test.cc b/tensorflow/compiler/xla/tests/while_test.cc
index b1566dc8ad..85212fa56d 100644
--- a/tensorflow/compiler/xla/tests/while_test.cc
+++ b/tensorflow/compiler/xla/tests/while_test.cc
@@ -861,7 +861,7 @@ XLA_TEST_F(WhileTest, WhileWithDynamicUpdateSlice) {
     // Update.
     auto update = ConvertElementType(Broadcast(out0, {2}), F32);
     // Starts = iteration * 2;
-    auto starts = Reshape(Mul(iteration, ConstantR0<int32>(&builder, 2)), {1});
+    auto starts = Mul(iteration, ConstantR0<int32>(&builder, 2));
     // UpdateSlice.
     auto out1 = DynamicUpdateSlice(input, update, starts);
 
@@ -1299,9 +1299,9 @@ void BM_WhileLoop(int num_iters) {
     auto one = ConstantR0<float>(&builder, 1.0);
     auto update = Broadcast(one, {1, 1024, 1024});
     // Starts = iteration * 2;
-    auto starts = ConstantR1<int32>(&builder, {0, 0, 0});
+    auto zero = ConstantR0<int32>(&builder, 0);
     // UpdateSlice.
-    auto out1 = DynamicUpdateSlice(input, update, starts);
+    auto out1 = DynamicUpdateSlice(input, update, {zero, zero, zero});
     Tuple(&builder, {out0, out1});
     body = builder.Build().ConsumeValueOrDie();
   }
-- 
GitLab


From 8d151b6c60919a9ed871f96c12f68a25831bdf2d Mon Sep 17 00:00:00 2001
From: Doe Hyun Yoon <dyoon@google.com>
Date: Mon, 14 Jan 2019 11:39:36 -0800
Subject: [PATCH 0686/1095] Replace PredictCosts() with
 PredictCostsAndReturnRunMetadata().

PiperOrigin-RevId: 229222942
---
 .../grappler/costs/analytical_cost_estimator.cc | 15 +--------------
 .../grappler/costs/analytical_cost_estimator.h  | 10 +++-------
 .../costs/analytical_cost_estimator_test.cc     |  3 +--
 tensorflow/core/grappler/costs/cost_estimator.h | 17 ++---------------
 .../grappler/costs/measuring_cost_estimator.cc  |  6 +++++-
 .../grappler/costs/measuring_cost_estimator.h   | 10 +++++-----
 tensorflow/python/grappler/cluster.i            |  6 +++---
 tensorflow/python/grappler/cost_analyzer.cc     |  4 ++--
 8 files changed, 22 insertions(+), 49 deletions(-)

diff --git a/tensorflow/core/grappler/costs/analytical_cost_estimator.cc b/tensorflow/core/grappler/costs/analytical_cost_estimator.cc
index 09cddad8ba..5baf306f6f 100644
--- a/tensorflow/core/grappler/costs/analytical_cost_estimator.cc
+++ b/tensorflow/core/grappler/costs/analytical_cost_estimator.cc
@@ -124,22 +124,9 @@ Status AnalyticalCostEstimator::Initialize(const GrapplerItem& item) {
   return Status::OK();
 }
 
-// TODO(b/67607683): unify logic with VirtualCluster logic
 Status AnalyticalCostEstimator::PredictCosts(const GraphDef& optimized_graph,
-                                             CostGraphDef* cost_graph,
+                                             RunMetadata* run_metadata,
                                              Costs* costs) const {
-  RunMetadata run_metadata;
-  auto s =
-      PredictCostsAndReturnRunMetadata(optimized_graph, &run_metadata, costs);
-  if (s.ok() && cost_graph) {
-    cost_graph->Swap(run_metadata.mutable_cost_graph());
-  }
-  return s;
-}
-
-Status AnalyticalCostEstimator::PredictCostsAndReturnRunMetadata(
-    const GraphDef& optimized_graph, RunMetadata* run_metadata,
-    Costs* costs) const {
   GrapplerItem item = item_;
   item.graph = optimized_graph;
 
diff --git a/tensorflow/core/grappler/costs/analytical_cost_estimator.h b/tensorflow/core/grappler/costs/analytical_cost_estimator.h
index 6275c62198..d058ba4115 100644
--- a/tensorflow/core/grappler/costs/analytical_cost_estimator.h
+++ b/tensorflow/core/grappler/costs/analytical_cost_estimator.h
@@ -51,14 +51,10 @@ class AnalyticalCostEstimator : public CostEstimator {
   Status Initialize(const GrapplerItem& item) override;
 
   // Predict the performance of each node of the optimized graph and annotate
-  // the CostGraphDef with the corresponding estimates. Also returns the
+  // the RunMetadata with the corresponding estimates. Also returns the
   // expected cost for the whole graph.
-  Status PredictCosts(const GraphDef& optimized_graph, CostGraphDef* cost_graph,
-                      Costs* cost) const override;
-
-  Status PredictCostsAndReturnRunMetadata(const GraphDef& optimized_graph,
-                                          RunMetadata* run_metadata,
-                                          Costs* cost) const override;
+  Status PredictCosts(const GraphDef& optimized_graph,
+                      RunMetadata* run_metadata, Costs* cost) const override;
 
   const VirtualScheduler* GetScheduler() const { return scheduler_.get(); }
 
diff --git a/tensorflow/core/grappler/costs/analytical_cost_estimator_test.cc b/tensorflow/core/grappler/costs/analytical_cost_estimator_test.cc
index 8a56344378..eb7ee8dc0a 100644
--- a/tensorflow/core/grappler/costs/analytical_cost_estimator_test.cc
+++ b/tensorflow/core/grappler/costs/analytical_cost_estimator_test.cc
@@ -100,8 +100,7 @@ TEST_F(AnalyticalCostEstimatorTest, SimpleTest) {
 
   RunMetadata run_metadata;
   Costs summary;
-  TF_ASSERT_OK(estimator.PredictCostsAndReturnRunMetadata(
-      item.graph, &run_metadata, &summary));
+  TF_ASSERT_OK(estimator.PredictCosts(item.graph, &run_metadata, &summary));
 
   EXPECT_EQ(Costs::NanoSeconds(9151), summary.execution_time);
   // Note there are totally 17 nodes (RandomUniform creates 2 nodes), but
diff --git a/tensorflow/core/grappler/costs/cost_estimator.h b/tensorflow/core/grappler/costs/cost_estimator.h
index 725d81a881..d85ae0b77f 100644
--- a/tensorflow/core/grappler/costs/cost_estimator.h
+++ b/tensorflow/core/grappler/costs/cost_estimator.h
@@ -216,27 +216,14 @@ class CostEstimator {
 
   // Predicts the cost of running the given optimized version of the grappler
   // item.
-  // If a CostGraphDef is passed, it will be populated with detailed information
+  // If a RunMetadata is passed, it will be populated with detailed information
   // about the cost of running each operation of the optimized graph.
   // if a double value is passed, it will be set to a value that reflects the
   // overall cost of running the graph (e.g. the latency of the computation).
   // Returns a status that indicate is the performance could be estimated or
   // not.
   virtual Status PredictCosts(const GraphDef& optimized_graph,
-                              CostGraphDef* cost_graph, Costs* cost) const = 0;
-
-  // TODO(dyoon): Delete PredictCosts() with CostGraphDef as RunMetadata is a
-  // superset of CostGraphDef.
-  // Same method, but returns RunMetadata.
-  virtual Status PredictCostsAndReturnRunMetadata(
-      const GraphDef& optimized_graph, RunMetadata* run_metadata,
-      Costs* cost) const {
-    CostGraphDef* cost_graph = nullptr;
-    if (run_metadata) {
-      cost_graph = run_metadata->mutable_cost_graph();
-    }
-    return PredictCosts(optimized_graph, cost_graph, cost);
-  }
+                              RunMetadata* run_metadata, Costs* cost) const = 0;
 };
 
 }  // end namespace grappler
diff --git a/tensorflow/core/grappler/costs/measuring_cost_estimator.cc b/tensorflow/core/grappler/costs/measuring_cost_estimator.cc
index 833205ac6f..088ce56658 100644
--- a/tensorflow/core/grappler/costs/measuring_cost_estimator.cc
+++ b/tensorflow/core/grappler/costs/measuring_cost_estimator.cc
@@ -51,8 +51,12 @@ Status MeasuringCostEstimator::Initialize(const GrapplerItem& item) {
 }
 
 Status MeasuringCostEstimator::PredictCosts(const GraphDef& optimized_graph,
-                                            CostGraphDef* cost_graph,
+                                            RunMetadata* run_metadata,
                                             Costs* costs) const {
+  CostGraphDef* cost_graph = nullptr;
+  if (run_metadata) {
+    cost_graph = run_metadata->mutable_cost_graph();
+  }
   const bool running_simulation = (cluster_->type() == "virtual");
 
   std::vector<double> times(measurement_steps_);
diff --git a/tensorflow/core/grappler/costs/measuring_cost_estimator.h b/tensorflow/core/grappler/costs/measuring_cost_estimator.h
index 3e741c9199..67145f5241 100644
--- a/tensorflow/core/grappler/costs/measuring_cost_estimator.h
+++ b/tensorflow/core/grappler/costs/measuring_cost_estimator.h
@@ -54,12 +54,12 @@ class MeasuringCostEstimator : public CostEstimator {
   // This implementation always returns OK.
   Status Initialize(const GrapplerItem& item) override;
 
-  // Runs the optimized version of the graph on the cluster, measure
-  // the runtimes of each operation, and annotated the CostGraphDef
-  // with the corresponding measurements.
+  // Runs the optimized version of the graph on the cluster, measures
+  // the runtimes of each operation, and annotates the CostGraphDef of
+  // RunMetadata with the corresponding measurements.
   // Returns the average latency for the whole graph.
-  Status PredictCosts(const GraphDef& optimized_graph, CostGraphDef* cost_graph,
-                      Costs* overall_cost) const override;
+  Status PredictCosts(const GraphDef& optimized_graph,
+                      RunMetadata* run_metadata, Costs* cost) const override;
 
  private:
   Cluster* cluster_;  // Not owned.
diff --git a/tensorflow/python/grappler/cluster.i b/tensorflow/python/grappler/cluster.i
index b0c1f71a85..af9276c508 100644
--- a/tensorflow/python/grappler/cluster.i
+++ b/tensorflow/python/grappler/cluster.i
@@ -176,13 +176,13 @@ tensorflow::Status _GetOpPerformanceDataAndRunTime(
   tensorflow::Status status = cost_measure->Initialize(item);
   if (!status.ok()) return status;
 
-  tensorflow::CostGraphDef cost_graph;
+  tensorflow::RunMetadata run_metadata;
   TF_RETURN_IF_ERROR(
-      cost_measure->PredictCosts(item.graph, &cost_graph, costs));
+      cost_measure->PredictCosts(item.graph, &run_metadata, costs));
 
   if (op_performance_data) {
     *op_performance_data = tensorflow::grappler::CostGraphToOpPerformanceData(
-        cost_graph, item.graph);
+        run_metadata.cost_graph(), item.graph);
   }
   return tensorflow::Status::OK();
 }
diff --git a/tensorflow/python/grappler/cost_analyzer.cc b/tensorflow/python/grappler/cost_analyzer.cc
index bb8c6d5b85..9aa5fbca38 100644
--- a/tensorflow/python/grappler/cost_analyzer.cc
+++ b/tensorflow/python/grappler/cost_analyzer.cc
@@ -44,8 +44,8 @@ void CostAnalyzer::PredictCosts(CostEstimator* cost_estimator,
   TF_CHECK_OK(cost_estimator->Initialize(*item_));
   RunMetadata run_metadata;
   Costs costs;
-  const Status status = cost_estimator->PredictCostsAndReturnRunMetadata(
-      item_->graph, &run_metadata, &costs);
+  const Status status =
+      cost_estimator->PredictCosts(item_->graph, &run_metadata, &costs);
   if (cost_graph) {
     cost_graph->Swap(run_metadata.mutable_cost_graph());
   }
-- 
GitLab


From 0d8b2d70ac5a7387bc46454b40ae4f1388eaaffc Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 14 Jan 2019 11:55:32 -0800
Subject: [PATCH 0687/1095] Internal change only

PiperOrigin-RevId: 229225901
---
 tensorflow/python/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 2c6672c81e..905b47ea75 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -11,6 +11,7 @@ visibility = [
     "//tensorflow/lite/toco/python:__pkg__",
     "//tensorflow_models:__subpackages__",
     "//tensorflow_model_optimization:__subpackages__",
+    "//third_party/py/cleverhans:__subpackages__",
     # TODO(aselle): to pass open source test.
     "//bazel_pip/tensorflow/lite/toco/python:__pkg__",
 ]
-- 
GitLab


From 30796c730b0d137b0367d425cbfbfae66c43a9ff Mon Sep 17 00:00:00 2001
From: Thomas O'Malley <omalleyt@google.com>
Date: Mon, 14 Jan 2019 12:02:17 -0800
Subject: [PATCH 0688/1095] Enable Callbacks when a generator is passed to
 `evaluate`/`predict`.

PiperOrigin-RevId: 229227149
---
 tensorflow/python/keras/callbacks_test.py  | 36 ++++++++++++++++++----
 tensorflow/python/keras/engine/training.py |  4 +--
 2 files changed, 32 insertions(+), 8 deletions(-)

diff --git a/tensorflow/python/keras/callbacks_test.py b/tensorflow/python/keras/callbacks_test.py
index d5ffdf789a..7af9a9a2b2 100644
--- a/tensorflow/python/keras/callbacks_test.py
+++ b/tensorflow/python/keras/callbacks_test.py
@@ -27,6 +27,7 @@ import tempfile
 import threading
 import unittest
 
+from absl.testing import parameterized
 import numpy as np
 
 from tensorflow.core.framework import summary_pb2
@@ -88,6 +89,23 @@ class Counter(keras.callbacks.Callback):
     return _call_and_count
 
 
+def _get_numpy():
+  return np.ones((10, 10)), np.ones((10, 1))
+
+
+def _get_sequence():
+
+  class MySequence(keras.utils.data_utils.Sequence):
+
+    def __getitem__(self, _):
+      return np.ones((2, 10)), np.ones((2, 1))
+
+    def __len__(self):
+      return 5
+
+  return MySequence(), None
+
+
 @keras_parameterized.run_with_all_model_types
 @keras_parameterized.run_all_keras_modes
 class CallbackCountsTest(keras_parameterized.TestCase):
@@ -113,8 +131,10 @@ class CallbackCountsTest(keras_parameterized.TestCase):
         run_eagerly=testing_utils.should_run_eagerly())
     return model
 
-  def test_callback_hooks_are_called_in_fit(self):
-    x, y = np.ones((10, 10)), np.ones((10, 1))
+  @parameterized.named_parameters(('with_numpy', _get_numpy()),
+                                  ('with_sequence', _get_sequence()))
+  def test_callback_hooks_are_called_in_fit(self, data):
+    x, y = data
     val_x, val_y = np.ones((4, 10)), np.ones((4, 1))
 
     model = self._get_model()
@@ -147,8 +167,10 @@ class CallbackCountsTest(keras_parameterized.TestCase):
             'on_train_end': 1
         })
 
-  def test_callback_hooks_are_called_in_evaluate(self):
-    x, y = np.ones((10, 10)), np.ones((10, 1))
+  @parameterized.named_parameters(('with_numpy', _get_numpy()),
+                                  ('with_sequence', _get_sequence()))
+  def test_callback_hooks_are_called_in_evaluate(self, data):
+    x, y = data
 
     model = self._get_model()
     counter = Counter()
@@ -161,8 +183,10 @@ class CallbackCountsTest(keras_parameterized.TestCase):
             'on_test_end': 1
         })
 
-  def test_callback_hooks_are_called_in_predict(self):
-    x = np.ones((10, 10))
+  @parameterized.named_parameters(('with_numpy', _get_numpy()),
+                                  ('with_sequence', _get_sequence()))
+  def test_callback_hooks_are_called_in_predict(self, data):
+    x = data[0]
 
     model = self._get_model()
     counter = Counter()
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 598766db74..9f544ceff4 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -1023,11 +1023,11 @@ class Model(Network):
     # or a non-distributed Dataset or iterator in eager execution.
     if data_utils.is_generator_or_sequence(x):
       training_utils.check_generator_arguments(y, sample_weight)
-      # TODO(fchollet): why aren't callbacks supported here?
       return self.evaluate_generator(
           x,
           steps=steps,
           verbose=verbose,
+          callbacks=callbacks,
           max_queue_size=max_queue_size,
           workers=workers,
           use_multiprocessing=use_multiprocessing)
@@ -1146,11 +1146,11 @@ class Model(Network):
     # Case 2: generator-like. Input is Python generator, or Sequence object,
     # or a non-distributed Dataset or iterator in eager execution.
     if data_utils.is_generator_or_sequence(x):
-      # TODO(fchollet): why aren't callbacks supported here?
       return self.predict_generator(
           x,
           steps=steps,
           verbose=verbose,
+          callbacks=callbacks,
           max_queue_size=max_queue_size,
           workers=workers,
           use_multiprocessing=use_multiprocessing)
-- 
GitLab


From 3768e3e1748a68dcd8e35ec1e28c7b09bacc1b31 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 14 Jan 2019 12:33:14 -0800
Subject: [PATCH 0689/1095] Add `alwayslink = 1` param to optimizer build
 rules. Without this, optimizers might not actually get linked into a binary.

PiperOrigin-RevId: 229232974
---
 tensorflow/core/grappler/optimizers/data/BUILD | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/tensorflow/core/grappler/optimizers/data/BUILD b/tensorflow/core/grappler/optimizers/data/BUILD
index 57b0df3cb2..3bfaef4be1 100644
--- a/tensorflow/core/grappler/optimizers/data/BUILD
+++ b/tensorflow/core/grappler/optimizers/data/BUILD
@@ -25,6 +25,7 @@ cc_library(
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:ptr_util",
     ] + tf_protos_all(),
+    alwayslink = 1,
 )
 
 cc_library(
@@ -48,6 +49,7 @@ cc_library(
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
         "//tensorflow/core:lib_internal",
     ] + tf_protos_all(),
+    alwayslink = 1,
 )
 
 tf_cc_test(
@@ -212,6 +214,7 @@ cc_library(
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
         "//tensorflow/core:lib_internal",
     ] + tf_protos_all(),
+    alwayslink = 1,
 )
 
 tf_cc_test(
@@ -248,6 +251,7 @@ cc_library(
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
     ] + tf_protos_all(),
+    alwayslink = 1,
 )
 
 cc_library(
@@ -266,6 +270,7 @@ cc_library(
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
     ] + tf_protos_all(),
+    alwayslink = 1,
 )
 
 tf_cc_test(
@@ -297,6 +302,7 @@ cc_library(
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
     ] + tf_protos_all(),
+    alwayslink = 1,
 )
 
 tf_cc_test(
@@ -333,6 +339,7 @@ cc_library(
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
     ] + tf_protos_all(),
+    alwayslink = 1,
 )
 
 tf_cc_test(
@@ -370,6 +377,7 @@ cc_library(
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
         "//tensorflow/core:lib_internal",
     ] + tf_protos_all(),
+    alwayslink = 1,
 )
 
 tf_cc_test(
@@ -409,6 +417,7 @@ cc_library(
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
         "//tensorflow/core:lib_internal",
     ] + tf_protos_all(),
+    alwayslink = 1,
 )
 
 tf_cc_test(
@@ -447,6 +456,7 @@ cc_library(
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
     ] + tf_protos_all(),
+    alwayslink = 1,
 )
 
 tf_cc_test(
@@ -486,6 +496,7 @@ cc_library(
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
         "//tensorflow/core:lib_internal",
     ] + tf_protos_all(),
+    alwayslink = 1,
 )
 
 tf_cc_test(
@@ -521,6 +532,7 @@ cc_library(
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
     ] + tf_protos_all(),
+    alwayslink = 1,
 )
 
 tf_cc_test(
@@ -555,6 +567,7 @@ cc_library(
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
     ] + tf_protos_all(),
+    alwayslink = 1,
 )
 
 tf_cc_test(
-- 
GitLab


From b23dc8cb365c0678931b893d6919f20630fab61e Mon Sep 17 00:00:00 2001
From: Tim Shen <timshen@google.com>
Date: Mon, 14 Jan 2019 12:36:07 -0800
Subject: [PATCH 0690/1095] Fix Windows build by moving `LOG(ERROR)` from the
 header to the .cc file.

Some Windows header(s) may define ERROR as a macro, causing `LOG(ERROR)` fail to compile. The .cc file doesn't include such header(s), since there are already a lot of uses of ERROR there.

PiperOrigin-RevId: 229233490
---
 tensorflow/stream_executor/stream.cc | 9 +++++++++
 tensorflow/stream_executor/stream.h  | 9 +--------
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/tensorflow/stream_executor/stream.cc b/tensorflow/stream_executor/stream.cc
index 9937861d88..0dceacb228 100644
--- a/tensorflow/stream_executor/stream.cc
+++ b/tensorflow/stream_executor/stream.cc
@@ -5678,4 +5678,13 @@ string Stream::DebugStreamPointers() const {
                       ",impl=", ToVlogString(implementation_.get()), "]");
 }
 
+void Stream::CheckStatus(port::Status status) {
+  if (status.ok()) {
+    return;
+  }
+  LOG(ERROR) << status;
+  mutex_lock lock(mu_);
+  ok_ = false;
+}
+
 }  // namespace stream_executor
diff --git a/tensorflow/stream_executor/stream.h b/tensorflow/stream_executor/stream.h
index 85d65c1061..f698d50e2a 100644
--- a/tensorflow/stream_executor/stream.h
+++ b/tensorflow/stream_executor/stream.h
@@ -2100,14 +2100,7 @@ class Stream {
   }
 
   // Checks the status and logs the error message, if any.
-  void CheckStatus(port::Status status) LOCKS_EXCLUDED(mu_) {
-    if (status.ok()) {
-      return;
-    }
-    LOG(ERROR) << status;
-    mutex_lock lock(mu_);
-    ok_ = false;
-  }
+  void CheckStatus(port::Status status) LOCKS_EXCLUDED(mu_);
 
   void SetError() { CheckError(false /* = operation_retcode */); }
 
-- 
GitLab


From c45a87f7c7080eecf42c430c66adef0e9249f5ac Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 14 Jan 2019 12:40:41 -0800
Subject: [PATCH 0691/1095] LSC: Replace calls to deprecated googletest macros
 *TEST_CASE() with *TEST_SUITE()

PiperOrigin-RevId: 229234323
---
 tensorflow/lite/testing/generated_examples_zip_test.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/lite/testing/generated_examples_zip_test.cc b/tensorflow/lite/testing/generated_examples_zip_test.cc
index bade6590a4..dfd8eb059a 100644
--- a/tensorflow/lite/testing/generated_examples_zip_test.cc
+++ b/tensorflow/lite/testing/generated_examples_zip_test.cc
@@ -287,10 +287,10 @@ struct ZipPathParamName {
   }
 };
 
-INSTANTIATE_TEST_CASE_P(tests, OpsTest,
-                        ::testing::ValuesIn(UnarchiveAndFindTestNames(
-                            *FLAGS_zip_file_path, *FLAGS_tar_file_path)),
-                        ZipPathParamName());
+INSTANTIATE_TEST_SUITE_P(tests, OpsTest,
+                         ::testing::ValuesIn(UnarchiveAndFindTestNames(
+                             *FLAGS_zip_file_path, *FLAGS_tar_file_path)),
+                         ZipPathParamName());
 
 }  // namespace testing
 }  // namespace tflite
-- 
GitLab


From 36af98052e0f92c303a85244eebdf7876e5cce19 Mon Sep 17 00:00:00 2001
From: Kashif Rasul <kashif.rasul@gmail.com>
Date: Mon, 14 Jan 2019 21:53:22 +0100
Subject: [PATCH 0692/1095] fix var name to follow style

---
 tensorflow/stream_executor/cuda/cuda_gpu_executor.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc
index 7705e7e8f2..6e0bd38149 100644
--- a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc
+++ b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc
@@ -1058,9 +1058,9 @@ DeviceDescription *CUDAExecutor::PopulateDeviceDescription() const {
         CU_DEVICE_ATTRIBUTE_MAX_BLOCK_DIM_Z, device_).ValueOrDie();
     builder.set_thread_dim_limit(thread_dim_limit);
 
-    int clockRate = CUDADriver::GetDeviceAttribute(
+    int clock_rate = CUDADriver::GetDeviceAttribute(
         CU_DEVICE_ATTRIBUTE_CLOCK_RATE, device_).ValueOrDie();
-    builder.set_clock_rate_ghz(static_cast<float>(clockRate) / 1e6);
+    builder.set_clock_rate_ghz(static_cast<float>(clock_rate) / 1e6);
   }
 
   {
-- 
GitLab


From 2a33341c8605dceb6b95f48e688347ae5d437781 Mon Sep 17 00:00:00 2001
From: Igor Ganichev <iga@google.com>
Date: Mon, 14 Jan 2019 12:40:58 -0800
Subject: [PATCH 0693/1095] Add basic while/cond tests to eager/xla tests

PiperOrigin-RevId: 229234361
---
 tensorflow/compiler/tests/eager_test.py | 52 +++++++++++++++++++++++++
 1 file changed, 52 insertions(+)

diff --git a/tensorflow/compiler/tests/eager_test.py b/tensorflow/compiler/tests/eager_test.py
index c36c12a0e3..c9fce39f6c 100644
--- a/tensorflow/compiler/tests/eager_test.py
+++ b/tensorflow/compiler/tests/eager_test.py
@@ -24,6 +24,7 @@ from tensorflow.compiler.tests import xla_test
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.eager import backprop
 from tensorflow.python.eager import context
+from tensorflow.python.eager import def_function
 from tensorflow.python.eager import function
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -31,6 +32,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.layers import convolutional
 from tensorflow.python.layers import pooling
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import embedding_ops
 from tensorflow.python.ops import gen_random_ops
 from tensorflow.python.ops import init_ops
@@ -555,6 +557,56 @@ class EagerFunctionTest(xla_test.XLATestCase):
     self.assertEqual(9, dy_v0.numpy())
     self.assertEqual(15, dy_v1.numpy())
 
+  def testWhileInDefun(self):
+    with self.test_scope():
+      @def_function.function
+      def f(start):
+        c = lambda x: math_ops.less(x, 13.0)
+        b = lambda x: math_ops.add(x, 1.0)
+        return control_flow_ops.while_loop(c, b, [start])
+
+      y = f(constant_op.constant(3.0))
+    self.assertEqual(13.0, y.numpy())
+
+  def testAutoGraphWhileInDefun(self):
+    with self.test_scope():
+      @def_function.function
+      def f(start):
+        x = start
+        while x < 13.0:
+          x += 1.0
+        return x
+
+      y = f(constant_op.constant(3.0))
+    self.assertEqual(13.0, y.numpy())
+
+  def testCondInDefun(self):
+    with self.test_scope():
+      @def_function.function
+      def f(pred, value):
+        fn1 = lambda: math_ops.add(value, 1.0)
+        fn2 = lambda: math_ops.subtract(value, 1.0)
+        return control_flow_ops.cond(pred, fn1, fn2)
+
+      plus_one = f(constant_op.constant(True), constant_op.constant(10.0))
+      minus_one = f(constant_op.constant(False), constant_op.constant(10.0))
+    self.assertEqual(11.0, plus_one.numpy())
+    self.assertEqual(9.0, minus_one.numpy())
+
+  def testAutoGraphCondInDefun(self):
+    with self.test_scope():
+      @def_function.function
+      def f(pred, value):
+        if pred:
+          return value + 1.0
+        else:
+          return value - 1.0
+
+      plus_one = f(constant_op.constant(True), constant_op.constant(10.0))
+      minus_one = f(constant_op.constant(False), constant_op.constant(10.0))
+    self.assertEqual(11.0, plus_one.numpy())
+    self.assertEqual(9.0, minus_one.numpy())
+
 
 class ExcessivePaddingTest(xla_test.XLATestCase):
   """Test that eager execution works with TPU flattened tensors.
-- 
GitLab


From 418359b3bc408e1948d9e34618a4b5af3e416b56 Mon Sep 17 00:00:00 2001
From: Taylor Robie <taylorrobie@google.com>
Date: Mon, 14 Jan 2019 12:57:02 -0800
Subject: [PATCH 0694/1095] Fix a bug in Sequential models where activity
 regularization was sometimes not applied.

PiperOrigin-RevId: 229236960
---
 tensorflow/python/keras/engine/sequential.py  | 28 ++++++++++-----
 .../python/keras/engine/training_test.py      | 34 +++++++++++--------
 2 files changed, 38 insertions(+), 24 deletions(-)

diff --git a/tensorflow/python/keras/engine/sequential.py b/tensorflow/python/keras/engine/sequential.py
index 3627091198..425c8c3ef6 100644
--- a/tensorflow/python/keras/engine/sequential.py
+++ b/tensorflow/python/keras/engine/sequential.py
@@ -245,8 +245,11 @@ class Sequential(Model):
     if not self.built and self._is_graph_network:
       self._init_graph_network(self.inputs, self.outputs, name=self.name)
 
-    x = inputs
+    outputs = inputs  # handle the corner case where self.layers is empty
     for layer in self.layers:
+      # During each iteration, `inputs` are the inputs to `layer`, and `outputs`
+      # are the outputs of `layer` applied to `inputs`. At the end of each
+      # iteration `inputs` is set to `outputs` to prepare for the next layer.
       kwargs = {}
       argspec = self._layer_call_argspecs[layer].args
       if 'mask' in argspec:
@@ -255,27 +258,34 @@ class Sequential(Model):
         kwargs['training'] = training
 
       if isinstance(layer, Network) and layer._compute_output_and_mask_jointly:
-        x, mask = layer._call_and_compute_mask(x, **kwargs)
+        outputs, mask = layer._call_and_compute_mask(inputs, **kwargs)
       else:
         if not layer.built:
           # Build layer if applicable.
           with ops.name_scope(layer._name_scope()):
-            layer._maybe_build(x)
+            layer._maybe_build(inputs)
           layer.built = True
         if layer.supports_masking:
-          mask = layer.compute_mask(x, mask)
+          mask = layer.compute_mask(inputs, mask)
         else:
           mask = None
 
         if context.executing_eagerly():
-          x = layer(x, **kwargs)
+          # __call__ handles activity regularization.
+          outputs = layer(inputs, **kwargs)
         elif layer.dynamic:
-          x = layer._symbolic_call(x)
+          outputs = layer._symbolic_call(inputs)
+          layer._handle_activity_regularization(inputs, outputs)
         else:
-          x = layer.call(x, **kwargs)
+          outputs = layer.call(inputs, **kwargs)
+          layer._handle_activity_regularization(inputs, outputs)
+
       if not context.executing_eagerly():
-        x._keras_mask = mask
-    return x, mask
+        outputs._keras_mask = mask
+
+      # `outputs` will be the inputs to the next layer.
+      inputs = outputs
+    return outputs, mask
 
   def compute_output_shape(self, input_shape):
     shape = input_shape
diff --git a/tensorflow/python/keras/engine/training_test.py b/tensorflow/python/keras/engine/training_test.py
index b8d4752db4..fa26385cc4 100644
--- a/tensorflow/python/keras/engine/training_test.py
+++ b/tensorflow/python/keras/engine/training_test.py
@@ -350,16 +350,21 @@ class TrainingTest(keras_parameterized.TestCase):
     self.assertEqual(len(out), 2)
 
   @keras_parameterized.run_all_keras_modes
+  @keras_parameterized.run_with_all_model_types
   def test_activity_regularizer_fit(self):
     loss = {}
     for reg in [None, 'l2']:
-      inputs = keras.layers.Input(shape=(10,))
-      x = keras.layers.Dense(
-          10, activation='relu', activity_regularizer=reg,
-          kernel_initializer='ones', use_bias=False)(inputs)
-      outputs = keras.layers.Dense(1, activation='sigmoid',
-                                   kernel_initializer='ones', use_bias=False)(x)
-      model = keras.Model(inputs, outputs)
+      layers = [
+          keras.layers.Dense(
+              10, activation='relu', activity_regularizer=reg,
+              kernel_initializer='ones', use_bias=False),
+          keras.layers.Dense(
+              1, activation='sigmoid', kernel_initializer='ones',
+              use_bias=False),
+      ]
+
+      model = testing_utils.get_model_from_layers(
+          layers, input_shape=(10,))
 
       x = np.ones((10, 10), 'float32')
       y = np.ones((10, 1), 'float32')
@@ -372,15 +377,14 @@ class TrainingTest(keras_parameterized.TestCase):
     self.assertLess(loss[None], loss['l2'])
 
   @keras_parameterized.run_all_keras_modes
+  @keras_parameterized.run_with_all_model_types
   def test_activity_regularizer_loss_value(self):
-    inputs = keras.layers.Input(shape=(10,))
-    outputs = keras.layers.Dense(
-        1,
-        kernel_initializer=keras.initializers.zeros(),
-        bias_initializer=keras.initializers.ones(),
-        activity_regularizer='l2')(
-            inputs)
-    model = keras.Model(inputs, outputs)
+    layer = keras.layers.Dense(
+        1, kernel_initializer=keras.initializers.zeros(),
+        bias_initializer=keras.initializers.ones(), activity_regularizer='l2')
+
+    model = testing_utils.get_model_from_layers([layer], input_shape=(10,))
+
     x = np.ones((10, 10), 'float32')
     y = np.ones((10, 1), 'float32')
     optimizer = RMSPropOptimizer(learning_rate=0.001)
-- 
GitLab


From 5cfd3b3876ea52608b3a64740d71eeef5ffbbec8 Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Mon, 14 Jan 2019 12:58:49 -0800
Subject: [PATCH 0695/1095] [tf.data] Turn the `map_parallelization`
 optimization on by default.

This CL:
- turns the `map_parallelization` optimization on by default
- fixes a bug in the optimization code that would result in a segfault if two consecutive map transformations were both parallelized
- fixes a bug in the OptimizationOptions._static_optimizations() method that would cause `apply_default_optimization = False` to override explicitly enabled default optimizations

PiperOrigin-RevId: 229237263
---
 .../core/grappler/optimizers/data/BUILD       |  2 -
 .../optimizers/data/map_parallelization.cc    | 28 +++++----
 .../kernel_tests/optimization/BUILD           |  1 +
 .../optimization/assert_next_dataset_test.py  |  7 +++
 .../optimization/filter_fusion_test.py        |  1 +
 .../optimization/hoist_random_uniform_test.py | 10 +--
 .../optimization/latency_all_edges_test.py    |  1 +
 .../optimization/make_numa_aware_test.py      |  1 +
 .../optimization/map_and_batch_fusion_test.py |  4 ++
 .../map_and_filter_fusion_test.py             |  2 +
 .../optimization/map_fusion_test.py           |  1 +
 .../optimization/map_parallelization_test.py  | 63 +++++++++++++------
 .../optimization/map_vectorization_test.py    |  2 +
 .../optimization/model_dataset_test.py        |  1 +
 .../optimization/noop_elimination_test.py     |  4 ++
 .../optimization/optimize_dataset_test.py     | 48 ++++++++++----
 .../shuffle_and_repeat_fusion_test.py         |  4 ++
 .../experimental/ops/optimization_options.py  | 18 +++---
 .../python/data/kernel_tests/map_test.py      |  2 +-
 19 files changed, 139 insertions(+), 61 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/data/BUILD b/tensorflow/core/grappler/optimizers/data/BUILD
index 3bfaef4be1..d95edc5623 100644
--- a/tensorflow/core/grappler/optimizers/data/BUILD
+++ b/tensorflow/core/grappler/optimizers/data/BUILD
@@ -447,12 +447,10 @@ cc_library(
     deps = [
         ":graph_utils",
         "//tensorflow/core/grappler:mutable_graph_view",
-        "//tensorflow/core:lib",
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/clusters:cluster",
-        "//tensorflow/core/grappler/utils:topological_sort",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
     ] + tf_protos_all(),
diff --git a/tensorflow/core/grappler/optimizers/data/map_parallelization.cc b/tensorflow/core/grappler/optimizers/data/map_parallelization.cc
index 8e49f908a7..d335117585 100644
--- a/tensorflow/core/grappler/optimizers/data/map_parallelization.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_parallelization.cc
@@ -28,6 +28,10 @@ namespace tensorflow {
 namespace grappler {
 namespace {
 
+static const char* kMapDataset = "MapDataset";
+static const char* kParallelMapDataset = "ParallelMapDataset";
+static constexpr int kAutotune = -1;
+
 bool CanParallelize(const FunctionDef& function,
                     const FunctionLibraryDefinition& library) {
   if (!function.signature().is_stateful()) return true;
@@ -45,16 +49,17 @@ bool CanParallelize(const FunctionDef& function,
   return true;
 }
 
-NodeDef MakeParallelMap(const NodeDef& map_node, MutableGraphView* graph) {
-  NodeDef parallel_map = map_node;
-  graph_utils::SetUniqueGraphNodeName("parallel_map", graph->graph(),
+NodeDef MakeParallelMap(const string& name, MutableGraphView* graph) {
+  // The inputs of the node to be parallelized could be changed by the
+  // optimization pass, so we need to look it up in the modified graph.
+  int index = graph_utils::FindGraphNodeWithName(name, *graph->graph());
+  DCHECK_NE(index, -1) << "Failed to find node " << name
+                       << " in the optimized graph.";
+  NodeDef parallel_map = graph->graph()->node(index);
+  graph_utils::SetUniqueGraphNodeName(kParallelMapDataset, graph->graph(),
                                       &parallel_map);
-  parallel_map.set_op("ParallelMapDataset");
-  // TODO(b/114475558): We want to set `num_parallel_calls` to a special value,
-  // so that dynamic tunning will pick the optimal value at runtime. Because
-  // this feature is not yet implemented, we set it to 2, which is the smallest
-  // value that introduces parallelism.
-  auto* num_parallel_calls = graph_utils::AddScalarConstNode(2, graph);
+  parallel_map.set_op(kParallelMapDataset);
+  auto* num_parallel_calls = graph_utils::AddScalarConstNode(kAutotune, graph);
   parallel_map.add_input(num_parallel_calls->name());
 
   return parallel_map;
@@ -70,7 +75,7 @@ Status MapParallelization::Optimize(Cluster* cluster, const GrapplerItem& item,
   FunctionLibraryDefinition function_library(OpRegistry::Global(),
                                              item.graph.library());
   auto get_map_node = [](const NodeDef& node) -> const NodeDef* {
-    if (node.op() == "MapDataset") return &node;
+    if (node.op() == kMapDataset) return &node;
     return nullptr;
   };
 
@@ -82,7 +87,8 @@ Status MapParallelization::Optimize(Cluster* cluster, const GrapplerItem& item,
         function_library.Find(map_node->attr().at("f").func().name());
     if (!CanParallelize(*function, function_library)) continue;
 
-    auto* parallel_map = graph.AddNode(MakeParallelMap(*map_node, &graph));
+    auto* parallel_map =
+        graph.AddNode(MakeParallelMap(map_node->name(), &graph));
     graph.UpdateFanouts(map_node->name(), parallel_map->name());
     nodes_to_delete.insert(map_node->name());
   }
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD b/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD
index bf868ebe79..93a9aca1f2 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD
@@ -190,6 +190,7 @@ py_test(
     ],
     deps = [
         "//tensorflow/python:client_testlib",
+        "//tensorflow/python:constant_op",
         "//tensorflow/python:control_flow_ops",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:errors",
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/assert_next_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/assert_next_dataset_test.py
index 9b8248a78d..e05dcbd9d5 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/assert_next_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/assert_next_dataset_test.py
@@ -31,11 +31,17 @@ class AssertNextDatasetTest(test_base.DatasetTestBase):
   def testAssertNext(self):
     dataset = dataset_ops.Dataset.from_tensors(0).apply(
         optimization.assert_next(["Map"])).map(lambda x: x)
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    dataset = dataset.with_options(options)
     self.assertDatasetProduces(dataset, expected_output=[0])
 
   def testAssertNextInvalid(self):
     dataset = dataset_ops.Dataset.from_tensors(0).apply(
         optimization.assert_next(["Whoops"])).map(lambda x: x)
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    dataset = dataset.with_options(options)
     self.assertDatasetProduces(
         dataset,
         expected_error=(
@@ -48,6 +54,7 @@ class AssertNextDatasetTest(test_base.DatasetTestBase):
         optimization.assert_next(["Map", "Whoops"])).map(lambda x: x)
     options = dataset_ops.Options()
     options.experimental_autotune = False
+    options.experimental_optimization.apply_default_optimizations = False
     dataset = dataset.with_options(options)
     self.assertDatasetProduces(
         dataset,
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/filter_fusion_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/filter_fusion_test.py
index 3ce921b5ef..525ae2c54e 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/filter_fusion_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/filter_fusion_test.py
@@ -71,6 +71,7 @@ class FilterFusionTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     dataset = dataset.cache()
     options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
     options.experimental_optimization.filter_fusion = True
     dataset = dataset.with_options(options)
     expected_output = []
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/hoist_random_uniform_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/hoist_random_uniform_test.py
index f080891f2e..08a44e572b 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/hoist_random_uniform_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/hoist_random_uniform_test.py
@@ -22,7 +22,6 @@ from absl.testing import parameterized
 from tensorflow.python.data.experimental.ops import optimization
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
@@ -65,12 +64,7 @@ class HoistRandomUniformTest(test_base.DatasetTestBase, parameterized.TestCase):
 
   def _testDataset(self, dataset):
     previous_result = 0
-    if context.executing_eagerly():
-      iterator = dataset.__iter__()
-      get_next = iterator._next_internal  # pylint: disable=protected-access
-    else:
-      iterator = dataset_ops.make_one_shot_iterator(dataset)
-      get_next = iterator.get_next
+    get_next = self.getNext(dataset)
     for _ in range(5):
       result = self.evaluate(get_next())
       self.assertLessEqual(1, result)
@@ -91,6 +85,7 @@ class HoistRandomUniformTest(test_base.DatasetTestBase, parameterized.TestCase):
             ["Zip[0]", "Map"] if will_optimize else ["Map"])).map(function)
 
     options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
     options.experimental_optimization.hoist_random_uniform = True
     dataset = dataset.with_options(options)
     self._testDataset(dataset)
@@ -107,6 +102,7 @@ class HoistRandomUniformTest(test_base.DatasetTestBase, parameterized.TestCase):
     dataset = dataset_ops.Dataset.range(5).apply(
         optimization.assert_next(["Zip[0]", "Map"])).map(random_with_capture)
     options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
     options.experimental_optimization.hoist_random_uniform = True
     dataset = dataset.with_options(options)
     self._testDataset(dataset)
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/latency_all_edges_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/latency_all_edges_test.py
index 8af86da852..4fd982d122 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/latency_all_edges_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/latency_all_edges_test.py
@@ -35,6 +35,7 @@ class LatencyAllEdgesTest(stats_dataset_test_base.StatsDatasetTestBase):
             ["LatencyStats", "Map", "LatencyStats", "Prefetch",
              "LatencyStats"])).map(lambda x: x * x).prefetch(1)
     options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
     options.experimental_stats.latency_all_edges = True
     options.experimental_stats.aggregator = aggregator
     dataset = dataset.with_options(options)
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/make_numa_aware_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/make_numa_aware_test.py
index 2386dd5f11..d79ae4387c 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/make_numa_aware_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/make_numa_aware_test.py
@@ -34,6 +34,7 @@ class MakeNumaAwareTest(test_base.DatasetTestBase):
             batching.map_and_batch(lambda x: x * x, 10))
     options = dataset_ops.Options()
     options.experimental_numa_aware = True
+    options.experimental_optimization.apply_default_optimizations = False
     dataset = dataset.with_options(options)
     self.assertDatasetProduces(
         dataset, expected_output=[[x * x for x in range(10)]])
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/map_and_batch_fusion_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/map_and_batch_fusion_test.py
index e2ff3116ec..dc7bb9d6a3 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/map_and_batch_fusion_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/map_and_batch_fusion_test.py
@@ -31,6 +31,10 @@ class MapAndBatchFusionTest(test_base.DatasetTestBase):
     dataset = dataset_ops.Dataset.range(10).apply(
         optimization.assert_next(
             ["MapAndBatch"])).map(lambda x: x * x).batch(10)
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    options.experimental_optimization.map_and_batch_fusion = True
+    dataset = dataset.with_options(options)
     self.assertDatasetProduces(
         dataset, expected_output=[[x * x for x in range(10)]])
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/map_and_filter_fusion_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/map_and_filter_fusion_test.py
index fa1d673065..7b0cc56973 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/map_and_filter_fusion_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/map_and_filter_fusion_test.py
@@ -83,6 +83,7 @@ class MapAndFilterFusionTest(test_base.DatasetTestBase, parameterized.TestCase):
         optimization.assert_next(
             ["Map", "FilterByLastComponent"])).map(function).filter(predicate)
     options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
     options.experimental_optimization.map_and_filter_fusion = True
     dataset = dataset.with_options(options)
     self._testMapAndFilter(dataset, function, predicate)
@@ -101,6 +102,7 @@ class MapAndFilterFusionTest(test_base.DatasetTestBase, parameterized.TestCase):
         optimization.assert_next(["Map",
                                   "Filter"])).map(function).filter(predicate)
     options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
     options.experimental_optimization.map_and_filter_fusion = True
     dataset = dataset.with_options(options)
     self._testMapAndFilter(dataset, function, predicate)
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/map_fusion_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/map_fusion_test.py
index defdaf0440..b3a7304b4e 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/map_fusion_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/map_fusion_test.py
@@ -74,6 +74,7 @@ class MapFusionTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     dataset = dataset.cache()
     options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
     options.experimental_optimization.map_fusion = True
     dataset = dataset.with_options(options)
     expected_output = []
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/map_parallelization_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/map_parallelization_test.py
index d8dd31fee8..60649cd3ed 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/map_parallelization_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/map_parallelization_test.py
@@ -22,12 +22,13 @@ from absl.testing import parameterized
 from tensorflow.python.data.experimental.ops import optimization
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
-from tensorflow.python.ops import random_ops
+from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
 
 
@@ -42,36 +43,58 @@ def _map_parallelization_test_cases():
     with ops.control_dependencies([assert_op]):
       return x
 
-  def random(_):
-    return random_ops.random_uniform([],
-                                     minval=0,
-                                     maxval=10,
-                                     dtype=dtypes.int64,
-                                     seed=42)
-
-  def assert_with_random(x):
-    x = assert_greater(x)
-    return random(x)
-
-  return (("Identity", identity, True), ("Increment", increment, True),
-          ("AssertGreater", assert_greater, True), ("Random", random, False),
-          ("AssertWithRandom", assert_with_random, False))
+  return (("Identity", identity, True),
+          ("Increment", increment, True),
+          ("AssertGreater", assert_greater, True))
 
 
 @test_util.run_all_in_graph_and_eager_modes
 class MapParallelizationTest(test_base.DatasetTestBase, parameterized.TestCase):
 
   @parameterized.named_parameters(*_map_parallelization_test_cases())
-  def testMapParallelization(self, function, should_optimize):
-    next_nodes = ["ParallelMap"] if should_optimize else ["Map"]
+  def testMapParallelization(self, function, should_be_parallel):
+    next_nodes = ["ParallelMap"] if should_be_parallel else ["Map"]
     dataset = dataset_ops.Dataset.range(5).apply(
         optimization.assert_next(next_nodes)).map(function)
     options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    options.experimental_optimization.map_parallelization = True
+    dataset = dataset.with_options(options)
+    self.assertDatasetProduces(
+        dataset, expected_output=[function(x) for x in range(5)])
+
+  def testMapParallelizationWithCapturedConstant(self):
+    """Tests that functions with captured constants are parallelized."""
+
+    captured_t = constant_op.constant(42, dtype=dtypes.int64)
+    def fn(x):
+      return x + captured_t
+    dataset = dataset_ops.Dataset.range(5).apply(
+        optimization.assert_next(["ParallelMap"])).map(fn)
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    options.experimental_optimization.map_parallelization = True
+    dataset = dataset.with_options(options)
+    self.assertDatasetProduces(
+        dataset, expected_output=[x + 42 for x in range(5)])
+
+  def testMapParallelizationWithCapturedVariable(self):
+    """Tests that functions with captured variables are not parallelized."""
+
+    captured_t = variables.Variable(42, dtype=dtypes.int64)
+    def fn(x):
+      return x + captured_t
+    dataset = dataset_ops.Dataset.range(5).apply(
+        optimization.assert_next(["Map"])).map(fn)
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
     options.experimental_optimization.map_parallelization = True
     dataset = dataset.with_options(options)
-    if should_optimize:
-      self.assertDatasetProduces(
-          dataset, expected_output=[function(x) for x in range(5)])
+    self.evaluate(variables.global_variables_initializer())
+    self.assertDatasetProduces(
+        dataset,
+        expected_output=[x + 42 for x in range(5)],
+        requires_initialization=True)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
index 65fa2bac17..405b038eb2 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
@@ -349,6 +349,7 @@ class MapVectorizationTest(test_base.DatasetTestBase, parameterized.TestCase):
       dataset = dataset.map(map_fn, num_parallel_calls)
       dataset = dataset.batch(100)
       options = dataset_ops.Options()
+      options.experimental_optimization.apply_default_optimizations = False
       options.experimental_optimization.map_and_batch_fusion = False
       dataset = dataset.with_options(options)
       return dataset
@@ -357,6 +358,7 @@ class MapVectorizationTest(test_base.DatasetTestBase, parameterized.TestCase):
     optimized = _make_dataset(["Batch", map_node_name]
                               if expect_optimized else [map_node_name, "Batch"])
     options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
     options.experimental_optimization.map_vectorization = True
     optimized = optimized.with_options(options)
     return unoptimized, optimized
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/model_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/model_dataset_test.py
index 0f0274b41f..c6b9db0d52 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/model_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/model_dataset_test.py
@@ -35,6 +35,7 @@ class ModelDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
         optimization.assert_next(["Model"]))
     options = dataset_ops.Options()
     options.experimental_autotune = True
+    options.experimental_optimization.apply_default_optimizations = False
     dataset = dataset.with_options(options)
 
     iterator = dataset_ops.make_one_shot_iterator(dataset)
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/noop_elimination_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/noop_elimination_test.py
index 8058f53eea..74f620e37d 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/noop_elimination_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/noop_elimination_test.py
@@ -41,6 +41,10 @@ class NoopEliminationTest(test_base.DatasetTestBase):
             ["FiniteRepeat", "FiniteSkip", "Prefetch", "MemoryCacheImpl"]))
     dataset = dataset.repeat(some_tensor).skip(5).take(-1).skip(0).repeat(
         1).prefetch(0).prefetch(1).cache()
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    options.experimental_optimization.noop_elimination = True
+    dataset = dataset.with_options(options)
     self.assertDatasetProduces(dataset, expected_output=range(5))
 
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
index c111567c1c..a08016ca89 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
@@ -106,7 +106,9 @@ class OptimizeDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
   def testOptimizationStatefulFunction(self):
     dataset = dataset_ops.Dataset.range(
         10).map(lambda _: random_ops.random_uniform([])).batch(10)
-    dataset = dataset_ops._OptimizeDataset(dataset, [])
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    dataset = dataset.with_options(options)
     get_next = self.getNext(dataset)
     self.evaluate(get_next())
 
@@ -114,7 +116,9 @@ class OptimizeDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
   def testSkipEagerOptimizationLargeInputFromTensor(self):
     input_t = array_ops.placeholder(dtypes.int32, (None, None, None))
     dataset = dataset_ops.Dataset.from_tensors(input_t)
-    dataset = dataset_ops._OptimizeDataset(dataset, [])
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    dataset = dataset.with_options(options)
     iterator = dataset_ops.make_initializable_iterator(dataset)
     init_op = iterator.initializer
     get_next = iterator.get_next()
@@ -127,7 +131,9 @@ class OptimizeDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
   def testSkipEagerOptimizationLargeInputFromTensorSlices(self):
     input_t = array_ops.placeholder(dtypes.int32, (None, None, None, None))
     dataset = dataset_ops.Dataset.from_tensor_slices(input_t)
-    dataset = dataset_ops._OptimizeDataset(dataset, [])
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    dataset = dataset.with_options(options)
     iterator = dataset_ops.make_initializable_iterator(dataset)
     init_op = iterator.initializer
     get_next = iterator.get_next()
@@ -147,7 +153,10 @@ class OptimizeDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
     dataset = dataset_ops.Dataset.range(1)
     dataset = dataset.flat_map(flat_map_fn)
-    dataset = dataset_ops._OptimizeDataset(dataset, ["noop_elimination"])
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    options.experimental_optimization.noop_elimination = True
+    dataset = dataset.with_options(options)
     self.assertDatasetProduces(dataset, expected_output=[0])
 
   def testOptimizationNestedDatasetWithModifiedRetval(self):
@@ -163,10 +172,8 @@ class OptimizeDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     dataset = dataset_ops.Dataset.range(1)
     dataset = dataset.flat_map(flat_map_fn)
 
-    # TODO(b/120558523): We use Options instead of _OptimizeDataset directly
-    # here because of a bug with chaining _OptimizeDatasets when there are
-    # nested dataset functions
     options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
     options.experimental_optimization.map_and_batch_fusion = True
     dataset = dataset.with_options(options)
     self.assertDatasetProduces(dataset, expected_output=[[0]])
@@ -179,7 +186,9 @@ class OptimizeDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
         threadpool.PrivateThreadPool(
             2, display_name="private_thread_pool_%d" % 2))
 
-    dataset = dataset_ops._OptimizeDataset(dataset, [])
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    dataset = dataset.with_options(options)
     self.assertDatasetProduces(
         dataset,
         expected_output=[list(range(10))],
@@ -193,14 +202,20 @@ class OptimizeDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     dataset = dataset.apply(optimization.assert_next(["MemoryCacheImpl"]))
     dataset = dataset.skip(0)  # Should be removed by noop elimination
     dataset = dataset.cache()
-    dataset = dataset_ops._OptimizeDataset(dataset, ["noop_elimination"])
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    options.experimental_optimization.noop_elimination = True
+    dataset = dataset.with_options(options)
     self.assertDatasetProduces(dataset, expected_output=[0])
 
   def testOptimizationNonSerializableAsDirectInput(self):
     """Tests that non-serializable dataset can be OptimizeDataset's input."""
     dataset = dataset_ops.Dataset.from_tensors(0)
     dataset = dataset.apply(optimization.non_serializable())
-    dataset = dataset_ops._OptimizeDataset(dataset, ["noop_elimination"])
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    options.experimental_optimization.noop_elimination = True
+    dataset = dataset.with_options(options)
     self.assertDatasetProduces(dataset, expected_output=[0])
 
   @parameterized.named_parameters(_generate_captured_refvar_test_cases())
@@ -217,6 +232,7 @@ class OptimizeDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       unoptimized_dataset = dataset_fn(variable)
 
       options = dataset_ops.Options()
+      options.experimental_optimization.apply_default_optimizations = False
       options.experimental_optimization.noop_elimination = True
       options.experimental_optimization.map_and_batch_fusion = True
       optimized_dataset = unoptimized_dataset.with_options(options)
@@ -252,6 +268,7 @@ class OptimizeDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     options = dataset_ops.Options()
     expected_optimizations = [
         "map_and_batch_fusion",
+        "map_parallelization",
         "noop_elimination",
         "shuffle_and_repeat_fusion",
     ]
@@ -265,10 +282,15 @@ class OptimizeDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     only explicitly enabled optimizations will be applied.
     """
     options = dataset_ops.Options()
-    options.experimental_optimization.hoist_random_uniform = True
     options.experimental_optimization.apply_default_optimizations = False
-    expected_optimizations = ["hoist_random_uniform"]
-    self.assertEqual(options._static_optimizations(), expected_optimizations)
+    options.experimental_optimization.hoist_random_uniform = True
+    options.experimental_optimization.noop_elimination = True
+    expected_optimizations = [
+        "hoist_random_uniform",
+        "noop_elimination",
+    ]
+    self.assertEqual(
+        set(options._static_optimizations()), set(expected_optimizations))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/shuffle_and_repeat_fusion_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/shuffle_and_repeat_fusion_test.py
index 594b59375f..824cc680ab 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/shuffle_and_repeat_fusion_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/shuffle_and_repeat_fusion_test.py
@@ -31,6 +31,10 @@ class ShuffleAndRepeatFusionTest(test_base.DatasetTestBase):
   def testShuffleAndRepeatFusion(self):
     dataset = dataset_ops.Dataset.range(10).apply(
         optimization.assert_next(["ShuffleAndRepeat"])).shuffle(10).repeat(2)
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    options.experimental_optimization.shuffle_and_repeat_fusion = True
+    dataset = dataset.with_options(options)
     get_next = self.getNext(dataset)
 
     for _ in range(2):
diff --git a/tensorflow/python/data/experimental/ops/optimization_options.py b/tensorflow/python/data/experimental/ops/optimization_options.py
index eb95484ac5..9fe1f9499e 100644
--- a/tensorflow/python/data/experimental/ops/optimization_options.py
+++ b/tensorflow/python/data/experimental/ops/optimization_options.py
@@ -104,28 +104,32 @@ class OptimizationOptions(options.OptionsBase):
 
   def _static_optimizations(self):
     """Produces the list of enabled static optimizations."""
-    result = []
-    optimizations_to_enable = [
+    result = set()
+    all_optimizations = [
         "filter_fusion",
         "hoist_random_uniform",
+        "map_and_batch_fusion",
         "map_and_filter_fusion",
-        "map_fusion",
         "map_parallelization",
+        "map_fusion",
         "map_vectorization",
+        "noop_elimination",
+        "shuffle_and_repeat_fusion",
     ]
-    for optimization in optimizations_to_enable:
+    for optimization in all_optimizations:
       if getattr(self, optimization):
-        result.append(optimization)
+        result.add(optimization)
 
     if self.apply_default_optimizations is not False:
       # The following optimizations are turned on by default, unless the
       # user explicitly disables them.
       optimizations_to_disable = [
           "map_and_batch_fusion",
+          "map_parallelization",
           "noop_elimination",
           "shuffle_and_repeat_fusion",
       ]
       for optimization in optimizations_to_disable:
         if getattr(self, optimization) is not False:
-          result.append(optimization)
-    return result
+          result.add(optimization)
+    return list(result)
diff --git a/tensorflow/python/data/kernel_tests/map_test.py b/tensorflow/python/data/kernel_tests/map_test.py
index e84391946f..97ab6b21bc 100644
--- a/tensorflow/python/data/kernel_tests/map_test.py
+++ b/tensorflow/python/data/kernel_tests/map_test.py
@@ -84,7 +84,7 @@ def _make_coordinated_sloppy_dataset(num_elements, num_parallel_calls):
 
 
 @test_util.run_all_in_graph_and_eager_modes
-class MapDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
+class MapTest(test_base.DatasetTestBase, parameterized.TestCase):
 
   def _buildMapDataset(self, components, count):
 
-- 
GitLab


From 92971d7af78b3ff48d0c66fb57bfde886406bbe8 Mon Sep 17 00:00:00 2001
From: Igor Ganichev <iga@google.com>
Date: Mon, 14 Jan 2019 13:27:41 -0800
Subject: [PATCH 0696/1095] Add some more pfor benchmarks

PiperOrigin-RevId: 229242903
---
 .../python/ops/parallel_for/gradients_test.py | 50 +++++++++++++++----
 1 file changed, 39 insertions(+), 11 deletions(-)

diff --git a/tensorflow/python/ops/parallel_for/gradients_test.py b/tensorflow/python/ops/parallel_for/gradients_test.py
index 66110f12d9..69635c5a79 100644
--- a/tensorflow/python/ops/parallel_for/gradients_test.py
+++ b/tensorflow/python/ops/parallel_for/gradients_test.py
@@ -70,9 +70,10 @@ def fully_connected_model_fn(batch_size, activation_size, num_layers):
   return inp, model(inp)
 
 
-def lstm_model_fn(batch_size, state_size, steps):
+def lstm_model_fn(batch_size, state_size, steps, inputs_size=None):
+  inputs_size = inputs_size or state_size
   inputs = [
-      random_ops.random_normal([batch_size, state_size]) for _ in range(steps)
+      random_ops.random_normal([batch_size, inputs_size]) for _ in range(steps)
   ]
   cell = rnn_cell.BasicLSTMCell(state_size)
   init_state = cell.zero_state(batch_size, dtypes.float32)
@@ -108,8 +109,9 @@ def create_fc_batch_jacobian(batch_size, activation_size, num_layers):
   return pfor_jacobian, while_jacobian
 
 
-def create_lstm_batch_jacobian(batch_size, state_size, steps):
-  inp, output = lstm_model_fn(batch_size, state_size, steps)
+def create_lstm_batch_jacobian(batch_size, state_size, steps, inputs_size=None):
+  inp, output = lstm_model_fn(batch_size, state_size, steps,
+                              inputs_size=inputs_size)
   pfor_jacobian = gradients.batch_jacobian(output, inp, use_pfor=True)
   while_jacobian = gradients.batch_jacobian(output, inp, use_pfor=False)
   return pfor_jacobian, while_jacobian
@@ -181,9 +183,10 @@ def create_fc_per_eg_grad(batch_size, activation_size, num_layers):
   return pfor_outputs, while_outputs
 
 
-def create_lstm_per_eg_grad(batch_size, state_size, steps):
+def create_lstm_per_eg_grad(batch_size, state_size, steps, inputs_size=None):
+  inputs_size = inputs_size or state_size
   inputs = [
-      random_ops.random_normal([batch_size, state_size]) for _ in range(steps)
+      random_ops.random_normal([batch_size, inputs_size]) for _ in range(steps)
   ]
   cell = rnn_cell.BasicLSTMCell(state_size)
   init_state = cell.zero_state(batch_size, dtypes.float32)
@@ -298,6 +301,16 @@ def create_mnist_per_eg_grad(batch_size, data_format, training):
   return pfor_outputs, while_outputs
 
 
+def create_mnist_batch_jacobian(batch_size, data_format, training):
+  images = random_ops.random_uniform([batch_size, 28, 28])
+  model = Mnist(data_format)
+  logits = model(images, training=training)
+
+  pfor_jacobian = gradients.batch_jacobian(logits, images, use_pfor=True)
+  while_jacobian = gradients.batch_jacobian(logits, images, use_pfor=False)
+  return pfor_jacobian, while_jacobian
+
+
 def create_mnist_per_eg_jacobian(batch_size, data_format, training):
   images = random_ops.random_uniform([batch_size, 28, 28])
   model = Mnist(data_format)
@@ -479,7 +492,8 @@ class GradientsTest(test.TestCase):
     self.run_and_assert_equal(pfor_jacobian, while_jacobian)
 
   def test_lstm_batch_jacobian(self):
-    pfor_jacobian, while_jacobian = create_lstm_batch_jacobian(8, 4, 2)
+    pfor_jacobian, while_jacobian = create_lstm_batch_jacobian(8, 4, 2,
+                                                               inputs_size=128)
     self.run_and_assert_equal(pfor_jacobian, while_jacobian)
 
   @test_util.disable_xla("This test never passed for XLA")
@@ -569,7 +583,7 @@ class GradientsBenchmarks(test.Benchmark):
       for _ in range(iters):
         self.evaluate(targets)
       end = time.time()
-    avg_time_ms = 1000 * (end - begin) / iters
+    avg_time_ms = (1000 * (end - begin)) / iters
     self.report_benchmark(iters=iters, wall_time=avg_time_ms, name=name)
     return avg_time_ms
 
@@ -581,7 +595,8 @@ class GradientsBenchmarks(test.Benchmark):
 
   def benchmark_lstm_batch_jacobian(self):
     with ops.Graph().as_default():
-      pfor_jacobian, while_jacobian = create_lstm_batch_jacobian(100, 32, 8)
+      pfor_jacobian, while_jacobian = create_lstm_batch_jacobian(
+          100, 32, 8, inputs_size=128)
       self._run(pfor_jacobian, 100, name="lstm_batch_jacobian_pfor")
       self._run(while_jacobian, 20, name="lstm_batch_jacobian_while")
 
@@ -630,13 +645,26 @@ class GradientsBenchmarks(test.Benchmark):
 
   def benchmark_mnist_per_eg_jacobian(self):
     with ops.Graph().as_default():
-      data_format = ("channels_first"
-                     if test.is_gpu_available() else "channels_last")
+      if test.is_gpu_available():
+        data_format = "channels_first"
+      else:
+        data_format = "channels_last"
       pfor_outputs, while_outputs = create_mnist_per_eg_jacobian(
           16, data_format, training=True)
       self._run(pfor_outputs, 20, name="mnist_per_eg_jacobian_pfor")
       self._run(while_outputs, 20, name="mnist_per_eg_jacobian_while")
 
+  def benchmark_mnist_batch_jacobian(self):
+    with ops.Graph().as_default():
+      if test.is_gpu_available():
+        data_format = "channels_first"
+      else:
+        data_format = "channels_last"
+      pfor_outputs, while_outputs = create_mnist_batch_jacobian(
+          128, data_format, training=True)
+      self._run(pfor_outputs, 20, name="mnist_batch_jacobian_pfor")
+      self._run(while_outputs, 20, name="mnist_batch_jacobian_while")
+
   def benchmark_fc_per_eg_jacobian(self):
     with ops.Graph().as_default():
       jacobians, per_eg_jacobians_pfor, per_eg_jacobians_while = (
-- 
GitLab


From 26f2dd9346139dfab43d56783ea76dff90dce1eb Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Mon, 14 Jan 2019 13:31:35 -0800
Subject: [PATCH 0697/1095] XLABuilder -- Uniquify constant names when constant
 folding xla_builder

Constant folding can introduce multiple constant nodes in a graph, previously we use the same name for them, which could break the invariance that the names are unique in a graph.

PiperOrigin-RevId: 229243764
---
 tensorflow/compiler/xla/client/xla_builder.cc   |  4 +++-
 .../compiler/xla/tests/compute_constant_test.cc | 17 +++++++++++++++++
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_builder.cc
index c98f5b1140..4b360c6563 100644
--- a/tensorflow/compiler/xla/client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_builder.cc
@@ -2575,8 +2575,10 @@ StatusOr<XlaComputation> XlaBuilder::BuildConstantSubGraph(
       *const_instr.mutable_shape() = literal.shape().ToProto();
       *const_instr.mutable_literal() = literal.ToProto();
       *const_instr.mutable_opcode() = HloOpcodeString(HloOpcode::kConstant);
-      *const_instr.mutable_name() = HloOpcodeString(HloOpcode::kConstant);
+
       const_instr.set_id(handle);
+      *const_instr.mutable_name() =
+          GetFullName(const_instr.opcode(), kNameSeparator, const_instr.id());
       *entry.add_instructions() =
           const_instr;  // Add to the result constant graph.
     } else {
diff --git a/tensorflow/compiler/xla/tests/compute_constant_test.cc b/tensorflow/compiler/xla/tests/compute_constant_test.cc
index 960f496977..ef800b8ef6 100644
--- a/tensorflow/compiler/xla/tests/compute_constant_test.cc
+++ b/tensorflow/compiler/xla/tests/compute_constant_test.cc
@@ -166,6 +166,23 @@ TEST_F(ComputeConstantTest, GetDimensionSize) {
   }
 }
 
+TEST_F(ComputeConstantTest, MultipleGetDimensionSize) {
+  for (ClientType client_type : client_types) {
+    Client* client = ClientOrDie(platform_, client_type);
+    XlaBuilder b(TestName());
+    auto add =
+        Add(ConstantR2<float>(&b, {{1.0f}}), ConstantR2<float>(&b, {{1.0f}}));
+    auto get_dimension_size = GetDimensionSize(add, 0);
+    auto get_dimension_size_2 = GetDimensionSize(add, 0);
+    auto add_2 = Add(get_dimension_size, get_dimension_size_2);
+    EXPECT_TRUE(IsConstant(add_2, &b));
+
+    TF_ASSERT_OK_AND_ASSIGN(auto value,
+                            ComputeConstantScalar<uint32>(client, add_2, &b));
+    EXPECT_EQ(value, 2);
+  }
+}
+
 // Test computation of an expression interspersed with param nodes but
 // the expression does not depend on the param nodes.
 TEST_F(ComputeConstantTest, UnrelatedParam) {
-- 
GitLab


From f6709559bebf6cdc9a015d4740226d905ac6f6dc Mon Sep 17 00:00:00 2001
From: Bixia Zheng <bixia@google.com>
Date: Mon, 14 Jan 2019 13:48:19 -0800
Subject: [PATCH 0698/1095] [XLA:GPU] Enhance kLoop fusion unroll heuristic.

Avoid unrolling kLoop fusion with instructions such as kReduce that are likely
translated into loops in LLVM IR and can prevent vectorization.

PiperOrigin-RevId: 229246919
---
 .../xla/service/gpu/ir_emitter_unnested.cc    | 27 ++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
index 34ddeb1d41..294a454931 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include <algorithm>
 #include <cstring>
+#include <iterator>
 #include <memory>
 #include <string>
 #include <vector>
@@ -2080,12 +2081,36 @@ Status IrEmitterUnnested::EmitTargetElementLoopInThunk(
   return Status::OK();
 }
 
+namespace {
+
+// Returns true if the fusion contains any instruction that is likely
+// translated to complex LLVM IR, such as loops, and prevent vectorization.
+bool MayPreventVectorization(const HloInstruction& fusion_hlo) {
+  CHECK_EQ(fusion_hlo.opcode(), HloOpcode::kFusion);
+  return absl::c_any_of(
+      fusion_hlo.fused_instructions_computation()->instructions(),
+      [&](const HloInstruction* instr) {
+        switch (instr->opcode()) {
+          case HloOpcode::kReduce:
+          case HloOpcode::kReduceWindow:
+          case HloOpcode::kSort:
+          case HloOpcode::kDot:
+            return true;
+          default:
+            return false;
+        }
+      });
+}
+
+}  // namespace
+
 Status IrEmitterUnnested::EmitTargetElementLoop(
     const HloInstruction& hlo,
     const llvm_ir::ElementGenerator& element_generator) {
   int unroll_factor = 1;
   // Unfused elementwise operations are usually memory bound, unroll them.
-  if (hlo.IsElementwise() || hlo.opcode() == HloOpcode::kFusion) {
+  if (hlo.IsElementwise() ||
+      (hlo.opcode() == HloOpcode::kFusion && !MayPreventVectorization(hlo))) {
     unroll_factor = ComputeMaxUnrollFactor(&hlo);
   }
 
-- 
GitLab


From 4bc799f0b27369b55eb0b6301b6ba9d48811b94e Mon Sep 17 00:00:00 2001
From: Saurabh Saxena <srbs@google.com>
Date: Mon, 14 Jan 2019 14:00:07 -0800
Subject: [PATCH 0699/1095] Enable a few control_flow_v2 tests that are now
 working.

PiperOrigin-RevId: 229249264
---
 tensorflow/python/kernel_tests/control_flow_ops_py_test.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
index 870dd8285e..104e9e82ee 100644
--- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
@@ -1816,7 +1816,6 @@ class ControlFlowTest(test.TestCase):
       self.assertEqual(0.0, sess.run(r, {p: False}))
       self.assertEqual([2.0], sess.run(r1, {p: False}))
 
-  @test_util.disable_control_flow_v2("b/116743589")
   @test_util.run_deprecated_v1
   def testCondWhile_3(self):
     self._testCondWhile_3(use_gpu=False)
@@ -2755,13 +2754,11 @@ class ControlFlowTest(test.TestCase):
       r = gradients_impl.gradients(r, v)[0]
       self.assertAllClose(1024.0, self.evaluate(r))
 
-  @test_util.disable_control_flow_v2("b/117519152")
   @test_util.run_deprecated_v1
   def testWhileCondGrad_Simple(self):
     self._testWhileCondGrad_Simple(use_gpu=False)
     self._testWhileCondGrad_Simple(use_gpu=True)
 
-  @test_util.disable_control_flow_v2("b/117276490")
   @test_util.run_deprecated_v1
   def testWhileCondGrad_UnknownShape(self):
     with self.cached_session() as sess:
@@ -2882,7 +2879,6 @@ class ControlFlowTest(test.TestCase):
       r = gradients_impl.gradients(r.values, values)[0]
       self.assertAllClose(np.array([1024.0, 1024.0]), self.evaluate(r))
 
-  @test_util.disable_control_flow_v2("b/115920078 (gradients)")
   @test_util.run_v1_only("b/120545219")
   def testCallGradInLoop(self):
     with self.cached_session() as sess:
@@ -3045,7 +3041,6 @@ class ControlFlowTest(test.TestCase):
       all_ops = x.graph.get_operations()
       self.assertFalse(any(name in op.name for op in all_ops))
 
-  @test_util.disable_control_flow_v2("b/117954949")
   @test_util.run_deprecated_v1
   def testWhileGradGradFail(self):
     theta = variables.Variable(initial_value=1.)
@@ -3116,7 +3111,6 @@ class ControlFlowTest(test.TestCase):
       self.evaluate(q.initializer)
       self.assertAllClose([1., 1.], self.evaluate(dy_dq))
 
-  @test_util.disable_control_flow_v2("b/115920078 (gradients)")
   @test_util.run_v1_only("b/120545219")
   def testIssue16504(self):
     c = constant_op.constant(np.arange(100), dtype=dtypes.float32)
-- 
GitLab


From 1965757cef0b20c5933f7dd276f41cf4f350d032 Mon Sep 17 00:00:00 2001
From: Guangda Lai <31743510+aaroey@users.noreply.github.com>
Date: Mon, 14 Jan 2019 14:07:27 -0800
Subject: [PATCH 0700/1095] Fix merge conflict correctly

---
 tensorflow/contrib/tensorrt/resources/trt_resources.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/tensorrt/resources/trt_resources.h b/tensorflow/contrib/tensorrt/resources/trt_resources.h
index d613c6ae70..11930f3671 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_resources.h
+++ b/tensorflow/contrib/tensorrt/resources/trt_resources.h
@@ -50,7 +50,7 @@ class TRTCalibrationResource : public tensorflow::ResourceBase {
     allocator_.reset();
   }
 
-  string DebugString() override {
+  string DebugString() const override {
     std::stringstream oss;
     using std::dec;
     using std::endl;
-- 
GitLab


From f6593d64cc39e304c77e4ae36530b7c04320bacc Mon Sep 17 00:00:00 2001
From: Jianwei Xie <xiejw@google.com>
Date: Mon, 14 Jan 2019 14:08:01 -0800
Subject: [PATCH 0701/1095] Automated rollback of commit
 879af3c38d2405db3ff4c96d6104cdf82a1ab368

PiperOrigin-RevId: 229251035
---
 .../python/data/experimental/ops/batching.py  |  8 ++------
 tensorflow/python/distribute/input_lib.py     | 20 +++----------------
 2 files changed, 5 insertions(+), 23 deletions(-)

diff --git a/tensorflow/python/data/experimental/ops/batching.py b/tensorflow/python/data/experimental/ops/batching.py
index ff8a9182c3..f0cf7f0a99 100644
--- a/tensorflow/python/data/experimental/ops/batching.py
+++ b/tensorflow/python/data/experimental/ops/batching.py
@@ -553,12 +553,8 @@ class _MapAndBatchDataset(dataset_ops.UnaryDataset):
     """See `Dataset.map()` for details."""
     self._input_dataset = input_dataset
 
-    if isinstance(map_func, dataset_ops.StructuredFunctionWrapper):
-      self._map_func = map_func
-    else:
-      self._map_func = dataset_ops.StructuredFunctionWrapper(
-          map_func, "tf.data.experimental.map_and_batch()",
-          dataset=input_dataset)
+    self._map_func = dataset_ops.StructuredFunctionWrapper(
+        map_func, "tf.data.experimental.map_and_batch()", dataset=input_dataset)
     self._batch_size_t = ops.convert_to_tensor(
         batch_size, dtype=dtypes.int64, name="batch_size")
     self._num_parallel_calls_t = ops.convert_to_tensor(
diff --git a/tensorflow/python/distribute/input_lib.py b/tensorflow/python/distribute/input_lib.py
index c64eea1604..3ba80c9287 100644
--- a/tensorflow/python/distribute/input_lib.py
+++ b/tensorflow/python/distribute/input_lib.py
@@ -574,19 +574,12 @@ def _split_dataset_batch(dataset, split_batch_by):
         "The batch operations can be followed by a prefetch.")
 
   batched_dataset = _get_batch_dataset(dataset)
-  prev_dataset = batched_dataset._input_dataset
-
-  num_parallel_calls = None
-  map_func = None
-
   if isinstance(batched_dataset, dataset_ops.BatchDataset):
     batch_size = batched_dataset._batch_size
     drop_remainder = batched_dataset._drop_remainder
   elif isinstance(batched_dataset, batching._MapAndBatchDataset):
     batch_size = batched_dataset._batch_size_t
     drop_remainder = batched_dataset._drop_remainder_t
-    num_parallel_calls = batched_dataset._num_parallel_calls_t
-    map_func = batched_dataset._map_func
 
   prefetch_buffer = None
   if isinstance(dataset, dataset_ops.PrefetchDataset):
@@ -594,6 +587,7 @@ def _split_dataset_batch(dataset, split_batch_by):
   elif (isinstance(dataset, dataset_ops.DatasetV1Adapter)
         and isinstance(dataset._dataset, dataset_ops.PrefetchDataset)):
     prefetch_buffer = dataset._dataset._buffer_size
+  # pylint: enable=protected-access
 
   if tensor_util.is_tensor(batch_size):
     batch_size = tensor_util.constant_value(batch_size)
@@ -601,22 +595,14 @@ def _split_dataset_batch(dataset, split_batch_by):
   if tensor_util.is_tensor(drop_remainder):
     drop_remainder = tensor_util.constant_value(drop_remainder)
 
-  if num_parallel_calls is not None and tensor_util.is_tensor(drop_remainder):
-    num_parallel_calls = tensor_util.constant_value(num_parallel_calls)
-
   if batch_size % split_batch_by:
     raise ValueError(
         "Batch size %s cannot be sharded evenly across replicas %s" % (
             batch_size, split_batch_by))
   new_batch_size = batch_size // split_batch_by
 
-  if isinstance(batched_dataset, dataset_ops.BatchDataset):
-    dataset = prev_dataset.batch(new_batch_size, drop_remainder=drop_remainder)
-  elif isinstance(batched_dataset, batching._MapAndBatchDataset):
-    dataset = prev_dataset.apply(batching.map_and_batch(
-        map_func, new_batch_size, num_parallel_calls, drop_remainder))
-  # pylint: enable=protected-access
-
+  dataset = dataset.apply(batching.unbatch())
+  dataset = dataset.batch(new_batch_size, drop_remainder=drop_remainder)
   if prefetch_buffer is not None:
     dataset = dataset.prefetch(prefetch_buffer)
   return dataset
-- 
GitLab


From f2202f39a6724da9fa4578f82da436a650a59829 Mon Sep 17 00:00:00 2001
From: Yuanzhong Xu <yuanzx@google.com>
Date: Mon, 14 Jan 2019 14:13:33 -0800
Subject: [PATCH 0702/1095] [XLA] Fix HLO verifier for mixed-precision select

kSelect should not be whitelisted.

PiperOrigin-RevId: 229252200
---
 .../compiler/xla/service/hlo_verifier.cc      |  1 -
 .../compiler/xla/service/hlo_verifier_test.cc | 36 +++++++++++++++++++
 2 files changed, 36 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/hlo_verifier.cc b/tensorflow/compiler/xla/service/hlo_verifier.cc
index 2c69c27ce5..36340d3d78 100644
--- a/tensorflow/compiler/xla/service/hlo_verifier.cc
+++ b/tensorflow/compiler/xla/service/hlo_verifier.cc
@@ -702,7 +702,6 @@ Status CheckMixedPrecisionOperands(const HloInstruction* instruction) {
     case HloOpcode::kRecv:
     case HloOpcode::kRecvDone:
     case HloOpcode::kReducePrecision:
-    case HloOpcode::kSelect:
     case HloOpcode::kTupleSelect:
     case HloOpcode::kSend:
     case HloOpcode::kSendDone:
diff --git a/tensorflow/compiler/xla/service/hlo_verifier_test.cc b/tensorflow/compiler/xla/service/hlo_verifier_test.cc
index d27c62a879..de0335a5e8 100644
--- a/tensorflow/compiler/xla/service/hlo_verifier_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_verifier_test.cc
@@ -499,5 +499,41 @@ TEST_F(HloVerifierTest, BitcastCanNotChangeElementType) {
               HasSubstr("Bitcast can not change the element type"));
 }
 
+TEST_F(HloVerifierTest, SelectMixedPrecisionNotAllowed) {
+  const char* const hlo_string = R"(
+  HloModule Module
+
+  ENTRY SelectMixedPrecisionNotAllowed {
+   p0 = pred[] parameter(0)
+   p1 = f32[32] parameter(1)
+   p2 = bf16[32] parameter(2)
+   ROOT select = f32[32] select(p0, p1, p2)
+  }
+  )";
+  TF_ASSERT_OK_AND_ASSIGN(auto module, ParseHloString(hlo_string));
+
+  auto status = verifier().Run(module.get()).status();
+  ASSERT_FALSE(status.ok());
+  EXPECT_THAT(status.error_message(),
+              HasSubstr("Seen floating point types of different precisions"));
+}
+
+TEST_F(HloVerifierTestAllowMixedPrecision, SelectMixedPrecisionAllowed) {
+  const char* const hlo_string = R"(
+  HloModule Module
+
+  ENTRY SelectMixedPrecisionAllowed {
+   p0 = pred[] parameter(0)
+   p1 = f32[32] parameter(1)
+   p2 = bf16[32] parameter(2)
+   ROOT select = f32[32] select(p0, p1, p2)
+  }
+  )";
+  TF_ASSERT_OK_AND_ASSIGN(auto module, ParseHloString(hlo_string));
+
+  auto status = verifier().Run(module.get()).status();
+  ASSERT_TRUE(status.ok());
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From 2af4bcf9ded4a872d6675397ebff396a3bdbde5b Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Mon, 14 Jan 2019 14:16:43 -0800
Subject: [PATCH 0703/1095] [TF:XLA] Bump open source llvm revision to r350980

PiperOrigin-RevId: 229252812
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 46c1261be2..9a699ae243 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -502,11 +502,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "llvm",
         build_file = clean_dep("//third_party/llvm:llvm.autogenerated.BUILD"),
-        sha256 = "1b5e02d72413ae0de88a4457ff549434ab9c7b4f583f628eb6f29a9d199e280f",
-        strip_prefix = "llvm-ec5ab87c7698f6ede8905f833db67cd16449e360",
+        sha256 = "066d58d687ea0a97d36135d4aecf6ea31b900cfc74582b2349cadf0c7e4a04fd",
+        strip_prefix = "llvm-06a887ac8a688a23aa09252f44e9bb95797d057f",
         urls = [
-            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/ec5ab87c7698f6ede8905f833db67cd16449e360.tar.gz",
-            "https://github.com/llvm-mirror/llvm/archive/ec5ab87c7698f6ede8905f833db67cd16449e360.tar.gz",
+            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/06a887ac8a688a23aa09252f44e9bb95797d057f.tar.gz",
+            "https://github.com/llvm-mirror/llvm/archive/06a887ac8a688a23aa09252f44e9bb95797d057f.tar.gz",
         ],
     )
 
-- 
GitLab


From fec769a899962e2b4792041b4b6baf80b30111c6 Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Mon, 14 Jan 2019 14:24:49 -0800
Subject: [PATCH 0704/1095] Enable optimizer v2 lr with eager.

PiperOrigin-RevId: 229254683
---
 .../contrib/distribute/python/keras_test.py   |  3 --
 .../keras/optimizer_v2/adadelta_test.py       | 11 ++--
 .../python/keras/optimizer_v2/adagrad_test.py | 11 ++--
 .../python/keras/optimizer_v2/adam_test.py    | 11 ++--
 .../python/keras/optimizer_v2/adamax_test.py  | 11 ++--
 .../optimizer_v2/gradient_descent_test.py     | 52 ++++++++++++-------
 .../python/keras/optimizer_v2/nadam_test.py   | 15 ++++--
 .../python/keras/optimizer_v2/optimizer_v2.py | 11 +++-
 .../keras/optimizer_v2/optimizer_v2_test.py   | 14 ++---
 .../python/keras/optimizer_v2/rmsprop_test.py | 11 ++--
 .../training/checkpointable/util_test.py      | 10 ++--
 11 files changed, 107 insertions(+), 53 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/keras_test.py b/tensorflow/contrib/distribute/python/keras_test.py
index 40916afcfa..8a607dd070 100644
--- a/tensorflow/contrib/distribute/python/keras_test.py
+++ b/tensorflow/contrib/distribute/python/keras_test.py
@@ -940,9 +940,6 @@ class TestDistributionStrategyWithDatasets(test.TestCase,
   @combinations.generate(all_strategy_combinations())
   def testOptimizerWithCallbacks(self, distribution):
     with self.cached_session():
-      # TODO(b/120946189): Investigate why default strategy + eager fails.
-      if '_Default' in distribution.__class__.__name__:
-        self.skipTest('Disable the test for default strategy.')
       with distribution.scope():
         model = get_model()
         optimizer = gradient_descent_keras.SGD(0.01)
diff --git a/tensorflow/python/keras/optimizer_v2/adadelta_test.py b/tensorflow/python/keras/optimizer_v2/adadelta_test.py
index c95af6a8ad..442a1d3189 100644
--- a/tensorflow/python/keras/optimizer_v2/adadelta_test.py
+++ b/tensorflow/python/keras/optimizer_v2/adadelta_test.py
@@ -170,11 +170,16 @@ class AdadeltaOptimizerTest(test.TestCase):
 
   def testConstructAdadeltaWithLR(self):
     opt = adadelta.Adadelta(lr=1.0, rho=0.9, epsilon=1.)
-    self.assertEqual(opt.lr, 1.0)
     opt_2 = adadelta.Adadelta(learning_rate=0.1, rho=0.9, epsilon=1., lr=1.0)
-    self.assertEqual(opt_2.lr, 1.0)
     opt_3 = adadelta.Adadelta(learning_rate=0.1, rho=0.9, epsilon=1.)
-    self.assertEqual(opt_3.lr, 0.1)
+    self.assertIsInstance(opt.lr, variables.Variable)
+    self.assertIsInstance(opt_2.lr, variables.Variable)
+    self.assertIsInstance(opt_3.lr, variables.Variable)
+
+    self.evaluate(variables.global_variables_initializer())
+    self.assertAllClose(self.evaluate(opt.lr), (1.0))
+    self.assertAllClose(self.evaluate(opt_2.lr), (1.0))
+    self.assertAllClose(self.evaluate(opt_3.lr), (0.1))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/keras/optimizer_v2/adagrad_test.py b/tensorflow/python/keras/optimizer_v2/adagrad_test.py
index cf6f6a7832..1599a54745 100644
--- a/tensorflow/python/keras/optimizer_v2/adagrad_test.py
+++ b/tensorflow/python/keras/optimizer_v2/adagrad_test.py
@@ -400,11 +400,16 @@ class AdagradOptimizerTest(test.TestCase):
 
   def testConstructAdagradWithLR(self):
     opt = adagrad.Adagrad(lr=1.0)
-    self.assertEqual(opt.lr, 1.0)
     opt_2 = adagrad.Adagrad(learning_rate=0.1, lr=1.0)
-    self.assertEqual(opt_2.lr, 1.0)
     opt_3 = adagrad.Adagrad(learning_rate=0.1)
-    self.assertEqual(opt_3.lr, 0.1)
+    self.assertIsInstance(opt.lr, variables.Variable)
+    self.assertIsInstance(opt_2.lr, variables.Variable)
+    self.assertIsInstance(opt_3.lr, variables.Variable)
+
+    self.evaluate(variables.global_variables_initializer())
+    self.assertAllClose(self.evaluate(opt.lr), (1.0))
+    self.assertAllClose(self.evaluate(opt_2.lr), (1.0))
+    self.assertAllClose(self.evaluate(opt_3.lr), (0.1))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/keras/optimizer_v2/adam_test.py b/tensorflow/python/keras/optimizer_v2/adam_test.py
index 49a9de41cd..837d0220e1 100644
--- a/tensorflow/python/keras/optimizer_v2/adam_test.py
+++ b/tensorflow/python/keras/optimizer_v2/adam_test.py
@@ -505,11 +505,16 @@ class AdamOptimizerTest(test.TestCase):
 
   def testConstructAdamWithLR(self):
     opt = adam.Adam(lr=1.0)
-    self.assertEqual(opt.lr, 1.0)
     opt_2 = adam.Adam(learning_rate=0.1, lr=1.0)
-    self.assertEqual(opt_2.lr, 1.0)
     opt_3 = adam.Adam(learning_rate=0.1)
-    self.assertEqual(opt_3.lr, 0.1)
+    self.assertIsInstance(opt.lr, variables.Variable)
+    self.assertIsInstance(opt_2.lr, variables.Variable)
+    self.assertIsInstance(opt_3.lr, variables.Variable)
+
+    self.evaluate(variables.global_variables_initializer())
+    self.assertAllClose(self.evaluate(opt.lr), (1.0))
+    self.assertAllClose(self.evaluate(opt_2.lr), (1.0))
+    self.assertAllClose(self.evaluate(opt_3.lr), (0.1))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/keras/optimizer_v2/adamax_test.py b/tensorflow/python/keras/optimizer_v2/adamax_test.py
index 339c0fe6e6..fe658761c7 100644
--- a/tensorflow/python/keras/optimizer_v2/adamax_test.py
+++ b/tensorflow/python/keras/optimizer_v2/adamax_test.py
@@ -364,11 +364,16 @@ class AdamaxOptimizerTest(test.TestCase):
 
   def testConstructAdamaxWithLR(self):
     opt = adamax.Adamax(lr=1.0)
-    self.assertEqual(opt.lr, 1.0)
     opt_2 = adamax.Adamax(learning_rate=0.1, lr=1.0)
-    self.assertEqual(opt_2.lr, 1.0)
     opt_3 = adamax.Adamax(learning_rate=0.1)
-    self.assertEqual(opt_3.lr, 0.1)
+    self.assertIsInstance(opt.lr, variables.Variable)
+    self.assertIsInstance(opt_2.lr, variables.Variable)
+    self.assertIsInstance(opt_3.lr, variables.Variable)
+
+    self.evaluate(variables.global_variables_initializer())
+    self.assertAllClose(self.evaluate(opt.lr), (1.0))
+    self.assertAllClose(self.evaluate(opt_2.lr), (1.0))
+    self.assertAllClose(self.evaluate(opt_3.lr), (0.1))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py b/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py
index 9a4178db46..333a6f288e 100644
--- a/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py
+++ b/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py
@@ -263,11 +263,16 @@ class GradientDescentOptimizerTest(test.TestCase):
 
   def testConstructSGDWithLR(self):
     opt = gradient_descent.SGD(lr=1.0)
-    self.assertEqual(opt.lr, 1.0)
     opt_2 = gradient_descent.SGD(learning_rate=0.1, lr=1.0)
-    self.assertEqual(opt_2.lr, 1.0)
     opt_3 = gradient_descent.SGD(learning_rate=0.1)
-    self.assertEqual(opt_3.lr, 0.1)
+    self.assertIsInstance(opt.lr, variables.Variable)
+    self.assertIsInstance(opt_2.lr, variables.Variable)
+    self.assertIsInstance(opt_3.lr, variables.Variable)
+
+    self.evaluate(variables.global_variables_initializer())
+    self.assertAllClose(self.evaluate(opt.lr), (1.0))
+    self.assertAllClose(self.evaluate(opt_2.lr), (1.0))
+    self.assertAllClose(self.evaluate(opt_3.lr), (0.1))
 
 
 class MomentumOptimizerTest(test.TestCase):
@@ -667,11 +672,16 @@ class MomentumOptimizerTest(test.TestCase):
       opt = gradient_descent.SGD(learning_rate=1.0, momentum=0.9, nesterov=True)
       config = opt.get_config()
       opt2 = gradient_descent.SGD.from_config(config)
-      # assert both are equal float values.
-      self.assertEqual(
-          opt._get_hyper("learning_rate"), opt2._get_hyper("learning_rate"))
-      self.assertEqual(opt._get_hyper("momentum"), opt2._get_hyper("momentum"))
-      # self.assertEqual(opt._get_hyper("decay"), opt2._get_hyper("decay"))
+      lr = opt.lr
+      lr2 = opt2.lr
+      self.evaluate(variables.global_variables_initializer())
+      self.assertAllClose(self.evaluate(lr), self.evaluate(lr2))
+      self.assertAllClose(
+          self.evaluate(opt._get_hyper("momentum")),
+          self.evaluate(opt2._get_hyper("momentum")))
+      self.assertAllClose(
+          self.evaluate(opt._get_hyper("decay")),
+          self.evaluate(opt2._get_hyper("decay")))
       var0 = variables.Variable([[1.0], [2.0]], dtype=dtypes.float32)
       loss = lambda: 3 * var0
       # learning rate variable created when calling minimize.
@@ -679,14 +689,15 @@ class MomentumOptimizerTest(test.TestCase):
       self.evaluate(variables.global_variables_initializer())
       config = opt.get_config()
       opt3 = gradient_descent.SGD.from_config(config)
-      self.assertEqual(
-          self.evaluate(opt._get_hyper("learning_rate")),
-          opt3._get_hyper("learning_rate"))
-      self.assertEqual(
+      lr3 = opt3.lr
+      self.evaluate(variables.global_variables_initializer())
+      self.assertAllClose(self.evaluate(lr), self.evaluate(lr3))
+      self.assertAllClose(
           self.evaluate(opt._get_hyper("momentum")),
-          opt3._get_hyper("momentum"))
-      # self.assertEqual(
-      #     self.evaluate(opt._get_hyper("decay")), opt3._get_hyper("decay"))
+          self.evaluate(opt3._get_hyper("momentum")))
+      self.assertAllClose(
+          self.evaluate(opt._get_hyper("decay")),
+          self.evaluate(opt3._get_hyper("decay")))
       self.assertTrue(opt3.nesterov)
 
   def testNesterovWithoutMomentum(self):
@@ -695,11 +706,16 @@ class MomentumOptimizerTest(test.TestCase):
 
   def testConstructMomentumWithLR(self):
     opt = gradient_descent.SGD(lr=1.0, momentum=0.9)
-    self.assertEqual(opt.lr, 1.0)
     opt_2 = gradient_descent.SGD(learning_rate=0.1, momentum=0.9, lr=1.0)
-    self.assertEqual(opt_2.lr, 1.0)
     opt_3 = gradient_descent.SGD(learning_rate=0.1, momentum=0.9)
-    self.assertEqual(opt_3.lr, 0.1)
+    self.assertIsInstance(opt.lr, variables.Variable)
+    self.assertIsInstance(opt_2.lr, variables.Variable)
+    self.assertIsInstance(opt_3.lr, variables.Variable)
+
+    self.evaluate(variables.global_variables_initializer())
+    self.assertAllClose(self.evaluate(opt.lr), (1.0))
+    self.assertAllClose(self.evaluate(opt_2.lr), (1.0))
+    self.assertAllClose(self.evaluate(opt_3.lr), (0.1))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/keras/optimizer_v2/nadam_test.py b/tensorflow/python/keras/optimizer_v2/nadam_test.py
index 44fad751bf..b877a80cd8 100644
--- a/tensorflow/python/keras/optimizer_v2/nadam_test.py
+++ b/tensorflow/python/keras/optimizer_v2/nadam_test.py
@@ -161,15 +161,22 @@ class NadamOptimizerTest(test.TestCase):
 
   def testConstructNAdamWithLR(self):
     opt = nadam.Nadam(lr=1.0)
-    self.assertEqual(opt.lr, 1.0)
     opt_2 = nadam.Nadam(learning_rate=0.1, lr=1.0)
-    self.assertEqual(opt_2.lr, 1.0)
     opt_3 = nadam.Nadam(learning_rate=0.1)
-    self.assertEqual(opt_3.lr, 0.1)
+    self.assertIsInstance(opt.lr, variables.Variable)
+    self.assertIsInstance(opt_2.lr, variables.Variable)
+    self.assertIsInstance(opt_3.lr, variables.Variable)
+
+    self.evaluate(variables.global_variables_initializer())
+    self.assertAllClose(self.evaluate(opt.lr), (1.0))
+    self.assertAllClose(self.evaluate(opt_2.lr), (1.0))
+    self.assertAllClose(self.evaluate(opt_3.lr), (0.1))
 
   def testConstructNAdamWithScheduleDecay(self):
     opt = nadam.Nadam(schedule_decay=0.2)
-    self.assertEqual(opt.decay, 0.2)
+    self.assertIsInstance(opt.decay, variables.Variable)
+    self.evaluate(variables.global_variables_initializer())
+    self.assertAllClose(self.evaluate(opt.decay), (0.2))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
index 0415c97a0b..bc210132b5 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
@@ -95,6 +95,11 @@ class OptimizerV2(checkpointable.CheckpointableBase):
   opt_op.run()
   ```
 
+  ### Thread Compatibility
+
+  The entire optimizer is currently thread compatible, not thread-safe. The user
+  needs to perform synchronization if necessary.
+
   ### Processing gradients before applying them.
 
   Calling `minimize()` takes care of both computing the gradients and
@@ -411,6 +416,8 @@ class OptimizerV2(checkpointable.CheckpointableBase):
         backend.set_value(self._hyper[name], value)
 
   def _get_hyper(self, name, dtype=None):
+    if not self._hypers_created:
+      self._create_hypers()
     value = self._hyper[name]
     if callable(value):
       value = value()
@@ -431,7 +438,7 @@ class OptimizerV2(checkpointable.CheckpointableBase):
       if name == "lr":
         name = "learning_rate"
       if name in self._hyper:
-        return self._hyper[name]
+        return self._get_hyper(name)
       raise e
 
   def __setattr__(self, name, value):
@@ -573,7 +580,7 @@ class OptimizerV2(checkpointable.CheckpointableBase):
 
   def _serialize_hyperparameter(self, hyperparameter_name):
     """Serialize a hyperparameter that can be a float, callable, or Tensor."""
-    value = self._get_hyper(hyperparameter_name)
+    value = self._hyper[hyperparameter_name]
     if callable(value):
       return value()
     if isinstance(value, (ops.Tensor, tf_variables.Variable,
diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
index 93a6a7e85e..374ba60460 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
@@ -266,19 +266,19 @@ class OptimizerTest(test.TestCase):
       opt = gradient_descent.SGD(learning_rate=1.0)
       config = opt.get_config()
       opt2 = gradient_descent.SGD.from_config(config)
+      lr = opt._get_hyper('learning_rate')
+      lr2 = opt2._get_hyper('learning_rate')
+      self.evaluate(variables.global_variables_initializer())
       # assert both are equal float values.
-      self.assertEqual(
-          opt._get_hyper('learning_rate'), opt2._get_hyper('learning_rate'))
+      self.assertEqual(self.evaluate(lr), self.evaluate(lr2))
       var0 = variables.Variable([[1.0], [2.0]], dtype=dtypes.float32)
       loss = lambda: 3 * var0
       # learning rate variable created when calling minimize.
       opt.minimize(loss, [var0])
-      self.evaluate(variables.global_variables_initializer())
-      config = opt.get_config()
       opt3 = gradient_descent.SGD.from_config(config)
-      self.assertEqual(
-          self.evaluate(opt._get_hyper('learning_rate')),
-          opt3._get_hyper('learning_rate'))
+      lr3 = opt3._get_hyper('learning_rate')
+      self.evaluate(variables.global_variables_initializer())
+      self.assertEqual(self.evaluate(lr), self.evaluate(lr3))
 
   @test_util.run_in_graph_and_eager_modes
   def testGradClipValue(self):
diff --git a/tensorflow/python/keras/optimizer_v2/rmsprop_test.py b/tensorflow/python/keras/optimizer_v2/rmsprop_test.py
index 67662aae20..d4bc7ce50c 100644
--- a/tensorflow/python/keras/optimizer_v2/rmsprop_test.py
+++ b/tensorflow/python/keras/optimizer_v2/rmsprop_test.py
@@ -434,11 +434,16 @@ class RMSpropOptimizerTest(test.TestCase):
 
   def testConstructRMSpropWithLR(self):
     opt = rmsprop.RMSprop(lr=1.0)
-    self.assertEqual(opt.lr, 1.0)
     opt_2 = rmsprop.RMSprop(learning_rate=0.1, lr=1.0)
-    self.assertEqual(opt_2.lr, 1.0)
     opt_3 = rmsprop.RMSprop(learning_rate=0.1)
-    self.assertEqual(opt_3.lr, 0.1)
+    self.assertIsInstance(opt.lr, variables.Variable)
+    self.assertIsInstance(opt_2.lr, variables.Variable)
+    self.assertIsInstance(opt_3.lr, variables.Variable)
+
+    self.evaluate(variables.global_variables_initializer())
+    self.assertAllClose(self.evaluate(opt.lr), (1.0))
+    self.assertAllClose(self.evaluate(opt_2.lr), (1.0))
+    self.assertAllClose(self.evaluate(opt_3.lr), (0.1))
 
   def testSlotsUniqueEager(self):
     with context.eager_mode():
diff --git a/tensorflow/python/training/checkpointable/util_test.py b/tensorflow/python/training/checkpointable/util_test.py
index 4987387dc3..78a8282f25 100644
--- a/tensorflow/python/training/checkpointable/util_test.py
+++ b/tensorflow/python/training/checkpointable/util_test.py
@@ -437,8 +437,9 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
     on_create_m_bias_slot = on_create_optimizer.get_slot(
         on_create_model._named_dense.variables[1], "m")
     status.assert_existing_objects_matched()
-    with self.assertRaises(AssertionError):
-      status.assert_consumed()
+    if not context.executing_eagerly():
+      with self.assertRaises(AssertionError):
+        status.assert_consumed()
     # Optimizer slot variables are created when the original variable is
     # restored.
     self.assertAllEqual([1.5], self.evaluate(on_create_m_bias_slot))
@@ -830,8 +831,9 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
     self.assertEqual(12., self.evaluate(new_root.var))
     new_root.optimizer = adam.Adam(0.1)
     slot_status.assert_existing_objects_matched()
-    with self.assertRaisesRegexp(AssertionError, "Unresolved object"):
-      slot_status.assert_consumed()
+    if not context.executing_eagerly():
+      with self.assertRaisesRegexp(AssertionError, "Unresolved object"):
+        slot_status.assert_consumed()
     self.assertEqual(12., self.evaluate(new_root.var))
     if context.executing_eagerly():
       # Slot variables are only created with restoring initializers when
-- 
GitLab


From 6fd7a5c291c2b2f7246def6d5e7551ace5dd6cf9 Mon Sep 17 00:00:00 2001
From: Andy Ly <lyandy@google.com>
Date: Mon, 14 Jan 2019 15:11:16 -0800
Subject: [PATCH 0705/1095] [Grappler] Change MutableGraphView mutations to
 return Status.

PiperOrigin-RevId: 229263687
---
 tensorflow/core/grappler/BUILD                |   1 +
 .../core/grappler/mutable_graph_view.cc       | 243 ++++++-----
 tensorflow/core/grappler/mutable_graph_view.h |  56 +--
 .../core/grappler/mutable_graph_view_test.cc  | 383 ++++++++++++------
 .../grappler/optimizers/data/filter_fusion.cc |   3 +-
 .../optimizers/data/hoist_random_uniform.cc   |   3 +-
 .../optimizers/data/latency_all_edges.cc      |   2 +-
 .../optimizers/data/make_numa_aware.cc        |   2 +-
 .../optimizers/data/map_and_batch_fusion.cc   |   3 +-
 .../optimizers/data/map_and_filter_fusion.cc  |   3 +-
 .../grappler/optimizers/data/map_fusion.cc    |   3 +-
 .../optimizers/data/map_parallelization.cc    |   3 +-
 .../optimizers/data/map_vectorization.cc      |   3 +-
 .../optimizers/data/noop_elimination.cc       |   2 +-
 .../data/shuffle_and_repeat_fusion.cc         |   3 +-
 15 files changed, 450 insertions(+), 263 deletions(-)

diff --git a/tensorflow/core/grappler/BUILD b/tensorflow/core/grappler/BUILD
index 55d642612b..475fa64201 100644
--- a/tensorflow/core/grappler/BUILD
+++ b/tensorflow/core/grappler/BUILD
@@ -232,5 +232,6 @@ tf_cc_test(
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
         "//tensorflow/core/grappler/inputs:trivial_test_graph_input_yielder",
+        "@com_google_absl//absl/strings",
     ],
 )
diff --git a/tensorflow/core/grappler/mutable_graph_view.cc b/tensorflow/core/grappler/mutable_graph_view.cc
index 5acdb02dee..13a530181c 100644
--- a/tensorflow/core/grappler/mutable_graph_view.cc
+++ b/tensorflow/core/grappler/mutable_graph_view.cc
@@ -27,6 +27,7 @@ limitations under the License.
 #include "tensorflow/core/graph/tensor_id.h"
 #include "tensorflow/core/grappler/op_types.h"
 #include "tensorflow/core/grappler/utils.h"
+#include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/stringpiece.h"
 #include "tensorflow/core/platform/types.h"
 
@@ -35,6 +36,26 @@ namespace grappler {
 
 namespace {
 
+const char kMissingMsg[] = "missing";
+const char kInvalidMsg[] = "invalid";
+const char kNoErrMsg[] = "";
+
+string FaninError(bool tensor_id_valid, bool node_missing) {
+  string s;
+  if (!tensor_id_valid && node_missing) {
+    s = absl::StrCat(" ", kInvalidMsg, "/", kMissingMsg);
+  } else if (!tensor_id_valid) {
+    s = absl::StrCat(" ", kInvalidMsg);
+  } else if (node_missing) {
+    s = absl::StrCat(" ", kMissingMsg);
+  }
+  return s;
+}
+
+string NodeError(bool node_missing) {
+  return node_missing ? absl::StrCat(" ", kMissingMsg) : kNoErrMsg;
+}
+
 bool IsTensorIdPortValid(const TensorId& tensor_id) {
   return tensor_id.index() >= Graph::kControlSlot;
 }
@@ -94,6 +115,8 @@ bool CanDedupControlWithRegularInput(const MutableGraphView& graph,
 }  // namespace
 
 void MutableGraphView::AddAndDedupFanouts(NodeDef* node) {
+  // TODO(lyandy): Checks for self loops, Switch control dependencies and if
+  // fanins exist.
   absl::flat_hash_set<absl::string_view> fanins;
   absl::flat_hash_set<absl::string_view> controlling_fanins;
   int pos = 0;
@@ -183,31 +206,36 @@ NodeDef* MutableGraphView::AddNode(NodeDef&& node) {
   return node_in_graph;
 }
 
-bool MutableGraphView::UpdateFanouts(absl::string_view from_node,
-                                     absl::string_view to_node) {
-  if (from_node == to_node) {
-    return false;
-  }
+Status MutableGraphView::UpdateFanouts(absl::string_view from_node,
+                                       absl::string_view to_node) {
   NodeDef* from_node_ptr = GetNode(from_node);
   NodeDef* to_node_ptr = GetNode(to_node);
   if (from_node_ptr && to_node_ptr) {
     return UpdateFanoutsInternal(from_node_ptr, to_node_ptr);
   } else if (!from_node_ptr) {
-    LOG(WARNING) << absl::Substitute(
+    return errors::Internal(absl::Substitute(
         "Can't update fanouts from '$0' to '$1', from node was not found.",
-        from_node, to_node);
-  } else {
-    LOG(WARNING) << absl::Substitute(
+        from_node, to_node));
+  } else if (!to_node_ptr) {
+    return errors::Internal(absl::Substitute(
         "Can't update fanouts from '$0' to '$1', to node was not found.",
-        from_node, to_node);
+        from_node, to_node));
+  } else {
+    return errors::Internal(
+        absl::Substitute("Can't update fanouts from '$0' to '$1', from and to "
+                         "nodes were not found.",
+                         from_node, to_node));
   }
-  return false;
+  return Status::OK();
 }
 
-bool MutableGraphView::UpdateFanoutsInternal(NodeDef* from_node,
-                                             NodeDef* to_node) {
+Status MutableGraphView::UpdateFanoutsInternal(NodeDef* from_node,
+                                               NodeDef* to_node) {
   VLOG(2) << absl::Substitute("Update fanouts from '$0' to '$1'.",
                               from_node->name(), to_node->name());
+  if (from_node == to_node) {
+    return Status::OK();
+  }
 
   // Update internal state with the new output_port->input_port edge.
   const auto add_edge = [this](const OutputPort& output_port,
@@ -221,8 +249,6 @@ bool MutableGraphView::UpdateFanoutsInternal(NodeDef* from_node,
     fanouts()[output_port].erase(input_port);
   };
 
-  bool modified = false;
-
   // For the control fanouts we do not know the input index in a NodeDef,
   // so we have to traverse all control inputs.
 
@@ -236,18 +262,17 @@ bool MutableGraphView::UpdateFanoutsInternal(NodeDef* from_node,
 
     // Can't add Switch node as a control dependency.
     if (to_node_is_switch) {
-      // This will only be printed to the log if we are trying to add a Switch
-      // as a control dependency, which if allowed will make the graph invalid.
-      LOG(WARNING) << absl::Substitute(
-          "Can't update fanouts from '$0' to '$1', to node is being added as a "
-          "control dependency when it is a Switch.",
-          from_node, to_node);
-      return false;
+      // Trying to add a Switch as a control dependency, which if allowed will
+      // make the graph invalid.
+      return errors::Internal(
+          absl::Substitute("Can't update fanouts from '$0' to '$1', to node is "
+                           "being added as a Switch control dependency.",
+                           from_node->name(), to_node->name()));
     }
 
     NodeDef* node = control_port.node;
-    modified |= RemoveControllingFaninInternal(node, from_node);
-    modified |= AddFaninInternal(node, {to_node, Graph::kControlSlot});
+    RemoveControllingFaninInternal(node, from_node);
+    AddFaninInternal(node, {to_node, Graph::kControlSlot});
   }
 
   // First we update regular fanouts. For the regular fanouts
@@ -287,7 +312,6 @@ bool MutableGraphView::UpdateFanoutsInternal(NodeDef* from_node,
     if (CanDedupControlWithRegularInput(*this, *to_node)) {
       RemoveControllingFaninInternal(input_port.node, to_node);
     }
-    modified = true;
   }
 
   // Because we update all regular fanouts of `from_node`, we can just copy
@@ -301,7 +325,7 @@ bool MutableGraphView::UpdateFanoutsInternal(NodeDef* from_node,
     max_regular_output_port().erase(from_node);
   }
 
-  return modified;
+  return Status::OK();
 }
 
 bool MutableGraphView::AddFaninInternal(NodeDef* node,
@@ -351,43 +375,46 @@ bool MutableGraphView::AddFaninInternal(NodeDef* node,
   return true;
 }
 
-bool MutableGraphView::AddFaninInternal(NodeDef* node, const TensorId& fanin) {
+Status MutableGraphView::AddRegularFanin(absl::string_view node_name,
+                                         const TensorId& fanin) {
+  NodeDef* node = GetNode(node_name);
   NodeDef* fanin_node = GetNode(fanin.node());
-  if (fanin_node == nullptr) {
-    return false;
-  }
-  return AddFaninInternal(node, {fanin_node, fanin.index()});
-}
 
-bool MutableGraphView::AddRegularFanin(absl::string_view node_name,
-                                       const TensorId& fanin) {
-  if (!IsTensorIdRegular(fanin)) {
-    return false;
-  }
-  NodeDef* node = GetNode(node_name);
-  if (node == nullptr) {
-    return false;
+  string node_err = NodeError(/*node_missing=*/node == nullptr);
+  string fanin_err = FaninError(IsTensorIdRegular(fanin),
+                                /*node_missing=*/fanin_node == nullptr);
+  if (!node_err.empty() || !fanin_err.empty()) {
+    return errors::Internal(absl::Substitute(
+        "Can't add$0 fanin '$1' as regular fanin to$2 node '$3'.", fanin_err,
+        fanin.ToString(), node_err, node_name));
   }
-  return AddFaninInternal(node, fanin);
+
+  AddFaninInternal(node, {fanin_node, fanin.index()});
+  return Status::OK();
 }
 
-bool MutableGraphView::AddControllingFanin(absl::string_view node_name,
-                                           const TensorId& fanin) {
+Status MutableGraphView::AddControllingFanin(absl::string_view node_name,
+                                             const TensorId& fanin) {
   NodeDef* node = GetNode(node_name);
-  if (node == nullptr) {
-    return false;
-  }
   NodeDef* fanin_node = GetNode(fanin.node());
-  if (fanin_node == nullptr) {
-    return false;
+
+  string node_err = NodeError(/*node_missing=*/node == nullptr);
+  string fanin_err = FaninError(IsTensorIdPortValid(fanin),
+                                /*node_missing=*/fanin_node == nullptr);
+  if (!node_err.empty() || !fanin_err.empty()) {
+    return errors::Internal(
+        absl::Substitute("Can't add$0 controlling fanin '$1' to$2 node '$3'.",
+                         fanin_err, fanin.ToString(), node_err, node_name));
   }
 
   if (!IsSwitch(*fanin_node)) {
-    return AddFaninInternal(node, {fanin_node, Graph::kControlSlot});
+    AddFaninInternal(node, {fanin_node, Graph::kControlSlot});
   } else {
     if (IsTensorIdControlling(fanin)) {
       // Can't add a Switch node control dependency.
-      return false;
+      return errors::Internal(absl::Substitute(
+          "Can't add Switch as controlling fanin '$0' to node '$1'.",
+          fanin.ToString(), node_name));
     }
     // We can't anchor control dependencies directly on the switch node: unlike
     // other nodes only one of the outputs of the switch node will be generated
@@ -399,7 +426,8 @@ bool MutableGraphView::AddControllingFanin(absl::string_view node_name,
     for (auto fanout : fanouts) {
       if (IsIdentity(*fanout.node) || IsIdentityNSingleInput(*fanout.node)) {
         if (ParseTensorName(fanout.node->input(0)) == fanin) {
-          return AddFaninInternal(node, {fanout.node, Graph::kControlSlot});
+          AddFaninInternal(node, {fanout.node, Graph::kControlSlot});
+          return Status::OK();
         }
       }
     }
@@ -420,8 +448,9 @@ bool MutableGraphView::AddControllingFanin(absl::string_view node_name,
       new_node.add_input(TensorIdToString(fanin));
       ctrl_dep_node = AddNode(std::move(new_node));
     }
-    return AddFaninInternal(node, {ctrl_dep_node, Graph::kControlSlot});
+    AddFaninInternal(node, {ctrl_dep_node, Graph::kControlSlot});
   }
+  return Status::OK();
 }
 
 bool MutableGraphView::RemoveRegularFaninInternal(NodeDef* node,
@@ -473,20 +502,22 @@ bool MutableGraphView::RemoveRegularFaninInternal(NodeDef* node,
   return modified;
 }
 
-bool MutableGraphView::RemoveRegularFanin(absl::string_view node_name,
-                                          const TensorId& fanin) {
-  if (!IsTensorIdRegular(fanin)) {
-    return false;
-  }
+Status MutableGraphView::RemoveRegularFanin(absl::string_view node_name,
+                                            const TensorId& fanin) {
   NodeDef* node = GetNode(node_name);
-  if (node == nullptr) {
-    return false;
-  }
   NodeDef* fanin_node = GetNode(fanin.node());
-  if (fanin_node == nullptr) {
-    return false;
+
+  string node_err = NodeError(/*node_missing=*/node == nullptr);
+  string fanin_err = FaninError(IsTensorIdRegular(fanin),
+                                /*node_missing=*/fanin_node == nullptr);
+  if (!node_err.empty() || !fanin_err.empty()) {
+    return errors::Internal(absl::Substitute(
+        "Can't remove$0 fanin '$1' as regular fanin from$2 node '$3'.",
+        fanin_err, fanin.ToString(), node_err, node_name));
   }
-  return RemoveRegularFaninInternal(node, {fanin_node, fanin.index()});
+
+  RemoveRegularFaninInternal(node, {fanin_node, fanin.index()});
+  return Status::OK();
 }
 
 bool MutableGraphView::RemoveControllingFaninInternal(NodeDef* node,
@@ -507,31 +538,42 @@ bool MutableGraphView::RemoveControllingFaninInternal(NodeDef* node,
   return false;
 }
 
-bool MutableGraphView::RemoveControllingFanin(
+Status MutableGraphView::RemoveControllingFanin(
     absl::string_view node_name, absl::string_view fanin_node_name) {
   NodeDef* node = GetNode(node_name);
-  if (node == nullptr) {
-    return false;
-  }
-  NodeDef* fanin = GetNode(fanin_node_name);
-  if (fanin == nullptr) {
-    return false;
+  NodeDef* fanin_node = GetNode(fanin_node_name);
+
+  string node_err = NodeError(/*node_missing=*/node == nullptr);
+  string fanin_err = NodeError(/*node_missing=*/fanin_node == nullptr);
+  if (!node_err.empty() || !fanin_err.empty()) {
+    return errors::Internal(absl::Substitute(
+        "Can't remove$0 controlling fanin '$1' from$2 node '$3'.", fanin_err,
+        AsControlDependency(string(fanin_node_name)), node_err, node_name));
   }
-  return RemoveControllingFaninInternal(node, fanin);
+
+  RemoveControllingFaninInternal(node, fanin_node);
+  return Status::OK();
 }
 
-bool MutableGraphView::RemoveAllFanins(absl::string_view node_name,
-                                       bool keep_controlling_fanins) {
+Status MutableGraphView::RemoveAllFanins(absl::string_view node_name,
+                                         bool keep_controlling_fanins) {
   NodeDef* node = GetNode(node_name);
-  if (node == nullptr || node->input().empty()) {
-    return false;
+
+  if (node == nullptr) {
+    return errors::Internal(absl::Substitute(
+        "Can't remove all fanins from missing node '$0'.", node_name));
   }
+
+  if (node->input().empty()) {
+    return Status::OK();
+  }
+
   RemoveFaninsInternal(node, keep_controlling_fanins);
   if (keep_controlling_fanins) {
     int num_non_controlling_fanins =
         NumFanins(*node, /*include_controlling_nodes=*/false);
     if (num_non_controlling_fanins == 0) {
-      return false;
+      return Status::OK();
     } else if (num_non_controlling_fanins < node->input_size()) {
       node->mutable_input()->DeleteSubrange(0, num_non_controlling_fanins);
     } else {
@@ -540,35 +582,45 @@ bool MutableGraphView::RemoveAllFanins(absl::string_view node_name,
   } else {
     node->clear_input();
   }
-  return true;
+  return Status::OK();
 }
 
-bool MutableGraphView::UpdateFanin(absl::string_view node_name,
-                                   const TensorId& from_fanin,
-                                   const TensorId& to_fanin) {
-  if (from_fanin == to_fanin || !IsTensorIdPortValid(from_fanin) ||
-      !IsTensorIdPortValid(to_fanin)) {
-    return false;
-  }
+Status MutableGraphView::UpdateFanin(absl::string_view node_name,
+                                     const TensorId& from_fanin,
+                                     const TensorId& to_fanin) {
   NodeDef* node = GetNode(node_name);
-  if (node == nullptr) {
-    return false;
-  }
-
   NodeDef* from_fanin_node = GetNode(from_fanin.node());
   NodeDef* to_fanin_node = GetNode(to_fanin.node());
-  if (from_fanin_node == nullptr || to_fanin_node == nullptr) {
-    return false;
+
+  string node_err = NodeError(/*node_missing=*/node == nullptr);
+  string from_fanin_err =
+      FaninError(IsTensorIdPortValid(from_fanin),
+                 /*node_missing=*/from_fanin_node == nullptr);
+  string to_fanin_err = FaninError(IsTensorIdPortValid(to_fanin),
+                                   /*node_missing=*/to_fanin_node == nullptr);
+  if (!node_err.empty() || !from_fanin_err.empty() || !to_fanin_err.empty()) {
+    return errors::Internal(absl::Substitute(
+        "Can't update$0 fanin '$1' to$2 fanin '$3' in$4 node '$5'.",
+        from_fanin_err, from_fanin.ToString(), to_fanin_err,
+        to_fanin.ToString(), node_err, node_name));
   }
 
   // When replacing a non control dependency fanin with a control dependency, or
   // vice versa, remove and add, so ports can be updated properly in fanout(s).
-  bool from_fanin_is_control = IsTensorIdControlling(from_fanin);
   bool to_fanin_is_control = IsTensorIdControlling(to_fanin);
   if (to_fanin_is_control && IsSwitch(*to_fanin_node)) {
     // Can't add Switch node as a control dependency.
-    return false;
+    return errors::Internal(absl::Substitute(
+        "Can't update fanin '$0' to fanin '$1' in node '$2', to fanin is a "
+        "Switch control dependency.",
+        from_fanin.ToString(), to_fanin.ToString(), node_name));
+  }
+
+  if (from_fanin == to_fanin) {
+    return Status::OK();
   }
+
+  bool from_fanin_is_control = IsTensorIdControlling(from_fanin);
   if (from_fanin_is_control || to_fanin_is_control) {
     bool modified = false;
     if (from_fanin_is_control) {
@@ -580,8 +632,7 @@ bool MutableGraphView::UpdateFanin(absl::string_view node_name,
     if (modified) {
       AddFaninInternal(node, {to_fanin_node, to_fanin.index()});
     }
-
-    return modified;
+    return Status::OK();
   }
 
   // In place mutation, requires no shifting of ports.
@@ -629,10 +680,12 @@ bool MutableGraphView::UpdateFanin(absl::string_view node_name,
     }
   }
 
-  return modified;
+  return Status::OK();
 }
 
 void MutableGraphView::DeleteNodes(const std::set<string>& nodes_to_delete) {
+  // TODO(lyandy): Check if nodes have fanouts before deleting and return
+  // Status.
   for (const string& node_name_to_delete : nodes_to_delete)
     RemoveFaninsInternal(nodes().at(node_name_to_delete),
                          /*keep_controlling_fanins=*/false);
diff --git a/tensorflow/core/grappler/mutable_graph_view.h b/tensorflow/core/grappler/mutable_graph_view.h
index 0a441d9115..59740f2c71 100644
--- a/tensorflow/core/grappler/mutable_graph_view.h
+++ b/tensorflow/core/grappler/mutable_graph_view.h
@@ -28,6 +28,7 @@ limitations under the License.
 #include "tensorflow/core/graph/tensor_id.h"
 #include "tensorflow/core/grappler/graph_view.h"
 #include "tensorflow/core/grappler/op_types.h"
+#include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/types.h"
 
 namespace tensorflow {
@@ -74,18 +75,14 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   //   1. foo1(new_bar:0, new_bar:1, other:0)
   //   2. foo2(new_bar:1, other:1)
   //   3. foo3(other:2, ^new_bar)
-  //
-  // This will return true iff the nodes are modified.
-  bool UpdateFanouts(absl::string_view from_node, absl::string_view to_node);
+  Status UpdateFanouts(absl::string_view from_node, absl::string_view to_node);
 
   // Adds regular fanin `fanin` to node `node_name`. If the node or fanin do not
   // exist in the graph, nothing will be modified in the graph. Otherwise fanin
   // will be added after existing non control dependency fanins. Control
   // dependencies will be deduped. To add control dependencies, use
   // AddControllingFanin.
-  //
-  // This will return true iff the node is modified.
-  bool AddRegularFanin(absl::string_view node_name, const TensorId& fanin);
+  Status AddRegularFanin(absl::string_view node_name, const TensorId& fanin);
 
   // Adds control dependency `fanin` to the target node named `node_name`. To
   // add regular fanins, use AddRegularFanin.
@@ -104,42 +101,35 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   // end up with:
   //
   //   fanin -> Identity{N} -^> target node.
-  //
-  // This will return true iff the node is modified.
-  bool AddControllingFanin(absl::string_view node_name, const TensorId& fanin);
+  Status AddControllingFanin(absl::string_view node_name,
+                             const TensorId& fanin);
 
   // Removes regular fanin `fanin` from node `node_name`. If the node or fanin
   // do not exist in the graph, nothing will be modified in the graph. If there
   // are multiple inputs that match the fanin, all of them will be removed. To
   // remove controlling fanins, use RemoveControllingFanin.
-  //
-  // This will return true iff the node is modified.
-  bool RemoveRegularFanin(absl::string_view node_name, const TensorId& fanin);
+  Status RemoveRegularFanin(absl::string_view node_name, const TensorId& fanin);
 
   // Removes control dependency `fanin_node_name` from the target node named
   // `node_name`. If the node or fanin do not exist in the graph, nothing will
   // be modified in the graph. To remove regular fanins, use RemoveRegualrFanin.
-  //
-  // This will return true iff the node is modified.
-  bool RemoveControllingFanin(absl::string_view node_name,
-                              absl::string_view fanin_node_name);
+  Status RemoveControllingFanin(absl::string_view node_name,
+                                absl::string_view fanin_node_name);
 
   // Removes all fanins from node `node_name`. Control dependencies will be
   // retained if keep_controlling_fanins is true.
-  //
-  // This will return true iff the node is modified.
-  bool RemoveAllFanins(absl::string_view node_name,
-                       bool keep_controlling_fanins);
+  Status RemoveAllFanins(absl::string_view node_name,
+                         bool keep_controlling_fanins);
 
   // Replaces all fanins `from_fanin` with `to_fanin` in node `node_name`. If
   // the fanins or node do not exist, nothing will be modified in the graph.
   // Control dependencies will be deduped.
-  //
-  // This will return true iff the node is modified.
-  bool UpdateFanin(absl::string_view node_name, const TensorId& from_fanin,
-                   const TensorId& to_fanin);
+  Status UpdateFanin(absl::string_view node_name, const TensorId& from_fanin,
+                     const TensorId& to_fanin);
 
   // Deletes nodes from the graph.
+  // TODO(lyandy): Check if nodes have fanouts before deleting and return
+  // Status.
   void DeleteNodes(const std::set<string>& nodes_to_delete);
 
  private:
@@ -171,7 +161,7 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   //
   // IMPORTANT: If `from_node` or `to_node` is not in the underlying graph, the
   // behavior is undefined.
-  bool UpdateFanoutsInternal(NodeDef* from_node, NodeDef* to_node);
+  Status UpdateFanoutsInternal(NodeDef* from_node, NodeDef* to_node);
 
   // Removes fanins of the deleted node from internal state. Control
   // dependencies are retained iff keep_controlling_fanins is true.
@@ -181,29 +171,13 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   // Adds fanin to node. If fanin is a control dependency, existing control
   // dependencies will be checked first before adding. Otherwise fanin will be
   // added after existing non control dependency inputs.
-  //
-  // This will return true iff the node is modified. If a control dependency
-  // already exists, the node will not be modified.
   bool AddFaninInternal(NodeDef* node, const OutputPort& fanin);
 
-  // Adds fanin to node. If the node or fanin do not exist in the graph, nothing
-  // will be modified in the graph. If fanin is a control dependency, existing
-  // control dependencies will be checked first before adding. Otherwise fanin
-  // will be added after existing non control dependency inputs.
-  //
-  // This will return true iff the node is modified. If a control dependency
-  // already exists, the node will not be modified.
-  bool AddFaninInternal(NodeDef* node, const TensorId& fanin);
-
   // Removes all instances of regular fanin `fanin` from node `node`.
-  //
-  // This will return true iff the node is modified.
   bool RemoveRegularFaninInternal(NodeDef* node, const OutputPort& fanin);
 
   // Removes controlling fanin `fanin_node` from node if such controlling fanin
   // exists.
-  //
-  // This will return true iff the node is modified.
   bool RemoveControllingFaninInternal(NodeDef* node, NodeDef* fanin_node);
 };
 
diff --git a/tensorflow/core/grappler/mutable_graph_view_test.cc b/tensorflow/core/grappler/mutable_graph_view_test.cc
index b44e1a360d..99a69ef263 100644
--- a/tensorflow/core/grappler/mutable_graph_view_test.cc
+++ b/tensorflow/core/grappler/mutable_graph_view_test.cc
@@ -14,6 +14,7 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/core/grappler/mutable_graph_view.h"
+#include "absl/strings/substitute.h"
 #include "tensorflow/cc/ops/standard_ops.h"
 #include "tensorflow/core/framework/function_testlib.h"
 #include "tensorflow/core/framework/types.pb.h"
@@ -44,7 +45,7 @@ TEST(MutableGraphViewTest, AddAndUpdateFanouts) {
   NodeDef* new_bar = graph.AddNode(NDef("new_bar", "NotImportant", {}, {}));
   NodeDef* bar = graph.GetNode("bar");
 
-  EXPECT_TRUE(graph.UpdateFanouts(bar->name(), new_bar->name()));
+  EXPECT_TRUE(graph.UpdateFanouts(bar->name(), new_bar->name()).ok());
 
   // Fanout nodes must have their inputs updated.
   NodeDef* foo_1 = graph.GetNode("foo_1");
@@ -93,7 +94,7 @@ TEST(MutableGraphViewTest, AddAndUpdateFanoutsKeepControls) {
   NodeDef* new_bar = graph.AddNode(NDef("new_bar", "Identity", {"bar_1:2"}));
   NodeDef* bar_2 = graph.GetNode("bar_2");
 
-  EXPECT_TRUE(graph.UpdateFanouts(bar_2->name(), new_bar->name()));
+  EXPECT_TRUE(graph.UpdateFanouts(bar_2->name(), new_bar->name()).ok());
 
   // Fanout nodes must have their inputs updated.
   NodeDef* foo_1 = graph.GetNode("foo_1");
@@ -138,7 +139,7 @@ TEST(MutableGraphViewTest, AddAndUpdateFanoutsWithoutSelfLoops) {
   NodeDef* new_bar = graph.AddNode(NDef("new_bar", "NewBar", {"bar"}, {}));
   NodeDef* bar = graph.GetNode("bar");
 
-  EXPECT_TRUE(graph.UpdateFanouts("bar", new_bar->name()));
+  EXPECT_TRUE(graph.UpdateFanouts("bar", new_bar->name()).ok());
 
   // Foo node must read from `new_bar`.
   NodeDef* foo_1 = graph.GetNode("foo_1");
@@ -197,8 +198,18 @@ TEST(MutableGraphViewTest, UpdateFanoutsToSwitchWithControlFromSwitch) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_FALSE(graph.UpdateFanouts("a", "b"));
-  EXPECT_FALSE(graph.UpdateFanouts("d", "b"));
+  Status s = graph.UpdateFanouts("a", "b");
+  EXPECT_FALSE(s.ok());
+  string expected_msg =
+      "Can't update fanouts from 'a' to 'b', to node is being added as a "
+      "Switch control dependency.";
+  EXPECT_EQ(s.error_message(), expected_msg);
+  s = graph.UpdateFanouts("d", "b");
+  EXPECT_FALSE(s.ok());
+  expected_msg =
+      "Can't update fanouts from 'd' to 'b', to node is being added as a "
+      "Switch control dependency.";
+  EXPECT_EQ(s.error_message(), expected_msg);
 
   EXPECT_EQ(graph.graph()->node_size(), 5);
 
@@ -234,7 +245,7 @@ TEST(MutableGraphViewTest, UpdateFanoutsToSwitchWithNoControlFromSwitch) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.UpdateFanouts("c", "b"));
+  EXPECT_TRUE(graph.UpdateFanouts("c", "b").ok());
 
   EXPECT_EQ(graph.graph()->node_size(), 5);
 
@@ -277,7 +288,8 @@ GraphDef SimpleMutateFaninGraph() {
 }
 
 void TestAddRegularFanin(absl::string_view node_name,
-                         const TensorId& fanin_to_add, bool modified,
+                         const TensorId& fanin_to_add, bool success,
+                         const string& error_msg,
                          const NodeDef* expected_node) {
   GraphDef graph_def = SimpleMutateFaninGraph();
 
@@ -290,7 +302,11 @@ void TestAddRegularFanin(absl::string_view node_name,
     EXPECT_NE(node, nullptr);
   }
 
-  EXPECT_EQ(modified, graph.AddRegularFanin(node_name, fanin_to_add));
+  Status s = graph.AddRegularFanin(node_name, fanin_to_add);
+  EXPECT_EQ(s.ok(), success);
+  if (!success) {
+    EXPECT_EQ(s.error_message(), error_msg);
+  }
   if (expected_node != nullptr) {
     CompareNodeInputs(graph, expected_node, node);
   }
@@ -298,63 +314,92 @@ void TestAddRegularFanin(absl::string_view node_name,
 
 TEST(MutableGraphViewTest, AddRegularFanin) {
   NodeDef expected_node;
+  string error_msg;
   // Add input to node with 1 input 0 controls.
   expected_node = NDef("", "", {"a", "b:1"});
-  TestAddRegularFanin("foo_1", {"b", 1}, /*modified=*/true, &expected_node);
+  TestAddRegularFanin("foo_1", {"b", 1}, /*success=*/true, error_msg,
+                      &expected_node);
   // Add input to node with multiple inputs and 0 controls.
   expected_node = NDef("", "", {"b", "a:1", "a:1", "b:2"});
-  TestAddRegularFanin("foo_3", {"b", 2}, /*modified=*/true, &expected_node);
+  TestAddRegularFanin("foo_3", {"b", 2}, /*success=*/true, error_msg,
+                      &expected_node);
   // Add input to node with 1 input multiple controls.
   expected_node = NDef("", "", {"b", "a", "^c"});
-  TestAddRegularFanin("foo_2", {"a", 0}, /*modified=*/true, &expected_node);
+  TestAddRegularFanin("foo_2", {"a", 0}, /*success=*/true, error_msg,
+                      &expected_node);
   // Add input to node with multiple inputs and controls.
   expected_node = NDef("", "", {"a", "b:2", "b:2", "a:1", "^d", "^c"});
-  TestAddRegularFanin("foo_4", {"a", 1}, /*modified=*/true, &expected_node);
+  TestAddRegularFanin("foo_4", {"a", 1}, /*success=*/true, error_msg,
+                      &expected_node);
   // Add input to node with 0 inputs 0 controls.
   expected_node = NDef("", "", {"a:1"});
-  TestAddRegularFanin("foo_5", {"a", 1}, /*modified=*/true, &expected_node);
+  TestAddRegularFanin("foo_5", {"a", 1}, /*success=*/true, error_msg,
+                      &expected_node);
   // Add input to node with 0 inputs multiple controls.
   expected_node = NDef("", "", {"c:1", "^b", "^a"});
-  TestAddRegularFanin("foo_6", {"c", 1}, /*modified=*/true, &expected_node);
+  TestAddRegularFanin("foo_6", {"c", 1}, /*success=*/true, error_msg,
+                      &expected_node);
 
   // Add control to node with 1 input 0 controls.
   expected_node = NDef("", "", {"a"});
-  TestAddRegularFanin("foo_1", {"b", Graph::kControlSlot}, /*modified=*/false,
-                      &expected_node);
+  error_msg = "Can't add invalid fanin '^b' as regular fanin to node 'foo_1'.";
+  TestAddRegularFanin("foo_1", {"b", Graph::kControlSlot}, /*success=*/false,
+                      error_msg, &expected_node);
   // Add control to node with multiple inputs and 0 controls.
   expected_node = NDef("", "", {"b", "a:1", "a:1"});
-  TestAddRegularFanin("foo_3", {"c", Graph::kControlSlot}, /*modified=*/false,
-                      &expected_node);
+  error_msg = "Can't add invalid fanin '^c' as regular fanin to node 'foo_3'.";
+  TestAddRegularFanin("foo_3", {"c", Graph::kControlSlot}, /*success=*/false,
+                      error_msg, &expected_node);
   // Add control to node with 1 input multiple controls.
   expected_node = NDef("", "", {"b", "^a", "^c"});
-  TestAddRegularFanin("foo_2", {"d", Graph::kControlSlot}, /*modified=*/false,
-                      &expected_node);
+  error_msg = "Can't add invalid fanin '^d' as regular fanin to node 'foo_2'.";
+  TestAddRegularFanin("foo_2", {"d", Graph::kControlSlot}, /*success=*/false,
+                      error_msg, &expected_node);
   // Add control to node with multiple input multiple controls.
   expected_node = NDef("", "", {"a", "b:2", "b:2", "^c", "^d"});
+  error_msg = "Can't add invalid fanin '^a' as regular fanin to node 'foo_4'.";
   TestAddRegularFanin("foo_4", {"a", Graph::kControlSlot},
-                      /*modified=*/false, &expected_node);
+                      /*success=*/false, error_msg, &expected_node);
   // Add control to node with 0 inputs 0 controls.
   expected_node = NDef("", "", {});
-  TestAddRegularFanin("foo_5", {"a", Graph::kControlSlot}, /*modified=*/false,
-                      &expected_node);
+  error_msg = "Can't add invalid fanin '^a' as regular fanin to node 'foo_5'.";
+  TestAddRegularFanin("foo_5", {"a", Graph::kControlSlot}, /*success=*/false,
+                      error_msg, &expected_node);
   // Add control to node with 0 inputs multiple controls.
   expected_node = NDef("", "", {"^a", "^b"});
-  TestAddRegularFanin("foo_6", {"c", Graph::kControlSlot}, /*modified=*/false,
-                      &expected_node);
+  error_msg = "Can't add invalid fanin '^c' as regular fanin to node 'foo_6'.";
+  TestAddRegularFanin("foo_6", {"c", Graph::kControlSlot}, /*success=*/false,
+                      error_msg, &expected_node);
   // Add control to node with control that already exists.
   expected_node = NDef("", "", {"b", "^a", "^c"});
+  error_msg = "Can't add invalid fanin '^a' as regular fanin to node 'foo_2'.";
   TestAddRegularFanin("foo_2", {"a", Graph::kControlSlot},
-                      /*modified=*/false, &expected_node);
+                      /*success=*/false, error_msg, &expected_node);
 
   // Add fanin to node where node is missing.
-  TestAddRegularFanin("foo_missing", {"a", 0}, /*modified=*/false, nullptr);
+  error_msg =
+      "Can't add fanin 'a:0' as regular fanin to missing node 'foo_missing'.";
+  TestAddRegularFanin("foo_missing", {"a", 0}, /*success=*/false, error_msg,
+                      nullptr);
   // Add fanin to node where fanin is missing.
   expected_node = NDef("", "", {"a"});
-  TestAddRegularFanin("foo_1", {"bar_missing", 0}, /*modified=*/false,
+  error_msg =
+      "Can't add missing fanin 'bar_missing:0' as regular fanin to node "
+      "'foo_1'.";
+  TestAddRegularFanin("foo_1", {"bar_missing", 0}, /*success=*/false, error_msg,
                       &expected_node);
   // Add fanin to node where node and fanin are missing.
-  TestAddRegularFanin("foo_missing", {"bar_missing", 0}, /*modified=*/false,
-                      /*expected_node=*/nullptr);
+  error_msg =
+      "Can't add missing fanin 'bar_missing:0' as regular fanin to missing "
+      "node 'foo_missing'.";
+  TestAddRegularFanin("foo_missing", {"bar_missing", 0}, /*success=*/false,
+                      error_msg, /*expected_node=*/nullptr);
+  // Add control fanin to node where node and fanin are missing.
+  error_msg =
+      "Can't add invalid/missing fanin '^bar_missing' as regular fanin to "
+      "missing node 'foo_missing'.";
+  TestAddRegularFanin("foo_missing", {"bar_missing", Graph::kControlSlot},
+                      /*success=*/false, error_msg, /*expected_node=*/nullptr);
 }
 
 void CheckFanout(const MutableGraphView& graph, const TensorId& fanin,
@@ -368,7 +413,8 @@ void CheckFanout(const MutableGraphView& graph, const TensorId& fanin,
 }
 
 void TestRemoveRegularFanin(absl::string_view node_name,
-                            const TensorId& fanin_to_remove, bool modified,
+                            const TensorId& fanin_to_remove, bool success,
+                            const string& error_msg,
                             const NodeDef* expected_node) {
   GraphDef graph_def = SimpleMutateFaninGraph();
 
@@ -381,10 +427,14 @@ void TestRemoveRegularFanin(absl::string_view node_name,
     EXPECT_NE(nullptr, node);
   }
 
-  EXPECT_EQ(modified, graph.RemoveRegularFanin(node_name, fanin_to_remove));
+  Status s = graph.RemoveRegularFanin(node_name, fanin_to_remove);
+  EXPECT_EQ(s.ok(), success);
+  if (!success) {
+    EXPECT_EQ(s.error_message(), error_msg);
+  }
   if (expected_node != nullptr) {
     CompareNodeInputs(graph, expected_node, node);
-    if (modified) {
+    if (success) {
       CheckFanout(graph, fanin_to_remove, node_name);
     }
   }
@@ -392,70 +442,109 @@ void TestRemoveRegularFanin(absl::string_view node_name,
 
 TEST(MutableGraphViewTest, RemoveRegularFanin) {
   NodeDef expected_node;
+  string error_msg;
   // Remove input from node with 1 input 0 controls.
   expected_node = NDef("", "", {});
-  TestRemoveRegularFanin("foo_1", {"a", 0}, /*modified=*/true, &expected_node);
+  TestRemoveRegularFanin("foo_1", {"a", 0}, /*success=*/true, error_msg,
+                         &expected_node);
   // Remove input from node with multiple inputs and 0 controls.
   expected_node = NDef("", "", {"b"});
-  TestRemoveRegularFanin("foo_3", {"a", 1}, /*modified=*/true, &expected_node);
+  TestRemoveRegularFanin("foo_3", {"a", 1}, /*success=*/true, error_msg,
+                         &expected_node);
   // Remove input from node with 1 input multiple controls.
   expected_node = NDef("", "", {"^a", "^c"});
-  TestRemoveRegularFanin("foo_2", {"b", 0}, /*modified=*/true, &expected_node);
+  TestRemoveRegularFanin("foo_2", {"b", 0}, /*success=*/true, error_msg,
+                         &expected_node);
   // Remove input from node with multiple inputs and controls.
   expected_node = NDef("", "", {"a", "^c", "^d"});
-  TestRemoveRegularFanin("foo_4", {"b", 2}, /*modified=*/true, &expected_node);
+  TestRemoveRegularFanin("foo_4", {"b", 2}, /*success=*/true, error_msg,
+                         &expected_node);
   // Remove input from node with multiple inputs and controls, and results in
   // shifting of ports.
   expected_node = NDef("", "", {"b:2", "b:2", "^c", "^d"});
-  TestRemoveRegularFanin("foo_4", {"a", 0}, /*modified=*/true, &expected_node);
+  TestRemoveRegularFanin("foo_4", {"a", 0}, /*success=*/true, error_msg,
+                         &expected_node);
 
   // Remove control from node with 1 input multiple controls.
   expected_node = NDef("", "", {"b", "^a", "^c"});
+  error_msg =
+      "Can't remove invalid fanin '^a' as regular fanin from node 'foo_2'.";
   TestRemoveRegularFanin("foo_2", {"a", Graph::kControlSlot},
-                         /*modified=*/false, &expected_node);
+                         /*success=*/false, error_msg, &expected_node);
   // Remove control from node with multiple input multiple controls.
   expected_node = NDef("", "", {"a", "b:2", "b:2", "^c", "^d"});
+  error_msg =
+      "Can't remove invalid fanin '^d' as regular fanin from node 'foo_4'.";
   TestRemoveRegularFanin("foo_4", {"d", Graph::kControlSlot},
-                         /*modified=*/false, &expected_node);
+                         /*success=*/false, error_msg, &expected_node);
   // Remove control from node with 0 inputs multiple controls.
   expected_node = NDef("", "", {"^a", "^b"});
+  error_msg =
+      "Can't remove invalid fanin '^a' as regular fanin from node 'foo_6'.";
   TestRemoveRegularFanin("foo_6", {"a", Graph::kControlSlot},
-                         /*modified=*/false, &expected_node);
+                         /*success=*/false, error_msg, &expected_node);
 
   // Remove input from node with 0 inputs 0 controls.
   expected_node = NDef("", "", {});
-  TestRemoveRegularFanin("foo_5", {"a", 1}, /*modified=*/false, &expected_node);
+  error_msg = "";
+  TestRemoveRegularFanin("foo_5", {"a", 1}, /*success=*/true, error_msg,
+                         &expected_node);
   // Remove input from node with 0 inputs multiple controls.
   expected_node = NDef("", "", {"^a", "^b"});
-  TestRemoveRegularFanin("foo_6", {"a", 1}, /*modified=*/false, &expected_node);
+  TestRemoveRegularFanin("foo_6", {"a", 1}, /*success=*/true, error_msg,
+                         &expected_node);
 
   // Remove control from node with 1 input 0 controls.
   expected_node = NDef("", "", {"a"});
+  error_msg =
+      "Can't remove invalid fanin '^b' as regular fanin from node 'foo_1'.";
   TestRemoveRegularFanin("foo_1", {"b", Graph::kControlSlot},
-                         /*modified=*/false, &expected_node);
+                         /*success=*/false, error_msg, &expected_node);
   // Remove control from node with multiple inputs and 0 controls.
   expected_node = NDef("", "", {"b", "a:1", "a:1"});
+  error_msg =
+      "Can't remove invalid fanin '^c' as regular fanin from node 'foo_3'.";
   TestRemoveRegularFanin("foo_3", {"c", Graph::kControlSlot},
-                         /*modified=*/false, &expected_node);
+                         /*success=*/false, error_msg, &expected_node);
   // Remove control from node with 0 inputs 0 controls.
   expected_node = NDef("", "", {});
+  error_msg =
+      "Can't remove invalid fanin '^a' as regular fanin from node 'foo_5'.";
   TestRemoveRegularFanin("foo_5", {"a", Graph::kControlSlot},
-                         /*modified=*/false, &expected_node);
+                         /*success=*/false, error_msg, &expected_node);
 
   // Remove fanin from node where node is missing.
-  TestRemoveRegularFanin("foo_missing", {"a", 0}, /*modified=*/false,
+  error_msg =
+      "Can't remove fanin 'a:0' as regular fanin from missing node "
+      "'foo_missing'.";
+  TestRemoveRegularFanin("foo_missing", {"a", 0}, /*success=*/false, error_msg,
                          /*expected_node=*/nullptr);
   // Remove fanin from node where fanin is missing.
   expected_node = NDef("", "", {"a"});
-  TestRemoveRegularFanin("foo_1", {"bar_missing", 0}, /*modified=*/false,
-                         &expected_node);
+  error_msg =
+      "Can't remove missing fanin 'bar_missing:0' as regular fanin from node "
+      "'foo_1'.";
+  TestRemoveRegularFanin("foo_1", {"bar_missing", 0}, /*success=*/false,
+                         error_msg, &expected_node);
   // Remove fanin from node where node and fanin are missing.
-  TestRemoveRegularFanin("foo_missing", {"bar_missing", 0}, /*modified=*/false,
+  error_msg =
+      "Can't remove missing fanin 'bar_missing:0' as regular fanin from "
+      "missing node 'foo_missing'.";
+  TestRemoveRegularFanin("foo_missing", {"bar_missing", 0}, /*success=*/false,
+                         error_msg,
+                         /*expected_node=*/nullptr);
+  // Remove control from node where node and fanin are missing.
+  error_msg =
+      "Can't remove invalid/missing fanin '^bar_missing' as regular fanin from "
+      "missing node 'foo_missing'.";
+  TestRemoveRegularFanin("foo_missing", {"bar_missing", Graph::kControlSlot},
+                         /*success=*/false, error_msg,
                          /*expected_node=*/nullptr);
 }
 
 void TestRemoveAllFanins(absl::string_view node_name,
-                         bool keep_controlling_nodes, bool modified,
+                         bool keep_controlling_nodes, bool success,
+                         const string& error_msg,
                          const NodeDef* expected_node) {
   GraphDef graph_def = SimpleMutateFaninGraph();
 
@@ -470,10 +559,14 @@ void TestRemoveAllFanins(absl::string_view node_name,
     fanin_strings.insert(node->input().begin(), node->input().end());
   }
 
-  EXPECT_EQ(modified, graph.RemoveAllFanins(node_name, keep_controlling_nodes));
+  Status s = graph.RemoveAllFanins(node_name, keep_controlling_nodes);
+  EXPECT_EQ(s.ok(), success);
+  if (!success) {
+    EXPECT_EQ(s.error_message(), error_msg);
+  }
   if (expected_node != nullptr) {
     CompareNodeInputs(graph, expected_node, node);
-    if (modified) {
+    if (success) {
       TensorId tensor_id;
       auto retained_inputs = absl::flat_hash_set<string>(node->input().begin(),
                                                          node->input().end());
@@ -489,48 +582,50 @@ void TestRemoveAllFanins(absl::string_view node_name,
 
 TEST(MutableGraphViewTest, RemoveAllFanins) {
   NodeDef expected_node;
+  string error_msg;
   // Remove all fanins from node with no control dependencies.
   expected_node = NDef("", "", {});
   TestRemoveAllFanins("foo_3", /*keep_controlling_nodes=*/false,
-                      /*modified=*/true, &expected_node);
+                      /*success=*/true, error_msg, &expected_node);
   // Remove all fanins from node with control dependencies.
   TestRemoveAllFanins("foo_4", /*keep_controlling_nodes=*/false,
-                      /*modified=*/true, &expected_node);
+                      /*success=*/true, error_msg, &expected_node);
 
   // Remove all fanins from node with no control dependencies and preserve
   // control dependencies.
   TestRemoveAllFanins("foo_3", /*keep_controlling_nodes=*/true,
-                      /*modified=*/true, &expected_node);
+                      /*success=*/true, error_msg, &expected_node);
   // Remove all fanins from node with control dependencies and preserve control
   // dependencies.
   expected_node = NDef("", "", {"^c", "^d"});
   TestRemoveAllFanins("foo_4", /*keep_controlling_nodes=*/true,
-                      /*modified=*/true, &expected_node);
+                      /*success=*/true, error_msg, &expected_node);
 
   // Remove all fanins from node with no fanins.
   expected_node = NDef("", "", {});
   TestRemoveAllFanins("foo_5", /*keep_controlling_nodes=*/false,
-                      /*modified=*/false, &expected_node);
+                      /*success=*/true, error_msg, &expected_node);
   TestRemoveAllFanins("foo_5", /*keep_controlling_nodes=*/true,
-                      /*modified=*/false, &expected_node);
+                      /*success=*/true, error_msg, &expected_node);
 
   // Remove all fanins from node with only control dependencies.
   TestRemoveAllFanins("foo_6", /*keep_controlling_nodes=*/false,
-                      /*modified=*/true, &expected_node);
+                      /*success=*/true, error_msg, &expected_node);
   expected_node = NDef("", "", {"^a", "^b"});
   TestRemoveAllFanins("foo_6", /*keep_controlling_nodes=*/true,
-                      /*modified=*/false, &expected_node);
+                      /*success=*/true, error_msg, &expected_node);
 
   // Remove all fanins from node where node is missing.
+  error_msg = "Can't remove all fanins from missing node 'foo_missing'.";
   TestRemoveAllFanins("foo_missing", /*keep_controlling_nodes=*/false,
-                      /*modified=*/false, /*expected_node=*/nullptr);
+                      /*success=*/false, error_msg, /*expected_node=*/nullptr);
   TestRemoveAllFanins("foo_missing", /*keep_controlling_nodes=*/true,
-                      /*modified=*/false, /*expected_node=*/nullptr);
+                      /*success=*/false, error_msg, /*expected_node=*/nullptr);
 }
 
 void TestUpdateFanin(absl::string_view node_name, const TensorId& from_fanin,
-                     const TensorId& to_fanin, bool modified,
-                     const NodeDef* expected_node) {
+                     const TensorId& to_fanin, bool success,
+                     const string& error_msg, const NodeDef* expected_node) {
   GraphDef graph_def = SimpleMutateFaninGraph();
 
   MutableGraphView graph(&graph_def);
@@ -542,10 +637,14 @@ void TestUpdateFanin(absl::string_view node_name, const TensorId& from_fanin,
     EXPECT_NE(node, nullptr);
   }
 
-  EXPECT_EQ(modified, graph.UpdateFanin(node_name, from_fanin, to_fanin));
+  Status s = graph.UpdateFanin(node_name, from_fanin, to_fanin);
+  EXPECT_EQ(s.ok(), success);
+  if (!success) {
+    EXPECT_EQ(s.error_message(), error_msg);
+  }
   if (expected_node != nullptr) {
     CompareNodeInputs(graph, expected_node, node);
-    if (modified) {
+    if (success) {
       CheckFanout(graph, from_fanin, node_name);
     }
   }
@@ -553,49 +652,82 @@ void TestUpdateFanin(absl::string_view node_name, const TensorId& from_fanin,
 
 TEST(MutableGraphViewTest, UpdateFanin) {
   NodeDef expected_node;
+  string error_msg;
   // Update fanin from non control to non control.
   expected_node = NDef("", "", {"a", "b:3", "b:3", "^c", "^d"});
-  TestUpdateFanin("foo_4", {"b", 2}, {"b", 3}, /*modified=*/true,
+  TestUpdateFanin("foo_4", {"b", 2}, {"b", 3}, /*success=*/true, error_msg,
                   &expected_node);
   // Update fanin from non control to control.
   expected_node = NDef("", "", {"a", "^c", "^d", "^b"});
   TestUpdateFanin("foo_4", {"b", 2}, {"b", Graph::kControlSlot},
-                  /*modified=*/true, &expected_node);
+                  /*success=*/true, error_msg, &expected_node);
   // Update fanin from control to non control.
   expected_node = NDef("", "", {"a", "b:2", "b:2", "d:1", "^c"});
   TestUpdateFanin("foo_4", {"d", Graph::kControlSlot}, {"d", 1},
-                  /*modified=*/true, &expected_node);
+                  /*success=*/true, error_msg, &expected_node);
   // Update fanin from control to control.
   expected_node = NDef("", "", {"a", "b:2", "b:2", "^d"});
   TestUpdateFanin("foo_4", {"c", Graph::kControlSlot},
-                  {"b", Graph::kControlSlot}, /*modified=*/true,
+                  {"b", Graph::kControlSlot}, /*success=*/true, error_msg,
                   &expected_node);
   // Update fanin from control to existing control.
   expected_node = NDef("", "", {"a", "b:2", "b:2", "^d"});
   TestUpdateFanin("foo_4", {"c", Graph::kControlSlot},
-                  {"d", Graph::kControlSlot}, /*modified=*/true,
+                  {"d", Graph::kControlSlot}, /*success=*/true, error_msg,
                   &expected_node);
 
   // Update fanin of node where from and to fanins are the same.
   expected_node = NDef("", "", {"a"});
-  TestUpdateFanin("foo_1", {"a", -1}, {"a", -1}, /*modified=*/false,
+  TestUpdateFanin("foo_1", {"a", -1}, {"a", -1}, /*success=*/true, error_msg,
                   &expected_node);
-  TestUpdateFanin("foo_1", {"a", 0}, {"a", 0}, /*modified=*/false,
+  TestUpdateFanin("foo_1", {"a", 0}, {"a", 0}, /*success=*/true, error_msg,
                   &expected_node);
-  TestUpdateFanin("foo_1", {"a", 1}, {"a", 1}, /*modified=*/false,
+  TestUpdateFanin("foo_1", {"a", 1}, {"a", 1}, /*success=*/true, error_msg,
                   &expected_node);
+
   // Update fanin of node where node is missing.
-  TestUpdateFanin("foo_missing", {"a", 0}, {"a", 1}, /*modified=*/false,
+  error_msg =
+      "Can't update fanin 'a:0' to fanin 'a:1' in missing node 'foo_missing'.";
+  TestUpdateFanin("foo_missing", {"a", 0}, {"a", 1}, /*success=*/false,
+                  error_msg,
                   /*expected_node=*/nullptr);
   // Update fanin of node where from fanin is missing.
+  error_msg =
+      "Can't update missing fanin 'from_bar_missing:0' to fanin 'a:1' in node "
+      "'foo_1'.";
   TestUpdateFanin("foo_1", {"from_bar_missing", 0}, {"a", 1},
-                  /*modified=*/false, &expected_node);
+                  /*success=*/false, error_msg, &expected_node);
   // Update fanin of node where to fanin is missing.
-  TestUpdateFanin("foo_1", {"a", 0}, {"to_bar_missing", 1}, /*modified=*/false,
-                  &expected_node);
+  error_msg =
+      "Can't update fanin 'a:0' to missing fanin 'to_bar_missing:1' in node "
+      "'foo_1'.";
+  TestUpdateFanin("foo_1", {"a", 0}, {"to_bar_missing", 1}, /*success=*/false,
+                  error_msg, &expected_node);
   // Update fanin of node where from/to fanins and node are missing.
+  error_msg =
+      "Can't update missing fanin 'from_bar_missing:0' to missing fanin "
+      "'to_bar_missing:1' in missing node 'foo_missing'.";
   TestUpdateFanin("foo_missing", {"from_bar_missing", 0}, {"to_bar_missing", 1},
-                  /*modified=*/false, /*expected_node=*/nullptr);
+                  /*success=*/false, error_msg, /*expected_node=*/nullptr);
+  // Update fanin of node where from fanin is invalid.
+  error_msg =
+      "Can't update invalid fanin 'a:-2' to fanin 'a:0' in node 'foo_1'.";
+  TestUpdateFanin("foo_1", {"a", -2}, {"a", 0},
+                  /*success=*/false, error_msg, &expected_node);
+  // Update fanin of node where to fanin is invalid.
+  error_msg =
+      "Can't update fanin 'a:0' to invalid fanin 'a:-2' in node 'foo_1'.";
+  TestUpdateFanin("foo_1", {"a", 0}, {"a", -2},
+                  /*success=*/false, error_msg, &expected_node);
+  // Update fanin of node where from/to fanins are invalid and missing and node
+  // is missing.
+  error_msg =
+      "Can't update invalid/missing fanin 'from_bar_missing:-2' to "
+      "invalid/missing fanin 'to_bar_missing:-3' in missing node "
+      "'foo_missing'.";
+  TestUpdateFanin("foo_missing", {"from_bar_missing", -2},
+                  {"to_bar_missing", -3},
+                  /*success=*/false, error_msg, /*expected_node=*/nullptr);
 }
 
 void TestUpdateFaninFromFaninToNodeAsSwitchControl(const TensorId& fanin) {
@@ -607,7 +739,13 @@ void TestUpdateFaninFromFaninToNodeAsSwitchControl(const TensorId& fanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_FALSE(graph.UpdateFanin("c", fanin, {"b", Graph::kControlSlot}));
+  Status s = graph.UpdateFanin("c", fanin, {"b", Graph::kControlSlot});
+  EXPECT_FALSE(s.ok());
+  string expected_msg = absl::Substitute(
+      "Can't update fanin '$0' to fanin '^b' in node 'c', to fanin is a Switch "
+      "control dependency.",
+      fanin.ToString());
+  EXPECT_EQ(s.error_message(), expected_msg);
 
   EXPECT_EQ(graph.graph()->node_size(), 3);
 
@@ -714,14 +852,14 @@ TEST(MutableGraphViewTest, DedupControllingFaninsOnAddFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.AddRegularFanin("b", {"a", 2}));
+  EXPECT_TRUE(graph.AddRegularFanin("b", {"a", 2}).ok());
   NodeDef expected;
   NodeDef* b = graph.GetNode("b");
   ASSERT_NE(b, nullptr);
   expected = NDef("", "", {"a:2"});
   CompareNodeInputs(graph, &expected, b);
 
-  EXPECT_FALSE(graph.AddControllingFanin("c", {"a", Graph::kControlSlot}));
+  EXPECT_TRUE(graph.AddControllingFanin("c", {"a", Graph::kControlSlot}).ok());
   NodeDef* c = graph.GetNode("c");
   ASSERT_NE(c, nullptr);
   expected = NDef("", "", {"a:1"});
@@ -736,28 +874,28 @@ TEST(MutableGraphViewTest, NoDedupControlFlowControllingFaninsOnAddFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.AddRegularFanin("c", {"b", 2}));
+  EXPECT_TRUE(graph.AddRegularFanin("c", {"b", 2}).ok());
   NodeDef expected;
   NodeDef* c = graph.GetNode("c");
   ASSERT_NE(c, nullptr);
   expected = NDef("", "", {"b:2"});
   CompareNodeInputs(graph, &expected, c);
-  EXPECT_TRUE(graph.AddControllingFanin("c", {"b", Graph::kControlSlot}));
+  EXPECT_TRUE(graph.AddControllingFanin("c", {"b", Graph::kControlSlot}).ok());
   expected = NDef("", "", {"b:2", "^b"});
   CompareNodeInputs(graph, &expected, c);
-  EXPECT_FALSE(graph.AddControllingFanin("c", {"b", Graph::kControlSlot}));
+  EXPECT_TRUE(graph.AddControllingFanin("c", {"b", Graph::kControlSlot}).ok());
   expected = NDef("", "", {"b:2", "^b"});
   CompareNodeInputs(graph, &expected, c);
 
-  EXPECT_TRUE(graph.AddControllingFanin("d", {"b", Graph::kControlSlot}));
+  EXPECT_TRUE(graph.AddControllingFanin("d", {"b", Graph::kControlSlot}).ok());
   NodeDef* d = graph.GetNode("d");
   ASSERT_NE(d, nullptr);
   expected = NDef("", "", {"^b"});
   CompareNodeInputs(graph, &expected, d);
-  EXPECT_FALSE(graph.AddControllingFanin("d", {"b", Graph::kControlSlot}));
+  EXPECT_TRUE(graph.AddControllingFanin("d", {"b", Graph::kControlSlot}).ok());
   expected = NDef("", "", {"^b"});
   CompareNodeInputs(graph, &expected, d);
-  EXPECT_TRUE(graph.AddRegularFanin("d", {"b", 3}));
+  EXPECT_TRUE(graph.AddRegularFanin("d", {"b", 3}).ok());
   expected = NDef("", "", {"b:3", "^b"});
   CompareNodeInputs(graph, &expected, d);
 }
@@ -771,7 +909,7 @@ TEST(MutableGraphViewTest, DedupControllingFaninsOnUpdateFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.UpdateFanin("c", {"a", 1}, {"b", 2}));
+  EXPECT_TRUE(graph.UpdateFanin("c", {"a", 1}, {"b", 2}).ok());
   NodeDef* c = graph.GetNode("c");
   ASSERT_NE(c, nullptr);
   NodeDef expected = NDef("", "", {"b:2"});
@@ -787,21 +925,24 @@ TEST(MutableGraphViewTest, NoDedupControlFlowControllingFaninsOnUpdateFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.UpdateFanin("d", {"b", Graph::kControlSlot},
-                                {"c", Graph::kControlSlot}));
+  EXPECT_TRUE(graph
+                  .UpdateFanin("d", {"b", Graph::kControlSlot},
+                               {"c", Graph::kControlSlot})
+                  .ok());
   NodeDef expected;
   NodeDef* d = graph.GetNode("d");
   ASSERT_NE(d, nullptr);
   expected = NDef("", "", {"c", "^c"});
   CompareNodeInputs(graph, &expected, d);
 
-  EXPECT_TRUE(graph.UpdateFanin("e", {"b", 0}, {"c", 3}));
+  EXPECT_TRUE(graph.UpdateFanin("e", {"b", 0}, {"c", 3}).ok());
   NodeDef* e = graph.GetNode("e");
   ASSERT_NE(e, nullptr);
   expected = NDef("", "", {"c:3", "^c"});
   CompareNodeInputs(graph, &expected, e);
 
-  EXPECT_TRUE(graph.UpdateFanin("e", {"c", 3}, {"c", Graph::kControlSlot}));
+  EXPECT_TRUE(
+      graph.UpdateFanin("e", {"c", 3}, {"c", Graph::kControlSlot}).ok());
   ASSERT_NE(e, nullptr);
   expected = NDef("", "", {"^c"});
   CompareNodeInputs(graph, &expected, e);
@@ -816,7 +957,7 @@ TEST(MutableGraphViewTest, UpdateMaxRegularOutputPortOnAddFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.AddRegularFanin("c", {"a", 3}));
+  EXPECT_TRUE(graph.AddRegularFanin("c", {"a", 3}).ok());
   NodeDef* a = graph.GetNode("a");
   ASSERT_NE(a, nullptr);
 
@@ -841,7 +982,7 @@ TEST(MutableGraphViewTest, UpdateMaxRegularOutputPortOnRemoveFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.RemoveRegularFanin("c", {"a", 2}));
+  EXPECT_TRUE(graph.RemoveRegularFanin("c", {"a", 2}).ok());
   NodeDef* a = graph.GetNode("a");
   ASSERT_NE(a, nullptr);
 
@@ -862,7 +1003,7 @@ TEST(MutableGraphViewTest, KeepMaxRegularOutputPortOnRemoveFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.RemoveRegularFanin("b", {"a", 1}));
+  EXPECT_TRUE(graph.RemoveRegularFanin("b", {"a", 1}).ok());
   NodeDef* a = graph.GetNode("a");
   ASSERT_NE(a, nullptr);
 
@@ -883,7 +1024,7 @@ TEST(MutableGraphViewTest, UpdateMaxRegularOutputPortOnUpdateFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.UpdateFanin("c", {"a", 2}, {"b", 3}));
+  EXPECT_TRUE(graph.UpdateFanin("c", {"a", 2}, {"b", 3}).ok());
   NodeDef* a = graph.GetNode("a");
   ASSERT_NE(a, nullptr);
 
@@ -910,11 +1051,21 @@ TEST(MutableGraphViewTest, AddControllingFaninMissing) {
 
   MutableGraphView graph(&graph_def);
   // Missing fanin.
-  EXPECT_FALSE(graph.AddControllingFanin("a", {"c", Graph::kControlSlot}));
+  Status s = graph.AddControllingFanin("a", {"c", Graph::kControlSlot});
+  EXPECT_FALSE(s.ok());
+  string expected_msg = "Can't add missing controlling fanin '^c' to node 'a'.";
+  EXPECT_EQ(s.error_message(), expected_msg);
   // Missing node.
-  EXPECT_FALSE(graph.AddControllingFanin("d", {"a", Graph::kControlSlot}));
+  s = graph.AddControllingFanin("d", {"a", Graph::kControlSlot});
+  EXPECT_FALSE(s.ok());
+  expected_msg = "Can't add controlling fanin '^a' to missing node 'd'.";
+  EXPECT_EQ(s.error_message(), expected_msg);
   // Missing node and fanin.
-  EXPECT_FALSE(graph.AddControllingFanin("c", {"d", Graph::kControlSlot}));
+  s = graph.AddControllingFanin("c", {"d", Graph::kControlSlot});
+  EXPECT_FALSE(s.ok());
+  expected_msg =
+      "Can't add missing controlling fanin '^d' to missing node 'c'.";
+  EXPECT_EQ(s.error_message(), expected_msg);
 
   ASSERT_EQ(graph.graph()->node_size(), 2);
   NodeDef expected;
@@ -935,8 +1086,8 @@ TEST(MutableGraphViewTest, AddControllingFaninExistingControl) {
       /*funcs=*/{});
 
   MutableGraphView graph(&graph_def);
-  EXPECT_TRUE(graph.AddControllingFanin("a", {"b", Graph::kControlSlot}));
-  EXPECT_FALSE(graph.AddControllingFanin("a", {"b", Graph::kControlSlot}));
+  EXPECT_TRUE(graph.AddControllingFanin("a", {"b", Graph::kControlSlot}).ok());
+  EXPECT_TRUE(graph.AddControllingFanin("a", {"b", Graph::kControlSlot}).ok());
 
   ASSERT_EQ(graph.graph()->node_size(), 2);
   NodeDef expected;
@@ -958,8 +1109,8 @@ TEST(MutableGraphViewTest, AddControllingFaninNotSwitch) {
       /*funcs=*/{});
 
   MutableGraphView graph(&graph_def);
-  EXPECT_TRUE(graph.AddControllingFanin("a", {"b", 2}));
-  EXPECT_FALSE(graph.AddControllingFanin("a", {"b", 2}));
+  EXPECT_TRUE(graph.AddControllingFanin("a", {"b", 2}).ok());
+  EXPECT_TRUE(graph.AddControllingFanin("a", {"b", 2}).ok());
 
   ASSERT_EQ(graph.graph()->node_size(), 2);
   NodeDef expected;
@@ -982,8 +1133,8 @@ TEST(MutableGraphViewTest, AddControllingFaninSwitchWithIdentity) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.AddControllingFanin("a", {"switch", 0}));
-  EXPECT_FALSE(graph.AddControllingFanin("a", {"switch", 0}));
+  EXPECT_TRUE(graph.AddControllingFanin("a", {"switch", 0}).ok());
+  EXPECT_TRUE(graph.AddControllingFanin("a", {"switch", 0}).ok());
 
   ASSERT_EQ(graph.graph()->node_size(), 3);
   NodeDef* a = graph.GetNode("a");
@@ -1001,8 +1152,8 @@ TEST(MutableGraphViewTest, AddControllingFaninSwitchWithNoExistingIdentity) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.AddControllingFanin("a", {"switch", 0}));
-  EXPECT_FALSE(graph.AddControllingFanin("a", {"switch", 0}));
+  EXPECT_TRUE(graph.AddControllingFanin("a", {"switch", 0}).ok());
+  EXPECT_TRUE(graph.AddControllingFanin("a", {"switch", 0}).ok());
 
   ASSERT_EQ(graph.graph()->node_size(), 3);
   NodeDef expected;
@@ -1029,8 +1180,8 @@ TEST(MutableGraphViewTest, AddControllingFaninSwitchWithExistingAddedIdentity) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.AddControllingFanin("a", {"switch", 0}));
-  EXPECT_FALSE(graph.AddControllingFanin("a", {"switch", 0}));
+  EXPECT_TRUE(graph.AddControllingFanin("a", {"switch", 0}).ok());
+  EXPECT_TRUE(graph.AddControllingFanin("a", {"switch", 0}).ok());
 
   ASSERT_EQ(graph.graph()->node_size(), 3);
   NodeDef* a = graph.GetNode("a");
@@ -1049,7 +1200,7 @@ TEST(MutableGraphViewTest, RemoveControllingFaninMissing) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_FALSE(graph.RemoveControllingFanin("d", "c"));
+  EXPECT_TRUE(graph.RemoveControllingFanin("d", "c").ok());
 
   ASSERT_EQ(graph.graph()->node_size(), 4);
   NodeDef* d = graph.GetNode("d");
@@ -1068,8 +1219,8 @@ TEST(MutableGraphViewTest, RemoveControllingFaninExisting) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.RemoveControllingFanin("d", "a"));
-  EXPECT_FALSE(graph.RemoveControllingFanin("d", "a"));
+  EXPECT_TRUE(graph.RemoveControllingFanin("d", "a").ok());
+  EXPECT_TRUE(graph.RemoveControllingFanin("d", "a").ok());
 
   ASSERT_EQ(graph.graph()->node_size(), 4);
   NodeDef* d = graph.GetNode("d");
@@ -1087,8 +1238,8 @@ TEST(MutableGraphViewTest, RemoveControllingFaninOnRegularFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_FALSE(graph.RemoveControllingFanin("c", "a"));
-  EXPECT_FALSE(graph.RemoveControllingFanin("c", "b"));
+  EXPECT_TRUE(graph.RemoveControllingFanin("c", "a").ok());
+  EXPECT_TRUE(graph.RemoveControllingFanin("c", "b").ok());
 
   ASSERT_EQ(graph.graph()->node_size(), 3);
   NodeDef* c = graph.GetNode("c");
diff --git a/tensorflow/core/grappler/optimizers/data/filter_fusion.cc b/tensorflow/core/grappler/optimizers/data/filter_fusion.cc
index 89b568ecf1..e3f0a7e6c5 100644
--- a/tensorflow/core/grappler/optimizers/data/filter_fusion.cc
+++ b/tensorflow/core/grappler/optimizers/data/filter_fusion.cc
@@ -109,7 +109,8 @@ Status FilterFusion::Optimize(Cluster* cluster, const GrapplerItem& item,
     const auto* fused_filter_node = graph.AddNode(MakeFusedFilterNode(
         *first_filter_node, *second_filter_node, *fused_predicate, &graph));
 
-    graph.UpdateFanouts(second_filter_node->name(), fused_filter_node->name());
+    TF_RETURN_IF_ERROR(graph.UpdateFanouts(second_filter_node->name(),
+                                           fused_filter_node->name()));
 
     // TODO(prazek): we should run some optimizations on the fused filter
     // functions, or make sure that optimization passes run after filter
diff --git a/tensorflow/core/grappler/optimizers/data/hoist_random_uniform.cc b/tensorflow/core/grappler/optimizers/data/hoist_random_uniform.cc
index 60755256d8..6aa450e123 100644
--- a/tensorflow/core/grappler/optimizers/data/hoist_random_uniform.cc
+++ b/tensorflow/core/grappler/optimizers/data/hoist_random_uniform.cc
@@ -266,7 +266,8 @@ Status HoistRandomUniform::Optimize(Cluster* cluster, const GrapplerItem& item,
     const auto* stateless_map = graph.AddNode(
         MakeStatelessMap(*map_node, *zip_node, *stateless_func, &graph));
 
-    graph.UpdateFanouts(map_node->name(), stateless_map->name());
+    TF_RETURN_IF_ERROR(
+        graph.UpdateFanouts(map_node->name(), stateless_map->name()));
 
     // TODO(b/116285210): we could also remove map functions from library if
     // they are not used anymore.
diff --git a/tensorflow/core/grappler/optimizers/data/latency_all_edges.cc b/tensorflow/core/grappler/optimizers/data/latency_all_edges.cc
index 52b4b785a3..ff0fdbc131 100644
--- a/tensorflow/core/grappler/optimizers/data/latency_all_edges.cc
+++ b/tensorflow/core/grappler/optimizers/data/latency_all_edges.cc
@@ -97,7 +97,7 @@ Status LatencyAllEdges::Optimize(Cluster* cluster, const GrapplerItem& item,
     }
 
     NodeDef* latency_node = graph.AddNode(MakeLatencyNode(node, &graph));
-    graph.UpdateFanouts(node.name(), latency_node->name());
+    TF_RETURN_IF_ERROR(graph.UpdateFanouts(node.name(), latency_node->name()));
   }
   return Status::OK();
 }
diff --git a/tensorflow/core/grappler/optimizers/data/make_numa_aware.cc b/tensorflow/core/grappler/optimizers/data/make_numa_aware.cc
index 72c27a1d4a..2940fbecd4 100644
--- a/tensorflow/core/grappler/optimizers/data/make_numa_aware.cc
+++ b/tensorflow/core/grappler/optimizers/data/make_numa_aware.cc
@@ -47,7 +47,7 @@ Status MakeNumaAware::Optimize(Cluster* cluster, const GrapplerItem& item,
     if (node.op() != "ExperimentalMapAndBatchDataset") continue;
 
     auto* numa_node = graph.AddNode(MakeNumaAwareNode(node, &graph));
-    graph.UpdateFanouts(node.name(), numa_node->name());
+    TF_RETURN_IF_ERROR(graph.UpdateFanouts(node.name(), numa_node->name()));
     nodes_to_delete.insert(node.name());
   }
   graph.DeleteNodes(nodes_to_delete);
diff --git a/tensorflow/core/grappler/optimizers/data/map_and_batch_fusion.cc b/tensorflow/core/grappler/optimizers/data/map_and_batch_fusion.cc
index 84c4d82f6a..4ff749c57b 100644
--- a/tensorflow/core/grappler/optimizers/data/map_and_batch_fusion.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_and_batch_fusion.cc
@@ -120,7 +120,8 @@ Status MapAndBatchFusion::Optimize(Cluster* cluster, const GrapplerItem& item,
 
     auto* new_node =
         graph.AddNode(MakeMapAndBatchNode(*map_node, batch_node, &graph));
-    graph.UpdateFanouts(batch_node.name(), new_node->name());
+    TF_RETURN_IF_ERROR(
+        graph.UpdateFanouts(batch_node.name(), new_node->name()));
 
     // Mark the `Map` and `Batch` nodes for removal.
     nodes_to_delete.insert(map_node->name());
diff --git a/tensorflow/core/grappler/optimizers/data/map_and_filter_fusion.cc b/tensorflow/core/grappler/optimizers/data/map_and_filter_fusion.cc
index 233d7968c8..128a2fefa2 100644
--- a/tensorflow/core/grappler/optimizers/data/map_and_filter_fusion.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_and_filter_fusion.cc
@@ -155,7 +155,8 @@ Status MapAndFilterFusion::Optimize(Cluster* cluster, const GrapplerItem& item,
     const auto* filter_by_component = graph.AddNode(
         MakeFilterByLastComponentNode(*fused_maps, *filter_node, &graph));
 
-    graph.UpdateFanouts(filter_node->name(), filter_by_component->name());
+    TF_RETURN_IF_ERROR(
+        graph.UpdateFanouts(filter_node->name(), filter_by_component->name()));
     TF_RETURN_IF_ERROR(function_library.AddFunctionDef(*fused_function));
 
     // TODO(prazek): we could also remove functions from library if they are not
diff --git a/tensorflow/core/grappler/optimizers/data/map_fusion.cc b/tensorflow/core/grappler/optimizers/data/map_fusion.cc
index 6b8015f96a..504fcc1f5e 100644
--- a/tensorflow/core/grappler/optimizers/data/map_fusion.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_fusion.cc
@@ -130,7 +130,8 @@ Status MapFusion::Optimize(Cluster* cluster, const GrapplerItem& item,
     const auto* fused_maps_node = graph.AddNode(
         MakeFusedNode(*parent_map_node, *map_node, *fused_function, &graph));
 
-    graph.UpdateFanouts(map_node->name(), fused_maps_node->name());
+    TF_RETURN_IF_ERROR(
+        graph.UpdateFanouts(map_node->name(), fused_maps_node->name()));
 
     // TODO(prazek): we should run some optimizations on the fused map
     // functions, or make sure that optimization passes run after map
diff --git a/tensorflow/core/grappler/optimizers/data/map_parallelization.cc b/tensorflow/core/grappler/optimizers/data/map_parallelization.cc
index d335117585..1274af7160 100644
--- a/tensorflow/core/grappler/optimizers/data/map_parallelization.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_parallelization.cc
@@ -89,7 +89,8 @@ Status MapParallelization::Optimize(Cluster* cluster, const GrapplerItem& item,
 
     auto* parallel_map =
         graph.AddNode(MakeParallelMap(map_node->name(), &graph));
-    graph.UpdateFanouts(map_node->name(), parallel_map->name());
+    TF_RETURN_IF_ERROR(
+        graph.UpdateFanouts(map_node->name(), parallel_map->name()));
     nodes_to_delete.insert(map_node->name());
   }
 
diff --git a/tensorflow/core/grappler/optimizers/data/map_vectorization.cc b/tensorflow/core/grappler/optimizers/data/map_vectorization.cc
index 3401dcc6f2..bc1d6d0ea6 100644
--- a/tensorflow/core/grappler/optimizers/data/map_vectorization.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_vectorization.cc
@@ -264,7 +264,8 @@ Status MapVectorization::Optimize(Cluster* cluster, const GrapplerItem& item,
 
     auto* new_map_node = graph.AddNode(MakeNewMapNode(
         *map_node, batch_node, *new_batch_node, *vectorized_func, &graph));
-    graph.UpdateFanouts(batch_node.name(), new_map_node->name());
+    TF_RETURN_IF_ERROR(
+        graph.UpdateFanouts(batch_node.name(), new_map_node->name()));
 
     // Mark the `Map` and `Batch` nodes for removal.
     nodes_to_delete.insert(map_node->name());
diff --git a/tensorflow/core/grappler/optimizers/data/noop_elimination.cc b/tensorflow/core/grappler/optimizers/data/noop_elimination.cc
index bd405c8329..b71e9e5420 100644
--- a/tensorflow/core/grappler/optimizers/data/noop_elimination.cc
+++ b/tensorflow/core/grappler/optimizers/data/noop_elimination.cc
@@ -79,7 +79,7 @@ Status NoOpElimination::Optimize(Cluster* cluster, const GrapplerItem& item,
     if (!IsNoOp(node, graph)) continue;
 
     NodeDef* const parent = graph_utils::GetInputNode(node, graph);
-    graph.UpdateFanouts(node.name(), parent->name());
+    TF_RETURN_IF_ERROR(graph.UpdateFanouts(node.name(), parent->name()));
 
     nodes_to_delete.insert(node.name());
   }
diff --git a/tensorflow/core/grappler/optimizers/data/shuffle_and_repeat_fusion.cc b/tensorflow/core/grappler/optimizers/data/shuffle_and_repeat_fusion.cc
index d9af78d38c..f03648e7e9 100644
--- a/tensorflow/core/grappler/optimizers/data/shuffle_and_repeat_fusion.cc
+++ b/tensorflow/core/grappler/optimizers/data/shuffle_and_repeat_fusion.cc
@@ -86,7 +86,8 @@ Status ShuffleAndRepeatFusion::Optimize(Cluster* cluster,
 
     NodeDef* shuffle_and_repeat_node =
         graph.AddNode(make_shuffle_and_repeat_node(shuffle_node, repeat_node));
-    graph.UpdateFanouts(repeat_node.name(), shuffle_and_repeat_node->name());
+    TF_RETURN_IF_ERROR(graph.UpdateFanouts(repeat_node.name(),
+                                           shuffle_and_repeat_node->name()));
 
     // Mark the `Shuffle` and `Repeat` nodes for removal.
     nodes_to_delete.insert(shuffle_node.name());
-- 
GitLab


From 01c19850c0fd6b587f44ab5938e2d874744a2cbf Mon Sep 17 00:00:00 2001
From: Dimitris Vardoulakis <dimvar@google.com>
Date: Mon, 14 Jan 2019 15:21:18 -0800
Subject: [PATCH 0706/1095] [TF:XLA] Fix crash in AR/CRS combiner, when a
 cross-module AllReduce is immediately followed by a cross-replica AllReduce.

PiperOrigin-RevId: 229265445
---
 .../compiler/xla/service/ar_crs_combiner.cc   |  7 +-
 .../xla/service/ar_crs_combiner_test.cc       | 67 +++++++++++++++++++
 2 files changed, 72 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/xla/service/ar_crs_combiner.cc b/tensorflow/compiler/xla/service/ar_crs_combiner.cc
index 35a32f537b..8e327ee84b 100644
--- a/tensorflow/compiler/xla/service/ar_crs_combiner.cc
+++ b/tensorflow/compiler/xla/service/ar_crs_combiner.cc
@@ -249,14 +249,17 @@ void ArCrsCombiner::KeepProvablyEqualInstructionGroups() {
       auto next_0 = instr_0->users()[0];
       auto next_i = instr_i->users()[0];
       absl::flat_hash_map<int64, int64> visited_pairs;
-      do {
+      while (true) {
         if (!InstructionsComputeSameValue(next_0, next_i, &visited_pairs)) {
           all_reduce_map_.erase(all_reduce_id);
           break;
         }
+        if (next_0->IsCrossReplicaAllReduce()) {
+          break;
+        }
         next_0 = next_0->users()[0];
         next_i = next_i->users()[0];
-      } while (!next_0->IsCrossReplicaAllReduce());
+      }
     }
   }
 }
diff --git a/tensorflow/compiler/xla/service/ar_crs_combiner_test.cc b/tensorflow/compiler/xla/service/ar_crs_combiner_test.cc
index b12b63b2dd..08eb77f032 100644
--- a/tensorflow/compiler/xla/service/ar_crs_combiner_test.cc
+++ b/tensorflow/compiler/xla/service/ar_crs_combiner_test.cc
@@ -706,5 +706,72 @@ ENTRY %entrycomp (p: f32[]) -> (f32[], f32[]) {
   EXPECT_FALSE(changed);
 }
 
+TEST_F(ArCrsCombinerTest, ArThenCrsDontCrash) {
+  const char* module_str = R"(
+HloModule foobar
+
+%sum.1 (a: f32[], b: f32[]) -> f32[] {
+  %a = f32[] parameter(0)
+  %b = f32[] parameter(1)
+  ROOT %add = f32[] add(%a, %b)
+}
+
+ENTRY %entrycomp (p: f32[]) -> (f32[], f32[]) {
+  %p = f32[] parameter(0)
+  %constant.f32 = f32[] constant(123)
+
+  %all-reduce.ar.1 = f32[]
+      all-reduce(%p),
+      replica_groups={{0},{1}},
+      all_reduce_id=1,
+      to_apply=%sum.1,
+      sharding={maximal device=0}
+  %all-reduce.1 = f32[]
+      all-reduce(%all-reduce.ar.1),
+      replica_groups={{0,1}},
+      to_apply=%sum.1,
+      sharding={maximal device=0}
+  %multiply.1 = f32[]
+      multiply(%all-reduce.1, %constant.f32),
+      sharding={maximal device=0}
+
+  %all-reduce.ar.2 = f32[]
+      all-reduce(%p),
+      replica_groups={{0},{1}},
+      all_reduce_id=1,
+      to_apply=%sum.1,
+      sharding={maximal device=1}
+  %all-reduce.2 = f32[]
+      all-reduce(%all-reduce.ar.2),
+      replica_groups={{0,1}},
+      to_apply=%sum.1,
+      sharding={maximal device=1}
+  %multiply.2 = f32[]
+      multiply(%all-reduce.2, %constant.f32),
+      sharding={maximal device=1}
+
+  ROOT %tuple = (f32[], f32[])
+      tuple(%all-reduce.1, %all-reduce.2),
+      sharding={{maximal device=0}, {maximal device=1}}
+}
+)";
+
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
+                          ParseAndReturnVerifiedModule(module_str));
+  auto crs_before =
+      module->entry_computation()->root_instruction()->operands()[0];
+  auto replica_groups_before = crs_before->replica_groups();
+  ArCrsCombiner combiner(2);
+  auto changed = combiner.Run(module.get()).ValueOrDie();
+  EXPECT_TRUE(changed);
+  EXPECT_THAT(module->entry_computation()->root_instruction(),
+              op::Tuple(op::AllReduce(op::Parameter()),
+                        op::AllReduce(op::Parameter())));
+  auto crs_after =
+      module->entry_computation()->root_instruction()->operands()[0];
+  auto replica_groups_after = crs_after->replica_groups();
+  CompareReplicaGroups(replica_groups_before, replica_groups_after);
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From ae37e634c2f39b9514c73faab140a4fb980f976d Mon Sep 17 00:00:00 2001
From: Marek Drozdowski <mdrozdowski@nvidia.com>
Date: Fri, 16 Nov 2018 17:01:15 -0800
Subject: [PATCH 0707/1095] initial commit (add lru_cache class, enable
 trt_engine_op to use it)

change deconstructor

Formatting

Add test for changing input shapes

Dynamic input shapes test passes. Static mode and cached_input_shapes not implemented yet

enable to run network in tests multiple times

Fix and improve input shape cache. Change cached_engine_batch_sizes -> cached_engine_input_shapes

Formatting

Implement SerializeShapesString for TensorShapes too

All tests passing

Revert cached_engine_input_shapes -> cached_engine_batches.

Rename cached_engine_batches -> cached_engine_batch_sizes

Improve hash and remove < operator

Forgot to rename hasher

Fix formatting

remove commented out code

remove lru_cache mutex

change deconstructor and declaration order

Formatting and use default deconstructor

Use default deconstructor

Use Resource manager for LRU cache

Implement feedback. Bug with destructor of resources when not using singleton

Move allocator to TRTEngineCacheResource to fix segfault.

Merge changes from e51fa30
---
 tensorflow/contrib/tensorrt/BUILD             |  22 ++
 .../contrib/tensorrt/convert/convert_graph.cc |  13 +-
 .../contrib/tensorrt/convert/convert_graph.h  |   5 +-
 .../contrib/tensorrt/convert/convert_nodes.h  |   2 +-
 .../tensorrt/convert/trt_optimization_pass.cc |   6 +-
 .../contrib/tensorrt/kernels/trt_engine_op.cc | 223 ++++++++++--------
 .../contrib/tensorrt/kernels/trt_engine_op.h  |  29 +--
 .../contrib/tensorrt/ops/trt_engine_op.cc     |   2 +-
 .../contrib/tensorrt/python/trt_convert.py    |   6 +-
 .../tensorrt/python/trt_convert_test.py       |   4 +-
 .../tensorrt/resources/trt_lru_cache.h        | 175 ++++++++++++++
 .../tensorrt/resources/trt_lru_cache_test.cc  |  57 +++++
 tensorflow/contrib/tensorrt/test/base_test.py |  28 +--
 .../tensorrt/test/batch_matmul_test.py        |   4 +-
 .../tensorrt/test/biasadd_matmul_test.py      |   4 +-
 .../binary_tensor_weight_broadcast_test.py    |   4 +-
 .../tensorrt/test/concatenation_test.py       |   4 +-
 .../tensorrt/test/const_broadcast_test.py     |   4 +-
 .../test/dynamic_input_shapes_test.py         | 105 +++++++++
 .../contrib/tensorrt/test/lru_cache_test.py   |  62 +++++
 .../contrib/tensorrt/test/manual_test.py      |   4 +-
 .../tensorrt/test/memory_alignment_test.py    |   4 +-
 .../multi_connection_neighbor_engine_test.py  |   4 +-
 .../tensorrt/test/neighboring_engine_test.py  |   4 +-
 .../tensorrt/test/quantization_test.py        |   4 +-
 .../contrib/tensorrt/test/rank_two_test.py    |   4 +-
 .../tensorrt/test/reshape_transpose_test.py   |   8 +-
 .../test/tf_trt_integration_test_base.py      |  93 ++++----
 .../contrib/tensorrt/test/unary_test.py       |   4 +-
 .../tensorrt/test/vgg_block_nchw_test.py      |   4 +-
 .../contrib/tensorrt/test/vgg_block_test.py   |   4 +-
 31 files changed, 671 insertions(+), 225 deletions(-)
 create mode 100644 tensorflow/contrib/tensorrt/resources/trt_lru_cache.h
 create mode 100644 tensorflow/contrib/tensorrt/resources/trt_lru_cache_test.cc
 create mode 100644 tensorflow/contrib/tensorrt/test/dynamic_input_shapes_test.py
 create mode 100644 tensorflow/contrib/tensorrt/test/lru_cache_test.py

diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index 9b798425b6..3df56271de 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -222,6 +222,7 @@ tf_cuda_library(
     hdrs = [
         "resources/trt_int8_calibrator.h",
         "resources/trt_resource_manager.h",
+        "resources/trt_lru_cache.h",
         "resources/trt_resources.h",
     ],
     deps = [
@@ -264,6 +265,21 @@ tf_cc_test(
     ],
 )
 
+tf_cc_test(
+    name = "trt_lru_cache_test",
+    size = "small",
+    srcs = ["resources/trt_lru_cache_test.cc"],
+    tags = [
+        "no_windows",
+        "nomac",
+    ],
+    deps = [
+        ":trt_resources",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ],
+)
+
 # Library for the node-level conversion portion of TensorRT operation creation
 tf_cuda_library(
     name = "trt_conversion",
@@ -493,8 +509,13 @@ cuda_py_tests(
         "test/concatenation_test.py",
         "test/const_broadcast_test.py",
         "test/conv2d_test.py",
+        "test/dynamic_input_shapes_test.py",
         "test/identity_output_test.py",
+<<<<<<< ab254f392694eb2f7231526c64fc301ef10a1384
         "test/int32_test.py",
+=======
+        "test/lru_cache_test.py",
+>>>>>>> initial commit (add lru_cache class, enable trt_engine_op to use it)
         "test/manual_test.py",
         "test/memory_alignment_test.py",
         "test/multi_connection_neighbor_engine_test.py",
@@ -550,6 +571,7 @@ cc_library(
     copts = tf_copts(),
     deps = [
         "//tensorflow/core:lib",
+        "//tensorflow/core:framework"
     ],
 )
 
diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index b1f631bf5f..87c2d367f8 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -239,7 +239,7 @@ tensorflow::Status ConvertGraphDefToTensorRT(
     const std::vector<string>& output_names, size_t max_batch_size,
     size_t max_workspace_size_bytes, tensorflow::GraphDef* new_graph_def,
     int precision_mode, int minimum_segment_size, bool is_dyn_op,
-    int max_cached_engines, std::vector<int> cached_engine_batches,
+    int max_cached_engines, std::vector<int> cached_engine_batch_sizes,
     bool use_calibration) {
   // Create GrapplerItem.
   tensorflow::grappler::GrapplerItem item;
@@ -301,9 +301,9 @@ tensorflow::Status ConvertGraphDefToTensorRT(
   TF_RETURN_IF_ERROR(GetPrecisionModeName(
       precision_mode, parameters["precision_mode"].mutable_s()));
   parameters["maximum_cached_engines"].set_i(max_cached_engines);
-  if (!cached_engine_batches.empty()) {
-    auto list = parameters["cached_engine_batches"].mutable_list();
-    for (const int batch : cached_engine_batches) {
+  if (!cached_engine_batch_sizes.empty()) {
+    auto list = parameters["cached_engine_batch_sizes"].mutable_list();
+    for (const int batch : cached_engine_batch_sizes) {
       list->add_i(batch);
     }
   }
@@ -689,7 +689,7 @@ tensorflow::Status CreateTRTNode(const std::vector<EngineInfo>& infos, int pos,
   }
 
   if (info.engine_type == EngineInfo::EngineType::TRTStatic &&
-      info.cached_engine_batches.size()) {
+      !info.cached_engine_batch_sizes.empty()) {
     LOG(WARNING) << "Cached engine batches are ignored for static engines";
   }
   tensorflow::NodeDef trt_node;
@@ -703,7 +703,6 @@ tensorflow::Status CreateTRTNode(const std::vector<EngineInfo>& infos, int pos,
           .Attr("serialized_segment", segment_string)
           .Attr("calibration_data", "")
           .Attr("max_cached_engines_count", info.maximum_cached_engines)
-          .Attr("cached_engine_batches", {max_batch_size})
           .Attr("workspace_size_bytes", info.max_workspace_size_bytes)
           .Attr("precision_mode", prec_string)
           .Attr("use_calibration", info.use_calibration)
@@ -984,7 +983,7 @@ tensorflow::Status ConvertAfterShapes(ConversionParams& params) {
                                    ? EngineInfo::EngineType::TRTDynamic
                                    : EngineInfo::EngineType::TRTStatic);
     curr_engine.use_calibration = params.use_calibration;
-    curr_engine.cached_engine_batches = params.cached_engine_batches;
+    curr_engine.cached_engine_batch_sizes = params.cached_engine_batch_sizes;
     curr_engine.maximum_cached_engines = params.max_cached_engines;
     StrAppend(&curr_engine.engine_name, "TRTEngineOp_", t);
     status = RegisterSegmentFunctionToFunctionLibrary(
diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.h b/tensorflow/contrib/tensorrt/convert/convert_graph.h
index 1f39f56f63..5616bbefd9 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.h
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.h
@@ -82,7 +82,7 @@ struct ConversionParams {
   bool fixed_input_size;   // Assume non-batch ranks of input tensors are fixed
   int max_cached_engines;  // maximum number of cached engines
   bool use_calibration;
-  std::vector<int> cached_engine_batches;  // list of cached engines
+  std::vector<int> cached_engine_batch_sizes;  // list of cached engines
 };
 
 // This method extracts calibration information from the resource managers
@@ -101,7 +101,8 @@ tensorflow::Status ConvertGraphDefToTensorRT(
     size_t max_workspace_size_bytes, tensorflow::GraphDef* new_graph_def,
     int precision_mode = 1, int minimum_segment_size = 3,
     bool is_dyn_op = false, int max_cached_engines = 1,
-    std::vector<int> cached_engine_batches = {}, bool use_calibration = true);
+    std::vector<int> cached_engine_batch_sizes = {},
+    bool use_calibration = true);
 
 // Method to call from optimization pass
 tensorflow::Status ConvertAfterShapes(ConversionParams& params);
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.h b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
index 8f2271ee3f..f9f7c0bdf1 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.h
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
@@ -108,7 +108,7 @@ struct EngineInfo {
   EngineType engine_type;
   int64 max_workspace_size_bytes;
   int maximum_cached_engines;
-  std::vector<int> cached_engine_batches;
+  std::vector<int> cached_engine_batch_sizes;
   int precision_mode;
   bool use_calibration;
 };
diff --git a/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc b/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc
index d57f2300f8..e7ca126f55 100644
--- a/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc
+++ b/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc
@@ -50,8 +50,8 @@ tensorflow::Status TRTOptimizationPass::Init(
   if (params.count("is_dynamic_op")) {
     is_dynamic_op_ = params.at("is_dynamic_op").b();
   }
-  if (params.count("cached_engine_batches")) {
-    auto batch_vec = params.at("cached_engine_batches").list();
+  if (params.count("cached_engine_batch_sizes")) {
+    auto batch_vec = params.at("cached_engine_batch_sizes").list();
     batches_.reserve(batch_vec.i_size());
     for (const auto i : batch_vec.i()) {
       batches_.push_back(i);
@@ -258,7 +258,7 @@ tensorflow::Status TRTOptimizationPass::Optimize(
   cp.graph_properties = &static_graph_properties;
   cp.cluster = cluster;
   cp.is_dyn_op = is_dynamic_op_;
-  cp.cached_engine_batches = batches_;
+  cp.cached_engine_batch_sizes = batches_;
   cp.max_cached_engines = max_cached_batches_;
   cp.use_calibration = use_calibration_;
   auto status = tensorflow::tensorrt::convert::ConvertAfterShapes(cp);
diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
index ae5e64e823..e1e64fec68 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
+++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
@@ -136,14 +136,13 @@ TRTEngineOp::TRTEngineOp(OpKernelConstruction* context)
   native_func_ = tensorflow::kInvalidHandle;
   OP_REQUIRES_OK(context, context->GetAttr("max_cached_engines_count",
                                            &max_cached_engines_));
-  OP_REQUIRES_OK(context,
-                 context->GetAttr("fixed_input_size", &fixed_input_size_));
-  OP_REQUIRES_OK(context, context->GetAttr("cached_engine_batches",
-                                           &cached_engine_batches_));
-  std::sort(cached_engine_batches_.begin(), cached_engine_batches_.end());
+  OP_REQUIRES_OK(context, context->GetAttr("cached_engine_batch_sizes",
+                                           &cached_engine_batch_sizes_));
+  std::sort(cached_engine_batch_sizes_.begin(),
+            cached_engine_batch_sizes_.end());
   if (VLOG_IS_ON(1)) {
     string s("Engine Batches= ");
-    for (auto i : cached_engine_batches_) {
+    for (auto i : cached_engine_batch_sizes_) {
       StrAppend(&s, i, " ");
     }
     VLOG(1) << s;
@@ -240,32 +239,30 @@ void TRTEngineOp::ExecuteCalibration(OpKernelContext* ctx,
   ExecuteNativeSegment(ctx, helper);
 }
 
-int TRTEngineOp::GetEngineBatch(OpKernelContext* ctx) {
-  int num_batch = ctx->input(0).shape().dim_size(0);
-  int smallest_engine = 0;
-  for (const auto i : cached_engine_batches_) {
-    if (i >= num_batch) {
-      smallest_engine = i;
-      break;
-    }
-  }
-  // TODO(sami): Need an LRU here
-  if (smallest_engine == 0) {
-    if (max_cached_engines_ > cached_engine_batches_.size()) {
-      smallest_engine = num_batch;
-      cached_engine_batches_.push_back(num_batch);
-      VLOG(1) << "Running with batch size " << num_batch;
-    } else {
-      string msg =
-          StrCat("Engine buffer is full. buffer limit=", max_cached_engines_,
-                 ", current entries=");
-      for (auto i : cached_engine_batches_) StrAppend(&msg, i, ",");
-      StrAppend(&msg, " requested batch=", num_batch);
-      LOG(WARNING) << msg;
-      return -1;
+bool TRTEngineOp::GetCompatibleCachedEngine(
+    const std::vector<TensorShape>& actual_input_shapes,
+    std::vector<TensorShape>* engine_input_shapes) {
+  const int batch_size = actual_input_shapes[0].dim_size(0);
+  int smallest_batch_size = -1;
+  // Output shape will always be the same as the input but we will overwrite the
+  // batch size.
+  *engine_input_shapes = actual_input_shapes;
+  for (const int cached_batch_size : cached_engine_batch_sizes_) {
+    // Check if compatible: batch <= cached batch.
+    if (batch_size <= cached_batch_size) {
+      // First case: first compatible engine found
+      // Second case: smaller batch size engine found
+      if ((smallest_batch_size == -1) ||
+          (cached_batch_size < smallest_batch_size)) {
+        smallest_batch_size = cached_batch_size;
+        // Overwrite batch size for output
+        for (int i = 0; i < engine_input_shapes->size(); i++) {
+          (*engine_input_shapes)[i].set_dim(0, smallest_batch_size);
+        }
+      }
     }
   }
-  return smallest_engine;
+  return (smallest_batch_size != -1);
 }
 
 void TRTEngineOp::ComputeAsync(OpKernelContext* ctx,
@@ -276,23 +273,20 @@ void TRTEngineOp::ComputeAsync(OpKernelContext* ctx,
     ExecuteCalibration(ctx, helper);
     return;
   }
-  const int smallest_engine = GetEngineBatch(ctx);
-  if (smallest_engine < 0) {
-    LOG(WARNING) << "Failed to get engine batch, running native segment for "
-                 << name();
-    ExecuteNativeSegment(ctx, helper);
-    return;
+  // Get shapes of inputs to engine.
+  std::vector<tensorflow::TensorShape> input_shapes;
+  for (int i = 0; i < ctx->num_inputs(); ++i) {
+    input_shapes.emplace_back(ctx->input(i).shape());
   }
-
-  const int num_batch = ctx->input(0).shape().dim_size(0);
-  EngineContext* engine_context = GetEngine(smallest_engine, ctx);
+  EngineContext* engine_context = GetEngine(input_shapes, ctx);
   if (!engine_context->cuda_engine) {
-    LOG(WARNING) << "Engine retrieval for batch size " << num_batch
+    LOG(WARNING) << "Engine retrieval for input shapes: "
+                 << TensorShapeUtils::ShapeListString(input_shapes)
                  << " failed. Running native segment for " << name();
     ExecuteNativeSegment(ctx, helper);
     return;
   }
-  const bool retry = ExecuteTrtEngine(ctx, num_batch, engine_context);
+  const bool retry = ExecuteTrtEngine(ctx, engine_context);
   if (retry) {
     LOG(WARNING) << "Failed to execute engine, "
                  << "retrying with native segment for " << name();
@@ -301,12 +295,15 @@ void TRTEngineOp::ComputeAsync(OpKernelContext* ctx,
   }
 }
 
-bool TRTEngineOp::ExecuteTrtEngine(OpKernelContext* ctx, const int num_batch,
+bool TRTEngineOp::ExecuteTrtEngine(OpKernelContext* ctx,
                                    EngineContext* engine_context) {
-  auto& cuda_engine = engine_context->cuda_engine;
-
+  
   VLOG(1) << "Executing TRT engine: " << name();
+  auto& cuda_engine = engine_context->cuda_engine;
   const bool kRetry = true;
+  // All inputs must have the same batch size, so just get it from the first
+  // input.
+  const int num_batch = ctx->input(0).shape().dim_size(0);
   const int num_binding = ctx->num_inputs() + ctx->num_outputs();
   std::vector<void*> buffers(num_binding);
   for (int i = 0; i < ctx->num_inputs(); i++) {
@@ -417,48 +414,45 @@ bool TRTEngineOp::ExecuteTrtEngine(OpKernelContext* ctx, const int num_batch,
   return !kRetry;
 }
 
-TRTEngineOp::~TRTEngineOp() {
-  // We need to manually destroy the engine and execution context before
-  // the allocator is destructed.
-  for (auto& eng : engine_map_) {
-    eng.second->cuda_engine.reset();
-    eng.second->execution_context.reset();
-  }
-  allocator_.reset();
-}
-
-nvinfer1::IGpuAllocator* TRTEngineOp::GetAllocator(OpKernelContext* ctx) {
-  if (allocator_) return allocator_.get();
-  auto device = ctx->device();
-  auto alloc = device->GetAllocator(tensorflow::AllocatorAttributes());
-  if (!alloc) {
-    LOG(ERROR) << "Can't find device allocator for gpu device "
-               << device->name();
-    return nullptr;
-  }
-  allocator_.reset(new TRTDeviceAllocator(alloc));
-  return allocator_.get();
-}
-
-TRTEngineOp::EngineContext* TRTEngineOp::GetEngine(int batch_size,
-                                                   OpKernelContext* ctx) {
+EngineContext* TRTEngineOp::GetEngine(
+    const std::vector<TensorShape>& input_shapes, OpKernelContext* ctx) {
   static EngineContext empty_context;
-  // TODO(sami): This method needs to be re-written to use resource manager and
-  // with LRU mechanism option.
   tensorflow::mutex_lock lock(engine_mutex_);
+  // TODO(tmorris): using first input to get batch size - is this reliable?
+  const int batch_size = input_shapes[0].dim_size(0);
+
+  // Get engine cache
+  TRTEngineCacheResource* cache_res = nullptr;
+  auto status = ctx->resource_manager()->LookupOrCreate(
+      "TRTEngineCache", funcdef_name_, &cache_res,
+      {[this, ctx](TRTEngineCacheResource** cr) -> tensorflow::Status {
+        *cr = new TRTEngineCacheResource(ctx, this->max_cached_engines_);
+        return Status::OK();
+      }});
+  if (!status.ok()) {
+    ctx->SetStatus(status);
+    return &empty_context;
+  }
+  tensorflow::core::ScopedUnref sc(cache_res);
+  auto& cache = cache_res->cache_;
+  auto allocator = cache_res->allocator_.get();
+  if (allocator == nullptr) {
+    return &empty_context;
+  }
 
+  // Handle the static engine case. For static engines, the cache will have a
+  // single element containing the only engine.
   if (static_engine_) {
-    if (engine_map_.size()) {
-      if (engine_map_.begin()->first >= batch_size) {
-        return engine_map_.begin()->second.get();
+    if (cache.size()) {
+      // Batch size of engine must be >= the input batch size
+      // TODO(tmorris): use match compatible function?
+      if (cache.begin()->first[0].dim_size(0) >= batch_size) {
+        return cache.begin()->second.get();
       }
       return &empty_context;
     }
+
     TrtUniquePtrType<IRuntime> infer(nvinfer1::createInferRuntime(logger));
-    auto allocator = GetAllocator(ctx);
-    if (allocator == nullptr) {
-      return &empty_context;
-    }
     infer->setGpuAllocator(allocator);
     TrtUniquePtrType<nvinfer1::ICudaEngine> static_engine(
         infer->deserializeCudaEngine(serialized_segment_.c_str(),
@@ -466,47 +460,71 @@ TRTEngineOp::EngineContext* TRTEngineOp::GetEngine(int batch_size,
                                      PluginFactoryTensorRT::GetInstance()));
     auto raw_static_engine = static_engine.get();
     const auto max_batch_size = raw_static_engine->getMaxBatchSize();
-    engine_map_[max_batch_size] = absl::make_unique<EngineContext>(
-        std::move(static_engine),
-        TrtUniquePtrType<nvinfer1::IExecutionContext>(
-            raw_static_engine->createExecutionContext()));
+    // Static engine will have max_batch_size for batch size so that all inputs
+    // will map to this single engine.
+    std::vector<TensorShape> engine_input_shapes(input_shapes);
+    for (int i = 0; i < engine_input_shapes.size(); i++) {
+      // TODO(tmorris): will all inputs have batch size as first dimension??
+      engine_input_shapes[i].set_dim(0, max_batch_size);
+    }
+    // Assume engine_input_shapes matches the input shapes of the engine.
+    cache.emplace(
+        engine_input_shapes, absl::make_unique<EngineContext>(
+            std::move(static_engine),
+            TrtUniquePtrType<nvinfer1::IExecutionContext>(
+                raw_static_engine->createExecutionContext())));
     // Runtime is safe to delete after engine creation
     serialized_segment_.clear();
     if (max_batch_size < batch_size) {
       return &empty_context;
     }
-    return engine_map_.at(max_batch_size).get();
+    return cache.at(input_shapes).get();
   }  // static_engine_
 
   // Handle the dynamic engine case.
-  auto engine_it = engine_map_.find(batch_size);
-  if (engine_it == engine_map_.end() &&
-      engine_map_.size() < (size_t)max_cached_engines_) {
-    nvinfer1::IGpuAllocator* allocator = nullptr;
-    allocator = GetAllocator(ctx);
-    if (allocator == nullptr) {
-      return &empty_context;
-    }
-    std::vector<tensorflow::PartialTensorShape> shapes;
-    for (int i = 0; i < ctx->num_inputs(); ++i) {
-      shapes.emplace_back(ctx->input(i).shape());
+  // See if there is a compatible engine cached. The batch size should be <= the
+  // cached batch size.
+  std::vector<tensorflow::TensorShape> engine_input_shapes;
+  bool matched_successfully =
+      GetCompatibleCachedEngine(input_shapes, &engine_input_shapes);
+  // If matched, use that engine. Otherwise, we will look in cache for that
+  // exact shape and possibly create a new engine if it is not in cache.
+  if (!matched_successfully) {
+    engine_input_shapes = input_shapes;
+    if (!cached_engine_batch_sizes_.empty()) {
+      // If user has explicitly defined cached_engine_batch_sizes, we should
+      // warn them that their input was non-compatible (batch size too high)
+      LOG(WARNING) << "No compatible cached engine was found for batch size: "
+                   << batch_size << ". A new engine will be created.";
+      cached_engine_batch_sizes_.push_back(batch_size);
     }
+  }
+  tensorflow::Status engine_found_status;
+  cache.at(engine_input_shapes, &engine_found_status);
+
+  if (!engine_found_status.ok()) {
     TrtUniquePtrType<nvinfer1::ICudaEngine> engine;
     bool convert_successfully = false;
     LOG(INFO) << "Building a new TensorRT engine for " << name()
-              << " with batch size " << batch_size;
+              << " input shapes: "
+              << TensorShapeUtils::ShapeListString(engine_input_shapes);
+    // Convert to partial shapes
+    std::vector<PartialTensorShape> partial_shapes;
+    for (int i = 0; i < engine_input_shapes.size(); i++) {
+      partial_shapes.emplace_back(engine_input_shapes[i]);
+    }
     // Up to this point, calibrator_ can never be empty, since otherwise it
     // means calibration_mode_ is true and this path won't get executed.
     auto status = convert::ConvertGraphDefToEngine(
-        segment_graph_, precision_mode_, batch_size, workspace_size_, shapes,
-        &logger, allocator, calibrator_.get(), &engine, use_calibration_,
-        &convert_successfully);
+        segment_graph_, precision_mode_, batch_size, workspace_size_,
+        partial_shapes, &logger, allocator, calibrator_.get(), &engine,
+        use_calibration_, &convert_successfully);
     if (!status.ok()) {
       if (convert_successfully) {
         // This means it fail to build the engine even when the network is built
         // successfully, probably due to internal issues. In this case we don't
         // retry in the future.
-        engine_map_[batch_size] = absl::make_unique<EngineContext>();
+        cache.emplace(engine_input_shapes, absl::make_unique<EngineContext>());
       }
       LOG(WARNING) << "Engine creation for batch size " << batch_size
                    << " failed " << status;
@@ -515,10 +533,11 @@ TRTEngineOp::EngineContext* TRTEngineOp::GetEngine(int batch_size,
     VLOG(1) << "Conversion is done";
     TrtUniquePtrType<nvinfer1::IExecutionContext> exec_context(
         engine->createExecutionContext());
-    engine_map_[batch_size] = absl::make_unique<EngineContext>(
-        std::move(engine), std::move(exec_context));
+    cache.emplace(engine_input_shapes,
+                  absl::make_unique<EngineContext>(std::move(engine),
+                                                   std::move(exec_context)));
   }
-  return engine_map_.at(batch_size).get();
+  return cache.at(engine_input_shapes).get();
 }
 
 tensorflow::Status TRTEngineOp::AllocateCalibrationResources(
diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
index 38ea224063..4f39a4c548 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
+++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
@@ -22,6 +22,7 @@ limitations under the License.
 #include "tensorflow/contrib/tensorrt/convert/utils.h"
 #include "tensorflow/contrib/tensorrt/log/trt_logger.h"
 #include "tensorflow/contrib/tensorrt/resources/trt_allocator.h"
+#include "tensorflow/contrib/tensorrt/resources/trt_lru_cache.h"
 #include "tensorflow/core/framework/function.h"
 #include "tensorflow/core/framework/graph.pb.h"
 #include "tensorflow/core/framework/op.h"
@@ -49,7 +50,6 @@ class TRTEngineOp : public AsyncOpKernel {
 
   void ComputeAsync(OpKernelContext* context,
                     AsyncOpKernel::DoneCallback done) override;
-  ~TRTEngineOp();
 
  private:
   // TODO(samikama): context should go to a resource manager!
@@ -78,28 +78,25 @@ class TRTEngineOp : public AsyncOpKernel {
 
   // Execute the tensorrt engine. Returns whether we need to retry by running
   // the native segment.
-  bool ExecuteTrtEngine(OpKernelContext* ctx, const int num_batch,
-                        EngineContext* engine_context);
+  bool ExecuteTrtEngine(OpKernelContext* ctx, EngineContext* engine_context);
 
   // Allocate necessary resources for calibration
   Status AllocateCalibrationResources(OpKernelContext* ctx,
                                       TRTCalibrationResource** cr);
 
-  EngineContext* GetEngine(int batch_size, OpKernelContext* ctx);
+  // Get engine for the input shape
+  EngineContext* GetEngine(const std::vector<TensorShape>& input_shapes,
+                           OpKernelContext* ctx);
 
-  // Return engine batch closest to input batch.
-  int GetEngineBatch(OpKernelContext* ctx);
+  // Return engine batch in cached_engne_batch_sizes_ which is closest to input
+  // batch.
+  bool GetCompatibleCachedEngine(
+      const std::vector<TensorShape>& actual_input_shapes,
+      std::vector<TensorShape>* engine_input_shapes);
 
-  nvinfer1::IGpuAllocator* GetAllocator(OpKernelContext* ctx);
-
-  // map to keep engines and their execution context for given batch size.
-  std::unordered_map<int, std::unique_ptr<EngineContext>> engine_map_;
   std::vector<string> input_nodes_;
   std::vector<string> output_nodes_;
 
-  // keep device allocator for TRT.
-  std::unique_ptr<TRTBaseAllocator> allocator_;
-
   // serialized protobuf segment or trt engine depending on static_engine_ flag.
   string serialized_segment_;
 
@@ -119,12 +116,8 @@ class TRTEngineOp : public AsyncOpKernel {
   // Whether to calibrate INT8 engine.
   bool calibration_mode_;
 
-  // Whether non-batch ranks of the inputs are assumed to be fixed or not for
-  // engine construction.
-  bool fixed_input_size_;
-
   // Batches of the cached engines
-  std::vector<int> cached_engine_batches_;
+  std::vector<int> cached_engine_batch_sizes_;
 
   // Maximum number of cached engines
   int max_cached_engines_;
diff --git a/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc b/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc
index 2f761c8b8e..73cc046bc3 100644
--- a/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc
+++ b/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc
@@ -40,7 +40,7 @@ REGISTER_OP("TRTEngineOp")
     .Attr("OutT: list({int8,float16,float32,int32})")
     .Attr("static_engine: bool = true")
     .Attr("fixed_input_size: bool = true")
-    .Attr("cached_engine_batches: list(int) = []")
+    .Attr("cached_engine_batch_sizes: list(int) >= 0 = []")
     .Attr("max_cached_engines_count: int = 1")
     .Attr("workspace_size_bytes: int")
     .Attr("precision_mode: {'FP32', 'FP16', 'INT8'}")
diff --git a/tensorflow/contrib/tensorrt/python/trt_convert.py b/tensorflow/contrib/tensorrt/python/trt_convert.py
index 92377edaec..8de17347f8 100644
--- a/tensorflow/contrib/tensorrt/python/trt_convert.py
+++ b/tensorflow/contrib/tensorrt/python/trt_convert.py
@@ -99,7 +99,7 @@ def get_tensorrt_rewriter_config(rewriter_config=None,
       based on which the TRTEngineOp is created.
     cached_engine_batch_sizes: a list of batch sizes used to create cached
       engines, only used when is_dynamic_op is True. The length of the list
-      should be smaller than maximum_cached_engines, and the dynamic TRT op will
+      should be <= maximum_cached_engines, and the dynamic TRT op will
       use this list to determine the batch sizes of the cached engines, instead
       of making the decision on the fly. This is useful when we know the most
       common batch size(s) the application is going to generate.
@@ -156,7 +156,7 @@ def get_tensorrt_rewriter_config(rewriter_config=None,
     if len(cached_engine_batch_sizes) > maximum_cached_engines:
       raise ValueError("cached_engine_batch_sizes should not contain more than "
                        "maximum_cached_engines items.")
-    optimizer.parameter_map["cached_engine_batches"].list.i.extend(
+    optimizer.parameter_map["cached_engine_batch_sizes"].list.i.extend(
         cached_engine_batch_sizes)
   optimizer.parameter_map["use_calibration"].b = use_calibration
   return rewriter_config_with_trt
@@ -199,7 +199,7 @@ def create_inference_graph(input_graph_def,
       based on which the TRTEngineOp is created.
     cached_engine_batch_sizes: a list of batch sizes used to create cached
       engines, only used when is_dynamic_op is True. The length of the list
-      should be smaller than maximum_cached_engines, and the dynamic TRT op will
+      should be <= maximum_cached_engines, and the dynamic TRT op will
       use this list to determine the batch sizes of the cached engines, instead
       of making the decision on the fly. This is useful when we know the most
       common batch size(s) the application is going to generate.
diff --git a/tensorflow/contrib/tensorrt/python/trt_convert_test.py b/tensorflow/contrib/tensorrt/python/trt_convert_test.py
index a7b2d2ea50..60c31580db 100644
--- a/tensorflow/contrib/tensorrt/python/trt_convert_test.py
+++ b/tensorflow/contrib/tensorrt/python/trt_convert_test.py
@@ -71,7 +71,7 @@ class TrtConvertTest(test_util.TensorFlowTestCase):
     for key in [
         "minimum_segment_size", "max_batch_size", "is_dynamic_op",
         "max_workspace_size_bytes", "precision_mode", "maximum_cached_engines",
-        "cached_engine_batches"
+        "cached_engine_batch_sizes"
     ]:
       self.assertTrue(key in trt_optimizer.parameter_map)
     self.assertEqual(10, trt_optimizer.parameter_map["minimum_segment_size"].i)
@@ -85,7 +85,7 @@ class TrtConvertTest(test_util.TensorFlowTestCase):
     self.assertEqual(2, trt_optimizer.parameter_map["maximum_cached_engines"].i)
     self.assertEqual(
         [1, 128],
-        trt_optimizer.parameter_map["cached_engine_batches"].list.i)
+        trt_optimizer.parameter_map["cached_engine_batch_sizes"].list.i)
 
   def _GetConfigProto(self):
     """Get ConfigProto for session creation."""
diff --git a/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h b/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h
new file mode 100644
index 0000000000..4d63a03b40
--- /dev/null
+++ b/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h
@@ -0,0 +1,175 @@
+#ifndef TENSORFLOW_CONTRIB_TENSORRT_TRT_LRU_CACHE_H_
+#define TENSORFLOW_CONTRIB_TENSORRT_TRT_LRU_CACHE_H_
+
+#include <list>
+#include <unordered_map>
+
+#include "tensorflow/contrib/tensorrt/convert/utils.h"
+#include "tensorflow/contrib/tensorrt/resources/trt_allocator.h"
+#include "tensorflow/core/framework/resource_mgr.h"
+#include "tensorflow/core/lib/core/errors.h"
+
+#if GOOGLE_CUDA
+#if GOOGLE_TENSORRT
+#include "tensorrt/include/NvInfer.h"
+
+namespace tensorflow {
+namespace tensorrt {
+
+template <class Key, class Value, class HashFunction>
+class LRUCache {
+ public:
+  typedef Value value_type;
+  typedef Key key_type;
+  typedef HashFunction hasher;
+  typedef typename std::unordered_map<key_type, value_type, hasher>::iterator
+      map_iterator;
+
+  LRUCache() : capacity_(0) {}
+  explicit LRUCache(size_t capacity) : capacity_(capacity) {}
+
+  size_t capacity() const { return capacity_; }
+
+  void reserve(size_t capacity) {
+    capacity_ = capacity;
+    DiscardOld();
+  }
+
+  size_t size() const { return objects_.size(); }
+
+  size_t count(const key_type& k) const { return objects_.count(k); }
+
+  value_type& at(key_type k, tensorflow::Status* status_ptr = nullptr) {
+    tensorflow::Status status = Touch(k);
+    if (!status.ok()) {
+      if (status_ptr) {
+        *status_ptr = status;
+      }
+      return not_found_value_;
+    }
+    return objects_.at(k);
+  }
+
+  map_iterator begin() { return objects_.begin(); }
+
+  map_iterator end() { return objects_.end(); }
+
+  template <typename... Args>
+  std::pair<map_iterator, bool> emplace(Args&&... args) {
+    DiscardOld(1);
+    std::pair<map_iterator, bool> result =
+        objects_.emplace(std::forward<Args>(args)...);
+    key_type key = result.first->first;
+    if (result.second) {
+      keys_.push_front(key);
+    } else {
+      Touch(key);
+    }
+    return result;
+  }
+
+ private:
+  std::unordered_map<key_type, value_type, hasher> objects_;
+  std::list<key_type> keys_;
+  size_t capacity_;
+  value_type not_found_value_;
+
+  tensorflow::Status Touch(const key_type& key) {
+    if (!count(key)) {
+      return tensorflow::errors::NotFound("Key not found in cache.");
+    }
+    auto rank = std::find(keys_.begin(), keys_.end(), key);
+    if (rank != keys_.begin()) {
+      keys_.erase(rank);
+      keys_.push_front(key);
+    }
+    return tensorflow::Status::OK();
+  }
+
+  // Creates n free positions in cache
+  tensorflow::Status DiscardOld(size_t n = 0) {
+    if (n > capacity_) {
+      return tensorflow::errors::Internal(
+          "Insufficient capacity in cache (capacity = ", capacity_,
+          ", requested ", n, ")");
+    }
+    while (objects_.size() > (capacity_ - n)) {
+      key_type discard_key = keys_.back();
+      keys_.pop_back();
+      objects_.erase(discard_key);
+    }
+    return tensorflow::Status::OK();
+  }
+};
+
+// Define a hash function for vector<TensorShape> because it is used as the key
+// for the engine cache.
+struct VectorTensorShapeHasher {
+  std::size_t operator()(
+      const std::vector<tensorflow::TensorShape>& key) const {
+    return std::hash<std::string>()(TensorShapeUtils::ShapeListString(key));
+  }
+};
+
+struct EngineContext {
+  EngineContext() {}  // Creates an empty context.
+  EngineContext(
+      TrtUniquePtrType<nvinfer1::ICudaEngine>&& input_cuda_engine,
+      TrtUniquePtrType<nvinfer1::IExecutionContext>&& input_execution_context)
+      : cuda_engine(std::move(input_cuda_engine)),
+        execution_context(std::move(input_execution_context)) {}
+
+  mutex mu;
+  TrtUniquePtrType<nvinfer1::ICudaEngine> cuda_engine;
+  TrtUniquePtrType<nvinfer1::IExecutionContext> execution_context
+      GUARDED_BY(mu);
+};
+
+class TRTEngineCacheResource : public tensorflow::ResourceBase {
+ public:
+  TRTEngineCacheResource(OpKernelContext* ctx, size_t capacity)
+      : cache_(capacity) {
+    auto device = ctx->device();
+    auto alloc = device->GetAllocator(tensorflow::AllocatorAttributes());
+    if (!alloc) {
+      LOG(ERROR) << "Can't find device allocator for gpu device "
+                 << device->name();
+      allocator_ = nullptr;
+    } else {
+      allocator_.reset(new TRTDeviceAllocator(alloc));
+    }
+  }
+
+  string DebugString() override {
+    std::stringstream oss;
+    using std::dec;
+    using std::endl;
+    using std::hex;
+    oss << "TRTEngineCacheResource: ";
+    oss << "TRTBaseAllocator = " << hex << allocator_.get() << dec << ", ";
+    oss << "LRUCache = " << hex << &cache_ << dec << endl;
+    oss << "Containing " << cache_.size() << " entries: " << endl;
+    for (auto& item : cache_) {
+      oss << TensorShapeUtils::ShapeListString(item.first) << ": " << hex
+          << "ICudaEngine: " << item.second.get()->cuda_engine.get() << ", "
+          << "IExecutionContext: " << item.second.get()->execution_context.get()
+          << dec << endl;
+    }
+    return oss.str();
+  }
+
+  // Keep device allocator for TRT.
+  std::unique_ptr<TRTBaseAllocator> allocator_;
+
+  // Declare cache after allocator so that it is destroyed before allocator is.
+  LRUCache<std::vector<TensorShape>, std::unique_ptr<EngineContext>, VectorTensorShapeHasher>
+      cache_;
+};
+
+}  // namespace tensorrt
+}  // namespace tensorflow
+
+#endif  // GOOGLE_TENSORRT
+#endif  // GOOGLE_CUDA
+
+#endif  // TENSORFLOW_CONTRIB_TENSORRT_TRT_LRU_CACHE_H_
diff --git a/tensorflow/contrib/tensorrt/resources/trt_lru_cache_test.cc b/tensorflow/contrib/tensorrt/resources/trt_lru_cache_test.cc
new file mode 100644
index 0000000000..a0959479ce
--- /dev/null
+++ b/tensorflow/contrib/tensorrt/resources/trt_lru_cache_test.cc
@@ -0,0 +1,57 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/contrib/tensorrt/resources/trt_lru_cache.h"
+
+#include "tensorflow/core/platform/test.h"
+
+namespace tensorflow {
+namespace tensorrt {
+
+TEST(LRUCacheTest, Basic) {
+  LRUCache<int, int, std::hash<int>> cache;
+  cache.reserve(2);
+  // Insert 10
+  cache.emplace(10, 100);
+  EXPECT_EQ(cache.size(), 1);
+  EXPECT_EQ(cache.count(10), 1);
+  EXPECT_EQ(cache.at(10), 100);
+  EXPECT_EQ(cache.count(100), 0);
+  // Insert 20
+  cache.emplace(20, 200);
+  EXPECT_EQ(cache.size(), 2);
+  EXPECT_EQ(cache.count(10), 1);
+  EXPECT_EQ(cache.count(20), 1);
+  EXPECT_EQ(cache.at(10), 100);
+  EXPECT_EQ(cache.at(20), 200);
+  EXPECT_EQ(cache.count(100), 0);
+  EXPECT_EQ(cache.count(200), 0);
+  // Insert 30, Evicting 10
+  cache.emplace(30, 300);
+  EXPECT_EQ(cache.count(10), 0);
+  EXPECT_EQ(cache.count(20), 1);
+  EXPECT_EQ(cache.count(30), 1);
+  // Touch 20
+  cache.at(20);
+  // Insert 40, Evicting 30
+  cache.emplace(40, 400);
+  EXPECT_EQ(cache.count(10), 0);
+  EXPECT_EQ(cache.count(20), 1);
+  EXPECT_EQ(cache.count(30), 0);
+  EXPECT_EQ(cache.count(40), 1);
+}
+
+}  // namespace tensorrt
+}  // namespace tensorflow
diff --git a/tensorflow/contrib/tensorrt/test/base_test.py b/tensorflow/contrib/tensorrt/test/base_test.py
index ff317e43e1..0f6293b166 100644
--- a/tensorflow/contrib/tensorrt/test/base_test.py
+++ b/tensorflow/contrib/tensorrt/test/base_test.py
@@ -68,9 +68,9 @@ class SimpleSingleEngineTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[(100, 6, 6, 6)])
+        expected_output_dims=[[(100, 6, 6, 6)]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
@@ -125,9 +125,9 @@ class SimpleMultiEnginesTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[(100, 12, 12, 6)])
+        expected_output_dims=[[(100, 12, 12, 6)]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
@@ -183,9 +183,9 @@ class PartiallyConvertedTestA(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[tuple(input_dims)])
+        expected_output_dims=[[tuple(input_dims)]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
@@ -253,9 +253,9 @@ class ConstInputTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[tuple(input_dims)])
+        expected_output_dims=[[tuple(input_dims)]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
@@ -286,9 +286,9 @@ class ConstDataInputSingleEngineTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[tuple(input_dims)])
+        expected_output_dims=[[tuple(input_dims)]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
@@ -320,9 +320,9 @@ class ConstDataInputMultipleEnginesTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[tuple(input_dims)])
+        expected_output_dims=[[tuple(input_dims)]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
@@ -369,9 +369,9 @@ class ControlDependencyTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[tuple(input_dims)])
+        expected_output_dims=[[tuple(input_dims)]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/batch_matmul_test.py b/tensorflow/contrib/tensorrt/test/batch_matmul_test.py
index 384b84cb02..1b015dd369 100644
--- a/tensorflow/contrib/tensorrt/test/batch_matmul_test.py
+++ b/tensorflow/contrib/tensorrt/test/batch_matmul_test.py
@@ -71,9 +71,9 @@ class BatchMatMulTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name, w1_name, w2_name],
-        input_dims=[input_dims, w1_dims, w2_dims],
+        input_dims=[[input_dims, w1_dims, w2_dims]],
         output_names=[output_name],
-        expected_output_dims=[(12, 5, 8, 7)])
+        expected_output_dims=[[(12, 5, 8, 7)]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py b/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py
index 49f5e4cc9f..8a5f8ef5d5 100644
--- a/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py
+++ b/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py
@@ -111,9 +111,9 @@ class BiasaddMatMulTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[(4, 6680)])
+        expected_output_dims=[[(4, 6680)]])
 
   def GetConversionParams(self, run_params):
     """Return a ConversionParams for test."""
diff --git a/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py b/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py
index 169835956c..0083695a50 100644
--- a/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py
+++ b/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py
@@ -63,9 +63,9 @@ class BinaryTensorWeightBroadcastTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[(5, 23040)])
+        expected_output_dims=[[(5, 23040)]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/concatenation_test.py b/tensorflow/contrib/tensorrt/test/concatenation_test.py
index c3576f81d9..e6afe4e3f6 100644
--- a/tensorflow/contrib/tensorrt/test/concatenation_test.py
+++ b/tensorflow/contrib/tensorrt/test/concatenation_test.py
@@ -73,9 +73,9 @@ class ConcatenationTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[(2, 126)])
+        expected_output_dims=[[(2, 126)]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/const_broadcast_test.py b/tensorflow/contrib/tensorrt/test/const_broadcast_test.py
index c1c883312d..56a297afb5 100644
--- a/tensorflow/contrib/tensorrt/test/const_broadcast_test.py
+++ b/tensorflow/contrib/tensorrt/test/const_broadcast_test.py
@@ -58,9 +58,9 @@ class ConstBroadcastTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[(5, 12, 12, 1)])
+        expected_output_dims=[[(5, 12, 12, 1)]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/dynamic_input_shapes_test.py b/tensorflow/contrib/tensorrt/test/dynamic_input_shapes_test.py
new file mode 100644
index 0000000000..9985561e1e
--- /dev/null
+++ b/tensorflow/contrib/tensorrt/test/dynamic_input_shapes_test.py
@@ -0,0 +1,105 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Script to test TF-TRT INT8 conversion without calibration on Mnist model."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
+# pylint: disable=unused-import
+from tensorflow.contrib.tensorrt.python.ops import trt_engine_op
+# pylint: enable=unused-import
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import constant_op
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import nn
+from tensorflow.python.platform import test
+import numpy as np
+
+
+class DynamicInputShapesTest(trt_test.TfTrtIntegrationTestBase):
+
+  def GetParams(self):
+    input_dims = [[[1, 5, 5, 1]], [[10, 5, 5, 1]], [[3, 5, 5, 1]],
+                  [[1, 5, 5, 1]], [[1, 3, 1, 1]], [[2, 9, 9, 1]],
+                  [[1, 224, 224, 1]], [[1, 128, 224, 1]]]
+    expected_output_dims = input_dims
+
+    g = ops.Graph()
+    with g.as_default():
+      x = array_ops.placeholder(shape=(None, None, None, 1),
+                                dtype=dtypes.float32,
+                                name='input')
+      conv_filter1 = constant_op.constant(np.ones([3, 3, 1, 8]),
+                                          name="weights1",
+                                          dtype=dtypes.float32)
+      bias1 = constant_op.constant(np.random.randn(8), dtype=dtypes.float32)
+      x = nn.conv2d(
+          input=x,
+          filter=conv_filter1,
+          strides=[1, 1, 1, 1],
+          padding="SAME",
+          name="conv")
+      x = nn.bias_add(x, bias1)
+      x = nn.relu(x)
+      conv_filter2 = constant_op.constant(np.ones([3, 3, 8, 1]),
+                                          name="weights2",
+                                          dtype=dtypes.float32)
+      bias2 = constant_op.constant(np.random.randn(1), dtype=dtypes.float32)
+      x = nn.conv2d(
+          input=x,
+          filter=conv_filter2,
+          strides=[1, 1, 1, 1],
+          padding="SAME",
+          name="conv")
+      x = nn.bias_add(x, bias2)
+      x = array_ops.identity(x, name='output')
+
+    return trt_test.TfTrtIntegrationTestParams(
+        gdef=g.as_graph_def(),
+        input_names=['input'],
+        input_dims=input_dims,
+        output_names=['output'],
+        expected_output_dims=expected_output_dims)
+
+  def GetConversionParams(self, run_params):
+    """Return a ConversionParams for test."""
+    conversion_params = super(DynamicInputShapesTest,
+                              self).GetConversionParams(run_params)
+    return conversion_params._replace(
+        maximum_cached_engines=10,
+        # Disable layout optimizer, since it will convert BiasAdd with NHWC
+        # format to NCHW format under four dimentional input.
+        rewriter_config=trt_test.OptimizerDisabledRewriterConfig())
+
+  def ExpectedEnginesToBuild(self, run_params):
+    return ["TRTEngineOp_0"]
+
+  def ShouldRunTest(self, run_params):
+    return (run_params.dynamic_engine and
+            not trt_test.IsQuantizationMode(run_params.precision_mode))
+
+  def ExpectedAbsoluteTolerance(self, run_params):
+    """The absolute tolerance to compare floating point results."""
+    return 1.e-03 if run_params.precision_mode == "FP32" else 1.e-01
+
+  def ExpectedRelativeTolerance(self, run_params):
+    """The relative tolerance to compare floating point results."""
+    return 1.e-03 if run_params.precision_mode == "FP32" else 1.e-01
+
+if __name__ == '__main__':
+  test.main()
diff --git a/tensorflow/contrib/tensorrt/test/lru_cache_test.py b/tensorflow/contrib/tensorrt/test/lru_cache_test.py
new file mode 100644
index 0000000000..74b51932ac
--- /dev/null
+++ b/tensorflow/contrib/tensorrt/test/lru_cache_test.py
@@ -0,0 +1,62 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import nn
+from tensorflow.python.platform import test
+
+
+class LRUCacheTest(trt_test.TfTrtIntegrationTestBase):
+
+  def GetParams(self):
+    """Test for running network multiple times with different batch size to
+    check that lru_cache works correctly"""
+    dtype = dtypes.float32
+    input_name = "input"
+    input_dims = [[[1, 10, 10, 2]], [[2, 10, 10, 2]], [[4, 10, 10, 2]],
+                  [[2, 10, 10, 2]]]
+    expected_output_dims = [[(1, 10, 10, 1)], [(2, 10, 10, 1)],
+                            [(4, 10, 10, 1)], [(2, 10, 10, 1)]]
+    output_name = "output"
+    g = ops.Graph()
+    with g.as_default():
+      x = array_ops.placeholder(dtype=dtype,
+                                shape=[None, 10, 10, 2],
+                                name=input_name)
+      conv_filter = constant_op.constant(np.random.randn(3, 3, 2, 1),
+                                         dtype=dtypes.float32)
+      x = nn.conv2d(input=x,
+                    filter=conv_filter,
+                    strides=[1, 1, 1, 1],
+                    padding="SAME",
+                    name="conv")
+      bias = constant_op.constant(np.random.randn(1, 10, 10, 1),
+                                  dtype=dtypes.float32)
+      x = math_ops.add(x, bias)
+      x = nn.relu(x)
+      x = array_ops.identity(x, name="output")
+    return trt_test.TfTrtIntegrationTestParams(
+        gdef=g.as_graph_def(),
+        input_names=[input_name],
+        input_dims=input_dims,
+        output_names=[output_name],
+        expected_output_dims=expected_output_dims)
+
+  def ExpectedEnginesToBuild(self, run_params):
+    """Return the expected engines to build."""
+    return ["TRTEngineOp_0"]
+
+  def ShouldRunTest(self, run_params):
+    return (run_params.dynamic_engine and
+            not trt_test.IsQuantizationMode(run_params.precision_mode))
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/tensorrt/test/manual_test.py b/tensorflow/contrib/tensorrt/test/manual_test.py
index 1187c759b4..aad7b9f307 100644
--- a/tensorflow/contrib/tensorrt/test/manual_test.py
+++ b/tensorflow/contrib/tensorrt/test/manual_test.py
@@ -67,9 +67,9 @@ class ManualTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=gdef,
         input_names=params_map['input_names'],
-        input_dims=params_map['input_dims'],
+        input_dims=[params_map['input_dims']],
         output_names=params_map['output_names'],
-        expected_output_dims=params_map['expected_output_dims'])
+        expected_output_dims=[params_map['expected_output_dims']])
 
   def GetConversionParams(self, run_params):
     """Return a ConversionParams for test."""
diff --git a/tensorflow/contrib/tensorrt/test/memory_alignment_test.py b/tensorflow/contrib/tensorrt/test/memory_alignment_test.py
index 104bac43a0..6142e6d8e3 100644
--- a/tensorflow/contrib/tensorrt/test/memory_alignment_test.py
+++ b/tensorflow/contrib/tensorrt/test/memory_alignment_test.py
@@ -62,9 +62,9 @@ class MemoryAlignmentTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[(2, 15, 15, 10)])
+        expected_output_dims=[[(2, 15, 15, 10)]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py b/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py
index 293f93d8a7..fcf89d3a74 100644
--- a/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py
+++ b/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py
@@ -75,9 +75,9 @@ class MultiConnectionNeighborEngineTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[(2, 4, 5, 4)])
+        expected_output_dims=[[(2, 4, 5, 4)]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/neighboring_engine_test.py b/tensorflow/contrib/tensorrt/test/neighboring_engine_test.py
index 3e1e4b088b..840344aeba 100644
--- a/tensorflow/contrib/tensorrt/test/neighboring_engine_test.py
+++ b/tensorflow/contrib/tensorrt/test/neighboring_engine_test.py
@@ -59,9 +59,9 @@ class NeighboringEngineTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[(2, 4, 5, 4)])
+        expected_output_dims=[[(2, 4, 5, 4)]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/quantization_test.py b/tensorflow/contrib/tensorrt/test/quantization_test.py
index e425a36746..fb09980552 100644
--- a/tensorflow/contrib/tensorrt/test/quantization_test.py
+++ b/tensorflow/contrib/tensorrt/test/quantization_test.py
@@ -60,9 +60,9 @@ def _GetParams(add_quantization_nodes, dtype=dtypes.float32):
   return trt_test.TfTrtIntegrationTestParams(
       gdef=g.as_graph_def(),
       input_names=[input_name],
-      input_dims=[input_dims],
+      input_dims=[[input_dims]],
       output_names=[output_name],
-      expected_output_dims=[(8, 1)])
+      expected_output_dims=[[(8, 1)]])
 
 
 class QuantizationMissingAllRangesTest(trt_test.TfTrtIntegrationTestBase):
diff --git a/tensorflow/contrib/tensorrt/test/rank_two_test.py b/tensorflow/contrib/tensorrt/test/rank_two_test.py
index dd3b72d212..85ce89e151 100644
--- a/tensorflow/contrib/tensorrt/test/rank_two_test.py
+++ b/tensorflow/contrib/tensorrt/test/rank_two_test.py
@@ -63,9 +63,9 @@ class RankTwoTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=input_names,
-        input_dims=input_dims,
+        input_dims=[input_dims],
         output_names=[output_name],
-        expected_output_dims=[tuple(input_dims[1])])
+        expected_output_dims=[[tuple(input_dims[1])]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/reshape_transpose_test.py b/tensorflow/contrib/tensorrt/test/reshape_transpose_test.py
index 207944468a..530c4992db 100644
--- a/tensorflow/contrib/tensorrt/test/reshape_transpose_test.py
+++ b/tensorflow/contrib/tensorrt/test/reshape_transpose_test.py
@@ -72,9 +72,9 @@ class ReshapeTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[tuple(input_dims)])
+        expected_output_dims=[[tuple(input_dims)]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
@@ -129,9 +129,9 @@ class TransposeTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[(24, 100, 2, 24)])
+        expected_output_dims=[[(24, 100, 2, 24)]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
index 671abba6a6..a105981298 100644
--- a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
+++ b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
@@ -161,7 +161,7 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
     """Return a ConversionParams for test."""
     return ConversionParams(
         max_batch_size=max([
-            dims[0] for dims in self._GetParamsCached().input_dims if len(dims)
+            dims[0] for inp in self._GetParamsCached().input_dims if len(inp) for dims in inp
         ]),
         max_workspace_size_bytes=1 << 25,
         precision_mode=run_params.precision_mode,
@@ -283,13 +283,16 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
   def _RunGraph(self,
                 run_params,
                 gdef,
-                input_data,
+                inputs_data,
                 config,
                 graph_state,
                 num_runs=2):
     """Run given graphdef multiple times."""
     params = self._GetParamsCached()
-    assert len(params.input_names) == len(input_data)
+    for current_input_data in inputs_data:
+      assert len(params.input_names) == len(current_input_data)
+
+    vals = []
     g = ops.Graph()
     with g.as_default():
       io_ops = importer.import_graph_def(
@@ -297,32 +300,38 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
           return_elements=params.input_names + params.output_names,
           name="")
       inputs = [op.outputs[0] for op in io_ops[:len(params.input_names)]]
-      assert len(inputs) == len(input_data)
+      for current_input_data in inputs_data:
+        assert len(inputs) == len(current_input_data)
       outputs = [op.outputs[0] for op in io_ops[len(params.input_names):]]
-    with self.test_session(
-        graph=g, config=config, use_gpu=True, force_gpu=True) as sess:
-      val = None
-      # Defaults to 2 runs to verify result across multiple runs is same.
-      for _ in range(num_runs):
-        self._PrepareRun(graph_state)
-        new_val = sess.run(
-            outputs, {inputs[i]: input_data[i] for i in range(len(inputs))})
-        output_len = len(params.expected_output_dims)
-        self.assertEqual(output_len, len(new_val))
-        for i in range(output_len):
-          self.assertEqual(params.expected_output_dims[i], new_val[i].shape)
-        if val is not None:
-          self.assertAllClose(val, new_val, atol=1.e-06, rtol=1.e-06)
-        val = new_val
-        self.VerifyRun(run_params, graph_state)
-    return val
+      with self.test_session(
+          graph=g, config=config, use_gpu=True, force_gpu=True) as sess:
+        # Run for each input(s) shape
+        for ind in range(len(inputs_data)):
+          val = None
+          # Defaults to 2 runs to verify result across multiple runs is same.
+          for _ in range(num_runs):
+            self._PrepareRun(graph_state)
+            new_val = sess.run(
+                outputs,
+                {inputs[i]: inputs_data[ind][i] for i in range(len(inputs))})
+            output_len = len(params.expected_output_dims[ind])
+            self.assertEqual(output_len, len(new_val))
+            for i in range(output_len):
+              self.assertEqual(list(params.expected_output_dims[ind][i]),
+                               list(new_val[i].shape))
+            if val is not None:
+              self.assertAllClose(val, new_val, atol=1.e-06, rtol=1.e-06)
+            val = new_val
+            self.VerifyRun(run_params, graph_state)
+          vals.append(val)
+    return vals
 
   # Use real data that is representative of the inference dataset
   # for calibration. For this test script it is random data.
-  def _RunCalibration(self, run_params, gdef, input_data, config):
+  def _RunCalibration(self, run_params, gdef, inputs_data, config):
     """Run calibration on given graph."""
     return self._RunGraph(
-        run_params, gdef, input_data, config, GraphState.CALIBRATE, num_runs=5)
+        run_params, gdef, inputs_data, config, GraphState.CALIBRATE, num_runs=5)
 
   def _GetTrtGraphDef(self, run_params, graph_state, gdef):
     """Return trt converted graphdef."""
@@ -474,26 +483,30 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
             dtypes.as_dtype(node.attr["dtype"].type).as_numpy_dtype())
     assert len(params.input_names) == len(input_dtypes)
 
-    input_data = []
-    for i in range(len(params.input_names)):
-      dtype = input_dtypes[params.input_names[i]]
-      # Multiply the input by some constant to avoid all zeros input for integer
-      # types.
-      scale = 10.0 if np.issubdtype(dtype, np.integer) else 1.0
-      dims = params.input_dims[i]
-      # TODO(laigd): add debug options. E.g. we can set the input data to be
-      # continuous natural numbers:
-      # seq = np.arange(np.prod(dims))
-      # seq.resize(dims)
-      # input_data.append(scale * seq.astype(dtype))
-      input_data.append((scale * np.random.random_sample(dims)).astype(dtype))
+    inputs_data = []
+    for inp in params.input_dims:
+      current_input_data = []
+      for i in range(len(params.input_names)):
+        dtype = input_dtypes[params.input_names[i]]
+        # Multiply the input by some constant to avoid all zeros input for integer
+        # types.
+        scale = 10.0 if np.issubdtype(dtype, np.integer) else 1.0
+        dims = inp[i]
+        # TODO(laigd): add debug options. E.g. we can set the input data to be
+        # continuous natural numbers:
+        # seq = np.arange(np.prod(dims))
+        # seq.resize(dims)
+        # input_data.append(scale * seq.astype(dtype))
+        current_input_data.append((scale * np.random.random_sample(dims)).astype(dtype))
+      inputs_data.append(current_input_data)
+
     self._VerifyGraphDef(run_params, input_gdef, GraphState.ORIGINAL)
 
     # Get reference result without running trt.
     config_no_trt = self._GetConfigProto(run_params, GraphState.ORIGINAL)
     logging.info("Running original graph w/o trt, config:\n%s",
                  str(config_no_trt))
-    ref_result = self._RunGraph(run_params, input_gdef, input_data,
+    ref_result = self._RunGraph(run_params, input_gdef, inputs_data,
                                 config_no_trt, GraphState.ORIGINAL)
 
     # Run calibration if necessary.
@@ -503,13 +516,13 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
       calib_config = self._GetConfigProto(run_params, GraphState.CALIBRATE)
       logging.info("Running calibration graph, config:\n%s", str(calib_config))
       if run_params.use_optimizer:
-        result = self._RunCalibration(run_params, input_gdef, input_data,
+        result = self._RunCalibration(run_params, input_gdef, inputs_data,
                                       calib_config)
       else:
         calib_gdef = self._GetTrtGraphDef(run_params, GraphState.CALIBRATE,
                                           input_gdef)
         self._VerifyGraphDef(run_params, calib_gdef, GraphState.CALIBRATE)
-        result = self._RunCalibration(run_params, calib_gdef, input_data,
+        result = self._RunCalibration(run_params, calib_gdef, inputs_data,
                                       calib_config)
       infer_gdef = trt_convert.calib_graph_to_infer_graph(
           calib_gdef, run_params.dynamic_engine)
@@ -532,7 +545,7 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
                                         infer_gdef)
       self._VerifyGraphDef(run_params, infer_gdef, GraphState.INFERENCE)
 
-    result = self._RunGraph(run_params, infer_gdef, input_data, infer_config,
+    result = self._RunGraph(run_params, infer_gdef, inputs_data, infer_config,
                             GraphState.INFERENCE)
     self.assertAllClose(
         ref_result,
diff --git a/tensorflow/contrib/tensorrt/test/unary_test.py b/tensorflow/contrib/tensorrt/test/unary_test.py
index b6e5e32db1..f630d695cf 100644
--- a/tensorflow/contrib/tensorrt/test/unary_test.py
+++ b/tensorflow/contrib/tensorrt/test/unary_test.py
@@ -100,9 +100,9 @@ class UnaryTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name, input2_name],
-        input_dims=[input_dims, input2_dims],
+        input_dims=[[input_dims, input2_dims]],
         output_names=[output_name],
-        expected_output_dims=[(12, 5, 8, 12)])
+        expected_output_dims=[[(12, 5, 8, 12)]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py b/tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py
index b29626d2c2..1a4746e8d0 100644
--- a/tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py
+++ b/tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py
@@ -70,9 +70,9 @@ class VGGBlockNCHWTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[(5, 6, 2, 2)])
+        expected_output_dims=[[(5, 6, 2, 2)]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/vgg_block_test.py b/tensorflow/contrib/tensorrt/test/vgg_block_test.py
index 9b0b189626..051ced055a 100644
--- a/tensorflow/contrib/tensorrt/test/vgg_block_test.py
+++ b/tensorflow/contrib/tensorrt/test/vgg_block_test.py
@@ -61,9 +61,9 @@ class VGGBlockTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[(5, 2, 2, 6)])
+        expected_output_dims=[[(5, 2, 2, 6)]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
-- 
GitLab


From 48891a7ecc94058e9dc18498dd3d0b2a46b75abe Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Mon, 14 Jan 2019 14:07:01 -0800
Subject: [PATCH 0708/1095] Make DebugString() const

---
 tensorflow/contrib/tensorrt/resources/trt_lru_cache.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h b/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h
index 4d63a03b40..02407d023a 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h
+++ b/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h
@@ -140,7 +140,7 @@ class TRTEngineCacheResource : public tensorflow::ResourceBase {
     }
   }
 
-  string DebugString() override {
+  string DebugString() const override {
     std::stringstream oss;
     using std::dec;
     using std::endl;
-- 
GitLab


From 9887a0312b59d288773d0a806c335fd46ee7d6c9 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Mon, 14 Jan 2019 14:23:55 -0800
Subject: [PATCH 0709/1095] Fix clang-format

---
 tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc  | 11 +++++------
 tensorflow/contrib/tensorrt/resources/trt_lru_cache.h |  3 ++-
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
index e1e64fec68..06a62fed37 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
+++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
@@ -297,7 +297,6 @@ void TRTEngineOp::ComputeAsync(OpKernelContext* ctx,
 
 bool TRTEngineOp::ExecuteTrtEngine(OpKernelContext* ctx,
                                    EngineContext* engine_context) {
-  
   VLOG(1) << "Executing TRT engine: " << name();
   auto& cuda_engine = engine_context->cuda_engine;
   const bool kRetry = true;
@@ -468,11 +467,11 @@ EngineContext* TRTEngineOp::GetEngine(
       engine_input_shapes[i].set_dim(0, max_batch_size);
     }
     // Assume engine_input_shapes matches the input shapes of the engine.
-    cache.emplace(
-        engine_input_shapes, absl::make_unique<EngineContext>(
-            std::move(static_engine),
-            TrtUniquePtrType<nvinfer1::IExecutionContext>(
-                raw_static_engine->createExecutionContext())));
+    cache.emplace(engine_input_shapes,
+                  absl::make_unique<EngineContext>(
+                      std::move(static_engine),
+                      TrtUniquePtrType<nvinfer1::IExecutionContext>(
+                          raw_static_engine->createExecutionContext())));
     // Runtime is safe to delete after engine creation
     serialized_segment_.clear();
     if (max_batch_size < batch_size) {
diff --git a/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h b/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h
index 02407d023a..57267e5b71 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h
+++ b/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h
@@ -162,7 +162,8 @@ class TRTEngineCacheResource : public tensorflow::ResourceBase {
   std::unique_ptr<TRTBaseAllocator> allocator_;
 
   // Declare cache after allocator so that it is destroyed before allocator is.
-  LRUCache<std::vector<TensorShape>, std::unique_ptr<EngineContext>, VectorTensorShapeHasher>
+  LRUCache<std::vector<TensorShape>, std::unique_ptr<EngineContext>,
+           VectorTensorShapeHasher>
       cache_;
 };
 
-- 
GitLab


From 2cfc9e6ca77544e90090c45c498d024ffd79fc84 Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Mon, 14 Jan 2019 15:40:37 -0800
Subject: [PATCH 0710/1095] Disable InterpreterFlexTest

This test currently only works with --config=monolithic. Disable for
now until the select .so supports execution against the shared TF lib.

PiperOrigin-RevId: 229269278
---
 tensorflow/lite/java/BUILD | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/lite/java/BUILD b/tensorflow/lite/java/BUILD
index a9d4a422c5..bd9d8f4806 100644
--- a/tensorflow/lite/java/BUILD
+++ b/tensorflow/lite/java/BUILD
@@ -163,6 +163,9 @@ java_test(
         "//tensorflow/lite:testdata/multi_add_flex.bin",
     ],
     javacopts = JAVACOPTS,
+    tags = [
+        "no_oss",  # Currently requires --config=monolithic, b/118895218.
+    ],
     test_class = "org.tensorflow.lite.InterpreterFlexTest",
     visibility = ["//visibility:private"],
     deps = [
-- 
GitLab


From a3a6bd35ffde82f0d6ff00b004497a5b2f97e6b8 Mon Sep 17 00:00:00 2001
From: Katherine Wu <kathywu@google.com>
Date: Mon, 14 Jan 2019 15:59:05 -0800
Subject: [PATCH 0711/1095] Fix bug where TensorSpecs are replaced with
 UnknownArgument, which caused issues with f.get_concrete_function.

PiperOrigin-RevId: 229273199
---
 tensorflow/python/eager/function.py        |  2 +-
 tensorflow/python/saved_model/load_test.py | 17 +++++++++++++++--
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index 6429022df3..b5bda35916 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -844,7 +844,7 @@ def convert_structure_to_signature(structure):
     """A representation for this argument, for converting into signatures."""
     if isinstance(arg, ops.Tensor):
       return tensor_spec.TensorSpec(arg.shape, arg.dtype, name)
-    if isinstance(arg, (int, float, bool)):
+    if isinstance(arg, (int, float, bool, tensor_spec.TensorSpec)):
       return arg
     return UnknownArgument()
 
diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index e94cda533e..ca9e16d14a 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -37,9 +37,9 @@ from tensorflow.python.training.checkpointable import tracking
 
 class LoadTest(test.TestCase):
 
-  def cycle(self, obj):
+  def cycle(self, obj, signatures=None):
     path = tempfile.mkdtemp(prefix=self.get_temp_dir())
-    save.save(obj, path, signatures={})
+    save.save(obj, path, signatures=signatures or {})
     return load.load(path)
 
   def test_structure_import(self):
@@ -152,6 +152,19 @@ class LoadTest(test.TestCase):
     imported = self.cycle(root)
     self.assertEqual(4., imported.f(constant_op.constant(2.0)).numpy())
 
+  def test_explicit_save_signature(self):
+    @def_function.function
+    def func(x):
+      return 2 * x
+
+    root = tracking.Checkpointable()
+    root.f = func
+
+    imported = self.cycle(
+        root, {"f": root.f.get_concrete_function(
+            tensor_spec.TensorSpec(None, dtypes.float32))})
+    self.assertEqual(4., imported.f(constant_op.constant(2.0)).numpy())
+
   def test_nested_functions(self):
     f = def_function.function(
         lambda x: x*2.0,
-- 
GitLab


From ca4c61a35b36da49e5dd2aeebc33b99587f243ba Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Mon, 14 Jan 2019 16:09:17 -0800
Subject: [PATCH 0712/1095] [Grappler] Relax no-inline constraint for functions
 with Switch/Merge control flow.

Do not inline functions only if they can produce dead outputs.

PiperOrigin-RevId: 229275596
---
 tensorflow/core/grappler/optimizers/BUILD     |   2 +
 .../grappler/optimizers/function_optimizer.cc | 106 ++++++++++--
 .../optimizers/function_optimizer_test.cc     | 157 ++++++++++++++++++
 3 files changed, 251 insertions(+), 14 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index a67d6ac5ea..ee216f80f9 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -145,11 +145,13 @@ cc_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core/grappler:graph_topology_view",
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/utils:functions",
+        "//tensorflow/core/grappler/utils:traversal",
         "@com_google_absl//absl/algorithm:container",
         "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/container:flat_hash_set",
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.cc b/tensorflow/core/grappler/optimizers/function_optimizer.cc
index e9c30fee25..8095d6f31d 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer.cc
@@ -39,11 +39,13 @@ limitations under the License.
 #include "tensorflow/core/framework/versions.pb.h"
 #include "tensorflow/core/graph/graph_constructor.h"
 #include "tensorflow/core/graph/tensor_id.h"
+#include "tensorflow/core/grappler/graph_topology_view.h"
 #include "tensorflow/core/grappler/grappler_item.h"
 #include "tensorflow/core/grappler/mutable_graph_view.h"
 #include "tensorflow/core/grappler/op_types.h"
 #include "tensorflow/core/grappler/utils.h"
 #include "tensorflow/core/grappler/utils/functions.h"
+#include "tensorflow/core/grappler/utils/traversal.h"
 #include "tensorflow/core/lib/gtl/map_util.h"
 
 namespace tensorflow {
@@ -264,6 +266,8 @@ class FunctionOptimizerContext {
     InitializeFetchNodes(item);
   }
 
+  int graph_version() const { return graph_version_; }
+
   const RewriterConfig::Toggle opt_level() const { return opt_level_; }
 
   const GrapplerItem::AllowedOptimizations& allowed_optimizations() const {
@@ -1259,6 +1263,74 @@ Status InlineSymbolicGradient(const NodeDef& node,
 // because they are pure functions of input tensors, and can be freely
 // reordered.
 
+struct MaybeDeadOutput {
+  const NodeDef* dead_tensor_src;
+  const NodeDef* output_node_dst;
+};
+
+// Finds all function outputs that might return a dead tensor. This can happen
+// if there is no `Merge` node on the path from the `Switch` node, to the
+// function output.
+Status MaybeDeadOutputs(const FunctionOptimizerContext& ctx,
+                        const GrapplerFunctionItem& item,
+                        std::vector<MaybeDeadOutput>* maybe_dead) {
+  DCHECK(maybe_dead->empty()) << "Input argument must be an empty vector";
+
+  std::vector<const NodeDef*> dead_tensor_srcs;
+  for (const NodeDef& node : item.graph.node()) {
+    if (IsSwitch(node)) {
+      dead_tensor_srcs.push_back(&node);
+      continue;
+    }
+
+    // Regular (aka 'direct') function call can also produce dead tensors if
+    // the function body has mergeless switches.
+    const FunctionDef* func = ctx.function_library().Find(node.op());
+    if (func != nullptr) {
+      GrapplerFunctionItem func_item;
+      TF_RETURN_IF_ERROR(MakeGrapplerFunctionItem(
+          *func, FunctionInstantiationAttributes(*func, node),
+          ctx.function_library(), ctx.graph_version(), &func_item));
+
+      std::vector<MaybeDeadOutput> func_dead_outputs;
+      TF_RETURN_IF_ERROR(MaybeDeadOutputs(ctx, func_item, &func_dead_outputs));
+
+      if (!func_dead_outputs.empty()) dead_tensor_srcs.push_back(&node);
+    }
+  }
+
+  // If we do not have dead tensor sources in the function body, it's
+  // guaranteed that all output tensors can't become dead.
+  if (dead_tensor_srcs.empty()) return Status::OK();
+
+  // Names of the function body nodes that return function output values.
+  absl::flat_hash_set<absl::string_view> output_nodes;
+  for (const auto& output_expansion : item.outputs()) {
+    for (const auto& output_node : output_expansion.output_nodes) {
+      output_nodes.insert(output_node);
+    }
+  }
+
+  GraphTopologyView topology_view;
+  TF_RETURN_IF_ERROR(topology_view.InitializeFromGraph(item.graph));
+
+  for (const NodeDef* dead_tensor_src : dead_tensor_srcs) {
+    DfsTraversal(topology_view, {dead_tensor_src},
+                 TraversalDirection::kFollowOutputs,
+                 // Stop traversal when reached first `Merge` node.
+                 DfsPredicates::Advance(
+                     [](const NodeDef* node) { return !IsMerge(*node); }),
+                 // If we reached output node, add MaybeDeadOutput edge.
+                 DfsCallbacks::PreOrder([&](const NodeDef* node) {
+                   if (output_nodes.find(node->name()) != output_nodes.end()) {
+                     maybe_dead->push_back({dead_tensor_src, node});
+                   }
+                 }));
+  }
+
+  return Status::OK();
+}
+
 // Returns `Status::OK()` iff `node` is an indirect function call of `func`, and
 // we know how to inline it into the main graph, otherwise returns and error
 // indicating why the function call is not inlinable.
@@ -1293,20 +1365,6 @@ Status IsInlinableIndirectFunctionCall(const FunctionOptimizerContext& ctx,
         SummarizeNodeDef(func_node));
   }
 
-  // We can't inline functions with `Switch` nodes in the function body, because
-  // they might have dead tensors as a function output argument (we need all
-  // intermediate tensors to compute the function gradient). `PartitionedCallOp`
-  // invokes functions with `allow_dead_tensors = true` to reset dead flag,
-  // and return default initialized tensors instead of a dead tensors.
-  // TODO(ezhulenev): Do the liveness analysis and add
-  // `IdentitytWithResurrection` nodes after all potentially dead output
-  // tensors?
-  if (absl::c_any_of(func.node_def(), IsSwitch)) {
-    return errors::FailedPrecondition(
-        "Can't inline function with `Switch` nodes in the function body: ",
-        SummarizeNodeDef(func_node));
-  }
-
   // TODO(b/120991525, b/120986912): We need to lower `If` and `While` nodes to
   // `Switch` nodes after function inlining (one more PRE_PLACEMENT pass?), but
   // because of the reason described above we are not sure that it's safe, for
@@ -1346,6 +1404,26 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
                                    ". Error: ", item_status.error_message());
   }
 
+  // `PartitionedCallOp` invokes functions with `allow_dead_tensors = true` to
+  // reset dead flag, and return default initialized tensors instead of a dead
+  // tensors. There is no way to express this in a regular Tensorflow graph, so
+  // we choose not to inline if a function can have dead tensors as an output
+  // position. In practice `mergeless switches` should not exists in a function
+  // body, because tf-eager will only use v2 control flow ops.
+  std::vector<MaybeDeadOutput> maybe_dead_outputs;
+  TF_RETURN_IF_ERROR(MaybeDeadOutputs(*ctx, item, &maybe_dead_outputs));
+  if (!maybe_dead_outputs.empty()) {
+    struct MaybeDeadOutputFormatter {
+      void operator()(string* out, const MaybeDeadOutput& md) const {
+        absl::StrAppend(out, SummarizeNodeDef(*md.dead_tensor_src));
+      }
+    };
+    return errors::FailedPrecondition(
+        "Can't inline function with dead outputs. Dead tensor sources (size = ",
+        maybe_dead_outputs.size(), "): ",
+        absl::StrJoin(maybe_dead_outputs, "\n", MaybeDeadOutputFormatter()));
+  }
+
   GraphView::InputPort control_input_port =
       ctx->graph_view().GetInputPort(func_node.name(), Graph::kControlSlot);
   GraphView::OutputPort control_output_port =
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
index cebd002bed..827b0658c5 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
@@ -1025,6 +1025,163 @@ TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithoutSideEffects) {
   test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
 }
 
+TEST_F(FunctionOptimizerTest, InlineIndirectFunctionDoNotInlineDeadOutputs) {
+  using test::function::NDef;
+  using FDH = FunctionDefHelper;
+
+  FunctionOptimizer optimizer(RewriterConfig::AGGRESSIVE);
+
+  // Function output can be dead.
+  FunctionDef dead_outputs = FunctionDefHelper::Create(
+      "DeadOutputs", {"x:T", "cond:bool"}, {"z:T"}, {"T: {float, double}"},
+      {
+          {{"switch"}, "Switch", {"x", "cond"}, {{"T", "$T"}}},
+          {{"if_false"}, "Identity", {"switch:output_false:0"}, {{"T", "$T"}}},
+          {{"if_true"}, "Identity", {"switch:output_true:0"}, {{"T", "$T"}}},
+      },
+      /* Mapping between function returns and function node outputs. */
+      {{"z", "if_false:output:0"}});
+
+  // Simple proxy functions that calls DeadOutputs from the function body.
+  FunctionDef proxy_func = FunctionDefHelper::Create(
+      "Proxy", {"x:T", "cond:bool"}, {"z:T"}, {"T: {float, double}"},
+      {{{"dead"}, "DeadOutputs", {"x", "cond"}, {{"T", "$T"}}}},
+      /* Mapping between function returns and function node outputs. */
+      {{"z", "dead:z:0"}});
+
+  // Build a graph to compute:
+  //   a: float
+  //   b: bool
+  //   fn0 = DeadOutputs(x, b)
+  //   fn1 = Proxy(x, b)
+  //   out0 = Identity(fn0)
+  //   out1 = Identity(fn1)
+  //   return [out0, out1]
+  //
+  GrapplerItem item;
+  item.fetch = {"out0", "out1"};
+  item.graph = test::function::GDef(
+      {NDef("a", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
+       NDef("b", "Placeholder", {}, {{"dtype", DT_BOOL}}, kDevice),
+
+       NDef("fn0", "PartitionedCall", {"a", "b"},
+            {{"Tin", DataTypeSlice{DT_FLOAT, DT_BOOL}},
+             {"Tout", DataTypeSlice{DT_FLOAT}},
+             {"f", FDH::FunctionRef("DeadOutputs", {{"T", DT_FLOAT}})}},
+            kDevice),
+
+       NDef("fn1", "PartitionedCall", {"a", "b"},
+            {{"Tin", DataTypeSlice{DT_FLOAT, DT_BOOL}},
+             {"Tout", DataTypeSlice{DT_FLOAT}},
+             {"f", FDH::FunctionRef("Proxy", {{"T", DT_FLOAT}})}},
+            kDevice),
+
+       NDef("out0", "Identity", {"fn0"}, {{"T", DT_FLOAT}}, kDevice),
+       NDef("out1", "Identity", {"fn1"}, {{"T", DT_FLOAT}}, kDevice)},
+      // Function library.
+      {dead_outputs, proxy_func});
+
+  GraphDef optimized_graph;
+  TF_EXPECT_OK(optimizer.Optimize(nullptr, item, &optimized_graph));
+
+  GraphDef expected = item.graph;
+  CompareGraphs(expected, optimized_graph);
+
+  const Tensor one = test::AsScalar<float>(1.0);
+  item.feed.emplace_back("a", one);
+  item.feed.emplace_back("b", test::AsScalar<bool>(false));
+
+  auto tensors = EvaluateFetchNodes(item);
+  ASSERT_EQ(tensors.size(), 2);
+  test::ExpectTensorEqual<float>(tensors[0], one);
+  test::ExpectTensorEqual<float>(tensors[1], one);
+}
+
+TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithMergedDeadTensors) {
+  using test::function::NDef;
+  using FDH = FunctionDefHelper;
+
+  FunctionOptimizer optimizer(RewriterConfig::AGGRESSIVE);
+
+  // Function output can't be dead because it goes through the Merge node.
+  FunctionDef no_dead_outputs = FunctionDefHelper::Create(
+      "NoDeadOutputs", {"x:T", "cond:bool"}, {"z:T"}, {"T: {float, double}"},
+      {
+          {{"switch"}, "Switch", {"x", "cond"}, {{"T", "$T"}}},
+          {{"if_false"}, "Identity", {"switch:output_false:0"}, {{"T", "$T"}}},
+          {{"if_true"}, "Identity", {"switch:output_true:0"}, {{"T", "$T"}}},
+          {{"merge"},
+           "Merge",
+           {"if_false:output:0", "if_true:output:0"},
+           {{"T", "$T"}, {"N", 2}}},
+      },
+      /* Mapping between function returns and function node outputs. */
+      {{"z", "merge:output:0"}});
+
+  // Build a graph to compute:
+  //   a: float
+  //   b: bool
+  //   d = DeadOutputs(x, b)
+  //   out = Identity(d)
+  //   return out
+  //
+  GrapplerItem item;
+  item.fetch = {"out"};
+  item.graph = test::function::GDef(
+      {NDef("a", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
+       NDef("b", "Placeholder", {}, {{"dtype", DT_BOOL}}, kDevice),
+
+       NDef("fn", "PartitionedCall", {"a", "b"},
+            {{"Tin", DataTypeSlice{DT_FLOAT, DT_BOOL}},
+             {"Tout", DataTypeSlice{DT_FLOAT}},
+             {"f", FDH::FunctionRef("NoDeadOutputs", {{"T", DT_FLOAT}})}},
+            kDevice),
+
+       NDef("out", "Identity", {"fn"}, {{"T", DT_FLOAT}}, kDevice)},
+      // Function library.
+      {no_dead_outputs});
+
+  GraphDef optimized_graph;
+  TF_EXPECT_OK(optimizer.Optimize(nullptr, item, &optimized_graph));
+
+  GraphDef expected = test::function::GDef(
+      {NDef("a", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
+       NDef("b", "Placeholder", {}, {{"dtype", DT_BOOL}}, kDevice),
+
+       // Function body of a first function call inlined into the graph.
+       NDef("fn/x", "Identity", {"a:0"}, {{"T", DT_FLOAT}}, kDevice),
+       NDef("fn/cond", "Identity", {"b:0"}, {{"T", DT_BOOL}}, kDevice),
+       NDef("fn/switch", "Switch", {"fn/x:0", "fn/cond:0"}, {{"T", DT_FLOAT}},
+            kDevice),
+       NDef("fn/if_false", "Identity", {"fn/switch:0"}, {{"T", DT_FLOAT}},
+            kDevice),
+       NDef("fn/if_true", "Identity", {"fn/switch:1"}, {{"T", DT_FLOAT}},
+            kDevice),
+       NDef("fn/merge", "Merge", {"fn/if_false:0", "fn/if_true:0"},
+            {{"T", DT_FLOAT}, {"N", 2}}, kDevice),
+
+       // Return directly from inlined node.
+       NDef("out", "Identity", {"fn/merge:0"}, {{"T", DT_FLOAT}}, kDevice)},
+
+      // Function library.
+      {no_dead_outputs});
+
+  CompareGraphs(expected, optimized_graph);
+
+  const Tensor one = test::AsScalar<float>(1.0);
+  item.feed.emplace_back("a", one);
+  item.feed.emplace_back("b", test::AsScalar<bool>(false));
+
+  auto tensors_expected = EvaluateFetchNodes(item);
+  ASSERT_EQ(tensors_expected.size(), 1);
+
+  GrapplerItem optimized = item.WithGraph(std::move(optimized_graph));
+  auto tensors = EvaluateFetchNodes(optimized);
+  ASSERT_EQ(tensors.size(), 1);
+
+  test::ExpectTensorEqual<float>(tensors[0], tensors_expected[0]);
+}
+
 TEST_F(FunctionOptimizerTest, SpecializeFunctionXTimesTwo) {
   using test::function::NDef;
 
-- 
GitLab


From 44c657a5323eafc35e01639506b41732ee363698 Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Mon, 14 Jan 2019 16:12:24 -0800
Subject: [PATCH 0713/1095] Automated rollback of commit
 c45a87f7c7080eecf42c430c66adef0e9249f5ac

PiperOrigin-RevId: 229276115
---
 tensorflow/lite/testing/generated_examples_zip_test.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/lite/testing/generated_examples_zip_test.cc b/tensorflow/lite/testing/generated_examples_zip_test.cc
index dfd8eb059a..bade6590a4 100644
--- a/tensorflow/lite/testing/generated_examples_zip_test.cc
+++ b/tensorflow/lite/testing/generated_examples_zip_test.cc
@@ -287,10 +287,10 @@ struct ZipPathParamName {
   }
 };
 
-INSTANTIATE_TEST_SUITE_P(tests, OpsTest,
-                         ::testing::ValuesIn(UnarchiveAndFindTestNames(
-                             *FLAGS_zip_file_path, *FLAGS_tar_file_path)),
-                         ZipPathParamName());
+INSTANTIATE_TEST_CASE_P(tests, OpsTest,
+                        ::testing::ValuesIn(UnarchiveAndFindTestNames(
+                            *FLAGS_zip_file_path, *FLAGS_tar_file_path)),
+                        ZipPathParamName());
 
 }  // namespace testing
 }  // namespace tflite
-- 
GitLab


From b30d3f5ac037c2d84f99130d1e7547d4dc6bbd5a Mon Sep 17 00:00:00 2001
From: Nick Felt <nickfelt@google.com>
Date: Mon, 14 Jan 2019 16:18:48 -0800
Subject: [PATCH 0714/1095] Expose tf.audio.encode_wav

PiperOrigin-RevId: 229277218
---
 .../core/api_def/python_api/api_def_AudioSpectrogram.pbtxt | 4 ++++
 tensorflow/core/api_def/python_api/api_def_DecodeWav.pbtxt | 4 ++++
 tensorflow/core/api_def/python_api/api_def_EncodeWav.pbtxt | 6 ++++++
 tensorflow/core/api_def/python_api/api_def_Mfcc.pbtxt      | 4 ++++
 tensorflow/python/BUILD                                    | 1 +
 tensorflow/python/__init__.py                              | 3 +++
 tensorflow/python/tools/api/generator/api_init_files.bzl   | 1 +
 .../python/tools/api/generator/api_init_files_v1.bzl       | 1 +
 tensorflow/tools/api/golden/v1/tensorflow.audio.pbtxt      | 7 +++++++
 tensorflow/tools/api/golden/v1/tensorflow.pbtxt            | 4 ++++
 tensorflow/tools/api/golden/v2/tensorflow.audio.pbtxt      | 7 +++++++
 tensorflow/tools/api/golden/v2/tensorflow.pbtxt            | 4 ++++
 12 files changed, 46 insertions(+)
 create mode 100644 tensorflow/core/api_def/python_api/api_def_AudioSpectrogram.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_DecodeWav.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_EncodeWav.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_Mfcc.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.audio.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.audio.pbtxt

diff --git a/tensorflow/core/api_def/python_api/api_def_AudioSpectrogram.pbtxt b/tensorflow/core/api_def/python_api/api_def_AudioSpectrogram.pbtxt
new file mode 100644
index 0000000000..38e4b38f3f
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_AudioSpectrogram.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "AudioSpectrogram"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_DecodeWav.pbtxt b/tensorflow/core/api_def/python_api/api_def_DecodeWav.pbtxt
new file mode 100644
index 0000000000..d6fd4691f7
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_DecodeWav.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "DecodeWav"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_EncodeWav.pbtxt b/tensorflow/core/api_def/python_api/api_def_EncodeWav.pbtxt
new file mode 100644
index 0000000000..edcf2bded1
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_EncodeWav.pbtxt
@@ -0,0 +1,6 @@
+op {
+  graph_op_name: "EncodeWav"
+  endpoint {
+    name: "audio.encode_wav"
+  }
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_Mfcc.pbtxt b/tensorflow/core/api_def/python_api/api_def_Mfcc.pbtxt
new file mode 100644
index 0000000000..21a0c8b8b2
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_Mfcc.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "Mfcc"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 905b47ea75..589605e347 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -112,6 +112,7 @@ py_library(
     ],
     deps = [
         ":array_ops",
+        ":audio_ops_gen",
         ":bitwise_ops",
         ":boosted_trees_ops",
         ":check_ops",
diff --git a/tensorflow/python/__init__.py b/tensorflow/python/__init__.py
index 31f9dce8e8..5172dbb836 100644
--- a/tensorflow/python/__init__.py
+++ b/tensorflow/python/__init__.py
@@ -99,6 +99,9 @@ from tensorflow.python.summary import summary
 from tensorflow.python.user_ops import user_ops
 from tensorflow.python.util import compat
 
+# Import audio ops to make sure the ops are registered.
+from tensorflow.python.ops import gen_audio_ops as _
+
 # Import boosted trees ops to make sure the ops are registered (but unused).
 from tensorflow.python.ops import gen_boosted_trees_ops as _gen_boosted_trees_ops
 
diff --git a/tensorflow/python/tools/api/generator/api_init_files.bzl b/tensorflow/python/tools/api/generator/api_init_files.bzl
index f496e90dfa..5182f53beb 100644
--- a/tensorflow/python/tools/api/generator/api_init_files.bzl
+++ b/tensorflow/python/tools/api/generator/api_init_files.bzl
@@ -4,6 +4,7 @@
 TENSORFLOW_API_INIT_FILES = [
     # BEGIN GENERATED FILES
     "__init__.py",
+    "audio/__init__.py",
     "autograph/__init__.py",
     "autograph/experimental/__init__.py",
     "bitwise/__init__.py",
diff --git a/tensorflow/python/tools/api/generator/api_init_files_v1.bzl b/tensorflow/python/tools/api/generator/api_init_files_v1.bzl
index 2d5898f31c..3e2c00c87f 100644
--- a/tensorflow/python/tools/api/generator/api_init_files_v1.bzl
+++ b/tensorflow/python/tools/api/generator/api_init_files_v1.bzl
@@ -5,6 +5,7 @@ TENSORFLOW_API_INIT_FILES_V1 = [
     # BEGIN GENERATED FILES
     "__init__.py",
     "app/__init__.py",
+    "audio/__init__.py",
     "autograph/__init__.py",
     "autograph/experimental/__init__.py",
     "bitwise/__init__.py",
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.audio.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.audio.pbtxt
new file mode 100644
index 0000000000..ce29615f72
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.audio.pbtxt
@@ -0,0 +1,7 @@
+path: "tensorflow.audio"
+tf_module {
+  member_method {
+    name: "encode_wav"
+    argspec: "args=[\'audio\', \'sample_rate\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
index deb1d100e7..016ae23890 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt
@@ -292,6 +292,10 @@ tf_module {
     name: "app"
     mtype: "<type \'module\'>"
   }
+  member {
+    name: "audio"
+    mtype: "<type \'module\'>"
+  }
   member {
     name: "autograph"
     mtype: "<type \'module\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.audio.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.audio.pbtxt
new file mode 100644
index 0000000000..ce29615f72
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.audio.pbtxt
@@ -0,0 +1,7 @@
+path: "tensorflow.audio"
+tf_module {
+  member_method {
+    name: "encode_wav"
+    argspec: "args=[\'audio\', \'sample_rate\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
index 7e5f86d7e6..92f4704b49 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt
@@ -68,6 +68,10 @@ tf_module {
     name: "VariableSynchronization"
     mtype: "<class \'enum.EnumMeta\'>"
   }
+  member {
+    name: "audio"
+    mtype: "<type \'module\'>"
+  }
   member {
     name: "autograph"
     mtype: "<type \'module\'>"
-- 
GitLab


From 681f6a6ac9344425549c8a38fcbfae854e7deddf Mon Sep 17 00:00:00 2001
From: Ayush Dubey <ayushd@google.com>
Date: Mon, 14 Jan 2019 16:19:38 -0800
Subject: [PATCH 0715/1095] Add a CollectiveReduce implementation that uses
 NCCL.

This change adds an implementation of the CollectiveImplementationInterface
that uses NCCL for all-reduce.  The main bits include setting up a NCCL
communicator key during collective param resolution, and calling the relevant
functionality in NcclManager.

PiperOrigin-RevId: 229277364
---
 tensorflow/core/BUILD                         |   2 +-
 .../base_collective_executor.cc               |  33 ++
 .../common_runtime/base_collective_executor.h |  18 +
 .../collective_param_resolver_local.cc        | 159 ++++++---
 .../collective_param_resolver_local.h         |  15 +-
 .../core/common_runtime/collective_util.cc    |  31 ++
 .../core/common_runtime/collective_util.h     |  21 ++
 .../hierarchical_tree_broadcaster.h           |   5 +
 .../core/common_runtime/ring_reducer.cc       |  43 +--
 tensorflow/core/common_runtime/ring_reducer.h |  24 +-
 tensorflow/core/distributed_runtime/BUILD     |   1 +
 .../collective_param_resolver_distributed.cc  |  21 +-
 ...lective_param_resolver_distributed_test.cc |   2 +
 tensorflow/core/framework/collective.cc       |  11 +
 tensorflow/core/framework/collective.h        |  32 +-
 tensorflow/core/kernels/BUILD                 |  24 ++
 .../core/kernels/collective_nccl_reducer.cc   | 204 +++++++++++
 .../core/kernels/collective_nccl_reducer.h    |  49 +++
 .../kernels/collective_nccl_reducer_test.cc   | 331 ++++++++++++++++++
 tensorflow/core/kernels/collective_ops.cc     |   3 +
 tensorflow/core/nccl/BUILD                    |   4 +-
 tensorflow/core/protobuf/worker.proto         |   1 +
 22 files changed, 916 insertions(+), 118 deletions(-)
 create mode 100644 tensorflow/core/kernels/collective_nccl_reducer.cc
 create mode 100644 tensorflow/core/kernels/collective_nccl_reducer.h
 create mode 100644 tensorflow/core/kernels/collective_nccl_reducer_test.cc

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index b655fdcbff..caf7b4cdbc 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -3713,7 +3713,6 @@ tf_cc_tests(
     srcs = [
         "common_runtime/buf_rendezvous_test.cc",
         "common_runtime/collective_executor_mgr_test.cc",
-        "common_runtime/collective_param_resolver_local_test.cc",
         "common_runtime/collective_rma_local_test.cc",
         "common_runtime/device_resolver_local_test.cc",
         "common_runtime/device_set_test.cc",
@@ -3829,6 +3828,7 @@ tf_cc_tests(
     name = "higher_level_tests_needing_kernels",
     size = "small",
     srcs = [
+        "common_runtime/collective_param_resolver_local_test.cc",
         "graph/graph_constructor_test.cc",
     ],
     linkopts = select({
diff --git a/tensorflow/core/common_runtime/base_collective_executor.cc b/tensorflow/core/common_runtime/base_collective_executor.cc
index 92e56df181..03c0e9ce2b 100644
--- a/tensorflow/core/common_runtime/base_collective_executor.cc
+++ b/tensorflow/core/common_runtime/base_collective_executor.cc
@@ -296,4 +296,37 @@ Status BaseCollectiveExecutor::CreateCollective(
   return status;
 }
 
+bool BaseCollectiveExecutor::CheckDependencies(
+    const CollectiveParams& col_params) {
+  for (int32 instance : col_params.instance.impl_details.dependencies) {
+    auto find_iter = launched_.find(instance);
+    if (find_iter == launched_.end() || find_iter->second != 0) {
+      return false;
+    }
+  }
+  return true;
+}
+
+void BaseCollectiveExecutor::WaitForDependencies(
+    const CollectiveParams& col_params) {
+  mutex_lock l(launch_mu_);
+  while (!CheckDependencies(col_params)) {
+    launch_cv_.wait(l);
+  }
+}
+
+void BaseCollectiveExecutor::Launched(const CollectiveParams& col_params) {
+  mutex_lock l(launch_mu_);
+  if (launched_.find(col_params.instance.instance_key) == launched_.end()) {
+    const string& task_name =
+        col_params.instance.task_names[col_params.default_rank];
+    const int32 num_devices =
+        col_params.instance.num_devices_per_task.at(task_name);
+    launched_[col_params.instance.instance_key] = num_devices;
+  }
+  if (--launched_[col_params.instance.instance_key] == 0) {
+    launch_cv_.notify_all();
+  }
+}
+
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/base_collective_executor.h b/tensorflow/core/common_runtime/base_collective_executor.h
index 09826a8814..b711aa6d50 100644
--- a/tensorflow/core/common_runtime/base_collective_executor.h
+++ b/tensorflow/core/common_runtime/base_collective_executor.h
@@ -135,15 +135,33 @@ class BaseCollectiveExecutor : public CollectiveExecutor {
                                client_locality, done);
   }
 
+  // If we need to enforce an ordering on any portion of collective
+  // implementation, and the ordering is encoded via attribute on the collective
+  // op, this function will block until all dependencies for this collective
+  // have completed.
+  void WaitForDependencies(const CollectiveParams& col_params) override;
+  // Record that this collective has completed the portion of the implementation
+  // that needs to be ordered wrt other collectives, to unblock any of its
+  // dependent ops.
+  void Launched(const CollectiveParams& col_params) override;
+
  protected:
   const int64 step_id_;
   const DeviceMgr* dev_mgr_;  // Not owned.
   std::unique_ptr<PerStepCollectiveRemoteAccess> remote_access_;
   const string* gpu_ring_order_;  // Not owned.
+  mutex launch_mu_;
+  condition_variable launch_cv_;
+  // collective instance key -> number of local devices for which NCCL ops have
+  // been launched.
+  std::unordered_map<int32, int32> launched_ GUARDED_BY(launch_mu_);
 
  private:
   Status CreateCollective(const CollectiveParams& col_params,
                           CollectiveImplementationInterface** col_impl);
+  // Check if all ops on which this collective depends on have launched.
+  bool CheckDependencies(const CollectiveParams& col_params)
+      EXCLUSIVE_LOCKS_REQUIRED(launch_mu_);
 };
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/collective_param_resolver_local.cc b/tensorflow/core/common_runtime/collective_param_resolver_local.cc
index a8e3f4c881..8907f6d56a 100644
--- a/tensorflow/core/common_runtime/collective_param_resolver_local.cc
+++ b/tensorflow/core/common_runtime/collective_param_resolver_local.cc
@@ -26,6 +26,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/gtl/flatmap.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/types.h"
 #include "tensorflow/core/util/device_name_utils.h"
@@ -39,7 +40,10 @@ void CollectiveParamResolverLocal::InstanceRec::WaitForOutMu(mutex_lock& lock) {
 CollectiveParamResolverLocal::CollectiveParamResolverLocal(
     const DeviceMgr* dev_mgr, DeviceResolverInterface* dev_resolver,
     const string& task_name)
-    : dev_mgr_(dev_mgr), dev_resolver_(dev_resolver), task_name_(task_name) {}
+    : nccl_(false),  // (b/111897089): turn on NCCL collectives.
+      dev_mgr_(dev_mgr),
+      dev_resolver_(dev_resolver),
+      task_name_(task_name) {}
 
 void CollectiveParamResolverLocal::CompleteGroupAsync(
     const CompleteGroupRequest* request, CompleteGroupResponse* response,
@@ -316,29 +320,28 @@ GlobalDeviceMap EstablishGlobalRank(
 // cp->same_num_devices_per_task.  Requires cp->instance.task_names
 // be sorted.
 void SetDevPerTask(CollectiveParams* cp) {
-  cp->instance.same_num_devices_per_task = false;
-  if (cp->instance.task_names.empty()) return;
-  int dev_per_task = -1;
-  int count = 0;
+  cp->instance.num_devices_per_task.clear();
   const string* last_task_name = &cp->instance.task_names[0];
+  int count = 0;
   for (const string& task_name : cp->instance.task_names) {
-    if (task_name != *last_task_name) {
-      CHECK_GT(count, 0);
-      if (dev_per_task < 0) {
-        dev_per_task = count;
-      } else {
-        CHECK_GT(dev_per_task, 0);
-        if (count != dev_per_task) return;
-      }
+    if (task_name == *last_task_name) {
+      ++count;
+    } else {
+      cp->instance.num_devices_per_task[*last_task_name] = count;
       count = 1;
       last_task_name = &task_name;
-    } else {
-      ++count;
     }
   }
-  CHECK_GT(count, 0);
-  if ((dev_per_task > 0) && (count != dev_per_task)) {
-    return;
+  cp->instance.num_devices_per_task[*last_task_name] = count;
+
+  cp->instance.same_num_devices_per_task = false;
+  int dev_per_task = -1;
+  for (const auto& task_dev : cp->instance.num_devices_per_task) {
+    if (dev_per_task == -1) {
+      dev_per_task = task_dev.second;
+    } else if (dev_per_task != task_dev.second) {
+      return;
+    }
   }
   cp->instance.same_num_devices_per_task = true;
   CHECK_EQ((cp->group.group_size % cp->group.num_tasks), 0);
@@ -398,7 +401,6 @@ void CollectiveParamResolverLocal::SetDefaultRank(const string& device,
 void CollectiveParamResolverLocal::InitInstanceSharedParams(
     const GroupRec* gr, const CollectiveParams* cp, InstanceRec* ir,
     const StatusCallback& done) {
-  VLOG(1) << "InitInstanceSharedParams " << ir;
   ir->shared.instance = cp->instance;
   {
     mutex_lock gl(gr->mu);
@@ -412,8 +414,8 @@ void CollectiveParamResolverLocal::InitInstanceSharedParams(
   }
   ir->shared.default_rank = -1;
 
-  // Sort devce_names lexicographcally, keeping task_names in
-  // corresponding order.
+  // Sort device_names lexicographically, keeping task_names in corresponding
+  // order.  Also set number of devices per task.
   SortDevicesAndTasks(&ir->shared);
 
   // Get Locality data for all devices.
@@ -605,6 +607,25 @@ void CollectiveParamResolverLocal::CompleteInstanceAsync(
                        "intended only for non-distributed deployment."));
 }
 
+// TODO(b/111897089): we need a better way to pick the collective
+// implementation.  The ideal way would depend upon the topology and link
+// strength before picking a particular implementation.
+void CollectiveParamResolverLocal::AssignCollectiveType(CollectiveParams* cp) {
+  if (cp->instance.type == BROADCAST_COLLECTIVE) {
+    cp->instance.impl_details.collective_name = "HierarchicalTreeBroadcast";
+  } else if (cp->instance.type == REDUCTION_COLLECTIVE) {
+    if (nccl_) {
+      cp->instance.impl_details.collective_name = "NcclReduce";
+    } else {
+      cp->instance.impl_details.collective_name = "RingReduce";
+    }
+  } else {
+    cp->instance.impl_details.collective_name = "undef";
+  }
+  VLOG(1) << "AssignCollectiveType "
+          << cp->instance.impl_details.collective_name;
+}
+
 void CollectiveParamResolverLocal::CompleteInstanceLocal(
     const string& device, const GroupRec* gr, CollectiveParams* cp,
     bool is_source, const StatusCallback& done) {
@@ -641,48 +662,57 @@ void CollectiveParamResolverLocal::CompleteInstanceFromInitializedIRec(
     // custom operator= does a deep copy.
     cp->instance = ir->shared.instance;
   }
-  // Populate the fields common across task, also default_rank.
+  // Populate the fields common across task.
+  AssignCollectiveType(cp);
   SetDefaultRank(device, cp);
   CompleteTaskIsLocal(task_name_, cp);
-  // TODO(b/113171733): we need a better way to pick the collective
-  // implementation.  The ideal way would depend upon the topology and link
-  // strength before picking a particular implementation.
-  cp->instance.impl_details.collective_name =
-      (cp->instance.type == BROADCAST_COLLECTIVE) ? "HierarchicalTreeBroadcast"
-                                                  : "RingReduce";
+
   CollectiveImplementationInterface* col_impl;
-  Status lookup_status = CollectiveRegistry::LookupParamResolverInstance(
+  Status status = CollectiveRegistry::LookupParamResolverInstance(
       cp->instance.impl_details.collective_name, &col_impl);
-  if (!lookup_status.ok()) {
-    done(lookup_status);
+  if (status.ok()) {
+    status = col_impl->InitializeInstanceBeforeGroupDiscovery(cp);
+  }
+  if (!status.ok()) {
+    done(status);
     return;
   }
-  // If broadcast, may need to wait for source discovery.
-  if (cp->instance.type == BROADCAST_COLLECTIVE) {
-    CompleteInstanceSource(ir, cp, is_source,
-                           [col_impl, ir, device, cp, done](InstanceRec* irec) {
-                             CHECK_EQ(ir, irec);
-                             Status s;
-                             {
-                               mutex_lock l(irec->out_mu);
-                               irec->WaitForOutMu(l);
-                               s = irec->status;
-                               cp->source_rank = irec->source_rank;
-                             }
-                             if (s.ok()) {
-                               s = col_impl->InitializeCollectiveParams(cp);
-                             }
-                             done(s);
-                           });
+
+  //  We may need to wait for the group if:
+  //  * this is a broadcast, for source discovery;
+  //  * we are using NCCL with more than 1 worker, for the communicator key from
+  //    rank 0.
+  bool broadcast = cp->instance.type == BROADCAST_COLLECTIVE;
+  bool nccl = cp->instance.type == REDUCTION_COLLECTIVE &&
+              cp->instance.impl_details.collective_name == "NcclReduce" &&
+              cp->group.num_tasks > 1;
+  if (broadcast || nccl) {
+    WaitForGroup(ir, cp, is_source, broadcast, nccl,
+                 [col_impl, ir, device, cp, done](InstanceRec* irec) {
+                   Status s;
+                   if (ir != irec) {
+                     s = errors::Internal("Expected ir ", ir, " and irec ",
+                                          irec, " to be equal");
+                   } else {
+                     mutex_lock l(irec->out_mu);
+                     irec->WaitForOutMu(l);
+                     s = irec->status;
+                     cp->source_rank = irec->source_rank;
+                     cp->instance.communicator_key = irec->communicator_key;
+                   }
+                   if (s.ok()) {
+                     s = col_impl->InitializeCollectiveParams(cp);
+                   }
+                   done(s);
+                 });
   } else {
     done(col_impl->InitializeCollectiveParams(cp));
   }
 }
 
-void CollectiveParamResolverLocal::CompleteInstanceSource(InstanceRec* ir,
-                                                          CollectiveParams* cp,
-                                                          bool is_source,
-                                                          const IRConsumer& f) {
+void CollectiveParamResolverLocal::WaitForGroup(
+    InstanceRec* ir, CollectiveParams* cp, bool is_source, bool init_source,
+    bool init_nccl, const IRConsumer& f) {
   std::vector<IRConsumer> ready_waiters;
   {
     mutex_lock l(ir->out_mu);
@@ -692,7 +722,8 @@ void CollectiveParamResolverLocal::CompleteInstanceSource(InstanceRec* ir,
     if (!ir->known[cp->default_rank]) {
       ir->known[cp->default_rank] = true;
       ++ir->known_count;
-      if (is_source) {
+      if (init_source && is_source) {
+        // Initialize source rank.
         if (ir->source_rank >= 0) {
           ir->status = errors::Internal("Instance ", cp->instance.instance_key,
                                         " already has source ", ir->source_rank,
@@ -702,13 +733,26 @@ void CollectiveParamResolverLocal::CompleteInstanceSource(InstanceRec* ir,
           ir->source_rank = cp->default_rank;
         }
       }
+      if (init_nccl && cp->default_rank == 0) {
+        // Initialize communicator key.
+        if (!ir->communicator_key.empty()) {
+          ir->status =
+              errors::Internal("Instance ", cp->instance.instance_key,
+                               " already has communicator_key ",
+                               str_util::CEscape(ir->communicator_key),
+                               ", received second claim from device ",
+                               cp->instance.device_names[cp->default_rank]);
+        } else {
+          ir->communicator_key = cp->instance.communicator_key;
+        }
+      }
     }
     if (ir->known_count < ir->shared.group.group_size) {
       ir->known_waiters.push_back(f);
       return;
     }
     CHECK_EQ(ir->known_count, ir->shared.group.group_size);
-    if (ir->source_rank < 0) {
+    if (init_source && ir->source_rank < 0) {
       // NOTE(ayushd): changing the error message below would also require
       // updating CompleteParamsBroadcastForgotSend test in
       // CollectiveParamResolverLocalTest.
@@ -718,6 +762,13 @@ void CollectiveParamResolverLocal::CompleteInstanceSource(InstanceRec* ir,
                            "could mean that there were group_size=",
                            ir->known_count, " BcastRecvs but no BcastSend.");
     }
+    if (init_nccl && ir->communicator_key.empty()) {
+      ir->status = errors::Internal(
+          "Instance ", cp->instance.instance_key, " device ",
+          cp->instance.device_names[cp->default_rank],
+          " did not find rank 0 for setting communicator key.  This is an "
+          "internal error in collective param resolution");
+    }
     if (!ir->known_waiters.empty()) {
       ready_waiters = std::move(ir->known_waiters);
     }
diff --git a/tensorflow/core/common_runtime/collective_param_resolver_local.h b/tensorflow/core/common_runtime/collective_param_resolver_local.h
index 365bddc787..fd408e4ef3 100644
--- a/tensorflow/core/common_runtime/collective_param_resolver_local.h
+++ b/tensorflow/core/common_runtime/collective_param_resolver_local.h
@@ -130,8 +130,10 @@ class CollectiveParamResolverLocal : public ParamResolverInterface {
     Status status GUARDED_BY(out_mu);
 
     // These fields are used to count the instances that have called
-    // in and become known while resolving broadcast source identity.
+    // in and become known while resolving broadcast source identity and
+    // communicator key.
     int source_rank GUARDED_BY(out_mu);
+    string communicator_key GUARDED_BY(out_mu);
     int known_count GUARDED_BY(out_mu);
     std::vector<bool> known GUARDED_BY(out_mu);
     std::vector<IRConsumer> known_waiters GUARDED_BY(out_mu);
@@ -197,10 +199,10 @@ class CollectiveParamResolverLocal : public ParamResolverInterface {
                                            const StatusCallback& done)
       LOCKS_EXCLUDED(ir->out_mu);
 
-  // Complete source data for a broadcast instance.
+  // Complete source data and/or nccl communicator key.
   // Precondition: *cp has complete group data and default_rank.
-  void CompleteInstanceSource(InstanceRec* ir, CollectiveParams* cp,
-                              bool is_source, const IRConsumer& f)
+  void WaitForGroup(InstanceRec* ir, CollectiveParams* cp, bool is_source,
+                    bool init_source, bool init_nccl, const IRConsumer& f)
       LOCKS_EXCLUDED(ir->out_mu);
 
   // If cp.device_names contains only devices local to this process
@@ -216,10 +218,15 @@ class CollectiveParamResolverLocal : public ParamResolverInterface {
   // current ordering of cp->instance.device_names.
   void SetDefaultRank(const string& device, CollectiveParams* cp);
 
+  // Sets cp->instance.type based on collective op type, and attempts to assign
+  // best implementation.
+  void AssignCollectiveType(CollectiveParams* cp);
+
   // Helper to grab status under lock, invoke callback out of lock.
   void CallbackWithStatus(const InstanceRecCallback& done, InstanceRec* irec)
       LOCKS_EXCLUDED(irec->out_mu);
 
+  const bool nccl_;
   const DeviceMgr* dev_mgr_;
   DeviceResolverInterface* dev_resolver_;  // Not owned.
   string task_name_;
diff --git a/tensorflow/core/common_runtime/collective_util.cc b/tensorflow/core/common_runtime/collective_util.cc
index 195521a078..bee4a13d18 100644
--- a/tensorflow/core/common_runtime/collective_util.cc
+++ b/tensorflow/core/common_runtime/collective_util.cc
@@ -79,5 +79,36 @@ string SubdivPermDebugString(const CollectiveParams& col_params) {
   return buf;
 }
 
+SubContext::SubContext(OpKernelContext* ctx, OpKernelContext::Params* params,
+                       OpKernel* op, Tensor* output, Tensor* input)
+    : sub_params_(*params),
+      sub_inputs_({output, input}),
+      sub_input_attr_({ctx->input_alloc_attr(0), ctx->input_alloc_attr(0)}),
+      sub_input_dc_(
+          {ctx->input_device_context(0), ctx->input_device_context(0)}) {
+  sub_params_.op_kernel = op;
+  sub_params_.inputs = &sub_inputs_;
+  sub_params_.input_alloc_attrs = &sub_input_attr_;
+  sub_params_.input_device_contexts = &sub_input_dc_;
+  sub_params_.eigen_gpu_device = nullptr;
+  sub_params_.ensure_eigen_gpu_device();
+  sub_params_.forward_from_array = &forward_from_;
+  sub_ctx_.reset(new OpKernelContext(&sub_params_, 1));
+}
+
+Status ComputeBinOp(OpKernelContext* op_ctx, OpKernelContext::Params* params,
+                    Device* device, OpKernel* op, Tensor* output,
+                    Tensor* input) {
+  // Prepare an OpKernelContext that is identical to that of the original Op
+  // (i.e. the collective), except for the input output sizes and identities and
+  // the Op itself.
+  // TODO(ayushd, tucker): Is it possible to cache and reuse these objects?
+  // They're mostly identical inside one device execution.
+  std::unique_ptr<SubContext> sub_ctx(
+      new SubContext(op_ctx, params, op, output, input));
+  device->Compute(op, sub_ctx->sub_ctx_.get());
+  return sub_ctx->sub_ctx_->status();
+}
+
 }  // namespace collective_util
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/collective_util.h b/tensorflow/core/common_runtime/collective_util.h
index ebb5731bec..01fb8b8c81 100644
--- a/tensorflow/core/common_runtime/collective_util.h
+++ b/tensorflow/core/common_runtime/collective_util.h
@@ -32,6 +32,27 @@ Status InitializeDeviceAndLocality(const DeviceMgr* dev_mgr,
                                    DeviceLocality* device_locality);
 string SubdivPermDebugString(const CollectiveParams& col_params);
 
+// Used for executing a sub-operation, e.g. a merge_op instance, with
+// an OpKernelContext based on the one passed into this Op.
+class SubContext {
+ public:
+  OpKernelContext::Params sub_params_;
+  gtl::InlinedVector<TensorValue, 4> sub_inputs_;
+  gtl::InlinedVector<AllocatorAttributes, 4> sub_input_attr_;
+  gtl::InlinedVector<DeviceContext*, 4> sub_input_dc_;
+  // Used only for Binary and Unary Ops for which we require
+  // the calculation to be in-place on the first input.
+  int forward_from_ = 0;
+  std::unique_ptr<OpKernelContext> sub_ctx_;
+  SubContext(OpKernelContext* ctx, OpKernelContext::Params* params,
+             OpKernel* op, Tensor* output, Tensor* input);
+  ~SubContext() = default;
+};
+
+Status ComputeBinOp(OpKernelContext* op_ctx, OpKernelContext::Params* params,
+                    Device* device, OpKernel* op, Tensor* output,
+                    Tensor* input);
+
 }  // namespace collective_util
 }  // namespace tensorflow
 
diff --git a/tensorflow/core/common_runtime/hierarchical_tree_broadcaster.h b/tensorflow/core/common_runtime/hierarchical_tree_broadcaster.h
index ceb9baad30..76392b8e59 100644
--- a/tensorflow/core/common_runtime/hierarchical_tree_broadcaster.h
+++ b/tensorflow/core/common_runtime/hierarchical_tree_broadcaster.h
@@ -41,6 +41,11 @@ class HierarchicalTreeBroadcaster : public CollectiveImplementationInterface {
   // and device_locality.  Also saves the CollectiveContext in this object.
   Status InitializeCollectiveContext(CollectiveContext* col_ctx) override;
 
+  // No-op for hierarchical tree broadcaster.
+  Status InitializeInstanceBeforeGroupDiscovery(CollectiveParams*) override {
+    return Status::OK();
+  }
+
   // Begins async execution of the hierarchical tree broadcast.
   // Must be called in a blockable thread.
   // TODO(b/80529858): remove the previous warning when we have a dedicated
diff --git a/tensorflow/core/common_runtime/ring_reducer.cc b/tensorflow/core/common_runtime/ring_reducer.cc
index 092f15e49e..8ed2fc2f1c 100644
--- a/tensorflow/core/common_runtime/ring_reducer.cc
+++ b/tensorflow/core/common_runtime/ring_reducer.cc
@@ -394,37 +394,6 @@ void RingReducer::Finish(bool ok) {
   done_(s);
 }
 
-RingReducer::SubContext::SubContext(OpKernelContext* ctx,
-                                    OpKernelContext::Params* params,
-                                    OpKernel* op, Tensor* output, Tensor* input)
-    : sub_params_(*params),
-      sub_inputs_({output, input}),
-      sub_input_attr_({ctx->input_alloc_attr(0), ctx->input_alloc_attr(0)}),
-      sub_input_dc_(
-          {ctx->input_device_context(0), ctx->input_device_context(0)}) {
-  sub_params_.op_kernel = op;
-  sub_params_.inputs = &sub_inputs_;
-  sub_params_.input_alloc_attrs = &sub_input_attr_;
-  sub_params_.input_device_contexts = &sub_input_dc_;
-  sub_params_.eigen_gpu_device = nullptr;
-  sub_params_.ensure_eigen_gpu_device();
-  sub_params_.forward_from_array = &forward_from_;
-  sub_ctx_ = new OpKernelContext(&sub_params_, 1);
-}
-
-Status RingReducer::ComputeBinOp(Device* device, OpKernel* op, Tensor* output,
-                                 Tensor* input) {
-  // Prepare an OpKernelContext that is identical to that of the original Op
-  // (i.e. the collective), except for the input output sizes and identities and
-  // the Op itself.
-  // TODO(tucker): Is it possible to cache and reuse these objects?  They're
-  // mostly identical inside one device execution.
-  std::unique_ptr<SubContext> sub_ctx(
-      new SubContext(col_ctx_->op_ctx, col_ctx_->op_params, op, output, input));
-  device->Compute(op, sub_ctx->sub_ctx_);
-  return sub_ctx->sub_ctx_->status();
-}
-
 // At the beginning of the algorithm initialize a RingField struct for
 // every independent field of the tensor.
 void RingReducer::InitRingField(RingField* rf, int chunk_idx, int subdiv_idx,
@@ -632,9 +601,9 @@ bool RingReducer::RunAsyncParts() {
           --recv_pending_count;
           if (!rf->second_pass) {
             rf->action = RF_REDUCE;
-            Status s =
-                ComputeBinOp(col_ctx_->device, col_params_->merge_op.get(),
-                             &rf->chunk, &rf->tmp_chunk);
+            Status s = collective_util::ComputeBinOp(
+                col_ctx_->op_ctx, col_ctx_->op_params, col_ctx_->device,
+                col_params_->merge_op.get(), &rf->chunk, &rf->tmp_chunk);
             if (!s.ok()) {
               aborted = true;
               StartAbort(s);
@@ -647,9 +616,9 @@ bool RingReducer::RunAsyncParts() {
           if (!rf->second_pass && col_params_->final_op.get() && rf->is_final) {
             rf->action = RF_FINALIZE;
             group_size_tensor_ready_.WaitForNotification();
-            Status s =
-                ComputeBinOp(col_ctx_->device, col_params_->final_op.get(),
-                             &rf->chunk, &group_size_tensor_);
+            Status s = collective_util::ComputeBinOp(
+                col_ctx_->op_ctx, col_ctx_->op_params, col_ctx_->device,
+                col_params_->final_op.get(), &rf->chunk, &group_size_tensor_);
             if (!s.ok()) {
               aborted = true;
               StartAbort(s);
diff --git a/tensorflow/core/common_runtime/ring_reducer.h b/tensorflow/core/common_runtime/ring_reducer.h
index 0848e37b52..a5aa8fad70 100644
--- a/tensorflow/core/common_runtime/ring_reducer.h
+++ b/tensorflow/core/common_runtime/ring_reducer.h
@@ -40,6 +40,11 @@ class RingReducer : public CollectiveImplementationInterface {
   // and device_locality.  Also saves the CollectiveContext in this object.
   Status InitializeCollectiveContext(CollectiveContext* col_ctx) override;
 
+  // No-op for ring reducer.
+  Status InitializeInstanceBeforeGroupDiscovery(CollectiveParams*) override {
+    return Status::OK();
+  }
+
   // Begins async execution of the ring reduce algorithm.
   // Must be called in a blockable thread.
   // TODO(b/80529858): remove the previous warning when we have a dedicated
@@ -52,27 +57,8 @@ class RingReducer : public CollectiveImplementationInterface {
   void StartAbort(const Status& s);
   void ContinueAfterInputCopy();
   void Finish(bool ok);
-  Status ComputeBinOp(Device* device, OpKernel* op, Tensor* output,
-                      Tensor* input);
   bool RunAsyncParts();
 
-  // Used for executing a sub-operation, e.g. a merge_op instance, with
-  // an OpKernelContext based on the one passed into this Op.
-  class SubContext {
-   public:
-    OpKernelContext::Params sub_params_;
-    gtl::InlinedVector<TensorValue, 4> sub_inputs_;
-    gtl::InlinedVector<AllocatorAttributes, 4> sub_input_attr_;
-    gtl::InlinedVector<DeviceContext*, 4> sub_input_dc_;
-    // Used only for Binary and Unary Ops for which we require
-    // the calculation to be in-place on the first input.
-    int forward_from_ = 0;
-    OpKernelContext* sub_ctx_;
-    SubContext(OpKernelContext* ctx, OpKernelContext::Params* params,
-               OpKernel* op, Tensor* output, Tensor* input);
-    ~SubContext() { delete sub_ctx_; }
-  };
-
   // Current status of a RingField
   enum RingFieldAction {
     RF_INIT = 0,    // Just initialized for a pass
diff --git a/tensorflow/core/distributed_runtime/BUILD b/tensorflow/core/distributed_runtime/BUILD
index e388d3e6f0..197a3f24d4 100644
--- a/tensorflow/core/distributed_runtime/BUILD
+++ b/tensorflow/core/distributed_runtime/BUILD
@@ -594,6 +594,7 @@ tf_cc_test(
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
+        "//tensorflow/core/kernels:collective_ops",
     ],
 )
 
diff --git a/tensorflow/core/distributed_runtime/collective_param_resolver_distributed.cc b/tensorflow/core/distributed_runtime/collective_param_resolver_distributed.cc
index 1dd10d309b..9f94a24fcd 100644
--- a/tensorflow/core/distributed_runtime/collective_param_resolver_distributed.cc
+++ b/tensorflow/core/distributed_runtime/collective_param_resolver_distributed.cc
@@ -181,6 +181,10 @@ void CollectiveParamResolverDistributed::CompleteInstanceAsync(
                           ir->WaitForOutMu(l);
                           response->set_instance_key(cp->instance.instance_key);
                           response->set_source_rank(ir->source_rank);
+                          if (!cp->instance.communicator_key.empty()) {
+                            response->set_communicator_key(
+                                cp->instance.communicator_key);
+                          }
                           done_and_cleanup(fi_status);
                         } else {
                           done_and_cleanup(fi_status);
@@ -283,8 +287,10 @@ void CollectiveParamResolverDistributed::UpdateInstanceCache(
   using InstanceRecPointer = InstanceRec*;
   InstanceRecPointer* irp = new InstanceRecPointer(nullptr);
   int32 source_rank = resp.source_rank();
+  string communicator_key = resp.communicator_key();
 
-  auto continue_with_ir = [this, cp, irp, source_rank, done](const Status& s) {
+  auto continue_with_ir = [cp, irp, source_rank, communicator_key,
+                           done](const Status& s) {
     if (!s.ok()) {
       done(s);
       delete irp;
@@ -306,6 +312,19 @@ void CollectiveParamResolverDistributed::UpdateInstanceCache(
         }
         ir->source_rank = source_rank;
       }
+      if (ir->communicator_key != communicator_key) {
+        if (!ir->communicator_key.empty()) {
+          ir->status = errors::Internal(
+              "UpdateInstanceCache: CompleteInstanceResponse for instance ",
+              cp->instance.instance_key,
+              " gives communicator_key with size =", communicator_key.size(),
+              " but cache already holds communicator_key with size=",
+              ir->communicator_key.size());
+          status = ir->status;
+          break;
+        }
+        ir->communicator_key = communicator_key;
+      }
       if (ir->known_count < cp->group.group_size) {
         ir->known_count = cp->group.group_size;
         if (ir->known.size() != cp->group.group_size) {
diff --git a/tensorflow/core/distributed_runtime/collective_param_resolver_distributed_test.cc b/tensorflow/core/distributed_runtime/collective_param_resolver_distributed_test.cc
index 40b18d321a..823d7d5eb9 100644
--- a/tensorflow/core/distributed_runtime/collective_param_resolver_distributed_test.cc
+++ b/tensorflow/core/distributed_runtime/collective_param_resolver_distributed_test.cc
@@ -268,6 +268,8 @@ class DeviceResDistTest : public ::testing::Test {
         EXPECT_EQ(cp_[idx].instance.device_names[idx], device_name);
         EXPECT_EQ(cp_[idx].instance.task_names[idx], task_name);
         if (idx > 0) {
+          EXPECT_EQ(cp_[0].instance.communicator_key,
+                    cp_[idx].instance.communicator_key);
           for (int i = 0; i < dev_count; ++i) {
             EXPECT_EQ(cp_[0].instance.device_names[i],
                       cp_[idx].instance.device_names[i]);
diff --git a/tensorflow/core/framework/collective.cc b/tensorflow/core/framework/collective.cc
index 7fa58347f2..b83d183f14 100644
--- a/tensorflow/core/framework/collective.cc
+++ b/tensorflow/core/framework/collective.cc
@@ -17,6 +17,7 @@ limitations under the License.
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/hash/hash.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 
 namespace tensorflow {
@@ -64,7 +65,9 @@ CollInstanceParams& CollInstanceParams::operator=(
     device_names.assign(other.device_names.begin(), other.device_names.end());
     task_names.assign(other.task_names.begin(), other.task_names.end());
     same_num_devices_per_task = other.same_num_devices_per_task;
+    num_devices_per_task = other.num_devices_per_task;
     gpu_ring_order = other.gpu_ring_order;
+    communicator_key = other.communicator_key;
     impl_details.subdiv_offsets.assign(
         other.impl_details.subdiv_offsets.begin(),
         other.impl_details.subdiv_offsets.end());
@@ -76,6 +79,7 @@ CollInstanceParams& CollInstanceParams::operator=(
     impl_details.subdiv_source_rank.assign(
         other.impl_details.subdiv_source_rank.begin(),
         other.impl_details.subdiv_source_rank.end());
+    impl_details.dependencies = other.impl_details.dependencies;
   }
   return *this;
 }
@@ -91,6 +95,13 @@ string CollInstanceParams::ToString() const {
   for (const auto& n : task_names) {
     strings::StrAppend(&v, n, ", ");
   }
+  strings::StrAppend(&v, "} num_devices_per_task={");
+  for (const auto dpt : num_devices_per_task) {
+    strings::StrAppend(&v, dpt.first, ": ", dpt.second, ", ");
+  }
+  strings::StrAppend(&v, "}, collective_name=", impl_details.collective_name,
+                     ", communicator_key=", str_util::CEscape(communicator_key),
+                     ", subdiv_offsets={");
   strings::StrAppend(&v, "}, subdiv_offsets={");
   for (const auto& d : impl_details.subdiv_offsets) {
     strings::StrAppend(&v, d, ",");
diff --git a/tensorflow/core/framework/collective.h b/tensorflow/core/framework/collective.h
index 0321429702..546e3938a8 100644
--- a/tensorflow/core/framework/collective.h
+++ b/tensorflow/core/framework/collective.h
@@ -70,6 +70,8 @@ struct CollImplDetails {
   std::vector<std::vector<int>> subdiv_permutations;
   std::vector<int> subdiv_offsets;
   std::vector<int> subdiv_source_rank;  // rank of source in each subdiv
+  std::vector<int32>
+      dependencies;  // collective instances on which this node depends
 };
 
 // Data common to all members of a collective instance.
@@ -85,9 +87,13 @@ struct CollInstanceParams {
   std::vector<string> task_names;
   // True if every task has the same number of devices.
   bool same_num_devices_per_task = false;
+  // Task -> number of devices on that task.
+  std::unordered_map<string, int32> num_devices_per_task;
   // If passed in to GPUOptions in ConfigProto, defines a good ring order for
   // GPUs.  Assumes same GPU configuration at each worker.
   string gpu_ring_order = "";
+  // Valid when using a communicator-based collective mechanism, e.g. NCCL.
+  string communicator_key;
   CollImplDetails impl_details;
   string ToString() const;
   CollInstanceParams& operator=(const struct CollInstanceParams& other);
@@ -269,6 +275,21 @@ class CollectiveExecutor : public PeerAccessInterface, public core::RefCounted {
 
   virtual PerStepCollectiveRemoteAccess* remote_access() { return nullptr; }
 
+  // `WaitForDependencies` and `Launched` are used for fine-grained control of
+  // execution order between collective instances.  These functions are intended
+  // to be called in `Run` function of collective implementations, and may be
+  // used to make part, or whole, of the collective execution ordered with
+  // respect to other collective instances.
+  //
+  // `WaitForDependencies` will block until it is safe to continue the callee's
+  // execution, where safety is defined as: ordered with respect to the
+  // collective instances defined in the callee's `wait_for` attribute.
+  virtual void WaitForDependencies(const CollectiveParams& col_params) {}
+  // `Launched` unblocks the dependent collective instances by recording that
+  // this callee device has completed the critical portion of the collective
+  // execution.
+  virtual void Launched(const CollectiveParams& col_params) {}
+
   // Used to designate an invalid group or instance key.
   static int64 kInvalidId;
 
@@ -347,7 +368,8 @@ class CollectiveImplementationInterface {
 
   // Initializes the portions of `col_params` specific to this
   // implementation.  Called exactly once for every Collective instance during
-  // the CollectiveParams resolution process when the graph is first executed.
+  // the CollectiveParams resolution process when the graph is first executed,
+  // at the end of `CompleteInstanceLocal()`.
   // NOTE(ayushd): This is effectively a static function because it modifies the
   // `col_params` passed in and should not manipulate any data members.  However
   // because it is virtual and needs to be implemented by every derived class we
@@ -360,6 +382,14 @@ class CollectiveImplementationInterface {
   // object.
   virtual Status InitializeCollectiveContext(CollectiveContext* col_ctx) = 0;
 
+  // Initializes instance params at the beginning of `CompleteInstanceLocal()`,
+  // unlike `InitializeCollectiveParams` which is called at the end.  This
+  // function is called before all devices in the instance are discovered, and
+  // may be used to broadcast data via the shared `InstanceRec` object in
+  // collective param resolution to all devices.
+  virtual Status InitializeInstanceBeforeGroupDiscovery(
+      CollectiveParams* col_params) = 0;
+
   // Processes and moves data according to the logic of this Collective
   // implementation.  Relies on appropriate initialization of op-specific
   // CollectiveParams in InitializeCollectiveParams(), as well as appropriate
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 0792b398f1..4f21b466cd 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -181,12 +181,36 @@ tf_cc_test(
 
 tf_kernel_library(
     name = "collective_ops",
+    srcs = if_cuda([
+        "collective_nccl_reducer.h",
+        "collective_nccl_reducer.cc",
+    ]),
     prefix = "collective_ops",
     deps = [
         "//tensorflow/core:collective_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:protos_all_cc",
+    ] + if_cuda([
+        "@local_config_nccl//:nccl",
+        "//tensorflow/core/nccl:nccl_lib",
+    ]),
+)
+
+tf_cuda_cc_test(
+    name = "collective_nccl_reducer_test",
+    size = "small",
+    srcs = ["collective_nccl_reducer_test.cc"],
+    tags = tf_cuda_tests_tags() + ["no_cuda_on_cpu_tap"],
+    deps = [
+        "//tensorflow/core:all_kernels",
+        "//tensorflow/core:core_cpu",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
     ],
 )
 
diff --git a/tensorflow/core/kernels/collective_nccl_reducer.cc b/tensorflow/core/kernels/collective_nccl_reducer.cc
new file mode 100644
index 0000000000..113f1487f0
--- /dev/null
+++ b/tensorflow/core/kernels/collective_nccl_reducer.cc
@@ -0,0 +1,204 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/core/kernels/collective_nccl_reducer.h"
+
+#ifdef GOOGLE_CUDA
+
+#include "tensorflow/core/common_runtime/collective_util.h"
+#include "tensorflow/core/nccl/nccl_manager.h"
+
+namespace tensorflow {
+namespace {
+string NcclCollectiveKey(const string& exec_key, int step_id) {
+  return strings::StrCat(exec_key, ":", step_id);
+}
+}  // namespace
+
+NcclReducer::NcclReducer() : col_ctx_(nullptr), col_params_(nullptr) {}
+
+Status NcclReducer::InitializeCollectiveParams(CollectiveParams* col_params) {
+  if (col_params->instance.type != REDUCTION_COLLECTIVE ||
+      col_params->instance.impl_details.collective_name != "NcclReduce") {
+    return errors::Internal("Unexpected collective type ",
+                            col_params->instance.type, " expected ",
+                            REDUCTION_COLLECTIVE, "; or collective name ",
+                            col_params->instance.impl_details.collective_name,
+                            " expected NcclReduce");
+  } else {
+    return Status::OK();
+  }
+}
+
+Status NcclReducer::InitializeCollectiveContext(CollectiveContext* col_ctx) {
+  col_ctx_ = col_ctx;
+  col_params_ = &col_ctx->col_params;
+  return collective_util::InitializeDeviceAndLocality(
+      col_ctx->dev_mgr, col_ctx->device_name, &col_ctx->device,
+      &col_ctx->device_locality);
+}
+
+Status NcclReducer::InitializeInstanceBeforeGroupDiscovery(
+    CollectiveParams* col_params) {
+  if (col_params->default_rank == 0 && col_params->group.num_tasks > 1) {
+    col_params->instance.communicator_key =
+        NcclManager::instance()->GenerateCommunicatorKey();
+  }
+  return Status::OK();
+}
+
+Status ReductionOp(const string& merge_op, ncclRedOp_t* reduction_op) {
+  if (merge_op == "Add") {
+    *reduction_op = ncclSum;
+    return Status::OK();
+  } else if (merge_op == "Mul") {
+    *reduction_op = ncclProd;
+    return Status::OK();
+  } else {
+    return errors::Internal("Expected merge_op to be either Add or Mul, found ",
+                            merge_op);
+  }
+}
+
+void NcclReducer::Run(StatusCallback done) {
+  ncclRedOp_t reduction_op;
+  Status s = ReductionOp(col_params_->merge_op->type_string(), &reduction_op);
+  if (!s.ok()) {
+    done(s);
+    return;
+  }
+
+  Tensor group_size;
+  Notification group_size_ready;
+  Status group_size_status;
+  if (col_params_->final_op) {
+    // Create an on-device scalar value from group_size_.
+    // TODO(ayushd, tucker): avoid this copy by either reusing across
+    // invocations or providing the scalar to the kernel in host memory.
+    Tensor group_size_val(col_ctx_->output->dtype(), TensorShape({}));
+    switch (col_ctx_->output->dtype()) {
+      case DT_FLOAT:
+        group_size_val.scalar<float>()() = col_params_->group.group_size;
+        break;
+      case DT_DOUBLE:
+        group_size_val.scalar<double>()() = col_params_->group.group_size;
+        break;
+      case DT_INT32:
+        group_size_val.scalar<int32>()() = col_params_->group.group_size;
+        break;
+      case DT_INT64:
+        group_size_val.scalar<int64>()() = col_params_->group.group_size;
+        break;
+      default:
+        done(errors::Internal("Unsupported type ", col_ctx_->output->dtype()));
+        return;
+    }
+    group_size = Tensor(
+        col_ctx_->device->GetAllocator(col_ctx_->op_ctx->input_alloc_attr(0)),
+        col_ctx_->output->dtype(), TensorShape({}));
+    DeviceContext* op_dev_ctx = col_ctx_->op_ctx->op_device_context();
+    // Enqueue copy on gpu stream.
+    op_dev_ctx->CopyCPUTensorToDevice(
+        &group_size_val, col_ctx_->device, &group_size,
+        [&group_size_ready, &group_size_status](const Status& s) {
+          group_size_status = s;
+          group_size_ready.Notify();
+        });
+  } else {
+    group_size_ready.Notify();
+  }
+
+  Notification nccl_done;
+  Status nccl_status;
+  auto* compute_stream = col_ctx_->op_ctx->op_device_context()->stream();
+  auto* gpu_info = col_ctx_->op_ctx->device()->tensorflow_gpu_device_info();
+  // `AddToAllReduce` performs consistency checks for the NCCL call and enqueues
+  // the `Participant` struct locally.  When all local participants with this
+  // `nccl_collective_key` have called `AddToAllReduce` and
+  // `SignalMultiNodeReady`, all devices at this worker are ready to process
+  // this NCCL op.
+  //
+  // The `NcclManager` uses a dedicated CUDA stream for NCCL kernels.  At this
+  // point, it synchronizes the NCCL stream with the compute stream, and then
+  // enqueues the NCCL kernel on the NCCL stream.
+  const int num_global_devices = col_params_->group.group_size;
+  const int num_local_devices = col_params_->instance.num_devices_per_task.at(
+      col_params_->instance.task_names[col_params_->default_rank]);
+  const string nccl_collective_key =
+      NcclCollectiveKey(col_ctx_->exec_key, col_ctx_->step_id);
+  auto done_callback = [&nccl_done, &nccl_status](const Status& s) {
+    nccl_status = s;
+    nccl_done.Notify();
+  };
+  auto participant = absl::make_unique<NcclManager::Participant>(
+      compute_stream->parent(), compute_stream, gpu_info->event_mgr,
+      gpu_info->gpu_id, col_ctx_->input, col_ctx_->output,
+      col_params_->default_rank, std::move(done_callback));
+  VLOG(1) << "NcclReducer calling NcclManager::AddToAllReduce num_tasks "
+          << col_params_->group.num_tasks << " current task "
+          << col_params_->instance.task_names[col_params_->default_rank]
+          << " num local devices " << num_local_devices
+          << " num global devices " << num_global_devices;
+  NcclManager::instance()->AddToAllReduce(
+      std::move(participant),
+      {nccl_collective_key, num_local_devices, num_global_devices,
+       col_params_->instance.communicator_key},
+      reduction_op);
+
+  // NOTE(ayushd): We need to synchronize NCCL launches across nodes to prevent
+  // deadlocks.  In the current implementation, we define a deterministic
+  // sequential launch order between potentially concurrent collective instances
+  // by introducing control information during static graph analysis in
+  // graph/collective_order.cc.  This can be either in the form of explicit
+  // control edges or via `wait_for` attribute on the collective op.
+  //
+  // The other end of the design spectrum would have a distinguished node
+  // dynamically signal the next collective to launch to all other participants.
+  // This has higher degree of runtime coordination, but it may be able to
+  // achieve better performance if the (arbitrary) static execution order
+  // assigned in the first approach turns out to not be good from a scheduling
+  // perspective.  e.g. consider a graph in which c1, c2, and c3 are three
+  // concurrent collective instances, and the static ordering assigns c1 -> c2
+  // -> c3.  In practice, it could turn out that c3 is always ready to execute
+  // before c1 or c2.
+  //
+  // `WaitForDependencies` may block if the collective instances on which this
+  // op depends have not yet launched.  When this function returns, this op is
+  // ready to go.
+  col_ctx_->col_exec->WaitForDependencies(*col_params_);
+  NcclManager::instance()->SignalMultiNodeReady(nccl_collective_key);
+  // When all devices at this worker have called `SignalMultiNodeReady`, the
+  // `NcclManager` will enqueue the NCCL kernel on the NCCL stream.  Thus the
+  // implementation of `Launched` keeps track of the number of devices that have
+  // launched.
+  col_ctx_->col_exec->Launched(*col_params_);
+
+  // Wait for nccl op and group_size copy to succeed, then do final_op.
+  group_size_ready.WaitForNotification();
+  nccl_done.WaitForNotification();
+  Status final_status =
+      group_size_status.ok() ? nccl_status : group_size_status;
+  if (final_status.ok() && col_params_->final_op) {
+    final_status = collective_util::ComputeBinOp(
+        col_ctx_->op_ctx, col_ctx_->op_params, col_ctx_->device,
+        col_params_->final_op.get(), col_ctx_->output, &group_size);
+  }
+  done(final_status);
+}
+
+REGISTER_COLLECTIVE(NcclReduce, NcclReducer);
+
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/collective_nccl_reducer.h b/tensorflow/core/kernels/collective_nccl_reducer.h
new file mode 100644
index 0000000000..dc70b280c5
--- /dev/null
+++ b/tensorflow/core/kernels/collective_nccl_reducer.h
@@ -0,0 +1,49 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_CORE_KERNELS_COLLECTIVE_NCCL_REDUCER_H_
+#define TENSORFLOW_CORE_KERNELS_COLLECTIVE_NCCL_REDUCER_H_
+
+#include "tensorflow/core/framework/collective.h"
+
+namespace tensorflow {
+#ifdef GOOGLE_CUDA
+
+class NcclReducer : public CollectiveImplementationInterface {
+ public:
+  NcclReducer();
+  ~NcclReducer() override = default;
+
+  // No-op for this collective implementation.
+  Status InitializeCollectiveParams(CollectiveParams* col_params) override;
+
+  // Initializes the device objects and device localities.
+  Status InitializeCollectiveContext(CollectiveContext* col_ctx) override;
+
+  // Initialize nccl communicator key.
+  Status InitializeInstanceBeforeGroupDiscovery(
+      CollectiveParams* col_params) override;
+
+  // Hands off all reduce to NcclManager.
+  void Run(StatusCallback done) override;
+
+ private:
+  CollectiveContext* col_ctx_;          // Not owned
+  const CollectiveParams* col_params_;  // Not owned
+};
+
+#endif  // GOOGLE_CUDA
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_KERNELS_COLLECTIVE_NCCL_REDUCER_H_
diff --git a/tensorflow/core/kernels/collective_nccl_reducer_test.cc b/tensorflow/core/kernels/collective_nccl_reducer_test.cc
new file mode 100644
index 0000000000..ff33c1322c
--- /dev/null
+++ b/tensorflow/core/kernels/collective_nccl_reducer_test.cc
@@ -0,0 +1,331 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/core/kernels/collective_nccl_reducer.h"
+
+#include <algorithm>
+#include "absl/memory/memory.h"
+#include "tensorflow/core/common_runtime/base_collective_executor.h"
+#include "tensorflow/core/common_runtime/device.h"
+#include "tensorflow/core/common_runtime/device_factory.h"
+#include "tensorflow/core/common_runtime/device_mgr.h"
+#include "tensorflow/core/common_runtime/device_resolver_local.h"
+#include "tensorflow/core/common_runtime/process_util.h"
+#include "tensorflow/core/common_runtime/test_collective_executor_mgr.h"
+#include "tensorflow/core/framework/collective.h"
+#include "tensorflow/core/framework/fake_input.h"
+#include "tensorflow/core/framework/node_def_builder.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/lib/core/notification.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/platform/test.h"
+#include "tensorflow/core/public/session_options.h"
+#include "tensorflow/core/public/version.h"
+
+#ifdef GOOGLE_CUDA
+
+namespace tensorflow {
+static constexpr int kStepId = 10;
+
+std::unique_ptr<OpKernel> GetKernel(const NodeDef& node, DeviceBase* device) {
+  Status status;
+  std::unique_ptr<OpKernel> k = CreateOpKernel(
+      DEVICE_GPU, device, device->GetAllocator(AllocatorAttributes()), node,
+      TF_GRAPH_DEF_VERSION, &status);
+  if (!status.ok()) LOG(FATAL) << status;
+  return k;
+}
+
+std::unique_ptr<OpKernel> GetAdd(DeviceBase* device) {
+  NodeDef node_def;
+  NodeDefBuilder builder("add_node", "Add");
+  TF_CHECK_OK(builder.Attr("T", DT_FLOAT)
+                  .Input(FakeInput(DT_FLOAT))
+                  .Input(FakeInput(DT_FLOAT))
+                  .Finalize(&node_def));
+  return GetKernel(node_def, device);
+}
+
+std::unique_ptr<OpKernel> GetDiv(DeviceBase* device) {
+  NodeDef node_def;
+  NodeDefBuilder builder("add_node", "Div");
+  TF_CHECK_OK(builder.Attr("T", DT_FLOAT)
+                  .Input(FakeInput(DT_FLOAT))
+                  .Input(FakeInput(DT_FLOAT))
+                  .Finalize(&node_def));
+  return GetKernel(node_def, device);
+}
+
+class NcclReducerTest : public ::testing::Test {
+ protected:
+  ~NcclReducerTest() override {
+    if (col_exec_) col_exec_->Unref();
+  }
+
+  void InitGPUDevices() {
+    std::vector<std::unique_ptr<Device>> all_devices;
+    SessionOptions session_options;
+    session_options.config.mutable_gpu_options()
+        ->set_per_process_gpu_memory_fraction(0.1);
+    session_options.env = Env::Default();
+    Status s = DeviceFactory::GetFactory(DEVICE_GPU)
+                   ->AddDevices(session_options, "", &all_devices);
+    TF_CHECK_OK(s);
+    for (std::unique_ptr<Device>& d : all_devices) {
+      if (d->device_type() == "GPU") {
+        gpus_.emplace_back(std::move(d));
+      }
+    }
+  }
+
+  void Init(int num_ranks) {
+    setenv("NCCL_DEBUG", "INFO", 1 /* replace */);
+    setenv("NCCL_LAUNCH_MODE", "PARALLEL", 1 /* replace */);
+    InitGPUDevices();
+    std::vector<std::unique_ptr<Device>> local_devices;
+    std::vector<string> device_names;
+    for (int rank = 0; rank < num_ranks; ++rank) {
+      if (rank < gpus_.size()) {
+        local_devices.emplace_back(std::move(gpus_[rank]));
+      }
+    }
+    int num_gpus = local_devices.size();
+    for (const auto& device : local_devices) {
+      device_names.push_back(device->name());
+      VLOG(2) << device->name();
+    }
+    if (!dev_mgr_) dev_mgr_.reset(new DeviceMgr(std::move(local_devices)));
+    col_exec_ = new BaseCollectiveExecutor(
+        &col_exec_mgr_, /*remote_access=*/nullptr, kStepId, dev_mgr_.get(),
+        /*gpu_ring_order=*/nullptr);
+
+    // Initialize collective params.
+    col_params_.name = "test_nccl_collective_op";
+    const int group_key = 5;
+    col_params_.group.group_key = group_key;
+    col_params_.group.device_type = DEVICE_GPU;
+    col_params_.group.group_size = num_ranks;
+    const int instance_key = 23;
+    col_params_.instance.instance_key = instance_key;
+    col_params_.instance.type = REDUCTION_COLLECTIVE;
+    col_params_.instance.data_type = DT_FLOAT;
+    col_params_.instance.impl_details.collective_name = "NcclReduce";
+    const string task_name = "/job:worker/replica:0/task:0";
+    col_params_.instance.num_devices_per_task[task_name] = num_ranks;
+    for (int rank = 0; rank < num_ranks; ++rank) {
+      col_params_.instance.device_names.push_back(
+          device_names[rank % num_gpus]);
+      col_params_.instance.task_names.push_back(task_name);
+    }
+    for (int rank = 0; rank < num_ranks; ++rank) {
+      instances_.push_back(absl::make_unique<DeviceInstance>(
+          rank, col_params_.instance.device_names[rank], this));
+    }
+  }
+
+  void Reduce() {
+    int done = 0;
+    mutex done_mu;
+    condition_variable done_cv;
+    for (const auto& instance : instances_) {
+      DeviceInstance* di = instance.get();
+      SchedClosure([di, &done, &done_mu, &done_cv] {
+        di->DoReduce();
+        mutex_lock l(done_mu);
+        ++done;
+        done_cv.notify_all();
+      });
+    }
+
+    mutex_lock l(done_mu);
+    while (done < instances_.size()) done_cv.wait(l);
+  }
+
+  void RunTest(int num_ranks, int tensor_length) {
+    Init(num_ranks);
+    std::vector<float> expected(tensor_length, 0.0);
+    for (int rank = 0; rank < num_ranks; ++rank) {
+      DeviceInstance* instance = instances_[rank].get();
+      instance->InitTensor(DT_FLOAT, TensorShape({tensor_length}),
+                           [&expected, rank](Tensor* t) {
+                             for (size_t i = 0; i < t->NumElements(); ++i) {
+                               float value = pow(10, rank) * i;
+                               t->flat<float>()(i) = value;
+                               expected[i] += value;
+                             }
+                           });
+    }
+    Reduce();
+    // Confirm that every rank computed the same correct value.
+    for (int i = 0; i < tensor_length; ++i) {
+      expected[i] /= num_ranks;
+    }
+    for (int rank = 0; rank < instances_.size(); ++rank) {
+      TF_ASSERT_OK(instances_[rank]->status_);
+      Tensor* dev_tensor = &instances_[rank]->tensor_;
+      Tensor actual(DT_FLOAT, TensorShape({tensor_length}));
+      Notification note;
+      Device* dev = instances_[rank]->device_;
+      auto* dev_info = dev->tensorflow_gpu_device_info();
+      dev_info->default_context->CopyDeviceTensorToCPU(
+          dev_tensor, /*tensor_name=*/"", dev, &actual,
+          [&note](const Status&) { note.Notify(); });
+      note.WaitForNotification();
+      for (int i = 0; i < tensor_length; ++i) {
+        EXPECT_FLOAT_EQ(expected[i], actual.template flat<float>()(i))
+            << "Mismatch at rank " << rank << " index " << i;
+      }
+    }
+  }
+
+  std::unique_ptr<OpKernel> GetCollectiveReduce(const CollectiveParams& params,
+                                                Tensor* input,
+                                                DeviceBase* device) {
+    mutex_lock l(mu_);
+    NodeDef node_def;
+    NodeDefBuilder builder(
+        strings::StrCat("collective_reduce_", reduce_counter_++),
+        "CollectiveReduce");
+    TF_CHECK_OK(
+        builder.Attr("T", params.instance.data_type)
+            .Attr("merge_op", "Add")
+            .Attr("final_op", "Div")
+            .Attr("group_size", params.group.group_size)
+            .Attr("group_key", params.group.group_key)
+            .Attr("instance_key", params.instance.instance_key)
+            .Attr("subdiv_offsets", params.instance.impl_details.subdiv_offsets)
+            .Input(FakeInput(params.instance.data_type))
+            .Finalize(&node_def));
+    return GetKernel(node_def, device);
+  }
+
+  class DeviceInstance {
+   public:
+    DeviceInstance(int rank, const string& device_name, NcclReducerTest* parent)
+        : parent_(parent), device_name_(device_name), rank_(rank) {
+      TF_CHECK_OK(parent_->dev_mgr_->LookupDevice(device_name_, &device_))
+          << "Could not find device " << device_name_ << " existing devices "
+          << parent_->dev_mgr_->DebugString();
+      col_params_.name = parent_->col_params_.name;
+      col_params_.default_rank = rank;
+      col_params_.group.group_key = parent_->col_params_.group.group_key;
+      col_params_.group.device_type = parent_->col_params_.group.device_type;
+      col_params_.group.group_size = parent_->col_params_.group.group_size;
+      col_params_.instance = parent->col_params_.instance;
+    }
+
+    void InitTensor(DataType dtype, const TensorShape& shape,
+                    const std::function<void(Tensor*)>& init_f) {
+      tensor_ =
+          Tensor(device_->GetAllocator(AllocatorAttributes()), dtype, shape);
+      Tensor cpu_tensor(dtype, shape);
+      init_f(&cpu_tensor);
+      VLOG(2) << "cpu_tensor " << cpu_tensor.DebugString();
+      auto* dev_info = device_->tensorflow_gpu_device_info();
+      Notification note;
+      dev_info->default_context->CopyCPUTensorToDevice(
+          &cpu_tensor, device_, &tensor_,
+          [&note](const Status&) { note.Notify(); });
+      note.WaitForNotification();
+    }
+
+    void DoReduce() {
+      col_params_.merge_op = GetAdd(device_);
+      col_params_.final_op = GetDiv(device_);
+
+      // Prepare an OpKernelContext.
+      OpKernelContext::Params op_params;
+      op_params.step_id = kStepId;
+      op_params.device = device_;
+      gtl::InlinedVector<TensorValue, 4> inputs;
+      inputs.push_back(TensorValue(&tensor_));
+      op_params.inputs = &inputs;
+      gtl::InlinedVector<AllocatorAttributes, 4> input_aa(
+          {AllocatorAttributes()});
+      op_params.input_alloc_attrs = &input_aa;
+      gtl::InlinedVector<DeviceContext*, 4> input_dc;
+      DeviceContext* dev_ctx = nullptr;
+      auto* dev_info = device_->tensorflow_gpu_device_info();
+      if (dev_info) {
+        dev_ctx = dev_info->default_context;
+        dev_ctx->Ref();
+      } else {
+        dev_ctx = new DeviceContext;
+      }
+      input_dc.push_back(dev_ctx);
+      op_params.input_device_contexts = &input_dc;
+      op_params.op_device_context = dev_ctx;
+      int forward_from = 0;
+      op_params.forward_from_array = &forward_from;
+      AllocatorAttributes generic_alloc_attr;
+      op_params.output_attr_array = &generic_alloc_attr;
+      std::unique_ptr<OpKernel> op =
+          parent_->GetCollectiveReduce(col_params_, &tensor_, device_);
+      op_params.op_kernel = op.get();
+      OpKernelContext ctx(&op_params, 1);
+
+      // We never actually execute the kernel, so we need to do the output
+      // allocation it would do, ourselves.
+      Tensor* output_tensor_ptr = nullptr;
+      TF_CHECK_OK(ctx.forward_input_or_allocate_output({0}, 0, tensor_.shape(),
+                                                       &output_tensor_ptr));
+      CHECK_EQ(output_tensor_ptr, ctx.mutable_output(0));
+
+      // Prepare a NcclReducer instance.
+      string exec_key =
+          strings::StrCat(col_params_.instance.instance_key, ":0:0");
+      NcclReducer reducer;
+      CollectiveContext col_ctx(parent_->col_exec_, parent_->dev_mgr_.get(),
+                                &ctx, &op_params, col_params_, exec_key,
+                                kStepId, &tensor_, &tensor_);
+      TF_CHECK_OK(reducer.InitializeCollectiveContext(&col_ctx));
+
+      // Run the all-reduce.
+      reducer.Run([this](Status s) { status_ = s; });
+      if (status_.ok()) {
+        CHECK(tensor_.CopyFrom(*ctx.mutable_output(0), tensor_.shape()));
+      }
+
+      dev_ctx->Unref();
+    }
+
+    NcclReducerTest* parent_;
+    string device_name_;
+    int rank_;
+    Tensor tensor_;
+    Device* device_;
+    CollectiveParams col_params_;
+    Status status_;
+  };
+
+  std::vector<std::unique_ptr<tensorflow::Device>> gpus_;
+  TestCollectiveExecutorMgr col_exec_mgr_;
+  CollectiveExecutor* col_exec_;
+  std::unique_ptr<DeviceMgr> dev_mgr_;
+  std::vector<std::unique_ptr<DeviceInstance>> instances_;
+  CollectiveParams col_params_;
+  mutex mu_;
+  int32 reduce_counter_ GUARDED_BY(mu_) = 0;
+};
+
+TEST_F(NcclReducerTest, Test2Dev16Len) { RunTest(2, 16); }
+TEST_F(NcclReducerTest, Test4Dev16Len) { RunTest(4, 16); }
+TEST_F(NcclReducerTest, Test8Dev16Len) { RunTest(8, 16); }
+TEST_F(NcclReducerTest, Test8Dev128Len) { RunTest(8, 128); }
+TEST_F(NcclReducerTest, Test8Dev1045991Len) { RunTest(8, 1048576); }
+
+}  // namespace tensorflow
+
+#endif
diff --git a/tensorflow/core/kernels/collective_ops.cc b/tensorflow/core/kernels/collective_ops.cc
index 82e2913b64..ea582836c2 100644
--- a/tensorflow/core/kernels/collective_ops.cc
+++ b/tensorflow/core/kernels/collective_ops.cc
@@ -87,6 +87,9 @@ class CollectiveReduceOpKernel : public CollectiveOpKernel {
                     "final_op must be one of {\"Id\", \"Div\"} but got ",
                     final_op_name));
     OP_REQUIRES_OK(c, c->GetAttr("T", &col_params_.instance.data_type));
+    OP_REQUIRES_OK(c,
+                   c->GetAttr("wait_for",
+                              &col_params_.instance.impl_details.dependencies));
 
     const NodeDef& real_node = c->def();
     col_params_.name = strings::StrCat(real_node.name(), ": Reduce(",
diff --git a/tensorflow/core/nccl/BUILD b/tensorflow/core/nccl/BUILD
index 4be33b2a0c..a19e1af888 100644
--- a/tensorflow/core/nccl/BUILD
+++ b/tensorflow/core/nccl/BUILD
@@ -20,9 +20,11 @@ cc_library(
     name = "nccl_lib",
     srcs = if_cuda([
         "nccl_manager.cc",
-        "nccl_manager.h",
         "nccl_rewrite.cc",
     ]),
+    hdrs = if_cuda([
+        "nccl_manager.h",
+    ]),
     copts = tf_copts(),
     deps = if_cuda([
         "@local_config_nccl//:nccl",
diff --git a/tensorflow/core/protobuf/worker.proto b/tensorflow/core/protobuf/worker.proto
index 74058c8465..4284dd119e 100644
--- a/tensorflow/core/protobuf/worker.proto
+++ b/tensorflow/core/protobuf/worker.proto
@@ -535,6 +535,7 @@ message CompleteInstanceRequest {
 message CompleteInstanceResponse {
   int32 instance_key = 1;
   int32 source_rank = 2;
+  bytes communicator_key = 3;
 }
 
 // Request for next agreed-upon step_id for the specified graph_keys.
-- 
GitLab


From 92e0eea7b892d7169b994a0cddca04d8170f3db0 Mon Sep 17 00:00:00 2001
From: Blake Hechtman <blakehechtman@google.com>
Date: Mon, 14 Jan 2019 16:32:04 -0800
Subject: [PATCH 0716/1095] [XLA] Recurse through copies and reshapes to find
 bitcasts. Layout assignment may create an alignment of a reshape that
 eventually introduces unnecessary copies since there are many valid
 alignments of a reshape. Also platforms now get to entirely control what is
 considered a bitcast.

PiperOrigin-RevId: 229279534
---
 .../xla/service/algebraic_simplifier.cc       |  90 ++++++------
 .../xla/service/algebraic_simplifier.h        |  36 +++--
 .../xla/service/algebraic_simplifier_test.cc  | 132 ++++++++++++------
 .../compiler/xla/service/cpu/cpu_compiler.cc  |   6 +-
 .../xla/service/gpu/nvptx_compiler.cc         |   8 +-
 5 files changed, 163 insertions(+), 109 deletions(-)

diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
index e28386d1bd..8a02c48146 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
@@ -122,23 +122,37 @@ bool TransposeIsBitcast(const HloInstruction* transpose) {
                                        transpose->dimensions());
 }
 
-// Returns true if the given reshape/copy produces a result which is bit-wise
-// identical to its operand and thus may be replaced with a bitcast.
-//
-// This function is conservative -- even if this function returns false, the
-// reshape may still be a bitcast. For example, a reshape from [28x28] to [784].
-bool ReshapeOrCopyIsBitcast(
-    const HloInstruction* instr,
-    const AlgebraicSimplifierOptions::ValidBitcastCallback&
-        valid_bitcast_callback) {
+// Recursive helper for method below.
+HloInstruction* BitcastingOperandOfReshapeOrCopyChainHelper(
+    HloInstruction* instr, HloInstruction* operand,
+    const AlgebraicSimplifierOptions& options) {
+  // Can't replace chain of copies and reshapes with bitcasts if the compiler
+  // used a memory layout which isn't compatible.
+  if (options.ReshapeIsBitcast(operand->shape(), instr->shape())) {
+    return operand;
+  }
+
+  // If the operand is a copy or reshape try to see if the operand's operand
+  // would produce a bitcast with initial instruction.
+  if (HloOpcode::kReshape == operand->opcode() ||
+      HloOpcode::kCopy == operand->opcode()) {
+    return BitcastingOperandOfReshapeOrCopyChainHelper(
+        instr, operand->mutable_operand(0), options);
+  }
+  return nullptr;
+}
+
+// Returns an operand of a chain of reshapes and copies that is bit-wise
+// identical to first reshape or copy in the chain.
+HloInstruction* BitcastingOperandOfReshapeOrCopyChain(
+    HloInstruction* instr, const AlgebraicSimplifierOptions& options) {
+  if (!options.is_layout_sensitive()) {
+    return nullptr;
+  }
   CHECK(HloOpcode::kReshape == instr->opcode() ||
         HloOpcode::kCopy == instr->opcode());
-
-  const HloInstruction* operand = instr->operand(0);
-  // Can't insert bitcasts if the compiler used a memory layout which isn't
-  // compatible.
-  return ShapeUtil::ReshapeIsBitcast(operand->shape(), instr->shape()) &&
-         valid_bitcast_callback(operand->shape(), instr->shape());
+  return BitcastingOperandOfReshapeOrCopyChainHelper(
+      instr, instr->mutable_operand(0), options);
 }
 
 bool IsUnstridedSlice(const HloInstruction* hlo) {
@@ -273,8 +287,11 @@ class AlgebraicSimplifierVisitor : public DfsHloVisitorWithDefault {
         shape, hlo, zero, {dim}, AddReduce_computation));
   }
 
-  // Convenience method for replacing an instruction with a bitcast.
-  void ReplaceWithBitcast(HloInstruction* instruction);
+  // Convenience method for replacing an instruction with a bitcast. If operand
+  // is not null, then the bitcast will use the specified operand instead of the
+  // operand of the instruction.
+  void ReplaceWithBitcast(HloInstruction* instruction,
+                          HloInstruction* operand = nullptr);
 
   // Replace old instruction with new instruction if old and new instructions
   // have the same shape. Updates uses and root instruction. Returns whether a
@@ -403,17 +420,19 @@ bool AlgebraicSimplifierVisitor::SameShape(const HloInstruction* lhs,
   }
 }
 
-void AlgebraicSimplifierVisitor::ReplaceWithBitcast(
-    HloInstruction* instruction) {
+void AlgebraicSimplifierVisitor::ReplaceWithBitcast(HloInstruction* instruction,
+                                                    HloInstruction* operand) {
   CHECK_EQ(1, instruction->operand_count());
+  if (operand == nullptr) {
+    operand = instruction->mutable_operand(0);
+  }
   CHECK_EQ(ShapeUtil::ElementsIn(instruction->shape()),
-           ShapeUtil::ElementsIn(instruction->operand(0)->shape()));
+           ShapeUtil::ElementsIn(operand->shape()));
   CHECK_EQ(ShapeUtil::ByteSizeOf(instruction->shape()),
-           ShapeUtil::ByteSizeOf(instruction->operand(0)->shape()));
+           ShapeUtil::ByteSizeOf(operand->shape()));
 
-  auto bitcast = computation_->AddInstruction(
-      HloInstruction::CreateUnary(instruction->shape(), HloOpcode::kBitcast,
-                                  instruction->mutable_operand(0)));
+  auto bitcast = computation_->AddInstruction(HloInstruction::CreateUnary(
+      instruction->shape(), HloOpcode::kBitcast, operand));
   TF_CHECK_OK(ReplaceInstruction(instruction, bitcast));
 }
 
@@ -574,9 +593,9 @@ Status AlgebraicSimplifierVisitor::HandleCopy(HloInstruction* copy) {
     return Status::OK();
   }
 
-  if (options_.is_layout_sensitive() &&
-      ReshapeOrCopyIsBitcast(copy, options_.valid_bitcast_callback())) {
-    ReplaceWithBitcast(copy);
+  if (HloInstruction* bitcast_operand =
+          BitcastingOperandOfReshapeOrCopyChain(copy, options_)) {
+    ReplaceWithBitcast(copy, bitcast_operand);
   }
 
   return Status::OK();
@@ -2196,12 +2215,10 @@ Status AlgebraicSimplifierVisitor::HandleReshape(HloInstruction* reshape) {
   }
 
   // Make this a bitcast if possible.
-  if (options_.is_layout_sensitive() &&
-      ReshapeOrCopyIsBitcast(reshape, options_.valid_bitcast_callback())) {
-    ReplaceWithBitcast(reshape);
-    return Status::OK();
+  if (HloInstruction* bitcast_operand =
+          BitcastingOperandOfReshapeOrCopyChain(reshape, options_)) {
+    ReplaceWithBitcast(reshape, bitcast_operand);
   }
-
   return Status::OK();
 }
 
@@ -3223,15 +3240,6 @@ StatusOr<bool> AlgebraicSimplifierVisitor::SimplifyConvToDot(
   const Shape dot_output_shape = ShapeUtil::MakeShapeWithDescendingLayout(
       convolution_shape.element_type(), {conv_width, output_channels});
 
-  // We cannot insert bitcasts if the layouts will not be compatible.
-  // TODO(b/33178038): Consider inserting a transpose if a bitcast would be
-  // invalid.
-  if (!options_.valid_bitcast_callback()(input_shape, new_input_shape) ||
-      !options_.valid_bitcast_callback()(filter_shape, new_filter_shape) ||
-      !options_.valid_bitcast_callback()(dot_output_shape, convolution_shape)) {
-    return false;
-  }
-
   auto new_lhs = add_bitcast(new_input_shape, lhs);
   auto new_rhs = add_bitcast(new_filter_shape, rhs);
   DotDimensionNumbers dot_dimension_numbers;
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.h b/tensorflow/compiler/xla/service/algebraic_simplifier.h
index 1acaadaaa0..ff3f638b22 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.h
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.h
@@ -25,21 +25,25 @@ namespace xla {
 
 class AlgebraicSimplifierOptions {
  public:
-  // Given shapes 'from_shape' and 'to_shape', determines if it is valid to
-  // bitcast from 'from_shape' to 'to_shape' after considering platform
-  // dependent effects on layout like alignment restrictions. Precondition: the
-  // two shapes have layouts, the same number of elements and
-  // ShapeUtil::ReshapeIsBitcast returns true.
-  using ValidBitcastCallback =
+  AlgebraicSimplifierOptions() {}
+  // Platform dependent callback to determine if a reshape `from_shape` to
+  // `to_shape` is a bitcast.
+  using ReshapeIsBitcastCallback =
       std::function<bool(const Shape& from_shape, const Shape& to_shape)>;
-
   explicit AlgebraicSimplifierOptions(
-      ValidBitcastCallback valid_bitcast_callback)
-      : valid_bitcast_callback_(std::move(valid_bitcast_callback)) {}
-  // If valid_bitcast_callback returns true, then the pass will replace reshapes
-  // and transposes with bitcasts.
-  const ValidBitcastCallback& valid_bitcast_callback() const {
-    return valid_bitcast_callback_;
+      ReshapeIsBitcastCallback reshape_is_bitcast_callback)
+      : reshape_is_bitcast_callback_(std::move(reshape_is_bitcast_callback)) {}
+
+  // Use the platform specific callback if set. It is not sensible to return
+  // true here if the options are not layout sensitive.
+  bool ReshapeIsBitcast(const Shape& from_shape, const Shape& to_shape) const {
+    if (!is_layout_sensitive_) {
+      return false;
+    }
+    if (!reshape_is_bitcast_callback_) {
+      return ShapeUtil::ReshapeIsBitcast(from_shape, to_shape);
+    }
+    return reshape_is_bitcast_callback_(from_shape, to_shape);
   }
 
   // If is_layout_sensitive is true, then the simplifier preserves layout during
@@ -47,12 +51,14 @@ class AlgebraicSimplifierOptions {
   void set_is_layout_sensitive(bool is_layout_sensitive) {
     is_layout_sensitive_ = is_layout_sensitive;
   }
+
   bool is_layout_sensitive() const { return is_layout_sensitive_; }
 
   // Enable dot simplification on platforms where it is profitable.
   void set_enable_dot_strength_reduction(bool enable_dot_strength_reduction) {
     enable_dot_strength_reduction_ = enable_dot_strength_reduction;
   }
+
   bool enable_dot_strength_reduction() const {
     return enable_dot_strength_reduction_;
   }
@@ -71,6 +77,7 @@ class AlgebraicSimplifierOptions {
       bool enable_permutation_sort_replacement) {
     enable_permutation_sort_replacement_ = enable_permutation_sort_replacement;
   }
+
   bool enable_permutation_sort_replacement() const {
     return enable_permutation_sort_replacement_;
   }
@@ -82,12 +89,13 @@ class AlgebraicSimplifierOptions {
     enable_window_reduce_to_reduce_replacement_ =
         enable_window_reduce_to_reduce_replacement;
   }
+
   bool enable_window_reduce_to_reduce_replacement() const {
     return enable_window_reduce_to_reduce_replacement_;
   }
 
  private:
-  ValidBitcastCallback valid_bitcast_callback_;
+  ReshapeIsBitcastCallback reshape_is_bitcast_callback_;
   bool is_layout_sensitive_{false};
   bool enable_dot_strength_reduction_{true};
   bool enable_conv_simplification_{true};
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
index 74a3e8cca8..d2ff0c9f21 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
@@ -46,17 +46,9 @@ namespace {
 using ::testing::ElementsAre;
 namespace m = match;
 
-AlgebraicSimplifierOptions::ValidBitcastCallback bitcasting_callback() {
-  return [](const Shape&, const Shape&) { return true; };
-}
-
-AlgebraicSimplifierOptions::ValidBitcastCallback non_bitcasting_callback() {
-  return [](const Shape&, const Shape&) { return false; };
-}
-
 class AlgebraicSimplifierTest : public HloTestBase {
  protected:
-  AlgebraicSimplifierOptions default_options_{non_bitcasting_callback()};
+  AlgebraicSimplifierOptions default_options_;
 };
 
 // Test that A + 0 is simplified to A
@@ -1464,23 +1456,77 @@ TEST_F(AlgebraicSimplifierTest, RemoveCopy) {
   EXPECT_THAT(computation->root_instruction(), param0);
 }
 
-TEST_F(AlgebraicSimplifierTest, CopyEqualsBitcast) {
+TEST_F(AlgebraicSimplifierTest, CopyOfReshapeOfCopyEqualsBitcast) {
   auto m = CreateNewVerifiedModule();
   HloComputation::Builder builder(TestName());
   HloInstruction* param =
       builder.AddInstruction(HloInstruction::CreateParameter(
-          0, ShapeUtil::MakeShape(F32, {1, 14, 14, 64}), "param"));
-  *param->mutable_shape()->mutable_layout() =
-      LayoutUtil::MakeLayout({0, 1, 2, 3});
+          0, ShapeUtil::MakeShapeWithLayout(F32, {1, 14, 14, 64}, {3, 2, 1, 0}),
+          "param"));
   HloInstruction* copy = builder.AddInstruction(HloInstruction::CreateUnary(
-      ShapeUtil::MakeShape(F32, {1, 14, 14, 64}), HloOpcode::kCopy, param));
-  *copy->mutable_shape()->mutable_layout() =
-      LayoutUtil::MakeLayout({1, 2, 0, 3});
+      ShapeUtil::MakeShapeWithLayout(F32, {1, 14, 14, 64}, {0, 1, 2, 3}),
+      HloOpcode::kCopy, param));
+  HloInstruction* reshape =
+      builder.AddInstruction(HloInstruction::CreateReshape(
+          ShapeUtil::MakeShapeWithLayout(F32, {14 * 14, 64}, {0, 1}), copy));
+  builder.AddInstruction(HloInstruction::CreateUnary(
+      ShapeUtil::MakeShapeWithLayout(F32, {14 * 14, 64}, {1, 0}),
+      HloOpcode::kCopy, reshape));
+  auto computation = m->AddEntryComputation(builder.Build());
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Copy(m::Reshape(m::Copy(m::Parameter(0))))));
+
+  AlgebraicSimplifierOptions options;
+  options.set_is_layout_sensitive(true);
+  AlgebraicSimplifier simplifier(options);
+  ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
+  // Verify that the copy of reshape of copy is replaced.
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Bitcast(m::Parameter(0))));
+}
+
+TEST_F(AlgebraicSimplifierTest, ReshapeOfCopyEqualsBitcast) {
+  auto m = CreateNewVerifiedModule();
+  HloComputation::Builder builder(TestName());
+  HloInstruction* param =
+      builder.AddInstruction(HloInstruction::CreateParameter(
+          0, ShapeUtil::MakeShapeWithLayout(F32, {1, 14, 14, 64}, {3, 2, 1, 0}),
+          "param"));
+  HloInstruction* copy = builder.AddInstruction(HloInstruction::CreateUnary(
+      ShapeUtil::MakeShapeWithLayout(F32, {1, 14, 14, 64}, {0, 1, 2, 3}),
+      HloOpcode::kCopy, param));
+  builder.AddInstruction(HloInstruction::CreateReshape(
+      ShapeUtil::MakeShapeWithLayout(F32, {14 * 14, 64}, {1, 0}), copy));
+
+  auto computation = m->AddEntryComputation(builder.Build());
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Reshape(m::Copy(m::Parameter(0)))));
+
+  AlgebraicSimplifierOptions options;
+  options.set_is_layout_sensitive(true);
+  AlgebraicSimplifier simplifier(options);
+  ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
+  // Verify that the copy of reshape of copy is replaced.
+  EXPECT_THAT(computation->root_instruction(),
+              GmockMatch(m::Bitcast(m::Parameter(0))));
+}
+
+TEST_F(AlgebraicSimplifierTest, CopyEqualsBitcast) {
+  auto m = CreateNewVerifiedModule();
+  HloComputation::Builder builder(TestName());
+  HloInstruction* param =
+      builder.AddInstruction(HloInstruction::CreateParameter(
+          0, ShapeUtil::MakeShapeWithLayout(F32, {1, 14, 14, 64}, {0, 1, 2, 3}),
+          "param"));
+  builder.AddInstruction(HloInstruction::CreateUnary(
+      ShapeUtil::MakeShapeWithLayout(F32, {1, 14, 14, 64}, {1, 2, 0, 3}),
+      HloOpcode::kCopy, param));
   auto computation = m->AddEntryComputation(builder.Build());
   EXPECT_THAT(computation->root_instruction(),
               GmockMatch(m::Copy(m::Parameter(0))));
 
-  AlgebraicSimplifierOptions options(non_bitcasting_callback());
+  AlgebraicSimplifierOptions options(
+      [](const Shape&, const Shape&) { return false; });
   options.set_is_layout_sensitive(true);
   AlgebraicSimplifier simplifier1(options);
   ASSERT_FALSE(simplifier1.Run(m.get()).ValueOrDie());
@@ -1488,10 +1534,10 @@ TEST_F(AlgebraicSimplifierTest, CopyEqualsBitcast) {
   EXPECT_THAT(computation->root_instruction(),
               GmockMatch(m::Copy(m::Parameter(0))));
 
-  AlgebraicSimplifierOptions options2(bitcasting_callback());
+  AlgebraicSimplifierOptions options2;
   options2.set_is_layout_sensitive(true);
   AlgebraicSimplifier simplifier2(options2);
-  ASSERT_TRUE(simplifier2.Run(m.get()).ValueOrDie());
+  EXPECT_TRUE(simplifier2.Run(m.get()).ValueOrDie());
   // Verify that the copy is replaced.
   EXPECT_THAT(computation->root_instruction(),
               GmockMatch(m::Bitcast(m::Parameter(0))));
@@ -1744,7 +1790,7 @@ TEST_F(AlgebraicSimplifierTest, CopyWithDifferentLayout) {
   EXPECT_THAT(computation->root_instruction(),
               GmockMatch(m::Copy(m::Parameter(0))));
 
-  AlgebraicSimplifierOptions options(non_bitcasting_callback());
+  AlgebraicSimplifierOptions options;
   options.set_is_layout_sensitive(true);
   AlgebraicSimplifier simplifier(options);
   EXPECT_FALSE(simplifier.Run(m.get()).ValueOrDie());
@@ -1774,7 +1820,7 @@ TEST_F(AlgebraicSimplifierTest, CopyWithSameLayout) {
   EXPECT_THAT(computation->root_instruction(),
               GmockMatch(m::Copy(m::Parameter(0))));
 
-  AlgebraicSimplifierOptions options(non_bitcasting_callback());
+  AlgebraicSimplifierOptions options;
   options.set_is_layout_sensitive(true);
   AlgebraicSimplifier simplifier(options);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
@@ -1804,7 +1850,8 @@ TEST_F(AlgebraicSimplifierTest, NoBitcastAdded) {
   EXPECT_THAT(computation->root_instruction(),
               GmockMatch(m::Reshape(m::Parameter(0))));
 
-  AlgebraicSimplifierOptions options(non_bitcasting_callback());
+  AlgebraicSimplifierOptions options(
+      [](const Shape&, const Shape&) { return false; });
   options.set_is_layout_sensitive(true);
   AlgebraicSimplifier simplifier(options);
   EXPECT_FALSE(simplifier.Run(m.get()).ValueOrDie());
@@ -1835,8 +1882,7 @@ TEST_F(AlgebraicSimplifierTest, ReshapeOfTransposeOfRngToRng) {
 
   auto computation = m->AddEntryComputation(builder.Build());
 
-  AlgebraicSimplifier simplifier(
-      (AlgebraicSimplifierOptions(bitcasting_callback())));
+  AlgebraicSimplifier simplifier(AlgebraicSimplifierOptions{});
   EXPECT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 
   // Verify that reshape(transpose(rng)) is replace by a single rng of the
@@ -1887,7 +1933,7 @@ TEST_F(AlgebraicSimplifierTest, ReshapeReplacedWithBitcast) {
                                   m::Op().Is(dimensions_wrong_reshape),
                                   m::Op().Is(layout_wrong_reshape))));
 
-  AlgebraicSimplifierOptions options(bitcasting_callback());
+  AlgebraicSimplifierOptions options;
   options.set_is_layout_sensitive(true);
   AlgebraicSimplifier simplifier(options);
   simplifier.Run(m.get()).ValueOrDie();
@@ -1917,8 +1963,7 @@ TEST_F(AlgebraicSimplifierTest, FailureToSinkReshapeDoesntAffectChangedBit) {
   builder.AddInstruction(
       HloInstruction::CreateReshape(ShapeUtil::MakeShape(F32, {4}), add));
 
-  AlgebraicSimplifier simplifier(
-      (AlgebraicSimplifierOptions(bitcasting_callback())));
+  AlgebraicSimplifier simplifier(AlgebraicSimplifierOptions{});
   m->AddEntryComputation(builder.Build());
   EXPECT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 }
@@ -1942,8 +1987,7 @@ TEST_F(AlgebraicSimplifierTest, FailureToSinkBroadcastDoesntAffectChangedBit) {
       HloInstruction::CreateBroadcast(ShapeUtil::MakeShape(F32, {2, 2, 2}), add,
                                       /*broadcast_dimensions=*/{0, 1}));
 
-  AlgebraicSimplifier simplifier(
-      (AlgebraicSimplifierOptions(bitcasting_callback())));
+  AlgebraicSimplifier simplifier(AlgebraicSimplifierOptions{});
   m->AddEntryComputation(builder.Build());
   EXPECT_TRUE(simplifier.Run(m.get()).ValueOrDie());
 }
@@ -1968,7 +2012,7 @@ TEST_F(AlgebraicSimplifierTest, TransposeEqualsBitcast1) {
   EXPECT_THAT(computation->root_instruction(),
               GmockMatch(m::Transpose(m::Parameter(0))));
 
-  AlgebraicSimplifierOptions options(bitcasting_callback());
+  AlgebraicSimplifierOptions options;
   options.set_is_layout_sensitive(true);
   AlgebraicSimplifier simplifier(options);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
@@ -1998,7 +2042,7 @@ TEST_F(AlgebraicSimplifierTest, TransposeEqualsBitcast2) {
   EXPECT_THAT(computation->root_instruction(),
               GmockMatch(m::Transpose(m::Parameter(0))));
 
-  AlgebraicSimplifierOptions options(bitcasting_callback());
+  AlgebraicSimplifierOptions options;
   options.set_is_layout_sensitive(true);
   AlgebraicSimplifier simplifier(options);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
@@ -2055,7 +2099,7 @@ TEST_F(AlgebraicSimplifierTest, CopiesMerged) {
   EXPECT_THAT(computation->root_instruction(),
               GmockMatch(m::Copy(m::Copy(m::Parameter(0)))));
 
-  AlgebraicSimplifierOptions options(non_bitcasting_callback());
+  AlgebraicSimplifierOptions options;
   options.set_is_layout_sensitive(true);
   AlgebraicSimplifier simplifier(options);
   ASSERT_TRUE(simplifier.Run(m.get()).ValueOrDie());
@@ -2651,7 +2695,7 @@ TEST_F(AlgebraicSimplifierTest, ReplacePermutationSortWithScatter) {
   TF_ASSERT_OK_AND_ASSIGN(auto module,
                           ParseAndReturnVerifiedModule(hlo_string));
 
-  AlgebraicSimplifierOptions options(non_bitcasting_callback());
+  AlgebraicSimplifierOptions options;
   options.set_enable_permutation_sort_replacement(true);
   AlgebraicSimplifier simplifier(options);
   EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
@@ -2680,7 +2724,7 @@ TEST_F(AlgebraicSimplifierTest, DontReplacePermutationSortIfNonIntegral) {
   TF_ASSERT_OK_AND_ASSIGN(auto module,
                           ParseAndReturnVerifiedModule(hlo_string));
 
-  AlgebraicSimplifierOptions options(non_bitcasting_callback());
+  AlgebraicSimplifierOptions options;
   options.set_enable_permutation_sort_replacement(true);
   AlgebraicSimplifier simplifier(options);
   EXPECT_FALSE(simplifier.Run(module.get()).ValueOrDie());
@@ -2703,7 +2747,7 @@ TEST_F(AlgebraicSimplifierTest, DontReplacePermutationSortWrongDimensions) {
   TF_ASSERT_OK_AND_ASSIGN(auto module,
                           ParseAndReturnVerifiedModule(hlo_string));
 
-  AlgebraicSimplifierOptions options(non_bitcasting_callback());
+  AlgebraicSimplifierOptions options;
   options.set_enable_permutation_sort_replacement(true);
   AlgebraicSimplifier simplifier(options);
   EXPECT_FALSE(simplifier.Run(module.get()).ValueOrDie());
@@ -3292,7 +3336,7 @@ TEST_F(AlgebraicSimplifierTest, ConvertConvToMatmul) {
     auto module = CreateNewUnverifiedModule();
     auto* computation = module->AddEntryComputation(b.Build());
 
-    AlgebraicSimplifierOptions simplifier_options(bitcasting_callback());
+    AlgebraicSimplifierOptions simplifier_options;
     simplifier_options.set_is_layout_sensitive(true);
     AlgebraicSimplifier simplifier(simplifier_options);
     if (!simplifier.Run(module.get()).ValueOrDie()) {
@@ -3869,7 +3913,7 @@ TEST_F(AlgebraicSimplifierTest, SliceOfPadLow) {
   TF_ASSERT_OK_AND_ASSIGN(auto module,
                           ParseAndReturnVerifiedModule(hlo_string));
 
-  AlgebraicSimplifierOptions options(bitcasting_callback());
+  AlgebraicSimplifierOptions options;
   AlgebraicSimplifier simplifier(options);
   EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
   auto root = module->entry_computation()->root_instruction();
@@ -3890,7 +3934,7 @@ TEST_F(AlgebraicSimplifierTest, SliceOfPadHigh) {
   TF_ASSERT_OK_AND_ASSIGN(auto module,
                           ParseAndReturnVerifiedModule(hlo_string));
 
-  AlgebraicSimplifierOptions options(bitcasting_callback());
+  AlgebraicSimplifierOptions options;
   AlgebraicSimplifier simplifier(options);
   EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
   auto root = module->entry_computation()->root_instruction();
@@ -3911,7 +3955,7 @@ TEST_F(AlgebraicSimplifierTest, SliceOfPadMidNonScalar) {
   TF_ASSERT_OK_AND_ASSIGN(auto module,
                           ParseAndReturnVerifiedModule(hlo_string));
 
-  AlgebraicSimplifierOptions options(bitcasting_callback());
+  AlgebraicSimplifierOptions options;
   AlgebraicSimplifier simplifier(options);
   EXPECT_FALSE(simplifier.Run(module.get()).ValueOrDie());
 }
@@ -3930,7 +3974,7 @@ TEST_F(AlgebraicSimplifierTest, SliceOfPadMidScalar) {
   TF_ASSERT_OK_AND_ASSIGN(auto module,
                           ParseAndReturnVerifiedModule(hlo_string));
 
-  AlgebraicSimplifierOptions options(bitcasting_callback());
+  AlgebraicSimplifierOptions options;
   AlgebraicSimplifier simplifier(options);
   EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
   auto root = module->entry_computation()->root_instruction();
@@ -3952,7 +3996,7 @@ TEST_F(AlgebraicSimplifierTest, SliceOfConcatScalarInput) {
   TF_ASSERT_OK_AND_ASSIGN(auto module,
                           ParseAndReturnVerifiedModule(hlo_string));
 
-  AlgebraicSimplifierOptions options(bitcasting_callback());
+  AlgebraicSimplifierOptions options;
   AlgebraicSimplifier simplifier(options);
   EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
   auto root = module->entry_computation()->root_instruction();
@@ -3974,7 +4018,7 @@ TEST_F(AlgebraicSimplifierTest, SliceOfConcatNonScalarInput) {
   TF_ASSERT_OK_AND_ASSIGN(auto module,
                           ParseAndReturnVerifiedModule(hlo_string));
 
-  AlgebraicSimplifierOptions options(bitcasting_callback());
+  AlgebraicSimplifierOptions options;
   AlgebraicSimplifier simplifier(options);
   EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
   auto root = module->entry_computation()->root_instruction();
@@ -3996,7 +4040,7 @@ TEST_F(AlgebraicSimplifierTest, NegateNegate) {
   TF_ASSERT_OK_AND_ASSIGN(auto module,
                           ParseAndReturnVerifiedModule(hlo_string));
 
-  AlgebraicSimplifierOptions options(bitcasting_callback());
+  AlgebraicSimplifierOptions options;
   AlgebraicSimplifier simplifier(options);
   EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
   auto root = module->entry_computation()->root_instruction();
@@ -4016,7 +4060,7 @@ TEST_F(AlgebraicSimplifierTest, NotNot) {
   TF_ASSERT_OK_AND_ASSIGN(auto module,
                           ParseAndReturnVerifiedModule(hlo_string));
 
-  AlgebraicSimplifierOptions options(bitcasting_callback());
+  AlgebraicSimplifierOptions options;
   AlgebraicSimplifier simplifier(options);
   EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
   auto root = module->entry_computation()->root_instruction();
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
index 34a4f008ec..eafda68510 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
@@ -282,8 +282,7 @@ Status CpuCompiler::RunHloPassesThroughLayoutAssn(
         /*rewrite_inference_op=*/true,
         /*rewrite_grad_op=*/true);
     pipeline.AddPass<HloGetDimensionSizeRewriter>();
-    AlgebraicSimplifierOptions options(
-        [](const Shape&, const Shape&) { return false; });
+    AlgebraicSimplifierOptions options;
     options.set_enable_dot_strength_reduction(false);
     pass.AddPass<AlgebraicSimplifier>(options);
     pass.AddPass<SortSimplifier>();
@@ -350,8 +349,7 @@ Status CpuCompiler::RunHloPassesAfterLayoutAssn(
     pass.AddInvariantChecker<HloVerifier>(
         /*layout_sensitive=*/true,
         /*allow_mixed_precision=*/false);
-    AlgebraicSimplifierOptions options(
-        [](const Shape&, const Shape&) { return true; });
+    AlgebraicSimplifierOptions options;
     options.set_is_layout_sensitive(true);
     options.set_enable_dot_strength_reduction(false);
     pass.AddPass<HloPassFix<AlgebraicSimplifier>>(options);
diff --git a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
index 70485c62a9..48f718b514 100644
--- a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
+++ b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
@@ -199,8 +199,7 @@ Status OptimizeHloModule(HloModule* hlo_module, se::StreamExecutor* stream_exec,
       // elimination has to come after that pass.
       pipeline.AddPass<ZeroSizedHloElimination>();
 
-      AlgebraicSimplifierOptions options(
-          [](const Shape&, const Shape&) { return false; });
+      AlgebraicSimplifierOptions options;
       options.set_enable_permutation_sort_replacement(true);
       pass.AddPass<AlgebraicSimplifier>(options);
       pass.AddPass<SortSimplifier>();
@@ -272,10 +271,7 @@ Status OptimizeHloModule(HloModule* hlo_module, se::StreamExecutor* stream_exec,
 
     // The LayoutAssignment pass may leave behind kCopy instructions which are
     // duplicate or NOPs, so remove them with algebraic simplification and CSE.
-    AlgebraicSimplifierOptions options(
-        /*valid_bitcast_callback=*/[](const Shape&, const Shape&) {
-          return true;
-        });
+    AlgebraicSimplifierOptions options;
     options.set_is_layout_sensitive(true);
     options.set_enable_permutation_sort_replacement(true);
     pipeline.AddPass<HloPassFix<AlgebraicSimplifier>>(options);
-- 
GitLab


From 3c922d7df747ce3c25a0ad75a41f23c7e8d1df1e Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Mon, 14 Jan 2019 16:50:14 -0800
Subject: [PATCH 0717/1095] Print out bounded shape in HumanStringWithLayout

PiperOrigin-RevId: 229282536
---
 tensorflow/compiler/xla/shape_util.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/shape_util.cc b/tensorflow/compiler/xla/shape_util.cc
index 9821790cb3..f1ef202da0 100644
--- a/tensorflow/compiler/xla/shape_util.cc
+++ b/tensorflow/compiler/xla/shape_util.cc
@@ -530,7 +530,8 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
   string result = StrCat(
       primitive_util::LowercasePrimitiveTypeName(shape.element_type()), "[");
   for (int i = 0; i < shape.dimensions().size(); i++) {
-    StrAppend(&result, (i > 0) ? "," : "", shape.dimensions(i));
+    StrAppend(&result, (i > 0) ? "," : "",
+              shape.is_dynamic_dimension(i) ? "<=" : "", shape.dimensions(i));
   }
   result += "]";
   if (!IsScalar(shape) && shape.IsArray()) {
-- 
GitLab


From b0cc10d27608219e44d36f04c97a5aeff9b79832 Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Mon, 14 Jan 2019 17:10:11 -0800
Subject: [PATCH 0718/1095] Set the data type of the IConstantLayer output
 correctly.

PiperOrigin-RevId: 229285591
---
 .../contrib/tensorrt/convert/convert_nodes.cc | 49 ++++++++++---------
 .../contrib/tensorrt/convert/convert_nodes.h  |  5 ++
 .../tensorrt/convert/convert_nodes_test.cc    | 12 +++++
 3 files changed, 44 insertions(+), 22 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index dcb84dfb93..a1c519d908 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -334,6 +334,20 @@ Status Converter::GetTrtBroadcastShape(
   return Status::OK();
 }
 
+nvinfer1::ITensor* Converter::CreateConstantLayer(
+    const TRT_ShapedWeights& weights, const nvinfer1::Dims& dims) {
+  nvinfer1::Weights trt_weights = weights.GetTrtWeights();
+  nvinfer1::IConstantLayer* layer = network()->addConstant(dims, trt_weights);
+  if (!layer) return nullptr;
+  const nvinfer1::DataType trt_dtype = trt_weights.type;
+  // NOTE(laigd): calling layer->setOutputType() does not work. This may be a
+  // bug and we should report to NVIDIA.
+  // layer->setOutputType(0, trt_dtype);
+  nvinfer1::ITensor* trt_tensor = layer->getOutput(0);
+  trt_tensor->setType(trt_dtype);
+  return trt_tensor;
+}
+
 inline bool DimsEqual(const nvinfer1::Dims& dim_l,
                       const nvinfer1::Dims& dim_r) {
   if (dim_l.nbDims != dim_r.nbDims) {
@@ -1104,10 +1118,8 @@ Status Converter::PrepareTensorForShape(const TRT_TensorOrWeights& input,
       *tensor = layer->getOutput(0);
     }
   } else {
-    nvinfer1::IConstantLayer* layer =
-        this->network()->addConstant(dims, input.weights().GetTrtWeights());
-    TFTRT_RETURN_ERROR_IF_NULLPTR(layer, "TF-TRT Internal Reshape");
-    *tensor = layer->getOutput(0);
+    *tensor = CreateConstantLayer(input.weights(), dims);
+    TFTRT_RETURN_ERROR_IF_NULLPTR(*tensor, "TF-TRT Internal Reshape");
     if (precision_mode() == INT8MODE && !use_calibration()) {
       // If we are in int8 mode and not calibrating, we need to explicitly set a
       // quantization range for the output tensor of the IConstantLayer. Here we
@@ -2570,22 +2582,18 @@ tensorflow::Status ConvertRelu6(OpConverterParams* params) {
   auto weights_ptr =
       static_cast<float*>(const_cast<void*>(weights.GetValues()));
   weights_ptr[0] = 6.0f;
-  nvinfer1::IConstantLayer* const6_layer =
-      params->converter->network()->addConstant(dims, weights.GetTrtWeights());
-  TFTRT_RETURN_ERROR_IF_NULLPTR(const6_layer, node_def.name());
-  params->converter->ProvideQuantizationRange(const6_layer->getOutput(0), 0.0f,
-                                              6.0f);
+  nvinfer1::ITensor* const6_tensor =
+      params->converter->CreateConstantLayer(weights, dims);
+  TFTRT_RETURN_ERROR_IF_NULLPTR(const6_tensor, node_def.name());
+  params->converter->ProvideQuantizationRange(const6_tensor, 0.0f, 6.0f);
 
   // ElementWise Min Operation
   // Min op is a nop for INT8 execution path, as the input tensor
   // to this layer will only have values in range [0.f, 6.0f].
-  const nvinfer1::ITensor* tensor_l = relu_layer->getOutput(0);
-  const nvinfer1::ITensor* tensor_r = const6_layer->getOutput(0);
   nvinfer1::IElementWiseLayer* relu6_layer =
       params->converter->network()->addElementWise(
-          *const_cast<nvinfer1::ITensor*>(tensor_l),
-          *const_cast<nvinfer1::ITensor*>(tensor_r),
-          nvinfer1::ElementWiseOperation::kMIN);
+          *const_cast<nvinfer1::ITensor*>(relu_layer->getOutput(0)),
+          *const6_tensor, nvinfer1::ElementWiseOperation::kMIN);
   TFTRT_RETURN_ERROR_IF_NULLPTR(relu6_layer, node_def.name());
   nvinfer1::ITensor* output_tensor = relu6_layer->getOutput(0);
   params->converter->ProvideQuantizationRange(output_tensor, 0.0f, 6.0f);
@@ -2989,18 +2997,15 @@ tensorflow::Status ConvertSquare(OpConverterParams* params) {
   auto weights_ptr =
       static_cast<float*>(const_cast<void*>(weights.GetValues()));
   weights_ptr[0] = 2.f;
-  nvinfer1::IConstantLayer* const2_layer =
-      params->converter->network()->addConstant(dims, weights.GetTrtWeights());
-  TFTRT_RETURN_ERROR_IF_NULLPTR(const2_layer, node_def.name());
+  nvinfer1::ITensor* const2_tensor =
+      params->converter->CreateConstantLayer(weights, dims);
+  TFTRT_RETURN_ERROR_IF_NULLPTR(const2_tensor, node_def.name());
 
   // ElementWise Pow Operation
-  const nvinfer1::ITensor* tensor_l = inputs.at(0).tensor();
-  const nvinfer1::ITensor* tensor_r = const2_layer->getOutput(0);
   nvinfer1::IElementWiseLayer* layer =
       params->converter->network()->addElementWise(
-          *const_cast<nvinfer1::ITensor*>(tensor_l),
-          *const_cast<nvinfer1::ITensor*>(tensor_r),
-          nvinfer1::ElementWiseOperation::kPOW);
+          *const_cast<nvinfer1::ITensor*>(inputs.at(0).tensor()),
+          *const2_tensor, nvinfer1::ElementWiseOperation::kPOW);
   TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name());
   nvinfer1::ITensor* output_tensor = layer->getOutput(0);
 
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.h b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
index 8f2271ee3f..25949b7f07 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.h
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
@@ -468,6 +468,11 @@ class Converter {
                               nvinfer1::Dims* operand_l_new_dims,
                               nvinfer1::Dims* operand_r_new_dims) const;
 
+  // Creates an IConstantLayer using 'weights' whose dimensions are specified by
+  // 'dims', and returns the output ITensor.
+  nvinfer1::ITensor* CreateConstantLayer(const TRT_ShapedWeights& weights,
+                                         const nvinfer1::Dims& dims);
+
  private:
   // Verify the provided batch_size is consistent with batch_size_ and update it
   // if necessary.
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
index f143f56d20..36dbbe3f6f 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
@@ -915,6 +915,18 @@ TEST_F(ConverterTest, GetTrtBroadcastShape) {
                  "(tensor #dims 4 vs broadcast #dims 5)");
 }
 
+TEST_F(ConverterTest, CreateConstantLayer) {
+  for (auto dtype : {DT_FLOAT, DT_INT32}) {
+    TRT_ShapedWeights weights =
+        weight_store_->GetTempWeights(dtype, GetTestDims({2, 3, 5}));
+    nvinfer1::ITensor* tensor =
+        converter_->CreateConstantLayer(weights, GetTestDims({3, 10}));
+    ASSERT_NE(nullptr, tensor);
+    ExpectTrtDimsEqualsArray({3, 10}, tensor->getDimensions());
+    EXPECT_EQ(TfDataTypeToTrt(dtype), tensor->getType());
+  }
+}
+
 // Class to test various op converters, using both a TrtNodeValidator and
 // Converter.
 class OpConverterTest : public ::testing::Test {
-- 
GitLab


From 1f1a4a0a5eb71517b08ebb5e670c1400c944c996 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Mon, 14 Jan 2019 17:32:18 -0800
Subject: [PATCH 0719/1095] Add a revived type for auto-generated list wrappers

Adds a bit of plumbing for the proto and adds a bit of flexibility in _ListWrapper so that the restore process doesn't make it unsaveable (previously any __setitem__ would mark the list as dirty).

PiperOrigin-RevId: 229288388
---
 tensorflow/python/saved_model/load_test.py    | 11 +++++++
 .../checkpointable/data_structures.py         | 29 +++++++++++++++++--
 2 files changed, 37 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index ca9e16d14a..ff6b72053b 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -428,6 +428,17 @@ class LoadTest(test.TestCase):
     self.assertEqual(2., imported.variables["b"].numpy())
     self.assertEqual(set(["a", "b"]), set(imported.variables.keys()))
 
+  def test_list(self):
+    root = tracking.Checkpointable()
+    root.variables = [variables.Variable(1.)]
+    root.variables.append(1)
+    root.variables.append(variables.Variable(3.))
+    imported = self.cycle(root)
+    self.assertEqual(1., imported.variables[0].numpy())
+    self.assertEqual(3., imported.variables[2].numpy())
+    self.assertIs(None, imported.variables[1])
+    self.assertEqual(3, len(imported.variables))
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/training/checkpointable/data_structures.py b/tensorflow/python/training/checkpointable/data_structures.py
index c46af68155..759709aa3d 100644
--- a/tensorflow/python/training/checkpointable/data_structures.py
+++ b/tensorflow/python/training/checkpointable/data_structures.py
@@ -406,8 +406,11 @@ class _ListWrapper(List, collections.MutableSequence,
     del self._storage[key]
 
   def __setitem__(self, key, value):
-    self._non_append_mutation = True
-    self._storage[key] = value
+    self._check_external_modification()
+    if isinstance(self._storage[key], base.CheckpointableBase):
+      self._non_append_mutation = True
+    self._storage[key] = self._track_value(value, self._name_element(key))
+    self._update_snapshot()
 
   def append(self, value):
     """Add a new checkpointable value."""
@@ -734,8 +737,28 @@ revived_types.register_revived_type(
     "checkpointable_dict_wrapper",
     lambda obj: isinstance(obj, _DictWrapper),
     versions=[revived_types.VersionedTypeRegistration(
-        object_factory=lambda _: _DictWrapper({}),
+        # Standard dependencies are enough to reconstruct the checkpointable
+        # items in dictionaries, so we don't need to save any extra information.
+        object_factory=lambda proto: _DictWrapper({}),
         version=1,
         min_producer_version=1,
         min_consumer_version=1,
         setter=operator.setitem)])
+
+
+def _set_list_item(list_object, index_string, value):
+  item_index = int(index_string)
+  if len(list_object) <= item_index:
+    list_object.extend([None] * (1 + item_index - len(list_object)))
+  list_object[item_index] = value
+
+
+revived_types.register_revived_type(
+    "checkpointable_list_wrapper",
+    lambda obj: isinstance(obj, _ListWrapper),
+    versions=[revived_types.VersionedTypeRegistration(
+        object_factory=lambda proto: _ListWrapper([]),
+        version=1,
+        min_producer_version=1,
+        min_consumer_version=1,
+        setter=_set_list_item)])
-- 
GitLab


From 5e347da46abef1ef77356a7b85e2c55ceb44326e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 14 Jan 2019 17:33:29 -0800
Subject: [PATCH 0720/1095] Use `str_util::EndsWith` to find suffixes in
 latency_all_edges.cc.

PiperOrigin-RevId: 229288546
---
 .../core/grappler/optimizers/data/latency_all_edges.cc | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/data/latency_all_edges.cc b/tensorflow/core/grappler/optimizers/data/latency_all_edges.cc
index ff0fdbc131..0b7b2eba86 100644
--- a/tensorflow/core/grappler/optimizers/data/latency_all_edges.cc
+++ b/tensorflow/core/grappler/optimizers/data/latency_all_edges.cc
@@ -24,6 +24,7 @@ limitations under the License.
 #include "tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.h"
 #include "tensorflow/core/grappler/optimizers/data/graph_utils.h"
 #include "tensorflow/core/grappler/utils.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/protobuf.h"
 
@@ -72,10 +73,8 @@ Status LatencyAllEdges::Optimize(Cluster* cluster, const GrapplerItem& item,
   // TODO(shivaniagrawal): Add Op to return Latency for the particular Op than
   // for the edge (e2 - e1?).
   for (const NodeDef& node : item.graph.node()) {
-    if (node.op().rfind("Dataset") != node.op().size() - strlen("Dataset") ||
-        node.attr().empty() ||
-        node.name().rfind("_generated") ==
-            node.name().size() - strlen("_generated")) {
+    if (!str_util::EndsWith(node.op(), "Dataset") || node.attr().empty() ||
+        str_util::EndsWith(node.name(), "_generated")) {
       // TODO(b/111805951): Replace this with non-approximate way to check if
       // node corresponds to a `Dataset` op.
       continue;
@@ -89,8 +88,7 @@ Status LatencyAllEdges::Optimize(Cluster* cluster, const GrapplerItem& item,
     } else {  // fanout will have size 0 for last dataset node in the pipeline.
       if (fanout.size() == 1) {
         NodeDef* output_node = (*(fanout.begin())).node;
-        if (output_node->name().rfind("_generated") ==
-            output_node->name().size() - strlen("_generated")) {
+        if (str_util::EndsWith(output_node->name(), "_generated")) {
           continue;
         }
       }
-- 
GitLab


From 19f14821f261b290d4dc85a50f33900274434ed4 Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Mon, 14 Jan 2019 17:36:26 -0800
Subject: [PATCH 0721/1095] Fix several TFLite tests

PiperOrigin-RevId: 229288943
---
 tensorflow/lite/delegates/nnapi/nnapi_delegate.cc        | 1 +
 .../lite/experimental/micro/examples/micro_speech/BUILD  | 3 +++
 tensorflow/lite/models/smartreply/BUILD                  | 1 -
 tensorflow/lite/profiling/BUILD                          | 5 +++--
 tensorflow/lite/python/convert_test.py                   | 3 ++-
 tensorflow/lite/testing/nnapi_example.cc                 | 9 ++++++---
 6 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
index 8af159e6fb..94fcc8740b 100644
--- a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
+++ b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
@@ -13,6 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 #include <cstdarg>
+#include <cstring>
 #include <iostream>
 #include <memory>
 #include <vector>
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/BUILD b/tensorflow/lite/experimental/micro/examples/micro_speech/BUILD
index 702b893e6f..171d07cf56 100644
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/BUILD
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/BUILD
@@ -216,6 +216,9 @@ tflite_micro_cc_test(
     srcs = [
         "recognize_commands_test.cc",
     ],
+    tags = [
+        "no_oss",  # TODO(122853023): Resolve issues and re-enable.
+    ],
     deps = [
         ":recognize_commands",
         "//tensorflow/lite/c:c_api_internal",
diff --git a/tensorflow/lite/models/smartreply/BUILD b/tensorflow/lite/models/smartreply/BUILD
index fccbf79eb9..9b2f0da64c 100644
--- a/tensorflow/lite/models/smartreply/BUILD
+++ b/tensorflow/lite/models/smartreply/BUILD
@@ -50,7 +50,6 @@ cc_library(
     ],
 )
 
-# TODO(b/118895218): Make this test compatible with oss.
 tf_cc_test(
     name = "predictor_test",
     srcs = ["predictor_test.cc"],
diff --git a/tensorflow/lite/profiling/BUILD b/tensorflow/lite/profiling/BUILD
index 89ad726782..64723ba385 100644
--- a/tensorflow/lite/profiling/BUILD
+++ b/tensorflow/lite/profiling/BUILD
@@ -2,6 +2,7 @@ package(default_visibility = ["//visibility:public"])
 
 licenses(["notice"])  # Apache 2.0
 
+load("//tensorflow:tensorflow.bzl", "tf_cc_test")
 load("//tensorflow/lite:build_def.bzl", "tflite_copts")
 
 common_copts = [
@@ -65,10 +66,10 @@ cc_library(
     ],
 )
 
-cc_test(
+tf_cc_test(
     name = "profile_summarizer_test",
     srcs = ["profile_summarizer_test.cc"],
-    copts = common_copts,
+    extra_copts = common_copts,
     deps = [
         ":profile_summarizer",
         "//tensorflow/lite:framework",
diff --git a/tensorflow/lite/python/convert_test.py b/tensorflow/lite/python/convert_test.py
index a29d431322..e270abaa5a 100644
--- a/tensorflow/lite/python/convert_test.py
+++ b/tensorflow/lite/python/convert_test.py
@@ -410,7 +410,8 @@ class ConvertTestOpHint(test_util.TensorFlowTestCase):
           _node_name(output_1.name),
           _node_name(output_2.name)
       ]
-      self.assertCountEqual(hinted_outputs_nodes, expected_hinted_output_nodes)
+      self.assertEqual(
+          len(hinted_outputs_nodes), len(expected_hinted_output_nodes))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/lite/testing/nnapi_example.cc b/tensorflow/lite/testing/nnapi_example.cc
index 22df8dbd88..0ff1fbcd41 100644
--- a/tensorflow/lite/testing/nnapi_example.cc
+++ b/tensorflow/lite/testing/nnapi_example.cc
@@ -25,11 +25,14 @@ limitations under the License.
 #include <fstream>
 #include <iostream>
 #include <sstream>
+#include <string>
 #include "tensorflow/lite/nnapi/NeuralNetworksShim.h"
 #include "tensorflow/lite/testing/parse_testdata.h"
 #include "tensorflow/lite/testing/tflite_driver.h"
 
-string dirname(const string& s) { return s.substr(0, s.find_last_of("/")); }
+std::string dirname(const std::string& s) {
+  return s.substr(0, s.find_last_of("/"));
+}
 
 bool Interpret(const char* examples_filename, bool use_nnapi) {
   std::ifstream tflite_stream(examples_filename);
@@ -65,14 +68,14 @@ int main(int argc, char* argv[]) {
     return 1;
   }
 
-  string base_dir = dirname(argv[1]);
+  std::string base_dir = dirname(argv[1]);
   DIR* dir = opendir(base_dir.c_str());
   if (dir == nullptr) {
     fprintf(stderr, "Can't open dir %s\n", base_dir.c_str());
     return 1;
   }
   while (struct dirent* ent = readdir(dir)) {
-    string name = ent->d_name;
+    std::string name = ent->d_name;
     if (name.rfind(".txt") == name.length() - 4) {
       printf("%s: ", name.c_str());
       if (Interpret((base_dir + "/" + name).c_str(), use_nnapi)) {
-- 
GitLab


From dadbcf8c205ec948938e28e21c4a5dd7ebdb672e Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Mon, 14 Jan 2019 17:46:16 -0800
Subject: [PATCH 0722/1095] [XLA] Fix special cases of complex power operation.

(0 + 0j)^x for real x > 0 is zero, not NaN.

PiperOrigin-RevId: 229290163
---
 .../xla/service/elemental_ir_emitter.cc       | 11 +++++++++-
 .../xla/service/hlo_evaluator_typed_visitor.h | 13 +++++++-----
 .../xla/service/llvm_ir/ir_builder_mixin.h    |  5 +++++
 .../xla/tests/array_elementwise_ops_test.cc   | 21 +++++++++++++++++++
 4 files changed, 44 insertions(+), 6 deletions(-)

diff --git a/tensorflow/compiler/xla/service/elemental_ir_emitter.cc b/tensorflow/compiler/xla/service/elemental_ir_emitter.cc
index f84c115e0a..727e0bfa52 100644
--- a/tensorflow/compiler/xla/service/elemental_ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/elemental_ir_emitter.cc
@@ -812,11 +812,14 @@ StatusOr<llvm::Value*> ElementalIrEmitter::EmitComplexBinaryOp(
       auto c = EmitExtractReal(rhs_value);
       auto d = EmitExtractImag(rhs_value);
       auto aa_p_bb = FAdd(FMul(a, a), FMul(b, b));
+      auto zero = llvm::ConstantFP::get(a->getType(), 0);
       auto one_half = llvm::ConstantFP::get(a->getType(), 0.5);
+      auto one = llvm::ConstantFP::get(a->getType(), 1);
       auto half_c = FMul(one_half, c);
 
       TF_ASSIGN_OR_RETURN(auto aa_p_bb_to_half_c,
                           EmitPow(component_type, aa_p_bb, half_c));
+
       auto neg_d = FNeg(d);
       TF_ASSIGN_OR_RETURN(auto arg_lhs, EmitAtan2(component_type, b, a));
       auto neg_d_arg_lhs = FMul(neg_d, arg_lhs);
@@ -828,7 +831,13 @@ StatusOr<llvm::Value*> ElementalIrEmitter::EmitComplexBinaryOp(
       auto q = FAdd(FMul(c, arg_lhs), FMul(half_d, ln_aa_p_bb));
       TF_ASSIGN_OR_RETURN(auto cos_q, EmitCos(component_type, q));
       TF_ASSIGN_OR_RETURN(auto sin_q, EmitSin(component_type, q));
-      return EmitComposeComplex(op, FMul(coeff, cos_q), FMul(coeff, sin_q));
+      // 0^c is 0 if d is 0 and c > 0. 0^0 is defined to be 1.0, see
+      // Branch Cuts for Complex Elementary Functions or Much Ado About
+      // Nothing's Sign Bit, W. Kahan, Section 10.
+      return Select(
+          And(And(FCmpOEQ(aa_p_bb, zero), FCmpOEQ(d, zero)), FCmpOLE(zero, c)),
+          EmitComposeComplex(op, Select(FCmpOEQ(zero, c), one, zero), zero),
+          EmitComposeComplex(op, FMul(coeff, cos_q), FMul(coeff, sin_q)));
     }
     default:
       return Unimplemented("binary complex op '%s'",
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
index 5245c563ec..95a78408b0 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
@@ -675,11 +675,14 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
   }
 
   Status HandlePower(HloInstruction* power) override {
-    TF_ASSIGN_OR_RETURN(parent_->evaluated_[power],
-                        ElementWiseBinaryOp(power, [](ElementwiseT lhs_el,
-                                                      ElementwiseT rhs_el) {
-                          return std::pow(lhs_el, rhs_el);
-                        }));
+    TF_ASSIGN_OR_RETURN(
+        parent_->evaluated_[power],
+        ElementWiseBinaryOp(
+            power, [](ElementwiseT lhs_el, ElementwiseT rhs_el) {
+              return lhs_el == ElementwiseT(0) && rhs_el == ElementwiseT(0)
+                         ? static_cast<ElementwiseT>(1)
+                         : std::pow(lhs_el, rhs_el);
+            }));
     return Status::OK();
   }
 
diff --git a/tensorflow/compiler/xla/service/llvm_ir/ir_builder_mixin.h b/tensorflow/compiler/xla/service/llvm_ir/ir_builder_mixin.h
index abc06fb7b4..cf5083e8c1 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/ir_builder_mixin.h
+++ b/tensorflow/compiler/xla/service/llvm_ir/ir_builder_mixin.h
@@ -254,6 +254,11 @@ class IrBuilderMixin {
     return mixin_builder()->CreateFCmpOLT(std::forward<Args>(args)...);
   }
 
+  template <class... Args>
+  llvm::Value* FCmpOLE(Args&&... args) {
+    return mixin_builder()->CreateFCmpOLE(std::forward<Args>(args)...);
+  }
+
   template <class... Args>
   llvm::Value* FCmpONE(Args&&... args) {
     return mixin_builder()->CreateFCmpONE(std::forward<Args>(args)...);
diff --git a/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc b/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc
index e4cc8b4199..7379fbcc22 100644
--- a/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc
+++ b/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc
@@ -1443,6 +1443,27 @@ XLA_TEST_F(ArrayElementwiseOpTest, PowNonIntegerF32s) {
                              error_spec_);
 }
 
+XLA_TEST_F(ArrayElementwiseOpTest, PowC64s) {
+  SetFastMathDisabled(true);
+  XlaBuilder builder(TestName());
+  auto lhs =
+      ConstantR1<complex64>(&builder, {-2.0f, -0.6f, -0.6f, 0.0f, 0.0f, 0.0f});
+  auto rhs =
+      ConstantR1<complex64>(&builder, {0.5f, 0.6f, -0.6f, 0.5f, 0.6f, 0.0f});
+  Pow(lhs, rhs);
+
+  ComputeAndCompareR1<complex64>(&builder,
+                                 {
+                                     {0, 1.41421356},
+                                     {-2.27443288e-01, 0.69999846},
+                                     {-4.19847531e-01, -1.29215783},
+                                     {0, 0},
+                                     {0, 0},
+                                     {1, 0},
+                                 },
+                                 {}, error_spec_);
+}
+
 XLA_TEST_F(ArrayElementwiseOpTest, PowZeroElementF32s) {
   XlaBuilder builder(TestName());
   auto lhs = ConstantR1<float>(&builder, {});
-- 
GitLab


From 7648a9554792d77a1506e39cc3a7bfc361126786 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 14 Jan 2019 18:10:30 -0800
Subject: [PATCH 0723/1095] Fixing what I think a bug introduced by
 cl/228373932, which resulted in some of our eval_test/eval_train jobs to fail
 with: AttributeError: 'NoneType' object has no attribute
 'save_summary_steps'. Here's an example: http://screen/UentDg6uAZ6

I'm not familiar with the code and it's not clear to me how 'config' gets its value. So, feel free to suggest a different fix or rollback cl/228373932.

PiperOrigin-RevId: 229293332
---
 tensorflow/contrib/tpu/python/tpu/tpu_estimator.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 075ecbc52a..79340a9f21 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -2656,7 +2656,11 @@ class TPUEstimator(estimator_lib.Estimator):
         if self._log_every_n_steps is not None:
           examples_hook = ExamplesPerSecondHook(
               ctx.global_batch_size,
-              output_dir=self.model_dir if config.save_summary_steps else None,
+              # pylint:disable=g-long-ternary
+              output_dir=(self.model_dir
+                          if not config or config.save_summary_steps
+                          else None),
+              # pylint:enable=g-long-ternary
               every_n_steps=self._log_every_n_steps)
 
         if ctx.is_running_on_cpu(is_export_mode=is_export_mode):
-- 
GitLab


From c33db572fe31cd48432dab84e17434359786bac8 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 14 Jan 2019 18:23:54 -0800
Subject: [PATCH 0724/1095] Fix transpose_conv

Use the proper filter height/width dimensions.

PiperOrigin-RevId: 229294726
---
 tensorflow/lite/kernels/transpose_conv.cc              | 8 ++++----
 tensorflow/lite/testing/generated_examples_zip_test.cc | 2 --
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/tensorflow/lite/kernels/transpose_conv.cc b/tensorflow/lite/kernels/transpose_conv.cc
index 59eee51068..343f2ca59b 100644
--- a/tensorflow/lite/kernels/transpose_conv.cc
+++ b/tensorflow/lite/kernels/transpose_conv.cc
@@ -119,8 +119,8 @@ TfLiteStatus ResizeIm2ColTensor(TfLiteContext* context,
   im2col_shape_array->data[1] = output_shape->data.i32[1];
   im2col_shape_array->data[2] = output_shape->data.i32[2];
   const int input_depth = SizeOfDimension(input, 3);
-  const int filter_width = SizeOfDimension(weights, 1);
-  const int filter_height = SizeOfDimension(weights, 2);
+  const int filter_width = SizeOfDimension(weights, 2);
+  const int filter_height = SizeOfDimension(weights, 1);
   im2col_shape_array->data[3] = input_depth * filter_height * filter_width;
 
   im2col->type = input->type;
@@ -197,8 +197,8 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
   // Get height and width of the output image.
   const int width = SizeOfDimension(output, 2);
   const int height = SizeOfDimension(output, 1);
-  const int filter_width = SizeOfDimension(weights, 1);
-  const int filter_height = SizeOfDimension(weights, 2);
+  const int filter_width = SizeOfDimension(weights, 2);
+  const int filter_height = SizeOfDimension(weights, 1);
 
   const int stride_width = params->stride_width;
   const int stride_height = params->stride_height;
diff --git a/tensorflow/lite/testing/generated_examples_zip_test.cc b/tensorflow/lite/testing/generated_examples_zip_test.cc
index bade6590a4..a9a31ad088 100644
--- a/tensorflow/lite/testing/generated_examples_zip_test.cc
+++ b/tensorflow/lite/testing/generated_examples_zip_test.cc
@@ -105,8 +105,6 @@ std::map<string, string> kBrokenTests = {
 
     // Strided Slice chooses the wrong dimension.
     {R"(^\/strided_slice_buggy)", "119786029"},
-
-    {R"(^\/conv2d_transpose_filter_shape=\[1,4,8,3\])", "122735908"},
 };
 
 // Allows test data to be unarchived into a temporary directory and makes
-- 
GitLab


From 5634719232f2aa801adf3f9718cb92eb019da754 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 14 Jan 2019 18:41:19 -0800
Subject: [PATCH 0725/1095] Add stream and stream_executor_pimpl deps to
 host_gpu_executor

PiperOrigin-RevId: 229296366
---
 tensorflow/stream_executor/host/BUILD | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/stream_executor/host/BUILD b/tensorflow/stream_executor/host/BUILD
index 59472b14c1..127452aee9 100644
--- a/tensorflow/stream_executor/host/BUILD
+++ b/tensorflow/stream_executor/host/BUILD
@@ -100,7 +100,9 @@ cc_library(
         "//tensorflow/core:lib",
         "//tensorflow/stream_executor:kernel",
         "//tensorflow/stream_executor:rng",
+        "//tensorflow/stream_executor:stream",
         "//tensorflow/stream_executor:stream_executor_internal",
+        "//tensorflow/stream_executor:stream_executor_pimpl",
         "//tensorflow/stream_executor:timer",
         "//tensorflow/stream_executor/lib",
     ],
-- 
GitLab


From b74605a932fe9b7bc878fa83999d09d12d0e5e55 Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Mon, 14 Jan 2019 19:48:23 -0800
Subject: [PATCH 0726/1095] [tf.data] Add counters for tf.data elements,
 autotuning, and optimizations.

This CL:
- adds counters for tf.data elements, autotuning and optimizations
- sets the number of iterations of the `tf_data_meta_optimizer` to one -- the iteration of tf.data optimizations is handled by the tf.data meta optimizer itself
- adds the `alwayslink` attribute to all tf.data optimization BUILD targets to make sure they are always registered (without this, they would not be registered for the Tensorflow server binary I was using for local testing) and further cleans up visibility and dependencies of //third_party/tensorflow/core/grappler/optimizers/data/BUILD
- introduces TFDataOptimizerBase as a base class for tf.data optimizations
- moves TensorFlow metrics into tensorflow::metrics namespace

PiperOrigin-RevId: 229302097
---
 tensorflow/compiler/jit/BUILD                 |   1 -
 .../compiler/jit/build_xla_ops_pass_test.cc   |   1 -
 .../jit/partially_decluster_pass_test.cc      |   1 -
 .../core/common_runtime/direct_session.cc     |   2 +-
 tensorflow/core/common_runtime/metrics.cc     |  25 +-
 tensorflow/core/common_runtime/metrics.h      |  18 ++
 .../core/distributed_runtime/graph_mgr.cc     |   3 +-
 tensorflow/core/framework/model.cc            |   1 +
 tensorflow/core/framework/model.h             |  25 +-
 .../core/grappler/optimizers/data/BUILD       | 183 +++++-------
 .../grappler/optimizers/data/filter_fusion.cc |  11 +-
 .../grappler/optimizers/data/filter_fusion.h  |  13 +-
 .../optimizers/data/function_utils.cc         |   6 +-
 .../grappler/optimizers/data/function_utils.h |   6 +-
 .../grappler/optimizers/data/fusion_utils.cc  |   6 +-
 .../optimizers/data/graph_test_utils.cc       |   6 +-
 .../optimizers/data/graph_test_utils.h        |   6 +-
 .../grappler/optimizers/data/graph_utils.cc   |   6 +-
 .../grappler/optimizers/data/graph_utils.h    |   6 +-
 .../optimizers/data/hoist_random_uniform.cc   |  11 +-
 .../optimizers/data/hoist_random_uniform.h    |  13 +-
 .../optimizers/data/latency_all_edges.cc      |  11 +-
 .../optimizers/data/latency_all_edges.h       |  13 +-
 .../optimizers/data/make_numa_aware.cc        |   7 +-
 .../optimizers/data/make_numa_aware.h         |   9 +-
 .../grappler/optimizers/data/make_sloppy.cc   |   7 +-
 .../grappler/optimizers/data/make_sloppy.h    |   9 +-
 .../optimizers/data/map_and_batch_fusion.cc   |  11 +-
 .../optimizers/data/map_and_batch_fusion.h    |  13 +-
 .../optimizers/data/map_and_filter_fusion.cc  |  11 +-
 .../optimizers/data/map_and_filter_fusion.h   |  13 +-
 .../grappler/optimizers/data/map_fusion.cc    |  11 +-
 .../grappler/optimizers/data/map_fusion.h     |  13 +-
 .../optimizers/data/map_parallelization.cc    |  17 +-
 .../optimizers/data/map_parallelization.h     |  13 +-
 .../optimizers/data/map_vectorization.cc      |  11 +-
 .../optimizers/data/map_vectorization.h       |  13 +-
 .../optimizers/data/meta_optimizer.cc         |   4 +-
 .../grappler/optimizers/data/meta_optimizer.h |   4 +-
 .../optimizers/data/noop_elimination.cc       |  11 +-
 .../optimizers/data/noop_elimination.h        |  13 +-
 .../optimizers/data/optimizer_base.cc}        |  38 +--
 .../grappler/optimizers/data/optimizer_base.h |  47 +++
 .../data/shuffle_and_repeat_fusion.cc         |  11 +-
 .../data/shuffle_and_repeat_fusion.h          |  13 +-
 .../optimizers/data/vectorization_utils.cc    |   8 +-
 .../optimizers/data/vectorization_utils.h     |   6 +-
 tensorflow/core/kernels/data/BUILD            |   9 +-
 .../core/kernels/data/experimental/BUILD      |   1 +
 .../experimental/map_and_batch_dataset_op.cc  |  11 +-
 .../core/kernels/data/model_dataset_op.cc     | 270 +++++++++---------
 .../core/kernels/data/optimize_dataset_op.cc  |   9 +-
 .../data/parallel_interleave_dataset_op.cc    |   9 +-
 .../kernels/data/parallel_map_dataset_op.cc   |   9 +-
 .../core/kernels/data/prefetch_dataset_op.cc  |   9 +-
 55 files changed, 581 insertions(+), 433 deletions(-)
 rename tensorflow/core/{kernels/data/model_dataset_op.h => grappler/optimizers/data/optimizer_base.cc} (51%)
 create mode 100644 tensorflow/core/grappler/optimizers/data/optimizer_base.h

diff --git a/tensorflow/compiler/jit/BUILD b/tensorflow/compiler/jit/BUILD
index ba9fa93654..1d72d8c2d8 100644
--- a/tensorflow/compiler/jit/BUILD
+++ b/tensorflow/compiler/jit/BUILD
@@ -638,7 +638,6 @@ tf_cc_test(
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
-        "//tensorflow/core/grappler/optimizers/data:graph_utils",
         "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
diff --git a/tensorflow/compiler/jit/build_xla_ops_pass_test.cc b/tensorflow/compiler/jit/build_xla_ops_pass_test.cc
index 48a23a4c17..390ffa694b 100644
--- a/tensorflow/compiler/jit/build_xla_ops_pass_test.cc
+++ b/tensorflow/compiler/jit/build_xla_ops_pass_test.cc
@@ -24,7 +24,6 @@ limitations under the License.
 #include "tensorflow/compiler/jit/node_matchers.h"
 #include "tensorflow/core/common_runtime/device_factory.h"
 #include "tensorflow/core/graph/algorithm.h"
-#include "tensorflow/core/grappler/optimizers/data/graph_utils.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/test.h"
diff --git a/tensorflow/compiler/jit/partially_decluster_pass_test.cc b/tensorflow/compiler/jit/partially_decluster_pass_test.cc
index 38a54cc5ef..1d81a8f4fc 100644
--- a/tensorflow/compiler/jit/partially_decluster_pass_test.cc
+++ b/tensorflow/compiler/jit/partially_decluster_pass_test.cc
@@ -33,7 +33,6 @@ limitations under the License.
 #include "tensorflow/core/graph/graph_constructor.h"
 #include "tensorflow/core/graph/graph_def_builder.h"
 #include "tensorflow/core/graph/graph_def_builder_util.h"
-#include "tensorflow/core/grappler/optimizers/data/graph_utils.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/platform/test.h"
 
diff --git a/tensorflow/core/common_runtime/direct_session.cc b/tensorflow/core/common_runtime/direct_session.cc
index 8a1c648f61..80b62f273c 100644
--- a/tensorflow/core/common_runtime/direct_session.cc
+++ b/tensorflow/core/common_runtime/direct_session.cc
@@ -719,7 +719,7 @@ Status DirectSession::RunInternal(int64 step_id, const RunOptions& run_options,
       exec_and_lib.graph->ToGraphDef(partition_graph_def);
     }
   }
-  UpdateGraphExecTime(Env::Default()->NowMicros() - start_time_usecs);
+  metrics::UpdateGraphExecTime(Env::Default()->NowMicros() - start_time_usecs);
 
   return Status::OK();
 }
diff --git a/tensorflow/core/common_runtime/metrics.cc b/tensorflow/core/common_runtime/metrics.cc
index f4c94ed7ec..362d3ceca8 100644
--- a/tensorflow/core/common_runtime/metrics.cc
+++ b/tensorflow/core/common_runtime/metrics.cc
@@ -17,7 +17,7 @@ limitations under the License.
 #include "tensorflow/core/lib/monitoring/counter.h"
 
 namespace tensorflow {
-
+namespace metrics {
 namespace {
 
 auto* graph_runs = monitoring::Counter<0>::New(
@@ -28,8 +28,30 @@ auto* graph_runs = monitoring::Counter<0>::New(
 auto* graph_run_time_usecs = monitoring::Counter<0>::New(
     "/tensorflow/core/graph_run_time_usecs",
     "The total time spent on executing graphs in microseconds.");
+
+auto* tf_data_autotune_counter = monitoring::Counter<1>::New(
+    "/tensorflow/data/autotune", "tf.data autotuning", "name");
+
+auto* tf_data_elements_counter = monitoring::Counter<1>::New(
+    "/tensorflow/data/elements", "tf.data elements", "name");
+
+auto* tf_data_optimization_counter = monitoring::Counter<1>::New(
+    "/tensorflow/data/optimization", "tf.data optimization", "name");
+
 }  // namespace
 
+void RecordTFDataAutotune(const string& name) {
+  tf_data_autotune_counter->GetCell(name)->IncrementBy(1);
+}
+
+void RecordTFDataElements(const string& name, int64 num_elements) {
+  tf_data_elements_counter->GetCell(name)->IncrementBy(num_elements);
+}
+
+void RecordTFDataOptimization(const string& name, int64 num_changes) {
+  tf_data_optimization_counter->GetCell(name)->IncrementBy(num_changes);
+}
+
 void UpdateGraphExecTime(const uint64 running_time_usecs) {
   if (running_time_usecs > 0) {
     graph_runs->GetCell()->IncrementBy(1);
@@ -37,4 +59,5 @@ void UpdateGraphExecTime(const uint64 running_time_usecs) {
   }
 }
 
+}  // namespace metrics
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/metrics.h b/tensorflow/core/common_runtime/metrics.h
index d3430c9f03..9641265362 100644
--- a/tensorflow/core/common_runtime/metrics.h
+++ b/tensorflow/core/common_runtime/metrics.h
@@ -19,9 +19,27 @@ limitations under the License.
 #include "tensorflow/core/platform/types.h"
 
 namespace tensorflow {
+namespace metrics {
+
+// Records that a tf.data dataset op executed by the program used autotuning.
+//
+// The `name` argument identifies the dataset (e.g. "ParallelMap").
+void RecordTFDataAutotune(const string& name);
+
+// Records the number of elements produced by a tf.data dataset.
+//
+// The `name` argument identifies the dataset (e.g. "Batch" or "Map").
+void RecordTFDataElements(const string& name, int64 num_elements);
+
+// Records the number of independent graph changes resulting from the applicaton
+// of a tf.data optimization.
+//
+// The `name` argument identifies the optimization (e.g. "noop_eliminiation").
+void RecordTFDataOptimization(const string& name, int64 num_changes);
 
 void UpdateGraphExecTime(const uint64 running_time_usecs);
 
+}  // namespace metrics
 }  // namespace tensorflow
 
 #endif  // TENSORFLOW_CORE_COMMON_RUNTIME_METRICS_H_
diff --git a/tensorflow/core/distributed_runtime/graph_mgr.cc b/tensorflow/core/distributed_runtime/graph_mgr.cc
index ee5823e314..1065f021a1 100644
--- a/tensorflow/core/distributed_runtime/graph_mgr.cc
+++ b/tensorflow/core/distributed_runtime/graph_mgr.cc
@@ -451,7 +451,8 @@ void GraphMgr::ExecuteAsync(const string& handle, const int64 step_id,
       cancellation_manager,
       [item, rendezvous, ce_handle, done, start_time_usecs](const Status& s) {
         done(s);
-        UpdateGraphExecTime(Env::Default()->NowMicros() - start_time_usecs);
+        metrics::UpdateGraphExecTime(Env::Default()->NowMicros() -
+                                     start_time_usecs);
         rendezvous->Unref();
         item->Unref();
         delete ce_handle;
diff --git a/tensorflow/core/framework/model.cc b/tensorflow/core/framework/model.cc
index 96cc7e583a..47c8dbe61b 100644
--- a/tensorflow/core/framework/model.cc
+++ b/tensorflow/core/framework/model.cc
@@ -496,6 +496,7 @@ void Model::RemoveNode(const string& name) {
   if (node && (*node)->output()) {
     (*node)->output()->remove_input(*node);
   }
+  remove_node_hook_(*node);
   lookup_table_.erase(name);
 }
 
diff --git a/tensorflow/core/framework/model.h b/tensorflow/core/framework/model.h
index d9f7185362..7fac1753a6 100644
--- a/tensorflow/core/framework/model.h
+++ b/tensorflow/core/framework/model.h
@@ -359,16 +359,26 @@ std::shared_ptr<Node> MakeUnknownNode(Node::Args args);
 // implementation of `DatasetBase` and `DatasetBaseIterator` respectively.
 class Model {
  public:
-  Model() : collect_resource_usage_(false) {}
+  using NodeHook = std::function<void(std::shared_ptr<Node>)>;
+
+  // Creates a new model.
+  //
+  // The `remove_node_hook` argument can be used to specify functionality that
+  // should be invoked before a node is removed from the model. The hook can be
+  // used for dependency injection -- to allow the model to invoke functionality
+  // from modules that it could not depend on statically.
+  Model(NodeHook remove_node_hook)
+      : collect_resource_usage_(false),
+        remove_node_hook_(std::move(remove_node_hook)) {
+    DCHECK(remove_node_hook_ != nullptr);
+  }
 
   // Indicates whether to collect resource usage.
   bool collect_resource_usage() const { return collect_resource_usage_; }
 
   // Adds a node with the given name and given output.
-  virtual std::shared_ptr<Node> AddNode(Node::Factory factory,
-                                        const string& name,
-                                        const string& output_name)
-      LOCKS_EXCLUDED(mu_);
+  std::shared_ptr<Node> AddNode(Node::Factory factory, const string& name,
+                                const string& output_name) LOCKS_EXCLUDED(mu_);
 
   // Increments the processing time for the given node..
   void AddProcessingTime(const string& name, int64 delta) LOCKS_EXCLUDED(mu_);
@@ -390,7 +400,7 @@ class Model {
   // Removes the given node.
   void RemoveNode(const string& name) LOCKS_EXCLUDED(mu_);
 
- protected:
+ private:
   // Collects tunable parameters in the tree rooted in the given node.
   std::vector<std::shared_ptr<Parameter>> CollectTunableParameters(
       std::shared_ptr<Node> node);
@@ -416,6 +426,9 @@ class Model {
   // tunable parameter (because the information is used for for tuning the value
   // of the parameter) and never stops.
   std::atomic<bool> collect_resource_usage_;
+
+  // A hook invoked immediately before a node is removed from the model.
+  const NodeHook remove_node_hook_;
 };
 
 }  // namespace model
diff --git a/tensorflow/core/grappler/optimizers/data/BUILD b/tensorflow/core/grappler/optimizers/data/BUILD
index d95edc5623..413588b15c 100644
--- a/tensorflow/core/grappler/optimizers/data/BUILD
+++ b/tensorflow/core/grappler/optimizers/data/BUILD
@@ -3,29 +3,29 @@ licenses(["notice"])  # Apache 2.0
 load("//tensorflow:tensorflow.bzl", "tf_cc_test")
 load("//tensorflow/core:platform/default/build_config.bzl", "tf_protos_all")
 
+package(default_visibility = [
+    "//tensorflow/core/grappler/optimizers/data:__subpackages__",
+    "//tensorflow/core/kernels/data:__pkg__",
+])
+
 cc_library(
-    name = "meta_optimizer",
-    srcs = ["meta_optimizer.cc"],
-    hdrs = [
-        "meta_optimizer.h",
-    ],
+    name = "data",
     visibility = ["//visibility:public"],
     deps = [
-        "@com_google_absl//absl/container:flat_hash_map",
-        "//tensorflow/core/grappler:grappler_item",
-        "//tensorflow/core:lib",
-        "//tensorflow/core/grappler/clusters:cluster",
-        "//tensorflow/core/grappler/optimizers:arithmetic_optimizer",
-        "//tensorflow/core/grappler/optimizers:model_pruner",
-        "//tensorflow/core/grappler/optimizers:shape_optimizer",
-        "//tensorflow/core/grappler/optimizers:dependency_optimizer",
-        "//tensorflow/core/grappler/optimizers:function_optimizer",
-        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
-        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
-        "//tensorflow/core:lib_internal",
-        "//tensorflow/core:ptr_util",
-    ] + tf_protos_all(),
-    alwayslink = 1,
+        ":filter_fusion",
+        ":hoist_random_uniform",
+        ":latency_all_edges",
+        ":make_numa_aware",
+        ":make_sloppy",
+        ":map_and_batch_fusion",
+        ":map_and_filter_fusion",
+        ":map_fusion",
+        ":map_parallelization",
+        ":map_vectorization",
+        ":meta_optimizer",
+        ":noop_elimination",
+        ":shuffle_and_repeat_fusion",
+    ],
 )
 
 cc_library(
@@ -34,10 +34,10 @@ cc_library(
     hdrs = [
         "filter_fusion.h",
     ],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_utils",
         ":fusion_utils",
+        ":optimizer_base",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core:lib",
         "//tensorflow/core/grappler:grappler_item",
@@ -45,7 +45,6 @@ cc_library(
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/clusters:cluster",
         "//tensorflow/core/grappler/utils:topological_sort",
-        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
         "//tensorflow/core:lib_internal",
     ] + tf_protos_all(),
@@ -55,7 +54,6 @@ cc_library(
 tf_cc_test(
     name = "filter_fusion_test",
     srcs = ["filter_fusion_test.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":filter_fusion",
         ":graph_test_utils",
@@ -74,7 +72,6 @@ cc_library(
     hdrs = [
         "fusion_utils.h",
     ],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_utils",
         ":function_utils",
@@ -94,7 +91,6 @@ cc_library(
 tf_cc_test(
     name = "fusion_utils_test",
     srcs = ["fusion_utils_test.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":function_utils",
         ":fusion_utils",
@@ -113,7 +109,6 @@ cc_library(
     hdrs = [
         "function_utils.h",
     ],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_utils",
         "//tensorflow/core:framework",
@@ -127,7 +122,6 @@ cc_library(
 tf_cc_test(
     name = "function_utils_test",
     srcs = ["function_utils_test.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":function_utils",
         "//tensorflow/core:framework",
@@ -135,7 +129,6 @@ tf_cc_test(
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
         "//tensorflow/core/grappler:grappler_item",
-        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
         "//tensorflow/core/kernels:cast_op",
         "//tensorflow/tools/graph_transforms:transform_utils",
@@ -148,7 +141,6 @@ cc_library(
     hdrs = [
         "graph_utils.h",
     ],
-    visibility = ["//visibility:public"],
     deps = [
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:framework",
@@ -162,7 +154,6 @@ cc_library(
 tf_cc_test(
     name = "graph_utils_test",
     srcs = ["graph_utils_test.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_utils",
         "//tensorflow/core:core_cpu",
@@ -171,7 +162,6 @@ tf_cc_test(
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
         "//tensorflow/core/grappler:grappler_item",
-        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
     ],
 )
@@ -183,7 +173,6 @@ cc_library(
     hdrs = [
         "graph_test_utils.h",
     ],
-    visibility = ["//visibility:public"],
     deps = [
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
@@ -200,17 +189,16 @@ cc_library(
     hdrs = [
         "hoist_random_uniform.h",
     ],
-    visibility = ["//visibility:public"],
     deps = [
         ":function_utils",
         ":graph_utils",
+        ":optimizer_base",
         "//tensorflow/core:lib",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/clusters:cluster",
-        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
         "//tensorflow/core:lib_internal",
     ] + tf_protos_all(),
@@ -220,7 +208,6 @@ cc_library(
 tf_cc_test(
     name = "hoist_random_uniform_test",
     srcs = ["hoist_random_uniform_test.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_test_utils",
         ":graph_utils",
@@ -239,35 +226,45 @@ cc_library(
     hdrs = [
         "latency_all_edges.h",
     ],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_utils",
+        ":optimizer_base",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core:lib",
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/clusters:cluster",
-        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
     ] + tf_protos_all(),
     alwayslink = 1,
 )
 
+tf_cc_test(
+    name = "latency_all_edges_test",
+    srcs = ["latency_all_edges_test.cc"],
+    deps = [
+        ":graph_utils",
+        ":latency_all_edges",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
+        "//tensorflow/core/grappler:grappler_item",
+    ],
+)
+
 cc_library(
     name = "make_numa_aware",
     srcs = ["make_numa_aware.cc"],
     hdrs = ["make_numa_aware.h"],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_utils",
+        ":optimizer_base",
         "//tensorflow/core/grappler:mutable_graph_view",
-        "//tensorflow/core:lib",
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:op_types",
-        "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/clusters:cluster",
-        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
     ] + tf_protos_all(),
     alwayslink = 1,
@@ -276,7 +273,6 @@ cc_library(
 tf_cc_test(
     name = "make_numa_aware_test",
     srcs = ["make_numa_aware_test.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_test_utils",
         ":graph_utils",
@@ -293,13 +289,12 @@ cc_library(
     name = "make_sloppy",
     srcs = ["make_sloppy.cc"],
     hdrs = ["make_sloppy.h"],
-    visibility = ["//visibility:public"],
     deps = [
+        ":optimizer_base",
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler/clusters:cluster",
-        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
     ] + tf_protos_all(),
     alwayslink = 1,
@@ -308,7 +303,6 @@ cc_library(
 tf_cc_test(
     name = "make_sloppy_test",
     srcs = ["make_sloppy_test.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_test_utils",
         ":graph_utils",
@@ -327,16 +321,15 @@ cc_library(
     hdrs = [
         "map_and_batch_fusion.h",
     ],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_utils",
+        ":optimizer_base",
         "//tensorflow/core:lib",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/clusters:cluster",
-        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
     ] + tf_protos_all(),
     alwayslink = 1,
@@ -345,7 +338,6 @@ cc_library(
 tf_cc_test(
     name = "map_and_batch_fusion_test",
     srcs = ["map_and_batch_fusion_test.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_utils",
         ":map_and_batch_fusion",
@@ -362,17 +354,16 @@ cc_library(
     hdrs = [
         "map_and_filter_fusion.h",
     ],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_utils",
         ":fusion_utils",
+        ":optimizer_base",
         "//tensorflow/core:lib",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/clusters:cluster",
-        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/utils:topological_sort",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
         "//tensorflow/core:lib_internal",
@@ -383,7 +374,6 @@ cc_library(
 tf_cc_test(
     name = "map_and_filter_fusion_test",
     srcs = ["map_and_filter_fusion_test.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_test_utils",
         ":graph_utils",
@@ -402,10 +392,10 @@ cc_library(
     hdrs = [
         "map_fusion.h",
     ],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_utils",
         ":fusion_utils",
+        ":optimizer_base",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core:lib",
         "//tensorflow/core/grappler:grappler_item",
@@ -413,7 +403,6 @@ cc_library(
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/clusters:cluster",
         "//tensorflow/core/grappler/utils:topological_sort",
-        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
         "//tensorflow/core:lib_internal",
     ] + tf_protos_all(),
@@ -423,7 +412,6 @@ cc_library(
 tf_cc_test(
     name = "map_fusion_test",
     srcs = ["map_fusion_test.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_test_utils",
         ":graph_utils",
@@ -443,15 +431,14 @@ cc_library(
     hdrs = [
         "map_parallelization.h",
     ],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_utils",
+        ":optimizer_base",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/clusters:cluster",
-        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
     ] + tf_protos_all(),
     alwayslink = 1,
@@ -460,7 +447,6 @@ cc_library(
 tf_cc_test(
     name = "map_parallelization_test",
     srcs = ["map_parallelization_test.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_test_utils",
         ":graph_utils",
@@ -479,10 +465,10 @@ cc_library(
     hdrs = [
         "map_vectorization.h",
     ],
-    visibility = ["//visibility:public"],
     deps = [
         ":function_utils",
         ":graph_utils",
+        ":optimizer_base",
         ":vectorization_utils",
         "//tensorflow/core:lib",
         "//tensorflow/core/grappler:mutable_graph_view",
@@ -490,7 +476,6 @@ cc_library(
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/clusters:cluster",
-        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
         "//tensorflow/core:lib_internal",
     ] + tf_protos_all(),
@@ -500,7 +485,6 @@ cc_library(
 tf_cc_test(
     name = "map_vectorization_test",
     srcs = ["map_vectorization_test.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_utils",
         ":map_vectorization",
@@ -512,22 +496,42 @@ tf_cc_test(
     ],
 )
 
+cc_library(
+    name = "meta_optimizer",
+    srcs = ["meta_optimizer.cc"],
+    hdrs = ["meta_optimizer.h"],
+    deps = [
+        "@com_google_absl//absl/container:flat_hash_map",
+        "//tensorflow/core/grappler/clusters:cluster",
+        "//tensorflow/core/grappler/optimizers:arithmetic_optimizer",
+        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
+        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
+        "//tensorflow/core/grappler/optimizers:dependency_optimizer",
+        "//tensorflow/core/grappler/optimizers:function_optimizer",
+        "//tensorflow/core/grappler/optimizers:model_pruner",
+        "//tensorflow/core/grappler/optimizers:shape_optimizer",
+        "//tensorflow/core/grappler:grappler_item",
+        "//tensorflow/core:lib_internal",
+        "//tensorflow/core:ptr_util",
+    ] + tf_protos_all(),
+    alwayslink = 1,
+)
+
 cc_library(
     name = "noop_elimination",
     srcs = ["noop_elimination.cc"],
     hdrs = [
         "noop_elimination.h",
     ],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_utils",
+        ":optimizer_base",
         "//tensorflow/core:lib",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/clusters:cluster",
-        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
     ] + tf_protos_all(),
     alwayslink = 1,
@@ -536,7 +540,6 @@ cc_library(
 tf_cc_test(
     name = "noop_elimination_test",
     srcs = ["noop_elimination_test.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_utils",
         ":noop_elimination",
@@ -547,22 +550,33 @@ tf_cc_test(
     ],
 )
 
+cc_library(
+    name = "optimizer_base",
+    srcs = ["optimizer_base.cc"],
+    hdrs = [
+        "optimizer_base.h",
+    ],
+    deps = [
+        "//tensorflow/core:metrics",
+        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
+    ],
+)
+
 cc_library(
     name = "shuffle_and_repeat_fusion",
     srcs = ["shuffle_and_repeat_fusion.cc"],
     hdrs = [
         "shuffle_and_repeat_fusion.h",
     ],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_utils",
+        ":optimizer_base",
         "//tensorflow/core:lib",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/clusters:cluster",
-        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
     ] + tf_protos_all(),
     alwayslink = 1,
@@ -571,7 +585,6 @@ cc_library(
 tf_cc_test(
     name = "shuffle_and_repeat_fusion_test",
     srcs = ["shuffle_and_repeat_fusion_test.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_utils",
         ":shuffle_and_repeat_fusion",
@@ -582,48 +595,12 @@ tf_cc_test(
     ],
 )
 
-cc_library(
-    name = "data",
-    visibility = ["//visibility:public"],
-    deps = [
-        ":filter_fusion",
-        ":hoist_random_uniform",
-        ":latency_all_edges",
-        ":make_numa_aware",
-        ":make_sloppy",
-        ":map_and_batch_fusion",
-        ":map_and_filter_fusion",
-        ":map_fusion",
-        ":map_parallelization",
-        ":map_vectorization",
-        ":meta_optimizer",
-        ":noop_elimination",
-        ":shuffle_and_repeat_fusion",
-    ],
-    alwayslink = 1,
-)
-
-tf_cc_test(
-    name = "latency_all_edges_test",
-    srcs = ["latency_all_edges_test.cc"],
-    deps = [
-        ":graph_utils",
-        ":latency_all_edges",
-        "//tensorflow/core:framework",
-        "//tensorflow/core:test",
-        "//tensorflow/core:test_main",
-        "//tensorflow/core:testlib",
-        "//tensorflow/core/grappler:grappler_item",
-    ],
-)
-
 cc_library(
     name = "vectorization_utils",
     srcs = ["vectorization_utils.cc"],
     hdrs = [
         "vectorization_utils.h",
     ],
-    visibility = ["//visibility:public"],
     deps = [
         ":function_utils",
         ":graph_utils",
@@ -644,7 +621,6 @@ cc_library(
 tf_cc_test(
     name = "vectorization_utils_test",
     srcs = ["vectorization_utils_test.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_utils",
         ":function_utils",
@@ -656,7 +632,6 @@ tf_cc_test(
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
         "//tensorflow/core/grappler:grappler_item",
-        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
         # For ops we need registered
         "//tensorflow/core/kernels/data:dataset_ops",
diff --git a/tensorflow/core/grappler/optimizers/data/filter_fusion.cc b/tensorflow/core/grappler/optimizers/data/filter_fusion.cc
index e3f0a7e6c5..25dc10806b 100644
--- a/tensorflow/core/grappler/optimizers/data/filter_fusion.cc
+++ b/tensorflow/core/grappler/optimizers/data/filter_fusion.cc
@@ -57,8 +57,10 @@ NodeDef MakeFusedFilterNode(const NodeDef& first_filter_node,
 
 }  // namespace
 
-Status FilterFusion::Optimize(Cluster* cluster, const GrapplerItem& item,
-                              GraphDef* output) {
+Status FilterFusion::OptimizeAndCollectStats(Cluster* cluster,
+                                             const GrapplerItem& item,
+                                             GraphDef* output,
+                                             OptimizationStats* stats) {
   GraphDef sorted_old_graph = item.graph;
   TF_RETURN_IF_ERROR(TopologicalSort(&sorted_old_graph));
   *output = sorted_old_graph;
@@ -120,6 +122,7 @@ Status FilterFusion::Optimize(Cluster* cluster, const GrapplerItem& item,
     // they are not used anymore.
     nodes_to_delete.insert(first_filter_node->name());
     nodes_to_delete.insert(second_filter_node->name());
+    stats->num_changes++;
   }
 
   graph.DeleteNodes(nodes_to_delete);
@@ -133,5 +136,5 @@ void FilterFusion::Feedback(Cluster* cluster, const GrapplerItem& item,
 
 REGISTER_GRAPH_OPTIMIZER_AS(FilterFusion, "filter_fusion");
 
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace grappler
+}  // namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/data/filter_fusion.h b/tensorflow/core/grappler/optimizers/data/filter_fusion.h
index 91a0364a46..ac0326c0ec 100644
--- a/tensorflow/core/grappler/optimizers/data/filter_fusion.h
+++ b/tensorflow/core/grappler/optimizers/data/filter_fusion.h
@@ -16,13 +16,13 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_FILTER_FUSION_H_
 #define TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_FILTER_FUSION_H_
 
-#include "tensorflow/core/grappler/optimizers/custom_graph_optimizer.h"
+#include "tensorflow/core/grappler/optimizers/data/optimizer_base.h"
 
 namespace tensorflow {
 namespace grappler {
 
 // This optimization fuses filter transformations.
-class FilterFusion : public CustomGraphOptimizer {
+class FilterFusion : public TFDataOptimizerBase {
  public:
   FilterFusion() = default;
   ~FilterFusion() override = default;
@@ -34,14 +34,15 @@ class FilterFusion : public CustomGraphOptimizer {
     return Status::OK();
   }
 
-  Status Optimize(Cluster* cluster, const GrapplerItem& item,
-                  GraphDef* output) override;
+  Status OptimizeAndCollectStats(Cluster* cluster, const GrapplerItem& item,
+                                 GraphDef* output,
+                                 OptimizationStats* stats) override;
 
   void Feedback(Cluster* cluster, const GrapplerItem& item,
                 const GraphDef& optimize_output, double result) override;
 };
 
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace grappler
+}  // namespace tensorflow
 
 #endif  // TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_FILTER_FUSION_H_
diff --git a/tensorflow/core/grappler/optimizers/data/function_utils.cc b/tensorflow/core/grappler/optimizers/data/function_utils.cc
index 311df15bc2..351209f92e 100644
--- a/tensorflow/core/grappler/optimizers/data/function_utils.cc
+++ b/tensorflow/core/grappler/optimizers/data/function_utils.cc
@@ -171,6 +171,6 @@ void SetUniqueFunctionNodeName(StringPiece prefix, FunctionDef* function,
   node->set_name(std::move(name));
 }
 
-}  // end namespace function_utils
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace function_utils
+}  // namespace grappler
+}  // namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/data/function_utils.h b/tensorflow/core/grappler/optimizers/data/function_utils.h
index d4ce824652..ec08c11dce 100644
--- a/tensorflow/core/grappler/optimizers/data/function_utils.h
+++ b/tensorflow/core/grappler/optimizers/data/function_utils.h
@@ -101,8 +101,8 @@ int FindFunctionNodeWithOp(StringPiece op, const FunctionDef& function);
 void SetUniqueFunctionNodeName(StringPiece prefix, FunctionDef* function,
                                NodeDef* node);
 
-}  // end namespace function_utils
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace function_utils
+}  // namespace grappler
+}  // namespace tensorflow
 
 #endif  // TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_FUNCTION_UTILS_H_
diff --git a/tensorflow/core/grappler/optimizers/data/fusion_utils.cc b/tensorflow/core/grappler/optimizers/data/fusion_utils.cc
index b3bfee138f..d5308ad31a 100644
--- a/tensorflow/core/grappler/optimizers/data/fusion_utils.cc
+++ b/tensorflow/core/grappler/optimizers/data/fusion_utils.cc
@@ -471,6 +471,6 @@ FunctionDef* FuseFunctions(
   return fused_function;
 }
 
-}  // end namespace fusion_utils
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace fusion_utils
+}  // namespace grappler
+}  // namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/data/graph_test_utils.cc b/tensorflow/core/grappler/optimizers/data/graph_test_utils.cc
index 202dfb5ac8..82ca0146b9 100644
--- a/tensorflow/core/grappler/optimizers/data/graph_test_utils.cc
+++ b/tensorflow/core/grappler/optimizers/data/graph_test_utils.cc
@@ -107,6 +107,6 @@ NodeDef MakeParseExampleNode(StringPiece name, StringPiece input_node_name,
       });
 }
 
-}  // end namespace graph_tests_utils
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace graph_tests_utils
+}  // namespace grappler
+}  // namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/data/graph_test_utils.h b/tensorflow/core/grappler/optimizers/data/graph_test_utils.h
index a2707ee7b7..3750e2d5cc 100644
--- a/tensorflow/core/grappler/optimizers/data/graph_test_utils.h
+++ b/tensorflow/core/grappler/optimizers/data/graph_test_utils.h
@@ -56,8 +56,8 @@ NodeDef MakeParseExampleNode(StringPiece name, StringPiece input_node_name,
                              StringPiece num_parallel_calls_node_name,
                              bool sloppy);
 
-}  // end namespace graph_tests_utils
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace graph_tests_utils
+}  // namespace grappler
+}  // namespace tensorflow
 
 #endif  // TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_GRAPH_TEST_UTILS_H_
diff --git a/tensorflow/core/grappler/optimizers/data/graph_utils.cc b/tensorflow/core/grappler/optimizers/data/graph_utils.cc
index 90208c1fba..cbafb9dc82 100644
--- a/tensorflow/core/grappler/optimizers/data/graph_utils.cc
+++ b/tensorflow/core/grappler/optimizers/data/graph_utils.cc
@@ -293,6 +293,6 @@ Status EnsureNodeNamesUnique(Graph* g) {
 
   return Status::OK();
 }
-}  // end namespace graph_utils
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace graph_utils
+}  // namespace grappler
+}  // namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/data/graph_utils.h b/tensorflow/core/grappler/optimizers/data/graph_utils.h
index d130fee204..8f2872c146 100644
--- a/tensorflow/core/grappler/optimizers/data/graph_utils.h
+++ b/tensorflow/core/grappler/optimizers/data/graph_utils.h
@@ -140,8 +140,8 @@ void ConcatAttributeList(const string& attribute_name, const NodeDef& first,
 // and renaming nodes does not mutate any edges.
 Status EnsureNodeNamesUnique(Graph* g);
 
-}  // end namespace graph_utils
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace graph_utils
+}  // namespace grappler
+}  // namespace tensorflow
 
 #endif  // TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_GRAPH_UTILS_H_
diff --git a/tensorflow/core/grappler/optimizers/data/hoist_random_uniform.cc b/tensorflow/core/grappler/optimizers/data/hoist_random_uniform.cc
index 6aa450e123..237e4b63e3 100644
--- a/tensorflow/core/grappler/optimizers/data/hoist_random_uniform.cc
+++ b/tensorflow/core/grappler/optimizers/data/hoist_random_uniform.cc
@@ -220,8 +220,10 @@ int NumberOfPlaceholders(const NodeDef& map_node) {
 
 }  // namespace
 
-Status HoistRandomUniform::Optimize(Cluster* cluster, const GrapplerItem& item,
-                                    GraphDef* output) {
+Status HoistRandomUniform::OptimizeAndCollectStats(Cluster* cluster,
+                                                   const GrapplerItem& item,
+                                                   GraphDef* output,
+                                                   OptimizationStats* stats) {
   *output = item.graph;
 
   MutableGraphView graph(output);
@@ -272,6 +274,7 @@ Status HoistRandomUniform::Optimize(Cluster* cluster, const GrapplerItem& item,
     // TODO(b/116285210): we could also remove map functions from library if
     // they are not used anymore.
     nodes_to_delete.insert(map_node->name());
+    stats->num_changes++;
   }
 
   graph.DeleteNodes(nodes_to_delete);
@@ -286,5 +289,5 @@ void HoistRandomUniform::Feedback(Cluster* cluster, const GrapplerItem& item,
 
 REGISTER_GRAPH_OPTIMIZER_AS(HoistRandomUniform, "hoist_random_uniform");
 
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace grappler
+}  // namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/data/hoist_random_uniform.h b/tensorflow/core/grappler/optimizers/data/hoist_random_uniform.h
index d1bcf6782d..94db9f72a4 100644
--- a/tensorflow/core/grappler/optimizers/data/hoist_random_uniform.h
+++ b/tensorflow/core/grappler/optimizers/data/hoist_random_uniform.h
@@ -16,7 +16,7 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_HOIST_RANDOM_UNIFORM_H_
 #define TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_HOIST_RANDOM_UNIFORM_H_
 
-#include "tensorflow/core/grappler/optimizers/custom_graph_optimizer.h"
+#include "tensorflow/core/grappler/optimizers/data/optimizer_base.h"
 
 namespace tensorflow {
 namespace grappler {
@@ -30,7 +30,7 @@ namespace grappler {
 // `stateless_random_uniform`.
 // TODO(prazek): for now only `RandomUniform` is handled, but we could handle
 // `RandomUniformInt` similarly.
-class HoistRandomUniform : public CustomGraphOptimizer {
+class HoistRandomUniform : public TFDataOptimizerBase {
  public:
   HoistRandomUniform() = default;
   ~HoistRandomUniform() override = default;
@@ -42,14 +42,15 @@ class HoistRandomUniform : public CustomGraphOptimizer {
     return Status::OK();
   }
 
-  Status Optimize(Cluster* cluster, const GrapplerItem& item,
-                  GraphDef* output) override;
+  Status OptimizeAndCollectStats(Cluster* cluster, const GrapplerItem& item,
+                                 GraphDef* output,
+                                 OptimizationStats* stats) override;
 
   void Feedback(Cluster* cluster, const GrapplerItem& item,
                 const GraphDef& optimize_output, double result) override;
 };
 
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace grappler
+}  // namespace tensorflow
 
 #endif  // TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_HOIST_RANDOM_UNIFORM_H_
diff --git a/tensorflow/core/grappler/optimizers/data/latency_all_edges.cc b/tensorflow/core/grappler/optimizers/data/latency_all_edges.cc
index 0b7b2eba86..4529b89bd4 100644
--- a/tensorflow/core/grappler/optimizers/data/latency_all_edges.cc
+++ b/tensorflow/core/grappler/optimizers/data/latency_all_edges.cc
@@ -64,8 +64,10 @@ NodeDef MakeLatencyNode(const NodeDef& node, MutableGraphView* graph) {
 
 }  // namespace
 
-Status LatencyAllEdges::Optimize(Cluster* cluster, const GrapplerItem& item,
-                                 GraphDef* output) {
+Status LatencyAllEdges::OptimizeAndCollectStats(Cluster* cluster,
+                                                const GrapplerItem& item,
+                                                GraphDef* output,
+                                                OptimizationStats* stats) {
   *output = item.graph;
   MutableGraphView graph(output);
 
@@ -96,6 +98,7 @@ Status LatencyAllEdges::Optimize(Cluster* cluster, const GrapplerItem& item,
 
     NodeDef* latency_node = graph.AddNode(MakeLatencyNode(node, &graph));
     TF_RETURN_IF_ERROR(graph.UpdateFanouts(node.name(), latency_node->name()));
+    stats->num_changes++;
   }
   return Status::OK();
 }
@@ -107,5 +110,5 @@ void LatencyAllEdges::Feedback(Cluster* cluster, const GrapplerItem& item,
 
 REGISTER_GRAPH_OPTIMIZER_AS(LatencyAllEdges, "latency_all_edges");
 
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace grappler
+}  // namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/data/latency_all_edges.h b/tensorflow/core/grappler/optimizers/data/latency_all_edges.h
index f6c71a9ec7..313d108286 100644
--- a/tensorflow/core/grappler/optimizers/data/latency_all_edges.h
+++ b/tensorflow/core/grappler/optimizers/data/latency_all_edges.h
@@ -16,12 +16,12 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_LATENCY_ALL_EDGES_H_
 #define TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_LATENCY_ALL_EDGES_H_
 
-#include "tensorflow/core/grappler/optimizers/custom_graph_optimizer.h"
+#include "tensorflow/core/grappler/optimizers/data/optimizer_base.h"
 
 namespace tensorflow {
 namespace grappler {
 
-class LatencyAllEdges : public CustomGraphOptimizer {
+class LatencyAllEdges : public TFDataOptimizerBase {
  public:
   LatencyAllEdges() = default;
   ~LatencyAllEdges() override = default;
@@ -33,14 +33,15 @@ class LatencyAllEdges : public CustomGraphOptimizer {
     return Status::OK();
   }
 
-  Status Optimize(Cluster* cluster, const GrapplerItem& item,
-                  GraphDef* output) override;
+  Status OptimizeAndCollectStats(Cluster* cluster, const GrapplerItem& item,
+                                 GraphDef* output,
+                                 OptimizationStats* stats) override;
 
   void Feedback(Cluster* cluster, const GrapplerItem& item,
                 const GraphDef& optimize_output, double result) override;
 };
 
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace grappler
+}  // namespace tensorflow
 
 #endif  // TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_LATENCY_ALL_EDGES_H_
diff --git a/tensorflow/core/grappler/optimizers/data/make_numa_aware.cc b/tensorflow/core/grappler/optimizers/data/make_numa_aware.cc
index 2940fbecd4..d8ea9adfb7 100644
--- a/tensorflow/core/grappler/optimizers/data/make_numa_aware.cc
+++ b/tensorflow/core/grappler/optimizers/data/make_numa_aware.cc
@@ -37,8 +37,10 @@ NodeDef MakeNumaAwareNode(const NodeDef& node, MutableGraphView* graph) {
 
 }  // namespace
 
-Status MakeNumaAware::Optimize(Cluster* cluster, const GrapplerItem& item,
-                               GraphDef* output) {
+Status MakeNumaAware::OptimizeAndCollectStats(Cluster* cluster,
+                                              const GrapplerItem& item,
+                                              GraphDef* output,
+                                              OptimizationStats* stats) {
   *output = item.graph;
   MutableGraphView graph(output);
   std::set<string> nodes_to_delete;
@@ -49,6 +51,7 @@ Status MakeNumaAware::Optimize(Cluster* cluster, const GrapplerItem& item,
     auto* numa_node = graph.AddNode(MakeNumaAwareNode(node, &graph));
     TF_RETURN_IF_ERROR(graph.UpdateFanouts(node.name(), numa_node->name()));
     nodes_to_delete.insert(node.name());
+    stats->num_changes++;
   }
   graph.DeleteNodes(nodes_to_delete);
   return Status::OK();
diff --git a/tensorflow/core/grappler/optimizers/data/make_numa_aware.h b/tensorflow/core/grappler/optimizers/data/make_numa_aware.h
index 48a7d8145f..81dbb31e6d 100644
--- a/tensorflow/core/grappler/optimizers/data/make_numa_aware.h
+++ b/tensorflow/core/grappler/optimizers/data/make_numa_aware.h
@@ -16,12 +16,12 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_MAKE_NUMA_AWARE_H_
 #define TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_MAKE_NUMA_AWARE_H_
 
-#include "tensorflow/core/grappler/optimizers/custom_graph_optimizer.h"
+#include "tensorflow/core/grappler/optimizers/data/optimizer_base.h"
 
 namespace tensorflow {
 namespace grappler {
 
-class MakeNumaAware : public CustomGraphOptimizer {
+class MakeNumaAware : public TFDataOptimizerBase {
  public:
   MakeNumaAware() = default;
   ~MakeNumaAware() override = default;
@@ -33,8 +33,9 @@ class MakeNumaAware : public CustomGraphOptimizer {
     return Status::OK();
   }
 
-  Status Optimize(Cluster* cluster, const GrapplerItem& item,
-                  GraphDef* output) override;
+  Status OptimizeAndCollectStats(Cluster* cluster, const GrapplerItem& item,
+                                 GraphDef* output,
+                                 OptimizationStats* stats) override;
 
   void Feedback(Cluster* cluster, const GrapplerItem& item,
                 const GraphDef& optimize_output, double result) override {}
diff --git a/tensorflow/core/grappler/optimizers/data/make_sloppy.cc b/tensorflow/core/grappler/optimizers/data/make_sloppy.cc
index 1cfaef3ffb..1de0c46427 100644
--- a/tensorflow/core/grappler/optimizers/data/make_sloppy.cc
+++ b/tensorflow/core/grappler/optimizers/data/make_sloppy.cc
@@ -25,8 +25,10 @@ limitations under the License.
 namespace tensorflow {
 namespace grappler {
 
-Status MakeSloppy::Optimize(Cluster* cluster, const GrapplerItem& item,
-                            GraphDef* output) {
+Status MakeSloppy::OptimizeAndCollectStats(Cluster* cluster,
+                                           const GrapplerItem& item,
+                                           GraphDef* output,
+                                           OptimizationStats* stats) {
   *output = item.graph;
   MutableGraphView graph(output);
 
@@ -35,6 +37,7 @@ Status MakeSloppy::Optimize(Cluster* cluster, const GrapplerItem& item,
         node.op() == "ParallelMapDataset" ||
         node.op() == "ParseExampleDataset") {
       (*node.mutable_attr())["sloppy"].set_b(true);
+      stats->num_changes++;
     }
   }
   return Status::OK();
diff --git a/tensorflow/core/grappler/optimizers/data/make_sloppy.h b/tensorflow/core/grappler/optimizers/data/make_sloppy.h
index 9dcab1038d..cf42e84198 100644
--- a/tensorflow/core/grappler/optimizers/data/make_sloppy.h
+++ b/tensorflow/core/grappler/optimizers/data/make_sloppy.h
@@ -16,12 +16,12 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_MAKE_SLOPPY_H_
 #define TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_MAKE_SLOPPY_H_
 
-#include "tensorflow/core/grappler/optimizers/custom_graph_optimizer.h"
+#include "tensorflow/core/grappler/optimizers/data/optimizer_base.h"
 
 namespace tensorflow {
 namespace grappler {
 
-class MakeSloppy : public CustomGraphOptimizer {
+class MakeSloppy : public TFDataOptimizerBase {
  public:
   MakeSloppy() = default;
   ~MakeSloppy() override = default;
@@ -33,8 +33,9 @@ class MakeSloppy : public CustomGraphOptimizer {
     return Status::OK();
   }
 
-  Status Optimize(Cluster* cluster, const GrapplerItem& item,
-                  GraphDef* output) override;
+  Status OptimizeAndCollectStats(Cluster* cluster, const GrapplerItem& item,
+                                 GraphDef* output,
+                                 OptimizationStats* stats) override;
 
   void Feedback(Cluster* cluster, const GrapplerItem& item,
                 const GraphDef& optimize_output, double result) override {}
diff --git a/tensorflow/core/grappler/optimizers/data/map_and_batch_fusion.cc b/tensorflow/core/grappler/optimizers/data/map_and_batch_fusion.cc
index 4ff749c57b..3aeac021e6 100644
--- a/tensorflow/core/grappler/optimizers/data/map_and_batch_fusion.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_and_batch_fusion.cc
@@ -98,8 +98,10 @@ NodeDef MakeMapAndBatchNode(const NodeDef& map_node, const NodeDef& batch_node,
 
 }  // namespace
 
-Status MapAndBatchFusion::Optimize(Cluster* cluster, const GrapplerItem& item,
-                                   GraphDef* output) {
+Status MapAndBatchFusion::OptimizeAndCollectStats(Cluster* cluster,
+                                                  const GrapplerItem& item,
+                                                  GraphDef* output,
+                                                  OptimizationStats* stats) {
   *output = item.graph;
   MutableGraphView graph(output);
   std::set<string> nodes_to_delete;
@@ -126,6 +128,7 @@ Status MapAndBatchFusion::Optimize(Cluster* cluster, const GrapplerItem& item,
     // Mark the `Map` and `Batch` nodes for removal.
     nodes_to_delete.insert(map_node->name());
     nodes_to_delete.insert(batch_node.name());
+    stats->num_changes++;
   }
 
   graph.DeleteNodes(nodes_to_delete);
@@ -140,5 +143,5 @@ void MapAndBatchFusion::Feedback(Cluster* cluster, const GrapplerItem& item,
 
 REGISTER_GRAPH_OPTIMIZER_AS(MapAndBatchFusion, "map_and_batch_fusion");
 
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace grappler
+}  // namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/data/map_and_batch_fusion.h b/tensorflow/core/grappler/optimizers/data/map_and_batch_fusion.h
index 2c64831105..ef3a218bf3 100644
--- a/tensorflow/core/grappler/optimizers/data/map_and_batch_fusion.h
+++ b/tensorflow/core/grappler/optimizers/data/map_and_batch_fusion.h
@@ -16,12 +16,12 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_MAP_AND_BATCH_FUSION_H_
 #define TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_MAP_AND_BATCH_FUSION_H_
 
-#include "tensorflow/core/grappler/optimizers/custom_graph_optimizer.h"
+#include "tensorflow/core/grappler/optimizers/data/optimizer_base.h"
 
 namespace tensorflow {
 namespace grappler {
 
-class MapAndBatchFusion : public CustomGraphOptimizer {
+class MapAndBatchFusion : public TFDataOptimizerBase {
  public:
   MapAndBatchFusion() = default;
   ~MapAndBatchFusion() override = default;
@@ -33,14 +33,15 @@ class MapAndBatchFusion : public CustomGraphOptimizer {
     return Status::OK();
   }
 
-  Status Optimize(Cluster* cluster, const GrapplerItem& item,
-                  GraphDef* output) override;
+  Status OptimizeAndCollectStats(Cluster* cluster, const GrapplerItem& item,
+                                 GraphDef* output,
+                                 OptimizationStats* stats) override;
 
   void Feedback(Cluster* cluster, const GrapplerItem& item,
                 const GraphDef& optimize_output, double result) override;
 };
 
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace grappler
+}  // namespace tensorflow
 
 #endif  // TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_MAP_AND_BATCH_FUSION_H_
diff --git a/tensorflow/core/grappler/optimizers/data/map_and_filter_fusion.cc b/tensorflow/core/grappler/optimizers/data/map_and_filter_fusion.cc
index 128a2fefa2..1ed4eae4ba 100644
--- a/tensorflow/core/grappler/optimizers/data/map_and_filter_fusion.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_and_filter_fusion.cc
@@ -92,8 +92,10 @@ NodeDef MakeFilterByLastComponentNode(const NodeDef& fused_map_node,
 
 }  // namespace
 
-Status MapAndFilterFusion::Optimize(Cluster* cluster, const GrapplerItem& item,
-                                    GraphDef* output) {
+Status MapAndFilterFusion::OptimizeAndCollectStats(Cluster* cluster,
+                                                   const GrapplerItem& item,
+                                                   GraphDef* output,
+                                                   OptimizationStats* stats) {
   GraphDef sorted_old_graph = item.graph;
   TF_RETURN_IF_ERROR(TopologicalSort(&sorted_old_graph));
   // TODO(prazek): We might have some problems with performance if we copy
@@ -163,6 +165,7 @@ Status MapAndFilterFusion::Optimize(Cluster* cluster, const GrapplerItem& item,
     // used anymore.
     nodes_to_delete.insert(map_node->name());
     nodes_to_delete.insert(filter_node->name());
+    stats->num_changes++;
   }
 
   graph.DeleteNodes(nodes_to_delete);
@@ -177,5 +180,5 @@ void MapAndFilterFusion::Feedback(Cluster* cluster, const GrapplerItem& item,
 
 REGISTER_GRAPH_OPTIMIZER_AS(MapAndFilterFusion, "map_and_filter_fusion");
 
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace grappler
+}  // namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/data/map_and_filter_fusion.h b/tensorflow/core/grappler/optimizers/data/map_and_filter_fusion.h
index ba25ca0591..8b3c95d37c 100644
--- a/tensorflow/core/grappler/optimizers/data/map_and_filter_fusion.h
+++ b/tensorflow/core/grappler/optimizers/data/map_and_filter_fusion.h
@@ -16,7 +16,7 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_MAP_AND_FILTER_FUSION_H_
 #define TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_MAP_AND_FILTER_FUSION_H_
 
-#include "tensorflow/core/grappler/optimizers/custom_graph_optimizer.h"
+#include "tensorflow/core/grappler/optimizers/data/optimizer_base.h"
 
 namespace tensorflow {
 namespace grappler {
@@ -26,7 +26,7 @@ namespace grappler {
 // component. The FilterDataset is transformed to FilterByLastComponent - a
 // custom kernel that filters elements based on a value of the boolean
 // component.
-class MapAndFilterFusion : public CustomGraphOptimizer {
+class MapAndFilterFusion : public TFDataOptimizerBase {
  public:
   MapAndFilterFusion() = default;
   ~MapAndFilterFusion() override = default;
@@ -38,14 +38,15 @@ class MapAndFilterFusion : public CustomGraphOptimizer {
     return Status::OK();
   }
 
-  Status Optimize(Cluster* cluster, const GrapplerItem& item,
-                  GraphDef* output) override;
+  Status OptimizeAndCollectStats(Cluster* cluster, const GrapplerItem& item,
+                                 GraphDef* output,
+                                 OptimizationStats* stats) override;
 
   void Feedback(Cluster* cluster, const GrapplerItem& item,
                 const GraphDef& optimize_output, double result) override;
 };
 
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace grappler
+}  // namespace tensorflow
 
 #endif  // TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_MAP_AND_FILTER_FUSION_H_
diff --git a/tensorflow/core/grappler/optimizers/data/map_fusion.cc b/tensorflow/core/grappler/optimizers/data/map_fusion.cc
index 504fcc1f5e..66f8404c05 100644
--- a/tensorflow/core/grappler/optimizers/data/map_fusion.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_fusion.cc
@@ -77,8 +77,10 @@ NodeDef MakeFusedNode(const NodeDef& parent_map_node, const NodeDef& map_node,
 
 }  // namespace
 
-Status MapFusion::Optimize(Cluster* cluster, const GrapplerItem& item,
-                           GraphDef* output) {
+Status MapFusion::OptimizeAndCollectStats(Cluster* cluster,
+                                          const GrapplerItem& item,
+                                          GraphDef* output,
+                                          OptimizationStats* stats) {
   GraphDef sorted_old_graph = item.graph;
   TF_RETURN_IF_ERROR(TopologicalSort(&sorted_old_graph));
   *output = sorted_old_graph;
@@ -142,6 +144,7 @@ Status MapFusion::Optimize(Cluster* cluster, const GrapplerItem& item,
     // they are not used anymore.
     nodes_to_delete.insert(parent_map_node->name());
     nodes_to_delete.insert(map_node->name());
+    stats->num_changes++;
   }
 
   graph.DeleteNodes(nodes_to_delete);
@@ -155,5 +158,5 @@ void MapFusion::Feedback(Cluster* cluster, const GrapplerItem& item,
 
 REGISTER_GRAPH_OPTIMIZER_AS(MapFusion, "map_fusion");
 
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace grappler
+}  // namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/data/map_fusion.h b/tensorflow/core/grappler/optimizers/data/map_fusion.h
index a6a06592b8..c9960c7217 100644
--- a/tensorflow/core/grappler/optimizers/data/map_fusion.h
+++ b/tensorflow/core/grappler/optimizers/data/map_fusion.h
@@ -16,13 +16,13 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_MAP_FUSION_H_
 #define TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_MAP_FUSION_H_
 
-#include "tensorflow/core/grappler/optimizers/custom_graph_optimizer.h"
+#include "tensorflow/core/grappler/optimizers/data/optimizer_base.h"
 
 namespace tensorflow {
 namespace grappler {
 
 // This optimization fuses map transformations by merging their map functions.
-class MapFusion : public CustomGraphOptimizer {
+class MapFusion : public TFDataOptimizerBase {
  public:
   MapFusion() = default;
   ~MapFusion() override = default;
@@ -34,14 +34,15 @@ class MapFusion : public CustomGraphOptimizer {
     return Status::OK();
   }
 
-  Status Optimize(Cluster* cluster, const GrapplerItem& item,
-                  GraphDef* output) override;
+  Status OptimizeAndCollectStats(Cluster* cluster, const GrapplerItem& item,
+                                 GraphDef* output,
+                                 OptimizationStats* stats) override;
 
   void Feedback(Cluster* cluster, const GrapplerItem& item,
                 const GraphDef& optimize_output, double result) override;
 };
 
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace grappler
+}  // namespace tensorflow
 
 #endif  // TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_MAP_FUSION_H_
diff --git a/tensorflow/core/grappler/optimizers/data/map_parallelization.cc b/tensorflow/core/grappler/optimizers/data/map_parallelization.cc
index 1274af7160..d8f01d720a 100644
--- a/tensorflow/core/grappler/optimizers/data/map_parallelization.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_parallelization.cc
@@ -28,9 +28,9 @@ namespace tensorflow {
 namespace grappler {
 namespace {
 
-static const char* kMapDataset = "MapDataset";
-static const char* kParallelMapDataset = "ParallelMapDataset";
-static constexpr int kAutotune = -1;
+constexpr char kMapDataset[] = "MapDataset";
+constexpr char kParallelMapDataset[] = "ParallelMapDataset";
+constexpr int kAutotune = -1;
 
 bool CanParallelize(const FunctionDef& function,
                     const FunctionLibraryDefinition& library) {
@@ -67,8 +67,10 @@ NodeDef MakeParallelMap(const string& name, MutableGraphView* graph) {
 
 }  // namespace
 
-Status MapParallelization::Optimize(Cluster* cluster, const GrapplerItem& item,
-                                    GraphDef* output) {
+Status MapParallelization::OptimizeAndCollectStats(Cluster* cluster,
+                                                   const GrapplerItem& item,
+                                                   GraphDef* output,
+                                                   OptimizationStats* stats) {
   *output = item.graph;
   MutableGraphView graph(output);
   std::set<string> nodes_to_delete;
@@ -92,6 +94,7 @@ Status MapParallelization::Optimize(Cluster* cluster, const GrapplerItem& item,
     TF_RETURN_IF_ERROR(
         graph.UpdateFanouts(map_node->name(), parallel_map->name()));
     nodes_to_delete.insert(map_node->name());
+    stats->num_changes++;
   }
 
   graph.DeleteNodes(nodes_to_delete);
@@ -106,5 +109,5 @@ void MapParallelization::Feedback(Cluster* cluster, const GrapplerItem& item,
 
 REGISTER_GRAPH_OPTIMIZER_AS(MapParallelization, "map_parallelization");
 
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace grappler
+}  // namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/data/map_parallelization.h b/tensorflow/core/grappler/optimizers/data/map_parallelization.h
index ac9cf7e12a..8e71dadcb8 100644
--- a/tensorflow/core/grappler/optimizers/data/map_parallelization.h
+++ b/tensorflow/core/grappler/optimizers/data/map_parallelization.h
@@ -16,13 +16,13 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_MAP_PARALLELIZATION_H_
 #define TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_MAP_PARALLELIZATION_H_
 
-#include "tensorflow/core/grappler/optimizers/custom_graph_optimizer.h"
+#include "tensorflow/core/grappler/optimizers/data/optimizer_base.h"
 
 namespace tensorflow {
 namespace grappler {
 
 // This optimization parallelizes MapDataset when function is stateless.
-class MapParallelization : public CustomGraphOptimizer {
+class MapParallelization : public TFDataOptimizerBase {
  public:
   MapParallelization() = default;
   ~MapParallelization() override = default;
@@ -34,14 +34,15 @@ class MapParallelization : public CustomGraphOptimizer {
     return Status::OK();
   }
 
-  Status Optimize(Cluster* cluster, const GrapplerItem& item,
-                  GraphDef* output) override;
+  Status OptimizeAndCollectStats(Cluster* cluster, const GrapplerItem& item,
+                                 GraphDef* output,
+                                 OptimizationStats* stats) override;
 
   void Feedback(Cluster* cluster, const GrapplerItem& item,
                 const GraphDef& optimize_output, double result) override;
 };
 
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace grappler
+}  // namespace tensorflow
 
 #endif  // TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_MAP_PARALLELIZATION_H_
diff --git a/tensorflow/core/grappler/optimizers/data/map_vectorization.cc b/tensorflow/core/grappler/optimizers/data/map_vectorization.cc
index bc1d6d0ea6..848642041b 100644
--- a/tensorflow/core/grappler/optimizers/data/map_vectorization.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_vectorization.cc
@@ -212,8 +212,10 @@ NodeDef MakeNewMapNode(const NodeDef& old_map_node,
 
 }  // namespace
 
-Status MapVectorization::Optimize(Cluster* cluster, const GrapplerItem& item,
-                                  GraphDef* output) {
+Status MapVectorization::OptimizeAndCollectStats(Cluster* cluster,
+                                                 const GrapplerItem& item,
+                                                 GraphDef* output,
+                                                 OptimizationStats* stats) {
   *output = item.graph;
   MutableGraphView graph(output);
   std::set<string> nodes_to_delete;
@@ -270,6 +272,7 @@ Status MapVectorization::Optimize(Cluster* cluster, const GrapplerItem& item,
     // Mark the `Map` and `Batch` nodes for removal.
     nodes_to_delete.insert(map_node->name());
     nodes_to_delete.insert(batch_node.name());
+    stats->num_changes++;
   }
   graph.DeleteNodes(nodes_to_delete);
   return Status::OK();
@@ -283,5 +286,5 @@ void MapVectorization::Feedback(Cluster* cluster, const GrapplerItem& item,
 
 REGISTER_GRAPH_OPTIMIZER_AS(MapVectorization, "map_vectorization");
 
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace grappler
+}  // namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/data/map_vectorization.h b/tensorflow/core/grappler/optimizers/data/map_vectorization.h
index cc56a8ee5e..cde6ef6839 100644
--- a/tensorflow/core/grappler/optimizers/data/map_vectorization.h
+++ b/tensorflow/core/grappler/optimizers/data/map_vectorization.h
@@ -16,12 +16,12 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_MAP_VECTORIZATION_H_
 #define TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_MAP_VECTORIZATION_H_
 
-#include "tensorflow/core/grappler/optimizers/custom_graph_optimizer.h"
+#include "tensorflow/core/grappler/optimizers/data/optimizer_base.h"
 
 namespace tensorflow {
 namespace grappler {
 
-class MapVectorization : public CustomGraphOptimizer {
+class MapVectorization : public TFDataOptimizerBase {
  public:
   MapVectorization() = default;
   ~MapVectorization() override = default;
@@ -33,14 +33,15 @@ class MapVectorization : public CustomGraphOptimizer {
     return Status::OK();
   }
 
-  Status Optimize(Cluster* cluster, const GrapplerItem& item,
-                  GraphDef* output) override;
+  Status OptimizeAndCollectStats(Cluster* cluster, const GrapplerItem& item,
+                                 GraphDef* output,
+                                 OptimizationStats* stats) override;
 
   void Feedback(Cluster* cluster, const GrapplerItem& item,
                 const GraphDef& optimize_output, double result) override;
 };
 
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace grappler
+}  // namespace tensorflow
 
 #endif  // TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_MAP_VECTORIZATION_H_
diff --git a/tensorflow/core/grappler/optimizers/data/meta_optimizer.cc b/tensorflow/core/grappler/optimizers/data/meta_optimizer.cc
index 0fc0cf43c6..584759f85d 100644
--- a/tensorflow/core/grappler/optimizers/data/meta_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/data/meta_optimizer.cc
@@ -121,5 +121,5 @@ void TFDataMetaOptimizer::Feedback(Cluster* cluster, const GrapplerItem& item,
 
 REGISTER_GRAPH_OPTIMIZER_AS(TFDataMetaOptimizer, "tf_data_meta_optimizer");
 
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace grappler
+}  // namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/data/meta_optimizer.h b/tensorflow/core/grappler/optimizers/data/meta_optimizer.h
index c39ddda4cb..b65e702777 100644
--- a/tensorflow/core/grappler/optimizers/data/meta_optimizer.h
+++ b/tensorflow/core/grappler/optimizers/data/meta_optimizer.h
@@ -50,7 +50,7 @@ class TFDataMetaOptimizer : public CustomGraphOptimizer {
                            GrapplerItem* item) const;
 };
 
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace grappler
+}  // namespace tensorflow
 
 #endif  // TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_META_OPTIMIZER_H_
diff --git a/tensorflow/core/grappler/optimizers/data/noop_elimination.cc b/tensorflow/core/grappler/optimizers/data/noop_elimination.cc
index b71e9e5420..e474fddcc0 100644
--- a/tensorflow/core/grappler/optimizers/data/noop_elimination.cc
+++ b/tensorflow/core/grappler/optimizers/data/noop_elimination.cc
@@ -70,8 +70,10 @@ bool IsNoOp(const NodeDef& node, const MutableGraphView& graph) {
 
 }  // namespace
 
-Status NoOpElimination::Optimize(Cluster* cluster, const GrapplerItem& item,
-                                 GraphDef* output) {
+Status NoOpElimination::OptimizeAndCollectStats(Cluster* cluster,
+                                                const GrapplerItem& item,
+                                                GraphDef* output,
+                                                OptimizationStats* stats) {
   *output = item.graph;
   MutableGraphView graph(output);
   std::set<string> nodes_to_delete;
@@ -82,6 +84,7 @@ Status NoOpElimination::Optimize(Cluster* cluster, const GrapplerItem& item,
     TF_RETURN_IF_ERROR(graph.UpdateFanouts(node.name(), parent->name()));
 
     nodes_to_delete.insert(node.name());
+    stats->num_changes++;
   }
 
   graph.DeleteNodes(nodes_to_delete);
@@ -95,5 +98,5 @@ void NoOpElimination::Feedback(Cluster* cluster, const GrapplerItem& item,
 
 REGISTER_GRAPH_OPTIMIZER_AS(NoOpElimination, "noop_elimination");
 
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace grappler
+}  // namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/data/noop_elimination.h b/tensorflow/core/grappler/optimizers/data/noop_elimination.h
index a65fccd882..11d86ad2a3 100644
--- a/tensorflow/core/grappler/optimizers/data/noop_elimination.h
+++ b/tensorflow/core/grappler/optimizers/data/noop_elimination.h
@@ -16,14 +16,14 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_NOOP_ELIMINATION_H_
 #define TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_NOOP_ELIMINATION_H_
 
-#include "tensorflow/core/grappler/optimizers/custom_graph_optimizer.h"
+#include "tensorflow/core/grappler/optimizers/data/optimizer_base.h"
 
 namespace tensorflow {
 namespace grappler {
 
 // This class eliminates tf.data transformations such as `take(n)` (for n < 0),
 // `skip(0)`, `repeat(1)`, or `prefetch(0)`.
-class NoOpElimination : public CustomGraphOptimizer {
+class NoOpElimination : public TFDataOptimizerBase {
  public:
   NoOpElimination() = default;
   ~NoOpElimination() override = default;
@@ -35,14 +35,15 @@ class NoOpElimination : public CustomGraphOptimizer {
     return Status::OK();
   }
 
-  Status Optimize(Cluster* cluster, const GrapplerItem& item,
-                  GraphDef* output) override;
+  Status OptimizeAndCollectStats(Cluster* cluster, const GrapplerItem& item,
+                                 GraphDef* output,
+                                 OptimizationStats* stats) override;
 
   void Feedback(Cluster* cluster, const GrapplerItem& item,
                 const GraphDef& optimize_output, double result) override;
 };
 
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace grappler
+}  // namespace tensorflow
 
 #endif  // TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_NOOP_ELIMINATION_H_
diff --git a/tensorflow/core/kernels/data/model_dataset_op.h b/tensorflow/core/grappler/optimizers/data/optimizer_base.cc
similarity index 51%
rename from tensorflow/core/kernels/data/model_dataset_op.h
rename to tensorflow/core/grappler/optimizers/data/optimizer_base.cc
index 38d3fcb234..7fc0da3579 100644
--- a/tensorflow/core/kernels/data/model_dataset_op.h
+++ b/tensorflow/core/grappler/optimizers/data/optimizer_base.cc
@@ -13,30 +13,22 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_CORE_KERNELS_DATA_MODEL_DATASET_OP_H_
-#define TENSORFLOW_CORE_KERNELS_DATA_MODEL_DATASET_OP_H_
+#include "tensorflow/core/grappler/optimizers/data/optimizer_base.h"
 
-#include "tensorflow/core/framework/dataset.h"
-#include "tensorflow/core/framework/model.h"
+#include "tensorflow/core/common_runtime/metrics.h"
 
 namespace tensorflow {
-namespace data {
-
-class ModelDatasetOp : public UnaryDatasetOpKernel {
- public:
-  using ModelFactory = std::function<std::shared_ptr<model::Model>()>;
-
-  explicit ModelDatasetOp(OpKernelConstruction* ctx);
-
-  void MakeDataset(OpKernelContext* ctx, DatasetBase* input,
-                   DatasetBase** output) override;
-
-  virtual ModelFactory CreateModelFactory() const;
-
- private:
-  class Dataset;
-};
-
-}  // namespace data
+namespace grappler {
+
+Status TFDataOptimizerBase::Optimize(Cluster* cluster, const GrapplerItem& item,
+                                     GraphDef* output) {
+  OptimizationStats stats;
+  Status s = OptimizeAndCollectStats(cluster, item, output, &stats);
+  if (s.ok() && stats.num_changes > 0) {
+    metrics::RecordTFDataOptimization(name(), stats.num_changes);
+  }
+  return s;
+}
+
+}  // namespace grappler
 }  // namespace tensorflow
-#endif  // TENSORFLOW_CORE_KERNELS_DATA_MODEL_DATASET_OP_H_
diff --git a/tensorflow/core/grappler/optimizers/data/optimizer_base.h b/tensorflow/core/grappler/optimizers/data/optimizer_base.h
new file mode 100644
index 0000000000..45af5a4b7d
--- /dev/null
+++ b/tensorflow/core/grappler/optimizers/data/optimizer_base.h
@@ -0,0 +1,47 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_OPTIMIZER_BASE_H_
+#define TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_OPTIMIZER_BASE_H_
+
+#include "tensorflow/core/grappler/optimizers/custom_graph_optimizer.h"
+
+namespace tensorflow {
+namespace grappler {
+
+// A base class for tf.data optimizers.
+class TFDataOptimizerBase : public CustomGraphOptimizer {
+ public:
+  struct OptimizationStats {
+    // Identifies the number of independent graph changes for an optimization.
+    int64 num_changes = 0;
+  };
+
+  TFDataOptimizerBase() = default;
+  ~TFDataOptimizerBase() override = default;
+
+  Status Optimize(Cluster* cluster, const GrapplerItem& item,
+                  GraphDef* output) final;
+
+  virtual Status OptimizeAndCollectStats(Cluster* cluster,
+                                         const GrapplerItem& item,
+                                         GraphDef* output,
+                                         OptimizationStats* stats) = 0;
+};
+
+}  // namespace grappler
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_OPTIMIZER_BASE_H_
diff --git a/tensorflow/core/grappler/optimizers/data/shuffle_and_repeat_fusion.cc b/tensorflow/core/grappler/optimizers/data/shuffle_and_repeat_fusion.cc
index f03648e7e9..f30b2230ff 100644
--- a/tensorflow/core/grappler/optimizers/data/shuffle_and_repeat_fusion.cc
+++ b/tensorflow/core/grappler/optimizers/data/shuffle_and_repeat_fusion.cc
@@ -34,9 +34,9 @@ constexpr char kFusedOpName[] = "ShuffleAndRepeatDataset";
 
 }  // namespace
 
-Status ShuffleAndRepeatFusion::Optimize(Cluster* cluster,
-                                        const GrapplerItem& item,
-                                        GraphDef* output) {
+Status ShuffleAndRepeatFusion::OptimizeAndCollectStats(
+    Cluster* cluster, const GrapplerItem& item, GraphDef* output,
+    OptimizationStats* stats) {
   *output = item.graph;
   MutableGraphView graph(output);
   std::set<string> nodes_to_delete;
@@ -92,6 +92,7 @@ Status ShuffleAndRepeatFusion::Optimize(Cluster* cluster,
     // Mark the `Shuffle` and `Repeat` nodes for removal.
     nodes_to_delete.insert(shuffle_node.name());
     nodes_to_delete.insert(repeat_node.name());
+    stats->num_changes++;
   }
 
   graph.DeleteNodes(nodes_to_delete);
@@ -108,5 +109,5 @@ void ShuffleAndRepeatFusion::Feedback(Cluster* cluster,
 REGISTER_GRAPH_OPTIMIZER_AS(ShuffleAndRepeatFusion,
                             "shuffle_and_repeat_fusion");
 
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace grappler
+}  // namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/data/shuffle_and_repeat_fusion.h b/tensorflow/core/grappler/optimizers/data/shuffle_and_repeat_fusion.h
index c8fa53edce..3738d141c3 100644
--- a/tensorflow/core/grappler/optimizers/data/shuffle_and_repeat_fusion.h
+++ b/tensorflow/core/grappler/optimizers/data/shuffle_and_repeat_fusion.h
@@ -16,12 +16,12 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_SHUFFLE_AND_REPEAT_FUSION_H_
 #define TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_SHUFFLE_AND_REPEAT_FUSION_H_
 
-#include "tensorflow/core/grappler/optimizers/custom_graph_optimizer.h"
+#include "tensorflow/core/grappler/optimizers/data/optimizer_base.h"
 
 namespace tensorflow {
 namespace grappler {
 
-class ShuffleAndRepeatFusion : public CustomGraphOptimizer {
+class ShuffleAndRepeatFusion : public TFDataOptimizerBase {
  public:
   ShuffleAndRepeatFusion() = default;
   ~ShuffleAndRepeatFusion() override = default;
@@ -33,14 +33,15 @@ class ShuffleAndRepeatFusion : public CustomGraphOptimizer {
     return Status::OK();
   }
 
-  Status Optimize(Cluster* cluster, const GrapplerItem& item,
-                  GraphDef* output) override;
+  Status OptimizeAndCollectStats(Cluster* cluster, const GrapplerItem& item,
+                                 GraphDef* output,
+                                 OptimizationStats* stats) override;
 
   void Feedback(Cluster* cluster, const GrapplerItem& item,
                 const GraphDef& optimize_output, double result) override;
 };
 
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace grappler
+}  // namespace tensorflow
 
 #endif  // TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_SHUFFLE_AND_REPEAT_FUSION_H_
diff --git a/tensorflow/core/grappler/optimizers/data/vectorization_utils.cc b/tensorflow/core/grappler/optimizers/data/vectorization_utils.cc
index 60c557d557..c57a7b1256 100644
--- a/tensorflow/core/grappler/optimizers/data/vectorization_utils.cc
+++ b/tensorflow/core/grappler/optimizers/data/vectorization_utils.cc
@@ -46,7 +46,7 @@ namespace {
 // Describes a tensor with its operation Node and output position
 typedef std::pair<Node*, int> TensorDesc;
 
-const char* const kRetValOp = "_Retval";
+constexpr char kRetValOp[] = "_Retval";
 
 void ReplaceEdgeSources(const TensorDesc& old_src, const TensorDesc& new_src,
                         Graph* graph) {
@@ -643,6 +643,6 @@ Status VectorizeMapDefun(const FunctionDef& outer_scope,
   return Vectorization(lib).Vectorize(outer_scope, map_defun_node, result);
 }
 
-}  // end namespace vectorization_utils
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace vectorization_utils
+}  // namespace grappler
+}  // namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/data/vectorization_utils.h b/tensorflow/core/grappler/optimizers/data/vectorization_utils.h
index bd7d390900..f5183fd4ff 100644
--- a/tensorflow/core/grappler/optimizers/data/vectorization_utils.h
+++ b/tensorflow/core/grappler/optimizers/data/vectorization_utils.h
@@ -90,8 +90,8 @@ Status VectorizeMapDefun(const FunctionDef& outer_scope,
                          const NodeDef& map_defun_node, FunctionDefLibrary* lib,
                          FunctionDef** result);
 
-}  // end namespace vectorization_utils
-}  // end namespace grappler
-}  // end namespace tensorflow
+}  // namespace vectorization_utils
+}  // namespace grappler
+}  // namespace tensorflow
 
 #endif  // TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_VECTORIZATION_UTILS_H_
diff --git a/tensorflow/core/kernels/data/BUILD b/tensorflow/core/kernels/data/BUILD
index 663717ffb0..fab1ec8b6e 100644
--- a/tensorflow/core/kernels/data/BUILD
+++ b/tensorflow/core/kernels/data/BUILD
@@ -54,7 +54,6 @@ cc_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/core:protos_all_cc",
         "//tensorflow/core/kernels:variable_ops",
     ],
 )
@@ -189,7 +188,6 @@ cc_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/core:protos_all_cc",
     ],
 )
 
@@ -205,6 +203,7 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:metrics",
         "//tensorflow/core:protos_all_cc",
     ],
 )
@@ -262,6 +261,7 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:metrics",
     ],
 )
 
@@ -295,6 +295,7 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:metrics",
         "//tensorflow/core:protos_all_cc",
     ],
 )
@@ -488,6 +489,7 @@ tf_kernel_library(
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:dataset_ops_op_lib",
         "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core/grappler:graph_view",
@@ -504,7 +506,6 @@ tf_kernel_library(
 tf_kernel_library(
     name = "model_dataset_op",
     srcs = ["model_dataset_op.cc"],
-    hdrs = ["model_dataset_op.h"],
     deps = [
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:dataset_ops_op_lib",
@@ -512,6 +513,8 @@ tf_kernel_library(
         "//tensorflow/core:framework_internal",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:metrics",
+        "@com_google_absl//absl/memory",
     ],
 )
 
diff --git a/tensorflow/core/kernels/data/experimental/BUILD b/tensorflow/core/kernels/data/experimental/BUILD
index 7433303f77..583fce6d89 100644
--- a/tensorflow/core/kernels/data/experimental/BUILD
+++ b/tensorflow/core/kernels/data/experimental/BUILD
@@ -124,6 +124,7 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:metrics",
         "//tensorflow/core/kernels:inplace_ops",
         "//tensorflow/core/kernels/data:captured_function",
         "//tensorflow/core/kernels/data:dataset_utils",
diff --git a/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc b/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc
index 5d8e7cfabe..0ef56915eb 100644
--- a/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc
+++ b/tensorflow/core/kernels/data/experimental/map_and_batch_dataset_op.cc
@@ -18,6 +18,7 @@ limitations under the License.
 #include <utility>
 
 #include "tensorflow/core/common_runtime/function.h"
+#include "tensorflow/core/common_runtime/metrics.h"
 #include "tensorflow/core/framework/dataset.h"
 #include "tensorflow/core/framework/partial_tensor_shape.h"
 #include "tensorflow/core/framework/stats_aggregator.h"
@@ -37,8 +38,10 @@ namespace tensorflow {
 namespace data {
 namespace {
 
+constexpr char kDatasetName[] = "MapAndBatch";
+
 // Maximum number of batch results to buffer.
-const int64 kMaxBatchResults = 16;
+constexpr int64 kMaxBatchResults = 16;
 
 // See documentation in ../../ops/dataset_ops.cc for a high-level
 // description of the following op.
@@ -126,6 +129,10 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
       };
     }
 
+    if (num_parallel_calls == model::kAutoTune) {
+      metrics::RecordTFDataAutotune(kDatasetName);
+    }
+
     *output = new Dataset(ctx, input, func_, batch_size, num_parallel_calls,
                           drop_remainder, output_types_, output_shapes_,
                           std::move(captured_func), &ctx->eigen_cpu_device(),
@@ -163,7 +170,7 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
     std::unique_ptr<IteratorBase> MakeIteratorInternal(
         const string& prefix) const override {
       return absl::make_unique<Iterator>(
-          Iterator::Params{this, strings::StrCat(prefix, "::MapAndBatch")},
+          Iterator::Params{this, strings::StrCat(prefix, "::", kDatasetName)},
           map_func_);
     }
 
diff --git a/tensorflow/core/kernels/data/model_dataset_op.cc b/tensorflow/core/kernels/data/model_dataset_op.cc
index e1a0d94f5f..20254234e9 100644
--- a/tensorflow/core/kernels/data/model_dataset_op.cc
+++ b/tensorflow/core/kernels/data/model_dataset_op.cc
@@ -13,180 +13,182 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/core/kernels/data/model_dataset_op.h"
-
+#include "absl/memory/memory.h"
+#include "tensorflow/core/common_runtime/metrics.h"
 #include "tensorflow/core/framework/dataset.h"
 #include "tensorflow/core/framework/partial_tensor_shape.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/lib/random/random.h"
 #include "tensorflow/core/platform/cpu_info.h"
+#include "tensorflow/core/util/ptr_util.h"
 
 namespace tensorflow {
 namespace data {
+namespace {
 
-const int kOptimizationPeriodThresholdMs = 60 * EnvTime::kSecondsToMicros;
+constexpr int kOptimizationPeriodThresholdMs = 60 * EnvTime::kSecondsToMicros;
 
-class ModelDatasetOp::Dataset : public DatasetBase {
+class ModelDatasetOp : public UnaryDatasetOpKernel {
  public:
-  explicit Dataset(OpKernelContext* ctx, const DatasetBase* input,
-                   ModelFactory model_factory)
-      : DatasetBase(DatasetContext(ctx)),
-        input_(input),
-        model_factory_(std::move(model_factory)) {
-    input_->Ref();
-  }
-
-  ~Dataset() override { input_->Unref(); }
+  explicit ModelDatasetOp(OpKernelConstruction* ctx)
+      : UnaryDatasetOpKernel(ctx) {}
 
-  std::unique_ptr<IteratorBase> MakeIteratorInternal(
-      const string& prefix) const override {
-    return std::unique_ptr<IteratorBase>(new Iterator(
-        {this, strings::StrCat(prefix, "::Model")}, model_factory_));
-  }
-
-  const DataTypeVector& output_dtypes() const override {
-    return input_->output_dtypes();
-  }
-  const std::vector<PartialTensorShape>& output_shapes() const override {
-    return input_->output_shapes();
-  }
-
-  string DebugString() const override { return "ModelDatasetOp::Dataset"; }
-
-  int64 Cardinality() const override { return input_->Cardinality(); }
-
- protected:
-  Status AsGraphDefInternal(SerializationContext* ctx,
-                            DatasetGraphDefBuilder* b,
-                            Node** output) const override {
-    Node* input_graph_node = nullptr;
-    TF_RETURN_IF_ERROR(b->AddInputDataset(ctx, input_, &input_graph_node));
-    TF_RETURN_IF_ERROR(b->AddDataset(this, {input_graph_node}, output));
-    return Status::OK();
+  void MakeDataset(OpKernelContext* ctx, DatasetBase* input,
+                   DatasetBase** output) override {
+    *output = new Dataset(ctx, input);
   }
 
  private:
-  class Iterator : public DatasetIterator<Dataset> {
+  class Dataset : public DatasetBase {
    public:
-    explicit Iterator(const Params& params, ModelFactory model_factory)
-        : DatasetIterator<Dataset>(params), model_(model_factory()) {}
-
-    ~Iterator() override {
-      // Signal the optimize thread to terminate it. We will then join that
-      // thread when we delete `this->optimize_thread_`.
-      mutex_lock l(mu_);
-      cancelled_ = true;
-      cond_var_.notify_all();
+    Dataset(OpKernelContext* ctx, const DatasetBase* input)
+        : DatasetBase(DatasetContext(ctx)), input_(input) {
+      input_->Ref();
     }
 
-    Status Initialize(IteratorContext* ctx) override {
-      IteratorContext::Params params(ctx);
-      params.model = model_;
-      return dataset()->input_->MakeIterator(IteratorContext(std::move(params)),
-                                             prefix(), &input_impl_);
-    }
+    ~Dataset() override { input_->Unref(); }
 
-    Status GetNextInternal(IteratorContext* ctx,
-                           std::vector<Tensor>* out_tensors,
-                           bool* end_of_sequence) override {
-      IteratorContext::Params params(ctx);
-      {
-        mutex_lock l(mu_);
-        TF_RETURN_IF_ERROR(EnsureOptimizeThreadStarted(ctx));
-        params.model = model_;
-      }
-      return input_impl_->GetNext(IteratorContext(std::move(params)),
-                                  out_tensors, end_of_sequence);
+    std::unique_ptr<IteratorBase> MakeIteratorInternal(
+        const string& prefix) const override {
+      return absl::make_unique<Iterator>(
+          Iterator::Params{this, strings::StrCat(prefix, "::Model")});
     }
 
-   protected:
-    std::shared_ptr<model::Node> CreateNode(
-        IteratorContext* ctx, model::Node::Args args) const override {
-      return model::MakeKnownRatioNode(std::move(args),
-                                       /*ratio=*/1);
+    const DataTypeVector& output_dtypes() const override {
+      return input_->output_dtypes();
     }
-
-    Status SaveInternal(IteratorStateWriter* writer) override {
-      mutex_lock l(mu_);
-      TF_RETURN_IF_ERROR(SaveInput(writer, input_impl_));
-      return Status::OK();
+    const std::vector<PartialTensorShape>& output_shapes() const override {
+      return input_->output_shapes();
     }
 
-    Status RestoreInternal(IteratorContext* ctx,
-                           IteratorStateReader* reader) override {
-      mutex_lock l(mu_);
-      TF_RETURN_IF_ERROR(RestoreInput(ctx, reader, input_impl_));
+    string DebugString() const override { return "ModelDatasetOp::Dataset"; }
+
+    int64 Cardinality() const override { return input_->Cardinality(); }
+
+   protected:
+    Status AsGraphDefInternal(SerializationContext* ctx,
+                              DatasetGraphDefBuilder* b,
+                              Node** output) const override {
+      Node* input_graph_node = nullptr;
+      TF_RETURN_IF_ERROR(b->AddInputDataset(ctx, input_, &input_graph_node));
+      TF_RETURN_IF_ERROR(b->AddDataset(this, {input_graph_node}, output));
       return Status::OK();
     }
 
    private:
-    Status EnsureOptimizeThreadStarted(IteratorContext* ctx)
-        EXCLUSIVE_LOCKS_REQUIRED(mu_) {
-      if (!optimize_thread_) {
-        std::shared_ptr<IteratorContext> new_ctx(new IteratorContext(*ctx));
-        optimize_thread_.reset(ctx->env()->StartThread(
-            {}, "tf_data_model",
-            [this, new_ctx]() { OptimizeThread(new_ctx); }));
+    class Iterator : public DatasetIterator<Dataset> {
+     public:
+      explicit Iterator(const Params& params)
+          : DatasetIterator<Dataset>(params) {
+        auto remove_node_hook = [](std::shared_ptr<model::Node> node) {
+          metrics::RecordTFDataElements(node->name(), node->num_elements());
+        };
+        model_ = std::make_shared<model::Model>(std::move(remove_node_hook));
+      }
+
+      ~Iterator() override {
+        // Signal the optimize thread to terminate it. We will then join that
+        // thread when we delete `this->optimize_thread_`.
+        mutex_lock l(mu_);
+        cancelled_ = true;
+        cond_var_.notify_all();
+      }
+
+      Status Initialize(IteratorContext* ctx) override {
+        IteratorContext::Params params(ctx);
+        params.model = model_;
+        return dataset()->input_->MakeIterator(
+            IteratorContext(std::move(params)), prefix(), &input_impl_);
       }
-      return Status::OK();
-    }
 
-    void OptimizeThread(const std::shared_ptr<IteratorContext>& ctx) {
-      int64 last_optimization_ms = 0;
-      int64 optimization_period_ms = 10;
-      while (true) {
+      Status GetNextInternal(IteratorContext* ctx,
+                             std::vector<Tensor>* out_tensors,
+                             bool* end_of_sequence) override {
+        IteratorContext::Params params(ctx);
         {
           mutex_lock l(mu_);
-          while (!cancelled_ &&
-                 last_optimization_ms + optimization_period_ms >=
-                     ctx->env()->NowMicros() / EnvTime::kMillisToMicros) {
-            cond_var_.wait_for(
-                l, std::chrono::milliseconds(
-                       last_optimization_ms + optimization_period_ms -
-                       ctx->env()->NowMicros() / EnvTime::kMillisToMicros));
-          }
-          if (cancelled_) return;
-        }
-        model_->Optimize(port::NumSchedulableCPUs());
-        // Exponentially increase the period of running the optimization
-        // until a threshold is reached.
-        if (optimization_period_ms < kOptimizationPeriodThresholdMs) {
-          if (optimization_period_ms << 1 < kOptimizationPeriodThresholdMs) {
-            optimization_period_ms <<= 1;
-          } else {
-            optimization_period_ms = kOptimizationPeriodThresholdMs;
-          }
+          TF_RETURN_IF_ERROR(EnsureOptimizeThreadStarted(ctx));
+          params.model = model_;
         }
-        last_optimization_ms =
-            ctx->env()->NowMicros() / EnvTime::kMillisToMicros;
+        return input_impl_->GetNext(IteratorContext(std::move(params)),
+                                    out_tensors, end_of_sequence);
       }
-    }
 
-    mutex mu_;
-    condition_variable cond_var_;
-    std::shared_ptr<model::Model> model_;
-    std::unique_ptr<Thread> optimize_thread_ GUARDED_BY(mu_);
-    bool cancelled_ GUARDED_BY(mu_) = false;
-    std::unique_ptr<IteratorBase> input_impl_;
-  };
+     protected:
+      std::shared_ptr<model::Node> CreateNode(
+          IteratorContext* ctx, model::Node::Args args) const override {
+        return model::MakeKnownRatioNode(std::move(args),
+                                         /*ratio=*/1);
+      }
 
-  const DatasetBase* input_;
-  const ModelFactory model_factory_;
-};
+      Status SaveInternal(IteratorStateWriter* writer) override {
+        mutex_lock l(mu_);
+        TF_RETURN_IF_ERROR(SaveInput(writer, input_impl_));
+        return Status::OK();
+      }
 
-ModelDatasetOp::ModelDatasetOp(OpKernelConstruction* ctx)
-    : UnaryDatasetOpKernel(ctx) {}
+      Status RestoreInternal(IteratorContext* ctx,
+                             IteratorStateReader* reader) override {
+        mutex_lock l(mu_);
+        TF_RETURN_IF_ERROR(RestoreInput(ctx, reader, input_impl_));
+        return Status::OK();
+      }
+
+     private:
+      Status EnsureOptimizeThreadStarted(IteratorContext* ctx)
+          EXCLUSIVE_LOCKS_REQUIRED(mu_) {
+        if (!optimize_thread_) {
+          std::shared_ptr<IteratorContext> new_ctx =
+              std::make_shared<IteratorContext>(*ctx);
+          optimize_thread_.reset(ctx->env()->StartThread(
+              {}, "tf_data_model",
+              [this, new_ctx]() { OptimizeThread(new_ctx); }));
+        }
+        return Status::OK();
+      }
 
-void ModelDatasetOp::MakeDataset(OpKernelContext* ctx, DatasetBase* input,
-                                 DatasetBase** output) {
-  *output = new ModelDatasetOp::Dataset(ctx, input, CreateModelFactory());
-}
+      void OptimizeThread(const std::shared_ptr<IteratorContext>& ctx) {
+        int64 last_optimization_ms = 0;
+        int64 optimization_period_ms = 10;
+        while (true) {
+          {
+            mutex_lock l(mu_);
+            while (!cancelled_ &&
+                   last_optimization_ms + optimization_period_ms >=
+                       ctx->env()->NowMicros() / EnvTime::kMillisToMicros) {
+              cond_var_.wait_for(
+                  l, std::chrono::milliseconds(
+                         last_optimization_ms + optimization_period_ms -
+                         ctx->env()->NowMicros() / EnvTime::kMillisToMicros));
+            }
+            if (cancelled_) return;
+          }
+          model_->Optimize(port::NumSchedulableCPUs());
+          // Exponentially increase the period of running the optimization
+          // until a threshold is reached.
+          if (optimization_period_ms < kOptimizationPeriodThresholdMs) {
+            if (optimization_period_ms << 1 < kOptimizationPeriodThresholdMs) {
+              optimization_period_ms <<= 1;
+            } else {
+              optimization_period_ms = kOptimizationPeriodThresholdMs;
+            }
+          }
+          last_optimization_ms =
+              ctx->env()->NowMicros() / EnvTime::kMillisToMicros;
+        }
+      }
 
-ModelDatasetOp::ModelFactory ModelDatasetOp::CreateModelFactory() const {
-  return []() { return std::make_shared<model::Model>(); };
-}
+      mutex mu_;
+      condition_variable cond_var_;
+      std::shared_ptr<model::Model> model_;
+      std::unique_ptr<Thread> optimize_thread_ GUARDED_BY(mu_);
+      bool cancelled_ GUARDED_BY(mu_) = false;
+      std::unique_ptr<IteratorBase> input_impl_;
+    };
 
-namespace {
+    const DatasetBase* input_;
+  };
+};
 
 REGISTER_KERNEL_BUILDER(Name("ModelDataset").Device(DEVICE_CPU),
                         ModelDatasetOp);
diff --git a/tensorflow/core/kernels/data/optimize_dataset_op.cc b/tensorflow/core/kernels/data/optimize_dataset_op.cc
index 04cc48a0be..6047dc5f3f 100644
--- a/tensorflow/core/kernels/data/optimize_dataset_op.cc
+++ b/tensorflow/core/kernels/data/optimize_dataset_op.cc
@@ -40,7 +40,7 @@ namespace tensorflow {
 namespace data {
 namespace {
 
-static const char* const kOptimizerName = "tf_data_meta_optimizer";
+constexpr char kOptimizerName[] = "tf_data_meta_optimizer";
 
 // See documentation in ../../ops/dataset_ops.cc for a high-level
 // description of the following op.
@@ -99,7 +99,7 @@ class OptimizeDatasetOp : public UnaryDatasetOpKernel {
       // prefix is used to identify checkpoint elements and since the
       // optimization dataset is excluded from the checkpoint, adding a token
       // here would result in invalid checkpoint identifiers.
-      return std::unique_ptr<IteratorBase>(new Iterator({this, prefix}));
+      return absl::make_unique<Iterator>(Iterator::Params{this, prefix});
     }
 
     Status Optimize(OpKernelContext* ctx) {
@@ -129,7 +129,7 @@ class OptimizeDatasetOp : public UnaryDatasetOpKernel {
           ctx->function_library()->Clone(&flib_def_, &pflr_, &lib_));
 
       // Create a FunctionHandleCache.
-      function_handle_cache_.reset(new FunctionHandleCache(lib_));
+      function_handle_cache_ = absl::make_unique<FunctionHandleCache>(lib_);
 
       // Some functions may have been modified without having their names
       // changed (for example, nested dataset graphs from FlatMap or
@@ -301,7 +301,8 @@ class OptimizeDatasetOp : public UnaryDatasetOpKernel {
       RewriterConfig& rewriter_config =
           *config.mutable_graph_options()->mutable_rewrite_options();
       rewriter_config.add_optimizers(kOptimizerName);
-
+      rewriter_config.set_meta_optimizer_iterations(
+          RewriterConfig_NumIterationsType_ONE);
       auto custom_optimizer = rewriter_config.add_custom_optimizers();
       custom_optimizer->set_name(kOptimizerName);
       auto* custom_optimizations_list =
diff --git a/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc b/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc
index add123f34a..ddd81d4596 100644
--- a/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc
+++ b/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc
@@ -18,6 +18,7 @@ limitations under the License.
 #include <utility>
 
 #include "tensorflow/core/common_runtime/function.h"
+#include "tensorflow/core/common_runtime/metrics.h"
 #include "tensorflow/core/framework/dataset.h"
 #include "tensorflow/core/framework/partial_tensor_shape.h"
 #include "tensorflow/core/framework/stats_aggregator.h"
@@ -34,6 +35,8 @@ namespace tensorflow {
 namespace data {
 namespace {
 
+constexpr char kDatasetName[] = "ParallelInterleaveV2";
+
 // See documentation in ../../ops/dataset_ops.cc for a high-level
 // description of the following op.
 
@@ -87,6 +90,10 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
         ctx, CapturedFunction::Create(interleave_func_, ctx, "other_arguments",
                                       &captured_func));
 
+    if (num_parallel_calls == model::kAutoTune) {
+      metrics::RecordTFDataAutotune(kDatasetName);
+    }
+
     *output =
         new Dataset(ctx, input, interleave_func_, std::move(captured_func),
                     cycle_length, block_length, num_parallel_calls, sloppy_,
@@ -121,7 +128,7 @@ class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
         const string& prefix) const override {
       return absl::make_unique<ParallelInterleaveIterator>(
           ParallelInterleaveIterator::Params{
-              this, strings::StrCat(prefix, "::ParallelInterleaveV2")},
+              this, strings::StrCat(prefix, "::", kDatasetName)},
           sloppy_);
     }
 
diff --git a/tensorflow/core/kernels/data/parallel_map_dataset_op.cc b/tensorflow/core/kernels/data/parallel_map_dataset_op.cc
index e2c9057cb2..34f341d1d1 100644
--- a/tensorflow/core/kernels/data/parallel_map_dataset_op.cc
+++ b/tensorflow/core/kernels/data/parallel_map_dataset_op.cc
@@ -15,6 +15,7 @@ limitations under the License.
 #include <deque>
 
 #include "tensorflow/core/common_runtime/function.h"
+#include "tensorflow/core/common_runtime/metrics.h"
 #include "tensorflow/core/framework/dataset.h"
 #include "tensorflow/core/framework/partial_tensor_shape.h"
 #include "tensorflow/core/framework/tensor.h"
@@ -31,6 +32,8 @@ namespace {
 // See documentation in ../../ops/dataset_ops.cc for a high-level
 // description of the following op.
 
+constexpr char kDatasetName[] = "ParallelMap";
+
 class ParallelMapDatasetOp : public UnaryDatasetOpKernel {
  public:
   explicit ParallelMapDatasetOp(OpKernelConstruction* ctx)
@@ -64,6 +67,10 @@ class ParallelMapDatasetOp : public UnaryDatasetOpKernel {
     std::vector<int> indices;
     OP_REQUIRES_OK(ctx, ComputeShortCircuitIndices(ctx, func_, &indices));
 
+    if (num_parallel_calls == model::kAutoTune) {
+      metrics::RecordTFDataAutotune(kDatasetName);
+    }
+
     *output =
         new Dataset(ctx, input, func_, num_parallel_calls, output_types_,
                     output_shapes_, use_inter_op_parallelism_, sloppy_,
@@ -108,7 +115,7 @@ class ParallelMapDatasetOp : public UnaryDatasetOpKernel {
         parallel_map_functor = absl::make_unique<ShortCircuitFunctor>(this);
       }
       return NewParallelMapIterator(
-          {this, strings::StrCat(prefix, "::ParallelMap")}, input_,
+          {this, strings::StrCat(prefix, "::", kDatasetName)}, input_,
           std::move(parallel_map_functor), num_parallel_calls_, sloppy_,
           preserve_cardinality_);
     }
diff --git a/tensorflow/core/kernels/data/prefetch_dataset_op.cc b/tensorflow/core/kernels/data/prefetch_dataset_op.cc
index 1f2dd6ca95..f0e835a27c 100644
--- a/tensorflow/core/kernels/data/prefetch_dataset_op.cc
+++ b/tensorflow/core/kernels/data/prefetch_dataset_op.cc
@@ -16,6 +16,7 @@ limitations under the License.
 
 #include <deque>
 
+#include "tensorflow/core/common_runtime/metrics.h"
 #include "tensorflow/core/framework/partial_tensor_shape.h"
 #include "tensorflow/core/framework/stats_aggregator.h"
 #include "tensorflow/core/framework/tensor.h"
@@ -29,6 +30,8 @@ namespace data {
 // See documentation in ../../ops/dataset_ops.cc for a high-level
 // description of the following op.
 
+constexpr char kDatasetName[] = "Prefetch";
+
 class PrefetchDatasetOp::Dataset : public DatasetBase {
  public:
   Dataset(OpKernelContext* ctx, const DatasetBase* input, int64 buffer_size)
@@ -43,7 +46,7 @@ class PrefetchDatasetOp::Dataset : public DatasetBase {
   std::unique_ptr<IteratorBase> MakeIteratorInternal(
       const string& prefix) const override {
     return absl::make_unique<Iterator>(
-        Iterator::Params{this, strings::StrCat(prefix, "::Prefetch")});
+        Iterator::Params{this, strings::StrCat(prefix, "::", kDatasetName)});
   }
 
   const DataTypeVector& output_dtypes() const override {
@@ -392,6 +395,10 @@ void PrefetchDatasetOp::MakeDataset(OpKernelContext* ctx, DatasetBase* input,
               buffer_size >= 0 || buffer_size == PrefetchAutotuner::kAutoTune,
               errors::InvalidArgument("buffer_size must be >= 0"));
 
+  if (buffer_size == PrefetchAutotuner::kAutoTune) {
+    metrics::RecordTFDataAutotune(kDatasetName);
+  }
+
   *output = new Dataset(ctx, input, buffer_size);
 }
 
-- 
GitLab


From a6de72009482036ef3c367e59a267ca30d0903ee Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 14 Jan 2019 20:21:26 -0800
Subject: [PATCH 0727/1095] Fix bug in the default tf.distribute.Strategy where
 the return value of make_input_fn_iterator() did not have an `initialize()`
 method.

PiperOrigin-RevId: 229305002
---
 tensorflow/python/distribute/distribute_lib.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/distribute/distribute_lib.py b/tensorflow/python/distribute/distribute_lib.py
index 66112dceda..421b2a6520 100644
--- a/tensorflow/python/distribute/distribute_lib.py
+++ b/tensorflow/python/distribute/distribute_lib.py
@@ -1685,7 +1685,8 @@ class _DefaultDistributionExtended(DistributionStrategyExtended):
   def _make_input_fn_iterator(self,
                               input_fn,
                               replication_mode=InputReplicationMode.PER_WORKER):
-    return input_fn(InputContext()).make_initializable_iterator()
+    dataset = input_fn(InputContext())
+    return _DefaultDistributionExtended.DefaultInputIterator(dataset)
 
   def _experimental_make_numpy_dataset(self, numpy_input, session):
     numpy_flat = nest.flatten(numpy_input)
-- 
GitLab


From 1573389a17eaeff49b7af29fad2d1deb52187e1b Mon Sep 17 00:00:00 2001
From: Albin Joy <albin.joy@huawei.com>
Date: Tue, 15 Jan 2019 10:06:51 +0530
Subject: [PATCH 0728/1095] Typo error fixed execution_callbacks.py

---
 tensorflow/python/eager/execution_callbacks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/eager/execution_callbacks.py b/tensorflow/python/eager/execution_callbacks.py
index af1afa3454..34fa3da39a 100644
--- a/tensorflow/python/eager/execution_callbacks.py
+++ b/tensorflow/python/eager/execution_callbacks.py
@@ -64,7 +64,7 @@ class InfOrNanError(Exception):
     """Constructor of InfOrNanError.
 
     Args:
-      op_type: Type name of the op that generated the tensor that generated the
+      op_type: Type name of the op that generated the tensor with
         `inf`(s) or `nan`(s) (e.g., `Div`).
       op_name: Name of the op that generated the tensor with `inf`(s) or
         `nan`(s). This name is set by client and can be `None` if it is unset.
-- 
GitLab


From dfce752a5a2c9d69186401b4f5bdc4e03413a916 Mon Sep 17 00:00:00 2001
From: James Ring <sjr@google.com>
Date: Mon, 14 Jan 2019 21:45:54 -0800
Subject: [PATCH 0729/1095] Add some helper functions to C API.

This change adds:
  * TF_OpKernel{Context,Construction}_Failure - used by C kernels to
    report failures in their compute or construction functions

  * TF_OpKernel{Context,Construction_GetAttrType - used by C kernels to
    read TF_DataType attributes from their compute or construction
    context. A macro is used for the implementation, as there will be a
    handful of these methods in subsequent CLs and their implementation is
    boilerplate.

PiperOrigin-RevId: 229311735
---
 tensorflow/c/c_test.c        |  5 +++++
 tensorflow/c/kernels.cc      | 32 ++++++++++++++++++++++++++++++++
 tensorflow/c/kernels.h       | 27 +++++++++++++++++++++++++++
 tensorflow/c/kernels_test.cc | 22 ++++++++++++++++++++--
 4 files changed, 84 insertions(+), 2 deletions(-)

diff --git a/tensorflow/c/c_test.c b/tensorflow/c/c_test.c
index 100aefd58c..b86d8eb8e3 100644
--- a/tensorflow/c/c_test.c
+++ b/tensorflow/c/c_test.c
@@ -32,7 +32,12 @@ void compute(void* kernel, TF_OpKernelContext* ctx) {
   TF_Status* s = TF_NewStatus();
   TF_GetInput(ctx, 0, &input, s);
   TF_DeleteTensor(input);
+
+  TF_DataType type;
+  TF_OpKernelContext_GetAttrType(ctx, "foobar", &type, s);
+
   TF_DeleteStatus(s);
+
 }
 
 // Exercises tensorflow's C API.
diff --git a/tensorflow/c/kernels.cc b/tensorflow/c/kernels.cc
index 673aed558a..c45efd7fae 100644
--- a/tensorflow/c/kernels.cc
+++ b/tensorflow/c/kernels.cc
@@ -159,6 +159,38 @@ void TF_SetOutput(TF_OpKernelContext* ctx, int i, const TF_Tensor* tensor,
   }
 }
 
+void TF_OpKernelConstruction_Failure(TF_OpKernelConstruction* ctx,
+                                     TF_Status* status) {
+  auto* cc_ctx = reinterpret_cast<::tensorflow::OpKernelConstruction*>(ctx);
+  ::tensorflow::Status s(::tensorflow::StatusFromTF_Status(status));
+  cc_ctx->CtxFailure(s);
+}
+
+void TF_OpKernelContext_Failure(TF_OpKernelContext* ctx, TF_Status* status) {
+  auto* cc_ctx = reinterpret_cast<::tensorflow::OpKernelContext*>(ctx);
+  ::tensorflow::Status s(::tensorflow::StatusFromTF_Status(status));
+  cc_ctx->CtxFailure(s);
+}
+
+#define DEFINE_TF_GETATTR_(struct_name, func, c_type, cc_type)                 \
+  void struct_name##_GetAttr##func(struct_name* ctx, const char* attr_name,    \
+                                   c_type* val, TF_Status* status) {           \
+    TF_SetStatus(status, TF_OK, "");                                           \
+    cc_type v;                                                                 \
+    auto* cc_ctx = reinterpret_cast<::tensorflow::OpKernelConstruction*>(ctx); \
+    ::tensorflow::Status s = cc_ctx->GetAttr(attr_name, &v);                   \
+    ::tensorflow::Set_TF_Status_from_Status(status, s);                        \
+    if (s.ok()) {                                                              \
+      *val = static_cast<c_type>(v);                                           \
+    }                                                                          \
+  }
+
+#define DEFINE_TF_GETATTR(func, c_type, cc_type)                     \
+  DEFINE_TF_GETATTR_(TF_OpKernelConstruction, func, c_type, cc_type) \
+  DEFINE_TF_GETATTR_(TF_OpKernelContext, func, c_type, cc_type)
+
+DEFINE_TF_GETATTR(Type, TF_DataType, tensorflow::DataType)
+
 TF_DataType TF_ExpectedOutputDataType(TF_OpKernelContext* ctx, int i) {
   auto* cc_ctx = reinterpret_cast<::tensorflow::OpKernelContext*>(ctx);
   return static_cast<TF_DataType>(cc_ctx->expected_output_dtype(i));
diff --git a/tensorflow/c/kernels.h b/tensorflow/c/kernels.h
index 721a4aca0b..b015d01039 100644
--- a/tensorflow/c/kernels.h
+++ b/tensorflow/c/kernels.h
@@ -111,6 +111,15 @@ TF_CAPI_EXPORT extern void TF_SetOutput(TF_OpKernelContext* ctx, int i,
                                         const TF_Tensor* tensor,
                                         TF_Status* status);
 
+// Notifies the given OpKernelConstruction that kernel construction has failed.
+TF_CAPI_EXPORT extern void TF_OpKernelConstruction_Failure(
+    TF_OpKernelConstruction* ctx, TF_Status* status);
+
+// Notifies the given OpKernelContext that the kernel's compute function has
+// failed.
+TF_CAPI_EXPORT extern void TF_OpKernelContext_Failure(TF_OpKernelContext* ctx,
+                                                      TF_Status* status);
+
 // Returns the expected output data type of the ith output. If i < 0 or
 // i >= TF_NumOutputs(ctx), the program aborts.
 TF_CAPI_EXPORT extern TF_DataType TF_ExpectedOutputDataType(
@@ -119,6 +128,24 @@ TF_CAPI_EXPORT extern TF_DataType TF_ExpectedOutputDataType(
 // Returns the step ID of the given context.
 TF_CAPI_EXPORT extern int64_t TF_StepId(TF_OpKernelContext* ctx);
 
+// Interprets the named kernel construction attribute as a TF_DataType and
+// places it into *val. *status is set to TF_OK.
+//
+// If the attribute could not be found or could not be interpreted as
+// TF_DataType, *status is populated with an error.
+TF_CAPI_EXPORT extern void TF_OpKernelConstruction_GetAttrType(
+    TF_OpKernelConstruction* ctx, const char* attr_name, TF_DataType* val,
+    TF_Status* status);
+
+// Interprets the named kernel context attribute as a TF_DataType and places it
+// into *val. *status is set to TF_OK.
+//
+// If the attribute could not be found or could not be interpreted as
+// TF_DataType, *status is populated with an error.
+TF_CAPI_EXPORT extern void TF_OpKernelContext_GetAttrType(
+    TF_OpKernelContext* ctx, const char* attr_name, TF_DataType* val,
+    TF_Status* status);
+
 #ifdef __cplusplus
 } /* end extern "C" */
 #endif
diff --git a/tensorflow/c/kernels_test.cc b/tensorflow/c/kernels_test.cc
index fdeb04c84c..531eb598fa 100644
--- a/tensorflow/c/kernels_test.cc
+++ b/tensorflow/c/kernels_test.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #include "tensorflow/c/kernels.h"
 
 #include "tensorflow/c/c_api.h"
+#include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/kernel_def.pb.h"
 #include "tensorflow/core/framework/node_def.pb_text.h"
 #include "tensorflow/core/framework/op.h"
@@ -42,7 +43,17 @@ static void MyComputeFunc(void* kernel, TF_OpKernelContext* ctx) {
   struct MyCustomKernel* s = static_cast<struct MyCustomKernel*>(kernel);
   s->compute_called = true;
   if (ctx != nullptr) {
+    TF_Status* status = TF_NewStatus();
+
     EXPECT_EQ(43, TF_StepId(ctx));
+
+    // Exercise attribute reads.
+    TF_DataType type;
+    TF_OpKernelContext_GetAttrType(ctx, "SomeDataTypeAttr", &type, status);
+    EXPECT_EQ(TF_OK, TF_GetCode(status));
+    EXPECT_EQ(TF_FLOAT, type);
+
+    TF_DeleteStatus(status);
   }
 }
 
@@ -64,6 +75,11 @@ static std::unique_ptr<OpKernel> GetFakeKernel(const char* device_name,
   def.set_device(device_name);
   def.add_input("input1");
   def.add_input("input2");
+
+  AttrValue v;
+  v.set_type(DataType::DT_FLOAT);
+  (*def.mutable_attr())["SomeDataTypeAttr"] = v;
+
   return CreateOpKernel(DeviceType(device_name), nullptr, nullptr, def, 1,
                         status);
 }
@@ -78,7 +94,8 @@ TEST(TestKernel, TestRegisterKernelBuilder) {
   REGISTER_OP(op_name)
       .Input("input1: double")
       .Input("input2: uint8")
-      .Output("output1: uint8");
+      .Output("output1: uint8")
+      .Attr("SomeDataTypeAttr: type");
 
   TF_KernelBuilder* builder = TF_NewKernelBuilder(
       op_name, device_name, &MyCreateFunc, &MyComputeFunc, &MyDeleteFunc);
@@ -129,7 +146,8 @@ TEST(TestKernel, TestInputAndOutputCount) {
   REGISTER_OP(op_name)
       .Input("input1: double")
       .Input("input2: uint8")
-      .Output("output1: uint8");
+      .Output("output1: uint8")
+      .Attr("SomeDataTypeAttr: type");
 
   static int num_inputs = 0;
   static int num_outputs = 0;
-- 
GitLab


From e3aa49701d60a006d09786f5a240530ee5f47e25 Mon Sep 17 00:00:00 2001
From: Yifei Feng <yifeif@google.com>
Date: Mon, 14 Jan 2019 22:18:17 -0800
Subject: [PATCH 0730/1095] Remove cuda dependencies from dummy targets.

PiperOrigin-RevId: 229314246
---
 tensorflow/core/platform/default/build_config/BUILD | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/core/platform/default/build_config/BUILD b/tensorflow/core/platform/default/build_config/BUILD
index c0c1022c16..664218d11b 100644
--- a/tensorflow/core/platform/default/build_config/BUILD
+++ b/tensorflow/core/platform/default/build_config/BUILD
@@ -85,17 +85,17 @@ cc_library(
 # Dummy stream executor cuda plugins.
 cc_library(
     name = "cublas_plugin",
-    deps = if_static(["//tensorflow/stream_executor/cuda:cublas_plugin"]),
+    srcs = [],
 )
 
 cc_library(
     name = "cufft_plugin",
-    deps = if_static(["//tensorflow/stream_executor/cuda:cufft_plugin"]),
+    srcs = [],
 )
 
 cc_library(
     name = "cudnn_plugin",
-    deps = if_static(["//tensorflow/stream_executor/cuda:cudnn_plugin"]),
+    srcs = [],
 )
 
 # OSX framework for device driver access
-- 
GitLab


From 7225c69b4cc3affa65dd0cb61b12cb6ab644f4ca Mon Sep 17 00:00:00 2001
From: Siju <siju.samuel@huawei.com>
Date: Tue, 15 Jan 2019 13:31:35 +0530
Subject: [PATCH 0731/1095] Update ragged_array_ops.py

typo
---
 tensorflow/python/ops/ragged/ragged_array_ops.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/ragged/ragged_array_ops.py b/tensorflow/python/ops/ragged/ragged_array_ops.py
index a6b2442f09..c088afeca9 100644
--- a/tensorflow/python/ops/ragged/ragged_array_ops.py
+++ b/tensorflow/python/ops/ragged/ragged_array_ops.py
@@ -448,7 +448,7 @@ def boolean_mask(data, mask, keepdims=False, name=None):
 
     # Get static rank of mask.
     if mask.shape.ndims is None:
-      raise ValueError('mask.shape.ndims must be kown statically.')
+      raise ValueError('mask.shape.ndims must be known statically.')
     elif mask.shape.ndims == 0:
       raise ValueError('mask cannot be scalar.')
 
-- 
GitLab


From e8e58aef6f8d8a3278d92718b5202e2cacb9a1c4 Mon Sep 17 00:00:00 2001
From: Siju <siju.samuel@huawei.com>
Date: Tue, 15 Jan 2019 13:33:19 +0530
Subject: [PATCH 0732/1095] Update ragged_boolean_mask_op_test.py

typo
---
 tensorflow/python/ops/ragged/ragged_boolean_mask_op_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/ragged/ragged_boolean_mask_op_test.py b/tensorflow/python/ops/ragged/ragged_boolean_mask_op_test.py
index 19f7d216d2..6f5fad13fb 100644
--- a/tensorflow/python/ops/ragged/ragged_boolean_mask_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_boolean_mask_op_test.py
@@ -316,7 +316,7 @@ class RaggedBooleanMaskOpTest(ragged_test_util.RaggedTensorTestCase,
   def testErrors(self):
     if not context.executing_eagerly():
       self.assertRaisesRegexp(ValueError,
-                              r'mask\.shape\.ndims must be kown statically',
+                              r'mask\.shape\.ndims must be known statically',
                               ragged_array_ops.boolean_mask, [[1, 2]],
                               array_ops.placeholder(dtypes.bool))
 
-- 
GitLab


From 52d0e458875714cd11d5cd37ad1b00981546f492 Mon Sep 17 00:00:00 2001
From: Siju <siju.samuel@huawei.com>
Date: Tue, 15 Jan 2019 13:46:28 +0530
Subject: [PATCH 0733/1095] Update tf_ops_compatibility.md

---
 tensorflow/lite/g3doc/tf_ops_compatibility.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/g3doc/tf_ops_compatibility.md b/tensorflow/lite/g3doc/tf_ops_compatibility.md
index dcfda72137..8b85c08a32 100644
--- a/tensorflow/lite/g3doc/tf_ops_compatibility.md
+++ b/tensorflow/lite/g3doc/tf_ops_compatibility.md
@@ -185,7 +185,7 @@ Inputs {
   1: a tensor
 }
 Outputs {
-  0: A tensor of indices of minium values.
+  0: A tensor of indices of minimum values.
 }
 ```
 
-- 
GitLab


From 82e9fd50c1fd44a96b322e6317b645db338e9f25 Mon Sep 17 00:00:00 2001
From: Pete Warden <petewarden@google.com>
Date: Tue, 15 Jan 2019 00:14:55 -0800
Subject: [PATCH 0734/1095] Add documentation to help with TensorFlow Lite
 Micro porting

PiperOrigin-RevId: 229322903
---
 tensorflow/lite/experimental/micro/README.md | 128 ++++++++++++++++++-
 1 file changed, 124 insertions(+), 4 deletions(-)

diff --git a/tensorflow/lite/experimental/micro/README.md b/tensorflow/lite/experimental/micro/README.md
index 931c8d9d31..4c3eab38fa 100644
--- a/tensorflow/lite/experimental/micro/README.md
+++ b/tensorflow/lite/experimental/micro/README.md
@@ -296,10 +296,6 @@ Building micro_speech_test binary
 
 Lauching Renode to test the binary, currently this set up is not automated.
 
--   Until https://github.com/renode/renode/pull/30 is in the Docker image, patch
-    the change manully: `sed -E -i 's/"rv32g"/"rv32imac"/g'
-    /opt/renode/platforms/cpus/sifive-fe310.repl`
-
 -   Execute the binary on Renode: `renode -P 5000 --disable-xwt -e 's
     @/workspace/tensorflow/lite/experimental/micro/testing/sifive_fe310.resc'`
 
@@ -389,3 +385,127 @@ This will create a folder in
 that contains the source and header files, some Mbed configuration files, and a
 README. You should then be able to copy this directory to another machine, and
 use it just like any other Mbed project.
+
+## How to Port TensorFlow Lite Micro to a New Platform
+
+Are you a hardware or operating system provider looking to run machine learning
+on your platform? We're keen to help, and we've had experience helping other
+teams do the same thing, so here are our recommendations.
+
+### Requirements
+
+Since the core neural network operations are pure arithmetic, and don't require
+any I/O or other system-specific functionality, the code doesn't have to have
+many dependencies. We've tried to enforce this, so that it's as easy as possible
+to get TensorFlow Lite Micro running even on 'bare metal' systems without an OS.
+Here are the core requirements that a platform needs to run the framework:
+
+-   C/C++ compiler capable of C++11 compatibility. This is probably the most
+    restrictive of the requirements, since C++11 is not as widely adopted in the
+    embedded world as it is elsewhere. We made the decision to require it since
+    one of the main goals of TFL Micro is to share as much code as possible with
+    the wider TensorFlow codebase, and since that relies on C++11 features, we
+    need compatibility to achieve it. We only use a small, sane, subset of C++
+    though, so don't worry about having to deal with template metaprogramming or
+    similar challenges!
+
+-   Debug logging. The core network operations don't need any I/O functions, but
+    to be able to run tests and tell if they've worked as expected, the
+    framework needs some way to write out a string to some kind of debug
+    console. This will vary from system to system, for example on Linux it could
+    just be `fprintf(stderr, debug_string)` whereas an embedded device might
+    write the string out to a specified UART. As long as there's some mechanism
+    for outputting debug strings, you should be able to use TFL Micro on that
+    platform.
+
+-   Math library. The C standard `libm.a` library is needed to handle some of
+    the mathematical operations used to calculate neural network results.
+
+-   Global variable initialization. We do use a pattern of relying on global
+    variables being set before `main()` is run in some places, so you'll need to
+    make sure your compiler toolchain
+
+And that's it! You may be wondering about some other common requirements that
+are needed by a lot of non-embedded software, so here's a brief list of things
+that aren't necessary to get started with TFL Micro on a new platform:
+
+-   Operating system. Since the only platform-specific function we need is
+    `DebugLog()`, there's no requirement for any kind of Posix or similar
+    functionality around files, processes, or threads.
+
+-   C or C++ standard libraries. The framework tries to avoid relying on any
+    standard library functions that require linker-time support. This includes
+    things like string functions, but still allows us to use headers like
+    `stdtypes.h` which typically just define constants and typedefs.
+    Unfortunately this distinction isn't officially defined by any standard, so
+    it's possible that different toolchains may decide to require linked code
+    even for the subset we use, but in practice we've found it's usually a
+    pretty obvious decision and stable over platforms and toolchains.
+
+-   Dynamic memory allocation. All the TFL Micro code avoids dynamic memory
+    allocation, instead relying on local variables on the stack in most cases,
+    or global variables for a few situations. These are all fixed-size, which
+    can mean some compile-time configuration to ensure there's enough space for
+    particular networks, but does avoid any need for a heap and the
+    implementation of `malloc\new` on a platform.
+
+-   Floating point. Eight-bit integer arithmetic is enough for inference on many
+    networks, so if a model sticks to these kind of quantized operations, no
+    floating point instructions should be required or executed by the framework.
+
+### Getting Started
+
+We recommend that you start trying to compile and run one of the simplest tests
+in the framework as your first step. The full TensorFlow codebase can seem
+overwhelming to work with at first, so instead you can begin with a collection
+of self-contained project folders that only include the source files needed for
+a particular test or executable. You can find a set of pre-generated projects
+[here](https://drive.google.com/open?id=1cawEQAkqquK_SO4crReDYqf_v7yAwOY8).
+
+As mentioned above, the one function you will need to implement for a completely
+new platform is debug logging. If your device is just a variation on an existing
+platform you may be able to reuse code that's already been written. To
+understand what's available, begin with the default reference implementation at
+[tensorflow/lite/experimental/micro/debug_log.cc](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/experimental/micro/debug_log.cc]),
+which uses fprintf and stderr. If your platform has this level of support for
+the C standard library in its toolchain, then you can just reuse this.
+Otherwise, you'll need to do some research into how your platform and device can
+communicate logging statements to the outside world. As another example, take a
+look at
+[the Mbed version of `DebugLog()`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/experimental/micro/mbed/debug_log.cc),
+which creates a UART object and uses it to output strings to the host's console
+if it's connected.
+
+Begin by navigating to the micro_error_reporter_test folder in the pregenerated
+projects you downloaded. Inside here, you'll see a set of folders containing all
+the source code you need. If you look through them, you should find a total of
+around 60 C or C++ files that compiled together will create the test executable.
+There's an example makefile in the directory that lists all of the source files
+and include paths for the headers. If you're building on a Linux or MacOS host
+system, you may just be able to reuse that same makefile to cross-compile for
+your system, as long as you swap out the `CC` and `CXX` variables from their
+defaults, to point to your cross compiler instead (for example
+`arm-none-eabi-gcc` or `riscv64-unknown-elf-gcc`). Otherwise, set up a project
+in the build system you are using. It should hopefully be fairly
+straightforward, since all of the source files in the folder need to be
+compiled, so on many IDEs you can just drag the whole lot in. Then you need to
+make sure that C++11 compatibility is turned on, and that the right include
+paths (as mentioned in the makefile) have been added.
+
+You'll see the default `DebugLog()` implementation in
+'tensorflow/lite/experimental/micro/debug_log.cc' inside the
+micro_error_reporter_test folder. Modify that file to add the right
+implementation for your platform, and then you should be able to build the set
+of files into an executable. Transfer that executable to your target device (for
+example by flashing it), and then try running it. You should see output that
+looks something like this:
+
+```
+Number: 42
+Badly-formed format string
+Another  badly-formed  format string
+~~ALL TESTS PASSED~~~
+```
+
+If not, you'll need to debug what went wrong, but hopefully with this small
+starting project it should be manageable.
-- 
GitLab


From 5596b6196d07d59540e62ecfca5a5b70291d9dee Mon Sep 17 00:00:00 2001
From: Siju <siju.samuel@huawei.com>
Date: Tue, 15 Jan 2019 13:56:33 +0530
Subject: [PATCH 0735/1095] Update README.md

---
 tensorflow/contrib/quantize/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/quantize/README.md b/tensorflow/contrib/quantize/README.md
index 9085d9fa71..5b8da92491 100644
--- a/tensorflow/contrib/quantize/README.md
+++ b/tensorflow/contrib/quantize/README.md
@@ -110,7 +110,7 @@ See the documentation for `tf.contrib.quantize` and [TensorFlow Lite](../lite/).
 
 ## Quantized accuracy results
 
-The following are results of trainiing some popular CNN models (Mobilenet-v1,
+The following are results of training some popular CNN models (Mobilenet-v1,
 Mobilenet-v2, and Inception-v3) using this tool:
 
 <figure>
-- 
GitLab


From d292c630911d6ba75fd0219792ef1cb47f531579 Mon Sep 17 00:00:00 2001
From: Siju <siju.samuel@huawei.com>
Date: Tue, 15 Jan 2019 13:59:24 +0530
Subject: [PATCH 0736/1095] Update options.md

typo
---
 tensorflow/core/profiler/g3doc/options.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/profiler/g3doc/options.md b/tensorflow/core/profiler/g3doc/options.md
index 7f2cd3f698..19f33253fb 100644
--- a/tensorflow/core/profiler/g3doc/options.md
+++ b/tensorflow/core/profiler/g3doc/options.md
@@ -57,7 +57,7 @@ cpu_micros: This is the cpu times.
 Tensor memory are usually ref-counted. The memory is released when there is
 no more reference to it. It will be difficult to track the release of memory.
 Currently, profiler only tracks the allocation of memory. As a result, the
-accumulated memory request is uaually larger than the peak memory of the overall
+accumulated memory request is usually larger than the peak memory of the overall
 model.
 
 It's recommended to generate timeline to see the allocator memory usage over
-- 
GitLab


From dc07d3662531af7b3f049c6f3d3b010d4252df50 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 15 Jan 2019 01:02:19 -0800
Subject: [PATCH 0737/1095] compat: Update forward compatibility horizon to
 2019-01-15

PiperOrigin-RevId: 229327462
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index c9f2397fa5..83f31bd533 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 14)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 15)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From b329214d7bbf2e451e8631fee86ae40694869dd2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 15 Jan 2019 07:00:53 -0800
Subject: [PATCH 0738/1095] Add a ragged size op and register it to the op
 dispatcher

PiperOrigin-RevId: 229364271
---
 tensorflow/python/ops/array_ops.py            |  2 +
 tensorflow/python/ops/ragged/BUILD            | 15 ++++++
 .../python/ops/ragged/ragged_array_ops.py     | 30 ++++++++++++
 .../python/ops/ragged/ragged_dispatch.py      |  7 +++
 .../python/ops/ragged/ragged_dispatch_test.py |  8 ++++
 .../python/ops/ragged/ragged_size_op_test.py  | 48 +++++++++++++++++++
 6 files changed, 110 insertions(+)
 create mode 100644 tensorflow/python/ops/ragged/ragged_size_op_test.py

diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py
index 81558db04f..014fdd25cc 100644
--- a/tensorflow/python/ops/array_ops.py
+++ b/tensorflow/python/ops/array_ops.py
@@ -357,12 +357,14 @@ def shape_n(input, out_type=dtypes.int32, name=None):
 
 
 @tf_export("size", v1=[])
+@dispatch.add_dispatch_support
 def size_v2(input, out_type=dtypes.int32, name=None):
   # pylint: disable=redefined-builtin
   return size(input, name, out_type)
 
 
 @tf_export(v1=["size"])
+@dispatch.add_dispatch_support
 def size(input, name=None, out_type=dtypes.int32):
   # pylint: disable=redefined-builtin
   """Returns the size of a tensor.
diff --git a/tensorflow/python/ops/ragged/BUILD b/tensorflow/python/ops/ragged/BUILD
index 46f7fa62a3..972b14955f 100644
--- a/tensorflow/python/ops/ragged/BUILD
+++ b/tensorflow/python/ops/ragged/BUILD
@@ -875,3 +875,18 @@ py_test(
         "@absl_py//absl/testing:parameterized",
     ],
 )
+
+py_test(
+    name = "ragged_size_op_test",
+    srcs = ["ragged_size_op_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":ragged_array_ops",
+        ":ragged_factory_ops",
+        ":ragged_test_util",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:platform_test",
+        "@absl_py//absl/testing:parameterized",
+    ],
+)
diff --git a/tensorflow/python/ops/ragged/ragged_array_ops.py b/tensorflow/python/ops/ragged/ragged_array_ops.py
index a6b2442f09..399ddbe597 100644
--- a/tensorflow/python/ops/ragged/ragged_array_ops.py
+++ b/tensorflow/python/ops/ragged/ragged_array_ops.py
@@ -1194,6 +1194,36 @@ def _coordinate_where(condition):
                           axis=1)
 
 
+#===============================================================================
+# RaggedTensor Size
+#===============================================================================
+
+
+def size(input, out_type=dtypes.int32, name=None):  # pylint: disable=redefined-builtin
+  """Returns the size of a potentially ragged tensor.
+
+  The size of a ragged tensor is the size of its inner values.
+
+  Args:
+    input: A potentially ragged `Tensor`.
+    out_type: The numeric output type for the operation.
+    name: A name for the operation (optional).
+
+  Returns:
+    A Tensor of type `out_type`.
+
+  #### Example:
+    ```python
+    >>> tf.size(tf.ragged.constant([[1, 2], [3]]))
+    3
+    ```
+  """
+  if ragged_tensor.is_ragged(input):
+    return array_ops.size(input.flat_values, out_type=out_type, name=name)
+  else:
+    return array_ops.size(input, out_type=out_type, name=name)
+
+
 #===============================================================================
 # Internal Helper Functions
 #===============================================================================
diff --git a/tensorflow/python/ops/ragged/ragged_dispatch.py b/tensorflow/python/ops/ragged/ragged_dispatch.py
index 970d88663a..5284750198 100644
--- a/tensorflow/python/ops/ragged/ragged_dispatch.py
+++ b/tensorflow/python/ops/ragged/ragged_dispatch.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import collections
 import numpy as np
 
+from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.ops import array_ops
@@ -415,6 +416,10 @@ def _ragged_expand_dims_v1(input, axis=None, name=None, dim=None):  # pylint: di
   return ragged_array_ops.expand_dims(input=input, axis=axis, name=name)
 
 
+def _ragged_size_v1(input, name=None, out_type=dtypes.int32):  # pylint: disable=redefined-builtin
+  return ragged_array_ops.size(input=input, out_type=out_type, name=name)
+
+
 # (original_op, ragged_op, ragged_args)
 _RAGGED_DISPATCH_OPS = [
     (array_ops.batch_gather, ragged_array_ops.batch_gather,
@@ -426,6 +431,8 @@ _RAGGED_DISPATCH_OPS = [
     (array_ops.gather_v2, ragged_array_ops.gather, ['params', 'indices']),
     (array_ops.gather_nd, ragged_array_ops.gather_nd, ['params', 'indices']),
     (array_ops.rank, ragged_array_ops.rank, ['input']),
+    (array_ops.size, _ragged_size_v1, ['input']),
+    (array_ops.size_v2, ragged_array_ops.size, ['input']),
     (array_ops.stack, ragged_array_ops.stack, ['[values]']),
     (array_ops.tile, ragged_array_ops.tile, ['input']),
     (array_ops.where, ragged_array_ops.where, ['condition', 'x', 'y']),
diff --git a/tensorflow/python/ops/ragged/ragged_dispatch_test.py b/tensorflow/python/ops/ragged/ragged_dispatch_test.py
index 1aa612633e..04ef0d7cd6 100644
--- a/tensorflow/python/ops/ragged/ragged_dispatch_test.py
+++ b/tensorflow/python/ops/ragged/ragged_dispatch_test.py
@@ -687,6 +687,14 @@ class RaggedElementwiseOpsTest(ragged_test_util.RaggedTensorTestCase,
           op=array_ops.rank,
           kwargs={'input': ragged_factory_ops.constant_value([[8, 3], [5]])},
           expected=2),
+      dict(
+          op=array_ops.size,
+          kwargs={'input': ragged_factory_ops.constant_value([[8, 3], [5]])},
+          expected=3),
+      dict(
+          op=array_ops.size_v2,
+          kwargs={'input': ragged_factory_ops.constant_value([[8, 3], [5]])},
+          expected=3),
   ])
   def testRaggedDispatch(self, op, expected, args=(), kwargs=None):
     if kwargs is None: kwargs = {}
diff --git a/tensorflow/python/ops/ragged/ragged_size_op_test.py b/tensorflow/python/ops/ragged/ragged_size_op_test.py
new file mode 100644
index 0000000000..6ffed11b13
--- /dev/null
+++ b/tensorflow/python/ops/ragged/ragged_size_op_test.py
@@ -0,0 +1,48 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for ragged.size."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from absl.testing import parameterized
+
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops.ragged import ragged_array_ops
+from tensorflow.python.ops.ragged import ragged_factory_ops
+from tensorflow.python.ops.ragged import ragged_test_util
+from tensorflow.python.platform import googletest
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class RaggedSizeOpTest(ragged_test_util.RaggedTensorTestCase,
+                       parameterized.TestCase):
+
+  @parameterized.parameters([
+      {'size': 1, 'test_input': 1},
+      {'size': 0, 'test_input': []},
+      {'size': 0, 'test_input': [], 'ragged_rank': 1},
+      {'size': 3, 'test_input': [1, 1, 1]},
+      {'size': 3, 'test_input': [[1, 1], [1]]},
+      {'size': 5, 'test_input': [[[1, 1, 1], [1]], [[1]]]},
+      {'size': 6, 'test_input': [[[1, 1], [1, 1]], [[1, 1]]], 'ragged_rank': 1},
+  ])
+  def testRaggedSize(self, test_input, size, ragged_rank=None):
+    input_rt = ragged_factory_ops.constant(test_input, ragged_rank=ragged_rank)
+    self.assertAllEqual(ragged_array_ops.size(input_rt), size)
+
+if __name__ == '__main__':
+  googletest.main()
-- 
GitLab


From 9825f1d7534cb6d0516813346750fe9a59df42ce Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 15 Jan 2019 08:43:07 -0800
Subject: [PATCH 0739/1095] Split NeuralNetworksShim.h: - Type definitions go
 into NeuralNetworksTypes.h - Function definitions stay into
 NeuralNetworkShim.h

PiperOrigin-RevId: 229377846
---
 tensorflow/lite/nnapi/BUILD                 |   1 +
 tensorflow/lite/nnapi/NeuralNetworksShim.h  | 330 +-----------------
 tensorflow/lite/nnapi/NeuralNetworksTypes.h | 352 ++++++++++++++++++++
 3 files changed, 355 insertions(+), 328 deletions(-)
 create mode 100644 tensorflow/lite/nnapi/NeuralNetworksTypes.h

diff --git a/tensorflow/lite/nnapi/BUILD b/tensorflow/lite/nnapi/BUILD
index 467a2b7a7b..bd3a8a69af 100644
--- a/tensorflow/lite/nnapi/BUILD
+++ b/tensorflow/lite/nnapi/BUILD
@@ -8,6 +8,7 @@ cc_library(
     name = "nnapi_lib",
     hdrs = [
         "NeuralNetworksShim.h",
+        "NeuralNetworksTypes.h",
     ],
     linkopts = ["-ldl"],
 )
diff --git a/tensorflow/lite/nnapi/NeuralNetworksShim.h b/tensorflow/lite/nnapi/NeuralNetworksShim.h
index c39502f4ac..2ce6e50de6 100644
--- a/tensorflow/lite/nnapi/NeuralNetworksShim.h
+++ b/tensorflow/lite/nnapi/NeuralNetworksShim.h
@@ -20,6 +20,8 @@ limitations under the License.
 #include <stdio.h>
 #include <stdlib.h>
 
+#include "tensorflow/lite/nnapi/NeuralNetworksTypes.h"
+
 // helpers
 
 #define NNAPI_LOG(format, ...) fprintf(stderr, format "\n", __VA_ARGS__);
@@ -44,8 +46,6 @@ inline void* loadLibrary(const char* name) {
   return handle;
 }
 
-typedef int (*ASharedMemory_create_fn)(const char* name, size_t size);
-
 // ASharedMemory_create was added in Android 8.0, so safe to use with NNAPI
 // which was added in 8.1.
 inline int ASharedMemory_create(const char* name, size_t size) {
@@ -81,332 +81,6 @@ inline bool NNAPIExists() {
 // NN api types based on NNAPI header file
 // https://developer.android.com/ndk/reference/group/neural-networks
 
-/**
- * Operand types.
- *
- * The type of operands that can be added to a model.
- *
- * Although we define many types, most operators accept just a few
- * types.  Most used are ANEURALNETWORKS_TENSOR_FLOAT32,
- * ANEURALNETWORKS_TENSOR_QUANT8_ASYMM, and ANEURALNETWORKS_INT32.
- */
-enum {
-  ANEURALNETWORKS_FLOAT32 = 0,
-  ANEURALNETWORKS_INT32 = 1,
-  ANEURALNETWORKS_UINT32 = 2,
-  ANEURALNETWORKS_TENSOR_FLOAT32 = 3,
-  ANEURALNETWORKS_TENSOR_INT32 = 4,
-  ANEURALNETWORKS_TENSOR_QUANT8_ASYMM = 5,
-};
-
-/**
- * Operation types.
- *
- * The type of operations that can be added to a model.
- */
-enum {
-  ANEURALNETWORKS_ADD = 0,
-  ANEURALNETWORKS_AVERAGE_POOL_2D = 1,
-  ANEURALNETWORKS_CONCATENATION = 2,
-  ANEURALNETWORKS_CONV_2D = 3,
-  ANEURALNETWORKS_DEPTHWISE_CONV_2D = 4,
-  ANEURALNETWORKS_DEPTH_TO_SPACE = 5,
-  ANEURALNETWORKS_DEQUANTIZE = 6,
-  ANEURALNETWORKS_EMBEDDING_LOOKUP = 7,
-  ANEURALNETWORKS_FLOOR = 8,
-  ANEURALNETWORKS_FULLY_CONNECTED = 9,
-  ANEURALNETWORKS_HASHTABLE_LOOKUP = 10,
-  ANEURALNETWORKS_L2_NORMALIZATION = 11,
-  ANEURALNETWORKS_L2_POOL_2D = 12,
-  ANEURALNETWORKS_LOCAL_RESPONSE_NORMALIZATION = 13,
-  ANEURALNETWORKS_LOGISTIC = 14,
-  ANEURALNETWORKS_LSH_PROJECTION = 15,
-  ANEURALNETWORKS_LSTM = 16,
-  ANEURALNETWORKS_MAX_POOL_2D = 17,
-  ANEURALNETWORKS_MUL = 18,
-  ANEURALNETWORKS_RELU = 19,
-  ANEURALNETWORKS_RELU1 = 20,
-  ANEURALNETWORKS_RELU6 = 21,
-  ANEURALNETWORKS_RESHAPE = 22,
-  ANEURALNETWORKS_RESIZE_BILINEAR = 23,
-  ANEURALNETWORKS_RNN = 24,
-  ANEURALNETWORKS_SOFTMAX = 25,
-  ANEURALNETWORKS_SPACE_TO_DEPTH = 26,
-  ANEURALNETWORKS_SVDF = 27,
-  ANEURALNETWORKS_TANH = 28,
-  ANEURALNETWORKS_BATCH_TO_SPACE_ND = 29,
-  ANEURALNETWORKS_DIV = 30,
-  ANEURALNETWORKS_MEAN = 31,
-  ANEURALNETWORKS_PAD = 32,
-  ANEURALNETWORKS_SPACE_TO_BATCH_ND = 33,
-  ANEURALNETWORKS_SQUEEZE = 34,
-  ANEURALNETWORKS_STRIDED_SLICE = 35,
-  ANEURALNETWORKS_SUB = 36,
-  ANEURALNETWORKS_TRANSPOSE = 37,
-};
-
-/**
- * Fused activation function types.
- *
- */
-enum {
-  ANEURALNETWORKS_FUSED_NONE = 0,
-  ANEURALNETWORKS_FUSED_RELU = 1,
-  ANEURALNETWORKS_FUSED_RELU1 = 2,
-  ANEURALNETWORKS_FUSED_RELU6 = 3,
-};
-
-/**
- * Execution preferences.
- */
-enum {
-  ANEURALNETWORKS_PREFER_LOW_POWER = 0,
-  ANEURALNETWORKS_PREFER_FAST_SINGLE_ANSWER = 1,
-  ANEURALNETWORKS_PREFER_SUSTAINED_SPEED = 2,
-};
-
-/**
- * Result codes.
- */
-enum {
-  ANEURALNETWORKS_NO_ERROR = 0,
-  ANEURALNETWORKS_OUT_OF_MEMORY = 1,
-  ANEURALNETWORKS_INCOMPLETE = 2,
-  ANEURALNETWORKS_UNEXPECTED_NULL = 3,
-  ANEURALNETWORKS_BAD_DATA = 4,
-  ANEURALNETWORKS_OP_FAILED = 5,
-  ANEURALNETWORKS_UNMAPPABLE = 5,
-  ANEURALNETWORKS_BAD_STATE = 6,
-};
-
-/**
- * Implicit padding algorithms.
- */
-enum {
-  ANEURALNETWORKS_PADDING_SAME = 1,
-  ANEURALNETWORKS_PADDING_VALID = 2,
-};
-
-/**
- * ANeuralNetworksMemory is an opaque type that represents memory.
- *
- * This type is used to represent shared memory, memory mapped files,
- * and similar memories.
- *
- * By using shared memory, a program can efficiently communicate to the
- * runtime and drivers the tensors that define a model. See
- * {@link ANeuralNetworksModel_setOperandValueFromMemory}. An application
- * should typically create one shared memory object that contains every tensor
- * needed to define a model. {@link ANeuralNetworksMemory_createFromFd} can be
- * used to create shared memory from a file handle. {@link
- * ANeuralNetworksMemory_createShared} can be used to directly created shared
- * memory.
- *
- * Memory objects can also be used to specify the input and output arguments of
- * an execution. See {@link ANeuralNetworksExecution_setInputFromMemory}
- * and {@link ANeuralNetworksExecution_setOutputFromMemory}.
- */
-typedef struct ANeuralNetworksMemory ANeuralNetworksMemory;
-
-/**
- * ANeuralNetworksModel is an opaque type that contains a description of the
- * mathematical operations that constitute the model.
- *
- * <p>The model will be built by calling<ul>
- * <li>{@link ANeuralNetworksModel_create},</li>
- * <li>{@link ANeuralNetworksModel_addOperation},</li>
- * <li>{@link ANeuralNetworksModel_addOperand},</li>
- * </ul>
- *
- * A model is completed by calling {@link ANeuralNetworksModel_finish}.
- * A model is destroyed by calling {@link ANeuralNetworksModel_free}.
- *
- * <p>It is the application's responsibility to make sure that only one thread
- * modifies a model at a given time. It is however safe for more than one
- * thread to use the model once {@link ANeuralNetworksModel_finish} has
- * returned.</p>
- *
- * <p>It is also the application's responsibility to ensure that there are no
- * other uses of the model after calling {@link ANeuralNetworksModel_free}. This
- * includes any compilation or execution object created using the model.</p>
- */
-typedef struct ANeuralNetworksModel ANeuralNetworksModel;
-
-/**
- * ANeuralNetworksCompilation is an opaque type that can be used to compile
- * a machine learning model.
- *
- * <p>To use:<ul>
- *    <li>Create a new compilation instance by calling the
- *        {@link ANeuralNetworksCompilation_create} function.</li>
- *    <li>Perform the compilation with {@link
- * ANeuralNetworksCompilation_start}.</li> <li>Wait for the compilation to
- * complete with {@link ANeuralNetworksCompilation_wait}.</li> <li>Use the
- * compilation as many times as needed with {@link
- * ANeuralNetworksExecution_create}.</li> <li>Destroy the compilation with
- * {@link ANeuralNetworksCompilation_free} once all executions using the
- * compilation have completed.</li></ul></p>
- *
- * <p>A compilation cannot be modified once {@link
- * ANeuralNetworksCompilation_start} has been called on it.</p>
- *
- * <p>It is the application's responsibility to make sure that only one thread
- * modifies a compilation at a given time. It is however safe for more than one
- * thread to use {@link ANeuralNetworksCompilation_wait} at the same time.
- * It is also safe for multiple threads to use a compilation object once
- * {@link ANeuralNetworksCompilation_wait} has completed.</p>
- *
- * <p>It is also the application's responsibility to ensure that there are no
- * other uses of the compilation after calling {@link
- * ANeuralNetworksCompilation_free}. This includes any execution object created
- * using the compilation.</p>
- */
-typedef struct ANeuralNetworksCompilation ANeuralNetworksCompilation;
-
-/**
- * ANeuralNetworksExecution is an opaque type that can be used to apply a
- * machine learning model to a set of inputs.
- *
- * <p>To use:<ul>
- *    <li>Create a new execution instance by calling the
- *        {@link ANeuralNetworksExecution_create} function.</li>
- *    <li>Associate data to the model inputs with
- *        {@link ANeuralNetworksExecution_setInput} or
- *        {@link ANeuralNetworksExecution_setInputFromMemory}.</li>
- *    <li>Associate output buffers to the model outputs with
- *        {@link ANeuralNetworksExecution_setOutput} or
- *        {@link ANeuralNetworksExecution_setOutputFromMemory}.</li>
- *    <li>Apply the model with {@link
- * ANeuralNetworksExecution_startCompute}.</li> <li>Wait for the execution to
- * complete with {@link ANeuralNetworksExecution_wait}.</li> <li>Destroy the
- * execution with
- *        {@link ANeuralNetworksExecution_free}.</li></ul></p>
- *
- * <p>An execution cannot be modified once {@link
- * ANeuralNetworksExecution_start} has been called on it.</p>
- *
- * <p>An execution can be applied to a model with
- * {@link ANeuralNetworksExecution_startCompute} only once. Create new
- * executions to do new evaluations of the model.</p>
- *
- * <p>It is the application's responsibility to make sure that only one thread
- * modifies an execution at a given time. It is however safe for more than one
- * thread to use {@link ANeuralNetworksExecution_wait} at the same time.</p>
- *
- * <p>It is also the application's responsibility to ensure that there are no
- * other uses of the request after calling {@link
- * ANeuralNetworksRequest_free}.</p>
- */
-typedef struct ANeuralNetworksExecution ANeuralNetworksExecution;
-
-/**
- * ANeuralNetworksOperandType describes the type of an operand.
- * This structure is used to describe both scalars and tensors.
- */
-typedef struct ANeuralNetworksOperandType {
-  /** The data type, e.g ANEURALNETWORKS_INT8. */
-  int32_t type;
-  /** The number of dimensions. It should be 0 for scalars. */
-  uint32_t dimensionCount;
-  /** The dimensions of the tensor. It should be nullptr for scalars. */
-  const uint32_t* dimensions;
-  /** These two fields are only used for quantized tensors.
-   * They should be zero for scalars and non-fixed point tensors.
-   * The dequantized value of each entry is (value - offset) * scale.
-   */
-  float scale;
-  int32_t zeroPoint;
-} ANeuralNetworksOperandType;
-
-/**
- * ANeuralNetworksEvent is an opaque type that represents an event
- * that will be signaled once an execution completes.
- */
-typedef struct ANeuralNetworksEvent ANeuralNetworksEvent;
-
-typedef int32_t ANeuralNetworksOperationType;
-
-// nn api function types
-
-typedef int (*ANeuralNetworksMemory_createFromFd_fn)(
-    size_t size, int protect, int fd, size_t offset,
-    ANeuralNetworksMemory** memory);
-
-typedef void (*ANeuralNetworksMemory_free_fn)(ANeuralNetworksMemory* memory);
-
-typedef int (*ANeuralNetworksModel_create_fn)(ANeuralNetworksModel** model);
-
-typedef int (*ANeuralNetworksModel_finish_fn)(ANeuralNetworksModel* model);
-
-typedef void (*ANeuralNetworksModel_free_fn)(ANeuralNetworksModel* model);
-
-typedef int (*ANeuralNetworksCompilation_create_fn)(
-    ANeuralNetworksModel* model, ANeuralNetworksCompilation** compilation);
-
-typedef void (*ANeuralNetworksCompilation_free_fn)(
-    ANeuralNetworksCompilation* compilation);
-
-typedef int (*ANeuralNetworksCompilation_setPreference_fn)(
-    ANeuralNetworksCompilation* compilation, int32_t preference);
-
-typedef int (*ANeuralNetworksCompilation_finish_fn)(
-    ANeuralNetworksCompilation* compilation);
-
-typedef int (*ANeuralNetworksModel_addOperand_fn)(
-    ANeuralNetworksModel* model, const ANeuralNetworksOperandType* type);
-
-typedef int (*ANeuralNetworksModel_setOperandValue_fn)(
-    ANeuralNetworksModel* model, int32_t index, const void* buffer,
-    size_t length);
-
-typedef int (*ANeuralNetworksModel_setOperandValueFromMemory_fn)(
-    ANeuralNetworksModel* model, int32_t index,
-    const ANeuralNetworksMemory* memory, size_t offset, size_t length);
-
-typedef int (*ANeuralNetworksModel_addOperation_fn)(
-    ANeuralNetworksModel* model, ANeuralNetworksOperationType type,
-    uint32_t inputCount, const uint32_t* inputs, uint32_t outputCount,
-    const uint32_t* outputs);
-
-typedef int (*ANeuralNetworksModel_identifyInputsAndOutputs_fn)(
-    ANeuralNetworksModel* model, uint32_t inputCount, const uint32_t* inputs,
-    uint32_t outputCount, const uint32_t* outputs);
-
-typedef int (*ANeuralNetworksModel_relaxComputationFloat32toFloat16_fn)(
-    ANeuralNetworksModel* model, bool allow);
-
-typedef int (*ANeuralNetworksExecution_create_fn)(
-    ANeuralNetworksCompilation* compilation,
-    ANeuralNetworksExecution** execution);
-
-typedef void (*ANeuralNetworksExecution_free_fn)(
-    ANeuralNetworksExecution* execution);
-
-typedef int (*ANeuralNetworksExecution_setInput_fn)(
-    ANeuralNetworksExecution* execution, int32_t index,
-    const ANeuralNetworksOperandType* type, const void* buffer, size_t length);
-
-typedef int (*ANeuralNetworksExecution_setInputFromMemory_fn)(
-    ANeuralNetworksExecution* execution, int32_t index,
-    const ANeuralNetworksOperandType* type, const ANeuralNetworksMemory* memory,
-    size_t offset, size_t length);
-
-typedef int (*ANeuralNetworksExecution_setOutput_fn)(
-    ANeuralNetworksExecution* execution, int32_t index,
-    const ANeuralNetworksOperandType* type, void* buffer, size_t length);
-
-typedef int (*ANeuralNetworksExecution_setOutputFromMemory_fn)(
-    ANeuralNetworksExecution* execution, int32_t index,
-    const ANeuralNetworksOperandType* type, const ANeuralNetworksMemory* memory,
-    size_t offset, size_t length);
-
-typedef int (*ANeuralNetworksExecution_startCompute_fn)(
-    ANeuralNetworksExecution* execution, ANeuralNetworksEvent** event);
-
-typedef int (*ANeuralNetworksEvent_wait_fn)(ANeuralNetworksEvent* event);
-
-typedef void (*ANeuralNetworksEvent_free_fn)(ANeuralNetworksEvent* event);
-
 /**
  * Creates a shared memory object from a file descriptor.
  *
diff --git a/tensorflow/lite/nnapi/NeuralNetworksTypes.h b/tensorflow/lite/nnapi/NeuralNetworksTypes.h
new file mode 100644
index 0000000000..de8b84a823
--- /dev/null
+++ b/tensorflow/lite/nnapi/NeuralNetworksTypes.h
@@ -0,0 +1,352 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_NNAPI_NEURALNETWORKSTYPES_H_
+#define TENSORFLOW_LITE_NNAPI_NEURALNETWORKSTYPES_H_
+
+#include <stdint.h>
+#include <stdio.h>
+
+// NN api types based on NNAPI header file
+// https://developer.android.com/ndk/reference/group/neural-networks
+
+/**
+ * Operand types.
+ *
+ * The type of operands that can be added to a model.
+ *
+ * Although we define many types, most operators accept just a few
+ * types.  Most used are ANEURALNETWORKS_TENSOR_FLOAT32,
+ * ANEURALNETWORKS_TENSOR_QUANT8_ASYMM, and ANEURALNETWORKS_INT32.
+ */
+enum {
+  ANEURALNETWORKS_FLOAT32 = 0,
+  ANEURALNETWORKS_INT32 = 1,
+  ANEURALNETWORKS_UINT32 = 2,
+  ANEURALNETWORKS_TENSOR_FLOAT32 = 3,
+  ANEURALNETWORKS_TENSOR_INT32 = 4,
+  ANEURALNETWORKS_TENSOR_QUANT8_ASYMM = 5,
+};
+
+/**
+ * Operation types.
+ *
+ * The type of operations that can be added to a model.
+ */
+enum {
+  ANEURALNETWORKS_ADD = 0,
+  ANEURALNETWORKS_AVERAGE_POOL_2D = 1,
+  ANEURALNETWORKS_CONCATENATION = 2,
+  ANEURALNETWORKS_CONV_2D = 3,
+  ANEURALNETWORKS_DEPTHWISE_CONV_2D = 4,
+  ANEURALNETWORKS_DEPTH_TO_SPACE = 5,
+  ANEURALNETWORKS_DEQUANTIZE = 6,
+  ANEURALNETWORKS_EMBEDDING_LOOKUP = 7,
+  ANEURALNETWORKS_FLOOR = 8,
+  ANEURALNETWORKS_FULLY_CONNECTED = 9,
+  ANEURALNETWORKS_HASHTABLE_LOOKUP = 10,
+  ANEURALNETWORKS_L2_NORMALIZATION = 11,
+  ANEURALNETWORKS_L2_POOL_2D = 12,
+  ANEURALNETWORKS_LOCAL_RESPONSE_NORMALIZATION = 13,
+  ANEURALNETWORKS_LOGISTIC = 14,
+  ANEURALNETWORKS_LSH_PROJECTION = 15,
+  ANEURALNETWORKS_LSTM = 16,
+  ANEURALNETWORKS_MAX_POOL_2D = 17,
+  ANEURALNETWORKS_MUL = 18,
+  ANEURALNETWORKS_RELU = 19,
+  ANEURALNETWORKS_RELU1 = 20,
+  ANEURALNETWORKS_RELU6 = 21,
+  ANEURALNETWORKS_RESHAPE = 22,
+  ANEURALNETWORKS_RESIZE_BILINEAR = 23,
+  ANEURALNETWORKS_RNN = 24,
+  ANEURALNETWORKS_SOFTMAX = 25,
+  ANEURALNETWORKS_SPACE_TO_DEPTH = 26,
+  ANEURALNETWORKS_SVDF = 27,
+  ANEURALNETWORKS_TANH = 28,
+  ANEURALNETWORKS_BATCH_TO_SPACE_ND = 29,
+  ANEURALNETWORKS_DIV = 30,
+  ANEURALNETWORKS_MEAN = 31,
+  ANEURALNETWORKS_PAD = 32,
+  ANEURALNETWORKS_SPACE_TO_BATCH_ND = 33,
+  ANEURALNETWORKS_SQUEEZE = 34,
+  ANEURALNETWORKS_STRIDED_SLICE = 35,
+  ANEURALNETWORKS_SUB = 36,
+  ANEURALNETWORKS_TRANSPOSE = 37,
+};
+
+/**
+ * Fused activation function types.
+ *
+ */
+enum {
+  ANEURALNETWORKS_FUSED_NONE = 0,
+  ANEURALNETWORKS_FUSED_RELU = 1,
+  ANEURALNETWORKS_FUSED_RELU1 = 2,
+  ANEURALNETWORKS_FUSED_RELU6 = 3,
+};
+
+/**
+ * Execution preferences.
+ */
+enum {
+  ANEURALNETWORKS_PREFER_LOW_POWER = 0,
+  ANEURALNETWORKS_PREFER_FAST_SINGLE_ANSWER = 1,
+  ANEURALNETWORKS_PREFER_SUSTAINED_SPEED = 2,
+};
+
+/**
+ * Result codes.
+ */
+enum {
+  ANEURALNETWORKS_NO_ERROR = 0,
+  ANEURALNETWORKS_OUT_OF_MEMORY = 1,
+  ANEURALNETWORKS_INCOMPLETE = 2,
+  ANEURALNETWORKS_UNEXPECTED_NULL = 3,
+  ANEURALNETWORKS_BAD_DATA = 4,
+  ANEURALNETWORKS_OP_FAILED = 5,
+  ANEURALNETWORKS_UNMAPPABLE = 5,
+  ANEURALNETWORKS_BAD_STATE = 6,
+};
+
+/**
+ * Implicit padding algorithms.
+ */
+enum {
+  ANEURALNETWORKS_PADDING_SAME = 1,
+  ANEURALNETWORKS_PADDING_VALID = 2,
+};
+
+/**
+ * ANeuralNetworksMemory is an opaque type that represents memory.
+ *
+ * This type is used to represent shared memory, memory mapped files,
+ * and similar memories.
+ *
+ * By using shared memory, a program can efficiently communicate to the
+ * runtime and drivers the tensors that define a model. See
+ * {@link ANeuralNetworksModel_setOperandValueFromMemory}. An application
+ * should typically create one shared memory object that contains every tensor
+ * needed to define a model. {@link ANeuralNetworksMemory_createFromFd} can be
+ * used to create shared memory from a file handle. {@link
+ * ANeuralNetworksMemory_createShared} can be used to directly created shared
+ * memory.
+ *
+ * Memory objects can also be used to specify the input and output arguments of
+ * an execution. See {@link ANeuralNetworksExecution_setInputFromMemory}
+ * and {@link ANeuralNetworksExecution_setOutputFromMemory}.
+ */
+typedef struct ANeuralNetworksMemory ANeuralNetworksMemory;
+
+/**
+ * ANeuralNetworksModel is an opaque type that contains a description of the
+ * mathematical operations that constitute the model.
+ *
+ * <p>The model will be built by calling<ul>
+ * <li>{@link ANeuralNetworksModel_create},</li>
+ * <li>{@link ANeuralNetworksModel_addOperation},</li>
+ * <li>{@link ANeuralNetworksModel_addOperand},</li>
+ * </ul>
+ *
+ * A model is completed by calling {@link ANeuralNetworksModel_finish}.
+ * A model is destroyed by calling {@link ANeuralNetworksModel_free}.
+ *
+ * <p>It is the application's responsibility to make sure that only one thread
+ * modifies a model at a given time. It is however safe for more than one
+ * thread to use the model once {@link ANeuralNetworksModel_finish} has
+ * returned.</p>
+ *
+ * <p>It is also the application's responsibility to ensure that there are no
+ * other uses of the model after calling {@link ANeuralNetworksModel_free}. This
+ * includes any compilation or execution object created using the model.</p>
+ */
+typedef struct ANeuralNetworksModel ANeuralNetworksModel;
+
+/**
+ * ANeuralNetworksCompilation is an opaque type that can be used to compile
+ * a machine learning model.
+ *
+ * <p>To use:<ul>
+ *    <li>Create a new compilation instance by calling the
+ *        {@link ANeuralNetworksCompilation_create} function.</li>
+ *    <li>Perform the compilation with {@link
+ * ANeuralNetworksCompilation_start}.</li> <li>Wait for the compilation to
+ * complete with {@link ANeuralNetworksCompilation_wait}.</li> <li>Use the
+ * compilation as many times as needed with {@link
+ * ANeuralNetworksExecution_create}.</li> <li>Destroy the compilation with
+ * {@link ANeuralNetworksCompilation_free} once all executions using the
+ * compilation have completed.</li></ul></p>
+ *
+ * <p>A compilation cannot be modified once {@link
+ * ANeuralNetworksCompilation_start} has been called on it.</p>
+ *
+ * <p>It is the application's responsibility to make sure that only one thread
+ * modifies a compilation at a given time. It is however safe for more than one
+ * thread to use {@link ANeuralNetworksCompilation_wait} at the same time.
+ * It is also safe for multiple threads to use a compilation object once
+ * {@link ANeuralNetworksCompilation_wait} has completed.</p>
+ *
+ * <p>It is also the application's responsibility to ensure that there are no
+ * other uses of the compilation after calling {@link
+ * ANeuralNetworksCompilation_free}. This includes any execution object created
+ * using the compilation.</p>
+ */
+typedef struct ANeuralNetworksCompilation ANeuralNetworksCompilation;
+
+/**
+ * ANeuralNetworksExecution is an opaque type that can be used to apply a
+ * machine learning model to a set of inputs.
+ *
+ * <p>To use:<ul>
+ *    <li>Create a new execution instance by calling the
+ *        {@link ANeuralNetworksExecution_create} function.</li>
+ *    <li>Associate data to the model inputs with
+ *        {@link ANeuralNetworksExecution_setInput} or
+ *        {@link ANeuralNetworksExecution_setInputFromMemory}.</li>
+ *    <li>Associate output buffers to the model outputs with
+ *        {@link ANeuralNetworksExecution_setOutput} or
+ *        {@link ANeuralNetworksExecution_setOutputFromMemory}.</li>
+ *    <li>Apply the model with {@link
+ * ANeuralNetworksExecution_startCompute}.</li> <li>Wait for the execution to
+ * complete with {@link ANeuralNetworksExecution_wait}.</li> <li>Destroy the
+ * execution with
+ *        {@link ANeuralNetworksExecution_free}.</li></ul></p>
+ *
+ * <p>An execution cannot be modified once {@link
+ * ANeuralNetworksExecution_start} has been called on it.</p>
+ *
+ * <p>An execution can be applied to a model with
+ * {@link ANeuralNetworksExecution_startCompute} only once. Create new
+ * executions to do new evaluations of the model.</p>
+ *
+ * <p>It is the application's responsibility to make sure that only one thread
+ * modifies an execution at a given time. It is however safe for more than one
+ * thread to use {@link ANeuralNetworksExecution_wait} at the same time.</p>
+ *
+ * <p>It is also the application's responsibility to ensure that there are no
+ * other uses of the request after calling {@link
+ * ANeuralNetworksRequest_free}.</p>
+ */
+typedef struct ANeuralNetworksExecution ANeuralNetworksExecution;
+
+/**
+ * ANeuralNetworksOperandType describes the type of an operand.
+ * This structure is used to describe both scalars and tensors.
+ */
+typedef struct ANeuralNetworksOperandType {
+  /** The data type, e.g ANEURALNETWORKS_INT8. */
+  int32_t type;
+  /** The number of dimensions. It should be 0 for scalars. */
+  uint32_t dimensionCount;
+  /** The dimensions of the tensor. It should be nullptr for scalars. */
+  const uint32_t* dimensions;
+  /** These two fields are only used for quantized tensors.
+   * They should be zero for scalars and non-fixed point tensors.
+   * The dequantized value of each entry is (value - offset) * scale.
+   */
+  float scale;
+  int32_t zeroPoint;
+} ANeuralNetworksOperandType;
+
+/**
+ * ANeuralNetworksEvent is an opaque type that represents an event
+ * that will be signaled once an execution completes.
+ */
+typedef struct ANeuralNetworksEvent ANeuralNetworksEvent;
+
+typedef int32_t ANeuralNetworksOperationType;
+
+// nn api function types
+
+typedef int (*ANeuralNetworksMemory_createFromFd_fn)(
+    size_t size, int protect, int fd, size_t offset,
+    ANeuralNetworksMemory** memory);
+
+typedef void (*ANeuralNetworksMemory_free_fn)(ANeuralNetworksMemory* memory);
+
+typedef int (*ANeuralNetworksModel_create_fn)(ANeuralNetworksModel** model);
+
+typedef int (*ANeuralNetworksModel_finish_fn)(ANeuralNetworksModel* model);
+
+typedef void (*ANeuralNetworksModel_free_fn)(ANeuralNetworksModel* model);
+
+typedef int (*ANeuralNetworksCompilation_create_fn)(
+    ANeuralNetworksModel* model, ANeuralNetworksCompilation** compilation);
+
+typedef void (*ANeuralNetworksCompilation_free_fn)(
+    ANeuralNetworksCompilation* compilation);
+
+typedef int (*ANeuralNetworksCompilation_setPreference_fn)(
+    ANeuralNetworksCompilation* compilation, int32_t preference);
+
+typedef int (*ANeuralNetworksCompilation_finish_fn)(
+    ANeuralNetworksCompilation* compilation);
+
+typedef int (*ANeuralNetworksModel_addOperand_fn)(
+    ANeuralNetworksModel* model, const ANeuralNetworksOperandType* type);
+
+typedef int (*ANeuralNetworksModel_setOperandValue_fn)(
+    ANeuralNetworksModel* model, int32_t index, const void* buffer,
+    size_t length);
+
+typedef int (*ANeuralNetworksModel_setOperandValueFromMemory_fn)(
+    ANeuralNetworksModel* model, int32_t index,
+    const ANeuralNetworksMemory* memory, size_t offset, size_t length);
+
+typedef int (*ANeuralNetworksModel_addOperation_fn)(
+    ANeuralNetworksModel* model, ANeuralNetworksOperationType type,
+    uint32_t inputCount, const uint32_t* inputs, uint32_t outputCount,
+    const uint32_t* outputs);
+
+typedef int (*ANeuralNetworksModel_identifyInputsAndOutputs_fn)(
+    ANeuralNetworksModel* model, uint32_t inputCount, const uint32_t* inputs,
+    uint32_t outputCount, const uint32_t* outputs);
+
+typedef int (*ANeuralNetworksModel_relaxComputationFloat32toFloat16_fn)(
+    ANeuralNetworksModel* model, bool allow);
+
+typedef int (*ANeuralNetworksExecution_create_fn)(
+    ANeuralNetworksCompilation* compilation,
+    ANeuralNetworksExecution** execution);
+
+typedef void (*ANeuralNetworksExecution_free_fn)(
+    ANeuralNetworksExecution* execution);
+
+typedef int (*ANeuralNetworksExecution_setInput_fn)(
+    ANeuralNetworksExecution* execution, int32_t index,
+    const ANeuralNetworksOperandType* type, const void* buffer, size_t length);
+
+typedef int (*ANeuralNetworksExecution_setInputFromMemory_fn)(
+    ANeuralNetworksExecution* execution, int32_t index,
+    const ANeuralNetworksOperandType* type, const ANeuralNetworksMemory* memory,
+    size_t offset, size_t length);
+
+typedef int (*ANeuralNetworksExecution_setOutput_fn)(
+    ANeuralNetworksExecution* execution, int32_t index,
+    const ANeuralNetworksOperandType* type, void* buffer, size_t length);
+
+typedef int (*ANeuralNetworksExecution_setOutputFromMemory_fn)(
+    ANeuralNetworksExecution* execution, int32_t index,
+    const ANeuralNetworksOperandType* type, const ANeuralNetworksMemory* memory,
+    size_t offset, size_t length);
+
+typedef int (*ANeuralNetworksExecution_startCompute_fn)(
+    ANeuralNetworksExecution* execution, ANeuralNetworksEvent** event);
+
+typedef int (*ANeuralNetworksEvent_wait_fn)(ANeuralNetworksEvent* event);
+
+typedef void (*ANeuralNetworksEvent_free_fn)(ANeuralNetworksEvent* event);
+
+typedef int (*ASharedMemory_create_fn)(const char* name, size_t size);
+
+#endif  // TENSORFLOW_LITE_NNAPI_NEURALNETWORKSTYPES_H_
-- 
GitLab


From a6b1f39d10bf7312d7e3c43dcb7dea74b24c42e3 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 15 Jan 2019 08:44:29 -0800
Subject: [PATCH 0740/1095] Replace calls to deprecated googletest macros
 *TEST_CASE() with *TEST_SUITE()

PiperOrigin-RevId: 229378037
---
 .../compiler/xla/client/lib/cholesky_test.cc  | 10 +++----
 .../xla/client/lib/triangular_solve_test.cc   |  6 ++--
 tensorflow/compiler/xla/literal_test.cc       |  2 +-
 .../xla/service/algebraic_simplifier_test.cc  | 18 ++++++------
 .../cpu/cpu_instruction_fusion_test.cc        |  7 +++--
 .../xla/service/cpu/cpu_runtime_test.cc       | 11 ++++----
 .../cpu/tests/cpu_eigen_dot_operation_test.cc |  8 +++---
 .../service/cpu/tests/cpu_intrinsic_test.cc   |  8 +++---
 .../xla/service/hlo_dataflow_analysis_test.cc |  6 ++--
 .../xla/service/hlo_evaluator_test.cc         |  4 +--
 .../compiler/xla/service/hlo_parser_test.cc   | 28 +++++++++----------
 .../xla/service/hlo_rematerialization_test.cc |  4 +--
 12 files changed, 57 insertions(+), 55 deletions(-)

diff --git a/tensorflow/compiler/xla/client/lib/cholesky_test.cc b/tensorflow/compiler/xla/client/lib/cholesky_test.cc
index ba9580a3d3..095dd4fbf8 100644
--- a/tensorflow/compiler/xla/client/lib/cholesky_test.cc
+++ b/tensorflow/compiler/xla/client/lib/cholesky_test.cc
@@ -157,10 +157,10 @@ XLA_TEST_P(RandomCholeskyTest, Random) {
                              xla::ErrorSpec(1e-4, 1e-4));
 }
 
-INSTANTIATE_TEST_CASE_P(RandomCholeskyTestInstance, RandomCholeskyTest,
-                        ::testing::Values(CholeskyTestCase{1, 1},
-                                          CholeskyTestCase{1, 2},
-                                          CholeskyTestCase{10, 5},
-                                          CholeskyTestCase{2, 20}));
+INSTANTIATE_TEST_SUITE_P(RandomCholeskyTestInstance, RandomCholeskyTest,
+                         ::testing::Values(CholeskyTestCase{1, 1},
+                                           CholeskyTestCase{1, 2},
+                                           CholeskyTestCase{10, 5},
+                                           CholeskyTestCase{2, 20}));
 
 }  // namespace
diff --git a/tensorflow/compiler/xla/client/lib/triangular_solve_test.cc b/tensorflow/compiler/xla/client/lib/triangular_solve_test.cc
index 703227c949..284a2e9d18 100644
--- a/tensorflow/compiler/xla/client/lib/triangular_solve_test.cc
+++ b/tensorflow/compiler/xla/client/lib/triangular_solve_test.cc
@@ -439,9 +439,9 @@ std::vector<TriangularSolveTestSpec> TriangularSolveTests() {
   return specs;
 }
 
-INSTANTIATE_TEST_CASE_P(TriangularSolveParametricTestInstantiation,
-                        TriangularSolveParametricTest,
-                        ::testing::ValuesIn(TriangularSolveTests()));
+INSTANTIATE_TEST_SUITE_P(TriangularSolveParametricTestInstantiation,
+                         TriangularSolveParametricTest,
+                         ::testing::ValuesIn(TriangularSolveTests()));
 
 }  // namespace
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/literal_test.cc b/tensorflow/compiler/xla/literal_test.cc
index e67bb5c32f..b54a71ae68 100644
--- a/tensorflow/compiler/xla/literal_test.cc
+++ b/tensorflow/compiler/xla/literal_test.cc
@@ -641,7 +641,7 @@ template <typename T>
 class LiteralUtilTestTemplated : public ::testing::Test {};
 
 using TestedTypes = ::testing::Types<float, int32, uint32, complex64>;
-TYPED_TEST_CASE(LiteralUtilTestTemplated, TestedTypes);
+TYPED_TEST_SUITE(LiteralUtilTestTemplated, TestedTypes);
 
 TYPED_TEST(LiteralUtilTestTemplated, Relayout2x2) {
   // Make a non-integer for floating point types.
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
index d2ff0c9f21..0dea498456 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
@@ -2989,7 +2989,7 @@ class ConvInputPaddingTest
     : public AlgebraicSimplifierTest,
       public ::testing::WithParamInterface<ConvPaddingTestcase> {};
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     ConvInputPaddingTestCases, ConvInputPaddingTest,
     ::testing::ValuesIn(std::vector<ConvPaddingTestcase>{
         // Merge this edge padding into the conv.
@@ -3097,7 +3097,7 @@ class ConvFilterPaddingTest
     : public AlgebraicSimplifierTest,
       public ::testing::WithParamInterface<ConvPaddingTestcase> {};
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     ConvFilterPaddingTestCases, ConvFilterPaddingTest,
     ::testing::ValuesIn(std::vector<ConvPaddingTestcase>{
         // Can only merge interior padding on the filter's spatial dimensions;
@@ -4197,7 +4197,7 @@ PadReduceWindowEffectiveBroadcastCases() {
   return *cases;
 }
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     PadReduceWindowEffectiveBroadcastInstantiation,
     PadReduceWindowEffectiveBroadcastTest,
     ::testing::ValuesIn(PadReduceWindowEffectiveBroadcastCases()));
@@ -4248,7 +4248,7 @@ TEST_P(BatchDotStrengthReductionTest, BatchDotStrengthReduction) {
   EXPECT_EQ(has_no_dot, dot_should_be_transformed);
 }
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     BatchDotStrengthReductionTestInstantiation, BatchDotStrengthReductionTest,
     ::testing::Combine(::testing::Values(1, 2), ::testing::Values(1, 2),
                        ::testing::Values(1, 2), ::testing::Values(F32, BF16)));
@@ -4305,7 +4305,7 @@ TEST_P(DotStrengthReductionTest, DotStrengthReduction) {
   EXPECT_EQ(has_no_dot, dot_should_be_transformed);
 }
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     DotStrengthReductionTestInstantiation, DotStrengthReductionTest,
     ::testing::Combine(::testing::Values(1, 2), ::testing::Values(1, 2),
                        ::testing::Values(1, 2), ::testing::Bool(),
@@ -4479,9 +4479,9 @@ TEST_F(AlgebraicSimplifierTest, DynamicUpdateSliceZeroUpdate) {
   EXPECT_THAT(computation->root_instruction(), operand);
 }
 
-INSTANTIATE_TEST_CASE_P(DotOfConcatSimplificationTestInstantiation,
-                        DotOfConcatSimplificationTest,
-                        ::testing::ValuesIn(kDotOfConcatTestSpecs));
+INSTANTIATE_TEST_SUITE_P(DotOfConcatSimplificationTestInstantiation,
+                         DotOfConcatSimplificationTest,
+                         ::testing::ValuesIn(kDotOfConcatTestSpecs));
 
 struct DotOfGatherTestSpec {
   int64 m;
@@ -4687,7 +4687,7 @@ std::vector<DotOfGatherTestSpec> DotOfGatherPositiveNegativeTests() {
   return all;
 }
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     DotOfGatherSimplificationTestInstantiation, DotOfGatherSimplificationTest,
     ::testing::ValuesIn(DotOfGatherPositiveNegativeTests()));
 
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion_test.cc b/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion_test.cc
index 46c1d4c38e..86cc72dd7c 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion_test.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion_test.cc
@@ -943,9 +943,10 @@ ENTRY main {
   return result;
 }
 
-INSTANTIATE_TEST_CASE_P(GatherLoopFusionTestInstantiation, GatherLoopFusionTest,
-                        ::testing::ValuesIn(GetGatherLoopFusionTestSpecs()),
-                        GatherLoopFusionTestSpec::Name);
+INSTANTIATE_TEST_SUITE_P(GatherLoopFusionTestInstantiation,
+                         GatherLoopFusionTest,
+                         ::testing::ValuesIn(GetGatherLoopFusionTestSpecs()),
+                         GatherLoopFusionTestSpec::Name);
 }  // namespace
 }  // namespace cpu
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_runtime_test.cc b/tensorflow/compiler/xla/service/cpu/cpu_runtime_test.cc
index 1ae3aa5711..4e8c986783 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_runtime_test.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_runtime_test.cc
@@ -162,11 +162,12 @@ TEST_P(EigenMatMulTest, DoIt) {
   CheckMatrixMultiply(*a, *b, *c);
 }
 
-INSTANTIATE_TEST_CASE_P(EigenMatMulTestInstantiaion, EigenMatMulTest,
-                        ::testing::Combine(::testing::ValuesIn(MatMulShapes),
-                                           ::testing::Bool(), ::testing::Bool(),
-                                           ::testing::Bool()),
-                        EigenMatMulTest::Name);
+INSTANTIATE_TEST_SUITE_P(EigenMatMulTestInstantiaion, EigenMatMulTest,
+                         ::testing::Combine(::testing::ValuesIn(MatMulShapes),
+                                            ::testing::Bool(),
+                                            ::testing::Bool(),
+                                            ::testing::Bool()),
+                         EigenMatMulTest::Name);
 
 #ifdef INTEL_MKL
 class MKLMatMulTest : public CpuRuntimeTest,
diff --git a/tensorflow/compiler/xla/service/cpu/tests/cpu_eigen_dot_operation_test.cc b/tensorflow/compiler/xla/service/cpu/tests/cpu_eigen_dot_operation_test.cc
index f8f5f392da..0b4ac9dc29 100644
--- a/tensorflow/compiler/xla/service/cpu/tests/cpu_eigen_dot_operation_test.cc
+++ b/tensorflow/compiler/xla/service/cpu/tests/cpu_eigen_dot_operation_test.cc
@@ -102,10 +102,10 @@ std::vector<DotTestSpec> GetDotTestCases() {
   return result;
 }
 
-INSTANTIATE_TEST_CASE_P(CpuEigenDotOperationTestInstantiation,
-                        CpuEigenDotOperationTest,
-                        ::testing::ValuesIn(GetDotTestCases()),
-                        DotTestSpecToString);
+INSTANTIATE_TEST_SUITE_P(CpuEigenDotOperationTestInstantiation,
+                         CpuEigenDotOperationTest,
+                         ::testing::ValuesIn(GetDotTestCases()),
+                         DotTestSpecToString);
 
 }  // namespace
 }  // namespace cpu
diff --git a/tensorflow/compiler/xla/service/cpu/tests/cpu_intrinsic_test.cc b/tensorflow/compiler/xla/service/cpu/tests/cpu_intrinsic_test.cc
index 9b10c49f4f..3fb0e3cd91 100644
--- a/tensorflow/compiler/xla/service/cpu/tests/cpu_intrinsic_test.cc
+++ b/tensorflow/compiler/xla/service/cpu/tests/cpu_intrinsic_test.cc
@@ -140,10 +140,10 @@ IntrinsicTestSpec CpuUnaryIntrinsicTestCases[] = {
         HloOpcode::kLog, kTriple_android_arm, "",
         R"(CHECK: fadd fast <4 x float> <float 0x3FBDE4A340000000, float 0x3FBDE4A340000000, float 0x3FBDE4A340000000, float 0x3FBDE4A340000000>)"}};
 
-INSTANTIATE_TEST_CASE_P(CpuUnaryIntrinsicTestInstantiation,
-                        CpuUnaryIntrinsicTest,
-                        ::testing::ValuesIn(CpuUnaryIntrinsicTestCases),
-                        CpuUnaryIntrinsicTest::Name);
+INSTANTIATE_TEST_SUITE_P(CpuUnaryIntrinsicTestInstantiation,
+                         CpuUnaryIntrinsicTest,
+                         ::testing::ValuesIn(CpuUnaryIntrinsicTestCases),
+                         CpuUnaryIntrinsicTest::Name);
 
 }  // namespace
 }  // namespace cpu
diff --git a/tensorflow/compiler/xla/service/hlo_dataflow_analysis_test.cc b/tensorflow/compiler/xla/service/hlo_dataflow_analysis_test.cc
index 888886865b..4a7c4963b7 100644
--- a/tensorflow/compiler/xla/service/hlo_dataflow_analysis_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_dataflow_analysis_test.cc
@@ -1901,9 +1901,9 @@ ENTRY %AddDependency (p: f32[3]) -> f32[3] {
   EXPECT_FALSE(analysis->ValueIsDefinedAt(root));
 }
 
-INSTANTIATE_TEST_CASE_P(HloDataflowAnalysisInstantiation,
-                        HloDataflowAnalysisTest,
-                        ::testing::Values(false, true));
+INSTANTIATE_TEST_SUITE_P(HloDataflowAnalysisInstantiation,
+                         HloDataflowAnalysisTest,
+                         ::testing::Values(false, true));
 
 class HloDataflowAnalysisTestBase : public HloTestBase {
  protected:
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_test.cc b/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
index 674df0016a..d34fa48efb 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
@@ -126,8 +126,8 @@ class HloEvaluatorBf16Test : public ::testing::WithParamInterface<bool>,
   HloEvaluatorBf16Test() : HloEvaluatorTest(/*use_bfloat16=*/GetParam()) {}
 };
 
-INSTANTIATE_TEST_CASE_P(HloEvaluatorTest_Instantiation, HloEvaluatorBf16Test,
-                        ::testing::ValuesIn(use_bf16_params));
+INSTANTIATE_TEST_SUITE_P(HloEvaluatorTest_Instantiation, HloEvaluatorBf16Test,
+                         ::testing::ValuesIn(use_bf16_params));
 
 // Verifies that HloEvaluator evaluates a HLO instruction that performs clamp
 // with 3 operands.
diff --git a/tensorflow/compiler/xla/service/hlo_parser_test.cc b/tensorflow/compiler/xla/service/hlo_parser_test.cc
index 74ef8a2fba..6ba16cc82a 100644
--- a/tensorflow/compiler/xla/service/hlo_parser_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_parser_test.cc
@@ -1371,20 +1371,20 @@ TEST_P(HloParserTestLongProto, Run) { ExpectEqual(); }
 TEST_P(HloParserTestShort, Run) { ExpectEqual(); }
 TEST_P(HloParserTestShortProto, Run) { ExpectEqual(); }
 
-INSTANTIATE_TEST_CASE_P(HloParserTestSuccessInstantiation, HloParserTestLong,
-                        ::testing::ValuesIn(CreateTestCases()),
-                        TestDataToString);
-INSTANTIATE_TEST_CASE_P(HloParserTestSuccessInstantiation,
-                        HloParserTestLongProto,
-                        ::testing::ValuesIn(CreateTestCases()),
-                        TestDataToString);
-INSTANTIATE_TEST_CASE_P(HloParserTestSuccessInstantiation, HloParserTestShort,
-                        ::testing::ValuesIn(CreateShortTestCases()),
-                        TestDataToString);
-INSTANTIATE_TEST_CASE_P(HloParserTestSuccessInstantiation,
-                        HloParserTestShortProto,
-                        ::testing::ValuesIn(CreateShortTestCases()),
-                        TestDataToString);
+INSTANTIATE_TEST_SUITE_P(HloParserTestSuccessInstantiation, HloParserTestLong,
+                         ::testing::ValuesIn(CreateTestCases()),
+                         TestDataToString);
+INSTANTIATE_TEST_SUITE_P(HloParserTestSuccessInstantiation,
+                         HloParserTestLongProto,
+                         ::testing::ValuesIn(CreateTestCases()),
+                         TestDataToString);
+INSTANTIATE_TEST_SUITE_P(HloParserTestSuccessInstantiation, HloParserTestShort,
+                         ::testing::ValuesIn(CreateShortTestCases()),
+                         TestDataToString);
+INSTANTIATE_TEST_SUITE_P(HloParserTestSuccessInstantiation,
+                         HloParserTestShortProto,
+                         ::testing::ValuesIn(CreateShortTestCases()),
+                         TestDataToString);
 
 class HloParserTest : public ::testing::Test {
  protected:
diff --git a/tensorflow/compiler/xla/service/hlo_rematerialization_test.cc b/tensorflow/compiler/xla/service/hlo_rematerialization_test.cc
index 22c3c40a93..c4bb475619 100644
--- a/tensorflow/compiler/xla/service/hlo_rematerialization_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_rematerialization_test.cc
@@ -588,8 +588,8 @@ TEST_P(IndirectUseTest, IndirectUseNotRematerialized) {
   }
 }
 
-INSTANTIATE_TEST_CASE_P(IndirectUseTestInstantiation, IndirectUseTest,
-                        ::testing::Values(true, false));
+INSTANTIATE_TEST_SUITE_P(IndirectUseTestInstantiation, IndirectUseTest,
+                         ::testing::Values(true, false));
 
 }  // namespace
 
-- 
GitLab


From e0c7bf29e7486f1b057abf50baf5cd4058e45eb0 Mon Sep 17 00:00:00 2001
From: Andy Ly <lyandy@google.com>
Date: Tue, 15 Jan 2019 08:53:33 -0800
Subject: [PATCH 0741/1095] [Grappler] Don't allow creation of self loops on
 mutation in MutableGraphView.

PiperOrigin-RevId: 229379324
---
 .../core/grappler/mutable_graph_view.cc       |  34 ++++
 .../core/grappler/mutable_graph_view_test.cc  | 186 +++++++++++++++++-
 2 files changed, 218 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/grappler/mutable_graph_view.cc b/tensorflow/core/grappler/mutable_graph_view.cc
index 13a530181c..2ba53a685d 100644
--- a/tensorflow/core/grappler/mutable_graph_view.cc
+++ b/tensorflow/core/grappler/mutable_graph_view.cc
@@ -388,6 +388,10 @@ Status MutableGraphView::AddRegularFanin(absl::string_view node_name,
         "Can't add$0 fanin '$1' as regular fanin to$2 node '$3'.", fanin_err,
         fanin.ToString(), node_err, node_name));
   }
+  if (node_name == fanin.node()) {
+    return errors::Internal(absl::Substitute(
+        "Can't add fanin '$0' as regular fanin to self.", fanin.ToString()));
+  }
 
   AddFaninInternal(node, {fanin_node, fanin.index()});
   return Status::OK();
@@ -406,6 +410,10 @@ Status MutableGraphView::AddControllingFanin(absl::string_view node_name,
         absl::Substitute("Can't add$0 controlling fanin '$1' to$2 node '$3'.",
                          fanin_err, fanin.ToString(), node_err, node_name));
   }
+  if (node_name == fanin.node()) {
+    return errors::Internal(absl::Substitute(
+        "Can't add controlling fanin '$0' to self.", fanin.ToString()));
+  }
 
   if (!IsSwitch(*fanin_node)) {
     AddFaninInternal(node, {fanin_node, Graph::kControlSlot});
@@ -426,6 +434,12 @@ Status MutableGraphView::AddControllingFanin(absl::string_view node_name,
     for (auto fanout : fanouts) {
       if (IsIdentity(*fanout.node) || IsIdentityNSingleInput(*fanout.node)) {
         if (ParseTensorName(fanout.node->input(0)) == fanin) {
+          if (fanout.node->name() == node_name) {
+            return errors::Internal(absl::Substitute(
+                "Can't add found controlling fanin '$0' from fanin '$1' to "
+                "self.",
+                AsControlDependency(fanout.node->name()), fanin.ToString()));
+          }
           AddFaninInternal(node, {fanout.node, Graph::kControlSlot});
           return Status::OK();
         }
@@ -435,6 +449,11 @@ Status MutableGraphView::AddControllingFanin(absl::string_view node_name,
     // dependency: add a new identity node.
     string ctrl_dep_name = AddPrefixToNodeName(
         absl::StrCat(fanin.node(), "_", fanin.index()), kMutableGraphViewCtrl);
+    if (node_name == ctrl_dep_name) {
+      return errors::Internal(absl::Substitute(
+          "Can't add generated controlling fanin '$0' from fanin '$1' to self.",
+          AsControlDependency(ctrl_dep_name), fanin.ToString()));
+    }
 
     // Reuse a previously created node, if possible.
     NodeDef* ctrl_dep_node = GetNode(ctrl_dep_name);
@@ -515,6 +534,11 @@ Status MutableGraphView::RemoveRegularFanin(absl::string_view node_name,
         "Can't remove$0 fanin '$1' as regular fanin from$2 node '$3'.",
         fanin_err, fanin.ToString(), node_err, node_name));
   }
+  if (node_name == fanin.node()) {
+    return errors::Internal(
+        absl::Substitute("Can't remove fanin '$0' as regular fanin from self.",
+                         fanin.ToString()));
+  }
 
   RemoveRegularFaninInternal(node, {fanin_node, fanin.index()});
   return Status::OK();
@@ -550,6 +574,11 @@ Status MutableGraphView::RemoveControllingFanin(
         "Can't remove$0 controlling fanin '$1' from$2 node '$3'.", fanin_err,
         AsControlDependency(string(fanin_node_name)), node_err, node_name));
   }
+  if (node_name == fanin_node_name) {
+    return errors::Internal(
+        absl::Substitute("Can't remove controlling fanin '$0' from self.",
+                         AsControlDependency(string(fanin_node_name))));
+  }
 
   RemoveControllingFaninInternal(node, fanin_node);
   return Status::OK();
@@ -615,6 +644,11 @@ Status MutableGraphView::UpdateFanin(absl::string_view node_name,
         "Switch control dependency.",
         from_fanin.ToString(), to_fanin.ToString(), node_name));
   }
+  if (node_name == from_fanin.node() || node_name == to_fanin.node()) {
+    return errors::Internal(absl::Substitute(
+        "Can't update fanin '$0' to fanin '$1' in self '$2'.",
+        from_fanin.ToString(), to_fanin.ToString(), node_name));
+  }
 
   if (from_fanin == to_fanin) {
     return Status::OK();
diff --git a/tensorflow/core/grappler/mutable_graph_view_test.cc b/tensorflow/core/grappler/mutable_graph_view_test.cc
index 99a69ef263..597bf8a7ee 100644
--- a/tensorflow/core/grappler/mutable_graph_view_test.cc
+++ b/tensorflow/core/grappler/mutable_graph_view_test.cc
@@ -400,6 +400,12 @@ TEST(MutableGraphViewTest, AddRegularFanin) {
       "missing node 'foo_missing'.";
   TestAddRegularFanin("foo_missing", {"bar_missing", Graph::kControlSlot},
                       /*success=*/false, error_msg, /*expected_node=*/nullptr);
+
+  // Add self to create cycle.
+  error_msg = "Can't add fanin 'foo_6:2' as regular fanin to self.";
+  expected_node = NDef("", "", {"^a", "^b"});
+  TestAddRegularFanin("foo_6", {"foo_6", 2}, /*success=*/false, error_msg,
+                      &expected_node);
 }
 
 void CheckFanout(const MutableGraphView& graph, const TensorId& fanin,
@@ -540,6 +546,12 @@ TEST(MutableGraphViewTest, RemoveRegularFanin) {
   TestRemoveRegularFanin("foo_missing", {"bar_missing", Graph::kControlSlot},
                          /*success=*/false, error_msg,
                          /*expected_node=*/nullptr);
+
+  // Remove self.
+  error_msg = "Can't remove fanin 'foo_6:2' as regular fanin from self.";
+  expected_node = NDef("", "", {"^a", "^b"});
+  TestRemoveRegularFanin("foo_6", {"foo_6", 2}, /*success=*/false, error_msg,
+                         &expected_node);
 }
 
 void TestRemoveAllFanins(absl::string_view node_name,
@@ -728,6 +740,22 @@ TEST(MutableGraphViewTest, UpdateFanin) {
   TestUpdateFanin("foo_missing", {"from_bar_missing", -2},
                   {"to_bar_missing", -3},
                   /*success=*/false, error_msg, /*expected_node=*/nullptr);
+
+  // Update to self to create cycle.
+  expected_node = NDef("", "", {"a", "b:2", "b:2", "^c", "^d"});
+  error_msg = "Can't update fanin 'b:2' to fanin 'foo_4:3' in self 'foo_4'.";
+  TestUpdateFanin("foo_4", {"b", 2}, {"foo_4", 3}, /*success=*/false, error_msg,
+                  &expected_node);
+  error_msg = "Can't update fanin 'b:2' to fanin '^foo_4' in self 'foo_4'.";
+  TestUpdateFanin("foo_4", {"b", 2}, {"foo_4", Graph::kControlSlot},
+                  /*success=*/false, error_msg, &expected_node);
+  error_msg = "Can't update fanin '^c' to fanin 'foo_4:4' in self 'foo_4'.";
+  TestUpdateFanin("foo_4", {"c", Graph::kControlSlot}, {"foo_4", 4},
+                  /*success=*/false, error_msg, &expected_node);
+  error_msg = "Can't update fanin '^c' to fanin '^foo_4' in self 'foo_4'.";
+  TestUpdateFanin("foo_4", {"c", Graph::kControlSlot},
+                  {"foo_4", Graph::kControlSlot}, /*success=*/false, error_msg,
+                  &expected_node);
 }
 
 void TestUpdateFaninFromFaninToNodeAsSwitchControl(const TensorId& fanin) {
@@ -1125,6 +1153,30 @@ TEST(MutableGraphViewTest, AddControllingFaninNotSwitch) {
   CompareNodeInputs(graph, &expected, b);
 }
 
+TEST(MutableGraphViewTest, AddControllingFaninSwitch) {
+  GraphDef graph_def = test::function::GDef(
+      {NDef("a", "NotImportant", {}, {}), NDef("b", "Switch", {}, {})},
+      /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  Status s = graph.AddControllingFanin("a", {"b", Graph::kControlSlot});
+  EXPECT_FALSE(s.ok());
+  string expected_msg =
+      "Can't add Switch as controlling fanin '^b' to node 'a'.";
+  EXPECT_EQ(s.error_message(), expected_msg);
+
+  ASSERT_EQ(graph.graph()->node_size(), 2);
+  NodeDef* a = graph.GetNode("a");
+  ASSERT_NE(a, nullptr);
+  NodeDef expected = NDef("", "", {});
+  CompareNodeInputs(graph, &expected, a);
+
+  NodeDef* b = graph.GetNode("b");
+  ASSERT_NE(b, nullptr);
+  CompareNodeInputs(graph, &expected, b);
+}
+
 TEST(MutableGraphViewTest, AddControllingFaninSwitchWithIdentity) {
   GraphDef graph_def = test::function::GDef(
       {NDef("a", "NotImportant", {}, {}), NDef("switch", "Switch", {}, {}),
@@ -1190,6 +1242,115 @@ TEST(MutableGraphViewTest, AddControllingFaninSwitchWithExistingAddedIdentity) {
   CompareNodeInputs(graph, &expected, a);
 }
 
+void TestAddControllingFaninSelfLoops(absl::string_view node_name,
+                                      const TensorId& fanin,
+                                      const string& error_msg) {
+  GraphDef graph_def = test::function::GDef(
+      {NDef("a", "NotImportant", {}, {}),
+       NDef("b", "Switch", {}, {{"T", DT_FLOAT}}),
+       NDef("c", "Identity", {"b:0"}), NDef("d", "Identity", {"b:1"}),
+       NDef("e", "NotImportant", {"^a"})},
+      /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  Status s = graph.AddControllingFanin(node_name, fanin);
+  EXPECT_FALSE(s.ok());
+  EXPECT_EQ(s.error_message(), error_msg);
+
+  EXPECT_EQ(graph.graph()->node_size(), 5);
+  NodeDef expected;
+  NodeDef* a = graph.GetNode("a");
+  ASSERT_NE(a, nullptr);
+  expected = NDef("", "", {}, {});
+  CompareNodeInputs(graph, &expected, a);
+
+  NodeDef* b = graph.GetNode("b");
+  ASSERT_NE(b, nullptr);
+  CompareNodeInputs(graph, &expected, b);
+
+  NodeDef* c = graph.GetNode("c");
+  ASSERT_NE(c, nullptr);
+  expected = NDef("", "", {"b:0"});
+  CompareNodeInputs(graph, &expected, c);
+
+  NodeDef* d = graph.GetNode("d");
+  ASSERT_NE(d, nullptr);
+  expected = NDef("", "", {"b:1"});
+  CompareNodeInputs(graph, &expected, d);
+
+  NodeDef* e = graph.GetNode("e");
+  ASSERT_NE(e, nullptr);
+  expected = NDef("", "", {"^a"});
+  CompareNodeInputs(graph, &expected, e);
+}
+
+TEST(MutableGraphViewTest, AddControllingFaninSelfLoops) {
+  string error_msg = "Can't add controlling fanin '^a' to self.";
+  TestAddControllingFaninSelfLoops("a", {"a", Graph::kControlSlot}, error_msg);
+
+  // Adding Switch control dependency to Identity consumer. Node `c` is
+  // consuming `b:0`, so adding `b:0` as a control dependency, because it is a
+  // Switch, should trigger a lookup of outputs. As `c` is a consumer and an
+  // Identity, this will introduce a self loop, so no control dependency should
+  // be added.
+  error_msg =
+      "Can't add found controlling fanin '^c' from fanin 'b:0' to self.";
+  TestAddControllingFaninSelfLoops("c", {"b", 0}, error_msg);
+
+  // Adding Switch control dependency to Identity consumer. Node `d` is
+  // consuming `b:1`, so adding `b:1` as a control dependency, because it is a
+  // Switch, should trigger a lookup of outputs. As `d` is a consumer and an
+  // Identity, this will introduce a self loop, so no control dependency should
+  // be added.
+  error_msg =
+      "Can't add found controlling fanin '^d' from fanin 'b:1' to self.";
+  TestAddControllingFaninSelfLoops("d", {"b", 1}, error_msg);
+}
+
+TEST(MutableGraphViewTest, AddControllingFaninSelfLoopsGeneratedIdentity) {
+  GraphDef graph_def =
+      test::function::GDef({NDef("a", "NotImportant", {}, {}),
+                            NDef("b", "Switch", {}, {{"T", DT_FLOAT}}),
+                            NDef("c", "NotImportant", {}),
+                            NDef("ConstantFoldingCtrl/b_1", "Identity", {})},
+                           /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  // Adding Switch control dependency to Identity node of the same name as a
+  // generated Identity node for pinning the control dependency. Because there
+  // are no consumers of `b:1`, there will be an attempt to generate an Identity
+  // node, with name `ConstantFoldingCtrl/b_1`. As the input node is of the same
+  // name, we will introduce a self loop, so no control dependency should be
+  // added.
+  Status s = graph.AddControllingFanin("ConstantFoldingCtrl/b_1", {"b", 1});
+  EXPECT_FALSE(s.ok());
+  string expected_msg =
+      "Can't add generated controlling fanin '^ConstantFoldingCtrl/b_1' from "
+      "fanin 'b:1' to self.";
+  EXPECT_EQ(s.error_message(), expected_msg);
+
+  EXPECT_EQ(graph.graph()->node_size(), 4);
+  NodeDef expected;
+  NodeDef* a = graph.GetNode("a");
+  ASSERT_NE(a, nullptr);
+  expected = NDef("", "", {}, {});
+  CompareNodeInputs(graph, &expected, a);
+
+  NodeDef* b = graph.GetNode("b");
+  ASSERT_NE(b, nullptr);
+  CompareNodeInputs(graph, &expected, b);
+
+  NodeDef* c = graph.GetNode("c");
+  ASSERT_NE(c, nullptr);
+  CompareNodeInputs(graph, &expected, c);
+
+  NodeDef* d = graph.GetNode("ConstantFoldingCtrl/b_1");
+  ASSERT_NE(d, nullptr);
+  CompareNodeInputs(graph, &expected, d);
+}
+
 TEST(MutableGraphViewTest, RemoveControllingFaninMissing) {
   // Actual node.op() is not important in this test.
   GraphDef graph_def = test::function::GDef(
@@ -1233,7 +1394,7 @@ TEST(MutableGraphViewTest, RemoveControllingFaninOnRegularFanin) {
   // Actual node.op() is not important in this test.
   GraphDef graph_def = test::function::GDef(
       {NDef("a", "NotImportant", {}, {}), NDef("b", "NotImportant", {"a"}),
-       NDef("c", "NotImportant", {"a", "b", "^c"})},
+       NDef("c", "NotImportant", {"a", "b"})},
       /*funcs=*/{});
 
   MutableGraphView graph(&graph_def);
@@ -1244,7 +1405,28 @@ TEST(MutableGraphViewTest, RemoveControllingFaninOnRegularFanin) {
   ASSERT_EQ(graph.graph()->node_size(), 3);
   NodeDef* c = graph.GetNode("c");
   ASSERT_NE(c, nullptr);
-  NodeDef expected = NDef("", "", {"a", "b", "^c"});
+  NodeDef expected = NDef("", "", {"a", "b"});
+  CompareNodeInputs(graph, &expected, c);
+}
+
+TEST(MutableGraphViewTest, RemoveControllingFaninSelfLoop) {
+  // Actual node.op() is not important in this test.
+  GraphDef graph_def = test::function::GDef(
+      {NDef("a", "NotImportant", {}, {}), NDef("b", "NotImportant", {"a"}),
+       NDef("c", "NotImportant", {"a", "b"})},
+      /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  Status s = graph.RemoveControllingFanin("c", "c");
+  EXPECT_FALSE(s.ok());
+  string expected_msg = "Can't remove controlling fanin '^c' from self.";
+  EXPECT_EQ(s.error_message(), expected_msg);
+
+  ASSERT_EQ(graph.graph()->node_size(), 3);
+  NodeDef* c = graph.GetNode("c");
+  ASSERT_NE(c, nullptr);
+  NodeDef expected = NDef("", "", {"a", "b"});
   CompareNodeInputs(graph, &expected, c);
 }
 
-- 
GitLab


From 39ebdee5e7f5fb7d7f0626977d363426ed22e308 Mon Sep 17 00:00:00 2001
From: Sergei Lebedev <slebedev@google.com>
Date: Tue, 15 Jan 2019 08:53:37 -0800
Subject: [PATCH 0742/1095] Fixed handling of np.longlong in as_dtype

PiperOrigin-RevId: 229379334
---
 tensorflow/python/framework/dtypes.py      | 11 +++++++++++
 tensorflow/python/framework/dtypes_test.py |  3 +++
 2 files changed, 14 insertions(+)

diff --git a/tensorflow/python/framework/dtypes.py b/tensorflow/python/framework/dtypes.py
index 9386cca379..465ea4229b 100644
--- a/tensorflow/python/framework/dtypes.py
+++ b/tensorflow/python/framework/dtypes.py
@@ -560,6 +560,17 @@ _NP_TO_TF = {
     _np_qint32: qint32,
     _np_bfloat16: bfloat16,
 }
+
+# On Python 2.X `np.longlong` could be a distinct type used for long
+# integers e.g. 42L. See numpy/numpy#9799.
+for pdt in [
+    np.longlong,
+    np.ulonglong,
+]:
+  if pdt not in _NP_TO_TF:
+    _NP_TO_TF[pdt] = next(
+        _NP_TO_TF[dt] for dt in _NP_TO_TF if dt == pdt().dtype)
+
 _TF_TO_NP = {
     types_pb2.DT_HALF:
         np.float16,
diff --git a/tensorflow/python/framework/dtypes_test.py b/tensorflow/python/framework/dtypes_test.py
index 719fdc0953..7dd2a792d1 100644
--- a/tensorflow/python/framework/dtypes_test.py
+++ b/tensorflow/python/framework/dtypes_test.py
@@ -295,6 +295,9 @@ class TypesTest(test_util.TensorFlowTestCase):
     self.assertNotEqual(dtypes.int32, int)
     self.assertNotEqual(dtypes.float64, 2.1)
 
+  def testPythonLongConversion(self):
+    self.assertIs(dtypes.int64, dtypes.as_dtype(np.array(2**32).dtype))
+
   def testPythonTypesConversion(self):
     self.assertIs(dtypes.float32, dtypes.as_dtype(float))
     self.assertIs(dtypes.bool, dtypes.as_dtype(bool))
-- 
GitLab


From 4ff0c987c3358df220632320869d48bc8b51262f Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Tue, 15 Jan 2019 08:58:32 -0800
Subject: [PATCH 0743/1095] Account for the possibility that fully qualified
 name objects may not be normalized. Improve the consistency between function
 and classes in namer.

PiperOrigin-RevId: 229379978
---
 tensorflow/python/autograph/core/naming.py    | 65 +++++++++++++++----
 .../python/autograph/core/naming_test.py      | 16 +++++
 tensorflow/python/autograph/utils/misc.py     |  9 ++-
 .../python/autograph/utils/misc_test.py       | 15 ++++-
 4 files changed, 88 insertions(+), 17 deletions(-)

diff --git a/tensorflow/python/autograph/core/naming.py b/tensorflow/python/autograph/core/naming.py
index b8d79daeba..245795c3d2 100644
--- a/tensorflow/python/autograph/core/naming.py
+++ b/tensorflow/python/autograph/core/naming.py
@@ -18,8 +18,16 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import enum
+
 from tensorflow.python.autograph.pyct import inspect_utils
 from tensorflow.python.autograph.pyct import qual_names
+from tensorflow.python.autograph.utils import misc
+
+
+class _NamingStyle(enum.Enum):
+  SNAKE = 1
+  CAMEL = 2
 
 
 class Namer(object):
@@ -46,17 +54,52 @@ class Namer(object):
 
     self.generated_names = set()
 
+  def _as_symbol_name(self, fqn, style=_NamingStyle.SNAKE):
+    """Returns a symbol name that matches a fully-qualified name.
+
+    The returned name is safe to use for Python symbols. Any special characters
+    present in fqn are replaced according to the style argument.
+
+    Examples:
+
+      self._as_symbol_name('foo.bar', style=_NamingStyle.CAMEL) == 'FooBar'
+      self._as_symbol_name('foo.bar', style=_NamingStyle.SNAKE) == 'foo_bar'
+
+    See the unit tests for more examples.
+
+    Args:
+      fqn: Union[Text, Tuple[Text]] a fully-qualified symbol name. The qualifier
+        may include module, class names, attributes, etc.
+      style: _NamingStyle
+    Returns:
+      Text
+    """
+    assert style in _NamingStyle
+
+    if isinstance(fqn, tuple):
+      cn = '.'.join(fqn)
+    else:
+      cn = fqn
+
+    # Until we clean up the whole FQN mechanism, `fqn` may not be
+    # canonical, that is, in can appear as ('foo.bar', 'baz')
+    # This replaces any characters that might remain because of that.
+    pieces = cn.split('.')
+
+    if style == _NamingStyle.CAMEL:
+      pieces = tuple(misc.capitalize_initial(p) for p in pieces)
+      return ''.join(pieces)
+    elif style == _NamingStyle.SNAKE:
+      return '_'.join(pieces)
+
   def compiled_class_name(self, original_fqn, live_entity=None):
     """See call_trees.FunctionNamer.compiled_class_name."""
     if live_entity is not None and live_entity in self.renamed_calls:
       return self.renamed_calls[live_entity]
 
-    if isinstance(original_fqn, tuple):
-      original_name = '__'.join(original_fqn)
-    else:
-      original_name = original_fqn
-
-    new_name_root = 'Tf%s' % original_name
+    canonical_name = self._as_symbol_name(
+        original_fqn, style=_NamingStyle.CAMEL)
+    new_name_root = 'Tf%s' % canonical_name
     new_name = new_name_root
     n = 0
     while new_name in self.global_namespace:
@@ -73,7 +116,6 @@ class Namer(object):
                              live_entity=None,
                              owner_type=None):
     """See call_trees.FunctionNamer.compiled_function_name."""
-
     if not self.recursive:
       return None, False
 
@@ -84,15 +126,12 @@ class Namer(object):
       # Members are not renamed when part of an entire converted class.
       return None, False
 
-    if isinstance(original_fqn, tuple):
-      original_name = '__'.join(original_fqn)
-    else:
-      original_name = original_fqn
-
     if live_entity is not None and live_entity in self.renamed_calls:
       return self.renamed_calls[live_entity], True
 
-    new_name_root = 'tf__%s' % original_name
+    canonical_name = self._as_symbol_name(
+        original_fqn, style=_NamingStyle.SNAKE)
+    new_name_root = 'tf__%s' % canonical_name
     new_name = new_name_root
     n = 0
     while new_name in self.global_namespace:
diff --git a/tensorflow/python/autograph/core/naming_test.py b/tensorflow/python/autograph/core/naming_test.py
index 2db98836d1..cc8c4314a7 100644
--- a/tensorflow/python/autograph/core/naming_test.py
+++ b/tensorflow/python/autograph/core/naming_test.py
@@ -45,6 +45,22 @@ class NamerTest(test.TestCase):
     self.assertEqual(('tf__foo', True), namer.compiled_function_name(
         'foo', foo))
 
+  def test_compiled_function_name_unsanitized_fqn(self):
+    namer = naming.Namer({}, True, None, ())
+    self.assertEqual(('tf__foo_bar', True),
+                     namer.compiled_function_name('foo.bar'))
+    self.assertEqual(('tf__foo_bar_baz', True), namer.compiled_function_name(
+        ('foo.bar', 'baz')))
+
+  def test_compiled_class_name_basic(self):
+    namer = naming.Namer({}, True, None, ())
+    self.assertEqual('TfFooBar', namer.compiled_class_name(('foo', 'Bar')))
+
+  def test_compiled_class_name_unsanitized_fqn(self):
+    namer = naming.Namer({}, True, None, ())
+    self.assertEqual('TfFooBarBaz',
+                     namer.compiled_class_name(('foo.bar', 'Baz')))
+
   def test_compiled_function_name_avoids_global_conflicts(self):
     def foo():
       pass
diff --git a/tensorflow/python/autograph/utils/misc.py b/tensorflow/python/autograph/utils/misc.py
index 1b06caf0bd..046e6cf97d 100644
--- a/tensorflow/python/autograph/utils/misc.py
+++ b/tensorflow/python/autograph/utils/misc.py
@@ -23,7 +23,7 @@ from tensorflow.python.ops import array_ops
 
 
 def alias_tensors(*args):
-  """Wrap any Tensor arguments with an identity op.
+  """Wraps any Tensor arguments with an identity op.
 
   Any other argument, including Variables, is returned unchanged.
 
@@ -48,3 +48,10 @@ def alias_tensors(*args):
     return alias_if_tensor(args[0])
 
   raise ValueError('at least one argument required')
+
+
+def capitalize_initial(s):
+  """Capitalizes the initial of a string only."""
+  if s:
+    return s[0].upper() + s[1:]
+  return s
diff --git a/tensorflow/python/autograph/utils/misc_test.py b/tensorflow/python/autograph/utils/misc_test.py
index c78df48d62..24b5753a91 100644
--- a/tensorflow/python/autograph/utils/misc_test.py
+++ b/tensorflow/python/autograph/utils/misc_test.py
@@ -18,7 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.python.autograph.utils.misc import alias_tensors
+from tensorflow.python.autograph.utils import misc
 from tensorflow.python.framework import test_util
 from tensorflow.python.framework.constant_op import constant
 from tensorflow.python.ops.variables import Variable
@@ -27,11 +27,20 @@ from tensorflow.python.platform import test
 
 class MiscTest(test.TestCase):
 
+  def test_capitalize_initial(self):
+    self.assertEqual('', misc.capitalize_initial(''))
+    self.assertEqual('A', misc.capitalize_initial('A'))
+    self.assertEqual('Ab', misc.capitalize_initial('Ab'))
+    self.assertEqual('AbC', misc.capitalize_initial('AbC'))
+    self.assertEqual('A', misc.capitalize_initial('a'))
+    self.assertEqual('Ab', misc.capitalize_initial('ab'))
+    self.assertEqual('AbC', misc.capitalize_initial('abC'))
+
   @test_util.run_deprecated_v1
   def test_alias_single_tensor(self):
     a = constant(1)
 
-    new_a = alias_tensors(a)
+    new_a = misc.alias_tensors(a)
     self.assertFalse(new_a is a)
     with self.cached_session() as sess:
       self.assertEqual(1, self.evaluate(new_a))
@@ -43,7 +52,7 @@ class MiscTest(test.TestCase):
     s = 'a'
     l = [1, 2, 3]
 
-    new_a, new_v, new_s, new_l = alias_tensors(a, v, s, l)
+    new_a, new_v, new_s, new_l = misc.alias_tensors(a, v, s, l)
 
     self.assertFalse(new_a is a)
     self.assertTrue(new_v is v)
-- 
GitLab


From da47d5a9a6cb6b10b166d4f3169e50bdafec2683 Mon Sep 17 00:00:00 2001
From: Vojtech Bardiovsky <vbardiovsky@google.com>
Date: Tue, 15 Jan 2019 09:47:53 -0800
Subject: [PATCH 0744/1095] Refactoring: store structured inputs on FuncGraph,
 same as structured outputs. PiperOrigin-RevId: 229387670

---
 tensorflow/python/eager/def_function.py       |  6 ++-
 tensorflow/python/eager/function.py           | 43 +++----------------
 tensorflow/python/framework/func_graph.py     | 40 +++++++++++++++++
 .../saved_model/function_serialization.py     |  4 +-
 4 files changed, 51 insertions(+), 42 deletions(-)

diff --git a/tensorflow/python/eager/def_function.py b/tensorflow/python/eager/def_function.py
index bd70b9dad0..977f548b61 100644
--- a/tensorflow/python/eager/def_function.py
+++ b/tensorflow/python/eager/def_function.py
@@ -25,6 +25,7 @@ import weakref
 from tensorflow.python.eager import context
 from tensorflow.python.eager import function as function_lib
 from tensorflow.python.eager import lift_to_graph
+from tensorflow.python.framework import func_graph as func_graph_module
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
@@ -459,7 +460,8 @@ class PolymorphicFunction(object):
     for signature in self._cached_input_signatures:
       flattened = nest.flatten(signature)
       if any(
-          isinstance(arg, function_lib.UnknownArgument) for arg in flattened):
+          isinstance(arg, func_graph_module.UnknownArgument)
+          for arg in flattened):
         logging.info("Unsupported signature for serialization: %s.", signature)
         continue
       concrete_function = self.get_concrete_function(*signature)
@@ -482,7 +484,7 @@ class PolymorphicFunction(object):
     if self._stateless_fn:
       concrete_functions.extend(self._stateless_fn._function_cache.values())
     for concrete_function in concrete_functions:
-      signature = concrete_function._python_call_signature
+      signature, _ = concrete_function.structured_input_signature
       equal_to_signature = functools.partial(
           function_lib.is_same_structure, signature, check_values=True)
       if not any(equal_to_signature(s) for s in seen):
diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index b5bda35916..b0140e5c98 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -568,6 +568,11 @@ class Function(object):
     """Returns tensors in `self.graph` corresponding to arguments."""
     return self._func_graph.inputs
 
+  @property
+  def structured_input_signature(self):
+    """Returns structured signature of the original function."""
+    return self._func_graph.structured_input_signature
+
   @property
   def outputs(self):
     """Returns tensors in `self.graph` corresponding to returned tensors."""
@@ -824,37 +829,6 @@ class Function(object):
     return ret
 
 
-class UnknownArgument(object):
-  """Signifies an argument which is not currently handled."""
-  pass
-
-
-def convert_structure_to_signature(structure):
-  """Convert a potentially nested structure to a signature.
-
-  Args:
-    structure: Structure to convert.
-
-  Returns:
-    Identical structure that has TensorSpec objects instead of Tensors and
-    UknownArgument instead of any unsupported types.
-  """
-
-  def encode_arg(arg, name=None):
-    """A representation for this argument, for converting into signatures."""
-    if isinstance(arg, ops.Tensor):
-      return tensor_spec.TensorSpec(arg.shape, arg.dtype, name)
-    if isinstance(arg, (int, float, bool, tensor_spec.TensorSpec)):
-      return arg
-    return UnknownArgument()
-
-  # We are using the flattened paths to name the TensorSpecs. We need an
-  # explicit name for them downstream.
-  flattened_with_paths = nest.flatten_with_joined_string_paths(structure)
-  mapped = [encode_arg(arg, path) for path, arg in flattened_with_paths]
-  return nest.pack_sequence_as(structure, mapped)
-
-
 pywrap_tensorflow.RegisterType("Tensor", ops.Tensor)
 pywrap_tensorflow.RegisterType("IndexedSlices", ops.IndexedSlices)
 
@@ -1347,14 +1321,7 @@ class PolymorphicFunction(object):
                 autograph=self._autograph,
                 arg_names=arg_names),
             self._function_attributes)
-        if self._input_signature:
-          python_call_signature = self._input_signature
-        else:
-          python_call_signature = tuple(convert_structure_to_signature(args))
         # pylint: disable=protected-access
-        # Save information about non-Tensor arguments with the concrete
-        # function. Used to serialize PolymorphicFunctions.
-        graph_function._python_call_signature = python_call_signature
         # Tell the Function to clean up its graph once it goes out of
         # scope. Function does not do this in its constructor since it gets used
         # in some places (like Keras) where the FuncGraph lives longer than the
diff --git a/tensorflow/python/framework/func_graph.py b/tensorflow/python/framework/func_graph.py
index 9603c1536d..5202e9b8f9 100644
--- a/tensorflow/python/framework/func_graph.py
+++ b/tensorflow/python/framework/func_graph.py
@@ -58,6 +58,37 @@ WHITELIST_COLLECTIONS = [
 ]
 
 
+class UnknownArgument(object):
+  """Signifies an argument which is not currently handled."""
+  pass
+
+
+def convert_structure_to_signature(structure):
+  """Convert a potentially nested structure to a signature.
+
+  Args:
+    structure: Structure to convert.
+
+  Returns:
+    Identical structure that has TensorSpec objects instead of Tensors and
+    UknownArgument instead of any unsupported types.
+  """
+
+  def encode_arg(arg, name=None):
+    """A representation for this argument, for converting into signatures."""
+    if isinstance(arg, ops.Tensor):
+      return tensor_spec.TensorSpec(arg.shape, arg.dtype, name)
+    if isinstance(arg, (int, float, bool, tensor_spec.TensorSpec)):
+      return arg
+    return UnknownArgument()
+
+  # We are using the flattened paths to name the TensorSpecs. We need an
+  # explicit name for them downstream.
+  flattened_with_paths = nest.flatten_with_joined_string_paths(structure)
+  mapped = [encode_arg(arg, path) for path, arg in flattened_with_paths]
+  return nest.pack_sequence_as(structure, mapped)
+
+
 class FuncGraph(ops.Graph):
   """Graph representing a function body.
 
@@ -69,6 +100,9 @@ class FuncGraph(ops.Graph):
       inputs coming first.
     outputs: Tensors that will be returned by this function. The tensors are in
       this FuncGraph.
+    structured_input_signature: A possibly-nested python object that was
+      received by this function. Note that this structure might contain Python
+      `None`s.
     structured_outputs: A possibly-nested python object which will be returned
       by this function. The Tensors in this structure are the same as those of
       self.outputs. Note that this structure might contain Python `None`s.
@@ -101,6 +135,7 @@ class FuncGraph(ops.Graph):
     self.name = name
     self.inputs = []
     self.outputs = []
+    self.structured_input_signature = None
     self.structured_outputs = None
     self._weak_variables = []
     self.outer_graph = ops.get_default_graph()
@@ -407,6 +442,11 @@ def func_graph_from_py_func(name,
     func_args = _get_defun_inputs_from_args(args, arg_names)
     func_kwargs = _get_defun_inputs_from_kwargs(kwargs)
 
+    # Convert all Tensors into TensorSpecs before saving the structured inputs.
+    func_graph.structured_input_signature = (
+        convert_structure_to_signature(func_args),
+        convert_structure_to_signature(func_kwargs))
+
     # Note: `nest.flatten` sorts by keys, as does `_deterministic_dict_values`.
     # Variables to help check whether mutation happens in calling the function
     # Copy the recursive list, tuple and map structure, but not base objects
diff --git a/tensorflow/python/saved_model/function_serialization.py b/tensorflow/python/saved_model/function_serialization.py
index e5d3cc4bd2..8e8e3bc856 100644
--- a/tensorflow/python/saved_model/function_serialization.py
+++ b/tensorflow/python/saved_model/function_serialization.py
@@ -18,7 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.python.eager import function as defun_lib
+from tensorflow.python.framework import func_graph as func_graph_module
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.saved_model import nested_structure_coder
 from tensorflow.python.saved_model import saved_object_graph_pb2
@@ -64,7 +64,7 @@ def serialize_polymorphic_function(polymorphic_function, node_ids):
     function_proto.concrete_function = concrete_function.name
     function_proto.canonicalized_input_signature.CopyFrom(
         coder.encode_structure(signature))
-    structured_outputs = defun_lib.convert_structure_to_signature(
+    structured_outputs = func_graph_module.convert_structure_to_signature(
         concrete_function.structured_outputs)
     function_proto.output_signature.CopyFrom(
         coder.encode_structure(structured_outputs))
-- 
GitLab


From c223fbd116f2266ba7129eb8257c8c8a9eceb499 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 15 Jan 2019 10:15:20 -0800
Subject: [PATCH 0745/1095] Upgrade bazel-toolchain to the latest release

Also removed bazel-toolchain support for Bazel version prior to 0.19.0 and updated check_bazel_version_at_least in WORKSPACE file

To fix https://github.com/bazelbuild/bazel/issues/7113

PiperOrigin-RevId: 229392830
---
 WORKSPACE                                     |  2 +-
 .../preconfig/generate/archives.bzl           | 32 ++++++-------------
 2 files changed, 10 insertions(+), 24 deletions(-)

diff --git a/WORKSPACE b/WORKSPACE
index 1c59686f16..957b8d8528 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -73,7 +73,7 @@ swift_rules_dependencies()
 # files, in case the parsing of those build files depends on the bazel
 # version we require here.
 load("//tensorflow:version_check.bzl", "check_bazel_version_at_least")
-check_bazel_version_at_least("0.18.0")
+check_bazel_version_at_least("0.19.0")
 
 load("//tensorflow:workspace.bzl", "tf_workspace")
 
diff --git a/third_party/toolchains/preconfig/generate/archives.bzl b/third_party/toolchains/preconfig/generate/archives.bzl
index 0850893589..bafc7d4943 100644
--- a/third_party/toolchains/preconfig/generate/archives.bzl
+++ b/third_party/toolchains/preconfig/generate/archives.bzl
@@ -2,26 +2,12 @@ load("//tensorflow:version_check.bzl", "parse_bazel_version")
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
 def bazel_toolchains_archive():
-    # Not all bazel versions have set native.bazel_version - if it is not set,
-    # fall back to the more compatible version of the toolchains archive.
-    if native.bazel_version and parse_bazel_version(native.bazel_version) >= parse_bazel_version("0.19"):
-        # This version of the toolchains repo is incompatible with older bazel
-        # versions - we can remove this once TensorFlow drops support for bazel
-        # before 0.19.
-        http_archive(
-            name = "bazel_toolchains",
-            sha256 = "41c48a189be489e2d15dec40e0057ea15b95ee5b39cc2a7e6cf663e31432c75e",
-            strip_prefix = "bazel-toolchains-3f8c58fe530fedc446de04673bc1e32985887dea",
-            urls = [
-                "https://github.com/nlopezgi/bazel-toolchains/archive/3f8c58fe530fedc446de04673bc1e32985887dea.tar.gz",
-            ],
-        )
-    else:
-        http_archive(
-            name = "bazel_toolchains",
-            sha256 = "15b5858b1b5541ec44df31b94c3b8672815b31d71215a98398761ea9f4c4eedb",
-            strip_prefix = "bazel-toolchains-6200b238c9c2d137c0d9a7262c80cc71d98e692b",
-            urls = [
-                "https://github.com/bazelbuild/bazel-toolchains/archive/6200b238c9c2d137c0d9a7262c80cc71d98e692b.tar.gz",
-            ],
-        )
+    http_archive(
+      name = "bazel_toolchains",
+      sha256 = "ee854b5de299138c1f4a2edb5573d22b21d975acfc7aa938f36d30b49ef97498",
+      strip_prefix = "bazel-toolchains-37419a124bdb9af2fec5b99a973d359b6b899b61",
+      urls = [
+        "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/37419a124bdb9af2fec5b99a973d359b6b899b61.tar.gz",
+        "https://github.com/bazelbuild/bazel-toolchains/archive/37419a124bdb9af2fec5b99a973d359b6b899b61.tar.gz",
+      ],
+    )
-- 
GitLab


From 9d1b0f9ce7f31d9ed217c06e6c5163abf5f2944d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 15 Jan 2019 10:21:55 -0800
Subject: [PATCH 0746/1095] Automated fixes for core/kernels/BUILD: Remove an
 unreferenced import of the tf_proto_library rule. Remove full package
 qualifier from deps on targets in the same package.

PiperOrigin-RevId: 229394137
---
 tensorflow/core/kernels/BUILD | 35 +++++++++++++++++------------------
 1 file changed, 17 insertions(+), 18 deletions(-)

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 4f21b466cd..1be749289b 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -48,7 +48,6 @@ load("//tensorflow:tensorflow.bzl", "tf_cuda_cc_tests")
 load(
     "//tensorflow/core:platform/default/build_config.bzl",
     "tf_kernel_tests_linkstatic",
-    "tf_proto_library",
 )
 load(
     "//tensorflow/core:platform/default/build_config_root.bzl",
@@ -509,12 +508,12 @@ cc_library(
     name = "batch_kernels",
     srcs = ["batch_kernels.cc"],
     deps = [
+        ":concat_lib_hdrs",
+        ":ops_util_hdrs",
+        ":split_lib_hdrs",
         "//tensorflow/core:batch_ops_op_lib",
         "//tensorflow/core:framework_headers_lib",
         "//tensorflow/core:protos_all_cc",
-        "//tensorflow/core/kernels:concat_lib_hdrs",
-        "//tensorflow/core/kernels:ops_util_hdrs",
-        "//tensorflow/core/kernels:split_lib_hdrs",
         "//tensorflow/core/kernels/batching_util:periodic_function_dynamic",
         "//tensorflow/core/kernels/batching_util:shared_batch_scheduler_hdrs",
     ],
@@ -1147,13 +1146,13 @@ tf_cc_test(
     size = "small",
     srcs = ["ragged_gather_op_test.cc"],
     deps = [
+        ":ops_testutil",
         ":ragged_gather_op",
         "//tensorflow/core:framework",
         "//tensorflow/core:ragged_array_ops_op_lib",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
-        "//tensorflow/core/kernels:ops_testutil",
     ],
 )
 
@@ -1170,13 +1169,13 @@ tf_cc_test(
     name = "ragged_range_op_test",
     srcs = ["ragged_range_op_test.cc"],
     deps = [
+        ":ops_testutil",
         ":ragged_range_op",
         "//tensorflow/core:framework",
         "//tensorflow/core:ragged_math_ops_op_lib",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
-        "//tensorflow/core/kernels:ops_testutil",
     ],
 )
 
@@ -1194,6 +1193,7 @@ tf_cc_test(
     size = "small",
     srcs = ["ragged_tensor_to_sparse_kernel_test.cc"],
     deps = [
+        ":ops_testutil",
         ":ragged_tensor_to_sparse_kernel",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
@@ -1201,7 +1201,6 @@ tf_cc_test(
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
-        "//tensorflow/core/kernels:ops_testutil",
     ],
 )
 
@@ -1210,13 +1209,13 @@ tf_kernel_library(
     srcs = ["cudnn_rnn_ops.cc"],
     visibility = ["//visibility:public"],
     deps = [
+        ":bounds_check_lib",
         ":gpu_util_hdrs",
         "//tensorflow/core:cudnn_rnn_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:stream_executor",
-        "//tensorflow/core/kernels:bounds_check_lib",
         "//third_party/eigen3",
         "@farmhash_archive//:farmhash",
     ],
@@ -4857,6 +4856,8 @@ tf_cc_test(
     size = "small",
     srcs = ["string_format_op_test.cc"],
     deps = [
+        ":ops_testutil",
+        ":ops_util",
         ":string_format_op",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:framework",
@@ -4865,8 +4866,6 @@ tf_cc_test(
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
-        "//tensorflow/core/kernels:ops_testutil",
-        "//tensorflow/core/kernels:ops_util",
     ],
 )
 
@@ -4899,6 +4898,8 @@ tf_cc_test(
     size = "small",
     srcs = ["regex_replace_op_test.cc"],
     deps = [
+        ":ops_testutil",
+        ":ops_util",
         ":regex_replace_op",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:framework",
@@ -4907,8 +4908,6 @@ tf_cc_test(
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
-        "//tensorflow/core/kernels:ops_testutil",
-        "//tensorflow/core/kernels:ops_util",
     ],
 )
 
@@ -4923,6 +4922,8 @@ tf_cc_test(
     size = "small",
     srcs = ["string_split_op_test.cc"],
     deps = [
+        ":ops_testutil",
+        ":ops_util",
         ":string_split_op",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:framework",
@@ -4931,8 +4932,6 @@ tf_cc_test(
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
-        "//tensorflow/core/kernels:ops_testutil",
-        "//tensorflow/core/kernels:ops_util",
     ],
 )
 
@@ -4953,6 +4952,8 @@ tf_cc_test(
     size = "small",
     srcs = ["substr_op_test.cc"],
     deps = [
+        ":ops_testutil",
+        ":ops_util",
         ":substr_op",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:framework",
@@ -4962,8 +4963,6 @@ tf_cc_test(
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
-        "//tensorflow/core/kernels:ops_testutil",
-        "//tensorflow/core/kernels:ops_util",
     ],
 )
 
@@ -6618,6 +6617,7 @@ cc_library(
     srcs = ["remote_fused_graph_execute_op_test_utils.cc"],
     hdrs = ["remote_fused_graph_execute_op_test_utils.h"],
     deps = [
+        ":cwise_op",
         ":remote_fused_graph_execute_utils",
         "//tensorflow/cc:cc_ops",
         "//tensorflow/cc:ops",
@@ -6625,7 +6625,6 @@ cc_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:testlib",
-        "//tensorflow/core/kernels:cwise_op",
     ],
 )
 
@@ -6636,6 +6635,7 @@ tf_cc_test(
         "remote_fused_graph_execute_utils_test.cc",
     ],
     deps = [
+        ":cwise_op",
         ":remote_fused_graph_execute_op_test_utils",
         ":remote_fused_graph_execute_utils",
         "//tensorflow/cc:cc_ops",
@@ -6651,7 +6651,6 @@ tf_cc_test(
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
-        "//tensorflow/core/kernels:cwise_op",
     ],
 )
 
-- 
GitLab


From e0cf2939f71c7742a111de968d9008fa3f880570 Mon Sep 17 00:00:00 2001
From: Guangda Lai <31743510+aaroey@users.noreply.github.com>
Date: Tue, 15 Jan 2019 10:33:10 -0800
Subject: [PATCH 0747/1095] Add comments and fix build.

---
 tensorflow/contrib/tensorrt/BUILD             |  3 --
 .../contrib/tensorrt/kernels/trt_engine_op.cc |  9 +++-
 .../contrib/tensorrt/kernels/trt_engine_op.h  | 13 ------
 .../tensorrt/resources/trt_lru_cache.h        | 44 +++++++++----------
 .../test/dynamic_input_shapes_test.py         |  6 +++
 .../test/tf_trt_integration_test_base.py      | 33 +++++++++++---
 6 files changed, 61 insertions(+), 47 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index 3df56271de..17717c8403 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -511,11 +511,8 @@ cuda_py_tests(
         "test/conv2d_test.py",
         "test/dynamic_input_shapes_test.py",
         "test/identity_output_test.py",
-<<<<<<< ab254f392694eb2f7231526c64fc301ef10a1384
         "test/int32_test.py",
-=======
         "test/lru_cache_test.py",
->>>>>>> initial commit (add lru_cache class, enable trt_engine_op to use it)
         "test/manual_test.py",
         "test/memory_alignment_test.py",
         "test/multi_connection_neighbor_engine_test.py",
diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
index 06a62fed37..45f3b8d0e5 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
+++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
@@ -249,6 +249,10 @@ bool TRTEngineOp::GetCompatibleCachedEngine(
   *engine_input_shapes = actual_input_shapes;
   for (const int cached_batch_size : cached_engine_batch_sizes_) {
     // Check if compatible: batch <= cached batch.
+    //
+    // TODO(laigd): here it only compare the first dim a.k.a the batch size,
+    // we'll need to to support non-batch dimensions as well. This will be done
+    // as part of the offline conversion implementation.
     if (batch_size <= cached_batch_size) {
       // First case: first compatible engine found
       // Second case: smaller batch size engine found
@@ -466,7 +470,8 @@ EngineContext* TRTEngineOp::GetEngine(
       // TODO(tmorris): will all inputs have batch size as first dimension??
       engine_input_shapes[i].set_dim(0, max_batch_size);
     }
-    // Assume engine_input_shapes matches the input shapes of the engine.
+    // TODO(laigd): here we assume engine_input_shapes matches the actual input
+    // shapes of the engine, we should verify that.
     cache.emplace(engine_input_shapes,
                   absl::make_unique<EngineContext>(
                       std::move(static_engine),
@@ -484,7 +489,7 @@ EngineContext* TRTEngineOp::GetEngine(
   // See if there is a compatible engine cached. The batch size should be <= the
   // cached batch size.
   std::vector<tensorflow::TensorShape> engine_input_shapes;
-  bool matched_successfully =
+  const bool matched_successfully =
       GetCompatibleCachedEngine(input_shapes, &engine_input_shapes);
   // If matched, use that engine. Otherwise, we will look in cache for that
   // exact shape and possibly create a new engine if it is not in cache.
diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
index 4f39a4c548..c9b325cfd2 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
+++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
@@ -53,19 +53,6 @@ class TRTEngineOp : public AsyncOpKernel {
 
  private:
   // TODO(samikama): context should go to a resource manager!
-  struct EngineContext {
-    EngineContext() {}  // Creates an empty context.
-    EngineContext(
-        TrtUniquePtrType<nvinfer1::ICudaEngine>&& input_cuda_engine,
-        TrtUniquePtrType<nvinfer1::IExecutionContext>&& input_execution_context)
-        : cuda_engine(std::move(input_cuda_engine)),
-          execution_context(std::move(input_execution_context)) {}
-
-    mutex mu;
-    TrtUniquePtrType<nvinfer1::ICudaEngine> cuda_engine;
-    TrtUniquePtrType<nvinfer1::IExecutionContext> execution_context
-        GUARDED_BY(mu);
-  };
 
   // Execute calibration
   void ExecuteCalibration(OpKernelContext* ctx, AsyncHelper* helper);
diff --git a/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h b/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h
index 57267e5b71..c9648420d0 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h
+++ b/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h
@@ -22,8 +22,9 @@ class LRUCache {
   typedef Value value_type;
   typedef Key key_type;
   typedef HashFunction hasher;
-  typedef typename std::unordered_map<key_type, value_type, hasher>::iterator
-      map_iterator;
+  typedef typename std::unordered_map<key_type, value_type, hasher> map_type;
+  typedef typename map_type::iterator iterator;
+  typedef typename map_type::const_iterator const_iterator;
 
   LRUCache() : capacity_(0) {}
   explicit LRUCache(size_t capacity) : capacity_(capacity) {}
@@ -39,31 +40,26 @@ class LRUCache {
 
   size_t count(const key_type& k) const { return objects_.count(k); }
 
-  value_type& at(key_type k, tensorflow::Status* status_ptr = nullptr) {
-    tensorflow::Status status = Touch(k);
-    if (!status.ok()) {
-      if (status_ptr) {
-        *status_ptr = status;
-      }
-      return not_found_value_;
-    }
-    return objects_.at(k);
+  value_type& at(key_type k) {
+    return Touch(k);
   }
 
-  map_iterator begin() { return objects_.begin(); }
+  const_iterator begin() const { return objects_.begin(); }
+  const_iterator end() const { return objects_.end(); }
 
-  map_iterator end() { return objects_.end(); }
+  iterator begin() { return objects_.begin(); }
+  iterator end() { return objects_.end(); }
 
   template <typename... Args>
-  std::pair<map_iterator, bool> emplace(Args&&... args) {
+  std::pair<iterator, bool> emplace(Args&&... args) {
     DiscardOld(1);
-    std::pair<map_iterator, bool> result =
+    std::pair<iterator, bool> result =
         objects_.emplace(std::forward<Args>(args)...);
     key_type key = result.first->first;
     if (result.second) {
       keys_.push_front(key);
     } else {
-      Touch(key);
+      TouchNoCheck(key);  // The key must exist in this case.
     }
     return result;
   }
@@ -74,16 +70,20 @@ class LRUCache {
   size_t capacity_;
   value_type not_found_value_;
 
-  tensorflow::Status Touch(const key_type& key) {
-    if (!count(key)) {
-      return tensorflow::errors::NotFound("Key not found in cache.");
-    }
+  value_type& Touch(const key_type& key) {
+    // Check that the key exists, and let it return std::out_of_range error if
+    // not.
+    value_type& value = objects_.at(k);
+    TouchNoCheck(key);
+    return value;
+  }
+
+  void TouchNoCheck(const key_type& key) {
     auto rank = std::find(keys_.begin(), keys_.end(), key);
     if (rank != keys_.begin()) {
       keys_.erase(rank);
       keys_.push_front(key);
     }
-    return tensorflow::Status::OK();
   }
 
   // Creates n free positions in cache
@@ -149,7 +149,7 @@ class TRTEngineCacheResource : public tensorflow::ResourceBase {
     oss << "TRTBaseAllocator = " << hex << allocator_.get() << dec << ", ";
     oss << "LRUCache = " << hex << &cache_ << dec << endl;
     oss << "Containing " << cache_.size() << " entries: " << endl;
-    for (auto& item : cache_) {
+    for (const auto& item : cache_) {
       oss << TensorShapeUtils::ShapeListString(item.first) << ": " << hex
           << "ICudaEngine: " << item.second.get()->cuda_engine.get() << ", "
           << "IExecutionContext: " << item.second.get()->execution_context.get()
diff --git a/tensorflow/contrib/tensorrt/test/dynamic_input_shapes_test.py b/tensorflow/contrib/tensorrt/test/dynamic_input_shapes_test.py
index 9985561e1e..1c6d411d11 100644
--- a/tensorflow/contrib/tensorrt/test/dynamic_input_shapes_test.py
+++ b/tensorflow/contrib/tensorrt/test/dynamic_input_shapes_test.py
@@ -34,6 +34,12 @@ import numpy as np
 class DynamicInputShapesTest(trt_test.TfTrtIntegrationTestBase):
 
   def GetParams(self):
+    # TODO(laigd): we should test the following cases:
+    # - batch size is not changed, other dims are changing
+    # - batch size is decreasing, other dims are identical
+    # - batch size is decreasing, other dims are changing
+    # - batch size is increasing, other dims are identical
+    # - batch size is increasing, other dims are changing
     input_dims = [[[1, 5, 5, 1]], [[10, 5, 5, 1]], [[3, 5, 5, 1]],
                   [[1, 5, 5, 1]], [[1, 3, 1, 1]], [[2, 9, 9, 1]],
                   [[1, 224, 224, 1]], [[1, 128, 224, 1]]]
diff --git a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
index a105981298..db6be3828c 100644
--- a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
+++ b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
@@ -40,7 +40,15 @@ from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import tf_logging as logging
 
 TfTrtIntegrationTestParams = namedtuple("TfTrtIntegrationTestParams", [
-    "gdef", "input_names", "input_dims", "output_names", "expected_output_dims"
+    "gdef",
+    # A list of names of the input placeholder nodes.
+    "input_names",
+    # A list of list of output shapes of the input placeholder nodes.
+    "input_dims",
+    # A list of names of the output identity nodes.
+    "output_names",
+    # A list of list of expected output shapes of the output identity nodes.
+    "expected_output_dims"
 ])
 
 RunParams = namedtuple("RunParams", [
@@ -159,10 +167,19 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
 
   def GetConversionParams(self, run_params):
     """Return a ConversionParams for test."""
+    batch_list
+    for dims_list in self._GetParamsCached().input_dims:
+      if not len(dims_list):
+        continue
+      # Each list of shapes should have same batch size.
+      input_batches = [dims[0] for dims in dims_list]
+      assert max(input_batches) == min(input_batches)
+      batch_list.append(input_batches[0])
     return ConversionParams(
-        max_batch_size=max([
-            dims[0] for inp in self._GetParamsCached().input_dims if len(inp) for dims in inp
-        ]),
+        # We use the minimum of all the batch sizes, so when multiple different
+        # input shapes are provided it'll always create new engines in the
+        # cache, and we can therefore test the cache behavior.
+        max_batch_size=min(batch_list),
         max_workspace_size_bytes=1 << 25,
         precision_mode=run_params.precision_mode,
         minimum_segment_size=2,
@@ -317,8 +334,9 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
             output_len = len(params.expected_output_dims[ind])
             self.assertEqual(output_len, len(new_val))
             for i in range(output_len):
-              self.assertEqual(list(params.expected_output_dims[ind][i]),
-                               list(new_val[i].shape))
+              self.assertEqual(
+                  list(params.expected_output_dims[ind][i]),
+                  list(new_val[i].shape))
             if val is not None:
               self.assertAllClose(val, new_val, atol=1.e-06, rtol=1.e-06)
             val = new_val
@@ -497,7 +515,8 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
         # seq = np.arange(np.prod(dims))
         # seq.resize(dims)
         # input_data.append(scale * seq.astype(dtype))
-        current_input_data.append((scale * np.random.random_sample(dims)).astype(dtype))
+        current_input_data.append(
+            (scale * np.random.random_sample(dims)).astype(dtype))
       inputs_data.append(current_input_data)
 
     self._VerifyGraphDef(run_params, input_gdef, GraphState.ORIGINAL)
-- 
GitLab


From 431165fc97f12d19d6311dfb8b882e3c36d1ca54 Mon Sep 17 00:00:00 2001
From: Guangda Lai <31743510+aaroey@users.noreply.github.com>
Date: Tue, 15 Jan 2019 10:40:05 -0800
Subject: [PATCH 0748/1095] Fix a bug: when building static engine, it should
 use engine input shapes as key instead of actual input shapes to lookup the
 cache. Also fix build.

---
 tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc  | 6 ++----
 tensorflow/contrib/tensorrt/resources/trt_lru_cache.h | 8 ++++----
 2 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
index 45f3b8d0e5..f089f2c1cc 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
+++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
@@ -482,7 +482,7 @@ EngineContext* TRTEngineOp::GetEngine(
     if (max_batch_size < batch_size) {
       return &empty_context;
     }
-    return cache.at(input_shapes).get();
+    return cache.at(engine_input_shapes).get();
   }  // static_engine_
 
   // Handle the dynamic engine case.
@@ -503,10 +503,8 @@ EngineContext* TRTEngineOp::GetEngine(
       cached_engine_batch_sizes_.push_back(batch_size);
     }
   }
-  tensorflow::Status engine_found_status;
-  cache.at(engine_input_shapes, &engine_found_status);
 
-  if (!engine_found_status.ok()) {
+  if (!cache.count(engine_input_shapes)) {
     TrtUniquePtrType<nvinfer1::ICudaEngine> engine;
     bool convert_successfully = false;
     LOG(INFO) << "Building a new TensorRT engine for " << name()
diff --git a/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h b/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h
index c9648420d0..ba455b9b35 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h
+++ b/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h
@@ -38,10 +38,10 @@ class LRUCache {
 
   size_t size() const { return objects_.size(); }
 
-  size_t count(const key_type& k) const { return objects_.count(k); }
+  size_t count(const key_type& key) const { return objects_.count(key); }
 
-  value_type& at(key_type k) {
-    return Touch(k);
+  value_type& at(const key_type& key) {
+    return Touch(key);
   }
 
   const_iterator begin() const { return objects_.begin(); }
@@ -73,7 +73,7 @@ class LRUCache {
   value_type& Touch(const key_type& key) {
     // Check that the key exists, and let it return std::out_of_range error if
     // not.
-    value_type& value = objects_.at(k);
+    value_type& value = objects_.at(key);
     TouchNoCheck(key);
     return value;
   }
-- 
GitLab


From 347bcbd4ffda686e0b4df95b451e0101c29f9a72 Mon Sep 17 00:00:00 2001
From: Shashi Shekhar <shashishekhar@google.com>
Date: Tue, 15 Jan 2019 10:37:54 -0800
Subject: [PATCH 0749/1095] Add quantization for some activation only ops that
 don't have weights. Ops included: - Squeeze - Softmax - Avg and Max pool.

PiperOrigin-RevId: 229397364
---
 .../tools/optimize/quantize_model_test.cc     |   2 +-
 .../lite/tools/optimize/subgraph_quantizer.cc |  90 +++++++++-
 .../lite/tools/optimize/subgraph_quantizer.h  |  14 ++
 .../tools/optimize/subgraph_quantizer_test.cc | 154 ++++++++++++++++--
 tensorflow/lite/tools/optimize/test_util.cc   |  10 +-
 tensorflow/lite/tools/optimize/test_util.h    |  12 +-
 .../lite/tools/optimize/testdata/README.md    |  23 +++
 7 files changed, 285 insertions(+), 20 deletions(-)
 create mode 100644 tensorflow/lite/tools/optimize/testdata/README.md

diff --git a/tensorflow/lite/tools/optimize/quantize_model_test.cc b/tensorflow/lite/tools/optimize/quantize_model_test.cc
index a25f2cc91f..52ed16c0b8 100644
--- a/tensorflow/lite/tools/optimize/quantize_model_test.cc
+++ b/tensorflow/lite/tools/optimize/quantize_model_test.cc
@@ -38,7 +38,7 @@ namespace {
 
 std::unique_ptr<FlatBufferModel> ReadTestModel() {
   auto model_path = tensorflow::io::JoinPath(
-      *g_test_model_dir, internal::kModelWith0Plus10Weights);
+      *g_test_model_dir, internal::kConvModelWith0Plus10Weights);
   return FlatBufferModel::BuildFromFile(model_path.c_str());
 }
 
diff --git a/tensorflow/lite/tools/optimize/subgraph_quantizer.cc b/tensorflow/lite/tools/optimize/subgraph_quantizer.cc
index 27aee83cfd..56f2e6cf76 100644
--- a/tensorflow/lite/tools/optimize/subgraph_quantizer.cc
+++ b/tensorflow/lite/tools/optimize/subgraph_quantizer.cc
@@ -14,6 +14,7 @@ limitations under the License.
 ==============================================================================*/
 #include "tensorflow/lite/tools/optimize/subgraph_quantizer.h"
 
+#include <algorithm>
 #include <limits>
 
 #include "flatbuffers/flexbuffers.h"
@@ -33,6 +34,8 @@ namespace optimize {
 namespace internal {
 
 namespace {
+const int8_t kMinQuantizedValue = -127;
+const int8_t kMaxQuantizedValue = 127;
 
 TfLiteStatus AddQuantizationParams(const SymmetricPerChannelParams& params,
                                    const uint8_t* buffer_data,
@@ -131,9 +134,7 @@ TfLiteStatus SymmetricPerChannelQuantizeTensor(ModelT* model, TensorT* tensor,
   // Calculate scales per channel
   std::vector<float> scales(channel_dim_size);
   std::vector<float> scale_invs(channel_dim_size);
-  const int8_t kMinValue = -127;
-  const int8_t kMaxValue = 127;
-  const float half_scale = kMaxValue;
+  const float half_scale = kMaxQuantizedValue;
   for (size_t channel_idx = 0; channel_idx < channel_dim_size; channel_idx++) {
     const float half_range = std::max(std::abs(min_vals[channel_idx]),
                                       std::abs(max_vals[channel_idx]));
@@ -159,7 +160,8 @@ TfLiteStatus SymmetricPerChannelQuantizeTensor(ModelT* model, TensorT* tensor,
           const int32_t quantized_value =
               static_cast<int32_t>(TfLiteRound(val * scale_invs[channel_idx]));
           final_buffer[index] = std::min<int8_t>(
-              kMaxValue, std::max<int8_t>(kMinValue, quantized_value));
+              kMaxQuantizedValue,
+              std::max<int8_t>(kMinQuantizedValue, quantized_value));
         }
       }
     }
@@ -288,7 +290,9 @@ TfLiteStatus SubgraphQuantizer::QuantizeOpWithBias(BuiltinOperator op_code,
     return kTfLiteError;
   }
   auto input_tensor_idx = op->inputs[op_tensor_info->activation_input_index];
-  TF_LITE_ENSURE_STATUS(AsymmetricQuantizeTensor(op_code, input_tensor_idx));
+  if (IsSubgraphInput(input_tensor_idx)) {
+    TF_LITE_ENSURE_STATUS(AsymmetricQuantizeTensor(op_code, input_tensor_idx));
+  }
   auto weights_tensor_idx = op->inputs[op_tensor_info->weights_input_index];
 
   TensorT* weights_tensor = subgraph_->tensors[weights_tensor_idx].get();
@@ -317,6 +321,72 @@ TfLiteStatus SubgraphQuantizer::QuantizeOpWithBias(BuiltinOperator op_code,
   return kTfLiteOk;
 }
 
+TfLiteStatus SubgraphQuantizer::PropagateMinMaxForAvgAndMaxPool(
+    BuiltinOperator op_code, OperatorT* op) {
+  TF_LITE_ENSURE_EQ(this->error_reporter_, op->inputs.size(), 1);
+
+  if (IsSubgraphInput(op->inputs[0])) {
+    TF_LITE_ENSURE_STATUS(AsymmetricQuantizeTensor(op_code, op->inputs[0]));
+  }
+
+  auto output_tensor = subgraph_->tensors[op->outputs[0]].get();
+  if (output_tensor->type != TensorType_FLOAT32) {
+    return kTfLiteOk;
+  }
+  auto input_tensor = subgraph_->tensors[op->inputs[0]].get();
+  if (!input_tensor->quantization) {
+    error_reporter_->Report(
+        "Missing required min/max information for input of operation: %s",
+        EnumNameBuiltinOperator(op_code));
+    return kTfLiteError;
+  }
+  if (input_tensor->quantization->min.size() != 1 ||
+      input_tensor->quantization->max.size() != 1 ||
+      input_tensor->quantization->scale.size() != 1 ||
+      input_tensor->quantization->zero_point.size() != 1) {
+    error_reporter_->Report(
+        "Invalid quantization information for Op: %s, tensor: %s",
+        EnumNameBuiltinOperator(op_code), input_tensor->name.c_str());
+    return kTfLiteError;
+  }
+  auto quant_params = absl::make_unique<QuantizationParametersT>();
+  // Nudge min, max to include the floating point zero.
+  const float min = std::min(0.f, input_tensor->quantization->min[0]);
+  const float max = std::max(0.f, input_tensor->quantization->max[0]);
+  quant_params->min.push_back(min);
+  quant_params->max.push_back(max);
+  quant_params->scale.push_back(input_tensor->quantization->scale[0]);
+  quant_params->zero_point.push_back(input_tensor->quantization->zero_point[0]);
+  // TODO(shashishekhar): Log a warning here if overriding existing
+  // min/max/scales differ from input scales.
+  output_tensor->quantization = std::move(quant_params);
+  output_tensor->type = TensorType_INT8;
+  return kTfLiteOk;
+}
+
+TfLiteStatus SubgraphQuantizer::AsymmetricQuantizeSingleInputOutputOp(
+    BuiltinOperator op_code, OperatorT* op) {
+  TF_LITE_ENSURE_EQ(this->error_reporter_, op->inputs.size(), 1);
+  TF_LITE_ENSURE_EQ(this->error_reporter_, op->outputs.size(), 1);
+
+  if (IsSubgraphInput(op->inputs[0])) {
+    TF_LITE_ENSURE_STATUS(AsymmetricQuantizeTensor(op_code, op->inputs[0]));
+  }
+
+  auto output_tensor = subgraph_->tensors[op->outputs[0]].get();
+  if (output_tensor->type != TensorType_FLOAT32) {
+    return kTfLiteOk;
+  }
+  auto quant_params = absl::make_unique<QuantizationParametersT>();
+  TF_LITE_ENSURE_STATUS(AsymmetricQuantizeTensor(op_code, op->outputs[0]));
+  return kTfLiteOk;
+}
+
+bool SubgraphQuantizer::IsSubgraphInput(int32_t tensor_idx) const {
+  return std::find(subgraph_->inputs.begin(), subgraph_->inputs.end(),
+                   tensor_idx) != subgraph_->inputs.end();
+}
+
 TfLiteStatus SubgraphQuantizer::QuantizeOperator(int op_idx) {
   OperatorT* op = subgraph_->operators[op_idx].get();
   const BuiltinOperator op_code =
@@ -324,6 +394,16 @@ TfLiteStatus SubgraphQuantizer::QuantizeOperator(int op_idx) {
   if (OpHasOptionalBiasTensor(op_code)) {
     return QuantizeOpWithBias(op_code, op);
   }
+  switch (op_code) {
+    case BuiltinOperator_AVERAGE_POOL_2D:
+    case BuiltinOperator_MAX_POOL_2D:
+      return PropagateMinMaxForAvgAndMaxPool(op_code, op);
+    case BuiltinOperator_SQUEEZE:
+    case BuiltinOperator_SOFTMAX:
+      return AsymmetricQuantizeSingleInputOutputOp(op_code, op);
+    default:
+      return kTfLiteError;
+  }
 
   return kTfLiteError;
 }
diff --git a/tensorflow/lite/tools/optimize/subgraph_quantizer.h b/tensorflow/lite/tools/optimize/subgraph_quantizer.h
index 2543dccea9..9d6ca7fad5 100644
--- a/tensorflow/lite/tools/optimize/subgraph_quantizer.h
+++ b/tensorflow/lite/tools/optimize/subgraph_quantizer.h
@@ -40,9 +40,23 @@ class SubgraphQuantizer {
  private:
   // Quantizes ops with bias tensors.
   TfLiteStatus QuantizeOpWithBias(BuiltinOperator op_code, OperatorT* op);
+
+  // Average and Max pool need special treatement. The scales are propagated
+  // from inputs to outputs.
+  TfLiteStatus PropagateMinMaxForAvgAndMaxPool(BuiltinOperator op_code,
+                                               OperatorT* op);
+
+  // Asymmetric quantizes inputs and outputs of an Op that has single input and
+  // single output. E.g. Squeeze.
+  TfLiteStatus AsymmetricQuantizeSingleInputOutputOp(BuiltinOperator op_code,
+                                                     OperatorT* op);
+
   TfLiteStatus AsymmetricQuantizeTensor(BuiltinOperator op_code,
                                         int32_t tensor_idx);
 
+  // Returns true if |tensor_idx| is one of the inputs in the subgraph.
+  bool IsSubgraphInput(int32_t tensor_idx) const;
+
   ModelT* model_;
   SubGraphT* subgraph_;
   ErrorReporter* error_reporter_;
diff --git a/tensorflow/lite/tools/optimize/subgraph_quantizer_test.cc b/tensorflow/lite/tools/optimize/subgraph_quantizer_test.cc
index 7652429e32..96ba20d4d9 100644
--- a/tensorflow/lite/tools/optimize/subgraph_quantizer_test.cc
+++ b/tensorflow/lite/tools/optimize/subgraph_quantizer_test.cc
@@ -12,13 +12,15 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#include "tensorflow/lite/tools/optimize/subgraph_quantizer.h"
+#include <algorithm>
+
 #include <gmock/gmock.h>
 #include <gtest/gtest.h>
 #include "tensorflow/core/lib/io/path.h"
 #include "tensorflow/core/platform/init_main.h"
 #include "tensorflow/core/util/command_line_flags.h"
 #include "tensorflow/lite/schema/schema_generated.h"
+#include "tensorflow/lite/tools/optimize/subgraph_quantizer.h"
 #include "tensorflow/lite/tools/optimize/symmetric_per_channel_params.h"
 #include "tensorflow/lite/tools/optimize/test_util.h"
 
@@ -31,22 +33,31 @@ namespace optimize {
 namespace internal {
 namespace {
 
-std::unique_ptr<FlatBufferModel> ReadTestModel1() {
-  auto model_path = tensorflow::io::JoinPath(*g_test_model_dir,
-                                             kModelWithMinus128Plus127Weights);
+std::unique_ptr<FlatBufferModel> ReadModel(const char* model) {
+  auto model_path = tensorflow::io::JoinPath(*g_test_model_dir, model);
   return FlatBufferModel::BuildFromFile(model_path.c_str());
 }
 
-std::unique_ptr<FlatBufferModel> ReadTestModel2() {
-  auto model_path =
-      tensorflow::io::JoinPath(*g_test_model_dir, kModelWith0Plus10Weights);
-  return FlatBufferModel::BuildFromFile(model_path.c_str());
+std::unique_ptr<FlatBufferModel> ReadConvModel1() {
+  return ReadModel(kConvModelWithMinus128Plus127Weights);
+}
+
+std::unique_ptr<FlatBufferModel> ReadConvModel2() {
+  return ReadModel(kConvModelWith0Plus10Weights);
+}
+
+std::unique_ptr<FlatBufferModel> ReadSoftmaxModel() {
+  return ReadModel(kSingleSoftmaxModelMinMinus5MaxPlus5);
+}
+
+std::unique_ptr<FlatBufferModel> ReadAvgPoolModel() {
+  return ReadModel(kSingleAvgPoolModelMinMinus5MaxPlus5);
 }
 
 TEST(SubgraphQuantizerTest, VerifyConvQuantizationWithUnitScale) {
   ASSERT_TRUE(g_test_model_dir);
   ASSERT_FALSE(g_test_model_dir->empty());
-  auto test_model = ReadTestModel1();
+  auto test_model = ReadConvModel1();
   ASSERT_TRUE(test_model);
   auto readonly_model = test_model->GetModel();
   ASSERT_TRUE(readonly_model);
@@ -145,7 +156,7 @@ TEST(SubgraphQuantizerTest, VerifyConvQuantizationWithUnitScale) {
 TEST(SubgraphQuantizerTest, VerifyConvQuantization) {
   ASSERT_TRUE(g_test_model_dir);
   ASSERT_FALSE(g_test_model_dir->empty());
-  auto test_model = ReadTestModel2();
+  auto test_model = ReadConvModel2();
   ASSERT_TRUE(test_model);
   auto readonly_model = test_model->GetModel();
   ASSERT_TRUE(readonly_model);
@@ -237,6 +248,129 @@ TEST(SubgraphQuantizerTest, VerifyConvQuantization) {
   }
 }
 
+void VerifyAsymmetricQuantizationScale(
+    const QuantizationParameters& float_quant_params,
+    const QuantizationParametersT& quantized_quant_params) {
+  const float eps = 1e-7;
+  ASSERT_EQ(float_quant_params.min()->size(), 1);
+  ASSERT_EQ(float_quant_params.max()->size(), 1);
+  float float_min = std::min(0.f, float_quant_params.min()->Get(0));
+  float float_max = std::max(0.f, float_quant_params.max()->Get(0));
+
+  ASSERT_EQ(quantized_quant_params.scale.size(), 1);
+  ASSERT_EQ(quantized_quant_params.zero_point.size(), 1);
+
+  float scale = (float_max - float_min) / 255;
+  EXPECT_NEAR(scale, quantized_quant_params.scale[0], eps);
+}
+
+TEST(SubgraphQuantizerTest, VerifySoftmaxQuantization) {
+  ASSERT_TRUE(g_test_model_dir);
+  ASSERT_FALSE(g_test_model_dir->empty());
+  auto test_model = ReadSoftmaxModel();
+  ASSERT_TRUE(test_model);
+  auto readonly_model = test_model->GetModel();
+  ASSERT_TRUE(readonly_model);
+  ASSERT_TRUE(readonly_model->subgraphs());
+  ASSERT_GE(readonly_model->subgraphs()->size(), 1);
+  tflite::ModelT model;
+  readonly_model->UnPackTo(&model);
+  auto subgraph = model.subgraphs[0].get();
+  FailOnErrorReporter error_reporter;
+  SubgraphQuantizer quantizer(&model, subgraph, &error_reporter);
+  auto status = quantizer.QuantizeOperator(0);
+  ASSERT_EQ(kTfLiteOk, status);
+
+  auto op = subgraph->operators[0].get();
+  // Model has a single softmax op.
+  ASSERT_EQ(op->opcode_index, 0);
+  ASSERT_EQ(model.operator_codes[0].get()->builtin_code,
+            BuiltinOperator_SOFTMAX);
+
+  ASSERT_EQ(op->inputs.size(), 1);
+  ASSERT_EQ(op->outputs.size(), 1);
+  auto float_graph = readonly_model->subgraphs()->Get(0);
+
+  ASSERT_EQ(float_graph->tensors()->Get(op->inputs[0])->type(),
+            TensorType_FLOAT32);
+  ASSERT_EQ(float_graph->tensors()->Get(op->outputs[0])->type(),
+            TensorType_FLOAT32);
+
+  EXPECT_EQ(subgraph->tensors[op->inputs[0]].get()->type, TensorType_INT8);
+  EXPECT_EQ(subgraph->tensors[op->outputs[0]].get()->type, TensorType_INT8);
+
+  auto float_input_quant_params =
+      float_graph->tensors()->Get(op->inputs[0])->quantization();
+  auto input_quant_params =
+      subgraph->tensors[op->inputs[0]]->quantization.get();
+  VerifyAsymmetricQuantizationScale(*float_input_quant_params,
+                                    *input_quant_params);
+
+  auto float_output_quant_params =
+      float_graph->tensors()->Get(op->outputs[0])->quantization();
+  auto output_quant_params =
+      subgraph->tensors[op->outputs[0]]->quantization.get();
+  VerifyAsymmetricQuantizationScale(*float_output_quant_params,
+                                    *output_quant_params);
+}
+
+TEST(SubgraphQuantizerTest, VerifyAvgPoolQuantization) {
+  ASSERT_TRUE(g_test_model_dir);
+  ASSERT_FALSE(g_test_model_dir->empty());
+  auto test_model = ReadAvgPoolModel();
+  ASSERT_TRUE(test_model);
+  auto readonly_model = test_model->GetModel();
+  ASSERT_TRUE(readonly_model);
+  ASSERT_TRUE(readonly_model->subgraphs());
+  ASSERT_GE(readonly_model->subgraphs()->size(), 1);
+  tflite::ModelT model;
+  readonly_model->UnPackTo(&model);
+  auto subgraph = model.subgraphs[0].get();
+  FailOnErrorReporter error_reporter;
+  SubgraphQuantizer quantizer(&model, subgraph, &error_reporter);
+  auto status = quantizer.QuantizeOperator(0);
+  ASSERT_EQ(kTfLiteOk, status);
+
+  auto op = subgraph->operators[0].get();
+  // Model has a single AveragePool op.
+  ASSERT_EQ(op->opcode_index, 0);
+  ASSERT_EQ(model.operator_codes[0].get()->builtin_code,
+            BuiltinOperator_AVERAGE_POOL_2D);
+
+  ASSERT_EQ(op->inputs.size(), 1);
+  ASSERT_EQ(op->outputs.size(), 1);
+
+  auto float_graph = readonly_model->subgraphs()->Get(0);
+  ASSERT_EQ(float_graph->tensors()->Get(op->inputs[0])->type(),
+            TensorType_FLOAT32);
+  ASSERT_EQ(float_graph->tensors()->Get(op->outputs[0])->type(),
+            TensorType_FLOAT32);
+
+  EXPECT_EQ(subgraph->tensors[op->inputs[0]].get()->type, TensorType_INT8);
+  EXPECT_EQ(subgraph->tensors[op->outputs[0]].get()->type, TensorType_INT8);
+
+  auto float_input_quant_params =
+      float_graph->tensors()->Get(op->inputs[0])->quantization();
+  auto input_quant_params =
+      subgraph->tensors[op->inputs[0]]->quantization.get();
+  VerifyAsymmetricQuantizationScale(*float_input_quant_params,
+                                    *input_quant_params);
+
+  auto float_output_quant_params =
+      float_graph->tensors()->Get(op->outputs[0])->quantization();
+  auto output_quant_params =
+      subgraph->tensors[op->outputs[0]]->quantization.get();
+  ASSERT_EQ(float_output_quant_params->min()->size(), 1);
+  ASSERT_EQ(float_output_quant_params->max()->size(), 1);
+  ASSERT_EQ(output_quant_params->min.size(), 1);
+  ASSERT_EQ(output_quant_params->max.size(), 1);
+
+  // Make sure the input min/maxes are propagated to outputs.
+  EXPECT_EQ(input_quant_params->min[0], output_quant_params->min[0]);
+  EXPECT_EQ(input_quant_params->max[0], output_quant_params->max[0]);
+  EXPECT_EQ(input_quant_params->scale[0], output_quant_params->scale[0]);
+}
+
 }  // namespace
 }  // namespace internal
 }  // namespace optimize
diff --git a/tensorflow/lite/tools/optimize/test_util.cc b/tensorflow/lite/tools/optimize/test_util.cc
index 83b9f0dcd3..3506142901 100644
--- a/tensorflow/lite/tools/optimize/test_util.cc
+++ b/tensorflow/lite/tools/optimize/test_util.cc
@@ -19,12 +19,18 @@ limitations under the License.
 namespace tflite {
 namespace optimize {
 namespace internal {
-const char* kModelWithMinus128Plus127Weights =
+const char* kConvModelWithMinus128Plus127Weights =
     "single_conv_weights_min_minus_127_max_plus_127.tflite";
 
-const char* kModelWith0Plus10Weights =
+const char* kConvModelWith0Plus10Weights =
     "single_conv_weights_min_0_max_plus_10.tflite";
 
+const char* kSingleSoftmaxModelMinMinus5MaxPlus5 =
+    "single_softmax_min_minus_5_max_plus_5.tflite";
+
+const char* kSingleAvgPoolModelMinMinus5MaxPlus5 =
+    "single_avg_pool_min_minus_5_max_plus_5.tflite";
+
 int FailOnErrorReporter::Report(const char* format, va_list args) {
   char buf[1024];
   vsnprintf(buf, sizeof(buf), format, args);
diff --git a/tensorflow/lite/tools/optimize/test_util.h b/tensorflow/lite/tools/optimize/test_util.h
index 26da41e9cc..1981068acc 100644
--- a/tensorflow/lite/tools/optimize/test_util.h
+++ b/tensorflow/lite/tools/optimize/test_util.h
@@ -26,13 +26,21 @@ namespace internal {
 // channel has at least one weight as -127 and one weight as 127.
 // The activations are all in range: [-128, 127]
 // This means all bias computations should result in 1.0 scale.
-extern const char* kModelWithMinus128Plus127Weights;
+extern const char* kConvModelWithMinus128Plus127Weights;
 
 // Test model with single convolution where all weights are integers between
 // [0, 10] weights are randomly distributed. It is not guaranteed that min max
 // for weights are going to appear in each channel.
 // Activations have min = 0, max = 10.
-extern const char* kModelWith0Plus10Weights;
+extern const char* kConvModelWith0Plus10Weights;
+
+// A floating point model with a single softmax. The input tensor has min
+// and max in range [-5, 5], not necessarily -5 or +5.
+extern const char* kSingleSoftmaxModelMinMinus5MaxPlus5;
+
+// A floating point model with a single average pool. The input tensor has min
+// and max in range [-5, 5], not necessarily -5 or +5.
+extern const char* kSingleAvgPoolModelMinMinus5MaxPlus5;
 
 // An error reporter that fails on testing.
 class FailOnErrorReporter : public ErrorReporter {
diff --git a/tensorflow/lite/tools/optimize/testdata/README.md b/tensorflow/lite/tools/optimize/testdata/README.md
new file mode 100644
index 0000000000..b6b331b828
--- /dev/null
+++ b/tensorflow/lite/tools/optimize/testdata/README.md
@@ -0,0 +1,23 @@
+# Test models for testing quantization
+
+This directory contains test models for testing quantization.
+
+## Models
+
+* `single_conv_weights_min_0_max_plus_10.tflite` \
+   A floating point model with single convolution where all weights are
+   integers between [0, 10] weights are randomly distributed. It is not
+   guaranteed that min max for weights are going to appear in each channel.
+   All activations have min maxes and activations are in range [0,10].
+* `single_conv_weights_min_minus_127_max_plus_127.tflite` \
+   A floating point model with a single convolution where weights of the model
+   are all integers that lie in range[-127, 127]. The weights have been put in
+   such a way that each channel has at least one weight as -127 and one weight
+   as 127. The activations are all in range: [-128, 127].
+   This means all bias computations should result in 1.0 scale.
+* `single_softmax_min_minus_5_max_5.tflite` \
+   A floating point model with a single softmax. The input tensor has min
+   and max in range [-5, 5], not necessarily -5 or +5.
+* `single_avg_pool_input_min_minus_5_max_5.tflite` \
+   A floating point model with a single average pool. The input tensor has min
+   and max in range [-5, 5], not necessarily -5 or +5.
-- 
GitLab


From 1cbfa21e2a3f18b0396369e7194e3dbb4bedb08c Mon Sep 17 00:00:00 2001
From: Tom Hennigan <tomhennigan@google.com>
Date: Tue, 15 Jan 2019 10:39:11 -0800
Subject: [PATCH 0750/1095] Support Python 2 slicing on ListWrapper.

"Deprecated since version 2.0: Support slice objects as parameters to the
__getitem__() method. (However, built-in types in CPython currently still
implement __getslice__(). Therefore, you have to override it in derived classes
when implementing slicing.)"

PiperOrigin-RevId: 229397602
---
 .../python/training/checkpointable/data_structures.py       | 3 +++
 .../python/training/checkpointable/data_structures_test.py  | 6 ++++++
 2 files changed, 9 insertions(+)

diff --git a/tensorflow/python/training/checkpointable/data_structures.py b/tensorflow/python/training/checkpointable/data_structures.py
index 759709aa3d..d1e617ed46 100644
--- a/tensorflow/python/training/checkpointable/data_structures.py
+++ b/tensorflow/python/training/checkpointable/data_structures.py
@@ -304,6 +304,9 @@ class List(CheckpointableDataStructure, collections.Sequence):
   def __getitem__(self, key):
     return self._storage[key]
 
+  def __getslice__(self, i, j):
+    return self._storage[slice(i, j)]
+
   def __len__(self):
     return len(self._storage)
 
diff --git a/tensorflow/python/training/checkpointable/data_structures_test.py b/tensorflow/python/training/checkpointable/data_structures_test.py
index bcec6e0100..79191f3448 100644
--- a/tensorflow/python/training/checkpointable/data_structures_test.py
+++ b/tensorflow/python/training/checkpointable/data_structures_test.py
@@ -271,6 +271,12 @@ class ListTests(test.TestCase):
     with self.assertRaises(TypeError):
       has_sequences.add(data_structures._ListWrapper([]))
 
+  def testSlicing(self):
+    l = data_structures._ListWrapper([1, 2, 3, 4])
+    self.assertEqual(l[1:], [2, 3, 4])
+    self.assertEqual(l[1:-1], [2, 3])
+    self.assertEqual(l[:-1], [1, 2, 3])
+
 
 class HasMapping(training.Model):
 
-- 
GitLab


From 2e5ee61e2e0d876f91a66a1fe6279105b14d7534 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 15 Jan 2019 10:45:54 -0800
Subject: [PATCH 0751/1095] Delete deprecated and unused methods from
 distribute_lib. These methods all had the
 `@doc_controls.do_not_generate_docs` decorator since we knew they were going
 away as soon as references to them were updated to point to the official APIs
 decided in the accepted RFC:
 https://github.com/tensorflow/community/blob/master/rfcs/20181016-replicator.md

PiperOrigin-RevId: 229398974
---
 .../python/distribute/distribute_lib.py       | 161 +-----------------
 .../python/distribute/mirrored_strategy.py    |   4 +-
 ...orflow.distribute.-mirrored-strategy.pbtxt |  56 ------
 ...nsorflow.distribute.-replica-context.pbtxt |   4 -
 .../v1/tensorflow.distribute.-strategy.pbtxt  |  56 ------
 ...orflow.distribute.-mirrored-strategy.pbtxt |  56 ------
 ...nsorflow.distribute.-replica-context.pbtxt |   4 -
 .../v2/tensorflow.distribute.-strategy.pbtxt  |  56 ------
 8 files changed, 4 insertions(+), 393 deletions(-)

diff --git a/tensorflow/python/distribute/distribute_lib.py b/tensorflow/python/distribute/distribute_lib.py
index 421b2a6520..f112787ba2 100644
--- a/tensorflow/python/distribute/distribute_lib.py
+++ b/tensorflow/python/distribute/distribute_lib.py
@@ -505,33 +505,6 @@ class DistributionStrategy(object):
     """DEPRECATED: use extended.broadcast_to() instead."""
     return self._extended.broadcast_to(tensor, destinations)
 
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def run_steps_on_dataset(self, fn, iterator, iterations=1,
-                           initial_loop_values=None):
-    """DEPRECATED: use extended.experimental_run_steps_on_iterator() instead."""
-    return self._extended.experimental_run_steps_on_iterator(
-        fn, iterator, iterations, initial_loop_values)
-
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def call_for_each_replica(self, fn, *args, **kwargs):
-    """DEPRECATED: use extended.call_for_each_replica() instead."""
-    # Handle old *args, **kwargs, and new args=(...), kwargs={...}, to
-    # allow transition.
-    a = kwargs.pop("args", None)
-    if a is not None:
-      if args:
-        raise ValueError(
-            "Can't pass *args and args=... to call_for_each_replica")
-      args = a
-    k = kwargs.pop("kwargs", None)
-    if k is not None:
-      if kwargs:
-        raise ValueError(
-            "Can't pass **kwargs and kwargs=... to call_for_each_replica")
-      kwargs = k
-    kwargs.pop("run_concurrently", None)  # Ignore old option.
-    return self._extended.call_for_each_replica(fn, args, kwargs)
-
   def reduce(self, reduce_op, value):
     """Reduce `value` across replicas.
 
@@ -546,58 +519,6 @@ class DistributionStrategy(object):
     _require_cross_replica_context_extended(self._extended)
     return self._extended._reduce(reduce_op, value)  # pylint: disable=protected-access
 
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def batch_reduce(self, aggregation, value_destination_pairs):
-    """DEPRECATED: use extended.batch_reduce_to() instead."""
-    return self._extended.batch_reduce_to(aggregation, value_destination_pairs)
-
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def update(self, var, fn, *args, **kwargs):
-    """DEPRECATED: use extended.update() instead."""
-    group = kwargs.pop("group", True)
-    # We temporarily support "grouped" in addition to "group" for backward-
-    # compatibility.
-    group = kwargs.pop("grouped", True) and group
-    # Handle old *args, **kwargs, and new args=(...), kwargs={...}, to
-    # allow transition.
-    a = kwargs.pop("args", None)
-    if a is not None:
-      if args:
-        raise ValueError(
-            "Can't pass *args and args=... to update")
-      args = a
-    k = kwargs.pop("kwargs", None)
-    if k is not None:
-      if kwargs:
-        raise ValueError(
-            "Can't pass **kwargs and kwargs=... to update")
-      kwargs = k
-    return self._extended.update(var, fn, args, kwargs, group)
-
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def update_non_slot(self, colocate_with, fn, *args, **kwargs):
-    """DEPRECATED: use extended.update_non_slot() instead."""
-    group = kwargs.pop("group", True)
-    # We temporarily support "grouped" in addition to "group" for backward-
-    # compatibility.
-    group = kwargs.pop("grouped", True) and group
-    # Handle old *args, **kwargs, and new args=(...), kwargs={...}, to
-    # allow transition.
-    a = kwargs.pop("args", None)
-    if a is not None:
-      if args:
-        raise ValueError(
-            "Can't pass *args and args=... to update_non_slot")
-      args = a
-    k = kwargs.pop("kwargs", None)
-    if k is not None:
-      if kwargs:
-        raise ValueError(
-            "Can't pass **kwargs and kwargs=... to update_non_slot")
-      kwargs = k
-    return self._extended.update_non_slot(
-        colocate_with, fn, args, kwargs, group)
-
   @doc_controls.do_not_generate_docs  # DEPRECATED, -> `DistributedValues`
   def unwrap(self, value):
     """Returns the list of all per-replica values contained in `value`.
@@ -612,50 +533,16 @@ class DistributionStrategy(object):
     """
     return self._extended._unwrap(value)  # pylint: disable=protected-access
 
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def value_container(self, value):
-    """DEPRECATED: use extended.value_container() instead."""
-    return self._extended.value_container(value)
-
   @doc_controls.do_not_generate_docs  # DEPRECATED, -> `DistributedValues`
   def group(self, value, name=None):
     """Shortcut for `tf.group(self.unwrap(value))`."""
     return self._extended._group(value, name)  # pylint: disable=protected-access
 
-  @property
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def require_static_shapes(self):
-    """DEPRECATED: use extended.require_static_shapes instead."""
-    return self._extended.experimental_require_static_shapes
-
   @property
   def num_replicas_in_sync(self):
     """Returns number of replicas over which gradients are aggregated."""
     return self._extended._num_replicas_in_sync  # pylint: disable=protected-access
 
-  @property
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def worker_devices(self):
-    """DEPRECATED: use extended.worker_devices instead."""
-    return self._extended.worker_devices
-
-  @property
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def parameter_devices(self):
-    """DEPRECATED: use extended.parameter_devices instead."""
-    return self._extended.parameter_devices
-
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def non_slot_devices(self, var_list):
-    """DEPRECATED: use extended.non_slot_devices instead."""
-    return self._extended.non_slot_devices(var_list)
-
-  @property
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def between_graph(self):
-    """DEPRECATED: use extended.experimental_between_graph instead."""
-    return self._extended.experimental_between_graph
-
   @doc_controls.do_not_generate_docs  # DEPRECATED, being replaced by a new API.
   def configure(self,
                 session_config=None,
@@ -691,24 +578,6 @@ class DistributionStrategy(object):
     """
     return self._extended._update_config_proto(config_proto)  # pylint: disable=protected-access
 
-  @property
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def should_init(self):
-    """DEPRECATED: use extended.should_init instead."""
-    return self._extended.experimental_should_init
-
-  @property
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def should_checkpoint(self):
-    """DEPRECATED: use extended.should_checkpoint instead."""
-    return self._extended.should_checkpoint
-
-  @property
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def should_save_summary(self):
-    """DEPRECATED: use extended.should_save_summary instead."""
-    return self._extended.should_save_summary
-
   def __deepcopy__(self, memo):
     # First do a regular deepcopy of `self`.
     cls = self.__class__
@@ -1241,9 +1110,6 @@ class DistributionStrategyExtended(object):
 
     Args:
       reduce_op: Reduction type, an instance of `tf.distribute.ReduceOp` enum.
-        DEPRECATED but still accepted values:
-        `tf.VariableAggregation.SUM`,
-        `tf.VariableAggregation.MEAN`,
       value: A per-replica value with one value per replica.
       destinations: A mirrored variable, a per-replica tensor, or a device
         string. The return value will be copied to all destination devices (or
@@ -1256,14 +1122,7 @@ class DistributionStrategyExtended(object):
     # TODO(josh11b): More docstring
     _require_cross_replica_context_extended(self)
     assert not isinstance(destinations, (list, tuple))
-
-    # TODO(priyag): Remove this when all callers have been updated.
-    if isinstance(reduce_op, variable_scope.VariableAggregation):
-      assert reduce_op in (
-          variable_scope.VariableAggregation.SUM,
-          variable_scope.VariableAggregation.MEAN,
-      )
-      reduce_op = reduce_util.ReduceOp.from_variable_aggregation(reduce_op)
+    assert not isinstance(reduce_op, variable_scope.VariableAggregation)
     assert (reduce_op == reduce_util.ReduceOp.SUM or
             reduce_op == reduce_util.ReduceOp.MEAN)
     return self._reduce_to(reduce_op, value, destinations)
@@ -1276,9 +1135,6 @@ class DistributionStrategyExtended(object):
 
     Args:
       reduce_op: Reduction type, an instance of `tf.distribute.ReduceOp` enum.
-        DEPRECATED but still accepted values:
-        `tf.VariableAggregation.SUM`,
-        `tf.VariableAggregation.MEAN`,
       value_destination_pairs: A sequence of (value, destinations)
         pairs. See `reduce_to()` for a description.
 
@@ -1287,14 +1143,7 @@ class DistributionStrategyExtended(object):
     """
     # TODO(josh11b): More docstring
     _require_cross_replica_context_extended(self)
-
-    # TODO(priyag): Remove this when all callers have been updated.
-    if isinstance(reduce_op, variable_scope.VariableAggregation):
-      assert reduce_op in [
-          variable_scope.VariableAggregation.SUM,
-          variable_scope.VariableAggregation.MEAN,
-      ]
-      reduce_op = reduce_util.ReduceOp.from_variable_aggregation(reduce_op)
+    assert not isinstance(reduce_op, variable_scope.VariableAggregation)
     return self._batch_reduce_to(reduce_op, value_destination_pairs)
 
   def _batch_reduce_to(self, reduce_op, value_destination_pairs):
@@ -1565,12 +1414,6 @@ class ReplicaContext(object):
     require_replica_context(self)
     return self._replica_id_in_sync_group
 
-  @property
-  @doc_controls.do_not_generate_docs  # DEPRECATED, use `strategy`
-  def distribution_strategy(self):
-    """DEPRECATED: use `self.strategy` instead."""
-    return self._strategy
-
   @property
   def strategy(self):
     """The current `tf.distribute.Strategy` object."""
diff --git a/tensorflow/python/distribute/mirrored_strategy.py b/tensorflow/python/distribute/mirrored_strategy.py
index 1ed7eef1ed..f279c3ba1b 100644
--- a/tensorflow/python/distribute/mirrored_strategy.py
+++ b/tensorflow/python/distribute/mirrored_strategy.py
@@ -880,13 +880,13 @@ class _MirroredReplicaThread(threading.Thread):
 
 
 class MirroredReplicaContext(distribute_lib.ReplicaContext):
-  """ReplicaContext used in MirroredStrategy.call_for_each_replica().
+  """ReplicaContext used in MirroredStrategy.extended.call_for_each_replica().
 
   Opened in `_MirroredReplicaThread`, to allow the user to invoke
   `MirroredStrategy`'s specific implementation of `merge_call()`,
   which works by delegating the function and its arguments to
   the main thread (the one that invoked
-  `MirroredStrategy.call_for_each_replica()`).
+  `MirroredStrategy.extended.call_for_each_replica()`).
   """
 
   def _merge_call(self, fn, args, kwargs):
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-mirrored-strategy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-mirrored-strategy.pbtxt
index 863ee9c210..c554df489a 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-mirrored-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-mirrored-strategy.pbtxt
@@ -3,10 +3,6 @@ tf_class {
   is_instance: "<class \'tensorflow.python.distribute.mirrored_strategy.MirroredStrategy\'>"
   is_instance: "<class \'tensorflow.python.distribute.distribute_lib.DistributionStrategy\'>"
   is_instance: "<type \'object\'>"
-  member {
-    name: "between_graph"
-    mtype: "<type \'property\'>"
-  }
   member {
     name: "extended"
     mtype: "<type \'property\'>"
@@ -15,46 +11,14 @@ tf_class {
     name: "num_replicas_in_sync"
     mtype: "<type \'property\'>"
   }
-  member {
-    name: "parameter_devices"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "require_static_shapes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "should_checkpoint"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "should_init"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "should_save_summary"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "worker_devices"
-    mtype: "<type \'property\'>"
-  }
   member_method {
     name: "__init__"
     argspec: "args=[\'self\', \'devices\', \'cross_device_ops\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
-  member_method {
-    name: "batch_reduce"
-    argspec: "args=[\'self\', \'aggregation\', \'value_destination_pairs\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "broadcast"
     argspec: "args=[\'self\', \'tensor\', \'destinations\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "call_for_each_replica"
-    argspec: "args=[\'self\', \'fn\'], varargs=args, keywords=kwargs, defaults=None"
-  }
   member_method {
     name: "colocate_vars_with"
     argspec: "args=[\'self\', \'colocate_with_variable\'], varargs=None, keywords=None, defaults=None"
@@ -87,18 +51,10 @@ tf_class {
     name: "make_input_fn_iterator"
     argspec: "args=[\'self\', \'input_fn\', \'replication_mode\'], varargs=None, keywords=None, defaults=[\'InputReplicationMode.PER_WORKER\'], "
   }
-  member_method {
-    name: "non_slot_devices"
-    argspec: "args=[\'self\', \'var_list\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "reduce"
     argspec: "args=[\'self\', \'reduce_op\', \'value\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "run_steps_on_dataset"
-    argspec: "args=[\'self\', \'fn\', \'iterator\', \'iterations\', \'initial_loop_values\'], varargs=None, keywords=None, defaults=[\'1\', \'None\'], "
-  }
   member_method {
     name: "scope"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
@@ -107,20 +63,8 @@ tf_class {
     name: "unwrap"
     argspec: "args=[\'self\', \'value\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "update"
-    argspec: "args=[\'self\', \'var\', \'fn\'], varargs=args, keywords=kwargs, defaults=None"
-  }
   member_method {
     name: "update_config_proto"
     argspec: "args=[\'self\', \'config_proto\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "update_non_slot"
-    argspec: "args=[\'self\', \'colocate_with\', \'fn\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "value_container"
-    argspec: "args=[\'self\', \'value\'], varargs=None, keywords=None, defaults=None"
-  }
 }
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-replica-context.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-replica-context.pbtxt
index 22f8160c96..c3b7991175 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-replica-context.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-replica-context.pbtxt
@@ -6,10 +6,6 @@ tf_class {
     name: "devices"
     mtype: "<type \'property\'>"
   }
-  member {
-    name: "distribution_strategy"
-    mtype: "<type \'property\'>"
-  }
   member {
     name: "num_replicas_in_sync"
     mtype: "<type \'property\'>"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt
index 1c06f7873e..cce83be3d0 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt
@@ -2,10 +2,6 @@ path: "tensorflow.distribute.Strategy"
 tf_class {
   is_instance: "<class \'tensorflow.python.distribute.distribute_lib.DistributionStrategy\'>"
   is_instance: "<type \'object\'>"
-  member {
-    name: "between_graph"
-    mtype: "<type \'property\'>"
-  }
   member {
     name: "extended"
     mtype: "<type \'property\'>"
@@ -14,46 +10,14 @@ tf_class {
     name: "num_replicas_in_sync"
     mtype: "<type \'property\'>"
   }
-  member {
-    name: "parameter_devices"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "require_static_shapes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "should_checkpoint"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "should_init"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "should_save_summary"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "worker_devices"
-    mtype: "<type \'property\'>"
-  }
   member_method {
     name: "__init__"
     argspec: "args=[\'self\', \'extended\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "batch_reduce"
-    argspec: "args=[\'self\', \'aggregation\', \'value_destination_pairs\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "broadcast"
     argspec: "args=[\'self\', \'tensor\', \'destinations\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "call_for_each_replica"
-    argspec: "args=[\'self\', \'fn\'], varargs=args, keywords=kwargs, defaults=None"
-  }
   member_method {
     name: "colocate_vars_with"
     argspec: "args=[\'self\', \'colocate_with_variable\'], varargs=None, keywords=None, defaults=None"
@@ -86,18 +50,10 @@ tf_class {
     name: "make_input_fn_iterator"
     argspec: "args=[\'self\', \'input_fn\', \'replication_mode\'], varargs=None, keywords=None, defaults=[\'InputReplicationMode.PER_WORKER\'], "
   }
-  member_method {
-    name: "non_slot_devices"
-    argspec: "args=[\'self\', \'var_list\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "reduce"
     argspec: "args=[\'self\', \'reduce_op\', \'value\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "run_steps_on_dataset"
-    argspec: "args=[\'self\', \'fn\', \'iterator\', \'iterations\', \'initial_loop_values\'], varargs=None, keywords=None, defaults=[\'1\', \'None\'], "
-  }
   member_method {
     name: "scope"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
@@ -106,20 +62,8 @@ tf_class {
     name: "unwrap"
     argspec: "args=[\'self\', \'value\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "update"
-    argspec: "args=[\'self\', \'var\', \'fn\'], varargs=args, keywords=kwargs, defaults=None"
-  }
   member_method {
     name: "update_config_proto"
     argspec: "args=[\'self\', \'config_proto\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "update_non_slot"
-    argspec: "args=[\'self\', \'colocate_with\', \'fn\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "value_container"
-    argspec: "args=[\'self\', \'value\'], varargs=None, keywords=None, defaults=None"
-  }
 }
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-mirrored-strategy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-mirrored-strategy.pbtxt
index 863ee9c210..c554df489a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-mirrored-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-mirrored-strategy.pbtxt
@@ -3,10 +3,6 @@ tf_class {
   is_instance: "<class \'tensorflow.python.distribute.mirrored_strategy.MirroredStrategy\'>"
   is_instance: "<class \'tensorflow.python.distribute.distribute_lib.DistributionStrategy\'>"
   is_instance: "<type \'object\'>"
-  member {
-    name: "between_graph"
-    mtype: "<type \'property\'>"
-  }
   member {
     name: "extended"
     mtype: "<type \'property\'>"
@@ -15,46 +11,14 @@ tf_class {
     name: "num_replicas_in_sync"
     mtype: "<type \'property\'>"
   }
-  member {
-    name: "parameter_devices"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "require_static_shapes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "should_checkpoint"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "should_init"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "should_save_summary"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "worker_devices"
-    mtype: "<type \'property\'>"
-  }
   member_method {
     name: "__init__"
     argspec: "args=[\'self\', \'devices\', \'cross_device_ops\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
-  member_method {
-    name: "batch_reduce"
-    argspec: "args=[\'self\', \'aggregation\', \'value_destination_pairs\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "broadcast"
     argspec: "args=[\'self\', \'tensor\', \'destinations\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "call_for_each_replica"
-    argspec: "args=[\'self\', \'fn\'], varargs=args, keywords=kwargs, defaults=None"
-  }
   member_method {
     name: "colocate_vars_with"
     argspec: "args=[\'self\', \'colocate_with_variable\'], varargs=None, keywords=None, defaults=None"
@@ -87,18 +51,10 @@ tf_class {
     name: "make_input_fn_iterator"
     argspec: "args=[\'self\', \'input_fn\', \'replication_mode\'], varargs=None, keywords=None, defaults=[\'InputReplicationMode.PER_WORKER\'], "
   }
-  member_method {
-    name: "non_slot_devices"
-    argspec: "args=[\'self\', \'var_list\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "reduce"
     argspec: "args=[\'self\', \'reduce_op\', \'value\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "run_steps_on_dataset"
-    argspec: "args=[\'self\', \'fn\', \'iterator\', \'iterations\', \'initial_loop_values\'], varargs=None, keywords=None, defaults=[\'1\', \'None\'], "
-  }
   member_method {
     name: "scope"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
@@ -107,20 +63,8 @@ tf_class {
     name: "unwrap"
     argspec: "args=[\'self\', \'value\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "update"
-    argspec: "args=[\'self\', \'var\', \'fn\'], varargs=args, keywords=kwargs, defaults=None"
-  }
   member_method {
     name: "update_config_proto"
     argspec: "args=[\'self\', \'config_proto\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "update_non_slot"
-    argspec: "args=[\'self\', \'colocate_with\', \'fn\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "value_container"
-    argspec: "args=[\'self\', \'value\'], varargs=None, keywords=None, defaults=None"
-  }
 }
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-replica-context.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-replica-context.pbtxt
index 22f8160c96..c3b7991175 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-replica-context.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-replica-context.pbtxt
@@ -6,10 +6,6 @@ tf_class {
     name: "devices"
     mtype: "<type \'property\'>"
   }
-  member {
-    name: "distribution_strategy"
-    mtype: "<type \'property\'>"
-  }
   member {
     name: "num_replicas_in_sync"
     mtype: "<type \'property\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt
index 1c06f7873e..cce83be3d0 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt
@@ -2,10 +2,6 @@ path: "tensorflow.distribute.Strategy"
 tf_class {
   is_instance: "<class \'tensorflow.python.distribute.distribute_lib.DistributionStrategy\'>"
   is_instance: "<type \'object\'>"
-  member {
-    name: "between_graph"
-    mtype: "<type \'property\'>"
-  }
   member {
     name: "extended"
     mtype: "<type \'property\'>"
@@ -14,46 +10,14 @@ tf_class {
     name: "num_replicas_in_sync"
     mtype: "<type \'property\'>"
   }
-  member {
-    name: "parameter_devices"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "require_static_shapes"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "should_checkpoint"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "should_init"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "should_save_summary"
-    mtype: "<type \'property\'>"
-  }
-  member {
-    name: "worker_devices"
-    mtype: "<type \'property\'>"
-  }
   member_method {
     name: "__init__"
     argspec: "args=[\'self\', \'extended\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "batch_reduce"
-    argspec: "args=[\'self\', \'aggregation\', \'value_destination_pairs\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "broadcast"
     argspec: "args=[\'self\', \'tensor\', \'destinations\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "call_for_each_replica"
-    argspec: "args=[\'self\', \'fn\'], varargs=args, keywords=kwargs, defaults=None"
-  }
   member_method {
     name: "colocate_vars_with"
     argspec: "args=[\'self\', \'colocate_with_variable\'], varargs=None, keywords=None, defaults=None"
@@ -86,18 +50,10 @@ tf_class {
     name: "make_input_fn_iterator"
     argspec: "args=[\'self\', \'input_fn\', \'replication_mode\'], varargs=None, keywords=None, defaults=[\'InputReplicationMode.PER_WORKER\'], "
   }
-  member_method {
-    name: "non_slot_devices"
-    argspec: "args=[\'self\', \'var_list\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "reduce"
     argspec: "args=[\'self\', \'reduce_op\', \'value\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "run_steps_on_dataset"
-    argspec: "args=[\'self\', \'fn\', \'iterator\', \'iterations\', \'initial_loop_values\'], varargs=None, keywords=None, defaults=[\'1\', \'None\'], "
-  }
   member_method {
     name: "scope"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
@@ -106,20 +62,8 @@ tf_class {
     name: "unwrap"
     argspec: "args=[\'self\', \'value\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "update"
-    argspec: "args=[\'self\', \'var\', \'fn\'], varargs=args, keywords=kwargs, defaults=None"
-  }
   member_method {
     name: "update_config_proto"
     argspec: "args=[\'self\', \'config_proto\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "update_non_slot"
-    argspec: "args=[\'self\', \'colocate_with\', \'fn\'], varargs=args, keywords=kwargs, defaults=None"
-  }
-  member_method {
-    name: "value_container"
-    argspec: "args=[\'self\', \'value\'], varargs=None, keywords=None, defaults=None"
-  }
 }
-- 
GitLab


From 0bdf7e66fcf04707e3094b1ac4648c06bceb2db5 Mon Sep 17 00:00:00 2001
From: Yunlu Li <yunluli@google.com>
Date: Tue, 15 Jan 2019 10:47:16 -0800
Subject: [PATCH 0752/1095] Update test data multi_add.json.

PiperOrigin-RevId: 229399229
---
 tensorflow/lite/testdata/multi_add.json | 119 ++++++++++++++++++++----
 1 file changed, 102 insertions(+), 17 deletions(-)

diff --git a/tensorflow/lite/testdata/multi_add.json b/tensorflow/lite/testdata/multi_add.json
index 97b931dba8..ae559255a8 100644
--- a/tensorflow/lite/testdata/multi_add.json
+++ b/tensorflow/lite/testdata/multi_add.json
@@ -1,46 +1,131 @@
 {
-  "version": 1,
+  "version": 3,
   "operator_codes": [
     {
-      "builtin_code": "ADD"
     }
   ],
   "subgraphs": [
     {
       "tensors": [
-        { "shape": [ 1, 8, 8, 3 ], "name": "a" },
-        { "shape": [ 1, 8, 8, 3 ], "name": "b" },
-        { "shape": [ 1, 8, 8, 3 ], "name": "c" },
-        { "shape": [ 1, 8, 8, 3 ], "name": "d" },
-        { "shape": [ 1, 8, 8, 3 ], "name": "i" },
-        { "shape": [ 1, 8, 8, 3 ], "name": "x" },
-        { "shape": [ 1, 8, 8, 3 ], "name": "y" }
+        {
+          "shape": [
+            1,
+            8,
+            8,
+            3
+          ],
+          "name": "a"
+        },
+        {
+          "shape": [
+            1,
+            8,
+            8,
+            3
+          ],
+          "name": "b"
+        },
+        {
+          "shape": [
+            1,
+            8,
+            8,
+            3
+          ],
+          "name": "c"
+        },
+        {
+          "shape": [
+            1,
+            8,
+            8,
+            3
+          ],
+          "name": "d"
+        },
+        {
+          "shape": [
+            1,
+            8,
+            8,
+            3
+          ],
+          "name": "i"
+        },
+        {
+          "shape": [
+            1,
+            8,
+            8,
+            3
+          ],
+          "name": "x"
+        },
+        {
+          "shape": [
+            1,
+            8,
+            8,
+            3
+          ],
+          "name": "y"
+        }
+      ],
+      "inputs": [
+        0,
+        1,
+        2,
+        3
+      ],
+      "outputs": [
+        5,
+        6
       ],
-      "inputs": [ 0, 1, 2, 3 ],
-      "outputs": [ 5, 6 ],
       "operators": [
         {
-          "inputs": [ 1, 2 ],
-          "outputs": [ 4 ],
+          "inputs": [
+            1,
+            2
+          ],
+          "outputs": [
+            4
+          ],
           "builtin_options_type": "AddOptions",
           "builtin_options": {
           }
         },
         {
-          "inputs": [ 0, 4 ],
-          "outputs": [ 5 ],
+          "inputs": [
+            0,
+            4
+          ],
+          "outputs": [
+            5
+          ],
           "builtin_options_type": "AddOptions",
           "builtin_options": {
           }
         },
         {
-          "inputs": [ 3, 4 ],
-          "outputs": [ 6 ],
+          "inputs": [
+            3,
+            4
+          ],
+          "outputs": [
+            6
+          ],
           "builtin_options_type": "AddOptions",
           "builtin_options": {
           }
         }
       ]
     }
+  ],
+  "buffers": [
+    {
+      "data": [
+
+      ]
+    }
   ]
 }
-- 
GitLab


From 158af766b665509ff5e47a092f64279848549d5c Mon Sep 17 00:00:00 2001
From: Akshay Modi <nareshmodi@google.com>
Date: Tue, 15 Jan 2019 10:47:23 -0800
Subject: [PATCH 0753/1095] Use the correct nest in autograph

PiperOrigin-RevId: 229399245
---
 tensorflow/python/autograph/impl/api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/autograph/impl/api.py b/tensorflow/python/autograph/impl/api.py
index b1c16b1169..6633441e9f 100644
--- a/tensorflow/python/autograph/impl/api.py
+++ b/tensorflow/python/autograph/impl/api.py
@@ -35,7 +35,7 @@ from tensorflow.python.autograph.operators import py_builtins
 from tensorflow.python.autograph.pyct import compiler
 from tensorflow.python.autograph.pyct import inspect_utils
 from tensorflow.python.autograph.utils import py_func
-from tensorflow.python.data.util import nest
+from tensorflow.python.util import nest
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.util import tf_decorator
-- 
GitLab


From e7af439def1647a7b8da9e4a5aebfc2b80f998ce Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Tue, 15 Jan 2019 10:54:12 -0800
Subject: [PATCH 0754/1095] Use `strategy.make_input_fn_iterator` in estimator
 instead of the deprecated `strategy.distribute_dataset`.

PiperOrigin-RevId: 229400527
---
 .../contrib/distribute/python/tpu_strategy.py      | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index 518c704b89..b9d3924a24 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -303,6 +303,20 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
         functools.partial(self._call_dataset_fn, dataset_fn),
         self._input_workers)
 
+  def _make_input_fn_iterator(
+      self,
+      input_fn,
+      replication_mode=distribute_lib.InputReplicationMode.PER_WORKER):
+    input_contexts = []
+    num_workers = self._input_workers.num_workers
+    for i in range(num_workers):
+      input_contexts.append(distribute_lib.InputContext(
+          num_input_pipelines=num_workers,
+          input_pipeline_id=i,
+          num_replicas_in_sync=self._num_replicas_in_sync))
+    return input_lib.InputFunctionIterator(
+        input_fn, self._input_workers, input_contexts)
+
   def _experimental_make_numpy_dataset(self, numpy_input, session):
     return numpy_dataset.one_host_numpy_dataset(
         numpy_input, numpy_dataset.SingleDevice(self.get_host_cpu_device(0)),
-- 
GitLab


From 556ac5a846e5b03859eda02e17349704387c465a Mon Sep 17 00:00:00 2001
From: Akshay Modi <nareshmodi@google.com>
Date: Tue, 15 Jan 2019 10:55:40 -0800
Subject: [PATCH 0755/1095] Add helper to get mode strategy and replica context
 together.

PiperOrigin-RevId: 229400832
---
 .../distribution_strategy_context.py          |  5 +++
 tensorflow/python/eager/function.py           |  4 +--
 tensorflow/python/eager/tape.py               | 33 ++++++++++++++++---
 3 files changed, 35 insertions(+), 7 deletions(-)

diff --git a/tensorflow/python/distribute/distribution_strategy_context.py b/tensorflow/python/distribute/distribution_strategy_context.py
index e6648bf7c4..6c1e250f96 100644
--- a/tensorflow/python/distribute/distribution_strategy_context.py
+++ b/tensorflow/python/distribute/distribution_strategy_context.py
@@ -201,6 +201,11 @@ def has_strategy():
   return get_strategy() is not _get_default_strategy()
 
 
+def get_strategy_and_replica_context():
+  per_thread_mode = _get_per_thread_mode()
+  return (per_thread_mode.strategy, per_thread_mode.replica_context)
+
+
 # ------------------------------------------------------------------------------
 # Defaults that are used when no tf.distribute.Strategy is explicitly created.
 # We create them lazily in a function so that we can workaround the circular
diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index b0140e5c98..0ebc7c915d 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -471,9 +471,7 @@ class Function(object):
     """
     ctx = context.context()
 
-    for v in self._func_graph.variables:
-      if v.trainable:
-        tape.variable_accessed(v)
+    tape.variables_accessed(self._func_graph.variables)
 
     tensor_inputs = []
     for i, arg in enumerate(args):
diff --git a/tensorflow/python/eager/tape.py b/tensorflow/python/eager/tape.py
index 56b68b9eea..56d696bf92 100644
--- a/tensorflow/python/eager/tape.py
+++ b/tensorflow/python/eager/tape.py
@@ -61,8 +61,9 @@ def watch(tape, tensor):
 
 def watch_variable(tape, variable):
   """Marks this variable to be watched by the given tape."""
-  strategy = distribution_strategy_context.get_strategy()
-  if distribution_strategy_context.get_replica_context():
+  strategy, context = (
+      distribution_strategy_context.get_strategy_and_replica_context())
+  if context:
     variables = [strategy.extended.value_container(variable)]
   else:
     variables = strategy.unwrap(variable)
@@ -76,8 +77,9 @@ def variable_accessed(variable):
   Args:
     variable: variable to be watched.
   """
-  strategy = distribution_strategy_context.get_strategy()
-  if distribution_strategy_context.get_replica_context():
+  strategy, context = (
+      distribution_strategy_context.get_strategy_and_replica_context())
+  if context:
     variables = [strategy.extended.value_container(variable)]
   else:
     variables = strategy.unwrap(variable)
@@ -85,6 +87,29 @@ def variable_accessed(variable):
     pywrap_tensorflow.TFE_Py_TapeVariableAccessed(var)
 
 
+def variables_accessed(variables):
+  """Notifies all tapes in the stack that variables have been accessed.
+
+  Only trainable variables are marked as accessed.
+
+  Args:
+    variables: iterable of variables to mark as accessed.
+  """
+  strategy, context = (
+      distribution_strategy_context.get_strategy_and_replica_context())
+  accessed = []
+  if context:
+    accessed = [strategy.extended.value_container(variable)
+                for variable in variables if variable.trainable]
+  else:
+    for variable in variables:
+      if variable.trainable:
+        accessed.extend(strategy.unwrap(variable))
+
+  for var in accessed:
+    pywrap_tensorflow.TFE_Py_TapeVariableAccessed(var)
+
+
 def pop_tape(tape):
   """Pops the top tape in the stack, if any."""
   pywrap_tensorflow.TFE_Py_TapeSetRemove(tape._tape)  # pylint: disable=protected-access
-- 
GitLab


From b0ce2166c0807e538ef346dad1d647d1983e0d4c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 15 Jan 2019 11:07:45 -0800
Subject: [PATCH 0756/1095] Remove unnecessary dependencies from
 core/kernels:resource_variable_ops to :mutex_ops and :state.

PiperOrigin-RevId: 229403710
---
 tensorflow/core/BUILD         | 1 +
 tensorflow/core/kernels/BUILD | 2 --
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index caf7b4cdbc..1fc917a5f2 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -1406,6 +1406,7 @@ cc_library(
         "//tensorflow/core/kernels:manip",
         "//tensorflow/core/kernels:math",
         "//tensorflow/core/kernels:multinomial_op",
+        "//tensorflow/core/kernels:mutex_ops",
         "//tensorflow/core/kernels:nn",
         "//tensorflow/core/kernels:parameterized_truncated_normal_op",
         "//tensorflow/core/kernels:parsing",
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 1be749289b..ea6343f24d 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -2282,9 +2282,7 @@ tf_kernel_library(
         ":bounds_check",
         ":dense_update_functor",
         ":gather_functor",
-        ":mutex_ops",
         ":scatter_functor",
-        ":state",
         ":training_op_helpers",
         ":variable_ops",
         "//tensorflow/core:core_cpu_lib",
-- 
GitLab


From 3e0268dc76f8cc36e72a5cff980d8d0f27c609b9 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Tue, 15 Jan 2019 11:08:09 -0800
Subject: [PATCH 0757/1095] Adds v2 Mean IOU metric.

PiperOrigin-RevId: 229403786
---
 tensorflow/python/keras/metrics.py            | 129 +++++++++++++++++-
 tensorflow/python/keras/metrics_test.py       |  96 +++++++++++++
 .../tensorflow.keras.metrics.-accuracy.pbtxt  |   2 +-
 ...rflow.keras.metrics.-binary-accuracy.pbtxt |   2 +-
 ....keras.metrics.-categorical-accuracy.pbtxt |   2 +-
 ...low.keras.metrics.-categorical-hinge.pbtxt |   2 +-
 ...flow.keras.metrics.-cosine-proximity.pbtxt |   2 +-
 ...rflow.keras.metrics.-false-negatives.pbtxt |   2 +-
 ...rflow.keras.metrics.-false-positives.pbtxt |   2 +-
 .../v1/tensorflow.keras.metrics.-hinge.pbtxt  |   2 +-
 ...w.keras.metrics.-mean-absolute-error.pbtxt |   2 +-
 ...rics.-mean-absolute-percentage-error.pbtxt |   2 +-
 ...ow.keras.metrics.-mean-squared-error.pbtxt |   2 +-
 ...rics.-mean-squared-logarithmic-error.pbtxt |   2 +-
 .../v1/tensorflow.keras.metrics.-mean.pbtxt   |   2 +-
 .../tensorflow.keras.metrics.-precision.pbtxt |   2 +-
 .../v1/tensorflow.keras.metrics.-recall.pbtxt |   2 +-
 ....metrics.-sensitivity-at-specificity.pbtxt |   2 +-
 ...metrics.-sparse-categorical-accuracy.pbtxt |   2 +-
 ....metrics.-specificity-at-sensitivity.pbtxt |   2 +-
 ...sorflow.keras.metrics.-squared-hinge.pbtxt |   2 +-
 ...orflow.keras.metrics.-true-negatives.pbtxt |   2 +-
 ...orflow.keras.metrics.-true-positives.pbtxt |   2 +-
 .../tensorflow.keras.metrics.-accuracy.pbtxt  |   2 +-
 ...rflow.keras.metrics.-binary-accuracy.pbtxt |   2 +-
 ....keras.metrics.-categorical-accuracy.pbtxt |   2 +-
 ...low.keras.metrics.-categorical-hinge.pbtxt |   2 +-
 ...flow.keras.metrics.-cosine-proximity.pbtxt |   2 +-
 ...rflow.keras.metrics.-false-negatives.pbtxt |   2 +-
 ...rflow.keras.metrics.-false-positives.pbtxt |   2 +-
 .../v2/tensorflow.keras.metrics.-hinge.pbtxt  |   2 +-
 ...w.keras.metrics.-mean-absolute-error.pbtxt |   2 +-
 ...rics.-mean-absolute-percentage-error.pbtxt |   2 +-
 ...ow.keras.metrics.-mean-squared-error.pbtxt |   2 +-
 ...rics.-mean-squared-logarithmic-error.pbtxt |   2 +-
 .../v2/tensorflow.keras.metrics.-mean.pbtxt   |   2 +-
 .../tensorflow.keras.metrics.-precision.pbtxt |   2 +-
 .../v2/tensorflow.keras.metrics.-recall.pbtxt |   2 +-
 ....metrics.-sensitivity-at-specificity.pbtxt |   2 +-
 ...metrics.-sparse-categorical-accuracy.pbtxt |   2 +-
 ....metrics.-specificity-at-sensitivity.pbtxt |   2 +-
 ...sorflow.keras.metrics.-squared-hinge.pbtxt |   2 +-
 ...orflow.keras.metrics.-true-negatives.pbtxt |   2 +-
 ...orflow.keras.metrics.-true-positives.pbtxt |   2 +-
 44 files changed, 265 insertions(+), 44 deletions(-)

diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index ec09ad9ba8..245dffe99e 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -241,12 +241,13 @@ class Metric(Layer):
                  shape=(),
                  aggregation=tf_variables.VariableAggregation.SUM,
                  synchronization=tf_variables.VariableSynchronization.ON_READ,
-                 initializer=None):
+                 initializer=None,
+                 dtype=None):
     """Adds state variable. Only for use by subclasses."""
     return super(Metric, self).add_weight(
         name=name,
         shape=shape,
-        dtype=self._dtype,
+        dtype=self._dtype if dtype is None else dtype,
         trainable=False,
         initializer=initializer,
         collections=[],
@@ -2175,6 +2176,130 @@ class KullbackLeiblerDivergence(MeanMetricWrapper):
     return super(KullbackLeiblerDivergence, cls).from_config(config)
 
 
+class MeanIoU(Metric):
+  """Computes the mean Intersection-Over-Union metric.
+
+  Mean Intersection-Over-Union is a common evaluation metric for semantic image
+  segmentation, which first computes the IOU for each semantic class and then
+  computes the average over classes. IOU is defined as follows:
+    IOU = true_positive / (true_positive + false_positive + false_negative).
+  The predictions are accumulated in a confusion matrix, weighted by
+  `sample_weight` and the metric is then calculated from it.
+
+  If `sample_weight` is `None`, weights default to 1.
+  Use `sample_weight` of 0 to mask values.
+
+  Usage:
+
+  ```python
+  m = tf.keras.metrics.MeanIoU(num_classes=2)
+  m.update_state([0, 0, 1, 1], [0, 1, 0, 1])
+
+    # cm = [[1, 1],
+            [1, 1]]
+    # sum_row = [2, 2], sum_col = [2, 2], true_positives = [1, 1]
+    # iou = true_positives / (sum_row + sum_col - true_positives))
+    # result = (1 / (2 + 2 - 1) + 1 / (2 + 2 - 1)) / 2 = 0.33
+  print('Final result: ', m.result().numpy())  # Final result: 0.33
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile(
+    'sgd',
+    loss='mse',
+    metrics=[tf.keras.metrics.MeanIoU(num_classes=2)])
+  ```
+  """
+
+  def __init__(self, num_classes, name=None, dtype=None):
+    """Creates a `MeanIoU` instance.
+
+    Args:
+      num_classes: The possible number of labels the prediction task can have.
+        This value must be provided, since a confusion matrix of dimension =
+        [num_classes, num_classes] will be allocated.
+      name: (Optional) string name of the metric instance.
+      dtype: (Optional) data type of the metric result.
+    """
+    super(MeanIoU, self).__init__(name=name, dtype=dtype)
+    self.num_classes = num_classes
+
+    # Variable to accumulate the predictions in the confusion matrix. Setting
+    # the type to be `float64` as required by confusion_matrix_ops.
+    self.total_cm = self.add_weight(
+        'total_confusion_matrix',
+        shape=(num_classes, num_classes),
+        initializer=init_ops.zeros_initializer,
+        dtype=dtypes.float64)
+
+  def update_state(self, y_true, y_pred, sample_weight=None):
+    """Accumulates the confusion matrix statistics.
+
+    Args:
+      y_true: The ground truth values.
+      y_pred: The predicted values.
+      sample_weight: Optional weighting of each example. Defaults to 1. Can be a
+        `Tensor` whose rank is either 0, or the same rank as `y_true`, and must
+        be broadcastable to `y_true`.
+
+    Returns:
+      Update op.
+    """
+    # Flatten the input if its rank > 1.
+    if y_pred.shape.ndims > 1:
+      y_pred = array_ops.reshape(y_pred, [-1])
+
+    if y_true.shape.ndims > 1:
+      y_true = array_ops.reshape(y_true, [-1])
+
+    if sample_weight is not None and sample_weight.shape.ndims > 1:
+      sample_weight = array_ops.reshape(sample_weight, [-1])
+
+    # Accumulate the prediction to current confusion matrix.
+    current_cm = confusion_matrix.confusion_matrix(
+        y_true,
+        y_pred,
+        self.num_classes,
+        weights=sample_weight,
+        dtype=dtypes.float64)
+    return state_ops.assign_add(self.total_cm, current_cm)
+
+  def result(self):
+    """Compute the mean intersection-over-union via the confusion matrix."""
+    sum_over_row = math_ops.cast(
+        math_ops.reduce_sum(self.total_cm, axis=0), dtype=self._dtype)
+    sum_over_col = math_ops.cast(
+        math_ops.reduce_sum(self.total_cm, axis=1), dtype=self._dtype)
+    true_positives = math_ops.cast(
+        array_ops.diag_part(self.total_cm), dtype=self._dtype)
+
+    # sum_over_row + sum_over_col =
+    #     2 * true_positives + false_positives + false_negatives.
+    denominator = sum_over_row + sum_over_col - true_positives
+
+    # The mean is only computed over classes that appear in the
+    # label or prediction tensor. If the denominator is 0, we need to
+    # ignore the class.
+    num_valid_entries = math_ops.reduce_sum(
+        math_ops.cast(math_ops.not_equal(denominator, 0), dtype=self._dtype))
+
+    iou = math_ops.div_no_nan(true_positives, denominator)
+
+    return math_ops.div_no_nan(
+        math_ops.reduce_sum(iou, name='mean_iou'), num_valid_entries)
+
+  def reset_states(self):
+    K.set_value(self.total_cm, np.zeros((self.num_classes, self.num_classes)))
+
+  def get_config(self):
+    config = {'num_classes': self.num_classes}
+    base_config = super(MeanIoU, self).get_config()
+    return dict(list(base_config.items()) + list(config.items()))
+
+
 def accuracy(y_true, y_pred):
   y_pred.get_shape().assert_is_compatible_with(y_true.get_shape())
   if y_true.dtype != y_pred.dtype:
diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py
index d28a615c02..bf2f4d88fa 100644
--- a/tensorflow/python/keras/metrics_test.py
+++ b/tensorflow/python/keras/metrics_test.py
@@ -967,6 +967,89 @@ class MeanRelativeErrorTest(test.TestCase):
     self.assertEqual(self.evaluate(result), 0)
 
 
+@test_util.run_all_in_graph_and_eager_modes
+class MeanIoUTest(test.TestCase):
+
+  def test_config(self):
+    m_obj = metrics.MeanIoU(num_classes=2, name='mean_iou')
+    self.assertEqual(m_obj.name, 'mean_iou')
+    self.assertEqual(m_obj.num_classes, 2)
+
+    m_obj2 = metrics.MeanIoU.from_config(m_obj.get_config())
+    self.assertEqual(m_obj2.name, 'mean_iou')
+    self.assertEqual(m_obj2.num_classes, 2)
+
+  def test_unweighted(self):
+    y_pred = constant_op.constant([0, 1, 0, 1], dtype=dtypes.float32)
+    y_true = constant_op.constant([0, 0, 1, 1])
+
+    m_obj = metrics.MeanIoU(num_classes=2)
+    self.evaluate(variables.variables_initializer(m_obj.variables))
+
+    result = m_obj(y_true, y_pred)
+
+    # cm = [[1, 1],
+    #       [1, 1]]
+    # sum_row = [2, 2], sum_col = [2, 2], true_positives = [1, 1]
+    # iou = true_positives / (sum_row + sum_col - true_positives))
+    expected_result = (1 / (2 + 2 - 1) + 1 / (2 + 2 - 1)) / 2
+    self.assertAllClose(self.evaluate(result), expected_result, atol=1e-3)
+
+  def test_weighted(self):
+    y_pred = constant_op.constant([0, 1, 0, 1], dtype=dtypes.float32)
+    y_true = constant_op.constant([0, 0, 1, 1])
+    sample_weight = constant_op.constant([0.2, 0.3, 0.4, 0.1])
+
+    m_obj = metrics.MeanIoU(num_classes=2)
+    self.evaluate(variables.variables_initializer(m_obj.variables))
+
+    result = m_obj(y_true, y_pred, sample_weight=sample_weight)
+
+    # cm = [[0.2, 0.3],
+    #       [0.4, 0.1]]
+    # sum_row = [0.6, 0.4], sum_col = [0.5, 0.5], true_positives = [0.2, 0.1]
+    # iou = true_positives / (sum_row + sum_col - true_positives))
+    expected_result = (0.2 / (0.6 + 0.5 - 0.2) + 0.1 / (0.4 + 0.5 - 0.1)) / 2
+    self.assertAllClose(self.evaluate(result), expected_result, atol=1e-3)
+
+  def test_multi_dim_input(self):
+    y_pred = constant_op.constant([[0, 1], [0, 1]], dtype=dtypes.float32)
+    y_true = constant_op.constant([[0, 0], [1, 1]])
+    sample_weight = constant_op.constant([[0.2, 0.3], [0.4, 0.1]])
+
+    m_obj = metrics.MeanIoU(num_classes=2)
+    self.evaluate(variables.variables_initializer(m_obj.variables))
+
+    result = m_obj(y_true, y_pred, sample_weight=sample_weight)
+
+    # cm = [[0.2, 0.3],
+    #       [0.4, 0.1]]
+    # sum_row = [0.6, 0.4], sum_col = [0.5, 0.5], true_positives = [0.2, 0.1]
+    # iou = true_positives / (sum_row + sum_col - true_positives))
+    expected_result = (0.2 / (0.6 + 0.5 - 0.2) + 0.1 / (0.4 + 0.5 - 0.1)) / 2
+    self.assertAllClose(self.evaluate(result), expected_result, atol=1e-3)
+
+  def test_zero_valid_entries(self):
+    m_obj = metrics.MeanIoU(num_classes=2)
+    self.evaluate(variables.variables_initializer(m_obj.variables))
+    self.assertAllClose(self.evaluate(m_obj.result()), 0, atol=1e-3)
+
+  def test_zero_and_non_zero_entries(self):
+    y_pred = constant_op.constant([1], dtype=dtypes.float32)
+    y_true = constant_op.constant([1])
+
+    m_obj = metrics.MeanIoU(num_classes=2)
+    self.evaluate(variables.variables_initializer(m_obj.variables))
+    result = m_obj(y_true, y_pred)
+
+    # cm = [[0, 0],
+    #       [0, 1]]
+    # sum_row = [0, 1], sum_col = [0, 1], true_positives = [0, 1]
+    # iou = true_positives / (sum_row + sum_col - true_positives))
+    expected_result = (0 + 1 / (1 + 1 - 1)) / 1
+    self.assertAllClose(self.evaluate(result), expected_result, atol=1e-3)
+
+
 def _get_model(compile_metrics):
   model_layers = [
       layers.Dense(3, activation='relu', kernel_initializer='ones'),
@@ -1094,6 +1177,19 @@ class ResetStatesTest(keras_parameterized.TestCase):
       self.assertEqual(self.evaluate(auc_obj.false_negatives[1]), 25.)
       self.assertEqual(self.evaluate(auc_obj.true_negatives[1]), 25.)
 
+  def test_reset_states_mean_iou(self):
+    m_obj = metrics.MeanIoU(num_classes=2)
+    model = _get_model([m_obj])
+    x = np.asarray([[0, 0, 0, 0], [1, 1, 1, 1], [1, 0, 1, 0], [0, 1, 0, 1]],
+                   dtype=np.float32)
+    y = np.asarray([[0], [1], [1], [1]], dtype=np.float32)
+    model.evaluate(x, y)
+    self.assertArrayNear(self.evaluate(m_obj.total_cm)[0], [1, 0], 1e-1)
+    self.assertArrayNear(self.evaluate(m_obj.total_cm)[1], [3, 0], 1e-1)
+    model.evaluate(x, y)
+    self.assertArrayNear(self.evaluate(m_obj.total_cm)[0], [1, 0], 1e-1)
+    self.assertArrayNear(self.evaluate(m_obj.total_cm)[1], [3, 0], 1e-1)
+
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-accuracy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-accuracy.pbtxt
index bad488f59b..e3e8dc00fe 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-accuracy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-accuracy.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-binary-accuracy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-binary-accuracy.pbtxt
index a1e7601a51..60b4f992c0 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-binary-accuracy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-binary-accuracy.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-accuracy.pbtxt
index 5f2c2f9807..deab5a7a70 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-accuracy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-accuracy.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-hinge.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-hinge.pbtxt
index d173fc879f..10477e6662 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-hinge.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-hinge.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-cosine-proximity.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-cosine-proximity.pbtxt
index c50638740f..cdb3e1a79a 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-cosine-proximity.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-cosine-proximity.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-negatives.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-negatives.pbtxt
index c153e9cf4d..f4d067243d 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-negatives.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-negatives.pbtxt
@@ -108,7 +108,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-positives.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-positives.pbtxt
index aae2bd9988..f898e00eea 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-positives.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-positives.pbtxt
@@ -108,7 +108,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-hinge.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-hinge.pbtxt
index 7f13c75583..393ba964c4 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-hinge.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-hinge.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-absolute-error.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-absolute-error.pbtxt
index 896320004c..a86b26e618 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-absolute-error.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-absolute-error.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-absolute-percentage-error.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-absolute-percentage-error.pbtxt
index 37ab1a409b..5e8f321a06 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-absolute-percentage-error.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-absolute-percentage-error.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-squared-error.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-squared-error.pbtxt
index 9025abbb0a..a2eb10c5f6 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-squared-error.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-squared-error.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-squared-logarithmic-error.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-squared-logarithmic-error.pbtxt
index 808853994d..7df45a8caf 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-squared-logarithmic-error.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-squared-logarithmic-error.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean.pbtxt
index 904a2fa9ca..899502cfad 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-precision.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-precision.pbtxt
index 794da447d7..3bdf3a4167 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-precision.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-precision.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-recall.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-recall.pbtxt
index cae610b9ea..1ac2aacca0 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-recall.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-recall.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sensitivity-at-specificity.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sensitivity-at-specificity.pbtxt
index b70ef32bca..5c4f26929f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sensitivity-at-specificity.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sensitivity-at-specificity.pbtxt
@@ -108,7 +108,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt
index 2e693269bf..48ac5c2fdc 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-specificity-at-sensitivity.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-specificity-at-sensitivity.pbtxt
index e62a2df056..702f711722 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-specificity-at-sensitivity.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-specificity-at-sensitivity.pbtxt
@@ -108,7 +108,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-squared-hinge.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-squared-hinge.pbtxt
index 38f94048cc..6c3a9748c7 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-squared-hinge.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-squared-hinge.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-negatives.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-negatives.pbtxt
index 1a524d73c0..3f1a99b83b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-negatives.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-negatives.pbtxt
@@ -108,7 +108,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-positives.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-positives.pbtxt
index b9b4f565c5..e5a4207a32 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-positives.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-positives.pbtxt
@@ -108,7 +108,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-accuracy.pbtxt
index bad488f59b..e3e8dc00fe 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-accuracy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-accuracy.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-binary-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-binary-accuracy.pbtxt
index a1e7601a51..60b4f992c0 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-binary-accuracy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-binary-accuracy.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-accuracy.pbtxt
index 5f2c2f9807..deab5a7a70 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-accuracy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-accuracy.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-hinge.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-hinge.pbtxt
index d173fc879f..10477e6662 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-hinge.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-hinge.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-cosine-proximity.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-cosine-proximity.pbtxt
index c50638740f..cdb3e1a79a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-cosine-proximity.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-cosine-proximity.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-negatives.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-negatives.pbtxt
index c153e9cf4d..f4d067243d 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-negatives.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-negatives.pbtxt
@@ -108,7 +108,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-positives.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-positives.pbtxt
index aae2bd9988..f898e00eea 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-positives.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-positives.pbtxt
@@ -108,7 +108,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-hinge.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-hinge.pbtxt
index 7f13c75583..393ba964c4 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-hinge.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-hinge.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-absolute-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-absolute-error.pbtxt
index 896320004c..a86b26e618 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-absolute-error.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-absolute-error.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-absolute-percentage-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-absolute-percentage-error.pbtxt
index 37ab1a409b..5e8f321a06 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-absolute-percentage-error.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-absolute-percentage-error.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-squared-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-squared-error.pbtxt
index 9025abbb0a..a2eb10c5f6 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-squared-error.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-squared-error.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-squared-logarithmic-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-squared-logarithmic-error.pbtxt
index 808853994d..7df45a8caf 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-squared-logarithmic-error.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-squared-logarithmic-error.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean.pbtxt
index 904a2fa9ca..899502cfad 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-precision.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-precision.pbtxt
index 794da447d7..3bdf3a4167 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-precision.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-precision.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-recall.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-recall.pbtxt
index cae610b9ea..1ac2aacca0 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-recall.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-recall.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sensitivity-at-specificity.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sensitivity-at-specificity.pbtxt
index b70ef32bca..5c4f26929f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sensitivity-at-specificity.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sensitivity-at-specificity.pbtxt
@@ -108,7 +108,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt
index 2e693269bf..48ac5c2fdc 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-specificity-at-sensitivity.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-specificity-at-sensitivity.pbtxt
index e62a2df056..702f711722 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-specificity-at-sensitivity.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-specificity-at-sensitivity.pbtxt
@@ -108,7 +108,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-squared-hinge.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-squared-hinge.pbtxt
index 38f94048cc..6c3a9748c7 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-squared-hinge.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-squared-hinge.pbtxt
@@ -109,7 +109,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-negatives.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-negatives.pbtxt
index 1a524d73c0..3f1a99b83b 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-negatives.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-negatives.pbtxt
@@ -108,7 +108,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-positives.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-positives.pbtxt
index b9b4f565c5..e5a4207a32 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-positives.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-positives.pbtxt
@@ -108,7 +108,7 @@ tf_class {
   }
   member_method {
     name: "add_weight"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'aggregation\', \'synchronization\', \'initializer\', \'dtype\'], varargs=None, keywords=None, defaults=[\'()\', \'VariableAggregation.SUM\', \'VariableSynchronization.ON_READ\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
-- 
GitLab


From a082de29448daa8d91f5cc24a3b37a296f5aec4b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 15 Jan 2019 11:09:26 -0800
Subject: [PATCH 0758/1095] Improve batch_group_count documentation in XLA
 operation semantics

PiperOrigin-RevId: 229404046
---
 .../compiler/xla/g3doc/operation_semantics.md      | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/tensorflow/compiler/xla/g3doc/operation_semantics.md b/tensorflow/compiler/xla/g3doc/operation_semantics.md
index 59ba9bb658..c5f9377f98 100644
--- a/tensorflow/compiler/xla/g3doc/operation_semantics.md
+++ b/tensorflow/compiler/xla/g3doc/operation_semantics.md
@@ -636,11 +636,15 @@ details, see `tf.nn.depthwise_conv2d`.
 
 The `batch_group_count` (default value 1) argument can be used for depthwise
 filters during backpropagation. `batch_group_count` needs to be a divisor of the
-size of the `lhs` batch dimension. If `batch_group_count` is greater than 1, it
-means that conceptually the output batch dimension is split evenely in
-`batch_group_count` groups, such that each group consists of a consecutive
-subsequence of batches. Each output batch element is the reduced value of the
-batch group size.
+size of the `lhs` (input) batch dimension. If `batch_group_count` is greater
+than 1, it means that the output batch dimension should be of size
+`batch_group_size` where `batch_group_size = input batch / batch_group_count`.
+For convolutions with `batch_group_count` greater than 1, the input batch size
+must evenly divide into batch_group_size and output feature size, which implies
+that the output feature size must be equal to batch_group_count. Conceptually,
+this can be achieved by performing the usual convolution, and then scraping
+`batch_group_size` number of elements on the diagonal of the matrix formed by
+output batch and output feature.
 
 The output shape has these dimensions, in this order:
 
-- 
GitLab


From 8154ec0b4a893657e48b838c06e1f988aeac4115 Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Tue, 15 Jan 2019 11:48:17 -0800
Subject: [PATCH 0759/1095] [Grappler] Lower V2 control flow in inlined
 function body.

PiperOrigin-RevId: 229411946
---
 .../process_function_library_runtime.cc       |  10 +-
 tensorflow/core/grappler/optimizers/BUILD     |   1 +
 .../grappler/optimizers/function_optimizer.cc |  74 ++++++-----
 .../optimizers/function_optimizer_test.cc     | 122 ++++++++++++++++++
 4 files changed, 169 insertions(+), 38 deletions(-)

diff --git a/tensorflow/core/common_runtime/process_function_library_runtime.cc b/tensorflow/core/common_runtime/process_function_library_runtime.cc
index b236343a0f..76c75ad3d2 100644
--- a/tensorflow/core/common_runtime/process_function_library_runtime.cc
+++ b/tensorflow/core/common_runtime/process_function_library_runtime.cc
@@ -568,15 +568,12 @@ Status ProcessFunctionLibraryRuntime::InstantiateMultiDevice(
   DumpGraph("Before running POST_PLACEMENT passes", graph.get());
   TF_RETURN_IF_ERROR(OptimizationPassRegistry::Global()->RunGrouping(
       OptimizationPassRegistry::POST_PLACEMENT, optimization_options));
-  DumpGraph("Before running POST_REWRITE_FOR_EXEC passes", graph.get());
-  TF_RETURN_IF_ERROR(OptimizationPassRegistry::Global()->RunGrouping(
-      OptimizationPassRegistry::POST_REWRITE_FOR_EXEC, optimization_options));
-  DumpGraph("After all optimization passes", graph.get());
 
   Device* cpu_device;
   TF_RETURN_IF_ERROR(device_mgr_->LookupDevice("CPU:0", &cpu_device));
 
   if (options.optimize_graph_fn) {
+    DumpGraph("Before running graph optimization fn", graph.get());
     Status status = options.optimize_graph_fn(std::move(ret_node_names),
                                               &data->overlay_lib_, device_set,
                                               cpu_device, &graph);
@@ -587,6 +584,11 @@ Status ProcessFunctionLibraryRuntime::InstantiateMultiDevice(
     DumpGraph("After optimization", graph.get());
   }
 
+  DumpGraph("Before running POST_REWRITE_FOR_EXEC passes", graph.get());
+  TF_RETURN_IF_ERROR(OptimizationPassRegistry::Global()->RunGrouping(
+      OptimizationPassRegistry::POST_REWRITE_FOR_EXEC, optimization_options));
+  DumpGraph("After all optimization passes", graph.get());
+
   std::unordered_map<string, std::unique_ptr<Graph>> subgraphs;
   TF_RETURN_IF_ERROR(
       PartitionFunctionGraph(device_set, std::move(graph), &subgraphs));
diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index ee216f80f9..70e0f226fc 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -181,6 +181,7 @@ tf_cuda_cc_test(
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/utils:grappler_test",
+        "@com_google_absl//absl/algorithm:container",
     ],
 )
 
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.cc b/tensorflow/core/grappler/optimizers/function_optimizer.cc
index 8095d6f31d..1394aa5047 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer.cc
@@ -27,6 +27,8 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/device_mgr.h"
 #include "tensorflow/core/common_runtime/device_set.h"
 #include "tensorflow/core/common_runtime/function.h"
+#include "tensorflow/core/common_runtime/lower_if_while.h"
+#include "tensorflow/core/common_runtime/optimization_registry.h"
 #include "tensorflow/core/common_runtime/placer.h"
 #include "tensorflow/core/common_runtime/process_function_library_runtime.h"
 #include "tensorflow/core/framework/attr_value_util.h"
@@ -1365,20 +1367,6 @@ Status IsInlinableIndirectFunctionCall(const FunctionOptimizerContext& ctx,
         SummarizeNodeDef(func_node));
   }
 
-  // TODO(b/120991525, b/120986912): We need to lower `If` and `While` nodes to
-  // `Switch` nodes after function inlining (one more PRE_PLACEMENT pass?), but
-  // because of the reason described above we are not sure that it's safe, for
-  // now just disable inlining functions with functional control flow.
-  const auto is_functional_ctrl_flow_op = [](const NodeDef& node) {
-    return IsIf(node) || IsWhile(node);
-  };
-  if (absl::c_any_of(func.node_def(), is_functional_ctrl_flow_op)) {
-    return errors::FailedPrecondition(
-        "Can't inline function with `If` or `While` nodes in the function "
-        "body: ",
-        SummarizeNodeDef(func_node));
-  }
-
   return Status::OK();
 }
 
@@ -1488,22 +1476,46 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
 
   const string prefix = strings::StrCat(func_node.name(), "/");
 
+  // ------------------------------------------------------------------------ //
+  // Grappler receives the graph after PRE_PLACEMENT, Placer, and POST_PLACEMENT
+  // passes, so each node has a valid device assignment. Also V2 control
+  // flow ops (functional If and While) lowered to V1 control flow (Switch and
+  // Merge nodes). To keep graph valid for execution we must assign device to
+  // every inlined graph node, and also lower the control flow.
+
+  // Control flow lowering and Placer works with a Graph object.
+  std::unique_ptr<Graph> func_body_graph =
+      absl::make_unique<Graph>(ctx->function_library());
+
+  GraphConstructorOptions opts;
+  TF_RETURN_IF_ERROR(
+      ConvertGraphDefToGraph(opts, item.graph, func_body_graph.get()));
+
+  GraphOptimizationPassOptions opt_options;
+  opt_options.graph = &func_body_graph;
+  opt_options.flib_def = ctx->mutable_function_library();
+
+  // TODO(ezhulenev): Should we run full PRE_PLACEMENT pass here? And
+  // POST_PLACEMENT after placer?
+  LowerIfWhilePass pass;
+  TF_RETURN_IF_ERROR(pass.Run(opt_options));
+
   // ------------------------------------------------------------------------ //
   // Before placing the function body nodes we pin input placeholders to the
   // same device as their corresponding input nodes.
 
-  for (NodeDef& func_body_node : *item.graph.mutable_node()) {
+  for (Node* func_body_node : func_body_graph->nodes()) {
     const auto input_placeholder_idx =
-        input_placeholders_idx.find(func_body_node.name());
+        input_placeholders_idx.find(func_body_node->name());
 
     if (input_placeholder_idx != input_placeholders_idx.end()) {
       const int input_idx = input_placeholder_idx->second;
       const GraphView::OutputPort output_port =
           ctx->graph_view().GetRegularFanin({&func_node, input_idx});
 
-      VLOG(3) << "Pin inlined function input node '" << func_body_node.name()
+      VLOG(3) << "Pin inlined function input node '" << func_body_node->name()
               << "' to the '" << output_port.node->device() << "' device.";
-      func_body_node.set_device(output_port.node->device());
+      func_body_node->set_requested_device(output_port.node->device());
     }
   }
 
@@ -1511,8 +1523,6 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
   // After placing nodes corresponding to the function inputs, we need to assign
   // device placements to all other function body nodes.
 
-  GraphDef placed_graph_def;
-
   const DeviceSet* devices = ctx->devices();
 
   if (devices->devices().empty()) {
@@ -1522,9 +1532,8 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
     // of a batch job for graph analysis/optimization.
     VLOG(3) << "Assign function call node device to all function body nodes. "
             << "Device: " << func_node.device();
-    placed_graph_def = item.mutable_function_body();
-    for (NodeDef& node : *placed_graph_def.mutable_node()) {
-      node.set_device(func_node.device());
+    for (Node* func_body_node : func_body_graph->nodes()) {
+      func_body_node->set_requested_device(func_node.device());
     }
   } else {
     // If we are running in an active runtime session, Grappler will get the
@@ -1536,24 +1545,21 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
                    [](string* out, const Device* d) { out->append(d->name()); })
             << "]";
 
-    // Construct a Graph object from the instantiated function body.
-    GraphConstructorOptions opts;
-    Graph graph(ctx->function_library());
-    TF_RETURN_IF_ERROR(
-        ConvertGraphDefToGraph(opts, item.function_body(), &graph));
-
     // Use function caller node device as a default for placer.
     const Device* default_device =
         devices->FindDeviceByName(func_node.device());
 
-    Placer placer(&graph, devices, nullptr, /* No session options */
-                  default_device);
+    Placer placer(func_body_graph.get(), devices,
+                  nullptr /* No session options */, default_device);
     TF_RETURN_IF_ERROR(placer.Run());
-
-    // Convert Graph back to the GraphDef.
-    graph.ToGraphDef(&placed_graph_def);
   }
 
+  // TODO(ezhulenev): Should we run POST_PLACEMENT runtime pass here?
+
+  // All following transformations will work with GraphDef.
+  GraphDef placed_graph_def;
+  func_body_graph->ToGraphDef(&placed_graph_def);
+
   // ------------------------------------------------------------------------ //
   // After all nodes placed we need to prepare them for inlining into the
   // optimized graph: turn placeholders into identities, update nodes
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
index 827b0658c5..f4752bd584 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
@@ -14,6 +14,8 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/core/grappler/optimizers/function_optimizer.h"
+
+#include "absl/algorithm/container.h"
 #include "tensorflow/cc/ops/functional_ops.h"
 #include "tensorflow/cc/ops/standard_ops.h"
 #include "tensorflow/core/framework/function_testlib.h"
@@ -1182,6 +1184,126 @@ TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithMergedDeadTensors) {
   test::ExpectTensorEqual<float>(tensors[0], tensors_expected[0]);
 }
 
+TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithFunctionalControlFlow) {
+  using test::function::NDef;
+  using FDH = FunctionDefHelper;
+
+  FunctionOptimizer optimizer(RewriterConfig::AGGRESSIVE);
+
+  FunctionDef add_func = FunctionDefHelper::Create(
+      "MyAdd", {"x:T", "y:T"}, {"z:T"}, {"T: {float, double}"},
+      {{{"add"}, "Add", {"x", "y"}, {{"T", "$T"}}}},
+      /* Mapping between function returns and function node outputs. */
+      {{"z", "add:z:0"}});
+
+  FunctionDef mul_func = FunctionDefHelper::Create(
+      "MyMul", {"x:T", "y:T"}, {"z:T"}, {"T: {float, double}"},
+      {{{"mul"}, "Mul", {"x", "y"}, {{"T", "$T"}}}},
+      /* Mapping between function returns and function node outputs. */
+      {{"z", "mul:z:0"}});
+
+  // Compute: return cond ? a + b : a * b
+  FunctionDef add_or_mul_func = FunctionDefHelper::Create(
+      "AddOrMul", {"cond:bool", "x:float", "y:float"}, {"z:float"}, {},
+      {
+          {{"if_node"},
+           "If",
+           {"cond", "x", "y"},
+           {
+               {"Tcond", DT_BOOL},
+               {"Tin", DataTypeSlice{DT_FLOAT, DT_FLOAT}},
+               {"Tout", DataTypeSlice{DT_FLOAT}},
+               {"then_branch", FDH::FunctionRef("MyAdd", {{"T", DT_FLOAT}})},
+               {"else_branch", FDH::FunctionRef("MyMul", {{"T", DT_FLOAT}})},
+               {"_lower_using_switch_merge", true},
+           }},
+      },
+      /* Mapping between function returns and function node outputs. */
+      {{"z", "if_node:output:0"}});
+
+  // Build a computation graph for:
+  //   is_add: bool
+  //   a: float
+  //   b: float
+  //   c = AddOrMul(is_add, a, b)  # is_add ? a + b : a * b
+  //   d = Identity(c)
+  //   return d
+
+  // c = MyMul(a, b)
+  GrapplerItem item;
+  item.fetch = {"d"};
+  item.graph = test::function::GDef(
+      {NDef("is_add", "Placeholder", {}, {{"dtype", DT_BOOL}}, kDevice),
+       NDef("a", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
+       NDef("b", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
+
+       NDef("c", "PartitionedCall", {"is_add", "a", "b"},
+            {{"Tin", DataTypeSlice{DT_BOOL, DT_FLOAT, DT_FLOAT}},
+             {"Tout", DataTypeSlice{DT_FLOAT}},
+             {"f", FDH::FunctionRef("AddOrMul")}},
+            kDevice),
+
+       NDef("d", "Identity", {"c"}, {{"T", DT_FLOAT}}, kDevice)},
+      // Function library.
+      {add_or_mul_func, add_func, mul_func});
+
+  GraphDef optimized_graph;
+  TF_EXPECT_OK(optimizer.Optimize(nullptr, item, &optimized_graph));
+
+  const auto count_nodes_with_op = [&](const string& op) {
+    return absl::c_count_if(optimized_graph.node(), [&](const NodeDef& node) {
+      return node.op() == op;
+    });
+  };
+
+  // All `PartitionedCall` nodes in the optimized graph must be inlined, and
+  // `If` node must be lowered to `Switch` and `Merge` nodes.
+  EXPECT_EQ(count_nodes_with_op("PartitionedCallOp"), 0);
+  EXPECT_EQ(count_nodes_with_op("If"), 0);
+  EXPECT_EQ(count_nodes_with_op("Switch"), 3);
+  EXPECT_EQ(count_nodes_with_op("Merge"), 1);
+
+  GrapplerItem optimized = item.WithGraph(std::move(optimized_graph));
+
+  Tensor one = test::AsScalar<float>(1.0);
+  Tensor two = test::AsScalar<float>(2.0);
+  Tensor three = test::AsScalar<float>(3.0);
+
+  const auto feed_args = [&](bool is_add) {
+    std::vector<std::pair<string, Tensor>> feed;
+    feed.emplace_back("a", one);
+    feed.emplace_back("b", two);
+    feed.emplace_back("is_add", test::AsScalar<bool>(is_add));
+    return feed;
+  };
+
+  {  // Check 'is_add == true': a + b
+    item.feed = feed_args(true);
+    optimized.feed = feed_args(true);
+
+    auto tensors_expected = EvaluateFetchNodes(item);
+    ASSERT_EQ(tensors_expected.size(), 1);
+    test::ExpectTensorEqual<float>(tensors_expected[0], three);
+
+    auto tensors = EvaluateFetchNodes(optimized);
+    ASSERT_EQ(tensors.size(), tensors_expected.size());
+    test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
+  }
+
+  {  // Check 'is_add == false': a * b
+    item.feed = feed_args(false);
+    optimized.feed = feed_args(false);
+
+    auto tensors_expected = EvaluateFetchNodes(item);
+    ASSERT_EQ(tensors_expected.size(), 1);
+    test::ExpectTensorEqual<float>(tensors_expected[0], two);
+
+    auto tensors = EvaluateFetchNodes(optimized);
+    ASSERT_EQ(tensors.size(), tensors_expected.size());
+    test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
+  }
+}
+
 TEST_F(FunctionOptimizerTest, SpecializeFunctionXTimesTwo) {
   using test::function::NDef;
 
-- 
GitLab


From 7b8464b172fbe96464e7a800f89883e5796b1458 Mon Sep 17 00:00:00 2001
From: Thomas O'Malley <omalleyt@google.com>
Date: Tue, 15 Jan 2019 11:49:02 -0800
Subject: [PATCH 0760/1095] Adds Multi-Worker Independent Worker Mode to Keras.

PiperOrigin-RevId: 229412056
---
 .../distribute/distribute_coordinator.py      | 15 +--
 tensorflow/python/keras/BUILD                 |  1 +
 tensorflow/python/keras/engine/training.py    | 91 +++++++++++++++----
 .../python/keras/engine/training_utils.py     | 10 ++
 4 files changed, 91 insertions(+), 26 deletions(-)

diff --git a/tensorflow/python/distribute/distribute_coordinator.py b/tensorflow/python/distribute/distribute_coordinator.py
index 78c995a578..89bd8537b4 100644
--- a/tensorflow/python/distribute/distribute_coordinator.py
+++ b/tensorflow/python/distribute/distribute_coordinator.py
@@ -721,7 +721,8 @@ def run_distribute_coordinator(worker_fn,
 
   Returns:
     In the client job, return the value returned by `worker_fn` if
-    it is in-graph replication; return None otherwise.
+    it is in-graph replication or INDEPENDENT_WORKER mode; return None
+    otherwise.
   """
   tf_config = json.loads(os.environ.get("TF_CONFIG", "{}"))
   if not cluster_spec:
@@ -819,19 +820,19 @@ def run_distribute_coordinator(worker_fn,
     if task_type in [_TaskType.CHIEF, _TaskType.WORKER]:
       if strategy.extended.experimental_between_graph:
         # All jobs run `worker_fn` if between-graph.
-        _run_single_worker(worker_fn, strategy, cluster_spec, task_type,
-                           task_id, session_config, rpc_layer)
+        return _run_single_worker(worker_fn, strategy, cluster_spec, task_type,
+                                  task_id, session_config, rpc_layer)
       else:
         # Only one node runs `worker_fn` if in-graph.
         context = _WorkerContext(strategy, cluster_spec, task_type, task_id)
         if context.is_chief:
-          _run_single_worker(worker_fn, strategy, cluster_spec, None, None,
-                             session_config, rpc_layer)
+          return _run_single_worker(worker_fn, strategy, cluster_spec, None,
+                                    None, session_config, rpc_layer)
         else:
           server.join()
     elif task_type == _TaskType.EVALUATOR:
-      _run_single_worker(eval_fn, eval_strategy, cluster_spec, task_type,
-                         task_id, session_config, rpc_layer)
+      return _run_single_worker(eval_fn, eval_strategy, cluster_spec, task_type,
+                                task_id, session_config, rpc_layer)
     else:
       if task_type != _TaskType.PS:
         raise ValueError("Unexpected task_type: %r" % task_type)
diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index d1ec5f0ac3..9e08c766db 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -152,6 +152,7 @@ py_library(
         ":regularizers",
         ":saving",
         "//tensorflow/python/data",
+        "//tensorflow/python/distribute:distribute_coordinator",
         "//tensorflow/python/distribute:reduce_util",
         "//tensorflow/python/training/checkpointable:data_structures",
         "//tensorflow/tools/docs:doc_controls",
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 9f544ceff4..d690df2986 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -24,6 +24,8 @@ import numpy as np
 from tensorflow.python import tf2
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import iterator_ops
+from tensorflow.python.distribute import distribute_coordinator as dc
+from tensorflow.python.distribute import distribute_coordinator_context as dc_context
 from tensorflow.python.eager import context
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
@@ -128,6 +130,7 @@ class Model(Network):
     # passing distribution strategy to compile rather than creating the model
     # under distribution strategy scope.
     self._compile_distribution = False
+    self._distributed_session_is_configured = False
 
     self.run_eagerly = None
 
@@ -216,6 +219,7 @@ class Model(Network):
                       'create the model under the distribution strategy scope.')
       self._distribution_strategy = distribute
       self._compile_distribution = True
+      self._distributed_session_is_configured = False
     else:
       if distribution_strategy_context.has_strategy():
         # When the user builds the model in the DS scope and cross replica
@@ -273,9 +277,6 @@ class Model(Network):
 
     # Set DistributionStrategy specific parameters.
     self._distributed_model = None
-    if self._distribution_strategy is not None:
-      distributed_training_utils.configure_and_create_session(
-          self._distribution_strategy)
     # Initialize model metric attributes.
     self._init_metric_attributes()
     if not self.built:
@@ -781,22 +782,52 @@ class Model(Network):
 
     # Case 1: distribution strategy.
     if self._distribution_strategy:
-      return training_distributed.fit_distributed(
-          self,
-          x=x,
-          y=y,
-          batch_size=batch_size,
-          epochs=epochs,
-          verbose=verbose,
-          callbacks=callbacks,
-          validation_split=validation_split,
-          validation_data=validation_data,
-          shuffle=shuffle,
-          class_weight=class_weight,
-          sample_weight=sample_weight,
-          initial_epoch=initial_epoch,
-          steps_per_epoch=steps_per_epoch,
-          validation_steps=validation_steps)
+      if training_utils.should_run_multi_worker():
+        # Multi-Worker mode runs the Keras training loop on multiple
+        # servers via the Distribute Coordinator.
+        def _worker_fn(_):
+          """Run training inside the distributed coordinator."""
+          self._configure_distributed_session()
+          return training_distributed.fit_distributed(
+              self,
+              x=x,
+              y=y,
+              batch_size=batch_size,
+              epochs=epochs,
+              verbose=verbose,
+              callbacks=callbacks,
+              validation_split=validation_split,
+              validation_data=validation_data,
+              shuffle=shuffle,
+              class_weight=class_weight,
+              sample_weight=sample_weight,
+              initial_epoch=initial_epoch,
+              steps_per_epoch=steps_per_epoch,
+              validation_steps=validation_steps)
+
+        # Independent worker only for now.
+        return dc.run_distribute_coordinator(
+            _worker_fn,
+            self._distribution_strategy,
+            mode=dc.CoordinatorMode.INDEPENDENT_WORKER)
+      else:
+        self._configure_distributed_session()
+        return training_distributed.fit_distributed(
+            self,
+            x=x,
+            y=y,
+            batch_size=batch_size,
+            epochs=epochs,
+            verbose=verbose,
+            callbacks=callbacks,
+            validation_split=validation_split,
+            validation_data=validation_data,
+            shuffle=shuffle,
+            class_weight=class_weight,
+            sample_weight=sample_weight,
+            initial_epoch=initial_epoch,
+            steps_per_epoch=steps_per_epoch,
+            validation_steps=validation_steps)
 
     batch_size = self._validate_or_infer_batch_size(
         batch_size, steps_per_epoch, x)
@@ -1007,6 +1038,7 @@ class Model(Network):
     """
     # Case 1: distribution strategy.
     if self._distribution_strategy:
+      self._configure_distributed_session()
       return training_distributed.evaluate_distributed(
           self,
           x=x,
@@ -1134,6 +1166,7 @@ class Model(Network):
     """
     # Case 1: distribution strategy.
     if self._distribution_strategy:
+      self._configure_distributed_session()
       return training_distributed.predict_distributed(self,
                                                       x=x,
                                                       batch_size=batch_size,
@@ -2644,6 +2677,26 @@ class Model(Network):
     self.output_names = training_utils.generic_output_names(outputs)
     self.built = True
 
+  def _configure_distributed_session(self):
+    """Configure a Session for use with Distribution Strategies.
+
+    Raises:
+      ValueError: If a non-distributed Session has already been created.
+    """
+    if not self._distributed_session_is_configured:
+      if (dc_context.get_current_worker_context() is not None and
+          getattr(K._SESSION, 'session', None) is not None):  # pylint: disable=protected-access
+        raise ValueError('Session was created before `fit`, `evaluate`, '
+                         'or `predict` was called. With Multi-Worker '
+                         'mode, this is not allowed. Please avoid '
+                         'creating a Session outside of these methods. '
+                         'The Session may have been created by a call '
+                         'to `keras.backend.get_session()` or '
+                         'functions that use Sessions, like `load_weights`.')
+      distributed_training_utils.configure_and_create_session(
+          self._distribution_strategy)
+      self._distributed_session_is_configured = True
+
 
 class DistributedCallbackModel(Model):
   """Model that is used for callbacks with DistributionStrategy."""
diff --git a/tensorflow/python/keras/engine/training_utils.py b/tensorflow/python/keras/engine/training_utils.py
index f7cbf4101a..c36b637829 100644
--- a/tensorflow/python/keras/engine/training_utils.py
+++ b/tensorflow/python/keras/engine/training_utils.py
@@ -21,6 +21,8 @@ from __future__ import print_function
 import abc
 from collections import OrderedDict
 import copy
+import json
+import os
 
 import numpy as np
 import six
@@ -46,6 +48,7 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import weights_broadcast_ops
 from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.training import server_lib
 from tensorflow.python.util import nest
 
 
@@ -1492,3 +1495,10 @@ def convert_eager_tensors_to_numpy(structure):
     return element
 
   return nest.map_structure(_convert, structure)
+
+
+def should_run_multi_worker():
+  """Whether a model should be run using DistributedCoordinator."""
+  tf_config = json.loads(os.environ.get('TF_CONFIG', '{}'))
+  cluster_spec = server_lib.ClusterSpec(tf_config.get('cluster', {}))
+  return tf_config and 'master' not in cluster_spec.jobs
-- 
GitLab


From b58039978dda1ddc4fadb2ee528cfdb9878e000a Mon Sep 17 00:00:00 2001
From: Ayush Dubey <ayushd@google.com>
Date: Tue, 15 Jan 2019 12:06:18 -0800
Subject: [PATCH 0761/1095] Automated rollback of commit
 681f6a6ac9344425549c8a38fcbfae854e7deddf

PiperOrigin-RevId: 229415244
---
 tensorflow/core/BUILD                         |   2 +-
 .../base_collective_executor.cc               |  33 --
 .../common_runtime/base_collective_executor.h |  18 -
 .../collective_param_resolver_local.cc        | 159 +++------
 .../collective_param_resolver_local.h         |  15 +-
 .../core/common_runtime/collective_util.cc    |  31 --
 .../core/common_runtime/collective_util.h     |  21 --
 .../hierarchical_tree_broadcaster.h           |   5 -
 .../core/common_runtime/ring_reducer.cc       |  43 ++-
 tensorflow/core/common_runtime/ring_reducer.h |  24 +-
 tensorflow/core/distributed_runtime/BUILD     |   1 -
 .../collective_param_resolver_distributed.cc  |  21 +-
 ...lective_param_resolver_distributed_test.cc |   2 -
 tensorflow/core/framework/collective.cc       |  11 -
 tensorflow/core/framework/collective.h        |  32 +-
 tensorflow/core/kernels/BUILD                 |  24 --
 .../core/kernels/collective_nccl_reducer.cc   | 204 -----------
 .../core/kernels/collective_nccl_reducer.h    |  49 ---
 .../kernels/collective_nccl_reducer_test.cc   | 331 ------------------
 tensorflow/core/kernels/collective_ops.cc     |   3 -
 tensorflow/core/nccl/BUILD                    |   4 +-
 tensorflow/core/protobuf/worker.proto         |   1 -
 22 files changed, 118 insertions(+), 916 deletions(-)
 delete mode 100644 tensorflow/core/kernels/collective_nccl_reducer.cc
 delete mode 100644 tensorflow/core/kernels/collective_nccl_reducer.h
 delete mode 100644 tensorflow/core/kernels/collective_nccl_reducer_test.cc

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 1fc917a5f2..bedc78ac11 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -3714,6 +3714,7 @@ tf_cc_tests(
     srcs = [
         "common_runtime/buf_rendezvous_test.cc",
         "common_runtime/collective_executor_mgr_test.cc",
+        "common_runtime/collective_param_resolver_local_test.cc",
         "common_runtime/collective_rma_local_test.cc",
         "common_runtime/device_resolver_local_test.cc",
         "common_runtime/device_set_test.cc",
@@ -3829,7 +3830,6 @@ tf_cc_tests(
     name = "higher_level_tests_needing_kernels",
     size = "small",
     srcs = [
-        "common_runtime/collective_param_resolver_local_test.cc",
         "graph/graph_constructor_test.cc",
     ],
     linkopts = select({
diff --git a/tensorflow/core/common_runtime/base_collective_executor.cc b/tensorflow/core/common_runtime/base_collective_executor.cc
index 03c0e9ce2b..92e56df181 100644
--- a/tensorflow/core/common_runtime/base_collective_executor.cc
+++ b/tensorflow/core/common_runtime/base_collective_executor.cc
@@ -296,37 +296,4 @@ Status BaseCollectiveExecutor::CreateCollective(
   return status;
 }
 
-bool BaseCollectiveExecutor::CheckDependencies(
-    const CollectiveParams& col_params) {
-  for (int32 instance : col_params.instance.impl_details.dependencies) {
-    auto find_iter = launched_.find(instance);
-    if (find_iter == launched_.end() || find_iter->second != 0) {
-      return false;
-    }
-  }
-  return true;
-}
-
-void BaseCollectiveExecutor::WaitForDependencies(
-    const CollectiveParams& col_params) {
-  mutex_lock l(launch_mu_);
-  while (!CheckDependencies(col_params)) {
-    launch_cv_.wait(l);
-  }
-}
-
-void BaseCollectiveExecutor::Launched(const CollectiveParams& col_params) {
-  mutex_lock l(launch_mu_);
-  if (launched_.find(col_params.instance.instance_key) == launched_.end()) {
-    const string& task_name =
-        col_params.instance.task_names[col_params.default_rank];
-    const int32 num_devices =
-        col_params.instance.num_devices_per_task.at(task_name);
-    launched_[col_params.instance.instance_key] = num_devices;
-  }
-  if (--launched_[col_params.instance.instance_key] == 0) {
-    launch_cv_.notify_all();
-  }
-}
-
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/base_collective_executor.h b/tensorflow/core/common_runtime/base_collective_executor.h
index b711aa6d50..09826a8814 100644
--- a/tensorflow/core/common_runtime/base_collective_executor.h
+++ b/tensorflow/core/common_runtime/base_collective_executor.h
@@ -135,33 +135,15 @@ class BaseCollectiveExecutor : public CollectiveExecutor {
                                client_locality, done);
   }
 
-  // If we need to enforce an ordering on any portion of collective
-  // implementation, and the ordering is encoded via attribute on the collective
-  // op, this function will block until all dependencies for this collective
-  // have completed.
-  void WaitForDependencies(const CollectiveParams& col_params) override;
-  // Record that this collective has completed the portion of the implementation
-  // that needs to be ordered wrt other collectives, to unblock any of its
-  // dependent ops.
-  void Launched(const CollectiveParams& col_params) override;
-
  protected:
   const int64 step_id_;
   const DeviceMgr* dev_mgr_;  // Not owned.
   std::unique_ptr<PerStepCollectiveRemoteAccess> remote_access_;
   const string* gpu_ring_order_;  // Not owned.
-  mutex launch_mu_;
-  condition_variable launch_cv_;
-  // collective instance key -> number of local devices for which NCCL ops have
-  // been launched.
-  std::unordered_map<int32, int32> launched_ GUARDED_BY(launch_mu_);
 
  private:
   Status CreateCollective(const CollectiveParams& col_params,
                           CollectiveImplementationInterface** col_impl);
-  // Check if all ops on which this collective depends on have launched.
-  bool CheckDependencies(const CollectiveParams& col_params)
-      EXCLUSIVE_LOCKS_REQUIRED(launch_mu_);
 };
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/collective_param_resolver_local.cc b/tensorflow/core/common_runtime/collective_param_resolver_local.cc
index 8907f6d56a..a8e3f4c881 100644
--- a/tensorflow/core/common_runtime/collective_param_resolver_local.cc
+++ b/tensorflow/core/common_runtime/collective_param_resolver_local.cc
@@ -26,7 +26,6 @@ limitations under the License.
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/gtl/flatmap.h"
-#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/types.h"
 #include "tensorflow/core/util/device_name_utils.h"
@@ -40,10 +39,7 @@ void CollectiveParamResolverLocal::InstanceRec::WaitForOutMu(mutex_lock& lock) {
 CollectiveParamResolverLocal::CollectiveParamResolverLocal(
     const DeviceMgr* dev_mgr, DeviceResolverInterface* dev_resolver,
     const string& task_name)
-    : nccl_(false),  // (b/111897089): turn on NCCL collectives.
-      dev_mgr_(dev_mgr),
-      dev_resolver_(dev_resolver),
-      task_name_(task_name) {}
+    : dev_mgr_(dev_mgr), dev_resolver_(dev_resolver), task_name_(task_name) {}
 
 void CollectiveParamResolverLocal::CompleteGroupAsync(
     const CompleteGroupRequest* request, CompleteGroupResponse* response,
@@ -320,28 +316,29 @@ GlobalDeviceMap EstablishGlobalRank(
 // cp->same_num_devices_per_task.  Requires cp->instance.task_names
 // be sorted.
 void SetDevPerTask(CollectiveParams* cp) {
-  cp->instance.num_devices_per_task.clear();
-  const string* last_task_name = &cp->instance.task_names[0];
+  cp->instance.same_num_devices_per_task = false;
+  if (cp->instance.task_names.empty()) return;
+  int dev_per_task = -1;
   int count = 0;
+  const string* last_task_name = &cp->instance.task_names[0];
   for (const string& task_name : cp->instance.task_names) {
-    if (task_name == *last_task_name) {
-      ++count;
-    } else {
-      cp->instance.num_devices_per_task[*last_task_name] = count;
+    if (task_name != *last_task_name) {
+      CHECK_GT(count, 0);
+      if (dev_per_task < 0) {
+        dev_per_task = count;
+      } else {
+        CHECK_GT(dev_per_task, 0);
+        if (count != dev_per_task) return;
+      }
       count = 1;
       last_task_name = &task_name;
+    } else {
+      ++count;
     }
   }
-  cp->instance.num_devices_per_task[*last_task_name] = count;
-
-  cp->instance.same_num_devices_per_task = false;
-  int dev_per_task = -1;
-  for (const auto& task_dev : cp->instance.num_devices_per_task) {
-    if (dev_per_task == -1) {
-      dev_per_task = task_dev.second;
-    } else if (dev_per_task != task_dev.second) {
-      return;
-    }
+  CHECK_GT(count, 0);
+  if ((dev_per_task > 0) && (count != dev_per_task)) {
+    return;
   }
   cp->instance.same_num_devices_per_task = true;
   CHECK_EQ((cp->group.group_size % cp->group.num_tasks), 0);
@@ -401,6 +398,7 @@ void CollectiveParamResolverLocal::SetDefaultRank(const string& device,
 void CollectiveParamResolverLocal::InitInstanceSharedParams(
     const GroupRec* gr, const CollectiveParams* cp, InstanceRec* ir,
     const StatusCallback& done) {
+  VLOG(1) << "InitInstanceSharedParams " << ir;
   ir->shared.instance = cp->instance;
   {
     mutex_lock gl(gr->mu);
@@ -414,8 +412,8 @@ void CollectiveParamResolverLocal::InitInstanceSharedParams(
   }
   ir->shared.default_rank = -1;
 
-  // Sort device_names lexicographically, keeping task_names in corresponding
-  // order.  Also set number of devices per task.
+  // Sort devce_names lexicographcally, keeping task_names in
+  // corresponding order.
   SortDevicesAndTasks(&ir->shared);
 
   // Get Locality data for all devices.
@@ -607,25 +605,6 @@ void CollectiveParamResolverLocal::CompleteInstanceAsync(
                        "intended only for non-distributed deployment."));
 }
 
-// TODO(b/111897089): we need a better way to pick the collective
-// implementation.  The ideal way would depend upon the topology and link
-// strength before picking a particular implementation.
-void CollectiveParamResolverLocal::AssignCollectiveType(CollectiveParams* cp) {
-  if (cp->instance.type == BROADCAST_COLLECTIVE) {
-    cp->instance.impl_details.collective_name = "HierarchicalTreeBroadcast";
-  } else if (cp->instance.type == REDUCTION_COLLECTIVE) {
-    if (nccl_) {
-      cp->instance.impl_details.collective_name = "NcclReduce";
-    } else {
-      cp->instance.impl_details.collective_name = "RingReduce";
-    }
-  } else {
-    cp->instance.impl_details.collective_name = "undef";
-  }
-  VLOG(1) << "AssignCollectiveType "
-          << cp->instance.impl_details.collective_name;
-}
-
 void CollectiveParamResolverLocal::CompleteInstanceLocal(
     const string& device, const GroupRec* gr, CollectiveParams* cp,
     bool is_source, const StatusCallback& done) {
@@ -662,57 +641,48 @@ void CollectiveParamResolverLocal::CompleteInstanceFromInitializedIRec(
     // custom operator= does a deep copy.
     cp->instance = ir->shared.instance;
   }
-  // Populate the fields common across task.
-  AssignCollectiveType(cp);
+  // Populate the fields common across task, also default_rank.
   SetDefaultRank(device, cp);
   CompleteTaskIsLocal(task_name_, cp);
-
+  // TODO(b/113171733): we need a better way to pick the collective
+  // implementation.  The ideal way would depend upon the topology and link
+  // strength before picking a particular implementation.
+  cp->instance.impl_details.collective_name =
+      (cp->instance.type == BROADCAST_COLLECTIVE) ? "HierarchicalTreeBroadcast"
+                                                  : "RingReduce";
   CollectiveImplementationInterface* col_impl;
-  Status status = CollectiveRegistry::LookupParamResolverInstance(
+  Status lookup_status = CollectiveRegistry::LookupParamResolverInstance(
       cp->instance.impl_details.collective_name, &col_impl);
-  if (status.ok()) {
-    status = col_impl->InitializeInstanceBeforeGroupDiscovery(cp);
-  }
-  if (!status.ok()) {
-    done(status);
+  if (!lookup_status.ok()) {
+    done(lookup_status);
     return;
   }
-
-  //  We may need to wait for the group if:
-  //  * this is a broadcast, for source discovery;
-  //  * we are using NCCL with more than 1 worker, for the communicator key from
-  //    rank 0.
-  bool broadcast = cp->instance.type == BROADCAST_COLLECTIVE;
-  bool nccl = cp->instance.type == REDUCTION_COLLECTIVE &&
-              cp->instance.impl_details.collective_name == "NcclReduce" &&
-              cp->group.num_tasks > 1;
-  if (broadcast || nccl) {
-    WaitForGroup(ir, cp, is_source, broadcast, nccl,
-                 [col_impl, ir, device, cp, done](InstanceRec* irec) {
-                   Status s;
-                   if (ir != irec) {
-                     s = errors::Internal("Expected ir ", ir, " and irec ",
-                                          irec, " to be equal");
-                   } else {
-                     mutex_lock l(irec->out_mu);
-                     irec->WaitForOutMu(l);
-                     s = irec->status;
-                     cp->source_rank = irec->source_rank;
-                     cp->instance.communicator_key = irec->communicator_key;
-                   }
-                   if (s.ok()) {
-                     s = col_impl->InitializeCollectiveParams(cp);
-                   }
-                   done(s);
-                 });
+  // If broadcast, may need to wait for source discovery.
+  if (cp->instance.type == BROADCAST_COLLECTIVE) {
+    CompleteInstanceSource(ir, cp, is_source,
+                           [col_impl, ir, device, cp, done](InstanceRec* irec) {
+                             CHECK_EQ(ir, irec);
+                             Status s;
+                             {
+                               mutex_lock l(irec->out_mu);
+                               irec->WaitForOutMu(l);
+                               s = irec->status;
+                               cp->source_rank = irec->source_rank;
+                             }
+                             if (s.ok()) {
+                               s = col_impl->InitializeCollectiveParams(cp);
+                             }
+                             done(s);
+                           });
   } else {
     done(col_impl->InitializeCollectiveParams(cp));
   }
 }
 
-void CollectiveParamResolverLocal::WaitForGroup(
-    InstanceRec* ir, CollectiveParams* cp, bool is_source, bool init_source,
-    bool init_nccl, const IRConsumer& f) {
+void CollectiveParamResolverLocal::CompleteInstanceSource(InstanceRec* ir,
+                                                          CollectiveParams* cp,
+                                                          bool is_source,
+                                                          const IRConsumer& f) {
   std::vector<IRConsumer> ready_waiters;
   {
     mutex_lock l(ir->out_mu);
@@ -722,8 +692,7 @@ void CollectiveParamResolverLocal::WaitForGroup(
     if (!ir->known[cp->default_rank]) {
       ir->known[cp->default_rank] = true;
       ++ir->known_count;
-      if (init_source && is_source) {
-        // Initialize source rank.
+      if (is_source) {
         if (ir->source_rank >= 0) {
           ir->status = errors::Internal("Instance ", cp->instance.instance_key,
                                         " already has source ", ir->source_rank,
@@ -733,26 +702,13 @@ void CollectiveParamResolverLocal::WaitForGroup(
           ir->source_rank = cp->default_rank;
         }
       }
-      if (init_nccl && cp->default_rank == 0) {
-        // Initialize communicator key.
-        if (!ir->communicator_key.empty()) {
-          ir->status =
-              errors::Internal("Instance ", cp->instance.instance_key,
-                               " already has communicator_key ",
-                               str_util::CEscape(ir->communicator_key),
-                               ", received second claim from device ",
-                               cp->instance.device_names[cp->default_rank]);
-        } else {
-          ir->communicator_key = cp->instance.communicator_key;
-        }
-      }
     }
     if (ir->known_count < ir->shared.group.group_size) {
       ir->known_waiters.push_back(f);
       return;
     }
     CHECK_EQ(ir->known_count, ir->shared.group.group_size);
-    if (init_source && ir->source_rank < 0) {
+    if (ir->source_rank < 0) {
       // NOTE(ayushd): changing the error message below would also require
       // updating CompleteParamsBroadcastForgotSend test in
       // CollectiveParamResolverLocalTest.
@@ -762,13 +718,6 @@ void CollectiveParamResolverLocal::WaitForGroup(
                            "could mean that there were group_size=",
                            ir->known_count, " BcastRecvs but no BcastSend.");
     }
-    if (init_nccl && ir->communicator_key.empty()) {
-      ir->status = errors::Internal(
-          "Instance ", cp->instance.instance_key, " device ",
-          cp->instance.device_names[cp->default_rank],
-          " did not find rank 0 for setting communicator key.  This is an "
-          "internal error in collective param resolution");
-    }
     if (!ir->known_waiters.empty()) {
       ready_waiters = std::move(ir->known_waiters);
     }
diff --git a/tensorflow/core/common_runtime/collective_param_resolver_local.h b/tensorflow/core/common_runtime/collective_param_resolver_local.h
index fd408e4ef3..365bddc787 100644
--- a/tensorflow/core/common_runtime/collective_param_resolver_local.h
+++ b/tensorflow/core/common_runtime/collective_param_resolver_local.h
@@ -130,10 +130,8 @@ class CollectiveParamResolverLocal : public ParamResolverInterface {
     Status status GUARDED_BY(out_mu);
 
     // These fields are used to count the instances that have called
-    // in and become known while resolving broadcast source identity and
-    // communicator key.
+    // in and become known while resolving broadcast source identity.
     int source_rank GUARDED_BY(out_mu);
-    string communicator_key GUARDED_BY(out_mu);
     int known_count GUARDED_BY(out_mu);
     std::vector<bool> known GUARDED_BY(out_mu);
     std::vector<IRConsumer> known_waiters GUARDED_BY(out_mu);
@@ -199,10 +197,10 @@ class CollectiveParamResolverLocal : public ParamResolverInterface {
                                            const StatusCallback& done)
       LOCKS_EXCLUDED(ir->out_mu);
 
-  // Complete source data and/or nccl communicator key.
+  // Complete source data for a broadcast instance.
   // Precondition: *cp has complete group data and default_rank.
-  void WaitForGroup(InstanceRec* ir, CollectiveParams* cp, bool is_source,
-                    bool init_source, bool init_nccl, const IRConsumer& f)
+  void CompleteInstanceSource(InstanceRec* ir, CollectiveParams* cp,
+                              bool is_source, const IRConsumer& f)
       LOCKS_EXCLUDED(ir->out_mu);
 
   // If cp.device_names contains only devices local to this process
@@ -218,15 +216,10 @@ class CollectiveParamResolverLocal : public ParamResolverInterface {
   // current ordering of cp->instance.device_names.
   void SetDefaultRank(const string& device, CollectiveParams* cp);
 
-  // Sets cp->instance.type based on collective op type, and attempts to assign
-  // best implementation.
-  void AssignCollectiveType(CollectiveParams* cp);
-
   // Helper to grab status under lock, invoke callback out of lock.
   void CallbackWithStatus(const InstanceRecCallback& done, InstanceRec* irec)
       LOCKS_EXCLUDED(irec->out_mu);
 
-  const bool nccl_;
   const DeviceMgr* dev_mgr_;
   DeviceResolverInterface* dev_resolver_;  // Not owned.
   string task_name_;
diff --git a/tensorflow/core/common_runtime/collective_util.cc b/tensorflow/core/common_runtime/collective_util.cc
index bee4a13d18..195521a078 100644
--- a/tensorflow/core/common_runtime/collective_util.cc
+++ b/tensorflow/core/common_runtime/collective_util.cc
@@ -79,36 +79,5 @@ string SubdivPermDebugString(const CollectiveParams& col_params) {
   return buf;
 }
 
-SubContext::SubContext(OpKernelContext* ctx, OpKernelContext::Params* params,
-                       OpKernel* op, Tensor* output, Tensor* input)
-    : sub_params_(*params),
-      sub_inputs_({output, input}),
-      sub_input_attr_({ctx->input_alloc_attr(0), ctx->input_alloc_attr(0)}),
-      sub_input_dc_(
-          {ctx->input_device_context(0), ctx->input_device_context(0)}) {
-  sub_params_.op_kernel = op;
-  sub_params_.inputs = &sub_inputs_;
-  sub_params_.input_alloc_attrs = &sub_input_attr_;
-  sub_params_.input_device_contexts = &sub_input_dc_;
-  sub_params_.eigen_gpu_device = nullptr;
-  sub_params_.ensure_eigen_gpu_device();
-  sub_params_.forward_from_array = &forward_from_;
-  sub_ctx_.reset(new OpKernelContext(&sub_params_, 1));
-}
-
-Status ComputeBinOp(OpKernelContext* op_ctx, OpKernelContext::Params* params,
-                    Device* device, OpKernel* op, Tensor* output,
-                    Tensor* input) {
-  // Prepare an OpKernelContext that is identical to that of the original Op
-  // (i.e. the collective), except for the input output sizes and identities and
-  // the Op itself.
-  // TODO(ayushd, tucker): Is it possible to cache and reuse these objects?
-  // They're mostly identical inside one device execution.
-  std::unique_ptr<SubContext> sub_ctx(
-      new SubContext(op_ctx, params, op, output, input));
-  device->Compute(op, sub_ctx->sub_ctx_.get());
-  return sub_ctx->sub_ctx_->status();
-}
-
 }  // namespace collective_util
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/collective_util.h b/tensorflow/core/common_runtime/collective_util.h
index 01fb8b8c81..ebb5731bec 100644
--- a/tensorflow/core/common_runtime/collective_util.h
+++ b/tensorflow/core/common_runtime/collective_util.h
@@ -32,27 +32,6 @@ Status InitializeDeviceAndLocality(const DeviceMgr* dev_mgr,
                                    DeviceLocality* device_locality);
 string SubdivPermDebugString(const CollectiveParams& col_params);
 
-// Used for executing a sub-operation, e.g. a merge_op instance, with
-// an OpKernelContext based on the one passed into this Op.
-class SubContext {
- public:
-  OpKernelContext::Params sub_params_;
-  gtl::InlinedVector<TensorValue, 4> sub_inputs_;
-  gtl::InlinedVector<AllocatorAttributes, 4> sub_input_attr_;
-  gtl::InlinedVector<DeviceContext*, 4> sub_input_dc_;
-  // Used only for Binary and Unary Ops for which we require
-  // the calculation to be in-place on the first input.
-  int forward_from_ = 0;
-  std::unique_ptr<OpKernelContext> sub_ctx_;
-  SubContext(OpKernelContext* ctx, OpKernelContext::Params* params,
-             OpKernel* op, Tensor* output, Tensor* input);
-  ~SubContext() = default;
-};
-
-Status ComputeBinOp(OpKernelContext* op_ctx, OpKernelContext::Params* params,
-                    Device* device, OpKernel* op, Tensor* output,
-                    Tensor* input);
-
 }  // namespace collective_util
 }  // namespace tensorflow
 
diff --git a/tensorflow/core/common_runtime/hierarchical_tree_broadcaster.h b/tensorflow/core/common_runtime/hierarchical_tree_broadcaster.h
index 76392b8e59..ceb9baad30 100644
--- a/tensorflow/core/common_runtime/hierarchical_tree_broadcaster.h
+++ b/tensorflow/core/common_runtime/hierarchical_tree_broadcaster.h
@@ -41,11 +41,6 @@ class HierarchicalTreeBroadcaster : public CollectiveImplementationInterface {
   // and device_locality.  Also saves the CollectiveContext in this object.
   Status InitializeCollectiveContext(CollectiveContext* col_ctx) override;
 
-  // No-op for hierarchical tree broadcaster.
-  Status InitializeInstanceBeforeGroupDiscovery(CollectiveParams*) override {
-    return Status::OK();
-  }
-
   // Begins async execution of the hierarchical tree broadcast.
   // Must be called in a blockable thread.
   // TODO(b/80529858): remove the previous warning when we have a dedicated
diff --git a/tensorflow/core/common_runtime/ring_reducer.cc b/tensorflow/core/common_runtime/ring_reducer.cc
index 8ed2fc2f1c..092f15e49e 100644
--- a/tensorflow/core/common_runtime/ring_reducer.cc
+++ b/tensorflow/core/common_runtime/ring_reducer.cc
@@ -394,6 +394,37 @@ void RingReducer::Finish(bool ok) {
   done_(s);
 }
 
+RingReducer::SubContext::SubContext(OpKernelContext* ctx,
+                                    OpKernelContext::Params* params,
+                                    OpKernel* op, Tensor* output, Tensor* input)
+    : sub_params_(*params),
+      sub_inputs_({output, input}),
+      sub_input_attr_({ctx->input_alloc_attr(0), ctx->input_alloc_attr(0)}),
+      sub_input_dc_(
+          {ctx->input_device_context(0), ctx->input_device_context(0)}) {
+  sub_params_.op_kernel = op;
+  sub_params_.inputs = &sub_inputs_;
+  sub_params_.input_alloc_attrs = &sub_input_attr_;
+  sub_params_.input_device_contexts = &sub_input_dc_;
+  sub_params_.eigen_gpu_device = nullptr;
+  sub_params_.ensure_eigen_gpu_device();
+  sub_params_.forward_from_array = &forward_from_;
+  sub_ctx_ = new OpKernelContext(&sub_params_, 1);
+}
+
+Status RingReducer::ComputeBinOp(Device* device, OpKernel* op, Tensor* output,
+                                 Tensor* input) {
+  // Prepare an OpKernelContext that is identical to that of the original Op
+  // (i.e. the collective), except for the input output sizes and identities and
+  // the Op itself.
+  // TODO(tucker): Is it possible to cache and reuse these objects?  They're
+  // mostly identical inside one device execution.
+  std::unique_ptr<SubContext> sub_ctx(
+      new SubContext(col_ctx_->op_ctx, col_ctx_->op_params, op, output, input));
+  device->Compute(op, sub_ctx->sub_ctx_);
+  return sub_ctx->sub_ctx_->status();
+}
+
 // At the beginning of the algorithm initialize a RingField struct for
 // every independent field of the tensor.
 void RingReducer::InitRingField(RingField* rf, int chunk_idx, int subdiv_idx,
@@ -601,9 +632,9 @@ bool RingReducer::RunAsyncParts() {
           --recv_pending_count;
           if (!rf->second_pass) {
             rf->action = RF_REDUCE;
-            Status s = collective_util::ComputeBinOp(
-                col_ctx_->op_ctx, col_ctx_->op_params, col_ctx_->device,
-                col_params_->merge_op.get(), &rf->chunk, &rf->tmp_chunk);
+            Status s =
+                ComputeBinOp(col_ctx_->device, col_params_->merge_op.get(),
+                             &rf->chunk, &rf->tmp_chunk);
             if (!s.ok()) {
               aborted = true;
               StartAbort(s);
@@ -616,9 +647,9 @@ bool RingReducer::RunAsyncParts() {
           if (!rf->second_pass && col_params_->final_op.get() && rf->is_final) {
             rf->action = RF_FINALIZE;
             group_size_tensor_ready_.WaitForNotification();
-            Status s = collective_util::ComputeBinOp(
-                col_ctx_->op_ctx, col_ctx_->op_params, col_ctx_->device,
-                col_params_->final_op.get(), &rf->chunk, &group_size_tensor_);
+            Status s =
+                ComputeBinOp(col_ctx_->device, col_params_->final_op.get(),
+                             &rf->chunk, &group_size_tensor_);
             if (!s.ok()) {
               aborted = true;
               StartAbort(s);
diff --git a/tensorflow/core/common_runtime/ring_reducer.h b/tensorflow/core/common_runtime/ring_reducer.h
index a5aa8fad70..0848e37b52 100644
--- a/tensorflow/core/common_runtime/ring_reducer.h
+++ b/tensorflow/core/common_runtime/ring_reducer.h
@@ -40,11 +40,6 @@ class RingReducer : public CollectiveImplementationInterface {
   // and device_locality.  Also saves the CollectiveContext in this object.
   Status InitializeCollectiveContext(CollectiveContext* col_ctx) override;
 
-  // No-op for ring reducer.
-  Status InitializeInstanceBeforeGroupDiscovery(CollectiveParams*) override {
-    return Status::OK();
-  }
-
   // Begins async execution of the ring reduce algorithm.
   // Must be called in a blockable thread.
   // TODO(b/80529858): remove the previous warning when we have a dedicated
@@ -57,8 +52,27 @@ class RingReducer : public CollectiveImplementationInterface {
   void StartAbort(const Status& s);
   void ContinueAfterInputCopy();
   void Finish(bool ok);
+  Status ComputeBinOp(Device* device, OpKernel* op, Tensor* output,
+                      Tensor* input);
   bool RunAsyncParts();
 
+  // Used for executing a sub-operation, e.g. a merge_op instance, with
+  // an OpKernelContext based on the one passed into this Op.
+  class SubContext {
+   public:
+    OpKernelContext::Params sub_params_;
+    gtl::InlinedVector<TensorValue, 4> sub_inputs_;
+    gtl::InlinedVector<AllocatorAttributes, 4> sub_input_attr_;
+    gtl::InlinedVector<DeviceContext*, 4> sub_input_dc_;
+    // Used only for Binary and Unary Ops for which we require
+    // the calculation to be in-place on the first input.
+    int forward_from_ = 0;
+    OpKernelContext* sub_ctx_;
+    SubContext(OpKernelContext* ctx, OpKernelContext::Params* params,
+               OpKernel* op, Tensor* output, Tensor* input);
+    ~SubContext() { delete sub_ctx_; }
+  };
+
   // Current status of a RingField
   enum RingFieldAction {
     RF_INIT = 0,    // Just initialized for a pass
diff --git a/tensorflow/core/distributed_runtime/BUILD b/tensorflow/core/distributed_runtime/BUILD
index 197a3f24d4..e388d3e6f0 100644
--- a/tensorflow/core/distributed_runtime/BUILD
+++ b/tensorflow/core/distributed_runtime/BUILD
@@ -594,7 +594,6 @@ tf_cc_test(
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
-        "//tensorflow/core/kernels:collective_ops",
     ],
 )
 
diff --git a/tensorflow/core/distributed_runtime/collective_param_resolver_distributed.cc b/tensorflow/core/distributed_runtime/collective_param_resolver_distributed.cc
index 9f94a24fcd..1dd10d309b 100644
--- a/tensorflow/core/distributed_runtime/collective_param_resolver_distributed.cc
+++ b/tensorflow/core/distributed_runtime/collective_param_resolver_distributed.cc
@@ -181,10 +181,6 @@ void CollectiveParamResolverDistributed::CompleteInstanceAsync(
                           ir->WaitForOutMu(l);
                           response->set_instance_key(cp->instance.instance_key);
                           response->set_source_rank(ir->source_rank);
-                          if (!cp->instance.communicator_key.empty()) {
-                            response->set_communicator_key(
-                                cp->instance.communicator_key);
-                          }
                           done_and_cleanup(fi_status);
                         } else {
                           done_and_cleanup(fi_status);
@@ -287,10 +283,8 @@ void CollectiveParamResolverDistributed::UpdateInstanceCache(
   using InstanceRecPointer = InstanceRec*;
   InstanceRecPointer* irp = new InstanceRecPointer(nullptr);
   int32 source_rank = resp.source_rank();
-  string communicator_key = resp.communicator_key();
 
-  auto continue_with_ir = [cp, irp, source_rank, communicator_key,
-                           done](const Status& s) {
+  auto continue_with_ir = [this, cp, irp, source_rank, done](const Status& s) {
     if (!s.ok()) {
       done(s);
       delete irp;
@@ -312,19 +306,6 @@ void CollectiveParamResolverDistributed::UpdateInstanceCache(
         }
         ir->source_rank = source_rank;
       }
-      if (ir->communicator_key != communicator_key) {
-        if (!ir->communicator_key.empty()) {
-          ir->status = errors::Internal(
-              "UpdateInstanceCache: CompleteInstanceResponse for instance ",
-              cp->instance.instance_key,
-              " gives communicator_key with size =", communicator_key.size(),
-              " but cache already holds communicator_key with size=",
-              ir->communicator_key.size());
-          status = ir->status;
-          break;
-        }
-        ir->communicator_key = communicator_key;
-      }
       if (ir->known_count < cp->group.group_size) {
         ir->known_count = cp->group.group_size;
         if (ir->known.size() != cp->group.group_size) {
diff --git a/tensorflow/core/distributed_runtime/collective_param_resolver_distributed_test.cc b/tensorflow/core/distributed_runtime/collective_param_resolver_distributed_test.cc
index 823d7d5eb9..40b18d321a 100644
--- a/tensorflow/core/distributed_runtime/collective_param_resolver_distributed_test.cc
+++ b/tensorflow/core/distributed_runtime/collective_param_resolver_distributed_test.cc
@@ -268,8 +268,6 @@ class DeviceResDistTest : public ::testing::Test {
         EXPECT_EQ(cp_[idx].instance.device_names[idx], device_name);
         EXPECT_EQ(cp_[idx].instance.task_names[idx], task_name);
         if (idx > 0) {
-          EXPECT_EQ(cp_[0].instance.communicator_key,
-                    cp_[idx].instance.communicator_key);
           for (int i = 0; i < dev_count; ++i) {
             EXPECT_EQ(cp_[0].instance.device_names[i],
                       cp_[idx].instance.device_names[i]);
diff --git a/tensorflow/core/framework/collective.cc b/tensorflow/core/framework/collective.cc
index b83d183f14..7fa58347f2 100644
--- a/tensorflow/core/framework/collective.cc
+++ b/tensorflow/core/framework/collective.cc
@@ -17,7 +17,6 @@ limitations under the License.
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/hash/hash.h"
-#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 
 namespace tensorflow {
@@ -65,9 +64,7 @@ CollInstanceParams& CollInstanceParams::operator=(
     device_names.assign(other.device_names.begin(), other.device_names.end());
     task_names.assign(other.task_names.begin(), other.task_names.end());
     same_num_devices_per_task = other.same_num_devices_per_task;
-    num_devices_per_task = other.num_devices_per_task;
     gpu_ring_order = other.gpu_ring_order;
-    communicator_key = other.communicator_key;
     impl_details.subdiv_offsets.assign(
         other.impl_details.subdiv_offsets.begin(),
         other.impl_details.subdiv_offsets.end());
@@ -79,7 +76,6 @@ CollInstanceParams& CollInstanceParams::operator=(
     impl_details.subdiv_source_rank.assign(
         other.impl_details.subdiv_source_rank.begin(),
         other.impl_details.subdiv_source_rank.end());
-    impl_details.dependencies = other.impl_details.dependencies;
   }
   return *this;
 }
@@ -95,13 +91,6 @@ string CollInstanceParams::ToString() const {
   for (const auto& n : task_names) {
     strings::StrAppend(&v, n, ", ");
   }
-  strings::StrAppend(&v, "} num_devices_per_task={");
-  for (const auto dpt : num_devices_per_task) {
-    strings::StrAppend(&v, dpt.first, ": ", dpt.second, ", ");
-  }
-  strings::StrAppend(&v, "}, collective_name=", impl_details.collective_name,
-                     ", communicator_key=", str_util::CEscape(communicator_key),
-                     ", subdiv_offsets={");
   strings::StrAppend(&v, "}, subdiv_offsets={");
   for (const auto& d : impl_details.subdiv_offsets) {
     strings::StrAppend(&v, d, ",");
diff --git a/tensorflow/core/framework/collective.h b/tensorflow/core/framework/collective.h
index 546e3938a8..0321429702 100644
--- a/tensorflow/core/framework/collective.h
+++ b/tensorflow/core/framework/collective.h
@@ -70,8 +70,6 @@ struct CollImplDetails {
   std::vector<std::vector<int>> subdiv_permutations;
   std::vector<int> subdiv_offsets;
   std::vector<int> subdiv_source_rank;  // rank of source in each subdiv
-  std::vector<int32>
-      dependencies;  // collective instances on which this node depends
 };
 
 // Data common to all members of a collective instance.
@@ -87,13 +85,9 @@ struct CollInstanceParams {
   std::vector<string> task_names;
   // True if every task has the same number of devices.
   bool same_num_devices_per_task = false;
-  // Task -> number of devices on that task.
-  std::unordered_map<string, int32> num_devices_per_task;
   // If passed in to GPUOptions in ConfigProto, defines a good ring order for
   // GPUs.  Assumes same GPU configuration at each worker.
   string gpu_ring_order = "";
-  // Valid when using a communicator-based collective mechanism, e.g. NCCL.
-  string communicator_key;
   CollImplDetails impl_details;
   string ToString() const;
   CollInstanceParams& operator=(const struct CollInstanceParams& other);
@@ -275,21 +269,6 @@ class CollectiveExecutor : public PeerAccessInterface, public core::RefCounted {
 
   virtual PerStepCollectiveRemoteAccess* remote_access() { return nullptr; }
 
-  // `WaitForDependencies` and `Launched` are used for fine-grained control of
-  // execution order between collective instances.  These functions are intended
-  // to be called in `Run` function of collective implementations, and may be
-  // used to make part, or whole, of the collective execution ordered with
-  // respect to other collective instances.
-  //
-  // `WaitForDependencies` will block until it is safe to continue the callee's
-  // execution, where safety is defined as: ordered with respect to the
-  // collective instances defined in the callee's `wait_for` attribute.
-  virtual void WaitForDependencies(const CollectiveParams& col_params) {}
-  // `Launched` unblocks the dependent collective instances by recording that
-  // this callee device has completed the critical portion of the collective
-  // execution.
-  virtual void Launched(const CollectiveParams& col_params) {}
-
   // Used to designate an invalid group or instance key.
   static int64 kInvalidId;
 
@@ -368,8 +347,7 @@ class CollectiveImplementationInterface {
 
   // Initializes the portions of `col_params` specific to this
   // implementation.  Called exactly once for every Collective instance during
-  // the CollectiveParams resolution process when the graph is first executed,
-  // at the end of `CompleteInstanceLocal()`.
+  // the CollectiveParams resolution process when the graph is first executed.
   // NOTE(ayushd): This is effectively a static function because it modifies the
   // `col_params` passed in and should not manipulate any data members.  However
   // because it is virtual and needs to be implemented by every derived class we
@@ -382,14 +360,6 @@ class CollectiveImplementationInterface {
   // object.
   virtual Status InitializeCollectiveContext(CollectiveContext* col_ctx) = 0;
 
-  // Initializes instance params at the beginning of `CompleteInstanceLocal()`,
-  // unlike `InitializeCollectiveParams` which is called at the end.  This
-  // function is called before all devices in the instance are discovered, and
-  // may be used to broadcast data via the shared `InstanceRec` object in
-  // collective param resolution to all devices.
-  virtual Status InitializeInstanceBeforeGroupDiscovery(
-      CollectiveParams* col_params) = 0;
-
   // Processes and moves data according to the logic of this Collective
   // implementation.  Relies on appropriate initialization of op-specific
   // CollectiveParams in InitializeCollectiveParams(), as well as appropriate
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index ea6343f24d..358d3d656e 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -180,36 +180,12 @@ tf_cc_test(
 
 tf_kernel_library(
     name = "collective_ops",
-    srcs = if_cuda([
-        "collective_nccl_reducer.h",
-        "collective_nccl_reducer.cc",
-    ]),
     prefix = "collective_ops",
     deps = [
         "//tensorflow/core:collective_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:protos_all_cc",
-    ] + if_cuda([
-        "@local_config_nccl//:nccl",
-        "//tensorflow/core/nccl:nccl_lib",
-    ]),
-)
-
-tf_cuda_cc_test(
-    name = "collective_nccl_reducer_test",
-    size = "small",
-    srcs = ["collective_nccl_reducer_test.cc"],
-    tags = tf_cuda_tests_tags() + ["no_cuda_on_cpu_tap"],
-    deps = [
-        "//tensorflow/core:all_kernels",
-        "//tensorflow/core:core_cpu",
-        "//tensorflow/core:framework",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:protos_all_cc",
-        "//tensorflow/core:test",
-        "//tensorflow/core:test_main",
-        "//tensorflow/core:testlib",
     ],
 )
 
diff --git a/tensorflow/core/kernels/collective_nccl_reducer.cc b/tensorflow/core/kernels/collective_nccl_reducer.cc
deleted file mode 100644
index 113f1487f0..0000000000
--- a/tensorflow/core/kernels/collective_nccl_reducer.cc
+++ /dev/null
@@ -1,204 +0,0 @@
-/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#include "tensorflow/core/kernels/collective_nccl_reducer.h"
-
-#ifdef GOOGLE_CUDA
-
-#include "tensorflow/core/common_runtime/collective_util.h"
-#include "tensorflow/core/nccl/nccl_manager.h"
-
-namespace tensorflow {
-namespace {
-string NcclCollectiveKey(const string& exec_key, int step_id) {
-  return strings::StrCat(exec_key, ":", step_id);
-}
-}  // namespace
-
-NcclReducer::NcclReducer() : col_ctx_(nullptr), col_params_(nullptr) {}
-
-Status NcclReducer::InitializeCollectiveParams(CollectiveParams* col_params) {
-  if (col_params->instance.type != REDUCTION_COLLECTIVE ||
-      col_params->instance.impl_details.collective_name != "NcclReduce") {
-    return errors::Internal("Unexpected collective type ",
-                            col_params->instance.type, " expected ",
-                            REDUCTION_COLLECTIVE, "; or collective name ",
-                            col_params->instance.impl_details.collective_name,
-                            " expected NcclReduce");
-  } else {
-    return Status::OK();
-  }
-}
-
-Status NcclReducer::InitializeCollectiveContext(CollectiveContext* col_ctx) {
-  col_ctx_ = col_ctx;
-  col_params_ = &col_ctx->col_params;
-  return collective_util::InitializeDeviceAndLocality(
-      col_ctx->dev_mgr, col_ctx->device_name, &col_ctx->device,
-      &col_ctx->device_locality);
-}
-
-Status NcclReducer::InitializeInstanceBeforeGroupDiscovery(
-    CollectiveParams* col_params) {
-  if (col_params->default_rank == 0 && col_params->group.num_tasks > 1) {
-    col_params->instance.communicator_key =
-        NcclManager::instance()->GenerateCommunicatorKey();
-  }
-  return Status::OK();
-}
-
-Status ReductionOp(const string& merge_op, ncclRedOp_t* reduction_op) {
-  if (merge_op == "Add") {
-    *reduction_op = ncclSum;
-    return Status::OK();
-  } else if (merge_op == "Mul") {
-    *reduction_op = ncclProd;
-    return Status::OK();
-  } else {
-    return errors::Internal("Expected merge_op to be either Add or Mul, found ",
-                            merge_op);
-  }
-}
-
-void NcclReducer::Run(StatusCallback done) {
-  ncclRedOp_t reduction_op;
-  Status s = ReductionOp(col_params_->merge_op->type_string(), &reduction_op);
-  if (!s.ok()) {
-    done(s);
-    return;
-  }
-
-  Tensor group_size;
-  Notification group_size_ready;
-  Status group_size_status;
-  if (col_params_->final_op) {
-    // Create an on-device scalar value from group_size_.
-    // TODO(ayushd, tucker): avoid this copy by either reusing across
-    // invocations or providing the scalar to the kernel in host memory.
-    Tensor group_size_val(col_ctx_->output->dtype(), TensorShape({}));
-    switch (col_ctx_->output->dtype()) {
-      case DT_FLOAT:
-        group_size_val.scalar<float>()() = col_params_->group.group_size;
-        break;
-      case DT_DOUBLE:
-        group_size_val.scalar<double>()() = col_params_->group.group_size;
-        break;
-      case DT_INT32:
-        group_size_val.scalar<int32>()() = col_params_->group.group_size;
-        break;
-      case DT_INT64:
-        group_size_val.scalar<int64>()() = col_params_->group.group_size;
-        break;
-      default:
-        done(errors::Internal("Unsupported type ", col_ctx_->output->dtype()));
-        return;
-    }
-    group_size = Tensor(
-        col_ctx_->device->GetAllocator(col_ctx_->op_ctx->input_alloc_attr(0)),
-        col_ctx_->output->dtype(), TensorShape({}));
-    DeviceContext* op_dev_ctx = col_ctx_->op_ctx->op_device_context();
-    // Enqueue copy on gpu stream.
-    op_dev_ctx->CopyCPUTensorToDevice(
-        &group_size_val, col_ctx_->device, &group_size,
-        [&group_size_ready, &group_size_status](const Status& s) {
-          group_size_status = s;
-          group_size_ready.Notify();
-        });
-  } else {
-    group_size_ready.Notify();
-  }
-
-  Notification nccl_done;
-  Status nccl_status;
-  auto* compute_stream = col_ctx_->op_ctx->op_device_context()->stream();
-  auto* gpu_info = col_ctx_->op_ctx->device()->tensorflow_gpu_device_info();
-  // `AddToAllReduce` performs consistency checks for the NCCL call and enqueues
-  // the `Participant` struct locally.  When all local participants with this
-  // `nccl_collective_key` have called `AddToAllReduce` and
-  // `SignalMultiNodeReady`, all devices at this worker are ready to process
-  // this NCCL op.
-  //
-  // The `NcclManager` uses a dedicated CUDA stream for NCCL kernels.  At this
-  // point, it synchronizes the NCCL stream with the compute stream, and then
-  // enqueues the NCCL kernel on the NCCL stream.
-  const int num_global_devices = col_params_->group.group_size;
-  const int num_local_devices = col_params_->instance.num_devices_per_task.at(
-      col_params_->instance.task_names[col_params_->default_rank]);
-  const string nccl_collective_key =
-      NcclCollectiveKey(col_ctx_->exec_key, col_ctx_->step_id);
-  auto done_callback = [&nccl_done, &nccl_status](const Status& s) {
-    nccl_status = s;
-    nccl_done.Notify();
-  };
-  auto participant = absl::make_unique<NcclManager::Participant>(
-      compute_stream->parent(), compute_stream, gpu_info->event_mgr,
-      gpu_info->gpu_id, col_ctx_->input, col_ctx_->output,
-      col_params_->default_rank, std::move(done_callback));
-  VLOG(1) << "NcclReducer calling NcclManager::AddToAllReduce num_tasks "
-          << col_params_->group.num_tasks << " current task "
-          << col_params_->instance.task_names[col_params_->default_rank]
-          << " num local devices " << num_local_devices
-          << " num global devices " << num_global_devices;
-  NcclManager::instance()->AddToAllReduce(
-      std::move(participant),
-      {nccl_collective_key, num_local_devices, num_global_devices,
-       col_params_->instance.communicator_key},
-      reduction_op);
-
-  // NOTE(ayushd): We need to synchronize NCCL launches across nodes to prevent
-  // deadlocks.  In the current implementation, we define a deterministic
-  // sequential launch order between potentially concurrent collective instances
-  // by introducing control information during static graph analysis in
-  // graph/collective_order.cc.  This can be either in the form of explicit
-  // control edges or via `wait_for` attribute on the collective op.
-  //
-  // The other end of the design spectrum would have a distinguished node
-  // dynamically signal the next collective to launch to all other participants.
-  // This has higher degree of runtime coordination, but it may be able to
-  // achieve better performance if the (arbitrary) static execution order
-  // assigned in the first approach turns out to not be good from a scheduling
-  // perspective.  e.g. consider a graph in which c1, c2, and c3 are three
-  // concurrent collective instances, and the static ordering assigns c1 -> c2
-  // -> c3.  In practice, it could turn out that c3 is always ready to execute
-  // before c1 or c2.
-  //
-  // `WaitForDependencies` may block if the collective instances on which this
-  // op depends have not yet launched.  When this function returns, this op is
-  // ready to go.
-  col_ctx_->col_exec->WaitForDependencies(*col_params_);
-  NcclManager::instance()->SignalMultiNodeReady(nccl_collective_key);
-  // When all devices at this worker have called `SignalMultiNodeReady`, the
-  // `NcclManager` will enqueue the NCCL kernel on the NCCL stream.  Thus the
-  // implementation of `Launched` keeps track of the number of devices that have
-  // launched.
-  col_ctx_->col_exec->Launched(*col_params_);
-
-  // Wait for nccl op and group_size copy to succeed, then do final_op.
-  group_size_ready.WaitForNotification();
-  nccl_done.WaitForNotification();
-  Status final_status =
-      group_size_status.ok() ? nccl_status : group_size_status;
-  if (final_status.ok() && col_params_->final_op) {
-    final_status = collective_util::ComputeBinOp(
-        col_ctx_->op_ctx, col_ctx_->op_params, col_ctx_->device,
-        col_params_->final_op.get(), col_ctx_->output, &group_size);
-  }
-  done(final_status);
-}
-
-REGISTER_COLLECTIVE(NcclReduce, NcclReducer);
-
-}  // namespace tensorflow
-
-#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/collective_nccl_reducer.h b/tensorflow/core/kernels/collective_nccl_reducer.h
deleted file mode 100644
index dc70b280c5..0000000000
--- a/tensorflow/core/kernels/collective_nccl_reducer.h
+++ /dev/null
@@ -1,49 +0,0 @@
-/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#ifndef TENSORFLOW_CORE_KERNELS_COLLECTIVE_NCCL_REDUCER_H_
-#define TENSORFLOW_CORE_KERNELS_COLLECTIVE_NCCL_REDUCER_H_
-
-#include "tensorflow/core/framework/collective.h"
-
-namespace tensorflow {
-#ifdef GOOGLE_CUDA
-
-class NcclReducer : public CollectiveImplementationInterface {
- public:
-  NcclReducer();
-  ~NcclReducer() override = default;
-
-  // No-op for this collective implementation.
-  Status InitializeCollectiveParams(CollectiveParams* col_params) override;
-
-  // Initializes the device objects and device localities.
-  Status InitializeCollectiveContext(CollectiveContext* col_ctx) override;
-
-  // Initialize nccl communicator key.
-  Status InitializeInstanceBeforeGroupDiscovery(
-      CollectiveParams* col_params) override;
-
-  // Hands off all reduce to NcclManager.
-  void Run(StatusCallback done) override;
-
- private:
-  CollectiveContext* col_ctx_;          // Not owned
-  const CollectiveParams* col_params_;  // Not owned
-};
-
-#endif  // GOOGLE_CUDA
-}  // namespace tensorflow
-
-#endif  // TENSORFLOW_CORE_KERNELS_COLLECTIVE_NCCL_REDUCER_H_
diff --git a/tensorflow/core/kernels/collective_nccl_reducer_test.cc b/tensorflow/core/kernels/collective_nccl_reducer_test.cc
deleted file mode 100644
index ff33c1322c..0000000000
--- a/tensorflow/core/kernels/collective_nccl_reducer_test.cc
+++ /dev/null
@@ -1,331 +0,0 @@
-/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#include "tensorflow/core/kernels/collective_nccl_reducer.h"
-
-#include <algorithm>
-#include "absl/memory/memory.h"
-#include "tensorflow/core/common_runtime/base_collective_executor.h"
-#include "tensorflow/core/common_runtime/device.h"
-#include "tensorflow/core/common_runtime/device_factory.h"
-#include "tensorflow/core/common_runtime/device_mgr.h"
-#include "tensorflow/core/common_runtime/device_resolver_local.h"
-#include "tensorflow/core/common_runtime/process_util.h"
-#include "tensorflow/core/common_runtime/test_collective_executor_mgr.h"
-#include "tensorflow/core/framework/collective.h"
-#include "tensorflow/core/framework/fake_input.h"
-#include "tensorflow/core/framework/node_def_builder.h"
-#include "tensorflow/core/framework/op_kernel.h"
-#include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/lib/core/notification.h"
-#include "tensorflow/core/lib/core/status_test_util.h"
-#include "tensorflow/core/platform/test.h"
-#include "tensorflow/core/public/session_options.h"
-#include "tensorflow/core/public/version.h"
-
-#ifdef GOOGLE_CUDA
-
-namespace tensorflow {
-static constexpr int kStepId = 10;
-
-std::unique_ptr<OpKernel> GetKernel(const NodeDef& node, DeviceBase* device) {
-  Status status;
-  std::unique_ptr<OpKernel> k = CreateOpKernel(
-      DEVICE_GPU, device, device->GetAllocator(AllocatorAttributes()), node,
-      TF_GRAPH_DEF_VERSION, &status);
-  if (!status.ok()) LOG(FATAL) << status;
-  return k;
-}
-
-std::unique_ptr<OpKernel> GetAdd(DeviceBase* device) {
-  NodeDef node_def;
-  NodeDefBuilder builder("add_node", "Add");
-  TF_CHECK_OK(builder.Attr("T", DT_FLOAT)
-                  .Input(FakeInput(DT_FLOAT))
-                  .Input(FakeInput(DT_FLOAT))
-                  .Finalize(&node_def));
-  return GetKernel(node_def, device);
-}
-
-std::unique_ptr<OpKernel> GetDiv(DeviceBase* device) {
-  NodeDef node_def;
-  NodeDefBuilder builder("add_node", "Div");
-  TF_CHECK_OK(builder.Attr("T", DT_FLOAT)
-                  .Input(FakeInput(DT_FLOAT))
-                  .Input(FakeInput(DT_FLOAT))
-                  .Finalize(&node_def));
-  return GetKernel(node_def, device);
-}
-
-class NcclReducerTest : public ::testing::Test {
- protected:
-  ~NcclReducerTest() override {
-    if (col_exec_) col_exec_->Unref();
-  }
-
-  void InitGPUDevices() {
-    std::vector<std::unique_ptr<Device>> all_devices;
-    SessionOptions session_options;
-    session_options.config.mutable_gpu_options()
-        ->set_per_process_gpu_memory_fraction(0.1);
-    session_options.env = Env::Default();
-    Status s = DeviceFactory::GetFactory(DEVICE_GPU)
-                   ->AddDevices(session_options, "", &all_devices);
-    TF_CHECK_OK(s);
-    for (std::unique_ptr<Device>& d : all_devices) {
-      if (d->device_type() == "GPU") {
-        gpus_.emplace_back(std::move(d));
-      }
-    }
-  }
-
-  void Init(int num_ranks) {
-    setenv("NCCL_DEBUG", "INFO", 1 /* replace */);
-    setenv("NCCL_LAUNCH_MODE", "PARALLEL", 1 /* replace */);
-    InitGPUDevices();
-    std::vector<std::unique_ptr<Device>> local_devices;
-    std::vector<string> device_names;
-    for (int rank = 0; rank < num_ranks; ++rank) {
-      if (rank < gpus_.size()) {
-        local_devices.emplace_back(std::move(gpus_[rank]));
-      }
-    }
-    int num_gpus = local_devices.size();
-    for (const auto& device : local_devices) {
-      device_names.push_back(device->name());
-      VLOG(2) << device->name();
-    }
-    if (!dev_mgr_) dev_mgr_.reset(new DeviceMgr(std::move(local_devices)));
-    col_exec_ = new BaseCollectiveExecutor(
-        &col_exec_mgr_, /*remote_access=*/nullptr, kStepId, dev_mgr_.get(),
-        /*gpu_ring_order=*/nullptr);
-
-    // Initialize collective params.
-    col_params_.name = "test_nccl_collective_op";
-    const int group_key = 5;
-    col_params_.group.group_key = group_key;
-    col_params_.group.device_type = DEVICE_GPU;
-    col_params_.group.group_size = num_ranks;
-    const int instance_key = 23;
-    col_params_.instance.instance_key = instance_key;
-    col_params_.instance.type = REDUCTION_COLLECTIVE;
-    col_params_.instance.data_type = DT_FLOAT;
-    col_params_.instance.impl_details.collective_name = "NcclReduce";
-    const string task_name = "/job:worker/replica:0/task:0";
-    col_params_.instance.num_devices_per_task[task_name] = num_ranks;
-    for (int rank = 0; rank < num_ranks; ++rank) {
-      col_params_.instance.device_names.push_back(
-          device_names[rank % num_gpus]);
-      col_params_.instance.task_names.push_back(task_name);
-    }
-    for (int rank = 0; rank < num_ranks; ++rank) {
-      instances_.push_back(absl::make_unique<DeviceInstance>(
-          rank, col_params_.instance.device_names[rank], this));
-    }
-  }
-
-  void Reduce() {
-    int done = 0;
-    mutex done_mu;
-    condition_variable done_cv;
-    for (const auto& instance : instances_) {
-      DeviceInstance* di = instance.get();
-      SchedClosure([di, &done, &done_mu, &done_cv] {
-        di->DoReduce();
-        mutex_lock l(done_mu);
-        ++done;
-        done_cv.notify_all();
-      });
-    }
-
-    mutex_lock l(done_mu);
-    while (done < instances_.size()) done_cv.wait(l);
-  }
-
-  void RunTest(int num_ranks, int tensor_length) {
-    Init(num_ranks);
-    std::vector<float> expected(tensor_length, 0.0);
-    for (int rank = 0; rank < num_ranks; ++rank) {
-      DeviceInstance* instance = instances_[rank].get();
-      instance->InitTensor(DT_FLOAT, TensorShape({tensor_length}),
-                           [&expected, rank](Tensor* t) {
-                             for (size_t i = 0; i < t->NumElements(); ++i) {
-                               float value = pow(10, rank) * i;
-                               t->flat<float>()(i) = value;
-                               expected[i] += value;
-                             }
-                           });
-    }
-    Reduce();
-    // Confirm that every rank computed the same correct value.
-    for (int i = 0; i < tensor_length; ++i) {
-      expected[i] /= num_ranks;
-    }
-    for (int rank = 0; rank < instances_.size(); ++rank) {
-      TF_ASSERT_OK(instances_[rank]->status_);
-      Tensor* dev_tensor = &instances_[rank]->tensor_;
-      Tensor actual(DT_FLOAT, TensorShape({tensor_length}));
-      Notification note;
-      Device* dev = instances_[rank]->device_;
-      auto* dev_info = dev->tensorflow_gpu_device_info();
-      dev_info->default_context->CopyDeviceTensorToCPU(
-          dev_tensor, /*tensor_name=*/"", dev, &actual,
-          [&note](const Status&) { note.Notify(); });
-      note.WaitForNotification();
-      for (int i = 0; i < tensor_length; ++i) {
-        EXPECT_FLOAT_EQ(expected[i], actual.template flat<float>()(i))
-            << "Mismatch at rank " << rank << " index " << i;
-      }
-    }
-  }
-
-  std::unique_ptr<OpKernel> GetCollectiveReduce(const CollectiveParams& params,
-                                                Tensor* input,
-                                                DeviceBase* device) {
-    mutex_lock l(mu_);
-    NodeDef node_def;
-    NodeDefBuilder builder(
-        strings::StrCat("collective_reduce_", reduce_counter_++),
-        "CollectiveReduce");
-    TF_CHECK_OK(
-        builder.Attr("T", params.instance.data_type)
-            .Attr("merge_op", "Add")
-            .Attr("final_op", "Div")
-            .Attr("group_size", params.group.group_size)
-            .Attr("group_key", params.group.group_key)
-            .Attr("instance_key", params.instance.instance_key)
-            .Attr("subdiv_offsets", params.instance.impl_details.subdiv_offsets)
-            .Input(FakeInput(params.instance.data_type))
-            .Finalize(&node_def));
-    return GetKernel(node_def, device);
-  }
-
-  class DeviceInstance {
-   public:
-    DeviceInstance(int rank, const string& device_name, NcclReducerTest* parent)
-        : parent_(parent), device_name_(device_name), rank_(rank) {
-      TF_CHECK_OK(parent_->dev_mgr_->LookupDevice(device_name_, &device_))
-          << "Could not find device " << device_name_ << " existing devices "
-          << parent_->dev_mgr_->DebugString();
-      col_params_.name = parent_->col_params_.name;
-      col_params_.default_rank = rank;
-      col_params_.group.group_key = parent_->col_params_.group.group_key;
-      col_params_.group.device_type = parent_->col_params_.group.device_type;
-      col_params_.group.group_size = parent_->col_params_.group.group_size;
-      col_params_.instance = parent->col_params_.instance;
-    }
-
-    void InitTensor(DataType dtype, const TensorShape& shape,
-                    const std::function<void(Tensor*)>& init_f) {
-      tensor_ =
-          Tensor(device_->GetAllocator(AllocatorAttributes()), dtype, shape);
-      Tensor cpu_tensor(dtype, shape);
-      init_f(&cpu_tensor);
-      VLOG(2) << "cpu_tensor " << cpu_tensor.DebugString();
-      auto* dev_info = device_->tensorflow_gpu_device_info();
-      Notification note;
-      dev_info->default_context->CopyCPUTensorToDevice(
-          &cpu_tensor, device_, &tensor_,
-          [&note](const Status&) { note.Notify(); });
-      note.WaitForNotification();
-    }
-
-    void DoReduce() {
-      col_params_.merge_op = GetAdd(device_);
-      col_params_.final_op = GetDiv(device_);
-
-      // Prepare an OpKernelContext.
-      OpKernelContext::Params op_params;
-      op_params.step_id = kStepId;
-      op_params.device = device_;
-      gtl::InlinedVector<TensorValue, 4> inputs;
-      inputs.push_back(TensorValue(&tensor_));
-      op_params.inputs = &inputs;
-      gtl::InlinedVector<AllocatorAttributes, 4> input_aa(
-          {AllocatorAttributes()});
-      op_params.input_alloc_attrs = &input_aa;
-      gtl::InlinedVector<DeviceContext*, 4> input_dc;
-      DeviceContext* dev_ctx = nullptr;
-      auto* dev_info = device_->tensorflow_gpu_device_info();
-      if (dev_info) {
-        dev_ctx = dev_info->default_context;
-        dev_ctx->Ref();
-      } else {
-        dev_ctx = new DeviceContext;
-      }
-      input_dc.push_back(dev_ctx);
-      op_params.input_device_contexts = &input_dc;
-      op_params.op_device_context = dev_ctx;
-      int forward_from = 0;
-      op_params.forward_from_array = &forward_from;
-      AllocatorAttributes generic_alloc_attr;
-      op_params.output_attr_array = &generic_alloc_attr;
-      std::unique_ptr<OpKernel> op =
-          parent_->GetCollectiveReduce(col_params_, &tensor_, device_);
-      op_params.op_kernel = op.get();
-      OpKernelContext ctx(&op_params, 1);
-
-      // We never actually execute the kernel, so we need to do the output
-      // allocation it would do, ourselves.
-      Tensor* output_tensor_ptr = nullptr;
-      TF_CHECK_OK(ctx.forward_input_or_allocate_output({0}, 0, tensor_.shape(),
-                                                       &output_tensor_ptr));
-      CHECK_EQ(output_tensor_ptr, ctx.mutable_output(0));
-
-      // Prepare a NcclReducer instance.
-      string exec_key =
-          strings::StrCat(col_params_.instance.instance_key, ":0:0");
-      NcclReducer reducer;
-      CollectiveContext col_ctx(parent_->col_exec_, parent_->dev_mgr_.get(),
-                                &ctx, &op_params, col_params_, exec_key,
-                                kStepId, &tensor_, &tensor_);
-      TF_CHECK_OK(reducer.InitializeCollectiveContext(&col_ctx));
-
-      // Run the all-reduce.
-      reducer.Run([this](Status s) { status_ = s; });
-      if (status_.ok()) {
-        CHECK(tensor_.CopyFrom(*ctx.mutable_output(0), tensor_.shape()));
-      }
-
-      dev_ctx->Unref();
-    }
-
-    NcclReducerTest* parent_;
-    string device_name_;
-    int rank_;
-    Tensor tensor_;
-    Device* device_;
-    CollectiveParams col_params_;
-    Status status_;
-  };
-
-  std::vector<std::unique_ptr<tensorflow::Device>> gpus_;
-  TestCollectiveExecutorMgr col_exec_mgr_;
-  CollectiveExecutor* col_exec_;
-  std::unique_ptr<DeviceMgr> dev_mgr_;
-  std::vector<std::unique_ptr<DeviceInstance>> instances_;
-  CollectiveParams col_params_;
-  mutex mu_;
-  int32 reduce_counter_ GUARDED_BY(mu_) = 0;
-};
-
-TEST_F(NcclReducerTest, Test2Dev16Len) { RunTest(2, 16); }
-TEST_F(NcclReducerTest, Test4Dev16Len) { RunTest(4, 16); }
-TEST_F(NcclReducerTest, Test8Dev16Len) { RunTest(8, 16); }
-TEST_F(NcclReducerTest, Test8Dev128Len) { RunTest(8, 128); }
-TEST_F(NcclReducerTest, Test8Dev1045991Len) { RunTest(8, 1048576); }
-
-}  // namespace tensorflow
-
-#endif
diff --git a/tensorflow/core/kernels/collective_ops.cc b/tensorflow/core/kernels/collective_ops.cc
index ea582836c2..82e2913b64 100644
--- a/tensorflow/core/kernels/collective_ops.cc
+++ b/tensorflow/core/kernels/collective_ops.cc
@@ -87,9 +87,6 @@ class CollectiveReduceOpKernel : public CollectiveOpKernel {
                     "final_op must be one of {\"Id\", \"Div\"} but got ",
                     final_op_name));
     OP_REQUIRES_OK(c, c->GetAttr("T", &col_params_.instance.data_type));
-    OP_REQUIRES_OK(c,
-                   c->GetAttr("wait_for",
-                              &col_params_.instance.impl_details.dependencies));
 
     const NodeDef& real_node = c->def();
     col_params_.name = strings::StrCat(real_node.name(), ": Reduce(",
diff --git a/tensorflow/core/nccl/BUILD b/tensorflow/core/nccl/BUILD
index a19e1af888..4be33b2a0c 100644
--- a/tensorflow/core/nccl/BUILD
+++ b/tensorflow/core/nccl/BUILD
@@ -20,10 +20,8 @@ cc_library(
     name = "nccl_lib",
     srcs = if_cuda([
         "nccl_manager.cc",
-        "nccl_rewrite.cc",
-    ]),
-    hdrs = if_cuda([
         "nccl_manager.h",
+        "nccl_rewrite.cc",
     ]),
     copts = tf_copts(),
     deps = if_cuda([
diff --git a/tensorflow/core/protobuf/worker.proto b/tensorflow/core/protobuf/worker.proto
index 4284dd119e..74058c8465 100644
--- a/tensorflow/core/protobuf/worker.proto
+++ b/tensorflow/core/protobuf/worker.proto
@@ -535,7 +535,6 @@ message CompleteInstanceRequest {
 message CompleteInstanceResponse {
   int32 instance_key = 1;
   int32 source_rank = 2;
-  bytes communicator_key = 3;
 }
 
 // Request for next agreed-upon step_id for the specified graph_keys.
-- 
GitLab


From bdfdaea03edaa8c91cb9d9606e283afed38eaf23 Mon Sep 17 00:00:00 2001
From: Trevor Morris <tmorris@nvidia.com>
Date: Tue, 15 Jan 2019 12:46:18 -0800
Subject: [PATCH 0762/1095] Fix buildifier check

---
 tensorflow/contrib/tensorrt/BUILD | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index 17717c8403..58c188b314 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -221,8 +221,8 @@ tf_cuda_library(
     ],
     hdrs = [
         "resources/trt_int8_calibrator.h",
-        "resources/trt_resource_manager.h",
         "resources/trt_lru_cache.h",
+        "resources/trt_resource_manager.h",
         "resources/trt_resources.h",
     ],
     deps = [
@@ -568,7 +568,7 @@ cc_library(
     copts = tf_copts(),
     deps = [
         "//tensorflow/core:lib",
-        "//tensorflow/core:framework"
+        "//tensorflow/core:framework",
     ],
 )
 
-- 
GitLab


From cee860e07b3b3daaefb7628f8b31748aed3aaf05 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Tue, 15 Jan 2019 12:21:35 -0800
Subject: [PATCH 0763/1095] Remove the old mnist_keras example

PiperOrigin-RevId: 229417867
---
 tensorflow/contrib/tpu/BUILD | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tensorflow/contrib/tpu/BUILD b/tensorflow/contrib/tpu/BUILD
index 0fe9d0cdfc..d898f05405 100644
--- a/tensorflow/contrib/tpu/BUILD
+++ b/tensorflow/contrib/tpu/BUILD
@@ -263,7 +263,6 @@ py_library(
         "//learning/brain:__subpackages__",
         "//tensorflow:__subpackages__",
         "//third_party/cloud_tpu/models/keras_colab:__subpackages__",
-        "//third_party/cloud_tpu/models/mnist_keras:__subpackages__",
         "//third_party/cloud_tpu/models/resnet50_keras:__subpackages__",
     ],
     deps = [
-- 
GitLab


From 97aa5daaa5e2b7728d8b2a19695a39884fce0147 Mon Sep 17 00:00:00 2001
From: Martin Wicke <wicke@google.com>
Date: Tue, 15 Jan 2019 12:43:06 -0800
Subject: [PATCH 0764/1095] Increase version of google_pasta to 0.1.1 (for Byte
 string fix).

PiperOrigin-RevId: 229421387
---
 tensorflow/tools/pip_package/setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py
index 55b7046e30..6e182a5e5c 100644
--- a/tensorflow/tools/pip_package/setup.py
+++ b/tensorflow/tools/pip_package/setup.py
@@ -51,7 +51,7 @@ REQUIRED_PACKAGES = [
     'absl-py >= 0.1.6',
     'astor >= 0.6.0',
     'gast >= 0.2.0',
-    'google_pasta >= 0.1.0',
+    'google_pasta >= 0.1.1',
     'keras_applications >= 1.0.6',
     'keras_preprocessing >= 1.0.5',
     'numpy >= 1.13.3',
-- 
GitLab


From 8d8fee60bd2215858f574b27dcfd07aac4473860 Mon Sep 17 00:00:00 2001
From: Haoyu Zhang <haoyuzhang@google.com>
Date: Tue, 15 Jan 2019 12:48:04 -0800
Subject: [PATCH 0765/1095] Add fp16 support to color distortion ops for better
 performance on GPU. These ops are typically used for image preprocessing in
 data input pipeline. When the input pipeline gets complicated and becomes the
 bottleneck (for example, in the SSD (Single Shot multibox Detector) model),
 offloading partial image preprocessing to accelerators can be useful to
 alleviate CPU workload. Adding fp16 support can further accelerate the
 processing on GPU and allow the model to have larger batch sizes.

The ops that are affected in this CL are:
  tf.image.random_brightness
  tf.image.random_contrast
  tf.image.random_saturation
  tf.image.random_hue
  tf.image.adjust_brightness
  tf.image.adjust_contrast
  tf.image.adjust_saturation
  tf.image.adjust_hue

PiperOrigin-RevId: 229422273
---
 .../compiler/tf2xla/kernels/image_ops.cc      | 54 ++++++-----
 tensorflow/core/kernels/adjust_contrast_op.cc | 59 +++++++----
 tensorflow/core/kernels/adjust_contrast_op.h  | 24 +++--
 .../core/kernels/adjust_contrast_op_gpu.cu.cc |  3 +-
 tensorflow/core/kernels/adjust_hsv_gpu.cu.h   | 17 ++--
 tensorflow/core/kernels/adjust_hue_op.cc      | 32 +++---
 tensorflow/core/kernels/adjust_hue_op.h       |  5 +-
 .../core/kernels/adjust_hue_op_gpu.cu.cc      | 14 ++-
 .../core/kernels/adjust_saturation_op.cc      | 32 +++---
 .../core/kernels/adjust_saturation_op.h       |  5 +-
 .../kernels/adjust_saturation_op_gpu.cu.cc    | 17 ++--
 tensorflow/core/ops/image_ops.cc              | 15 +--
 tensorflow/python/ops/image_ops_impl.py       | 97 +++++++++++--------
 tensorflow/python/ops/image_ops_test.py       | 56 +++--------
 14 files changed, 240 insertions(+), 190 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/image_ops.cc b/tensorflow/compiler/tf2xla/kernels/image_ops.cc
index a31b5a2cfd..92b20fe0ba 100644
--- a/tensorflow/compiler/tf2xla/kernels/image_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/image_ops.cc
@@ -26,6 +26,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/core/framework/tensor_shape.h"
+#include "tensorflow/core/framework/types.pb.h"
 
 namespace tensorflow {
 namespace {
@@ -186,19 +187,20 @@ class AdjustContrastOpV2 : public XlaOpKernel {
                                         factor_shape.DebugString()));
 
     xla::XlaBuilder* b = context->builder();
-    xla::XlaOp input = context->Input(0);
-    xla::XlaOp factor = context->Input(1);
-
     DataType type = context->input_type(0);
 
+    xla::XlaOp input = context->Input(0);
+    xla::XlaOp factor = XlaHelpers::ConvertElementType(context->Input(1), type);
+
     const DataType accumulation_type = XlaHelpers::SumAccumulationType(type);
     auto converted = XlaHelpers::ConvertElementType(input, accumulation_type);
     auto reduce = xla::Reduce(converted, XlaHelpers::Zero(b, accumulation_type),
                               *context->GetOrCreateAdd(accumulation_type),
                               {height_dim, width_dim});
-    auto output = XlaHelpers::ConvertElementType(reduce, type);
-    output =
-        xla::Div(output, XlaHelpers::FloatLiteral(b, type, height * width));
+
+    auto output = xla::Div(
+        reduce, XlaHelpers::FloatLiteral(b, accumulation_type, height * width));
+    output = XlaHelpers::ConvertElementType(output, type);
 
     std::vector<int64> broadcast_dims(input_shape.dims() - 2);
     std::iota(broadcast_dims.begin(), broadcast_dims.end(), 0);
@@ -234,8 +236,10 @@ class AdjustSaturationOp : public XlaOpKernel {
                                 channels, " channels."));
 
     xla::XlaBuilder* b = context->builder();
-    xla::XlaOp input = context->Input(0);
-    xla::XlaOp scale = context->Input(1);
+    xla::XlaOp input =
+        XlaHelpers::ConvertElementType(context->Input(0), DT_FLOAT);
+    xla::XlaOp scale =
+        XlaHelpers::ConvertElementType(context->Input(1), DT_FLOAT);
 
     DataType type = context->input_type(0);
 
@@ -250,15 +254,17 @@ class AdjustSaturationOp : public XlaOpKernel {
                                       /*dimno=*/channel_dim);
     TensorShape channel_shape = input_shape;
     channel_shape.set_dim(channel_dim, 1);
-    auto hsv = RGBToHSV(context, b, {red, green, blue}, context->input_type(0),
-                        channel_shape);
+    auto hsv =
+        RGBToHSV(context, b, {red, green, blue}, DT_FLOAT, channel_shape);
 
-    hsv[1] = xla::Clamp(XlaHelpers::Zero(b, type), xla::Mul(hsv[1], scale),
-                        XlaHelpers::One(b, type));
+    hsv[1] = xla::Clamp(XlaHelpers::Zero(b, DT_FLOAT), xla::Mul(hsv[1], scale),
+                        XlaHelpers::One(b, DT_FLOAT));
 
-    auto rgb = HSVToRGB(context->builder(), hsv, context->input_type(0));
+    auto rgb = HSVToRGB(context->builder(), hsv, DT_FLOAT);
 
-    context->SetOutput(0, xla::ConcatInDim(b, rgb, channel_dim));
+    auto output = XlaHelpers::ConvertElementType(
+        xla::ConcatInDim(b, rgb, channel_dim), type);
+    context->SetOutput(0, output);
   }
 };
 REGISTER_XLA_OP(Name("AdjustSaturation"), AdjustSaturationOp);
@@ -284,8 +290,10 @@ class AdjustHueOp : public XlaOpKernel {
                                 channels, " channels."));
 
     xla::XlaBuilder* b = context->builder();
-    xla::XlaOp input = context->Input(0);
-    xla::XlaOp delta = context->Input(1);
+    xla::XlaOp input =
+        XlaHelpers::ConvertElementType(context->Input(0), DT_FLOAT);
+    xla::XlaOp delta =
+        XlaHelpers::ConvertElementType(context->Input(1), DT_FLOAT);
 
     DataType type = context->input_type(0);
 
@@ -300,20 +308,22 @@ class AdjustHueOp : public XlaOpKernel {
                                       /*dimno=*/channel_dim);
     TensorShape channel_shape = input_shape;
     channel_shape.set_dim(channel_dim, 1);
-    auto hsv = RGBToHSV(context, b, {red, green, blue}, context->input_type(0),
-                        channel_shape);
+    auto hsv =
+        RGBToHSV(context, b, {red, green, blue}, DT_FLOAT, channel_shape);
 
-    auto zero = XlaHelpers::Zero(b, type);
-    auto one = XlaHelpers::One(b, type);
+    auto zero = XlaHelpers::Zero(b, DT_FLOAT);
+    auto one = XlaHelpers::One(b, DT_FLOAT);
 
     auto& hue = hsv[0];
     hue = xla::Rem(xla::Add(hsv[0], delta), one);
     hue =
         xla::Select(xla::Lt(hue, zero), xla::Rem(xla::Add(one, hue), one), hue);
 
-    auto rgb = HSVToRGB(context->builder(), hsv, context->input_type(0));
+    auto rgb = HSVToRGB(context->builder(), hsv, DT_FLOAT);
 
-    context->SetOutput(0, xla::ConcatInDim(b, rgb, channel_dim));
+    auto output = XlaHelpers::ConvertElementType(
+        xla::ConcatInDim(b, rgb, channel_dim), type);
+    context->SetOutput(0, output);
   }
 };
 REGISTER_XLA_OP(Name("AdjustHue"), AdjustHueOp);
diff --git a/tensorflow/core/kernels/adjust_contrast_op.cc b/tensorflow/core/kernels/adjust_contrast_op.cc
index 47e10f56df..1aef0060b0 100644
--- a/tensorflow/core/kernels/adjust_contrast_op.cc
+++ b/tensorflow/core/kernels/adjust_contrast_op.cc
@@ -189,11 +189,11 @@ class AdjustContrastOpV2Base : public OpKernel {
                          const ComputeOptions& options) = 0;
 };
 
-template <typename Device>
+template <typename Device, typename T>
 class AdjustContrastOpv2;
 
 template <>
-class AdjustContrastOpv2<CPUDevice> : public AdjustContrastOpV2Base {
+class AdjustContrastOpv2<CPUDevice, float> : public AdjustContrastOpV2Base {
  public:
   explicit AdjustContrastOpv2(OpKernelConstruction* context)
       : AdjustContrastOpV2Base(context) {}
@@ -378,23 +378,32 @@ class AdjustContrastOpv2<CPUDevice> : public AdjustContrastOpV2Base {
   }
 };
 
-REGISTER_KERNEL_BUILDER(Name("AdjustContrastv2").Device(DEVICE_CPU),
-                        AdjustContrastOpv2<CPUDevice>);
+REGISTER_KERNEL_BUILDER(
+    Name("AdjustContrastv2").Device(DEVICE_CPU).TypeConstraint<float>("T"),
+    AdjustContrastOpv2<CPUDevice, float>);
 
 #if GOOGLE_CUDA
 // Forward declarations of the function specializations for GPU (to prevent
 // building the GPU versions here, they will be built compiling _gpu.cu.cc).
 namespace functor {
-template <>
-void AdjustContrastv2<GPUDevice>::operator()(
-    const GPUDevice& d, typename TTypes<float, 4>::ConstTensor input,
-    typename TTypes<float>::ConstScalar contrast_factor,
-    typename TTypes<float, 4>::Tensor output);
-extern template struct AdjustContrastv2<GPUDevice>;
+
+#define DECLARE_GPU_SPEC(T)                                         \
+  template <>                                                       \
+  void AdjustContrastv2<GPUDevice, T>::operator()(                  \
+      const GPUDevice& d, typename TTypes<T, 4>::ConstTensor input, \
+      typename TTypes<float>::ConstScalar contrast_factor,          \
+      typename TTypes<T, 4>::Tensor output);                        \
+  extern template struct AdjustContrastv2<GPUDevice, T>;
+
+DECLARE_GPU_SPEC(float);
+DECLARE_GPU_SPEC(Eigen::half);
+
+#undef DECLARE_GPU_SPEC
+
 }  // namespace functor
 
-template <>
-class AdjustContrastOpv2<GPUDevice> : public AdjustContrastOpV2Base {
+template <typename T>
+class AdjustContrastOpv2<GPUDevice, T> : public AdjustContrastOpV2Base {
  public:
   explicit AdjustContrastOpv2(OpKernelConstruction* context)
       : AdjustContrastOpV2Base(context) {}
@@ -403,20 +412,27 @@ class AdjustContrastOpv2<GPUDevice> : public AdjustContrastOpV2Base {
                  const ComputeOptions& options) override {
     const int64 shape[4] = {options.batch, options.height, options.width,
                             options.channels};
-    functor::AdjustContrastv2<GPUDevice>()(
-        context->eigen_device<GPUDevice>(),
-        options.input->shaped<float, 4>(shape), options.factor->scalar<float>(),
-        options.output->shaped<float, 4>(shape));
+    functor::AdjustContrastv2<GPUDevice, T>()(
+        context->eigen_device<GPUDevice>(), options.input->shaped<T, 4>(shape),
+        options.factor->scalar<float>(), options.output->shaped<T, 4>(shape));
   }
 };
 
-REGISTER_KERNEL_BUILDER(Name("AdjustContrastv2").Device(DEVICE_GPU),
-                        AdjustContrastOpv2<GPUDevice>);
+#define REGISTER_GPU(T)                                                   \
+  REGISTER_KERNEL_BUILDER(                                                \
+      Name("AdjustContrastv2").Device(DEVICE_GPU).TypeConstraint<T>("T"), \
+      AdjustContrastOpv2<GPUDevice, T>);
+
+REGISTER_GPU(float)
+REGISTER_GPU(Eigen::half)
+
+#undef REGISTER_GPU
+
 #endif  // GOOGLE_CUDA
 
 #ifdef TENSORFLOW_USE_SYCL
 template <>
-class AdjustContrastOpv2<SYCLDevice> : public AdjustContrastOpV2Base {
+class AdjustContrastOpv2<SYCLDevice, float> : public AdjustContrastOpV2Base {
  public:
   explicit AdjustContrastOpv2(OpKernelConstruction* context)
       : AdjustContrastOpV2Base(context) {}
@@ -431,8 +447,9 @@ class AdjustContrastOpv2<SYCLDevice> : public AdjustContrastOpV2Base {
         options.output->shaped<float, 4>(shape));
   }
 };
-REGISTER_KERNEL_BUILDER(Name("AdjustContrastv2").Device(DEVICE_SYCL),
-                        AdjustContrastOpv2<SYCLDevice>);
+REGISTER_KERNEL_BUILDER(
+    Name("AdjustContrastv2").Device(DEVICE_SYCL).TypeConstraint<float>("T"),
+    AdjustContrastOpv2<SYCLDevice, float>);
 #endif  // TENSORFLOW_USE_SYCL
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/adjust_contrast_op.h b/tensorflow/core/kernels/adjust_contrast_op.h
index f4a53c2ef9..3e501bccee 100644
--- a/tensorflow/core/kernels/adjust_contrast_op.h
+++ b/tensorflow/core/kernels/adjust_contrast_op.h
@@ -87,11 +87,11 @@ struct AdjustContrast {
 };
 
 // Functor used by AdjustContrastOpv2 to do the computations.
-template <typename Device>
+template <typename Device, typename T>
 struct AdjustContrastv2 {
-  void operator()(const Device& d, typename TTypes<float, 4>::ConstTensor input,
+  void operator()(const Device& d, typename TTypes<T, 4>::ConstTensor input,
                   typename TTypes<float>::ConstScalar contrast_factor,
-                  typename TTypes<float, 4>::Tensor output) {
+                  typename TTypes<T, 4>::Tensor output) {
     const int batch = input.dimension(0);
     const int height = input.dimension(1);
     const int width = input.dimension(2);
@@ -138,15 +138,19 @@ struct AdjustContrastv2 {
 #endif
     Eigen::Sizes<1, 1, 1, 1> scalar;
     float num_reduced_coeffs = height * width;
-    output.device(d) =
-        (input.shuffle(reduced_dims_first).sum(reduction_axis).eval() /
-         num_reduced_coeffs)
-            .reshape(reshape_dims)
-            .broadcast(broadcast_dims);
+    output.device(d) = (input.template cast<float>()
+                            .shuffle(reduced_dims_first)
+                            .sum(reduction_axis)
+                            .eval() /
+                        num_reduced_coeffs)
+                           .template cast<T>()
+                           .reshape(reshape_dims)
+                           .broadcast(broadcast_dims);
     auto contrast_factor_tensor =
         contrast_factor.reshape(scalar).broadcast(scalar_broadcast);
-    auto adjusted = (input - output) * contrast_factor_tensor;
-    output.device(d) += adjusted;
+    auto adjusted =
+        (input - output).template cast<float>() * contrast_factor_tensor;
+    output.device(d) += adjusted.template cast<T>();
   }
 };
 
diff --git a/tensorflow/core/kernels/adjust_contrast_op_gpu.cu.cc b/tensorflow/core/kernels/adjust_contrast_op_gpu.cu.cc
index a451bfe29c..1a1c2a4e1e 100644
--- a/tensorflow/core/kernels/adjust_contrast_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/adjust_contrast_op_gpu.cu.cc
@@ -26,7 +26,8 @@ namespace tensorflow {
 typedef Eigen::GpuDevice GPUDevice;
 
 // this is for v2
-template struct functor::AdjustContrastv2<GPUDevice>;
+template struct functor::AdjustContrastv2<GPUDevice, float>;
+template struct functor::AdjustContrastv2<GPUDevice, Eigen::half>;
 
 // these are for v1
 template struct functor::AdjustContrast<GPUDevice, uint8>;
diff --git a/tensorflow/core/kernels/adjust_hsv_gpu.cu.h b/tensorflow/core/kernels/adjust_hsv_gpu.cu.h
index 49df5ae296..dede7d249d 100644
--- a/tensorflow/core/kernels/adjust_hsv_gpu.cu.h
+++ b/tensorflow/core/kernels/adjust_hsv_gpu.cu.h
@@ -91,11 +91,10 @@ inline __device__ RgbTuple hsv2rgb_cuda(const float h, const float s,
   return tuple;
 }
 
-template <bool AdjustHue, bool AdjustSaturation, bool AdjustV>
+template <bool AdjustHue, bool AdjustSaturation, bool AdjustV, typename T>
 __global__ void adjust_hsv_nhwc(const int64 number_elements,
-                                const float* const __restrict__ input,
-                                float* const output,
-                                const float* const hue_delta,
+                                const T* const __restrict__ input,
+                                T* const output, const float* const hue_delta,
                                 const float* const saturation_scale,
                                 const float* const value_scale) {
   // multiply by 3 since we're dealing with contiguous RGB bytes for each pixel
@@ -111,7 +110,9 @@ __global__ void adjust_hsv_nhwc(const int64 number_elements,
     output[idx + 2] = input[idx + 2];
     return;
   }
-  const HsvTuple hsv = rgb2hsv_cuda(input[idx], input[idx + 1], input[idx + 2]);
+  const HsvTuple hsv = rgb2hsv_cuda(static_cast<float>(input[idx]),
+                                    static_cast<float>(input[idx + 1]),
+                                    static_cast<float>(input[idx + 2]));
   float new_h = hsv.h;
   float new_s = hsv.s;
   float new_v = hsv.v;
@@ -134,9 +135,9 @@ __global__ void adjust_hsv_nhwc(const int64 number_elements,
     new_v = hsv.v * scale;
   }
   const RgbTuple rgb = hsv2rgb_cuda(new_h, new_s, new_v);
-  output[idx] = rgb.r;
-  output[idx + 1] = rgb.g;
-  output[idx + 2] = rgb.b;
+  output[idx] = static_cast<T>(rgb.r);
+  output[idx + 1] = static_cast<T>(rgb.g);
+  output[idx + 2] = static_cast<T>(rgb.b);
 }
 
 }  // namespace internal
diff --git a/tensorflow/core/kernels/adjust_hue_op.cc b/tensorflow/core/kernels/adjust_hue_op.cc
index 52dec94305..06de5ea3fb 100644
--- a/tensorflow/core/kernels/adjust_hue_op.cc
+++ b/tensorflow/core/kernels/adjust_hue_op.cc
@@ -82,7 +82,7 @@ class AdjustHueOpBase : public OpKernel {
   }
 };
 
-template <class Device>
+template <class Device, typename T>
 class AdjustHueOp;
 
 namespace internal {
@@ -196,7 +196,7 @@ static void hv_range_to_rgb(float h, float v_min, float v_max, float* r,
 }  // namespace internal
 
 template <>
-class AdjustHueOp<CPUDevice> : public AdjustHueOpBase {
+class AdjustHueOp<CPUDevice, float> : public AdjustHueOpBase {
  public:
   explicit AdjustHueOp(OpKernelConstruction* context)
       : AdjustHueOpBase(context) {}
@@ -245,12 +245,13 @@ class AdjustHueOp<CPUDevice> : public AdjustHueOpBase {
   }
 };
 
-REGISTER_KERNEL_BUILDER(Name("AdjustHue").Device(DEVICE_CPU),
-                        AdjustHueOp<CPUDevice>);
+REGISTER_KERNEL_BUILDER(
+    Name("AdjustHue").Device(DEVICE_CPU).TypeConstraint<float>("T"),
+    AdjustHueOp<CPUDevice, float>);
 
 #if GOOGLE_CUDA
-template <>
-class AdjustHueOp<GPUDevice> : public AdjustHueOpBase {
+template <typename T>
+class AdjustHueOp<GPUDevice, T> : public AdjustHueOpBase {
  public:
   explicit AdjustHueOp(OpKernelConstruction* context)
       : AdjustHueOpBase(context) {}
@@ -265,17 +266,24 @@ class AdjustHueOp<GPUDevice> : public AdjustHueOpBase {
     const auto stream = device.stream();
     OP_REQUIRES(context, stream, errors::Internal("No GPU stream available."));
     if (number_of_elements > 0) {
-      const float* input_data = input->flat<float>().data();
+      const T* input_data = input->flat<T>().data();
       const float* delta_h = delta->flat<float>().data();
-      float* const output_data = output->flat<float>().data();
-      functor::AdjustHueGPU()(&device, number_of_elements, input_data, delta_h,
-                              output_data);
+      T* const output_data = output->flat<T>().data();
+      functor::AdjustHueGPU<T>()(&device, number_of_elements, input_data,
+                                 delta_h, output_data);
     }
   }
 };
 
-REGISTER_KERNEL_BUILDER(Name("AdjustHue").Device(DEVICE_GPU),
-                        AdjustHueOp<GPUDevice>);
+#define REGISTER_GPU(T)                                            \
+  REGISTER_KERNEL_BUILDER(                                         \
+      Name("AdjustHue").Device(DEVICE_GPU).TypeConstraint<T>("T"), \
+      AdjustHueOp<GPUDevice, T>);
+
+REGISTER_GPU(float)
+REGISTER_GPU(Eigen::half)
+
+#undef REGISTER_GPU
 
 #endif
 
diff --git a/tensorflow/core/kernels/adjust_hue_op.h b/tensorflow/core/kernels/adjust_hue_op.h
index 983a4072bf..6d6699de3f 100644
--- a/tensorflow/core/kernels/adjust_hue_op.h
+++ b/tensorflow/core/kernels/adjust_hue_op.h
@@ -27,10 +27,11 @@ typedef Eigen::GpuDevice GPUDevice;
 
 namespace functor {
 
+template <typename T>
 struct AdjustHueGPU {
   void operator()(GPUDevice* device, const int64 number_of_elements,
-                  const float* const input, const float* const delta,
-                  float* const output);
+                  const T* const input, const float* const delta,
+                  T* const output);
 };
 
 }  // namespace functor
diff --git a/tensorflow/core/kernels/adjust_hue_op_gpu.cu.cc b/tensorflow/core/kernels/adjust_hue_op_gpu.cu.cc
index a4fe5f755c..c30085269c 100644
--- a/tensorflow/core/kernels/adjust_hue_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/adjust_hue_op_gpu.cu.cc
@@ -24,19 +24,25 @@ namespace tensorflow {
 
 namespace functor {
 
-void AdjustHueGPU::operator()(GPUDevice* device, const int64 number_of_elements,
-                              const float* const input,
-                              const float* const delta, float* const output) {
+template <typename T>
+void AdjustHueGPU<T>::operator()(GPUDevice* device,
+                                 const int64 number_of_elements,
+                                 const T* const input, const float* const delta,
+                                 T* const output) {
   const auto stream = device->stream();
   const CudaLaunchConfig config =
       GetCudaLaunchConfig(number_of_elements, *device);
   const int threads_per_block = config.thread_per_block;
   const int block_count =
       (number_of_elements + threads_per_block - 1) / threads_per_block;
-  internal::adjust_hsv_nhwc<true, false, false>
+  internal::adjust_hsv_nhwc<true, false, false, T>
       <<<block_count, threads_per_block, 0, stream>>>(
           number_of_elements, input, output, delta, nullptr, nullptr);
 }
+
+template struct AdjustHueGPU<float>;
+template struct AdjustHueGPU<Eigen::half>;
+
 }  // namespace functor
 }  // namespace tensorflow
 #endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/adjust_saturation_op.cc b/tensorflow/core/kernels/adjust_saturation_op.cc
index f0c6ae499d..87d34fcfcc 100644
--- a/tensorflow/core/kernels/adjust_saturation_op.cc
+++ b/tensorflow/core/kernels/adjust_saturation_op.cc
@@ -81,7 +81,7 @@ class AdjustSaturationOpBase : public OpKernel {
   }
 };
 
-template <class Device>
+template <class Device, typename T>
 class AdjustSaturationOp;
 
 namespace internal {
@@ -173,7 +173,7 @@ static void hsv_to_rgb(float h, float s, float v, float* r, float* g,
 }  // namespace internal
 
 template <>
-class AdjustSaturationOp<CPUDevice> : public AdjustSaturationOpBase {
+class AdjustSaturationOp<CPUDevice, float> : public AdjustSaturationOpBase {
  public:
   explicit AdjustSaturationOp(OpKernelConstruction* context)
       : AdjustSaturationOpBase(context) {}
@@ -211,12 +211,13 @@ class AdjustSaturationOp<CPUDevice> : public AdjustSaturationOpBase {
   }
 };
 
-REGISTER_KERNEL_BUILDER(Name("AdjustSaturation").Device(DEVICE_CPU),
-                        AdjustSaturationOp<CPUDevice>);
+REGISTER_KERNEL_BUILDER(
+    Name("AdjustSaturation").Device(DEVICE_CPU).TypeConstraint<float>("T"),
+    AdjustSaturationOp<CPUDevice, float>);
 
 #if GOOGLE_CUDA
-template <>
-class AdjustSaturationOp<GPUDevice> : public AdjustSaturationOpBase {
+template <typename T>
+class AdjustSaturationOp<GPUDevice, T> : public AdjustSaturationOpBase {
  public:
   explicit AdjustSaturationOp(OpKernelConstruction* context)
       : AdjustSaturationOpBase(context) {}
@@ -231,17 +232,24 @@ class AdjustSaturationOp<GPUDevice> : public AdjustSaturationOpBase {
     const auto stream = device.stream();
     OP_REQUIRES(context, stream, errors::Internal("No GPU stream available."));
     if (number_of_elements > 0) {
-      const float* input_data = input->flat<float>().data();
+      const T* input_data = input->flat<T>().data();
       const float* scale_data = scale->flat<float>().data();
-      float* const output_data = output->flat<float>().data();
-      functor::AdjustSaturationGPU()(&device, number_of_elements, input_data,
-                                     scale_data, output_data);
+      T* const output_data = output->flat<T>().data();
+      functor::AdjustSaturationGPU<T>()(&device, number_of_elements, input_data,
+                                        scale_data, output_data);
     }
   }
 };
 
-REGISTER_KERNEL_BUILDER(Name("AdjustSaturation").Device(DEVICE_GPU),
-                        AdjustSaturationOp<GPUDevice>);
+#define REGISTER_GPU(T)                                                   \
+  REGISTER_KERNEL_BUILDER(                                                \
+      Name("AdjustSaturation").Device(DEVICE_GPU).TypeConstraint<T>("T"), \
+      AdjustSaturationOp<GPUDevice, T>);
+
+REGISTER_GPU(float)
+REGISTER_GPU(Eigen::half)
+
+#undef REGISTER_GPU
 
 #endif
 
diff --git a/tensorflow/core/kernels/adjust_saturation_op.h b/tensorflow/core/kernels/adjust_saturation_op.h
index fd28ba536f..c21ce4e360 100644
--- a/tensorflow/core/kernels/adjust_saturation_op.h
+++ b/tensorflow/core/kernels/adjust_saturation_op.h
@@ -27,10 +27,11 @@ typedef Eigen::GpuDevice GPUDevice;
 
 namespace functor {
 
+template <typename T>
 struct AdjustSaturationGPU {
   void operator()(GPUDevice* device, const int64 number_of_elements,
-                  const float* const input, const float* const scale,
-                  float* const output);
+                  const T* const input, const float* const scale,
+                  T* const output);
 };
 
 }  // namespace functor
diff --git a/tensorflow/core/kernels/adjust_saturation_op_gpu.cu.cc b/tensorflow/core/kernels/adjust_saturation_op_gpu.cu.cc
index 37cfb26a47..6c70490d46 100644
--- a/tensorflow/core/kernels/adjust_saturation_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/adjust_saturation_op_gpu.cu.cc
@@ -24,21 +24,26 @@ namespace tensorflow {
 
 namespace functor {
 
-void AdjustSaturationGPU::operator()(GPUDevice* device,
-                                     const int64 number_of_elements,
-                                     const float* const input,
-                                     const float* const scale,
-                                     float* const output) {
+template <typename T>
+void AdjustSaturationGPU<T>::operator()(GPUDevice* device,
+                                        const int64 number_of_elements,
+                                        const T* const input,
+                                        const float* const scale,
+                                        T* const output) {
   const auto stream = device->stream();
   const CudaLaunchConfig config =
       GetCudaLaunchConfig(number_of_elements, *device);
   const int threads_per_block = config.thread_per_block;
   const int block_count =
       (number_of_elements + threads_per_block - 1) / threads_per_block;
-  internal::adjust_hsv_nhwc<false, true, false>
+  internal::adjust_hsv_nhwc<false, true, false, T>
       <<<block_count, threads_per_block, 0, stream>>>(
           number_of_elements, input, output, nullptr, scale, nullptr);
 }
+
+template struct AdjustSaturationGPU<float>;
+template struct AdjustSaturationGPU<Eigen::half>;
+
 }  // namespace functor
 }  // namespace tensorflow
 #endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/ops/image_ops.cc b/tensorflow/core/ops/image_ops.cc
index ee8b1e58d6..0f1555f49c 100644
--- a/tensorflow/core/ops/image_ops.cc
+++ b/tensorflow/core/ops/image_ops.cc
@@ -408,9 +408,10 @@ REGISTER_OP("AdjustContrast")
 
 // --------------------------------------------------------------------------
 REGISTER_OP("AdjustContrastv2")
-    .Input("images: float")
+    .Input("images: T")
     .Input("contrast_factor: float")
-    .Output("output: float")
+    .Output("output: T")
+    .Attr("T: {half, float} = DT_FLOAT")
     .SetShapeFn([](InferenceContext* c) {
       // The contrast_factor should be scalar only.
       ShapeHandle unused;
@@ -420,18 +421,20 @@ REGISTER_OP("AdjustContrastv2")
 
 // --------------------------------------------------------------------------
 REGISTER_OP("AdjustHue")
-    .Input("images: float")
+    .Input("images: T")
     .Input("delta: float")
-    .Output("output: float")
+    .Output("output: T")
+    .Attr("T: {half, float} = DT_FLOAT")
     .SetShapeFn([](InferenceContext* c) {
       return shape_inference::UnchangedShapeWithRankAtLeast(c, 3);
     });
 
 // --------------------------------------------------------------------------
 REGISTER_OP("AdjustSaturation")
-    .Input("images: float")
+    .Input("images: T")
     .Input("scale: float")
-    .Output("output: float")
+    .Output("output: T")
+    .Attr("T: {half, float} = DT_FLOAT")
     .SetShapeFn([](InferenceContext* c) {
       return shape_inference::UnchangedShapeWithRankAtLeast(c, 3);
     });
diff --git a/tensorflow/python/ops/image_ops_impl.py b/tensorflow/python/ops/image_ops_impl.py
index 24d049b726..8047743cfa 100644
--- a/tensorflow/python/ops/image_ops_impl.py
+++ b/tensorflow/python/ops/image_ops_impl.py
@@ -1250,14 +1250,14 @@ def random_brightness(image, max_delta, seed=None):
   interval `[-max_delta, max_delta)`.
 
   Args:
-    image: An image.
+    image: An image or images to adjust.
     max_delta: float, must be non-negative.
     seed: A Python integer. Used to create a random seed. See
       `tf.set_random_seed`
       for behavior.
 
   Returns:
-    The brightness-adjusted image.
+    The brightness-adjusted image(s).
 
   Raises:
     ValueError: if `max_delta` is negative.
@@ -1271,7 +1271,7 @@ def random_brightness(image, max_delta, seed=None):
 
 @tf_export('image.random_contrast')
 def random_contrast(image, lower, upper, seed=None):
-  """Adjust the contrast of an image by a random factor.
+  """Adjust the contrast of an image or images by a random factor.
 
   Equivalent to `adjust_contrast()` but uses a `contrast_factor` randomly
   picked in the interval `[lower, upper]`.
@@ -1281,11 +1281,10 @@ def random_contrast(image, lower, upper, seed=None):
     lower: float.  Lower bound for the random contrast factor.
     upper: float.  Upper bound for the random contrast factor.
     seed: A Python integer. Used to create a random seed. See
-      `tf.set_random_seed`
-      for behavior.
+      `tf.set_random_seed` for behavior.
 
   Returns:
-    The contrast-adjusted tensor.
+    The contrast-adjusted image(s).
 
   Raises:
     ValueError: if `upper <= lower` or if `lower < 0`.
@@ -1305,19 +1304,19 @@ def random_contrast(image, lower, upper, seed=None):
 def adjust_brightness(image, delta):
   """Adjust the brightness of RGB or Grayscale images.
 
-  This is a convenience method that converts an RGB image to float
-  representation, adjusts its brightness, and then converts it back to the
-  original data type. If several adjustments are chained it is advisable to
+  This is a convenience method that converts RGB images to float
+  representation, adjusts their brightness, and then converts them back to the
+  original data type. If several adjustments are chained, it is advisable to
   minimize the number of redundant conversions.
 
-  The value `delta` is added to all components of the tensor `image`. Both
-  `image` and `delta` are converted to `float` before adding (and `image` is
-  scaled appropriately if it is in fixed-point representation). For regular
+  The value `delta` is added to all components of the tensor `image`. `image` is
+  converted to `float` and scaled appropriately if it is in fixed-point
+  representation, and `delta` is converted to the same data type. For regular
   images, `delta` should be in the range `[0,1)`, as it is added to the image in
   floating point representation, where pixel values are in the `[0,1)` range.
 
   Args:
-    image: A tensor.
+    image: RGB image or images to adjust.
     delta: A scalar. Amount to add to the pixel values.
 
   Returns:
@@ -1327,10 +1326,14 @@ def adjust_brightness(image, delta):
     image = ops.convert_to_tensor(image, name='image')
     # Remember original dtype to so we can convert back if needed
     orig_dtype = image.dtype
-    flt_image = convert_image_dtype(image, dtypes.float32)
+
+    if orig_dtype in [dtypes.float16, dtypes.float32]:
+      flt_image = image
+    else:
+      flt_image = convert_image_dtype(image, dtypes.float32)
 
     adjusted = math_ops.add(
-        flt_image, math_ops.cast(delta, dtypes.float32), name=name)
+        flt_image, math_ops.cast(delta, flt_image.dtype), name=name)
 
     return convert_image_dtype(adjusted, orig_dtype, saturate=True)
 
@@ -1339,9 +1342,9 @@ def adjust_brightness(image, delta):
 def adjust_contrast(images, contrast_factor):
   """Adjust contrast of RGB or grayscale images.
 
-  This is a convenience method that converts an RGB image to float
-  representation, adjusts its contrast, and then converts it back to the
-  original data type. If several adjustments are chained it is advisable to
+  This is a convenience method that converts RGB images to float
+  representation, adjusts their contrast, and then converts them back to the
+  original data type. If several adjustments are chained, it is advisable to
   minimize the number of redundant conversions.
 
   `images` is a tensor of at least 3 dimensions.  The last 3 dimensions are
@@ -1366,7 +1369,11 @@ def adjust_contrast(images, contrast_factor):
     images = ops.convert_to_tensor(images, name='images')
     # Remember original dtype to so we can convert back if needed
     orig_dtype = images.dtype
-    flt_images = convert_image_dtype(images, dtypes.float32)
+
+    if orig_dtype in (dtypes.float16, dtypes.float32):
+      flt_images = images
+    else:
+      flt_images = convert_image_dtype(images, dtypes.float32)
 
     adjusted = gen_image_ops.adjust_contrastv2(
         flt_images, contrast_factor=contrast_factor, name=name)
@@ -1560,7 +1567,7 @@ def grayscale_to_rgb(images, name=None):
 # pylint: disable=invalid-name
 @tf_export('image.random_hue')
 def random_hue(image, max_delta, seed=None):
-  """Adjust the hue of an RGB image by a random factor.
+  """Adjust the hue of RGB images by a random factor.
 
   Equivalent to `adjust_hue()` but uses a `delta` randomly
   picked in the interval `[-max_delta, max_delta]`.
@@ -1570,10 +1577,10 @@ def random_hue(image, max_delta, seed=None):
   Args:
     image: RGB image or images. Size of the last dimension must be 3.
     max_delta: float.  Maximum value for the random delta.
-    seed: An operation-specific seed. It will be used in conjunction
-      with the graph-level seed to determine the real seeds that will be
-      used in this operation. Please see the documentation of
-      set_random_seed for its interaction with the graph-level random seed.
+    seed: An operation-specific seed. It will be used in conjunction with the
+      graph-level seed to determine the real seeds that will be used in this
+      operation. Please see the documentation of set_random_seed for its
+      interaction with the graph-level random seed.
 
   Returns:
     Adjusted image(s), same shape and DType as `image`.
@@ -1593,7 +1600,7 @@ def random_hue(image, max_delta, seed=None):
 
 @tf_export('image.adjust_hue')
 def adjust_hue(image, delta, name=None):
-  """Adjust hue of an RGB image.
+  """Adjust hue of RGB images.
 
   This is a convenience method that converts an RGB image to float
   representation, converts it to HSV, add an offset to the hue channel, converts
@@ -1601,7 +1608,7 @@ def adjust_hue(image, delta, name=None):
   are chained it is advisable to minimize the number of redundant conversions.
 
   `image` is an RGB image.  The image hue is adjusted by converting the
-  image to HSV and rotating the hue channel (H) by
+  image(s) to HSV and rotating the hue channel (H) by
   `delta`.  The image is then converted back to RGB.
 
   `delta` must be in the interval `[-1, 1]`.
@@ -1618,7 +1625,10 @@ def adjust_hue(image, delta, name=None):
     image = ops.convert_to_tensor(image, name='image')
     # Remember original dtype to so we can convert back if needed
     orig_dtype = image.dtype
-    flt_image = convert_image_dtype(image, dtypes.float32)
+    if orig_dtype in (dtypes.float16, dtypes.float32):
+      flt_image = image
+    else:
+      flt_image = convert_image_dtype(image, dtypes.float32)
 
     rgb_altered = gen_image_ops.adjust_hue(flt_image, delta)
 
@@ -1696,7 +1706,7 @@ def adjust_jpeg_quality(image, jpeg_quality, name=None):
 
 @tf_export('image.random_saturation')
 def random_saturation(image, lower, upper, seed=None):
-  """Adjust the saturation of an RGB image by a random factor.
+  """Adjust the saturation of RGB images by a random factor.
 
   Equivalent to `adjust_saturation()` but uses a `saturation_factor` randomly
   picked in the interval `[lower, upper]`.
@@ -1705,10 +1715,10 @@ def random_saturation(image, lower, upper, seed=None):
     image: RGB image or images. Size of the last dimension must be 3.
     lower: float.  Lower bound for the random saturation factor.
     upper: float.  Upper bound for the random saturation factor.
-    seed: An operation-specific seed. It will be used in conjunction
-      with the graph-level seed to determine the real seeds that will be
-      used in this operation. Please see the documentation of
-      set_random_seed for its interaction with the graph-level random seed.
+    seed: An operation-specific seed. It will be used in conjunction with the
+      graph-level seed to determine the real seeds that will be used in this
+      operation. Please see the documentation of set_random_seed for its
+      interaction with the graph-level random seed.
 
   Returns:
     Adjusted image(s), same shape and DType as `image`.
@@ -1729,17 +1739,17 @@ def random_saturation(image, lower, upper, seed=None):
 
 @tf_export('image.adjust_saturation')
 def adjust_saturation(image, saturation_factor, name=None):
-  """Adjust saturation of an RGB image.
+  """Adjust saturation of RGB images.
 
-  This is a convenience method that converts an RGB image to float
-  representation, converts it to HSV, add an offset to the saturation channel,
+  This is a convenience method that converts RGB images to float
+  representation, converts them to HSV, add an offset to the saturation channel,
   converts back to RGB and then back to the original data type. If several
   adjustments are chained it is advisable to minimize the number of redundant
   conversions.
 
-  `image` is an RGB image.  The image saturation is adjusted by converting the
-  image to HSV and multiplying the saturation (S) channel by
-  `saturation_factor` and clipping. The image is then converted back to RGB.
+  `image` is an RGB image or images.  The image saturation is adjusted by
+  converting the images to HSV and multiplying the saturation (S) channel by
+  `saturation_factor` and clipping. The images are then converted back to RGB.
 
   Args:
     image: RGB image or images. Size of the last dimension must be 3.
@@ -1753,11 +1763,14 @@ def adjust_saturation(image, saturation_factor, name=None):
     image = ops.convert_to_tensor(image, name='image')
     # Remember original dtype to so we can convert back if needed
     orig_dtype = image.dtype
-    flt_image = convert_image_dtype(image, dtypes.float32)
+    if orig_dtype in (dtypes.float16, dtypes.float32):
+      flt_image = image
+    else:
+      flt_image = convert_image_dtype(image, dtypes.float32)
+
+    adjusted = gen_image_ops.adjust_saturation(flt_image, saturation_factor)
 
-    return convert_image_dtype(
-        gen_image_ops.adjust_saturation(flt_image, saturation_factor),
-        orig_dtype)
+    return convert_image_dtype(adjusted, orig_dtype)
 
 
 @tf_export('io.is_jpeg', 'image.is_jpeg', v1=['io.is_jpeg', 'image.is_jpeg'])
diff --git a/tensorflow/python/ops/image_ops_test.py b/tensorflow/python/ops/image_ops_test.py
index e7249333bd..361befabce 100644
--- a/tensorflow/python/ops/image_ops_test.py
+++ b/tensorflow/python/ops/image_ops_test.py
@@ -886,44 +886,6 @@ class AdjustSaturationTest(test_util.TensorFlowTestCase):
       y_tf = self.evaluate(y)
       self.assertAllEqual(y_tf, y_np)
 
-  def _adjust_saturation(self, image, saturation_factor):
-    image = ops.convert_to_tensor(image, name="image")
-    orig_dtype = image.dtype
-    flt_image = image_ops.convert_image_dtype(image, dtypes.float32)
-    saturation_adjusted_image = gen_image_ops.adjust_saturation(
-        flt_image, saturation_factor)
-    return image_ops.convert_image_dtype(saturation_adjusted_image, orig_dtype)
-
-  def testHalfSaturationFused(self):
-    x_shape = [2, 2, 3]
-    x_rgb_data = [0, 5, 13, 54, 135, 226, 37, 8, 234, 90, 255, 1]
-    x_np = np.array(x_rgb_data, dtype=np.uint8).reshape(x_shape)
-
-    saturation_factor = 0.5
-    y_rgb_data = [6, 9, 13, 140, 180, 226, 135, 121, 234, 172, 255, 128]
-    y_np = np.array(y_rgb_data, dtype=np.uint8).reshape(x_shape)
-
-    with self.test_session(use_gpu=True):
-      x = constant_op.constant(x_np, shape=x_shape)
-      y = self._adjust_saturation(x, saturation_factor)
-      y_tf = self.evaluate(y)
-      self.assertAllEqual(y_tf, y_np)
-
-  def testTwiceSaturationFused(self):
-    x_shape = [2, 2, 3]
-    x_data = [0, 5, 13, 54, 135, 226, 37, 8, 234, 90, 255, 1]
-    x_np = np.array(x_data, dtype=np.uint8).reshape(x_shape)
-
-    saturation_factor = 2.0
-    y_data = [0, 5, 13, 0, 106, 226, 30, 0, 234, 89, 255, 0]
-    y_np = np.array(y_data, dtype=np.uint8).reshape(x_shape)
-
-    with self.test_session(use_gpu=True):
-      x = constant_op.constant(x_np, shape=x_shape)
-      y = self._adjust_saturation(x, saturation_factor)
-      y_tf = self.evaluate(y)
-      self.assertAllEqual(y_tf, y_np)
-
   def _adjustSaturationNp(self, x_np, scale):
     self.assertEqual(x_np.shape[-1], 3)
     x_v = x_np.reshape([-1, 3])
@@ -977,7 +939,7 @@ class AdjustSaturationTest(test_util.TensorFlowTestCase):
           else:
             raise AssertionError("Invalid test style: %s" % (test_style))
           y_baseline = self._adjustSaturationNp(x_np, scale)
-          y_fused = self._adjust_saturation(x_np, scale).eval()
+          y_fused = image_ops.adjust_saturation(x_np, scale).eval()
           self.assertAllClose(y_fused, y_baseline, rtol=2e-5, atol=1e-5)
 
 
@@ -1438,12 +1400,12 @@ class AdjustContrastTest(test_util.TensorFlowTestCase):
 
 class AdjustBrightnessTest(test_util.TensorFlowTestCase):
 
-  def _testBrightness(self, x_np, y_np, delta):
+  def _testBrightness(self, x_np, y_np, delta, tol=1e-6):
     with self.test_session(use_gpu=True):
       x = constant_op.constant(x_np, shape=x_np.shape)
       y = image_ops.adjust_brightness(x, delta)
       y_tf = self.evaluate(y)
-      self.assertAllClose(y_tf, y_np, 1e-6)
+      self.assertAllClose(y_tf, y_np, tol)
 
   def testPositiveDeltaUint8(self):
     x_shape = [2, 2, 3]
@@ -1455,7 +1417,7 @@ class AdjustBrightnessTest(test_util.TensorFlowTestCase):
 
     self._testBrightness(x_np, y_np, delta=10. / 255.)
 
-  def testPositiveDeltaFloat(self):
+  def testPositiveDeltaFloat32(self):
     x_shape = [2, 2, 3]
     x_data = [0, 5, 13, 54, 135, 226, 37, 8, 234, 90, 255, 1]
     x_np = np.array(x_data, dtype=np.float32).reshape(x_shape) / 255.
@@ -1465,6 +1427,16 @@ class AdjustBrightnessTest(test_util.TensorFlowTestCase):
 
     self._testBrightness(x_np, y_np, delta=10. / 255.)
 
+  def testPositiveDeltaFloat16(self):
+    x_shape = [2, 2, 3]
+    x_data = [0, 5, 13, 54, 135, 226, 37, 8, 234, 90, 255, 1]
+    x_np = np.array(x_data, dtype=np.float16).reshape(x_shape) / 255.
+
+    y_data = [10, 15, 23, 64, 145, 236, 47, 18, 244, 100, 265, 11]
+    y_np = np.array(y_data, dtype=np.float16).reshape(x_shape) / 255.
+
+    self._testBrightness(x_np, y_np, delta=10. / 255., tol=1e-3)
+
   def testNegativeDelta(self):
     x_shape = [2, 2, 3]
     x_data = [0, 5, 13, 54, 135, 226, 37, 8, 234, 90, 255, 1]
-- 
GitLab


From 43bb4d1f6d3d91a2f0019d187204f665f52c3955 Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Tue, 15 Jan 2019 12:49:03 -0800
Subject: [PATCH 0766/1095] Avoid using a temp directory with a space to
 workaround bazel errors.

PiperOrigin-RevId: 229422438
---
 tensorflow/tools/ci_build/windows/libtensorflow_gpu.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/ci_build/windows/libtensorflow_gpu.sh b/tensorflow/tools/ci_build/windows/libtensorflow_gpu.sh
index 7dfee8f371..9c05db974b 100644
--- a/tensorflow/tools/ci_build/windows/libtensorflow_gpu.sh
+++ b/tensorflow/tools/ci_build/windows/libtensorflow_gpu.sh
@@ -41,7 +41,7 @@ run_configure_for_gpu_build
 # build_libtensorflow_tarball in ../builds/libtensorflow.sh
 # cannot be used on Windows since it relies on pkg_tar rules.
 # So we do something special here
-bazel build -c opt --copt=/arch:AVX --announce_rc \
+bazel --output_user_root=${TMPDIR} build -c opt --copt=/arch:AVX --announce_rc \
   tensorflow:libtensorflow.so \
   tensorflow/tools/lib_package:clicenses_generate \
   tensorflow/java:libtensorflow_jni.so \
-- 
GitLab


From 0336900e0c9b9047444027a31bdbf5bae4fa3695 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Tue, 15 Jan 2019 12:59:27 -0800
Subject: [PATCH 0767/1095] Make XlaCompiledCpuFunction::StaticData
 inaccessible; NFC

This CL will prevent tfcompile users from accidentally depending on StaticData
internals.

PiperOrigin-RevId: 229424173
---
 tensorflow/compiler/aot/codegen.cc            |  28 +++--
 tensorflow/compiler/aot/codegen_test_h.golden |  23 ++--
 .../tf2xla/xla_compiled_cpu_function.h        | 116 ++++++++++++------
 .../tf2xla/xla_jit_compiled_cpu_function.cc   |  34 +++--
 4 files changed, 126 insertions(+), 75 deletions(-)

diff --git a/tensorflow/compiler/aot/codegen.cc b/tensorflow/compiler/aot/codegen.cc
index 347a365dcb..d016632da2 100644
--- a/tensorflow/compiler/aot/codegen.cc
+++ b/tensorflow/compiler/aot/codegen.cc
@@ -384,8 +384,9 @@ Status GenerateHeader(const CodegenOpts& opts, const tf2xla::Config& config,
   // calling HloProfilePrinter::profile_counters_size.
   const string assign_profile_counters_size =
       opts.gen_hlo_profile_printer_data
-          ? "data->set_profile_counters_size("
-            "data->hlo_profile_printer_data()->profile_counters_size());"
+          ? "set_static_data_profile_counters_size(data, "
+            "get_static_data_hlo_profile_printer_data(data)->"
+            "profile_counters_size());"
           : "";
 
   // Use a poor-man's text templating mechanism; first populate the full header
@@ -449,7 +450,7 @@ extern "C" void {{ENTRY}}(
 //   arg bytes aligned:  {{ARG_BYTES_ALIGNED}}
 //   temp bytes total:   {{TEMP_BYTES_TOTAL}}
 //   temp bytes aligned: {{TEMP_BYTES_ALIGNED}}
-class {{CLASS}} : public tensorflow::XlaCompiledCpuFunction {
+class {{CLASS}} final : public tensorflow::XlaCompiledCpuFunction {
  public:
   // Number of input arguments for the compiled computation.
   static constexpr size_t kNumArgs = {{ARG_NUM}};
@@ -464,16 +465,17 @@ class {{CLASS}} : public tensorflow::XlaCompiledCpuFunction {
     static XlaCompiledCpuFunction::StaticData* kStaticData = [](){
       XlaCompiledCpuFunction::StaticData* data =
         new XlaCompiledCpuFunction::StaticData;
-      data->set_raw_function({{ENTRY}});
-      data->set_buffer_infos(BufferInfos());
-      data->set_num_buffers(kNumBuffers);
-      data->set_arg_index_table(ArgIndexToBufferIndex());
-      data->set_num_args(kNumArgs);
-      data->set_result_index(kResultIndex);
-      data->set_arg_names(StaticArgNames());
-      data->set_result_names(StaticResultNames());
-      data->set_program_shape(StaticProgramShape());
-      data->set_hlo_profile_printer_data(StaticHloProfilePrinterData());
+      set_static_data_raw_function(data, {{ENTRY}});
+      set_static_data_buffer_infos(data, BufferInfos());
+      set_static_data_num_buffers(data, kNumBuffers);
+      set_static_data_arg_index_table(data, ArgIndexToBufferIndex());
+      set_static_data_num_args(data, kNumArgs);
+      set_static_data_result_index(data, kResultIndex);
+      set_static_data_arg_names(data, StaticArgNames());
+      set_static_data_result_names(data, StaticResultNames());
+      set_static_data_program_shape(data, StaticProgramShape());
+      set_static_data_hlo_profile_printer_data(
+          data, StaticHloProfilePrinterData());
 {{ASSIGN_PROFILE_COUNTERS_SIZE}}
       return data;
     }();
diff --git a/tensorflow/compiler/aot/codegen_test_h.golden b/tensorflow/compiler/aot/codegen_test_h.golden
index 7fbf83af33..35994fc785 100644
--- a/tensorflow/compiler/aot/codegen_test_h.golden
+++ b/tensorflow/compiler/aot/codegen_test_h.golden
@@ -59,7 +59,7 @@ namespace bar {
 //   arg bytes aligned:  192
 //   temp bytes total:   126
 //   temp bytes aligned: 320
-class MyClass : public tensorflow::XlaCompiledCpuFunction {
+class MyClass final : public tensorflow::XlaCompiledCpuFunction {
  public:
   // Number of input arguments for the compiled computation.
   static constexpr size_t kNumArgs = 2;
@@ -74,16 +74,17 @@ class MyClass : public tensorflow::XlaCompiledCpuFunction {
     static XlaCompiledCpuFunction::StaticData* kStaticData = [](){
       XlaCompiledCpuFunction::StaticData* data =
         new XlaCompiledCpuFunction::StaticData;
-      data->set_raw_function(entry_point);
-      data->set_buffer_infos(BufferInfos());
-      data->set_num_buffers(kNumBuffers);
-      data->set_arg_index_table(ArgIndexToBufferIndex());
-      data->set_num_args(kNumArgs);
-      data->set_result_index(kResultIndex);
-      data->set_arg_names(StaticArgNames());
-      data->set_result_names(StaticResultNames());
-      data->set_program_shape(StaticProgramShape());
-      data->set_hlo_profile_printer_data(StaticHloProfilePrinterData());
+      set_static_data_raw_function(data, entry_point);
+      set_static_data_buffer_infos(data, BufferInfos());
+      set_static_data_num_buffers(data, kNumBuffers);
+      set_static_data_arg_index_table(data, ArgIndexToBufferIndex());
+      set_static_data_num_args(data, kNumArgs);
+      set_static_data_result_index(data, kResultIndex);
+      set_static_data_arg_names(data, StaticArgNames());
+      set_static_data_result_names(data, StaticResultNames());
+      set_static_data_program_shape(data, StaticProgramShape());
+      set_static_data_hlo_profile_printer_data(
+          data, StaticHloProfilePrinterData());
 
       return data;
     }();
diff --git a/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h b/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h
index c7341cf8b9..e1da19cf1d 100644
--- a/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h
+++ b/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h
@@ -59,45 +59,8 @@ class XlaCompiledCpuFunction {
   // AOT this is backed by data compiled into the object file.
   //
   // The contents of StaticData are XLA-internal implementation details and
-  // should not be relied on by clients.
-  //
-  // TODO(sanjoy): Come up with a cleaner way to express the contraint we want
-  // here: generated XlaCompiledCpuFunction subclasses should be able to create
-  // instances of StaticData but only XlaCompiledCpuFunction should be able to
-  // read from StaticData instances.
+  // should not be relied on by clients (and therefore are private).
   class StaticData {
-   public:
-    void set_raw_function(RawFunction raw_function) {
-      raw_function_ = raw_function;
-    }
-    void set_buffer_infos(
-        const cpu_function_runtime::BufferInfo* buffer_infos) {
-      buffer_infos_ = buffer_infos;
-    }
-    void set_num_buffers(size_t num_buffers) { num_buffers_ = num_buffers; }
-    void set_arg_index_table(const int32* arg_index_table) {
-      arg_index_table_ = arg_index_table;
-    }
-    void set_num_args(int64 num_args) { num_args_ = num_args; }
-    void set_result_index(size_t result_index) { result_index_ = result_index; }
-    void set_arg_names(const char** arg_names) { arg_names_ = arg_names; }
-    void set_result_names(const char** result_names) {
-      result_names_ = result_names;
-    }
-    void set_program_shape(const xla::ProgramShapeProto* program_shape) {
-      program_shape_ = program_shape;
-    }
-    const xla::HloProfilePrinterData* hlo_profile_printer_data() const {
-      return hlo_profile_printer_data_;
-    }
-    void set_hlo_profile_printer_data(
-        const xla::HloProfilePrinterData* hlo_profile_printer_data) {
-      hlo_profile_printer_data_ = hlo_profile_printer_data;
-    }
-    void set_profile_counters_size(int64 profile_counters_size) {
-      profile_counters_size_ = profile_counters_size;
-    }
-
    private:
     // The raw function to call.
     RawFunction raw_function_;
@@ -134,7 +97,8 @@ class XlaCompiledCpuFunction {
     // declared so we don't have access to that information here.
     int64 profile_counters_size_ = 0;
 
-    // Only XlaCompiledCpuFunction is allowed to read the above fields.
+    // Only XlaCompiledCpuFunction is allowed to read and write the above
+    // fields.
     friend class XlaCompiledCpuFunction;
   };
 
@@ -280,6 +244,76 @@ class XlaCompiledCpuFunction {
     return *hlo_profile_printer_data_;
   }
 
+ protected:
+  // ---------------------------------------------------------------------------
+  // Accessors for reading from and writing to instances of `StaticData`.
+  //
+  // Classes generated by tfcompile can call these because the generated classes
+  // inherit from `XlaCompiledCpuFunction`.  `XlaJitCompiledCpuFunction` can
+  // call these because it is explicitly added as a friend.
+
+  static void set_static_data_raw_function(StaticData* static_data,
+                                           RawFunction raw_function) {
+    static_data->raw_function_ = raw_function;
+  }
+
+  static void set_static_data_buffer_infos(
+      StaticData* static_data,
+      const cpu_function_runtime::BufferInfo* buffer_infos) {
+    static_data->buffer_infos_ = buffer_infos;
+  }
+
+  static void set_static_data_num_buffers(StaticData* static_data,
+                                          size_t num_buffers) {
+    static_data->num_buffers_ = num_buffers;
+  }
+
+  static void set_static_data_arg_index_table(StaticData* static_data,
+                                              const int32* arg_index_table) {
+    static_data->arg_index_table_ = arg_index_table;
+  }
+
+  static void set_static_data_num_args(StaticData* static_data,
+                                       int64 num_args) {
+    static_data->num_args_ = num_args;
+  }
+
+  static void set_static_data_result_index(StaticData* static_data,
+                                           size_t result_index) {
+    static_data->result_index_ = result_index;
+  }
+
+  static void set_static_data_arg_names(StaticData* static_data,
+                                        const char** arg_names) {
+    static_data->arg_names_ = arg_names;
+  }
+
+  static void set_static_data_result_names(StaticData* static_data,
+                                           const char** result_names) {
+    static_data->result_names_ = result_names;
+  }
+
+  static void set_static_data_program_shape(
+      StaticData* static_data, const xla::ProgramShapeProto* program_shape) {
+    static_data->program_shape_ = program_shape;
+  }
+
+  static void set_static_data_hlo_profile_printer_data(
+      StaticData* static_data,
+      const xla::HloProfilePrinterData* hlo_profile_printer_data) {
+    static_data->hlo_profile_printer_data_ = hlo_profile_printer_data;
+  }
+
+  static const xla::HloProfilePrinterData*
+  get_static_data_hlo_profile_printer_data(StaticData* static_data) {
+    return static_data->hlo_profile_printer_data_;
+  }
+
+  static void set_static_data_profile_counters_size(
+      StaticData* static_data, int64 profile_counters_size) {
+    static_data->profile_counters_size_ = profile_counters_size;
+  }
+
  private:
   const RawFunction raw_function_;
   const size_t result_index_;
@@ -313,6 +347,10 @@ class XlaCompiledCpuFunction {
   const char** result_names_ = nullptr;
   const xla::ProgramShapeProto* program_shape_ = nullptr;
   const xla::HloProfilePrinterData* hlo_profile_printer_data_ = nullptr;
+
+  // Add `XlaJitCompiledCpuFunction` as a friend so that it can access the
+  // `set_static_data_*` static methods above.
+  friend class XlaJitCompiledCpuFunction;
 };
 
 }  // namespace tensorflow
diff --git a/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function.cc b/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function.cc
index fabbcd04fe..884dc45cb1 100644
--- a/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function.cc
+++ b/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function.cc
@@ -135,24 +135,34 @@ XlaJitCompiledCpuFunction::Compile(
   jit->arg_index_table_ = std::move(arg_index_table);
   jit->program_shape_ =
       absl::make_unique<xla::ProgramShapeProto>(program_shape->ToProto());
-  jit->static_data_.set_raw_function(raw_function);
-  jit->static_data_.set_buffer_infos(jit->buffer_infos_.data());
-  jit->static_data_.set_num_buffers(jit->buffer_infos_.size());
-  jit->static_data_.set_arg_index_table(jit->arg_index_table_.data());
-  jit->static_data_.set_num_args(jit->arg_index_table_.size());
-  jit->static_data_.set_result_index(result_index);
+  XlaCompiledCpuFunction::set_static_data_raw_function(&jit->static_data_,
+                                                       raw_function);
+  XlaCompiledCpuFunction::set_static_data_buffer_infos(
+      &jit->static_data_, jit->buffer_infos_.data());
+  XlaCompiledCpuFunction::set_static_data_num_buffers(
+      &jit->static_data_, jit->buffer_infos_.size());
+  XlaCompiledCpuFunction::set_static_data_arg_index_table(
+      &jit->static_data_, jit->arg_index_table_.data());
+  XlaCompiledCpuFunction::set_static_data_num_args(
+      &jit->static_data_, jit->arg_index_table_.size());
+  XlaCompiledCpuFunction::set_static_data_result_index(&jit->static_data_,
+                                                       result_index);
   // Optional metadata is collected and set below.
   CollectNames(config.feed(), &jit->nonempty_arg_names_, &jit->arg_names_);
   CollectNames(config.fetch(), &jit->nonempty_result_names_,
                &jit->result_names_);
-  jit->static_data_.set_arg_names(jit->arg_names_.data());
-  jit->static_data_.set_result_names(jit->result_names_.data());
-  jit->static_data_.set_program_shape(jit->program_shape_.get());
+  XlaCompiledCpuFunction::set_static_data_arg_names(&jit->static_data_,
+                                                    jit->arg_names_.data());
+  XlaCompiledCpuFunction::set_static_data_result_names(
+      &jit->static_data_, jit->result_names_.data());
+  XlaCompiledCpuFunction::set_static_data_program_shape(
+      &jit->static_data_, jit->program_shape_.get());
 
   if (cpu_executable->hlo_profiling_enabled()) {
-    jit->static_data_.set_hlo_profile_printer_data(
-        &cpu_executable->hlo_profile_printer_data());
-    jit->static_data_.set_profile_counters_size(
+    XlaCompiledCpuFunction::set_static_data_hlo_profile_printer_data(
+        &jit->static_data_, &cpu_executable->hlo_profile_printer_data());
+    XlaCompiledCpuFunction::set_static_data_profile_counters_size(
+        &jit->static_data_,
         cpu_executable->hlo_profile_printer_data().profile_counters_size());
   }
 
-- 
GitLab


From eadd8aca53dedb40f5664d616288db665dd95652 Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Tue, 15 Jan 2019 13:06:12 -0800
Subject: [PATCH 0768/1095] Making sure dataset "output" ops are not pruned
 from function graphs as they might have side effects even though the ops are
 not marked as stateful.

PiperOrigin-RevId: 229425577
---
 tensorflow/core/common_runtime/function.cc | 2 +-
 tensorflow/core/graph/graph.cc             | 4 ++++
 tensorflow/core/graph/graph.h              | 3 +++
 3 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/common_runtime/function.cc b/tensorflow/core/common_runtime/function.cc
index 48f32df275..3df09eac29 100644
--- a/tensorflow/core/common_runtime/function.cc
+++ b/tensorflow/core/common_runtime/function.cc
@@ -812,7 +812,7 @@ void PruneFunctionBody(Graph* g) {
     // TODO(mrry): Investigate whether the `n->IsControlFlow()` test is
     // still needed. It would be preferable to prune entire loops and/or
     // conditionals if they are not used in the graph.
-    if (n->IsControlFlow() ||
+    if (n->IsControlFlow() || n->IsDataset() ||
         (n->op_def().is_stateful() && n->type_string() != kArgOp)) {
       nodes.insert(n);
     }
diff --git a/tensorflow/core/graph/graph.cc b/tensorflow/core/graph/graph.cc
index 3ea222c13c..00d3549312 100644
--- a/tensorflow/core/graph/graph.cc
+++ b/tensorflow/core/graph/graph.cc
@@ -85,6 +85,10 @@ const std::unordered_map<string, Node::NodeClass>& Node::kNodeClassTable =
         {"CollectiveBcastSend", NC_COLLECTIVE},
         {"CollectiveBcastRecv", NC_COLLECTIVE},
         {"FakeParam", NC_FAKE_PARAM},
+        {"IteratorGetNext", NC_DATASET},
+        {"IteratorGetNextSync", NC_DATASET},
+        {"DatasetToSingleElement", NC_DATASET},
+        {"ReduceDataset", NC_DATASET},
     });
 
 #undef REF_CLASS
diff --git a/tensorflow/core/graph/graph.h b/tensorflow/core/graph/graph.h
index 289a3d2a23..f65e4b921e 100644
--- a/tensorflow/core/graph/graph.h
+++ b/tensorflow/core/graph/graph.h
@@ -174,6 +174,8 @@ class Node {
   bool IsMetadata() const { return class_ == NC_METADATA; }
   bool IsFakeParam() const { return class_ == NC_FAKE_PARAM; }
 
+  bool IsDataset() const { return class_ == NC_DATASET; }
+
   template <typename T>
   void AddAttr(const string& name, const T& val) {
     SetAttrValue(val, AddAttrHelper(name));
@@ -254,6 +256,7 @@ class Node {
     NC_SCOPED_ALLOCATOR,
     NC_COLLECTIVE,
     NC_FAKE_PARAM,
+    NC_DATASET,
     NC_OTHER  // Not a special kind of node
   };
 
-- 
GitLab


From 75d9981d6c79b178340c0e011921f4b407f59341 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 15 Jan 2019 13:09:06 -0800
Subject: [PATCH 0769/1095] Add LSTM and embedding models to keras correctness
 test.

PiperOrigin-RevId: 229426039
---
 tensorflow/contrib/distribute/python/BUILD    | 40 ++++++++++
 .../contrib/distribute/python/combinations.py | 14 +++-
 .../python/keras_correctness_test_base.py     | 75 +++++++++++++++++--
 .../python/keras_dnn_correctness_test.py      | 20 +++--
 .../keras_embedding_model_correctness_test.py | 75 +++++++++++++++++++
 .../keras_lstm_model_correctness_test.py      | 65 ++++++++++++++++
 6 files changed, 273 insertions(+), 16 deletions(-)
 create mode 100644 tensorflow/contrib/distribute/python/keras_embedding_model_correctness_test.py
 create mode 100644 tensorflow/contrib/distribute/python/keras_lstm_model_correctness_test.py

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index 4c25f49f16..1a54cd167e 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -660,7 +660,9 @@ py_library(
     srcs = [
         "keras_correctness_test_base.py",
         "keras_dnn_correctness_test.py",
+        "keras_embedding_model_correctness_test.py",
         "keras_image_model_correctness_test.py",
+        "keras_lstm_model_correctness_test.py",
     ],
     deps = [
         ":combinations",
@@ -714,6 +716,44 @@ cuda_py_test(
     ],
 )
 
+cuda_py_test(
+    name = "keras_embedding_model_correctness_test",
+    size = "medium",
+    srcs = ["keras_embedding_model_correctness_test.py"],
+    additional_deps = [
+        ":keras_correctness_test_lib",
+    ],
+    # Shard count is set to an odd number to distribute tasks across
+    # shards more evenly.
+    shard_count = 31,
+    tags = [
+        "multi_and_single_gpu",
+        "no_oss",  # TODO(b/117919883): Fix python error.
+        "no_pip",
+        "no_windows_gpu",
+        "notsan",
+    ],
+)
+
+cuda_py_test(
+    name = "keras_lstm_model_correctness_test",
+    size = "medium",
+    srcs = ["keras_lstm_model_correctness_test.py"],
+    additional_deps = [
+        ":keras_correctness_test_lib",
+    ],
+    # Shard count is set to an odd number to distribute tasks across
+    # shards more evenly.
+    shard_count = 31,
+    tags = [
+        "multi_and_single_gpu",
+        "no_oss",  # TODO(b/117919883): Fix python error.
+        "no_pip",
+        "no_windows_gpu",
+        "notsan",
+    ],
+)
+
 py_library(
     name = "metrics_v1_test_lib",
     testonly = 1,
diff --git a/tensorflow/contrib/distribute/python/combinations.py b/tensorflow/contrib/distribute/python/combinations.py
index f6c4291659..304762e69e 100644
--- a/tensorflow/contrib/distribute/python/combinations.py
+++ b/tensorflow/contrib/distribute/python/combinations.py
@@ -321,11 +321,12 @@ class NamedDistribution(object):
     return self._required_tpu
 
 
-def _get_tpu_strategy_creator(steps_per_run):
+def _get_tpu_strategy_creator(steps_per_run, **kwargs):
   def _create_tpu_strategy():
     resolver = cluster_resolver.TPUClusterResolver("")
     tpu_lib.initialize_tpu_system(resolver)
-    strategy = tpu_lib.TPUStrategy(resolver, steps_per_run=steps_per_run)
+    strategy = tpu_lib.TPUStrategy(resolver, steps_per_run=steps_per_run,
+                                   **kwargs)
     return strategy
   return _create_tpu_strategy
 
@@ -344,6 +345,15 @@ tpu_strategy = NamedDistribution(
 tpu_strategy_one_step = NamedDistribution(
     "TPUOneStep", _get_tpu_strategy_creator(steps_per_run=1),
     required_tpu=True)
+# TODO(b/122327153): Remove below two NamedDistributions.
+tpu_strategy_loop_on_device = NamedDistribution(
+    "TPULoopOnDevice", _get_tpu_strategy_creator(
+        steps_per_run=2, _disable_training_loop_on_host=True),
+    required_tpu=True)
+tpu_strategy_one_step_loop_on_device = NamedDistribution(
+    "TPUOneStepLoopOnDevice", _get_tpu_strategy_creator(
+        steps_per_run=1, _disable_training_loop_on_host=True),
+    required_tpu=True)
 
 mirrored_strategy_with_one_cpu = NamedDistribution(
     "Mirrored1CPU",
diff --git a/tensorflow/contrib/distribute/python/keras_correctness_test_base.py b/tensorflow/contrib/distribute/python/keras_correctness_test_base.py
index 9f3cab49a8..08ed933f29 100644
--- a/tensorflow/contrib/distribute/python/keras_correctness_test_base.py
+++ b/tensorflow/contrib/distribute/python/keras_correctness_test_base.py
@@ -51,22 +51,46 @@ all_strategies = [
 ]
 
 
-def all_strategy_combinations_with_eager_and_graph_modes():
-  return combinations.combine(distribution=all_strategies,
-                              mode=['graph', 'eager'])
+def eager_mode_test_configuration():
+  return combinations.combine(mode='eager',
+                              use_numpy=False,
+                              use_validation_data=False)
 
 
-def all_strategy_combinations_with_graph_mode():
-  return combinations.combine(distribution=all_strategies, mode=['graph'])
+def graph_mode_test_configuration():
+  return combinations.combine(mode='graph',
+                              use_numpy=[True, False],
+                              use_validation_data=[True, False])
 
 
 def all_strategy_and_input_config_combinations():
   return (
       combinations.times(
           combinations.combine(distribution=all_strategies),
-          combinations.combine(mode=['graph', 'eager'],
-                               use_numpy=[True, False],
-                               use_validation_data=[True, False])))
+          eager_mode_test_configuration() + graph_mode_test_configuration()))
+
+
+def strategies_for_embedding_models():
+  """Returns distribution strategies to test for embedding models.
+
+  Since embedding models take longer to train, we disregard OneDeviceStrategy
+  and DefaultStrategy in order to prevent testing timeouts.
+  """
+
+  strategies = [s for s in all_strategies
+                if not s.required_tpu and s.required_gpus is not None]
+  strategies.append(combinations.tpu_strategy_loop_on_device)
+  strategies.append(combinations.tpu_strategy_one_step_loop_on_device)
+  return strategies
+
+
+def test_combinations_for_embedding_model():
+  return (
+      combinations.times(
+          combinations.combine(distribution=
+                               strategies_for_embedding_models()),
+          (graph_mode_test_configuration() +
+           eager_mode_test_configuration())))
 
 
 class MaybeDistributionScope(object):
@@ -393,5 +417,40 @@ class TestDistributionStrategyCorrectnessBase(test.TestCase,
                       testcase=self)
 
 
+class TestDistributionStrategyEmbeddingModelCorrectnessBase(
+    TestDistributionStrategyCorrectnessBase):
+  """Base class to test correctness of Keras models with embedding layers."""
+
+  def get_data(self,
+               count=(_GLOBAL_BATCH_SIZE * _EVAL_STEPS),
+               min_words=5,
+               max_words=10,
+               max_word_id=19,
+               num_classes=2):
+    distribution = []
+    for _ in range(num_classes):
+      dist = np.abs(np.random.randn(max_word_id))
+      dist /= np.sum(dist)
+      distribution.append(dist)
+
+    features = []
+    labels = []
+    for _ in range(count):
+      label = np.random.randint(0, num_classes, size=1)[0]
+      num_words = np.random.randint(min_words, max_words, size=1)[0]
+      word_ids = np.random.choice(
+          max_word_id, size=num_words, replace=True, p=distribution[label])
+      word_ids = word_ids
+      labels.append(label)
+      features.append(word_ids)
+
+    features = keras.preprocessing.sequence.pad_sequences(
+        features, maxlen=max_words)
+    x_train = np.asarray(features, dtype=np.float32)
+    y_train = np.asarray(labels, dtype=np.int32).reshape((count, 1))
+    x_predict = x_train
+    return x_train, y_train, x_predict
+
+
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/contrib/distribute/python/keras_dnn_correctness_test.py b/tensorflow/contrib/distribute/python/keras_dnn_correctness_test.py
index f21b1c6752..7afacab0dd 100644
--- a/tensorflow/contrib/distribute/python/keras_dnn_correctness_test.py
+++ b/tensorflow/contrib/distribute/python/keras_dnn_correctness_test.py
@@ -28,6 +28,17 @@ from tensorflow.python.keras.optimizer_v2 import gradient_descent as gradient_de
 from tensorflow.python.training import gradient_descent
 
 
+def all_strategy_combinations_with_eager_and_graph_modes():
+  return combinations.combine(distribution=keras_correctness_test_base.
+                              all_strategies,
+                              mode=['graph', 'eager'])
+
+
+def all_strategy_combinations_with_graph_mode():
+  return combinations.combine(distribution=keras_correctness_test_base.
+                              all_strategies, mode=['graph'])
+
+
 class TestDistributionStrategyDnnCorrectness(
     keras_correctness_test_base.TestDistributionStrategyCorrectnessBase):
 
@@ -65,8 +76,7 @@ class TestDistributionStrategyDnnCorrectness(
   def test_dnn_correctness(self, distribution, use_numpy, use_validation_data):
     self.run_correctness_test(distribution, use_numpy, use_validation_data)
 
-  @combinations.generate(keras_correctness_test_base.
-                         all_strategy_combinations_with_graph_mode())
+  @combinations.generate(all_strategy_combinations_with_graph_mode())
   def test_dnn_with_dynamic_learning_rate(self, distribution):
     self.run_dynamic_lr_test(distribution)
 
@@ -104,8 +114,7 @@ class TestDistributionStrategyDnnMetricCorrectness(
       history = model.fit(x=train_dataset, epochs=2, steps_per_epoch=10)
       self.assertEqual(history.history['binary_accuracy'], [1.0, 1.0])
 
-  @combinations.generate(keras_correctness_test_base.
-                         all_strategy_combinations_with_eager_and_graph_modes())
+  @combinations.generate(all_strategy_combinations_with_eager_and_graph_modes())
   def test_simple_dnn_metric_correctness(self, distribution):
     self.run_metric_correctness_test(distribution)
 
@@ -153,8 +162,7 @@ class TestDistributionStrategyDnnMetricEvalCorrectness(
       self.assertEqual(outs[1], 0.)
       self.assertEqual(outs[2], 0.)
 
-  @combinations.generate(keras_correctness_test_base.
-                         all_strategy_combinations_with_eager_and_graph_modes())
+  @combinations.generate(all_strategy_combinations_with_eager_and_graph_modes())
   def test_identity_model_metric_eval_correctness(self, distribution):
     self.run_eval_metrics_correctness_test(distribution)
 
diff --git a/tensorflow/contrib/distribute/python/keras_embedding_model_correctness_test.py b/tensorflow/contrib/distribute/python/keras_embedding_model_correctness_test.py
new file mode 100644
index 0000000000..3913f9bc0c
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/keras_embedding_model_correctness_test.py
@@ -0,0 +1,75 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Correctness test for tf.keras Embedding models using DistributionStrategy."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.distribute.python import combinations
+from tensorflow.contrib.distribute.python import keras_correctness_test_base
+from tensorflow.python import keras
+from tensorflow.python.eager import test
+from tensorflow.python.training import gradient_descent
+
+
+class DistributionStrategyEmbeddingModelCorrectnessTest(
+    keras_correctness_test_base.
+    TestDistributionStrategyEmbeddingModelCorrectnessBase):
+
+  def get_model(self, max_words=10, initial_weights=None, distribution=None):
+    with keras_correctness_test_base.MaybeDistributionScope(distribution):
+      word_ids = keras.layers.Input(
+          shape=(max_words,), dtype=np.int32, name='words')
+      word_embed = keras.layers.Embedding(input_dim=20,
+                                          output_dim=10)(word_ids)
+      if self.use_distributed_dense:
+        word_embed = keras.layers.TimeDistributed(keras.layers.Dense(4))(
+            word_embed)
+      avg = keras.layers.GlobalAveragePooling1D()(word_embed)
+      preds = keras.layers.Dense(2, activation='softmax')(avg)
+      model = keras.Model(inputs=[word_ids], outputs=[preds])
+
+      if initial_weights:
+        model.set_weights(initial_weights)
+
+      model.compile(
+          optimizer=gradient_descent.GradientDescentOptimizer(
+              learning_rate=0.1),
+          loss='sparse_categorical_crossentropy',
+          metrics=['sparse_categorical_accuracy'])
+    return model
+
+  @combinations.generate(keras_correctness_test_base.
+                         test_combinations_for_embedding_model())
+  def test_embedding_model_correctness(self, distribution, use_numpy,
+                                       use_validation_data):
+
+    self.use_distributed_dense = False
+    self.run_correctness_test(distribution, use_numpy, use_validation_data)
+
+  @combinations.generate(keras_correctness_test_base.
+                         test_combinations_for_embedding_model())
+  def test_embedding_time_distributed_model_correctness(self,
+                                                        distribution,
+                                                        use_numpy,
+                                                        use_validation_data):
+    self.use_distributed_dense = True
+    self.run_correctness_test(distribution, use_numpy, use_validation_data)
+
+
+if __name__ == '__main__':
+  test.main()
diff --git a/tensorflow/contrib/distribute/python/keras_lstm_model_correctness_test.py b/tensorflow/contrib/distribute/python/keras_lstm_model_correctness_test.py
new file mode 100644
index 0000000000..9ed2dfa206
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/keras_lstm_model_correctness_test.py
@@ -0,0 +1,65 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Correctness tests for tf.keras LSTM model using DistributionStrategy."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.distribute.python import combinations
+from tensorflow.contrib.distribute.python import keras_correctness_test_base
+from tensorflow.python import keras
+from tensorflow.python.eager import test
+from tensorflow.python.training import gradient_descent
+
+
+class DistributionStrategyLstmModelCorrectnessTest(
+    keras_correctness_test_base.
+    TestDistributionStrategyEmbeddingModelCorrectnessBase):
+
+  def get_model(self, max_words=10, initial_weights=None, distribution=None):
+    with keras_correctness_test_base.MaybeDistributionScope(distribution):
+      word_ids = keras.layers.Input(
+          shape=(max_words,), dtype=np.int32, name='words')
+      word_embed = keras.layers.Embedding(input_dim=20,
+                                          output_dim=10)(word_ids)
+      lstm_embed = keras.layers.LSTM(units=4,
+                                     return_sequences=False)(word_embed)
+
+      preds = keras.layers.Dense(2, activation='softmax')(lstm_embed)
+      model = keras.Model(inputs=[word_ids], outputs=[preds])
+
+      if initial_weights:
+        model.set_weights(initial_weights)
+
+      model.compile(
+          optimizer=gradient_descent.GradientDescentOptimizer(
+              learning_rate=0.1),
+          loss='sparse_categorical_crossentropy',
+          metrics=['sparse_categorical_accuracy'])
+    return model
+
+  @combinations.generate(keras_correctness_test_base.
+                         test_combinations_for_embedding_model())
+  def test_lstm_model_correctness(self,
+                                  distribution,
+                                  use_numpy,
+                                  use_validation_data):
+    self.run_correctness_test(distribution, use_numpy, use_validation_data)
+
+
+if __name__ == '__main__':
+  test.main()
-- 
GitLab


From dae187c176cd0d59f597e694ae193ff6a5dfbdd2 Mon Sep 17 00:00:00 2001
From: Peng Wang <wangpeng@google.com>
Date: Tue, 15 Jan 2019 13:26:33 -0800
Subject: [PATCH 0770/1095] A partial implementation of RFC "Random numbers in
 TensorFlow 2.0" (https://github.com/tensorflow/community/pull/38).

PiperOrigin-RevId: 229428968
---
 tensorflow/core/BUILD                         |   3 +
 .../api_def_StatefulStandardNormal.pbtxt      |  31 ++
 .../api_def_StatefulStandardNormal.pbtxt      |   4 +
 tensorflow/core/kernels/BUILD                 |  25 ++
 .../core/kernels/stateful_random_ops.cc       | 265 ++++++++++++++++++
 tensorflow/core/lib/random/philox_random.h    |   7 +
 tensorflow/core/ops/stateful_random_ops.cc    |  36 +++
 tensorflow/python/BUILD                       |  32 +++
 tensorflow/python/ops/stateful_random_ops.py  | 241 ++++++++++++++++
 .../python/ops/stateful_random_ops_test.py    | 156 +++++++++++
 10 files changed, 800 insertions(+)
 create mode 100644 tensorflow/core/api_def/base_api/api_def_StatefulStandardNormal.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_StatefulStandardNormal.pbtxt
 create mode 100644 tensorflow/core/kernels/stateful_random_ops.cc
 create mode 100644 tensorflow/core/ops/stateful_random_ops.cc
 create mode 100644 tensorflow/python/ops/stateful_random_ops.py
 create mode 100644 tensorflow/python/ops/stateful_random_ops_test.py

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index bedc78ac11..7ba9afb5a8 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -1101,6 +1101,7 @@ tf_gen_op_libs(
         "parsing_ops",
         "random_grad",
         "random_ops",
+        "stateful_random_ops",
         "remote_fused_graph_ops",
         "rpc_ops",
         "scoped_allocator_ops",
@@ -1256,6 +1257,7 @@ cc_library(
         ":parsing_ops_op_lib",
         ":ragged_ops",
         ":random_ops_op_lib",
+        ":stateful_random_ops_op_lib",
         ":remote_fused_graph_ops_op_lib",
         ":resource_variable_ops_op_lib",
         ":rpc_ops_op_lib",
@@ -1414,6 +1416,7 @@ cc_library(
         "//tensorflow/core/kernels:pooling_ops",
         "//tensorflow/core/kernels:ragged_ops",
         "//tensorflow/core/kernels:random_ops",
+        "//tensorflow/core/kernels:stateful_random_ops",
         "//tensorflow/core/kernels:random_poisson_op",
         "//tensorflow/core/kernels:remote_fused_graph_ops",
         "//tensorflow/core/kernels:required",
diff --git a/tensorflow/core/api_def/base_api/api_def_StatefulStandardNormal.pbtxt b/tensorflow/core/api_def/base_api/api_def_StatefulStandardNormal.pbtxt
new file mode 100644
index 0000000000..d963c55278
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_StatefulStandardNormal.pbtxt
@@ -0,0 +1,31 @@
+op {
+  graph_op_name: "StatefulStandardNormal"
+  in_arg {
+    name: "resource"
+    description: <<END
+The handle of the resource variable that stores the state of the RNG.
+END
+  }
+  in_arg {
+    name: "shape"
+    description: <<END
+The shape of the output tensor.
+END
+  }
+  out_arg {
+    name: "output"
+    description: <<END
+A tensor of the specified shape filled with random normal values.
+END
+  }
+  attr {
+    name: "dtype"
+    description: <<END
+The type of the output.
+END
+  }
+  summary: "Outputs random values from a normal distribution."
+  description: <<END
+The generated values will have mean 0 and standard deviation 1.
+END
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_StatefulStandardNormal.pbtxt b/tensorflow/core/api_def/python_api/api_def_StatefulStandardNormal.pbtxt
new file mode 100644
index 0000000000..fb3c3dbdb1
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_StatefulStandardNormal.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "StatefulStandardNormal"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 358d3d656e..374aa88d2d 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -4293,6 +4293,31 @@ tf_cuda_cc_test(
     ],
 )
 
+tf_kernel_library(
+    name = "stateful_random_ops",
+    prefix = "stateful_random_ops",
+    deps = [
+        ":bounds_check",
+        ":dense_update_functor",
+        ":gather_functor",
+        ":mutex_ops",
+        ":random_op",
+        ":resource_variable_ops",
+        ":scatter_functor",
+        ":state",
+        ":training_op_helpers",
+        ":variable_ops",
+        "//tensorflow/core:core_cpu",
+        "//tensorflow/core:core_cpu_lib",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+        "//tensorflow/core:stateful_random_ops_op_lib",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/types:variant",
+    ],
+)
+
 tf_kernel_library(
     name = "stateless_random_ops",
     prefix = "stateless_random_ops",
diff --git a/tensorflow/core/kernels/stateful_random_ops.cc b/tensorflow/core/kernels/stateful_random_ops.cc
new file mode 100644
index 0000000000..e5c1c25864
--- /dev/null
+++ b/tensorflow/core/kernels/stateful_random_ops.cc
@@ -0,0 +1,265 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#define EIGEN_USE_THREADS
+
+#include <algorithm>
+#include <cmath>
+#include <memory>
+
+#include "absl/strings/str_join.h"
+#include "absl/types/variant.h"
+#include "tensorflow/core/common_runtime/device.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/register_types.h"
+#include "tensorflow/core/framework/resource_mgr.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/tensor_types.h"
+#include "tensorflow/core/framework/variant_op_registry.h"
+#include "tensorflow/core/kernels/bounds_check.h"
+#include "tensorflow/core/kernels/dense_update_functor.h"
+#include "tensorflow/core/kernels/gather_functor.h"
+#include "tensorflow/core/kernels/random_op.h"
+#include "tensorflow/core/kernels/resource_variable_ops.h"
+#include "tensorflow/core/kernels/scatter_functor.h"
+#include "tensorflow/core/kernels/training_op_helpers.h"
+#include "tensorflow/core/kernels/variable_ops.h"
+#include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/core/refcount.h"
+#include "tensorflow/core/lib/random/random_distributions.h"
+#include "tensorflow/core/lib/random/simple_philox.h"
+#include "tensorflow/core/platform/logging.h"
+#include "tensorflow/core/platform/mem.h"
+#include "tensorflow/core/platform/mutex.h"
+#include "tensorflow/core/platform/types.h"
+#include "tensorflow/core/util/util.h"
+#include "tensorflow/core/util/work_sharder.h"
+
+namespace tensorflow {
+
+using CPUDevice = Eigen::ThreadPoolDevice;
+
+using random::PhiloxRandom;
+
+namespace {
+
+// 'Variable' doesn't support uint32 or uint64 yet (due to reasons explained
+// in b/111604096 and cl/171681867), so I use signed int here. I choose int64
+// instead of int32 because `VarHandleOp` doesn't support int32 on GPU.
+using StateElementType = int64;
+static constexpr DataType STATE_ELEMENT_DTYPE = DT_INT64;
+
+using Algorithm = StateElementType;
+static constexpr Algorithm RNG_ALG_PHILOX = 1;
+
+using SkippableRNG = absl::variant<PhiloxRandom>;
+
+// This function is for hiding the implementation detail about the
+// `absl::variant` index of each algorithm.
+Algorithm GetAlgorithm(SkippableRNG const& rng) {
+  auto idx = rng.index();
+  if (idx == 0) {
+    return RNG_ALG_PHILOX;
+  }
+  // unreachable
+  return RNG_ALG_PHILOX;
+}
+
+// Fills a buffer with random numbers sampled from a given distribution.
+template <class Device, class Distribution>
+Status FillRandom(OpKernelContext* ctx, const Device& device,
+                  SkippableRNG const& gen, int64 size, Distribution dist,
+                  typename Distribution::ResultElementType* data) {
+  auto algorithm = GetAlgorithm(gen);
+  if (algorithm == RNG_ALG_PHILOX) {
+    auto philox = absl::get<PhiloxRandom>(gen);
+    functor::FillPhiloxRandom<Device, Distribution>()(ctx, device, philox, data,
+                                                      size, dist);
+    return Status::OK();
+  } else {
+    // return errors::InvalidArgument("Unsupported algorithm id: ", algorithm);
+    return Status::OK();
+  }
+}
+
+// The following two functions use the contract "lower 32 bits for the first
+// uint32, higher 32 bits for the second". Note that this is endian-neutral,
+// unlike a direct memory copy `memcpy(output, &input, 8)`.
+void Int64ToUint32s(int64 input, uint32* output1, uint32* output2) {
+  auto u64 = static_cast<uint64>(input);
+  *output1 = static_cast<uint32>(u64);
+  *output2 = static_cast<uint32>(u64 >> 32);
+}
+
+int64 Uint32sToInt64(uint32 input1, uint32 input2) {
+  auto u64_1 = static_cast<uint64>(input1);
+  auto u64_2 = static_cast<uint64>(input2);
+  return static_cast<int64>(u64_1 | (u64_2 << 32));
+}
+
+void GetPhiloxStateFromTensor(Tensor const& tensor,
+                              PhiloxRandom::ResultType* counter,
+                              PhiloxRandom::Key* key) {
+  auto tensor_flat = tensor.flat<StateElementType>();
+  auto tensor_ptr = tensor_flat.data();
+  // tensor_ptr's index is added by 1 to skip the algorithm tag.
+  Int64ToUint32s(tensor_ptr[1], &(*counter)[0], &(*counter)[1]);
+  Int64ToUint32s(tensor_ptr[2], &(*counter)[2], &(*counter)[3]);
+  Int64ToUint32s(tensor_ptr[3], &(*key)[0], &(*key)[1]);
+}
+
+void WritePhiloxStateToTensor(PhiloxRandom::ResultType const& counter,
+                              PhiloxRandom::Key const& key, Tensor* tensor) {
+  auto tensor_flat = tensor->flat<StateElementType>();
+  auto tensor_ptr = tensor_flat.data();
+  // tensor_ptr's index is added by 1 to skip the algorithm tag.
+  tensor_ptr[1] = Uint32sToInt64(counter[0], counter[1]);
+  tensor_ptr[2] = Uint32sToInt64(counter[2], counter[3]);
+  tensor_ptr[3] = Uint32sToInt64(key[0], key[1]);
+}
+
+// A helper function that does the actual work for
+// 'MakeRNGCopyAndUpdateVariable'.
+template <typename Device>
+Status GetRNGCopyAndUpdateTensor(Tensor* tensor, int64 delta,
+                                 SkippableRNG* rng_copy);
+
+template <>
+Status GetRNGCopyAndUpdateTensor<CPUDevice>(Tensor* tensor, int64 delta,
+                                            SkippableRNG* rng_copy) {
+  // The dtype of `tensor` should be `StateElementType` and the first element
+  // is the algorithm.
+  if (tensor->dims() != 1) {
+    return errors::InvalidArgument(
+        "RNG state must have one and only one dimension, not ", tensor->dims());
+  }
+  auto tensor_flat = tensor->flat<StateElementType>();
+  if (tensor_flat.size() < 1) {
+    return errors::InvalidArgument("Size of tensor must be at least 1");
+  }
+  auto algorithm = tensor_flat.data()[0];
+  if (algorithm == RNG_ALG_PHILOX) {
+    // Delegates to PhiloxRandom to do the actual increasing.
+    static_assert(std::is_same<StateElementType, int64>::value,
+                  "StateElementType must be int64");
+    static_assert(std::is_same<PhiloxRandom::ResultElementType, uint32>::value,
+                  "PhiloxRandom::ResultElementType must be uint32");
+    auto counter_size = PhiloxRandom::ResultType::kElementCount;
+    auto key_size = PhiloxRandom::Key::kElementCount;
+    auto min_tensor_size = 1 + (counter_size + key_size) / 2;
+    if (tensor_flat.size() < min_tensor_size) {
+      return errors::InvalidArgument(
+          "For Philox algorithm, the size of state"
+          " must be at least ",
+          min_tensor_size, "; got ", tensor_flat.size());
+    }
+    PhiloxRandom::ResultType counter;
+    PhiloxRandom::Key key;
+    GetPhiloxStateFromTensor(*tensor, &counter, &key);
+    PhiloxRandom philox(counter, key);
+    auto old_philox = philox;
+    philox.Skip(delta);  // do the actual increasing
+    WritePhiloxStateToTensor(philox.counter(), philox.key(), tensor);
+    *rng_copy = SkippableRNG(old_philox);
+    return Status::OK();
+  } else {
+    // return errors::InvalidArgument("Unsupported algorithm id: ", algorithm);
+    *rng_copy = SkippableRNG(PhiloxRandom());
+    return Status::OK();
+  }
+}
+
+// Gets a copy of the RNG and updates the variable. The copy can be used to
+// generate upto 'samples' random numbers, and the variable is updated as if
+// 'samples' random numbers have been generated (e.g. if the variable is a
+// counnter, the counter is increased by 'samples').
+template <class Device>
+Status MakeRNGCopyAndUpdateVariable(OpKernelContext* ctx, int input_idx,
+                                    int64 samples, SkippableRNG* rng_copy) {
+  Var* var = nullptr;
+  TF_RETURN_IF_ERROR(
+      LookupResource(ctx, HandleFromInput(ctx, input_idx), &var));
+  core::ScopedUnref s(var);
+  mutex_lock ml(*var->mu());
+  Tensor* var_tensor = var->tensor();
+  if (var_tensor->dtype() != STATE_ELEMENT_DTYPE) {
+    return errors::InvalidArgument("dtype of RNG state variable must be ",
+                                   DataTypeString(STATE_ELEMENT_DTYPE),
+                                   ", not ",
+                                   DataTypeString(var_tensor->dtype()));
+  }
+  TF_RETURN_IF_ERROR(PrepareToUpdateVariable<Device, StateElementType>(
+      ctx, var_tensor, var->copy_on_read_mode.load()));
+  TF_RETURN_IF_ERROR(
+      GetRNGCopyAndUpdateTensor<Device>(var_tensor, samples, rng_copy));
+  return Status::OK();
+}
+
+template <typename Device, class Distribution>
+class StatefulRandomOp : public OpKernel {
+ public:
+  using T = typename Distribution::ResultElementType;
+  explicit StatefulRandomOp(OpKernelConstruction* ctx) : OpKernel(ctx) {}
+
+  // Assumes that input(0) is an existing resource.
+  void Compute(OpKernelContext* ctx) override {
+    const Tensor& shape_t = ctx->input(1);
+    Tensor* output;
+    TensorShape shape;
+    OP_REQUIRES_OK(ctx, MakeShape(shape_t, &shape));
+    OP_REQUIRES_OK(ctx, ctx->allocate_output(0, shape, &output));
+    if (shape.num_elements() == 0) return;
+
+    auto output_flat = output->flat<T>();
+    SkippableRNG rng;
+    // Multiplier 256 is the same as in FillPhiloxRandomTask; do not change
+    // it just here.
+    OP_REQUIRES_OK(ctx, MakeRNGCopyAndUpdateVariable<Device>(
+                            ctx, 0, output_flat.size() * 256, &rng));
+    // Fill in the random numbers
+    OP_REQUIRES_OK(ctx, FillRandom(ctx, ctx->eigen_device<Device>(), rng,
+                                   output_flat.size(), Distribution(),
+                                   output_flat.data()));
+  }
+};
+
+}  // namespace
+
+// So far the 'Distribution' type parameter is only used when the algorithm is
+// philox, so 'NormalDistribution<PhiloxRandom, ...>' is fine for now.
+#define REGISTER(DEVICE, TYPE)            \
+  REGISTER_KERNEL_BUILDER(                \
+      Name("StatefulStandardNormal")      \
+          .Device(DEVICE_##DEVICE)        \
+          .HostMemory("resource")         \
+          .HostMemory("shape")            \
+          .TypeConstraint<TYPE>("dtype"), \
+      StatefulRandomOp<DEVICE##Device,    \
+                       random::NormalDistribution<PhiloxRandom, TYPE> >);
+
+#define REGISTER_CPU(TYPE) REGISTER(CPU, TYPE)
+
+TF_CALL_half(REGISTER_CPU);
+TF_CALL_bfloat16(REGISTER_CPU);
+TF_CALL_float(REGISTER_CPU);
+TF_CALL_double(REGISTER_CPU);
+
+#undef REGISTER_CPU
+#undef REGISTER
+
+// TODO(wangpeng): Add RNG ops for other distributions.
+// TODO(wangpeng): Add support for GPU and XLA.
+
+}  // end namespace tensorflow
diff --git a/tensorflow/core/lib/random/philox_random.h b/tensorflow/core/lib/random/philox_random.h
index 058ed95ffb..f4bbc689d4 100644
--- a/tensorflow/core/lib/random/philox_random.h
+++ b/tensorflow/core/lib/random/philox_random.h
@@ -49,6 +49,7 @@ namespace random {
 template <typename T, int ElementCount>
 class Array {
  public:
+  static const int kElementCount = ElementCount;
   PHILOX_DEVICE_INLINE Array() {
     for (int i = 0; i < ElementCount; ++i) {
       data_[i] = T(0);
@@ -131,6 +132,12 @@ class PhiloxRandom {
   PHILOX_DEVICE_INLINE
   PhiloxRandom(ResultType counter, Key key) : counter_(counter), key_(key) {}
 
+  PHILOX_DEVICE_INLINE
+  ResultType const& counter() const { return counter_; }
+
+  PHILOX_DEVICE_INLINE
+  Key const& key() const { return key_; }
+
   // Skip the specified number of samples of 128-bits in the current stream.
   PHILOX_DEVICE_INLINE
   void Skip(uint64 count) {
diff --git a/tensorflow/core/ops/stateful_random_ops.cc b/tensorflow/core/ops/stateful_random_ops.cc
new file mode 100644
index 0000000000..4c9277eda5
--- /dev/null
+++ b/tensorflow/core/ops/stateful_random_ops.cc
@@ -0,0 +1,36 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/framework/shape_inference.h"
+
+namespace tensorflow {
+
+Status StatefulRandomShape(shape_inference::InferenceContext* c) {
+  shape_inference::ShapeHandle out;
+  TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(1, &out));
+  c->set_output(0, out);
+  return Status::OK();
+}
+
+REGISTER_OP("StatefulStandardNormal")
+    .Input("resource: resource")
+    .Input("shape: shape_dtype")
+    .Output("output: dtype")
+    .Attr("dtype: {half,bfloat16,float,double} = DT_FLOAT")
+    .Attr("shape_dtype: {int32, int64} = DT_INT64")
+    .SetShapeFn(StatefulRandomShape);
+
+}  // namespace tensorflow
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 589605e347..adba0b49fa 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -1833,6 +1833,11 @@ tf_gen_op_wrapper_private_py(
     visibility = ["//learning/brain/python/ops:__pkg__"],
 )
 
+tf_gen_op_wrapper_private_py(
+    name = "stateful_random_ops_gen",
+    visibility = ["//learning/brain/python/ops:__pkg__"],
+)
+
 tf_gen_op_wrapper_private_py(
     name = "resource_variable_ops_gen",
     visibility = [
@@ -2768,6 +2773,32 @@ py_library(
     ],
 )
 
+py_library(
+    name = "stateful_random_ops",
+    srcs = ["ops/stateful_random_ops.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":dtypes",
+        ":framework_ops",
+        ":math_ops",
+        ":stateful_random_ops_gen",
+        ":variables",
+        "//third_party/py/numpy",
+    ],
+)
+
+cuda_py_test(
+    name = "stateful_random_ops_test",
+    size = "medium",
+    srcs = ["ops/stateful_random_ops_test.py"],
+    additional_deps = [
+        ":stateful_random_ops",
+        ":client_testlib",
+        ":logging_ops",
+        ":random_ops_gen",
+    ],
+)
+
 py_library(
     name = "stateless_random_ops",
     srcs = ["ops/stateless_random_ops.py"],
@@ -3046,6 +3077,7 @@ py_library(
         ":special_math_ops",
         ":state_grad",
         ":state_ops",
+        ":stateful_random_ops",
         ":stateless_random_ops",
         ":string_ops",
         ":template",
diff --git a/tensorflow/python/ops/stateful_random_ops.py b/tensorflow/python/ops/stateful_random_ops.py
new file mode 100644
index 0000000000..c7a14f2015
--- /dev/null
+++ b/tensorflow/python/ops/stateful_random_ops.py
@@ -0,0 +1,241 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Operations for generating random numbers."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import sys
+
+import numpy as np
+
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import gen_stateful_random_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import variables
+from tensorflow.python.training.checkpointable import \
+tracking
+from tensorflow.python.util.tf_export import tf_export
+
+# A seed for random ops (stateful and stateless) will always be 1024
+# bits, all of which will be sent to the C++ code. The actual C++
+# implementation of some algorithms may only use a lower part of the bits.
+
+MAX_INT64 = 2**63 - 1
+MIN_INT64 = -(2**63)
+UINT64_SPAN = 2**64
+# 'Variable' doesn't support uint32 or uint64 yet (due to reasons explained in
+# b/111604096 and cl/171681867), so I use signed int here. I choose int64
+# instead of int32 here because `VarHandleOp` doesn't support int32 on GPU.
+SEED_TYPE = "int64"
+SEED_MIN = MIN_INT64
+SEED_MAX = MAX_INT64
+SEED_UINT_SPAN = UINT64_SPAN
+SEED_TYPE_BITS = 64
+SEED_BIT_MASK = 0xFFFFFFFFFFFFFFFF
+SEED_SIZE = 16  # in units of SEED_TYPE
+
+
+STATE_TYPE = SEED_TYPE
+RNG_ALG_PHILOX = 1
+DEFAULT_ALGORITHM = RNG_ALG_PHILOX
+
+
+def non_deterministic_seed():
+  """Makes a non-deterministic seed.
+
+  The implementation will be changed soon from pure Python to an op.
+
+  Returns:
+    a 1-D tensor.
+  """
+  return np.random.randint(
+      low=SEED_MIN, high=SEED_MAX + 1, size=SEED_SIZE,
+      dtype=SEED_TYPE)
+
+
+def _uint_to_int(n):
+  if n > SEED_MAX:
+    n = n - SEED_UINT_SPAN
+  return n
+
+
+PHILOX_STATE_SIZE = 3
+
+
+def _make_philox_state(seed):
+  """Makes a RNG state for Philox algorithm.
+
+  Args:
+    seed: an integer or 1-D tensor.
+
+  Returns:
+    a 1-D tensor.
+  """
+  int_types = (int,) if sys.version_info >= (3, 0) else (int, long)
+  if isinstance(seed, int_types):
+    # chop the Python integer (infinite precision) into chunks of SEED_TYPE
+    ls = []
+    for _ in range(PHILOX_STATE_SIZE):
+      ls.append(seed & SEED_BIT_MASK)
+      seed >>= SEED_TYPE_BITS
+    seed = ls
+  # to avoid overflow error from np.asarray
+  seed = list(map(_uint_to_int, seed))
+  seed = np.asarray(seed, dtype=STATE_TYPE)
+  if len(seed.shape) != 1:
+    raise ValueError(
+        "seed should only have one dimension; got shape: %s" % seed.shape)
+  seed = seed[0:PHILOX_STATE_SIZE]
+  # Padding with zeros on the right if too short
+  seed_size = seed.shape[0]
+  if seed_size < PHILOX_STATE_SIZE:
+    seed = np.pad(
+        seed, [(0, PHILOX_STATE_SIZE - seed_size)],
+        mode="constant",
+        constant_values=0)
+  assert seed.shape == (PHILOX_STATE_SIZE,), "Wrong seed.shape: %s" % seed.shape
+  return seed
+
+
+def _make_state_from_seed(seed, algorithm):
+  if algorithm == RNG_ALG_PHILOX:
+    return _make_philox_state(seed)
+  else:
+    raise ValueError("Unsupported algorithm id: %s" % algorithm)
+
+
+def create_rng_state(seed, algorithm=None):
+  """Creates a RNG state.
+
+  Args:
+    seed: an integer or 1-D tensor.
+    algorithm: (optional) an integer representing the RNG algorithm. If None, an
+      algorithm will be auto-selected.
+
+  Returns:
+    a 1-D tensor "rng_state" with:
+    * rng_state[0] is a value that identifies the RNG algorithm;
+    * rng_state[1:] holds the RNG state itself (size dependent on the
+        algorithm).
+  """
+  if algorithm is None:
+    # TODO(wangpeng): more sophisticated algorithm selection
+    algorithm = DEFAULT_ALGORITHM
+  state = _make_state_from_seed(seed, algorithm)
+  return np.concatenate((np.array([algorithm], dtype=STATE_TYPE), state),
+                        axis=None)
+
+
+def _shape_tensor(shape):
+  """Convert to an int32 or int64 tensor, defaulting to int64 if empty."""
+  if isinstance(shape, (tuple, list)) and not shape:
+    dtype = dtypes.int64
+  else:
+    dtype = None
+  return ops.convert_to_tensor(shape, dtype=dtype, name="shape")
+
+
+@tf_export("random.experimental.Generator")
+class Generator(tracking.Checkpointable):
+  """Random-number generator.
+
+  It uses Variable to manage its internal state.
+  """
+
+  def __init__(self, copy_from=None, seed=None, algorithm=None):
+    if copy_from is None:
+      if seed is None:
+        seed = non_deterministic_seed()
+      state = create_rng_state(seed, algorithm)
+      self._state_var = variables.Variable(state, dtype=STATE_TYPE)
+    else:
+      assert seed is None
+      state = copy_from.state
+      self._state_var = variables.Variable(state, dtype=STATE_TYPE)
+
+  def reset(self, seed):
+    algorithm = int(self.algorithm)
+    state = create_rng_state(seed, algorithm)
+    self._state_var.assign(state)
+
+  @property
+  def state(self):
+    return self._state_var
+
+  @property
+  def algorithm(self):
+    return self._state_var[0]
+
+  # The following functions return a tensor and as a side effect update
+  # self._state_var.
+  def standard_normal(self, shape, dtype=dtypes.float32):
+    return gen_stateful_random_ops.stateful_standard_normal(
+        self.state.handle, shape, dtype)
+
+  def normal(self, shape, mean=0.0, stddev=1.0, dtype=dtypes.float32,
+             name=None):
+    with ops.name_scope(name, "stateful_normal", [shape, mean, stddev]) as name:
+      shape = _shape_tensor(shape)
+      mean = ops.convert_to_tensor(mean, dtype=dtype, name="mean")
+      stddev = ops.convert_to_tensor(stddev, dtype=dtype, name="stddev")
+      rnd = self.standard_normal(shape, dtype)
+      return math_ops.add(rnd * stddev, mean, name=name)
+
+  # TODO(wangpeng): implement other distributions (`uniform`,
+  #   `truncated_normal`, etc.)
+  # TODO(wangpeng): implement `make_seeds`
+  # TODO(wangpeng): implement `make_generators`
+
+
+# It's not safe to create TF ops before `init_google` is called, so this is
+# initialized to None and get a value the first time `get_global_generator` is
+# called.
+global_generator = None
+
+
+@tf_export("random.experimental.get_global_generator")
+def get_global_generator():
+  global global_generator
+  if global_generator is None:
+    global_generator = Generator()
+  return global_generator
+
+
+@tf_export("random.experimental.set_global_generator")
+def set_global_generator(generator):
+  global global_generator
+  global_generator = generator
+
+
+# This function creates a new Generator object (and the Variable object within),
+# which does not work well with tf.function because (1) tf.function puts
+# restrictions on Variable creation thus reset_global_generator can't be freely
+# used inside tf.function; (2) redirecting a global variable to
+# a new object is problematic with tf.function because the old object may be
+# captured by a 'tf.function'ed function and still be used by it.
+# A 'tf.function'ed function only keeps weak references to variables,
+# so deleting a variable and then calling that function again may raise an
+# error, as demonstrated by
+# random_test.py/RandomTest.testResetGlobalGeneratorBadWithDefun .
+# The function 'set_global_generator' below also has this problem.
+@tf_export("random.experimental.reset_global_generator")
+def reset_global_generator(seed, algorithm=None):
+  global global_generator
+  if algorithm is None:
+    algorithm = int(global_generator.algorithm)  # preserve the old algorithm
+  global_generator = Generator(seed=seed, algorithm=algorithm)
diff --git a/tensorflow/python/ops/stateful_random_ops_test.py b/tensorflow/python/ops/stateful_random_ops_test.py
new file mode 100644
index 0000000000..92419a0770
--- /dev/null
+++ b/tensorflow/python/ops/stateful_random_ops_test.py
@@ -0,0 +1,156 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for stateful_random_ops.py."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.eager import def_function
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import errors_impl
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import gen_random_ops
+from tensorflow.python.ops import logging_ops
+from tensorflow.python.ops import stateful_random_ops as \
+random
+from tensorflow.python.platform import test
+
+
+class StatefulRandomOpsTest(test.TestCase):
+
+  def testCreateRNGStateIntSeed(self):
+    """Tests `create_rng_state` when `seed` is int."""
+    # using leading 'F' to test overflow tolerance
+    state = random.create_rng_state(0xFFFF222233334444FFAA666677778888,
+                                    random.RNG_ALG_PHILOX)
+    self.assertAllEqual(
+        list(map(random._uint_to_int,
+                 [random.RNG_ALG_PHILOX, 0xFFAA666677778888,
+                  0xFFFF222233334444] + [0] * (random.PHILOX_STATE_SIZE - 2))),
+        state)
+
+  @test_util.run_v2_only
+  @test_util.also_run_as_tf_function
+  def testEagerAndDefun(self):
+    """A simple test to make sure the op works in eager and defunned mode."""
+    random.get_global_generator().normal((3,))
+
+  @test_util.run_v2_only
+  def testOpSeedSelectionAfterSetSeed(self):
+    """Tests that op-seed selection is reset after reseting global generator.
+
+    Fixing GitHub issue 9171:
+    https://github.com/tensorflow/tensorflow/issues/9171
+    """
+    shape = (3,)
+    random.get_global_generator().reset(1)
+    a = random.get_global_generator().normal(shape)
+    random.get_global_generator().reset(1)
+    b = random.get_global_generator().normal(shape)
+    self.assertAllEqual(a, b)
+
+    # Now do the above again using accelerated ('defun'ed) computation
+    @def_function.function
+    def f():
+      return random.get_global_generator().normal(shape)
+
+    random.get_global_generator().reset(1)
+    c = f()
+    random.get_global_generator().reset(1)
+    d = f()
+    self.assertAllEqual(c, d)
+    self.assertAllEqual(a, c)
+
+  @test_util.run_v2_only
+  def testOpSeedSelectionNotSensitive(self):
+    """Test that op-seed selection is not sensitive to trivial changes.
+
+    Test that op-seed selection is not sensitive to trivial computation
+    (i.e. graph) changes.
+
+    Fixing b/32087099
+    """
+    def f(include_print):
+      shape = constant_op.constant([5])
+      if include_print:
+        shape = logging_ops.Print(shape, [shape])
+      return random.get_global_generator().normal(shape)
+
+    def compare(fst_includes_print, snd_includes_print):
+      random.get_global_generator().reset(50)
+      fst = f(fst_includes_print)
+      random.get_global_generator().reset(50)
+      snd = f(snd_includes_print)
+      self.assertAllEqual(fst, snd)
+      # Now do the above again using accelerated (defunned) 'f'.
+      # Running 'f' with two different Boolean arguments should cause
+      # two different graphs to be generated, hence demonstrating the
+      # insensitivity to graph changes.
+      f_acc = def_function.function(f)
+      random.get_global_generator().reset(50)
+      fst = f_acc(fst_includes_print)
+      random.get_global_generator().reset(50)
+      snd = f_acc(snd_includes_print)
+      self.assertAllEqual(fst, snd)
+
+    compare(False, False)
+    compare(True, True)
+    compare(True, False)
+
+  @test_util.run_v2_only
+  def testSameAsOldRandomOps(self):
+    """Tests that the generated numbers are the same as the old random_ops.py .
+    """
+    seed1, seed2 = 50, 60
+    # note how the two seeds for the old op correspond to the seed for the new
+    # op
+    random.get_global_generator().reset([0, seed2, seed1])
+    shape = constant_op.constant([2, 3])
+    dtype = dtypes.float32
+    # create a graph for the old op in order to call it many times
+    @def_function.function
+    def old():
+      return gen_random_ops.random_standard_normal(
+          shape, dtype=dtype, seed=seed1, seed2=seed2)
+
+    def new():
+      return random.get_global_generator().standard_normal(shape, dtype=dtype)
+
+    for _ in range(100):
+      self.assertAllEqual(old(), new())
+
+  @test_util.run_v2_only
+  def testResetGlobalGeneratorBadWithDefun(self):
+    """Demonstrates that reset_global_generator don't work properly with defun.
+    """
+    shape = (3,)
+
+    @def_function.function
+    def f():
+      return random.get_global_generator().normal(shape)
+
+    random.reset_global_generator(50)
+    with self.assertRaisesWithPredicateMatch(
+        errors_impl.NotFoundError, "Resource .+ does not exist"):
+      a = f()
+      random.reset_global_generator(50)
+      b = f()
+      self.assertAllEqual(a, b)
+
+
+if __name__ == "__main__":
+  test.main()
-- 
GitLab


From bcde9269af0b177e1047ca9b250234b57c5522e2 Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Tue, 15 Jan 2019 13:44:39 -0800
Subject: [PATCH 0771/1095] Fix arena_planner_test

Use proper tf_cc_test rule.

PiperOrigin-RevId: 229432697
---
 tensorflow/lite/BUILD | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/tensorflow/lite/BUILD b/tensorflow/lite/BUILD
index 8fca01624c..18fd1c1012 100644
--- a/tensorflow/lite/BUILD
+++ b/tensorflow/lite/BUILD
@@ -60,16 +60,13 @@ cc_library(
     ],
 )
 
-cc_test(
+tf_cc_test(
     name = "arena_planner_test",
     size = "small",
     srcs = ["arena_planner_test.cc"],
-    tags = [
-        "tflite_not_portable",
-    ],
     deps = [
         ":arena_planner",
-        "//tensorflow/core:lib",
+        "//tensorflow/core:tflite_portable_logging",
         "//tensorflow/lite/testing:util",
         "@com_google_googletest//:gtest",
     ],
-- 
GitLab


From d9c5cfc2ac1465146c8c6e19d1c189d07ab0c55b Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Tue, 15 Jan 2019 13:49:49 -0800
Subject: [PATCH 0772/1095] Internal change.

PiperOrigin-RevId: 229433602
---
 tensorflow/lite/g3doc/performance/gpu.md      | 230 +++++++++++++++
 .../lite/g3doc/performance/gpu_advanced.md    | 279 ++++++++++++++++++
 .../performance/images/android_gpu_demo.gif   | Bin 0 -> 1475626 bytes
 .../g3doc/performance/images/iosdebug.png     | Bin 0 -> 17762 bytes
 .../g3doc/performance/images/iosmetal.png     | Bin 0 -> 36053 bytes
 .../g3doc/performance/images/iosrelease.png   | Bin 0 -> 75374 bytes
 6 files changed, 509 insertions(+)
 create mode 100644 tensorflow/lite/g3doc/performance/gpu.md
 create mode 100644 tensorflow/lite/g3doc/performance/gpu_advanced.md
 create mode 100644 tensorflow/lite/g3doc/performance/images/android_gpu_demo.gif
 create mode 100644 tensorflow/lite/g3doc/performance/images/iosdebug.png
 create mode 100644 tensorflow/lite/g3doc/performance/images/iosmetal.png
 create mode 100644 tensorflow/lite/g3doc/performance/images/iosrelease.png

diff --git a/tensorflow/lite/g3doc/performance/gpu.md b/tensorflow/lite/g3doc/performance/gpu.md
new file mode 100644
index 0000000000..14ca54a158
--- /dev/null
+++ b/tensorflow/lite/g3doc/performance/gpu.md
@@ -0,0 +1,230 @@
+# TensorFlow Lite GPU Delegate Tutorial
+
+[TensorFlow Lite](https://www.tensorflow.org/lite) supports several hardware
+accelerators. This document describes how to preview the experimental GPU backend using the
+TensorFlow Lite delegate APIs on Android and iOS.
+
+GPUs are designed to have high throughput for massively parallelizable
+workloads. Thus, they are well-suited for deep neural nets, which consist of a
+huge number of operators, each working on some input tensor(s) that can be
+easily divided into smaller workloads and carried out in parallel, typically
+resulting in lower latency. In the best scenario, inference on the GPU may now
+run fast enough for previously not available real-time applications.
+
+Unlike CPUs, GPUs compute with 16-bit or 32-bit floating point numbers and do
+not require quantization for optimal performance.
+
+Another benefit with GPU inference is its power efficiency. GPUs carry out the
+computations in a very efficient and optimized manner, so that they consume less
+power and generate less heat than when the same task is run on CPUs.
+
+## Demo App Tutorials
+
+The easiest way to try out the experimental GPU delegate is to follow the below tutorials, which go through building our classification demo applications with GPU support. The GPU code is only binary for now; it will be open-sourced soon. Once you understand how to get our demos working, you can try this out on your own custom models.
+
+### Android (with Android Studio)
+
+Note: This requires OpenGL ES 3.1 or higher.
+
+#### Step 1. Clone the TensorFlow source code and open it in Android Studio
+
+```
+git clone https://github.com/tensorflow/tensorflow
+```
+
+#### Step 2. Edit `app/build.gradle` to use the experimental GPU AAR
+
+Replace the existing `tensorflow-lite` package in the existing `dependencies`
+block.
+
+```
+dependencies {
+    ...
+    // implementation 'org.tensorflow:tensorflow-lite:0.0.0-nightly'
+    implementation 'org.tensorflow:tensorflow-lite:0.0.0-gpu-experimental'
+}
+```
+
+#### Step 3. Build and run
+
+Run → Run ‘app’.  When you run the application you will see a button for
+enabling the GPU. Change from quantized to a float model and then click GPU to
+run on the GPU.
+
+![running android gpu demo and switch to gpu](images/android_gpu_demo.gif)
+
+### iOS (with XCode)
+
+Note: This requires XCode v10.1 or later.
+
+#### Step 1. Get the demo source code and make sure it compiles.
+
+Follow our iOS Demo App [tutorial](https://www.tensorflow.org/lite/demo_ios).
+This will get you to a point where the unmodified iOS camera demo is working
+on your phone.
+
+
+#### Step 2. Modify the Podfile to use the TensorFlow Lite GPU CocoaPod
+
+We have built a binary CocoaPod that includes the GPU delegate. To switch the
+project to use it, modify the
+`tensorflow/tensorflow/lite/examples/ios/camera/Podfile` file to use
+the `TensorFlowLiteGpuExperimental` pod instead of `TensorFlowLite`.
+
+```
+target 'YourProjectName'
+  # pod 'TensorFlowLite', '1.12.0'
+  pod 'TensorFlowLiteGpuExperimental'
+```
+
+#### Step 3. Enable the GPU Delegate
+
+You will need to change two `#define` flags in `CameraExampleViewController.h`
+to enable the GPU delegate. First, change `TFLITE_USE_CONTRIB_LITE` from 1 to 0
+since TensorFlow Lite has moved from TensorFlow contrib into core.
+
+```c
+#define TFLITE_USE_CONTRIB_LITE 0
+```
+
+Next, change `TFLITE_USE_GPU_DELEGATE` from 0 to 1, to enable the code that will
+use the GPU delegate.
+
+```c
+#define TFLITE_USE_GPU_DELEGATE 1
+```
+
+#### Step 4. Build and run the demo app
+
+After following the previous step, you should be able to run the app.
+
+
+#### Step 5. Release mode.
+
+While in Step 4 you ran in debug mode, to get better performance, you should
+change to a release build with the appropriate optimal Metal settings. In
+particular, To edit these settings go to the `Product > Scheme > Edit
+Scheme...`. Select `Run`. On the `Info` tab, change `Build Configuration`, from
+`Debug` to `Release`, uncheck `Debug executable`.
+
+![setting up release](images/iosdebug.png)
+
+Then
+click the `Options` tab and change `GPU Frame Capture` to `Disabled` and
+`Metal API Validation` to `Disabled`.
+
+![setting up metal options](images/iosmetal.png)
+
+Lastly make sure Release only builds on 64-bit architecture. Under `Project
+navigator -> tflite_camera_example -> PROJECT -> tflite_camera_example -> Build
+Settings` set `Build Active Architecture Only > Release` to Yes.
+
+![setting up release options](images/iosrelease.png)
+
+## Trying the GPU Delegate on your own model
+
+### Android
+
+Look at the demo to see how to add the
+delegate. In your application, add the AAR as above, import
+`org.tensorflow.lite.experimental.GpuDelegate` module, and use the`addDelegate`
+function to register the GPU delegate to the interpreter:
+
+```java
+import org.tensorflow.lite.Interpreter;
+import org.tensorflow.lite.experimental.GpuDelegate;
+
+// Initialize interpreter with GPU delegate
+GpuDelegate delegate = new GpuDelegate();
+Interpreter.Options options = (new Interpreter.Options()).addDelegate(delegate);
+Interpreter interpreter = new Interpreter(model, options);
+
+// Run inference
+while (true) {
+  writeToInput(input);
+  interpreter.run(input, output);
+  readFromOutput(output);
+}
+
+// Clean up
+delegate.close();
+```
+
+### iOS
+
+In your application code, include the GPU delegate header and call the
+`Interpreter::ModifyGraphWithDelegate` function to register the GPU delegate to
+the interpreter:
+
+```cpp
+#import "tensorflow/lite/delegates/gpu/metal_delegate.h"
+
+// Initialize interpreter with GPU delegate
+std::unique_ptr<Interpreter> interpreter;
+InterpreterBuilder(*model, resolver)(&interpreter);
+auto* delegate = NewGpuDelegate(nullptr);  // default config
+if (interpreter->ModifyGraphWithDelegate(delegate) != kTfLiteOk) return false;
+
+// Run inference
+while (true) {
+  WriteToInputTensor(interpreter->typed_input_tensor<float>(0));
+  if (interpreter->Invoke() != kTfLiteOk) return false;
+  ReadFromOutputTensor(interpreter->typed_output_tensor<float>(0));
+}
+
+// Clean up
+interpreter = nullptr;
+DeleteGpuDelegate(delegate);
+```
+
+## Supported Models and Ops
+
+With the release of the GPU delegate, we included a handful of models that can
+be run on the backend:
+
+* [MobileNet v1 (224x224) image classification](https://ai.googleblog.com/2017/06/mobilenets-open-source-models-for.html) [[download]](https://storage.googleapis.com/download.tensorflow.org/models/tflite/gpu/mobilenet_v1_1.0_224.tflite)
+<br /><i>(image classification model designed for mobile and embedded based vision applications)</i>
+* [DeepLab segmentation (257x257)](https://ai.googleblog.com/2018/03/semantic-image-segmentation-with.html) [[download]](https://storage.googleapis.com/download.tensorflow.org/models/tflite/gpu/deeplabv3_257_mv_gpu.tflite)
+<br /><i>(image segmentation model that assigns semantic labels (e.g., dog, cat, car) to every pixel in the input image)</i>
+* [MobileNet SSD object detection](https://ai.googleblog.com/2018/07/accelerated-training-and-inference-with.html) [[download]](https://storage.googleapis.com/download.tensorflow.org/models/tflite/gpu/mobile_ssd_v2_float_coco.tflite)
+<br /><i>(image classification model that detects multiple objects with bounding boxes)</i>
+* [PoseNet for pose estimation](https://github.com/tensorflow/tfjs-models/tree/master/posenet) [[download]](https://storage.googleapis.com/download.tensorflow.org/models/tflite/gpu/multi_person_mobilenet_v1_075_float.tflite)
+<br /><i>(vision model that estimates the poses of a person(s) in image or video)</i>
+
+To see a full list of supported ops, please see the [advanced documentation](gpu_advanced.md).
+
+## Non-supported models and ops
+
+If some of the ops are not supported by the GPU delegate, the framework will
+only run a part of the graph on the GPU and the remaining part on the CPU.  Due
+to the high cost of CPU/GPU synchronization, a split execution mode like this
+will often result in a performance slower than when the whole network is run on
+the CPU alone.  In this case, the user will get a warning like:
+
+```
+WARNING: op code #42 cannot be handled by this delegate.
+```
+
+We did not provide a callback for this failure, as this is not a true run-time
+failure, but something that the developer can observe while trying to get the
+network to run on the delegate.
+
+## Tips for optimization
+
+Some operations that are trivial on the CPU may have a high cost for the GPU.
+One class of such operation is various forms of reshape operations, including
+`BATCH_TO_SPACE`, `SPACE_TO_BATCH`, `SPACE_TO_DEPTH`, and so forth. If those ops
+are inserted into the network just for the network architect's logical thinking,
+it is worth removing them for performance.
+
+On GPU, tensor data is sliced into 4-channels. Thus, a computation on a tensor
+of shape `[B,H,W,5]` will perform about the same on a tensor of shape
+`[B,H,W,8]` but significantly worse than `[B,H,W,4]`.
+
+In that sense, if the camera hardware supports image frames in RGBA, feeding
+that 4-channel input is significantly faster as a memory copy (from 3-channel
+RGB to 4-channel RGBX) can be avoided.
+
+For best performance, do not hesitate to retrain your classifier with a mobile-
+optimized network architecture. That is a significant part of optimization for
+on-device inference.
diff --git a/tensorflow/lite/g3doc/performance/gpu_advanced.md b/tensorflow/lite/g3doc/performance/gpu_advanced.md
new file mode 100644
index 0000000000..d0ae123613
--- /dev/null
+++ b/tensorflow/lite/g3doc/performance/gpu_advanced.md
@@ -0,0 +1,279 @@
+# TensorFlow Lite on GPU
+
+[TensorFlow Lite](https://www.tensorflow.org/mobile/tflite/) supports several
+hardware accelerators.  This document describes how to use the GPU backend using
+the TensorFlow Lite delegate APIs on Android (requires OpenGL ES 3.1 or higher)
+and iOS (requires iOS 8 or later).
+
+GPUs are designed to have high throughput for massively parallelizable
+workloads.  Thus, they are well-suited for deep neural nets which consists of a
+huge number of operators, each working on some input tensor(s) that can be
+easily divided into smaller workloads and carried out in parallel, typically
+resulting in lower latency.  In the best scenario, inference on the GPU may now
+run fast enough and now become suitable for real-time applications if it was not
+before.
+
+GPUs do their computation with 16-bit or 32-bit floating point numbers and do
+not require quantization for optimal performance unlike the CPUs.  If
+quantization of your neural network was not an option due to lower accuracy
+caused by lost precision, such concern can be discarded when running deep neural
+net models on the GPU.
+
+Another benefit that comes with GPU inference is its power efficiency.  GPUs
+carry out the computations in a very efficient and optimized way, so that they
+consume less power and generate less heat than when the same task is run on the
+CPUs.
+
+TensorFlow Lite on GPU supports the following ops in 16-bit and 32-bit float
+precision:
+
+* `ADD v1`
+* `AVERAGE_POOL_2D v1`
+* `CONCATENATION v1`
+* `CONV_2D v1`
+* `DEPTHWISE_CONV_2D v1-2`
+* `FULLY_CONNECTED v1`
+* `LOGISTIC v1`
+* `MAX_POOL_2D v1`
+* `MUL v1`
+* `PAD v1`
+* `PRELU v1`
+* `RELU v1`
+* `RELU6 v1`
+* `RESHAPE v1`
+* `RESIZE_BILINEAR v1`
+* `SOFTMAX v1`
+* `STRIDED_SLICE v1`
+* `SUB v1`
+* `TRANSPOSE_CONV v1`
+
+## Basic Usage
+
+### Android
+
+Using TensorFlow Lite on GPU is achieved via `TfLiteDelegate`.  In Java, you can
+specify the GpuDelegate through `Interpreter.Options`.
+
+```java
+// NEW: Prepare GPU delegate.
+GpuDelegate delegate = new GpuDelegate();
+Interpreter.Options options = (new Interpreter.Options()).addDelegate(delegate);
+
+// Set up interpreter.
+Interpreter interpreter = new Interpreter(model, options);
+
+// Run inference.
+writeToInputTensor(inputTensor);
+interpreter.run(inputTensor, outputTensor);
+readFromOutputTensor(outputTensor);
+
+// Clean up.
+delegate.close();
+```
+
+### iOS
+
+Using TensorFlow Lite on GPU is as simple as getting the GPU delegate via
+`NewGpuDelegate()` and then passing it to
+`Interpreter::ModifyGraphWithDelegate()` instead of calling
+`Interpreter::AllocateTensors()`:
+
+```c++
+// Set up interpreter.
+auto model = FlatBufferModel::BuildFromFile(model_path);
+if (!model) return false;
+tflite::ops::builtin::BuiltinOpResolver op_resolver;
+std::unique_ptr<Interpreter> interpreter;
+InterpreterBuilder(*model, op_resolver)(&interpreter);
+
+// NEW: Prepare GPU delegate.
+auto* delegate = NewGpuDelegate(nullptr);  // default config
+if (interpreter->ModifyGraphWithDelegate(delegate) != kTfLiteOk) return false;
+
+// Run inference.
+WriteToInputTensor(interpreter->typed_input_tensor<float>(0));
+if (interpreter->Invoke() != kTfLiteOk) return false;
+ReadFromOutputTensor(interpreter->typed_output_tensor<float>(0));
+
+// Clean up.
+DeleteGpuDelegate(delegate);
+```
+
+*IMPORTANT:* When calling `Interpreter::ModifyGraphWithDelegate()` or
+`Interpreter::Invoke()`, the caller must have a `EGLContext` in the current
+thread and `Interpreter::Invoke()` must be called from the same `EGLContext`.
+If such `EGLContext` does not exist, the delegate will internally create one,
+but then the developer must ensure that `Interpreter::Invoke()` is always called
+from the same thread `Interpreter::ModifyGraphWithDelegate()` was called.
+
+## Advanced: Delegate Options for iOS
+
+There are a couple of GPU options that can be set and passed on to
+`NewGpuDelegate()`:
+
+```c++
+struct GpuDelegateOptions {
+  // Allows to quantify tensors, downcast values, process in float16 etc.
+  bool allow_precision_loss;
+
+  enum class WaitType {
+    // waitUntilCompleted
+    kPassive,
+    // Minimize latency. It uses active spinning instead of mutex and consumes
+    // additional CPU resources.
+    kActive,
+    // Useful when the output is used with GPU pipeline then or if external
+    // command encoder is set
+    kDoNotWait,
+  };
+  WaitType wait_type;
+};
+```
+
+When option is set to `nullptr` as shown in the Basic Usage, it translates to:
+
+```c++
+const GpuDelegateOptions options = {
+  .allow_precision_loss = false,
+  .wait_type = kGpuDelegateOptions::WaitType::Passive,
+};
+```
+
+While it is convenient to just supply `nullptr`, it is recommended to explicitly
+set the options to avoid any unexpected artifacts in case default values are
+changed.
+
+## Advanced: Input/Output Buffers
+
+To do computation on the GPU, data must be made available to the GPU which often
+translates to performing a memory copy.  It is desirable not to cross the
+CPU/GPU memory boundary if possible, as this can take up a significant amount of
+time.  Usually, such crossing is inevitable, but in some special cases, one or
+the other can be omitted.
+
+If the network's input is an image already loaded in the GPU memory, e.g. a GPU
+texture containing the camera feed, it can stay in the GPU memory without ever
+entering the CPU memory.  Similarly, if the network's output is in the form of a
+renderable image, e.g.
+[image style transfer](https://www.cv-foundation.org/openaccess/content_cvpr_2016/papers/Gatys_Image_Style_Transfer_CVPR_2016_paper.pdf),
+it can be directly displayed on the screen.
+
+To let users achieve best performance, TensorFlow Lite makes it possible for
+them to directly read from/write to the TensorFlow hardware buffer and bypass
+avoidable memory copies.
+
+### Android
+
+Assuming the image input is in the GPU memory, it must be first converted to a
+OpenGL Shader Storage Buffer Object (SSBO).  One can associate a TfLiteTensor
+with user-prepared SSBO with `Interpreter.bindGlBufferToTensor()`.
+
+*IMPORTANT:* `Interpreter.bindGlBufferToTensor()` must be called before
+`Interpreter.modifyGraphWithDelegate()`.
+
+```java
+// Ensure a valid EGL rendering context.
+EGLContext eglContext = eglGetCurrentContext();
+if (eglContext.equals(EGL_NO_CONTEXT)) return false;
+
+// Create a SSBO.
+int[] id = new int[1];
+glGenBuffers(id.length, id, 0);
+glBindBuffer(GL_SHADER_STORAGE_BUFFER, id[0]);
+glBufferData(GL_SHADER_STORAGE_BUFFER, inputSize, null, GL_STREAM_COPY);
+glBindBuffer(GL_SHADER_STORAGE_BUFFER, 0);  // unbind
+int inputSsboId = id[0];
+
+// Create interpreter.
+Interpreter interpreter = new Interpreter(tfliteModel);
+Tensor inputTensor = interpreter.getInputTensor(0);
+GpuDelegate gpuDelegate = new GpuDelegate();
+// The buffer must be bound before the delegate is installed.
+gpuDelegate.bindGlBufferToTensor(inputTensor, inputSsboId);
+interpreter.modifyGraphWithDelegate(gpuDelegate);
+
+// Run inference; the null input argument indicates use of the bound buffer for input.
+fillSsboWithCameraImageTexture(inputSsboId);
+float[] outputArray = new float[outputSize];
+interpreter.runInference(null, outputArray);
+```
+
+Similar approach can be applied to the output tensor.  In that case,
+`Interpreter.Options.setAllowBufferHandleOutput(true)` should be passed on, to
+disable the default copying the network's output from GPU memory to CPU memory.
+
+```java
+// Ensure a valid EGL rendering context.
+EGLContext eglContext = eglGetCurrentContext();
+if (eglContext.equals(EGL_NO_CONTEXT)) return false;
+
+// Create a SSBO.
+int[] id = new int[1];
+glGenBuffers(id.length, id, 0);
+glBindBuffer(GL_SHADER_STORAGE_BUFFER, id[0]);
+glBufferData(GL_SHADER_STORAGE_BUFFER, outputSize, null, GL_STREAM_COPY);
+glBindBuffer(GL_SHADER_STORAGE_BUFFER, 0);  // unbind
+int outputSsboId = id[0];
+
+// Create interpreter.
+Interpreter.Options options = (new Interpreter.Options()).setAllowBufferHandleOutput(true);
+Interpreter interpreter = new Interpreter(tfliteModel, options);
+Tensor outputTensor = interpreter.getOutputTensor(0);
+GpuDelegate gpuDelegate = new GpuDelegate();
+// The buffer must be bound before the delegate is installed.
+gpuDelegate.bindGlBufferToTensor(outputTensor, outputSsboId);
+interpreter.modifyGraphWithDelegate(gpuDelegate);
+
+// Run inference; the null output argument indicates use of the bound buffer for output.
+ByteBuffer input = getCameraImageByteBuffer();
+interpreter.runInference(input, null);
+renderOutputSsbo(outputSsboId);
+```
+
+### iOS
+
+Assuming the image input is in the GPU memory, it must be first converted to a
+`MTLBuffer` object for Metal.  One can associate a TfLiteTensor with a
+user-prepared `MTLBuffer` with `BindMetalBufferToTensor()`.
+
+*IMPORTANT:* `BindMetalBufferToTensor()` must be called before
+`Interpreter::ModifyGraphWithDelegate()`.
+
+*IMPORTANT:* By default, the inference output is copied from GPU memory to CPU
+memory implicitly by the framework.  This behavior can be turned off by calling
+`Interpreter::SetAllowBufferHandleOutput(true)` during initialization.  To copy
+the inference output from GPU memory to CPU memory, explicit
+`Interpreter::EnsureTensorDataIsReadable()` calls are required for each output
+tensor.
+
+```c++
+// Prepare GPU delegate.
+auto* delegate = NewGpuDelegate(nullptr);
+interpreter->SetAllowBufferHandleOutput(true);  // disable default gpu->cpu copy
+if (!BindMetalBufferToTensor(delegate, interpreter->inputs()[0], user_provided_input_buffer)) return false;
+if (!BindMetalBufferToTensor(delegate, interpreter->outputs()[0], user_provided_output_buffer)) return false;
+if (interpreter->ModifyGraphWithDelegate(delegate) != kTfLiteOk) return false;
+
+// Run inference.
+if (interpreter->Invoke() != kTfLiteOk) return false;
+```
+
+## Tips and Tricks
+
+* Some operations that are trivial on CPU side may be high cost in GPU land.
+  One class of such operation is various forms of reshape operations (including
+  `BATCH_TO_SPACE`, `SPACE_TO_BATCH`, `SPACE_TO_DEPTH`, etc.).  If those ops
+  are inserted into the network just for the network architect's logical
+  thinking, it is worth removing them for performance.
+
+* On GPU, tensor data is sliced into 4-channels.  Thus, a computation on a
+  tensor of shape `[B, H, W, 5]` will perform about the same on a tensor of
+  shape `[B, H, W, 8]`, but significantly worse than `[B, H, W, 4]`.
+
+* In that sense, if the camera hardware supports image frames in RGBA, feeding
+  that 4-channel input is significantly faster as a memory copy (from 3-channel
+  RGB to 4-channel RGBX) can be avoided.
+
+* For best performance, do not hesitate to re-train your classifier with
+  mobile-optimized network architecture.  That is a significant part of
+  optimization for on-device inference.
diff --git a/tensorflow/lite/g3doc/performance/images/android_gpu_demo.gif b/tensorflow/lite/g3doc/performance/images/android_gpu_demo.gif
new file mode 100644
index 0000000000000000000000000000000000000000..31aee24c34b6451727abffd95bb7f2f0d40f55af
GIT binary patch
literal 1475626
zcmZ?wbhEHbY+!O={4TG_z`&rwz~I5aki)<*fq`KQ1H%mlhCg6Y6%_^#6^0xYh6ySR
zTT~css4)BiD^>Ae@bF;B@nD$X!LY@H;f4ppAFzIv90rdZhMXLR2{{Z~au{ypF#G`<
zsxpDWV**3Y1cnI{7`9AcxG{m@57;J^Eesx87;?5SOxVJ(WedZNEewCa)~noL@VLQ{
zbAw^R4Tdc@7;fBP_ycy6${z-gKMXm47$*E-*z$+r#vg`1AcuykFob$Agyt}WPGAV#
z!Vr3cA@mPJD1%C<ib|-5N@$Ks=meF}Eh?cmR6_r#gfe)9s(6HYc!cJ7gii1X-Qp2?
z!z1*MM<_#1s7g+#M^0!?PUwW3&@DNkH*!M%<b*Oz2vwO7>M<cSXF}+N387migx;7C
z`e#BY!<JB$EukJ;LUXo+PS_H<WlQLdEunw5gfiR+Rk;!BaU(S6M(Bhap<8Z*-nbF^
z=SC>QpHP)Qp&oxibN+-*_!GM2Pw0(5p@05_GBB)CVOZtCuquaP)dYrBTNqZ|U|98s
zVHJbQDixJg9xAJHR8~z;S+zxF)eV(Ze^gd6c&t+KSmoidD#v5h1dml)JXYQCSoOza
z6+_M{m7G-`IjeGVR!zuRwIyfOjht0~a#k@+Sfw&ymB)ltITKb*n6PTggjF{ttok!y
z6~mTQDqB{0Y+04FWz~c&tF~-ebz{q_KU-EY+*qY@W0l8^RXI0SO}MdY%Z*hxZmjxq
zV->@nRVsg0dHh+G^JmqBKdZL<S#{&jsy}~LF);jBVfgRC@IQy){{)8rTNwV|VEF%s
z;Xi}Qe-)Mg9xDHHRQ^v;`M*Ww{|%M@e^mZ6c>GuK`0wHIKgZ+$1dsn)JpSMC`2WY_
zKSR!cm7M<`IsbEV{!hsHza{7Yjhz2~a{e<+_^&eIzsH3CITQX*nDBqgg#R}t{QooI
zKf{*)DqH@0Z26zF<^O~&|F>-Ue`Cx4KU@AY-1x6@<G;s^|2a4QPq^`a%Z>jxZv6jq
z<3Gcn|0;j}d;IyI^XLDBKmWJ<`G4cj|382JEB@#9a}5c0b_{Se(lcOYWME)W{K>*1
z!XVF}!@$760Lmr|9RC^aZYg|x%2%?9O;Aq9V#2~h?R?yNJ~z6OW*bR(7ic=S9XZw)
zbD~4Uv&-k?WVPT$I+j{b)r=M&+jB=nIPL5-eeWV8&COHyMy<`-n(bOLWm@R!YpSuU
zwrE~n?zh-Zc5Buht-w(CMA_w~Q!D}(ZE5;?>#KHHVbtcly}LwK2d(nDyQ%l~wsjWU
z(!!tK<^HA>5%}oHA6XrVoW#I(Vg9%`TlMd6u8@7~yL<Ykr)#H6xBp!oTJU^-7q9WU
zo=V+2q058yLw&!$4O;U0SVorVmbn4Z>!J!D9jZ2ee?O*%{lD%Wi;s_`Yo`b1*;G#N
zT{>-g=ed<W%Z)B9bw3~X*XsMr)Ljk#_sh+-`SIe#`PJt6^RARk&W+ltc(v<l&7xed
zty)J@Pk*?#XX3&+)%TaCFQ{Ty>v-VEqju#%GoKsB!xkYo54JLiyd{ZkGE*JBq;*?8
zx;hjOxg7J3p5x)xrLpS7qb{wsD@vN%wgf2$Y~5hwrM5<6E0^};4WAZ=Ixh+n4RQ-g
z6%F-~^ArsW*d+RNN>G^R)2U%;olmDlm7RP#rMq$J$@p0lEU%@-*1mi?D{EV3>a5tQ
zlGD@*54~JA+kex{ry5pWj$8qz*Iu0R`zLzq{=$EZ$D<b=ustr_F73wka>=AHua`@w
zrFA8ip3HrfxqN}wcg5Nz+k&)KYRgGoS~a7pYGJi_*#GZY?N<|@%v&o_Ib(G-2X9EA
zCP!$<gbf^`DLETY$a%kUSo)+qaGgTo5<$t;eOo4Z-H3};^1ZXlVsh8@X<Iy{zuqts
z33QfJy}B!4>*;mDVMkX_oBL%^=e1aGInDK<ULSu?+5i5P>WA0~cU?amV4BZ5O)Jq%
z=T;(LyQkPxHuatl2St;63Nouw+tktpon;F5`n}37+VA~#chsqO>lSfJZhg3YcKspE
ztj}jHm@_u#_r1Eqs(wCFJ+{;(Tm4JvUD>r?N`3S*<~0TAb9}7{slTD4*)A2nxq<1`
ztGFu(-g^^TMW*QnI5E$(eYkm6-j7wDIa^%3?igH*Rq`{~Hseuu_2K^>8ao<qo$%^g
z^WlebsH<Gxl)i4|Je8on<f-ZN%9m}OIWNYq@8`4fxT7251HOF~f3YO~-p`kFHfLJy
zE$VZ6KBp{i-OeXDXX8|#ZvDQJbAe0lLu>J18L2(>2S11HS+y$Y_lx`qY<J66pLe%k
zSs@)LFV`k<RZG_Gs&{(rimSa@uYUYGJ15JdGs!)7wRql?+I8>p`}h74f7CTm`iS2W
zdzW<gEC2uV{=DxPHF34klm|{1yTUx3gx2L)2E>?Editnvtm9ggB5b$w;?AtZDe|mu
znsuASPro=cb@P;^I?0aQ$JZ!{*E8z0_y@YjPAQsx?ab4*qQcY9&n)8F%+O}ApmRpt
ze;H?&6A~XiEVoF+^0_%}o3bI9t9wzJi?i#IoMXmw3{M}K5MeZB=do_LUr7?jImE*{
zlDOC0INw>fy>qtdbNhlC<@2*P%iR0(Ch*itmkVEZ2)=JvZgu77;|dl@*^g6>XC`sq
zRutDX`C`#+73qAgp5@C5)~4s#oo7;RoH*vWwRF?u3y&|XNH9_g>Ql;FD&wlPck1@A
zPrj~73l(~EM5Zj*u(-fhC`iwC#p%*7LY{W71SfTD*;KRe-Hh$4=Ne_UTc=oE_FCk9
z*LZp7&Z@}Cvu0%ry)~M?Bs8N@MAh)hhX(g^k0rYTdy?eD=ZJf~+_`z~<W+qY>x|F8
zz8IV#v2(f2Qs)15{+;V1yS=6^tlMn&^5`n(4U^8TNlUeleW(>7u$E(^+N?EQe5=EM
zoX&79Dm}ob7*KmPYk`NXh2zXCPoh<`9(dnM)LV36nb;DGE(1r8V2Lh)_{lr3b6#I$
zWN_`va<kTjJee7Wg*AFf9<q-Fwbc$wajjXox<M<%Sj}MdgDnZJv3JuZyfWBWvnJ&e
z(_`*Nt*P=8lelwp9$OY4yS8nYsHHu3)I|%^ElVYPZ977isC{2`eb(NsQF9M1iDO?a
z=eO+bWOmCn>WW`3zUWM`E8w1Z_l4Q^+?|(P4XOflqAgE5@b-zedUNU4?CiV9b#lvU
zhqVU5(r2!j@p;~!vh2-`CXqG&uZV5iv3G4==4r9e4`SN;wyw=tUvlN4)bv^LU2|V<
z*`;>dvV5jnm+t%-OO(R|{3~wkeOKE1UAkB=zw$%P@(TUby`Pd!RNdJf_vg`a+qbdm
zdnYlel<oR0t#!k&_7zX+Z>ueFKYp6;W$?>!5mplX=d04d@ofHjy+>^?3ce`IJ?wA~
za+<^*6_KYl>p^E;kT6GF_EISoFWno8ZGk;zn#R6{k<G8xuU1pl)p@drckZr7QtY9c
z&atieovZrJf3@hcjgjF0f5}<?_m4I|-$lF|m7G<TE88r5yae|~9W_ogY^|+K+E;T+
zPyCR8S$FdB&K(P`{VQOYTc5jH=B<bOvj>9z`z5%ZlyTfVVYQrnea{}_`0rhwiyrnc
zTW^-U&UIzs3y!k$TuwT-MD|9o+h*@SFD<I&nR3c2f6)}(Q(sc!H{BHJRVuJZa@RId
zjJnqM?aH#Wx0$skPlv7l=HX%Ztnhlvl-+N_+7ixv%`STzZSXs+ZQow+f{#YoPsD!T
z2v1t~GVS-B-6k(@-c?a6UHNGDgUH=;ZtVNERs3k!n+?T#9`HT2Gj|CuD)zAXePy|A
z@z=PzUAOP7K9qNEMQA|9wAo=CXTLj@rUrJ%y}o~9QidDjx;1R#8y5feU9eThCTPlo
zBTGb<D70;LWs{!UZMIfnqehWuRA2F*V+!I6_grhbJWuZAF^&KHm8XkrLeF<9od~$~
zkgu~=d?i;-r~lH*kaE4lGS){|hMbi+ykl4BgrzIJ)ch@XpV@WHh+m}grH^s+-&dP0
z*^b|Jb&iVt-1B0cuvv~r^WrNr&BJ4BYtN?(8?^A9{^n}_%Q4qr#V<RR?I%}l+BDI0
z;<Df07Ul2d$e+I~weM9hx7E~dAC|lAc@&s^>#+VgcXdCRO}n>5KfD!n+@sS*cY?^z
z$S=Q4=eY~5GmNji!*~0}iupB97c9xKefo0S`LsJF@vC>p-wbnKyuxhT`s&=blP2%p
zvr&J$>6g0&()CrJU(fq=zV_ZTefzyH{;mJ>qU!wwne)33FE8KmT{_|I(f47DEEfM=
z|B4?-)%>CON%OF)fLvm$@q~vT_9w7B?>Fa9=04(hbHOqHnjdl>|Nr+j4AxQ?)n!Zd
zV=G~v(cpDm&+=GG&d20H)nXsBV!n!ofUsiz8I3yIMJ%VKct33HUl`K7P%<E-DY~qP
z>xNN{Qd*{{XT~ws^lxDy;_j1DN>6@FGnddRe&$}3mR=c}{%L7?x_L`!h;hR)Y4;A}
zGaK#RDaOwz$%sF$lX*NNWQAESL*|>uiW8@LJuu0fnijc!OZkQ+N)`vL7lu_NZ>Y$c
zQIWGEYt7e+0*UO4ORGYIvOS6_?KTzfG~>2A)b8=HeKkWz(~R~O4xdHsmF$zNx;)yu
zPx##Z&ywq+RNcdDxyyuW=XTax({g9ssDARL#QbsYaq%u0r`m^4HJ?q*o7W`RYEt_{
zOzX8s&COPgRZKxkHMDmf3gDaCy~nBGpIO1a6Ws?m0{Ny1%0CR0+giZOUSD1uWS*#`
zoT~hlK}6y>U$O#M@AG<D_o(YT8dx%nv>vD4nbGu6L)3PAv153MbhClQx5gJ8jqf;$
zq&-4HO2v2{hpGsx-q+}RH_1EVamoIVO`9Z2lf#<}zDk!rk(nD*7Fix%-IgBreL~=2
zP3B6O)aP;m91|ZHw&pL-=yEkHWcL@#EU#=YuV^=~Ymb~|#-_1TX-_iu{LqTU8`_Lc
zwyjxKdC<k@(*L6!#Sv9DoE=*lvh8lR`zB7V=a{lsJ?6mD$-N!XfkjnbJJnAu@!j&>
zVsb=mdvIs+g6f$wI`>@f-2Z**4fFU#7Sm2Hk4yV$^XzIo<FUA%m*W0~YJNEy`_0tv
zsB?Ewcu&^M?vKt1`y%St+5#D-C-6<IlMJ+fyIfGlG5_R_8Is2mEiWdDwdjhei|AeD
z+jNlYjY1To#gy4A`;$@|VnmXSw`D(mG4lh*Ebq2if!kb+uQ!U{&@(yKxXCkQ<w~a|
z$+Hcg<!A?Ycun_yQ=pN`*37^u(<#(^;2@``#{{cj_tP)aEKkgdXjin6NOxPT@JunD
z&$H#0rR;%;E%zid&U=R2{@*e8VqojDoe_VQ#w$!pjL}e<dUc+*WZT*;ky@ROP2wWW
z%@RvfvpS|{t@t*7LU>1!hiK=IDeg<urnBc%dCaQc!M9nUgO6)rdxr>D)B^9yg<`_0
zVp$7LPSNjk_OXBD*e$$hi-A(q&Q1-l>ATgtyFTSE^_aHqhvuWQX=@~Ee_ok(<)__y
z;l)Re$40-LdMUV;t5uPGYyR#Ev;C6-_gzY8c-fs75a`cU@G9IPrJ>;U=fIwsORKK;
z2(s6&N#MTtB52D)tJM=`&W)V)MoC{U%uvcp()|0fjI3FVpZk2K^nI={v}!NTS=Cn&
zH7j81?8qR)q>x$mhng}(C;UILs7q;r@c+s=WlvT-tSns;VZzxtF~)t~v|sLbBW><?
z8keR{oIYjc)14ES-15v>JWt&*a#dCN>Q_Q*ROagk&0n{R)#YKN<srwYgd{na$*bFX
zXPQs(*p%&I*)H+R$41j&`?Tm2zoKQL)?B#ib9UF{{=?B7#+?VfxUZg4SO2wG>s7TD
zSIvHRwnORD^`q7qWKGM|ti9L1`0ny``J9@Xzo!2D>i6<!?dQd|H9PJ9Enfe#s*csH
zCsiPkjU`dOvFGFRo-fa5Xc`uBd$Tnk4idl4x5gnW?UV?UMBl$fDt;xBUan0`7K#~d
zn`K@iRj4KEm*y16vT57TjoRGF2G3j!c8j@Rad{vSqW(X+NmqOG?#vDambAdenK>Ic
zGrn5i*~WT|v-EIdT0YCl$yd^H+L|w%lqpi3du68VO-qlSmbnqys|0qg^f<Bd!A=>U
z3=_v66P{OY6P`5h(&u^aU(SnL;Z-g?zvy(PM&#-ya|^eHt9NhNzD~8hjywDAlQm7!
zYn!8Y$Z+KbDovfTWA)0TTw4^j+>u(Zb*l66krkh_*3GD1_ra{|&9$AYPON*daPdXX
z=@YAM7QL9Z<AnBsB7xPXmoPu+K2oH^D6-U}%Is{AgT%xFNvC|#WlNR67Ti}V<S}P6
z5n49Ya^vEFjjGv3UP1clP9^70OUN(InZ~^NQ<2n0&)NQM#S5ADdZ-$O{7)-!E%%kF
zwqzF$)8KFlbIoyQSz+bv6Xdu_BP{BT1NWDN{okUwFJ{E<+Z^_9^@_tcS9EPkJNr^|
z%C6E&3rlbRG>Pe6x$hujFvG!gwu2rsjOS;S&FtDHcxmDb&iEOJT9j|D%JGg|qtd1+
zsjQc@-MndgR6=ji&lt;>K3A;OlP_AC)u^pa-x055Rcol0ps2p+!Qn*=N1SxjmnvHs
z*N9%7U3H<$cgs@mWnuf@1n_^kz#kfO^y>xwuMhYm-tb4C;YwCu-EX`z?&Q=|o%|EY
zyG{q^&zik+;TPL;>a`y)=jCbcDvg;oDQMSz^=VIcO}o$}*dChja(dwU-S*P0b$_)4
z&;LIZ_%)b`f7*=q&v&a#-5|s=lV`V({#Hk!)ts3gZq}<eu9~PHHBD@q#ok2$z4Jdx
z@;6J&Qjkcxa4IaswQJ(3FqhMrNvV!1Qr{#_?~#$(E90^*$928TnInHr?}|BnY>w;2
z-L5-M#;mkFWfgrYQgQ#6?)_f^_`hD@j=aNv=gir?7tTIdbN1exvwt6Of8F5rPjF60
z(G~_zV-M@*4?J7XNp3B;<{sjm5ymao_N2L|WNvi#N|zVu*UT6Nd72Zf4}6tb)t0tx
z>8%TWZLQ~p?N(gbR=8MsnMr0`#-Z6}LbJ-|Z_kRIb-+UWl-0W6OQ}VdoNbOI8K}+N
za7imxO}F=wk?oOn|D7+dZB|>Kte)0%$$riyW8TY}XD>O*9MRLgqI>p|YwQ*0yQh*r
zT{f}ZXOeTuy?tTd=J{I=@O>#b8n)+Z<ejTgf3C*$9!)rVHD&M7wFlUO8#>e19FM)Z
zHg0EF*P~-8noHiREcvds>+lNO!#l1o@a~$Zw6nUV#%xyi)5?hUoEv%WCxR<(q%iBU
zmpBL?W9^tZgMIbV(;9m;j~1*=*z>|@<Ib$k1+k}|a9`~TJagpEsTF@uMK#=7efQR^
z2e&r-y|vQz_F<pfYhrHisJ*@2_x7IN+k4is&SbcAVDF_ve{UVky|XX&_UgHJHpHA+
z9dqlD%w46$NBVZm49`8g)q(%f9sc_~=kERgd-uWHySMM$y<>ClPVBkoI$H{QSInES
z;;{L|k22fxmQCP{xgTDwz<2V%^jin`H7^MGTo8)6a8h#KqD6AzlBQB`F1%8Y>klno
zqZP?NuY5U6n|9?S{f)x9PTqQ+6$vt|s=U?;{~jvUT{87mSLwUtyZ4H=?U{qMR}SBO
zbV&Bj#1D`3_dW8cz2e#X*xc`mdER5Yb&u`kp6L4B(YtraIPQsy+~Xs2S-sXhcKrLq
z$@a2l-Q}yZqXK=7Mz6j4_rcM-d(NilUQ76QHF4h2<bO|-?_P~N!xd$4Y@@`^d$*ot
z$V^+WaXpXc_@&9~q?_W_WE|hwB*&({E9d3GBg>DMTE<oVt8Mu2BPbSlqO0~qpPoZ+
zo#1Qrn<)$#R~`w<CB9S)y~(jUaUS1H%a0CQE(X20#5H@u>h&?bE7#pVeCE}fzqdEZ
z-riJq=V<KfEq0HO-MzJC-|JoXUhkB9y?-97f!&(}x^Ir&dvt6b=gEC<PThNR=--=D
za-8S&-d^N;d+pvE!*x$K&AmPEqKb3n{x1{m-h6xZf$zO%cK2S`oqKie?#pxUUhlj6
zq3_+l3;YRLuX3I=c|^>)(Z1!5*nN`~#=g=DpKU%AZgY1uPG5ba`RASU(W~$OsV(ET
zVHGi8W$<Ga;b}hmQ)B4`Q;z$koPFD#T6q;zUo7apsO<kxYHg(U%7;H@DklrDZc>zX
z{;%_S#otRd>n{1uee8Me$-%QvjOAFPK74ju&*~8W#ijm>+x#!?`d@tZfAQo0;>-Wl
z-~LO$d{)=|uO9th!uEd+n7|q<&l>aoYtsF%@%rBq;=d)#{}y%Nv)R8Zo|`Je=U$DB
zyOtjJJ<j)8(!B2#>#vsQKdX?xRww@>I)?w-zh^CcH>P&yirqQZ^ZwX7)?II}<yGE&
zzRY5mB$G{uz(M<}otHk_?TEP1dAH2lN?^(Q8!~S)Lb_f`tpBAfcyi9X9>qgFGv4-m
z-+t4ysc@SimmtHNP3L~^d;j`o-y5TOtj2n86Y~Fr$+Mo4=R9@)&*}Gn&Yu5szWz_3
z!rN>9e{R(Oz2*Ps>i_t^H_!h)qyP7A{ol*;|6IQRH$Z^(B|qz%_5WV*zdfoek?gSP
zy6n4$_pUyXz4z<=(_i=hGhFn4;Lya%E9Wv}L;U)>eXp)cZA}%P-YLAjVCRg9r`>wR
z&FhZ*xR~rQLDhGj$;p!`;!|Tz6=*#cdUj^IX=0gHb8!01Aj`^MZzM%toS$dWDyBN?
z%Zi|7-gC|V-kfxJ3TMc2pSfL^(tLyLciYvzTGkV>Df#R)*Xpp`TU!k87AZasdUx9>
z`ox4R<5zmK1DAMwyt}$w<iWuv&dE1jj^#Z%+9hc1=ksb~T-dg#$aP1xKF?5zGRi!;
zNL47~;u81vzD25Ev#zd<+<$J9)wWkR1OI2aZ|t=;6}hvq`tvqko9==~dyDza|3zDh
zJULTo-T$bj;`6h>2`<~^=2{isUToZX(!}e^s*kHP)-HBkKmS+N-&@x|f4AHB`{#$7
z>&xfgulx7=@6Z4F=l3%(9?5rL=5`4<!1A^D{PYgJSxQSd6sJt+=Fs1=!HGxniNWDs
zr6(DSXSlR@uiNormbR-%c*sUqvG|g}qk3U;E-p2?vrBN9O#YFL%Vg`nY*gF1W5FRO
zEdv)J6^WO-F1pFw-msB(Q`F2&r#7yT@jj)!wq)^CvrTh8otF00n8>Vm!o*n3=ww2$
zk@@?k&%TzAmjrJx6q0`GXY}bvvA@;RoW-)yw=$PbjpC41{m<pOB_U+;B^lnc7q1x2
z?7iffd|FA!NBL!M@Ex10Z!dF{NQ-U{T5Mq&`amgSYiMxnQQojO3ldDNH0Cukgon@X
z*1N7@a`lU`?-7C48-d3nkJ?mSTHQJ?>Q*#IR2$E%Fon#8XZ3G2uDctyAa29cu;{|g
zf44;!o|ZL_DLVMo>`s1eqN`GuQ_#yrzGfFP4y|!|GF_!!KWpWZ21l+P5BhngdAj)R
zn$Y=hy39M4M@DIHIG4@SSoiYL{GOuKkLDXZlSuAf$kd%Ox%$@%t*PsiDwU0=K6&EO
zU;XRVldXAa-05>~du?oyNX~kEb{~s!YsTX5I<wX<lFwd~xom#g8^a*BqdxzQ!y;b3
zF}=R++OF*D2E3*@cPpNLwc5Pd+B$d3b-UT-Tkoe?=WSZ<eb(C0L@<A9V1!}++^QEZ
zpW6Lk>tFxjprp+o+au!PHic}WAu$C^%JDIuj>)I*DLVTwJGH0J$j7MUg1flw$@A{N
zIueDZ|Jhl3Exz3L+pM%Jh9_=izkhk`PVRF-wVC&nY_)mpo93>&wOeU%&$>GkHdSug
z;&Qt3#oXhQDmQs)u4XZ+Q4>CEyuZKuto`x%oBvpSzY@IOj#XKHW7sRnEuv1h8#6+5
zRdjQ=Tzuu{mvgt~b3B9dQmwG7TwDSUPK@(|53ndPHtuYCu$Ir_V9ecDjly$6^EIAo
z`1Uyd7g;QERO{M7t}=tukzXCAJFRdPDBXBEiv6G%cVfo698r}vlaNHGycH8<g_cAy
zb@C@YGo2_fS5L)FhEphT(~(BqjVd~c65cxuCTr;RsF|-k=~vb?MVGHNIsOZ$gtwX7
z?1>wb6lR_}wPBJ_YoTjp>L1T@Pj(nvbuHyN>3y*?rbT?ui>LavKa&&wFwJlfJLVtg
z{3KaObLI-)wPMpWL((DySv4!pPTD%Nx1cXrX8Tdzy!l_2s4u)=w(G2#$^C=3E52RZ
z9`(c5_RUIv9Zq%mC?&sXUsk3*%DcHE_S>hT-#h0#6)RC-6)i4eF?v=Vv9VM4iMR8T
z=CqeuR~B+so^W3EUplBeOx-$9Z2jzY*+K6so-DaFF?f!HQMyIDim!mCXIaIb#r+Gy
zZS`|c&Wq)`Z2R)3s`;Bfp1Y5mX6{+8wqwTm1#g9=etnp-@5|3jzgNl43tk0k{M_0k
zGBu}n*{;i)eR(G4%qpQC@mIBDECrg`U7A=mXIz^6>s9@fUp^Z{X6gvd+7S89G2D-R
zQgHd#EwT2t)BT&dLgopDu5gvj3Yxg<%3`ytEB?K3*LE}1Z)iQDBysaV%cic0YjPTy
zJ#CcS?b;0*WzVdUR?PN}TV&Acc}CTu@Z=e*X@>fFXV!U4TrKf1<Jexer^;c0$qx3P
zjJ6)rSy!U^SVlDZ`0Tw$)7L+K{9mDc*_Oq;yqT?$QA%^CoiIGG(lA(f%EC2fo;^{M
zvSXN~m+fFa5YY6=>iojl>F>(7?^!!Z>`lfsUsiSR!w-#$_y5T)Xta)b_-eUbk8w;<
zqqa|?(Dj{1d7l}Ix_4Q(PIY?IlUC-Nez$(6iW=YVW#&hv<E|fUDH9eBs!E)+=kdoC
z+U8Di6Lu|H#LeknaqZ2r_fM+NzqIYwz4ZBg#D?_q(uMvT?lG_U!M7`+<yWVF@`K6E
zvR3<!v}{}SCD}DwOjKusr21BQ){QqrG7j)?u`k;!%G%-@aXFQdul3TYg*=yCoyD(h
zIHtd*kiXQ%x!EDxPXCDEE*Ya0>5^BYE$<ZeC;#V3k-oIU@vXokmsdyc_T(&_CBDgM
z>ee?ddgVXY+?eu6w6@7r_tw_#)MMGPKX*ObH>prfk9ke#nJ=3r=d`AoMd!?u%yzFh
zb<Wp|`-!}&cA}%WP}T&;l5}O`nY$y{j2Gp$ig<NL-3*B`-5>b2J$f?hhYMj-YZkYr
z75w&!`L<1d-<P#|#XG$pue(*WBCK>(;$82Eb+e{@%uIcF>mXOC_VlY$y7M+f@7<ZG
zlu>Z{2#4glKR$2Fs!Qf)t9MNCRmz%NzUO*bZ108FLb>JDt`Au*y*>BySixeBlHi$5
zFT8?m3g)NI`}s!5{-vi_c)`l5r5Xo&|NXRW<TBICPAWKX^7w`St?Sm;cJ4gm!t1(%
zXSo`8<c5%Czt}%|ImT=|JZa^<jHpAxX0M~IY&T7u`s&KWZkd&@jVif=^@^r^Up)1F
z@~eeeiTw394vQ|`&}u95h`;_v)3a|ISpSsxeV!z~_o)x-tBdkRp_-RpcSPQE3Y)$2
z>Wb{pc*%vo!d;DyPBsx-oBs6I^`rZ)wM?~D7bwmSu`3Jzaw;?>=lP6)N9Vpx+&J^B
z@j<USs~>GT*cZFMi1+k7R(tmwFF*Fy37T9~Nxpf~$Zm7Ty_`k1){?6_w~4h*V#zbp
zjrJ?Q@L3_HHTU9T$*L>O(>I5_+mQS><{J0+9c%6P>gMv^z5aij-ByKGTbc6g9~b`L
z5fy8_zB!9;eOSk?ACB|+ix=*$xpBFDSDE3W%}eLUW>_0`6*1~>+nS#%I`ewP?XvE=
z=bP@CtG(a*RNeg7dHsKaSHovrGqazuBxG-S^T~hLcK?s`Qe0-lu;qR>(|of`gT`u&
zRwwyyiSnOU8Ys=4AC}Or`(lAW6l<7-V&Cbt?G+nEoQ&84jnqzWT=M$>Uyc!f$tKm?
zE5B!`Mm<oJNpe)P-dd&5TGp^CE0LqEpd;+U&dyhyzXW&vczLk%&{}EZgNk<!WN<E8
zwtCfz%|<I{H|b>0%<8gIb&}Oz?6m%|l{VM@wLfHyxRs4xJDL1GWG3vqyP500=VFWK
z#p@QYo~0@I-?sSh+>?v1B~D(z$>Ydn;-IWzlOlgMXo++366Ycl=Z$;zI_`Cvvh`-M
z3}ffgltXiFMDLB?yf=Q)*05Erg0l|rAClgGu|-Fq)p?5TeXFfe#r;tR{RcO;ubjCw
zh<Vq_g_h+E`|f>a52~17|6^x;!j8vHOP_g}UVVHtCd(xzxhExh+e_tRJPViYzhyOD
zLrCwyGQop(?{3?@{cM(3%$@ssU!jzFq4xHo*T&qQ+kblrW$zZO-fUjkyuDJ`piYaK
z`Stey&D<X)4^)_FG#MXYSu%OX#GaP!B|kN1DM_yqH(n*RhQ0mAG)Wc9CE8XkAt#!7
zriVTd@q5VYf0D`9qyM1*^Zy?j9CMpiiMT8b*dr}EM=Y>`!FNS#+l&=u1sxNNog`d5
z>pSMltg)PTV&@!d5uvJSttLB#V|Fgu%vvU}NacxDwV|7u$|>%JyB2S;TEnERt)kt;
zXx*Z?dxefp`^43Ur6gDX={$VNYe%y6&c(Y8MVxn99A0(8dLOs7wKtbp&*Fp0r#CqW
z&Ns9XeYp6v)2yS>QpXSL><*DXVQsQ}V$U+oDXVtcbUZNiNZC6@=0r%&zN0RB$71##
z)7k3c$fSE?_Qh<~%dgvg8k&3z^vV)geRvisNwan=ZauiLmHAZ5w-4-97noEcSPyBo
zeqU)DG3DsJ&8#77EUPtG!v#bFh3$@}?EC*9=FE=>wWA!XlU&ZF8z0*c<u--E_)+7r
z7d>v`iTm$w^t+e1zM^aW+be#Bv$&`IbSu5RO!cH+&943bqPSmZGMDObm+EYI?`ZHr
zQ?PQ$d8RXyf9Ggao6K*q7WujIWT1xUv)^L#KeI*~1XyRVS{npJD_oG#5kIQvD9Ga|
zo$bg~dN6ONB1dW~vlOeW5VOh&rmqnl7gPGqKbYcEaW?7%3&#tWMhRAyt|q=x%R3S5
zzc%a)TESYs;$W);$Nb6i@h>c^e_T}9s`3Ad<-FY&R8$Wso-w>Ld%9-uBDFb9YZe7*
z-d&|RS762DU1?2DT3;^d&Gt$=;xy|3r=HGcW8T9i$(#T0R6cCx+chmha?_r!DL1-y
zHFudG3Et`;Xxnvq*A8wglO@(HDz;PBINm;e%!9`+W{WGYkK-Ah4h7b#7i(Q?T;eO1
zoIY^HfoIp;Y8RI|rv9v<{v|Tui(2`GTEk8;sYtMT_L>In5EcEx9@x<rsu1ei!56lm
zo%d{0X-7Z5X!Df>_WFeOhZ?@I#SSYT>CVy?iHNy2>vMZqhi}vu)9~LeNvpyh2ie6X
zu?98RrCrwJE_71=6PCGX0>9(2-iTv*9^o%kCcc?`u6*sWilxUYS?#mm2&#K7m(yHc
zap{=CPl1x)?T;FlKUuiFX|8>-On238p_X3r%?~D)K4F@l>AsU;$NyhjZnSi5=$X6W
zlVtO$fF1wtm@-T?ocF1<O~Q(GYjag`^VCx2@Ey%n#jLER0rR~T^C~-1Kb{e6x-g6N
z;M^q#zgzI{<y?44h9mMXpO+->ov6UBl4)Yrn|OPf;+D3D?Dc57;o*DZ%q0<KjU67M
zy|+Ri+^V_|Dk!_D=zwFhOLYB)6*7Ar=Pb6I9&>HZ>YeF3PcFXXDQ7x;e(1^8p8hb0
zLn&9Riq~3cujV>5_0plEL2JBDt>V^RZo_Hyc+-|7?akV2cJA`p+N^yh*V^=P%-&1d
z=DC;mcU`t@joHj>vp(Xq&Dr35E5U`HQpdHo>{7X6`{wkmzuwC-uROTSBan3Un9iL4
zQ5!-|I|Nuq+%z(1-dxZ;-Sxr)<vB4&T|O-6ebB+XNOs+o)mslmP7^Q+6<Wk!<uT=U
zM^k9R)j*GxACBsl74-98wdobQe<+gu^@FXKdjg*>YJDxinkdUBRJG&l1tyIjXEZLb
zJT7t#n-P{g)$xbIlrRmMxycqYg|~W&p5rK6_oKn_@t$>stg_m|*Ppz(R3_E=?8&D0
zP1gne)_;ti_$p+*j?MbdyZk<Num9GyUp8|2-fAcQN<on}&OgTfuV>o7Ub$V4&ETsD
z_dnweKUf@CcSm&BZumXLz&=a<?1K4hJU1Cw#ieCD7Hnpn(i%C<wE3~C{$tbDX}LGe
zGFa!lVvll&`romckK+W3sI;Qs(Um-V*M3px-!0i(y5Ld#R*M-`zW-AD3O)FvJVe%o
z@&`IFXuL>nst9~`U`Jp=d)C5MkA#bH3GI=0H5(VOzm8~aEx4NaFsiX4zT-~H>bLFF
zzwCT?`rynIb{UpS^HNxURi;c{63z2xp?6%6tdV2J&VyYVoGO<RbWHC|{rNO;+NHFt
zWyk(rN)g+=fqQfN%|iwzmvy{?&wV{LDKU5hOYA0TZ%bjF8IEhLbG^5TOuk{b$4+XC
zlXZ{dS7lq9C3Y?$_ARsAWPMf_rf@7%jouORXe+a#S#zCHv)+&A4RYK;8b-f&tUVpm
z8WMZem1l3@i57k@{_h=6{(qgi_O->;stDHZV`s#=`+Lqc`PLn*n{eiOO6%(g*2qsU
z_*m6qegy<NEc9q-VpyiSBJ^cc!wC%m))lYVqggLbR+|$#MP8}dgte}Nx$Ih*!Coe}
zy%Uy&#5K){y>c|Jnj<LSTpD+{&fzScdwTsd%!<EesYYJU5`Ory$E|wl^~@#Qxz6f^
zLD}_B!*g2ncP#6+@HWqQrPef6sP*XbJ>MQS`9{2UJYH87(eP=%k>&~&H3y50lkF07
zxVk4VkUsO1(^k}E)g14e(?uUoefwO`tl7-q!XiGYr;3kv9aWs2c5!|Z`>g_zC9=$(
zXH&Y(7Ow5R6(`XeotfJlaB(8fh1`oQ|5+3~%XWxPI5zFO>ywG1923v9Oj`4Hh3?7K
zXKyt!+>S|XZFG37ae^fx){}kC)5tvW<}Wv!x1{b7&0ppeJ(=Ya+uW075v$ccap;<Q
zY1`iUF>lvV+fy5>yryW}-EcX7h0dC7Su@stnPDVrJpaHclg}~Be#GR275LqFHr4R-
zDOpMDx13i+pW9W-8Z9_{WOD4DCElfVdrmNaJeka6#COD<*On=6tCJ+RxkB8g7wLab
z?+j+!752mSiM7Db1YZ?_=HC(R4+~PhuV5E?`}ko;%caBju9duu+Hh4!HtKrbS=oHk
zz!_&!ANKnB%!%3|sy*HI)7hqkuW@dCds~k69@^6HBFOqbte4+W@0wG{E3@>^=F4q8
zYl@eaaa32R@Ms4WKTRmmP1C%Z^>od?y3FO1ckJ7!_Ru-};rrR#Vtx<B&K-KwvtPn(
z|En)<t2ey<X!7v>&DXxmg%re+8D5vj-rN5*=J?yj^TF}L6+Ul5%MCb+OIcJ*j`j)f
znmhmhoIGjmo3~ZE%nr;8dXXf2<BeGXE7Q>vQk%r)mF`@+^wG5|Ocu-7+fyE2N+}cM
z^)%Bv=_SMLRmo>~VC}s=mQ!*Zs@LDHy!ugE>3g>i$A;df=;JSZr+5TD*xFQZtH*{z
z>Pg^US@9MRqm~T^+dh1sWcyy(HGhHf57wnZ`jK6=lOxnWJ++HJ^uy2mzuKOxWHY7I
zZ9g{WaB3En=sc>(;5*D59?TN{At#Q@;_B{cg<gi-d;TX?ZtGDmD&w>}<y^YW+2MA9
z^V+*EUk{rl<b)l#QaMd!zF%1QgNb!Q$qj5TbT&4-d!6x|>-+b|g|y`fcP_M4eem(v
zDIIh7<=Gt5Pgk1~Z0Dp3w*C`s7F<=Nkk#K3cJ<o@pPAp*`fPZ&<eaMN{n_`l7VZ<g
z>Oa*q@XZzHIoE=!9j*M&xS#HMw0XsX#z`MGcK<XH%)M1?ub-y5<CXpUYeid^W&G~g
za84~<<@E=n><`f;QWLL>XJ*&PZm)S8Ub9tzFM!EXR`x}WY#Z;_%Ezyp|GuvNq;Ay7
z`QNyc``&l+F!q08`a2@(C%Y=YXpgY?&!Z@=b)x;l8HGKqVJ{l=7`{#0+Z>*8bIxJU
zT*D_C44y^X^Ue!22PRBweDLYFM4(1TOSC-4T(x)CL&|**Ts<$-mnPn^_9_!cDgUR4
ztMA!5>i$-J{jB~$!Lf1gE6tRO4T?t^`GmcCc3gPi+WGI#&sER67w_%g=X7hc>!l^S
z(Rop8bAnH$N*bk}n(`p1jhj=?wD{VamBH!a^JN=;Jvs1n#l^)j+jO2DD0w+$sq~_}
zn?^@pwX6=_lyo(#^t9S)@6|;&w@l$yUz#~}lkcf%-O*d~``76Pe}5Oa+x&FNrRY=Z
z@9i`DU!>)H!yx|uk=p-yel;^TCLO6)_g?4obJNo^GYsYLzOvMA*IYR9)}Gzsn`axX
z3}2mh_Se?eH#U5~wZ`}NHs7FJkBO33XN$ZKTvEfWestN7!Urdt`QQ7dT`o3?UgX~%
z7i%T?;QI3Io38E+)eYNvW24m5e#KvPU(fyCeEeSWNxifutIa+in<^c&_S*9I>;Ku;
zuU43N)arjK6RXJLHNpJ0@8@1WuMz0r=w0>q0+XxAY6pu`T3a?apVms^+0kYHWy2&_
zv4ABrT-$h!ww%&h%&_8K^PjUP<~Lisn(?^T=GKqLy(X_TQu<u4C3{<MZ8+|wy-U*F
ztt2KvU3Kb?$jE44^VFY{r$t;l<{I)pvB`4TjMPb-&%)zYYEGM(X7u%0WJvGHkdTmq
z#OE<yu{)RN)_&TVo><3Lb$h`!(OuUUE{%-5;Zt9_aJo;ct5mS$k^rrhab0ygr-}r3
zq%E7j@YPc{=_Tt@vR1BKl%-X-YT7BGH7n0P3HICis6<;bDC(5-`kOy*X>ZuiC7n}{
zoMUuM<Ajy_oehWIwWnxbR6DuQnLX6{qQl=mm%7AbX9$D@tlE2p%aQx6#)NvNwapK-
z_<EZX8#yipcRQ~+@H@c%jfVx7ZL-TGS2c!o!A^<SDk0}94A*^1-1;Nx^@n-^dmY20
z{DCQjj(<zG6nZFhmIQe!E@sJ=UV9)#Q_^nbpJ4z0j)F^teO(%(51rAuy{G83sif%j
zz=B^k>9aDctiD``$t=Bg-hbg#^~=82drib+mxM~F2fcRvdbKe#HX^<^ciGJ3&Qe*)
zl=#`a-)?3~PkS8_x%02)`Ha+9iPQ}@b!TTSePla(#nz`)!c%sqZjY%vrC7e>VW;%*
zHKjLV*<&AHkYukcxvwdCuVPJyQb6zA-{SXc7FMs0DtBh;jO*Q^AlO>Tb5WU1B&y?)
zqVA%Og`BPb)*M#SUU5CRUoGi`#e1!%8#gWJb`px5@cz{Q7dt;+u+QK1#ZBtwL|yl$
z?J1Le-(6!<5Ad7$`hMuQ-rLi(%R4`xeY^R-YkK-N&HaCpJU@mm{~vzWe*cVuWB$v7
zUYRWWvG=K3<;BZCg?^~B{(2c`R``JFG{eEqhFSW?D;M~xnF`O2DNw7PImIVu@l;M7
zMeQ%i36i%Qc3s`#*s2@(kSqSF)3gcaT1`A9C3_~Vp7X@3JtRwPV}r&K1)<{+dkq)m
ztElVTXIU=xZAXlC)I^oAo<yavEgrUYiTyuk9N>vH@sZB?C|Df2q$kB>`o0Ah^s;-E
zgadzE-)~~r#`blwn8Bj<qjm25Ms9DCcn>QnToj#X7Z<49_U43}+oThHy(uahf-`O3
zIBc5!=V)`J*G>PnMVn?!ZJ0c#BvQRtY~372*U1asG-*bsoeG_{X{uE)7yE}NBLA0a
z@cM-ao)+s_c6!lTO&uRY?%->iOqL$pG-FBTEQ9MxTlYLzYVzUqtz-u0?HhbI8(ve1
zzOhcycm2Y(M$>|3-?9*#wSDi~8OtPj65cI4w|C=aUd5gTQ!GB(K9Fdiw?A^uqZ<xf
zVGlOX`%`jGVukXB6N$&?_5SWFH*l8=oRMASsMRU%lrDei@1YXMSLZ8VP2|))v7+*c
zB&+s~kJhh$J}<G&n&%wMwLofZNtKE%zf%7cm3NjaMFSr7PFa?0S5dZddCafOUwgLL
zE#SIZ>y{cH@bb9*(n*>b->YAVeLKF)ST)!0%jV#adrbS&l_o6LSh65oaO;YVSz5tq
zvLaFXu9{Qk{tsO{i|O+16(3eI7M1$1PV!t8^Fm8(R#)5ll_@?ssUp!gy=ok7-UtZK
zUUO8bFtV#QY?6}nJguZRF)1>f;=3QP1y5Nc_Aevjh|_QPlB-E^O)|Piv|83@`{rz2
zS?lrKWSiX0t`=oE<xXY)NkUV7gwz_2F44#oyE9WTe8+s5JpaDvJHcGjcj{-aJLOP$
z<r{BmdcjUT-HFTYE@AD>xObN8;wmSx%{FWIJX_iwbD`VRHL*AQiq9Dn&BbRo)aK@V
zW=%K$SiS8-;lkYcAJ0TDV>=tTv3M?9nJ-6nkln36xmGDRcnfX?%&FE}eyHPhdeNi~
z*`_%6!@DjTltet7+tSr8`hU#@7Uw0`FV)?6EWeCn?v6VSdnbkQFEv%WRP|xT;}<t~
z9Ji@k*|%Z)-VY`3J#&6M^xMyMet}ZyF5kLMlNa+mot1OdkKxVy(CJcYe;(#@1%yrx
z&6PZ0m|S`$@bF~D{UNh=Y3)2~)B1qr@F_>_+z)2&uRLJ?8kBBxe0@vsf&&-WDum7M
zw}r0$xLP}HMTW#KMtg&h#K63#&T{_*Hcs=)+0ex?HzoI$<-w1l!t*!FYiw(heLPdy
zjjQt5*B^^Lrz$yb5eV~e^2@o}xMizkXztMvX_ko!kxz~n{q~sbu}O8xmxMD+tA&j|
zu<SVA8d<cM_iZNk_nq^7o)$a)%$b$A`_?_Z|IFGu-+G0lMNeJ#gh_jC_mt&lHl-i#
z364AwG*z(Apk8>x_fRv#spZ_Cb@P6HkNGt>ZpF>knP={J$1jWYJ)$w^A>aM9G^xuU
zHeK94`~DgIW15#Gi>lI0st$2q<I^cB3-UVoYh`+|fcu=9xEqJgJ^bKc&NJtQN8S<d
zQ*+d^4({pRc12z@vsK$`;@<Bc_AGI=w{lD8c_LH#q?2=L=EV5klmlONSJ-W}3S1u-
zvS#)9TA3GC=l0bZZ}h+UI<-Fj&z3V+j4Vx81%BN${cfF!WoP7D_k+Thx4lYZRXeq&
z=6U9QX8&v3ngoKT2EATl=IZ1>Gr)(dg!xJ6RCDL|q4~>_wy6G}S*v|0FD`t>zduXu
zV_nh=xi>COX!e<^ax{4J;jO7lw&v)oow&Za>>K}Xr7APC=sr`S?RM-pH{=H9OrFWJ
z!F*29sjO$Y(-^1!w0Nv?dK!1^Gp+btOTKI?UR?e6O}hQB3%cKnd!u6CoLjnT|CH?&
zS6#MLZ;t26|11*8Qg`nsyM*p(o!&dQxM%!2BtHA&+hw(#zDH7J7++0Klwm#I*p|U^
z$GA17(e^>(!(&Z9dvh&LK9D)bD&@!aF8H(|-`U(7>_O++UD<_gWuJAhzgrU0D=0hR
z#SO2W3a?(=;Cjcwo7Dd;hHvs58EG4dDNjOv`YZ@HlnngY;vd^7FmLV@RYf7a{|kl8
z9?po3oE8}R_XYoSThTk3j(47(XPDSHr{v*W9s!9*p5iH@e`GE&o|Kf*3Yha&*8lFz
zhoJ&&E3eL*d9_~i!OS`5dDglux*CwZW5Qz7TMK1pah*No&vUKxYrxXIQ7;<j31r?n
zZFz0E>!acYaw{9ztveXC@61}4skB~JXtn93+?g#KrnYF`V5<8v?c-AyAD3C*0&j1t
zoV2C&(x$I1&!#%)GKK5j>(V{;Xxl{TZ>6*Ki|&8pRsPW_{ViACK<V+0*!!g`rDIY=
zCGJf!X>u)3bp7Yyu-t>u?4GpQp?HJ2tp_&FHk^3>vPtm4hw+iBcMf}Aw(^TH+nW48
z@Xmipg}T`Pcj|hBWz#}~W+cb=%&|YlW;4-0&!p2%^SE7N#A(@-Rz~)hNAHxmxI6ep
zET7nPc5Z@hhSU2E54t4hdS6uVOzr1<w%~gN7c18i2R(u7xpTd;WWE21<R;wpUYBz1
z?30@lV}<Qz`Wh|tt2#MjrR@JLb4AOz#Iz2w@I2l7@KV{@yRy@+FW9IUu`nRYGBtW?
zXq2d2bfQ<N(7m_>(Z{@JGg79;?reJ+87IGNLe3PX_|$uPUQy4V9tkvfq!cIhQq$YH
zEoy^<yS0P2$k{6r6PolDj=Za6aj$4O_F!6KpOV*-Bbw8C@+$@SHZJ;Y^X$wXC4Ia5
zpA<v&KDC_9P+R)sksr_hHt!ED2B*##iY+z}oBsW7&zIh`OuzfzcRtH9n_hZWE&317
z-OlTKB3sJ~yQU{D&d*x>JGS-z1^Gg@IR`ghzr%3shR&TE0e4JU6-u}s*oY~Vwh1#m
zQ_k4X+F<#>zHbiuS5ASjlVTxV4-`~R&s|>miS10F-*V4ByIKWTzL^yb$JJ7nxZhLv
zI43f3?ws!)p8T^?W-nC{+Lq$OcXgUb2GdS~sB1HGcN~&^;6DG&l9qcat$r_BzAXt#
zni=j9YPN4~fLCL{vAN<*0W+u0oH0d`kyDCk?!u658B&)bSq?5_PxNAY8rgeKBe#K*
zRUnl!apBAbGjm^DUBsv)FZ6Py$jkhd{~wh;ZQ*};!FQ^9jN8f=ih;?kvtoa7#i@me
zczWk6r0Fg>vQk9e$K;gWJtbYUXq~LvUk@(Q4ojPT?$KEjdEG}or<PpsKI6i7^NbIJ
zQr(<KO&eY2uj9|2CuE?Oysb1^Uu;$8tlRo>uXeOX8)O9;A8OTk`B?X_(5w}mda=p{
zWy%H9f}^fn+LbE)lQHJ+Lh)7e7RQ<_xXR;YvQv3Y(&Qs2wbou!X1t}$m^WvB;T%iZ
zm~tnDriJ_!K?#*|?v)V|RZ7}Uc8{EeLWIuBJneCE<+~}7bMWL_Hy?%<{B9X5mZW%B
zY5Ts)+!1wQ`>GV~n~Zfk8fE{mZa14H+Y+$&<{|5;{}1lkwc4j??v$rX{9a^peptP~
zE!LMQr?o5FSE1G4?CLCsP&2=S0Y*B9#g;B_k=!GzWAZ3=-kuc;dE$<7bsg&xJs$Kl
z;v`FSug;^))DWo)VIMLcM^2r8M>Kx#sq6!9o*bQ_`j_Q-q1?|SOQ)Efj=8pU`9~ee
zCr_t&1#X$pC?e6iB<|Uwu=%Z0qR$i4Ccn*G^m5VXo&MSjR<nP7nf^{`0TW-8-&!A&
zS&1FGTbHI~2gPSs&HtkN*6`S}+lSQdELvOKE1z4p@cfed;V;s{D<sX!R^2aKJVzk?
z&V_iRr}0Iv<cqb^|GTQEm^`<>lw9hjmw1PhvruM1V8hY0v+Jv#TL=DM!1C(UcY}!=
zixxZ*@vGV;({WVhsZHPdk{M@jGEFXF*Yt6?TFEJ>wy;qu^xdsUpL=>fzdD<bwX87E
z$X&2n`byjGvUk2Fv$Xe2*1p5En@LH0i=(E|3qih|%W3-G^70QZyL)S+r)brR!)7`^
zkI6DHPWY}D+SR2vK~7-dPK`hJj!Ah5^qiad_uPx}NAG1Gt@xxB*Pq6^;O@iRCkxL=
ztvtEHZMDowM(bs1Qj3_Y=bdvixKMOUuyTRQH?Fv_MIw^A>=w6YZR!tP(UYzxJ?qd)
zom-32Rw+d>Wo0cWU|U)+M=O(EIP>HV#@%H>i#?+yS+rtm)=tZ~cVpKfUy0T)LF;+t
z|8LY@kUeXe(B)~-!qUpwTH)%FufAwzEp{?ol2-WR;<IxBNB2Elw(MGT{;vD?wGNuO
zxvp?JyW$Y*?vs}+j`==Zw#qW;?CL|wvexNq?bbbcS7>pQ)vhQ;=WTM_7v&BAvK)Rb
zn5=T{rKU=e!6wH~lQ;cqYctj6V4EW3+Ba#2LdAqbY(43}r)X>RuhuA^r2XJ{)3(M*
zJCAwuv}w-s63V}jB`|gGyQ80l-hW|xpIPB9p!~5}PQR%s`#|5^)=#U2{%L>jdM*A>
z<u<G8omP!Ntws@>&C@+~M1piA;&T3YJw2JX`MBTGlcyd&>|DbhyEOXe-P}KsoJ%(^
zuX!rY_VV6?t54Vd?|Q<Tbhhr<*|1yZyb6|uZb+VbPw1ytIZy7E8D}mgO-jqszq0MT
zqyB8c&Du#0oT`!^HQ%vvGq18{Oi~rh<d~f+#BJ!O@ZsYr14)6ESLE{5&vJQOZn<+z
z*L(k3@9ic&>rFEEeYxFsKj6Y!<4Zvo8aCdZ@G<}27D@K>Lb=NUn^`3rVxmQlt+SY8
zW%l(NPr~C@hEM!TpDbf7`!_QqpfTEJvem?;is~1(O-iqtwQke2Qg);4GrmSE0^PUV
zh>4!;6#A!VyHHM`;)<yy?;48^dR4f4_n5Q<z1LNkrCTvUbJ3@HRTp$Cwp@|g=B@Nz
z`|>B<14Ueg0&d>=b5C}?4O{V5_P?=H@wab$uCr1#W*Hd&bTd98^z>iG^^Ptx6Q{>r
zX`j2#t)Ju=F!$=V-dF4AJ&lyvwQxm3Xdu%EA(x!FFXl!{y=-DiVtl`lNqz(G^py!4
zRh6D?F7bc4Ec9j6{ihN<H>X{5vfk1tQqXwJRGMf1rKo+Ev~KBGJzS}Ej)hx2YTmm?
z*VnNuRoUo!@%5E=o8oU9@PAz(IK5Fztx&VYVD_Ve`JXbU`Iy8!DVLnVxMQc1=>9K0
z##znbg)Q3iMHUBrjodiP?v+n=#jIJ<e&?Uf_Hhl5Nd8@z9PnQ)e2HQAcGle+XWL0l
z{^NKyP=5KUU0MkXCRe}QA-nmpY`9FQv);OYM~~M2pWOHC@l(AAfe~k)*)6b^e7NDA
ztCBvSih0pgb#0G_mrt$yT6IWWdP92j>E)9mSTFo6O8)QJ6lk`mZtcfIMh(l|^mnId
zPBGGY-u3a8i*E22*^r=w%f}R^IcooYK6kYnxA@g82f2^8iZtT$cY3WbTzzKoW}hYB
zk1TngZQyLRZ`#7$8|VIud3bo*dXweba{Y9&w||~gE}CPny0Ap&gq9ck)?d==rmhi>
z3(L#ttMam%EU@6@)Z>L;pY*2M{QtF5<ilL8+qT9l>(kh$yegjZ>rIm5BZ1WZSG*<K
zIS1K(6tHPVOkJS5eLmaGNe8b>ZPV;ud-B1~n*A-BtNS&jkMf;1_fY?Dp>XPm%kq-l
z+Yi;7n={R3Jnay#@$q-y*CZcy8((glgZwsGVm3ajbz9pjjQtwxCkjh`|03CZ^wOD%
z!1M9F!Ms0P_#erwJFYKt<e}kFR!jNjvq8eg<o}B=I4Tq^vg!14w&ePExfT*Fy`}6@
z=cekvezd@}x^BVLO<SMxNXGa(p5*u_IGyWLSjmFm_p%)M7t?j?Y$E1PbxPRH<bB4n
z<(`}5k;-)uW!et!Gr#=Qz0b()U3MlqpXuq=4bSq@x7&Wt+jd*|OWN}lyO*vH&2cE0
z^dmIkVW~*lj8Nm?g)6$`0wR`M9s753@sWrBvi9aZ*PlOCr|%mt`-(O5SvNcu{Vu=l
zzrbQc^Wy)B8=d8sT52wx=&$MX{hXB&>+jD~-IU6UzdN`G>K*;AnY4DNaHV8S?q{V9
z*O?xfe_UB(f5SyxbI!GD|BSV9*0XfKeXmlM_W!Y+Gyb~T&X=7076-(qdny0XaNY53
z_p@*Ee^RuRzsGO<F8uh5dvsgB^d-Ml9pxK?TiOEe+}_Zz_|N_yQk!d5E?OnE-nDn}
z<6e0ghXT#uRXq+(bJ(Ui-tbIxYGIdkYw?(nbf`=G^aQUJg30dP^0q~1G&U-TYY9*D
znXzcnQxU!pi-Lzu7kp>VGS7W>WM^>t`6hc-HrA*KJq|%@J+>+y?Gl+C=yOvi*QzVz
z^3ug~CLQ`!@;~e9nppRVH(k=YBQ~X-?vqX4_VUJt*-=|JU0o#>8&-PKBU5?0!gRBB
zIe~M$r*umnNI7*vH`t``!TgSsTQp5)bZdu~d2QWz^Hgtuo+{VwmoXof9XOHNR;9;L
zRT^+@e(do)TWPbm*B2C?Zks#%`@38F^MAjyjrozzx1(QNy>HIW4^K|dx4*AjTk+-9
z>Dt@VPO2_D^X?97-pmcGnwJ)(?96#+lv-Q;?fu>3>HA}!{anx!`rLnSe0}}D|E*3Q
z3XIZHAJmxyopcUP6p86M*d}EXaOldl*$$jj0u~)o{juW3BDJ%3G9If5<|=(WEL3>O
z$M}kaawE5RVvXU787CB$9+S@fe@5`QZ0Vbc$K^|TUh5jQs0eu|7q3}5Z`Y%&qRE^d
zhHH3dE-E?7dt{AHtJ}F4$Fm!>1%C^9n``{q#KV`fDd_C13hl)#=6C<B=@xnxlDFMz
zWAR27qYaCTEzN(n^s*T7Wb<yf6#Qy@(N#2e%0+kCUXe?ls%xiQ^42{oa@kk==_2M2
zcD<e<{+_DIUAmK!p1S$H$ewEVHKpn5)yUY?(gVLDp0@PFNIRXl9(Ow7#r0MxA+sAv
z(n=kQ{WDlk-b`Cq8+I#b*Mp{|Q^HP#Eq!10=;ZCZTBozde@=*Mtz5OHscY4`8>L>c
z>#zOwx~Ft})pwokXQaGW)!dxmy>9o5EY<bLe*M>CHfTDv;i%UUuFXLU!rnYsBXU^d
zcK4Y}ZCc5$=c_KCOMXyN&8snyRjAxvBsXcsmeVHPz2-I_L-d~6y<PG%VD?iPzO*^<
z*JeJS)%Z`+n(vF1mS*fu1Jk|Vt~8&|oP29?rrj^EPv^53HtvhczEWotnYu6}Phcyn
z_dBUW$G`oqS}`?fmw{@|?bih^0?~{YUCtGKGTOXnXP=>(_<y$ddy4zNT>NmTU!;g<
zyO_C=j<o!K!%vHEZCpB6O<$e$l>ScTD-Rp@#eDW@yr%Qi_exUL<>%)gh0OgryST5`
z|4!8Q%dTSf%FgE<?(@{1x*7T3$XNUIj@Xi0re81nx=k;S{pynXKiwq#g>-K0uP5qn
z8!o^7J7?eT_tE+Ly8T~X*;nIo@_wY>=hrPknx9KjLv8&mCWwadJ^TI6I%@moQni`8
z60}=Sq%;UCU2vJyx~5&Fppkn`2lH}Mp{5R%sEb0Jte%gw|7>k(4qm_@%JD;2uu0pM
zM>#U<T$0EVmsU-$#;%Zv8+sCPM`I%z6+_#mOsjr-A<mQ2JF3rU`c4DSgdhdxBV1;N
zlfR~TZIO(uYx=xVL(IKPvDnwoPr~Nnl1vk)pL3qqEjr>9Xqe$(9CBer@|@Qe>AibA
zlR4Kcb~8J7u{U>)pY&sn<Ik>4><W6&bkWPPZL>*kroVAYjRVJ_u+-j((VzeGH!^LJ
zXOZ=oG$ltM$wX-zyKcs$RgEcG76po|2^pLsGn(3N2`>^{yD3^p=;DzbA;RlUauhsR
zsoFcqZO7raLWO%;<~%-T(3POHa=K~O(Md0gc82Y6DsFwr{i$a1E~9nts$Uhyo!xP5
znc+4Mv*aY%cM;BegnZ73ntNaN-^60ol)_p4_Q^f#yORDD=Yr>beG(@b;?#EL$&32g
z5m9Tl728LC>|gNLc!f`esZN6Cl!eEY#8|W?4tDs3ERq-OTx)gXApfKZi<PIS9zNwN
zG4a>an9xMmJpo+$lb3zb(%Q>4aZb{bjV}s3Y}W_|e{y9D<1kDp&1@FD%4YQT$+Mn=
zidx72$FOZ$ylY|JDo@D(v$DxycNU6vPh1*s$|1BQ&a@{sDl)w#eWrB>cbU`f$&nYM
zHqSiyZq1>{*WYj5+MIdLbnOw%!(I=HM3yXUpU`5X!m(T|f+=r#R<Fq#W&UilmKi)b
z8OqYv#a7>lv^rtZIdxX6L5U*&yh*kSx1@t#s$IQMRA?Nd?ED~Lwu$6J56z=rmS(HE
zFFD4!*<-Ty9YfuA%cVW-=@qMFP3DH%I6md`HT9hPu0?qHO%EMKwbfZEUg|c+yezZW
zCO@?-cfKXGq&VqyjkD$6>oy<VylPG<udiHLZu|At<h@^>O!+GHbni!7t=f;a^A~lr
zt-5y0qwfFa`Q@L=|L=eMhJPXZ-}p;8q7F<-YaJwSUiIMla)HNh#v;*m86IYn5_@z$
zY`XR7gPZNGzyjNi8j&?;eqWmUNW$%?wt3D+&+HfLR?ZR8HQgq@@5|L&TYFNM|6A4V
z|M!RK=CF^+b51K|@49Mm#%DWE*#oK2+YGVmn=W?EF}@M8_X?}N;Co(HZm-N`5#hV~
zcCM>5`7C!{$K7_7<=O1A;@J=PxZm(uknwEh%S(T><ZU;zZ&;`0vGq-Iq~q)7li8Lm
z3Dl}Q$)>L{ng0?;lKPXGTswJHgKy*{x~%aMT)(1hG1r~9ESe{-_epKD3%avXw%uaK
zmE?|7Ijq`d8(NP(Nh?q54RnkN5efL;%w{<0WvNc=#UrLGIZI-j-c9|d^hR6swqvBz
z<X4Bjs|HK3nYdlkn{(lIT<-4j+jpOR+pc@m&Ua!0m(Ye&tNxs}|H`vv!B(RU3=i)L
z^qPb$JeG7~Q(tOgZTyV;7e7t)G<x;WhyRb7qpW6KU6G!6)Kaar4w}J!SwSTyy|o1A
ze+)^P6)bz}b)m)Fm421K9`9g@G`>-|DC<g_#O#j3q}%|rmw6k_*QRaNZ!2t_qIfvu
z3Qrr)x)rM&-+hQ|kCa?(A?Y<&%Jb~@Ni$z2P0Kc2DS3TIlyOG>)~d+q|BU%m7P|d@
zs(tyJOxZ!U)D5ZaUlx2{{B&(^?<VJqDK{VR-8`vx@qf-g!yOJc+HP)Jc|?Cn>Z%NL
z=86khJOMk@*7X!v?vQd_dZpJsCO7r;juSV(O`m<kJMZzLGZF7oy)LaOJ((B2%&5@3
zBz~3Fbe6~!TDhX8N0QAfg(vx4Pk$MF%{I=g{<qa#`}IK|mR_lN&Qf!aTR!+HZ=H3o
z{P`!%)%#9rzyI|lI^M&zzb0nF0kJ*jwCcZ!FSv6u{@<<Gg!21cf4Ddknt$F-U@^N8
z$-O6G|3mpe9>0LYJRv98KF_Of;<q`~ZhLxXoCbq_-qAddSe^oHz6%~^=Z+P16|}X8
zIO!!sIdW}VAY#C5`fL4rH|2bt4^x-CE}oEKsFddQ?~th6vu9GyG5>Xv4Rs!SX|S0p
zw0Y|ZT=r;Qv*1KmhtG;<t5!@pwPMPums1}1-dn-^hU09{gtJfHaNK><_o9RIz7OZ)
z6+BB4nB5pPZ#jgYO1gU_k9&FB-E;4_|1X{_r87xD*Qw2k=i-EHuC~d1xfi86xr36X
zXykIPaG2`Cl3(W3Q{-gi(2^~t?DnT^TFC0x9%}`p`9BE+UcI%TXwpj0s6_5%5A?nC
zCs-^NjdAg^S^j^M%GaLUo;BHq3Ge4zdQ$Fj-fKsSp@shJviEDQdHI($moMY1oN@kk
zMxj{lMzOuVck1$)dpz79J)hqY$lYrA=hoSJp{#`y1QkslEqb(m(X;iE-`7U=q+0(s
zIC3~7>A#zY>o4UcA==7Qk{;^#9Xm8p`&i-92ds|*v>uCnIF@qi@eL)WnJL^_0f{jM
zFYMZ%u^l?elF7pBG|lVaYALrHZx&qNpp&1mA!W%aF$wLLb&oq4mTX_N&VSvxwWqi^
zroK~J%>Me+E6Wcp%HK1T=J&non(*>e0{6P}uQz>oeM9*?U*?-d%c9Qky**HneMpT*
z{j`Ia(YqJxHYs(h-}ckE!l$9eyV<0h$u0ZCzdU1`?z}$&=6>8C7hJLvTkkH>uT{HJ
zOE!N-R`!g#lUEYF^roy7o6^bsMakQP;hW9|?>|Y2HWMp69?Y>>ZoF-^+Pz7)yx4ZM
ztS-;<7hBJ%|3CimWuM2o2TyMQVHDVM{(HrFgUxRZ#6R!3ciM2ZyLjP8G4s}19pTzV
z{Iv$3eLpdU7PN40%g@}wfBwU9YnGn+o{tMe*8E{vA$;zA__h8+Z~FhudMdeCQ0jHR
z<ll5vlhsli1CRVzd}P<fn0<w9EE5=*1sKzPEK}FCiA-R8+qx{uyIo;|pzAxq<qC;)
zenQI0GfmtK*_;&jb160}G5=IPlhT}j;qw!lH2y^AsTR&AYc$rjWwD<;@v=#PtFW14
z;s*Qq7wjMNID8Fu;4C@I)g&ICI^jaH>dt`kSDYulSggV3t#(n5m-p=CC-0o@1~^~7
zz;klL!F$Y`pA;}nDpLOtp#J}Z0KZ_Ge;%iejc%q3uQqR?o3Q#-p^e(*DLPuqWsh7E
zWKT<cb?Ju6MUlliJLagrHY)nna9p&#Uq5j1?gKApx8>gS`k2<&Z=`PQy|Ki5Ye=BV
z=OC^}zsi>E|MN2R1CK=K#05d0!(N<S=+^XC;6>w)vu1^#&9XO#ryQEH>Z6qN8vo)+
z(vpjHlomg_z^Xap(8F^Zm#$-y<t+$gpAc8n_4rKE@l}U(T9)Xwc`&Vgs<4;ksmvz(
z3!hh96MpBm_0}dAlU*DqoAOT2I$_eYhTS-5t&&mL!&OQjFNfXqQZ6nH&z>4S(L|;2
zYxTX3b(dBcE32z+o!}U4WXIu|^`~b;+9ap{wpZWzEqiC=cA+WgqL#NpQ;@07Tc-o1
zOnpN9oyUD9Y|7*4pB=PXK}di4vPteKS3U}`3#NC^P;39#<)ZgWQSV5R{--A9A1q?U
z%B38|-xDT!mh0CvtTwySvV!Yd$*M1<HZHnrSL=6m&n<FGTrW~1Rh%q&IoWz!mBQt{
z`?jP!e`zH9)O)*cYUbC(k1i9|EEUgB(fPGH?KS%WALI0ci$njk+vTK&7o;BKcyTc$
zSn_a&nedCh0v&$JT#G)f^%G{gZ{iWX;i<X1hN+KE=BYy&6M~QST+Z&_%rJd2V`+wL
zXNr}&sNB;g5fz4&MFm}V*74uJX1pe+bDdYH<~7kg$DRLqlJbvlRycD&?F_@|cbj`o
zC9LHu56chba4+JVyupEaO25i=&bbCfpF4^khH)MF@_5452~r|<v%bz(`C_+d>xCR4
z=fLkg+M1>ITS{*mY`Ss5S+m7lH-*b*%2YkI5AQy>q`h=yEELEzsp!@eROEMWe$t_{
z+<o?wc-C#N=BNj(@t$zvq)$>x8Pn6Uc_GGMPV%++-rUezvhH=LZ>Zj77lVgKe0ELo
zzG_~3^ZWNVuT$HZtC&+uTvNHt7TlS1z&N)h%`w;{I#uHGl=O7f@Uo`|rLrWIAB77@
zG-h62^lH7*46&@YTmIe%eelOb`oD?&x#aArN?JTu1NSS*EM*D|&XM`Ac<yMCMo{r9
z-33g?A1MXb`xI}wY}fhDaL4!63X3~S7V*wmqoDHmeaCZsksxD{XZEKa?B3N@Fj*qe
z;&{yR-ZgcmVyRP=qQeRujn>&r>RQ+rw$v|t`;PE~H)bu8uysAEs@A~lrr9f+(97(=
zD5StRMPd1tYsHb99G%6(xUwylmi8~68m(!far0cZl#+U@P|Jo^7iRs5-XgIlk8Jii
zEt-7j!;P!4rJU}MLMwO$<JI)H6&-n<;}ZWfbY@)^Gt)KE^5^BpGkcuYC;e}#UaWKT
zn$Q*>L&Fm;yfb-g+}SJd9P#pT)ZhNfc$tVF({<je%AXctNv_i>-ON%t`}e3^p7DSC
z@s!)%vqX*eobpNzb$|L#$mZSmI$am0DG9xz3Uh=V7$+U*J^Qow^3FN;BjV3E{pQoT
zyKBA4&c`O@*V6CnW}Mdl+c7PJJMCVVnYpJ^rZ(4oEvEYdQB9srQk#!vx$|7=Vmi9(
zTg&cki;ixy{JwNq*@~Q>OSP^&YLDBo{6&+9!pGqFWLJr0dH>gm<}t?Z6lYa;HrO^<
z*Zz1nn~>={2VLgtJavZ+?j7JQnpJa?MKO1So%J!L<uALMOC#97*ypMhl{eeVhFdI|
zC7RCsazYpLxe6vV2PQRzUe$)Vq6>OYR;{@hx#r@pHRr3i?fQys%Oj_Ua*KQR|Gu%o
zM|`7?`TGCi;ibI2Zwrc4{#2D2YL|ZwSe570^MS3rPTasfJkGE(F4tsJe7D}TBUf)F
zPO)?{5?Unh?LMDNRrvT4pR}lH9qr5)HU*fd-Pn+=clSx>>@7S@-@8{n|9NZsw(H-w
zZ9Z$@n?3vVjoIN&#YZFO9Q`%ty2YH{>N#gEcb>1@Id%8W>DG*s6c~FtWFPMQeeTPC
zCW-k+-`C&z((p7_Ma&}ep!H#i8;$Be_9%NeXsKB1&sdx%<PfyNg1@wd^J~k+z(?<X
zYn&*IeIwl-5)o9m=5@@C<8Kz}ZFA87bJQSg>q&{3#(lho(=?a9Eo2tASzdeN>!nZc
zJ9e?H4P~F-kyatllX|TCzw4$hUz;AqpzfnHjzr$+kKJSXfxX@=fyqpvcYjr{>Vn?W
zk;l(Utu4OOdsynk<y*&ZTArw#bDS}uSagfdl&@>If9OjtvJhOiZsnYQVeSbR-PEmP
zHaIScvDbEG<xdn--s-?SJ>y8hDx<R(_Xz0!bQf`-Y&lb)GjsVj)6cV9Z2oN&&aCpX
z+3oH0ar4^MqQ^9sJ30$R_xpZQ`;ioF_)t6X?CFaWC4=m?c~0GVY4XoYvn!LuTW1B9
zU6v0nE83Y*_?$m9JEZ3K?87^ET#V$r@^j8<%Q-WDUk$IlmMpvT^v$c+EqCV2?kcw3
zRr|NscJi*j;R{N>&FAoG*k0``+GN-vcJKd;zq@&vG9QNAVwF-Bl4@KRyU_NL;-@Kl
zWpBMGd2DZSBjdowZ0%Xv2EQ!kGcmTdh%6GeQoq{V-D1P>=eXCy1G<M<4qrZhbaF|a
zRML*Lle69w%cS}5=$cTvAtZ4syJX#wtrEpobynqneJH=3Wy!Ci-zvS&u0?J6wR(RP
zlgNP+S9hK8v|H1A=ZWxxC+#&)TIW0s|M%2AZkKu0gadr*wr>b^yM9W()9LH1Qq7go
z$3K+T?^E2uqu8~Nw|Nowjl~;wyYlz?iPnEF7tKE_yZcqA`PDT=?(Y_DlHJIAWU<iR
zz;9mu7dEcGv5x!J8}EHHZf?6NxBVsOl~TzHp7k0wXY01Q7XR<JzIt)y&ceCx=SuJF
zzx|$d|IU*?-{<e$$shlr(stMVAJ-@EiE|Ln_<M4GTj09<Yr8hj-kpA^Ve-b$qFf6c
z|L;D$dyn#u1zTemKF_t?7ps1(wWM81_59Q3-?IZ9EA|UtZOM+;{ukXUzk&CK>XP-r
z2~!prPV>y)zIiv_y-hU&%F~mN)?V#iDsfcv-;sJzmAU`BZEei9?0URcYRzfOCzUbB
zUFSWCwtIT={-4wFPcQ%fb5;ItV%(a_KYz|!t+`)uGIQVhc`@hbgl=6F^PF)Jk9Z>s
zyOd7HhXsdPc*N!2I0-B|+`)2s%EOe23y(IYXqi2^w5(@#n4<eM6-}Y19_Fb|(;oc)
zG>KVUC?;jn%9Bsk+I3SrzNS2$EOKGKW3`ZJl!@l0rA|}DWL?=_En5*%>%?ni@<JkL
z>#M6-p{rw8Z(O3nqjPoH6zTB16BlYC7jfS)l1(_Z^|s@?yF#hYjy?Syo}X}_=r@<%
zpX&w>4|ehMm+@2<J~`PhuALY2vnXkQ*<YtO)&+}QXIX83Gsl2+D~sxC(H(va(T+l_
zoFW<vb)~<*y}3R6{kFNkzvtZB+0LxqC$l5}?wRcKWpdV@UtV5VeR|nE2dg)_G0XB^
z`Lzj)zq^0({ga?#u3w)`cRqIEkGHA%8S|vwz5e4f*FO^HU#YuJ@MmW12-Fmx>X;zO
zCgyRWiA(Oq{{@W<ZV_(k{6-lI8pYCnJZR%D;&{}q)FzQsuQc~WP?yHK6-m-udqXDF
zS$^EY+LSxtl2`DW7aphjteL(()^Oz0oHAkY2E~+#Yh-4sOj=Um&gLKN>3KCId@GY^
zNc4t_UAnPNtZI@%69T)mvr08@b+aAhxI8P5izR$^!8S?tnCe0X@p&a>+tTOO{(G6e
zpyAufbk0_$UDxC4&snC=={cnt=p^G~_;QJ`M!`xJE{}j{ra6zUy<9%8YS+sZbLM4b
ztyp<UEo=4aWv{eWtyyKIy<%%wmiD?`&rW48=VaKx+8a2@Fm=k~!i900PUx*(v+2&t
z_IcGGp1R~ZU-N5uyY*Ju>HllDs>!7!?O^HLEwGbaZc12`T4zg`m*UdfL3@}ky1e(X
zxR?^`x8%-=kT8#*uSJ5H*mFcfBc)=l9`Gm?y%xbY-84Y^HCI%6N;8{R<guxlXFeX2
zE9cpiBFa21&`maQb#RsR60=s8pdC5~3vQm(i9IXmB4~Wp>iV46LOW^pFXxZ{)QKr@
zy5D9}uzlyX+1Y;k)3PrFiC-^1d;Me2SF!uKzB(I^7qNf4e%#Ny>}Ja4_OjZaiATOQ
z=I-xJxWjO+R^cwE*N+8vmEXlG-LIScqx?aI`ii87I_rMqZS6XJ!otJC?p^XEZ#Tau
zPi(#VET6hgDzelJD&<Xm5&Es=>5KTf(Er@p@mm+NNUEzJNplo=e{;p0ymNL%^Sd6+
zTQPUX<(t;?>tFt}?kSIz7G5gsqi}HP%xht<)@<st(#|n-S8$aMa+%N~66i6p<#aO3
zZ<$r=?p4`+eRX`7-P*T{*T4GmvU`;S!}f1ljei&aTh$<Pm`hvdt4((#OUboO7npUE
z1G(+HZck(0&3cgC%OHAU*OUyc6@u(hDzO<aA9#pdVVksq)n&;+&r^OKM#~xAXs`rJ
zghl-jSsbC67;bqgblV|=4IQaEYbG%GXL)d4Dq+!8nJ6LoDu`+OlOv@zfnLJS7Gf5O
z%ZwGAA0>Vg64y?9mfZ7EHmQYO>OcvLiCCkcWQvl&%>Q%LO!bq`UYf#ueA!#IG@%(>
zn#VTS&URH~4D4_cQd+>NyCI1uP=eVy!qCHaOOxo@701OkOq=~wBHeOIPjYd?;Vs@y
zyy1S?yvriCF<x#-W1h-0t8`xR+~Ba@v{IF6*HUiGP3}By`$lrM&zZ1oTh6xIXH7nT
zt8V6k-HsVEBm~u(`zjZv1a&y3OD=1lcT@48Yq0ZWk-$d2OA~XpHS*l9x!!rNbkpn(
z;he*>F8<GBSyKBoW3GMDbF0gpOUi^a=XtGKKG`LFP0gf4o+a86{(kJs)^Bm*ik3JU
zu_vYb`qoEc@ta&V^OBC`^%=Bq@NzlrJ~HjLESH;ft0(&>mLMnARsR=n{Ve;ICE40+
z<&>zYvl!T?wXd9!m9^~S?J2%z*|q)8x^eGwPziUt=q{hRNImL;ldj3e#kw*^T!E6V
zwnZu3o2Hzvy;ObmPaNy2h(s^WGtrWDJ5H{fc!P0qS;0}=BO(mH-e#@co28vI>4K@2
z=gSBtodvP8S>-biy3b*oc#-A!*Xta9Pof)b#ho~;1$tbxHu5;WX$%uP_4R>D{JPhk
zl2%F!zRpPF*1qkTa`+ghc$Zk);?Nrh{kjb<JvyOnT_T~qWJ71zmcwFiUv%k4TojJJ
zq3rIKDa?7|0$25gGM!k(b>U&r-OigHxy3z6IuUTnRjZ3ftya-l=&Z(sDJN4>SN$(s
zd`WNSmJfC2nJ0hpR;Dg)t(x$#f3by@_N#+jWmgU@Sghb`z2(BaZ+T7qKen=3A7c<*
zJ@ZnAuk+K(7C%j{dn~97=6LF7VQ@@fU5oBIjn%E%JrB2L+z`Fh-FG1%(`#02Zu$mK
zhYe<iZ0mA$rzK_=#uait&g@J-_F^ah9j9oY?N@#Ctj`tpeSM~|>uc`JQzgqx7~7w4
zY2I{J*tGI<QrE1hxd$WvJBy_(EqJQ!FBxgO^I~7om#3vkp2gD^Kc8gb?r*Sr^O>)I
zW$YZycO5US)MD_Rdzj-^*SvQp#Qg6vxgNa2bzQbaBreM}XZI7~updenVrJ&r?g<Hc
z6w4EM)OzKAcbOojr+Z5({r748S(UZ#!&8yv^R8vSTGY_;<=A%lRmrVlCI{2QL^jDL
z2gn9R241*wHA(nW@%-i9OILCxL^iD3{l@cJ5_h=D?lmX(Icw|udBf#iz2-ujnda43
z*VeRo)qh<bm)pR4V24z81M`vW*oCDBGC5ls>qK)B@AAIipH!gGoqA%AYKDpb*^9jf
zlbbgD|7%pw9&_Vw+0Fg4PaLY>Yw*zEOxNM)3IlD%8yco8N=J)t7_?_Pa39?Id2#cj
z4YIo$Mf7)iO1He8thaKyu&n>j-jvr%pLuQP4!fe1Y9c&KrLRqSdx+%eUD=+`4cIN$
za^Bz9n^tmDr1qsd|E$XH|H6#xTojoWZ#b%U&wJ{0!K(c?xu;oeZ!m7!wq{a*_lE3$
zcM}4q<s3?l^}cP)^L_j2xrg%af4Fn^0INmQjl-+foIA7Nu<`#~r)<r-)XG=JX3>`>
zP21Vi|F}3wKK`7AJu{d8oe7>PO46H8%rG}<wK<}Y5mfxeOKtB9Z+{=w-d(jz;<8ty
zpOR{_d-wF!or%-$a-LUvX?G@T+H%|1+^RdvdgmM0$OPE1$LKHQyRd2d+nnP@3Uz|B
z4(FbF8}Y{`x_cXwhWOV>-AcME++_3rXwSV|HD!C_Ge^s+T}kn;Uvg*%?yvv)xqkm2
zSKqef|E^Rg)~PriWH)}O-0=Q+iqM15tB3ykrhdwiC}ls@mU!AMSM(X1m|3HcN3LiF
zyTpn{krRy)*VxuK=H{q2-DG31`(l4MNGCBluY9XcqO;je(R{-ndBz&erZ*xu7MiRL
zG;vinj&M$LNN!pc8l`nbgg-rsqd7`U++d?=^xCHe(FY3S8a>`PM4KJTP=6j!8YrHc
zz|H5X7V9C9Gf`1HQNe^gwmLAjK~QRXX{73w*g0Q|b4}u$4>uP_#Id-w-!E-{bhN#A
zM|0f_>3@eggWcRWKKGpQ#botEqY0{B>n7_7IVYC2CyJ|eN~d*B<S3siQ9h%hv+O#T
zutG_&a+m$*=KAf7&Fx*9&rI9Rb@e2Y4VHO#wEs_@)}Cy2&5X0Cds{{2ZjF?~M-&T^
z6(tfC4|h}@si?Z((sN}>k9Szl8H=7T(}VAY*>4nN*tjuQbs9@FgXQfT)iaANQ=au&
zD)qf-u{`lT&A6rSuZXC}q#DMf9tjIsqbH=l-Jp^<F(CCqKzx%x%7fbUhe~hRnnfJ*
zc@L_oIB1?aY%_P6efUz%7zNe^Y`tH_Ctl!~D3odU=6T(h^oAsJxnGQ7+8Y~As!f`#
zk$W|4lFG}RtJ@mIe&k5J;L`j#NoQuG__YS(o0F2CvL%<>-=3zuG)?>J3;SpTJ9P<%
zlA}$`Ee`r=0z5|?OkPYe<CyCI(!uL{36qDZ?T_-vm9i>o|NC7|IB8!I(MxIZ+~_1Y
zJ<7y|ZAYlc#;2lIPYl*>V7;jvb1IELFF_%FVPbivtI0$+wnn$2q}V#m8S_Nj4G!nq
z&M=NRY<n{>;Cire)CIoRr7}M4B~xaWc&d9=ddQrfXfpH3%y}%cic+*^Z!%fYD8;|6
zP3d5#N@e-%7iA^vv)60RUgtS`ZRYHap0iUJOuYh3cPx}$$ueiZX2Ojb44I}C>S-0L
zXH@82lS{o^+}~cYo27Ekl!$5KKDNiY8Jp%hUQ@8X=5we*;oypt`#%+!73VEspZ9WQ
zionL6SDA`WZgL89)_*yidv%4jDx36Gj_}RL=i7bq%RZ+3WlHtF@U-_={-+fz^FO$}
z@9WaqQ{n;90q$!9xP+uKt}_KTxiZ~(7_@R>-Fe}r1!@fki?qIr8wWXLFq+#XgmVX(
zE?S~izwFbZui}fXTpK>KXZbzLXa5<Rs=z3-pdoo{4&RBKHJX!i3mV0*HELBZG0$u?
zvTXFRn(QmJG<esfkf_O`YHF`0=PI9Ax?e~~eMXbp&L-^_UDaDH%w9wUu3Q$>xh&Pd
zWO2t-JB{Vm%8ai}TFPHdT@qU8F~cC`r%3LFmV8UG6b057hBihY#Tp`u-d~ivBU;e*
zGtoVG=1mKa;OSX$hNXcYvZ4%1Jyhere#rWGmFvxdkhKD=A)O_aH`;4wc&=Wh$?5iA
zYkgI)W!0=@f{8-RogS~atPN+Ew$I*ot8?|uSqHh+%>1!>$I8x=T(k3++g3d{UMS%i
zZk$xCCEME76_s4ENyY5aqVV-MDoo7g<~Vn6S<$^CVx6^{InShZyEx_@xncHhRm$5}
zJr6zCf6D6l{A-=K;{2$n`Bz@7-<mjI@3Md5v-vE$({`6{h=0~Q&Db)pZNcKtmPKU-
zEzkOnur9n)R5zO~b>pereQBYqk83^L%(^dtKe>S=<T<~pG;3LzhnuO#k_Y@Atnq6S
z_}NSYerHV-Ia#%F*TjV;I(NQIT>F6Q)UgI9t4^L1k#1JG<~z5jdFHBDE^*JE?4`9-
zZ{`wx&ZQC7|Cg%%Tp}4US+8;_?~I(72OIYU&EIix<A=}Mc3+l0bW?Wo%qwf1lDs+3
z`(}7{*Y=!M+l?Y-7PFWf`5CrUWBIKU(TOjY$Io2O)xLwL-Jr~*P&iuj-zLtS4>gkN
z(XW@9E_CT&{J!S0)(*jf(x4B`F2*Z+GWky)U3t<pKAB-vYY4wz>ZX^W+50}kFWw-!
zO<~uuMw5jvO}vXc@<YnFz8Ud|rDk0h%2ntTOXJ>EHG6Lsw~7K|Zo!(pR(rQb?me5m
z_sZ(MH=}3oiuBs&DSOaT)`+>Hr*f@ETbItVeHRx+PAWCKu+sbc#E88qb8krc{1S{}
zUNqNkTX*?PpGP-2V-<R06aLR*iBV*eIlvlo;GO1y$CU?|ZF+d;9N@1x@XWK`c-PVe
zP4jOyrvH8Ex1HUe?Nv1s*Mhz23;dPxeobkyJk-a_CAN0kqR*U*B`xPKI%X?4fgvbi
zQQwqa6Qx~7Zx+pIvfXYP|M6?w*A<+B3pm}US}{E1-=-bEsyS)N2M^0?tpj4KXEdcA
z{uZiQsOFFr#;36@;CGJK>n%RkTVtcygb(xwcrQ8g`&fR=(&9f`J*|$H^&Ah4KCbe!
zQS!`ENgf7E8P(kARIL+D6`{*gH*fErv%U9E_^h9aP1+|f9Sm8+Ad=9ze8!#vSB}CO
zuAS}CB8^%rUY8nM?OD0ay438md;OA?GrRuBGc04hS>f^a0r#Q_aos5yN^Hwza`@M5
z;M{eT%iHIu%@k!pFQE{hfMBMTOOndCwrMh5kBt5heJsr7<nGR+yY|kqSaZUPRpr3m
zQ(0%%zg&H^Ywy#Vy;r5zNPX+PU}gG-XJ7s6=3|{}pG$kUKkqv8a_xs``G*J3Z>>mv
z!*$_*&D{Sr$-HMc*>n%)C13a|wcaYNC$?ZcyYGcO<B69t=3nO6a7{v*V*&#QL-gZ}
z4Qf1ntiO9zV>e84?z1oI6S!?!6sB^m-D*{H#)hd@r_uvdy#r?L;+(gcb$eMz(-f<{
zYeLo?+qL?^rtU8u*MuiNRAvoSWqrH!@V413A3NL`ZrJKO|6jB$s6o0R^$=6oaj{8>
zGD`$1j;YO@6zhE~_3g2%+ZwDHkNbO1mJr*<rx6x$ODJ4wYijHDBHhN~y<6*KCs&=l
z-sF2D@55GuO!nXnlQXin_3&=6&79xksoe5sTcPzbx0zG8Q?@g{JTdp|iTSnyE45@A
zR!=?Rxwa{1NAk)Y3)bA4trL~Txzp=~et_VLwW2fc8;kC>DK*d*Tfh6r*4Z)dpT@O6
z5&g;P@p&n0xl4wQR>HxJUP{)CK4-NSaoTbnlRDz9X=>;qRJLZyiM`ME?0xX}uH;t6
z=Q^|3E>N&qF19X!EjoZn!c6m4&b`O7_deIo-t}|YKF)o$>iZhXWnUHUYg_)`wBxw!
z<}3T&oL=!uXRg^c`5fo{pKqHz%bfe)@ZqoS%CFY@J5vu#YCj-!=OLHRL1CE#%zF-q
z$~@vYBX6?mk^HaY-&Y2={I<v{tNu6B&xCJ*>A(52lozmTS%o~3Ef*2hi7AX<SYr?)
z<{-dm!y4lGA#QO($eRZtYZ_SmLXQO4#K#6^yYj@#s9#PIsCWGHG)gz;;+><KieV|L
zOHx9QT}eB}ZLu|G_mZ-NW3k@PD`TIpsXT6U@}7{wIhD@xKqcYqgNuy<ndTg2EpuQK
zE_fk&j8*zTf2H5jN|~3nv0KBv8>?krW-xVi+csrdH*IFjbMc&#RCQv8t!~ZHo3|z{
zTX6ROt9iD9uI^pSbzfUE?|Pxq()s$<&VRS&)l73#tPQ!S>UY9<=Wf<-4yQtwa|$UO
z{akuy;!Wi)6|p2m){}dcQx>r5DK9$9yL`=~{5eO3kE$i^lsbOfX7+=eHD7BxpXjXl
z_V&5xwqvOmX5}7;R5xafJ|NWfj%`~*xp0Hnl%~Yg3unL7zGGhh?&h9Z&)%^`FStKh
zWo@;&=PsY5lQ!qqP1&d2cK+v|efMvj|NF=LU(CA4r}i_i?UstWkkfoYt^X4*?*Xpb
z3&L~Of2g_;qi}$UXVbn(7tQ)FTFT#)**jlhZ|^_NOZM|GsqNjsahmNVhwA*rU(B0o
zBAuS-_gF1aVC8sV6e#-N_@_CW&pocKZd`YjJ!}uH3<|ia-^{{LvB_%bm9^~ldY{ud
zre4c@`z&AndBMFcMe*M~q#E;OZ)nW?zDBY!Pk}L2VXOX0)~>k8Vv2@(rAqow*>Vs3
z(2rzWShBTU_j+XY%Zhi`+iY*lioY?F|9ay$yWK_G`gMOvz0qv6xtVu+Vw=x0-3kZu
zgHvX|eYMu_w^_%kLa$rVmak2Nmv7s*W98nH36|3iuNR42)V0yKkkkG4isaj}Vs?#T
zYg=X=E|hF&{c&fbU;FAO=ca_*d!hH?5%0efr{3M;zqfV+V~oSz3#VImMQLUoe<2*e
zR-V9}`{4aO*?&*$|2^(~Vd!k?#4f-o@?XVMVZwrgP0Ye_ArTW48gyrG^O>@vv*(Cz
zj9KBy9j%8&^b{S79!{9J#FI}kwCLfIO{piQrzGvuxjZS@ccw-Dq*Eu0)g|>(E7(*+
zgEJ%-d1SZ6NKH{&uIFg&y4JIFRmketyfUMztv%rz!lt%8T{Km5Ig8LF50Benj*g9m
z_s>n`{;qqci9^UE({+`;c2KSdqnb>`hWm$`xS7+6cNjiC+$!&`b~H_7b)fjkoQpXZ
zlb)SV(F;Gvvny0PdTrLqw7skjj#~vUt($u`Z0%c>{QUFRe5=2|-?r^8o4@waUTy2O
zZxq7csr~%){M>B!<#w@_#n$I0CSRX7_gC4A2S?l0`Ty_pnaP`btUR56uWdzt?y37q
z#)+ptS+Bh_W#i#gCUNt1yAMw_Q+u$0$9%_)2JTPyKJ!_!E0^RuacI^U>}E7vQ{WsO
zQ|Qcf_tm0o7eQ~8kLy&-f+rexTixp6o{-d1=q8!^CS&P@h9oTwxv8%{y304S6nQ9i
zt9<fMp6pWOsX95tp|f$3O2#P`zEz*Rv^F0RI;|Ta*uyjL5KFO-;c1o6KE{`Gc+Qw=
z3vV{L_^8GBtp3rIrPHeXE`K)BTos~W?(ly~(FF%?)h_|gysbuSgQi)(3*=XBy%OlD
z-TLK{hvrg|%a6^@e!1*-T=+b%k=?9@o)1o;p;tnuD<^XK+1^QN{_hs6nasr(qv_Zr
z<l?$Yh3`a$=#{r^y`sVXr!SX;1O}I?Zsp~fbm_d4_$IT{_S<XLF6vN{I=s2V<Wa`4
z33)c0?j;wucB+=;PI?*ZSF|hh-n^c!@5?LSUNyg8d43U>R>R^;UaQx0J<HN=-n~le
z!O8_$I}%!+i~d;JxKlyzNDGt43B6v{R}qc<U%B$S_jxVLF+9TXTRU}Hc-hYlQw^1K
zOin-g;$wUvN6gnC=+2QhTdssf>zZC~;QsjH;SHy#mrEWe{d(Em9PD8GVqG<#^_un9
zZe^`@J!|m#?I-Q`ub-T{v51#3I<wiHf2YIkUb9;h8hFA_yp!6$;qbFtd|v`Y^AG$t
z;(7DpK=dgNvHT+&&s{jU>$BgV>=Ux@*L^%LyZ(3Hsrj|jxK~V{rrog9;rkDl<@~cU
z7K!RR{XHg?Sz@a^A>sT*71>IbRi0|a$7{>)RzLsyeZis@p@3}JCU5V@D^2XUjG{8v
z_`W#YGtc+s>g950U*5GbHlFA7*zTOsZJFS$Wm%c&=fk!w_~!ZWxlzb12f?oR3wH!2
zg)Mk^)4y9JeY)wk8(x*K0)J()atg3G{O?^Z`rOh_dRxN$I7=@_j#&zIQHk?C1+^PF
zOAd>8G&p!pUBr^IKx(zakNvz^0j(N;LRL*l?tXHEscpr?MBSV~?u9GYil)4{{AY^8
z+DEY`>v$Cx{hzv|U{_a(T8!*X1`msm6DLej=~DB&?iF`PZ^n}oYQB+Ndpj;ozWtL$
zJ-(zXsO`|v<$q4cN8I!Z6F7EkeG02~!56NhZz{wd8T6J^PwAfeU=iyPjbrI;jx!a)
z%_L7+xJ}fU-IMAuar5krD~yYne5Chn=UlK+tv%&LvU=;|odUI+G8g8|D#%&fU{v59
zd8x9m^o_#&*D5Po`X2SvIVj0L*`nwms}w4If<a`}+$YM<Z(ePjcGx>+>ch}gMgiJ+
ziVJyTIczT7JW#u6=UhJC7h*R=LfgZ3T6~|t>(tGhZgN-Q9JeOB=^Y<m>%h<F!YVW;
z)a?}3ys}-|u1#;g8dC#{#sAOeuN*yBx>otc>s1^!Ex-I0a;Cm;YIIqltD5f}wdlpv
zB?s8TQv7Q&l`gwo{1Vt#w8bR3A=vZemFdQJHT|F8zN~g_x`xY9k!8kHGpC3wU|@*U
zYOv~^D(m%eK9A7`(W^NMo*qhTe&sCWNm0ns6x*^*ZBeeH<5AIdM?Oy!Qu1#7JNYWx
zjH3s|nkKAH7R}y#jQMc-NdZ%{5MI8E$GdvM)D{<{vTNUHRJZxMNN(?ocKu(8-GMF%
zUIMCdpQM_W9uC?PE&6lXE2B+<H$x|Og)Yh2y+K2G+5zr3tA{y(Kep;uYCheZIwc~K
ziOrx^HYf4-rt3#IzHMljdUOlZ<hKVdY})d_|7lsW<n7!TE~Oi~TX@nAhe{T<M#nt)
zm1<%neWoar?WE^3pIyCgZfY21a)+f%nmkuJwS4FA*yXL$r!3EzXa1edj5~}sug^$8
zJTJV9MRrBA%!PjUJ7(HK^H$}ubH#Y&wJR`BJv95aW;2T)tL3CAjB{slpKsb9_P}rM
z_JdWy)1E4YTk@E%bH4p%g7sZZ2?6D0DdkSSwl#@KBEJ{Q|69@C6}#t=*zIOf&k4P9
zzAE=F^0t>$d(KtyH(%Gp7OARSDcE0DV*TA}%R=w4qRE0>F26YgPlc_@n5NR;`8w18
zwA9(6Zw{{l-^K>{CI6E9ZZqrdr;S@gpDVK_y)Iwz+x0(ZU6Edr&qwnW*4!!FIy?c)
zrx(m)n|4b1P}WiRrmtE;B2`=~kGu-7t(f%L=c?%Xt}kL<AD%NS&B_(@>y*w}xJvxf
z&lZk9FPFK`bqdUWb=kxwjW>VBVcSHJb(`8=&wb~`zh~PH`I^v;@u6p4$F8@yv0uz=
z{p-Tn2aQ~A9@SRb%r4DRulFtb{-qNxCYM~ME^xUW7&6^^&s^8<c29L4ADbTV>Z|bb
zJMT0X$!-t67BSWGnBl#YiRF7wCQZ9}Oj;{w=SMsDFNS;esJVV#v!pOS?86LRqlps|
zEixte?utFkIWzl(<av+Do(ogMbJ-13uN>g^df&GFv8;c8>oqZv{(?5{|Le2v)p)K@
z2s!I@aOr~sEJ9ltjqW^b<d}6R#L)J>*`3CfD}+oqr$~MI{G=o&F8aKd?v=R_UJ6r1
zmsOeXlyrVp@?-8hpFZs?X6GxeH67XJ@Ub(prYv#s-J`qgs=IH#E8Dt!@6jlMH5$2Z
z-q+36d&DreW5QoM<>IUB%6!!JOy-TRP!FwK)E)k3ib{fnL3N~WBwOZFXREZymrnZJ
zTzfq0+8XsBj%~|6FV1}Fe&0^Rkuz-mwkyncpUE?5?&7si6=+=3b=CiTR%7=zS5C29
z?HEbZsVXN{>NBo2`!TB^^5n0q&pWwZv4+O}Ix+JhZ}YRT^Vg$(Z(a8~*7|>K!v2Uq
zhu-~X|9LjP?#uG{zpvx>$Df>#qaft}?~C)5kL$Ly=6g!VNJw7y3J8j1IHGjAXu9sM
z>1Q8Jzwk=DYNvQ@;dY}Ut|rU199!0M&e84M=}@^^g7?n$TEXtyUfdTMXI^-y+pp+2
zLD7-(LHFaRgU!(g4|PaBpCu`&!aH}T<jYBukG<eN9UvK2pdL}YX9Dxie(pnlT&k=$
z8jK_ixL56Do7Ld5nR#Y$bK0ZXS+DxDvYIE~=%4ancEUqdqli@uD|d09mf0O?Wvp_z
zkcppj^D556vw1JiiN3L`yr}t&5zl1K-5b3PcU14*`r29Iv-8%+hwWvKIIy^EQ*PCG
zp|@*7gTN8PcTp}3|1B1sm0;ed%-r~E_qEP72ZK46SZq4{+vN!J(N-=ct<CnGTCPWL
zZ92JIE;Qw6U)0gC8sWJ!g+Ejtof2jI`J%8{aQBbJ#y`4^+fK|k|E+i_dcI}zq}h|2
zRx*qJf589c#%9miO>u_1rf6_}HJF_xBF2BQKR9|}o{wq1%htRn44N6uuMY1!m)!gI
zwp~;Ld(;8b54W{HE1OlxY)jC#PdI%%>GtueE63}exK}?htzXgV(z5^1BKdUh6Hi`>
z{kh$8Y}4}NQq$F)Iwww0TKvdNW~Wl`l|^N*7a1CDFA`QREIx2m%F~3av#Zg3>JtaP
zni<u%x9hD?w%laCT417^jN|M#Yc>CWUOv@u$q@rR$0&<a9H(3t@wn{b=?I=U`{TTM
znXQ2@BrhJY@O>nF<Aa6!7Oxe8OFw>E^1(`!d!y*e&jK+M`-2=sw{mjj=BTc@thRZU
z-SX^R+vY4=uC1QANFaIze~<_7!5xR+?3!$y$-c?hdWW~Q)1AY1GH0B0e4Jx^j;fqF
zqI0B5g8BZ1jdiQfxQcXKpU~vVvO-K!<GjeG1CNiWaJya-IeO`j?6KLKuH@Lr-Ch{c
zV-vw*eEP7Qv7u(vAGyO0!Z9JnwvJ}j)_b!wr-)2pNMiB(7Ra#6aaxeUf~N@!=5kG4
zby@4>AGcRJ{9>yNUg)egyKp9~;&{wy?P?u+(dhjj|EJi0R`LI)?f=zf|0kRCUrPMH
zJ>D8vq1iHLf2#JHt9y+8?&j#*U{?HcKLg8w(wUuSj2zf(0xCTN^sNqD)6(s4@x0O$
zz_rEUcGTLog$_5Wbgw<)TDhWgwvMBK?LmR7x@UAc9vIDhlI1Ac+jB7@a7y7R$s;{d
zbCz8A(IdlSydz7!_wydRjb4g-PKkuf-+rS>QzJ-G_0aN%OrNa|8=aBjJUoZ>_N-Nb
zVrMd)**42)%Q~;UIQzjbcE^@8$NyZa67V-nJ!4>c*(mpnajaqW3RX>tL$ygRLXESu
zZuVS_6wKaz<h;p>i7)p1ZIN}b^}Q?;;;73U?4$EL*;o5^%k!5;?*CI=^_s7^=N>(?
z`l!+3dA(7_(E-NOZZ)M8&u?W2P52_LI-x;x2FKkSOt%9<e||jr?cn?v!Q(tk?on&c
z73{eh<*Udp#a#bDyT)dJHBVUWpU@=bYYi+O^)}b)W5QB%ucg=;>KdF#lU|g%IPCXj
zi9AoGf6Y2&H@6k<^w9og#;6i7sfNqomy*8K$)dL&lXI?@d3#QqqRhL*e1in{Mvkta
z4QGUIO6+=}yH$c&FnQ))ksA+RaWBf*amXT~HG0XMi7L`Z_AGnSc}b#Y!AvjnwWlWT
zKDBlWuj+%Q*+EQlQXF~=g6qyi)c^AORB=gjuhqvB%UDlos<tfSPL!E)G-&qK2{Zq1
zJuQ2Q$zh7MNv`!r=QAd;!HY_LOl70CdGGdpA-wov_@3R}3_F75s|9_&ocSNZxxitQ
zlWNF1+p|~yM7i!YI+$_AZEDCZliQ~k+iX3$$8fQ5*bTWD8{T_gLVlU-z4Uqc?TQ6=
zF5I5wwfFHIhP*jf1BLIrcycWIXa5>4?Lr^LO;L77RmD6uvIbpXiWT<%vFFa|SpPG3
zW4=9!IscWtKB2j3js5R8uFWwvnkVerV`7g>O_!dzhJiP{K>7evD(5+;0OqS}m~t-k
z>m1;4@#KhA=9&_~u_S=ASE8eF#*8Z${C>#H5ea;u8u&t&<1Eh&5mSpdy%*ouN<LmS
zbIBe@V`rX=|0`4sPAWzO%geh4ePp`n`mjUu!Wmadr=@`nB4^I&T%Duw*I4Kge|(Yt
zZzWN_8Pcr5ho}19+_*AGW2*t{xeaSGn>ALj&bN&+QN28WY67=f)RMn9!#cut7ti8X
zox3}`o8`sw%Qw%6GVZZG>r(VF_~Mr%Yil3+FX=k$9dgxVUhtBKnw;x4mQMDt<=OH!
zQDNU6@2NL#@=P{k@VRHx`+aqE;Mb!8t@E{mrBpK(=`slZp40uu@!0HH4N-<-OjT2Y
z7Dyd_*dG*W*}p+m#b*Mi!-QW3TaO+Mjb=UmJlI(`fi+$@?2C8UY1VDF=aTKyuBGs%
zq{yZ?s6E;DOWSva(AF7Z+5dB&WS*U-5}}iSlOykB$Fv2Dy8hVrOu1Y5^hAO3_Ith;
zN`9vVg@p&pdDeKRhU9r(Keheo(#12EgfHLPeg0wi9*sV0o*OZGdd*McT(rCv`b4y?
zo#`>jqHA`<k%}e9Ud;4uHN8@>QmH5HgVD_(t()C5XQj`avng_?PKj2w)&`>pg+xP{
z*^4bBHcmBldmhqtb9>H&vT3KkI;iLNolbV*)l_J;7rC{A`Icd;b%o6N(;iVgW|ukm
zZq2`%ezatFU_|%Hnp=@}j%S~o5uP@8m8ixIUnB8%QN3-6ZA{Vq>o)0eKH9{)Qf`~8
z?rh)NYmR=Xylr&T_e*BG>bB^|Q!;0|xpDq~@yLItD4z!FX2v@|o%((`h#r2}By_{l
z{DSR1My=&VvyX4Rzo95*`;zp{MXx$OXvc@{kN<5~lhPg<(H<(%8k)eqFNrm90c)Z(
zb6`Wl`Pl4))YoU;YKKKUaWRYy*qiio_auf3=X0L=d$C<8dbxi}M{MC;59YnGcdy>P
zb#~FAZ?WaC#V;puhPJ)2UZopucdyb{VqJk}^TjTq#O`}tal%Vd=WU4-Q^^&-n)*0N
z?_|Nc$R&Xn6PhnL^vvR%yin0_<LmCe-xuZICfr+>7VNb2v?BkE+iFG=8W?_du+5t7
zG`nFd<8#mCyp2A7i@8KM1}<P;wySv2hW#52G+ZC7+SuCpUn98cKu@SZ>$-RCRS~T|
z39K>EtcDV;brI}_5v`kx^UGE|Xx#UJ=UV;+)e}<Pvs`%&@|WFv=+Us^ueukHhJtsZ
z$h%-AW6mWGMr%cna`L-M9+Q<kSK!pU;^^+gYgc@x@ja5C7b3Vk#K^i(sqN@>H(o#6
zUb7z|mx|>jmtWg8ZKb;06xYQ0x1y(pzZ5ln#((3m(*iH8M~8R*t#D%E-1_{@G0VEg
zF?&O;H?W#bVA{%et!_bkpag5wi#?$d%vBxvx;x(2tzZq5Xs%ko>KoCT;9MMLkm9KK
z#L4X1YPqoVRzv@(*)1hcvU1HHUGzD&sw1o0!+V>D2GjMDn<s9nUZ4EGB<GCMQ@8vt
z!D->1@z;;9`@+i-*SNE5%9`tX3r_mw$F<M9C+poH?D{sQZf3A~>7}q6CoPsd>k@9C
zx?@sTL@M9pw6xyKD_DaTr2S+K%Dxxj@8oy(qEykpsT)cbOmg7b@^JRnNc}&x-;Px7
znsxNuign+n<<7}{WN>9hOqjt7<LfM81-rHN?alJv>+CPuBIaLJk>C6*|F}qolTJoO
zetA{?#sgU|+RJX;QFti#_<YyB;8TxY^oFhAet)aqu0rc(i>zFxUh<EVw>55<ynLG&
zedmqFG&N&02A`dGC%;Wenl&%;@t51Pn~pt=_RIUn@BZ-Ei=O1@t!2xn*@bUbl<{&G
z|Nq`QD!El-LH?%{cEbqvFUwj(U%ZNoC=WdGX~Vaxfdb4`0ft85#oOzuR+OKQ``jKT
zarc3W&YJzdbS9=aynlWDtl7bL+g#S)Wj&(9{5Iui+ZU!O;<vYQhW2sk9NNBP{u|yo
z0X#=@_MCsSe#O&WAKnP^=H8Il5v%w1wQi00oqNtrlb7b?b{7cC%J3RYJazlt*Dr5N
z?}pu9%W~({zFF%Fn|J<{o?05DCHq%9_0sIC2H_7jq)$9t#<bh&%vT}dGlmyfoY%CP
z{4Xlk`K5pTufr17&vC2^-e>)}moK(Bp<Msp<NFUz*0?avzkIH2)|oAry+R(Y66GxU
zd6|R%%AJ)T<R8{G{=Z(ZgrWZOk;G5mKXg{@sbCU5Fd?K}zVLhaQR&j7BHKem`fH36
z3%@M%YBM|L9%j}MFu}o5sF8(N%%|hS0;g6l7Ox76qrKW)Wm>y-Wn2uMX(VNw^hd);
z<z%m#<2;$nO;dFzY>V4e6L~spZP?nZO^Hj5m;24#a^vEn@YVM&)Pasgy?#DCXtAez
z=E~(FD=)@<4eLw_T_3;Ka8btB=%p*KU0mjty*7HQmd2%(X|uPLMv1NqUX}X%(O&KF
z^>_D`{{Qtk-8;CYjZ;WYZAL=Eq@!Ju>Ulhpg$$arx5b?EdG<+6Z}yIoXNOX|GWZty
zP8VA{i(CEbTG#z)y0gx_Hd<?W^S{{K*{Abvtt-9G$5&$^et%DVx!^g~of*$gTb5SJ
z+5Y_EcX{FGZS!oaB;VfM8GZb2rb1=&)05VA`ErR&nz<fZoAmAN{{H;-`g*$SE8Bm6
zmwaJ<5#R84<41YToSscL)0X5ce4xgyX2GTT>1FtWx~*H1Py3p04C2;|Qc-Xe5^9N1
z>J*shzNkxc)`>@5`r9~^dxSSUX_H+WQ?gjs?9q!;vg@`?QTA|pcH?os%Pk8P4|gq3
zl?m(ad|J{LeCrBV{lAOX6WevJKJE^Q*LvC?;@N1qY)0ZYXK%5>xidv<*K}+;r4eya
zNz{7Hh32#wX`fcA&#c?VnGsygl=W;uyHMAPMd|-<9X{5fC>HXl>Q$iC%O&lNW^T>1
zCtZ}xoZly=wPJD8vZ##J69Z-PS5G^*YW3PfzgDf;c*$z@+O3bOR<GOnsp@s&elszh
z4Trg+bv8uH%*bgxtu<Rm@|;ujgUy%RyiaYp=2We_`HbD|+?IP~-MQP>>=D<Ke4Egb
zIMvKkP|}&*@LQhqr%TrB_kOWk(!lg?;g$7X|E94RxH3-;St7eF=F6oiTd#x!PhIn5
z%Nq^bjh})8oFi4I&v)Cob$YN@;hyRCUosBzolyE7%pLIM#?y_b^$*KzK4X0OPtkGr
z(vw{sUr$NT+})vR63N7<vVk?ty?$EB%GI4^T9^HuWoKX6boZIG`mz6ky3#q3`L(z6
z<I2~XUOi{XE?cxcP<`&r>w)dDx9^uGMHg*5C}plbRmeB9s_&`pg(RVji}5wTzPj&d
zY!04b@u-8pV#njQ%irHNbNa1Qc$(7Fq)^K*ckWW#)(uIQ#TZVUQ0iLlJ?mHJs@dm$
zb!qqo9MiSEmr*czi+<;CSD$}3g5C#gVp;NBDy(w;G}D`xmVWAFyY+njdGqsoKL1TQ
z$n~Yo$bQA=i}Ct@=gEBLtoznf<*ye0_d;`>{+kBB(@{~|lv+F1Z?=2+>e<?P|C1Ob
za!Z!~FFzK%NJ?P^i-y!C5p|Q5mwH!KJw3qoPOFK3&VyAulC`;`CNyn{I<sC(c_v@e
z{}0W_vU)_03G7OX)o#5Sq_{PzLsxv$ht?Zi{X2zHbfuH{VvI$mZ_Nwp*V?++OF?R)
zvl!D%WjnSw&nZ1gZ@wC{+ALA{spWj&pXnqSMIAM(CeKe&X_GHF_9R*D6#m^2#J~DT
zt7cwga%A2!akFEpeI<>_!ABIjcJ7#{SNZX@T%ek}%_rCX+&kVePHEGlI9lZ#BcGfS
zo9VIS#!>I&f{ybOP8{*JVw>zV+3CN@=b7hfE`=<0RLI(KpmDixZuTmZzQU@{Qfu}a
zO*?BTT^Kdf>-n2B^Av08@;lE?M>eU?xGkAhu9fQJT{xlHbHbt8S(-c2UOl&Mo1OM(
zNv1;9wd1q?bDVnEylbX2r)!2&^XWVFZjF1+=Vp|A?hbtYrQJ$m?`79FQv}7Ewk%Qm
z`NETP@}&<2UtAC7b|%JI1<yb5MZH<bB;c0zd?(3Wwkvy*cy2V`a}qSY*rwOD=*i_3
zKhJseS8d?e5EXvuFL1N_OYYM}Vqe)cZ<X#(J|r?zK=JOfKigLZP1IV>(lRScOyl7E
zEP)kHyjp>^rcOzUUstH4IBRdas<S*RyVa&8aa+oYwZhi~V(veBxMRgtUHMhnvC$T1
z&b><wn>W{V?T;oU&RH!6GcUBL@iH%#(A>6ZVM5M?j88XCnuKlM&s3J?k$5zMDLZ1{
zuAKEPHp+oQKZV04d`#*z`TtVEd0E8l+B5Q7JD8#*9~m#uHC)+kIqRa)<C#IM0aJV*
zYi5T(-nugO?bevnQaPEwXRkZ>X<Nu;RsE~ew%z}=DfbE||J?=7cXkvmw5ff(;Qp1%
z+n@Gbn|bNzv&SOgcaO$iFMRxDS@VwSoy@&Tb~!g#8q5-fc@_0b)+#W+ydio1`!)T2
z@22E4h-_fh3Vdn%tMxo*wBUs(9+8For5cU9)9sjx^$#+)$ys|{v-udye@G(q!}l}C
zjlOZVE_v^qVjFO3DrbDjL6LRG#pF!m{{+5T`XQH}fwANA+*cxv3!~Pql8CqxwRBae
zW#5Coe4CKS)DOou1gTB5)reZ^l^~pZU{&M)9jOZ>oFZ3wml=glUn;!LErZA6h$4Tc
zhMxSbBZ+1=St6gCN~&fRb+UCPo$q<`OhC?!J8{X3l#3S2oX!<a$iGrN@97M8E2dfJ
z>-ka^JPzXajW~b4@r(Anml2}Jl!PXV1usc%O>%RP+H`)Fo=F1RNs;)TMFzIcXJ%fO
zS$To4C1d8np7e;bl2@k{WiC&N%86MW{d9xT)TGI&g%!Epk3LMZ*gZ#eL%rtQ<nn{o
zr^M0>Q>B8{iUM5DSE+5CU8#B`HhTBfJ<A%`bneKl+Uz)$`MLS=OvTSt=ekW5nU{F6
z$Nybvag|FUs`jPrmM;rDT-B~QePXhC)G|$TA%|F@qo&RO4<8Fl4oQkmo@V~)L-Qj3
z+KGIcx^7aL{OmPLx$pltY;C7ey+MhywK3ht``o-mg<4My;wl-_|7?-twtQwcZ|9lv
z(kHW(Bkjbhw9c>Waaw65y2?moS6FF=Ue46lrz?0L?eYkGc5aeX%gGp@dmTJcTh*2r
z1n#_<xclALS#x#X#P{FSIAXAD!ot0Jy`@WzD*g62w#_H4c4~Ff>%NE$`b*`43|<R2
zFMU?Q=^HU8W@~0}_K&4o&9b)7Tbpw@Ej#p<*!HbUm)_jBE&Ccn_>L2N>k23TmHoKk
zZQc*Pi!W3eVlR1nab2xbD~;y&eZ5Q0e$wZ4Pxo~y94gG)^PKlZ+bw~&V*e#u?tPZK
z^&!}{N@Tj;PeZARVr*3(o=e;OV$m;XYTY#LrSpEdH4(vH%JUv}?Oy*#Yj=e$ON>TK
zv&7#U)9o7nKIGQzxpU-gj%?S%)eH;;n}W@13wz454oLnES#5Bk@8F!iwWs<FFEE_)
zV!Rd;m>9r%^3Dc{nHzi*&gC-hG?`gr<K4<IH#D?+s%z-f$_3$fCtNIy+);6J$Cg9<
zH(4$pJ$~6UaAK*ztkg@68zKcgC#IXWbWE5kbmZpEkCF<Sx90IaT*i4zP-x+zEv&X{
z8Z|l&YKSN<PmF3_>wG)5N7C_9n8HE@*N2M)Z;6X3E|W|Y-_~kB;b6yxo4je4q@skR
z%`X1mC@}BhoUrsQ$1>-zX>Ogi?&ZUrH;PY|2Ce&;s37OQQSi2c9ecr-NgG(%Z_C^+
z7nr<hX3SC-FLuuZVr%A=#3reEO*ixP+4O?_*@Qz+D=zP7e5CP#{rLu`ilx%;w>s@U
zds$m6UgyxGPiOr_Cb)i?5&vBw#C2kShc4IrD@-C%r`mUV26DL0SrKyFH$*q3DU9RJ
z&%zj!syilL$);9wLN*+_w=-JhmjCMw5vNMI{%^d)uvF~w5-!GnB34`rR?Sgnxp>Fg
zY{6Mg(Q}GZMJ6;({SsrxcK5<dW&5s#5ZTzULeWcwp)Uklx$7?eNszr__{8bmf)g8=
zuJ^{?*?C6trtGN`uK$lMIiqEx7}3CEy~gR+g~JjXoKGaMMjq+8u{6cj=HZQ<Pohii
z-nn}>dc~4O5z`X3ct?d^;+Yrazb`SF#bJ^n?`qk&K(#nIy?OF-Paho>%Z}*1YH|EV
zKwQYZwmT2nZl37PXTARHY;V~f-#aCTia(x>I5qp@LibLC$!bn~Zz99%HmH^zz1I@q
z{q(|9b6;=uZB4venRYD{Zx88A(YU|+?EHjNi}rash^$fg>KpGR=w-WuKRlterPW`l
z@XXvZCp8wF*Gh5n*ygFP@OWWmO3;bUlL`|0dHz8ihcwpQ&(czJp1JIoqw??A^C}bX
zbn-5%DHO=KF-2xl8^hQ6JtreF);#`Su&RwwN}WaZ!77ahg@=-hnIu_+rt+M7T(Yg*
zSEQpplH<sSm??jg*^8F5&suP?^ri!!>x`WX&b6H2l@gq_*L$Vz5vOH}jxI+MT^KZ1
zOcYmJD6q^Y?8U{o60SF&dY;k{`YmzPs^_5Al0_0T&UmbAI<-Z`w&S1{gV3A{O^x>=
zXMIqYPMsw(?Wkv;qIlDa<45MU)oCoB`9d)7g<w-uVvQK@HpK@!S0oAe{+|%GT2z&N
zf{>==*{7PR{hwa+|5IG2$*!mAt|$6(!lg@v29NS}Z!5n#8>x1=Dn`grf%|nLr&`iW
zRkn`psR<1sC*Dkaq}HO;)U)#3)XS9%Cm(*2l6X$(|7@{koq`iPCO91sjp<<Ivh86?
zTQp@x3~RW6m{*+I+?ZCiD<>DOZAw;>3OM5VOJVW6jQMI$R?gdScuqjFejBIzGf$(e
zIYwcxmPI{Y9=0m2b58ZrzRi}m|9<Fy*t$5sFZuMz2!By^7DI(IKSS1@%CfOaU$>9T
zHYk0A%FK$dkL`9%IDd4)dCBZh&J>4F0hbm&a9r12uc@;4NVl_@$`#Ee9WDpdemEsu
z=((ZUx@773x-Q2koh<t`%@Rw_?_9RJX;sg(t4zzfPHJ>G>CbtsVIdS=!5Wcq#-r}&
z!Zqi0_bmy!z_e&@%B`QFZii%JLLMH!5PR#S%9$%q4rlq;W;90aeE$Dv->vl;2Ltxa
z3Y>SUPf1lv^=ZG|y{D#6Prh?|5}CRoHdX&DZ(hdJNqJ98R`{0R(1~DjV0w~s?j1k-
z$2I4Q0<J#o-Ez}q+O6YJV&|gM#LY^R9{Y5}PI?qJQM2a7(<4h}$8S1)f6Gdnv*LSp
zp1Zz}L(p@|mzo>yohd(3wRDf1Om$LQ$knPQ^XfoT>z%CoX-Y4SD6Dc{_AG2kTD!uE
zkQ1GjUpY^&xt}d3)6&QE_2Z*QU7Ww(aO(Q_%t<-g<d&AX=)V53XPH`IGja}EmA=ib
zd-syzz`llKtJ=6$f4l!*F@WJ%R*~HD*WacT>{4ELY2CVMj2Fz@&Y7_>{o#85Z57Lh
z>;Fru7L^q#gjPnDE4}AvQlG}EGX3I&&q7Ci6`rsySzhB7=&~mA8{ZM<0~t<JuI@ha
z{F~E}uWa5dBGcL~9oAan>T;HIXH@?!xh(q)hj+}<-lcWa)9Hm(k59{~^<o@)N^jS<
zv>n}(x8&EOlOY?<h_OX=O<E||c}PNH;X$cZv7n=~SzaWVz6vaOV&xI#vXsAJUIue}
zL8o4;{RStIFLR}5-Kt8Ow0@nG#>qsDT*kg>$BtjRb>f@ighv|`{za|lTr^Q=k)l|5
z*r#cyi&B-FmMWJWKgXmiY+Weq{PCPy!z;<PYGKlsq;ySLIcC<Kep*tf$@QL1jzjy)
zQt_kwk-2MU?^^W#`Ld6na__oaJe0P{C+iuf^^#@wYn;P0Tx}I@ExzVyUtu6GpJjXF
zb=ZodM?{Juc#fXXDLRyywcP2`f~aZVYHm#b(=sFA#dLwNjBn0abw_6$*=YRh<6<Tq
zm1eCKY{w)w8~&Y<b#mf~#25E{7v`;ACALc8Ipd>Gh1(vVKD+MiHNACe3_=WR4L<7$
zs5BUI7oX!YHd1GJ&s8iGp|pW3d&9Tnjd|*(lNZF!d$5TwMSB6aQ<vYIRq9*soff&H
zEPi15o`lsRSD3c9-Tkw!Ij|x__s!Z{S*?GxGXCCL-MFqPwC9jjgsXD#(W5a-j+p7V
zm1UnbiD;<e`O~It>m8!+z*_&P_`l8WlM-u~TPJnOva-(c5V4!C;$!s2r|WJ<pNX&0
zn~p~*_dT<PH>Q98)w7CMRbye8Z}Xd=Um1MzOLOL~xf;;xznC{H?CC9z(!$Jpw-3C@
z*Ja-n=`LjbRoJ;y#M|`cjPUwIO|k6E=(w-1YolYU3?;lduKLYxnP6zudNX3H;omFI
z99G0>|6$UscsskRV~*$fg_e)CCM=oJ(K<ivsN($Ay-cmH{XHpKdO1;gUnKRu8!UGJ
zcJfc!A&rz(dwyx&Nn;H>uvYDU;IDJq_is&%VmZ3~&{}2x$I9y88g3qz<Vwq$uJw0m
zmd<=VUFPLkt&58;nH3&-mviJ@(Z5al*S#kA3jBX3S8Tv<%=AL<g4AXs=C!s>6F<Z)
zFAL&kSG%kJt6f5Pb@{B9#m4!ZS>G@I6fa-g{e|gb#8lC5Ve5a)G8f6z7x~*OlGC8Q
zrIBAKusSK9Z_&aUFGrVicRgJ-*j{HxHGDeNvXE)H31|0)2IGRxpf_3*G;D8Ql-R)2
zu&%%95a+xMNtLOx0R<`c{2y$$3wjF`G|K<rn7=~GGPZ`Dch%q6o?o(@H#2%q8Su`Y
z)OWAI@BN1UX*%n71_>~46ySD0{b`3F`)x&~LW55C5b2*Guh)o$e4G?*I63~Mm~xTk
zHMyOQ!4lJjC38F_@3Ta$E}ZV&*jDmWN<a8fSV+W`lP5}U{C^szl^MTSbNA0<Uk|5>
zckN+&d?csn)EC1)hjQmVczJ7?($Z%eo0PkvSYFJQwmP0U?avAMS6Ak1uV(kXX_B*E
zCi|uq^TnK^u&Um--<Cb+I-3<xAYHtIz5HBPl@a%L1MbbQGM`V0H8%?`-oX*O=6{S?
z{^m_-#x*G%7VEEfZBO^MUc$1aN!!D5>$3$yE-oxC2c>p=ZjjgUTAt$Ox#94<FK1kG
z?w*a^I(v(a@CF|zi#Y2<o6RaBNgD3S8tvbkq6Cgc$(-^@61(U5d$)FCaD`{*$1_`2
zS#WQ5-%zk7uPVp)@s<-ze|q#&uO1Lnh<kc{O-^9^Qvvxs{n}qIUw9LK@6CT9<)0Jn
z3olyij9#T0uCE*Iw6i>SvbcY4+`XsuU!I=KKV-HkbKlVpp&d@zJ`Pc{J?3;dWv9hI
zliHT^>W`-TeXTu(i;isEmvl6>+$g0!J8RaNxgx(<cBftL-nM+(&ZoZ)+n4C&ttwmm
zod4pD<o^>K^S!KA$V|vN{Vw-%*;=iD*UyZWAAiiett<aL7sCXrJMx?FFf$g)Z@xRb
zDp}#+dbag5%gXO3r*E!^{HT@Bp_O0S{=CduU%R6G@QwuQ1!jDUzE{qd`tJ9yW^sA*
z0rRTs+kaduua&MA>d6%d6Kg!&>MB<8YWBa*wjYAW3?ogXTDU&^TA?5G$3-Ua!z=zD
zOt0(yOR2X0{<}|#u|9$0|F*nr>4Ir9S61*ITETy)B%r9Rd+{-z(>pZZX7DA~_kPw;
z_*L-z{g3bbrAk^ae+U;&dMC8G(p@v6=BRU_&>#27hwG17@2qQCWOlt?eEQ4JUfXxo
z^2IQ_FmW<*@|i>kJala3l+-JkaU?-VGpgq6=XAlxuDxG>U3?v$zu10)g6^a{C$g@%
z_ew^tE4eB3%zLI`=D97LljcneHU7PJRqCW<&)KG(e5O$%F9Rp;dv#Ok>4XJMD>#>U
z$wo}g3R@kyUd@9c;k1}h^o|L04?WF^+LC>Bp6c&yI^sKvUrw4C{p?-L?xOFPo+gPK
zi0^4;6<4ddQIK?``@g7pTF-Xw=_e*CTAT4Oa4;}%nB}t_vSggFKHgE;GeJseW!Rcq
z6SB^>iR9een0bEN*;S#rcej;)fA)8_$jk?aTgBDu=J0AiK6HD_s;ld!etxm`mWn5X
zVQ15u+grn*&YP=Mu6p;Vw#5IrHQzr!QPr3@!KM3T-Cl#bC0p~Zt}<Nx;``@#`NeTB
zes<bLxd<L!{$KA?z9Yl`;(&u3l2Zf@u<0*4e~4W(C*bh^s*5Q+9O`olmI@{seN+`O
zj|o(+Pbs>nBoRAjqFa5<#*1>%vsOK~4C}a^!mWAFo8R`!q0f9LmG|$~W7Isw_1bgC
zlAhD+wbt$tI<52fQ46=;&MBu)>plPUe^QUZ-ZziD4J|dBg-vdj6rZ)YS!3*Hp}yJ3
zXy>~(i-gUsH5UsrxCu|W;K-}HB|x(%I%KPxWbKuUZnZ5_E_q1Ky>iLN@ax3hz!M)z
zCi|b{xOzFrT2#+G#8a_VG<r$#R<Y;vvO2<6<g69F-f?fTLRVBv(n>Mk2+4pcfo`u>
zh?TvP?3@-DaB97LncKw{u?0@QLYHK3K6))GFLjdt?Yvl_?ej9X+nOsEWCw}gE!e8t
zb+=-3?y9*_5g#W`^*weWsas=9hf|lzq62>6ZQHCWR;?3x)|JS~Fo8jD`;A#U*6vWT
z^VaKGDbu}v&yz!|A5ZwNw0cATXQeaij_U-^d^%NC`+v`->Dtvkn=hC+D;PUv|MLl$
z=kKi0X}BV|SM)}h@vd!)I-m6hUvJB{3JuH5_O+02oqhI|+iJT!OaJE2Q6Z0}n5~;y
zHMd!BSJ|xzkF4I_TAgLH>X8}$yIq@SSMS|()%nxz?VNeXU3s3*&X+a3_OVnnux72Q
z{U4Wwtp#BfBB?xmT+7`*a2oSZ_{?oBf9LZBNBP(fm+Z}BOAh{(RNdEQbjRlF<<$Kr
zN~H}~$(Bl|@44uDuk^e6aoOd+t9TUDg@U@Cl;6$s==ieFdav@*<<(WM4}JEsezW$r
z-I{mH?=&$_vOHi>qkC|R(b+E-<tP9Bd{=++di{sx^1inFQ$9cb{dM{OHRr8ZUw^JI
z-*lfr_3|=?b(#U~K?;lbR2zLmxSD2mJz!CNVZ!xmh3OK8scf#HT6}#?%~6x3a*|lJ
zU#@XuW^MFX7g3=tzREx;sKZJ7(o(^)HA=#(6WsleIq=uNNN@j|(E8}lX1=I3Nl`Kj
zS+_<VTDQa|P&2@VOV{Y(YK=I-F26t5f-0WOsqSs!{q<=2eg-wcH!N>#GfW<TDpQ>x
zy6s~tx3ff;SlNWNCY?Oui*LwjFEgAJc(SJSVo-d@G^6QTc_y~|_?$^<^Ej8mHfiFQ
zGiR-ujW2jK@?6Xek)CqR<U!3P^Mx!=uezKx4c2qd*<i>g9hJzv#aG=b=kU3EQ(XR=
ze_6BK!{t({-puFoR9qTFRk;MLd7jT#bkZvfQ+dXB{o>)cHBLO%OWGye9%|oM>*1>P
z$g_3BN=3fX;+Uv}<L16!7Vu15p=z2n@qNY<?&=-E?&l-VuIk#(`dIUP<>_MSgH<=@
zJ==TM_Oaj!y+TRb*tqpoY_UNMGB0d3YV{`wPJQufRf^iT)|0hKR`Ij<wWwKhwk<AR
zHTl2IuT3W<3j8%p6&5fYb(pSK5D>LOfi=@A*faChvI)0BavV6^f_h(N*|RR>DBi3k
ze2i=5+8a&Mk8e0FKQ?jxnh#<2O{$^wYn(-_C68(x({%B@(j>RILZ)uT(TRVSEa`kV
z>($N^OT(7_@6cBJ^wM2SW&6=3TTIt}{ouMcB9J?*H;HE!$EE!#5(bB7ecX_hqHKTj
z#Fdsk6I(eZCm)hfyLqbZc+$+bkB^)&67Q`o)2m<NH1*U6_eq6s^yRdB6eD*>&CoQO
zxN+*u9Lb%V=HA_wZq3|#-eq&^3kly$_vl*%t+S;c<mH;JQc}&av6Oj`;yZOY%Wci%
z%Uv%PoytjVUK+H1^XeUYeHAJ;cU2sot@5hWJ+HcW-ffm!^Ei5x3g)h6H~qWWW#zS`
z)qRs!HH2;KOALK&|J6bEWt#b}sXP(~S^Wy$Cq1!dzn;vqxS*iNAjO*PdtUkO>hup)
zCHg^Ef`x@V=1sVxyyUCVw*Pxo#5R`Sy!}Y{_u2eyJsa;`icbDAuRQO;&ceR8)Anw+
z5t)#$IX~u~#6^>fuC<b`MZZm~?0fAjPkv?AWc+BJIw2@+>D&|Y(^lRI)PK#{pxgS=
zTT;Sg_BD~S>C>X}3`#ZU)OCq0ymf^ArO)gT%WgB(DSdkvZEkJ16}=FkD<QKnc;|!(
zOXMSude(~{=~MpqB)076qa{l-M0B}QRg^fVN$Bl$-TP^)=s6~xj$<A-Oy@=QcBSeh
z%X+_Xbl$dd<FTt@Z(fA3vM_kfzWOz4*^-T&UR(4VC%)Z~7@c#><dWgEH<Ia7WM^Ic
zaAuOn1cr+ioU^Z6^qOC1d%Dg0WbVBMLU)$`mx<0jw=6ayZ_)03KK_mikDrdc_37@l
z<VoS%#2bCDFuA>0((3+4Sfr>Zc*Zl9^cIb^BBgSB=E}64yV0<OQ`bY0DQLsd_a(=t
zJU<a^*^s;AQ;P^|T<x9d+GZQ=nZ1f8PnUVB&c4F_x^%!UTmE|n>EC;MU5Y=ynlasH
zQj@b{)~)#ecTQyO(wH)HPD-!W&N}6<E9SUaE!P(Ru%x=F%0t-WnPIaU*LSba*RH-v
zoBL?W+4`ps{9CKGE_7SFV#$Tgx&_LzMq8{lZ4>igSS7^p=H^_jisi`__cpisEH7PK
zP^ukwGwbC+Z>!6v9|`8|v*4<g?cb0Z^7uf?sq{ehGVZ93n*RsbrX35`7yGiN%=8Ys
zY{upN6Ru6!`sNj{ngD}_gYMkBTji{0Z@lnn>4~aY-t%(&Q|@&0=(nl#oaD(gN|L{y
zx^?c%#IopF>E`hP9t^%hCJuKVuw6fQo&DXn_G<m?<oUnvE%&`TMJ)Dn{aTC5yylX}
zYPVMHt+%=$>B{8vFZR<s@#k?fxBvTj|Non*=6mDbOy9zAU##bVi+k~om$P^-l>S;+
zCb!|B5zpZ&_lHr9vX}C?W%w#9RG9+K>@G`vqL9SAaWV5vjVEsJSnM}E`SkMWtMf+_
z-q<8uuu0j#di*pSqrjd6-Bl$Y{=L)xzic_XFvInRC&ycSYEB$~zV6A18967l{&y|v
z&|BQ|?lf0N!;&?p?x{3v5(wDPb9%)EZtqiEtMkqr(_z}u@cP__*IFyiUh~O{Vpe^y
zCdA64*-Evghx>fbvZ>$JyuHWwbxYH`FX!I<6R4I_yKs-g`iSPG4NKd!dIftAxeEDT
zzHu-vK<V5uhEJzXeC9j-*-Y?inC{hc8CRX&Uv=Afm1pA@zlmRh0<VRA{Nl6T)1<pU
z>0*E0daojV@0`F1QEoqCxPMj!PMnZyFemq>i?d19v70{3jDC-;TG}SJWSh+~G&|*Q
zKC64@l8d*!ICmcFw>aSTcSoD)9Bqp^i{&2WSM2JpJh1rxkMmVId{u0#?%va7yPa*V
zq&K6DX)f3Qb!s{~ZfYAI6{yS;^PAex5^pBJ>ga!z@6r>eiLAaKn8i2!6Pol?N$BYf
zZl&OJilGHh*;ef}xpXwCZ-L#XX;nSPcPyTEM0<LdfnF8gi?B<aa|IU9Tf@0<!V8O2
zb#Gc0|8ZGj$HlYP=hd`xuj-5#1su5C7<n%i@G8I1xEk=rVQKTMNz21p&$=Dt4OQmv
z*Rl$kC=eN_EwA0%)cIlGrw<JIaVBfm@6(9?mbyWWDM97Du&v<}5uq<37e!`vC3p#n
zK3pNXJxKqv(K|7vudh753MFuV$@pfGty}fpfH707w%?Fnc&kI3_~%7Gl!AZuoHgoy
zyLFnr#B|?)Ne6x|T5qEFzw>XG_wNqv-#h$w9Pu~x{`_Y~w^_(%)1@1;FZG+veJwp*
zIroWXZqZ?ud;|8k>+`bSnAZwhoL<eUtnm6vU_rt2vRR)GX`DNl=HgzGVjA{1_*Lc+
zmvi&o`mDEKw)Q?_?fStw{Ed?AB&DO5ZL$MwGbh+S{>XZF^HZZuM|+Ab3O87kZ(wg0
zKK{knKDTF&Py$!O8IIFQCyu%6wfT5Y3phDr!wV@BE?JS6lh&MC6Hvd6r;#mr=`sVS
z-8>p=`8c;J@OB%yxPN*5VuGgaiRL#w%ZnD9E;_8W{BYH`Ah+Vyga@kyMGJMZ55|3X
zt;7E5q1k$2%a6jBPv~`*eCaFE6IJq@{(p+d)GvC89s21ZUW>1MVG0pjza>e^n3J(n
zui&Cri|*DAUJ3olKGDl2P6;&eQT{dMqRFiile(9UQ<mM5dir|{@9lSAB-a{Bt^a7Y
zBlqu$)fRche~w(dvq(8i%1D_pi|x2Gqs^xL2f@eA73POc2tRQ^?uX#RJeOeY2@l_B
zdTa@{Iatgb+@}z(^VBc&X+UUb@YJJXlM>`61(|=cNer0J@`3gD%DWMzO3{ADEQ<C@
zO+98~#PG3yT3L_1>!!f#zKmsls>_!cMov;)dciT(=+wS9YAYwabkR}|T~)i}jhB_k
zvPll1E?=516)ZpW?)9NQA%Bq-2QpT?Fj#(Vjn;DIu&Doz9xF<`a#@ugaI46DyioQf
z?#t7UbtSr{8-+uCMP3`}eGFm}IiMgT7O!j;uR1MBeH;JZO<xxzF>wfcf9EiCTx}?B
zZ0O&eqHsAS^mU5%W~17`)DSb{R5gv6E+%V|eY2-|1}&EKo@^2robFw0y7T?59&htC
z#s2Od(%Hs22Yds%mCfThbJZS%<`(RIGudMGG`q&)ti%si+Zg`cTG(t8D&#P=ch}^Q
z!;@{!Z?ks38ghM`Lh@Fn`@(tGh3!&5WIdji_x_sg>#+Q%m!EDj%YQ9gaI%}d|EfJ_
zxqV>(ml=cVLH83U+*6v@scwJ5x%8{!%XF^(bxvz}>h->y`jhUo^Z!+DL-Ckctw*aS
zyZB$%$jw^rvqiI-D{kU4-i}LwpA1&MnwqN>mc;wfz4^;$k+3fhE_uwn;<50HN9Xm5
z_d?gQR4XOJ^mN6_=Wbv5@6*aj30xN247i0CWqo|k#OxEe%<x!=Pvkdo-&b2Tnti7h
zR-Y3$%`4-USnvAui*#@I$4zS|FH<v1b@sRS^nYuzbDLo1%I>Tq%S3m1m@iIdKE$n@
zdNuK`aK!V?;nGvt<SVABSt#8O3)im*HI`^Lk7)j^*(@&FVt?bQUul@)RomZXPYb@z
zzh~6w`R-|4gtDk<e(Th>>~%J6Qv(w%BAd4!5iwV-zY)1Gv_mnxNV)vvy3lCj|0U6r
zcc^6+Io>_~YEodp5{oqXW7`A7yC+YIQC+(H*49}6r>5yyag)1t&--5a@KSlQuBUWT
zB{Of*!s|T`Bl;fC2-8`8P3?5+6a|ljk5$I2zX=JaFz)mU$W9IjGxBDeI6Xi0^c0_D
zx9?}nCD$rXx@exZHo11O!sL6?)6H-GvamF@URgJ>?av|pDZj-tcN+#UTLheTGcPUX
z=5XUu*kQqZJu7o&)=D$Ga8ujc!hyP%o^X9qI(WG`bE{3NK<jg{nQz6iKZea*T4wR|
z+D!ilyZoEe*tZv~pP45WF?;jO*^{r%mKJMTexQBn5r>1v3;C~juWn)Psff~?9(8+W
zgUKr{<Ns5*|Et%(yUD%rLUi&ZVYll60uIZzMJ(93!)15Gif12UWm(UQwU&K9V>zv>
z@A-^Mbt|vi5pL7VdsVF#nMW;7^{rBj(01CD=pMzW=FtCpM}KPn^{^`M!i!>@nm*~-
zKlV+Zcyzi!)uknQVaqR1Up~`?dsedPJ@fSF=E){EO`nuxecFC|TGj0o;Q(LFtVP)k
z`tPPjHS&j-Hj0-n5xp^EchoHNORepaYwnuOu&ii4dTWhoh0@6=C1;C3_lT!mKXs$u
z%*ZY+xF0yH+Os`%YRlcg_U)GK#W!ZN&)&dYy+NH}qj2?H@!8xS4Ll0M9m>_-?K3uQ
zbaA{d`D$68-?TnItN+p^<&(Z|O6fYjAwX$oExXj#shgICE!FgVS}JZ8q_#9vYSwv$
zU5hQHw&zE0FW#MWU!vST>g&B7OWLcMysDUvrO)+tFgTT9aOwc>jN3cqe{WP?mTbOy
z^S_9GyKmp!(sn1_O4(;U@pIC|+*?Mh9!75723nEJ)hlaXzin)5GwHQjvDSZI^)@Nn
zol_#0{`vHK*M;-@59>C}7yMUtdDZsg^Eb_!l|D1@{;Zk8G1GDlR{t%t+q<jf<*Y-$
ze?8eEwJx?``gF^g{gQdF%?{0<c|>&2wEt!^Rqs6AB(<UZ#)kMiI|OGlt!QA*n!|0?
z!0okwr)19Y@;k?U4;-)FbNnNd<NYM&JO3BW;hkc5z0!Ne9sN0T9Op}HUOuPj`A+VQ
z1>E!V=6$SGU%bcrkENt?r3vS(Gr_Y1Ber+PUN5~_5PYoSyvIT9CpQ*VM(aP1xv)d%
zyzaGgFZT$@CyCbnPLyRzQk=0vm0{yV>*LdJ^PEb!7|XCr_AZZC!*SKx<K72YbYd^<
zesHOA56cOMo!+Z=InVtTVYPXSxOak-@yY0`y3EUWCoGTOyF7TYRpjB?yVGy2ID0KV
zxh~Vz(nfR2`;d&ukIn01Z!CT7ALweirEnFyjnxSS>yR_Ho2rgJ4ZdYuv)~l>;l;6R
zt83S)Zfd=H>-NFeJC|?WD!sb)xz%*dqQj0BcVn*}dVI0f{eMKn%DuA_?`)7Z+o;~a
z)Hj=ZlJ$xDKlfX09<awf;GVa0cgQB;eVZ13-88kJcFv5=QzQJW%sY#VUYUt_OLTO3
zSDOedC@EZg#<{LLeEOR+=iY3YkrDHB#jmWeCpSD#Jc=uie)hBGS@u4~7V8VFhZ4*8
z?Wn$Yp}(4IQncZM**m=#a98|!aryvD-@QxPcUi0)F3q=lwNUP|`P`SXvag%}Y-D9#
z`u)W&*2vw5?N02vJ>j5rYPQr{!No@Mkw({Md)}Ds8Sr?^L+iTN`}TdDJ^5&U`pc00
zlat>cUB7>_*wpr9Kh`}{FD|os@jl1*&Oz~rBX5$JjwnrYymPQ{=aI#dpI?g|dHsLZ
z=giW(=JOA!%FJ4L^OLR3=k0%UBW%8g$8Siqx!*r$WAD8O>GL0C&;Q1|@5RF3mw(sl
z_02i4@A&t#KOH6-ELyOqC{bu$|NJ6Nk@?$WieJurWc<DOv1RRnKPF}8&bZDyW2t)f
zrH*T?@!LBE+p@#Mp8YtXuD~cF{x^5t-^_b||JyvzZQbE*^`bHEMd!So;<XnW=j^nU
zRq&eeBFHIWW|JBpmrllo7RQ!Y8MQePZHs0$v+SJm;83E|aZWA=FOLNZ4xX%v%L@We
z7A-wJ%^-f)okOi^zEjP!cRg7&S>1P@L-nMGMq4zd8@W#|^Xw9Nd1;x?VyC%UTeYsP
z4x3sO`Ku(m>Awo+gc@1yZEqqr2A$ZFb+jvY_O><mcC~7o>&5QKiFkT<_4F%o`)hx0
z`slri_u;_~X8Ew586uL$C&+vE@yvYt_}JQ8S1;Ykdwa{wQ^aH9f|e~WFL8NjYxG8$
zzP`M{JO8G@udS1=ZY;cg?d<H*{L3rzZ<opL{_)}QvB}Er^Ny5!zPo<@>oPsZUGHC?
zs$PFjuckU{%7eq*|Lq(PwR{#ft9Y^^@bSU#s;71|_}lOM+x7MH{rl_p_tgJ)X5D&F
zM7XC+I)YI}z=46mfa5^3fZmD&BGEUNOm4ORx+F0`rtHPTc7-;Lr1m)l2R$X#m)&UX
z(phvvL|uE=j72>rt9FR?Sm=FK{LjFtFoB`pj_GG|zq8r~R?W2vD^n)<=)Fvt9AKxJ
zIwi!<lS@0y&9f_h@dP6->BSR1rcRH^vy7P3oH%p3?%EBJ-rD)wf=+pqs0BQqTlTE;
z`MgR$hlTSRK51qwXyMDsSlA&oD{4`X+^dYm6W;B7F}GW5LHg4Bo&rH0LOdr9ipb;`
zh6JZ<acry4{bL?qv-;HX04D)Ql|XhOK@|bVA4l1>>%?bXxaTHYyJh84Rm~sD&Bs68
zSg`4YR`r^R7AHl5y)B;}ne26)sr9YjowOZq{Zsz5i1<HNbJYuYYBg0f?9IB@dAna7
z^HzzlYnsXsnJzhVdbG!}tM6ylempI9faQPohyMq{za8~Dz_Wbihr@i<HaCw<3H%e5
zoWAYm%_!;ZO0G=nS<_c$mEBV-I;lL{=T6RI&xH*o>Z`>*=V~mrG4@h<UbEng)#E*z
zPaXOwQ<Cj`*>cPI%ZcurvkboX6kT>u*8OzlUZz!C#jK{5dsQ-jy0q&PzL(!$mpJ|J
zy!wxS%e5q9Kbs^p`bh77uu0$=ughJ5dw&<)V|X#Q;hu2V3Sqy6pT4YoD4FJwbo)(X
zgi@!RU7k|Fq;Nk=f9*-XExKlv*NODFmwpOS*PB!qqN%q??pN@F;yUGL{p(-6dE;yq
z{azySS=`fiyS|^Ro|082x$Hwy73Z>9;aWRWCxvf%sivKO?a-<J$FJIE?VTHRWlN(-
zw2#!w@7LS;|NW>obGQ3>&wJI%t179xGArl#IXDGL|6Ts7e)plM2~OWcQ(n7lpXA{x
zbxGjQkL96Xv;8-mGuD}Gv7wA7YZ^<Y`@vh00#0H+hs2htuH_5LR#WKd^lw<SLx5F7
z)6FtGJnY9hu|ua@4RW~l9Z(Y6d?SEqaY8rWTchc6pFXrV@4S&Dux-PUCrdk8KZ@`F
z*W|Kq18ZgT!WkmkVKc%KcsvUp^`u|>wDFS7bCW%SH;X!grI<tWEiVfy7tEM8`>{%I
z&#BJj6Hh#j8{P`+vnZZ_b=|b%YfInuzcZYzaP*q|rl-g2I#dq-ar$b;y!TBU=YI|-
zmXB^H_l16%RI_`DTK$&=EGhiX;#zZB-CRxu6?qgnIzD}>@O_u`(HAcBR||F}x$F|Y
zx`}7|oIt(!lT&3H^@?K_#7^iqvbg%A2Uq&6py-<8XYQMR_B@>v^x;T|jb7)=ij&U4
zpEi6^Ge5XeckK-)h9mChK2QBT=UGn3uQw(u%U0<JypEB#e)+`jQ-PQ5d@Y$^h9~mo
zch1}GxOBfpO=x2D%U7ZQy)&HHq9!(!F=`%o)Ud&2(k%9K>mtSXD7a0%dNQ?e(&AH2
z3?BAgLAM-FH*5S&*s^VFQ0`8XXk`_y9T!|L7q1M^{@At9FaF1&btg8oedapNvwsuo
zjv6-o2$P=w2c*gj_9}=b2Bc<%^kx{S2u<t?UUFlfg76WYs%6PhKe|}Xcx;)XnA;b)
zJMwHxTG&>rwdqRDQu#T@uB|+H(Y#D|)}?b?($k-s_CJn{_qZ_GU+>I%mK2F4Nt`=x
zY`Mu_`d=|od2X_f=O+!0xnE|deV!J1?%mnV)BfnF{&c(LHBD?u<K~kMSw%Z^W%QOU
z>shN+nPhouT~X1tv$_Vc$8VqAx~_ED;nY`c)mBoqeWmlF1NF5tb}K46FhyN>Xgt^R
z?#n21-@~7SK5p`s&@o(Zz}bDz{OIo1l0~2QJoC-}@#y8b%CxL`A4JzAJe)4q|7S^E
z$DIE9s1Ni1-dM%L>3V{J{lC&`CZiI&e?A`T<yJZL8w6ylY|L<*Y5kBZ_{QQhqANY*
zmn1Tj{BW0jx5C4C$H6JHEYvImU$%NgoDzP;u{?TZmw&le^1NLIE6nCRl(_ck@UAx-
z^c?n`4w=4leS_{YJ;yEHf)j5^DE?FJis=0mwP4Q1vqv(!pXQzuziV+!Gv|HpiNNH@
z?_JkS4h5bm_4!=7<Feb#kAm4x9Q&8wNH%jeo3^pVW%ggcqjGLSCnj=qJyDSrQB+`I
z;HdXDStNe;;Uy+U#^qO2)J#2JdPUE2Tp4SqktluJuiC0eF=$<{k@?~v?&&4lT}mYb
z%9X-SU%ImINvc7o=j+64-du}Uw3e<u{rZ3GcfE=<CE*!Odu>z`Cwt~LT{gdN)aJ3V
ze2w%H^{VA-<0`zY_kEp`Z>_m%YjqV%$Xmy&^?zn+)xSEmaAQ`+8kS3qrNTEByfS;>
zSa)<HZ_N>#sa^q1dR2+szjxnf*q`7&+cb$sYR_U0`IYT+OD~Dc-Vt>t>7}>wy@h%K
zjYo~%2JFh>dMtVS$79pF@PNju^)kwvWxYa`4o|qVeu{X`D&Nw|V9swLD~)tk2ktu)
zwd2)vv#XC+&%4<Y;Vvw*Sh07V`|sCh@9f&LOuF`E!CGy}rDml|!{25-wd`N6a;lY&
zWs+UlvCnf4FH7G#wK3Q3rraGT;fxI*GB)SG&$(l}#@yrd|G)X=?Yr+j-}n7O+qU<U
z#n#?emaW)-`>kWkG=|okprq?n+Acl97jp`|f1bR)?bC+SHQMQwcg|<N(_B~fRoB`7
zPU!S^uIvB&d9Ic5T1nRP`rf|1>r>-Zbp&Vcz9HkYaqsFSd41Q7Q#MTgxgu75Vn{U8
z3HyJW@2<}MxoPw5$p2;6g);2xe|=WkTgA`7>9aaqF{I($vHrTJa~%?H-ah?9?xg3p
zzwf`X|M~mwe|bB1-NB#wcmA>R8?stksqhwk_^&SJ6jM-m|5K*ir=(YhL=}(gZ7>n#
z-O&)%m~0`&82v!f?W?nWS)<*QqSB29`45xJ0#lT3=rb=)X*?L>H?27E8r%P{BgHur
zTxV}|U2DLV)Ec&-G0io-IdxlePHTAZa+8EDX`aW8!#BG}IY-2p*CuUmNp_d-*pkj@
zBwa6No_0{DVsm-f_l$nA@~9n&<%cs$jwiN@x6R+0;e4qzdPZx+kJf3|+VXz1O`npv
zz{HC!qP;Ama#nMdWl_wAvZ_^XRx1`~ZTpf{6`^dSsL~OU9dF2XQN(xi_l_;ots@R`
z9SZAAn`Cp6MJ-@b&ZDhj2i5(Ll-E4C9JgRb?z_Xe&7wLFxB36O;-6X|s3#C`%e<Sl
zFya34ybss3J{Jn+O%Qx`UGSA-;A?Z0H|+JtUex^*4?4c1K*7!NWJD0_^a54Y-ZK>i
z|GCTy&Mjh5WouyIC=_B37MWh?!YGnbP-wWd(CliV07ucAisY|Mo|a)^79}BHZg3SP
z_B$;r@@NzP6VdeVMacgjMP}*~0@{ivH%eH4muOokX80o1+Pvg-aF~~RY39)|kMuBc
z%ZZ{t(;_#UJ`gkedALRCq<rdhkH;1Y%tcZujXG_|%B#Z54Q^%_dA4O+Og8nLZ0Olm
zVP0+>IXNn0atnKuqh^J3r}^^I4E{wGO~zhpm-^cZR;o_%S}-kohWpg%<x^*EkKVtu
z>X2#2TC<pV$!W{fr>#0%c_=hy_4EAikCX#HG)??5Z9+!ufiSh0CYwF&s$Q;K`>uE9
z1R5>;G5!CU<2B1O{M|eJZ#dOHZnIBvc3@6q+7c9S+dN^NlOV@4t<(qIG7l5(ZmT=G
zspsSL#CejlPE^$Ai;D>c^q%af7ys6?Vr9?Sh#<+Oy>}!Uw9<5$cO;c@i0ChCcza`Z
z*vE!_mZCxu{Xz~+QZxD;w>AE~k$gHb<@XC`rH&B!6De92uKGJBOgPvSrq=9P<d*GX
zax5{esx&-rnbCud?hUP_&m1GZeJf3#tgS3L|NF-l4?~eUv#<~jN8Y52?y`*f#fh0K
zT06g$O<2}uSvi@tYjSPy0{fK<qF+QxyS0^0ZtXg@Kv01*Y5`}m0he@_YS4kU0*OUR
zQJ#(yqP$mT$_Y*VzjNoJpq0`4L$Vfq^V&2e>o9A_vUV#Au4ye9S)0>*wme&0Cd4|A
z(YHXdx-Vi$kkRzRZqp}wNLZP0?b$ANPsMg#WA1ww&2X=-x28s6UNbI<=Vd?cVppmA
z;h69-ZQ1*2%ebxvh&|N$_ECGmBf%eLjDpJ;{(PHR!Bz0(dC$6=v(~>{-mq(V#VSr~
z726Q4lFh=s*LSSAd}775h#-ln48a$IHlK97XHj@xL&Vrs&%Uw2<fzlZoqdb6oIiFX
zAGJ(<P||-gvtM`ms&&64|4-_dd_1Sbu-GtsHAnD-t1l%^mq^-%bK5_ku+BjJtYe9Y
zXQ}AUi7z>YG&Y*nDFrrMX}RRp{r_*^q^~#AgCZ34KYMT{MOyz{YstB;X!-(M&hq~+
zCppYq$F*x6N7aINR+Ar`S`c-BZ;iwHHxAt2ytq6KCO4~3Szy}cmKnv=Z9Z=cW3h(U
z`lnNFJem^nQ*rwBMaxa5hTe>xpRT;a#oNlN%F3(C%xl{6_GubHj@h0n9s-O(3b7uu
zHWfxzm+V}Uc_}vXQ0J*<l8(au(-+yjoV@wM_oYwEwbCCi`>@fT=V2{V$QHJvGg4=5
z5f7}JX&I;?m{>G<d64$35bLe=R$H59t#CTUcd2>(76#iT3am9z^|iBRu`OT0bba}s
z^2nfom0L80`L^rotSVf*EBJt<cTnN>4@Ps0LmK~Ef7yPVv&n8sV}tgNA2S*qrny{P
z>HFtXV@u+MkdnE3ZiODWH6i;U*IEO{qrc|ne@k;;Q5y4E`;+Ca<|!WCQ`R=H^&d+r
z(0MtZkt<WQtBuEMcZvJ%&E0L>TDxN-+8U<so^y3UeR<Sg>nLXFg$qp=uyO77NLv3j
zYW>;;oX4Uol&$todFH8Z)xKk=V)5?18y`ojB~7z%>eyLQ6<4|V+3dxQ8T)EyC>;$^
zIp?ZU7Sz~#Lglk}(>HEa_p1HgllE)vww`ul(@mEq%PlHK)v0oq*L<5|`flsypIcbl
zxB9PC%FAfZ+wo!s$HlxiWd~ojEla7&o4!*}RUvTJ%zX90t!g^;|5rzD)B3Y~&CIQO
zdk!^n9TsED4A8RWPF@kgwBpK+!+oa?_j0X#e8e{IcJIT8BR4)JIR>)L&eAh13tsr^
zNU8RmrL&R)Hza>uv1)zQj;Nla7c^I|-4&u^o?<Y4^=>P}y<BlA47LYvP1tsUuVk0I
zn@2>+vw2IRkAKdb<ea!cGgD#JX=Tp?lPbke*z9cmXm#T8uXPrj3mA4S@MvI2x_DyO
z?h|jiPP|>UZmP}6rkwJYto3UX*6(6Va+N&w&uhx!JL`|9Y?!qzYTeci+_No0Ju3yc
zH}Y3UZ)Np4^|W$s`Nq}4R!^*_?K`t^S^8qx*QZUgym_Vvs;aZjW~lzNx>5W0nZ3)F
z{GXaJ-Ozfeu{EpgOn;e%8Of{Ce?PQ$j!x*6ba)~&)Ah9Wj7X-!U5vAp7=^-m!m1B_
zvPo1DWenWFDz%F<c>!zGg%h9m^sI@rj{H4SX(?+Jr?{^b*FT^0208WhT1i~$7sRh^
z%eS6A-O5Qky-%C#sLWXr-X9nBg)S+XAN?^&%qFd|q5IPDmCow9A-a2=-B~7_;k;}l
zd+b-ovD7o6d!;0@1No=zHe@~|pJ=|eJZ<e`39b2$)PJtbKk4N8ylUO&@2#9xSJ!(l
zVA++i<@c4eAG>z%ZtIdc>9K%SW)-Jo)HNw7*6r4(%zu_o71{eo>r_*?XTaT7sV$s8
z*03%XWEGsi^1s%7Z=G&tN4bw^(IQ>fcCB4|=ZBqEWy&_n%Bqi899x-nux8(sx%<{F
z^H${c?BrMyRB++T@0%`F`_J;+(um&wH(T}69@QxlOLGkmoJczxWF?)NkT<z<i&)G#
zMz;JZH(9x#$?s!2<a_&E>8kuPsq-IgF8r_ATB~(Yp^YVa1E=IG)z!Y&0u8t_?p{dd
zUcqzr(2cWq7kh5I7u#$1XT=lSd)d*}7tD@mu=dTIwenen-n?0T((J1O_cZWaoBe|+
zL~XfP_?)9LdyfA5yUHP?xcjuTuEd11KX>ewa!o(;VC{p;d^06w6j)v6rl&WTnLbp%
zb*ik0`9zz|3FT#xit{3s|2_O)9llO=-PI3L>!#d!WW>p`W7@*Ar}wO0z&g=@El^<3
zv^S4t+<82`=Fu~)tB(J!E#<kstmcVx<c99$r#Mfal2G5cu_k(x&*_kTmHUobZ85Rh
z>r*9XeP)_&OuSvk>cjiQS~Av7XPJBU%#ywPT~F=5pwrn>SgrqilZN+!Yj=EO3>e)L
z1NvXi5V<&GPm`d))l!iQf+7bTUa2RZ-238~@75EsiL(;!p54v0Wdq+DhnI^FoR1bb
zT>0xz@4uG|U%s4^cPL2Vh^Vfxa>@;VIg$2W%Uzzq0fl<9XB)y7Zdd5t&UZquCR<ER
zxB0=8OZtCbYwNzz^4%e>u|wiSzxOp~%iLqr7~}r`esp>6MZ>iYY$9=M{W(N5I?wJa
z)OWn_@b}AIT`{}2-o9GOt{5lV7Eyb(_-;mb&bw{eyPx`9{Wx!TyzkX-cJDn5*iz2!
z`FD;h#kV<!W6%3r3+&%z9Ik%C(EZNo-y`cxPlc?C|G%zt+kX^X)GO+wc;!;mou(U0
zTiPcyD{r`bWAl?6bs9HcMBhwU_o*pDdDG&X2KGMlrZ_J<`)pzCttGa%%>O?#_Q)yy
z`T5>fpFpk6+Zg<|Ij~jbY2|YrDm=yb{;|VRrt|;*ya+!3qHo?^F_&u-1J-ZZz{P#}
zD|ho_uD^+k&)r#k@6g?~cf;S;3GlvpyjJ05?W?!FD@)D`Ka;rc)6@6AGXDPF%-7RT
z_U*P4+go=@_wW5S`3GM(CbU~W;IKQ!>pXYD>(%`q-&7j@6!>@fTHc0&$3`;u*2?Lv
zTl(|ig8C2bYt}uD`epZTlBVB@(AX2p=l?Q#dFQX)<1PNzoQ^+U!vA|${p7{-Cws&_
z5vYpd=fB=r=B#Uc!`si|)c#k3r@hYp{}Yipb+yaJojfs7_nz+G^XZVyP0jZkcgAq7
znVzwI8*50^XG?$8yuN?SV`II#Is>>i?>@{`&bLkIfN~K3s~`DafA0UE6`#l;m>M$0
z@O8%8MX3`+*9JzJWimD}I5c&Nss<g2FnsLRE304jL_+9^$3#`nc_yA8lf8N!gj6O@
zaQWXgMb{)V@1SK`x1Uk;4xgWrA}<0Ky4Et;o^oAzeu?)~yH=^tRhNwXSC>5u3e65%
z7rnQP^>=8_jSX|Q%sSh(ttV<zQv5z&rtrK;hHs<p?mf-5{$5z_mw$JqXXI;#)v*e@
z@%&hyc(ha4xNXf2UQM$lCnAp3L>fOkmGdP?@64&<)n!}X+_Sx_#mO*ztuTY*1t|v?
zh0V$FExy*@vZroUf7X{Pz2p6ZEv@p`*Ghl;^x(`~@%*~qKMP(8tqx9K_s8n>?PHsr
z%jY><mhg;<*qGElZ>pN+>|<-zy`61Z^kMUJ>G{4f_J9BW+<994oE>BQ<MVb5ye<Ov
zjN&N;N-EDLIdpOS-+m#b+gInv2Pa<RCj#noCU9wK&R?dq(M8z5r)%=ckPt6dzg-1|
zx)RqHUDT1ze6yrSru@pr<3Xo>7<wqQi)`{xs$FB~>8Iwzuvksqi(xT~(we7wK?1u!
zZ4O&^rpQ}oKTD6d!QLxDK89OeE_)kaRhfJ?&|~47vlfvvCvVcbo^y0s*s3+3)<t<t
zWKwmER5DpK(a!0Ll541yNCO9tZpl(dbzKu>Psyhm7dcciJ-U6|B=x!4k2xp>`Y5`7
zy%ZuI`!ysqeyK>PcVuYkwh7CooLn|*+XbfO^D2t+yW?B$c1FxC`80LqqK>3(8cQ3h
zzG-GOcWt{hZ;H}kt=yHjrfKK>-?(d)ZsBny^EG-~tghcFx;}UH-5rxIG)&dvea)?3
z_0o0Cz1{DgcHgi4{WN>ag@1d!6Po`E&DePA#ho>Y*0O3nJ?HhZm&(}%Z`KZ&Ubp$G
zIg=&Z&*X`=@7CM>@o}r%@n66ri92Y^k9AM}+3EE?yOZ-S!r~5>P2{u3GrA8QGQ6W4
z)qS<<ndQ?ZwQjpI=Qyj)v}1g_S-+8ec8&QV&g!F{hZnYft8(6GsWK~vr*_wbmAVri
zCAef+PUt!+5?<q#w{zjH-FbVX??}6?idbT}TK?>*i)#+Ho6I?Oxc<z=6UX{%{(L;b
zYXACVuTH*DC&!WluYJs}$80{MZ_EF;`0{%j+fu#%FH;_$4U9^kJLSep6P*&<pLdG=
z!xUDTT)S6quX*vtq6v@oM43H2TmG~$z3xJ9e794G=9UhnrrEYpcE6u(pZ;~bM)D-5
zU$OZ@w{FaBvfUOj?+Sm}jfL0kx78-C_gxkF+xf|%+|;S8g;6V37zl<<Q94j4S`eEm
z$Q^xd(}6829~$Bcr|(vA=(2mtsxQ7LA=&7~jAJ}&nq^YXBwI-wIy&ch;!MYdhc(os
z^g`QAmMQx3ykE5O`hzrc^Ju=sQ<f?GjZk;sj0&jPB`o_u!BOE=BD>^-4My*;sHa`M
z$XC|pF<0Z)gie<&k4k2Jo<CK0xkFseqq+kgN9KFBIBLv}YW{Rs`v3G4Uni57S8qJY
z*k;-FHoI)fW7(G#X8dpU>+4+t6<8w|?fIyi7gKSudk4c}$-6JyY_wwf6)IJfLnlp|
zt}<tmL()zSQ_k?@8^W(Hshl#Kcxg(qlkCfYon99Xa+Z}wNxXdVWU0;5bz(IeS+%z<
zuytJ7RXuSQN9+^RGf_QFs{5X(en^<Q(#`Pl%p29qRxYsA%=Fr<vSZfCA7!a_MmdYr
z4>K+Q)pd1p%EA8GkH4syU;gQvxVE_N$gh*PrlpGf@jX57(xLRNI~WeQewy46`Y_C4
zC(}AtsS6AEL$2?*)EdBH_aauXr&}yVV#Yomwgi^d8i`8I8yZuuG;=veaB@!GFrn}N
zmliFT#8i!IU8hT)c`dyud@O1q>&qS<OEcEUvo>X0mcPBQ{K!QX=^GcjzRYQt&G9(o
zU;NBOrtD0c&)vw&Q?x>t`CeV+v-^Clzz2nPHwLx*C&@Psn1$I)3hhtQjgET1BW%mQ
z^Rr!EKfACibK%asChpAB@_theF>c$LJ3)QZ?Rx^6Hx5V@ZRlo+Nm?m%(nF8W-NS3n
z1f_CMMOU#xTcJ5_M$;SAcB*(4dN!Q?@?=KHw4$ktQ<lEEp<;DmWnNiX(p`<XZdLI#
zPyg+km+DR4c{^T?i!*t_0|U;_>YN|0?bz%3uCUo%dqTyv)ekQ>t=lop?$x5tv)?+)
zTQ#z-d0n)9?*Hqo@5_(gKKo%(`JUG$&u!jc*GaWJs;RW-n%)0jYa8XR?BkK09`)t5
z1GCeX*dT@GqXskddHB)}i2YGWFw+QPEuPTK#Qe}R*y5pR*fIUdioLoiS<`)E(!2b8
z&$VB_VJz=-F(oQ=rdaBuv(tsPib{RnAU@yXvF5!EM@;+*dy|)KHe7Wk;Dn>0xMz*g
z?->)#!m2yNx3V3#y*9BgT{XMy&Z%pf!fQ{3Z!4O%QRcc%@ytDCTeIc=-njR3+RZb?
z#!K$!TsfiQJ89!2mr1HmeQqjS7U}xznmR>8iBb2<O2g$YxeI(x-?}=>@51sJS)J7A
z<&kx1)3$9@jVrQbd#d(5<hJ+!CNUMp<@T(nE=}#bYP3AkH>|VeieHuIk2Im!xTh|=
zpW93mJLBmWvEO*kX1V31=cD7ke7bdgx17Gsa)t9>FMXc(%ciX6uX;d3)#;mO`NZp%
z*52PRCGl3&MDA+}TC7})*y6>vCb%7ayS_hei$S>5_MPUh0$We^9TE=H+@)Qk%6Qmd
z*;<FV;9{%IE8beKvWk_eUn7z9cgZg&&#UGK7H+#D_3znA-u4~)r1Lh-3%mNrd-sXm
zN&>AMYpwdya<fl3+7)e0lgugNl|J>7D}37NQW3rOo#yd+pJz-?d={7YGg`MKd~cmb
zeQtG^zXjuVW8q18?It?if7)*B|H~Vb?-z6Be|5@DOZ#(^CYS0wG?;E<8~;`{`b6+_
zyN(x*tG8`UH{X)!=x$wp+-O?Xp$&7lJrTe2Xq{I|h1?4JTA#h&Yu_(^w{)T2(|`{z
zU)9{sU2b`wJ?FF8zUw{pdmh*8R-W*;`!s#KjrG;tKY#BPpD_RX>);=rN=(!CEx5Ss
z+b)qC4lMJs_pjlf6CT$8w*8u26T`aWjm9_Hc-v+l;4I^5s(+vG!26l;N4J%Sb#hd@
z7DgX7^x@UKF~e0hJig1$%&3AbulAMy@vQBdSNk~gl60SK_+!3qq1fsjbw<C`?ZpdU
zc&i-v`10bBli%}=rhQ$X`fu@_|0eaT{}*fPNgtTqygKlQ(RA<s4l|YYS!!16Hy>c1
za$xtb#fGa7EIRDqsI}x^)aES-#@5YN7N?h3Dmz*|p6C-bNyE|9rdjCRBOSZX!VcQj
zk~+FlHftqqWVg*$wY)jW-MU{cMmFq%f!)OEQA(_*KkmM}X_?BO-D){gr%l)&b90&D
z1k)3<P0J5${hGM^O!a~HuMTmrtk<7%z+j4vVT*%4i?gxG;n-sJv)bE!XL6;UR<{h{
zs{gb+uhA@5QKM<swkxl@TA5rnMyzQ5q<{OgR8YmfZZF<iPxGEn)7@HJii~IISV~NG
z+E<>fc}7FRt7heGOJ0ddcAd&IR~*uuvujnuVy)L4``>qW1Swj~%{(epJfXe#ziUVF
z(RT^1vv1llG}^0pH}ytuQWxGiiP>_y(oV)Rma|vyoSl3uHN=4-#4RJoEwjWeYs;~W
zEj!t+xUsdk89cUJc7wtG5z~UhvmI|8*srzbaFF{_Z=C{0nPami9;?z>sqJ{|;F9Vb
z9WkCxwT*Mkgw~p9?la>&VIVnoOSbMs#Vsz2CtvScx+8e<Eva=;n|AN+;8xq>p?<{U
zejz7Ih2G(4n^Vl5Up}pyaAwabW6w!8o@cyIPI5UpE#zc&<>9lN9loDB&?L2B`2jAw
zFNf_?6mGsgc|CcB(;nLms}=5A8+2QFbxGMhl=gU3ZQ$dhY5j2T$qzg{rzc-Ja7tp5
z-TTa5>DB)?hRf{#czff<C;JS#Hoo@Qzd5Dv!)>jtcly3(yT-G)ZP;YQ;^J_6!|9wk
zmRj07|2iMjHa?Jb#_fB?u{|jcyJed9xSTn9<@B+V0|#5q9G`L~x5wwy9G|?FGlnvU
zkI!*1-0fhr`s4+ZT_)Ztk{dT)p0n%9lZjPJHm{m}T&%@cwCAjbV-tsB^ButxhKeVg
zcROy{?0Cg%i*BQEScHiJ%OU$plS3<Z-;rF~b;jeJqgv`FkN76enF90Q7j17nbM8%v
ztl5Dx3_g24^k{!Pvd+ZB-_+*(mn%MBQ_g=aIsdKY{8yL5Cxp$;%~nsDeOOSjIkQkY
z<EQ$ZBhI#K^yl3<nRS^f`}BV<mc3@8TlSuB(%`(nVia+t*3#hK;uZPDE^Nvxd1N&w
zep=y~GNZhEd&O<>Q!l!w8}U{%yK+{{3yj%Oo4ljt@IIv=*Y&F`=52KS&}=Upy(pyT
z=#Ss4a+<V#MR;2t9ueTZ=$#{x{pEC?$eGL|ZYOJwX>2`nq$fzb*7E2T9<gknGbzV%
zw(#iBy`;a_&1h+mdaX9s5$*Z6nTr0{@Y?JWQQTGJV|7SM;iwn)^3z@Bw=Y-jS+geF
z_k^U-n#)~BJ15-?IjHebN8G33`6P{HC8zG{SbMH@-0ZDuoaj+LyXShBb;p}+Pu~bX
z-*e(=rle-Yxu->wo(suG-e$2dJmKWL+ok)z>f0RIAcynQWe)#TIUKk9YV6n0Sk^ED
zo@+6pVbfDQW9IJpwekGFpUVGutS+88nXVnc)Vp^bgV*MQz4Nw&u?u-Q*KqgUZt1*a
z%j|uM@$v!>)`iPLc&2;pTd^hJbVSeUmVMk>>z=G!`TDb0O{)F-<USwctIuy-uTkB9
z_l4H4%S~K~8!zrq*qsouX9IVU?}pDGxX*XUP7ey&Q*tK5=gfCaAFZpG`p-s85WP8}
z)JI44=0w+<J+@~~`P`g3*XQt=GbVq|%w+YwIA=Yt&vA)BmCGSQ_O&V&n|C>+p0)Ho
z=sfl8V$&%0wddkK3g0V<lBv0MX`^H0RN=QKe&xGw>8_k~TkHR!=&Fsut$qv%f!@kC
znkiS^KHu^<t*WGQXxY`X_s$%AFlBPo-ZMY8-2Q&#_U@~<_dLD5_v`J1wdZHFtWUXo
z{=(*yD^574S=%zY+GfqYwk{xm^{;b@;`F@Q3omc7aPyk;n1=K9x)dmHFXg_wKjZpY
zg^R+e%{^J{(HkUY&e9Y=>l%0`u)TPPnUbr_uGr8kcPFeo8pIg1_T|*wIhuP5VnzQR
zdHhxQ`rH+quBXCgxJ<U<FFVWg&^Pi(%uR#Po0C${=v=)w`R%1QSA+Dgx;@-`@1yPI
z&!)k~s;%mCgG3oO&*MJME#v-I@@%Pyj?l?S>$R6{p5EX3LwYU8!NU`eAK|>UaPR-)
zM;|V!Ejf5DDPVn<-m<kz@35TM{#B+aMD?D{i5+J{WT#jMWi$r{OkO+HBiblb?aXeq
zDV&?%s)e7O)LHV-qxzvlTjE{M=;b|E&qsRPtdhOH@ywSip+9x5?GsJ<yk<{a>R|=D
zYbti0slA?yI-C#NZni18qv^Ilt8Z_1t-|TA*WzoGIR&OwXU8p;2{(-MDv>?Wts5(R
zSEzI62HVV7i9@GicZF{;v6r^CZ}8pUDQx5MaFxnp`(2lgwAL!N`Ce?xy-|Jol#5=9
zMO=z=S%gvBl?xql))V7ISH-=TjqKU`#P`?}zh_UTeNFki_V)C%_s*Ts=59%vZF>Co
z)!8mZ?k<o1-@NoEQY6QCf#|JhGa)9oE3t03`j*U_xufUyPu~@)&l11+%HKJ++Skum
zF!9bVSzU%`uY+n9n|8W&d)z5}CV4JR@YyrLdujP;e$U?Ax>FSWkjYOb?8Lj4o4aZq
z%GjLGEjgo@m%e{$deye)Re8>8(>yJE4yQhD$`M?lx5N49QvEBJJ+<6ktl>Dqa`ut#
zwMVL_)woXWHT-5<bob7)r*|%j_T;aQSkbe6rAc7x+SY5nahDXP%9eKCTYB_vs%uP)
ztNAPeHk}|zqkEI~ZJ)jF<r<eKBE2i!<TB?UJ2mTA=2Ov>V&=ee9*VDWeR@)}3`(<>
z=4CBCcY4J&9^Iu`Z<fAVS@-|ds%Kemu4b)xmuM*Z`h?77wkg`*{~p`(YIVBnaf_wF
zzwh2Z9A)+YZM@CW*Eg82><OD=_4{)4vg3Y5jy8|oLq4mh%zidg;L`3Bs>W{N8=J2>
zIiCoRwZ5?^N1?<+;Mt_SeQRwT=Xsr;qhWA+VdBGwQt6UrZzZq2DY*CM>OQY)*9@+&
zTY9$d;X2i28-wSz-b@VQxId%Tzhd2U`F-gsW=RkA4nK0cHZ{h1M$EO!uEQ%c%tSn2
zyto&pJ@1|NKhM0<7jN3S&(7>reZ2X$j)8|>IFod8X|0z@o4IM*b+dJ>)fp+50v4^E
zwNLW1>(6LYhq#sQ*D_x`&U90I@}KP$1OF>V@&6x0Y9BM}fB0XQ^3m_Q=ej2^-=_H9
zd$peLssFkU+}l6!yib`f`ZU)kJEZRNB5U{UX;1%#8oMP~T?zSkWshKWjIX2VgTosI
zYkd;p*xqO=%FQ}a7{;7<$42#T<Lwo)>sE$7dvGMG+}ZEUwOcpBa+94pU!76=y?N(@
z!_Nw)J<LB=q|KgwGp*>xw@-TP#d`nKb}xQg^zBv)i_@c?wWmbxpXJFmOZUJ3?5*YW
zV$0*tAME?Qr&YbS?Qqp8{~XQ>vsGR&vUWGu#k^D#Kbf1MyD#Q_-QsUMe5-@Kt{8ar
zu*LG%x_C}8c>1_Uc(3o_4cy(8$!)64y}y$W$;8caySH##=6|{WPoC&LaqKH)a{q93
z?brEwy;140mv59tKmU5|?bo>a{JGOgW-7%A8fkj_yjpkf)#|)&>FlpF(#tZ>zv68F
zmU;iu(z<VE>e+eIgYx1p&tu+YwlrSk<oz9bYnC==?@T+sY<0Hn)rtIj)-b-yuDbJD
zXvuLFHeK7x$6c15eed`tewu(LPxwBI;$wADUahxJmRalF4B2^^Ti{<$+P5?-%gNER
zJOW(b_MZQw|NVzv{M~C}Kd;OEoS^=5viZ*``*o+Xf4+IF;(_(<8<%ot&#pYY=BNJV
zP@DeZx#x=?sa<<)7B)HNU6Qi#;lJnCJ`i)U|FU#{-VJGSx9jiL7k&B3eeC+e61MgK
zUwGQ{lrHsJ5n}hud4_FaK9hQNxccvK{oh;s3u5A*1e$-?{=GEtf9B5RzbD@NT9t5j
zkNDT!&wuX<uU@q7%jaV*S^K|boloI$ui*{<b0q!dvGnKs%Oihojhv%fXwv+8sdb@i
z(h^bkvjR0APgWIrPug|pWqF-@)MfKqF^)N}r!c+NvElwGyywSc1^u*J$FFFwyrt~^
zOfKY;e{N-g-`fZ5{~o62cK!eHnE&$)v&yICxzp@_J`pdzqE}oTX8ky3%e9S`?^`|x
z))l{F|6alN|NWKQR?jQvrrYpuJb&=-=X$w`N2XqDO8fPr|6Oa)mu(Hd^5#}?KHTxD
zvD2QdBQV8dYu5j(%M=cUy^Y%FbYny8>^3%4Ih_-24_w+=mGg2=yn1xBQOUDa<!ILx
z2Q#+7O^<G@3$u;>nzePZ=j1fUIr`$;E>&)Rer~pYI^WXF%^ALn+^1{hW?jj=wA>)~
z*qq4HRiVp*7RU9z3f1siU|Ff<b7F(CvzeUZq8x8G1+N{-_q(QKZ`ZrGqw?h>Ddi9E
z@9nN)|E$aNLoeY_k@-c@m>C-#PNb*=FFP%f`ZO?ED|w#H%+t$sqaE@ka%XXi^DPeS
zf7bHLH0%0?<o(mm9yNP+YggvycS^U-Um5LB<>vRhW2*izb;p#KOYT&D3OYa4yWj8c
zuF{vcPKP}2n`_;`!f`~M!O5=Xe-q1#tIPHM>;C-s;JM^Nzfx+Q2<PX?FZSQJ?-C2@
z*?82I=jtIZUE9i%1&wlZuL?M_SZX|s6mh!Y)uJ9-BGe``%O$8~ZBLV88|ST{VC`*7
zb}Z`B-Sp#8r{1=XMLi~)Je2#4U(HDBvw0`6xZjb<GNs?;Uqs48Tg{mYley$hENB*Q
zo1y5rc*3XaQ}a_fZ>U9AT}w@g?>ZTgni6HEHY;sa=QFXKT|3ie7wpqan^STwGi`2V
z)w1WA!37W17u0h}t(d=9^wiO;&3oOBEn2U8{Keukp1Wk0PL7JwTsA#xmge%=d9&^;
zU67=;YQ?;16CH$BteT~@diAbbT5DE&DI8t1_0s=Y@j`+hrfaX+nziw0YL@7W1Sf9K
zlxrK0t4+~~R-S4SB(0NZ5#Y4-MzNAl0_)bb+wQWthHQ&z^%mV3I#v7f&hVML^L9q;
zJgv9q#i7@F+rPGjPK%UsJ$Y?&-cHr)`x`meNW`_9mTfr5Dc)mnn9I6mLma>N9fPAW
z^H!=YlAOQoV|J(3FT)et->Pj|wDaOI&H{Du<E#Z5{N=NYPVSU3K6}B`JhsRz|N7kG
zb8p|omS6gLC#K@#)^L;Rmut6wx#)X7{C@4fx2xo@?(u4HsoM2QYO6?eNvwLxk{XXk
zH$<W$6mP0;Oj_!hlkbt*uI!bOq-%7jV*B07$GnSs?eEPfzwh+#|Jm)jrT=qx-0$RG
zx4qL_uX8~Y?;_pQ_T21>h-ot`#nhh9u3l&PLcCrt?Rje7KDAewtL9m~UcI~T`Mi`Z
zH=iwR;hR<XcE|o_&vQ<*KGzVRz;wTL<<au0Js(dh@89!j^1fd+Yc4wXuUZ@8y>K;4
zV8+>1>n?leo7Qi+r2g+m5R1osu~`*!|B3C-5>oi{tXqEn_04&XURzSOxLgQ`s(fnC
zuu$m0+AZDMQ7^q1Z!gtjH8RlVo5IC(kagYkj-^dMw7fT6+9W1$RyBd;_ku(=u|plb
z8(R7-7jgu8NSsu7X2@~-%uKKEViIjDjT3LNWH=~Gxx6jlUgda}KX|f6|HL&NYR3N`
zo$#5qY3jtB&f|uBA~Qd`JS%CcFyG84ws-Q;1+nQF_K%ZFdeq#$^$2EuP*koy)I9Is
z8?UUO$qr0kgd15hdiz!`QoeSqdFzK6?)^^=O39tPe4ORU#0m8xPbzO1pKwx3RLH%-
za?__}iE?0&$K)+3s=X~IeX}-Ap8h38jX8b0+C-zGKqYA{`y*|EYjUSXSE)`@n!Hp0
zY|u3GlDoRm%THhbSTud*W#Qxo&D}RxC(k&nqh{28bNTI??*dNSoXwGFmCE$;syMSd
z?em6}XLGzJ1~0Xm_x;dM+xJ(y?03AnXK`8cTxp+h_B#`mYf-IReV_C@*(OfkA)~Rr
ze5rQq&8P>={~xOcR0@_XVAIub;$O9{pz^eOisI=_aw<A}zg9Ohd7p_1*<m2G?L(r>
zS_uzBtBn)ewk%;YmgRFkY?#ruW~tuXvx3R9j?CQjCROiqZ$W|9=2;hY(rsC#0=8e?
zyg+Db_tJz}7pF|~SZXBN>*0Ls%C}%s%Ugwgf(%bWH@97_NHevKdAh1HXzPh@JVg`y
zeN%$oAMet2Iqgu*t@_}Rmxa%@x0C0~$Y|7tTt9R)V}0FQmALQMEQB53c`mC7(6T&o
zN-H*VDi8Nu9%XB*lZ!%1rk-8;%HZ|sxT0;P3Eg*%W>;D}XXcd4&VH6vqrUP!=lNNa
zHYu5J*Svk>lGvPAdvvYim;RRzS!ld{$KSQ99%Y^g;oQDs-6W<ha)<nvyG)4VjnOM%
z+~wckw!QIk?YffZSMRJdD2Yz~ndhi*YGO}K%ObI>8dbq7;slmW-z!qA&uM9V^JiLH
zTkpb!wUUpeF24!rSpHQ%+bK=@#)&EB9wr}7FWE7Bs^vwCusM>aR&*HaR<&EYeLZ(!
zlk*jx;2k2VIkTjMB_Dm6^Knfb&&P@-r{=HP6cHL{yF@-Y$3s$l)3L5ShVqvyl9zfa
z8p%ykiLuPQuJC;83Fl?2CwA6^i<)+x?5h@P^7FI|-Bcv9u4n1A4L*`V3-YF=JTjVP
zu=?h?qHmi|w#=P#DM#IO@y{7~=Y)+P&#}Jre_cr6m6VWd)2Fv?e%iV1{+qNFdo*7-
z{N8rqz|*M<h41=R82y^tCzr=G!Mp4C+^6$6kDe+O?aFLb+j4o%zb(t1=f>9iZCEIF
zCfw!p?KNGSsuGoA^Eu*mUq@V?wM6*$inYG0vL%<9w3n(q@^1FM%q%|ffYNE+qLRru
zQ{Qg6X}|TRpZ=eti-bZCU$tBvH2Kw)Wu~IHGi3N>FHXK1n;B{&XtB8WuSS?l(%Lm`
zCFLE(GVgPl9hf#R7JG6*G~y^P=X%RqtK)iEKI{|Pqt?OwV?Xz~@Mi5l>v__D?7P28
zMSb$74cxN=Zmrm|ZP%4g2@{s)^Em1~Gibi!lN@*X<jFlZr%D(9FWV)oJD0Up>Rzf&
z#p;F2vWn$wY_qwx)&8r@?Y+)=Q)}0YcC%k!TC~rXvroVHx-NZHkj|{%TesGqd7JLA
zabM=z?ClGp&+oExmo4+Jm)-SWyEIU%Lcb$ymhw~EPFL+)al(hH-j(gwxNE=ayU-?s
zClik9r+l)WDaW?)gELQAW9Qt2NfQsI_{@29N`j{{C^9fzm`!=>Qs1|_S+{S0{`Wz0
z^ItjVu6xIH)+xVCU-QeUuhiy4S^u6y{oj?VFV>xCozR)2l%o2%?)yIfPw)GAY7Xqm
z*x=u1G)2H#mA6w@Z^Id*F9*L^>d#;IOK1PSQ}0vGm~D@oaWCI8BVO;h#eb=kiT@A%
zJlDAH--7JV=T9+y`8~b;%i_!R|6bRa{F8oL60-LEy!h`{D_9q-;9bBV_F!qP)LxS*
z<*nkYL|fclSUw+}Ta`G4J$1_Jz2Y8%XV$)yTK9Gqd(;`v$ZP3S<h?m>d#m1SsO90}
zlP>zv=J$8XW=rYwF~_%DO?Z~bez`<?d+uz$R}b31CR}1!_<Z5jZM70_U)~U0G~?~V
z=yxx9HF)nFe%WzYS4Pw?(Z}VXNLu3I+{8nrilRl664rKnloXqya(~K%6G^c%lUo&1
zf6kTOrntUS=45Wzsl@rR(`Fh?Y|MDO@ZE+hXBp4GXL*zzDtvzJe8pqeF3j{&ns!iW
zmRI$Iu&OKfE&jVM`l~BjXxm>HyGYF`O6?TrMpZWlf$NM5N;~f|3SHSMk-#`D_WIgx
z4X%KdD`oxH#NN&g*dL)#^K`D>y;!|v(+}+scZ+S~KDh8frrU!;HlB%2e7R1U3a8ja
zyacYwuyfqzQj*|a<aR10xydt5#P!}8i}set_SV?r7E#gScOR7H1P0G!?`)hQAoMU&
zcHwKmhYE88ZIv$BJd3c~)!s8xwD0AN$}_VgT^+7-oM>Ir@s4-CxKrYL+l5YMOFp<R
z5$<|8*|KwrYS`2*lcpv}KAXb#)l+_UWc;_*Co^;BdrnjF+7<b|G2W;)KBIU3+X>Sa
zF-Xr7O!Ade^4lc;$G7YE|F27OLzM!&k}Nc11F9CWI0!lYnXe_J5UjT}*i7~2)<sNh
zNB2HCy>H5NmaK%tkOy~j6V}x}Wp7jUtd)8FBP#Rf!}^&EHby3&jCp)g;o>IQ=+jR^
zcn&>ydQrjZ+0*k0fqeTGzfint-E{B8SJgzlP(it6or-EUdTLApDGfc29SSMEA&TNb
zYGPr}M1vC3{)KjBJWQ>7CN30{-nA_C(`=2sCnTa`{1RlI>RdY(dDCs(LYYZVrKipD
zxR&_s;WOE)YjZwy{`i=9=Az8myD?dekFo?G6&(nhr@Gwl*^`eZk3OonOBS3e$>XY&
zkyYh-dQtGH+BNxdty^n67*#x50&g@2{a>WywDb|fs^^Xy)pzZ99(+t)Gj1-+-6K9H
zl$<umy52}yow@Yjm!!LnXY~9Y7*xsJYfNaor1qTQ9#1PD-$!wiS#Bmwv$=LHeH59|
zWE%0oC(gX-vFKeN%d#7mw-#7V%V?K8`@DBvf?Ar5l45e*?8Hqk?Asz7<d*F|`&eRO
z^pZ+mw}h5`wkne<AHFkOlJ2$efb7Z%S~7<mRUQYZNX9++l&X?rci*il=2^_k6Bi$P
z?CP62F>-ocy!WOjvpY5COcX0PcJs{CuoEe#ip-YJex$OJ=hljVuHT04fmzSziY+ZT
z*`9atRE3m>&jIz2Q|b%(RxM<Eu)AdX%23r!1}rSI!s}OgDX_AwsE<>Kh+7qrmUZ8j
z=atUmNF^_mRjJ;g$}!WlHp$-JDB6Bz&w|a4w_A9x25|PqozoI8WxA}uE5MYN`0N$m
zyH)X@9<2ZMYVEES+f*I@Pvmg=bx0+0;i25LRI!(Fmf9IkuQ!Li-Xe7WuEhMwfp?Dj
z@@Xwycw1pb)48<VKJ8PvFY^{@%ZXjyyEny$tIOxqoHKW47H-lk+NZfsN~y%?_&y`0
zr6KCeSy&hpf3mQOFfcLbFfcGMG%z_ZGI0E7VC4|Gk+s0FIgd@urouqcxpAEwmxi-j
zLT7U-OVPv=feI@d7h4tm(O94~Gi=)}$u(E=dep*{lI>)U8!R~zD&yKH;t{NNe4ah;
ztDcpw5-%>ao2<2!GvtxQY)g5iP_NPl0fiQ8)#hrIE^i23yv0HE!m7Et)j4hx6<ytQ
zZ*R|jd}yxrcAmTa<&TRlX*cLD-zWN;@6V6vYu$}n*{2D;DctaQlFIdMm8ZKzq=TmF
zZ%!)RDViQM@8*Q3Hc?v~F3)G(kQLlj`k+yg_0QbDJ665Dy@U7Iw4E1~PflcVpC~o^
z%U21;*<x~QmcF#`nfZuCVEPHu?O$Kt*qnVoFZ#-tSGNxS{x8y6Jat*%nk>$M`Ip32
zZ(DO^(e|F3n`(?^3%tI+J6z%IzfG)c3H86Gewkj+EUOXS#G~}$K@-2$3<=Gp6O1CY
z7o>JPY?CTGk?44;^oXKlG)I7vvyj}Di}8C+Peg<Yx{55yHTJsV<&>wxHk(15bIa_~
z{VPkQqgk&iH+ZeRa;VSyj^q^)=clgMpG=Wmx#)6$E9aD{K92%dPAR|SX?ZF@vGnCq
zm)9~=mN{m3i3s_ctUdIk>g|S*ZriO-mM*WWd%1PKXZ6?22lM}h9$~FtZ~J9_QV-XX
z#SHyMU!>!j_L|;IopG)GW%|_AW15*c2V8`*C#$|naNDRW_HyR+x8EL5*_--5<(1=p
z$t|3r+xKmA?O6LIs#_!JqS3ixrL|300@f&)#)d}coqV-9-ceh3vmx)(Hx*GoSwt(I
zOf>J3SiE7nir2iG+pnb-J`E45t6nMnz=3I>n*N?ITqdg>xsH7a7gt)lBOysR?8m}(
zmU@qAOKNj>T<=`J_t%9y(W|Edq#J$J0;G)+?@T-<p}JN2*t!CpOM63gy{~$@%UzkI
z7A<70>(BhPCD`M7)y-uAGAln%IL92jJ9Qe<3Z=f8`pjRagy%idnaki3?p~7j!1m`=
zt!KRl`MmetN<KZWN$khF>&(Zj1G-ZVpZ3h(Vyh9L&a~m~#M`;MGcGLYi}bpmHmg=l
zDx>QA|GPJ{#JplI^{t$?@WCp5Z|}mOm=u+wy+SkRgmj)XoN{mb;tN4K&PLw43<3U|
zwVpaIR+kGtyKMQnyBmGB>jhk1=Pf>M{o`9n-nxf2PQ5SoW8%Tq_iBIbDh?~#yKM5;
zbbFDskLkzjKl^J#D-sW0$*(9lB$*^r<GP}3%AY<vu4xyO)+_Nnb`UfSdF=es_=&-f
zTQg0TBz#@r9$@`!;km3?^ZGY;JzsVD?AZk(2j+ING_2&F>o1@&p>)lm#sgmd)~AA|
zO*!$@zuPPD)`IOCTQiqbD&z>ZJo=^)YRVN9eCYJf7tM-iHwY9|Y!-N+7QAZz6phNZ
z(~9am?M|OX3%uSx_7M30^4i2`&m$eomKPteY|?K2l4ZMCqWNTu*xCgw?%wuGPBBdV
zwE4{o{qAjwe6LiFFU@&+EFyx1TYJ%guEMKLG0~ULzpn@vvYO^tQ9ei9`3uJ(ey*1I
zTZ~J(j;RTY>2?^@&b)U(EYfz{l#X)sU5i(%IPKY=a%7Ugrif&g1)j%j`V`;Jp5bxr
zyG6J~i?S`#e2&RVJC=C%iRf%>b-Z<uYuZNUo*gaM#P`qep84UELedA%ms>0*=oTGm
z<Xdt}Ys&&ou0=<5);?I+xWL@^sjSNcw#cXQ>`iAg^_~?lH%i}8G4i{8AUEjN=?nL-
zS<d3#x~NTU67Q`yth4{!aGj-5a)oob=>H>k<^*YOx|MP%Z?jjqq>61(&;!Tn%n1|r
zyi`2a<2kF#$fxnc8^!CZG@K859J{@tPw}FpcbMD4W6usuijmNAuB$s9^k6DWfAj0O
zA2mgLnhp3J@9xRuuspP<XUgSx_DONdk|!5d>@%sk>%Ho`+NK4bbt=mKt`AjCKA#-W
zrQ+uGFv32)YHmMArh<ovM?~h;=}}kmr)`xvs_W`>G}E<Yv8Ut%*QG7O3o@TC%IrGl
z%DO19`9;opc1NzT6`!{3%VM9&@AF2>V<D%}mc_gmzi^wnuW&ro^<-Jhi?ePUeu@iM
znayVxOuut#r<p?ND*c(iW-q_BVODwF64(2)d6NRR9zOT~o4}TWqrK8kmWy<5-DGxR
z<JtTd8Um5G`bCoiG-j(T+PUpoir1uU;aA%{xR=CCN#87SLWkw_#Q105cE|K|`kva{
zm9nx>B--QLArF>4y(WD98=Le$?#L3b3wk<<>4lo}sb|Nt`vl^7K8Y(ayI*OVn!*)R
zv84Y&_o`!trX7=ysHoL{o)y^3WqvAhzN1B2Xt0%*S*pj<yb~L)E-iUtp>)t&`<3G9
z#tWZp-W2Fb@N(Z<wdbo(?y1$1s!TVQIHc+<3|^Nju$wPTrP%n#!kjeS*9X<^XPfJ;
zT4^WxMNn98>&b<pof(2`Grd2)%{S3Gar4}^Q=5H{oay*@kx$);OK#et|2*@rE}v5z
zJfZ&6X02O`PvuWNw4rf}gqp#{Bc&$l0+P)ondf(HblCJ-*zNd}U8a2|t9$OO`LyWx
zr8PHVZM=_MdiHZ&Wo6FGW_REFYA-{S{yfdB6Q6R~u4zIgi`*+Qp)+v;+rlTkKl)WA
z_msjPx1N;`wI90O=bSxj+Kj0@jvs@vTu$e2Y`!2<xNc#tTBdY(wC~Y1mtGerr7*gv
z`*v;ft(=hg<ZOuA$?c2Iex7pa;0&2p&9N2&cJ{lqwoSc!uTcB#%nOnhCw@h*naTTO
zQfTd}TZbgiZQuH7Yq$I5Zq6?Xn>U>itzC5F`ij;Fg`i#r7Tp6gY>w}~_VSy<hgaUu
z^vWMj{4ZwdR=rF$ETPIcu;s}z+qDz@_m>~EnBUR-@}z*{uRoD1%}$)(e$u9%^HRL1
zW{;9}ijvU9s%rxM>(>?Y?(b{mpZsw1hQs<{tNo+qYL%IJ{EA4P5Hjh2yGY};e!;&2
z7dBKVyvjVf-6T&SgWcec#(J;*!o_`SUi2JWBJ4T4ZD)BigPre#ml;8uf}VUjc0=FW
zETt>V<U@|t%?f_bU#rr8`CcfiJKb``D6>%dTziAu+&NnUWLfJp8r3;+r5%0VJiYsj
zKm6wB%J08wzt^Zq-+FDSt5>4i%X2ZunB_flNX@GTwoiMk=PUE=<?}lgd+*b$yk}ft
z8>-89{hekNu`=Pxvj3~CFJ+znUVgXp&4kP8XaBw1UG?h2eai+-%^C6i{VRovXSDGf
zDa-c>ZAsmb{q<<T)X75Mm|deb@SP2HE%~m)^QGn*V@2q~z{4ty*8~GkeYd>hnCE(o
zPhwIE*Q2`Mp&nhA1aGP2haR_?A71t2oA{5m{9ngppKpm@6W+Ye+-#Ps$L~V_DMi(X
zzSVFptmZ!)64Yj}?4W>`N|@&tfuA3xCB>8V8_Q)AeI#dim#GRLTT(P>S-@HGqDN^0
zA&-kb8nX3^NZguU`EiRNk8nJ<S-iK2hrLP3UNQ3@swH<ss@5NqIAEMT^JsfaLxT0x
zwuNRL_t|q~1rxt8h32X&g($Wd{Z~wy_)Ty&LnO~cbxsf0*bHg@A4X9-tQ5Wm`V}_v
z8~Ow)3$<OfpQ#aOUYPRc0Nd;daqGh>U$oc#EzMHvXn*%PahFHc3OA1&<vRP0YCFa%
zcA<ilX^}Ds%_k!&&n}MKpPbfq*mf(sW}aG)#`6xDgNEOQ3i+3d{A1EzyG3%QL2kyO
zvLhQi?}xSuaOjH7kWD($@N@^)7J=m14M{h*xA?et%$e49N8IjCa);?Q{ldfgInEP&
zj+)tRtz4~ESs7u*^t?S_dcwQqkssAP88ij!CzdpS7TkYL;`la0&x3}N+Y%#FgipBz
z&pyDG=BUmT$h}5E?8yz4r-GR;I2skq#rXekbznMRAA8W!c4v{7BR^9B|K0%pBKB^1
zlZoq=RbD?B|L2gtq+ouLu=zHL=B1`pLJ#YA8cLhpn7U@UpqYk0+al+(i3NKqa&Kq|
zs%}fqJXmJ9(8Y0+*q`N6`j%Y1;_@DgbzgIY&vZ~-`NR2Cka5216#k3+e;@FFZH^OZ
zZ=ZF%&84HV@P~js`_v7IY#b-Eueo@5X%=uyi=6wjUFk(!%L{9<$Smg<{nEnKUWuW*
zp4Pi|$nH@W{@EzRWhj;OLFm>&@3%L+LnB%b9ubr&7npUSGjFG*vtmTFk?3W^p!%B)
zx}GyaFY+@f%-yTN&~cM%eMgr1jwwtNdv<sfJTtG`^WQN2#Abgb!Ja5j(U_GzjN5zS
zJuAJZMQ$ojyYo41`*-Ps9C3LcCh}kOoULN`EmYL=g@pD?7p4WQ=Vn^H+L3%+a86~X
z@d6D2zFmdEm!=0gF0?CCJ2F$c<tG2vh8b%%u&NuedO9xp`NQ$I#l&)#nSWMVS*cE7
z+Ud($*}lMhv3!chyQkJdoe4I(7Au{|6xFO(wP=v!w2Zm7$Z+Dc&~m{QLJ}JTr&b*k
zOb&JFOKm79_ZGet@W@m^edlzgg{@7;3eTv_VP43|QMHV{OM3sxWgf<AEG{+73oY3_
zR4<(rkbAZ8>Pv37pIsg%3;vom)t{2pSRozi?zi1_9)IJ!POauo|DQ~KcXR65jAqtF
z&b5Kl>?#Ca&U8(Do~M5(RmhQRQipWOX3lTMdb*j*w6-tcxtJ?vr5af!%Q4Ga_|__+
ziA&|TPrh(d@SlOeZY$lLT5}b$Lf8@)9Xhr0`E}Ni3B}r6_K(<gcPq9Rd8`Oe@3?zq
z&7E@hdEaK*u3BqX74qJ7)=sr0LW&*&&IP)+?AAoCSTj>ub)sN;__{bDN!f)J8p$fJ
z12d8ZSYux;y(zd_=HdDsf}CuO8+fyl)Y8kpMa^XjT+V)Q1FOqirUlD$4Z@UM%+wDt
zY8;xoVX8EfhVbJMiw7p@3&e$gX37>Y&p&Zvrp$r%cQ5Botm;Wgvye^{Z~U0H;eX(!
zom(w-I=amLC9(0OAhVQE-K|Np6<EDKt`f6a_D4#XrCNZgS|Cb*l}TZ1fPp|z_0}+L
zhEQ#RXl=pB2_ok=s&Pe&UYNO>IdMH>ii(pb|A8oe1H&!b47Ye$W*L30VX|CYv3i2{
zR~w64J-4}L)!$y6Ah~W=*~A&gCvsn!nAIukw8F5At@ExXgG$2MkH#t*!RlNWg@U9V
zqCRY4`?%HU;ZA18T{j$e#oZR1=&&nLVHeYZU6Hp1m;JU{^JoKi`*Oi(e$hgu-x4d1
zotpc`Y{{;TX@Ng&c@%RB(|Xww?M_*%>L?Y+7_e&Z-gM%})QMUJDchR2mPT<+Uuplv
zBKoS*{{NxcHorHY2(#SE=^D6U=fY}fUu~hc+`D&f-1^>HH*mEHzu|JP#%&qd0>5`}
zIcT+r-*FAwLYE_}Tvl$Ywc4KWgv;LJm(GQ%_L$aqp))DXPbSoH?~st$u|TXqG;>Gw
z5kbFMA}@E<Gd~ni6l$?~bZ`Ra>;^&MV5t~YM&ZA=`Ib!Bx7lhd^TS=1IszQ4_IW-$
zyo2!wlj4!<f{e@?w+1L2nYduztJ$nux-!;m2y^a~-r_BqEG-<$6JGVoAf<Vu!j^vK
zED!Tnn;vye+_}S7P{-v#^5!cU(a)Hdaxs~AS*72}Zqlqieqr^dx)<`tW<(m@wm9@V
zZLVqlGtM~Gu<fBAXKDT~6WF~;VA}?P9nMECH?hcE+#0)DB-Z<6Ot$QlH7D(KPL|!;
zu;a>p9>-H5iKn(FZj*huP0eW$lgxJh!~=>~SQkZhJov5?5GDGjX#482nbRe61k@$O
z?$k=goZ&cDH}6=`hvkPD%=CXw_n$p$&uzg5qeP8Mo7T&I6mzy*Zqjp@c_QaVZGnx_
z=kD34y6G^?syP?K&B)fs^1O%TjgG*pnj<`d=YzF3h)MaJ`sTp&A*W!uY~&e1x298u
zg-4kKr|=XWjm$aq>$VC{^pt%C2D_ryWON*8ywNW<cjMDe!OaUpCaVO0Nt<@e+Nn5a
zTE{Helx;ngfA*Y@ZeAbWoa{C&UG38S|6AvWY%G4VyTd0#di$jlx}4=p7O>8GBm7Tl
z^?aVK5vK*F-%)Y7b7VnHX56AvjJCS}7Mx<cxSA>PD$iUImbb#+cCG&Q_!RTUQ|ew9
zQ*}=tk2=j=cKZG01D7`)xNLRse)CjO87D^Z9Y^ojI7FRp?LNVLqPT5Ehpgb)ZGvSR
z8F!vv<fOOf@{$d#@`?Ofhq;0T7~N)XxnOkO^N+x{9D#2&Cj$)53uy`bes$hE`WCa{
zRlAL*1v{p2img0*B2#p?!nulzQnu@Od@Po8=FHtLv3UEQ2(_~%F(+<ZoM&UjXmjgQ
zOv4@hyHyP~o4*}7_D%E7EU80Ref%38H4POFS4o|oqQTJr-$!g(hU~_wlTLfC@F(6|
zZ+(Qx;T)Ud(U8VlMvYeu74NTUx_Zg->ZiR|<DBk$CSDC@IFQmSyXj5bmOTe~k6pVu
zO=<U;kQpU2L!Lg!{~?vyy!cqm!?QEaJh*lJ*QSXwe)Ia1?V^r`ygX&7<SZy&SjO}4
z=&DM!%Z+DKSiRFr?I+2yCcWCZGF$fL?0x)<N1SxntR{*E{<+MQc!9O{RwU1T!$cMC
zT7lSFwJI%v4$i0C(x;p|FYs62a@J$mdx8Jm!nWPJZ&jVGUAuSY-yIM4-&L^ulE9|E
zC+wug@!!wdvwBO{$nKx9>3sGbo!T?zH>Mixel}&-Ug@g`<}6sUPf^P9tU&GmT^&cB
z-YpPc<XIpcAH6^GlwcskhJ%%vvld$>SKoiHE94P3<Iiou`Rim?oVkCw^3}Is{;v$)
z*E*l_TwER2>%G>2?f+eOM!N$)GOiu!x^_1Bf%)=c_PX1qYVDItW(pm9aH21Dw(7Hr
z-s?YqtZm6Y{bhP-|G~sJ$+D_>vz7(7=+|gZVY<2b3fC+F#wdmj{xt$iyw52ry0(7H
zamrl(qubm4pMc9d)y3<cel0eA@b8t|BB6f|xW8TCdRHJ8{6|1H?bSEG_v{;23--z@
zy~<*<d3Nb``})|AiV~HZI%YbRE97acJ;Sr<%8$ftOXC-a9$zYcN27mbch#kL*V-%W
zSdaZ}Kk@(V)S052FZ{T3wPUmCH?5X+)Ae-(ryM?^@%Zk9o*3s?-CxDt8|5-1dp?9X
z-al|kmghD9mM=1X_D7?Fx%MV;zgh6AL$dt)Ev{_Qr%T_g4qq$9|G%9{;o9~*+4LEU
zPwBjsi(PT!<08+*?-pB?`Ok%fxQ2Gz_TqUNXLSFK#GEPH-F9r{O=!{;wAJ@gi9E#I
zc!R0&(Vo?^TOU2TImf`bXD-JswhtN%W#!Uy-d$C?_+;7TtEE{pvb2SHy0^_g_jExb
zKTq{??(BxB%Y011>R%Z?JkS-`ex6Iq@G1Z7<*$t{ruMc7aZVKVKa-+Yx6yRTC70;T
zc_BH6ZLaz|nXL6!<bA>QKi%p$$L9zCKN-u<H_)B`XQFd1+l!rFl#i>eJ33R^{`?Ph
zFQci7Tzt_-!}W!Jo`15-OO?UV|A0dy(=T3^837LuRQ+Yyw79ikss`h%jEm8%sUZhj
z|FOx%MEEy&G;{pDn7sAX0{iN<9#cbZte>~PY;Dxv&Ckz=t}grUa!c6ZRG3q>L*&NK
z7N3+)F5&jIYL%M$*oSeAS?<k4siz$Dn76xy?qFSbqcP>M(wkFLdm{8wcT5m`u+8Vr
zwyekZWG&3~Blgw){pV{j{qkMEa&~UB9`T0{T4QCo3)X~wnsRcgX7STY*EYtStvXmW
z?{|>HY%}gy#=fX1ht;j~gb#(z5Hrm(TKxY<UXhknhIaK<kzaDmVk%c2a@L-8n-^pG
zpzh5Uo}NVp!P82&nkZginUu2s5qmY~lIOFl*9RtDQ+zW$(EDJ$W3kY7*RJjQ)!V$z
zzE58rs@d|TR{m?QWgCl0-qzJ-6MY#rAGwvq&1Z00Yct2rgl?x3I%jk?t+P1k({NBo
zrFr9Z0j@%QVbR&%>-`+0SBUJ5(aBN#C)O3XP|9^pp&Mh#6+=Zi!B>JCrhA+$=;G{g
z3DV}%c>Ktb*YXU{DI<%X?kSq(Jz5@2>BY?34jfp;t$6p>rL)^wat`yF-Z}OqX#Xmf
z1a}5wTP?l~%eEac-f8vUs>3>JL4#6PsGFyz5+lp1^#39%TO7I;R)s~TDg^dPbx!lx
z6yVt~c}v8UsS|%)Wa;v8Smm$b>ny~eaWNvRb!vQC*2|?~Z9)rVqdp(mI&Jl)Q(4;a
zg}xp-iJ~oQZXCCI{brrW)YNOH3G=@e7&7kaDAW75X76j&9dTT~4Y@~k%og&qF39Op
z-L7%n=%mq^9^-P+$z=-;?$ce&d$A<t?fT0h#mtIVSzeScxOD1+&_Z^0CC7%Ay|Nv=
z8LrK%AGDr1(`y=)E3`l?$W4(`ZGT<UtFV~wE-R85J^!tk-tz0sQpKuG4o*CiFTPF}
z;ZW-IPzy8aR1#ub)~v*R`;}mFVB@buRxXwW0?l$WEqME^<6mhm{Xf^5HDKrTI-8`Y
zwtp*(PfRa6wPU?pbi=3P@>3S}a;Z$smQ3YlDOcU5AE$ZdLtdiwln+j2{h=N%f=6Xn
z9NEG4IOUXxaC%MT1)=zjt~x)p-?;zNn{V@3Ve6OfV#b9rpFA#}H7!51<zHR*<c?7P
zJMKC+{qz#{CA>c6eX&G3_{5#lMOwaXg8DLAVG$QyEc^SUPxQ4inaF*6>i4T-u19#}
z@_9Zkxfu(re=lU^baYQr?b0wh8Ck`+=)^fo6`QZ_Q#3XSFmO%%C1`)Cv1pS2lBQpO
zH0`2)iJx+O70ea>g+-%wrsOJ<qf>MzGHIvm=n0#Wys$tw#WC6Obd<tlu^j~y{_DxE
z>5iFEs+4ZYZ@FnAZ}+#N8CkC0aUZ#44$B3vD_(lKP3X_;-G{zNwwzho{6Fig15=sQ
z@)PGYmum0|%$>BM+4fjV?Z>&nCpK_)|7{aVUceRcC(&r)&PRO93lB?+EZVbBfj>L@
z@d>L8#z|WZm#4_T4Au=$YB20c5`D9x)kE&<zDIlYg;%dQF0rS4UgLv{3VOl}LLo`A
z9259$`?;=IWnDWlF@uqdDdPAKJ*9meD|%F|Ke-0IGBKUE<dk}9#&L%i$=qREnpD16
zihN!)=|H4as&(K-cIzDm=G;H|w0Una81ODL=gNJMC;rHx^$3&de%EYgKf##rbtOCY
z7__-w`5!f{Q)&BR?aFhl4!@Q?VLLuu-t*j?^~wV69_G8012a5lT1rgW!LU>>Ak%BZ
z#!0VJzPK<}vUThzzLAsXzHxs}lgO@{ERrRY^(_Sbrx`YK>ar%V8-F>?<G1nf8k^D+
zvs7JpA}=(ZXJca3U9(6teD!59{?IvX8<H0@NlaPzDv-a6#g`$HW0u;HWNDQYM-jn=
z?6$Y2d;H*4;l5cIBC=q@-06yq3uXn$AAdEmRiog|vNtc<f150nTX^rh{C7s~f1g4{
zf&>J^oHJeLupQ!hR{13B&lNdqnU3vo6SJE{zntFC_Ef?sgKf4VPvT}b4*ea+HW+Sp
z>`C9}s?R9W=6b8^;s59i3)YgAt7GaeFXTuUpOQ0|BfcwfMW%pIl;XZ+EVZ5!IJ1Qf
zSJ;>;&ek}4lr4z6Ea77>PvA_g*UX~kJ033L{F&<%p)A~Pb9}|b1}A?_reNnfmTa}+
zj^!DS9=aV#qVCG)e`+bU&F)$x_IE|=ZzV%6o1~WXnF*|ySQr@JuR9ZLUfn0LX7}8x
z0x#~bD_qrX%#NPe5We5a_x$fW(>vc^QW9oz$g2)d;t!wUHI>QYwnv1MCgX%jvTIHz
z-(Z_`sz=mJYu=jUBITapNxa^UQ$!sM&N_vQ2~VD}f?4L_za(G3gw4l#cXOMuo<4i3
z@$;M+n`Ny2KRR2?^jh^*%5#yX|1-~|l%2>AJT<{q^7zfWl@^L_e@=_-QQ3HhMZ|Pn
zsE5ejm>pW{-IjAT#?=cXbzl18T~<}-%vpYgElfqVx^2<%`3n@drrBkQ&Js8r#`%)}
zd5n-^QU~AV4N{uR#g@pmt(h6bxN>jR1*eraE&G)&$}D#~?NZHlGit>q1>r!QRe^<1
zr{wx2Et_$cWzxy5^F4!pUTWF!HH7ifW5&OmwrxM>wv_u}N26AB_~shctr?DH;Z3cM
z_cUkg=AHD8THq^wym09)l`TdgL95>=teddmpOkw2F-w8FE@_*l*@}tpny~PU(^2nE
z*%`qh7E+b&OGW=%t(_x!=;<N0<C-GE7u96cHvR9p`t1tG$6iTMrz4Fs7!sDR^tUy5
zru%bCbLGy|4xa=yW3zW}(nTx%ME1PCzqC`)Z|XWtvHQO-S0%)#xt?}%jQaLz-ZRUC
zQPLgPKApZdM}5iuu2W25*BdVzzR?ey*duqQB2S>ILvhmG$_%HdXB*erE>)Hb_LE-Q
zS6W|j$wB!XS9?pV9xLnSH)2n^nk%+4p2%r@yK<|5v2lvdewWX!8m|)X?LTr>Y;VS_
zhc4<*H-s#f`r2CjG(}q7seQ@CoyJc7ZAGt^c>4Z((RSio_vf6l6E|!1TSSt-?U`d{
zHrrG&MTD!AOYZjXu+4Kb^bhJbOGka=ZuxUabW7Ip!n(bi+G76e3oG4RwDe;6#1~!u
z!e2A0{FC3_Ur>A8?9pU>ZIzn>fyVi&?{=%$TBvp<<ZGqHzCEF_XoKCm8*Q1Pn*JAl
z%=s9Yp~&6Vq_t!V-x@pDrPn`i4?gCxQEX4Fn191<vwJ%gwi!&@cT3@b*Te&>w&ncV
zmh-s#-~utp?uF%#b&fTydGK)BjU5v-9~>y(wea+!rZW-S%d{63{+iZ$c=tXF-fIeL
z<vuCy<2caHDR-k|(FzS=qYYE<8|-ME(7E#BZZq$R-IHvedWt?Nl(yjFoYigF{DL(w
zLnUT)>%!NXYl@thnmcE4N-bZ>9wxx5JEQOOMV2s*C7Mrc9lMy;TxwB|nHjf(@4t2l
z*OEpijt#waFIdYStkd~0)5m~6l1ZO&0qcuTmUasz_bJR&ZsF?r&E2|Np0R*cX_tAC
zv|;i^ZC1tI!rI;ID|WBRuola<UZY^hnJB)i`{4c+%NsS0?w&0qyL*yo<DAf(HQ`g{
z$W4|x^K#?i&xVTDi`1E9k2z0Loo#)h+2~r}9IcZwM@~&PyRd!53U1XP4g+C^3*H^g
zP90J?W{e-^Uascc<Y3G3_}I-^Q_R<FkkoA6nZar;alG}whSlC3ai7}*I+;FRuv1yU
z5hcMIc%o&(rgakoMBF0XKQ=Q(W$!GFXnkp6@b#gcm+-zvo+81!C%Y>;e=*=}<TyO(
z#v=E_Oa8C8JjM2<X>^pkZQ}`l<s*G}PFz&lo}z8|@AcGww}nKSdQLysmK9@|<hU?%
zvFCxVZQ0(A>{EI<zHrY=@f2e^b!5lx3`<8LL$S3F97=9?ok<W9V&UeJQEHvR-Db_3
zUdXD`-DPB<+orgtweY~c1rjYy+)b;e8g7txi81~Y#mlj|z2PO3umQ87l2iBwyMi80
zS4-#S6YG~6u3FpF@m65J#{yG}KXPj#CA}ve*65M{+%UN@XUQDSGha?wzT0qkev^ir
zi^7H(yH1BFthsGEA*3Vz1G}q_sF4&$Taj3uwBW9a2|EjTHWu?F8ZO)9Y`D4DdijYX
zcXD?7giNSBB(QJc?*FYDb_ahE$lSC{&dP7UbWd*4#DfCnF+REnr8y3H^Bvi}M>kMb
z&tuckX7w{2djvA4Uwf$b{K(NmS<F{^xNduPH2kvJ|3Tzi$qXar?iMZK?2wL{pZ&p!
zds~07hhON{Gibh1+-$wWJnX{W^F3SYuZUD>n11;<#rCGt`_8R1CmedcsMT{NU&jTO
zkAjE4JkWa^WPkt0q7S>)Z7-S?94X1<sJVQ{tl!EL9t+G2i|SQ0R6PG;|H%`=3?fHU
zt#ue$%t~+bq(mNI`{NbF!gJ)tCN2}N6NhAZr|2wEkv-kCr_6Ybs_of1D#zJUEG{lG
zZ<6e&xvf(dwf5<OgEytthDfn)IpXs_^izAZ1}nb+L;LJ$Mh302F9V)O1uqTI@BPu?
zagpPf)s!7w^F4oac;7VM;kcp2Ld%e4tF8^d6X&*xiUN~wo@k!1f?fISZRQ!5bDFai
zXQVvgp6A1NeaaEB%>ru<bO{S{i#i*YJUr|B=7`^w*|Mcdd#W5Vj9j0{xXsaNK6GMY
z_?N2(xouiFt|>CBpZU2_{nP}NXd|^kAqnn@0-DU)3&YMiUDIW}dhYb8mWX3d7nrt6
z2;W)YR({&;vUJD71)|$FE%xs9;QW04m;v*@2L@L@v~!)c@IAq_aOS*5k=DuG9P@WM
zRY|aZ$<mxLY4T!^mF`6jlO#{>D_G4N+|he#x|VVGe<e{JvEL3GRzygA;X2--aB9bb
z=`0f!dbctAIK*AG?w%`lK;Y^W-!)TYPjQFzJa6G%_Jm2*;}m}n&-M&)#ZvPbQ@vU`
zPBpIZa+8T_EtpvBGo$je*!qy0jUTpOP!^1yd7U@8)nw~rv)W*Ni4F$c=E#etS02pM
zNLbIqy#Cu|J(ksa9}BM7Ke4rY(jD#8_b}kHO)FQKEuZU_*4xpJOTGrRT1YUyXlTv2
z*`g7XT6^~T7nit&zPFBWi5Bm^r6RUXn&YmHTIAW;@jZqq(sQ;fnDl6lnABmj>#til
zQqJWy9o?lY5bYE8WQWbNJLfd6PSD^!&$mhF(-xZ@r}ik{-P33w{6Aw(?~jsWD?C;h
zZ*+@1bUyG#hbXHr=Wf=P1=G5<TBA8`Kig_wCDH8BAz~F75Y-`KYIRZMtd+^*?#W4k
zB3~ummEQQ5vG?i<R!Ls2t1E6^wYjwX?Y7+qA~cUqO}==ufHN{m^k}KHBdg4soYdKY
zHj0^nI*WPacv2+P-mVch+Aj6UL8jJd`J3+4#2rB#64y=yEqcp+^}?zB4iD0A+OPYw
zX@LauwFx_n_eS+RIZ;<|d1r&Sxx=D42khmp@SWciE%)z)ZlL757biZs-se*83b`5m
zbfWE&v)uYOW3GHS8F2N$^cJx|*%*crLoUbAQ(ZCPOLbI#Cy8!8!YaA@sOi<6+KK<K
z2+YZfnpPzhl=<_jc$A+s<F)8L=aOZ^cD!1BRC=yzrFufowQU)5vi4rHRukCk;kNe%
zPgV4Jn@s=TF&y`I9?O;CZrdfg<0i+71lwr}N({Wbn^Yz|oUy+z@%kYvxyhCHFRQRt
zf1GBtBI3)Ic=niS(pPVuHrlrFN0*n4*~yHP(pO!$w5BZ$;n32EJiEg)t2lB&t<pT3
zI}A2=rt``P|5>&`hF45!&N>fq%@dK_YSZV{ieL8*Qi|nR^Q5~$k(s$Kb^i`WF}IG8
ztafp+X!D74#C)FL+;Qci2*WLJj_3!^SLd{<?whl}v@6i&j=EG#>x-u5FuuUMr*3AP
zRr@$Q{@K+3iJN`HQm-n1d=$FWWwZCjyGLS<&K9^Vc=W*2Ybn2Djy*V;nRIJvu8t_<
z@^u<KHw6T5c09f^H8ZKSYnGeMVe5sZk6i8Jy!blyXl72o{dC#%0>LG9N=0o4cWUs4
zRXn*Sr17)nRO(`-<tti4C78EX-Dum&9<QKubNAG?v-YJWa@&PESJevFXFkw>xP5Aw
zZTH-5Eg3H~x4qu&^dfFuYF=5_e^%k-oeQT-dC|z0Fv<0qV(NjvtUZO@T*tm`Q<?g#
za$}HwV%nA&J<2Z~q(9AO{Fk;u!u-k}?kcevTT5hBc6IMs_v%p6WRZP~rW;7k+Y$BS
zbE4SRE4xIKT36hb-{<t&x9fkeiSng8YcGA;C=z(Kb>-Dl(~Kj|WoxyqZrXNrznxib
zwZK9<F>&Xs@lMm<{7ZWKEH7TMn@w}$YrEttmG5p!Y!423IW>09j%AOf6O)c`UpsX8
zW#pAPF)gwxkB=&UOg=Gr`Sul)GF7EFX}X>9K0kk}W4^BE{iVIr7tGLIu}DI;IsdEa
zhK?)eOcwSgrOtGHsO0*VXI*-om*9k{JyXs^B%e-=X*($#ZWbr^QEb1on&wNcOM$yA
z(gc$nN{)#>PkR^hv~bVnw4)7g7ToO)O!4Ad!oas@A@|8ODHD~hB)DCAwpKVOPk5E`
zidL>Wj(cZTKF{BBW9Gs1<qqN2H-h%ZF1i-*;(wL#CE=G%ZFw2Xb6?+lQLsPYg{6O=
z@pEqLcn&N1j)SG$=|`mluFQUO>|IFdRpHqUyBteg_MhFvZM~<_H(-i&vf|6jhMDKr
zF7p;o7cG3L)Xnv3j^BX==bZMr3O{N}@Y+<?#Zx(H(lyS4=X0)oYMG`veY2TF+~ciZ
z^&aoh+3dC>fOq4LtVuDewbbJ`=XJ^Sbv=I3+wGZJ-^eTD{+VBWf<^yl9>2%xN-q_%
zKJML?x@Go}Ufs<8wLAyU#j^=AB#EWTn&uoi=Xq2uaq^cOiKYVvwFw2Cg@+D3Q+;@#
z>{5=Z_@#TX9rcW%%f!}Qv~zEL*qz<J`YKzGsU&mI3FT*r2mhBo-FruhZ;R$5ra+Db
zFVfAck6hRMp|2W~GTY{`s*ddS15?_#%*wmgsxS*pFl%P76uQv;`A)&KHd&kYu4$#u
z(ldo5PRG37_wKFOuQz&zdHeH>>q_S9=RXmM4e>d*Z2o(RLxMKTCVoGbtg!n1A)(#D
z>ay=s#Q8Gk98pY(us`->Mv)DtMwjGuPrfcT`5k#Y-=eNR;mW?hmFfDm3&&=;^I0~(
zo+=-@zd&q_N^_lHamM=e>mucX>r;2HlGYRuJDzT){`ZDhvqJnmvuP?0lT4o-zPE6n
z`JZ(8QyFYU`w}LeGZj+3kbm~v!n*4^PX&JD=6*ll&vko+MM=txJ+{}M2;TZ%pVlik
zvwQ6mru8iU(#<Oq3V9!=J-F!3A!PCVfWup%4DPlh-f#<lql}*{8xvF2E@irN*uMWb
zL0w?Z^ST$l9efQh&b0d8xHC2O)VtLS)~q%t@y@xr#zXj<y!9!E32xW_2fV4A%)+6^
z#K!eRWx@l8W~N;QQZE7=B-;g*`Q8|95OM7hm**0=u~PYXqr9^mPv)k?6G42TvrJA-
zQr4WVwzFWS(Lpsou3!tFmmZV59raRpinM}7l#aFOM;qn-ny@IiPcWLdN7DG!)fG%0
zsaJQUDo8Tsx>?@2QmUZMxb{Sbq{q~(*&Ae@axLKgy7Jnds_#lV#o^hKMtLkZ1Rorm
zI{W`_;nQ~<c?=h=He)%xr6}~-hLnAk-dTC4Za#e8C}U;4<wh#kbY0Wn&2De2w!YR~
z!CJFcH}!PxtzCj#Z~jVcef3~X==m}aSBLYDIr_h|-JIPM&=I5NwC3M*lM`oee2-eA
z!SM9j<f_bDAx5ds^XJ>2_Q^m0Pp_hHrtGdF4auU*zpqD6y5YA=J2om;NKSL@m0)i7
zwG)gSOaG=GpXhMca_jt-e~}S~O4_CdH8JohT@ezWv~!`5cc-I9lBP_`3%1b9Ta6-I
zl&5VIa2E<&F+J17tn_i;+*Y$-_LvnvgneJF+;*~HP3|Vv$=-g7(o>$>-VFBN%DEvW
z*C%UbRDgq^=ac6Dk4;7IJGj-l%a;1|9=SM6YpZ6%?1FVi1UrlJN|v`SxmY4R-{Pam
zlj$asrWt`@t<%%Je^<H%hNs&eVT+G()MAMA<+TuqOktccEt+GMYa0urmq}Jg*Q}IP
z!Az~LZnI_;7T!oun`n4HIQ34@{bVMKu+_SKj^z=xsZ)}?wcqS)ifE8g+NHC(%JoTf
z_C+D_+#K7NZPMFz)_xV(x?$}$R>?z*+iy9aa@1fbPF+$hP@S6gLvH0&)7P?lYdn7_
zhX`tZjg9@&^-3d{mos&;h_FtfTn@`t_Pf>ZGFw;I?($OI7@g$1W-i0zbE~Ju7=~7@
zaGsfW)ysODMN5J-_q3Gh!T)XSG_ryXzTZ|{t^YPkt-Z0=Y(jI4hBq6h<SVT;9jPlU
zCUh1}b(q+ry=h5KTiFJeiK$vw**7NEPhF@Jr?_xdID6&Y7n_`8wu^l0xzqP8*qyEL
z<gD9qax0fkj@I#%KDa5;`oW#r;>M+S8aL~v<XhLzh@Hq)TRy=l?bPoI7qdlECOW6s
zZFx3ZsdIY3Q|bKUoc^`nB2yMPPIWxBVD5~aA)7b8D-{UkW9*rGtSZ<0b_D;#gPBda
z9)Xvx^Ur*1>6p8v$o-94QqPyo&7Nzgt!vhNeSJlpDch}$>!y3Q$?o3ws!LW;DC}9n
zHWqt<>xY+HO|G|(I+0iE$z}fk{wMW6D!Ko+zwDpQu=AqY+P^QJOSK1ZS{`HUEuM4P
zpkm?6!XuC8qzhE+m{fBtHHA6S=S9$jx`{9KOfAp*o;xg@!FkkHZBO?%mBmiKR63Hx
z9(R08aht^M*2*(oDBuK7NO#$r3%cC5n%Xm7h<yEdCadtJxZ51J`8y^w=<SdaOx+c(
zw%YYbfx^ZUt4*#<>?rAXR%Mvgt#V9j)s^<x*acFqdOKvfex0!Y?j=28hrkkt%1({1
zn-@Dy*s{d-(pt0U6P*~>mn{ADa^i&YC5wMA5|TRok-Le<QrJc9yMWQkC5rMtJ?0o`
zF1LvFQ{zvZE0B7W_2R=PYFbQJ{4EScoCH^CKEExo)=Bfk{|R>lg1Iimh6{vrhizG8
zq4rTJ_)7S;r?z_O(=El4{ut@>Oj%gHP9gByk#swiV@qq!cb$;cU=m&TlQrx~u}^}}
z6Qw7eu35Kc_avsQ^h@(iG4qrPFE}(+s?x99BH)(1-F8v`Uya@|7o|Lk_$&^VZB6fv
z*}%c9xZO{n^jR<O#*PDju6lGfwQ9~^=;3R&fTh{@fDQ8-jx4_y2FqHPh;HOLX}Pyd
zd>2RY!s4Gd;ye<aPtI7Av^IftvW$}R+;2K65>+S8MTCjZUs&MGm#I{nqB7&e2CweG
z)luvJIkBzgZDT%u<=*C&>H5zUO=mg^hkfZ;rTk^8r&7l6s3$5KdlG;BzoOzeOYmmh
zD$%c{jJZq#X%luiShZGgdRDA?Ec^D`c00RFp<sq5m28`i?=$+gewE;*<Z`zieG+<w
zuAfdm5-JMxoVInMWaM3!*@ryMEK__tuf%v<Q#mqy!t%vSQUrY4QmihkKAk!%Fm7JL
zsVQqrUMVhF5gV7MF4o`pRLitiHgsD6Thi94YtHS}{^PLdti;h*uKhdXtYkYB^|D%w
z&Zs)CXZyT5S@){Z?d;|s?<TmMx}&r@&toOCcjN8IgJ-s#-K)L1PDJW=)jNfVHyrlQ
zcXk%soN$1dO?=Xunf^6GD)ZRYHZYt_J>am2yYcK!gTroFvMz>`79Lq6_&e#ACC57c
zCI4mT9umrPV$EKgW~*MZAoy9a-=7ekbibD?y>}b2&3%_DY$EGnEpv(M<^qk3zc0_9
zTQPm6QBA0`R#%|rGO4|<dOS`&Sd!JS#_H)}#qREDm8<5g(=j};gE1qq@XV!x>cyAY
z3Y&zBH0Nl~&<dIMZbJQEZTX)ToO>CAxa!uOP5R3f@o>Xi?d4Com8FBGxNgr1klHwz
z!ASh2XZ)l~YAa2$Y&p{>?GaPw@>UIyICP5d_wqBHvl83FehA!aZMn)<xHW&f@q#ry
zO+3d+IX1TKX`R8+9df0(<Mj%0$(IMbX6aZy`kr;mbOq;Yv7<qG+H>v-=KD#{IWE(2
z<=y11?K7l~oxl2@r{4Uoo#L{s+^bfewhO&_o=JJe0sl?UCx58Ck@GRX?u*Lx?O90&
zB>e=e*h;U5S-;LL%zk<(D^}9=dC|JLN-kdO<~I55`MQ*qOSkpHY4103M)Mow`}8&|
z&kIe7@Sa@J!?QME+wyBhtsNTG2BFKeI%C$kEDTHy6LscsTd_-U<Fwv0Zq^=27f!GW
zIiyug@|^D}qq5m~dR7S2$yX0q&V5|AI#4@weMju^Q)?Sj_F7k{p1Aex)@&OlvwpKR
zi|<}Kcl@@*w(~+SK386<Q`xbRZPP~9>W4QvBF|{~6)x`zUm`zK#G+BmW{I5cvoB2Z
zj-*RXiTYXL?GS$I<bTa23+9~*{C{deltplhuzG9FYWYRCHw%>O&${HhHovwocK%J#
zHA>DVw;R@II=w&h=&#&01*fiGHxhf!h<1CeOWu1Uv$(tAr}fVIo03cFf7fQ!-u--3
zq_p9(M#Gns@<#SQ2|9Nt-}^PYN$8Kwd(FS^@7(k1-+Jz<jI;KKSm}9I2jzEM?(j46
zdCsKS#akvfSK!DerPk{!cM5rVvK;=%)7g1j;)U3<2{9)oIG+4)!Ff%qh=b%|0k+V9
zz>gUmPdR*RU&y5L_)6Heq&m13=fryKICFhM<IOc({l4=<?y$Bkxq00n_RrM^*G>k`
zy&HLx@#xK`GjF+a=E+Q(zxVF#yDXEh@c*qjnHTXt<csUQj?&{#QsjaHT9z!Gw)ASJ
znrhr5-Fdu4eBv)$mUp`GZ4=r&=gLtVKFN+QCNo!`1G8H12t>|keKGN%#*!G8O>W<K
zLN`jDH*s{)zTtSbH*$vPvGtDp8b`f8ykY;7d;EMvh@#f9P}d6&V`O<RF5tc=AO9je
zGWF`$rP56wAAA$Nu{+UwPwMGC2W69<-rn)2t!3%_oiAdUJddj}%@YWG^l8eG-U(3=
zHEgkOl0xRNi7$!h%Dv*(5Yine!*I#7XYQ4tEg~HrH?1U3Ng1(9PnmaeW(&)#nC~W$
zZ)dht>hjh)%)IGyByuJT*QG@VO3ywnIhf(oU##iYsdDk#6b8}6{|UjoEtRq{e|YAs
zj0l()GvRc^q8MA3Ylb0Vg6AGQh>Yhwdf#A)++T&sTkeW^9J<Z9@V4SL!DGC>E@3HO
zT{!JdI2!Oh?wOH(^mOcsKGncJ)!180GWi~vCVC%x@@(mmIW_^=OK&_eJfV5fCBUF%
z!33$y2iIC!+9cmii|aWQ+7Rb)=bS{yVuh<wvX5plyb#h{z-FY_^1UE>hvKQQb1u?*
z#5*GVHMcM(|7mq_Iab2s9afqkzex1c*>1I4D&f2Ce^rfe-ZuN&&giTJ?#VhczRO<w
zagx7i&is9hQydyuid6#gH?TYL%J4F#sur>QEfnw&i}BrY%}thP>KZmtj#J%=|4)VA
zn!&iO<)O%fV-{Ww9X`QVZk=SEIYG2rq^GUNgN0|=;`#&e$=6QTpFFL1jKj6!Vthbg
zhJY-`ue36W=aJJQB_G6mDM=N#d(M05CLdRn;?D*AVqyu47R~o{IB3%ut2D`BgS<}L
z64gElZC#gVo{E{NQA>|{R{PDWK9!Og86|$qC7>bE&uLCo4X3kB&}NbO3|unHExM$i
zvA*BX6x7z!#20Za<ml9tJI*%_HGOHxG>eVn2wuG+d1g#oq?6z76Q@Hph4WfHj2e%e
z4Uz7-!R02?UwXIe+mz+HdLc1wLVsM2teO)Y%N6lW_Tlcr#`sgWzu!Hk#U$`caDttX
zi};@Zi*~PAT%6}~_2DARSxbVyD8=76VsNj;uVTsn$}}UVjC7va$)~z@cq;o89FYn;
zWqR$fg93;C&vR@W7UV5@e4vo+)(VAlOH>>Bo?X&S=P7Kcnaaeu$7SOLL2h66D~-vu
zD;{)j5VQ3<6B*`iv{R<^L}Z-F%N?03=0s`j-#eATBYpY=*_*m2Rd}xD>TupZJ6q*X
z<iVIHl{p7)+l1XMbVzBOyH%!lcB=dR%HApGlpf}`1O@QTvyiJk_k4rggH0c%y|HLr
z)Y><-tB)bzKwTB*v#ov&Ed3X9*k7t<d;Ga(qWAFam0MvotdTj4CwW>n-E`7C(Y9{g
z>rDavie+ch4K9D(@}E&**TL^MUMJ4*Pk6yqwNYF{*UfC<iiLTBol{n<pV_tSnu1Q1
zgz&wrRIlSMDIC6cqMJG&B^aKZQZfJ7qD46?lD6{*v>Hx1I5lB$ZiI$mkmrIo&swfE
zyST8mw(py;<YC}4*QGO<Z0`7ekryv{mYyXu&*kXG(2#bA=TqXmTFn$fcPyK*^W_CA
z@$@Y-4?PtBb%OQEw`CDm&K%X8Izf?l>eD+HPF!cT(wgG9YOm6T%_(l3Ilet5EQ^kv
z%CqyU6k2;HDXF@q(<v(b*1|=uUGqiu+!eUTwkGR|KMQ+qgy5=)Y$ARxRns_U8q8aI
zcF8Q4yk%Dt!dP6FCuT0c7I`NsqoL@3_sSi&A|1RMjgG#sVr@=4){xUQecj6mYv-J7
z)BId}%yGig6?-~*nUZ2l`cE&N{E<aAWsA4uMw!(L$4VOfrS83U-6H;V2BTWu?Y2*?
z><3f4e<&Z+)seAv>4|!O|6unn&KZ+9BqmQd*gr3+Y+lljUzfKuteN;pI;qTMlHSX8
zZJ8<8PJcPJ$cb}pMS$kNSNa_eF==*pZtZcaesT5s6S?3eFV6Hl3pC9=qBCV-N2a&W
zsWrKdQ8y0H%6rT9=jBAtBNtS=K03Ihl|&w!crk}<E?d?_!N}f^f=Oi$+qwDA#{NC}
z#`3~Xo4F-_=knezm{asL_mt+cZydXmjyuR5nJjp6Iotnh&PnRG3!f*3K3Ub*7i-1-
zqC_RtvSqE^8>NX$Tnes<3Ex})D{D!h;%TwfHEB{QZpxn9#0;&p*#jDv>vackP5yO)
zHD-%TlUpm(q(gnYEd?`=NVKr7p13+>mUgI5;~UN8>r>gQWRD$vm}k7zW8=m-@pB}r
zK5C0UNfcST;tEH!sjRmO7q`;v0;St(n&obX@_HV|J?ark=`_{%S@!AhBF@-Vm+sUi
zZT+Kq{FVQ{pTgR0YdmG#lcZ#h?0dIKB}!X+(%*@lXB=3knz^+8^GLt<#_jH!v_65X
z4>?YA{nO@Lp2Rp=$@Tc1XRd2EI_a)>>AdCe1Yd^NH8&N@Pb>;z>vs6Bwa4M&BD1U0
z9VP9WbN4HFb13_s<8_lZ2;T2-w7%6%nQK9BhV1j!&&6>wx*U!_WfYr~lA~#B=$CUq
zB*$UF0WJ6VX%bO4F4jDmQFA<CO_!Eu;w+1}rMZbdQ4hWB*9CK|w$i+nI*naZvArZO
z&wIxlo{!=!aYuDCwcV%W&CGI(NIAA+-MbJk1z*j}x=&<wap`n_SsgV?V_(zdk1WUb
z=538SaAF&$uZr2x4+?8OP5pEyNWUY~t;cEp;Z0wI_i)GD*4TCSa(+p?<|d^bXTENc
zNY;NJ-O77JyX?5*M(#xO36a`sq6~|<)e4N&H*8dDVY|7$)6>{pw#Tqc&Fr_%b*KNc
zeD({hE-47E@NmgEb5XDR%$q4J`9~#=wAfa3O~`Fx;o-KHZcB*hnZmp3nvi~b567aS
zo7Feg8BXN7VxncY?22or>14w<s{b}PJy#1}^K~|-!qT`5f$rOd@<*Zt`KlSX%JUCu
zzGX>yn-ZJA;j-QL?$K2XKeD6>78+>#r1?&HS`g#4o^Qjnjd2$5&Rkm3{LcJ@f%nt0
zz9%0~O5J90I4b6E<ejO%J-8=sX4AcV&1s%B*^vyNj^`X?=RUC0=<7xswv8O8%SC6K
z|7?{_xq9W#v*=}A9eXas=$`t#Z}p!nw$FF-#ScA_w#;$dlwT+NvfW4^=KUi#zcv0X
zto`n9tOfr+bN^ZDvE_Sfs<ENHKuXs7lwCz<bj<#D>A3IsX!1CxweZ=wD3^3&*K}vy
zV3DPFn0i8|6njo=V)<~(=C|uH!{re%!f^qQ7S6Qji+%5;)4NDO)LkX3SgiMNs5!r}
z$-?fFC*Gxa{J(OdD0YF%>owc*pJ}C~FL3cZefm+=?PrNnnU;D_wjG(i{_-pdm(6?@
z^KG=xS8ZvXVEOyZyQ8_;Q9Be17P{7RA80glmDf+$e}SvH?TFsQMxg_8VY=J+)3av0
zt6VpQtL}@`_Lhp!)2$m0hA(v99`%@m*VFp$G@~eA_CCJMU$3Hn?Nnk~dwT!LN3Z7@
zEqc%$<@)VkQ01RV3x0Y2zmS?YWzY0qU0Wl>6_V`Dk9NK~S0|FtRPMIMz94YBOT&x8
zkX_%RSL`fG)W2227<;#3zqLn?(dPVZkK-%mo|jY?>&^YxC45zdZMoJL&ShUCBCRub
zq%J*G82P!?Sm{)+^K_QdSPzq$8yk+_m|O17-RZvSyZfuTskUXxo+rX$*;6)c5p83s
z*mGq1eZwWSlQwZ}t$%5AZrY@k+YML!J}#Kq{(R+*u$rFhvyVt!x}{U;8o^?kdx7=T
z%qc7{N(`o+-O803B60a^KtRtOZ<jrepXX?NisE0F!m{Q`=IO^4zuru5Wl8A}Jg;?p
z<`>nj-#1>wN$=dg{``qoqE0*SEA86)e9r%kV!>(2-Cd_P1r{$j#+2}^^X2mG8js^P
z@Lez{@UY<UI#RW8vP`JPY<|xJ;ZwG~S#&<+&$bDb0T<SUmaYl!{rK&@9Lpl1H_|hl
zB3ZM(pV_=wxpmzP#@m~N--Q1^k*ptc(W7q$TU$kkzR{8gMJ1_P?jG#FPFW?EnmlXo
zYG1VRiC0nL<Ettc|7~CKnbXfod&@S%=Y?)Mm6{G4gbuhlI5IBg;tJ`wkZ`DtpLK%K
z4$dT3#uSz<lQIO7ougT!w@%#D>Lc0B<YT6jeCe#Sp4!_5I>oC)4;o3tb7cfCTRMAc
z49kv9mrqU8HjlX@A$s9cpYL?F_cL5pU3qY2ns)5YC`pn3sR8D(L1J%`3fA11oOaPp
zr+QlM;jOv%!&14oDBRt~^l0AGfQ`;a_f%-#Dm^W>CStOhH^W_}gi~$Df_4-rZf@m%
zsyTIM%(XK&9!xn<^qOmd(gW4IMoov?_U@kbHP%nvI_`<Z@6s&Bt<NV-la<PNqA5Iu
zL0qnetNrwn+gr?Td`g@tw7PMc4y$2Vbd$h(`!jJ97cz9^tQNl$viaKKsI_W7fxgcw
z4?nN=+^oaA|3S&xrB&Osn&bD^nw$0>I@SAf+N)Q6K^_L4+MBOQd29ak(m336Sk9=c
zTj)>m;pO{w6=kjA_y6Us`t)2-@0xG7j4o;l$v+CP6#x4wK}YoDFCCNRC;lrqn(^&;
zkg)OovIz=1nnVN~I@?4VdW6qRTe9Zli5_L$Q*yS#4el38P8XeQisQbhAbk4PC$na&
zmyfm@vnov3s>k}`h=S?Wl+C)P(oLUzx0!W$=;>+v$nE6($6~a17fbTS1!h)G7Z=?)
z<kh$Jom5kaY15}Y-Aj73dfzHHM`gNp@!r_wxzy-gf$F{>JFi!Z%{sU`yUlc3uJ%nm
zwc+s8xqKy>D;J;2csgyqm5#=td8aCz4^KClvPxn84DK@5Eh4FFd-l7ioLMh*ev@~!
z4)<9x@9r0i&+Gk?QYt^6!6>Y7>$1e-1g3KCSq_Yx&wL*!R!eg+Ec&80o5%R95%-PF
zTJQhA^4)&VZ}9~)XURMdzvde?yo=j+uJL$uQ>82SQB$ndj=T1v0(wtWZH(7N3r*h5
zdACa<_eN^7R%*qRuv<4pmiSp}pIl((<>_O(wn%Ej_5YWibn!&oRcL1Zy^Cp<>RhfE
z&lgLVF`rtYcV@w<C7a6b?pnQJTdT)I{rG9y-fX@Y=66IW?M~O;yH#&CF1Wvh;oJgc
zZZC&?#y7r)6!!NYnZ9=7v>GF$Q+(@v8jdO$S-fdWf70UgR7tMb%3NQ4bB5XS*L#$#
zE-(9IrniFO$`^;rvmF??&XtMmI-0!t<f1T%TY8t;M7nM&-3`CIV&Mm=Rh>(g7;Q3r
z`DBLiN;SPrS%+Uusr|2;sv14bT}pLP!`2Cw|JUrxS~;c0D*I`A<`<*2S>ev}6Tkkq
z+PQG@U2~DUhy1*GP90NY2>r^*lE1pXLuG=#Qm9IU(2S=29!`>%7<dwjU-$Hu7|*(N
z<47$xx9s6VOnVYCdvbOh^mU9XTdXs;MQ^5LU%@noMpcgqy33k4LKn0N$@Z>LG`{j6
zM{mZqz9*S!Og}s-Ew;95D)?7@yLrA|=o;7in~qgpE&^^kM}&+@92|YEy*QbkEaDVh
zqQ6RWeiXyyed`!n-L9-UAzrLu-@Jy$t)}z7^DW{1hi#s#Ssmi&uX!@#il$-%lg6a`
zT#Hz0e!i_w-za{MWkL%JgOF#>n*SY4P0o_)iE4$>1~>bcxNP`-vNb8$sx$YL6PNMb
zWjcAUg1lC3vz&f&rCoUwhj#ZhuSmO=@;%kPE;Y(B_DlUH>v`YW>>)A#ng^GXO$&#%
zO3=#JCC{zXZ!Rq7b873op`*O^#v!qQ(5+8yH1X_aN(>TUJ-1Tlbi10Ob3@a%IVM)!
zrjMKFpWbD<HhZgEt&)vo?Jp)J!A&dbj#ggJXHn3v`ub7PbrXkUc=LrVnh$4j{F#1$
z<=cVH2IrC|8J@j<W2awt$t2%Bw^tl(N)@^=^T=Y^;yVX@B<4(DRc}>ioa^fKaM#^8
zsWnRkeaiAyT%2^TPw{{<>*^JezIq;vnXXT|=Kk+lWISJ`BUofky4z*ZtNRyhw3OQV
zDwO^30s|&BRl!{+eGP(+?|2#*z~qoIk?FYkwI97}%cHZEw3$x)JNZ@RThM{jX<In;
zY%&ce#oUQs%_84?^Go!+8jq(8!eJ{msrly}in(Yb9(m%}q)o+#R=J!Lk&Md|TcE$X
zk9BA1ie-o1q%v7@8|5Cqao_iHJClaLZi~e3H8+w}1AKyioTxCXT-NXU^~g1=`y9tM
z|M;et7<eU`@AITbI_1p56Q;lV^TM6;!RZB`4`m(ZSaqSv=ey1yuLsWJ5`j%$Hz|nR
zotCM0>fmcJ@%H&PI@b0#rvAv&Y2J15!M0@vt@ay3rWa?4xy)Gnf3sv-_WDJ(-dXEU
zooNnuu9$LzMLLUZV%P*nuB(g_JS{|~>FxNuayHA_g|{R^rB?Cg9^nn!`smTtTCVI~
zBgID7k2_k=dO3Qqxh_#=UwQ4CLXl}}3a?V-9|8A!s*|cyFFg_|m-Mb`^V+32qmt8f
zSJ%O|9A2f<hphtJuKT2)j#~F-Zk$)7=~K>WssHZG^qjKXSTenT&ex^lt6yE|4sl<`
z=4eo!v2uCRBT1#ROEMe<4moE=EfGt(aX?Hav}?_u^1ZEBSL~E3^Kb4reevDNS6R0{
zPdif5$X#0)KL4AzQK0jY1s6jd6us|Cz132`nxnbE(QEIkwiCY;g;snh7v%_K`(Ko{
zi0$R-<9bSxJ$~<2$+X_sxI8DKUGesYWB+7NTV30+>rdL!^Qtplg?!JY@vhjbE@_Zy
zB9S-OZk6I~Nmj`Yr9`gcMu-15PfBiFl6Z}0+r_WX-flj|G^IxSl|g)Kv>daSfz<4+
z>zucJkL3Ah-unElqGMji!SBl+H_dZlnsRY<+}~T_Xa4Ne3Saa+OYWMgkw>0+cWXy-
z|J<D~JzO?z$Vr`{TYdL>z{`?7DG~l9*QBIQzRTGa+~GLkq(SngX;rKnSvA+)e5)TM
z>T0qmYs-73W1{olZ-1{7o#Z!V+WOdxogXGOILMzAYgS9xC3?JrWzvP?DlQIOOcE9h
z#W!9}*Hrk=uy$JY+S)bx9V!bKPEt8sSR!k{?$oj<;2B3%q06>r``C|#WqYO`z4jyB
zzqVxw=eei8$2YebSlw0-)O{RYlfJEE-oYCiIShAxKGU7k#Pe35;dQh6B;6FA78y-W
z&2NV{e`|PqbkoBhu4b3mVz|tYs3%H`>jq8Z6Im5?M&C~2anZtCt5u=`qKaPKI(MA0
zYi_>h1twRsg{>NYbeTLl97M!kertA7<Vsdx)piqPY}w$f%;-}ZtvjjU|J|imIRhFT
z*|ky{RF3U9t-zALP9pE&1Q$=GW^RLXCsZ=MR3;?Iu`3B=2QFFrB!O*e%u79^<qDrv
zSUj_*T{+_Gv4Qz)j_Lm^YqU(bOPQW3%B1GVHAOUdGB4D%tmt#i>J$m!XOUmT{&7v2
z5|g6aLYt;lVk(as`qVU>9?g!?(3lr6BkxdP9hb_S<1QK8ugo}=G~T{C@YJ|z^{O}N
z2@j4cd}H_QI1<OQ+~eIk#S`ncMkw;JHoWCa;$GsQ@Z`L8*21?w-J&K7#cnl<b~tgd
zY3}CW<9*n7h3C=LZRf9?=zHVzEO?r>(Xt8ZB5Tz;4rwe~;E~bm!n49i=9E!M?{$xh
zN_D=54jwm-Jv;2TXtLk2oA<PDTD@GXGr{$dHc##=a|3@)o5Pd5PVAFn`u^a^9FFx0
z3U9x!NxpNdiR1LzgB^?H<Q4^QN%;0Zi%}|ez1*z^;S0T0KHVZEDux%<O64w;wMv!A
zJSO(aan=Pv;jhWu&h51i&Ss~nEYV|HQ|6Mlp<nRwDuoaH>wauhYIvjaA#GyBLb)6H
zJg*XY{sk~_6^Yz9$bUMSTj-PG^8%GDwV4jBx)PjgA9(RD?taMk;s3w$*VGo|NgWnH
z`5?n!qq?7xTFfQU6AO67-!bs6yP}pbNhj~|gg4i9PR?`V5Z5m-bMrg;=nRYeB9?<W
zn^aC~sJ@;W@If<OGk1rMcxh(9g4Z5)OW!&q8_KXNPq-+u-J$ZF6U%X<a-Jr|3O#no
zNg~r;22NuBcf&!cYs1#M6I)fBxQ|R!jBe$*U^MgpBxYqVhKsKbihC?Hi0>~uSE#_k
z^z94t7X|+#8(MY<DPPukWTLb<wrg=e%c`qi_^ur~#~XTff=Yr&jw08&^<rTQ6}T6P
zxh3-4SSWRLy_{R4=#504Ee)b(jUraA=36%j^JX@j(i4(p7KwkF@qyVs?nJ<j%xfGO
zg0{xI+h0#`__+H}ccW3W$jR61zhxR_8MN@N+is;}u5n)FQMXO*1=SM6QZ>B^en|;m
zPVmSx?UG7T(Q5kQcgQZ%<={JGiRkxV<(g!~pG&enFnzsH%^)c_Ynkl)W}an-<!(JV
z_~zjYwzYDPjFgQIiM~<HjB#<B=k=2($$4wv(M>7eUvQ|e$=CnyCA2H*?86&PuGfoA
zW3G67YJ2}Ck$Vrnl;>tM&qkg_4ocAnO(Gi)op2CPJ(OQo!hPY~TCcYZFQ5LeczL0s
zC9J|pw^vV%=ct#ild!+i!yQw2&uXd5PcTt3V4A7pwWft>v5$9Hv#8^R(@T~oH6$%J
zS?>4cjmilgEn9)JM|k`nw*=ha@rZeLHN)4=JLu4>;&QhQe%gn&)O;wbTB~U0sBlZ+
z!k)!yD>_Bb9e&Fa$gp6en97vG2kT|FFX9n+tHl1+=Gfw`73Vk-^<3qi_X<2N=s6r~
z=3xB%=q{;`)>9Av7rJmQr&06<L-1Ay?hBKYCN=OJIsbdhC66sdC$1PUs5>zIZ=0+V
zw}_|WnSI+LrI>?hU58#Q`k=BxB!)XlqhO-TFFs?wV^-$M*3wL;b{KhgY-GFslIi8t
zn+b(Cvkc>tH{N{GeN)QG*vFmm=H{F@-brGIOWz3F@6J_t!xI>pVqLaHZGoYbxg&D~
zqpYNn{?eEFs!GnhO)GO6!($rNf-Z%<x)?Ht<Mo|FIm1V}N-k;vO72a%$G&KIu&rz0
zILQ!w#k~HE{Nt_@+pes=^lo|CmbVNmRVIItU+Jxq+pFR~$=$=jDq^YbhAT=Nwq)IG
zx_IoW!n!g`9hXP%4jtcf<?Xfyk7NoSU-)*t>Y{tY+olfQUMnyEGjBFVtn>ZSqBGSf
zKl*C%f4>h;zfH;iye(;33wMs5N#o}FCw%cL3JhB70<{m7hX|->wFc@Q+A?F~`5ljS
z%SA5ed{tbL&a*^8|M3Q~EXN;F-sam&dy5W<c{TD(c;stp&@9KkbKeq1Lu0Rf@50VM
zy29jPS8AGDlk;Ausa$cRN-Kwm_oED*i>)ckHSHFKgdI-dJ`@r#O`Bnf(wzfdo6D;f
zZ*<^Uq`AY1A!jM?6TXEa-;J|*jS9{dUYV+QW9tTq_Sgk)%dek2;&#UR?C0)Jt2eiO
zcWw1ac$~$p@x5tPa>Mz{g}0k@e{NPw$TvM%D6Fy|+4zu{Q&}<ZAq_9Zn4Cw_bCWNf
zXm<%rTC;6I8n@~1|8L&w-N@rUc3$Czp5A0J!P`!pUcT=te0J}+vUXmvY1fheWfGHK
zv`+u<LN&?Db<4E6%NZI^iaq7urRz5Jdnm1!Sw3k|%E_LMK6fX2zeo~OK0o=IQg^Ac
zUrf3GYcs9)PZGo4X}rDW*E30T{~5n1zx>HXJ^7FQrJpf~UQ}Rc$X8h~Q{t1}4F(Qn
z@e3)>6jVfm4x1{sgeXNl)qlTenTAK*-J`GPafjP~j&yHw6;@#CyE<!<1J|u{ll8B4
zOt+|b|0A;7Tw6brStLQV;+SaB2F*N2h6CFbSD4qxZ(k|#mUF+U<<9TBwv}>Qd=zM%
zX?b4BQ&Ow(k7}&@$J*|Xv5!0D{!1>4ncAzHV5sr;>#Wp^aT_aViNB~#o{}iE;Lw)G
z<xh(ylur>^_3>*1M=+Cys;`s#i{v)FEXVY+7HQ@M6Q=d|u}tLiWR&fE`@?8r)DGF4
zDSW+QTv3x{YZ65kCFn&v3g1{DC&IO`t!LsF{`SA&Zr^YAAG_9HQL(^B$V+vx%9_y5
zDJvY_H1Mc7lx-;cd?hnGJo)yHX}VFrRJwK~ZsS!;6?&BQN%?G+%9&|OKfKgWcN;H}
zjNe(A?QyQ@f_h%GX4<l%GoRk{%-)nCw|c7F>)x$WecONaG@muRb?5=th5Anx`TH(8
z{d}XcD_OZoukp4}v0{=42U8LMqT<&=6OZmR`tSUsF2gj1$*aK1c=7um0>9I^ZyglZ
zVB$RH!0;zgSYVf!L8(Z`RMTijGxvky5}V{6oEPhvmY6c>(Y()*&t0SJS!YfakLFdH
z#;r7Me)#m;4os&WL}%Yz8m$~tF1faF_MCmw=fq@8(`BmK`F-2gv#P0)OC7DPO0TUs
zS8m6D;yuIrokzVFd0hMu6uACr=#=ZnbLJ@RbK|iQ$(S+wa=LEQl%H)Y8cOG=v)YJ#
z)9Wx;uJve+wzN{q`8P+d7|JZpG`DI$ZvFPq`KxlA`AQDLEC=MW9EFt}6c&8s_e|W>
zJwbF)f@0)Bxh$rC0-wH5dho6$*kciQzDrTuj~B0<Zi#<?-0^?U_iecgmu(g8+{tis
z<L!?8rPKG#><sQ*C>OAv@7VflX2*Ls=0v4U$$MhNUhdQ2B<;KSw3a&S`bvq{hka+t
z6n|6pJ$2yZW~o!r?@so<I(53?X@IBJi${z83qE9<oiD~IdH8CBV1=o~)v^cux?<8n
zs)tJ0m4cd=&zcpmcye{av1#EK1=8IQiJ2+3ia%c@=~=jUz1j^&hJOWmf<-(N9JpUN
zDtr@A{A-{%fm8pyRNH^|E%UW4mV7nxO6K~h9DQ{9H1pJL3*Dw~{o2%gh_7q$^x|qZ
zMWq#S%a*0z++Myraf3+B%w&xhhXrgmDtt**xfVY2`c@IkHod5A+IMbuY_+)gf5NR<
zn=Nnsyn1sozd~`x{?}_HO|`5=zUhePq<5{{bUm`@;=4cR&QIRozu=h`6N>`ZH9f8^
z4hMXLg^aGriCnT@+i6-B!IhE5bBj@c_xD1<NC8vD2hoh2($|}sRrZT+JUrX%NW6|}
zWNT=I(6aPfk5Yf-<Q%og`}JsV`IXGwN?E&>b?<l*e`{H0)RUv(8U{^!gik784Vb8r
z$93vkj<(mN77y!#8$LyCbjdoRH|?+AB+bB=CFk4j<u7$Vbz9}s)Jvy!&RRWpU-6=S
z8EyYo)c!hmQ;Bg=g2!zoUH^XF0R1=nZTA)jta}+$rf@qrZM~=n(@eWVlP~6RZ(%6e
zo{$!0%=KSnr^D@avK1HPqCVd*E#|&q8~%CyTbovqA9h?1_@}rh9^_7n2#w&<iAwJ_
zPw|*-q5UyxXINRvUp=3&9l1M|7Hn6&a;Y__ipk-<P7>1&@7BnD*RO0pdo6e8<LRwO
z-#cy7k5Ea<6|6neS8dAhW~s{Z%eBn;2}^dz?pN)(dD1#srl{_<;dQ5bafj;;@4mM0
zPkprV+qk=5cHHkbUA4T)Q0atng^3gQk^_Hh7RsKPFY+eQAu^cz-(IuWY*Cg5xg!jU
z0(VShZMaOLxfz<yXoP(|;Ss2yu+XWMgH^+Gg~G{cx}4@RQ<M@Hy0vhLcpT73JR~wv
zX2%qrV6LUoy<E9B6aRA>FFP~SBs;6*s+*JKY_8awg-@4-NHgX}{ngodiFYP%O5CR!
zF4HU@^R;FQJ`V~Kn^gRF>V+63a!>b-(NGi7PH2wU7|fFM@WQv;=xH%51*^DAm%Hs@
z`k|%LJl)4BsEYN4@5&84>WoX1<hGrTD135MQ*-)D5y|3H$23hB_ni6pIqkv{cjZ2x
ztW6D9jE&D7IrUlj%0e!8&#NbPl{tm^v#N>Q(fY1-kSm(g;NGgxDQlx&ZFCY!oNOPI
z%yQ*YNcR*G-+!+-6H4b!ttk6vvN6frZL)rtNb~Jea|O1!{q&UD<fFj2zlk%*!|&lC
zSILwufo=gIve$$Cx9ylTLD$gm#DAe&CI^iJUOYiF1X?B3c3kkXiH&$DyVmEybtikx
zt-+G1Hv+lDg$ooI*o7wqbV&>61RT?K@N~G78(^t=K-Rc&hQI`)pHC)7a;84HmA#fz
zp;1V9YH0Y>|FI{JPGvfyI(bs4)ieoz)pJ`U{8O_^o;hV++LGqfUCJmqC)erOGN<Bu
zKSRTKesTyG@8LYaHecuK4uOTH2h}I%a<Q%o$Xc=`$X{1VRh?0Y(<6~PU+Ai_n^4Y`
z&92sKR2I4}Sv<j!%XMo5`zp1QTMwO*2;?Yur6zgOeX<bOgX{72!7E&uBoB$Xh6Wih
z{+u+?NOiTqL|Lw?+urBwUK|rK(E1%LnLp9-zf+HC-<HYRsS&3ih&bO2yei`2XFYk7
ztK4sqO<as8^V(7szFctfe4uq8%;4(s2{rP$j$)Cub6uZCMekx&-jd~X%(GupYFS`n
zYvardM|Udj3P_%&dL%N9sif%U3Dxp2sbsBonOkX#*76je)`*n}cH=PBbYQS~duu(*
z>@z8MeAZ}qtzJ-H=WgO{f0!jE%X{+2<;{+pFLjBW*{Yn$68&S7r^w<PkFv5ndmK$Q
zm$+;^x-u$fq6<rgyQ1rDCYKG*&v8z$n9{!1;>32R@|+!sPLB7VZ18D5UTx9N&B-}A
zaNUU+7w)?<eNEbtTX3;WTg79-PFcO8hd)`GeohJ7Vft6(|IYi4%$kpOyGprRZIRey
z!8k?Abmsj@8=mI<p7hsgW{XJhgvKzZNV)Z?pG-c6PLDMCo{{nJlD|H~9L~siTb~>G
z?+f;Qv#eQ};2x}B+kf?}%ZkNow>(ZLJQ-M`xm0N5@vE0UsJC2vAO3gU#Be4C9>H}s
zEp|LVJ6cY(J5<;nRlV|=Q&Me$1>=o{JXw*`Sz=Wh@>VY8-KKH4dxA@o<^`Qbl^n5v
zcgwZce^{-}#dtbuU(58Z2U-(jf;<lQr0>~vXQ`i5W84fQHHJqMoI0u$&+u!f%`~m_
zQJxy;D?B@6hDz5N6ZPnuC)d50IiXUm(|GZj+^`j!cNMZ~3u*E#NI10O{ePV<wa3j|
z$15hydT@m;i`Qte?ve!pni`sU{*z8}B^S@<oboXLt@jB=PWDASz8wXZyA+zTg4Xl<
z1~_alxDilr$g+3NiKyvYTBYwac=GXNxH9cH8lX18(maRJy=Stgw)YB$-aSE0LOHJe
zGnNR<xvS<StT=D#gy~ClStqNDEIcv2Nn)Acp`w{Nn}oMKo1!h0m8+P!L``bB_}+sa
zs#S|4;uX7#_y6X2kk%TVxbNRn0agvinJR0h@2R`8Cbjdl$~CP!UK!U#eC_-u2wk}&
z)+ID8(>Fj-Eiqjs^ON6<hzY9fbG!V#7M}k5RlniW&wFpb?BGmRdefQ{84>N%<h+OB
z#DBHiNfJitVQk9l(iE#aCm)r$x?*ackD~t+o}+q;CFUfxsm^MQ^!L0rbLPY|2DZ6A
zPBzR}<~RweJqhuUs64)Zh6@8@^vP!r*vz-w)p(T=8TvR+<dU16F0Xkb*DZ}RZCak5
zXRWT?bYkL|A@KjJ-#XKaPknTx3^)rGPPiYkWWhSG$p-^7N*#aCmgQ)fK7Y=MLUrCp
zY-R_nIL>Zza-TIpwxVRs+>U?<`7O-q%Yzg+4NpuADs0{zV;C8A$z>C_W=`9k2KU(x
zrR4^hLiMMVX2gGE_0C<X=(Fd^w0#U}$rU#{J!f{tOiR$;xlcI#;UopdAel)mSEqPg
zN?4?EFl4ew<NsH$3t5^)IWL$hK6iOgVZP{`w2?_1cjAIhJrR*ItuI?0nV;@(d9GNP
z+sNXua=EO{OtDK|Pc@<o)SGt*tT|*dWv5M{Sk1%~rN=j#^1rTVE{$5a<bS4x8pE1S
z233V=^Ci#dhWt9OWq)hKvx7@q(zRX+&fhLEbzWOi3Dcu2KINHrOahNhvh)>hmy(<?
zmC?^mE^>mhbIR;*630D`YO3#AVKnbgp!UZ`551)YCQoKeRL$gB?Yr!?!R8|ua~3BY
zUKw<mZ`zF{VQriDZx^&)pYfnwsZm)*MRj>X$<&2<YbONO@k*;K*uv=A%+lp?@&TJ(
zj)S@z=OxXlheQ<r<`&P?mieD08r#KslqFVQaMJD&>C<6Wt+A6nN4Evt_By8eX>a$D
z3A*2wJw3?7dYF4-w%+P9!Y{m5R<JXtIn`KCPWrY>|Cno<*?cBnHv8h6I-X`fRg_|9
zE!nusc%j*rzPpFb_*!Z?J&U&=e65nYy<m>{ACnM=m0o-%R*Kr1%mFM62XiFP-*wjH
zj4^YFTo%LUzUt0_U8)`jpFMrsw8;2k?k8(K1L>H^uS!CDpPg3c(7dLuH}gXaYeILF
z$11Ut3vW3^9Ax|0u}AF93Rl~zK>m0at!*YR4Z@TT3D#bfS{!pD@sQV<4Vzia-43ih
znY2Jt!pLUlMvYoS{=&;El&<|q^w6m~Y4Cq@u<~S{S3%(?u9VI)T=BZ(b-28ib*0)0
zgSP8B4Sy|s_MEx8w&`ncy8nq%VWVqucAw5<irzVP@UT>L_iR=3;OQYIEmn(pTw9JF
zJ86Hk)vNGtdvue|VV<K?CobHf*79yy$%L@wFC;myHMh>pIjk&W$v;VPD_eL>c5J$E
z@%C+(xg$-FCH{T2@WQjeekT436;+0*+DGJ>KAdxUb#d<JySsMo-7TB*rrG+1*Fqhx
zpXc)?Zr_lgt#bJAQAHn}ZQtK0m!Hxp{9L=pp)D*zy7BTqZvT6IB`JQ@9o5O6+A2?^
z7*E~&UDx6}TRkgv1BZq4rK;~wzWFdQCS@dE6qi20>ZSPKs_cV9N1J1HoB8S#k);kw
z+Kx)c1KHd@I?T9SrQN8^cHQ#Q<@mc!w)3Bfh!-hsX|dulNz5vhS$m*RNJvA;Ty9H1
z(ZnGBzX}X{H&k;y<mPH@HoC~aZ-K^r=X#@u_C-&PqXi6W8^z9)C-EeU8E+9+DlIW-
ztZfZaGjo;cUg&)8i-DG_gyA)f<E$yE9_rrHB!Wa-<2uxIu5!kHNSU!w&B3kaoM`AD
zqtu8-`L0JE?w3=upQK(nTviaS!7(W`&6#WKf!yO83vUUAm$K?sxeE&^FlsN%{1eW`
zQe>FVXtnl%<K0gV;vQ^0)AF>GJSLPk>IvzbNbweK$eZ=tQPuIk%}NpF<R(4CK+mqt
z3FjYK$V^By67-z+Oi@+BauTaj=M#k&kGPtZW4EwtuRc^LdA)hBd$0YA=Ia@~`_vh(
z%@DXXL*QFO^`@=fc^g@eB-`vURXZXQ6BAIozcBa~d;dobjfci|kIUVjhUZ?I=5TAP
zz@1c?dtoxU38p)@mCFTk-B@IK&57;%w9e=N>#L9Qw?`y!{}5J8kzliM6!vgr3Shbu
zq_DJEZ;^>&-U220lX(k{39b6B^ZZynW7(wFp^+Y+gvG+SC65=%eAoKUQDkHxr<Pu%
zalQHS3c)?cg<Lp=-8!cjEo{EKkhRp<&qp*gbXohK$D&*g#e0thhHMhpU-91{@MCdV
zQczQ1j)RGz&O#s8kZC^Y!Ra%6M4dyb9F3g6m-s(#D|nfrdYCg-pfuDYbw`?TJqrU@
zpj%V0Q|#8D<A%Wjg5^vfOuCFj)_l~^IVde5=Jx4^y3hg!lf@BTY*x7p;@_4EH@D~Q
zzr<!KsBo1@+3XPGUsVZxMu~=pit;zDrp?Sc7Qr@khmNC@_oT;MJ6~|E|CY7=M_sd^
z%y|yEx8hUoX3R8Xtg&Z~%@;GMc|K+9Hi6Ub0q39BTuc`}6I^>fxz23coQIO48v<K)
zZ(;rIZuiYYxcP)hzlp@d=dmXRqs^R!3$F?MzEoQg8u!>ORc@khS2$Cf*u+Kc4Q&(u
zH|j9ys9h9!^l75&#!0-3COzUT5WXNG@?A(cEz#^{)7u59hC3!dFO-umFS6tib~@>E
zM?=){2LBoZ?I)t?Za0cvsWAQzR6Dy_?8mjK?94f_o#NJ8N}4}8r#^CS|G7B!$K1pj
zhK6QKt_G`J``l(y7BWGwv?wFRF5Jjun?&``#o0I1{Zi{q4+?&o$dK2Z%lWZA?U%@_
zN)6Y`(%T%l_6TaE9d6=2SQx#)*dt=Tk>mWPCTlJQiP}<j$s1PdI)sg#B2@zV7CTR3
zyTm5GYBryd_dQeL(y(;?q$-n#I+NH$f<7$YU{;;$P`w~)j+A+?=JQ^uiM=L;t5mkE
zy3Nt@@4$a9-<^q%RBZ)6+D99R?>rj4T`}j)jJ^P^g$J`{UJ%Xo)>@<UvFuNTMl<sS
z8LJ6bk^^6*qy{YvH%P3wdnvE?*@BNHR(H6vzE5$u<Yf2#30Ib2k<xL#Ee`y4mW!-r
z2<%~BZ=Es4G;xZCmZ+zt@HZ`n&t0oHFY;d(4D|DCslA!Voj6rwVzf+Z?fz-SaVOiJ
zsh7O-n3i`^*wQumL!x2i3N>Yu65-b+fwLsSx3|S?hzb0)<lL0$fs(@dS?%G+gfqU2
zG<ukHcE*}Mls^7&S(=!M-i@eeff-j0Yx^jvR!p(4Wl;?e6kl1|y(wg+&BiTLpVsvn
zuM0SBA(q6Rv9S7Q)&KR!wL}l3^QU~6QyNjTBRE>-o1npynhVD_TuQfjmLajfsx?M?
zd%r~Zo$_|mhw8=L!Q2~X6iv5ym|pa5yU9oMwPuC$U$x{%PjHepU=7O@)E75+{?xYa
zx}Jc0WO2090vDUD5BUG=68IH0#l3T<*+uTZ5BT3)Q27=)?}bRXu~!nu=Bb)$t+y)Y
zMoS4FKWdWwY3g4NNoSE{%Vn+p2W_txhZGbU26w52M@gu9tDl)|yN!EUfM;sem(b<M
zr6bwG;-)V9<q`H`d2!)&<@U?I63^>W3YK4Hj^(pVmu`>`b`aD#9U?s;Q^ZJD>QQX7
zYtvs7&$Sb}>yD~(f2-90scbj#zgF&*Rr9rl?>&@V5beayqsyhk@ZdyI6<4~a0c-I`
z?<gkGy|<=1e(zh%eQ1MAe?Ve?YL(kbRSEVZb$iA8nl5kWTkIE8ARzQG%p_-f%tQNK
zO03Txt!0q$=f3K{o>^R_VcwxrI+K+rtZ|saqhsrnBRuc+dQOp+1&8dPrpoJNZ|6}|
z^@$J*J~VZIskH7h=VTwlBu(KP(|sj(Ctbh1(f3wTxW@Fztc|%V1gk1-gR_DQZm`x&
zlo!3a$wY7l=LECm(={F)lg{~m$ni+K<B{0e3W;gy-uBwIKN*um3b*NfIcmDNz3IyS
z{uW(bMT@tOg`2~a`BW5cEmF!g5c+?m$~f?U)_RkOU~B!pFS8|{?X_b%syJu;zYUxv
zkKK>BZ_^PBj5t`7x%%K1?ql5DKHIBZcD*t9Bsxz%@ledQc~8WJx7%pm6x(s_VQx{1
zb$!ZwBf-PD6ZfSWGTu`zezZjUwTl0W%l_<5!pu#=TPqBHyD;^sJFN>~3RY+;=0320
zCu?#7zhU6|>nobCbu_=*)%^a|8Saf!Hr!goR^6iWB!+9kTw%w))Pp;CrIoYSoZ)cU
zywb2F%}2fEQI6_v;bW(lOn<oK*y^Q)GfE#s?#>AkFR<wU#<=tQN|i<rk$0TlSH8K`
za@(f1&0sP#IT`37{rv3e-y&f*?I+!kDmS)zXL(AqDWQGWe{=7`2FaF;)g=x26P5e6
zvkHd@xn0_#<G9~+VwS<iYf_9y4!;oIwuE(GLv!if^~<BK|C`YJ&w=0KhV0)BTpH!Q
zd@;gny>1vhJi@VO{aS_9`EA}8CNb;|vVP7{Yh9Il_RfWwvZ8j$GFe_{Go*x7|IB^T
ztkuq9I$=@AyUFDNiQ6M}L|<K*H7iBtx2nXmAU&BSr&g~NIcRi*^JU)4GN(`6-n%7q
zk2`TKO<Ht)#dWT`{C_WSe{pEu<H&99!JoW=uk<Wm(g8lcSf?LWQ)(7n-?xBs-yT7~
zouWxr@-ol#V^<m+T2>tMbG4nhaOuQjGjflKMVB1eEl_ai{`uGU?@U&5X8+$_E;(Iy
zsbI*dB@+^tRQYU*S+S{}Q+TQ*11Ce>iPF7>6Q#4%E%INLz1S?k+&Clc)rppyQEMCK
zC;h&4Sp8h(E#sI4%H|V=*RIU)-RNZ*#Fh2X(uXl~f$?J{Bj>dz7*iEi9-Uz&Gbw9B
zv(B2Q*WLx~SKJimafvPPfydd}UCxKO7CCVL-o>{_f$O?9*Xj$`<Nf$=SIDj|xG^Jp
z*SW^Shwe<@BYY;HsAhYaredpV_HN&E!uL5@&em*tu9I5wq2pfNO8p>#+^XLqp3K_G
zGdfmtY4`Jb|F*nX_tKuLKzG&?#|6pUzhf$Y+UUL7c773y`0CD6qKlL^1|Q+SwboDC
zo3r%4@cLz}OD0TNTD>dDfa}#RuGoA0TNZGx-q3q4QSP21*Ce5uMGG#>+S~hD@deMp
zEBogNFg}oykd4VN6f+JHkC}Ax0{0Rl-SX9nC7M^?I)@xP;AN-lYpD13Qe^qNfU2dZ
zmO9&38Jtr~|7cfyrsUxQU10~=<u`oJ9#piud!>%EvHIjj{nLDNpXP3<f4S@5@4Q7<
z*!Xsa&k7auXpGPj5}C71@bB%D>2IF|WDCE3RH^Cdeey(o^<viK4XbMQa?MhBJ<*B3
z#+Gf-1%72Kfqw>EvG=O?n6o9H6Y5+m(8epgcGfEE`=77x5cv0iJILW)UtV^Au3x?=
zgVfS({tH#!`5rqa_g}R3+}6*F3yd0buRZ&^_-)@jU*Q#-VplZXntX_>k8g|Jq{EL}
z7H8}gHV}HXWr@)A)t;Z5zOS-V;rLqK6<R3PUMTgvX}ytY)PhgHUp4>N;!4`E{_BEu
zm+$f~R#?5*fa`U@u3hJ(tIl%Gn(%ZdkM2ER+tS=!e?J`kaQaPM%3Kw-oa=g`ySS%a
z&8cfuoThAhDRoba?V~-_IqIi(UkaRZiRs<3-TiycZadC%?m=1NWxjtR6Kmzoh23^D
zC0&*7H#j}-gqA?zk~)QZ7NN=phopoSn27dzIi^kQ*>Abh#qZHgnT!?Jubp;^TI2p?
z?@EsiYI8PS?^*n+Y0~=7uJ2`UFbe$5`rjM1v@^^$Y*xg@*WvyR4E0}FI#z|QpS~hc
zNIvh$jf;<uxBqkK`=T-F$?>Xx9-JOdK~7EW`pI=u7=+XqID>a^JXjRG{J5t2_5u~-
z5QdqP`A@6d-7#fJ@Up<gPD^ivyb4_tDD0tl`IX}Ai5{EnQd&hdXKzYoUGrpy*qd8n
zU$Zz@T+K^f&1*1K<Hw2b3*#9Zu9mf&+?Xn~pHpqirs7My;!Aqj{n=i6ZDO#V7HhlZ
z@zrT-qqaTSw(9Ds*Wrw7A{M3|T~~UvYcI=*Hjmx%y3K07-##DrJEO(;x9Vq{c?^$r
z-=^Oeqm{En9n1b&U7E+YadFAyYfG!|>@%#s^ya?#x7W}AdoP%BsB!YPFEiv)Cb8)D
za@{zzwC;kgOcU3G4~5SU|6d}sdNJcfrRK?7`D$J`EVqyhSi5xB4G*Uk7F_`iMpg?i
zteU$jXq9mPt_LaVf}SoD7Bggfowr@X<(*)6%FpZd;_YU$J}$g$rj$B`wNfQhWWydQ
zfwg^ml9q~oaa*N%OhPqaLGM|!+Zx7<OZM<Av9{QBNm<jn=<%uEO<z`Qj5g@rWEjn|
zVd+#;%dF-pVIGy6MZ>aIZWfD|EAsGP7PQKpN4{Bl{ru-F3sR?sx>&wgwB_5K6H09|
zulQ8lEsM5X+GtRlamn3&(m8)!v#6kuAoa-#4QnUdTDg4Qv8IX3S^Ezag#Blpx#eqE
zOmXU~qh@(3E8Y4UmKG`}Hc!2JETS>Bdu6I+lkk$THmR-3-%6%>aWYPs@W64BRB^J3
zf#u<xnEb74CK;c-!lHY2r-Sxc6RX=nzNTdv3)#)O(>~|Le>D5+q4G&j>abPclIXsv
zrE#}Cw$-iOx;1gq2JXJ?pUgTNG!L336)w(V&emi&@aA^Xik}|K`WOE@W>vW4Ki88K
z%_RwC0oS6~dp<mk)SnWd$*A(B(rJPGt)0&&YOxfm2v$l(H7sU*kn&vAlPh8IO6yr>
zo#C?2G>)&hq|`b!ZE;ff+Az1DyH4!Px|x)`b}~~=i2CUln@;z?oA8Nm-Eo_>8&6AZ
z{=M*j^b5aY-|(Uq;d9EbRo0%X`Xu6TdE4KW%N0IZ6>$`uC~lpx#M}Do_s<h5ygSX?
zUc7u7*Lg#ecjb>sv$kECEaIWixQg+K=8C^FxISnsUiHZA+8V7KY3+x$p9(4_Mff;1
z`35=LtxUSz6{i*dFh@PN?^a~=hCWLso#aUe-h0&atK9gzDt~o}@?G7lp_}KK-d9~a
zb$i(C@_SY9r>>UsTg+6kCtA>pTl1h-#SEvplX%(^uYR2FQRTQXF2Y6cdlg4f%w+rO
zgKiA_65M@N8fNrrEQq=*-OaK6)BImKiOl>b&)0p~Jn`qB=l0sxC*O0Fh|Dp3p=hZy
zE41hPL~h5_Oxc?k{<rm%Z{gwHsBB^*H+i2r1E<TuHkFT4-9KFuOFH>NkICSU`=M=u
z3WXCzQXU9uc6<@sdiGqY$waO@n-(?9?Z}KtzBOydIn|{HH@-B<t>~D;=f3>Xmeqv{
zhBKB26wN(e={NDOu!v&zM9!!Wn%X@nDvC>;JnI8|icUxJeUmHF2w1hQWO=rS61N&d
z^$+Ec)6Gh6?)*4*^Px-FnnSW-K4s4H4!(&v|2S2D?Nk=c6EDLq&Ys86!Ya0LO^WHc
zS0}`0zjVF4WTnTN&rd^dEN;+U@?J4rYR)dF;BJRQ-PzxFC+L0=pIxhJU_EuqwP!k-
zmbz;W&E1)lUhY!l|LMu+MnRKw71sX&>#HQWntwSjSnzJ<Ik&u1^8XZ^MdF?derny2
z{Gi$2+pO`><U*D5$Vf@v#^-^vPI{$nm3rxR<WTF}hiiEJH+OZ^9J@6Cs7Z#(!I_th
zo^0uBSRt%^x5ICGl1bpkUIYHb=&LFUt*c`;dTc)xJ;x*QSUU@sik*|!oFhrww3e-{
zR9y3H9m|9U(Rta**-ER!dBoKn?rHk&dl3`9+e^-up<VekOTzqhLUI>PTPA-OTFP+E
zvD>P1(v&}^WhRSxacORNUdo@F`qNK|ks;@3+1gtx0<4lmjQ+S+uu6Fg`*OK#k2<k7
zr`VrqO2U-XV=KC3HJ^D1m`_^!?^O8WjH6%WlBWN^9kkit#-oma|4x&#K2A&ampS=n
z!mOj$9`uW?x;;OKi)+_}p4N$v64QI6+)m#7lFYNH;`zRL)<T`BMw10|IxL@mI>~iQ
zyP0J!?|*@lbD8>D6927mon)qTM~pF?VbhrxC*OUZAZ~Z0HIPv=(BZW4?5->An|~=b
zZCrF@@(-R)sZSh2l06HVR;NaBRw(%_nXq+@$ya5|Pode{cTAJ*^bHHsRhIN9ytnO2
zlIqKsH?*ZsADeli<#6l6?)ZRKt*>8Z9OgeLrI}Nxe%z67w_Ij+tiGn`naFo4Jv~PG
zEDCJB4w4PC&a8g<V&n4A7NHGW1#*`7%nb0BHPUMBo$-Y2*^QY8{@=_pku0id+oUyr
z2UnQx@0^Ukf1a;i;<b|L*3yju%zG#M=m!MxhGqS}(N^YBoL%!p*X8PAA;F?ItDWb{
z`do>~403hg+8Uv8@Ytu=xO1PUaL(~j%Va#+*TJ6fYs!Mkmx3ps@Lk#SGb*9vD(Ah8
zE3A_CNgO?@BWw3<OJ2=%J|BU&6#0zihyykc8@801Pn!Ahje|<*g)`e^FGU>nIp~!A
z(mln>gk!$SVSC$&AM$3!_lNZAZ{GYZi@kU1RU_`4oP;gz+oD!)wRz&J_bf>^*OBYk
zv6SOdi*MDj8kj2B+qm7i?X<5wJa+BR&NCAGwQV11#@V~3tWWu;qV2G=tM~KyGyik!
zif37-d;D-JH`ue$U3bxfhh|D@4CVXoCX_WrX3OtTKCWCfUw@4l$MWAQN13;*FcVa{
zc5l_HfVUk-H(Q9UoF~}Bv_ZFL=RLOH+l&^?+ve$BF}on`zv~VK5mRk1UaLg+Gbg?4
z^6z?nig~y}IWWSH>-gr|+`c`GrT=5{vaeM8eLr>Qx{u+4BN1mS+NUm;i)!GX=r}1e
zt=r$gS#6cb0$%M1)>dvtF^6|ea-!#yb}SQDTY0hj-GkLCvSBaOlrEn8p7-SA4TIn%
zXL@Qvr`@;KcP~72@#<o&23Kx{ZM8mXpM<|8nHH`!iC%GcQ~z1-84R26M?4IfqS~3J
zu3foKxaR*9PZd$Yi5HS&nJ%<kUw_H|WUmwVvo-EZyiR^v{oudKuYBpUnnc6X%A(Sj
zH!ws@n!jMp#)zqGuh+2tUOC5d*Rt*dUlZoZJXC&SxG_mfOwMy(zXG#+G`CrTu(p%X
z42Ruuht~hL(lXQ7x?}dHi8pw6-JWx5<_wQ0*0vi{qJFGC70LSj75jaMzKCL@Wg9jz
zM(t_Nnj!Yanp0$omll70Li23t&i9OpXBLaS-?^oWX@bb7{!7doPCZ)D|G~`g&bB;d
z=LeQzdK?0=pLA|#9{&GBaw($#r=noL#eCUY94whqJU2D%e7G)s(q-+?(cExg;|}vp
z5<K17+;zLVo+aD8IK8<2|AxtzA9D!jNWWdYyiRfM&JXRO5gZ?wEI)U11U#A_D8Sr$
zp`&9)N1Fs|;0?aUfQ}UhEIb#s>O8Q2^3mGiGke&AJu6lm`S4Tg{O-A7Gq=Tf?p>Je
zkWyl>tXuGwvU$47taKSEvm;zWi;n3EEL*EMdrO3F-VrXI6d`^QF{2G@`!d)!c=K}#
z=#}nS?;Gs?u~TZ%B(8lOiwswYysC0^IkhA-g1vfDU*HN>C&P|6yX?b%YnMs11}<QI
z7||XW(eColF-m~-pf_Kc1oKtFR>_Gn4hLP2eYQ}D*{W--!mPDIg=O0eZ`0@tEf&cZ
zzS?@ze0ERcXqvTwUt-ci@lA(&4|s0=?{VyEGgmST=d$EoX)SJxWKIbvo|+S6yFRJ&
zwY1ivS#uXRO>64rvil_JR3iM)c%P)CU3%uUy6Oprk2;K!n$st4@#vX$Yo(iJPghom
z_wp%Bx(58tla9WA!Twxf!Gn$t-x${JGfO@<@%f1`yH@svEonEDX#M5g(VV~<cw_Qs
zWsWDmJw6rm>2wQzI>8k7;KcXa_O48x(IqoJuWk>t;F;ba>t*75IAHOdHA~n0aN@9Z
zWA8c3`dRg6wA1VfN|!mhU0$nh2rzkWy-#}Asl@^ZdpB@jj*!-HI+&FrC{nU$Zv{t=
zXOohN>7)s!4{r#c^b~3Nsot=pHSolljx%SX7M$_;FJyD)#F@GXcAdk$_Z=jpzZ`tL
zqpy)cMB=ncn1kgj7mfcf*&SYP?f>BK9cA-w!HhZ^*-2OYe;S!Gf0|!gIOkIWtMDGR
zl^*;z1vjO8E6x?!JNu3PR*elgzXMnwcJQ3`ocrXEiO*rPEw-X>nWiuah}g>Wr%KJa
z<lN^m%a@t6a`JY8B0)wDwdk8((TcNI882IsA?H@T%AjghS?9hjiL8rfthgb`ee9E~
zMi<j}?zzV&&hcHO@JZlkZ-+{~#?cm+6M-KbpLcLHePI78bH+u{_<4oQ9fRea8^q;x
z^q)U)uNUlSxiaP7&oi$-um@)NOqS_9bV|E*Qulr)KDj5o&wqHR{eOJseX-^4OTwR<
zIc6JglRiCl_KwY`ihL)0xFTeF?D}a*y@+GcE6*;Pqq%sGD9aCyiai(aO==Qw5i>kA
zHQ!}kLC&;wE7lg}wE6~^^;XVa{a|fn_R>QVipzKoh_omrZ0dervEXTh-UE;BpeODx
z14ODWoNMS=5*NX0xPbe)Ku5vR-kuJZLRP*XDeVsvj+!cP9160pFVg;<v)`v_&z>)Q
z4=+rwF5>vZ)A#euk*~p1Za272$l1j{Nhb4Q$BRF@ZKYnXl~+}AyGgt{B*wJ2SHgh7
zJJ2cEZSn7uQc)Wg@<fI{?G#&jSYYAB_8A+bbOS4d8^Zps5`U@skfrm1w%w)zwMqXo
zrn%a1-3pq#aZz`e0&@WO+P;ZQ+JD8X5?sG(ceJi(-@S&t)uVGsu4~wf(~Voj!*V)Y
zJXPLlaX*+b$4Hb<u14hY!6}D6T~0`D{qdx)%5;Wr6Gx{8_oYg{Hxf5`TW-4~N*vZX
zbNGr+e9`Tv%$(BFcfzV<Sv4FIQ+%UeIf*$5F4?nT(PZD7TYTf3ntB-+7&pY6uH{e=
zXgGH?=28uZN=L)9#MYUzYAX#4*ZlBe6gaoeLuL1YwHB6P(?cT4zI43kIprzS`J$&m
z`eyG(mop`2{kj&g8%EeaoN%J5!$W!wx6bSKK#3WbEjcE8NL+2~sQI}y!lsiov^!jf
zo4IJ~>;GH*8+|(-S1kCr=6L7B)1?y3*{W0WdljB-?ReoDJ;7qd&R^nx|MIElv}#C1
zarSm;rg-t>##~H}VYwH3oVBy>kn@~f&MY5SFgOO9t6UVR4dmKpE2$(UHrIJ>i?d+t
zA_2*ZCp-A>y`I(25Y+#{HcIx=#2a>fH&~jAr%7vi&$nH_AT(@^!2IbG8gzf~gco=(
z>sWMEDejYn|Gfm)Hir{dm3)=6qK>^ea$IregAc*Kr(RhiuwVSo<hQeZ4yH)t*FG{l
z(PC2C5hfzL%T=VTz|`oUU(XWulZo~XZ+uL%EnRm#vd?9YJ9FjW2Zhy!>n=&VvG{d`
z|8%-19a{KXbMBUf!vE8_Le^e<zA@&)Uk;Uun`eE)rcG$raPwHNMeG&}HTUc%&M}E$
z5B6?95IMnuo5_&ZcV_J2j2jaJyagY1>po~Pt>k32u$Fb(&GA66{)Op74cC=_dmkj6
z@i^t^650BqqGQwAj*}m5Pu987TDi&7^Im?bOf9d6L!igAt!ZsB?NJ}X+IIBr?(sO-
z&2jL^^alogseN1_o5gN>ob2$rR{n<j#T409lA^zFF#J*2yCxx$+1fdo=i(9xfjKg#
z7zA%xm_&-Urc6A#*XqiZ$B$H3Cv&Zf<lMGHYm$O;qx9){vX{ahDDM||w9$isQ-d{n
zMuu$O$$k$eqYEM@6I(Z^PH7fk`1`+dN>G<zbj!07FWJLRw7mRu?aF~`Yv#%XMqGa%
z!f`L6-J!OlDI(<56#t$rkK(*It1ASAW(8eo*?e=Q*wN7L=uaIk+ql@zEM=b)b80RZ
zM-SJTrybtUI?mPhFFYZARba>Sz}Uyt4}0@snG)wbIwH7j#<bR}U8jSYG-fuKDQ&9s
zVY>9TwQiz}g!P3Nuj2al=)EgFcsyHShVY??WOm&P90~K{U8kOSAn^RD#Ef1Qk(|<J
zE9>~a3v;Hs1;or;{_cf)e8YwBKaNg2H%)uW+!=dS=lDoAsBK^`OX%-Pc&zts&XYsj
z>kbPaO_;A4$(I)>VD@ghm(+w$Gv{u+C~>g!z*aG#|I!;L3mB#uZP>u*J(ErO{ELqA
zk25Z(2u)UNjh@g{8P~6?5prsStM0j}YT26@_V!hsXsVdIddXA476I1aCutGg9n0gA
z{9B@{H@>uc`eu6d70*Q<K7L@Y_IT;@F4^AZj_!xf{~to;b7#Z^8Mum>2QKXi&6N(8
z?VMhE%A1YJ+pXcmp|dN$bOiTt?MZWaK2t9JP~?(=*sy0EI~NGs|5J!Kd*#`lLl)Vc
zVKW%Et(>O${vyYW{m0W5cpi#BmAa&SpS7M}pRb_AKgH*dJ2<}I?Tco9>$URj|0!MX
z4_V}F<E=@#&ATh>_SWZZcO?WKy{&HRe|#(b&x=D0l3jNW-gtN~d*lCQJabH5Onl4n
za@k%^Juj}}V_yDTQ$o2JK3v>#O-WgDcKeS!*`L1aBRK1#-z#Odp1M3s#UUqnqMv)L
zitgF{d-iPg<6p4n2*>8qtzk7HU$$gwZvGIxU*+Sb_WB61tzRBrQn?d+l9MIptYh4l
zjdso_YY%L!`{?JMol_n;spEB+Yr~Fjcm6VcUpwQfjQWwwrY^VtoJv>ZvY6O?<LJaN
zgM(`MpKQ{-CZ%x&Dx1C#ntRAwV$YxHHHY{Xr?ob`kJCGU|IbqozeM%ZVgWU}9jrqA
z`uv_9=@;Iz+Fy9MqdH2X|2u0nL*>O4+*Xysi#L>Ohg^FXdHg`fInh%A`YocYe9r9u
z_c_njf5#U0Ap3HJb<4Y($~P@rj)~Q}E#LW0_E6qau~{znr_Riu{UPq*M2*sw0n3`0
zxi>g|U8{JoMqamXB2)aqb2+{qE(<w246aO=z^&QF%Mvc)^Dt9&O8C~^uO53J`yBjm
z;xb=Aj?B&#lh;eK8eM47z0joF!KAD3XWb)%B~~RX6+W`+F`vDqdhU}fPvn!hpk0f%
zh<*Q1*8ZA9fq{WRA$FBczFpAMRZ5puz5dw$LNxj1#|;PFCU2T27_;!`tyfQNI42)7
zoD|T+(ZRH{*R*Kc!MxhI#ttoMpC#|F?6#eI|KIJLXib%Vhxn#ix!WiBZolAT_g(R}
zA}9XGj|uk+e*Dj5U#--td!t470*mehrZ4rXnjaL7c1?Vqa^w5`ZpGp<Mvm3L!v8E$
zc(`Qw$~6M14FU=idK?&8Sc^1QZ18q$W0F)8IiQ*3%*eUBKx;-|qATNEi=ZbOMN3Yw
zczR6qSy`ldx@m3HzM7L>4^}kIX1TNH<ixWpX0CNxm}2^B>4~VdS)QI3iZp|#d2xGr
zC@oxeqVB6(r{K{pIbGi^zrG4>?FwBRxcJ7;#o^v>H8y6REqZgkMDOk{mCIcVv&z>y
z?DT&6?X&ofdG?3C=4@Sk)JtQfZt|9#iHlUb8Z?5UygY*fHaneT678A7P`V|fb*bgN
zun7s>2Q+I~H-t%iTyXGKYR=>T`)X%DcsFs2=)F4KJ&PM29G%Ui(ZSH+z#=o3J3%i<
z%Zo9GMPO^(hO-5`uGjbPiJKlWdnW7230a}*ryVHM+ioKo5w&^J)Ol~O`L9&q%-{?>
z_Cq-+aZQI2m+jvT%EzoYx(@a+{Jo$QW}mwD&^fJH8;ll4wDgKBaF8ovQSz*_-7%>{
zweCcqzRyqAX&T8K0uMU1Ce8@5-^XxllD+k(7v1*zZah9+)v4>oCAm0fqh|_J%(5ey
ztd^P+(*2&5&&mi7`uQa2XKh)$x7VBH2U5j09*N2+%{>x0E4$1*XqLjknHOABj6&24
zvvzf=yG!s+KQJSgD?!e^iEH{?j|ZF!u6yYJ7knbDy-kNpegRX1TF_G8qv{PTR-7LE
zF;bg?{KK~#dGS)?Z;uh1_1`r{$26F;HgVZb-V~zT%{npSyoV{5T7%lQWlOZ<8$`p_
zByQcXSR#34me9?VshwZaz2{{<OWitq;*zCaw-%Yr%H;HU7?thQ$t~@^aiU<L<f~QD
z(ld{4)SghNB^jRY`1#VV=N>;!-Ht6%Qk-yP=G{(rA?-(kS`!xPtqmy%eo?qwplYqk
zvI*5|6||OmO?PWu>+m)3^~R;Fp<Zt#9K%Ar8(S)O`RQ-&bK<sL-SJ6OcV*uuFMpX$
zht75Fdk}TKtM)4E77w?@63LDV-@LxG|J$(m=o%HaXJ6LI{(tu(`C2>gO)l#kiOwaH
zW*MAZQh$L<#PzrWH^Z{+rG?o`kEs-P%Da0tYO-cacbqoQk(r{c_kgKTL1=cH@z-ZT
zTO{41j~(cJGJ!!rGVl2uZr^q5-FZI#mSULwY<GAWxAvS#6<vHw1VZg5HU;@8$*MZ&
z&bE1)`a)H7%1RHRjKuz{hOe}|><{NlEC{gT{;goU_r<I~@3)t;=ftQA?T!<B)BB|6
zLL+aQ&ZbjLTTdpR3v!F}s$X<}?%NCLr>C%RdIadjuDj?j^mxa|vY7nd&G+s1`ki&v
zWMXNUf8s*dn+=MIOdT^E&K!1N`<1U==GkA#VPtr7>vDy+3w-OH%vCis|Eu^E3wL;K
zQCP$9l*4hg0#6Iy5|d}VzL}RMU%X^hV!A8ft*OngzzJ^>yVr?!^31-#v@qq$VuixB
zVsA58wJ#X6sb)HIPv6<|lchj++tT%#+g?j7y_4oT(KlV?_3GPOj&L}&7I6Ax&2f0(
z_Gnh)LQmx)k&Z_pO6tEa7WP#Maqi!tu=AYdk^Z_fb6kRgjvLjq==(c1iLDZLoH{{I
zl{=f&%j2QJj4Q_i^$yCF1uWV<wP1BEQ^l#@T2mXmMO7HQ0v#CKH0NtvQ3~hE47@Op
zeeV-h|Ef!u<u2MBQ9dl<X81t4RqdE~>q(zL20@2I8Vx(;!VCg6{AHR`RDzFga$T+P
zKT;;~gh|=jb2df#h1Jn_9GLgppUhXyRq@=@cX@sJp>FfHBAy=aQvEC1f{P|Jr$1+D
zYH4C%wE8b|Qb~9L>zke<6W1?X;&9;FfhLPJR^2=MoTV$~&a81);L_!ICE_%%W$AL!
z36D6AP5N;1!U91rr8hzno*qV%`viIeCpuZDP1&+cLAQ<T;!Ce2i!H0}8mPLnT;iW9
z;j65f*%1_>wPlIvBty<Z?g<l{SSCpZ=Pq+Gez{uc<SXaU#(=A<7qY0o^DvVQ@#0(?
zZ}K{O&xz1J4F{9oo%h*9F1cAO3ub6FPMIh0Jbao=i*TPcf1+TKzu%Y7v67K%-1N+M
z`n@n#?RfHka)FN#8*5vz3g_gLiv+Y7yw@!Woc6|3J=2-%bB{^cwt^?WOrA!C&zsb;
zB1APea%I@A1NvM`9zXqJRJ~Qi?nH!>(6b~r!O(2G*Ry;JS9xt)IpNHf5R0{6bxQ*$
zw-oGU>l55~dDpL{v(J20ixu-ZnecPR`d8~zHkN-*_hRc}P|3+k6u6U7we9kT-hFM0
zv<~KbIn7$pykxnoa;S%P-ed-&psZSdPlo2hCnPytGF4(8@o24{(!w#rz3R>8P`;^Y
zN7j7Yu)>=sDM<P7m6W2)+4Y~YZb#`%`lWEAC#BS|_vERyw=Y>7yUeQ`oq5%_??}Zl
z`E0|UnJn&%Pee~vuS{0_|6gnNv^AYPJ!cnB-l-i~?$;+AEAhnnhU85(Wub{RiU~p|
z8g9CCac$00<IJ$V^~_b0bJ3CqDUNB6PWA?#+CED$Rpgw_>8Tk%!s-NjHa1mGc{FL^
zj0u*@Qa79w%)HsR(<bj_fBK$u$#l`Zv+8oUm$e7|)JS03;Q1-;gad2k*3G*=IGL`t
zbZS#x&NuJ-<LAr1YaaVi^T}@OWQO|CN9mI`{akoR@6CGAPp{(^t2eSTM@zeK?G96M
zopbMzfBC26NgO36uG^MO^xAUFx&Ft5Z1>((lV-fi@OIyROsMWl@YbhCpP%XtEA)OW
zdHvNX#mBStWL_8Uu{=9zW7V$QrJbuqZgBi}-`~4h(<iPX{_UiVpOSu_j+m#MdUDCr
zrZb+8YFgGf2xW>hxHw#T6a6`;>+Hq2>pLg+<;c!8jNlIHh`AgW#3gj>!0r7$>#L5f
zVviJ9oAJqWVa>Uxh9xt%JT`my%hdh<N6-42FH!B6S8r@A4c=ls;Vs`MGak?IHNi#h
zHAb#`|4p{orzJag;vZp7wM5?Xb%*DE(s(kdF=z(!+u0XB{`i$}Yo>QV5W88|M)%I8
z+0I)mof1p--#9vTRdklXoLT2w5`-PHXL;3dCM{x`DC*%mY4d?wL92y!9(6vi<~?tp
zb(Q$eZl%oydtZh<zU1;_>qc>lefp`tZC1}-`(1OU<l3VmoB#I<m`uMG@rw13TZ{OT
zx~kgb*)z@vbzS*fAjG*w&a)}v!=;bAW4BG7tC1Jca^>Zt*oHO7UpFyu2FM0X^eIj^
zJ-y*lV9@8H^6Lu(cD=C_Z#lR2hLiTfJb8)0^WQ{n8SkAFu%YRPkEl`N)EbK@qs}|O
ze#{Hvng2g9fps6-frmHKB_vlGvdS#v-nAskx^V95iQ$PC=kL8B^`%3Wi|fGtEejs5
zbYXP8^!7}Gn&fTgv-dZAoV)3&WTl1l8;?_74`!U;S?HZPu|_6@|L+C<-sn!<(8+WB
zy)H!W^L@C>bovWj1!lGw@0LlAXUfGoED%kbCAiEbN7q$w*%6DktxOB19^_>Bf9T*s
z2c9zzUQC^AI>*UEKGtE*k%T+8zA~Ou=07QLv@N{z{c7;D0>7WE0?G?zu3S}kv!O*f
zu=VwvF#b))8#r46j~o-vnstxk;*W^Q!3+7*dluCuNJ$7yVf}SObJ=XIL%m9&qMtq<
zoq6ZtbxXh9Caz8y^6n*m`gTFz|31`<>rUgjB3^mYL2^nuCkxA27HyjgcMkIXUfMFB
zafXMAfTv4H@P`EzKNOM}7J5FJml=}SvUK9^UYD4M(k&P7S*hJi%?Q2Ob8lxN8>`$r
zv0u`jHWJ=X+L<iw*%n>gwQ!yl-%P1Y1#jP|nt<8bff65X-j+y|s(r$@&F6v0r^g?C
z8)tHy`yX5A@JsVfwnx02=D8`EE=Og<rk_0R_TsXFr=Qop)6;V0m7YE1=4#>o*X?Q3
zr~NBR{bam5=f#t6=awae)LxZ8_CO`R=7NFLV$EryJV{=GR!@V!&DOa4IQ@oPOXBrE
zTaK{D#jf7?U}wqQ@L38+N>zV*$UIMoG^}dlex$rUm7OJ3O!#Q*->G~meeE-Q`&2tc
z^tUp-b#R?}=i1Vih5NR~9q@EqniG0+&(vi-$s(^7+dPxrvoz!kXXL(uMNTHN+ChnX
zRb?gTg?QhIo9~js^Xx|Yl*g|767P4gxJw1O7~ENMXNgzFo!t>G9#$@jN%Nn+;q+>n
zu9VazqH`?Ri1j4{>wnIB57HO1lxN&iJ#@}7<k>P-wO})qw0Eq|QE6I}6bq8lEspYP
z?z}d2Mri$kex@s-vuAYfys{|A>3&cEOGQeLe%SP>mDB(Eq$@9t@za}N6t}q6@NxEp
zJ6<Wb{k9z16nUH{NXTN>y=ikMN-axHE{l4UnpW1<_V$Bv=DWVjl|66YKDJvXvb=EO
z?#RxN8lGtho*b+*MQ!gzHL|_8P0P3yT^g1cyFpRf%b~~5q4(Mgk(?)SH$y)O`u*kT
z`(K;#)l|~@>%z6VN2l{DDW*z(f6ID8R;1<C1CN9$`2{Zb1!uTT^z%{Rl5)zNUZr`Z
zG_73YMNbshpE;9eZhFYla%<K#jsFh{7{sSE%qa<%+`=fn;#S-eb=#TTe>o4V^!0O5
zS?M~ZJ8YJMeA(UXK3V-H@##0C4clI7s2-Hq;a<_w&h&Ia%)ceGOczC8ik&LvqI^sB
zujB>(gMG@LrzaPz6fk0yx+gaM&r{25(NmAi=#xz8K6S%B=*eQHnI&eb@eIpK*k)=z
z^;;TqD(&8E+jp<J4u$p@t?s?2!S0|fzGtP=lmlK<93+ou_7@$gC}NR)r+G~FkxR*O
z#apg9ld_XF`*OWvY}Y=SyTzqQ<@nFLEhil{_h#Mm;&{42cEYhGS?lXwmMYy<j!i3@
zH)Wk(YH3$m&ffDqt1=IJ9bofmn0-i7Z(1X#i2HxZ9SsaK7`GMW6o#yJm~r5El#ApF
zwb&)<*>jd>$=$5_l(}%%n*X)oXKuCfY;y^4@hLr!y76kPvuEb{DCMP9S%<eUl?xmx
z-_sH|>5#qN3fm~p&VAP&YQD(OQVUMWh|Y}K|46RAPT+BEu;V%21HFMER&OUyc$|LB
z{i9_rN1#{C#TA>~)@&(qWIyoIBSL3EOtz$qOXaS>{#o89fB8?i^-zIHYp#Hv>yc;P
z$2@lz-9Osd5;f__Ez=~{ORG2jUGDC<*n{WjX4wg83A)P+*4+A+{i5w%uB7%!4uOhQ
z#|~}EJ$!4$k$tDjcREVWV0<#;seG>sdq>V)E(gvj4V)zh-tBVue>7>$+<B|5GGA|B
z)sof4Ev~bE$DJpKMORLed$(l6g2<-18*iP<`mv^AmAtpf#HZWh7reN8cdEkTImdlo
zsA@mZTCp}O<5TjV4bOgeD84u(+I{P6^Q2t1Ywsls&orApU*b8vv`?|ORzT7!C8I}o
z^S0dCht}TF%1rBgbNtn@yGh#O0xk<qE&6!sVOG%0>%A@?A7y^ow%T*oYIl(%?kX;d
zs|<3ceeiHN&OJ$T(Z6itbu4L;UbTrzWik`2Pt6vZa8SfSLegW|3!%bN$?UGQwKL;9
zE6j3M_%a+}^4oPp^PW_@bjF&z9Gw&I)~zi)c)ZNv?m2JHKYGqJYmUxp_%QF|v1tzf
z#n*VxndWfhTE>b@u0I-$_oec!RFBNO@W^Az^G|sPEBmfjl**o7^+@l~<d>n`>u<X9
z_VwLuypsNQxd)4D^fpe%8SforCd6%Px}=t^zS(864}-YRf!c%zxvsh4x!eU|YrQUH
zeROPX`6QjWr`Jn#=H+E8n5@@q-uF`Pc&@l{>xo-$#p1PDr!(H;(i6{Nyzk=>@z$N`
zLfC&s2NQM&fg}Ag%O=ZJr>^ykpP;t#LDUh2%FlBO7#}Moxw9;i?_zOlng8TSJ9k;y
z3C-kSpLO5a)Pn`Q=D6l9_|tMI<=rBO*$2CVnS30K#6JcxY`K0ZpH)of_o6n2lgBkT
z>K<;pb@<VX|6&sw=D4{?PdJzx@$t>9!h$)4$97%b>gV?^DEnDRw)mB`Uga}C<gMZK
z$(d8cAY;BUYr`bt)h_?frZ5Qv$m$=ps$G4*>A^iNEvFkB=he;sB6R!<*YPh#Dp65_
zwMEzb{gM~%xgX<l$vbA1SHz|jlUg^txX&gtH)`TayG8fet{nGVs{2wa^_p6m#ido7
z?%lSR+pu=gql35d<IaB);ZhMYe%)h~^ZHOO_jVS8>#i*C-?4}n8|Hs8RR1iZV{C8s
z*_gY*@3o0{gqgq!p;*zD`+q*NrK>#52%pEq#UY#-)W0EU^Rukqf{#~iuuFAOGTh3r
zXM>l4d(naGfzB&@=3JW{s`<apQ8+YU(!!o|_U(NR`%epqe+g#V!cdvc;b@Yu`Q>4^
zH)1}oj~q+iDz^D(UU|)?4-&;LlHb=cq~u6(=NtNeJaW$Dfk~m}#cAHLWvR=Bf@_2r
zzi!EUpK(Z(<y2Kc);seYh0TezvAGjoSIqoY?bcVKlq@14u>9S0j{^3<KWS%^9^Z@1
zoSG!U8*Ccf@+x7a`x1wg=A;FgGo~8(Pt?rM`1dUOxzg%Y>s)T-&9d4h%51ki`|hWB
z>%!hVhC6<*-%4!r2+5w1|M95ax1`CAyLc9LI^RgKF4ngE>C@LzQqc04t7VFI%hby%
zGmrASseP?Cman_bx>L@=mN(C0vgfhJ&;KWF+vYss^XIuc=kV<C+VOx}I^^jy<APc3
z`Pm_NuCkaI`tV+J66iZqWPUXLUF$lzk^|=q6O2F3aw>G?U$iQ4Bg2t*xmz0a))^+T
zzxQhXD4AX8_~o5^<eO#5I?~}eKA$Uky&7iiGWRP!GWqy+hlp))r_HCBw--Iwwf6m&
zT4jB2hOctwuAh}!J}>gFU%Z?{-!yUZ(i3a^4V4^EY;-KU^RrejdBe{)F*i26%&|y_
z-QnnPxhi1-8|SY5N;%s$XsuBy>Cb(!QBET!I=ywPMd-YFB5lc5Suc9sOZvjU&yL%Y
z<Mcsn^~b#;#hL+zl6x{IIhz^J4$)>kxJJ9>@aNX4;w=BC>bKAFc4uO6GztG)-+kmk
z#P$bA+WOY9Ozy4ZSX{QLyLeh`%QV#^E?a7HcP%+y+LY`XcJi^+%tp(qv!7#L{Y;O2
z6>AijrRlOjT4UjLsYUC<UoK0k@Y?S3x}uanY1^%rj$0b6b+`R<DG>j^&f(9x!U=_U
z`?kM}D*Ui-%bPGA>DA`G>sR`+GibYB@O-(&*?NnU#k)K5+9fx>-{NFvm%3)kZhCz3
zH&)>v59Zq#EHi8BzU({U-0iol=I7=*9h7zPyih98nm^m!Y+BSlvnlhr4=mYr#L+`-
z6;ovI>guLtKXletznX3LWx~FW>zf=_>FzUMz4_6Lgx{O;6~7sXTP90y{=dcUo00xb
zPhA@m_u`Mt4h%B?KW=^z9Psm|=gukS_6s~}raJF5VQ!f5;_Z%v&kLdyIOI3atp99&
zf1=fTOWE9+R^@Z9_wHAe`y3-6(0u*DIq{PD`(2*IDaB8-I}r8Ef!lC5a|=TwGpCqI
zg+QWHE4L(<$Au3I9a))FMb>CIG9KgPGAo;NAd!n*Pj$x>P0wXL4My6mJ0c^Gxq3Aj
zg>t>PwBg8sIk82@LL@KyND3==c0DmjY-;J15M}WYNKm-a7QB+r#Xx}JT5}|8k*GrG
zfzVb#Sw0nR28X6Cp4&Jwb{CvFu!HIOuC-bTdee7RzTT&5{6XJIkC~lYN9Vx@-^Q;0
zx%WzaCKx7f^5j#JoZ-T>GK~GUPKxu^&yHdJoPA=t4>l(oEn%6Fx3{WTmvIyOHZGlj
zP6t+D))1K!x6AW2g_n4ANZ3>yy0fd^Tzo-7<-r#$n%~td?p7AP*`XO_BT-ZS<AcY^
z_B#rj4xeaN^9VkxX^`GICH2XE7tc2Df4b7k1g0-=X;PUd<lxY8=GQ94ja*9Jj!jZm
zy0q4FOe#=R;Q7R*yG3P!(}Hb3inE*f9ECDmcCl4;GdJ@I_iPXmjtX==#AThcK}vA<
z#SPNCAE@k?-7V2GS)P@nXoH`Kagc&ayUd$qa_m_`$JcUg3fK{5md$@!NBY*8txax%
z9j1*w$^TO~Ut!_!U7(%1x^)VR+#_G_rb(ASEqp7d%)PK(<M3vM2SPhEl>3;za;Qx7
zFyahkjLFDUnD(st{ethi68sgs^0gZG3YmyD*b1NGICXh}*Na0clPoOqyLcy@(r8>Z
zt4sLWagmx1|K=0BzB)8z=$^jDebuW$clV<^*$P@8w63k&cB{+#rf{;zi}eQ=sje|p
zlitmJOeT46=j~MuwXeOJFQlxPs>f<{dBNFZvMY?QTm9C#Q+i+a#GRE1#^J^-Io0p1
zRxNt`LE+JZE9<Pgra9*+JbCcvob^ktZSU+m=W+8Z_nvsnkgVs@^>y73mJd7jO@DK^
zzx(emZ9h&{mYA7g3=5q8do!%tR28tGecdk0w8jRHgy(Y)fBJcF^(Lt(Z63udJDvI_
z-_vqtT@tWu9bf<5q$6ol_emZ9lXJytzx3MDNzqERPh1-O+GK(ZH<rd!8y?wm?z01{
z;Dm(ks8_Ks^45G6o4i0jTDn!#$dmK4LbT83R?|%nVs7lTQ%~S4s#8vvxfaF0>Z(Oo
z)YBLKjuW+}XfOt7tyG<M=Yr-2SM9$uf{#5AUeMfV_2NsAa79qbz8PUfnW~d~<~m$m
zTE_k0>R!o1Thmr$=w(#uPG4TIW-+se_F9&06}z;OmfqW^kj(k`w87i2P2XIzw)Rfi
zcx0Z*z6D?OtL+!EEK*oJw?J6Q=4?~M=KuQ@|9o}ZuDCk$>`u4I`nLp@$j*J(J6Fzl
zc{}F;zl#g+B=eOS)O9XTlG!=qn46G<hhR&PSM!wP50p1eopMO!2P0?wzt8gKf*e}Y
zoaNTvm~JC|>4IZ$<2=V%%gwbES)6?*A8P+*ETAOxF@i~BVmHU6wHg;s)I>EU_2sy@
z9{JG0ZT(7W;)O63Mah%h4njixt5hc_+-&Lbdy;rgNTD_G#@e>RHC)+$p6&lLb7c%$
zWbh-E@`HA=DNQV#Pn^Wsom&-n0;E39Vd*>6xUJ=#&qSNpbg`?uZ?AL-R!E-ftB~3x
zSeB*W*xBh_;y!;-OG>D8pu_48_Gx<~*R>ZtQDhKs`0u?s=AFZ|*#YZ{%AD7qDbU{U
z#k6_DlD)}CS!W*K!fiP7<(ej$w5ZEXLAQ4;H8Bla8okbXp2&#<`&%lE6<-H+SSVXg
zHJiZclWD}uAX&8j{2t?YrVMw-H7c!Ud@5EvHXU;tKFc!vRWoNal_-(1*myT(f{nDv
z6}8l#Rug6xq(~XhtPeV7w8ElQd-o&B`327u)+sz_XW`5YU%?X8cx98eLGcCt%|@5z
znR0Nby<K`H!EEykB}PUMhA@#u3JldsDNA_w3Pde?zD3vP;fnmsG>?v-Vu_qzBO_l+
zUC%PPs<UgYXQaq&$?_62<%zl;3B1wU50r(jdh}^Y7~?cCkHQ0&|4W?j3r}{9VL!yS
zzmU^syJJU=!1v(JzhC>`9`yKQ(jL&~;Skz)q~FtJ`lNGn^i=HxJACX`2)Fi3m^>q>
zXL;Qxq5HioQ)L3Up2w61t~fMl`tikF2C|dIG$$kpuVIkUWV+s^p~JsDVsc{0is#E-
zTYl9$R-#qF&182(VtdMkPp5?!|JeF<EAOs=*!#XEnc58#jr|rgs^90TShs7JTK;pT
zn2P0_?3)fOlg$%#&=z`e{mz0njhu_S@<07FSjLiN&N<h?!(xWq-q33bGk9+{xTjC)
z6jSHak`54vshl3m$*|mE_tLDT-C<HjFQ#d+G6?-~^DoO$T+z>280w}ofq`M+{}uku
zs~GH#B&|H#+UZ%Ml9c3gCwm^hVoLT2k-cVDwZCz6M~OV1usdR#zMN%`tJ>p9i>6%L
zB9yBey&${#MD)YP3xVsuB#ApkEWBr%*XS=SXsoYpk@oD1@x<m!2@AJ5G)X;8v0EZ^
zD#dw)C&T%jwLG#M)eTA_=O?YwkoSC=7x8lyTeRfGt8=>+?QUZ5&4^^!lHjguc5(HK
zy8@Smw;5z7tNA4Hy;eBs^K8?!nL5GhDzjc_D^FSFDW$$oYIf*Eho$Dp_HUNGRtTHr
zb??lPwmox;8xFehT7@2#3fs^qvhbphc8=ruuUQ9+bpNOyl2c3jC=<3{{-MGm*<Xgg
zI^Ni+{C~_;ud(Vuy9KM3<B6H4Pc<#y@+8QqXIfCiv$dP=e7j_rq^Y@=Rp9c1CXsaM
zUd{d2ZG+c0gm5oPWY?}b^|rA`LO07$D=2)<k(HrqUHHN;_N;pKczbQkYWq{q!um_L
zZJ2iUiRSe}v$Iu?*R`lJdA@k&xNOaYkNZp&Lyrl?Zq`w|#<R|Z-9#onnlUZQ`CTE`
z>yGDkLQ#thwv}i;oV!S^dhNGXftDF}b(X*2nf&v{IvyL&FM-p7UWAD~xxjn$!_vn)
zY$`ubx_qXXLwB)&UeuZ<vCWFEhnm%5CBI#(x>4;D|KyPmV}*WAt9{-haeb?%9LK0L
zXD^zpUFX{y%xXOK-2MONvcl`$Djm6gPxQo&nBd~$OVaA!+iXj_bup)DMK-tJ?8~+9
zX9p}UIJlqn;IDZb-JJ5BO<EuOT@^8V;$*99@pjjwtNW(Dw^{PDx_x^4ZQkpCA0{tT
z^;wsB^pI&_?aSpN41SOK3!T4&1Zf-U3GKb8oZR#HnVidUp2j56y=}*&CI+7Hy7y?}
z<dziW$!BxUWb}OVUoGLA@uZ!n<lS$N$Ddga^j$dOb8Ep`mIGhe7K!C7E*8<$6zbmZ
z!DRTX`IW~@)mJ@sYOifGoD`DOjjpMeS+EA%9X;>$bV^?mGs{tnE_O$;34YJ?7$-Pz
zEpp&qq`+jN_wJrvl|vSf&W6*|3V7K5pEH#6c{gjlt<Bq&Z2ljocwX7C;flxEtaGp3
zmK}A|(4W1o?f9t)$p*^nZ1hc}0&YkxGR}IdV7J)#p26IJ#U?KKH$Gg5o75|+5^(d{
zYNq&pkr&UtG4#CdaX-*-Uggu_i6?kdMSL7C=qe?swaFYx&v|zDm#$J>hp>EKn$bjI
zYp%=>y6r0t-ES-4aZ}{nzTna>{`(B)v{IfaB)oc<W$2i;@$QS>^c=xcRhr9|G>D4z
zZS!d42?^xNTXtkkPR6CA^bO0ox!?auJM(Z^$KxHxa;^AQd+_zT^sV+gCmqW7^bFgY
z3u(J{WH_c7GA(Mju_;@T<M@{aT=maRF+RR{I_<ET%Krqu(nGI*^cY$l5a3QyFH#D=
z*DP;)G`Ol=!%Ja{TQ<`fJ@)Jq9D=>m8XUOH5_uW|geP5Ce#}AG;)$b@kkZ3*hLaq)
zCMj@Dc%br|w_}NZ+p6c=J)C#&oL898tu}Ra=Ze&*bMCb_jz2ZfSG*9Qcw>TM(Btbe
zi@h(nFzq~J807CW!FS%gO$sUnOian&H#}+Hx3>J9vx--n1&i82r9g%&xr%(}6bfEt
zcnOLd_bR?S`)f;z(&7Y>Eegus$s#6>+-8gklRkb*^Kd*dNu;-}cG3E+4GNFuTs-Qq
z#QD^Dmnj)LGTwZY5sv12bnHw)_LeTsCZ6soIX#!2b+aAH*~7t*|Nn?g?DJJ~cn&pr
za(y}>t9X`i<)gfN8y7M)ubXp;&s{&B>7v}}1f_j^=OUg5J!n?g$roa?TICpDMb3n&
zazX5Cc~o{QahW7?D<wAEXcTixez@d;UiB;G_eTzC_2?EJ(_j2SM${?qoysY{T&+z3
zwp$CX`1D3j;x@duDCGq|PiQVrc)@X%V#Szs>XJ+vTi2W2n3(^_;5k>xmussPIwsAP
zP`X+2R{u|d@vB6IXPJB10()!xRed^mj`ZH?;CEm-;@jk)yDur+kB4jFhb2{Mx<Aiy
z-riJn>v5&bV{vyyA(KlAHxfk_Jy1$kJ@O^UM!qk7-JvwDre!IHX$#sjM3UNT{+H;i
zR(dq+qVBxFtXZyJU)ioDJ;|+lvu2J<_J{r_#{$<l3HuxteCib3v+T9|#X#BCNv598
zj9=THuk%qz^WGaZndx$i*%}w704IeSCk+qRH$JRWvkEk$I?X4&=S*$2o*sNzfNANQ
zPrWxB`yT~ycVx4#S9^D1lZ~&iF5_ia&yPCJQ-95P7S41&NK{FUmFc6@WQD9vACIZq
z@hYi@>hAgw-1TL%J@+A_E1!&v9y`=e^_72o?bu7cyBl~{FA}v%HVsOirx7Bcy@<yw
zWRb(7I~vS7!b$A`*=;|9Dle6IC>Mn}l>W-g48AMGsFLU|nDlT_0=J3cv>m+REg_fh
ziRya(Pi|)k_Usp7;Fv6_oR_MSpl9<$VOqxr6F1kaH7CCquln}+@3v$A-kdk;eADx<
zKj~<p!jUT{COvVR(xUL_eXu_lqy73XPp&9DdS$xsmDiu;hlNT`uj^SKm%le^p@rLP
zw)1^~`O}h(zB(;R5Po7*a3n>PCE?P(CFwVh|GuHV-6c@Oa;vp;>53}u4@^r`mNsO~
zf0nte!Rzv8uFV%*u5Dg9=jopZmlKuGCq}L{jM5D|bK$SWvGByQ&r%m+d>(t<n-amx
z)X((g>x*k2WhU^hX%TrMV5)Rw?X#7lDhvkhSA+tmcm;%{XFR=Mvn<Zh)a_)k&D^jM
zVWuM2R|+$}=$n6cn()6t^zI4;F+DxwqVUj7_uiP6s=ryxc5ca>2}|Sjmi9f1OaHEC
z#8Pg|!WhA3+{J7y+^49u_R<D3FaAZ5i`-VM3-Mkr_+(Yr)ooqehfJQHX{*|Ms%rK2
zs<S4$mKfBfyz}GTQvc?P>B>jXc$D^JoPPExeO=08Z?z3KXX*N<nq3r2+U~YBeAecL
zFRmVR_<qqOQIvtb;7yQlQAoY)*36sdqQ#nY!%KRDUWGI0Kl2LWUh_?rJ4rb2@XDMm
zy2lu@?>`FZ&d%oL|M)AZM&W#DVrb51rcW;EtuOP{*RHdzc%!zwKuals`-XPJS@Y`$
z!s7*)G^3}bdM>^`>FSMhN4@5J{VMdoPy5WJlQS&5+oqVT(zf}nyZlmMy7Y~rLe6w9
z3AfZXBe#XIPlNQfJBUj2rLSHTV_WLwp;dlSY1uWVh_3hLtP6rtgA_ZiE0`1sw`Etb
zGId5>)pc|EyX08H+>@>uOa6ZP9GQ{gR>ESuo#O<@h7<c%`RHGpbe7}kx|dImlor=^
zi}JAN_nFRJ#bb6kynXu<pEkp}Cpyi(rW+R+r!hY3c`1G`i!<U_LcyGhY4VR3*LZPT
z6p83u(mOk;yZq*%Jh$?NVKbJ9acJ%cnx3s-a^9^qb=LhwvpAJFj~`HqwAgVYc&6Cp
z*}DT;d7bU{E^B>#K5=`v;*|0>;jp($7VtjP%)9-6%4adAIc;CR7D>cid#Sf8(>QN&
zKI;OxYb=d(v^*|vDm=}!<WcGU2^WiFuWKp%(#iZNBEQzD{esx(tusu+6)mpo$~Ki3
zo?bk+WO~uEB~4ySSIlsIdd%sT#mwBw)JuiVYBTx-lV*uO3MsDAYAN^PZQ{PTx$VZx
zs5KiOcT`F1eqM1=&1Ca*AKkEI!<$Ku&nI6LI=mx^Bc6L&?@v9qZG7BLa)Mt@#m!`j
zPV`;#q<JGt{o@p&VlQsD#JpVZdhs363s=2AG_^=q*LUL#6O*W!mzfxqW~gkO+{Bu|
z^XkX4+*upg7i}=#p<wf4!}bGirb3%UqZLg0dA2>+^iZkg;LVjB|KA^#;BWQc(Yky=
z&dP@4I{e;0g8L>t*s$@)=NGMW8A?9aZJi_Mod0^&ws?Q1%c<NG6qcUf`2Isw`0CYl
z+Tz864(@wriGDJA_lvJfFlxPhSZ0b>oc{Wl-@48}L?4tri?7`t`*KF8^zE4+EELb}
z(vpzgD;%w$w0je)!{@H>j~v$~9JXFnban3C$Xvy+6QA^rcXIS--AtPMD`(y=g=M=$
z_Gs-WUgl_~(tbPHw7fUPb!LoDWPQ}(x<{I^8osMvn(JN(3`$>Yy6cUE`J5wf&AHm$
zbaQU5XPli?@BM*KX_M~M=IhcGmKL*xoAy3_(PX&WOZZUs{+o|(ZJxPf`JCfobNm0_
z*|%9`|82{#3kIuST524OI`#fS%o-NH%`UrtRBG8Bp6`16M$K9=HmT01ojhez3lj}1
zzfE2I;_a^2y}Kf>h;@E>sKQXNrr7Ch^sdKq7SAl}<!$;Yq*5l?<h3lK%s249O;-HH
zq}`3I5h=T}L>FxpbXvZ9!X;k*^P9ysex9)@KW6qxhl=?hv(NkgTCs7(TCLUJq>XoH
z`ApxnlKswAP9+BRM+f>Vb@og*^NO7I;+w2_@~IsvK6){09*X;im+5b9iA~Nn+4W|x
zsHnGD)Q-xN>)tOq>npP~!*k}uhNJAeH!PNnnkGA&U&&zGPFuCxx=OKqI@QPjGo5e#
z!*%%f-N_uSfB!q#J-ut-v%9C{lS07lSqV$6{5~rvIdI?5_TA%o{aw?8V%;zAmYwS3
zz23A#k>R`J?+adKv*(>q;QSVP{;}!$%o&QCn*444rB?+o+$a?hD&VQHR$ON5oyI2q
z@vGSSW6MPT_Db4vCiyWfT96elb5Y{A;Z5tk%PaeDN58oHTjAdA+s{Q5|2e(Xuc)wH
zGSfx&*o)l<ge^}l_Hg@fw<|z)lB!v**|n;rR+DXo|CM+g)D15=oND^_c}-!TqDWoM
z+_T<qw(*{PvMz6PLBZd&?4`n!9k_q(J(4*yd@BFp36V$ZTX!$LHdFcci(H$HmUB1#
zzgKZ-?(wxBj%(L*Kir!*anHY`{}GAvGe6tjS6H^|<aUL<O_%C!d^lC?HzzMzamySn
zp}H>;`$XBKj;LRG^z>`=^4m^-bX{$~KV`ie=>Dsyag|8yyv)8WOY93JZ&xfVUG;2n
zg7b<n?@eNtR~Kk`$-Qj7cjvnFv86K=j>K-<F8N}!<e%a_3O9c1Tm8K}Kg*>>X2oRj
zaQk;s8(-yYHGFDNc{SShT5_x+pNxVPr-a9b1&+-OY;GM+UJYxTS*15Md2C$N(8(+v
zbtgjb@xd?_+bR_e!6yg8mPBo>$~c{(%ed9U^MS|aWxh?Ed9UuA3|4RIwda2(BO$`T
z#K1X0%Xfl<laOZM7LS`op{K=~QrPmeERLmJ|KAwDz2&dfhgav;+Fs@|*gZ`>sw4m6
zq?;~j-gk9#H2oBJlqbY9)_)e-b8EWy!^6xwMNCeob|)Eyh?EO1urNG$DoZoz9*<!0
zvtG_ETUNPmnCm%N(Z2l2jb~3>8P`Sy=d@1bb-XQ>yZxALVCvp>woo-sk1GiQO~?Di
z|5w(mV0g~Du-WY29<>Pz9rC=k&)i#eu;t8Z;p3e#3>69tmlm6Iu^jxW!a2!ff>C_;
zvaLC+)7P$;zrSH!_(aoVYK;57&U9O)>M@~#+brXPM(jkxE`}08iyLt*YmYLvN%LKK
zSZ7=4G@&E2aG69}Y{4Z~am}KGi@Lf>jcz(<b16J_V4A4W@V~>d?}c(73nxp8CcEmE
zxfM)*T8<P8<+Mh*ZE#c(^b9F#dEg=K+oCWfLg>pA-!(QfgIu>wP-<<;Shn)nERLDe
zugvyP6gU{{?;9H0z*f5Sc|CX6Q-#?LY+5T^OKyg)@U2$my0NG$>BNggl4%@D>C@(V
z+*~?|(?EIlyx3<i9cR4M$ZGQB*s#<$KIx~bYwV)V+isD(N8DztnJ`gEdvhP##LU1R
z^8lv1p(S$*4~m88xZKq&f8%hPYeJ5zyB2%YhP$$wbE{7+V?R)JW1sl_n$Nk84|Y7_
z^SBuLb8G9mU2i6(PH1F3nw8MPc8>Mh{BKess~LWWnLKp(AM`|TA#-qq!GBjqbsd8P
z#b=MQ?qoIn^<r0Zk;}_Yy`X7@%e4K>7I!hfjF`}2^DblYNhZ!pBX<`rmy{?E*Thx2
zYY&BN&RWi7B`9qv?lOCZ!p4*U-8FL}4gKAVRNk&{Qrmka$oJ&*mXPQS!?p-fuMGXz
zC|Qy0T?*SozttyPW_f>+QQmf2bb;WOsWGkhj$Vu};otURX^rF)iKHop*RJm?wQc+U
zK!eZg<pb7St`{@c?g_b-BYjeAPPV!y-$eI~vMt(<leRmqUL)|e`u6kSwwD*&*Lpb0
z2p#_!6<26!xIJakk*Mr@#V;P#U2(L(xzA+lQO>DPHk_2*J#VwS>VaL`J#|9nt3B;o
zWcpt-lCe+p0E;h2qU*K8t;=uQ%GMZ!U$5phcgdW6YU0k!ZvKlu-XH&;@J-MuLa~l1
z=fJ~%Qs2HN`p-4;U=U27$t3hHNVh65)ncNd;L!{>E$b628bpkm_MA~rijMq{vE796
zOrA#22`}#}WlOK{JV}cX>de+Dm+qd?Cn{m+Y0btYGdt5U*NU+rSgFDJlfaYAf}neI
z<~W(IzL_ebe6fQo*H|zt!=GWXLhF}B8he}P^s4=Rsd&5W(tMp;Pr5j}X2+hHbadCx
zr-6NwB9^>6rgtpiRPZ#dJ1J~4B(3VW`UFlYtgs7SrF-L8VZyFESqg`obZ=>S@iKaP
z)iF<Qn&HOf_+N6G<E+xjX`<O0ZkilIvs74Be!QA=M#-ZrEq#KjD0lRm!v0T>gy*+<
zBx*h0G<9v-*IB~XCZ*fVR{dJ0>0S48Q<aUNtjNjldLJ@3)`+}}^gZyy^+WEV*zlBP
zhs36A-M-elKBHyI4VF}QX0>Ckg?A*n1vr%ZLnoxDX-isZdpPP&3StP_vCuiJ)F=8~
zii*a>&|iB>4(VT9HgQE}jP{-g|BgLIj+~52K|%)0!-H7$g_2yo@>nO;f7_hS8@R~r
z?N2H7c^~2>Za)9R#icQE(ier;D<^NiIuz6;d88+&XG-EHH%a4jTb6Zd`lOh+wl>MU
zE3kdi(_wbix9dipa{902|E^q-f>rmPgswMA)a6f_y&_K~Jc_H+D~sbqSd5|d@uXPS
zB~#`2kL6X!sdT7MR=KvtNZXE;vwyQorFGs$KFtrAa<vJTY%eD&h5uO+GezV2?CVFC
zb<W$(aXw*3gl?tEbf$(iQ}`b3+m?G?;nW%N2QyACuU^=%#yc}0<0`xA8(Edw=!Nr|
z_9b1au*|5ATFlLSdy)N~gktU+A_x9|I}ttk?UEJcIx@W{ZXS?2xx(mJ>w^60t2c4H
zJ7Re<bMpEM3xO3J4%}g)OI(+UOzcW9@eY3aB0>Aps>=&likUOn752JZI4{8&z$Ln<
zW|6~G#)(Ob#I|y-t=o40-#;F+;8QDpP5Hla=ZS3wGtGA<)i27N{H;hu@0gGGO}|%W
z(~hlGEA>u}Hd(WRM{`yF)T+fxb&QW&ugh#Wa8`N20%iXWi&Ev4uU3SFZl1mRmG(p%
zW6v440-di{d|h#6rKexpnKe%o7$b^VwF7Tmmb)hV^W@qdrycWldP^Kywy0O8<FSu$
z#MBSLs?YXrn!fSPEVWZRIs^W$o>loQDsk1ybS2e-1p+SVT(fl#`KLsfYHnGlYmpV(
zxKbkEA&1|dPT^#^>WjzCmRgjo`kZp^uIG}=lP1qRFj?Jvv$4q2m<b}6mWjQ#_0+yo
zbLPjXmzrPZb}dx-6~E}>1+~{lITT{&rp|6`$=KfJEBwED;{O^ZiOn*e3*9Vi&#aS-
zHeI{cbCY{ij&J(2T#>9B65AyWXKnhZ&~Y=`?|RYYy#Kk&mF*TVwyr*USo7SuY1|n{
z<7DReq?>Qr(zGz=(3VM2^IJr2Gs<o0d8+eNk!9+Epwzgz?B0bhgYH@-njc!P!*IDJ
z@=T+w&R*Wx5z(8H?&X~GShoB0M-yFZ$4a(u&Fv>2aONI;a(10WWKyUrC+~^4iknB+
zP8HqeFZJkGebwVyCv~SQ<k=20uRLp2i6x8p@4k?#nHyDKVZCf&=Z%$u1&z^D4}CZl
zvHPHz=rgrLkNYPqYPR*v5SZB1qIm1&)5Rw?&I~nvy-;w;+YOD?kEMPJExP)DuIM_m
z&vm7}sqy?bT4nA_W-iVU4vP8sSn!#LSKuwP^z>uhg}V)WjrJtwWmojJ7V@MX(0z2S
zc-fImuZ@pqroB9xaWj3UV=vdm)9*U^!_|cjrrPg*H*5L=l?Nt{soMn@p6>D6xqjQJ
z_tSU&nRt>@?CYwTiB;ar&on}>oEEVEH*@{E*(X*>ZmAbHxZ?i)uGYkFGuoefp8lS*
zaL<>Sw@d_W8TYO%b{5}d^vEQAk@lrKPsLnP#pjr~c3d%fm}{^u*GOP#Q0xrREZK+U
zchXDM1=Z@W`d6ekY*mmEP0Zp=RTpS=5lI!Wy_UJWT!3wdpnQaYyhT%fc$2I~w#E<t
z`HQpvM=cfn>5{W*xm}J!1?P+eX+vMr6+su(+)5kk(!VqtmYN$c&2@Q^dpj&xeN*rP
z^*o=7yyGr;7AYodh0cMkf<8+X{1+u>PZH9)*wD7MvCv#_<q;bebv?H2-fx!G9Q$6i
zD6B?wVcV(gaRomt@~$@;2)4(Baq}&2zrbG0;~o(h6rrwSDVgrQs9CynTJz1~X6cQ!
zXIm<=4@L1R+lmHBiY-!Pxu_U@fRXQ5OY<^A)5JX6=fR4T4QG9FU|AUBUS2P-u--%5
z@uFz=)j}C&55b^}*#8qfc9=;kW~eH)mp)>xSkW%HL?Mt{gJFrIz=4k*E-c=>&VjK^
zZ6<<QrRn+qvmE`o+j|AY1Nw~1%hHX@%hUe5rj;)9@l*-yh=^ikbnBZTHpjGTu7<`X
zwcL(LzP|%of)ga1SM*Pu*k3E`K5w}uhsp$2Q%94;2^QgoWrk`wCj`0KEo;B0K3*y$
zEpF9$g{{6cV(o$ClRpH%Ov_%oJWt7T;<|EytP_I88NIyQdw8!;dbu^Vba|Y<rSmJ-
z$yGNd8K=aZf8K7mteo{@UTCR6=Zl=T5fzKogHA8Az3SY_o9z4MlAB$Iutr+aTlKtU
z8h&%r{SI#xv^`|RRNyF9==kURR8z(>nT4!I$pQh4fvO@sYp)9jT<_C3C_O)|;nEJ_
zy)znay=YU(Y$|Cl|8KT9b=l=?!xsPLP6>Jyf{R@fC(ID*jcgXUW*TGMug4lZr9F7Y
z&i<Ok{T?@F#x7{#V3}pJeX5G7gWn1hfr%zoh27UvwR0JSj<^fgo@msmFkbE`{5z<1
z@3U6xZG!5Vf;Ty4msCtX9iCm;sb=(Y(y5=b1sEsix3|}xm>kC>v^TKO_**U8#J*Q6
zItnZ!WiCeb-xNMn61m~%6rajYqj0ghWr{C@LgpQ0d?P;pb@|lkjP;8>COqEaFi%j~
z#4s~~Q}AbcNs4HSZ@KmRC$h|eRi=uf#f?J$ztl8aM$}~pWS7qsjTBh>ae~xyRrAAh
zc+(s5JGJzV%{?J5Q~JDDW<~jb!I!m>K|Q<`3q_n38D_cmhWpI=))y6MJMnt6m}8|-
zv!e0Teuq_+b8bp_a!l~}<+kMH+=YvhSXEQm91Efx6fS+9$aj6xp~yv1UJKW^ORb+I
z8t~Nn)n%^G(!#wrC(5o=E8J0Y#<O?BwK;W}?d#J7bHyi@WlgqxnJmDbCKa`isi2kZ
zTJ7!9DbYV7Gny86ci3_YNBizHO-V6-UM{>?)4?`8W2)19spR=nCT2W8mhomwjO3H~
zDL?14D^BBbioK+y8{{Fd+Dl;CHJi_&E|X7W#dKu%UUu1$mUcz5d{>6QXlM_ETF31d
zt-H$RG;Nxm{i*Q;*Bs?n)30a<iK*6baLoDNSs9<GwfJ?0utmq(<7~N>vrI1^6PuQq
zJ1MDU?#o>DNByo{4zF&tto*T#vtpLY(gjQg!dJEm3Mr%pe(;J)5PfW<@br^UovL?e
zTicFDi&$2hep|eOQ#vx%!&}SH`CC9h<}NkS-NJISCb3PNQ~n|~{8i1m?W<=R3p8j|
z+zg#^M@wVkPPOS*BcFDb`DS+Zzq0>VEIPSr^Wu(`Q>^mJpRDxrQvP$(k8`EtvMAvt
zS%UV4a_1ge9d+~+HEb|;TyJ9{!gJLo)Y_%wkw}n&Y4qx?zkh8F5Zso~ofUO@Ynrui
zaHG&~r5a7`#dji_uFfcbB(c!^W79T~(63C|6FswXwHN=NymRqiMZ@$jiyd#}_H8ux
z;L7Woy@X-sk~`*G=Kos4e$e^5)?)8XDoZD^L~pQ+d}t}HxqNx?^vsV!(i*)nUzeFW
zHZ6GFSWw+w-Zh!$dQFW~ZDYk;0l_J~9;zDM#xp<0s{9gr`B}fm%GLK|`imy<0LlK)
z&doNr_k?#EKG@P?>NG)N^HhuJMibHmk~6n3cnGH5-1}srtao(m?<~Pjj)EJ!1hz&9
zoIPxvC(7vkNak$u0vW^Dpat6&ea$?hReOC$ufl4v{hZ~;IFtXaUU)owxkg15W2@?^
z6d|uAGomgDbaXj;tX$hUtLV<jgsWFpJj}{%`mJHHG<W98b+5ScROYPv-}}1rgI7>0
zgP^WYrj6r%k;KfvfSrBY_n8##+a{<jsLde!dZF#ajk}$epPVJRdD+Iuu*pU$3oljH
z?7o#5b~K{#R^jgMwa%Uz2ez#hzau>7Xq17s`16-$+e3>RpUpe%J(YEfB%{en?~au`
zOC$`0O5TRlvsG>p39R2N(Xi$5zWa#+>8$&F3k58E1io9TbF7->?krQ_YWx4tw%XHz
zt5%!kNB1hNSY-I6D#*O&mU@M$)SmNUi<^xOc3Jc{ygpdD*loh1nZItVbJt3mb9{$a
zmcXY~V!@TdlNopR>uAhey>HJM!P!$x>$#*~nQk;u+{K=F#wcq|0oP>Hm2+}WEI+?p
z>c3`4q2tfI1IudftyB=LiMr`?ROgoSqM9i0n=78Q?|E*X__BQd{2P^u*EAnalPI1t
zUtnX#2P-Yr$6bk>3;ebpx0^WY-<d7eE+^j14&za6xMF(3l1GsFmgD0_hL0D+4Xd;_
zJmk)K<e%e_ru51~Yv;vM?TJ#qRG55DHs17?d1cHiBXgjuqvG{;v7D%#GK=fIXWC?)
zYR;}Svpr-vb%#dHx<!@qU7l9x>o7RlUeT@zGCHHiq1%wRR9OCq{&|%wq2Fzdi_UNg
zom6_UrogNHoX?yZiL{us1EI?;cXTJ{&)uC>sP$grT%58!clJE~#|BZ3$NnapPhKT?
zpG7OX=Ga;93I7*M9Cr{u-q7o4zew1XTka3fJ|17;$B8$rCNf-{s&Kzs$8D-`Sfbc}
zmLvMPR|~l`IK;Q-r=P4^wfpGy)jlRiwN9_i>UB2E)etkgeD&Mnn=Lz(a+WZ1p5E|g
z2kX*0W#b#Exx(&y)!f6xk{#wsM{*zW5j=6{cBW!)%-u6*-y|hptGV=ixthtf{Skd!
zn|4R#2r@RGT`n{)N2soQ*TE;^i=XfDoqJ-B-STr8G4rK_&woCC-7jN;?WL)ifm>Gm
ztoOJq{9(1gk}0y6k~~X9y=1@KxbWyk>cpt|)0X-ky{S;)Gf`&Z&5ElFyt8*!^{Bp_
z`<!*n_tsmwy$5T#bG2F5eeTHpzud-fR%J_@w&ePnby_(NcBij5Z4-Pgb7k+G3;(8z
zdCryO_<MWFm&e}jg1&zPw!b`la_`OJDy!o;!tUpGl}TKRc>AO@V$PAB8{K8f<+Cs5
zM$BE~B)#-+_bJ;Iw==JuSteZL9jW(sQ^KF3*&hsQ+={0k?=0b+|2q8n);Gbkem#2E
z_1t>V`~n*#d1rTv_6fgi1oVYt+j4I(#L0d1w!VJoe9_S>j{B~72R)cPMXk$5<V3c&
zvCCEI+yjfdcLn=BT-^7>=HrBUZ!LCj%jTVFyx%He&Y!dCx<M;z(@m;g_wTtiMRKa0
z)NR+Q9b0m5=Z7R+ioTMXJHh1R?Hjiq3n(5wb<X5J-&57YGS65;mm9BaH~)FZ&TMzC
z$Lzcvme<zy&D42zD`tf$$Gpy2qVYE3IeGW`tM0{xTz_+R^Lv@o_gf6#&t2JUyjR}%
zM&Gd*^~K5+F@isMMJ`W#e6i5_!qN+D`_FgmN%~l*7IHCA@WmM+%d1w6H(3ieUiJ{K
zl)A+$Sspm=fNHMyoM%mXUaDKyxjsC8Rxdw2^RJ$a&gDm5mo@m7%&wG}Gf`A?&AoCy
z4b8Ck7VCtq<uo|XzG46PX4aROY(r7K455S!ezp@IW&8z0`2|DQ-}IMz%ItH*zwgMI
zGc_AlKD~D@<bmFl>ub#(=&tCFNbd+gdSr<(50h&~_;*nyz5IP*|4q}yiuOsCth-mX
z_g=Mc-ZIPRTE64<;mUj>C4VfnzU{sJGv@{K#S2d#T5&4wV+znZ_wfVUybljf=i3YF
zED?FRkpE?|s24NiliD@;Gyls7?PpzeZ_>PX6V7VQ-s<lkHD66JYLSN4rJ~d2+w!+}
zWu8sl`$@&?)LPrz`oLUM-rN6w{PDbXY@^NTvTf(=&)>dMXR>w170ZVK##4k|m43Zh
zZ{j}Z?ag;*7)^Hzb4}cum~iBN&Z5>@j>8=p@&EgMUjK8ld*yepZ<i>;M%M<1CT3nS
zkqH5bPI_F@MkNx0Nr&5o#JN_?_^2q}C1Y6i<VI_-2V<;Ik&z~Ys(P!A^-7TsL5tMR
z%+!yZ*E93;qJ|)@>Npn#h68?!xP?VrXN9~BVsxKr7b;=0@bWToMi0-9sZ$w(wnp7t
zrM^Jy%0}zcPPJ9rG9tF*&fFHnA)a?=C+GQ9qN_Jt3){Hnm6NLLbdmUVjqW>L=TtmQ
zJlZR}_-=_|(IMXymI*8Ouo_FAWfJ!b<gi-!qA|uH-|%d}mdvhTjifCz49zZ1QJs3N
zu4l`mLw7d|PunRWz@Ye(g%vc4tHZ#+z|g?tz{tSypMjM_BqA+gcH>eWr4WmP1*_Y5
zSXFdF0}?p(m^^t@lqXDK=o4`6%Gr^*#9@N0;<TL9)p4uCn6ftt8E0MH6{=SJD#pBE
zPQz?F)(2N4gQjROE<PeFx}apqjz;g9|5|=*QS@(o+;&f7N2tQN^)Uxm2}MoKys|X4
z(<xSJ>k5ZLXT?5KYlC$smhnDbr_~+q5WR%iomEW1ZNZ8|i>Gmk=}g#oet(vtQD4m^
z<);(Zu%)FbClseM&aq7wbI2-w*|*q#a$xPOEicckj-1GKR_n`~b6X3G^QO)Y&AT^?
z_0XO*)2^I0TQXttLzYS0ZqH*Z9H+mVQzib2X+!a=ra*<|jng{2|L?!k<}>r%(UUzv
z{!u9vjk8bWNZQ9g5R2B4PP-Ojyg6)x$6BMA!2!Heu2padl_oUtG<}__(X_T`TDVi_
zi3hEZEniIY>`v@RXrIq}g)eYgZ<3^@+{UCwIg3v`TU2PUP3Uot(J77Dy(Z5(9``MM
z`69X3kc;8+1UV&*Wa<00TUu*U3JwNN_P6>eSaqH=;OW$`w9eE?mbN073Y@y0EcHA*
zH8D+6b%Dx(N~x7U<q9X~8XfasUZt``_w5y<MGRXvD7UGZiE3U74Hi5loXjA`aWPQ9
zOzUb>aH~s(pG~M%gVl7tloxTWw~o9FtFC38-r5zECbGDWGfSb>CsRT(esbggh*cGx
zs@qoimliF3xo(o1RAyR&n#8>gIcKgVZ=55lExqZOUiO+z=gp*bx10%*Uc32>-tC*Y
z^Himk@_yBF%V-~-Xe}_~K&6YIW)@$GYRN@kb$z#6vD5GF)m)i%$#KfogGVc;sBV|b
z5LlRz?)-ISYmjHnvvBVDYpyIhDY1G((`Db+7mPGlD=OZ)5Lm{$Ouv1p+tTYNT;`rB
zJR!4q&H7WNjv7~jbh>#YPHp77wum9B$0Vs+?r3^ay7sD$MHiPco_*YFtH-N+Ilx|b
z>*WyhzsVC!+a%YV)?Bz;#m#@$>%8)FnVR2<8UA;hRmDt_40O#pc=y7D+&PsKZ!k?!
zm@sAa*8lFW9e9}iK26u%Ya(Es%&_ZR#uBcq41fE-*LMU@*n2d)sh_)L<x`v7xE1SX
zl(DX2NN$a(^eFB$34bP#o3rzobFWlo?3TwO4=s8+m9y8a6x){J%<Ok#^|~X!vqE<W
zy#4vsL(bGc>i!!k>F6)#PMQ7vav|=%^0(Xaw`;#%(u$7GeEfOa?fe(>_3!Ky7uyo!
z9FpZ3yXWfJ*|9forM{G{h*7SPJ225OVz<X($;m2jjM#Hrt#bs|9bV&<Y`?IT!9?xg
zgqy9U{fAEmo7}LUI3q`E<?e`9GY!$xOZ8G!-z?x_UApkZ2R~iQY@URqip`QvF2Yqp
z9L%f<{cQzpB8wB4FZ`ddOi7$~qq{!SB%$;pTU7Nnb_VaBCA`!_dF{eAJTa^MZ*2Q9
zQPcEzP6U^-#O5!uRi-IEOpeJju1%l3cB!F{@Wk5=@6?v+@d@}b%?xgen6$jCL9=j8
z!CkJ>qr&r3mTJYH?0U-bdCGE@)Cn_w%lJ!4YPs+|X~`Ds(E3*-vhR)8gbADO1RIEY
ztq+)_Be-fIPqEO{sFj~4J`!BwAt64w?TWL}+s=DgWh+m|ty>m#?{wIYJ?wVbiP=%t
zrP2&1dh8RwsAe*2;Zg@9Wnam;-g$A&@_Qbb?$15fUHAQ0pGEOZ;W|f`K1)Y+moqCT
zY+3o#S|{to28qwh8f7jt#QK~Ri#zoHK>NuhVv83@=(Z_2HU?~{5EkKNQJ-NMHKAZ?
z%<pb>>tfNUX`9k27s?(t;5^ya#*!u>dXiiF`0mL|KP@hmJR4B7)Mf!^g0O>1(#5nY
zlhv+kv&^?<iWHx^tl{@@xs%a3DgBTxm4zxfFB~~<tZ^|jj5|3u+xYe@g+*?SGcNZ!
z=XyBpGnvq|WsySKE@?@VNfUdYY@WA@$@7BKCf1-Ehvf7n{j4Jv?r?afvSi|sQz15K
zLJ`YO8OSZY@>EB3mw)7hl9Yu4Q5QC?y1cS=%8JvS&uzjL{U$!uXmXsI9Ga<pXRA=r
z#v^A!<-)w>PH|MddieBn&B>_Of|IK@T%DH_BYgY)|AW_NTzWJw|H$#^q;G1qmwS4O
zmTcZWF<@oR7w<gtx6|#nO%+y(4o$4Q<0)b!!_v{Hso=6D+y7CGdLtkAJB=*{U9E0z
zO)q>k{Ioo@K2Li(aj6LZs(DI#SRS$jE6nZUC@sBz;Fb2ii6`?<NBj(*>vK%qqH;;R
zu+OT+J<k$<9PHoh5jaVdb7P0Y?t<<<4iyc%i_1g)eDHVnNmA-$_xxnpSsT(5V!g{E
zC%Gi6=WLi;jVR~pV(wtcS7CytvQv3h9lgL^s5$B978i!PGr{R1;ww%~(hB^wDMV56
zdAR$sQ|b)g)<@}xPTLjnsa$bkK3BZCz<EZA;Ea2rvr1c^-RR`{uTwfHG}&#Ub@a&#
z6J+mjrGHs+AnL|pu{BSZE?~aveVt2q#_=uJ7EbQWNM-wc@j{WBkUg_+W)`<)YQprA
zW0I489PD0skEitSwB0YOXKu_@ENu3BaO~B(S1ohHVuPo8luTO4x~AbfPmIT{t>-^X
zDici>Xsy%?zHpK^!m0W5gyh!taJ}<Q8nb7+E?p9vwlm|WlZ@+xhX3CT7b-|EHr>3M
zNB+d-z$=SHpZ6KWguAqHo&J(x;v3~(xa;!jv(d|ymL&-{XC=+8aIw2Pv;R87I+Y1I
zi7P)^dj_ZLByWx6>}Cn<wtQ$-IB`LDZG5cotOIV>R&J_jEzdbI=U!FbrE{f8lK)pv
zs8`tZmRoRDk$d2WsoZ;RX7{(0#?F&Xc)7`Ij|_*=>^)av3_^Drhu_wkwE5)fX>vmQ
zUwz$Nm6{kKW^P#aqVqg=PWyuE&MzF-U)y?`r8=)Rd-lDy+Ba9F@8_LN|CToO{q|R<
zuNO=YHk&I`o`2%$GxO)mW&b?%uQw|zbxe@Wp7N8s;iaMAd$urxb6cuy;<BCVk{-=Z
z;J>ZnJo9Kk!KWPFgE?nd!=|P!H4-yi&U#A9x7K;y-<(E|*<UTT?|kDrUn?Xx<FhZ{
zI)-D0Qckt0qO%mjPeumuJM2!gHQ-HZGuLrZh?_C<=dp`>-Onh$n*QV0-@Fr!Gq$Lx
zpOvZ$v#R(nF8J?JP1UT2FV+jK3g?(?{53+wwr^fL%i?T_)l0>WeqWrnO76te^>;q|
zFVpJXqP5JtI4rDqv$*8s5F=UU;@#gHQa3j61~v#+ln6LRUR6t-e@SbxkXA~Vy5Vv!
zz0aZ&$-<2;aS_Ya^Gl;*uQ%=4l5s$!`NWir6H78KDP^X&N9!#ryE93e)wz83hw_ON
z3l?qjtvMc(BI4I5F7(PpPHACQ>ve&IslLjIv2UCV%$sB19*pf}6xyX`@gXrx+e}`k
zF;C#KWM^^Z{X-V3H&m8hY-7GyWo(?F9TNZ5w7pYR<l;8V)eVhpO6@ZzHHbfIn0Pg?
zK2Z3~w7`Rcnyg<%#r_)yl$f#!2Db^SDIaUBdBvD-D4OreQv2gu=XbF#1-JZ*>aLNY
z1?=T@+H8U{j&&^IF<NcrcN9YtrkHJeVY*hVaAG1Gb0U`@qZ+HDpwxCjS$4VF!=_nF
z!z>+((mGN+6xq6st+W?c@hExk4o|mQY#nhia;|GJt5b>Cid4lNz4;TZyHh2epAhML
zDAAiLHkZY+EJACZv&O>D4Ia~rUD~Yu)q}2hNUFThRP8WqG~qgU-JQ#&XxEX_=4DZ9
zLUnc>ZO)x;yS_F07|X;Q_cDo{W!dR=$399=o0$1=lD(y43wOHek<^yDaQiQlCM`1)
zczr#v$T{|(n}ar!9BWIM>wh7ORFBHug`B@keMKMV{$p(C3lerr6!)JPd2fXiM})%S
z?;^>9qSL;JzH*q_!8&!TcvX{!OpAx8;MA$@J2aS-6s#3R{(g6O?Z6*)oUg>JrXq>^
z`8LjLlWR_GO4L}CD8Z(CH9g<*QR4OI)31rouzDoRpk5d7sCFZVAg_7dTs5KYoilQU
zd*sYR`5A?$z7()xR4Xh|D_df|`<tMJrQDbF1htKvYbVq@e=@#s!|Cq}qc2+}mM@BQ
zdnmqql7w5Jc5jmQ=cQcCKl&og<Rmn_wwBK}N}j#ZbN0rQ8Yep=x4zUk>p5rF%Bido
zV(uH$vK|VzZ>pZtq<y_3G9fKzM!Svh|0AF~mYzn=+v>o-H-P`|g?aC8av!+n9=_dS
z?S@kBmrWC<%{SRGzt3%A&$lRx6F%3PWR5P9kv~|j9Ne;KTg>unljbbbYEfqp+~xNy
zQI1R5K<1<Kl!;NRIoj?nO<J*S^1-LlGX)$Y0}Tr<wk`UgklT^d@gi5aNo(@PND~u9
zFU~-dBU5=YsuNB&Si7q?FqY10h;)w9GMTd2Y_fvs^(miHmoz&DNKQ6<6TtuF0)J@L
z(!U4hg+}rJJ1{S5*3yJ1{w)hQ^&d{-PSLr+8tl2aGyB#uPnNE)Ze0aYrn1+j-(Xj6
zjhw-IQAK^BtF$3&b@Gg0hq?_M6<w8PFT!I&Bw{W9Uu&89GHY#tz?+hgYAe;c#ZFnD
zT0dOmsz0XA%oy;bwQ50X|B|XDi?t-?Jr(aToZT5H;r4J<yQ9R$U#qrou3nwB`b4M5
z=9l7kB#O5!vfg*gbMLLW?`DW>s9e2u)|xH5)@;7DW=GcSQ@7Tfc{%sc;@($3`aVjO
zD*dola7u4rm7G~P@0ln6%bW9FXRdp5bKc9Ub$<`6OSrY}^{S=G4qTc$9KJq?+P!2#
z%eJUJB4t_M*Z-T@ye+jX-+cpH^~Br*T&ow#%sR%({(aJst23AmGBP`Eywy^EzHrhN
z$8x_*<=Q_tdd<-PS1QnvE_CXK>pVvX=7X*5&N4EB7I~&JOOEm_nZTO-f9E0#ZAGio
z8Yj%E>}F5>b3?H^a0!bhmvhyYqzRmE+FV|%IqNrVaXBp?xpK<x%egMoa-6cAU0;j4
zpWfnqdrREvElIPtByw*}TfKFq)7B`?z~&hWx<X5(ocUjME{%A#G?8mrN%qo;>h1Br
zmX>la(>3ax%8{XQiK}b5-04F#(rt-zcT8Vuk<?JRyh(G$waqHixXm~x7qn?Hd=8XP
zy3(aLZAQnh8Oc}7dbnbPO$DvYLgS|Hn!GbP?NOlyW7xv#l}>DJd5eT+Utr@-arF>f
zCE%2P$a>F?$UQqN_Z-XKvn5L6jJJf}>pk_0Sp8OWUW(p(`M3Iw-+RvP-rI6<@9k=d
z2miI%mMiUhBE9dS_Ufmn_p~qEd;Rvlo4@y6k=}pl_r43S_uj1*Sy#3H7x(JL>}yY*
z);MN4`(U<n&@rwjmg`>ZT>5V30iH7lUcNd|zIxp|O$OdS>jdA-`)<Wmt2Y02*?c*l
z493@GW;ZevWhM$&9y)zQhsV2o_P0#Vu8pocWCAvAyijQW*u-z)OohH(ew)HJC1v=}
zY!lHi)Zq#inw7Ol-b?C-q4_>R)=ou5|2K2W%#OMnF6m-abeX-)-I^=m_R;v$TRgQH
zW=_}=WpHd}z_FB^W1imI{BCba$vzfuvm{REc;TC^B{j#(<{Vem+LrXjI_=JJ8{=b4
ziYK!79QV;a)~>Uy^uNvVN}nxW-P=5(#6?7w7Omc1|NB7PEB@$PORH~hue!aRxzL;W
zBL89^ex?T}%T^y)IcMp=1JeYRcch-+$~wh%QaxCLE#FP0tIvA*9-EzE!OJf`S79&T
z8NXPC>-$2@ZC!k;lV*6YXb>-YzI^BZP38r{Isq1H7HVo72lWq0gvorDkzp|Ne`K@j
zkVJC95hZV-+yMT6OXM%zUbRRd=Mne$r@vSE?cVpK`uvUV^FQwFyY~Bhy3bz5*kf0%
zFQnI8_^-2{DfYre>%A;%FL1_Q5aPY~Df@!3?}eEYSlQQJl!!ehGxs8!?A~OZ3-wA2
ze72V|WcEMF-oHgkp-Hg!(X4q-xz<1XA9e6e)#Z;}>#TfFZ4)`gv~ix|MQ(wOmo4^Q
ze$N%K=D@j6ku_YiC&>Rf$h3N*nB>*%r4xf>H!xIB)VgCMGdo64Mku#gPK?_=xwl*-
z`tX;-8?*Lql$$MEGwYmes$57!tL(@89>F5ngPS-$sTpdkEKyYnykN{S)57idmdnvM
zJhYGJ@41m@v#sy$#mEJ$6Bn>H+TNV%dvj9n&1rLQPOrT=vG&;fwXBf=Cp=eka@5|E
z<-N61_ZExn&B<?X&62%6SNCS50BfWG=hm|~x8&YlfA`jE+gt3i$CPu8OfM>BXl{R6
zb#l#_r9wH|&(&_PtoFWCd&(;Dlw#tkYq~<$a!<~`b9dGMGuyQuO*<0KUDT=cdSQiB
z+w`vAN|(*M>?Q^8dvn^-mFv)(4c}~4lT6&0j_>sJF^fu?5t&q9VZ4iFU0sff;2ULu
z<*OgWUS(rvR7(!xnp`EQa)4EIouI_K+L~F1IfX2Wgd|lj?k+S{TC;$4Qh<b@??uhJ
zi(30GtgyXV?Q?sF?c?pSk4@?xZ`XY+$9qZY?qlh{kL7qD+s%6{*?U`I?_=A%CuaMe
zSlB%d6yWr)yB&Dob|Bx=ZD${Iyq#>P$&ef%%v5>6XzjeO57vpsJmbE@&Hd*9kB)$i
zuD~LmD|zoO7ur3~ay<3c^I#g^LEaeteGL|ZlT&`QtNZuP*I(gdBe6i`|DXAp;tP8I
z%|CGD5ZAA(eKQp@WV3X5vNF%g$o{tbaKzpqbK%rqGnY)vOf^WE{;~DDh>+YxGeI+7
zXC+qK$;W2a9^1?NcqZ@dE$41+UH5v1?2Uc*UhkA;WqR;>tL>YkdACo<y*cIg=8WIX
z)9>D#?R#@!9cw_q?Q7@WT<3cm8Sr)+FKb}H+k1TP?%Tb4#P@EeENAT*dF`w_A};(P
zUQ6enIT<tS<m#SNYt|fy)_T4?XX)>}Q*TykEm|XXyZ8OvH+SdWS?a-i`p0wbi<h~a
zCU=(Y+>y7+WY3z@d(TYYU6X(3`38pY`w!a%V-@OR9aw8;seW$}<YZnEX0$>+{>&oo
zx>l{TTmEY>BtPN`n32pL7wW9~kc%%Yne|+3%0`E2LNaxd&*lg(lzjAeiOrjao7{iD
z$jg3ps(X5E-o-t&x07Ptgy}sEv3nY3_f*i}X_Ec7l=yF{|G#GFf6ILTHS7Mjdv@Ql
z|37)m_cY)B-GmQsnKpcF-gCa>S{vJ(13%KQn9aRx68qzw)Ot(C=T~$Yoc2zwh`Z9}
z|HERfl61`k*)w{-tX{OPi&o0HnsI%;_{oX?q9*Rxy5Wsbi~EX63y+1gN3CFfrzaLY
z>0{A?gu~a)ZM^nl^Fwb7kN?Kk&lJZV5SCCdRM1#>T{_CZ`_0Nj3t59cygqmT^@aa`
zPU^iqVfW^m{og}+-yY8Ud;5Ps%j0){@A3V+FVFhu{@*8d|8C3wd%phPOaFiG-v9lu
z{~r?rEAzr*uk;l9EtWo!x*L1zKSSX8`3I{0a!9#E1SC4OePyvI(QtU=QYEY#)l(6;
z*rh{UyJ*e~LuHSNiu!$T)EN}_w*8IJYngdLes$oQtgX8uSI=D=mf~?yWpn2h4Kvfu
zKB=fFX@NmYZv0WXs`V;lW$4Pdty)vBUR@i#dDE$`(42@(OHSlio=)wK+Lq0_q4sp^
z>F8Z0&+`^`hv(nxth#+jRaxP~vCfoda@%H0e2igSs<y4FX5*P7CnuOj<=I@^*l<#H
zM}^u(!!HRzlR4+R)fVz(1u;cM-%&|O-#vTU|H$6XMXHk<JoZ$kcTQ3jDtHjs#Lu4>
z^RwvL*}2-{0<I1%F9TP^w+q=imAwwwR=-{DuJsg+$EWMf`B|H)z6D%Sj@*CmZ`EU=
z%_)}_dG@R@Tpac{zFo8|W8v~?RXlH&%D0H5Y&gg&t8?NYyYv)|rISo{eNdUK_e$0I
zkeipTvw-6!4<}(o6T?G%qHC6_-HJ20AhtQ}%Stz?{4E!cEiWtKTD2~sGHbO$waBL9
zij_P<I$>QYlRVV>OSn#4JM*Ze+xN+%MP54FpKLnay=RT~)?GV#x_NYWf0-m~eErE0
zQ}d%O#(w8^7_8mU>oIwS#v!)q&F8z5E^}Pyy_MplG%sgk$A3{q6GK1l&LzzzR-Ti)
z^}0KCw)!{doVXlv$yd>J(H;#a)&47itQ?}8A^yIlp`pRDQ$r);drd<%qH;~GeTz@)
z3LBb89$lfDnYdhTTAb901u2|T8IkFtRbi1C?o)yp>eW6ySDtQGqj98n#oBMqmRoC2
zE}eQ#cSTHT>Z<l-a|*JKPAiLAxpHOT35QoJ7G~TMUpfEJ+bpg1JA`&5tmSEY^<dK>
zsT-@>j#_E!Zn^fzDQVl2S&ugK+`kpUu!oDaa*>new2tnc>$hh3_x>>PSv1YtZ_N`c
z&ckPv!p(L?E}Em7<-929l)_>jJqv*|=Pq`YecoEYDec-27#1D5=-Mo`od0HXbq?v;
zKVUi0vg2V{sGiWvuI_dFew|u&&f?XU%cZ+JOGFiRf8W;3V7L3RbCTVUd)@wd94e3e
zcKrTydj0-?EFu#em;`$^9ApuV**O1%dicho6QOf7RP$VWHj7)@Ugs#`GZ)(Ha><`x
z){jj$d_w6d{rz|3%Z??uFApnN)xKP&aL@HM_dedetFE>*#wbB!w#~eXr;}6URz2&U
z@=EhM&uO94#_QdSU+C&`Np9SsqV#L+hxTO>X~(vh1na$e>J;zIcqe@G*`?2?=cG*4
zSL)?8|9XD1>A#oTJ+fI;8*VGE5@TA?y2wni_09>-;JKSZSkzVJULHByROEcsC2p!?
z=ZaPTn+qmM7i^ipYM7EFGAp80<DYYPslaNnr?bK~&oFhC+?Eo>`|_G=&?e`pbKHu>
zcZy{--Ei*ZF>G%!RN8ug)q$sSb^D7X4xZc_6B@5139nrtyC)}*Thg>O&Y5xXz9T0@
z5`UiP;n9vbz_;vJ%hrpD$&JZjS4<4MOEr@vF5gI8F~Pm_&C-53<|VGn9vDvko2r{(
z>6;)Dx$&LmGQG(weX<lh9ZqdYG@5<!go>kb!v{_UllV`<!Vg`xc51ve+P1=7`<dy)
zHLgzF@xp#fn}WA-`7N1Nlr{VQvCG@{|2;eHn50bIm&J$vuTd&wwVL3hRIZeEY)L6k
z<+7$Du8U-(REn?vzjA`#H+*eXg<i)j?YU;Imgh33b<BM3GO<O=oNe|VkEz=>xJzez
zv6i_S?C$@Ib3#|T&2!0>^KG<*96m}0)GK9rT|FH4O=OnsjH9hS;*$5>_>9G0S(Yq(
zb6RNjz3qWT$2$FbcHTdALgD;@58ZXk-gM7-@_Zwcn;XMygV{6WGA42y;?3K}ljx_-
z$}0HFOE}qK;f4rDo#dIX71&?8Z|PeVvdquh=aEa2n_be8pRwJmoaUTh?6)}E=&@wA
zLyVHC?x%^DH42qqdx?uIopEi3!fG|Mjh=g+bm=ePcpCrr(CWZhPLmsW&#dS8z538C
zi5mx*%=(&U&WO4cw9#5jsJ~xV<^ORLwo~?DVGPYj)|}*cDK_0RB&BC!_X(Hk88dxT
z(#2m!rl~z*o4arsPwEGsSi8<Gi+a~A7RtPJIih&hwFggJg)9~FHZAB}+v%$mYu$TG
zWviKg<*MvMclsY@2`yk0?U0kLn)@VXd+d$O?Ym`_9&OhuiT)IIMEu&WRjyfq)?d6>
z=fxeJW?z%YU%zDg9x+W>v*69n@2alvzj;>AH)N&P+$)FVj7|yKHm9Gj<9lD}CnVP*
z?PH@9_@lyYl91>6DHmQX>|~DkwIRl7jjdqu`Gqwrb=d8eC4FqwG!R*PDca)Bz6+~P
zZu@@riTmRx!j)dW41Z4N{(oZPV8N<p7LxsvQTYE06ShTeP0JM@`Zq2;G;2*$>y`~C
zH}+`hN|ya>Gh{U83tV*gLzwp7TPm;J7G&w>oXy`G#;P>?y7k1$t;rn)`>s_!>`DGR
zt+ZJE<|n6XQZF>Qs(6-ET=xy|)UUg%9iI}?6)YF4Z}!c!w{>gDva>%nv>r4}YBy1d
zmfAhd*32w<`7@`L+?F1Dj&N?C-mByK_DbXo%^-vBLX~70VbANkrc4q#$fH&tW+YUQ
zxJ_uL?!17m$G;zDDpxo@T^}npN9WFSyY5rx*Dv~Q@;_jDM7Q@dql=yAj!nI-pk1}d
z%fekLw{7Z~!YE_@?JE_pG#D#;rm}0lu-?V;^ROJ#kL|WGfnEQ@b||}j`_XR8W}{!`
za7;Y=S+S+D`q{$pJ&syQ0Zmnf`sJ6@%GX2`oREAY+&pd10`>g`r~Y^P%UxaJC9^N%
z%*8K}CR=A6o%3aJr?qymUgj24?Ou!2J=>SteUi|0oZhh@s&<y{7Lgr0&Yss=b%sm(
z_L7LGHqYRtOV@1Gni~6I>Gz6mi)$OZZO;{MTJ`LkO&HV1%8soHtcwDa_3Ljh66q^s
ztxkRXg}HgDlHg9`2QR0mS#rj$ZBN%;cw0-i{AlP!_D$J~KW+_gc)I#xjcsxwt9QgX
z>9Ws-caAI<<<1IDUioE$?%q{_WfPZ1r@UP6_H|9B_G_oTYf^VCo4j8464y1a|4Y7`
z-1g9{`WAJ8r+3?A&oYnv)zSB?a+TL+F3%BK${AI%c|XIxeXm25EPD#F>jc6-=f?kV
z3R{yb`dEUw{)S>hDbIAV2Xo5*1gkStad7NglhLlf;wpdq7U%m6&p8_|@Bf+A8nl7+
z=PmQ=39RMKlm8f5{|S<1{4nSJDLbw$4ku^Mz4g)}M9K1g<c60smsh_!Aedt?ZMNVO
zO~<Lt^CV}^<J~;(jpRn%PDg>y1_IHBU4h&pj~T*lH#qK^d~wqXzT1<&%;eVMT-w1s
z>--9_%>q0@mpIgQmR|1Yz2?n3C2Lp3NiEl%yk#pE9`)WMv1s9$i$>EtwpadE{Gh%2
zQ<U+lRqX#Caq-Pl;-A;m6uO{2)S*cyp*7pHb1Sp;>ndx}M^4!b*#jGzLN|z2Z)jf;
zp;X<l`}-uRkWbrPN?2PKuP^(twcgoz$!EKNS)GpDoBi+X%DdE}>Z#y*yHk^qt6p*O
zNo%fsMqM7Ox18C$#k6Sgnd-%Ule^3~_8wu{8W*!Q-bOM(#4SYmSeww~aBG$B4O>(9
zm_{=@#VqEzx{JeS6Gzq)o=g*7Z=qSfN>=HI4TE+~cd&H&Y9-)3hhtyXobKki>X+vH
zEINK?V|%%c{)a53LnmA^wQN3~T4v|XTXmr+AW$)XF?-a6j?x1Dum}6v6BY&vuo|x5
zY+ulB*ui3`usSfJ|G!}eN0h+q(ihXJc38)MV83|LqOZW>=c;z24aWbnd6>5xv2vc^
zlj5LW<iK-tVWIc&s~dNSNOIlYDSIox;-28n1ui=mnrPGo9h|H!RQbv<XoHZVjqto?
zuQsij0k^w#Z#as+R=K!|dtUbZX_>+cx_3O>EPv%jlhFi|;3%`G)nZyMy%%P8m>qCp
zyx?>vs&)ON^&cN87M?QJUDf*elCjXwRT~bA9sF#k#$~f!$;V84v(QBrjT0?Z3l=UZ
z;`k)Mnwiud=)tF3;T#xYf2M`Wb?3Ce6SL265i7ef<vX|1@fRGuKek^J;9#5}7%tFJ
zvB$bIgS~pe!lnrvhk`jbU7BY1`0Rh5GYfNWDsI`NxYpa|u}H`5f;~GPb?!;(+I(g8
z%RPIl61VtX-t*?op7X+MR^4#zWIUfB;vZ;i`^jZyNU=_0jO{iX-mqY|wJs{jGBe#4
zO}>BGwtK_*tDk0GS#`{lt2bG>N9m8-a!F<-?gf_@NHK4mTfC;TMP%|F!G@okSfe*E
zM;9=MZ(!y*dVzB*Yq`&X<y8)L-P;A0+8>Q<{}O$`aEEi%hxR3t1gcIjsR%H4{5h^8
z!7AJvsJlYe=?cq(Q!E@OS~M2e?R=ssc6CmpwL(X*#f?4FLQdI#DQaEO5%A^GezCcp
z@`nt6>^^Y(=X$Owb9xT%5KCk(KP1bNe6VWPyk(M>|5tQuP|Un6R~alaC73h#RCAGI
z&!NkDk{fN8ItsoH{v^2Z(?xDkZNvVX4R)^_4H;)j2G3t765yM|EUUd~Vpf;@VcqT@
z^EIw;EN|#_(AqSG$!w>yg6qW7fgaP2%IyAZH6?6Av)P7bqXOozsjHo5Ub0rUu3FK~
z$SIJV6lme8AH7t|W^#w>LL(;5MWGkY2r2c+9rf9{XB|^wd&`Sfp8}ue4Tm)(SoNY?
zB}7}9I{AW4&+05-bz6BR@C0A##<O83oI_Z*DF>ZZJT>#?nnf8`U9NDPbNqZHKxd2J
zlxYc-)17yARwcTIwQTW>=JGo1=eyX|w_3vNm5JF8zxXdlKdK!2pSa~l*V+gLA6}QF
zV^P|(#IxN#Z{%K>8QyzfNy6eK4=0;3{;-{@%V_kWXU!KC>q4`q!m6tpXL;WZITUaq
zbW*QaWGjav&-st1nHLA}mNqXjZ5GmCHZEx8-pkDUl#_We$I9FN@fUd?2K4zjocM6~
zwA?E-o<zl!ulqw2+F2?)LKW6M6qwx^W3S}OpdrvKCN-fhf%R*_(sd_$_g8ZmMqG@$
zz{1hd^1WGxRgjgXqv?Az>pZ1PlBul^BG~tzy=L2Tc16eWDv1Nvrv$ik2h>mCTFYfo
z`Zuy|wOuENdhL?!mpVOVPn|5eWbroZBIBNz310##oKz(xn{Qvd%k-+#^selGff$3y
z!f{JvG=y_3wK7jF?>Y7KtelLGTj$#)E~);rxDUN~dFqGZ<SNlwmsU-_RK#0(NqAM_
z%%2CR_ZkE+o`_^*TC!tHM~!jl-8<9Fvc>ee#Cd#gEolwCv1sz2y#dx4t!4_LQ&+G)
zRp{GuFib1C)#S2FZj^rVLe@Y5)&#>NQ&t?#I^CPq%ye!_$5GjJM<iS8uQcsSX%D;@
zm~MDBpm3??W#3K(7mXEm7tb*1m1u>o^sW+MUY;nDy^`Hpr*D@G=ZPQN+YfB6pFO>_
zmg7V~IP*$tmM-x!j-)!9)j7Y;a(z6@{>1l}v(3@ZM*6SW`=i$f#Y}4A<ht9-)wFxA
zX6C{FCf^)9l`TWJ_`NmJIrG@h!cM}>Ld8hp{O2o=oo+q;cx18Nz6iU#CF#}vb_+GO
zdvLGavo(Fq<o~*nj260#56)kEHS_W>TMw?4A5(cu`gY8zwKJ?#pMO-hXO08s9FB@#
ztko+n@c7<ZQOaywk!sxVG|b?%(T|yW8qC2HmNPTWEnB%RDx&q%ffI*JIc6nzwCmih
zTfypfu>bij5snKj%)4&$#Lf9t9UW!B7g)h@!ZJ};V}g$ZgYN;))PI4mg4OQ&OxT^e
zzb>NvYXs}bD>B_ZE$d_xziS*>(sXf;WK7K#U!xnHMOQc(Efu0y%+2%CV&O7xd|;PU
zcA=oNLz{Q{jmFsj{|?8nEV)}@vw_)K;Ql1_&1VEn7Ya=mekr{&^}gxL`)k!3l@40&
zH4yTdzjcMsjJAWi3nlEn2HP%u>2P$Cvr>q|)>F<_A-y{cU5^P#TKBY6&6m1-Z(&;R
zJjt%k1+y5+W*uxe<YIOIibePF3E~fwLc;`DPw-shSbLvmEAyXH=6|iB;X7EvH&~xD
zVAd;Op7uv;b<m5b0{+6d4v|-)h82ec69VgbMAxNqmPOprSaBx7jcNb0r0$Z$lR4AG
z)F#v~@zDPqonJStQzB7?<5rbM|H0h*^-Jt+uROD!l31T1rLy6T=(-rj6|$>Wx6f7R
zR7eu*xsmik_wdr4kFMO<d}oT{|G&*=^x3!@9lI|4K6hcYgmkp9)xkKw8*$Rw%+H?j
z{EQSn7$E#AY{}DY6PHa`9+ShRK2iNpm2&IWqboVMx*1bKto?gJ{Xd1g|Mt(dSM=4c
z2sf8+NBd(_?qsjE2)tS5dh=|-&BF{A{Q0cru!{Ki@%q(iuE=BMxT<w)2XE`KW&ezs
zgLhouY7dP(R<NUGc{xwU><xCgRm-|2Tx>P)-MMxy)5kNCv1teI?z`qF>b<g6EN)6*
z#zGFkvl<;?>^q}5BHHUNu&|tHiF%OEXmO`@p1^}H77l^dl4%_t2GO%RF6z7}%v{8p
z#aon_C1_ZYF5Vw-=4;W^j6m(s<NC7CXO&F&|1Nm@Ar--@CC9an-r+c!!9Vpv&$hc^
z1*~^-o9AotN*#PRJz04A=NC`Ey^ySY_uydO>vzQyx4koEb$l>);=@)UkNFe39}3B&
z#66oDvFeVF(^AW2e8LyHcD}xPL}kr1xy>QLtFjn&Wd)Q<?#Yk}<2ca$AxkyQH~PR+
zJ&V^d?N{{-nyps0#!m?U_32ZZqVw5E4sG_=jxM=+N7BQ7h%~h%N`{F!i8Nd)o%OKe
zu)sU^;=ew(%Vq^e9>`d(Wu3A=jZs_R-POX_sb|j|z8-QS)L?>f%d|6%9k<NN75Ua1
zzwte*c-YtFv&y3EL{@9F*Zb#iynA8D-4|MMxMN0tQia>Cxc^VhdOGU!y2A6i*Z;6w
zIr(F#cc1&LAC>bDW=x*%JuB#gjnz8W4&QDYv6N}cZ+y|Qb?|%d@XawfrSx3J)-`Tx
z_TTt<i)rQ24{;uLy$jCg&-*C3K#+&Qbzh|S>8h(&mWwYio56d=V}aogy{mN{pI>DJ
zHF$R#H0!-!h^xQFJC$Ps%ZAGzqjlb~_bz$%>+-X^3}VwHSdX}#IHCAmGhOieT@lUe
z9vN<D?|r!~@vS|y<Mz5<jxxRe@_);+rCN0aSQX<Z{`%a{c+<{ozr|(N;L9`ToVvd+
z_Zdg+{o{W&h3?;y(Y@^R<?XEDJBoLlDPCacEyN*Z^uyrrjK7B?RFr()ePMk0zjOH{
zhwrsc@Bcn9pL}2bUj|R*<Mwxtp09Xgdgxht-TaI%?)q_)H&5(-`SSJlk~!Bhg(h=9
zzx(p-eaEdg7#2LsTu|4oYZ0<FVdJ7c!@jJILctnRv$;3y(ch?3y3T%AuScn2=B!v2
z4n`(+A)OOTdk^R`N8Zd*-5|EQaTd#!&*}^AyE6WJ!&#tuAW=JdTi#Ty%ST;in&-W1
z5lL7rd$jIr%B3?A0SnYL0y9FFENAtc=9ARYm2}i^+5W244ePwD3PMFxR%-?MzUsew
zKJZJ#%QGoku4x$tZ0ebG()5(Za;e`}PpK@(v~3jQ{I<A#OV-u8FzvKE-Ga)ltIgOi
zpXIn=@bG_QJOBBsFK#T-@7)^b)N-Ti%G1-i9uprrrFlQUFwcGZ!=PVVUS3%dzB=#h
zE}rGrH~OA$`+I9^gVE0N*ZW@1R#@J6kd;q7V2{Phpd~CS>RT<1HN!40_nuxCX*Jt-
z{!X8V@Ag_(ypP^Dng4$6o=UwJr#Jh*k5dq5{K%~p@*;BKpO3$;em*a6Z&LG#Q^P@X
zw`I!&2M)0(6Wy6CzeMMyOcA(WV?I&wLeXi*9g6KNy-F7(m|8d%wz1}3^0MW<>Lg&T
zc*-R}pXsY{vi{dAFS>2><~(U-NZWC6f%IC7FG(?-nq?O?JX{JzqGu{Fw2Hmn;B?;A
zIBKV#NPthNgziKK1unP$t2YFD+itzUEGm+~b-+t4&}E{lN`c$W(>-5$0<ZM2O!efS
z+kQ7Cb)v~xi^~h@!e*woSDi~;+{@(h%-go#>J@+A)LY3bb-$%#b!*RIyXmH#J0-+T
zGNr^!kuzq=tXC~-SDk8Iy<*>~gBgoBUEB)hJlM5*-QKoq+Bth3MX_maD-?^~c$De&
z>rID_9a~elq4eXM4F|Xka`HB&YCEWJF#Mj|C|P$}QZr{lw%)F%tFrah{dv1xUR<W<
zIP2c8x2pB`{e1L$T~+jl-|HEoa}*x1eZF>EvGv!z)12aaFU`(y-0P?lpD7@``(|rS
zO|`J5s-Mn*c23_tchnx6eVM?pb;IQ@jsJ#!+pfF)+mSR$l4-_;R>{;IfnL&ryx*oL
zg$S{T3a$CDwaYKzmFeWzmP<^mBAKF^4AzkfipOT=XRT$OvS`B05c}6_lul_Zy5Pir
z!}jjc8E(eDZ*Mr{nX83|?%NwNVWGL{Gyi+l{3+euYp*C-XkW+-%U-)+>9w1=y^qxl
zvwJ3+S!PZ0erNG)oo!xJ-W+b0o%urkHFFB?vpttdTN!PBw{*r!u3MFlHg1=DyZN-7
z^}9bi(|=bpcgZN!&1bFI{qBEn`P;3cVLt?$c<W3qG&8>xvvoW--Ed)z3P*$OCzgp1
z?;L+9Y~prOdfNVv4)WVR3w_ny6#O{fBuw(A_S_PM36lR+O9I)>H<@b%^jKWsU+1tu
zr&P#W?2H?KRLfEgriJce3k=wtY+XcB425;C6)q3#N%At;G--j8g)rln%z(n(9=$xG
z2Y7o8446O8j?q{uDEX@?X!<6_KC`8o6K^Pa%zqT=c_LsH=hUvy4OU6)eJ`}4j;`YN
z2}u&)63+DgZWoV?$Gv)qvb7ImJnVLSQaWrSe5|Z<V%$NeYa9Ap*O(aj^`4AY-nwn;
z?DL-M`%1n{li3?O>4C@!3#ONk9gM!%KC5|ebU%5z!pp7meNS}9_4Z6Z`0VQm&s*A)
zzDn^Q<|&)UoAb%0S0qCHs<FXWn@@>RCMQ%>HRY>&H%@)TEuY0=<ocg)sbBKxixMYT
zgeK};y5ubWD0PwD7sWpY&s8;|Co`BNnshVxx^*1vIFsS#&hF@CQ2F_6=prMoeH<ww
z`BL52Tn_eqS(_y5!Fen8+avM3UAeh6KfSH{T?EskmdtbW65)CHY=*n9TS1HQoR#lB
zOJ0oeD_*MjNRWS)lu}-QaqyM1I@^yccK$6=5ZCH`Dwckscil3xRUb2+l_&?@yHdEM
z^HcK6j>$ofr@V7oX&m#)Z1bGAYtqZja{1q%2o9ce&%}m#p@8EX;SCG@5-VQbJg=p#
zwczIqHpSF{3l4MK_dd6DF=rLhba}i~i!tN9Q}`aywm8i&!Q2zctby_m!mf0ct@xkA
z!?s6l?xQg0Nq5e$*crJ>=RE9oioHCuWTC5?P(=6PE6UBQC0s2t163sjxdM%t6jPK}
zOPu6-lKc5c`fY_^L75<~_UXUY@KxUElJDvYS*o?DCGG2~2*-_`YhN|}OX9w*$J4US
zJn)`>@$Xe3rEDuil5S{*O3i=JanT~BL~L(GsoPT{?}#Lkux%AvgC5lQMJv@#%iR>(
zmRH&oeNQ)_oLeQ1X=3<<C%@#59dYF<-}pU#dd>PnBG2Of{^*|a<?nSdtC{ito@hHT
zAN1FJ9IbEEcQwi3ZJgt-6#^d^gBf<{7&OI4ZsM7%ld375&c5^J!mcws>1h?s=b}Dx
z?=dMh_+KpDaq88=#8M%x+@8`?S5`2cU3Sssu&P+LUGX3D=^MVzH!SU2$s{FpG%;lH
zc1Mlu<f{Q4UM7ozYl<eTYMt(!6k`7IBCmeE<<1^4%Ox^@iym942c4T)(w8Quo!gvz
zd+NMC%TtXj6^pkATMFyjY&{+8U%4v%1J4tioqy*B_Ik<vk0`O5E7%~N_lRqSQ%_aI
z)T^Rf*{{u)6FM|)-j_C~Abv%`7SY8XibsAWatD<#rdQ~)b_LzI{=+w{jzMM0D#yP!
zM77ddKdjba(3V#2e)FqU|4$CD_S2gex>Vd3^NI@iYbz?Qo5r)IXq!;t)mD{h52`Mu
ziRLW*d+XM{{8LOb{(tj5FgLm|`eyn5z<Z+JlQ-P|cg@LD^<81~%sXLWt9yGci{?J7
z%{TZ|-94l2PJ!Dg``d3OI#k(M$ZyXsu&bJ`@I>nRjuRU?e{uQPvu@U}34ZC}t!4Ch
zmPo3GHs7a-&6ncmO!05f=agr-ojz&xk?=kGdlxpVt?HgpSa4$B)X(pFnPx0D&_0nS
zmDr)%KEuW@+ePV+qonwgBSjnCwR2v#34|B7te$py;<d8LWxLL<|8(Vf{+mqyy+3Cu
z{0W)ZZ+@67xR9-o<*$)bVDppKch43EmVI(qz>}KqEjg`I>SgOc<po{){=SW6Gk&zU
z=Ev2LogY)53A_x^s+{IGuVjVsle+((YOc)5syjb#duGEg;}8e&ugjX<<C?jrEHLvq
zZ{8!;;V$q=;o5YCt+#$>HSK0EnfY5SMfp*LUj1gfZHJy{&NqAL`Y1``kW&lq8in9B
zb9c^|eL!FD)PAdVo-?+`IcwdJV%(Ol+qyXV&4u9OY@wN+N8empqjk%(Z2z{q^P}F?
z-ZtO&_3mHoEomF|@>hPlp8H_$90s`?xf`z(tQ9KQ%3WaUbJ(Jr*(`js%mc=-xKh{d
z(nRsHsV!``+_oFcsrblW**l$Mn)ngUAhoU6)whaQzvZs^c)j}J_1dT6>R(Tb{OIPL
zc1NV(Qk`Vap+DCHB+s0F=q+KW;Po-CC8S68>AmL-|Ih16CYqZ{)GqN?;B+|Ia^}I3
ztF4V^Bo_3hI-a=a!`6AX<${H5TB2+VLt9^BY>ElXt*eLn=T5sVDfv`o-ZO(KcOS~$
ze|YzvtK@ylhuXbUK61=cSah$p;^MTEiH@#(viI7a1n|tXeKe!-oJnMqpqkgShgar^
z_Rp=AQ!ktMQgCY3j|aYQ`f5tpKiT+v5^(vlVxEJ_E#DoBJ`1>bp1BzI_gHvmw|w21
zbH0h@jUCGhZ#h-;CYPRwcL}q<AeC@omiV;0Jhoivo$?u(UCT<kv}?FF@y-!&l;(7p
zy~c+_IQ91L(A(Lnk@q8~EcqBDxui=->d28d@p@bi+eKYm{!aNe<^M5JSspt(9%ioT
zj8g5IXQTJ>EZj5oVzFS#9@XnwUzNpK=d5#?UUu}D9IuRvkjx>;m<r7p&lYiS#z|g`
zmjs-IG%F_^ec67jG39u!qo(Jiqpf#%nuJcwjNyH>fY0wBOUIH1!Qk2ghw>xd0!9jZ
zcU-dG#<<zBXXS}0%~M77f4P<1Iie?YH=wm=0>in;Ox7DUcW-F2T@QWQn|L|7<Y{Qv
z+{G+@sUqk4StDh-jy>9X?2e{L^dYaiPi14CEzCdj*!tAtBU2dP1>W>+=$m>rygtYM
zlgK?qtwoO%<)Y3#%(js8bmcx0vS?PIT#DPh+|)(tWs6d)9@TyIdm-!ZI7Lp4>A&d1
zN;c1pN{%+7@$cH2VnpBGyyx)Xp6K66-d9zF`}jZnWz+0&^w@GZ%9UR`gGqPBgs-;O
z=D&z?O<)PNSiHee*`zJZqve#Tn707$MZvA{7RTmWoVsrzcQOCM(*Jko*FT)Uq)@cx
z%Hqta=|{iFJ65DS?s&X&ZF=tC`)xT>^It00K9euKd*7WcWgmx%gVh;-vjq-i>M2oI
zHr;Sz<BM@o(_q)jC@7eIuvejF(slWbA$7l&wys*BdN#c7;1Zr|34B#6_)a}(KbKOw
zc6JSG&z?UCFJ5vNTz#79=#ejQ`RU)72u0x|HW9f_Jr@HN`{#LhuQ}1x;BoszjO|C(
zB{vSGeEB~kb;8n}TN>BR;L31W`S76ZRHJ*cpB~;ls9HWR^0BYdYu~xr&)UxII3zpe
zQ2!0b{EogHtHhmx{+dB2K63agE^AY^Qkt%)Ht*icxz}Dk>X<oyUR#o1(vJ=5|M$L}
zo0&8>cG;pTjq*3sMRHyQu9cb7wTNNDqVgE-^ks|k1NybSd{*RmlvvH4`O>YzFLCFZ
z%tZyqeopcDsnMdFBV|x`#N`L`TD9lAH?#OWmH9&-|5S}OGvYJPnrbe0(R`oS*2w6b
zo%g@y@IN^->BX5#vr=ceUVOZE&f_gznU()STnZ1&_!VF}tt<0G*0!$mo6kLUIHK&h
zV0HcqZLPJJY-c^$y=ivMoc}_FnoAl!Jg<ND+EHkU|F>+3zc=?*P5dJA!lkUc$xr1_
zk+*Bs0@q2?-RyK)9=ty0tFvcS%CX8zeqHX~d28CQrL@<*X@3^(TNLEGDcpZm%KU{H
zfiqQhzsy#RniVX#Apg&$CPf#Q8Qq5+J(}OnJKqs{@ncZPueRuwtx{}#T@#k36|ha=
zyE&!mW|Z2LXg|rxhBITnJ(RZ-6?wOG=B`HBn(R!zBmHO2Z2jZ0h*Qv8g~MRwOW&NA
z$5-f1KPMCXmTl3ozE-}gbGV*8G<`cg^-7{&rjlJ>l2=l@huTBG>-iVn+|gV0sL4Ie
zY1QMZjI>u%s(!UCohN6oQLf2A%E@7>bk6_07aby7W=2=+d$#q>yqtG3KYPy1n!x#+
zqt7AZ_)pvAn{+R(R+;!#Y2qfen>nU}?F*JS2q|PR>4<7xeK4_M<~bkTpuoJ4$7eqH
zm`*z+H?46;)v=FeqMs+}e)=W9(2wWu(c|C#-G9R8VK*zI{$vj0v-@?gmY-ggU&E<U
znz*1bFZ|CjH}+Hdf77OKFA#6C%i#Ey-?AyGT#$YJ9q+cJ6^y&ZE6zOV&@)gs{m|a`
zhUt3B$xjLXkGfASS`l#c`u=^&LUnJKJUpJJz|s59?R?oSrveZ0x%_@XSvxb@+gRQj
zc6Nvutd+4kcW0+p^u2}AuVkmaQ*_K(IcwR<2a`@+;r*C2>3`<R8&9W9%x$P>5fDAn
zFOei&()H`-h2X7+WOZ&OEqWEvc||o(Y3@v=@`{PICss-ZpQwDMt8{9iqM6d%Ya1N@
zYR>G<5^_56-Ab{w<>RAIA&-spo?LsdnCX+4&8x-tW-dt0I#snyV)i44*wxy7V&K(r
z_rB>!-(R_MNv6lOBEut3WHk4@`Ydon&Er6pr}S3e*_w~vwahVkXZgDNoQ$JKl&+iL
zGSAl)T{l0^OAeP^A8_q)*v9z0o>To>9BV?7r7zt7bjwkDiI=1W%O|7ankh}PCFfcv
z==x9d+VxSZiPun0Z}o?JEW4G?_#b_JIx^?zzHr92dwc&S@B5|gAoXPbqyO3Vb?UiG
z1YPgt8$8a>D>-e~J*6pbiCbL)gSUnwvq9xtKk?cW&sQlPlVpxtnRLu+KYl6Yc$mq_
zuq`a*?CMj5YA$pstEObHd$U_D<#e4O`{L!dn%Wpz41Tf5x0eK*__AjC%S*yt=g;>x
z9%1E?-s0l8g-Oz*LHXVW7vJ3b4-+kmK0dz7CA|CX71a(8Q&Y(w7czDp(w)F1zC&ql
zQ?BVH&6&S4wad8lre|t%846c#`0*pJ`d_SP?Z$bstLivBBv*fqJXARG>)RN4sYfP!
z9xIQ{XM5*sp*EY<jCI-PwE@rMzF&E)woSqNAe+ZQ5f4WHgY6RiNlU|4y*Z+(n6@_0
z;(!0MwoU2vt4i+8%fFNSvS^dYubhr6OdoAKoF^P&dGjH#d3|2$`EsrIR@2mO=6zC$
z<EZQs`TVJW1&8WhpN-NS7ek(#-ONhJGZx(B#qs8Ehpo~qcOUie+;3MxXPbKG7pZS=
zcyVmj+L`YkiP_d0CqBNmddmaPZ3_Ht`qsD9ySXH49w$2*aBsZwoh>Q+?<4c(Q|teI
zGynf@{eQL}6`kJe1PZF(E_kQ#L%#U9kJ%2_;1V}?i<ZkvL@M4?oj=a~X?mB`+(S7V
z<Y&FVt;Hf~b9|PGD(7!E&iMu>mKe03$dI0L=BJ7JgkL<88V9#}@bLfM5ZC5+B*!q!
z)HKn%{~S-{gctvh_1`(Pu{5{vvw!Q~4&8{EUv4pJPdlWWqF=-CD*XPzfL%<p<{b;3
z=_Z*{7+n9lKR;J+|JqF9rE<GXKQ1`_;^F+e^EN8ax|ndy^6?ZyspmiMw@EHY+WGv#
z$Jk9GFIbK*SX^0kk85wpvPItw#aTWn<gZw4ZKh!VjfJIX_VbTRgVxPvQu{b-D|`97
z^zS>i^2WY9+pe~}(s9W<jU{>d``#U!aPk3*W}C+WF_DW7CpPKb|D~%uBd@$qXYxAz
z<+pV{`7d$|)vUSt)o<&R$&Q)@UP7OZeoR^G-J!d=yhS?as>pM;)g~g}3$$-e;`)5*
zxJaq4;L5lev!p%0&cDspal2}>knsPO9ap#I?alTI%Go>Lc-Mcu1AOAYz1G+qH@5yi
zCHG?6Mu`_w3z>2~y|<~<s2lug`gieQ>=Smsn*V+;Dtt3~gEI0pO}yKJP6TZ^5ms`d
zeNW4Do7Qxb>bVKWKR$6^zJ_6m#_7$M_B>UZ_FwkZjY(bGcb(^$toO0TC)uN`o!{sA
zmgbNdTIUMP)>-%by8HZvk6F{onQWUkdvBNBkhDC;V{?60UICYr#m*V6Eo?=9LwC1?
znj25zciF6eYpQ7Bl{!C>-L=mfG_TBJIa}KCV3Ae-<i<tS^Ja6q1pc($^?2v~XJ2Fc
z-K6!EX0X0`a4o(tCNzcZTdRPTx2s*HeA(h(8<Xt+TP0hUwHDVc4c;YWWA^-H>ArlW
zG~s#a-aO4~OkP*Jytd@Hl|J>`60f*8ZM|9BI<22-teKkC?2_$&KuqBKuJijUgzP`g
z4x7OEUB}4!o7w-u?zLspnXH1|8%%Z*VV^S5d;crJ%_}2*3%~bqQH`>V)BhZ1%UPnm
z#=~ci%Ae9*T{@Z<zaFXIcm3mneMWQozyH(=UA5vn<9B-pjt84syBV6<dBs9BCOmX%
zjZtu$a-uc(NSCm5+LI$66eRsp?b=u@Ck0QRpys(u<YuPI>7dlOV=b1Or!>qm&wX|z
z^74ao%{GEuD=G>VrfWto%dwbMvNCA7kGjKLp{cJzR)=o%TDU6oe;3Qz;4Yy#KMa%(
z>dIKKiCTPH7}b({XJ2phvV=PeQnpP|WSu^R!7T6pEztv~9%!_*T%B}N<;R0*&DE?E
z9B-ToTOGxerI@GM%JVdg?X(P!t<|OmqtsgyEZ;8cnH`iDbF=d8DPQgIwN8h(g>K%e
z9lelsOJV8iDX*f}-MaGEH@snW^SqbSCa&40Hr;HBzxOiT9ldkI*4&-(jH%{N1*7UI
z`6o?wyDGQ8b2Z);9bQ-a`(y7j_TBOI_5Tt$nGEckc#3kmws2WZuxp>8-Ot0Wxa9Rg
z&daMe?Bzav_=D5O24h{j3y)qoY4AIStafg(nv%2lf>}@?gW&z&=j|6ySX8)QYWn|M
zPFzb$o&<Jjmi-CTR&AK$(8<|ma>+xc`<H(M!|!hHQ@S4(<?njH({y^l2bPSby%*+O
z4)T6pCAF64%O$4{9wDoyH~Y+AOD7xt<}uEg9QJDGn@!KvZZMhYd1pRW)w7tPYhskv
zyt?PcgxPD&_KL0L3AB!3)C+Xl%AFs0UEu7bfCU@=c%D9SYJ+owedgg@+Y1+jx3IKu
zS%uhs$#~mz%{aVvD@*kC*B_4Th^rN}7rNVIs3F+vdtw1!LT&!dw4DjUN_<DOx&k({
zEh&rIxMgoyRG*A;$W2z`pfcTsI$q*Ri}Zrd-)&hY{a$H;0+XjvmGA8n&Vr7Qt{<Lh
z*L6~D2HXFAD|`G(s@^PbtxmF7-o|^YV}<S%CoQe+>AQZc>fQLLV%7QyJd!Ev7D=s$
z&R!YIty8!^w`VbHXVd0=PhN1Dt%%>O@^tflf!C+g!+4}yr&tL}vt}^%sLjfl`POBZ
z{!x~Qo++nzZf!m->%B46Iy~w3n~*mRpL@-{4l6f{tYcMb=DESr^xFJni}JgJMjF+f
zJZpD~Jm7o9!nJ<WZJ({Fd#?NCzBS={os@rd*4>~xA6qWiyjPfgb<rQ^q^E}?4Ypi)
zB(v~D^Qvr<oK&4!x69t*4-Q-n&6i!2rF^P`o9p>J*+cA#i#90o8o!zQ{l!1td0Q5I
zQN6eJ(M0{VNBdYRs$2vo{}1Obm9EUmI=;Bh=T((#w}{Y5rA`~88r7)@LSI{ccwb(?
zyiwP8(|6~iHa~QpvcKChYq9Rz;_tk#yz7_E3{&6QbI`6lt6lfjM)mc%3EYP&uf$~t
z3vWw&<^Q<z!JS=g2lbw^Szdf4u+VRdW_Uy&i~6GjnM_B8_13oYsAfE@O7K`6voSzn
z%I=`eJWXrF-+Dz@eOdUYNWzMNFVNsg*EE+6Q;%%W`KaSn^2q6#vbCmFijvF34pCL5
z-7j07)Xw>E!sFA^iTcmHRW&D`^k#AsZu)sdUBFu+;Kq+*%^quXL^pQ@Sxs~APjyYH
zebXhhN2yU+x3%+{(UT}YxA}#MEJ`jV|4(O#NKV(?#%R>C@kzq0a{2!|Sq%$kPLBO{
zNu1H|tkH~0zH3|DjExqYwMv>jn?Z|N?bew-o+)0pSNf$oK5BSo953as{cgdbYYlCt
zJb}G0+=6`~wzk`HCdTj<hswwL`W3NlocZ+4V#`F!NwXESm|yydpL@yKF~_%RQs9O5
zeQsS(=dcQLzdIGYzT<-HyiaTHzOQ+1&G)vr!eNX5rWx#uM0-12<)mIX-eppCdX`ar
z^KsmSUF=IV1u{JzDBpKIYTjD-Y`J1W;Qh%_;*v}U`(5HLr3E~Ap?!G*qwadA(^LJP
zY|cBSlfh!s!NA%fFz3-pv1KM6+EELY*~N}GEsOGH|F3#@+vOu}3@RHQEWY-*Yp*O{
z(TYmWbseQj%P+mMF=RP6e_hsD@gTpL%*M=Te#{pXRvnGV+B;L=`6RKeWeqW9iPv@u
zqzG)ud0m<oc*^0Qmmve|v<(bal8p{p4Ckg^Rot}r#OZ5G^}0N!#qPY_QJrS4uYc-w
zoG9nwOpnG%{WI8ljTPT|#5EZ-uAP>U*RuF<-0Ne$mp+#IUwoYFzD?AgOUScHWC;@k
z=ZVF0Qd93fI&ezpkkQkkl#P$Od_L}2xx&NfSk9C?GZTusOn!>2oqX%BO4-C~TtN<J
z4VJ8ta?&u{8fmwpsbul3oOcUWeVfOn_v~j-5zD1{f2O$?zVD2wKO6erGK=emUbb<}
zJ-$b6)-xC*nA<LUrl0b?Wa2qT^xlh~5!<awxLO<{mn?c)BrbUC?B%i_CL8TMnfKJ)
zQ?F7kn%M31c+b<b5tYR~-E|W-HNJTn;eP)y*AJzri#>dji}&pP(6-nl_IS^fdQP!3
zou8I1onFCV>-Y7})Vu04ecg80-At8|*0`*?V3j*p_ou#Kb5*&AIWMixWj@TvxO!;g
zrhvBHmqgdUvbnpZWtqY;Ep0yemr~aa+#~l2niS2-lDd`C-kyE;A^*)^58R7xMsgLt
za+wsdSU0CiVb-%R%UHi0$tsP|?KLfRmO3dGy8f4vTw=$OH?D3I&+O{S{KMg$cUS1j
z|5b*4>yLUKuKH^*MK4FgZRgw2B5$MA?lanZdDrDnWr$%lSUTaslMN>m#H|&VMV)uH
zS(dQQWNv`elSxnbymo5VPVn!WG$}<QHMOcsAUem9sr+|eWml%$OAmKj-bb+;5?0(#
z)jE=<IZeh)Smu!Wbv7HDmjSH4kK6Xh?g;v0TfwtBbAI0a<7)-wWLx`XintdS+pluo
z)O6wHxzZx(Bf9PCwx27{*UXccWIBJoeSfJy^3sk4A00CsEB%tUzH(oD$C}Hvm*t5{
zYROAivH2__3XLpN-kf;tUDV(zmAAFvrL}$O>9SdempU&zUd*;r(W&NGo9^=4x{l4a
zYL{)E!Tf8>f8|s?*SWj3oOiwJVt&2x>XWKc?Hx(QRw2Byv;R%sf9c3Fk#1L;t~l1&
zFZQsRhZ|i9_I7N#zUPQ#`S;y(b49D<n6|9FZBl!@bwyBvlf-(9l2^@sW)b`OrhRKz
zTPwWrY`9aU$wq~})870#_tw>|OeaF+Uy3-(q(6Dr;^wfv756lprMY~^mdoys5B&Nx
z>CO7Uz5`-q()-tM_;;|2yW-D`yz}dG?)|%X^W4AOf~B%=l{w9n+H}`7_5EI(X}akO
z<J^@cPaoHQI^_H3k-`mz<$u_hce(98<(=r!xO+yM$gyn0S1k8@GW|~^Fnu{|6ZdrH
zF0OhG9Ss4!3vq`(7^xr3I9%}G#d)TKa`p<YHy$a|UND&jJ@)rnTJX&~JV~QaCa1o`
zW4774;yJ1nZhlXC0_OPW9Tri#y`qEjVW*RU%d?EUXJ!{3{NhYsaQc|iv141-9NT4Z
z%;retnLL*(iz-6SRqsi!xl*ubl3$I%suPR!I7Ji~_Z+I35TKC45+lT2_ki8ht0ltW
zsI%SS@Fj;=-eA!^rFZdO`r8}l?9Qz_y^EzWEAfnjGnYlexhjql0jGAR4JIP}F;&cJ
z517urds;Wk%T&hIO6I77*3&Hv>n)WQvp8+EbJ4KL;S6`s7Y}^TE-kq3T9VG0N0%j5
zTy9&P^dkRqQ{a_pFG?41L@ndE8m22XF<<BZC6}H(9Y5C@sxEr3yzOGosTIF=oKI-V
zl@UJnJ8AXrmPd|f&i7Y!YcsvP`9Sd20^NU){FB*KqP_CCR-~?<F)?7*TFG>-F1e3-
zztThUvUXdX)HTagXnr)=u2bX+U*@u85gh|#o<m|Dj!ardX36BRsGea{_A}*sV||N3
zujO#aAw7$v#|0_MzOyuE%s8AK@MQIy<sF}R-dA0`cq8wS$SRK7PaHp$9VQvf{ZQ~U
z?fKk$okw12&QogP%IurJ>cg{$_w@;9o^M+9oTV#b)udy~7JFU#z<bv~r&4=I56i1l
zLKB1%_?GhRcvDc-b!jofBrh4|1p+;yuP^ECQL!j6<WP3L^Z%HE`fWd@{jcqw_zB&9
zxV*rbYsZ1lHGvz24iyXN*Lx)R+OX)VeKuiXRa6rWjXB(SB4I_sqrfOfvkN*GHB_xK
zlFy~AP!oT<;>?2ZfTJJIuvnx7ioacRULslCm)}()S*y73yiarF3#A`3CMKnw|2E;X
z1o!C_p6=*5!QX|nQzg8u;(MR2>e#fQxa`5xO)pwEA4}e|Bk)(**=s)B2@GioW=_Ak
zmdPAGzct~t?cHK-%S74!mcRcz<et9&`z83_3&UHJ9GKQ9S$^){!KSqHlmE`3G>&wy
z2O+O2G)^j%YUzm?Wb~<JG$p-td-ztOxuJzk$-I$=BSTniqtO?ZX&#M@|0ggzRbF<v
zlHwip*fivEh{`)tuT8E8*YT9C3~4xYc2;xRn#EId98KRXdvxe9uMVG0(1p5By_SB>
z%);&ab<WK)csGl|_-?~z^^GDL6Ak2@wRFzCyfwq%NLKohWgC3H`c~xi8BSWxz29K+
zs-R<sCTPXx`29ChtUJOh`hfSd$b?0G{<W+97y2<hG0<ff-C^=dee#6GuaZvYb@I3J
zPJhI%$Rudumh3B{e^KuI8?Gk7)mvukOfh=JxI$wiug*v7ckdK#JTVIN_!ZOa+j3Y%
z`l(ey>4kldjbFUUzIE)Z8EfmT<^(BLG5d`0)QP5fOCy|}*JgeGJb~5M#U-+IUHiPP
z!T)!k>@Zk+Yr;l};w9fWIKNK`l$^QozmKof(W$yl?mhFe^EL?na#Q+sLp11sq0Md6
zUsoo}ybSGobmI4;MOJ;<Df9UL{n={IIL-d=X1UE<b^E<bzrE6_+Zgq$|1`&1yA@B2
z<u`a6ac1l~t=H$KyXQ^P+oZdCB^k?-x!p`O42mxASmu#9rJ04--0!K|`vjA+$NQ?9
zvg{UHNiEqcw)9ig)uwk2Dw6AXkDT+6Y&svp<Mr`*A|wC%9a*zxe0wi-R(YnIdFkZL
zA8Y1p`I4o*M&0y{!-os0A@8zRFL|1fm*ag#^K6-O*R^N=CWx&SdU4Baa=_!~RepKD
zAG!22@gDv3lEIYkzi31L->`t^0gOxzy6cy7T-B@<E9JL*q^GWY>POotkwZrRdwf-t
z!*~Lsl&56obr=b~FwmS9vVFnXT`g?(#!XI}gS;M|+0nqp{@kkSi}C3vt0sktC4O9a
zZb^aFfr6gWvO>Z0dEs7_(^tPawAQQt%OcUJ>)RsU9X8L`d9EbhS;T*3(q#$m?90<W
zOPK0zVEeLRS!!z6A;YWfIv1AyxUpyx*P~4_XKh|Bz50yRMzPgq@p)0jr&|tf_PJy%
z^YC)q?c)0Lh60v}@yi)@d^1kqoMHd8Afc5zxhTh;>$NA}6#350dYcLyMLmo8QW=hR
zryFQ8IePxN!mBTltf#tx*Qmtrf5WQfURxZNJxDq3?4Ijn_9E%*wPTiS>E=IlExaC@
z%}{PqIp-mnZt9bKNQ%X`>UiOtFB##kUY13jt{Xa?1+wN^bkzo^%X*#{y69}>kiEI2
zdt=#RE)~T?syV8UI6cq!oV9SiFs1Tw(t@l<zK=zCZeOW<HO<evS@3;W{(q4IPSXPR
z(0<Vc0hf(*n-}E2{dh=7w2+%cUnVrD+St2+SubYADIs3L$4Pe#!VPj>`MM?YL>v@*
z<#>i~5wFIwF!>bL;LjzN8Qyc4eRdo<>vh7X%p)?rJhJekPtMV4*}-3=R$uscJf3L{
z&lv^QiQB*GSw5EvRD7Y;<ofpV1i>rowpRV`QQBbV8@*(E^z^{!d4lT?mB#2UicwXI
zb90G1r4gsNRN8fs^y6~1C!zXZuFHJ=9@pM0b9}v$-LeM8Zw)U4`M&<_7k1#=&ot|6
zq5UU+d26ljn>@I8XWkd_e0}ASqoJo-vs0$r;&h9$=JRL&zj1!cp`79`VJxtCc?pYI
z%t4!^%r@VnysuuHq%hmeni6zXtatji4*LQ}k>$=SwlJ^kEaLNd)Z}FG?ttaLb#_03
z?7sBa-RaWro@MCbUc0=dd+x68_Z#Qz`c^8zr0MK4?;E$tscp7PKIAO^#Kq{b^hufP
zt%Nnaue8{+c+3~+e`)2}kj}(pS#Y_i;BnPD=IQHLzpwlMeixtfyMo`zdY9QYFskOv
zd#rGPZ;|NHpjfUut5%=nGB8}xy^P1nPp#Td<F>BaChx3;#rBJ*Z%bg($ZwBa+;V1t
z$QgyTU8U^hsqZ=T!sm#su6SDEsPbV_g|KhrC!x37c-IzX7alG9{6%J^^D;B@*U4X<
z(k&KlpEdL6w-wXBt(m?vzkk;D>;FpLPmnpZ`GRWFrs7R3OuKJaSzlgO`&@1A#YmgW
z9(4yd`n;KLo6KEb-&_CryUfcD%}<fHzD3UeJ#+s1=>0z{Czw22zv8U~@0J}2>bsS7
zOzlq1o^gCvN?Wk`NvDf?i=53`<eOU@MGOOHygT8*$;ZBV!jHmvoNZHfto}bQqN{VI
zr|pM{O`M+F{bq!7UVL;|G;Xuk%8+fjHjkw?W;woFSt*k}TS`xwX+vk_<8wbW*G}k|
zmfm-}!td#hzPU1e53l(yjp5;E(&<xK$8LW5bJXdNvrccfS$~>+Rr8(iYEQhmr1efb
zDQ-TLeP&OQuwm%z9(9RLGmL-wa-5jVx<FQ;O4!6`QnbeVPkiTRecj))!sLKehmLqx
zz~&k2ST?tXed1DTnxQUo_whTY=Fj&U488R)T{^q=%8ZVA({<-`%$!}hqvFWS;v+LB
zZoE8cVyE5O3BM-B_~%}mA9!2MqV{cTcgEW}`!`;H7&`a*=D6mwb02TEt9e$+RFUvq
zdjDjF{r_kGz4_bn*8Et;e>*i<Zh4AmUzs-XKcCF=Kb@1#=-x3d|HJ3Z^gDerhx48*
zN(MHZQ_OZbSXDJwck!qwuq&pwWvx&SzcTUe0g(>V*>Bo&xZDFzJxHxxDw5}DbFX#7
z>1MX*X{L3*_#y&U&P-b|b$iFEh=ZCw-3xa;-fXr?lu6G$>k0RwBHJf%U(%U(-|G4P
zl3zL`t>PX3Y9sBt+wu<H<@aU2vwMSDSG&TNL+ke1oECO?(fsc;GlS=?PbGh(P86r>
zSuc65@ohPeklv@Fgs@fbjZC*kPYLEMS|%rWNcq-{`d0>D{}dYwJ$7M!U=n=Ms>L@T
z@*>+>&%66XT$KZE@f`d=Q|pb^IT7*eQwuqLBeG_=RIj{qfalewi+M4Qlc!&9_bB!{
z|A@6=oy%Nqv&Ji$_a*#gKX3fHbN+0J<>I?$8h<HVFR#2~_tA2x;Io@|+`bwlcYSB@
z<_llrPR*`6ZM^riaoxqwzdu~JeYmq;RxRH2sz%-Fo6HKgy66Aswf{NWmSa-=1d$oH
zn4~OO9jXOa9#Ef=wK$|MPh-kD@5wed4%$}y5pzBvyy}qhV!sl#!)x1kL>^3Kxc9(b
z#q8Zt>mX%gcJ`%-kHt0}W|LfJ$#7_@v>wx)JDSQ44xD<a9BMkn8x~53Gwqn7Fgd_s
zWpp0Pl#P#<B_3*J)t>fplA-dUDLRqUGX8TGFF8EJB)F)da&xNRG(JJ4C!1JTT+nph
zDPWsrqIqel*KDKOS0-8^%N#q|_G*b}i#Nq+SN+))q<VNG@99al*=D*ow%TRyN<FEW
ze5a+*aFS5N#&r$5%l3b|+bv!Y-%?-cHFJWCBg4@iW|fFH7M~uUoFH!;X5@Nm+Dczx
zr?op=9bU|ojOH|u-JHA9ELC9AgHKiJod?VfCn*L$%X_eWPxbG%m9te>hJDT2>NI!4
zlos986MN>(71~wIxcM|kN!&7D?YF|Wr_8#s(Pw4P{n*Ow@0N*Fz1ktT&C8!D*{Pk|
zdx1zoTax65RIfM7R|I(GwS>ou>;FujU%OxT#_d`jyBYs4g$a3`@C<XvZA@&F%Ck6{
zCRcVtu}!h4;$i6Q$d1r1C*KoE-Fo|WOa+~nbR+Cjp%Ysl&%+a*lD7X|B<DF!5>lDq
z#`aQhLc&*%C$TPSCUH}wd^J<2hQw*API>avOfJ^Vb3qFer;nHmQ>s^3+N_jWC)1MJ
zeHov#Da^BIog-JJy1Z4-X@*eV659}A&D=R^4BEOc(=wbo(?ssX=CFjXV94Q^eo2G7
zFG)jkTHYl#?QI)Ex*2jkHiSm3{psT7vC1x`y(;Fzb2sg6Tf(|@*LJ7~IIg?VC@oo*
zqkcelomu;8&x7-B1$yc8YBXpnPTe{`LVadR;8yOrA@_Z59oq8$ZH;8<>#5sy|IVDy
zELeA1ch{q3A=A1-MZdlaE;&~%GB@b%Y032W@3ciS%!H1zN;+1GT!~1T@Ki90Yx&lK
zLw1HwHh2krzB#F;@g?hvSdN!V3#AKF%dcliF>f<W5cM^5Wa^DnlVq5@^NqxrDVl1n
zX|tYYU#SU*4Gmlsz!w#~>A<G4i%y#z-RClFU3%roVqM=`vyavuQYrPG9dO6XdfS#{
zsjMYF2VyEW9+Q%<c)GQ5w!_+$a|6nEJvw@@{;Y4vw7M57)hD>WSGAewuF4~Et%gG@
z<v}f%R!U$!n^H(c6C>Ay1v@1@?*&Cn^q13n_B4yn(qFgqfxwFi-Fgi#VkhwZx9S#U
z+iE2~JK;rD+xl#YMRTKX^i{tNtE|`^TKSFd%6l0Rn?GSk^Fs}~jlxV4Tca)Q7x$h!
z&AxG?nx(vovy0J7VNHpjKlXl?v(<d^<L>Vza#OTdichZ!+hw0x5bL)x_3xMa^1%h7
zevHeQJbk{|AD`3d%M>%2fq6m;=K>y=vQ?Ac*OVJQRP9$Nekgp)<&yL>4lU(?WKVX@
zQjG@&tCDuFJSibzsu7g)%49xQ=nA1rT$%+7!`}b&cRw&$FnmTkk5x_|pKnlCnBoyW
zPGJZB(+aDv?a>N1*Xda4b7)7~9+u|&lO9FH1qn`4@atkcl;m^btF(SqUd+oV@f`<V
z%vS%;#iNm35*|41lj7o<6LA)+mG?bpRkB^1;hN1eLtc{YL@TG!gj`LPqe(#rT~4J=
ztj$z8$-?Y;w_<tA73at#E#;ZhR1c)isky24clu0=qGu`JwIq04j|x3u=v7_)v+LrN
zZ`0I6-%b$N6ymh*o5G_Trz|uXXWuq(+WvjUvZ}{|9QR(7nZ>rM<dsam{mAK3<5AfY
z4xPaPk2%uMo>YBNbmvWsR{)b=e%It_`3q$%>pw5z$~{#2CPuw2ZTrHCOM1@AQWI?~
zOU~Q)3Ap`f3G7>vzWC>p73o%|0{dD@mdL5TT)^p?F;Qlg%EtpwY9m%<OnYa(zsD}4
zRG8!P3?olTrnZv*4c(QKnanRAnBwQiz-Qw1@mr_c{Vj<)rW5@$PF!evD3BFs`cRYI
zmn}rX=~N)+MpfT}nSwqKpJs)B4)N|&KCEK4gjeJtQ|Q|ZQj8pip*t^~;N8FYh-2py
zb)|)whn97S^-FLRvaa%X+oRz&<G|Oo&uqjc&%T&uyh^*+@aXGZZ)VL}FSDk%r8K$Y
zNyPdiN@DRPXCwn%nvLrsRW;vp&FGqtzF1dgjpA(CCFz`<8#cM{1T6V`%F^4y)S*zv
zK=E|uACpH>6ByO>b{TRdg^3yaKif8WCojW{2PMZBzgVL=RhM(2k&?-bJExTYHu8$+
zJ(9X|Xi}a0pG>DT&AazPzU!+0f74sGt!++APTQ)Nj*Dbt6a~9DmaO97^sM!3RnRMG
zD7d(V|HYf#JANfiE~wLvd48yJS3%x7Rfd1-8_lm@KY2EM&T?e~pEEZf6sF}JFcrM(
zFXXhVAaUZFGr!gQZ>&6bV{dSoirhz=bn)GMoo*&dhXUR;eRo(OlYi`F^4#tP*KHcu
za;ge1UUMjq(UN_dws{H1r|$(SH$Jw$uaWbd?^g3A<?Bj)uh-03tWUSA{?6QD{)^em
zUDV)!0?UF1c9qRDnI3FX@pzD?p8JxiS4sV%M?h)&mPgSnZfy^9E-W``6tUd9owG4j
z#Y9)}OB|bP+v><u!7C~sW?D^I6&8@9v`xYLfAqPcwYp)ayJAYR#diPL5uok8E?@Vn
zhez+^Q*V!`O=VWz$@KkNXu8Wa&V^s&x-)LY%wEOKUwtFNR664<*X~|1Cc&i2<D#PM
z<u}WgoRZ7Fv*@5y&aDM&zolQ!@`%u1(WLp%OqsR8@LJdHiLH|6D(kCbr?hfTo}E*!
zUitbYi_e3!Gje)C6>P$}%xkmPICfn0Xg2A2qjG-v(xrjSqE~tfyt0qWZV@^0!0yWq
zrD*P*`HxrJyw>eqQC#}f_qc4#hY6qAuB~$uce-^r#=bDjN#;S*91)J<MIjyTY<~+n
z;@dy#KCyYOc5SCke7b)X*WDY^m!8Z0u1x0AjIF%)$m{IE|JxV0zw20NXL)JTsqn9B
zW=~rb#wD4sqtAX~^X=2G%@%lEZCS>$;-_zMiGr7GhB{-;fq0+i!lIc;voCOXEq|!%
z>lSl*lIzw*f%^j$%Q6>*m?=yZ+2+;3?a#b>W8+nw$U9qHZ$!Ofp6dO1d0wZOdW_qJ
z|6i{yk>4ACbmoK8bLBQPy!8xo)sA}X+sG{Zpn*xPpqkag;k(x3y-XqnN(s+vHyqS^
z6JS$f9C%JUuG@6m=b64YU-*k9oZmLFMJi~Oeb)S_xm&k=SH3pwlHJEGVyC%P57rtM
z_CHU4A%FhFt@eLC6BtvqlDD6^$9uqZcV6$8I=f&G%ShJZilrO9I3CI-x&CL<4X>*(
zx8vLxtFy7e|FJZ)2Ez^$ws2>z<OIz+HBHTpoH+{GyydY$4dN>{3Fv+ovQ&(h+?ep-
zlDe+AbZnvCs_lh~ofET<O3Q{9?fs}OeoTnFP3ZQaMAeF-vmd3{jcpDK7k@U(7g~_^
z|8q;0dTYRnGQkC`eml~H7O+MeaIrnLHJh$%C2no~yjYgWKRqHTbw%4=<q};soBR`u
z493ZSSZqsolw4#^VN?!ax|$sDT-h-#C3tztv&$(yYI2T;?EJ6Uy;c?qJ7l7qV&5`d
zMk&Sqr?Pm;cZuY(GKM8(*~^Mq%Dv?;rB!Zn&|sC4-r#@lS?nSsvCSJBwkg(Eng6fo
zy41+I(W}<X$<L|NFj3lLs!-c8A<LysJ)2B*6S?OIh%}mnZxN|`5mL2iy8@?}@V0AJ
zuQp{jT<YNvY~OO6QK+GM$M$NW1xB0-eMv!mPgbP8(CB-;qVI(VSM-Ax2_=K%4arB+
z`wyin>0VPb6RtUKCi>-4&4uSV(?e@@lyZ44*Ip^hy(b=(qgZCJP<-`Mj~8KiPtx)P
zjp{WI8@&??@_JykbU}LabuNXK{fs+BzNIN@8hYOUSibtASNjYD-R&mk7ezPh5a0Q6
z!hS|m_V3c4J~l>O=Ic{qc<aEX^1`;-Jf(`G@Rp*`_rey&;whD7a_&E4qc+D_dU{_v
zAtZk6|5TR>t@ujmIujfHa*3b|_KBaW6NS_|CrV_mWXRh&O~^oI%>k~(4o!jrtWg52
z6)UF;Zs3e+&<r$Sdvky*MmQtzXKQ6zz-yPblogUGH;O-$_)q4vmMl^X`xxZjUSiOc
zEWF7kc4PAO&5>d3cDzSzm#-{+akxYGB4_xb?#VAYB#x?b7m5FV+@gH6({lrBQ<2Qx
zo3rmN?q7Vup}@UdxJi0Z(VT4^UaL(s#E%Gs2S!{Ml{z#{di#%#$rVx^i(+;)%H$lF
zymE{9{g+~$+bi~{F!Y;ONEFFzOX6Oaz`gyU_NFB6`9@i{9&&9`%KUM2-ns*tn+|ck
zUYR?`xtVo&_%ikXRU6pF1e^tKTQJ_AQT^~n^>dAdPdpaBInl>7fq_$zkyDX%+X1fV
z1)N(Ca8=#pS`)ynb%ZNffOV?@SMy5N|B|dR7Hn0|l$SIp&rVp}xomOcvBi>y`cE#G
z%+qiWWt?#Cdc;{V(Wz5Kq8Sa?C)eJ*9vQG|0#9V_gXP)6!37VuH3-hEQ%$bxeG=6t
zGEp*8^2c-ri6E}yhB8^VW}mQ7@A@jD6Dks|sBYwGQ|FShed3gZ#n$;6QZ>8krCkg~
zUN~Mmn(C=4bM2(l*8&;MgKV=DwZ9%<o0r6&lqxqds&bvgN>#3=UUlua8^pII@T>k3
zV0qZQxGhoVhihf1+oH>&+8_T{tt}LcG!^sQ;lRbB_IGi$+}CMImsV>DPcL|BpRdW#
zIZ+_1Y>nhZ7F|Wg<ASW50*e+OVEcN2>&<~hix2Q6GpthF#kuwXUqz<$+66NzCeFBb
zxpnb}&fcA@lQyvSzbt;$)s}X`T8^pY3rq6%U93l)z0ObV^89X7uAx|P(fISn_F(hU
zJH{OzOp{xW*zt6C+<KNG$lV)VW<Tji`g9A{q@uKmjcGyx8T&Xl9lyxta6L`6T11`I
zX2wy@UDK;&O&NtYFq#XuR~~UN6W**bwMJ`NaICQA^h?35pJs7BZ_H~9NuCf~nkl^`
zK}Pq8i9{0jQAe(}B<`gNhPeV-i93a&FZ^fwn!xS8(^P(zY4yZ()n88A7fqEncS|nm
z%{iF8>s!p`@M?|AoA$Xcd}h%n7PYXld;3wXzM9!<B)N7Rl@gYoWg67L62-7;$pik*
zMT?jMWC9gfA4x6B)-q7~*}Ak~@sb0JXU)!-!|ng(*Wzdc&Z(T%`M1gzU6JHVbZ790
zVSTxzE~WOGx-zHd()p=N8Gk7x99bH?xa&pw#QoBqH&de(rluwTo_{=gQA)Pdvf28F
zzv}H>#gaaA^+oT!kr~sjW>34Bz4!L-f?XFK`DaA&YtB=AvUhi)=Gp-680J+nOC;5z
z_&Gmn@?I<o4&vwiD5cL>xM)^JZUTSo?DR#mg1=|Z|MuWNTS|e2=?>imKWBzlX<gY=
zRQ9XrmD=8npQ{gZtvP&Zn#7&eIhs4X6d2^*9L(Ri=46!cX{$pEe@*xLZPjBXGdrQR
z`FTs{wzN($DLHH9>#Z}oGS_vMHNJmUJZr-`g;$4PStWf?k(yNL&ug-tW#>#^8Rb|(
zag*?Vd#6&bPXSz|N4dI>M$B2TM0B>uQdRMzjS4P)zaQHxE=`-WK=!2crmS1*47W>i
zsi<pxT`s(2dwI|H=Py=Qtxnths`Kuu;F@JIb>BlCXl_w6u4pUwHuyMqO@VkUN5<>Q
zd6VbNo4DFkO-noE_nffbn(W02QL9ZK8PCsMu=TU2$`4PCM9J{X+rrr&{|l_Yezd^5
zn)lTL;a$h~n=gEmvAwK&q0}saCo=?VwHc&m2}*kT9kDrcd>8AokMn0Mu<q*AJhvuo
z!tK6@j=qEHQ<p3_tF`6q;sa-GP90t^yz37~PVS9$c2@2#TKXQljJ;p&GH9*k);zy~
z-Efg=9skdzdloKj5?uD@k9bT}v<9;z7Xt&=hM;22N&j~)YYv~X@XIo_na01p1dhe5
z;VM`o#nYN@qZV@F;@lI{1Rd5r@HREL=y>C5`HN%H`!pDAb~*-aURk?*Ui4y3rUd>y
zLh`{uE3X>G%Luq^D{x$=pp)7h$Fopwc~jGskjluTF`IA2oV{37a?9GMVsG}d)d3u8
z$&3D9l-_gjZ}OTt?H$Lv&KM?MjE@yQVRMbi;F{tYp^LKxvTV;}X-fH?SiNVK#O`0!
z!U_zZ+q$F|w6~W{xiu%fVei>li!`0r6h|$Pbl_QUCq6T?<H+HcNrvCH9douR-!-%J
zg=p=Hn++KooP<g}KMBP2^yUa|e5}4vcVpf9(9Q`QA)lL5SH}u}KD<OKT21ccY^B$;
z|7)(hX`Nq?w5d?%rvKc8#JPfExpy+&N@T3Pvwh9%s%t{sN|$&Rc}Z^6u(O<}{kCD(
z#PUfxQx++*ZauJdSFO|(_0Vt@Rn|R`TIIQCo=z~|*t_|8Z!!16%A9Fc)~v2GkBi(o
zeR9vjBfi2SbA=ez{a<5HcxZ3owdA+=A7^ae$+g*0OKN*z!=WSde$13>-o80-_u0NR
z5AU5>EN;Q(u&4FUiFMHz4off{kr3MTTf+U!^LjtU<z-7m^*`;}>izg@%=vJsqAnNl
zJ%$sXeG|8T_jva~*0h%r>9^K$&E0dtTIR!wv>6jOyU!ANB>ME|+%@Vk4;pVvH)yj?
z?h;geE7nq#epA%j&`?uv?!K3&g+6F#%unLpcu7D^ef6iUP3DD7)4xl_-;}wpH2M5e
zlkHV|_x`HRv1}D&Xl;pW^>lcl|K?D~xfjU+{4HsRWX}jqymwXL&^5`2*Rp-jq|X&P
z{`TeRyZ3K)cNyG}yT>TEyXn>QrLR`@{eQJ_ku3X;vm5nZ2kzvw{bS|O^V(_84VN=F
z_MY3LEhjod^^M;ho0++97Do05ikF7@%vvHDc)o9U)Hc=lIjNf@kFD9F5EL}q?V<hY
zM#0=Qw=K7_=4A$L-{nx8BU&%DVSnPClWQdy=LwvgD`afdZTnj6Yu|grhi6-r=G|F&
zwtLMb*HVqc7h5M?TB{hbO_}4N%c6VNCxtB$jWRs6Kw9DccJYPc`yT{tV0^ivHKm}n
z=Uq!n-;Lw|?i_}*r7KyZ9&mr~cpC9jEb;71gNHAxaz$U}zA9Zb-{I}N-Iusl*Tt1z
z_`K$ywsZffOsPrk_7&3q&d%X}<hZA`>+QowHRl{=ox2qNe+kpgU02i_t{?w;XSu<a
z-4h;!+gi-ae&Vv?+4H6QR)2dxz3%6kr6MLzjAbTy^gVH%*7xj0KmXgDe?iysY}J_d
z-Jd*9NN#qpUvT#x!Mm-eZngbV(48wZ@6d`~F6Y?_j1gx9yhNV6y>MT&u!Va{Yu3rD
zk#i50yj`95(|-BK7ae+>Yd7qeu$!}Q&GaaT8E+nND`%}$o3-|<1KVk-L+Nkt&zvW)
z>-x3Xet!<e-<e(a=ivV8jJ;_^bJjBkzdB?hxGIl(wI7pk!K>wUahLbUxolB5#PYgv
z?!!&%G__J*JI#4*QU0&*=If8Y%HH1(%AcYB$Lr>?%Sv;9t~V@tWB<+Hnn@sL$Nwo0
zRVFMr*u=!a6>wt%k8?YprEQJI0ujj$ZtqYX)dpdgHfif79?rrgu9Gz-Ck1tGdg48u
zDZ)s{B9(ik+1jFQmri!2O9!Pa(eQ9GaTMYVQoAu{Q;B!*>}azb!HB8co{XLwnnE_1
zIyS5oP*qWMUUl_MSoGzjR8!%U=(b#iL#ij)mfYSD;og{%TqfeixVYk$P(b&R07hof
z=?@Pja<y~HsQ3gpL^d^@78269F~`9;ozrVug_>>H=a-i_CA<U@q7F<8`yY|%XtZjc
zeNdFgRjH$^?#>9@x+Nzu+<UTq*_w!}w?eu_Uhm-vO%A;~H8K1R(;Ds>ew%iE;}kOU
zdAR9+>lp{5T(6y(smrJ8hL^=0+;43EuiE_6*Zu0o3w!?kxFfK@mhp{-fqff`kGI2w
zn8pm<ww#M6b+~R@t=0{zx;$Y@pvTRWDZa-pq;x-YyOY!-5PZj}OEfxV;?Wg55?*Oa
zo!lJgCX;`~&|PNbr;Az&tCMsLkDW_?bK>YEmm*J9_O6aaFBYA7t-NN%9WG_<%{D%2
zj0Q<3PwVYu5#C^MR+wQ$SWS@e+089hYge3|G~?0H<tb~5k84UiVzN?Yc_F;Se(7W*
zea?bI*%x*R@CILe!6$k^??6P<IyuH$i5j{KCJ8xoEpd`Qy+WmrsdUX^FW#?L7H`W)
z?^aRjaB9%#Yx2<kzjn_fzpk)Xg??S0kr!LcBeFfuUDBwLcD}CX8Mw7~(UcA6BHMmV
z5mP#|Znuwhj^X8LTm7b1tuZt@rJ<_+?M}hb7>3Ke?_<i&mX^mdK5Aak%jr=Nz`(ZM
zXu`tUk5|og=9b**5NYBzyYV0}z@@O;SE6i3qNrkB1-GZny`)FoGiNoe{5Vf)z0HLB
zKZYKYyv0r|J`%L=)#6SI!%I)62`fKd+^80`BDHD3nV)GIlB+nD^tnCqXz&Z*nw2r7
zOYddIjJ~p6stcm6UM`tg#x-O4d^1koD?w8uUS+R1!1nXW<i&S*7QH$kb;TscGqLp|
z<HcDo->&_ZbRtjB)9ST%o7u{0?f;v1q$E#1$W`6l6X410%XuY$S$8YPqzQ{cPcCIR
zH{ExM!W|vWu-ja_qBcLeI_<%<*S%8>Cv`Vjgv}B@uszBuzE4r}#0@9ocRLoD?OLZc
z`BC!c3)zc#%&%p?w++v8yl4C4PVn`gzn?tb?!YwneE$K)<+~OiTr2ZMZc+8))D`<G
zwyWzL`W{u-s?McfvU~e?%P$w4jqL6$sS|m<eE-Almnk0~$)DwLD?23I&sCwkYJ$50
z*O7@yJcn479XELTEPB8q`e?~KQ<l7oFPj#f_~O`+kQtcxi-W`aMw`&09##27&);Vr
z32)xeA3Fb*55vAi460{Vx|;10?q7GvT2<vi&dy|y|1TVpTqfi+iq^j5o@u6`y4ygd
zE1HYDYsZuA&JR;scviNqQ`j<zV}nLwy(Fu2gkWoc;=$1Do+dGqg{>Mc4#I0Z`qUN)
z$dqk4B&W2A)%wJNF5R3~jm=TMYrj+o$6s+aXS(3BH0oksSi!40$BR-y+YA`}cvFpQ
zPx)<CF_2v0)5<Ks6E1n;YFGJ5_SiQ;Pj`j=^r~!BiRZDJzUYp^)F?Ya?QJi@_is2P
z_jSw4gw)LLgC{O5+<!+uQ8P&5h?v^O?vLjaWxBnNFS}s4QpHW{$<3zZQ;*Lc3s`Cx
zDycewOG#kX!kkni>*tPHZ2}ehKBw88_OLC!)~Q#sL|EpchrRv(OA{C6sw{|p*%8gd
zHL=-@cOiGHr&Fqu`0B`BZf8v|*ZoRb>sxuX<boC0KQ+0$j5ywQPg2t}OSSRr(x<rs
zi=yM69GWoANl+<Rh4bJavAMIo3=48@a6i14ykqACC9e7{t?S$_N?)q<;NBB-PNsH}
z%(6{R;gePz%~=%CADEG7xn@J_wO!7#PTzDVMZE|Rk}%u3^qhcZc5U$EEuwM{Bu`Fx
zZ75g!MtT0TyI(8L-G2RbN%?%nuSL@Y0t3FwsqE|VP>65db^YhHO}6G!605gsICGv-
zSjgC^U%O7@Xr#?=J08X2y6?A-{BvkgO1*l%jIa6PQ3o^S**qT_*p|G_k@PV7Khx8o
zN%YBD*3xB;4_*l=MXkCNwuWWWV?J}An&cBRj(oWzcIwIIoUGQ_4^$c%G!%B<<~@5X
zD<oRxqvo2XU$5A*w;x$>W=hu~R!$B>CMQ={^OmY?cU^^(0^YXjTtO17FGCJ<RnIwU
z8NH%?eV^)ssyi$PnKTNdxz}*cn{-fagVy>fJ6c83Emv!BJ>cQ9oFn3+m&(i}vDE2^
zt6tbT@7UEW0y@E-lMd>Ai@!MW<CkqgZ2x~0Y5z0|kUV61ZhF~f2d<e+oJ(3xH83vK
z@#f=F4fVd&X?iBrcisG>Ywj3iUF%qUfA6y11Dq;_vv{_LD{ej|q&#7!s@i<H$juDb
zWQ=G1_|KDW^Z2HnWYTk<e>$o@&bJ);k4<WNcQd2e&hyGNvB?`kc(d2voaGuQ+OpWg
zVX@M)XE)X{U3zVhvgFp?u3Lu|Nowv8$SdmBcm6zWn*LAS<;`BFt82X^4u_cPuC=(j
z(W9}&`d0GF(-&D@F519Zzri_ZztdcX65r$3Zz`F$emwcMZHD;k6>6q41<$|V;o0_O
z$?=MH8~o$m-rDoc#sAI);{g4Y%OCG);dgpCS!nms=Xs1*j`Ip>Fz<PKmQ|>xJ4X0M
zLz2Q|;S&Zcc!CYTH>;F<__B4;<|P`ehrNG)P&a;aGW8*6`t2u+WrYgbW7yW`R!%vv
z${=9QbHmL^9V!R!efrk)fB(|-(>H&A+SBntQ8VXVjlEH^P*G>=rBcP!Ryl{?W`*x|
zn&ZHieEYC$u4>2)r9}mWTaUQAp6m>2{~7zQOx!qWY4_F#oGz=h7)|q<4AxCDSbgKe
z>?#F6?yt2A^WIcX(_N!CGsoEXk<r`b%lz99|CL+1<CMc5k=Mt?Hsqu}4DQjBFjko!
znr^e`jxnR&l^F)pef%Y5g4@C@+!(g)VD8)gfKBho41>z1@+kq8%T)M#PWp(6SKs-(
zU`|5yR<nuER99F2{>0$_{SJ#d`;nK`jZ=cpypFkf^4!d4<!blmau*i6t@v_#PVSTW
zq6eSPy`Gc4NH=Bvf1l1v3#N&0_c?pzVg7$lr_)S<K|WJ7+%_hzxK$r`EOW}@P{HL2
z<?O|*)>>93r6RtftW)+}+^=y{bLI^uvmcs=891Z7MdwYB*?5`dUl!}yRVum)y1EWs
zW>Kq8NOp64=n4Dia3@j3sDWQ7M!aBivzf>AZzuX0F7h*W^6!jrtV<NIt~hw4oBQL+
zjYo>NU&_{bcd+YHwt#g7PsL)Lrs7s>3Gtm1xQsg1tX{F*Xu<ZM0u0L8%^x`DOPpEr
zMN*)}dB@kEhrYZ#^!VVKFM&p%Z?s%55?u1y!fcN21ZJU0+QxThDOfF8px(*xG;5EB
zr=F(Z!f#HJf<=1!7VdnSHQ92d;zDk#1;V>dO<;OIOV{bYk&<GQ<rJ%KH7UjalNYU+
zymI>C*=|3bS+7j}&BPhBa;}$?=C0&9ON~X>Sx)`BS|cFRe%WLp))nT5tEZPcu3`Bo
z&iK-`-ccmJr?2kf!3`G-i>24ZiU^e4=IZA<rl`%VWMZTIc=zfKewl8YLmYemH*}=(
za7$|MP@F835M-Ni$%uhT;l`s){f-I`q-Q={&1Gz@KGoCM?}YkeXGyI}<E9hKYmZ3I
zUdUl{S>e5~$AlLOlQd>EI(az0=6F{v)q04_L3zuk$3oL?v@FU#%F<x+{q%{RBMSwZ
zyC!VpnslS@^^Gl#FXz6L=(e}&=4}#~c4f9p@`~v-Cl3lXgf?^dyJ&>`zkTFO=T`QK
zO&;02=LNi^jChY)o?3K;Z%K{W3k8vl51OI_HZo0|UgRxMviYDq%lwktr(W!s{`TT2
zZ%O$OrRJJyP1YyV^$&4<>NID6*+0K>P2=I|nmsz!3e77En0GczcX-rf_Mqvb!Ss(0
zk8SB#S`obGI=4G}WOuEn&(;Y7&ocX4wV8Xeb(dv0pPsXpb<zG!nmd(N9=3|Hyil@p
z{zYTkE2_d$3mvxXZ1Y$+H+kmuh>7PNj=bHq(DjVo`obeji@J7eD>)jj{Ma?g$y3-j
zWsgtG&b=P%H=MAVpdh2~<*-rW)S3yd{FnJ6TBd!i-j=brkLz}q-Op8mZ;napoSklM
zTk!wO(t;)%+Z<+14aN%Qy_^g^W-ru2b}cJ#wlP1@QCKAYv1aEZ#SX1*E{0nyIj<xb
zZ|uGE*p`KNU!T;zDW2+;IqDp}>eX|WTQ5<6!Mx7w^1iMW$5$U%&S%=W@L-pdH|MA9
z6|G!b+Sh3IYxIj&9__y6;jvk<H-us7hiOZHEc<nN?puYEzm*rIf1b1CgU%!a?+FH{
zAL;nHd3raWTIs;VxrbAxXMyy(EY|rg91C2!l(M|;NG^K!AZW|St~YNkYTRskW+`y!
zlw<6jqgz7u9R4Nu-csJY*8JCvJ@P6%878WSvYOS7OxLa8_}1Z6z4G9hQ(jK2^Q|*j
zPyF^;v!g4oipS`G0sk=ufe(cTf8F4plyD|ums8kfSD&+Mj;J`FW9kZ8z;$!iWYaGe
zA+@^R&BrZm`tN@7nY#PJUl)-_PBSgMeXqr=n{)W8Zr1wVLNQkvs|C9?-t9UYymw3F
z&2wumsNP`cQl6{t;u!R`t8eA$*{dIOtJkWm^Y#w=$tQRBV&u^+nSl+w3TKjfgo;<6
zdU?>M^#Q{rp5uy-4d;I}+&SUQQjNv}p1t2c@z-g#?*8NYPsK@ES*PoOP3c6{qs>c;
zOKm1;_*b|_OfZ=6HPB)K$Aw}SciqKml9BHDRXh_b-6y78Ws|-6Xtuh=;p25akygn&
zA1vZ*UgO?jaAE_)iWv(cKML=6{$DLMSJtFsj-=$y6TMR|c6SB#xp3t(%1u1NRS+8G
z|Jfu*viqusrt3`4zs1~pHqJJU<y^g5e)gWQ8GjaNK3$|Dv}oBG>${n|Cf6+5G?(eG
zhR=s7lYS-e9MCzlZ%ugJS&ltD;<j&3f7;-5L0Z*lM$`VohC(YiPP|f+pUogPyTLf{
zN=NSY!;736_Fnn5L7RDH$N{BJ=c)6yq?~EDH9G5bx{;OhvmnD$MIQZ<*}oYTn0EPG
zzdO-fr+>=D$rds~M_({S#PVD?v$<;4RduhkvrdP`+4{!$_Vlh)yrmNMw0H6ni)*hv
z=5QU}qHGnt)?$6w=h$PLl{6Ph9}wvI^=j9*s9kUVpO|~$V%N(Tm$)8ZylZp&C2J?k
zjE2k65w07Vb7R(o8QjPg5jZBHTJFnpBAMsQ8~!gdx*onXKVmGfvw(;1#FcvPL-$+b
zKh&tcX_aQ)$QoDG&%LcR>cOtur=6-nd|#Lzh6eN{KVs5Jp27H3jrr~}=Fk9fx#bF)
zh80te8&wB9?q1%!;KIc>ciS>Hzy2+GCw24G%9{##`(|yuI%UJ8jw3f`L?i_-mI_|X
zd7PtnAuq$iy-C4>v(4|w%7upccE`OoSp98gx1#Ko%F0F4W+|!IPEk9{y{1;=hbA}Q
zvug{!@O_o_bk&UB=pe$m!O6I-WBr~(TYILT5EA&&;l#Su+<XR;(f<O06U7sb8!XNK
zW&OW{U$NV%-%4m_N6O2caz-<FR_qb~ReJp(v-ggj)6HHyW;ob&bc<n9NsNMvxv%Ud
zu6@&^7X%xY^qV*eJ10(lTEXwNvfbrV`|TaQhd8=U`GlsXIp4K0HtKc`KO1ZP<StLu
zB6%qhJC~#<r*=lS>gBPy6r2m4G-Jj28D|r1(-#W&GEM69{Nnqqi8p9Nw`P<6l|7R<
z<s9}M=w>$QSr^TH<R{;bk4ZD6!>gp-qUKGH{B6C{V1m#Dt|MESWrSl+3*QObak}f6
z_<7?^*8@0C%#PkUgKK}bQ+NXpUj#p^iO~^nfeOpZ$Nw&ETRQJMi+e}w^;DA=jsJE1
zxwk!-w<L()Pxz^UvVdRTqYI*o-nCY}U|;f%y*?mo**Df@?^^3TmN2f~+jDo5UT}i`
z=ZFW)k()GkPTGCEswJTRgk*h+-2{c~xd-id{wmJYJz?F<@&5C^C$}PZO*^q`-D|y-
zocz}5eNVC%%knSTn>OjombWEQYZ{_9EfAQY^my};t~I5eTPh~Z>6z_4vvXES&>WUI
zb6GgoJyx23!zL)>d3D;9w+Xz_DVHmhy=!(a(^Y89o5ir(cH;?yrB$ACyHi5SW1sx%
z<=LkqH+RMKZw%9ai!gi&==dfv??iL>nb^~(U7hl5PRo_V1WsepOW3|OB*gWt_?vI1
z_Bh<t4VnM_{{jJ%#)T#;SMmC(eA&mlH0=G-y!>x<@4tyPEN$cPcr^d|-uGU~3apnV
zXHQ!&x%%pq%dwHUH!t?}&1Z~@m5sY<bGGoA$P?4_#J!4(qn+Ogy!iGs*?H%Y$ffBy
zN4v7Fa!>l;VstTV0oTH%yREMp#7BSZ)8}37800nS#W}I~Uia4RLLEK58C`t`4F9ww
zcx>wsG#1Zz(r`xj&cu04+EaY`KdqRfz&No_M|%pt$nVwL_J+Jv)TvZFrk-`o(a`+R
ziU&N`GS_XETEO>^f0=UBg7&qm*;ll^T3YtGuAz8^+bhfd&&%JjuX(}Bc%az!d2#GV
z<<6aZyEiT?-E&dz_p-iUd++~OmVBsuQ&iWEqbXpM-0SC$6T4<~yxq*fyL8ef%|}Yh
z_9=YZ*Q>$#?sR0~L=Sa2i76HuB2OndNSDr^%cOPek-~;Yi>7Xo&b=~u=i|E~nqSu%
zYzls}r$Io;_VI=z2Op|*&e&uXym_vpAdkhf*#{Q{&9Lge{KEIx<xA%sMOZVQ6rG;l
zc$?{`0%K?y^D$YuYZIpPD$L7Hy24v=<=78?j$6m}vhFFGJl*K*YAF@-^24Vb|CUYL
z7ZQB!%Fhqpw$}1irt<PG7T+V}<K*6d`j^Tk{Vb_0zdQc@x3sLJzW28Gi*3|7y%IYn
zv~$=T|2(0*@4(UgsqP9gfs$@>-8284VbuSi`TykMf-t@3LeFRS8_(N*wR2VjyLi|_
zbE{jM3chijs}OVGe;=B)v@%C^GV>Nzk6BxH><Ef0|HYloQP{rStiz4*#K!KlenDwf
z<}%jnYK%3^dx|o}-)E{GNcp`}^+|Q-#t(BcV=ozg-EwHoy_dP6lg$_&8o3vn>SujR
zd$q*xmAQS*(eOWg`ah4ouV^dyv*Aa@iT6MBvShWrkLzvDI#AcM=2+la6Up`K65Z1!
zw|qNp^W=E_p2M>57Hqqx5LjNiTW(8;`K;jmuWz=0ouwcuEu^I5)}_atvsH4JWdEc$
zhrTgzTskl_^h6)uiFuu~a~{vJIk&e{y61saOFVZF!>RKJ;tu~06X8=XIxaQ8F*qjt
zfMHo~E|cY?|2I~buDtVP;i(8N^XEH7N|(xAJX-(Pea)M5Q4InL7y1q~F~vyQ%m_$W
z$r<xzOV^FID@QulMqE8~<5ZY6<JX$2pT&(*>?i*7nq<=XX^B0f4(qq(od$_MGtJT;
z&G|V|bfs?evOiClO1dsAa_N`y?JCK<sO!3uLzJPkE5s;zRnf#KHSOtZ<90XcDj#@#
zV-wTSdAc2D>bJM$@lFyv;OP+4So-qRQS0zGcXpRwU$=Mmgl-W|P7xLl0S3dzN4=Vi
z&dyx&;K_FG?OT$ZyRQ0}O)IX7)3{-@vT5<Hd14QCzSA&!o3%B2>W1DaEK@}}{`(jN
ze?N4Jv+}cEOa#Mwl^wUWCKw#;)i!6#^|XDlEOhp>Ycp?eoHuu8N%(C+t^;y%e;k_6
z+AsFczbTe`W{ttM)uHRJ*G$^$^=)_kCi%L*KY#wee#zJHtCEASrR<CZN3MG+0gXIT
z5dt1tdyI~$*(F+BVUsG-xY8DrbC5AmQRrxEN5I?+r4Vf{mZ>3nMNG*hhDDBSy{QF4
z;(eJPGp0q(S)&>t9?5yZ)zM&4C|A@R2~7ue-?B}~nykK&vnB<{X*Ns>o7R{r8B-;>
zY+8KZ$)z)sFCFdd5YkvUBR5ZNLW7X7kE42ryin)mL?+gNfcZOR(*sh4axQUcZaS$J
zu!!~l*1!vkdi7XVGITssS?M?_%k5O=>>Z)A7A@H@QA%>jl}!^FwsIVq$q=oQC{f79
zs&&w3)43%FSPpY492MK;=lD9hG4BGK3yb8Gxw;&c&+oe3<D77J!vo9Lr#3znyq)7z
zW}zOv?V;$CIS%)Ve!um3Amx&`k!fQ3J)gA=FS_m3dXEI{(R`?$;F<q%`Q;{)vn2~0
z|GnESaDeIRV?k|kK32&TFT*oa4)a)lUDRz_<mz@*q<(5)MC?@6X~$nR&bb`J)H~<8
zlj`@W;W0~(PTh2xbs^8@GeK7?ZzP#(PiSD-V{~U){wq&Mx3pD)ZzD4lA8);I(L&ZX
zDtc+9)`cv#txVC2y8q`TMlUe_duH04lDvz{grq$bFZl1-xzxomD5QOI1$(afyq%A_
zUduR^iI<e$yVGQCzWwU1+O-c#mb*{Tjmb!qZ}{`JJm06^ox}3UG~P4s8yMYhJV<Am
zcZyGPR!-qUe@_365B;SVom<$Uu9UhcSo@Hib$I33c~@hKH&*OEY4NG<_oQso$S2+M
z>@OEPCP_-xOo}q<PyKw}T0d>t<#NuwUoUefER8kUJaaRX=aH2SA|;o8E({Le*=n|C
zE0=<!P_&FxX>@nc-gSSvZq3iiV$=OQbK#nvM|#@~rZPk*O20@sYY@#mbA6a8<J})F
z=^-~KJ4mfq&N+e6iZfDVix8{e={x@yDk$!8mG(Hm`n<(J@phEQE|wzp4|fhWS1yjP
zjES(%-pgu}I)SOgugRq5?}Z{Ko<#>e3`G`dyh-%Rkr4J)VOW*&RLgYIqcELqqH7D5
zb|p%3AFm3rQEIxVpn3bA#Men%jDIav$gGh(t2RMqj_QLjZs$|^8p1ATQl8Z%)^uMk
zD4O**G^3vF<CQBc1+#8&EKz1Vc`LW*@HAFE7jvhLXKxA#IC6bbOJ{YI&RY_|miAYm
zfIIVO$+@8A!tV^HzS`)lx#dFFUzH^ex)W0!Sxj8c>|9#3>rVL7f=j2*98DLAW{^Ep
zleFtn;foSkC$$bg7bTIY9%V(1f~L9QzS1X8xc_em<Tg+Lv}7W$r-#d`WigYs*gP-H
z)bk2T|GJ`F?Um>Hxu0s%-?tbssv5A(f4ur~;F;xK-)r+j+78{<P&c)2w#!nRkrd(>
zqRQaGIw80&h9h0dmCN6i#cD+pH$%+QeG_EWn;t1U-*j0uuYC(sG@Gvnhv^DE*PRP_
zay6QzZxyr~Mf9))KRCd-b3-F1$C0IX7A10Doaz)VQyTL;szc*p+q#(z?#)37tOt^U
zHnfVoSvBLgYE@aK<u(TnCl>EeL(l6+EX@qpDodr8=q=m0J*g-8i19qF%fUu-j;u*d
zi<x=mi`S;TvvfA7&ORg}GQr={Zp}K;p807HCQYz2TB}>T=fC&=v_<Z7d|V7dSG&6?
z1x@hH@SQ5X$#)$K)0GR`TyvQM892`|>oqa-P2uO|;gnmYBv_flBolB+%*?@S{xXp$
zuT<}?UDlJAJ!9GVbk<2Rw};LK+m1|2V3iDCSglnhqf>HgDc8%snO5)Lne1^YRO>kP
z+v>*F(j=ozcYALAzP_JHbKk*iA@NDqH~!8u_Af3ClC)}E_S})TDy8Z9jCCsKE8P4K
zu-y}y>5(|KS}VctkBYpJd5FLfi6@KMHa;l)_GDl8zkZWxk}smgT%C{4Qaj4_@QQ}h
z8p(4hR|30~n$MZ6k&7?2IBwcEHM(GPYjW*}iqsoTM=b=z=O0_jtJJ0G_y4W~8(+*)
z9_brSWw*lT8hI?R(o}iD#;)Wn$^D>JPe<#-CDmsJhcDzLU(k}sbnI_Rm@eCOc6!ot
zW{3G7-gH!%aYapkkuH|V+8wbaDTec{;m)$a*<GPeGvcPX87&Izous$ep~1*bRp6U?
zMy>bts?7qN2HP6aL;O?4|IT#MS|utQxFp?O@D_)&k-B`UaH>bQ|6>XF!-cD+vWRS8
z3^*6yqN>sCm+jrAIc0%m^b7~hq^u`GU#3}weB_ff$Y{?hJ{Yv@o>`1pAot62Epnd(
zJ3Jyb@)fU75?!&XRl}_?G^!&>l;hCTu%Dq<E4g{p4_<lKs<!auJ+;a6Pkjlncc1cq
zQS<bU<wB2{dR5GPvz=<n4Ta<Sr??wEk!<ns&8n#RzuT|h`{bNAEWwT5eh+kK{yCue
zU1<6(i-*&HKQZ1IH=iM@b*6J;<Q4NA4X5y50g|WPZZip1P2Zk=qG^FYgHW%Of0SX@
zr3$rU(xw`c2cDm3iuiCjw=lG=(5SfC=Ro8+#(wECsWi>to6GqZo@Pn1N=@Ismq#XJ
zxArN6M*m%kFHe_tZ{tcz6Vod#IN)|(x9SkrM3I)WTO^KZJ=)v#`-+qB;s|!#Fhee1
zksQ%k4WBh!9Mh}Xx892vT@+D#=;o%Zs`K@6wNb|Ru5HT_T$g`TKQ^@T`nEg&_BI{m
zJ{_{{_vHUGKd!Ii)j8YZ6d|aaGkK-REf=p`$>)z*H!1GEpPH&8!7R3BGpk~2Zpzov
zuQl>1!XBABM81}Nd$Z+T$iugHZwoBa$U1p)zKTa_N9x?xA7}F$%f0`#-xiDJxgaJZ
zw_|CS#s;UFHHRhEFxPe*)VSmBlX1|B<A7BMqwSfX))tn%4_O!=ML(-}u)oGlKjC`o
z0=f1ajvF6>UujJ4sPU+9l$^5mL}lU8BYWcBK5^~qxX62|=Uu>@4-%Zm&R#Y9=rN^-
zRWRzjlSXH~?v3dsp2B($n>25H6`3I%r1<RuM{D9kkw2VkauzZ?aJv>8@%v4uhzQrY
zhetX!r*#TWTFwv<>oMhj*UA}bC)`#@Ou8Z|@UNl&oJQD!BT^E1H^i^W^ZvXdmUFbt
zr#<4sl`sQ$pRJ7>Sgv!2Ms9LgwCTV#$!h|9rrhj;;@n3k+}C7zz{yZ@r)i7FX}2wh
z#5$PRxg2&!CU{<O-nT{C`U|slhMevdX5EY?-DRg%hD?9<;}QF+OQBbp{a-{1DGI(;
zTpZ-$-hCiWO!U^V39OM7f#!wMm!?dNTN}7U@~$Ln+mc-6vjW!)(^`9P&d8sU?8L%*
z&N1H9>%O^Ei-IbP;zicWD`zoiT>8d1McisqR^+koHzqoMNOqEOpUrWqZL7cLi?c<A
z2O_?RIkmOtSa$2WbZ_oy|NGK+YUtztg_$Z0^SBuJ0(2WvuBIHT-f*Zpvq@ruq|XKC
z=U1KVuP|9}XxhbjKr7|Ylcmw#-<WMX4l;4ESa&d5-w4v*)*4>9K<gX}<1u#Y9!A5z
zw+$CQ2xglz*FgAyV{qsyj;g18mW6lt`+5YvEePLq^jPdfK{rn8nNMfTd3us}#;KL6
zO>1Lj9#B11JEwEcoYtkslfJO!1qA(h(BizX-Puxvci}=uiz!Pc98{A@^V_57>=?Un
z#;GfsUUEgU(LWTgMLv_$bog61lc7h+B_#T<>S>o<2VIm}wud~*d^KPF9dC5bO(zF;
zZ%=QHX^$8#s+CFzxXN&8Z)s%AJhHNZMLWgO`%}<=`;vLAmkvKNec+?E-1~vMkHI}}
zvwO8XOWBwnzX+WkkS5yN<7jB*Gk@jP@}ldjI+pWVEq;?5D3G=!_Uw@(KjY%IrhnMG
z{N2Xq6J3tkPhFVi5Wmtf-r6qAQRb`_m&=uq`xkm|EjiNuEiqPkYiq9}yPzt&q)N(m
zlkRd2&Y!wUn?t<+3;I1!3~ztd;(qOrdsj;OL>1{NDxRB|tar?;Il*3gr@h{wPn(M&
zK+VPb!XY0i51)*tJr5sgnLOY7<&agxJ*_p)-rHPCC7(L6sFwblz;RFVkj+FR&CBJX
z5mT3)d({!%F=x8SMTJVe<m#i>`|kAh3f!M4!BQJ}r|$0cj|MIOEqhxhZP9G9d?=FC
zKgUjSZdl?xr-dOFtVSni+5O@yc;j}KQ^alQbH|GT$$J+^pGa&tB%z%VlJLyyd+Wo6
z=T7-dXxwCQRawouvSmWi(<3wI-c*-z@$MBWxsawYOR2=e<xd2QTgDNFN*0d;+}>Bb
zy&k-(OJNFq$g-#Rkj9244Uq}<7o2ykJkr$R;v=(qX;Yy6mBTt|;^ke7U-_;s*SqNS
zgfk+9Rdnj}%2QJpedXA^=*Ah9BXK|C`v0_=XFcA<cRzM&xNM`(ng`GFs@T%M^vA!O
z_+{pcgk|j)E4*S>%)091wP?es?=APQ?)7r|knC{I#bMFeH8XwhY+>EJ<e8e$)&KVx
z6kJkrI@(fJ#vE~-@uIp#$8%Fg&6VT3e>i#W@vi@)QE&60UZrEdiGt40Ztp`&x+b-_
z*Tm_vo~VBkt;gxmn3JF@s2hDlXh-NdZ@tuDwac4$SwsG{gm^9eIfI{P)`@P7J8w+x
zblb?hoxwA+>2<hR%h62Ni8uIemhsG4^z>HUoS1d0r}w5tAJdICdMGr>NjR#rcY<c#
zhllAe+J2unbxwk7znhoe1MT~USJIxb)E`@%<8U`~#-l5WUb$ByPK2mkDv@G8lIC<I
zE!*t$ojZ&>mt45BCrxA3sx`A79XO|*7w6(OrAxQQ<;}IMy=z?Fu4>u)LC8RhgDr2t
z+k4%fKPLTOyUhK?-#5B{7dfU}eSD#rBh6*Yvbe2k$zR-BZ+57QUP<8PJ1II>{q4q=
z;!A_xO<no!X~OP9>eaLIe@=S+Q)_;z!HSy(*BZVp&k%XB<Ji#@U6;H?w=QzB*r>Vu
zlUi~7694Lo`SW<PRF+QqedUgDA-h^rmg+TuRZS~&Tb6W8JC)b;rnBwM*=cLKzd4^;
zwq~svABR3)gG}a=86QfM__Q;aQn-A2v^oOX1lirDdyDAuT6<i$=B_KVyz5gz*RGpg
zNz)CcF5p_GVC0%RbzQ==Y1czHc?ie9f1{)P<CWh#*7Gy_)_$ltV<L3k$~18{TgJBs
znKNFzl(2s|XWB=};Dz5d#QYZ*GT5UfaP6*)l#t$LJub-)?=PQuC_nRsdtSh`7Z0;E
z#U!{s`LK9T_!$!@+4r*R(aUvOt}oK^>T>e0F?n3bT6b%m>iu^gSFwMTYVl`E_44c6
z@kr?1x6_-uZhzY3>=q)Yo8Y3Q|6bcp`_rjzk1b4xWFCL^(%$N&f3_xbo7v(8g%=}#
z7@0OL)i|E-^ud+CPkZ*W1oOPdi)BBjmqamfyIgo8=wC2%?>`yqya}r^*dFsuy0W6}
z4@=JZo2);6wB;GR`NgH;H%Hlyv*oZF%i2GNw>M-}ro3_U*q~W9EjP}f>(z&@SvjYE
z9cQ0haDLyKdJpx<*FNxCm+TKvKj0!4%DMLcJF_o=*FH>&YSTK_zT@tr!gmI`A2v7`
zbDW;n;>6t1eTi9f!h`(_7X;ho`tZwyUixy-^P-cL%z{kSi4FPDXT%NbmmcGEi8NE3
zuqF4z)|V%aKD|AwQ1sl|xtG>{TVeQl#ql*;Ppvv~UPe5`@#L|Gvx?7iE<68LDOyD>
ziEmQQ5!Vcdtg8)@8Gk=6jXL&JY)RDwr`$z(th=+qj%T@_c(<qHgU)Z$V!LpsB~SFl
zcn-*zIX5SK6gnfZ<k1UVvqbj5u*#Nm`xfn3nW4S;llQ9)99G$>m-RR<pN@ar&Tgme
z|6_LKOQVWS-;_GjFI8%q)V)3)@^O}wan*|V?_X}K{w61VH2uF;W@D$6@4jtE>)oQF
z+&}+2l$gGuDPhZv><-gd3HAk5yq8XFR(|O=C*%9N)N&yX-nu-tle2QV`^?_%`oN*j
zr*o3!(mvNo&swV6c?uUTaoq9!qR?BO^BWvRIBH)MIJ9r+YAUI}&aS(#{P7a=s@WTS
zrpVQ6r+QEBn-Z1TlFu>C?D9_?!;U9kz8XDld!O**o6+?;&9}M1^)n|o?#&cm%zDK@
zamE8($=yHZ9-2IF)*KPPlUW--Iz4#W_Ob2v0shX7tO?s%!{5(|ez1BK%Y}3fkywV;
zHq%aBS=hDEC+&C4v*-}Dw#vk$n4NhFZM_%0R$nQ7b7__B9X+*VqhhUZiw>Iqf9mor
zi9fk7Z))x2=w^%8k7pQZUuJr`vB{&`Bt8CHI=9)($=g0GyA*o0sre(9p7i4^Z~f36
za_!gqzJ&!ET%R|y?X_H`{-r-(oKmWs7Q8i!UA(KQ*vaQ}i7?lRM`;pMXPKq@ulW5u
z)$)ey;l+QpG_F+eS`>2kevtW^5VrgipV#g6D$6gIE8<Q4kh|K$k;UcU!-ZcwmzQkT
z|M86H{l1Wrn%f2JO$#=COJNN=w?%n<J^PolzX}?zyZg>Cahm>{`BbG#_dIr?_xmyr
zmPrPOoZ520H}&eizN?S62=@mcDU|VF?VcN5=yxYK>eIPE?X3qFrv_@*2<iQ4H2o@h
z*&<rZ;Q#L{QNMlK%0H`K;`r&4V<Y7zdtNdqDeb&;(WxXg%PSX;Mn#8BQP)xUyw=XF
zEy=)bU3A)mYcmob?2%(ycXW5Zh{g1`Sw$M|+n$7^Z}Gpr_oj>fgEbeU{q!>qF@|l^
zeE!5cuPp8U&$Ppz(&ft*{C=myZM$Xl&eW8vEioSRzDB;9Tl>W>&?%->#CGG+gg`Ox
zS$DcOiL8HqmpT64^z6tbKA9@TM&?o1Stp7#cs=YkUMYAs<3qZXXZ+u`&*IK?PA1!%
z-{xChzOeRkO<vCTvh9Dee77HbB=w@Z?%Dax3<|#`*63`{FRD)d;>ND))_nUS-(wAv
z1lPj`(N~%!I9_`+fBDZ}z2&WZ35WcbO5wewVt-3S^IvXk)qk*o%e!%158o~6LP2L$
zLH`}o_PrL)w(oen_r!_(o6A-%5<GvvI8pH3iM!ig-rcqK*L7V{H%=+(8^8BgSjV1k
zeEOsH>5ut){^Zvz6k%}D4qD41Vp1WH=orSvsaB$q@bF+8zpR^x!?Q<fU99e$3P;lx
zFqkbCOIl+ZxX_)G!Ax$-1c%0k>0Cl*)^G-=H8mS@f69prXmZqC{O9WLN|Be2i#=t(
zY2V1dda3=(tE<gk*;gd0{+WF%)_S0QXhVW)=3DpkFRxeq`z4zCdfVGuJMy0f$($C+
zzrL(`>aV2ZUZ<vMHtpn@7c^r-;*s|MqQ-iEZWN|;Me`I1I?UvjJRNf?LhOQM^7E!t
zmJ-8`!jgN7mxNlU^nNgPy*!`0Xwl!9Ti)K;Qab;euEX<VDs0X6cZ>`c9c@~n&y}%s
zkHV>GIkU9YzfQKi_gZtic+TZh(~3Vj39E44D&EmB<2COl@dYi*h1y>xv-$T2?DP40
zgtcem^VjRHMLk-zU=d$YmqW7vi?<`AkVt~8Vc1I7m7y;L7t9NFd--%_YX)b4LkG)>
zrMm0A-aI`bB3u)&b+?7hhJ(wKM7S0C9QPEuEze3!<@RL}&S5^RCNRn4V4KOM4GKGN
zIC-k^-)?Gd+R?E|fzRggAtS#HA`Yi@Y&Wmn_EaY6|GTX>miTCFx$U9ZJlW`)PKNJo
ztHj`jpch*{TUqBG-t1>~`od>_`?qt1FE}!96~5@q|8}#8o2=`Wa{`ySwG0}Ua9>>1
zxny^V4dX3=d__OrOD7h*b$h92)$gM8bXU|Z&5lED9upTW``u@y@3O4za+9vq)J?Ay
z1u_$tF9;3$cy+<jUlY6JqQVker*00rm?B!XFDh(G@@v6X(Z#of6r!1nO*>=_Gb&%-
zjyufsQE_Dg&&0chlZ{r*i*mcEWN^6d%Jhv#6n>i<HOOmcaJ6e3_GvgP<QK8o#IyRw
z0;8vMBTNl;nw)slQ<=SP`|C1p;V0di)0%q1Zv8l+@akBq-tPMvhyJffH5A;n&N_rQ
ziB;v<KJN9RFEe}(*>;8Q*0*Pzbirxe7MZBR%;p`3x(+e9*Mv31Z7SWcpz7tr4b%7E
zxOl;d@x{dn(@Kv;EElgjvcf8sG2`Jh*|qnItZy?E{#hyJyU5q$<p!ZIyEB(WoY9Vx
z<W2wZp<J?L)%2>UkA+<)8dZ!=DC*8%RCH#!gwY8F%L{XuHRdn*q|kc)$feEGa-w6O
zT-lYe*|NiPvrpMAxwgCK&lj{$-fC`JWPHK&W5|^YF1`BA0gtBI%UyDt+kd5UjsVwz
z<t=ecTrX7mHitO85EA;T@nO1h>EhQ0N*PODuXt@~!<BtAkDH<3@5*1(3ij?d)yyjV
zAAT?NNJM(%$yKb@J2r6z&d3l+kai7S=GvY1a;@B!3(<})hF#pL*}8v(8yNQ{DOB}5
zYVeqNYL`?<zhOsGLDz53hb&7Rk1%@c*PIq~@{y2`ey43{`Q|zwtJsDnQ60r8njHe-
zZIWkuju|f~oOvWj*+l&wTUY9gUXj@yvoG#x^AW6m<hHHA=u2gnh=k_JRY}t}EveAg
zOgb~wVa8#$4;{<X*)BR&syN6m;W-}0HnG3*LGT1SuLb?@8PB;XDKKbmSsd<LEx5_x
zi0LlT*O6g{x5NHypUSauu~=5LtG&_|QQ@YgF$Kx3TYnk0306K0m+%SDza}UucC*7x
zg{NEpfYE<0!wi9GL6?_0ip`uLvdW<8)S=U{1tsPwhU?__NgQSSRJ!D>=*yWioiCY~
zZeOCts+l-{bx>-{j-@S?nkl-8N+Pn4B&XFr^UW)gc%LJ@-0tm5zotp)v$<OLIH>AZ
zo{7FU|JRjH_q&-BSUnvWo)rtsox)z7EYWe6#btu8qDshIr4!48LItL(E>V_AT(N4J
zF;~-uMaqdXf!249cRjaB=id6rKj>Nb1e?4i7HqC33Km`H+M%NtDe;TTmQ6xzjl_ni
zB;nTE`<8J=U1=7b<q#P3-CQ@!raRi>Dof|DiMp#&x+C6Ltg;VF^omk(Sv}9DBRb}W
z-o6Bll{zU;0)H&On*M)c%aq$zuOhcsOxei57H@w_obTt+>vG=$80>vc9TZz;P;9om
z>+zcGqcTYwn-B0QPN>y6rZHRKcv#p1#YqR89E&ctR9s8t->H$2swUKu(6Z&M=HgXm
zt0wqw-6*=wiqn~81E<xh_6(7ekG)xEYCA}sRsQF(@cMymeapKNMMRQZL*8yZ5XDof
zn{_)t{cqvPx*J8ZzfY&wpK?9Gt9D&8)>pwgV2T<0uT3tST$UP!M+lT<Bx;L1<hs1$
z&_b5C{EG42`HT;rt=Vjv$0KQ~#+kZg%VEJEtsH(b)*noloGA1wo_YGtQxzHCv%c?&
zZ`E|X+_TeQ$p${Nbw7F1bHbBuzJ2$<OyGCy>`Ob<9x}eK_<3i!U8YU)3rjBs9Ze1<
z9;bk5EDDS|j1Ciwvpw|n#55l*_Q)1A66#yjuVRucukGa}V02>r35Ud$jvua1E_7WO
zt(l`>J4tHWIw9>wzE|xm-K7reGnkx({!Ezj`{Ny!rZ3J9PJM1Qs+e|cA@_wy?SNPT
zBa3h6PAF_W;WE$h$-2!4vP9!+x!4R<wgm=spO=<lX51Tekh|+`Ygh8a20>frmLE*<
z<}zpEq!%APx@VUmgE5DIdhpCsG9QhdOoGK$-;gr)eRDl<&Lfe?;AJ}Bj&<7Tb@Z_=
zc+-^8C>rPEmwkH1t*@HPZ|O;|?O5@6Z;#TG+bjMjEq2LCoMO@UYMb=R`#!9ha*bXI
zTqy!cK6ToY7j0ExTFelzf;GXLeWpNecHb&S`zQVH;=*}m=!iF~l!%2LDQ{SOC+q69
zPa>roSzRUUC&p%%oY~yBR8DBs`iL*Tr|yXL$er@8Uqj=_!tIUPT3c$Ox69cs;@!3O
zu@TR;DB<bi^*xSUj`JOi7ryja9Q<zP6`4z^8#EhuWtPmo`$fni`nu4wC3>ao6C*NQ
zyk?oGift_960v=5S+@DQ#M|Fr(my}ix%`?zv|7*uxk<rGpS&;C3iUg7+TFQI_S?-l
z3L;9*zOPPkhTk|O=K6ST0hdu**JXp187sBgc;XzJ`;$ac5C8eUX&QG>^1>DJ7gt#<
zc*^vmq;|D~YF5CWD_ZL}?J#hjCBeGU&E#NQhLgyhT^D9@PC1a)nY8ZjzjbZjp7*z|
zc{IJ`dY@R{jhcOS^W*EM@&tEyL|lHjtL$LdKe4}(YL_$KB#Zz4x1%9p!YBC@i|6JY
z-+ZKUr(E6IyKw8T%!!9CT=zN>y0&7fvu)74uOe^nHM~|VI<2rRfjjD;oLa(14JXdJ
zYzwo6;&Mlx2=(dmbhymp)|vBYWfRxbQ>V3jSaf{OyIgv$$*0>oBW-#}$GRu4pH+D}
z{#mZGM)iqXVx+@~&$C#S<{WC7^t!oWeO4RS<Qr|e^Ik{HU^@6K?W+*aa*=lJL;scX
zRUEk5-W4hIaG9oatrg(5RzFm?;w9GyqplOz>YS464sAGhGO>W2cdy^pO9BBU>J?Mk
z-(2$JOYfOKOFigVs)kPMNr9%|UmH~4<Rw`piJWU!);VQ)Z^bDWr>dk^>brdG_ncAA
zbL5I!v~<P$k9<#dZtIG$(ae3ZsQJ|*mj_B;-3ot4J>u}rRSsj^C+F1Eb2MVxQKxm@
z3^x|a330Sc&~Hk4zuTds(d3|(%g3f;i@sL%NhEeWH%Q@Gr0`;y!_`k&S2>!m#;m={
z#HP7QpJ%#q=Om7eFZA_~Y2MhPuetA|!ZMH7TR3hW2<(nx;=HEMc<#I%OM>k6i77s9
zN-U4k9R6qD;hDHt;n1x<CKWR#(|vChqO$LVw4FM`tLmY6_lmN;mZ2ikg)oLro;Qiy
zo7lrEoT|PZR%KZ*A!{M0Fgwd{#m7b)_Fdxs8uCO!{t#=o;K>-4Y`!Kjv9(!3PP=ww
z^O&$a-=`Iob+CTLQn{*w^_#M@Pqnf=Zo3??QlVm1L5ij(PtRjHr<p1WTnjD~EpeEo
z^XA36PcPJ#8_en}5m0)yMCG*-w`P#~jWb1^TVA}*U^YF}aa7Lj&8yBg&pyq2HR<ga
z&iVIV&cCMTa!Q|ZTRMO1F(tOoX<WsM9Ku!}ii@+Bos?)2ad;(f%Ox26$Wln*u0&zX
zjtAn~8q@@sjG43k&imghT6Hv9q($$U8Vi4miR_`)wv8fBQ`vaeu8MiN(nvAp2IJ=&
zOj>>`w5F|YzNN%*RBhrhudh85-npG{%yQ&vINdhmq~t<=t6B~tCT~-Y)4!yS9a)gE
zHLke(SiqmC=bH|3ZkAc}=gNA8M;{fG5`=FZSmN_q{-KNf(^jh>?b~um*53macnz(6
zUWLY~hI%aQ`{!@Ubm3mx3Q?^~it<hOqzpKf9Jt(q&5tsDjVt2R7AX)p^LWOgeN5>y
z&b`VPS)#M<Xl7pDsuzhoUzVNoc;qsxHO|Au+}Fsd(^De6V7lFw=Mfr?87mbsR%&Q4
zbFDPun*WlkJ;bAX%b_VOCCeOUPx&9B<NaL2fk)Bj`L$yy(%jxYmsq@W*^eGM$IF=N
zcY<&Jn@N1#&l4}KoA1<ed{P3BkCwqlrrrxL6c#Wqc_gxUmvWk!qIH0p(uWf3e|mz^
ziku~nszQ_<d6Sq#8iZI=G}??6U$6#?F*#3E3YtIZeuR){W=53C1%;ke?ORRDA7w{!
zEcxPQ8tF0F!R_tipwJ6prY?UPJEkPOnDNo_dso=sEUEvh`fCJVO;~Z=hDU$T#3Ye!
z%`|4k6C1;y9=*xK#3`iUt9oE@L0cM^@FJ}yzpZC`R$c88JEZVDII3RBUUqY86r-YK
zlW3Zgvh=089cPxx1Tj81e0R<9r4OzyYUz$W^}kIt>ZMcAC53xw-^4uKcNYJyaTL{`
z*tX};J{KchHzSt)O@ZQ>s@5iof37P03r*U{#>&6w&Z)O5^O||ywVqZp^k~mJwBeG+
z<Z~(geOApUt{u8`_|Rm{V?w?$ZP#y1O1mmh;j?BQ*CFrIA@i=Cn|H#*=k>|-_HTU7
zPaK^WiEtXLPAiL&Pf`>qt`TywbXs-)!~;YAjkP}}gmL6=+z}So>7;dM>F1v8rUp)p
zk371&R^>2G%N6Z)sX3~v^G>DzqUp}dfom7VEl3q{OXAtW5I@78d&)(dPaI)O8zkp$
zmi($J^;_%Ov7^6grq1R$?UlI9GGW@B8`Csn%v>Jm9cSZYDEa?p0bi2r)^-uUHe1&x
zQNFJ?`@Q~a5dK5x(7#nzmb_XjTlMUY8AFDgQMlL#LE$C_-jyF;gl3f-xqQjZ>D24&
zn2WU*Uvn2RX&hY;?CBe%Tb2}AP?h>5@wis@7KWd3FO&~IN_?(8ZOwG2t=rw^rtPb7
zthsz$zUH8u(Y3+_sy&;Mgts|x&v@^?yhw|Ed*3hf^7D%s9mBY~r+9R3IW$erb7QCH
z`6p7JnO?8^*ck7Ymh4`hz*csG;pje-qb3!8cFBHrKhi~2eH>@B$g{QBR+#MacTjTR
z_VbEkn3y?rTjuh`Ucyew>zYHN5(VZu8*MIgm&wXzzgk*)TEV0-aK2OS(f?_mt}%UH
z<mzm+N=`3D*1?0>z2!pDtIkuef<g`2B#x-J?~q9BZ!<`7FJoCUX;uDq1uiD@f~5`x
z$_@38Z`A+3p1pQ?LF(49`GNiOKZj-NhKGwu?=?tl;ZRnRD7tLeC9qRwK8Gy#p$K8g
zH&IHFvP#RYU0ix>L3)8&%`VZ>tyhC(l+tJUOiuo$V0GI3-DHK?j~5*&-KL_nlp*u3
z%p;pU+X5ypQhfi`rmrnQ`0TWl4=qycJ9)O*b1}tlJoIsg_QQY_<xS?{Ga{d}rKr7c
z(yrU-dd|!J_I8JLk+W}#Xa74aw`FzN({i3y<-!c*Ld%PU&L-E_U$MDV*~ibskQ~{6
z@Bd@>#TVx3KR?)IGJoqOF8#+dgSIKmd&k|Gp&WCqsCa45Hoi%=JIaEb%bslVFWL6S
z;)cJ~jW>DD{!di-CyMo~{Hng9c_GWk(sZ_3wnvehzoqeAF-qUL*j$P0-nPJ*r?d_D
zb=R!YSf<)#p5MG|)wu_2IHIFWw@55@@LeI^9ap26f1iEHv7PxhBMO!#6fE4;x5SEb
zMO7bXen6{*#g<#f=0ZE(-O!r5UQ)a~z92M?@0O?T<5xAIYn^?L>XmoIhHW}#9A2@;
zqUp`Lr3ZR7PAT{;Tcpvn{y3{&=yd693C$IUvs9awWPRVVY58^*flcyF4y>Jm>ekz(
zHZVS3H(|oFj=TT6nHDnraQOXj)_3*YTuOba``TES?Y`l5@_Nnb?H3{fYR+GGymH-9
zrNhY~rS@X@j+XAK^4qhG&E`ZZ_MEW1>R?uXQ*y#Hi_NT=mQS5OK9l_GB)oma?pR@w
zU9WdGaL(AhJ(}Clhhbk)c>Ffi3ssvAUFOL)v)-J)*Z9Ufom;$jH?-Vd(jvL*_XKs9
ze?<%Do!~b-`by$d&4VMG%XTP69286H-j<`dG|wqRX45qG3kq?&;(E3AxHLAL=ut7!
zZ$6?nSwmmfrSSPKizB@s!=+4Dqz68UC_N%;n!RgTk9@I!sbcTa1-+|(ty~<{S8StE
zwx`_j^E&pI$D8jQmoAJ=4CVarKVtoan*QGr;m$1!_>-)4zt2&UT-O*C(f_scO7Jb2
zMZU>1V!S1lewA1YM60+(PE~G-{#vZO@2Yjg!erG&ap&He{OGvLvE_7wn*FWN(+!X7
zHqAcW=B&<i)reJgp+br3@wUrs9y9koQuhpaY{IAGb}4bSxxSd@&nK^E>0aA*d%DA&
zaE5!^<xIBywGw-ABVoJc-r0A<le51odnC@;zM!s|Z|C{Qo$7`SzsoJ#ne7%t?^^o%
zpuCNK(N3X!+Z5+*yZh%JGCMw>-)X_zyp98tte5XOwK+rg;KYc7?@C!5S~X|sy}!Na
z;qT5#2eRG@t}sv9;8`ghxYU1k@*<u)&(hQa9sd_^yy48ag8jsbbbG7M3J2bNG|9-j
zkeV21ETS9wv+LHH7fa*LNE}O-DVX|kr77delM<6x@EkjNz&)d;H@{|8Va?;`6$Rh-
zly9zDZEvh-H~GnSkL$^%Yd){CoZDZ>`IuX?&8R`6l<TIKuv5~F-)m3T{&u>qcx~gK
zGt8DZF0X&M+0gswq6>EvnX0=r%=HDL?|8ftINr>|6dJpCmf(F+2If;nyYIc;zt2X(
zP(y6XgimL;{p5|k^eFnuycRnlx0ugK3oG~4?B1TeyY|2GmDjDZH#+Lxiq}t+-fSDE
zwY*9=ZthQp_?@~!A7{_L&i!cT$Hmt_H`x_PDwf^3E3>h$fhl(7f9vTVlXtS-xUs!@
zH}|2UIeFZ(=X!B;?~$CLv}bZ@7`v&3Vw%z8DcdKS#6ACZQ02CFa=FkU6SEzQd@X*w
zwEm&)9r&DY#rfJc-7EpuDurITjU0EBd1MYMHYwyVI{5$7xW;_q3iJKuH04U0z~lcP
z|4u$^DblTd_Q?F1-P)n?JDZN4jL45%_#t)9F~fVk3umobYju37lmd%~zzi3!gUxKb
zULKn^C_1+>DRa%3k?_c&TbMO$$&A9q4E>_cb#G>DTyk=<M(`?|%8O5qsxVFQ2{3X!
zC8lYb|7?q;Yx;Rjhn;`s>_}aDvB$NOYlhK8?;s<sh4ZFpF=R<HhO4L9p4z(l`v0Ej
z{b?))f*pY>(ury}53S<WWL#skZ>~i6np3-SU-`A%>Jquyne%MZCF}C_3})KgvvZDi
zozXZV9KCh5YD3XUGcOg7zy_mSS5{6ZwU04@oavm)Bxi}usCw}7nwOiGk5J-d{rbIo
zz8TN@y<~mozdCLXzL<)F`osC^?sYK@w$Wb~T+cdw!*Hf=^p-nU&iqYu*5oxdW?65Q
z=5x;NxcMB3*?TL$z3@FAm%C|i<qW0^!Z!T!YPGxd7G4SRWKO$ssqqKzlR%$4o`wfS
zge^E0w(y5>JZu#+`uc!PfNO(myLWDb5WA0NsuH_z>DO>KZK-ETo-sM1Vi6{@jvV!h
zJ&+OfUp=-U$W1-U_la_Ugy7HQ2y>>42Wqunmp+;7wzX-}l!;4Cyt*|e8g(%&@!*^q
zw>2PBQoGiZ;gaW^rIHsNa<WpM&E`B1sSwW1HQ|WFSB3OTdBPt#mqwhps@$-k=~~d~
zge4n{PHEioT-dre&_nR36PK*ZOXsOpE(xB}GcyC1&y_mya`}>?C4!#rS2I_6uG)0#
z=KKxQp2WA9<~B_B-G1xWLtpu#9ohaHjE;r`9=0l79ISbdLo_t}WXILbXYHg@)?5j*
z78H_9j22>IO6b<DV{&nHi_YD<R3brgt6=J`(wC`g!=KMO!lvC=C>g0eCD1cfI!I;0
zOqWN#OB{r_)|h?xe_+MY=@)C5o*$RX$~D@t%tg?3fvaxSE6s+2n8G=N#fMvYjEeW&
zFwW5ZdgYK`>EfQoxr-khy`ymC(Z!t2)f<1N%N>3<H6{MMT4Q#6PSBe__b$5g&K2CT
ziHk?C?$^TQ)2?_YZxL|w(EoGqstEIp$TNa%y$^4QO|xZfPvt(K*zUdX#6hwAYK`(h
z&xKQ#c+HtHrTm@~*MaXLp@oj2eM|+8JGOM>m#$&3l&dQjFL$<1**tg3!^IQa<MMt^
zto@%mT_m>c-#V`ly=j3jSEMfqfB7OR{F2(b<wwM)ueqV5Hk0Xwlh=Ojt(MPUXg*xd
zYAv-|<%QSr`Z*UrCd!@iS~R<7;nDvR6S4vWmT*+2ul)8}HbY^>3ciYGt28@J_T{`-
zU-fUrf$jV@K76NCTGv`LKE3?c-c@gVxT@aNiy}@R1vahvG-bsG4(%pqg=s-X%HKPW
zM!H|T<|Xt&m(gRP=-P%JX4VHPeDnMaI}&+Po;B^-HN~&lEzqYl#A4>DIW6VPNw)$o
zES_(1^i@_+M}vsLH^avo>B-96Gi|-tWM3v+(ATZLS(c(^{MBNkwRoV^`M$LI%jdeM
z`+k=6`nOnq>YtpF*fS>%tTGdnoK~0_qAc$<@0<0yo1HzHALmsbFjL%o^P^MF>=Uha
zCXD=Zdo`3MMR$6bOx#h(=-RqiY}$%LYot{9RHOe}&be!}$X>zsh{sgzpni+*%7-fb
zJUMSPy)df~x;FQm_Odf4rmZp(ydk?mW%F#=q(dp(Q*sWrEfMja%XNsW{zs<LN#!dG
z-ZbhPIr>-aX_~S`sA*+d+^d;98z0X0bM^HP-5i>AChgY)4zI;?@@v-z#p@mSG>JOP
zAHK^<+P_Y93cu>gdM}}A|GTD6IqK+Ie{ok==z^oC3b(Cq)c6>-oo&`pbEbu6J(_cO
zU6?w((<FqUa^j?2YF=u>zl;q|Tb<MjHd=o`fz2>C!&TR?!*PiUgHrVGN1p#wH~u=a
z(pGSG|Mx`_dS)*=JH9Gt3ay_!*Zr(dn(6ObXChX)>aGgOcJMy=KQNq2L7R<Z<=2~!
zq&B8aGPvcL#U6ZGnzQWKx_yBTafORlxG6as=S?={T_w<Z;#ipASHax+n%DBT&DQL=
z=&SSV_3XI6(<bi!*gJ7e&cug3hgy?Gw{5U=(s}$&Y^QQ+koMHsb5dq6-RWPY%9&!Z
z$lf=6>A9k7&00GzO$}(8A?o>3clp{gZ@tRBH|B-uL?yU>k@~8k>%vt%^_1@SJ;sKX
zyIzJobB-^NW;=IX>idopyz^MkR@%?`(xU#BMZxBEnEARd{_~<a5+2SAIxzV~aN~KV
zHRZBDMLn7LoBUTLH0^R(Uon$K-~ZKu=od~-Q_hHQ(Q<s~dFS-OunCQ=TetXl=KbfC
zV}6;wGKuGTxa_o$DKa~jJ`l`GixN(JCs4X7)-c;Kyy(iD*opSM&cQz~EYZ4dH}$Z@
zvdK-+M;}?ViLTK(X)wEG*EQ>?7tX?tv*mxEaJOaKwCkTRgChUVNqZjg7I>IF*dpP6
zQqBKK!K5>@AO1VGcu(Y}^>dGKshiC_DgQB);f_^T<e{8{Eo#%Fgj!E~nI#JCuLzj7
ziq|0aTa$OryqT)^I)hDavh2&A=CwmP)A08@n}Z_C;TKxU?$?}|=@<9+N%*@txBtB<
ziv54Lbhl<zpxgf0{zp`AY0r(@G-=+pR%Npg{g#QRoj$HU_<Zh?IN_Pyb>2*p<#S#1
zUqvM(nng|ezhj4_fzn(aPvcFUlNTL3%4zp;*#s+v8A=ivL06;>FT0Vr(Z)I~ZOf`n
z{Y+bp`Zi`W3C%k2@A8HpE^Ak_pFV$OM^$LP`t3Eh+O~OZ@Ho2Sl!@7#M$IVMkmPMq
z8A`QlCNQwv)82Ygc=u$sBCno3+x0(heW<!|O`_2+C9h{+H?-^CNf52xe%4caQ{=~q
zC(O-vDlwKsODY;OsT^3Mx7TcHx#6i-4~)2UTg9fyx#YTKymYFa$T<DuzG4Y`;TMa#
zTa?}PcUf$;eB^Ka_vQR)4*QagCP!Ir-}#Q^TCD!vsa~%QPfUFLPUT>98%y1G?H9V7
zQ;d`DssHEP%b>cMMK!}!Gv$9+%eFju&eyl6UV0jPHg_7gm)vB>palg+9q-?0TuYv}
zK!5JR3HGb%&S>Q=jeC2G_nv>~%v)PDvo>Z3GuB?MOUpA%sw+7=?Ukp=ng@>S&wq@6
zxhQ{n?4)AuH%eRwYGqz#B=hj@akhF_*zatjv+dy8dRN0u_D64@cS*bT^Md-FV>9Qd
zXr&*S6nQR2(re>R?c&zx2k)neIbC;L5!`(?Y*i2k|K)hg%d-^cY~1{Ff2US>X7IoC
z(2rgb`Tzd?wm)JrMV`xmk=deQXOrLSFBy}zmBtqqO#9YwWok-FLfM}!{&T*0C0#6g
z{?+5kB36kh;;Y+)b~tAXOx62%NP_QZ{D(=Z|Ff1gX-P=teH1^arm8BM{cxest!4RQ
z4=avu%MJNj@0zN+jlGg(l7gFr$=2fuTBg-+6Z4!WtNBW_KAz^5^(lAn<|<E#P=jSA
z?*dy-P7@MVU=%iJ3qHWuUeczLWcHb*;NQ3EqOW0D$|>yb9x=-c;vR<Ael4g{wpQP0
zVKzynb!nl*^1@Ej+T7>i(+;QhU9ELrSmbTYwwR@59&2%FU~y<;oidBXO3~~>H}N%}
z1>Q4Ndo9Xcq|v#oqO0DiE5pe-Q>cJz1Ea8mQd&gW7769l3vAmux}6;DU!|0OD=xKA
z%v9SX6gRyfA>7>aP)%yOXxelK&oYtz$0<I~%C1=S{^x(<@xnEY^{9!Ccwqf>39m_3
z^FoC*I^xw`qMjehmI<xsx#D!W&}+$oiqNH9iqlMAT+{saF(I}vH)cia{>iPelNDUA
zHrrY_$L~m(^gJQ%qSoIP{Vp7#&EG`jCQjJCz3c0YYKF|}WQ(@68xvz3m|_B$QWYkt
zSWZ%loTTdMnHFL8cbXZKc#Z6&HoLHz=9Zcm#lmT>Y}JQrY%I$z9FNGE?r~{)M9*PP
z^Q1V-NwqVkOqp?|gE>^t^Mb11r=}Ga>A${;7k?BhP*YV@3DOHuUE2`ksougKId#Q}
z`fusek`}6YMzpg0m?p%~tr8TWI-xtYV0vBvTjfh8wVO=Jkxc*90@&0TW-tXXsTv5j
zRnAbI5FvY^`%jqgmLjvL?G1-+l>TRK)HZCac5Iw0ru#Tm{7m>%A%&?=C5pF&iEl{L
zpA*rPxolR9pgy;$#L8<mae;aPfl)t{G!CaJd<p9hP+~oHh_&`hzY<%Xy+xSakIb9j
z)bmVCXHPeEDOK>5Xt8JxVSPNC{bp|Di`F}wl`jO_q$Rs$653=1CaG7>)2N)MSvaHO
zsff<aYO}NgrA=n35%ZfuX2v#7uDanrM<MR-Pfwxk1&uFinlmC=Ga@t+Q|p9_Zfy_$
z$}S`*P~{sm)xf>)Q;^WK8A3r#SrY@jA4ep-2<|#+nGk)!oq0-q)=Sa<N?fJR9@B2@
z6k0PQf!U&<P_mmrM0uM>cU9%|rjv_nG$+|>G1zm>h)tNmbb+yTreJ&L68|)&j+aYn
zBxl%HO*fnryFI9MN0R+-;hw`8vD0rZ-EbwAWy{PnJNy@RHd<^?iFOpd7||=WLgi+M
z(5)RNOUts{KL)H8YbyKLRM_Zr|3(`37NKpPeUlU!-WKSKDLQ}oF+n-X<o_ndA8CR=
zXUK7B3OGkNTT9r5EzMnd-PE9TrTEQ~Ov#B~-B*1oTQ!k=VtmEQz{zS{2fCOA=gE1_
zbM2b9ZPuj3hfCBKuD&jza*=70u*ZDP&!$<a?s*k!4zVju@~p8uCSpFR=8{BV<%<RX
zg*qB-S7u(4=<P1naf_UCHM}<KyN-|LI@e30H-yxEjG8pFL^dC53cbX(c!O1(W<tm%
z{g3Q6W>=zqY+v_hmUmxxD=WvmjS}4!s}|erT3jwUBkjWCX2}`6(KB4Kmc$y&P<^;$
z&#Ki;noG_;Vu~=FWDzx8#$bcALfg*i3|s;X>a!Ub1Q-}5Flb1tn*^F{FB0}T*uW5!
zIW=vwo|47M84k0L`KxYR_T;O2{;x9sq;i#|%VQTV&%3x~&I$=%YjJ+p?0iA-ZD|!s
z5#o(i;<bgdLmM46H_x`bHv9ZdH}{2p9L<%RzbA#b&UvQZb*rq!cCox};oQ`h+nzo%
zxchUiv}X7J^cnLSs#mM0>;80M6aHbAA)Ltkc$1zqL*MRA6Sx^BS?|>G7Au<+CQ`KK
zgXT;brS3yc?UUJ5D>w>kOP85f7B<e<wJLi-v{^)6dK}NH)S1SD#a_ErHi|8Bo~pTU
zcP+b3>q7~q2VPQK;?Ih#<g&7(8Ki>3rx`@8^mb(1@o3Km_EnF!tkBL{rNL$M{MfXr
z+48kd=CW`UFk3JdMYi1x-LU<{;>w*Hf~}US1uPMi-tV(*NoyvPtiwj8hW&p+*lH@L
zM+r`9H`yTlbCp{6CXL&hIM!^^I=zXj=Kxc}&ZmphR|E<>Bsv~W>Ph&u)N$KEvq+^|
zu52e(i26V5Ia|I~wWsmF)9#e)?=`Nwg$lw~X+CRQTaZ=vDq#0-QLodY>(WGC6*=u+
z7FaIW+@ht?-W4AmAh(EJ?uDY=xoDx!n_0IdIc_<=B7|qc)wbrykO?Ptnns_IT%_J`
zCc0XVQIGGX;1z3yB{y6Rqb6?pwf&joj>hhd;<E+!a4k0pVoPZ_uJBT%GuohAnt@Y*
zg~w;7gThYEfD_u?Czt|G@ZS+kV@%UzKRB&A=-9D*#_}~*(@x4BayNOY*ixdTy>Pzz
z#L3sZA|yDboM)fno;f9W+o>+r$QZYE_Dlyg59;1G(tCP+{q|#ZQJ%B&1=kB3a)ku#
zk^MDwM>6+z5%oLKx<#ATFYlOkx25$z+s{Q@nbmU`r^|S5NDZG}QL};l^}hN}(N4~d
zV%0N*v(NEbZ@l_qb>+>C$~ReW*La8sY<dv6NqzRF3AYdYnsXq<@w8`n%mf=})<Qp1
z>2|>r_Srv5#n)`MI<#4M`~Hs09yg*2-7X&TX9{TEUA1^-z^hfWL~n_0Hw%;r(OZ2@
zT=;1D`>WaNh4F{i%!>B5$_vY0E3j=@$&pzf=lqih6E4dU7dsZSu=!-?<;afhOGMmG
z%Pd|$>x!4ILCxuj89$CSN$*$^vLki-@c<h|PGhMCo)a84C#H6vU|_h`%EQv0v$KDV
zR?6QK-ETy<*a$KTuy9S-$yDxcm8ST`Gkn_@l|^eN|JVC8pL17x+2mcm6ARl~3tMX=
zvaegX&g__eS-EwZpr8P2$|IHfSs7FJ&RV!t=T(M&@XO`z!d0U`sOf6$U7aP;J!_)f
z+C}GHpHb$Tup#M6=$~oK>5KD?8`)Oxv&)*K7qI$$^^Co~OT@aDG-%#&w-S7BEhzIk
zQ)c&u(wlb}ea^FTNNJwF8h7{VpEa9$-)Q~+vq1aUU17$9a)JzV7D<ZoRF!s!yyz7=
z8qpJTTuJ%Qe#fLk%c>9QH=VMVoRY_t<*a=+e(Oc!oUNvRdZkSCl`QsH<t*GC<JG7t
zwlhp3rK!T?gwleJxn6&oc~1&4p2%7Gsrj1pvBkM_<{VF$yKB#yz+3;Pow`!>dzFUW
zZRtw0g3sFnb63xsCRFNk-0h*xT_3iz7xN<KO7&=;m^k~y<n9xU4o`Z{=;r#?Z`=KZ
z@$QqY+-X_1V)2e^n9~aw*=r*Hwb@T}pD$zX#J1L<tvyNczFp@1w0Rz@-mJ}fcjL#~
zTF;kHeC3`BJ!qV`;!;5PLe@TsC@0YlLGgOUGlh(2ekEBmE%Oo-P+wCkB)ow&g~30h
zW^d@r*2>vJccNe3^IrTkyX{WTzT5+g8!M-GXD+FieAP1Z+%moylCL+k-&}n9u8zvj
zjWHJ(Kl3QEW~hBXbM^O}*MDl#&h5F|AM3WoX{N)qd+DnWmfdLQuWh`T)VQ>!OeRC<
zzf{IW_l1p;dxer49K*5~R>)PnbP^XSjptjsPD%GtnO)Pfb5_&$>cvS*9tvx2^J06P
z*dLN3FJ$!oO47EIZO8sC>GGO;<*xNU-FFiUb#@rX>CWvGZL9VOmsO1PJ?XgNz+~x<
ziL%#R?Vcp|GGzTdAw2&sf6mTHH9`yly94)KOHC9}-Y=B<_io-@LEW2*$Fi=^$)5jh
z=0OIfXZ**n&wX5A=D)VN_gT}6&uwdWx!klkbzQmLx&BVmI?kiJcg2XVX4#_}DDg~4
z;%(8MHNyR;d~WG;so$G<>&l##<?b)l?tQ(o=M4Kw`RQjSKI5LWwPX4Y-AQ|OXYAqK
z|Ni%ew#qwiW-j^v)BE_(EBnI>brh;=F6fqSU&rvr=ESafcXf6P`SWS}zZ2}+p*1mI
zXtIBrfKQrS;pJz&2j}Dqy`Fn-q4cs3zGZ8kyh*XUxMk+rOFIr-JMrt%yrd7s<^SJ0
z7H(YlY{v3$p}#F<nx0ScntC?=_P(3b14V=S9k)5i{$PFoZ&HFn)X`<r4J7wTu6z4+
zOSz!y<mOdXNhQ1YYVsX3k(%eX_T@II83p;1p41mr<{#fT>v+nB<1rI<n0$V$6TQ>v
z-lmS2Ybk|Kie?r4@fTxgbU5I^z|6{Fk|FT$U<(_!S&Bu$Ll*@h(XgBjhGeH+0alwY
z8VU^!ES_wB6O2+%u`o)9an1N}>A&h}U47OybBcnOGw@kf=4GBZs;s_{$%41l<Jf{r
zjVXOx2~t9<jxS<riQ78st5%?1YzNneQ%hHebi^*<a!E4i6bo7#!KvUH_AX{u>D#Iv
z?unr>LHX?4q;7;WBrvYYdD*J1*vO*FD=JZAV_<ZIvseB8T<f2OOpMbN%fq;TeayJH
zRJA*7LBrFm8NyZjo&~jea~h@Q+&62C-rgy+vvRh;f!!|R_n7P1%{gKmX5P^|HAUm`
zoim?rY~DHH#(~5`Y@EU(8eswoH9uGwg%~X6%l)nTI^(5Zh{S!RBdzaFUfbR+uXLhC
zSyW@DQ&(eyQa}R}`<8t5xiwiF8fzA4d1)^Fe`-e8A<pGlSv$E5pM22U%=T-;LIL+L
z8ZKP^TfDo>A_`Y;n7|=@#!f<7X=!iEr%e-kC4@H_s_bWZ5y&8Wx44@ps3c_7)Uuq+
zR5yjC0S;W8LZ5opoRlhlW39UE%ZgKH_ALowx)dbc>vKWP!Pu8?M?|s3i+eSCeijdX
zKIn;@Ug4`_7Cx!yw2GIOzO4dN1V6)<1`mddTeOlw7!+7uaRg2hdgQO*Z2#zdfG_u@
zFUnr}ORrq+khsKQ*K9V6`)crOm5wzGlV$`hy;{(f85X^~;N;SIC3hF8GuQimSlQO6
zR2H$AN6Snlk$2TAE!(-ZVg9S<on3XSWd*Bt<Uc7d?M}o0w~`F?cOOaJ8dE%x=gs<U
zeRl#IqNJ+K4bSaaQlzKn{D@~!jbpRN=JS5tJ*t;wqJ2%TFN+8^YfaWZ+<RMQ-nqmV
zb7udreB$+@cSleC>jMhC(pDFDK09UL+#s}0i!sq8^tYeQ#~aO&wllYzmWKrAPUC<6
z(`_yH{@Sn7U(@^Z_F0D1N()seXmYV|IWBLM?QmSlo3tTw#loyvJ&RJLPaImh@0`J#
zwna+O>(XX^wa68?%63|r>+Pfj;q9|{E-jE(RJxugUMR9Xe{bWT?sf9-`7~GUoU-ZS
ziI!P5JR2mJdL*2_%+eD5a>6^`7jeh_UA=Rvey_EsuUY!!%_T32tZn^x|1Z^Ac*lDB
z>B-G;KV3W<RdV=EIGlt{r+jzTXKi}+L%_-DMQ#83omb~NMscV-WOZ1y_{GGI14}j+
z`FXsUJNbGz=adxF4@;vCr)R2tDn4{oJ-#w%{+iU2TC=OSupe6WZ~cxP|NpIV@;R|)
z)wOeSUnC}pol|M#c~~^*kI0NYfgA}U)s5T_FS$w_7I<b*cKgsCn?;WGm2D;rvqSdV
z8855<*d4#xQX(g%*yy&yakFz9d0%;LpY!<226MI7yv37*Wn`J$EmgYv7?v?l6Ww&u
zRQvO}iWspKGfY3)aIZPN?-`T4$J3rJtJxt;i#D+8_BaYJu29hQNVI*jlRYEo$UM=L
zObq`yIWjWYs+2D;NZV><=%K;4YT7lu5=D`ZfzGLu1#VcWoXB2ywm#>v{B5C6R=YeH
zVsyO{VwE@-du&tGch2Hat4x_7xAAp;&WY}aFM?EpTwFcHeLSphF<3I@__g#Iay&bb
za_FLur_VCa$uB${+AL?Dn|`5jkz-uD`pu4|>RVq4zZ06!ps-~5%yyxWMKYnQ+$VA=
zhu)dD!mdehf=ZXMYw?Mt7EcBJ8o7db1(&Zlv(oROYoq9!is#mujQ(W-c?zp`r5oH0
zo)EtEVwi!Tyib&q_MA6diR`8m*7K#vCz^^x^2>HOPc1e3m8lvfeso2|vaj}QD|&=G
z@7&zC#K2r^;(yl|p^yg;L~b~2;5;^|JNbJ5qo)&j?ir~t9)9WhhjsECt|>+quEIGH
z71FnNEHu0LaVh7n2E(OBTbX_S&WJ4f;e6kKedXmw#nv^&MsG8YTYLI2RGxY4vMDQg
ze&C%2ju&Q}{k@_v-No2L#4V6J>_U@h$<o%{Zz}gTX7G0x)MlTXlRn|Dsv}on%VL@N
zZpT}EbhAQ?Ig%xJ&skfT*e|Df`}VWr3s<?cF1q$hr<38vwu@!G>57To-9kndzU#Kd
z+TD7RE!`Bf|03%YPC?bkk}XOBuQ&{HGh83X%{~|wwXCIRqU!!LQ=~-LN=x>(tkZwK
zNajq&x7>}wr^V7How1*pKJ)*Ji?S8f?HXU^7*{x-K9Q;0>dF|ld99SE!$+qT=MQT6
z&psr4Pr~NYG{YsVd)K`3aQk36y>OX2=Y!Rp-Urh6u|9sHX1KG{TWHn3IU6{YgluKL
z+ePp1%R72F?59`6Ej5vg+j#9+QaqTH3@)i{?An&D8!pj)<WeKg$|X;BWm`K1U7kCy
zW%+{F8yy2qtZoTizJk|zRc=(mF6n>IEZ1`Tz3#ZoC?7P_qFd&KKx@WEr83dZ`<$y!
zpK!i5)AnPfftzvsf3H;!*WOgRF(=o<O|bE@T(3^u+t)SAUrd}>^Ta0i4pY)oi^wob
zGmXkQUJ85FwmFztU2Hm|+s>pZvUI|YDAjrYyVIU-Vii01!d=&|n5jNnK!Ww8+v0!T
zp1L=q?oQezx?S{ALep!#j+}?~+pqX!S$&qc_C&}m_gCtMi+`-vN>29gNjzi3m^zQ)
zO<uqg(J70UoYlxwdNoUYr;_^};|C^}o-5rC$#)6(I>C42<g~e3>brWkY59n{aB0Z$
zJQY}^m3KS!0JGiZ<vdsYHG9tAf5cp|wqTO>7u}57@WVbE?}-O?$Z;#lye&BL+FiFN
zZ^|R7wO!%wbQ)A#9JtpqT(5h}oWEzu%{-H7pC2XhE{-}A^)w}-C#&VoEMA$w6&o#A
z<(|3FDsk+jrOBIz!N$)*CUY)G76`m+5X-P=vT1Cpbiz!b|Hos}jPkYbt4-$;SS8N*
z|JF{$)GM278^nYTY%*8$n4qcs=Ma|@yTOKh{-|?*xVHF6S~xV<Jx-E-E$A%0>ur?r
z+ced`8E4rxEt=Z)CCz+Ig^Srez37ZdUa_Z77Jb#2w6V=&g2~M7Z6%j%y%G;cXNgEW
z;3{`plJS_W>TArUuR(2lTR5gjD^*)_3OhwjzH7L%+xm0M^qG%K%{~YOO!&MmP%pV&
zZqC-d^F0|GW_xrrscl=(X_%ArE9k|pm;-wIe`Rg1yz91nh1<g&2i~;uL`9fe+<uk&
z?efu|ESz#xziKBpFi%l4lC<#4QhNBsw?g6dV}>=WSF}6NW`Ak0?!bJ0@Ba=P9S;2U
zoV_Yxo`~>*qYZlsCUWnSuw+!>K3P0byio8!MR(K%!!wf@7G*WxULawaIai&TZ&{a-
zd83j1ETatpe8P;&ZYL<}y<pIKp*4-MUsXzb)ojW33l1jLA}TZX8D3!0oxs8vF-yOT
z>1p%EO{<muG;H43-T!$4)2qq*G$u5ubTrvl%58{V{qs?evCrzStBq$cZm|&Bl3--Q
zEHiukM=6IY6ZgZ14<>IpB7I<+A@>9?ffLMIEu=ZbD!ctZcbxW~uw7w);{#n8Mj7{%
zRzIidyFTb1NSN*|a*#!-mGi+;$xj@&ie+z1>@}SzJ0(Mz^T*_ONsB%gn59=Q>QdO(
zUh#iQmX>JAAr2F#!$vXM-&bw!c(Jc##rDn3^K@^pe9=Bqn$Q~Pu-~x5Y14|M`Mb=w
zDXAB&T3>kR*n|o>XUB#YD_x&6yKlbLQK)k$;O3M`6B>T~HgD|ioT;GET*dok;@lIf
zgn7I>TDdth8^t;|@8~+MQG3Lq<FZ-H3OUXjOuK?~S5|D?^<a;cqvoVy!7yjbv)zLC
zqhvy~dhX8>l5NtOY}mM*t8cNg?qiFI6E6x(Nu2z~Ve&hVUIT4@))l)H_v|VXaWI)>
zr+a~AYw+%<2KBTlO@SNaYCN5dOIBMgJhW&E2ZO}Er83JG>g?Vbyn02;ZsSEFGd-D@
zJUinzN;xfFys!EHn&V8TS1UGebGBNU(51Uzi@W#f^PhB%y*}Jiyl3K)ZrzM_M-|gk
z$)cxspZ0D#5~gi=<}*i_vA3e&UX{-72dmh3Z)x>YncFCEbd$+iVad6sH}z6}Pfis&
z{F!0jeak5iXD^FC(*Lg8A=i4d(H!o}Jo}4wZevL}Hm7BIOT+Tg4!bXhow^--A1B)x
zGuiK&+&$A~-@=2-jaa0fNUvv!kg{W(|8e5fFPZM_#eST#W&i)?T+QLI@__sAh&^48
zHq?GGV0&|p#l}JCcUPlC=fOuj|GQ>S`z<ZCvA?awV@<WffiDhjB@?Gv_ssm&&8fin
zY{K5{7dI)rV&JRjohm52lKaq%oc~_4LN46j!_DHs6f{A*%%CGwfPt%2{LAH*SRMDT
z-uhW<&c<K4U?Uv3=!=z6%2c(i!;a27tfB-?Bu#KSwB}&(7OOpnrAoIfsn~Jn(uzw-
zqK-K(av!;aCoh?>LVAyn%gPlh2To25I>hs!iTi@^q_7pORRRpM*+)2UG~HgHbLaB9
z<Y1<I+*8uJB{L=;&HR1F+}OH3!YeUlUyY)<*;)0?hkE$;o?W=ps-~+U@Q3H5*>;SF
zW>!viV^Tg;=DfUAiDUKYW5(PYt{mw}oMJX{h8yE*w|!AZL*8tkqM%WB=E|RJzgS0&
zmRaX^A8@bEu}FKU(e>J(huN^9M~o#TG;FKb{!{;_bKGE3^E`2K^~_sM-CtXDj|j|Z
ztWcOXN#S4v_kj&3b}!jlUO1=jHs7>44F<y6iX}?FwT@f;y})|cVy6RB`jahOrX8gM
z3>95Wj3)wC-mqKAvs=(~`eGls4X@QoN_Vf94iRV-JsdTwx>sje;o&(EE7tw-V*Pk(
z!<@tCnb%sooL=u}bEMjPyUq<eos9yVrxxT)*!6|mD@tL~<2}=}SWhQ!?d1*Pyd-$=
zims2(r2R93cH4BOojT)aDH5S#cfP0N#YFy@OOKU_Zm3Z_{^P^h3RyolQ-8~~lU+R-
zUKT6#uV78Pa`}OzTjK-Qil=fvG&-{X>KE&J3BR2wvN}X_Vw38V|4PYUqFyscCyO*}
zXyx<?-O!MHZcFKLc4v9cWcgP)vtI2v&U(tP^Ygjgu?P0J9@zVLgEN!MvbhJ$vV<l)
zz8d>AbXS4DMw7?>1smB89oWrs@cJy%KRxD|wMsch@0{7&-L=4OXO=?Q2JNs5EV?&Z
z3_l#L5Ov^t8@ZlaA(m&iRjj#Y6wl#J*BxHp(v@h|y>R!=)4P}MG8t*aGOpNnhxOh@
z8SmSr&PSsz?$X%tZ0SnTw`<ONMMy8b=h1EA(8J}EEb1O~KF;-8Ak+O`*DzO;OD!3?
z5r=!!EQKy6H>)kYw2Nm>*o~$~CvUpmJ!{$`vb4qR2eY~6l#cAl-S(b~zpv)5S^PgZ
zYi_u9QApyF&4v?L=6qQ++hl@DfnCdm`|fKGWzROxO$l(by;5YQ-ZjI^ZR35G#9N^;
z8_g~?E#<r|wr=10qdmS7w>N%`x_dU{x3WgLh()=|j*351#n(y(*>1^vsqvP_f6vos
zl|HfVPx2ki=8OxB)pPFe^pxY9)7`db;-R&|hk52mN%u^eHSy$L@8}Cnr?1A8mfqUc
z!K9nO8s?-uxjA-sLGa@>`vM;v3Y_rpg{J;OnZVUkuiNd5H(la&ZSIxzvtvX591`7j
z_x{mXH?@0ia^8Ql1U$v!`Y-4&-I3zama^6-uIf*?0^jL|r@MDaoL+Wf)*6Y^TaV0=
zzUq7~>Bfbw|EJxg?2oUU-SxFYNpz?9!UZ>yT5rF4x@TjA^3l}Y30(z@dk@bEx@k1w
z(ZZV(9Hw3kHFh|6ZAbLJb<Rw>68p<u=m$qWC}zFWd;4ZlVK95FA>)J={T}@puB-F?
z!fPkp`gZt1v%?gxUMA6NP3v+KlWQJcnwltS#^J@5(ZO-Aalw<82~QfIr6;q*-Ssv2
zeK@h)$6%|j#`8Oz@f$a^ICm-L-DtXdPUTv(?ZexlU(X3jbFVVt6y1GFOgZMdl8lbo
zo$1G}>ZFNI?=zOQTsbwH<B-G7gFDVg89z2=+t4^+_3mA(EpI0;Xj}9-!RxDpor%k<
z?=x<N&WzQJcwGJVR`oJFU%mhL?ti`eYBG2IyoY>@uUkGmZ&G`Gul7lh-x~88sWS!l
zj#p(fE?_#S%&li}BV=CJmNSQc%slM2S!ct~*PAq^Ph92wN%{W!qgPK$KRdilI>s&1
z{mlVKsmKY<L6Oz*#}#>uJeWKtMn;Nm(EcR(U#nxQS3qUj{;KAEz8&hEx;$Ld<ZG<8
zGt9KJE_L`cKJVV(Yi|73Bx=1QuR-bODV2Ji94pe-{=K#S?(@g*p6@ouZu<B9d0R(A
z*}IqPo;-Y;{<`jA<E1Na?_Ke;)o7jN&e$@Iy)twy$IE`P%nxU7*YS9moHK5W=nmWv
zI(c#C$`$Vm4@ytj6&mz9bC(oH^o^GJey>)u{x@K`(ezO{Yrg`kro}5E*;BhalE1H;
zDZm<7k{gxzFZqncyeiu@u4?yuHR1%sKMLr_tr3*I!<3!E5*NJh$ws@3-4fY97z^FZ
z;@q~~J#y*PiBFvMyifhVJqcX5b=e2bt!mAhJGyqe743a?xc8sVoi6W=rQJ&<?<DrE
zWntW!(3ivi=FvfkXG)?sd3A&oEL|8ETw=MDv%K%-yxK=5y8HfgWO1fMT}@+GefdZu
z=h>{U56s^m%k6wz&UUsS%8vK_8|(b%rs}&Nos4o|j()Ay9GFp}T=^xasKhh>i}$*`
z7jiGW`~+UE6ZS5D_lh|#;Q8~7cNsQc&71llq_eNHqi+M(=Kn7$MNaHI+Ox}6+UvCZ
zgkz<u*S>Dp5#8H;>f*B<0zxr9Etz{GJ-*r8F+6@%KJ&zHyE|vJ<1fx$JI}A<=#++n
zx~zVQ>xLcY8!N<hGd|>qe`mX^z~b;^)ib4l=dT3RS>5lql(EMJ34AQ$=W%cE7L+es
z!BAK`eS$=N-sZlpXMD?Q@4srges^nF;9lmUhTOoqvxNb1_m*vZt-M~|E$q|Yu+w|`
z0ykZV+o9MxP1y9vxu-hrL9IPGf5R+K-1{UvolB*24|CK+m7Q<gY(5u!-=)>U;hD)%
zBXu@|ZT~*4c}pu5WG}OnP3va0`{h*k_J8VjhTX2q_p5m}R`s87b&-Gf=-B`CTDC&>
z`6V9uOV@w@wSoQj#`Z7ko>y^7vi=SF?VUb#t9ZqaiQL<tr@M)NY<u=@_x+IOU7W2=
z8yDTUxL@C)yGbnk?AQKlUs?Ygj)>p)s&UWJlpGW5Z#ygeRaB>o-Oe;T=6~hzx5E4V
zE+3teQwqX53s%{#=2pLW`Mw>?3b|F+m`dVjl^!n_y8r7@>Pol#F1D)gJ1WZWY_F9}
z)2zDxYh!Gop!mPW8{Z#^&)%+oNYcFG@zW>L_fN@AuakdYSof6a_^yBJ&(y6xA@zEH
zw(@Z)uitk=#Z;owZ~W^#Z8RmKSwgeAL-6BSn+~6#RJ)(CU$?wJTJ&Ro{q0NL2J9VS
zOgAQE{SRF~&4H0ibdu*J22LghCIv5%h6F{o9&zD_5RO2G<2~H^T_GMu$tNaiSo_6r
zs60B%;*b(&(`l%Be3oP`ljh6f#G{imeal=fY*s!u*LdPTk(mb)CvkYHc&uFIdU~2~
z<nfHVN2Y4e3`{uMr{t`qb8A!j)kU${+jL{L<(%JDsvYKV!;tGe*N)lZ`EGk9-A#4x
zOj*xxFv<LB>Yf=r*L0(o`3M+P9-6{CWqHiGJ(9+0e6v|A!Ze~7#QCC)iVR~mnP%VE
z5U8J%D{VUcc8tZ-dv|A-=iOV;_?z#ab-DP1lbu~QGg+3j_->23nzZ%8E{``%9;sK>
z-7WQer*t%3CI6o3f6Y%^!dw}tbFC}Gwy|81(pS{@@#oj~W_~R}2c{n%%@rD1-?g`d
z2&n{K6${E~oEjeOVAj^s6Y9~{(lBx1!nPj6w(ISQUtPHp7f-Nwq>(dY#-lEcMJpb4
z>S}H2Y&5;)q1++iv}|!Jm+6nisgA#PT*z`RQn)0-t-3N`lJ6w9lnJhBAu8h0H_}wI
z!m@7OYLhlPk~lqqX<2|cS7G3@=_@KNZ|9hKIh?9_^Y-|>Ig(d*##KMqsyJaz?u{99
z9nv;7&UM_n<*>S2ki^#s4DN>PlN$SyR37+A$hy9Ca_+q%ylj#cYvy8e%`H<_c>aCz
za>b&ytU&*HMYl410s>Q~Hpiy^-x?YoU%7Qc++MYB+3V{jP8Hi&vNu&{({Z`ovYShC
zm+RCVnHHK9pZQPa#*9Y6uu0P;a=R}|+)i3OAx^lebluJgp4VokEvWsrZugALWivCD
zuif@!e%iIPwEVpnmqq9O|MpwIaF1ZR%%OdSlVpkx*=lKd3dT%1BJt^IW_l^pp6TyP
zf3Dnqp^~dMCSvY8!|p)s&sUdkblNsSiM2uI^q-ZDW@qQDU*^kedbNqqs^Vd*T#&`W
zmgt_{0o&v1G#c7{oS&KW_F4MQNH*AXWBOICq`gW#OG?(7cB?e)y|A6-(T@8!e=X(x
z+UM|EW9E5>b!tyj>>gEbpHyl5Qzfe+{bayIiCO<WpNfPQ1y)Rp>FZ3L5pU)vJ@fY~
z>z`+tJRIGuFF(E6HD77#qzliv?^-^1p^<#W@>Fie^_>--`bXS68{^L0b-dEKZh{EE
z<kd;;(wq<CJ@;uZdFi?Dw$GH6TD)H;t`sWF5@Go9)a8}Wid`W={vIu>YQL?{TVm@y
zRjJ`fuz#TI)wR$2wkm5THy+JSIPk7DcjuvN@_BKWkA1xsb6)6~4#U|`IzQ(4G%)^|
z-=U?xd9miP+P=bsFtY^7QydFo=XG9ZwKSf^J5SBj(~I}%tt0aFtZ(8ETb|<-4%_f}
z#e`NKYvBTx1Dvy8x>So_jPqFWX=hl$acik)NjEb!<F`F;Qtlf5Px4HdKL7gyzvR!B
zO4{25j9G&=c7^@%WD-$n)yO#L7{%ayq3nsG;kyjCKH*?V%d%~`M_2UfBpsE0yh6$1
z+{Nt039Yg(Piz%C*^wC@l5m?TQ<3MaZeE~O%H0V|wyRsNoalH+{O*_1rJhkM3!=Sb
zq>O*=$h+#+GU4|Ub#{@tVKS>6<_e!E&udp_G`Z-oI6F3RN{4tDM~gyTkgR6z#lCMR
zJcSK^o;d9Abk>B3^XfAeiMSW3*?!^-4%L>FGnRZhaSp4$;-$%<p;@ike^gIL{F==D
zf63Ho33vVL>@`ovv0a|d(#6qm)@T9aY*l}zOU3iPJvsM(RjYX6)ZU!uBAx618*SPq
zX&4amNjp60iu#t;b#BJN4dO0_8IBh-ju|bN6l%P&Wqsq<)r)s%_T@%tT<TjVkf@*2
zmo26;SC;8whvMT1Em5J%^F<7mqb}dr_MpS4aYB>mEl<hXrYc#@pwInnjPmteRjw>9
z44i=-hs2~!+VsOZToM%pWiBUgUGr^9l#x}S)l#)<%T3p2Dy+J={Sw!<^)og4%dWhV
z-Q9KEs%Yxkq}R#zKaB$xTj};R%PdLJd8}~IFD=j8OXa>$x4HVy7X@iiK~v?dcO1XP
zR@iPV^T^<c>M=vTXer)>PgpLn-dK=ne?HSoWLv<5?w>5yxj$Au&tlotw|3I>jc+w|
zT$TQxo;&Mk%9hps-svKfv;0!ZTNip7udNhb($=rB+3M{lF;<^<AN$HL9pU`p!Nf4z
zb4g8LCsXK_1)L`z<m+9C3iIU_)@9jrz5eQ~Yt^rxpFgytPw}no1DnT^YpXtqr~DI@
z+<a@df6+(LwR|}|O<Ro)o{IRe@Y5}x-DVw<r$eqZ2XHM-w25>wJK2_?!;<Og_~+rI
zc_nQ=2Oqw-YPxcH5x?KtnVJoo7vGyNP-yQevdF3Ix3Jw__A;BC^TNVRE=Chp9r$(7
zE4V9CnZ>uYJMKzg%*PXlHN{!B2dB*}P2pVpQ9)4IOJ}0A>$kO=CtBJEeLCrIc+c_O
zheG>{kLJog|8dObe;e<a<hT=YUyIYU9(hc#`DJ{k&LraSIbE+{i%Ch6D>K}UZW*;P
zKb@vidDBZO&wK9OHIH_PY%rME$UVnIc$S^@X0hwbOSYQcU77E$K7Yn7zcs#FvlEuP
zL~S}WZOxL@XSW1nmYsB&W#=ik!I*XB5|v<;C&_s~<363q<SMQd_EZZ|Vu;g>N!2c{
zG<M`@Ixf@QqtN6Pw$09V!M}A!>u1XZN1dE{eeu~JHc$4c?_vCt$5gdi<@K=>ao$Hx
z?)euS{p$_4lkAGsC7G(t5;hhtudgoY7E_zTH$CQn#jT5-)l(j^Tc111tClpCr^9iv
zhG|c)OtN%!W%Qd&)zej7MzQ;<{@bN^Gz9%GiDFVM?)r66?!1Ib>%v?mTbn6GQ&-lc
z{*Rh@@AkDB6;A7dr@bt*xo3FMVS(NK`s>#5Fa4gEtrIh|&TC(n`}J4M;y+(E{-1r{
z=yvwJSm!jOU!oW6AN4fzX_xQTy_DZwmh*5@h^M>rw)azyzPYa%YvS{`b#n{*TX#+K
zs1IjklzWeRuq=77I8cGBbZwum_Uxe7hZgWXU7>gWG0$hqrj|bInZmE$lsuo<nz26k
zs_fMDil64FczjoLj|^GWw`g~&5!dV&m%E}o;<KZ6AGbZUzi(Ui-NR3Wm%J``r>T9W
z>3DwiOC9Eo`?l9B<@LO|zVxbp!c9(|g637bZLa<|I{)2uS$xI!@+bPL33iNg@{Vly
z9nP>e@b5v>{p_FY4<=mf60x|kn_~fMDO;vbL}tW{R!#@@m24V&+4j3A<bCMQ<!H}e
zGNGe4q+?<P&&(Si{<!egJ`gLNn8M+5tVf1x(j1>e$;%h7_55w=>ts1Kv1z_aOU;y7
zd6q7sd%JoqLeJZ9r2L3+6uAFfKz^;^+zYI8g@Po1AB~>AbpijuzS5RECsS_9y3Vcm
zAQ<u2<yND6+)BxqyHjVMjhu6I>YTX?#qQ0N?3~m1M|8o@$OWlxcO=E!-u5b)N-cgk
zH~H<{2eD!Iekv+N-CE4Mi1Q^|oXefIR<X9foh&*nVsp+uZFJZAA1f#NZpQST$J)-!
zs@@|ucaEIX9X`zyM>ZQqySWG(+uq)&dU<2$bWO3_n;nk^yU1<LWsQAvVc{Jm&8LqR
zzL~J9^_-xeA~(}wgRC<P1Rj6ljo+p@e@!K?v6;N_uOr_sh6rDZ-oNwMc9Ds@eHWYU
zlD~K|II)L&e`%s=*j)dX(+OAlC1fRbRdl5IG>9n39E!Z)B0KAz;w8n@8A?G9+yiCo
zzb&@UQ#dBd-ssZv)Z!7>z9@l>ij6B`c)HG2*SMc@N%4HKVB*s`=X;J$xhPbc^Q1(f
zMR!A!`z^kCA?=)(q&%BcKJq5Ity|)y^|0(u#MHehze69M_;KO6s)}5ys92G#<^MR@
ztG=?9-=5U8K8zMsm3g}4hGOcC*1NH-^4Bz<Oc8OrGU1}cPTmC3Fb2L`cWx#wDonk1
zH+J%yg%PjrE}6P?N#wLt$+(At_fpp`jm|rfqjOJPbBXV^1s-oMH7tmUJ#@{fLaw6c
zUY4%Qx`zrUJQ#{M#(7w@e4CTBD&a^HSHt_v;7Fc$y`bZIQm39LEc4(JjZ2ctUOVg9
zjRpF3eCmrXsa=XP<dZk_d$`7Q{wK-gX)EXWbh>{tP2t%$-*i%M(jJAX9arja3Y)X3
z7tEV)H#7N9$Nayl%m3Y#FOKW>uwg5ayOHd2EOn2vi{ye9(~~W^2VG4cu$v}uC`!$i
z@jMXGne*Re4r^P){+O6UYo2$Q`Wz@|Dmt4UxaG)49TsuU2C*9)X1A99Tq^#(m0f6I
zUvA&=wi^k$Q=|l@&T>xRdhF^r*_I=mV_|iOyGLAxOI^}xTfzDS_hSoET4ya06pQzs
zw4zh*;hw+G>T;ql@v@wb=<N<T9sFqa-m4ndauua--t4|3C^>7P<fChWmn3^0U2S+X
z(`nbj(0z)DsY_W@g_3lyEppYAdzBdRZ{bw7q{z5g(R@nNUbszXJCji=@igIx(kFLi
zGgi%=3E4;cX8n5373HoHr&KSJ$p6vvl*|z)g(J^B9KIQxN;>AH<9A!RDEMRTIbA8P
zMXzQp;%Zp;bmihCmV*DON17)2yinlMe<!dkEo-4zmO<8;Y}UuwKSQ#FR$Pyf?>3X?
z`E>mF1or(i=Qb2Yb{$(~{%hGD-gM6^@_#RAR7|<K&PuguYHRWvjtxOicLlVSdQC|`
z@i<Lm&Y_8J<?GzC-`qUx%XjqcV*5DlZM&8qJ9r_lM&WEre~{-qCznQs39sWgSlXwt
z%$c{~%#oFOPp4?9E;-fMr<d6`*Ta#;qs8@KN6V2=Hzk%frZsQh+;Uoy?X)EO;6KiS
zN)6qM?uX|+@f4HSo%-U)tQSX4J*hgt?z3t|&&>FFPgE{l6}`E0mAGH*>T54<bS8#x
zof)$=XTqkb!T(Z3B{S>R#D)evl}_up`+twDte?Q$%G7%ynwbGFFHBl`k?EfNx5TuS
zOVhrdRY??*j1be9cujuh>zQdSEH!}%$1>B)be(n`)4Z#4>5QkgiTra7muC+&mo#yh
z^LnKwEn75cQkqcCo@p%$IhHvI=-fTl=kVrHRfFHsZ)<h=@(vfRa+{Wxv584tOE23m
z@p;ZX!)G(9GT2xBI+9zY6H=yZeoiHiD?NZo=1+rO(X*`LN$avF<duAjf29_Y=s4qb
zl3RL$I#Za4?V`5wTkqIEc2{&QUpzPc{iNm9Sr4kY)_a_s;3jg6_iDPY$I&`17xAg9
zn|^d;6&}}aU6vT~$Z?<6v#xnBrmQ;Ipptm&WJync+yA^J2fA1e7HJ$h$NpO_=eO92
zxePf0u3CQ=Eor^=zT?^Yqn{##@^WmKD$Kqnb|NcUE^%S_-sv+$+x+zkn5w5rr}Fks
zdl+Q&Sc31R*e{l|X?oigG-VDYz80D_;fKzJM~V?*x(accT=NT-t}Re3TdANX$Emnz
zhTVfRGqOE9RzFQ)m5gvt-!*C1w#@9%yfu3xt_v>IU9tAAR@TbgwA_pD|5#-$dG>bk
zHD}KN!-|MPw>Oi-p1;*Q8<twPaar5PWzX`L^C@TVd-2*Pd1chPZm-49zb*4$a(z|S
z;x%c?&xJhxEsXT+cr5kv^0oy>olK7%;C;PQB)dgXn=SClp_g;_wis0W&j{J(wn-&9
z+p|(z>?#MJ_7Q%+^&;7PT039PPk4XH^U+b4C42Aj97<~WHO)m!`_1A-_f9n|Ijr~Q
z(6p3}d!IUj-t@=jRA=azGQC*6XwvCrIae4}MYq24bu6gr&<WDp+`sAO^Q;Xak0fsw
ze2^&IkW$dbVf`|!EcIgD=kT`A5uaoqXy#gOSG6jC8Tn5(VZ&3FeZ@1=jy%rXlzroa
zzX!w4N!nXWkMfoK@hhu54@kVRPE-47SRKQ&1wV>)6%2npTRB0q<(Au*Q!+>N5;uy<
z7oLtOIdt#L-KeyAB_Hp^J=3fE=+SSoY~PIxwvy@^t;a%(eRis?%9=eZPwweHzUhn~
zg#X`K;Sn;Q>(VO!@;txg2{SLs|5R1qeKG#Uyu26d^n%y(Tc;Vn*|(0(^=tE4J>`1!
zeF2ScvrK1xS~rv7{rg$VvyOadU#rgUBU7EWAnJ6wo5sv#CDXlJTo+AQov6WKS1uTF
zWQoV7<K<gK^7hUCsX9Sq+sgivSJYpz2kdOyeJGz{cU100&H~;wDfjhtn^&ZI>!iJY
zpJn(^Hd<-!0e8P?t3;y`dnWj1#>*PK7Va}MkjRsj(o5}s6*wce-Qv=k-04ShuY0gZ
zZ@%ptu<)W{@>O0={_ULq6-8GRa4p}U;1#uGZt--k9>smeGx=0RU85(T?J_fDi&atP
zUZlCW?)%)DtZn~mI;Y(XoBQs9ptRSns1i|qmg-=`n)DVoZU@_hEpFS3oV!`-^tI~i
zw+kKD)stE!!T-ba_oL&}3|HLxVk8%P`DB#z@r~cUsax&~Y4~R(8*h<tepQh9^28_e
zEza>@%k8lGC*K+Wd9BLogSl<2J5SrpV@pnD32(nvGCSSlt8%ZuQD1Y&hp0{dchpzx
z`RadTvxiH=3K!P%H@-IMaD*K7y_0bCywB8>A2TnA@K#n{?5`0F6XD$!kblr?iAnML
z^e)|bHeXh+J(K2Urh9(V;ai_lmp}Q}b-e%UC;#@HCl0Kfs`~U)QljDc7xU-5DbAZ2
zyLL@<r%sHb0dxIM`Gu0VALiWtFDe%%cFipR>14LI7vA+=klM}_9CayeS0tO9)%wks
zw{6Z)^qZ-Y^rU0X1bvo?*<2H?S#|d9DnI&bwYRHI)b@}IR%@Q`D$hMwJ%Q2R@RieL
z^NKgRqKSrX64i+ny0>OoZtni_AnMuX$tBCOQZi;O^IBcv?YwSV_`BmrLMqGW76er^
zE-{liU%4Yc-(GO*i**l+q`t1@^I%@Nu4L8YT{(?uGM_@8?|!3a^zPekrcZiX`bx8A
zq_cFH_9iE6n`i17dFMk}_R@})29fitQd%08Sk-&922b#3G6`ie5dEp(;A3#rr=sR~
zvCWD4n)Qu;d>Brg`YzSr!N}jr+NAd_`MU0+|ItZj*SF7n{x)Ux&enC6v(wGe^3SC$
z*Em<c-ly$b%-$E<A~y8R>*<Lr2wqcm?dS@LQ-LhMSLD3TkiTnF|K^)?(&h!VAMOZl
zzbS5BwEXYXS(}%YR~7Lu;^`K9a%A&~*tcippS^K>=JaGQ!=BFVqU+Y(bUuAx)7^hA
z1#fjXZZ4`Tc(J7D<EcO2?|#~z=4IU|Tyb`GVR?0y)aE|_i_@Cx|6R#m?|)?ev#Okz
z*R$IGzb`nqq?nJRuYk4a)$tt{PRR4WE}3R=@NfLaL&;s?rnZ;7-(JgGsOi%4zlG=e
zkKXv~PtSkYKXVJ<@ZW4()AjZ55{-T*m-n{Tahh3PXEOi4s&rZXe{%n%uRokT8n*|=
z@%QFGx@@^dwC&!ms2{T)q_bUJ75VDfmiD%1_VW#PpFeHna3$%gSA#<nTco0khJ(ZE
zMqc(wLIM*LwL66*lTvmRDm!+IdTz{;FnXd9rs}^*QZ;ZXLr{*zu_>05(|l)XdG;Rq
zS)_V?@|vo5cQg~8uIP1>RNA_8N@nnKCNoFX&6!!DtD;vW?VUCCj96RrlAM{3LUnFx
z#-Cqyw_=)NG-KBC0#EJlx_5V!KYiA_+ciIKU-f6M57rU|F$Y_Bs+gT|HJGj!B4OO4
zQ;|7oxs0y4uR!pjV;s{RkGpIt*%Hhm<Tlas>?+>FWm{iycF4|p_-4j7%SZQC{Ga{p
zQkiu9Znn7B<*TE3d$(s#t*O+0dV0QfzMbu^<Ik8myK=6&{XWupNBt1b`M~7Ey=x*4
zis&cot>}GkKJC4&eR9q3uOD~sS#Q7p(#+&ZOa+Tqun258(92^oL78>Fmx047H%VqE
z&d|jMPOHoo7EIPPyOYwqaoG$87r~=ShDS8*Oafi_{A#*g8Mjp!9^1U5Vxyau#^cp)
zvWY%D%yOkEIvW+sRW7+J_nS--R@oV0WT?LT0ZZe|WpgGeDx{tAVbEqhFzNK!4No{4
z&!@avV?6VOOs}_|_u{jsd#|-@K0RZZkCW*NyQHIHO)p+wKEKSSOY=h4na7(KKF%^+
zr{F5=YVv<!lTg<c#TJ=WB@4W@bww8U>peQZ%)wSkQ#H`D$#cmx6|Jrni^F`qmd%NK
z%Apw%TY6F>v{}n*=@g$!CspR>wT6baas0WgJ+tqxvD&(AS=&~xVrkg6di#kcLAM<%
zj)vXdzT@gP{XGZ%hQ;jpp*4L&;YCp%BPHqCYq(EN>79P2Vs=)sNws-au($4hrNs_!
z<}{xbyO&|rcdt3wXvczH5w&v<uPdneHFV4j*4y=J+3ASKPdP+86(>4;(|ghzwfN-Y
z-Yd!ePnSLEp1N$B?OTz3ey=Crw40T@&oX^Rc2dEC<y~h#I4~S{IF-J1!l&y_t4*d{
z$mCIdB5;h~e$FG8Re^hc{dW;`Ju>Skf2>dUQTg0CnwyTO2p64Dn3;3??e>RxLT_i7
zC9lY9vGh7{sxsN@(Pm@T9o6S{FK21+(_%H4qoyZ%Kge(Qb_MP8w%nUs)mkkN&3QP}
zBz?uhv$o<TiG6%RcLSUpbwj>hu{D};v6Z#x%Yt^ErjjymZB^Cpopw^1YV$Z%ZI?`}
zQ<@qcRFO0_?*Ekj%&>1gk2otAOc2+){whCLE`Lw+wpWonGr!tJtx@97UXy+C)VAAS
zZoj&=?)!75?du9&K7XYD`@Ahr{gHRq<Li&MZCO`-b}jehoA-i$lsdF+IjPAh<IAKg
zEGDtl$!7h`vj@_gHq4CBGm&BGWzS@Cn)RP0J&CRP#=<V=IZsziu)L6Y?&z#$nZCsK
z@|&3)6*)WR_HT7B5%oOpQg_06`i^w_4}l87Ju^1{QQ2s3+!@fY;Di9Xr=Vl#l+N>2
zX-Bz3Gn@imKMI+5OmTyc16Nq^#f}RnOilK=y5uixxN(VTn<VEkH~UW$``(<8uwm=e
zpCTBPY;j<T!?Cluep#2Mn-y(8E21~CZ?@1h4s-Q3AI}0NiL*-b>E`D(iqxj2B=|~Q
zpKaC1s=~8y4qN|<1D8vW*sREYs?ED}g|jYKU{6)Z8<kXRpSMP(2UpJwc1jiO7Re9U
zV->fy!)f)WGhLM@diXCHG5JQT8b|D7;_|DRDx8xv`Twjfo?%+Eh3wYW^v`)L!z?*P
zfmL(DMjeHq``aGu-1qCOaO}ff@;g72@2~FSHtKHlvA=q32Gc(7Ijn1ag1=2wZO!s*
z+9P~2`%R-`v!6-HLgy>lTtbGuc0H|^Hs0Vi7rWH6@l%STw50D3lTB0BilplOzU!0r
zE$D%6V7l4nmCKV#FTWMd*5p#UaFM0Oq<$q^kL0JM!0uODu6Q^yy^eW)ZPPW=6{`Y!
z<)x%$4o|z9_0rHi<MHEL2WClZ6P&cBrG!h(Qc}vL$kT_X(_zjhu}1GxrtyuFRW$h3
zCm%aFRWWLUljheeT+uS=Q(Jz;=(tbcu)O4*W=E;&>#*<@F0!jir~aS2)HTuF`s_Ww
zApV<k-#c^%PuWy7<H+T?E9zWrU#jL;pUs*2d6K5>i;XVf+Ku<Jm;Q|EVtSi?j_JO|
zaZ5XfiH$20H5V5ious={<@($g-S_`2@b-T>&wKurt?!Z!J>7qRC9NTSv9E^tqyu-2
zR2Q0Pu4g!>J(*)(r^thi3v#%&InKVJyzlc$_GC+?L(3CNk`8-5ERMJl$Z$9yx%CvE
z;|}A)((~RV`*S7<-26PLXGeR7$=UN2uGtTHE+5;<zUWcA?Vddk^Clkg+B%`l;>y%z
zehYVG=?F^2&sk);ErYA{NLFvg7iFt=7gv{m*%)#7RAbr0LJygU>k5z4Jd_HV)=t}^
z{a@8QX@}Rbmy@~YoK%%ff9jPr$vaWVH#YT?=Lf$}>r%cM&78czPrmBhM&roTMB&+%
z=g!_#`;ofT=YfsJrFWLo)~$XzU-79%j9=IE8BI@5&tO`0d0xtlB}vXJ>yG+nsQHFn
ztO+&Fyt>LKeDmzBSC=ifyqm~#xlQY0kyydeo1&#nsa#$P%H6j${hyr@uKT?D`P${(
z{{K3&=ihwFl=5eVUZa(p5sT}K6`~sVetHCO?@EraRq1-uRQGPt)q5I8nA+a<3q>qZ
zuH9TN6%#aT3!ClpL``pV9%HpvK8HO^w%6r#ZLT;x``#BpsgOC3HdnV-IR2?q3hMo}
zWjW)Xm8{`SOjG_}xUwqE_gKhPsT&VIDQ&HIsw2`qvG1hO_H|4<*QB@>9XH)KWs4c>
z#FS|#r_Fy8=6fo|EA*dmv^kUKF0KVD<6hpKXTb5i>)y`xhN8_$pZ^qFI!}8sF;*m1
zU;Ff#Q|HQdg>bmoI0;^OwdeW5OMISEce^~6f2%TP?aeOcE}L`5Y<5b%=hC^FxyxKy
z9qw6w-F&QaYH0h@2aFRMgc54)HtKxW+q}=aB>QjqubaE-uHOCn@cZ<NJsILRGMleI
zzFOyy>L0godg7{VuWxa8Bzh9QOznK0%~W;ay!(E^sqcL}W>h_>=u6T%QpU~76V!Kf
zgA)6>h2<qDCQLfna*)sPD!;AT!~cqYJMMmt+&=$a?ZfGd{%UX<%@R#+I$H5}_YwEE
z0te4tzrv!D_?v6{t<E@^I;ri3+4&O}_gs({m$@AHxAINk#0UNM_C<3%JhuJ6%`@}A
zqT$Vx^R{SDmp*wa<<XgWO}_uuuQOZht8nt<A)Yf59~i~k^W+3L!*W&lniP_nUzPct
z(`!E`vYOd(gW9b%XFZ-@_`(zNXN?lW+CV1VAhxv^j9kx8dmpTGfl2&qg~D5v0`G`v
zCoe^~U8z&Ac6xicqkv;-)ABSX&Q8@&_ttUFcbz*yn^T2>K}dl?M1eEmY|WI{J0CDz
z-(zt7!qJ!`A7WmyFKBuj%aLW&(W}^_`{~XEyML$ttI90mW#PVcuRzo7y#ET_#cO&O
zuRFUq<-FA{-?k*DJBN6e>Zu-7$XTlL=Au>(+sWp;Dj$DbTD$B;?!9Mg9W#_NmUIaJ
ze7k&se|1w=1|Q3V0G=9;#vNg6c5GWOI@y77p4X}+s~#@udblAgTBpxY%a5VJh4=n?
z_cN_)j&VIX!S&4Gq{#=ir{9)&IW(_7r=@l%<yn?qlz~zpoA#zRZ}`juSd>@4h)Cbm
zv$!;*pg3vy%dB)oy`$|xUG`P&Y=OsbZsL0#@LGMB-nL`empTqrg)QAy;4MA9;BDKh
zw<iv19Y}h&?1AV?1BO)wj6w-aANX9p^>H)v*X`S&RI2-thhKR+H{b4-BL-|4%6C+a
zIJ6WP7!-f9u!=A+G3YQbFfcSQIWRJC{AXb05Q)fIaL|a2eVbHAzyqhoToJFF&}B(8
zg;H2jL{7LW%<Pk5o#f@QX-R`GlUuJ~$D}776FJSdt_t^dSX;(aFzJ#}*Hx>@4##<A
zOhXRr2z2T1GP25g8MsJu;WCq5SyxsSu0FChs=PEie4W!lt{*=qF9~Q&KhCG4?Qq9o
zd(KhaT<h=yL5sXyUv51e{%TcYB<nv}9lxm!p|9A~?bb})`0$|W|4A)fqHCY*t9@=J
znj9|j{H&%ew@7ZltBhkV>zA#&^VJ}{asAb0zS$GLIou8o&f3`ddEwfslsG?8zAZZr
z_A1))xzyd+QT*(jYg)?ZO}v^d2TvAUI@iiw{(jD`;{TlY=l=QeF7?E6_x1CFCG}Fh
zmiGI{@B91f$NSgf3G?g!G_ndliD|ef%5kOoy{mgyY08Xg(M~~IQx+8Ia5{A9@?SmV
zrTb>Xq)z`e45E(`*DO&DjjOrZniMN6_FT+UUrl7Or{TF9k9r+q*>0w?ee8JL&m^e2
zL~XHv=aUd;uAd7M+D&&Zp6sw~>ypOOk6Ru(t-TQ>tQoUuBd4TB@0H+8#Z8G9y!oe^
zBspa-o4L#}d)Ak<DXNV>FV4O9v{Rv`>eJ5W^Xu5SG@6*E6v|3|nR>CUMl?1&(2=88
zBy-`PK(5TCsXGca8=028>~c}5jk+_3Y1^eXSJjf1jJXDa*OdY_a{q4<3Q*p;i&OgT
zjZ1#odu|=+nws<1Vs>CKXKK#IDT38HVG)f_MZ+)Htrm;Sx@bAUiYeiBSa1H@rPqUs
zI92*W+D^_8O=_7tdq&usMYr>I3*X{csNu`{Q#Wu<t#nk}y=5GVlX`gDriMjC9$mA=
zzBM$rJ4bctY_+v-tS0Dw+v0S}jV~&C(oB`YH6Q!a+(agp-DypqSNUk^nop;mZfsvr
zZNU9tUd{EF)9;`C{dIZqQl`3`80TY(T=R>gS&k=otaJLV;1%t1T~M9LL|3S7o5~e0
zwdjjZYZ<bCx#(PX6e|r6Tt4-0{*B2K|1JoTH1FJYJ6rY1;`4#$e9b3#E-P6wDS!6=
zCCUj6#*sp|kKNQ=q!u+H==AK+=98*3#Ew;de^Q?RJoe)B`Y-dI)fV;5Td;uN%48vj
z(yKUU(Ma+6uO@5Fnz(3ER@90&OHb$}yyax%-BO(SE>6g4=@bW{VwEkcHhPKfd%LPY
z<LRl}4sSjzDJ`6&-g5hvkEPF!fS}gWs}c8|x$U;*u)a3lmd4$pyQ5|&>pP84>$Zyi
zh$@_VBRlS(SwzmC&l2^8n-yjLyi=08AW@({SMYkwq0)7SH?Y;mD0W3``?j?;_Pk|)
zq_djGM(sHqN94Bcm{D-6O}SrFS5f`TjpC*>Q<Isi0+vXeFW*-@Z`#d()VGQKd=DSb
z>pQ`2_eoRa?EeBcm$`q_(^<TfJe7iE)Gp~NacWKAkTzMWF!zIsE>mY~kki(%D=p4z
zFSuMgbKLY~NpJQY<{6X1p6H8kZcxcn^pfdRJ-)KQJLc%5+lRQ0C_P@#QPl8f;v}|}
z$4)rB^lTDdQn)UCmOy8saQEvg4zDhDMfX1OHsBOG?v|()RIo{9iPQ;6*`x&lvr?uw
z{tmim(7rs3t*dxZcj^Kb1r0C76uS<qDGP(byMhz50$Z8VmQ}6P@Zb~<w$bie!Ls2@
z>Z~I(g%U$tgIv9Z3L0lg-t;Kgw!(w4XQArmB2IJmrb#-UIf_z3%^G{uPL(lbMxN?=
zzL96rhRmCjYrgZOG;UM<uWED7Sn-EQ{1&!fYl{D}MB6uV%PbX9HeyPRx!<L+|I4K*
zoqnk~D>Hq<UtOQt%xf}#!$N6omyN7tcUZicGVX93zo{h3cf{wy#~D8HUcrrLwm6uJ
z9$2&?d6U$yDX9*;S$vf@S})$bWNV<aC@42<!m_YaE9YoU5T2VMu<GBMRW(Kzn%oY`
ztn0emwJ}TZ%93M8bvRXA6qPPnEptEis<BrzpfTGktWbQ{uhUw|-nVc6Q<@Umb3#As
z_iT;xMz)L>m<*<`?C$k55fC-~wlPh6x*V%g<jGgU`B};f6%{9Y9G-e?LRQi9c$pN@
zy9I02LKn|ybvxAMTTx`e$~93_-&@H1f1}qVxs!A9VqaR?)xO)lko(<&oRbrJKN#8N
zt&lXxR$uw_RNjm~mG6q?O3z*K&4(c`;e^A2p91k|#+HY?J=VTZS(5LRp?q37-LGWo
z<T(WyJhK+CTC)^I{Nh`(FM-=D?W&p1<~wThfAFkq$kbLo@$Iu?re^#d7At$U!>R^c
zVfVM5I;z{sSK(GBJDn+5XkAyFiOKxXUgp)4?ALo3)PzdL-(kyH8~8ybiEZJHi*ir2
z9($ZyYJRA&WWwR(xmn?D>aY2D1J&e|*tfR5eyCWrMO)z4nVgjYd9MT~XWo7EW1U54
zUzUqSDBHrf6<cGj>TvNkc`~P}tT8jx{_@7CZR@nw|8jh}>QOt>M4Qh}?On9twfxUd
zH#h8TU0=ykBKu!xzrFkqcSEO%zi*v*VfRR|zfIGmvHS50Pib|xyR+X!o_&7ZgsoF_
zmc-#Un-$vi87>|Zey8NQUGiMnBffpR>(fZrkW(+V94tynNS#+PTV1jHx$?e$b9CB$
z??m4+ntojO*~)plk0!AR2hEC_vSsSs(^`jbZCk=~wX9L-h+cSbcdHU}Rlm7t<jGf?
zT`IR;EfGE0cp#iXHYvF7;kFb14{hH6DaN4TyKK65wD&>A(2tIrOV>Jkch**{F_HUu
za$m0L(}!QWERM+Y8LX0cqs{tvg@YrP$B#{mWKA8_IrOhRm*G}e{_Fo0L!VlkP4n1(
zo-pXIJk_&yx6{El)6CPergZ<fGEctaq+Q-4hh?wk*lv8~BerN!bJVB13vaAE<08%R
zSuA{OOUJSeS0<m)`V;kB<nh;YWfwQ!@(FT3@J+O!txh^QQ)R}^jJ@;kTld`lyy*F^
z&4L@|s{TmTdHsHp#+~$*;OYkt(k#jjomAP<sGWYxc0#OUYu9SouD<kK!{#&J+%}o@
z<rbg)srzHs_Sv6v*XwWJ^qs@Hq*ZF}t9(n7s7HI0jGFf@j!^e`X*RX&;<~$U^Zi$<
zW&hiDL4>2+pS5~vVvGIFknKvLP0tQ`?$vl^vvRj7<9oraU)^r64ww4hncwjF_x=n<
z`~QvaPdwLRDmi+&X+w$_vz>(QghSkWcv2TAP5*AWH16wi?j2$~4y3$%e?;9)jQ3cf
zpqb?YGd49<L#6=!Z3nn+i<p^)$>=RjmOjWe`MHqov7jX;vd-?B&S5DE70z0R8FieS
zEVekCKCeEutjWnl-s*eP&xi6sY@rNEsr#MPL$0zp6{o6gRESYZli~;qTpY@*Xu2n;
zbl)VQqz7ru>27tbX0eIsrPne-cBF4P>bCr#9Y?#w|F5N$ZsqYA<t3_R5kCTI%iE4#
zG;0wtOK-23ZI*sD+0N-v(%OU|&8b15#$0Q@scIOiDKUyFACwXNQ7!x^Z2ESY)l;(P
zZ_i%Lp5y-iS*4b9j!AgVO7l*W=Ng?wKF79nE;q_O^rZ8MO70{Un`x7LmbfTnCgpN%
z3>Nv6%M@tiY2IPMq`jG`^n7@7@&wf>OB&v8;7Z!UXLm62OPIp<Y3_a#-M1C$<Sgv@
zCK8mVD02Qp0DGBZ$svE9@`4*2y{uwER~7~63rX~Eb0||(iOSHgRo444y-(0RP^esA
zh&}M%p&qI4wYBWwQ3hF=pQLXV_Dy*lrnWG7#S=s2#e#b3#cY)mUJ6#bo636}oxsQ0
zEFjrz;?dlAq}lI^fYl6z2Vv?wPFy()+JoGS99P5%Ifbz~hWRp^hAmFp{48nl0@G-7
ziKy=_(cwy(isAA9+r<;Nr}s^@%g7KYO_V5iZ?le^5}z@J?OVD1PeU(B*~WI;%IO{|
zf#S;y%~KYdeo|#{dE}b8NcbRQ_ylLJe;yqvKb03wQ7b>>HK9X&nR-?3&1o?PtlG|O
zMUtJ59If203RHS_u2HWRofxyn#CkJ(?%A!?e;&r{`L29=O4H8A@~uq*tBh<E3tewa
z7kV~T;Gs&~tD{`|6>FI`vhCOCVVhL@J$#mrqyC2#B8v+|{;8Tz-Pl{QG0~^7&tc}2
zZJHv#r_UDss{3lo>_q|fa$>W;KB@_EoV|3TIP;N2iBK~u$=T;R=RCY5x#DD>2y4Go
zm?P)3B%W{L_KoG$CKW0w@sIv{@^5kA-|8^$%LV?g7r4J(;QyN-Zah(KYk;BBN<#~=
z#!V-jxF-t!UKuQYIoLcU#eS-hnnjbUhTN7GncXcR>n~5xnmNI&+*>fy#j@BXc$tg0
zWM1GB1zw?!rsZX47J8*Ei<^6BlGGyU*D0b=7fUTO!>v|Mt~i#lb!WWftMaGq>0dAP
z-Cx8h(kglSWdDq#=A!1AzAMAig`=Arr)D4J`!Zpk-z|RsTl0ce%?t2a`ZXadd4a_~
zMX$vPvP&+sCvVS6SBkbMnr3mSGE*|A=S4@~3Zr@7m+5StCL8EI?Zou5UsaM17fx#O
zX^rf<VwQVxS<bSE=9L^Jp{3CWoNI0f3Pt^A5xRLr@aIo0^^HO^SK6ptG}#}xXpu<P
z#=xT6&h8$Dg;u-zgqDiGQVINb$h}{5v9N04rJH?^Jyz{|DY?gUb<zUP?W_EDWv$*;
zwU}vwe#s?wo1Lq2LsqR{r7x(u>dlHiQ4zDVmJN(%^PWa>n_Qgtu5#^-S!-`(tquD%
zk1dcphP7yiX%Rn1v~FthH_u?ni$%;;6W(!!9NsKzutfH=;6n5C;xDs`t&fM;nopd3
zxY;HmMv7C!)tT*DVsLtuvIe86>_HQE3xW72Nms<%5?-u6<Eg(eP=){aVmGHinP`<o
z7d9$!Z>kbjxuPzh&b`TI7H9dv#Z{L#8T?L66W*+1&872y_a;ScE}hw&NgFm>{oZW;
zd$Y{!%`+EFwal(_HImYc-t2vQi%IszfS(E>KNDM6TKG3{d1=Le{jel3Yw16Or7v&J
zON?In^?^))m(Vh{@b2a)%~bxa52B<RJ&RHmQnb_;xTy1Ql<oAWs#)F9tFio1;PgVt
z9gi40nGdRO2+67LoL*a*<141QlG$`^@rHBDsxOA-o^P$btyWV$J6C{l!(1h!yJdo0
zzh+c+nLIgG6Y(YV{B!~HhcjcEE$+GKuUa|FL$EMepm49}>e&Yu&ud(LVD{=Azjn`A
zICc9f)&tplG^_>mR!=<>y=O<)reeXp=YFrgcv|v?^`4(jdpel+ocO)>-v8O0_qF$R
zIPZ%(u=#HF>QlS-sh{3==Jy_z!o6>!g$_jRS(LbH?#8{pX7B&ATe39K?M=YE%USbY
z-sIm~u$FzzTF#sU*G|oQcXHm*i`>T)52_yI<ERieTqdaWGOM+s=yagWp}_esUL9QT
z?9K2hm`N&xb=RRs2M^tK^geqrMP>T}fn9QD*9BjGw7MQ>{5xx*u|~6<`HC%9Q{B{O
zrY9N7a&;_iH1uND-f6ao>7q%r16SlE#f;K@C9Z|%x>Tx@RkT)bvX$PH*t6PN`j}1h
zu?!uC^fkw9S07_OxS1(%OVOQUDQh<8?m3p8b38-mSar=Y+uNIGDeQYRoAvo>))t);
ztu-f}|K~nYaOPNH%%1(b_Am$Tsd#g&vSzcJcdg9qlZvZ1Sz2##|Fk%zY3pkTezl`p
z?^-Qg<+C)_TPW3=pPBJAlj3Pr$J3j27&iPlHF-+|(?#yt2e@J)OH+3roS<Cl#Um^e
zti3R$V_|6Y!qV)y%a_&tURR*CW5NofEnl*yRxI!GwyJr#V8d~Zvu!?eR^%MM(s@KA
zckzwt>tZEu-t6rDxpTV<+sa8=*GkT>DpEh?9yhrv&vl{7na^uY9=bAJ<Xf;!dgrgz
zM|NL0q`l|C?+bUiFAB+C6wKYnv_YDw@gnC&X_?xK;=UJ!V^2J_-lx)gLTxT<Q_m%>
zxt!W}FX`l7YUQ~suDh@Be~y6R-@Wp(mqga?yCQw^hV;qet)&MzdKi5UJojA7w`Q%^
z+ABd?4EJ`OUU%m-Ti`afk5`yJ&I{RlI&!XnKn#Cq<3ZtI!x+((7bga5O`o5g<fM2o
z`0cNCdA$pYql9i3%9@)W{-k9rr?{T~=3#w~Bmb(}TxAYRO|;~0@|=6c*EwU7!@@~i
zjfzYQ6{U>W0v9;$tlFGhbF=Br=I)*o69Z1n^*u3vE$c$tTTue6i*ipa=e;E(%PD#G
zmiXCQtJdCnbo=J!-dkI0Z*84>dwK4yJ+ZeU58U1_duJIh>;AX558S<dT=&*dS=OU>
zZ?4~a>w@e_4%^F9cusmsx7l74UJ^4ediU0p|G&4s<GdGcb?=exy#U$Mdu#;!Yqw2K
zJN-geAe<{a+{u*b0r%>wTq~yKr5sjG4zEmo>34DB_KSD6?=L&su-m9<_JbQT({I|a
zw%<<a-d%P3&Dq{+Z;nDMu}M~EMXlR1cgn5XIp_5m)x^-S%`V~xS+5;oY`)udD^XEp
zU(NKFf^kzzW*u+f>ev)mu*vH0g=e=PtE}aWbhv$V?rq1qeU9snxu3hVCU(m@T?X%Y
zPu%qw0{%S-Y<RLpmeY0J6Rp~(j`yCPk>w2h@HA15H9_uKs^7EZxMz-X&$9KNMXY-k
z^Y58^-&3Xo&!Wy<l3Tm4gk5({z!i7dZJxbrxqX)M{5in8=W4)z-h-TpSM3hJFn{!d
zN#WpwUxk<44HZMy&1?xi_b}|p#74!9MH6;%=FUB&cZPLo+ya(chh!E$QcKjZ>M5>V
ztHr<Sy0gi`z-QP07sgH&yK&MmZPnw6POlD%D0M4EDV%T&3;lB=!<5Y<;Y3!?t>%~$
z%kJJh5qs-e+}j&<Z?DHaFa3A>j-Eg$-#cc(clYDoozZ=ICidyedv9;Nd;3!FPNc(~
zGrX*U4)4G8u>>}}{}%WDXWaXb@7{ltd(Y&5hee+uAmIZ?{k!vh`+Ob4*EDSPImLI+
zcH85%^WwGdt(<dePtOa#w^wBHWm2U-S$z7aH2<pV`l~PJev--0ir>4GJHoSHS@F_;
zLqEkip2bbMTg<eqmTtd1$LKmw2wU8am<N?boXZQi4l_S|kiSYsHK$X^@8MClbIX*g
zS9o`wW3xVQW_?wyONm*8J5a#lK3kqc*S_yNm=exw=rOGMckr>>wZ)NnB0_1=B8^6l
z!e)UN9!N2U>YbR@^Q7p_+lY1V4%R+Pj{DJ8|D$96kFNbcdhY+|*#ACIfH6Sf=T|<)
z$^5L-&cFY??`N{#dsE+iucqua^t^I;*Sz3eSG>=zT~)vK!`yo!cYbY{fA1CNr=t(&
zsWQHtkSC;4e^1F?VEujW)dp+|<&PzT)`k7<Kj$cuR@BI`d&TpKhw^JDY}xYdsH4&C
z!s4E@H`o|WZ~XU{x3O7hv?IiEP1k>^>wEc3Zf;@aVVZO%Ug*$81?h!1jO<$s4w}q;
zbfY|8IOW2dS8=x(6cxH0nwWXT&LkNuJfIOMWL9%y!$JoY=FJX<5rGPBU2K+a7decQ
zMS4WNCy7*UT6A)XZsa-<&fv6D-ApVQol8_FOZYB!Su1rkOLT4N*B4){qt?&ARP}Gx
zx8hwUT9=oG&gP2HS}@OdUBvD>R_%s)XX|3O-?EVw%e}GLFLzfccbU$OEonE8#m?S-
zZCTjbtgVlt&WmXpE%xA4P;L;>3^U`mZu|4qFycUkE;qxSH13&aqjwZM{4<kV^kT5r
z{C_!CUo+3I+PPz1(&w!W*S7rMJ?&wT8$;gRJ>|{+=HB@J;m-ck6EqL6?0I*07ytCk
z&wG3|8JFJiu#d^ADmA~gws>ui<r?jEPd?5Pzdwg1Lr;cf!i!lq)YOkoe_JoRnR89e
zp;-0XPrNsLVOfzng+peO=HynTNP$HjDqW8bv?(QY9OBfTa^et&_8I}MMhzBj7e=KI
z1xJKDw{W^LDmXYXXefO6;Hr|4a?w>Rb55X}MqG~|(`==Un%zEIF1YXn9+OC&S{jkq
zE><;X&C+=qzh5pFU%KLm(dkvI_FVGPs^&cHHT%dO{WGUuX&L*PUXQuF(lBD>(z)w$
zR<?^CQv0oc_L#=)&HFYydECW&q)fP>+jqwUm;YN_T$8s1UbLt%xpZD(uE`~*ySqco
z__l4}4E|O2dOc6jjvKlce|m^MxcnqXYw?sPQKF(@;ptPihDGJ)Zrv5La&ez;{6uHb
zfUPGKly@EG5>60V_vuNO{W>?rWZ`qCT&|kUj@anx$Mijgl~Z@og+u#vSR}qH6kEU2
z;GB46s%L}P{ZjL|irlP;aqFJ#4Zm0Ov2}Y~qmY+}dZ3Bq<K+w5nkLIGUXrycak*UC
zj8z?LFRe;ick0j$?M+XFxb^GL?de`;blK^0_#UrqGtMsHSaU{<kuADM^l(nlw~fh<
zv(}uSbNWhT!!d^LX~qj3Kff`))Z?}`@J7VrLz}O;yw)<CowWGY|EPx?Z_n;{P^8^w
zac@VRsO3|k>UDN^V{{kqHd`>+Z}*nF@6Pq^sFI!VZruyF{?%LiqtDy^xZ};U@5l*@
zH-?G6er7lSuuPbimOe9ek0>K!_R_gb{5;|b?Pkxj7B!mKc(`!6%Wo{X<k`c(c_p-+
z>8nI+%iK5_&pn05WD>tzaBq_dDmbC!Dye!>b#~39c^YfqxOzX`Iz{5NwxOHw(@oc$
zBA={Z_P^`QYv$b*XEtnbx2f6c@vQD_UCO#WVjpuhE}#1E2*398ux}}k4IPDd$2a^r
z>9M8W{XcsG1M82OPK8UOm=1_A?LTDyuQRJ)zEs#|i;E6HN4Ubpw}i4)rmlWfBJjU%
znk8Qz1FOa}9xua1S<Q)4o~j>NBTy9IIIYftb(*ACTj4gvR$Z;d`=T~9i3Uz!+HtBx
zZ+1_Ml50+&=c818HN8OJ^~~Zs$}R~vbGB-1weSj3*yg&-=V*5<%j47o%QwEAnyKfZ
zcv|wj#lG!ZmTDIpMt;>%c4s`msu#cTvFzz5{ADap_00+;=SB&N9C~p)y{}O099zod
z4@)>R=5q30v~zA-y7t(l^`5sb&6_55^T;x@(8*G(cbT2Kw}8uhkqDn=FY_9KOYY|D
zdd?Ls+dSv>2_x&f5Bb(VS+;${mV;_#kGslpF0sX5TWg-uAX0OsY1N@xgTloTG2uKX
z51lAT70&se(SG{ZCY{a+uN|L!n)7_uriJGVU0s!aoqC<~X;JB`zPa*;PBbs^P*GO&
z^z<p3H2Ia{QoT}5@4tV99Co@aU6=7*C;85eS^+6>gT|!`Z~i`Y{MHK}Q`;$vvNmdj
z8&%C`+PFw=Dt8Yb=l3-m9%M$wz6=gO#4O?Xwq1MI!}!YrOEqTw>koQ6QN&Ms>*a45
zr<PCe@~vL-O!HyOsSrckt4zLAWghNW)g*pJlx?Y0=&Cpqy?E_|;a(b&fwJC_7qY@k
z#Wl`mu3gz?dFI(R3C)QGrKvGbmBM2lTCGz`I<juV>*-NUZ0mG76or|iT}7`ua+h6k
zO7u*W5Es2|&=#0eE!mv&|N92@i$-o&R>l03NZlPY?aZ}}eQ!S8NPX+<ZquieQWZ2U
zK1xmXn1$Y&gqxR>-#l49RZqxf`%;-Cg_kBXKb1|M5!0L5(tYxwnE2#YqqA377`)cC
zT$;R6Csp(0fq728<-6h!>)kkDw(Zx1wU!ypq9u2>$$ngyYrQQsG`!35%=#Tk)3s7V
zCA|)`e%1Kw-;t^2x_{S>Or^A06E6Fk-DbR$%epaUw)J(#W0^B`3{I4P5ow(mX87=-
zW9R(nsJOQI+Y@z4`>UfJBcqJirWq=gEUt*mnBnE7ewNFj+V1o++dmiif=-;26Z_oC
zP?Oj%dwA)>C#O%R`sK`T)e||Px!HTd<o|!f%6b-VV1C9mmqUGx_^(xUUlm&yd{Wn#
zp_jFcndO16r0TQ(q8pa8KJj9*G-&nB)v^<I;hDqVF3vXXRcO%Fgn7$PX<s~c+&oM3
z;o}<y#%3Pbtp;<7Hz#H2{`PURx6ElhU2#KVTT$z}fC}wpJ1<>R;7sUBe&HF=bcpLv
zlAva9uY`Eu-=k|z1Zv*@Y!G)XV`k*~mkT<Q-77Y*+gaQ^BKPgY!u0!1)1=GGHs`At
z$~@%QWHrrab8e(r<gG`?KGm}Jox7JFpd<eBgHD@>!{z%6SD#!n-*(x=Z<Q&Rl_WhM
zY}-BQon3F?WNYasm)Jtj<-W@e(L8kjMoLu7?sa9GPfP#*sdGFh*MaYLl&JT%w{3fF
z|LpVq6y}q-ZM)5OR&UK`vsWJ3k-l1O&t&6piL;Y}ero^f{L*mva94h+BDdAYEes}%
ziOQ_`hkmHYiLEtR+W75`x83uC^S^CML}u$>(^M1C%u3{MikepYZ-e_bufRSH*UaNL
zvs~A)%RH75zH8)cCfxb3-R`vO&f53ZB~uSyx^%s1j{OhmkS!^%Qg^jmE;m2=`>y5H
zd3$%ye7_~qIM+<z)Dpj)AxoUJA~<$vusvJWbgFRGJXQXyk(Ga!viBscTBNM8UM;&N
zm*1P`bd=h!Ii}HikG+!9-fdpUvvk8T=QU#L(_Y=2KTA_|wReiLb;*+D|5t84mRDvQ
zws4==>eTDge%4yoa7=F%sz0zqgLT`Di~TF=cgtFCYkxW=FY>gq-jU_5WiF{~%ct#K
z)7slP&G$xK(9tDxXI<aqljr>S_PuShP5dXD7k-^z|4{Gx|1+WMR&DIu%e|v9x&Nin
zW+oAf#?70L1n@n#=o60O`=BNL%0jw*^^WO>ES!EWjl1L!_h|OKpv@-CJ5S%z|2siU
zq-M?rjtx1T!c5MSo_NmVDVAFu!KJ^FUBA$<m&x#s@MJljO)pk%TPAJzp_^OTdD9fl
z?y3{&WeKf?3|#XhSk=z7XhgI(2`Tee8y;T7dS0q0Yy-Q@8?N{l?FR$o-LzVl-rg9W
z(f&VZhn?)>jaGqHT8g@6Mphd)&%1Q$u<Gx5n_sNhw7Tb&^vdpEHucgXGps}mf?eDd
zjTugfn7kJ8yV+!#y;)Pj#(eSucBwhP6{d0q_4)TqT|R41;GU^`Ov_F`l5+0h_ET(q
z5nz9zc<pJ8o-eBxu3NG2KNGj?B$KUNT{l0oAK_YTx^nAj4ZF!Vw?;E}C&g%-UDO_0
zz~R$;tl?ryRYJRtLsMvmgwZ9pwHj;8qgZ`kIMr)#2X-*|&X~8kdPc5D17mV){f>?M
zGT6&otiJDLmAuVWc0|ecm4uPk#*EvCJ5O=f9_h+v_Gt8R>T~Sw=bUFBwbpTx<oV*B
zDNNjxjm%3f>;5lYy#Icc`DF!O-$N>YZts7($b8)j#hT!Cl8?*<nJgA*F4$|ml-1+_
zM^0ae;SR2#9kZ`Y-F}ld#!GsuhT4k{4*W^$U;Q}QYrXk*l;i4G`llP_c>LD**fIIR
z%!BtXA70(PYxSLtGOw*pYiu%#Tp_&JOGigI@P?o+2iJvDJL^`odN?oy{t%j-ag4)y
z(!<3wbr_CcsTTBI!Mb7vyRJi%;faIG7HqUyHEF{SC%NjG`Hv0jC3=Di*w<{9(frsx
z-+12#M(ZOlz04ZBN+p<?0xj1T%yjfw;ug%^>9byM_U`{5H@Tg(o_A>XqMK7ZU+%uy
z<8u4Ml7rFfI2HDUIPUTH+~c%z+W!Yn&fb}!aD4Z=6NfiHd9vV?_0jHMQjsM*-z80g
z1h!iqp8PdiO)_Zm;T1>s-#9AgCDz}?RmyXy|Kh2r3r9UJZ~S^jM}cSUqtC}r{ZTlT
zW%%pM%&v&mqrY7Di!fJpa0Tw@V9Va=lhL!ahiUtruD}MS-UWx-BwE{7+KIiMcgD$G
z@-R=>ht_2o@_q%Z%crd9d}-I^H1o$6{-_HrYmz;z0(%ZmnE7A8eB0-B(~^3+TI}6~
z`hTw8-!o-p-<|z(Pxntq(PQlC4p2J4x<xNYajKGS5NnU;JE_^>k$kgzrfR5)aI5Gq
z<eA@p+2PIyX;v?V&@=kNDKh^)na8kjPdPoUl&Al{<7~$*KMsDncxv?)=I|N8<_j)c
zZ)J{7kY{u_Gv#L2V{eU(TMipW9#ZZxwR^**b4Y2XU~`~>&V&fo6%(%LuN3liXz;jk
zx=w;sLX!2yYbTD5rci-~&;urA1zd&>Obv#iWeS{O3l7T^I~)1PZTQeyrob%My=z~@
znW$XRZzsL#1&%M%o-b~EXp!=HqfHtw4Rv2{oMaear4jA^O;Y8x%bfGOrJN&O7_(el
zxAg9x>@sN!uV;y|x5{<>-p%Zr_*G0dpRW*ox<VmP#P#Wv@L-i)&rKBh_JkkxSm<(?
zds}vojpzQQN4O^+;`#bSan{R_=9bfm#{PMmo5MCRn-?^j&A9x(-Piw$@V0Kr?!_6C
z)FN4DS{~k^aV*i>Epl<I%Fj-Zr(IDmWOWa0(GXy6oyoEI1?!`U+d@}#gtB(FJUv}?
zfu-k$ddS>EbqNzVg&6`XCWU<UU%JBX<j)yOPc9!!*!b!)*9z?jgDvg#JGv&k?V31u
z+xCEM4tI}lls%UC=T^rmQxmIenWxu02-I~5I)BJvzx!*+EUB~k$`L;=2VTB#!sn8`
z+|$4bI(ieA%&I<Z{?sYvf3fE`4^O5$yYpNROzk;&M`On+-6^w7cC=XU_%f-lo0<E|
z3F-5HxxzE~7PHLZcUfW(>Sa}I|EJ(o?k3Ks#UY7DmA~$^iarr+y_fm>q==udPX0Hq
zXuZ4j@~Uc6#t8O60oJ(}IHyzy_{@&o<e;Hdd8^@$MAd|uO_?G6kGRSTZfYb5^qgP`
z{Og+C()58<_r~Yb8Yfu%Ki{*>>?%zV41X)8{l`-4ry@s1K%0WjS5fZiygD2P?s^X|
zhiR}ze`x1n4P&uOkoe4+u$J9u$3E%HhGj9wEfR&ka-Oc=<7_*{U-fm=nL}%s*{(6i
z`NVH!H@nevs`0LdBu}E?_DNoo{~QcEA8|J4>PqQ6UzvB-{-3PPXCKMm>$~cr?@bMx
zU`apbS0XC0dpx5=*o(rso02M2k3N;L3ghbTua-$Uc|GD!*EFWpM!x%hSD4(6Sa|AS
z#0Q>v|Ch=xK6lA0Z!@cvue5PNvynlwo<_5tLGzZECtox0bS`3N<(=UX5wM$uYvGOM
zA5|h}zleA!&Fagb6TG#vv>;OPBGdHKEqYh3bPF^uVpyxy!j$u-U8g}XYQ_woh=9h6
zEujWiS1(}gskjix!1>XKJ8ZApQibC_609l$&C4QM8P=w9tmRu?s>~J2vwZKy{AOXE
zy@m&Gn2D}koai(8hswpIUheeMH|~Gd`J6ISZWZgE+*aKiOiQh|Ge%3kdeil_*;VYQ
z?njU9%LQX*KDyoi<znTt{r@+oPxiU4YI}hB&$YUBdNW!)*WBdY#Hpxh`eK&Kluea9
zyFRMMpW<Ou;BOaW_#fZ(;-alD<BavcesLYtbll1<pK7?oz|1mDl6iT^<txF{_>^bf
zUK{bfNZ+#UrCC91L{Oh!#Ezf@=KU)sZK@DrTIj@bqQ&Q^(BZ6<c^Ov<*0dWk#Cv^v
zqO-%P>VvnghK^?Cq-6`(v)`Q8S;^}AK`_Yes!l+%l}vM21pB<xLK_)4*Dh$)$T+=y
z&opt#)+dQKB|5XWim-}Fay)0emghHr?P)8un}&zI<cw6})_TYl&N-zsGjxrG(R!P0
zzZs0QGasB(d${IE=(mG8;V&+jWwb_CT^CSl?pe0}`~)VW58RfWKG6xSj!Raop16C_
z#)mibj7z^hm*=~F`<cip$vwO)T>r0YP&n$GR2mw7?DLT_-lLk+!Xsa94r=n7m2*@-
z?qXqcp4I2|(!2U?JQJHkSGQkTTyo{8_0$`dWz3fEdaL%Zo?XUw{4kH0cHFT=>>*c~
zOs5I4Hr`as?C`$VHUFzvY9N>Aw`m)9bZAUSEiB;*D@c90SE#(PDey)6{N5(bOzsFb
zrtnnLr2<cUBD!MEF|o~;n}3ueNbiGeBU5DD>$U}~`<`%3-Q22vT8wkqymw_YT3D2O
zKDuwM&57imnV!A)+mk}s$wK=RT7MSGTX*s$NzPJEk9o^waql1Z*Nci`_gKVwivp7`
z&ir}3C{EJtSAW^G=M~?ID$-`@%dfk-g4g8m^Z&;7G5@6x<b|G`{>Qwa^v-GDPcx3p
zI@K%s<U-8s6pQej9czp9J9lfvd9C05vezi0|K%r(tsFZSUgn=rrul92yK^Cd`#WN1
zS{B|qYZ$G;YHZMIUXf{Tz-lZ|8aO{A`bVghp-%s#gDlSv)(NmK5X+j@%E0tBQ<ybt
zjYPJFMEjAj3mzXd!cMTrb51whap<2~r$_DScoxnBM{a7&*tgk3j=8<zkB*Pgs`mBY
z&VN6~;cIX&aK#~y18ztBvI-1C^(M{?wcygbs8f1Bvyfk->6Or$8FDR)ls|BK=lUf`
zu6}cBpVygZX67&6gsl*@X`k{_t76Wq^)2t4_#QL~y<lohulV2IUvYOv(;K^7<GKg>
zbq_yuiWSUDbZg1Iu}$;FnU#jY;l2+aDv0g5J;mjY%JuaINxxpWFgYnaoYp(@cqLm=
zIM*&?4Zpl+cc$7o?+D(Nm&|ECeUYDo<MY24{NC>2Nqd;%>76z!bDPo^_HYeWn`_A*
zmnGYj@mM(~-0JRH{DJGu#Sfu&w`46puGn#?tKuo^)Tc``o@!JCXiBmg{)n6O;!0Q>
z6U$nzwGTMze=LljmzvE}p0#@M?zX0_`EJv@-FziJtgV=oo^E$+k=e%nkJ9%gq^|F^
zJ@xGT`$DriNrm@{-wWp}sZTq5<GJ$Om^(A{uRou4{rHPV6aHRTf4eW^)8)Lx|Hpo4
zy_$0_@W<c#{$ls-m&`qQQGTg?|DTIhVwdjv-1)6=sow6=?QKW%{yx7Z{$dU1Z0qO$
zxSaa9)?8ZHk`WiBFTFHlZP5C<#p}OwX!pr!_Pfnlzv<l%gW!`d&a7$<o6p&`S<dV#
zTZhq-6H~G-dpT$d$69d|JU+EhH=2p%%j)azAGvmmsDGRNqEMtoV~rlyg(j75hN#sn
zDW@iR`kbm;Gr>W!d}FvR<CjGWlBtHtx@BuzTsw`r=FOP=#)E5xs+NjtRG>n~u9q9b
zR5+Nb48K}ToT_H%-N+Styen>PPpsyuuN{W!$9Kr?3|q51kRhh8a<AW+X#VBvtM;+{
z`1Jhry@&sgbco;Hbp6IO?dY{UL3*LbjAk08cojVKS!A4XZjp7lX5g$ZFRm;MFMfAm
zR(JO0c{@EaZmko2SGM)dRw>nnEBTE38io1oHd++yle5gddFbfQV!cbtgV*Q%wJOs)
z^)pZK!rtE^?`8H?f8UpQYMSQr<N4w1{_bV?@#*pL>3R*q;(jS#inSKcpW?t6W%eMI
zRj6V?Bipv^3eBgc9z5F2%jIyaFn7YCw*1?S?hnJc)yywyq~rw6YdNlY+)I1X)k`AU
zOdE_&Y5lwM_<StWkLDTLOiON@(%rt}lB;CwjKpKI>+Vcc($;=-mETs@@Md7Ov7~CU
z{Va#Z7O}t$ERVZHe;(%Z|GzqdCEPa9Vc|(vlV6wnd%mVHxvG5KFgZkQ(S@XGYLO0w
zSEj4IRC(-T(kH|j7m_z!cxu5~&Bu$9k1ZANJ#~>oHGWFetFY#*b2lz7Ypim4oUnCE
zkoP2wNtd`Z#U^chC6-d)=(bv5pVsT<t(U4^H!psAYbL{-sTZ|3sIo}wtUn|bt+SEI
z<Dl!L#a7yqnv3^H=t?ej=~mQS;$zLV?N-!k-5qOlKFNp&PpMwF^O@E1cRMf5eEqKE
z&A#q;JL6i-FYaY}vU|Pz&s(nrE7zB)8`KJyrW+i1>L~HLdEKq3*DeBF0oiE+VQV7S
zrn3BaePlmVc$YM<+>=mgZZVUIslvq{BWwQ4_-Hho7TG0pr(N~dnd$k9dkmH0uU36g
zJfCb5+P}hUoAMox?44J+UF)xY?zVQE^`MnAt;J@dW2WX=g&7`eCSBC<aF}pXE8}lS
zxc$}{np}Q<(^^!c3^oOG+ivMuv|PehDKtdI)pg-XFY&9JPHDWk@Kp4o{noqTOUrVv
z_}`weon1U!?y2j`rir&aX7sixKKq&2+xl)s@<iJ!UDA^UXGluU-+78LW5f0~skf?A
z6mPwqd-q;;=H7RD((jLb%Ztk4{upQTX@RQ!?Mk+-MYmV<@tW95Uj83u`|Vor`r4CE
z`lRnxTxvak^vC01_T8l~!r7~jylvgiS{*$n{a*R!)9dg3{V$h!oU`^vX1l@9L(coY
z?LBVy?{<F)i_=1eug8rTiV~UxR1Th2c#<YIr(><kk%b|tuT$j{1*Cr*>f)|!ac18w
zSrGM9(Ckfy(h?yCYd1qKx2qaKmNPH*yisIO<jhoZ(rT9WYjr-8s;3&z;1rs6q)+Ed
zi(K!k<3VXI{F6J9Iz%=GiM(6Fs*$3|RrP97zJ%+RPys)N*DKihGBw<`i*mKcDJsQ1
z30mA`v&yC5&YUG?%Pj*q9b7exu5#Qo5NO=FYQp9siFJ8TwJ-cR6@DP=$&m(A!9?Fn
z%2Q;fZ8TWvc`oEr#Jo3G|L-wo3|jFba?vdpg|$a@LzA>!)=askwYgR6f5b%X*e{!w
z>MjgE9sB6hw1qko_e+WRtmokg`=ZF-%qAx9Y{wI>_514KUy~&ne1nr27YP)cN>Jqa
z^fY{Fhl+{qr_~cHRoow)nVZhy(DXLvL(@B!i|1@MadH)&a54StrQ1<9`?d+Io8-!g
zZjXLUj=yo3C)VJg%bA2(Mh~9p{|`AAXXC`L*5z2ZxWh^1$OgWs4auUjE*!F&5ytK7
z;2<3Jc#8YyYur^El4K7>vRY>pnyD}3RW^HiB3kFT*s?nYeW!Mc#P2w?<p9sJ6-_^5
z5+`LzoVRT&nK04k)vX4F$X>Uy4S{iQor<L{mL#@sdU=;`uhYqOlR_(Inq+_2bSl(;
z-~YDN8+WeCO1f<I<$;>{`OQZ!>b1;ylls2u$jW)~t}c2nTKuP9>hvoUikAJ=Vl?HP
zqeSH^@yO3BmT!7$xiI`0vvL2Ko|?}y{c7Wm%CoV(Xyi0{%M$$ioKUQf(^{qGRV)pP
zb1Ij*it}8wG1NYPH;ltY?&ig=xD#FC7biIS74b}HNonhxxlm-gfRB{%)af1t7xn%|
zum<jvVAaiVTJ|qtrN*==;(IGIeUf~p+!mUuvCl;$SZJ4W^R*`{f99PNfBPX>o8ja^
zr(;}UZ7wOsxm!IF9Y0NGnf63YxF(F%xZ)sRo<XzvAx6yy$HdMaTCKb6`=YGgg6PYi
z!nSQw_4xm4)Aavp_nz*0)cUA+>5c@UN8<ao6|evDrQ7<n%Z#%en%WGPU(Gzf<m%B?
zD}5PwBa?4mSr)#|&9As>Ny^)qJf~LuGMZlKQ?K^vL6+*2&EI~ku@HVZv!G0Q)kIm}
zbo-4r6M3SxDyE*5%kG-fGNWy}VM-ED-z_iOcU!oon;l(da<s$$C6ndRL>6_cyIt;D
z>=!#vSO(93G{@dSJ@cr^*OkkSJ(`Sy+s+ppTa_Ox7qk7=)dxQ(C?@Kft}4&SVw`cS
zLO9cOjpIpyN3!d$9a#KzZEt&atbU8h>DQ&%DSzcSCoB@XP<<~iZKv%%z1=sP?z$Au
zJIq)*^?mF!=KG~r%I#9O@h86D{9k^@e%-vlMeKDmw;D0ryexPA-tmS@jtVVq+zUD#
z1sqrOD(u@o^C|1rg-Lu#>g?Xr=J0I&Y%ZBKVcQgArMuhITbUS&x!tcSEmWD<E4;9M
z$sHS&r~4RFw*GJ}dDwm2f8o(ism4i_!G6d0t&u4%D4czwzDv{e<;G_w*XM*6oU^o_
z#1*<C=;|sy%he3(n`W}4Jy90ildv)G`J~yd2A9=OHnQvQIB;;5Rm;l0uR=b1jicF9
z)f-*eG$$O4biFOu9{+PuSW?E@^=HI%gKHkeO`9=Y=UeKoFI)2y)ZVn5&<Q*+BeJB^
z^>gA?n?I)gdwdz6oR}T{xoaQmgB01u%i0%m?lSzJy4mloNX`2wv2%x=`=7pTs1#YN
zAoY|#h}(GmoeSM+J+^l)bSlKEWjD7apLzf2`RTo^7pznl#ZP(s%i-~a6&F;sg2Wd~
zcv+h&tq7hQ!hK|yYu&P65x%`ELX)M|^W|TudAX`=&;5vFljb|Fv)f;kcQz~GY}D#a
znqirb;`b!)o%(ORW88Xs-c#pu61dOZUOpqxzr#YBG3oqwkJ);HVN6~9NvdK${|d#o
zw5NuK{h#YzY~s!u^nkNgEIIS|8IK2BcJlmg5G(!ApP@Tp&(vuRPo%l!I-cjcO9zCS
zPZc;+AW)FNZT)mXc*}&Mj*b@-Pw-c{u6i(|Zv{v94vvTg7nf-KXFqngl`rxb|I$9)
zmY8QI4`n4!N#A8X_Lg<>m5WUpr%v3xv-qO;iWhB1o=!i%)T?8|srt7wq=jTJ%0{eH
zk-e&U_pHTS)wg$5uPVx?&6WT5u=(X_jmQ}*Jlb^U1ZkyuZ|}IVy~O*@!@2rPXQ<qp
zyXV5`<d?BMn(RG^Ni0%ldY+zUTO=*8&HaJs<qMX=GEqE7Q+iq@BqwfFnq+#@WkFxj
zgy8BKlaCbyJ-fId{7rk|mCM{(?&ppKpXXtJQ9ILSZd`%v*^3(vOPF}Rv%MuUiA(y$
zvA3>|Wld-K-{k2v;FWj1mC&$wc4ePMpWh3Ymd`s6rL5@tGWEchqkicV4lciXE^Oid
z%hfl|MFa*(Zi%`w@#>KW4_8+nSykDx`r}att|dit?yr(vvT9+}y0cR29hbOFy0wn;
zyo*r(`kPTMx1uC{X7pXTYWFboYfl8D(1Qe5=|3w@S9b9K*pl)qpxv7-;B>*Yy)Dvg
zUAKLoJrVIa@cWF<_R<FjY82GU#D4$quiG>G|BvHLye{<&9htVN+|Qy7$|kQV5S|{&
z!I|3;E+Es<643qOL0H{{eBGGN4DQwjNo|j|zC9BHG7e2TGqJjNVXwn=sX!Ouw1sE3
z9$#|j>Eh0X%P-7X-jXaoGx7Y~u1Kc2^_;O<%N8li&PaZisJb-1lR+i^%|q+Rw3J(k
zX;I5I*2pR({l9drby;&~+WHA<*RMWw`68b;=dQ@DLkxKl%7=osZ<*9_;nJPRBo#g}
zl}AGAQ5OupstMkjVf1%ywr0-*-L#9ImrZrmO=Bk)M%;VcDI7oLjAMwTiRq<NZ<6?L
zT@EoiF^%)UxeLLEKOR-PbJcy9fVR!!SqH?$A5EG6^WJODDRU0G1_~YvQax6yC7)~}
zpW-6=C9y3{Lg-Dwt!x2}cWXlb)K2{nalvZmjh$QD?SEY;Yq`HNbVZAuhGQOkB=3_p
zzqAcGDhg%un=UQ>mU}hp%>4BWkG1$TvP^M`eR@nJiDUEG6`N%rur5kToY-Be5XS7r
zexI{}H7u-(<z=OaW>tW~Kb!vv>|z479S=l09!!dJ|No{v=v({LRSSaMmS&ZzvmHF%
zG-HCpmd33ot{qH`?7ner`cDPHCdnDR5BVnrb;&4p?N}<x7`pV%(HR>A<K?6zPd$|r
zWJ|R2mST#P{4#yvHpL>h6aO71UA0Y9Yn4rnn^fwxJk(7nElh2C+Nx$nZuz2D@^!D$
ziyB2zdT%c6Xge$1ad487*&~N7r*fYOsjiDqJr%o?VY!+Xd+%K>fnTzQk&=aGC)lJe
zOY_AoVvF3rbJ4`iC~1qC+7^<QM;AT0`Y3;D&#?_>7(yit{5oc*tuo+ZH)ML;QE`)h
z+HzZ+#qMV~t@lJ$9~Gb1c&bOj?fC=s&%OW8CBNyYPn(t2!S?Jdo7=b7cCXGgGUcqT
z6lHs<yfkWs#<k1(ftR+fy4+IG(daR?EpEbh#U(`sPkzO|DbrbUv`J%qWXcDX74I9~
ztczW8d{)lynQyFo8N?f&WPLiu@GCSyRAGyw`+r3RZ$EdRS1-0cUE_01g5}-m334l0
z@A+4Kcydur)1OI^^OpNF-V0lN9cu1$pZ7b?82i>G;rtPUrCir$Pt)R<9>>G-?ab*2
zj#U+@oefXVKA2`$D_5{(rQ#-!fGtgG8=9o2o|w<H>@08E#XlDp@VQ)2)Y_;Oq;zuO
zQaL@9)U@)jnNJfZNH_ExGl|svlAf}I@o|>gv!G{}JE!_P{a;p8vhK>#`)B5aUSI0=
zvguN0)3fwtdN2Eyz2@US?r~6T3Rm_vCg}y29F9EOdg9sk8_)7yJ!3le>ix63`TH)X
zu%@qfy2e_2d^e|TzpB<+jr3wa{bOgMAN!?0Ir_-4=JJUrN*Oh5Q@Bn)6Ybeiq@A04
zX#1@*YFiH0ew@tRsIILM8uU{AbLgzkKhwXMKDR#C_BAs;S~cU3;FHC18u2~zt2-XC
zPIO2Uad~2vVZQL3vx|X=V*#tJi)KM6V|{#^+=lRq`Rwc|%_T3sZ=KcDmgCyBrm0KA
zm8pR1`K|D)@~_gKtetp}t>;zuJvZBk8)q(Q#9w_={6M2W&RyXBigRLy6({~jWC`g`
zl3F?OQ;P6=l`4UN3{TA|c?{_XeVFGxX!*cCH7+xFT|$=U(?`E<|Gy%<&Cey6Yfg7X
zKwC?|wz$aVD{3<gW#S7i%s!TT-q+>sz1$U7oHVaCC7ug<@G3&u?)?M48(Ai+o_oF8
zpt0_`ncicjz$boL7QQ~OEL<k;I4J%4gzSsksdt>T|8zNLY&z!PR^!5PWv2f2d7{}1
z)@gSi({5g+^F48)pF`!eS0dkzafF^$+IP401^e1dv1|Mmeb|?^HczSN=iQ#zu=~A9
z8cOe*Or{+$(N~fz+PF@sC8Ou)pSFp)$DK8foBJQRI)}$8X5QDvb!TMQ6Fx5b$)`MH
z<LBZ}|6NU2ow#ViTJh_CeU$Js+m6smDVwDDzemp<m=|wncGo{H^@vHvYn3lYbV_P0
zK3M%dWwrI7bIvT4ze4N`fv(C&G$tQBQ(+?6^Qk1j#7W?YM*U^y9JixIZ<-peHYMmB
ziwamXbq&w_H(bstC(GndWj*nGf6HoOW7UnTE4Jt-OikfWR!EGLPD-48ewx3J-|QcI
zymthOrK+wCb^cgs@zJm9c$hlR?uiRdd)!X*T<dR?P@1q-EZKjLB~OEifJA-n6V_y&
zMVpjo8?x6vogw$waGxRP#cNXB>|38qJKGcRqE%b&_U3XE_oS%L4-}2cjIy5DeG19y
zdFuD7rRP(VL7s>Ge-|_N<rX^+>HhIDyWe=~18aXo+YynIn*QswZ#2H#aqy5=S#v|4
zR=eJ^bK7z^Z~FhPO#ASsS8=iFpBAR8%~&x>E8FjU&Zl)sS>6WDkC-lH6-#gKyqUG$
zFRc8^hmVi44!LZedXl|_L+{HW#YchqW~MABR3x9a2A>E~e|l1>tZ35VxS%r`s~uJp
zRkQG(|4?ev9QS))#x<`-rsVieqq)CgwV2Aw*`nWAr@FcC_WA0|^J>Z=E8iVQEO;8X
zn4QVk@-%5?TlF=K{9F397Qbsgr_M29bv^UVGee`+fN`@$yPpo@b_?(RsFQO1-1;nB
z^>2E&f9Tu%!8iEHuc;r_3(P)!@4xt&Pa8JheNp^6ElIajP+siABJL3Bw?g4!{&AKf
z$vdYx8#bA*EeT+p;&mgfb3u}>@c)93V&#TY(=sLUU#5hs+Gw-0pInhM@vYkX2V5I3
z+nhDb-}};R<pmd+Z(TPIiEYq1^5fgXMQOS2-}bi(tDa9#b)OX4^v%5Y?U6I3&J2RO
zMMW&XIdZy-RQy?%ruX#EDbkqgIMJa;yI`AhOWE3nZD#s&YhNeweEycVzi8u=U-|E?
znhSe`);w)pICsmFJf%1f<+ukQ_%?6y{{A5;Y2C8pVkg#>dmQ{7@Ow+Rb(&<s!HnPO
zXT3{I&TMGguC?i$mX@~ZL?27nOOrGuH#-?UYyAKI^q(yP>prjfEoL<7Z5?wj!<O|M
zzbf6-E$Ol^IZ|>>)zQS+-=*1Zt&@JuwV;`-|Gm;nGhNpEs02o82AuGGed4hlhj>Ge
zw#^BR`R7u1l+|wkcKom#&wByRk{{_)8d*2@yLVTe$Y6Usaobyidym`pJT02U{yBM7
zse!ljlVf=~N6k{4FO}?Pnte&R-)-NAe&PN<AGch&Zt)?5$KURJ)}swBJx>^vEWOsL
z{JF6v$bM&VxbD`KPi{{4o}i>FqW*R&_kr8-J4H_Xz3uM9=zE-_ZR?+$b?n#Lw|O3C
zyxOk6?B}&jrE4p6Lv~dfq+OW4&u2#2g&fXA+230WulN+PlnEI9=SbxKrXjM(R-04O
z<G_K8cTE%aJ#YN=xR58Ou;<nDJ<IC<sBbs>e(T@~ha*hi)oa#Ng+Ak$bAF+_Mfm2j
zvJcI>EKb|0g<GXQ=Xg?QwcbWnxmY_lf1PS?-rCJ>FWZiN?LPMMm)UYY^VeoGzE}TO
ze*0wQg}kmi$HX`;z1tP5ur+HFcd@F`?mvsU4@G<|5BPoO@qd;V_W$zD9XK8+C3G`1
zv-9$WXiRwMq!+xSKv8+(qa#K!dTB>?1SY%trJhLP@#IR8oTPl3!(-;91nH^z(Xug`
z3q*WpYweuyAV}%CqY%3#hl27H5y>EqiE5#{rmQ>|$Zl?ETP2csC2ZTRqdB6%|5r>|
zU@zG8RnuY3!TISYwOp&Gd0$v)fBJ~3@f4qxW=l)96+T@yb@twMG1tDH4qt!YYzf!&
zSsDrXtBscaiSs$bxY#jVy#3bBJ)3<xTi6ZXnMR})Nb)WDKJDd&r9CT~m+F1e*=Y1G
z=VmO+gJXM3zQ1bTJ(0;><Ina>_jY-gc<!^>`TW_bh4%S+3A?_$xV~1u-)}GX=XZCk
z3(Ei5{w`w-I*}84f7RTzVQvq_r^?Siqox=A;r{jgt%s-m{`;N#!hS|poeTRlw>4yR
z@e0gIYGRdb`EZa^d5!}Em+l;gE>3-s6$?5Xcmy5(v2bKDGy3s#F>7gXUv%|TQfe07
zmA0l(Niy}niKqJwkEXN7k2n0;sJ0{~Yn5i-45?EKcJc`~h?u!qcJVG<mg1^uG|wu-
zYpL1F^Byg3iOs8Y-B&)<YF;C9KrHA(w(`2h6sIoLr9o0z=Pt$xo#M5=tI%t5B4%Ur
z(t}P~Pp4hEmGN}ij2$0d8#R8}#k$QXhcQR_*d30y(<bqWzEIk4#(Gje`jOIEUh@^+
zR~Oi^T3kKf6Loa0k-z0G6UE+?;L;uXm2anp-z?)2yOG$cJMBiy#JinVsdK%z-b|m{
z`z<nUsqKq{VG%c1iG^jYJU%V3Cn;4p$RjaTBy66_b`4hftm${l?_T9!#QeDR`n@XQ
z8jbsG=Vv!5X#Sb&z{$({e_qnzMt;2)3aye^I}$rsns(gnNdA(q+^z9yMOd%vD~ZJu
z?5+iRwshHOEK_=%<kK*TSF2Th!oDnR|AiULPHMy##%XyoE^^}XR<Aef&|;jIb=-$>
zlhA_{%^feg16yp2H8k42c{=qJLYvm;?}^II*|7hel+VU}F@;4NgdQ$g*>j9#QpZL!
zC+Dxn!c4kP>-)Q2?qs>b_e*=*l`dnAUFR%z`!2Z2^QhD4HruYrig$~&{kDJLSv1Gu
zpr*Ul=cQ)$Cq8|cc6+7uMQerFFSi5KWA|KL9o$#?DJGt8&&|~JvSrVUW;rzlT{-<q
ztNO-^qJ0k^-d3^yrz1D9xl*!+<G{8_|NZCveEB}x?huQ7$_K|cs%t(tv758hAL2F7
ziMAJ?&!45hCE+#E<$3HACbv!t!4AD&y0ZiqFS2_ictWmu%VOE5Yc!>PPMYL#k?RPD
zg70&y6BkZARcUdUpd~nE!G!}?Rof&rn_`V_&dgG55U}F3Uw7lQ)1xIq(|%-z)+(M2
zW!h@$F1jjc#Ujt856TX2;yog|E#Tq82WcWQMF(5TW^lW$dn92rBcPS(g3q#33s#ul
zJsVb=B${cVq&-jNY3jkQvyO3XQ*@;?vvz4IuU~d-iiuuoZ%j-Wr$vEI^4Y+?(v)c%
zSF!5aT50!Pd8sV3Fm;;BlJw{_qsh7vjjAP@Mn12&{?FU<O7vv4Z_uQvHQ`MLHx+!C
zdTQ9TZya#YG-y<PYB#^o!&$>}{qH*&$!nj?&^G)ual(n96sxG4j$USyH*ICnG;BP(
z=}Vi)#vQ3zEGB6|W!H=sDK4Gixzo#b%CV_oTO7HhGg>+2n^zkBNiov8*`37jl<md^
zUIX3T(vn=w+mtlbOv)IRr%5PDoiaGa=fpVW)`@3qlUTB*FSX=dW)Uo%s@hku)Ol{D
zg81$hx4ebCrk9@fN$xno>9^>>&O2KQ(%pOa8TQM{_Lszk%Uy^~Un04bPf}$=>(>p-
zMZd0K{g~jz_D7&&+3&=FeOa@DJ@Otd6YvV~Hw;l+$Tl_MiT10&|F$Cb;!`_Zw6rb;
zho%@V)pBw3o;5KbYsupN2}|mkSQboZ3UN@rd$J*3izBH^i9thA#n&rS+i9DL$=0x=
z-fTA&LijFu%&W{4j{d0VbZyFlz}n3nWxEdRwFWuM-C{C}@eo;QR5aByR7-ivt%m_3
ztHtGywRQ?A9v0R<>~c)yqmq^i^Xh`2C>2>gKgqisn^v)!8dsTY+tu|VxA$dK!qI12
z*S>j_|FXh;@}aW+qf6h)o!{KZvTf0X@XYp|XO4Q5bb2m(yGVR*P^RbGTiNk5=P)%f
zu*LanCGKJ2ep>l=sz9sbq$H8kdX0Kw_ruSLPgDIlgCV6;ecHoMrj?r(#)j<pe{|j-
z6%B=-CwnjKQn~Z%mF8lJW+VPj#yjHzn$DO;UGi&_caT+64e@!Nl%z4^^u5ix%WWUt
z3Ro&!warOg<C~yEpwF}&do#@<ltnmCED$~_ny2pkvhdNRZL1q=(ia_^sNt#`m7<~@
zSu|0)=%qujWrx<%o|)E*CpxGzG%wOs)jj8#&l@vy`oxE+a|2JlIJMxy!&#D%S<7VK
zc>Q`dQAolz`eUrOVCCi+Vn-`1WhZ{kJ9WA8@64*Ex)a~li7US4oqF-H^(qmSh7BIx
z440T1Cx$G3ZqsHll}X^C)8_@|p^`^e)NSnP@YD&K@^|XO)eL{K7}~cyxU%enms4=U
z)LBAHSN>n%ox$(3npNMUi|5+ZaQ{yW9RHk(5nS~$zGvf6!C0o7K|F<e3u15U25H|s
zp~t17*77Lb|IP|ChmWBWo}L>VlDE(L@YYv{@v}qjZ*8sSiT65JZ<pU`vDGNfvM~2Z
zdC;F;w*@Lk3lA)PfAIC}4+eUFR4;gSrpvv%X#Z(rB+JYLVt0}~o?Tlcwu|px$@x$b
zrqt<AZr^>UwtqLHU)G7q+UoBsT~{&$YB}x|-n{dO*Sb?(Z)6^R{(ItBMx{{nBTmIW
zWh>v9N0Zu5_Z;wd<zSkv@_J6{%Y$6os*ERJS}TxsRAKfE2hCS%6ARj0OcV}YNzE!c
zboq{iX|0xHR?6f}J6@apk6_!I{Bn`Sj2QxY;UURhw=PXfX=hSD;U;CYewohKHj6h?
z8uY#AJv!}m=D~Yq`}D6*J%vu6-k+j2H~&ZB=M!sgbl%%xqj<RK+T^L<t}uHiO?vR5
zFMR=n=8EMOYipY&ru=?a*uSDTZ%d%!+7?gl_#Z42R|Q>VzbfRLSa~-6Ux>2N{<e^-
zZd0xcdTyw%ecNn$ZF?*ym%CWqEZJuU&W%DW&%F-lDyDixJbHa-%BLxdOEk4=eU&!N
z^kvhu*>-o`gpf?%6PbI~8nG<h^lQncDS2TR-i9%(-!d)r<-QjM0*9u)%hf#7*ZN}Z
zqeo}&^*rA8<E^pC`tWBdYL|2$R-I!B6bSntvcV|q!sj<@zhreYf9v15sdeW%Cp)o|
zk6q5~stA}`bK%JjZ^vEU97nzP@GaZZ_aPxI@0p6jvlFZLHe7h3@#u=tGA{oQ4;+@M
zgd}O?tO#{`yno6f{U<Fv8qP<vj-@|I{_AwXL#iiz5>w-hmyHwN9@?>v*UWJzN9tR@
z46`3eCZ`VYaHPMS<a%5v>39xv&x5x;O+ELwX)0<kt2DUk6zIH<S)sb;Szy!SVgvWc
zn#I;E`o&huPfXJJFeB%roBFAk<wYJV6BoIq=`A`CW2m&@mD{uSt3hwYzN@?M@LF9l
zv0m@|m5?Ukg66Aki|f*kF<#D*h*dxP=uDYiR^+laZj}b%{}HFo?Q(5wNUNx5<Nl@N
zptNY&twoLkYb$wlFK#+6Q`^7dP<z4=^&c#1mss2wJNy}!97yUrUum=U=OICPWu_}p
zDlX5`ue^AEg(;{$Aa`}oqk54`^%6nX3XXcDba@J8th<+`spb>gvaCr<L%IF!x=Btm
z1r~d6Qn~4r(XvZPh3~?hH>dAbbZ>9ad8G7#F}<7l=!x)CC%QbGHN76^$T4q{VekLu
z8=+Njuj(nYa{)`{+0QWnYC#(`*I&?#obY)`kY+HeTIOq;9SzYtj<Vf#z9Hgq<4DTH
z4cc7#np4{X;!Rd^gy!wJAv9$f!!x#HoazTSXMb=!ap?(L%T$9j{e_<%{I}3oDLSTF
z!x_x<=Jc#Q(T4?xu1(OKbn=y2$Dyi|OD7#O@<?Q|V=KMqvBQu3uGj{}Yb!o5_%HH0
z`odvC+O58vE*H-j9#!+j2EPu8+&OFI^~y?1>y(qgBBAvkrJRoL%Tb!rrT8LAEH}l1
zU&wtM*XIewzqQ_f+LUEEyI8fS@bipdtGSc4l>Pc!x_LtM0>e&4Uz$+y#(DXX%~iYF
zol5dAo-vlKKYy_y|HqwTxv36v*?M(9nt#5z{4;FH&k62V99Z_hS+3M3>|SH|dlUbf
zeO(RTw71?lW;j);QS5}*8CNeA4_?K|e=CmuU35tK@dT|OUA*t|Z%%VzGGN=bkE2{=
zg4%!k2`1myYTkLkcjA=<|D!n}kMbTp$>DkQKWPiky91^cSrJCU+j=xVhHVVz>epOx
z$<N2wbLtZPnO`)YXJ3md&{{f$C0%X8HU}n;#ON)pqB#fUq7sFdWyc7;xa7w_=|-x~
zX`{&(ihTEL>uy)r!*$H@gf{1N0ln8-^jiI&2{l}*`k?o|_4%p^`g5nSa$bG2t$^7x
zZLZK2!-7RTlUp=;lK#4-8P0T?xyj|wqYcw<IcHj2_0-c7(#+C1lVq|;?RW~4ij;ef
z9+OT=+s6n~4W7k3@0wH<npGFE83k#ZE)op(Qq-7qj-S0Lr%1W<l(J>TQjhM%9t(3s
zJDH1|J{#|HsQrIS$wenu!HswI8c{c%Lkgk1SBnG{4}MiRx;47?qQY6zdYh$Ui-V&h
zb0w6Q^H?QHSSPAJ3RQhIRkDz&WQOYTS8K}utdgC>B7Jb_>m3^%oOtcIdL8#^uh_wE
zd$~|nvCK&&H_4Pm?y<1l(?GlF{4Pt{<=;<s{bVTrcd>$J@0B;lWBhw#g3qoND{2s1
zvqo*I--T0N1-%(hveK)3`Qp!eiWnbyaXwstIi~E`PM=MxGt$)Bi`;t8xs`4S+g;#y
zhedJMgfyO~v$HZ%i!R+;!NK63teo@ZCEtcnzoi08pG7?K(C9UKz|^OaeW{m2&u>kz
zh8DBu$}QRKrph|&J-?de{SPhhN<Vf_>6}6cU-YciVz->_q8MY2ZP|6<ijVHg<%hH<
ztC;V(wrO(T1>Lt-{GJ9es`VCs4w3tQ)L)r%pW6D8?_w{ed{Pc;oqq4z3kIJBXAF9q
zChH0H1XV8ORm^ez%%vE0s$l0i&!AN-C)TJh$~kZ;$za)6odf3#4{7Q#_!xY8X>?&_
z&M`ehxn}(r=Zuw-!$O+h?|N1hu4WRZYRbc9ZFbqp>tb2|oA)e_<QMAy^19~MBJ92<
z-?IE;ouR(_tjiWZ!#__mmRK&_@PB*k>SD|1TN_piMWt|ghp)2Tq!jx?*(TNBrqRo`
zn%`L|jMMZ+(tYDgM&FX`Es{eyFWZRO{}*+!pYOi%nY$SKqPkW6^_#uyqOM&%#I-%F
z+ppt@<Lv{D%a)|;ZgRZK<#UgJ=8ThG_xineDz)5MEIPyO%##$a+1phAv;?><)%W2E
z^->NrbIs(su<^{>{}m^KMZTo+sHz|C%lvtW_gJnYJF8%2_l~M5<~b3X6SjJ!FDjIH
zbSI9<^Jhi>%ujj#;uGeZc>NaB>_0cbe-qp6Y~LQI#-2lb<*WRqug3-H73uEueZD1O
zib3(~DV&a+{<8y^COI%#lur?Tu-j$&q-6p7))-94zY&le(luErtMa68%XiUQwL?;D
zzprc$uS(-7k?fe;lyT#5`xWO5y>m*_RP?xfvX5kH{J+F`blJ)LW9#(poSYT8COcy%
z>-R&YO3fw=foEMF=S^W|w#+vZbB}OiFFZHh!sWTSlVOPDsTp;XPR`3wULxwUCA7|~
zOkrcGSo1`&d_l$ir7@+23vvy`lQru(HZ5$Nc$MQ)Y<Hn*9cP^W`-Y8dad%B^3?4ge
zYp}k4*!wb9#w4$%ho_gVV3*aeSfLc=cXoABXa0{T7n=>mT5^9ivmUtYqLQ++f$PmB
zyWfE+KPBtfH?3fuRQJ++;qj@d(x0~7Vo0fCos~XKseNWg{fzW-F`sEJ%?;cBY;*9O
zbfVp7Qs;iF-ra1WLPxvSovggeta7U=m5J40<J;gekJNn@s>^cU|BpFnc<Mob-XWfV
zqjzR;JPPs-+oz$n``dP|jayHA^A6jhp;_wZx6&`8E$^?GKg-Rj#<xBHifO*Srohl9
zBG6ed`Gt3WF1x$l)WAN!aJ~eC?UT*TpWU0NXKOy?^Oa`L$|=GQj7$zpMS5JiZl6qM
zGp#gM75X9(xGZH#Ud7pftO<vrE-#&avtZ_yUxwA&*6(r6WSDkSq9r_5=e-nnaGR28
zKo3t~^SXr`;nq3E0XnJY^vwRKWc}!S-zFaM<!Xeru}NUZ$1d~!U^kDYvo*b**K*3Z
zWvN8}7H|71Zpj@{FIrR2d?n`cRK=JbI|P3y8&t(T7d12av&eSOr>4&~i!{IfzjN4g
zk>9s{feuw(-}NMZ-1xl0&T`qq$;+I7EX&<<tk^0(Xpf!zp_bY|a-1?Nzq+4DjQAb*
z)}deXWzN%=lOC<UxH4wm6NNKNH7~TLZFOn7wMuDi%#F4g%^N<ZyGpnCusJS!>-%Pw
zf%750AG_AtpYA$qrJARh#qC^kmUpYsp$(#k7&BY9Dj5h$&Q`m-;?c4St<n$^p7#s0
zg6GY@`aY<eOXSeS_(M@sw`}@5W0QIH4*k8#3$_F-{WKx-?7EOF?W}@M!SYz8pX_IY
z9QS2&8_k>?TdS8|y-hV?yXEw2>9LI8O176<GkDD0UKndvZ@oM0W(jNPX6wBk%T}gN
z*`4aQ)!@JAbDo)d3PqBaEVMnayKh$`TY9P4`dbGYw=*f;`;yzm^yb35*@<gH=S+87
zx#F15;t;X+g03F!Q`ZYxyf4oE-T5?9O>ODvHGD_^Jz_ZfJ9`d~sQmL2CKpdkQaY)@
zHQ(__?c?;;?=w^W&y?uhr63rUDqgkF@y->Ce=Dw<#&R0PYF2%IvX$H5&!>dNt3&QB
z+MM>egU@hT=F&|Y?LIj-1^>BpVzSN7^hD|2q`MboJN4!AYhGmAw76T^_0N+NU6lO&
zqn+}S<E>S#jGN}$&bv}@F}iZ1V(hmW4rx6iXN;ZhT$9_f>iVl~7xdF#IyviZ`g81n
zw@Y<!x0&Oos5ugcMV+$e{GaaHYR8rrvRZZO(f2=d7l?gemMqNbO)<FhfR|_Q3%=~V
zflCj6;&i(!_F#(A0}lQRkC!Z;9(SpE@@H*pEq8&-uJ)HFE}5`AsI=R+@mll5i>r3k
zZCDocclo~8Mb?qUseiXU-aMr~_ImmL@1?h|SK4wE+;s}7EJ;=gaJSE2czH*Dr*<Ca
z{|QcW6h5Awv#NK&Z54?tx5~e3R;i_JmeMq9<lR$i6y3;ufjRuRc{oc}#FWMFp2W`W
zG&Rn>{GQK$UZe8yAImg(e$QSfF3jlEz@V4QQuHw3RNrbN7V&9CkFypW=oHb8+S6h9
z*hP=ovfw9&(UKDrRoIIjooJoxV5GHkgUtUEMylRMdX^h?l8vAH%(cnnvgiy-@}4gf
zT6E~7z{`t4saBe`RbN(IUg<H@syFIbR`8PW@JXj`Eeq8yU+Xc^aCfWstu4woS3NaK
zJw1iDc)8Dp?(gg3b_c0xO#EPO@ZeyhsD@$Bj}4EGbWGML;Q6`n$%%<meI#~->9%VY
zm{-sHbaQj`B<195OLi7N%Uly5ov!<Ns-|Y|7FMGv8Vx+TH*!5T3SG#$l4B-ZpPiz5
z=h2(a)6>_>xTZ~Jcq!)YyTxu*PKnVq&zq}Ky&ZHLleT*(M7TLfcI53a(0OS0ue$le
z>*6^P^7|Wverr#Q<WH#D7ORrrn!DrCvHFxH$LCA)tup^F=)(NB++9=TuX;l(?@mXy
z>eoAqjyJQWT{-Ha8db80L35eM#afZlgRbIpcukZgGoO|vOIki-TbTFqW!mhTj~Ay*
zncymynKH>$=;fqIeoCB6t6zD}yfWowXysEE*1XJ%VoVb*&DLGZvEk^9g1tv0qcZ2M
zd^WSN@SB==kWb=tp^{&%%Tueo7Bs{+{Ji-*p37rG15=w=R0ac=%B~emU3RZBmP{~O
z#o|@4tMld3S@Sf7oPFx7o=jep^ipoZlbwM9+N_}_X)Uj&GRkwZb2>1s6}YzQcI4i*
zB@<U}{TnsARJicF>PDuCPIDNpd0y|*ecN$0D=%om#{d_mEsk#NF;g!5cdfmf7o8w3
z%P~c14f8zihh8c*FWj{AI35T(%Pg^8TJ?6@$M~&pTv=A{{xC_~!DVU17Qy<DS&_M6
zzc(#k=x{4cMQ`WhRTdi=gg0j-9n(F<J;`0Zk|)GVq1ETp37Ml)ZU;EbJhsXA9b>ET
zM1$Qum))2B{UUfyzdq#iIsNddQ-WWW1|?O+dX#<%^Dbola>-kD?Y+P>*`;xs>!uuH
zjmocm5_VmFx##ql>fX82Q-cJKMI?5wy&G|RimJeoWKNA)NA46kPje6wI@3BcW>(`_
zNzvJV|9;F+s^3!d$n5+X<FjcyzfZlq{wQ0H+xwj_?r(^j6eL!-f1_fK>%w}?3opda
zw*BArVrP}7Y;MB2ZLTHpR?>00$8H|#WQo4BZ9(1C|8*1F!}V@FtP<n;u&%0n$LlWH
zmphBRHCZ)H6rYI}h8)xVA$26c<^87g+f(!IG%D`;A7!%M;br)if4_d*71!&Xk*0R>
zq|nn_$^WAJPSsBdG<v!J-&cO8@I^vbCpbAg_}gN+_LNVW8LOlD$<(HrnNIO<mMpjV
zm}$@~B$B|R%6MGFt7Vdw;u61%Lqckee&0J6a2uy?R@3!tW8!%*k>kd?P7xJvF-y69
zrySTKE0=_A72;S|@kRB<1iwa~Gbdiugi62MayW8rMcC>S3ax7dCvtDvb<b^IhPG1M
z#W3MT*Bv|@+L-?v`t));O<2Y9NT7qk(E0zw#Z&GCaQ(31Nf7svbN_Lbu}pCV5C7w)
zZ3Zc}4?@=zsdn`r_h3Hh(R4(m)8qtG_oU577A{ja_4GXNWHe!puaSnaH&@t<Cebx6
zx|wqlR4)gu+_F62sn&YS1zu5$XRm#EDVwjQ`^G1xGaF+Ca&xrPZrw4H+{wzAcZP?-
zczWdaQ+HCC&66%aaA-c8&F42sabf34zs*uh->{fHnR52YtQGTEO;0MZ8inRpeU^U0
z!BX;BL`>V`q0Hle6p{JDVufkVb8hx@T5rmh&9!2eeRE||<RiiJGdvdVmd|<_{zoz3
zlhPLFlB*jl?KU0$c*JDFh6wKej&~k=Xy#3n(`#5?!?bdR<hul}jy(r$U9B%w`Au2M
zv`9EY@6}32J0oSYxtzx8y94zUTWsZ)mP9Q46cYTC{lN0D^;KqT7EWB#5*qZhWYM$7
zT5dT_$|jv(TW8IBFz1y?E6=K{`U*N%7TnQdbT#Y>)L656@tbF7+#g;IZ1{LvgvB9D
zVjZW7pRNw?>aK`2K}VuQ6W6S|ba;wp=|{U|mwbQae6s3R$zCpgQ)ffR(>ckN(`?Q$
z@wwhoVCCD9oe(@r{Cz`&%S%SX-gOzx)pxFH<%BMY`qOxRkBHI&$7SsO+*RRWEJ<z8
z_nzkA5DJ%k_n@n5pGUN+hI7$m@7Rc3@0i>F_uh~*eP+-(_rz{t#rNXjDihu6)+E1?
zJ*gZhv}wW=9|f(2`aUA<p7X!1N$vifrtx{T@!YRGn(q=%ola;GW)arSvOlS*mCgB(
z-TQ~Ba2Tgn_fh33GA+6?5#H@rox}z9cjl;=-JajmnRHS^F~2-G?cTF*N$Gl*XO=ED
zG?rCn-Fczar}?&1fBhLry@1aJF30rizginBx%SOrUB_kBU9vE)QnPoCaKm%0MgHfe
zy|cPLajsjGdDp!ON@a&u9Tky2HAhC9C7x4CVAZXu3mz(Yc!>t}yI<X`#ds%af{<6{
z)ELGk9XGCtUi0Gr`KCv~e9uF#Zn+N03E#CC@3PxXpMCy+>$X(Sgq`_HZy$egwVG?$
z9N@UYiMJ@gq~_IcMMLv2R~_EK$*+QUS6y4{r@*8g6dx0>a{YRCaKb~EgVtv~{WH%B
z&tp2OesY09URmz9*w0+a8PC7oEdQvT;Q9BE+QF*ZxAMNOxU^M^yZ6Kvu}Sw<Cz)#Z
z-eE5dxMC#iyL^sn+>*$*5gA(oPPc|>q{P1rR1Ow6GNpaWA$5*db6dic76jS+vj4Vn
zUE%xBZGRd5ByuxeFI~JxKB6@HvDPE8?N<9f_AeGs_K8WUoPFb7^O8BUb<W>(P14=;
z<Jgf@UG=l|h07+-Pkee{l}|xP45xQgR;w1{@(mZe<qE9iR=fV^?QDPe-$i`a1dAE9
zD<{vtQg%jAO*sEz%}1k!oR8O=f1YW`=ODDK)3^D6=j?2GozowsKVNymqUBNOn>}yM
zxw7yhZo5_$O*1~!cERZRdy&qtX~MhaUaQnp{*iK6tz#?a#*R#z$ATR(9CP=CEfBFd
zH~EAE^R;<jZZv6T?zz^U^Lios)DVWWj?G>>m)!Uf;=haMR#;6{_se_l)W3^2^5+JH
zoD^>i{~@;V-ipSqw+prGS2&uk3M_0EIvmCy;=wf4Job>-qC0Ni?k)SuyH%?C#%}BI
z{ol0bZcA9sEmggC8t0USK`-89o|a#Kaozs7<!f2?A98(rGCufQ{NIqkr?ugxN+%CT
zuqpi)n8@nl{6&ANt6<tjnX2b``<rExH~25#thMm6&Q}JRsp<kj8yL@C;$lgzVw@}@
zc8QD2SWQeRh9gOr^AO{yRIXPOV;={_eqC62=Mtl|d#vJhk4vm^k5vt26!la$DOIP$
zi4_+;d}{DL(9pA^<tb~8bf86iaI17#>*r5~=H_0Do)qnAlXjZu`S+@4hqI8XvXE7=
z1@kvik82hj*Sy%v5>~5-h8rdNEff`XODr*w7Jt^xZBj5-#LHI9tfEl8GO$#5LPzx?
z^|p_W-47+&9nH&vlMT(vrgn5DvM0}6p;7VC+=k6U$<5-EYg>D9$}+=*zR4oqYKcBu
zoHsoW-IMfRIsTG#n>s_wb1Q3B*)`5;;tMMp-MKD&tVlf+ASWRgt-$&wFkyRu={5y}
z1?ACY5A=>57T}zukk%lyFI{z;x%}M|y)(=NL{qD`D#kv2Vg2`E=8F}5l8jl;ANRez
z(HD2M{&R3v{zdPv&He7I!i*fDciFNHTl(*sH86WNu$8E?r?^Q?%;vTfOxPi)=i(@B
zpe`PglP#(t+>j%CMdM?-)5UK&-`FR8f3BgkMDve$gu=nx4Q>-UStD;Z7cMgmba)aS
z<D&n1vi8iaS;8DPHyj0%6RcJ?3rc6CB^#;bZeX3R*t=;_50|i@aRi%OVC<{H=C+jT
zsx9*O7rEb_Dz6a!KhSN4p~~{f;)z8*5rWApTf`ElZC$AQdTOi8GgDJ`PYJ>7ugud+
zJ13bOmRfejLO4l;)2+Q^yNSnlsfrz<O_?*?x3@o4%P)DU79{S|u|mA3qVzaR=OjUM
zW3#fP_L;L+b|zdeE&VK>|H$0#TlkGDlDW&h3p(2j#f|lgh20kxKT)YTkQ5!N)HrRq
z*wz>2`#(u<z8NX(=)a~(w$ojDVY5!Zx!|PhTvN;iPe_O@WY3Het-TysJ#||3^_A5(
zSN7fNtUmmFt|(*6k>{>9p_X<>-6a$i_Ph0GAF4@GwRaH>lMK_pv%U7JxV(*X#LrCQ
zxTf06*K6-?@8w{rixr)w8}vVHPGMMGqT}o13*H_NYjbis>sH^z#9$D%@Lb74xtBhi
z0#4mYj_DV<W-q8ONSbs~eFDRdN&ZdgRxWZ;MtMB#LbKB%r8inOidXDO65PHs`0cUD
z31K?t7}Z1#+14a*e>G5(lUO9=U~^?@kyXW1i;U(FPs7_&<3t4K>AJSWX2uCRPs@so
z^UIhQ_`<-+VyRT(v~(*4yQQ&V6#|aieC69(3s<#n-qgBSB>uzK;DsrkOG3KJv)WV~
z4VL^^&N8j5H+e;#M*I3eQ-{qfCK)HXiq8;S=CaArU93&mGbDbRQE9UB%&w0eQ*N4R
zyO}d8BroAIUna$}V$~|MW2HhHI1?5Bi(TBv8ZD5t<ySaUz-q;7%O<3x?5$eS`fApM
zV&Q#}(gD*ZOck|`{@}MgBlW<`H5Vs!e+&~`oT{^b6K7+)%n_#^h1Rs43zuwXU&rGl
zKY5${%V4HwS?hjGv|}w=cc5tL(_fh%yX@XYS-yYO_v%*>bCK`o74xQFQk!|uR3p&W
zw|%~G%6zSt{fk5FIc6`~JS{ZD$swdzPsm%4Z>NJ;;zqI8*-4Y@xMn$u&z#txBq<=J
zdFI;0OVb!6yB2Dm-lUan9u&Y8x=glkljbL3$BmOW@J2gJ1_{_k=h@C~$W@3oQPg%-
zSwCaz42!4*`<e1Z1)~<HHf{~z|8{MWz*md(D=MzP|LbS4v^W|Hx}J7T<ldGb-R2#!
zAT!ePeyH>{<)ufzyDtCcA*;6CKfIM;_AJ$dg0GviGIwmRIX$E7W>LUHU-gSZ@`Wb6
zY_q4|DqikVUfwaIopWc&k{Odvw@VgnWZJ#5c43|7v7M6_b{d(PGfr4l>AGrB_o}%w
z&7%)kuDQUq#+xf^Ve-w8SqmSQHE9WHE3de2Ufg0;UYWU3>SLGP#w-URYZ1p(my>Hm
z6#XNfZ3){ZbDDeYkCN_7t7Q+pNZlJDdps!h$Kp9>GzC_;s;qeya=1zNvD*Sp6UC#+
zF%pe&Gu!;8vh4r4DsRg~kBFq~HQUzz<#Nt)Qe%qlKf!+BJEIp*<A2+qCsnRC&v%(r
z%gr;vf9Yl&PgMc!I$@b^z7<=ZdMcMEnepzbYx}fdVv<G?YrQ;Yu4L9Bv8;u=g2l9X
zWRl%B&q&(r^tJv<dBlHC)%8lY_R<9p)wXOCba|~JIJ<4JSd-4)LoQZQI;WBv&leta
zJf$6&z2t5dcbK%;1}@!QOmbfp)GFD81w3@$Z<(`pgY4I9I^PP?(<57Av$kb)O^f%M
zmQ*!O>~V`vx~I3QLdO<+xrN6|4b9|^iS63uyY*La!LMa`Gg}+FPGqiFo@^ns`lN5h
zj`*JD6$hlexhfp0eyo@jRkE?LWK#6X`ES~%S0$=5&J0s$Z95n?H`=E_*}QAh|Cy_+
zRxQ;!t-p*VWr5b}+iYvJ7v&2jl`=+b<yw9}rORPq+diw4t9FQdTIlD%vc-1ynH%b1
zCvSDf30cn(Ub4f&f6mdP{68A@?5Ge;42WMOJ55|nVe1^;t6V_^3Q32~B^|On`{~@`
z=h{jq&VA8oN>^0VuCdv#bmYB<^oJRJM_+}0vsRE**e$HUD6#TFzNn_mEp>g7!@pjy
z|IO{psO=Rq!}x2bkmk${^}8=#$vnK3t<KVQBd?|cx25~JgA=|Ty(Iqn(oQyYnKuiJ
z|JJo9YN(x_sIhyK=&8$Jk6#W>IP0Rhc^{jz;9gUizXuizl@>otHhv?NQXn?VarHJY
z-%#7Dn^*ncwR_TJ?`X@bU4jmki$xW;K4#QCuEaR!g7lG#Tv2nRm42vhP2hicgLTs-
zu5St4A48U8tv#9+m>+zbZ|_6h7~B2TT7vdY4*wZl{I<r;UMQZvS)${T#)MjlDDlp#
zZ2EOR7QVUtvw1hjFS}VNCG?(k#{!dE6Isfa-?dow#A{=wcZqsQKlkkkvu7N4-TBn*
z_B8J!SsXKaSDac9Wj?)g*W&D5n__k?*zFU2fGz1D*Ume;5*1hPxl{Hkc=x0yC+?SY
z?Ts$CczA((ZHM>W(*=qX1DEf7$lGKoCLW}4&S>_Yv#SNq7Fs>oD3f|nX0zj*=xKY8
zf2w3Xxa&?;x64e|fL@ozi2vulOp}g&B%35yDU%_#<^lKH1{v)?(k)?Ai#iYHop!fu
zRg6}>D`UVYZNPZvZ(Yl#iQP(%WV$q3HaV)FTgY@F+(q}|AFqpVIE9Sl4*aT|Z=M%n
zbNIl8p2to~4T`zW{Hq^+F*@jA<nEQN^_Z>As!Ht*y9U2h_<Qw3mbMG+dJkF8HCNcR
z%gHuCKW6in6OTE5KjvQ}*IOhp@rOje24kPEME1L9>CDgb^%$?sIV+xX<iMuL#ZwnY
zZ_{Dgc;whm)!C_9$+~iH9pu&;Fz#l)y7?mCydbWiXxHqwY)c+QT~Fc;_R-yzz|V7$
zt&CUgor2oB1a7&f(L&D0(^mGz94+XW`(GqAK)dzujhaTYHlv%90yVbwy-r*Ax=&G{
zRY|}zL9kP=Yul`wJEgjg^u5V{wX98Nx%T(v+pF6y-aB#WpXU$Plaubun%<S-tGu!=
z%J2TZlbdsIKkmCdZ_aJ+?K1KgZ>Y1anp!DQ_*h~x$F9vbyEf*$pP|8c;38MffnBH1
z?%FQ1dn5O~de>Pl({^9WJF{X&%7ox@P6Y<FzDEN2k5u?>3iUs#GS;Xv)-Bn0Z`Rk@
zXA%WhGEKhTB-mlzb9Cmtt6t~t&N`P_D8`h)zxIO4(;k19Ray@`xV8t#u^;OB5feRU
z!5r>Dm0jW&ST3=KHwDK(krY0_$az=YKmO4~*_(p?Uxobte{4|__&M*Z@WQVmjBi9f
zK1$?&Y#x2l?0W0MMY~lFZtygE=I}4~TlIwKRuOKcjamz(%l12z-=9#OA76F<62B#@
z!tW<$xvy1QE`KP0s=NA!EiVI?0c#AO%o^#Rie8RQN9$$Z9&mCBHPc~;U(c}r-_K}u
zbIuP*^CN#P)n`2+6eYIiY}hfbBMYB@KU?9sJ1f-1wawi(AoluXJ?)wAxRMlKaXu=%
ze#wUQp5EGh4Y!@xzA0+&OSk7PQtkBB-h3&sZ)L}k67!|+J1(mHn(|R%?*G3B^!^DO
z{u5is@}ys|+W12d@9xYZ-(z+kuJ+HGb@yieNne)eTm3p`GA#V`m<3WS{&VUWPw+n2
z#B@^TNy~<Z&Mk8VrfE4}OmgiJH%)u8BCz3TzoK`iishvg55^Rhf|*8+T`a0xN#~|S
zPF6ds!Zk^S<Iv^w^9(l3EWAc04uK0eJJcpPEen+nTf4}w_0}rUnZD7>a%LQwq}{HW
z5MHDheeF%8U;4>qPd9DQx!sb>?zBYvz`MJ<xjwzST5Z09fwi7l-9y2_OFO6|@?q+p
z8|#vfm3eyh-Qjw_;`nsW-a}nq-%UHwBRo0iWZLA0ux(LSg#xs-vNWSlY$+^VedUl5
zJL`(DTGMh##sjA({H(oD@$rs!z3_G37}d@V!WKRY_BkGGdCjGg;lDtkvE|+U-P$)N
z{b%{}lO@Gt;di;HhL-QGlHXH#?(5b5ddFJw^!%mzdWQeP1`bT@CI$y||9EfE*88Td
zeQ?{UqSF?eSQ-iraTY6c@8mGGdFRaM%u?tgIFWmurAWK5-_i#WJ}>pAT5$5YF{ZGb
zTPG2}h1)_>ee%V1@+W5`ajB{WBw4=^xuB=6%KSlZgZkVlLZ=oi3DII?Sa-%~h3-a`
zobHP{lfBLy3D~W*jp?WK>qDEu4y|>z@K%|4L~`;4e}<p$P8n~p<z^P%X#KG$FhJ^k
zit-j`$-_BQ4E;898(&foZJ6RxDzkKIS;Xq4vb%L|J9HUE6+U+1+o5sricyelX{d~6
z#>bEc9u^);|8rOhG&Jk(n6*IlZBmfYvX2~-6V#K04fMH^l@-DwlecE1bE_zIICW_#
zNgNf^ED<{3ci_#FvfIDkoZ4(%`0<qgoqfmNdUNR&z0Km@aO<hpJ<V+e)8p3PoBKVk
zLb7LicYUF%_Jd6<Y!z#^3&nXSZk00Y*40x;^H$iTk(S}SgQsh?LQmo48B^|SSamFV
zppliKJVDcIXU|mMRWBA#cbv3SWrlxI<)(>MInz~V<b1MRHmBj5=gNh1eO75KY7^3W
zxuA?IOW171EUm27tCU=|*FL-ZDmF&2Z--N-Hea;f#?$+LC|r^V>P+3e@2Ix=fk%8g
z0X8359GY|mHopltA-1_E@BabT2hksPYo#VnIm>dRh3A4&@t1WMPdLaJhgD?ru8(nV
z*?Mqym-fQq-M3zeui;cVz_<JD9_IHaKSa3aN9e^VP2&mM_Ct?Tb5aKbr{)rgWRYsO
z&II9=Sr=1!qE0dM-#IXwKgP3~e~R7Nm#dfmt74Sx3;4a+prv>r&+{q8H6Ks^&vkn!
zrL;~_A%2d|ahcog`;UpI&beRr>-PR`RpsFCre0dC1zyZ5Qq%XO9MCD%<XpmWIr~f7
zETgSW!V$aY%~txN_9^v3&90x*Zhr`xVavKsb8P~1rrKj0kw-6Fo4d{|diTj<&zuPX
zOgloXb1w3&jyS3poxzpa@%iwRJFmhRZqNGfolxYmK=O2i^J&hFC7)d8*lg^)T{KPC
zY$Ah7#zMJ&2YS>Z4)TTdIC0Hh!7B0kAeZlyiK1UKj&5DYVg6U6v65+ma@;GA<9lyB
z?fVfZH1ESU<7(9talckwzqaGh)HR|)6Wi`Y?S6Ds-<>Nc;X{*vLisUUpR}OKSu-Ql
z4ZlhEUOcKOSL~tb%DHJ;smoO9#MCnxX`90*r5G)V^z_dAWIX@r67}7Cf`WQJonHF0
zC8B=TsTp0KQ<vu?hPUfH*{`+bq;Hjw5|5$kgD*xiPP9xgjNTD4Q=xcy{=KKOja=rW
z$P_!BkU6Kf?BkiaDH4{4I2RPH&ODo~(_+qTS)?tz;oQqz{}0Pv-KuBvVxxFfi|K{N
zY1($TCpC!VB=$#zWQ+W>NEV6N%xcv4sQXIMS&dhL+`U{0JZ}|_zWtCi|7Xjyn1@N3
zcRm$aUEY}Ocx~~@SgsK7SxNd1DTx#6U9NB7cGQ?}zce7QxcMM&R=`RHtBD0sTNX?3
zvaf2Lw2EtOl>M4d8E$5a;{Oy(S)gpn*}yEKFqLbC>K3U9-W8XlXDCTbJ+fhuSh(}#
zFrF<!7bLz`AFC7&nx3`j*OOHB(%FkSt_C}a?K<w-tvjVoNZQ}~qJfDK4{vwJ^p#f&
zUg;%UOxrYL>F%V~u+Gy$Yu~vg+GJ)ratCG1cIs5s`^*!@H8YD{YJY^=f89RCs5wdP
zscE;nOtvNSEV;;%G%w@04bw5ZCroE{ZcCnVNmn>7M!`|?RzR0o$10JA9=ZMBwlZ>a
zGE988>AKQ>Z;#qX+><7#JUXC|vLR@t>b~@u>my&Cw(YRxNltq_<JO;Ufh~<e2d*qh
zb9M;Z)Ty##>PnZ$voboHXY9O`s(w&1ZpS6#Sv~HIjv7%Hn>x4fYwK!SOD2X!IBrdf
zQq@}9ad%zWRFBHgO^&~ws@yp9ZR_T*+ZAFHPW071aolGRzVuUI((QjC%gjy$pPb0K
zd?A<kES25q#&a`cDwg(0oHQ&5nwH!<DOdb<;@7_R=O;bO%Zhe$s5nGLUvR2irn)og
z%Kv1Z_DJ8iK^baBDP6M36IQ!Meza!HP|9DokyUfc1J|GpNn1A>@^gGG@txkLZLPO)
zbxz%lHlrPirt3B|Yv(-dvaYywd9B7_pWR89=PZj_zG~S@h0~XUH=l^meQ^68>*T7}
z`)BdjzDT~`*AQ}b_n#H34<Eb#TkWY`%6b1b`$zL5PX5Yaj9cjy=cN!edBHrxonISo
z*L4cDrG&GD_7rciTeY@$?!kQ(ul4Q~m2jy#*mya9J7;yte<{OrCr$Tf0n>7uc?)Ma
zUn!V&KK_rAWL30oitED*Q2{&L+z)=Zeaez+jX>U0iKA{_nFk}zO=i%HaAvFCqQvEQ
zFnYzU1D%Pv2jc$Uo#uIdAG3~XF1sD?)V!4*4l-}1IbF^xU_EdBx7hD{%nKX!T^DXU
zh=r@$PH0X^Q<{Cw<MTdm$EkHH5e23vz0+c^U3jwVo0{6f^|O5yi!@Uk8RrNDE}NUm
zb?n3`i8~Y7-ESywSW>Oyf5Lttr|)#>cT2ZaYbFWCcSZzttPb(XIb#@^YU^0MQEBPD
zQs1<m<JN5+1#+Lvie0Pns!lD7op#;XfpyhM?=+tmu9r1WEs}HZKJ4?r&G~Tc`?h&&
zHu~tA2{!6(XcCRNbLhxT2~pi1ktJ;NS7&c|@S%!ByeD$aT9Jr}5<=5<E}gmm{8mHb
zB25(quBV3AHr!C2cRkjfYxk%BFY}Goeh#~lczL@(>2mwE|K=~BfB94U|3B8b;^q??
z3#~6}?)cA~_dnW{(P!(-S(@{zCUmV-;rKXV;im~jlZE#mGUYUx_}NlEe)g*6yH?5U
z6kJmzxc1U2UYnhKXXgJ^n6#<MOxyDybJRiI%I(`unMt&0AJO2y`f(eR;;Iif1uQNe
zEKb(qdopd|LE*fK{0h5;4`nlZEmS?aN=I$aD#=CLFBB-cPn@$mV{dXXkC-H@*#^bc
z2NbV(bS6$RzN^SuFR;@17E825tA+$?yn*um%yyg2wkt0%nXHkun6oso(d69j#a3CC
zyuZ~|IbGgXa5Bzlc)4rm!xQV?Xjv_IZ8Bfj%KbmjF83N2#s^z=ONjp2eZ((u&WoEY
z4pQqsYpt(RT)*~Wmw2|yd#CjwKI^|_xi%H9Z`7P2HQQCjnS0~sC9R6H|8uDZypq`R
zQ)SD?8Cw}8nY}kmj<lYdxqDmpZq8)4^f@zU|K!;8V~YAoPW3{m`3bWGF1zLAbcq@p
zig@pNqtu*}sl+O=f7*jtERwUMrp(xwaJ=W>jFa8RAG~PE*`-z#x$@lV{=*Wz1zkqR
z4CbguFR$UAbMht6u0|0ZZS}7YHl4m9+xofHw}VL~LRr^?X{Cgwkpolzrk=9``!26)
z*S)|}xrt}0gr3<0#g7m6ocJ-#MpKCMhWMQsyk9PF@#o(9J$wJR|K9t}8urgyvj2;>
zJ>v#u^9{`lWu|^{_OfO;p;FxHwWZrXoB5{4t~pE&|2Q1Nq#f2ykqdXWdbd%5`<3Iy
zIR_r!FkNnSI!@T^PvB`mooy-BW>XE!w%wWkaq}@@lhv8d3R_N1%D!y2zs7ND%_5cs
zTbzOf3KyFm=@EFUaQfp#v9i_DxlW1(j)#Q0dX96=IqjpUeZ}X>4n^ai(q<BB)e6lA
z4^88U=(#Mxy6lAUY6W+LgIrVJ$liQur*3rIV>h>@)g)!fnXWRs7G%44{_(xv<1+7X
zcc7&oqr*9$kaJIXcztsG{&>&cEU?5u>gdNs3S3FcR(PvOC?EB@E$&@py7KVqHUFD!
zK0n#DF}P!kGRLk2|6n84?>0-fI;*Do980O$9oDpAXZO-wtC!9@cx>^{-Ag4d<m8B5
zNoYN6t#(AYmD9len78{UYyWl0-FgpL<W^s()akIjc=)!8&i50Ocb}YXbdqO}gzfhR
zp3ByXmv?vX=R9$=aiK${?ZM2w(wkb3Z|HIAW?<=HXpiDoIkETCffcRHYwzv!xMdUc
zkz>XC-IM<WFwCpAU(jQ(xz*m=#OsH!u(s%_%F9O_b~X#&+`saP{p*dVZYW+}YqNEY
z$poe^nup(Ln)^D)_5|;{w3<WabaA$pjqPckCk_dp4;)Kcv(?2+K*ntKqxtVDwrwt&
zq!$^ICB0bu{~zz|N@s0sLR1xf=6{-(J}H=8r~A%~J(eG@2&~*!rrdMQYoVZ(jw+8x
zb++8GOWVb&7HZj;ol4;OIzdNIgh4-M`SnfWNs`l4g$^mCC?%S@@2#FRW5R6yB7RN{
z)&;X&yiNRe7kIr$>3QCAO_x>1N2Pa5#PPW&xn8NL1U@-eV&ZyS*!8Q``r@Zj^?}zL
zCAph-F8ST%`g4)1)aIjXoyTRkRTFuR#eO;3E7|ej!nLi=8<JyYChJ7ZI4HR-aAwQd
zj+S24-4dLq3}&_(2=QA>9k_f<Fqz9tVc`y@<IHQjR^Mo{;gQmaQd@aizb#XE>e-`Q
zs@>C{HqGe0*%Z(xKe7MY{~c%dC-g9I^d7X@B<8#6sKj2equq-{ms?v;(Ttv>Row6D
z*mF3NXU&S%D?4OVlUX@;%zod|KhxOQC|d3S!~R1VCiYAVOCt4ZV>DScn15@ZTI|BR
zxaQRJHP<dBU(9?vWqD6e;KXaYAKcD(d)xo@ndLhc7>J%{HVii2yV`_Rfc@qv)2UX6
z@9sKmvVHHS;A2-zw_FKs>EKB`%)PDW^!5_(3tv~AJG*>_!@+|WyxrcOcF(<g=yuDU
zm2Nlp_Wer`E}R|H@=!o7#V5DX=aAN+12>sgIOrTtauTarbEYKJ^<sbIq^qHCef$_L
zcAv1b*mU;sBqfi>7v5auuT9~xo5G-1a3u5p(%T|kw|C2WeL3s0+aP|I;VCnP=B&NF
zJAcgD^1&tlsKy#el{Y%BE3~h#-7vv1OMH!>ewnYpo~#(wgI26A4xFufI1g^*IVq7i
zM{UdR@b4zpTbns12fF>*az50_ZO3K*e^ZuD*s)vlL*KsBr}<y+R+-_$dexRS)JCc#
zV7}w=sGNi-9%=5a0W<F=rtIyX+k1vz^^kJ4ozUJejwYF_&b-=p&XhjlIagroaPiRY
z1yRyV_ue(=xg9C@?A0Up-IEI!O8+U~XH{svZxi*s;MC%;(fV(rk7YCmZoIgswBJmu
zA>ehi`H4A!hF<0yOppD1{PEBJ)iNp59PSv$-jU=sUm2ZZ_}|uHy~--zyRm|iKBxCy
z=JJ^syW2rMXN~$D2dRfzqKD+q^Uj^V;EIS(tXN2wW`p_l+}Qh3vg(2>K08dlz1Fnl
zq~qR8j{bL3A2WpB&5%AM@TfK<tU|l@igVKGCr{M2I7!$B9+{+Ae_O<G<~`M>*;9Xs
zSxQ9*eGO2&wk-GCh5U=%x-(o3B*yFc`kB~SJ)gnEcjdO<$#uImSR;4td~k!IF6;S@
z1v7p<W%|v#<o8*w*I(`jvm9;k3ollEP?i}}vR6F3X8m`q8ST@;+cP6&Uh8*nl#pef
zS@Si#ciV&iUbDAaNAw81<kfrGx;C=yYeMT<gVwdyyP|I-E3@vA<`Gz(x%7YGta4|=
zx$~q}X14ljL~3xoQsV56+4Ios<E%w;vu7OLIP>X4qYcN^cWy3;-kx~SEJz?KMa_0k
zMO>x!oaUN~>zl4#T0F(qLh+F&Z`C20wXD%Lp5jLDE^G<a+}X#Y$Kkc8Bzb?W*Xak*
z%ip}7`k=umBYAnqMC%7loC?hPrOhD^o6J0n8WyhG5Fs67xhv@9V@{6vz-GydjPlD^
zFEegD_4jVP%dwTtWiiYz<v2e+KD{>On2GmhpVJq9#O#nUOJ4nU8;{wVO)G`_%v_h6
z=~x~QYr7)3XD)-$W+@q=_QLKdD+HEYOPyK!&aXBvo=?zSIIVp5E0qw=C%hu!E2lhJ
zy8X%jQbqMX#oY`ZVxGM(bZ?nI3OKOw9l!57>y7=I3cEg+@_*iX>C0ZxGO>o&hmOS0
zV-k8{sds}xsYq040t27I)(rv-LKFIo8u<7Myk1J|dNL*c>yr4oWB1qWxW80;$=|<b
z{OiRT&fUN8tstmpms8krw`*21eaj*zC;az1zq7%gYhK}2<zwGfGPgNPw)?&mn6_JR
z-R{U$H!eT5X7cS4oYXaU8rNN8n-|K*XDxUboiu%2_B+njo3h6OBzG>Gd-vuAhm{44
z{6Al~A9c7@nY}n-?$rz2i%ssonj-PuTC;fHPLVCgqNTUa{oL?=O_}+z8x27mclPhJ
zx1M&*T43sa-*fVx-pBq=oO4Q#Ic8(CQNYeW!8;!^bT3ve_{PkAIxYRR)G4zIB>}?1
z<`-sMmgqPqcK(H?Pv~)}D87$JRG(bR+iE6y`Cmaw44?e9bERjFrYt{npe)XbS7n<Z
z&(pO{PM4p)70pR}DE3ZsMu^73D=JH4+3wj#ubbfUE;Xk=FIRnH0$cL)kCU%=o1e*X
z`k?kbY%h=N!b4Tr_xx^(=+rHYx1HXb%=2ANTq)TzudgS^jOUiXm#?*4LDG_u4;jQJ
zdNHwchif)(5D-1QZQX`~R<nqTR`m~46c}0gKLpNXcyns17-xaCM8UCXk7W%NzrN|=
zJsw}pnDIwfMZn&btIF`Ns{YLQ43Yowu5IhA+S^<kI$u;RHS9TdbpHMIz30O3Fl5&9
zZkXt4&?K9=Onv4rmWR!?5%T$&E7?EIjC|R@V5+bh&ti>$&mF5(-2+w`2&=GUC5dLS
z-r&l+tD?O;W9hU#Z4Fsf*NfHmcHgK_*tL;I*wmwG&t#S8sAGcJC$oAgtkqWs@SbL|
z&wg}oN>|T2-YJ|MZ_?ksHk!cK;8<|%oT$+T=Ckg-FIS}Nacni3(9recPnbZrM}N)h
zM@&b}j`%4zb1vvztlX&UA?nV;_fz=JWbckmbr<cw9bsjBTh+<?==nk+^As+pClTxK
zewee>A~N^I`O@ot^W)VI240iD#dlY5%IQ1n1pob4T(DK>;)Al-r!V>Vgf$Mfa6WnG
z^?bfz_Vjm~b_D#G{{N$*likjG<&rL;E?e_*qXq$m0tLsW6fPwnkAf)-Z5*DQ4IF_E
zvm4o2H>?U>Kb4`6*;CbHk;4IxR_4hwd{$msBGi-`v$N6i0^dx|m^)i;8MrpAH4^sN
z^2lImuda2B+LSEgR%dN<(~~=HZgOmpW(<Bi<>7`_=IPV@Sa)o?ylRo`jI`6owm4s1
z<1i=HV`8KBbRKo1+1u`%?Jj?RS2Ow#(>$gh`VSAbaVwjx`TYKorqxb{XPZj9o}7x_
zUU0I<()jtgdG^iE4r!L8?Mm5kLHDd>X}Di%!0x_Sw{35(ZN7i+|FXHi%U0f9SIw+1
zJVzw&(OyM4@c=K&&rc6Fn)knp+4Z3#U)4Wgk^;l`37p$DBwqEhS^eqh@x|izvuq_b
zxm*s_>&Kt0VXZTttfs3_|Kj4DU4M?VHf-jcaKM30OyqzUtLjEp&9yc=7PM$@bh_Xo
zsCeX<MoNyx1vPEQh=pxTYty^kWoCM~F$t})2=Ej>q%na->#W9N2D6(&)7+V+DjxJM
z%LzRuo9grQ3Y&SVX{)rT=Y*MIhEo*<eAr*^NQq3nt9W4Qx(3y2)55)8E@g<^HAzH$
z$%LR&YD?zq?2JuG*v2}i-qCG_Nc5eV>hsDjNzR<#dDZjg{3fR`jfL$}vosd9{7-U=
zOzWFB?Pm6J-&>;TQ}<0<xpY?1u9eH@HvL+;VkVcvs+F-*XKD2tMqSZTXOj7nwPw{J
zt%f!0uKju?7JWgyfr&vy!Bv{S>-&O@0!r0ys)9GHXjso_v6^Aad9&3Fb64bwX^Jzg
z-1gS-_C7Vu^w=0L)}8C;T4(ieh^?5wGJAs~m-N<6vo5)^p6$M{m)ojkp@R^oNWSBv
zt-;>z?CvU!9x4-C6n4K6`m)GL$X+MFgLC1PAWxMR)yWPKH$^5hSnphPh0T}qfa9qZ
zw>HK|=IShe8fmmpwKyuRGv~8IX5iK0P?nvpb7C@%rHaSJyqNlB-eeK8gSCF|ceOI?
z)>FG$+xtJX_v_VY?{71%#s-_soIE?z)GTvxYTLGq1q;tgWpAF^W_D+7pw{;6*n_U-
z_h#O+{hqt@+c)X_W@+;s4^AY`nDZ!KGyHB*M|{VQ;vRjz>%Cu8CpHMBY!SLw@v+Wt
z-t*+O?p=r8x3=u^n4#@3Ur5Menpt{Qxtn$RovO6mZ<ub?=BaZ!)-3RRKUYM26L*-$
zg+58XDes#^U%r0U&f#l+L1WE?po^`l+mbdMR7sPwIn3nw=Ftb%y%BalwQ~!fo=@0m
z$jF|1!NpB7X)4z;>7{?(c=eV4Sf6akbeh5Xo1u9HXO+myj;LKN0(yJ<UAB36&APBf
zjeC+)Re@8~l|PG?TUY)s@UODs;ouft;V`oxuq$dIS9HeA^K;bNrq5Db>Xv#q;Ngch
zt22pHe3q^XS7hm66;yJ}D_t0^xG_w?@`O;XsenRJqaEjiLmOAkSfee~&GYf2sQ6-5
z-HB^FI7<%ktl2bm=HEFV%bwg4+j+}EkMW>kti>Af-c1?}m2X!oR9*>R&9cdT@}G-U
zyq+goXI*z^%xF=5{j=kcpZoD92@j@QniGWPY-7~)S)nTRm?64q*)+2)i_#1QC8N`p
z?eBYfrjes_ftS>Zr88AjGF)eK$4pr!zU;0mSMY(<B#z6|kC-f*-pAN;O(FQq`q*c!
zg^&1+XWx{}-MdzA!@}<TGH3I<FOHf0zp-Xn&b-g&cQo{@P6+Z@E;N=s>AAe{Sqk5Q
zZ;z+HN$nSXXxRVaMDx6`E$*eCI?um~x^A`Ha`McKlx6h{CJSzK_%m=#5@^a&(Mjf1
zsg$UE;_S0&Ps@fRXIUYKudBi*Bo?Z~%V)KA<`iyt@7d^Tx@d)i)RXCD4?UTJ9x!=D
zn6et|EK*o_MbPiC(6YIFE#7j1!P829d6+X8tl(74eCd_AhQ}$i`BL{S_U|HxR<#+d
z4%_;Q^Y4*E>sM_EoXQ!!&?H#OxQyG>SR|Y``@~_{Ltoe0x?EjZ<*{00V@cTeUEx!D
zJl5|&#}zjFfKuYx9}9Hn<V9VqnkKHQd3jTwfN92`|I->)Oqi+n#+@N{!<w|?sW&a6
zy!z%Fc{+&(1vgq`M0~XJP*DGJ{Mx6B{Y)>#o!t!E_(dlvF#XQfewDbv)$L2k-;x<#
za@PVheHb<}O=u{ZY0a3?yIz$o>tv?45#xe=K`Z4xzdT;{?C#VCp$wf^&(OA~Atr*n
zzT55|$^E%W-P7Zt-~KsAMBcgFpZljMyrJ=y$dA|YNqN17G0xADcLbO*Pkw8>D|&X*
zO)+!No4kjF^3JxT2b*(z<~{WMlfR@@p)~()wa_ObGM(Md=L-Gag-*@BV-dLKoOru&
zntUhsnI3ijq{co`(a7@~woS17p`yU?D`oEMN!GiS-|Lkxkj!}dU$5J&xooL0^RKxr
zUWe9QR!ejfy5O{lL+Rp_kQa-@dtbPSu84MaJFlnpGW^>M(}-nUPpV}nIx$6UnKW_A
zdspKuSxJNAtlxb$CcaFb@oSoP<Lt)NmGZ$U<-&6(ME7JWM1JDreD~chtjN==OU+}M
z?Al=AorfYMU4_4S7;>%1Tevq$Kyluq(^+ditz4xnq85?;_Rxkfx3!tA%c2ZLug9Mj
zy)j$JaO2yo>xybE9sXyoZeo2DuBJ8d=DDz>#MQjhb+os2$NpKwBEoRN;^YP64PMGS
zw;yFaw@$EcdaE-3@n`|fo<yBbvrfdxSL#2PbWyEdqpSE=Re<&0>h1egC-&@%_@B`+
zRZQ_r?vr)XcDHgKaaWur(r`>ApljCJO;gvGq=p`0+U(Q&RKuGy_56`;fj;piT5EDn
zyI&9H;#erYSElHFtjR{!AIHM?RnOd7_g`to{y*X`ngV$mSaZH4dM(R2X_D<Qd)<?}
z2iNLd>iLzc5i>h0YxAo+cdR<fA15A|Dm>>Z;|xn#@o&$U6kBDbKaYOYq*!rG+h$&{
zzw{mEC-Ten?AA5kW&7~>%O8WC2lrUN$YBaF^pLxgq#QUy#I4ZAXm_>a&Z!zLYtpk!
zt3xK7HNMs%$CB<MH0NAd+4jayKj*ex4vEr^o!=X=zQOFnmqsQ{EvHi-jE&~Bv6u@o
zaONCpz4yO-DeouYN&dyFUixXje`4LFaEK-7(xuiHD<$T<T4(p-)x7(eS*laMu4Xh{
zvF?`DpQNe5+dc&U+tPMAC{MI(!)^OpyQXo4uAcQjHasZ*=BaOCn_`u|e0k;4X)$%{
z!sBWeZYD`8B(LpL;dEQ)>znCwEq&(f28&ZWW5ssWitTE-#=Et<B_YzugYo&1xldi@
zwHy#jh&<7w+VYtt=AeXV*2}qDB6>b3xHw(Cd8jbj>uShr6_(P#J3l-aBopqq@Ac7S
zXbm=Sb(iTZDma?$!w@2K)b#~R#ETZ`8%*bW9(aiytvV6?W$LY!y-n3O=cql2sd;&H
zu1Ax8<W-i2fD2!**NMjdmzI-tP?Qy~INQkCI=O%?yrxY~YQ_V@8}e=vtq0v&Sr=XV
z>f7++Z2b<m>$P_~ER*u5cwJ{br!3|7c*TsHTP3<PA3gFp_atRe%#T2urxDU3+(#Dr
z2~@PN|1-m0^d^hi%_d7(DchSmUR_iEvaQw;X_n|{wwdP@aF0pw&VJu1P7h{#y*V+v
zWQwQ3k>Hmi+A})7^R(!8$aTB){M3-?)tL8atw;BoI4_L@-XV#Lb{<;tx24gOK|{9n
z)Jgx~8~pD)83KMB>GyEeywMag<)Cg!k8DEJsiTv$O)s31<q5Wx7fU?9Vy;y6)j1rQ
zjq^-oR>y{ZKYJ^&;_BK~`E|ONjSOS|r|-D$wkh6SN_wLwN1W)zO+V)g%v9Oz810qR
z@jq0+Lo#gs+`es+3nr}f5L%JAFeYW)UB1kMX;PdX={qOJ#6H};;+pK%Q}UvZ9YvJ?
z-JBg5$Fl$IHTNI01C5p*C_VD&#|&jge%F)k-leGvMO}{GWhvh~DafEz+U8-M&w;-?
zn!*m85t`(9e`lkA&(jkPuACNZ(Q#2HeNV(~x)c9t>E}#E@lDQ-hmKa|I!dTLbMiVU
zS;tXhnYL!7;)Rz&GEvXY^G<Gf*>deqYrq>WKdsC6JJaeFZsy2baqK*HE%$O(*kzTn
z$q&B-bR2wURXN#w;pN8x$8SAc=CVW9W};HRZ{w>9_WxR+gtfI8ihkbB_$NC>QY2S{
zrDWqhr@)>bmBx=Q98NVeclXY0nKLV)@QOpqz5NY+dn}GrOu6EzD3sJI7vaZIU6t&#
zMnwEb+#*3v@h2YAKRUF(bc8=qty^`bx1iyz=|hW0^CoOLnEjzg`(m79%B?28t3Ete
z`&=}bIa4;e$cxJ@iS21T8yc$T_@w=u6h~?3{0SPaK~H{64eiQv`?XPnxAIB%w)j7_
zDt|judT-4ap8DYLLY2NxFIGFW^zTyHRL~?M5w}ft!7ImZAsvM-zl6y}ZIjcErRQFg
z%oN*x<i!t_J53%9vJVAA;}-6=O?7;7&E<^u^ncQxTMvgu9ntcd{-0ri%e1iMBPo5|
z5+^bRUA!Zbc@|yMsGXAh!QZ#Q@w7#66rXE!9wX-jM$R>?kp<$>BCL^hnms8CwR040
zMb$JD_}e9XC9)dN`u3`29F*+Js-3t>Pf@%mQcZf*Dru?U8&?)CzN6SQ(^uXucI_%(
zg-vYDj2wzWNo_aX!lxX{jGCJ{txHMjwu;j02Mu>s#r!sJy3v`d^wc9;&F}br4^@pI
z7tIe|_j1`^tw^d3nUp-^{+cp2`Dv@=FC}fvlGyNx&Gl-CvZv#w88;o?+;9o?a7|I)
z$Cv!sah|KG{DHZ%uGDyZy>LyTP}9XC`Fq2>&$&*#eQ|T#vU=v6Venkx|Kj0)>q#Q-
z0$bjGYzav8WT-vqSINnwIDh$4my>Q2J_fd)dUfLS)G4-|QD-(WRc9V*yC>s*Q|J53
zH(^sJxGW2BWzt>zLM5Xh<>H|ie4^1E7w@;sd9uYaxBp#+(8?$O0&f>F+?sewMZz;^
ze$Ul#-OF_<>FKH0wiR@HE4<~%YwSBan~N!Et;SPDL;m1pE;E%__J2LiY3BcF&4Nx2
zhgs`hhQE5r#+7%3k7eGs1r8dVTDf<QO=Fyx!+8GAnQ3><XcTt(teB&|lEr~3#cM@Q
z*~}@r4eNBTG`U@IvhHEG-FQfAgHvMMs+GNms(KldnpUY8q@A0(N^07wgb(6rW^JmG
z&(hMC{a?UwR9265joR~!ds>s`o}Kvf_!|Ma?6k?5$Jo+5^{>}P@T+Ry5It8gXYS3H
zX^)o7wTNB4pCf#GTHnKsn%Ybubu#%57ay%=n3#2rtt4;3k0&>aHl7w!>2^s?-|u#>
z?40(&*n0<>)D4}keEs9CJK@Tqvi_GjkLNwreCV{sY0c6Z0xp3+ul!-SFttPHWTod*
zy(o_K>C0TDLUl5ly=E<$nq$IsKv(BOrQLn?gO^29S*))(@rpnB`l5iX_t2~#1#Ayb
zI_WLpxGLT;O@piUz6+D?WIN4YYo{*VFm**o_r;)(8-7mcz4W5@*fh>(^Evl*c}!E;
z{8vNrOZc|Z)jn!3nI=vB&)q3{Iq;gpgxs{LZ)HAr?7ERz==*_vn%{vxQ(0b5R6KjS
zENfoKBVGCX+!{BXW-(otUHfR4&GHW(6K8KbEqnQ8<CPPyOD~-~c*Jk%B>%7d@m?FE
zW=zp&)(h6$z%)6j^2l79*#621>+-KU>wjss|Iw`T+u8n$lP!mX?V3F2H3v1;9I{%|
z^y=1z{Edg(+E)3rzLGc<Qu{D()vkr_b}hWX8L_&lce2F$HM3r8Yvq69Q!?j%w9e_S
zBA-Y8pDd+c=h_;*|NL`T30kibmh`}7_2y}^TfJU?cj8m@S55Jlw7shRpy~pVFLPxl
z`8w*qe|O`Q<X$fs?dVNX>~hXQPQC{>N<9AW@MqEVo7s*9E&Jw49yF9%%-Z9|TXffr
zrR7PW((Ea7ZZ<Km4sjBhRo!=9bdsZJnI3bmXM}=4LRf3dr=<a`%YsCY{QuyxByi1X
zv(}9(uJNS1xJ?ML{m~SBz1iO7tIqjPA>Lm@qf0}wOYJutl32qO_P!`$I(s``;k+Z^
zE3KY{{an+@rkgQIw0_Bap3sjmo8vcL?D8_v-DJD=$}j17d+94yx95C0HL+;6#Kx@`
zW?gihF<ZvCOh%n$MbCl_C4GBlu-x7!wy$Dk&dZEDR!dt%f+|8%tyMP7`j#tgvgwG#
zI{D>Cj!f-(H0#c}9cK1sQt`i(bz0x$&i+(&;&R^Uxl1oG{kPq-N#{3H=>4L?;8J^q
z@*?H!I{M!W&lfpeX;NFm6w2*l&*7lY;b1Retl98N;@KRZxuvOmvC>w)m6N|$eir}m
z>c|JK_dC*-6kW@jTxe`&tAB0fvrNDDU&OR??v<|O@csVp`?s?9--RklI9hrb&-6RB
z*-OSu`X_v8>&8pGZ$Ej-##&75-#2&vR*5ZQcUpd&D%v`wH13M)>TL?$FWYu-I<`D_
z4~Z+6YhkiiKDe=og{8sk;L5DkEaDFqN2!E5e%aylz~5`nhY9>k_VAvbD9qhs{xvwc
zv@m_^War9I_nr13<&H0kye=GmZB^+Mvh`q&qsWx4t7m3?&kQ{;_c8H$)PL#aFIX0d
zitM(GJ^b_c$@v{;cY18o4Su!azoDwcXN`G>*E(_>dz};gx<FUDIc?hlK^bi;S?jX2
zTJF758vnZ7n<nQ!GwNJWTG`$Nt7Y{^Haxh0_uFj6jn`xsYG!h?%+>9A;1~YTY0a-Q
zD^*TEVB2Tqnd0M_Gl}sihh{hHi+8_Xl=Z)S_08t4gzZ(Pg2P36mp>KU-=zE4DAf4(
zW^3!xXmh=A^D<?Yvf!N#DZ3eRs<-twNpKt};BMI}?xH8nWAr$trt$mApk)_kZm;B?
z>v!T|vO~q1Tf4O%CGMN7wySLO<d3Tq)E@1hotB*=@%32#)Uq32^|UQB-`hC9E}U}N
zCO_%FmFy1(g&i~8yy|zJYp#!$mD&<nX`FqdtM$3>X5ZAewqJO5nK&)kC9=TtDP!{9
zJF8v!&op&Np3OeNP_LfFE3`lLqK1Y4%Vk@p=Q{e9DZf5ivU>Y+-=`kF?56$`rUZOk
zo3~rVL)Wy&*>rNY>6HAvQ_BB_+LsrGTRO)4beK|mv;S{XjHPx8Q_=lRHxK{4nI0nW
zJSMU^LvXg`tk-`R79abV<;~X_Hs|op*z+0n))(X4*KYZ0>FKm~-s4hcrRm!>4HrF8
zk(_&Mk@>e9ejbajbE^hhr568^|6I6l;)AA!ZMVA;Lfp=sJbgKSqvOii>l~J7u6i8#
zB+w{4iZ!@%)B5>E|G)RR+O90B+>lpryr@XKx=4C^k+%IO^Xj7X@1OmDe@-s{qTN!O
zzk8edk+13Y))|bpcZ|NK|97}@$gq6-uIlYl)4s3VS1o<Yx?$yagN@~vbdO#-`2Cj2
z`Zss%6^zzf+_tx3W?62;#K|Xf!qq|CkBgT};zYoL!%eKJ(=<cdMBIACjf=MY_?YA|
zL5y{ZR^_D>uPICsvvN)rsrt;Av%}|Rg0OnKu2pH#%}0~H0~R`2PJHOJOhcV<35!PR
zuacEMEA^Mp3OQiJ;LXXIz_!Ec>uLthps0yj)}d<JGqRR%bNVfo7sHr$vnR<bZF0k&
zBGw)&YcE~N&1-YEt~xqt>TO+y|J%F-1g7Pu%;;MlRHqZ_$gpmvut(wLQ=!`COKwdt
ztKKELI;t)E=c?zow(hU{_v+-b+}+>a+}U1wTTCIksNiA2--9~mK2_|Gnwjx5?Iyok
z3FFr#LE-`3S0pr}7I{qE^t8I-L+}!xKqt9B(=<)5UN5m<`nT%m`<LhE%a_>JE&O@%
z&yyfG4Tff+5Br%IZ7v*O5sV2q$R-(cVc~xzox=;cA{Tvdnqjczg45CzC1y2&NTxtd
zK~@97qXMBig-3;F7fp0!Y}mxVV*a)l{E7m-PTYz!1zrWZ?XZ#YYfwornxwie!fDb{
z-6&1ntt)RdWO@lKR|#5X);4<*@3NRT+-FW~x$=La>Y-aRPwUPw^HkvEoD$&Rxn;@(
zPe!%KS(820rusCTSNW0h`Mk=`Uzh!rJrh|2%OVn&Ppguex_0`_UmsmB)v5G~1bP4N
zyb|PRzBOc3T!K)j6{DB!xsWXZx}x4I16)hBZXJ<ey&O<)?57pa@}gBP%<ZC6*Tx3f
z?me7K46fbKTsDJYU+V1CFx!l|YhS9(Uv>9U6yK-Uo!opapUO1!GPts03OlW)bGJ4H
zO~1o<JoWp6|6IE~7WnZ<eP8(B>ZMa0*XbMEmD=BkcHHadxOYOKUqnt*b&^Gt=8_qH
zi?SYd?R8mwC-1Q7_Z5W`h0L{<PiT7is5^Dn4b_!Ca__X&%r*aO`|8{jN_aLix$md$
z)^#Edr)ROQ=$;bz;T8X~z+YTaXD&y~nzS<Nmgbuc{LB*;`0iAg&9iK^(e0?s>x4g?
zT76t-hm@XX)<*Sh_o`O+zP)|x-P>y~g0}Lh?TTEhXYuOAw9NM*s*9xywzwG1GTZVi
z&GY@sUxg(i9~yl9_k2<b@Bftccr|~ql<E09w$kQiS$7#^W&S!|^3|99%Jui@1bv?N
znp!o9<aExmTd{kzwS@BT$Clq+Zq=pwpqWQbO`%cc(~n2dwccDt^IEcg3SN99v{vM&
zkdM}4?>?=yTKZ<2eP&);BemA{zs#2!J5%rZ{wv?`c)vbADd53b6(s?O?y^50|FZ}C
z{{4Tyer3}+hNo&XgJd@}6iz;MH>q{?%ML9)9+f_Zu!WsbC)yhKu4rFj62V<{p-EOj
z<RQE0QqF|R<JL#5u5Fy7s+)F~rOl!<k!P#x;jeFw*gM?3!B@HCaM9m@m^Y6kj=u{%
z(rJ^T=pM32aHhtj)>M`*p2<l^LXw1*g=|bR^1SK2Va22!HmNB~BqD<rF-?8DF(oyF
zr%y6fX!&Ul1#Qnur=%xuV7=m^Z)6eeU63TUWufOW{Us~9&#?v1*t(;YhoM0@LT>R)
zx4K^A-W1P2_cn>HRXJc7z0)gsLzoJq%frB}KhNgOYjzKwSERKy;;7bVHQ@-!cG08+
z_lGYY{y!MiGC_1p1gmaHuy@smtisC^Im=FLkWHR++~&lezNb%w<<s-{X3g2Xu=<Pe
z6q&9(hnH4&%>s}6GW&AOS$+C~h2<8-;;8rDFJ08F&aPZ2_EoQ3Ym(f@I(7fjTgI;9
z(m^SoT()h`Sjf9pfJb{zi0aA=&8jLB!6YT;Wd;YArWgtfCcb(oHfLg!Z(pPEUzP)+
z_6L{L=AXJXcT3p*A6GLc<<~r(y+bL;T(eVj3RB<xj9*^Ni(H$`YegSPS%}83brQ+m
zz{eu~LTi>1yV}+ZM<iGrx|jXh6nM#U&QhbPD-*h$xMw*eZU{S}le+UmWMc2MnpLgB
z^E0$HuD-gqZQs>v1^*d0#(pZg9&?%Xdc|GQ_{yc(`~I;`)Z^$;6xF<Oh_7rzpTe6B
zyumr1GS3X&h3ceEbUiI8?;Ux163^zWD>^R5zi)(Pns7EX-qgI?6tHr~(X@h#lU$!4
zGM@GKjGoi&(;Yp>gcfJ6*V%U~Qkx@LV&w$J$z`jSM0Cvd^yz!Mqu8app!qP@-3PtP
z^DXZLz7C7s7g(;fbdxk|xFi3{D?)|cLJYRQ9Qy-1npTx_$CR{MidLQJGG{80&d*U^
z`RiAk>Q;v<YTGW%{WK?i=G5x6T0L=J=Hzq5*X~4^OTIbFW*_0SdUAowGNr`ZzOAm3
ze$QK%3Mt=g<S4&%_-@Dhmx%(O_5ROOa{GA7uO`V<uf%h{>DhCO+#k;meLqjWsoOcV
zD0-?d>%#lRw-PTs5$ieUzqDt~<w?G?QbLk+nHC>VSix~4m~~(3@}rhL_EoO^bCteM
zJ$lio;<PoJZ^`y4YXn+@GZnjLeoYcdy1^^geVIG3CoyR1rO#7LTZ3PfZ77^2!M*X6
z@?kNlUuya;UqrGz!rY9l+ONNPxa4rE%Muxt&gV)ec4bV;taN<h;1#H}qW{Ij9*HpJ
zK%1PT<tJ|FIq_Uue@yA<7LLif(+Ybh+osOh@kk;rS=BFoB9rEnw&OExXKt>jd=Zi*
zz0Zt+O>K{<igSJ2oxV$)H!m6oWz7BjBx=p$ysiHuK9`yl=6s(PKD}3J{rW3+Vhg*I
zL@!(RRk~m75A8X$duO7nTGq!qJ32P)y>NH?$`{sNHc`vvJB*f<YluHc+qitD;Phul
zGQUjDeAdn#!Vzn>nKQB9`bWt_o#oaiEB&g2JT#?^CZ77}`Ke^noFgBm%xm+H>ALiR
zZ=vo!OUb;v&)RNBpE(;z7C$@`V?8a^YgzB5-sx^jO>W&-e))ERbFQ}d<fWcH9hoB8
zo?+_dQ@`9Qb~+t?<lR0qKh~29q7FBg9-6B0y+34G&*t)ASLv{d?|a>v`xaemd{)(S
z@a$`bKZy;uWK35tJ$r8LRE8UdY-L}T25(Mm7fai@^2CiVrzU^j^#5t3)0H=BA9UBO
zYG1!ifiuZ6d(BGszpoB@t(9I>y4LaGwKsRIM0N-nuK#mLJNDsI!@YG;>kis?91sfo
zb13V{5C1<WM5T8}Xjz3_+qmuP7xQYPSN3M9)7S3Pl$8`$t=(eir&YEsR$Hd{v`U)8
z<h7e3CrwXt{~cf+*1csXU(ou~g1XZVmU(T?U2eF6cUOkj?hLI$4G-arhs|?N2Q64D
zf9_3KSfXr3qDWYytW<&Mie%9v3in(bnN>gB+mXYwgeUJ;lE|stDJjijd>d}JWbV4A
z{a~FkcZOrogF~#bAMStY7TKa;J7Z;$!Z~(+9u7<OgBrX~?9Op`x|dE0nDZlPe$xNr
z<#8<YOwN=oDmc1o!_h~Y&o*4((tbT}(}W`*G`Qw0H`&GVYQ?(Hgr4ImO;wjZ{I%j%
zYsy&^bujSO!$m9F>MUOQUGw8V>942LoTYT2=^J~<nl+zaEey%)6jhp_dd-pNfiHv8
zhQPGcLZ^Z~Qi6<)U84W|-XHOu;d5b{S*nOkfAAzN5p({KR|_o-JVL`ZHfA_;haEJ3
z#JAE;UoY%w*sC?+KfHy?3@#QpU92m-nCBFI;nl?y{YxDJlDX$&CUIQu$&fTX+wmvs
zGS{N`SE)Y*o^>ob(J684>aLAl3;cgA@|RJ3{VPjXFRzz>xBuHaFK#V!;$rgbKcj!^
ziPBb|PM((k%Xn{pI2O^Z8Mb;mpYwhlFU}uFCb@O}Z*%_7BV_z2>Grk-pW;^TxsWR=
zAb3x1{mKWqQ$r?*89ikB%P9OivEW2@fsd=OT0zSnMU|a7B1?K?!zS&HXfzYb|D3{P
zRlpqP7O;!w_%4<OD)q^`)@55LO|n|BINZxfL^kh$oS!Cl&m*;SkG?r;n5=WK$kXb+
zpw)jNUMc5^m*4SaO><-NoOcD4MVvdb;DXNkr00tk@#N*{Uc97h&+{UWPpgLS`U@f1
z`d<Ad3|=fRw_a<iia508LVo`emsbp*x1TAnxa9v@Yt`12MTRb3{5LQD?s}(So0*f6
zbo-Bhnvw%of>6++=6ezUAKug2boSr*bN}+K7}g7g9}Eh*u+wY9{RS<wyKkE3Ih*}m
zD8_I&sH!!sZsmEijwxapr(#x~?|L=WC3X3P&w{28&6SRZW*Ie_c&KG0$}VbCy_bFG
z)9Z6Jp4QtEgr~TEJ+MmrxL5RyqhHrN{kknRT17YcLTX2z|9Ugs9~R5Lf6G#Ad2h$-
zX3u^){)h2TgQkr$&i|ZsUi#p~u0tPxsdOiaOp(bv%PW7y(UhBkCBaF1ij(jHC&MpJ
zk5_oU>)?N?B=@_>Ai7&G>2?1XCfzOR`cv0!m)g8_iNfL!`Pb!=xfOyf-YhZxm>uGE
zY4<h8dtHm0j%BO<&fz(6n(=Ke&$WZ{kN&S=3O{+e%2nhS6XVXD(>#Ud_t^Ho(cYc5
zYF81LcbFr$+4_Q%N7DuJM2#l0L<$?GEzOHqsB)3p>ea(NE4oC>Hn3M3aD2X^S(>5I
zbI9M7U3+~{fNoEKDpPXWb61NsoR&Un5iI30qI!D^xbJ`BQr#M)K2>#s1A~AAgNZ1I
zRcWweXz=n!sS%01r_ay3w}|IxP-yVfmp>NWywp+sK)~?Hh0F@=v=wgQxm(_T3wV3u
zoT24H-ZzZL9xz25Jyf^ofHAwrDu(sqe-v4LS*?ydm@fBm`X|3;hWx1t8|5#Z4J>;a
z)bM#<T0m6xr{G_2_We0?YTp~N^=BGqby;d1wRmxMWzhd%5e|Q069ENwb@L-@cOOd-
zohN!O<>j<9Iou@yku@JLR85Q&(QTWND)E$gZGne|imv42S8Ll=#9H-UZqoS9(G-6r
z^~#Ek9b9Q~J@2K&k9EfIstA<G#E1P-GU>LP;-NN0HD{XBbnh)^k}91{bZ2SneVNE>
z_E7GozTPu^t*D2o|2|rIh`3G8ezU=a@yjHUyo26CljhravM3s<zFcei`0BYwYxVgS
z%ge2wrZdTGh1Zmt$NR2{J=o>z&iqb2I#Vl3K<&jt(R}4yZ_WtidQbba@Z*EtuosLg
zp7hEwOlx8gW9^-E_XoG8YM!Rzc1`{@vsQ4rubSi_;qskrYO&%Ihe`hf+-`Nwd7z|o
zaZ!o(n<rO3xt=fzik|-b-Xs%+t0sOfdY3;vPYG8uDRr92z*uIU9{DJEx(kzsvf+<S
z-ltl0dAh2cO46Agr~549>Ha#8$IIo5p`OTQABoGEi*lI_IE|h^F=~H&z3-^#tpiJr
z@YOw0Vp{az|F`3xH8RdlNoJ~VE->#8mtt+QTe|AY^!mBNF~&<9?p>c^aDM8&tqtiP
z&wT1(IKR3e@Z#AihXvpJGOfHCbY}|F)uyPOIblk!A$LmH4{F&QJr!BCQSHp-HDyk=
z)f{$ri(5Zlwkv)A{R3CqTjA8d#rD|}($Q~!vQ25{F1aQp(vkXAw<Y4bS@))C?K;Q)
z_ow}I(ye0Cy^@r+>`1sw*0#De-U6Dps!of|5>rc<QggzeD`%mI6tljju;&s7;a9JB
zuPQ0rBx3P~yDzsWcM`MrGc)!7X_Zf=Sf1RPA*27XCgc={_+F0f`g{du=eP~deL3@D
zq1?F~^IMG~a^VMNWEN@fe6*so{hvv(_4LW|KAe&6Mp^1cVqbL(r*f?q2+|Loy0u{H
zeu0XoPQl4Lolh?^VJhx?+T^pOJY7sHtnlrEH!DSNJq+8K$Ng-%Z-eN}uOA|6ON|^B
zpX>?MnsUHUNrCg&0l8PNLza{-Td^u;{^9yb&5?>h4G)}}_LPW)3tIhbioGuB^PI=(
zZU~$D4i(-1`pvHwHkoYLleYA}($f!a??0Lx6_@fC)?$~C4ol?SB`$nxO_yM6_L3xh
zvE;@Rm!}1GSn0;U_?$ekqth!_CE)FqAhC@ORnqRaQX{`PsGjal3roxYc~Z&Y&$*_h
zu3yrlm+Ezu>4!~CtGMPhO)P7jvLVX>v26_so!uUPUV2olUd&}=v5mL>(X{+K6&lCC
z%qgF=^sR+aR)PBR)zO<D1iovNt-BIvQPBUe(rm#N%LRv57A)@NEzGT1Sa>AaR_;6d
z%pm{8FE!@}9W)4^%`|m=cfb>ua&1S+(!@vWRg_(W*11M}-=M8B_35S_nc2N*<z@aq
zb!Vi6*lPNpSRlLLNJ#Jh9lFcb@oe1Ty+r)jse&{g@2X&tzc&uKJP`;#uj0}-^UbY|
zM%JEMe=hhhNnp0hsyiTVbi~g%QHg8I0<$0PR+nzZW(w-YJP!PDVve$N!$y}m6Pfen
zFHODGW18wbb=T^GqUOj{ui$Uu@0KW=d!LVRs<3gN@m20@i?2m=(2nRo&CxX+v9C6-
ztKeUE<LJKf^-}ha>~nS`7#&h%*z<$WTa#l?BG>75@jpxanNqE^e_M5TtY$jYEzxOf
zvh48ke|c$gk*5z&XOf?~wyW`tiI2*Hs#{j94l7o>tvvaD$A)d2mNHG5ldO^Ys4i6b
z%azbO7eXxN&DLKdrO}?ew@OiXj`u3T|Iepgk?7;}vpN=<zu37~&^`05h`;(zzn8Zr
zwF*owU$K7opAwcsjt<jH7Pn71z2^v@=wY9kQ?K6%3hvRb2@mc~ar!!GyN_s<Pj>pP
z2Y)i$&IfWX-&@qtEwOd6?~L;-;qFVePnA^r(6a5bz*`&5><K9+=lJ+t%bPV>p?*=!
z8Mk~U_swy<72$Rqdx|y-e*ZJ)=$hC-+0zd1V*_2o_k>&(Js~!cYad%-_!BqLpFXo+
z>-BBq4u6xkAd+Jt|JeoIe-~V>Zj;MRO5|T;`qp-SVe)+2MR!%NnC<%Ea%eHP&DC{*
z($>F}-=r>Cz9O}J$CBV*?UEw051Sl+D&75=$=xNs?CF2Q8Wm-i+A_V0vU@9^xTLJk
z2wA3gs^vzGwqez*v^$5y3|<}0-+6TNHIGFLME~T@5W9KL<)D(P5aVa<iK(BP#Uw3q
z0`xZgT6W`f-fOefo7-3Jn6mfBjr#%fuKxFBP_R6ZR5$5)RMBgxg9eh14bIt{-PB-6
zDc&pd;$WwCyWNKN#T)pGOPZZGdjBjfZ`U+=^D<TaRVtra!8hG~{`wWKE7E`IUebS^
z<I`n*yYl;;1xuE{+3Z)<wRL)L#mlJMfjrFH1Z#En<oPKaUa@%Ew!`5@(tBSh9Weg8
z{j<cyvnviysD60w+Qa4RL>c${@*ME>%XE#i&fobnApDWiuGYmCcedQO`!Bql<HwW|
z)dltzYeb@Ku1ZT(Z@hSpYv1$T-(FPTikDw}v338)R}LI83U;@?<^Qn%FMQ|!)7!`A
zI-I<#!e<k>lxb<B#9iSnhr~l}pJ2FWTqE~+?qPdt@0`~ztNk(zUUzNoTX3U6lq<PA
zc#g;kiDL`r>rWA7IC-T>dxETqK!LNsTHg)w0{?w1H<|qi?RB@i-SFpeGqb{t%*5yG
z;~W_|gw@_G->~p-y8!DIZ;Odn#5Bc?{cdjBsC>LXct?qdr_qv=lQjHS`E*X=7T3~C
zKJ~;&ElpA{cE_DFJA;>>pY2f3#p_kFB5(oIDV5k&BALO<0_MBTc{L%}X?2Kvl4!zI
zp78&Sah{E$3TAI^Y>Gcy_13FQCwe0XgNBQOf?g~m!;4G46I_^9hSf)I6g<!t@^DSA
z65qU6rmG)s>=o_mogtC<B<yX@R^bb)LJye*3297R;NtM|GN-VO-;5_4n=bMBdkYv?
zAJT~b_s1~A!~e{RxW5^lQCIt?s)aFe{d#xY@64{cy{?Sn3SCnV-sRe;{pI@vYp3@o
zSq+Zx?W_6z`I%(Xf4O~OYGHp%Hs8ND;ZNnaJH_qi_urpx8!hH(cu<5{%;P~Ln_9<%
zCgwbY)@B(W!G#{ec?VltjZ;@7w#_+m<zbsb(Z@u|ITJD#`N>XOv8YpfTLhb?+98d_
zolEybxb<3Gs(Adr&+^#~&VGk)6^j{`3Y=U#(N)d!$s{kclTRkQT8XGm39-t2n#JX+
z;Htfb;hNYqK}B}8^pMK72{So28gfZy=Q&-~NMgJup}Ay3lDGC&jtS~c3~g$TOq+W<
zmog=9yEcm<S;8c+c2Q4~fbHU*NBq{k--7vdkH=Qsv0B69-f#b9gOQi+$HbL$7YOLS
znz1U<H>6);QO9<<RjZPYuX5qIdZ=sNqDh-P-GY1?1>K|txhHwZC}sz3(l$MP$xE#5
z$)qhRJ6j(5>LtDo-ey)BePz3Q>+jqkE7#C<!QP5XrtIQe+Z`azdGNH}s${oi7ozf>
zcE8`3`Sr9|G>59f^$?FcQ;+-?o6Bhy>BOzCla`t?&okl(Z)Mtg=Or%NX3XH=oi-y}
zoZEmaDJ|}rpKisr$1~FxbgCU(7%cc|!Wq7UG8>#sHj4#{2xeUBw%^xt-P<Yo$SZ%{
zUakwLtroA@)WxuRhZnc?nhU|+wg+BLE4AOJV)7+<xl&8)b&gtHhMSL6Tedl6Y+w7W
zJcW7g>^n-`g|SY*RK?Bns-7P=zwqrxy0`7t6=%4#m-Kvmtrq2>*!Z|s_2aC@V3`*Q
z&C}!~ET3`3<xOf<YTS3UC1%1o#x|KFQ-1l#cfEM{T5qdf^)}^ee!t&pFoh;}n!cJ+
z*lD2l^>^Q%*Aag{9(n&Gx%V{l&6-Khk20404>eT^QTghhbicqUxa_>`)FjcJdw=lg
zTP|ba_`lU)@0(&C!C3;GH<vNYZt0nj_$t2V)7)yMBd5d~m_&ZF&6vufkjQ;~=F}5m
zA6Hgwc$B$x(b08kLi2y9Y-swqQ<^dVs9k8`20PxusY|mC_64RSW@pasT3zF~BUW~u
zZ}NphVv82EYIrDen%6|~DSzK&KhZ7Xz$FfkHj7D0K_?<TYQ(%2l`d2}w?ysEqt5L|
zTwMzy3pJipb%`xLG(~8uwx;T&>!B<PZY?>kis?(dq|UCGYItw8u6Xq0@EaZaX0KQk
znP+>;H7YP@PFQH1WO(L?)ikx%4N8VO(khJVY4*n~Qc}A93kIj$y1;d|V3{!|r*FI)
z^X!%>YUwL8JI)GxGGgr6WXc)fs}+9oREV0Rso(`C-J->^@2)h-9eCNUH#u1A?u$#)
zMgJ_-EC1=ebm@m&KF(-o5ed1LBWvw{UgW>~q)qyu;JJcU#l!1O7PjVHx}b1{=T7~r
zg_3#`6_ao8=@3!)zC-R)!Q>?`A7+@XWH{OD#=vz-z4_b8ovf}LB|?)fyl_fV<gqk#
z;ouGII`KiVt*c|9Y2k~mB6b_G$SbY}t1>>Ou_!Al`g<twPV!i|e2JgV$tA8%FDI|L
zF7_$*<x<_XlYEwMkz&blto;zJvZ_dH<GdiBI)!3k6=5Tz!wx}y--S>A|M%gF>VL)y
z?N+%Q3q=GNt_GMgGVWN#u*FJL*Ce2Ew`NEAyJ-%*uiVz03Q-KT`yuhp=g8U}LjK&9
zqAPuS9JzW|FBHk?IK1y(!TR>qu5+&~>C`<@Ft_IJ!le5z&&e^Z(r0X0#m}4}rNHOi
zaZF2Tv)Sw;=Qg!;UheALvS5eq{3}noR;*o=wf2N-`Oa_MpS`Z{miu&T1(%jlVNP<M
zeDU-q#T4P(7n2g+eRtDY$eGkFc;L8*rNnxLbVJF-H>T@Mp5T_J@2kqKlJ;Y{Q7gj+
zea=9Jvmu`+R7HKAx<=%H*45vq)-KvUZ&}U>o5!4Sx@UvzZr${?U$%HlP_4UlTC+#g
z-v8SKQkZ!2>bwi?e=+l$GP78u%lM7!t&@JodzT&hn35hZ<35}F`-Hilj~b}I_Kd%D
zAe>Wuo>o_M^P{!X+#FaYH9fak?$2D=r~H>k)=@jf>WXB@!VSf;T``X?C`!+C*<!u+
zZO!ojX~Ul<^*?p>PTI8lN$Ag0$Aafgn_0eO!n<=G8mfXbmb*XGpIsSyCOYWHnd5uK
zbv>)yxZ=NXeRV(Z;mWp(ea1yMT)Ip(Q&vpc7@;_Qm%3rEB13)G&9LpWuC6khwJJFF
zY>t|2B%|N0fT)C{ksFQYF>U8M-(M)YE=ofy?8es(rdP6`h~2)iZ(CLS+I?I2!hN%k
z-<oCi#Mfk<=Sr{tUw`eoe!$FEKtEbXN$U3I6Jbh@vm*kOE~jk0;J0?m(W_A_9z^yn
znAat<{ZOv?xsSJ>v~mc&ayoT@WjpT=;RA|OyO(~Q>*<qIFh5t?b83ZodD4_EDd`us
zaPB*@dGgv22Q5*jd)HZB&0AvVYa740;N;EXhu#0SXELd>`kvQabN0&@qwmez`yQ+3
z|C#yh{tvTrMRI&|qko9T$sRA_bZI>Fzinr0^0|{Oi_@meo#K1eGIQ3CMbGAO>GvM4
z*4yc7F7t@n|G-5-B?XD=Gv)^zmuX>=Qj{~VzG?5bVnU?*6aG>~*ZB9lI`nR(h|k>n
zM%=gB)q2%p`M)bF?4(xSdHn91(Ut#qm*j~%+<QH5o#x*YRb}DEKUnyK)f+RtV~fhS
z9WQx)Y_+ssLgD2InWB7s)lmPbDXV5iIkKJNn{a8t^4=GoolBiP9fL$;YFu(uC$4+h
zwQgD8L)VbM;o-OEUF|u$H>Fti-M!W37xx}3ebM;;^&|G%-&cw3Sf78aCSE&%>EJG%
zjfT}Z=d^zRx}e0Fak5}-USamF2mNKDn=^86Uc2@uQgiXE>+c0V^;u6@=EEQ_ShV3t
zwc1Y3${WrX-U@V>s_c?Uu531J5H60H^iVX$$>WNu(bLWKH?N9mPFC0c9R06NV5wl9
zk+X){wM3~KNzxpG3!Y@py&|wwg;h|1HJQO=WxLOR$?fJG+Y*%0`I!QWz8+BRSnMmf
zt;k?|qU5z^xo^#Hj}$B4C|>?iibK&@x2$FHmJF`+MvosYI&ASiH|mUo3sxywhOdYV
z7R?j9UUDHlIgq{7;+q0{k>Vm{t7Xc%%NLh!7Pj6XT;`^h;@Xy?u_(o1qG4%8%A5r$
z%SB@Y!vaH=DK|#gG~Tc|8*I~_q0$hpl<%Ha|GXnZE$wP>+Qc0lT^^k`gwwQIRbL!d
zop!wL&G(o(H+#WGdyiIujwzKpgJLI46`gEWwZNnP<5teu%L*fuvWmYt?Nh3*m>RLQ
zta|?ozYSuRA6dJ1mG$h~o_$bUBZi}X;g+7W?gji4drpV<{=ekWdfr?|dO`1{<AQun
zF85yuo;2&bRUW`08u&=O?p25Di%qV45B0LPNZk<Y&Iu9CbrF4>W?Wk4(YVF%=kxjy
z#tHFB4WG>_BaC96PZ5~9MN%tSt?6mL!}BEf6%3r_RV_@?iZ?vXgf$+DO1rHP(8%E5
z`O)WM$fRo}rt><Aw60qiiMQzOEdJzDZ06o#yIjCIWAZ0cV?T+wb|+)In@M(_^_tHL
zf;vjtHkAasPH|aTVjEfFda@*5qRoG~KVQ()loNuX;sN59lB1>vXl^Q9TimDVq*Sv)
zDYa55Yx=Z=D<W}MMDo+i3r?mw6_poMmRE^a^m|PA{5b7dx@hIf^8dUOgOZ=AXoUvt
z2^C+Sp8mx{|I@SJ{3X8pjOtA#YOkARyqD(9KF&4a=)`VIwWtQ^O5yPE#o7&m)kZGP
z`z*3IE~{Q+Su^Em^|^|gqt|Org!f*ln0;Ad_FT<S;RP(yQX}OPbq+;Z9$yss<VECy
zP+dQlsLRK6mw(N9y`t}ly66k>=;!5iyCu85CF093y60Zt`*VFpxB9$~Z4L7UW3Np0
zePt&3$}@jT`vmS{BXwuBG$*d!ub$^5n)nX(tX?=@a3L$Nrn<nkNqwJ-6uu=I?o3n(
zD>hDVwEA2;FJ!WZhp6_lg1{Y<jf)q$-<T*pYj)7n7Tcc-WvgZzEiPdUUbM1t(SNRk
zMe#36{M4rglqaWX2r(~8j$YZ8)nOI;vNV@tT9UgkujmqctM;vj+p{Bctv5~gZA@*>
zNLBmP5r3_ta;IMKuIcgR(;M0adMnbdD0enkc3yAp%y}lkSrFVcbM6AQt_;Tz-sG;X
zEtTbywD~vB%y>03P=TRl>8t}QmiK;*uya{{;zV}SuI%d^YCAkzJ8lVF;Sg+Jwc^~4
zx|<b(w>4H?E}wcdZROF<+F4aXCuf!#ew^c|=xW~Bw|G|HvRi#uzx8biU2b(X_bZ2>
zL9zz#M$X^c@&p<^GM%_O110u&N&H|o{Bu0-jq-#;SrhmdR=oOVIQgTyMy2OZ@yX_%
z#zGDg|MOW*++?BlNHx@4GTw$|@;$9Wfmy|ZT~d5r>r8(x_#(7gHmf99BGdi(6ba5L
zZsk+_R!;HMH2x|z<=v@uuAQy&nJte_EsEn<9K6#%?4{*@mr|y!ZLud69j`4mQe2W}
zvC)++(4ng=ZTd95q9y*fHs+q}$cvn=$i2xcs$8LZQ$gjX@{{Gf6U$r6gAAkv?|O=R
z2Bg0_ys9EhY|?dsNuDuFS5BBAzFhFJ<Qo@_voGdbWP5GfHnU-s*8%sMO_E!;d5VTT
zU2%5CmWbQ)FMOYEB&KuJX--_VU_!Lzsi19cC3Bv7=w5kLyL74U(qG#yRIXZWI`?_d
z+_LJqfvo+W)|)vm&i&8%P&`H`W`)MQNedfR3$A(fEzgeCkaI;|Z>QS#YZA__Ytk33
zIeS&|^G>5QrnUd0R)kw`?TlLC;VpgpWYWi@i7wY$Y?n)U&L~!Y=II~NBGHxPdSaKq
z<U%E>-u_*?|LzRkyS>#tV^LJaqAilMRjc;MXzk&;D8(c_mG$?Y6bpsq7nUYo0U0wE
z>ut0W{Jr50_maEXDJ2s7yw&!_Ft}ao6kX4%@3nAWiiqgk1^eo!Zz{-Ks<OKL!|Q3^
zS1<V*of_~eDt1-pbd4Dav)lz7#XT0NmWg>pT03M}+4mnY`2Jl|d2v^j)BJ^wYq@=v
zUE|m#b7#f;?%4~rtl0B<#X{>t^G~n%FXwegD`tg`kN)}PEV^1cw@+*}{-Y2l-8XZU
zkZ{wso9&8Aq_)41ICB13&c&I1mphkin0oNbrrhrl#sZB7^C~=gD<%Fd7T7y$$F@^3
zQ+^9P>RdDV;*NKbJN^aF=eo&d>M@^lBjblHYb9??bmVMOoUyj`klnAy<4;SEue&&3
z;^w+PR_ni?T4!;2-KN*;T+_XxIpSN-v?^+>jy%!QyJnH_^*#Drd#CL2=d><0mlB?L
zYr~v7i*ru|#4w(6kUCY(Ay&OZDd9`0+9r`VzeV2sKBa2CKWC+D&CQLmD*N+SPH*58
ztPrnY`eUnhRiKVrP{?6*=ky(3F@~u@b`y7Rb`6y%3{wC9vs|q_iTlz~zmszg>Gm8_
zopbiyoP+v0hcB8RKCE;u`L^JZ%sFYhS6&y7a(;8}^vlEZZ`GcUJYV!$<oTrY71?t=
z_nhy0c%EzF_SfI*p0EklO)+Tlj_I0`+w$6ea-qeumjW40A&GYc{#5uWT$XGp_RQGC
zdZc6i+pb0#DM6*Z>u%pUc<t5hYf%d|(pnGxz7%a5r@6MqaPPXCr!H;m-fek##k|$C
zUwbV&qh2sUXV3m_FT1@5cUfQJa!k~XI_X!dAQ-gRco*yS*?Y5IY_MLXIL~H@ci6_T
zMx{Sz&c8XWmwfir7wyv@XJ7lU`&!toYZ<bea`&FTCYg3aSvBsJgvSC_i?3(@e_5nf
zo2Ku2)Sy*xhr8*)Kfwn%86DQ2yv|Z`=*{yRchB62oZWk9(^lW~bFtAk&F=_4ym3=@
z<KZ`Nbk3&f`fk}48!57&>inCC?a?lC8Se^IX<zuSbCb>2vr$`(WvAVNs5O^kJpO$*
zc<r`^Yvb*|<@O56Yfk%!Nhco@vS?&_aM3vPlHuiJJGzZ0{5@8)X7a>klg(robsf8V
zbMEe!UF&~kE#i5e<iUQTPv(SI#*`;#C%3(cTl4YW(H)^Oor_kqoZwYF`P1Yi3-{vS
z=~E+5=13_qJgt5(XOE)LYpdmNgkl~(i2p0PIHo_z&20{Yeti4Gx2x;c-kG-j%tra$
zr(Sn&+NrbYzs6~ilFaFPw>P~KI#c{nz|(-Ou`qp$t!UCgwy(xJd44RbXWTL;@T_Q$
z(fPa1Zq}_6-bw^ra0wC!m7kNlIQnsb-|@YjH=o2jy{Nt|zWdyx9p_G_J@C1*a#8Ao
z(<ir`xvBfjCU>Q+-ilkd{@!_(Si3!s@A<~96}Mk*KPnRw<8-u>+y0R31g>0<uiK7_
zem;6p)F^sSo`mhOnXlI*IQDqgP4xG>H1kO93um@|zRNOu_MF=>*C=+6V{LCy&Rwh6
zOGR%cn_QlJ<L*lh-#xNDdn`R?JHLH(X!WZ%v6r{KTX$0{F4#0vFG}Ix-z#iN48aO-
zu0*#5pWVQ-TTv$ALDIX$!mr=_k2`zS-m86)jHKwh^Y4`0UPtS#f8+YI``U(_@>PHK
zc}2Z@ly~h>-2UDh<&z{jOWz8VyVRFXIuQ6yqRzI9=W(8~9oGrQ<rPggL|#;G+uF;q
zkuiG0%ddX=?sb<UHhtjiHx~X-!&R^$&f?s~^qUg-AE#cvc|LdL!oQI$c>!nFDK7bX
z*okMmtB;8LnP;1NwRGj5PvYIdIeCr1!qo{ulE3z?ZV|lGu<yd_X))~+^CmE=eVrL|
zMs~rP9~RHMjz2ux^1W9egxCK{?Q4sh>vqb;Np`$4v;DgC+!vpWSC?`xhw>l)VKuSC
zCNzBidg)olwwzNOW`5KB`&Dwq>s{y8@8tU~WWe~p%ARrMWGmGJtf~QQjrKoc9Fhan
z-*o)H&sx3VcwWG49bt}(C!@~pP1Jli?felzMYm5)!qWHlZSFa;Y0tFx(vM0iAFYx(
zt#G<LZ|5e?i>D9ToZ%=G=a7-$|F33l*Og~9b9(u*HkK!qo7BJUyivw;PtZ_dzxs!*
z^Zz6)dNqIcE48=l1lNC;Iq<!~UL>{P@Ada=P5b{||Nq_mAfxxbC;KXIUR-`NdG^i7
zbAl<+mZuLLUb^-qpWd_7bGqklZga?q`qBUFiBjsE{lW~23wWmMG4XO)%-F!=)XF2S
zRC6Lg$)#OXGj7g|z{hUA9#&p=F0B$>Ju$~aF-7^K%JC^0lG`dz3jSAf4&z8yin-a9
zxO%d-=_Dm(g%>XKrP7&1Cj>27TgK#Kq%v8>QHU{!MMGc#i}s<lGV8N^6cSFewFHT1
zOeip26V;Y&xvTW_hSd!_Bt@_MJvBiqd~eLdqPM$B-`!ioGgV=VfW(?R+lmEug`N@L
z@MLw<(|L1i#9lw+6jH01cc8KRg;1*3e6zW?T(YjLjod7@A@Nkp&7chb4Y9w=^6qR;
z|Nl+)&-R>K(RIZ$m1dt7;|t!=bMMofW(K`0^NXv_vHt%0=JwWd_VckeGv7X$Kl{1U
zyedgQj;qyIBLo)yX=6QI61C%kU0vg!_wU&^>}U8Yyutn(%LD#{9dd`(F9_nR*&xjL
z-|Hp^BcqLwLz8dCBhQ9K510*I1l_N6^alEIt<nhc`E+5)#3;$7YW&l)mWnavq;!aI
z3QbVxlr=5tad|#zauT<kr;CxN>hzRNo+`~#JPZ%21w7dN%;H89iyoU+;%TEh8p=N6
zhjfxnjZc}ZZMo_bu$kM;_eH~5A*&geSr{L@nPkP6^SHRn?*Eaq=N*2l6m>mWX5eS`
zc$V|Ji;A+XUoO6w)2+Abjh?OkUR_h!4jx}y*3w{q>$@ci!QQT4LqabLXLvWiG1jo}
zRhrZhrk=gmG+gt9fTv2>tqFmPW}EGLslk{vsW4(r6jvC-jHn_pb^hEcPdBkvr!JSd
zQ9h?umi;ftNsnGQDZqh?lW%s0Udck9iSDZBQg!rC-ng5+`MBET@Xa-keZ^0l)|h<#
zzKZPTaIZ5W7rKnjimhAo;D`fvu8CgE;vZYPwDXD-9-Vo*+5Bz;bEL&Y=Xtu@oYyaz
z66h12R+;)<uhw$U`y;{$yJxN3ExGVpNm|D9-d!vFl$uyHJnX`6UhWr;{2VN0zdZ5m
z`pJ58rfuzBHc{-wtJS<qqMVisvdnhkRNwSrBgcdbO;e&gBs~|W6}MI&ndPIBJY`{+
zOlnk6)+2*kxse-}ibm(D%vmlg?A@)``28!_y2BSwy;y(j_9C0SI@LqoAFCPL_)nc+
ziu?1qY2}upE^W)Nn|k#AA7a_O#n9V$VYkq&IkNtykN4~?yOY}=+hF!qB*U~VWqn@K
z<@Df1=Bg~Q_qIJ#ulQG~$#Xa1my_U8Im^de>bLy%n4Vn{^kHuGxl4`ntp5iGn!E1*
z5$f(+`qwYIRU?CGW%E?KUt(KUDE1Vb<@j%qq`hP6Tc=u0Z|z%*Ym$Anqb!R$gQvLa
z&i>#V`N-+g?KzKhvLC8wINv<cF4rM4LuaX`|He)cp$~ei6;u)<f5yb!P@P!5ms|G*
zQ^Ye570a_<4T_syo{D;=<Z{~MnZXOMIoyKEyH5Xot?j_#6Q!Ugb;;*UQuv-5qQXXt
zgudw8&tCLMtSVWo$WxUe*l`Anh1s$HceiXbW8#p^cW|5IEc!g)bRb{xiFVbO5wC(4
z9h_3mqPf|zZ@JBF%Ydp4$|Ca{H8O-Q&KK97w4X`vh5ghiw^kcSZvFe@x#LEs6YA<E
z^8>wBh_FrRXw{k0qPyYj)M?GClY^YNc0bk4my4R{73b15`3y@zlX6U7NXL$|x_Vy6
zdQy$1FFCAkJJImlD(?gjb;g?o3CvMJqBFNhs&$<`XHqih?)yhimRVfnoRhyRSp0p+
zveILHvscf)k>T`trk1J$Bjb*ACWZz6bxaBjzwe|ooI7gc=zCu4l6HpE{}9{9CSNLU
ze~xq6|J?TD)cn+Wr$qiZZN2#8V1DTP+f!=WwwWwy{C`P%+oY=u?k*0gW|Nc!Z%kOP
z)-fT)R=`Cm=K*u%!8J3Lp5!0>)X8?YSw-ZYOQTQWsclgg**4F3-61`*JJrlU>_Xv^
zjybD&4m7pQtceY6`x7)hOtj2$z0<}7?r4dZ4$iLHX&2W<`1l@kh-5RW=<3>au`A?^
z1jD{x!VLASISGHYZq$0II%mI$NV=(Ue4=TYMY+wrB8gDVms`$kn$|nBLNC<#?Vjgp
z{a?DgoxjXk#JhfpYp~SCpxY;R{&L?Mk$kImLfib+rM}x@rBz#%LVUI>C@^_G5N)q%
z3QpDHZ2mBF_T83EIx$|W^Djtw>av6@GxGGQyLp}Ic-AyqcZ-1M|Cz~Z`4exdoNv53
zL-@d}&?{9-FXS9t9Iq7o`jpCUBj22|hdr@xKX@;{^L0B5>q{kTL8d)FOHLkG#_c=x
zLXv;YypKI~SD)MewoGgYia5j|z|bbTyg|i7vGQcG+~+Ckj|99IR<*raw{XYyq|R*;
zL87@=ich$+d0*U7`(kxr8JpAGfQhG$uZZsd+u|em%OPlzmg}bfQ*;if>U-Kr-0EPt
zu`Ohl)r3Z~)-b>IF>7X2EM2azxGqrrlcNeFx0>U{PR_|%f})!~taj^B6j{4VQ1OuZ
z?$F-U$cdfTHV1HO2TzG$zg2bg>H)rWkugI5Uu0ePeE2Lz=2-#LIf<5fHU^G{z5ln^
z8_#X<VP0@~_CJ$0K?atV1wl%VGrFX%o;aqt=2x;@v+2nUXLl>9cC!<K=VaYCyL&Tk
zN^PAJxuG%FIB9<S4GUr2P9;nE%ZIv7%s%p@!ALWj*>U!*>*t~*%c8k>zh7|j=x7ir
z<CK3SW}dg%rK3o3$D}D6npF0@@hoZVRGyP?z3tgn?*n|#r#-nC>1o^|EzJGQr<#*t
zFZ=uhY)a2o>J=8xPTDkM)+v>b=2It^3;)-j&9-J)8P@}!)eqBbb{}cHP?2`<`<ds>
z`ZbAM+xHyO_n)*<{nA5z|Ao&?*+rZ<mukFmELmh<Zn*S-d*tb&D=%JM&Tuzv-MoNN
zw}CUV%=7>Jc_{)Lrz}<3W!35QNonD&DH_5#)BJ7)OqqDVB{KY6$IZD{g_Aedwxoy&
zaxJuuoxjhe)lZk_Ky%3j_su&y6}9sB{oFKjt=qGN8n>FM3TlRzWNv-TzuRH;aPyX3
zMsgOFbGw$P#%3N8GP^RJ&y+#n!*!m?Rmu&i9BqO6N(xJ6C9iUOla%~6;`pUsDHD5j
zF6!^DKKpLo=KeK)n><-++C<)4o=GdqT<dme>Dscb>sPJKS?quNw%7h&A=Lt{EC~*w
zN@^<a%M{#C<>slYPCv6J#{9k+$K54teC}6{>K&MEY5B{zugFJc^`HGtR?XTD50@K@
z8eLE`XtcPoaQ&ZG`Ya*;9sVRa^sW}QmK1I@*ifNlIZ0WzI?3Q{^9J4+%Z8+#(`RpZ
zyIMw2$F%K%ov@2z%ZqKoI*y_)9P&R6_?zW~W;;HYR2JDR*Pdk9@oDFRW)aELe4jcE
zKX~$gUTOH*QMT`v;m^YPii_v}FwpI}F<&`*lS=X??Ilh+HI98Zrp7t$)(Y;NlckU?
zxN366>;y+=uB>T7lcov$T#$2;L(pjRQzPqxnOzA^9ljn7*MpVBC)xjfeYineNybWT
zOVuJxBU!!`i@G$oT+eVhahc=f<Ri+$4xN>Ky)P$v%r-gqTig5f5ueXTYPoFv51U*y
z-kay_+WW$+)5BG~<Y;GxYs4Kpl^g$^Z$z6O4bq9cd^ECJCvMMH>r0DIMM=g_mYkue
zl5D*%iF;qlo0(~6I4;cM&p0C4Qn6&k<|UJMEy-T8BxjD<{||1Nua{iy6v(x942tH~
z43OTI>HI1AcxOZBtQE6+c8GOe=uEiC`I1YfWr4ZTgNYk7ENl*QotV%q|Ixkj=fVpc
z?KMw!H?I~t&%t}SQ+?N^WtTW6yXly8EuMS-;tBJUCwf1h=(Z4E6nRJ}z@=)@q$3VH
z*^75hWjQ$Q%5olI!I`H;<yLhIzuC9dfl0H$QS#3QsV|<d6IaZ$p5k&)&h6vD#ed{Z
zDOpK7?@|eo4|-|1{Ey+(MJn274*ff5wPw$$mCiz%3ICbn1$XNeAJ2HWX^Sz}VvEB`
zKc{YuHs9*Ad5Hs;0F&TE4^E>Ks;0LU+AcURZxWaK)TN@;)8nAzbfu$Ia!=dE>CKH2
z+KGD{t5+ZIS-pl^aMGa_v5R}%FRx~~;IseHo_Zx$U+trQzqKz&n_g}9^bbCGE_m;O
zQyZ?T*j~!^y<Fqk*Kst@*tU&f@5+uQjSEb^63u}U&At~rqigo|-8}kOM)GFBIp;{f
zldnwgHJ^KbfG^>6-}zY`DZ+NW8uKz)1Xl;zJ)bT0)LM7xLzdqf{%bed-M%&J%btB=
zJ_o~I&Hj*m{HM>l$tzB8e6#dx_JZmqyA2-;HvQcFu}fU5aM?fR6Rjct_gv205!9&f
zY%MHqalqu^+`pIRa^3YX4^(H8o;N|-vcs`^bH;|6PX`23PYR_9iK!eEaSfb#WRBpT
zsVp}Hwk5Vk%?O+nuyF1X%NIKxojeU)7*ENVOi_I3xMazS51ob^E&7#n<}c|!rO-TI
zf!k<-B*(<6ZRtAuG^Xs*x$30Kyg=(qkQ%q(j4bAvQIkwcj+?}++8()T@`cW+8s6n8
zmp4vUSaCpXXG_1~)y@STorbQf_8jR9IW4fJLh(>ahl-cXdnFgk$699<=4#FC?Z~vb
zzQN;a#hx8a{hlE!|0po)Zd~mgXsUYQ%*qF=Sr(WCPBBohxVrz*mi=01FQ#1e5pM7f
z{(n(qPUv?N-z&SXUb%er=AF<PPp<Yv9F6l;ZDD9KOkfYQxR&C2%`iabQHjaDhOk?g
zU2lJOeREQI>O<vjBe%0!Dqm*Iyzk8a<$=yT#p@Rr9b2%{O~b`b_vo>-%PQWlj;)w@
z{c7j+tlis!m86d*E-imOp>&S_r)JLIHWQ;S&F-<#`&VLrXro}A%bd5jxR~a0eBlac
zOE~eh%H!e#A$?6o#w{}ne(C>RHTkCGa&yhe6PL~{eq>=Tz##r%hfzZ0)tL_8vaU?=
z*(p?da`s#G___WfTj%WJ;Fim}$aRrLYH4fe0@g5#DCy3I+9L6JTXxP5S>ZS_YQdIk
z?_N4C_-(mV#!6|)e?zt0Q;R=~$ax<MYn1ZU>Ud=lz3lKMO&?9YtEYUvUeXH{;(a-F
z)*}v)PgWaMrtO;CF>97W(2v`de?$+&PTL_f(Xe(}jMU|*9f#vGFPp4V{QX<}khjm*
zsMQX|XS`Z{b}q7UJM41Ccrk~tYhBjfOR~0p(xz8Lk6uyTdad{BrJl1lYJ59)_zIr6
z8k2kWijVKrId^ZchK08D-x3MCGo?c#fjugry(ghnBOr9I(z)ms(_>D4lNN?O^O0QJ
z>o;BV*oO^$Z(EMN*%FksC+MT<{mh<SizKB!{_%TnbAIJa-AY&gSDSV3cm*<cG=wTK
zPg>E*FSzQTtDdUlI<<)tg<p!TulVmRywY0gqWh$RY5!Ypupi^;TzX^6ga)n@KDMjN
z3^fn@oHe(9t}Ne+1OYz>!M4d=&gvXz_<7BGA89hOJZ50MJ#k9OPW~;CT$dx&A4W#W
zUh8oPd{}y3W}UqdXIO$OQ;KcW6J9Z0fxWf6xFlNUg?7s|X&J8g8Ypslp7y?|B|Utv
zR%k5ylBD9ZOO@xguVA!B?IGFA=RTgjR1$mr^W9VWwaIGAJ)ILai7(vLDKOpIPS0RY
z$J|G^?b@cz+>yL3<Q~_d<jNN3^&XsN2?@JJQleM1Z$C5fxTQo3gNbX5%h{_>E_s-A
zUl7sB<a^q<<-*(5p{y(XLqY?*pN7lsjkszM+57*hP|Z`>Wio-6L!<jrlS)%}ec-sz
z>>G0}^7@oedA?_Hf1f_|ous&|TOzUDri{z>j!D!P!6Um|!@J|mCxty+pd7P!O7_>q
zai5j*Y~5U)j-{GBFFZT5bAsEukjL*ruIu!wq*q_hd&5+fs{3t8xPR43jTxKfJ>W=D
z++MG{nayb8R0+K>h23T$s}7tzKG%UeYuAkqpSf2GcJTdE<Np^?uuAw}*IXerb=EzT
zC-4b(L_F+S>oA$+;;f@Lr*4aEWD^W!V$#$Q@|9rKz0hR2?PS!|OtDy=kGrPNUn;X;
zpLj6ys|_0_=*|c%+aM_Yu+z|?>GD+>^|*PnXFIOCw$ov!+`P;G9hLrC>OH&l#$uPe
z^)92h`G$PApL*GC_09g88~!mSeD!LMXU3Y_Qzu!uEsTG;OQrI%m53Ohkl3NOulLP!
zH!$Yh(Y-r;qx&(@hJr}rpc$)rThE-Bc4n1uhu53M=dZd1z4h^Z>=O88OQ5vv1-7Oz
z)wdV7#a=Yd4Heq^Hu~*5?zh*rOh{hGeci^FH{12zotAr7a_?Vh;jr+%pL+J(>AMf^
z#ir~_d>odRT6a|A$JqyWn(j<{sgiN^ii@(?uaFzn*KW?f_hirBnAvM`&fbgX-u57Q
z*JJJbMWrfRpZPt~zF%4vzL-(!sdC4s7Q369p1<6^?{?97o{a*9pG7&koYgrWl%Dzj
zI)+p1w@bF+6v5IZ?(?%UmX&dF-k2n&(0o}Tk;~Vic$P4?s+9H1j5Y)2w8ajUuXyZ}
z4J6e9r-gk~efV<j*$xc}R^J!CVI6`GdYYbJYx;HdV|dKLxZJ{9E&U&w`mZeQo^E~j
zs#w;pg6YMHpXR+esmW%z=H0~wDTZrGH|pNIrQ~x-HPva8++$UiV)Z$1S{3Y!=6765
z+V(r~y2@;))g{I29gd5iyzMNOFl)v%(=7_jVq#ylC*~Qx>CNa|zTvd1m`wGAC(@a%
zoHv+Uc3Yji&Ea~w@4W5w0Nz**?W+MMPkqjtTyV<`yOw%;ZfHVWV3et8v{`9Dejv~6
z+};4yXVd<FEOfXQVISr@p((JV!S8pjocXiZ>tEyeCq8*r8o{$SEY_8G^*i^?c><fC
zW!%l1uyJdipJd!=z5>1_h2O$b|Lo0)Us6)`((Ijl_7S(Ldn(mt0)=n(b#iQv`<T97
zV!uFU`KQ(Xa||-LZYiXfb=BMvTgKOGl)*glZ355RjK1arrN7qq$a2gKHJBCZDY`zf
z!RlRJ`;Q>zCfx}i0~KyPcD<F{+jKnP(V3qgQ%_yr@9}AV*+oV670TDuxi2a<ewwIS
zUbKQ$r1?<R+I>md=Wne&Yq;U>t-1ZD0v2{Gcrbs9?{yES+n2Lm?`(Q(uD){DyiK;-
zG#&L$Z~YRyQvOMjWJ;Fh|J^eTR&6c$b;z$k;O8mh&z({$*^kwB<yPi+FXedl`^m-?
zyl2kXu)lWj^A5dYclV;)-HX?%FWj?>X!D#=_Euy|F0b`>)w<j;(StYQa*v$fuu1aI
zBfDopCEVBN+|6sgC>ee&MU<iJ0@E|Mrt?ew+!f7>PxTFB`4&IbHJ<<8mFCd6+@oi-
zl~1G=J-+fTXWPA}E|MpOtlsdxFTA(rl})}%;`^_8&mXm?FOGlxZRz_DduG+-%~~{Z
z)~&8N_g8i8dK=DhE1u!~<M|irj8(Z9)4!JlPt>aRmUwvk<sm_~yw)cT>suo(h^bj}
z1#IB7C}e-EJ~ie?+xbYfbrVvUI9Qn2ghc);Y*0AV!X<7LGULEPCl=2WE|N#PuCD6d
za!v4xNNTVM0|$eRPDKC%Lz8;=s+<>}RDGtavYxTIxgmvt)l=n7h|$62Q|1RZX-Qf+
zo|>ZRJ~_|9D)sbb&6p)YbF;Q;g{_X>QgriFYftzFrX5orCT-J^+!TLxnXdJ=Gt;)m
zJh>#=aP8flmJpG5sxzj0i|=M)nl7nh@xkFpld!Ok&4rJTPE1fYuhVP{S$$^htqoo`
z)YLK<*WBQdxf{vwym1M8UE15Cug{v6+?b%Ge4xzvQclFyUAEN;tE0YFUC&~VIUuGP
z{fKR<l4IvN7Dgc!PKQlao(c?`qWu|E9Lql3+urZLvR2{$geFblb?%;<Q=J>xFH4u?
z3)?wO)BO5;dw+qw%ff|I4yZ+W9OAO(P}|}vlB}|EuEXuE<_oJu_)->DztL<_Y<tys
zEXYYgYRfza#tTzin5-?urv&NvEfDF^KP92uBYlA5YIpF>8&~^c3KXXH$IU#-s_rZm
zxg^p<D03Qv$E3!T$v#e&sc~}}rinX6G%b^u&Qj_*BQdqa>Bfw*shVmtQ@SGEG~e2`
zxYPzFT;occJ5lMtT&AvN>Q2i)zPyw^ccY=#yd`%$-R7<3yb$f!vn}cLVx}uW7sR-p
zn8<q0=yDfYKD+9c=86Y@EwxrIomCa=<B+lN<f^S3HVHa%nyomwjQQmM_W3Jy?pj^&
zG1k4JJ^#S1DESOqPS+`c9;VZ;1R1D(RSpUFoGrRF_H62t359E?ifzBW>1%FOa_jDC
zG205YXXH*`En6F(ymNI#Vk@iJ`^2WB(vgl6o~{;1T^z@zxtQ~(XTSk&&0A91T~cc<
zq;m;BDLBmf*i(AZj=gFO(lhUxNgbPcHF!=M%f!Z{itC-<Wlt$9G8{R@v?P6Ax!(FO
zpVr=<*&^imzcb}?>wm*7tJ`hA-??|;)mrxwFPC(SHD_%epSgG1Gk!+NWzU?2og#}i
z249d^(<jtec1u=w;T4O?wqGy0a=+c&maG4E(&Zg`OTOG<VHI%c5;9R=+--91$M*Zd
zH~w4fco6*O$KtIHNxLl`d+Wt5nN;j(IqeBcq~(RD28lZ>cQX~Od>ZlP+-Xno<XQJv
z)l-+$MZB83=4F*Sr>a1sgK+Ax=y}V{o;*Lm@-!~seNA1NdS(Y-NxJlsl5dgPdjxr8
zmQ0pQKdm-D>io;wyPiu4)_p&E-S)?ls`C@pvs^V15Q@zfJgG6e=g=yV<Q%1==ag&h
zb^aZBrN8Lg`-xHeZr=%+z+#iG5v;+rX_~6VauZd}lb)wS7Ku*H47?B<6DafaK+?3r
zokydS6t9PGIeKu<4p#ksM%jIPj_8+9onZKBm3VTJ#>`VTZEePD7J92};5>Wen5o^H
z`A57qFL&desKCkmzwfZXrP(*HJTo(V=$pPsV^-9S=D=f$9nC>1S}F}4-9I*7sB95C
z_U5W@z4>l=M}~$L4iOH{5b66r#F*UYcXrf2;}*DmL78E7uTx$4?L%r$)EhTAuIRCQ
z93UmU)67M$$a(3j3pXlSS#%e9xK>R*wCUS5r2~aV1!h$j)mLs>tgCR&-QiKg1UYjR
z)r~=3>!#e%UVLSLV5w(YyixG1f|cGTrHQ_K-yHFjo5a!-@iwshm+ZVrnnzN#f*tCd
zmc3h`sbSN5$ZUCI+UJfP8Xh;Df~Ksxs&iO!a%^O;$W{ecmLn_Az2#Kkik{$7w8df5
z#D?Mshv4Tw7&-)MPi?f@&ZYM2ztV=uDhWZ{rz~HHZP_rVRqwebd#sn5q7@^ffvRTf
zVhz^SO9NN;xE4Blon~8ax-PHxO7zoPVsW=VO%*w{LE%f1`0f{_>&4Yv41arh?2ph=
z(A=<4_H1&<hlE2bnx`%^wRPz|5aT@SipQH3tcqGkXJu@>lhMQDzQtGDE={JFHzkE*
z#*zzf$`yYKyj5KE>HZ~+FB7J$DYZ>^^=uLOX}08|i|~>JckL8bhH0F<nVFYwUSZ<C
zv9Bv-gO|2^>nu0Z$s2mQlWwnWStNFH^F&`K$xS)RoZ1n~U5i}0->kW;7I~7x<>91L
zH(3;dyKh}xwQcK)*o<H`t=k7mH*GW3y3=2l^50@*PHR!`Z`}gP>ni1`=e&+>o#WJX
z+p?UurTA>@yW&JCE77iV1}m!P`b2-<Wt!TT8+4;Zx1fC09mmZxxjA1somegKRc*4#
zo6tp{XZ>99Kxebq)x-C!4okX!$xIYkmvztbCHp+fHjSFiXVm3>XRiKL*fp<2+o0vQ
zaH8Nx<B1N!VQ(v%PChA!SQyQ4TEcEmt3dz3!>VyBirn5L@*PiA;4@vYiEoAuyA`XJ
zz?=te=2p|zYVdiSxVeC%*++aU&(7wDMv9ZXSEPtco8VnLlV@^9mPGWbfHup$n|RxI
z3Y2H5neuohWmJ5>^4MUF@odL@L2kF%PiDUEy!od$W0Tfi?f>7>Qld=w9bX7LxGTAw
zp7~ecJKrURuq#TA6}}Th!j2j5+SOuTSX<)F@MPiNPp5hcj2GWHwK#+E=vLb)Z6_Cf
z{yf*qYr&QoQ<~WPUPsxpP1?FH?#1fqn>n8dEIE22v(UMJ(d&ITuWeg6$E%O)`qZGf
z)9$NmwmqI?yF}xl(6kxe@1}}09lCJk(ynGp;cb_f+>Kx5=fl5KbPmTN^Ro+f?g_dx
z*ZW9MUK&@1shf@5r*BtgH{H$LcX!IYk17rK|9q0RjaD!@P~SDj@9XXdRa*1ywAU=J
z-}Yo4v!aHh;H)H}dW#9lVuq4BdiFxuO-I!BSsZ0uaG|m1&k-h$!WfGlnRYva|BFs#
zg>B9W3FJ)M@+>$bFLc|Zsq)PemMEF7WUtp-H+A0Wq~K5w@qeGDp9=Zj`eo54pZpcU
zwXM(MKc{&J2!0E(|N5-ME5w6;_AZU`-+g9_47IYe7Jpl;uacba$D4mEa86W#XfnT0
z=e)^m0n>9=tj({Tl^(z7`SQYV2R$zwQd=|YdS!TjYE7t0`zf^uBdK27?z?O5NNJj$
z{wKP*X4#)qcbBcF-c7!;>{^pk?B0)I>+fw{SN_#0M^@_Q^(uukH%r%NzCL;T)whBr
zB0*Zs+J97zZrXjqx8tL<(c88N&XR_cr}xZ@zk6}v$+)yvW^Z=97HXVyaK=rOih$F<
zCw<p?VsiDr&AgfO!@u7-TOKs|Rj;nc!uR{ve?G{0ZqL%n(jUG*GI?0s7aw%J!C)%I
z_^iL_ZGX&77AB4<ZLW=u8zYXnCOq<8z|!Q((iPFsD8XhG*wGoo9w$>&^FU(G9I>7U
zP9+acUX(gDG2xWwogT&0OqLT|Z%TxE2%LR7W6OsdY^ebeEv+7%7x=rnC$t8HZ@n?8
zRFqToVe*z66LlqLgh-3r61e(s!WCad=M2X%(KDG1GgoHF8nX&!T#-I_f$ivlBUvAg
z39US~sK%$@!pyt1S0nFUeqb0Bb+F~o&BQ3vur38T$qQXKcJwSMoK<gdNhT^vLF$q$
ztFN49lC#EH$DU~`b0c2;XT0=kqNBX(J=MO0s!@yn6g*<dlW{u2y1MsK*2F~`mu`7q
zxYu+euG@0Ty4*?ce%{Juy{+-}M1K#fR-(7IpuE1C^i&h+&Ait)vnKe4-kcH{{Z&<Y
z+tKdVG4p5M>G@tGzT{1(h~ng(qJi%g-26C0+$lvlW5M)80;jGsy4P%!H@|q@;mG`k
zr#yz=ZlAm}vtr9)o4(*@ijE?0T9}nWIMw2gMk?6HDLDLF>=5-p>mHl1T#BIDlan{2
zx@RVE@wGT_i+p=i&ZX~$)3$^nlL@CoMK(u>Y;iftduz^?k||qX%-Ooe)uZ*wg`Y8=
zkr%usbqmkrzS!v#CnH(>)AiET374;S`}~ibFB<jG_vIAtoP#xuil#;`B3m5~l*|=$
zJaa%`D!*lD;KZAYG(?3r&UJcoZ1&oli?%fDO0@g)wiK#Fo%(V44v#`5&$NpS-3|)Z
z;u+&!7`8lK%M*LaC#o?l{i>o;Sx3y%RH=(+Z-h)s*fcfc>DiSF@2%Q6FDWfXT`MiQ
zEJ^<EF%5+yYkbobViz%_EK1IM_Nr9v_1rsad{wmUQn-v%)-Ju1sdn_<hbWzGY8r3F
z9=*A?f%X1}t7!&m@=h|-i(dA9zWd}^&$%y=vuCBme_Px4>IC0LmjhCsj^FpnhpJ7r
zvYq^XPqfn%@oL}6SsLdTC9oZw!=^2B+050;v1Pj0pZhldnG{yO@v*t|xN_6-$Zs>-
zUM4%~vL}C2X1$fp8nmD;O(vsd8N(0t2CXF#0Z*K?o@C5YTRLNbtCLD&+k&^f54ehM
zIG&B*5j)p=@kVc}3SX2!zUXYWg((~>RRq7?5#W2V^Ul$UOIHXTQYn6_5_iTmQiE+;
zi1f6;#DHg0<{P%$+j20B^D2L+YS7V^#R5Ik_qN<*^mm-FFr>4`JHuz*8aG=P{|KQL
z5fujUhcjCq%1eAqJ*1j6?c15!HOC)3ja0lT&r}ulZ>h@;&I7ZYSPst(jNc_XXTq(n
zJ@YiTNvUp|rEa9A=Cn+$D9O|}&5=XPQQ(!6KvqiHvtx;Bi%*@ex6(@cU#I5sE8VeW
znfz3>Wqk`Y7KyDrDyH*nUhYGWhZmIcrwXzExHh4HVUz4hEz$Uovhxj|sclh|U%6w2
z&s5KXX??Ta2yI(?wVG$L!O#982A;H-#dD{~Z<;j!TgGFPMD<c8%?W9bEg#K(J2SmH
zPJ7#-%z2V7%y~14X3RLsn!?7HVn0pW!74CZmu0DtOM}^wHLCX<d>+*QQr>%v)&5-e
z&PS^=n%bI*&N^vngm|oQ*(<{w8Nw`jj`!Fyk9#tnWf}Zstmi+<OlFbwuDXyLku!Vc
z3Lmw$lE~TIAD8%tEfiTbS!71=i5AI>64_f*kA3f*DU=%g%kWI7UZ&HI*<qWQ7OC)O
z{@>CRo_BS^z3|fu+D}!ibn3}$*zh>gY!Rc^jG5bbFJ5RrywhdJT9(wQQ(wO1O<FZC
z>DkNIPgkZpWpMQ^P@Q!;h3(aiS-Q9S)}}1ecAk=(wr}n2V{6s4vQn#_Ew);g*7T_U
zoo?nY?WGA?S$gg{nL;aG#c2qhdl<mF@#?Y<u6<MPJlQOJ-Qbv3-lJuoeyTl*3b@qC
zm%y>SOmMmWUcaTidX8sa%H=X@mIxO%%>B`Oy(CY$L`k)zYw_-joYxtrT3!<0y6uoy
zME15%w`JE|W^$S%vP0M@QKpXXNo`m5QOgvKO>cAeXxAUp-YvH3D<8|ExO``qgfC^H
zdp<p{)6%b3d!O-town<xxKm%VfBnlo#kywGmOBUS0$NYyvs_i+3+v{awdB&rJM6t8
z0$hp=@j1J$ED=(Q6plMB$aU<J<_502+@jZ31e|K|KGJ`u@M>nk)aj`kcJ0i2zeUeU
zNAxUL=z)s4VU0dZr?KBp)VzPNU3}+RZw8K8Q7fn3JMMYmti;5ZcAB|&ZD!rL<KDGp
za^^uPg|{!&#0r!7HqNVCc7toxy=|)(c|8*<zxwl`o9WeKDN!3m9(P_mxXxQ*)z_$J
zx38_0xU4Igo}{*`WpR*xMxK`R`j1X~m1K4=dwA$XKg;Zu^MY0}O@I99QSPo+PonfP
z-Yk12l6?7L)(jox^<A9|TTUwRxmf*QI{98ne1+&lp_S81*NK~-IySH3L!g)X;Q*r`
z)2o>>nV)@Do0ncKS9ts0?vB`#7h3NpI*7h!UKUX|=@84UqPlC@4nj{pn`t!lJ@amP
z_T`hV;cxYYQlG!A+L$Zj<@0X&&PlKJZoh6|GG3{nabRCWPp-<Lyv@Bg9{er3;t|#D
z?X+fFk8#nL{0I9|E(*r_im@>Mj^TE^zk5T#j4Q={tg8E6H-$?*-Q0AyQdTu9=0owW
zH-#~>&Jz@O?P3YbWz#e{v&g1PT8Le<qP={}nFp-vf7Y;9bDVa)%n~v2xMS_?|28*f
zZ1PEb=$}|9Cc_sjG(qfImeahxx3|ug99^>3cmCR2|Jw>Bi_c1Im)gB9Zr7x>B9FiL
zHb1|W^~vm6pSZfqw^O+{&*jS7eht2!8=|kJvFzhrrxi<@mWBCeohsDoTa)(il+wc+
z9q*^AJYH71(NcU3*C%g=BBu?eUrO`VajEOozAKx`G+SZ0lfi<kQ*xw^UAOoqt^VKm
zjoU<vOP+h5a?Y)}U(UqtQhvGOktfrd=i0);e>Vg>E;;TXs?U<NNL7kUTcGRHgl)Q$
z4{7jlWw|HZN>aLAm;dA0rY~HZg{?n&r|5sn($CA+&rMg${$G~EZoXG<{T{Oy_kR_=
zUHI&0SHf?<3^VBq-LBjJu)Q%)XZ3uRztL1YYu%LHOXfu6l>Zl!75;C=CM;wiG%w(V
zNCA_#<Q@s--dh)(b`<fg<&Zpa_wbQ5_UxKI1q%<TOblEU>)igq`%3JDcg81J&bY*{
zJhR8ZX-nqIO`0u-PtMwwdD_R~>bVaOG^OsnZJ&2><-HYQzTs!ibXjy>E)ngvaC%b~
zeE99{XWyJp<)*&NUA*nJ0rx7^tdFAiEyV7h6y2ZuTCcR5{b$$i+@AiAJ<|)NlDF<w
zbD329YU!_wMFM)+eP4SIbo5Pmr5Wg_H>))~=clKhpMl=(DFWha_I+LR@1(A)V)Bzq
zvu?D;9rx3m5&G>!WuMrUT!jkb*iXVwf4)vip08{=|3BX>!}E)JjFN?;bno2O)aN?+
z|HAxtC%(^**yeO2`G~4}a$DI(v$AE{{KB=%m!EHQU1FBGJAA>SZCTT;R<)O{O3v4P
zt-jA|!9KC?Ci3Q~Vd{B{&ANrX?gXjjaTNP(sxsy;H{t)i_nvmq_kP!1>knvo{}Iy(
zDfw)szs4gfAiuNNbDr8Z_PmPlHy2co?1;Fq<B5vT&KKO8ktf{lRzEMkcE@tdBAwko
zH(%FH4V5x!<S#oElj@{@#OZa;!q0EQcg1fx7tdY3B_`LbWwF0!!_`Z2f7)2qu7B{j
z?d40KdUeS@w&mI@Crz{HGR{1vR&v}qU;Onq@#j`2cdzYMFP`_9t#YM}8viP%6+ceA
zuL{v#6>@ox+JC22&T5r6qO>xb-iGGuhHt(m#*-%#Z291cSdN-)RPoLz?(3UGTk@vu
zyz-heM&BjY{r)pKBjddAHw#34jx5($mgoGTxGpo<oa^F(6@N@cx2R9wRIx~T@%718
zrn)s@RewU}EX>$-FMrpv`seq4?=3sFPDIh{Bgd-@@qcq=_N|y*wP0S=$1UG(G<~ac
z%dZPtFLnLb$-_0zj<@XmQf3;eR&aXT2KI8p_p4vZ|88tcc;J<B+VOjd@r2+4$KHoC
zwuB!o%{G7cenv)9dFQt}hS?duS*HbsBqypyK0l@rx5a#_ZGlDXB(>9bMK`)#vpw^f
zW9p&_d-w2vF%0zy6LNXG?U3aEZEojR>3#4xpt6;{_6U#flJ-N3r~cCSy|;1Y)OQmT
zy&PWs)f6`E51SX6c7e@dOVZJ{DGp6*Jp@xsCtPu2<d>f$G+~3HOOr6?B*6)V%9_10
zH#sg?1TAryC}3*mQyFw%#S}gHLphv<SI*2b&)p^?Jo%YlhhX@!FC0@|9GP#@ul96j
ziQ4(V<bu3QnqM`q8isB3yQ=kI)uqLD`<?deYJIJ?Df*1pO{3JNDV#ZXH%<LL?On`n
z=_f@ety%S_>&h%Ec*qoC(5~Cc&o8E9u`!;pTb^6&fTvNa-em3YwiBM3&)esz=Zn3a
zS|Z-DEO36<-wi(Mm)FLZYkl4j!o<j-aBJJ#|1J%=40{#TCMvjgPg(pn=WmMZttS_s
zF6sB0A}5<vy#B1`R>2Q4i@t?1?Q2i}YWu{waa+_v_WuTzt2H<o4z%0ZYfiW=ILV_k
zsOGeo(QFUR6$)WlnMV&q9rG4g$jX{>;o$sb-4`@CU*6_h^wi+ai9<ZjhYCc(+!7xj
z7I0T=brE(onb;!~dn@Ircy!Ihqmutu2d<K2Juq?St5t_rF6*obd3Ahh-y}~Jsfj9w
z?Zu~uY&xd4@Jyg*$BC*Hr?@u$$?TrHE#S1S=CU6}T>B1)Og^*qLsa*M12?XmK5O=H
z&zZBw<*ZK|YOg%=S%>k&qeGtdPgR%$964o80xVy$d@*#C*KYpbttTRS;Ns1o*=K`1
z4fkHzqGPG5+x4)gTH{Llt;qL^?>4wxJv4uxasJ-0cvI7BVOwtZDfD%OXsIyYmRNQ5
z%d3^H42fs6UAYv59Gn<Z-+Fu9oImBMZYPJGs-N(d6AMlp$i1Dp^mhK<r)AO1$Cbks
z8qMyk>*X<wDYp(&k;+)usPpPYSl7F=;k&x6PL(a1*mi2#l16`}A4{iC;0k&;r{K_~
zT|et|f|8agg-K|&>F;})w0d5Z*P8iDtTNR%tkasgdXt-5rSRnAMyGXl+*;+GwmWTI
zCD(xuOg$U+^Q|u4IKPoED&vHJ?yPh39h`SvUh4mN3Af*=f~bZo3ckHteT=8B{V%*e
zr<%ijef^>J7B;=o%_51pfu>4wGxMsChfZy}y5~8Qj@aHOa!O$#Kc0N%;{5aF@%jCM
zVk#33e2LNw_;`drUgjY4Y1a#fD!A9zIGk2@(kOmu!k)ittD||A_2eDewy^@XF@7%m
zs$PZ4Vu@=EkKIgN|ItITI7BOD`|V|SUGL}e*qwa%|McI5cgw%OI`#DPk>c~ZZ{Msx
zV{rIQk<aVp2kMv&Hb4J&_Wk3W&xRl0@0oo5+4=H$=j@h$nOysgKi8!0yZBz8px?5o
zy#L+b*Y|RJ|1aO~efC)v>yx?RN(!t2vaU@3JY!b&1+AVx#VfC{+Cy4v#*_~ZEMZ%k
z0@kyLZTR`WLR;llVYqdc;F`pfx!j%}4cyB`Sv3XRxuXj_ollACF_*q>WPg;XTPNWn
zFVzt$^W;JKsjad~)3RgtWQn)kU~ttq_F9Q2vcqXjhkk#6O0?udC2?;i1H-nB2}XvG
z4{EfDEOTJDc1pZ`WK+n-meig;sml>zs}@bQdG@h_pOO3W1ovqhzo_e$bb6o3n08|3
z!>77cFHeifdCoY<;%4ahxM7aU;u$Amjv4Wv?D=c-dFI(0{3cU=a;iU1-ap^wt>Wxi
zv(jTEx31osmNlb`S37&MbnGiPhHnp5UD_@$T^w?{jomBeMcY#?j?Q;ZOCFwkKBZ~y
zk`;cMLU$4>4;=2=t+?*LNBGIGhceSG3KuPRcp{Q+d4A?OrkP&(Z*^pM2`zD9J^ML^
zSIWQn!^s6_-%fTqqayqA($$66SH5#*mVMC~!(`pMio@ZE^Q0+xDSIBdswd^=oI3o#
zLjIwprl;>B!71}pRG&T6)C{*$n0hVARx@XPRceyv%wtoQy*VpVW2zcIyK0%&=ZnuP
zpTE}p^GZBqG4IsCjzhPWu-w?cn602;hE?XYHsw`I&IE9$2{W*|&$z0T7ckTP<AOC2
z7qw$TJHnL$1&W$loj9Ca);g4|j@rw5q@dP&-GeM`F6}LT%1zJ1A1f`2XROv!IrO>Y
z-K^=mo|S5A8@a^y-d!Z_EOE1;^Fo5s|3*$viD|;iGq!1-iBU6Z6}hFLnwltPyw>kc
zM^uMO{?<C1$e<vzO;gwUr0N)R&)+7nS^s9w_O2=2=WZ<7yg#QX<EUw#nyRp5$BIuY
zmuRMHPiBjKvFKQ4K=UkJ)umgnZDI4~oN&N@pIXBo0e6Nd4Skm{#)tKOzAng&=6`B%
z>(2I#=T=|t>G0$!*}hILb@q!d`j7r~oBwliv7K9#esA7qi^HwYi@0{(X@0%Z<(zMY
z(?6T5_WCuecukiqSaI90Gw%AXmSq|)`+xI0r%p+_FX_K8&->%sq$d}8`TCEZ2=4N{
zxO}Pf>yzq#U&eW_KW_Ub(_c3ys@pgB)ycywYRf~l?*I3=ES~!8>y)}syZQ5NDxZEc
zd{y`B^Zfq|9M6`-ooZmoDGXZlYN`f@L!<E?u{!w-m$Js?VOxTtZ&!U?ue&OxWi5v%
z=k(^V=p|n@zMhyZ&>tC}$~19pd)b!_UalFu@4l~hd7>SEc(bl5-?ejoE^8c@WN2&k
z{pzv^=@8rfBO=s&S`Slpsn3NQLM=-_o>};Gip<;~ciT&X{joQb!%r=;-p;y7=kLq@
zJi(jMsh?Bzt-Di_c$y_LxpOD?8=m<!<&9^6k@@Kvf~s2k-@d&tCC${mGgVh}>fPwH
z=?y#ooZa33NczE_W!VQL^QWflcs8Fy-TrrAcfHRei}g02bH(`PmTvifd6!ONu)+%+
zgXNQi3mHA;JQCi%qrq_A^%Sn|$I|lGnl@MJRox4I^3|l$?qg&kr=zh`@x0Hkcy?K}
zOsa6zipuCo=LwuEfAX4XUT*87)sy<{I$!u`S-CGivP9<kEp6L(le{9cmQDAYyDZeb
zXpYd-(z#ms$5p<*y#Bpprd?|78yD7)f77@Qcp83Pk!x;z`J&CLC2K0r9_nPOYZLYn
z4_tC_$*T~ygIcdPs<F-%|B_I-%{0Spve$tNkv6v`{haKmee?qV$~_Ev_Kr&vL@zgf
ztw|`B@SmQ4Vng7T(4?&|B5IdLZCa~;V|DzODvMKvTjusgTo87f)s<GZdHGw3V^(3C
z{y(T*v$@tL@>I-g;akVdQeqe8_RZ0c-cq7@b6;#(?1xF4XB$RNa>zbY*eo{FF7;DJ
z!F+z5`O$Y@hJDX`yZzl|9aht|j_P|nIrA&t_r~3O5xwXBREGT@`|dNj>#oWwc*QnT
zMg5ZmyUn%E*<uq+1d=RwXw7a(RXwL{yGiBojphRu1<bY=9!*;|SD`6_<H=)-JR7@p
z4~1L~>KuRK(f5R<Wxk0zn<h(~QBH~8nZqRlM`GHKxTLt&2~@OPDC<&R<Z$dyU6O}W
z+OZh{$K+1^xWjgQPu{5mehj<Zm$vzw*z%$3*pU;5*iUX+cl%WT&IN0DF1m9aFW5P4
z^@>~UZ|_W~zsC1}<r9P88|Tiv5#W?pI;+t1{~N#kk=Lysm<19w1m5~FtxC1%YPLV~
zC@LoHSJ*PGBz7TpM%}1mg0dS^Vvd|QS<*S-upHZ2u?fzaYWa_Nc-OA;Vs`$>Y#Q)b
zLg3<)RqbUSF$~&I&MdvmroBe!$d@Nizqt5b@p>yB__liwhl*Nu56iQ*B>!s(z1O<@
zd!H%q(>kVJc3wN9|9aWmzOuE~tKKHJ^h!^C)F1b{V$~6ynl-=Ho&U9M{lpn(^;J5V
zl@7};=apk;zV++LEs4iGo6o*}(L8yQx3P<+rcJV$p7XRhUO`Kq-O0$j8@2de-DAa9
zYws~VFl%@md4${IMo!y{bxM`Z{~KSuRa~aZ_(7S;;ZmzxkP4@v%5qiCMGg1EKK)<f
zz>s)JHTaTB`X}Z8@71<Hu+|BfeMwo=l;7&wEA{@z4Gj7h&ECxrP|@Ud*NiN9;>CB!
z=D1@<0b6LEQ_cp(f1V30jyNUwJ*)ZfEQ#Hrpzm105}n$<=aUW{yKqTzZ_<l9MimZT
zFQ@paEa2hyKIOe&ozE?$WxE2Z*SzE2n7888`zaPnmoF1gTE-y1*~ft2HQ>|pdIe)X
z<;{F|`IPRxYcMOad*a9N;vD~SR|X{p3%O0^2`^fGHk$-~HVHL0jSM!Om|&V{%(+cr
z&6%$CR)r#le3l9tk6tbOsKcLM6)fU;(fsqV|L@J#g<N^HN<!E2tyjC<drRm34Q~A6
z{_0&`Z{r@gehtd+j_S9LU%!bX?ORd*w_}^1FPr%N)Ma<0V`?RDzu(zx>v>H2py9R!
zAKX0Bwk^otcFaj;_XU}Y7t=QN-`tY#`=llP#zzAl5l8OF>HG9~`kQVqV0Q9szP-qM
z%9Zz1)tVLVuK%0G`EQN(zXNJ6x<xK~nap45^MB`e{Vbs5^jRZL*d*S>#NC)FPM9%6
zL?B*RCB8VS=!;vW31iL{H(`cIfdr4nDfwMrOqVh!>If9d>1j3z@h>*<b}6v-`@pjQ
zabZxwysU)>z231#f0&n2@HFoNTcH5EwT`c=&yj`+t~U<+Kk^{tXvYQDYm>I#5_&hw
zz+YCYyyt_CCo?bO*4H(sZhR=J+V<giT9(h3M<*CQS06i`airii%T*r!%dbOTdTrAS
z`1OSMDf=0v;xjKctUPr_Pu9p-{?R*!%dL-mr)TAw$Of6nHz_TAs;+WCIiX27`=D@j
zv6rW^1$XZ!g-4%^^6jp1t)C)n^MFfi?&S7qp&RB*?VRMnG&@6lWvKY-7g4^v+V5Mx
zwiI5CS}W13a_!6s$wJ+}s-~nRhJ9;3+FGhf-<{gu@K)oEh>UgXjSG*K{`o8;*?IGd
z;Up$!h6jZ^xEA|8*_iOU@Xef@+n+Ahh=lE&v?$R!*(Bnm0=tpIMc!#T!vF7@d{ckY
zo7}IY^paOeDo9EGkX=^@!$1DB5nHmIg5Bn_DrIeP4gabbb)|LQ(NOPgP6f)GENUrw
z0wSH0?Yg$UKewj&TNmTm=EoC?G-rPEo;z*c9+9VCI@wlF^7)_ng=Lkt`xEWlb=lD@
zN55^+QD4QOdde@7N7dw2#RNUQho4@YaMD}8BxrF@Na5?_M^cvVNY|gKyY%0uR|hVv
zV4J-B%Mz}yg?0v^3@aR73oTtC!y2UWxlnWSO1;T>k9pgUoKCIpEb=@YGBrtA^}vy$
z7iQJQpTi18Ro_3d5?~4ZuKuO1wEy$kgz4*MOpTm>`ST*-PCxZ>e($(N9vim=TkpJj
zwPpK%iT%OfPJFJ&?T<M<SyI|9$&1_WEr-@Ck1Eq=Kh~7n^{@Z=rTFKe!mU1P_TN9>
zD0@BO-{tL0A_n)EcIYW5+?>8c?D6k=-_uv|7F?VVta3$L!^D|+)2>BryOzD4`hefr
zby`a1HEp+Pp_7wR9y6;r25b6Ea<RRpm@1~^H?2YYMypSmhw7&W#YK?<7My!Bg^xVW
z`@;6&mK)1v6Ys7|O-lK#lP<Ar4>_1p!Maz(N72XUXo;4lkHfl4KD7}_m!|Biy5M`)
z#IJpVU$?v3nQJ-cb^L9;{L9bz`)dX8f36H#<rC6XB``H#{3`eQb6n3|432*A7Y#n8
z7RswE+8esaU%~Z%-4pk>cBOnrb3zs_R-SpRZ|Wfv=^vqY-NH5>>0fv_MEOAKF^9NE
zStX8vqNe7h>$}!=9F0i)D5_dnmRcB{SAKaGNBP^HbMrdHB@c;<zrM0<YjpGUivGZg
z>4B1yr(e4?wen_A<@F4khnr*Td8O-prR(3RzxXO`c{J|0dHm_}_^*$2TPzJ<r!aGT
zlbvcQ_cHLcjPsjMhP=lkdf$s1Yivo1JUqo@OJ<a2@_M!6%ENumV&)IthL$JaOPJPF
z?QEV?(tmencAQB6!PUWUoExV$H6L^{>s?#$t}R@AW~-=gn3YE+=Pj0*+cf82%gm0@
zT0Tu{?KUlUckP{%w6{;|*t_ljvtyG)4sGK&S;qO+B+v8k;>k*?N2XQGyYb?|w_L|v
zdehexMEg{QWa*!6$xrRdXKW6f>{F<^W!B~;b^INM!mLH|;vueE75>jOG5Ws3Hb}Yo
zv4+v-`)QwJ0zAT-zcKqy&%4dJK56^9xWaXRy2`S}H@Nth3%-uEnz?b)i}L1;)`5~X
zkxHDGCvVPLAM<k7X0z4buNiN>VqA6K_{TbPnThY$&+NBb{Jvt!`mK3Sbn0f-oG+aC
z<WUXdG`}0+j^-;3Uwo`x?d$md;!^FF<;vgWQ(ro>Rv4`~ot4$u=r<+%(+m?o5p#y0
z9*&!ulD}j}nJ|8sVQw=c_`}ZW95a&V`*Hu6`G5Ae*BKA<m||0z%ui$-U1=dXp>UlE
zYjOqa(%Y<uLy8M3*f!4gaeUrhY|vq!;~UTJ7VxH{kezFKTj%s+ZVF!h#eTZhtzG?0
z`wGu=9dzMr$XT{-2DfBr;6s+>C*5Z;m(SwezWnqI1M$sMpFA;akK&uT>R@+O(Zr~g
z9|a~@x0)|;V7|g6&R;2F_<ohGp_lDPMJE~WsB)I6i%Pdz9lfhHDR$zd<cpsTc%!DD
zxV&QP7bo2*n@z1fD~}f*o#N^-_4w7PucczIub5i8UOngL^(i(d3oUQ_II*=NeY$hi
zR*q5&soOPAlXQQ5jlW%<u(!rw%1r+^7v<LXoT;rcQ2m;r$NB$&Z|0pny$ZjtD=hUs
z=f8WJl~~i+%B_b_&wG2L|D@&I$yet1UzslU(>!Lj{j?H?Uo$gQV^*{L%9v<$Y5r`@
zxtlJD&RQ7UmnFaJ=?0-}`!m`Hh02Z;El%?Bx>Clu-et+<q9s=+dAL|Dy=!(Yc-PYB
zVTX3Bzsjgumg2Q+*Ny!D)@M~^c!j?!3Yiy52Cq;T?t8vrh4kK=*4tMuWxb^x6!K%S
z!Zy`#`@)F1caQM*K7OZF_W0D%xyEabR&}1370Z+DyheEK)vB*MKHrTzyS6c=(p!7;
z1oLUFckX@NtG0q^>$2@y9MwN}nEwn|_JIBOPnLVt&1WCvd!LrvT~qB{oA0)%w(Wm{
zpvf(X!}=z2j?cf(oUHBNBYEJm%#GV=NymGg1LtLy-TwD%@x6aGjk22*f4kXrg*<b=
z*6e(3p0dPw)l1Wr7|ewhSfpv}$X(dt$-LU){KdUaJLPS6rnk<Ga$Rqgcgee;E%d`H
z&Xk4u7Q0n*d=zKxc4v3C@agFMq1>~h?3|q6e6GD4pSazaty6q+?|GYj=Ogk~ZdktV
zLqPi)?ypVz|K84DY55^D_V_e5UiZ44A~pxVA1{=xIis<8_Vl*m?RRsHHh-F3wbF3E
z((N2>DVA2Z?GxF37~Jm2O-VXZ?{s_ZtlQB)KDL#$PT9ORqNXEmPvmh{R#xv$7KO>W
zOD?b2{C(}z|5plPj<*+HeQ@`<Hunj^W%p)JU$=DQ&*VcVPK&Pls8OExGj{47ts{$m
z?h&+lWIVk;Kt|loJ?YoOucoKJFVbgWn0eE1=KehiPmi4W%ra}T(i3Mnr8|KiuJ5_+
z(dl<<&t!%sVR6P-3r>X$frkh6SXifcp7@Z^)Xc;Xu%+U{BbQF5296mso+Kab7O~A!
z@eEwz-ls0Oq~_(Pr42Jg)RWd&8mkL+>BYXA^CQ?nNHb`gN9NI0qAR1;=3L#ZYP_nU
zENJzz98uBLK5O+iR*8O?y81>G(~L=`(c7*@hGl1O^yLy?d1pt-)KjXd;_L72Vfymz
zuQyjgd|0EWpq)=wFGE<jyW#(|#}y(eL6h7A)sibko}Hd;G5O{fOOqGAOm5TdYA2Mu
zYEId9BImD_)|;yfVz2AnwfZ7<cUR$Uu?JCUAEqA;7TexsW9hp1Ot;m1qtwGwbT6;=
zUVd(A)wj37JMzU10@5m;J~^$v+^KLuwqELvl0<ufy5B!P$!;p|sIRN*N<H)YKYv2Q
zsa6%?Mt#B15bd(9!KZ`$IVGmr3Y>De=9oV9s6yLTgA)(iq_-Vf*rd?qkz}nhuVYcT
zO4HVe?w-9nBBb><eNvJ(*>uP)$wKW|a-ZEd4=zbZKFySgj$8pMle{KveB!t`;7iJs
z04{~psli?^pG=9&^ISSDHj4Av^w>#erzHNTPV;;wF}p{xVOGw*o6lwy?%TO+&isv)
z6J{4(TB+t*9dspqcEhS|N5zCa3=fKQ>aEIH++%lZMad*Lb5{1LW$Fy<4{{g2beLOo
z@Ss>skC&GC>qho@D-W!lm02j$V=hoNhmZYr3%hE`1qW8cE!s}Iiw<e8v-vCT=w))!
z<+981N##MI=7lV8z14lYgSR-ywmb+dFiiDsO}!Z<8vdlyH8A4orWK;W2`hiUZOz*n
z8W!|X>FKqYZ)&aL`^xWLogVa8?AiMG*1uaL5?JM3Bo4J%sxhSWXNE;4<TS7urY+2L
z`*>7bdHF4G+2vmhkJtUuyp_JAH|<m2l2RL)(}A+(0%!iO5!!y|wDED9;)Rzz4UZOE
zJG;wXuqj_2d-0y2to*8-%hKnZeX!4j)iv;R%e_k<x3bG!`*d~r{5tlp-%acN`+du9
zp72z34E&k2bzyKoq{qeE-}dNEZ3(-n>tK7b^0axV#Iv`FQF5z}Dm>(>%h=JRrlt4j
zk<uw`@i0%%bqji?UhnvsWR@j&qtE2o56cObT$Wb-ZbmaxUWpibs=l68$f-8Z-%8VZ
z%BK5#)o*uR_Is9@(dU^ue?Px|&4R{0nWD;yQ&%=-9XEe3Rod?4wRiC~^<9k&ooU-^
zOJ<jy%3L+SjceYrr|aGKRe7z?zj|ewmuzUwGyUGx4%_*@2>j!;6)|)EpYZs(eV0Ur
zl38b0V^8a3CM|Cj$4D{nmPrRD%CxARU*y5;x=3)M&V&XPMowl{50S+a7~>NPZ#({J
zlPT==ZDmhX+;#5jRIxA*-KNzKcR%{NF869``}a-P<LXXKZ{!R#aI*0^DDrtjdtjQ8
zz;2&3oi@YsYtE!BGd!C$ValG_8`Ziu{p`(62$sHgC#`4m$(ffk(#$@IoU%Nd$yc&2
zSoT#!mo<~aEZO~=x0p=Q%?fnpubOm>)o4e0_LrBp>i$ii^j|Tkzb9u&eU)0HZ^$}L
zzEAx3f{Hx5Qi_~JFGaW8y<EsAsprz;JyFx&(o*bT##Ftci9IIe%JOetxNngOdg=Gy
zeTnkIb)LJA_)Om<vMeF)<@6(SVh#7E_-KFi^V;hjJmrjt&2!FE724LmbMH)%vU-|1
z>+I#UIooEcXZ|`Fa%uNG%hlnwFIBvP=bpOqJt$Qxs52{Ub=SRBSK2q*WPMb9#ys`+
zBxAYNZ}=m)9v*S#Dhg;Wd!(rA$m!E|^#oU%i;|?tdEdNE6O$CvRRt9SZn@0R&%634
zH)3Je_LvtZ*6q0v6Bjr=DvB$3$Ifou&$F&a-`I9IYK^X5_Uo5Pe_Wj!S$Pa58A}`#
zI%W_Su`JO$^--Eu^2Q!d(^dPnC}r80ZPJc2?LK>Ym8rj)RCe3e)jwSpOST4Y-o%%_
zHr++L;)aEZ&SDPUj0=iVC7YV(S}XB}GWxEpTo&fu6_Q)~F4DjHkNCWGcbD6DiA3LE
z5fETd{K>*9!obX+!@$76(7@!t$iVTRfqjRRLqURL6LXZJiAKS~!_6Fm(<B4579C+s
zF_?7HBa30BnV54|kHx7Z&!)vjN^c}4P0_91rnBjiN!wk9*~}4&S3D=AA1UQ%6k^$x
zcXbD&Yd>E^X;Fhli$AAZX;(?g)rF3(TzgNIKDoB`)r56I<_>pPEldv=jrA(iJhWLb
zRM13vlg7>U%<tG_PhZWuzb9$qD_8A2(b#<~M{RX~mSiQb3R>8eW3@Bu^6C7oX-7}}
ze6=dnrgoe2jXqzOX@a$VCbAbxw8PdUeowJIu=UNHZN=(S)&`Wnzq^f7OXM6^%IqLv
z^+^w>6#iuJW8UGzV_AFPh~Bk{#ahjC6h3v`*DU(K+~eNvtDj#uH~IMAef5h`L0hWb
zziwaUx98Q9lU%#b{`>d;=X1gADI1S-YRXRC!YX}OFy(Z?;tr?N48ma^imk6NOj2yy
z(!eH?;XYO55re8|M|{<c){?O3nY9sDBlV8G5Q{9DoAJ2U;#o(0ly~Q*uKvJ-qBHw6
zCS|C2aTrcpGEsRN%cGz*Q(g#bFXm0~p1QL4(!|U^l}Ccze?L_TY8IT>Ibr6{-j&Px
zPES<4J3G}Q{qCHToguPS7ksCvSElX=^pZ}MNxxgzurEE(^^0Q5iuo)%4IAe7IsJOE
zXsOGu39dYbA`_QQpLTPCOL>%wo8$_?ohh9uOG1QZgiQ?cwq0`d|D?+riythxyf*1k
zmt34nscA!yQ`Wc1D~}{yeZ4VMlvN~_h4ZCOWa8PWqFa<&MXzm2T_`TL_0ly~iRk2m
zN0ma=qNXnL>{eV7vT0vgn9e5klUlDUx7<9j)aS>yEtmJLe7a;-L3-ECgyvk4=>oN-
z+fK@qaG0h)FT0kjUMMM?|6R8HaijaZV;$4Nb2h$jt=X`cWv_eDqJAZnA`h*|sT1d{
zeY-n##$nZrCu(93dp>98Zs00#S?sE*=p-x@@}*U3)1-%1Uh02UG8CsuUkov`njN|M
z{x**bI!>}{6T`wLi|RO;*5!P=`F!cq<o<}|=4MqhrdBNRNVk+s^)GwQ`&47if&Zp#
z+J>!HxbBN>6<X%5&A4XrLuU3rpAYt%%k6wTAzZKWDTm?l*4cT-JOk#OEEEhZT@v3C
z_ws>bROTzuuP3V(PdVf#xOlZOSNw6o&{;+<^S{l@Tt5BZJ6Vsb69od@Qm1Sbyrm*r
zKg(m?En|nNTYJpBe9d3Fa9iK%uKcq9ZBc1RQ0Q9gf9nsmo&U3L_Qu<<6B;{@By9=1
z=x^Ws_*rOjM2K?B{{OWbZ*7*DJma5huI6J4Po~-BQ%WB$&CUBLR=4jES6Ppfu+_}#
z49|{Eu2MYGthkU@yW;TV>lf6__9-TwIa0Dg<W`#9vCP&YgF`YwJ6>8|)V%xL$6WnQ
zfR5$d8GU8{k1)^csa%un5hGuBX2FWP7u^;6X7xLTwQc%Q)5D|ilehlQX4fW$72TGf
zZZd3Z*m*$lU54UMsrCuSO!ys7))ZTZ|G#8v+8x20?Vu^$b7zz2#4|||5jP)49@?~s
zaf)wehl_N7MT@DN=UKO+?j;kSh)r7Dup-~<aEi1s=T_Comy?+%+|y8SSrpyaX}{u+
z|8%}GOMlyS5gD82tS~ulX>qXAZwF843Z5yJuMJL>HvbA<(DHJ*lde{(`sOPq%OZ?Y
ztS$%o@Nh2wVC;L}^mFmqz<FkSJ9Z>(R0(DA6ty^dV^Ww0zv3pT7~gH?3*YUW+-l}@
zjrsER<1Xv?Baa2eZ~dnEUpG5d+mPW%|KD5b#}9QJ_GDMRxI<&7%8>>7O1x*hY|pdp
zi&0GBd2vT!O;nTe<vXj@7p~;BS-xem;5)bY$+Km6{<jE9Y2{d+x+&u*aqaZXx;|mM
zmK*%KVQmU+&zHMBxpC}G=HkhcUMi=T^q&kl7BbEB&B}*w)Vyq&7OX5=Y2V*-;+)MT
z<8@3t-iI&B`MEtZ?N;ekn6@YSv(Hxv7AMX6{NTU|0-uhrEq%Mr?e_&8j>@glk{2dS
zQe1WOorGiXT<^T(%eU8jxVAnhYKmA5^VxKvrlp@-UV2}Mj#;TE?tL!k+~)N>oVw=a
z-WMX>{gz&SIA_Mrs6Q`GPJi(7)vG?|-KqayoePro&Z_h&i&pnCu9zYm_ukAjMovv%
zd1-{)tsIL<%cf+hK06}Qc_cx*sUSaom(_Wj>sMp^|G92Ro1GrR#C0R@@8><tU(;RQ
zg)jc!>LUBwTmG-lBe52z0v)cNjtNRF`${=8wF~zOdal({Gs@af)%EyUr;Ws%$>*LO
z+)?=<yW_F+#913!x39ij9I|PHj8wS_k8*5B=`GowM0U%$3X8f97t2@6eYN>Kk^fZ6
zoimz7qFo%5-+4q`+tmN$N%!A}YBN|wQe+i53S}ARu$4C*+<1>ePtQtWl9Q49iHMzR
z8v-v?{5JRabET9a-c32KrNm>u7u)FvzMp$fD0!;;Ui$CR(t2y{jBUdFA5$jh&It(f
zOtesrou<0t&&H6_m>uUg75Q3b31>dOvh(t_mlt>Tm@Fyhd=%%mr1a<?VZ-d>mps0T
zl^u0?wtW7}=qZBJ%MY1OEt*t5@yW)POXg0!tBz=<tj<})`+Sa^W0oVAVU9<9#*^@>
zSCRgLE3@;m{K~GKIF$2=tK{Z|f3H10nP1$pk+tc5mGoBKL$NN8UfsTN_GE<lu9Uue
z94%^VS9qott-k)^iS4|(=XuQ6>|AkA!2QC^h$CgQ=5+ooNp_T7b!>|5q@|m#t?+R5
zFuFQ<#k99SY}aTSd8JKDS+%KgVf5J_Uc0h_r-zhP8Lxa$`ERL|ca)?(%YPZ6&<%fP
zEZbtULA`zFsq4G9t~@i{Lv#P9Q>lIhAEuqo%JZ8T)o=1;nS*Yo_hq@c)-D1)Rvc3{
zFkPO=T@snNd(#EYsj1f-t8Yaa{qnVR$<2-qla;^Z`DRt>c^9iqyKZjm+cs-m?ww%s
zm6KlfbqOwf)NQCbDeKqX9YT)x=6Wz@E30SD(Na`*G82qlaMy)<@r(<RT2)FGW!9d7
z5A&INbKlHa-|ZT#cskIjK4;6k+*lSF_YjfKBG;a~eqNMe8+ziC$iKPb^_J_Er(V?3
zE8V%L;ojSIp?dS*<cEvjag#dhrnx*pY3hfca&rt?7WOB;yj@|-{n+VSqFLFBF0<lA
zi*Nk*ogDn=YMHyu1xLQyjOR9feV}JsmA}gMTb|ah`#ZLtv*vfdxA`c;&v^Bi7OC^4
za(?N82}#^fkMXTtz&Tl5=frgV-PeoLgH!t((;lei3a!X_nUv%3T&!TBNzIek{A5MZ
zLwvV``M(74e-%*W5EFcUL^$e@`Er$vRZF}WFGqG<^E#t0wDpSRHWsU-<57X8nLAs`
zeweARRn}Oy*s5VtW}r&eJh9gESE7S7H2aJMB5nvoRtQ9A_&%R#v#K=4`bj~?WFe1D
zTFqikS2jCJAK{v?T$r&<UeHu>k$|FVP<us2d*zPyvWkG37wzk}xBDH87jPAISFGZn
z+94<!c=lRO&3}n%KgQ0D>TDAws;5<SR!CG=?dYto=nPWqoFJjtE7&#DT`8nkazce1
z>nG)d$5rwQJMO9FZE@h=Y|*{`68G1J?!Ozl_q^c$(qNloq5thc;Mwv#v1458FDk4O
z3<^%I)k-$_^ekx3g4)vuSr<7AEJ-!{b1Ya?x$c>I!2^vD%cS~@&E?$JHMovDFL+e@
z!_nBf(Bi`pA&Ko3Uw+syRj9UZZdlA9rM@t%d~<)1uu5o}-ttA9s!0>(E#b5*kI48O
zE+OHrvS@;z;zV&3E-ufB5|*MxPhu^XPrO&EC%K~9(7ZTnsiN9WzAp~^e<w6)-JGO-
zlmF`j{;v#^jUp!-OHMZb|B}DvGM{0ho6eQ=yDa^8U4k5TmRe?HEWFCL_-XS(VXG69
zm6v_ZaMNh9E)&dAFE4mr)+AJ3dVJ~@=jhNUKG&bfP2W`h{-D4vQ9;!YQ|52>^*R#6
z#^HA~LvZoI$t;FaZyY+8vQG?7>)^W5u2Co?=+xOP*_pIp!Xk&x$r&>`R?g_moH1eN
zjFuT)i!xNY1-l+O%ve$}bNb5}^E^e{I|XJrbS|7ZW3eSe`_38uA7`%mIb-e087nwv
zPEec~FtKv!jp-~4r@s*tUEU!SIMLzdr}*Nq?k@q{TUJc^yP*5z$?m-ey3ehgbHQ@*
zNlxv*4g&uggg!Y5e0?xEjL}Izqt^as&5{3tmRe5|)*c8FcI>^p!}9gXWGkayv#t4`
zIt_1rbbdZPX3^q0rs;-^<w|LqLjPwr>OY%j!z`aD)WG;c)!~?{+cr7QrUtQv4JI4=
zb(R-vg?EXUOpvUaD0Xw9_{>={Qi~Q%;8fOPQ_)%^bhC)#=0ZWq3A(=)O>CLSesiHp
z)MByBMa&a8Emkd_IbpGN)*@x8C1zcV<f4`+MopY$z##sz$op5CLZswbVe=E$Vk@U9
z>inFfRXN#Y<<d)>ljBY;HLzsZV6-erO7ME8fOP4yK31>vL+QcOyjTvZE`MU2Cgh#A
zIKx)MYH4cA%g*{^Uz=SyGB=9(9J|so=Na2N=JHb^)&c)L<WD_u3A-VnFtL@HF@O3c
zg+rp=oXN6l#6nxP+ihN^<uR>|*I4GNvi_R~C0jPFTJ5E@<<YA0741n3o$a2p7FVur
zU&*?C)#^Q6t9Soez4g`VZC<lBdal_TwdO?Dn&m6k9K5y0^VjM<vpAy^)?Tq%do611
zrB$4}q-GuJnsw;c+GDTQ9+@@!gp|;(O1Z~Uv;R74XDt*HP3b;5v-{JlIbWjIe|xob
zPscKspt)Bq8TKmVcz@Gpo*{Q{XWq5%HQmg11}e^%R@Cxk_g-J&^7EVX*PXI0%j-l|
zJ3DRN__tH}$&2|9z8cy52)V?V!C9CRSG4K;QsHQ)RV*7r&$U`e1}bgaP$T$%;*#ve
z3k|v^NOvvqI<?4T*6b@;i!OU@v5DH^e0#A?)*|~`ix=NqEO>CSrPU())muV-ZwcPL
z#q0JKyRKPQRt&D(+Y+QXqYSo139v>vY`f&OEpo!PYh2shS8t8c-pciIOL+D6u<Y$s
zzc+h(O)Lvhnx8o7O#t_sh9>=)lm9AA-o0Xn#m}YDUXyP{E_*(4*_7<L$y$@A&ECn}
zuv774p;%$z3d4k-n<A|xqTDoP&TM40d(oWu-O}i(<?^FZ8@#QqOH5s%D!celZEL!E
zjZ%5u_uaYX(e0;Ju$k9)2-TkrSvhIfp8lXcViRq$I|Z(LFfDR!oBzu&VUf@|)#-ZL
zGN#f($^QjdlON1lX1RK6)V_VY*52B+FZJ~5q~F`Fty-I~dhI1Ifv>mseUV=KYxe#>
z()<6-Ui*Lde#V^rsk2$VvJYII#X2$J0N0rV+;cd2c@FZ+9OUR(D=Kr~@9M>0Rxfe&
z-oJCzL9eQH%m!?~SLnR2nzNZ>@`cLvXHL$!Bsuq&)XsmqYRsyY{+^OE&k>sXRclgr
zBzKwGmFe2d&b92bU3sE=Z!fRCII;e|_<Tjdjn_HmiwM_=cN^aEn180l<xiwbj7^Aw
z^|TY#^@^H8g122bsx1C*U+~{>sd8Y$iiv`=1Gt|q6HrZ@u)S)r{I7-TTua<<Z@YAB
zTh8lk8Q$BPYqmAjoM`De(e{7NiAJ7n4Qp8YV@^z5bD}lo<dm9ikpisD0<1IdoSgIL
z<h(ho0RpEM{5jcOb86b16K_{95nDK0@RPF7Me`=>?zYoQ&3^9aoqf7TdYMI~a*EW>
z{W?1j`7E36eP*Wg<Xb0qUay?H^@7N}?jWNKFRoc$t5qzPW{YkN;aW0vxx<MJSC6S{
zT+6DgcGp-<HQS^vYv|*_5&fE{HO{kja#Qrjm~u^H0WL#<@D~D+9m$iUEc=XP@-8UE
zB-ls<GA#-^<|VS~vB|3QkN1}E%vN4=^3(0rKcX+b*WPA1>!O!6!^8;}W%nMCUwc4C
zmsS4kMH$^o3U?34>K^2qvsh%#C4;q>jASpH{Fh}lm1Q;Sy==Ajvd!JgcCwsiwwE2}
zUSgiW`fc{?&X>xnHHVI6&e?Hej*iULb2pcM*|nZ|!eNfat1K7S@1AirZtqpG#;bp%
zu7&U2Q5>9iw_AD9?4A=APPucnyBCM>TKB%@+*ma$O@ukPC4Hm4OU3+Gy<JLm;>I^*
zPs_<?o3*d)U3-j^@05^9aB1+>h$vHr=&zf+o&45*C@fjSn!0*h-tKJ^-fYWVeKN!Q
zR{HLfD|~OQ+<R+D@8#J!SJuoux&G|Q#Xh$-{XMx^_te~&+q+^<?R|T4CogND!6}ag
zCl}e=2@JRsV8C$5_SEb<cTVu$JvEp0^4>eHxpxH_mpBORkTlv6{(sie9_f3RH1F-u
zxhEEQINbNztkq{Sy9DNF?|ifO-qhVYKg-_#qI>^ZW@%1`{Av@`8NE{?mY2pZ-<AKU
z`I^VMP0_ozzOLO+oqtbfs&A*a`lqPIN#~+}PMyM3-gPST#f^uFoX32Y?opbjk$2&|
z(qVzafmSTFHl5<j(<bi?agSHN$YmF)`Y?K%WbGB#y_Yue9-PH<#lG({*PqLR8&2BD
zGTQP!v3q;j<J{$m7oNE1J+bC}DyVQLFyN_G?$Zc9R)Gt5V$a<XXkZCA@bt9rv*deE
z*XiB~+`V>@!=blXOSeVvuiw;t>gSx!-QD_kmR9;bzalBPLqjMk_v*haWpkMqe}AoC
z_P^;x!rm8#J!T9U^HfXDu1}QrjJbZH@xn{S`8O<r1Kt<~^=xd(m0iJTXz@f=!udw*
zoum}bM?A-0ExlH}VVTg3udef}7c37v`*LHbyyNn@-b%~Y-a62Ci)G=-7l~Kgd~apR
z-95SY&C%F9o({K;?mc<x?3>eicP`(1bEWR>&3$ig+<SY|j=_WBonqpf)-?y$%S7Hz
zp3|N^*>vU(i<9pxCErI!O`bjbUPA2LEGwa`UGLxBeSca<=x3c0<NW)*d>`24h5q<`
zc)uc>U)SsAQPsr?%L}rat!Ak8J9FKh^YQMU2Md|cZT$VwyySY-sVRNmnxFNk>rOq_
z!R#~LZH2eQ`JSj&ga7*Hr(J6Oc1Pf6&H1l220zxc9(`)Sk!W)&S#U|8$D}0{ii{RJ
zzd2gmy{U2cq{F<EYh~Z8>lJYI69~6wh>U*{lm9j9|JMk9*3h`8iTA&z>VHeM`xev3
zl6dc1&iQZI_n#K=e^2@Un4@O3%9(XO$xX*AyF+}Rf8)9us5|$i><h7lFTTzD(QbDw
zMpm%(-_<Vu7n9_L+U$OI?Eevech1c7KYHTk>MHAR_;lvjj~8|i^Z1tO8(i6NE#kV8
ze}Tc#>!Ejg`@Zc7?Rh2pyYAGq8_6@v*QJUsF>?K2u|Ye#L3v@})@7k(R$R-^%58|f
z!jX9Aq};npc5fcW{e9#oaJKI6qrAU2{_p?Gq4@U}|G!K8tdI4do$(booqH;hVF}NQ
zE3%W`wPv4wZ}<O$-@SMHmU8XC`eN;epZD%D7&<&~sAuN=wc31vyi+3&yM@k;2?>YV
zMU=y)WCSR?HHz7qZLu&^KHe?h?6oH%dx3868jp=a7F^F7rZcBs%Q0n0J2!`^I_r;Q
ziGrkF$~FPlQ%iZomwB^V$ufj$O<x|(B2?>Tsuj8_j9uaGu8=iBo081u)kdl5o?Vga
zwY4_-)Z2!jyeHT8W?x@>Zx{0&jx)EvzmIQhVc#WnA&nuav4>qtjN@lP(w6x7d<ue7
zebTn3uuNJd6ZPfl!ujkyQXEpc+S8<OdUP(zOjx49{C^YAirVVOZ*MJH=pyjL+TsJF
z2b)qS11nRJ2J<ZK^nErzi#RSUm(I5{@S37Cb62+@Yu%L<p{p}4TF0-R_p9!2)z$3#
z*$aPv`}+8BdH?^ve}Az3=B(j&=&F+BUHhNw$_s~9wx9sV{}L?)i`kT80u<OaYc4pk
zaPJ9lZoc+9gQ41NO;0bQj|j)nnLJsk%jVcL_XrzVBzAF2&3u%?u=d=kz%8=%G8yj8
zTNW9v3Oe;*Nssbmm7~U6*|drnBpHJ@c?GTN@jlsjkLirU$qSQo&K&np*?eY&i?*}r
zj2ji5B4*cfgk8<ECw<<+WbV0^_d-Z^t9oP3$xad7GcsSy?lu+Y{a^5=%le$pmU*X@
zxO9yKohKWbNcM})Jd?a!uBvL`^7&N)LZXVn?x4e}19eTWvg*$XIM}4ON5i={OO<1B
zp!}n)3no>_zPz4LeYSMLgo2}`i&Glc=0v93%ako)y!EqKA)YPE)pxF4@ygp~nMEg;
zGyeILdHd8Pr|mkMRy3{RUc2DZN1gQ>es!&`*uBf^?#6GSug)+{7SV9udQVD1Z^ySo
z-unALMeW$o!ut4<UbDbyPQ#-baXm&ovY(AN73ymzHcT*<-f41L=I#WZD`9EgTTgjx
zOg+LF64WVrGe>x>`CBbzzs2|F74tPQwus%@d28EgrAhC$TwvK_=vFyPv}}oA{{OvI
z-&UWO|GjOSzumtVk8e44Y5iWG!*zCE!Qzc)#eDZMuCoi+*YsrF@##Ar%bj@3$Z_Pu
zo8BMaCOGd`wb6ddC9Gpp&>SMvQ8<M)s`&O?JF)M}CDPY?bX)eWcF*UA%PfCSskndl
zQCODd614V8e~PU4^~Z}&f7Z9Xo>68r|4X^Dk%;zPlQ|hhJ5_s|jt4Jz#BNb{zW0`f
z`Bq1E*Vh3r3&k|LUCnph`{l89f=RcRyC7>#*qO&$#HwC3>R$~M52_Chip;HlbJ)6M
ziX6-Qy9MEsYp(uZ$b5GCwY7FmLK6*Et1zc<{BA0plxBEA<&@I{2Hs0jjRC7wS12C!
zKa?t*o7(-~b<t1GpuSCdLVl{+>UU11vOSv0c*fh~qtVQLDH&oc57+U`^PCjr5ZtzN
zieplR<BZrkjU)TcJWbrTxaWAx!?y0s9s@Dy%R>6!Tu$AXU_8SzIBnl0>syj3{#w#<
z;(^U#GkVW9t-0XDGR17dZJ$#C6IaN|m>!b8UEv-yD`D2<6T$OJFEq)9CiCCfa&7B|
zN?rT-!%`0}Eb{m@v7@fdI3Po$a$5G$o}7OP?*4xgyLoOsj$-|}#J(nxzrI5#qTfg6
zVTk4a5T(4SiyKv%vJ{O^%w$Vd%L>ehQ&B#fd9Fl?aZ>-ePlxI^uJ-g1@}8{YEP5vL
zm}j78(CZ~DRj=iq^j`E|aPskK*InHozDw;2<e3xmNzI)1h|A+8m$$n)c0FFGT3O|E
zFWoAQJ8NSSk4M8iw|V9ax+#f4Q+Hj_&{6FQ&FK<bv%+<`o#?6`JJ&CKS*)=xKvF7R
z^mN18Jtyk5x`e`oyd!r0%5cqg<f@j*SiEOdJFDxIweg}UE?<03c*?i$kB&51Vs<oh
zi{7b=r+P9(EF&*klx|XAcqpa$k<+W4U7JD{<*aji_h`=E9Sm3F{{^j@vzcpO$4;k3
zZ?rv)U4%_Sr>mTQGeMX2abiNuM*}53wxqnj-ANBt8SYQfQGXyKE^Pcwyr=SwdY`1n
z36I6DMpN`E?p)!JaA$OTDtoPa+MLM$DHFwK+jKq3^w;jcvcir3>x$>LZ9KQHIjold
z({ak~nAPcqYitUK7xFsgyv}I*QnrJqGpk^W=-I|wmj%9UY>L{T+F_d%b~0$sQn9x)
zp4z{;*i~k5edmd*vz(%CoAbX7E4(x3oz~{==#xri&U+rGKFbn0W0YIET1G?Q)w(VB
zOca;8TM1oyag1wMrk8P9OqGn#-rb7%QcLz%T6KGS2-?ixk#N0TB>9o8XwQ?4smf=i
zPb?M6vJ2R{Ds)xS{gpaA`@dVd>`sZg=6LSY#Zv|Wk%~emep+Pj|F~9p*CI~C&3s9h
z9&9a9RnB^sbj$Ie*wHic-+s6krz-`syA?Dtx@_u^{ogT7so7<DZ}#)~Ptv6p-`X^7
zW#NbBjiyr;^=w!fWSSUy+M!i}a|t(Fq9XTdF6Uk!MUU9IiIF~Ko9?Xi(_E_E%zLwG
z`ic)n&o^>DI&>@Qh*F!|dheW`xUH5mkL4^$`Fr(I=)c#NR~tU2oRT>qaek7caaxt=
z+sOHM_$*(!ObW}MtRuWCFfI1=u1!~YX05ox=FnZbAxSowgI{j%;?B<*j9ke#LYvZ-
zM(+`2X{e6ousyYSQj@`SRmFAe#qveE&-yk^S;uhJc8zY#+!)<S*ICWWkM50)Ug~>w
zqpD1n%yL0r_V));jV7j4IW3M$>pPqMjH&8-^?RfHEB1P21_gFqvHmY@Sgf$Bo2&YD
z_tM`{pLZumm6(11E1}o+%v16ThvoUmbBoS#&VB6SJH`CaCd*XGLldN1+oVrTJa9BZ
zIe6Vlzu0}j{Y_sMsjt=8#9`E>qmWoVMe2o?g*2x|hEtfP-L~~F7HYP-u=Fp=_{9*`
z=>I#>yY8fxtN)I-U4auf`M(Qzw<T-h<T8(vBkA><TwQCD<_LQ~vrM(z5OA5Rif@Um
z(dh-V4LH`Bv0hbs_Df=?U7E-t{i`bjUH618th%~JNiIHX-5HLbraz4<mpz*9Sn=vq
zmgHuk9=+TT3u7*6UEkFbx2{xF>D7I!nO&+A*VQato%C+j#^!KSV-?|>V%*-7Jum-X
zcS+~=?qw%$a7B5EnS8j|p(MP?jH$P1%C-i(FSajF7Jbj%wXQeyLCosEVOboL)&%ff
zv8*=T@z<UE2Tz%8gd&%DP?qf07wozjo4Xn}D06vqFj+IGaB*ZD$`xGM8X3sp8nrHH
zLu+>iYy1b%oMP5;3%#!bg1axSeYbn9+p9U9PTLE1&u+ZX61IS2*2x{`3@0)NcS~qm
z$Vjg5yuIo~wj$q$uFDUXetV=^WhlG+mW<}aNmn|TT~D;Q(I}gDN#BU`(B>EG?-|S9
zVeA#P+`y!*KL0@H{Xm1qksEwg_e{PmHn)1V3iHya53IXBJ4HWmN^o3|w$VYtdG{>m
zZt>e1lf5?nXWTHqw1Y__q4jH_eBcefAH~krDN-K<nKe3iKTEQz2ryS&kP35P(r9RM
ztdTZ!XbR;pE0<vSt9@AK&xDSQ_D8F%gM#K?nPjOv*>EfKQRUw*970E#xp=vfCfZh-
zABg5Xb#qgVVb|rA6BP<q=@l5SQrO&k;;7MWj!#~jMWr~*tq)m9b2mLcYHY|Yb%{sj
z$wJvzik+U?yC%$NV^rGa)zk5C(#{2|U1l#*d*ya3n&(hP>yAqtcdEP2D)0tLIGq>V
zxB7sBw=*~YoN31zWfC^)Ufdz}d^5ZC2G+U~=I{;8s!S*9rtEuYWc}{;>_aan6l-tq
zI3V3z!Jl?npe&-*PO9~Py%WEM5UaD7%F55qWey$Hr<lq(mV_?g4K$FCFLtSyX#FSL
z>><(mwS$GDU}N2e+0iF<J+kIt*wYcj;gF-aFL(1ce(w{vSJ}0^S@HIk1Am|?`^E!h
zcQ!83n%I6=Tkl4b@R3uiU%A#+%Y3@EO0(c##mDUp&O+Z`xvsr2ag)!Es|)qBioE|E
z(pPM;xITNw2B&W6ARfKW$t7B=gjaGFEbK8&l+?V_@nog&{1v+yH?aPd77uK&oWrDg
zG^e#;3$u9vvwT*wbwO)%0jmS!;q=uVJ01J&SDAj|koS>P4&J$Q)@jRxNJIaJ8c`Cg
zh9_D)C7FMDZ~T0lZ`nap|C{Pj5v>1x7qT`tI9s~Re;dH@)$oW$1naaTyi;E|+jmWg
z((vQBW29rk5%!~<OKSStKa<yFw+B|uJ}NVN+vJtTTQ+W8Z22|ES2xC}&q_1wM$6+>
z(=P1b+}Av3(q`40+%padFFd?kt6Fh0gYTY-ja~6J0SPj^CzX#a(&)W+a?b__w~e=K
zwiIz+*nQz*x0_?~>fHyli;id?Y1n&B`eIM!qHB-O?-1$mIL&kVwvyFq?`;ulB&S}~
zvF3hc&0QwZ9PrVO^`v{C(7HO76RMkhj5V6|P8b_YG?-Vk)^BMJn{Z;r2fn3>yPQL2
z-MYeZXNulp6M6eXhwmlrzqQ6UP~zNn!;=~eR=Nv#|HsXi^mS-(aWPwevE^9@)2#=6
zhQIiuIt~ZEVE^6R<iDA}enI=Z$CsmKTn@Z&S%+h4U`MZE1Cy?YJnI}WnXf8=TB_b_
zmj)S$d*52}?9-Jn1<(BAt#5?Q-%MJ&$1${VgEVW2op<Whm(_L;vs!g$a0>YJhL%jJ
zxGY#DtY7_dj;M+7mq!hnHyBpW68!EoS7V{;x(i(5R}Wrz=qR7FL$5Q?Skg(w$Z6Ju
zp2J)s4-41q=iy~LY-Hr*di0FS6c;<e?v7n6S<kbaP}+Ouv<!pFTPEebKGqYsuNiQB
zP+ak$)7dg)W00d&>m-gSj`^Qf_`d64S=c=1?TMDAC{~|DpAV;*{vVf_y0m+WPe7+b
zieFITe1{g9&k^k<qE{R{Pex_*P4NzIT*2#mK`N@DN#_EKNp64C4L*AlE3-BJCUYdi
zc8G~JUTV2;<l~vCix;~HHH%I9;L@R?u;lE{{|lV|2cGR>nzC^<YuJftIf~6{6PeCC
zxK0f^T4pn`_s6utpQn8l_N;!hNxEm1p2P*l8%HmEaIMT-xXX0{n{2mY<Hb5pw|%k4
zR$N%Pt>D;>hGU9KF%F*t>{h!Kg(zJ&=E?6_<ZR7d62g6Gl|_u}8bc$UV^X|kKMpi{
ztT0SyuQut({@Ge9bE&?D`G^6tc|g$C4XnE>WS^bvys4z?7IGyxo4-mx!@P^-zhTAx
z{~4b1H;a6_(DI7G%(=C}XF+Sg;r^%-E&rPL2M6!hXkf6*J*lyvHSjgNTQ5u4jQ(F6
zCe#&N_7Rx+oKuwb<7LB!TXrrk374gQRbTdBE&A1Y#j{f?f42H@^k|+JQkgx6KQQ?O
zmrOiUn6%*Uivd|z1Fi1PD16ZL<q)^+BpYpJp&grr9yAGQF!O0}8EIV83DKOpWz+48
z6IZV>XR_s9_rz;TqfKLmjrcu=4S$wOTZS#&At<wE)mp1-TkjmyUa7xp!Ll0*!xYsX
zWv#tt^6<R*NAHqFoV~wK?-lVmuH<9g$r;ls*&Lu*5_+BSg;Vs4Gv%ds4eIV5{lOaj
zqxIY$$N$kB+xlJ@thjtHB$7Rzncw&2;V#jRmCl#9D~jH^aPRT~d!tm1V95}b4$shq
z?VqKOJk2?Gxbdb&z|G*B;qy1o58crl_#kj@f{QN0$##pBKi)e<PN#&exSzRjx!}*p
z3{QL8dq*zBB-^I>Zr{9fac)#Z-r1;oQ?oZltqFB6)mgJxW6jbiE{n(FJolzgwMf{U
zJCir)*(7DH(kT-c2i%TO30T{5JL}%jOtId07Xkh3faERW(n@X_kM<O93plSVp?K>;
z=AH|!jLWayp3&pTbG*T}vTxCOYYV-uEe|8z?=McW_S|qvqSefUgPY0m;@YX3B$w_r
zVBIdo5F8Ms8p~t7BIJMA)>-Fo=_ak_?~}bNvRSU0#dL9GdtJlf%QvKEG&C7XHT$f%
z{NACrZbKx8Kx?>kgO3D*ug3}tmXH-km)OQedfD)DT)6iwg8l7A9p4Rdx*RXN`;uqx
zd$CmdF55)@-GZiTFWz12>9OwPzL2H59z`d1KAakAsv1@r8a{U)JDZ5P#PP6#ZmpBX
zMiE>*dX0JwuX)ZT?A-Qx$FbMDw`DgTczsCdn1rsySxNnP)1=;vhoWj>Nv2_vf)<<K
zxXSJ1`Wdt%gEuL2t+&ddH#@h?tzBnp+t@S9V%1@$>!OTj90YGybafW~4KujH@z?pv
z@g+hkUlq3oT&mJ-HviBW|6*51l)(R##1KoH2`ewBw7g(#=@t2voig>RM%jv#kd>{U
z4k+8F-1ED}<fOv!%#Qa<X5WmuofalmGqhdiM?bNPZN2qno(}8DlDn~&o3?)cIH5gi
zn&%U}e~z6zucz`vcf?(sAMK{6aWpDcZaJS{SArUMV&CSSZP^n~KDfg@lP$fVSL6D%
z3nmYhbNP}ctGPvI*vW3y-7KDF{OQu|?XjE1%--;77DzU2UcJN1>h<lUuw&9%yy-G`
z?38DupE-8eb%xCQ?(8U&;}dueFmW|=%sjQ1>)Z`n(XNZ-e|XN>zV&*|`KxD5U_v`%
z9{c$`*6<DO<{GTRFPMY{Jgg-c>Ut!ZUiR}jO8@_```-T-tbq+oGv@d%KGzcXVsZCc
zUc0^SFQ(r6z-l_XllOs0hwN5?s)QpRIvuwKn12e28a`OQs@TghcH3G*`SV=&xmNc7
zm5BefLO1C(N5sad!M{XzYpSyHly0iK`h4;}b=PdpeTrQB9`L9!@0j*_SKDi&8DFEO
zXGhr^N0}QRwAy^Ys3T7N)8>XxrP)rJ>v*<q@LD61!(i*(_#h$OoqLOmar*UVqTfC=
zK6r3;VdAl4bHtfGo=BUnxp7jVy^ws}=}g@up_#Isk8?~I!|qP(nf=3}e{uP~;G_d7
z%?Dj_doAV}O32OfICI3N*YZx1;7|4!i>-~zT9rjR%nO*$%838B{t&r8@m8b6EBmD!
zHqX?*xXlk<**<OSx~3B;zd2Jv1W#Uky3<`%!{(EvT`T|13oY}+UOuq#Fmn?PEtB6G
z=(|`|dNIq<Ij`qm`?I9%?fcbdZ%tQVHhWOEj45r8DtFSd?5z@??o6(VfByPT@s`a3
zI&1cGty{97=&X?`<J)6d&FOtdSG!ba=Gz=Pc9ii)_Uc1gb+483=Is%_etXN2=#54l
zT)nbeUj3Fk6(B9YWoMJ^;f%f4y3-4MYW~>>AFTR%QnmNaP4lN0?{LcV8g+P8)kxQy
z7n|}Y%>L3IEx{qI!7SX6vE^ILom>03Chljv;n67Z(vfHFXW0g4{e(b=1`mt=|L+z1
zZ?55;Qj%G=;Ovr#k(%Q7J#ymSYOnRPU+ca7RgkPX!$<zLKbQOwdmY3cR9o^u;p?N2
z{oPuH+1r<8v&h%)J(s=bSSjzk>nl258(VJvKK;*E^UYt|>wnDuA3Oj5_x%62iZ`&c
zbTDpX<56(^u;5@bm$=ZKBdx;IJA~Bv9t2HPJk}#6T6Dwcz+$I<1??h>h@!(&y2W&h
zln%G@xO7SzA8LtgyL{GYNy#+LqnFh$%wulg)3y4naj7+zPfzuR$x9z2?`Yl(*RHD!
zVd0ylRF0~>c3cy+G0HXK!|MiLiK{{~OFz792;7#HD06gG+_Ac^f8M@*oqr(6nAxYq
z@bXb#&Hv`iHL_2<FBm%RVgARrV0mhTM%6@imy<55^X<#l-1zzWy1(!gok{%bnii$T
z+6E>mWu9E5dU-`?mG|$(;r+3?WxId#-MHPKzrXq9&!D&)dwx2dtXi0)^LK~gx7R)F
zx3WS{Tdxj#xObA`=A)`rdZ|}7$G_ia`{%prmGYVKNeBKKef{`;|9g4+hVPmx3eK?s
z?y{2Xh1+L1#dem4I<bmgVbgpz^Y~FQ39l)WT;(ENCb)*guqcJdXH5}uRPNm}vHkVV
zCrZ_#svJsP%7=c)^}TA0IM!n$7vd(($&@KDVJS}lTax>>#FQi-rEO0p`yM(fGIbq8
z*@Ps&(3i(Tm{lwmPyeqXW7#kxjw|y?W2P31nq;2qi2$dREk+j@-fFaj>U*kQKd-yA
z#9=~0>X$2S+WUT7KApv-dOW~>Zw8a2TFKW-{JM)=1Y0E*_bl`j+u<8_USnp5<M~C*
zU!2Zsy}lCEAGh^_^J#|F)68Bn^v)`Ky=IbPR#f4stFxX5)-y`qEav&;d^5gBHd|x!
zKZES^`sZ#XpWd?W#-lFjwJEz({B5|b?pLW47{;?dz7`%Ysk&fw9_P!x*Ll1Dy)9p|
zi+N7<gvQhx-4h#G^_J*6GW_>W*!xn>#lZ2iR!g2I%SBg#xmkP+f-aIp2OC`^CVxqE
z5iXCJ*r_I`^67-4EKAT~?YfeH-v0_^GLw(%AGC2eEI!#{(;30LUmp3n|Jo89;KG?J
zaxufJ#>6eyX{m51OXAUEVXT>~U#|#w`3AH^FFbn9DQ5Rov1_qMQ@iK#?40@{{=(H=
z+Z_D&CbmhZ-bgyFU!54P<MyS)s6lLrE88oEnOD@>G}c_WJZJTJ+2`Vq&j@bc_V|or
zsMgcTmg$wZrp0gH`HVd`+`Ih#RU^69oGIUhUcG#}<JXQijotJ6elo7mKg1$9<I3BB
zr*a?q_Zn~6J*(u_;R_Klp1xeIVre_}v`Gq0nbfZ3sjt+jR8|rAMKkHjrddi$zbCy?
z>E}@Tv1YgVvVC?!Qx-VBsn$%K=p_}IBJDA4lStD4pZE1sL&KwXPMOMN!JsYJq7`;3
z=>N5lWz8F2)(Sb@YP_{&-}<9Z8>KSckGyaaUt_bTXj_tW@~;ws9Xl8CZZim(_l0Q<
z|EyIZT#?5mLbk4mzIr%x^{gvvvr1*+G@pkpOT5Zo+aW0aO)Jn@#bcht<L)|>)t%?6
z7U{ncbj`Y(6m|8~wQWtCgpDe?dA>D<zrInbXShi__Pfyq$=9X&wO_O2Ig6F|e_(a5
z-&h!*yJxY?#To9_pFSpJge;b2eX>|`t8=rP57$wZB*U&XEGfM_k}ST<7WJ6)qy^t#
zII%=r*sU*LS!iYFnVe0+3re*!oDzcqdzJ}=x)mvVH7}?Va+>;o=>*p7quEm7CnimJ
zl=)OAA=@L?VA6yrn;gZ}63uCwF7JH(<8c(Tz!~WYChEIa_9|YDnUiUBTqdWfE#gYZ
ziN_YlozJPV=1-2CVmVn=pt0h(^^y%;p%qQLq>nD2_nI;M^_OJ%8-JJ2|ND~Pt-*Xj
z9p{#*Y_=MDs?F0Fbwm_TR&#z{wm`Adt;Jb2d*WBGu)VDv?_G47PkAVC*?lN2V$gUr
z<JFs$%HO|eIL@h<KS5)Oiqc7wI$tTTNlTUlA7v;|V~;uUpRL28NoT@}*U<`+6W9;7
z$t`u_DVgl1WTM_AVicCJ()Fpt(mC;4lxx3dyK2AMsBQc6RM;}fX?)VGE-teFUBcUA
z{L0=en|stE+vrl%G}{lq)@Vi*=3PIOq&@A3uHGjP+Z2ItPCjcr{e7>yQ)}MXzZBij
z6lfOD+MK-aPDJF%qH8^K@9NasW#2qGsmy2EQRRjl_X}75t?T6!o^VJk=xT@J)1{)-
z9_|t=4QHNtldOCx!ZUx_-qs~=7O4bA_E!8h>O8YXS;dgkd)n%GGv4s1FMcDYIkV){
zqQsV|?0@y#zO;I;iRsNs{KyrtZrQ!wg0*|uC7l`Xhn#8Mki=7W`PuRh+#8R0To$X_
zImc<$$~k#jS!xAW^`8l?*_QM$eC|SKew&Hz^On^4FMYOp`5U8}4ZOGe4$H}1aC|;_
z%2xxIg#QZ1JlwTkTDLg_?O+c!e6O(WT%$po$a@j#zSNR=Jf2gI$4toRb=kDZoljx0
znCip{s+@6eD_0zP?9DmFw`x;YjD-6O<rCi09+!XWde(id7W(_<`_tdN4?MXSb}_DR
zdB!+x=2<3{2k%cziJFmg_K0`h1xGhUN1Od8n}S16tcuv!86}g|=5l3WQBB}sr)$Zb
z)u&!LiM3v9X3!4IusxS#u=a?$9(PyJlrWK>UTXuRy`vj>a{@TJ{eQP?wCiro*>kKz
z@3y2)oRiR#8TTUGZGXM)-E&Mw{;Nh#!tQNx3%{CfH2<BQqJQK@qs%wS`F9fyZ=Jr;
zsk<O_@v4cwZby<NKL20qBX@J78=JZ2<dB&vbF+oq{g&L_XL?e^Ytf^!>NTpmmO_37
zpEd_gE7G{_y0rZL8WF9tN6v9E&gh$yW)t0{>XgKI`f&crt?|oLXB51e<fN2#za-DC
zxZ<XF-2dtqyGvsZxLipqdQ#<gGS$zzwdF&_nn`;u_g*<<79T5~VZM7V^ZEs{^O8G_
zAKjAR-t)rEpG!^ngg}gyuAlY4k1p!F1QeZ1YIOCscV6I)>u@`w8k_Dh!Tp(#R`d@K
zZQDuHrixttdo^UkpY9;Phjjuuf+{Nt_*PBhFbZLlt2rtZq~$n6tH8`r^n8T3p!SNb
zUwk(tJ?ZoJ<GO0~sfXiq(WzCd<NiD9u8sdv8db--=JwCCu1DKm$99C~?2*!bT^8E%
zb+T-3`d_`@U+3=V+ZE@L=I^b4ZP~TW3%}Z?{@qc1mGA%e?U(m{jC_65>Q`MnoASRo
z+2+rM?brM9Sx;!ot2WGUUXiBtQ{soEn^A|xlEsHrJuU_Eyzk#qJ=tKg=hy8A7)2Pv
zcbUKZwYPH7`}r@9+`q;8!%>2Frs`2`+1O6c)=Rd&R~$I5KDoh^)f)OGX#P%dgTOgA
zLR@cNxGu4VsYx-S<UnY{)9FpU3C)HcYiwpzoRInCarET@kJiwcXZQBC{f+rjJ7X3{
z&%v7t7iC%boC5fg<aIJdu5muRI#b?R=Y-RO>5&Zo-Mnq)_A|~r!7)>WPf~Oe$M=l6
z2Lk6RuRY`8GE<x{(rf1Vsj3fuZ#m!6GBaV}!@oEB|2cRj?U0<U+j@~hAoN8ygCL7m
z@4P_CsWZ-S%rKb3xbNh=&<m_wlOF|!G8)PW2hLh_rfXTGT&(HE<!`5ol-}D~a(vr@
zsO=nF;;n(N=g6(L6q7hAB{@^@_0s;&vD4-X&UCHmWbQpFZ!}>+#p(W<snQAF2XuQf
zS*72doOprpuC(dx3$oJsPSRf(CmAjZ(`QrGSGp~~aN6dtr`amn=Z0JrTE{AG(Xw5)
zzxXDPa_?;crL&&0F26R)dn}pXwqv%H-^@Rb=l5*9t$s7wHzvI7>;Kyq7!<S|j|Pb-
zT>6k4;(IjAB_UL2!6D5#dM|@&CW;@e^qr^c@~|<~t>>VI!HlMb!A)mHdlLh!d;MC@
z-uXVq{l@`?*8wwcUi5p@)A_|S{{GsO_Y0KX?|3pRL2w%fkK~<2vMO!bcMkH)oiJz$
z_kX2i{BMaqleDpvLeK?IT?049UMHuJ4q+!*k-ifuiZicrCc3rUy*NYOqon6(Oh@OH
z6XKK3dtC_rIc3THjh++ET=ChtFhQ|@#@j3Zb}&wV5qtM+?6eEAv#+Yky<0N<#5p0o
zDGzjA64H(&ScN)=aj;6pIbQH&yVbdj=lrwWd8+4nZErYg?>Mxw(0kR#jvWQd)R+B_
z)3my_>r7Op?flGDD$WfjKOaq6$9i}FgOqIyJbaTFCOvqhdrVDyOQZIU#+i4P9e8pt
zfa&?c+-sj2rxhyAR}kbWQi|8&oBMX@qt~MGAO7C?d^KUKpvat<`8|;W&KdD}C*{A}
zPXEFYF6ig8Q!}z_NxWs)kubgkEHjQMxw33;eEjQZbkWW;E(city*%UhMqcypa@&8=
z401<<KP<N2HA(Q_>4|$dFPm~y+A17g8(vd-)-fPK((LSYfm1VipQtIMmbdV|^oeL~
z@%Oqa^ZLQ$ngvU`E-7SPQuylW<9~!Vp6S?}1n=H=bMHlV-+Z}@W9mWOgzk_nhjd%S
zC-%+wF+rkx<^T9edYbN<nUj@#-Bn*qPHS;3IbIfaRHJA5^bF;YGR-dqiZhZPPATdS
zH1ofHlBK9N<i^B|n+uctc7#YCTe&YM^6!?rQp*<J<_weh*3WQKbsNitn0E>>6%*%(
ziuy3l3$$9oVwn&kyQpl^A-0(5;=OXK6|be&sqNVzCCNE^Pt3IqhWDok&CR+b`Bt`b
z!G}KWX|E>UQOmXpko>c9<-29t3}<>KIB4EDxMm;AECqr5d&_QaU1r4gT8?w60b6?Z
z&qbRVpE?{3Tra2^#G`EVEpSsGXV(>{(6i#s1)84?{C})HvfEN~>bVw=Gxx<}gw7_g
z^z2=3weEHMnfrehy11{9wz2!q^6zHL-j6J{N#gq-CTpp_3Y1yt7H}e1CPj}|<sjFS
z!-1>^7qPNNW+W-GL>a`sFqk4T#r@}<Ij<Ky@q86y&+5VZE5_j53a0{2riTyKUOWHh
z=`8IIfdmFlw;L+e8pm`c?&zL-bLZ*Ppc_t_I}V+CwP4e~qyVYxz*U?ZKP}lG#4hfl
zB_ziZ8_?Rj^`Ke=hsZaHE3HR*G7`V6ICFKTkGAW}@0Cl#>q5Tv>VBJW_vk$STicfX
z+OsG`;)pMAXVfizzq>F0ZMiw`lIony&btp+<eZ9}A=9#u>6s&sT8O~AMS82;0t7Sz
zy0@muXzo$(x0tr`$MMX4VVVCp)~)St5XgG7_P_t(o>i;NCY*Yt<x}OIaFIv+=aH4`
zR^`4sF~9C~%F`$7&7>bpYm~ilC}@iNNx!Q<c`mGc6<5flzJf`=uqmuqNdAMadb+Co
zC%4BR4CjAV)9+4tZpd|iPUVpr*-IxcXq)FHySx$Gkr{HLLq5po;f-}1U2D{THKzYI
zyng)5{3$opy{4RTInq++H+@1#N9ofR{~zjQKjZg5N>q{5;1|ohbl_yOnt`+9imFK8
zYXKRoVKO)NT)rfm80x|BBO~H?nNDd#PN(9@S4yt@&sONxct(Bd^Pe(v<-=DKAFR@T
z@MOX{jlL+)15b4wj}-WR%2C&9^Gj<vF)hAx#mQ@OZ+984><sxY)FbF3RO}L`b>i@g
zORY8wr_bxDY<YPqrDcZLT-PZrJJ!Db*70`A+9Nk_U6v?YK1b~6oMXE4F1?-cHDnfN
zfd9dd^RiCQGC0B{`Ks~W&Fcz!Azve#TSOP!Joz}U@nEIy<ZFg+mulTU6f*Nk!5s#F
z<?fR@T&JYMSo66=^<H)7vH55}$P)Xh6)SZ0Nt&44r?mS`C$?%D1*Duz)MQ!nZ{Frr
zC%5qBZvHgyv)t^htReyDicYDqnmoF=_rklciP??}XL55_&%7k9Jx^UQ?xXnf4)=sh
zfA|hm?NfJ75ZbL(JaOmh+M>_fH)i`jnZ5U!%f4%iFK}dNKU{Iwt(E_>%X9^uBf0-u
zkJz3#$hPF#4%fqGDn~V0SOZ;ey7*{&9IVV~VP8LE_Oy?V(lQ+1{d}7~1gCJXi+UCx
z>lu3FtnRBf0Uy>RU0LJ5qJ6@%EROdlb|kp)PoFt;&Z-!t2}v=hWzVGi+K?!H;>8}0
z><gES4DOkfY9y<CO}Y3_=i)!VCJ)9VCNG%GeO+AV_GkKEbqo@dz7n$NhG(G2mD1a7
z1&m)kByZk(9&s+XW`^p;nKQDs$@=!1`dxe}t#$IOALqTF{Z`!(rCy)p&vDG+$a|2;
z^3bT|f$Ir5=c$jlg0*?hJCz(~6EM<D$%~&h?S<Z^l|~a({-0Pm?b0NXsTFsh&7YKa
zH+5p3Rj}rn|Gv_DS{ze~R!%&)mdWk<-A4f(-oX-`;+$U6JG)~q-L;+S=pDRnK??WY
z7v9Nh&350B`yi+PMM(dnTY90J`e(84g<k4!c$a^8W>n<AzWDH`FS;(e4Bvj`91^Q|
z|E+fYcDL(g$9qEaT!gxL)GAv{CoirvK2lo0`M1{Q-Mg08`Z?^;+*<YgOiARCb2=+D
zDn<YQjb|vH&M^IDcbhn?^pjAVoGVMFGn?OSR%&r_+Q2fIsfKq)ujG;Wb8m{|96Yxu
z$9i|##5RK|9+Oj**v{|TurJ4qvC!mSgm-7WclzfJ<EP8EmTXzTvvN95i+043Q_DU~
zyyG}il(YABtY`5Q*Zqm_&;5Vr(e-7O;o>QBuKs0L=F4mnaK0eauI=n&x#Q87_J5Lb
z3j8iUYp2(p5NWfV(e!ju=g)5?nxXzjLML0!?2--dPIS7yQS#i*=^b1DR7C#Sp%Z(|
zaJHsfz(>njrbm~sRm}WyCRRG(;osO0Mx)E~&WOEu=BpR*?!v2Ac}bEpq^A7IywkBw
zJ={z$%4ovARjJ=t^2@emyuSPQO`7JO6AUvxMQ-<>cH?3HkzEUewk2&jtvl=By3pw4
zkC#2(2k%+Zy&*oVVMa%>T&u&!lI&Y1TV8z%F#5T9`N@yA-*rD5c^~t-x#rZBnVP@#
z&Wn|O+}^+ZO9#)NzTBPt+pB{<J5DLD49+w={hvj&z+umAb=Dipr>tH5_g=KcbCzFk
z=4<|~u793nFC^m{%+42NFzaNojdIFCxqb(`nkx@B9}N|_W^=UXj3?WpAg>ua7RcXm
zVrHxV*x@@X%KP=2j?1$BQk*@n-~7DeBzx({snZs7erC+9<z!U#IRDJQ=XH0|jmD}i
z2W4OESea*{J}KhZzrNQ;I@X8JKAv^$xK{Q>t?F49wrmdCd)4^0QC5qe{1q{uH|>1<
znp2!yU1deA_-E`p@-tOdV!P!f55_M${)&1Ztgh+^33d$a-|IBNvWj2QHZpQz|AtkE
z*3NsrbMAe9UsfNV>q`%O5t+7PYw(PS<msN0BBF~fZFP&VOHBL!Twn1Wb9#2fMBW8k
zo7{6YdEQUcEMV|^rk7JRaSc<?nn_Q#POy6SN{hL$A!}dOe~Gg_M}B4BR=IU|>C5}V
za%$<xqV^e^1asaln)~o#v61?f)oVW|=H=^~f7RZ<m9f0oJO8uc{(SfSADC``GdI`M
z*8j3A{c=L~|J*|f$tTL6O*`B3e8<uK*Ea5T&t%!X-r9}lTg`6if`j@FJ~ItknMJvR
z*;IT^v_53;<2h;bL}=pa2BSGAB4j3P<XPFB>eVO`ndRUyNlkQ;&x)Xjr>5v89-Cr0
z`PmsWV^2+$qh5;sGpvi>`Dm0dTx7Hp&I?*$@=&AI&vR>*<dtcMR)ue$vL<W!|0%PL
z%42p)UO0Aj?X<Od6IGIrPhH&^6T@5Z*(@=1Q|4(Eo!eUP8McU>JTlk%z*X6OV*j7%
z1|Rd)439mXCgZ)#$L&<iucUPu6TLZSq*{f)S+n8Di&@6D^QX<+<vml6Id1j5oio+m
z-Z04D{zPGO^O?Jb^^e0O_LQHwb2wF<M{bX5?p=eacfLe#k3Gp4y~F3^zS!#Tj2yyg
zqV;oYu0464+~>c;@sx<B@U8v%H*2k(f8R+JTJv*S&$E9E7_azjoMmxyg$8r+DUIbu
zw>&;Luz5CJSY+n4(^G3N$Lw=Xt&d&>EZ%4KY0ah<k;OecC$(JskA9vr>y^W-8!Pe}
zqEdV`lsD!4&wRR{@dlIUnqI#@6F2R!I6YyrkzS0?iQc-H9>bldG!)k`dNEAwx#H9%
zw0XjpP0hwaR=YoW>3R!nJ|*C}MPqBrE{o5m0-vjb)ht988T$%6^$GUx4SO?r*>#3i
z-wO_$vR5uR3EGBSbm|R$VKKw2!X(I@H8z9cd6wh>vFb;Un0EWy@|Jerog@7AQt*4@
z(5ug<NH;KlFbmMQ)}j^>$kn>VJ2bq_;=!Y<@tj=W%rBPB^;p!c<TPz^|Fo|slqb7w
zdZ+R_X4SRx+1ZnNxL@{t^416|`ZR5MSkEKQm?dZ4ev8>ZC##TQ8C${i*q|k|uTPVT
zKGij4)ib8G>$*-Qy*xDg$p1~rN;_{&I@5dT(=Bb)jd`a|YabO8UTbueCB$P*E2s3D
z6Bm-AiqFncOP&1WcEGYLPo`?;T0Wg6R~ni6FrqFpZJKBI+9w{iw<FVMH`?jGaLb-2
zxonNmTs=kE?F!9yZ#R5ir}p^-H~;OGx4mCnV>An3+Q+#kEMvj16X!~I|16lAY4e{Y
zNBV%(!!Qpc=I<#52ic|Pd{8kjdewDUzVnU5;maNkCChe-CMkbBDjZbfdOTL8wMR_7
zK2Uw8S?1!otD;U#;n6%*aFlN!!?#VF<aIyoUUK36f=OOVpRFc)9rKA?b4n|B{@yLO
zt(A?v?pFJs;9WWA&fjuV``6m*5B>b(b3x<j|DHJImoLvuzVvGLak*bFC!YWJ^Zorf
zCVua>%h&%lJM2C`{VU_%y=N`%rkt%c-^qU^<#+zRohg40>nn#HoouFDZIQw~`vy~4
zO{)W6O%^l9$Ei!j9<B7&oe(ZCX94R$XVYgY>DkOL5}Y^|Y5RTY3J;l~;Kb7;AgZOJ
zePk2At3k71a-mDn*<T(Wzt|>So$Jzmd6Ss;I<X^G6C0!N?79$9r6i&(<f@UAkmPi3
zs&KjLQ_WDB))0a1>suaVD+^yp+ArX?y=B8v-L{Dl;m?jq+{$<sw7}E%m`lsd)+ug^
z=R8kHGm9*k*3;vwkj0!Vk<4~wWm<jhr|ao&GA6q3bW`h;`R`De!t8m=VUo#F!JGM2
zZcF*T^f1pkvqU}i`exs@pviOgD61KIp0tjd6lxH-w5#Ze$-9<UnfC*uD_cB-R~=mv
z#ua#oq3-3@c(251JQw*ycQp9CZTMjGr*QUdI}zU*A}drSH3Q<_JzKyU=#ekQ_^54K
z+Yx29b)qX4KW3QG?q_(4S4FZhM(<+q&IK|WF(#Qu_a=SXdEm~+QWdUybIZQ(o7b9F
zzN@-7vm{JVU~7s)V3tNs)iPD}n;v;8O=k{GE5Epo=S147&YM1Kj$g3!++3;T%D-65
zbcN@}{94`Am)@~b!7q+ZsnzriS@ceQv&+}kEXI;nM#t6z0j5hrOI<{{|F2ipx;25z
zdif?P{oJRH30_xoj&AaI+@Q54VdC-#p_l!s2Agy?Y8`7kFLg3<`P8t?wW9NW_H@pD
zB{KD8kGIB?McT0oBf|IgdB@&g)w!ocX>Qd!?s!k-qbwq~*lvZKnO`|2S=u~!mUEm}
z%4)^4sk)mkOjK)e;@;exW}Mw_R?(`<!rlEzF8b26nkyN`(oz57q@%B&D@(O|9XWeN
zOZ2>_-?r^leX3q69drNPv>k_@zAfxNJ!_`pq^(}ruFG-)xuX2cv(7Q{`>d<JYiDU>
z6FK|5emc{=jWc`qe2aWmdF|N^n@J+;3}!}4$xhn4$574bPRQJeMl9$0?jCV=ztfX;
z`R4ygvkSP3-lxd;@W@PNIWMgB=})K6oJ~6|Js-Lp+iWmfCui23uqiGzZ{mtx?2fE!
zTW6Re8JFsEWX9#Tyb=|S*6BS@9zM>ks9yJ4Zq5;PyAKoBr&b%SoVDV+jjZ^o8k^ZK
zC-%-sY1Es!OxM+D^87a`Gmm}QA)tF}s`Z6^D_IPJQ#n@EirhHFwvOHAa98K@Bvr?>
zQ>RuMyj+@P_rj5rrTE-^&Z#aHrJ*UixLOZbNo|y25?pV0MCr;IZr;hFlb9X`s8~Kx
z^j7#Ql)Y7SOUIiNks^n58U0yf-4h?hPL|N0ae#Zn;s>14Tq>^TGEZqOxp6@3jzr|e
zkTTcqlR5Ln)WpvIuWDTwU&z|P)ogL-j-tvr#ThYPh1}8qcU-Dp^mgkkZ{t~=Z_>+6
z@;Kc%V*-+*r`-LQtG<#YFR#7&_RVwKb{v&GRa{+t=SEwGmj7Y~qdT9SwoQ<^vMu=T
z3f76c-L^ic+Ge(zanH@^TZCpai8;x?(I}p#U9_yq*lfYBqb;*OhE6ayoHw1rwj)ZX
zv%9Wy`oaT@hH9;!c{u%+ESRmbW3B1R=OQm}PGY$qtmv8TF1_SHqjslAeTmeB&Kegs
zp%oU_Q$3!uy*3G)utD|u{%t2RUrSh-YVtSP-x9JGcV;toxLfr!_+p`!?6)12`HwPp
zX}a8I51w`|<gIVs@7s6f%6FWv{lC;~?G=Y5NySDvNh@l%^Oq%mUG;Z@dRU~O#lgEh
z^Z3dFIE&|fzOkli{u-v!=XHB*#MyioZ7^D`JhxqJvHktSTPM|A+B&T><pGzvlGc-s
zGSTa=W8eQOJzBT^vZAZ4XY5g*<0&Gm|D9ws<z8gUdxH09|Lpax<+tpYz3h*hDbeXR
z{q@EpOSSV1dOkGWx2jjQ{;}P{XPvjYp`E8pn$bkviwVEqd=Bs0v2f3~-|gJ>PfpqI
z{M`Tl@7wwRf4w)e-+D^0MBeHCCtYg=kKLD+nf0A&%2?c*!>MNX;_QKCYAO$`KCIo-
z)5&n%XL<mu$v@^<C-lFr>O8!sUHQSYyh|(v7Ekj3e@U9K#`VP}_OCk4n+4t+;Cgc)
z#ZN6uKrLb2ytYI0YVzHVIdPdSnAexSu*|u#>WG8Z;UiiuB`f-MZY6lk$}7=fTC9-#
zvXAAJgInbzC&Lz&@+c*pLyJTL*1Z&K>)Vq4=8QXUxel+j&XRr4*QP8xan0}KGxbx8
zd}==xoLbPszqIGw6}{6+t^&Io1Xu+RUs#@cs)1Q;!!t9d6$*l9ZuM{<%i%iKlbxj0
zJb`Ofit_Fh<~=G0gj%!0xtdn6HE^+<Z<+Nt=uPqk7G9-CTo;3$x9Rb=m%R|H?T9Lw
z_~k+8pSnXG6ExOWDCIXKNlZQ=v2aE7C$~(t2Hsn&#U=VNQ<P%<>$N<a)F!$uflJMM
ztD4qNvqvSTUg~Qc5p!`@-f>9Di#su<k;h_L?g^#M-(LEqG~KZAHtgWdV>lUa!2H$Z
z%zX(hvj-{`3`P?rGE=!W1xc(lWe{2#cWm2)(|_BvZ!3J-t}^NFttG}0tEOmdx??fv
zj?JgZ7jkbO$-R51cYah7(<UdaB}^eAek(t*?w;gqGOcqrkAdQW*ERxribX|SQJ*rU
z*3DG#+j}FxOzWH@*ONzje6vlIV(k2!pFGlx_Ma1QxRj}9QA5Ym2)=m|Zw^*mD16oV
zOlXl<Uk3Lc4i$|hyitx;dRaWB8l^>RIoB>0^=a?8v5EU6!=;j5qfIW?8J_X|S82QM
zwCaV5w}094@`~jK%38CoZFqg`-Eo=Ch94)snUlWtg5NuxV19jLW5z&Z%VJ~Ui2<h=
z6qh);P2ma&db+}Fabrj^*QP?#Xr(Zgb^bPjijUr?B|0&-6o{z4U-hHs(--E@64eWH
zG?j8#lueW(=PdnvM(axelWLvEr6SK83X^RF+G7rJUdiCM&hGy<?b){|rVU9=A|lT_
z)-4u2VI03~!RA9v_f{F`+z2c&VzT2rEaua)*=%vV*Fr6~6deQQ0xo7FJ8!)|Lgsr8
z@hG_VeR<&_#^ey?tgNzHUOm{T`plmz>z&s3)PKnT|03Vnc*?&`z0P0#x2tWsYg4kL
z?sVvX5e8l3rJ@m&b{QDh^97hjb(r4PiO*{_Td%6l&hW5i(nE=Nds4mxSs7^D%(lAw
z%47Pa>UEiFhnn_J$neTI#43J@^{y8C?1^j}w-j#r;#ImqL*|U@q09O?3*~m?=xj@1
zJ~VOju?t6HmL~eCZ~n1Kzk%b1S{l#K6a&`Hhppa}=e7Is7*{Bi^uJ0=`E<myqi=Cu
zj)v;yy>omwUrMht*v$VrAn5%CzK;gRH_!1Ke|^hQSj!_=Yga1px`08YL3qg`Etl3N
z38zMp59a5ln6fr8Wv5OLTcVa6lJn?|8T-0bULD~@Pt5}+Dr8*!q;*KNmsKovqjE-v
z*x6N)g@Ip`)_(cFZt49yr<I;?UAEAUIiMPyoFv)Jr1_~UdUm0R$HCa5_mU^KYP$4F
zzFxE_YGJV0;opz8-f(I!cGA0fY(o5w1Dic2c;}t9Q}pgPQSQ6<Vh+pHO-&jT=J+SO
zEZ1gP`A4thuh=Cg*-d}#UT$~!l4SbL#nkNXja(J$O)O49%C~b<T*FiXzbQLw-kq1F
zn*Q?i1vloai?c2q6AE9Uw#{vK-X$~MzUdt&)f=Zpoqw}>UgqAKi4HtV9%=P0>YMa<
zlja;Vm8|2B+}~*%uQF1+bkgWWYT>3!Y<st<iS?*GJDGhvA^uIH+qooRwFgIkBz?7L
zd3i0^^O=X%f-4J)miwPp$(QX)|Ic%I;hHll!ON;%r1N?%dAXub`Cia3j~;KgWjCuX
zdMF+>o}Cl?@mjgbRX$boV(a4K>v~~%hwe?u3KyEt%5px9eNk9I`Gc&aR-OwC`Kj3g
z`f5@a&n|0V>SCQ)^K?epVe_enTT4`x_MNh5EML<RZrMBCa<S{$s3!4nm+Ki!TMU||
zW=^zD?Ektgc>ONp>U1WXZl#0U4%qOsmxN7trsVecM^J^)q72<MlRFuGVpsZd4sTzx
zrm*ttrmD3s4jif!PI`UMq3?#$&#v_jn!<mrUfG=M&ljBW&!VhRTYIw4q(;MS&Z-tJ
z%0;(doKN=b3^Qvij=Ex!Xu^CsjeApDRz|7a|DvzL+<rMy`47S*W>gee*Og3b2w}cn
z?(uiA$7w##``&>MQ?`Un*nfwiqcK8%$5O3U|G9gn#2q^7HM7Edc7*rtDeU_0UE9~l
zr99d&O;yHnnTXMwvWDeB&F<UXoR4{^cxf~Th6{dO&}Sqlnz&~Xix;C#&>Ak$DPP-`
zUw>BMROsU`KDAx(RnSbI?LSH;C~KM4sh0~>J~#4S$g^`I>nQ_PuX3JMLBA(#XPQ}T
zZMxlxL0S1|=Dz7Mzx>QQPgx#&6S`TjG+C4>IqBm*=c%$ARMiT~(!ZN@Xoh-dl-GV0
z@!mc$;$lSNiwkQ^BSn2xrm=phS|PSj)GE~Ja!AW7msbn_=e2w_@?E#aSYl@2{*(>T
z)lG#Do?5T%V2@_0Nj#*dp>ssatc=ZU>n-s_`5%p$zR41cf6Vz=xtddFRl&_u4|q*Z
z^K|f<w6OS{T69UT*^sTI;j;fk9#zE|-WGYu3q6y2o@ONLo0S=?e#)_O(fyxEu}hOJ
ze<>UONI99?BR+$9$pM>_4h)H6Zu!@;?l9YhXk<TdHY*8H3%dS5YU4+d10Rd5ILk`*
zr2JT(e5tMT3-e^>OwpvwqZ*p=E(V{posTH#y%5{q5INT;?|IS&l^Hj^E-ubKx~+@N
z+stF37#j!2*Cr8_1nvu){4QL$RraRtntOmq){-3s6;jDA-!2AdFXs7wI`b&k;oiA?
zo-5XI*R^b&=VHipSY*`_!yiX@7oA-s?OG&Xv`A3Ik^5}L<j<QXT3Q-=?-jXZn&iI2
zMEspa<rZO!1v+nP^<0@=u%2JD%B|3|RZ62YTPj`D$&Sa*J$+WlMdcX{oS(W{KUEd4
z)^Tl#D{MQ`rEHeJ#%9r)3tVg6^2HYxe_zMSk~+cW^{J|lT;JAN&)GG3{f^tSubSG-
zeUWi#mW^fmAv2~~PREYjsNU<y^Hbxo=5jG3u3ax?yz>k?q;JUPx39_HDs}FeL)u&N
zjq?_+pRZqUv&vuUlM?%?gDY7Ut&HCK<ibW#Q%k7|7sZH0w|{S{YG$6)!1L~GuwT@X
z|N2W?q(2-n4Qz>y;uK)YNs7wOYI-1`x4qP8TD|6SVFg8%hUJBKn(M4Ga!qX&yjN&i
zAG$bW(S(?u7f)uf@$7%HThpD-rD<`&x6cP>eC*t5a(bi8>H0-Qe$GZsi$st8QV;l)
z#IcIw_&R~R*H=zE9wecxyZPKAo7dOs-rDXxbVSPS`XAA<H7S$xSF7GzQ1+>^%s`7r
z{20%#vR7=|C)}CYf6=C0c5BeYZvUI#8!a|?^cl}+-WfST_=v9bO!2o$Vd__(hy<Qq
zAbf9$@GqvT!cJG?ov#}9O#J%(jCrK5M5M`qCH8Tu+gP1AMIKzAs=%nh6B4meaRc+p
zc4kE%%dq6=(*H}jjxC%kTpF!DIXQi~)(nvY*-cBs3nixq7Al_6Y3ka#qyFZFDFH=m
zLJiw)irxG>W2*SBjjCLsDcOfTRS&mDZxii2^Yq?Q%h`!e8$_1JOn3V3lH0Xsvqs3K
z`-OenW?geHGS8QOe9lv)e(m|0zki3k)6^GYPS9zR5j}mz+^uhm{|4FXy!RG(Fs$sc
z{k_qrxwmN9Rwc8AqDxnlJv+#4d`;um!j4<VHU5~$n0MIp?6o}{T$jqE=D^tZ?}h4v
z>&y;}LJo|x_Pv~M$8fHSA$ZozrkE3<F*BAmY_FVlE7ZI!-}rd4(($u(3`KL67tcA`
zD6_)<@xv?M7tY(cB6U&bOfkp*25U8i<)Ytf_%P}0yUXn<lKx@qADbOkmMbit9s8z!
zWKL``i>kO<l1Sy|%||vIJ#_S)gprlEyUG7We7Oe8{jPtgex7hf)Nh~4wjHH5haHT&
zUw>Pn-@{%huzA@P_Vh577q$QTqr0?XpL4hhRG2HRX$y}uT-0VKyLsaO%hOe>`*v5%
z{eQ*3mch#L*UJ4$d|YO`8o13W%UKuw?*DiFp#7ivzb}v9e|h%)^+wy<n{7*)FZXS7
zx~cuKBzUK4Uh0e`tNt}+mSx`YUvu|5qf#f6s_?b~!xOEir?ki{GpVs)P-^es+{v?|
zqmW^FONzxr$qO5kCH<7WC)q@{g`6^48n#X6zbV5r?`ekFd>cAHFFQBarZP$<VA=8u
zj7y?;4XQSau3X~XZ)SSxl;-6XUdyePdTD6|2Klo0T>AMnEn-p5thy^VPl@?&UAv?}
zGj;XU*+F?t_jHPnPnF*5_;H)ehU>gD511!1m8G5z^)=5c`&W{D@#@?Kxp|U&ITx=;
zpIjz&z4UUb!waLb#(O-qlf60hQbU%<o!!j+hEX-+wq9e@mUDM@mVdgZ`{nz~*;~We
zP0!6T4S2ft_pE)ZH*fj!;>tq)w~<xPOI}~!T^?@X==g(^Nrc5<y+dOQ3)hYL<@fFE
ze*gTWI>&s6z><TyDLj00FUdAU^DsPMVUf*v;493b+Q?+?S@3^u^L5n)!mZ|!77k3p
zJGKV2bJe~`th>%TLq_tXY5AjKt-R}ZBNbjgS=6obX-R~%xv5!0+?}~!1v!`fj9`;Y
zpP2O6FNdS_iLWQ$hLp)xmNzd?vMIfK)i?Cp6)#)!tA{R&MQ>R+Bc^JrlaprI4kyX@
z1TL}QWF1jSHE#>2@L4&NZazDvE&fpBcI=L+4pO<bOkNtfjwj1B0(J<RzsOJ3`L$yH
z;?U(Q=38BLo3nH(llk2x9b(sA*|m19nCL2d_sd0h%}$rDRcj}GdF8cX+LB9ay!U#A
z#BbcUL(6kd(WTb)wWUEq8(Vu#9Mo+Jg(k<IiaT<Y#alunzQ${b`Gl>vHkJR^-F{7N
z!4bztYAF}o#NQry;~SmB@Id5^(2fTp!8$u0Fa+JuZWLmUS?#iq@0T@0%e}i(jxewW
zM||*;;4*1oViMdUATASfMZlRmFtE^N<;#?buF{1U1D%vR&uEvf+jJ}3`^>&8YkW)&
zZkgg^C297d)b(0T_*A#IH6iES?)F?^FwcHkeBQ^g^l8NG7hASO#0j=uk?`EK=$daR
zf5K^z=mS^1vMsf?U3Ya<yXGcYxHR*4WYgQ!_SmvJ744d96S*SN#ij;^=cbk<PS286
zW{)mfB&j~zb;ZTCvlq<NJ16|e`77I<0tZLEpF1ZT+kNkxwRpv<%yYHxes<X|-K}%(
zPGjHyT`yNHH(OuzWpAABn@z{X53q2*Ro8pBYqi~jwqThXAND=e)JSX=kee^pCUH;X
zz$>ez+g&>p{N~^Hw{7gP?Yd<wUiIqcT^FleP7}|)aesL#b#Y+V)ftQZtlou4PuTv^
z#ZA*puG4<<ujl+r{{{Z~ufj0vUD!QI*J&k}_*QRCa>!2NJ^SL))|V}uKIIHt_e6?U
z?!NmXaN$dZ#s5@h^EC;y$lJ*Jr#+DsTN$8nvry)-Px%V_l^MxPnLFKcB!v232&8U(
z^w2AW>4-en$8a@GC9l4WBa=$>YfKZTZJfk<R9%nV=Fg!p(e<hmbke$QqeMKIgcMly
z6H|JUx7=Z1+p@0g|IuuQFRxDST{mNGHtTsuAt|pz6+$d?pG>S|+s!G<s>FKxDW8kX
zff>#+iwx%mJd6?53_b2svPzc8wcYE@=`}e=C;4phNt?5#^UR#@i{{t(Ys7fSN3{rT
zS+YP?+ika(cIP3HWd*D{K8u%5JvGTo`(~53)yiADJPNlRzM*JTzMwnBBU$ikt(wNE
zMxWwo&5Iu0Syr%UrgUz|WsQKwRV|&)a|(pm&*j%$HlN7+{Bcw8xhr2>i`BAxtsU9U
zR@fEKyA!>TbLV2&S2gSAO_5U7h)xk`+Of{|wc?8!u2~E%T3gEgT%5Zj@4Mokj%*k6
ziB64QL_N%2X(yEU^s}%SU1%`%|6lq4%VIwk&An6VURb}=x%liy<#|cI))0eLnw#Gw
zomzOq!#!1-Q8G<f<>$|iwr|1<{5sN>Flc$MWOy@wO4%hXZC}lm8$_a*ggxvRe0XYk
zY$5kOxmI7NpKtwFJzQXRP|MeCX>qWOi0sn{4|9*58`f^S84|0!b;q_xqU+0JL*h?n
zPTw1;r`X3MdBRRiNQOyu)#|7>Clj6?j_{los<Au8tEp5Y(f3lzIW94gxg9!FO|w%M
z>gMiRk#RIPw{UAjiG;+?4HJucy<;}6<#0MzajfEOcdz=}&V3Iojz5^WrQcffhU|BZ
zDc0XqIJYHVchpkb*d5x#CA~u~QLsC9;?iuZx&N(iCV4B}@?Cah@)KPj&*qd}jD=0<
zt5Viq@T(MFz^kC;e7UoxIP)~4$=l5Mm-^vBKSdLguB!j!oWlBdnq_tzOM$)O9Pa1G
zckGaR7c_m=hN#!KCKpZ%4l#dNU~zh;^y|*a$yOgflwRApx7^97TL136@4M8kYUW8C
z(`SqIJeqE$rDRfOv&^62$c+n*ULlJ#Wxd|(NS(N^<)O}7bVL5}W7p3-3o4f#p3pq)
z`UaH~-BmGSf&Ly}{KP*Sm#$Wx?{_9=-;xI}($~h^4`|DhR^U8Y)b{ZHzjr=YC)tTk
z3zZ5A{rqOCS<F`TlnG4}!kkz5PZhmrZ+mtT-}5;Fv)o!PRIdKtv+?jD>F-Yt&Jpuk
zUVO&lwV+tk%8xf(i#lG(1_fQ6D7q*$`|9e5!%-0joF0{B#(Ju0c}G56%DJ(kcg=H~
z?qtry@NJiZk7|?~+6kym&z=AFP0O7=hYyc(_qKgHR(sKKN9mK!LQR%;Ja<`ku4<l9
zteWM~nUSN^vqvvco5@r9Uh6d_$CE0pHzv*4`zH0t+a;g(JwHEp!b2~cgYUIg78QhR
zv1(gi<YKHgnzm8rXoD@In*Q3htxIzr&g5QJ5M}u1<Bl|`BP#3XEWMcdY3s5*MNeXW
zw#Vk}ec!P*S$C~v_hFv1A{T{^F6>@u|5kI?++eTE3qnr3S35ATabMnXPKHm;^Z(mD
z(QAI8#<Zxc$!wOwyoxWjFAC33+3%OyTB`mgWLnXIDFU`Dz4JBnwpyN>D}4Ft(G|<4
zAG<UsDM?xW>kC&W!5~fVj}vFV-MP}IR4bT~w{X#gE!H^>r;a|>ZtZ+q`>93tN5JnX
zD;G4ia961}--|qxH*K?MaJ6b}iO+h?qc20vOH}us5H($SW%s^=QhRRa=v1swXsz2I
z_t>M1|Ml<PcMDY3HJwYA)_(oiIBe6jH+S|t*R8Q)RNZ6jd`-Cetx>|{Yn@gvUBqp^
z30^7d-Wm|r;5@f?cSg8Q?}NhYd&*8vuD+YHn2Gycf%vuK&r}r|89f{h2??;Mq#7{o
zsApNska*;OT{J`N>+KCI4GmkrRxjTaseLG7-eM!|LY*Z`^M5@vnjaF>UGB6dP-OWL
z<4;Sy)PKaiOKo~zD$Ma*_P1!Eha*?2qsTUcguO0YahqgQ1XOYv9M?VI%3UnGXo-x|
zB{mxlk>-nBWeVc9*M;S#Nz1Xd?rM}uRnVGHT(qcMc%gW)N|}?6idW5H^F>R9`2%fD
z+S}$_F4H~Mc1zhmSuMCUJ=w&qeen^Oy9ae96t_z?8pjBDcpXoE$P~!unv&&idofr+
zY-`}}O)0jFihgBs6FJJ_kJ~(KOSK6s<5ljIS`?&}>bB!(Qj(d!ABR_UM%Q^q@nwn1
zc^6u46^aRe6coPr-%#I3<$H1Z&&?q_GeZ6!?zSja4KWhQxxg1{obdJnTiZ0T4#S?W
z0^-#ZCAKB-Z%^P}n%MH~GS~GRY?+Dtg@y%N6Zo%ZaKAgCHqWi(iAHejiLyG^B&ibd
zB@g>NUMLG6X3R{nmUhy;zDSP6P~pNP!CsfFGfe&yrsxUHa6V%0`E{C$<B^(GV%e*w
zX>Bh{lQI-qoS1FlsH2jo<N780Noj51lZlI)wd9j?3XbTr8b)&zPI3w}d>`Jg^Rv!o
zVaookxz_^?7&9jSVQ;YT%nNZ#eK5`a_i|U3jhuev*4jS>xFigPCkx7No+9Yk?NO5d
z%}x8qwfsNFJayYz+&v_1e*70^zbGHKvp@Y}faXN+RLdr7K@;xlURgU;0xlZ692WHy
zD*S0$n6pfHNdR~Ma<OdzTya9;^^x5BE|{iVVB0=PqAO5pjfD8tgu-nMKDQ%8ZddeP
z=V-cXn6&7*W7G<(>dWFa%*oLbf_#RI!U~L!7s`31%d<TV<SNKp?xvKty-n*{+w9M6
zHtgkwZL@D5l-X_B&KTZjzASkKXW(JZ(zY9lnd<tV7O{LvQiwd>VRKQTZi?Jl<xbno
zx#B^J=8O!s7fa=*>c<3#TuPgxEn2+sW|Cir(*l+z`HPd@DD`tFR<3o@lir}0=`#OI
zrN9?WHy5XZj^*8L-{-fQi>wt&2rm6Eo3fzS_9ENsM-d^({4<l4aufK=9JnsEEZ7#n
z?{<*w%n2@L_TGSE;fSMZRZpvK1TI?K?yt@j*7c~*&Q(h9Rpo4RLEeLUUvKFDZPLBf
zlyhqm%fC+sH<arC>}ZX>S?!t?;ocSd_KTsiaU)x2&E%&Mfn7Dsp5?w(*_zC83tJ=o
zk1SovniD@uxQ0WxIzx5kAy(6_e&)vc7tNwYHp^>gT3<}IKJ#*t*(%}vTNyr^$2c#n
zw~}P|E6!5S)u8O!`N*u{{<X!guX3drOmTg=LhN(QhsY&;PdpB)DXS$*t!rAUaxq)_
zhK^yf@ba4qXCr3?J(wrEi81ksUSrCtRkKz_3;cIWJksQP*x0c+e)BHj?NO^gY!&vJ
z9G`Sh)Vz55)`I4+%hQ7&_$?0L-o3=~<_a+xb(5G&THX^EMi)33pWq6e*rP8}wQzZ&
ziHLN$V@s`I#GTIqi#6q5O3B%V$+J%lShrd3<E#L&O>*T?^L8AS&(N5?$YS=Ntl0%S
zoJ_;o=l`5-k(~D6@sws&{SQ_hPTxxAR&=~GXZ0*AIdFKcnydSpiSAb{J8x!oN-{BU
zZkT&(rSKI?fu#XlUXE-6(J8MEdHmzrz#8pbBjIj(TK{L$W^-*;p$(gDPj9xrz1b}}
z-P?3aMVV|_+m_uo=JQWq9Z_VOv2lU4!hE4uGdkG(%_LOnLjT)mU1Xc1u+V+u)^8hv
zZ+A#kIR<b1-a6x(<+~HxPJWzcYNfwsR+Qx{-IHmX{jTZOh8T2b#8jQ`kl4EHz1j}h
z&w0<8CiH}43201E{5e4?aKaQ3ZN5q&%O|qoQd1+V8pR{oW>x3UKI%F9c7DJW&&9vf
zy%#$7Y3E8ix#e29d8`z8!?ayjb9X71jge*ybMRzK$%Y246)E4B2d8z~YcjeP_AXf<
zaKh80{fmH5uv_mJ!<vo_U9L4d(wsLXdCpuS^mCc#!fc7pTI=syuivtG*1gqowyRja
zWENyrhOD1bka}`;gi-u%iv{jU)AgI;*B(%1z9f^W=<h8kzNWF#saZsB)_;jL6U8pS
zYOYu=bNArP9qt>f7VcTxxo%me&AV5DVlw;gS+8%avf(_MB0py?<7XSOqV>|uY5hyq
zKjYl2H?7#Ie74RbrEA4vdnEmfG&ZxHF1PqRRorn-V0g#zn~KqCfu}4tp6yg%Sy(FW
z<UTWEMdI^~irkx+1vV*bAFaD$6SP2N^34^GBW>5uJfvgO^|CU#c)R}B=`nuCT;rca
zMLwy|_EAh}2>Gae+{V$wuc`ar<h5%qu&oggZ7!E(bYo*(W$c&D@3hfGeqzrx7LB|F
z?wZaF#VseQ9!eE6s$?9kOirz~`JrJWv3TvmDHlT*ubH)(?{nb4@Kc@wj1HQN(H{bz
z9RDwO!_(mWO5szcr=1gzZIH29_Nqblb3^ZwJ@3p`3Tm7=$X>%<Il=r(&0?ozH#WwK
z%~<MZA|?5<*1WKGfp*-Ir$$l|%lJ)<%Q?Nz9GWuc!^y`+{TGhu70sHndG$Gg<Rhll
zyPZBxj^AFtXSQNv)TX<PQ?6Of?0%*Cu3SK*!0`FDhE$Qpix<zRnW)U&yz+*1)ch#n
z1+Q|q`WS5Q*;wo%|IMlJt!tC(;)~l(t$w+CUx2Adq9fPerQOjRIoaB^gbWHUz2G{v
z(40AeUDq(I@8SBGownA7{W1oOypFS$c^;JF*>`WHqVvoE+gSn86HM8PZItd@v2eb)
zs#!5_qww#JXtg*0XDu%)&fan8Z`C}e*&3JFuU+iUG%fB@=)88}bbIDE*$FMyM^0|K
zcv$a@<q@`^jcz^H&mYc}*}ZXwN01WtjoY3_Yo-Kf#_rKtZE9mPZ+ok4;<93v*R!t|
zUNha&QPp*RS#ys|b+=pemV+<sLyL|FZQ4>_R^j<^q3uRa;RDS@jbRh++<JAw{Dg?g
zi>;L{ty8}4*t~pZRM~C)pb4i|>afmUz`DZcj-UYR{<WMvnRlWVa7GERMknygU(7wX
zxnpYW@=Y?6>%A2|bLgrV?QnW^x_S4M`?WjHN8eM_oM<lMDYW9+s^X<{qI7m{+xba1
zXU1)#d0X}_@!qxX$JxFuyHZl-mumfgu&MgUzaZBgv+H-AuAlN__x{_$|JdC)cAfVs
z-1A0U>4K}<zqfn(R^NLtCxSoH^ShYK&u=Sv?t1?Hso<2T=qGdNYn7YEx_z&-Mb<32
zRP;wA;3$`I#wEYQk5de}q722}?4GgZ#-YOpr8fCmTI<e|Ua;;)Q^^e_Yv;WOT{mv_
zRA625fPaYstLFyR*#>Mu0<3`tM7K^@7<l0L)&qQN7O-wJ;1Xmwq%`MI-;Za3vGr+p
za#v-i#8v1e9D5#RTm1V}TZ3NL*3Q|^D=xgSEGm3q6FDPv#i8~qJFcme?Jx-u)T!PR
z6@1;}+8oce>+U|fLYt0ADy62BpKsI3Q>s>M&OI8<@ZZOE;#${@f>FsY<rW<>`E%&6
z%4WTw=bkc$j)-3L*~WQDYQKNzaVDt+8au35yl#E)KE8afaE~VAnMFeKf}tBV45Jlx
zTU*!NWqN3<#kz!N%FR!Uc^<#EjSAfQfPYQFos)ClF&nUDp5xoPfHOIOf6an5Zx*nA
zZFrXA$M)5MjZ<0i*Q9`}vBwnF$knMBu-`W*TX&uDv4F~<yJCzVTW;?;ptIx|?->uT
zGm6HZw?u1eJu(C=Yd^jXb1*!c_*eYwq{i>MaeB<F4rIg`=3nb#ulX=@$|kwp%yJ%|
z51nH-65wWtIazvsxAlszJ}Y$Axie?1JvL+Q+6ld8Mdvs*E(it-9zOG5ihb_vUsF~p
z^WVSy{DO*~vC!N{X*HdT-d$mw%);D0ZOg8US|?T?dKV|R_L6~z_u>sAzD|#SEEQXl
zP$&E7^3s{>e7BVZ_Fb{r?cg$3areC|{<ZFFAMk&5;7W_*-}m5MrW|Y30j{kRekS^H
zeO<u0FW|ks9mA~mkCy$LeOXs-ZPy-Ir@#!BuJ!&(JzJ8@u3z19fBl5Hhm&kK?1~Sn
zTybr3Z`%HP*;tG2{x63GUt5cKZ=Cn;<)ewq4qd(O@&Cc{yjLsEZOn`pxw-%RUqRc|
z=TkK*e-~FAzB^Gr^-px*TbskTxuQOXcuL>$VEWIvE8X*O`re9=dp|0|J_YVydn%AY
zfOFD+4iyUnPnKp@K{p8n*Hbg9QWj-Mx~5HOVdP}Xk(sbW*?ofa(k-iwuev*<Y|V|U
zw?eu@Sv0ranAmxv=zv%dbHvu9*3G>ltX^y!s#a4LH!*rlRZCsv&@_cJV0q4sLyg>=
zMyXy~GoLP7z1nQaj+}=^4b72Tn0KrR-F@wC^p3*E+jL)-F{p1V`#A0Cl4(BoHz~66
ztL@pSzuc(9JJWa0j?)dtIs~2DY<3ndKf>&ld2P+q&uJItI`{vJu_}2uVXnoqJFm-5
zO<TE9-DhU(>~C*ZY@O<JV{J;~qy+~XnbqH1OrF5#D8zC>LZRWo^OL4(0dg)(n)B9f
zd4J_z*BP}N8_azk8vSo&uH0Gh{hz9FL+#UV_2y<}`j($+gHk`ee?F&pbNxAsPaIoW
zCQMpzU@@CT!D8n5S*q%NQJotMI9XR4EMeEb<Ke`o+%jQtpo7RnWiz**1&hthF7l~F
zN3Be42%A)p(cOOMc8jQ{iiHNJ?9vYzN}5)e9=Uy(bfal%RMf|G5#6Ybts>JTo^U8@
zvY+-8o|JR($Pz;*8HZ(ti!OAkhpF9qvRil31*PM?CSC=~OC&-Pnk-M=ZsO59rLefy
z=#Iuw-ted`N2f+JseTDCzowMm#V8?~!`;UetbJ*=+T9g;TTiX<4z{s8`#IR(`RStI
z0QbLJY<-_|>s*~+(W-Mk{B*}u<^Pe-7mJ$46z?rv{4M9}RolL*rEe~Is+xWaay_lh
zzwOhq8LQ0iEov=`$`qX;Y>~~}QM~vCN3rqZ9)^tII}=1CGS%N)h`ngLNi6hQto3!F
z(BP`{zgw>*em)u=URU~8Yzd=$?owU}_JqX`nj<COJm{6Ub~2$uT5Y0oE9>fr4Z`dn
ziW>MmGao<dU9iP{LzKrvhoe<FE1x@ty^?em)?G9qNyN-!p~upw88WVFGq*%|PS*A6
z%1jMwY5nZ`sE4VnTUX9zW{=E!mDFaHGqtZYS%oGrSgc)J>b1fueeo%+zgs?W1z5*;
z-sX$GmKU8HexXk~&p}u=f6eAAR{r@OJ2!ac|KGVLNMC>3w_U3Xj{Nlcvwo{Y_#K13
z4?;dg6C~EJ*?3fI_jdhLa?C4v&X`5<Y&j$LI(zeF-{84hE;E(bUgx*``FhW%wBl=K
zca~1tcI#owXWN$z^QAj1pBQm1x_yxCmItTNuHQR$&vlY}{Z4nq{@d?&T)SLnckbg^
zpZn3NSH1R~v9^xge~#fz_=bkrNnTCM40U!m|9ZZC|3B`J^EWs*ao<*XB2alU(KPv}
z#<kXkVLmS0y4sVE{khl|wZujETStf5tAl(~uOxd*<h0wHJe;J_(J5-K?AqP5WOajQ
zB6na#m&lf5i^b(U7!9sWWLc*=PjkXUzsgsa=SL;7tIK8uv`U5iztxqoV(Y=T9SS>-
zG6;QL_vs6FQ1B8F(cfV@Ha}B?mn{lkxywRde;Sv6q>5PT5q9m5GXr}weIEaMXC!g1
zb&3AlnUAFv-Q5$Ox`dUkQTBM`vBo}*E6!pLQ^*tp)~vVBqDmh1@>V`J&Czs-KA|kI
zW0|pprx&C3%Ts<|EH-g!9P*B|R=FzfxYg~$){DJ6UU5(DIyA9<Z4lR4;WGzwEH_3!
z6Pm7ADB;0ovGHm|XYy8-YgrzfCW^ZTs7rJ<sLfNEV0a^$M>*5|!YPrnlL~oMEt6Yq
z3zE9KZY&m!$U4q&%du^h;Zg&agB}HYwryQ&>XH+>(<|)Mg|@Xdu6dqM)bC9Czr(+t
z?*O0QlMZP%;{e~lRGs9T9`_uY_I_~*Su`UkCYFPJy1Cryp4ByrYi(xDh?9L*{b+N)
zq$so16eTv}8Hsmaxjeo!jVsM~(P7c-m}cuc4~|W^bnDjpnp?Yj71ArN>`d9NaZ~NF
zN&;8R%M^9xmSxv<y2?^!W<0D=-TmuM>Jp{DFWgPjt@;+~FUyTmoqF$zz?&2K&RMD2
ziE>-QxOH=qrElIi<`KAP!jmU%Qch|;ixej4Zc%m<ywSq0ykYaS2~KJHK@r+w5&ZKq
z8<lh}x6GI^Z5i9cF0ZKIg>#LzF0bBpP`g=atpNYlZSuc@x0W~DX1JnuK|U{I!MdZN
z3)n8O{SVYz71-Q(#jWb}1tX^Ig-2F-i}_AnsjDQub4k*>m2+mV@?Yr{n!Y6OM?$H7
z%-ihRABPri*i*iGU%~g@N4)E0oRcoP-pw<&_I>hh9joVh-=Im8)}$~o6r3zP=vk5S
zAf;Yj&C1Bdbjso>;^KQPv_tE@hs%ieTx^-NI%1I<SIkWTZ6UUe++T~P?`yfCulg<V
z$`chANp8>o4oaI+ic~y>9GqCTTu^GuIVkr+<*?@-;|a!77m3cUI3;Gbuu(?R#hq!U
z;>4-$VM}LTotipB+W!8rZFS3h^MdPiJ$Ozf96IwY^!j6l(9h4z-F}`uUHY;phJ`(F
z=Y^*QqOrxZw@%+-8vXy=*~W!3&-Zw9g}n7Dez1A!51-?18~WZYT`agdoy9lf($<n?
z>!QCb;@mv_^8=HpEjP~!HGMj9FZI^)TS144PI$>K<~!!g_K~yPD)SMCuD01_=l<xZ
z&(8v1Mjotb^LZAawd<1TubQ>5#hV;9iO80+s4JYj|F>uD{$pZ1>yEiiZ08bo=*{dA
zDYxBt?8brX96JMi%WY5S$Syjhae+h8bcrI*aare*^lS0bGZ`egQeCbx8J=P)J?!u{
z;evcz<+gPS8vA))<=={{H29-tk;{3t%jn6s*xjWe>so$CYln3$iqKiN^4O0UX7lin
zW4hOh{3G?v76na>m>2WJ=iXiOZSMX5HRaV#y`CXF<#U$7jG)tt%E~=5bc3GTZVhXE
z^1Jv~*R4s?isJ4}1)F%jPFELMJ*UCd^OQkd;FDl=OPT56Uk}cClYC^#4gLI^rEz*Z
zGXx)p`<=1#*(AMLPGz-7bHgN&2O+-ijF;?od{y^-=kEOI=E{PpcbvChzcqV7li`YK
zj*C(c7qfiqO8?^~rvIby)r2bUwWmaO{|VR{uyxh!;~BHwHfbIxsd+T1=yCO<;Q2oi
zzUqJ9dB@W*dH!`{H`5ljfJKSFB%SZ?EA4&8vEF*uG>_*d=6io;eyk3cT_m=C5>uAP
z=LOG0Qp+>4HZHt3^XVozd#+{NcOM@~ewQqHBc81|)%yRIv|H*kSht>ZP;i_uY4U;l
zJA|Tdex7c>?WO&~AD2yU{d#VH{KKkie353k*%MwW99T0^ajoVAmNT<@5**^S#m~-~
zr1`K(?bi0*9n2FRm><00a7&8en$ZsKFWdRQ9N^h=;NTC3rp>KKD|Q8BP0`5OagU3$
zW4DOM$2HqlDDB^|=+P_b7q@o2IAo!F%JIpfb^YC@6L#xG%{G1eO8h;S*oQ?upSqkh
zmh?=W>{RoU{~U*HQl>axLzn6f%Xzbe*Uwy^b8^;CCaY%+ODAm5SMl!p9xQH=vWwSK
zXhou!*b$4B+?J|SOl_5WRF7=>e7mEBsq^)N&gO{|xE~u#V34%ru@*=^())jh1n=$r
zd!l&F5?tOySl@GESQ6C5<T-1}1&)9ehX9u84u3cfzOZ&&BE<TDf$zzRBgK1~g;w-t
zNV93Ph8>nZtK7Y6$CSMvrQ><r9(*=RV%c|Kqee=I@!?1}i)1%5W)-X5t2RyO|E1AA
zanaOwk__*E8yj<NiLsXduVHfA+QeD=c;Ok_dy6$oQ%okWcGEjB_uh}*d6!IlH;X;s
zXi~a7ZDG(pMhmTimmQo3`rcmXd!ESJd}M-1W!KFE3lBzi?>)hzvc&EEDYth!r~BP-
zvzOjqXrmo?+uHGq$M((pe?_iN;y#e9Ezz`c{uGmKU5(rJNHu3z@1Bq;5w?Mqdt<Av
z0Eg*+7o*L;+~=p*PbgVD&!tOqgQtLwW25%LrrR9h32U7UEqF?F%@vrNlh!(RdhzOr
zELS<Wl4k>Bu*Kb^DQ&YYy>4ne$lC3-Q%P22Z@00Y|IQuT*X&T}vD$UTdDUXmuuWDr
zw-#%*$V5h39$=aFscGjl>1F$uI2}2%`17JI88>%&bg9(195%LbKDc8^{zR1@o>pgj
zj=1gNlzge9vFbGQX7Al1J@#96O*(9KsAK+?PrLZ4ePg?PZ^&5lEAz~hQ4`wj!mX)3
zL1TGwmi&Ye*Q+t|ss}oFFE|{W?0T@nHENGT%W8pYZB{;k`DK%j9(mklbmH8NOH=f;
zjpMGko_c+*<Mq+j|5@Ee(cE!1=UX4yq||VmF4kD><u+@x$geZ2PDbuMdBA@{mhCTw
zmHnssCtJ;rl{UF!?e2Pc?>+AecQ1?XIl%j9vHRo2)-`vIKVj~C5wNdu#?p5gTK-CA
zZyRPUn9y*0N9XI=-R_t7y_?MQ{;}5k-E(e8%N}dE=o-9<o#o`W?EQ`*f`4Y)PrP$6
z>9QzC<?639SO1UhHCxd!>j}r~nu&}zSj;;&JbZJK<I8Tw84Qim2L<08(7EcM)7rzC
z!K_!Rv}(^<&V)_T4}^mmrx?v(5R#ZGBQ>M!r<bMZ(q0Mg%{r&gEOxa0s%v8^w%uo)
zyiW5zLp>+fkYFuU7H4IZIi^>04t@H+NmN^Az50{WhnIvLHaW9lhLbmI=k*^>u30LE
zUuH3BI-hiL_W$kK@KFD(%k=+Qp`0ftGI=)Llu)(N+z|9@$A^<wZI9^1ot-}SiN_U-
zqf0IX%(XbeIrGTDfEn>kE}NXg5-;t){bokCv6MulHE*S030LPnVXi=zt4n@{v4t!T
zY}s&jR>KXa-kY0_>IiEX&1kSW$;J1OVU<oR?*pdwFE$A&Ft5&NE_*vwkDL3$nmvhU
zj=tJB`Qwhx=CxeSTPJU3u=}jzW@3GQ|Ia-;3yxVP8+X@kK9wPJ^5*$}XWZ;hZ;m=>
zeBwm^q_d{8n2tL-AD?P^;Znr$k`(tDz85|RiTHG1*zup6Z|CB^8WoO5FQ)|xcR2Y>
zZ(`~ECn>S{vf#rFym}R;eWj+CvPHY@Y<~Y~U-z1=JvN$>wX#=_aLm#0{CQD+qwUF!
zs-6;xtYHEhC#+aKVW%%Mm*irePIXWFFQ=_Irv#}^cG!P9TB}vq!1k2dp#wYi>NaO7
zowzW?%v8gCMuU%Gznx+K4n_VA2K}1~?wDx=Z{2vTWp?nnvup2K>3d1;?R%`-*Wq}v
zbe+@O)7{CZ4WtgOn7!-LkzE$Ax{p`vJd$E*C**T??G^9Xoo9C}zyGx7u!z`~ET4d<
zJ5TCdeKK{)(+xh0q#M)%r~7hodK_7^>f$WtBJu9k-Z$Rfd+%jnYNUVSzoeg}&4QaO
z);F)H^8MbmMcO*i%KC?^wScum$nQzL8SG{P*GxmC7F*aqJfc!psmiI*dVDs&ZD#PA
z7k(EOaM>>8T0f&f*I-49GUMhMOwG3#)U{e~CiK{4wpOuD`SF0M^_O2`@8!_o8=pLG
zyh@Jv<iNeBGU8j$!;Y%+dlvZbTXFtWpslQ*?cXUqS8w-k`nYN>m-gD;z{?3YKUrVc
z*cExGz+{%~9Ni2Rj;mXC-oDA18}}r*ug9YA*=<h!wtezn`4-IB{cuKTr;$v|N15C=
z`%ItBJAPx|+ui-*y@6I`k1Tcu#&|Z=SMC@787L>qljE(eD0;Fl(##_zx^L%dLF1?$
z|DT@S_eetXZS<#91J(qiz6U`@HyU_7Rv4W<XXLs=pI1b1*(uHk4PiT4jTUryYBJAR
z!c%o>iv3L{p$P)ED|hZ%aQWQdQ|Eh?r1|cOsjagK<<`>h-uB?2<J~EazG;eo9Cxg-
z2(~&j;k9x6+!Xr~r<u)HH1C}D*?Q%}sVi$a&Qvo!@2OJW6rl3t>NKM-JD>jD`846)
ziy3^%H<!HeSba_>c4@3bpiFnv(ZfnEq9$HWc5kj;{<F0|Wq!=fX-<1|xvM39<%Yc~
z%}Cv=`+e!z&T02&&N=DSb|fsLT}-gY%)&KL#kG&&`qP`!B&E70tG!(8_%i?DL`}u!
zh%K$ArAN8NR)qXt*UhPr5bhw#Z?;%dp>ggq9-$jd2BpnLC!}jv8O5k@Z}#v%9~H4|
z<GfbYh}S3gw6Pv*Jo>7ml>2q@iB8`+%UX^1CUi@?JU&`^qVKF*h+<@9WcHM+{1cBJ
z+i+;>=S=?mlA9`Zk7nG>-uv!#sY%bRnMwC&TG!^pai#VtKNf7%Ing{NW5*@Fpc_qd
z=H>|UX1BN83Uqv2wq<elosPGkb8@!aS~GR;xgXgHv6%}*T(&QMzT?y5jkRw#x4zxF
zOCo5->TSL$n<r232{lxeWtcrDx?YE8U#P&NXVKH|Meo)XH>x<=>yWz2LpoCPRZt%D
z9joq^-MZE<)-K$)*80RbvjYz6|8MY8%{t}0lexwCVGNshY}u6LolhBs_r|fMwRP?2
zj6Qw2^zP1@XV-gUvz|?N*rjYeDWp*C3}btbPw39ud(#|eH3jGj)P$rzO-+9`qu_Ut
z;@LTI=e7tKX`G9&b+AaB_SW_abNA~Ts=lj!zh}_z3BGcFwuiWDNybf=`1WZZCGzhl
zbj8OeGWTtZ@6%W=d`UF#>a~B*;_qwOM9FBFH^f-KxW473k?CG8J&ndIR?^FQj1D|j
zZ>-YwU1+11F12FWzNVuKKT7Z$+9$sMa_)mlqGVk0&ovKI?r8j;;&Joi!(Tu4DWviI
zi%~vea8sdAM#U{awJ!3iBk!ddk@ie>Y~pX`{Fm*}QPX;)xQb`53*S@ie>XdPckF&B
z(RJ$Fye}uFnx{QB&wFedckyGg=9F8vf-X#BnW*?R+d2O5#WPC;<mY}(oopY?AN^C&
zu&2}WP{(c6UKd@DyhmZa)8o2!xxQWUpo!~X(EA5WD$BjL3hFv1uH7Q|zFawf_L6Tp
z@)|rovnO=S4*co&Rqp$_(u47OVqdpCwd)mGHt+qhg<2^G4+cs;bC~LV@yy}+d!c(|
zJ|(>>Pb>`i&>8aoMq2*8G|$=)@0D{6uZVHPr<d1N90?1pe0RZZ%?!<g56m)G*A|M&
zZVK)BoSr%DS;jTD*u5fdzVjCN$Q|ks=8Zk+9FyT9V%^#KLj1pa&#g5-v$UVMu30K8
z&m+#nxlm$)h33)K+Eo@4n|NPr(dtbS4XT=A_RB2Lh+B+jc8GLzi&10MCiQOVWmBYe
zl$)cTx-OIK-l!iD*8NIf{qv0jid&9Li`6CWxSrM7`fzRD3BTnLJ+|4EH|N>zHWqqb
zobWR)M*C)*;?13A^Z$9sGqFx6&?%W&TeHvX%dDj)^SiE=$*nr9*RftTO5bg3mt~Y*
z#FqyV^G=oBI=wIF$^#kGe>s!Bzj?Rw)|5%M$0P#O-M*Q38uXl~y;i2Z>F)e{?YVy$
zC*JzEB!hcF=HnUMffE0u?%X?f(nGH7g?ye<$U@x(ldiA0ETQ5ObF<-n+Ns0;o25UN
zt*zdjQaO3;GagPJo|vU2x*_i>3jOnce2*#L`1I%W;EUN0f7V!ZdwF*@=QljK)Owns
ziS2ZO;|c@C!>t@Ac{GFz7ai$h*7G`2QK)>pPtLyS&xy>(9{tLS^JFF%sjlqQRoo|Y
zA?T6MOy-1pAv-5>&(sv2qQN0hvf{!bmrgZ}Ss{m(g)N!m#Jh9iRR&F=Ng5st7_yl-
zz4qkIJk*+gcvFg$+Y`^iwXU79Hx6ypDAkMY=UTEM^J*Hy?A}`ae^0e73hp0j=9l(+
zQ@-(GpKf$q%^tx|3}MyQ+cudl2-_VcW3D=3)slBdFEYQ&n`q>~IbD-$O3T{ps>f6I
zmYLoAUuJxucXjvn`p0qwvlr*xm8s9xOOCcEe!#Ia=iwnvt1r42*Cz94*B+j-;m-c@
z`AqY6f7fK*mb2rWtQGsl=v$lj`_-=b^CJ2E<MZqP*E#<E%i9+^p+Wg$h>)aesfmz!
zs?H6C=F|zF7BmZcWn65OI(bwub?J=|t`_D338fC@u82n+x-(BKbXSX8lH{(JRdJ0a
zXwSB(y-utX9r`TU7A)@fRGGjMoU`GQ*F^VCK~EUg>fC%X$uBllIbgA`Wn;wepYBny
zu?2@xCFj;ewzVhpX)c?glBd!bCcbv%GNzn;ng=uGR5u<Co{@XxiEEYBP4)Tpep8oE
z^ZB&%qDC8Efre(ze~qXYzDqKWO-!GaD;VWol=U(&`~|OqW68o?!-l2BdvC5}5}r_a
znxXv5wyf1<TdW+`NN=nXTs`Zq)dAg&-@@FgbGHPpUcGHnk(lJC&Z#-sr?qD1M69)y
zmfL*U%=_&Y)=+J+ZETslb4qT-vBd6RY2AI9Aw;BQLIeB6uK|&K6ITaB<sRJ{=q8*w
zW#zsg#}Yv=%b(fneNFCOIp)LW-m<|(SyMdtz+*4N#xS?QIUxaJ%p0bNtg*SeF?edN
z%T>3Of~U_`EI+iZb>sVKt$}-(_O}L{VLqZG5Y4%^^<uQ#+dcUn^R`{`ig)AFjgVd=
za4m3>XRxS5g1qJ48<)+FR%E11{vTQ<={~pg$xVg@OJ^QoNMd&V=H@UpOzT#T*2T+V
z(LE}z*^YH8X0vCMX}L<SZ)2I`UMK#_O)_=DK}V;x6CMbQ=ebzT*~?nA(eueQ-@dz5
zFG|~gmOY&KT=@CPIJsTb3&p?Oo61m>AkY4HY2Ex+zY3Eo>MlQB9@OyXVqUe=<<=t#
zE<%6LoqTWUXi?L~FK%6w!gFfM#CBy{ex=VcPWFFi$}Q?p>sH>j<Kj1iD*1hFYyX;v
zOt4UUn%r-rrobxcF63$QM@fEfiuBqEj>=)F*MCeE3iEnyx5ZkZZpkHMy^lG{`JIw8
z#GWkdm~zrtahF`C+gCx$+*Y0`2PZG)xT@mN`2Rz*(w7JG+%yikOgxk&WpzbE!I;Z-
zqEm`=&y^FqjKuex<jfbpn7DjRvgWDCrP@wj!OOdT2<qK&43{Z%6$({dqw=aPw0XC)
z_Nfi5LN3dB${#pY2^?zV+_6w}^<!S%)KbCd2?ryzoTqKw#Hwq4Xk%|t&*NQ>Zmcbx
z`p$-lYpUb7hzV17a>sF7b}PJgU0;3obh?70fV+g}B=@)!)&4)?ep;Er-p5oXTNch}
zSoUDn;t(HATdyXzg^xB`pG_4Gnh_CpPh|Q3o#{F)DKniP7;R>~^d_ynrQ5Gc%+vMX
z8%?Fh9+#H2NgR{;oY|G>V`Q2ld@`p%Z|271?jC|Faz!q=|C4`ru$nt52iCG`N8Fq)
z<Z;0KiAxJhiUOnfx2B_J8EIU}HZhV<BqwT4I4HXPCPT4V09VA%WtM`1_ez#2@AOSz
zUwCt)U&V&Y7tT*rUvz2Jc?P%Xmand~TQO(Mt=aJ8Vt>+0dnMI?CMLFt+?E*&M5SId
zhW%Q=F?ETqa%aHzfG2wxOPs3JUrq1}zH)i~x)-i@ihF!gmhA6dt6ys?wUni$(nQ(u
zt4E;erC?7(*1}sSazZzqf3rztl8^3&#zdvbGgTtOgx0=PDtwi;{L(G8J`cqVImQsq
z*RRwaCoXF)|0|IivO>{qBGXi6hZ`{`6kHO70@voAT^P12$+<t_a!2%m|Dx-;k}O(s
z8o0MTNLpoP)!4m*G5u|)M_TT~E+dZxyvut}tg$HSem*r(__&pr+e+r+8HU<XyS6@-
znZ8vw_3yUruen0yejBfwEAzVljaT^YS=t_RPHOMEa8$fhxbsGBU{Iwd=Z!LRPNkNI
zOA_ztwjNrsZ1UV6DpNc{9gaP!@>+4PGvKhy&G42(r~D?KoxCYTQdcr*UFVfI(=rPq
zT^Dq3t_d`s{&Q2bhlI^^UXINzuCqT(n!vVckzldm=hj(QK7@#$+H%CGc&)4LDoZV&
zo4&cXe4;bYsV!Yw@${(km1WI^D;jt|oHbv2LMLDARO4m8gL7DCHn=#2a+xn^_x$vR
zSMq;ADm!0pV*YW<$G#>N3QTe)3!~1(Jhw_;x}oM~@zch>7bkK8jW?&4ERE(De{ebN
z>RvzlP1&DopUEt^BWYf~C0W4nN6q9JdK^unjXWQeoVsS%Zm2jNy=7)p%HCIw{Js;W
z&GW5&wTL&-*X8c*F9ESEc7ELX+)u6N|7sFhqRa8t!{2q*s?fhn%yzJOMEKqgKO!u9
zi&dbAr7*=^;K%9s`QN>Hc!F6@SM4?anG<5_RvP<4YlVQ*?H0K?W$p~Coy63*S|=2W
zEdFJ}Ahj*2V}kD*!LL&f7Zs~)l4X4Cn)bAfv*4iUdfR^4Qw;e6g$i4j{#cZHFrsVo
zleEIUMb6SP>Hm-A?_R{qtJIt$op;+T%RP$!sBXTt-kp0)+xyu%&$)K2cDGyiTIbB%
zr#kbqKUF?`vr9`~b3<iq+}xD#$uqv>Rh*4p*O$+zAtLR?<Rm(E{{HVRKWx3CL?4L$
zowW3cZt~2A!k1rOzWY?sduHirL2tI5RW63dbe1Jdx%o9&Z}ZFu=8b19gY5e@9Lk=X
zc|7IHSwZ&o&y+Vm<e2*C-lN^OZQo1JbkLT0zT@iD1cuJvA1jS!TM684EBJPKxwgc2
z=~}MiVhpK%4sQdFCJBn1p6tBBNmlCP`R+&?;rSX*Hzmt`Zh2*O{!mP|9{c>hbL!s|
z=GHQruA9vCmHUF|%nP2a%Y&x=4?5v!(0ZYH!*v@c>s!miPtRN0xGLDseE#K<Oipz(
zhs7*Q^xd6qwHyq~U83R}y2>?+Rs4tgxvO9J8-7S8ybU|IbVC)VPl~Db--}xgZ&`LS
zC@<?Z-(Jm#K8t9EM^oDc7ckrkb&r|7t@L(6qszjIvy(b}dt;~Hp4i5B)Kg@umr$(B
zqdBhCBJD*}8lTP-6P<Z&U5~<(lU!?7C<tFX)unM~s;SIj+dJDE?n&3q6L}+c?B#-^
zAFpi_n7%_IypWYcekHfF#@v&8xlf(sZc?0dU`@=jlJNaLLT56kmv>$lZ4y5;;rc<B
z%lk!ILOU+SvP{#Qn0Ud_^#Y6F`JF;_o(*?yu>aq4MyUAY!-l1<4ObTm$wf3Zx(IJN
z>1q+Na!;(;($3!oovnhtlfN9j_{H^d$c+mTGp-pZskHfQRZLpuBFwDit9|7#vmYzl
z&xbd2F6VS8X*{}nt4C?Bhbv!ef~0L+f1%)Hk9l@CneuJq)?8Rru;rRiZp#zacI`jW
z8%4TvB(FTnWppZSH>{fXqA{|fMY3{(oO7<9iJfw}$i0~}&qeHsd%Z`}tVnd)5y^LV
z&du#*o&WZn#k<G_KmAINihLB{_ccA=bYaR&iwU1UBz_6#|Kc*MOXfC9=&Ua_{Xa^i
z9I9qHSh4-oaQwByUs-F(AA==v8lF=Mj!G?>FO)Kk;iS;Ihd2K>uy`m><aWR0<M|+H
z#+hWl-cEa_C&ITl8R}fNY<v<sRbZ>-DetOirXXK#U9J-`ETLNta=-MHbWB)zKz8TS
z+pN9{9vTf<xi0w{PuF=&JaF*TT7!F!GusZ<M(@7q#kOq$+qNd{1UXHUIpL=sY=3g?
zMPpp#rw4qCmP+TIF37!e?BPU5hM;=IM8&t4gwMHNt)1%W!_wp-<bFox;1!v6jrN3X
zjO`u^MIOyz%3&4X6P_&Obs@KL0*j+I>jAfpvnyjdFRkS{nR&;dVwpq3Gp3t7dowuW
zoi3>}2?r<Ky_tKrRhDggX~dO`yV+ec3~evx#yxV)=)IitC@qF-S1()h%-;X*EX(hm
zlr<=2mHZLwzGeXz+wuD>>;)d`PXm`f&S(pcm}y`XV_2mAT=&Y0$WHZ0&WahSM$?u)
zvk@`%yYl*|TzIO;+q3uTTa<U+nd!CU@l@SA^S6pvNlAS?nA{q0;{C~x#eYLS`(9|A
zwWRS@yo1+$&#EJh8OMWP98K7G>wD+@kca03&)#zJYOv{LUHA5QOv#eCf46=wd~Boj
z;+w`a^<~okN?8<FEs2#(VB|~iDVpFNch)uG_9Bh0ZK9{PZEW(7d#?3YE$N}a&On7-
zy*$E+sSdGEpTwpHxxHNV;-v7?B%zA|oNT@cxl9pdnTG@Wj%2d)m9fuUc5KGA1xI?H
z$^^#zkNS3v{~gQm+$REk$;C$(3Z7cpI5Dl%a-p2kjMz!X61C1BesaVoRPi9sv+}Dh
zzCN;Fr%p<C(vr+`nzCV1wCvrN1}!>w-35zQIr6X+-&`2o5IX;pq6f=Uv)m&qJRFaQ
zv?=d;<s_A~Cg{dx1J27O5ep=gGGiXac_jXsa>LCgVa1+FD<sl2*F|iMO1n2VX;UAY
zLQ$ja4;7Cm5v6wO`gOMpJm%?Utme4Ytb3(t+chWcf@9xBPiv&8OZGUGuX=6h7E@8x
zS7DUA$1B9RNYbS<YVV?(#=B;kz0;mksQmWjOpAB7TxTgC<P#Cz_V_)IysL^-j;nvn
zjm3U87nG--SgiWu|HiM2S#)1GILSM*P37jBW^mNM_1LSwQ{uz+Mr1MwxYbR0w<E`G
zn#LK6sP76Mex*6hQcv9(Qu&TvZ8MGbIPj#GZEeVhld4IxeK$GsB%YlY(7L0Hcj7Xo
zNnB5_#7$=^aL|6^<sKqdJmZ?=R<G;3m~QkXgl4trrX0I@k3Y2Z%&M8W+FJs+YgbOM
zd(Jy=<;+J9?oM02*=EY@c}t^yO?$gjHRe|4ymP8iE^`lf$S!!*omutpwC5z%jV^uy
z%M$uH614oi1&$=Yag{8}t68{k_M4dGtq*H*p4E2DIVSor{Z!AxUz(YFS~K-}Vi!q;
z2Cm4vbl&aF(VVK-Yl^`)Tze<Bu>8MSdnB*OCBKNxy`_uqY;3dUJKo&JLXImXR+-zf
zR^JTE*KNuE#^yVzApiR&ChIFqwi6=suQcm_Y5w`E+5QKUbpg+=MXx1#nEv!NmRCJj
z$!Yvsb!m51-_#4*FG?j{G?N+J?^Jcf?JEk&-I-qVN4utKipNy$MjH<ArBmMjN&nz#
z(C{uUw#dV;k#qUPCml}~e|z)z=)c8HY*Ux9zRzuY^LNdy6NkiI*f>uyxgIG=KK*TC
z;oT3)tQwhKZJ!KZ1ZZCCI(PD);A;6t30wFQdLF&#F)Ez*V~vka!nUm~S4(dD`ng?Z
z)AR`8itdg6>=u@i>)?~7d9&{J%~@-;U$h2u2LIPx5}P(9;^(zy`y0(N%h%f<DYU<_
z(f&c9%<_Ee4Tm)TYz*fr;?8U;{lYTq(c8Ozj!N@0XWDho>{@wm-$j-0Yx%mCMzlSS
zd^26<m-sxfbtR$^8p~r=nysI9hdXIn$eU@e_A6wyPZ96ZxO2l+_>jvh&54Ww8z$BA
zwld9d&;8g^t0Ep*P_*3W`n+qcBHw&gMuof2c{PW{P%*CU#09+%hhD8VQd(Ol+xGRO
zDL30&H{QUfD{h_*S88K9`DK3WG+*~MXEQlEHQQQ5wm5G0oBYj9bfS*$qz(E%@{(*P
z<m*4!`0JnXuWyGPUT?G)H<A0^?D+jKgU%PH>(0OT=^kFVdY72df2W5FR9cpYdNA%O
zDtXa*)BW^n_uH$Dr_J0K=e}QW!`#{x^-uUL_|~j5<oD(cng27+r+byJ`kQ*LsB<?K
zADx!s??3Ap*VeBoZ$AGuI4)<JnUg2Qel~b3^O}V^6Ea+<-k4Q$DUxyPmIL?Zi7=hF
z3F}|{R4qt(YwVSyJ8yETSIsuzT)V~gw$S2O51Cx9*t1H0Y?^OeCn;>yZSWN=FWf2n
z(o$S=<rz+m)?+Md-rkb)+>zI&f1prKd5fI2S<3s3DZyq=#$Qt7U7VA@xn5yXU@qFZ
z>_S-<V|kRjlK!VmYa`{!saeZ4@9)~SDW>S{-qe>x&(zfuOFjm@H(7uBV_96>rT_W%
z@ACCmiSqSSW(t`|`aE~y40|)Dw({uKIqk3SoHIC``g&T(<4@fM?2FF391y$Xe=VY|
z<74MF8{W$1xfe_3w!VCqc%ntA?(U^&vhR*9Q7bBQpTMJV?Cz&UY8qz6?g{tnnJXHS
z<>NPV#=ZIcW8V8@?TS?IZw}TLPWmNj?Jf2>EUnEft=E};SAF}#CU*Gj`dwa*9rL%j
zbw79e?Q*0=ZqF|5|6L}P%bwrc6vw##c(qYM@8{!n3qq`dKCphi=OV#h-<8599_`X`
z^$1_-_YMwEsoU(z8C`5DT>V?8d9X37ZI5{x6ZY~%_d*M^dlfa$Hq<RjZt19OWGY<N
z;}ylp`v3pd&^a8>#cV!$UDKFzI%AH!!UZR{?LV)kFo?x-=ufQBcAt@}8!?&n@78H+
z!u#hO3*tTM#_(w3Do;MugU9|HPQGuL8eX-OpCP5$rNunoUgczZ@sHbmhtzr)`KFcK
zowrR$r|9moW2<hzolzB}JX=+Dd#38)ZQCojb2*Eh7(P1+C_a3pDc5Cm=wGJh`Z-$R
z3m05tIXvOj0Sz8q=iSftU4Q;;lhmqR(zV_4r9}#kvK|~-o4tiLDeG)ryefoYsluvf
zUS6BmgefUAomwrIv|6vNH>lWj<%UA$w${YY41ZoR&ni}U_5H>i%ZBc*kj<VMON_0L
z-}tvjsAp<#xXXcK3}5~`%5Q0jchS#bITl~h8o#YI{Z~xIp5yT?`YFGhj|*hXUcEEQ
zq`Ym1TYH7H-$9k0A3X<N8<l<OdY=EE&A>By@#~f6rhdrgT`ymEX6D15kb{T5SY9&m
z72e(d{kh-v*27W!{{H9Pa}4Z~YLA%h4D8-|eA>>Cm|H0szS;#$JlU-w``H-IR(O4O
zR}nnY%FwfL>lEvv4&Ownb9I0DwS~@SO*nN{BX-M@&5B&x{!YC#>#lE#;nc0N;_AD!
zL!Tu$*ZsHMmF!#hR9w)E^SN1X)3SV)<=Nk!=WlCi-WQUw?L~gs%km>?o0^0F7#?3T
zA-vB~vA;~oyHV)MzSY}*xfySI_k7>~HILWMdbf6Me9X%;?edLt9^5MXBt5BgvWCIT
zWg#~8g~wf<9FsYlx!pxV?P|mHGW)h*37@3_Y8M#;WKO!@G5;GLx?eSuW#{LWZZ)&c
zuX%O+&bmB4eN~}+!=E|tjM}R9n|m;@tYl<n<ukF^;Jdn+IeM#fM7H8=73L`#0ZxSr
z9a%li%AWkVn0%sB(sP=LMr-eW#=jxoj8_OfYnY+Mf5{{=_}S?>#^sZGUUti_Ua&Yq
zP}Mc<$_(FVliZ0atXUzPUMe0pl~TK=tcqM-GS|qFD`I2(Nk0wiROYCyncICbZ=2=D
zY|q-6v$Fd7J2Ae>_eCFnix(suZ2r%uaVC=?=?L?Gado*Ht-?=?CR+Nc1@8==8I;QA
zbL~%M@$++2-HYe#*>Jhx%F>YS=jK!@WZz^?yS}Wpy3Fm`#=`qw&RQos8U-&a^C`Hq
z=q7K~=Xa_+KRz8eJ%@dQR`7*{cef9Szx#Lh_U;x&u6_DbE_H?eZ03@bO0j?N_xGfa
zcmJy$O%pj8xa8gQ{rWLG7_%BrC`3oCY!%@Y5?HUodNNqx5G#8?fx<_LITsdiUp<{6
zQXf;4=`3LW$L~n9i%wxfRfJ&cR58!YjT&KQ6AO8C*X;`A)GYF0S|z<UNJ(3M{Ua{z
z)m=Q7bXQNx>F(Y+E2XGQZE4S>Q<`hn7<ujHd7QLh*RCf;r;T<z`S0X&_>{_IZlen#
z!9J!pV>TNbJvQ;((zs%E@evz$mmcFICko!2v;WDm#p)uPZHP(ZsSM{px7Ec{OdDC8
zbk3@%dpI!q2phaUwR@9FGPmhc=F0~6Sxm1iI3|#s8|oCjb#uTuk-aZ>^cdKBa2Z7f
zcJThV^m%Le7l|a*h@{5W{D!1z-}ToUn|#G?rtxMSR!Ex|%dtDhIH_4QYi{c|1txYW
zPGuh6RT&GqPW;tvEYeguf47^@%jvL+Qb!U`eU8+LWwo`JP9B-V_Nra3qbjm}TF;-4
zM-@cmliJ0~vQ{pi7I;LWRjeyIC4Fk|agQvKWm_t|R%t9)^=j3s1x(tj*B;{f|5|&-
z*;$o(n{RE3Ts!-@*J@9{mzT7?^ei}TSuSMUd^+RMwkMN>_X`#8$~0PBSH*B*$CfIS
zh__qTZMu+Qox3^ImixEyk%Zv4Th1IYlYX=9XrJ%q^|_DF?cTL{_iSrhUZEHIJ1fMb
zUM~9I^~-3#o^WZ<{Wr^`^Cyel?^CG|&oWomberg(Kgl58|MM?1_b|n<4bmF>|4uTs
zRW|bY80OH<wqfZ37UT0T4&BPV8Z2BNeycX&o_JA3!-5wd=gt#jUywfKkJPf|^Q+hv
zwTec$MW#;WU;aK~N>Ap+7SZQ>*DrNlq*p(2g)iHUL+-jhW}mcVn{9;Z)-_B?V*WO@
zCR<O#`sn|n|9;-Gmlfh}Ijx+KCSfXD;LBNanMssG-K?|Cs`uh-mh|lcp>J~;mj9Yu
z!nO1u-!ZP1&6lRFF_O@B&lPB92vKB;oS`QAkH0PU*SQxTt}G9^dO@LFDUr2U)auKE
z<-B*L7+yz|Pf+sHR$_a3RcPuB7d4}g(U%S_SmJf4?W`tC*;mO$B4SGGD`p<)uKe^!
zLUYkV#*BlNwhyO!{<*rIag9bI`^%#rSX8=REnT6pT7m1Riy~XgS|^@2FIW|B8cyi{
zx=1c5qP0C`(WJNn9gWoz4##`KMV5B>a7CR+2{sWF_0M9Rl2j>q)}vBv|4gHFd+Dgj
zVVo8R3m#Z*6ghJ_=EZ*#NjVq8<%vBdT9>`ge-SE*YhG~cRKd1O4hGq!Uc9#}HqY8|
z^H|=k#xw)%9g13huL|Z^EHK+CDSK4JOX1ef-f})CzrBIyRHhVPXi=NgqC4SH@#mK@
z)h{Y_t1m5|_hU<XRacV0|2J#H|0Fs)vhCW=sM}&2#nB*{IbCs*!Ic=nt#hlEOgQ?9
zAzwAK=JD~2ja?lN3q{UGw`$CC?|ySJx%#ejrtPnb{JtKOMD@R}aF4V)S!Gb9P%}m4
z$ANU4#UiaDvn86<vPzV{q+ap}n0e^WG%b6jz3a7qR0~c0a@>CPFMqH9(hsKXW6?U?
z`_hM}L^wkFgu~?%oqkcPpGAo$9d`IH$Q9PybajnDsd|&blE8;0R|GzHIf}@&gy@Ut
zcv?7X?R=yewB@7NYAfftMswWS`_3l1UHdA{nUfTL{g+JDyrpY0WV^$U{}M~T&!Ky%
z?c0`v2Oq>5e_E}v?U2|y;p+@b8vO20_^2i>d9wM5=~Tuo+C8FC9TgW&7|c1M;>xCZ
z<I{@mT%EiJmLxaa&^mTt&yo!#vd3Sy9~CfbFkG`({<Q$J?VpExCTwz^D6o+Av)atd
z{JaOmwVWipi=w!v?3*Z?&2TfIU5hQS=FFtIQ%*G;44OLg<QZ*?;0;kzoTsUM^qCr&
zm3lGZ<#aA}-)Y{9pSCSdGhG?GHgoYrpNx_jKC<iH{?AUE+;i(fo8+qvGma+yo@M+z
zOyij6=EDBu*S_mG{cLAWE_95`URxLaq%5#cd5UUtcKMl|6-pu93$!^!zXw|dY2<g6
zFP-L)_0X!DFNK@?>>ZcqTd#GxEEIbm>!GW1ZR5;!FT!8Xkej{IeZ!Nf+vV;APt;JD
zB-gRVN%*yD#OFVEf&zOERbEl{`gZBYMU$=<%XBm+wr4mQc5ZAsJM&xIhloUFe<RT=
zk%^MOKZR%g@%*}KrCd$|dxXQmu2ZcXZ=51C|2kORmGbvpw_v`^=F|3^PeN4wlrHkp
za7vmiaI<>r#>HhB$?5S0(=XopxadvE5y6|KXFjmpU4P>+SK`c;sJ;KK7Hv5pE12vm
z81wR<*tUI}cg7u^rm{8lIfGVqbk79#?XgQ@3oe)H)u(re=9Q!bDX85G^vPKwEZwY8
zb+O+uY`a)lil-8L!G_1I`W1f@*c-S{?&({tlEj-o@8FBq3#V-hyp$co=~t;#Ff${E
zhk4zpGme=L0_QEAVyS*w^Rm&F&{s$GYbHLuz_>VZUESxVeQ$2**UywVuVB1(=e!)l
z`F*)-_qydhc*&c;Xk}^a>sf8_;=dgj3>X+x7QD%ncVM`XH-Ul2g)!Bhf3o=!|8q}^
z=Y8+_RLvrE;c_Ck{h!Ef+qicLFgvDjGc;(-lzbnsV?KXvpZvT0MHlu?>M?Pxd^hjE
z{fgZ})|0|rnHC-QTDbc8+tf8;s}ok*H(fHAZg5_8qMzW>RQ=33*H|hf{`%QQE#&ll
z<+0N8kk^XdFZ;INIq!LLeb)a?EC=**uTHxdpEx~1_8QB%TWg9h?ThlZDf_xW;13(;
z|CqqmobajjA1XrsFZy-rACJ~<h3eFAe|9?Oo|xEZ)vwLEB!d0s$xpNVRk9fb4k(<;
zQ{X62m~cQLC!2eZuI-J-w;vsj@Oa%jE0syVAkyGN#EchP)Glm!@lIh&>sBA8Z7fGv
zc$0*M8e(#~&+s`pmt-plIdGjiAe*;PL?uB*Ez9wT?~YZ6#RHDWS1H=(b;%t|5S7YP
zh*&89gLA3d|3u!1Ri7gotMa^DXS651c>UzpL-uH=OHzGmyC&@WvDBriQRRez@|gm~
z3m0}gIIW<S#-rlMbj>~0!Sjk6r*;|NbBQ%+Uj+0vwWK#3JG8Ck(7#8A7#0i6JeJ9K
zSl^~+T1ltzt^-HZxWkUM38gOnZIyV$$NAVfw_|3_$2R4$eAPehW|+U{yofo=35E1I
zU)0{JBybzO7WwR=m=<`7&#7dU@&^xf7I*a$r+^q8uCfyjACKvU#2ge=%5rf_;xSuW
z?$p1u>R{iTcG+di_XsW1xVC!F7VYH{C%*-)lin#P<A0>QN?o~Ved4)~izcm>wesbz
zaQQJStIqB0k9X`p{d}*6{MT(u=m=qJ;9`1FA9Z3)*4i}-`V@3MY&A6e89uZv@l|X;
z`={ruqWy^*7R{y_pKd%`ALuke<q?19tpoqsR@~y~R@jpo`0c!k?IzxD=U>%0H;EW9
z@-CVyc1X39RZ-?bbd+L;hjvFBtKurwPClnDFKvzeM{hY?QaPme{=iZe@k5G12Ffd%
z*!d52Yk4a#IUpg~!d|T%Tk~4MERpxR$K$@^&Mk*jpJ|-+Iaj%B<I+tRRDI5Ui#Z?P
zP@p*Bwe_tIm1`H&J{YM#(*Al&o$J|zXE}b)*@|_!g7tnC=-2fzvrK%^=69Iy@XG~j
zulq62P{?>`bXeG)*(~quQ7h)xJtvJV{tKGVTWP*w@iCW;BFTZK;l*p0INUzwH$SQP
z{gxM|6PM2a#VkJC<HLm3C=sWdTu&T~Je=~9KB}B&HD4-{)7Y(*!EK_IkkZeUxai5g
zK((Cr%NUlan0U#Y{8;XBNanPl>~TXIV<%UulPMJ}TK5X;+Lrw2T7P9(*Uw+-@;evE
z^D_TBa3qtB!}m<fnjgjr=eZoV^S2u%sg}Rl@Xt%-<oY9Ns_P0mnkMP~yVd{ySJwY!
zuIu$eHh$4E`{92==(S4VX5)^IbxZivgc&$|oEXnLE3!1*cG$qYdTB)4(aiy?U8g@~
zzP)&}-Rek}3pXbe*(RiK<8We?cKH-l5R)LJn3Kp{^xuF@{DKFslW^8U?&p^tt`g>y
zUAL6&*|%MjO>2{R_kYq0eW%uQT0TNic*_Bsr4RJ~8%@c})A905|7GC!Qt1%W#X}r|
z{_&ay7Cc$MLbBeZ81CY4*C>2x_1Tb3>DZ-bM-?_64bpjS_0cNtu!-lySyo%TwKYV&
zm4ZE)T+F{(3q6>tchx*2(9CnI6_?Y3xF;n+TdnRLj;;|Y^GLKf-p3u~$fe?*zop@w
zpNh=!VwZcbt!tgndbrq@ytlSvb4>E%dckTtf0c(Dug)7myZ0BpzFds@{!!seu>AQa
z^+#BKXZ0!+`mXg8RIEO$Ao=Nr(~29HN&~cnTJDI(=@%&lZ&G*uUt;cIq#78c>Uu@R
zOGvf-L_D)u{M+}+svA{eiWHMB#Tl#?->b{nxFT9ftfQw$vFuX#ila%_bvrNk-Z^#H
zUHnj@;3=^=#SFI?9dEhXJZROD(^oFIs&e9Kn0nLw8ton(6}6=fUiphuHx&0)IQ7k2
zonGCf*L>}1fm32dSVBpYUTHh;(JK?<ytqSCM6RgwW#*-(9P|CS;b4WIZX4SnG2tA^
z&4$UIhnKOvyrVK*<FMhBb0!{(U+GMK^<Z1uowAoY7lp!|UmwX7DLfP8dNqId6o$zS
zYB#x9o^rnD3xA(8(Y_#9;ahWIa*10T$7!x3EYXcxpIH{oxpMD|OZeenmu3G?U))h_
zEuJLh&d7UovNf|?iS6{#gO^Kdk4K0vl5=wu(R$nK;mCa}aAjAE?d+~A%SwN)F!f?f
zs+hYzWZx0_-xIyRZ2ooPl|B2X#vSkTy_5cUtyp*N{Mvv1an2!5yQf$0FaK{~Qe!-&
z=6HFHda3ivw2fk`3-9FD*ncY0FL!#dQatyf`s52q$yXSYO*qrPFsFwwXNq*06tQG>
zERZ-P_J&cZZlY)o_uLmtJ<MHVgTfZc^lM4Fa53d6d@<V7vtcn)W^;KvkG5{oiA{2D
zUc0VbP}p}xAtb`*`iGM6h_?70t=by;j3+IB7G*u1aKNckXIIj}XC(*adc4$p1enze
z*#18j_$eZ|G|g&9rqPZCMzVr4kD0dbVH3U^W};x2XZksCK4;FZ<gF_@^R=(`l=cLz
z))c*N(R=l9aA={6Qo_4s<;Pu?M|vHPjC*2dp%Rhx`Sb(jlP`AI=sf@C!cy7x;Y*QM
z)w+w7Zv4qB)OWk(tG1bnC5M;Ri^>Y8Z8KZzTBENu=T+(i7Oy6AyVXyk{BL^i;5zd!
z{Th#l!hG@Ti~Qc4nmR4=Qq}dVO2t3TN_NJ&D#w-noL(Lo`~KA5F3b4C;>nyM$=X-q
zK7Uo;`&jv7X*@SmBEJ)(;Dmb#n;2VJ)qkFhRtZaT{B&C)b-}U7j+n1=>X&MGC(T{`
z(&3e2cg%A3>;IRQtzf!e^+Bo1d%m{H{VzA27F<f>Ws0Bit;O_U|C`hOE?mi7?XFsR
zJHuQi#Ux2i5A*L5v!3~F$E0bQ=25*b9J#qN^^8yPUf$w+|BFt^vnkuQ8Pu)|IWW!g
zA5-SBX$A{+WZsyTyZPAkL#8oXEvCit7<Zq1y@gM6ci)QlZuzY~6DC)kF<9L+Z_PX&
z@l*3=vAjQ3SZ*RNZ(uuP*St*{7TJlGQ-7HkvD+*U6p2_R7T1-uqg|EnrmEE{u_ro<
z!%~u>#LHe*mAG$Ld3S5w%PQ3!?&-ltc<pDdeD}d^#i<SJTc+?XuIT<=Aszke&qnVk
zN#A2!(=MLc#2iw!_J_!7^BY_Kn^*H|S4(d>lF}S^`T(!iC0-4Ye*vvLY9ckUw`*>i
z*Pgz<^{)7~y{xnIk1P0XX}@<O>9%A_>Xq=G4s*xTwl)UynQlb1eM<IxG&}pCoDox*
z#uKFroviX)yQNpL%g?Z2SK^vwynem6s$E;ck%UQSH^}*Tak4+S@ao}28;jnU9hu@y
zt*MLlJ^#IrpUKC0#(wc*-u#Q?_&n1k)%K;dWM%UmY^mVCd?@`)Qieq;gV_N)@ymxM
z{P5GSS;l_cUwWdEN%B#r&ACgb2i)E^LnV2J#z$fE!$Q$VtuJg3IJF~K)AU$6yG!!N
zUSoESEe!K!Nw3Lovi;TUULIT$BU)m{UY7H}RJm$S%r(`?bp_=Ly?Zx*`MmA)&U-7)
zx%rE*IrV38CA~Ni9$bE0Co<xya^<BU8_#QUIgKJoe%vkVH@q#J(ki|>Kyp*6M|Awz
zRxRt_IljwTLj%6L7>QoGE_1l~j_CzgWmAQZRn3+&)Pt0+Z`~Sxd`7tXrK|51Sj<#D
zn<`)TpY?SsFVBTHPItHYi7;xoyT+G_wVv3Xdr3|I+SO)_nTBRbok3T<7rDNknw0uc
zU9>5MH;G$e2fMlp;|3w^)2)tew(~Q^low2z|8ZuT?hys02R-XH%dF)7@+G73x?}$w
zZqCcQ6YfZJ2ftf1G0f+`)Z+iCdYOmzhx)}QIK{t_R!jP?l|Hvj{a$TiMZRzbpN_;G
zgO3xk{WWuTvg8=9(NC$GrZ72E?(p;zZZ8?OzVdK3%9-^lW=)=|sEtSQEF-&o>E(0o
z>^N>ePlaWASIOCxi*G&go~I~Q?#{ko)oQ7EmzYoMx!djYxff<!Y|8XpwBfDLgSbm;
zRF<fFtE%a4-e{%yR@%PhQl8V9srw?1%bY3Ja+PRXZguRt!>)CVcN8A{)Yy__`}(8p
z<n>clo38Gees#ij?^6wxzl%407p~HecKCZIfjcDMBKprOyUiEhzjMAl{r#~&F1z3T
zeX>ne^+RTK4ZHm|?)vTg8&nz?QuW>EM!%ljc_L9+{!{zzddJmny7S#y|Igle{Kk`R
zja^S~vTRSVP<5YMu+?Jsi<|u>oPBG)_CNmG@BA&gK1OAM?4wPs{dG@w6@K4aaOu#M
zeH$nHysp0cyVWP&>F(>jPuUL0)hM3TZ2KP5!ZR&NxXxeu#uVN~zwfzkJAHv!bJ={`
z!V5~vl5~E^E<KsVuw|N;!k?cSF&Qj^|FrKIo-I54^8aDCX777uujDrRafvFNnxb{s
zGhrqpvsbC3#fD&~CVp?*CsQ^kIR$ZO^BvLHxcESikYtmL=B6b_Crbt{i^()va%!4N
z<f%C?H>LSZH!gNdysEYQ{CtP>Z#+^{uB>2O>@hLPm_b=OXbszzlP5$HSBGs&xpixj
z0_Vm5>zPGbc%>B_Z*ERI+BNm1Xrd5jVAj{kmfPp=-&Ousam(rQr*~%7AAH8cvn|f;
z@YhY(l0vy(X&B8umT^<b)muc<)4==0pJ|_;om%78d&y{7(bLm&P1OS@I5@q&=C#aY
zp-<(3DZ1N>A0IoL>h=EqzGmkCwhi7NA0L~XZZfZsd%Et4?&+R(d#%KcV%B7zKNtJE
zq4~l7beoO|dmi+NbOxyyE7Vqh`4AZt^Fr>S)9=Uk<)`e3w{iORKlRAHDqr3Ma&GL3
zH`v_RyN(Ao@|s1sN&gX@v7k{*uVZ1;pTHS{3>=ITCw4HaFtl_st1M*HT<daFSUY!*
zMQD%VHi=|6JwAoU{~63q?Rea4b4X)qNFZ;()Cro5H-kfU5~o}a;V`_}mB_JmvXJEJ
zlPZlu9JxD#qB&F*p3O*^G;^X*+`gqPOj&u`p3ctTQ()AtIy>b$L)BHyh<O!hN0!gi
zeEE{w>7Pi~@`d%^UiRyMGsyC`-MZ!Squ>d-3p>SvJSMzcmbd6<*s_c^uIIDo@46|u
zGH2PYTdP*C(>%%&d_e29i1d*x?KN8;Wo573>a_hJgA7A;%m$epK^^fEYOCLDI-_=5
zZu5Id_rNj{(G*?B8*!`OI>lymudR3>%JR<PN!^veM*exL^ITtMnOxebbF)M);?us8
zpuJztS+5s!e);<Sey7+cR|35S&n)@h?V4tCDl}3t#anwTPr_+6&Ob}SB2z-wI51>C
zRcV~Fw(-ra6H1$(ghsEqGF329d;1>oSfjw5lg?<erWN~r{c-7B($*d!DalzmhBHNi
zTo$sX&sdrE>e6zvzaK9vS?hj1v3}}{Syx4mo88XYUbr{9U=Qc?+XYGL&!bB?48vtg
zj;^eg6}=RwUM_z3YV6(en-68@Rn{=~C<Ze8)3pewojzUOwN|<AhgT!}yr4(-Gc6R|
za|Jvsn<cd>Dxb{upEu!&)HJ44KM$1#hpyO(ikWT!N|sCd9Mn!)Ik|o_nflhz=<1@-
zH80|JPu%kR-0v69-}3!Q)N$=u$`rKZOW}+})rE!s-J-ipwPtaJ=s%xzJNkU;QJv5`
zHq+1cFS@=!XQ6$=uf~<{?H0b!=d%A(`75aYZ||+-)8{5la$?g=zHvm%tWk4O=BtLK
zPQokPwmP&FXzIjNJz!ArIJk9<W>e0r1*}(vrm<)w_{zluN1K?gj=7i>zU@%&x`d^N
zZYw;=*<a<yv$H@)YTMd~6OR}+TQsgo_gvj>^TcWU1Xls2Fm9#$p64CKf_6+1Nl_4a
z>9SqT$+gI~MLG4rqvkCYM>}?C$C+~~`}i<TshSzs^W7=g>zsvKy=Q{(+JZ^<pIIDV
zrQ_-@8F(r<qVTvzYD=QdK`za8--K5LzMY_0Ehj7KZrrf(O{(#K$1@BHTaD*gr18ws
z;9aIOdCQfGPsa1VoQYoHZu?>C3h!ks_pfRlI~DB2pR(e}Eba7_8#!}&&32tk%|GO}
zEskrsMS5_bLI34Bj1K8DN)Mkao0hr#T|;-@yUt}kuY~7+S7@1aea*9)d(QG7EuJXc
zmvOMz-g$OD&k4sDdjeV%j`*EXTrpR=>v`+5P9dSCEUs~mhZvSPw9VsL>i&0=V1HDg
zvLfS^1qND|dfHr+l_Fb9xq4r`EjyyJ;Y*9HPolr}8k2?&-iF0F6ArPbDrp6#>Nn-T
zy5TiXT;0O=L*~L+4}6q1bjCRdDZW}g`OmrOE9`Ib`}Y)ys6O^+3uKvK#P%fQ|FT;l
zmzd2=n~YjlvB*k<Zs-wdvOAgZCCF6lTEW(2d6kE2k6Eo2ofi<kXVKIR9ja@ds~nFw
zmpLuw>8ryTuGwAJy7+}IPF`E^vowZn<Dy-gUaTvvJhGwH_l@MktwM7+*fvfG&5@Gq
z);GQMvBcv7liQOWC(iv-NwjI|n#gl)O4G@Sq6?PFh8cY_=%2Kx(kO6c`roasZcJ+G
zRbD*SV!~2EE@~N@ck*OedEN;yIy+&*6ydMeoK8fRt(m%hNvfNV<>U~*a09Oms=0fP
zZH;YuyZat*{>_F=vlOAvdyY3smp^hdPupdlUv507CY^iAv8U~-&vy3LA6aa1^z3(q
zf3gjASN~5F`fHb0$Z$-+RpnCrKQ6t40@wOmq-7rVG;sQGTwEUh!}Y?7k{Rv}s{|c0
z!!KOC`>u4oaYk3=@kdi7@`JQJa~tcDY?i&yGt};kkT#pT$hb69ZPQgDr;t@AZcH*!
z4*KxouLx_%U*4_d|D=zKE!(qliPDEt-leOit(+yKSLxa!wdB?G^GmMs+8LaPiQ_C?
zFKst-Ns+ExkKWHCQP(!k`fyHjizxfD1nI{|3b^anEnJXqaebGSc88sg($+VNT~zOV
zpUI}_p{~H>+C1UVrPvmw-A2AQdMg4qJTnTHx_#nU;OwecrG;<A{d<oEO}Ele@?w%)
zbSU!Ity}6|nlt3TYTWuiC1i>yZ*-@w$HLa=^JUy`BpPhhEKf$g5!>qdQ#aMZ@=n02
zXSe*k*G=ED*6Z~lp<S!zq?Ww@IoY&-_Svkm&AWE5iPg+GkiGLy)QyVBZB@d@_dGLQ
zwy$4u?oQLodnfQ!RImh}YsxMzxaMPF$*S|Puxq(fS8+#Uc+k0-9>%kupV@Ipc1__K
zNxcKF*L<B`^+t?y!-_Jytc`t9M?STzvvJp(^`bXsUO=PSl?ropq27ujn}a=ln(AVw
zenc!fvB>_y1l``t{r8_tUOc%-_3Y>Q3@d9KmX^J$is)Y%k~%AGn$Og}47cKoyZ<~c
z<+t<+Z!Pw^rZjC^OK4+hpLOK(zVwzq|6hggi+X*l!}aivb6?+|voaCjKi8?rbtC%G
zt!PKRzejYUetn$&Olbwb_RSMf`#u>)DnHXd|MTqnU7J$Q{P9=XFuCrmN}^HU#*M!J
zCpW&lnBez{Gb?ktdqVLPcfrl9+B=dqRxZ4mAI$x)(B1QdQg!sM%ieaoPRhm>OaK0<
zQ@C9urTO%${<ptAZ~LzQUiM7FI^EneFB5OEEx2d&Av&)x_*v<k%I)9YPcP^<^Y@cc
zSD1OiUisJB4b8J>#Q&*JeWb(K@Y}uc_tN_3ES4NEgnvB|i#?>Yi9Jd1sB*=ZMCCSt
zjh}^CTb21sRrWl#7Jl03pjZ@UD3Wu*T1X-%U^>Hwr^)}*%jFWL3d%Moim^58{0Q<d
zwT;;>6m4D-U@9E{z0~=7>g{Qc31(tx>0XKHDQCCWyeO;ji%1Fis<pn!skALUE!C+y
zO?v;O^wWtt<|WRBt<DYC&Fn+kN?S8*ruce>$XyT0oXQr_Tb`NMp<Arc-uFDSLL%#J
zQ^d++S!xdjW<3|;+o;F=O-{(L<I$JQ$xqvReq`AdT5%{aMsKiQ$zJ=#RAhFk;BjXz
z?x0TRM}b101CBkZ(fL?CZAZ}Bk79Slyzd-qIrqHfx~kam<29E`^Okee-YoV0(B{2&
zk@v0V-ruj~-&6O=ajyH-mVbL;9mDgw>g0ND!-DtA<t~T#`mxCWzdObE!4uzGE86Cz
z7Tl>Ql%DQ;SEV65$=67uId7qr@DG0-aTUg=eWv35rcNqMtk#Z7#rj4zc0~dD%k^_F
zaG9B@*e;X^E2`e}TsXj^`P9*%bBkkApHJ`?pJ){ky8VY}{B&VYi<CRgMf^7>#9nIF
zTvTRM;i#=BCjK%xF44==J-t}HT(8q9%7S6PkdE=o*7J&ORxB0u;w~`<Cbx^prv&Ia
z-VE;ypYlMkvhR3>Cue23M1)_Y-dypi^S0}CXb5$1baZM=jhY$Z!dPXg$hu^Mgp^0M
z&cjYoNs+*VY;2Q7yalV3Jgg5clUP_WLA7$4i(>b3mhNpUCP>_@)}0~ze@Vvlrj@2w
zkH>yui@VW2<FI2~&&k^J6U_=Pn>}ckxjeD%_1FA2?D4D;^>Zxa|2fq&{>b=weI`%4
z{6fp<UsGrOOP{4}IP3P2S>MgQ#m$=kJn^sp;;#5C(LAV8qpVRguz&gYM!jbNMu|3+
zEF#`QDq$D-G$K?NzexQ(WkUVTiEa^d6B6^d4wR{#lycrSH{EUS{go4LS~lxils&PW
zsC#0rY)D?3d-prb*h`ke&mui;y0)gLrt8j3-<2eN^hJ90_f|`hwv&@4pE|_J!sWsd
z)z;-E;QZ45(IGi@#)T4-TzR?VihsyXEz9I_%$$2{VT4O1@5jtkgN}u4ixxc-438B2
zuPL>t{f9uuj)W=GrW$1l>vjorL}W$0oSG-b=(&M4UvoxxOJ_}si0?s>J!#VtPew^D
zv<VIFa^RZrwsP8eQIYjOmRv}mbY;8g`)#p1E_TNUYajU0{aCc;$d3FIjXJNE_ssYi
zclJj-lX!vs!(J8*`O80gOQrf80}IQedP}N$D^4vhUe%!E7`&`plvy;uCUM1{Vo@)p
zM%JrEs@wWGO_l9GDVaSl3Q?5EacFWrB*HRNNLY9V2j|3qj1vBxxgSqXw_LGmO_kiG
zpR2qw=Ej<?u8*AVKi!t^i_MOxGQ$;RxzCnlrcJu)nNnobs$aS0XQuSFgJC9|I%b_Z
zb;~DPUzz;>N|%mJif+keIp@xb+ND#jFPh>Zxvp=zOI_;}>5mlwnG5|RBh*)QsC6ys
z+`7o<mB5!-g639Jf2_(fI2a*WXr=bq-m<aENO42T%Bu942J6F@upX+?*=Vmhv6}6Y
z*Okn!15zR)uOr2)H#(#(mEjiK88Ksb;*5@&wTWCky*tfL3f5_L%N7co=VX~zF=u`-
z-F#Lee)h|m2@7Yf(QJ6aVPN}O*y{GIf8q&`JsO0?6P|Y_s(tt0v2wN_cVoco08x*|
zohuvHE$%=0QrI}$h9Of$ZAT#AZIx$blO%snU9n0(^>wM~kEY1dl-t5{rLE^Wn{~gb
zl)K`(QEK)Cb<;JmNo)T9>|CQ3;#9-toGqNeaI4(XWWnXEwVbmu979~LCCPY*EWH2F
zerolUk{LU@-P=XG7S6I3oR}t{;<d=YYmu7OF1=a1m=mXJ-P$$lo9v3yi<DpO`mD8R
zqHES7bB2GbcFPMf1|`hdmNxDD^iJ`ED%Dejj_g)gaAQucfmhARZO;u;Q#P=s98i_I
zz46BEy>pi`2rn>=uJVq(y!2z4)0ePKNnQEpChj|RXw&QE%k+2qR2{C%zqRaqMOW3T
zy_a7sf3RaeySqsAf#UY86-;a92vvw~`x)@|L{a#`;<pz4`qLY~?@;csT4`M*H#;lI
zZBvx|)`PO99(ES1mV4#Je9#tl;JO>V`oG}MRr{x}`h2pqOQp<7>`?A$$1KBnCN}dd
zZx{-RG`l`4NoJg6=+W%(W@7c~601KZb=_;ca<ro?SkG!X?_&(l+gg56Yi*IJi(}_9
zr<0tHnZB_(3NdHaCcUYcy?due<<8VUQ<SxKEqc8u%O;ce<FU?&^(8jP%VLgyvDzK7
zQt11xsV1kUR%v$fXzyXE-ry3blC9}3A?V)PBd~yD70-=rMcs#VEg07x^v}?q5!|(Z
zPR{CiXZFfl*G3z&K8!v!#qm_Ua@@1*O)0NrCfzi96FqbK&wbY$%>}$RpHH#4aH2k5
zaQ1S|Epu&z9j(t832tfK+#u7L_$otm_e>SJZGHZ~6aNQwD+d@QZBtGbzPd#x@a&Px
z#SR-+1|~KAkKi^D*7Q9%A@+Io2hCa^@zsT=w@>0ZzdCCBCN1^M*?Ae;TOQq9wMQ!Y
z8E06I`09kEt4n9jy?J==#z{x&yXS9kI<@1I)4oSX8otf9WNGWHzIeK5L0ZSr{?iw4
zsqE}iTiB+2$<4AobIp`l+jcF~-o4b@b@H}_z2dv8eB^YpIur~q8|@X~2@|yHy}atR
z-uJHEMv6+JlP4woJmC;{;?VCUb8k#@)rvaKtvb8adV);O+6Q|k>zvP4pMP|s<|3&x
z3BCI-aHj|#*n3C2^}U+({nOpMHg@Zk_&k@s{$lo~ziBhKypc7v-amWi>Hn83qQBqn
zjsG}{XM4dTi!IH*y>^PM|0O&YSp;;hTJbMr1;ZPqw>yg7X;d?~_D^0F^~ob}R#sK;
z#+3^)&dIV)bTFK6yQeIO;plIcL#jIu`98mO?(yNLl85tu-*QYl9NT(0GH9d7&!C-E
zW!Jnf@QGY-R%6_B)G>Ye9U%k8bG+JCB6ni~m}=YCoZ59kTV?XC+Vb0L%a0Z61o5m5
zyQ6cxs_l4TVQ;ZZ49~IKzjR%GhQI4Qrv3Kbq&D5jW%sjfGQalT|1<Y~`Jek=vlf@d
zT#lG26fyI7?VR;9*q7RJ8Kzop5Qy90o!hy0nFz;35%G^_R$dq0dSC<Z=BDf3C%>L?
zI?Md%|M}DBFL=+5wvE=5Te|$lqtmhH8F-t86&SY$=~%hkP2KP~Wy9mV4UbblJa(va
z%Dij3!lrgv&yyW0aZi7DKiOIv=yQFNWccORo9-;EE3@6)=)8Z=n`Lowy>=I$nrBb9
z=RUKv^4XC+n}y7ZCjN1oc<Ag!o5r(e&iH#*Gj5%&)b^agC+Un&lHiHiVrN!5up4gM
zT9V?v>Q_^caP=e6>hs!h=jZ)-k(_II?5&qpYm0{8olUFe-d#E2{M$QzHjI~InNl5I
zEsJAQ3wX6+-K*vEUM=YpsNQyDlbrS~*~zDWIn{|BtuC9~zWU<TU#~N6?2yToiLXhB
zKX+Q}mfTaWh2mKPW~W^K>+e0bfm`m^*<-~XyX@y4|9+8`ecdi&Te<IsyT8wUyG46_
zU&nc70oK1Q?m~9@y?<(^-sp7O6nQ*)LuJVUFRs#!7ON+(yQBV3>O^dcqTID4+4HfX
z*G%^@E_x?@IoI4_-(%Gatk&~hiNwEBb@(Wr|55D%t6IUw%wQ(YT<<L1Q?C1#y7}$X
zxZl0{&6AX_O^xEGOOv1Gth#=JZ+Y@N^L=klv+NfVO5bwl%jf!Dfv647m4CQT<h`k9
zc0gPG0QaAwnXxxDuiu=xw|HT!%_ie>Z5xffSKZagxt;j%y=7tAAD$E|p5_-mC%vX0
zn(TKnJ-XRs#+_ZUho|OU*mCN9?4iBqdmsNdF}b_u-uLDIzSr6dt0_oWnY`Zg&*}Hd
zF!OcgrjgEhruX*CRWu*JcmIT3yv&qo^}e;G6|?>Cf5@Hk#B1F}y<gwv-d}eA{$s!6
zkN(|1A8}l#@xeE*-E}!x*I7GfB^`J^v1*t7G{)@g!X@skn-scZ=6Ijlup>Oj$l}9R
zm6salUQFP7{E6r9)u-pqo&TeBuR2v$j6?s`qI(~187exwV_FsWQ97PU{66cm_<wTu
zKdN0|eDnUFs=~jO|72I{{ZCc+cmE;Nor}^-VxBN;bUh`a$;2r&rzs%uAY+V>U*Qhc
zM=tH6nsHAq2`qBcla)@pb7`ZGyPmT2q>~YoxKB;g3EmbGnfCvH$4ut5TYEG&Kl7b!
zUH<3^XUGcwIWE;=u2mr~f|fDwGN`>(k{PnxVrAUcS*2N*R4jIL-My665HUL`vRC!?
zs;f5{b8qljWS8mO+3E0jm+Wn}bm@(2Km6Kz+G}m}#;U?en=)sJ&Svac%9XlUY<cp=
zuMO7@?Nse#SvDuPN{+$GxLvoOU1LM1)bgx_YkcRYotm|E($%f;%lrOHE@!#EGIV#C
zfJ|w@{nhF+?&li2KMAGIvM-mD{q^OAjqjbhHO9|<g;dOae(E`PHmPt;@tI-d$W`_A
z&GqH&_6c1z-#)yqcmE&zk8QT}{`5cj419Kv7#2NsQqpu}wfxwzgi+$mi^c5!Z6C3y
zv#~2IcH)`LwMxye=U0LAqO4a>G-vwg6fW~C@!HrUny$69hd*9N`51rln-m5a_K1%w
zWwhUHTrN|7r*MU0>=nV2K@~nl9%{KoMV=aklY-o5GaTS-WNbLV+1Zp;nZ43r<K!$K
zqw_gl`e7c4%!X#3iQQ8UwY*xbvmoc-$!WSm8#>QF%=#^Ce)U$hkD1BF*J}fegj?5c
zx0lZ~F`uI*@n8qj%<h#-jLb|I^9Y+wUDxv5>qp2Hp<M!3SwcMD9%2deec^ERb(-qd
z9>3TPq05Z|d|zIAEb_p&k)g$xp&_Z8mx19kM^OLGMy{%|<_vbG^$Xfne)%bNZ#%jo
zI`6-B){2D_1Y{O$=~=;Ht~%wK@5*kUOs&ji6<Zf?Km2w}^egqtJzQKGwfw9S)~j+Z
z1+3uOXce*g`Vxi3vb(OV+Pr4VuSsY8IyA-BJld&{maW@&TIkHOzUP-tKb~-XlkkJh
zhn{3NR2i&3`j9c{nvcm!m&LkUE=+lFHtoiRC%)zj>&3XXEGU<g(pWsZXv)?FjjdC5
zTvnTKAc${DvY*x3mD-`N8{!vF=b2)qb;0(dh=)QW!xg>-xeSw7AH3c1yl?kgmWnw2
z_ut+;-fg%4cVE8U2NwMgyALgwC~%l05VE20xV_iPg9YmBs~&LKABj1}W6J;0@tp0H
zjrW9Czf$@<Q;*+vx#a)Ylv>y8(dz}jT~B{5=y|>HlwtXu?D9?J+QoB<JRh!=sr#{X
zm5A4AmTd~G#&hm9{i}TGnx<9tYW;HVYU9oOb<c!(IQAGGVQVN}^RZEBwT{j7OPA9Z
z2+wG^xT<jR{0!&U?62w;ecqn?|KIED_ZNsbto#wlRT%w(Z;#niwgAHomOHZgICmth
z3O#r#bd^^qJ6izLoB{)_`Lj6JIYqFs?Q#rOeIfd&PPLJR<8*6w^z1tmQgtLReTXoT
zVVD*+xusd-U?lrYF@e?@0@<!9O1G*`Y~SN5*f+(^NnWwpH|UbU<P|Ml`jMSpi#&uY
zU#7%;3_7*vO{;#j=F*<@C)fA$nHv1Ryy2o+MCIZ`4<e>o%-X0ac){iLF(L6OOV9V$
zAMBK8{<*<n(Hh-C&LxQq#?yDdQBCPO;+?GWQ{uQun@-Qcl+;;HZD&i;%*`eqx@4!^
zx}@<^?OMkv>N6)9FV*a-3$pHIS!Kq^ccI8q_|B~5B{#RO3w&-UvxU<j-*}OZoS#j)
zcl51mVlBHE7tSp7ZZPguJ)3dv>`Z2d2EMH+lXK!S88~nFXng%PZ|_v$g8JS44}LVy
zV?3$uB&{iMlcPo9(uA{xB8Sr56igIr(^Q;oF9t|isstGK^(`=W415_;zWBn$6K-o>
zJH3f}u6$ebrFx*_C7*dtDyoK=p6;JwKX!aq(N6qt?9J@Z6n^N@o)>F#s_xvr_idk9
zu=UYQ-&_s(fC8-(4!4}rUQ0LVnmydi;MP?i_<LDK7ng&>!b^b@ow7nWEe>!><S}c6
zC~$7=(q?<KDtP{`1;uf$t9XyCYTEeeLFtSjw)IxWT{K)2IYV|dGBRaHl<$4WJ9%oj
z^f%M?+C!Z4<u;#PrsBwT{#W8X4yCn6yu6i|4qn?jX@`qyo_6d@t=^q%4eK=1KB`2v
zbTZu+om@RNtYq2K3Ht-8Z`JTlS(Z_x94sVuyoK?-W9Y`*V3iG>TXJ}GMVR%XjMQd!
zIlkHca&p9xj@!}wxjqK{F_FdmPbH>rTWXlT(Z_mD`sA{LOX>gDyz$9DusX!^=6N%B
zWBuUw>T@phXjyK5!=Sg)_<l^&hpo;##XUWm=Lhtub7m#azWqSWJX_2n?-I)aQ{gkd
zdb+Od^+zu?m;QLLpUrHJkVJa*iAq1d#l{7{9Pd03QrFWncsEadpZb%$)#us&PEIf@
z{FrMPzDpyoPuPqtL)J2ISIylYB@9<RpM`DNb@i-Ynft!E@1~qtq7rVhefR21?(SMa
z6S-VB+;@4&u*iY!$)PVxuYKP1ZIyYN;dDvs`_4Rmi`(x^{5nN*n?ftgzDf6f%bC^%
zOY<%LkTcmj{FNz_$^sTM2XE$!E~{0~h_c+2l8xDUignqp1kM-(hVYv%Yv)P*cUWw^
z_OzRRRj@{m2WQBM#$8?y)*b137?-^wYwx3iL+Ve{5)yMSMO?S4Ty}9`BKsrfBTn8O
z`_vwZH2J!yh|S(4@$f|I(icbAN{_^b?^)!%M0dSz-NsvSYfg%@dbzEVxpI3YYx&$(
zBljZh9QCJlJuFYY%+OOdO0AqIq4YFXyo%e%;8OGxC4SAA*$J0SrlvTb(CQSM$+>j*
zToEo!-=8O2s@0~f;Cy<BUEpJ9_ofV=rSe5Nm0Q<boIZVL&N-heS|W)|OP^R6HfKL<
z-TLLrbekuP1p<>yA20-5es*EXvuPVz8rZxg)a}H-pVzEW<!Tl_EOBi^^^|*w%bvT>
zZ4s7f_<v;S4rWt7OKzo;c8XIL_7%uFOMTkay?92lN!N+3(JL446|!2qbvK9W!tRM(
zB2r6aZ6!T+X9Z0%%epwB>dK-+uQR4_TYEjev~<^<YTg<ArfM_iY1Za$diirrw#Sog
zOe@N}jQ^d|Uh$80)yh)S{MhAHGKVh~acdmZlAN{qz}jtgp-HUTk@`Cv6T)U)a(ZbJ
zE`C>YjbDM~mNika@f;7Nw#`~!V=I!_T=(?4M!ja6E&ux2OCw(S&)*o7@aBP%^vzR7
zf-!UV=5#8vO!~0AJ9Z)K+YHz4+$(aeJzuq6+&nE;GV9p1Ue4-M!CBqWmopSoq@&k8
zvC@02wT{81`^lvl-Io6y7`DvlF|_H|-19PtYmWu@%$s@=+WY}t7XK4P{_Nbm;%6^Y
zmA=oy^&Ct)W4Lx*)?zl=u}jxL`ahEa<Ludt-UiI89e15nWL>=>^6=IjKX=O4^enU%
zQCwqSXJ}REWwkeRm+GI{e<c|HJ?LkA*x%zZM|Xv~$l*gZvnGf~cb)Du)Zb$$Z_OoK
zxv^=bOta?P)iVq)8Oln|w7wB2s4`jhUZSAN<GI#THrdSCWV>Y3B&}HmPLc-R@&?LB
z^o?EKZ0vUC*}UvU_nbxCrpEJkNJwr<Xgxi9=7Li;w=WpmKQ?B3b>!UR&0Z55qLf^<
zdpH9U_p&=pb*)zL6JG5uEGq7>n$h9EcErsEp1b?*L~q$v)ZFpOHt5gPAC9xdowiCE
z9ouJR$ECa`b&hG;7Sr@Qj)#<-ela^~Z{~b4o9SisG|mfdO()#Gp0I13(4SbnM)}jU
zl;3t89Gry{7k%s&&@!6-+1ox-dF`|pi>iC}e}6n7FIxNO?`=Png?~45St^~_;yj}+
zu}e3j{hrhODVlrilDe3-NVNAXarHS;<7`v&+1zVZ&(w?ElYZ{!R8hTkLh|bap05v#
zyOjA7MO-f}Ub=dw!qwI5U;MBT?on7Kux#>az9kp<${w!Y{BwP3km;OewVAV*?R7H9
ze6*9}x3+GgSALUMRJU8qYPSW^U2NPlGUvF>=4i;@^?$=2$6Y+i%XvQ=@Gs{7@Phwe
z!E*IK9hzGXRc>0Y^Mq;b>s@PC3y9qoka#>{<81p=mF^2VbQdW&3*R){*?stM%3O;W
z*~5FBm2dA>t~_&p;u)JaEA~yE#C~~_N=L(m8%<FetOg&ArVIIYha7P-+5CGF_otnj
z?sJ4sIm^%A#JQl!SbdhrN{%g#yG8E*HVMe=TX$+|kfip59^1%HQLoigBM+KPynWR5
z^(w{(wo^Y$W#Z}!ZeEzO#c%&DuayP+4jS?5e)OB%bnNYvQ`*UPg~I+zA4<M_eeBik
zW8Xx~jtA+yoxJAV>NTIVr<d(nQ*lPAGDUoPgnjPglUIc<GzaO5zVXiD`R{QlanhgF
zT`h)(mv%H63M}DRGnq4X-2){Dh7=Db7QLQ@7anpg`Jc>sct@|75cjc;C1DxtHidHS
zneAo)JKQc>1U+f`p><GriGG&I+@xDg^IIHKXIV;d1<lc(wv>fw-f1=Ety{U0Pwi|p
z*wq*$vxRd_iNS&kiYgrq8XXP)A318TJ$7B;kU?tKn>UmHADX>b_;ityx7w50?+Y%g
zyPVb#nXochMtHI}d*OuLdvx{gT(<jqXub6A(+4~E&(UP_J!~doWNBh_D>B5j)p^n;
z-P2DFPdsFF;M1hQi1~WEJr-uz9MwKM-RsEZ6!X~<@^iGhKXFd(vy=~8(aOE0^}LQa
ztHyuB|5v8O-kh?ou#qciOL)s(J`d5yTNL=T&Us%x8hz3}V7F8h*Rhu}`y_Uqd!5sz
zZGHY-&ateuK?j(+awjzzUT6ut!IF2iXX%7v`dhEv76^`3KD6$QuzXO{;YV(1mi=D?
z&dYOc`{BZ&eCT@e?`djXGJjkIj5^%^_L%7^MN~ggYGz$ryKpfB>xq4T+}mq;OQIt>
zr$#KhVbp%{3QNuQT?rmtZz9;1Y;W6RzQ4k}cD2V!1`DggB^+Bix!(lN+~OH~>f(Hf
zoedv3jb89=l9b7FQ7h<jT%vL+lj~O2p-I`NuCa0ZFVpZZTzg8s=9KK0gU213G=7Bn
zUSL{%G%BxGO<(qZFiS3%;u|&X-(DuX-lnlu#YWygCUAVIP-h4{{=dR|(?a!~PnPex
zx~pJk^vVa``)b^sYI%w-Psrn5A*MO;YUc_?Z=dbPKHHP6cg|k1r+eawCoA?XzLRrw
z)A|7OXo<6DpN1TccDelK&hZQzqm{xQK9g;4-SE=cVRLW-XGXJ8#odQrL~bfiy)|{}
z?b%arzuw}%Q}o`P1u1J)Uz%)9s|`yz;<tY3vCPz4i?252_qGH|u!beDhe@;=b_D1~
za4D||S*hSZOUOTCi_ym)Ze_LMm9p^_uJLacPkUc+{^ynJ#eepFIKAdKi+!nLL}kl#
z-CKOdkG#JN2mCh;xF&VM&}#8-tN$L(u>q{Ln_9AGIG(z|c{Sj^)q(aY5^Q?{_@{dC
znJ#gXy6E|9yQ5dn$>@vTM>yLxdFQUUxqhLz+*AF$7R%Hm>EwwAHP$Xuwhdx%2+FYv
zn(VbfCd^L0XT$QXK}NM3G9C%fpS*#SJ4r$8ks@0cheYe@srNMJ#g$#yWH_Pat;*$w
zg_o?Xoi>T8oA^30W$oO=BJ-zWm-*CPg2MLJdoR~@$?P$a;9i*Q$jY^Us;<>EF1gzq
z57}HXDY?@slyYe9?myZw?zva)tn@iC_lnl$&M8t2XQwdvq}m9a^SG#LeCbGvaresJ
z-Xq&2=D+{=#JcdR_m%rGuIFM(j`|BKKA)ka#qi(lrOdHcQywPk+>>VNS~~Sq!_T;-
zOEu4|IkrUfdaf^1SVCyo0@jo`_NWBb*Rrf0|85oWT76V;`ozOv>Ab6~*Dmi#T4gSO
z?iGW?;Et0{O21sTb#ZVqf4p9BaZSVDHI1TcvnHpv%1*4+@#x5n_+cGTvU^4KvKz;D
zMD)#_G3o4$e%TDZ((My`m-Mo#xS#627~AF5r6jaf_;7`!&@mStMGd0}$7$6E#o5$%
z8+t8LeL3;m&dK+7W?xGEaMw{^)IZj4@;h0p)k|J3>pi8-*ra=*MPtRIr3+dEJDBD=
z-dnhBgFfHm#d41~nj{;$C2#%0U%F)13O@nsrQR0XZcpCez4yxhS9`A9uKRPl+DyaA
z&1YlSolR;n>XS7(4>|8kPMI*tN9VDS-N8G)5%VuDnK$iI%sE|`bFN2x{A^Yh95GDn
z;oZg^y@6-p0b!#F409{GL>@If;1+wOvhSqXy%d#Kr;p`4-Mf`#!z~Suw5Sc8r4kLH
z8#phqy?S4hX>{>T#@%~QYkSXj*lBj``}j2NQ>_2j1Jk}v)#=ojP*ivSz4Y`cGw%P4
z<@~$kYQ;4rCcXs5qg-rfpZ}Fjs0~wQR`XyxD=1&d(-s*ku<rubwFGv(bxyMq_-xnR
zmwMkA=dfwP3;w$84cgtJTMx`EoMFNJ_@l?{|6ENhC-S8h&6adJm2`8`ekP`aM*nYJ
zll!pDr9geHG{@|q=bKMumqsnnC{W*LI>+Jz!~CFy0ZEIZAI;9=dehq!IDyMoq9L%O
zVJ6!QUA5~x%QY9XB^&w$oB2j-^jx+Ko%0}1#yn3(^zvo7^~v^EAD?^~f10ysg+kHt
z50CB2oK^lOtA$^%w|nB^mU2x|sQ=S$iGxL6x>70=8gx&DT<~!@^5Kf_z9$#{#sq{O
zG5kGmhsON5QCx=kH@RiFpL3o{E(}Xwn-jkI<#n&z>)Y<Aeob}wUfj^=me6;qkhSE<
zfqPL7&t^J4(~xU0Jn-;#L~i8aH*Z6)XSd~kd~h@OTiR#VZLI?7Sra)DPhbD#bNt*Q
z-$!gWCUg9csB+m~xy}8J0hgYDN3)5!=Bm=Vns;rXYa5SzO{g!r9B@Kt+KDe>-K|eF
z`*J<{a%X<f>uy_T?)pmER_V?46-)X<-rHW{_Rfh6vk+4;cqHxDFy+xCS+|u=_lutk
zr>)E>U*=HGw*HHAW9H;naqQKmi#``{xHEW4G&5dc5@hev_!lO-Zgbo*zLxDqiccmp
z7+g+%r7z*W%V^n;rFM_E9O2x`GhMnrYt^+wtM+BB6SJQ4B7RxL@f%$c^?WgAJ&Rk4
zxHh#_PH)#Zk(%AdWTS4m^8K4GmIDS}Q{?uZd)>3<$zkqLWA?j-^F6Lkkvn!n?oi)M
zYtAMA4=t?SU-D$`t=Mb-&#lVusw^+hF`A|kP@?-FOm{-lGuIxKw?R2)tKOZ7(}~W^
zH0#}Bx$Mi7x8-wlzxk|r>C9hP*HN*HW555`Prpw;+~*&5|NP<W@!x`#YgeW-#6`S)
z@O4#0o{`oCCO(NN=ehno-hcc8lW*uvPO%pg&L%wSdcd?~#;18R7;I{{N?bguFj*t(
ztV7fLX}T6+9SzE@Df12&IG(TkFy~9u;fH$Pel0w$CbIu!@%?j*(^-rRK0bLUq_(Yq
zU){+i-+*bq(X6mX3g;BpigYRWHeHfv&KIrM^=Mj}lk~>o`g+HxtIwhqd;53j-{x6e
zVK$Y2&W!$^bZ@g`A6Hi=r!sdiGFkE}{P!qekZfiWFN%B-kZ`zNNZxIZ#e(jmtrDzd
z53hWDe6(B6w(ZXi&g7YXn&GQ*PJVjYU}TtFCKYgM*;ym=@<)4i3LHG7BDiFW(2Z5o
zoWq#A)M9@HXkP9KV%1t|wRPo@Rm@wgTD`ii1O)|%Z#vl#qI4<DbKR!g*@7!vcI2rm
zT{K!Hs=ZZA_)p2{@CWyI=8H4Uh=^Dav$KOyfk8mAbBadPB$b3S6%GlV?X2g|p0P}R
zz{04aGI7C?EiXB})I0*u-YR{4b>s3Qb-JmirsxK-`8=Go`+NR<-d27EKbc2jPq-&L
zt4yBbJ@a|W4j+AI+h?V(Q_{SqXkPgBz4cn4-5rTZi~k?{^5WuYllOD2|9t;+^z!_S
zdwUN5es$hkH8nd?+lMt{N&~A<#uR5Jri_FTUO&+V&3>Lcrn;ntdR$}36=9v`uw=u<
zQ}yY#DvZe<C$`>9bWkvxn56S<t5Og17Kz2q^Hjx-O1#;!NZC3lhV4eWV_TX;qH9*#
z6X$kD1_l9dr3+6c1<dPE>B>8+vAEk%?ODK7W*yB7)8pi(F{Cc_Y`isd?b1x?*};3B
zrDf-xI(eHV_`|jdbGdU5Twtg!+Q`a&wD90Tu^67H7YhzcMqOAm$0RGmiOE$;WvST9
z?Q-?M60aQf6fZrc={0xR6<6`amqfB8C8kcgRBpQVmDcL@kDjbb=XU*nVM<WcQ?G@g
zVQ*Kxb~;ex`f%OePhM{}?q|~u49v}Ws@Q(c>~)B)jFrkF=l@$bMK8@&ytp<YD5FVy
zXQk)C>r)@Bn!Pl^A@o{A^3=WEk{MsFSw+MM^Li>z<IDXiAi$~OFiV8hd*ST7;5o}$
zL{@7q+>j+;|7Jrr|M_Lh=1M;>V`V>BsS}fbwA1EJ=t*1iPbF6!w|^=#+Szl#>9(g-
zyodgNnYikgxwGQVy}4>L`K-g?=NGD7;!9sTnZE7W60&_;(D{}?RSt!#oO~P)gP7Ci
zmIm@V{n&b)C(ia;(3&Z!51lg>zg@FAr(%X;2lv%WeGVR;f*ZPwp4lF_*TQ)5QChUp
z{|1)?hriuV)7+b8THI&ybnw!C`$pobm|9eb#A8>Zd3o`_+1@Oj(cQi8$@Hb|H!Wr?
zo?K@YUH#oRWq+0T*C%i4h2zp#0zD=?6Wg1;sa0$*_kPX~hnDZ${h0a9KI`l=t8b>w
zy?8qE`P@n`(ToN4LRlK$+mvqoU35p>WS`?iEtRW|M#XMLu418A?A%ovP5yg)?l#Ht
zmfibA=>NvL?OOg7N)u*ZaZwO#IGxP4L5#(5sn~%@QCFuuI*{U<5)ly@7H056WM$7$
zZLf)9>5C4nEj|0xOYMqy_o4$_CwQ}ke|O1p-afkSFl(Rg_bl&+H=gQi>8cnn{kc$#
zPtD~)uVO{C#{WehdMw(SS^GQrZ=TJSaNDREwx(-S=Zx9!T9)fp^mHHNlsu`BB>1dj
zjb(Izq}0aCo8D|F?0vP_<6_%1&ADd_ZETpO#P0;l&uU%XUvos{64Ohu=nH2ZqOF%)
zV+%RKtjLia>nxzWUQN-}FV|^u$u#FH(*${W8FVTeyL8_eGVJ19BI|L}DN$$ouF$4;
zT5Xd(?(vmdoD*sD5xT(=(Pd<ARHNNyRHHaOrp5BcR-t&wlQD++%a_Ory(`=1Gg++e
zy7i;LV(zPJo|&v&>1$Lp+47(;hsgBuWtV$`=bWvMn{!XZw(-Wwxni7CBsQCCIOe_&
zIQp<*k<=rN+O9K!9G>xK{_`CYYn)^-fqmzBNAE=!0t~-|n0Sh=_7W8iJ*pLIS$LJf
zpewNZPL|@^ALlnUr0T>ddPhi~S>V|ls>9`{qBY}8z!f%wgY5~~4F7ko6w7<zT~Mg!
z`O(YCBs`)!@Xo5`)~X8w%QXb&ZhJb_IOrn3vam}_W)e^OotEMg4OdsEl?sRbeI>Ye
z*BnNX{xt`!vp-(2a?w!BNn8;*Lsa#f)V+ipzu*fq#2Y`me(Qas#XFI&l&SmJ)SWz5
zhLe*|w@kaSfVJ1APc-VB1b5`DyE&WcdKTBdSg|hf%voKYSx=NXznnO)*O3)<dzSLY
zLt9y9<}ze#K5<jViOVRYm3LW-l<!He|2xiqRAsDOEo!v!i^{FPPf8QK!e6o+Uh<HQ
z$7uEiCLyEaR$*38O6PN@#q{i+u&0!>WEr=bQL?7>-h-^g&qRYC-3eWGSJY?r6&dC*
z_l2Q94wdhV+pAwIpJcS@)AXJ^#l>R3tQ#2Q&#yi1to~(Ldbn;>ihuQ?xI!+~#F{4m
zn%{5a=F0@x`JL!laQD*&mj~*TI#o>S<R`{^J}IB{Z{v*6Kf7P7nbaoSF7iQ<-AAri
z&ST+XlMT(&dLG1tt-8dOwZT4l<&sFTqTrcaM_S#VzU+RLI-{I1A#SQy*7hfpoa;PU
zXW0HJdb~(WQcbLXb%gEau**HqOjn;@l`PvZyI(`V^1pQo&#%_c=5Oc7sF+<$$zIBv
z84xU`6?ZWy=TYz1cfU41YS_5`{pk;LetOFu5OcTI307WqP<Z~E13uO_G<Cd`7G-wY
zOuHhWc)t3>=HowO!X`A_;lI2@iPh+avrDJRgeN?e*=|v1H5WcDRaWFYd0CI8MO^5F
zH|wW`$)&ne7Vi<8!>KMHYT6lh!*p5DVx80H_iXa0Pv)5t_GS8mb*Ez^oThKpG*vws
zI^E~|ow?O_v*V3=7VZ1mw|?(&h5J>?DI&}tzK1;6o&M<H%>Am>n-5l-r$5uEVQ_L0
zJ|g(XhSfiT;fnVbXSuh_b$?{eYg_ev@fXP=w?Os<#@0V48!yTVPS*OL?ycRqB8l(f
z<SAa6Yj^EjbHVw5#PlZp`@0^#>tedJVok^rX`NkcQ&SHn|I~V^?pmaMgy}Me?~-Nw
zg+-?}+f7f*+WY#?t8HBZA0$?;+ghIaY}L%GYrd^Ll_ShuxI5$lpPsdM%e$g7<9XkE
zuGw)1ng&U1Il@$Lc<sS5za1;%jyoK?e)PNO!}`*aQ!J-9c`v@Bm_KJtN^_dY$$d7T
zr!75SqZKU|p)F#&D(+`UUL^OOZC{S_DV1%I-Fa)>TdC|7U%zH@vmE3;x=m+}N^9G#
zgl&tj%sM%3xex#P?1}roF1>v;cjD*M-A|MA%x?!Pes}KZwt8iMWc%Kw(<kldyLTax
zt$f#i;~O8=c-}X@9ewug{qJQjbnENi|Gxi!+4sK>{p<g5vpo1F{+fY7;m*P95}Y^c
zWEU`goBm<K)Vz+#%t{hRefy8@oc#5L=X@T97D2CX9ucjfGZu8n*z+Dq=a|wy^??(I
zNc>h-&RI7)I*(kcIMVg-{J%2`yr0ClU18yq5@nijQtXHO<*yM}&Rps94G5IE>HOy2
zW!<~1x(~1XiF!SiH@b%@xbkkOPhxsan`m0(l+gGP-iv~njiGbz-s)AHeQ<7=Z_AyZ
z^PPXpm(pXAu4}8GIQ6cg%-pLB&3{VDO`E&$tzzzhV{A4)Vh`K$KOBoQy?3IeN5b*e
z<F!6ZdD$yiu9bJps`&4*Nc!v%DN(T%XOAp)y(agQXVqDjOB-&<pS`$xs*>id0FfNW
zI~7x|Xq;&D@XLI9OZ(WB4Tj$DK1#h6mDb+Uo1NIP$&h=qXTV2aUP+yR$6qdmWLy_J
z6cC@|HzieZ|Aup4JXwvbj%*Kz{;KHrhNo{gr>lqv&z{)HzdDuc9!l@~8vW;^^xl`V
z|9qYO_v`$>dlk<tkok8utbNN(CxHihIawIZW<KRraCj$K&pC%FZ}Q2M$%h_Z6|w05
zydv02Q(|7k`FSdfPUOTKKPh~o@d5j!J3n|_TpA+W<BmQJjdPqR=YB-S`Rbg1D~`CW
zIkIZXsnsu9T;I-FbLB}FLu;9$<GY-8-TysdEV8T@1tmQ-PxDU`VHH&MW|iI~Iy-pJ
z6al51&NrAeOGHGAqC=O(+~}1}nQ&8Z;arJLvJqvWf6gwpS8BO4_v!70iBZ#18T?vo
zCNcfp$Z8(QReWP+?7h3@x(ecNpUzM0i5HSP_Qlis?b5iHQx}^`{c$-`_c!#3sC%;2
zqtc$9<-YC}Dq`mn=BfFKRhk?*x8u(F8l^PbmKU<VX$#pM_sDp79ep>UQ8SW7({^F{
z-9`SF=J{(qeUo}tFYQS7*Cd~(u7Mqse6~K@ytehzRSuoA9N%L7c7@)bWODhH0bj6Y
zd`Xey3_-U}o&($0T>t4Byl3j;U$x82#boXt?Xh}w|Ih!q%PW^Hm!6~&9nrn(%i`LP
zi|;&5X3Cn(DAa$5v%k(OrpdGaOUQZa*o2}VoNR74=9eh^$axa8W&uZ5gh$tc_Etex
zhg%^fE_FInx(ZLNxFW;5>&ZVG7lyA#-jvK|v5*No(wp@$;;xN=;5&({CIalP6NGlX
z@Q6IZCAo0J#%V4(H=TT*h8DRA<un@p%W!jGOX-mmoT?ZzNl|j!)0$nalCu`t>{FiV
z_j1ZHfv8PO?;Ks|GNb42(VMY#q4S=m=1mBVXL=?kaVz)9v0Mw5f+trC8Ya%MlXC6q
zDY>&UnU6iyP0gue*8DiJ<)Mp~pJe&dvh3oHTbi#X%CDT`srIUl<!H!%3-N5BNx^%T
z=_pN7yqKi)cS*y87wb5sgC(T%`(EYgMGIeJD)2iJwL-`!ESZzj#nkD(*&(eYhp7{y
zj`Ymv=!#J;6ADkXnaVV=lgaadxff&kz3Y|drqyU_OP&d<Svs?(kZphH$|YCit+cY$
zuP)y9OL|+`>c2OW-FBq+?|NGGP5$oN#ZrCGE7qmXWlfoE$bL{#rS4gx^Rmm%zm{8X
zyzC;Du_Q93B~it;*59shiM^p$v)v0$p_i?*k`xv$ads8)j+x?fh|BG5i{F_WzQ_DB
zcHAz?aD4JbWx`I4Af?pHyh|@neX~RCsY}PI+-EDU=*-mJz|^Us6Mdw6^}6l=nU^8|
z8y|*FbgI#Psll-J!aM1hGL8c)&wFHPPFc0|Qdv+8Q_Mv+-7A;kuY~Dd{T6>UX|2nS
zxsqzfXKsBO6V$cq(9OG=b3$kO#C^M)`{(H`p?k4gS2}4ed8D{(@zjZTxg<{NJWGC-
z`DE$Dr(ahl*WFs0IZtM<RK~f$^nX1XQmdAxOgn3<x733%Z04nGro|xvvAOq>-rQ9)
z2%4LhbmFD~Yi{jX1_t$@IqROdIBMVZPG;at_;AhNfG_op@1vZXpa1fHzN_+iV&4pv
z18;d&Z_GNl@!Itq(frMF>p!aLe=57=yeCn*tRq3@?dMPO|E|hM=w#a(t<N#kk0{AM
zwQl*=j@3KXCEEO-rf=iNa_Qdm$!rPsLJHSs`RqCNp7E7>o{!AzJF8ikUc^Q&Xco<I
z&Uw&Sq|nsW-?GU6kldPPtrjNboGd$*4ntnYT(7oCTxU0O`KE-lUwd-2ZH0e8`mQ@(
z4UGM5U!FvYB!=sBcg@<+buFXkoQ80k%N5C3ZK<Oh_UMHE=(y(l_~{LfwPlKjzP!=c
za8TpUi|~%40@5GDH#l8T+IZ#fqm!&3FGd}iY~?a7sB7vp?XC0jw=>+y>(h_irW(!0
zIlbxSoSipkyt{dDhUTFanev+qXFobN?}_Lfqo_I8HY!TZxvRIR*y35Si~BscB;~Y-
zCy}=ncqu6)ySJTnUs?OCPA5<O|27qs`gVnh#~1xN>g1v4e@aOw%(o?G)v~ZjDi@z=
z<Z7u4%L-*(%ARS^uz_ino{MdXm__dV)$g<pB&7FVC@NJGew8ub#bWZzmc^!u@3~B0
z8+5yV+{OFJ?DhA$!srUa^p^FmCwzW<y1zXyds|Vqsk8pXxR;@8vfZ}m9~4w^JheJ*
zmFHeJ{(t|T|NC@o-@Dbv{$?MjyK~^3uuF!D<Bei_*$vE>9|ta+x6EeEVW%8krx2g4
zf)2h5e2hoeAM?|nnz7`i+vZ6Hp=U${uQds)r3Pwsb862GEc$Zl*RAFV)(!*K{z;)l
zP2EOm^TW1XKX_#NZGploM@^ob+2V8jgU|C*0{_1&`AyDt%V`gK^DN!Ru)57q%zy1q
zw_Ka3+|P>?+s~-CZ+IBIT=n`p!|Q2xr%%eAF=_4mt%dU2-rmacd8jz+*ax;mF{Q_%
z224S>A{Y0}+h%&-OHNUG+H=)usahe)cLP_gofRE%<-#7BCrK{7p<H_!k7}Q1eRgQW
zvqR5ZpJ%8w=IOV+3)|3`Z^e3L!@Q5Vjqi8mt-e)cocXQnuj{*S1~Z?WdzZmzs<{64
z7DnIA6SCG%xV>ka;rDqi1s;(-Pa`h9adz13`l2kaUf-m8{r9}}X4(3M%;h4VOs0M-
zIWtG~Zm8*;!nGFN%8{Pi&1M?2Jks9Br13xR`~M_`1srb}gbU>UE9)@5{>(M6f0xOJ
z7c=K^7aeMA^72k7=w_VYQ?#L@%)s~%N2u44kezouyf3&gCvt_GgmFA{InCSG77)?X
z`kYNQ`Sz8f7X?cu9lK#DxMf{{@8y!XHDztbOSbRWkic=%<%(Ip{Sir_t*$otV(Tr;
zo4qgW3hNG!6ZQV+a^zv%#I-Wh3X2*RhLt}J;|>hJ>~?6++lL#4Bll~{s{brhNW1ZA
z<7EF?bJ>iP{eI4~OH$-sHGkbj#q3!%Tlg1QeM;5UiU0i7Y39Mx3+4D$zEhpwDK_C?
zV8FU8#UPz;Qp>n*d%wN4v@UDIk!iyAF1JHk&xkyHb=9|k?|^pl-H-!Eqt+ZP`#-Pj
zW0UVXSH2bxzqo*^29Enn`Mt9Z=LUH3=H6Z$<gwY`xY$H~+M!S1O!&=rRBbdCF0aoi
z?B93sOP#Uk>E{RT{wg_~|Eto}D%t#~<j#G1EH<GkHm8l-B5$0yxOtO(7ssbyjxv=?
zS+a|ox|{>6-C9q%?s0JppK?@gi<5KKm(u$TOBNmEomk)ehV`ce%S@5o?N?YTO_n!*
z*%iI+!^I4f{d@r_>IWk%!ZLTbepflrZ^Fr}8fOtO=kkXzm%BTfegtOj(tq{r^R%Pk
zJlkDj=KV4;mC)aMdEN3W|DrGE)f_*Z9P{m4PWIa4v-jB7y}kbL#f>`Aux!)WYCYlX
zI~N@+wEmyj^XTHnM5*Jl-4`FvJril4c7NN~gvj=br?<wGOQi)pIo;}hx&B`2wpQ1-
zKE5Z8#D6}Xt`pD@YgC;ms@*clmHXIw*=<kd7V7G$>YIJjkKzcPa`Y<CuWYAY&52G6
zSo00^UoUY#cjwLCCjol*U)SEZm%egQ$t}92`Sq9N@{U%A?$zOc5^tKFaoMfE)oS%`
zv+r^H3faw{<<~un-{%rz_^a*T{kXZ4cjeb{%2+NdeIHZK@@x6T@C57lgarZ_cCF@&
z?dR{lE3S0T@X-5kuxUvZ{}h(j{%vtAD>+<x&z9Hbf2i%&*qw52EytO>oWBjc62Eer
zf+DtjQ?_U*|Ifw0(BpsP+>e!E9^t&&Jl;>6raZau#pD0(1N-;?X0Q1(y*_Tq@5^sR
z+I8k$OugVyXz0MBVA{}ofRUYD%x1@j2Wmz<if&UZK0eUs6w^#=>6plSj8VpxuVP2R
z<Rg=$0`=}Jn)Gy<p?2&c8RLYWX;byL`+$!soolQ9;fm&$=OTK+Y!X*jmOKsAW1iz=
zs`d5dp;ZBE({imAXwF=gI<M#>SLvFF%?T$99IYAN9Nd<3^iXX?n&RD#vWL55FQh8o
zZ?3z!Yzl{jLF~SYk4$1RhK@pAK`IX4EW3PHN4<^ODs>@m;j_+pl1*|CEQ^m^V)txL
zyV@1%*vcZlFYZK^$*b0_uWn|g|6hGC?cVhI?Yr+}nWsHGuAjfIwA$g*vpK3w)BZ^+
z<#yy6McHdqD>%KqFIkrE{3`L_M5*)h<7Mh<tG~Z_xPSeQ34acKQNMiK-}2d$24D04
zi!~4E{=41ez|48ap=*J}oFpc8#hL&oj)`7b&CSbpd{E%qv}*#lp2dwv&U|({f~)xb
zYr2F*!b3J{`H4jtDoL_81gf;<v}m|bS$D{AkzS69&WXKzyH-s;+?TR?ve6H%r9FOZ
z1wvlx-kqCzw3eo5G%^ZEFm$%=VmaV#z~807X{_4e(Z#rE!lJVpTP8H^znHUejcByM
zsyv2UTdX_I-(>h}yvUyQEXVE(e|pwgc9c1<{eRB={hnfmyC2qkS>VRMO60Pq<yn!X
zcHifm+css(YrlP+8P+07H#LmDF1Gm`^4{&gM3BBpi_R(0>*0n<uGgcVC!biHxSRXr
z)wUBFCl^lj%N0|ap3(Jf=}ZBcRm-!*pJ?9JG+JY@a!wyp_{tS0j%uyxoblA_Zt+di
z^4RjjOx|%@&v>1VtGYbN++f!a-kybrzo}l@(7<}QTWMdy6YqzN3t}V^)A_S>7N5|V
z*5hEr)|9vTo6GAnj|2TzuYDYv+<kUelh@`P#d{I+Ow4<y9X5XE%Dy7~*-Xi(b7p6k
zDX-lYD9$W;<#od1T$}4`MxlF>zgoXo_(!3wBlFjaK99`iNz1JMXGd&&XLWwfK{JmV
z;o_68zFej{%k!=8kJ-Tv|M~v-zhmBc%gK>VyeHs2=j2znqdWBW7#tDs77;umB<OBf
zDDw2Q;IZS$J%1FAt6slo+ur%6iF?1<{ngwmyCZ_WsL$`YRHEhYKle<>E}4YWtuoRI
zOe|8-!i!90vyC}9%lw_V=0EwOaVhGJ%v!NqI?n$p9#6b49PmoRsqWSM?<`x*J3sVj
z{Px><|K9JZ2i-lN_lLheV|7j7ZgJ&0<6Tz2p3ij;RSFXp+FiTiVjlmE6`WG43ln{r
zCjMJ<T=`{E2V-QBz)Xz~sxg7mO|Q~)bH6UpaJ}i!dg!RgN|RKrC$qdJ^38Bq^S@z(
zwu+2m<UXPGo7V_v<t_~8Uj0c-^4Ns7_Lp}K{0kD7eHNfErFe#~^T^>9o;eKLw3A%Q
z#98ls<nx`dvD4XOp}^UMV@4q|=R~}nkDm2l?~Zs9?6~E#)b#_;O!*()oV)J%%J)By
zrR~m4E_m?c*qt+mY1be7X6)*iHKFsA$+^oupWDh-yzelyTgT*o^;qz{NKcmfKgB9X
z`b_sux$-=RNs-wtXX}K&u1oA0AMrOhWG-NmT;aI%&g^@C(iX01xZoW9;6jI=%Z|eb
zy$f}c=IDloSRQ}4$t_~xohMq~=gBv%(`Wa*G`ZvChD)iOrDswkoMkpI)!Un=cqK>H
zWx|98mL-}0)ump}6{!+$%>AZkWNr9dY|>Nh)*bo*=Y9u&dbHj6RbqR(60_*}oW%a#
z*epe<&*}Dzr~Dh=O<tfWdH;t>(7u4bIT=znXZ^j$JkP1ot4FTz>QyPr0~5a-uitxW
z>iV*)7h_phN2D;X^!OV-agq{u<SDN(25Bduna93NyjY~*yP#G0K_a(?-ODgPy^MuR
zEmC7b3}5YZSRpuLMfOUrlIvlIiu6}r%WnO7Xgb%=(!;fxq6vSUe0<rN!#V^El?5$4
z-kh0oO~s^bZFXsP@~nuBCysQk3%bzc`|j6~9l4I{E7rb#|3fNbk5Goc#%G_RH-+xg
zulA%C=e$X-7LGiTc50*k|J{%6MN5xWwfPtbwaTB|7QCsnudM24<t?SNYVw|&{IjB7
z-*SF@c<buc-mSCOg#P>Tc}w7&urT9~{rOW}wYRaBI}}SVZQhi6ic?oyvp$)loynZx
z;7%c}btZQn2~1b&{L-Pc|4pi1qvXA~a~Xz?w<aF4Xxu5I$1~F_=-CUGbX6VO&0az~
z)l8K?sovb1S9WU(!yL8+ty7k)^kGhT^lp=-U!j}A!K|qc&5CP$+2_CXxw)o*DTAGX
z$KByD$BTTXGfP;5D%`ARz2tS8DJ&==pK??r)9_48%<QKh4Ce_*-gLPkx#j%6<#mD&
zb?ewFR{gyhrYw7LiCfcaIi+BgSl>L)|IEon!R+f&H0)<S2-*?qxG?2HbK=VccK;j7
zENZ4_q8uO2Jj>)P5jFL!e#vK^yG4g3^0pc{n9QiD({S0iD{pOJM2Dbi(zVWj&1+lh
zUN7XGo8hwl>Y>)WlEt~{$1X;wZDPKVo-fp$c{J}<-`r(umcE~n8L(Tz>hR8x<3BWJ
z*&pJsT=h#wadnz~kc^Px(}hozj4z$|_V#Cpmo#UN`p?(iQ9_fq9#dg(-IUVtBg|nz
z4u_B1%$O%e({D*6b+m1Gr1&!I`&5Olr4Kz=rWVY8;o@2p7+P_2k<`W$)8CwXy8F8A
zV)4Hj9!_gE$4@$9P-FaZ?~>V<j2P{fcI?daVYb&k`0;=A*;bVq<;)#-T@IWSaCF{$
zKu|FK;Axg56@wWEh3;9r-SqO|1a$@u2|ouj)0m<O>JC1RdGqY9FE$htH(Aj0=)R-m
zrdgjm&m@~q`+7dodsSJRS)ia{(T0@ns5X{LB{|zo4ijv*PX6?2N$7tK-@l)O+714O
z#5~{SR+D{T!+Dh$>%E(rc6?RHi=7aCe-;z_G1fHw?cOVWOpABT+jM1ZiR!A5+|MB=
zg~ZM}rhfSHDeC~=|4{c&OB0XY5scukbzyc-e6%O;+TnwKa;Id<lxA!*Nw%$@;Pc$L
zx>I!T(rw2Dj;5`;GN+MG+;sNLd7QCzLV;5+cDYT^P)f0JTi%hWD<A#;`lj}|mSwp&
z8hYj27V*}fJ%7KT-aF=@Q}lc1gVI~Sn>BDoL@Ql5%GJ(Re)n}!c;T~EsS&68^Ad`#
z@9r<}|6M)v-oX>|_vSoo+5g+tN-OrI*D_&?DIDh>+)DO7;!>uR=KUwzR^!Y=9l!kv
z?n(yl8C9E}1T`@zCve|rcx{nvX=ZRVBQ4O%fW>|SvwJ~uz?tVo$C%Ueo-Jv<meHK)
za`;D=zIEA}Z;vu<!nC&+=>C}ZPN_=gnnu%NJ*6LWc%qj1oJwlqZcx|I=+Kx`@OFo*
z=A=cZ`&ONCd7-tTV5(on$_*QI_5`$E@zMRE$HT6#|Ke!d7p2uA=i_e}sD9Gvco6X4
zuxb6;Gb!HxPCu$J*e~YCWXSh<Myl|whV+8uuO@2+9DOhQ@eA@7UXhqsoztfnb>4VO
zhw--#w#({DF=zcn7MT|G*u@lNifC@F;n{AM#PQkSn#RfNP3uBi&ME~J-bj0TEy2)s
zO7G1TOK%o@oV??a{U%kT4e!Nw_Me%zXzH%D5(jzz-a6`XqIdc!-@9`r-aW$Y_F(Pb
z63?Pj2i*2`%_z{gzr*mskN5WvTr3fD@)4NeeImtsk>^7>J=x2JPo#8I)$|^6_&(Iq
zV_2`L)WD#`C7|@~V@Ft$!Yub16{hE57ZzwV9aBrP<m<YY<71GP{bSbA&0MV4RG1de
z(^sA2m$fMG{|U20uj~$mHu&sF>k8YDr?iJ}*}cOj=X7arY31C$G<t!6rreRY9a--l
zJZ+ihs&vdJc8=HkH|NxMGp%BJuQqw(hi9p6Li`^aUW8nEEn>Pz=JiF*uL7L`O&bCP
zp4pxM{Do6{`y|2LleEQ|bi&!JN}SXCl9N;XzNzV&>J{3?6ew|=%qTmbt&pYEuznIp
zhGE0og=rJ?Ul!?C8>r5CpX1kUF!f!g4Ab=+YRmi%<<45QqwLYIPtU|>Yflwpx_#jN
z?^WI0$NK+l@|HZGzuP6)V8`mhe~0#5;PzN2=ppuOuT#3>gYJ@V`uiSbuQ_ALz#3o|
z*Zx$6$6U7WfX2HAFD{mTn)rW~T;AhP7j3^8%CG0F*mfXf8HbWbftBwDE7u7u>|C4e
z)D+n#Szb%8ZD3<Px=eJJGFvpOsP4vN9<DY#O`#{)47r7qER6K8KYr~NbTcTZB_+vz
z;soQhJkeWn-c~qiEh|{Dujk#N9QDhsTwgXjhEGzz_$IdN2zTqJvl$xaa!#~enUHX+
z!0^Y}do0RKcb#2~l^*)Fy2zhoY;<5UNo3xlV9cj#zU$KGj4RsCYrhnH`BHMlO>yGU
zl)U6m9L7>cHiAnRiYW=NP8Oaz#kO(7LLp8w7H5&qAFf|8ez|6uSLDWPJ!>bK@atz@
z^!{k9UK%L=&~3^ZpPhc0YTCi-TM8tSU;g)XU!v);q#!Hh&ZYBr!Z^7vo^_RYbZ1kO
zTiHtg$GLkNj_7XKsK+OIU*zhHkYdkW52W{B^kT`g*qXvHN%)~bpQ7GX3r;^5-@K<{
zht{k(=@a9A!N)}PTcJ`=sf=op?}E?FteX@#H?eSEvaR8J9@Z1^D`S$vq-_evHrMET
z`EC-KxGww{+p9Hf5zF3%SG3kzot)@5Y4VTfjYWs%EMa)Nu7@-6)7wYyG#=<lb_%I~
z@8MRRqWXz1?h;d6%nJ=(|J9Q|om*lQ_jGFgB^AFD3+Bm8RNQBzW6!U%JLt==squ+t
z+%I0T+_u8l&sq2Oq^gX7@Lc`?qmLOS{i1h{+s@2q&wlk^ajnP;i)-P%{y*IeQ}R{p
zDzj63uVrr2d%cAJwZi0;ld_F2i#P>oI_niH8cjA%^_*s7T)4$sa&1A2gnP`ySLyfr
z?}beM_wCU?BlY4w7PmeXvxQ>+mC{1JzbG^mGcYY+$}eX4HZe<d9fSMUQlHW?j(6VY
zS`@=Vee_ISUOJ~cG+S90JbNUzARs62=gXtpiVcgV9{tsFtS7`u#q(7|K|sXY+JI(}
zSIg^io-U2ZiYsOHYM+|Q{Mq)9iMp|q<3s_+EvgR2PdH~TT5-em-Ijm=SB)0A5Y_D)
zd_&8fv`b<SerSF0D0cd%iBBdd1(d`orpYQ6*j`fceYe$Y%cd_ZW&ahO`jw7Ma+eM?
z&+>aV#k^l)T9v@1g4+kISTfC~l+<yp+*+b9!a7-`e`?O0RJPP^ui0wsu}ekg9ut-3
zym?PJ;@u@X@k6;K9~}f-qP;oOHuB%z&}$rh(s>W>v_~d>)z|*acslLd7LPEeyQgv+
zc_sYkZoIqbqh^Ck(?-KUX66L3uQTVEl}zaT_{Z0bf02A-JOAEoGV*=FrwhIJrN~J-
zwVeGpYuA&Y>)Uc_i%ZXZ^xnnNew^jVq=v|r_KHl?j`Y$FTY-H2K$}$2uwp&avcB;B
zZF;hrp;snP2z#Z-=Hyj)CPF;f&RcM)gIARCqe$-}mFang56TUDE;yXwi{3i*|Jxf^
z@=vkGv~M%{d0gFpdr35tw$_!H8==Jq4)tGk(tIdd&be~}`<srs?+S5g6@R~eX80D*
zylrB?x$>Wr373TuZx&fk+!4^_XWPFerNi$#pK6y~Sjxs5wq7^&|9<kCoHtcgQ^lh!
zrTlb?zUTI&pU2&b8or!!ShjY$X7TS|%hFiW8H_$p?Wi<9nlI@*Io3(I<$$Tuj@L6y
znf}$NWb!xv+hA#G9^q+Wk>z^4H8#Rv$;o{=!OADhT5^MP*w1BmPYyC%H~03>S!cx5
zA{46~7MrizY&AK;x_Zi_`QjODX3u|o+Z^@5W?}e`AWPfcEsE>gLIpl6``y%(7gs%M
z5qEya{~mdhI!O-aHB*<bdAfL|PxFd`KKF9l0P**yM3=UH<hs4(gIh+M$(3^{yCT_c
zPFQ@5@w0%%$px~@wx|6vz0xMG)MBpGw|zxu|9snHT`eWYYMym0d_Lcu+4J|3D#yt0
zvr6h-9kx+$x^7TZWL=ni^!lVNA!|-rtx;ZdW8OROK<>3u`SkBAJ3MGxxhCJ}<M%0l
z(&|I6*LQ#PJ$m+z#FUx*J0!oF`2XD^*|=>(ZsybsmcXw}&H6t!-qzmK&&lxh>|TXO
z57%Y3wF;YGzSttlCwo_N?)z@W&>d~JXU;0wA@9ChHv0zu#&z@94~S2y*yjD=2-B^O
z&t<kEm)J`AOw0H>>(~9CviMk%Boq5ewZ(a-MfTrV^2#`Jc4*Pz89hs@dsfa?k@ZwP
z{m#ccT;t9UjY*H99nE7lUx*C&9us)IL~)9u#FzC)qhpyP<EvIU=T_}|dAq?zc|Oy(
ziL#cHeo9aJYrS9f(fk;GT~05L!^Vl)h2g?p{w1ZW6w5-NU7A|YpRD-u#{qFwmg!Sp
zf123kyZzGjX_=MNvUe(FRW4HAuD-J=ddb`A+MCnNQ*Ph+lD6`7TK`Yu={p-2&RoiP
z`;XX-KQAnl11`_H%$*r&eBa~yOdaW&WlLIeZ?taS7F0Yr`($L<p`CL$^R`@+oO|5z
z^m)nIuO^06&(LOBV?8B&{=b!ZicIq_&-wrCij7j8$kn9KPVq?#&bKRWEkDe@-S*8L
zMRxN@x0+pZTf6s*FV5DAV&zcfl#Nv|>0No$Qr#o^f*+STM~SPsR+yE&ez>y4mrIID
z-2ZR2_7oNK^gCalu+MkzgyT7VN|h@VnY05BU2V2nF|~ZbkCk1UG_JOZteXFQUYc|0
zl3(YF*Z4aKZr>rLs#RVyG4@(Z?wZhz$<zOy$a`!1ws~!0pmN_`l{vca<HDR?ML9JU
z-agUe?6BhYmt7u5WL<9DZjm$DP^q$^a8KiW!99=G20ScCDA4HGvC;IqlBWnWgJtPv
ze^ZN}g%<uhf^#Dtlzvwzxb;A^j$hiYU4FwB={SqCTK{9c7SD_B>Bx+pICV#urP}u7
zHDQgWbr0?=)UjdzdS`Osn+TV}hJb&!&fkjEvbxB)cc18qH&t4vryc5T?uyRdw6}lG
zgz}=x><urT{^-9Q8b9gNm6`J<)HZ3K3f})c^wsz3S3hUJQvWH)DPw7uytyJz*FW{V
z`l0K)okLZ^6sqz?{ihwO*4?r3%j&y!3w!U1ES2@){p?-8^hmq!5kpRU6RD$#Nw<@)
z-8NR*Ww7~J_8;d-k|yV^wH+p(nG>0D)O6*&Sc8D#6SBuL|GQq{T<;O_SxfKS1C<9M
zd%vA*KViGhtW1TW?R3jm)(c<rg)Mql7&Y8DV;^w#xO#7ie23+}@3mH8%6sH?+-0^A
z?MnQ;ZF1sKo6Y6Pri&J~w0TNqDhN$3)VzMHdO~gPtqbXP(}R_Inddi6S-`pTsYXfN
zAHx?8cUvznUiQ&=?q%D#AL7JcCQ9kO(%CmxRe^y)@h1zb2m>>N4g&)NLj#ioBLl~O
z26he^kAelOn>d8UY&<rcIS|Aw<0cYuY~hin9JVx(9fc0Zo0gpDkrF6U(VWD^c}*%~
z^}SP3i;wUuFm1T%Q)ZU=Y|Tky)q|zXja;%D@~&Dea+~-|gd=3Zj->&M)pESHXoaqN
zHNj4ImDfv#($zEAbTzUv!&Wk#uY0@8?CllC=<v?y(2|CW6N^6D$r=c~ZP--6*~et_
zK*Q;9=us}7*Wquh!e$-giMY8b%r^cl%dIaz4HH+NWr^k$(fv?-{*vZx1s2~4Q(rNz
zjpNT?zO^hPY;DrvzSvd1D~*;k@IPpoG}+mRQ(sKw-j1E|#>{tJN|T>2DCxMe_OP5n
zY-#zsi`%{bi|bZD$bQ0dkze8RGZoF7PtTsd@6%;^(3e@2+eOai(aFG9f+^gdvwBP!
zY(sck5(2Ewez~EwhJkfqOY6s{bDUZPkKJ&HY@NEme>0EBqYj~SP7gcFSj!%E>1>;E
zsV(v7j72@t>mtMw{WDiQaxie)aWlDl<0D1wH9dg|8O%3~6+G9pY>~~p&Fj*Y5wa^|
zY4=*6pZ7dp8*X1TT||`SVonOz6F1FEOF8Gw%Hd;(oz3F8{fcX<!}2>*HDy_z)(Kzq
z6e>#X*{L+W#!E9)`y0=LNe<noT!ej@Y^^RY;k7cq641rFB|NF3jO*&su+~|k0a>Z3
zTI~z#R=n!t$laysw1Dfs>-3n-eaRwgAN5?B-nyHs<gr6)tLY5~<=PjTA!eD0@wVG;
zC`(DlD6vPSum1Su=GGmlr-dhO*g0)&?#(#wWm%7|eVMiGMcB@_>A6RaMOPo<O}SVl
zJdrWYaW2mUeV3nd*-K{?sy&gNd4*Rd{@|yRniFK1xz`9R_~2_f!T0UWu=trwu|B$W
zOmDq196$D^d0S8T$@+2$50~rdKm(!OB8e=iH)ce!tUMalF=rj~%5%OmJw$Y6&s|dN
zcTqlajluo*YLgJ%-&eFl4F0p`bahYM8qv#ksVH=bK-M8$$FF}bspKpch+3L!Tf67W
zt*Eq_Oa05O#JA5>IQ?DVZuxG{2@4)7{$KLp{y!s;t9w2%t%|AVJ9$lTtDNGKc^>t%
zS}p|zY;&35qq(uTXbE%fyU80_bEUSfU$O4)%WE_9Dps9gl094H6nRwjTJWn4s(o6i
zPiIV69V#0-<^A?svlhj5zuo#!V|^5_)f%=<CvGLbk?@SNWll*u@71PvRWPGEOYBzK
zU2PY=AE~#d2e>WEb$_>8pjAYu@cn<wfQC<8DjN5hAMBrUXf9Kqc?Hu%-&w1a<qxgx
zOkkIpQRZSiuk8b;rN9#L`diKdTuD8i;%oR9TBIo+U&v;$L_y>mi;=>G3(AV>HQmiB
zOHCC+Rdw7eW6vyWn-KrhFKg{@uB#?T4~uucXxE8($nd{r8<*0n3nd;Ug4b3()0`IC
zQ6hIMaamBwCaqUFq9&5fzDCLu%6>3c=S%lW1Ues6DLa)LEp=jVpQ}u}!;#)XDdE`{
zSPn3MTA%;Tc(TmP6Z?PvJ>Mg&d1BAAud}%~X+%v5mJShHx*}?W6VK_@i$o%~?BSaF
zRHK7&iP~bOUA~E{+LwBJT*;U|!Kdw1n7^ZH@3W-oF`j{G{=c7Gn${sOr=qti)@PFL
zq3uda50^#1G(Nk1SE#U7qkfE;=k&Ce&b3LK_$>|<#!L2GR_DByKTAZ>Z{HTpWxkI*
z6W1rsko_p3Zau?u4zq&7p?4kAJ{4$spWJfoXrA!W*qg@Hd@|=83OxTGD!a36lIF#z
zQ*4i1)S`MzIF2s7%CNcXBI~hIW=|=;@WL&dALMk*d=$B)%59;H+3|&ZUzbVFPrI<e
z<?Ir+&<+RLEf?3EUE(3*a(2;!mlMlR3FnGGTKe*qTUXGW7U3_>C+>tjS#JAGzv0eH
zUb(;1g_aqarf6PV=@_wOmV@aL|LVzHkA;$iPV7wAlzF1`?Z!e@{#I2dmdu&YEpFO0
z9x2G$x$8-vlh=iSyj39!mq^ZAk>qu0uj#_|x4WF?3Qgg4yt4R_fNLZ}l<3XOr3O8M
zaw&ILO^98)D1wzyHhFQ=*0T>>6y8W2NZ&3HRmXRrh;wFG)<q`OPk%P`+;h%yjM+Tv
z>;Ic!D^(c#_WnB}^DA*{-LFX;|9+KTv021Ar|j?yhUs2@?VO=KpOwOkc-iC*GOlS#
z<XSalNtff*0G-@qXAd9VtHHe~Z?-UfdM<K3O2G4Ni>bjA#n^@00$NWMWbIeFyS^vf
zr00>(dSjPIHLkDDE{^!t>F<(VFDP_Gp=+AxDJ_r9UL4X+doLea5h&icc-2xf22-W%
z7^%Wt6Xrbn^|nFXujAjjB}wxStW>xYd(rXTm*|c!%S3K%y;Q%bNa&T}YE!W+uHG__
z(@R5)*2%V<PM9{w`bp#_r<H32)Eq0r1GI8a<;5Ds=N(Bo%6IJaB8h8tzdwbqY@Hcl
z?zK57Q{(=#6RH0<^;CvBFWshnL_(s|`QC(|c}8s$e?$a&R$cqKNJ%$ea`McFOlsQ?
zO<!BAxNFAUtZ?lr;W_jD@+=x&T{2Ppk(d)^top}l)xTNHp>IxIo4^t1TjLeG-QLZ8
z&YKo5$rA>z9WH6xPF{S;ru=Vg_uHKca^9ONMLZ9Mtw|B-DO?yUv6(AvagN$|olW~4
zUd_4w*Gsf#r`ha_Md$l+rTv-j6*RE#o*AcEeg2uVSwe=BchrW(Vs93(+x{|qU-jeg
zbk>iHGFf70#eV;N|52Nv|5wgCQ_keB*raDz9C7B(f6J{q`SL4dXKGm-6g=|i?V0%n
zteSS8m0Aj~I6Yaq_pQlIo{X0I|A7UU)n*E<dc&1=V)Ds^6g@6Ojcw1&msdvalX*5<
z*+0DVtZYIP-*nU0d>eAMeVcro@r%i6myQ1Gf5tE2`?~V-pI6cMEax&BuhIR@WsvP>
z^l6LTmy3pTH~A=TUdM6ydf(o4w^<{1re-$_pS{>DR=(U__to9*sxOP>_r7Si|EKpu
zzt72JieSein|l_02K&r<mHVCIc1zycQugUXXKvmt<4hwPzPcBML3X)+ZyU^vP&l%2
zqtxfJsufD#ugsi&>UDJPi8HIE8ndKsbPM*qc2eZrcFI?8%c;4O61;PsFh6{BBJ9%h
z(`H|OE-I+|vi$v@>HOO^?Rhj+tz;=@V9x&EH?N2O|8cZ#X@T|GGk5*>e|KN_?_2f%
zzcoMYesAR~|9HFp&m(?|+IrUSvtHx`efg00*{tDjLVjI(L-TX~Z)z3XD;oGL7`P<j
zu5Ok+7nJC`NG`FYQc|oS<B51(lEc4CZbzr8)tIPV+ZwWXnZV*{O)ICxggZ&C5%c&m
zS#a*gP&-GBP8HEt8&wvlvdy}XBWfg6Zq_2^)UxYRs--!D(Z&|98!bLBngdpFdTX=>
zeQybIFHK8JGcPX=-rj1qy){9iE$K&z--?!?iZ(k7hKwC;fj?SOJ=zK^1ZG@rVG?XN
zUXm8~y;Se9xUyq$_VJX*=7u(n<t;19zg{SBbg$^z(b03Gqve0QK>c(6*B{e$F7lm7
zZ01UAmVW4cqbTFV7t@Ysos1>PyG-r(rpm}~i;kU?+~nf>tH}I!k@>3QO?;OP0t>6}
z3sxU2uU>ww>Hp&HJ>l*g3*`j1WSMPM|7;>9-WGhLJWg$)#qOjirj0^T3-am>(=ShN
zz1Uvz#Jv9F_PSToIp3@I1$p!>n!uS=(f2Z=?}<hI{q4mMp7*}vsDJ*W?m>8ey<%TY
zV&6lHzQ+~}FFN|}icd&#n83z4f%RoS^Ul8iKWbk_C@`;_@Ik%*_V)HC?E)&QIXBF?
zoEDi@xaYHSOk#^@;5*U4d82`SMT5%AWUdtge;)|z8FFpCz?ZPZTjL^E?f;8W|H2aY
zvwLJ*;YwH(yWeyQYq~1mMYp^Hp^T<z3FB%XQ3Fl0Le7r``!uEsnp+433Mx1X1+W%1
zmu1F^<~(=rJZa8$)q*QoAT49%G@ohHjJKzneV<mZD4faJ%DjLzy<@uF3f8oa>2)hP
z>nf)=Sk9=joKaobmUW{oJ7PxH%(ftn8Nm`W+bn0cu4J8=FmndyjOjaP&Nw-<W#;q+
zl{031p3$i(u;k~A%#2wRZ?>cgmKIBt-fOd5q#(JuDaV!3wsl8&<Mr}-@rvf<9i1E%
zE#h+;%?0#l3e~31Ii)FZ+>+sJ<(z#1{7VF)UYJfkyGTmfF=u&T^U{ThvjQt;AJpCQ
z|7rIwLDkvenco&gUzw7*?7HgPmzl2&Lj(kqPCj(^Uf6xQbZS^<*6+;jT^U)dhShw*
zY%voAj_i=Tbkt&hn)#9gY^j^IWmF_PkEq_d>2I)*HT_tfw_3f-s)hH{RlfX~v9`0%
z>%`3IGg%pK&WxJC%Gx<IYX)o0&6%?%aI$tzV5{si^;-OB#o}o*Cs=hYww|@vE^30Y
zmayTkMeb6I4Wu}u1X!aEEb-%7>VIoV@Tr*wyJqUHT5Ps!Y1FU9QBjM&p6LHpQQv)V
zVnmDUYlZx3_DNr#HE?KDsC70Jt<vDv=$xb>P{!5xjjge!YdLe_@|s!8Stc%T@LDc2
z)4To_*9k#4_5UBGqL^y#U1>bS*i>BPyX3f=aG_A%l4{isfprS*Yzyt@zcSZ;k;EBT
zSkY3byJ9L+AQN*S6YHz$zX!X{eGC;p<oeT$A<$rY&W+W*o-><fE-AMZsNtNzoVaM3
zrohpvndK+f7;?=xxN2to%$aqaYp$(YbUkbBv00oNx7IRWSUYh7>qDz`kD}H+ky>|W
z*UZQXYj50Icd2ToK;$f=_MB~!itpOWtC!Dif6?*hRL8-e<+_*VuoTYO%OJ|4y`lNn
zq`GM}W|^udQhb&c`Upxp@Xz*{^jv!BiOg?{Hm<x8T_o)BWAT(V=H1K76}P8X`&!1H
zR9&!T`uv0CSsTryH_j3;j$ZKo|Hi1hkF&)rq_<T~RDQlfzH#+;Mb?Q67e~&T@HAt(
zcgIqn-HU>`7VmzwB=Gi<U0z#*qPGTlFOA&2)qnNY;8R-@UT;m5W{nhB8Wp|Gw`*I9
z^w!kbtN|C+=6i1|e7!B#dVBWlZ9%KIRdH|iyS=8^LgMQ6iB`@H>^ml@SME?d*`O`i
zxkIIsb%#G!;EFSx8BO1p3utU8@7fS1)VN@x<l6?xU2Q_**Yd@fS5CgJWOKC3<>x$B
z@%aIUn*$t^5*JM=Vp?UOU1-Q%_}g<T<1Tx?$6WUf88&2RGdnQFERgFyTygSs#Zhs~
zA5Vl&nM*oFuf4*(_R8%wr+2M6ZY41Bz+Pqn&Y=GrR!<aQedN6_$Y9@7?`<!%*FE{Y
zH_%|+ljwCXPw#tmd*AEo{U5vcf8D+Rz4yMq-uqts-p|yt@7?SD0Sv7Fz4w3Gy-veQ
zAmm1kE2Cmry5zKrvk!AFpQtfMY|p_%FXza`2(W4kDDrfk{&`UC&m7gBLz8zOVqJJh
z&1S=~o7^%#)2}6SuUQa#x>{;NyFm3M=?9XcTdr1qE?)We<u1;LsvdJ<zilpL30x?k
zxLfsgjK1~JJ*x%$Esv^4S8Kf%K5X3+q!hb}Q8nP|RHlQD0#--NSNmrzu89%Zm-KsU
zfzHzC)vHTnwwBrm?A8*f^f_KN=lI*x$9LU2kh1!C?(S`^cedT{I*|QedTajgwaq@;
z^QBLg%svs=a9~=@$r&{#r}wY~9yqyc)k$Us)`c>sczqU6+*o@tec8J({x27}bG4Rt
zyk546=XCX~<!3op?8w<!r?sN-*6Eg4rw{x&eTe7G?mNqm%{krdW$Pb&SoE5l`ozX5
zn@uuYx{pK`G^eaw%`N&RQb6UJ+djvpIa;!dXN9P|SgCa*DX?)B)52BE1p@DELSNrG
z=kBV$PHWFvFE&+&Y~dpnD|Sykcd+@M@cxwUzPq>g-if-9`g?EVoBiBx_w)U|ARv2D
z$o8TL??u4}`^94~3d){hi#fTNhgDGFl#=Z!j>dH}XHFCF5#L$aQRiN<zw@BM-Syos
zlxF>3UEdpc+4Sw4rsI4m0Z~&A>U3VJ2%Na*#pljv#g(s$eBNG^6Ku&`zC%8n<*2xk
zZ;8?-rPWo(RI}EktDb&3|M8=yPY*XG?cG!%(j0WKvgw4k$<}Kpe;5{|C+?9D&C5An
zQhU7e&Vl<@0u_6Ym+G?4yK}PY%*m#^CtG?CG@m``8E~WP?8&y?lkL7Y`ekoUuw|I%
zbF;x$Ah>#|-HsHQlM~l4<gd}0q-uFfL-O>7IXfy|9lCTwX4dS}Ww&lw<j$FN`w*w#
zZEc&|)4K&)t2&H*oR)u0_PwMc8Mw<P@{XZe$3|9B*@|rEGrP{ko;`lBaOro0jD<2f
z+ot>!&c3@bNjK=|%KyLa{?a^G9>SFRV3p{^^J`|=f1G%jW#ZxJP@x|o3%ksFdnO6;
z>F#Gev+usuK4t~hiZ{mt1sItXA2ir7u*3-nPGH$p^+2$Jh3Vo0rh5;j_}&zJ_h5Gy
zi>TZsfrf`38y-&C`%voLgMbGQnHd;ooIUVCYWkN`T(dss>0P{3YhJNWbB+<;a^t$k
z;ySmP4Gyt9oUI)*=giEVnmqzG?+&@e9Wpp`NN<jSgWqj0yC?qZPVc-ce#6;T^u(Uc
zUn~B^=ziJa^H{QS-9}aE+jo|juXKGWJ||$oJhzoUrc_CbcVDULPLaL0D|%``<EH<S
z0$XNfnTh&u61*ef9Q-uU?0e$GcMIog{}<eQ;=07Ywa3eR1^C`Q;Iezkyx_qEyN65<
zAIz+Kz})z7rk}u^eJ?o_FJ+uw5|yCRu=*6!j%8bNZf*YaierVqYTFe-eYcOEIlZIq
z%vQapTjsq!Y;$IN-p>8&UQhacx~}TYaUYd83>VgZ;EwrX_HN^ythbtxj}ke*$0Q~>
z$p6T^Q+LGerSz&}Q||Xvf2%B96t$|7={$Ggs`5C2XSNJqV*~|DHk(wa+TPxraj;qY
zAXm_Yi#fZGSIiMwxK4oCUqC_kl+wS8()&Kh+C5S__dzZ1`jg+&&Dam>^sU!FH(QT)
zx&GV+!?PW_yc_)NK50(8tRy2k=>_+(HJ@zl1Z>`Y@{udp_J6OEE1&S%1O8{u2+B8i
zI=!`hqqNcPPN!GzJbAZ*)$IbeeL_}f$f#7uwHVD`&+&Zjq@yLm*MCUL>2Ozn=QO{1
zRxV}HWUJHHtY+tl-PC?(!Vujcz?{VW>8<clsSl^LPB9r?>Wll{Yj`Op`_#&N6X&1h
z`#ND#zSpfyGRxQBJDpKD=fs@bUG^0#*WTXR_ha|HAG_ZD*wy=En*Pt8c!8PoH(32U
zG|B&Uz`38Z<_igWaGh?KI#*P2`{S8Y9MQS+?{G&YI(XhazjxP0sUDVDZ_QuGoJn-t
zH}UQ_W(&R7ch{wz+y1+I$L*L_5y7uL=lPREo!;Jyzxi2Z`NV)iAxGQET}+9R|L0!k
z`1q}U&uZ3qf%-GY>vg{Gs=BY)SA0(WphUjp?wt+$0xBeX9`D`RpvYO_87Ht~${Yqm
zhX)7h{;>&}SS+}Iu!Wmdtj0s&;o%l$o?n|h0vE4u`d4%F^V5xSM_ZKH&1_}_Ep?u%
z8@tJdW3#IFEFtB4KAM~3*E23*yCoEHKFqptZPcdBldDp-mooQk3%u2}@_-rhM5A2i
zRXzTGDQwd+Im1#V{g_Yha?zg9ePDCKUcN7y7t&X*Wq!f+MrGQ&n^TrvTD3;&!r21{
zn}Wl1Z4^Ex9cuqC$MeSG&8_1_%E7W3BE~UGCq_>U>Y8$8S>qDczNXv3o3A#zux{IY
zG|M}6df<PyjK(5^?gMuMH@RG7lBp2WopmHbN;&I_M%bCit4ZPAar#w%-QvERZ(8u}
z&E1Xl-~A?;-Rj~oo+X|U@YnO(OXGFz=j&{%W;4A#x=jDx-+xmcJwMHEVE6r>^oP9+
z{5k>qKkKYoE_NkO_{6l|Y&*I{%pOI(UBHvseR}EnTMxK(H?lFTKC<4!<>C<|jo^<i
zjbSDmmu|Wz<*%^5hUxTr##LF1ryLZx9cZW+TI78ENQ@-M$t_cSIz5l^2zPj@EcYp5
zP}{Gxm_=x_%3_}Nc{<L=rL;e<JZyA4=G0OTwyq=F?%tEp==RujV#$t2M+8)N8oEWM
ziiX|z8N4g%)yt*x>iBdu{s-0z*_K?4lW;Y;<gVPi(rA{<q)asv^P?+O=kcsMy2w>N
z>cyg}sFlwb*U#B>T)v-;LsKO@wRWo6ys$?m;jyv5w#+k}vMOES#v4PfmCZfAoHvt=
zlT%kIum^QbT{Z7gmx$31uEOiFdslfY?BCbgy++7V@}gdGd9Tjh#tI%Gk1})VO@<X$
zRfF!Ovw!N^bbOcTtIaF;n9i<j_$&J3{yes-f`*pbstsC8^kjT*GB$MO_U!MHX107f
zCHb7?izSQC?Rw+#Y~7O&2bl96O-`@wx0`-@^9jYjTfF_#W@jI>O#Amo_=V+MYuOX?
z7~d(gYP30IEuXgJc3{Dk`JR)0E}1P}67^qk_Nr-7s%tJMSw5fiCy{U4BDNJzr=Mqe
zc4j5xqC*=F$Jl2-U8Da|c18Dw7?(l@vG3cekBG+K*&QR5`Q*y#O{Y$rTBjuXUF_Zw
zzN$NMC*IZ?#huVNZlkKpdTUS5x%e$Trc+hKI3h2{ZZdn=dU=&=#-`@1RAF5=&%B*>
z|9Pg&b7a^3a=~5J_sjRW0;2oA&*i!6`8<JLqeNlytn)9Ib_PYw`!$ots$`K{qAkDD
zceb~;e)rU+=U2CGcz;6ag-Xr}m2NwsZBxpkJfmLxe%Ai>$}ht#N!6JD+ovpJxMBN0
zk~i&Z;fcjECJhO!-5TFGL?61iD`xl>_^dtQ6}u~Fan459|M6)PC+@Sen6UNCS-nz0
z;d=)znF~#2jhVgc%&rs;RYk^=g}t2*cgrk3EaBB=mNG*uO6hP{gKA!`yQOcYz_Ja^
z6A$I)NjGHonR0zt_$DQMOVX*5t6TrcT<HFDlK;#jXNATP_wqLjSZ&!3^aXY(IqDWh
zT@{<8VGzYqzxtHFw$oOv=qXNGf0Spn{7dY2ywxPKu<Ee{?-zMS-JqWSJCZ6Z)jX%A
zwp%(2PSbTV?K&T`LPMfSOyp|VV*b{Q0*yyuU2F#qiEZU5QGLXu<GI4zs9@nVgG<xA
z_dF6m@^Xdy8z~<H)o|6RF_|+0vpA=+IWJ8zYx9!WBDinMlckw%mz!q)SDNc}&P2{7
ztC;sz=9EQBrsoQNDTOK>Jh66>`ULlht5gr3n&P`@yPrwud&ASw9nD5%GFB%wxh5vG
zu3xZ#wMrw%;q+<`{R=<lRjqLn_WjtdnEi28w4;hj_U2Z5wwGODDJCl04Z3XW9!~sy
z<+1;(E23WPi-O9kPA%1}&Ge}`IOi4XYr9z|pKWq;Kd5ZBRO3>Vc<c@)r<|vs^p!)r
zy#gxMiI-+F+)<jY#m}}*q-a(9+e2AT^rjz{yw@EU_x4ragjv%R#e8yeoL5J3UK6^i
zd(lupSoP>Nv#xhJ?v92V(^X#YaP_oXmN{j~6Q>&!SyldQwpe`ONYTF~8#G)N&R)ET
z{lD&_!+nhxnnf2aXgwG9arPOjRU0{lX9X=&(^<*$N&cNdS7z(AuF#v#8W&H#zPwEA
z_|0qAvbXK+l-WCN>-<(j+k;*=Zm3-mR$BFJO-69Gm)R1b9W#ycm8>|r_iZvi5IWuE
z%GRejIhuE;cm%Ahc>ONC=Z5mZB@_01Ty<ySze)F!7j0LKSH0M=DP{4w%9VOcI?lW-
zi`vEcY)WFDZA4VdwFL{VEV5iVD_#6<^Tde8CwAK!3|gdX0;fuT)qmA8@9T{4#HnUM
zwKE$U)G`|PJo@Ua8Mt80RXu4L-4rEme$^`)ixzZ7S-9|ca1@#=3$R92xGY|LM9xd;
zm2vkCr~W(LynkQ(H($|zW|dmOn-E5A(*ygD1XoE}$}<Q@2vq&)YU~j4vXu*)IY+Nm
z^w^}Zt!rkA<tR=SR{HWlhy6pE<?0rrU%Rex?%z6f?UGMZRW^k^es<`Jp{kmZ{i&>K
zW%hW<PZP}Qe=po+wOP-&_LF8tRBVi%aLjaxE;AKIPu(7ySAEwGi7zW#qEs%n)x+ph
zx9B}>+1)cHoY-{ODoLV2Vn_1zi5rx;JU+bCTJRw$?1!gtWWymXsVSWr3(^$#H#%*u
zUM93C^R2={(Gv>Z+49c27Rfb#abNu=Gl0eSQ}6S#C0noN=9PW!QTUtC*JjcwIDbv)
zl!YqmG%g37YMyf`zm!@3z}o0_|6jg3`ye~=bk3IQ(}qVibj_Uv_0O~VeEH>Jm}De$
zj6LY;loi(v&g4uoUMq4<=uCO>#gZ)lLnWcN=Bzv8>%+S|=#0@cn=a*zpN{J*<*<j}
zIVhU<qVmF9frtGjk%|J5A(c`Z8r+Mk?RlGYe_mU0BalU%JH*9T&82lk+Z81t0g0nd
z5rVB80frSue%m_RV*{!#WQywPX)I^|(EoGmsZjoh^H;xmw<`XM?d>B?XYFU@ObA=r
zvQ#>+$~&~^$KmTM4zn4h7YBWESX@!=V|Uam*eGM|qB4VzQy(%Na;_J!JHPbH<EYI$
zmi%&B95*%gRnhgm89zd=x0{ClSgUZb#C}7=(*H|$c(T|DKP^%`+jA)Q?i>m0o~9aB
z{S)E>?=sHceb8nUApgQ{yR*CXtgR<DHHpa<uh8fb=G5(QQu95mo09pkwk(7{WLAP<
zfy<c%p(18n6;Z6B(or|s_1uM5TXa<IFpQOE@Riu66*Z-`p~+(fd%R;);EMJ$O%7od
z4pkKb(>PSFU*uoJsN#G|qaa#$!2@Nb<gLpe>fYM5R%FSni-&ZXZ_T=A<@ml(G;G5R
z4$ZD$>DgB;W<OTadm<(MOsRFpNAu_^Jr852SG(3VIv+BM*wN~}N`1}5dgi(NKg|9d
zCDT3IqC1*LIdIFg6?2?V%Y47pvr1r2<}7{bMvHkt&VQp0|NrMCYvj<I%Vn@d!r<BI
znL5b^lYX&QOYE}V&}v=KZ0;arY`}b8liB#k&Uw|W<`<MFoahMs&}-Joq;X)QhXtPv
zr^aW+1yd&I+(}#^s%$TQnq{d$M`*_6BT=n}3EMm*nkPoEb2u<H3C*kfV5)M%(DEcx
zActtr3wB?J1|JTwn9EGFJf{RYH0dO;*MHcmv11KmMC+RayrDe}QzVvXRLSpdmX9!8
znQ(DMRYr4P(}F|4XZ&o`Wezgeo-Ow&gMY>k?Y*;9dS-~qYg<*Stn8m4z?s>s>o7Gh
zOZHF3VbjYim8E8{z9DFOTR|YHTOf$js@kc|#KJne^I|Y}=ayxAEf)S)Ix>A*g;u^I
zckx9FiK_MMlJpO+*n9o3)?|}aJ7$<1uvo3IXZq7Bo+}ehGI00!ddSRgnxI^*!|cf)
zZDAT+z&dYp^GjpqSJDi|3WCNjn3n3exRo%68O)k@a(|L^gHMN9*b2Tk3epadtS5~W
zq73++D)8NJU^0wgZ(Y!OtZGGRMqj9hVN{1%pvIIpE7${P9BF7e%_z`1d1Y%x^@?w+
z`xs2zbUoazbTIiZ<_PTACiR+q_m|!UAKK44H8F1BEt64o@0n00Al2ibQdra1nKhO1
zx2~Sj*^t-T^K#6u=$xHZXv#KQV$;RN&oibe9MR9d*`%u>#Va|hj<MG$^5oV)&;Jju
zI5s>qSMOOSTD4B^0n_fpb5l0VwsPLlyn1P~u|=zN$ISrlw%g*NN;{@msLELCcL}Q<
z7Fc$a)AG}nNj=i%Cps_tmU8}k$)t(SbLuWnnq+*qmq*s*<g)3LyQf^(-uy!0iMQL~
z*X{i~SgSuc&;PtL!D`Y%<4tm$lc%npQWzz|;Bst<wMbdO(Yjf}XM{RRe{7YEUK4tu
zDS6A*zy^jik>xD{tQ;p=I=!swIi?CoGgST9YA7)O%?)0eqZb$%9YkfVH&k2c?%;j9
zfK_1jnHeuz#e9Q!c(-qjH2M=L^<iRQMc34ji+p_rL6MKA##Y)?wAh3%be}W(xb2qX
zzjtW%CAs|P&}}wbs-~yOsMm4XLX|Ok7L(BpyXTTz>nBM3<MCJ}u*$~!#G+q&w(Kyz
z$Zf0lfT31#dY0zic|U~CYjYPZyt4AsiDk`uC+C>x^9UW=uu!UNy4<Hp?#xPCZ|+_8
zVDFYod(Uu4BxkOEuvlRF9OLbU3s%G^2+K$<FJKMd!TPeed6CO0#)@W*n}_3fw8xdU
z20hS-HsE~D$?q}8XSY(I#)HZ863ygZx9@9d5`W4k;L5~sW({Kpi`~S&N~r@)Q5~;W
z*r&W{_h?}9{js+F!mh=y`(`wpPTRaBQMi?DiL0T621ANK*a_Z>)CtOSW!J7`zrL}5
zL85|TtIs<Qu4`x3sz)#Xzq{+;-JpY$5)5v<(74$$`<6=cguClRcP@=Eij=UtIa#$?
zcLI}i$ZXD)+qZR2yC*HtsN(m>%0o%zrqU9gry)+St2|!j99qz7+<3sxPtl^?nOjOz
z{L?2vy(P92b9%nAI7_XXb8=(UQ?Bl_mWMO6S|ySWZzw%H?T3EuFYZ=VE_SEhvO`AI
z8bZbft%p7c1{*Xk7U78&urb^l^e}P3Oo0f+tsGSy$0kOIOzvXhxN&`(G=q%jyr>Oc
z=0=)<KYHtx%$yppR~{AFu<(qac3`Zre4~;^`dx$i4Hwz(^ldj@lg!LlwqnJb8D32p
z6JE{;(EZxsa`DnbM!D7*A~P~<-f!IWf7!>46>~%<NCdO*KK{FabwSCgu#SLTFNDrd
z41S@@`10!<y&3m}Hr#u)_WYYz)pL>!FJmY2S{N-}Xk*Ibu~C<cXIHnSvW(N*%`a9N
z|7i8F){a_qRhsXD^r?;zyJAKA>Q#K7q}K{;>0LeX*6xLs6L0w}wV0gKywIonoe%e&
z-7?c>Y^i&>ibpg0;pg*eTs)@PoXWux^R@V|Yc?-kv(IP)b8vu{utu}F1kd6Fm$ZY9
zoBlFwh@E|?%I$9Coq!nDsuk^bFSJyi>N9j;GStv;4qW`SU}K84Ql>X&j5j|s%Q2Z!
z<>OA>aaT^eWjg%NR$jAa*874mJ>COVPZv4O-Na<r|G)bLM~{Lj^Y0sLI3g$cMow9p
zJh}I>=+0zI*NK^%?lrFPxBnNnRfAdc1J}+jrTPrk^%`O|&g(LEZZ|)7GvrhJ6B~>9
zvFGNMcE0HGdn?7@x6RLauG5RGr466YHLrAve|4qXc}p+1+SffhK25rv7BwkF%2F*e
z_>!^n?^lyd{siY#@mT&je??-$iZu#{g{Lk|;QBRjg0%*#@Cznkjw9h8c5(>MdgwT1
zYF1~IMa)hcUXBx6SFIKfeDU!9N_M5ITe}*}#n*1secD-n=#<derc#M1QnJk%(e2KP
zkGOj~>Kcx4f4w{F=ZqN#10H+snxXNc{B+>znaZmlcD8<7YufgrLgn&*!>R7o78k0w
zu+EDP{@NR0o*Vr8%%m4b@4Ys4nI!k}{nmTWas<V=QaRktiO#ymd*S%?gG(M}Iotp3
za;U!VS$n_5<gw2@Rg2q9d=jttrNsZ0b}zlFuwvJG@4uPTTCxspX!eioSvgH{t&;6&
z=eaUxdX$a0LVd4Rzn0{l=^63WztfxZ`a+588C^m$vmAeQ*#2QpHezPYWerbY{kO8;
zNbC-KjNH-&CXE2TlS(s}7$4Blm|C(m;8f6JWwqGE(@p7DE+n3A;gwynL7PKpso|=X
zrs8k<UnlrZPBiVvJ$`S|4NdL-%li&D-nbFfaW-Y0uf@Ab6PLcV+V(a=X`-IS#OFGR
z|LyM0v9o(Q*}41qo47`WcMfdbyeH0{`Oy*fV97n64eQ<XUjE}zU)8n!!cA3|LwXCC
z9>2T!KCg43uSHbTp~az{x(T;-Tl&4c^s3HU@B_>Fp4SuQKdN?f`~T)UJaNk$J(<}-
z0`G(@Eg!Ni3VLvP<KN@V^_DwlJ2M7zo_sA=#wB-Kwjo-Af&W9(;*>;}6qk^T*;S`F
z*IsUEdzXDk<T6J%NB7C5mkk|t41pR4ybM<eKMuINI{NvE){l=Qo(C=LlS;n*S#n;<
zmDr9SB2`cC`doDRd*Whvp;6~LcUu+Bp0*8dqA#ys+HLmreeAyESF)LJpMHAqEKV#*
zb*0uj!*WgocftS0-=*K!7N7e#$@*+?o_C5$)sE$#9!x2S-?D(G-Q`5b91lzB)qdBO
z_Fh?Fm%Y$-`D}5)2g0u=^s8)P`p3;&_a)>Q58FKvg-M~3n;x+3Pq^*O-96Lu@#F+<
zPJ?QuhtG<`KI~{e67g{9mx(-@Zzl<~MuaGLOSInqkmMZMZs*A_`Pi`9>-34V!Xr(E
zAB(+p{j8i1@-2C6+Q#tqpTqH(NZ<5+mA!IY%Egaw`J`CfTQ_O%+q~(QBVIkecP`hI
zKjppY_x%ES4*&L8K4qF&7MW)~<L>Vz8)x3!9Qi6&{m{!J=he47VSJHiH2r?$CO?0+
zlEtN*e`-BA)b_rfCH?<z_m^W6|F8Aha^rKz^ni-*J?E=3)l?>~|Iu_g_MXQb=lIWi
z)&BkJ_L}U>sQgShNVBvkW2%Ff)0R)yCQdcXJhO`DdX8$~{RxHJje{!Ye!V|wO=OOM
zQ=*8{@e3^0R!j3|9(}z?QJmv?Nb;QIA|p}F$F7$(@>@02gZuwwOgtAiS?}e>{rB{r
zpL<uTYIxsoy4us{M(<4PtGnY>Kg?zNXv?v5qQH^^o_vbUd=sxMfA{%?kJiEe#SFgr
zn*^>L-!V<dvt(70R$unsb&pmtyKy^vcOK&J{x9RVZD#pKyG5%GoD5H0WU)tJqld&L
z2Cj{UQcD9C3)LL_X3}h~amvWzLVQM;#VL{NvKs$8v^X42PCl||QKY=&<c3+hjOy0%
zJl^$4>YmV}bpN+KR~@#Bnc7z6-a6A|wqfbbXWuNJ)!n}KwlMwN+tAN;^Qx!ZyH{-A
zVIj-?D6PIx`tx&<ci;Mde=o0p^o>iiV_{Z4YtxmIS8N@OYdxk)I<#@mWaK)X6M5pO
z;t|Hw6&i1x3LB1g@f+raya-G@K0)5rOy;HF!sAWcwrV=fm!6(#(e~{N(JW-p?95H!
z(^z_;_dH`B+ZN7SZW@98?)6nNr$QbEE%9I4R}vMR)Y&e3dP+uSfa=P{Icz<l5!2Qj
z-<)-M)6$v6BIhPr249j%IG?;XO782g;2Y<Y_wQq7;{Ra!VEXF+Ld<7o6g@uGYa9RV
z&)>hV!#^+_oF@8f@%FB}X{Yx7n)oiX-{|F~#^1ecI#$N7uPipNe{=iw<FrG~zonkd
zy*XdM{*WqvKhK{XPvZaO+}xddUaTZKFUpraY70-<x|9v+Pwy>dx&Hp)!RZ3=afWW6
zVmGYY9=Uzd@vk4aPc65eoc6c>t^2k2^_TZoOZ+ugsD1PI`6QN@8<WDVnXe>I*Uep+
zI?a=%(S$)=I(5Z`Hi>=ZlRW3ttPo(AE=*h$CO`AXMHc0cA#9p)nl5hb%o!GoJIqgp
zcu6Y#G70xr=K2{I5+*uRB*aT|rpn}S!K)W11lC?THHj(IQ+1kL;LD5A%Fn($4gBw@
zx#gtC+Z#@*+RPV}Q{tQMKK2TB4y?RYJ!9+4o2kofUkIkwvX#wB{rkf5(mdx+k>Po2
zUpb}I+s(3O=x&`8>8-tWgHeEVqM*w$N4?Tht=<cptX8d9q!kuWnOYMrShac?yZjoa
zW3S@tHgWBcTleQ~`szBah2q)kng5(xy-;5B+FTKS*A-9ozb0@#iHWp}KBmTbvL&D^
z>f@J3t>V2Zms<HvpX7QnPpN*_absOdup_H%$%P*MyCz9amd8RT_@|024c5$kaa%vg
znYVPx0pE)(>cIs{HU|!s>aH<3Y<u&|>y<H{DY=uf12bMkzIQyG8sn6=bB$9gSK4!r
z#b3XATmMh}a3f!PErZvs1e5w1(OGIsr)@qbcX!!^^BX?8MdTmnZI`)pP?L9#?S7l@
zB_;iV*H4w&ByGQ27E@R_ukwDSuUvRyGw;1U=3BGNZl>2C(Yuv(`RF!J#&zBcYW<eQ
zn%^lq?=F1%Q{(Z2BJR4k9~7@ux$?0^{^*H??MJQtRctf!nx{ME>8=a`b=#<H{k?N8
z+{k;z@>J#7<E8Z?Q>S>W%1a3f+c-~X$``rTWeyjX?Of`#<c{5?cU%|k7-HmE`E5?f
zFK(@F*=BO$l*St#wi%0No4)PRPW9QoAn|;5fBNFP>D!juSKo?}p5J)snQhvW;<8UC
z%O7&CTr%~T57#ooy!H!!|1X`iYyZDZtEyJ5W~x{dI#u0n-z$%`4XyI=KhwjwOB7gg
zuR3r_88iv+Nnq_&71YZ~&^x{8Qt<;Pd)=4kyw)t3|D5NNK+BXC9^D+p9c&NQ{>WLv
zZF;4vCaFWO_00tpy%LXmYqlPemr7C2lNP#Nv}pZKhNtR(wOUITtgyNwnWMF?`^Iax
zO$Mtyj>M)a9A0^iZPUaIox;>VCnSrX3ksfkT5|Lfx9$#g_bHYuEVx%4&HOcK#px?O
zeH}sa7wdwzEcHywJmtxEdCsvpuDc%j<SvNP-jr<mE=4``@y;ie2ALZ-Ze87wx4Qq%
zy~ziXT8*X%@`&9pNfqL`wlu$dw|~W@XP*BVFS6UD%wUq#GHu|vddTP9VcACoOv<81
z`-+uUUF>!GqARdcBUa|Y!ktl(<`a(i6(v2H<W*s^YvP1+CBaKLMT6Z9uA08oc>L!?
znT&heiU)mjyFTBINlRa<lFR9#CV6kVhoIQeiAln1GbT7mxu}-)DEanTpH2BPWuKKL
zmw$QImASW4B&X<xDDLgNza;5cm(SJr-R}=g`{#6Oe_-kH?ElGE*Kld*^l#B>-nmG0
zqn(O;@S!csBm%FljhS^&aA*25&Lb;*%1zIv{hGiWbVJBNTx4O(gLCCuPhaT?lkhFP
z`6B9r5l7%!O=kgDJy-cnGM@LCo!8G<8>=~K)i#Gzjbyd|E}h?RO`pVOHc8pkFv)LA
z(vfX5t_$+Isy<27OxV~t%_Og<r#ba>L{sF#3A`^YnJ;uZWd&a7{FcMuxY{QtVDs_?
zwy$biJ*6_57t1Wy6w10#*ebi^+lf`F?AehoPi1evX8N_@)A|U_?AsUEyzfryRli;%
zT~MffC;XOf{(jv6&5veR?jG~qndhBXyx7}&x=(`MgthF&E0)}7J>ub_6eVc9m(z0H
zBy$z%n}>|I{t~Q~{H&rL^!{m-(ykpHD`!9Y8}Q0*lI!2T3{7{#)YxyG*0*?fv@kcS
zH$AUBKaq!jen9AA&ofU#S`R#}sdK#V&UH25RE2-_za6Jj?xainNZlg#|I6hoezzDd
zP3$sR600%QC9G8|VlMkqp`1+LxHU22iOX|@Q!<%DrGg_lXHFH_vcPZRnkUM}H&6KW
z@oy4xGHU#>NOXI&Oyl!U&J%eGJ2rHSZ|IS*6wTVIY-?leD-|T<zxC~!)V^hXS5BI1
zvOet!{^&7xbxQixn<`pn8&6GJvPDoxCndrwC`VFQcj4Qzlk<91mpeAF1nl+LR=w-v
zn}SasbCyWDZ|Q8gRV?+g%iXViYvk*`Yf0Yo)-IKry!q;q^;R>tZ&sY3wlB4F?d`AY
z_3Hwa`hJ#vy}_2?yz|rcW6M&tmhLuB)&FkGc<a2^H|1QWvzPKyZa8>WtQ0st>EMDq
zmH*5$^-`2Y0)y|owpH`B_#T(h*21h^vc30#Z9{qy<KFDpW#N-MH%fY@?6Y6gS?fC~
zUg_n}+22)9t!+*5&x~F9(6=b7PMXIis3Tzhr<(qmY*t+lO+#vxKk_ay%G1>5Us5;c
zlSp^4-lcaJRD0Ggibx25wYqurldLmu7bU-Y&=tMopk}JaTHQ6<KhHbNwZi$^OD7?X
zCz(ofkGc7s^q-wqGW(EvxQE=+Nq@^G>ppQg6t>A-+pX|H)3tS?8-qW49O%-mI2xc5
zQu{mPq0F`B{j1M@aN6y8?Ob%pcG1<`hR^rE%h}L$b?Sbu%o*+qWhxz}p7FEqs7NUH
zzph<YxLe5SapM2GLeupPZ>vnGeWP*Wg4#v1wO`J4MwPDkcH~VA@3u8zTPu&K{odcS
zHaA6jokXkeoyT#)+a(V3mCAjdu`~7g^$_Q`EBN!<ySE+`Ek6;SSN#6Y3t{o?PJ1m&
z-haCz8*abjq~7g~=}dBZ4~nK&-&A#2X>sQLwBIgI>?(F0(!M%*ippLyr~I9(ZtnQo
zp1tbValuH>Bd1wB4+zdV&=@>bCc!fF<bkCwhqhj_(!9~0a&xLE_i;gf>5YL~jtKkC
z>2%o8q!uvi&y0wZtw;aXcy+Io`Tt@@<VvsIECN$*&N#F7%7zy=d<*9n{yidl<xY|1
z{p1~u=XN&UF}bh%fkWKplI{Ud$pimATVz~?e(`Jj-eoq8oMb4em3UVqDKQ{5R@Ck$
zTcY&IIh{eSv$j~w+Br8i^x%b_3UA(pS7*<Zj&KZq>EL;Hr%Uyj`$mk{&&KlY-^vp5
z&^s`q-RVZS@KedSnMdZyo;>WC5H8`Oe2Zl{&mzl<EQvEOKGhX#wY|02biq&8Fzp*m
znm!CG8ZK(=yH>vE-s`jTR$mO$sTIo4d^lmF_t}HKRy%tH$^yN1bURL9ko+U1^5~Y+
z3z_v(ow9Zw+^2a=x4~uo!fU5@H0mx1Gn#bidu8yqPQeDB<juUd{a*5JihaCgXHbQK
zOZSJM<^=H@wHy_dSEY|laKGXaw)S!Bp6I`4llQFrf0HfZ^8dH-Z8L5@>2c}1z+v?@
zx}Sw}!WHp(oSdS%3cdlolX&i^e46sOXTswX!Pc3blL9)Y9B};*Aj9T&$F7jmy{@Ij
z)kE9k;K`*i<r7YMot5xj_k>64)|p&o9joYI8@K+mE|M?iNS54pw^<ml<)CiZ(Q}FI
zQFjkb+t3@R=i=BSCC<|tXg1OJS;%EsNx!{UCW=O0@=aX7XNpLiENkT45GjH1sHfc%
zQ^Sv4*{IMW!g0?1iqq7mM;bkkh@D`$=Nc?CuO+l}YS>Oi`)jIyOoe5OSZZb-`4c(c
zsJ3JA-)>9Bo&yeo$y}`>JDWsAd*T%1VtH7PDD-%p;8_20wiN4s*CfTTM*$wHLSoNn
zE}T$v@44=2=Y~MX4N|&iFBQDJS2xF(A@BaTkF(!DeROq-qc(@~zZJgwzsyi!Y}MtO
z?{tH4A4l?*sScWV7(E)=4@?m)<KU5;&=6%Sf7n5N{oiHUZ&LD}s`&@aPds<)USa%?
z()ksy<f<PnsQ$$(?!%yNBKlD0w3}muy_09>A62VK*SF^|HrTc7zAD{#>$Y>kZJBu-
z|Gq9>o7fk_GuhuIV*<}37o~(rK1mjVtx8fdrf%IVQA-vcS*n({=))Sp=^GBRy4`S}
zkaG9H6CN+&&82C@CtsWwTyC^$xtQp}^&3yG=WOM_btw7EqR*a!hyP3uuzA36=fBh0
zpDn4YjvQLya!@6?l<xpv*hA4z>Z0?4#PeQi6=qJW)6|}OO615xb-RumhUeVxsP6W?
zx@|*Cn8A_i7Z>hS20uL@>5~(1Yw68f3yyrPJ+W5vqWm|OR~Bgsb{7x(rX5g;J1CnJ
z6yf7zwQ8}{kusC3?HW;uinF8^s@3_OI8^$mnd6G>r6=tS5@N3wiJv`@_Df0k)FNHC
zNy)o>jI7)Y4|)2%iTOH(X@Xne=bMcEE(+_Dj$SjluKi<<(;EfOkV}F}@eijmvPbmy
zJE?~ytgHx4Hhaaj^_ymonD{2w+jkvq*Z=W-aKl;ORjiEl){F@qJ9$0r7CrvG_xg;3
zk#q0ex0U<<V#n6S6Lx5u-n_qJ-(`g~;bcjbR<#AKMH#OOCLYjJIwTnVj%P+wDyOSe
z%wesQoTVzALj86Vo1#iQQ!ciMoLL$$bNOSLi7Tqoyf_w~405-6I%kDyqOjV_WQU1s
zwOoV_aSBQ;bnwtz;qbE8ZMv@&>!BxivRCB@-Ag(*D@Wb-NJvy_PpfKV+tTndhRbem
zFIfphXsx~MmU}r$_sX$T*P<3W=NwQ|=?}J?W^?iBoJDj0t)08er7z^_(_Qzj-~J^Q
zS{oXgIMr;Q``(19(W<O%Og#(2SW~Q86>m)y3l)5r^lb4tnVhAQeFElXM7^p`$dWF+
zx2*3(tU-s<9M2Wsw89eq^K8G7_WYNu_YRgaJr~_i9Sr|w38o243M~F`^3@03#4OHL
zD{dz3S7R|2YdQC|#klO^+OIRth4x5t%znouU#{1p-{f8zHc9JRT=g{dzDer&jcc|g
z-TxB!{!YYuxv<x}rW~9xPhQUOc!Wdfj;GJ-ytET0h5x-PpV;TJg71Lsy~TT$O8YK&
z$n{dbSLIGk;k*{F7Y7R$N$Vc{RQRH9<%}1frnvOE#2Bsd40zREF}qW5TKgjwb+z*%
zUsxtcX?{8P?7rgq$Bsu&uX<DB@c10-(F<HF16N5126;;Iyj^kNr0x=7fvHX0>Fapx
zGh1xtC#$48GALg3yx=D!8>ppPu`H%$#{W~dr=8Q#-8lWwBJrxVOu9FxdTsP8Yzz!M
zImNZ@E?4LDp8?#BN2VFRkUlkc*0b1(r^<q{qCP5x4n^F185X|~YgsJGI?-#9%C(IP
zyaGdBTzx2+clgI?#`{m6WL=-&u})Qt`)S^zjyr;$Eay%aaV^YMTfJ#sGE;OZhy40&
zdF$BX4XavYJ=%S4Fi!2@)cml>m}}7ypH)-7DW&Z4DXMBQh~htQxL)=5i#>Oe>Nh;f
z{u$bMAz{fciC<bDAIE6_C|u8Ycdenacx=UL-w4^#iHDx^GCbU1pgHHdwN%%?x5+y@
zrT48{-<5Ol_dVr?bJ~AS-Q6X6y85R4-a_M9r(7SW3Ag<FzuH_?r9DpLf#0Hoe4-7t
zfv=`09!^>^HALm`&dGLxI__Bttk+#<IZV58%V`_y%9E#D6z-`AY}#<nHdR|CQBck`
zYbVn?siod)3Ra|iS<(7wkBiWX2_+M^bO$rZpWepwb<U9hmb-oPSX`g4@#^XFWC-xu
zcgJl{OhV<^t4eRzhCIEhwbm;oH&EL&IC*Wb`PV83-4OnUklS9N&rgL`yM!I*S}`M~
zB~()K2*<3Ib3+>o{~wHv{U%`bTs=Cv_+O=(^zzJaR({n>T>^icj%8jX>&z-A?Ajv7
zI_HV;L!)_%zTMc+aPjB|spTpx2Re=>wyLGXD?OU=>{;){jvgoJ7`>FO|C{3`G>W9?
zso5Ky;d71nFZJ%+yRs|mqLzm7pFdG%ar7i-UZTJHO`Zdvc7A$o#_F%2{z~ipV_EOd
zZN6(3vcG$oXjne)o2vQ8oy$)=diE}Kb;YN+`A+Zi)~0_s#9NVbKqTN~nzKjjj`<Ro
zGynEQhhLq&lOw}Y+vU%+VvEh@f2I}QoBP>4;K=Q?K9?sP5h2@_v2B+2J$fLrwVLPl
ze5(Qn{|!vdFD6)gVC}llyj58GYU|Sw{(vX19==@RGNDBNc89mleD~cg!KXe1ChZ8G
z`sEYr+<di%K{`vjzDB4XKhpJ{UqCbY&59*Y1OjF}GwNkMz0qQ>&$o){|9*t?N!DjS
zy_ov{VMoMU;nNa-CLNm67@%E|GjG!ofsdKfU8g>IuyD?)hkLb-e4limtM{eIgm}}n
zlG1!>b2dqC<2u>>hQ&NrY2htHE&GWvoN`L%R^%;e&$fH9-}9U5ESF5S8y!)3&$?9n
z?`&&TNp!5)6fBamv1@jyfYcVQxdQi8)(hP_xP<r0E}m5h$K!)DOSaw8;q=zcTtC4)
z`b6I23Bij^l*;+;i5n)bUm5T7Gc)>G&`Gfd@#cNgt?Ry<t-oz~I(bhc%bv#1apzwC
z7P@@<ReY6i<Gz3MSN@v&G2&Z1Q-QNoLi41c<wkex*-f_lb8crYiRp3jNJ?;w*O7`%
zFmb-N)$pRmGz<TL|5|fQQYS6bnaZ_k(yz3MdPf+lH?-Yf;&-|tIA+V|z~i&`y`FF-
zfj2Gns_2A(^tE1wHixa>t`Aun=)W~I&Gc%k>ap&p!Q399`8G8*dybX=IhtMSr|A;P
z92S<b{YDgLNrvUFlAG>VWADc8JiGd<%sbT?`3KK6?~K08Qv3N=#p36Czuvv|<f}8^
z^H}4Ahf+CdK}TaaPjB0E&1;2sM2=@lyIN+{qZIvFEU(LBJ0`aUMxDOHl6|;{`TYkU
zht(Ud6y_e6D&I3tU${KidVTKHsrwI1E8DZ^=9xt$g2l6zr{}6|{_^i{sl46F<?3tH
z_ZiNws_uB4{ML0RpKWFP_Lmv0yM_L*5WT;v@m6#-|4G*U=4Q7~@YHr&F)ip@GU3S^
z{aW9QRmQf5oNgV}&~Y!_?J8tplj_-|TQI*^rt8s>_u3NDZ_3|Ik$-T{(fdWrI<JZA
zzJ?}AUljPiyKQx-i-(^vzi9WwH+v%b*HsprzngT{awBJiuGZB56Sa^0=uZ9BoGS9&
zLYR5y&&~T^{+Xe%u<EaM-_=r9NexHGEjA82vP>IJH!w!?sF`#KD2f~L$_w?pxR7wL
zl~XV-WXHs0*QQ{jCKZiTZV%01hedBPHuM~ws&BhXCV)-N$4J`qY|iB31AcwELc26s
zuP9tFb8c2sxn-Og*k@vX&%~px#l3mSv`sfptopAU-W<hSCCgBj<G#t_;vtoz$L=*$
zZDYG4#UTEQ!F*lXG>yk;4xD<a43~8lPd5<X%bdq&#`Cq{{=rz0X$BmXMTd4x7Y<Up
zQQ@$1CfCAgyR2tDl?``l<-4$EXIl34^@+!)-QA^^ae90D<#!!3m!_VqtY&Bb7nvx2
zKxo?o(~Q;k7=mK9op^Ghwd!cooK?}P5r&JMUa(mjJaH&EUGYNt#NzN3aSx}oyyI;;
zv+v^5KV8CXJ<H!u<WX-8s?=&;-p6+4t?A-~=&Y49mu${l-pC`xafnf6$pxprBdZFW
zcrB$=75UvCY985Ik)(WN)1#=ob*s&8Eb29ymbFr~Ddo>;w`n>5|Gd?oR`+`K;`<7V
z*SJ4w`*Tq*+-LJ)-hEBlhfU1CS@rhp)@jx{aVF&Sq#fG5kAt=v?Rg?}M(6O6Pp3{b
z$prhDY<|<=qkCT`_#ES%FOz&vys9xiZ~XX((Rtg4PbT{|-&wO}hh6KRY_2a4noh55
z&Pwu7>JXU}64cAH=Dl*u4QU51f7`i_g1_tCUAKTO_OyoqbNH^z1B|w>PZ~>1Eb7u{
zX*|@ezo%krc0(L5(>60B7460dqgP6gmo2>J<)OW4-rW~E>8G2vMfGvz<lbVuRd`xq
z=bT3kH$vI9uPYp?`NLyeWF8f|<5r$9@A{@CyLeX_O*rBmcW}WXp$$QA3YMD1o&Eo!
z_il6f;ocpsym{ewTcq2jo43zzs(9G>Q1slk^2F0WRQn^}uHfElF(FBFXO801%TxW<
z@?8z~?VY)+P2s81o~C2c{Ck@&yxwX0y+N+w%5|Npom>C0ec+FNKI7cCRTH9vHz==M
zUcHvbNNh^)YR>panTPqzKNaZm>z+Azgz-e2=aT(yW+oR-v^bY3Yb<>7<8;gWP0Aaq
zU$dRvV)jq#saOST{!-Ni?;ONdHNDrU@Yb>1BwQrSC)@DpWVrDOmQ!Ze=X^e|I9<2+
zwBz<YpYEu%R~21NOj^5?b4P}WlhFh3(nYs3k8RogF6+{i?yau3lCJRDg|7I~XY{J6
zYr^Z+UH6{N{NEO+HNR?A`HBeMU7;(N2_4GTR1-cl??y?wm6$>5=>z+g<+2NSD`s0J
z%|9@UtwvjMXTzfdo2!hMdm0~Db!5`zKiMYCPtN{~FcOKHxIo0;>TG73(Sf@R0ZreH
zJT&=~H`JRh%g-;KekelR>1KmZi}Y@ZD>l=dT4R@a9A-Q_!HrE~f<uR;_eK9YDG|A*
z5O(WXO8kx`8JpW!0_~%3L^|DgJZUOtu5$2<11si5Y+j}6Vr92vk#Cw47Xzb*!`zpT
zE?X?xC!n;VCD^g2c*isWzO@&fm?OEQl}i?GnxK@cAS9^i@#Gj|E>q0iD8W`QA(?Ay
zQcTY7jyljVY5M||w!B=2|7U#|o^>^F@@Q<2nDl_<+lCgOli~9_r?M^;(Mj2PXwv?y
zm=}Di%M2nrr%n#Mto=M?j`hlyzU4-jZ{L}=(}>Y)dPmk}{pwD23)xrxTfba(V|zK#
zNzu5(Z1N!onN$UD$E;a1PMux3tijh$l0QTykV#mdX>*fKe&xo8vYfGtw=9`!sJ`wO
zr@z3Gy^B>Yg&+8J)wcMmr|CWplgGx>QkaUTe%bpz>SEWm@O7;Lt&2JqW@g^<+B8j|
z+GVoS+6WQN1ws0=?i`sgagP*3`=a$`9-o?&OPR09Rfu0+((mQmC@`5JgPlj}Si+>p
z%%BfX+qPM5FrM%EGTHBhVN0S;qR`@qfdBnbhP{mitzycnf+j}^i%t_P)t$(onRAhE
z=ak0s+l`(jE0q0a%CcCQcb|(qJ8LuJ4<0ke)pJEY2p_$<AWPYN4wt$`mxN!A$a>3<
z+LsfQ+^)G!(X_wGA8S5SAZUqUuguR`PEx;xHe4~xIcGV$bx+fA-vV#NKB*HT9ZK70
ze(25%kX(5$EvY?hN7EEr&BMC?BDkV?o^1Ur^-^}x6E<naR<-8cx7PodWT^MysENN8
zZ~VVa;&q`cK8+LkQjeLeh<+B7a<zBPbA$G~uP6EKp6mVWRlv+x=hAhhx0Y(XiaR8=
ze~KfQ`9h7PUAJc4O>Eh>>7#5--SG*NbzZ3)w~ljCoABTFt;CW!Q#6-nYcTD-W22w=
zp(TAy-=w{-m%5p>ZZ-I^@A|s?JZV~?h8^uwFLC|cx?Vd?(=g{Bms?ZW63eeET&-oc
zwsKBc#Isi8foJ5bAgS*6dVjX?wAf`$k2pDn#Y8ZIx!HMpkkpeY%L<wwe>(JVH$$6o
zlcEm~XYjhiSDw~C|GMewxo*EDuh>?p)vRh0-t<sz<;-NqOQ+c$e7^Vg$MJSom88>t
zt}dTu9PfDf(KspoQ`o|X%V+<1nCN7YcwMt<Ybd98kC^D9E%#Ub*wdn_>~rML{GJev
zzML;xWb9>co@(OAtw=eXq&Zi~-YiQ&&3RI9bRqXGyG*Zj8(4eT40cTXfA3e;E~#~r
z#cStS|5&xXq4O)VX~*7&R_hW1js^CnrATzI`R<mR{H1(h^<ATzll&JSR(;xV#VNo4
zwpZ05RfCsT)T4{c{kbyDcWHmlOH9j~c~m9D{odZYCEQ(V!UtB#th_0>JuhY7H`&!@
zcTX`c5L`cd>+D13--S0n?KL+4m^!!q(<{GOXI3q*o*CMbDr(pF#mn-?p69IXrZ0=9
z`fs$h%y3$nx^}wsmIVg$=U<jPI<ZsAW%b*FqiNBJ+>Ljin0%62n;!EpnU&F?=J2w>
zf_o2U_`W(CfAo@3dwcL~oy|8B|7{jOY-cf9x;bApxc61i>2n6%Gbg<DJ^JO=%&xMn
zr|<rMqk3jWoX3TGSF6<1v^Dwb4sy-iu;}ZW<y!npYK~pI!SXh^!1P2tbB_GJ@D{E~
z<-VV<tPE!?6<loj@yzq;U#YDDyq6o6oo|l&bmPkEiC_CnPiQQeWR??arfvCgf_!L-
z+qFv^FWn2LxU!{BcyRrj-Ud}`-?r`=V*^=(<xORdJ<PR|Ut4o^B+Vy;CAB%83VO4y
zqWSyp(pjo+T?3BU-2HiUwd$vJ^(E7#-+SHa|M$IpUCEQs`rnWLPpaWJ-t+XTdCAU(
z3a@qhl#)Ij&<tMZk+J2avKCWDi@WN#)oKcP3{w_l2u%ppSkxoYlU(FfRitOCx59YK
z@#+|7%LR)RHsmq<UzOyxse?z%`$&Sb_MPPm<C?VHa_Zue7tLB+ZNX?Sa5DVO17oH(
zuO)puT@F89@#fWq)YnV+PH#AOYllvW+OgYNkB{9^+x}!}UYB02*Mr?I{-s8`@Akbp
zbH-=YopY@Z)L!q<V?EolTE(HK>P6S10_6uhHs5&Glr1}dF4<b7-*(!Ghxd{%bg8df
z;=sh#aDlIot3>1Tv&RS4IDJZSTAcUz;;&9s8~zPuOw}x>9hN-1^nma3hBW1UYFBQp
z5b+GW;_=pHol^fiw&y}hv$A|%w7d&vU^no}cAIeYXHLqms6+Z@$+s5d?>&)zcE#!3
zh$PPwe3$1P@l@b2@!)KnRmkA8c*_5fmv@f!{(HgyyX&IVY29ggnfJcw%5-ze9RE1u
z#oOFd6L+mUa_`dF+*i7a_X1~JI-@v6Az5c#P=eE%EzWC>XsUkFR-Lip<D<nDS?x^1
zOphLUWn{63${2=9xwR}<K6g>NlC{7S9iuRhck_&llrNu|U$ex$E<3V_Z()YtGl^9y
z*?9{^vQ=XZvtP05J$ZCA$c>5V`NU62DI3=0?)qS`AxC1m)|)F&S&nYd(_xB#*YG$(
zDf!zY(Y|-Tbxs(S9Wkjn#j{CZ)wwrX6JIsoXnkjAxGG3}@0kr}ju^b%7m)IYscgrw
zFG+_KMAol4$7NfRw)RO;=PhM{yIKd>&UVVY?-aTu@Spc$u-So6TNW$UrXM)xaZqRR
zXSrAF9r(YT;yB!Nh)+*fGL~5@pm$>w{{y39)ewE16KR{ma&jKM(9Y4{pwjSyErDZZ
zo_J=L+X8)Q-lg1*9!ArSnA~g7U`=+Pr?+6&@+}Kk3<|j8q?W5n9B;~C+Ro9W-;^!B
z^yA4V{r`IerH`-vw_@||5Bh$4PVYXmgnO&7y1Bbq^AhXO!~eY4t*+{)>{)BN?xK;$
zqOe6w^2Q(JlAR9xC=A}DsVZln8t{nmZ@02QA(KFnveNqlYc$M+ax{{ih2B0hzVh;*
zOF->`<xey|KRNK+d_|M{1lLFf?u8MJq6ZE+d%iKvDbRL0mdSBi?EkGpN6nhls=JPz
zNwqyIeA_@bp6yiJ1cAs+0Zl>1OL;mhf;3(kJrgZCrLZqwEBWb<U7K`Dncc#b@2Dx?
z`LjG?O|I9`wYyH8=bm};&8u_f_gEMFXyxf>SF?I*^{x5bj>8HclslhX>bjB~_~@|S
z%oCjrYF%?)2z=ipz;CGdPw2oM<<D#n4lqo;xTLv!fx4l{B8O8Bm-s@TI2rUfsb$-I
zw3S{P8<6X4_&F%@y@=`LzI9Dk;?7^uyKL!FtgNw$DRMHaa|h3D2QAJMC-NRP#)VpT
z=b3zK+Ii-IM(pLp7{*g=`5wJK8VTAOIh!Y6yC`M6*lYX8l37=zyQg}4E$}-~tp2~S
zNZq<2s(WdaJ6rVp(5SVgQJX_e=Y5U2bLMP?X3X0OGaeY0eA3+&QLy6CVg;21;RTCU
zvrXOis5`ArJ>H%v{e!V;hkJ-qtHS-q6#|<QgoOk(1DLb|6i;ySc&~6gf1*ca6SHQJ
z`jjN$rz@U#=`GmNz>@JQt4DS5jg`C!1<#WBo&^*g&B}`o;t}R<*<itxs?&TdYIA)Y
z6Kkf*0yC}SL5Edc4vYPJ<`I`-GQn@_hojvy_)h3Bd96ERw(q^(qIJK2G`&*D_<3Ne
z;RZf!@e*x+4{cLT?R;jVJ@330l|;!mN1wlzv)%7>QIOZgZ4tkma(?<}Up*K5Q`I2q
zkWGun!~aZgIz_@5r#LXOm~(DZP=9inAvMWHpiJ+n(S=7Q`ky!DKg=rnY2^R6h(T|<
zvWmiNfz^q&-@9}^*ejKlh@Ljmy__07WtC2EwGOjktoT>AV6%|?vS|MPSx=n{%0o6g
z<s4v&u~K?rlV<ASJ?%x>mXK4&6g0yow!B=T6v6#SvYvI@y0)<PwPNP0Vs$)!Px$EB
zewugM#ljQkO^ZK!`-Q4+Km9l9%B~Z^#ahpco~T&woofB~TJGcR+y{KsOM{}fUai|(
z9QFKreYsG>+w!`<(<Kz$(>I^x6YXPGOJEW?&?p;GAgaKq7Qx7r(4^YXB&^UZ`XJb7
zk)upgkiyYfdk)M0Z)8`MIH$sJZT5vPv!^gPL`yjQ{v<Hffk|#h+x^95Or`5iC@??o
ztGvdkExSg;&fju!6HBae%5ip!DXz{FnKbM|*c`Vwr%qsd+#V(7Rkvwrs&@-p_QIoa
zJgHHPwjE{$7G3H6AzoQIr=GN?v??8U37GUPeajNT<|d`c45fq_ZyLL{h%DO1ePBDw
zv?FF!lhoJe<feZ1T)dg*Y2T_R*V1l(%TcXVSGSy``E!!4<z)WKNwUn7r(cN4N?x49
z*8X5kzUYH|)g4XD)0-wcOpTq{#H+-}!_@p&MDa|C{P``)V&54UKl*+xy6|R-0Pn^8
z&sP+_mH1mXspz??Nqt)FGCleKHjf1XdMxp+fzJM=VddsspE4SpuK0YMZNK*Nfk@+v
z%jPg<yUa<>j(fjpPpaz!Ca1pMt?w3y3UyAImQu<1SnS8j2_l|~UGKvcI^HZ<ytPV8
za`NlUvkMX!G(5KW$1pGDt~Qj|eZ=#ism}r7dykgf5wonFEgHC}G-h{c^yBjSom=a?
ztD-q)L|ZcCMOvM`q+>JL)96l0qk2Q5;*TkXyPEVPnzbXEjb}91Mm4iLOfw9hW^OS}
zy`n`q&)Vy$>aoMNuRho@pR{F}q@aEwA@fH<{ExO=35VQ>#Qjo=tY_ZUNn{!HWvUb=
z**7O?3n}~iDL;&gXy9GY(HiG6Ei&wo%KyNi_2Ji^iET|aU&Nbc;JQOqZK8(8lcO1`
znF;}38Xb#7@{|%74)<27_IIuoIUpXhw2<Ma(&nkB^#y*qnpYmSXxgHEEN}n072Ce?
z?EI#_nPtVLYm={rO;(TOoO*%N{r05w?~D(1G3OjhdlJ&dY+ew&b4P4sLA!K;=mCX&
znw&hBb}p=DShRbmu+y}~($f-|)1)3*ON2j=D}V51Q;~l89>eW>CPpf*t5S}uShIG;
zC%f&+BE1KBzACQ%DWo4Ld_1GNh%YTsS;Vr~*w}behPd;Z9|4*C-Rvwe8eS>|EIYD%
zw>KESEb7fR;;f#x(Jl9K*25=8Z6?3w>HeD4o!fHp*Z+6odHaJsj)le?e7p2+xZ<1j
z$~VsMSylWmsMDJJ#*&ZMb02>@QfkvsYjga6_~zC>$}gALWbJB{{?Rx!rb+Y04*eff
z=F6<G{4u4uYK3vd3eiWWx^K<+y5;ohYKCdMLwed;INyD|#jL6nkioL8HCdv4@2s}{
zQp?p7n*DF?Tq$$r|0X8xPyX6|O2>jqMM~ECb0p1Vs!=O(N)StF^wrwH#ulB}<v7LJ
z-^j(&q|5US+jO<7UuG+pZRnhlay;+U%dD;DZ3W4lAB>N+sy+C&b@7|5>t#>vn7ORo
zJWp;~^Izs)p3(Nwy_3piC+8dK2V4*rW$q|DvAuS8{@fL9owHANS5N&vReGmt#m&jl
zw-(1XU948_uu`0)aNGRQPV=ND^#{`=&l;>v?l^Hr@1xP4!_xkOA7@0~Xw?)bczn0@
zX)ROsaev9MfXKFzaHhop=j=0h%xxCWif2xCH!?ePLCEr~shHgy*N+d?pY#Q7yz=K%
z?Y~tUlYD-#6>RCtm5gr6>nre>ti9-H_$|4rMY~<R^`iWie2sXl>$k;l@yxCM6`C^h
zr%XS!Am+VF4EGM^3Y(>#vrZ(=t(kU<;ia?S#CiGB6|0piPO@%4nJdw(YH_Z9)~V{K
zg2r9T8?8>C-F@14`yNyAmD)+nvP@|_DNJW$G8EoDOtSFb`)f+J#Jk*xT?b`OPrtqM
zf3MW(k98}Q{aYG7?UCqN>mL4?!8`f+vg#K*V%2h4O|Mvbn7LSoL~pQAJ+Be9>Cm%(
zTpRAIUozd>pQ6O-xSNM*<}a?~>y6B@a%r(a^<UMWl|;?|dT`U*6K|4>l)WndZu6Hq
z&aE6ias6_xrpGeIH5U9EEVR~~Q{M44DDmf{qKWhMr|!KqS^C!Ec*dwm#Wl0veV?bi
z=dZ%j=zCX<<oSOWJa#UcUf*^;dM~fgy_b8ZGEe{edM=YPe;UVU|D6f#$7OGP@hUMt
zZ~td*CPUL@h9GA4DMlS19yGM_NL$t1*ud-D$$!^G=P>J{V~nXDTW|d+eB3ZWS#na2
zhtbo+fqapAD*uljOLdvXB7UtpLtA;eS*qBypF1aTA6qENK1HBwi_df+?&(scQA}CW
z8H1Obm1Hf_3~dYBp7nRuRPC!95_U(aW<OeUt2OiHyuGKy-W^`YbNf*2E!FIhW!3YI
zLQidYe!bO)vs$$ChN5IUlc`>qO~aXKJ(J|!(^eFpnx-i|#V6oHq$3k2r-+7vuGH5T
zmpFxX%PAx}Ts*x=_4+-f*$+jo@0Ql8w0OAv`NM<F$K@6<SY|%qiMB7%t4b`nzau{X
zNbKG8gxBkIqs4q?KmImLgnfrpK-{Cs_6MgYC?4Ff;pvGbx3(l1PZF&Q`z86v-jbn?
zccYMk3)8-^%M3=UF9It0t_J_-c2SvdQ8?aKH`Fcuwr}aeiW0{Zv2rCfwtKDuQ+M2T
z*67-@s6xxC<zbxUu^+*0rvHwzvTu57!O9-&^C5YH%oB;34z7tu0+O4iUYh9WYvg&s
zF;MJ@lBV}0#R*gO4W}qfX0PoGaGO(Lc(Xr<C4Cl?*OHxSnb~R+?=YM*oWe4<gOh#Y
zjGe4sC-if_w1_U_mu0`rAndX60!ypVsSEks4}Y#;V)O`?az5F)>={GBI!jjZd7pk>
z$Z>h+sg%MzRZ(EFzHX<MQ>@dHfF!;sbLk|Btp^u)$gkY8+Eeu_*Ba-&c`Fk(@=bIS
zXkK%mWsNWQ%}WBBI}|fR)~*W;m52?LJ@!8^G`dn+X-9jb_KY2}J+H5Ae_D4sFET0f
z>Dz?z#AlO38;%}dD0!pn_>}&Cejg$aEZ~(XSgve#MIw2+>YNV;rA>8&B<Ids*}}4H
zqg&K5iSSb|76}}6o3&^`kz<>m>h)<87bvBgIpiyOPhYWc-pQPeEc@q%J)cvYSF&>H
znNGGPTiGYSS#-FQbBn|!*2?1-l32bTS>W}Yb+>@zr=8)YmnA1Uxz!p(w%>0|6t^h5
znIgX9LgR6<DX$xjZ!5a&%Y5Y$-`%D{U$=Wz&t)ZByHl?OKhX2L@x5B*b!^+J+gro8
zJZPHT@#yg*OS{UaQ?lE4KJ7NITlCc7D#x$+%r8zH6tiQh{J-LC&bobRN9TQg@pAcb
zvy~I4#O!(%JY|b^%okUI>+1^CH%NsTy;{5aq}8XBS5NL)R%REXzGBHMPg`d#{tb-Z
zIz!LQ={OYoYZ+64lDMt&WHbKf%N+gH_O7V3dm?-I)>f|xla6efE8x6Q&gQk#-|dF-
zM=aLzu3?zq7P3izWtvK(K%hb+i^}ATqbj*iIv03sIzE|a+Cko?OlQrUhh5PcErnBl
zbFwpsvst%X5G`Zq3O{jZZTUfsSmq7fJolZPrL$PtuW5#JYP;`}^KD>eP`+`%r+L=p
z9G*_D6X%kcG?X;jmMaFC%n}to+$yhIXu9&?4X3he8(TdT81`mL<$093-TuFzkl}iE
zho8abmTA1NOj^14iqrzsRZp$7<nIjr{*S}hewAjo)8oDc64Nag8|r3v`b_*UwA{Y4
z!eElmFHXr45lJ5_7Q3A6qqUzl%WwJmypj8*|1B@cQ|%LilvdVwth=>rqS=;a#myl-
z+Bu2*F|UNw)~;Y}dDW=RKXIYRw@lZ&RX6!!w=K%vvBp)qfYaez?8PwY&Y0(SBqR=A
z=}@+9^grpby_J{cWLk#N*;!|HT9vE{t~{~&#!R*cY-)v%@`H6Qtz=rfyT;0Men8PB
zC%#Eb8B<n<e%iBqGSe!-4n`5BPruVY#!UAs+xF6rIlzgjjCtd$kC{GJnqC&3p>B6A
zQvUlk)O0MCdb;OB<f@vd&OJ;PmsbXgZkk}bg!g*fr@0}VU&<1GE&o@yYvK|WR$G^;
zDy>~kYGz?-Hd7UyD|vpbX8XnE5Yqi>L)@VSfk#?IbR=0f-EinibzP!ZR4pc&dyy;L
z#&|k=B5PE%BUk#ihaM>f&AN3DJ-1&wES7h6srxsp?Gq!8ML*}dnDA)pqE#(RwQN1P
z6Lx)wP;tB#*YmbXvX1HIy<Wc#7m*w3S<8;-m2FI0Y}uP>yv*f7+MCi8rOU#vGw0ra
zYtpc3?ORdqn*mj-CK~o!y4fzjqn|hQC?5AbAhs&_sPU%4oi{F?kUpw;y!VKjFK=dV
zDZ2nm?-!p5_WxhMTfmcMqF0bHOKGC~WsahIr)5}5=NR5iNebVzuGV<Y!<Lt9FRJf9
zmMd@k`?<f~ztU@S=USB?iBE4{nYQNk!ATC%jbe9|zNoJ&%ng^#Y_?_l;Jrm`nR<5X
zYlfneTt+oV<aV#R?W_04<m|cr;z`+poaWnoF9g0&cafgdyZykuD{GB?g?#T>26oMP
z>?ktVi7n{cy@)9~iJe}{y}~97%}Fr$`q{g4@!iSmb5yl?rwGgLj9x9Z$$r5z|5F@{
zo1OQC7!|JaJMmT|ep!Uqa*LhoYj?iPa4&XlJn}|u$-1BSK3-C{dv2<}`(~|28S{h&
z1(pSVwvRsNtYDaEW|n&?;@89fw`#1GYJQz!z3i(_hROy;&5p^;=|ZI|ooiPFtd8P}
zp1>q-bLauH*VIP-hgX^MeTCSf3lvReCNyL*g$Lbo%DeFCs))g*t&9t%rYw;#i^>k<
zj-KK%jhW-4lgFOU^j}SlrazBXrM%D+oat!u>zC(FWAB~yg$_*f_<S>htD`E?dCacs
zPMO28b<WiVW%rlrMg)mB#eB+J(#52|!@+LL?c*sMjZL{33PKmpYV$d6#-JX0G}G(3
zj#&y<$0NxcwWY>(|0eD>kUp?vkG6Wd@~wxSGnnT0ww-8lZpmayjOj8g^}D*$(|48k
zp;@X5tg{(wUpVqtrP%yD8TRG5^1?s<jUNAh3M}8euz@vb*OmNQ>u{%*S5gb5y;g0T
z5w^f0KWRJb^<SqPS2<jgo3!Pqu3b=9^n&NjJ9!c}#x^T0{NXaYhM&tV=L~D%qIFlC
z&i}NWxV^Y^)w!T`t23PY4%@cB@OgeJSaxFXn+Z;3OD=ttJ~1i3YE_7B=9gW<wJ-Ur
zF6b5STrhWmykmKk?|iQQGaFacPLfcvX_@QCC73Jx=<$&~7LuIbtdH`#mzTIXD2A%k
z{0mHS>YJm`vEKA@yob=7;@aLhF@Bso_6haP-g{>M`=>J}C;7`USLrUSTNw3y<?qum
zcWqx)|C2f_BL4FI=ZjZPDz96q#k{^Mbdy^)=cUkNixkd2`mfCrBv+mu7V+ID?YqR<
z58UUT2yRozogwVR?#Q<N0oS?>!Uf@pM~<k(d=PI8jLa%m>|0!JyD-oGP@bDh)GE>Z
zKg$e1o5h%W#C&`%bu_g3*R@Qp2kH|A3)ZU^yJocP-YT$jYV-{ek;oL8eXND66=myG
zgw8&d%2#pkxFntGQg?h~?d|7<x3>ytsmms0*iB+8k@zmfQr_aZ!ytb~oK&IA!SdSD
z8?|@MYs(_qeO(-~n&p<IDDWvJgkI-jdE%+Wl&Tcu%@=0=)<GzBgOmHhGR2a_Z3*n^
zflSg*xFjZa+82r}C>BvoFf}{myD(T;WwNqmQ*zoQE78ezLKE!nn*T4U(qNpkfpvD0
zao=~1zVe`bf^OYS>gu9BJ0&VN6jrR=oR-C;(&uOr5tJEoEH&y{YGj+%D)ZjZWBw7>
zQofh;UgGF&+}yG~J#F*%;FNCyiQnotCacU-s>pfP_vo|Hu2kC@6GP@ZsV(^|ReepM
z>r*$6V+LnJ+az|ubB-D3gl!)&m;5%B+~LSNV@pl8N9Odv%u7=gUZq=a`&{=zqu|0r
zff-F1o)dC91heDavv;&fa2*j~y&<qyEbItVxV)m^HipD<c3r^}jm4&6^H>~IFUvWZ
ziqxr#xPP6z%h6?SsLR_hsZVb8s})(lJ}mY>>9*|v=MT4-LZ_+ChcX2cr=E2DU;pV_
zZP*EqHxE5>BeZ5-R&u=|YN#&W@zG7#m@#^Z)Xbz-k?k?u8>O!sr|(@@_=H_hsW|BC
zGC}KRnfQ#NR?8XRzfBQpkd9%FyCWEM-*MVxv1!lJgJhX&uRWLHESM=YA=lI}A*{VL
z)Z8n2yQ+|z5L=o9r<zh-Mpy~6uEY^Rl^cpzn@sPd%$}R!`OeX5l}1Hckn!=3;)63;
zf)rQ<C$I<#utphhZO@o<;-=Jzm8`P@xZgC)NjBh0Ucee1!2emW@~DJcNlE^t!l@55
zJ&rTZJJ#WH;HT9r7JI>+l`@6F?{B7lI9b^k9PmAJKFiZ88Abokom#O6QVpL-eP3z6
ze)@j{shQR3YJGb)E5{_rytgo3U{rJbxj@6Gz~W_sXP0Z}3-|LE7F}DK!9F>oX$edB
zL#|7gG;bPeEmO9AB_Wq}GQ`3(;QyDZi%f#+C-pVnl(B7=xX4_RVlZhvV`!-9lHjcl
zEGI(QPE0ycH0$cMoUJKg<<I4uZ;E;(D%D@gkS>^FFk_0zjwzCp>!m+9?l7FQ<^%r{
z1=hfXxl0zXW}cGY7s4HMfUEC@_v`@f4Joy*9<5Q8`F0#p`pr2TD=jaX_-+dge&`w(
zTv(8#7=3PPr5{J@(us@KUKO~>IPHCNO3aSJj*Sa%rOPsM3NlQ{@a+uCX=zn5S}`%R
zaKb{9q$$lwSN;bz{FD-UAo%29!p1PU0QC;`YYLNgt~T$om%bR!`!T_6migHevkk6x
zIk9JKjA-B7(SG)p->w&{Hcn&+3gG@1z&*==;nGX~H3F<_0{GV);M-a-_p1S$@<qO6
z2F0hl=Jsi@o)hd5o!Wg;QRXXGrCqVH(T&*YLB{Eq;%1dh%UdDwIc3B8uRYz2eX1h8
zcW<QJjp)7fqW9_v&QL`G#$8G~UDFO7-pG7mBm2Y!xe>yd$EuySvC61gUk{w#a7f_c
z^7M;G7lx&@sf$l&4V!UYo%Q}k&QnW0AFxckwQ$2!;VsiIS6XPLZ7c8Ny%==MHGkbK
znaiea4(_4qFSS)yI)q1W?R@w@)L^56B4du=k%nvAC086<%Kv@oFZS-POzznUb26=^
z<SzGLQ(wOC0^b9x<(Dnl-W=faT)_JDRamAKSN3b~H3x*EA9%`!7pa;mHHjE3P-9(s
zIYjab*K`gi84ah@0NE>_CO7Wd*<!UJt<u$yeetTSh@{F0YgRcAH~o2wBt@B}*qlAM
zUkXf<(%QthdzPW0Ui$RHErMl71e6}TH&wb`OO4?ykrTSSsC`wEn}(~&bEQK+WQs2Z
zRsRxbJ1yXSJYm7flI?C@w_opFq@u7mGU07`rjJ?!GsDtTYU;;N?mNL*a_na9htvB$
zMz^21Ip^d`&PQHsS*Q34O1sw<PW|@GIH~=A;-gdY*}nxZcJBL}$e+A{^Tq4^vjf;1
zx3BN?o;S^@b!l44vcs&xlPtbHWLqb2s4Ss#T>-ml4Yz7e<Hsv|dN-!%XW1`Uz2Re}
z<foOX3%Az^M)xi?ILx(L#n5J&oobbHPu1Gzi#~{N>S)~Zq5O#W>lVYKtQy(<TPJN=
zqLf*t8N|1dq3x5@UX6?^vn6hq*X|Y+D)^c&`EUZ?O@TL>)4G?`@if)F*dEOKwazU&
zts_t~;f#iksQS$^SC_2NLrF_GT^f!)3f<!-kg!$9_f;ZaVR=e``s@b>T5g-J6<|GU
zIp?J1oT!GmK@(V#8M0qmag`qC`{%%w9AGZ@@jzvl!nOtfSs!KbufM|=ov{4rEw1DR
z%l@=4JF=5CY60uv&K<R1jD;DpcD#<V4rH@=$aX&Ekao(>vu7I13%oD*oR!JoW^C!2
zv%5)qg=Oy-srlXI2Fq4$U%rCrR6x7pE`I0!iH&Y+w@#Q6HN9_^>+Frwc@FNbd8sis
zOZ||xP(sd(XP0YT1%=ibsBOrW&9OMgsV?+bdG(!%o8<%LdDJ`|r8*ef%>J1sFno(Q
zTQzHANcM$89Y=3X+4qG1u@zTA^4#PFoa<w_*CfoneDhMR_qvkJtWUK#qXKd#87x1x
zWxurH`ePbNYumKDJlR~`)iNIWwH{A)VwYy%R&joGP)XvD@)@U4-$UVN|2wI!af;5p
zs$;?s!+T9>kA%)EDZQL)p~cr^4E-m6?#;>V70T}A<?i*r62QQ<VAAGb&R42(r>`rV
zzAo$CCpGiJcJ-qhlW$y0y0Lnf;H*=-=k7XM`>euSdy(CA1I3GM+g40}b6K4^{P48T
zho2uePz{>M>R9c*da>Q-3i-<ule(90owhY1dXh?HZshDqf$qZW6??Z|=1SNt_{fXv
zuqD@hs~xi+aIe|Gy7&Qi)oH$c6ZSv7#rG{?IkUqYU0Z=Q8#psck1}iC-9BU6&*$5I
zKl5AYc0T&RWWTMBXL8P-TXXNF?pduX_s-_9Ghev(>F?DqviINXp1r7ZNJ45hSGCG*
zn-wqr{8w6F{a~JTrmoqI<<=5C%J#MkTOKLeJAHO}Z?m#Ia<};NB0Y81U9Y9~X}eXQ
zI=@2qk#u8*R{+~?ro<PGyZ20dAozH7_m%F;;*Wf+cdH9#-;7qM^+<5}D&T)reg)^g
z^vR`9wNFU#Ug|r;b@u0hB@J^@3KD}7eBN-KOiti0-=*_rl|mBxsjF&b@{VkquW&Uz
z(%6}i-}h-t8@u$IP%Gz4sn30{zFT{>Q0|)6muKgb1fLxgxF2?(Tk!s?-&ezW4{dwE
zRy-}m$yn{%kr2H%9!&QvRw>Ol^*NkTdv5CCBgJdxn`jBR^;EH}ULZO1fuD@^(#M*M
z<{9NovAwx=Wz356Og#Su)l)ZjX$zS}x2NPdOj6V^I(lpa`%DMz#gjE>nx9cvWc$Qh
zr_yFu^LnmHYhw!nL~rvg%x?P=s>c}SqotEu{AAXp+LyPuE*`$KBy0I%0md7zI{q|X
zE)90r<HKexU2<T~w)mFgAujix$@-sO<1}^my(@F>hw?gI+VlQ(A^X?A@3}v|FFG!e
zC-8wMk)6%|K4boevvc0dSiD%>ZDiBg^-*<&^H*n%e`ifoGj;96k1Ut%=Gyflb)T!S
z!v+<twO2K@*B3sTZX_ue8^f_TW{!s7sZ%$mN*r6am_d^<MMr1rfx|VLb*Eo{%-i8{
z_plAGt<n}LAtB$3n`BE58BLO`y|^aw|18z}tN8=D#h(di?u%I?Al)y>s`RY#_{_~Y
zAtf&C-Fes6WakHIg!XVLuuiDBIxXXSY5&#YcUNomlxo%s*6{zR{`Wkw_E58(O7)^P
z4-eNaKKtyyhpxW3CXu~ih12!$km?Ed<!1I(*Pp$fme%K5Z_Orc{XX@E;|yCpZoyv}
z)90?z=-+EAx|HFP`h<6OZO@D~Z$7O}k=GHp`$<l<U$1L}tC9rcuGf1vn{tV*Nq#2v
z@6MJlV(K!%cD^UytjpcHKKJj{`)?waF1dJ4)lo}OrSPqJeXxT$+v|5HH%KsMtgc@<
zamKy3cY|IC6wcO2J<#~>?$vMg_ZimzXL|phhf$!M?LUu-g~Njb*6RecOgs!8I<+vr
zktj($9_rM+J!a=arwQCMb(yW&o&+sRKH10YGs)4@?Wy}z!_zzkmc`Ew%r-0(I&*XK
z^Ye{*+<dB0S6&1zVcz}c%@Sj^<qPe&oqS(yxx%nQ)0OS%q13MM4e`rGY%GuFoM23z
zs3mFPc#@?_n1v@Gpfn+tmGK>u!isW*`wW#!CnR}n5e-}G>b&Z1=Jk({7dmA!EuF36
zc<{8#WQn!W1<WfCGe_{s+D;IazPOxyOVHV>1nH)A;nU?hc9}U|$@RLe7P{s8f&0wP
z{98>o#2$RaJjo+s-yWlTDqM^BCoFK;#q|30X8sL++pE8xySuYo&FlZZb!$6a?(*o&
zsY$5%F4Vb5eMa2B-@jbniqFYluRZ$z-G-&SYq?ZbX|LgUq_UJXbCH(g)D@F-)K99-
z$yqjwW73)hypvC5HUyn&&~jN}wNvv*qmN9-5yn|n8&<Af7xmFlQgL&k){$M0nwU45
zNn~^F=t|1z+|ne&pr<Sz@p_%YyHz@y7yrpJYGBcsoYe5RMSI26uv<4$m?v&~kg!0u
zf6}Ry3nmG5P2qQMDOe$uEu+QwDKBN60_Qe~cWz6*#prHZ#3wvOA*@QebxI(c#gqm2
zZYZRh+{nvro#Mf%rDyj_Xo5oUGnES^mcHz}c)XfyR5`qY-uiedgh^MeSolb8R|doX
zUZGW2uQKM${CX{#eTT=w$Wq-G3f}rzN0p-MpJp&j+Qzl==Hv*W%%#%;j9x~~+_cp*
zdil98-_)lYZJBUhdSTS6l}qJn0yIyh?K-uby}Ee%#f4T)TI$TZg|v0nvvGC0Z|+LU
zUeI{k>XXN|S6N0^7k|(?p?_wDkHW?#x9p|UJhx{|i|MnL<ItL3w>gKQ?c~&^o(mDp
zYl9Z$NS}IoJH{<H?M_zT%oX>F+`P<Zu&+2euQ4mrJ)?1_vPb8uK0OVa#oTGHUoKjx
zmyz)4m`KL{KOKjp?ANL~y$||*=-e;6C-<~lnT_tO$jF^+aiMVa<ZYjJXc?(&sa{^7
z`enzvxh^`&NB#@=J~^`NAa~J5*QQys4BZ4cCtd!yQ~7_yaos(i3YiodO}aKVKMeY`
z@r}$nu1AbtxE6VFR!NtX@?Cu7<YT0%wkl81_{gdEhg^?j#<D(KV!q?J|JupB7MY38
zt7uv_IZ4=_*L&4Z`>j*#s$R9<zI@@9kZNnMtMB6S2KVl+^-qFMtDpL@@WS!U73Vl^
zB`u1UQtD7lieFp5;zzXVir!9(pxsxScPQ0w*6NTxb)n7Z#zK*l84b6uESqSwz%@@;
zXTlacp|z7|q(m7+p7hQ-<T!K6lGrJd*IO$l&EOM!>RcSnwL9j9?x~tLn(mse(oMoL
zGK#9&1<Hr_E!)7lu_u+=i{r-sz&nRzRb>kqxNn^K(3Uh${g<jf3!?)QQ}I&Ut4(H0
zH@fEUOPVg~)t4;h6`H=xc=;-g2Pt)0yx9TCd#~|0q;0ND&dfM$Y{l1*YVo-@u<h}t
z`MgJLPjZIFPI@M@H1Sx5OJS6f_)m{T6S#b)2}xgnF?qF@S4!j387HouOPkE=p=O!q
z;II8JO-Sg(id<_YXXz6U790KPawz!hq3fEoAo`f!bY3Yg{rArnn3x{Z)ST0ktdk-v
zG<TAM?iEk<T>*<`Ypg3Q^pbg!R^n?|rju@UO6J*~lRLMFyi8!T_Bz;NFi-s!Yf_j`
zgV2tIy;ob7$;JrrY3hhCGvrP25&aavqx^rf(%pp|F1vP56rPwq-B)!=q{W-yCkxp3
zdi;H6-nmHQhs@O#e_iwar+=E6*5zdPb*6splT&A^SSHzj+mIE$JB!2X@%1*&Gf$QX
zc}7RK2r21JNZ>ZkJS=h{jBWn5gS_#Y&C=U8G_rK9<durNwz<nlC@StkWN>fv))iMB
zb8@{`-+3j?vt9H|Qkt;Qp5(M$YzNL5xGgO-esy7Y*Q3c&jXI9%oZXQI>EW6NX<UBJ
zmk!!Z-@th0s2iKC=v1Z!(=Mu*1$=zMB9N*ccH-8gl%y#PpSyxIZ^^WZR<D^d|5=6f
z1*O)7)3-+2Hf>vKc%gOTgq&N-(w+wHPHQJ_Qte>Mv2OeyCUAPv--Z1Re?H%4Tywlo
z_=cvWM?gqxfn%=pjYCxxU$#G=mc$&hfrE3|g|@JSbv&sGjN-Q*aLLt{M~3#k(mgrb
z+h}I^rl(BWj-4mXCGC#o=bV}w#VvGf*?|>22D6?;{gvKx;@nzu?puN8yCVgZbQi8)
z5X@)AtF@}pVU6eV&1cm99ogl0beU+(1g4k+Un2Mlm12yZ+<oxsws6L$B`y<~3x11q
zs#~cp^-7V*b<Srw|MH>5;w2pCG<Kv1zHQ9!Uj4A}v%%9D6Xv})^mOa`cxQ3+!!^hG
z*xzt^++myj)n|Kd%7*3{7Mjb#b}Vc(bevtl`s38BX+Im*?l`Jl-sPR<`TyEV`OIs_
zR;+TIl2gzuzonC7KiA=eUzX8_SYFpM=ZI|i_ax5!VQf|7XQ?c^Dh;!mvpp*oSx<NI
z)^VQW(BApuma>*&C-d(+&5Dy}9ta8K+O=@rcb>~#x*_ew@i(0~dna9U-kow$#d+%S
zAa!Y{o6C9&*CflvUv{&c#b&zrrR3(RbKgkcUZe0<Lglxb_4X#-6yuzm+e&(tF?=aq
zrdQO&-?-~Eqv6{ZD>!uym)tJcrdu<8=Mot$_L#)G@7u~1l3xaCUTMpVXWYuNZicG%
z)n{!|Z(pb|gzxaUBX+g@_}Pm^&H}7&k1n`(bbHGmosuApFySDNXB#h?lstA5xxf<Z
zI_v+$Z|@h{ev{6t+kDxr>GZOmkS!~D{TY3&LgU$tmTHB1`hQvSytP$%h3C~jUMY^J
zW=;sYCNj~cjX!JS)_fLKO@`Y~%Np<H3aGwx&egBcTU=seJxhB_`s9;0r)aKya{b-2
zX^qUnKlFUEl^Wk~vz$^UaXM)ICp$fbuY8UE3%orZ30c=H;Vm~%b!BTQh&2>lrBD{H
zJ2xtwYg+iK+55FPw%@wCw$5%%z247j)s@%G-%7RREq&(m<m<Z`J3`|gXm|A_<j2Pz
zKC7K~b9MBThzYER!xAS5&du^ZevTpJn}65Z85u2`9hS&?atXaz_44f@>lpVu?!B45
znGtK<FY`SAt<C(u-myw7YS!70fBG+;m5VN%Qzej_v35ewue9iM3q96uU4PeJC0Bd<
zUkQQvyExdin2*giXM1E6&af(?OHU|(Gx~zod#$B2ET;7`2&DX!ViXkUow4*w){a$O
z7A%$0LNW)tR$KJ{Tx$MNqG{FK`3iF`9A0+drT)~#j?yVIuZ6XlChy!XF=N(huD{0n
z=38^k3+`JGt-Sc~hQ-;;njW*11G_!7y0`+xmp*6_YwEgl^3c9O!TT$BefDgabaAId
zv5@m6;l4@@C&no<!G~XU%DtJW{Pw4zE!TWq!A*UR^S5rcvJqKnyGEC<(E9GA-QOb}
zY;SciN(lEA%;s>Cj6SiF`;gwU|4Ey-t`uebHMM2Ok@$-ozH40g=O}Rga<QDOV=>!i
zf&eG?CI#Lq&N%`LGbK&<s*dspDXu@|xMK3rxe;4jfAe}-FSvf$H9n+slXO@75yywC
z7D_0t*!ijJ-s^)3mEGqqS{^spx+h_|O2u*$6|P$utkE}EG#_kx^}^&h*XmU#_MNbr
ze1^f<?qtv8O!gZW49nlxJv^Z^J%KrCwWN8~wlhMy_Dyn^7nwD!Xk8g7RiG@s&133W
z5BWTAxy6l>c4g=}YAI})vFYRvbH}KDp^T+lH*)m8(d~Pp+q;Hi)6WzAEel$;Pp(U_
zn6gBBQpb*t(>CtT0&ztrt2)=eF76Ry5r43wbHaaTuEt$lU5OJ{7fe37aoPV<2dCe*
zRe0(6#-r<@=82CSO%Eqdx+Ey770GwibLJFHjyI1cDOyQ!zSyu>V`08#hggx?RD;uh
zwY*bwIHrAeI&R_d<>o$5!&Aq1$hj(ZIImohQoPNogfp7Kqwl2i@fl2-HyAz|bT!Wr
zUhA;3LAXov^^&cgVkg%eN%<(6c+zA0<5^7$e8M~Su(G%wU$Ae<jy)F5t1KTcT(wIq
zJM!#>&wFNUkTrRzbvb*>mDh7Ccb+}A)Az<<-<!&({#H)EUA=v3=H6+S_b!>k!1}?q
z$?@D0O}pJ1`!bgZF$6SB2w0<dTS+goRn11@oq^D$&+1Oamba$NvHAaS($qxOn;G`6
zUasjWGPMoj-KU~^IiT0BsOL=8;!Nf<i!;v6yKHuClJ1#Kr)P)e=tk_iVb!(KV$*#l
zX|}&>dr~+i`gE<8<eK<L)JEEI`$8GswR28g6y&${+@-XBmDEAbB>hDa=L>rTSTC;9
z`C~krVP4Pcxl$=IQhN_RT5`HCa>9}$d~#>EnRNBNtT0%xS!jykE}6q#eg%h?GJ3mp
z2WpvYjEnN#Y9oGRqT}KdJZi5GH*=ZKdSP|a(fPR{*S&<+E-veLFL|YtR_qj!f5@=>
zlSh{=i~E)nlXZ9S^xnMmXys&Hjn*HMT_?_nGR>OG`BsHXwZrbL&##3Sd_)w2na*1L
z|15s=fiS0(>uF<GErX*xr>^kd3JHC4C1lFc_eU-+3*K^-`54d3qca>rZM!CJTD(Q-
z6>qXkN0N(0Wa^v;7sUB0x}UIY-_>FFM8xS+;iN?q{0v0St~w#{cE)61M@NV3uoDXI
zvqePPbhy;57N4BN;Ag>;sNI!lyzOSk@#MqjlYUQ5Yc?wuJ+|t=I<JH0+gVOT8R*_{
zns@rc#R^j{brojK1AOaOOuBB#<zjWl>++eOnq7x9dM1c0@ms`iZlKq^W6AFy{Oq9$
z3mF`zyt&Y2>)E|`so}whRhMjL`f$$Rxp{J-=i&uUlX(62F!B`!Movu-ZkQz);k&cJ
z$x@4DPUEUT?f*yBwtewkEVJSCM!p5L8<t0Uy>9SYED@yrYisyyIon0{%)Q}nW-#0o
zJS{ls=+YUTt6gpH-;A6ayXmh^N9gCEKwqvIC%qpC?7qMx|Kg#>*26u%f!yxBZo4{G
z{^%5SICEzA)u}T#iFPHN5f#)knz8%sr`xL|uUK<T+;P_`x%SFRtFx0|F!`qX+8<U(
z{KR>&IxH}D%e|7lA#Y>OUG;r%=gwV`bD^xqq_nhdd*7A3B>tp?oAX7_^}l92duOmP
zFg%qu`o4tsf=0k2!+9?}8fFW-f9N?Ixb~!QZ{W#^-WP7H`IH>@Dck*h@x8w%-8W4&
z+r7f@&JE#Nl3fce`8XG?ZF+f?`G4vXxk-Jz4_KZZy8f?uZH}>=rUo<XS1E7D-QJcQ
zt!o!1OA6@i@F>w{4xZH%Af)$q#t~MYi!qljtb1_bdZlbhaspp&0;}D@I@v(xX*)j6
z@^!ndb<1+2d-u(SErHs$$E0K*{yP=fS9p5wO{VQHE-p>EzicO;f-Kj9#T!^SqL*Er
z^w7aSh~wVU!eeT-_kL6@yS?$yW~E2zNorD$ZmO;3((ygBX7N1+=F4Vp7Ar9Kb<c7(
zU23bNDXk*B@~dQr3U?0+!y=|o4TEowm<_xBY@El@()+`(^J|1l>ue)=M+Ko73|lps
zcUWxtdGiQ=VMy}Q5YM$X9&;_2Eo~+pFn0Ch%>4iIikq29jpV&c(zj2^D#pB-7JDWo
z;%bV($|)WTmq}>4h5K3>y)?M}DD<|`yg1=Q=O!o}7P4A=;`ouQ)*yveZO=177e%5j
zb0tZ6r%z2;EMaLc5~f&UxNF9~3kh3g&KW-2>;6%CTZPDVn_cb~1-5;?oRm{1`S!2y
zHeHU3G7_PBTrqxfA1;Wj`g!;DitFZcXR*JT<#J}_+@Ma)hWN#|8VqjhRi0bUxTl$E
z5A*CDvo@aW;E3^zeRRY5PFJsO1{>dG{$|;<lgo_?cTA13$Phdl>vXWYdX?d!lO~$Y
zbHBV0o*CI-wR8Gk{fTlbCKn6O(e)B$GZ5+1k&S)0AT^$K#`WF*C7#b;eC*0Wp>H{+
z*G`;^SRD3#!mYmBbH$?+Rjx(NnxeLJ-6iSXHBSYj7f*S$#q_pCjfmGz-fI`nx7%z=
z*t)CVOT$<4^}iIaB^rHt+z$@C<}m4&XA7FmJImTUqt$RD$83cij2^1-j0#bKVrg3i
zPu<?-ebh$qXh{Fk&M7ZA9cMq;z9>Z@@6v>sJ@-0t?#!`0ZFctt^Rdw0Da=W>XW|rB
z+z1TW*Q4a4Ze+4%&k5n@&5vHYTvNF`<w?n9>5VauN_o;YO=pN+>~T})o}JkX+w(7f
z+O2((d+pqcc?Xs49M6A~>8L9SueZ6NkfSR;O?S~z?ps0Ws*Ag8q`H`M^cZU#*#1xH
zIPlxLT3MjlY*o#n30E&@3cc8M|C8t4&lgq%Jurw&sF|fu()^&#_C@rpjGk*YLcBTC
zcXJ9$Z{+=FGk2#Vhozuca;9|dOO?9JT`u=)w({Lx^iE=)L6e27)V4%@-Cb`&;@`-<
zI=VJ@Q;gab;YT9+Nt;r#>^L=@os`vX@5^b^DUZ?lp0iK>ww!V7lmwP3habG!^81$Y
zxjZF#P38O<TV@0pM)yVw9MN<(w?4KzY=hqP3oCa$HtN*S>N+6Bu<FQ`2Wm|d-p=$%
zc0RZ4=voDrWvhawX!*~*Gi7bcl+qmImA5W>PdI<H_|{|BggZ~JD|bbFHJ@`sIp-Z$
z@VV|_v)sr3FL#Gzt0z4QyLjT<+Z+Q9=Iz^mC^3Fdb#eZ<Xoc(2d7Sqe7iB9;28tg#
zbx7~k!Pw{0TvIE}EV|Od{mq^%^Z)fk*)v=6#mD213a%%{=}zK$?h)h>7`c1<iM3g~
z&F4Q_$G9Qgn&Zx2Q$4?^12cCVU_GkB_H`|z+611~Rcwj}7cWw{{`1@Q%olBLo<@xs
zCHCLHM4j}`-7%dda>CUN_wJK7cbX)u+b8w<<OH@7w(TZQRK2FNRY<M0-MC5p;<La@
zAGay4@mk3I@!7*kIdeoFFASI4E99o8c*^6I_@|P^SAtq^O;K9<xz8|IPT1Px;KIWj
z6#uR<n|AP}p4}FS{n25DODFv=;Iw(?bK`-J*Akb#Z(lR)Ix>yx*OBN1R<+_~n;(2$
zRD4;-ciHkgSN2p~EVi0h&vWnO-V@7mZXN!0`dv=d%N!P|*atEt+q2I1{ZL%O$+KmD
z(3bPuTQuI?DScbGujgsUTc7+lubLHm^&_tw-nU#$eCgHRDbL@2n;duF&HtX?`Lu8Q
zo~nKMF2Z~4zueiV;M*767W_P}bJG6$q<5RXmCiZ5Ur>C0ua8aI@dH~wPUR4dIKH9v
z$IlDf6*(8NbN8>9vsU0jv)_xNa@nmGog8g1IL#cd`&U^^p5uL4I(il73JHV6Ir_6o
z3)bl-N|&CVI`7w{uUsEG`1VSqORY2L{q@14JnGv2Rk}+OdDaRTirRAdKmQ^AM4!{X
zXo*7q`X#T7j}#WYG*sBU;xDKCYsTKqv*Yc|iW(*uBrI@fVrCby$tX}f)XXW&7qGw}
zX?F8AkzKnUznZ3Pp2C*3Wpd$Thww7CJCX(2iw;fU3g1@}+QvOoHzv>TVN&ramwC)P
z{_t1{Xk46c$<LI!vgG9fU7tub*#{*WK}$^QCwW?ZetwP7*q%?PIN;6YO=;)nslG1N
zx!ss>y^U9E+LfqX#jmS!lf&OP1d0BZ(-E!M5Z~Hl$@uI@>M0ggE+(6476C;KEE6VA
zNGuKb>~N0RCR^=lR(Fevj;Kn&0*9taE1B=D>b+a$AiibG^lSg+X8hjfbYz>l*SsA|
zMei@26}i>w#;M{@ou_Z|MA&_({K9zQ_J$_Ey`tYAGym6GVjZw2=sZ{S)lKCauQ7aM
zJ|Z5n$>3mqE32?gh=%;bx@*jnj+6^XYR)}!$+P^1(xgV_*dq@bdHEO;n|L;r1WL}C
zuyBU2)IF7{A<{QYrus><xF&@t|J%~q>37qFf!(k0pkshOlWKCW!6k{u0WtxCAwjZY
zUoUs2GA_O0uupT!!e}PHFViM7Pc3>_o#M)=IyH!k?P-E{D(8%Z*ruINBV(_Hs|A-7
zUYfz=88OX4fSvWCTEpeU?{`bi&b`j18GRysRuGqlVWWScuB6x`_R@#9^Sb~0O^H?)
zwq3P!N!q3<@s5WBXRVApJ9D`}$@`6tT`Lwgd2yvL(iE4i6mzwhATsOFsY&y_JGV~q
z*6=+N=(W9RiO{-jk7fyNFmTNXTCIK4Nx03Sb;k_9ZeD|oP}jiMi?;?Iou%0=cj!gp
zR+ri@6C)lUn!0wUuxwSpE`P;ug0b%&eO(g!UiQno>f9+8+Y)}teG%VZ))c<}fVrk3
zW6D%kg98UA?p!vZe8FAMV|D+1TjcB94{n>5^Q6coy7sl!(y(no6B^aiMRnWfp1St+
zwAWnq-90g<MMF#Go@T#kBUp4Kb=C9oSzD93ZMm*q@{9cB`{&9L4}OREOYFL~43~d#
zsx-RX_&<wtp_9<Is)@Ss&3{XEw_Qp0)=rr)DX}dk$0=lU{%eu94l3tt89G(0TFrxW
z>~1f*ZDKb2k?;9q8J8b8g?WpHIv%Q8u|uM$XyJ~6iA9N$Q#ZS=o8Fl#_iXyyX1QlO
z7hiWjm9%tgX8}{rq%*Or{0pz2QuAlzdJ)p`bJ>&Zy*H<Yh{wDsyRdj7pTnPg0n3yy
zq2LKg+_7^f?Gu?-G41(=(!~v4?<=m&d=*oAKiJ~p8sVqM1Acy;D}T{ZNRvTi19OVV
znxG@!ssuM_?fmpr*YR`yw$D|sR8un#Ur@^mnw<VLMg4+MSAX9loy7u28+ne*dNjl7
z;PM^EOr4tqBWAQtU0L)0sJ6+>-W(25+sz$obTm^sBLspE6>Z%RbNln@xED@`_uL4G
zZ?No;-t=X<G+zo=aKPC^TFo=h<*0Rf?72Me*zyg{4sMK1UXzZ^NI7(?Bh_4d&3Q5Q
z0*zlAR7_JR_J~AX?6dOZ@7?yWr({~kv9~GeRtq;y4mz<}Yi{K$(-(<eS1+Vb`)9-2
z^Gw>Kep%20PTnUI)PF{`w6K|d?O7|g)q%fo!erU71fj`WC#a}b2lB^XVb0A+mKL|z
z(y4nRrA+ab1Jg@0R$*C}=CW+Aj({b?(knGwoi*?1wx%eHzs+#JJ89{JvdlAFn^elf
zV!C%6Q5M^fu#PA6l{e?|%(Dj-I6eQ*)4umi%~j1%l5LLH&u5%d54G}oJTzSG?ZC9;
zgN>CHd&X%)Wyv_z)h9*HsQaC|yGvrJ^aMpO#km}EA5YBGe4?wu89gJ}GC6f#@WpJA
z#X_Q*9!q7;%~)<3mmTVyoGSHgqf5lTC{9o1DKh_D(<f<ePSK4p^xV`aH09rh<34{A
zFIU~TvP?XYRo%n!0JE94x>M;i7G;l+#e83*gtuJdG>({>tNSiWwmQ+>K5SY}T&eEw
zPm2z`D-y3bczU&!$mxB@q&6^KSQ8)cQAvDB!iM7>3le=ACCWXLrGD+vO!Rc%UEpJJ
zRqL2*aqprlc}G62*zV&MEOf)Ac#`tw^D6At+JFD|q~tH#H2Yw2YNqSs)Zn%-@2vhH
zJLl~5OU#PjA7o|)HQ&k#J#BJTXj9O7)f1ORHic_$QOudr6I*W7=~<RzwBzdwSNoca
z{d1JQ?>P1Kc+vFi%jbCta&u?v-44i%ooB(iyXbRg_Pb+iMHU+6HE~}I*v7D9pD63}
z!rh!b4J;~4_0~Bb-O_tYseR$~JJ*Y(4zJ>=bzXXb@kcS!yJye1b~zV@nTDAr_A*rj
zd74EORb37|S2RaxPN#)hfZxOXWgVY(&%UBo;r?jVP602w)-8Qi-aa29-$hId==Ab;
zsJ`2xzA3Ov^XL&Pzk3S)!V^#ZnNd3-a6Ze42V56R(k<VuboKdf-+6iVvyz<_w(q9=
zHDq`eRmt#I!ohZnjbL!+ss>gbK}n&CD@)uAqSE6wGxsk_Tc)?QDK?}lcy^GeWwl33
zq2P6nzpo6%ZoBFJ+tbUvQ_A^fLa6SiUwsunik$b?tZc7+ndo|lLA8FZz|?OxE5C-F
zaeKGNYXYaVN0y%1M8UR#*3&X2wOrnh!i`hcFWL9OVNQoYNNt<y{9kXD-QX~^{HNM?
z<L1|r8O6))TX0&&+I3Xhh%9IeS#@o-sqg$h$x>#G6YbvQ1Yf?<nJe?W-R|T~mEzv)
z;NsuRcc1Cy6`!u1p!0pPIQzu!4mE}oO771V74nT{6g7^0J#W&U8>y*VIL`f#J?5jB
zA9V6*N7&3oqK-DMJ4`0_<s~W0{(a&q`9ADXi_-n&FZj&<sVYAE<D-$C^wO)N^U|sR
z-B-{1c2%%_Q)^OJvT~@O$y2B3ry2A8^)WYhf6hqZqu*vFmYm(Gqx4Qdz3#a0J=JSw
z)_vbxPW_0OGuzWfORitm`6$<}j<(*bg>&6RA4x^eW?eEdQ1;N3bB~{d@;dxroZuAs
zkzZrG_4~NAlhc02&41c5lP6j1593*zka;ry&O{4n^$5jZ$zC|Kk#}l#h`Ytk{bgrb
z)LA4QrXH#+J!z2Amgc?d#H!Fe9b4DfwQS+Ls3mda8h2{)f4MKqg1L+?)GAHz>=kLB
zB)a{7L|yr;>mkQW4^MNSdGCw&LQ}Tb@MM)^J)0g(u9LQvj`Th?FD>c6NSZRw>Zdg+
z*IzQmThDqszsYm<mAm)Me&5#jSY>(aMRwAMxYy?&G%d7ey6CxX>gGM)H-EW5b?<D3
zU-SRpyuJ5lv+f3F$sVSMosycYcf?#2v{Ypt<%(-_da>=BbXR3gh>DDs>CtU3czb4q
zcb+-1b;kAEDKde3!XKUBSb9Lt`^xk>PO;FnbEbO;o7$dgIC-b>qRg3}31=mzI3~&l
zU*O4q>$C1IXW<Rui=NSKo(;1Pa&d>TzH5jJ-FrW1$}t9mFu_Gfzc}!Jc_S^Iz#r8p
zIq!tr%~t=G8&iXLB>#VYD&Z8-!}6eAW)Yk0vh%i<7dOpyz3<y1|7)QWLsH}oXWbS_
z&5oIjtfI4Z<+gBKEi+AG-E@;VFllzJ_Z-iA`B!eT{FM`5A{Db>)`zb%#s9T+Y`pN|
zBj2LddCx4Pq(!25pB`Om8nwh!>a4Gel%3d-4matpS%((5CD%+>_dEK2X0U_c$t`!>
zj>k^OpK)+{h>O^&coV-i=PiChf#)MHUOc<yVtT>-7|Z)+#{^5~xG#HgbaKH>hAH>w
z&zX_q$<qAM-P(#xF?Pmf6W;=nBWpM&)$H*S;dxNZ+afRbfPrO+6VDSzmJ~*%1&&Gy
zwl6pz_BOF_&0`4@*YbVvNcWzI;++2inF|)gJYnA?q;=6(Pjim0;T@gM=|?^1=(7eN
zbDcW(q2v|S&WUSy>qJkvthjYDwsUHP=jpo-P9L20fk9Q^-kmeDH_lFcv~j0gxks3)
zYkS;>PGPTOaZB4od>DfMoQO(H(k$&rU|^jk^YnIZqRzux;b~km3S~=trfl?`k(_(i
z?VzIL6-7BS7x|eHp{$F7J}gwY)h2(<+5X8>hHcIJ6qS^O?um)adlb6BH7Y^C)b)_(
z!AH5*XRe(2I8;e>UX)svw?Ud~Sj{7!l4n|zmO81(hFx`g-W!oC;Xhg9NJMK`%$d_+
zE92rs*v>xbj$gs5xqv6{+`XuK^J-EazGLj#lEeL<_2_~~1HXwJ3uj2Et~oOGeC&NM
z$K_{^@MV;`rN&N6iM?|A&zY=MqEdphb5)MJUpeFOB=zEk<L+COw;UB~_~L%$$hGQq
zV%{#i|2)tBy!e!%=ir5uNQPOg^+F1jYzwA-WwPDiuG`a8DY8I+L-Rh)dlEJeG#yuz
z1};b{d*Y@vr?u#bcIbk8HA~vG=9KrU7+gKUY1geU$;U3h+7;)`-PiZ@hB)uL>pNWd
zPG3!kF7eEgy3z1b<3+~FzBaX`GKtax?1gh=FJ6^j;3Jqe>+%DG%-X+8x=XLDux*iW
zWSO17=KL_wIaf9~XJuu`6$!2O6%GDUY`m-2EEL-(J8RL=D|avdXYgoL`?E0j&r^H8
z15K9>JLW{%ZeX(JVX$4}B+2KZE08hWbJ9wW5T$LsOosOs<sD46S|(t)Y<AQV5eKbD
zx^60dX)0M-vmzhO)zl2mQSudV4bN#+UnI3{hB8lQT<FGIt3I`@nzx8kQ7rz>(T=NG
zg$?ZMo(1qZ&MG=~??$KF`P@Sy500<+nUuxi5US@oVT*@vR6sxt%e5!mI@g{p5mfX}
zNRI63XX$ww|AEJ5?ZwE-X+o1;e>*v`%t3yF+3|HV1+Bz*%a7gqb?5lcDd*>?i0svC
zwasm;i;Hx;c6;y5+dEaCOpTP(?qJgWGk>b$8-|`k5(2O0Z#ty4#%cQ6!&(VJ{}1Oi
zH9MuW%)8VUwqQYqN~lrWi6o6fuHJ2K-tJCm?90}88r?a)jzceY&N0z*T!(J<i*Rmx
zl4BIR&?(2~Sq4w9-{kO~7=flUy-cYVZ(o=xuGRhbV#XFH%^Ziek5fag^_9u8oQ;d|
z)^eLD@Yp|(|J=sTD!=4R5f{namzNhEy*}~bs)oDQPtGuP61$nhyI^7L%o%MW8NK&E
zzIFWabgtYHjUFalnH6azH#L4NJy<pApx6oxnd2H;L^Nfz68YvOo4tB!m-ft$F*GRa
zd0t*<lK9f7gd`=o)=BS{1}zIxy&L)E#P*zJD~gV;Op=RKa?2EodVFWy+rAm+FL@t3
zb2@uo-2bgw@#`i{+xMXRquBfXHzw}85}f%VRM4|Q#7F5E$Nf!8%Izl~WKBB$D%bNh
zYvRnDgAqozDv$Vljh$N>rv9x~?36?Q*TgGlCMciVaK_Ro`ln>?>J4s=YuakIiMgCv
zUjM=UpB?Y7tLj@Ws@vb2?eycmL&c;$oLag%(fS(>&tG@QY6JVBKBmTXhb8X3)UwF2
z&oQw5a!8}&pq0xTtqsjA#Ra+>4ojHm9P>)xS$s)FL+wQ08<qk+zSjkO@frNn*RbV<
zA3S@>qif0bB??uthLy7C9-f(=G0oGg@JzSR+U`Ru7M^(V^n&*cfv3;kC^YKkHdJoB
zRV2t3r2Ag8L8QT>FGlA7WIe+orMB=@Yg1)dBozA<OqWY^U#nQ~^sB+Gp5Dhr1--Qw
z*gs!Qoh4<o=8Tufl|!*|EV>(*9{$tRjac)TujR<6mGhO>&5xTIpT(5k;S_(VNligV
zyJXX}i_Y=an$)f|`F5UqBK0geOX^wDyJvolLRX5CYu4!&zk6o(>X}@Nzr!kav#e8A
ztCH5;JX`fDQsA<`#oU$0@**-=KQM@81c}ONw`e|Fx4P}^i?^@JEs}M=wXSP3w$OjR
zB&2V}w$S7ip93@l9<+3SHR|)-)@_!3o?p~GVap=pFrH14XA3K`=hkE|&rSa_QEX?N
zh<DE7+?wT)952&c+b?=t3R2<S_f6vemYdn14kXI0n`Z0DIR96Ab%zt%^+RmynWw+J
zt;VTW7%9pA{ji458he?%xz|?OZ!xewu!Z}*CfDIFQWYiMFK2V_{vejo#1o(L{9VD9
zM+Vs{rF@fhPX783%xrkbx7B@?f~Q-I{>CL|<B|*|PkKAeog%RI<Uy?qw-ik8K6rlT
zL2yj4_-50DEn6=ab(-e;-{Rrx%6jTs@-}vT@AWF(NjC%|nGP_pd`sXiOAKdY$cU~m
znH`hwwXW97wQ=V4&?%1nvf3APSDSE{%;P>e!*AP`LrmI7v^t#Zzc_1cK2-Qu(|XGn
ztqaUP+IhAe-<aj!FzDn|XRuA(Xi%LX=@%>M$C~z^Igd#*<KR=NWlyX0ma=u}s&BV?
zZKU_U=-IhV`@CEY#W$;{&3l!!J@<>p)5PszAs63piM{*u=H0vr9TU}}b3{LA$tSG&
z9`=>{`!2b&B5oG{yB96=E`R)8RY|?*Q(i>H`<m%f>wiy;Jow!~;UI&|k4EVqjv7B4
zLzLHlluD}&D~Vcezxw^D1&Ke;@qB9(D`i^w$M5)Gwyf$7i%T3IHj5V2=pD)Nb<F#F
zNOo^nO^9#xuFULLs?!gCvH!(<=#rDYfS&e+=6!R^?AJKWb<@#)AfkTIBr{pE=;oEe
zulIMp%w#rOqn2?n*=>bL#?K)4_nnXBgN(QGIe(oIt)V^hC!3bcf5Foyy02Uk+$=7=
zz0ljtZ;?l5t>paqYbTgJe9*Vel)osLpRYffSt2=|`^H7?SsI<;TE7<Fwpb|s%|XB{
z>H1v7pl@A<+ooSTE!Q8mjp@^cf-B2c^S^XSJh|mi^uJ?H{w#%Gn>9o#=X>cCUu`<N
zj%jCL<p0o{Yttpyz23pZ{_3dGBQc)uEq2TFu5S`i*&g&*UQgXz_jQqe_PXbO<uAA{
zKGl0=82xO5!sM@2rf*)&<Vc)&tm2W$>RY~#49${1FTAL=_R{GsRi7;`3kp2Un&u%j
z<4pB}#;yHRvi32vO6IWc-`+jth}*VXol6t5`7%Atzh`MiL?!UE2q;(tn^+{Xn*4WQ
z-ct1E^HtN-(>=<2<h}&YU@uwlD|myWgZnh6jvd~c!Xg!>?7jAImdT8pD}G!)<CQDO
z^6krg2R)6-OH2=bE!|^lIc2f*w2iZ;tu)#9HJgcl%|&~04Z|1Cd|NMbZ=IH1`EOU~
z!6lccN88Tm{&~&3l+9`@+qyT8YL|HOZ9IK!jaO7mm)7ICs;=SR*#a6$b2PX(8`Baz
zJk^)n;a#RySzdU0rYrxZBlqgpFP?ogKHqqX(7(t8w<3kQN8#?vJ+@sh>`%YIIxps%
zR9<7+Y;PtX$C>90*W5{6``xTKQ}X5Oxl<<AeG$>p7MY;-#?vKj-NwyrZ?qdXnzu3B
z-&AC6&R4(gf8g(b&$&LoT=s73VM%dC<*hHQEBenp-{^hb$}p;AUe1lvU)b{le`RqU
zeC7}^>ri=mO?=dqiPm?gGw$DhbPd;!mlG#<{p8MGCAW{~AA`l8Ci5SS%pB`#7nh_=
zIrBuTJ=mg@$wfMhDVjGWwBP_k3n#Bw&fyP<PMyM<VLcujeRp?my>(UULExfEEkbHq
zik1!sPco{gE)mK6w3KC{kouY_8pQ|B%n>d0nz(^=$$mzs_HUoB>pu)!=%s2jcZZ4Q
z#f6@;g<4<vzq-71->Z#Mrq5n4Gn{oSYof~0<^D5_L^U#RhdJHaA-u$AhLd=H?A{9Y
z8(*u%H{8_?HsjHVspw8T+WLQ|M&6mk1CLL{SQKom*;$x&no~Obi_OH~LwXB^+n$;1
zI^AHjN_AbF?Jl3?H<(kl^WE)KddaxUoA*Vp^bVIhd$rl#`~ER@eR6W9dHFw?s*;zN
zSB8JrSsJSS`u5K1G}DH&E<yMAPEs;Ws5$icm2lQ})=Sfl{16f?@z-lS{p#7*x!FD)
zCJU?jq!!F$oWL}};XD`51}m0@!mH$}BNAQLaBaWQ#G|?Gf&r)IqzMb0C)<exF5{ds
z<Ixe}r4EH%qM;@mT_wKGN^0O`f1K1MT|9@$UDmkUXVJMC-aVTYdVO^4RZhRiIoUGp
zN`AwYIXP$dY1vLTveXIA<~e=Usrl0BmRkz{-!3w{mi>5#p{mvAGbWwIk1Z^OCkgK~
zVP9ZurD3~!#m=X4Mb1hV|8-OsJ+2An^S%FL2}i?|NjIFV+eCyiF1?Rh_~p`jHjQ8<
zKg~;DHn$3K<ue5MZtd&!n4F!yk1<NtLHC!<<Nifc^1L+EwwHLlIQ-isNPFG#MP8?-
zcm2@}TanZr^zFuys>I{Ut7puec4KYF-m*xJY|d<M-)$19-M+i7GU+DVl?rU+f9ln}
zZpWRwtJi5hyQOV#fK6KAcnim54Z~v{Q+m2}6*lv1I-$iJbmE+lp07uz#Y-P^t>tqf
z9tEXswlL|LyJxNcExxO~+gsW`-+0tgH#x&{N~qb*Q`2Je{#OPXTF?FYw98~o>z)@5
zb2FaLn)|yi<l`Be&`aN9zUnUwJNaSeA{Lbbsa-0w=DnETDfTD-Q0SglM|U!<|CTi+
zYF*l`*SRMZyt_9ZyOG4Bo8#i`!tXf6iE~%L5^Fc{#5LBglG#~>%)I3?Js%EMJ`qe*
zUMxI`OS!(rW|P|I?n$Nw+S7k*?pS?!&!P((mYgfra*nP#^*Jp1eZVyHx4B!b!slsR
zv^Jmmcj2vYeUY<wtaJDJ+*9|OdBNoLiP{RMdiyJ-S0^5~{c$UJ+KP)^6C@ITX`Bc!
z-TiGtc*pNf87qz5&1P4(D7@v!;`rOX&;5nMm)*NV7Jazk@JsoBoaxDB6F;2)-?{Y9
z$NZNn*(XK4SuBKRKVS*ppwzjn<ItM8HA$iuqK-<u*f_DOV6DcNgrimw2m9hmTt$*U
zEC}(i5M2|JtrZg473$L=y6(Xl?bM4cVtYcJi>97d51i1zciCxO`QHHj_oq(BcX%jE
zz2uJPwCr)=VwP0rOGub<uqSO<#1gHE%e+Ih4(IATpVcM7)pO$Wj1xyBj!kUUu{@uA
zOX}tgxA~ckB8y%{UYyr#WgeCy@@S!l_NC@|rkU;i?>(ccA6;f!{<0;ke2$--{o$_A
zo<(wBSG1~SOcdblJEXluL%>??i-QkegWPI^3C*`66gUJ|h&}$<wcVs;nnnf3G}%iR
z)HYrI-{m{Oo7-!_?R!2g(|28Ubu}}J>E~O%xKn41rOT>`j~cF6=y|+!UzK?2$-ENF
zuDv>@@mYP-wsr0_aO^Buqd7P0%oer#7dYqU3Qe{OZTGuAl_}sk&phkEw%yi}{sAV&
zakhp|yJtQ%p5L|Wxn=)!BNmmED&fw13{p#5%~q_koiS;}QWd8CrD0w^)3heL&3>%e
zT#}wH;LEZ4%Zv#Tsx4uiQ?9OEHf;h+h`YKmXL|6CLtKG82b#Y~9Jb<M5!-lk%cKQs
zTv=T=cGcNTaEM&k#Wnrtm9&`~RtD-m3_Hc9>sQCRI{SjumRAqEmtJX-O~0)u(0F5A
zh3H59ADg@rg%+J@{qHKcZjNSziMg@lITIc8m_v(PHdUNacsR9a(MgGurwZmA`jytr
z=d5$Ot$5=BJN3#_flpGtUy=1cdrryD`;f$z;(3-q*HWKx@@v28kh`R$()VehN6j;4
z(<u$tQs4aWxBsQs_5O;pFxw}0Te;3Dw@;OcY+1YHM#Ahlt5r^9&ui>b@ib)H(vmT~
z=JV~YFGe$XCDs*~Gffg*dqqM;_j6&&gF~0*xtu-y@}%XQ+r8-}m*c+W^(oz~$+wuE
zW@RttpLRy`^5-0OjoVMl%eI}K`;l#pwd+pLO_@rp^IwVVt~xaNz;B*;4uYl(3O-L3
zh^4J))9-2F=uxVkcxc{_J;yhH+tU9(h)dD1lA~$jx499uFP%S@1ub3G-f~>*n@5Mc
z1=FOdXIxaZ9>z0;C#|0Q$3o>7w*uqjnAP!fU5@kq@z-}tymGfssoyHKJ$(DtuBr=%
zwoUre)u3h3wZKYr)p6TvVOMgT)-MZKm6!K<PWz)n>+S@#+f3n%eAybNtTwT;FYt^o
zquPd(A!)`TM|g~9KE8M9&9Uz5XXZ?H6LT}}o0yX1qWr;Tv74!w_?3n4{6vDL&N(mi
zwr~F0j>-eh6F-00XuD$a)_(0vUHnT!u57L9yS}ekr2bBcMrfMr^p_9&y{(FzyjCW#
zt}NcVQS*fHl9ay9s-JQezPKT`E#qDBcb>^{Oq*S$|8tq|D^g*t`Kn>m@mMcm`)Qe!
zZMS#(&04l+dCJZhYxVA1BE9C<vv+<L@S6L9r{ax}>BB~+3cton&tKPtCp7b~dNBR%
z7F$-XfSPsd9`VjU;XQvci-l53wUnr(o3ZOBk;z`xw$dK1VR5S#9PK%#xHV*T+)-^-
zrpTolbFZG8scfqBou%>5#~m(HOc_!-g|eAnwAvrI<rpUNJ3@TVg1`+<rSU&zMP^?6
zRd_Mx(9$f~*4(g%v#;Ast-7)4c0r=}?srAkHed6e>D975dE0@oQ!-KOCPbIJDlEJ4
z<VJKce>PA4wbh2b3qMx$ZoF}Vt=i_9!n}tQ_isO+@NUj2e=+M-W?%m=SG#Zfbwi@f
z5(e7~0q%C47jA_vKQF)T?u*aMuDUZaE(m2=6j0pxWwEyR#9cuwDI)i^pML)@8Ss3r
zbNc$iqhf14QrjF@9P+=vb-l7Ry0@%F`Go7{ySD?=lJ>3so>e9I=MjtVX&KwO7pC5~
ze7;gYJEDBWLs|a)YlCP0Veh)ha6A6M+^z!YuH*HOrfz&N&E0;^@q+?u4)RQ>v|(n{
z%V99eIV_fS*v$J#(7F(jBt^A=YMrM{;XSo(YmR!Q*4{c^qs4X9L^Y#inbM)d$1ReS
zTGl8vya}7ZbMn*kh6>eKAqMv10QORY*s2L^vlnTtT6fNmLwm|1j>h*Kuh+c3)bnQB
z|8tx#dAK&KD%Kp7TVZhAVo{ApBhL}nm?uh{RROVgj;dTs?YO76wr_=!!s5;a9lY~1
z5_ohkswwkH>RR6E7C!NS&vGG;M`Cx3Gk@q~{=e&#gtC=-ob4}ge2O@BxuaiDy3lHg
zLe2}0t7;dmRh&M@xrLgfikwUSBBEsL!egs+v@gg1x!1?7T1{W=n7=VSSemq6G&O%|
zSB^^1Bj>7AEddsXIfuUI<^Ndbd`shzc<ot-2XCgdI7zfl)SBcysV&#Zz*Far8`l;E
zt}P3kvo`)*^U;})({<Nb7q2JN|2%V9aejsn`|lgMR(HB)&da=Kq5MbV*u4bp`w`B2
zg_P#(TY3N8f5YS#oDX05``vo!`_J$phn51PpO46!11y(PBzO+k9iGROHGdBCe7$u`
z1x_(0rq*e#V)E`|+I~DkBkPHo(NUGY#X5oq^xhjcJ!A3pSA532=$UiYnYSm?oR;Pl
zFcfW2EBv5VXb_uaq;h0}ZUU2@M@s#o1x)i7z1Y<EV#Cvw(<VjDDR{Gk?@dRb(uT8Q
zuMUdcQsmxl5M#!odyFfhMu~^@h5AjWtaD3m*A%_=IHZ5?-7dRk!=5L)+-+RXver$@
zeE)_^pLJq!+++S&=43AIjV5k$s+yC1)C}~W32gSan4_{u>|?jpSwYvcDz`Zpw>5|q
zs2S~4{$h1;%bOQWlbg2&{4ZP@X2_V*FB-4?JgGo+&auo*A2WZkTe++fT>nUZTDL>Z
zLghB!soOT>nz=|NXXa%E=B+Bsi`x7%tXRe3U1ruN)BLqRV-87IYyLWv>pqF2aGk&5
z>g3-t?J~2E%sP^<^d(ttvGd+zn~OEv?|%uBJL>=M*5$u<{Qd51{`+k5Ux|1AOg#a=
z^#2JhwdLgd&+(=#Z~6aO?G{{b%4Vr63T{z2s>0atU_o3KV|>?RDFekV3P-fEj%;FA
zR`hZ<7ZUNjP~+EHBc=586njljkJXKaBL?0-OWr8ITD~@|KuJV_G2#7$b3r<+8XD(6
zsWs><vr}dYIR1LcrIl-%I66OY&Y!@w_J6>n4ME&n17gxLVrz<cE(*DqKT_@E@t(U_
zw`h^~W|8n8IUNkiozpzt{aVU<dCCQzUYX@5WL|IJ`@KQ`Yk=8~j-D>(R32|Lv$M%P
zi+dEbOBqa!*|(;tOi)qXC=|)Zx;CZHua&9&n6RagNwSIQPv(X>YN`vKnZ8{dR9WOZ
ziQ8)4TYHT|5{rYhc4?UhpVU5=#=YQU+bThejfEEL3(toIv)N8nk2VZ14-HojajN{f
zH7d|*|Jl4#xiW=IJno%cKdCeRQX#{fl{`}FUe_+l8meD;B<S_zbkUa1=<CKl0)789
zn0hy;2%c5&I~O|dL}%%#mn9;tzD!sB?mW2vPuub0f0uwqp@O18hmO4PlerLY>&*1%
z#3C~zWp1-YR+FmjHfb0esbr?rs4WdPVGZ^(ungFAL}x=t)<IYM!>s8dR=qiHfnAS<
zI434e46ECsWVZ0dl7e%yCY%#m#I<>?@-4<MCmN#jR>st=@G8IN_5Hbf`BKX}O(y~Z
z)pnYQ{ZCInUb1$UsoVWc9``QAKRT5#;Yq@~B4fEUt9uj8_=JnylCzE5vc<hE(;oX7
zp8fdXYpK#}W4qpf+h&#<oXz;RTIvaIbGbOjWqrR#fnvD!SGEKmwgX>F^h}H2=C3}x
zV~gMMJ<qfXmx-DcPObSoHLlY_eR^I~ao(e4;nRdI&BKeQPd07*|1~_;OR1)`WOj3j
zt9gp66Jy8LsqdZ^YzU5g>l}4*qD=5((*wfgcgyx3`YflJ_V-S)S3$CA%Xi!J(=5)E
z{`>LS_HB3NYhwpi9s5rAa-}4P<<lMh>Q{Y#U-_1Qp-E1a5LbcXQ)9zxOpKW|p*=O(
zoi(SYD=N9xXt=T*S9iI4z50b&ozI%!C`VUzg+lfP%5GgeS5&PMCa5!*Sy@d}YFgBK
zEypu;n}WybwCzP+PZxQq--xZ-pfZU$qc*J|@|8%SaQKhKc1e$pf9rYsZ;=o4S)ZE)
zyt6s@SeN}4dt3O5YoA~9dauL$%5U{wf5}$v^*{e2K*+GSWO|3DH<OM@p5}kS!09tS
zOKk1)Ixnp2n$X+U^<Fu}^|FYyXm_b@w{5bM<PoifPZi8QN-A!T(GZu`pC0Dk(r_z6
z?Ea7D*N?-iMN68umN=VVar<$bX?el9w;dC%M!q+)PM-3<Po>D{u=U=}QR=UYZIYvl
zuSZvx+sgCqUFNd4#l>qvIl~!+lwW~yCp|09D%k$b^?P=83jdliznKc^Ia6iN#m3K^
z`tpg=ga_TS6BySo@6s2W;CJRTM;Np1rs+G6FOs{q$nMlpg=>~NV#=X8nI>fu)RY=#
zuTib&>wbKrVb+;h@e`Cj95;$Cb$+x+Tk}TjAp@Q1JU&}RQWI4(mhMambW-y-6Z1Ep
z{rvyWZtv|%`z)UKyb15>So`MEe03(@s|H37KD=9YZNbZ|53g3emwRh-bMnHkkM()q
z8|>^zR(PKx7hE}kxl}6nQ`e2fbDg<7678nk_#AVrZe=oysp=P>?Ea9V+Tit}eoCTS
zcg&i3%%W9z`Spr0(@@K5E6emk^#WfNwUEBOH|D7qJ?{B6ulVisb*ua1r<Z-ro3Nd^
zY`25#{`Y&<Y+YFVc<;8VKN3+T_q?+YTSezrc-=Z*p*T79$>oLYg$yTN@B1-1cIkAx
z+oi!NzOlvabMN?9b~<M*d%k|XvT}>0bNSAi$7TH{BDLRFEaBX(aoRoT2>T)*1FMjd
zO}yV-BR9z2Qs5H#f8fp;Un{d6th^Rt3!TH)nRo|R_u9?24!UR+BW)_G@Z3dup79S>
z--@;0md-D)Hms`Vy<4zgZimsf2Sy#gjh+M;Klozvb;8c4Chd$LcOKLiIQ=a}M>qGg
zNUnX<?wyYVA9XB|_RRZOo_KI}-;%|FSC!MxJys0)l@x#Ls=$xt^%e81XIQ@Y-CUaF
zbo*vs{niz(+gJRbt<ET8l@YZsareGr-+oqi_ZpSElT0Tpzh!yPSN5gvteo;9rRUc7
zH%{95c;dm%`@62n=|*mJ+Ozt&$CU76ydBdkTw@NOoaN=Se2zh%!dt6je_yOS`EpAB
z9EH+9iZUDK<u^@B`Rp{kvQ|@Tk==hIr48weeE4QKPU;f8ZLG1at8;aJ{1-(gkGZ+C
zyO-}25sWyo<;Usy%v)@kGtTC=>`p#2S$SgGt(2)!;ZLirIp!?YnX22nV*d8Y6Q1*B
zv0G(5UFGr5HIb>doPDkdhv0|2>qR-{UR--~r02$OmcC?ZwfJgP@0D$fRc!-G%{dEp
zZgD9|^v~VnT)Ro5;LADX{FmFOM>|dk(YCw6bv@zziidl8e@TbGlqhB1bM5XOOPAXs
z+jm=s99jAQjn!q|vYE>a=Wr%YJF;ryE?Lc_!?nwzrytw9<nP+QE9BmHdv$F8TcF(i
zqWF^e&UO5%raSH?zxnuQ&flZ2g)1VN&Fik;+IV;0q5rn`)RhV<le$z(r+=26X&th$
zSdyjNE#!;X#*OMhd22T9K0ZU(<?EmDxslP=>iJTe*H}HceSI<WwXIj`U1scay`A2?
z>+E)kYv<>j>-Ko3rPLZ6v7qsGXM#n7#ep4yZVRQ)z8A{&J9>Gixb3ckmv`NN>3`Tm
z(0FgI;oawV%bc`>lpctk@(g-<$U1O##FhDRd-Mx!?LJ<fn!j;pd!5He%Nu*Hd*6P&
zTkg&B!~5jz_NgZ^rG0w6-R^B~*p;yDc|BhXEFSm&uXxAu)7O?+*7EQw{+WlA*RECk
znR>5qnM%{X4~CPrc79abdB@c9sBqKA$o9#h%AcZZrkwq?(dy0z!}&+A{!dxUJAcyB
z=f+3Gou^BR*oOphd^_RUmsoF^ICIZ*!=rigolo(HJycJ!yTtT({Y<4pbJ&`7)XLAP
zpSah2y6!vwjcu)G+&9lTbN%1r=QH2z{CD7S^fKSDmei*$7eCCOIZf%;&V+^|@xSgQ
z$lS@2JZ3B=HEU~C`FFYO{bCyrh~?D(yZv{Cx!JE>Q`2)b?^?C&`=?y{|Gm5d8;^$S
zh6M+knFXa>3It9tv<ax@WL?c-(C%i{SGqDo@Ud&3f^}Yq$0n7NW~?_ST-xy|LAqCz
zd)uba(+k!zns4U_37D|lalWmu9E;YL6&DwK^fHASHDp{`9<bO;*2?kJ6iw-6IWvtK
zR)_sxAG54L(qNm;&CS7{eOFgrUURc4^#WhV>}?9_ySd(TU9oOh@3^1+<QG|mh=iB}
zh1R?;n9LJnno`d=l)Z`g6mz<V)hKLF<wu5dtj3R1c`Dw=oGN#{Y@0jd)02~H)wRQB
zJXs)OWXS&H-rC;@NADX|bF<95{ovgUky3T}IDw}HVItl3sh6df7T<l#b4_Ml@F|A(
zPO1_qRt_}*%|U88Hx-Vqbowq-BlI=1Zux;<zP^vv&z4sV{QS<8bxO4t&yRTytdovR
zZv5uxdXK@&L!q@r*e5VCP+DWkLI=sT69*aO_&zLjj43$i*r_3AGA$%e@yLy?qEg1i
z4lGxHL^`oeoHXHouZ`N0<bIZm5djm}1y9BYy78@CGQo+d^2uZ_)qnsdL6wE9n&zD<
zBJP%2N0v^D=MtIZnXGk1NLzR7rAx(Uc-3Y7SYMhr<nnSIO{izpbUY?jJxTC^|MQ2H
z&li-rNop+kW?8DfD8TI3lc3Ph9jzkK6HkanPH$T}txKm!>q1Y)RK@g|){W0r1TK}D
zlI6IRtL%DK`%EeAHO;DJGm|Dh%xYWLb5<&9?f!kgva@HIe$(8rmec!<;}$J#?U0}?
zm(F?Tg=J4FxzsxIPTslDl0fY%MZa@*Jmg!jxKA}ueZsD@oazC#OsSR!%I-S;ke$c+
zc6q#`aDic?kn{vY#wPY(;T!%322J~(KF|GVdA>jQY?q9=jA1nem0T~o4d?N*od~HC
zYuodXp{Ob1!U>IPtt3bJz8yx9GFw!|d$ce0tlBtx(=|`Y@~MYDJJ@r7S-jI*$X8%j
zDo?LopO5g%l8gScMM^IRgx^)U%%*x$<?8ZjEk_uF>gFDhh_7pzI74n}?6#TFN}|g=
zHDZ?td5bS(nKRwwCX2dckk+axbA?qH6mB%?b%h-9kIebd)L!<sVllVh0*BD>+|-@n
zv7%F6M(Q?2iHpu@{};A?`O&Sn9xq#JcO$TTXIJ=|WfPyx+_3WQj~kgg?rBt~@8-W&
z{dVi*BjO+CH7=9r+49UUAbs1Xy5vA1-RG(Q^N;NOeS6y#=__l$o?<=ztT1MuCBtsl
zK=wPOm$K9YtheP%GN=f&wUBEO$~ov-{Zp%A-Z8e1_kYw%Hed69ve<&*-`Rwp%(H9G
zDBCkw>ID7@YN$~2FktZ$ND<w2;p7A(qvn!~RAr-!%ah!u33Z-pag~&d`<G<pHzhIM
z`t!|m&mK8Rs`cvmy_$HbX3JtZD^A7W*-_WJ%#O(X`q1IVvS?CHi&d+Gi$drjU)_6c
zW?KS3=CEr{7F+JFru&mkIqTQ8S*s#s!fq&8vrZJh|EBf8VMQ0FV1YBttf4!aeR<+b
zbCegqt8jOFyRp;LP~h<^mW|6oT5YE-zOq!y)u=J$)kLHJn<GQK!i;A3c5ycu%AWI6
z7m4Cb4GqY=s^-2^=c<WX`RS<Mj4(@`^CGXOuCw9L_72uIzVOcEkkitY8H?qEV%GZ}
zFzItx_jp~xwnxzsF3<8$Z9Svk$UQSv`M%?gqk8vv+H2peK3?{$_4pqjjYer>E}8Ex
zM&EdrCj4s-V*HzNU<*%1frzm`Q;m^-SL~bmw`clUI4kCH{<u6p_`v}-zmSD%G7O5P
zZYi{jd|Jpe^@TIPZX!pz*^F6lQ(X^Bb4z}?<g{x?LAveZNXeQo=ZTX<G8F#?UTRnt
zq^P^r+x_*!YfTwjmCuD}Y_|09EACp-^kmAiXz546^CY%4F*rC~+_>oftxI7?b*xHN
z+;=L4bRU|yd+HLUPnV(-**x9<O7>`So#^uWclBzX+PirHnhO30y3Q|B=(BISvOGr8
zJ>1#PkLll&hKL;jOnOreu20Qymb??d7WlGMJL#|2QiJU~H+7_@>aOCBJ}2a9lpr*D
z>1myf8Jc_cREQ~V;aM9H@?&>|k-y@iz`OA&FLyVy^lv@-XpMhi`K;>~!W4D}&kH=J
zu=kCkd(E*YXNx+TWgh=As=lJl_bte1E@O;)?1e^Qr<{+MCY{jN6zZ73qFi#f;2Zy(
zublb&c4^18Rc;S>X8XMI>S{xFtEIbrXC}l*FKO8E%cbc5oCBSHSFQ_9Ik=xeubK1T
zf9(V&_8lzDClXHhT{RNOQ%sB5F#qqJq(dUI*;^O9?e>VB5jpYFhQ;#tBsKF^1kL}M
z({AXy@nV&Q$t<alM?HEBg;R5aR~@W4)?a*rLFvGR_c@Ey4&<(KT(RTq+nkLn-Tl(s
zw%t5xAhub-N9?ks=OHf5(lvn}CpJ8|*L7{u8sq-dx5`y#SA`kydldaG<eDYi6aJTN
z%K8RfW=SsZ8&<+MSj+Vc=J^^l*nXe6o-tHssnp7y+cH>#=U3%KsAx`KZt!bUTja%f
z-{;}26-Co6n4i3EYJC}`#Jz1<uBvvT<*Nv(C0AD3s_KR3YR7L)GfSHh?78GX+lz?J
z)dzSs6}JDG@;|+)vwF`p4%3;t_1;^GtG%h7?vNGKy);&`?!v!!Q+R&(wwVT8&a?5b
zG7o$El`UZJgJt^m(o^np6+XD5_GoqE_1XtazsnE2*W;Mq|0$f6zehzj<b|V~nOOU}
zb{+nC1st9`rd^o2L0`hubEU`8#EE@BBG)cF-SOV2aLT4Dj|}H>cZF94-(m7KG_RZI
zCwl0z(%lp3j=G#D{QX?k?TpD*d1?6Yg~@W=gqOLBHy7R9wPV{{?Z^{3!bN$Ho(VnC
zQaNS5(^ET3Sx#@ixwh<==tX;SjvH2s-ie?0tL9_m^>x<^i^}gd%>Ma0+~RWO239j?
z$q#Rugbv((%ha};)9XPK<C*`p2a4DKW)R5vZuh<NE%S=cF&h?%YASrXFx}f`napj=
zi=L^QdiUq3aW0NbOSIP13VF*FoBb**YoEr-wx=uWS!bR5w(iRo+qZt%+EX}uSbFbG
zmfM!HDCQ|k<+tNMm#!_G8)E*l>c*9Ye1@fq1*2~`$8Gi4w(mi@$kz#pT(_r{Ri3yU
z_e){bUcsOUYtv;6pXz2$h@Y=5%Wm1o`sDka-=XEZnR)Wd*WWIjWpa0p_ZHS_eG4s3
z?!eP>I*S^P^BhWTRAM@j;juB$Cb?lwLc<&*v7Utua^D(w)lFCpldMbhY($F8HWm48
zDhi+MqOTrodQ4gQYopT_70zM-PX*V|tttNxC#J-F2~nL`Y%n7wH7O^F#rL3Ks@TNP
zf<-0w9ObgF$$u{s$S||K$*!t?!cT2t%T2K~w!oHbvDUW936l?}B`pt636GfER#vvG
zta@A7f7kNDbf(yX^0MRQ#pVpM8_IK!r^o+j%V<w;y&B(ejPVz1nXmu@*MfFN1;xom
ztW%6KR4<E03yO75&1gGTq4n5OEzpx&B6EU;*Q_aCOG+x|g^RB!>s-_xHOC{%|Dn#>
zWm!9w{Pww2U%SM%Bi-kkQOuzhvt7#s4m3933$i=%OfTny;H6Enw^&lwB*>llZg8MO
zjwd8Oe~O%{U@gbR+}myP*S6PQVDg`ISU4~u@5S`I|8J(JzjsS`yS?|*GJ(%!0{1fn
zYQF2#r$+uxRTK_V;M~y1DZubnf`NgdpFyDC;X$8BlIZvLfL|4w+*kY6igo@y%NICW
zZ(!_nJjgitAg9KbN*Px1s*MG_H>KEq6nGj}3T+R3_Q_a--NdBHS-Uidg|SHgOQA%0
zk+S5Z&8?H<+=VnWojDISsy>|LDO9C)z0g#$nBTCtMVR&C(h$3JGy9u@4ktsrMRSZ_
zxO!yfWF(vWy9wQ9tS-33b?;(HprT`lr$}g_La;?$>e7~qC#9vXtwmE4k|J9xP0M61
zgikh0kKt%*Ia-!9qdeK7EnP#P@_KpAx86$5wu*1lqCDFF%L)YiV<{6l&|ZC`kCTC+
zbLNZ>J7#qKoY7~=P%F_fxu_z?v7&qG%q~@>u4fgKzINmpW)=r#E|RR5y_UILDN1sh
z*s7D2YaUDXJha&uDYWUS4co>lZOtN;g^c@BLf3NS>|^adSZIEGa@W<xQ_gJk*&2~^
z#?tOwsGxUn&9?=1+b(eZWb4^~!tdrygTTNzA>q7VTNFxEdvm9Szql5ED<a|T_PUod
zr$4yS%VgN+nmDWITln(`(VI&dJSQ;xmSp&++5cIhpGj%~OVk2pgMRi^$qWo;o(28i
zrd!T9?5OtH(dwb%A9FEfp#tWN#9p`A%@ZqEZ%$m@*r0IC^N3TUq^8M#?bbysj)l%f
zK?b{ux)`1HE0qm6lf#oI>pd%Szo;B=DOoT;?&76rhv!{ZC$g6-g&K6|?JR8eFB5d{
zDhc@3%%s>FS`~ksqwaUeQeTnQj-;g_76v~|%hJNj($mxHUQTCjOaFItSzdWtZbw3%
zdwG6Wd4h#_ZRPU+p<=2EGh`atKObrD&|J{NIkW%fjJB!;OodH85kiwKH7Bg{xSwd*
z7m^X5*paj_qrWs!q*Y{^i&(vA=9Fcg-0j9ocj-*C5MAmfv|1}`u0&R8qPNO-oegT)
zTpL;622|%7#T?z*wN|70@RQlckEr>I$#}h*b2d|E=PKm~O|rXA<P;cnsBE13|3#zw
z)o%)?e$-xiKKE1MQl_H#$<DpkPUOAY-ur50?}tc%Pdm$BW(ib@SM&zXe<^1C<EGr_
zEAu%8SUyNFI2iQ*ky`NAs-KaYfq8YZ`i=!0jT<ayZs5|I$vaE#$BzYqyZV2Ksdr}>
z37Sn{E}t-e(gd|76Xd)l6i#W)ye2VO-HRzuVoAe9{`NwtpPLj#H|uF{Hqn}_q`p{l
z<!sZaCX3%Ds@E6mE_dNlG~1+RVISsWF@1{5$`HGs*^Zf0d|wy4PR~B^Rqb}5Or+6N
z4#Sq1sHth!{2$!f8r2n_GBcdfG$8HNv<#{C9PR13+tW+Jrx#UEX9_GY^A<1fPS09C
zEj6+Izf428kjRSmmwj!oX0%1`_?V%wVzNQM*UY9Jj?<YJc9gE{7peTiSUHV#)e_Cj
z<(8c*f9+h)RkdNI^}1857EH_9;aRmJQ+h|B_@m<L)2SuPG-9p=$6P<GdMPC5%9mYR
z4dw<+%ehpgFE=?Zjx+w-7U4%T_b>(J{W4v5{YPt=)ZV9E)32<a&b4tJhhTlmhWM}P
zima1Zes#|HXSv{s_XdXO1uUypFfH8h>h``jvo~<9PUa01Vf?URqL2|6+rnSk3+0#8
zD_p4;aII&`C<rv%bkn<1^;UtPMB$o?n@mz91aAbISRJ%CUTl53Na{zU=JiH}2p1E!
z5dTRmK?TLKhTX1RuK)KOlg>5{J!<Icwy`PxNte;bAW^}k?2R>6j4gi-=_hJWvy*AD
zJ+qa|s3rYpeHvFp_UdH`JC+r)Z!eTuUOm07SZck`8L5_3iP#I_FU)6L)Y#F|y&|D!
zN5`st>KesqIU#;6$6iMZFdF1`zdFXDEYQo{QSV`NuP{R-dSy#lM)$S~@otg6ADRof
zGN-6doEy@~mbl1$(Fy;Ht7iW=u|eu&pNy#L^{kECb}=ck<}ys(BULRrarJ&vhMR@i
z3*SgzyK?eC=qV?|9JYme`@aeqGV1TH5Z<a_m%UWr!ZC;F-R@5;8K2(V^T&1F+viMa
z3CnNJ=q<as?%s+sJZtu<2Pv^l?fd`mb>A<qeGE|x7^~0zowXoaM<D&p*{nPJ+&=ER
zyGKC9@f_E}cHt&RiIA0lew-847EnE$FVVeGT+7MevY5t}3F2E%sI4v(Ub%@$*F-{7
zyJ}*T%r)(~7g*(8#kp(6jgL%J**Vc{ca<5pi+bfGEp}A{^IV?55S3<Cb)9DCpIf|B
z%{E(1@%fz{yj0h<e9plVfq<E+vlBd0t1l;YUAFq$lB&%Z`EYyc@@cu5>6NdJ6jzoP
zN0ocbZL3Y+&OUK_BYQ>agrhMR7@r?I`t|A2=IR|CRXaL8k9AcG7^R<k5OdAa@)~>N
zxs+NJ^&7Pf935{Qk3T4!`%Y%(l<b{7FIM&)XZ>$9Y10hX`BTd?7if#F-5IqxQ)or{
zuC2ziYYyJnzo=@9>J7=wtS^goJwEO_(V2a$%zUG^jN>sb4Z)hN8n<3Q(rt1STybOf
zt%*5rr%ZhrC?6xbZ2z1(*$GEqZ=3hB^31Eqy{CHwuFXENJTp>I?Cj6Z_3=I{x;)SJ
zMD{aP@4KUOj!XJ3*TQSe8_setuq5m5S6O&>{%i(8rj6gL^Lv)|`%kP-f4E;ztU*1<
ziTOt-SNj1a?L{BnCW=Yk;N?u>krm(UX~o|Wt<SR9VbR6you;N&FNQu|Y%@#DQ$uyi
zT-CJ?4)L!O6tH`^iNozyqmQnHkZ;$V<G+eI7OEMp&5qEHed0Flf6{CJr1I8$wJX`L
zTPJQjV({)rfz|Q~_s99C)|I7ORhWs_?c83gJ>z0TeA~=Btx-=ps&^#(J;u>^?dhEr
zfd>Vy<O*Hd+u~8!l)FdR>%+B1jqCp`G-Ed^wuW2Q*gWg>I8d1@(C8yJ^S9VSt68ld
zuM3DjKTxaHb>_t4a*>%YlxE$ioRA|lGm&lYChhG_C;u7?F4H-+mA!g#&x?h7UhLjw
zw>xZiyo&6C99_?or;Z%!I-zp<LX>dPm(#I@O0o;;cJj9E%3W9d@iu$oUe&^r%mUfV
z9vf>c3KyLe`*UaCFR!z|XPsr7&9GN)!}Z<`pSoE%C$j{{3Ha2BJn30*+m_+TJeL1~
z8|L5WeKW1&+^6mh_uejedq-5<=-vk%5#`CxB(wsRRn%Y4RZ{3KSe1OBHK%e)#))S|
zo7OlL&MC?K^;c)+#tX8Vlm5g_vh6--Z9VDeqs^PSwpeuuxT>mh9GJiM_GBZUt(q}Y
z<oueQB0m~+EIHI-=XQOyhD|K1qh|DsBY7{UCH>gu%d<Sa<I2fB-g!UDU2VlfVwu8o
z+kdD&sort5?bcQIe={ESSQ$Nk^5D<sEB{;s6<8N<;7s1Y>As&eaKQ(siEq+kMFQln
zz4ova;O>Z=dGg;3qsgl)s;-LG-l=G~<B_-V*|gC6ODkucwYkyYGT9_^SG(=5HCyg)
zX1%kz+*(D@_J7X&o4zM)kFC79<ZMi1eYa$w;L5kVPrE%*y}*`RweDs|`TLXdUn(QF
z@YS*|>U;BJeqzj%J@3|^dHb5B`mVsjWd9|{-gLj^<?eqd^HlZVFZM1LH+!~O3s^VD
z|K23Oq&i;Y&&szGCnaz{dpm2g$lq)uUbA=7QHk7@&$t3FtZv>Uvn{acVZ-_RJC{Co
zx+ZT@D>!?O!*`p>_pkFi|9@L3_GhzWw`rMNlfn;YmVaLYt|ar^KXjkzp?IEcxb;K%
zcOimthxBed^b9-1VAyb#p=m9fgo#JM!$V=)M66UYy00+k<(-Ns2~=C$DI{&p_rPL7
zPp1)!bCZsx!eh5yX0Iul2bcb5Y3CER@R_kdVR;J!R~3_ofC6K{BE}9r*QzNm7#O(c
zTDg7*d3Bj3c&XdgQz40=O^hN*t*=5(uP~ey(VKL1)wLCYQO9y788PTMu8VwoFP3B5
zyF)CVY8o?@R0|&HHnEGSMO2(>;4NIPFmIEi;`HOe8u<$|DnF&|p21l((}&Gi+-QO8
zEIsF6B`2?}an<tqIV<7K4esp2$84>)=j_?Lv@DQg&Vmn&M|#=a-`&oWG@k2XR)6m0
z&k)YV3PL&8wH6+F!^2(@o?-C2?Bi|zTfQs)?*7&MgnOBP{?y(-O<}JE*F4?QD<Y)3
zc~$_2L(`dn4u|Wf6qHT*c05Sv;#@Ytfs^(Bq!|V)Segv97!?>8vJUfEUYT&1(MD$D
z%9YP{y;{BY%_?oJ4L(!2+?J<o;qF+oDC4wFS;ozc#~Zg@NYQ^9)KhqZanB-cPR=DW
z9J*9z&q-)tUp!~hf}>)agA{exZ%i_5wUKUUKC5Ew<I{EKNmKVljbA(_em>D2R!vV-
zlDAux9-eG|%R{kw@}yTjC(qyQSe(0kQiJy5$$Ed@sHnO=^(kox%9~um@Z4)-$R!U~
zro2nu%5$eE_{^>D3fW^I8mbtgExSNVWsX-<PuQo`NB4XdIdy(rW7^%&1)tfJG87U;
zrNkoQd8N!E;(K+=v|89Q9`8v%tUN7hxq?zEH{YFrm&#55V**yH_RR3s+&Uxgkwep*
zq96sv`S(P<RxWuK>ZPsx{-}BU{eT)SZJEUpOWmW`7PN}Wxpy{A(cmZwykFD$CTlf|
z63?2GU5^h{H0%&Nbz>v1*6)gLZ9`>CAG3#F);^i!nEdm}wH+28Os|*BzNzW;b;`}e
zx#e|UyB^o*>Ah&}Fur=FwP)KZtLU7kHzTv>CjAOqc{jOZ@A7BL9IqL-oU>Z^^NRF~
zeT*`zmd^e2?J>jE8Ijt(2X*c8j1H9vFNoIIJ@JqL2k*pB7c>OTr*JcfH##`vxB5H@
zII>J&{={Q-;+rnUiHdgF=pVK`qV@4`?dqCUCuUztnyhg{G2=mNM520$?*DnO_PE`b
z317EfHNAY2r)K{8qEp&CpVXe#s}|mzYHBIrd)DN#jikOMdqkbzv$>C!_`i7lW^&a`
z*0-B`p3T2+m+@fV^OQiBX1fi66Qx?eWVCwR-BSD6acb7pR)eMYSB5@)zgBBQlIr?t
zagAT^J2IWnW_*{`!2Tzm@nf<>s^n?~#*k|vlfH_ud#$w-TQcp&y2k<&169<XAL8^A
zQD9^}a9Uk#QaIOSB}-vPA3eU}#5Zc!O_~n8(=&@X@k-2XSE)s-?14*8JZpodi&^rx
zxjgZ1S#m(-pn)f6;H8L!H7y%8ZQKkd7V@O6(%5v-My+!GPVO0EO2-ZhEHT;a*qaqq
zEcM@k_kmxSX2`rp8JnL<c%>gd$tCyENoMs7M~e%IeR*?)Jyv=o2w(r%7kVUkt4kHT
zO{6A&&ArD{+=F<VgD=jyw&<(r>aC6uYXzc}KR@pB)i~fOv?y9gM}%chYR6QEg8}VZ
zgjm)xoYQJ}JU#TlBJukQ9}7b!b_rx=EZNADAhJ_QVx7shGfKR{>9w8?Z|8+rYd!Sy
z;yZQeT}`>T(MC_BtY2!)cFueBH=Uf5@aCS=y%m1<MOO9tclOFXx0rKX;Z)7EuE$H3
z7;E{~a%IkWrM{ZQO7xFlNGQASRqpt$6I5~-BeyoaVD?|`kz&;vxocC#R;?Rr(manU
zT>Q1d$)RfXTZ#XtZTK9Y1U_=rpQgv^W>J#nle%h7%sh|j9ulo;$AY-aK3rLDDRtPX
zHRAFH4q^SBB9mAqZE)$f4D5<|6(O>Cm!WR$8{=7mai0RN%{EP)Y7(!@yC7kjv9VWb
zpZVeNg;{37PhVVI>!*3^SDOlB`=+m&fyGiP^WC<-J)=IE>!bFTh=XitGoRaiZSb(>
zaPs06edpvX8JMD-Zozx;Xu%}q6Db>$lm!(N3g@Y2#bvN8zUKI}*mx~hig3Gy;K!W8
z?rx{ZuALLM?J(q8cvaFTeTsdUmglT1As(j1%g^5Xx@`8Oi9Iu#mao;7SikeQ_H(Cg
zR#Tt)hHI@_d2h}&P4(A&xqkm2amRl?wtVka!6kvueM71)?vDGq>Dhb^(HDQ&xb8Ql
zv1-@eJj5cO;qWi4yJ62N(XD#F;yIK)gkSAiuzrf`0WPQBsAnr5xJZ2x;F_FRID7TN
z*lJ7XC2YkjrRoy@rEpDA*^tUJiQ!VGbNhyveP80-d7ibs+_Wgnb;Zj1Mxl`98KSH7
zj&x5y=@V{w_^STYi6)CJA7@_BxVCc6)xM%N%2Sq}nZ0|#*0m=Eqo#fSF~^>7lEdYh
zT_U#<_;+iVX+Nk`@s!P+=Mh(6Gr4AS+(OgmYg*YO*bYi*KPXN&ey+apScr#^&F?AK
zijGcOTQxVhrt^g)zh+Uhn-h!KMXn%@lPi8z{ht_GB7O5%>WmB0UWXXW4B0kKNEK%G
zXnJPz)<^hlQQM4Mlh?nJPG$N9Cx>j371^jc^}grQ*$aL?yRETEx2q#~^$G25Mb|Q~
z-Sv6e>!<g6%1;kF*SY8Cbr;ERdvB@kRmvZ7e_zP5s^IvaMz`*;SS~5o(>Rc+t8zd(
zw7`v>;b4XNjC<ls6ZXi)%uoMnoh4k?<?OLBQ-EJsn^)EKX4FEjJu$m7cbq-FdcBf`
zkhdaNP=M;u#+gyOpIo|nK%{U@?1~0;re#k=AIDDFc12~k%QB;wBdfyJ&lWXU`z9q_
zBrf8m#O5_0=A1gyrhefiN790en_onIi(A$p9RB&68&|Z!>~sG`vbFbUU0NKze|4zZ
zCes5&b2l_Ty;Ut!AsZFCE3}2%`&rVKjEMB5TlCT*^i3VP%C;<e_hteEcjgz%<BSis
zv^NJn)QNUJy7rge(%j!pf96^6>+ain&gIWyb?NVId(UZe1~9$ND9=_4n_PbL%A}$R
z*_)os;xRecoOk-%%h^vap6p#8zwOH$@x3n&-t)h``|rXHIdeX+KL2)uZ~eC|Z&y6u
zn7`Mg`uz^=(EZiAChsfPYcPLo7D$mRxOYANL8Ed;0+W$S_z59}b-!jmsP%t)@Izh)
zo93CW)=j2Nhe{T5+vnx6C@7Y!4d7;|miZXrfArzrMhm{f1`*0iUD90)J=V&G|5qt7
zSr)J6mMom!X}NRK;f?B>)Z}h2($?acxRdAf0j;wPiv^bKRBWECyqM!6NALL$mbWX6
z=FHqwDP)yV>{vU~tZLK2`wmtv9~U!z+_dI}p>ej#YKB8=yf?d6b(wb?e+%fh2;TmS
zp}(bL+Uk!Qw!$krFBl)))nBF{ebi|8!k{e@)`BV>_EOf{`(_I&Txfa|;Qa1JlRNju
z_QZ+H6n8W|-f#YAe_QZUm5zq{QIm9Uw5W3Ooc*n<amrNt;?_sEmgxj`JXUhlmz-Ca
zB$+bV#oVy>UFEj4;12II!oD?o#qZeaELOiO?D}b;nSYI3gVp?oPgA@crJik;{}t8o
zEx2RP?*Ee`Uv6jkY!|&|#or`zmS&9)!ON;n?<<|Gu=myW+TZ5U97{JQY6#4BX4o@v
z%ZHV(SF@hq;cRth`jN*=B|jhfw|jbakA<q>an)M~bPbLdUtub_b6|P$@uNSwH8m`o
z0xbI?m#tj2XhNkW|Cd$8uhy*$kW}W@-q2*bNov>J(~^>FjP9(E+&b%E;ws1G47|ZV
z=M^Ve6-qlA9p=l?*<p5Lm(^#Fb)T0^tm1DmFy5rhvH7-+*rV<Pnu`uSoX+Pc`1hxi
zQ?s=Q=bojvTC4-tlu0xniQdZ4A}iTBN9>}-HYc0T#WL3`6=zS`YSYu85g}A$a@_aB
z;Z;jctzL6#<(;MymdU}N|Mwj0kj^aPdU(r}U2)yrgWVT{Pa7nzd;GEI-K*mzLXNj5
z%iU_0)61}ZHqo{tgu_yBzE1R%haXPgW7c0ef$zD)nVSjv&Yb2omFtVN4<EU)E0$;f
zo7Z+#k6k~!-j}Rtz`1(gKc-`;oc#=02HsibwYx>5PfX>t>Ik^(k>BF`%lVX)Q<utx
zrk$@n45plAV{xx-lx3MRhgC;t(WJRcjr@vgcIhfK&q?vS7~%K)PLpBBxvMkJ9sbaB
zwUhgeOhbuENAJoRXBInfw<yWJ_FI|NQ+>rjhjG)_C&vx6j8vk9&pRBv+TrN)#`5<Z
zNA1i0f3_IiuUNhN$6^Lqf!!}QwlEx0eDc3<&n|8g?+eOD>{&KTc`CUza#*)Xtg(E(
z`h$d1YEMU4N5gW5rk}zW7gs56YhJkigTV19E#C<Z={;u@FZnL!I>mTq3(FOg-3Js!
zr_Q~_IH`D#myv;~LM^wV?AFya+<~t>S1T+&GNJ2gB9GdoJ!iP}G!uo`S6*U&ILX=P
z$a03s@x?BQ+Kaa@ak=a;&-`F_>gg_Xr^|jNT&91INY|Xu-7Hca<n!#NNX09WAA+Jk
zD)oaKeO4Fva!zn9>t50E>S*N5?f(*Iz0kIIaWyE{K6c?qcbRub;|br!4Xi9j`&z4g
zYj!%X3s@-r_DT(--_ht(<}IQ<uMaPOeCm0MUr@!>h|>R8pC9Q~spWci$B$|4@#tVC
zO^);XC%CT&vV6bAefr}`Y=WB}92CExA!%ULtCD=ehSN$cM*C0BCb5)*fv$WHHmqC9
zE%p3E*dB|msUnAFuG)3a;=0C{U8QsTGXzhr4dicj(0ud5j&sAr@QFuHX&nwSwaMUF
zdv2oV=|ayf#a-L8=UyrjlnmTG(?sUrV}pY?bq{CH<>o$TGlgMA%%YMbrvjx}6~5l+
zW4+lg8aW}9W0EYZ!{n8-v=_e<Idb#h+;h7R<i6UvCgDiFk?rj4$!j*~y<D*O&7^e}
zmagw^dasOFURN^j(Sg1DMfm2eJ$S^#&ie1p-z}@dOOF1UC|~#U0^@`Yr+ofjc~#7n
zvZ?i>^nzr`vmdpi|L@wK=yG;%)hu^kgQC=9Q4tckp$GC+4)1>9Yt_TGBYBHQ^eT-D
zOl2m=BXiIBHiRDfdSi}FWK%@Ul+rsZElkb3V#<^@85Ufcbz9=d4T<=_-8)t|$P_D0
z)Aq0WvPk#WM)_#l?<qloE;FxO;PrdsZ`ONv_Dp5lUDq;oPuwWwdSq~GIl~g!J&t=a
zG<46Lc*J|C)#3shuanxHE54?OzIODxblp?jGHYFQhcDN(^@*&dUsaj{u6L|x-&EKk
zc6W{d>&2g&HNV&f<{w$OXX?U^+uh%!uSlEP7?zw`8ntM-@g*hM$o{93`~KeVzZ#_=
z`{1bR{|Enj%~;+Y>CA{RQ{{-3J$O}iv6A#+jnYV`BM)x>-1BG?w_8e(k)U$XqzB6l
zZr!lH?Bnt<<MerRS1XIVv7dj<Tkd;w{_f!VRl$E|nuX7iKf!6z^y9*sw?|rj*hLkF
zMB7|pFn*+T?C7?qJ~flKtLAN3^LYDirzPzc0aJ`-|6P1r_gPZi#)+K=@0WgYetyRD
zsZR2d+N;NUV}dg798G;ZIi$n<+AWK+$5zw2PR@;4*^sb&!nt`<{LZ}aob=G|Y-mq#
z)A8~UCT+p<nx$O2(|YRP+&vc&xo^td<}HgqMIEeOc`fsFij~g6mAShLauZkIa9(#Y
z#OqPgo(&=8so{SQ1Q<X5za&f{?%J7tGns~On-v3MPYN1tVSMuN>s&65E56dQ7o|LP
z8(3}DR7?%n93gi%!oH?ELoX=P@6JEl$VsVpnC2y0*xkIN(%kuAviI8)h0E^v<s@G<
zoiu4;)3J=ml^^!Z_;kth*z{YMPy0N%`RZZhyO+J@UBNq3<d&x1dN1|#z@O!<XS#lQ
ziLQ>_*Q(blsoLRjSweS)=%&Ie>(^$iYYjQAV{jqm%2|`!zd9ay3uT^~V}7cIL)|wu
z)+@7S<HSOdm~~V9OzQlMr)Apc-JG2kv&J%g?cK~NXMN4<x&kL8ug*wV`Ju<Qjw_lq
zMSW|`q?31gE=s6xGW@V)lFPLO-fIWW%>AD-)mM^h+H;j^E_2ftzCA{JI@A|dd)(R@
zHcv2M-pW44hTBg=<5+d=-x*x^oN(`}!969O?C@)GZ~w+=sN8!W;P%nBM=Y!BTqajX
zp-i3O{f51_7j-pp98h&BHeGzF`#7KB=dFP&H>euMJd=HUxVP5NF7VECo}hoK+~!fa
zim7kssXXv`^Ng9zgyorVp4@}UdnVsL_cr%iZo#!(cl&a8cieIKmc<!&Nye&YL(hbp
zC->~w@an?njk_hzFK2kzDv(&Yjmz67apI0#b6KIPZC&ku&IgJe&DzXu(YZR*bKlRb
z9nsUc?%!%Us%zk4dc{T7_f@rm;xvuzd*7>HJDL!}{bj;`F*Uv&yt2|qZ!vvm-DrD<
z>AKF2SxyZcKW-n;Jw8M4&SBF#hvsI6m(DPaOSTq!m0kB((0rMYd!bPHM`88CV|RD4
zO?$lX+H}>g$xRYZN_^apcPu&+JV|iTD;Kt+gx);YIV+!(DW_EL&Di>6-@HvNM`Gu%
z4%<5I+L{}3_gwipr>qXzIbBzQl`l4P-p|$hKizA7;50#qzr#W2$gX{+zFw}(Tt+b_
zY;(>g%WY8=GHH4n?j*c+-({}fz6?F8$N!4nEclo9U-!!d-UkyxOI}5OaSOk^NG<)r
zv@e$PzIZMF;=R4ZV%rz*tB(VNJEWdp`rh+EsX6FFPM+pd(;CUW^GzqGrT*W$Vc+vx
zw@>@eJ8iOb-ExK(`&#B5_+lHko!cWb|EuSV*1TQI9WrLjdf(Boq$5h|kAQu;`)!t8
zJ`9gu?A~Peeu33Wg_l}kWr4{B={^i~uAu@;3woT-9kk6|sKy#*aHlV~ye$3kQ<*I0
zWx8|ZCfS5_oU~0}$oHx<fw}H~c~HdnJiZ@m87oTd3!AqeI?DT_vHs)yd!G((2wn28
zB<ASV<@-!k4NJLHI$YP~=__{R6-i8;>c2T?v5Nnv$?WgF%#OAl*|d4utrb&$iu1+I
zV!!wCn0U{_PmBjn6y41(_1@gGMP*^T@l#jhWjg~jzbsk5aC6*3gAbzjt9A&7X6?T=
zGk*Dm|MyRva{jR|e3!r~31-2_l8t+NT|*XsP`$|1D;wmL{@aQF!S?@Y_s@0Zt$TaR
z?!j*J2YbvH-)t+{qmXXF_UC?I^2#682lZ>rV%2NnJ09sR-mXx|W#-Nq$7AR6HIZrC
zl~OJP#mj3yr5}0!Zubu1k8zcES&#1a*k{qo`|!f$9}~N}T&Aw#Fun2qAtOUl>xFoG
z;iTWkL}PghGTzI_eX;vvrm*Vt)IY^tcWjPv9t~M}B4q9FvfY<b{|9SzZ!XxLQLd2M
zT6+E8o9F-D%73gkS2KKWwL9anM10{qzM8<(-@n##)uw;^p8g}ZuYS(Fey&v)?rHr0
zT+VegoX?)ELzkJY=jMN<f&~YfnYVdJUI;iPqA5JZC(!7?BZnSl^Sn6+3Ku)*$-7T-
z;V62@Fj4v>kHX=lJmS+*!?wv7Y<_ZPrcv&vJwGq1yU(|*=6ew|>B)r!?$hf`vPu?Q
zW}Itgr|UKKWsq*rmYgip17g=2-yWKz>%Hx)n;G+!qLkeiUo(W|KH$q>FLSWnu<g~u
z)X6CflNjrn`8_rmG721)=45s=5a3KY&?V{2W+cSW(8$4H;m2U<*`y-0WWv2oCru=S
zR^D8pcJ$ZQ*Vk4@&(`(Mdi?eda|!>Ix7FtF9_(#p{%`xI;=^t4sm$)@=2Ut<PCK3>
zW%2BBX&;yHq^JK1wy^Z^vinH}?ECh=tzTx_;wKtGs=<EJPx+VFAFvDi!yMW7<h`A$
z=eL9I@-;CJo=>U!=4*PPDb>UxNR!#c;zCPGYS|Q)zRa{44pEX%rs@`4WeV?L-eQ!b
z8=q_P$X%VUL`hp?Q&N(y@h*?Wy{3<DEb1%S>pH1ld(xCgflFG#r6-9OEqUm$NLA5Q
zL@v+qm|9BDAui2`ZBD1A#TNZsIz2jT<(zP{Ni$RPjro?S&B|K$R3>^|uT$5Yl2xwH
z8H)QhU7W}Kb*B3K#$S;_(zRk)&*x;HjZ|OUu2mJiklE7eW!`pOsm!_kMPHuJSKatL
zvy3ZK@q)x+A(l%O?7iXh+@&=yzG4ubV;)e&arQr#!lbvoA;+ee9ewiJjg@bSu<!4W
zCT}*L<k~72aAs0S=sNGvP|+G?-`!Ir{bx3&ut;aBI<DW=bM)el(%g_d&$mBbOx>lt
z^|sQU_tTDaMSbPFz_wRyRi}Q`q)S&%#g^r!blWy@KD+6_nXw`wQATs8;A9@<1j&p=
zo`$nBqkASrWvxAN)Jt2j`3{$~!sXLXPs*)cx@lj|)6)}Y8!i`p>v3K%-T3U0t2M^w
zt$+U!Jm=$hJiY9OtgGzG<#U(FI&sNvWnE)wY<r<<YhI!$J9B4=fVfiOqDggMJ$)zD
zt4_837Wz4Iw^_)EZF?^^$%jgqd(M<hyAfZW|K&Qv-M%9MS&ydumsm2vWnP6vcl&G$
zfo|pnI~G0C*ZY_M$c3rWGPd`4>Y^R2v$dz`u92~vkZNiFbBRcFhNJ4-*jYVFJ#we{
zjAq36#ogYocI%z!``yJ$Vx)H_S|)!unz&PXT9)ekrP;>Zv(z=KUukB1Y52A>X3@lb
zoGTX|*!fg!!EJu4x*rd$|EDduUhXyNa!PAe>_u1pgjKINr<%(}3inKp`~AN6?f(B6
zhbGPCxe&?n<mAnoO~RXWyft%Ip9-i+nv&+Jt5vxAn!k$5*0zj91_^0b?Kg+Sw@jR%
z@<NIuYTtLkI^8_;lm*__Sr0Au@+^^Cd*FVu*yoA+KNOfmU0}%yG2SF_O{vX|=YJ%}
z#1%(W+j0ucX5D@B;i;i!-I-+Rz{@?armol7cQS60R^qxQGn0EJ%X^%E-@LKJ*=^Rz
zeub-&C#3%`Tm7xJ#WvtIM`cmi>fR;kwyz^O#J648x~|dC;kzOy+gYXg8){W1#y&Wr
zEL}N;T~#$wx-+0P>+q6E8fuI@+Mch%j&to5;L~2Xc;b_|lpSY(Y80+{BdC2aVzJ22
zM=jZtoYHh2&p6Pb=CyFiq-hBziv$FP6OQRTGY~XiEb%nMH@s$<@#4;@cb0By_4#>y
zX55u!W>Y_wpT8*gWkSk|H@%aqC+!Ncd|Kl7Zewt8-b}$Id$#ZMwbhSe=e+8%&Uw|h
zk^;r=zdPLiTYXwJaSnTPr%Mn+|01CoGkVn=7B0R$>8U)c)$3f}l`jGpw2Jm7r7k>d
zbU5VHhP4?Va~1xo_`cfpX{zwl)k}MnukTFKnkHs;ap}8BnkMd#M7J&ph%f#*aTkNx
zVL_oaqER0fI+{&WRS}!m;U+RYxO>O+6;^&8`dig@XWep9nU%5nhtb9G=#N*cZ))g;
zncO_}>F6f?cVCqAf)sBxeY2i4=VFi5;@7d$o5P=YtnIxf=_xaH*?B9yx53gAJ=RQX
zIy~u3pyJDev#ve+A;`$OdP1V*?2QwSTwL1nRxS3!n%hPz9ap~e^;CU&BJ;Q9{<R*7
zJlieR9=SYGx*@r)Xw?->cdwuS7jO7*ZdUl#WjhM2)<rU%(L1WTtdO<*&SRt2wv$r?
za}&2Hr|T6LUtOzrtLe(^xA*dV5A0T&wMRQ{pIm$JRqqgs)6@5U&h)SB`n71mf$MuG
z9uTd)WS#$G6?=riG5?UV8V|X}-%P$XY^?vM`Ji#Gs3S`q%VwX232f81Ca^Ed%_(7?
z;Gu9r$GBr!w7jHj#kQcOw|kCuSZvifd*=A%-Y?yvDFu^bxu#9uQ<HkEdS_wWye#p?
ziy!nt_ZIdR@=OyEop{1<&Pw0><9Ah`e{;RfI&<HHq)j`%bgO7>5uX;gCw%v$z;$1*
z7@dE0>e#Csrn<ruc`t=sCQ}rqsdT&gE3?j=VE?~q;zSuv>3WNa51hMlRmz?jFL>)b
zUr*wT=KQUk>M~0*X2(cOE|9*h&MwsGQEkBW_O_D<n-c34#y;;2TkiyVRW3Gq8j^PD
zYOZPE+Y}}K1LFFw;fA$u9#3NFQQj;!qswFFB7wJ2g%h2h_IbRun0ZiV`O$B0^Cs$A
zX1(&8eSPb@XC6)Wyac;~cW3Hb-nLwslb&_^*0OEi-@IC(KRJJ1V&G?|Kf5Ld#Y!Du
zDxK>5?Z%BOy}WnX6%RW~aOuRWN~jrq)Cl{wMyJQ=N0dj_g0w=_sf`U+)Rfnn2Roe+
z?REUAeb`q#CTiA4wa4F-x0Sr^{?w{(cHrqzKQ)OswuJt@JT3p9YOgQU{Snk3d%8y7
zaQn}mkB2@<-miJ4q_ZhB|J28$tfkuvGL<6YMZz@Kb>0lix%ups_x;bFzCD$1mtEM_
zBf2_YRy%n{`k!yrTFX{S-+fl-u{zInnyr!VJ&u#N+^*E6Z+*&hK1Dms*C_ks&U6v&
zxib>fPkp$#Uw*X~my&1uQ!}AGxkvJt4RqaYGAHq^EmIJx2ollT|MJqBb^a%NN^|7h
zZ?C%a!tLywWs`P$SJf8g=_E*UudrAD-GBLB%kxQFk5=!wB;P7mm{z%M_L6J2`{YwA
z4h!D<c(#4-&Nlw4OTQm|YQ4PoMSt}#`IYa#pQ*n8d^P_WH{~~%<czc#FMC~9{eSY7
zWA@>>-*+5Q{TsMpy;#qKJ%{&C5;%79p82c4v4u0##qZTGOO4WZb<qELA+zU3#9Z%*
zk{6l80%LU+?3^R8>%_U)8zzJY2=1?F%{m}(s`8xMht?j>nY~YCIGJvqoBM!+Npdys
z&55=NlMaS>KI!8zlZ^VnHziPzEAe7rOr-eR`13bJD<@8vk$Y3%8rz>9S(#M!nTEXO
z1?<5NzO$!ZNal68wjjQ8io1u4*e#u)1-eJ)-IPpwd2{~Pn}I4Fi&*c4ELix!^@?h*
zSfs(3)*ipZH7!gU$0n*Yu%z5_+9T;{!O564Z}Nn;_MTHyLQbk};rpE9S;%sH<x7vA
z1Kmn$w|D(tb4z(wROrEZJC|^%?Ys3VGf6#d(S#hOcUxoDadNI-DfB+$+={}MKAzbd
zU8Qy6{8p((o61JdI^tSk=sM>|Pvzd})>9|z=iRnqRkn0(|Iu*y*NoX)C;BJYCfdpN
zuFPO}abS_FY`I-{WX+kb4O3d3{&;!qi1)s8+o|{QhCf~VueLN79&vdn>$*`<$@A=A
z*`tRek209d@o2qb<8&)i_L|#^h4*t<7~9+&)DAlErO3HCJd5S=`Z>X=&ihqHQ~R7_
zD{`8|dc2&qR5BBIvaj-&*0!*pnsI999FL|49&IX~O%L4G-0@sB=iJXZXJh9)J(99Z
zK-PB?x9`Wk@QpKc+2l9>KlteMg8r$jsksHQ6Ep5!o#`4Ys<?CN(V3O9UMr+ME=+X!
zkQjO5E`!fP<7F*g4Zcc-mn6)l+*_)A@8|KfsV<RB94x*^7aF!^sK&LX2u+?6;(7Gx
zy&e|Eq6-xPOWAXeJe6%r@R4HlyY-Y&&GAuUB3l}h!m=cxCGtkg)Lj`;6dKYsd>qu&
zl7Ah)%qp(UHrMB0SbE)&^r}Zby=t119%<YQdl~Adb?&sWVDy$V3I4mTmOo^hec<|q
z60a|{Ek<6_^Pi*@&ycQO%39@n{>zlxpLHJJXo;=nx%MPbc@4*mP`9vOGmrjCoKebj
z!F*G^?~B>Zx;I)BUH`SJw@S)zO14J*mzz{+71MI&$p3}D89Qb^oyf9O_UPRZpNu=p
zYoDpty<1-QO}+l!@^@2wdzPwri}f^$X?XJ;Y1}8%RHxFUm2rCM1CCs^iCYvh>tsF}
zN<K7*=AAUPe}`H?+RUFn6fXQxnACGIaGFM!rh3B06`nrxl3x1XnH$U5$?_|6X_#Mn
zokURGlgN2Tk1KLUiED*yRk;*$;&^Le<HFWyY?e|jF+!6|js<3TuxOk-6mvq}jx+ul
z7vsOD>0h2^mTK}#`b_pXmK+fElELkejN8is4w+V-)s=Ttm&vI0vBqD?^r>7l@7%pP
z$L_h`nYH%s4J|&|r&p46eyL14^2}teXQZ#}q8psH{{<gg9u+rGdc35M-{{tvxhY;%
z6aAN%$rl(sHU1asx#9Lom6qZr7PlM*dA~-dGhH`NP88FUVe5QccO$DpsQVtPjCGiV
z`<$h&FXe-C#Hy`gJSyYszCCt-q1ZX;s9oQY%#Fu9N)jS0G+N@an<lleuX}Pp;Od!`
z9&c)w&9RD&Rgl%a<hsA(>b?&uy8C?gRZP@-#uF%%x|uCeLh|V@*|WP?r|mtMA@Z+(
z`GLFFGR~T@U0QQVcEQxv10S{eXQf6k+zb9Q->ontdYfYFg0R>k7Y3Q5n<r_eR85l6
zlXGuzkJn?puW{_8liTEqOX`is)X%CPS@F{0M(;75mS~FyY5PRt|1UXlZkf#ax`fcL
zYcA9&sq3}s&P{u{^x68O%g(UGpF5EDYT*Q7#{R3O{P|w-`O~8F_?CZt5Z<-z=<*`@
zy3%-~NiM;sR{egW{pZePdqbAhYb1N5t{t2-VMEE{QZMy&Tjo2<ww4RI-WQA6e9Aqf
z$K}qJ<<mZ<%eCd~oist@L;Kq+tfw7cG!`9slD2wlR9>@PhQ~F5zNc#*Z&}cJP$55&
zZ^D^`_C>DU8xQT%S-0oW>aJ-oq*7&H7qOJikStAT6>bW;by0)!;9P~LFX~zH4)JA9
zP+Bu#lHz^a*6TI_i&*bPi5<N!c`JO{k!YhHS+~qHRTu26jvXpsNeo%}q;^J4A4~H8
zk~fcPC#ANmPg%C|aDuDzgW08mCtI#fcysYR=UM#=_wxTfd--n?%cWHwr+a5<{X6lJ
zQ##n<?Td4Z7A8%8qEK))XI0JIi(682m-QXVV@qDL?RrqmhCI3FL4Le%pS=#caq;HL
znRlM%tyvWDe}%H_+{NoI-S?fL{8!Pt{MRbp_wgH#E&d*<e_vzWv_Q7tDenX4uyN?}
zZLNCX5wlJxd$s$r6wjiT?Ph*E*yidSo8l`s*Kg@+vD->vds)R^KIq_54G?<Be@VM*
z=Eleww{xEb&v#Ij$#pU2lkiv~`?K+4iJH)zT3$)H_bE?%BA)S7PJ7yF+sbGE@@O5)
zV_na0lUk+}y=VA;;pu5HHl~tpetkC<4>gCrBPzdBBW!qrH8>es+F$vci>-KY{-4g@
zMMbAB=4<_4qy0Gl&0dAIj33^eRoWR;czK`FSqI&Aw|3_ET=w55uT|w%WyTx6z8)lV
z>RY6LenI@N1}%f>6W-4Ax|riw5@qbd$?{)Ari*utdGTlSEZ+JfESB?=z5cAdFLadU
ztbyCvcZUo+LM7H&AAa+wOec9mzk|p}X1^Cg>W0F<jkj58cogY9fAqeqVcmQ?UEN~~
z{NfUhu6(C2eCkZg#@rKnw_3C+OC_)E{Cs`pO@<>MyI-B^<~kxWA--f5>+RMrcdpI+
zD5fvIZ;Q9G&V9lBV1r5SjuC$UeU3)&dU(_-N;BboP{pRSDM!^*d+x@3O8(}nH2sLZ
zlbYv}^+_V7o*Z92diqY<CY`#gdBJbRnYBi7KbSb0O2byqi75Zt@=mjjNlSNf5Ci*%
zb*UL!wkA1Ec5!LH<+OO-X4$ObEx`uix7p*fKDXxD*sd~|&Qhe5exu4Da?iw~dAT0O
zEbedge$R1#_bQ`2<l34%<%SlPKgU?oPZwDJo1O7_cEzV$>wex1MlyDCtF5=2ah_c2
z;Bi2@mFLt{7yINq2VsqEY8yTJj?CTnB){^RPgm4VJ=RN|qTN@wG95oEHkH+NaiHY=
z?AK*;S&ffs{Fu6QgW<|aE;m%qawIQyzp_#8|K7RF1>PLYdur_`c7uUKemzUA%#AOn
zwjBPKmAfWU)8fQpNh5#bleYFo$J2CXvQ<1$+!R0Wjf~WfSzb|1=O4N)(fs|ZJdI&%
zJ$qYXz>kbK$Bir(*miu?+;4PY)28s^ukE{w3glg*9^cZ)%D8k+EW2!p$<zxxUT;5K
zoEnt1w%7P`p51(d*A|s`PcE?doTsB~^tf%pd|ge8ZOO_fmw#Y4-M6;#fS1?w`&^fK
zYHs9;9n<wnQ2+h6>)tkb%l>aS*KFSSr!aGRK-IjT*1NaOVy-xT>ZylGOHO*slQcaC
z^TN5odX0K3gq;gJs`KWwt(!AxS)u#d{G&U>t-oA;S!&e3an^tBrHkuzmo(pe8?>2Y
zedN1&NA}MviCEI!>*w1Tu`RVS^`nyhglR5|=I&yd>hi#FR>+f*pf_9m%{3joOvC;x
zuIK+6SM*6Htz$`Pka}r;v-DSGb%z$=-Sy5}9|!(Ov@liI-F-A>V*i8nTPAOf4E`OF
zU$Hg$cUbcywOOnSCUNZ0FY!N?;-dZC^seb9*3Ow0!X+z$7tfe(RhhN!jQF*GN+zdw
z7e1>_a5p}ibMUvo6z3QhIaZrHQPa}oZPG4hRhO9kX?whuk9*y#Xq!ptisF~HE-^cM
z;q#ks+fRN|oZWqVwv=J>GqaPAx1GFi*0N}c<MZ`VzjsDv$cU9sy>zzb(#)fm|8L|G
zPdr+@Ur%cNPQyhPukVeInB*%RUS(&v?b3=bv)kKyk`_mXoL+k%RykMoYQM|GX$w`(
zn#gZ2KNnK-@Da;_yW9Pu-!yc;X>#vbH(eyu{P!%Yw%OiielQu{;V(_FXnXwh?2m6B
zD)t1#2zcsVxzy2Z(ZO9epWp66SN4BB)_00;doB1L`V_8I%u-xw5t|U6cjV-`*Sih>
zKdVgifBwE=xmWaa=AV1tS5LUkGwJZgSK^m$_Q-4Wwp@8XKQYqMVspEB`M1)fvQ>}g
zTfdW^_1osd<OyC|@8l|P%)4#*>G94da{nK=zgv4$<ptZX7g^I^9rQL0mAw4$!QK|v
zXW~IY|4MuR@6Y=n&wJ^*p3td@vtoOGX8sED*y~a7t4t!tDYNWDw^jPx<32W*d^v9Y
zdvEW+^TH&-VP+FEw@`@1f+>gExRrH$HUvC8!YC*ZpfY2k0z;F8c9zV6iHnXh%DHBl
zSO%$hPE_Dj6YO+J6ygjLnY8GJU&>y_Z#9Bzo}X5~u*jue$#mA0=NFbaYF?8%5S-<;
zdYj2Mr3~-a4C|tIYw=i@F=#i1vR^TpU6dQSI_%`W*o<wWnwxVuCn+6Gn-aH=>-W2<
z)-xVyOl)KS<7?xvJn29`r;?n)Ors}<+I79vd<xEpbwwW&_~KKs<(MYR&H$giH?uOY
zb9t{1^ZmHx*u@l^>2eMKs~uilO|APMmT_Uhu}5nn%L7-;QTWVwZen+g<lJ8gGp}xZ
z9CKsURNd+b*IU6F^Avv0e|&Mee|_9vr;m1f56LUcPv0B$cEh6D?y&z(^H~1h*3{X!
zO*!L0>#c~@yj_o!Iz-LZG)++8oOoGN|Bb<w6Nh*#FQ3`OZ7Y+|bIQd<XUjo9l?>s#
zK}i!A+`2a_VS}!jLvQnxu*Xi@WCJgEOqNdn9JsBufoGEN#J(?qi}faj@LBZ=2x%-e
znxUP%AZSgB#_9GTqnB1)8IJ;%=%>zl;XP@eO2#4s_5~+Zj75u#oAht0d{%DF+vKzQ
z-;>wDiuU4Jug-6ICLN;T^z^g;ItI-{r~U`7=Q7zaZQT#$FFM;qc4-G2246ZA%p_aV
zZ5YOq{ASY}lULeC2h8T22|fI)X{tzMW~u1alWDc4`VqNjr-qA~-f>Y$G*>QTNM3Dy
zG2*S*s)-7cxxpD8e$makyRH|#&Sf|s^f)Y9Z`X`zN{@u9CPk@Sv<<&odNbCe>GmCN
z|9d=VyTkWXbK8bos5;<Xa>-kN?JC*75~^<wu(7MWKESd&X{qQQv&9=?d3~>*d=&k3
zSH<Bl-?=jmi-=1l3s1CMea+Qxq3nUk)`l}-$8}4tY&4&FQ*`=P70$yq-HR&UE#aBV
zexOppZ;j3`g{Dn5QHqmdlsuSTdN`fbT4p-wfYNGbsR#dGF1_EB$uN0|o2F9lo_&5y
zecRH?Ca;fj)C%b8>zciC=k!(5D{Ea}^&5Opi0yd4o+m}%3Zs<IYTb_KLYyD{*!AZ)
z_KDxpHaKo$w`OCj4bR1EXFZ?TY(5{nR4gTPZBZH5^(74#A76J--uqQ9@%pyXeE)YJ
zx8LZgK395Id-;hw_Zs=<Eh^_!YF1zS;p&M!Pp>EMv;DSU{q#lOd1OWGe}C8<qPN#n
zD`NFy&L?>vKDx#hTt0R^>8WF&=AwW{XU)X!X`SFaT~mC<kh^yKIn(W4DPoc44Vo+b
z8cxg%>)o-Tb(&zN;--VK`EJg}oLrjjqC8Q%)-ahII&n<lf@RQ@CzDJb{r|z^9p2R$
zb16hvWK)~U<((eW_n9^5&usD4)Vw_B-I7DXeEEt;eoe1j!nC4>H?Sk$`ida8sLR{Z
zeyjdR(hm=@yZV1^uNJYC`7PGI(4SRL=b82W)&oBjPb(gC+IT2*?U^5|jed3Yrl<t2
z`{FK`qIo2ubjf2`^<~ntGgFvWZWil*@_kn>@22Qmr>?!>ndu=XI*H3$sI%ysgomf!
zrr7A)ERwr|H%#&9kG(W`-uxe;^VNU3Yb}jYY7k=;|K!?LkQ~{qy>*kqaa|Ut_uO%R
z^h!2vzo1_y{VM)L9p{>fg4!l3!V{gGgdMUASGk$)y4V}Ud^YD`j*080uKWzO-P!N*
zR!o|cAXM1#%y_Bf@pNb5M9Xfmr7Ru3ipkP)?n(uw-A6X7UMjqGRjg@S<+cgg`XQG~
z3sx-afAD!8t84>Pn9*|Gj7FX%8mZ+XOrk2Uvh&(E&Av13knK+o*U7Vg-VQw$baVon
zpJJA!x7fxR3z-e|LNqgd5)OIn|K)nTuJDQci+9(pQ>W!bFAnZ_#i1dU#v3$Yb@Vc`
z@*}gI>;h(Q^!5xeJ#_G)&(&l`Mh3;7EUY37%nUjV3=9knOb(0;9RC^EIb=LQ2c>Xu
z?~;13>g)j{7DYY{#SK1^MyXzsD#jC%or2iSRc$=lLbR({Utg6{O;cE1x|(f+lt6Ks
zPm?k0GnveSyAJc2YPtSa7Th@y$lTNAWtH`E$1+cL3)zM#%R}cmu&3NzwKY3nQ9S1=
z;jJMukt<V9Zu6O4wpPP1<?JF`?eYeTmCVz*VpjQuUYKq0{}Ruht_=@otUk8ZDeKCE
z;K0Rl{j=Vb7zH1j#u~OXOZ1~v*er`{LO0G$vyEPB6L<Hd)xyKg3haBP>2A3Crg0nl
zxqY^q%MXUFb$EB>rBKD}pkt<d4_-{1>~NLSI(DvK?UEx+d2A`=kF$O|nQxBf{Li9)
z;oPCy%t!j~%!~V7@iz6ALrdI5%OfAQT-|>C|16Q&flG3>9Svk|+`=iDViD5TD7V!(
zAahN_Hep?JL6?P&TX?1<wiy>%FzC)-`$@<*w(~`>kK4i%GXrP84ZRR#@JRG=k5Q4^
zB8QINpoRH1r&KO?sy2B{p0HZLGo_O`G(xt~<7MjONdaz}s*Z+1k&lBsQg_}+3RF~i
zlrp>L+XIH>TffKW?0RedbXJa@W?Vr*(-F_&D}{j*=7yhKu26cDGv$~=y<Ev$mvtK9
z8D-pCBV)^#?kS6P<&QLR$jMp5DKsl1F7%S;wkyUB&2QJJuvs%tc$M70_NSR@xU=u|
z8(n3t|Bp(fN4kAowJ31Yt*Zf%E1RcVtU7Sw+J+-yw?!h4Py3p^hUxCr%MAPeu?qV<
z64~ap^-5Iq+9bDwUseU`#JNpPot&C^GkEK^jH?CndyU#wGG9%e<uKLG++i>K!)VQY
zFF3EXdA^lAazFni@AFBHf900Q7rfk>7`=GOl#ojd=XgJ49A&nAKBq+D@hih6Y#Z4(
z$=2>toG8n@@K0Rj^}W}Y?|=6*dsB@pZ^Ls}(Za&Hbyp`gE!(9MnPaHA)J12)jF&eL
z@n!6I_T^^3zM$nJs~~RG(yldERz=3e?2dRkb#=nYr&{KnFBZ3^Gpnjx&5EvFG%<hw
z+i;&^=a*0J#@5$_w$=#$-y(TI|Jh&j#^~!^OM4@Q&)xKywSg(?((;Xm*ffp*rf;}6
zLt83cy0|VU{rFW?osu)mGZX_Cgxk*gxhU^LOq0v{>uc^XoVsW(w9keyF2FN)K}nSM
z+-pu=-b){SDOtH!xAxm73tQ=F;TMX(+BmL%^fw}8s^!YxyRu(t*?zgmXA%~9bYH7j
zOxD@Z#jzz1pFaBi?!A<@&%U75SGB%YyM6n6*UnNjjX^|wSIW{IEgG2y`Zt?)JxS$X
zqR0|IeX{2=HP+1&zjUYX$&8q#6)~Yo@{9_<^|>ayAcfnUEG2<KcV`8(Tdm&Xqg3u8
zSzYEV-DJUHtM4u1d?&c;%@^-diH&y;{r?m^;iiwC1GD4=2bWFOw+f!br_AVV_Gp&k
zJm~Greq>^(4U1CPj1%4ZjL98`GL&ZZc)0pLlIp!yu~cg5N4Mox61S|rEaVSd>ru`V
zbf=9)MZx!FLEGZ1uh>?)nXg;ln{;_%pVVP4w_Qhaq87HQ9W#)*^25pY;6ksZ70V2q
zsy_Li=;W2OayRs!<X58QbK*_KBIjF@OQtRh>5!IUHF9+HDhZgSP-<2xRoS>oY>hy(
zzsFLWkfVX2XJ+%Qe{y}*>W0%{0XyYVmKjZrK3e3v@6*-TcS>CeQ{T@nmJD0-p>0BH
zUFyusyXFUQ9X`{tCoAmyoT)aux2y{4U066TsI&Rhm;Vm0eDaEzF8k(0O`Kfss@cS>
z%BS6vJiGm@dXn4e8#MtlHa4#LH0x^O`O|S}6YRgf<g>DoWZyXXRLjcaDNc*e*W}C)
z@H=!Y{c7bR2R&26C5-1Cd%e{Pd4vwG*y*coe>7QQqg#maGLOR93nx0B;56GCqT$RT
z7!+W3WShskLeb4D<aKLAh1aE(GSmrrM67MvZhkDkKu-E~g!YzeOmZx1{iWvIc;+>A
z&DAGros&B!?eX9)S5ytzYIxS`$Yj&lq$v~fjXLMND7n!+X<e%H9xv|7Ll+xE!W1@K
zkuLu=<&;O@hPLlqvm9TgM(kH~Q4)A7JM$6inQ3p9sIfYxWlu{v7yEzX=9!U|>ett)
zOBOn`8r?X3`_;bi=to6s_0lp=|C}~^-?LY#A}4a^JJcMV-n?M)%<X6Ma(%T0?zVMr
z-*78#Ql5y!Pu<)}!7>eAY!A9MUmb6;K6uwdkNtwut2DJ1+00ynNSQ|sAKI2mC^{OX
zItfp^q#$zkvU5)4>2pEf6!br9WGFsz{$7*belw8&Bu|pu??cOZWg=J34G}(atKpiW
zvx#BX`=`bS7=4#rpE+YWThpwt45n>yS3JeGI=OS(uI)B_sod#Ta%{zdiIL%jlcr9q
z5?E@)ZCEC9qHR&nac<6=`Z2DPHeYjD9GLN_pkdFmX`DsV*YPbm6KnW-61U~z4S`Sp
zt3@3W>~nWA)z4~5fAny9zt4%2a<bd3W-H$+TKCg(p=P?Z?UladSW8)U|MZLO7aykH
zydt~WgGEWQu%jkFeS){=N|hXk&OIK%++`V#d$??u8Ex$L%U*gpI`-$Zv}HG^L|y)M
zZJSzn$gwla!u-8=ujAYHciWt4$t%~Z<~e?2=$Jlz^M<9@LOi|y9PmoNz3%g^w|Cw}
zv;K^Y*qe3yl*;3k!AVjdb8gN5_wtDcN7U@bMVd`#rIVchw`TfVw~I6jNzHR>f6(j@
z<<)nrFI9fcu9HG3i{-BIdM$rEVbauxseQXHFUV|`^V?-&JH`9^WS`SV1m5$h@P*9k
zHJ@xM$C&D3|KH0i-|q9w#fu`lqkN`IZr%2A;(@QvuHN*ypTKvVi!ry}R{nj-B(14F
z2Tr_*KX)Xq=bWU$?yJex{8r~5c8JSfy!tAr(D+*Fq@_oCm5wWMS-1YsDl7Y9>f~X3
zR^Mpm$>uQjdwJnAk~G~O+21w@-psk<uHo+oQ#tRvXNmM!>MORplfy-M#y$@>1KSD3
z--~@$OpTZqDeV*ZB%<$hbG75I?S4lOxVYcjB=$mB`PUJJqhHRh>e4v2Vfr+aqhFs@
z*KSE%|7(>^YgO0l`0XD5E3<js>i5f^d0Fc{xllt=|D@Z$>C^Yt7-l#robjvsb%$NF
zS>d}{mtfEPt0(`K*y(HE>DB%pyvthha{s)X{qw}Mr=^uuT{!gj%L-Q8szCl<{AI_s
z-dL`iGD(=(d|&6(%QnWQQS6S+Gt!gR3OX^01xg&|<bEV5I8{K%(3&$$;QzLSfTn_m
z>*f8ejhxTR%|qm7nieQ#MD~Q2vm~lpeQspD-1I2XMb}Kg$ArN+!kFukpi!Ijia_qk
zUsOv&MOD9-KE5FBtRbv3DV&uh)k}k`<6w*5iWU{7RJMny%nLZ<%38x_w1#VV6l_)V
zmM}~B(UK(5mf#_r<WUl*#udM<ZBau>){nNl7j0oDT20#nn-}Zln6&w<@aPUKay`WN
z<pAHOaQ-hB_**0R|0)O=n=y3W==k?PfL}03;Ozr`jYPHit_IcavW1^ymX!F$8ZcZc
zbYlL{$s8`AbtF)Hg}K%eXXPDg+!whv9}yNy6qMT-)c3vK=A&`x55bfL^-HeteP^-V
z{787ywOD3Gz34-_OUxx-Y?65Dq<7vU`{J_{KcU()?K!g=dUX$0c^u3M_GmW>w|HQo
zvokE`8Aoosv%ocu+Vj(U-@d2`<>-I+qUID^ZElNzOS#CB=atcu>h@gsD>~L*<6-t9
zT;jDm_gnS)tqX*@BLw~h$o{>+Jz+^lN|Th4YeRo~r*4L*c)I}0(YynKPV>~f)LSz7
zG)z(h3^g?ac`FhYWHja;s$i=~TIM0$b37>E|D^JXrkSE`a_biw={Z}?GcD|5j|!Yr
z_;r({sYmmKBKbuvBEQWUey4LUOtRbEQOtVLJh8ay@|LN75!qoK*@>R56F%02Bu-;*
zoR)PmHgbpQti);25v>>0L{lQCpAVl_QaQbBW@+-x=_NaJz0C!@(nU{)x9Ux5+npg`
z&6XEt)E2*7IVn7~=>%U(M#tX^6PO<IcUmyanmKd!&6#sN1!i{&^q-J?IW;}d)6(jw
z@T}v(Rf{9$Gz<KjZn?yw%V4G877-(XZRN@{oLN_N@&qQbI0~+tQX%0ey}e^%S5wqv
zC$>s6so&G<4|15S5f9%ME;`eY;m!{6yB59$AFEDHwJZ4FnDfkd9`nOq=7p@x3+BDq
zIgfqeJSN8ZEE9W~8mFCgvw3D*n?JGlx8=M)p8bC%tHUf8yohZ5u#z)sK?%pMc^tDi
zqc$w~5HUYslkJvv)no?Ar%Af3KPA(qPx!JRj_u=wnLAY$&lF<6IHB_f_umg4Q&I$;
znyJbND&{l<s7;fzVJ_#MnzbWQTDV1%<zXRvp|R=74CambEQZ!AJtn`uSkSOdZs`nx
z+D(bSS`E0=3N)4m{Yf+AbaHVMiBxG3@cOct`&&c3tAMoy*Qc2&m5RkVR_&Q5+TQ+X
z%Z_NxdQnr_Rr9fV-rJS)&R?G=bSgDwv)IfHVyDAOk`hXqyH>Q{YU}?0sxLod#mC^@
z_b=ytnz^hwv!7#OTgj@{<eyXhS9sL8nl^dxPmS#8(GXbtbEc}5!0N75idF(jQnS`q
ziFW)bWlfaY=ir#zF0i&TVz!5uXoc^B;MwiRmnh$u-FK~vxp2+IZEI#uP82oM*f6=G
z|I*US6Xz&<$_g^uR86aA7GS-6Eu}qWYU`?;XHo0EdfL3rnDH%hM*S(_w_Ve?W=&(>
zxbEAjb?>?as!uK8ytSV1)`EY(7X00{;G?BLWPt#?w*aTMz|0LBm>(>7{c}CPSO4c9
z^N*TU$^5E&zkI^i_L&>1R?V~!T6=2MoSXcell|{y@ypH>n7wS4&Bcl0%jLMY2Jm0z
z;{GooC}$C<t=aXrZ4!rImwu*zXO^gOr1y@HCQk7t<C$Ip-xGGQ$O|7Awt6h6eR`tZ
zCwb0`N~;nB1UGZ8RZt85xiw?e+{sV+z8}f)He~ZT-dDYFTDw=j*Yp+q)hmjnw=)ZH
z@>g%K?B33PdOLgJN+!m2mm9Y$3vxzn*ig$oziswBro<hQ3t0QOcebx)4K&y}arcf%
zvo~~A?_jkSXqwIXK#FyN_s+%HJNT-0^;Wf3ORVCE%vxGCQ}Nf#4YO7$du>`5wP|zJ
zZdS$Jmjzd|FwWZjn*Y?|<~OQp`AiD(!BIg%GOo>*Rqg^uw7S{Fy5oK=nK{G2=I3m&
zBeIi{^xM=0j{H9<lVY$%=XNIRr;-g1=d4zp`s)bSgI&|u82A6YweJ0@1)Nz6zG-i)
z=w8rvdOzo@4O6Wd*jLYMI=zEQk(Di4fX`ZpS9&A+?*mLS2L=8dm^guzH~OIXpAC@>
z2c>fkO0PLsQN6O_^ua~9H>l<uWXj(7R9#7S)y8F9i*~=B5N^0haka)Q%LtZ-niDq%
zZ%h>xzOq{Sr<7r}jG1@2;N?z>Y=NI?nPH8FA{l`$ryHeKXlypjwCbFE)jN`Ld%pII
zS!!WhnOa@EO@sti$k}aXol~;R`_gu1#~oF-cd$P^#u>PB@~xdc+VeSj4m3p{U_E%e
z@%P#WnO*a|cQNVg5~?||VD^c&|279XC$a`SI59zI*Q7ZM@};LGXw3YSK2u@!?%s-3
zb9;8PZJep9b!w5Ptg`pvvhN*{j~ym2S9?A!Nn&f~X~8w7I*pz^OLkexPEHd&HS2Wn
zquCoI1TB&cwK5D{Jq12()emp9o@dr@dE;8q&ic0o!hsGO8oYOYlst5kXJ`58d7?Qx
zMEA^7K694IXNP=_K%UO=phw38gAO#w><avFuE9qj;KSNL2G;L$&VQ6Sf8S=q<IaU_
zFBfuq*6;nWk-zf7re7Pk{z+%utETm9!d?xv;47M6g7(eXCY<oc!BTOvl1Ss?kih0|
z7bAD6?eJXE&9=oMdZ`YFvE59ya35*WZKv05obC6Yt0MfAu*Aiq(lfo+JdnS=cx&^q
ztvL_2Zk@0?SW-)2A`8dH17$wP*$fW|_MCm4dE!RQ&V|uegA`8G$DDs>BjEAi>XpFr
zOp4c-9j>wZT>B!!>M`MbcJKK-*^^VePfq(&T7P3#`-<I$IRbLhs~6n7zUtM?ZLfDP
z@;Sv`cxvnH3kPnms;w2Mi9L1M@l^i~p?h9ZDU((!2+EwQp7Q*)nwsNj?@dRPePtQN
z_o}X4vc&kP(c05BPM7PKpYHgsA=9WX^08r4;MS9!0yF1!3kP@3pH#nAfmQIqyt+46
z-bo&h^_6{Nb3S$N@kX5;8NLD!r0%5Z-VG37{E;JYHTHZwFRQ?Wy8{0&T)VsX{2!U~
z!q%R3Te;qB-}rgD|LR?<INt8&+Iw=p^!;_WZtTpy-?8_C9P|Bkuec|8Xmv+OY3-hM
znNikSQO1=!|5#(b`y4|JiLLD4_AD>0Fz|dR7WYtW$=(H1Z`m|DU$ef<G10)JMUG$n
z;o&omKNiXVnxwDbSiEeiDo2oGdXL<SL~i4cSG^}KQ)XP}_4jyV_W|YwSDfUYxY%7g
zS^MO2Ba7gMYd(CeKI^Xive`9tcB}ZUcGp8(GyfdoaNJ$-)^LT)sfN2xcYJzU8T%~o
z(EWfQ{(l0UX;J+DFL2Fnsr+fG>d+(IwRQ77Ymdz{o7f!#&oK(t`&s*VSx?;>*mw1G
z`2WTixkYQ%CL2vUB=bNzNvKh9XJ_|m8=+l4&g_<*%KYGVl)#-&GUv;31Wxka<=l9D
zk=!f64d<`zeZrD>H=&p1zU}Kxaj)0L-TnRM;QwE97nD!<czgoO-jnZnPjOtlaOL8S
zHrx9tbvNep1RnDfIBzF#QBUBq9mCaqYUVavYd3UQ?WzB=P-^=L!Ba68ZO^PWJF@uO
zr;D<|<uZ<I=K0G054)uK*6YdX0&k-i#=U#Bgda(2Oi?!6YB2etZtIlWlQpBQgL5}<
z?)tKI%?4HrJE2dOcUb>?@RDV9uzT|5jBv`|t1|pg{r^4bw>cSH=Rf&j`_gGoRd;p7
z>^pf<@BX&zMaqWB8UOh{r^VgaD1GC!ozMa6-QUlC<~#ZH$KKD0e4Co)&YWQ3xb^|}
z9$D!hOU;v*%FE{^RG&M-B)E60%2sco+W~Lnl>R-OVSF^=&Xy3brBlC2X}Qi}c3?X9
zXN&9=<N0hte?+tvF5*0>dv#IX`P`gWflAi|46fbJz5DR&-5)Zn*L3g6>TG;w%NBpU
zj_c=ZgP7f`v=(Wf>d4}IwnFRYAKRZht?%2;7qHBKt7ZQ=>E2Jyvp<b$ZY=7zbo9SJ
zeLvTti!Xn%Kes8&GdBFXgmF^rt_OU2d#?l<#tN+IZ_4LrJni1~@cX(;(iTf2c^<9p
z-R!}=bsPT&Yu)(bd0VCk_xWfFTwN*k=zpNlk^+TK%S@6#e3&=;fUoS1H#^UNK65Rg
z@4Q0&`5?LLkq)0$|87^Wzn<{#q@KUW{P~>>5AzQkVEp>$s`8A0hfZz0vUV~T_>~+R
zg>~aXDkfZ85yg3%cSCx2f_BFmwh1nsg$<r<Ydv<#9MDvhoN6jQtx}^{VLGEt=&2)-
zSqZZlUGDM}GzTw~UdR{gXjAp^<)M|XoI+b)zRHlC<K?ZPb8sn-<R)e>!PMkphg*#~
z!jpom8+cYmDGKtfxVrAzG7jcNKYBlz7aZE(!YuFBW8pnTXQov4MHRJAD-0KhtxY=G
zC${(O)>mIwsc&3l-?)^WL&W#vmsgF;#J`<du}kUjP455M)_eXMY(LT=QaO9x+x7Du
z&M)ELwuya{QvQT}QOxBhY~Gc;lK9QL-n{MZfj-vUo6g=7mfdcB!Z|C2`Fo1@<TW1K
zGOxCY`2Mm=Hrf)uS5eY0HH9nJOK}HNOozkYHm6go7H?3}&{{2&WZ={$Gp9p?P5hRY
zX3vpLuMX|-aLgC!Fq7=CTm396S$NhQ<rT8gQHso+Et?cF40R>CAGt{vKMCa4J$4~Q
zBrr^}NF*$5Ve8bW8y|BNUe0Kqr0|ksm-o^Kb+ad#sz+z9IMunQh2u=?rXG(4RY!bQ
z$f@!Qb%;c#8J|$7<_qfZ(LA{&*!*hF=^%@?4LQ1}E`KVyB(Kc=YsG)j=$IT;-AQ^?
zUzDavRB^7;{if`(@@0+G0`9hqB#+BGP8?Xp^OG%!>kudZl0K)ceV4Qp{SPv2T^=QA
zvR-%g73rz#CJ3BeEf%D7!cbUuQ-cq8vhw96n@;KN)-jHnvF5n(jU7H77L7HMdP<I`
z?`+!s;M*Istu1HXip3PH-_5)24#zGtD~-U}9os8}cWs*=*Y<h2XxP0qo3~fhY;1o}
zKPk9`VWNj}MMM84Ro>e-Ud_F+s8Ds@+uNP@&Tf+rbMg6XJ+VYK<I*dixXcA%QJT%+
zJC7)@;GUZ%dtiT~OK3;%>|I$L4zsi6R){QRND;ltCi}@iF??#BK*nsHBLbRpq=mZw
zJ5Rl?T&df1r|7klfM=q!%LGoPz?@iR*#vGQk5})Gx<*c1w5h+}r|4wWi<hl(z2|jz
z<?k;&>=)gv(pAh*Rh6A_N=v-_gu<HLA}8iq<W_oK4C8pWT~u7~SBI$B<3peQHQoPP
z7uq_?NUFxm^<7cfBI>pB$*h>BmzR9<_n*w4v%D_bYR&^8QT4@re!m>TdA6)rwdPKS
zvgW#mbyIuR@2lGRGElrp`*+;L&8dZn$A9?vyeRv;Ir{mr-s|V`-#(dP_WgCu^gq_W
zIrod5dvP*{;mog<uEM%}?lQT*rs(B!vYY(=)M}q|a7jn&YI)8zNB4_F^oLD(WX<`<
z)jH~7U*n&{|K(Y-Stm;r>38^a#)&IVaq_U3Aj<YZanHl~6D^(_Ot{e``Z&-ds%WV|
zK<fE;&O#CGOU&{=UDlUee#laPX3+$OmCw_89OGiT*iG4Gk2(f8PpF&rO@Q~KUdVz&
zk4j<=ac$_$Ra(4&Gw{+1C6`_m{k0cZW=UPLvt{K<u9mvu`e%u1-_ih8j?T+kMurS2
z4_~-N3Aoz33C?(@kYdrOJZIaNuoZ{bI+<LTY>;qsw6DLBocdoP*s<@kV0FVL(ejr|
zyx#?Sh28p4>b`|@u9ld=oi+N=Sw%OioS2LzEqJ52evZe|OJ>3|p1w=lv6<ss)>MIc
zqMNN80;L*ajxNykTQY0?|5bjUBaVscw>?cw(Ks9_^JrSCQ;Y7-H78?sEH_Bl$Xvs#
zBKzpuQ}vdb4GAYb`<(2KZ-4s6&FimD!oowP?xlhs=C3;Vgu$a??&}g3#()b;?%bF-
zGpd4zD=?vbn@4beW^9|7->c(RTM|w>Zd5XuEyHKKg`u@vE1;Q``|M7k9U4z291|0h
zVPIn~I-d~Q?aQ+E<k2lt)_y;`sl~OoulU3VgZ9!V)+U`HYt>9w%Kj6m?n|0^`V{BH
zKLSfuv%gYwys%Ov$j3M1TjwIzuIaloqn>nyzZ5XKo@E<SIZ;t2Nx)$93ezdBFS3~p
zZaa9G2pBt+DlBn$6Z64ng1GKVQ5nM*|EKd^@{BK8V$Hg;PbP7u+L?U;(KjSng|=*%
ztK~XLHqxQVv*mJsXvac#ODWcj2*y^OloE?Y5)B!-(cK4TxcE&eR-77ia#C~cCzF$k
zt=eYSH`H8HDbWzQoAz_svgRvEIi0WOdB~K`J07d6xUreztxl;=lAN%*$>Yc>*-Jj@
zLM`fcGjD~}Za=ataH9kNO0T+^o}wKZ{R;n%zW3bLB>XGDUom+ix8xDY>04%W*z7cU
z>=KYNT};>0db{I%7r6}o_#21#<m3*t+xyHupJ}1of8|<$ce#^@@Q2L{7Ru{hJIJ-S
zmx*cOixWFqg1dbb6ej*_@J-mVc}ZZ364$H`JrWWBrIt(od^-0{hm)F?*R1+IQ%sp0
z6*A*y9f;1kDZk&Mv%Bcj(vGd?`?D(bbe9{e3VWx_`>N2Kzw(Rhq#5nm&sy$!M>tKH
zcJjO3rO-~H#Tp`WOIn4EKED#Wm?^A%skGv7w#ocPH{b2+xRj+n6`vEh6Z>j~dJf;k
zSEsuoru;nMddBxJN6KT**_NVb#I_Z%u6cif%Ua;cRn9~~1y=K$7h2lnyrY5&*t^t@
z@#by1^srPnZ4cM)shx|@GU@cWEmTu+X%uCz6q-9hfvtIi6SoJ$%?BBbtQsAM<d^^D
z{$F(Emc|7qUjf@WbKX2$*2wo{`;r;K>cMUT!SA+AUhrnk2G(`*|D7%d9$mfdF}LeU
zt|^^Q#0?}Hs(hBo?Pc3sHls<kMmTAA;;B~d(<Vt5{iaS}&}_ZbAji?bzMyF8WEbtT
z!cmr|9hzp>vL1Zk^SV*2SFrRS*XF9s<E^1TPIRbDNZ7yOc>DW=ZtIMrqG=H)bZ7m0
zb^2Yl=W2=836q}AIN*}B<+82#H2IzSVs=Ky-t0KTFvFy*tmEK`9?1qVh10h*6cm|e
zMI3ce-Xwdm&&jCc%*K5=J&&u_JX$0f`eMPch8CSaQ6lbPWg;#cm*`|PiS_rt&3v=c
zQrGafJbN<hY$FdLG0lS1uL6>BH!gNxJQBQu&3tp0ugwLMW)r2)SHv$~PL)|)|35%s
zv4gax`>{z;Zv{H;xY`Li_s(Bb=#<X$@IdU-ps={7MQhmK#g?wBFj-tWUr~-}|5D*C
z_xGl4<eeGE5$Ap=^ZMJ{iv7QCEkC+zLg0a;fuC=>Bt0w=pQyvNVc!zH<_)c<noO+=
zTCHz%F?VyB&)7OuLb5r5UG}74k+5!{z@*a;mRRf*s$bA<{IXSYGf$tvwrihuIXS8L
zNww-UG_85P;(&uupVD@n7kqD4v}-glXjq7)O18eW;LSSSlpVc9V}-^UqYl?5rt;Sv
z8YiqYHY};T(8BmZUL%3^P+(t>#nJ}_bNoKB@4BJ+++yjwmkO<!tO?S_rp&vIT`XF;
zmv%c${-4L#ci`fIM#sHT0-d@iR9#Olzs|UR4da3LLdy<Dbwqs>ne%|t*Kz#}M>XG%
zi@rPUY}vs7VBz{t%9iiDk1X1$w(^I*c|mKCqXCmN$L!TJH*L^<{CmpUgK}FnOwAl@
z-)FYVmh?Ib@-^*b<!9cwteQFG;*l5|gW!h#3%}>6-t0MRut!5-rP0Odou{mn7qWX^
zYV!E8+p()fqoXC$db>u0M%GNRC<#7|=uHlZtsV^wfjgwHR4r#()t)b{o4u1eZ>5_?
zN5{=WE9#Th9v0@S;OQw9;EOuJ!jWJjm*W0JNRsPtcjLzPJqtKae_DC(b8DkvU-%mH
z4h6-klQS5yjCqr%{};Qxn8k{>^^0xO#}gAcI>U=unM<0(61oZ=sm+?fG)qGAVK)1$
z86rP-ZT@7X^OSq1TZ`%(OP4@Nfv|}Jp_XdlnOj%`IXo{f+Ev6XT-mWoP|oji|8b_R
zGZPjE1~2}bJ@KoBNb3dDE5aw4VjNZ!GyCpojXp7Di2*bBujxlVwwG>jn|6RN>ce~)
zM==?h)>m9TxtmqaRoH00VUJp|ZCA;hYZ`m%{w%NJSbOUgyYl6h7B9nFUK$TqO`H7a
zK)!Hms*H@>V%_qSd@3up$y9GYb7zv}MTOImQ}#qmd4FK>Ga;?Brxq_1nqs@DL*&q;
z^N+TkY|@NgaM)1B`TGhHGli+I9xnVpSA$vaL6fe7^rRnCzs;WFv2sVN!1_LI^_hn|
zobPb{_{3>cv9IO96z@Rkts7PcbXsgKVE(FbYGFo~mX-gz)h0eZmTdt$r)ZzQ>&YP`
z(EU@RYtHPx;zfNs6_|HR%xp|)4poq^Pv|nQXk8X<sa?foe8RPwVY8G~r=iS@6I>!u
z8;;$mT9Q9y&$$KdhC5aiGW1SewYKrWw118bM{V58bEcL4(8w2LWtp(4FS7NX6UW6}
z+;^h){J*Iz5p^P8y7{uj+UzazVHa4E3^~g>9XgD*y*bemYGE<|(c(+DmtUQ6skidP
z5yjm%wO3qBm~sB|-uNFz?4b(myeGr8Pfp5d-YhWd|MQ95OHcT{k?x4pWL)>4iMK-4
z>hi1>28W+XJ369gPVDCVBHgk6*XA#;rEhB<iD_E@C*a6-2GNBNME}HeXbRac`)Ic?
z=KLa`2@fT^mSi-)FIn%qd)D^?j>$8$SrnBIM41E$NbfMvF*>pTW%6=&&9iHktg}p%
z^iRBC<k7X*pxLN`nJt%fiS>*mU5S%Hngy>N)oqN*ID11K)?S>%acsq;W)5B1*v=ha
zxDQC55$sjzUuniD+S)C3$b-u%_sQfVElo#d_`)QbudlM%{)V^hHlNm?9&K04n?BCT
zCdXoDPqv8kxRbMH>O<e37W?y+?3V{dm`^=Fd*Mt*j&7p~P27P~{zrXaosr<Tyk)M<
zZi@#YUM@8p&T}~Y6eC+7SXkWE@yO{iF_rT0j+oQKy|$y@XW~2~1B2O%`wKn^1f7v8
zGIC{9?BHnT_$Ol5^r5G}Q2fYEgHxH3N=BXwHJF7HE(n~S_StfW)x-I6ms@IIvRCc6
zkd@dT_+h?}0CU=84r$%qhZ1|lLN(4sv>OJT$&_T6*5vx{A`8caO_fJ9s&@9>>|#)l
z<?VdaUjEcfYa#2J%{+odCk{Mad}GF?8=HF#f3DG$UD3BC#=zBT$!^vOHKq~PmwyP%
zG7aVWnHBQQE8?DE?1aBZjBYUKRfuT(6;(c@7?iW&_Z#CGf*~_h{l0gddbRlE|2Boy
zs*|oTH_kHPQeJRHz*A;F3y+~+t$@Go#D%ilD>D72F4)n*JUiHj!$irH#ZgCUmz4O#
zg%z*FBCO|y|89*gXx1}m)~jHS*vWeO=bRsjeHMbvxtq`Q?O0*>!D>%*tFhqH<K0bJ
zZ%=3KHggwJ?Xy~0cjL6|lKKA@2G-BGcxBe~9lsrG{+cpq%`y&le$F)|d9!aE_l-Mi
zS`F8TH2?MOt_WV9c=>r?m!3wmo&lr(%&8J>96u%nZ=BIIS=8C!$<jB~9M=VxP2PKQ
z_63ViK{wjJa_Y@!61sVgx3k+VlyjPHYsk$j5BG+AeIq@sd38jjw(FAf(*o}?U-W;w
zwc}@j;QvOOTQ{<7e<@zP#K5bk&ALLmVWnvRvkGTJQL@dhy~j_p<UVn@GHK5lhRLrb
z+~nq{m+d&HIY+G2$7$adJz2wP8E0B9d=?HAV0Q8Ah~ItP^p8UKMBdpK#MMuV*WI<%
zdumX9D8}e--&s?K^QN8`Ts_UUO0BkxO<}s{!r~wHvdQvkgJuRZ-=^)mDg<w8_RlZ1
zx7&RFMe07s)})ROyO*~+=XCB96`S=SB`v*e-U7{#{R>09Le6_HSuJq*isuxm2TNu>
zyknA}6#71De_(9Cp_cLiuBZi9r6x686}s6HwL_=1aAN<(gp)z>O$#0`C}`fYWzxYb
zO!t!>9Qd)~q^`)eDNg_IKhRZ8dEk7C$?$_!(v^fif~*P~d+OwNm|1JRNSex3W>znn
zcxCe(p=tcnEvDW{jd&<>PI;Q>)ul6~_GPw~a?A~jjX5{P>aEt3Ev(UZ^%#1lOrORn
z*Cy#-6Y=)W%eDnf_iFd4=Otx6%k1&EYi#Dv?Rnm<dznrf$CJwQ|BKH5liJ{u82nj_
z`^SlAZ5uY*@Gg5fgMSg5nCZigYteUJKbY1o&7eOaN_azL@Wg=284If=`$8KU%07$j
z_`@wFeW_n;`+*478wFvhKjTw{C#5HDJ1}KZwe0-WwYE386<#P9FG`Lv`n<PyOAHTJ
z@DgFki7F=}{DQv*aY(%ri8WK3a`x%}?g>c?9Ph@)-DsO}+2{t-<Cu<j2Nid)oxi=8
z->EgS{Q<+=hYWq%TV`FHms`d$^98?g-#bsyciX$9JZ0p9AIkpRlv=)w@0rN0pVB%J
zysNh!*k9+yDJ|97z&F{ii1pjsCS!wU^9P~f3f@N+-aj;Ftq#MTrWYN~uLJ{Us8=Zn
zZBym&TQ;fIRJPjNxp}e2YX#ZI36llbZdl4h%us7>J{x<F_tmVITvo10pQZ99Y<-y0
z^y)=l#G@%qk8}DywCv~-PU=|P-|rkTRpQ+28Lt{OKK*@QA-6AU^4GUPIUl<j7PEQ!
zPm@j0o5wfR`sz}PS-x+$m|u7L>$pT~e$=|&Z^U>0|H7m5wtCubJNP;(V{co5czwtH
z{nKul-*5>xusXj^+$U(7=R;nO?L21~6nLVY7c|+ncTE3uv!F3(y8kl|1KVBKx3!<W
z8xzr-Y7!bOI{(>(){gt%zPx8EZJKd;{S1a33Y-g=JKYmIS30HrEO@s+@?l08N96T$
z=BZg4T=!x>D(SRoe~?RNv9$E9?U>)Qdg+6!!s{iYuO05ad$_#s`P3QDH!@1zH`Q6g
zFxyw+aEpa+`P%UAA8W13?*BjL-0)jDUvME;xOeT*@aRJy%C38r^!IJyeHa$)QNJN#
zn%u|f3Zmh2Ib65Aywevvr>$do->ZiEU-av<?(EBK`d;2%%aLzo@V_ZMu~T=s>&xkH
zZ>?*6QTTO2Y{|a2Gd_0ear~U$mL=y`)U@O4vuPjiw<QH~?th!zp=&koV;M)`F@5bv
zU8!1(vlLwB_%0555ScQsBj#gl$A?)K%@>wAaVS?7ADi2|&#%u<gE`pYa%1PZ#_SlQ
z;EiHlyu}BVB~D$scj1!dBaiEGsb}YYIrlYY=GwQ}?_3^C`Jv2o^+DOkzqhjPvAt5i
z{(H~)SM3TvEblgGc8E0h#O}N%x^CgV)TVa<+dI_aiuNpjZQSn?H+@l3xPb2a@3X>Z
z?e^dQ^6HHp7o3kg5_M=Xu${_na_HN8E6(5VoGbpXwM>hT`zs#e<b3OO!8H}FPg@KA
z-z_gU{}ZmuQ`ngMvPt)&(Eo^4g8ehgD%Le57EC`c5$-Zy@5gjqsesf!Z6`8k-)`dF
zak#Z*e_MmS{PVZ<$6qSzf4U#{u=n?;?uzxo@}E-d)l%xSwfrxeY)$&Shhv7j<=Tkg
zR#pxpRW<>ch_*8fMjW>Va$bB;JklW$>8ax3rF5u$$%ze{QqPC&ZTp(loAmVj^uu+6
zp{u@_FNl>6TI=?!SaV~zYr9^w(3<6!67{Dsw|x72{8F0jZ04MKIloT7^sL@*cWd|8
zi(+BsYqxCEyt<VqqABH+z>`VE4%6E*IVZK=`1bJj_H_0Ww$bH!_x4nM|MWJx>5Yid
zT7_F#?Gt<t{cr2wG}768QBl1;joqa!V&|hL`<RXV&&@GkoVKx(-Nr6j>r0F|Z>Vre
z>_i>sW4wvy^G<BoID7YuOk)|J3)|l^9%yuyv1<_3G!u8Xm2vH=bZ!wzR=>}3=UT&|
zt!zDs|AN==tq)@V<Gj{f<>32Q?R>7U&flx9_k8=)D{B3|wP8QAw`QqM4)MuVFnJJ@
zTNv0D$$rZ<Ks<Gc>$NDpRaXPjvRzXWa|NbO5Npq{G`Wz*kyQ4ug~#uPVwd`>XNoS_
zxhoXgSUrz8y2MO5aFi|D#^i?k63+06^@}DRJlZfvaf)P%W#u&`7b~lu7baKmdS2_g
zXzP+NRaCKLVodB&GY04Xt9GhAV@TQcY=ZRindQ^8nRhITicV=%yXH7sbIS#W?x#D`
z{a9X=3NEP0EeY(`-|KQoi+AUeDc%~oyF{v4eg=gs;eV(yCGf52*39ac3!iugFX(!8
zK6vRQEzwZ9n~zSdirM-oRg-<I>w-v*NQuX@xHnDB?p%M7HQXhHY4_`Ok!cn=8%{`h
z>%^7ry?QjBBX5Dc<6EA9e)r~=F9n?XQblg2?rC#VZZ&=9x?p}y>F;dk<tsJyT;h*$
ztL@!%Hv7GE`nIrVn?&TU+@5vpubPY|>v>nU66VQ{T-uAbIL;~9w&C)IxYtX?1*=3?
zid^z{(~X)^`)P0ZeEqFI7RjIb`%^q&*Z<<KttpX(2bU_u<~M3wKP$hhG$$%8@y9gD
z!~f>Ian7E}pnu6z+jg2rspvHMq+9=fNNX0UElGBo+!=B0Tu$J`$*s}+4W@fss;=p8
zPnVC8p3byw_xe37d!wJtT6eTpf7aTq6VoD6c~}-@l-!Lxw%Rke<wlbD%cq+nmaLfd
z(PME>QILxxZ>nYBT%i=<+O9UPt0BRaDpq0JZhU+;J9;ZqnB>O3sfy{x-?eIYzqzGX
ze7)qa?X>&5Kb_k(D?{7X=Ht<3-fJu={9>2fyYjumuXY7r3)mqT>VM(u?3vnvXVP_V
z?tR;1{(8cfV*U^7ejdBtU6Qx+rR(QkN9USK{IwQ3(^mZ7(s6t0UWPm+TgNTI++iI}
z`mZ8IYJ@a4O}P8qZtIEjb-$Pu7#6N}a7h;b)8)1%`s*o0ZKjF62cEBIo_er0#Zo~0
z)`d1}fg_^j24U^bnig$aqS2PYD9G-6L_um3N4s2;STn<OXW41p*`lu!c3ODsp6B_*
zEp^7BEoC9|8GDpG{$?|)Uotg&tD4~<z<5?jHQ42?U<%iVOA|SA*_Y~`%F_?^nx9$p
zNwxOvX~pfFe%#WZ)YiA8nkas{9kIiKN$GEwV!LJcJcr31_Icb5wU2^JI9lgs%XWz*
zdz@l$Tk07!SF2=&mTXu|BKy_CquLu^E&g$7h235TzKGnP-dlb>oB4l#<5S<&nU^DC
zcDS40_|znqlkC@{YMd_X9kD+p=v`l@PWz1ykvo5#W?1%NLtDga<+T<n88U~Dh3@O`
zNq)j)adO3}_Kt!%hfa&;J}dRxWq7);A}9Vq*<ytT!SnX4x38(KxFK_sCoS=NPls;A
z!7kq)PK}=ro@b6eIG=G{PE+3L4yQfo;&+dDI`J9>HaApQ&wJVvxu7yux*&Zwv*Hq8
z-4x$BA|>7CQwtcjTRioOJ-BP?tL(XVOr*>`r(9-~XyHD&`M$f?rp3o6H0HQ%ogAyy
ztZ3ZEpB*Qn?0DkYgvIArvyaJ~owQ`v+4CaBRX>|NG#_l4weitw@8ToMamSvh8D0Ia
zW-O`L7-7WZUSZq6I(`c0whO`CWjqDkYb8=O_g``^dm7Li>-&t=_1D2Rj)}Tkwn%P{
zNEg`jMW|U$NAcB(BKDcPOm%cZIrJZ<TrycJy3yKJH_KFbt>%wVi`zF>y9-=q>uShU
zw(2@#AwOyF-hxBMPh}QgO4zm}@zh(z;;AR@^2`?BcePDt)z3@OlXgq3KHIwF)9;|a
zPs}&GHEXFm`SIedZQL3kKQ0#y>R6%Azgmay<AjdZ(CD{XN=JA;nWU-ato(jp(@nDr
z({*DWJmoe&d^-BdQ-^gsjwbMhbWLqLu~aS9fU7z*;?_-$oUD%vc{@2Ka$V0mRGt~Y
zZqnhy+xkQCzpBQ^b|)ES(efMnWhO8cekqm@b6LCX;~g!_y(_oAa%1Ve=Q1l;SXpQ1
zpKJX>rIORn9hX>NQyNj?((8Ho)b{NuOIX&wUO9KhnF+hJ)C?az{&Xqjli=QM%J!D4
zCMQk2AwF9#xNJ*{YT8fl)A4UVx^XHxZci!oio10%e@5@*cU~Lgg{FnMT}a97h}p4g
z!@DAe*{;#EnrGhl5j5*;<-)lijMk)iW$#FfnZUr1eQ-~dO`F$)Qy=^NXDbwjZB^QB
z8g8-sxsjITq}i8P!=&E`EPL=~SKrrDXV;$WTwv|gv%6UHTYuA(IW3kc_1|t^a+~Hp
z;q8;<qR|Q}iR~56^X_@w7EJoT%<-PJ(``{RZw@JgSc}W%ER9ENm;cN!lX;t9KI3R>
zk>&SBe-AM>?`5R|Y~0N!4|u<MV10m@<JPkD<(I{tPbphDb)oQi>Gy3j)`*IH^w&J&
zbX9j&>&g}PZf#ckd}WcD%j%uElfEB0t}E~SB>UOPW&NiFW^Y(!lP?gM?H41Q+Ww|y
z&9<K2AIF^99QAne`Hy?Jh+XTF+q><|S_|Vd%DW3vwT$)F&hgdt{O$e9N^nA=WcRE~
zUkom3ariA4Onr5#VV4QxjU~G*w`8$N-uiXT=h>&SMH<PWT-H-G!*;n;SoiDwIOzUF
z-?Azqk>~2-RvWcBPg*2*J}a7eGPu8*x6_9Ge}vzJGx=>X?zTUr7N0U*ze{KKbj|pT
zGruN8+l#OL6|~p-=hJ<&zRxZ^QReZN#cXTJe-F#8=jPi97Yn`$2sF^2nNs#;bF1!!
zdp6<It9nlqSygS8ZY@+y{C#Mb=+sSF3^n^*&t_yL%xX$Kn6LCO?e1~gvqfSLa<!_P
zqq4X+&6xkI&u)X>-SCJ*UM#udsTGGpTio8>U)DO&Wq|~f8B5pB>-@Y7-5YP5&pF6u
zA=ID4o*8icm}}_7$dl%o=L1s}*0V&u?7i2)aQT*p|LqIj;Xh{0?3sG)rcC@78JCc2
z35L@gj)=xiVVdE}@+jm8Ul+Gz#PJ!4>^WPR#95AA`^z=e^nbYY&zVzS`qkb&aeqR<
zeG9?8Ek{?kgvvGEy|ePzsx5wHA9xqvbX;?1e)?0lxjQcG-grOFM&5s`2=~_6x_^&K
zpLyt+Dz?F9=DM6K>yCKbc(X8*^TCy?eoG2(t(Q<xe8t-ShgFh=ZL3OLnFL$xgRqkm
z6|&aeddlnYELBcC=5|_)v__s>|J9CTxoj3suZ8YCTgdBQ<Qnd(a{b<tq`I5_es|(r
zB--m0j`It#lw7!65!=FAHeJ_aLFA3tI2-x15%aH2n6f6IQ<`JW_uSS8J3JR%yb>>a
z<JE*l?Hke&bNCwH&er}g{r6V?eLtfuCJ8ie<ZIa2SG2WNFE^~=@AcY+CqwW4zn$~v
z{(r;Ehb-?zJ_zZY!C?PNz=2I6o@dDeNfyZ*hMtG>RyBBv33i4}y?8_Bp-a@g7rGC{
zCniWW&eJmN>)R3Yf^}Y+hM$iCTZX|+?l#%@2QsF=X65Z*)Nc(b`NNy+tE_nUqTa0I
z{6^B-7hFBG#qn5(cl`v%%N;@5Pk7GUko2DzaD5}6c$@N#txLq$#m;^+@zz{EiCauk
z^JFDg-Q6FX>{)hyip0_{TRBtj9+axO?`$(Ga?c@YjcL*|9P<?J$miXV>k@hR^y%S;
zKFvQuG!xqM6S{4`2!Bg)d-701Tt@BN8`d8wY*LzO>X$mw6MWU3rn<bj##W=$7s__#
z?8%qB|L0|Mc)y-IwRmMhHU~e~T`61H<GSzU-o3r|P{PN>pnY>JOTiJBi>#?fCNO<o
z>($}YCaT8vZRu=jgCqCu$~T<7{bS>mhP@{vwjA=>$-8>bxp231$s;0Myff}y6uIFz
zqpV3eTZ6;qkjkRF;;~*+Vzp$}?MP&t7vZ9_pl61=<N<ZZ3+b*m)ZgBC-tn}hPVDSE
z9*4^YPZ<6*<xiO7JY(YV&UsB@7v~)D(@bOYkm5g+kg#f@%nRF~yBmb>Zdo8VNp$vs
zlbnA~@!sp{SaAIg%Upi7W1rV#tmE;L=1KJUFg1}y<9_4A)jw`VHM)8lUjO3x_{-FX
z;z})H)0|u)Q$0(UO3M9z8T9Gq`Xh;w_ZZ7KG`ZJal`)!?QMw}T8LR6`aiypW(_SV2
z-x|5GW5x`rq~&upm+s_{RN*gpbCtoQKRi`ad@Ad;3BIX$foV=G>@h3mMJ2hNi*mXW
zm35T2#Pvv|@AQd`;T;nWu6=ng`&nzg!I@1@S^j+pp7+Op!M;p}ziql7)Wp|tH3_nm
z3;C5gJw4EPw0>&xs(n)|yOc}jEw;RLyP@&u>NbzRSEGHl3N)Qe{Qc&v`^4z=D|~so
zPH%Yj*xv8hMFE#hi!!J82{IbJRxo<ttfSo>@Vdn{yY<)6MhC%M&c!~iDXtYs6|ugY
zY1&n<W=wNA-nUR}l43aL!j!XHxMef{E1q8{S~PL$j5$IZGI}rgc1|<Aq~*!)FSW{5
zr{`m>%S_&e+4IEzRd#+qdFAHPPVt$3lJm}`O*pW6)9rgRm(HHB@DA_FjvWI2oVR{I
zebKh%pd7Ez!wlBb--NUjl^s7R&J2pJa-6^ZgyuOrnJrTec28`yW_dQdY*nfi%gzrg
zzn8QnKXXYdOPa^$a9M9fUg*7~ZF3gIb-IM`{+!cqw)RmRi*yvHw|13A^osah$9nc&
zdh{pc$kDI!4!jfAS335=kY}&RT-%(iBR)@7&3L+M+Qa=b=Qpf;wT5rDRNvyJPWj&_
zvntO$z47XH?b4HGQg>QS!kCLLG=6w|p+lwS>Er4DxVCEj_hpKc+V<^vUDE5A8_Qo<
zJ*YNI*!A~Hc3g01V$v?N4Am|<jz>pV9n^a>Z*^M4tw;-<3eCvLxf*-3H1^%_eZNV^
znyqi@(MfY%W-DeMQ#|X+|BZE?37_PY$b~LR(jIe;%6VwNnLc5j|HOL-U&SalO*wGv
z((QPk_?Ze)u@@E;%{}VSa_yF9l<%?StWWu7KTJJ)Z;cRF+|&>2Jbvk4N;r{~`b;Hs
zvi6h9-`|K_nx%Alrt9T2HCF3f7aQF;k+<LGxbE1}y5QNGi%+@hOug%OK8q|_?c(60
z=eO!=*Rc<RKGIKu=P@k4`^sHY=f2s!rV9&a@3(FFv6Hhgm+jOs<=6>pjeq{v+#+;w
z*Pf#~hSSbJm>^tw&|%jJzLVl-B$hZQ6n?jP9hXq(@X2+vU$(P^ikNZQnI_FFiQd+l
z^SuR)A}6|ejr%6PX${Vdd*hHBcW0eT9$R*00=N3Aol3EvzqZ~EX}Q)Z`)kG24|iti
zu5?Sjsm$l+d*?)$Uv0q~)>j9uTq<)J*=zED=N|nzl_9h(EL!4Y><Xd(xAb>=Bu-)p
z6!_WewpVdZ<Ay3u1Ks;Z+PbPM0w>R&a5T}*;OSGBe8;Je7rvG)kLycSyP05cDZrt%
zNn1f|x^ux>?M0yr<2GGC#d<SLeD~LNe=S!1Rhj<YwB<`jf9@$3OKz=GIR`~17$u*1
zJfrFVI?idQcf~zv3Yp)$=LAEZhh4gey)$>aOz*bF=!X}VOMAWh@xxW({&FQRmrb+Y
z%`%v1GdVMAj$YT!6OA4RC*3%-+P3p%$`R(*g;kFt)@_*Px%o5OjFNfJ-Z{IxIQ;Rv
z{>qcGckjIF%=JE|&@r#bDIwD3XWn#67r$4o%762AojUE7pf7kTL)t^ZRJ{Lu$gHUw
z{p3wMG%w`m9gqyO4tt-lRoH6b7N&#;xp^~YxyQb0jd-8u%>F3i`1`2t)903bN${B@
zC>40sYi*j$=3}ZH>e&m_%vg-)A7v{@ajoI~;d1wXSj$h7(n~7h=W>hd4718z%RYK!
zEq%m#z(wodl_u6x|4ZvO6gT%y`ToaO=lArvfgeAA4fHyx7<DWD*H)3;j@`WsUwDsh
z%gFD@y=>;d{N!`zZH5xPe{~)XzqVa^GJ&&Up8Yqo8IwL1roR?nY#6dpy}m8aks(kr
z=O7Ei1Z{`nof&aAAANl@>*IxK6P?^H%Q;^aUO4NmlISIoO@Z6pe(j0-wMQjxhfXw)
zZ0u(5%q>hw3k;Mxxw+1@2FmN?Xy$0Ad}q0!vpFN={fl!=k6(oCF3Jv2)s9xY|57k^
zMq_9Yr|+#Y-&<xa2QHlxJJQ5oP`tWdH$AhH|6b&FVP(IADLa~V4+OfjU0xmejQMwF
z(zK(OydK0=%;{(UCUN!0&NFP9KL3xcUO6f0zPGRX!s}fhc5=n8OA(K&2zTbmnR@uk
z6%ij%#w(_>KPJ9eEPPhz+s1Qgf77JhBKEC3X0DYyQ@%rUkBH-v`FdL)DqZ`@`_p*d
z_szNQH!r)pzH{mg$M7Q;C$tu1pJ2>7_S~+;b$MQ`mcsfe7Tvw?!e1vdF<tD{SmG&q
zt~`6HJAdc$=dES!ZC=(bA`BUo#d9vr*}=_xI`hK)+bJ1xo=0ZC@jdHs`Gs!&**hoq
zE6F<Q1o<<4^?&af8@XvJi$%Z6*Ern|rRv>ROSg(}x$Lz~p5riib9KOln62|O!&Xn;
zbn9d9i9?fg%D29Iqq8IZtD=|0uEi^P&KhUk{3O2o|Bf@e0{^eF+OBwlWy&Elu3S0x
zO|kPH%@O;&>LN#PNCE5P*5gyHgp!ZBZQ8Qrq|ygIt@smjHkOD@aB<mQroE~0GgtUG
z=O@A?F)pd!C;wN7t`ktrcpuQR^!Bv{4+}nJtqQvD+R6EAr)>0+_=nSSG<OHMF{aoZ
z-<Egy+r0Z_7nr{|yeP|;FP9A0mFzdXv{|U&)cecME|-)qZ_?X)#l59Sd}5H)BB|B>
zdAdpw8jfx|^jxdl+$v(0FTSX-_}F)q43ocG5BATyz5F}F;x@x3&l{@WikF^Wn6={i
zrSq{S7M((!pWmIF{^h|#={n~R<?Elj-u{&2CUkrO%i2SIE;A<{V)^jj@A^$GYcn@b
z*NU?dOBYXNVxQ6|dB7-w=g1VT6VfL(HBPlmTli3Js^`qf)!SEA?>eue_uh_Uy+yS8
z>2qrR+B=x{=H5&8yZU(MQG@e&hS@0>wbU-g6~rn0VVUyFuzyROhFWRlC)YV`3l^QP
z+8y-g!WUyPkESKT2VNXaIKq<47=JU(QZs^m`s7BIJ?HdJillNHn&-9NpQ!IJMYH(y
z)B}v`?i6SqPHk{%Wxm5R!D7J~+0GaxwUWoL{4_htdGbmgKl7C|kM;6!xtYq~JzY0?
zTEa@BXJ=*^gid>*5U75>(^mZ#&#TKvE=+M|Ut%LwqJC+a^Zb83T~oBqcDl0rOqKZh
zf5qA5X3e)mqNl#{UXpy)?~diS*8x*xy)@=*S~z8GSjw^qQ+K3l#P6&A{YunkO5(#d
z=6`HcjS@aKa5_%qIWfDUAocWg)k%_GT}c<rgzdfm$W$G8-m%!*zbC^|=}^#oUbd??
z8+lyyX5HQ*b+*eydey#(O!8L0&4bQ06|$?kwr#eJ{;F~<E4_OPOOR5u?3UnFJ=)gH
zC#KwrXghO^@sZ%9mvS4+j_=s&Jtb1>Vz1`sPnNqT?T}j-x_t(#u=mR8VUslXonu(D
zW*f`Ir~(c4=B5eTPODs5?X*cRMZkHLk<RKhKXy!5wEMipfgb(CdP+TtonAa$-MSKw
z61s&XZzV8DGyXrLz5bY7a=}r?6%0j-6m+9E9zJX0Gr@%6?uuTEgJP2mtd~U0%s9ok
zj%Sru=e9LnB4HjM-|jfIYL3wv-%WE)tTS^dIV#r3R<yL&=*<QHg}Yd^o=&^|Wr@bQ
z!@rC<m$Wvm%3m@$D(lmRze+Bzj&Gjg#kF+)jtQD881Fl;4D#ijBci?9&NOjZfcxH6
ztK28f;=j6N#jNwoLUm;uLOSh^YOaV{d1&iW|Hq#Kt_8*#J#?RTZIP)?)bUL|4!6~A
zG+CWo@b{zpDK)P%w^YQ>ep|F>)6tEF$0ocATrb~wWZFS>b(?j)H*bCL*kfobm9>>C
zTv>Zd&yL9f`_8Ccdb90%&}Uui`_ul<@z}8-D)H%qg<^IUXHHFvS!Wr&OnSkzuoaun
zO=}YQ)Mwqf!?|B+;+M~QnthMf_V4QmbDf|wTgK+o)6RfN`S&LZ8Z|GU#XDtvT5%&^
z)Qg3keg<i5{eE40b_(R|Ox-V4W%#Ny>dB&`jO+X^9+TR-)yKVHxxjT-k(Ewc@2EuU
z-MVv2Mkm5_X6%}?xd)&58SD<TsIF-6-LoZVk-mFmgwpP)iAiR5oL(;$SPM9H<WBU7
z3JA29R6P^+pUwH|!lkD?ymidaR3tB%U?F9zG}Fp`?*Y%oS^jTXwlZmHZ(UL4wfc0a
z?X@eXnQQLaC*`tsr`>DZ-pj0c>w<K27UPs}2K&tZ|M>O#sMO<WI?n>P-ur&g<g#_<
z#Yef<ji<XVUUhHPChjYqAGGEdG(2^%HupR@)m_W-rdr#%M=!lHxAFc}IW+Ojlao9f
ze+K3TsP5Su@>HXsIKGCDW#X~9{;FvON2T99mii^qqQtr>^1Q<j+xway>_c~6;MAOQ
zh;8Ku$9Kx}xdV4No%UG4y>;dRX9g4HyuBV-H?uCrdx;#8<_%P`@Vd~qz;*8ej|8sF
z;wy=#L{*e53#-?!4wn`wJ?gYpzxc^Y>Bvhe6SZSkddKd%J>9~X>-X2!wag!UXa2j^
z=6hyYQggQJ!)*)XzIse(O8xw}h2!h)zgGm>EgMgG{0Y{a>Y~{oZS#LVlkz9Fwt^#e
z9cwy$Tv|^~?cma!Gv#Wi$cvK!h8dcBlSKVjRIC)$b)BMj$@^Wx%TpqWc@^_ruIbi?
zo{nN%T|Qx#hvck?88-_QbI%-}_H0>!(h-v<*^?e#6Iu0L_r#x#U7uESsjD@ez2c)%
zvno^GWt*oMLz8hs-U;!RKdyp80WQWV3e9<5#>)&fb2MVPx*qVDO))qXtiNNUdNJFB
zIWMm~PAZ#uK;(pTQ=hM&qj2SwrliLUW;$F*GYR(6mbk9{^A2Z=#hJ+-F%lx(4}8+u
z7jn27hAzljH)W|qcjCpkT@KElTohtD;<Q#DVwBCzb*uilP%yW|bdJ)|s_uz#{r{IJ
zFdp}DS$X=8pjq^^>6;_hWpkt`N7@AK|0U{czd@2KGvRPRq|mWO4&feK*2#09<a+j|
zyo}4*bXX>==h0jbMVZS{780ynOFXwS-jyy2Tj*uEM3Gxb$K}Se<&7J3yb6QnNIQHw
z+ds8rqP9?M;%)^k<CJaOv(GNk^^BOx_vBp6#@%-la>7N|%t+n3{nXX480$S(te)B*
z&GOz<lX|gpQ=aNv#T3bH1<tEf+@ewvI=-c8N*tDOTORH4ta$PCBMTz<cRq}I<=8H=
zb{Z2$?3|Uo?kXbdKNozSt(L1&yW$d;`3iAAr<?xO(ud}h*KO4Lu_xW`b@!BQ-!At)
zO!3p6qx^qjna6Rxi`tBZYkuv$bwmAN-;$2<FU)bL4!)c-sk@;1#|e#(8gCuM?@afQ
zn>w*;*7?H%Yht`M2wgsXKxfHq-Sw?r9yWZr?=!Y?zFiv88f5WS!ms&h!_`CEW-Od^
z)Xl8vUSdne5hXdnD36JUlGPM^mztHXJmvc?;NsL9NBmmmzmr;DvHiR2hD*m){7XcX
zxy_D!@nc!bne^J_huTDo`_qjpgn45(2Onp=aa1h&2E(qp%BMSKu(xyl^w}qI{kqZP
zo<e2T<*rIe?2~sq65E&=^;}|RMk~*f24;<f=M1Y?aC;r$=@E`iG5@LUyJ$zq%B)ho
zT+Y|0CVaTMWG$C|`zG(0|3yZdwG?%oyI8XWFWp*r>{e)0-K58Eg%y_rrQSGM8L;yC
zBrtQ$XkbxG$j$avVSFU@?CU&nkw;?IJJr3^j8!)?YWh69^JW@{39GQ=@e{MPyedLY
zFS~VNA>%BqS4<`HkK3nIWp1<=iAi59;x+H4k7kmPU|(&!af6niyK&l)^+i8!vsI<F
zi7;$CI!|}o^RgAvc08URsuiL%r~Lg<|K;b8CHsRl?5dWCmcBYC8|669Na2>5kR~Tf
z<*`YD@+rzzn;-1)xOUuV%9mq$Wxp<m?|H<g)yg9x>QVDxiF>P1u$z@_#uU{B4V@ja
zN-6SPOd<+T4o_H{dW)w^aPIaervk$N?{b{R6SUGNwe!W_yGK;ix{q~DS!ow?-=lE%
zW#7w>w#Ob*OuZ|2??B0hze+ESE4=UhJ`u1qily~@T^NVtvlM|tUF;LN6~8#<HM)9<
zFX55*_o&`+xvg)(b_Gep%E@LynT;xaiqo!4k#q{TS`vNr$!yi~RZ&ylL>W!cjIhiN
zEVvqVgY(&zEzNJg{#V{}@sL=>-S?iWS}#RUPuSHa9ebj>MkhDZU*Y!KaEBX5^S&`I
z*!w1N{{MS3fBjqfsoLyHQ&d7w%iDSGf0wMTQd_i3rRkoUQ_d=#{|ovuvJO|<IPW=-
zlgGf9cqa4sfyWve=Qd1oe!ODQ*$%cxzdU1qEqy#^vD|+RH>)MhJT56-4yV2dDAgKW
zur6BnjEm`uLt5b<mL&@5&u=8ZFw1E+TWx)>;fh1L^Od$6(|9vQ-nXV?99v}I$dU51
zM1B1iCrvvJzYXWo)}4F5OK-=aG%ghf&Nu7dyz9TM)^ty4F)#Z8Ug3f__X-$<68MxK
zf6!9Sl`wE$pn77#8UFce7?<ZT{NTIO#<lZPZts_;`4dvuBVRc&<eAQD*_UxxvB<gS
zQdY*5HKO0LKm9oR@CJMS6|W?=1+o_W$xYonX3LB<)|%ZpV&>3ufn%*%UgKPcoab7-
zN*qZ_7D-A@Ywg{h6|Q-8)hF|`Myg|$#x<9t3I*KP^H{IPET7EceEq+JYIqj!l#-XB
z5-MA~G}RN*B&V-yt2=$XA?KDC%c(Q#_=FqYJn`q#T`YUj>HV&^W=DAMyeicAq*Qh0
z?456iR`3Prt2x|b3Q}U)r102)acURi#_koHUNc^Qadg6?PYO=4L2Wv(Qs+b^$MN|n
z@%4qe`0w*d_ByuWtb>5}FYiY!$z~7SADvRSkoahHqru$o!sC#DdW~bOw-1QjV75Jz
zX>(<<h)&Y;rcENRQvBMK><iR5*!h%Z>s2h7P_A(4qfC+3-PPJ_CI#*>(B3WZx6a3H
zpI*+C1?~?f@~uqL+33o7W)t7(pjp#8b^fi>+L6lt@RR&uH>F9Z4BrX8VVSs6M_K3p
z@@Kk|MjFeyJT^URW#}~yaNYBT?M%b61P#HB66d>D2z}btb?%q<T9rN#bN<8?$;=*&
z3bz`#|E2q92qwH@3CMZq$HDZRk9A{P{)7`}9a~(bd|zlLySAt;(eTRrU2x`FhVk{D
zCQYUmsqE~ZT}M@QmTp&B8oWlxa;~uYxqfb|1lOh)hGGp0C!H9t1X-U9l6~B-zuZTc
z-SGX7ApK)S0@FQ|m>71-J^jCGsqS;3oH^QeXK}{N%F$TxX4ksY`piYA&V2a$WbMi$
zew;m>T84{_BtPyx!R^4+SG4M!r;@6wh@b5PPbJfHhKx%RPc*TtWK!{LwyZfU_GY1o
zxxbY}Gtd7!EDi4*)a7+vsvH+dICN>+dQC-x?N&!5dJ7H21huUnFD(dm-m=EMRm45e
zE3@YHk^>jAri3hC^!)W10p0l`Z$%h%FZOvYH~F{Vi`V8YUi*vrbQkj}D|0fq_^e*U
zH`#$vz@Yfh+T|-ab}l(?_gUE1u$gzI(hQw;#T5?%1p?jWmE7xD9Ak6Nh@Jn)b<*fi
zhw9E2M}x^CU()tJdSm?Rjdu|9f{+QutCP8}ofegPF1AjkNTYofSGGs1Nr*nXWbab5
zq?0#RUF161YtgyvTEf=JalGnnypa)WgL|GjL@3Fy9{Mp!SJk3{t3)fJJ@oI61)WVV
zBl<&^Py2dup3iNS1lj-h^JRq<qPBjHT0J#tt!ebu;^@Vt8dr>LyW67r^9|OqhJ5^Z
zcV4IN$KrcJLI3xaMm_guIua7c8ECWf?Si&54$BLiK3s^oq8;z&Wv0Z^IAbwSiL=Vz
zHm^Ty7TcEe+&Hcp_(^5wYSEw~OSV^!&H9sWv~d4uNpVV!59K+Q))Bwl$a&qTC0izO
z%RV~Zv`G8v`vto$z5mx|eCJ*K@obB{X5l}6Jp8M8n1j5Ri@Z5xv~tlD?}=h^hfOu6
zh4JbqTd&^gpzS<U&FQ2_vqCS|?pKoxKMDWeQ#R|-1>S`vd?!o1Hg1kFWjUzyu2w;1
z>I2tY7SlQJT(u-U_4z_`HG+M1<T(HL{ba1xXKeQQ@rEOM+Y-2o7W);v*Ypmpn6z>0
z0jIPjM&{Nh%o{fw<t_iU!SGr_(iSyUwTz@lji$*rmK|SYY-RbN<a++UJL~_>dNG5~
zIscwk*=-LczW(-E8vUO*Y+eLLE`1xhSvRV2di3h40*o#zcTK5iX7XlztY6jPDDvU%
z3laOjn^yiSW;n6MT76OGRl#cEg7}|Tt5+(;9b2h&ER(HQXu_&<Q}1Y+1}s)RteJSF
z&G{r#ot;SXd}WO#heiMNh?K26?vc>1GKEK^%b72QTeGY7n}(-S_%b=c2@5u8P4(Tl
zt?2j;hlRT?oj(?!zx0aN!Y{A?^zm%yG>jKo-FE2oe~ZH{w>9$9w)bxEY~9?oXAjS+
zj+LHE9Wq|_>m1seId|K=HvxM6{L3?1Iwm+Nd^ED(EmpbuQRU$+Z_b7Iod5Fg$$3A?
zq&usW1g}r4Eh;ha`@#4;Y|Wib8%i#$xzls@+L^qKDIyZ38do~jrIq(>Sn^nHs<GBp
zFAnjAd5<?NV)<fnIJnY#D$~jhX1THZG{a2G!=~nku2qs^T{)rLx7^O#eEMb{7jc=h
z3vP-e$#^FGUB)YWSnu|x?a@oGMy>Vb%v<HLLi59_&PVG`xjuVj@48c<HBFY6-7D-#
zgfENX0|O17B8Amwcx09T?VFmp`MOQ(O~Xr9Ql@@rj1N*$WMaF@t@;0oxA~728!^|1
zM#*M>S}pYg6OX>Ic<{Yfo6kg!ZN7=~S&p<8#;LWhrt=t0Td-`gY}CQ?PHN^aewyBK
zS&-wXuI*}f$1~}aLs9^<3cGu=SVUtnx3PjsYH<&D<r{ACN5^;M=`jTQu_imLH`R}8
z7WpFR_RFm${D#r497mDQo>3K!D;Ei`;9OchL8R(hs^#Oh)<~aHiT2a$ZZ|Qd7hZGa
zDvn?26T>y}-Yr4h_2wMMf*4P2<CR}*z<RiA_Ju|lso2(KJkL`6&S-x8!1{ejlr5L?
zge_+jPaG&X{F3o}2k!-m#`O^if7*DiWEj8vp*QEKSRUiL+=t>Ziq=1J&Of;I=f6ww
z`FF=uq7~PzY!*A3A?lbtp<roonWuWe(NeMBlLLH$4xKOMVcB$i{pM5rm!;Ors2_N}
z!S}X}B=62%*}H*!n@uZDPv91KWbs4Sa|35=kyNfQOZwwHji}$XQ42*(wu-4ML?0Ek
zIp3$%Q_gisbKkoek6K>dWpiFRIehxh&-Uy=Z&IpeG`2Z3r)|CJmg^bj+LOII-?{&n
zd5TEijI<Wt%xmx28ie_le`0#KTCdqeM{hqvOWoAP>zH(`ZXFagOJtt+)7YiNT13xh
znX>gaOB<Id1qRFHikw|?t6d)Mlx3Oe-dGYcRq}Yvp(AV7Eo60Ct$QP@<b`(POdZz8
z%a2a={=YDUX<m+=%%k^l(fT0~`g`6uc+Y4Ftzt-%S(-iP*m>*Ow><W2O6JO`P+E80
zr*V$XdU4*#2eLLUby*^&X*b95(fZDI8O>AQe3mk}ZecfAuI3STbQdeL;rDBT#n)oi
z`QBA&j?Dk=x<gujTIeFx!#i^-jb@1_JhjQU-FRQ@Tdl=mk@iD+F$IbYT**5Ye_Z9_
zr=e<I_fXH{VV(QKKD%|^8fnRUy292LUN7LC<lcEz<Is<a(w90v|FoE<-ORsz<f);{
z@9CF6TseDAXEjsB@~xAXCfI2<pZ@t;Sb5#H%)gA0O*s*NA~y3**;%~Ix#r)N_D3te
z2`@hy^SXHFVbgzQ)p7qDYYn=h_<Ut*7Wz%SeS8%U|K2lt4rdKk`q`X1U@Nfm+pbqe
zr_2<}@1}WgT5|SMirK0b<6SB9Zk#cHa^2$H^ZXk>E$1aCzh=yrJ3RN6!hw^I9gd}#
zJvuzs<%;N!>y`qs2jd_2y_prvy}np;pLBIe(|W(xQPU3F-kE-I>YHnVTb`G0IjL6_
zxOZiml!Lj&TeF1I3t!!qbD15o$!JSg%fu$8$ZDm$7^O5D4Ux}#W>0B87;13f%a!9^
z*TtXePMxiDW2(-JlFo}pF`ZRg)=t-9N_anEYu2J)T}vm;Kk&yWXzr8C0e-Vr#ogba
zFx7!8uJ&Bhvpj8OYbAeuHU722p?!Q)&;RF5@2@*3=49OO>};&2>J|0y!hg&8xsAdC
z-!4erl{IsF6u(H*i&JM(@MeqE;hda*A1#Vlp3A(!*Z#TxzkSAEP78Idw99ClewDe%
zPTB7o<C1N$@y91hYSquEO|hFfuP17w=>L$pd{bA=4YM?uc4z8}2Q%E~Yt6H@o@g(-
zyZ49pw5506`CB_P*;Z*B`BpKJ$$R-09^XH6H>u3kZRUOD^GJxz*n85wr^|o8l&+6X
z{H*dI-sK9f$3kxV#bO(O?7ee9Y#Z~}FN<wt+7B@-I@7=y>&0Wx@gZTRP^y$w&5a4%
zl6o>4c{dGGo2E1}Pw@y?#E>#+0<&9@jb>x=sj2GGWg-6!7?ycYH%i_WQncyWfeDJL
z9yc#c;&#^)VxM5xRkG~T<fYThWN&<V$q;0`GN|-s=i-o+k?du%6TZH_v{830pTaD`
zB~f#kC!Y!N4twFblX=@5Sq%}5fKANic`^o#YVO;WybpyK7;!vebd)I0sWECuKFKV^
z;;|s*kVs$1xl>!M1x)Tvo72Z<ayIZm>{8$LepgRTUmraqFM0VA&z+l^ScE3!$ldNr
z{%|8Xl2Jb4&nu0-*%p09fA6SjhAnQ}zU69HFN;x1%*|a#YaJx@nCf>1)f)V52+CWs
z=$Q59(-rMcUY}p?QOMQKDWl@^WBmaZMme(yEPGO?Iiyalbz#f!zRL1n(OvMJ%0r(V
zPP2d}Nj-}NX-8^3WU}5jdL(svl^zuAQY)Gm*ueRwY<3Wrhk%2Z^1ZLO1CI2XF^GEm
zOkCV!#;X#UCl)qCc+y18f=C7*Ed|zoH9ya%S&x>srRG@hR;GA*hD~!2;`F%ark%@E
z@XTk;hKoVc>lCM|x#yg-oKRTG=b}D4weZU`PxrZ*Qkp4SlDch~uNXUMZ{3p2ttrg$
z^px(ph{!hS=ei~Vjunb3SDg4KUJT$|(4-|;CzG=<a@9=U4;S4HzKSjL-+omrjm6VL
zfK_r$#>AGjdp$3BX-i*y_S7M|OhnoJc-_rQn}R=?EUxqPJ=xh=%XcL`Th}!7f3D`L
z(nYhjU!7;77x~o4by|peXX!16IRZzO{1r46Cbv2FNnQz%Inz2lR-|x?xVz&qu^SVQ
zgk|I(lF8Iai)M}~n>1r)?2(K6eumAwGW%hh^?{wnQ+HpTazxH&TARPxLxp2n>RmFL
zRa#qR+*h!EPcJo2ynd`~s+sG(h)<pA@fRNZ9yury^I^JRf~=5)aHH`jwWDG|5-)@$
zm*f;=U)%5I+od1ma<*$h5X*1zlUmnP^fzZ|pU}!@l<K|Gt9@SOX^`CR?vHn6>+ekJ
z5|6f=HYLVGaY5tmvmy=$FQ2=6>A;btzH=_S+WGACH4sWOmF#QQFn?TC-80K!?!Ijz
ze$hsM|0k}Q&Ej1(M}4o)ozpqnHmOB!sQ<BiZg8Y+gYCA0EgEG)Ie~1NsSR8col~X+
zWVtR+n)2$3^vt@2o7ddCvPvlBhH?%|U{7Md{w$XToZ81cg^p@$-4NVu8?AHPPy1^`
z?&&Mn_dmVlE_AG7_RCXByVo4Pklnx{v%q~xg5eJxK5wSXge)7ksadhvwi^OZv1}}z
zWK?jdQ9JnhHiKzXvpheTE>Sqewv2819F2rxFTpNu|4r*z>M}T)1Lr)Nc4>x!!#Y)~
zH9q|2PnJ&XNMTx9;L5q(L0s<!YbWzdpL8|_=S@AwRbSkc-W5}PQ0U;1ICk%8-a;Fk
z4L+%bUrUsUFL^%c!T*3(Uc>C^B^QnunYXU(Vg4ao*5W$lchB;S-#7akSWHx^-llXL
zcr{$P(Wc@Uv_f^i)KPn$sVtfs4#|I&Xco&!+*fwRNqFmx(D$o?`~7Yui7Y9Q7Z9{m
z)Qi=qUsE-UCB||=c$SN?&a93=mP-rDTF>?Dd6Rly;<<U<-U3aXSq<(LpBK%LxTNt<
zXd)B)0jKFFR6UNVa7Qbot1P|Ytv)4D$erPX!Tf?JvHNtEPko|hHFZUB<T|SxTH<Zn
zqIr+_9DV5V_SA{4#S0AQ>&{+2rzLpz?TL9h?h7x!Tk&Xj0AE62%H(+#`K*(E7oLpU
zkk!4m=4e-#iy~v-gsDMmzOPnkc>ljV`oa+g2~)1HE6w4_5+1e@f~~q&nnjizSaS8)
z2F~b^(^VRYOSsZa4uoIw4*PfVgos(g{KuwCeDWk3Wamg}M(=P8uzJ*OTy8GR%NJsE
zca=in7Oiz(7pTwBED9HInIyWa##Jlp>51@bzZA<J=B``+kmbv>qm$2GSQ>Lx$ZKm*
z*z{UeK^N~GiV4RKiJf>e!J>B}&#8GqZMr#%?xozlL8~|{G@V>mpSS7Ew&4&JIC;6-
zIP%Tfe@TAAt-ZeDd$>X$iO4+OaxhKZ_N44mhGX#`*;0d@pLvz-p5)<X#046UbZeB=
z&0w@#D{v}lYSC(4lT~6}vrc)tC-(bhFz!72KePR3%|@A^l1-wMR9O>5rzqV2CAMM{
zkHEt(n_D~2q^IX)91WiOX}2|xL(rt6=71R$B4?JB9SBv*=6-K<X4NI%gMVgqK3kIJ
zQ>9sG!l@=0>#%v|*`<?|xedZPO<4R2cDWq!-{c)~%AoagpXw|YHSwTTTTgQboiEyW
zv2#YivSY?~*YRe^c+V-hBXnzLpQz2Y<&K5zYIixk_UyCnRJ*;9Ra54AR^sYh5k0RM
zPY*?|+p^7SivsMvz4a1an{j-h+0ES>CMH(NI$jH^x@>Ls<VrxPN-Im&t-i1r)#5VS
zx16_Q9GLb6O?!X$fS8sO(-Xr|g~?JU-pibw=)QZ&O6HBt|Fv~=Cuq!DW-?dw;tr;q
zDTRg8OhO-Co@Zk%%4-#1D*Z-h<#kub3+GgHbdN=|bkCf5N-wq3P{!fFs+5RLJ6Mg>
zF9tAZOg?)eg(Fq>;vA->SC#u#RA{*ToX~K8$0qKHVlMiRe2dxcB-bx93^XZ?Jagj0
zoZ{ry{yNErOFSkni*oqY7j>hl;`PIBMw#p4CncRljH0^Vt<!(7B%%H1pNW^e)Et+V
zO*B_?Ox`wgPT2XNh`ZO#Hnys`{G5<?an`kKzuh?#Tsl83U0>T?+->Dn^>l*Tx!Q$Q
z-C-`<^zx=et_?5C)smbsT{O<1tEFg)_^DP`hjgXxH6c^Z98>iOUZ9xd|Nqj%X)3#K
zsCY40WPf>^AQ^bwe17z~(4ZMwot|9HQXaEnlGSS-tt`{KcsVqqO(tZ!%Yw)$l6yLH
zzDHd-EXEZpQfjQ%=)G;Rc@)Reir0tuqBFM3?(2xZbNxD3^@hNYf-7FCm6&QA{m<`d
z{l)WOfW?c-lUZ(;jJGTlp8LYcaO*{_?MG#E3gbPkw?5S>?_8qq``azLis4?N>+>B^
zpLx<=dU@^C`p9Z}=&YB3ZPZe&s<N98$`@YsDsY`1{_fD`b;4UrF0D7)`q63neHQ8D
zz~|mg+eDVRPEwtn?BqLhnNei8v0p(X*UUSb61OJtwyCa(neprORq4yiYSjg0s*Xh&
z3ICVfEi9XpB(t$q&BRn7O)RseDO&2dMz|X53>DT{5BNJT@kK8yXWk(5v^e&k16T3^
zt{Gc2riti03Dw!SFfQb<w34`~n{a%}1vZNtTwhhIe?M(zo>=5?qR4n*{2p_GUrXb+
zEvS~+SnaYSe&QotrV3RVlLqDpUF&eA$cN^okIglLbeTT3CVWh)QMT}SqTsEP>!lcB
za$K<Pqhd~@@Ou{}rD?*B2CPdSa9wlcI(xKK{h&+e3Ug-0wj<^C%MYst3#pxU@lQK$
z)YG9Z|0s{CRA^VT+oX)pj7hSU&#i?jg1RRa3L2PcFD?AKAaHN9%zk0U(kbarwpvw|
zH3b$|iZK6Iee@)8>x#N{hosxp1Q#5Vv$oJ&lkUD%y(e^q$h0!{iI1uSQk2{s*`$wT
z>K*KfIa;$&tVUu>4Ub6H*$k=18TwnMi3SR0CsovL`qq0SB5Nt5{=MZTUL``0&4qQI
z^d(;`eD=Iv=3xZqhun}YoQ((j>O6{rEUXTxcN}?W_#i^{_BJ!!i3S}XtJ6gCyaO99
z8!~L#>>yF1cJN`|YEywv*Q&ZUu_h|9nwm1KjNlGYWMwpGTk=3Wsv%GEx;VG9kzl&?
zMWLSklZzZJdYm(&<u5fCH@Em#a0gbD@TC<qXHIF+n4oLcE79C4?<8*TE}-(svSzb)
zbE=5hiiGE^5pyU0SN2W!{a@^>GAU=;M#aP=DLxO4r4zip%msAP`br9O*9jReUFyqK
zsr^$!#@Nw)*))xu?`qa=osZHS!(2rx%B89!+__5A))t6xCYfqG8p?(06o?6L4HRCo
zRp+*v@XVDg)r!>{-Nkm4%gQkJIBR4s|5>CyaaK=Q&xsw=RGDf;JEt6;UVN-sH>AV6
z<;DcPi{bJr;g33M4{&B*Pw!iDtyXNI^8ZhY%#TanddRc{PESr$+3qA-T`3i0Auwy3
zh`Cd!`$@y*nf3Bb6HL>LPCXG8ikwqrIk7OjW=V>_PO|#llbxk58nZvde9|avUO1oc
zbK<{`A-0wbt`ZAm8Yf$8{%>Kon#IvI*}hUpb9ua@gr@Pq_&|^1prjhlne%Q`Ofgq4
z(G6rvyCA|OxQOdn>r7SAno{w^iYYctV$PBl%npp+52qO`vb6uu(R-*O{jlMBpi{x*
zg(^bR1)rKEiEC%5rxaZAUT55S;$X77n9_%B9c~-jtDmQAyTIqJDEvvpC6}#C*HP!V
zlhMCo;SN!SYbV7SR*CGNu|VZe)%37g8`!IT3o}k-xSh4|oBd4P)wB1KM(_C(y+=&M
zo_2`p9@JygoMUe#c3d)I|3>}&+me_s%xMp;i`umG2%D1Ck(~FMvXh@G%wIV7?W0=n
zOp#iP+J}-+ZB}BBSJwSHu640>MW)LCiEF013RqYlDlF#~P3e8K^yf}LyQ}s4ti*1s
zFbX+jhA1u24_~nE!-7**)l-kPoUal(uAzD96<1zCM(&0+WeNi6C)(C`id@d9S){cp
zxG4Ah_5_(<-to_q6fOy9Xw_G(a&URRn5Qg4At~Z&i4apm$5#&(p$Wm6;d7Tb$KCk3
z%CtRl_R0wz70Jm0OS70{_;=3R^0>^)P(7zTmnC#Y;iJ4y#c4aFbb5cLy;hkl@p<W+
zDUwSS798qWW|gT~RxoQ%dS-f3rpf_U->)IA6`32Bui3SHbM6I!>27TQ9httdwl<1N
z2Yp&zW~IBaGFO=?>*@|=v+&5rO5#a|u8mg0o5KEY*v_Ue!RVYUxRTj$Wz9n!o1@}K
z%*t{mq#FyW^D0HgZxwkO6(+rFoqWnF<Ae2Hi$vHq39D^%y%)BDWyieeQ2%QW)r2}V
zgjcL?j8=3gRhtyHK&7as?1Hg-MP^I@(>K%Qmv=SiCS-04V6&8nOufLRxm};viD_z)
zV2Z)QRgG;KFZ#{Q1<yo?T=dw!{k8WD){>f!%R?rKzA%iiRnLy9Ry|z3yyU97p33?^
zpR$c#%}KUq_?4w3u9TFJlFt|Ic5a!-$D=#;+}xo(xg*8hy&`?btS;THzdBw<`x`%8
zvN6##$HD)M#nKw}{DYVI(u?LBbl4mIRJ`(7u;}u#smqqxy#Bw5^$yqaBu%C_o2NN3
z1z%tb5-kx;)O0;8;2O-fF5#e&Ijc&6TyOx}{fRx#eq^QyTFq&bdX<q_(Yz|<<Lv!6
z_sU<Y{WZ}f<zd@iua$pRiUdDjTJ9z@`O&tLS&2uEIDRv1-NhE()KwpR$=B>waLLVF
z7H=_u--`T_6OSEq3&_qA=X9@nrnbYY;l{_+x0kIFQf<%w;pHy4bK2_#*1OkCzRmVE
zHM7jY&^ef`+<{Fd;CRdi#?m+0%n#x{RF6wvVA~$R)-Xr#TOiv5MiIx(q%U0ZeacD;
z9&V|Un*GG<B<t#R&wrVheinJF7#6W4VUgOYvuz2p4lBthuvSeybxu3bTk-#1C#QaX
zX4N+*r^|G%-nL+x<};zsQafHA(fPV5QPsGtbI}s-g()?ebFY0glF}}JR-7eqI!5N9
z+WX}*<TP|Lg$rX<=Eb{2#h<cQP!yTJa*b8C=CmR;@i&_|)DBA6FscM_sV<P0{c}*&
zfKmL8dx`^7^nrt}kJ*Am&-4213QywBT_7KFab@6b!?2Yl?`ADGH9tA$a>8murG%?B
zMVF@}-3UAK^N<3QzME)HU+}SHFO_bSP2VbntX#^x1ZT|1aG8}T-uJ6|#?+JTw{#bq
z%~27$w6QQbz^CVMq8O)zz{QVI>l1fWdx(CXsd2LPaN}>UpiP^<7iyhhmT=ZSUU4Tx
zH1R)o)tO!E9GHYJgmk~jjw#@-dUHH~&WWlyCvpq4W~8;PydD34YTLA>Nw(#Q7k;cQ
zKf9&JxivIm(WNJ0o5iLoFh#C;7#dV4depgXUZROZQ`Cx5p)~E%&`lAEN%@<;PvGs4
z&1O@ny|Fh^@XTti=5kN#wCOV9o+*0D<r0<Zxk^V@-z{tIwr&#IFTAt8$7$)@8R<K&
zo|=%`tx%+suxtC3&8&ad6j}a0UMI7~wEJLiKzu_D_qD8+-~gr#%a6YiJfBx^_lM(V
zwu@{r9}e8Q#kN1<oPqeE*?TS?-QII0!)EQa+4a5VcRaG6^D12mH<2q6GR;xi6Kgqf
zuCuBl<6h~MG^78lTQbC#C%LMxSy*H!A1rb4`g6mNu}XeMmkcLLpFCp!YD%<c!;C#C
z%T$F{f7qoSW#jMm_GrJG#uL|e-Hl%2ehF-w7?)NHo$z21`fwo6fbp*3wdx%9Uxp`A
zH|Uo+u(hR$Wb?Yl6tMSbia+pWJGyI@Cij$khfdW$-p94>!Qtl@)Qm1mo@#CQ<uk$b
zM&xd}qr0Rc-*vpLQa#13k}25Fn&cO@U3mINLG8Pq(wXX4QeE~3HqUQdc;v?Quoe}y
zkM&+fisoX<nMvxd&u)EZ56F3>wrpbA!#4k_q>U}F6PhLk7BMZ|yIJFN$b3$w^q;?*
zigya%Kl7~HL|N%>nt~#$+3Eiog*7IOuecg=xMTkwY%D!~&-Ox0LzTCke^TB8t<y!n
zQZ*B7uJ^?V*7d&mT)T&N@0KZNZq_J@KfT0Rx7je*Gu+&F&!<>1m%#gaiS{WUZ>!lp
z6iL#blDKk;?u-(X`3H+U8a_(rYtL5w^v3FvNZsQ@T8j;)YWl5>og<Mcwo!1yzmQaq
z+D9o14i_soulg0UX^E76c~5G<`K!AaoxM-2v=+!~Xioi5w7PRL-zh<UtCka13ohRh
zv1B^B;dg(I##0Wpi)ZJ}R{d0Z@y0#o9mV3YO6w-SeLO2Pq)=pUr`WNU^)au6CUd<y
zQdqikYJYg*=`D8}ayJMXEM#$bq+lH1>h$oxx#e6d>sYs=Je@Pv?TgX34!^i$!(YW|
zd>TQkw$*H(xN1ekyni=V?yJ<2b1Ay%7qCd{$VK~rpx2oq7p_VjFr7JhPso9wP_E^R
zGK&h|*lz#y{@0<yNA7h`ep^?!T}t-nt#i9wCThtDO2}?|mhUff^NK{~Y4K}<I%U^{
z<b+g|7F{S>sI>T?;{zo*#V@)&k9}7iIr3_YwCMHADS=0ucc+_eKU4eCKX20)VGlOV
zdq)n;vJYUkWjC_W^HmBtyz3ChCt3dm=FVa~uUhm9H=pp>8FFatw68t|Z-xA?tPQc7
z@Nib>^qikwAD8D9m<N}BTyXSj+3X`FzeQqsYMOt1Ud<%H%<^Arrb591rbbR4u8<8E
z79MWnXJ_%4(BiYYbt})cB-Mb&4Po0nas_9&96F%Mw0(l%iJ(W`GYnI?bdoomX7J;l
z>~ba3;H1+$er>f0MS%&kXL=f*dc#q~AU(mC-6NING}Ajs$8b~5WYNX8joWTDimbfw
z{N!pOPQQmnu8aqDg&uhQuw;09h$UkxPrxDvM#*R)=H*jgcrB1^F3VAL)45y_VZH9o
z6e)*KiqmJr7;xW-xtZ|%oRK5<nVDCXIvXwJ)L)}HA@#YDrT5mpl~!Ah__1lOe|Gow
zcW1v`^&NM!Kdd{mi}}p27zu}j2`y@#mbE)Yl2@0dERl)N___VyEOYiJVjlnNY+koM
z+i%=*GT8B+%DwC7+dDUEuYSq4wol18>*1HBY!0hLgR&$~D@JqZbna_WNzh)uXvzeI
zLkr@9H1t_jCiHJOn6m5e)Myr_i&M>Z6dYb4z`84$;Ru`bMZRf4Nd;<WJU;PlzF1JR
zLhV*Y^a~R?c8j-LB)jK$nqFsnk+Yq#EW6oLgZI?=<E>@ZA61u#Ica%+Sa!Hax2a2b
z#R9{(Q5?IjDlOt*;9WWUl)_4uS1YE5nVI<cD1W}PS&7q(aiwS&!`m#a)jU!m`&Voc
zxe(B~qb68s1=rrLo;6IOK_2UKRxWpDK5cYzN&5$r49iXtkEG@u9FJPoafbQksd0)v
zc%b_KFq83{YlmHIayA}R>ORAJl8xaSub$e2MZJ8gNoT~&Jsv(dc7747ud(XNtAXz4
z7KH|#S4!^ojc)e(Y`FZ7j8^!9mos$y&GT;m?c?TKHe<p1uRg7uyHq-o9$T3|HF;sh
zq+5M;!ITMmx=!ikd9T0ArR#G-J@{nD<-7MiUU~%mXF0Xwg3jz+c~cm~y*5m3Onv01
z#BuUO0H@Kmg2=?lpJy%F*zO?n;Ba)z&j8nOf!`PUjP58b?lpUtv6M$|!m|TMj67yE
zv={}IoOsSywC(ZfuWwp7Z{_Un<IJ43Lt>Naruapda=KnhTklt2%5puRkX0*nliEwM
zuy<;#i)L>Mny~5jk^ef2Wf(Faa%6cNVtnSRZ5gyah@qjYBWA_Q{-}o?=ZsuTU#wO6
zY-6qVk&R>SwhQjLb8N3nz1-;^G%e|9pojb^{gigAX&rCt|F3<ua*M{*F41W>K4dXe
z^ZdCk7Um)PIdqF(__@6*Y8~DRngMgm7w-1c^J8K)6No%|RE7H*Z`x@Gu?(Ro##zip
z9ygkpPj0)RkYQw|z-;)N_tl1QvDl5ScOxxROjINnOk%Q_^=?nEvBB=HgD*a>&DC{D
zTvhmh;asZYhnCgmue_v>?lD<kX{r;#BN=gW(PmaFPL&jG83*fI7r4T69;F<cD9CKQ
zV{vY4dch)X|CcHi^DeA(S2BKb_J6fn%1XYvc84Dmr|hgSHxw1~Fwm4~;C!ikXo-%x
zbI^`Sjs?%UjMk(uoR7LMrWD-9xy8{arNH^}r59X6K8-bbGnhD4j9E|Gs-$n%V3kaH
zGE?PT3m2PY$~J)r;b{t!a*~zLJZGv@nQ<k>D!bUDU*xFX9!`(g=v6{$>#uB|{=~)j
zQ&&eE+l2}9V=~liJRKN*u`G8#Sfr>Z**SBEvbe*uj}zl|ck(tKnq6~rdFi!Ck+*WJ
zyHB3v3*eu4<+V(bsEUx9cHl;@Wp56yEK^z0rG9Z<)TB?7pQyNp+~yYju6ChIRCZbV
zHVtRl=!<MWkMh@*Yc&7uabq%4$YuJj9j0mb-^DH5X0_VOnhxC}wiEX&j=SA3O4%x)
zaUix$D0gl(XDZLJgC$RUo4Etex+&i}_~689{iI8G^fz|sFG}f**E(}|f0Sd}-iV8i
z`ihg7B~P%*dmWi>d}vXu&!VmU=YppB1zuQM_vMJIEtiABY$Z1>p~;d$hW?ck4|H0I
z%(N~zSZ%3%nB~p|E<PUxqfYJS#7h-hIH!pAX6=dkm3MKUZb@KP$4N<>W-Y<@B0`Je
zf<15i(OPo0!PmsR=Rs`!XUl04JzhJkxsrb_GuM9TTQI#wCv%%=tEPmWQg89SXDp9o
zx*}8@0=}GZdtt)5CFF#P>f2t`!WEa7w5TyIzkRYPz--n3ge=$2ZPlGiV;*gsW>;M9
z+tc~}-<60?!S$_joSxbjlv>+8Om}UI?d9x#EY6wpg>PFyl%&|=(677)s&iM&?>usv
zRp>x>Z>k2fo66=X@ppA1nRc>n<|#VRdgOG&chv`s%oc9zZyKzB@F*&AtE(gLr(?^e
zPU-OWaFmX<TGAx6mq+b<@(rclPr8DUa`9DG3@=*BxW!G<xYs%>80}@zt5Mad%9wO;
z&L@NA)^j^c{5z+gl@dH0zO(P>siX6I7ILrTR}AQMS8F}@E<#Z<h#@zKWs%Vl?@Xb9
z#gk4<*phs4NpBg)dbvOr;W?H?$1^Xk?p+dk<E%%YsvXaz89R?odU@eLo3Z6jnW<{O
zli2K43WNl&YdF7{u2TIiaqXN#te)DUR$m46FSf2rRhlDq{hh(Ki=P$>)O$+UIf^R_
zpXLu?QGK#xQMdOI*DC>EFTY$iAxiD?lF7}JcdzVC+3aU$n(7mgJz-Lp@O7h!6Iu*}
zC8Bu`h`DJy+U<TdNt-uq_o23vHHRNfnAiK_BHu<HakDL~*~w{P*Ur6Xvc7yjTQ{Il
zr%z*&z{>;Lhg*y!V{bTXX(!IvaWl5{_6LVmCj?9+RgT;~<Js`%?K0u%jJs@a-s!sN
zdqdC1qQqG2gzu5b(%#E??HrDNSXlX8ev_kWh0f80iOVy(HCb+X$k$24?Km#9(z7@H
zf6CsOJ{e}hP3tmTEq7n?(%6^0S5M{nhT4;7jg{YDwJFIdY1^LKEB9V&Yi{0qv1t$9
zOxwQu>XPpH51rzL7BD=IVP)X@v3Q{`@1&?4rd=*)1fpx~-h0N+day!ys+jMi<>mX9
z#K_LHHkv6E^XGtP#b%jxCR;a6Sr@QM?7EFW<SoZZQTdlG4*V$yFgrabI89^S`DHd|
zZ@r!oBl*P8II-uy#^clftMaprFY2F>lajj9Kilnqr-3i0^vmlf)|zhq`EWNI_ZOvB
zla?h#=Uoa`->zegZ!Hn3t^UBG(&_R`%S5fG_kHbS@7YT<RL$O$EZE@^$rkxrHj{aI
zugoOfjy=tu=l**$nK{(zU%uYGm-FJ8=zYg;CP;tZe{pqOHp|8KTPn>-^Y*`z?ki~*
zSTt)v&#V)iPr8lU6%Oo?Jg|A@niEY{vjkM_i@4Xkwq01Ddb5yYm4sc;jlStWI9Vn5
zp8cL>b6U9nRd0gS#;KFKc1Ej}+_W~ox!dC7mXO^$+!prrb*LSB(6vf|^`YRrum;oq
z38vEmI`TG|2R`JCl~$}ycKY@}hI8WLzZd1TdN|WB?`eIp=bW(F(G9Eyi>6h~-Z1aB
z7}F)qpEr7LD=4%!PQPVrFrlhr<4&=So>CT%jkzCdFyuJfys_`noVJgHfz@Gh8)p;C
zj3)6SjxWL;Wf`oJS%=yK7+(DM+C1$9r=iH)OC5&uj63dF9F(-|Tiq=1aI*R9WX&&^
zRl_p6mTi!56`48V2Fu6G!a9$1-rZ~nnz8yw#Qqrzq-L&`n!9sqv(!Xereoz?=5~9!
zeld$bE8cP-N#S3y@r=kFPqH^UE6(O#CCg&c;or1t<?R(KC-1e(=H_HLvZlD>f>GD%
z4>Kle%=|9Ev9n;N`)cDBZ6W!f=>=UKReRQIDRGG~b-uIik~yNlsJQm3$B`S1ogSXt
z4}V%e$&!t@J^#%Dxue#q&yzbOtM*K~FmK)=JxvEOmdcJ}5z99|64l$nI!Whb_9|B2
zm8=a5CkN+vx(D*w8*Z3#gQYb{s8!4E;%c+T{~nxXA`AZiTAy)q!f$Sa|G#%Gn%#Nh
z^x?%!C$23vn9;!NyJbVi>m^AooIwok`%g@NK7->Lqs7#sDO0cTonY?Tx}(=cOWI^M
zzv3*f$(o|=jr^K7Sf0GxU;J=-_eF=^g<SG)rtj6!h^pFW7_7Q^LIcZ<<pP{NGd?RN
zd^vj5N8tFA1M3(>`T{(sZfTABz*_yGy((aW<>D2q&PX*kh@H7%x%k%^pBm0HJ2*eD
zSirr?`Lx%XOIuXb8J8{=m{8m}H*X0i-=z8WDXQU0o%L^Yw=bOhlJU&bh6#dGI?p#A
zyM0@1(d<<<nq1}^t^1r^N`7u}Z***2p{DusV0zHPcfqDJA8;M`|Jkwa$`mmp0cOSK
zgeaMpyS5fS;%<*{`gw=@$V2X(GbYUH=$Z7!e~Qh?s0$nonVw9ktYs@&s~j%GtzeIr
zV2J<F&MwPPKOvyGAz;_$1y@#VIBRkE-GP(Oo^0FO)S13g>|DgGMQ3bzd3F3U+&Dru
z+)Fs^W8^#YK@0B$rr^w_EDA?nJ{C%uHMzd9HR!|6?Kc=ak91xs==5^&l`GO{f6XJ|
zE9_$|dOG3YKWEno3amOuXS|hZ-RbRGlG&9rOLgLh-p)CCWm~%UI0WZOoe4hGt^2|0
zI8Wya1`#V>k+2zOPNxL0iFR;4z0CN6ovXK#^RCaU6P+`B{ECW=FY#QwX5@S6|HO`q
ziaKq#c#RvJmmP8Tl$vwkq?=!l8^@E`Iv+Q$)^>l`!a4N{->!hUM?ZHzJg}L~+1uyJ
z{w+IrXLtr3XzApxv|jY;c);!@SzAInGrHb=Hcy-@W47V)+ZbN2g_E<QuK98X_g<Ky
z)2R1MQmM<Q<Lo5v;EUI;R`H)}w7WV}w{Y&-kX0SJ8W*Ax*qL1eSiZ74d=0kC4d8lv
znQLk|N9*<W53DOUtn?Kyn5r;&^XqN<9`!OTvRKFHY&%EL&1hBMFW&y511tX6m%r|f
zpSpe8O!4B_?K29dSSB8Md_jhN=2S(^-dR;07hdvY-0696py$pDp6IW=yI$BT3+YP!
z6_>x=KjVMGqM*+QEhKyPzv0aby8KD`Oz9sTnWY?sIUGxMcLpxx=9T2J<-Ovtiv7{b
z*1!|i%XoV2tpbhK+?3aHUXT>HKK9a<fCKTm;#>Ak*}}SY+eH;u#yQd|YtFCUwRyJ0
zE@i)EcTerP!R>x~p_zAg#N4YHD-{fDj=q?{z#wTi<Ah^M)?B$O)<F%jeG@0^-t_Jf
zn5S#$Dm`b)iQe6Bb7V78&8FtCs_Yhy`NdgfD0+3RZgpT^VeQ?URr{_@^)E>6i0f|9
zkYL_0BcO`+O7+xR(-&O7#~W91)#uNFprs-m&a#Iu9MQgUYTNk>o{cxQwMyMM#c=9!
zh^>pQm7m9wccPQt+s>Z;|L>k<dz{NEx2wxWXFh5=vDR;u$HE6oCYwp@__-zarRTi`
z*%n!a7N=jGnZCEn=;$@>#v{&m9L#6$jy!Z}-=A|SUoVD)96c1QIB~)KWv0Sy47zKg
zjT}Tw+BqHs28J%b%JQJJquiHkRd1(oTW8gbfHkHZ_MIo6iQY+C<GCp0+V$*>?p$|-
zVmc&}C(SB|wAOLhdUUgouNGg`ft`krcHQi~R$*{6LQ`%vtDnT~poufjajv=Kvd_iW
z+O2fa!JTd$8l2TNozlJiFH<z-bx&Ni5P9XZY{{$Lt1s-ZcjW5LmQCcotT-orkwxF}
z-4}zqZeC%WYRu~6_|KtTLX)$DN!KE=@_%VSjc8o;+6xb-UT|BN(phl5|3iP~BkgT3
zc0Sr1H~+N#0<DO~y<Af7lFXO%vmZ{JVIXmPLWiHP!{^@4h-Zh3G+RZyT&xt&@kMt`
zoWy^KHS9sa%@#)Ss~++5ltd5Z2sa(s%MiF#L3?YG(Rs!n9gkNEAC|T6^kCAwz?9wA
zDKxLY?n8Tl8HdBtTU~!G*?ujSTD35w?m6G3Sxld=$eOMB&w3!dY@^311=f&r*AJ=q
zHAU|zy{e|y9ICb{r7GU)W?y4rpTCOm#+_42s;~aZ)o6Ql=6cGhl4rdOXY6}?>qgSq
zJEz~8SZ`DCdnEL`WAbTU4}aq+cW*k%$8<f&jQ^5i_U8YAwD@b?Z3ibGDRy$NnL77b
zH-`t4u0*%zH{K$)xT0rw3*{Q(1Ud?qT{nEtpX=%MRF=V>an;(X9Pc&lP8ZCa!H~^n
zw3$UK+u)3Ss;j{@=KGgD_O7nje!R<KM_<HO(@DBFpRW91ZWSqdt*U$Gt+-W}c}#c?
zyPtaIzl8fm(YD@u8|A#ySM2a+|99zSN5ZBVTdS3K9!hhc+|lHslfJ?%UFh1GI**5i
zegR<;xlt0WaT3p$O}oIU$|_M77aZ5IHn-cz=LXl}8Jn2{yDmMtk&=4YDSP>u&dI&z
zigvr&ysVrx#o2F3Ki?*#?fLxXLC1(gPZe}i7HIfzwK;pf>B&^~I`jX8y69K!Z&iM_
ztMA`?k#VnTvTt!$($nK{u@CN9n3(mco@;$S^B9LOXHQ1#kq^lkZ%#%Zl6c&{z<<&q
zf6kkIogdp5+r4Vd2w>ZOtM5fHTX;BIYOCRb_ND9CqZYJBy=ae$Xf0YO^175`){<>=
zRc5q)c*SSh;a^rbyXIrpxg!mC56fFx%<K@ldE#T_ve#w5ugIzQM2Q`}HRb;63dN0U
zxHo&KdiZR7YQ6c&0x>V1RjXIW%KU!(eMMpVKmNI?YAzf5wdcAS)~+n{k=8zRHMgtj
z@;}~dX6a{RTJ6`iFZatTd%-Snoh3>ly)1(DzXWU4f?GWk=VW_wI>|RUeeVhWHgDE{
zm1A2{N)G(v+mzP1nMdgUzjysy=FgP#{+!L-G&kb&&ds+ALnHeF_v~EoQtfDB7W0);
zx4A-?x-JWP)_*Dp*17f~J#hXcrH7M!Po#>U@teMFL2~S}Z+quXtDEBQoOxR&Y&WaO
z$C^%gpM^8;O`r8F|8(KeT?dvPo$T@IRf1ue%ZuITS5FXl*>T}sYF2Um^`-CHS!Ee4
z)t?7SFqg`<hF&PBYWU76%3SuLy(*raC6+bNAs{S*b$Z9!vmYj}o!akYU*a5J(H>Cp
zvFgXVwvQdtKmP6#>x!><xNL^#)85-3R%rHLj7Ssu?C@-tU)Sz=ZUtP|w@dU1)|TcA
z&wg)u>0C^X^8b^&79F)R%DX(%)cyCmQ%eJmw(pS8YhC(n+k!pa2?^Djtro>v_T1eI
zZV9F3p1pD=ZbAFSd1plZF07Vq2#sKUe6*Nfyft(|RA9pQI*n?+>n-c}E5a7ESAF>G
zAU;X_SyNr;kLGqg&4hH(_k7zj(ihHQVk&xGc5;8`GNIT{xp%6stk%d2zdF&KH!a}s
z!+6yL%MTx0oiO{RCTq9^C+E*2Tr5WpUHM)1EpN+_{ke=u-wr*R@j@wuOG?-I{puH|
z7EO2jwr|Pu1wYNU{artKa#}m5*UjlEfB)t*ZT*+#VmGt@u}x>2h2*mPTW{ZOVO#s;
zQrz#*8Lt>`6bGIt=A2$!*6=;<|B5=!x9nj*zHg6jXO(5Wy6$^mLU>igg_UWo<<np6
z^e>#nQd8!)kyFCLXyflqeq0SVj1nF+=t{E{C`JS*I_k-=S+(paSm4kmrqB1pB_P?Y
zo71pNF~UvdB;!)H3>(V;)q{TGma|k&2r2u{Gz-6WW@WMZ`FX)=8i4{Po=pr+>OPt$
z91~ph<b~B#nhg}yjW|6gHhOkFxYih3-IOYkaNx#9X5k5{8D@@5oIz|GxU|dl&WDM)
zYyPpCaF$_r?w)N*p|2OM4GZE_TUfm@_x6#Ji)s@mr1;o2{bl>|Ieh&zwXiR*dZmt@
zlegXTFQW5X`Oae7s99O5aZ{_Vtg8CwcJ%+(?OtZDb})w>UF+(#{(x`L_C3B<g%6_J
zg72BhUx;}yZOJvwxNyG!%aEhRyM?D%6(yW#Y~Q<h`Ok@pJ{t_~E4x@O`17aDu>7Cw
zjQRur{(jf}6)vQiI>-H3Ma%?4R`DPPRYxW+4TIJuuDpY;tsh;NUaoqb8Im9wTWcX#
zFSb%-foq<@l!zpTm?siRoGOYFVrw-`lKsv2u5f8vZ2A<Qz+5tQew@R<P!-MGHz&B_
z6DBr>YMX6(xJ36G&jmimw;b$?U4?iKsQSm1t!!aQ%Dd>MDeNL9>BwbpM9tZYXMzh$
z)Rjl)8OrXx<j+j*xyGMW`c3Fo+M<MQ&#jhp8gl8r`QKycrC(=tOUiaH&x6xxU#<j2
zE}hOBz-=AWYj{B|I_J@yQemEkS1deE+qG6PXOzcQiRT7PY~|NHa#UnTllc8L`n(}*
z?#fd?25vmaSCFvb?ZV?i-g-MjE_<H)7qxb4oaNDjBGNh03xn@B{nl0YOPxBkCMtHe
zo-cz6Lj#M^DrtSkGS+9(6L>Z@-e#ydnmRpy35(eb*Vc)QHyQHI-MS{W?Z&6>vZAFc
zU2XptSWU=cE}3@IYUS0<T^h4^8>9;sf4R(`v4&wg*YdAx_5>eYv%t-3<poF4*pdxq
z-?LV6-QD9@bR#!>V$s#WmJ5ep$3{=AJz19$y1~m>Wzilh&tAFOFaI@LOsmB_d%w6w
zht>v3<^@D^b*XK=D0=f3@5xIIvU1#0FEfjDTx^p#mi}PFhlx+*w_Z}KF1yOS!6M)x
zvw}mwBWA{l4I(KIPE4&U4CK4MS2=3l<EIk3p22nhj7*ZBSec%5)2{avRh16Amh8Qp
z`NpMl{aHdR&mB0DTduh#dl|S&esjpR_C0i1)0HKNqi4DPOx_DC=56V4>Yn>%L(=WE
zE1R8PI*2~+oVRpskI*ZL*(TwuN@KK+D>%n(TXo^^hP>sM9l6f(n>ubYI->3S&G<>_
zf9cAU%)sFDu59LOXa8|my!dOrtJsT4GcT|-1<zjKbf$Ic>WB-O5f@K{U3e<Wwd{X)
ztV2cjfj6cC!c7Hw#!9T(+l;!_Jz$Nm{m8qQ#aKXFaEU=#Wxy_(FYDqz<g4dR>Jbar
zvA1qblNj^Oo@Hzoj;I>7RBv8&CN1KJlWvZqkWc10ktq{ILTUx;m;<M}E-;?((~zat
z%Q<;n&<ug4tc4AlF8)qirmbty3CL0XIjio|rQ@@|v@0}zx$U}S)6`w1%dMq6m(;8{
zv~|*zw;s*Hf*VyN7EGMcq2O_XQ%LB-$rleaivBH3?8_0E!SGhyD@OAYJLjTPwVEoP
zH81D*Eplm-y?9o8f;7X_tdJQNxt$8ht7HwYtNR#gHu8K^a8AlOzBHZdveMDwZo##x
zu00PpjsM@;a%1kU$dy%3x_K5$g*v*awiZ`;R$OdZ=2RlK|IGpaFfmIGY4(QIvkrW$
z^HNNRn6XsAug68}xg@7@<|4rvG6oAdViq{@bqMzwlrZr7Jdn}unz%kkb@8O^rLIqp
zu44%~5_eT3)7>!0`Tm{_`Z`l&UMPth9I|<-tG6rm(7n=b#g{F<yC(ODe7JNZyNz{)
zL+A-%&Y%$Mz?EHV4En^DIJ3Pd3FPu(IK{9`YgywBJ`Jm4(eoNf#_Pp;Gk)bPcDb;4
zu7%NuiCMl6uj(Zo-*U^hLBg`r-oE8P_dh{pB`rZ`qs}#3RJ~P>8wG^0Y3uBgG>c?%
zeWU!<XvcwnsWu70|2rO-nKLL{P%-Qd_Smhmp;Nu}=oGG(G0*>0P4rRHTt6#WNYkg?
zD)v@u7t`TMv1wL5v#Qh?g%yteS;DYvm6))yP>0isnuRt^4?O)(#J)3|T`l=6^Ssi{
zkP`tjU%PzhU9U7-qFLo!r(epWzKX5N%4<TtPHwb$6Z%QU!S?0$IaSyE*YP;6w#^Q@
zZoBWQ(o@Ol{{<>0bzFK9A;c2q&c^g$(z7F5IM$s^*4*h{xs}^(w@sqoz9juQVOQd{
zW!I)Au2c?Y;#sz~f_qL{RM1^shNH1vJ!(3O1+HeO9dkYEanvYyW~fVf*WD#TXSQ`8
zEmUH+44S$s<<iRqtCkh_q)y?R{QvBYIn4p?4LxtxCvEwBY*t5lfB#=T+1JA1?w5{c
zSIv37KA_WbLrAUzo79AG4v|G>{Va=jh8Q~=_$=PcyI|8vDSp+TIv&1EQHpBsQd0IQ
z-7>D{+p=-V#6{<0l-+}OtY7YXDL!MR@EVnu20^c$CS+agi&uM=#`SZpvFIYd?gxH)
z=T!8jPKnl#zMLehA>DlHOG<Y4OHJeRp}x9dhHa%BYt&BGeso>-`bgA<B%WC}n$)KF
zuIj4Te&PYI*GIR##zwQkPredeTz&Dzy<dD!6u)mdu|u=s{=_HY3oq>I@cwgBVAr=N
z2e{{%@70!ixP8HgL*gsdB7W#jpPY71Tlr{af*bq)-7~U06<7Kx9x4b=(fx5qY1g_|
zjfjJNQ5TMxnZ0ATowH)r=>=yVElG4uxAL2=7Iw@}vwV^H#JyK^GZJ}%8Z3(>oRS_K
zI@#?tAxnI-a_g6c=5;>LKE<q1h~Ffqy`fN*KX%)k$?vw9HrOoOv|dg}-ttrWfrF=1
z=II{_pSeI|LaCov!s8gW&C?ZgXB^S4`)4*`-SeCMk<;f~vFa2&!>3Z{(mC;s%k})W
zBdOYEt75d{RtoI@mFBtFxT1CGrznMuiSC>BChA6QWW7C4YJ2V}&O=A4r#0UBw*8P#
z(~&)q8uedh`OlohoAc{n-`zDyvRhub>)$YB@M%47_T|Ckm#Tg9p8S7wGI&McHZhZh
zt&gScZv{%^O<XL{sF9H<9Q9$XB<H59pQlY%%R1w^^JipfQ^|hy*M$c5{Lx(>OPc<?
zWmD*6(Re@YiuFI+LM#6l^=ZE^x5mEH3SIJ)Uu=I)M0@YGuo(dd?@n+MY17SdWJ<Yr
z@I?IQtDYirk7;Eb(%Rr`&(USSM@~C~E$a)@=@PE59ZWe_oWGp8!Lo9W@J%Lb7B*>}
z*&-Q^A|3}nsWysSd2lB8V4uL5Ge4(^uk>H@roXnrcYTca`a5SC9?p;XB7Rj>@$%ox
zeik=ou->T8n46V3GwDZY^Vy?ut?uon4{tjzbX{>Hspe9pL6o1tJiiYd4J>l;A+!G9
zIoR5FCdl8UGsLFzabj!2jpo-eo^N9AUAWrvBKC;DTR(r5nKK;bPrY+%?gcsP4#)W$
z4sj?=T>OG_amxAEv2#?j+!nJ)ySdCMi4->zjQ$?MB7E?S=Ako3uO3$CYq3?hEs??a
zIaFRDF8ab*)>TIWT?-flk9l8;JrZ2tSoinT@4we$f8IA#T3o(?n<;})^M=#YD=ck4
zXUBw0ZjC(h;Nx+xy<+<XT)1PCI}{%rym?5?rJ+-I4)Y?ZDiayU1+Lb|P8O6tTKAPX
z{X$djf@2wP*laV{zO1>&vCZw+9k$#VXJSIGf4ksfwp1bL?n(QC$wv*IIDNZRntSl6
zr)cO^CA*WS{!fi$sa<+p=+)G#E4!O_&K9nU^whh{(5v!0ccJ(pS^pMUzaAIOC+#A~
z<{B1V@tM&v=c>wsH&1F8obcGP#5G5ZX@j@AsVLt^LH`#i=I6Mxe3K6x4U$T7dHJDz
z(iX7;3_UrCp35cLx^MKb2t1m_ljO(YAF*ihhc63CL?ZKM%$xT{&RU26=*B}feTSq1
z`u9yZCiQ^r>x>xv1g}!jCw>YkeiG8oB5H{STweuS@?WwDT<ZGPDDTp8-!tg0`#RB0
zH;y-`9QnCa#qWWOp_Ig*%4hW!0e&HgMseLu507X5ohjNT5Pc`e|HjNeOV0*x;_bcT
zy;mlIZ)=n81SggrMgKX<|NU8>TWdA&a|S1eOwRh^lzxZPmVrI(2j{7ju0UQDk%ei|
zn`BB&r_cRy*uG}D?Uq9;p3K?isJ5m?n#1?zcg1c=twjGFvv*hedXy|<*gI7$>xg*L
z-8vRk{{>-OzF~e3BE2^qNoG9oP=oKDCJPe-W3!h7&%-p|A5%lTmb$WZE@AEH+p{p|
z*22V;<S8j#uS=w+ZDBUPxM=GKKBJ{CW^CwWQfZvM!L{euia9-p)YF8QSF{D%c6G-d
zOEg>J_hZF(0T%7Pq<Jqo3vb>k|8-dZN3;H$hjQDTrE{91?_5iYW3jo|VlC)WwQb_X
zk4f%aiWOJVosYISwLVKf6mPg{rOU>b|9_`GpM5B^YooWw10nZ=4|Aekut)~-3c7f)
zc9^TlB^a>CZ)++jxVP}pHRG+W({)}=l=Wc{X=QSF)hy9!?R2bQ;z7<=Ez6`r-?t|D
zUzuTM8}8S#YHo_#q*t9^Gum`>j`eN0Q0A+`<HkPw0-y5>pP(x-j(>Vq96c?l=X+^q
z`Za|c(R;5}W%xQxh)+1!vi?-$tw!HEi`VjJkKE*$aobbM?aY(-Jx}Cc9`#9j?Z>mi
zXPryKfmk7?gAWB%dpsHzK5fa3I+wF;%F{V~jA`edpWzO1;*@1N^D-s*VC}86GpAp=
z9PBJR`D;Z+fDF%}u*|NdGkS!MHmkT5hDC36bz3QM`+wrE=5Gf5C9Ikpn&FeRRu(oc
zpKPVM`q;Vl#G_@oO>8w+!*_NVherD*+;`2$w0?N`;=`Pjw%cBtR=O5Ai3hX>ON4kQ
z@UqNJo1?Ti>r(5bb6x+=a`x_cJ^R<u8;RO`<M=F;Qu}@^6Jc4t=$8P0W9O1C-6d8J
z4lQ(DH8bL!$1yP<AEUli>vY4{z2vNDT5f-%IW58~Q9=4ghp>*4JI}d<E(7<iYr0Bo
zj(5uOR<G)ws`)O}FLtZWYrhNbKL3<r0@wN#$U4qwd=|$t&ozv(@kz&$-WxmbHayI%
zVo`0{#_elzE+vBV#hS1;Y;Rl}!Wb`gd%9{oh<(SIyCTT!<m5Ax{)gUTmEYI5eCt7-
zw$A1)i+hf(=+<K3kBr{h(K+YB$$qhyF#<0qh)EaSl$vzw)(Nwha@+de>{w-Cdz-;T
zRACi|!AGrw&#w3-L{EFyGbyvfNn-BwX;-(FFgh?!xis-*#;3E_N?cq-7tdCD%Xs{y
zQB~N}+$l*tA#J*<4@$G$UY>aUW76I^%3Y$%51n-_%rbc4dQj{}oMs$LfN1E7e^a`b
zzPh8fa@97!g@?|p{<U_E+3^Fn=5#+WNP1Clgz2(X-8?^&$zcnhdOpifJ;nMsiA5;9
zw{BUxfsur>%a!TNMB8Flcm}+_Rih^MJEihV`tx-xVzW6V>a-hN7H9Wm`^QbW5qspn
znx@~wJGWa}U9a4GZJNfHS?c=aV#kA6zDXw(bs}9hygNPdO!kx?+N;^3pT3<S=^K2e
zEA^n+C%-o#Cbpkm>~z%@Njv_@<$GJsiCfMmxB1yF$hqHl%7sraVoQF>0@WEFZVJ!(
z_>x6m@UHu{L!a?#j?m?ni*H;0T{!Z@&DU$j{OdQ5q<wo+-=k@u{GrKES8{q}#0Kx4
zk}jFU3tb(It3(t+;<6QvCqCN{5F-+rA9c@R#|bw5h-4ALqZO+h_pLRnOrLb^?cJz;
z{RB>J4)&}sx?&#_c(~S`<@D#vmb2Jh<hijkbY<6Dn=`+61lCAwsSA}Aj*qoJxZF$W
z^aP8$ihHB~Cxou9m)sna@wy|Uuu~@YcVme}a|ui87uO9OX|jB2xk2isLjRwxV~#HN
zmc3<aKg+1UW2x!AeL|cGHBycxQW=(fu^g>297(<_R)`#N{kG0{RbBp%s!hvZddT(}
zxnF9T%=jq0(e>86<Qt0?NARgzsLxc6{`%+Hk?j|ScQi^r4r&w;SKP>QtwQEPI7>l>
z)Q60^FNTl5%DkN4;<z};BbnL$X|UUdXAe#kyouh<bS#X6XJMFJ=zjy>iVapxaVb4a
zcg?o0-CJzK&ZgPJB)R8-*og%^zdnbShiWn}dd?_rZ*O#{<MSTP23MhtPs<|zMX~4=
zGwLl~Cdhul!PLa<%Jcsh-MS?=9NFN~Eyg`*W>(<@md=@an?=j!dL<MFJet|5_##r_
zz^;8;7XNIk_fUOgnEL-J%l=pEbbcB-GZanP(ikh#>GhQTz^-$Xd(s2$-#ObdFW_*I
zL8$2@TR(N@l^>ri{F%piI_l*lMg2&=AFc^oqx>eTIG;%hIbE1^)!0|KS>yP(t4C~A
zSyn0VUj6xM&4u~b4Y;p52Gr_Z$bbH|GyZhk{?>z$$??f8GSQ3o@O1VvOlu7*Fc&|@
z#5CD&%eJJNuM&2CYaVUpoxdsh#shn9DQl+S!bQ2^^`>3js&n>ilh~T?`cOZwS<ksy
zla+DPU)6`PmrZ6KOmXdhcBfN&tK7Z+g{NJlm(1NUX=h!>YXwIsY2lWpcd}O`=6K)D
zmTH^i=U}Y$)uS>wQqIov)``T9oqZkOmc7YodbcH?IqAsc{-~9;oteKsN9<l7nqB3u
z^22Y%r>zP;f%cz5mv&4qnb)$JKj_9Ao5K6wT_v4jIxbu?i)Ao!c~Lg=(XU3!5a;#L
zvCco&o9CwIGH!}J9sm9QOS=sw!rQd(8n)ch4F7W?DSK_ZmC*|SPZ3%3ma<J>nftr%
zxl79Df=R^%Rzk}sO}E~A^^;qWKznkUqHJ>e>Ej*--fQ0<S^NAwqu_^HhH10szx2y#
z_4!vVG5hQmmZjUbE!!+%?mG3S!-bz|vV03YBkue9{eSbVeQ&^vMYh$F$$tAH`JSFp
z^k1;O>i9wh`Pu&8)ihONzFcZq@|tDkTbCyOkCO_m&vPW5`Ob3M`bTm!_i6F1E4Fhk
zInD2OKydww{!0gb3tw`%8?o$C;WEz|^BFk}M3{eHsTO&-!8Lg51cpm2T#;+9?ceb(
zX^&}e#Fe7OdDm|Cr@VC#=sD=}>+J5m&whS5HQPGH+st=iME5z{tIbEJ6iYWfEOYwv
z_tJ6s+G!<vPv0kdy4E;t`QOI+SM_!6q$Bq`wnzmlK0eO!^qlOXjmw-aSkB$79Jt5z
z)_a%5ThkWHO1(V&uKadMmMv%H;}XS*HV3|N9QgCvp(i6@$)B^8?C16y{ht#4bH{1c
zmMbh@xb{z8-#J^sXKh(^OR;qq<KC~inGsWNx2a58bK5#{rn_Dzhrac@3%jIeysrIW
z+;aGQtuglz<6Cv6?^rx5+c)OhCbBQ^Y*cpJJH<65PWns5hveVSCR#f)xHK^`bBf6<
zIFN9tiHTh)WW|Mt4xBuqK|K;56<xddjf=kg5M1J<8O`P=>0y*|YDx^pCLNEbijqyK
zA=kD<1}r;1hdHC@QRW3z_xUFBzn(-a6Lwsj>Q!rErSS5=a_4lu8LK{Nu58TRBy`1U
zLGQJo#JzkH+S`0jv3RQFNL}bthzX0%m&#fCA@9%@#pir)c9-g{-cs$nN%88lb=MAb
z{Lj6)I&?#*?pjY9FNKbeMQLYQb-hE39&GkHBOom%Q@de{%Y+$A7nt3!+<LH=C0+ea
z#jnRl&uwSFCO3h>>`kMvkI<a}a}9Sr88$U}2H_0ve*e{cODb)bUepU>w^;AJ;pKr3
z_fJkYub12F$ndJ3KjOc%Lz70zC%zxAuO~cUjJN;vJpRM}{(n**7Vdtv*jq8^o%90H
zAgML1dd)eROGO(sx_O;1C4D}-<&44M+|BI{I{b=!yi$gHMb0~OC{LLZ{84S%q{*IY
zZ`L$#w4Pn|%2RQEnUsdZ%q_nzu9<&E`IN`3NlueIjb|lYTJ?hM*1KbkEhZeEAJ06_
z-_t)S!n>Q{wblO#3PzG=x~DRqb?I5adXUL`gAr?TYwv`ND_w^gqc~SjEz4PXd>W%d
z%Z7uD8riz4BJ4p<Om5D<HB266Rc%?|QRp&7!P~lJ;`SC5R^BUpRjWON9_3lR+tD~9
zxli?3$k~YslL8(ca+O)RThLl9-#25=vRync!)01!r$tP?XUx&aGQopk$2FnN-HTc8
zZpn=Lw~TdzudSp`v54W@&DU15i_ZF>*XFZ_b4S|t2=l{VOm4loTjnhE;*LP~-tFri
zXX(CNGGWi|@5i6~J6g^#Gf&k>wPotB6XE7*uSylnYg)4uj2EpswNvNyr5!8egpC$2
zNH{3{;?OstUq+p}*ILf(*>h{c|1gKvWmo&pF~0ISxgpo{WT%MPyCNMYw(Om&pPXOv
zg6HyeyR48V%@a48xP#0JUIfd?29>znkXdA6Au2548(1xTNF(REoz}VqoISxOcRr3N
z+P$FmV%x{~*4dYhQkrK;MJP^mRg?IjRLN_z<f}lzdUF-GRjJQdK2Nrcp7H$CHRZn_
z4rfPC;8?xhu~1kkoQu`#-jz2SZU3wdoTjsQ>$=NItF|wzI;}S6+mxPb<qaWATc#FF
zS|FVe|1!(`L|*f{uFeJ86HDGbQnzfXSkNs$sjmFpw$GiX?nW-k^4cq|*P-@eF;D$H
z{iO=Z8Pl?Ng+>3qv;2d;(Q9wv{kz{;7o4-ofA-%)?XbY^)%8u8uE|VaXE<(6Vs4jm
zxOAb|MAfH(dsi^WQkTWEs#A4?L}G5O2w7y>RB<6i_{^28n8gj<Do+}Vy*q9RDLFRF
zv|Ad;G^~Eaw&&T0wQA=cvK(jIAgh<7?`3y0bfVre(Nn!V`AyRIgl7Jja9H$-EoX;6
z(>8}L&Ykaz*f;}H60RND_vESeZ*Px<n_eie_i7sjHQX(~WXf13s2!Kqoh+Q|ENxZA
zQZ{WS!_ux`8NDAD`}Ms71Ll0TZq(hFcQSeQ$9>-g<#_d~SWd=OR{WUo^y!L8Q*};!
z{Lpb+KeNeTrt#$1Gbzg~T)DgZnK{qIESoe(RZYG6*G&f|*Z+zfxeZEEBAn@V5t_?F
zPc_NRl$DjNn797sv#hy+2lC#mP@JuyU~=Kh854E|M&Y>2Ez6`BxQn-MYur(wdi%|F
z=O*<%vr_!sTDEw73_3D@ox`NgN2jH2vXzWpac0~8lsWKllIZ*?u52E^3uByi&CqRH
zHpzi8_P&Ji@>A`1-uKM!J|pHbefHTo2ByM=%96`Av}!g?E;(E@_lAX%K+mGICDOaD
z?|vbAsETp<ES*^gj6@BBXI~3plAdu$-jPe_*_6q7=M_)9H^@|(@F%;Ye2dWgch^EZ
zzpZWOI(hFaPm^C)slIjS-c`QuG_S6$+N7%`x%5wvr9I1D&SD;|46&zQj12$R9jUKW
zI-L08%Vwr+%>ip~G%-n9SYOjAV49KDs~7%6a8sg2wwz>6j?UsU0kLc?GB2f8_+)HL
ztm2qGbM65qwF{BwixMnjG#Z^O7C0sG>TG#&bLw5SYf+86u5JCY=Jfp+9=BUcLRSX7
z@fCUeSZCqc)G3OYN5#1h*?n1Xasivx2@SSS*S=jn-t(1HM3;R>t7#jH%k{Hw-M8!(
zn0jX8%z3vumZ&KSKG6BR$9U^9U-b=*oKv<j#~k20Arp3=(LyKb`&mJ8fv?ARGm6Hy
ziriCKY-Lk+OVjFA311`krU$Qb!W9)(M}+94iE+A5vX;E2a`M2HeQjzbJ5oOt%sU_y
zGrQ-~{~c$yb|%)iEu6zq!0>HD>-;Y!$*=nYHgQ@?IT^S*t`gj4Br6hjl_$w$HRI-4
zk~tnrjIy1w(}J1i{mseZG`iI^<AL>QLtT%ZKPSwSYH4G3nB9Hl<dq2ynh8w2UH)Ey
zY-YQ6dgRotb+9cuGNpjC^T4Uk8;+z$*T|kO_$|XAutbCL{<0lW6SyZN1$3@$PW!0e
zBUbqIR*U)xmk_6`Ycm!p?DpB(a$$pLW^6@zy5P+R5m!DvQdkk&wQI(@>C4Ph^D`|!
z#Qp4=rk=VYUoxg3KJND}H8qK~Q#Z!6$%`FP$<XHWsFzAGtUluAwlq2D^`|MO;y=_q
z7itM`Z<^a3D82gG|0P;&i>C-=3#WK`sdP+U;yZ8a8rB2y4`sBT*LwX{yi$9>HhXs5
zvgMY%<vV@_tN3voO8RlO)V^b7YY%6UgOP^8mGy~JL`rTw{lu_y7S~2KPkrf+$CV^Q
z`n#U7o8EY$<{f!z0qfeS><2z+^D~_0@7(FT!RF1h_~SR?_jMi>eiVOC_IB87lj5lr
zo19m=?AD1gS|oNgR^`^Mo7Hj?mpkox^}Ib|Z<)W!iK-c_44OHI#Co;EmLyI|>U+Yv
zYp(NRT{A|dhXK8c*X|q;I}vauaB@e)Vnrjzia6gHQNkbMSHFCap8Hzp&MuzV1t-^O
zAA9}!%Ptm^jhnb*@=|B>x7?Un?)YEQ=fRJ2`S*BIE%t2uZ_)aw!bFiF&C~MnTp?$#
z%)`v>1zr^vSC_ZVKB#!I_FeIbtDM1imBrU}{JfwZ)4Xca{fyc#?V(9!Y`n`JaNYEb
zc=0&NW@Do$_xz9Zb8hyvPd|R4(RSiO_0?Mx>i0aJrkeTFc<a`+4}YATwO8%b@-vKl
z8|FOSdu-D`UZc;`)l;6O@%_(R$K$e|fq#PLsRo`N-Yx$+ESS7YT+|tZJkCVDSN!#E
zHiy!gDnGA=bv6-6>RBx!CT#~}oD$1aI)x5J-}CHzlevxQI4ftGSwn-uq}HxAD{SWY
zFYH^t<A>5zIj`tfi~ZIey>#k`<Czqv97k;-2Ilbp%`;c9DK64k8nCLwgLk)=)*cf-
z?ynk;`JAqC^nbW_`a@ps7N`D?HEXZ4ExLYbf<lY0q8k6_D;rog_}nn-|73QaS4RJf
zPXo{N$9tC)Fzr)g5K&-YTOh#u(Pc_P-lW{dnymeDfe{Hz?P=$g)dW@AI&SUpT`iPk
zZDr_{!@qqAb4Skl+lyAMb~5~ybGYW-qB4&)YbJHCb?Q`@#Q)&ep`BdE6nGZ@ic(Bx
zSs}`>RPJ6<z^#S63hVTGcv_F-2c1cNV&u%JcPL1zbvncOC!0F|<?wEa>*Etml4e(8
zNL<Hz*ibPi?*@;-CjkN3oeey78Qfa$B<FwhW>R2eS|IRxqV2Z-c?x`jkNuX;=4xV&
z;?LUU>=x4ER&Y_v#Ldm(;Q=--?Nf_1_prX^Qqugh<`s{U$ojmL3~i;;&D&?N-eXgE
z)8ghY_qb5#6#G?mRf*#ZkGxa6@a~eECg=1IGfkM8AHR#caentMgU3zD9};}89htyv
zmhk0^w(cA4{d+!M4{G`p)u-9Hme>0LpRB%?@_AO5(^}bUujOSaaR>_XJ`y<VT;lUu
z(Q1|Tkxdag6FfxJz6DJbvYa5C->+P8-sVZs-4!hw3sUZW+Ek--K(|S)wyxk3hqGAX
z#rtY&6>dDS`F2D<&rq>sVqMZn1G(g5YzyR6Hp_K={5$QU&9`$U7EUK8`IxTyZ#etV
zhRq>-*B=#Zi8;XYE#H*g!PNH9|09Ajy9H&G&zqI<%JNU(k=h`;Tw#NhE7Q}7a=agH
znHE@d2P^ies(Wj3usF@0WuWTtD%a1c-0Z@ckn?kv_<yT-;qDf|+Ih&jvrl=!JGEI!
z9cG(!Z*mk`ctl=nvU}B}y6~JzwSfIKt(SEZd{}~6rMn!Jx>QcPa60zxVOsvjV)JyB
zre!roJeg;&_3&DsGvL2ELBq_Sm$QkNal?NTe=SRYe%JF_{QTP1*&jq41a=B>&(z>N
z^XZF(VHi(Ryv|4K6jq_1ZMsK|Z8vwVI_7(450?sujz(Gj>P^p;<aJp6({xoDME*^V
zDEU9Zo_F$s8%K)HElCUMd9dY0{esu^4TTCk9TN4;jV!CV0zFjioSzsSb(nkRiP(Z=
z0q0CkPL$5fGkg3+dT*iG*^3MpC-PjI7^Zcuv8h<@wIIVQL5?>Q6*3GId8R5e<SJ~@
z3$$G%pe|r_k~#E%k0M8=!d$J@Tx@sNCAaq^De`_&{L}pMLr>K)<~tH@s(;i%XTM8o
zn-Y<8;etz%w>yu)%~yq80!395b`(twpJ@`!<?5@XRXb^dXkJdluGep`aPx4V|L1W)
zOEzTx8*Qh;rFSP?G7a3U`E-Mms)^R_t=xS4E^pUwUh-j9i?O`uBklc0eHu-H9}5Gs
z1>)|TDsFkF-T!~7Sx|`%Z<7wU)ACu$6Z}?Jtq4v2aA}3zR^7FyRyKUmZFfyn?NM8?
z<j|Z~A-qpLCV$bNdqug4yFTLLnjU}sw8EY^wIfbVzYcjA{$Npgs3Z|=rhD)FyCYkI
z76m*pGXBSxWa2(~ub1(boXMKs`)!ienX)Tn94_4Av%s|1?9-<W44*&9h<<2SI${3X
zL{3<NgTd7DZs>M4Cri;qv6qvxxvvJ+E4N-r<B~X`Q~f?v<PcNhQ7h|X@oa2~E1bjm
zf^0gNl41_!ecqNAd?9c9E0NcSLNDsNUI>n3?KKln2=mQ}Sm)~ba<j_EvthHREMz&g
zrY|7iO_I}^)y5pF9e(=F@%#UTJ5~2oLlIY)-@bb){N-z}f4acOyQow?iI+R+liK-@
z5#r|7=Xv&h)$$H^N$(P<FSj^vs(5MY?x3|-gBE4;PPdsfT_LDMC1*u^km}8V^(qC&
z<=gWjn$p$vL_?fEsHuH(ljf;5f4Q<mB-mL=#IsP?n=5tOgQUaZX-+Te{5-!+%hKk1
z%<)sj_r-Jnw5b6atGH!ZuU2(RC7x9eJh%6azVu8ZzTiVoQzrYpERj~1cy>zRi<7>2
zZ?3HOYsUK<7ik=h**(c(<2P#oQ^il53_%ugwvVzuJ`KF-#AWay$KI(t=0scei@Xjl
z+jY7JMLubiUGO+@c~aV;7afa2Et{&m{%?I*_q5}Ar_g*2ri4;%sqjKeC(o~MwL>PQ
zs+?UY;?<ok+M{Y7BB>Ja%PGQeyXXmqWfd=trkIv2)>+>0DN@`Z>Z`ZY>PH{fG&{w9
zZfrlx`&@u;x2XANgD(fBa$7m@Y%LNHk8FAE6njxLJNn1vADodMKX&&VRi5O)HP2kd
z?v(54Nh&|s+;W*zPaSa0I-2lsyWW-U?(a3c6{a0BdgQ$*fk$n+Xl~`2C%VOx{EK1>
z9ltHn&j|Ftr1ZefNlogA!LOEcKaTik^h~?8Q1sS1(?tb)kA0Qe_|fEWplPJAnWd5$
z^9ED%^`B>mnO`n!w$I|4@l0~2Drei~EPr1`fzkl!|DoCI&E~!TsTd@|!N9;FkTC!H
z<W_s9N#SeNm>Mo5SOiH2s%Fhl4q7UE+;e5wvEa$+!Hq|5L<z9o74lxERKOgrwD8p8
zkFAr~nK<j5Blo@Qviim@{XOo$(~BOL-b~TfeU@bKQN^KA<;aQ(rz;gq4=ft*zy5b(
zgV(asDE%v@mTG0aw|Eot%2;ofaaWda-lQ%*Q@e9|=PvInH$)i(6zoLQxt1t!y17^0
z4^`-Ett@0|i*<Q!S6&tQMR7-Yd;2*{x5%lXPc&;p^zP3p-=+0kZ_-^8HIe&oYwnrX
zJbwP;QTh&!<a$4W`p-ukzkS#H^}R^(h^U7n)3yV?+EaM9e{9_Uf1=sejb_=Coo9<N
z@}H0VKheK&veBc90xwnV9yBk#m2CE<Q`x6N@rmU8EoSroNHU6~3m;1m2}ly&roi=Q
zrcl30TW9OS;79W7!?)&XC}u2ja}ZU@nzF0DNp%6sxm&zHLPEVc%BvRn?YY>zolS{p
znkwg^lwBfDo||tJR!`_o+p6=9dD%?91wxOyZFSb}vRI&Hek4rHA$lQ?TEa)2DH=-e
zd3$Fy{;xRLA2s3VP3ME68l3B0IKOIN6)xy|dPw=^r0}x^+pIW`IS6la;C6{QvfV-0
z&5>K_f!s^U1qWA8esw~CH<!EW&1|RU)+xPnzD-rWW~P#s+9s?t_4TY1ir*Bk{GYB@
z-~5Bax*$d<swzN_!THoht{L5t#Z_+F4-TdLPWi31d8T%B?z1n-5$m4K5;C7`W-e~9
z#loWPx}^p0M*(vg!KXId55E>9{HgHIU2r~X!CTAhuaXQC5_rrUxm^wx-Eic|h!mDc
zTHt@<;>Vk6OrmX<)#87qE_7aGb$<F8C#IV9(kgqxD-KLf-(szIdV2Y$6wM>%lV&Lu
zM8-xOsE%V@yX0mhS3yA7)$|O5ZFK@&OTTa4@#^>9+bt>wkNTcDXaDU`kDj=Z+7e!l
z8mH9he9V&OA2&G1I(%GbG|_#^s+V(4TufflyIpxn$Lj44N8KOV^LB{N$QGIO?Dpoc
z%Ab2CCI62-#=IbKilIu>;mJpRmVZ0y8u3){<*uOUB-yv$)!X*mPCQa0-Zpik?jGAD
z>sc<k(_&Ue2&o+Fa$++UPZvBbdprG&vi_UKX)H(mqY{};Fnro1+H4$Q?#Z83FDy56
zsV$dbK)rGW@5BqwcCNYKJV|)LeECQ7uR2>c?<=@Ad0Xlqo2ZDu?R^(QVua<I#BL-q
zm?ZLe?V1<N^iok)#V0~elp*lSvDXvd-qf+RRWm(fw)aedi*C2yu6m_#nUfw@Hd;;6
zx)mV%!y)O_l5^E2MG-jx+-`|`pZ-31vU%xMm;JX}de+@gUh(7H35jmG-_y98V&?o^
zc{;LObVqq6SH<dC3OoMW9!q4o=h}KT&*`<>{`0*{w|h2DTUe*C!0z{rg<>X-+-8kR
z>-N1mE;TuyY1W-t@gXaY85?{(@u}ZQ(w!so!BxThjV&JXJqi5x)F#Ok^vv0<vQOdO
zcDsAiPe!}#eCGR;ZS$#Yw#VPy#W&a3ziOHJKV{dSC2}#0joTcW7ltjrBKgcQqUHWg
zg%|SkUf+2!{qyH=i^`2s3TzxQ8Ul<0O&r2nF((Qb7#bNE7(5Lex|W48NARW`?RzZY
znLNov#&gn>!)**qECCB#4mEIcB=XHLnxN)5TU9Vm;zXk2^m)wNd7ey~pyA}ld6Fk&
zR!HK^Mt@ejRI3Kfl`Vm*o$lT;SncqCF~@F&vokk@%=BZvI4SmY*@Kge9Be8c6I@QI
zMKc>HU9sK}s@qs|TkFs6jRj7<mZnN<cg)u(w}eLuik(<>HMDhG)YeH#j*TrWTw!by
z231=gTwW2pI&Q9ko=a<f?CGi82OdcHMu}yogm;Us)`&7&+p8U|aJ2QPyIj6UAm_Qp
zpas+CY)U*O_Uc+>zsRpS3ob2xz+O?hYVV#`dR!NuhuAZ$arpM(@~r8zRUAc(-adQn
zzjkNnY{vf^4+=<!PrT%oDiPZ%sGa*Ho!@cmmd9S&LVH}JoY^_2JY-<c^|<B9$gn}C
zeO>Ad1_6H8Ct;52N2g6_Wloil7T9#~$D&?VMVFZW?du+9#C0)ve4H3izx7g}`z#N`
zF0tQ*D<-sO&z#o6a*>y1vX8#*71>DZw<;5+I}5s8neld^nV56B-cz}-D9@vup&_O%
z_bQ6JwhKg67=@g2sNQz<IYZXRn<|d#t_ByUL_hd?!=p~KMbK?;>J^F9zE4Z1yG{2s
zlk}Xev?SVT;>}GGn$vhE$UDu<HPgu6)N3&zXTv_NMSWabttPBKrrDys)^gvsM;_aL
zX)O|&-uvkG#vQpvY(gj2{>aW=e{I$4wJN?ZV@ka4T$vzZl6xSx%=qaRwMZx3uc33+
zEL=Tz*Xm>6?|Ho8jbf|%XciJL;-sl{VWldonZW)WS&Nm+`({NI{7+rO;UUn-q_yWt
z=-Vv|A2hDrlI(5EoM6+S`}WGEQ)05uyppysD+sQxJh}1eqBcR#kU$od^V4RnwL7#m
z>@3gM84YU8Ih$J5?AdKki7DOWxqVS}=bH--%XD*Prntph1PE~z9Gl={Q99k!GlA*q
zg;`Ivrlds|%+eOv%{{fYYz9lG>xJM_zLIA*VrIGuM5V1XT^~}l`1|u!mxPY!)P_je
zG#qDA^n3ZRL9}zmqwXlH0_TpOvjdj2d#<`r-_7~uKErfTH!aP%vt~u=hBB?Q+RCS@
zpd=JELGR8h%eP(vg^ip2UW?71u!&Q0wvJ|NyZUD>W2fD>_C+xX#c0oUxTz*0`+qyr
zQoDfC#LIq@g6`k6ocC?Z_1#htZ!YegCaHWiTwL=PS6XG|+}-PEKI4m?I^oD?^*syX
z_=T<%T1RgXbnQ!C^!RRGrn=J!N0tcb$Chi;v^LFjRFh~mlvyX6qA|;H&J%~ouUwUU
z8z(x(s7uUIRB`28$URjhK%O^OKy_8-#QugS+Pq6Q*y(P(H+55sz}mUXyqQ1D3fLni
zqL<RI`%IYQZpSCv&9$vQFC@bw-k2~f>B-HDd+8FgMMFSu8LLf<4R;t*s95Wpm-!mP
z9xMU|(>YWeHQ!2itnet(Jf@<j_+GgqUL@GPDI=wBy0phjp3M`O&Q2^@(luY^Z_$yT
z6DFAM{+~6q!=;6x+rrIgNkil*uT3X?C%IIfFN(TiV|i-HRD(1Y&$xG8;d3sEysZ=s
z^q9Jfd6`^SW@!q??H$JsuTxufZhE$$d{#@Oknxpn+XH(NS>A0DsQcKhe=daMzzHK?
zYbBLg4Vlex8q+pR^5}6`cB#8wYqES#W>4}gM{&)CUmcxnm$WWmNz_P87GO|G<QC8l
zvW}S8(ypUE@2uBCo4H3<-4XOESF`c`$Y<fdeZdMd-PsE|3SLZVNjN!e#mdwGHU_oc
zX_`5`Q)cg&!(sJ4V52WbW|GlKCA%#ON0&CNQd$t!C9vT^uW81a$SYiv7*x)%&hMJk
zBWHeztHp6upzEi{|Kp5;LTrpw4CW_!ED?Dba-Op#q+NPyDhKn1GdGgT9(eO=WH+-|
zB(=y&u4L6*BYv$dXOhy6=Ikw#HV522xixZQQYw2P`)UzIhDG(Ia}#c!*t5%JVS(Yv
zurC}%flV6Q)=K~NnwhnWRr8X_rWvUnA}>uFG)^t==5(Cz;BtD-FONo^uMIJ2H-)b5
z6$wq48+@vs$%Tv2^Ub83AB)Y{6ZrRNysA$zxcPuZ>%!wRUMT^}E7+ZHoSW0(A-s^;
z$2v-|RpPFn8$0W@oUAWmQk|D}t@M1+J3;N#_BA|B4qY#t|DTGR|I36kFDCKT0WFyg
zT&Y!hD>Hn=4qfZ{^kl!SV(<)>|2C6Uiq>m++RP5*+hfKi+#(#f@lo}X6)uYoupjd3
zSy_>=YwCv|lX^H9H#+<)adYhbk;a^OErvm%(Zw?_uu1S$9NWz+#~F$gdzEG=tX#Fz
zsp;K>21VJlsKo9U>%L7s<=7EWGdV1bBXurYkK`0jqo*@wPG4Z5CD)hrOzX3giPD?7
zDNHvyT7F2CaR=RK61x(!&aKZ;BU&T;)vnDqCYfwgQ#i}ls=j;H+NC0jAv@QE>?jJF
zBry3xr0183myIVlb4Mw>y3I6EI_T^qpY4kMNmrh%>wGM|pMTC2#^+0K8u*1-yyz3`
z3SgT$?XIv^xtYNtj-IAcn}$V2%b9-0rmp(p|GznFyKZP+2JhWY6YiTUQnyyFa5?ZN
zV3&pTzQ74hB3mY;&0nj!gm-7Y>nCp8h-;N9(ODkXC747s|1D80ah6`dy>^0wwV=*4
z*-kI(>Px4qQgzz({QYt*7|geQw3EKP*n)GWxMc7t{h4`l6vS0lp3>N}DkyZ7(q|8W
zHF{YkF>kxWd-r7gl~Lim(iVBVbe8hoYobx>9%;u!J==Alt8$Uf6th{6a}LIAJKDsg
zd3_c;Lr|lEX6a?`6C(R+x4yDH?|q}u`25XdB?mcLeMHL2qvdRi4lHy(vMO+4z_0ID
zRy|j_%#q!dsIttr$Ka^F#LKn+Ev6k5-^5YAQf6CWp{Zcp+5a2%oZA!-^T%@Sd$uWi
zw7l{jNJ$)&T6E$4p<SPPC!DQF=?|#=IJazTr8(OMvnBg%SKByNt&`7xIbFwuv)0$I
zIDMO7#d-dJw{}VE+$;>dq2czh<+hwn^~6iRV{4|pxqCw-m_u6iKDXw_eU_cy8F#ek
zMM<<SDtc3p&9+IgWnxc~jOmF<d{<{pJ0ug$C9?8#&ey(folnoL?`7Y5Idv|p>wm|m
zQmfnlh@KG;T{B~O$aItClIQZnI-asS-G6Js@|>BmA$dbF&%rP=M`7U$Ld+L!&Oc7S
zqAU>nz<K9#*%j$QmLD^^%t|}lGV+t<4zbi|s@kPBh^0D2ohtNaWVe{wmi+&}k-_;T
zmh&fz^RO#0Uog;kYPof(!oMIxNoK{Brm=sI<a4zoZ3@fYAy%$*+3|(BlA5u`NyRwR
zBq4!?ak`u14t>q}bksU(h0Y;EF6M)U%nw>E72E!Zx7kk4<+<o_L(O}miBLqd>B_*W
zcZGSv79!b`5=u|xt^C-y=3zz-V^eBNDa*v*0`YQYmfRm3Yiz$2^f1?XO||3*tnG1@
zs0>NwI+(0%Xw@AgDYo2Q`g(kvc-OCq;o9P9%YGD<D6(EmtIwKN->|Jd^kV(S74^J~
zHmMzLJdUDbPr7&xs<=&$;5p&!#$FO^;Ju(j)-_RX(RACaW~sd<@*&Hu=6uzAoYeGx
zcewQjRo0vdQ6kT47Tu7T)!50L*vuYOW;(;S?qal*vVizzN%is;x62NTro;%S1ZXZU
zKc(9L<3`rnWfh(7GIytXNjoX`E|2SX7Z)sum~=eNJG_#?LoCHZAVp$gnudVLO@V|A
zvkfzfy^l{+Ep!Ppu4CHhY`sNF?rZ$|>Ft~wT^hEz#2gUO&ouKi5uMzc$QfAezhqJZ
zW1`YF*NZ<Q6_+Ji7S<J9bSqWNTwY|=w8ikOyF2USDN_{vE~-zxtX??bN63=N3XX0P
zbDdIFZOX`BYU^?`z-xxmr}Ak@VQFoyxyHvQ&RL!^v9xXHF_8@`^0JFVr*D^!{xIQR
zdm2Zh?*Frs8=TE2g)wzm3g+bs<|S+vkn9LLyF==;S;mg#?e{Z$dasGSN*1@Rkh+m#
z@z+W0u7uxTMFpwEW-rF9Euwy=86grI`zBk&20f5E%-YYb7^iw%;NA6{qvFQpsm8XU
zvvodaSr&!A4xJ--F__&_b&g_X)P}TcVr?ar-L}diQ^WPnTQbag-WqEmATo0z7h_vO
z#l)2dr^`!Dw=`!w{j<{QNPKSLB-7A5yTC^C8+mg!q@P`(7jt1ojf9W7s?UKf<|-T2
zn=+ElKB_zRIJk8y>#6NcE>}CeU+S-7O|E=s;qIDzc6vcsilu~Lv&Aw6m(;?`Po`$?
zh-$jpwz{=z#R<3nrGfRv+tZX!Nj*&}-s~Z;X@?kJu$=Twu~SWRZC*{YZ5Ixj(4f4M
zC1?SQ>Q5mq1EHXTNsq6ooUdrSw^_cyLL;a}ESSslCP&D+lVUup<YTp(If@r^B}Q*G
zwSRC$e(MJX*~Nx}kNS)b>)1__(NuRh!mcfBn7jUolR;61)#KQ$D|6T^1>Y}q;*w0C
zvVCrxgjLFk)|uh){3lyywa=a4wV0`~SU7Ux{SE<s&3TzO#J(TRt(WxXS|Pjic(L1*
z$t+<)DF$MXjl@<*u@x%^|31jN<yOX~;{r#%1Q{(gUtl^@J<QE-hS#Z1|B1m?XVv@W
zEXs_punY;EnjNSWl^N2<A@P6a$3^bn{Mv4cX>?7~I<n~d@v8pF#im(nSA4YQP+a!L
zi7Ul}OG;3Ev4X-2rDcg6>z|(xm=iT`)~$(Lis@I4S9#77aG9R_U{Md3;N;kijjKOR
zR$aJZ`ph14%P6kI-m8n8GTp4AcZtQnve%s<pg%+S@>h-jAMK_c6q%va9H`m6wqRqf
zmdqp5**iRy-aiTRbGFelvDhRj{$_g|n`NMFYwPKqtuwl&uNGc;de-7=;#*cfo-6UQ
zKFNY@PL|`7UlZqD&t<%=vWsQ@i!!OZ#nV+D%K09V^S5$kHB+DKur0-~#<9}t%@&!D
ztTLk6TC+cCm|U}H46L2HLVDT`3pHuW?g*j(vobVYmx?RSk}NwZ6*OVFv9rb<tu)RD
zZmUjAiwdYb;LNtTK$PXuI=?S7I2$XAZ{}@I?*6>eYDuKTr>ONytov4I3-E2;WhuCo
z<>2%ePV)pf8g&g-=Oxzsw_>=cyoAwu!>)2Ei;1g}t|xwVo4G|;uE$gEuuJ~D(8k73
zdzuzHF)v^gjpklCb!CajjuTb_De23sW+eYh-n&*oM%pOGEZR$kWwY{Ql{1f)7l)Q#
z5A{DO7V)mNm1`xdL-M*8K|3atuH;snH~m#BQ_o(}>B3AaWNp8%oO4mAKegr(v&A8A
zv8>N(mW}OvV%0v1i7U4s+<83mvc=>_7g<eOx0N$Zx%8<2ztY7WrGd6wzr-C~J;c2V
zbSgR*Sb7Ar7RtVoxOcrK#(nMD1N-}y?6~daY5R0Xo^<*Y@0|%Ag}VZGn!Y&V#C>$7
zA(!Wxxejxryk-l0*V=MXbl#oF?q3gg%@$w9BQqn+sOnx&Pr@SaUydS%QEI=l)n{=Z
zt3Ht-aHiyl%0^~|jqeVLoePWA*~7^s<bMA6#!D&^lE3voKbE(3Q^;7{Tz5sD^J8<C
zX}^4k%ohts9pnA$o%h>vSAL(pKiPpRWkPd^qJfIzspqqH_3ey`+#|OB&(Q^+<~9jh
z-II*nwPPo@;_e?(BAGIicpGQf9Sjui*y_7{QjGQS)yD-UM(*KXEq^~b`2VLw^Crcl
zy)QTII%MOuj5Xze_$0;c6)A_O1X((X?wI&yf%zTrnj;<8+oxKc2|2rcX`Cn5K{Fv|
zorw3UjP8G|_b2PHUfTZqpk=FI^U~c0Mr#Cuc&x&0+EgyCF{tVLXuNC9>X0O_(DgBk
zS1VP8KANZUv7w<jL2g<%hxg1A)pAFEZ<VQ#s;uc1t|(ohxc8T8v6GnMyp<c<=16t+
z)P9&Fe0TT4``!}Iju?surzjT+sC^bNQCh)au;P`os?m4rC1=*%<~}Ur)yx)qb#tkt
z`Kqf6y@S@&q;z>5n6OZ4^WoFaW=b9C30!&6W8rbeLvy4mDnvF#nXO)RCL^hOla+}4
zoYnu;)<`u)Xhd!k=<629(aiMT86rJ7*<-6zzp;ch+l7UZL7YwY?UT!@Cu%7*I(_FU
z){)v7kSpnG>ap*u#>Bl_s-%-=ZxdVdU_ZBF-`1^SNf8$|*KA?cIX$6q=gb{z{&!0y
z-Rb6>X#2M`{mRGg3xauj@0K*O<xhHfmG8D1Tep<2i(QjraKj3*X)CQbcM60!u}ux+
zZ&@R2sk`v^rW1;{ea*Wh=0C0t7v0pbq)bv+agh+C&FL#4m*(sVKD9HoujzE(?@xDQ
zr~BUNz8C!V?3TR(+f#Ra$-2O+c)NMiQJ%!pJ8f?#=iU;?Y)rI~l{hD6UASuVYTaG#
zE|Ox8e1-B$Vub#G+No7PDgRXSjtR@$j<KJ()T|#Ju&d&t=G)-yy>l%d+}4y+KF51q
z>_XJZscXf>yZy4u>JK<`b-Hw}jJ?e5c%m_})7SIBQp;uWcaI946m#RYU*;_&`uBp+
zJr(vf0xTzr_etm-@e#XeC0{5sdC!)-SI0K2vYP*I<(|?ta>^(6Opnx`yWn_A;{;_#
z)u%zXrj(udtz^#;e3j>*!r#K^46jU4=gXQeX3bI(czwfA+~x9&KKF(#%^yBn_T+@J
zOuYAet`yI^(v5qx{;!epO1x&9evRkjwTyXE3h%DHEqqjdS1dbd_4a*ditcW8chujm
zWv+R1b*}J&T3@lo+?w?*43FJ&|J$)Y<}ti=p?qr9ktgl4o#(s`Z|TvTX=ZP$b1%p7
zMYf?k@7*UGc%HDI)wp;3Du3PL0G+m1N^R@syt+O4sZ;cW__G&w>{y>>+t9n^nVOVr
zzU-a@+|xr7MT+wB%I?iLP`5!qU!6Dmc%tgwl!x6mFBZ&cJKQc~KR0tzPmdti=UZJ*
zYA5$@I=f}&o}-3;&i36n@~M5Fq5S+ct2ahJKNqEJcy-lk*)?w;2+Vu`+DeG;UGh&o
zi4AfM-W#75N(Nm~p0sMCu))EH@<+vX>=RST+jFLOwdz8-AJ-y9U+g)2v7GsX*KtFo
zU?%S?%epuiWh|e+Xf%=hm+NlMv+wdQcZYWiCtj5P|3L5ELW%G9I;5=ruj$x(%4xp9
z@;a$@d!g2DJwe;&TTV#r?)Z`%qjx^gJJkBMscYleyS8tN=AYxSIIHu#=a?Kb*P4h|
zLT!>Im*NEVypAsUw2_V3;C|mHh5t!<8Vgs=5oB?Cwe@L1-#W=_r=wK`-!VD<Ipi;P
zjr-|iz8|>@U#yz=Bk8V+y}y}8>6fXu7OM!FF|B{dA($|^{u;O7^|O1$GAA8861U-@
zzWU*P=?xJ{$MU>R2)7<9m&se=)@r1BX&vj;P}Qt{0S3jy0|yv`wn^M^N)yozS}V0_
z73;>SYs0px#VyG&e0-owMwf3z#Kfx&6BRh;sd!G9a<Wa${+9DiuK!ODl&M+rt<e1Z
z%&*Bb{T~b0m1(|VR?Tv*vqCb17W;Ba`JPG>Ik=MfhQOX(A}_s`r1TZZGJIPeu~uVs
z)?K5t2h(OIZ`mNrApU|ym1U|==kLjDV|J$Y7d1w=ElAkQytj%~YsP8+L(Ee1b9!!k
z^gpS-`^VqPpDUg<_FD@+>DignvLr~wMY5r)!C}Gjr5+cP3NBCcP4T#StTx)**~~R^
ztL81Qiq&OnYphciTYvv_+dp+u7W1aX8L#Zy`S$o3SO=`|nZHPf)8<+EnHX6<os}lq
zUpK#<IM0@I|F1tYQ`m3)uPhTj^_TfK^V8G&XPPa@U%*)x)w5t<fR})#A6JZ)QPY1d
z=^UqDr(QhSbet{g!lrX}zt`+i;qZE}TxhaW!U7IA8HH_E8YVR@kl;)bTq-qLz=KPy
z^o${wW{ru)iRLzuO($;D)##e3CJGldM0q40-QNEyaMkX`S&HYEF>SZzQu<QSByi}<
zff%nfUk|w$uP~f0)vEpDj!&@ef@44O)%HxAl*FX@F2umsPS@wuYF<{^1}83I@6Cr?
zY-Kc#G@eo52r~AHoN`3gzsvBL28&R~7C&}{wLxblaA<Y1xaFu?zSLSIv?9z^mO*Vs
zz@twpI?bDcwoF{mmAAmy|B=?NFSicpuZXX2ooYKZA$H|W6YbmU&n*^n$<${GJT+xk
zHs_;+Wqki%{w`a_Zkb|mXY#SRFE9MGxcDYok7M_lKoiD*iT4_VSiY)M3m^FI67W%$
zp;IWbI#=Opl=I7A&Pg0o6&l&fW?WeyCD-t9<{McCjwwMbN{uWL%1^Z($9+m&79D71
zd2>OQZ2p2Ng|7OGQbXtUE3RF&i_3mfQJDUkdDEl(wK(q1o~Dqw@@m<!rBWv0EBBpP
zu+3<vP)$#!i?#Erjl6qRS7ct?#-`<BY$84T*(I)|vk&KMnC-qY-^aIU?GqQhcULE;
zt>t?^`HkC2xoM46lFt@Ly)<2Mh%@QwnFLn$_sdPwown&LVxKYRSyJNEe5cH2C##jr
zTmNR>Ub;(l`|DR1V!MP(|1-OudOdTOL#(!!|EyZg`gEolRxCmeO1aX$Pee4kgcD}x
z-b-H;@o0g^UD*zfEgk2~9=Av@m^Uf$>&mK&MXDMFOO$q9J-uyrNzCb)a{Ki^eXMu$
z{CoEE?)(KSSM8c(b@Ga<Pv#W6`GWcvedRe`t`J_Zm4UUsPSA%*n|0fT3qA%@v<@%1
zW<FW6c2#1|>;<3v79H%G+oj8uc(@?zp!ux8m!UPEJ9S@uG*>TVSG{?-XDV0uj>#fV
z)%SLuV!1fQlEd<Jq?UEh<w={SpXYU{*D8+P#3#J_($+l9SPpUVNk3*D4cXZE`Q+TL
zKdI|E-n=ej)(|}A$FxB>u^@N#$?3QESqS{^7IaBkC?$C8h=fRAr4EzfrCy(9C3}}n
z2&iU$ko^7H^Szq{gu-4r-sh`roqC4ztd>XN6VJNRoi3jO+SMCRznb)ExzfB8X5&YP
z=1$U>WgmLlV5SbYxXDbtwI*%G;W7+5lS8*LI=l*;&f+C<X2uHXOxH>&o2e{W!O{~%
z*xOhxN<04Hn3D4AY5v;;#o8U=Vp2@0zOSP1KhcR?nLCZu{;p5g95w^C0x{m;88^eM
z0+k%j73MO8Wk1r0+&LxG?^)>r&#pc1POgYJ^+`i@)kROY5Tz!*gua|Vou3$vEefdR
zS?FLW`jJJd(NyWZMr6#XcRs4DR=p1uuWXLHy=lA8E0_PmsZAy-dhG53QeCPkTCB6;
zADD>E3OcvU{ipx)i1JCNWF{(9J(}#N?RsF3#Yc@vf<CsqD>P=PvF7-;YOzlhI&<@i
zTT4LfGTk5xhRJNI)yKb_^PClQ_S?ku+6{+JEjn=0dG6UWI(3@MqhBift#WwLvD7D|
zNlZDUfKkBFCU=oF`vI2)CpKjUykY8SdZckw>8Xp*hj3TJtbipxVhKXt4HC_st4^q?
zGD&@wSn2k~yGiECM#;}J*zX&0*}hVl5Vi9hS9a6Vn^{|1H>bud)tMAzq#6|B=Qw$0
z!L!Xj?v^mj?p6u7S1$4+#&u-?t4RDut;Gv|Y-v~`@+k1f6qisVQKSD;98YO~jMR*r
z=4kBSnpd^8Xq8y5BX^vlpLX0;pZya;_J(FTuPX^UyYbi3f*VI-Vhx3_8=W-O$klxp
zeRxaETAy%}H-Eetc23>(``zXBb&78svbM(wH*QQ~*}2Bm^ULIuY*%{K{U(N7?`U{Z
zIiphJ;*89fqR9fE&oytFdC!F*^S;;I50g4fB7<tI9XF|EFXKGcqxN?5*{$b_)oNV3
zEUr3*vx_g$57o<R^s?M)XUc0>eQ^ns`4R^w-WFr7oL5Q5QcrUo4q26&_&$1xL)*g>
z;Y+`rJr}Tu^V)o$%{qRHGAfs=CMmH$-KoWVI&z<;)P_GBR<h`IzgKWs@PFlZM@hT?
zchizqPC5N!vcj)9uN-Q&AB!z2jQ=)?VV}|k{i?TmuIs}tO5eAcAXI3Upw4!++jN>6
ze{<ngy|BRjENzPgPaaEM_tY@phmElOYLl8hS0@#`+ODyfN3t;S)E%#LSC+YR?XdgA
z+~wWRtuxC}YI15Icl3@6_STg~CCfy3r)=JBwaSp!@UE+Vz>}SJ#x-j9uNWHoFz;It
ze7DeJb(ERU=b9;1b9iofPxL$zJ?;MDHO~ti_I;Z^$!7kk-t`v~oz5H=_Tkzk)Uxi`
zI=*SoohEp(e^U|g6Iph4nv1{qG=-M6ciY(aecs-=Y<j`J4}oR2Nec}0ne@fv4nI<p
zj%G>M*!N=9ssDfGh_r9Nkm~ib!`;H@>$Zjdj~+Y!ta<fBZPV$kT2n2pmvpWYwOqRN
zbonCgpbwLl+wwLzd~)Lo+}X2b@0ZT(4>jomyv5g?9}8W{b^GJ#H<dwc;zEl<p5Iiz
zrrq9am}lkgX>nw8%9>+8Pu*(O%5K>ID2zq>X!{Y@lUsyMQztL}dUCc+fV$!p-Aivn
z4|Vc;HK)`~IcD<WzQ@fE3`Ks6PX1%rq@cI9%R}Y(Obvwx+Cp0&)hi_nKJGhmqW;C<
zj_EBoE~LHo`_#%AcxlxN`KMN2=5pq7hfG>^|JA3ROV2+Kms<Ag;hKq_i66pveB|ZY
z_Rd`p&sDm5x8{oO1wWQ-y&%7*`v2aFT^sgCIj}hj7_HbItkpd$TiT3!a^vB>CXst~
z7<4Q$P)m`T|MQh}O435XAhifay|o-$TtC?yIk;oW4<Uxd`~N!cHTh_-Vl8r&dCm8L
zu74UkUs(xw&hF1ulx3}1KEGjSZTE(QMF*Gyd)W3^F6FR`^IE~^A;`OgzdmrD&`G5T
z>BVu2w^rYj(Ai^Z7$exfag(GV-)*P;^E>u5b{w9}wOV1fRmN|tS2t}GqvdC6m|Scy
zkvnC-;DDN(%=RB1;vUuWw?=m<++n(HEU?5vpw!!GMzhwEm5WqtI1eOih2PpT^T%F;
zpJLp19KOCfpcu*d@Py6OS$md5xX$trSw3O^mahMfl52MS7TUSsg+&DCp;sTcELRF=
zM9!MOh+kQzLG{X!lsgV;JnQ{!rk7W(y1RRy;TAiCLo;-hb_E6RoB3h7-XS|<nT}>F
zJCk2-4U99g=eTv<I@a&mblsw3YK4&Cn%3)HQ_O8R?^GTAxqxH;1?N`=N2E?!M8|YZ
zT(MvEvRNwQ`q_^prCe;>WYm{(OxK*ieKBySYoeR_qsa@5w7z_DS);iAqt?+0OH6}5
zI&jYDdU(Kg|3RyZ6+3<?ANisoc0b8uV$1v`4C)(S9G#oB<+tRK!c+EzQ9Mi&_bxR!
zrY@tZIg69eL%_S{P<oD&x(r9=l`XzYysEhb>y7ju{x-R-XtiJC|Gs(E8ZTGtNGYD)
zz_a7TsujkcBse*Qel&H=+Q43QMAi9}yW_2SwmPm~8z;rCIVtbcr963#+?>wuClAhl
zang;Y%lhN|BWpIibaF~n;m~S0z;e*5deO<#CbN$UPx|-5ea0c_>Mf#{U;0)#bFI7F
z<?>5^jmMlxGd#tQ9DJqF?Q+;hee=pd2cD!|^7~&b=~}|S=fsinM@Md++3S0mr{tu;
zjmA^_j$MT|oguFcrWh=aUwq8yiD{kSR$<4}o4&Xi#q`#BdgsN=Ziw=h;S~BjV^V~p
z+D;$;GacR^7YpsuQQ7U%Wm9dxEaKpAEB@I_WR7g`Es)-@&&PJP#+d_iIvJn#FZPn~
z*xMm^Z_oM_g*$yFb-$c+CSukF^-BvI7#I|PvapIUFf-^dFfcGQFgY+XaQtUr=aBK(
zu)x8bO;}5(V?*zuAWl!t5{(IG9lF^J^<*r(5@$B0sNC9YdLV2!<JXAHsH1&i)?v&s
zho(q=PCdhD%zjA3BYW|AMp@QAX@d}vi;QmVTruS(ceR%JE##YU>dW%0t0S)R#+Y7M
zv7_-(2xrLgQihw1{yUe6PB+WCy**&Q-`w3L33pbxzvj}l&|9%%!Q-AjA$6|}2M)4b
z?(?!Xc@?s=@9U+br}BE1g|WQmm8dLKNb9yX_X?4)Qd~V(A~G|DIr!_*xih%8WvOnQ
z>ejlE_nDkSp@MkR95xRgJ%OacH7C|8#n?^Qd9Xjtt5%2kV&Cds4)&DqH+O$8ZQQNS
z_5Y9WggpoPSq$6t-bOxn(Kwr}Pg7zF&wIvHkBPS~c@<YhujvuIlEQqkdAd{R*CWAA
z=3Cn)`!ScLTxs!8RTl`4T5H4|ELG%j+RC=ILRfe87n9Q+=5tp(V$hy7<3eZZLf41k
zZk!>DTg4YGP3~i^S$d&?sg^aR-=WZDiKE@4AHglrqBk!Ftln^F;gmHlJN>3kmfaz$
zySV40T*ba0?$SQr62<OSc`aM1)_3e{+Wguu*6)Fuxl=B>&1Uw!KBugPFJ+EXno0T`
z?Rvhf6^^%Acd7NQ+wt+a|D}i1FXWymHCf`Xn48vJud#9qm#tpwtsrmHk4O0Y`K?y0
z^2^dHowB+#@W1KwHJkLZUOUA<opmE9%hGjP^npXFI%^|ug-!@zduTdkwQ!$O*X9d$
z+PM>>P1$a4EkD~8(~}yzl5-Bz+>jvcZ<#FXT&Hk-dB!02R4hu{dM=xs<A#gbxAyG1
zY3)#9aj^USp0yS&_ZVL5v0OaJY*V$m^^9fusUm)Ukrj(rb~;X!70E4+cP#&6HbM5q
z&Bhmw(!wly`K<qcU!L(ME-NMdo2AHSwT+$D4%S;)Zd_c+dhzk(CUaelMGlq=D?_GC
zTc<SRlIX1&0^41RRKq$0YAvp|OV8p^>gHthz0k?lI9JEf=X8W(>&e3CYg$PQuYPk1
zc>Y85N&w#%vm*0*KNe4vb^L$VT)Vz_h4NKnS4rWxtcy%P)V2zgT=G~vLsKT~jc&|O
zr|FZQ1#51Z>UihmiCUG3bJ?x6<}Fzvo^tE}i{$ZLg;M+_cgr{F*JLmLzEE9u2lM2Q
z6P;gfnl136J5B1HgGIlP$O>IUdxjN`*F#FaI7sJO$K9}ZTOBh=_sLg>r25?75wWtj
zbJxGv{q=Qp&d*hQw@pg5-<ERb(^H+m@8742OmMS2n!T;hQLORH*99#%Qh02#WxQ$<
z`|GE^c6{N*m#r4Gvt%Wk>B5O8SU;EUnzYo*oN@7GhZ6@l^$MC=o@gc(C^*e&(#WX^
zzsdJn!ib?O&_{6#xBr`@?E(|Gx?7hS-E{kJb4=@k3X3scVylalki5^^?u0`(ourF|
zrK2}ES90kDgx07DFsrIKCb)=R*tdLI;hH2v@5KVQxR|!`oL%aiu~91AW{asHuWl8~
z&v{N?b|`PT@FFKG==1|N$;B-yo*wSrDR)`S%_V*<$#<Q#xNljBs`l9>eat@t{8yDo
zP2F%@?AwR-^l3A8@>_kH!ZnfOxyME8&l;Rci-h}~({9Rq5hzqP&$(NCVY0@THF-&?
z99MV(BL(h6rutoZ>HA??=7jG%aujD~DJ*{#$o;f8BV^Gjwew~twU~s0qZy8fS+M``
zjZR%(yE=h8@|yUXgpI;=kMDF$Gg~xuVa-wv_sXD&38w%3lCHi|Upj|NTgzDG?2YDl
z)=1uGCfnCnRH`jFxiTgvf^q!~7M+BZ%;FzoW^Q0CIpqKJ4<~=N#>z(C%}b;Y&u|D?
zCEK2Hrky!K)A5}_(4@Q1r>Sc<zps<HI8B$uLyU2*vxnfkuqQbpvkE#Le^?x=7R=3l
zz4DY6SESjpg&Q*?dZXnHTc$<snCL9sm3);?eZq&*x2_MYC&;8v+|Io=FkiZHmO|y0
z(AJEp+PyhXrtn+}TmDmYb?s5Dn@XG(TM{@kn@@LM;9I_6EoTWsa)Q5Y-9^3`7uqtm
zH)%Nd-?Z#3a9r$Vl<EF5WY?rpbwv^DOugK!saL&DZGF0AuA{^AUE=?ZiWTEGP59`M
zG--BdFwc!Oo*f>plb0ox)vS{Wn3N>qm$28r;q#uZ8%Nl-yj;b(?q79})x~zd&?Tw0
z86QKV=iNS^6q(`BSbDQTh4Vm#)RC&=Pj5QJE#eFdFpgQEa+0OcWzVEkK2!0F3XXZ3
zwb)sC_HEudc{Yoa%;$#g#j;nQ7VellVe-5w3LDlII0;Q%ZtGj2z;}|DUyv#I8lS^9
zSvK94bPdZH5*Hm`9y}=|oTT~P!z<yCB&*~K4FT;{DlUmN^P;{cnZA;gJXRh2c$So>
zz+S&it6y78R0}?0`g+kRJv}2q8Cgw_tsRR_H*fKpG;7V`J$8|+St_*^Ztps_-%05I
znb^8hs>O%g%_ebl_uck+e8sry`<%|@im{VtdR%N1HmXQ0dZqF5U5kK|<PtxrZC#7~
zW2~DOL?}$3wnf?Xg|MyY^Jy1Ec(=OsYA!M}Fz#i}Gm~yP?jW6_;mkEn^0uYUi369f
zMih!h|5-6DcKPP(`}xGI<|ZABJj^<0+rQ<l)1EE~uid!k$g6wRyhW?B_!rz-BXCsW
z$kGE<Eh~ML-}ty5@ttrdN+a&;uCDzWzt-KHbKF*i<?ZaKj0sMzKeQIGef8J7bE|1i
z$bGpliN2yq?GxDom!BvMFN*qjrbS|wLhg+tefiI3&RO~7)~q8>rZyj2z*;Gz9-7!1
zv?+GqYPM_Y<^OrRZq0HDUs<|UCsB3vneLpe8qJ~Jty?C<?AK9V`g70omNnX8?@kpR
z*mvedyXO6M|GsTM^7XBAs8Zk+$tw}IMo&Y0Ym&4-{Z_EM{i^7hTzqCmaMy<0@05=e
ziY)eA{Y|{-)>a1fXC9f;Gk5)FF`ige{wiUm-n~5q2f__^J9W5UnKb2ziiL8km{+Rk
z>$fVqK0WT9_~EC?(lv|pGY<ZFy7TmmbL;_gJ{pI;aJQGN(@OQIGwcbxwDZ^gnwwvz
zmtT8keBa!-ZF|p)$5XdTJ-&BB_(<qi?_cqpO*@`9{o8oF_o@SX)!W8POd)oQ-BUJq
zOgej^EvV0bsZ-VA$2|-ccOGP({hxQw`J%wFzZV{E<KlModdN5Lo2A^hg!ahQw|=U(
zbW~Zr%&Xye)!%cYSzub7*5y*xBf*|36^Fj(O*gd^cCWQt#>wPrG0CJcU!<`sC1S%g
z|FhH73cjVvy(pId(dbyN=#wO18rHP<T0z*wf|!fuiX2S_KbrI<N-Vb<m_`WXu$hZi
zG%0tahA3-&b>QFX!2j2wh4}*itcfj5ie3g2UAF~r?<<JhXvj5vapaC|0$VP!y?4y-
zo073%Ny5?#W$cOhAIe+PIx>G76`XfuGh}wH-L7`7MYJ`%mN}4ni2_^55C4BGxhLFm
z`X^Qvvxd8W5BD-tJkKOzqh8(oI6G~NMDPC{m3<x66C-lcO{%B;5U-um;CDIAi`_%0
zNu*U#w^dMgiH7d}Wy-aWvwJ$KkEP{Q{m7}F*f=X9E?Q7D<dV&K^Omgv{Cf{{f4#tO
zr|8AR*t72f|J5t<y_>sdJ!+e*WOI31&4m{p|CkIc%mm_<@?W?+iMJVWG%Ig;A~-j#
z{*j>dytbf4p@tC~W%mXKi(OWiy=E;CF88Q0AfmCVd4`d&BUcK8uvu}%+6kQ9fe|4c
zO==Y-THz(u?%{{0r_DW)wkf^KNTNB;Y@);WVzHCOUXPokG?l+q6frS2Ixsf6DNgce
z74Zn|l9ZgNlrhn1yWp#@ChpH0JrpO}|7dpdXtDU;*>c!I;6R4J-wTtu9`f6A_WYY5
z^yZ_*m&OKzl-6?hki!S9m#aj@%`hmRWWXy}pH<PeW@_ewuyRht=*Y_afRef*kK_nP
z<2HAJtq1s`6u5$dIx_@2`#3~W4X1A@pZ@Ynf#bu8oPiRtifm>|YzsAJv?|Uxmo#Hh
z!;Fs1NoOzVm~EO~eN)u*(ewow%9Ax`Hs0*C;F#gTIHT2a#)8TjmzBEOXOuKGcItLa
zpY^@N@1o$^nPQ%To+UqLZR4oj>A}CZL+F6V6lTR9&Wqd*o&pC?PUigB{r3WQo3g;}
z8y5Q%^cG%>pQ$V;bdWD4Q%d5BFI(``=oNY~D+InQvu3{Fr2F5c?Z>u6g=tp(W%J&A
z7k-}}wdZ2r#?QU17w5@t5t#5T;p<6Oj&hexA6*QW$r+~!d`#3_{CNJMh5bSg8{S^v
zic(MzmYgWMa#HrO8PkGhEm0GgX}};kYa#bS^H~pO&Qjn~v|6Op6{5=3sBFbBt6-9L
z*CN%Kiwv|DX=yE1{58=uXtCBT0qIo>=X5Tz^b)YhTC7|(>5`DHNYoM!t%=4RX~Im+
zl^#lZj{I*7T3jQR?$?;(-l2SW=F+U7DOX);quDL_nDYK-E{NhRT$9okBjLT|u}1E8
znJ1i@Vv{m0wy2d<)lU&?Tevuwe~G}s%kgO|n^+w)m;(7#7S36-fK_{;P3(W8S)UVU
zwOg<)+>urn*)eD43}%KIvvwKGnYnW2uZ8oq*yOoZE!wqm*{hYytX3_xTBWsXmF=v>
z3Y@F0v{q}ZTCru-igmMQ?DCqid)4asm1a|`G+lQ|9MMX@!BTtbM0ZS-z&=jrXe)-j
z4WbU7Q@#pBvL>$h#t<kdGS_FN$~05g@=5ZCF2*nT+QyQcaamC2)hvO+E}1Wu^Pg8v
z-L_elxjZOOp`gOb;FG6dY0zBe1@k{GSNUS0d3R;&%uD^-5<+GzV7<noD;hLOu~I@?
zYGZcTVqvC9Mz2<~D6+|{TF5<d;Y0(5$N&N9QyXofHqBJvnt5QO&hAZmw>RllZ!&tl
z$vAtH$^Yuj%mSMvUTw7Ay;;$E6N}&`z16JxuQw^J+T=cali2M=Jd(P0<`aUS3pq=)
zgnG?kdpO(S=9;i-?aQww$L?Meb8GT`k5=Xd4vD|GCr{qi#}Y3lBz^Z~mdw<QRbQ4Z
zX>EJS=^WQ78^3dVs)uu(b@YnQf+`=RGheOeo~YOKYA)YK&8$kHKm++%T`T8W?PPY`
zX&)u9QED~U#>LXJSY=kRI?ZNv;@&0oYNwRfmZh^fU3Tx{UcGDG?p^Cw?^?Th#R_YN
z4b{7(yLPRs-o?DJg6-k%+13J1r+3?&-raqCrexQmrpy_!n<Tc~usQuw{$kfsd&9je
zzN|Uo((^YU;$I+l|7S_-|Al&DlcE=HT)=-+OL~diwH^9V7SZ#Cvz}>1J>ZmK{<ux(
zS>4Cu`7d_V+BvR&JAHqIp+We{{5*ygoQxY77SBuL$}mcs@TqX2bn}9?a94}X74oxK
z4R&vm=-IUP)IsIh3*~h-F*9yCv|HnVwm|pvL)&ie)}C`{;(|k3J%@Dv9MbMN6qs;0
zs^GAd4TE`(fc2llHgg05AM6e=U~tvh6gYv^`OhJ@-HVoawAx!NwXAIMdNG;%p?p+z
zi<70moI{hHcWzDCt#)Iz%D)dn?l1XwGYWfkM&8|}@#K>rTTSc!MERQIWo2)c9epBP
zmcD(;vE$Wq)XG<>T))W1@ljBf;kd%YI-~!OdjhU3@A>1w_ec2G4~spa+uj<mnZH?l
z^7TeLuEyDCcFLXJIgfkGOou~@P9IjTKE-tLuw#z)qUyt|a!x6)*}Q6x+8UlCEP||o
z46M7>oL;28YmN2{KC94hroC65bYI*xn|Z>~^sKhz8d2v+U+cm%vkr0f>9{nw>n}OL
zw)o;PzNNxD1=n5Dba~;J|FNx3%~@vN=5uo%Sjwu-`+9oaYtHpwD}z70+W&i5;`^-i
z|8LKGzp9-3VZv9P4JR(nRTLE3%PPki%{}YDDVD-R5sVkv91pQLUSujf64`Lk{m!Wa
zvoEfQIpWE4Nlx~X{NIaSYgm=`UgX+%QBzkyjQ4av!6j##Bb)wfuiC_ZG^l#ARpbiJ
zk7vRz9*wHndTExxzLRH4b{xI1XL3T#R!u{$wB7m(mK5a1G;oXdPRd!AC3IXYNiL&Z
z{;hOd&hx$>HOtzfmzR4xuUoL+=}6n0ZE6RNJM10V-fqzB$v)Y?YlDQ>sVdt;+oDe?
zy}7iR=k(eb0lu{=+wUA!ID2?k&XHzWR>6WJf)`Frh!qfU5C~vkRFJ*7cg~TC4|eWY
z#mS_kaGHIu(~It~Kf-Q%w;ssY>$&y{>%ucD?w$$zxRsf4&Y~^3CoKeK3pt)RxNZBB
zy#GuE4C(UEPxMHA-S@n4UFG(=7k8W(f0VzgycTI$@V9;W$BMfzb_y`%)PL$U3j4os
zevE_5^KwguE<xr`D?S*mOIy`1>sY(blxyNz6|Fg^lz6sy&N(8!_mp+a5f1}~zjyDm
zExdFh@WyJN8*Cr%2R5+mp7UUz%@K(^izabd1Rm!KKE2eT_EzGXTUV;r+P)PQ|98vo
z;pOFD)~MuZe+}UGi{19)sN}Rz0o`)}i`taAS6tn0<ddJhKOn?P>B+JxtIXSmnJgcV
z+Y6qkT%+dF*LUAh@NfuI<)_FTUYj!#*H@SnE?+ih`kmc#?_OW>`+C^HQ?u^gU>3N~
zcJ77~4{N`zfX9a$A7hWC@!ibT6PVh2v*_Nl*>7*kugdLqJaaDm)?SIVYxbULh`Yl4
zfP4SRHLHBDaR2}KeD&Oit@mzOD0!*&J(RY4xy`A^ZRMQQ?11@(HggvSWL#SNMZ)0v
zjT3MG6#SQZ>gRH2={)&IR_EA0ZhM<`_gm(9y;Av)k{iC+^u4P{{4egz^(W8d-}>K`
z+RW7pBgOMlE(mgDhb&fL6+C;rq4$)>oKp&O4=4LQyRlA%Yu$rAYaSTSJskDn!L)@o
zLg%=GW-qn2wUd8$d5i4J9=+Qi=E*qRJo-;TpCwT!V@}Oosktu?PGMMn?P=DXZ_BTJ
z+>;;5c`f^R#;u2CAI+a|C+|qCtZ$9!V+~{~_A$6pC>qr(m@RXC${TsLM8UuXoXXx?
zRQ*qHesgJ}!-ETTr<nv9t^VBD^<U>1^TGQ?^PWvVd%ExJ&4WC*&RXBPID7UbYk`)1
zXAVeC;hy;Jdi9I?dk>HAoP6W=nlnDPPdI)(*zuz8-n)dH9@ecrVX+Md7rWd%BPaIt
z^^b-9ojHo{=6w6nxNglBr$@JXzq7smd}dx+ep2~->ubG%7T4}ZJpBIN%qP#`!@iH!
ze_#5xzn}lU@Xhxh3L=VvCXeqG=jI)_ebLVF{^fZ$)~vb2eeU$U^+$|k-<kbQwXj_p
zv}SUsPrHKNON&nryC1#Vru@k9>@Nwu-|N<X-4--uIpZ=Om#nv&4y-OJV~Th@@tZK?
zirV5!>#fhJpEFEccKP^)`P*IP{_?Ires%qJ$GG5_15a4~H_Bz+iqE(G)9)kDX&pV|
zpyY|eTtNbiK^rPm*Y9FlxVb6zBKN|>dwG6&#+));n_FQQ%5bntR<~*^*R|K7>+dtP
z@%%NdIk7?fNSm;9+>sfD51soY4ejn&C@8tMaof)OVo|j4<OGS@sWp<9=kKrDYQh)M
zIr-t~R<S@UpP#`m0v3sOznXI6ytua6SGFrT+c(S3WXzEY+B!=x+bxJWLNxcYsIquZ
zh~%V)Dc_WqFV2>n_K;;e&&r*tUc4_(8N6rM#`BwRhqOe&?VemV2{)SqzRrhvq|J4T
zXJkA%(GnOY_o{8xt@$wq+c&-0D5`l$&OPzYlC7_og#}7B$t7gJXVB&SpKZLS*U<a0
zPT-aw=N74UOZXmf?~H1#5m_C&CE{vQcz@jfmwxrLTxQAsKKfQ}?czUud#f*g)Hpr=
zyoJzW&slNoH$JDYpL*k~)BedkbN8?K)%N$D`5*n=%jd016`3u?bXG3<j&_&bqBLvH
z%{?kMUb^QL4u4oX`9xC3;nN?O&h!S|(r`T@@IGnH!lM#V6IUGn(tL4~sOFv(JVI;}
z1deXgOVL<8SwW<$hb8ytI=QeV9Z3!M*iNLF-^sY_(*4LqF|Q|ZO-!n6Sj58A1<hHG
z84WE}MypoDT{wJVdG|RJ<71Prn5^0}``MFo)2pIZ9vA=TwdmxsnRS`$Cbs&&+z(cX
z#eDd$^0A|7^B&u+ul0i#B-E5FuR4~Het<En^OW4`XV%^t?15Dm4~pA<3ec5e|L{SH
z+qUVo?yeIXvezB_mb7|x82jZw_d{~tZx$tl9?{qo-1?Q{#>oPmYiBo2D9vHGkZ`!e
zE^Fmg?Zua8xvp7#HR|BBg;5bVmAV2=B%P-`4vSo8sW*S|aZL-;wf%nS?@PCNFz9E!
zS6}|`u%5&FKZ;KsraaO%u?yd2bw%@sS?<vX%ul!In>6m69I|{)L6pd`sy~r^iq|{7
zET8Hxz3JNGz^~rDH^UQ;yh%+n-Ewt3quw3U#k-YWui_E%6uG$JXhRr}t1x@P(eSny
z&fb?jbJRrRmmOBs{=aFJ>7JQAf~$p29q-=rFY~tS?x27t%cCSyr}}0)X`Gqw_9#EF
z>vM64%5HC~RZACKdvb1m-Jfr}=hq1_1%HT|)g6-e`GfGB#rwVU(gSu!emW{=puK(R
zTq7mBD{>Cen~%!18Kpl>cI#BS9Jlbo8uPP@)jm}>T+X_+<+6+IE`gj4@w+|NBuk(A
znYNwn1ylT%3#+zH*;Vpo+WLb<tbqmx`-032Qy1q7-VBXA^5Xl#x_raxcek|`UoH5|
zymrgNsAsY+*EVcjDjI!5v31|h&qu#6>fZagqqN8L6T5B3My{})M;6H!S|%Mcko8S*
z6lq#~-P6Z>!T!ImH5)&3+3IPm-Tt%rzvza*0L>ZS`0M{D2i(w_qC0Q0{h<>t>m-XO
z+uztI)9m@Wc;UkHr{-*3^~i8Z`+2TVfmU}(nM$`bIj*?gImeA<{;qMk%slO4&==tq
zbB|OnWR%}{Y`Mge&iu5at9Ue56dje@xIrUDLE0@#Xp>jv6Q7O6UG{NFY8>0n^(6@j
z)FpTr{x3<76KRaKzaqx!^h?d_s9}KZmIf9>i`0U8={Zgcm-t^?Q1Yphc2>Lctb6}P
zWv0Llo%3=6`=ZXUH1KD-`n-9_b^VNs*|r_*&s@rOUi-=4ws7Oqy&+0!UV81h%o38?
z+lqwbUk0?tDLxL|Rz7Ke#S8s8D`rW&T(l?im{)kkf5pS=mY7SlY3N1<PvO}l^tdf-
z&MAXqN!OC2O7^BDzSW3JxoBjZbY$`ze-oKA9ia^+g5MfX7$n!o8@ONZD-;c|mp^^k
zH7modE$UcPO_Zz11c$?~HXM<k)!-z?q<J`0L|u510IROcAwFFPN1di_CYA*Wp8Gd4
zaqj%sZXNZRpK}_IKaZi}ybP(}f|KgSE?F;hL)*<4Ic2SQV9Dvr6>(TG_w+oTpQnP$
zIaYHec4(#C<k{TeF>}X_H%gvIIHFe^ns(xiPR69pM@fBUTPBADc1w8Px^(ZA+KRbZ
zS>2k<ttSpG6xxwhoNBZ9Ls_{XM`y1_Mh$;JdZhhj-zC<HiNgQg>lQ9x4ZX-GE;v^v
zJ2Rx_>5}Bh!mB*OoP_^hStPI2dLZ*#i>FFTqg>tVqZ(C<SpyXc7y<<jm$)%D`phtE
z2yAd-F-g1?*TKZJsA92wP9T@#!;SWfbX+y2Mt89~ZWnpWaktu|$bSAG!xcI;9sU}}
z`wVxm%}7uZ3SBYPdfnd^-X#f#!@ntAeJ^LGcg07p<Hn}=eKVF;p1Pf8zP~+;$=y5h
zwa;_$i$5iloK{!AXGu@KrEs3J@v_lFsoq}(NfFOYpXi0W^pbWcR&d#Pc!BHB@R}OO
zr0qs$)wokVDrABjXKhKZe-as6@1gDa=0Zxtxky*FB`y<OSC!fP>v-Yy|Gyd6gLO{L
zT#`qX#3TCe^(<Pdf3;=jt!9=mohQkk5?OsGYt0Y4uqEJu(B>wOjSEb!Uq35fc=<s@
zvEWbHuIvpDH>~KHuGsVELTHWMX|Z#M7Jdp`S`!`0E!5nya9z)eiz-%v7Y{u@yG>>C
z+@Heo866qZ_Hmt`Tf(_~Gu!P6Hvd^RrZNZUN?5yJKN#r0X5SlGwhwIPI}Y;jR<b(l
zN$(2pzNWzTQb_yS!F7*297W<iIr@x09rI}N44%W2P}E`Eko>%J(yRstkxdFIzCDW+
zW;Qgau_#RlO<AR~rNfCO%yF`-nUT`J7p{|bpW({gU&ioiovYfS1AR5_2kxXV%G7od
zPyWwv_K0|+RD*`#KH-a7gZgrkzb+A*KJSR*sf`OmL;ip8Fzn9ZQ7_ocp7izmyo(N^
zsp{uX7{34Gdb;GHe#^s#fDKa9v-~zICaHKwDNNRm4xAh=w=JoqwNIhA&N%9Y442^M
z9j>Zc$I@+faS7Br#n&ixY*=z*TI4UuRfb(34Q2}jxf!YNwn<}IxNigNqL{??qcaZj
zPTg7R98<`BA?4KRzXGjdN`<#|92j>*cC-{6SzyMQx8WFTr{Ddw7lDQuLCxO(dZ)Nu
zRMF^Y6g3fK7FuKw<9R8CXV&=@_ZK_-n5b7I`n8~I=a$R;J`Rq;j54f3b7F!FHZpOA
zz3R<mU-Ilt-~SqhR;Gy_S7bgma(}UWbEEG-ici62^>ux^BCF<1xo_X6a78}(flo-*
zL+*gU)9zP-cBbkr(c*DEc0S(ZON5?M#zV<l#_ykJ^_;MpvB|73`?1cgu04|BOHR(M
zn%X#v!DR89liGgGqDB&n+b+ni<nFEIn$%<{(kj(pxS_A_WBbWjYi@Lyu4r6tw8PZA
zfK^a<$186>ZN=TwquS$(mt3;y=zHBR8P#_=L+HMT<3Ud&Lyy^k24?rWG>QW`S`=1J
zKg9XCi)mt4hib5<Objd2tiDMnwm<gj*tgiBE>OkomqZ!E9>Wj3Ekev15)G$jFW}1R
zoo--JJwbJ;(cI}7y`Oi@{=bz`c>W6x3r_vs-}7yD?O1%O^TyBFiyn#ZzPRSpL7D0u
ztfw}yhAXHGZsyjz!6aC1cw11XM?z=j#;!o^uIbgAf3M=+bzthhQ#-#*=AM09&Fi<?
zYej>yUvm?0t_s+##P*xhT4UE@WuDCvV(E&!KOSv}Ub1<1=FAP7Ie&KYmO0F}v@$yH
zImJ4{dXH2?v^KLq4u`Qp^P@;NK`+%spZb@-XxBO2^8SUe)=3sa4<TQJLsb$gcdWK)
zU$hD2Sm1hzsaU&n|3Y@n%L*MG8hXZ)t|c9>FI=e;(O&h@Msd%I9xH*CRqRex3sfhs
zIGZh_V{^QACTCE@;WmlsQ68!nXS$hoiTuB)xo_K!jSDLlK3jd*cFo4k)*?SUB?<~X
zf-;y5AN8#1>fvpez~U%<uh8MNg2~s@{42U8zezL9-NU6<u;2MJkNH6!wIY+_H>3_U
znCNNsNc@y#+%vsYNRaFGma_+XQmQt~3-0KV5a&wS+P7Iayr9**f;BSOYsV&5!`aGV
z0tamvIsbWdoGI3AHxR15+@kYx1!D)xRz^vV4Nf;!vhPS~f7H~p_wja5Cvy#n_J0<W
zqb{(pTxbywSs6G{(P>ftJ13_AE8V6PO)sHVjT0@Cs}9}P?tIkMv`1%htAfThM&Wd3
zuCrOr*L^JdJ9e1m2+w(8Vcj&lrMts%r^ER`f%`tY|2wHHca{-+dU7@2N*>qK9eNs!
z(leR1Mx3>o+4b$ymaLx>@6J3qe}l_<C(ev6*IfsW>{-Dx_c6EL1A+gmc;+)3ED-MU
z7Uuf6(Tkhgbk;$Rlr>w$A~(#Glv*)+`6?Gqp5Oj_OuctcTJ>`nuK%rm;YWXm;e>Yu
zGoH@cu&Km)yLPjAgj@8FR>l(nZ46UbZ(f)s#Jd0U<TEDIdmZ-bXdkKyvHiBdylM?=
z;EzN1D%bi<+^dvmuCm}j)CX(6#j;n8t(Tr!;j@x`;;AMb1BIKfdM_<hh<f41ZYk6B
zp`#~dMK8;YO}EAOd|dO2+ta#p{twCN7bSY9<S_s9*;Sd<A^(Vp??%&qtyP;@1NC2>
zah-g6Vb6>1g)UP+C0jlU-0<!3<+cl+j>_B)tX?}$$O~+?|1CW0u*1apf(EXpJpUf=
z?=Cp?Z}Mudr6FxbE~-lVd1fyAA);jD+WCej^hU;v&r0hnLim?%XfM7r(I=H-JF}zp
zj=5);xa@>f&L+1XxVhG6gKw3_f)mm0aS80F-tc=#HC%ZmE1SZhv4ZzliBI!K>8K7C
z4F|od2~L++?knS-(0ky}#ly2TvM1%quzFwelDjRfJ=LP~LTBuw!*8<|-?_ti_sZn>
z3CfnA&N}kUb=t!0tfS7bdg@HZtEnpue(mYd%kaF=(RD?|(!y4U)#c=h*ArIvxXN5S
z^55r)%NB$4dp~hIcXRvj7@S@)FGy#@rV@2A&E+BsH+-)--_Uzw!IE8zEav}8<cN8)
zJVvG2_XpRJO`LC5?Ph8;?1~T&)IJg&A(&p%%xG{dx}(?XjK#zW+qr~Vr*q9b#=KI`
zaE|QKLrSmbmujqJU9?@J;Ud?n_H#N;2Q&8`f6|xz#YenrZPDS5iiPcQGx}mCMLgT=
z(e+sP;c7!;8BcxFlLl|OHb3xxdgU^w=KSp(rw*Am+S*>`O!S*Sr73x3YkUUlip9G=
z-}E#8x$v;7{kPyFd^4NomQ2+>vCMh#RGX_@4!@0g6}3IO-4=B&(&n1KY~uO(4a?s<
zx^?XB5IV8yy2s7`e>AtwuZUQi(;k@6zV3~Li>F0(Mf2_tO*@>U7WGOPui>z?bgFW`
zcF=HLkJI!*g$}vRVHyTH6^FMytXk6l*hk|*Xkkyw+0#NSpUzzVyhmfffnH19_^53c
zYi64}h97#Nr*l>!Sg5o8f$z)}b55QLXN|P3>piu}>Rd-_LRe<ZjGub<?lR~xOlN*6
zGCM&(DagTJc=4H5!S_7chxT^N_<Wh~<H^3Y+_SPdGNK;FcJeqg`ybfA{q4!J`36UC
zsYVt^&0^Q)N{-~&w%VU3vuFF|yT5*My}!6&Ps02)(yi+z&;9PShE-P}YRCLXpJldR
znh;jd{J~qR^@_LhN9%Ko6}Vot|G(#On|s%k=~^5vS_)wq3LpQ9M|BA0D!Uhnaxzxg
zbc@XR&G_WPFP;0RH~#MEcNROT*E@sX>g1F!Ce!{3gn9Zs;Ov#tVm6ukK+1Bzjc&8%
zji$g(nNAtj3kE$a4skG)&R@te@6pW|PQ4rJg%*iiJ9**nDo5X#DF-~QY7-y6oH_sF
z%(a25g#ukQt7gST&iea#f6tECzXhRFUToQXLF({=BN4~uzHicRD&}g+V)s#e(7aLF
zb|<Us%ci4ewcTWP+{_7|v!XTARc~Uzl2eaW|GyBj|LJYO!g=s!Xp@72E=TySpiA8k
z9-H%Um0jJ@_k!bvwC=}-@aYq-S1k?a`E#}=_QC&$Z>~q3T+QBh;}PR(g)p`L8&6Vy
zzGzgP_v}f7W(IS)hsf=fcLJghtgjT46MXpI=^<l4uxV|g-h!R}a~{tAc(Uj0joF^u
zV(V_W{k9BTv*o8uB=h4$_GqsPA$iU~?W}tOZoQWoY6UJST;SDK4!YK@B`4eVVEY8e
zx!k9=rhHK}jyPtuTH=|vyNUqo>j?`D6FPP+5xO5R+jhy?CzEBvI*v0w*xvLaJGI%)
z_?OO`3JKBElWi1F{+P*sz}J&y$}5h#)Ky6x>HEY_vu){(n>yvIz%!jiybnAxPlm5n
zo<D8l<)<xAm4sgEJ(wV`)NJ3H;K&^t#WwF!tNoYDcYRZLh9`FaU(|C<xO0oEY1X0)
zK|Q@k8I@9!k$0!x+v1kny?#d8G8G-80*TJdc3sDGSH|?`rp&<`wwgr<M16hcDLLoU
zR?ht?t$VCh9#5UolyP{@3}wb;XWKgR-<){2Zpq}Gjp46)uQOe{aL?tzmf(a9B8Dz!
zw4YkWhU>)~yV`$w-EH>st2J*hT<Yt!-kG*_fs53;^(tx0V(qvNeHEC!Ec2Uorf6ID
zCDyI84n~B%<mT@b?TVc@@ASimai``-h=%BRr{2t&s(V>><}QiG-l&|8SR3BMx4P#{
zUB)r#1k=y-ML#4K`5t>@*leD`T6H$>cfhLT&$h27^kt>o($1BwE!=!uN`SGg|G)9}
z$)dkkPGXrpWlfAbUxbR)o$twFhj?G|f6rjO9%1bJ_HJ&6(1X=;c0QTZI@MS1%lb!5
znjGTMH|zu$d8aZ@o4$C$+<O_bLkfBrLOAb<-kO*DVB^#Ks$bp;Y74B~A8+RMooCmQ
zC(As1pLv_+J<j7^c5~*dl0w$@H%B`14*YuH`RBpxmSDpwjmEC7zz^$eW_1?0y?&>*
zT(D$a@m<wRt9D%b;%r-$zwpG_nFTTjW`3LB(b-n_f}7<-ho7~x$&p8Y!h7cB9J=vw
z(T?JYe&7E@mS36recOf<y#mjtq8g`HJ~MjMvrfVP-=lNv_762Ty!89hvtVAg%je9n
zLbblDFSEE@{~Ig0M)-XFR<qiyz*T{7d8*sgKYPS#8s90YZ<Wt)z9O*rih|(=y$QKD
zPe^=juP&JNNONLW&Ml5bhpjjD^!cP$HO_W!>(_sLX=2PLej`sq-WZ$kEqhi;1uW0u
zi;S^<p5ayYtZ!P?-xXhEHJLScO!)C|-#0-n&X1n++ET*Vs^u4@>MdaEPq077aW`_^
z+j({-Yj1roOZxE2BxL@}r|qT8i>^uDo~vE+u%bWX*}@$Y116@49XdC0`;Xb&)|#^4
z=9}_cZ%i!coKbGgwf&8BZ=r1I)0%$cls_H$6MuYT{^wg$%KGrjqdL`z-Nz=f_<soP
z6J)P1zbg9UR=VOdoo3z{P5;9R*!ijg5?AfI(mQjTK_b6=_sn;n&mZj&O7GYlpD5yQ
zugEp;apT7iTM`$UrM;5e74SQ8nVjpc<rPfXf{PQYs;szH&j^11Jd53vcY`<Y;UMlr
zC*>dyUhQMMrm=r!Twv|C|I>S>jw)t0J`s-shy9F9yh>lpFU+&=_*QfEb9n!vwPBLk
zTeDnGB})h8p5S><@<HWPW6Y@)HW5vyPwMF<E}AlP(gNQ(%qh=gZnQpSn8n=4r!cEX
z_0&{@>1?K}Ht_^6^K2Fp(K1oJvMS1d+DpqR&;6$8m`|{}nboj2YBArYLs>?uo{Lm!
zgKS?+&AX(@yzR|i&w%y!8mqQX2;}-|5c9vOxLeHil+cUlqpIu^bfVQ#W}jYqizj3A
z&jd|#7R%~qHVZ@?uQ11~w5upvpxv}FQMhLBE&X?4i{#~1I5+dWU$|84vE7MfI|`>{
zv+M1z+*5Adx{dEz^468p?Bg|s7hSPCApfIr?>1%i7X=Frf4(1Vb>*GiUj6$0Y(F-A
zKRr+P&$gVMn->4Bmt4xiAyeSWSUT13spbxa<`b5!x0`!RH4=R^@47zp75D3y?c+8z
zfK4)Z!orITTV2BYb>B|-BI9_xFw8Ad!*!ctukO+#*TrOf4mx_r{P18?S7EfA-tQ<{
zImvU*l!Ypj4Xiq&B3En;Q}$X?TDCM}S>Vov8O(yu7exM--qso&ozo-q*mriwPEN_x
z*znU1mw7Mv>8|NH$mhgubmW5Lou`okjx~A`aV}!6ErGg{ny0utlsZ+OOtbvTa>bt|
zuu34n#dFq-h=WgC5_~zYg*sIV8fCniw(`=CpaAI)Po<(RPc0Rj#k|$<aZ<e730BdH
zxm#w0Hu5UGaa_MlMMu0<v@N1wO6FFHkV4D0#C6l!N^WhPn=5u}TjVygrE4GXU3lvG
z<Wk1E9QCu`R(rEdoRHHKwXEvSbe5}KbM{P`n|j{!>lP!fIi_ky0_>mvti0gz<>2&t
zwcnqXch~*id9h1&UB$Duh9X8+wvN)uJu4$3aywS_I!@F|<SbmOvuV|T-Q9nT&X`a9
zb%{;N)jhrB+l^<c%@fb=n0nlf*_NT7SycAQYIix$2^;79d0~>8&v=@(-#eJ~Nxq|)
zYQhwQd147!Kl;LA_H7e0H$3??GjiMJ6N;^uBCXpkFU&|;d^6^DUF7v=aYg!8Zg)?-
zm0aOeFST!;tB7jfBDb{#PXZS!@AY}Pw5(M|EP{Jp&826)g+6x@Q+F8t+Oa^4{gY={
zYU$NRm#zOqPK)fT`{|hf_-bgxv}+}rzU)a^l}mSOhqZ~sFMHvwtupuMqik=@*E&mr
zBTW_Ko4=;Maj^Jxbb8QfztfZQ&s}=5C-Jh7N=fRuX|8N(x6;CH?a)}+Ryh69y6XB}
z|97nAe(k$%g13TW*t4aRCnoAGO%P;aKcI9+CcRPjmVoCc(?t=k%N5q%IFij*HsSjb
zb-VkSGi(*uHvXNk!T$fpGYt~SA~UXNr|J2ed}gO2=`+>QoTZT2dqU90#ybb=4KH6g
zp(A3=%V`+mHbu6r#C`YupIQy61y|3pG|8t=)%Da55p62kvRF>kGi#z@>?<~*C2E?T
zB5}eKrucn6v~_8V@r$ZNKJgI!oxUf`wn+-#<5;A&e@mi>(;<a9w-nW_Zm!g2_vUxe
zDiRb?JyqEEyJNA(X1jT!=~XhHZUk&fvYbC>4fpKTb3Sxz5}7t*VTvasAKznf_j!s>
z#WrR-^Opy6{SV)@aO&ohR|1$Sj`$wxQJcH_QDD~#Cx4?howS7=xk)LLR)=J3&ssBA
zNZQ$2FG9bX+bJxb=kko5IgPD%dAt(1E{hz$aZZ1ow8NBjTi14bcIKv4KcC9xw61jG
z>A5M=(ivP=q`fC*CF^VqI&-RILiOFv=BWq!6<9CzaZh5j|53<VuV!X=zQ%Wc{^lhg
z^hy?T2nISX`}Nc>X~m?O8;>M8eZ0VsB4K|q!KYJo#!_v*W~u+J&&*h3&dMHPJu~;y
zmgUSp1th!787u`=lykZGniixfFa4(zdhF0V&A=ob`GYK34pv?lS8Nkqp{uEvCc|<4
z$`9R*58HZooV>InD$&0p>%XQ`s`t{S2}~Dm38)m9o)S}?>bzZh?ouNOo0BC*Mhu!U
zt6P_HO$`y7V)^i5YH+t{`rCq+W*6@?8SlNU8gl1BSG0h~=JEnAag)<Gm+?FhOAqYT
zwqUz#e(apnl+T_mNw2kgPnne}<R<A&&(K_dpk;f{RnFHnGlTSYoYGN|afp5)A*ys$
zwaQues-1t3?aO;BCY^dEmptv0uU<xf_4U*+A@#}ozLdJD%qo1mqr__e#-`AE=}l+<
zo!K<|&McmqO5??T!IvETHfHZvKe;$w#-nZ9gX0Qnn^-l~CR`6$dUp9VCIQvd7yFMp
zq^2=HIn;h9f%{_0j$q~&iNRiir(^b-O#2^qNrx|T)8QYDKh${p)bs<VpDunitutoj
z#bm|TQ_>ncix1p%I?LgtE^@Qk<Zxj@^RC%e3tyb@@P4V(+oH5YZKmM<1n*=i^CQP<
zB0HCA_ulvQxPEK1)pm=_&95C^M97&e&r~?>si&#baqzl;sdDZFrnZbVP7+0WOJ)B?
zm>5PyZ=7&&^KzMRKKa9~XIl8$r}SoK#C~iusxi7$9>AHdHQVacqBK>5+~(!cQI~$&
zZBLbOEouumt1wC6nyIGID~7UZI#Zu;t=j);>hUjYPFCHVwd<M)&*e>$x|u<0navB;
zF7GLqxGlLfN^NIPszy|B=ImwB3ExiJd@i3o<D!VO`tSd~_K_XBGdA9g<C*lpkvV$N
zLbV-pcGP~)f648;v1ay?)hBrvf4<fe+q9%z;nt1oIT8!>zPdZe&+C;nJ2UBmO-IPT
zd$+RgILzVS`%rYH$Xso`vyD@2+Lk(mn4WO9Qxx!I@J*lis3gqt{nX1#R!&}cG5GtT
z*eL}K)rWV@o9xY`wff4Pj0+`Eo9;0wub=m^CoAWD)0P&!FQ-1YiFMzPN$)wY@{fOh
zwylYcddRzqYg=^~z6mwNJ97o~H1Td*nXo2Wk@4`M-EK;*C$~oyabHt<k$XVVbGc^r
z;f|s;JX?5jGizr^My;AQ?cyJ^RN>fLe&Q!CuQLhImfJtcsry_|!k;pO|5IPQo8NUO
zbw%-;d_k4T!P5^+)}F|;BI+Y|Im=P=QvbW9^N)X$^jms;Wz@US9o=s<-|W-6|5q;0
z$0)8uY^llf+<f2YS@$;R*55v7AMW+&weu}$jqR)=o|SW6A9zx*-;KNHvm<}egED5D
zT@PKAyo)wH-gCA!G)19tx`~^@l@+3<t86<m-UNtr{p8;#RIV&_H0W4HgjV61_NUkV
zL+|DFNVOew3*T@=THuKE4kyjOAvpmL_!R?h&x~x|bK`jGl|;vmq?2caeH(LUc&y<$
z{<<}$Jo0FnX!peztf6NU3io)O4{g2b;v3U*u_0k*#N6Aqu6L)#%w1c;nLW4n=fh*q
zj{cAG%8_mU(;ffd>}<ot=q-%1=dj*d8mTSP&v@;|&#CSqJNz;jSywQ&*Rvd594qA~
z*`a;Z&HLy1MY?eTTO6N6@-P=g1z$<jWjua=BTGym>+zG5Gg4#<4zMUbyY%A01j&k{
z%R^XK&Tvrna}7Bv`uyPBfD)0_F_#z|9yt^&5=>nb|7Yf^JuZ?t2U+f%j!bbecbr<j
zHR<(J-wl7I3XjCZN^oz?;qa*lf8TRnOX+q=rvD}u<)jm{e=g<UB07It0+01Y|D8R@
z{@l6Hv_sS+NjiK_yw`#I7Xt5lS}0yuJYKwj(^1iV_7x^4Sr<2-17Zti)J{B@Tk5xN
zLc^(wsKyP-l1=|_p8LQqwMyv!+cvi!{tIt5$vSl2IVzHsBXKV?%;8(K&&HNRijS8r
zymWBkr7t&ITUI`H3`yuY81Xz;#XZYUazaCg#U#tBGsjjQS@}|C>W(|>a~T{(uCM=k
z)N_i;g}pMqLcLxlHz&P}x$w7bvqh(8$j$36Col3ooigL9r-bU%Ese4hZuPG1&is?O
za>Ly_Ih}ic2<GXqr|a++z4Z1fart;fQCL-Mv#nd2k56F7v62Rf2exsVN7!^j@98eM
zc>iId%&aipotL^NI3B&h^n1#+$TcpqO&ncYg?6q<Jtlicj_<&dM4@F*Zsagdes9z2
zR(WNH<Mr^0#67$Y>~F5L=BWAP{8x+hkz@C0ynp0Gs79nj;-xc&99g@1&RY84e!yY+
zRnClavDw}k6@v2(rya8jR5q5A4a(!p?)5vCk@Dy0{HWZM-wrCB+2X1FfjePq)9MOI
zhLa+~Rk!wjecJU@-uG#|HcLk*tH;q3K}Ry5m!A`<^plAC7Alz{<GAl)i>6AlkBX9%
zdQpeciIwqhmhjg3b=2i4D7P(2J#;(8Oe|!|osvCnpY}X+yS3n}&D3{^{h?<P_%6*6
zyBZS~_~L3$;B1D@$x~Q6>U#OpB7^u6D>sPUUa-PuDes0q!RC37n^xRxT5&S!&Bekq
zUUM`j$R{o=TzD^XN0WBI8HOM1D^%m`ZarP`^?&RUQC3NpD^btp#!nKP<Q5Za6!&1}
z!+^iXv+gu8+TBa@jj${0d?9dA_0tW{3Z;A-g*b;t%J*71@7%Z@xJc#KQBS|5-iS*l
z8`4y#`OGzXc6x@~#~M%VGqZ%2b#1sI=aV#X!_8aozRK~G^_vMQ*S}5MQhGZ5kHRHQ
z`G2|`J4;^KsxFVepxpZ4x?@4xryUoHs+7I{h(0tqSHhS1|K;2YySoQ2ia4wF^ZkA5
z&~&!VagKda1Cv<C{)NwfOD1WZy|V46w9kSzyPG#^ySmmDK1lX^-J6)^9+!Q(L+PwU
zx9tul&4PnHGQNw1T6VDA|2apSXVH;_oCCT_t0w<E;j>HT>Vy9tlYOU7l;FH3(t3^O
zuD?|5^$php_GL`<nR{c$ip{@R-R=pj>X3f3rDxX3vymH^G<5`K8?amH9xMDZHE)~i
zfh%pgIS0k2IJtz*x0lL{txK2eQJ>tW6B6Q+b#iX%$+_#|G{r44tG9^+Etz!p;f1(&
zTywdEUo}k3?oB#V6PM+7>gAG_d14u@JT3E<MJ4NUteA38B*$@W3&#qDtV?Gucz#=X
z`iY!E*JM-1V`n`>Ur+6yDR#Uf_H^Y|-G7DM(>IDRnZ?&Ebh#UMTfyw^&)8|d<{T^9
z)Dg1n%=OI23_VWTI!7E1c)Ascr?aFmuDnsXN=LG#&3=`{5>1t)AGcR5z4C0*|1Rkn
zPFE{=R;-wk<M7mN+5Cogm)aWc37)*ske1dUwYJGXS3mU$ccoBB&C`;IhgrWgS7n|T
z*}>)A_u6*{S8I@H-V2Y(xfgefadKy#oiIf(fMMyuzY3jMGY`n6eiLHNUg&k}(#!6f
z7n^eC_Ph&>)AZjH;1->dxW?E0M382OMdq@VqA^ZVPDU$T4;^`Nqvc%N(FKW9RChW`
zTg+M}!D=XbWOh`9^{RF8do(ZHXi--(aNM&p*rH(JyiApQE-dHospNg|=L}chwQerg
z+c|DG*_Z2u>{xQHcJg{b{WDYBcQ0GbeLve=)%#3f<Kk(#uUt;&)+HJW-8anRPF^E!
z9XMs%h5voar)h*Q;Ib}D+!XVQ+0bLvtLaR<NA3z7dBB$QaM`2IBb{A7jnW&RA3V6|
zpYF=%$K061R=>aY?x2}&il-JwlJ<pt>twEPZ2sG#d1?N;)QjiyJa&d%58n7h?9QWa
zPfmP1rOp08iq$}WXV}%tKRm)kUFUe}N9cBbZQ{M8$#MHd<jQS<%{Hoe^ZEsr&5S>h
z5nK9nsz=VlSAt#>Ttxk3@0a8R3WiVqB@o~v?>i%}Vxob}{4TxJxU_GPyIs_3k6w%A
zk!rpZ)_?Y0lZ|eRjKSMS5@#l4?%C*aibd3Wi<7x*mQK(m?y8fkCpfOp<mWlI;R=(|
zr3(vo-IKXF>#fGZwM`5D^9xANm+Cy1UJ_ZgP&8`kF@;6fO!7{AF_qz7dS>?%Q9-*4
zBI*yPxm<9NT~{K&;?8pQ$*$o1R#yGrpT0Mp_nvbmxyEC&k@41Th0QL-ulMEEhpD~c
zy1Y3^jd}SxDd)G|>$${z4lJr&ZOLpbc5#sc`+EkS_3=NxeEhWO(>sn$EL-^|1r{)*
zcJyg{-r6%&=8JF7CcWnTZ(1Fihk0W!ayhmp_Z;E=ahB8T!_uayQmr0ciyUA7n)p?-
zlP{lTk#yTT{pKFd8GIb;GYd67vlJV|z0cgepoO(sDdBVEvs9%-_6xBhX7TYSTql3^
z+-~B-e&J=S&Q-3p7WxuXZ>+wV*T1XizyH-H;n1rEn*UmRIk*+)uif-^ri|~g)FZ67
zw_WKt#J}}yZiw)kD`$m|iEu1k8j^otzLeyH7{}7y1(9OfY}z&Q={K4jw=^lN>=2pI
z@b_kVbE%MqBx~c0LtU@>Hrxx)+~Boc@C8Sk$u_fXCeF)bcW<n(>*&+*Jm0w1^ID46
ziYKnxGj$)e`Fo3pGRiikT>Zqy?e|DwU(wVHwO0eatoCdX?PSuCSB>RR`(f?-_0&%T
zf6q6)d@VmG_0Fm(RhX$PDH6_InH%~eaj8#~b=0jL7nV1!Seka!O|$UYbEy+gT^HBh
zl2Dib7W;ae{^~E~^N(CQXSnrLp22eq>D|4R>bc1a{x7!pT=a(XsQKrQ-gUZ;J*6K$
z$-Te2;WMAZ`Y&8Jw-uKzd@Cf@{^6yX!1d2NY)!A9JY98_e@({8yWjYQy*O@5E|~nL
zXWE8XCf~vb>!Qq$-Ru;Y+Ii=u{O!yW9a%XB3HBEQszg0}Zo0JHG}^T2`E*`Ong5X*
z%TLXJsP4JDWJ3D01%;bVD;W3Wceuaj_{8;-JM-#SoozyL`CcNtqCBScrxyhBckZiN
zv5$>?ih%06-2nxwG-_?1?zr=_UHtgrMW5mlO*ON>`s%oO=(t6fWJGds?QYbu@0}u)
z)|&t0q1TRs>5Ru^rv$y9(ik<ZQM2b-@PR2)lJ0Dqevswbq}E6Sjb;Cj#Gk(%$Gl6G
zX=l8D0o&o~dZumqUw?Dn{?W3aDy!+y()eBVQk@+i=J7;N=H35jny|j^re!4$Z+dFn
z5b>}K{W$OCgl(la(`G%gJinoAOJnlNRc(UI=0z4ucfB!Qa^pHfE#Jq+SDNb8qQNJ>
zO{_`sxG+!N#rnrj71P2%fq=H!C4mV&Ept_6?&ajYZ;>yPkzuP9pEBv<^eZ(?7v%k_
zD%1B~yI$zO=g6_U3KEu|xTf8@)We_05b3}D$>hgw2{|&YMTg`MvmEz+r2P7I{u7Ol
zkzS4BvKb~@Z(T5p{#$%4bn82Y>cc{Iy?Zuo^nc@d=k$lY!E;t}28xDiZpc1fa;KB~
zzsC~Km?McXr_NrwcE{;ZU6A+@q2o_O!Wb?rSRC=<u-o?i5p#CU(0q8#|3=GenawBu
z2zf_*Fn;`F_d`DB#a`;2T@{bpjIFn^{I^K@*=l_KZp!*a8TVZbPRSLQeV&%V@9ozp
z((~0p!*OGR4Y#t#wD3z7y3%?7&U~2o=i|Zt?K6D;R!G{f_Ab1$mN!$ydYQ<Q1KoRf
zc=RUdIPeHq6eP@M%;8WA@tAOip_%74=N8iqVY|(O&8A2_@LC`po_DIE#Ce&Iw)MJG
z86_4Eo-`OqJ9g;=Cn=t_UUz2Ck&{QyHq29v<N719WwGQ0=G{6WUL`LNO!tq~>eVXM
z3SAO5GfH(v=zoRltE2XoU42!O7}OefSkK3L+S>-b)Exy6jHYu(?+Sa(wWhlK-L#fq
zmS3tn5*Z$DV_JS8bdI^AXrs(BhiPldKgKl9*;Ul!<z~FnXol@%sTC8yyg0mkX;tlv
z4-YRMjtSGw-8$jwfm>YZ+-K$%lxbgFBA}(R*;C<A+b;GAl1pnGj<jrx%DwsM)#Afp
z+oP5qv1@pK{9RcM-;Sr7uB;78J$2{Fz0J=Z^eoNyt^9Xs>+QuZlIlx5CtJT-<}&{g
zlkJ2Huk#uHeaSklBg|oxt;5c4lz5OsUBttQTloyPBa84U{{=!OG8{+vU3n(166DwU
zc$jN<!h|EO5}zlmlDNBwQ%{fKzi>{2Zo!?2iu-pcFnfF#yL8cB_U|nxu0|f|1g6F*
z5+0qpLXQm%CD|ExHa2c~GoeR;Rm5P`))wg(J65=8FYbNJleJjHJml3&#i>5U9;)V|
zUAYUsMcmd{Xe_Ses1S5QW9ig8dEOcqADyx3xWFj(x47Y^)UCH0+P<hLFREh8TE56^
z$>&a!iC&q>`-1YcUh++M`sBCd`zr0k70ZM^aUEKJC#ze7eK*6Zmbm#^heam(JnY-o
zZ}G|hX6nq`Yd6yux~>i1^`y$jfN$N^je34M!5xMwznLdBAK<7eiYc18B<p00*$SQg
z#q13|r}=p2bZ_2eC;ZytJlomtHWy8pEEKI?{;vr;KmCh}#=|~_qpN$`)^7T8^Ok^+
z?@Ht5=uQzQ^OZ}dJ>zKV^Lpi`#krg5=%RNP(H2#k4=lcSd+NPa1)*9iXBW;jd(O{l
za+=pTqIUIjqcb}W^Y-UCuHS9magW<!>w+>v#g#{`PCwgJ7S<8I_Q&q6YOD6tZAe>v
zh%b62AJ3YdEn5F)ZTBvk{N>__3}K<1lcnq(mpt`WuUV73t)a!c!0>Oj$CfsO<g-Z@
zw{=1lyWJ-n+t`JBhiRRTe#!Zi|KyE+ZXp(%bt;Ww%e0cO@L28D_4z;NtXz{;R_bI<
zFUIwYU*(l2-<baCmW}4jwaa#g^`*^PnKiw}RWfqf_t#!_3r_$4@xmrNPssDfl?_e5
z3^t$ao946YB+u=zO>9!zZ!0B+tl7W!&f})z6_snckLw<j-G1`&yK~X?r^FU#ZPGcP
zI`dzlo5GyT6X^mkCY_tYSj431o4IbgPIEwCO|oJP3+JZUE??6aq@_KW**9=r+n8sT
zBjL)y&$^q@p`FLOYBF<vqUEGB8QRA`OSN+ZO}}sZRCjyTX{WH%H3mCX)D^T2zwdFA
zpU=Y^w%W=6>!r;$uUc8#C6c(Wx};3y_Fd}~ym*2re~MD0j)!xS%f+@+KNcAZdD=@o
znio=c)J?>b)%)j0pXraLEtZR1na4cycIK;y?SE8Ln5rHN#jaCPKAw~5R`B!xqzA`J
zR=OoVQ=Rfg;iXR7a`WCU1p%1@b02M4-ul5)aa~2nN0CDdcU3G)U|TsQBqUtgM>lI4
z$Hu3QD~b-aPG}UFSrof`mhx_O)>iIKS^T|Q+O$KKX@rLfa=v5vGAWu>Td}-C_+#JK
z$?~U`u&m&EEO=2({-CFR70V~icLp>3O4m)Ed#33^%&Mu*@<+K13I%(9?`ZZ<DS96B
z{f^UQj?eDfp77^;&2Uh9en^-9>Qe9gX!+E4Q|BB>Rn?d`bHxD#_YGE!`Kn8}`wwM(
z^=$K8nkZf}?Z}Ns(pNQAlGCd?GRzt;iEdlen!+$8yQxT9R_SQRUZGin-8<bD?pji1
z$9d}T{~bcUGd`?SHxKj=PcU1wXzp_B?K8w;&OGQ^a`0)WQKixat%^n4JbE47ZM<)8
z(3rY5veiCAI$3n)%J~b8Hs^SFW;bssy592T)Y`v46OI4QGCQcnV$gc_lHA`HI{Xg{
z=P@0W4iyrPlqo!D*q8Ytf=OJ>Lyz0I&HJ3{{ICgAAFNy~Z>n4Ea#E~si%#OvH+LAN
zRcD1w$(y^JS<$mxQ>xl0Md|EDR!ud5UFRods_lP#uCC|%>L;bVE@w^Wx!cVYYML-f
zVV%WEW7bKlQsiFjT)(2<yX)tjpF1A$7EW3em#v}ucCMQ4{ce@bcdGY%d1tz$nD_OK
zH?tQ#vt+g9|1D&<Rp`I@FJ8^Mcg6;Pgi`h&`q5$VJ=uHuhi^BUcb#?)SrMPgc*5@M
z)z*eN3J+#D7&i!TpJ`OeT^_W8d)oO;tV!x_efc&X0<l4==F-xxT!qeToHs&p^nPj^
zx37p?-g=y8mTy<sQR4~LXD%F()pX|hEi81oYm04~>@9{cQQ^KBU*^c|bZ&ehEVM}W
zMenMR$1>tg_d>E*XJ#Dfmg3x`DEhGUT)|x>$9FkmEP=e~zaCx{JNv<UgBGuzs)Pai
zye}%6G8XgST})1_PKhv!D!5c#d3!S31{Gc3hZ{o{UP^h+G{ZG5Gp^^0&<q91``r_d
zPMUTmMa@)8Yhv4_NH$CV@Krm`Jo*12$bZVLDMyuDCro`bDe&;Lm0=Z{b8hYF3uZ24
zE%1EE7QcPQ?qf!AUcb8Mr3h+hd@Ae`Tx#=r`K>&KvK`AT&5y6@li^rns&!8~PI10D
z!&+{+g;Bd==j7MUuUK2K?wgN?(PV>@u6oil4mr+OVO;6)KyP{^N7|&653(mu&wKZJ
zdE&pBW({@`+B5FWW)WH(@bzf+mFZKT$GthK%WAYFenBEv8q>Cm5trX4EX@_}I`8tf
zrQp%C-)`p?9BZ0z>rET4;nbHoOC8u$q)%)9xaB1|<JeW*??>*`Y_hsAiPclxishuw
zL#Jzd?`r8rN883$96n$0?eH8Hsc@%x`58AaX>U9B^}k8&TkX02m)~z*8u~6PG&<&~
z&mzHxJhruZC5DfMW^yW2F$v9RV3Pa)gHa_xfir;NDf^;^GujUiRQ?R>=SYfZxBFt?
zIg4qkieJDXp(7Kj4wu{)a%R;@4C`nPHqGNN_dLHW^_jM#ZkDZ<$m)WWZC-+p50}Xn
z&%S@{`RdDSo$93T+<VsMAhl8EW86}$DKm~tnVqxg=?R`G`NdDwBh=)43dDt~nlHY$
zi+Nv@IkkKZ>-p!8t<SuDcxrwQzl~m}`wkH<&ChHOpH2Edacua+5}+{2!E1@i4we<V
z|1^zM3fylTuMzV(D04<jMOX1%8WV$oqKMx^wKK7Di))k)v)oQM{LkaD$bz{sChv&P
z;_tK0JP$f^GK}ZuiH2=_k8j*aO4eD{nC7EU;_08Ym^bZlve63BN$W%-+?FiM)9Cq-
zGEc|))gxt(N4_%(l%t+bzVUcPm51FO2cLa9yIW52o>5XiGvVd4^==E&>?bu998vk7
z#_x8{r{|H**?n$(TTbwaFX7icl6~S7|I$OQzw#J-7~%^w9JtON63b`ONMKUp|74QS
zAUr`qEl|s#UqQrAQLXUeuSYfKmgoAsdvfVn^bTk5DJlj_61dFPUS6^2Yh3=OH7%P=
z1Zo3#uV%PK7Wo}1`gpa;|C+~P-zQ5F>I!upJlXqi35&-f#fqM7WzNlPhs>jR6#v&S
zTekUXHBC@#XudJ2StRRWDcb_2l4en}28(CTHUUklCQN2MUA7)d#Q{wTD#vfG=(g`k
z^3Mv``($<Xi}gSMT(qB|^x>bg<VEI^eJs%m%n$keIL!hzjV3A>eN>#C%E+1|Bx_Kp
zmaDiS)t!&)YnobIz~T)y2HrM&mzOM1lnzpmDq>>m;kIzpdyqNNBH>Dp{`@AFg-Wkf
zzb#$#iS@;l_mjNNOjhyG`r=Zgrr{sbdwSOTmNiM@?e8{grJ2o9mYQ0?J!8Tu8z$9t
zuep*vtlOGi6-`hLd#x3-%uXgn|3`D}i##2H7L9@xYj_Mzxjt4f9Lf2_&(FA;`N|p1
zXGTu{Zyb78#A*=S;&{nuYR~I-Hg*?*ppPs{7ey3U^bHxbiWuaB6?iTP&1O{<e<bwZ
z$-`!S15@%hvk83auNOX0y=)<17}c*U<jbJ$y-CP9vL$Ep6jo)&&6`X3x14ZVeC3Is
z+v+U`&THkY)vj>=)7Q*$PQ@d|+f2!0*}6dOj82iX!(y|1m7)@vw;kYBb8=X*JXPq>
z6c47z4chueht_;})%#Cy!@GRmi^971{2kXVI$!1UAz<Zxj!ngr7XROH^ns0ms%4?E
z=0sh+iAvwo891gmh+c3IXL1w_P+aD+Fd$i(KhH@zN5LpSn`bpM;|FHpJO-x=&Whev
zJZt{5IjOn6t^fbS-Ow(FwbbZ{Q;&j!#<P}BNA@hT{S^3o+k<ank0cj=G^uG$;%Ks%
zmaJfOOhKiMw{L=Z+F^Y)r&k^?TZ4G+vuUrnb;gC|oYS*_bw55DJn)J<=P{4ry#1~z
z>+UoeL@$|9a!lvU=MT>`9x4SgDV+bLqtBq{&!qF-P?ujxThPU{P~lIo>R*$9|H1*B
zp^@vJSg>%#_~c#|(r(|<!X0x_D@Nao=ai?!r3Df~UO_^i5;mLoE9a#JYeudKbl4mc
zyNFlexVS9SHIH_Qxt>#eI-j0mD2Ypv+^KxiOWVIgL@V)%RQXf|A=c2+JRP~C+oBq8
zPBQTB<I*U4W&4?X!vnte{}-87U%Gts6;}?^drgi@6@AJ29FG^MFa(Px#7uqgNn}Bc
z?)sRMr7`EH#-9Efd-rP0#Zn>VO|h4+#^`Q#RQHd+sl;g^?6t%x|JGB+gCWd6i}UBH
zFuNBjntfFHzRZwys{h&~7WWc0`Q$G>`O!grPpvZ4Ihmd{e43QDW$vBLrfs1XE~WRg
zdXn=6BGjKw>Pa?RRji=a<erni?u?J_)FhwdQ(P{b*4>=3-|3Xwqe)sn-mTf7q0KT$
z;Aod)!?DcYdT;eyX5CS8Xy{@Q;lF64FLXB4Z}+zBw_E(4Uhz9wmLt3%=jgVai{El?
zhUMP=mU}u(^Iyrs7u)=Ix_JD`;FeO-|Nn<6`F5KD!(ju4WKrcaSA-5J30@0&xMjkF
z5GJt?hKp5%E!PMC71NjV++1|g-ayE_Sl_oKSjQ$@Iqr~o{W<?t*@`DLGuRyhBBm=c
zsNUEj*gP*-JMU$4-_-3ThsAC!Y@OsRS<4z>$MRs)GnX#)>(>O{zgTAbm)X|eSz)5~
z?LXa7N*9=a8wc~6Fiv8q7CjI@`9R_J@amo7HQTqxUKfpfSQ>kAdd+Roxc67%ua?$c
z-5zuIdJO*thVLZ`d?k*;hbHKRJhhvuAhO2P)=wnx^2T2!dCb!oHBIx+i9XrTs+aOU
zc8g1#*vZc!Ay0Wcj72&cKe?L3eQ8e9ZSh_cwVzw(e?l|+@h11_ts)v}$2txt^(BkA
zIG6Idn@liBIniJ+;e51Pi^`|Rs=J=?S+nSA`L4OaufO$5_q8`2KPP6Tv}9fvy0A>s
zaOVPMA^wZUjv77;QB^j|c{nY1>EYayVhn$`=6>8#`2I@w)f+iSxAoq<)_Yx|@8*wQ
zWuv*v4j2owxD_qAB9<hoD{A0Vq@ZM)^wresLkYuMk%^zTD!rW|s_y+Q>2tkvsC#*5
zNVHi>b!n*oR$aR!v!tv3PDy4Lnt51U6})~HTZo0{CQohjQ^;$6p*M5sj>6`IaE_Ht
zrFI#KS&hpk^##<-oW8H+delMDO^<!teui;#%vgTNI4#?J!x7v6cUML>wCg_Uao+N^
zy`k%Y(uD_yzB+IJQNXFhspP=5EkSt7k;0AUOq;e>Z#L&rN#MTqK;^5Mr9o2Av=A$o
zFRG8(t?wRx@T1siwRzm-@3m*c7rgzt;Hsz}YZMb(W;}nh<LTwmC1LZ9sCu4z`ap~)
zX{pm?mLCRT!U<2OH$D&NjQH3ntS@rYCpmwTWcZsvt7}iacv6BxpDk*ga>!S!z$-}V
z*hIgu`w@*lb1YM)Z|112WJn8B@^IvSFq3oUEzuk=6SXGJ+(v(gBffW9lpieN<ZqQ-
z=wrj7+c}jf>!Xi?)q+`bO|`xirQgq5v5xEG11BS?jf<H};x+x13_lB<H{<$$Yk^$Z
z&+SVT7*9odY-<oXroh=SbJ?0rAt9!UR^QnK(-m?)n-;cIDV|>-!_Q!tBr1D;f!yaF
z>)jKi_>FXw*_{`Obll`!_CUyVvR0V2!G{@=&)s!@YHV8Z$a5J(F!z+NCbJS{ixip~
z)_woF-LarZ)q2v~M@7NCc?FY;9V@=OWR*mhPIfH1D)z`G$19yREBXY3RP0Q%8B2My
zW(Ec{^F|z;%I3PvPW9!jMyu?FaybX(0=|~XKi$jyHZjJ%JUVgj?S-Ous>|z7&MTKt
zcvu`c>-ell)-yl)9~dw#u$@w`Y3Q#xdHeV6{@2B;RU1Td9;%o>+^+H<VABPTzl(T<
z+L!+So*J+B(tJsiglEok4V|YmHXcoXJJaToO2(aCTTY4Yc)&2X_46O`0=0Gd=6Mag
z5((FrI=9}5JXX%|+<fAUIrFtv*S%OiPvw`I<(b20T#Pr!XgNr<upA3*J^q-3IXbRJ
z`%#;MQmA*;t>=eshlDTYb!$w@iBQaH6g@T5D)-^BJF`V6ToKQBbu8wf*nu9NI|<wQ
zSPxr1JyymY#F4apPmIq)4ySKDXP7SP-tf!dTYq7bn9zKdPjc6spOzRo{<Z(Vb-QkG
z&-pUO+(d4H2ZwSK4@WK(jgy`C{ANSO5i{??=a*I{YIsU)oRFGy;`;wT#~S~NmAp|D
ztk97um>>4O*5ho!|Lsq<IZAJrHOY4g$d@Xr7CyA@cvEn<^p{&oK6>V7!_y}fUA$KN
zNun_@Kt1qtU(LatM;`zC^58^dOMt{=FY{87y=985E9wQ8DyU1Vw!Uac)?z&r{4JX6
zSOe$JTYFzO^6t4Pw`8JdPU0$Sb&nYW`>H!HZvQP7=We3&=+Rq+nsZW@Hft>DVy`^F
zttQ-WZ@#6_uy~#O->0IMiX2~r)=vLwCRp^^WZ_hX4Y`fN61O<wzilqtc!b^cPT-#K
z`o%}WUwXTodAU}mWu?;Mc`J{-O<0xtXIGH)q1O?b3cogN3ehTZi8F{=Ao3$daz>=s
z1P9g2TIWg+Es~k{NupMYSKV+&;QyYu6_p?3U9y(BSFi;xel07tBY9q;;C1G+_nv2+
zdB6K(VcfQ3liscf?>M)8!QQ`(4|5&QM>d|nt*|UdG4sx1UbTH673IwA;;pKiQ*##b
zcI<F{mHqC9LHMO}-ySsy3g$8^2qu2~^@*+C=zr+#)PDvn|GRzf`mHsxYKY8FuPs*m
zt^EALoyv)kGv>G5P@fX{OfBh!TEkrKJ>PO1zc{NG=b84qOC5XJobpv`dq2~KuR29n
zoG(n^5c#q7k<j_Cm#wZISC8B|`~ODU(>ocH8V;B(Wt$=yP~aflD#T`zqY=<}fKf<Y
ztJ6a8@evi~Ng*LhiKiRFgd=s&%=ozE&{U1UO^W|bgH)v(mxOF{v3&K+#YoL~l2Y*b
zuz<$5Y+JHbQ++O$aYx_W?0QNw#At2aP3^-=k1!Z5U9(Fu_}I0zVKHxa{Ia@q_GVLT
z3D*^_@`h`jsjsFnMo;IK)RWov<*q?$lL)7RY}^$K1;?YELgL&t7J^1D?RoOy=U!MW
z5@MZgUHp$H=v0elv`gOIQ=L9p^O;UwkYNx`^S&vZ8Q<3XyY$U1b73W!124<mwq~7`
z+Me3Z{o*{6rIr7k!sL?1;BS6BdNmCvG;f*v%rRit6Fp&`d2mi;g~P8N8FjTCcjFpg
zXvS<SSa`m*+hL#5-*@s0;_Ipk7s(f_-FoDtV`{>7g}mJt!~Rdm*Y%t+*WrC+$E74u
z-?W>_;)|9<2(1g%kkPg}n3CkFu~q1{k43MCu#aA!%VlroE0=<F*G@@09UQpz5qIR0
zFB#t2+jczi5@%<;(KeC2@S&Tw`qaeB^-No@UYo>RvM^NAtEe%Qv&Q!z<4oZS_IoV7
zD>WOL9z|+2ibSWlE%KAU_bqYWbX5h$`E}cFE@!IQ<tbkqo%=9qk;<Wt2braIT$zmw
z?_9#CxZiukwJcXdA!9*ZSyW=}B)?sgJ*vJ=mo1m{UBI%sC-6vS`^&xCZ_J%~ac1Je
z?{8PFDoFU(CE63^bW5mb9qaO^8+Jau)|L~x%XJ3R+s24MCUL*nEi4+7O8!T7F)#6$
z)O~LeOY+ovMXQ7T@^-ep^A6!n@OQI%x1({Qf9F##>4d34G5eylGAC>*Qq8^)mO3Sf
zDNcqXLM@?*Q*3#ad}+nKYC#pjMv;_^#6l7A{=oin^P98oR!yH-BkaQEu;fOXZuza7
z88Qo&&Cg;_OEKQLapFelZ1r1zE@yB4nx@dY_}&%HjYm>nb<LVz_{u2#LgI}})!$~8
z-7R(Z4_R>HVdnQ(@1O-KammjouDw@sp-`k{%O%tDU;)M!bJm2fQ)k{kv*g|q`9|hf
z%ina^v!+Kk+b(vIZV?XjSW_+O81}Hb#!E2tacN(*@GhpaDYwI3aoJwd(BH@u9H=3>
z^M9*I>Yo>F2fV#brF!Rz_JqFuIWcpYL*}}ig$v#ZZVQ~V(tqzZ7SEiET-rBJSvJhs
z*5R`0>5<zrAAadO{PX$zwU^poEbKDU5-8K?^sl?t8@8=<?#t!(gBm7#X=O_0eR$oa
z8(h&<|7~Az+f1gPjr?}|?y_Z{)lTr%Qd+^HrN*IhDD=x6W7gQ0Sq%)gy5EYZ*gljw
z9k{&lq@&Pr$F7AbQIeGl=1yABy5ftA`mE+Do%2d3OX_-gJ<&Q9tQOS0xJFl*)44OY
zu3~z^$LIQuoM#f)f>(b&!e#uX^RoArhK(8<4#}l_m?5%d0jH*mBh%spR_mODT^c=$
z#NHgZDE7*c?fHL)mKJuK=&FEeW>XxV7))B2oFa8u%}-Fx&!~ESLsps8JWjsCJs%~P
zSbbk`R8m8E-@<@PYG0&}T|Fr()O@Qb?wR29ttu)%6~oloAIPqcP0M*=IQ!<EPcmjk
z6B;EGQ}0|co)ycPk)QnOu?}a;toN+xrkzT28XY($H)-fOb>-Y?SNWu9Ht(&GXk;84
zYtznCTLKjBD6zh**xY$7C0KDWL%g<|;%vi59I94I&Br849ouGhGUsvezc;(-H@9F#
zR<`p!`-Dj?hCC1C*IYQHR`Nx+{@0;dUstq><y_<o+j3ZJcEc%Sf6hgxzMOQ<4Z6&t
zvVl>P;hDqg%a^`0{q*r_E&0Es`gF&E03rKzD@7a{A6_{n@_FS9wyv0OCpoUIV>4?M
z*kE*L;UOU-qt=Z#H5-e$8<@Io9q2ieJ99?hAu*K-`FPHaWhz%YH}bd`@b7YUR_U5&
zBwS!K<-i8jH6O$@)07S~o;fjFcxIr9_K|hl@>*3sM!o1|dvIa3ZH|HX#V4zUl2>zA
z>TpHBP(NniCTQmO+CX%o)3L0F{Lwv1TuWBCI+_M{*CkzZ`P$JW@|tnenSCkTK9Or&
z1wLCe?)kW7Rq0Ca$104RJ#J|=jKP|G%ce2zIK+2iWxV#gpi9hG*A(wN(zoFVyVG5r
z)ka5LPfW`57H_;`X2Gf%|6`Mz&r7jlhW`I0+&{m4+Hf}F(j@(8rTKr-7R<eumc4gZ
z=gBjNT^^m)<>Z{P{>dZf`>&r)&-(JU!+nRk_nwcg>b1Y+^_V#>J=I(1D8ltD-$C7T
z(z)V!F>kM!t-fV3$;ViE{%INCh>H(<L^53}vw~f`Juc>$x_2|(b(zC5Q!X`%zjY16
zV}^exxr~~6L_IDz^|>eV`~J8g7nR}a6xk>u*83qeM)%|VuxD!KnNxf^n1m&7DcCD-
zJ;axHaLV-aPHp{Kt5!Wa?J?6(M(ZKVa^38@`$~bkf{Z4(ndqO5Uv9Tmbnh0H&?}M-
z)!NEJ?^mo^zw7jgDk-Nci}@H<s;~12^gXGr>XBHz=T_+dg1{`t1W}%|*Z00-^Ii48
z*R8Go`t9ITCfB$ot$3nhn%EifG-{F4rdvfet1s=&(^wy<@}R5e(h&v|$F}pXhD^*I
z4kAkwICb3|gp(zXO31v<E<4c8GsD))dfSAx&wH%;`X@F8Pf-$?c*99}%?ftutT&IO
zmRq?tiFCTXJ>g>OG_&9JPDA{Gkm*4tCwZ0Tb?AQJ`oLJQQ}wuE_q4Q?!aQYD92YPz
zYtAcrX3XFFkjZjo`PuEUNom{tJa4P!v9K6^)H`l0^HYVj@s@>?e+r{d>UoB+$oniT
ziqEWO?sP~y=kVO)wQ*3#s&$`=P5181pEmc*%&eHr3+C+oX~uW0LdEibtYS>5%ihqt
zA9BxBT`M`sG3!OvQMI7A-IrDUyWF%gBJ-EDGG;h(NinNG^|{iOzWmmvtm_{&{ycos
z<ktG?O7KSANkRQ#9USv*UO&HaV_&O=N4v8ByI@yN)d#mu_Qyv*yQa%l@bgTv-Ge#j
zPdqg)YX7xU>C3hi-oD+eU$2<=)tb+oTM#pU;==v@=W?1d7sRb<c=aLDI_1XSm;Y;K
zYR0&%Sl1+WN91sZNH8yhSkk{Q(^h44sU&c-G_6{7<NInK?(WOsYb#3j-WO^-7-(3p
z`_S;_WJ4*(&qkYG{&JbL{a@n-rM29JXLsJynjr9+=U>)b?L9qCzHbA#mu$#xJsKcs
z{NI%+ko(*-e(oK~$szo9&s?W%P1Z=s3v#f^P2l%fXm(}_i?o1@*Mz`NA0^%|bywJE
zv}j@Acd@j@Ybw9X#54-og$<hzUlcOz5SnsD=!9ay-cs&u1wzbi360Lecg0OkJ8`u&
zOK*3TZn<3B^CWQgwJN5IoEy^9gfGU1tnhd=xiz9fGWJHnhK=IiC%P*=H&W;*Fk_Xe
zRTN0QBzxtd?3#A7wrP?<>H^<H8H~1v6eZhUF&2z>=PG+F{=?9?Qn>m+VnTy(>@lap
z*=(`rTx0W>S=^dhRi@!oGs%CIyN;Kd^xVUidoGIn{qD<fRp?|hi#&6bhl6F}a@MaO
zvMbcx{x$qhQEQ5PbHR1r0?w7!ayCUoSTTt@vC8L7v~4L2Y|3y9cdIN*P?@bRFyCD8
z+>0J7!-6vvR_1}tOy&#=zo&8s%Fn8h6nftKmRU1whh24XftG2>_l=Be4{2^&UcIO(
zSofgpUZ$G=2W1~QdKE5Fdm3&l+!*5R>d(o*FzIoP2IGXzqsG=jnXN^_%O+)pEUh}Y
z(J`D=cpanDgU<p5DV8^lxK=Jxk~0%{@2+!ls$k|+*{x=J+rQ<1a%zyc9I#7_H79{z
zNPyLH3u{P;&AJ=DOB94=wg#M?oTW3VLTUr&-gKLMWv5lHf{PRLxi=fAY|uAY5&bW$
z`Tw>_bH)2u+Ix>Z{LdwJJu~^4GwWh0Zl`H7R}^MXYw0X$U4O8EX|e*RlXduuLP0Y-
zjf)-u+ogAh$Tu-caesEt>=61JsK&ckD_=>*eyf~7sieX5cAk(B2F{t?YBLWW=3IHL
zx@m{@zl&|lE_og_RPbn*O1arFJ<Md!!x9mv>5-ffg+Z19>7}JkY`4F(s5%N6Tx5$k
z_jPp=N?<iwa$w^BWx8t%0%kvm+@LrmYdh=tX=O}`tZx&Y4ozebc3?YoLugicW%x?r
zi!buc4SVMtOT9HibLoX@j<%{*nU-;tl0`4X)}^<mr?s+_No^8Wc;%t7_q)BRs+N^w
z|MtWgUN6OTj!SSGm@_@-|I+ebk?Cd)`;i(&A<-!+>D-c`Z;e9}7Ipf27@P=_RNLXR
z<4|Y^W0*99ut|yK%|g9Rg}$ar;c6#K<Zjm6FzaitbauUz>s4mc_@nXmLB-ZW-7SkE
zCOfLPUToJk4%^;1S9oJ||DxvLoz3QfOKxZg>`9gBNiI=(D&|sQ+*j1n%q-2btt9%D
zcj&L$`_~Ka|B#-<)Ymg*>h2Wty2GqFr<Q%XA^Vv_Zqby5DZ%YG4b9D#x3fk`$_vfe
z`ZJ`=SnZc$$>c}os~9~jl!N99+no=cbt2S*|C`1s6aPiqJWn36`H?<PyU3#Lh~q*F
z;rmxqW-3<*Gxp5B<tC;yclz<vaL(S_k^krVyb`<HDRjAF>Sd4BR*dt?3IyJ6tzFQz
z>S4vQlv8Pu%=W1hOlp(+E-noX=TLTEEPj!z|FTrS$PrnkW}`Mwbs@$D=H@eNxnw!C
z1kRPK=RVHe7g)<JIYX31N2F_>N@{FGs)Cqi>-NOXzc1ugEXy+Am?eC1{msHW`I|at
zKM8RgE}8mp(dX-nrhXM!R}t=-WW&oCrflNA=4ftP+Qwt%a=A`yzEzT*UP=nlis3hx
z>`@gk_3+%iRcyj0Z})bod&>(7Ivo!zR4|fmD^c8>@40MCaa!yQ6Z_Y#Hz&?o#V)+^
z;IjJ8Ey9Y+<0q@#{1|+-c>Se~Ui#)UCst0-E4R^E^<Q9ifYj1L^E-#z7>(NWv$rH?
z3ViI^oL4yOw&)rKrm*OO#N5+Px)Xixia2Q(mfbfJT5@5Q_NQ{;*F_=1L4nbxLI+rH
z&2*G|z43a5^p07*>5Qr-6=KoPq^h<Bg=w|Ek6LK{arMoQ^GY98ow}7-s@d1QDQH{D
z{BMo&dp4zeot~rdam)6LYXu9phTqQMJQd=t${63CB6W>zNn-8dNwu@ow$Gd`G&gu<
zbf}{Kv)$`vDyiQpeY<>v)~ULv)nNu7MV2)yb4=ZHXhziK;FU&UPCB2%=PWWeKbY_J
zN-W&d)Ad$#MXFftg65rBQ}>kircK<p?Ezbi0i#G2`w7Q+OJ)}w6aBx8C0LTZvBa}t
z8C#9rfm55AOk%U5XC19BtTb!mX+BulZ0#GJ;c2>i>O-%4Rx2+qT(4B6(Rj+XX_rCA
z5A&~8+s%tx0wg*Qdv*32OP+ka_|$U2MAw6of_v3IbqZfh{GugX{abJ8kL``KtL}Cu
zO;ToDb!pY5X3e$6y}@T#KmFeAeaOVPL$iECYOcf4#os#`oSB3UFs3rh%lnZ2F8N?0
z=jKAmy2zCpT^5^*v^~?^)_?gWHUC&_w9$IDMQfa=?-DpQVV$+{cZuC=gA{e#)hoNy
zk2o8-g)aVMF0}50@PX(|hM-h_&xL}Vg?TXsqLNuEQ+1UR52;>UZ_MPJdqH?s)PIll
zKQ;7jab=Y5z5RN_?cK@~75C}3M-|W5_^CvAj??l9la?%YXOj+KyDU+qI^oRH3;Xw6
zwQ}5&n)+c%%m&7o1hHC2fm%V;Z){6<n{H8YI`C|=u{YCngWuB@SRQ{dS=KmOz+rXk
zhdak(G*(`{dic!ILmt{%Ar3QSYNWNUpYE2Fty~de^+s&ROd*p(d5s!#h9Ym#Bi_AH
zX}m&(63b)ns@j$R)DU*f@>wG+bWz}#*n#bgrw;{Qx}svTNpO3(t5Cu7`en*V(I4hW
zd0)1eBip6O7`SHYRKcoS9hI@1av@3uyIIB34H;DmmKX=KMLRG>m@%qe;QCW`EZ2Zh
z=7Ch*#Q$gN0!3#&lG<f1xGYhYEl}a0(lycaoyWNA0+!F4)txAjr8@u40jq-w5?RY4
zKd<4dwQkAD{2no5ZIQ&^n?mw6!rQVh{8*LIyu8lDN$BeLkePGjg->0}ICkJvP@0jO
zlep1k-z(ciAL-_c`o4A2GZ6~k5~HrT%kuN_4fdYC9HvT6rH7kNpDqfKyj)aQQ0yRV
z;LDuO{r~seU(#2d?^>w@80@iP{C-Sj-GQ?wcvaUHoGlGhD=j>`$CW9#A@y4rn~J0T
z<yf`{bAmr`rZ3KpoojuQbxzvvjiyiEXt*)UY`z>j_rZ$rALk^GZ~LC3Fz3Pa=BRvE
zmKzegCu$@X>x<81yY*k@;LF2TR8;3MNGx4==x5=I-W578jg<s-9>#EaO6khT&6WL_
zSS?X;>ENwP6K@F_FItr2AoN(|@=Vz^;+NF43VXL-V2fxIWqT|h?I4qqB=CjtSjr`~
zatAi<WcIp^j~(8zh1MSBn)o;+;mnCSGTg#Cxdq&Nqr|5#j7$7$TeYkp<Lx0uR{JH9
z2hX=GEYDeCzA#j!a?&J@M~Um?SDn_b2-f)iV*b(g^;V7xN{ueheUy8x`;yASUA=#c
z`FE~-@Z#FSH6B$;4`M?Xv)T#&e<ig3n$VgzV&|NK6t#o{14a0L*RA^~m)$bOYmIP$
zO0ra^V8bJ+qi3R19n?w%B|bT`UCR8=zPj(&wg9F(jdx7e?N52cb@?vaHiNstmaNxn
z#j`dV<bBAG77#nfDt2?nZEfxC8Py7D)w|BuN$$N8C_Vk@V(VR=%AS7i_g}6&$hY|Z
zue&c_U0mC;`(=ptjT$a<uYx1&dpJ4kOn=o}Oa1vE-TPv!)v|;iC0Fi+${l*I9BkNn
zy8O1ERZVrW;yFiyB0Uwx%lzIAn!;yhc^Ksu351_Ky80aJLUFkxhK#pk*@PE}ZNKpN
zM#ZtU`wX@foJnE$l>X+C_g?YETJ|?zEHTJm5?2=e)k16Ub*Tj2Yst3P(md~F{$0mW
zcD*oW&${CW3ipbhZ4!8SJ2U;JLSS4W=fN-hVvqML{Z|kYag0%56;!Da|NSA|`1#Vb
zth~YrQ<|mM8u-Pgb8ioon$aUaL3WLsd@QTcl53CNha`XRoG7__+q*l9ELA$|7fRHx
zGduN<D_bQ<c=I0NKtqxLW_}?ovhF`M<e6S=@O`JPtf~ET|NhgAHWB)#B;NQxx)b2!
z86x(EZ<j#ep?qVDuSW5!ZGN1cajAFTCygCfo`z`MWBPJ!`Y{czH!;B-La`jyj<=6T
zMI64Zcb#wV!TWa6jSuIG9b7B9&g}P5ulI%T*0tD3Zv61dU1+bl)UAipdHKG~SuOm|
zR`9*$!-*LWox2~ZC~drae8ug556v(7KG^a?ai^s!U&BAK|B9>voNWJjWEM0$aBSh^
z^zv{>J0RP#{Zzq?q~oEIoT)K4Zys8BcCJzCwh0d&JxkO!&%Lu{)8Y$H4@PrE^A(h?
zPFronyq)L9&B+SW8AI3@?s8sv5wOI8eZs>>2FsmSFc<kv^)h(wy@q8m)6yza#pP=f
zk1Czv49mRg7wX+>`=Zq0)+#Pf-CE9RK9YJBr+pq8r5+YJAiTt9qLJfC5mgpmz8IST
zr-Mz7Y%DX*lr=cCGKy(y9`0K3oN+D3HK`k?LN$$ow|$s<Xv)faJEo;xU(0oE_2HeC
z&y(h@jYwqD%yCiw!ejO317j|Sl6=CP1x3>@RP!sS##ns~dz*7r+QX_^T5}J(ilWDV
zxzz`bOpxUFS>w_;={Uz_{u%4%?QNR7P2g71;nu?^g<Kx<TZU&SdF*fe68z!7%ZyDY
z7Fp}v%53;AwM9UCgI<V+vyp}7VdtPdi@Fpxr+9O(W4v~$;3&6e;;QwW%Q_0(B(s;i
zcRRt8rK9~>i`z-zBikgOg}PCRe3s8Rlvv&N?I@VU?KkI5!GhH`X9^fLta)?Dd8N^p
zRo%^dWHNM(g7-{NJTAt?ZOl38#vx0w$2_^`cHUT!H)Xn;V)D9YFTSjq+w8HqRYbGu
z_QwS*DjsVtxU_4PbbYN0_F&|jRFly#Ey>FxxXGf`Q!QX)mZsC}oTAqYI44TI&<P8*
z?O54(VupbJf32xjR~5C^Cau02Q95<AxnOOtuA=Da2a|V++*T0Ywe^U}<<rUiypz2)
ze|WWe{edY0XAJUQ2m2gnDOvJnL1FA8-4jQJ?xt9r?}=HrCCnk4RYx<dSZMwAPeR|q
zr#J02y1vt^QSZcQ4k3Qyk7YhiO}@{bT(_{B+q}cXZsO+RZDvt{=M>F6GMCClGuW<>
z{m|g8VYg9gvO^c|wg*mK-+y&UUTT*5T((ftr-)ODO?b<M^qX%s990TFUvx5~b%I+*
zc(~^guJB!9S6?k}o-uEi?xsgwLMybrcdm+T3b5gD6+NpmEq)98qb1IKSxY8HOWy9j
zV&}=eNT-v9$s^$e;}5$IoByAdC?s2W-|iG&_%P|ofs<>O_f*}OexSzt)AW7+E}YqP
z`kFxMIZ@&DXAfRZyBM@<ft`oR<&QC0dY8Usr*b#xhb(x?DKsO2`-bX)S~FHPjl-9x
zTd>vzH>D|Q95VBhRZ#0_klU&jK0&=pU5V4H^jBE_F1wX+#b>W-p4b@lFYA-*{I_1W
zH-u#WopWm1wdyr@j=$QqYT2&S&gzlIJR9?N+L|0!(NKt)qWB<3h=JkQj!eA*F773K
zmuD?|;u^1PH2>C|wypJR`{OwyZp2JD%yrSBy^%ezugI!w;|7N{EnJz*+5sn~z1oqj
zpw1#vJ}r622OgFJn;?%`5rx+2XW!WFPWJyFaqshCGX)MsRw?hsMdcF~-h9O8e0YlP
ztqX#Tybhh5H$6m@r%dLO<<<6Dz2MirOS5^r)_Om_E#P=y+QBnBH}(|A%&Nb$ttD^o
zafAPdPw}-**}gaQnX1r-2q%yB37k9$ss34BkC(ZMrKfWE<b>P~u?lSSUbfUn%;~11
zmFM<GhRjsH#gB7bl%?!Uef;$kkN)=fc#5%ur)&M?g@Wo+C%6<UmU~~Ersp_w>b;4L
z!N;sQF4&n0%yv1UWU{l_>(qidbH21Q3oAHFw}tO%cX%&mv2oVdJuBV94cBUh{!SCm
zDfL@1Q%^gllJkm2q}TKv!L@H!pZ#_1)4nGzr+sQB`Pqy9f5a9vB_PT&vpIH+==Oh^
z-qGEj(|53}XkZd6FblGXak|IuWM}CTXt*n!C6U>_;@C1Bzu>rjF0oTlX|2t|Nqm2W
zl$>_TJZa@fY~7=@Y=U6hQL8}i6=e=)4c8?3%D7(#9{Qy*|N9a(#@<U$#kf+}wMe9-
zE;qU-;@-NlS0p2Y>9&WanENENm=rD<j+0_$4MKW34?=uwIxcjS2^PIl$qbpSZL$01
zB#l~E&CDsjQnKdz1S`*mD@t+L-Of1p<7A<aV(P_fX-h8HYK6GFyK3I6E194$MO%@t
zP_xiuN9*@BIm|tih1$O@6*$G~QZ$!=>uFQ#Ns)tPlhdj><!w1b|DQOb&iOUzO$p!1
zUDLIeBs=(a=teoV-N^Ldn*XNPK}3O3FGjfX>Ymfie}dbrw=^l0Iq*u}acDbwb0N>-
zze!5^MvRjSygJWbR`Yu@ON%!*zshytt-v?4S9|g)Y>oUa_vFdV+Z$tE)_&1xmVUHC
z`rE{Sgp(KVIQ-$9+aTxSdY!W~^bFJDd%9QZ+dL=tsW{K%=l*2FD*Ql%XZGdh3posi
zMZGNX!uyOql}1^5a?2;DN4GE>S3F~Q?g$&x@`k1l3Rge&B(}eK@cc|_qp|Gi2RdSh
z8E*vG-@D0@Kf`*v+`}8uos2OxH7}%_4!AR>1SmBrnKWolNKgvCz`=RrjQszE|5>`J
z1;*?#7u!s{C-UrL?tI^}&4oSmWA~Fi>tiQBQh8^asgvTexYg>5V${mF7bBbE!@0Ue
zRJE`5=yyK8$*^Xj##N6%_m=+ZJH;WBnNF%c3p}uk=jQV18mhL7YbPn|GONWXht*D*
z=B0M}!B!(viFsTFmzVboUlz}En!jT8jit|(PBY7z?>y#qqWGb=K=_VCCY6j<mqXrd
zyd0NBRZm1pYdmT*UEZP;mE-L8!?^J%Z_AA7N>i&}YE8>dPOWZxR=%c1X=-tozv6bG
zYmxU(-be`KTrlCZLeuA&O3d2c>D_O)Oq!&cxYFm2*7Vb-MOHA}<ndN`!lu~opmJFH
z?4ppT|GO5hNRKy~v)rz2vE0-L2bU)opOTbO{LiHjU2)j`ZmN!nE{~pH;#^TX!=ESk
z^ZhobCok_6oyKc-w=W|raYtam2gC4fPgBK`gPxhaT9WMFA!KGCptj9r&*D<%`eRkk
zP1386G8Z_zpQ<X$jPp7h>@)A!M$Ij`PP#o;CTm{`a=p*VYq##9562?O{-t}@ecmv;
zb>qe>p0Y85+p1^nSC781{AE}2oLjq_CZ_oMX36<h+tfx+UCQ@z(ifkGExm8<J&tj*
zykoxhdho|tYMI(O7X_oYyGrd%ow4UsON`G6udKPRCq&n*P|ITaTo&NIhx=XEmL~4o
za;Mnt6g=M(vY99A{r@#iJ(ZKM1XzduPqV4*bz05wO-t+3_4fyoiz{EWr!EmIIT2OB
zywh^=+UGMis6;CYC2eRC5Ikh1@jp;xqA|}z<!w!tme;%`_kA=B<KkU#!zwp(QcWZ8
z?9*L;U#+fMwEXBK;kDTkA-{N3gcUEom?6$%)Lp#u+$Z%n6@82WyEfdItj;Z4Z6W(s
zfmQqTrs+2}eM;zV_3TiI;b(mztA5CA;Y2k(g~q-c9KDrlyOOo%Z|u<(n&r6Le5vG~
zMJoiRTkcu-L6vKw)Y614pI0u}y0Yv4s`&zpg2^}4&lE~+ylL{Fd-2Uv_BUsAzl~-J
zl0Fjb+!0dTsp#1dqRBTa$#G4Bq}%^ZTSG0mu0CoJ+~Tq|P)hrtiXY21>nC&BGI><0
z_TE#P8{D)s+;JMCL~C;9(n}tF%d+i`I2zVHwAwg<f%C)eZ9=@^1;(*g7R}?F^*C}~
z&T09UfQ?&DnjeamGVtm0sCHXAVQK`Yg~VzT`z!W|lA2~&%(YG&c|Ow`cJkFJb#!Xm
zYdy5kyRbp6W?ez%p((=L4nlL}ojdYu=IVCvKcBTz?DvWjjMHjQ?v3%3mo(PAqjW+h
zSmBW60XG$nrK=BKNKkk-LG0JUqthF99iO>NWA>)T741<oxEHOOx#q&!T_+7|_N=|C
zy=d0Nh9CpAnKL@V5>6(xC~Y&?9DQM?Me?yV9E-XB@ARBk!}%?G&+c2RPn<qtoTdJo
zS@U0!;K~a&JeA#ZFPrQM>bTIoGFxThCFLW^uXav;BhBZuQlU}b;-_#>3O}dBYPU&3
ztq1HIbiA(KvgLod*MADHjPZ;WPE$@xsCK33@K<e}?%-O>wCQ2PwhIEi94oxkL@b})
zSTrM~BYTqGXJ?L}lboW#$5wsm{H?Xc#rafBCD(1^Gu(@K*GL+q6k4SGw6BUXU{`if
z*=<pr!|iaVTl2--(wY^FO&asPm$be)EmO>+x?%dP%hO+;FwItRl)AjYH+kYzrMYvG
zn4c?HMQjniB)CHC6AwrA(e9rL))yTY_e?Jl>Xnz_)m3UeXXx4ZU%{i<#!2<gDY?i)
zs(-|$hHz}Vq&36EdgFyd+YYHcp5*2wqdZ-bYyO4K|2NK;dOOdN>8M!jay;6`%1eFM
zib?#PTpqKzEx4pxKI|2?Id{L=>-?MEEnR0XTkmDivOM!caK9u!=L41?6IbVw9yQIi
ze1Yn2UK|fYX6fD(@_RBVez)_3%@cNpIB80-My5>M)1p|l#5F9&O68+Oa89d}rT@i^
zuDfD5SU+$rD4drl!?9rJGXF-7XN<>KF7m(FskGGL7_aNpw2)(*6WtnCPpz|DYvjst
zT}!@dkHYp7yyj6myt&O%J<m>8n%o&}!Fgyy`=y>+S9<P#Su$<*!J7`ksuz~_{bzFA
z7IE6_Y*&J3_us<-Q;RrTFK+CLv1)z8YZ1s7-g5MEq}@i3<s8n6MhUG^5BQ|L1TJRy
z%d9-@>^ph-4;k5}OWzvSy`0h6yTIS)qvYQW?9m!wNo!l*u4oMlxVBJRaObW`Mo&67
zOLt`YO8vNWGFy4^%ZuARLl*COwY@oD&vKEKIZyNtcC9u1DYf#>7Hz@h=jWW_uDv|B
zc+!cg6%3bn_FSC3KhpM~=Ojg4>t8!$#aaDUHO|^RA*AWV);TklczcE4Z@xYyvHK*K
zD(j@{mo;@RFzAJbbg0;>C|7D7PMK^NEw}N(CGSSg7YjLO8|Xa_3VYl&J?w)-SnAcV
z4XjsQbpHLn<;Jckw>Oz?xp&X}x){0dmksx$S*tb#|37@;wuJApxdu#A`dWe~#<!TL
z7xp`~YOIkJ50&oMJ+Y<RXjfx^;i(Eeg<rm0ySBEhwO@15XVF#Bj@XH<fd~D2`DAkV
z3a4laB)PP0S@~wlG=ZXp_Z~QgIb4lW2)n-WTI5v$b>05|hAYorn7b`__oK)ONo((<
z#D<+oonEn0Gc2J!>O)N5n)bLA?C~$y<0Tp{e`T-FX!UIn(#x3fJ!{Y0-mNAjR}81z
zJ?6Y6FY54wrm0(U!n1DP*idns?PlQIt*104wicYczL!zu<PF7e=Sktkd>0S+PMKnR
zeXG#r&cM6Pm$$5S$!gr<{hw=_MdPI}K2uaU=EnaFzxhL9<Bl0=!F@BPDjr$5^ypjN
zbA}x(7TU8J+`?DfnZ^6y;FOycfnm(+S}%WP4R__JmuRSyXke>*(72(UZQ8N_DjY^@
zeVP*XZ@#g8nW6fwCO5X)%lek5Za7F;t42M&xxemH@TcDEjRqSe(iEE~36`I^wdU+>
zMu!E>GiC@^A3c)Fao~n~6YHAyqTI8!A8u21uaNQlePP#kL)WK^Z(rCT{MbOTW|d#i
zCX4dK_&vFbp{ZBHd$;LM2vuS+zA9~fuc5ywV#XJNqn~56x|C+Nr<|LWbS{kHZdmTs
zOK)47Cp@@Zn$SAoL1RWj(+YMYiPr0(T*m+BtlrGrmHCzXf6Wul!bt}g&&;~(>FZ{*
z<Uxw5*M*%z-T9_#Hw4VG>^Z%8;>NTiT$?Ti_zQDCo~yj<2E&Yl;m=u@$Yr`doqVsH
z@y3SIyPKEX-6(Nz)dZRKjNLc8p56+Yywh^!N1mmDQr<5nPxqGQ2=z))7B+gNvn0Yv
zCc0s6{7oxYF7cu~9@%TxJ8h?*zQGVwaC4Sx!n+OZ@jni-nLUZ0_+W-A>y1(llPeSE
zux|LdXx_`8o}rDrA0GO2t1jlb<-a6WesL+sZh^R61-DWPZwXzRW83OFH*&*Dj)}Vq
zCOy5xGqG(2vz0=d<F3WRw!3n!=>9mssWkti;xmDi@FLy=w*UVueWqpRzUM}-=~c-|
zk+LO<GncyC{Oeh=TH?Zma}_~BGe1cj&fPab^D!es<P;v`)!%yMVplDV3UrlHQSD&R
z%&?2!@VrGK<%%i?r`g>`gV$>&JaBC6(#X6!SNCv^skV<=$L7svjJCwySt>B6bY_?)
z*WItGuD{bSHf~8gc(Ei_$83&RXxh_K+qm<)z4l(5Eq`(Lg9!@-v!=7ybbJ%Iar5=P
z=-<=-6mh4BIo#Z-T5)Bru3qr>tt%QWf_}Y>2wu~w+tKjbRrrjB$n7(NPjvWoKk#h3
zcfxt!gok1&VppDP-pn>S`(SsKjn$_I&yU?rw@EZN-PYVZYtH4Vm(`XBTZ%3CFOk^o
z?E7ijV%EtIgD&i_G#2}&)}>p(Kk?Yp(7P|!yUNIVi64BiqQ>*+K0A)2ye;0TiuQL;
zneN<JclYHkS0QN`?u!gDQ_eZQO66W5xo%0@hyKn@&w|5@E_h}1T?#nVGimaH`JW3G
zS9N}!leOX8%DNB9Puw`1!m>H?vNy@`1~K%uoz!;QHjncm_r?V&^5VHgx1JQ-I+<Fs
zKcA^6aGmXSmihmZ-v0=^9x~4+Rd2S<l$<pa0%is>#oXgm{&}pjhV%IorcV=Q$1$a8
z>@<Jy_1^8<)7KR~jemLZqk-YkDc2t!SpI*u`^yP&<$^xCOzVEX)iv{E(!FrQe>Hn}
zjWf4d7tjA`cTYc&D=_kzZ}4ep#mkv&8%*TGeM)a1x9tA^djIjpj9oLjvrZ)iwYsK8
zDs~=rZ2I)#^iJ*r5(_jPE?Oo|f3oiD&H%$N5|22FcugOj310c-WcCgoQ^(DjzBeNz
zG&L5sgiLz9;F;JFNAZ{i^UoD$oIm)y$3P|c)s~Q;?kA23D!h)F?Y#3}`9&QHer&bz
zb=I#7S2y0U(v)DXv&>_&<EUTJI;%)f{O2{lcmDG}sCqQ6-}7x*gFe@t&TC5^l>aXY
z34Pho^8T8!NY0r*qN@s)o6q5r`8sLygVpV=skfXaYPsiXi+8_m?)N=0qvvIjvf-ph
z7h*bQHpn&hi-m4syD9YkKVx^l`TTp=LW&j5PW}1B^T$M)^-=l6`|T?0{@&o-q-}Ta
zhOX1}@>YrDE#KPN<&szGKdD;4TK3|h)cy3GPRHiI?G9ZybDsv69Q*Z9y$fs~6Be`_
z^L)DA$n;Ue5~=z|nI(cV3V0+EF9wUBdm;Jf*aU`d0mEZKGww%zue@QWw~JAUUxzDV
z;sUku>hsPVr9uYm((V)D(_)x5?~wJ@pP$roEAv@Emh$1-lBaGjk6y6i@v%1?A`7xu
z%{rP`c1W(v`@T^_YfVRkh6HofyziT)eXKiBC075SZbfV9g4T!ft)UP2>OZiSWh`~M
zCtSzYeb`uP^QF%0)z{{pc>U`7QulwIUjOg?>ub=Dz4Rnh@Kf&7n*m~{PVIb`z&QVO
zb&lrr;Mm}ov%|0a(hu&uGGj{qLOr*FhK#M5434T?Q*s_Gn!v5z$|7r~qEV<M*_o<S
zrgb>eL9<WD+K)%GNaf@tMlZeqg}_5znki8Wdo+vHd?y;*_E>YG>w(5Zi}FJ%rVP){
zF}`)VB^0$et!bj~bUz;@N8yl}fB&qNI(jPJzUkkqtBb#zzHX@cS2iihwCl=&Ugi@y
z4~<+^#5+sn&&^dT_m10H_w|)*W%+>ztJ*xT`T9J2ceGncw!Nih$FkcKT2;f=S^Zda
zp<7n<w(NzQj~V8$oYoM18?1Eh+S2&hb-Gffxl7l29Q=RAcY^2{M%UW=_nZtoPjOZ~
zR+3kk!yVHks@>EVS($wF$~os{=l)GhcynVir`m*7SFE#x?p^fTvo<>EqS@Zdo0F6$
z^1lDE^6}<6-Fi*Qg=@bFw?tcS;|b_?*|z0yn+T_eAwwav#{w2dK`)huUaX-fCV5E3
zX<Tep^^|B8`+DXHzh}(CimQ=}cpUh(-(0!mufOl4#VL)njmB)&YbH3I(n#47I<bI-
z*Yb&rw1&FiBt_pwl}U#jceYM+vl9_dj4p7P5gQRVbB5FDLdLYDC?T^OsVb>$H$Cz~
zQ)ZSdsk|cSu}bhtpnBn|$g6XT>#i@Kf1~qSl=i!=K`$13-FhfmJLSL0j!PmkN48E_
z;vHKOuDN36vFGy*ZA~Y#aIO$p<uccfHEVH6+n3wRRxewA#V_~V^!vWfybke^9uqey
ztX^&Qb&5!kM8u`gG~U}+8N$B?pVqRzXw+c;rbmfOdo_=Om+sdMhm<-OHB8mjwO+fG
zGulB*{8?PcsaEm$(vMg55?=c;uAjN^r|2!`Z||&xM3?gl#O@bktcl$>xwP@Ui+C<e
z<iX6aDQb?HRmTFQZ=E!oxJ;u|Wy+j&2Q#JfoIkP^<tuJ?N^PlcmAo|}^$x4{{(#Io
zZHL+Ci_PA@Y-&^R^0rp<B>`)WY?HrW$jG7KXs~mRe6{yp_emFp|Gt@9X)v)OVd1X-
zei?}#TAfEqJ)%y3S=cHhP`*keCA)A+r&`rjZriOpc5vzH-8$i?e=_UoZH?7eg1a@6
zXSnuDtp0MiPa^5*i#H6-{0I0X?X|XZPKeG4Qkfd;V{lQVn0G?>bLl6hPZ|9C)}>7H
zXzTnXwra7R!!(w?=Um^c7uFP<`t6*xgUhbVEU%0IA9{6TX=&c}i*qe`%|4$gma}*G
zeD?iL*^kV@`hjKZL~pe%x+U$!^*mKtTy06zJkON@*IG*qzcTO~u=DcQ6mVeT^e_nc
z`n`C0NU3)>i%6NZtg4FVPT7AMn>KU0F3e<+yryOHQ8z53Dau%?HR4;^wA|R!ZpUA>
zYy5EI_Ko>}I$tqGu;-GHp29&@pPH9<xGb+*irw<#ik@(YEk$Cz^uMRAU%1Z9cx9^g
z=j#NY-k;*G_dcyx6x*QkbhXdBiuMgQANzt<bxNeKF&A3;<8_*-ac{u4@_n~u-X_12
z<nrNhvVY33tmaQspJ2L?@zT(ojtK{^eDQFa*Qt=8$y3#tuU6!~;!V2-%fkL|4<>6Y
zUdg4FW6)o>%4q8X&J$vx#sW>ZHY^BkZE@DQc;SZIp{<)e9rf5hinLg%2&sA~a0xlY
zhyS|RAIR`dyZLd~_Fsl=fnN^Et4SSZNMqSicOjW)^#@mjn-@jHI+kjmZW7d%GiN&)
zsyEAdR(FA!Im2U#=dJ(e9lm46$+Sy#>bf#Nt-BVjmv=oa@T`fFlAkFyNu@dTz0yJB
zIFZ*c#d;5X_A6@g`mEEoy!M@Or{|Ww1$UA!Xlf{MvY(V$lA%6%Nv}&S)7_Bs9*znQ
zzE}KLF->7g7d^(*xkPNujYFQFrZ#X!T#>7f@zRP6+?rguRBp9EtG4Bi$?0$3>g-`j
z3)^SJu<FHHxl<+r2EDEVLSKEg)z`=rg>@XuY&((@JYk`TnYpvrBZIh>hj-$(?UIpW
zdlpw~bZO7D!<(kNrYW_440mG>FxcqybZy?#yL*JpCaH&QG<mFeuY&29zxj{Q;Kz>h
zE6<4^yR+qxXp!Smp(mo^a#1G@c)9<d{Kq6B+%@qSuhPkzajKJ!Pu#%I`F1kL?@0$1
zOcYsa?kM@hqvhK1lN`B8cN)+7efVrL<x4wnYu4h*zE77HPkpl`J-YMSmS<Z#D}1x{
zHfs3TwJ0{plsgB#>~y*Dj6Z&j(zLG^5;iS)dWvyPV!lC9m~F<z?Nw(yRC0`8G1LV1
z-?lKG6_cP9zov6%_9OAxEC=U^Y&dx~r>J<A9?NqR?wDOyVqeBZ3f(VCE7#xj=(S$?
z&ATodbCz2&tx53k(RTk}xrZ-*iE@5R(yU*SLN7e^Yc;23L}wQ5UE9PL#PzUYiHMM(
zkx7fCvY+pls6XlqXY#pUG~UWyxGC_D=h?0mp{Hv9Ur67oYx30joN3RL?H9PB8*<gG
zm8BnCxh<L<n;JQJiMvOgwWRP~^O(m;T9S{hE-(-OJS$)B<m}~E&gENQuQV^Y9(QSm
z+ozwLUhl75DiUL!J8dgZLQd|-4#D6_FDF#^{+H^VeEUN3`xX8EGR?~3p_g6#&$n^w
zNS7v^Tjge?X}EUQr=F$OITMX86mHwp`Z#&b=bJM;ZaRKdKAGqJ$^BicN}xpSt%+|V
zHr`|P*lw}%gi^HOf~uG;vRgCS9sHz)`Ife*2~B9yjd3(vVdCNB*>OOv=HXN$_SMr3
z4Yw!WSiQsK{->E!W`9>RGY`%AX1>2@o<sfbxA*JIi`5>>{IFs?`~Sf%of~tx%1<ou
z`<v0|+9vF=$NQkxx(p?|HHW1R3w!#^`DksMfAoLL<o?y|H+u^XUC?y5d>_$wb5c0}
z3C{=0UU7V%C!U$bd*!b0+9}gKf|gV(ZHovA-cxqq{hTkSX2^&x7dUZx){bbEG&`-3
z8)4k0-g<HUdy7M@m$&Rp(JN*V{JqcGJ#_W05377{tvCBQ=jxFU0#_$1W*58cl?hqz
zbA>T`v4eQ+`raw$9|jy*D;&r=De|;g`fOE8N0Z;-`>Ikl{Oq<#{piUSJlpEfhnh1Y
zCog9m{XJ1cd}40g_REnM+FZ6cD7<BGdA;vid-_!O=u`Q(n2+C<+x@INC3jVYywIKh
z&$nIQs??gtp1{9Yf<tYqf@=lahA)1396N;{MLBJMJCD~RBPZCcGsGbxwsXU7m51rK
zUDmyx`q5wI)zkOVn)5XCCBx30{WGtAzp_hNUH<HU4D1{De!oAo^>D!35YE>NZuLy<
zY!;MyXEJf`of(H&{C$4J9*FTuHSqV@5OFLq!8SBOHuk~Ui#%sEBv&O&n_T%|6@vqd
ziO0r|b37&}oH0FnPV}bBhZCZbtU&^@|1MnkX4u-yF>}L#DZiH9_u%2a6A~MFgSo%M
z`{|!k^G;s4u6dBfCQ-z%fvM%LX{~IX??K@(QQotWizl3${3cRjkx#_Qx$`%iUhsE<
zfb2ynKanM=3je)0<YGc^Jn)T@jk@+_OO(eNk#h%+WoLxRt#jk(ire3D@2Te;#Z@Pt
zcD5+(b4}jDDdgH8ztK(d0n3`uMVBlj+hUKrS}W(cXI@6<G!F&G^Dh(wR)k%iv52RO
z<=xr1s*qL%%SoFgFDh+3*dR0C#piae%-qC;*_ys?-*Tfjtm$=8=)Qg7P~U`(-5bS!
z79PF&(btDzq5hWUs6F@h1d9Hf%bV<a!p!IzgU5;4FYnjioo+T!`Sw#+%ZZcUwoHB<
z)A{Pejp+t4Hq#bLce;r0NVI)=Bs#?{@rAI++hBX9DJ~6jK65N_4{1Bz8^fvf#OdDo
z*Ng|&3Et#7mf*7@f#cmNZm!n<=e91=a`L?p=_@T0u-rwwL^bfDXG_P51mi4LZHIu(
z6-SM@uHF{uh`iCEF+<$Mi=}hMiH}PqqYLi(TPcd%lGh4U40L)J`093u<@~v(r$RN|
zgKq`Q?@bgq7%Jn}^5R5d@t&in3M7stN<=xj{yZV-dEzP)7hA}k<kTZ)UOx1BUV3cJ
z+K6nKxMMX6(cidLyW~{=oeSboipmu9%((XA@985w_Y`Kii8i+VW@t?Ea?#i{PyL<X
z+N&-XJ)V5{>8ALw{k`vzcZE*^5|-6$NXp*pdg6?j{M|=ybmKoK3T;chS)4jMyH0tV
zYIukxhi8uTyPr<IYnW^qF6;bs)_-%NPi6A|O?yuGTx7a;$HQa7e4hwe?K_uvG8@$j
z?rtkgZjoszHM&-sCcob@-lFYp?aT1zDVOUcXILzJ*tqr_<EnHft_4wZRqc|FwG_tr
z`YdpdVPjn;;b?L`q2<o;g;I6b1fAIyH8{1NeG|*_iAyMl@xou8M!#h~2OjWjTHx;G
zvd|^c<IN+FZ=HOTT!cIrd4!b9Tu<M*(!6EHQLQzHtRzloZD{uYxU9JEP<RTrz@;Mw
zLDD*bq5)owW?74wau)X+T?=%3xHnf(q$zf>hN}3#CG)R#S4XA#%vreL?}Yaf6J-+T
z^e^?4vbr_-+|*9rX{mvNs}5WY|20ego1((0V-J4@$8TEsKV=1flHbG`bw{2rb=S1G
zcks_FS01HYfko-O4swhKUN2-3P-zf5Bj&3iz<Z7*{gZ$GmIn)^+G-TeW>09`a`Uij
z;}nmp&w{q7{nKFNvb(!;qPXr24%;tIwj7hIdk<RO={4pGm7l1kZ+lu|f@DZbyocpE
zwG2iL&qkGibhD)AjvT8UKfGA~YX1J8(RH1R3u`8aRZf0sll@x3bDEBXwbYzLnQ~Ew
z9v=NW*=-7=Q`eCW-X->blO5SsG%w2DCDnGuaf*-4tMWao=Y*-H+O5hwaO7O=8IL#D
z{!|2AE<MZF#J<jxF<|P=hznOjO9Jj^FlifH&|nDmyLi}V?yCEHJp8oy{(G*N?<Y0?
z<4vZ%EJu+)r^GB2Z{{ox`LV+3r^bpoZvyV!zW#N(m{jK6Ld9DTB@=9J)R_j#@UgXs
z-kPApc4Y3={U=15o}G4^vUJ*ss~5vm58M!qc&6;RMzy?wCBgNWm#?nes?23MQnL;%
zb+MS2pedQSX~BKpT=r@031Vsc733W6%<|w^m9BVDi!<IwWtGb>e$Oj%u58hYkzs)W
z&r(B=Ecw-zdFuf~iC1B0YmQe-Th<Fr&lVovyBiZ+b#6{Nyy%t25ssBcZ@NAm$=h|^
z@ZORt&P3mY(;9DHt#@hAn%f&9(c&bydc&@FrA?DIHeLGjG`#-f^!ksioBLQATisqO
zJoqoEmpk*sLYE8YT24Nn>UAW-kL8O6oBAHnuCEau7bg4Y_}8t{b{1ND`P}kP>&|Ch
z%}xL3wrAT_?nOuH{+y{^;eI#H_w!pN&pq!`zHoW{;aaxtzW>n<`!~H26J`dr>J=Q~
z?frMs;MZaM4=LI`hj=_2+r3zsC+qz6U|pdh!cv@knk!RGN>l7q&e=zjQa=S&&q?H%
z+%k32%T*3ipLeOpNb2r&k(&AI^uH<hmL7E}pRzJLN7pyw)E7Vfvo%uvsw!SG&l2C=
zZS-?l&avuu2^W)!BmXIu%y0f1R*J2EsQcbjW39tRVg1%6pYD2JIkdFRMb~8de3QKW
zX}+b~@|<$BDm)zjKY1l&{pQ(5ndGXoPwsN8sJ`m7?p+p3&+N>Pg%v*v<-L!J`J8(r
z^l6)Ip6D;}7d^{88ZKL;p4_rbc$$mX%njYsdEN(U^uNn!u@z$Ba@Kv^=~BPy#!jJr
zm9+Poi;Xf}4~Vgx{_*L&%Ow}ru=8$;2E6BT_i;_Q^zrp?gV+O$j(Fq={pHwvaj6SK
z0q=?n{vuZn6-}FY^pcDAU9PLT6N>s+jOWij=66K*AM?+zy!n4Rgl^3gYSH=gcAD)U
zCryK{0Jnz$=j6|XO`W~9n>R5u-1Mc*C6<=EYZ(5m`II*!o-5;LLc}KzDQ-Wx7gLY1
z2EApU*5cyplbs{;;wjJkJ1=iNTmC=!OX@7Md7px|#JRH1Z#X8$msJt6YD475#6Rw-
zi>_&ZpCfwIJvUUNx<zg3rm%Vyuh}NT+9iff0xYUq53?P4U#!V+J?HVw0QnXP7q1$R
zm1p|@oatxzoV|SB+58CO-Cqx{^(z(#`dl9I`Dj~9sPW<z&r~F&T^A~Ne4F63D<vpL
zh{fXF?M?sYyD&U>xkWiK!12SebCq3FBVX}-+PBbtH|Lctp0#nOj?J7gcSn@(f(Whc
zH{^5k-z{`W{_n}e&>+Ha_wNa#)oa*wgiXXAv;DrtG)1u8!|i^HLx({<*Yy+rhTUCE
zEl;vfxoB-Y6n6hkcyj5_*f+OwHtYRtwykNc=AHY$YQiCf;<*7`+ge1Pig0C~`E)zv
z>IBQ1eiHX&r=Mf7%RaeJrL8UWrIgdRa+6#2UbhbI;Cv9cUhwON6S;xB{YT{a8mjzD
z-c`zFr|2H*Sm0XcqP^$k2Hm&~u|6d=Vhme!e=}G#-&pOncv_~&Y^K<4``n(SE;@ff
zVRp>~#}i-L^Dnq-$7pr32uMtJyjjz`qQfOvTF*#bL~1jaR<E@7omDx)kH1en5|+O;
zbbg1+lQPEm>9#)(wN5WJW;G8kOy5!YxMTmbz{x3|JmPPw(#>;)51JZThrN#N<~(fM
z(h}M#9X97k;I!&hsv7fWxGSXa{ojz(T>0(PoPwB;jXRCJ!d?I0J7!cn{YzfJbhlTE
z1$)ENU$546xanVgRzm;Xyvr@~UPlS-2>B`z`b3KRr{spsXMTi#*|w$9Slgp-O2AZ+
zhBxiTcmJ^Itn6VaT+veKywF_k>1m_yVtT4J`?=3;I<+rmQv_@7g<qxJoN{fcnUZ#a
z@yQ7}DkXAOa?y8Ry40+l#(#}h++^+v)g^6fzyG={%e6c4n2S@f{-3TSCLWfpcke9t
zArlj(veiDs?$Vb<PgpL*zG{&1u%6)9R;BDIBffLG%RFnnwg`@$k)}Q0Idd=X&_A=g
zxBmAQ-!qTT%w9OlciD979IyNRhtD54e&oiHW!8_QD~cwY{d~7qXJ^qKzUczT{{P^}
zXXJbTF5yq+oy#m{$J}DJ9M#WFn{sf@o#dY{3T8}yP-gx+aYfDVjx9`g+Y=|~H(Cer
zsJ)u~{5a1$F~2qXo+_-_%dOAkMhcb8%xyN!UH>Cjo3Ho%wngQeJ?<Xc;jvug&8wGP
zEbUbuo`1fiWK`{xFW)biIq6np*xl|D{nj{PH}L|-vdJ4ZUT=AD_RLDrGjm1Gq|bSJ
zW{UaQBj)SBcUJ8_Ca)z|$#YT8%0!`T!F25{J};`~zWK$lW?u57Bd<E$Jyj~?H|9?a
zwUfWqQm=AKajMI1w+dfj`3nrPTYOu#bnbRqYwFDzve>0T>_&d4#pyl259?pqp<7eE
zQ0LMUkCXq)ZRYQs+|tTmS=7iC^x3jw!Z$U=noUeHFI=)cF8F;hww$uWr^Nkd%7QuD
zy=1<4#kfW6Y(JQOOjhOiLC%xHDyJ6jPf~7BY=6HhW#1k-x4<X7b6#ufOqzI}`{=db
zmXCjBUuJY^V)Wd0McUxOeaF`SZ~lHheqp|C{a3YLS>fHG+CfXqdM|ErJvB{(d69?Z
zj3*03R5e9srK}WCJJrmUpe0iA;Tgj$)5>pOUKTGu$EYpp6OeS}g@~r-(wqrKsaz*?
zgLn8eo^(~bHa~j3)}2S2xu-U#ZkeQN+>j&LnsvOWG<x#e+j_QFi=3h_>z&!bzV({Q
z(hu(&%$D-;y_;Fi@Q^X>|Ftr!7i|Tahr5Keb)G(Zf6t7|T)nJ#Lcs&GC05?gJhSv)
zoHJln59^M6n=x-S_kPzPVTLm-ERlO<FT6^4e{X*xllr^g&p$BMw_eXV`Yd1jmi_rd
zD#ClMv|k*^y}IUVU(cEOj@4cbJ13=|`)s57Cd5Jb?*)#aoWGx+^6%HL-u9+8Uv=)3
z*V`llx*20uY1+-;3lZ4a{3!U*-i1v{`Wj!WPMvg`lThTpkezo*f~uxUgx8U!%Vs<{
zB4Yl6bG4|YM1rf{wup}kQt3}#yS5&YYUth<affT;R<;*SlZ^s4ubyHwAw~P-o@c9g
zmMPRP>QO(nCP(13&el7VPV4QxW5mh1FX+FX#gh{u9y^WwjW6py=y~1AJHPF9&y@dc
zD_Yf~|D6)Bw3B`^*=DxNipe5jZi=bhhE9{5x+l8*dg-8f<a6ew`C*?vpZ2M3&Tm@O
z_bX&2?;@$vRjW2wg$B>tXZ1pRW5Cnb3pQT~T5Yv`$(>Wj*YH(quTy{DRl1WgYpv;4
zZO7mf8<Vdty0rajnD<)sixI1HH=JD-t!H-SN|vv|(TwD2TeIt1y%&1%@8VTc5t`7{
zQ?|im-PC*czIg6h`>fUTPV>9c;vE*XyWiQrFuQ$jL9^bj&2!H(J<WQ!a8u5V)dIXF
z721=<N-CE280K|84lxgTv6r*7tKi8*^(8M;=6S5kR98L2{Qu)&A$OOFX`+cC9mi%T
zo(fzjZu#St_L^-kHoD98`h7jAWI6Hcb>(cYGKLlWPCO@1aZI*)HA|`a(&Ke||4F^^
z5%x&vesRdIBie+qds^<6>+e=h4P4f``^<xPp&sWM4c->LpT)o$`Tl_auakEgTULoA
z9gM77{eIv1ds$9Prdeq%30$r9b@{#`*LCYd`@dza+0+tx;`)vSwSUxizq<5QXHC}8
zsjJWKbkm76K46;DaQ>KU_XeYHLcBkYq!_=w_2Bdb-eqfx?Z187GW{)Q%zcKAgvnQ*
z?cJ?rdhyJ5%k5j{OYgeB?DO_z*IP}kcfDoXuypRb9`6XQPd>L!+I+kAy6EAr^hy6O
z9PGRAH9=cETY$lw)uQcTiaMvqgX1FVQ$Ciwa8{f(vnywt()migblpc6d)YR$$`n3q
zp8t7Hn91?U(Jx&+axZ#cl_@se@1UQ}esC6#?%}Ch7V@=QUP<&9SRlagrj2Lqfy)g0
zu5X)X^4#Z^_ids3-<+OvJkvc;D3)Gj>@wrVBGKRjQy%ABTCSJH(Zjo@vx1A=Qok+G
zy!*D|E1Rn;m-sk_1W#OiF+g><&)XHvyR;t0rfuD=H@CpM?4@_i2Bj0?v;0e!T%MZz
zjL9_ls6ZyO!<qPVm!y&-y$zN)-cIXT;<VjyV$@8Xdl7jNy6dJCDhN%Oo!?a<Q)@fX
zxTN{1c4g9knaZojbkjR8mobFQu-baw`hVwqhMg`4OyxGKH*(E9<*;qT5!K7)0$%6C
z984B*F7gPux#worj+MswaxLlhzi0MXd2F#d#O1BN#e3p~Lw+J@Yg?zze0VEng-yym
z;rZ5Q&N^0EoR#-F=y&|!z3EY%OQnmR1~4Za)_U!B#GJQXbg$+O$6d`wbvaZDWA1Rx
zx8OSa*Y-?9Vs?bzyqV{>FL=`%w6H7m#ShDpU0WQLuS|715g<1E+r$Vp+3T+!O$zqk
zXX0hQNOAq7aEYTc4u+eFYWdhpuJKpmFYl?84tX9XwJU<H+~(!-`<I+2nEz#u<3B5+
z{psoK^{0fYpFBQU+R?l(?EjU+V#1~o+S?TN^DcC+pY}^5ux7FS{$EFCuy2X?|L|(s
zj{=VkXL1yh{!Tl<DizSk!K0a|7_n$pXIE&<$CzHTjK$(#Uo_jCR<`Os)_FFtS3M_D
zJ=$vFlC@t#R(e{Az0Io7nR`!cd+e=OjtvG3EFKFSzH4b{2Va$X{Og&1)?8J+`e4_2
zR~Ct;91xi9@hb3WY02W4m(iZJ!FRcyisiVjoFfy?wL0AJtGD<WW$~<AaTmV|?n`HW
zGC{@j{<bA+CoN{a=E(bU#`C#(E9@9I_y?#ib?Q4hVI}XiZSS0BHV0i=DOgt^yjOJZ
zJGVrhT&qXX=bkKkbM@1-{XbW=i#}S`pf&x!`jcesf;+GDb{twFyY@y4lS<kC|AEEc
zt)e>R>a(@vj6Wow66rcI<)zQ5x+zJ`r87G>aJo+`6IQ$XCe32MCBsFoUiW7@QUSs0
zo~jcQGhc^XnsL}?_3j_5_PM6Y73uKAncpv49W8poPPl<Vvxljhed8m4zE@ixnswA@
z=$)I_c|*GVDoghz-%ZOhCN+F3@#=}3r8h}B?2?!B<l|C*FZE06PFdDdzDH+mS7r3C
zAZxP~Em1EI`CdHK`lG7oz)CA`rP}0Ozdn`drmct#Z_BbS(sV1|=`9vr6FyD&T<}3I
zx3&3xe?2!{5MFi3={m!*T1Cwb4=-$qc~`*3C7dUIeCn<LFFzM~$2>ZF<h6p<GQSe5
z!;iwYyg%yP^HJ^3a>LdqS9j@7J|J?yr|m4C*!)UY{$B1KOXc0mHoh#>NzuPjFt26B
z$s0#*9@(yRS!1)-7RSp|k8Vx6wX#&^pyK^ZuIVMyUs<g6+@1AG`Rbaj=KE#Ze}{PF
z`)``PN${5U@l?Lq*N(WXJJY?;B_u;9{j>NS^<N4<!gL<!v;^+;bYS}D7AUE=?4^m`
zUEMkRtj=6NvC`jPwQsRf>AG(#G#6~$(wCN5TJyWZdH0!JYRl`aeP7<2c8`-Y>)+Zl
z4gFlgZcat}qJLWZ&$9iL$!qcc#4pnsE2e8cJoG6l#W$%arqWz<pQ_(372p4RQ-akW
zoG4{5RAW6T$s+P(O$wKg#{rf%Nu?ZiHEK%L3fCCB6)wx~Op(9(>r_(wlL=wBPI#7I
zIIR(GEb>z&UHtjUDQoW-Ut&8uXJ(Z83{HEMGNZFw^SF7h>tvd)E0p~c!~SUQvtv{3
zmBVvmi!W<UwT^K<%;MiE=1~(<|J$8^#Xt48hD*QK-~I4jAXwwL=93jgdhB~w^`G9d
z<c+|F@Doo9+m`M-^L*>8cX~pchAHVcFZE<+v1e^a(&IXBK8Z{Jg|}B&de$F}2_i}#
zG*Sc74x1<S#NE>Ra^URLKIc1jJeIc{tVNtIWThtD(6!jce2;B`+!_yMvEF2c6RDf>
zlKTEPHOpi$OnJ9$NdVXRXERrEpEOufr<VKjU)IZ>^Jzw>&HwSvnf2aPgTXza^|9IV
z#|{@he9>MJwYD?m`Hq_Ar&+K6E1Z0vaAMoNMQ7#I%~_mc8MrLY@mrRi%blU&+p@0u
zhfc&k^|e_aRz@rq>suGMNPu&5_N;fgU)zq^_%Q8Wu)%R(w(w4_-e3H7JF>d>Wqd2h
z*cEp%Kg9oQ#-h4?eM?RJc<kHL&vm{zCQx$hU2#s9-mQ1uDaU#LpV+50A?D4Z5)se$
z@7^gqQ#xqGzW3idrh2Bc3Y^(FXVuqSWXyh1xvN{@+KV+;mI-#QlR1CZ;#@ELMCXJz
zDt?ca@-`fnd$7Lt-~UAgOx)jp9Zy}6<o3YBTkq|*A6&wc9<wxbf0(&^yLWho4a>I;
zDk(~vG}IcD4o*nZSf@5A|IvaoJl3DmUwEAM$XFJ{dO#+5?yJ}Oenp37y;>mmlK*qT
zytq$Kt5&qj<c6}X>Ufu`TD-3F5}VNMj`Ib2>d^=9tynGknSW1959h=)OI(hvmszW^
z=u)p*1M`YW++GK+ZrY?}(GarekwTuJnv1T4yO6gI-?FlltGALQ_8aD?d|;?Pq?_A&
zZIYq%_CV%lEoni6oF_%)oBE}@eOVJ%h)68HRg)BZCuqT`jlTPQ|0V_0%*tMStv71k
z<LMn6rhT~hjV(h{nN@Acgv(p>HnRM`bZn1`hK;bO`{wTf&uq0eDXpHw?C?U1eE~~K
zuwqQZhaEyZ#cs-$-R7%#!jf8=Z9E>E<S|~<WL|&DYMP6s_GA4NqeC@GcUH0RFFtO$
zcbV1x_3AR2R>dpYtGorirkO8k5ZUD{v~}&mDIPEO=>AoCqg%Gge9>j~6YgduLeIan
zYIt7WwQ0lue{KIW_?OHG>UA-6NHi2Ne)aMlukcxC#s!yp)h21j3u~WDGK*iVc4q-k
z&SfY5LuR?cx*H~GAJ*%0(|-4b<L$j5M(>b_nBM-dDc)fh)MKXTFdKU>Xp{VUA!CV(
z^m$IT8ddjfC4Enkuz&9)>lcNeQc#oj@xHc{chmp1-dmO*7TJ8~im*|4ay-l8+ZrEq
z{PN@)m93u@GQ1MJ*uXsXm-|Mylq9>)-{amN<2u*K(x)jK^q*@|-K>|1DnaTIla(wT
z{LgorrC(-Yd*J_niaLvEfM`<yXQ^4`5hec??J{HK{A1}>M*T6YDxMxH_m%Vy^qe`C
z<GQmjd(D!`+ujGypKQ7DvhKSNIa`dq<34%({!mzyw7ur^Gmpj#&yEU6y9gSy+ATJ*
zy~w8gXOW$&j|)cz)3$YL3;f*r4EZaQnDU&yo_nEw<#DQvE|;i>R<{$=rxGp&6YbwG
zjGKH{-Qd$@G1U{AYQ}#kCOgTgy-jQ0A?<KhtqPYbM<!iA_TPvx@%WWp387c|BzAt}
z&0ifh%TP)*Gok*}k||pyg-`A;S*xG#y2|fEJf}O;3tmg6RW=Jt)>TYpTe@Z1EtNPH
z-MhCgu4H>=dw;E%gZ~k;1Ru$?`)Um{;ufjJiJiC*;?1`1zm>6SoSMmW&6J4CrfY?q
zet(JK{_4$lHK4!D%zFXP6fxbJ<vd=F+CeE=D(|!ngUk(!#1*eLUzy-;Cv^JV*T)KF
ztqHyncQThvW6M3pW9pzBGV{pRNlPr}Y8(<ctSahn6+4xG{xR=$n?rX#vzoZLy=ZYe
zmoQ_4N5?)EyQzntUrBqemh)*xlGQntnsY5-FOP~nYWL~=l>CC<+d%a))Bh{pt3E`S
zFL6{?h+bI~y*|kG?AK`D<7VlXSr@35B{~IoHw3VY1_+lrpFZG}e4S^SnYQf|t?5pT
z4lG(BSqy8N;--pAOcnLsa40mZ%RMVA?8fTgz=HOKFDy$pX4i@2xKHsfb)CE8kl_ZU
zk4MaICTPgqRq`qOT6pA?p=-|(yEDGa7KgF$TAfph`x0)=7rx+H`??=4@-saf&K%Zc
zG1=4d{_7qC$Aq?qJqBFe;;D-S{8twFH=D&yd|Xzq6i}~}*}5%r?lzti4LWhc=Kd^B
z3Mn~E4Xw-x3=t8GGjv?D(ikF6nJ?i9n$53mcluNtTT9K3jJTtf{sw1$<(6D=>Nxqo
zENorA@3l26wyEX0Z{^o~T;qH7_nS<&*mZf|1#W)$P?#jD_U~lU)z4Lnn4j-DtKRx<
z!Mf!uw><IM+#dFFa@>nok=&R1d<E0HMH^!cm&~~uA-cn6*2)sq1m=M0(VAZjr@4Fl
zxf-ANsQL0$7NM&d$G<wAeo($qXtHgG*5V>DW(P(l1;&^kt)fm-+$GhS7+g34rsmG5
zd^+`XfuY+(=VeXD?pHjy`DDY@4f?IqzIs=DN!WSl+Mm}qzdcI)I!%}N^6I)pw-W+x
z--wv`yd~NDO>d?_+_Il0Hzx9)IOTP}%SP~-{SF3IJ{Nh*%}IIs4zA|juh`d@<eX6Y
zaq+_y)13>Lcl~$B{9B@LX5oFNw7K_MjBsd6ZF@`NW~~ghx?^gbPL_;K3#Qb*Q1Gf;
zsQ+Za%r%;aF0|ENo^A9ZErrFRI3;`cwCwYzgH$=695K^BqjT+*o`X+bmfySF*WCuI
z)cD*ds6We`m!e*^%VPevAKpn*)kGg{_hCvFRf>4MAzbsiVCkd~;fbFrOy#Z@=e`iG
zjyfpzDvO~<Y5NX?2{TJe;-5u*zA_<nx|62Kjbl+|{nuEHc9>oK8go9%nO&jGcLC3X
zUz64c`EQ@9rMam>^u*-Yh$*tOrr5Ggo$8<^=DIZN#)in7S``~z+IOg5@!oj9+qH71
z>&w;4#BaMNTUvYjZhn3!eBJ-TWu2>@9+}9qm#sSQ>Ppu$HcKU^wCSasbx)YJ+WW7$
z&*8)M0m2S{i~IXaITa&*d~r9uooxEB(%-vbUi&rO4>Ozl!`9Y5^8fpDe)rjybO|TE
zs#al#XNPvEuZynwrNWssV?pMQsrePNi!6fdv$H3go^Gl*E%7_ds_)l>3pEy3w_Y>5
z8D$jVCb}&9grSgs=+l$GKYa7v5b^Pwb^gcBa|{k#W{T(cIIpn&*4FZAMSRE6o)w+j
zQ{v9F2`%0wwqrpb>+-&fJJr7YSnMG4pz8O%B}UP*CZ)<-oJ4P~t<Rb?`L}{n<+19f
zn1{dCK8`AF-L55F;%IL<<>#~Y+nt_u-FYbT|9Go=%7*EG#10yX|2(AaD7E?iA)TAv
z+DpQhHoBhP7=Ga2`RSF9POseP-6o+OwD`>TMN3|qc`jx!oEegAE*5VkmUpGaOh<o3
zB2$XH`}Cxj_H4%z{K6FZp4lH#Zo7Kri<ryH1$>Uy#hi>g>l$`9eHC%)o?UkL7EjMK
zXVF9R%V%cj&&}a3HIx5lJnwdS+-7%<D8^mC7dXyrjo)!qylVcRsMQiyyZ0VB5g+kv
zSM;LHAKLxv7ftndF_f~f&2Dz;)<4mi(HFbqyjfV@QmYv^uBEW%o?Wvq`R&r3O8X0=
zKE}M5v1j_(ZI?BeH>zDpjXh?(s!+q=omy|?CdUWiFEW^G8U82ttk3ZGiAndKqy6r0
zM4*?}d#ib>A&O0J4R1UuuT;9+e|utjOnLmC^2+NEYuP7jE;zpU(`4<sh4Q-=`bOp`
z^@`@teSFtRA!z0X^(rlqrb<uMcZ=?9+WF+oxpjTFvfqCBQzbd`^31bKwuGOlZrc3P
z+g&vCd8+&EujiO1hD5Hhuw8fd>_N4c(|<h=*nLN?f9riG*_#Qn{M%=1a@p+K{biQH
z$qUX$L`@u3e_ZsNuEc$_rS?g4q4>^=Qr<oXF6VwO^W3|v&MhQ>#U)5~VYKaI!?@rr
z);)jgdSfbLW2Svb|M7%Z+H&2uEalSnwQt&;i%$2o?M)SFo#7U|e*5W776&r^NA2G}
zSNl!pf$tx@9OUnmcAdL$^jXBiSNH$ytV%lH@_2PrvY<3qUdOIK8UF=V&oq4|oByqa
z>+{_E%P*OlH?VALU`aBbxc}<WMoCNd&`IKT&UbG;)D@ZR=<z7>v+b8#k4*Qe|CslF
z`?@K`haQK2^tP#cy{!A#v^g6hceZ)|)4qG3!97={d$RV26)jEArfvL_aphf{zmx(S
zhm6OD1qYisgtbCWq$W&n<rVju<FWBfLtCnj+!K|6#|-@(lT{`dEx7A2nJHS=W#$Cc
zLuFdrvr<eaJZqR?p1kdnXTWmbu3WZlH*a?F`p&Xuy(S!?of^QZ!s#WkAmPB3Rm|!c
za}P9nU;i%@>&2rH==R24m^p^W!CO%A_Qvdoi(FqH;)ybsHGZY)+|+%aGg!?da6(H#
z69aRTf?tmX14C1Hvx>%!8AZ2S`nl8O8XOcF&dwEQvfNQ=@~oM|hh0E3)hj`>fiZuE
zQNy;kjayisne9k?Fs)IB?Mt3T%n#S~OcUc{>TD{HbjVB&Rx!S?*!7i|=`^j&6Q;NY
zo!%?I#7?2==jXY*3ViZ61WWeaW}PN=A@jhuU!V1Z&#aGE{Qv*2pvlcNPo60*ES@<B
zy@J^vng}#A{eAMlL94Mnpw(RTi?E{@U&6x{Q_Ul6tsE04iL`s}RGFZ=#KUkVlkfqK
z#VpbVN3V+2PCd~2-?f6XWxBK!qvize)D^B5Vpt_tPKfk+<f<~kWtO0rxNO0J#xNtV
z2@On)AxBdi62wGqIZgIm7BG|9r;ULjTaBS*<_cbuWsU`LX3Impd=!t0ReNnbIFH$9
zfwbe^T*IR|ZE98r@_9{n%~>qi`twE6eo+_MGWk@lNlWIGRozSFzxrLkaiUvRz!LtY
zuP!hwHVO||@>MWp?iv=x9TysBSFF-@RbVo`(7`g%{6NQ%Wk=dtb2luSq|0n@I>b8n
zz&6+LSWcB`k*tcDxlXqZwZ4tcsO6q6DgScm^!T_!!`TT!8jcaOm_;luBr#Z+9Z2J@
zy(YPD9_N!~X}h0Q>kF(@6=?l0)}7T<FqhH4C4$L|L#;4D<a*l2+})~IW;;krUym+O
z;Ai->r9Y_g-O2M`)8*6;O><{EJtsx{ZZ%u#^vnj$@{qe{o;`G*Uu%Ec>O##Q!I1bh
z?25747o514lpN;r3`k%K3SV_0kXd7QfVQkqi%5%2?T^cWYjb7<U60J3QC9!p;+4`M
z0lpM(?KZZrj~UAOcdm4qJu9{8KJ!Y~C7iC)Oj9|-Zb>X=iY~jkG&P|4<c9roK4q&h
z<YjF<snN}6(Dr<GPUDf($;>$y=CZ0TRG+_Qdzirdtq1K|uQC~0{dV0x`&HITp-5g$
z$*GeB1<S3twco6nsLr3Zdiu+KHI8Ce#cTgB^9Z-UaGCR=)b>x=<@yu4?wij@^wa$4
z^5`dbp!pTQlWbE<wY`^K3DI4<#j(h-jC0C<hr4o?0y_#0az~y@)zjnXXKv@}**#?f
zgR!ZaF|!q~(~nkxD-%5xMPwQ>4;|4r>&+91x^P6o!o1H=BS&P><pl=}JeGH=C~yi*
zaN_xDVt63tO&j;ih~rJlvM+hsxek8h=9j%H<M(5!g4>nzAK4mKK9Y2?k(1&Ou{04-
zHdE=e-E(vGlh5;vj<9EZN)(l7Y}>n}^P<9;?wE@joAfJAo{sliz}K9#Y40Y(497>t
z9_v*$9cLF!QQ!K5^VS7*f$y;yPM)3ej?cEWa(;QC;PwBP^#2(mJNg_}@}y~;?%QM3
zs(m!0u<!S!Jv-dJx)!V~kvy}&A?WjLflW=a%!%yUB5H?PBR1GCH4YY9azR~co0=A<
z<&{Nw>DF$#>K&7d;@-0{X+GodS6H=a<~c7<U(c<t6rWA>bH3y;Axv{~kL~3}m)ovQ
z{27tb%ifr{Zq?2CcCVJ_Xh{a1p7CQfBTFmKgO%}iw-Xjs8EEHSxaE|;OH?mkG9>u<
zBfhP1m-UV<eQEHIt6{I)WXWn?2E%^I3(0046V5aoGm(_JoUXTd=ES15xy%bS81At+
z`Es;~NeQ)zxEYFyI{8jzYEsN8)7t2e$kQ(JQX;=c*eInakz0NCl>e)59h$gIDrMfm
zMaw_(WN20?xr9da&J>eZGS0k~ZfkFS>nj)20@G=QnHjE`+-sQ*vY4(`XIsg7Ym1rS
zX4xlwx4AF+UNBSKSn1ziCEC&Hr*dNxufqCb<>m$dri<+n)thY9m7~5>X)$L`r&j-A
zuT-tW=Pn68nO3ywj>ChHOWUS|Ipmy*63N(P8+F2nE4hO!gSY5b_0p0p>@)A^Rqwof
z)ipJF^Y6C{N)~JAJ<02i40-l?#?KW-Azc+wQ<hm=^}5?qaCz4}ug723Fi(1T_Dz`J
zji${aOlzm)yj6Me_12G+pzEJzey@2Je8T4XcHtlf??|=CF0TZp$-X`dL~?KNNuK(D
zKzu#Nl!fnvPA1uS&oVfe=~IwoAkA!IIj3yPL6L<I_PH5%T|M+=DSwhXN9OU1x4*qU
zk{IMWt3=AV^Prl+uE;mJZP^=Tez94ck?kwuc{Asej9|CJIwh0U&QE3uN?I>b@SnuG
zIz`jTedp!9uU5TSEckuR8M|9QZ(8)4g*7jj9PDeHx`$`V<@*-is%4?kEhUq#ckT#Q
zY-X9P$a(0z@Ww~ZUWOh?t;HIgM?9FgK1_P77j@TZbFs0IUPt=PDJ(&2$`lM*Gj-#v
z**w>LTC%+Pdb;!OYo}QnrZ;_Ds^7dy+U4xKGB;nf8MAJ#NYA^Jw!v)6n)5RRH25WZ
zPv~-7TDN+_|Glj$Oi?RZJr9^?f0a@>%j>kwVL_Jv=V!w6-fn2MmuC?^$Em^~u=It4
zNO4q6V~NR(-5Zt#SFb;}ps01*)t*+%$)`f|1mdT?ez8mWcu+XsgTqV{pE9IrHC~+Z
zV;NVbPh!f6;Ca8UwAug7Y^eHdIMLFk<As&qIhCT+8M!GpnRiYJnf+_i>x{2TS8hHl
z3~F_K!KpG^_U1+R16tQ~Cr_QDXaCyqnukfO{*%)QlcHt`q@<<(|G0sbarWk!MW%bC
zq}3CHI}&q0J!|6GR(&dI?YwBmwqre;Zp;cR$tzDicyrf<BazcRa)UV;&bAgXyqWQ3
zN#w;#h6)Rf<MPeBlzXC2N&ZhUWX!lO^684yKK*YB-Kw4i$>u)Js_Ny(pWby^^G!P0
zt3B|-Q@^s1y16~#mZwgAE<L=bQXs_lX7H@*C(33nu4L|IsoS0POY!{e)po0v)IHv~
z$H8ri&Y^WP`yBK;V}w7<*v_3&!p7*7TRJtPMWrf!Mz!+tOzmgQ4E0L?*@8VMGMr)T
zX%bm%uuv>u#h)gZ`6i#;3{$kKE;eqT@xZp%w}*Y}>~G30D=fY}ENnZ$=yrAgaod)2
zM^|U1bSA8NEO0R9vPrDFOZN5PRFloJ{Dtx<F7mMg+OYy!d&Jy~p357UyXAgNU~OdK
zUBbHLfKHi+;=83f9_l45g66_b;rZ9N{^zoVa|wEgA961;wQF1IVQ6T+HlQX#y*ArP
zXvqYYxapa9m=kxjs!cg;@GPvHL4j3I)oIoS)>#Hz3z=I^U(YNMk8*jT87pL1cfGtm
zTtG#z?YCLmp(BP%%d*xi3lTY3!R)}g>YB!#g@rSOvj1*xK3}XQd{FRuh=A~mB2h<e
zJr&lJ1s)HBb5=c1GfxaZ<5vB<MMq62_wqKSuLn#$BFejt1!gsMPP<|7L9B*jf<j=s
z!2(BtL+yEsJuI_@q@ILjunSu)e5~|xnfAl-B*_r98_QS>r*yNbYwUSZ#}%H;wa{5E
zS;k$|<#JO<U1N{Ci_4s*qOh$pQr|TXC28H9V*LNrLcy2~?J-BSqK<~XyD0E3(blj$
zd%9qnSV!@tb}69`(h5yY4PT3Eo|Z^m(k?XisC=qm_^|lYGNHzn5);F?qRqm0)7y`2
z?<{JvoT1EWwo&2g^5*nS+I*7ThsDdLRj7-(L``%qFZ&+wmMN=F-0*6;K$X!XCdG=H
zM^eW^eX>@_ce!gSe2+Qo9J;|Rl(|Ck44W2vU=?>^?6za=^W6nwR8?cQ$t+g4%wwJK
zWUBk?){^H_r?mbkZh0cI#vuLCV!NLruGwiq-wXu4D&%|#4i-|Gm{2Zo$b16V#8#EB
zoyMZVZ!U1X{UGgF;h5f;$)>5Prka<;R91M>NyDscTZ`2H+@%aVI_h4CYp{zaZ~NH#
z&RpO_xb+c@0CC2dM`y@f6)If6$okv~^J$Ml&Xi7O+|+LMxc6QJTkHeZ`9W-{B7(I)
z1b;NnlCThbrlIL+Az*l<$#-JEOSy3Im$*(3`#q7u3Wu7it~Bkxsi4r5)7Ij#(_OJJ
z%r1@7^t`0Op6^ae63n<a`BhtHyGe9UY0jwRjLMoMAk`Utb7n?HQR~JZ6RXV`E14<+
z7pgNWwsEFtEKAeKyfOK-t3dDatguuE!<#NM+$S)t(t5Na^+{5Z<xMdI6RvFzIZRwr
zB`2kMCVCt&(h=(_44NXG%bd%+L8Rr!)cj43N>&2P6oX_~ql7hU|IghK6eN)T)`87q
zL9SV9)Zd`!l?%HnPR`wJ)NQrOraUux{zCnnNj3UkoOV?#VKEoolQF|vE19F*S<*O!
z$Gj(4%1A@F@QjDxGeN=Whm0y_xUzgKyjmf6_k;}VjNZgma$6?JKJ1vqQQ^Al*)03W
zMsstw7aE$J0#bqp8r?OUPIIcoGz%QzRQx()w%&||77vTP$|I76SL7aRQvA6fs(jiN
zb<QOWavMq_78le+Z=cBcc<v+j#dE`~V$G*56rK0Cspe_hgvwbA9t{dX8=^N{R%c$o
zy7^R{{LLj^3Ig#QK8Y`+>K{oRHDx_JCFb;#8OLVHoc|(d(Aj(9bI-}kT2?1*|6gaF
z#ik+1-sLGfacz{Cf}jHEOkLfIj@^^eepd)QEfs!um`gNE;H?+e>*RRrgX^?5PR-w{
z^HzcJ&!g&vw=@NB$}jg2_<DfLQ=$Bq;k@Z)f^4%FFYA)Le|-9u89|4RtFbsQVHaXG
z+`<~QVdK<Itg_Yu9LfTUucZ&pT4IzU8W}jr_m`A|=+d5_n@;SKIqYckBB{RjQ?mEd
zWcBYMo2CoQUSxgZbK%Zqz0s;}VjFuGAClv<=(CJm8*yzGhvV{%lIkpsHG<o`w@el|
zG*hwVX@AyZVeihV2TzLlRE8fbTTxP$V{l^58&&&>S}R}f$d{Y3HKKfCWVx5f&WUB#
z(^pmn{a<A<cgE*c1}dw!?P8suEHHoaj<riR{w`acq|v68ZTM#AJi*DE*ah3VoaQeN
zi^=Mk&-Ku{Kgs5cNcLr}?N_F_RcW~0*dma+W!ACjEKvuz&OKVN?Dmum?gF1obiS!J
zeb^PQ9AuWeVCrL2`;yP4vl=I!IX+kN>f--qYQ>)xtAAb0abUS%Nv2$DqJ`H)i>(>E
zKW!+nbadPml<TyY{g8LifsN}d*k&)-$24)r>7#9&2it!CluqnevNJ-!H)}?~E&1T6
zrF`Wf3_VNJRtbo@%ZaIn{$1&-J#!|9ploJK=*fjzmeR|fHqN?UQDjnVoOz`v@>#0S
zWP$Ri3Db5g&r~wK(m3V+dyW;2S?R|;mC8!pO&Jf@-SW$2o)f;(ecOj^Tcca&O!A)g
zWBn=T^39xO3l6So<X*jLm-<vAwy@29LWMi#84LLA(dd-kl$@|yDOxj-(Yl*yr`%1w
zHz()qnI(FlL_<xp;(+OzV=TK8x&#)S)(D<jr1^ZNP4~hX>bo3%POkFtnsZ$&O+eep
zlA%7vSZGdT&;_nXr)*QZJ8Hhf=`dEm{iW@l79nUP5}7Pwevs|!DV@W=r|mZvIIlkK
z<mwIO-dmT>*}!?xO=`<t$%yGCmEOx1Habbv%yV?q7F{CzdX;O}#%HrPzMNgVGb#72
z_E9FrGt3+P-bf#1S-5Zi!81Q^T3(Pj`~O#@^tb9w-Jv^Q9<ST6@vKRjOxlmL$3<p-
z`7RS)vaDd&=0D$iE^_qdF7#QtT3-0YET_}EWEr=vmvp->dMI6V!@@fbLKC#Ne(>K`
zzAdJEo5-WtA6VzG&NcnMVv2E%K*-CIY;D)aS==SN4@=xVRiQ2T;^eA)CsWTeR=-!e
zNQ&D?K0jjn`wZ*BBVDU^Y;8W;XSrJaWNYG?)?JL!TN!tL<ZRoTn5Fr%jcG@l`LZ<y
z9--b}&h4Fcaen36GaM&wT#n^qbC&nm?X}kZK(*upM&Z4)O%GZ#L|f*@-Q>#)H<-+&
zU$J}c^uq?Hweyzl$XtFn&_=6l8PAgWqP44M?0v^pt{}Sazhl+nnl6Q19Id_E_w5Y2
z!7X?uSMdys;+dY@+T%56UcKJ;S^Mn#+m>q8OEhnH`&%Xde!c1a^Ma#-2Tohu3OIH^
zulc}|poZPTS0+Bn?laQLnB~T5p*1Jq#PyV0GH*<Ztcn%>95hv5wJ1itW*4jckL3lM
zx3bA99_gHVvRAQPOVN18i5Hvumo!{>AAGkug<B&w!@=hCKdtGhZ}&*D?f7T2BkF?s
znq~pNJ$Jbd-eZ1n&-QeV`ObUhX*;}o)&%^qJlk`$?fOxZmTr^SogX46S;q=|xG9yU
zv!)~6M|Wk6_+5blnaRa9m4Ts?r(eG%vTySI=es1&X<qBfZaU*T@%p5{AJ_hmzQA=@
zx><A1ohv1~&n^=xIiu((=x@02WZc=4zpgGYc(qbO%`sPZn%$Ylx?8TfoRL&Mby%`&
zmFUguqR~s9aNl?qy<@Ji6Z6E{FHiQp?Y{JLuJ(h*8(n(^;^WRFCY=fDIkWJxr84hX
zIqA(`t!F$pKQ^njpsBrJ_S<6y(&r{u3Vz=n^5l<Z-Mxp2O%I>E5R{iwm~~ukH}9j}
zx#yqE2n+l@Az7&Qlab_}l0%$^=DQ_J;)LVdr>;0v*|bNpG;pUvS!suShhpAh0iC2>
zGcMnqG`Y_$=3bBh>)OQ|1AK1sGTWTq=9iLiqb*lpkDtIh+c(GWygtpgzx(>7d7Ga}
z-LA;hWBniFv9mf|AihI-zt#L3dGj6K?!35X{y)wlqm`ObEyp<pAJ1*N!q_!=TkTp!
zy_7r2OQH%69dWywwX{k>=<y$u$7>XH@+uz}U7o7%)hhKxKXa0#bI+-NL5n=@iDhwi
zZx^jq4cnN~d1__g;u_t3`AR}M+tjyt-FP1>u-HLPas8PE#jlwJ-#poP_e1Pco%I5}
zxdMrHf`;qQ$i?57TJ&tP?Ec?jXLYCDWINui&lZxKxYT2550BkJk0(ZPSqB15dr#kx
zsW~xI?B0tPHQ8Iw#;jVqt4?D1uCuc)o5=e&o9wSWudl8tIH6@j^@)qmmKO_$pLl!W
zt!WAW+=W`_9<evxjnEAL|K&0FgUzqDMwbN%Fdkbif9~`ujt8qYc)gafKdOHJQ)1H_
z@w-R*ZRKVcXzr0avp4VOwe&YWB|mrn`#JZ2$W;9|lkdL`iLnki)Aq~Or}wn?MY#uw
zG8!hb8vCTyaPHf=qxeg1jlhXGt->FI5Bmfz?{W#Y7i{Hw#4+ogl;77Si|iK*ow&f-
zVcYU9YPn^~hHtx_-nVz9?OWvT9=PyIqR5d;ior95QWAux<?wrG=B5NFC^ZHP>=#&g
z$7vyBP5CEDj!zOR7yh5^VZdOhaFC&ixj}&;L4a`ri-(k&hmc~Ih-TE5oSP<T!3P6b
zZR=Q87CN{$$(r{Gd2V`e(va0v$R={ae-V#aTDe6_W(Ge$H&N02P>f~5<!OvkomDng
z4_-`L>SgXRQ^B<<WTn?KFA0N%2_fr(xA4ti+LjP8FNSN@U55$W>|13fshOTW`G&zy
z;iSkO?+@?p8p)r1CTmd6kZ_=uvrMxi;_b~Nz2cu`7VMauG_Urwf?nvd%MC%yX7bB!
zRxW=rmxGH>WyXic2Us{a@J%?hZ0g#%YdE$^1*}$h*T1&pvs%V&W$k+h3eSJJ!L{RE
z|C$-x@_aKx);=~`c{wM&`%3nSyN+rM3XC1!=PV1XW@m`Aoy4(ja=;y>)|GBX(&kFb
zGvi-)9gweo_P2BSk_nGa+A+L6l-)4%U6jE=wy!}28(97;E(tiq)vEMZ_Y=phMN19)
zeoa_0t)okSALo=APMyN40f9%?1e6%M-bkC0#HE=wC#hM}gJ+HVL9-AokIm0+WgF=I
zkUFqQrBbU$?Qla&k5T`;mZ0TlHJp7mXxpyd<lQ9H!KB@N;>0@Hc?HIoy-nFYCac}M
zrP0C5?YOB|#3DM;_1x`(&Fu=&uQo0{6SgR#L*cSbQm;Z=$nOlBMjI*p&d&~3S1vLN
zF+I58%^ty_*eX-xajE0bniq=}GQ}8&Dx@FkUl68T5~@BmB(yZFKTb%~A*AxjMCbV0
z)>T?!?V(8xNzHGwnl?-m^<KW^lGAGUZEZX~+<G0GzPX<`v7&qOp8u~V6|BqI`}^A(
z#kH)Qv2(Zh_6Q$Y;uF64#FM{0r&TMJc#ST7`pb9GS@_5Fy&4>soVbiy7WXRLoxz-U
zxNYWcT}xBxWyQ;1%Bi{ToRgFI)GaFbYRTm<Zfzb--=9p*@BVc{qO~`0f$=)?Szlxt
z(kvt2<@^0}i<Ddu5_opy^SvIYcBapYySwB>|A*u#^#!H69*6lXb39kB>ehO-NWpVY
zpxx><UQI_uW3Rkgy^&*Ap?gzV$n0)SmgI8v9Y#DodCEVnKHb}-rQP8*?*VJ)rX9;m
zHoe!_EiAL-wB0|AH%&UXl9t((E}HT2?jDsswHG31x<zK+e`5Tw>0gd(m$T$q&Qh2E
z6E!-%x^pu*ZEY5EVp`blq&riq$3Urh%DtNvrD=1Ew$&V-ow4<5(x*x7rM^>Qnr<ys
z=<TvP5kK=&Y4?hCo4g`cZeohRxqgP|gpFI)@Sj;L*Y6arH{pP4kJ^q*CZ7%JozI`t
z9dq!$zVXB#{+LN>$_M&u(h6rPa_s-sy1wZ{x6Qh2UPq-5a{VXTRkmG_n9@}|x8v!u
zB7tTJ4=0U{da*iA#)W)&ZZqbF<v5leJsiR$(Zn#>Q<01Hc9YnP%bOmZ<T0NuEa#Z<
zOwf|m-_b_fGj4{t;(|sO`xQa6TI5t*wTxDD@y!bUvrIyp<z=eF)!(6htIW0vB=%;j
zF5<YMvVh_L#;qEz>RKJgk8NVz_A<jG|8wB<GfivdbUQ1AekF@@KR4G%e3{=G9OHNI
z>qJ+fC2?(wPo|wpU_0i>q7rq(MS9cXgQlV9tR-85r5j#oSw8f1EYCW6FfV!8$*YUx
zSXa91KH(7J5IUkLck+tkF=nj}2NkAWr#&v&n4da*k;Tu+#ZTHuOj|iDvqSjR!s)Xl
zeY5IV!VjvP@IG`y`?aU9PRLHKO$#207MW$MpO<2=TKsfcX70nNn^x1e9dMoAbmp@1
zw=bu6X^EW3I;%gwaQ5llcaE|i*z0dR-SUO9{Uyn^qBa{olPlAm+=Q=gIA&{NI3w5W
z=ANcMX$)IFE}4BoIB4e7{{;+^Dt^Z<un6(gO;lHLb~rhy!CCi8yNG1;vqM(D0v`xW
z?zWn#HlgO$+^{=emI&^;=|1Vz#9uu}GA8`=5BdIS{v?ML8Iem?1uv95I#K8AayxFW
z2u0^bhH<_wQPRydQ!Z`&apQ%inN{G;F41X)^YYgQS47{w@@<CQyOcV2&BG5kE{WH@
zd(!WJDW2iytMmWgE?Lm><i6jT1v5^*J8v*~$`VPDSz6w8F5H2egJ)|TxA5-@Q=e_E
zCf*%2!RGF=@}o;UbtZ2*?fo?<L}+4@TE+9-sq;!&wi@WB+!EaU&LMrpdM7^V2*Le&
zqSuyOZ8`HX<>}JM*tfI(MRs;v*nZZas%72(;D=lx*VM!|&G@Wb7W!Olfhof_fhBvd
ztP+cT_Gn*jXy?wK&C3%0orvg{FxtfG(v!C90&6AXIfI!ueVq3)U+njp67)BVzuYEh
zL(|@G%F1UvISMBqeAM%8#m<$!enHhHy{Tu+G+92n8(6S?Yk8YuxrE<OIxOl$$2n!0
zKL-9yTzC6>EOuQun<chgN>P_9a)M>{t?=!)j=U{nck%C&THoPxeY+E@>x17OeN!eW
z?+xPKC;e#R^phVJuXb|V-+blv!yj25s=Q(i6LeIjOWizvl-FUir;)Sf83WILg#zEX
zf7e`Y^Dw;3W`117#$<vS&zy9-&r{TMTZ_Jjbf2tc2{<kLe_2t~MZRYYUPm)8@#f~(
z?BXt-rO@_IBHw7b@T}g7<MN%0iYski&(+ze7P`rQ-I*v`-cs}G`#kwKXSmwEUocrW
zGqF4RUZ-y9*HGUdZ{!_6ywv@9w0vXblyvbsPX!BkIlWo#`KSx6ntaA&%0b?iGYgdt
zO#Zy)xA!8hk_nG`r}V9vrM5*<k<D8x$VEI~P2%v!mE29QGP?4vbY9~;61Muu9G<UR
z{;2gAx^GOL6Ka~7&^zyfUS8;)m^~iak7pjY<?~qe?fA8ZFMeMxuq&<FdMEta)mzcK
z%9Ny+YUrd%cgH?#a^KE)<xINd=hm32g@3<x%<r&$vi16(Xa5iF+<0%{lgt0@i_*^s
zsl2-QCWrO$j?#rTpSN2oR_y<Nceb$c)-<;*ld~s#ZnWEbNp9}GkAAKm!aVn_&80<i
z@7{Dd8?Ik7-&DDO&EDp3rVrhB$r(O+SQr%AKgVW`@7IoXCMJufHg9YykX^o^FfJ}V
z(k|+HRerVn<$XUdvD{kuEBWuLv)BKHdB(BC1kHWgnsDySzn9!9=a<+CH88Rys=sbK
zk^b1$yFcKA*}0z?U5l9?J$jINxcXJWqFQCccIK}u_PT!5&g7l6;rB{wp3BycQZMZZ
zcjip9?R&R+#_Oe5+~>V`FF)_=WA%zzU!;uW4oUq=+P8IGXr#9Jbt}Eaiy2e&b1o|@
za_&0b@qf|&22<tBGy5CX{A!wD`FavZ$ddn>8i!8mu<zQ{uPJ##!EUL*vK`vXcrJCm
z6W+nsw8g@5_s8Vc<WF-ZpAvo&!1rv$mN^&fyt~)(dd`sXaop>C(0HX#*hJw6nR@JZ
z_JvH;J8nF0Uc#QaJ7(o*b*bOt-}q>%+HI*D9NP+&*6Tgt{NlA-&T*T5l9~SRj?1Sx
zO_r!v-r~;s%w7LT|Ja9}%NIKo3Y!;H+a4-h!J@ofE3>~TLFAq=hnpiW=Y=T&g8NJ>
zIc-+5iYTvqkf2$eta~p(R5p6nYv~5z%QIwn4$U=K`@H&y9;5w<85>o+olY#6`RaAQ
zT9Zxh;t4Y+tU7o?Yx0Bt7AKUgR4;KyUljdyc*llSdsc3g%AO^+_T@4uM@ebJX=`s!
z+_F)7`$J{@M(zd7I)#OrhpKw^bvXYpwsw59ZIN&fPm3}463+ix9lS@3`){m~JGJXw
zMXyG|cE*V9D;s)sFRbm+=#tP`YNN>K``F!VrDH(1gmGY>c*~X`CL0^eWnms3+{Ook
z?>N4*ws`uQuSr?+F_)nS3+L=g2hKYJ!Cy{%{Jru>@w}tIC8FLOeX((!&KJ|yH=RC8
zuK)CC&zuXEpFPFDN=~<%rEZ$zP}I0=LFC?jr`)VkR2Z4uk6S3*_Eu<LC3LrY#j=>*
zCWC(c!rtSG9i1=uEDv#U&Ysvbqod2=zof}2#Y*8W{z%EGo>H4rx@3a9mtI)7<<A8E
ziM_4KE_M$)cYR(Gz;je&&WxwdJ5+4!AAB%W%(8j1K~$yagz}w5KMo5l=veiaL*Vz~
z&M8~w@t;(C<9Ryuh|LZyj@_p$w<$UpvrNj6nRLKNhjXLbiBpoQnVU@`RSw?bEG*IC
z*&=@S)@GaC;->|+t6Vw7C^2JIOHaAY8I^_}mJ2L4hE5*EOHFqkY?o<xdc-l%dxul-
zRC%W@2iDBIrgiAatG=(+KJOFcjxuwq&C#1sv2|6YQtyGO{X3kdD@|O^v8_nT?DM3P
zXL+1AXQ(S?nnj)Tocm!~o}yf7=j=);apMx-3m3LAO#Z*kW(_y<sk6;m=IvJI^L~gg
zv*G6q?zNX;;OzEV<sz-0c<TKOg(d^Z@@|eOC*Et49NsU*Ih>dlEjYLH&*aF@2m4<f
znOo7fXNt!@75}J%7eC&a(64P7x#noZ8mFa+M-}!?crsH<A<<~+>#ds}E#thrj?v`U
ziVHp)EavaMWt5XCoi43?o_m*aq2}&|llE+MP|4iS<+v+@+3SC$&hlX6n{T=qx_9kM
z2$=BVz=JpE-WAM!IHk$VV`)9}m8ODo!B_S*&%UVeX5Ygo;odEFzXG+?0w?$L_`fLH
z^U2#X{txG$)5{h%a2l~J-jd@v!({o4CH~((9W(Hew@Z>QoM?7=_44^|{vVrrrZesF
z(H)HLX18@J95rrLUsw_&eD!C5!c2t+Ej$nJa9)`q^3dg4Q%8r8Zdk>iNsJj`9uGO&
z7c5*{wN_+P>$byvrZWR4aD+?U<!n|yDk`!nYn6xC6OOOmi)^2I<~`BaSae2d>7j!S
zKJPVFFG{d9Ipw2MeNyjD(C?hfOsPC}i<(~DTx{dC^ODY_t9Op|URi!)iGu~}Bwa?{
z;u@hNJ0{uKoVoYpY~9VvX>TuQxlU<(xYky(p|E$E-Aa?`t*Wv<cG99dq}EFGr08xG
z_OW%;t7z4mSmUJevSXpG)Py<LPi%4j;&UtZ;nslAu0YWndV$NCT*NQ5=>0Eow$uq|
z&i%jluOYY2$@y%d-OQ>Q?p_*qS}trkYrBHwNRz-3Jx0GPhG8cvdK+I{wiCU1|8=lv
z?(I(u{G2zM%pS~~64WOl8ve;cSh{+tWNgeX@2MR<7W3coxt|VEk=e=JsHC_wbTyaJ
zskMjXiY}eH!kHj5Vf%qI37p-$Ggj+}@-W4C>g&$Ww&vQsGIY<3SYy**g+!ejEj^~a
zn@`LNwwb&{PI1G|2eDDr$L~HdoXjfJ(J|+5g<aE%Erq&Qcvs%*|8Q;QR}Og|dxedW
z*NQ^YH|f`F-+Cl?J|t80<dGe#H@f<N-MTb^YenyQt)F|gM)YLJ*oH4%9{%+H->>l=
zYbUJL-PQEVZSI8sJq*$cH(a9Qb(!x3pZfG*%H;_Qw&#XDt3A~iAbYuCK}&~bg^tTJ
z)_u!MwsTa?bvL}pG9$XbU<1cDj%%F!&4H34i~91uvdSgjlwqCpVe`frEA39oHe7l2
zu>5P{Q68Ha76NLUu2(%>cfI$_G;QUjAHuI3HDU~zy-+z~U2klwsL)0M9i@}AcX=9f
zuq4d2h<owq#(&9s)suE9{_7C2bKP^bGvMKb`6~`eKifEc>dmQFlbasI?OZ%{;iMza
z60S?!5jfwgc8L4rQ$<gWl}F~hi1c}T#beo_`5%_OX6*hRBBtNMb55rtg`20mOW%&m
z>+cfvxhCRfZ|_IGy=AV=T*I}!+3WwLT?Gln)er8cOcAY9<Vk(F<LzysYf~mDUKG#`
z<dkK&F8NT<jN?XNn(IOhvAzdiUd&qc@S5b-cBwSKmtR@8pMExV>$78L<BG2G8o!Ja
zNRm-F)UOaH+%03{DZ777*FmGtlF=%&7!nWldke&=@K5EAozSpp`+@z7Klmi)XwN7)
zHsgwRpx&-d*ZB+P810@Bd%)=Iov)r|yk4g?l8<>lR-HM2QA%>wrU$%@3ubzqUi|9f
zyB%>Fh1Mr@=b0sjoDFpRz{}_S%)9$Y`WM~!PrG00MIJg-eYEUaNc@~bZ=_=R7A{T^
zTp!fh^QlSt>#rx@SM90!b-$(U>f1BxL;l_^?CD{V`>$|*LeI_3drL|ZO3rXjzM7D3
z`ufU@uvt59Uz?Z^ZIpdkz)wWaj&str%MtGBZ9UOx(LSDOJLkFonbFI*;n};Gg?b0g
z<(i+oJQMk1!ppbTNl^#WYp-#uoeo!<c4+&V{a-EJt}IQG)y&#7LyY;4QAUVoao44P
z2KRP*=61}TtZsOCw?X#<+oP^h_imMKj(D44yHs`Sxz0(3Q(Jue+V|Xh6`I-lF*$VO
ziA~%ovdM4eq?(GY$qaRT;+<llzU5t1bjo5=R|UuOnpaGh{WbL#Tb`o0O@GaKgD+1~
zo7LO)-dgcZdxf3z@mpp(lDS7V9X=>@d&|<N9JOAZubyr@cj2=`m~+nm*Pc5s@aiTu
z{t$PoT-fi?A@X<9#~C8adNh;mdSl)_*#B(Cp{e(7G{xlJ>-3UdSSb2@uQLBsKKs4K
z>)cA-WjQ_Pwd7RIy02=QG<RoA)!uZq%aVuRcuJh_pU<{>;p6AEUq#dJyxDSK^^XHV
zn+l}2zA*ZB?2$p&CsnuH(?Ui$DLe-^XLK(W?OqqGw@~QPmd$t6KEL~r%lkv8@lD*x
zt;xdwC8nKod2Y6J-xj|)a!Fpcw`~01yzz6AJFZuCH{^S+QG;GqisEt~x%E2#_ULlh
zGD+@DKm7LHmsri4S!?^=hkp&#3*5l&cl;P*r$Fhc_;*j!Y+dGcWY4vD=&RV#U-qog
z;s1y0k{!Y7kuHu88z=IMXx)jDcystlZ@5;@-Vnne?YuchZ;F0T^xxEbOz-o@zju~!
zZc1Z+^=q@urwKj*%YwA0Hf>+DT9uvK>e046o1)Zbmwi<!XOeuo%W_gl`DTqnpP1CF
zR3E<RU)k+ym3(AH_l}5fHShV3A6#-bqt$HrR+*HT_m4RPxDQQID|1xiPvu`Ew|$Q7
z{fiy%`KD&1yt-Dbf75l!pQ8bH<qErQy)!UdmHNeh+q_k0&j*`4<^4I4d2ar~9INV&
zJQ3S$qu9b`)XQ=@@S3mFVEf<aWW!%--RrFQVavaOr=bj@=WgHglik*Oe1aqI)=Pbd
za@perWF3Cq<7NC`p<#Hv-TTMPXH}mXeq=CB=!%w#F8OgP*LC9Kb1HY9FIOu{c(Q++
z)5OK+CT^K{{y;_1wbPt`bjx*GVl%~q^)GQhUg{PZn`+CGd@W^{%cbwfmUAAT*|lfJ
zzTiFg?9?jW1<zQmzfNp>=1U9LXNG&^xT-t(|IV+H({%Y8$MVJR?DfSQVjr6PUF_9o
z?V0v@<NdEM)wdS-UJQPE&;RMlv^l!&0smKhnUnGK)??>?-%2KINC<NmNl%l#tnt-r
zQ5a`mDMS7>m&$+h&vQ<m`S^ao!r0|EcPjsX|M_0;MNVn&yjDTJxlGmjd#;rA{x_}6
zIeO-f(%H`5;{9s9Wjh%=7@64lZu~d8pm4B-OIS(g#L@**bc579W*QY9n4-zC+(*D7
zwfPt$zr2vj&JQZ5rf@A^U}6!Z;xm=$q|Or0;%5y;M&+Mm4rD$&KhLRMNH^=tivub`
zEMAhz8$z|01PNc0j`*5l8y)PwP5M;UgYc%n&^p=OUv;7ym$0pgU0t@|c1z*yLu-GB
zJ@s8Oo2BpO?sA9teLUPkZWbRuu=L3pay>OjVD>rQ!dkTQkCATLLdROQCn3i^H|Q<o
zWE0ih;2khG)oN|t-Ao6~8=I0Z&+`>%Jix&s{N&i&g3Nb~d&Gaqt=q8UJtKpdl=!@w
zh41bh7FO{0j7?~I#yN@O&ZecJ+AEk>-MT#M=Bxh>)%(^h3XkYs@pE!w_Qopx33fOB
zFumqqBl>+$;F_LA0@CV#O%JG5@=kfcz{ORn;8;IVQL&kispC>3Z|aWAVt&5YWL?BO
zRg~Oxcv*x}6=TJO19=6XJaXhd$f4X}Gtq_3NsKQ=%FAlu(ZxP?aTY2QT{M@ry70~n
zed6?O!I62B-)M$}hJU@mo)DfWdG%PUm6)mcWT(ICmwglXo+vtIzcqa@tC4Zph4@-i
z6Xjr$n<poB_N+{~<UAo%O;TJYKrp#io=+vxDdxhB%V{-yTV{sXaA`itTv+NPr0(!+
z%H`~(Yt(LMvs^tE6XaT`eqkl^!)<e%*6>}J<8?>(`Ko{oi~gUQTqrn4>moxyZECl!
zUt4FWQ@EIDd<}Qz&#N_2d<kzhXj-lo(oB{zRqz+ki<lCBTdU<Q!-K5bN(-GTEuK4j
zx`kY1kmY>x&RuaQ%jCs}^{OuIdATVg!1=3?%F7I^Of7+>iB3|whkVowV_cT(x$YIt
zDI6ktR4X>sJ4|?fi1!h3J<9`Iv<fvh`ds5n-E{K)t)tgYCFPYai1$1BXPKklNADLd
z6>7&K=4m;y%}jRWe-M${68bFYqKa_mtPG!KHWhJS{x6XS=Wh~w<+NBWWlq^%O%=z&
zLpxtZmmKGHS?zFuH{ct?dCuhmw-c-92C(EMZ@<0nKCj4peJ0PF+a0#KB+hwr&*kdo
zxQ%=Hu83~oR6HHLnMHE<nz~c(L^k=%@aE3*&DzLleERV_6|0wKopsY*Jn}MKp1?Y{
zJ8jqNJ`L8CD!vpYaji`mm%_epTN3bY%G{Rf^V}0dj(z0xwJM!x(<LUrp!k!8RfK_w
zL5G2XfuVuPfsujZKLaaxg||aEo3NHi#HrpxMgrnqb0P{vT#baY1-EDv-gW5b&QulA
zoFMAjsK8qBCBrcF)MO!FDHYDm&s?T6UtRfUht~p&*$&N%d}eHVzH1)SF-6gYjq1k>
znZg}qca$WCb{Vmz-1RD$d(!YN$F)_@wbwL+F&lEFC|^;~o|&7nNmL>{Vf7m3`)WsK
zPuFvtoWZ^=aK_fD56zaGnxN$D<?wh5lXS7*0nY;|jZ<0Umfb1a;Ltvwdzz+2#TKUv
zOaa0!8Rm%zO=WTH+YaA4=VHG2>bBmMp$f+vXS0RfyHa5ib(iUk)Rq+!7528AXT4PY
z^WuY&#x2)6-|W1*f?>JO<ZJ)`WC~{eoD{Y7`#QTVw!TeDk5@0B8T(*O)A#q+zt@$1
zKRM5SrP2X^TgT7`<sqHN?>jIFsdUu~+wKUg;QqMuLN)8gO{X+wnV3(|{dU9XLN?b)
zwuhY>|3WV$ubpsdQRWQJ=@(K=rX6hUo%YZoGS$2&$W1)8#OT%pmZ?P(CNk}4QgL0e
za+*N4Cfn1Q0mZr@sZ*J4#g^s^Z)~1B{nbHpfwmiK#YOzIb2XRE%3V5hnV;3)C6oP`
zH?2&Y^>(e2pTG3C%z1M~zokuTU7T^S&+Y70<px_7u4~t)Ho1kRE?&#HOJHgG*P~of
zb8cOA)9kXGmg+QT!bauvsY_2uwghj}I*_og?|;}V2k&W*Ryl9ra*3W)u2RB2C4f)l
zb_erSv!@Pu3g#Emm=9LyY}A}27}L2=L}ihKoR-L3hTB%=fwgRz?a#~Cn3!J3-|uN6
zs;!^udVj`;B=@-t4=$PK?`1x#w9ab_$BvNZ-JTH_f*6z{gyZZ5T{bK|-D=*|`sLfS
zxvihYo=jB}_<!hBa#UKB=4=6Bw#4JLu4&<#`etFPd`+H3JU*?*&2X_-(DHRghK2r<
z36uSd-V_H+=sLA*(f!=$jU_4<KkHRxyPf%RzHyr7*2}EAzNRyo?xe@>Gk7R+WsY_0
z>3K`zy30eF0)0=Yw#0PiY-EV*Yns-vb&K=%unNuF8B<T3{NIw<rt0sx=)N*jnp#46
zWlh*4fBn80Nj;8nk}K5wx7}DdLw~j9>Jw#o7dEr-wq<9{yPX#8xRr6WwiDl3u^cCF
zE*9k;_0p_IQE6c&6E~&%T2Cn2ovXdaN#UWi{N6a-7Z(qSEquM9f@y7bTG{db*lpLZ
zZxt`Q@%3E0P1EV<Sp{=hwa+~GmRcSfTd%RK`dvJ~d)8N<P<{W?8p$6-#V$r>Wj?we
zz<j4O;>n^TDQTe&nKw`8962C;m&3zu(##1dZU-9`PV#QF_So3fs*(`9LD7>b@jTNC
ziy0Gq7;+8@`P?a2dRmgLaQ@@1-F@mp>#i7-tK@L|vWaR=zAzyx>TrMk{~#u_rWa2#
zxE+-a-`O&GCC3BD{R{cHn-4k5ndRwA2zM+xlhl3S18*xc=dHzAOa~asUg^va@>!vj
zEHzcM#eK@dLn}jEkLy0w3STHWS1#Ulm-RP8Po|%t3fYS%Xs+~Ywzxg@dCGO23r_=-
zW*<1A-=MA()^W)BaC7HXGs|VQOP8xW^SXVjWch@ZSJZu8C0@TWt8FXy(bv@{tW|eh
zP-AG=`ZWHzFvo!#(-kMM2v$A0&G}iw#i!L`ed4h%3&Z!B>%BdBLFFK)cSIE9qU^72
zyf3wq8E%*DXrBAV^=#JZG?5jH*=9{nvzQg>Qo^=S@ZFn<=BFRHgm?&4i5;0I&Ui{<
z3G;vRyn`2V*FSl3#p#*P>%Tq=!lh4YH~%=@$8)5U?^fLPW77_A`uogLfc04($17+3
zOn%kx(#+OtECt*qGR<e=dF&z)E^fSaN@1fRPkdgurjTJ?;o6EhCs`cYPGw{oPUw8x
z_e3M)Tu-pYvX?IDVlSs({;+Xn%^hV&875WZCC_-*9DMa^n^wc+($I*7oM$*Hxk9&3
zV2e@}z8Ygu<m4{vuC2l>x9!hK|IR%VG=dk$9m!j9inX_2;fdn2c;O_4cZI(88(PlS
zbg@r*W7)pZUZiv1f%XLl4;kk^+a(v*zDeQ0q)9X0W#8L*Ds)onrqf;PYi2v$*vO^p
zuqLWO(RJs%RO>DO<H9u7I^0aYQgncM(;1@!r4KC6zEpnrt;6GWNlJ8FyNAP};Mr`R
zKG6p=1g7+5Eq747wbf(e`TD9(S>rq2QHzfpNelY2fNO8C@X^o0>P}D9Uk2xvPI<BU
zlh+mnNl`D+%e(c}cAd}bIiwi<!zkg^v=zD+*3NyzQ#@m?n_klvV-EEWt3<sL#=g=e
z$t{waOPdzBmDC(je);Xkxs7Fchq*R3r-$SSh{<!8qz3453K|RkyV;wu_D|TyPfRzz
z_~gW|PH1h6a8!u$o2cYnD1V^KTs^YO+wN1Bk<RMna<!%B6C(D#&7UMQPhs_W+aA^H
zpHdE;Zjq@}l({GnIL&O`)H9R*$Ia@PZ!-BY|KT+!w&ga?n0NV<uV`YqOPi)bYQ=N;
zBB}Bn{3#2p#Lp(KFucBVnv+M<)udT0(U}=c59GCvJh*7|(_nAl;e|C9Q?~K0WYJ6)
zej2zy$+h|3wgv8;rmKD~nex8laHYFnX}HhXN$u}kHp`28-TdTzrj<2%PhDEOk5#YG
z%nFyA53Y8p8KiQDB|I^0Y4h~S&k~q9O;k_Y)YCd;^U66=$2=UK_DI*Sntj?w!dRrQ
z{FzDeEDo-Ord~J0uH~~PT>R<UZ~1qzbkxNqJ}cA0LvlYVdRGWZq*fbjRpnU5blE0I
zCOCuX;@N}`N=i=|^!Hr7{ralU?lYaNm1iFQ|9x4bSR?AgoH?8H`+wH8cz#xTy`+XM
zasI}KFZU?6zkfdE?~}%JXH+jr?JbOZcAD?(_mr{=j|0A}V7L8c_;qFWMMIUPX${^?
zSKQ_bs#SmUzY#t8wpi+&?SVlHP7Bz^$TPll6KUyK^X=}P*4uNN&+;l{r~DDHTXx~V
zSC*QR_cBuiR%VD=s=cqN*PR+Vr`P|z^{x!nNqS-3$3J~L`)9rR{L16=qpo?a+&WL?
zzfGfW#;2nu9YN=U<yNi{*P3zv&&I|{YU)OTN}&%Yhwov!cki-O<`)TX{a1#6;zX9n
zvwm!6_;Zl2W?Qpvjd;xi^{u|Q4R;*u3k;J~kU7^gIkYckg~9(H$L-x&)LJ}5X2v=d
zJyj9df4TpS`dr_zKQpZxmWA({b=JwoSaMhRhoE(V0@KuLg~S^;Hp&0K82-IERWCi|
zeX=O$Cay&X)E5bO2?aJrPHQxEEb(yj=2mVL@o4mIY!tHK5;AJi_#rD4l=k*>!xJIV
zC9M_7(*oZV7YbW6F20~{<iWkyoj>%rZpidZrp6k!g*D8MEq@Q#yF_p=4luu|#Bj}|
zL4I-&!_)M;UrcJRDWz^Vx;MRS*5We0EoFzcW^{$M*(FI`+%9GMF=OFo$I9<21r-T*
zpVxCG#(q~3*<&1Aan!}_Vb#hq5#58?buZL?4f$F*TpuyG`nNQ8YX~s?|J<=`x&XgW
zXN;04(`S*^j?VjuP1~;7%(1BAxMZ_kOs;E(*@PDzLP}gKSD39-S6m#?+4e(r(f9VI
z4xtrg@f(g+%Dt#qeW82z_v-!TV#$szOb$J6J8F(rD7n1g{;Ck<w8B;4Q|VRJhD+Nu
z>zMOc4fDzzgIfbF{)Xh07fQ$;ww}3A`Yl_&b84b!S>mT{mJ;9VU%5+u{89hsT3hPV
z#6Zzd#_KA}o5fZzDyz8(EdH3tBw+n-nsg?I#U+o{$foS;q9R)taO!a=P6(f<Wg#GH
z(W$m$Vxii^SqfZooRgRoCQ7UnnE60@@{TUaofFM}xcDFG)RvsAV>#I<a<a+KNs9kH
zC-qBAV){5yrL$^c#zd!;jgEyW3O@`Sw>9roFLkge{W_uL`1GD59W_Tz*fT%mKe9vm
zRJ!8Vfb_M23BeudkG~qv335m~-kLF8>5NcJ(vI5P<7EPcZO)q#S<?lSkF=?@NO36o
z_og|1TM^V~Ud!T9A#jK<=|ZZ{!_LJUW^~V-vG3pvJ<lq!js9W_yH>8~($nN>ubjbj
za3<4(ne#jaW?Kp@%oLbBW9H1xnag+1TCsChQo_ucKWA-PIdhBV>{$u3lL}_f-#L3R
z=d9hF!d)6O&j*)e2v6C2qI*9_+yV8PBjqieg*|MEHUB0~b-c*Gq_k9il0dzAuY+Si
zN=om;8NpBgw@LIQ)}2r_zN0P>TOj!$UGj-|-DCEc(k%j?H6|36&wqY%`kN;q=O1Z(
zyJ1|~*>AxtC6U<Y=o%L7#Kk2gqMkA}xpAV<4ng4)BI-Xnq!lOjL^Rs3ob3N)qO4Wt
zuAd4?1(RkPESl@NNbT3`C7zSGHcnQsnxw$F$mrI>z84D(e=W3-T0B#M%X-#K)n7Ab
zDoj+1TBK1mNg{Kxj+cPjE9t_ph1$;*+&P=PPBsULxA<q4x_ZpL!8zAy;?zIMQ<)g&
zx)k@MR88$VB);`PPc?Ic@Wu2KWvvCfG}9s-Sh@TYBHGNI*easfs=EAMl(a2vY%>*{
zuwrvv_6`qj_Z8WZ0yqDcMZcUTaM!gndBMyjFFW@AbeZkBa{8~?yDTR$Kb$yk6=&3f
zCCV#Vy>~6K{kd}StCehwt8}<lt@ILDziZW^Ta&j`tyb4uHFE*$u2(B}uUfr&)sll+
z0*g<r-jXG7RBQF-%%;hwx*gea4@lH}{jeZBa_x!ZOB<Bto|JGn_MAKJTTAQ_39Tn{
z*EROCRu~J3OUw=uGB=&~l+9AhIpb@jfXS!451Q_xVIe=Gv}R9~xa;2ke^uR!?;+`>
z{oi)<bG%mBq$E|pWd8i(I`vHx1gC4RZ}gRYY@PFTI_E;cqOYqAUQLuSVzc91Y^yoR
zu5yM#=O!1cMLOP#m?p3?6-+Xa=45u<%=ABTGgHH6uExz~t66Pk3z%4MjxyNn5WU%X
z_U0o}oJUt}VS2FH`}Y>#(_8$cx32cu9B8oBE?OXb^%fVY#lBTjq@1|d1aQAmDBT-S
z8n9BsYscE0=&9#51XHD!nJrxUcfr#9*-Bpv>^nXwwuw)B?zNHU`m%c>u~8PwZBrMh
zlq_$5rd)ff>^Q4nr(p8>Bq_$Iop})}@@FXZZ<}5lxtwL8>Zg{V+bacfm$pkP3d-GH
zHD7Ag(pM9Y-dgRud-bO5Ra<ti_LAl_n7viMd$)PEKxD)21HX44%HF(RTVVg|%}2TS
zERNc<+j`GA={*;I@3|zs*Mxh|zT0~?SI=H#z}EA!V$Fg%TmLJpJ)FKhwQKG9pIO_N
zuHEOrf6haoD~Ug0Wzgc&bCZ5<zj7n@!OL}-TO?G39sgd}$^1EwNvDpfX2&X{*v~WO
z|B&=!KJ6h`rk%93|Dt;6C-s8vE5@9i3yuZuzx73m+e>iY=k@ugT@NyDK4vZ8;=N_B
zw}6_Dz`?35zSaVVwGJ~0vhI{-J<@epzxz;tfuL2*-lz|Ut>&<r-Pz(QbHt_Rh^r2R
zYtA8ynyvcQN0=KA8@xW^S+h0v_QdW;y?n=Q54}qRW-3Kx?JJa?>il9(!ELcJYr#~X
zW5GM8Mw<wvEEE?>-ydN(Ev>axHF!sBG?(_}6>N^}yAv~Pwk~fk%SicAFZrKGq+GI{
zyRdxfBB@?3<z?Obx`PA~vR3cRTD2zXuzC04WwUnAoUq%vXUlHyqb!1)PBura{v2j`
zc*^$8Nw$rr=II;}{BR_2!s$Q(#()W@1p^N6*I}@gIlbr2X|pw_BNcY<{5k8+>Am-V
z?|pcB&f)1b$0Kyp*9fHg%q`n}_GI{4A;*39dU{+X&OPfsd#Ylt(~o^pje;y{S*nTt
z2B`vbpXK@p_q`Pme>5Yyd7DJci48y9)=x`Z|9Z#z2kGk{l<)YPHD89U{FB56KHkvH
zO36lPNgvw#|5b$f6*)G=2zW3#2plh%amn!A=Yub_7!H0kU$;m4<g7JIa`w2#obuN>
z<70D_`F|q|v*2l?yJClJ1TB08_WrpX$iNzIvt{Q=Md?FFZ!wl8)@%>ld8{Z#cxKKq
zZo#X0+`_?~$AWnk1J?@p?XAf-JfCjp*_&j+>Ji-Jy>6Z1a@|W3woi?_xnkG~m-no8
ztBQ*0JMB=UX*|Vyx=-rKtPF9!jrkH2r8Uz0lT_Stl$5Te2<?&DJm<`!-LqIv>^>aY
zb0})Jfacp%nz5%%?w(#RbB2|X(M*?Nf6T2Vx(tiXo<7RMx_!=RPn}h_PVe3O<IDr=
zwP#nZecIh~D*W8tHEYv*ZVN9w`<z=icaOlnv)lIg3LIZsb6S_-c<tHCCtKD`IC-^A
zQ0q&1hU|{5llMQ&=*|5vkP&+$`nLE9=8LTHFXl1pxWCps@V@fEOL657Zx3+S%wx9R
z!1yblxBKGXY5kU;@AY|I3Y(;NH1K5Brb})*rvqzla?~j;)@ASzVCBDen!j$VNZu;f
z-sTvaZK>VcLVTT~=L*R42`j$4-tRKE>ypIV2Z~o!%x~qYD;_@HR=t3Ww_nLdS;#SD
zmg3{Y8#`W=oG4i7HtF?zmfbP6b9esc%v|3*o#mi4XJ)&-^IBsYf!$KObz=qgcb}Tg
zE9@_GW^K(aQ{6Kwd07Q6oZg*tc=4ar>vzs@kG*~9`JJwy?GJqf_cpX>Fr6!^e(v<~
zPGR3wmcV1V6Ym813dG($oBNQvs_|t0|8DWeRqgNA2qYHvzU(@0>AF0_Gvk@>z23BY
zU!yL3DqH^~Jki!Qq&INayAv0_xjp{BXM@l}sVU3_tsxuQxGuJ`oG_A`CcJw>@F6j=
z?+x;Qf;Y^QNt$@qFh_WG*5QplM|A%@jr#pSQ0~f!xQCGi58Zn!+KV)DW(zd?J@V<D
zyE}2aY@WcKKW77E?<oAc@<{Hk&XOCOrfUBFYRFwFv_iE(Pic8;^n$-N(OK;>y2jVb
z-afXQcl~vX^nAyi;x>X_|4y`f&3~35)sm*2P;!@zO*4Lq0E?es;E5wbm6LYtJ~Acu
zrf$qlbsvHKJ%^S1&zzXE_e_sHKl9p0x95cHd|tHscFq5N0{Zn=jriZ4JaPMM@AK=Y
zjxj&v{u`ist>?~PhPnS7f|v}x#N7Jq-2c3(@9nX<0#)k-g89E*-C1+?g!os1-UiKt
z=9@24wp*khy!+VlWp1zJ9#`wkodR!8-pl*{EqCuXp*F4e9``@*J+S(o<s#<-=XV<4
z&)(m4=|0cyiw=f`)6E2LUdhd0bxGsT5og_-;j39M^-bC0@b+OefBdSev0YbZ^51Qi
zySm`Lv8#M(RyMyU|6Tw7&$skMA~L_$)?Gc(A^!U_chHTwHEM-fzROcwUmM%?KVLEL
zori$CD_isd9o@r^pKfumi@PphsLi)pjs5@i8Q#4$FVDwvn7GbVTKizv`u{SUPuQJG
zo5wnd_x7Gyv(L_HjyuS8QLD$FQ=}~V>!}^mRrB6f*1bJ-o!=u~>8-+-u19<fiU;N~
zH2wYa^K+?yqGKDctkIY60S-snMby1=GA=5)^+>CJoBSfs!9Hv;$F-siZwBekH5Qwm
zyx1V>ShhCe?2(hJ&Ylg+_1LQ9+9vGmmm*PSSyi%vVF}A_o!k?xisF7TYvPu6iE6Vm
zMy>Wz_3%;(S?Rj8>uHGEny76&w^ylhggZ#L7pZ=G;xT<eTzkpOb0x;%4hj2tzV6y6
zy*c~vOcrV5IX}Y>Bp>fO+E$t&vMi}di!<vaQ&hK@;jCjbu9<jUSrw=nxpC3aRbsoV
znEqGUu8A#Bai1`!_CK3k%#IJY_BL)`oU<p<wtJt~&Q0I@6LPEny{gQ6x^rPs+nrMW
z=X0VPe%yM>nJ4mo?sn%grWmuF^z-+UPfbX@#cu!edEEjBPN`fDhlfwR&M0mWWA@f!
z6-=3MaN!A!=57x4DI1))^rr|o1)X|Ou#QQvTXVzxn8c1FGki)oj<&~e^5_|P8F(F7
zA(xc7`K;8gi3??9cj?%-Wo5fJ_-@H~Yg(4Eko{!TtH@2$tMY>1+jQn#=GddP^a}41
zjosXt+uzpooRDwkk?K6zpR;?#!rMtgo($0ufhWb=HJmaOy1X=(PA!|}<FPbs-RZR(
z&1$13U(n?LTKIo&dyn%=Tl1MLmsT7%DL$__O(yrn-tRAVuMD1@b8BU9Sc8yP!=f&w
z6<RA6WOZDfv8Bm0EOMjL;^S>coK9ak>Zi+mJ+5B#!P<m&Q8UAR-<bT49DK9DJMs{l
zvCpF9si8b;n1qk!EN#z8J-p<M#o;T8kB_mg+pclaS0q;d=u{7zo-2?0ET6qt**@=P
zL}aUI*`LIvCLg8<U-o|eOVPjKn}L*uM>%W24f*POSNu->+8~-2@IXiJyL;50)ESA<
za>x1~pVXMvX+N1c#p6Ks>?AGCCcc)+RV>0H2Q)Tp3n<`P$q;kl**b$iH=P#hKPhnD
zzR2~PV%EXO*_j{aOk2S+Y5B5W|IfTX)V*SQ-I={bcl&B?8!4}syd;%eCcP?EMAzh^
z(qtW_lS*CN!vzyWH8w41SFk)UU7TM(=gzG~H!~8~EeJJ<R5h=ku=(=BJNH~x8k<Z0
zN)4&56MP_FZ<p6PS9Sk74yB9h#jHzop1;}j>hlZ9#}DLLi%O19EL<b9_y%WEt7<^~
ztC!1zT9^c`{#Qxbx$se}>4(*8lAfMg^=xOwJuSw9)Q(>ZJtp2;W3)xw`_STU-KA??
z?k;!}d2WH8PwBZ=+}n-X-4tC@7$0o+`D^+5qifvex6gHQTrG|GZMP^iXe?P~x;Bz^
z(ld1i4VOHtzZOCJA~r6o+TbMo*z&yArhwg5UzGpnEjsYPA;MActI8XtcQfWX<OFtA
z$E-8E$$4pa!{!Y$E?g*l`)m4DJ0_v&E`Evo5?a^3X%_y|u-slvIrMDHbLk}}8})R9
zxleb68|xkMsIm(dI9IaFc+TD&!#J5!7mjo+<XpPcS>_XW%z|e+Z4SP!H>#8!Jwm#6
zEI6YfTjZhnW0LaCs~$>qPcD85*}nB|OJ=OXO=)K#CH3eEHNW2Oj`B0h=B;bg^Zg$!
z@`}lQSyW7b@YWaZN{WZLG+Yv86*l&<KHlba<Z7B|pa6p|ml2zaf`ag-FYLCDo~@{7
zS!p#rhwrgc5%+1qC*p=F67$)L=6M!r2DVzo)_ps!`fkA_6}A73A;;e|3Yk6ebKm+~
zO+oV0tipZIg6I9}^s8du6Fl`*h|9`bYo2JXUg)sY{mSY`;Wu{L)(O3?{wH$H`I5Te
zXUEl%xh^X%am)%>QTW6`c`<*%t#HkomMOx969ZExSjx(#9ZB5xVY2Qf<H=D^^y`&A
z#^*Dc$^6%J^SO4j!_n%-K@O{vDoH1#-AX!HcJ;RBMP6Xh?NjN}aX6^*cgF&aElWBz
zPWTE(uReO$AeB2v=X2K1l}9=BovwD?IVmc=SLOeaQ}%V0+5Rk<?*CpSukMr-?EKxB
ze(K3D0ZqT9de@95#Kwmm)hlvLkNvqizWkZ1zLtiQBe#@fdzgs{pLS-GqQ`${VJ=<E
zLmOQ<N{?=oaB7mYZRzWpx#Dfe^8@W)uFTB&b<`mwsoQ$TqOJKAtqggI?QT{_JXs|t
zU*p)eBueFEF>9u9`2tPBc!?(pdn1GGs~q`jUvf`LOlp@1a}L=#^N7m!l)j?uVgbf0
zO2PsHuKGVhPnGhNHz%)P*Hsd1{r190SZvK~`)i5~{#*(9YFm1Dt#B52`5|2CuWd)@
z#?(1$7V;?{etXMgzsj8^mF*KUSMz(hyxKQwuZv-o;`4WIPlbax^E)L=xjxOibTiCW
z#PQ7TQ=cswj9+BBD%omze_ON3*2sfHm&Gb7=Fsy9x%kzJIW;+nKZGQgI9@x!$0*pb
z|G&l5Y{7|!0(<)%43+G2?|K<-%~<XI_O-+HE)9dcg0-DFjY8!=n`KoaTQ&TY+kAK?
zG(-igD$H52bmD=KbB`4{bya@se0kwfQ&G3m?TCr9=Q%m@aO8+M3aE2!<y6R5SlKSY
z(hywVU?B80LLzwa2TP3w;nR{8JVcHh;M={>T)AXR;zNm<mQ7P0CI3zB(4CRYZ#hvc
zZoTr|%+s6qPn)H8eZOa^K<cNBTLOQa6#RQuaBA&LYb`gu(%z(?dAwpvbTT$rPoB4R
zr^b!k<RiB@g5I=VQd+phK_en{L3fNp%9GOnKQ>Pajh9}*D!%*2jI!_7rpX`e%)hGH
z<J%R}q5M{%;s3R57kP3Fn<Wp_w1?j~=)$RZLSq)!cAu78CW|AES{3Xx?aO8f^3ysg
z5#`LUbv0QuFWdN(u({mi6i1P>qOKxVXXlijh~fScz-PUvp+zG=<{aM}b`E1kuc%#>
z)@8}-7jG3{z2?%G+<1=Vvg{_G=nI~T0`vW&IFqivzECy$1m~rB+kP+gS;n1{%<NgT
zHuAY)=8*{;jl1>r%)j?vwbj^hFoCNu{As?>k$%0xPTxD9SC(|GI{b!b-nyWrt3!KT
zEk2iQ{2Y-ZTF&)p-m;?Sq4#D^+mdr4<0w<ci^4ksXX94$mQCo6o8da4QJ(k2lI`1*
zUQJHD*>9|N?^aJkS;}FPkN@rJ^FH}ZXaCG*%(I_|JzS%D&hKa@p>@_f??lY{oT?Ya
z@~dg0Krh1<hwWVVd`0%IJwEgEwZA$&NimCWDBNB2swDl|qLq$EEzezkJV|GV%cBi{
z`s{s}SR;*8E(8>am_Od%*1^~Mll?EF$lC&imnT+5aR}8nn6?QR9DQNhcR<4Fl;VR|
zS~FJlvE;DotZx1Lp;J($RrdnZrCEIO3asvhBDbRFZ}(dCaN<F^>P5SXx)K(eWiM>9
z>XI~ip=89dCik||!BrdfXz3i0T5Q-PVEjw8O55mc*P2$TJ_{iRk&RA!R~t+{$f2=i
zZx<JzVi1ph4aZ~&k?I|+)fyI49VRZ_+5Z1(R%`sl#r#W5U0?OpZIEtO<hYyEzLa^@
zj)nSR44ut|mZ~=ow@tA4p1}8RR!7_i6%igQ4hgHO8<twD*>z`F_Fb@KFmVac;;0f}
zR@t$$dxrv3k?dp#rac#>HFt7ucrCGUcklh*+dq3OdARDpSC4hkOEwp-oaEZHyK}<s
z_!X0{WG(40w6M-#{Z!ce;h^&IU1nQXY~(F4{qbPV$pcfoGdrh!P~&|)_prApW1vxc
z#-8?wsWS{kS!as!d2C$QAg33#am@)a$6fQJX3I%-8%qD4x%Gsh)Eecj54!7m#CB)c
zUTR*`U}L0mxTEYd?^z|Kl<rM4UI=Pcb1*U<o^rtSQ5Ng}aE{H^3&bAn);X2Aq&r9Z
z-Gm*CGW>tLXNNB6RY~NSvVc`%!TLKddcP=leC=YISlxTOYm4XUm2C^8brab6o0^v2
z;#<<hq<g?2tH}P_YU$}BtST!NqE0bg<}%&cDf2XGw#DaJzkjvAm~kL(_tqI5;xdP<
z`%X=|n_<mjvNB3QbK)tj1rH32GMJ4fc%PH7eZO+1(F7(2$7N+7+QV+}9ME2LD#7D;
zgJarigY-G8r>$HpU9u>ndz-4Vd-iFwISYOCE-2**uhISCmTkR8g=c$Fw$nBTbAHdo
z3c-s{HknVlxVR?9hx@bjzSqj!!iQaMp5CLpVS^Nph+*ra;*M&Gx%Y#Q{J*%9Bj{3Z
zjF#3eLE$DxD}hT(y>mGHclB<$*z~pNq>lt^l_0A}%#^@}1`mZf{wBNLF_?y!Z18en
zGSuMber4IzG5hUfzUvxuY-)sV@9JO?p6elW&gT=K5$B8;lhaeXI0Cz@>luz+F$yrQ
znUQGXmU3mWY%+JTw$sBIi>~E#>t<-b?eMKVZQix3vxG_1maD65mHM&n)f<hDM+P1j
z6>iYYIIgYb^vaR@!AbSAi}#f8;*nvt{m8kuc22jfum@wt-pk1ySD7ze>2XTAc(EnN
zL%gKp!Y9F96DD%5Q9O~s%%{n`Jg_@VVn_S|c}-dAO(rvR9Gdnl-XHpc{h1J}nRf4g
z$Jgu;ObfqXJo(hZ*!RYQzzqKBGOR+UmijL-5x8a2m(=(7_5KM9f|tJPE1j?}uwY-*
zA6~b&D=sVT-u!Y^r^D&j0jK%@9N<$uxT7b4QSrFy<ZiCavuy^Anwz}0-)Q0$SQHkZ
z{UO2eZ-m+Z6Nip`Ii`MyV~K^1Oi#CL%S^omO*)JA>=SZ2edNskNgXo`*QD^8b2kcb
zChC;jo^bJtyQX!}sa1#id<2gcDC@XLGCo+-rqO)$iBrMP-p-P<PbY9(H=O$?^YrTp
zx=kAUZb$`%cg*-Rr6a1N<$Iy0MfPlGXCbajO`m)GC+yIR334&4T<RssZ1Z&Oi(RvS
zZaN%%XL|Pp;s0S7#-+6#Hyt?lSK3*LNC$q3kk=H+2og7%84}-Oo$%*qd*aQ+l!?i2
zI)gmiv}Hr52l}W<txjNcTD;;q=VR-|1)SM;b}wAI;?U{I0->9N1l=D<xbOYdwZg)^
zG$!zQ;&Hahi$@R4lU(b*aKb!+w^Ls>axY$ZLFx9P#EWav9iy|au1SvEdtkxh8{M4M
z*$3N1&iqce<gjaM;oEf*P6t<Bp8ev4`=e^+>r9v4NboU6Ex7B@6eQIeSRnp*#{!=h
z3w#6{ei?7RCnZt!Lv#9tn4K5*UEna;^>E>(7q{R2I{D9Wcd4d?!CI-$5gVMjCi89S
z+mdoRP|z{x!4;R6OgnDGKHeGmU$~m%L95T+u1=u`O}san+BfVzusHg5a?c@|c@maq
zUWy9Dlya!AQjYNrV~d=y?#1f&s@G1*98&yZw)>IK`7dX-C`>bM>aNq7ptUwEb)nO$
ziL*BHE;+wxV!<!5TY}4F4X=OgG_^SEtY@gw^Wf|wZws-RJ$D=oOcbqs8yW&jObn+^
zdUE7I9mCdu7LGE|?dc|rC&GO{h`*6$<+?TF@&?cH!bxoFRAi0#Ca|7fEh)S$`(enF
zqb(X2c6zThE{RO(xoH;QroA?9_F4WNtl^y!wsM>ADX+V^>jo3|-i1GZMDDx2`;VyE
zouf(fJ0>107JcF=GUH&^p;^oIlP(&SZIoJV^MCV$sECW|n-wmWe_5{V$8$ltPppzd
zX|9e!>ZAK7f*#JeJ)?sE*F=uVJ65!7q;S5G*fJ@kN1#7CK<r+QX?VibcxDC9$y=Hn
z_V4=XSslrG&7g1Dqa#fVS}!b^ZJfO2O2y8(F4zAQg#1(7U|`u1*{u0z<CR;sO7mAp
zYN&2x4&0Qg)4c3Q>Q)2h7~f0AMcg*C0*;y_rGJiXZxGxwN8(vVYyOjt1JMV!cil@|
znAWp_JNx6a9XIaH;5_u<tYngG*eBbt&%CoFS1;mPxiM9@S@p^B4A$G<Yn?W^`gYEU
z&#`q&^*Lj8YDMS6?r_(c1v7nrC$>Li;$yMfVPp^^5yqn-xa<Gh8_wrN8KxMUe)r&=
z6y0}k*U^)q{n0AYE(^~ynq@xK@|L`I`QpQMEf?m*t6Vs}v)%YfVpVPAC&na&Ws!Xw
z4!%~Eo9+9^*6-26G~W*qX}h?laSCu7b1F_{h<eVtJbbOF83&inw76xu7YkV27cDqm
z#M1pD(7kw0)C<phrEj{PWr|kvbgd1QTa%Go=`(dhYqZQ;?%ZB6vm&V`o=x>~eY-{Z
zP6?zqYK4E@$u3%W+t_FApAEAEA5T)rymK|8^}ONqnA?+1*)4I`J#AeaoAqeYqlVOW
z#@x!2^S2(k5`FA$u9?7&2?7b*#HI<}OAF&sKXh;Q*I4O4smpe}y_>t}_PV$KXJ38$
z`sv&D3A2n0uBoqP-#?EtyDxO(1)rT++n!&0mKd3I|Cn~cJ0`sydGBH`u3UTOo!ax$
zGKb`AL|GgkuYTcEp|-t(P3+>6<KL!Ut63NJ^&t1|9j`dfirFoC^ygVm^Rte8mYH3D
zq(2xmK02-QJBmYcmsFRBP~Wxl`fFzee7X@+v$AtW;+-vtzl{oH)Go{rJI1n=HN3z=
z{@hE2bCJI-v*d0+UbQgWa@$p*jtQC%xY>m?y%tWnz0qyevBK5w9tr7w+`yJBzn^Pm
zZii9W#hl+trg_;XPRqqEluNsI(bDZvTgKk&&SK^n+nXmHy3fG<jq{Asr)&$hwdGGF
zHYlmHf1m%~sDgRNif|vHzOZ#3U5e8;CNJZik{b1`b)wM9K*P6xpMLH+bTzRpJfUvp
z1F?o*veO=(JHB9F=x^7Li~hZI>yO^<@$AnEv!!A8+B17Qgmo8iDEYN(H+jVUPV?Qf
zYjOUytHJi~3;Mbe_nryU4?FXuXXeL)lF1u6D$`R%uYRcuo1d}f%I@dKvpBTwr0<^A
z@yqV&wplAu&c9!9_5J_1yFYB0l>F<~RS}K|m%H~DOmel;W!JuWY2gi%ox6D3Uox-V
z-OgAzV{YVDg~H!+OBR=vY_$3)H@$puL(;ybd^0YH<R%_mwd_^Y#U%Z>Jq_V|*7}KV
zxbVtA>v+?8_Z6yPs}nYgg<kw0GPg<c=FH`}kJlzVmU<hthV{0~_lmkHVh?779b8bf
zw>e&UQ}l{5BaP~gV*=d=;ubiI<@`M$+i_>Nd+~<}2TnZCXf^&aalMW3*9Zx#(<<lt
zmfV>8^v_MTOUI11?}-hppPGF#)G0s7gI_H8Y37{Px4S0g-pcC`dhm;B=G(h+)%QZv
z9M@N^DqnL`jhSykQ^qs#yOEa;Em^!|>D8CH-M`+QdA;u)$HQvYiN{$vON-TN!rAj9
z-p2`5&y;GO`pGT7?vj9ZZ`ehj#f@3BY>tKXNPNp?y{@u0zoyQ&uKBF}p^nq9-)ow>
z#CxlJonRWLlfbU`Qva%}?#zPt>^_5sM`hH?|MRC$oG$yZfLG(nv~s(;lIXXEi_*S`
z)R#ZGAGrGaMvjdoKLj|~lfGT&i3)mAuJcZGsi=0t`PBw|aqaw{X5W0L8DY52!tkAM
z(OL)h$VBzi&Hpvd2d4Gqw^-e7(phoP;^TF*ZMT>?7?~nCbuu<gbqHb;?-M%V)^I3{
zmwVf$?Hd*x>Sq0&Y4RdJ*?oedbCF0y&;ieS`OsgRuV0F{ow4uL%~h$ZL)V6_^>~}J
zctQ`u0tbCQ(F02#1U9B}>4lyOahTo|xLB$u>R8X!<$k8qJa2_4gf4MA+7z3W^ziz`
zfXl0LvnRau*unH<*Hx=c-km#xy2RFeT`lX_*vQUxBXfp`_WyZZCvyaMxhf=|=x_ED
zv$~!azBI1C=;WQBhSN`YvgSNAYW043d1c`2G+WmLFRyQO|E%kwraAr2^5W_5WNwvT
ze{kR=i~6~hJVik#YRy!yE_+bic#+lT{R_9n+FNh3#%%B1lRGnrja_+7^u7nq%`bV=
zWOnY$)n(lBit9z_g?$I&Wc<G!-v3|Oi-C(_fj85SN9P;a|2;b2wExfNZd-v{Mhk7X
z?MPZEy*A}WTg6g^>mr2{GDXB{RolK?Ezg*^RiHCw<`1PF^<x&wy|TJnk5;nXJj507
zn$cB2dI^hXTk)xhL9?Ugs6+~=dzxxanB=a>sXBS5WUDm$-igzqr^WcWNoD?*-m^_9
z-O+UB?U^ypj<K?5ud7sZ**<mQgd)zptutd~3nOGpYvWAr7B|E_VT&~la+%>^+13(}
zRK~Tzvs&DCORyvF)#AV!;j2gGYk#w9Oz_g%DH3dJd^W|~+Wc-vchcGk4}+zrDM|*-
zOo$C(3snu6dMzZ#@$2f~y|=0a#Z`OK0@6>kMyyzWqD^Gl25l>AoegS}G{m>*^)0=+
zamS_^Z?~Q8+j^9ZYwP00wjWikyb{*#@_sozUMeIq;OnWXn_>GTuZD{M`ezn4v)@)N
zI(zflX;L}M4sJ6zpm_IKWajF%n(76j+~Ri%k7zFcSafBoU`BAR$yM3%8@f|uPu}hg
ziCy@A>()cdtC?)R$W^u7GP|_2cBA!!hQB`^Z}yb^nWWI<s(nJCN$^*)cdMx1i!SMa
z{V`na@^Nh8-DcT$1LL&r{Yva|nCiGm_D9~^tvA!}@0@y5<Izd8%^A(@+cu>5G(K5s
zseEPe8K+;eOZ-zbEO*=ri8ER1HfOD7z@CJxn?J;4_NcAj?69-a^4U!8y4egZmv?UK
z^4_Uib&GZ4n&mI0=FVFlbv&+Q-lFxZk7~F~<XVt<_N`aBK;VwWDwi7cSBhL-VRSJ?
z*w0FnEj#e6y58GhU8Qu<Fs7LaSHojBtJ=P}rR6G9<X$U!?FQ40g_|5=V#>SQmi3=s
zefy!3bp7q}UWx4?|I-%s*2k_3woZ;ZF=1)<4JQWHIaeI+Or3FfOQW#!$}=ee$}>5G
zjv1wV<YBXXENRfYtdJ$x_n=7I2E*J;AF(ylJGP~W2)uii_;1$?Hf!fODN`R>$OoT0
zE~d2N?$y*%CPm4U0*h8eDz4Ue-gwiA%~)>hg*63RC5}2vEis9AIg#mOegCJ?6E^<B
zjXa86UcpwM_aC~+6Rojn*7pNnr@eO3buQo4s_miPU~#XjwD6;x<Yx_Y+fBK8F*gpl
z>ZSJYRpc^skLsDovVFqE4KH0gyyKrAV^a6<yW|xnbb>+oy7bA3YLk<=ggM1IH>fAB
z^UgUWGe_ozWUInDKY@kvf9GwU@_*G=&44<tlO^k#k4PPT7qIuwQa6dE69RAWDIL3V
zqBvsV${%yQ72YZwRV!MgHFwV$%k{rFYyYrsU(>1`eop9d>YhVp8Ku0D$98#K|IswZ
zddI=`3_<=YUV#qMoGz&!L~hz#-fs6V(|h*aLyvEJU3XtwqFDETjq=)e>=(BFSwAOU
zu~}qQ@w%`dO72T4u7B!tnW^jQ)5N6BvG?6pk%fQKn&&)>Su3+VsPW9QrCgKG9!juU
zHbv@)(VRj}4&7zJ5><tMws&Xv7i$<z_$%P{BAEN?H8*jkRqo2KXU;s6b<O458I7Qy
zHAy>N!e?LnFu9+>#gFsJiNmw(QkSq69+f@f+*Ty@Uu8|I@)iF4i)K$s+-*EPUpr0G
zYKY#kxrcG?ZP|z4!mZ<c7qrcd4rdKuI<1&8@2rl3WQ5?7lmFIwxCEs<l+!$QbZJPT
zY{*M@=hqg7ydNs(?VY1}dy1urrjggNo+~e2x$laMOPe0Gu{k$)_Kj^{CY5iJKItmX
zv~oqzR7IwQXRaJhZHo+y{C$7+Nj+zpzU;};?25oQrTnjLPYZ@Ils8DJ{Oy?Xpyy2T
zb;gN~d5ZIP9@uPseyw^i*9!0O0~ei&_8zTf-hI3_cF6)J3+~zlS49~AU(dcCwW5B#
zx}Keys($JWrl(xW4UT3l>TeEZXc`$!Iq6x@D$>)lw_~Z~;sslH{x5i_@Fha(vOwL0
zL_Id&V%IyfuAX^jzVoDle`>@W*~b$som%-OJoHwc8oONd&z3vkRnk5umAnoG<YozX
zTP{wU#W77eYQ>#Hr^2-JQ#Z0k=1U(b-~6?0Yu3}*+yb$Q?VlUMU2CQ)$IX7;7Cq&x
zW<d_u)oZ_FzYCRZkNa!9;Jr}TN!R5wx9$kBZrCKOq{>#%ek4cb@azk%0vESVoAyN2
zxc1S>+DS2U{+VbT{xU<|_q4_m8`bG$i=)h!PAXn=#G^PVHe&giOU|`Pr;AH8Cs;8D
zwr|yuESHN?O88V3*sk?Xtt)zSt3|rX`jx&BTUMIAUD(?D_q0Nki`xnV_M>wPw){8y
zy(sMMA1Rkq?@Z(Hqd{%fX9_hR=p5bt(&)j~!p{eq&WiutHQjDU!{)m0-^};@nwxK3
zbU7^WCQp#rgg31QXT|DnpJcjw^}vaeEswNqAMR~?q(5bm$Wg&-9X|2DE={PBY5n?$
zM>OJ$l<rZ3;}WkkkJX&on9cWf+BVsffv=UWiKMwHRF!HKihoL3C~WJtIVk+xC(aY^
zb}iMoxOdjKEjruGw5}B$`xWr{kTS!nt@*FnZ00*~X?VZSJI=c;;`6*KJrlm(o4Jg;
zvh})tF?-8`K!rE=ewFT<ecx8RZ?^yYe21Mj*<1hA)-Tnb@?d_Ga<fd;A?rlu{y)zq
zE1cy#5~nNt?7#H1pGRA_UFQD$MQoYEvWx9TmnI&w$y#|~S!i-zaZs1a+*!{48_!KE
z-RpMuZDVZ6+g+}m2bc7}&sKk$v@Km@-)@hWv%~$i39+93eA}nsj{4k<y=z6Tz55pT
za<^=>2y0e)n-63DN7w40T{$IJ!sVZKrByU<uX%m<QThJvcw><xJXgxsZ|BGo%ZTj%
z-|)ZQongk3v;5m7t^~!*P3pXu^ELW4%dM#%j&H6U)_>EI<Z>r5;$BvdOU50a@Q$-=
zMRVBZ-stqab-XpE`(%jYgn+d>q-U;~aO!Q_><2ge&Ib6ujM(%;qWD6)_1gr4kCBoA
zvv2&7tF>X7z!EE{7ud@ZZ^!wcVG*Z;HA7HEMc_+|{*DD6R<^DJP5%Cs97TKLD_xG?
zTFZasf_QLByLQ0|*Bti~Z$c|q$d~U(>{xMM-0*1CLRZNhhoq-O&zgJg#*8M;ACd<w
z7ep`N*eCF)_^5x>gxUKzlbHM%{O5REUpcfhgd@o#EcLC+)68iP&u}~nnKUJ0($oW-
z`90n%Dy1}9@3C~;6Y*il?1(#QI{i_B!>Ns(s}=@n?3(r9%t>v_Q+ofR?|k4q8N!~&
zC=%QmAuD&=Nh5M2r?7sOi=pdrBi$oAr?lDJ4Pt(GyWoq6qySe;pu6^+BQhti-ih>>
z+v#W5n7C$+sMM=_c2gOh<~$5r!BbZmJ6rYtO|iSv=T*up?P}C4P!3Kxy3e;;>D2lA
zKf|}OBsuVf$ZwT#{wBlow)^Ohi7!fy9rRt|`PSR@-R*<D;oUB$*X5`v&zve8)S`Rk
za3ageM4eeDR|?Dsi0U|b%JZ+=@!WX~0gRf{Vtm#{<UC1nS79wb8F6Z<jNeu^-*;1v
z9+}hkb%xWE3-4}RDtU58=AwHUPoM<rQOlnx?Ka0lY_Dz-jlEF#^qR_DrvSd^A#+Q5
zLn2%s#$<T(m9)=#8Fw?`gt^#--*e}#jr4GI2n)P<@O*A}Oenkft-FSkc;aH+uDrP^
zk#tn{+f^l_q=Q#EbUOJ;biKvh62)VerKY8&I*H%lky$JHU+gXGz1uBQXDGPTcqAR(
z$>krRlzKJ5`%=4FTT6Cq=d(zm0}YD~c=)}TD3`w~NsB2+>l)X?j-y#?<yNho#w^%Z
zl*E$sG4AD)dD1+Oo$k!F7FCYakbJ|jurnrP%H4Ajk}du#j*8WAng_|}UX9$zd89V;
zMA?e#R$>9cH4m-B+{-37GTuBC{^U-vNPtY`#j+0fN|At@2LekLga*Hn44<l=B)CZ9
z?QQ-p!Ox;f3}LNNHfpsCI+)^o&Cap%`aSELtJ+cL(RNL)vuRmpoti>l>dqG@%!87=
zwslQX;PJ~#IKMUJbI37I$4eY--Db6&oNe<mTjU-$gb2!d_bi>m&G&z4-i>G6lBv9Q
zFL`y_wQu^InsZfVE}!VN?sgv6K!t!R14*Zfl`WiZ7bdQZ_L#ZeFxv6S8INtv6QWvv
z?s+P<a{0U1BXdjd8%naw7iEc`#1L@es-YB1eB0Hyk56YTdak=CQo1+s%7y#<o$hxs
zCbx!3_V{GUH3p|wcr43X=wI1URFNf8mzCC~l{PIa?bfWQWv?En#wFkDy36BgyFktN
z%Hb!WVGnv{{V!3<@e<lpacydW+iVF|_83pKLZKIHm0A>6&sd_gnae>dY1TsnA&pZ>
zt5$Yy-F7Qb<A~PGxN0BG)vq#7a&+nb?KC~tQL45uswCUk>(rsC9lB3uPEhr-S@d7v
zyQFK`mF$ye9$U+C{haD^PuEpjV8!{`+m1V$zP_Cr)WH)haiv06$K_tv(q$13IpmHW
zyKvZR$>g-;Ge>0tO5ga`zBzMs!s*mCr#+Xr{0je9d73N8JI{l$+pj}kLg>iWu8CYc
zNA^7Po^>bp%oE>9Nj-m7<kVcr){#}oU9eV$)q`)Tx3ACSzBMP(79YPhDd2C!o^7qC
zR`^u1To74w;<Q0vc+=DHRlcPfEv_LPOM6fH*09E1Tj_LR>D`@2OBO6=*mFqwilW@1
z%hE>z#kX9nZuF>FqkLa>Nw}-`YN@cUN1hqSrbP&{^H%AsY<sn`EbGZEt*3oiB4vl3
zrmbsF$a@<0KdocatK!0gENjG9EqJED$(bD$w&`Jt`;ldA_tetYiXBmDT|QB6v(oFg
zn-;w@>v^fjs<Df6t6ciC6+tSx&!3<0xYH48aEbkl!LzeBdW_T_Z;YKX;mOKfmwc9J
zu*_~d5?CnTmL*%x)B59Q>du$i2Ny-!9ugJ55x@1s(SSMqf3`-3UYP&lO;_8b7aehD
zLnW4!xn?k#z4?^c(yZIU@aw(h&W_7k3z(vp@E)4OEZy#(`-Zt&hedzGNjG`6*(<J2
z(O~D7TRQb%oJ!ZKEzYicS0ozraa>%-akInUNucv>M$gnJp5r1D)Lx5u929Yo;Nw2P
zrg89z$XcG=36oaIXfW|?GyR`yrnYp-Iaj{A<*!y8{WPy#RBMUh(-R3Bm?m4ggsV+I
zT+*`3Ds;2qa)+FJ#~p8O>bAtxx-*u}Ek7u{yOVG6EWfe^Yp(b7hy2h_%Mn!kH8(@c
zRb-jRvY@%eTuKT_t6b$yYIUxe=%BYE?A_zAbt}78t&nSP57=}f@7<F$Belg^t5&ft
zv~yEg#^oEr(ZAu)BF$sjO~*DQ3HeQmQuoSMnR`vcOlZ0U>&~8|>WbQ0kM#7-U*|u1
zrS$Mvu9x3dx#d}js|{1LcWem?KRPM4XLS$D&8;6Vy%&t0`AjoZ^s&&rp0_Hjb$N5O
z7lc|Izsw&v|FF^(g-aaEKE3F;8~?-P%#9WQXG%W^UG_<Ip-cS3S2EK??+L0K_{$y-
zY+9q&w&u9dm(!(e4)$-4s+bfQd^xSi{_Nb*LXj&Smp(Fmy4^hWw8=`YR>KKSQ;toz
zrQ^M};MbcqK2mQ3YpzV#)p~oW0`KZay#5FHPJa>cILPv&U}dN4rF{t;7uB9|whNkn
zIqkT?(O=iZSv5rHURZ=mq^nDGj_QTXjUlUfrF|ClZDE#az8e@ZU$f#go4!OuzhJFE
zRuszyXGXDKdRe_i?l!t=;#()SE$K)P(h~i=xi~SWK;wAfi>>~;n^dxm5}b_^?28h_
zw<XM8Rr1p-{Eh1>m!vh9TozfnKQR@USk~e-`^2-8>!$Mj*G_-DiNT|9#+#yd-!^TY
zW~ShIdwp7;j@0ruYUw)S@3T8M+I5GnTY7H6jcs@D&WW0{de66Q8+%*c#klOw+xV5~
zda_22yGwMa(42J~EsLd={15GJ2u%5RK_L1*qigbpqf;F36)D#`hTCRq-FG{?eWB=Y
ztE>vq0!NJ-^?|Jnd{?U1m%rpOVEVP$Ma`r=YigbHQs&R5MQ==5PtU1){+(5y+bR3|
z(L3er-pfunO=q*;5$JP3ET?2uK++Lb@%FVPoEw*Tb+<j`O-S9I!gIdh!aO#<*9Cm{
zw{&SY`fPsWwef*lnPb4GFPS?Y_($hVc6u_|Xx)@c*UDYEN*<L>Ejtq6@jvnN&zMC9
zl48P%A#>dYw>c%qBs$6@CX}Q{w-mY`(Q>-d>=E>$WgF{O(<w=fywQ_;)-{O!Xt>C~
zsN-Jg!k`zA_iZhm`MYpt@x`Al$LGef%E_&(TWI<zZPlC=L2BPjmA&;6g|{XAFIAQ{
zODN8hwD^^nou^j(F8xu5pUkY6zgVUSdj_9b;W$NXa!BX<f2%gE+N34CTGxBllO>BX
zz8k%}WwPdd-gAr2C8<)g&m?`=Bsc5MBBSrSa{M)I|3(M@yT#X<J$I9dGMC~krz0V<
z!J9uTS$v%Du6JXmdcwbrC!zwj+;#M|p33^=C)<44)^8{0OW%mM-r2~uR{BcEdEFU@
z4zK!O9lqsYxw~)uvCppp-q<l;iEI(vKY7h$HIoDKB@N<R8kncD2yb;_|I?^nbBMd<
zfcO;``(N!YTaJi}Jd3hfcO@>MfRUwc@$u;?t_xEp7KCU`7WMB_&DhJKbM41Q744im
zAJ`rktnp{;zW&FzJ!oyhy&mmcf8&+UGVj^@e>nCa;n2NFpR<m?G*HN5UJ>SXRlsd)
zcpmSZd&}o9pXiWrbB6G%C`Q+H5x$xmX3OPg)^tfP$!hs}bh6}~TzSzc7W%Eydk$5`
z`bN*+d)qk6mT#K=H%W)Lcbu+Yc*d|J-ua!9e$^6jyT#>#i{$q`;{Nu<#X@sRUb6X}
zi5r|AsLXzy#qxU6|D*5YJoDGw3FOgM5Y)ZcXZCXcuCCiA-OVNb|K80CSp912zo$Fo
za^j<M^mbp3W;c3hCc3&c?|0e8xhtJz;)6f-xy;;KqA)WezSgnC*CXli?$!BeD}GO%
z#@J@U8b66A`h{U_VWRZY{JX+kk_Tr0?e*Omzx~_7-M8oewLKqoZG*X7woA?H+xv`L
z_unq}7yR+p|HHxe5oaFSJ5112UcfWGDOEtoWJSRP$7XK+qAZPxiB2u4DoSTMHu`Ax
zaahNyXdF1>!KfB}(`)CY@V#M6;<i~;7M=E)ZI<mPxZv}%v$Jd)i)Lz0IW4ltjaB5Y
zW&-#5WzkD~CK@@OY*`t)F=*+n&HufF7-Ls(u%1?36S&#+qT1a<t}i?nd2%08mHM72
z*~s?(maa2bqV(jN^j%ywmw7K7QfC!byAi-3*`$)g$8xJnboS&mew$Y6Fl<geHIFr7
z+s()W&o3=x{lU6x)>q%^b>-8_f?uunT+e)RiogQtM9xMI-f3CNo2RDST6afbd92&^
zh9gWiyLLSeJh1kW@$D&_9*g~QFyF>Cz3}U?s}9DT{#EY#_AU8*n8SCc;bps`?BCz3
zy|%55|9^t7l=o3f55vg`9aEUTdtFfO6no=wkp1T;4JQuSITsFb8BV$2$R?X}VKM({
z4$dQj);bf92>becJj~R-`RLrRTN{tcO+BHow3pfZf8r}Ozp%#-)a4tax-|IiC{60t
z?GZ^}-a0ME`{Zo38A_)#m#(?AN^|ujC#E$j60cO`CTDf)D(`%tu*T^8lZLfMHy^d=
zGO~!ga<JU{t4J;Al{de}KNbrf``@=-ygxrVMM}i}QG~XLzfHB2tI3s|qBl#TvIG~*
zogKCEr07AOS50~?cQQ2eX7Xq~p7!BRbmxNT3EIj#t@VQ5PiuSU(ARUAC+q37D=fQD
z@Xqu~+`8ZiSMlLqa~&<yD*^Vh84JJM6566JoOAl}zGrEVjP1AEv0hl1r~J#zVdo)U
zgP2AYr||V9mO<C=lv{oZQ>r}Ld;MPJ`@I?}O-@cHlzYGAuDCCd{@-i+(WyC}5^6J8
z)?}_-`f5q0+Pt);8yB|sIlb1|ToUzZ<%(rpT&p!Vo$_j&xncw7;|-@S-FUkB)sY+8
zJ^PPt*DL!e#9?stRKVE_2L-gxKF|4jW}E#jy~8=Co{`mCOqbXDg*05xD_Xj2N|5#!
zg(DqH*NR%7+P&5PA;;Og^=2`Qvu}h|D_1SB=9rXc`2ClRzrmA7-!7kbT$;KzKl}ab
z=;%KJpS~z=Gtdes?3D1oaDe6KECH93p=)BCd6oSq7Kzwyxl?@3K76CKpmo;KFA{He
z8d@6__1sF7t&ZwmyEV=2(rVSh`cF$#s~24=*L2<&<gL0*M^5wF5glV6<Kr=xE4PXK
z7w>X^`A{U`qV?jq-1oEpd@9bEy^UYy<<%+2N|rhKzg}7M$zLjUS>T71!?ka6cTE%t
z(u!TXVscL8d9!H$B=)~kUYV?0HM=zI>grU+<vjCN^=*DJZH^V^2W{>z!JgZWNgR8T
znO*G6AFylF?7JlkSz=!{aXNYEo@^0_{+{{JC**`|US)UN*HsI*OB6ZpHE4-t37qaw
zw8MGt71a)bgAccNX-w)4c%rQMszcJkWJB+l(>y^w{QDA`HiTVSA?nv8Az8<CY)OY<
zwbtiJL6e$9mKOM^2QuChc`!qD?UbCpkcBg%4^2AOvo$9<)3JLpgQanS?Gi)wo!(Uv
zrHl97P^+o^^*=aqU%GJ4$F|Isp7QBt$-<{%*i4NWd(+L5XPsHo7I<^=$?R7L?Wb$J
z%<lYnGm%e)A$pEVgwdSnGOlFyS3A;eo0O&{aa@x7+wwuJX)}M-g~v<o8nn0YUi9Uh
zvH76Uzc=};j&o)+m7HaK%j#Ie$>(m9B(SgGh0}EHKugUOzmlSkxza`IFCIwls#w%f
zawyYP`p&~H4pENEtG_SJ_FFVrH^*UtRghe8kI9AGu07>u3nl#P%$KfYIIk7G$m`>j
z1q_-e1biA?F3-3$Wtr*9PVO5mmmAhR7qzimIYrm>ibIXqW!s;+)sJVVNs4~mH~Y+c
z6+377g>_PznkQ1Gsh#XA{r^bG+xs3<<{GUt%j%X2%{!YG`P0xjeBRWn2k)}1PH(y9
zyCZ;!;ci;wJf($^-ZvaXlvrB7SbFkoSM^-*A#GXHm4~sp5&E;IYRB8>gvWi{8ouY1
z!3J?ob$6Ss`sz(f8oNyo?-!GflvCZdG2%|Qmsg^)f#KRo#)6Mu>wGhwxppFRa0Zv-
zLyrK)nAVgs8A;jIiyFF5U0Yb8!gV@sTC{0V`j#?P@jc7h1zof*PO>|DulP&JiOEme
zGM?Xl;>*rz`r=;7G}o!BGtwd_EVO-|w;+mtqsnKOF9OB8t|y)I?C^~HAnCj2(N9&A
z`I5drPMNR!ow=^IyKVmAtt*#YjK1^8;?jQ$Mr*zziBmE!E?8Qg-MP7NZuWDVr;>^1
zO4;Y7X}hMs74}#-ql|0S*Puy7)2*+)XgRV`@#9ntsXeO`O1!yFWQ4w3vFVB6)5ReH
zmwV?wk7xYI;cw&j(laLN-q&*})<<nTl_L+E%;vIO(ejE@bILQjd0v)v_Q%fs^_pq_
z`ddkGcVkM`ieK9o@VFWY94y${p*H2S&=SkPmL01f<h1#<ZVy}kXiBl)Q62WW)V1m+
z+4EPL>MGtY+EA%m!aLc+Me!MHhx3^Qg3?7Up4V3^`Ra(eb8HgyRbJA?n4BYN<tyD;
zk#%|I<s8Q&Ct5?+-%4EBH_J$S+OfF>iHmbvltY#%F{u9cSlYMYYFAFDdNiv?!eaHz
znQ}i@L~q&RbJ)1pOmU&C_|s#oiWRRU6RMhG7FaG<_lVO=@nd>-YzkZEA{+T7=fqF>
zExK)#lW?V2IdgK*CbfGv<3bj=eL9$2wbC?ts!FYUP*;Fydy8@A=cFgk=0rIf@5!rR
zJv909M62(UW`1s6&!OM<tnR(_e&#n#pEpD<$nKNbar?_MYyW#Y^X)9=>dv2FvAt8D
zeSh#F)AkP?8Zx^I_hnXH^_H8%P^B_kpoA&m>GpR?#gm1m@3^-5=%uNfS^~wl|2V4G
z7ZhzZ%jQ|_79Y2lRiEdG+rC~{vXpi4{HdF_?|ol;Q~!mq@tTds8P0!8{@?p7_dP@J
zr$&6tE0vo5l!&X`-#b#b`f0D}jYvy6<eu)`)Z+5qb&;y1d5NOs?nnLFX{I`?Np-fn
zwS1!fuDP~(+tziz71wk5+Y9aL`ntW@<jZURSmV9t?7p>%<p?Y8&MXczyWDp5dXmCz
z&IkP|2`WqydqU>^Vbqo1V#T%g-@Mq{eA9oZ9IacizA^Ur8-DA>cb;>07SDcTcj%4o
z#rL{r^TIq9zTlZCb7$Ma4{DQk&rWG~NNkNi6`ucl4Zp@Od6ECQi{!La>sc0fztb+@
z+o0Hxk?W*-|AbGcP?F+<Wy%YbHCtAiA5pXETg)iz)bvO7k=MeoFHU)nxM%qtdbDWe
zm;Y;y-v8vCYWJ?PDChVh^#h+$4@jlY;W%@}!iin~$pNl4g=dbk9n9p&KljXTIa8)G
zXK%st70YkVQ}@YvuF{pqb^67^oJB9@^m%urb4RXuwI^Wlq9yA0QkI-+(JA0zyMDyC
zLd9>@@#E``zgu?lb|23S9Zl0CK0NO~+^Jf0av7&_MC0-c>uRfBolI+D_-e49%kMGA
zB1TEgT&YK)JS&<nE#~cbabnuyPeO-3JvhUwa42NLVFiPJvpGkEA2WT9IVy7XvG$Es
z+ULB@13DC#d>N7o#d4Ke6HbU#FWM{j=rLE5cyhnuB#n!5yx%5dNQkce_NukfZf&Pn
zv!wbWrq=f=E$c=8uhN*|!qQda|G+NA%xV#@S#P6U((I0=s|m|xl64m(t<|tPrpdy<
zJ-N@c>g}Q}1>06QF}CS4z7ga;zbN%b;msoN{xcoh&-m+ce44<=X>n_Px{PLq#HK7Z
zXEhb)JJ0e>eb{S_JTosGH#v3O#As5@lGo`n$r&GxtLZJ3>*CV?wD#<V)Yb#tXIq>e
zWN@DEIvo11b-K-ky&u#bMJ+tXvMyXr)Arsn5uXns-+EN09NxP}$FS}tOY{eZxu2G?
zotX6_Tg#iPY6GXThhAX<-wwV<8}@9{ZZB|>OV<`|(OLgVM{s#@m%wwoloAb{=hrTA
zc%~e$Sf`i0$miWA&$L6bo0s4CzhT1-zob`jJ+C-EbMqSWv^%|a;#zVnrE;0jizV%E
zRxLk%hVR5VKKnRN*4G!zSorz%J>T&>b<1lKvAbmG#cAxa`TaKm4eQH_IZdH|loV%W
zsYM(W^E3RsYO!eA;(Di6^OFmz{RG1V7KJ@|qi|+bqtc5;fk(Vft=w<Uth{rm$!A5&
zy)M(RNf}8R))&3^=WrBG_-NfJXmd39+c8(=EM}jU?kifa6`Yzj*}dIZb3XpjWa+2E
zu1_w<zH$8-w|u_Y8rj8*l9JxYMm|o-QQB6wwxYq8X~%gvRRvBF2KmVe-m{Xf9rM4b
zRibo0U%~jwdIslNTi)7zIF?b6oVVvm)w`4Y|6f0!^yhv0pG9|1aWH&7u43A$`9&$L
zp{OQ<xnshH2o`1=fn;lq2?}Ncic1z-Tw>c~=fbjcfy%uEbDzeuJnPg?Hrwnw#N%Xg
zf5}SC#m8n?`56~;@E#PIy(;MH5tAp6R0>-Rd@}{u4cpmr3x1w4dFG{c;K3xO$s7D0
z8~H!IRD7&+{xue^>n&wcS+k}jsCBqAaEJtoUOg&t)j;}d;NAeHW&X!9dUPZv`ySY`
z|HPE}*EaA5zp7fGZ5*EL%E@k$&*hrkBET$Iea<H}Z|T%ODGGDE42~@iHE<P<WYu#D
zw3)HW@PK!2#w7Q`&F5Y%^K@?#70<}-3Rc{=O6g9QX!v8%|E?Yvgw|W#=Mw)|uz!ir
z)kB4fM?OZY^V+l;#`Fs6?RcuGovXa(Xu-bL*w;)6S0+oXd?B^IQ2Omw<sE_2AHO<_
ztaSXubnBd<0x#DhuB-aIs)2%GOdmeVNs7n|J0wb)aL9!)G9@GlEAaFt>HhNg6Z7oG
zF&~Aw8=|lH|5fn+_u_-|+2dwWCk$DRdQRhcALV@4;pD$LJX7q7LX}gE=jz-xO6C5u
zsJ?BP$glK&WiKtXCS<!MS@{&?O*kC<hAsa1W6?VYM6Ni`xaU0M$FeUG0fMJHyjQzw
z9XH82ZNj^DQqJa1hjZWJ)^)Bw7xHw2k<rrSdM8C*yX75F(CLzCDS4IY|Nm==Zr~~3
z6!s%ZY7UH>1HzRTrzr(88yCu&E|$4cDEYNe)S)PG&6aydUT-^Bv_G%dGt1Y7FRxZ_
z>478jJ<Lzc+7uyXR2rx{>B)r<$Iq&DJbdoOY77QbS9GhF9aT$vxpJDf!JP&kA?6C%
zl{_V>hOc-;boC=<O^%o#dhSqmb}rMp9v!jzNt&##)-#Jte<-(X3$K#ySog$fP27dz
zlj1QuO>HikOK=+9dZ2gd!t@Q(I@af<ebZQ7r`#=-`Dx<|VY{coS5|zicVzY4e9^J*
zyy@{b8f8vfdCHkY#5H)XtL<9i$ay7Ey*x=<OkQ<cl4%5^(TpV92+moDoLqnCWT+?o
zcL;OdyXg6|CijwMiW_=O1HR>LJFfR7CAB&!BjAhCEd~Zl%`A_rb~hf(i%33wYoVqP
zv;UUnfNjdwf1IZKJ>u(P&XkytJx|>BPM0>DE<YoC?B25My<w>b%z`fl_#Ca!D$ViZ
z=H__lvg4A7pP<TyV->k?r{%xgk^g#!6PrpvYm>)?iKbjrxdpE?h=&ILwJ6fQ;VAfX
zg1R}QP-Kz1S8=*;Wx|Hy?dNj*yQ-|N9>4R!Xt7)Vuly7LmKeu=|Icu->dAAvXJ^Yw
z&DD-SQ%PezZ{})MG$rCnPUDBuEv6bR>#`J>)^ZzUw0z;!IBVMW!mD+O>Wqs%Rcp(w
zQ!m;rF%|p2*|qiC>7Y>GvSa1Z*I8!oy)3?+Q-ZzU|4zBh(Vd-n`LVs9=jgwcei$5|
zSrpHFWA0bed7r1+J)imX&_=&28OaY;^owsdZ96OH&m=Es%D}=PuYaT2++v|%)k5vC
zZCSZ3t~1I4x-RtbB|Lw$Xm+Xomm`y-oi$6#%vD(Lob&EBG@9a5l`gD0`O@3TPjpSH
zo8IfVuJCfsU9#Awsd?3mnc0pW*_P){yyC8Uf4aN*mUrnc*26n`9QFEEUh;Wqc6pPK
z?oOlJg}-v|m-(!WdM3x>62X^mo0R|d$LjA>IBzC-{)qTx@J)p!a)RJC(@6>js^Sxc
z7bHzqSSLAi-Sen*!VL2N6LZu}%!>`buRnOM*td)KxysTxDJAaVWu~9ow~A<}xOThG
zyl27e|KPcw*Ug%c=JvNLrY!Hw>@!;bEvwD+vSvTteY#=GM&Y(+UV`4cXKvr&z4H14
zG3QybOe+tT)a-1wpY*=AvU<+Ptgb5QjV-%vWwn3U^wv+5uK!tX#?)$ec3C~A#jfJ@
z+m7cu-PKqiSD~g95h(g4MeT=UdeZ{Ysy&js_DD<hDeQ7eJoduR!nwM;wccm(&feXd
z%69q%D9l)W=8M5*PS;G6!}=1x7tep@s;oIRL~<EtE93FyTcxXa?VHiv`Rkx`^r7un
zwi%lm>fXt*->&u2X!ezoxLppsSEm*JKNA*wf5!m__I&3<tKUnkiM`|Kmi5xdXVHp)
z3H8g?N>@&N{%f7E0!P)K<IE3^*Zx^|&>&gaGT6{OSp0af^;9Fx%SOvHU72GN;>|Yk
zE}inMJuvB{-HNobw-=_h{?v}^co0+g<<epn!*ybtuUolRE}J4*uwZ7vuf?rbUaN7u
zWAS{k$o}1ls~(FEv-B)zui0CCVXLUa5yKs|vnBqU?zlNMcIw8<4sS0#`#SgO&p2(_
zd7n$?-c@kYIV0Wv!sA-UwFezX#D4ViZfIoRwIJ~Bg0t4^G_PH4j%iMmy(a3g_p<aA
z?-`3Ucm0|3c7lxl$}f*C>$(=R+C*!0vOGBV|C+mrmz2r>tZ$Pv4jZN{3XQf(oBbwe
zSBzV&o6(2a4^J`NJgHzl<>jMW%gnFk-{Dg!%36GI^;drlnN6=>Z;ezI6b@{z=<e~B
zS@|)x=lD0nnO3=npRAfc>&dY%#akyGIFq?Lvv2j>7e{65{GTt~;*+CZbKHLWY|~p+
zrnjP!D)&5oJ8NC7&524I4kiT#m9MWRuxzjtFFt%^68r2oKUTgmyM0RJwpzj_roTxi
z+m`WaoxEp%?bK%X)6rFRCl~|-mp)THHlgBQblqi3uJpSz|DBz2w4Eh6Jgw6zwmM**
z=U0wLEpCUZmxs2nJZ;@JLu_qn%c`qunmDf<*tjj7{kr4dw@>Zfy_Dk(Ty^1p?u*rT
zsuglGV;H$>zrXT+VZNzj_lL*EJ1yVFujra9ZhuB9zTx-u&er5zHxKEQHKlB*=JI88
zWOdkM{r_|0p*``j*OKS1^Zx(k;_Yjp|L^ciw9dY8v#Gnab+xm~{X=z6zqQ`TIkeYs
z%6h}H`a5w&A(oHt+|ql;cE|8eZ`$%so>^;ndG735lg*u=<W@GVV!4}JMO~)gBBkhs
zK|bzoM$Bp~2@G!&&1!!Axq9dB)4bO=?O7LVhIvi%&CB_<Vo9;_pMUS`?;jU2a^ACq
z!HHqHUJOgoLzM{&99!n>u*i9_VF5!U)9Hdu5p5yIjFwtd?TP4G#Ly$_U1iewY00T6
zI^nx={&Rj_dS;4%xzEf3<L4}z!Yp5=7!)oNVs!3iv$ZnpXi(v7GVv97urhdg;BqhD
z15>h?R4rI@9)9||+F6UwvV%3+Y>~5(z|BodPfSqKWGsHIw70v2E5a!HM1khvqxuhw
zQf~F=TyEn@*jIh@Q|%q`^-oSt(T<*HA#NOGRuMfhOO;^@Lv?v><2|EW;j5#z)op!L
z;oY$IX6mUD$5*ee<%BUsY~G}LAUxpV*09NfAIhJ&bWU%cH6!9#-rL3Jz3=R+-rLgf
zT>0!By~lg2r{8M+KTF@R^)Sm<;al1-EFb=LVHH-_=&T4^@BBCBOWjlc1eZUSViV)}
z7{dgb9A(>Mn%xbz|DV%*R`#eus(@KWT%}anjE5EVl^*w8xBd-{R^K`^QK>?0+9Nmj
z)Ga3jdvvC$B=?%!lThw6KDI(IN$T&?>mt%Wj)YG9CcCnQ<vp)!d@-ZOhm<J+zLKF+
z1Dr}zr*Nq>1lPy0{Y;&a@JdxN=!|GfT0rK!PQ?T!jcEyzLOiOiEH+QK9284iI(1@v
z<t>&K6QUWcG_u?~+eGpgYF4dS)L~a8ve;44N-(`5_1c_7o=MlvxvXv2c2evs$8j#r
ztv4PY^H@6JVwdhWl`yB0i>)HJX3s72;wtF47Ifalz^OgF<OXZ@%rarg>M3hBUfcGj
zqoGhhklpFtk>I(W_pYi0d6`)LpE4n<wpw6EW~a8^mM3k$16%|acI!p+Nrnixd^mSI
z%=yI4+0TkTc9}>z2{*b-^3p!%QsBcT-t*z0=G*q*!*-kuA^tADF;kd41*X2O@#1U<
zWeD|^Y&q`ldQ{|OL}IFXtXSjItGiPQ6>}qhy<Hi$_x&kr@#24<b{j97q2wBWfr;bq
zy=tL1EhSq$^A(PY1sU|-njq}w`!%+0YF1{zk{+k&srKH>SeFWyrp#mbdgIc?Qpc;0
zyjKce4L+^0cFSXb?KK;MxV7isO7>oLT<iGVvU8zg78^I+N@J_o+IYg1(_p>&lZ&5@
znnxaUj_-SW<h^HOSVUaazxWvon>lw)=bPL5|KV;yPKOJnF+0|!y%D@uns_xQXxrx^
zX}+?H@#4MTt=%3keIZz(Cl+thqrYoK&1d6nH;#R}xc#Bh*UQs|Y`=6Q?zgS{UFe`P
z=~3}kmE*!*k*ZT)+0Q&OO-RL%RekN0M_$@(ot^(@#O>3RoSmJ-;b5{eGrnu*iJA#a
zQ%;7w(eU&Nnk%Qlpy_kOpv}^YRlM}z;*_&S^D;vMM7ADWT=Q1XesjojkK*Y856{Z&
z+2p$3&56xu%AwG_j5!<BnC7fgJQ~WBAb9`Bwe8Y=9@Z`*4r`j6=BZ3lZ)<FnJX3OL
za<;5MvHQk5B8p-n3seNUqkR<W&#@je=ql`a9u@9<u0o*V|ITjVcV6N<&fHjV=b&r0
z)e@gW?^YPtyfW-JDfBp8@o9r)U{4a)jceO(ZQr;cXN{)PWv>%<&qS7a1TjX>xV_vi
zw9U6-X=?IACefs4M~v3B_WthTc6#$^{elx8jlxXsi>|B?(e+c|bTACLs;0b+|Juf|
z@`t?n9)^e3ry6L8);tVsFlN)dlBBeF$LyS_z#02@JQ9~Zcv{zC^6Hxvr%dt}-TNc7
zcurHvL6-Z9&pp;C&lB!#FXX6NRAr&C^T!F{j42yqn5GF$=}vt;PxxWF-G9&dzo(qD
zpMUuN3p?>C7ZlrlerLZ3kN9p>>-nX0>+YqlxyzRwwBUA8(7M<e)Uv4m+OhwY%2y}$
z7P;)5{4QguR_ndKIZp+YPl|Ykd`S<8>QrU$@mwDGP;;utg3Br=Hq<S1oXzm1J>=rT
zqelNG2Lyk9s&@29kdd=sSIUo@b|*80Om<G;St_H-aPCHK6H^9rzTVHP{B0BZrcD$H
zx;61|*%hS_rXQ}6(MPvWnDAWe`wB&&+NWImIS=G-EuK@<a&=9bV4HzT$F&6|My-~f
z+Zx3-2Va=MWaxV?DEj{$9j4V?{^#4uwnV<U#&#!S+U6BzTYpqekH}!#SJ)cVlNK^P
z_wEh1s+-eFxAtB?d&Wg@?#8L{?kDfc^`CZsA{VYaXF^!`lno5dNo$hYwnS_z_~qdi
zwsfM>|H(^O?3%B6Z{smF*4EF?F55i&=hAFz?GrJ#R-bzK?@dzq!uPJtO`=9SzV4g)
z@>Q(OV>6*mipJThM-7@Utz>OwTeoe8Yn-l!z2c*#i`u=D#AVneju$c(Nf-3nbh}xy
zYH&{bG1Vcy&B;5;t3dCYZC7pdotqc;SsXKdc1c7cc-IBlka}THEyu6s0VXToEnFQv
zH~43F%E>j8()Zk++i}Xl=8<fFrA1U+_qv>?)8i`LZa$X#nw4z(rttjITfuT}OvbI6
z5}BS;cF*wT_b~ahEdLzmg_ZtmRa!r+TPC_r!S&xipJiOJ&+1$F5(<7^J>_`MWOeXl
zhU+fI*E%;GTElch|NkP5*vvgV7TIZ~X^|6SocukHiOqVu@Kcw^5$@24&aT9y<g<@r
zvTmw3a%BXqHGF=6<4h1!iKv5!(gc=RcLlD%2~66r7Pe|+I0`3EXc4(}k&AO$nbM}3
z`yp$4U9-+hB#FQN_M7G8&eDn5TP^P`(VgggtfXBqe(T=rx%*q+<-g@o{P^n7B=-)H
z8hKTflW(^d{hb?QpT98i!rFBQxjR36Qxwu<6kt$Uc<65b#5cRE{$2TN^L+k{O1+*u
z!>2O$Pv83y^!9AU%}3fF7rj>bJjv*HYGuF;gRZp)_XO>}a%RbgNr!ibctqTLy|Xgq
z+CyF$-gD~5yKQ~%bm?Xk3NO3qB(&xKg4Pr-MXux(Q}t)HZNJyDOf>3+ncdwU|N48E
zUni*lzJ5OMn@Wyi$#2Kg{df6Z?3lc6A<x>~4Hv!y96OO`)O2t6EiJ1ToR62jKVa44
z#{d7xk?xX8%Pbr3{3$_(+eJS0_t!j0E)hJ+a?*}lWB%IQv*%wcxz?87d+8@SA${*!
zx36w@KhC#Z{@|32Y1z*%<^QvkJ;bBclIIJ4O}(^h#s5u*f_A1%xoffc+OsV&W|O@N
zT&)&K1pPRwzoBi>+-D)#j%p8THC79Yy;Q!LnLfMmJBzRG76C@j4Tt1~FEUwYU0fNM
zvGCI3h-C~n9`cvT{Lb4t;n0S)vFcx1a~V4P8DC@{DEr?a{y95PfpNcB4&RLiwvC(_
zig_EpM)5B#tX`yUT-^9PRd37JycexeW|Q^ai8<_D8vRiz<Fi`Pk*iHxE;;{KYi2NS
zmOE&`W**BC9_yH3DD}AD7HiARsj-u`l+I_$b+fRKo2YJdxWzvt#q6{G>#4>cS&LSh
zSnGMXc5V~lbcuHo_Wq;dJ$p$}vWIwtamCDIl6wq&VoW8MTrdgV!0L2J_~Vn-XU_7R
z7gWBpwB4SbzQWP_wtJCLnfU#${Ck~B(}VfXhHy_(V2Dv>_-DW+xPWuQ3GLn!T(cdx
zz8bK3DzJXJ+UfnVGex{>@r~TMKkSyB=sdM7$M>>H?Zo6F_HMgx|EpMC10>qZ{};=&
zw2SQWXxqZ>z_GCV_`xP=Rhxh`>0=Sn!C^tCXQW=6oSLFm8L}-kHm&mf;neFHY3360
zO)l~_9qrSTU0-ardmN#<r_6IzxXjUu=_v}`Hf$AtA9rs)uI4S~eLbi>I5~LV6URx@
zGJA|N_okRGF-Vx<RxDN$e^@-MJke*yN4o>hCTNw3M0_kgpf>Sf+QbDKtjPiVYZv4!
zHb_vv?&@KfbNnLL*8}wxk2_9I>zsLlYnB7ow+*IS7u3HsV4Hn`Yj#5ew?z)$%LeO}
zlkGKGgBVIrZ)2Tpz_xfn{F)DS1;u7>8<HOd<*iVh`s|uzXr<NKrFoG*^Ii-6&tG+U
zs_QoWeW70R;qJe}B>!A<&h%`~j%?1kDZuO=$zERIyjVoY+{lr`NL`tAO;BNmYC-7C
z!gNNx7iNO_4-4Bp1?M)jp1qi)W-i%RDI%0ozqcYjDOhoiqR*s<9dSoHeT7TiG|m5|
zPh9Tdz4}IH&<4&m23*kstkDTt3l4HceURRpIBi#Y{OkwZXTNkTHsI=9!C85k>z6v~
zk`Hwii#dZ9aIUgsJ1RD1&WkzAXC?)kmqs1v%64a)@k6rWvewrJ{C7^y{dQsQKZSXZ
zPC9K{k(^W%vT?<{{PKBa+XFI$GTh5rw=2)ra-T1HNXGhmcx$IiySq|{yIjb#iVGGC
z)KdR1xXRJXA=NA2sPLrOaO*b}XW@n4Ed;}y(jQ4Mtb57zqNA_+8W-=yz5;X8FP@8<
z+$Eb>l9j_vb<;EZCP_{#t2y|+h;wHCe)Fs|-yQqHI;BracV<kHjns0jte+WHKhY>$
zIFNf*z@+(;YB*V<LPNc0W#pWgHpk-Sq$L~X98qJPG$F^vGUtd{POy2->1{dgl~c+O
z*ZaKW{<~q>qnF%|Z!Y^Muzc?Y{(lFS{Y&7k|2gg9qNvk?VrGnD4Zk817`<9=H5wGJ
zke6uMeN^uUlf&mu#W!IBxl^J)9dc7SV*F|8%5ui$Ki8(szm*?9&3#US@VCHhtB!(-
zjz~%67KO_m!H52fm{qn+YiX(ZCbe)=qtA@htCID<6s3GoW9T~B6jtG#c*1MfbcW|{
zGyfcwKFK~|x|8NP_S{p?CM~=;@t;DtV<NxWO3tqdlYT6pUF_WXW<rjtCd0~;re6~@
zZo5VOa48j9(iQb;&L=5W&jm|Yag-)6;M~Jm{6{@}@u#`}80I~ST+U)Wue57J*{Thk
zvo}1_tX*$m^-^+?@a}m(nq{_CY}}ENaolias!%n5QJ|`q<em<ZgP-SjP48hm)pPEJ
z&E+3G$!WdOYMb@7Hyaz-8yqrNzDt$sK=Xqan|+(?0+hCtEMuv8xp4lWs<li~1rm$4
zUzcfR+uF9wL1J;yEY_N-WeLAi|A&P;`}8f+5SfyBzA8}qhz56LCg&2)smroM-PtOa
zb*f3STC^-nKEO9EgFnT9tI~@teKTjyhKU!{a(9}C-YRRb_sse3wQO<0vhvMS_8ijA
z{<SRs)v^Msc~f_9n0|W0jNKb1Me|>m)XQM8dZxB3dwStN!S<FYtIe%qtF&iCaptdl
zJoVA1sXMk>O}BC{WGo7eC=P9&Ht*E5{oJdJ7#A(8D&R^t^c57jojN^C(DQI);pQsO
zub1TuD|Rgrj%E!~*vqv>&V1UDDoIVQnZlP;oVKp<IMx;`TwG#Oyj3V6ee29~%S+uG
zrDQs1{oBA9G-3Pf3Fd+<_32aR6f{qYN|?N);D2ZE_A<X_5zVf$vPJ9DZ~0Vgp7QYI
z+^NwU9%|0JGjpee%)HW72RUya<obP3hUZZ6*D2d1RD_~8ifDJg<(#kkXP)F~`<z1?
ztHTvda{^VPs*Q6FZ`nNmRAR+{&LdjCgBE!O^|uGzkm%)Fb;M(jgt=mpKhwg(X9dAI
z3nP+LZCb15GtF7O{HW}#k}J2j2rF$0dcE-$)66%oyfSS%Dhd;VH`nYs(qH%{KIE`u
zi~8CR>>`2-Snrq!mP@myJeV?HX#4B~)0ZTuyH?i9Fh@BnH$;ZSw`emQeztv|!NEeU
zokd<d^P)CP>)DWdi+}!_oik4_<M3WS<MqkHRogSREYG)$Ykc)z-rHiuMx7N6uUfXo
ztdNm#i|^DIGCY=jDY|!6T;PhPcdFvNi+0-;Yi@tC``~H!MNtNeUY#+xD1W58pr^)P
z>~aCq1CRAtMuCA_wtB68$dkL|fr*7+`{pz93?VD`hvp~VZ2F=hxuwP7QMZGT$?>P&
z$ICa)G&yxXN?B^M<jh^mSmz6F4?4gmG-aZrX72oqz&BCJ@y|2ZxeqP$xwt6iqIm8_
zN!~-#Eov9uPS#j+NK@yMC+lJF&<rabMIIpuiO}lgGlvD4*0e55+4LzTXxXN_lQ&uX
zP1WCh#Ps#$i#t-+9Y~ehSmtT8P?)QaSA2_jm*~1obN{mo53H=pm7bE2vr%#P|MFI`
z%kxXc8!JM;rfSR=(wz6HzbrW{P51aej%~s%`TsnM&s9mzDiTS%E&aAZ>cR8c3dM#L
z@qM+ilE=InM0zd?-QCa}dr+$P#&qj>g`3@C3>w$iOr5y5EROM5LXL3QT6KZwsq1G<
z-LzbPU9}t2#<*?WGj?>JpCi?LVD+v4VrLsRGG1-<uw5atxpsQr7m<y(rtjR#IjL3s
za?viAZI|9V&A3y&`)I9rm!{;yYVZ0P4*4~=_P1-sC{M83a(?$}jStq6#?LO8q)DFo
zdm_%9Yx?SgihB+|vc0I(bMNWg8;@)6alPKK<bd0{6MeeItY3I9HH7U|UVC5Pr+oVl
zwaW1MhyMQWo5#E9|Ka(vw=cPuNpDt9w7*kq$CPH}bYx1pvhCa>9(#^(91_0oQQ{bU
z)c;Yil`5;`#iOd{j;?j$3U=_0`PlZ8S1rY35wnZ`B*T57SNt0@T%)wNZ5PQ*eEqnf
z#(UZ!uH%L?EzJc_b%`%fJujRby6C8b;?CAZOP>fc<|HTZPqIEGq`SO*?aA1EPovJA
zjJ>y^_}0l7zYRt=3;WJ4e{OY?eND^PzY)*RY<Xs*yDsPX&aUT8UdF4rqj$WyB*eP=
z#nU|141xcP;*vCP9ZD@u-MMDVp)>t@X)+gYsWC3Ho*imqef0L9f=MBs(@j>-?+dd$
zA!@1k-1V8lv{`W-b^`Bw{;xT#6&0Hy`M|pFyl>mDYisQH6;0dZbEEf(q1)@*wT4j#
zcDDJxkzBLk3&SZnn;Z9T&MjoVf1Oj(G4TH7y8DW`Z-31VNxXQf%`M~orU&1CuJ|gc
zn7QfdX7hKG8Eu|iRF<B2`NZO$OPg(DpKa0;P2-f>d{N_ycCqmN7p+l;t^{tZHS64>
zHqZR!56y{3?|4l}3wtZClD+6a=z}|ahTSshQ4#*m!S}nBq~fGA%Yz;Ncf2;La_qTu
zjOFI}5asJCo#G$VGrE05{9Y|_{J5m|+l-hgNl^v|kG{LN@Qn5hZ|*k^%U2t4?M>Lg
zbDR5X!57>6{Cgev*KFWh{h{tn!}b(~+Kegxy(aqJ4D#H&MEiM>oW=~>xR;&JSw*Mn
zTz=jBY**IB!p?VMD@(jezkFNa^4f9Znbo=9t4#LX4qABYPxG1C(|tCw<}OHIV7GdS
z!`Zeg2A4Of^Cl@Aoa?pk>yM*v3xyZXm}>u8YkHD{-S@MKQU|V_pFMNmjLPEai6>^;
zo7U*EZcEtkX70UH_kR7o;3MzDZ)?xB+JV397WW$m?uuakzZ<yLeE7Y!zy82}F14S(
z{}q_ekhq_;!}Ibzd7<dHS6@H6VRz}<-hk5w11hJNryR8U`tpNKOtnJwyC0J80%t$S
z5t8L(lHK|-<^N6P+z0=9uiH*-fA}T;{kMB;IVHkX_VWMu1h|;~t8gq}U^vL)d16bJ
zYgeLf7{BtiO-DC~h^q=2iG?gEJT*l(ct?kb<^dJS$(o_-QckKa^`30Nx?{;nBTjEd
zPmvi{W)`XYO`XlMg(>RFioh_n=#5FnAr8Tl1N!}Pl^3i!74|jg+G^DWt36r0!n#a7
z%bxnq*1R}tO2Xz`ziCpwZXS!4&egB4tp57>`g!sDdumvpzn%SE|6$c?x2uc7SKqrA
z^(O0U^41;kYXeu`2wgVyhEZDB*ND!ft(74eXHq;SK0MMox#3!LLc7R@)aIPq!CCnX
zmZ?tf?<+R@KisQoKK=2@O6B;xYVIOFrR^@y9|f6<^KA;<H}B`B-LGFgNDrHT^Z%|r
z-q#GGcRz7y+gr`J<k9jce6#;u`TgVTavjwLAv6Cou@pEka;toBU>0^MILL52AizOS
zE#T4erb!MO94$IB8l0RmNgWHz?05uS+E`{N>4=oh3h2CF=iImK>!sxWJzo<RPdug<
z^J<Cp<ha{`Y76dZd3kUOF?j9!J1OLn#-at2`bEN?NV_gw(zQxRJ*X?D$b0d%PeSW7
zq9k+99Qv{9_<ZB@Cc!3AH(owJ6KJ$LM>OnDWV%S%8<|$Q`bR0MMn`{je)g$)bfsAI
z*eg$CuWNUdop(<unS8}X*nQQO?Gv1&JcB2>ecBQnU@vMK5~Qoz8FY7E@Ykd7j}@=%
z4$Ee`r13wpa8hR2l(=O5u-|20r(TbGUA?wvO2^;+9wQ6Q<eUGesisD<FU&Q&Wh9fc
zn8(~}W7zGWO{Ev2b9RfGN9P|GEx&EFN_e$G8LJS-!X}1{?3MiLt6CS@NAKJ|)pyAW
zkL4^P7gj2(+FdH)o)QpLbZtgM?pJla{;F<`$q|{#i@K)oib(FZUbgAfvKs{&OILr`
z9kEI!@YqSQskbtnxwf(xWIvmdam@3?%8r<qYiD~LTz%%yjz#U$N+TAwPYpGBy?WCn
zk=Z#WQF^D>$VIs<JUV~X;yBmImu}TWO^N#RGWAu|q<e4t>uVC-O|QKP<}`nt<-FD6
z(W|_R=6mWN_r7~nWIS>E|NYh3de%FGr%G>Mc__H%<A<13n>{T%GrpLfdb|3)=l18_
z6`vQ&rF<wp@62BN<-gJ#g9i<UG91o4x>J%C@+VKX{ibU)E5KDmeTvij=W^}3?jO>g
zEaLVndULgiu`I-{@>%&6{&dE&Ebb&O)`pqIZ)TQgX_~n?2C2Re{x$DWS7g%6wSO15
z+gDG`D~fN>aN^!&P$iOGoW0mF_=|~?=;!5|Yy#axzG__2`V;6-zrop%rRU85DQ>3B
zpPuIQ8S-d)T$Gz(<CUjo8Om<%D&*HH(4s68%IRHt$Uk<K*}N)0Pa9KVtvy<8_cCvI
z?kLfg;&Mo|dUDbGlIdjmYZnr?3OW22sLl{d5-p63i3r;?>xWLB?rGkDo@qN6rmWR2
zS{)N%6E41eg{#3bZe=cxh}~>P6Pi^Tx%oOhq!pDWwi~!`2|WmqJQ<<Fs^Q=yU%8NX
zdyBK8(MGlOpC_Xh9iKcy@sir6$Sww*W0TyiinIa}yMm`o5(<maRDGg6Uu|lL=oE*g
zvP>7ePRFrL`}XsRt{W5g`HE#i9V?Gc)eMx6dZEz}_-BoL^-1nDH^&u*PdrV!S00zR
z#yE3jD6i>i3Fhz<Wm5aDyf>M2^Jy~6;nQ0<pBeniII3~U+49K^4)cc3()U^>o0oh_
zvp9It*W-tp#Y+?SdDj)^Ff2-*RK*rl_)O7%&58fY^S->;;AR@t>l3Fe-`w?H<7?vl
zx?jpx9h3YPv93yTWj(^l+1jeW6%gwrbV+gPgVmzSQY$-tEEQ~A>akRE;-TE6q<K<@
zN)}io^?$TVysUTN)iSG7aXPFQ#5g&YE_6=q3~>>YpQ^ev#YNFedf)L$VIGZBUGC0Q
zI^Q(;yXwN34#t-=er@!=G{@7hW>T>JHOJ*FzdZdi7P-z9S*m7obgEDB-1xX@n>gJ!
zrUX^HhAuPs8oul2)s|-0#6*47RiRrqgmpiC$oKz-*3>^<;cI8*_-fsrd-~lfcczBW
z+BDZSvW?q1&q@XCaMRYk64o7kE$HpGbzhHOpLJUF(JWoDtE!CujrVCY2<?#Dd*{b;
zGrJ`s(JPc?PgcZSd0NO6lpuO|(vgT4az*NPSI>t;K8o;GdZ{(BM&<a1Ak`0bJd>x3
z%nwv{vzt7{M>W<lKQgOLcy_O=s*UiBGtW*7FZdU-e@}0N*dewp`_?4$)M%x#XbYP;
zi1y0u6}X+egDHI1xvvI-dOo)<9pf_MU|ptm(6{7{rsl4@zDHijd@T4Dww-_ekHd}H
zcOJ&b+?-*vu5jPtj}=l|SD$(+Q8{C=R;co?$DM4JU2IB6Jpy(qw(Q~4-aI>2a><0<
zQ||Gd(h=f(yzj8Z%ttGVEd1vA8n@fNn)FE9Hcw+pNYj${s!J6gAAS<-TeL6C;lGxL
zxiQzPph*l?UJuk?d8dYLX9(xGe8}kk;swpm)793U3cc81_hpfA?Xo>L7K^Q(*z2RP
zRGQE2l*`OXOv`!ITK(&zTI5wq^EqZa@wa(x?3&xjJ^yl+(l>$K?u;q?$+CA7{}yz5
zt+~3+eD`a;KL$69ICX_)PIXO*z8J|+q_vs(i0h3i!JhlGwu!1fpKX-Jq57QpUC=#^
zpst*Q@yE6%@9baQVkh@;=RL`kHIpJdBFZvX1>f~#dNL<&QspG?(5(EQ$94;SednIa
zCA~Dv>tx)5n$Bt5=Q+=XnJ`X%uygM9>h((bKQF6%`M&?A${D@Ci+m@1Pd%i}&ZBR8
zPki!nn<xL1Z>61nebHAh!(z@iAJGQ^J?5p8BA%<V`JF%ivv^N$%bX{BmfLFYb(;RA
zPt9Pa;pP2@&(91yr^4FGTeCt>KJnp^$}`HJCpG*jRh+!8i$SWh#b^to`Q(tdTQ<i|
zxw~)C29E?T84Lf_Oi}^TS3}%Q-frsKv&MK@!1~wC*XK>1_2}6r&!vh#wy8YyJruL^
z<1}vTgC;STnmoQ|F7y+5_Ghu}{mKjKR{C!fSP#96t=yLuRj}Zm?Ca^)t<%b+R5pFj
ze*6E_%3$$rEvuWp`ox{R@#djHgtN`O)XEt*9~Y`e&e`g}Azt{+TJhUAmMs03*}pa_
zGj{csi_h#NvZK4+FJH}bL+pPCi-hI92W7LB)@mG;VVriUVXu6t+{C`(%(;i7?a!#U
zt7PrhymqX2BUex1lJ1pUUsg?8x`~UGXXd*{+dCAQeS`F?E#%h=FhmQiTHK+(IfDE4
zLJ`ItyjHgux}}Bwb}_gs^|)}`in{P{ML8bOm^|&U!NP_qe6#yrZjhFFVkvvaQtpcM
zoZkocE##Fh*|?;5&h!a}lAAa3221el+PLbn;hNd~zdG!W-=4F9*;<EXuKP(1O%4n9
zNprW}W|?wGcCN-@&B?nZU-M0wAbK*g|AFE>na0BpZ)#fR$jw=>?(YV@Hy5|dIZt}G
zalzCXla4j7S{xu@yJYi@9f!S7bF$88iVW_o{NLQYOQO{+*~DeHi~C_0x5ZMcJtv>{
zcD(qx^(q6`mCIXh9oW);gF)=(+@?(18^YVer|jhyGWT13bia{k%#<m?(Obo<7P_z8
z>fR&qP}y$!jjfJGi;_glVxy0RCmvIC>VFz7rt*nDWw)LBDlx4^i_)7H>7P={`s|ju
zcw5HLj_VWo7kPB$1@Es_WPKzgamQ+1lfm`|$L*6mW=!EYb|ld}j?-q|f^{qwN1R=D
ze{|B<pTlK$q?1`?{nQucsuQ_dH@1J><iL|NfBOp#myph$4vQx@IJZm|xW2*W=cXOA
zJDLm?dU987n9S0{sVzL!S@>AS<Z}mj3?;Tqcx*8{$8stQPxSu@2WQ@1u=C)-IoT8M
ze(YTsvrg8=tN53r@aml*3;Eux*z`uB|K7sgCNWkLj{N3lHkmCE{E^YW^7SdXQ@gaz
z%+t~FW|Q2_rYYe5lD{i))eXg!{2S%AD<7W9)qOu=;qq*WuQ$9Ot~~rma^C(o*6VhA
z*le-UO|)o}Ij!orM`dMKwO~WwE}KPFTjx0Rxv$*gvP)~;1SYQ-NB=)sRX?;&Enu}O
z)LwCuXT!rI=NMeYb$F^jv<6*J;IroPDPDf<wzAmCv)7A{Ui)p}zuWTm;k_ZJ_a1z-
zcUHvV>Xi9Wm%SC7_B~&$qad{8X|hu6Ouvtd&lwf$dsDJVDM)PE3;ygi`(7^=`}F_H
zw1UF5pJUd3IWz73?|nZsn2V37-2b$;Hj(4TVdJ?P9SRQX9y3YwCMf>C%>TS=UBznS
zH<{}h&&)78wB&780HetPZO7xxd${algi3O_I8$b>&tQ*s*b}))f0_aJ_8H1iyW~15
zyL)mh_;zw?b}&rZ!Y;DLV0sQuS;wqRowK%Zojse;<0v_M?IQMMr^$v9ty7n5IPk)P
z!TI3aGaGmp?S4~nNt(w?X!qGrlT&hgE(xBt6kRQ?Q7d@rNUyk~)9R2ppI7wFtm-#Q
zIkfGf;ZYTv%{7;`c`VerWrJog>@soQG1*8f+sbUt#%mh$zADUn)!}?VB}D)BuCs}%
z%AO6H0$t7jb1ruXy3|irSy17l(-`t+2d8qSysrb#0?S#KJ!S<~2>;*7<9i_R<P9F*
zf}V>j+Se=b?wcXCpwZx3$d-c*Ch-}p$9I|UU9?+lWzb!ou*E615p(ti7>A{Z>I7dt
zDr~jJFL7&J&DQuQ3mIzHI0%{5wrosW()s$2qT?f-{8Ts9Q>HUIjy*Nz{Ls>S`O>uX
zgLY-Ey4qdg8LPLwOWvlW#JYSI|4$CR+L;{D9cSws-5YhcD>)vXB(SJ;qiAi4g!7|G
z908kJbtG5rkZ7&q`s;C~K6}Z+jti=d2Ud8Ti)uK*Wzy+6TTUoO{_iA7k*Axxm_1vK
z&rH0+6jSSXN}_eeOZK$?RXu?}E+$wlNO;9s_TuJ(r<3a>ngc}_TsYvVviIa1lb&V1
zv#S!CeJ5<*d6{pQ@x&Jw4u)Roc=||pq0Om9PY%wBHr(&oS8dhzD!?i{GTMIcskLt=
zN;gKE^!9JNxn|{KnT;(W7P{VBSmv(EXtvB{4KJ7}JkiKz?&&)-8QxClbQj@w6rFcp
zGvwr3qlkwx(UaEgK6*uWp@fDa>%`7U%XUm!rp49O>9Zyy;_T5?GbT2j^w_e|Y4XJv
ztw#h{Z?0|K&@^lDOP4blv0)Lu>o^uf<(?Ja(jGQJ=E%X$)mzoQ4887NQ5GwVyK!6j
z-W`VP$+oY0;{tZurm3C_dU*8LZ`)HPdrtn(>U^T?b|%a3%Td2qE=sA9_myt-O}Sx}
z%6a~Z*4j^drv1)cd(~)qQQ=zkli}?jCYaCS2tTmm`|b00Zk>N}Y4NYe3o}LBKI;ZB
zS{$i;yu?>YlF>)M=c8`dSqa812^@dTk9Di)Eu7J?_`v!MR>g_)=6C7$?9ms9o!?Vp
z@xy8Uwgud~FDx|*R7=0LYQa+B%@gllIx*kG&{9&iNBQj@m1!3xOnKHx9Q_y&$9I5_
z@6@fir+s@*Bx!9966B6vvS+sLR?FFor-XV&+lgMTb$VRuG_U!T%(0j=2Yv3g?{czw
zb~}vW&dtJB3*$RejGXT@?w*zuz5Mm%H8NNBv&?(wnPTx};=%tZH&uSRw^m%aV$0<}
zsY_eaW0_S~SU^+DjFbg8LJTtR`pWPwNo)<Csu0C-_QI7TQCp8h{&fl87kX&MfiR8c
z$6rr+FSfoJ;QDN;0Ov*H__U)DGSB>!!;-~Jg_6S3?;edSSy;srpCGd;D%<?opKE2e
z!wQ~WORL$Jdd4(g?)uap{eBDr#xtgs9aSoydp#xidBL6gs~Y^5XK_@faqxF)*>t&=
zXNouJ&S3ktSZ-C!r>AQY&+O-1w*Bvc!<`osrk%LH=taWR7YS1%Is4{XnK>MAyu*L>
z$BoGs#3%Z4^7)?WYIzv(Xmix9{Q|x!I~!(NZ9HjI*b~Qa$tLmT)ZnOPrBSp0+g+M<
zE_2o~?)VMt%d=*`?C5-Qwomx=qjg6gY3X_`mAkfnZ_ona<Vg}TE9Q7PZacLrT4u%W
zi9AJ}?-N&?IeB}}+x|0uH+kN>ZSmLez|qHog{?XtA5Xb;{`=)%lagHyRl7cLgzQ<i
zQt!q}z19449AhP>`LupXnO5<*>4>W5z5b{PTK;S~zO2GOAMUAASoUCT&cm%a7eBLC
z|6s4)(Q2*d^mJ`+jaTQBrS~`|FvMT;z1L#<CO7xq64yry_NG3KyJdFnU4!VCfWy+e
z>~mu~AO3#(K4M>@bNut*MTL5cWHsMCGV4qdK7Z+!-DjQiZ!X6d6zZ0zU2p41H%scc
zndQ!UF1_zR)B7L4&sR;^D#Mh~x;Cx-<u0Ext}j9>dCKy;Wn)|Q=6hvL>e8Ia@<gWi
z$c3(#T+<dhoDey9rY3QtN1*t&^&+{6Q+2sTQn@!NOwu^8bbe&gEYnA)-5yPeiBijZ
z#PvLro4Ih_JcsHH%~5fOS1)~}S$65dI%&<+tUW(D-h91vdrhI9-}2ek8_PdkDhP@;
z{rAy$tGn61tS2{hRz`EWsOBs%$W~LIu;rhCda&kpyF2Z_ZQt#fDsP_RyjIgKZk~$&
z5fwxGIUa9TCioal&+xiB&&0dfEWXpwyYutH2Ul_<zTeb}I_eUcmTI)~v(J{RZq<$+
z-K$MjvO2M^W1N>RT;3e^zkoR<@Z*AGsnc#e_51CR{LLgF`>1AyYlP0T2$`*^qUUb0
z1*FeQGgAJ#t8d+`1G#a@b~*pT5`?d(7Rnh*h?T}p`?>XgX63e+`1IK4r3G`8(zp}8
zDRWti-nerpy{|y<-s)^crVItij5x*&n-tXpW?Yw_s&v^N?>=XKOxeSC(fgnK9`N1M
z``hut+OKo5SLZqhi?;^dE29byFmrY1e2|FicHW~pBk>}~4VHO%w<iDIG5g-j<$nt%
z>r@>6&>rtG`?^YD^on-%`XAPP71jl<o!eQXH!w%<XzscHqqDr(NP+Rq3j;py!d=?0
z_9WWgyTs&sgK3Y@Vauw@8;d_mvs=mkk6B)^>*uQiuetL4tNtusV|ICWU$peb=s72L
z$tI>OInS?je~xC)ecjBW75AqZHd!0bIleu(VWCRtYO#KEo69Fs<<`4Jy3gY>5=vBM
z=c=_7TD*g+$28XBK9l9~E7N69u02(7=3cR*yKvAh@s5hcZ_iEd;;0GV(0Wk5CVEHf
zA@`c2<*Y~cGas|BIabcv?=LPiA?cDIr=ZQ-D{*>JcUpWlwU*5=F`B@1DgEusv-ckC
zIePQj-gmirckeCCI(xif+t${*?r}49o?L$RLS+A|C(oYfH5Zn}KigRP`fw@B%lOZ0
z-o1#cd%aDprj#T1XNQed*@t`nO%E5RpL-h~Xny~g{iFY%UmZH*?4ddFTh{4segS^3
zBzjNY`TGAhd+dcjWnHT}UUYxWU%aBVtYJ!pdt{^$S3`i3V*`UKiz<i8j0p<P%}i{3
zA_}Ta)1v<A{d(OlxLChe+Wyz07w@O<pRn)E+%2i+!*rw9-rCc{DZETNye#L|rpcGD
z$<DK`e#O&e{K9XMRkf1ttjkkp8>Q|jXxP<ycnWW@TFlw5($gAdYqG8y-#i~~8#a53
z$HWO`Z&@^RHy%2hWvUmpx@@)ILa)#RVtTbZ3j`S|3=$5u@GCz%#AzVum$Jh|&3VFP
zhEvnDllQHWELLZnt1tZKOo7SEOUrC$i`7<{9t=v2Ib5e}ZT9x&w(Q6M^^_lc%Vi9o
zs^KA2v*W`ft_JQiyIT_yp7F3t`|YtXSgv=~adFz(xus1PPv;7zq(=QIyR`PyV)25y
zJq$r_{MXCJt~UAcZnfx=^ZoVJLX3YWXB_l$W4>m*z>Qt1Be0SCR&c-f)-6HY+G}e}
z6SegcD;DZX+$!qUmadX`)V}V{!>$gwX&p+P+V?Ik?9@3lL#apZ-V--X->r_vf~|`J
z*)%8J40_<|^eegF{_7Q|QxZWrimj7_Js49b2m4%nA{J^CxpZ1gjwS2#_$o=Y8Oc?e
z5%FnkZpUVxd-{FO>>RFdYMG4}CDZ4YmK_Y3S7N6cF+cW?WJW?m&@++LmUm%p3wQm0
z8#XgI_dwd?><RnSUiMB*;##@XEX&Jl=~4$NtrhdU+*At~H5yi}TB$X`VKqnW%j?rK
zxE$C-&mQGWUsv`i>UrLdU)$v~zpZt9Q#ac{WPZg*&I`vj<qCWW+OpVDT7EOrhKF$#
zOcTDxZN0n9y{&rfoa1hrm&9m0SU>kHyW_ap)g(&jqtTW5PRlGsWSo8}ZC~#oEEqD$
zLuKxiOI{ZzSx@#^yW`?1U$>uAt_0c#+I;kN_*qhTRI<3oFu*z0=g~2F?ym*f(OZNr
zxysf)@z;H`!N_Z}->R=+Q=%UFOrI7}m^n2*gpqAaNQW!y+>0%`W|GOC;n(IQU(uW<
z={%*@Z0ps@{|%SSx)xd7TbdD@E?zd9<>R#3H&Ykan&oEsG0a(gfRp`u;qj|aQ;T<q
z9m^K?5Z@+obw=&ks6B_~x$W4);%T<+QTKW_UF{_s47nViHQm;6yyX{dnK?(|+>eJm
zj$Z<pgnI0L?crb8H~$rrN<#z7Vi`}d7NMXUT)SS>l`ecIbIWK^``-WmlHMB${ZIO+
zr6j-j<7xH!J)cf?Yb19_vqdO0N$IXw(6S`MsCdeK@0m-dM#V|4p7DJ0%ze{q*V?aM
z|C4dGMe4c0<F7Nylfw4@>-yyx+5i7De*nW17fFVzolMgf8dqwCRL@b8UcO@)&lD!n
z6)GMF10n-Lay6Kj&kA+)i~a8vvh@+S_?<_qLKkM0aAkD=`tW_9&D&G0duGkg?ASW@
z+(b97$3gzPwk@4^u_N7SO~bU!987C325=;Liu6~XNM4tbd9K($g(Je_a^FvtbBgbq
zPB`s~n9%eiJCA8%bYaBw2IEDBy_qU}Ys9W?nB|a@B+<P%=tahHt4AOE%D+U&IxahI
zlXpTa`9{+zvquwqH}H6RU1B+VW0I)WOC7!Pos&YpY_eP6`&7TZlP}Sta>mP=qRiUK
zx6-^iXYJ3`Em#tGHY0*_+qnamikSkUZ)JJTj=XirY|2mG!Zv1^?LLkAG6p<<ZDvZW
zJX)0VC2EdHn6PVPk$2$?XMW9TM<*yxde-^>>jHjFkMda+jwvOJxYJ)q2sON0!ZmkA
z5(l5>W_iw>#&Y>0!4Ea&iyp=1RQ{Ba`y6*=kErN+mmeYX9Dcvt({Qe%Qg7E$g|*Wa
z+O4j5xH(NaGv~;X?!J7l2&4HwqdM%qZ0z?-t&(3SWbSn2uD_q=)v2>RwABBI7W*?z
zSjq7B$nx1r1p$n2gs+ucmgK5?6*z<I)zXfvm7IB6K}$HMF86AbjGOl%)N4V|FP31n
zlMi}Le;BU}&WxB<CiJ9-Wn*}R<fH}_9e3-yRm%iV1|P}JT-R~7>(P#B7Or1Z`eGIr
z`ddp}x-KtwQBi1v`2KIsVb^9FGCp@nF37Q%)tckJf$PEl13_M4H&5H~Y<9k>6Zj*;
zu#fZTX2sdb^7d-CF8AqdwaRpMjS3U$|H4$AFjM!8n<1O;0#{w%EN+#?$n9%WTD@&f
zpH}UbHrdVlIitjDRuUWYspsExcm1+fHDXMUl}h@ab<H>R#*Wu_zgSG2_UP=>ie%*m
z-X%wKOQPq8vP`~n=uW0pucYd_Wi#apmMsrGlys?P!A!Yr+g7$uG3L+QwRCOnv}60;
z**v$3RAtyO>Dh*tiegr(2CQ}ej1Ql`{lSs<)k7ZrDTnoom?h;d3-gvsRLUkD6^n43
zru?-@PN?c;&&@z?$;*r7Z!Uf@X-!F>24iH6{8x>M?Ps*t7#&%%tTR6I|0XVVNi)m(
zAh|0%xl-!?HCOpFyb)X|P~qV6_Sni@f}dIBI~;k6w3@S)U74j_k!Bu$!F21QNlYHg
zr+xYQ)MrxQr^(S>k|#5`XWuYcR4=GKQ^l)WQ8?;s<+aV%idvUnEZ#e9^CM>6^&*8%
zT<7=&H=54cr!+(1bK|s!->%C47Vz7!s!qI4KxkL$zF#J6frr?l9HSQh5S&o|XU&n4
zx7m|4RW&85PZ?M$Y&!So<F$%4hsBOv;ETEuam-_tyZ#}gY0grgl=O|cqb@M1&h`w_
zt5}#aqsHyd{LD);IwQXu`tF?hYv=6jkN?(bZJ&1LotV`0r>gq<&zId>%`lm{GVR>1
zg}eXHbgCAPn|J$5+~>7BJ~P}XYKiN$^||cauJ0G|Lvk)l)}lo<i+&td$_xBrcSg9?
z?>*myOy@7R0w%|AIiy*h5FzcM(6G2;&0*Hui4OjC-BA+~h4sr`eC3$3V3GDDk*FC$
zKk~x%8I+cs2wK1D_cal9mCYu$K79-SRy{41IA`K#`uDyMYiQZrMY4_EQ6Uw#AIL4Z
zr*X#9G>hf=xq}Pa4BjiSC_1;zd2TY-WNGBxs%(izS1z_(=ViJ0BSLm}#pWt@wXD$d
z(UVofUEPBp=@)7rx|TJ|!T<BR7Z+D&3h>NxGdKRqVc8+JRYRxRv*Bav+`r<Uo2$wn
zxhrUIDGjvfzIpNgw6bK~rsOmIcQ;;{wnDAI#)ebonXph!<I`Q5&Vfvat2-7>yKcRa
zRdZW{O5@ry$~x7#KTiLT+H@p`dscwAvf|Fo-+v!XkDjpqPitCzX|>Rlnr5leZ`VvJ
zqJ7gD7F;iIW84+AcW)4r(t_tQayQ@HX1;ffW$MgI!8_HK(=`H*CufKs*6pj+nOx=(
z@zh1;Sg&u*jP7j)bG9-}WN48RG8N`LCDdwrqd9PX&mESw-WaFc314<NN#;2EPC3Z3
zrLpsejNmON=^KqKg%dh+1X6MwCHEZpQ^<Jku9Gy!9N7ts>{A%ee_bH3%cJXQg8znT
zx+1ZXa(5+_I40TjiBw3tIo|y*Efy)Wh=1*s&Pj!gH!_c36TIhLIrsmIlk&e5r(Hd{
zFeI{H#%r4CsV}NYN?kW~ER$F|y=E<yobB1gYT2q%cVX$9*n7DGOL~sYn|tL!W$fP%
zHy>tBV3(YzIZ0vXhNQ(;`zkB$2^b0r{S?@GW@^Qb*yx*!mIR(ykjXxOqj#nShv?q`
zXAZs>3XgYdJm%r*6WiG4FrjhN+bFraP0~KD=bj$%zQH1y!yx&>C3pfu%8e*V7gr0e
z7+;6`0<YXSzdXom2$L*Hw%Fx(q#)UZDd-#P?Y22S+BHWMe=58EQErN1$(hQU7b#K@
z$-2{5C@#RYT5$RS&N~TzTrX~jc5M?%kdXPWnfW9&#Miy6x9rFz#zPM>Tdsd`eBgOY
zNbl;Y%Pz;#cCfRbx@2W5*Rs^l$m!1Wlarn&-tjfNv+c`-tvlS0PCa&#rNMe88{fqz
zyt1m+z7BIW9VB)5?HlKu4}99Wha>a>$Jd)prrQn*`Z0dJc`#&xqx6&3)4mNGSKeWp
z(kRQpz}}-8;-YNhrYtzEL)PIai^J0|I~e2Y6hp;Ay5`6PU05pU)!6W-SyD(=O6%E#
zEl#efA~I#2GD49ak)g99=1!i-CiiOL{LZBGe~OL`bA2Yt&im7OW9Ho84z|S{4U#2o
z3oS1Exp6}0Y;54gG}T*obMhW7&){M`s<tBI>ea8EDLP*Nvt=$WJ+UnR-Q1@d^I8o1
z;y$Kqa#-&7?9sEAi<S${5Z1i$H1_hQ8Qm`yhGf2xd^y$a=-;bVOQyOQxK!z?zS<C6
z-RBl%bInm9)ltLYo$U-3pG3P;g8p}$(jLZ0zTjYg$|CM_K&&O%{FbA(kAum!Lu>^H
z&-gNkA7BY}c*vI1=r`qH_%5baqt=FP2SY66kL_R-U$OM~4kn)m$zAVUx(kk;Dros}
zaK+DF`O`78zin{7@b=p8SWc&D(f!|g3LROCe4ivlEZCL!g!SIT5|gtBW5XH563SKv
zX=^I%o7(1~=f`|%rsp^5O5X=d4z%sFO`4%5;e0@)ienaoNNQwh`|5xh|6g$&ciy8}
z6?ezO>W*(3clMc=k2fqhD)}_ru<!BK&Q~U?;yDpQW~>d9B);^jo+(srC_2D)g+<ci
zfIyonyTKWj2@SqE4$F2mvbebR9cYoAv$XSoviKb)OFKo$DGZVo9sGT&c3TgA6j4lQ
zQe5lyRBVD{(yWK;?5;S6-kzx9+3$OIgV!^eqJwK?ZeD$RYSPoh>xs`cx6Lj7@lc^H
zR^eOXP1yxmT93B53FzIO8+_(cURl!JiLX^BWe4Bj*mdY;-ZJg{T{EjaCNvp#^~Fqn
z+S0{dsiajk;n7Z~J3E^)Cx{j^<!qWTkIRpHv80ah&e*Lv+)osE8n?19)0L|7nOMG#
z?SDm2o3_H~>b&FZo7Ah%&04LRTxW8*^_iggzy4OIL{1+@&Kpi`*AiAHDaL(cIen<5
z_1+Q7ycfdzSR}6;vCLBy-;vX_?p1a~&qu}ECThAFE2rstw7D`Y$@a{R$Z6^MnS5qL
z(}W*QXL1f*zIrycLD$)CNnvWxg*CG$Zkl~RN9B45@3oM#X&0ok?#!Or#5-5&UPgwd
z*R<&~ZoSPciCQkm!ldC>&T@de>p^u&+@o*0H6n{EGrD)ioZk9HJaY$&q{Xy1hRM(R
z+TPrhoSg9P8Q*E1Rp;1v(rj53L)w-`g*_C#_V|M@qvRZoxJewH>r^{#sY-4+v~m^a
z7srRP9v8arxcoo0@J<5Pvv{^;@qB9Ht7fd-c0#{IUR&mwq}sDBKbdWh_}C}#Yp-eQ
zOLejDIHb|T6v6OXZN<qS1)V>CxNY=X_PggwXrXMzJH@Jwq$!EZGG?jWnj8Di$~8g4
zYtfpy|E(U}<#Nau+OT_8c0g@*?xR>n`Q<yLp7-vVt1qU$rR2g9#pOq04Ol-fW8lzB
zTHDs5cratb!Cg7+(UF%oM+9%#bCoM!{pnVNopTC=P6`<$ov!3~no$t{cB#7YBk|X|
zkNLD2PjNKp<{WT3@SHzcug>S7&8La)%aS`M91z(Mv(%{dEnA|=G^a+njJi{gJFV^?
ze$djDbzeeWUb6e+$E}WuP4fSLesw-`;ZWm+CXEeF_E(xTIvjs|bJkyTNFrjk{f5I5
zSHg4?&P&_~`<~(Hly<c5(b0mL(SH@iizL}xqZB48v6ZO4)DV?h=diLgK%9}yP5sCH
zJARtwx_XZ!<L>3X)UD0Dz4Z7@zd1(^Zqhzpv{x)j_+g4}p8L#qDUaRe95j2EnR1bb
z>D7%j0^)Oy>zf1zRc&}-H$!0Boe#PzIwxpI{yEg<#~_fms_~h6XX&FTDgX8}S{BQ1
ztm%qbtES&QVe==u^Cu6Sb5QHZ?PSsw;jr#Gq$P7wE8~z>kCW{cXBqBK6OZ`xy!_m7
zBkX#b=Px$llvVm07oEKS)$3luGq0_jTfW`=UlRB_BdLwC)R!riSLxGvC9Or0Ez_0n
z-OXF_`tHirp7%XGH-Aw{d&IW&^<J%cJNPzK&QN0A$~5g<ke~LvoK@}n<4)}ta^a}w
z%M{qkAW+J}llpj*693xc{(7bftQWO|PA}Rk8Y=#HQRJJlh*_og3`8d$s#_9Q$q`n$
zrO`I$zzqqOx;3uhN%76kmh<o08r*S4y4CP#$F)oUCT{vzy6LCsH`BG7ZXOb=n4kUC
zSw^_%+BGNJ=Ns*!-)mf8vh6u&72~3RWpmF>G3%5gS|<u!6^m}P>4up+xe+yKjgQK$
zs3)#w;eQi&rvzQUU+^Sb`hmvvT+PjGe&TX@EiXMjZ&Tj3>3`;rN7}a^X`kNoykx@)
z*>@ajc_NzS#s4Ol%1V{^@n!qy9cg;^II;ZMIrSG44(ZQ%{#<YAf=ybM(pn#c9)4Ny
zy>fo>`J1Z}iXNVm&96Rg?jWfDY1j9f{pF#*CST5C>{a8itv{-<q3LT^Q{TlVwFXw*
zE1PA+Ic$5J?6@s{WPQ`<I2c}%>*Rj-r_k%R&FeiXcg(%RE3dcu(UJA@Z@%|^n(zD8
zExXR-{~LXV@9VR~?lGK>%(-OYXz+pgc?F}jLH}#nBDpyK%RAVFD>-<7PRxI_UrSD4
z1{;IR3<iUrnP0BXDJnI&t5O=Yyj1O0X36Pk$(O^}<tl_E=6Bu_$kua|`S^eDib`wA
zma1!Awi6mfmfQ+jXZm8x(F<&4mtU36n{7CM`p5b6KT5uC@y=kpx@*O@lZLX_H|Jh-
z{@T@;wtb`Q<Bd(}y5ID+CAH^Amb_I{Fe<35nqgp6&A9Tf?~}~sx|7yzEA<FZZTU23
z2HVwwqT5l^r|2!nJuQ^?yKLR#U6IV@y624we^(W5_hi4_zG;ek@$D*qx$4Z2?WM-X
zCJ)`~j|i{((r&akIM_|&WNV`3;yD(@zdvbBS<_X$W3u&*+16IK8$~`W{_@KBThzB^
z?jl=*3b!jCwM?we2*mbC%C>Syx@(%W7h9y>^Gy5^e=SjF%c+1v)1Uo0@k3Ea>2ULw
zrw!Tv7k??a^RVL1%^#k(J0usB9N(_3mUUsq+(nNj{^9?a?fYH(<Z)U5=i8QXIGp<b
z<z)MZz;=Vl?mrJi$xOBWeMEE(=eJ+F%ezCPpXbMOzY}6P79^<66Lw#({lFr3qu^_D
zu4m#OfAu$L{phKD_5IfQ?7yx|E_|QEs3~&E?S-Ly_&2RJO=;6vZ7;l7KWV=8?rHjW
zGN#phQ+j+fwqx6kuuD6qt_|5CuDbkN*77R1C~d}FxoHJPS`TMyxMyy%-uGyZ=7WD$
znJnfy)qdqV?kx{KJ;|-hbeZF)^{#pi`?HYC{7Vx;qUW?|7MHzluWr+OaqVX7!|mNS
z8c%a(d|-Ii*-&hHdh7qH)gB*}^6NI2pA+2w>D>2P<M(fj*9)3SPjr~lBGY*>vHy$K
zvoG;26HDT!W&HT&u6AI1fJb1Oc#nwT+pe^4xrgfK-u%nO@Sg4P?)pU&eE%HZ-#giD
zLU7HahyM#-{qGn4*~{F*V7~Mghe}2OgHt1?q*#gafrkeh1#eH8l`(OFD+}jj6$T{+
z&ciJX5_wNFj1sjcY6Y$fxw(p0n{jQ%xfG7zWvAyD3V%9s(^&oR0t3ZuE+^X*B_~=&
zcIBvF3e)D)UD`2c)`wLMYuvVK2`o6WDv&kqV3V)Mx7XKs*S)*3>9N>5*>II@6CP|z
z?4G@&`u45Ho3C<b>q_sKlCb-GZhUmte@W%IH#-WGPE1w}ugi%voT<C*PwBVo>o@CP
znCDzACR_C-vwok?^0L_MD`J|_%Y0@y-8Or7du#b^F@w~@B6<f}^B*eAIbEoCa-x3x
zImd?2PcJSmF3PL5EYrItoFx80*5<Guk7eNdD@*_U_;{Cfg}T0+UHH$>JEtwIm*0P}
z>8I;E{acszGe}EqP-psSyh<y&X_Dp&=J(kb4z?)lIjP3zF>}HcrrBB(oEB%T<kDGx
z&Pc0~BT#7KLZLvP6-Na3201Ws-rLox8=f&`br0vMjHA4dqjnx$YjjFsaj$%=U=NSp
zn@x**Wjd$4c9YKTnRH5XZObPwY4OWKr*v0-De}@kdgT8kAKm<u8>ZGfM1S6FcC+U4
zS&N5rE^n@iy6I$g>x)R<_J@C-d_M2MC>s)Bzw*OmD?OoH&x<Wmt}ieCR!a4_=(V!A
z<l-lt(2&dT7illH`R1~<e<2U+2L7wTl0rs`LE=>>>>^rqR2MeNNv#NDJX7&Cv?p(=
z=;D}G-Ny@K{_ZYS>yBD9bzx)F!4skxtO_fa^amJuE}5`umBG@PS+@fAElXLIxi3u3
zWV^z=G^^>N1)r6h54KErv@M!((Jh6AtG1oZ*B1UL9LU6~mf>o6^3IO#ZnI~%7V$>&
zJ!|0E<xy~yPqA{(>K^MT#W!pFe(rj+ahsH|<&#OC$9jx**u+IXofet*-?=NyWA@Bv
zvvTEf)8+)PZtwLqo6)=dK@s<?9S?fc-tBnM7o69%bn;}rS1XoI<Fo!UZS$@@A2xj}
z$^O14kN?ej;i&WXH=VVUe*2GSkN<%!+YLe)IK)L(g)DwH?Lyu^wV2N6@<68Y_tSD#
zo?5N)FEzvUS3pdR&JostLMB$JWr7|%-)S_raGp3J@i^Mx-_fZu(%X_hcc@F$Hg#&-
zpZR1M`TWz@tFCReWjABj*LvTHeqU5B9V&Iy*uwg9q+k6j&3_dZlk+!k4m+`f_l0fy
z;@m2ywfbMG9QZ~38ua+y{=4L@n>r=vpP6p`AJ^Ru`+pm0mHrR;;m;dd7ZAPl{uMKi
zjsMpx{L54g3tg*qcwG{!PDCe5!xAl5#%xu=)Q4V*0y@*0)iw!qCAg>_&AM2#=BU!7
zoI|Q5)?9a9JQiM@lMz>8-R1GgTyMz_Ulq+e{Idck2}j>p?ez6V!^5aa7MdOhbi#is
zhBs*lZC=w+=q1tpc#1+}kK-~CX$9wKFN0R28%-?11&4L@)Lk#-E)a=VX_k#S*r+R^
zW~AW#SjzsG+myY5y?Wh`qjNU48s2D5TcdeaF8s38l@EdawJb|MZ!<7){>C=Z>G9;;
zw!<Ee3JzKv4fHG8_F>KoA7AU4d(M>{IDFErl-D+ad+PFIljVY%a&ta9rrwIEGz;G<
zq?+5kwDHg5|LavI6yz5)M?ZLR*jw~jubtLK`G#xH7Fcz?a8=Kmb5rQJs3<3c8`q9U
zuNqiX%zX|RY}a5;%yVSxxad)`aI?}gljGAm3LW)BAI+U|YKdRjp>uOTo|$BGJ<`|s
zW)n}81@GJ+kv3L0dHRnpKB@LaWbfTS&&+QGXM7Xu(r9<nwSTEBzwBJOU6<afMe`mt
z{dL<a*EdbkdA4!rDvwpGCQT~|*Svbwsl~;CVb{Z@`fsPMH4xHnoX{0|_L1l&mIZ6l
zZJ#V>>bU0IrWL{%!5uAa=*b?uM0AJgQ&o|sRUwTtC#|^FJ)>~5*8MG~1ePrMVCS_m
z!0|xV6t|bB<NrzpdSB@hQux1bYiHq92Jwh#GkPyx70aE$#3|Ee(7AD`sOpUu@m6kz
z6Fl2eI!z;`#jQ749{ZTmp8BRICy_fS<6(u_WgU6X!&2vO9Txw#!PWZARj!~3jUveh
zCX1*Yh`M*<m|52p-HS=lCQD3txBD4CF4nR%y`MLUJB7htV|!T4y1AF8Ec+vMGvQ{;
zbGz+1Tfc-eC8?S34V|jJZ;F3F+05AB%y2vZ%#&ggNz+oDE*IL`=s(`2)X2heW-p_&
zC)0@p?o*v7`ARniN}n)r?V7sKGjGDB>F;*?oj&-vY~A9trt0+FdUwSSrA;!J;E;ZP
z#TG@btrBr_{(R&=U$MjZ|AAzO-sKCTwq*YoS#slu#Fh!mH9Q_3^_<`&q@wWnro<Gp
z{R`vfSL}H#Yd&S+)F<<twKW;e+IX!IK6-xfj6Ey5(@GY1y>j~dBK+y^tyxRL#DXe|
ztPXrUvubI4*-A;Pi<cNyWht<3(PZvCy2eEP!&KoZ37b_G1xKb`aPhdoQF;8-eBO;m
zV!l`@ww7(0@^i&%1uaeO@GQ%#)2=wHc&44$x8;z))>+@%pG|$etZ_xdcdf#GoC050
z2pfp8<{V7!)eK6MoqcSV-L|Wm#mZ~cv+p0j)6~69O<D3T!(>*r?|KI=dd>QGP>R3(
zoA_$MwR1Bb?YS(p<*5CXZ7j=l!=>`07=26D=83G7&c7@+@%aD3C70NgJ)gY!mKkuF
z?`p$tS^uI}lW$*qQSMYJ=Q>y2Hg?DM<t|6_U;K-<TW>P)K=e}iD7iV#nXC)F9)0z8
zHji1ob4}vS1Kpv{@>2Fc_IN#%IUbj~ZF{FdRsPcRLJcf4WS*X^i3xU}!oauZv-j6I
z=`(Iwex9{G_@L$SpUt_s{!*^2%l?$kPn%>m_2|Lr$2=9BZF3HODVw+`eOBT|h9wJH
zC9YQ-GreZjdw+Jx+H%`{G3-g%^~=;5-)J7+JyFj2CSQt&l}X^`#wn36=LP!QwGEc|
zdga%e8m%1<^F0)f8cp-xr8T+g%92%5n;(VGde%B);o03%98XSfaG1Qw%VWi%KmVPi
zZd~Rpd;Qr>@l~LCMaZS)FO+S3I+m>W3SI3cT2TLb)%GZ-b<=vZRp0#5sypK4+;M>S
zQhj!Ni8^oi<Kv->o)Lm8XX<D9+sp6m(q7N9<;JF_lOM0HJofLK?&Mm_O&fKm?(WfM
z`SVO=X5hrNOLx_9whL`|v$`|=_~fu<wz*Q_vu_>0ZomBJ|FG-V%C;qN-(i|yk)xdX
zqvg&8-n(_Dx9fS{-NtPDrT5+@Hzv!4_xikdx^O-)o4{;+A&0?%%VWj#b*GsRw~F|*
zwnUwG^mwRsCVTBKS4JPcSw8EoFdevjE~#0<{ppj#+6wLow-y{KYJZlbUS85yCc$%1
zY0a#hx8ha*&s{TFc0EDsL(7|VH3udocJ6qUq6^+G8v+=%ec&!Fczx``>r)1A9=+b1
z&~nU$@8yHnd+!PGKJ?~{()M3){LQ6v$A75s-_<)IlXarYM{^d>d$pxC^LpO#e5|?V
ze#&lD;2U=jsfnlUw4ArSIi>M2<Oq-OVI9S?rJt2D8kNo~9P$y~n<K)hA>zNW>B*V1
z=guticw!gS|H`Um^&!@21?R=m&p%8vY%ROEddDJBG2Lk)=Q|==eolH{=%UhbPhZ#L
z9jhzTOaXQF{Yq|b?bjln{d#cXw*YraL(3+u=07#L9u^Ig5?24c#<jQZg!YBPzn3Qd
zz2(($!%KV1+5gW1<z8R>Vf6p)51X}@?Q(S<crkDaC>-=Pm*QmnnKFs*Q^2IdjN+H>
zZd!bI$6`h4PuBN(?^(Dqee`Ai-MiCkg0l4`=Etm$o+alw1+@e?vHX3ilC4=E(?31y
zVaqn9!bKBj`l!rgS;n*C<g)}l^`rgj`DZyb+mkAm@V4mfSn+{teE@g1(5!6%2~##{
zPB+qAIEj1tCidsvudjUI*(mg8n-Sk-gMBvy_zoNKJrCeI_0H_vJDrPuZ`V9Oerw&^
zMUSiR<tZ{xw7RCQ_(n)@u1?LYH#JsA^?Q9jSUf$&5^ONlX?o8YC$4oLjMkqCJ7s54
zxHQDesC21_!h1!9^E_IY4Lt=vTU{10^*`tL-!-yoRjAu$4Gun4CnZHEr>{Kjrm5#2
ztyuLnWJA8+iEldRF4lFbo_OVOfkm^;^FmnDr9Vq&R(#mgqvW#UZC6^arK0k6zqX`(
z8$aD~mpPlkw6DOWr%&smxAn}4+7G%V?`z3zRQk6mN$Xs&-8X;R-~F}^wQk=D?lgNV
zAKz`qx?$U!&32W^nLL{^dlqk(5O6T$+_6E8X{Q2H@9ONZ&K$Rs%F<UH?F|?gJy6MB
zKRxD><F{vd1sheEk8cT5V*TvAN9a*oT9QR<;9kEkkAtpAO0&D|NZ^#`dG_Y@JdHKa
zmmJZ*lb~U~T%-FOm$G_7%@@ttM%=<oJTotO>^9KK{^YUre~@SS2Gyu1+KWSUmS5q0
zJc0M%ChgTCZc`Gt^4{>hEzn)<?o-*r&z)MW^^||^<7$zO0w4WUbiCfPO{_KYa*t*`
zC9qjg-u1N1#50eUhyDr)KX%x}c(UT1GXZDc7#e;IIin!-m_5LFv#9W)fY#Q}Jn!vV
zE`ISlv@*reTzB2HHYWxr5wmo`kd0n3Y3pNaJln!D)UrJH#C&Q?SZ0xYku}id*Mt)?
z`KEjQT4Y5V*6%8m_FlGO(FReb{z*sHnz^}jo=EC+N(sNHBzyj%4dXG5C#$tLFePl7
zYO`^w&Be)UFDNOVT6FnImiK+*Csw)MPdCU<FO|P-YWKI0p)#54K&~DCf6s|+$`%SQ
z70#cvT=1B2aoXJm?;RSOvaC7{j4$1H%bdbztf=<IQU02;=OV2GM-u}@cV~Q@zGZb1
zd++XNheOO-roT&Z-MPpj{cFx5!<pL}+;nv`n*H1>Lp&CDa&MlXxp=)tsll%_Gtce8
znr8*m=T1s5-e%#Q@?_H#ul{Qplf!ga+r0Oh=w3Inxbn&3)+V3z?k9FU@cHU@>VSI9
zy^?qDL~^!Hv$8rGbfPRr@UuSixuEbYVM!;WnXE>}!JkS}8zlM-%_l}`tq(nw^Eu|#
zk^?#)6*e!~bX3FOYQdxg?u3lpDTQK;yx*EO$K@HD?d<x|(<Cn3r#P?arP&eDA8&Hz
z{Vxo67gOsOo^@b_QrnY1r`B#=Gl6^h3bpAc(r2#NbK!L4+^b=iPupno+LnKhDK(cn
zDqL|oxT5^J^<h)pBU^1RPmQ^LIp%8Ux_|oiAA_r27H^xvw*AZ2Obty3#w%94zAKt@
zI!@$H?Bz~6eZ6M=Q7yAXm(R@)#lC-Y`%wG1{E^u8Em5EHQ=CHb4;9Z|o$@!;bM+?A
zrA9oRBA%QxJd(pzy!7|3E7po#(%AYfQdy!=v7#wG)NAUurio$A`-_-nZR4}Is4DQ`
zci)l4{WNQF@`+bQH6JF~JqYo;%<h+Lp?QNncuG^<moKLx3T=OxL}+YYE*EO7-ftA_
z6}UG?xO)dr*Z&`Zwpq3c97ctz%eA(u#sz<!(6B3_-^tpgFxRPYdxUdxam*Ff?+t-*
zPm=8wRDS4OiTf6A*QO!nzO6ogdy}qiXNGuF?spl@i2m0%9#@tA{iIsH^nKLDk2Z-<
zbj<X`cyC56Tsfhz+%>yAR(WU4>8bLXH)FpBcK-NSxqLdq^6hcrD-|||PuXgIm*tA0
z|4v2rYf2AuvkP|I7x`uz)s%RC`n37oQ7i{k?x|^c38_B6K3!W+Jd#6v?+?2@5tqWY
zr1u8*6;DfP7MmH)m(h5waq-QWsume5BWG<do@MRV?=EI<CNambqIvrk@3TVj`$BYA
zUeooK2sl#q^;xt2_aeWe|F6A!_E!J5Q11C_jusc~9&9sW_b^u9oEx~*aJli+sHV=_
z%NLx^skP~f^q4Fv?5|=FYA3|d^69!b``Q=@@uL4c=JMigCn}~1`8qM_D!QE3)XA2V
zUDEGa-lu+J_THUKi%m<irTXt`+0=?_7tN}u-epr-UQr*uqVl^<^RG*1Oeg(ZUpcXV
z<?*X4?}y&Dv8-AyW#6A>?|RdId-!T&&m9IARtt+v`<$GdJ#)=V=IA`hXtk&{ep74x
zJ$gMS<QaVF)|?`$m!jsAdQoDUs==?h;Z-yFRr*3hehD2<k>8dgS+VT<jakCCA3tOE
znr_y7G;-7B4b8hnTDZA3d+s=Xt#Ypae~IkvVkdVf&0`Im7ax%`xkvACS<MQa_Mf?(
z{Sw=lMe~z?Z%zHZAdoX)az%dm?T+Wmg-UmCf9u?6I(6-_SzDI9+F=^CQ@-WVqqOh8
zqUs{QYQAw~P*^LQcVbfmhohL}1YO^f_sk{1BCg9ymp@;;c8X=caC+qBrl`}md+%1*
z@kC}^^_uwf_CCS%$De*r_#VCS=GFa%mXm%49=P~*lJ?F27Bg3^G><#J*5Z@tq<N3;
z{F|(JlXt4=Nr!`rS6j0xTPk`cSsEU_by#xC8uvMNPK#W;rd(K+wdUioT?&0Fu14#A
zvdxgZ{(ZqC`wXix=G%>rtDh!SZJhctJyN7;?v$Cm!n0TZ-&tQiVPp2KoI^KHs{eMM
zeQl2O46{XIS?04dZ$vKaS`+ky&v5bL!=bb19iL;%cTM2zw}jK*Y^9{^-XtsE-4hUc
zbbe;F&5f9CH}`P=5;=eH*f~)VyALtjf6uX><iMR)BgQ|sd#bhh-PVh$*Dsu&vE-^7
z!;0C82~$}$cb%xTvFF%*V(OmD_H$2!=9NsJ8R<GbYH#R@a;dA0*DrN>{BOIpqSkyu
zYV6(xQY$B}kC`sMYOZ+f=Czdz!&f!i+V?-pXiLtN+cT*un{n$bM%@4d!4m5h?ZdXJ
z&c~{zh0i&%*;Y}>L3_)A*hi}04&9xuziw*HOVu}3-Ts@L6IY04iB4m9Hgnd0$vrdw
z6kJH#(HgU?{?AhJ%9HWCW-pXHweiczy^}Vb3^H5x;Ehv7oMXj9^XSuSukl|D34C!W
za97Qi+g~1ObJZ>KnRnm)vF$xS+d1}U4(Q2?>X+|$5-3;mYR;2zIn_6Np7Ki>9ZYL*
zTI&CLo9Rns&SMP8|Bqcf!TO^4rD~T-(chy>PJ1npe0$MlOJe`_Wq~sLPX3<Y%qMqt
zw*8%n_Q#^5vupS4{@s87hjn$wKEav$=LYVVyqSLT-~Fkk2RQ%iSHJmI)$*OWWn}B!
zcL#anoqryj*}Q$;<Lx>&O1x!C4|_kb-*>XIaTa;HMZ~^FEbPhFSUa;%+sr4ZT=-R}
z!q8$g>yG*V`$?6V$E?kgKRVyFUgF508<PEgS#`{W>|cAAIT#;#b??3Hee=1q%m3W(
z{By70W_IbEdzJQ1ZQnkz%Qdsg?OW8@oWJG`%Y!+mg)-LE_|KWYdF!8mM?tMi&p0GX
zJboT|W_$e^%XtF3#Z>p5`EVp&<;1dce{0U4kN+E6cmDk(;g3dtZ(nm<cVlP9=9^1i
zMGMI;ab6MqPF{h9L*zu7!okKA9=;sa243yPT$Wp#Q#VXq8zwYKBV^G91@}fStFAjv
zX{XtOQkHN8Xa*&EOkh%NnIkFu%wZ;T{G%<J2OoG(SE|1!BNgoE&*)X9CA(`&S}>!~
zBqz}sUmgdp44j%Ywaesn$p7UotQr!lQ`SUo;#!p!db(`w?N!N(k4;r_Jk_FE{`t|}
z)!X&&?=Ls^5j>OlValNne*cGyW}Ft&o1pAH&u6BQy54ks=W9KZMv`aepDYkO(77ek
zZ=w5iJ>OGX4qRjM-af~7_qMmUcND)4<7oIc?b=#NyPVM13mqPFPgZunH|b~Y^K+87
z+H`GyIs|RCVl(#$OwgXjq|R~oZs5UbnnFSrYUlUu_1*aH^x^L_H~;%R|Hs?&_vioL
zSNBU-b;7*{Ca#Prjm&Zymz%gc)F(IdN~u0<V)I*YgoP<b;i#B&o&{@raE*p*heBP3
zQu1>{wF5~yM$eME4fbg)X3)FjaWsR;aoXem88%%l%Dv2gHB=_p@<pml^pHE5EV;N;
zanfWq%gAF6sWUEiGK9KmPLT8~OH7@>Tq?PYA$Hb9m6<6;JD*Nvs=1jqGj-F;Ws*!8
zCDBaL92fjtxkJ;VOOI_en!`{R<*u<Xm*v!i`D=AfrMs>Xd2+{bidM)3$E`OmUv#ZJ
zXC_y%ZEETiujqt~+kspQrQ&_p)J@F_P@JkN5W88<HGB2EMO9M*KV7^gU$46|YWBKC
zr&2>Uo;VizCNk}$$u*V>Zrr-;mSWGYZ*53jJw2h^@Y>qM9?oYoc0`M?NwW7BI=)Sr
z<GD;Sb3*CCo4dcf`l08w_@Op~*BVC2+x!0QQrnQjwXrh#zb3Qdor#B-402{4WOLlJ
z%408Qi$kubiphl~Oa*V|I-PF3v+=|&#+o}P)R+4h%@?$MWvJaMw`;l6wiPCE)jvIV
zxX$Bz!+yEK<mYqkn*Eo)ysl<sa-Dzimtk%3<^Qa#TQ4wu`BQw=rF-e?mKf&~3n~)a
zIli&TOgQVx8DkK6>{jZIKsV{sDFJzHvRNk{-VMsIaBWxQD!X^T_o&8>p625}A|_e>
zT2bNdrZjKIqz>n}Tm8C!IaFswaix5i>X&CX;o$V_o0iX(R7)DpO5%E=`aJKM8e@*I
z^sg5S<?G+9c-vbQ9{a9VPfNpj`m1AI$4ttvoMK~J$+BvB;Nc}l*cM;h`saV{ZmBJk
zzdAmyx_o(Cp}fvoZ^7RoKN4e?YKJ~q|8z=JK;e<Pkl0+-pv`Bl>E8>Hn05Bx&BE{T
zPLgU23mBLs4*ZuiaK7cGuuUvO;-E!<;j=8CotHvqa_!PKdV6RATWIsl3u~SkFXv3m
z4ojALz`#?sadq$Epc9AgepzGA(ZS%oX|cegnw$U`=_$E3X0s=(<S}1T;o*{@tkJrl
zAUJqMaK*1=ttFnh3c?DVofEDf(wcNU>x{62@O+LnacjH6C4{bt#eP-bbkxX6UVQD}
zg=J>jcE0V+)o$iyu{h3fEwHa*&SOT)CFc8nZggq*A|TE(^T;8=cN5xH2lkdt+2Qu6
zttjDtRm_1lm6OLz&nZj@xtr%?zN~KvUukAnu*T&QHQz2yulQM$81&L+t2|jLW-NSi
ziG=HHGmrN29X*j}GLD~?%3Bk(P1EDF)S)gHPL*J$74E#&tPAewyjRVPJQ*5yXzC5l
zZnNjfSJ$5K5DByloV{b`v(-_qIx&$?=P-Mo@_Cdb_R+>W>f$Z=tvoO6ZT|W9GyREY
z{A}q^_pW$;wlX7=dV=@?ktvNg9!*Q!_f@xO=IWU4!0=t)%Joj?Y%rd^^Ti&UVEwBd
zY4P&8zWW?BE;S0gSu)=^Ggn1TplxkHvF6*3=LueN({4R$T^+(9#4<ts<%J10inmkU
zYb-Xkx9GHU|J3|HpDR?r{q8x1bDd8eayGKQ-zd<1GdfDVe6sP<%B!oko}71iQ`weZ
z9iny-KB3La7Kv;uJ-ha@)tQ@@rfoCZ>dTNPa6sm+nY;Lu1?%glCMW!jIxaNh!TM%j
zorK?A?&cE~@>rj~-J!z0=;XJ+{5HmzF29r$=ah=vTrUOoo1ENnX;#^!i9Iq|ite*7
zGIb`*5x()%f&G;KtRTh3kC)6iClY<Fw8?KrC)3BU&=t$iom+Xb^3^+@r@LE4HY;xC
zjC0XTz49t$xi!<xVCxx{0v~+xm7|K|^X!^L^vd3Me+vwatbHc<I<&avY4U5|Z9+on
zSKsdwj%e6oaUfo=<UtOz=0S&;{{{(+GZGZo{R-?kU-;L(+B}b+PvN%O9<!4f7xg66
zR<wOA3Sg~@Sjgqu)1m%ihEVi}BVu<l+;&cwB{1p3Q9Zjvj{mNYZ^n2gu~ol3=wcMC
zUDh%+g1IPPXn|0dNPB?S-iJ9Y#Tt)gW}Y<rqpCgWjNFSWd8x;QJN?a%ZdSkd-qtpF
zg73aPt{YxG%S_|Tc0ZE2cgv$wt5VvO)<>isv$E%2moiyf`nA{cJ-d7r>h+wtm$_u<
zXu2F|bA6NecUQ*2dlNTK=v&OatHEr`$L%?Z!46C!0u3xI3cA(YU**-MJ%oBsxF|YJ
zOb>s%=(tXsTf25)mS*Rz$lJFjZaMdLZT8vh|F@U$?bye5Ni%S{n*Q9Co<6H`H#Wvf
zD{XgNRVBr-#d_27r)iN4+d@3H&h*x}_w|z1wS7<dT0K`h=i3-KOUPkT`|WvqW<J^N
zrTpIPsAvAf4L|p++cfXWlccGUM_J~+RbIa0j^;vZd++@prl>ctDm`EpzM;UL=YND(
zr=jh8{`@Z`(zm@1ni+Ml8J$pG5i{vQrawCa10w?yL%_bGDeQ`j;*;jJ@!v{p`RBUu
zIp>+UbI$AX9KXZ}FTKc`xg@D9^Ge*eUm97zOoNs&pLuC2cXq0map}}+Au3jRp8fJ-
z!HhRT6dl+OIh18>wMpFG7UIUYU|sO+#ixQ!<<y@LY_E#BdGWvBbftxg>X(G)=lQ;`
zvM=21aK}bYCn+?m^?Lt9*GJE`OggWSc*fl0)QJT~iww`p++g5*5!Ix(=Ir@Ze~jZ;
zmE8Bvic{b-=U5b|pRiZUtbbD9Von?T>zfj-oMt^USuT8_=-&J}-w&`md}#mwQl+CR
zzgFeNy(#^PPv%P%2!?!3kG#9+_z|f>zpP1pdBrO3NkxTRg=<yz^Plut7Pc%_{#))=
zJL%htR&L+fc|dK^+sAh!Zco2{PIhqu+ua8nrIjwYYUD6TJDUg^J3GI1U@LnRx`k2h
z$y31(pBrM1#${gfalRb)qS=oBMp#O!{o5-FoY&)5Ul-&{XH!dP5@9bCnV#@p`$iM{
zwkm$HW)ZhUIrYFMMc-7%qPb!W>erifzc*`7FLJPGv0qViNWI20K)lqj*z9^s^}>ML
zNfO#l687y$VKb6GZVEKm)@u08G3G~WoOoMkd2-^3x^0aHk=wQ6ugghIG}L93_7as<
zDNcE1CR5_!92nMq+@<{bwv-(kV_aM6L(;g5kGnRS3otEm%~NBPW(X5K5qE!zYx0&f
zA4ieWw8{e3rVhjKtY+2nuWl8g87*e+ZP$bhA_`Y{6!yEhugPFzFbSXLW_F>edbSut
z^Y!W-KdN_F)VQDM-gl$>zzhaY1=c#1tf{4u=co6~I;u2#YvkqcJtm*B)-BCu`dI(}
zYkT%?w_e7}a&uIyPMhVt3Xfh9%(g<ruhm12$1r#E#@xHhtgAM8-}}~n_=?a$VH@Sk
zQh(f~SEV#=UuZKqBmbY6*lSb2N#dQ)F588F%uiMndvj%i0*9`MgPF(%`;<#n9LvJf
z91`>znpn&V#ViXIUN)`h$oTW5>;K}y?8AjQ!jel5C2~z*(T^;$(O|I2=-#Fw;M7s%
z#=+p$!C-xJvi;5?PKGJE;!_T#w>WF|c+E&ODc3dqp6I`$xLQ!JJFxZHB0Z<$%C6g|
zN~`F59Cu9cl=2b}%2_Vu5L7z(acQhVY4*x#LE;81i={I=>P$inVz;;LF%)pPW-K)^
zEaY1If3I)OQEf)~>8a-$`&(|xcu$StGIR+DwSMKn#jGUszoPf-myWH0GpEn!OcoB$
zK9atWqpHEIYt@$mHbw{W$ijA2ftIIL-K^C$qSZb^-P@o0%}<-WTBv8&&+a`tXYb>z
zVOqqpXj;wWqcvyBBZE_v8kB3VlzUubw_H3?;8NzyrK)wi6YK7`2l+8YzdxRAcr=Gi
zaQY5YwSBI0H!6qZm#4m1Y2@bG&*WeuxX^Izg^=S11w<NSH$0i~^r`DE#m2V>MHy!K
zMNOO#5Loa%HU5K(+25;-Q>JxE?3}oUb6)hKrWMzlL{BwIR24FrG|Q?>eL9k;tUh^1
z<z!vOGTA^wPfdaUYFZ4g5mQ`lEYi1X(b_&)yM2nsjwwDf66Kz?)O~d5`53^TniTFK
z8F_N5q^PtMOPiNe@+w9BB^RfuxU|h{p3WP*RIjykr{mJCj493P0)h=JLI+~EPHZpP
zVN@nDV@6WD<J1}X+r1kvm2Fp)O;;D#_Q1eeFnvmg^#2p(u~8GEOr)(HD+QA(g(kRF
zO*1Qh+Evh^+UVA`e0liN%!$i6np;{UCZ@S>a?hOdEdGzGE^GPhIc}3JIy;u!S{{+S
zs>xj1_VVn5?F{ZOR_)<f&7iP)lh%}dlA^~fB7+s?oSkmIXH`w}#2%GHk!Nqrxw_n=
zJy__Xq`|ca*IPf<JZPWm-(>ke>yz5$EUU*m>+W-S<rYb=TO$4HhR|CF@5iUq-)Q9K
z-0FL?GWVBx|1_f+_ZY1|p3n%I5H@3{(A}x)WgpHjx@h-xsqnoojpf&73RGsMUtS;>
z)wISkLFm@nXrW2!GZHj97K(;X%3(JDzu8RwIO`oJaZ8OwW|@;kofk#8Y*NdrQ19;0
zw60LkUgYp>vNrqX&0bSXwWhe8oU9kM*-Cq}T_=Nw0Arv6W1vHi_lYUeWm~MPii14$
z_(TK4JM|_imb^A9k!TH^e=yJ@NZ%uC+Iq)n_D%+AH<u*+UXt^3TE6By?&hVL-qN<q
zmg+L5aA&9s8PwMsEvx@2V{0<QrPM_(Ip+WUZ!@?S`W#y<l=Nl!EoQMNF1A09%`7<D
zG2?jW3XZNt<~x5!By7D{wff~OHnWJ{t2>Kb`lhON3kR%74(#6LzG!pfuCv>AEt8U8
z^K15o+q2ibnjE!&HQ9h`%?8e7hn`(lO8cYc9N1m6Jk9)+q{aCgb1t4xKKW}NhoaC~
zDfwsOdv{KpW*a0m{qwZaou!)#)pm-u-PKmhi|&0^nOb^V=!3<KU*-a=iT$ru?_=Sx
z{u#ml!&BqytNeeR>z{;*GHJzSg;wUfZn&(}#Bq9qtfc&^k|uGjNjzc>d5^n{cWjiE
zUSw^tXpz;bRkK&Ee6?!p#zkxg4@&14iX2{KXmv=%=aAh@hX0ELK*#O9Il#5}fTBzG
z7NawZy;c{Sb2zGeo8xS?#Pd+nbV2Dwf>N@LK`|#BS8ts5>CuwRlbl{TQrR7CT!l+@
zpUy4GbmiV5q*1b+%~7_dql`Il*}ca0sBbB$tDHTKIe*yY+~S$K^#a$KlKfxOma|s5
z&52ymADwykRr*4US!*I^$ttd7+%~H-tZ?eBjEURAH;J47TdKF{_sZq3Gi4X<T6}BQ
z(yCQl8!NUno}9Mk<jP;$LYaFu-CDKY>eL?h)w?oJEjhq<bm5`34qUsXv-V3Bi*2ge
zq_ak8Qr3}EO4>zx&ULQ2xO2~zZ+ot+?7iRKyX#`_5@+e@o6j6|%Sqj&^-ALG{}*1l
zmvhQrN#?dHh2Ct-&1%=Ek9OX?WZmpZu_3I*rGmoSjnYd#uHSL}9A{Sbt4#&sJ0}W<
zORtDrC{TVN?LvWeMpMy~^LY=>^VuX$Ow@f5D3*F~`l4S&p~5WNW-vG=teP|@SmBPK
zbLXn5Ii{kMPpW2J>{&it#Cfw#^=6g1n=Pz28?NSDv*48OnM=E+50?jSasNHVucP<@
zlU~@%ErLs@+W3~(@>+6!och7&h~iSi6^YwEFirE6ndW1&E&S6~yKCEG#jQOiFxWbC
zxw*;IEebhWx;=aORoTLnyK81NY0juU?VWK)sA;#)r>VjjM|NCxGLznr&Lk$dNU}>R
zQd&eh;s2WBv(~ytY<RhoVarO{3oEx;iY@4xz47wN?%GwGyG&Pe-JG=c<SxgX^L0+G
zpM7&dO-qnL>Z&(4w>AhJktp76b^2_%;(qNtwZVJNWvsanp>lounnN~*m!d2^ilUwz
zoO{!H?aW}Sd$0GdJ$&(t`M$5^^}Cpj<qq#VR^q~QQL1{=wcxFsTpP|!|9<Y{pNX0)
zuHW=+;&R+5x$8jaAv5N^ljOCEEIUl~%?~d6b-`i7D)$qMDkDwb+TM3Q=_ThQ_uW=N
z^zkAGhRw1%3_sr{?+rY};`hK{Hp`j|d~43|tv&Ey%>hMk&CAAfwpgVf=1q}MI;P~X
z>+sZxQ=P=GtT3IssaGJnL-N1p+#^1BuEbRyN%Jh#N}0Z8Ve-S*a-0Xe1Pwx6ri5%|
zJeKR_e4;nymFBU@haSHw>wE1f-C-%6GJ#?7Zbq(Ml{qOH3(BweMRnDjEvV_um}xz!
zw{7JNE_b!a>UP8Kb4;ry``nq5vr0euWcAF8Oc#qd8(4w_SfdQKgzmjK`^+<iw-+VW
zKFjaBIXm{$5}Vb{d8ah)9Mjr!%3w2RkU>wG;Attb)2nJvA3Z&1?b*|vD!uEMtvTbJ
z{nqgIb;-5Q+3QZ(yt=90clylQlV^6Z39j4u@YPF+{<@5H?*BB_KA7<)d*8K-XJ;81
zfAi!Lc3@kbKC6DttedivIAa$6(P>tbyeD!j>Az5_Zo$Fn{BJLqbS~O?@W#9xfwpw@
zw)W@Dh4(enFY4VsXd}tWY{)iCfno82cY+u0SFc<2(C&Rg_&fV$(`95fyID@L)ZVPl
z+oIIV673-K_?8Bz2UoO1k*7v6kNss6nbz`&#X6sK{a<f+xk=gn@8P)*FQ2)!=TY?S
zsM%MP)+Z_5=giIgw9Bn+hv9SsIeGD7Lu*wAb3K7%1%}Vv^^e4ky*id6s(9>)wb66$
zeTI|!V{<ePZuYk8lMYV0ek|_1(#EIty*uYfXKs{ymT{Red0u8`?7@tCUqxz9%Im&h
z+Q7MZ0c-RE&czEjD~-5jFJN7=t778a=fc9fOZZ<j^)akl^6kI0(H74S?+%#vtcbm}
z^6u*6FBR*8U#@!_xyp3zWzBtk@4r4|SG%OO*M8C7sU~W-E$3}y-M5p`dqRKx^%I|8
z?@pO$6msd~ozLDK+}-E7?lhe~=PzTG*q_?0Fzdo0jmfr`o3HyF+<E@%(mBE6lNU|<
zn|x#6v)p-8zWCn%-z)ty_MMF;YwCK=)d~z-3<NUYe+y!GuO#?j=fyv4j)zR59x%jD
zIZ=P<!1p3<`G>|nmqp~JwlF$&ZBz<BsqAv<;g|c$J?Hl*#P#a1sW3RU35z!|^9nsN
z+MsYyZ;ljK!3ow!&WubdWla%*kKOv1yh^{U5KKNgQNeefivokgQ5GrXoSdZ-|BIfO
zX^=g$hx7B2V{@2_wyiQ1&Tw7elpn_PYVvZ1Ag}Uwb0UKqu1wBVnQ}8?(IU=XHLo2S
z%wlV1Z%92cNjG>I%k-_P%L-;3a6H(ulT(Cchql6j;C;2_+y1=%u9DE#!kg`LGU##Q
zQ7J~38(UPpe9L6F=2Y!8+{kcfwt03O&m@D4shkWm+zO&hv#zd;+|TyF>RZ;W4Z<ut
zbQL>URD-7Y{Lp3nA@b0x!+(NN>Zxf@PR}+DKbP87^6K*1@Z)v0yN<q%+2*10IhH}P
zTl8qSN`Ksks&7KtyY&CZDc5|HdCC2vNxojP^XK2!|JQRH|Cp38dGf7?n(8c_Oqq_$
zEu0M0*#FDE$y8>S+;d?iS8>*1^;I_n798$!S<|tKJ$ch{F~+{Eqtlk1$Z%Y1<TK&p
z8lzht+Q*Gf+?d#`TjRoK$vR2mlNsYA2AzhzEF4KEmzzniwL9mO-LrR!S@g*z)3&iJ
zI>oZ<3%AdNq{&G=8g2{Q#riT9wolPGF=5G~^JYSu&-K|X4q~v;j^b>%(eOc-!NFfT
z<bngYu1SFN|1ZLgj{9w{oYxmt2;$^ivg6Aoh1S_hC;aT^GK2&=^OiRIZ~qZu6&yPE
z-SL+@9vo0=N|~y$Fp7Oshw#rU!TNjm9nkhy_%70v5tgJp%R?#UV>7>9toA8?uL)j@
z%5GVQI`yoND!LW6JokU(pJ|Q@lB7;8&p$1@J({h&Yo%JLztP0ST+^MprgA-Q&1~9|
zqmr%r?^;tgGv|v%-b}kcI7R4Zil^;JY+pPjc*}AIW(RKHEfTH_Wqp@4G`hD>xN+J>
zaax(#WxsX0sW-#&)}~I0U*_A$Qt>(0;^8FicNTLd2A>NFslTRYy|7trSJ=laS^dm7
z*)%J=<+JO;_Q%fE>s-(9_g2ciW!vMz;|@k;vc@wyJeewDG%Ki!ccXUD#iP926qv8+
z`ndLOSBk!+ov%FmOp&JolfY&JhutiTRNYOv_MFoB8nfp7QQ_#CkD9WNFB_hGuA*yp
zOkH<R!MOu}f()!Budg|yG2xPjZ}0!;i7DN_cHVmy-S9spRD3Hm$fW#k`E%Ke9)D|0
zzB4o}2x0U#+*=v!?<u>r@|pMCLm@Bkcg?Gk7hEK&_;!1L{<XNtVj8x7C%R|bB%I_g
ztBEa~oAI|@rfI+8&#7lu*!8Ru>s0<T``ntvMe}!cUbDJpD9pJ*{c{#qfY1rUc?JcI
z>VgWs_d*4?JY)$rKHSal>5+<h?vm!1?uZ1dLo-fJEl^gmd3epE?a;)hoc=))58Sdg
zZVa5mnNo98_~jZFp%V*z%<M{Lo@_Z`G-IJp5Cg}7th8w?norJjnmwK=V)kLqbB*Ix
zD<<)Yb#HThA8=ghkzxF$JH>JWdT;E4Z+mRsCbstf&j;xa47d2VT-&U`7PZ3BKa;)l
zUBrBz&?T}_zaIxF?GW7jbBU;u(5VisB>ruJN5$SwXn3<FL{KG^g|or2>FucwUB9q(
z1~zl<o=7_GG@)lxYpBqHH%Zf-SyUKPf+SeA&5rejow<~h;&Et(*Ur=aT{<mRKAww3
zo?1Wf?3&T-q-H3a>XSB)&2zV7+%k);douklS6*$7Td_n~i!q>IL8jvB*E`L7qbkp5
zemHpQQq5Ms-yMZEZ>*<nlvp&O?ab#n7mED17#un__rdxom6P%A|0*YOJh?Pg*7||3
zNR&urVNWAZs;lf(R=qiMR`l_In&roHLon=yl9qrKCy%-1rC$>N&!xOkl)N5fwJ|uV
zh==LIp%AlUT{ANrx$~T!CM?^cD$Lb3wMHxHRFhT6Lb0tKyMOi;&RcOs!)&Xi-Q%Q?
z_G?E!Dey+5tGrx)ro!K}Z0oArCmoVMINdXCw<Olu1cbfZDkw0qS5)Pdz)hPJkCg_S
zlHcA4^YP4_ba}y%$xB`?N!X-f>Cfg>*5~CQJaL2INx^TPHKAv`c)nYnQ|y~wD|q?Q
zB!jlVoQFxX1RB(0+}SiUf+ooZKGkb1*m3#Rxxzip&kV}N_uOB~wr%&~DJDx-?(vKK
zTzS3a-WMNMzXC3Xl>3?2zD?=A+SkKh^Im0s@p>!m4-Zz&dp-NP&F%xO-)&tY|BGxC
zb2z%%p7GB5J<nxd9G)@5aBhiioOtZSkPF8SFvXR`WuMk?-fHI2r&6FLrFe8(&B2MY
zZ#XT<a^1kq$?Cmq&b#TFUsO55XI*CB9>U<U;f98rV;8Tx+Vp<WME}2^f{zPaJ>tgu
zNU*-6;SO87MKA}SNb;F0%gxlg!>43j+;>@4u0OChqsGf>_OY0Qe}2qUd&ysafb01G
z6B|#~N;=y<(4N7Wx#U#c6rp;@Sqd{YIIT09sNHd>&BJWM>NEd7Kjt+r^7Ll9wq=)(
z!6PZrRzsI9a%(N_CY5>|+PltR!@`H3tDJtGp1Q?|DNrMW>B}3*<BtrY_s{v@_v*?D
zciya-J%|3Ayq9@8D_t&%DKO*FQWgP8%{w2`cG;Gi*6)7he=(QqE{~|i&X=q5_jy&n
zmwY~{_l(~cUEdRlI$^GlPR3Rp3%()M8Fghw=)Jc_zm@o3w%C16kIQ2Jvgh&IsD=7#
zoLihGEPAgb{wiWi*qj#b#llN8UQM_p{Ku|UL@xAuaE@Rwi+R<O%@Z5c-Yvau@m=+j
zfZjtd>%%*=3x8i!Wd5Sg?EA1~y2R0Kk-SV?TCH~_wqG&0a!4#m>T*!}o-1PMekbzF
zWR_iuNVkplJGJ$O@5>D~$8X%+7`lJexw`9ndX1*<WH@!;*y@)t%K0Kbtkz9ov+H@A
z3>7a<H(gZ1sPJO#+5~3J|1L*P@N7-yu4*ghTCcKm8b@*A6tAUkx%LSkzt&gC;Bm-z
zVvywgDP`wtP6oYbxEZlodRcM&t?oPiMj2P)PH$R%^v(UdJiD&e=}nE-`Y}f@_xA07
zok2T$om3Kcu%2>1_G#VWKTl2PSDp#J_xOzC)@3!^ce1&zSLHkEec!V^=4lp7?fUk-
zH&2y|6SR+n9m(sMV9aFST@hY){*~T`&zb>W#P;8MeD}mZzeTg1f1eV2u_-@Zf@>15
ztm;IWyPiR}j*9CtODujZSKywLxL{H92V;&KP7T}Rl}vBvt>@qT;UK$Q;vth?uPbM%
z{4L*AAM%Z}!ytMiheK0mchd~6ow^dtEG_?~S$X<dd>l?mH}^O@aJFbZwmQJka-dt-
z@<qWSO(j15627MmeNPSaWU_Zo&lc4Wax^HCD!bJguf2cy#jTQaq<#f#(%W%RT1ESt
z#stSztXq_<w3-zr%$lz4Y&`kGVu3)XSHIUgp5l!0Qk`%@MJ{TF43o~j2t`ec*<lA(
z>#b?^4KO)+c>j;)HTxfF>V2Ga<<)M}Jx)g$gwIzvoBvsAd2*81;w{!Tz23W)q)$@Q
zJ+LFNc&_0M7DtgI4qNs(-Pv<?wavNcBd4QXCUo*}_9*$j)D7EwB<%N;yeQGA=BeR^
z>#m+QjhQVK_n1SxL_Pi>m$LM<77gZ#DA&Z-{3(Ik(z^d|ORqMI?cDaK!>r+w>*LMa
z>R;{B{JEj4qH}tHHB+}@3eWUZ8-?W-;(a^zvE33_6tQAuhTGB?`_?fUh)(R>x@yA5
zZpW=oO=?Q547X>5DKPKT-es4(Qc&u!gLRMb6OI#}y;VGmCTq@NQE;$N>Fe@vFuKw5
zfx)AZ$76@)iAI+L%`z^{B?nqnTv$sSI;MC$t~$_Te1Lb)4$UOK<yEE=Ud|L^nI$|&
zORsC^dy84?l{OW~Y$`isIi<vN-OVP2BS!ltoYak;G3&}P6`Mo7E7o_dVCOiv^vDjC
z^EY>E)R@>O>a;a)zVd;4$r{-UiHjPxh#Sli%I!XM(CbvC%9?#iYi<Vo=Q;9e&XgeM
zz6_JnCBml<&OH{r_QnloFU=L+nM$VBE2k?p2d+51dyDsO9uLL`Yxc<;*{|YqK*Yy|
z$LElb%Q<Tw*CjU3W+*+C-0S^k?ZZ!M`Bv+mPMSK?m@}wH@5b+W213(*R80GDu_NRN
z|AT7&d6z|#vgbcYHhW~O->_=i^*v_sOD6rh*sCnap!q|dvBIwE;jvcFE>T7CzMq_n
zU(d?^qo47_&g3)e79p#tA=+wLi^_a9v~us5CS@^i)rM0W-7kOGbxUK;-3jNI-*E6b
z_xIe~(8GD+tBl9DlJnnM{F{6HTUbui$DClB;_>Imd5#hX-535{Qyh9!`g8-d1Y{0Q
z(E0B;vBp$n%~ZMK<$V`y4+<Dbtm)tg)m{^H^k;yPUVzcu*Zh+=T$rmdfgx#nMKZr~
z&iZAyl{GhtukF<C6%fqp<lJ)cBzL3J<Qa$Fnpmvc=+rV>=b-Wq*{|9jn++Tm&f)g%
z+?%9(HezXW^d6UkhmSu#thVUV8EgM#2ZA<da_Oda?Y?qp^T(h)D)M{RcsuG`ZjfO$
z_Px9(B-qL4@<E#;7N$omeb1abyr(-_j<LZfu+TK(^$A~=<@sKFPumzt#?0z<a9t|t
zdp-FIS8Pb9hq|A_*`z?<2p!*)ZvLn%SE5t4{l2(dZ-Va=?iEv|+`PQ@7vG+c5i>D#
zs$$j?&cz9#4^940e|W`Ba#3h$vU%piW6xz&f;v=Quv|+B<*8J;rVz?m{l@RZnYABv
zOifpJd=X*DxVvH5jB6RD=QC58GxuK0RlWSZ=KRMS-oHXTnpDDnrkp4iz24Syq9exV
z6tkt~Ey+0-JsxWv<kfKO-M}|p+H;}d$vMFx@q*XZI!dXmUd3A~HScoA##h}MH=2~T
zY-*S)_V1Esp`!SPo4$@xZf4Fh2PEdI+FC1?+MjBa*>AwR!}|0ZmD6TAhApS}Z@4nq
zoMrN{)3UCUFZnm04(h&~arV;OyFv55-kisJYyR6y`liymb)pQ<o;FIoyfpNdv2E~v
z9iM|HM-D6rK9UoD%ty{vb?=Ea|L46lT=!^|&sF2)H*a&s73<yjJhky|$5%)BH5sm#
z=7dBt>VJQ@cUS7!*eQ!MKfB(v*>}^V^R@`Xv&l`y(OjaM2E38y?j6}z%IVkIr9a_=
z{=A;;n!oM#2MC<7HR!0_yggaMUVEjXjN#P)!-YcUcjR3Av_|2J&hebs*qqY4MJcTj
zO073iTW?N{y|wl3t<>0{8+Y%WJ*{1PK2O%;;n%PqI`@9tc(Cp9XG@9e$_eN7;p=|w
zq4nxSwXY@X)o|Hp#}5gPKManZ$q8|t8!&+*P~cR+rl$e*cRMl!SGQ_TFq>dh^q5=W
zO^5uQC7hNERtRjK#WB0L<6^0D^Q;#FY)%4uKK<|1xM8CCbLH8o(^%`cR*P(0<9(^}
z%rpa&g<D;MoK~*A%60yZ)4a!*ICed`Bxoo7{&ghRCp(d_M4hDyn;za3i+jj*>ynV#
zEd$diX}73zrxKUd1~0D-UeOwScuBBB%$Z|XE@xi4_1rTl$ICKr)x7n*%L|;=N%0&N
z<B1PG@<=-_q=Vz4=T^ONn{DrflGBWRjqLPeTg>*G-kc`k*75VIu0hN_nL`$N8sVma
zk$WSuW3)T91pi0w|FvU<Sa0Zz1?C;Dk<N7wgsmkzla<o0?wEdw<CC%fvVgl@+rsYo
zp1(IY_TJqmD=y#lyZ7YQ-;_YM)Szp3Bcxh$dSgTVQZuhcdiDLkw)+kXpYgpYx$tjO
z{A0wzn?kO$#$0E$(W~se9=<tzg66Xq8{?**zE``qZ?cP2L##wbn_qqI$uFgzUsI(t
zE4H7KwNiZZtWJ9K#>NFqS<EXLX6NU%G9TV`xoZDDj%8~ecN}s&HRbJ6=Vi|YF3mZj
zZBrF<+V}L_RINGNT9)l7x&GPZ*21eddl;jnzTK24GiTs7mdMjCOtW$_dbp7F^5Ux*
zOP;=1^7TbuSd>EFtrcsdj?R&Oc;T|@z2Ijv<qD+qqCR^B?AdCvO5Vj)`F*U>2Uq^D
z2e~e;k@uK-C4C|P)gJjfi=%yPx81mNdnsS&)W4y<9{ZnbZ{Xs5QW<zham|WQ-v53x
zUvRE`=DXIjt1S6YVQ|;|DdOp_DQZcElbjU)2TNYM%6nNiqffzpSKpJ}ZK<vI?%v3a
z4cvEkzgcQfMys#fn_Id$2i4vjG0QnB_GaaY*xR|We5bEv%+2|_=5*lh9FwCtMo-^7
z{QBmxZusArd-W-Cm-wDB_Qt)Ka4-C|+BG-P&j}Y^Kj2$@(^S_k<aXGVUWs`3uiUL~
zJ3iYk=QWMk^p@k>)M*~Kn)<r#Tff`R5qNRAhTj^yiTsgQS7e7W{_y0^Y@MNHBk*d{
zRD(+)T{1T_w;W-b$vH2fcgCTM2X=k4yH>Yt%a3IyECLysn}S65y;%_Y;?KGKxw^OX
zQrEYoB`z!Y|B$IYk!gBH;ku5_z89TwAC@*3u$&K)diMVBzK7iG!94!A7Jto<efN+*
zKiFbvu*;UqT@EifJ(8y8&9kdbn!$I)Pi485pQrY+oqYzbTwg8keY0H9cIDHG<(GPn
zzWJpW)3VJs_pEY$))VRcgA&c<#tCcG0%LNQ%wRAt{w@ESyDMG9@As=|9H)-;N~Pqu
zyf)Ih*}70f_lNndz!WFX2GvPTOm8f9z2V4stIx3dwaC2BpVp*=K6`TD+misck|4V`
zj_z*`nw1>0D{=b%*>75GSX!+2wQF8tmm~jWT)6hM(DZt&TR2<Ib$*#NMvb%;aToHG
zO4)QhS5>7szva$+*ZoQNzw*|vtFr1mS1v4V^V}abVbeSgC(g=<SveBA%(J(z>g)>R
zD$ZkGnBDp&@2dWt2(2lTZ*JVKab<IoZ11^?JiV%O?N=WEaj)#UK!*QhrrDb=F`U<u
zJXYMjEu+}JfVDo6ZTXAx^6wSx-z&=tSlu5kP<^O>^h58z7j^6}`O+^t{&^@r?bhKX
zd)ByqYzhD9t)lnp<`MrX9gd-2=SJ8rN$XD$mQT5TCg$MIW`%7nKla|gz4u$>*^I-k
z%k=C5TQ~pfY?vvya)n%1L8aY4C-H43uYOu{|Lcw$Yt#6((gjvJtPX2<vPSEwK=9{w
zYhLr~4nMD)w$l^bGO=rB#9gcB9lXpNo~Qih^3ikWS`?;}s((NB%?9_Jjo~Hp19CQt
zcinXTz0tjTv%2_J{Wn|9Q$wD8c3=M{XYZT)SF0nR{XXXw9KG(TkAK`HwLjrPmRIK8
zdllN()zQZq8j$u*NlrOW>U{2<b4jxI-^x7a2(Rbj*=yPDp&r4$n(t}N{tFC`g>p0x
zdp>@{tsGNZC74inF5b3FL-bqao3;u1SFAmyH|j}xspJ}8xm9LU6XZGblJ`lQb3T66
z?S7^51<&gfIs5;;*#3Ryit<@S<!|P5>CcYUzg;JA@!yB~@9fj-TK>KNd_1x4{D-g4
zKh%GJDDeIC@~M~ivplqked#}OD=X_2jd`LI7&-K0SU4X1clwav*uu^$l%mn#<J``n
zD70k7g+)h=SS&MDZWKOrYmT+(R5_WdBHlb_*NPO)1(UsJuq75%-CTP5^qjdxLU$?+
zIvH4)7&KHkLKe0xnjN)OIl}wE<wZfFZnhDtuR5*?5Zrc2%R&2ESm^$=&=Xl&w>AZy
zpClUKzc9?m;nJnG(Ni=USSr82dVAVDUr4iYY0gBWG-1gjOgC=1R$R+|yeR$rHb;*x
zQBAYy#*>8(-28m-%z~RepVmlz&OA3GIycX`%k<6l*^%p~$^PE<_Rfy-)9ZSFZhv)V
zaR9%*ZP%2-Cro|n`TJxnC7)efp5N5>!)m5(=XT|5_iA^4fAipY_WhtAd;U++3{qP%
zeN*FxhQ?@?GvWqE?f(4^-tp%B`TzUt|K%P$+Q6^MGSOw?)vtohLA6U3II*c}Bsz&i
zMJ#lRozf`WlIgkdsAE*lAvf!7HLNp3d=-}@wX<)$q2#!B&Q-~%siIe}CdFU<G|g5`
zDaAXzSkYwq1SXNn044{Oh9i?$6djgK^i*OvFxg+NGF6B(Wo8RYP|Xh3q{R~sO6D-#
zsF;wJvp}<9R<C19U{vlQi{~Llt18uJS3cUgd|vIRmCqvUe_3kGW7&FBa?t`st7YlQ
zR$6MBeQsPYOOk@JGMCOw%6cg=r%38q;=(+oSBVL;cFkiDUSid{N_fMsRjW51veIS`
zzVd(8eaFwLHw2=DET;&qKX@%P*q5WwiEU#(R{)!*=%gz;t;!o)E_rFs4Oz=@Bu!h_
z*X-?=wSEulq!}i$xHbp`x~digh;ZaCWR+ffL+D!YXQhz!q0u*`7l-tCn!b<fy=o-c
zlU(_ATF4R^R>inF%QNCk5nRun?!L|IzBsbvTC382*-fQ4!xb)z6{WLmE}B!YB&Mw>
zYs=Sfn@$^^H!F@XIXovicjMDL(I@(I%Rc4n7`xvpTB7J)QoO`<`<+XPX|GJKTq!g+
zEqfR`_pZcK!Irqh@1?%+RqVcn^0mKi9>0I{*Fsr^Tg>mm18NVwn-$>7UfXifQS#>A
z?d~=Duglj5{rw-+*3xphcVU~7<T^z^nJk^DV$l<Vx^2UIzgEQRUwSE;^6%M=b2I(+
zwr=Ql5L3P4=wi51MJUXO^Twnow*yb7w|lOVj7;Bn!%I5n<_k;5e~<hGj%99aojU6T
z`~8J;v!zyjTzFCV{PWM{o0u}@t%=!x<J+zIWq(uecGpLJyZb%!)|XDVT^X5^&2Ig>
zG}+E-|DR9EQT0pbx-I{&vF&^P+!Y5xR{1t`+9qd*t_r)lD)Qv1&~JqgM9d208+|r}
z9S{<WtrHdOeW@GG`%0Q)`qec#OIcL4G(COXk|xhwGNJXsCr<w)jfPf6R)*)lIHQ=9
zMGZ=iifQeN32e}qd0KLfZb9S!o`^{^#QHp+YOb>sbbMntH7w<&KL2a($wCT^2j{q3
zi#PLXv$aiK-qI>+WVKjw`f@wh#<q%_BXgP#If<P+d(mj&Pl+tE$+K+#<)rPq*n74@
z*~5Wvl6gJLIl1;@bDmqIPwWfnD^cQ{>*3|=b7SX{ihYYu`XrsTNpL=2YvwRNux(S$
zjv#3*#wA{if*nq!&L{4E5}V2YPq!tw^1@@iCtf^-iG|X?mOheUcW?1<WQe+ONFpwn
zUw)oPam3Bol!zskO+LoDdnXIVZaKtuUv#~wO^`=$lp>?r#9;5JS3%QRI9BGKn%Gl(
zBryDu<aw)Wt%l!Pyd$laoj;ZEbk>?5v4Q`c$|s!SSQYTR`Q9upuNAHDGOKw*U(9u3
zzW8!yVXs(d$R8C`**~#b!7_92FPpb*f8a!u1{ReEVgf6I*fxbUtt(o&HYsa<w4zIM
zk+sK0zU>D&JXM!VpA_5Fl%>$iA-Y1ba$3uTDBaA&?3KPxr*Xw=tXo-@6wBnn@G4KE
zfn!0Q2Ft{TQ00ae4j09|cfTg}JY(HZW4Sc>x73V~BTS0&g{<oi2+v5~;$*d|qSGMC
zqvzDAmhd{;r5iFultbc@kC`;JZA@Dtm|b*b;Um+w&X|+xc9!a!&f29JDsW2`{L?VL
zcxg>~i;Zrc$7|`!yLd9RmHpCt%(#~5F3&iYcq>P@^?&rz$W4>Pe&=2>=sfvBz}))t
zPpP7Z$_*QGdvh|Y^XeD;mfvo=#O~%sRsYu3?cq`#ySMz&dyqBjU~7p>b8E0n73%?u
zuaAoS1T~f!Ufk3gxMiWN&%)JqpRIiuLyXs%c1@g<lM=t8CRlOX3U}Q%rw!#jC(d#V
zDQJ4UGGW=7;CTT(g3ohWrXK&&ryv_A7d7L@L;mmDZnq~3E;u$_+0u6X(aIvj?z!KS
zE^e7|toD=K>@(?m-+#2I(R{b@yTdkhDdC^JIVtiP50*dqeldTs-@6rmO_pl@5BzDB
z;MJqf*|dtaD6&zo=BilSSBJkI6YRDNb*y8#E;0L+h{~P2dFB02%`N^bcPlEJaB}+J
zm^AA|N~_sHPM-r$EhPL>yTj)`blerC$2r9`{Nq(8L;I8&flET;J}o-s`0Z-g@)KLN
zrXNnTdurvj>(1A>ydx7Cq-=bwUInkSYVumXy(!|zJj05u({xtyI~fSPpWaz?XZbqE
zXR}Ve*w|mm^Y(PZ<ftn~yC<Bji_GfiwOzdN<;30=m2I<y=jP;FFS+`1ZNc)eJHA=Z
zC%0IZYxY~8diE?m_bp$*tq{uvOm8nKa2QQ+evqP2dw<ik-@d0?D@qmTEGggH`MZ3&
zl=H^xB}-*F>ntK$I&Mn4Ue@vHS<$0)IWZ)C>3+?ki^pZ(e3)o|XP@%)4Kms`{&kVt
z{<~^CsyHS#`{U71#YVR-fBmGE70`Ul^O#k-#zGaX5*t>o7fn`1mvm39aNU_->sw~E
zbiutR6M||_G3>FLX1g+oRc%9~PVLL-*`?a+Q>QN%*7ESWBsX#Md0ofjRljH7lhavZ
zb-Lxsq-$GuJP27<rg$Ob@9*s*u~Rr&L_H&2Io>{FzAvyfdnZdL(~`5s%jDDq6Q-$g
z)G6n=NS?LY+;`BlW!0k0rU`vl|1I577rF1{+BNe!1j}=td*w{6)>?7Jdrn2i=0i8{
zRF)Zu2XB+le`b7@OMZg&<eBHX<ZpEc&d#`McPm#kGTK!yH^c96$d#(qwW~U3|2}%P
zsOY5Jc7x5kbF%*zn`=Fe71*?pH?K6=_|M&%9W}ps_uT09`NJ9~5HEIT`hPKpLyQT5
zI`gK<wumedUJ@{4&y~Zqjx$bv6z}s8ekFM^B*87L<y?<J*USLdM&8TOCag`34?-Ol
zL|7z*PnftmH%_K;PP6QJmQQ{?84~t0Ig@LTrZh<PJ$=ya>H7J=jPrjN@Q0o6df7SQ
z=L|tMNuefxp=Bq33fx@S;&w=7rifXro8!&NzOrJs`2UC8^iXYkD<DuOaOB!bE}zP|
z*L!dJga*u7IQP$6uYYIf|M6vYJ#lmP*+{1aE*?)7az4GeZ3_$c*&`ueUHlkk1wIH+
zoaYyya7%F=_u|G5Myt5pCUQ~#9pn-#+|TJudm1utY2>`9+(l|>kM_(F%(4}esFYZf
zI!phJsNGUI)@O066T`T~rmd;H_h|!b%-Kgt9&<h(n7IDx#Fu~F&VPw3P?)q)HTr|4
zw2@u(r>$Wd1>F}!NGrQ4oA@Pv^$l9;GpRI!{X6gViV4!!x3E{Qm~CM-{iaK}<vjV@
zKiKcYoY))KvFT2<wOohIt6sM))A#?Kt@k!w|E;oqXvf1p_m%xv8Mz!pR9KxNu0A%5
znjUb*lT)C~;_j~+eyN2q;WpFKGDKYN`K2FtkebQ!vge#*jY|Da*OOD9oXl-K)q3>w
zS}EIDiQ^|H+8<r=Z3d5%$@z1PbGxrT;s1AM{(m0X**{c-`EH(H$S<-k&{gv0uRZ>Y
zJ?^@vrd+9<G9^@YN9{v~v&SW>W=fvp^113YbLH)8Q)2(DoO|o;lz(q8`5&Bn=joJf
zIxJfh`0l=xy>B};m~o!pm$`llDZ6BDg-PBE_hfZiclwP8+inZF#6_)+TnX{BydCtW
z^LaXoxZFy%Teie?-V)Ka=^2eFS&j)V&r;LU9=jfW60pF-tW7H}?9|UB@1$F=d<CQ5
zKGNQ$w(joKw`-G9dU|r}uBCG-f9wtW_)+cC-7`jVkI(;+`z(9>bLi|`Nv|)e&x`C_
zGFv#MTRIs$(!ViI&u^99+1XyX#l5IE`KM?2ubIkL+k}2kUEcCv<iuW9?!61utK|IH
zOT}wGNza`jt}n@FAJy@nQDwKzlMs<7k26D)-Yreyi%Vh1h|=K^y5K70&@!jz*8Y`A
z9FvxC_-WKf_Hm}Mo-w@`QgO$4p;Sad22at9_jfMyrXB5kDe>LsHs7}gT}+AHi~Iw`
z5?xrkE?j&el-4FB7s8*!)9;iRwZ%U-aG@w)rdV9!l*D+|TE!U!jEsii*Ar8{8)wcq
zDZ;Ai!s>WJx+^^JLcDC!g?kwSa-X<1-;mregXP}O$oZw-vyQbi$Sn#Nbcs`PX<Bx4
zSH{E?p_FzBmFY5%-pHs$8JwJ_rQ~$x+#8u&bNI9z*PK3-<K$=6s_)~P`M)nE(Bpye
z!ey`S&db^~Z&{j;?x{yvUTo(s#;srYEb~@cpMdfv*JxuV_0|8<jHH6(U84)_7U!nU
zHo5i6<dK$2?BeYcopwa}nioyF*3etk@>t_+yGHNh8*_y=FAFL;%l`XovYlVH{XXRy
zza#gqJpO-mwtdj*$XDt?G7lW|&NxbF?6}6d^VtOn3z2t>DjrQL9DOP$&t^wgzR>zN
z>-1FDQwN1RoB}2PzP8uBwvw&?gT&nhCRu?umK^-1a`<103-41tHQAtFD~@d9ce9cV
zmRfntD>cOHteU|}^|i;RmEOJXnmP026|sVb$eXP#Jxl*}UcV!9H0s<!mkjS{wW+Nl
z-E;rvc5gkB8saJ%dbef!67GkpuN>91oa9>OE2V@R=FZofuA-$Ibw?mzN%pQ?nvXnP
zg*vYCFus^5<I)rr%9P=l;(IQ~%XP^z*Wew&E2CDe+#0v+*t6SCM`JHuy!mO?igU|e
z&Wv0A_TCAG#o3Ja^SGWr5X;LKdwg<*&*!Pf3a#!JE2Lj?k^ka*|6)M;<(PNhSg)H^
z-7me^F0{h^hwSPb6Bggtb6az(uf(?d^)^SUZr#4U<&xDd`PbKY=boHYe)C<;w#4o4
z=5OG*yi@8KBh!PxjHx^St#gW58+#@!Tdrl*!4ny~Siek)Yw}#tVi&_X??J=HN6W)x
zB5E>@7pVN`n_-*j%KQJ(g07bjeAym$pS#f))bD>NqxYYRkd=<>)f>XQ1pjRdj16#E
zBH%A-$2HmX>9Hn}{~c4lX=+O7UH2-CJb7qJZOPM_D|4J)oN_pjxM722PeEqQE4_1K
zN9Imt;aEL=>w-*fUL{WES&NqWEh?M(ATf85q|&0LAHoVx&f{Wp`jNTw66->x1H7EB
zO?^`*^SoLpw`$HJ*CQ2dsk!Tq1-$YI(~|x8Hl*U!GQIb<nMXOY*5A4GWL;xi`xf1O
z?{2O7baA6%p1k4X2W;sDazTcEw+m#RKfd>B>rq1)_r>WMx3~MMmu4uJv1J$LF0X3H
zyV3D@7jKb-S+?TzWX0?4trwzO4K}s>-}A0<%IvL&W`3MCcS~H6s=Hf|itD8`qb9Cb
zjo+S@Y*}5mPy0(<!k4fYd;IiGZ)%G@f35v4cV*a<{g)KX-rhRP%HORu#ZOI#|J{;{
zo+5&FI)DA9{F6&v^kluP<kN|%A;;2|1jp$d_uDKU$KJ^C?8GsyDOGpFPEEO)V$e2a
z>AOj)QEQ&wc`39Z;>6Yub2(NEZ1WJc>E?cw(aEK}$SGm!u0Kz?g^$eXyBvPefbYAi
zZTqI#e1_Nit}Z;L+2Z1OZOg>4EzVkASzqk3Rt0Xc4fx`2G@0SYiQqN4-c=tT-%)x|
z8Rh<I-m<>d=TCIc?OepQ>Cp$)$8Eeh4~oJhh2K4t(#!v^H{W=f&ts!^k1t(Qy87{p
zB5&aRX~mB|7KaH1MW;W@dv|rl?CUI>Li%|f14=`uzY9(Ost|nr*WUZTY>gu7w?>rf
zMdlYw|8?8lqVN49d5?nC)6<N4ShmzH(-wR#aKnpX@juZQK8DY1^jrE;TG+}GTZ3CF
z^HW|Vy-R&B(%8z=w_@Ey{pWLD^xs_2_3o2C`@-Hxv%pRHhNm<7j1&XKBL98M2>h`{
zR*+?a-pmPNniIJSt~PSzq{r5-VP9Pk8`d^+^%X9bRz+XWf*KPS*6nwv8MUlGFp=$f
z4*TPWa*u9utj=^=va(9E#eYkIQkjvHTvqtitxh7kN<v2iRahT3ZV&uFhx_`cRKfBh
z!Oz7?*N$Fy;|XXup)tcFL}8oL65S|+LW`wo-u&0Nq70)Jl&s*>xo5}c!Onl8!fn;M
zjZMn4C8ZxrYR5|$-|zalMLBSt%=1^R+U~N<(`MvYTzOHrA&z%v?%AEP;x#qCo0hRE
zP1aKq@9;aQ*RwwP^)bE10`YCPZrjXTy*;kz%%QFQhqsDExSXzhqS>A@u|6fSb%9RQ
z`<E95Ynz3=n@yg-$(6b~%U;*5{p=Eb`Pd_FZE5mv4wac1F4s3wzMCPQderQ5@FRYc
z>=5_6{>S&3T@^VNnwA>oHG!pb?hip1R_znp+2*jOb5CQkvg&@ic}`|9=P$nxGZ+28
zoEmuIYt{Pgac@2^IsCFAPP9}wcPC$d*xAg=VBPDVHY`_vafst+%#2-J?mU4THcX7U
zx<vm~#0m~K<Mpn=TW;kYy_NOFW0UU6ZA!5U-|mV9)>Y10!M9`M=4aX4l6;PDIq6}M
zG4<p7n_Kd|c5e|*zo4<M(CYr9pm*J-lUkWea~FR5qWHB$%e{2o?%>Jog;Rfo2(Nxo
zr~Wc)Yw^iy=PBuXuUAK`*83efz4WPX^}dD97JW5re$G?EE#&*!{_Hq(xwDMh;V}1#
z#FW-r`@Z;QnMwZzOa!+z?Jdf-v)uoryzcU{#Ls^BoVIle<$T(baj~{CHlr-<x_)}|
zx8M6}GJ<o2{@-@(pR_HAE0ckLLw0gv@PGfLJ?){F%BJ7_ImOA~=8v-Z{l*biwf&oQ
zt~hgFZQd~N<HA`?YyD+ecKmtCb-19){ntVzeWw)BD&`jg&ACd;Y)gL^ED>LH`}p6B
zYFcX+dwjlTt|iv0d&6pq^^E3=fhP_t)o=c?^qKGmb_?H2N{#Q$4U~3d^lX$kt|h4~
z{nUELQ+1&%pB+U>(pj&wic0F*+U-7C9N#Wku&dibeWr9#=eo$FzZD;!3RN}|<v8%>
z{J(vZ(@VEbQkr~sU-<NY2b1lE?i}2HqVID06@8me|7RCz?(vwmOvL$4LVi!`#!W8a
zV()(5x_2&6D)y(Cc8g)T<>&tln-6_@@+;^~=Ii`(i|c2_PsvKwxp3U_&*`00mxh>K
z-!^wY*SzCX=3N$Xkh6E-5U`lAz(J3dw<vH$zyYT=ZfUzGms&47c8ObZp19$;n4wSF
zwoT;3#Ki|#7<`l*BorGDaIl1ObqEAIoSDfHy-Q|>up^TSbEeuw%MZ(aW^+tjC-dXd
z(~FH7tp7x`98Rk-uI0G3$^AmOE#tQuRo95~(;POvWw|2Mur2K77Uqq|_F9GK1Z`z{
zpCtOAY<=9W3eI1mJm2%8!kGDmeRL8EAMdJT?#^|6_2J3kH5$fgJ~Km}hb^|r-gZXv
zbH?R4{?py|MwzCCEskkcJIc0&;ntFz>&tBaZ`kH=hso{gQ`?w?g2zWrvfQ5%bD-$y
z$&(iSe6hcZUSFP`KXYB}X)(s=B|bmp>h_d31|O;S*H5VZ-T1C`v-ym9+_k^&Kh0lh
zAFm+!Ip&M-0sCeqmE^8w0ht#H&1|of_OVI_1vs&3_MAV&ZFnZ&Ft4RZ#_Ii6BImR>
zvwb?{a^Qp0Y3(idv~t$&yp_qi=|_wY*V=<DhqR7Ka$j0K)vPE-*-+tY($W)QGRZxn
zW+us#4AsPoHz{aN{N<~lId{(4?K(T}T=Hhz!XkWH|L~SeXByYIX!sf&R5@&Je)|jW
z4zp(ER!i&GEqwk)-`)iKzi|6$e8F+D^x|DFcHB6!n{&yGg#TTfECnJ#p7Lu;n!RrB
zp3<yq(Xc+a(Sl>60>@4c(?i^fAq59nB6}xvcS$vcDz{m@e6+w%gNxH?LWQWNI%C%=
z&Xw`aOFI|D`wB5cE@+9>a-JHZHBDnlQ*73%v~sI!H`nmYetlSoeUqvVquZVAJ4O0?
zJ{~<-@#do5#+Vdwy_T%Zr5mb6oqaZ+{&sgm?9QSVZR6`Thk3kodGGb`UJ_C3-P{^m
zedFOZnYJB|&SY4>+4i_g`KU#IBlAx43r)<QA5V66`)M($Ry*>ErTIsR-EY&*^`*_u
zJ;uLBbIFa!2Huim{vkgFiu`Tf_wiZ1T)LlGK#?K3_tw>wvMm=Jd4uN${6Dm&{oSqC
z>#hG>wP}oC*}YEq;<D{W%dV_yeY@%GxAJ>scX=Zel-3&sDTMua7jE1%gXhk+gU3at
zo@(S$ulb-Ynttby@zW{7MP{n$!rM=4Zl813hLgAC{5eKm*)Qi^_3!R_C)OKM>Z!bT
zN|2ZJ-;%Byma^3f!H=I!xUyE!>4IWsUB7^GZ_(Km;gg)SJeMu$iS1Nd&a&xc_zJG3
zuBADwi~jwbTC^*Ufippf<Iln)T?c+;PZssy5@3DRbw}XlCRf*=_q*!dc79mT9WOF*
zQ|r6^1uO^lUsW<>xpMv9zi9_lT#x0w;hvGYZj$7h53h|jos>vTXg+wuC8t^Vbi$D!
z=dO$Y3k;gYYk2eWCQCmOS&}05=*FC`&4<L!ZLmC3=*P=ky5jYl0xQcoGs{jn%gwJ$
z&)F8u@A$ykM0H&|mzRi#aKFZ!-zG<Fm=`^A<he4RN%TcX?3P>26E^MUdfL~M$`Z}A
zWI^MJy(fAeC<%6+3hn>9wnITLD7?ohMET^J?*5XAfqgYB;)-|I8oD=mC;ebsdQ7)(
zV&~xrGk2+|8M$uqnc{L;_4*7?pRS{Rv({Xm^T9<u@Yl)WCN9~RUY5Hq%;mH1NxJ)5
zt5wc@WBRm5iMJ{pj=BWg?VG+khvQwrbZf(^i=mP;Z>?b|m>B)^m9ku7n){UFIxR7)
zqYg!ec~4WFkiD?~e-vwG=axlHd(&JSX5H<Hi;dc}&!_Hnw5Q;8>9eBqH=PxBW8zuR
z`fa*M<;}Hmr8yTJVxCBF{aWk%>C}A|^W+Wfnrl3Hd}laWTFmbBG+CrPWyV1-4(X|#
zZ_Je+uQ+Wcb!kekh)I~CWYHAn&9l}{X}7uJS9U=7@`o>L(jA$jMG7;$f4&G=?osW3
zq2q@MD=SA<@Z_kbGbgs1=}lcJdFhY_)6I}23%u(y6K99sH51rzwYS$};qnzcT+<mI
zo%dKHAT)`CQ{}j(UqSZNS5e2Wgqf!170uYZ`k6ua&fV8ci|0R)neXMU5yCcYgPqOX
zMPbt;A`X2Dnl)>#uDvP)pGn{*i%YfIZB^fQKgbMI4E0W1+jD*51tFHoODU(m*Il(q
z&DE~<GCa9HL3>}$yf7w{MlYtMrsz35E=+f4d*z=j*O`6in#!vwOp;%wg*N3J)&BON
z`!|b$-o^|4I%x~%M?G4;j?Hevsy~<S?|)@+ZO@a-4R^gxF^L<mXX`U<nkso?A6weN
zTYK0R%Pv{ZCM)Ljw{F7zxjTEOsi{vBQ)_KAG%?^2j7gBz`(de@`m%ZF=ect)e#v+u
z=ppzx`KF3fAcKrrhrj)rg_`e{E3z+=5{(L7sQ=2(iII^(@h1zb2m=#?4g&)NLj#io
zBLl~O238Ikj|~eBHgj-Ji#)NR$I*z%YoelQq0;O|abd0ojR}u6807^gdEGb{cCu`1
zz%}W`onqEuuVW60Y{+}CE0no#lhA_6%Nyo0HU1Lmn7qQR=`mZE(1eoApeAqDuG5uU
z5&~C;ZqAam2u%oK^gik;x;msmYZKFjRl!?BVsAC(u^tmOD9^jTy=dp5yV+%{Zm$p8
z$@HzO;L7pk%+8nYSZ(<jx*_?-%~jJcO|y+=x-sFBO<=;Y`O=Q^DWw~yUTB(S+keJp
z!qr#j*YU5*`&_lPVWM^6_GP+J39Fm;NB>T{y2_&P@zL8d=CN}uKRrKpTjajSL}771
zrZvSU&$W6>zP-P@{r~rMd%Y_^Jl|dZ^4{EFY4co`Bm8;?9{y?k$i1sD`NG-v-<Mvw
z?k4$hs$*MYYR-09_w8GqPUSnjEt$n2kYvIoDW2zaOpGZfNWevTqEoc%>>EMdp4+xG
zMd~i@5xm^J<w%HtF7w48kzS^o6&I2nMUS4+5)?AI>h3MNbLvFzm78YxuQs`PL&s9|
zs*0b5qvzAft0OcUrbb%1UDe9dw7i<oGnMn&%y`A0VY9LqMNXTUR<}eVk=55jpr2)<
z(Q~aRj(|yBsyB;gXRn)RbShnxjdj-C0<%{u!nUbCd9Z{(%<JXSBepX%i)Y1I35ll^
zIJGq~RitM%t(dneYt@={ue4V6Px~Kvs)5%sB%pz@?L=gTtn3%bW;s!jM}b;#ZIis#
zWE|qM)-#R1yt#WV^W`m<eu*pz3{SefBHDLvY4{qphn#tf6Sqo=MZcJp8n*N4FW2k4
zOFx?GITlN{L?k@wIrMDrXQh&-p;0|S+^NT-o++e@w0^sJXb+S7oC2n&*Q4kC?MvI}
zDD>}$OrebR_KnBSHp+ZDakf`uRq31=$K$G=#7=*3>dnzJ^D1AjJae?lC|_NmhG|FQ
zamO-`oY&{Ag{RDyW}NoW&6!<pMe$*#))!H&UouUEB&E`BEX+ym(75jry<mmXf^A!N
zc<CDK%Mj>e+Hls)F(=;o<J>hWt(SuquSrswn=bYL%hXAMXBR!07RSZ1G_d2difU+^
zaomN6eWsS7ldHem>dkIZwY>1)71xoU0d7Tol}Fh(YU*VK#Vrl?u;*KHQX^%xMuYto
z%M)E~0^g5iT<vwUn(#jGwAkw0>;KGNE}NFO>d(hB<-ZaNMH+Vru3R5>|5er6wEE2}
zZ%kensJZpp_0<l1o+?VNvsKT`a6icQ-mYMp@3cz?PHuAY-~4@Zae%KrXNa!vs>NM*
zT8gZC@1&M|wDMw*p1y-ssIQUVr-PZl=)k50Pn(*B-PvwBiSzAxs>i7lymwpEjC@^B
zh2pMLbB;Ydkh8H_b*;jU6rEz}Zk090#my()c|2Zyi-X_Pt|UI|zftpS#k3aDJ&7TX
zK9^@b(|O+M=ffwm(Af3m44sg>2Oph{Fq`-B#+&Y#l$)hV*EWZJXiwj#dCQ^c@ce%Q
zPwc&g11<}&tm&vd>)5sHgx@1$#m>ef0TG99T??p~?R09no$|*Cx<}H4ChX8~GhEc(
z^J1gu_Nyzj?1UZ!9DB0tOr)Q0a9~i(rq&&iSC&k1o$0Fe@r#*{QQwrgU+$?VzqI6D
zr9CBI@bXNHlim|fZ1O1z4mlQeU8(WtIq`1`L=>Ep_P9QMH$9c<-X0rQU+>7t!83kE
zEWM?m6%e>S<iVDC7F*rVg*skV*cx@hu|Ra<?Ki9X1XezF=J;usxzDif_@eonW*q(R
zKV77y-i*;}Yg^~8V;QR(p67%x{6DI4Xkt(dhpPu`<w-~0dEK%1nx^j&Iwf(~AXVXy
zSC@zAt$;Q=H{s*}&s4q3T+)v&7_XVQ{P2tj^#{BK+C4&D@`l@1^q0O_9{5H=)YI)s
z*puZ}$u^Q_ZY`eo=}AttH|yiz`L~WOczY%+cgy0sTNUkpZY=EeSbhGI#G|>)57p;w
zU&xzVHfhD07~Sk9lN0V97fpY!Da_G4<a6^_!L}`)MY)^ybhQY47TJ4tQvTw{G7ls=
zmalf;E9z}p^H}ck(x#rin0zZ8)}NcY)=8Kd*!=9`>`b|7v^GHb@deKPZjTo)>PZs*
ztUP<cnoQ}(|6Q}nLpMGT%Q<}e{!g)>{cD#MOn+{ZBXr&I4$HO6C2f1(#`afV@C_)Q
zJ$u#5Szb0WO9NaEm1}=oc)sJ6G)FUs(s^#lCEGbPr+GdL?@w9VBzJnz(Z39*>>IgE
z8q7mOm}SeS2&H|?7p;Edpu+8V!g0!eDb8*U5uK?U3zjaDJL3KP-qgD-YpzYNv3~1*
zaOwW}bAlLtNwc=!ur^@4;cU#e$|GaS3xmaiDjnS1N{&H?yKV&NXqfc*=IM#e)$+TT
zd?d5IXvvH@lcwr~U%#kSK4aI>WmYp!nn^29Uh0$j_KDAG*21IbZd7b(eDk~~u5Gb1
zhsp7ed?&U~n-v{;!+^^n>&5@G%{nO^kA(gCkFJ}zM&huB0_S3*t(SZx7f-4`88YpZ
z;I3)$lB!|b4xMWLBdOgV7~E>Q$UFAg(@XA_JZ;9^5iJ`emV^c_N_buVyz0l|V^fog
zm&iRili;^2ezo8`)AYG#v*pTF7R|cnxFY>-QCRt$>#cg#o3gKzD5-=i?^3*Sq<X@~
zOTiJL2cK8n{=KViUxwPHQ)T`VRq8DL*-K{4ThU_w-TOG-jq2c+Vf$PbuAk7+dUw&<
z7c<>nOsm>2<=64kz$E?CWO1&Z1m?wxze3I|=w2{&&l<^dckI^2yECdTUwcg=V4`x!
ze7C|C=O_9VvGo~g`CF!i&0BTC-LL4qjL_*{|1G`3-amM1xci|)ira-ICuXI44a?T<
zyvq?|`{mL7i_5a2HT>dI*&i8pG@UoJQuHgneyHhMuSTHZnk2ny6~E|nx)*K*YVj*2
zaPS@VZ2rCG>ie}fkFI)TSZ=5|*Q(p%XvPk${Px+iY-L1h>rYNxv*OH+8pnlR6O|Hk
zR9jbH_W1GnpV+Oo_0Jz(wEpgKvTDZ_n|EuiP3}GAO0k{ZzpmK)e)czy@XwECW|eVX
znZUq#<jY$V+uSnKBVQM8sBP={>-dPhAj_Ri%`nP!o9NZae}C;O2vy99Jho7OSxbsc
zvZVJ%UyssF?E4jtuyaOwct;%y`QOCqxjXL1vcJW*Vih_9&-`zdTstxN(F;}Yg*`m_
zhr7BKtbfS!-!pT<i`goCff^?J>dy<mdwJI+CQ$NwGE+sNgIkJQsFC6(g<Bg_Ld8PE
z#EQJHHUzUZ`l}j#Ol*u+6L9$~_dd|Ay`=F`vUakWJBPS`R;cEOa*tXTsa4Yyn_8?h
z+KS4KdA$7~tn*QM!WOQzCW_V<g(V;Bvu@>TohowGyzFME<-D!RlSQ5CCracMWwaIw
z6smjY7^w-Cs?XUX@@SLO;%4)t>E`Fn0tzm1+bbGqFlwk>Q{capmc1m_#-f~4(RUYX
zOlNyVzk7B?Mf<!-vh3*sOOyn-nkwfkuiUpxD^kQb`hZ;U2b29%OdkGE*5+upJDFBs
zay&<MM-J;kiDC_bE!*QX47s8f<eC=>pZgx@l2NnIq}1QT+l?bDMZM<Lw1hP;JhPli
zbIQd%4MlhT(Ayo+_279+Vugsjn<x84es)LeeIJ~(LWP(TBsI(mxIQaRo1t!c&8J~o
ziI{VM!`5DT=R($qLi@G`(<_Ct;YH6X`d*x1l{wInb;X#qF;w?@U*Qsy<ZEncsr_t0
zo%(K?Ii}@3pTZ=B!e`D1i+e1vF-=KBN#KnE*C`_|`$g(52d(nNMJj@$gkPE$G6~HJ
znCQN7q7-Lq=<{xkClh%jMY*4Oi*io7C+yHVRXA%qR}oXs>}jI*5dvQw1eX0To6PKJ
z&2(@w>%+<aF7WStFyZR~57$iJs!dXB%xvsMs(Zdq=~lPdCRQ0@IW?@)clE;@=EO>-
zhRRsUj#<}Z7Mf2x_|53vmyqrq9qQ@~QP=u76b&Ow1-3BReP!%iF<o1Bqof3*`kT)=
zJ?t{m1FLm}x(rrUJMFMIzbx@LQ`RiC?l?s@X2zL;3ppofGR$HSWh$H*q!Abu;dI<x
zBzB?xNp_((+cKRbCby(anwqILD|04?W9ihNqWR*YhrWv*N*B1lJ!AH_veQgSQHed#
z3nxoCT34AFY`xIqI@9O=RD*3FB_B_=Hen6s6_*h#axrVu_|cf`+LC-oB*dQGc9~%c
z<Nu1nN1F34D)!x4n8N=fB*~@m`o=z^$MfE;?0dsF-^#gPQNvEpEX|#<GdWE@ZRvd0
zB8A&i(u9K)Yg(jorztKupjc2Qt$4|_L``0*DBbi1-x2}Nszv59QI;v&Cr-aPYht9;
ztevwO7S3F>VCLJfAfuTAN~<QN2rXKtxmanHMA8IK=ElWL3l|3zPEz;s5VzEqYMZq$
z#NmjAfYr;%?22>DHRo_VTw-yQE4@SM>j!T8Kuxo-3RB7Y*&aESKbBT;EQ{pq*zG0|
zdtqvPq?YMs+w3mm3s0AE8fh(FBo{n&TFioJ&!_lJ@DNy`-ucqGljFMfxwfzk+v}w+
zavgir**w#{?SG<#zo5|84mH<J3F@BR(~<>NzqDExDNw5<F!z^+oL9!oECG`$fhnB=
zMzdzl-nEM9g8I5!t2X>vxpe6&C9au^PnoN&66~>>pnWBgsc;Sp;~e{$qK8feZaJ)E
zm@?;NCO5Nq-2P2D=Qu-Bz8eUo7v8B@>v*;IR#(GEmp-8xeGj<SecRdh<Z<fF%<!kH
z1X&l&Pf;yC>o#FBQ@gvNP>$GyNa3k#uCdv;vuPD6B|lbQBf$DrjjO6SO3P$*!o!K_
zO^dm>)wjRWSUYni*Xj*6Tw<cy3^NT@nO6x&RtrpBxp<q^qIkvC8&-)LzM4_1yh!5l
z1Xjj1_L*5yfoqr)Hg7KO`TO5#Gv~tzuT`VhZs4q+KIPCsYx8BXzL_iPC8u_=*C%pS
zhR?K_9K<!{)0_t`%QB<1&XjJ=;;Kr0SsCL!%_w<!*{*4kqSJ5Q?8|-G_ms<yCy*)1
zlS$f<ZEm6+PqNz6)-5kZXRMvzzhnXH>;pn>>THVyW>4tapk*yEH);om<0jU^jUBgV
z&04Ud|IlLb)q;K5YICi2u8$Jv$ljs6O0eqN2A0A(DIRTXiffkap1k6>hC`&la_cz-
zO+5~a>WowP_g&x<o|u%}RnKWye`=u=r(^x6?^2EncRMypU-6vl+~zVXY2NX`dFK!B
zxf?O>>}{D?X2I(R8(!Z`37EF`K3CD3$fC#pcXg<>r@pWf`13O9>8bfjE3`g$#WcBZ
zIkYjjIz=&0%)>y!rr0gK{m6RbmbPVrtWga?a+Yk<cP;kbxY1(P0g2rQxb`e&`?zv~
zHbdY9QQ<YBqBSCs1_I(W2SX1Ye11heQh-%*&cUb&2l?i3HXEw*bqX_>3LM~EYT>!&
zFVo@Bhx{fydVdcnP5r3;akAaIh3@BE3_ngUF7cS`t-Z|MM|0t#sVpDc(|;Yw_;n=5
z>S(st@-Q9a;;7~2yN*`8(n#s{<370U=Z*GXl1c}R+CN+|h+5g{zLB%T%2`8^Ve0{|
zpbtCBWDZ0s?&O)XxW-3da<#xj1%`<Uht^u1U=rjMwmI1HU*|-mzzLxkwQd`MiEBg!
za|EQ<9Gs!U&=Pa7aSms@wBDsdIu@S0Oja+^i`i^AJ7D$g$&nX#Jx)42F>#LS#5p^>
z3#v7PuVhG>x=lV*B69tc!8NHfH@eo|+aYsBTJu_TpPcx*Gd+E;f1I&uZY&GgClRsl
z-%dfJ=?d?qOzn?nHLI`x`l?CWBs@pVFu7~O|66=(84gSgI23i^T)T}x-=7nmb54Hi
z5s*8xlKtb#i982I*RcMtIT%nNAd_>5`R*$AxvT*TPH@&<U=?H)K69R5_axK93u|W`
zY}7gM>(D|g31Qh~lg&FfixwYd+jwe4&7})2hYfTtwJI)kPM6#tpuzX4oZ){(^nr_Q
zW!YQGw0#{Kr}nqo*spC5Se?W6aH^ZFK)Tn~Q19h8oLR$jrUm>BNm1Arv*+mDiA8xS
z0tJ-<$ze>{r&)8j{Zbv66pRG-ES&DMMzq#)a{Kc6KLV#1xk#>IxKQ$UN7Wq8-kyz$
zbFPc*Jus{1MEjeA0%vxz2r{xK-jHg%!4!D#XUs+SgEyEL-ss<RL-_0s85!2Tp5xOs
zxvI=mtt#xNESdZ_q36KKa^vh>%nwiLoIS<z@D$U-TTBjZhq8J$&D~|BBVdtzYDev<
zV=Hd6ZoH-bc5>CCHQGnl<Xosbr`5|AC}p8L{X@CS1JON~Vw0^Go;%x^<iX;4Q`Ys>
z9+~t1gB#qg?l~VVd(&}Q;IX~e61NJz*z2D<|MlN{UuSl_(zws{vF};5!0TP>UTV9q
zog|QQ;q2X>;!nS%#UG}9{cU~tktx#w#f2C5_qr$sDzGxxJbZsfV)B}kzk43eUv*+~
zFUvQZgK~a1nFJp(IXq%9yr5Ah!2I^1TAhH(zeg;Bhvv>=O=<|zU=c|_%(vBnf3L$O
zwu?(VW(C;UZT61paX+}kao#EWxF<`c=df;^u*RYJ+k!dUoG(XJ8TJ_JJeXAepz}yv
z-I3ZAQ*Te+YASp*jqhl#)zPzSIznZ)z17%~`bj!V`dG?=ZHapY(vAwQT_Cw~VUCYZ
zaA|Zq$41Uvff-$=_)7o#Uazn{(X4Z*SMHF2?8`4UH>czsj8wd-tjEBfdqW`L;#@m{
z*>x8L4m?_{Cm@(`QE1JHxjF(3w{JQqojPT;>#u|LzPGz}-+f|~eQLAqnk{|1jJ$8T
z$*oyl^ZMYqT`u=-9o=_(f8CqMhm_VVsAJl&Mpw1sy?8GN`-&?bYj3>h&sI@XHa;U}
z9`Zn9?{OKenhx2AUkbDLy?b_o@l$W?@14`G>by_YR(SK<<Xqd7dv7!z%6R1T>}OOm
z6cDukzv^+2!ow2Y1GTk}Wa=KW9DLX$`)JdvgM4=%u{FM$*n4orJk~nz4cb23mCNnF
zT;R8ieC<E$ji=mOcg8!R2S4q2`{|6{8`1w^YZlqQF^m`WvAUx^QAzFCn$Wn(J&e!y
zPJEENR@CnA)1$3ly!Np2Fm-yyZcW`M8|%OtaAs<zm2vEB;}G5Z!Dr>(^KJVO_dZhf
zEayb|EkSHaJ>QJ43l>yM3$4<2>FxZN+xcIw+S*ldTA*an9X`(WtQE57>vaWs=bR8&
zdve0w8}0Egr4%pBy7zMVy_XZ>1v>U#2ozux^*K~_=M~>ip)*BG4&S@IkN5M&zo*Rf
ze{nwKU#9E5Y`>`5e~q2FxBq=`vf5WS`Jt5Am1~)MU4E=xy4PFE?uhec-n(b5|2*8e
z=X`UL#Pwvh8*Bl$xvqB0?p$`|{;qjTC*8ZufB%`qd>Ms(-~LygeKAGg`Ri-1Z{Po1
zV)t>Eu)`lo&R0n<>ZQd${cCjbNUQOfVDZ6K(t7IxPJy`}*bEObC^{@Spu^0~XVMX%
zct|5)5r>PCf|5(8sHmLC0@nmJ!&%2VqD;H4YDKNdy13eX{d|UcZt-;`mI7%$vrMzk
zxpZ89<~Pr_R&UMs%V`(pxv)x@zS?|mMcCV{t(&f%3kwJejh>if>ZPE)&}=7{imF1+
zjaeyzajxE4Z=<&6-rQFz{ashxi0RceU1bIRLwZa*IUZb6-O#YFl~dmAj`)Uz12H?M
z6uznOPB}GMHMq~Fl56_e*|NJ!j5qY2y}<20-LZEU%gn2)a*O3W+Crp*a{g!CRxQ1r
z_vUUX>#uivqZeMA|25}kchu?3-G+f%QZ7BR37V!6sbnZ3@ieiFk$-PF`+3>jKN@cf
zW(g^0ojDM?n&r}>@Y5P+rs-C%TXR0B{#@hF_)k|n7yNk>oUv>g)0KXQ7Lhya?c!OY
zR?d(6wQz&@-?U8y%Q(2dbUHG!UU=y+U*JuaV*M-0W|5y+f)4v0uiNBnc2|HYZ{e<p
zgA=EQ?F(8lwQ&d2B=?qmmo}b|?bq4Ft32Ih)5)+|S3Y{G&9B)s#i;19OM}kl5DsJg
z{Vc{lhQ~!FGoP0^oo&+i$t&2T^_Iwr-b+_}-Wpmn7I*U8y7cIjyJGCGVr|A760JQ>
z=7wATyKPwarEvY-zV1_ED|{}wa6Q+%%c=ZkQ-jmxMiZwMiw#XASyd)(Th%&u@~@Tb
z>T^Aoa9;b*;^n0lohf@!l_fQlLvwn~-wC4CFJ2$t7v()i^ya#bOCQ2g3R_l9U7m9>
ze{JBIUnkw?pE<NmZO6W;+A&*7M8frUwcO@4YWXMBVR-1q-__h&B8NBYmq}-zId@`3
z#~Yt|<JZ%<&sr$|FmAab!tn4?hEv3rD>2Qzca3UxXK%ZCV3TiTNv*P<Vvek=hgS8J
z=!rYM=U+azt7%R3iC{)0M$^dwSEZNk{T7kfE*AQFV!kT3?Us{HjdCpo3X4p`I5JWa
zkM<vKZ7Dk`U*8uU@ZV`h&C2DPjBkD(pT{5hDP48(o?}<D1iJTmhR>eWn|XL1$CJ>^
z`mDsR^yNLydgg124)d;Qyivee&{%dyGdYY${7$}H<ju`fk4>%lRH!D|FQluvvPY=s
z`072Ac=WcWeE;0I<AKViOS0B;zw8fVfAd8zsQqrq&A|UgrDx8R^e|t!er0vJneeie
zXU)7XuI9B?44(dQ2g}pKEk`q%=1g&3s?HqPa%@ts*`s>l1T9bH+(HrW#me%EeKtZ1
zu7vui8eN@S6_{!NufVo!!LNzO{=fNs<=V@-^zM26-t$@}RB3c*eETr#x`*T1#dA(H
zM!$};-t+80(!*71|4z#{)NMVu=tkQAsGyG%ZKDo~tPyz?QqL5`R?DO(Hf?olj^ez$
z50`dJ>Z%wtd-5ce+*o&U?K|&jf)N7d#v4wGOiY;jkZC`UT0^Vil1{sgL6Tk{HthfL
ztXx>xSu;a<nglDmMc1P^iHl2hR!XE>o>FXecQb68wU#NJ=|p5rMHoMKYKPOO?voAY
zuC1AWlO@CMa(vSV3Dd2PXYCvRTu@AAG8GU!F|p%~zl!*y?v*QFI0$n^*ViQSd+l2x
zqO#guo%O;L(XA)sg1B$Dy!)}A`=g7?Z|`fD{%A17_cZaI7Hl}7qB>h*qHCb(72iiw
zOr#_tb&@$ZOXN<m6x-P2HZw}cReOttbViV;?&SZ?L1w*MEN@kuD`pmq^*PZZX>&EN
zY?bu$84VtOZ95rL4m1b9uUJ0)vF$RosxI&OECL$y1-hy_w5Hr&C$1i!cc}WJ*EtQh
zgM49@0?qD?OIbxtgcF{wuiKLpz5dPdgHv~`Z+I!~s@5HPpi9r&{%yu7dxa9Ej!Rvx
zT3cB^E(mg{|KX$?BkXX!W})&Vv!GUc*IUl@CR~#gLmI<4SA0H@Bvv=iwN<|)F=S!Y
zblrx)B%#BT;~$jHsJ`1B7yCAGAD7uR#Yv$Z>8^`x>zfR=@f~pw4C#%^{4#sjl&0x>
zF5VDly|l4$*Gl)}j7y6)UGi9+dMU$m@zg6Dt|@SeUdTzxp8a3t=W$^-Hr??5E00<^
zeVX{3LqYgTqO0wZ1{Pflv#AD+QCZz)K|eWG$VGDMdM8D7Glyz8%{A$?-=)?NWDu~u
zs<7QlbLA!3jU1EHW~b_0U$$+#Vz1-A1;-^qO1Ed;Y&M(w_%_dTO@FCZN4+?I=Sd&>
zeb;Ehb+6}pm8z~r$GL@GF3c*7ne?D~$?2o!MeqMUd+wz^;crvl{<==tw`JjeOsm(G
zK40SVM&~f2f?mbJbf>$bwaX`5_Nh?ZzgYd#JJHG8-gN5Jd)@KdAtdJf_p*71ly~U2
zmI;9}A~nBdSZ!raUHVz-n&}WaEx=OZ#MBMS+#U<g6<aN`EBoOrBGBQqTI#>Wr6~u_
zFWeHr+AXz{OE;rY%yD6W+9C&;`i=?Of{P4YTyJ!(TY04}_SswAA}s-dAnhETr&s+R
zy;(DfFZDP}Otfz47uyvj{N~|1g+96|XWV1zuc)*vStyvB;WoWdxV>&moYLDD+M!lD
zky758I&7hS+fx-nLp$<!IJ8}zC}q)gP?4|nMd-S1QpRpN*=w3PUMpX-GWt@q$ba4r
zwW|{p*KPbcH>YB^>)w+FS1P3zi+|S=S<&;RXZ=;LjZ5yD{M&NGY08~j`V*45^YU{i
z-(RQO`tFC;v#bj)Yu3C`mgig@<-)`?YsaU+s*e|^tzqH}OkhvUH0*l*W%ddop`*@O
zif{iLDljrF`XOm+@m2KhHs&U#fTb-gPq?Q{dMkFT>Vp2QK(2BX_WUU|n|B=Ly&^q1
zI+<hBwm(sy6ui=wYECM0)sA_{7e3+8rHY5MTT0(qf8KQ6%Z$BhspZ4sMeDD>a1~!)
z@qV?hx-6&n{Wo4#A63KTml^xKW|0t=;8|wzbzP#K<BY|J^2B-7`_*QLowlpr@@>9=
zRv-iG;<K*70hz8`T7CxuC%R`v%q!R=w(W;?NJI$tn#8w_=3!osbU8xiU36B6ayX>%
z!Zzre3nS;21)|zAlUgVAI9UgFtSxiOno_-9DQCuYr-|G%%on|tRa90!YG3!dYYy8}
zIi}8^AxpmTUrzcjd3^u%8_v;+(%H+NdR^c8^MdNxWee?>{hU%}_i3Je?aOfcU(7d8
zU+6Ud;w?X4WY#;A9V@bw%O+O&MzF4qciUd~b#;8)nqnXE6knI7i}Ib{*W1sU@gltS
z&8$N=p0B<7W9zPj=~o51-+Y{Sdxy56Q-^@_HiJj28-L7b%`odft-4Eb&Euf83l}b(
zwt>}pL#s81<E_Peqc<=!GC1ZfRynM_W@59&--XH@lg)g*^p+QjEo|frv+SIwDOSH>
zLTChgm_Y0E2Ycc!u&`WUnaFK9Z9%I>Lqp&Ofv_EY^@}G&S!~kSu_?5n>Cz|mR^f(c
z7dYx8SQ`)b$Df$BV8X29RRZ_2S^sakxTDcyLEt9Vsh8Hyy4BBdQXqBaX4Z*oCKq>B
z2J5gp_N_Unzu>dXw@o|eC+ojY64t)B&uxi`d&+wEH!Yqi>%F!J`RFkFUOBS(v&;O+
zYD=rvpEx-uW0Ur-3+umF&;9mTsHEuVU$6a4+y?qLcb&dq#eRA&`)OGY<+*Q*ro^)x
z;GVp^Q`7p|MnV2)Lq|c8rBQAhnm4*wiac1swNZfEO|Vb5V(0F~(?Sk*Npg4nPPEJ`
zZjCNzwJu;TFH&dJa1Z|>cZgf{au<JC!=^P6^8+3DXByg0Dq2x=*qCV*duZXFd%_HX
z92SboT9dRa-d<n|5@h`;!Kxt5sv&*iLXkpMgU$bw!VJCz9gWdOj3=7PU$a+9tPNek
z9@W6axPV=EicR%o8}>&_SQp8&Ixb-SG^;vcLF0>AGnA#LJ(zjqm&THt&b?84W>jb_
zteOy|Fn!eo-q(w_JW+OAzGUBv$zCrOpIWiz)T`G_?$$jzFS*x=H2B@%U2u7lwjlS1
z$*XqUkPBGc8I<h(PP0eVsHZ}61JiGwhJ*WeuQA=@qkWu#?|<~_Krdd+Po~#aC?%Yc
z+bh^{#Lz4uXxq-D138T%yDhnPEo?Or*|~7#%2}Ug@$(!MtZ6PbX+98g+`OXIx`5Ss
z!n9y5vF<s2FEctOxA1)tV2zrv&AfZb6l<=z2PMTgrNbmo)HV10zbUgau%kue14p1h
z^ScWy0+E~?3c6)81bn1eHFj)&wS&+1f`f0x_R3}(!yg(Gt54_@*pziN`7D^N``Ssj
zqb2l03kS!_TOx|>tND*s*{+S;!)YZLbV$`|l39F*!1jfl3o4fsD+h!xTIyjXHQj;l
ziNneT#R1;hRxUjOUp~ot-Er|+ePrS33*K`s@cli)?`rjZ(WJ8zmz}$~er|-)s>fER
zUMT8slr^|Gcg^gr3YT`BJ1|#2vZMIVu1YU=%a>O7R;q7)Y{36n*g=r1zU!FtM+sfU
z6&5#J#6GgN9I}?F+&r&p@4L>RXp6I#S$xe6n#~i0jbAif;NdmTU^P!*ZGYANf0khD
z#0yT3T~1bIOznTgT6RG2-Vz(%7khLrFtv9yg-+O5b>WasM(|pJ)>bRQv!|N^5Ae^s
z$nrKpu-f_j-q-9=JIsAgbWNJjuT|@zTVT`M(`0R=d%l}V>?M2P2`1kc!Cf(}O&9vK
zt2YNrDHvNX4Hn{9XUG|Sc4=)@m+z}h!ZW*$p3-=r&9%dE*76mtdq1izFY$`6KC-Yl
zY@xE->Y9CzS6)jxa!o<j>ZnHZx)xqJrqe+N%sw*LKS`Z_n>=On9xkIBO!Izpm&uxb
zN#4I^QTR@l)e1~K^^y`Qx32H`5+1ijeZLIH|L7STvY8Lgk+{BrTPQ(Xh@t!N6QzBH
z2afn``#<Bt>MxQVeFvw>zLjHBm0T{+Df#Anz^3--7e3Yz?)rbv{CzE#SFJx^y4UwD
z`~NipjR8AOR38la<>VVM^L0c!*Tr*w7cbXVg=k1LFF4UsF1=&I><Lj9{NBW@^b)mY
zlyJTvd{b|W&G!WpRHsOt?`Ao3*@iKK{j`nX32oi~OInw(8tr}Es;sPA_2K-6mnvly
zmQ#u>7H4wK>bfn)sd|d*!f7X+77M9Gjy9~J98W$NKdqQFTln;bAC~u4Z8;w5ve)B?
z$Cf+kTU|V-9O1nhbNa0dKWnT&sD5C`uFNJYwYk?ze|Cq;%#Cb08a5}Kx!84WcaQX1
z*E=_vZp$j&xodUf<Hi5~gn7+>?BZf}yYgXc)gsQCU0h4GF10@3>Qh~|wfWeql@mpn
zt#WTJFPnPl;O5IUUyiN)Ij^vU*_z?3`Hvvu2f6}Rw#9E~?JjN&>aaX)ymrot_I@9y
zsL0?~D<sd%KIdvXu}RwF&kkje*e>r>)~N=%^3F`(3)cRxy&YB1adGvfstbI984tLa
z1ik+TPkD1C^g#cVo(Ec6%{flAxaB%M7I_eFx_HHg_9ZL&qd2?f-kGpK#p~GZ6$@QD
zW~WSCEios(=F)zHZsEV{|F7!SiHu#g<XTSXBZaMDYxZ2&x57o^-L>^Cyc=|wjT=^f
zUX=8xFreTh_x+6PWv(5XKTH@ok`EZ{|9_W3yE@YT$L!NrUxx4fbKO#Jb#%4XUJjq6
zITQCPH1FQ9`pv0z$JX3<!poP^I3<0tbl3{t%^%z2Gg?0tF$Whghj*mtn=*bmy4N!1
z;H+P6GbOe!(~Q29B=)_d<;Eg`?mgR;q*+yhpG>&CM?>QF(hEj!A8;@(I9pbI=&_gT
zTa)v@T$>Cdo=kfY{JupXQ<|eHgM&@X^R30X7ZF#M%4!;(VCi|{IrpFq<G}<*%{!-l
z&2mthz@9jvM=Bs>*M!B4Qq!z;S{J1J(>Bpr>E>K><W#83mr3ayQ)3oYU*O!mXyM~@
zuh;3kM*~i$x?JSF+hn$DN%4{SD^81S-L>+x*6PVT_jHy1-<jFOyWygy!DQ*LlNv=k
zie+x>6}b1mTm4E!&*P=ncY3L<kx<<l>^AKmr|_aU)!1#f6uNbVI@GTo+bz82`<#u@
zy_#1X+`}uF83lrka+$Bh#LudFXnp#G-$bK}QYRW$w0>K`S5d92V7zy4aKfY|qSm3I
zQG9EgK8J99VlQjhv-mB$uD8?1O5=c|2iX^iz7k-av4DM@0P}N+TWd|zSk$iVn`UhB
zq_f}ho{3%N>>CDZv3EC`JX*z~e&7bTTHK=(ZEsF4+p<Px*Paze_S^{5+V|*;UU=cr
zBf56ii?7bxVK8sE!sD7)Z)3M)L$w1pV)tJ!+h266r-oO;@>p*D(VlWAvHzx)H{xDz
z4fxEvVDn48SrPLr^7>9jaOOBnw|LU;ddjpwhj-7_<qIZN?O?6e$TBWyzGUK9_GDY|
zC&%P%olU~$*$tgoPPD9fvE4azYEmTo+k}MGD_fWBy}x7Wgh@I6z8<2dC)@Z4ove;#
z{~yzTNvPwoRI^{9&9s7pF1;R$Pv*<HaO6#sI{m71@yyPHXO&MrTGIHVHCnWTznt?{
zwQwa9-`1ejQk6MoC%sNh4VYcsbM9!2<o4$$wtke{f9K8OQ+^e5xsMjgsJ~D!f5E|<
z#3#X_T;5r2`zAa__Qus!-+}{AUAP-7BGJ9>lje=4u46AWXEfX`y}0ewu8TKJ`_m?u
z{7L$sv`1Luzy6l&xao3}{#O?-)!LX5!?9L^`%NzEoPTlxUQ^e6IA-v83G3ev@dnLr
z<+3jCVV;@I`m^=*2`&qrmNoA>TAZ)WY%9{?xVMQ-OCdyz`S@aXhZxSP7guT?Z|}*t
zta!)CO?O)0gUi=e`Id2TGXFfuy~wy&_|&vRU!}s<`TY~v<ay!U!MN!U-mbIwcrj3X
zCzIRpV|IEMy*xj=rnP@F-FQlE*_*88r_|hf&+H30Gw;#azNB-<9$nxmcUhfuBjCt_
zV7o(VkIQ^>TN}d5OifdhlfPEynzudP{Al`}Yvp&-)Tg=5vY98bY4*gf&KvhRo*b1)
zk-jOBy0X=gXU)Wf5C11{3ohQsUcT!|_w&5)jOHs-eiS~QP*vQqF}3q$3sX<bljLV2
zCx81Zt2~@6Fm;w>>+X9-w_dTANr+DU$nj#zwvbNqXF4%<Isps%r`qxSnlJFNF~B%0
zalvJmj<-L<8fPdm9NEf~H2L5fyLZ{A*8ETBNDW&%{X%71jMV+Ax_2KpeoyBU{j^2?
zlm8u|g)hFVAHOJ)y8lGSr_(p?p4(8Qd;XK&_Kfa#pY)zvS+LKOy=!`3Ez@NE^&1M8
z_GZ3ZW^{>d&oq^#_ZoM7p5-=W&bJ+cS)!YcJ&gX#9<0Hvukp%Im)W?tOKa=dnC(*y
zTbkCtXfJDK`uFT)T)|?8r3n^aJ#~Hd{$J{G?p;OStea8CSN#lioGo<GaKD_2!1Af=
zfmN9k#fpAjOei_iBHyvn-@f$cs^ezgzB$Y7PY@RQ|4;P1!F&1o)c5~B$IG4!tIs}v
z>UzTDb8fvwPrqsJ%PIJtQ*`#t#eMbOGWBoSJ2aWt@&pgG@`yL5a>@BvTzKHz9xXJf
zQDeb^gPm;4Rh}FwR60IE&b>?LWns7H<e0E^I?4;q9-d*4d`aZNtA}U9Qp=`kI;Jf@
z->AYiX{ATo6o#N2xlN8<#V;LJFf9pMnpyH-MSCQ>)mq8T(`=*P=3HH^x<G8V;nzQ3
zUtjHCdws^gJ0*t6$6D^rU3=hL)a`G%`*#>-I(BZFs;m3oH`1t6(9~_CTG-!TSEZec
zH`z0;xxxC*bc2rVd8T_@0&n#{J6u`Jy7}8z1B2{?tCIQus2<qHP`#by4p;W+Y1-y%
zTefS4&RO2DfO+Q*kqb`~iy1E_`}O~w^R@K3;A~_4Q*%TP89lI^`p~7#rt0JK)0@?=
z?mP4S>r3yA%iqtlQ)m2f`i=kjl~3!>u%3`B{w*2drv69ML%x~Y>V`rKzn6q!D~DFc
zyac|rDh>@YRw9BfF@MqpJ7t<O1jJ<)oH*#BS$D+IL->dVtC!I&i=%A8Cw2(2YqM%7
z`54=(q)fQ5_eq=g#kWUfCaYd@@sbXhbm`QTI8I9e?Jc%bRHYYB*m!dK|CAinWeky;
zp{YJ8TavtWlWtz>VtBj7$V)eW?N+huZ+{+lNtVw1*d}@C>$mfB|J_-sY8$DraVkUG
zH=|P;vpObnE@Ardl;8I2h2)h>H}kUG&Yn~EEYx<+G^<z3;~KlNR%~gMy0LQS%vZM}
z^Z7zfPYKxfMYuSsYHDoJ0Xr3)vcpwZ13MIp*ylT)*1avb+4tKjfh||fyx(rU>@{6}
z@`uXXx;yT7Nv~_TH|ct7tF`Kjpr-R%wGZt6IIZ#BuCxos-}!xbmvU^ESe|v>7DXkI
zIA;#Y5J6V~#W{kTWI2x%c&T{G6nbdPHDT=$>pl`Be(TpMDSy?1hm%iSn)-1vgPW*o
zYS4f0P?wNXhWr0qIFos*=JV;Ex;tJGsauo9qOvxcnj|dfQftuvw<m}zE{$dTGTVJ3
zu_cT7E3GapPA)K$N|^ceYIwrT&c7e8hHhRKb<<Ea*KBRnqqP1tdw5e$JLGXBY_B^$
z(<XqES@5^v1Ljci9SO~s|Gcg2oW0$m`o+oX7Ek`U^6jjCH!=PGlNY56E!AgtN60nI
z_l{G1DLgCA>eX`X83}b;Hf~Auk=@v$^l<&>4#RhHCucnF-TV5-?{_jc`KqNWggqxe
z^yHGu^VJs@Rh=4=*10tN^5qoUZxNl6%cdtJUH#-7V#o4qR`S&N)8|WWS(f-Nc=hts
zp0p_+-9E1U6Qq&(fAvPGY@a{zZmXvKnXYx{XiDgopX_%Yv~8){(dJviEEQBzx@pGI
zzO=Z`bHzeUdhG_?R)#Mpd!KP|c&_4aRd0N$r0S_|x@UTnR!!ohm>p}xj%{c&+4GP~
zm&u^$?S}S;LE`?`mg?8+d=k^N+i#yx2v^<5_D=7RMZ%UQ`rqEP*myh5*|JDfxB2KB
zXW5G!;s%c<cJ~OX#4I>;<dTPBudkv??wux)TNXlUTV1>>gn4`_n5Mjw;7ng?;T4)?
zCc553RZVZR*Qq^UY~y>I(vw!Uo^LwVcX>yvZtmwU`<z1ge_Imuza@C9{(rQ3@lGwX
z4KsPyUP?2YrD2e_wbMJ|Nt*Eg*&N67s+#@o7ewmB|9zV;A(JSdS>$x!YJtOl$Maqy
z;qyMo7#Gb}o=_#DzTmj%3tOh34#UYaeHmNV&6*w@`|1$qxx!_9GdW)dwD&Dt*io~h
zls}Q9Cr|yn?h}u4<4>}7_1jy;Iy+R~$xKRkH{<wpkK-v%K6$wovD%slPN|9%5}LzS
zq}ouFDYE2++)T5;WxZ|&i|#2iYs+NVeYx7?H{B{^CCd!yXq}BbhZMUPCansJUuhD?
z=JV1yElX?tmQ5j?J=}M5O2uCBzEp2o;#)T1srjL&lhk~_Dll&Tyq!O^!182MpZL}n
zIvKg5(UDJHZROIk5KHs!PI#&qRm)+Zz3l&Kr!TXTcHhZTDnDr2X|r>B-Md}utG7;F
zU!&?D#qAclx^AO##+?(+EKY`|M$_CZvL3&`rLkDkW8u6*(VoZIq9<E#S|*x%;j!Nd
zfhnt(<fOWHUJ9CJw0SkhrmbQvlG*lV7tO9T>!{dzN8L~~-IkCR%9b-l|5VVbGd|09
z<Z+%^9cFnzP)WRC+rhKWWzOb|7tV^7HB3%_seJ93#&WebX0?jLt7YT<or*B6<f}UC
zeJD@XxR5>b{i7+F^NKvz?K@pEr}k~N-^HJs=i6s3t5uv5UMMtU4x3(jrNW`LH!m(p
z^vf#nVcFDL(Zu2=S*`QFVfn+Fdnd|X&ROAdw9Dqn)<(<!;aA>F7L&TDyHR3~$3waA
z7PnM8t)6P7R9;rD7nA%W=@qic@qw1((#zjEr!U{UC~x+L)skAGoN4UE9%t>gM1A%8
znymM50o&=<=7OKzDylk&`?^@@ui!es5+|qF<tihtdT>Ikc1&W$9UW(xZ9mTzZ`;hf
za_O}LU8lQz);toK*{rvX%gWQErcadB{fLH>U>8r3(vsh<89KA3YRKH|I4t8Pe3UC;
zWA++}-bWg(oEnasyk;C&;k|d`nQvT2c=yf@&ojC<uVosG$i~uBZAl^;e*-l&KbMBg
z__WdK-Wngv-x12$+A)*1CG#w1P)+D7^Yp3HnR;sJH|Kk10`V#aE0}lwKY6($S!T~S
zm6bbPKI(V#oQZo@G~;=w`JRo&>;LTwpZ)FI#l4QleSf{$eg8JgT$8_deQT@xxLMzS
zZ=7P0a9Q)BBd@-}5&k_^EO&3agzdF>!?S2nH~*7FgI8Np%Ud4uGUs2BHrv_z<7fQ)
zu8M7}q4VSpeoc^aU+<;8OwMPG_@mk3`C4K}SDdq$CDMP$XNiTMytd!1<U=pNZ2zBp
z)%nhr>0w)%L@z35xb7>xdO*`uO;vf#WES2<nW+MYp8tJU<aMo8fA#hVzI#gzZr82Z
zI6F2n=IxBIy}#CMnrHex^35uq-J-i~h4;l+Kg^Cf`B~pU-SgwA$x5*=UkUBGcbeCG
z&!4#ePnRz^{93f5`Cg*0)a{oS?R&oTh9>vj+QAsOc-E#R^A4>{*8g-p^_thV^_+8l
z+!DU{DXu_6`o@kxYte+qJx%LP_nl3$b~5Wxon%-1EpPX={*ZNVr^s*S-M(|}+npDS
z6mHh;nSOm=Ncr*Ie_B^({kZdC%jBsI*Zf<n_pF|CZ|T0-{~pKc@BQ8R{_pSb|NmHb
zJTLHj#&Cz>-1Pl3z5lVEcxDq^rE=o1XUq{-1(sEOZ7p(LJYsG+w_GD`yg$IfY7_RM
z;Ks>?vw9cYNxkXU(HqLO?G9`2=`*~Z>tg1f+d3od$T6O2CtVLp%t&dSX~Nno*s65*
z#w?#$&xuY8WxQf4y8NI14`Jp_)P2)>)v|TMk_(+j=H5O!m(8_ZG9#?GL;ik7!0t$Q
z(S5S4I~%3%I2KrRdw*=xUhq(NM&iPS-r_S3l+KvR^~*PS;apKGcZsM4^5<e%w?@id
z4V)vGwEl?Xyo)n;XecQ^3wR(pFY2lH0^dh$fs3|>C_R?FC15%6=}m9-X-!W*wo6~(
zZf;T1n5NV^gE3*J!m^oxUOjQgRj)}@N=diQO4&H?&C^G3*0M-R%J`nRwVE;6=ftga
z#-sjUTEsb7UFU?cth}bcDxEDLZK8DPn@&*B6&J-`CTWe?xqp3ZJ3KFC%(r*A?)>9e
z-kG2sFI&ov@)%Xgp6F2gb5gd_wEy9k|Gl=0F5GRI{rh6n-H7ml9ZYvCyrsoNeC{}E
zcli8PP5FCQJ}7hMPg(97l?5)J?$_VFeUS0SfxojEv!1l&O#a8JU?S_GsUuTsGRM;|
z#&gjV7O#XOp$m@bva<&?9!orOT-4>IW>{aztez_z9Ft_)=iU<WIIwc*te!6&?ROup
ziM?<-clw!)ZBt@o&;LBd8#Lwo(*@HFx`HP(1s~w_*&{LO;SB*P=}R+MOV@TXBuo$A
z7ju2#&A$sHf;ij6n{LW+9s9j;PKm<FvI(*!3ls|$C{F6}mfmqlw?{?t49lDej0HCs
zPrYfA?&;y&82k2wvNMxktXg|>?6SM6!rl_BlX-lc5Bz68DA1$4OX2g2o8fzBc3c&h
z|M7yB)jh3E*OfLM4?3mxWNG)_6a9Nr;v}Cu%iDJ4+23b-pQt_CIVFBBPnI70(F2RZ
zHNuh?vUshClkSz$J9O{$%S9qRF8cE>wsR!s+_~t{BzE#ZSg}a!_l05=KW<H#IE~eJ
z=48?Pfe$B!n#A`l^q0;#82I6c;j>BJAvc}fL@S;y4E}m#=Fyg4cLSZ8;x%tL7F;;)
z&~>44S4;e&8y?Rd=kI9Dc$=JhPrg1&V`}Z0wo~&rF1gOe+uuI%MgN|2hCk!G!k#?(
zF#Q<^2lJ`+7YYk3S7y|E_Az^%U|yEe^5%utKMilSh!;nnuoX$Yn{bak?f(o8y_J0)
z>MIw{3b1?0xv$UXg3HG%ybdc93{S<(;*DLG!a2uh>5NMW-wIq$y-Dbq=@}H_G9|*_
zwLvvP?WkSjyuYgMbF<uIHI`26h%arBm%nwt?Ow09DZ9VQ5y^Ek9w^TDz0+i1wye;o
z^VeOc;)aOH9BKxB856cNm2f`0n=|3W8Sa!x0rj^uXEH8woOAoe%>eH$98Nq+va&2)
zE>BiBrLBJut9(sz-JuxnnQAJ_M0rcO)#4Uym~}z$rkbW%;9ip`;f0Gdj9zP+X>YN6
zU1t*EsgeHD^@dJdkNY0Ml!F(gA3mD=YSEimb@v5<`dr)wT$~0)VkHuf<z)FJUqn51
z`R{qU;ka>Dr!kw5sg}^g35_+=j>c_qmiAy|pTcx<N_3l^%7Y2<F<&NGPx5g3p?qKF
zvEPdOJ}RD*7WCB3S^Dc{<3X+ivNmc)(;TG_`R{p_JpE^jw@8L&&I#Y1C8c-IdQ3Yc
zd&4PNEhWZF?Qq|!!(6ih7rk(~rQyo<)+{!~P44YJK^gZ$FU)H%dwyEc>hSh5ljc>n
zmo5929P^ZT>FaR(WQU)X*yJLKQ@p8Dv>d1W>-%rXQoe5CRG}w9W$o7rV|;8<r}kv}
z`lQB$ObLw>@%^&EDKkV#a&Dy15iuLBnia94bt@_~PdYzXn)>X*xo3-$yxgTfa6K-2
zGi!&_(|ax_nEoHnGdR7VPrYWveOZC57n~}K?T%c1_Bdgk&T_wtw_NqkA9D*+Ri2z8
z(D`g#+Pl2;Ls|!4$W9l!x4ub!_SaQWTc1VwPRzb`HQF~$pj0mZURwS;(XT?w|FkF-
zmd$G^m^X0(%S+wmFQ*>S?OR{6D{kdcxzbmQnD)%`n4#UZQ9Jj-EW<;uC(jMjG-*98
z5Win3<l|4S4#(cvTMkEFQ)l17-X_)eMrp&mdx?!)FQy-zaPrKVCokLYPS7d1aOv0H
zp5xORmmQM2x982NrZ?qs8NWqE&K^3y<xWe3*YW8aby)dZ4w|*!ouKz;hx~$fn(cZj
z9ZfHIyH}Kz96A4LodADfC!g+r#{arTrF?E`8~0V7xbW=ys~6q<zgAogTPqsQv0X)s
zy=jhrS>X+<m6K<!mEc~=@_@tMa0bH-$BwB1B{6HB`kk&8OXD#Tx%W*bJK}uz!k2uf
zW{4O~nX!O>PL;gZlo=1J0_C3-++|%gjYq4rfN@et!Q=)VhoV)j_a;6)Db;L{mF%|4
zZ$WhP(}2fZ&$>=VvHw+ly^Be!eqHCq56a)Y=3BJS+<D}B<Pjb5E7xx>TwkKOa@RQ?
z)yt_Lg;#IV>YCu#9TT~J$0|eh&BZcF>+WfNT(|7wyzUKRvp5siEc$zMe&9qWhKs_1
z>$l8Ovars!a9^#-^*rLNQuda><tGJG0;jE<x&CL{|J63#tC<*<9!a^w#Wf||Wr|$w
zt3z{g*2%V{3(n`~V%r#cL{W0mqD|=+j<)T;Cvo7IXKh|X^^@l(6qHvjlZ{*$F)jA^
z^{3v~A}>75%{g<+CCSS@G3dH?gVNm#?%FFhKE2vg7$M$#U`zR#Nq08g-Fstk!=_IT
zR;w8HIL_d8S@0ln=Rswq)9+R!Ej+=pQBd44B>YJ*^V<(PYY(Wt?T9JezwunFLWf9a
zN_hCjh<HQRu;c8nmN2xe6<;hP;hg#A>gIrJo0(?JX+PK^eWTIg;n~N9*UH5{aaX-`
zidE*l*0Or%EXBVuJ+_`KwjsQACnLS?mTYB`^l6NpFQK8HlM^G^+3^37|MW(w>%7G!
zA5OZw@|eV=R=hzgoISIA!mBK!;tr#1ZayD_XW91~B^O;^Ze^q~FR>ux(GSO#n7$h^
ze$SlC9v(jCI*rvk*i`hL*_p)Fz{K1WZ+y0wsCU14p!e?9gp6NVn}2AUw4ctN<^Ad0
zvd60TKAYu7fAua{w>>XOKfg|?W`0slb&_|aT%1;%c)=~_6&*f72dw<3^-YOdwN$@x
z@%kNp2Y$_3%jDr8v>-}KP{5$AC$Lv+X5pooE5xck<yS9z?({uBry;_l^=QN&AL|{5
z)N&j&8QK?rk_&jVsXXyjfyePz%X++huG?Qb>vQ`^#$}U}t3ICG)AQ_}ueY?i^sE2v
zzWnDL-nGcycq95Wxp7xh!1L}06FzWQzOi|?@Xe(8kN%atshc_dxZY~PjWhZ!XL7Ht
z7mZ;Mi}}Ct1>+?4@MMmN{;hK+mp1O4>{DepZ~B6H+e_aZ-70kA>wIR_?tqu8cg9TR
z@?+g!D0*&fmtly{luTZs&?WxA?)bIJg(siRVg14*q@8k9;L(il6B|W*7IrokNk)k-
zzt_@q?(C~+Gk(40z31~{b^52f7Ml_-wJRUqP<LUQ_Z($OmsRY|29hmr;<qVIa_IIJ
z@h^F?lwr@Oxk9IHS1stcB`JUHq4Nxe(33^mrq7&~Eh)>#BKysENBPdVRXdB4?|8*b
z+qL<~uI1Ld)&B<@s9!c%%<)d|xM8-3vHGKoI&+mr0vl)bY-hM7%2{(j`t?2y!Eb$^
z&AW{cPu^@{Juhu#$I6Kd8qRM%Dqo@Jb)e~gWm9%fnV6N|k*FU}LQjN*mTXv=!Yr*B
zabEFh_L(hvHgHY~*^vIALBz#=VMG9@>;7N%S}(X9H~F@_>3c4^X5$pcGX`gVS*$V2
zVn6uBd&@$<hBxcX-uXqm;!*$EvS^{NOJ(Dm-bGuNRvGFAd8IkWRr+X@RvcA5x2|@2
zd1J8q*3M`0p|f{}T(;dS`F7vS`InS*8TmNEo!^?D-TC11&Q}&J*Yz|5yjDJ%towMf
zR1CY-g2P*rd3SL!PFZJmGl=j1`vlHQGVybF?TniK<bm3g1fMSnyoXu>GeUkIS()O^
zc+zN<cEPvjN@)cyMUI_iPE!u$O}Vk|dX?7WU#m}lIPK-$qW5ajnV>8CLn5p6ik@&^
zc)sO6&$*~;E^WJ>1|6QwpKgEqXx`TUFISrJc>A<S_Wt+iurK*vlo)9)5o!MWboDXQ
zGd#BQ-e%YB|30%~`sqhY&HX)!W4>}blxP_TC>(4|_2M-NKIJ26&LitobK}Fq1MT9f
zMprrtl2-P~3zumgc6IP%3}*2OFmmnmnx>VwZHi=Z6VnW{=tol)I5Y{d+6(`gW6;VS
zxJYfHQf~64sVBpjqGWY~o4h3%RZm3N&Y1e2M=~haV<V@wgU;+Fswd}&PWYx1y*1<d
zK3z|*l~K{(a<;Bs`dxD+=aQV8Mb6h3GgvQW*|AGCLFI}y)3jZgQC-a9=1g+COe0V6
zrRm9f?LByB=@Z%TJhojzv4<3&tqhwu$z{%ksV^t0?s{<SY(!rE{i4Lr|L)HIQn2&r
zWF_~$kew&*1kZonc5YTx-s_N^DcASStuB9m|6n`+c~9$~Z=PJ79$r6BMf~%-yVD<D
zX`Nj2>(fW>HS;IfJN^I5_{aYM1E<V|15AQ80S8&QpB!1D+14d+XxfqmnQAkuUU@RA
zntVBVnAh>o2B$?^v@=(3_G9sDSQU}Ovu^V~rKO^hlar3BNPGW36v!Z3TJw6P%t@(M
z9z}D{Y>)Lf1cZ#F+C>sXR^()=o{E~6wUk%0NqS|E{Mw?WE7#~~XKss*$y~h3NUXS7
z#l&{<n`w<6t8^6&B{#p?B>Fn%yvii6qNfJ>Cqsh0Ctk_QS!Md&XN!v6T<w!RlZw1o
zwXCjKthK{-t&!sn57V<(RE$>%IRr0RGs`5{$=x=4r)TNzb=<;hnnlecm9OSr^!#|o
z$$C-A)2$kMRjl8xCv3KUbzQXAcbl!y?gfjEyxDbOYWJ2+(NhBLcJH{hDkm}9|Fqb_
zzhQR@PF^iq@N12exP9{j9p@#+XHBp3Ogt5FLg}B@Cy&K7KaXC&-}>w6iTK99|C2W^
z(VVs+;30SND^JyS%PpFn^Hn$;8W#vjyt=+P=#TDcGd|f4mPsa?Pie2e<<KJ{ebnjH
znk`3~EcJH8`JGy`&F9jY9XqmC9-mhA?UpygevgT7HcV8l*4TJ%%DT0hcK0=o22Cz7
zJ$Y_fLbb^D2L-7mJC^u6_i(@R$Q23N%Avb5`k0#UFVkq&h`i@JpHI8J!{c$9>8`!2
zUsvxkvU;F&`_z`pQ-u%hbgr>|cklMPk4I&{zuS9qvvTvN%(I(qtbdxB_kUhxQ@+9O
z@1NWJ`&)d{3l3fl&7b@AdaV23B^-upVwdt-zcDy+*ZxiI;e}=jf!Yt(sQnAtdiZTG
z&xW)dxm6FV{`+KU%H_&Mop^n7(QD6LFPe1L&0d<LeoABEE+bu|tvab2C*AS!F<hg&
zizkrLVd?H4n^&!qTDhwK{O_Dg5~6c9N}f^7GdP=iW~#)MBWGiGADo^4%iZSp(ya6)
z8x!t)N^@k-3uu--6J5QcY5AKMT`h-i-Qes^a*AIY5OJz#+C~=*-4AOxqN^SWOKvIM
zU-Y&hCTf-cj=B!{V^`c&ST&mkS(&WLWeYn!Y&@LUm|Xw6uqcb%obJA2g?>*+U_8TV
z4!4*_EAhQ5>gI;YN92x8?nv!W7v}Dg7JYDRo{wUy+RoshcM;o~6I5H0t0zvp6vVY*
zN~=zCWnfTAnaB1QXFQCh|N9;H3F6+v;H$5id6Y$`c-Hk8w@f8N*1s*ArGKyF&0Xx{
zB`M{(@?OJRqbrskmu!qASKF|e<<IWYvMt*mJnd87o+I}|f24W8+)*@R-KE|Khm@T^
z>P((__RqP;H$Lw;&B<Upf3bhftI57=C+L-J_?%{b@TYI#DLyCRQ*+;6IB8v4myu*A
zc~NXx!c_N}85XGwH+R%1E7w}_YhGl&WM96;cv|9m5C2<nr=NXV+~BlQ?D6U#?IjLu
zj@!Bo<Gj1B7c7<ed$ete>dRS+$|_g8DynAIL@CYx@?5}on#k0sRlQ+v3Z#rYmT4NU
zo$RqDSSVuV+eKYQQ}s??3*;2j58cG{Y|a0^s<}&!^@eS^G@J8S+xBlMtGRV}!niiB
zIwbZkD>Bw~T6lfvnnRh#Bkydy7J0qMxjSr2yG==U^kY{2(u3XWyqAQ>@E0xXa(NQR
za(ViJZ8uyGvb|X!CG%#^rA@6|Q$MO_Er|%Y<*~8zZ^|aF4G{r)*^vu180V?XT>fQ-
zP}GSx8?{m-Zbo`+5DQXBnefYpTib(i^NmSQtJPLYI%T&Rua~)0xAudiYV|d)u$%<$
z6<?&;sw>TYho<_;?L8H8=a9yeBc7QPq&U{Kg|FYB*fLGyZBTN`OZD&qm&r#>mgc`H
zn{)rg+5Dq9`385ZHKg*~+rG-Kj&MJ*<7h0O-Q9=p>%U9?Px!|3qnLHwvKK4|&R*8N
zQL<9{dGle(M2_Fft3O<cwKC*cRCX-zux#E*kq4F2-ppc|6m?MjoY1cGpMMoE%$j-c
zt4E>jWSK41{)wkGQl^NvJFzZ`X1JUi@N3%ECXt4USH5Z|Q)WMr+*xsACQn*`@dTTh
zr%m1j_e=>7Y159r$(!<F(gKzbGb$rrH9ikL?N(Zq_RK_UdG50Wwuf)h_@5nXyD6p3
zy?IvpTDI=<8ae&z!dNmA1-&EgEPS<Po~n~fmq2iS>6RTgR=KElxy1UJbO@cldZbw*
zuxswV@C3))8v)Hny5o*`oYFcpd(o1Kho|hF!PN92%i>(r=8ebJr0ITpXz^e5SYK8q
z*UEL?ZFa|0wV7X@3SVO~ZIgu1#4OJz1yVnA9(#OBcigN}*!z3imsyodmTd6~oukHZ
z#Dmq0pHVV<#-#OMziOT?bu!QNJ3Qmion?MpZ-jLZ+en_=kZX0UGog6O>D;Y?!81Pe
z_HB#4dhc6<dz8!u{aWs~KArwCMNIGBALZ}y{gYALvGntf7Du~>6Z-GipV$@KrarBR
zZ}ps~+T}kx0$d(Bty~++=6p)2{hE!GrpSW7doG@5JoTtiZsmfT-a?v(oE9&=(8?3^
zY37twA>Fz!ykbhq-u?TqZ4Sq;ww;Z$eCI|L9;jKJc|RmX^BT*B8OOGCm}GR7C*4a~
z-W(G7zt(^Ai5u)q!O@vnK_VgE%e%e@FHG~)Ta=TvrYBN6+%k0Kwx-)5o=!?@Uqzm;
z-L@)h|F6(LZBsv|-aS|2*q@QTcJIZ4+`uJUkAz)mUmd$PUOX-~{(9BPx~`VHtKY8I
zFbMhodV|Er*NfDy+&IH0fBoooiI4s3Z}xa5a?dE-c;=<1>x*e!DQ@phT{vLmvEqzb
zTI-p4=XTA%omOstOSN)RzQW7fS+AGM1!!`)^SRF5SR1XoFlkHZvAtp?^Vzbey%PF0
zM_<px^Yj}9t!>^~NuB1u@*kL-$W%FO;?mj3RIq1WcMgl*k{4=w!=A4Ar?$`L!UK`K
zeebj%N%^&Tv@ol@fB3(p)#eTNtOJKx6E8es=<#2mT4J@l_!7H)nD;%go+mW{PmXD(
z9_eKD?>k(h^i=3T(j*1W#~Pf%37q#e7?XIpdJ8nBCB3dVtYvk~$M22qCjX;)1?fp|
zJW_J1+m@wf9P#W?sgdcbS>{)(bt>pj#)>s<+_w@m&#5d+T6LQ3QNyVOes+$jUS4<d
z8twBGwaU((_EQegT4eU>h}o`$`v;x|-dP~DKlk-Dt@;|~vkDo{CLDTyspqVL$GJyu
z&U#!t>v8DSx}=9*9OwV^eERlARQ-KK(!|yS%<W>nUvf0Wg`Zxm35;pdkf_bL%;yx_
z(b3uEad}aP#Pp4l%U67J(Nzjclls5;#2uHnQu0bY22Hb0DEyf8Ub<Ib=AfWV_@e87
zbZ_u6=|**#_o%pWbWS)IpqIDoX3ye3ZkZxciIYA#W!!4yS+aolP}=&S^QUh0=JF)(
zN#ayo;H3DNcUsIzt$PoypE{T8wLm08lkxYHcTubMoLMmSP{0Ev4HeeD2OgW41=l_B
zD9U%)VEtxwOU$ItHLf!?^3<kl+G?oVF5-NAr`fI`<8e*U>^cLD#sH1vbl0v8Pp27h
z%)7ujb(7}IAg<{~4D$+{&nfj+D7$98JpShWvkOU;-#q76ozuMH^x}%s!kiY-4ebja
zT~wK(m(FrPOp5E(odV_;i!^`mpLw^yAm{K~Hz%Y2IS1!?9TbxiG7@1n=5h|Wcl4gb
zVu7_xwPkOo-pkX@<}{nIIPP5135Ih!fBEH>b>%VS*SEbgzQ4$#>{X~h|4O0v9|AUq
zh6R7Tpz~?lsZS|_V*2{4Q-ar|OunG9e$6R|D5v#NTar~8gqKugeBGiH5_JB{r!33l
z^%66Mzb&|Y>CMs&1?Sf@Wq<eMmHx_lRVKyN;>opRg4bsRX!YmY9RHl8cA)wO`>!(3
zERXD5pVPXh(^Dt$>7*QUN$XDAv3}dT%@c}Jw!Qf{@tjjlL%05)z|%)~6_dL5#ArFP
z8z}C7ry1rojc2k}%*s<nEBzk5UA|$0T&PQ^iQvvQWn*7YmA`NI{twu~>=@v>I$8DV
zVpVqCiUwBJMprj;f7Zef_2Lq?(xL-ZTlSl@A8=AXePqSsBOMy;MxKs-9ITlMvqBVq
zHqUBWu0ErHbLt1prI)-G25C*5<UKcllOw-k&sx1*A9TNTRv!4IT>bcHTMEzle%+%?
z+g7EBWVs!GvPs1vk?T^@o;M#(Wprup^AQYx!pQ!}T~Ap*_Q~lZZZT2KAv&*`mYqJa
zYpLNZ=7wEuA0!ql8m(yj$2Y~uYsI=e3zy5m=bD;yR+#u2hCcTR{d;_IR>p;GU1sT*
z&#%i0OEqn6dfIx3xm`>B;;m-!uPICFCbl=NaP9jTF}L{A1VQ^r6EB(QO5SYk{69yq
zOYBhBJdO?P1g*CgU)j~pt;J%~d!F-TaLmcl7+tq5OalEvCH+0REi0C4pHA31>8#E)
z-i22}w#&RUSMX%0%a*_G$nZ<hTQ0=t#OlmNJpDhuY%Xdu)m!SUZxF<u@AKL1l*Xsr
zkf(8n17?_P+v_)J1|Og6?3b$18#gc0Q%f}6A5pZ%OCvS&(@Z8F^@oSmb_SNz8O7y_
zc@~CwmWny9$h=yU#I^8}=I#>j&a#Y?ljlwNHbv}2%Bc|DeV^QSJvWay^ZbyIM_-@E
ztdxvHOvfj^)4cWI&m|_k+XZ@N3Vu62oO*L4K>uRmWN)U<<tP5Bueh}A?X}5`tD55E
z4DQ9KaQ}asb;iXt=!#BK^<kIZ<GWm*h6t923HTQZpEDJ|ZX&z>49i-x<f~!2*F_D(
z#mxObS(F#gW3{j~7jK+;anI2u6F!&p3tw6=`CG(y%joa1B8NJjcquM;@r|u*{ff<=
zzf-Sl+gyGyDNyT)`o4=cXRmIWBfxOVl<|d|w#2n99%@^?n5sfo^F}%9__f%54BqDD
z^k>`l>KvDCCC841XisJN^6bcQxhA%ZH7bju9x6sS8|i#WdK|W^fotzQreJox6<40=
zoz_kXd8Ks7`>jM1^TV{WOnJ9onLlv5Jl(3y)!xAMtafzRr$Qd}M-{Ga+Zmb-Z?r^B
zVDaB}LFU~)CySy#s~`QZnc{AI+@@-q_0~roqDKxEnYERdIn4NCFC=2<b54DoiTB1Y
zoXid0^*b_xpKzt??Vc4@HSb&HwJ*xKEiVgvjy+4tSaPGfq(!eTEho28#9__as#nhK
z&h_hBOC~<zv+H(^6MpyV=v$UnSGf&u#U2NqNHJWc^InQ$xx?k1UajU1AM@XfPj@jE
z`Oh9IxLqWC>xAg7AFH^+jK7EJZ=WbB8E(15x<d3~LvV?re^2w&lbbf!Z3?(lbs}==
z)ri%8QPV>e60S$io;t<yZPfCqQ$5#8sh^TcjkGT3*Hv(-ID9omn4xlF`1Fa_WhOtm
zKBZ~Pj||_o<=O#iFJD}pFy-tvw@kVJA46}=J72RtLjT`Z*>x?<0h4lFMUMKgm~~_}
zu1{LV(mbW*tm2K*{|>IF<(PMC1)Z$mHJr07ftxQ`zaaVJPL<%I`OBFbjyR`?R=#Rt
zpT|+fWhSveP%2C~N=;Rd&3i|S<-sB`-h(j_ZMs!$?(f=;<}UU=nBH3+)4HwQ@mKq4
zHP?3?A5OU1O<g2X()avHlTLNB&g3H=1wGrAN!ah*R(;4!@3_0qhPAP4_>6bo%+vnp
zcj|#*PukaeOr~NQZ*@*CTgB9<_H`*+so?ITzNdUApHMmN^H9uZVc*@>_fwKXx(>&>
zoDnt@s4vwj^fg}NFI?oi+0=4H;my^{Pb?5kzC7{&*IyH*f0|kPN4?t>ZYy3Ad8Byr
znKl#4q{)-NhJ9ZmV|_U$f1R9lT*`FG=_@49CQLO~Dqnj!zU=+xuda8}B!4Vpn11?l
zQdQ;j{_?H$GnX_(&bS}E@0exP@$0hT%O>pN-?l3}?SQ!3uVT5<Cn=vq|DJR5x)5Ho
zZYS@u2Ig}&oph(2>~a<gKjI{+to3h#(|nD)LQ2|E3ynoG{I2<)^m;mNN6#(^=jL5Y
z1O=~(9MLg}YrL;B!Nt9EcIzr5%WwO9g_;}#R^4j032R|}*xa&=FM`dcSzqD^&n*pu
z2nLUigPe21xolcuC#AU0a0@JBO*J^|zoRHb;^2oVAu&4+8dM$muRqJt#L8>`y2X19
zwIXXWwwiD>S#XL}7aTllzGu_Yw~<-bA9Zdn>ncB8z2}<Bn=Q+a^3K~~An;tEK=?6V
z)8ZSGmib?5(LHfa@GeW&36_*o&sT^G_Fm6ZxcWTMi1W;=w+(aB=6XNe`}e{+i;2Sj
zXY?0xntZ$-tRZY_Zo2OF8WYx;6JFjqyFx9Up=#oHs|!D+#29~xZB3r!o>U@OebI5%
z<Z2HITggi^w@#T~ddW9(s_yN}a|)-<4phi4U$o3Hy1IOt^Iu1=-;3>M#Z*Raof^PY
z_Dpjb|E_CHQ`2gr`f}}FPSuG>yt!@qwr7H~&aeKnWT)N=N&PjJ`ctZzx6SVFlbrb9
zgTd?!i`Kg%{)-M7@Jwb9R(f;Hd#PE}vL!mEQ=Zk^PkSa4U^=gR@~ymE-b>FE{ZamE
zIsf6`yWell=d|8$=D6^q>_K*`gAX(B8}2wDYI&b^-v#BqM?bn-noH7FW<C@=b2xg&
z5ygE+_IEDUw7ZaZ+}lu2Yq|la3d6zbMR`|?r@vYG%;UUIv=uLF*X@EO%T5LuXPn7C
zukU%}o8LmFc{)sU7oRb*NPe@!`rReHoIOXx4)hwPdrcP6(4JGJDHFV9x60`YW@iE(
zPnmg3NHrjFyU<RNvI#$bgmQfs-u>@2=l75(;jlZm)=F+)`%voq{aY(VA|f=mhiR_g
zyus@o>)v;NtyuqSyGONb-oU%;{q8*_M<bq8a+KZ73;4iQrhL0JIb2a?cC1nLN5$Xv
ztBlJXqcfd;e{!F#_rjxnromQ5g}w(TwWAp4MJdf_yg6yfp`ezve*!Y2Cae`*74B9x
zw|Y+5_V`&x_K1J4-YeT4t26n~j=xjAB>w0hIL614>~}{&D~a>!A-Be|fVUy5csD&V
zS>$y4;AuzK=E)kYOrcqETYR#A*;v{c84F5;2v@kQW!vArvhi#5z4^`&hLPd#V*hdP
zIq<z!;{oUYpSB0(_qZ`8aZT5iW)U*kaWvszBlq@#8;4YtT=e*r`JNm)nRKi}+A!<S
z2}7lsy-XgnWHcK@yn<9&IW+zsUg@Jb!zlaQo=ju)b91e#KmB=WtZ`wXQ};cQf-NGK
zmU^*HcxdF<*~Gxv#Iit2<)DZW)2%H<u36neoL*=4lyq)86S-CI=Cp^OzUkiDS?sxW
zQ}*<A_x9HO{*|iQa8@H|@rfRxf*Y#d(_&e-WcAz#vyNV?al7bdR`(RMb#bTmJU!LR
za3S}>v|Xl~MPC`MN-q+8a7s+_+V)d>N`-%IU2VOM^-f`F^vaB_vA6c=-CKGjTkq8O
zePRKJ^h;i<eRC_X+?!vz_x94=Yx1tSh#SWqnXY;NX-(&slQ(DQub10d!T9#&`sMcf
z|9#ncTJhI@jev$24vz;7Y+@Y`8oAU?JZR#RO8KwgBEXdr*eba9>lB6<8&UC?(<isi
zX;W&GP>Pkk^dvFHa4AQ)&&#DRk|gKeP*d(TURIvmYjLY$UZ3eLv&Ff_jG40~ov($o
zO>~v3oGiU`M_}9J+$lz>(t$ypOQ!}U&78y#owM*j2xkDR1C!^3#PHOm6BJunJUtqw
zWz=bgx^br5=u{7nu{^3iD|FH&Htn1VOcK(K6OCRh&{I7%Bd)D#d&c}pRoB^KV{Rt(
zr)=F4>}?z87Z9nPIw8m{Bc$z_WM1Bcg;|nwc3$Ml+yC`iSc%^)uGqr#)a~NYUmUq|
zHXP#f&RJV37tOTkl-lVxo6ec-mfLb_m&>9y$;=&`;y0bP{C_+BUY(_@6RWDpQ4!YO
z?7SUJP1XTU@7lQ6ZRc89eX;((XUatvv3Iir-4*}74%{!e^$6!dHu0VUPd8nkki$Io
zZwijMDXIzwi)puRYzyqH3>6Aqa`RMj^n|4jLb(e=SH|Tntq_YnZ+Vq1uG}^(=WGUJ
z+l<6ELAPUSnw-ai&NN(nGwJ-=A40EE`!@cZ8oaHMZK>{DSIOBEwnv<7UDNN%Io&&E
z@1ITa=YP(cyFR1$`X=vRFORNXcVlI2ak<uaukY*g7MIVfOzqe{uOjWu-FwwFk9*^`
zG(JDI`n2Cq_ha?KOg|Ewn7#b+pYq4uOL)fT#Zl=ZVpqD*HNREPs$G8Gy8ntD3VjyW
zJ9T!+P0uaZdqX+K<y2htJ8Lz|>X<F+l1n7FU1r&n>vyW;&(ssfJ8M4sC%sV#x@d3z
zbXuIBqG%|ihk@&fDO(Sn6wBYoDH%I==arii%g?#34OUsORDEh0=PQZ7aoTNS$!knj
z+0XB{I<|aDL2E|yrC^4o&Rb?DdKpb(RM1p8AQZhK%iLYCA!ET6oy}YOBl2c@vw#2O
zxZ|a2%YLhw@g66{g|aTR)P9_Ld&1N?=_d`MjC&M$C%Y<dy|BW2&W+%0F^AT6=XgiQ
zXI<YTWW-piD%k1O67GC~)m2^Yqk>xsN2^}bdYyNtl^I@lh_j~&87x}TzCO|OU`)}m
zY5zTPdIhqTrSEeXb|tP!k}UQNp0`NSP}9-;?GA47B?}l<Y9!|Ard*8sW-@8!$uns^
zC7cOmVKWY@JWFq35@FE3$~E6tEwew9RZ_*FEvV#}XFzrAjSGil_RbN~?M#-|UdyZ_
z@=4YD=EUe*e=d1>xE>GBy6IPRW!n7jYt-#3rTHKJ`F8rpA1%)#iL+ZQ5+)q;yc{>-
zu}t&5*0ZMq`HJscnPYCg-IcrXoYOk**sbSI?h?{m<EoXFUKQptx%`-_`-e5lt!qUV
z%dm3Jw-K83amMrW4N9Dzep(t+W2P=`&dc<w`86rTW}4cm4W3M#1(TV+O)}cD<&f95
zlauELRPO&#sro;yEl6Y$-&T!(Q%xfNU+!k)_0aU@OlbUi<I1uiB}TtJT44)UPUa22
zabm%|fE6yfT4iY+Au`V{m~B=(JdfprK!#WfQ*PGfh;^s-K95;ld+XJzz57)4)zl0-
zgMGz!-4R`1u~d6@&QkH`D?EL5+lto}zrB9&n8mT?x2%0NrHkW#?>g4axvfv?)MF7D
z;Y~i1T2s`Tr$o)VrMQ2A=fp0PRdEGw;s-V)8Tj37Id_4{#Np{vLq}t&OFEZNhIu5K
ziMw3ww2_dF5^C3tU7Wh^Mev!;TYILL7&=}*V=Q}blfm@0m!}q7F1sW2?bEb_zTFI~
z4&VOC@XqFmrcm#y)4T7?U21XR|C0kOW(yiPTf!NIIPzUjMqm6g?YghfghLKrwzOvM
z`(Ai0X2R|z6VBS-E1%7=Xioip&II;Kz4#@c177TUpjfPBagZlW_SLbr-69wL4((gL
zK{K)4`c|ezS=x<spMvM-pPQaHlMkEJ;a{`oxhH>>PF&&FDTgNM@0>2Oj^n1<k)(?(
zF(Sqpmh!P_OgA?i3{l;D^Ti*zRQqEZUJSpBJ<rIjOqpsRx#gerf`#+C)DpHbZSIPE
zkn-z>CudG#NNUC__tPKjrcX%lHz}DFf8*1><r5k#?R>*-rFeQgxt1H@@HJt>hckDs
z=}vWvspMu~u&Z^WU*g8F*f&4Uq+B$2Rb&jj(DQ$$(u5Z8{ha&f`6lysf0N0p?Ckj0
zxhP`!2A7cO-dBD8T-qUY_ej|0sH<UX7CFng3)aR?jF0~FXuliNv4s2+DzP6*{#`I~
zUvMNv%{ZydcUG+OJO}9w^=r$5UUa<e2|U>s_qZe6YO&<8X*`oJbrz)jRod(tWfsvo
zX-T4h^K=b+w~Y3iDp`WNqk<$i)w26W7TH{xvrh9|_SM;M4ZkzZv(+!xOY8JI{8C76
zi^?AB3$oL4BQu!NW<5=L{?NO8zc%B?u9`isd4;R4|6bO$Y|h@tlbwXN2~6|NKK9h0
z|K>b9TcICLdc`?M*YEoF`}2a^<_`bo#xyXW*~yz*SmI?a`-bO#<l3f<n_bsbbJ`?c
z@hkI_a8>ULe4oVfP3~pzn-p;khlweV8jW76a))L(tqoDtD_ojj66Q5KXkCMX21A%;
z_tg78>y|9D{%Xo*lI*poX!?waB1_w&GQDKwXYAc#@=33>Ys&mr(|exvFEh(+Pu}(*
zazgHvPq{mMmc7}yDsACOf46lZy1SH8x1?}c7;Gs%`)f~L!pjRgC){fCR(l|%$-%I?
ziD$`2)}0ZLCaW?=XSm8oI<{%JC~yU>s1w<;p!H?Hqj1~ALmF8J`*!Cf8kI#(_Ogqe
z^ql9S{o=6fNKxMZp~v^f<mygeG_zveg56Sau79*zzPN@no^WN>Fo>UGz|Qes^UAV@
zYsVV$O(OU$#HEjNs(cPm{cN?)NILM5SodWM=8L)CC)-b%T7CFXq_wcb!teI}Yz{Wh
zBzQ!$`U1IsJQbR!#IR2xaGij(qH*xL0I5PpuAt_qX@T7P1&nkO`E`GYZWFX<UmU$B
zUCBYzDfO{f@QLazkHqr|yktu<m<l5F7}ZNYwsJmbeZDo~Not!&Ytif0P)&EXZwI)d
z4{*Ibz!hk~wq`@>>`?9~1Gcvg8kG^;OBS%sa^RZ%fV(z?>%rvWk4rl~Z!LB<SNNjN
zI)S6pVY#I1a$lGB&i6?rcaQjbrYkTna}_r9;}f&uewMJT!|KYXlGQsB_df30m7KV<
z_`kKNn9bG*VR<*<b75R+(`+=&Z1>+t@;@YWMY+dA(70|(8`py5l&1numgL85WV{z7
zQ2bFfj;Sh!LBnlZs#<BzkH@{m3!7rDY0NlkulY4-!XweDhpF2F_^od6MQz~Rn!vyH
z0N0uT{wWVd%N}@^CGu})koAgSn;FD^+@jwtgWLC5AJYTPMNa8wK6$GJ<@se89}&&y
zRbr4f5RY9Up4cJQ+#(kLLg<vaSg+colh@<ksIqP|;JOvgUmd}H|N5kd$GHqG*mO19
z0~Hu&2Tb0!K<N4O)ab3Di=N3(Di-ibsSh#;n`n{knNq)T+Z4Z<Su4d{w|#SA3lv<S
z(V*bipb+>!=TMjshZ@7&3qnbmIT<Ztr>Avib_!oOHtl?vNG@kY(Uk@(MR&2ox#c@W
z%3Ro1l!dB3o_>IRhL=NVfJ#kJ!^HO1iTkF{khIiNvS>PRDbMC{UYDW9?_`b7Yz`>_
zl3NS%*G_0JSTUjg#LW7LwA2lp_dZK(`P^)pD*iTs|M!VL{hg^_GPt)|3b$@H6KEHc
zW}jGfaCUm3jNW4B(-UQ77ffVgsHs+LOIjgr?Izx9S|e>KrZv6oO6J@*u5)$6xz|o$
zx&EBnU}xl-1?_Jh@UKx|o%LZ-;ST$)3pkS}<f$t9&fD7Ywbb{aQ0I3`R*#iaZagW@
zeLCO%`uyo9I^DzdS+)NAd2R0sn7+VYT}l6=RrvJM*mD2G<)wB;W!9Ul#iGixwyT&J
zsqnN0=m)AQb_Eo)tFnDf@tP=Dv$7}n;hcj^>V{U6-l{Sjdak-iRHHGdU$r#o|749R
z>`P{fY7~E5tXWYxy}fVEf=VWZ{x>T$eP&5UC1`A{?Eh^MsrRt*{|cGZfY}==TeK!N
zXEpY(Dr~WLow!RpW9oOAg>7PHT<X@(=E%Ji($`{8;G9%bxV$QGl1k<zRm(|Jp4D7^
zUSn{4@+?ielnZ<*AJh^X<FyQp@*h<%c{DjK(j>!hiqI#$5A9P{ESn$V8s=M4zdCJ-
z|II01me=27Qh2mTk2i1u!;Gn-kN#&fUY@%CqBBRv>bpOZEe(}3c1|na*>LjNYO{|G
zEmuV<S1K|q<z09wE`D^)nU}4luY?{4FP<`0eA>~}9jnCW9A%ql%4KII(V>*e@<e26
zV*XqehA4(vrlym>YedbfoY3GHy*OyL*JZB#4@6QFOf?_cO0iCO(I_upVxIPK_9aGN
zy~c?TOvMdXCChitu@)1}NHl6*vZh*aBU6Hrc<4sOn-j%eXUJOH$(=4z{Jn9pgX9+r
zHctiC<tv3QeJ|2&-z0Bs_i&QX(g1@`Hyf{n%%3%Je$vGGH=1n&H*Wsh8Sm`f=@h-?
zr|SZb@15S-R=(1ayx$U*XL#{u$#DO&TFcSBp5uSg#vj(QVJh;=3{;LO@4k_<$3shT
zRZ`5PZ3liNT{Z5x(v-#cU{P_Pkw~-==YuuH6Isq_wi#^ibw4I#aXF==P<*;*>Vu5-
zrXbHd6&h7S)t4srm2B)YZCyI+;u5Vu$@MEm+;8+veaN*jReBwRlwYH?-)9y5iH$2t
zWOpq0k}ArWAzowpN>*Ld<FLEfHfxVup_=W{%Xhw3lm0nzpZ4;7tHsORcOTE*bFzEV
zo1>eA1X!gWsMc65exfQWYAW_{>HLco8K#a}Q(mb~lgbMFq8hj|YhG073F)a`p8Gbr
z?eo>lUhg*Lh0@fpN>`DhsZpG(qjm~zKIQt-eYMgggL~HE$ISi<XLL?mHgQ^kWG?p`
zgUe!T1cRr0Ca$^FmLuw;dt%piVQ=v>Hy1HEEI-e+@rUc$*55Pk3C^5&#JGKRV9h6q
zS()5(el{sF7c&*K7>CX-J2-RR^PO9sEAp_48?f3de9?Zz5*oarHDKw+s*Q8148y8k
z$K@*2NJPg~e>^B~TU;t<qX_p#rPoDymyYVcKB{;6s6qGKH?DhsTIzKNrv(QkPZN~h
z7<j<ydzhe*CBv=FUK8d!Sc_?RZ}}G4>B4=YUgtz@&lcva6aU>4#6F62*CYntJ|T1}
zVQq71zs*UZSqsHamF{gmIs47Rze~!}Ck7~asVhj8EALXh_}R4Hgps>j%zE1YgJRig
z-g}PJZj8Fcv99rxM&L1xKBg!yHqD-*z740va~3RV(Vj4O``nyGGj}AiFPo$z87jP1
zSm*Rdspt&7jn-nt4=3gcoz@d%RnTFWA;wrN*gs2P_qo$--xTa>xioLDIa@t(_eIXR
zhrXY^e<$LMb?v#?=Pw;UfBE_OzdRS-E@ccfU}V|Inw=&c$Ebbw%!O?U?0mcz`R`sh
zYolt=DD>pd-hI!F_bm@weQf2k>PgS7Cw(YN_R3r(=Nl>)yK2s>`B`6AevbC_`MruU
zqrtL#b?QwaJ;iBOzJ|t+E}MS3?D%%t{$B_3Ox85DPG`1JD9t{)_Knb*H?lL!7GH@v
z^zoXwYQ+DhD#a+BLUCr3b$St-`U{gc8*Y`p(xTwn%i1mWs8uY$cYBh@a!!HXN68zb
z4hWfLZH$e{JbQf-;}Ndp4V>FPtWbQt=J4)~YRyNBc8OQmidXzSs>gm)Uz_!c`^~6=
z+Nu@pvjyfETTY(2ku``}YN76~#k!{Z8E@t1+}i!|*fs_GqJwqCe=KHloG5>@*-L?S
z+g~-WfX+If6OC_fJ93xI)>z=_ef;a}6N_(bS&`xAKYIc9wmVbKY~5(ly}o1XRbBDz
z$!t3^4w!7pDf-f_)3{BOt4A-a$4gM~=9=yPb2Ed^Ua4F?XX~Hsw_kYd>{<KuY>L=U
z@$0vCOk+P%^H6lM`(d6d|J{9Nt@!yY#A-{m)fc<WW#a9rH%eEnIW9g|F7jxk&!+bp
zwX+^@Ga0azoZ#~@W1AQ-nQQ06<N$l#5dN-Han(MHyL5MNe0{ci?($;8b4(4B4du=;
z$g)~g*3J^>NDh$rdV+86CC$A7{QDm8d%WZKp7+G=F_+NaCw#I`dHy~LwYwM`_w;!4
z{e9`ZOa;jT7oVCh)3aSY|L4uPtKyfm@5L<$@0`40^PH+FZ&vSHt>yaWw#(MCY(Ylf
zDm%eyKfyiMTz|cun)qeEe)DDPwFay<S4=)$-W8F{_vVGgM<JKFIc1$!gy$F?wG!xg
zdr$7oLGIN(5B}`F_-gz6$g65^MBbUroKo_?VmgccY{`}-s%bqo0`1RYg&AtB#oEqE
zzPiA*uXtkI`O2el$^82qxF+ZJ|64G*wq#bZWBb<zV`sZ3jkRx*cr;r*b?)C;ma&jk
zZ(;l54UwC7uKO3j|KuJ2w+H-xC%p4n_wMb!cdzpJ6AN$cXO#M~?$%ekTPx?@I;&&4
z&Oo^Pa$T>)?a7~SPwY5;wf%OYk681W5Bu(Z@c2EyJ?BJGX+l8uR&n{0d|e-fy~;j?
zEd188@Z*w&iu?V4Zz;<%Jf*3BYJtuvwNqu0hlEzzEML3l-d&s1^ZJ%AU%8}V(`S{f
z5;mt6$1Vyg*_culsOj3j<nLc`$<mqMBZB8Jea$J@SI&O!jOPEV1xDv8qaGh*uZv&7
zDwVKj=h<^r(u|E~H5V^ny)f^Q%4$x*gmahVSYMpvQ=2z$@dnlleUs%+Kj!RyY-YE6
z%G<Zkc$TdB@X)8~$=?Uuug*Pr^Y2~H{wMGFetN&-UbBI5t;5sl`A=u||D2t#Wsqnf
zmD~HLJbCf?XOkCfzS{mPUGI`w=db95UrGOtr|5Nt9-Z<_yyWBb`dJpoe=X7vF}U1t
z&ULfapR&@`Ho1Rt6|bB;f527tCBMu;ci)3YPl?ZAn0tZq>fC$coD=euW;iTACw@!H
z(_b_y_L0Qmqw0RMOpL!eq~!Fbu8G^nw>GLF-h3bH;tO1+Tca!%N1f_ud&eu_$?~5^
zWJZd&ZP=2WjjNZQoYr8-6ycd@S`cE(xYfW*Mx!BQO5p4zDw3KG3GP#vmzG>PRJ34P
zhp6?YRjLQXz3Q3XlpeWpe!hRbed{&ps4MD$^F9A6#!79OHpOh+yB(WUuTGU+y?ssC
z+*4Cug@s40-lZ7o<}g#2X<LEk?y^(UG-H;nn7aF@?j7BrPtWEmrFIEvHZn4+MOu8g
zx4laAee9hb`j2)M-d8(j9r-Tx<P6jFXJ?KU-#I_;bj35HC=+p`r8_wSY;Toj-&miz
zewl5-x3`yKV_u)zd%OJoowSDO4}JIWs6XC2ak}NXJ3h;emd={3tGZyy>{r(}Pqw|C
zR$cb~+Q#nxdH-#fda`;7ad_Al>^=EiW3t8zjn)w9u(c5z6;&smtz@v}Dp+9Vp_shj
zA4>-FQkK9+?;VVIL>?SqWKCE+^?%T+<I~D4a+b>WMsy}|2i-W)FB_(HNu^06>Wb53
zU6qMRlUufRI5CvjrMONu6maU4?Tk`%ZBkJQP_k84ec^Gku|ve6J1XMhBH4J4nL#c=
z4lG8KG%{pd8GNmF6)w;SlhSD7I?pV0qOVF)WBHT^J}=c6o0v|CFKBtietyB1Czs{z
zzV9i%u+&W9C|~r3UCI|%OSy`yc9-4Tb<uUBl=rTGmu|VNZG3YpNqDyvYv~pL<Q~Hf
z2UHwFL&K&fOjQZ-Rt?qv@2PenyKCRCAkpib_n1mI#XWpgdgG+ogfkJ_EEs$v&(C;z
zD3Z@ZxmP%I`POU3S>dO{8geZkZc~23_9-l;;Pg~J%a;qh&374H$(<fs{<w8|?8|Fg
zzsFXy7Hp5J`u)^=_lq;D^A@~eG3$TO$F}Z+3Zu}1e1%`Pgd04YX6Y0qX?_tddTe6-
z?Z)G-8xpNk0#8U(I<fpzo-id$L}bC!|2a1|91hF+<k9W><#YSgz%2<&=V@f$oHDi5
zLs2!O!)@i$jFyZ^EKAzlPVyho?U<s#xcaX5OO@X)cNV2LZ!rk&nzqPck?Z7h*`5r0
ztzNBPG>!dMr#hqAs}=Ki+xv0mFa34xbN>JR%>S)d=$_!v);_ZQ(yG-*`1dasU9+WV
zG0&SbN{e+F&348#?7ViV=CbeGSv`iwesP^CIph{38@Brd!_{@iJo^7$J0Z|~X4AQ%
z`#Imw*hH_sbtOpJ_s5N{{k%JGOprfo_B1;C-qz>d-TN%>F1Vaq^<<rTU-aWO^WVij
zv$8zh^Xe4G`9Gh|%lGekcd6XoruDM7KI7LbpMN~;+jgRVD%01;`HufoQWXCD^l#Id
zqA*itLxXJKO-A+%MGhy9RFPK_|Ib}sIP>Zv*|jU&wHHlj5L8s*5|n7wzh%fKkn7Z}
zzv^Urmf@zd4NbD!6po%hrohS9*O;mwe7dHqL?wJ<dXjTW^nVr)Bd0ABS1M;jb&4q&
zPUwhu;$1a)@*I{1^Eo9f0naxo`Tbzqaj-G4Ao)wHeGI#%r)F2+giWqbXE?VRIjj?v
z7HE(ba%JPoSkA-tDNNp2T|4sc#E|9`o>fa0tO}70nz)9mM}>iNmFEN1NnIO7*L_g3
z+VLTaYwxBfB0Z^VPHlK<l&5$$$|q7cWRX#H%z`lOwMRSkwtR|y?8T{j)O}rJ2fx*c
zMnC<JU%lhmF2^6<BfkI1SN$Zl&+%;95yt%AeEKbQRKG;>|NBrb$C;Cutn+x;y*)pD
z8K<0aQcpgt_AB`STTzYz`=bfEQ4jrQ`!4Ya+Hllm!Ug3Zg+`ID3eJ6t{u?YU_~euI
z@5iKm6OD#48_)LFbNH`yiO5gUP|S__{oMY#+0nk6ys9;qlkQ$>Im(zR;-DbAIPl*w
zk);N%OKX){&5klni#AB-R=LsiGtaS&*C(MO_{3sqwF8Y|9)4VHn_V?qLfFn`o+?eL
z=nk^qe&zbfbtQF<S49$*?mDpH+{F2ZPrv!GN#c~sGs7K^V<StaO9uV&Gg)BRo3$&!
zX2&D9b*nACW8Th^*(2t@e*a1B_|?Z{?k{*9{X*G4aclC8k5icHUN_%ws`1$W({zop
zz}pk1p-ydKU-IplZ?17Sy0U=Hki%i2tj8Uhjs=Wf9!(-V6FSA7DDa$GBf9zS0+AQX
z8~^j&DibzQS*YvwQPFs>&cX$=);5|=-LE<`)7Fh^!el<n3BEHtl%)<%o^4enH;M0r
zdVbSn`!(7P?=PKH%XjLmI9h#In{Ac2=g;5;w@$}7acXJQXR(G{yRl;B3e`SU0S5M%
zi){1Ex>lwZv~Id}U<Gf^LEd!*&S^hq>^vj;V&Xb!=5S6+kIN@dOpv~}p)K{Pm49Q`
zs@+m|)yy0`H3d%VE%=fWv#nu`uEXi0XSzOa@(^AdlJq)ijfn5Iq?eL;M~kMteI%#1
zzxDL9JFjPWvOU`~<;{~iUCEijYRgKQuAR}g4^HHgo?s#^w(-<5&1{uh%O<d}_?d)j
zq$q1k&ejZ3>YVaFqn$fT(mMO0pUw0#?$1+Bcx>@(Q+ctW)YCxe*^w=iCT-|V%(UZ_
zoRPGAB9q6G%w=EPk5;^#ly*ygt;U>ivFBYzFTNB^-t4Sxv0y>ij)zB6cCd<@Ikc_Y
zvqL1b@OhvmV`z}g)s-_>Z+NXFbJf%9>gqV%n4s+ES1)R&ttl|f-h6(S^;V{SlOwaU
zx4vH`b<M&5_Vs0P+m|nwd7QHT_PyYHMQ0A%ec9T$;kBK4^4;rp74FmOUN6jP{UF|`
zyCrekpWWx{-Z7_7*%Rhir1l{5|8ye`y9>E_zx`XnUK=)=?{WV2U;>+cNc=BPhLCk$
z8%0DEE;csABz@{$k*axZiNWF{p8s8@II)-o9<y__>K0{s>Gn13vse7bt{LxG1K+e{
zE_i=Ze%pmRweqfPbNyO-%N`tJ`E@(qNMRLEutKA*m}`@!%#EO#M@1y_YF~4oUUg_b
zQ{fHSmmZrkcN`A7@v4Q_g-h>`mj2Y7RU54GR-Zb+Iz?Y@Znyi-@4^50j$5kz(XM>s
zrT6DXf4Bbh9~D1!WA|mB+>>K8b>hzTiAnc9o!YX=*!H|nhFZ=nO?OX*RW}WrMapNV
zpR0V$s<P?d44*CO&mvC$x_qwun$LN@t+$-jxfR-UZ+$%|?jTadvSaVd+)GyS9%Tn#
z);xReTkz$9q;A+N^C`1TmL}g|T6^#k8>fVyc3H-M=CD%=jTY0Q3Ue(=rSIrB@Unc>
z-k#B_-NRM3g6q@ufYR7?jy%?h+y{;_9H=?HcahMf(=W9Wt9foIu1>E$aMyG7X^!3_
z%Dqn?n(-Xry?w&r<;&G~PHek&L;JmC-=l=Sr!)8(J=Yx35;&sOe>g}-yk_kQr_LWM
z4t}=a?47N9Hb}DGnx8q!;tJQHDGIuZb2zUCt>x<4!P&W%H)|&ErKX<W3$vLl85cO^
zFFK^~NuslA-I<lF);}lDUoqL_^`aNIHD14Je}8Ir-vVLFm+ZX~9kX8uFNj#bG@&(A
zqB(GZvxDLk+drM(KOMG~Z2b{!wOWG1dQ)>y$F!3(8&-IWxYx+c`fp__)F9)wk=4Cq
zIaA?|4bn?C-aZnvaO!1475?s3z7LvyZ#c64#xB;KbA&%niQXt0@n!1XX2U2OG4W<Y
z$?Rz%QhQ`4PYZn|rZ8FcIJ3AC_jG0JW1B6+%NFiY&34n6yeU0It^TD=)|%;AXEx~y
z+qV4NJH=w><Q-~jX3nV8YX0*;?Q+rHIoy){8Jlm+UUXhdSI%f{)DGiWl^xS>GOc;I
zx8TXKwJ$jyuAU*yVpsphed~#ROEjEImPk)f;(GIHMw9?+RK~ugFWAdou+NWReVyGE
z7{S`iv&;LoiDp5o+s`dC7OlLPVb8XD<uOV76O7#N4{!dZ+!}OXzoXXff7Mz6je-pS
zLwW*CRM_rx&oOjZW#GV^voi7bs(>bkr6p#8i<%B?Hd}FI-pv~aR@^bm=Qe+Qp?CkK
zd3+*{2PW}7)0kf@?Y*&PdujCiV(a-ICYYaGWPb9Ogwbtzu{9mPEf%!=kZ)+-+$5~f
zlC97wVA1NmgVWLJnv}rhEDOdBtX<540xM_wp0H5(wL@!*PKToH-i?R)7%Ww9o>85|
zr0-%k=kX$wjf>_~IKNuWZntuB^PbsN2c#DYFhnJEG)Ax*PMBr0u-!1CeV#OPS;Nk{
z2Q$xqnLR(a>!C>3jMePUol`t!b{RD+em$Yfw8-V@lyfgsT%w8%d}~zRs)$MiTJ!z(
z+vIF|bjy*C|Cd;FRWIke#N?(m^+<=S|3Q^IiH2#}8v|Bt4AQds6Qde`$IxEN&EBd%
z`jR-~SJhL=ZYNLgsr<O~Y;-{89HX>DZWoHxS~Mk2Z8_6-aamD_x{{AazLGk>uFtL8
z3wKDe-f3QbNvXMJ&#cuC15C6#IzLIiU2L)-l6AtbPVrTpZ3|l8u9{tUVa2Qshof$+
z3q9b`!6Fs-;Do`K&Oiy)s*DZ=QLcH3>;|tqv`qctZ!nqt;5v4ZYm&_z|6q|nMKZd+
zN7sL_`?cD1@yq_}51Y1K=GHZsdXtl9qh-%AgPAv*JZ*YU-n^yepe$rHRqNyo|79_!
z&P{N*@S^FyXYXZ;-tDur1^%~q@$Ko|t90;y65qoQ4ttCauJ7S9H<HLtoG%`;=2PIo
ze+p|}ub5C(?agp;+094%|8HLSd?Vlj(;-Gj1-4lVDsLnX#5m2II5TR+OkO3{6O#_<
zEIITbd6JIGqHa%PwTDh!9Xp$kEPAKpY;<Iz+)gE{HSIkj42Baf)h%cZ6=*1pVAV`z
zH;ibFI&e1dMWBrcLsbN8AjhT9i0gJ=+Qp?VJ-&UoJ%!co&tbbM>#M%9)=9CNC>?oe
za^uw;krv&E)}>q_C9Pp6{NCMR@~sf;y2|8vAj0x&#HN)I5;s|7CoL42s3<HL*~TC>
zW$C)I7kgC#RT4i=^`7M#ox+`xX*g@{#?=2CH>UlZ8}iHb-yX3%PPc0s;)RCXjAxe~
zOEzMzHCndTMkzarEmkdi&MocLQJk!MPucWyxpMwW5SnyBt@w?a*jket$?k<*KH;S^
zr)AEJKYKYox4C`OtdO-PkC;80WIVP>vKsuFG3)ik#s#c_4|Ga$&jxPTS0~Uq-||lA
zg7&%_C)BR8$bM~CWR>!bXf0*9-gC#}>K#jSk=EWn(yx+P`>$-VQEWAwuwO59&eWLL
zV}Y^PMK4cZ#ie(4Z?EC<^ODRn8YY@wjqSSIbSE}wp6A2`D<u~>oPG3_y}HBlY3O;S
z%s9;lJ2py%yi^Sdd~`F@nfK~~Z62ocimG{6^;~&(b>7<ldrqxmkuT%)ex`7GbHM#l
zYx!@v=6Z{_v`cESU+%xY$a{OGuI1+rCPjs&%cmnn&&cn&T6!o{fK@;>W%*GXjoaN$
zKRbOoe;sF>n4sviQ_E&&UI-^=2a^%Qlt(GPy$5FBs0rGA(_>LWyWs(kvKisMH%`op
zz7)5h{nivkv17hhQ(7z5wySR4R}!lq^+TwNL8hSglJ&=(7kQN*<}mA89BH$?VY2Al
z(#hx4+LAu4N&3Vhbu5Qf#Bf630cZVpNe0K9C%Q&jsl?i!6t0$N7MyhGCxcQb=d^vR
zdBlD%5euF+OIB5Cb=;pbhV!p(w6n4ao2BY_Z7Jhh4ySKoiC#~h1*EL_-@0kVS|j$}
zCmeI1{5X5-BD0;6;^qyqx35KSF1DJfsG}?Rbm2DDQ`0kBpM_rB`84H<_iW!6!c!)3
zSU;Ir$$Gu=ZQy~YiAR>+Ig#3~uc)D7`Y`T7OW+ERsuk_m?mV57aA^+5y+cPi7BK2}
z?%C(PI_S;pOTM?8f__{&#&~&Uz}@LtvHG&lqc^lpw+dc#XGW0^hvG$rnUP$Lyt5|f
zb}c^eEPLOx1q!Vb6K?MKyKsp}$g8Ds&U^je#D?T-UhRJO-uu>&n*#Cst6p4kh(Bd$
zzCmV<NXbFbBN;ty{0tipa-CHBvmt)kOUHvH6Sx!)v3>HMV!4^ki-Y6jRTD-9j-=Z>
zRm-RUmz)%0H%UGyRO*b+`nNOY+wtq9NUKRse6ses_Mt>Mv&4FbhuTuDraKasu51_K
z>ym#KUJ<*#(l=`*<D}3CeMv4ZnU%tgZ+FgA_Wdzy{f|u^4pR<q`*;1D)Gl8Q=GERe
zJT@NDIF_v~ci!6C|K_dvlkWD5I8L0rbY0bpoj;qxwwArF(tT`Xm%N=VV#em{8aEQy
z(;l1Wt+Lb;J2$O6Zen+o&$KyrZ`wXHl(o)X&+T^g+O+LL0WNw*H_Y7f1CPa4O~14v
zm$fu1U1u5FUblvyx%c_rt>pE1bZ^#Ko-I70(P3=rrB5U$D@tCQrF1qbdP>Fes+j1I
zum+L4#~)4OZ`8@-{LlAZom0eCu%U8k`>vVc(q|JHPdq&K^u%G^XVtuq6nT?6IUXHO
z<M^`8GXIf=@xT23C0Tk8E*<{3g~|N+28R!?TRbO*rT_hLwNfOa!t9aF&z*fN1w8DF
z7c6)c@-AJ}w<-1Y?w7J3o&N3);hOh0HRE|?#yeX%aeuj7?)kn`GZ;;|KRTK-dYVhv
zaBtB${dFzh_L$TKa+$)Fq34Svqgod<UB1tfbg<=ie6zOzbJY%JJO2MKJD69tvgRF%
z-zqitpM}Ba8L6V1J61VOSuF7^s#E0xld_!O(r<dp+BnKO9<G%O%Ad+9;+UVg_S(fI
zO2+@LD~riENVYaB_<!D#v|#1T+W(*B51LIB^oxAV>U=cp4bS>74*iz#60BkmpR@nl
z5jWBQ+_l|ew{ubx#pXTI{5|EsrFCNApKjV^<sR!Ub*!^FCalJs`b6#Z(sS7Xaq$5U
z`vN$&rW6EConYoJ7xixDh1vdg>SsGcnJm90yZg_udFtF1{q9)+=LVhaU-HC7J2ZZr
ziTL)kz_p`vtH;uhooZL#n{1SR;e1^GR6(}g%0)M32DhDDswr@mOKMid@+g6bdh;Z%
z_S9;=4~_djc#D@j5f!ZaS0EbNw1@jjgIeKPi5E>_y_XpdwuBYBuhs6IAL66ySYy~M
zY_ff`LAU?b7ykm*1c_c1bJ1CQbJq6=lk8LSSO330$Rd<*X;SEROM!h~IOg#u{B`J@
zK20Ik^%LjGe%_Y~cV8FD8wsf#^wD+++Y`F;=K+Zw1wwPv)~O11INx$=iOO+Y$$E3m
z>{%x`Ukhb_S~dCItev)zhs_Pwn^>-A@XS8xm%XEJy}f3#T_U^gi>!yOB}e#f{J2!T
zt9<<-`#;;J=^s`9bHx15F@LUO>{F^PG?|uFnf<c}75pm7{neLA?BtJ}*;^xj*G#+E
zw{70v$lS=c3u?cHr{`X)%e{GRx@+Ik^2n&WeaGC-)vgwOcUS)1%>VDUD0ZApxoXrL
zn(|)a#%uxe<OwYr0y_jJ?AZ7&GrW`Kf;-Q(*^@Vj>KRPlZeh6dzd@&{?B|v0!b*DY
zO``4QyzYEj_q{p4-n?bf<J{u!^Xt9W|Nq7=ve7g3^>HcH1J(^QHn}PsY>ZhZ<I_>F
z@K76%yj9PRR_>!+BBu)k53nk`^-5Xh@o*L`IWb9rb%DvsNhy+kGO=B6f|dnK&NMka
zMYFZ|ncwtSmI9#wHwVpyF1=#1v#$7F4DxBU(@jpx)HLP{;M!3os(pPO*A6em%d13J
zo2@yqO))i$L32tD%ND@{ZCCE>%zS<6?C)>6;X5N<|8kZ5t}7Xo!XjdKXZ!mb>y~8Q
z+<bLi*zPcm+qs@fj+@g|xoor}dxDDN&jzXG+??Rj;J_Ha)8nTO!xk3R*@0gF1t$D9
z)2`q5`zP}~{{saVkM)1owTrPRx_C73m$Rmg#n+eDY9rULv;F<0D(OZp|M^&(pC6x{
z4nHopN51^qi`L!S<>L3+8wD!4+}soCIO+T^f9@$!jx4zbu8pjGB@aY|b&ebq;j^j;
zXyFMondmB>bb?h~G}q%|+nfoENdZbsp-SqpJsC-zb5}@+`08&vBHUy0(j?i>=H-;f
zy;h%gEbg~u`nh<5gIuUc7^BJd2%(?}Nv9*1Y`GK`?I~rhIyGdI(shj`Jx<3oYR|R?
zO_{bgbtc2w8%@h*rE_g?Wtyd`;5a9|>zk-#(1I;T#frAgoI0oeRoE<s)v5=%$|G#u
z)x$ggC%N)#M|y?NEj)j<Ido}lXjNSK!>fl^E<cdDiK}34n)|A<Wm#TwtKM(4dbNDb
zx>v7C)?`d{VVU#y=FR$O55o!Txg}X9ZV=>+&T-`kY|)VvO)Lp?Q>+aM+M>QT<n5NL
zeJYpTZe2SfxIJiR%Vl4~M<RLw7QU)eBz3&5UJ3TwDXky)ewX2kuo%?>{jk4@U#Btz
zZ3(;?vF3woK{U(%trGEzA~RLP3KOpu9BEf=i<p!)vr}Tq+Z#dtx*79sdZqoj>b5C8
zu6J90UUS*hC?>9)!sRT%6BuWm+?*&Hwcx*-L0A5luZLC?cj$@9hHXtb)uplef^bQ}
zE6xYrMQ+@_car}8zpz~H+JT>oORpWO{cC(9X1!sT{(f8AvRk(^=gO6T*}3=IwW92|
zCH7^<)3@LIz`4D=prZM$zwWF%tK`$wezPx_$B}n1p^3$E-vcKeIhUPJL|k{cw!{=9
zBtD;!d}Cpo%%pdUt`3<URc^A=b_jOsEwT{qV!DtL(qr)IiO@TXPkxUl>}S6j;^)Ha
zzvSc5;GI#Y8uPZXPVsj;Ic?G9;QwON<BBv7Ot0LyNNo1B)S782nUm(H@2fwyQva(|
zZlh4vhKU#Je*MzDFS;=0NY&G&VV8ru8UB@cF|0{w{}y(zuWn5e&mxDY5|yoX&$nnb
zJx@F;k)(Q%dzn_V#GMC~uMF8@j{fKL`^EWCr0VKwMp=~zEfX>BPSuGmF&UBPqQty6
zq#l;?<5p9?xX`0UNO|Uu32J6HJN=HdPF}3Rs_EsUBs}wx%c?W0HNFSQPgU8_xiMj*
zPU_4_v74@JI5=TZb=gF33FgVucdO|5q_Ffv^CTN~DW>Z*J9<XHGTe8PqroWVMSt$E
zA2Zi82&t;S?#kV;Y16d_ZVo#HpX8J!dpK(;GR@x5nt$QjoZW)z!AuIFx;GrV>#ih;
zt`>Npm1QUsxZzOp;s{sUD#I==wswmv3oogzPc!A6cYM;7i%TDv1aimDTAsV($%E=g
z%JY6m1P7n=@xSh-*!btctAZU&0!=(F3$pF}9RKqqO=#v)S=d@J*-7SvfTef&LjK-9
zZVj*4uAnbgqI)mT-7;CwHuuTy2aNjjf80_3(6z<xDX*7Ka?_+Kav1?nQaX8~FHK6?
zAY{8^gQri@tB6^4Y1~r^m}(oWG{XNp)wdGV^8fmSebt&{d|_*v#1?y`mPEYduUhkI
z>6QiV_BS?qF|HG7So0y$cFHr8DHB#{hu)Ug_5ZrccWRf_vQG*56S!7?xUksD_H8?B
zd*)S+D1+7eRvxW&uyE>~Hz9m~R3cY&Mwqx#iqeUug*?ti&KU={Dw~IRZdrF!DDUWz
z=&HM7J1v6NDbLc@aLx1<^1j%%M5HtBQ`7Z50ZvEPsCY63Wc_D7rcu4YK#|8M#^+KD
z-{xuiJb1K4ijxy(U2Z$JWus`>mk8}Y=3+)F4}Ddmd(RbZTYvjnzc#~DuiTX?tsEW?
zr>A@0Ry%IMwPbB#BZrKuKFjt?yXSsca(-#6hMlBr#HM7PHwH^@B+Zmr$$2%SdsayK
z?qgdw%$%N=_1RrNVCKEuUv>9Q-6L(A@$lr^KCj*Or}eBayf)o>mUmzM<ot%eY7dyD
zZ{+=FT_Cn#rTpyzw{6=dHZ{qmC!XY0DVlcKz(TrZ&Dq+%VtKx~magfKE^q8B{n!($
zo_0Oy#?BYY`p-JwRh28>e;6^}%5L6=CU+a<g*=wKr#6Ue@VL-ZXmYYUv)k$a`t1Sh
z9CQ{5x(Xk^^2C1I6Ruh>vvYZcr#DSeP2_%XvX5a!_0zdc?o$Hat~`Gu@j%^$WKp~3
zms&}hT#t4bd2dWPH?39c^y;h)a;pW7u3pj|^!-HT($zOIwO%jeoYo{XwL$IK64Mv6
zZ95GvNBSLL@v#VNnj*S(k=^SJSGpFaUi=WLvhj6H_KR0xuMMu<{gk*Psbt0inYIlc
zcTAJ2c}3&Z1>>*WK6K#84Ao}WoRds0o1)HnxOKd2)yq^#mOf~4t4;3tmcVTx$2Bf@
zo!YXPx29%84vVD6x-~iJXI^{l`u%BY+4|~K*L6$EUM!5A5^z0h#hP}n$jI){b<G!<
zUbxO|oc#azf8IS)FMTxD&D|-p`merq#_S(@+UHEG!i^f|GNtJYo#6T{b$!Ye(<y%9
z5&eNO=KIeqwOBCy<h*YoQnvpRnwu44qz_N@;c{8G!F2P@;***y9C(5P%;i(Q<QF*>
zi)pD}_V;-tcy`@f?O9bl$vlgtZci@%bLUsz?s>}fK`$5U8h>*5edl?L-j<UK&IUyt
zP*44|^5vq4dGdG9+U=d7*D0zJnpN^^qe<8w>*ZfA%wrInyR_=cms2ipPyPEAzV)Nr
z^N)39S!di`x#O*_E-QUE<+zn=I>RsKr-7oj48J#?S{fwv#go^>u=X&QW<}`b&%sX1
z?Paq9>{TKb-?Mf0nXA+Ff6LZL#g~UZJYMzc@n)&^weJ?LF5EP6zmvSGyU3{krrj=&
zwrY31@;bHais_1eKS4`Q4sNmA`&=1U6(({E`~Ld>Xz$~a_4<)sJeii&(^_hG-<Dx6
z*IuHK`0q>`Q%y+jRWS!f5yo3$%3oQT-u0aGbj;_uo-g38&O6~!3TMUEONSF#D@s|f
zDs-{Col0s5U(^v^Jmo=UXb{V)1g<jQ>o=P9U5;g4Wzx)Wv~KaW-*9MgV(;-Q;Y&IX
zF6z8AL$kN^;ew)4@g~L_O|i#s$HWL(J-oF?y!J|y*aP;RrEHI89J`Qs<9uO6n`7U_
zTnUM)8;%vi&nx<#Pnhv?MWWTx$**=K{_hfu_jZx}c=24<k%v7OTfDEF=>4d8{pZ8J
zzc>2+MC><dXSAC1nJ46nMbBNykeD;~r<I(XT*x}5#{Y<k=hUhE(Le5oPdvt4I@3|G
zOXSCyg$z>}j@+D{(8_H1NK4PX;)3+^pG?_TnI2qJiahh^_!s$;sqSSTMRYqRX#`x=
z%4n*++Pw0{ZH<7bFMFLrLgQYUCYRR;yeWLN?C){5ma7dDL|1;C#~sMX6X_DoI)97A
z{nZm6f7;99ed3neiKs847k6qL6=alNFUh+5g!0^uxp@N8sk}XrYgm`$%$~ABF#3UO
zOwBZxR?j$({=*ID7rRLRJs7wxa9aEa*5fxWru4X`{+R#&Y$Kc36lr%Q_gx2PXWVJq
zb@KN9Kc^*b9IjcJRPe>0{oBlgk-ZgD6}$=*rk+vp<zjH1I^QlUT6;&M|E?z48%+VX
z4h4uU)y-+VS#U>tqSx_WMqgC{<1lyMP9ZPDUgM;v#zE}XA&F;gZ-o0rJP8Rl%~Xtx
znk+VLsi`POOON7ngS*l5f?uw=+!1gjX4{F_c^=kBJKmkS{O-@g_^P`KW~?`MvZZh`
zOcGf(DaS2QMM>3)O<8ZzAImn0rLogXjz({gRW+Kedxhyv?3Lp=jtm)BlGmQMy(jDC
zaZs(}IJ0Nm6T!2GRM`(X%7xuv_PsUnb|LGMGmBCu_y_KC3cPnHf0xt$0;c5!LeIqx
zOj^TmVZZGu9X_e*D^8`o?%EeaLQ|iX2F<Iu(sb_RJh#kSrgc$fUCYg;MLGS5yTmBJ
zdByV07T&Rv^Q%u?UA49)U`Fy<Tlt+H@n0FmZ20<jUQpg$BELhzHNc}aIq_)n-P7)#
zVq%fkA`-4O$6Y&E(E2NP8iT8k!zICO3t5(Indt2HENQ{C)_tx2DsM;pn7=OOA)}rf
zquPV%4}JdDJnFvI=IQ3hx>CXCkeZ(pdzgUCk+qNP-)My2XwD8iuglROThOX|r!i1(
zWq{dAZJW%=x0oa|4(d-8FDnTOeH30`aG~1r>=Dfy!fX+D1tMnfN!(_LxU=N?tTf;M
zwmFT>U){UTg`M=75g8XDo~JmkP4QBv#ET!AmX@;8Y*}3kCAuyuE?n2QaF^ekH;M1=
zbjW4Rz0UgVdf=--jgz4>)OJr~n>g21>C#25qAMFOvZ^GVzZJSDwdDMdBXei$I2nGH
zbH%)6VmFvH53MS^@{G-lT{Ui|%SqXAiI=WlwY(%&GB{|4-8op`q`65hyFl-C!K6cC
zCR10fkShyX9kz?>Ib(qS7ybvP5pMz)b7(qpc8YQ(Dp%$`-Y`YjB8+8~ZTIFWk9W^n
zY&CDO=ZRSXJ6iUBW3h>gxB2(nMr-1Ri=0Ygv&A?sY_si&%DL#+Bq-%LTf$R1TH;Nd
zLP~PUw8Sg-{}*m>ba~er6R;$uB06=4r~KVH=W0ThSxxGB=c9PBQ(5rl{)~gia*r4O
zaErf^a^*qRq^1{zKUBPSFoi3;+#r`3&!QdnB_~|r<wdTw0a|M(Ni7Z7#<=Cu?aDv=
z1}mSG1gzZ}6m_>GPWOv{?cM3MioQ3lF`u;+j(FED->K8ku{3I)>TRyr+j+vb#bl$s
zPFy<p^e&gyO9$E5ce(**67RoKz4mqC5+=2!TuKudXBH}?E&JBWA|U-m=H~LIu(e7d
z5}|67eUs95#U8ujl)g?Yqib32mAM%co;j}MzxS7$`6Soj+(|jR^1?lO@5oAr7rZ+t
zsJpSu^&;Qf^+`uBaA}8&y#D{?P9uXv=4QQY?F!EdojZrOCY5QaU*i*gR45l(dPz4=
z{6TK`^`NU;)_G}P>3SIO_)kU~laIrWDyf})&)?2n%{}v0Wt7JshWV`=_x~u#$KHIu
zy3%FOyCb!EEVX>C@jH_lPIreeWbEg1_%2%zpToMpVo7(KYidcBz1R8WRf4}_ueFK!
zu*tVeFFnd)EwV+_Ej`m`(w+GUKU#HpGPAcfGpzZb=5eE0a@mdABoD0z-M7vw{pI#(
zYdgp3z`O2&@2xk5Z?fVY&bW%MJ<s(1?I)c{i&%vDp9&?j3bX5O^fJ8YRVeCSC}wUb
z@_dPI0i)Nn>HSV}cb2@YdONA!^x@QRzW=pzb@_|l`V{8Ml*j3B*jOc-IO7-7!x?O1
z=QML$RK@ESigOjst4n-Z5^-s&V#mX^FCvaC5YviP%Tn2&xNu#c`-9bslZtvo*4c3`
zTUu2#_r<hH57;s@4r)wrO%BRa=DM)LYUb1<_oqBq*19UowUbk_WrCaP`KeRhg)>~)
zt8qSUOH=-m6fdFB_48)7ZaOLOI44j@Ka4^5!m>>_dGp@~tu&~-{^V}Z)=#p}GqYvl
z^VKgheY@s-+2j3|q+HtxP22B%vZ|lDyixrZSNFS_t2g=_xpY&vjYEF-q1SfZ$)7%6
z*z@kf|6AwwvKcg<QWjpy?X{u9Nt<igqGy?s(vcAvkrpEVU3@=C#F~ibicMM@=p0pY
zqGZKUyCsDi7+rU>CKdRkL_FEf*}(nUOWv3NwB4nXuiSE{uFH!Iob_eN!AI)~6?BVa
z3IpO#3YxPAb{`Yve*5KyljfeBtM6tWUb^V$N|n=z?6Q{&Z&a<VZOpyim3tw=_;l!M
z{SD4H_kF25qWS1T+pRjq^G9`?eHx!^F})BeEGZTHY=Qbcy`s*IlRGyAUs>s)>Kz-v
z;VNghYQhAjhyOb4d7o*m^O>>m<V4XaPpS&$yQ;N!xjA|EJ7s2k-*9)PL)Y!nb?MiP
z7<!&18LZ1RH&C3mCi$+nhgSdRh%2)!ijBX&d#`5d^<~xjOW%@s^u_WE|HpfLGzsq1
zbtn$!*k0gVoOjz%BqGyuYEq7#eo4aW=at1@>twzat=*RQzGQz_n63W(we!y0WGeSl
zzT3NuWkP<$%I)jtTzI?mz0K_On)O#}!dw2VdtUvhyyn{a8nyV<Ki9wIVf{MQ<Ev(1
zs6d9(^@ohiD;jT|ZtQDodbC)^*55u!EG)!eQ)*a3fb`Z#v$k!4(nsZ&#LV`X{G~0;
zCq{;iE0`^H1E=Pm4JQ^Y<CSI?jXj#zIrm)8lb%<1_|Cr&+u?YbZ9Rj}=1KR;!UZbh
zIkt-y7Y68A3I!XA`xky-vz(;2a?<xN`ONF{pD)qQV9;D)=xZ0(U)Ea7d%@@6N15v%
z|KIVEky`yx@ZN*j^QX_abX?-e@jl_3IUhHs&tRK)^sU0{%-g>z732G!)kH+K-iV!i
z+2ZI^DWz8`-bG!JMpqMC6EcgII%_TYr<$ntZt9wEg%W)Zj%M!HU$sx#FwH<xYZL2|
z1DmESWJ&G#Hgm_0)m9aW)82$Cc)mRLBKhcxEC!*Cg6W&Y%1vg+PWZhl;R?5x$?K`w
zUAh^J{~Esg^cBaq6tCuw@rhFZl2xqNI?rTNae0>1+nt*u_7+z#dGB2RK60nNU9sb;
zkHz(eSATh_zc1;^lFQbO#(_=jKN1$VNJ_9Sm%fy;@Mz?WEvd7w{m$INsa)fpY!c&<
zu{l&nCR1aE%ai|}{TZ(7D&!_Iot#pd^VIwJYM(u>(+V#bggrPZ^5@4Tt+$0Q%;pN1
z24z>iD?0jtYrC+r^@k*v=U2E-*>9XI{Wd8;Gwkrk2UBONbJ}`!SYDivAQh||VO+b-
z`)kFP2W4V?C!8eef3&;B+0H+_HRkx&+s8_;pDwjXJ@;HgHqLx*+~UbEmvAI(=d-!1
z>h*$Snew-UysEWjNfUcrpK$$Vc0Bbot76sRUz*{MW>%>E*mCGc*38|4NjH90XLa9Y
zo5Hx6O?Bh@ZMoOKIf`aa>TU~^&y+MMJkOWD;g;k1Ne3_IF`W0RE702bEZT14Jtp2w
zP5LFevhkmbH|w#~ZV!vw9oV}4|L3N8`8!sI&;0%&U2;eF>#F(rk$=0Y50_V7`u=|H
z@87G^W!{y4J^cLBL2JufeHl-F7wVKWoNL*ovm=oy=dqikT~OF^rO(ZID_Wgbu&V!Y
zO)h9vnw{dkj$`_?qYa*}-XUGgEqkNZW^EO|;KiWL>8auocp#vYiIr&z#|90DM@PGb
zw4&a4d`z0uEpMNtqv_Da(j9#3#-_)Yu6kRqGr6}h<Mc8&Mw8O0Gnxnd&d+dQy;E5E
z@%j0AVs9thbZ~1}yQ1vYl#MUCSj5fsmS!#C-2Ck7QXA846N66AUtjm{(zo3&!W#B3
zEjB%s`Z_;1dQaBPPgj3m&wsF|={KKQjD+EX!<qlhJv%l{NKR3mpzKt0;^U_jP2pt)
z_bya=Wt?BIz2V;($*)r`2YVIg*|we*;oGqD%(I~C)46xImVbYD*3<mM?L)gco+riZ
zXn1luUweOS+hhj4u&MDqaa>iBZ{}{9{;*HhriAf<{xmIr@!F~$DOautjZ|;#pieI+
z@5+c*uKW7?=kj#bhTgz8+yeGN|1NVZWHHS;J}pct=fi?9i6jk1POUfV_kSz7G+}SI
z+OGm8qrAe@Bi0@&t2G$&4o=XRACj|}x8;<=!XBZ}5?9S79!EG8jZSGeYYC}6Xy6gf
z-FUQD)1xDk%cwwQV-N3+8z&bD1*UAAs_S;J`}Ed`JNau3cFX(^K4Wy!=JFYni!s4x
zt8OHza%rk<5!l3~x#Y5+m1Ix?(~m!Q9{FE;AiQOZQ>68k3pYesdoQ}m=89Z$uYIg!
zuD5GL$u6JM7p8C<8BNU#4zRpC<w~G0?^mVOAzL+e&5Mf7Iu#xr`qlkioyY_C_bVRf
zFXC9Duy|@hbmG$9|2@7dZnWrRIjg7LR%qq@%i{4($!yh8{iyCmt|y|>PB%>xF*+nG
zuJz?c0;^&!Ys0E(RXLf<7euj0a2R!{{GMb#Rp-)ld(DO^ovRmo(B)dK*?s8bG_xHA
zW!<X1w@&hk)So%^FwEk>R34?f9YNab<mN5v37a>I>ttIcQ^bR%L4J|S(c2{cH?K8f
zE#O`^bNZ@CThpUGm1lOpv&wmHTq!rpd(qh^miqNKE<Q4>R<Dj~>{xQl=hX`7ERC&e
z952s$z1BCF_s#jV&A!{bELW6V+H!TC$j%3StM#sJpV{5_?p4aw6}vy2xm~?yAA6WS
z<99P5g@UO};t7XzpE;M?{g(MJZTojgYmEaVqs&wLpGNcbW4kYv3Y2hgdU-NhdGpsO
zow({<yLX=5mJBAnHCtGZi>%(0akP7#OBOfZCEHKa!V+Qvy@c;RXll8)FsY0w=JbpO
zJmK^BclO@9y<zdgu)K+{jyyRz<3O);v%U3-nLal0I!(``CjJdOaUsda{Q9cp_P>hf
z{q*^H^UnYHdc)t(Kh;}BPqwXD*!tb)nV)Eo!aRu?>Z)HfO&YYWEKt!{!6Bz*w)CXJ
zViO5XelM+{?F)MsRc&7R{6$*ubgrw*BX&567kPy&)e>FtBiZ2PoeN6Av;0cxOqR+Q
zK5|cENRm90plw{^$rRlYxS^|cLb|F-v{Yj2)&&}trJ;e*lP4#MJU`^hFi|_^brhGV
z@{^t2Hx9|YubRGxk7?!<0~J+KMb6hhScUp~QsQMFa(H~nm{c3-!d3Q!;Sj^KMPD8~
z(Gu)Fm2++C)M*u~Q<`n$*u5^YEuW%TBXeNl(GvwWno}9HW|SPg8I?47`i&H3=SQ7^
zecPAJJ<+1GyIQsC#D@Q@Oj6Hu%{FtKyP`aO#|ssomY3{_B9kSO<cbYXn55YryRhYi
zg9_uciE3#&izU}o`ev_w8I=|GdDe}Zwx9=ZWOH+xXWuSJ$laZJyC5%k&g}_}mW7j-
z7nj|TdDx*IX0e8E^_pV2?iuc-yEl0~VA?R{+l4@euGRfDEDKz(+D_5?zj0da2?H+8
zS1A#5f8S|#D4!7Jp~%E}CHw*3^2w7HwL8~W-xc#oS$gNCO6r-ale()W9e#1QiCa1#
z>DaahZDw=UI-Xp7D(4i#RNW~Hc~xc{3fu5}iJ8gEDvzUqe$~6YgeEq!JY97uutdo_
z^3qb?zzwdm6n1$ha5c#NUBDi-^9Wzq{|!m?OjELBUd}QJ%1HF9jCv3!S)tzX#j~k?
zkpin=XoPdAKuhP}2kgboW_K%21<&qU!xyiyN@5<@W}_BG?)(*prQOm)9NsvUxmml!
z_FfCy<@89crByFUvO6jzWtz+KrNUZ5r<ES;>d2O^nxAAlF`}>g+U6A;ZYwfM^->(B
zORPP*&g2kx+~-r1F25^vzq-+HsdR_u0fEeui{|yRFj?FV_Hj8CHYJf;b-_syuZGDQ
zjH%qWwmSQ<3hY~cM8nPZri0|E=&M3k-1f|MnZ%Hha%y&hhP!R#%^<fWjbVS@r0+Vj
zG*UB1Fn-rkxh)Z^TeCNE@ozqKc=ZZb`z%GSFcsbZ8San0yQ;T6TC`W<s8jN3cg@dd
zT5VX43m^J;WnSCXvkAf{Zu1!3a8owBX>iJ{_iAtQ<(IQd<=pc=l<*bYax<^K#e2H+
z@+JnWg_FY_OF~y~;$WE8)X+U)i=uBzjGOxN0PSbn*8JTl)5tJoKC|nJVsXw`clYTH
zmpvDh{IOiwel8}X+vkbX-<c;w{C+N(W8vDF`k=vU#u86+4+f9@8*YfbIk55VR>rDz
zKkoU=5i<NT>7YCJ<a=9U61n9w`88(bI5+K15{|BZS{B;1)PK#b$xA*Q5ev#_opmLc
zzw+Tzr8gg*?s3`3RTi;BZnc1eewtf<XpZKsXro|;>qp-1+_v&|@_&z}Ij^ob6n{4L
z54zDHvP>bu!F9q0;WrQVSnh~8dat@=i(>P>z#I9ew@ui@`2Rb%X>??RxXtFpbN%>>
zw-yB5<>C}GG=9WlP{`<VM~Np=M);!Yo<BOL?aK39w=I+SbbFG_Q?>nKBH?kLwA{R7
z48@cEPe|w(n{5$0UmLE?bE!`^WX18xFHd@22^K6Z&$0~{wvziR;XcJ@fvDTePM>8<
zRk|&nCZrcTYbS||ntwSb>c7>b_Ur8~zxEupX(cND+k~cVeDUt>%f*ud9_^D4&p0V=
z_jwXSPb`1d=evimq_uXwo><<!Cdq(F;;7cK4Po_Pnm*>un(lg6q%v7e_;c;f?*B&J
zH-GN=-7GSpFfBDt=j@79wy(2ZPM#3yKWqKYV_)a|RLgnf?RR_r=96zqHrZC@uPnVJ
z{NwaO#c8=3+OHTUh8|zeR~~e=rkmr+&%cfh>szkwD6#zVxUfwomwm~aohe(kRXtnr
zLTFY4>$}c^m&YfxO<~aS{l91ZtL3L(t8AEh^I=Ew`S}jsd4l(YZY(l8;gHzTvP^H;
zv`w#6?=<kFIVl&l>+DE>_u|oN9#3_lGi8_DHTSGyG*9Ga%1HCd5q;;w5Y{O7r}3jn
zBa3d5$QEBcn>F$V>UzH##Y`Tq%}D(CK<lu}!q`_`?EBZU-&`a*%Z=;HsypY--+A|>
zQSOBQHQfyi|64hExw(W{o8R<u32U3XWdy054$(XPK;Rs=vGx`&zK%l$?H;aMJhZKi
zSPCC4-a5m>K7o6iQ(x2`*6r&w?B{9Nx2;z9I2iq80{=nPXNNd;PD_fImKys{xwl7m
z7grMZir1MxUgQ@Yc3hUAvZ~u&ZGp%Vy@XfZW*^>b#yPEwa$>yL>2~V5l9HN+&IbE^
zTB&tSsd|q@XF8=`N<PTf!*kB*{e<3`E{Qxh96vmBWph2a=z`bBA9=^z8pTosiua`O
zOlcM4>es!oP&{5oy!TkTf{yO0k5bzUGwj|=)h?8qvFupYnJ<$Pj-Q)&yk^bUYl0`j
zl1-1Xoy>TjGfn6FrDGOX{&(gsSaYh$KX22T`7fUR*x_SY#+pBk>DN5{Uq6x`O<Eyl
zcJ9}$HQTQ|K3l@{TjP0=mim(^t&v=T=WWveSiO2Krg0%6<-#mh2Gd1O8?rS+f*2c)
zc+5+QYT@z>(XBXioKe?6vucsHo60+%CGX07-<7E|SMt8QqOMVzyL}tOx^2E6*Xgpn
zZjuvw&+7TX{>~8=@qV#&zU;3T%DWttFyy)EbB6u+C-&#3w75I&xNzM)*W2jjZhTKS
zK+8jq{q=@~<Nz~{`=2I#^XO>#_IB>Cw=GP@!66*M9C_RaKX3~R8=AU4Je9D8rQwX0
zpK`+kB?CW(MGZW+7IxaaQ<&t+QvKTA=)aM&?jvJgKa<j=u2rnILBh|jxe6TjIB%!E
zy~wZkmSXI!Mjn~Me!gBMqbBpxr2aX4${q{<2Q5F&;;tUVG(k^kLf4Vx9j8POr6<IE
z^focjNG@bb-?%2Gjn|C-Xhfn&*1=nMR;mAD{1g_Tyrft3PM7Qw-?(+W^<NI@PT@7!
z6Z~0^-`-aL^Cm;X6Aug*F??Ot`9<sT7oYsBphw4VZ2Wph&?L?I+kv!i8cXM>iR8|B
zb5ie0ZVCUX8hz*H_nU2&ZeMoTu1Ei;SBZjd(@#GU#hq^yek(I>IiM){s!*Ybf9rei
zjP7k0oE`J>&ReYd<8n&LYn96P#kMC7$en0dVsOZb>wm6CygTEZl<FuhwcL>M8Y64f
z4etti-|1aY*Ee9=x@C2fyZ40@7Wc%pw}K^f7s;`$m%sMtmPz7t4*r|hUfJJxC^_-$
zniE0(r;F<Ev8?~nqI-SP`UwSJZ}{H5qpQdM)cDET4PPz=D!ZP{Yd*<l9DHk~fq1Z?
zv1qWlsr9K3N*ck+O6QDRow!?$hAcf{e7ecxWdWaO@pg}M(aT!-i&yi1EZFYQx1%Tf
z*$btJ6B^NXTm`w6B8q~IzID8aP?@0P)Va+;{fWoLC6A}rEMrRS(n#9qsy88Zhm-P}
zQoS`-nX*4d9h@4y@~dd<v1#iJeKH(HCJC+$JE*PeB<20#<E`MEzy9<2>+{&|C^Xn1
z82i=KaD|}s&#UR5&WV{M%B*ag#TAmsS>mvs%~7=E#I80+`{ty>YI2&a$*&8)o0U0R
zm(9JSIe(i^%G+bsY?G`f_`3#&sB;)8em{I>l7gMoq_a}5e!WZ9R4Y>2+^<()R<t1X
zT-!vYTs@_!PnB<Ke7TToA9AU5MzQMrP{thxMC%^9TxhSI)>IaE>A)R6w-wE`S?slK
zLd>O0H1==Ns9&poG)%oB=-o0`t{le9XIBpEHOV>Tr2Tq$q++Gb^h0$yZD}9U{r^v6
z|9@Qm-$A(}YdM%j3x9fXDxUDa#jE#R=$_K{z~vtT*G|ySWZwHP$iR0+kmgl`{r@L$
zTZWdTt}tBwxIkyKvw<Izh5F)_=MG6%e`^Ru+8OVd!Wvz*s4O?zWV1$hLSIZ-sB)HG
z?5_ZZl5=LyS<N55i1T}-JZb7n6Q_8l&cD+Rz4)=TZ`$OG^FFJ7@p-$9S&hL>Gfz=H
z%%w_-O^kD;_6if*bzkzEjvv(Pe}6+G|MS&D({lBu<$it>Y~QO{|9-Ol|LfAs6$O`@
z8ZKpsbd@@szi8CJ_f=xXE;D7v?P`w75=r68W^y8P6t;Xj*EvVWMBY@yxs*TUC8t$I
zf1ZO$%%WtMspnk+ueC<ba1AW^86jq?5qzU5^{mA5?Q-p2+LP>;q%L~(#yIi2Le(M1
zti(S>|5x8pW4`Fq<-zNvuI8lD7NR_-gn7$z&owOJ=br7Jc1q06rQ2Vnrc0~lP4@l`
z5niXgL=TE+?CxTEzRvrknN~~hjVt1|8EyXGr)1nX<ooM}_=BfwSyDt!I0!R5^=CCL
zyg4aAamU>m8w!QDPJGOpBe0rFJ2GfpLGF@fgY=9N>z%==89b(;JZqUaCVeT<fA?t7
z$`IeF$}5llj;Ra{{TvqfRjEVb$%&5g=}O_ug#!EgjrVuF&p8;op!@l)12T^@D&KO%
z_lXwmk*L}2v~pU?zI8tm7cf<xdZ2n?HH(UGr_iw!lfvqkheWoU#;pDtbu!ZSn5b=0
zk-Ern>76ZJZ@(*j_`ki5_1e6hhVTC+e&&jp(Yp1CRu+#%0JjN)<NCBOeA5#B*C(z%
zCR2GPtBR+IThvs;B1zk7w!)VaNy;aU!jj(_I@dq8xYH@@A~kPGkFd`zdG~Fn8#olg
zqZHmRJH11BnfJESpPsh8Ox~F@E&ZWRQO>mGso(P>4m#ACDI2@1Z+T^Jw@JHC%;=wv
zM{Y=_{zU(*6!#@xR+ea`%uCl^)$X>cojGd>ui3H5Da%Dug1mCq_J()%o)*)19I^Ih
zRZgo=`JP4pPduo9(LVh{)uFs+({+3fy}Kdt(<0&LRo^!U<r}t&#dM3BDK0qfz|CXn
z_bBC#%I>&Jo8p8dCmP=_u~-x!`~P}?eE9~I?{{DE>g(?0GL)R`>QQVxy*QZTfs^<{
zC-H5zmYbg?J(|G47+1#Al4|_?gYoRA5e45v%dbC;>07>&!#?S1RLiXC<)N!v%9UTQ
ziOstF%%qrO;!nZK%aQLpDyt`}teAdbLb2YI&AfSxsXN>~VmMznHO&%k6Ut;-&2c7W
zndPe`Oto(##6C)Ep38{crtqHShEh%9yy;B-FP-N7Om6sWDa~mn$nIpPwQA9}2OGI(
z9a4VWD--t8NM@$KP2<9}M#JVtXPF~{VvE*iezj^|q*!JW_9q}EzxjA)^ZCr6i=0aw
ze=L67DWA;hGM%|~>6xXI^OsK5Kk-xZM7vGf@BhwR9tXv)ttqUjvahK~zr0v!YsSvk
zYku#`55M`KV{$}f?}w60`4`%zbj{wTKC8s`hl|T53ANApQ3k$M5s$i6d%52!FuYgr
zvglQtwcKrod-wCE+Mm_lxhA#Gn?$xPkUh6Vov+6J&7<jWzRSOAD)`AKBdxO0w0)zx
z%%-Id9IXXTOBfn${%(5ia)oj2oj+UwodKJJ=lacSE0nd|AX9nQN%@-k%xg<G6!}L7
zDwUd>WIxD=-j*3XMOUxjOwSSxVNM|ju3HDptk((jtL{*K71n)w+H%%yF;ned2?*4G
z-*Mp7<6TGXBvc}DjU%=&+%&!uIkPfh-FNR347G<XJxyNndHq)`d*_g{rAcj<<lYXI
zg)+xP|7mL#Ez%AW6gzFX|Lo4_%tFn5U*`m?xHl~l-NLYL+r~9=f5llITr5=*6kg=+
zrnq3H=wWS{C-!kqxNpt!l56z0dm%C_OD<|5b4g;T-3ytpMas7pW&c~$%OiJk*&M51
z&0hoMA8MZRn{~?TSBv|#7r|05V|SfkEwUGKSlG1m<ib76v%f4a6?5L=vErbxa{knX
zbN;NoqETYQ{x9fK$&>YGraPUvC>NiSo|T;*RieeHp5U36=r$u%JI{sja8v@D>^lB4
z8oRHo?Y!iDf0o{XZ`aC_dN~Vy&bs*izIEtBnclw}N>V~LoRJJ$7*w}7aLxG7raCEf
z(^7^n$92Cr-i`CJzmk3C%I75|-%>XB23(Do-z=?kO<r<~^7jCJj_;DHCSJd<75^<N
zW#%^D9&D7lsp{^J_}>p2{)Xh-Rw|I@P-@s(vHNktO#2rfrutqkOWwXiS!gG3;PTnk
zx0}t)W*&`N(cvh=vWw+_Sv}uw#<h`CcdE-Iy?CLq%9=&BvRH}j<(~Hs#P;zic<k8q
zc$JT~^)`-Of(h;o&h5Oi_bN3$K6L4lFgCh!V&h`>9zj;0IT9DcJo-|W^n84Ndj9EY
zdWM^7UYwq{wruOIt%V+0XVe&@WxP&WY+1puI7o#@bitO)OUr$yy3Cywk`=m2XieH(
zt*zSE|F7Yi)bLhHEQe`B(o_wL)n<)MTe;3O^{$@w=H_zY4}K@D6W8D0*YKZ<Po|>z
zzHTs2@y$gQ-0hk%I|^<H&2)ZpI>sXM*qoWgDSp!(ixril99~|Y=)0Wl>`4<zy>-#s
z+um*{eQUH`wn+2xN^bRgv1Q-I{(VdoGFlt8G0U~fvY2tn0{fnMhP%GLywn}6zpn7N
za~RWs&7ArPdk)Wi&bsAN#_DNn!`|oGsuY{$)iQE^czyo-{$!!Qd{4i}*GYH^Jc#1T
zn6iL@(=YL~!@4_07o1f~Rped8xh5>smWzFIp;azOM5&!`qDDifM%IyqH3IWI0$c^w
zUcboZ>bZc`=i86Ti~lqH<CqYvyJmwCm!^H@qYGj|852`Pm<%efI0x}8Z4vR}FnC(;
z9%`97HPUMW%hdQz(diNC6JIW6NnKQ#HY;b{%Cy;OvnsF6$-n0Ld~Vq@%jZ$S9*)WL
z>%AIP76g5nDZVh+r!itto8q$<BK?JoH<wHb^Ln{-TAG(+%EZ8DH&avJsI6jH)RvXC
za>G)uSF2WS^Ln*%!?a)0t2Q2*CD<s^c{Qf!q%QkB_x(Xv*eZ^+nd`W%k?{`P#Q)Yr
zr|Rq}k9iE%8y&m0rc9WmE6G$a`(j0|O8C7UOk0w@ZSA)OuVa|4QV<#*mULvPdrTpF
z!a7;a4UO6yU$ZB9NL5NKa%HHl5c+@6fN9Rf1DyJI1U+RPcdv5)9Apwwye8pyaG|&3
zOT!@lCZ0_J{zgl$o{X>jns-X4`f1VW<fVV6ohduYb3HooVa)Y2o;zkw532DLG)Zh0
z_5E_OQ&l%}$&|`nH<kyu-@TbWx6>^uZC&p%smvuapKZ;FZ|D7XBWd=t+1EETo}DXl
zYHRPtE#Jzv+`V(-V~BbAJxh_ClANAJ=9LBh(sG*SN7?V!l-xT0U@v3YlWXp5R`cFJ
z61pL%*uv7AqU1VTy~g`MVfvTK!(tOP9y%}EJxQ@@vE{TG4AKQH*^QEar?;>LdQ7Z7
zshF;LB`A<9^lANPKKb?2m;~x;J{`+dT{io?z5nw6bFzZ^)#rVCx$^JV+vWSy8N9w)
zW-NTtpKtfGUMp(FuTHO9cFR7bOZ{Iqz3>#P=tNfW$Q8evt)$kj<hr_=$>RW{R6z)X
z%~eL$4u?AyEZ)7zaYEY;tbQW1TJ)6AHTMgXH@;xWj+FSAP_~3wPVmy%yj#1bZQErk
z-y5+ZTC%uf=c~fWQES{|mv}Us+LWZqqME47sc@m`gRsl8h#sYhlP5V>L@4N~1~PHY
zIN&IhG^wSf&xxZtLX7>|gyz_klEuj#+G3a5f(p8$H&%BvT3y@Nb@H`hUW&5Y6OT<+
zpZfabl)X+=L`+@$p|DfC`X*oCw@q8ltkE;Go*c!K>9I|tK`;CNMDJ*o&IO*^T+=OH
z&x~kko9_21)xgO(>LSl&@grYcgL*T4mZ%(Fc{1c!@a%&<8Oz*GM@GG!w)$l1tqsRI
zuRAQyPU!Z_Uc@Y&G>g}4ZiH_E%LUGN9tpyGs&1FQn6~{@!tz<=yORpbl;l3mINo|?
z5y!(j%<~VgY_|&txh(M0_=uvBUfkP19P94<oWOE(vSyygiKlT_Ec;KWJIlY~XlLuP
zYESjB%hPa{KJmce`qDrK;U69@vO*WT>{J#_G=3zcX=MFZ;e1&^jNB&~qxBVGnvM#g
z_v!<7?U-uBzw~8K@Yggh^;u5mml-{s=ePHG@Z4S9ZjX&ce*S9F@H5J)ZT6D-pQN(&
z$`8xXx`3teSK==3k5b*xu<nr7>LgPKR;CTBnH-!VW`1|rq_n*0QCBoux9FN}Qr`I&
z9wo5-zozrbfje6Bju`iT*R@;Piq)TcO|M!oxxu%tb;+$eEA|?7x%5qW#25NT++zcW
z;Hjp*lBuO9oGx9~6=FH67+jI;Ho<&TpN8SaWmi2i!)G72uJhTLb!CMVE2r_TTidqj
zcy}xDpPcHPv}vNv<F`u2oyH6nM>U-vt6Z99?X_Rx*-o#}S;@1$ZSnJbyK56qZ&~%p
zyQ@pX%?{{D-#jM0ZA<Oj99hSP0FLc*o+P|cu72$Cz^&qtVAtoo8%y}KJpy$Yf2>vB
zy29__ssEJ=pKMT_`+ebao50f#(%ybP$k)8aLAJ=MCuRxH?4>7M#FroIJ2q_*@4k%o
z=^hFuwo8ueXnNq@dpPOHrY}cRYBfLigojO=_Vw7w?##KHrth8mYsYeje;X&f6>(9S
z>tFLUm}!#m)GKPrp@EaLv_j^4J^dBSRv&X={)%7UX9bi>uI5R35O&b#8SAu4a|uSL
z?*2<$b3-nyebu=6h?&IPpHkgc*Od6@xk}s@`WMuC`_O0SeK*s-BpBE{+r+u6py}{>
z6;(YEO}Tc<LpS)9!xq=R>1b4QU~Br}oxml1SN2ec;_12U=J&GJ1{lpc(qHoC(#N1h
zYyL+py?pGKuA=PP?71GB{zq*$)7<+ldBbmu*HNE8-q_=&lskD*^d^p=9T^*E<}2Pj
zkrBN~r(od{Wm(VMonLxt9==w&Fh$#Ik?N``?S*-}{59`}?z){B^0`NMtMvPovANTK
z?AxA|vU%1z&dWJ}?k&xkRynI@g=Khj#`@cDw0C4)n*Q+mM~~W#l}A682)Df6xS5yt
zyWH0;N>wjo(rwj!1t(g!%=^~${G_A{TlAenPeYb`a8Unb*;d+ZBz&WxQ7Jv}Mp)Eg
zrF$RRq*n`?r=@hL#v1SYrM$q@Ss-3-+DFf1o`b8b|Gv^$t2uAY^JOm?XTEmk`+Z$C
z;f@C9t2d9WuLS%JvHP+xHXv}_raQ}HH2>WE|K4c*$=(0=Xr--9D?PJ%7Vm1d_umh0
zmROS2q89b`&)j3xE^fzP<v%&eoPMtE!pF-8SH}haUAyI5SM;_G?Fq7%x{O(kj+|^O
zEqvs$s_tWUPx6_L6ba#53)f6+KbpKK<8i5r=WVM?uKfGfHRgX2Xn%KgH~YS2Q_tVH
zZ8bk_N2%QwLC@OzR?)Z0PDXC#u8t|t{{H(6Z~e~6%VsRGZOgpct6KK*tmL&TM#eFh
zrT*;aKJ%e(`t<b2F8?_D;<{Ftho24PzO+Q9@Nv4##P}amedPmve?N_{DUN?&A}bT_
zC#NoUiP=>py(avLpYR3+X$7s62@;m<fdX3<gd4O~+oIHVH2qgD*WOU7ud$-hC_={n
zdeG#wWaZ~U+RK~GUiiJ5qHZgeV&7rwq*3O!LXF{pp4+!F$E#%>hwK6}QVNfU1Y4Ac
zro~22YyHFA6eE$A&Q`%YB{Z(WA-YvC?6}D8rm+1+;Zhshjuuv4yW&{Ws-q|(cdbeH
z?k1;|-+UiF(lNZkX5r{;agob%qD+^GXz+2ttc6uB=|<iuCHC!^YnHixQ*xi=X2=sJ
ze!4m0=ri|YWu1G(yZn}@?lSLEbj{i5mOXDr*UYw@H78uwz35mR8htsvcFT{P6h_au
zh259j#9xNg?z9k}G=<^8XOTdmyt~t5=Bw15arFuat6O!&Yr;dTse$?bKd1G+X_xx3
zEq~*|-aA)&Z>^BJ9Z?|Vs33U2Z0{wreUlnQrZ;GODJT=}uVB`>{>WEqqj2M9IcfI<
z$#Cv|u_B2Rfd(rSBzF|KyHC(=FA}@js1O#YVcx7)F+p&ryrEdLV2j}2ixzTj6U8IL
zWREr*JPXn@)mJ*1q?p-ks+KGh+M-k0qIYvr>(&;dNDZ&F5VMtKjZ7i0K5N)Sww9!~
z`ZMQGbPWCfId$Kcau3f6ubESPEDhxy9lOmV6uy}Mb**eV9M*e1-sn=8Oo`l?PhoMD
zat+hMb&ga%QPR7!NanF%m1UrFOGH)MQ_-FsRXfGHPHnGV@FIK7@{ZD((+^Fr`CsZe
z{cw0!?a%1~oR*uvHch+MG5JODM6uclKWp@iYbQp|yreG1_(1Q(R}qg%JrCUT&REo4
z*lIFaNOMY4>+AO3ki)T`k9$8#txsb1`Pp9296nooru?g?dR@!=&L=jo-e@xlif6x3
zAsF6%bY=WQL#_vj>5|_Cex2x-b?*ol6u<J*uYG5u$k8U1iX=J168_4$;Ub+6G$+X3
zY%=edFgba`^Of_o+U1=)60EP;I!4IrYR-RuGWmO?x~(vSvE}68fXOaf%Pd~D1pY{Q
zubk@OsTmX+oAN-@<z;H(kJOZi3XhxR-kxpAt;tS>VG>8QFDn_IUa2F#M5o|!+ntB$
z2bFaH7gS6wpAlYVk@2}O<CURYPJ?h)nls}xndk-4GsR}OexL9tvt)-_wDT{Q7oM4C
zrq7W0ICsK{=|{HPtX;8W<B6<&?A^;=bR02TI{!z<>W-yhzq+Eey7#!x;8B`+$fGv;
zWB*OFnJI}Ldv+}AFIiSNE4SZeZei5&fRg0_PmDc3iFjSAoo$(STEc5#q(q&f=9@CF
zD<{0xU#YwKW5p-&{JSf3|A-YlSmFI>#>!vQ%~-zI^DBliZDL)*pz=z?LczuNl40SS
zj5++*`<7eD{!T4?BILK^dEsZ)xtBJv7E2X9Ju&y`$$2j-{e>)x4xO5CV3q=R*~I%l
z1y6g;`}AaW!^<Y=n<f7ZEfmsrSOgz$*6h^(64GqJQF>j<pLx;zuQM&4a!t6>xsD-H
zUY5gE)`C$eK|;oWaq`6lDH}ZhR+a?{Ey%jQ{)zDdr=uDH-!$waH~5=|{%2k&*5#0~
zd4u?^DV{F{1sxpU3QYYLR2ktJ9zJvGWOtp(?BVsymB~9Rr4r@#E|RlmtXh<=C#vmi
z-PJMs#p1`7L5ixGCfzp6JT`ZP*7#{foeiIHdU|%sucf?foh!|k1!ZN1{AzNaT@|eu
z9ahvmjkESDyXU@$<q@;DR!F%DU)U<6zz`qJl&Y{T#d=$s_BQnn6{!o$y*9Npa)~}L
zuWR+Hn~+pj8m;Nn8GHT3ifHDz|8=KL>wotqH`hONjoWrGP%yzdWkJDvSJ?t(r|+Wi
z6IM5T%JBX4V%3KmzS_H{ZntXW-^dy*=z8_$nmdlWj&7Xz)^gtCofB8x-gPEx9{<gG
zE4?$nMDG5Oxo%tc{GGeky*jlnW;f%PU92h%Y0H`tkIfD^k-bMRVcY53Yzt1e<g2sB
z7nQk$P8MFkQWH2ivpwWa^@0!03)rGHCnnnSb(Q}+oXX?9kpH3<_iKkY+-Z`t1cfHF
zNnD9L#MEARRVRDL{yDCzOSXn3-4xWm<v44*lgPo!jL7|u1G&wsEKU}#&-~?ndRNEA
zRr59;6LdJWxzEzw{(IKM8G({A=F?wv1^(&^s#^NrTe~}c*3!gPOJlXVVon`O(AwgA
zySpfBt9j4X%B<yqDvTd)Z9VXN>yn^tso6}a2}e$AZ*$voBrW@hOU<?&A3g3}pPBxA
z=3})LEnX{%xKm$TJ33jYcj`s2Pi!mStFOEzQU8F$XHUz@hdWj-d${sdD~n)(_{AqO
zX9e3!fAk$cxvJ`pY+cUIn+tb7J~Zdd%bk21Ijf@P9@escb~EtXk=5T-cC%J4=~iAN
z!MRo-b0YK4)rE(56)K<9WZ%s)bM54qBH4#7FK3;4HS5%StzBz*);scSiwR&_x+vgy
z_36{I19rx+oxXi~PYlyuo-_Mp&g^@`l(K=5bHaMJw)I!EO}yInUi-cNe_nYA6Zbw&
zuktIc8=QaI^WQ$}^KwIyhcusJn8d__?&B3QjT>i}A9y`y>Xh#r6C<5iYuu`@>O3vn
zbVbOS=gu_I+vf%T=ownqi0)wxQB~DUo?a0-!(4iEo#k|nw-=;rW}GoUB%-^;b9aQW
z?q>hfMl;fC;;VwJd)D=boK*2xzbQv@OU#ior%&sgJ)&28#Nh8Gqq|1}{v63&5D>kf
zC1>~L2CnV1Elr|z>MoSlU6z<~b?U5|-*sQMU&%Z(YmH|8wv$rFOSC(+8AYCGpW7%S
zwOi(pq}*Q4{vu;1ZLPwY-m<Z=tAjiRPbQwY@Ui(yWs}OclL~AinjahQ2u)yoxvQ!-
z>i@hw*H>kql#=M=zUlYw)?q90BNL2H?=C!j^7f5W+?S8?+&pY^Q%&KF$C*nL-`?=_
zVN+FLP+_oU5KujpQ}*xWng5l0+ZOF*nH3Up<5qy!K3Cb>Vb`pjBloRz)LQw-dskpv
zi+NjOMjH1O!9WK#(b+Ob=U)4_`kI9LqU5&~3R2S~4z~R~vq*Wj)BTbIuYYJizR1Np
zXYtyC^~Rt3Pn1u$t}gL%Q07u#<>l(0u&nx&xr^+b8Jkj<bmzD~jJ&r&%zeX-EWg|L
zeO?zYyV0d<bLhbJ#o@EMqyjUqM`!MtB^YmgI$+Kb9o=oGPapB}xhy{Kp~2dRetRAY
z=j}<#W=l1Aq&zR{|B^3yj#C&M85o3@uzD@r*7WF6+V#uxcJ?^#&GVeR;_?cK&DU21
zY~Jo*H)VyP&sX;+4@~=ZrmS4PFmB!~mSP>o#SW|TFW*Udn_k_sDuS<Y<*UBNn0`LR
z)z@=ZOE#_+cjEe(S+w-_YM~1Y3eJjj)ttPvYso>Y=W}c(E}#8OjAv~_@5x0zy97*J
z^jb?KBbueSqNUaDe>DkIJFC0JN09sBja@xAC%<Jn<n!`~&C98AFLh*J&N+8!q0{Nq
z1B~&f8SQf9xda%N&tvf1@oLq*SDqYuy;oQ-$}OuEQmu4UU6DK4^Qs1O^!95!w*ynJ
zRC-TYQZ(hBOa(vpxxKLt%Z@G-HEI4Iy7F8I=Yh*-!(?Yy>UnJvNtAm#b=u=Y2TYG-
zSYBk)xa@9GeadLIg7k;T{*LOlmy7eBOt<3R!XNuy@U7wJx(Al4b7HLS`)cn$`u%-?
zb+om+HLLzSKE)%`CJN4dE1=gPsHVUmnE%l5?L%YVkN#(lsO<ZwYPV-{U|^`ASgUIf
zW56edf=`SL44U!`4ii3c6nygJU@?$q(dT<@SNqwt?(x|d;x217-RI77d{?)cbN1uV
zC-+zjJl@v@{awj7amOOBRqvi&oqYOgRoK_*<~vh(8!LIPU2B>1Z|b6ERpq*;b8>Q@
z?aW@ICg#|bduquci8>|GA1|*L^<OOhzlJ$~-h|87|Ep;1VY+a$_nu(?KLP&tAEW|r
zivBy@R)1vL+LsgGp7y%Ks;Mja<<{0k-3lxEUagt;Q}g|+IrD$cz5mLrUnST3){_0}
z8CKo$xwd!mqrK~R-)!2upuGI{%7bU`KGND<o0hok_V)d6)(a)`3o>2^xg%%wt!eS1
zzo+l~{k`yy_1%lL{oi_Yu3cig*(Z4Y?ZTfv4JIvmEiVdl6<A;16)7|n`I55v=*PzM
zTUmK#b|}nw;C1`omruJ^vRa-@d+^uqe$uWPpZ6b1?mDz_sR)Cj!vdj3R#p##837Lu
zHL0=8D%3Do)X*WoJ*^_6V3A8#s?QY7hk_}R#>%JXxJ0&fd+|l@;FzN1_`kDB#YC9J
zLqLV0fyIG=f#bpC<rn7Jck`KQi7*B(_3h2NrKI{QL^n|N$dpN6UxlrSS?@Q4DJ&;q
zQ}XdV4ex2pk$S-wiXLtf(_`OJ`nKp{h`Iiqy%i59-Q<>7*RW>}$79=^8NvyNdcLdK
z{W1EG?8m_pA=csI03HGTB{OR(!v!vPX_Zietp~0$1<en88&#HdYjf`Qb!V;2UWakI
zZ&%b^u>HWp!=ksc(pOI{ie96tGGT$juaY({;Sl+V*v5vpH}<N(Ikz|A$7`;`$@=zj
z2TDJ&US9scueQ3n@x#?{_Z0$(b&SFrev3D-G@sB=Ry60)30T0w8eyO)E_G*xV)K84
zpbw7brz$oq5ej>>s9DouMgkM7OO0P6gO5*x`hpOjjjm$xC5CPi=~Fhki7ua{<u0?i
zOKY{}rb&;EuTkR&;!-MfxwKYuTg0Z5+h*iwpVC~iq{vHq^_tU5cb|CB<jokz^x50+
zuu8Fy@oAl41(W?DD;=9v6pk8OKJ8iWcUi1S*Z+Q#aE6IJx7AwSuv-ePy-uEr%z=z2
zJUV)tY>K}aJF>1|-x+mlrR##STN9ibXH2=#rO@mm+7Qy}VS3fCEkL*9>YF&-t)Y>L
zrCWDJaX-pe)J!_trx2ei^x$$(<JZy~A^cJq_JKcd`z!qa!91b4Rc6sQB}Riq2KzWx
zKkc~v{=c7+!MYvG1U%GcPulTv+1#>Uop<NoV?ChBATXo+-m(|5ua+<UFx5P+_IK-d
zby@a%N!;sqIsJIB{*qC5V*B}^O>3v^DDXVJTW{Oy^@sjE;?_SX#w=+t(J{}*@VJ}X
znhpJ&uMcf#GU#^TI-k`kyya3<C2#tr61laej~J$|O@9=}eaqrz%V}M^`Tg%UUkFTy
zK4IKCZMEypsA<aYcD`INJAdJ)B3%!c^z+)^vwv*~zWTcLgjmPhuWz>S-~4~5?DpFq
z2i9Fa%gkzZfu~KVX2QWHu@->>PRU&#R2t269v<ejn9y*9MPvt``kb&WhQ(6m*Y^~k
zd%g9emJ0j6uHz~dEJa+Z|7&!LzGzP4+2pA)cS&uT&iW&BA89<CC8Vc&T<3zfDR))o
zkB0|CI(?t?ihr(ndNU=X+j6=8itQKIh;!_C^<5<4l3nZL&w+M(<>Sx4+yDJ?fU|IH
zJcGMv(iYzH9)e$t87HlNe#yHv+qH{j*Q9gnLQ+*2r*VfYe-yg&mC72XQ)d-;bv%3u
zScTIbYK0!EQrMs(p|eM}MM3(G!lsUv34&GX_6$NDTO=L{a77+rz1k_@&XO=u@>qdl
z-RTp~9O5>N9^AU;XL<TwXceBjhMj{o@zm}~8S9t)Io;{glC*C@#gxS!7rQ>~dL+y^
zfkoGYRV(PG%fU5FOD?IN>2b-5kdpo8{9m;(RX1T`#QP^o>vn%pO|CkaJYDUx_$1wD
zsvSl=i&Q3WIw+!M)LtUVa80=Ja_+;dMK60V@&rjOd#GmS?R+*@D^vPzNs&2cB(FA~
zsg%3pS%n~LzoKc)rj0p_Rx#Cm6>p{qXa?jk@0jFQ9x!9uhbwRG9z@>O=QTRLjj>qq
z{Hz7_Ws+uTR^IlU51Cp6Hf?`3C8dD7aI*WQV-pUjYE<?~-!{+<mk+HySNz;0-Rj~?
zrK+tR30A949cVB)ue9);dvKu8<S#S*Z`Nv*|KB4!HO6SE{>~@fr9UT4-?c@pZQjX+
zys?wlSsCy3$P>;e<(i`Te#^xVsaH&2J5O08p(^0M_kVx=k&Ti|a{@Jewmz==`+L5R
z)6@e=bF-MEZyZ>cB{e_!rdQa?SDF6RPv<W9wI%%QB!{5i%{nbQ2WGM^^!_X|X>Ind
zlZ);~ak$AmGMABZDU8WF$#3r}%*dh`%P7DRb(M={%Q8_8+s4;|_qkY3C$z0A*?N87
zhgnDG|9G;_@5pQ6r4j}eoT>?Ju4{xiz1PooS)kP~nk=#Pfauy89}{vylGCnB7^-ef
zNs?gB%`jhP*x$OQU&HwA@sMnf%Sz93J#X}gd3^M^uw{|ha>H{44#_9)mW7yomcDf^
z?C|D0&vYgq=bf+WDST>0ZT_S~8(pu4-IRH_qbU2L;Q64h!sf|o;nV(abUCkI{Zi-M
zKKB{L!5m!EE3dwa_-bu-WZ46*EefkXtyR$9lK)`Wh8>SG!*e#tq(_~!O!!viK36O2
z`7@z2ek}48a~G_>9^_^|Rk!D1i6zse)>aV})juyhtg<e>{`ONv`zGH4^}b6EuP<5O
z70owb7w!14#B14;!WRoKSO@nuE}t;t<a*J!GbaC-cUr*nU(Shpm*cdzT(|Al+Nt1i
zZH40%@k=g-r<MdeFD-Cuw*R|xi}^a1tIO9ty1VF<JJXXU(OdH(^^`3QnsT@k>zpJU
z-=C4R_KVeGU3B%n%qas!-8Cm#)@(hL`8Sf`O3%hwd72FyCcLt8)aX#UuAJFCLwxNs
z)&B*RcQ<D?COk7@63`KImCXur-x$AKZ|nYFT?__!3$-TQJ*1{0(0WSi%%-3NZw{Y0
z<Q6x%)wAHp%PHFoj$bG{b3?}2EcHi(M;d=n|5>q~&4QVdXH^;|Mk}=HW@X!ZxkaA7
z^SMuT@-peCc}1C91*LLbJWqW{J%8f5_6e_<hfjq?h==T)k~;T7#Tlm0yZuB%5)P<$
z$nI%%pQ#md(n+B9#bHYiMFFjnxoawQGaEPCUz#aXe&tqvpZJY~^7REL?7s_NJb3wP
z@|&do7oIBD?oLklb0_^wrg(tJ^K(h%FPWSae+9_Cmsxc3@_99ls3(2l$8U?@mHSom
z`-yYt%v0T79E<<kEZ*z5RLwGc|E;Al4^ORY;96n7G?gh)>09K*EpHZYnjCQd$d#5S
z&utXF6z6Z{vXI%eU^UmSg{xj=3;d}{znJ*(LH7Box$|PG*0tF05&Zu3<*s|CiutL3
z1bbIZSXvVGiD|-~<>L9_dtYr;uj&<#ckMrVD9dnTi>~df`%_sAx7qb19y!&Ylbdbw
zDr$3<%jxJpGnXcPpSNXk?ArJcvp<)_?6)3HKU1Z`5PdFZciOkvOY%Br8{d1S^XtBM
zZ(-H$zY7d|6<uC0(U?2?iw?8auFuQzv`+{$ynGNno%`j3+55~2e%?vn`}dye&1okU
z_a>@4sx6q%Iiq9WZnZs|SM94_m>2b5X2qwzbr07q(l+q%=HQ&-(0pOf`N$1y-V<3K
zJMgw_;1RJDc)~e(au?U@iLyHlG$ZC#Ju;Xsyzz$Q##@>jXD!|-a(k{Av#>a`u--~T
z+2VOEvo}gCHhiM!BvU=vLek1wao6XQauwBj>daCaf8-W78oe<P2~3ioxQb7{SpIp`
zN<C)zAdd!v#e8p8O*XnbmE%$WhZPGrEvEhWylz9|Zn;U;69PJRYdf$_T=rv=aJz*-
zkHrKQO;)*23pYrte%-M4YlDgF9+S;l9p1l97#*5UIXAKUaGb4{{Jz8Hy$8R~n=L+n
zjs&FanQp0c_Q&e|PF%s&i?pP+R%vd%XuMZ_Qj>w!e~E}@+f%!?u0O!F(Lm_-<*nBX
zmA7@wNSwTQ_U!KNjf-s_&B#bTW?Qr^t;H;LGEbw&k<}}7XAAP$I9jzWSYmZhC4k9&
z(aPT9*}M-fOjdPX>0_w+#KHcN0`rR<=0cJazh_UY_g=QtNo|9pqD71}gQdmegNIwa
zIlk{&->Pt+Y03`PO3vd88@id-yYuLCOgYhcgzwA>7nP1C4UTEL9!-2AJNsJ<CZrg!
zB-l<~-NiL|xrWB#N+luw>gBag+%s1@O0ldk-q>w;ai{d{o$^;a+b{Ls^m6JFIrYP1
z{<6!5CTewf6dJi-l>4)B%CsV0)(=)QKkbURxUxloBPpsQd8d8Li~oJ=UiRAxZ|R@m
z(ey{G?Xwos%Y_Nj3uihUIe3_pF~jB1>^+Bz_Z*4tI^Vo!&z{w<ADrGL)Uh+mCh&_1
z_mtJAcK4jwz2|U@&lzW*GsQl;ynN0~ZVHrWUY^JvmcZU9!K}hz8**m)fv!d68y8)%
zUK63R=5ny<A+Lt;?llqJTPMxl8@<~$K1aeVihJjcz4wH5Q)P5*Zq4Z9^n1u{H-XV_
zhG*023BoyNj2@p5QI_8SSZm$fm%^{Lr9OP_effLIt24YGv$wyu=>0rf?@<PG#HpqK
zciGowtGzASoFvr1`@>zRLrp=4qw$N@!GpX1<g8~hZqz$4t;JzkqEP?r6DEGMPjI#T
zcks6fU_Q#DyJKxs0y{^oA7e)o{}%_rBNw=@Uf^Z1<oMmmS-r-PVbV&EgQ6jGr=1oQ
zZ;=&TJomcdM)9Sdja`%UU&<Q2nESWNu$AjjtI#32*eU;{4n5y!_)CZ5Wu=P3>J=+9
z^<LPhDl*TXzf<m0;UP5<K9%H4Pn3Ffwrp~<lwY}U{uCjG=nd^YH|O6|V4cM=b?ppQ
zjztSv4;ZhnStXJ+@Bb11-N_v}nG2hC?9OS_?+Vz_?kzCqRlk7J>}^5@J6k-C80+ua
z!n2>{sHcwW5uG#JGegcUxWZ|2!H?yr$C^IRwI_VPn)tH%=q9j6o$x6$2z@l6B|OJ9
zBIIazX=teM|Gh#jXQNtt*GMeB`K0UC>a(}+m~Is~W+`;7c~Q63r?WO)+fqy1GHTp1
zbKJ6e8k!4K&L227MXQ(d?Ap8~=RRy)`;qzbqK6v(QtrM|;Xywpzo?m_k<3~q<Nwdl
zU$8O!+Y{CLGrp#p6I6vg4h43wtm$jMvux4}|EVt+v?c_95(;pU^;)>arRz(G-`XuK
zu1sf_u!c>z&?M13;Y~ovsep+!6KC9HVzk&PV0%y|(ynUJa`{5XnC!VXI#1R$U7Xo+
zvrBPuYmkO)uExACJ7pdlH+C?+T)0BrH=@&IzHMTw=H;Lj#vBh82y2yk`MeC9HF0H3
zS6>CM#kNLkz7DPbd&Jmx{@-mE+R?e-vh|-R`?J>_YL%5|a%QbDIo-&=(@E(BkI$Ee
zJv(069P~M}f6AFXYmRJh3OUW}<IWnReChP@Gd4k6_H3VfW{2;cQ^ht%TW=o^_1Rym
zea0k|LzJN`qcu<>X8)?&=R?-)V_I|2$Tx)f?j>7aH7BMUHMUpJUX8I)W_lHRo8{Ua
z6}P*FZuj;qPX6ut(8uiI=h(+A`wkuGnRJ)s>5Q4P1D(`$jdGu!dvJ;O-D11j22!t|
z^gjI*rvCTb%}MJjtC!5MIA8gskF#UW@t+g2A8u~=a`{J=Kz*tH<`Rp1&Bly_$G>D~
z3p?IeuvVBwb;ElN{+_$>eqZ=lLj$;DI@(hIckEzn(p_+aV`+qaW^3P^iM_t+{aXU~
zrV4U?^N3n{_1fg+YdjpIPIrm+bWM?cC|VjRS#wiDberT|StA1t(~FLeJXbstoG0~6
zVD{UCN-CEU6q!|)%DqYDC<(cwzFGK_;v)^0;|_^d*Da#l9lgpwZ*=@9{6HYsK!wk|
zM0ri`vcJ1Htk+!rV!DOZr(@kB#rHEh?7ysHdAZ)~cF$XdfbRxZUOiaX_BF&?^~{+M
zM-HiO2=MiBpX%cIRLS=OlkbEk4F=nTPdfd%V*;;S@h^Si&2p8=?2gBjnbUSnk#FwQ
z%)08EcGa&gG}2XvbHmj@HRed=$m_NeCa;d(XqDJ-z<1k<|91;#b~m4L>`~NBPQG@p
zc3UjpzI0~E?B=!qPnzAnwQcsnW3R4gy!>r;`NG=Vl4q~~_ALqA-V$N-++=A^u<A2~
zzOWgSj53&cG@H3^N*_2V!1*zp@r5;`%Za+R{#`Gcm~KuG`Q&b1cHZLH{pw>^elRp?
zTxbfCWA&9__7!OGz0kD1Y2tRyDCV#m%@VD*4@rhq%<|o^fv3l$Z|e&_*9Mja8zvhc
z>^&o>%aAC#W=&<0=k)HbIg@rum>ACJz1XxWwfV%&6{l~`u3aD}w@dZN^F@~@FETy#
zEs*2E1E;0D{?!$yp69gA=IZPZ$TARnyk-@j-Q6e~5gDTi=QkL+6e}7yxBlPbF+1eQ
zVVhkgK7Mca1oFH-@h7YKMpnR7lXKmB&hB1)c6aoySuQFo&NN(W3Y1`FlwfAuk*(Ws
zW#cxR9bHF`wfY=N^Ql$4^U5G+b6@tE$qjm4ohk_`hrV<kQ85kqB7VWe^eo@rE6F;S
zRecvNxOz1<_L5_2%frx{SJ!NH=)NOjmb6(S>R4>-zt{(F?%r3idypIAHmNDm?%ur}
zYqe8@-mH;h-*@jtZbIXMuqWH>-nR4xB<Vf0)4Z}n>dWS}`q7>2j1F(_?OyvfZ&_Zt
z%WHiRrqe$+JfGwCJ~t|2%ephV9V}rH38zJ$7#vzH;rQZrUBs#63m$bTOy}-1gw6I~
zHsv|?{{V+s0r$Fu9^Z*VHYfM$NlcKPJLzk}m4_4gyaR=gE#FoUC=!;a6KE(kH&Wu;
z#W}(pLhT#n-v-LL7Ruh+$ez?(H9=VZoUV1D<_8w}itZqp^?a(vs+umFu4g38cRZz$
z)_kcz#4z=^)c*Nfe{?L7>txjN=xIn-o1;IqfqRR=0+vghTZ)cY=q+O2rtvxGg=grC
zKhN?#rn&sTcjZ9tmEE!Zfj5{uC)@}&4Gj}`5;%e3?7Ayv`WLM0QoORC_}PB;yOO+R
z-s6@JDdC|derBe1Hy1^v#@^7Lp?4|tR>)hkETM3=u!L&v$iD6yW?^@2!vfVyQ)A8I
zp0OnKUAwn!mZjnUXC6s$Q;ejN@69Y;Tl$czFJDwD&t_@%zTU;g9S@G#y_mJGOKMwz
zUL$*8SxH)1BgdTX3Db_xeBK(cq}MzDo7U&-_<U|3rl5|dz>WYmv#Xt{t@+oRq$+3U
z9>4!JFM{h#$c~dYD!$9qTs$Cj=e41MqqyAMiRZWk+TJMroH)nmWnGt}X5>r0_=loq
zD`Zlp?2jm%f9Im%!%3a@PtM9)so;GnTmIB4W=-dVd{&x8jti3(os;(w>?_-FaZ12X
zdEF!1RJ1*se{Nf3;;yrL&We8JCf$=;_<u8;IsEj>myYTFIiHWWJUJM1=a`Bs%R5ff
z_doaSJ-w^-^|94w^4AxK-1Ye&`TxneHMeJfKfQC&o5Q8i$6c#V=;j<zt={<m)polx
z2U1?ov#;_$_olWnbSXz}ZJ_Oy%ePq`C|wG>YpC!dc-@zfxToInZ|~l=6`lPy>G0cI
z&+gt|oqKsx&l8)uiRX4@_m<9O6Ip!FZ{fjt>QDDQtG#4lc>RT+)r&i&ulLQ;pUI%7
z(8$@~U$Hd(?S``6>pyb#KW7Z<(D+am#8J-S8#?1dbxHa6|8I*o@_ncgD|ub<{ZW27
zo9zeAdly*x1y8!mszgqyx&FHUE|bxO<=nwL>+c&}4ZJ9%mnboJZgYC3n3{%oU1wYW
zL*e?3KbAlIDeg3NMdYvLszwbT<?lUHW#wRGV%;Y7KjTBf!DcpLu8ga0iicadq@{A4
zy%;o&mfT1YIgpm59hMu_nsWHm0^RCux7PaH=<}JxVkmsVN5Q~@fswIEg+oc8!BK&!
zL19Kla6;e`=E+<MRv%YhSmwQ0OZU{47Z;l&*ZV!my0AKADc9UKUG8hI4zMiVl96<D
zotX9Zyy&&D-Xa_f3>+;?8(*4dT|3~--fhZVnsjBe8|U=9x2Ar)u%}w}@zv<cP51Up
z(^O^%FixAq5xB%-qC)CvwJWR4dp*zIDt&!*Me_1%Tb`D@ytAzQ^*dX~Q|~#0TZHBF
zYPvu2p6r)?Zsck?T~}sV!NWs|-lcj>oAS?}YYi^@aP??Ce}3Jc|35#yxIUTj{K~xz
z91Dd#ZltWf`ZsFrm+Mjy22bkOFua)Gz{HiZfPvLW<dT!RV#fl8zp5u5c(S{dJa7=t
z?3n1puNmUq%9-SHsaEx0dmz)il`|d%>+brY)MI#RM?{$MtqSEnJ+ZAvqh=lWs%-uB
zi-(uc+|p@_5-gw1Si)ec^)sbMJ8jydiNWhST55z`7P96sWf&e4OPI8gQ!=_IGA%n{
zm84pB+BVB&GZJU*Jes>bb7BjN*Mu<jyd^W;Zixi3oO+%cY-7$^)EU(NV$nIvTN+FH
zy}a(0-3VPJyNqeV#kmf%^H#+!pOwX>B^fg%Dob+Ns;X71Zu@Rw^;S8lr9DO9?Ei|y
zwagn239gqez4Fjih4V!~>(&)l{B%#a{Rr}@HGTagptV+2Ae4zgHK4a9(fe&c_{Y@H
zq+rF;>pLD#+8WN}!aMCsY=$p)-uB?PU#A9m+-MQo_i5JBcX5ALX*&qnY%*oj%$vBW
zQ*4&kL_LObM%KJtb*5LtciauL*%)zMYf0f2iQ?=TJAd!8-njc=)7Fh?FK%(qVqto-
z@@Q7(J~iXB>rbW`pEE!I>{d=t!Ai&cJ)E*81+L=dUy3}m)n$rK`I^sNa;@~a?A6f8
z&+lG+wsX4NwU>L3pLB@yS~!2r;ieVZw{#zQ3EomvVwqHL7FRN<ZLLVn#9Kl;V=s8A
zaiwr;26_BHnY6C;wMBB^63>sqljbdL-SN=S>GqSyx0PCt<ZR2lH;W-KVC5o5pDw=0
zrwZFGer}m)ZzdQJp{95=gXzjo)!FI0D%ED@b^UxcyXcVR^LK@P3s=5BI@yqG;rj18
z|A;K+T=o2OGgsH_g{4MabBb>mTWNlunB=~4@$JB0FC7*XRn?U;ExjMR@@0BBTUpGK
zu2oJ^Z_Kql->%`^-p&2#{I}aIm-_z4zEI(tIor)*lEa?4Pn0<=Co<Yknlx2qwZh3H
z=8TYOYb!+El4Y9bY>6t_bo}gUO-5k>O$oP+dzNJD^hhSY@>xD(`I!ktt&R-SObpus
zPvjKxFnev?z$$Ud@>r3f(*N%6fN(4EO9qDjjW~pOgc=(&6cd?dN}kKww%J6HzuS7s
z&2!skxazbO_N<VcoSCQ0Xd{@L;mplf&?fA@L^M6KN2ss(g9Yn~$abr1o$|}>Jh%Ih
z({9uDMM~-VV!5wdj;ncQOv+$M;oj_c!d^k@LW{s71qRy}uKb&%)m0a1{})`6C%x--
zutbp=k131&x;^}^9*T_zYXeHR^-L6rILNh0G_zp)pVYMz1X?fL=<YZ+XU>tlP8Vr0
zzfQB6NzcX7Cz#$`H*4o_S+n&k&l*Keo_%|dti^6gKcmd7**gLkf6Cjeestq!^Tw*>
zB4!4*l~P`oT~2&gl$M@<SNJ|?-6g)C|CwH`(%a#+;A-alReZa&LYHPKea+<j=kdut
z;oc=jvA_lE?LJMJIAw{0$VztBsaLuF>h>fFuAV03d|hSLhA&ILOFaq<^Uy8hT@w38
zNnA+9pljl`N4je|7>-_0osuo-spVJrD6+{)^!OBm_#aML5w~28W*^(q;&f*E(Uz*9
z<x-*1{yo<ZD;%CKW@Q)`CVJg*!ONwcJnPpoWoYx?*9zah?P_<~Qc3>E$=4s(7OuDX
zqb;7Tc4PNBo=UsJz0$WmOe`*4wCD}JwQ!!B>`R8CtkBi7Qde%<>a6u?)6<(dHx@}4
zgvS-xv+Y{aD)MNX&#hN(tdVPb11yw6x?Z^Mcoe-RLGwSu0nT-WEy1y;i?*KWR7vF7
z{pF=<@*{8MOA{6uIX&xHF#CvV(+AJPq8#(0TC5o-WD4IoI(PXA&gDgq@+uy@eOGz8
z``*`8+xKoy)~`M(eNO9}CTGke;ROrKS^sJ|R{e9GvS8hIjkh0*=l>RoUv`E)AbjQe
z8oqy1R^B+i@5jchx=*t^L~aD|&wI9hszT)Le|e$NxBb2-lte#R7SU+l%XPqci&BTD
z+6ke*irq^%H%w30C==LIvnJ+RkcY@dX=i1x?<H=!dA;sYj%67S^~}wL`lo9?Z9XHw
z8MpD%Q74ZlmwNfMmnL^j-jI}<W|vxYU6wnpDMP^bgrCB*vnMo6ifk<ZAD=1mtXDh!
z{;ccz=cf7;{#`Ub`tWrJcB^ysvL=i8-(Hthi@MOQW-^E4@6pQ#d^UFazDb^3cHH@8
zV86%=9r^XavV1AiCOp+tS!S`@BXskvY^T2d({GCHH@#JFVEdWk%yYz(Ez!eKuJrcJ
zZ*As$xtA7mSuXAQ!aOnj*eQj*QzsQK)}5e{(V}91D>XPn^KI_+E>AO|o+EA2*TVT*
z<}gXHYPuEXiEd65(W=|N>tgPU67DAnU&Yo}oLv3>%Ovr=f8VK9-RJ($m~Z^x?w#^?
z97`=q_1^n0t(+^QcH#a@R<C*AWekpSuPJ=*s($d_mnkd6-h7<+e{$9utE<7=9(eKX
zm-v4~X<A@MqTR&URFlPWmM5Goe#u8kYA-n{)X>|U6_gjty({9c*wbfA*G)gBz3k<p
zmp;=^t-s)LXopwcaj}l3r*)E76&!RTkLz4Jv?Sa6)!Eh?m(;&~S+ZUvCQ5a8q_5c3
zj>?|b@tRdPH(%PMCH162#_5~p(>W6la_{?4y6p2D-nq|@K90Ot?zd;Th<@b>|9zio
zOidQ6^S|6Hwk<80NouZMZsz?n7dI|DzolRb^IW$?_6he+u1VjWw)J${+Rahkr=(aH
zEseeOD`)GrgpMxO#fiILTxXkjVeLeD<=~<f_tvl|Fx<Une(Bvd(dY|i+jBW}RVFOE
z6}Cle(kA`7sah!|C;!_#J2p$Gan3$Jg(-4zZ4r%mK{tGlmTdfg_m%(u-6!Smeef&a
zy>#}Ui_&f@W-pU?Fr(&rmF5DbZQ|8OZ!*v0WH}ef61ISO+4KVwPg+iNv(lBZ*6bJ2
zIA8bkx+Q1Cx0E@JxiQBZ`F})s&D5CDY#PzbDxwf5(Yp0iu*(6pD~H;i_OIcY(9Mu|
zMv#SZ;tYnp90E*pJ1)%WT6n|3VuJIZc25-!ca8%aw;b6NalcoQvyahd%Z4e(mV{it
zdSSxG*cmewr*gVxcpUsSu~lM{yi}Z|Q18rXQ;%-zm^uAvqKuiRp>3qpGQMev-ep^P
zGC7W9ojKuVlejl?%K8Ks(O+C$j*Hs<XRx>hC_OwV`6zMGlT^8<fpY5CZmP%4(}<GO
z^jffZ?xSaOA4O#xWDA*fc<-YZGue{o&RdufC2M+op^N*9pA+8{wCm53GFTR+wDy*P
zR8XKzP>sT4u?bFfC%A69c&wZ2(WEhZUFoFOA5!amJw60-m>=uy?48ug5U167$!X6e
zZQdhae@=VcBX7NJ@s_#skIacb)-eAyi}=!uoXl0z9VHex*d%>UJhJ;PuRDv1<2!kc
zkMd4pDZ4k`{==HEM{vR3slm%%D3|-DR4l#DvMAgwPN61H!M*AF+Y{oFIS=L<$Q<@$
z{IW*SYgNjTz&n{7GZx*L%(+a#*URhmL>ZnmuAdK7-u%}TIklBzjf2Qq4-rAJrPhy5
zxL%d%4rOul5c#`z8bd=!Z|DVv+(ThsT@71hpR`2B*YNBL;J)nH>Ko&|wc@61UugKH
zy8(B)c?GX}E%g(4IfIwcExau@c9TeW#cA)gcW?dVzOFe};grki8E&Gvjc;CD-MN>;
z-j1`|beX%#B4w?mDN#zPev49-;xyCJ)U(vm^OmJ2&3p87CX3gCL!I+7pRv8Vn6%7u
zS}{k1JZsRKqiL&;s^wO>2e~l3+j!CADZA=NUXw-Fw0b3SOg(PZBzM*(s~XDx?VWMU
z!=v@Vw9Xq$juRMl_a<6*Ei9FL+S?#G-L^IT%HwHsCMS9=aSVB|^MCKMa^HkGg(utv
z<efx(oDZb`J*xgs_KD*o`MOK<cdI`6!}~yEi@Lp1LPOp32B8&=U208LOR5$vZ!*eo
zw_8!AdfrNu>r2amR;M|wLH?Xk{fp)-;9B<J`x=F1TNHRE`8Ss~az}X`4?UiJ>w)7=
zkJ>Fwwj1uOS#d6xci|)l)w8Z?s@ray<vil?;<yOk1<#m;o3^xs3LPykO7xi;`IP0R
zBa37-7lZ6A#g{uSyr>D7Xt`1@=%tH8W>}H8$kpz!Ra0hYw$ya6Ws0QDR(;l>b!4{M
zGKV8C?Y`Z0$mw#jP;0AUnXkqn{O_fMQQ}$MfRwN_)m74SWV6)Dl+?^#rB@}T|Id1~
z*e+`EQ@$6Oj5@zObrz+)l=OOaagnTIdx1~8+}lShj`siJxOC#AoZ-27tFy#*^0s*0
zYX5iU*58N=x7P?|Z53-&X<7fY%fw24(?O55SH<o>aL@BikJp)AW|qC#%1?x&!E4w3
zifb?D__4SydtPapP`>o`$BNZ^=1Mahn>|NQ)k8vM*RAwj>sIg5Qu)90c75CGJ*PDH
zN+~d&i)1WXeP40K_I2U=gC6Y9Q)!WksX2R-ZJA)^k|pyVoP2HY#>;Qcq1c%AJb&M?
zoZ~B3bp4xr>{Cip;+vys?MwHZFydOu&!%npXodN$Te=gSq%Xv*-w@&PL}sl`s*^|R
zraNyKmI{RZ-xbmynVHygH?T)@N9ofkmY$(YUGII(@tvv}F)1?U-%+u5IRR^8WxI~u
zi@aFTwb(I-V~UPi&x4gKTDVR$=FXq?_Uz1D=1LY}yJem{OH%9>*>@#B=sdOLT5kHi
zwRf+*&6YV|d(ks5<5lLTV>-*ymRYu^IQAy)OY{Ghl53WiyR45z=BQNfo4+Q?g5TIb
zEL>CIbfjR_8z&3(f^)rh&xCz0&D$vUI&Y)PSI_hst?BuetGgTc-W+<^YxCsa#HPAO
zc?@Dc&PrMhd`Ifzw5vb8ZaNkHh;@Pcg6L1@dKpjM?|yLh#ijI`RoY+IW!Jrucl`Fw
zHDk)&V=H(4j9gyxeqZ1EAOH0-nr7uU&wKA)_JpnNfoIc#c{*>THa_9Vd*SW3=9|Eb
zqeU{@eJlP8^`Bt;aQv%&x131p!%3%Fuki1a_%cy>qug>KsizkAq(r?IG0Ar9bSu#2
z><GE#WUF%8YeC24(!{V;Hzzk_28CpDsUJ8T+dcK^jUI>003BW9xrr}YUiKb%?#Ls}
zpWeZ5eZQw+t-yUv!S37}m);6C$IScn(tXKVJ=2fto<uDua`C(%zu?ew2R)_h|L)#@
zn7OEK<9*9l4|_i@nzZqe=PJqbl2XZOk1S_tPU6(O(s}<A+XFUU>Ais$D^_LYMXh+G
zRN~Pt-#P2b8h*uQuQ#m9t4=PvF2XuRX|ec!uhk+G7&C8X6};;;=X+i(c3b1&tU$LT
zW^DI!de>Din^b&ln(mcDUpDG_?s(UAMo%TbQQ6o}_wOYg@sIV-<aa+Sa(p%a_P0-W
zBU-v-TNt-Ve~3G>T~1+#TJfH5%6s&l^xG8+A3odZ^SLN&z3J;BC$1%XESKz<^xC<q
zcyD9zJ}Ig4i_sZnkJK8nf317ZIxSenJ7d1in)xPcysUD*+{phLu;yRg3a)!89eFv&
zlDdxnx_tcFf}eXjGAA7DaTQ@UFZjPF=1Iqq+mC$DZN0JP!WxFk4d<Wc9J$eb<(<S^
zy@{OMM`joW>TE9HXp6mgHq~a|(r%B;441~tHwE_d)_VMS;r@Tinw#%TOtc>fT{j7k
zcVSaE6zta(k~R&JHx=65k-Tl~EhoeI@47-hpStyl>vopMouZFo=hrThQ;=S6c(3hi
zxP8`wn>P>ME7Z|qT=v%R;k${^((go0oGZ=AS!J~D@x86-h0j-?&9CtKR6D^Ve_5C+
zw~Oj;twekEkAWQ74?CJ>Tq<yLS^IL5X7iO-W~$ol@lSmGbls(y?#!r(m?+ZV>ALIo
zm!Ko{A;wwrw`KXScid#A;rTxM{PKS%ziBiZwHfH=>95bT&Tlfw|GdqkW5T4&-|oA1
zmFeAo{_p8?hEIC;H1aKWYkqLs{y|OcN7#3(#rkE|E6UV0YVKaoyiwd1x1#m`DpT9?
z2lb+-eBOQjXLZWHy}$aqev_a{r&vMhj*OP09Qi*=C-JKG?UI?2k$2lhX`$WylU^bo
zvKAc5+}a)ulib!$*K_-sD1Kqp!$6Co%@KV4TTjfel?;0{K}1pc{>w;vFH_GgI-WCJ
z&UkI?ntr6q-LQNAj&AqDTVgIIi{IY-^<}a|>GF~zH_DE%p1zr!cZxOq;pdY(COU5w
zi8d6C|0w+0AjNJc|91lcd9A748y9sIN~T**o4-+NyJ5zn&2t{IT{yG;5l3$n``MZN
zz6)5|3tw!MeOV}{drIE<xcuB#D;D-E?)E>)nY{9%+}qr4Uz1G^J{PS7&wQDB&G;Bc
zi;l;}|C(hcK?SO|&rQ8LijH|~v^sIW#Go!>o7KN*uX;-Q5>9T>JttOkvP8VHFwIEN
z?{mTCU8}cSt=V6dBW}KWy;X6Ya`(qFljN%2Zjbfzk2sg)ojO&O8NYes<4sOx@eQ*!
zTkg)>TyI=*T6^!(;-@Q(S^i~j=gK({_u%c3kCvu7(>3~Rrk8VUN||>c_-Jl|-D-3D
z=$wvi=As;5PE_X9Hm})xyCbtf?r-gfHCMC(pPg-G_`#x_6qIx&VDS!@y%OFr3q#C~
z=(xV^)zzKpbkX620k3<FNa`Q4NjHOCR-1&bUbi*K@rQMN^5T>4XKi?M>tuJ!$=Md2
zn{B#Ze+jYQy8ga}|9|$ag3^US@Am|Kw&_`-D{AyH`1Pca*CxX1NxjOo%HKBJDJm5F
ze~K}#a{A<t(~Pt3b(yj(6I`}vYv!ubS(D?U>sR?$^+xYrcJu$mdF+!?Z<@wkUdFz*
zZ<@d<wKr<2$<m)#)m=r(T5fFXeY2|jjr_|#BPS1zkauDpxx2KsA3BsE?^@_n5SXoH
znq2a)ebdY^XO`PSX?ClgN8S1`>-lP{=gbod)_+?wxo$1X{Cw*Z`QaX~&d+)MS?lHU
zZ-t9rf8DqGozU$~_O0K&%o&5eZ;3Z=PTT&AZ@>Gb>$Y}x=7ngqP1=3<oA{A)!3~l9
z?d!H5J@&o&|NF1X5qmFx?`qPmlNWpOe<G*%2kX->`<<;<Hy_YF%_n_d{o(%XFYK%H
zoIM)duOGQw_1vK*K*1<js?)-OBjd7{gEiyYoU5y^<|S&ky_FHmaTQQH+9{+fH0Oku
z((zvQ=t>^WNmq|d5WYR-PDfDcsR??~8#Pa!Qq`WRn19P=h46BQFe}SVA2l6TFf4XI
zDHA#?gkj|pf7S_wUL{(ySA|cGI%^f08Ms)cmg%Wwn1)}F{n}Zs)zh>hf?PdidsmmN
z3)&s?agyZK5dHgsL0_5vy)G$8IMgI4A13o7|FL1O<m8(o6_=--ir%K;so=_#t~uYj
z-A^cr^W}wk*8UR~RUMjgEjq@(@voKH^jpDZJ`WH5Z*9)Ix6d-?=2^!Bd5dGZ#Gh}O
zeY)uBiJ9W*c6%*9KizGqD8O(auZg9sx9px$>Ys|0+p=zEg=YV)VtkX`xZV3g-pTb}
z`0q`ZxBnNg;jg#SQc=dotGxFuj#$aMW+98i`DtZ2nHLnAmUu8Qa2q~(=kQ_2?h7j}
z_FPKp@#S&hP~=pu3D_&@n=;Y$+m+_^wvw4K`i^~hUpyUU3%{H{F4y?y;);p;q%PU@
zesT^vsWLOB=mf7gkCDcFx5VTr3)ZP*X)J8IQ?PIGw2(;$7*EPvK4Ww`<>?yJomV~`
z*y+|ddCJ~LON#v%Umf{;ZrPih9KXH)R4$(1&oZ@V%1JKYDMlwHRHyv!(Us`!vhlF@
zUa<N6k^rZ!i<54IH2GP6-E#Trlo?K3!M=AJY;HzuJsWyAp*HkdMCRPkFs+!Qq1&JA
z5b)dasOIfd%ZFvkW>)%7U;A)tPKzz;4vbp)+_Bwf)(0j1kIvsts~<eQZQrsrS&!Ei
zPW8$>(fP<|r9$)@&l9nWZ{1|-WO>Hg6vxfVz_f>D5=Z|Y-=0ZbC%;Qg3A^=OA;jZW
zdq~TNg{=y8Hx{&WMQQBk<UaM@Q&m6c`jUP}zR0DM)0A#LnUpze<+52RPTd-Fd$h6+
z^3Lo!mA0awFH37q`8Lah-0nYKX?nS{Ix`lgINe&dy~OF4?&4QnyH@UgVqB25^jp-L
z{~Hd?d1bZfumZQW(Z-W<vjc;4yrYe;+?ck&<eZzf@4L%E(Yaf1L}lx4-;;b=cl*76
z^A?#iZj+d^UAJPX-^Zhp`Fjc%Z?S%5z4(KuLQ~yjiSv<ZOF3qUh~Ms>u;AZ;&He{3
zrcXP+FW>x4|AA#}d!kn6+;zUNqCC82(;@E4=ie`%KPEkKCCgMJk9{2Z`2j!g%*_Aj
za$#Y)+_4Mka||7&`EMDzzbSrm&;D)hea#xBlkCD-y(~5wwJJMVqck^sTO@Q`Cvu_v
z=>^;G3fO=787_a;n9)MPlyib^)+3LYH7ceyM|JABkKAICmN>ZBqrj+dQ^I0{h=V+O
zubQM>L>4bGU_12xps&r0TT@m9oY35!`pP(1L$uOQb#~}YHnVN3uJY!p&WRFD4`264
zv7}1E<=Ky8p}Q3qF(^IX$5D|MdCICQ^~46b4jY%)o5u0Kzo^USAN212Cn&z-TFR6u
zQ(D#78#(s8P+nW6%TxJQp;^Ra0jtJ5R-G$WUiGgEr`X(5b>jBykQbd~;J_C&flc{|
zye!-NE)$ooo6#!Aw3-AxS7}d@U-`r{ic#1nD=2mDk<N=f+k}n!jC9;@3k9t>G-cHf
z9_^J^ZiXei^jiPtiFT4<s(-=GEn8QlW)?Jd$J}A}S;2W(|ENljA@AffYgOH5#v4ng
zY?)?uCL|-IM4C7K-Z#yQ7ta1SuIyfTFUeRs?BX)hZJNs;G%(8~$+BBsvOE{K;_;jp
zA?}6OJC`%qm5QiHs4`6s?lN1vz*54@n_;f>MVBNM+wULJ6Ut-mI)}E;mz}!KXT#=O
z+ls!({j%V2I^`Wyziq|FKe8t(B%8yfgRcaNF6>yUT`JfzN8DCYb&A{d;0wO_OKj)7
z3h{C}B>1!Mips*a%oWd)<(vC1sV(JN>1&(yD&W?EUHYA!^^99y{pUHcEI5nXPk2x8
zyaT^NH(ugws^^*=DcO9DPw!Q;e9IN~lgcZ(VqY20x+JoqPJC6s$+IC(8x?aHPOp-B
zG+~3;l&O5JR}aT?az|V+4BOBlu_m51SkwFeleFh=l-z@OjwlLtpV;Y8r28vjO|-J;
z%QtOCdaKuj+09Ws5v_4juUyiHp?b~RK#el@Hj#X1_StXqdD&tfvV|wso2WQ5Ng8%2
zu1VUxHNsPK(vt3kGs&{tH%r~yHcg(mM(qykQ;F?d)2?K$PO~(AfA)fR<op>%nf4b?
zO5D{J`M~xx(?eRVLeI-|mq2jii8E303o^|%*iJr>DlC1=AXxHj%;ed5#%J$ry6L<7
zWAC9_?&Yn;$_ypn(h54Fms>AqPCDLOab)sp86nN(iXx9c&tkedhhd8V>*5kW?!d+{
zpSA6`)aUjkxXsjPc336yMD%5Y*wnj?9nX_DtkAHN70vqp+iBI>NuR!HXyk61dVQ+g
zRJ;DudRYQrmx-sf+?aCa!^abS8;;w~`}d-xv}E(d2Ri${s%D=1y*tU^b^Css-z!R1
zy?%7;Ma<RzytjOqV}$$WnQdkBJL?r!Iyq46?EzcI3yo9nb+cRUIJh-$R$w0M!uYM8
za=x`}A&X9}HghgrB|go)tx@((0&nyU*<TZ`IrVg{?VgdXBp0~%{}-MNgKTk0`$8k8
zRUBK@HhCEMc9fKF^WOaX3|C{RuWyRW1RtYm>$+D2Zp+$}tB{{9Qt<qu!V|vjHJ>v2
z3)Giv+p3Vay?(b_h4}P)0^WH=oNPheo`p(1DN9t8B|p53GT*JH`NPZT|I3q8Y9~ER
z5lq#Tw_7@&bLpO_uBBlqH@$k2ZM?SCROnrH60?d*)y@0)w6=Qh!aaYcKmC{b%xG)d
z3<i0L=EXmHvWy<@-gdV4;LQ|++sg!IFPL0B$JJbY%_+~;J6~3rA667A?0M|B(z!kS
z{KUS}4Slk&Wjw5Z8J)7TS7~5zVVKYSRA$*Cg;tFxlY<p!O$jm33~p^|U+9&XF`H9%
z_n*3@H{~4r^+T){vUi%C(tMsYxp4LmZT)#C^>}N(DC8>{UO#bW>iL&X%Wnmr{ms5Y
zYGra@XH|&$)AY~5Mti6E$zRn|6y3UT|Et%ROI|Dw{e9xd6n~cQCSkT;Gr#XPzp|i?
zU*o?MtJdY*w_g^Y`{ou9eX6CRbl=MFugpUJmWHm&o4O`>ZtYR4X|L|&=WJMYE@GAR
zKaFX*(iJH^mW(P7i&%9tZtXBLN(s*HnmhMVvyJ!0&YOB(>fW7dtV-LStc{(b)Su&h
z=iFI_^KTultYyf5epjJjhsuQs?|gTv9Ao-<xZ;~`@}kwu&aau*bMNNK+r4EqOFYw*
zs-(Ri+S3AFrq?~&lW-xuAuU7UT<WA}sc*Ws78zXp#{Klp@hq+*-x@moekC2?@i~yP
z?f^%QuvbQKNnf6qa^4nAE?J#Jua+Is%E>P{d?>2<@G^(PJna@j%YU!yJ91`2;Uo`D
zD-}ki_BTH?J0~doXIiu%GJzx5p#7~1L&g=oV`k~xPdr@gGJd4!zvuG}WcrvR)FG0U
z=<CO>sNiz?f^xD4`>g|qw`l}^R529hW%_7gc=}rF9i@+glb2TpwVXcm=JccHWviBd
z$~kgo)AKV46Wvojy16B(J>prBx6$*}n{!<JCm*GsZ<zS?kb1XST5ZonfsI`bJZBqf
z9_c=6(U`YUU!k{AO(X4&d(*N)DaN+T2EEeV?|<Aowf>kh6Vn3YWebAeas3L&mA!be
z^_Ny#gzkn6j<5vY-vt~S>onxEc{}#>^K5(HzNqidlt<TZxG`MQd$TB^=*D~JKb=Jr
zT$%WUls*S2UGFo#v@TbxNNI7y|2GPgl&`;geoSF<&sminzV`$I7-csIHD~s+G^oZ-
zy32MUHpz&opL=4C(0!u|$4gF6{KY$oO;4Gj?|~ZU<c>|MJ)c;l_0&C9s>dwTe5<6L
zBA}(sr)9uru)s}g|IsP4E<LX3d{WY>Rm1u8o1enZ?8Zlno+jx%OMCb9#-+eHO_9q^
za@Oa~UJ<}4;=uJh=uk-Vi)|BL9PylEvc{BS`AaVMxjm0xF5{bbYvu2r#a5>-7(VY5
z-lNjFC+~0HqIiYpMK4~pTzSveuf%`wFt2X93I7pmm&x;|JhJ|k`sNuc<DZvtF~##Q
z1r(_`aI>jANmwGraNhA-#)myETZH^oA1n8_P5z(I@p)U9vE@WzOX>F?qq0sb`HO0v
zuZsA%EK2Zm0ZWb7`Q=51UmVsi{IJ-4O`uKSdG~o;$|38!WBe~9oL{x6>zfK+<fqlE
zEBM!(>RR(^ve6Z-9}0&Bc#r;2d%RZ8RmSvikXvr+mseNj>0e!@FUNXXZmYi2$G|^_
z)^9wNzfo(7&cVe8Egn1TJ$LrpY;98F!k@y;)WU1mc6*1e>V93uXc5N562|CH8dr4h
z=4Gor&Sh#peV2oiS^vULwPQ&;GXB5e@mz6wmy-bt_c@lpw$z|ysR?O&_NcI}6xq9g
z$!1T|%Df9~v)#OlE_ff@@~vh=)|G6X<5M{L4Sa7;I&h(__*nD*_;~`RAst+y8GbuY
zhG_ZB`O_Bq&BQ!Gxqzp=U=nlT0_9g?&kfISDD-*bU{Gw`w}@BK)W%Ku?HqoSNFGx|
z(@3eQftEtX_N~hNF7YwJMOwx4Z{*KklHlOsD9Cz5u(q*RZ({eGb4$K>xcM6i^9p?E
z+3ebGtXNk3F^^s7@#53_)`;xYaSIR>Rb4N7`Ha=vsb}H}#jdaME(okyW9m~T*xitp
zRwpREbjqd`r|NG_ZaDGv!mN)^^3Gq}v*^+nZE4-LQd>Vt@4YB}yNlN#)kaH&vFQL`
z<cc8SEq>t{SAMO!`eI{(vWQc1s#D=x`R{@9mAx+fE_}|+^H2F3yM4Bf6aJqxk@d}+
zrex_-lVhq~9zolDF1aRO*%q)ZZ0l0SXd$)OFxQ8??(78<vWzAObh=mf?3}TB;tyU8
z)^q9gLdtFe56xaa2xDWF<$LmJv2RwI=96Zx@RgYhS-$SLG|lZ@)?uNTBSFdUou;-0
zT#V7vojBpy3AYN5g)R3=avO4IPkNwtJ&<#rfqwAgLw_daO?=0l%*?a@gL#*-MVqtL
zf;_`Y*LhD?8ouo;*zV_Ymz_^l-GrOV#b)Uro%A<upG}VHz1g?!?-l<y7GauhR~0pv
z8|`&5WmSF`^)R&3xKhC^b?Rq}*z)kH#lqL0S_ZEwi|&e;o&G7TaAC#w$nsZK%coj5
zKKZXwcF4N-yEVUx_vEbAr-Ca#9zWNX60>ystu0!1<|>!&e3g8wwC2_4)hfKcKR-(g
zss`{q{lR2nu)?3|jVhy0Yy5xHRuPln7hW>yFMjcTtK0bA>105JO7}Ij?(B)IA>W@C
zF-l}ug){K&_|xT^GPyS;^|b?EK<ZniCdmnI>08|Nccgk7rKN6a&itfSYtk~aJIO|1
z632Sa2_~NQ%TmIcJojpA?havHD`>Oql2@)pdYpmQ^zYtN+uC<eX+J2mze)2_$Bs+S
zCi^Jz9+(o~d-aOumIY?V3lEwtKKSZNIYU_Mt!H^Z+_-mEaPPJ#F!Cv|@d>!CX(?bD
zD13E(h)%=S<o{;2-+A9I^1IIO;`-TIDIrc{d-7A|#_z5n!V8RCziaGy*LR)2%dF8S
zO2wAvn7C9+iP*EUQ@J1b*d`rpo|wH=E^e!<VI*Vm)XAJ~B94+vyGo~c8b(euT^LYV
zUOPSV;+m<+8>_cWc3dSY&Z^?@%l(TVcjTfc;(|{k8z)Zp)~;GS-8NZY^7K^6t0~#r
zr%T-pRoM73c6Z?AC6bqqUFZM5%0X3uLr8(a;E46L?Xw?VvDP~hB=)8}<gM)5e7TS1
zb4#5Q_pVQT_%o4v#wH!{lIJs%`746IM)avImStNcc1v}^{i9*N+ZJd_L>gEuhz`5U
z5|w6KaW6xxh4tjllCb+T-u|Dcw@khHTe|&+Nt?P=B9b!t%{1-LJw9eUW7QOWoxJG1
zR~~z<o4&88O?d78XOSK4oVwi`X3qFAvnAc&m|)w35Mu@ghK7U{53j^^{c?Kay3%k-
z+=GzWr*GzM`Jn&S%%FFj!SAyC=EeET`^<$l2W))zYM%4!4TtARel@!I>$RB*@5|Qt
z2A`e9g-qX8#92(|mEWppEL>#f62GH)y}W4S*Ox`c?eo_L+Zshq44v+}_2~ltY2nj9
zmqv@1WnQ(6&8iH^sw~i1xXFEL^>nL7aqH%CYwu+h`#)E>=f_SjZ<F_}JSn_&;Twsi
z+h?Tm+sX50D0OHS@Y{U2T>biU_41oFjko^?+I5H=Yxw1&>O5h0`4sD&Gk2G}9d%^)
zKAhvc>*KV!s#C62usZ)X+<kh-vc1(VKO^QFe*f$8Bw1m}!kbGQB&*zdllI=NUMMTE
zm-Sb3=jpo*r~BpD5}hPGW@~@!n8G~siHE6EtLuhV&kwELY5T&gw?=O83cjV+U~%YR
z$f391PlNcXFHha{LD+xVj>F7njO%^{_x<Yklv(L5<Lo2DA;7>Opm5aC#rU2Y_s$F5
z-RHPX-yG|03V6!Arc$O?a`_*n=SOY@*!cxJD=Bm~#T@-`b1if9+TW`4KeLwXZ7vcK
zil4N?dBReot;&q2lDWkM7oNKv+f_V?J7Cg8_SEx#ZW~qppKMxr*7&B}ojsGwuij}e
zo#H83?t5{{!JNpS%~Ka%J-^gc?5d7+Zs0k;Mbl0msTY}B`)rZ;(o>DsOzWh2y(U<c
zT{*7I<tu6WcZTWThKhiW>!Jzsu6>v{v&q6{R^Z&VlitTAddxnub7x}h&eXlLm%d-m
z^X=CP>+6ZG*Nabt3Y?y+`z7$#$M6<g1;!cu`)@bQmf3LimiMotzQI=-_r6^a9HzQ<
z?%uaesku`Y#!kC^`R&5OE%(%ZrR9pL7ncd?-<oK&E4^7L!}!nx2dRe%N%tm(X)5z+
zWJT{SxX~6HV7KqemwiF(H=DMGNXMT2vRnFv&cSapt_9_szNR@tpe@AJ<e$ZVhAFpJ
zg#LLnwd&}9iN}_9D?R@>t?SHRG3&*eUr#P-uKqnOVEe7ryrqWXp9?nk&C9T}o>6;L
zrgYC#r#<;`yc1cCf2KBE{<Z#XW$)rU8cdQeS#JhTzW4Un;`<Q?H!Nk{$i1xeoyOT(
zp4X-~T_=CuARoP{Ke(*-_{SV`SC5?$yL&b--+Vs%)Tf&A>ZC|7tNtzdJGUI1d~k00
zLGF16?=QcYeA_m)|4GG#tp}a2uuigD*!KMgOPqgf&41$^HxJBm<XxY(_fxX%HG9_Q
zawZ${YOdSv|GZo}f93C8rOe?4OPzM}|2d|AGjjje^ffkTtoOC|e2x6JcW3Ovi_&Q?
zfA7<r;65??|9-=Qbn^=>wgz8<W*l(8(CR%QwZOpVtKHr_i(lXOrcIfCNd3<*ZJYaz
zYWs!cW@OCz^I`MhUW1uR|CX<fIO1FLIMn8G^q!;tZ~VP}|L@FON8IJ|e|`G5RiMD^
z4fn>X9>%8bhDIh%A(0cR3lFvONLxK=N?72+D9FGwMIg|DsZ&}t>WYS8lMAOj>y#%F
zMF~wDlCFzbECZh&XK_%uCez`<(8OV?Jnamp@N$<0QA<Q}kFE;Uj((eSQ#jy!m~Ch9
zZQc~FtJe;!jNa<ht9kMIm5l*M>twUH#ogMPvHY4)!q?p3o!ee`ebI7QeQ%HV&sVu$
zuZbn>Yh=_njo~&-3hT@^{Ll91#zuw{lU0M$&UiUITQXC<+|5cLQ~Uf{k6t_53ni~E
z3P(0iJ&|?gz>UqR+4C&AuH<%Zi=KQcX4ce~_Yb$S>-YVsSRHsMd9u<`we1ZrcHQ*3
zvqU=Q+x0uU`QPx)+uq=t;x<t+U+(YEPp?ldP7e(^CH6B&MP<SSmU<?J-@iZqe;?n#
zz^9Q=|MTMZqax896BNX`{1h2lf-NKxTZQ>l5?e(4G!{0;<UCx+px9UOu$^fl<Dw4D
zMI95$_4aWj_ZXe?c%03#?Xp`?;LXM1A<=t}h59)9nLNq%_bqu6>i^L~bTX5{%PAp&
z?=(_F!oniEr*1U~JwGj|^5#;8$UMtsGyaElYF?Z+b?f3Ovl=#9O3f+IG)<pVqScx{
zZ;|Fyk@;Rvf2gN7Y&yi2-l~);tKD)g?XE-bE{|i`6ZuS5GECj~<Hh2ck;h-=Uv``x
zwQQy5me^k7CiAwQr=96>tKUrgq*cAR=b_e`*&Yh_#XhS}U|Ij`BRdPj0VRet8#q{^
zbr_Cwc?)koVRkxxy_b&x>(*;_9=T0d+;+Sbk#^X@$}UiJ@L<!ES*IHv4OLSFoVW{{
z1om*AJT0JpPxYhJKFOUNQy6|Ul?r>vt~4pw|JSeO{Q>5`+JP>XZ+CC-QZ$sAEVA>~
zp_vJOotmP@UHwA3!}EDXj7~btdvh%^e(%(x(;2&R#AE(fnNAfy>+|+&___Aj)ENn#
zq01tY9r@ozB+n4-y_q{(bFE~?oYrYqQiA3$i^>dH?Do|mcgK$xE{m*wMU@x|syh@N
ziOe;-b!ekwOiuQDv&U+CVqeY8d04A{&!On?+V2%_c3zKraO2=E*TBzbm*21T^k{Tw
zWU+f!@#N3l<wry~(mHmw&Pwn5*(O(%@u)?iD{oPkuF{mGmrG}#s(QU{|2^5JDz}r#
zZ#SIY@nrI@=lfLN?z^t1IwhPhUUgbj+0E48w-a{Od_H4uKOr>VD>wA(6@TC5Unaji
zS^K@BNKNX;8lF`ei@NlpG?riXTw)h7)s1Vv%g6oaOK-ece?Rlz!fQ8fyZqnv*MHR}
z&J7`YujG5)sI6w%wJJ>G(AB1A!i{mB6T&$im^Nk<>F{WAb4ao{)mv~J`YV&&BF7TO
z)HH!X^}(yC`@h1PqF$_Juhn*Xbm@)@*QRyXcf3}#$vNQ@7c!~-t-C7w=1xB$hK<*G
z)~GrZo^<eHa-Hd6s39uTAuL)psk`JzW6+JuXBU`kS~I6pE99fsgqkHrn>ZA7(-V^?
z-v|=hz9v=AXtDR1PetPUUj!ys%k-S*D4ua7!!@H<F!+K}v&Z7lG~-E$J?S&Pop7I}
zrWv~W$yK$<b1${HDb0E^`+9=1tZ(b3hFK?{+-zGsCs@jT>i$GO?bXT-8!JDR-Q78x
z!Q+3W_s13K)8AI|N%k}?`1N6Vp>WF_L)CVle=H7+mnR((5jnEpT;VzenM(of>rN_g
zb#|1fY6ZU9Q@JQ_UdOa2I?FA!w=w>l;ieh$iFL}nPlk#UKe`)w1x;c*WWD|a$9&o9
z^bh|$ZK_jVdPhAHn!&eg&y&5W3p2eYe_Qf=nU!aVpUByHuD?~J%o4ZwH)I`Mz*f4#
zRng11`_-9WNiOzYC-axy`+3XLHgua?qt>$D!VgyDRh^64$2L8|aY1Fm*UMjTa24*H
z)F7nM!NTSBp-IL^Y|Fe~YgSmDj{NKRuyNhgwONVYPR4hf!m~Zr6)yF55jwaqlK+yP
z;Kl5<T#?!l8q?PQzb+D}!FRcnLGGD;-AmKXzezXtH7wfDP}rWxu;E6{i3K7-SCmq0
zxFpZ`xf$)5GI8D~o5iOhzHK^_RJK|BwT^6Vnz7A2pDne2x)nBlGhVdvTE-gRTNh()
zdo0t<HS_J3x@J&m8uCbN^6VT<%~-9|dsq!})1sFa+;WiFw)SjrZ1u%^E!A_G-j(dy
zxl#4OHUW#z8IBA=58l-*xv)C%k_zJ!+hvuLC3iABI9}BL_|CI?ukZb+JLeqoL7*e3
z{r=^deQPYvtgQd<z4%aWqHyJ}yT=&b$liOKckn>=hbeMCdpM-udvmHzcADFs_{{T2
zRq~S$SL|w{ZF0|7SKfN@{ZPBe1gZaCfj_c0{<?L2?v$vkV<pVo1<^d)ZVL+Jv)n0U
zsD7m+FMJ_yyN|+}KA(_pTn_faIiJ|)=Q#0uRqhF%W)!@VCpF%aadlK{VoRDut5ln0
z49}z!O+H80$*rnXGd{^76mN8-BXfxg<8Dr0o2GP$(1}X+8<PY=izajMxhDNf>h!u}
zGpW9CBG2nDlWgk>m#%xVQD@3loz$Jl{PBNQc*dXnB5nUyboGR#TBR!&ul#IeH0N#L
zmfD|-SO2}SIbwd;gw7>ADI%G#-Ck8~+s(+Esn%)g{i*E6+%1_KbCZLfXQWNpaA)K4
zS<Rl0#maA;t5hxd^*p)w)}?7XCU)FmSS}Y^I^EIm%S5M2=8|`xS4r<>4ssAud*G9y
zmZ9+Z*aR1al>F*sW9P-2)f|sxuKdF$Q>yM2>%hRE_>+ZIgn@}chk=2Cp@GSPk%8ks
z1M3z+hYg~RjqL0~CB+vW9BN}f&7l#xENP}t3SW_+L)#g*pcIEmRuu-yj+14)g=AC{
zuFUY&jhAcj+_bF0ck7$0S)twy2g4R0S(Cgq@9Kp`OdaclHf(u$a4E||qgt=4%dW0|
z#L^?I;HB-&_-J37s>V9)8-cOgXI-t@y7u<=g4=njGp^;_oW_1}-I?rf+6N1hcFM)P
zHrKv;Fz33ISWU#*YoVF^Oy_QITF+ouvAfS`&G!7Lz0rTq9sRvs>-L^m5nESZT^DN2
z_${ZCXQtp&&#Bvsg;{2nq`dE0Q^BroF(;(JY%i;Xdh4Ewf(K{n-{<Ku>?|@nwpx9y
z-6YX(ukY@!{?G4kx3c1?=j|!!rgNrNeZG6K`uodyp{sv?i{6^(nilMF{rk>KT#|Eg
zTqZO!DV%6+<eGG()kT0yK+#b!Oygmz;IyVhX;Ir3iIUQ-GlJZfOcA>4k<w!%RHVHs
z;;x3~H}_*=e>pd_hMG@e5$;`k<C3z!%|>IdEa$aO-qC9uj-GP#7TSJllDF<ml_|H^
zCOr)aStlfw(p0)NEGEBns;KuO*P~OWwmB~4iU`!5_ADgmXyjFm)hSO?rzbpnd2M>f
z%_Gkf>t1PQEXWqUr8>PtZ5P+fZnIe$j!`{_Zu!}91^8>vw%N|98PB_2R$DL1{6*Q~
zX<4sU7XECzm&d;HmVoBk1OK(M*KDbJF2Bb9nAPJR>8sO%nw8jQE3697^qA<HI$`3Q
z4$Z<V7jw?+Y7OeD*vTU7ef`i?ubj1|+*2gp2&f*gdfew7GOb|Q>nSt+JX2qV_2o{z
z8ll~(DY56sGi!}~pWn5LdB=4vnwjuF%x#&*f`_f|59Juz%sR~LerD319N#-P5B{$T
zTOY@!EF*b%d6p5I_2g}84Z1RC*%pUhcDwz?@vPx?nKF~}I>u$kZ_X(`Ykm5T@j3I~
zGGEG1H!hDUblLtr?&7<n?st~Isof>J#&>(t{i|W=x!RK7OaI<$%+r-nY7X;sS?G}G
zVKU)XX1qt@ZP7z_+cwGZX)E2BqjBABMNRtuBX0K%eyRWHVSk#DyxREWkL0Cmt{jq^
zxMhv-&xzrtk&~YmI!&z%@$OAl)t+as=@lI3^mX}y_@a|n!u*Ts)TW1(iB^ZaniIME
znaV@HX;Y)-S$>$?^h>p7WlLb0#0S&4>+%;g7+n)ubfH{pudnmH=gI!l<b1i7oBw>K
zwBl~<d+|v=)AA=jojTk9)vwc=U)A}pQ~I}W)tA%zwbpKa=KoqP`9PGHwv^OGu`1I8
z@tIE=5*{BCd6y>Is?egqSEAr46VYZC;1Vfr$mv#OEPa2AkGXuzOrH&{;<L7-3g5oP
znH>|%9h7m<<4M=Tz+<M)OEgq9Ju;_fK1o?7cPQ0xzTo7D|4g4Z2JL*UYcr?Iw@*X-
z$c5K>hd!Tqzv}Ds^%iY9lTIdOZmW<od)wN5ID4MJ>W-tUSuScPs+<+N&$P;8ZGq|A
z4X(BuoTcwvdD5(}-1&5lvdsO8SN*#y{ferDjb8{X?{~lKckYJs7Pmf^v=4^2DwHfw
zF7ws3y<2&H(uy|O)hmxr4!M24_Sa>$zbpbdD`dj7M3*S=xv)5I*fK#US@LvB<BMsB
zcFnHxaZ!3(vEKCn&z*IDl6NHCOHng>G^>A|xJsDAIgi|ha~PI*a;t95^Gcs|$u#=A
zW$D^?mE}Qk^A2gwFyG2uZgu6M&bck;v^f*R!i&`3@40E+V5zT{l6C*%9QXg{>b`Z(
z|L!t%;R#RmWg1<9+MSc;Y;oY4#K@Vt@5a2skSFs4S=_z!SIu9+wdj0bm59&B<g3b_
zP5URk$`3P{bah=u$blHS3rqE$X8i6;6^x&_#7r@eFSjCDddHK=Etxsq+OHD3^L$UW
zx_8dm*;OR6_|Av6mrK%puWeng({?n@=}6WQyDU*Pq0@enmwOIqDTnn)u1W5^nI-P-
zJgGg?P*31=SMnRnuAY~w^Lry-ubgJW=F4y*W!r}KbuNo$-@frBwcS?bf|$3*T+ZB$
zW{+=PXxet-_$?kAOJgY&-EPmfu|kU%T%DD%(JjiKuR|o2!Sh~LlR~K83x~C(+_%-f
zoZ|eyH8(e-dBPnz7xPfLfJo)1(gwy}F=jk|1+C7{ax)g~Nd2~=(6e#Q6P0k`tX2N=
z{40ZA#C+d%q4Ay1!e+To9pC+}W-j&f3VHPG((EJO!dgpxg=8(5qb8f&n!e}vm**eY
zwu%2^67gYXU6FP~dTLYCt75iq-5Ogi>}daXLS2mY<g!Uhdp|_ZsFkRj)n<Gq@yp8o
z)m5wR|9SIVYx1lgOLjeCS#J?^py$h}?YyrJF$tSIW;1kI#<XLj|EDJcd{?(9sXAQP
zQr6{ebMH&{@hxGT_nVv+eLAw@O^1l?jg2A8Z+$ow_;q1*?T={}HGb$s+HyypVUjp1
zkslD9wm!3nJ;5OV+5d#EUB;2md#)QT-`aC7x+i+)j^}#MSmWdKw2!ahn6+zOWli|U
zRe=kXRD35e^=bF`vL_!gvi|mELVJtf8H2qt52o;E_;*WQDDtuo^nX6_QKOD(@LsvF
z;2^uzL48-PEa$B&6bhf8<0{FOGWGhcC5|ga8nknbPDeg6b2`3G^=?6?`R=7*w%Shq
zU2VU&C!FeVJL03~Sd=0(<Hz%qZea)A8yC4=YEA9*EShAtP~?2sa^rcPg6@lL(&KCt
z{paMGO_{r=BhKRB?zyWo(#vK)dUsuNkLJQe;eGX9u6wqfpU<*ZV&1Wo{9nhG2E;JQ
zh48I%``Y-;^IFiuW?fd1Hw9U@*8Fdp*yZ-oT&qaph{5%6kL1`Sk=uKg=(J^Sj=j~I
zrhY-os_X<aYbckz@@o~xvM~LHrJ1%*jic2{POIG#aQE2WCOT0_JXl=!s>-IyN}khN
zQ7qrm`dNG4$XR#C{Jmqho<HlY9JhA(u_}qapWKP*tecN$vTba6eR^$8_>2eY$CkA`
z_71S}xp%@#t-)g1n!Zb`zO?=1xY-?aZIgX*&ePVk%FyU-Y4h&pR42?m`J?1`mZ#*}
zSKh1VuX`KCu)6o>&9wL1Zp~i%?YQ4;*9z`MEMNC+_0Hzp8XlI*_1pL9^{{sjBfjN6
zU$jR3>9XJ_m*)EFq+Z^A!`<NhA#Lj$f;;ZDWzG8^`mX!QHkOK#=WqX2-dxFdZ>xRT
zp2^?e%rJUTz5D5DXQjz%U(SE{^f<itTX;U}R7r*({wjiXOlI*+H#9_^srXFHQ!rQc
zpCrY*&FAX^ncIiCW+w%Tx+RLU*#$iolQXN#nWFx}xcG=sv65S{N@=miH+$7XLHcP5
zAsYm3)0@|P4GJkx2wspYSy=5*Az)QjEwUw9$T3(_MbNWcaPK8Smt(=x#9DPHDk-Z7
zvj~=kZRE^x2r0N?8mPwQwXrm7S!(_Fu=J3$Wb<^UX31=e@*EAW8IIxQ2g{B&r~kO-
zVs<e-DpZVv*|p$BTf>dE+!LaO8m`G=Y2m7AnWe5d>}s|D&20FCGbRd3J}Fnb<0|mP
zN$8CO_ZNe_WCJdpE4q!Q5tgcz9jWeBBC^K~J-mg4>YupZ+34QCMdkiM*%_(b8#z3u
zwixfek+p9}*6Tolt<Pk`owIkQIWjy+DqzgvG|E|9B5*iN*fB$KXL!!3V>wpeJzhC>
zFKm)KdR$4Xyy(c3n%m2APi!$d*J}H1A?th#)h?&}D5LzoOB&2+aT5yrUraBQ3@_yH
zko>8_HH%>aljH=(o2tLB7gRA7{@F3%$rk?~QzTg|Coov5Fm0$8n_17%nej6zjcaB>
zN0}9e#H6QN6NAlCzg*~i6TtuF0>5@7|K9-qe+-isTuPWyWTd%RbI$)qnr9Cd)fhKz
z`aapPJjrRMz4p>#x1GftTfAgcOZZeu!WSkVODYjdjPSfJ7~_x<RN?-8dkRl`Q$=8D
zeW`rZ0?zJf&M_fUwcE>^#SK3e85X9OF@0{2ZImpjl$^ehvvlV4vYGL+$>D{^+pDH`
zWKQo04=rnpoRMEyR$Q(=!%-lqVS3rh=~a~t9ff82GYzGkr34PA=a-5$Oq-}%7;ZYT
zbB%!BY!Ci95!`x_{Cgiv`ukzl2Fclf6&TiQGW=7>6AR^^>?APBEk!$7*tJY_#&(&h
zS1V_UM;?x>-1)*|`-{qX+f5dmcYoaIY3~?yaG~j&K<(IIE*7S0hnLZg9@P*3XI8Io
zQIr=_+CN=EaijU|&%K92V;Y?MPK#&nU6OQGJWs1-x|pfew~o4R%lelpyZyh?@3E|Y
zmf*yh3KM1qOkA{K0q3p-jGPO3qb9N^PGq0urWCm#J*+*eB;L7bfkM;-9<7BcR*QI|
zCdl6sn&lwmICa95hw<e~4HZgt{)Uq;mrl}JIZ5Z|B*T-7S4&O~|2T<7amkj-MwUSS
z)ec-r+l5uO%RG{>SCCjb*U9eJCU^NTEv*dE$Sh&)7s7fLQ(Sizo6b<M)?kPYSQgW;
zEaO(QNtr-Sl|{JcyhoAqicTpCH|j{22)eTgsRT*OO_{2ACD(AGZULik;g^)&<5JNM
z)7t)<aZMJSRT>)3p}exsicL<LP0pAty;W?6RoQf@>G>zx$}`0(R&pl)oDt<9uuO{~
zxpLL&TdS5vP3-qvSf<%gRJpoevtz5)^xaac8Z$&oG#5?KT)ix6)yl3_<u6xl$`Wn6
zG^2h-8|&xb-mLVx>kHGL=>;iSuDIE`>*cJqKWFV_nEg;{cG$y7?pCvJ7B6x35^~GZ
z{I`KG`9kY@#=Hnlr-m%i^)oyyn!6WFlbu~QXOl<qOjgfZiE;+Pa=Tt+?cI^}C9!9_
zS@zFBuZ==ERfkwFt`HQzon68uzTI8uA*bT=m(_beRi9@Qy4#_o9vyQ^(dUMnMdgv)
za1G_f4{9E%<t_U!RsOrZ{Lk`55-01$Zx*o6tQV7-=%UTWpS6W)0;k*U1s<;#xMgpd
zHGwlqVe6WxiC*3UUaz<C-CE$|y@g@t7XDvb6Hkk7e6^LMY7vvB#G+kdzSax5ySDaL
zZuPX@E)XTmsyR{YRZ;H6`tZa+gAm`~Ws9|MPO_7l+{QiGpi*G-&K=@AX7%jeu|;y{
z1nZrXqIaq|a!=YM)m_}ErIe_yZgcUQH|y`YOK#gaSME|!Z05WaRJM4T;-O`kQc4+K
zLbeskN=X(Ri2_220?Z2)x0|=PX((2_($+UixsVjG{Ko``sVP3sgk6s=Q8Q+n^r3TF
z*UD*CdkZ?I-_)MD^w*4gxA#8y|9kJF=rtQ|tqPsJYNge7m4^$0y|;UI?+dZs`r`De
zC(-+#X7B%2z3)r(zTdO=zxcf>lv|VW&4PEox4p04w>oRP$gV{bUhbV38UIhH-#u{F
zoEfw3YR#6=nQeBFd*beOxfj=kGah1BJoHM7VXddYKL`Hq?@~H45nr>q+@ERteyTj_
zIj1jm&Qw+t3!BK9<{PYP4o`QTb7V%<(xY;&IZ;c$m@YltBj>$b_Vp3Li)@OgdpMl9
zE__oueqAZD=V-*4O{PgsZIcBni<L!PH~+MpR%@zK%Q_)j$H=RDI_IuM`G2+)_)HhL
zHDkKtBLCT2XBcj+)!9060c+#})<zlDC;?WcgcBYA*PQ6AIT3l_L|e>>_MUx>JSRR$
zpLm{qa@v_yQ+-ZMsW~}+&B@Q!r=Dw{oXDWL+(%%!%*hox``1)$4g7t)@bp$~E2}Tx
zEhjk}X<F{kzj?ajHUHm)9S3<1?TuN||9XdN%$ZX;OI)JPOlw*v?!hm2=ir+xVW+G>
zheJ!$m4%v0>>hn?eEfLw2WGW~61zj0_7SU=zWQSq*R@M+S(4@UUF(1Evhff|k2;?_
z>patkWnP-g#TV@ks4?4jTG4Tuj>-jX_P2sE1yjQ<gEW)Z3*6P&&splNR@(c0&-z07
zw+3u&F>6GB?*A@*>G$gWi}x&=IcFd9oP8{B7DN`Dnkc{;c;QmugG>GYbuJtJy=2aN
z*-V#V;)P3*6F7C}ZgJ$jvfSs2Q|uMzxmP^)Ua^0B+2!t)h}Tzw?p|hkaCy}n)==N8
zHoRAze1&e^UODkq$nuk&Yk#i&Yp^a|_KZ}F@aCUuyQ<d}%w1P>cikDE>lb*=D$lvD
z`sa-6s_VtIvln~{U&AP^yCt}_biMmF57WTz*~=6EMVT0HmhGvHTzopq@=nCWly2_b
z-Mptgk4kv$2rJnq80~p8>YRtrIf>p?ce9uLJ+kHMkuBnairnI=hh#LpW$$a`YM1AJ
z%Ji`+60&HV-@BPJXu*lM)+d6x?=-zR*?eYIv&@O}wkMbA+`V%4)Py-FZuFiAmA%Yz
z@NVb-o~sXHuLd-nco=&*wD#V!+<P~B@80gcdjIa-cWdvy`FrpE+<VVs@86lr8o+Sx
z_ucys*4~{rXRGkZ`JPVPy_-*4|2n<j=1hC{A+x~i&QaHUq#sVP-uW!+q3F3YU8^5X
z*u73PZm0Y?mE;Qv=Uxj~e%|@z@MK*l0p0Lj3Qr%IK0K$fEXky4O3a0z<XOv@0-85h
z3uNzl{QJ$aeWl7fPEWBGb3E7`y-$0OtWb*MwhM9-FDM@>$xTeTa(HUhYcboT(shn(
z2D+SUXPsiT+3xywUuxe8y|b6I?p^jd`^@g_GXvYpg?7(N_TB5$xnJt{ylUU`ntRXd
z{yl%pd%1kyiB*6073*-#{~r<lylB=|2maK#2PgEc%fI_jHfNow<Dp+JXYy^&K0Wnv
zhTfrGZvmMYZZXyM8xpzy6z&O?Yt;xewx4s@aZlyCb;hSF=gc^IxW9BmPs(Ap9GSpm
z)0>$`R+r0rzM1NvsZ%`b&A%M8Y~!Q36V1c^%+q0-e^^}kc&X3Rh(6`S`R0zA_KTbr
z2i%!_=k9CY7th{_eX13GS9||P@6}tj_d0u6x^><^_I)4Nz`_*po=M^Tk8|&T*1iAD
z_u<#M_kjWLS?k}k=fD4v_kl<M1JnBZ?2Hel{+W?c*kA8$^hIHZUDS?_(~GxpKCG4f
zbhhT9!u>PSat}51KdI(lKjyPTYyKx?`?G5Q=YM)B^;zVZ#XY|pvyB3eFz%B3@p-`{
zyJtP;R3g5_XxTsV5jtjfZcD5{ywzjvQUy!C-KK22?dP56VtKORclIr_d9Ewo?9MJ>
z&D|JNa6zR&v5u?q<nC_=JzYg~OHZ*(D|qrWXaT3y-%G1>URc<^u-SW}N$z>M-@ChO
z?;6N5mgoIw?Z4Y*f1gQ!F`z)8|Nf63egfS5jNJ3zvj{Ry{r@8%;pdEgmcIO-0T+HQ
z@c;EQ@8?4KpX`m7H&pE{F%n5WyLJ`lx_hgxt>--RuxfVA+iQ7eukSnmspjpOX?%Z5
z-d;Z{fBm@qbrb(jlk6U<-uZo!|5I`Ax;GmRHOH1lM0UB0UEKa((c|dBNByyp>t47&
z$@lP15_lqLy1eZ5w7IVjoY=r-oxNbL>F0beVQ+zr<)UKTYQOyj7!Eo%Fz7L{OUay2
zUFgJ^%4R3?!r+l}yQpSd&WWWxZfydJlN7JEiD(DqxJ?vF5K^5vnJHpX%4dUTr+ibk
zO<1`zb-~$otJq1Jn>`)Q`mT+-X?1Ir*y^YyTU;tvxi*CK2inAmCYXAMF$V3Z+A3wH
zJw+qsq*7;-3q#b_EcR`&(&CGv*MvV*>P#?S@2<hWFr|^pbh3o+Vwbg2N3%qA!?xVG
zy7>C~c`5Z@wc@wUxjDb#Y{YBcDK%T~AHKL?<Fs!bQb{ka%x7JYW*cSp^7{J!vFH0@
zrMJDiy~p?bzqQd7c@GY=Ub%F{>tzw+e9uOePT{2ufy};}R^7Yw#38J7@AbF;WV0(6
zmqhuB&yV~2{qx(i`^)9`hiLnm<!U|I#A@%@6#U`$@}_Q0xkXM5O#CJW2N}6#3Jx@K
z$(&f;cq_w$flGf&Ktq_(k_$cEW;%;|jofq^xrKaOCbBU4sWfv>aL{`1I{n&e?cEDk
z><-*(*tco<w6dt58)d_mbSQ8sA2u>tt=u0Hq|@<Dh~bpF{HfJmS_@ZfTCef(75C|V
zx0)C_4Fm-g&KNwk3O=))t!cqYjZG(9xP709I8QNji%b;_yRoo!Uev3f!A6e16}Mcd
z`(rref55#0={%#`Ud3y7w#U3y-FhobctMa)V{yo(Mx~=(!BW?Qd(A#N*j_chtgz~h
zpM2w&i(=L1stc`*HtZ^0-d?nqYw1K5g-#VSFR7I`5?f8D-AL|K)xVK4SGMeCa<G@#
zt)%AE)Tl`fw$rA*iwe|>$v-F?zH>rUCUfi*3(2i_%XYrtSj?rqs64J(RjN?Ac5k-f
zu?eTVGp9s{a#cK-wxRi@N@MXMP51Vjz8M;`dz`pdMolTw+OWdr)+?>mJES_+thsPX
zt8>E@tL(K~SOjK1o%+72LvPKN7~>~8|1L#LHsrF_*?9EE%A@B`m|T7naJuK+o1ohc
z-CN9;&5w)9ypgw9>VMYNIek{!RxSP(lD%Sg-mO<__nxbIvq699a!U_&(ciBp&+y{X
zj6Y}W|8TmT6K7%q|GK<AyOw2tQ~abL^(X(`9a#?BBkW!LcE_JCK5zGz<CEusmOroG
z7oBA<n&7}FlhWbXuX4rTrBaqD@Nl)M$=)e7ryn$Q3;C;f92LuN{kT)I^66H+d*z}D
zJPX|f>q3mCWF4O--!DArg!1MNm(?#=8GLlqS3b#Fud$%V;q<$|O!8bDyKI7Mwtq6(
zbV5Uk(Z|p7{hs3Uw$1-G|2z5XrD~wd=lk<FU#%(>VQ^PI;P04~%j&XVfxwDwEM>29
zcv=!J^Q|!ZV|k&0yF_7`(g`yjzpVdFl5ZX?w|Jo`l*gMRSL2~8bh$5F^}O7+3O}Cu
z<=Rr`PFU47pJ>|=s`G;3T>G7$iW`KsJEz<Yh&b>mar=^~+Y{ej=)8F;VA~mG*8*Pd
znA`xqU0#YpYmU5Dlx0+Qx|BSj$%aMo!>g0!DoX3CDqZ4b0{hqRGM?PvyIi$=vGT!!
zr1^91bd>N3d)0OoO?i^*p}FRQlh7h2(KRJGTKz2^kt&{>CNDiYDQBn3X$7Vk-yBuB
zTO?WM)jX3p&eCh-a+qgP(9gN&w-mJqI8H6}NYz?-bW+PB&8y`Ro3*?J)wB0yh7{CT
zp1Ehx6?n+xTwX=ztjRHNlC}vn=)Stx<?Etu&GJ9^q{g=7#Zx%^Ivxeyd340`+<wz@
zbNY?>)jt)l{<CFr(f@<~4NO}UPAi_6H$PawPP)av<HZq2F;2l(a!!hk>n1u0Dhf)b
zO;-|m*vGT=lIXi*nTsD<`q*Cj63897uv4jUlc#*_rQ;6MY`;4Aa~EE6nfsu9dFh*#
z{$EzJGkiY2-}!3DRI{m3MzTjWZ@vl1)ZppW7O&8Z7Wz74Zd2+?6Vc0atyUTPJni|k
zK=S~X--Q+Ss_Oct8JdM+rMCM7B)E_6Qha*VYf<=1*$?Mt?Z}+E-1<V5*ybm@TvD!<
zt}9&nI`-|VBNHpMyTZPN?Kt47_gP}8Xlg@5nCR<?e_~R51tOFGr}#ggWd5l=UaU*e
zZ0qKmEeBoHn4)jUwJIqtbm2*=zT>n@YulWqIgZIaU%EX~e+#T~FyemySTae(xqJRh
zBje>EPcP~fYy9uMFqQvERGvkJY3Nkn)e+VUi<f$Di>^Dqo>@@HTE%oG`=nzzxs&rs
zPMY0&|4*n@mF2zOYsYR!tIxp(ES#Edn>zP!_T^Vv&v54|pPE~HdH?UP`VTW$9vp1_
zZ2iJ!xuxdu$>nK&_6wRfH0m6;T;rh8%RDK-{_>7pS9~)TOxAVfY;a<`HB&=zTlI?W
z8qvT9R}0$Awj`Eyh0ndj#1nod_+DGhmW!&6`P%mLPR(hFSsrk#Uv^CdyXN2jEE?~E
zuAX$i9FnOgagbYULIcm9O|0@)4`^mrHqX3dwA@6(;a$+o(*lX^+HXG2Qs-A$JiGE?
zjm2aBc$ZhL<=+(kiF{4{(ski{?Y~}KRgSdB^Tb<Ty<M60!jNs*ACAgPJ$>)8eoPcQ
zyu*6?tNhJRx-##&vUeJO{37UlR!2k3u&q?rbmh-qM`nZ+b~UiB@#wk4cD=H_Ctid@
zO1S8CtoxbkJ))20m0czrPjcNPw(@$1MNrO~J?<*SED>^+(Z};9B_%G|;VB>MbZahy
zs*^?Fxva*mQw&`a)7vh-)o5fhsd8`&Skdx5^_5aa?4_dBJ7aWoGF{#;()xV<|F<iZ
zYE!rTZ(kG1@IRv1+RKsa>ED$`o1;!8$RCzG%^+-Ur+e;b>&%#duO&0Lcy7^gtebT^
zL^#9q;Ov^0&n8Wt!lJ}=`<%nCZ($z=Zp>~!UA|AaC*e4!gT49Boz}O1KBy>4oHu{9
zx}EEG|Bi@?`E#2&F5eAMY%0=l<~tbh%E3kHreTPydeTIm+OLnkED?1#d!*KPB)D6A
zQ-HIMN}Tq8li<IqUS_KV6@u^2&3O0d_nu#QsnhqXecJV8(&I@hMGhX+k^Hr{O>T9B
z`_jH$S68d#W-)(swf0Elclqgbe#Oeovt~c$3YyZ``gKFA{e(qnk(>!<H6H%?%jK%P
zKY8o7t@Wwz?z*nrd+>i&<)%fUJnDD#KDnIUZ^)SBGe!T%)8<{XuURR5sSll(dQdd-
z(*0>7%)he=EpPwctj8)TG+R#ppH$D!Yo5Pq=e;?(NnYu+Uj63V3L7TnTub2GZNJs`
zkkFN`vcp;R9TzW6xVlS1MXEJqmBmF1>1U@j89k&QMyS15r1~_=;q#+e@3c(poq0u*
zSOX1u3_B*@?qUjCz;5WkWau&5=(E<99la+67FHeTi<0ONPug+u0*~M;!M14QY6aVj
zPweNaty_ehcqLieowYuBt^ee7=w;9$wVngZ5AyA(T(<avz^_&N4<FQjH^XutWB=^m
z`uaRFbGQxkefSx?`{z0ze(qt<HhaT=vm-L*Qw*w_tlq!i;GB7Q^$eMI16IDrhns{Z
z3TaNf!MJ;Fz{L3xh7X$!y^1$_ou2wob6(>LL*~m<!yNWLt(<x!V$Uk41$&qmcz$HH
z+RPq*!!dA*O|YYuGxMs=FOLWq85=uqF+RM>Lz30AYO+s*@F4-_vKQ>W4h=plSZ#01
ztNotrsj1o#!Ky33Dp|uD)iLjcvD9Vh4)xnT7qZ!}|86Zau&rlsRhTFmcxIz~<EG!4
zM;)czLrjhu8Xph+X!PXOri%{_WJ;~hNMhaW<>I!*UEz^Q!0RowXWZqS7RkL*)V}4e
z`fE!(qp2p7*b4^_tt5|#>aFjC#W(Nn`fO|;w^`@^#{iDfMJGPpX5zch`J-65`eUE@
ztqu#J&YWF*)1o<dU*wy~Ao1_C*}MzezIM-;Rk320g36K$6P6y}J9A^{#NQ`HR?Ud|
z(0chp-(8Kq>e&t)HmBw=i*32U5|z=n&~oY41zl$rus-5EXkFPGb)rQ>p!MPk9$}W#
z&pxz=YH&ww;5ld9@4vF=ZINxf0e|(6_Kc(*U6Z}LRvug2v0XP<p_hBvfjxVoe(g{f
zv8=aVwz}b<Qp#HYh0A_8akgHZt#L;>+uQTh9f4WD#T1QHcr#Cb{xR+RpQ-0^_^-b7
zSzgm&a%bmnhFuE}_b*_Utz4yk`ldRI<^KLJ9LqlUu?2S69@)h+v*Z7&%^g<?=Nd(<
z5EgV1-YqNIEI56F$<d@vX~*4P7aZvkTK!3)x!}(3sujjZ9-j+f@weW<YF)tk$>hA`
zoGE*P^^YVTx0f~vOkA*l#cPq2TbW1qs~sz6ziJO`Xb@boBeY}Q!5PBJou}#|TK8W#
zTDPLrLt?&80=tW4fbNeT=b2N38q}Ol&bR&TxLCvMx^P1l%lS_YQv+CzCd{$D6RdMs
zS$)0Hshbx>!X%naxF@w|uvf*bzCUq=%w5?mNr}|Lolj>g%gc&Q*f34$>T2b)+Pik?
zWqq8Mr#Zclv-6NpXPcz@gqrE^6F9yG@4s4TKXD1iq#p53o&KM%*JNGZ|9g>H3#ab?
zMGQ+ALXON%U}XyFQkHJDI?VI2iDS{4)~iPT;Q@2oHw6E84D^(o^<|a=XU;mISM5%!
zytbJ<fdb9`Oj1m>y{9xp!yLLpGx)+30wmY;yz6N4S<zl4!K@;{dhS9<aWhlk1|HuA
zCSL>ZbJ?vy7CWSQdQFYHnm@~X$#B#cv`6k`-?hhabIKtVlWR$?*HmK;t-HAWamM-#
zt1jE#jZ=@T&suwCV#ZF58vaQPXLG+^%VWKs#CiS4hcjodC>{;+o%fh?!EKq-l^e=>
z4Y)SzbnQ9O_hJsuY2PKU51(`rzal8Oj{C?u;SMp!jnD3IMz1t+ZP_R}X=B%_ji(nK
zkx8DnOrrU}PmHVo%FX^;T6<;v!*?);7c@^WZJtou9A3f9sBk`GlIW^M@>LE-D$>l>
zlAG8=rj+W4&if_ud*`e*Dm<2wtZ%JEs!lL%*>KcSkag=6?&A(TE3b%DaR~c<^<w<s
zdQL$^BIN?}7JrR|*1t>msxC0CUCXZP&=knvR<B?yImO0T<mj@5-SRs88A3<TJa%Ig
zIC_d>%Bn;1i+=6adcC9mvqkWe+aV!pp-)y_U$Nkp!qt}<ijgz-E0+4*xhN}XxcY?c
z)-$VO8lEU7DF&V9^*DR>j%IJr1cumG+?|zA`qZL4BC}6u2E}T3o%p`^q<)lrmG0uI
zqg=gHV()BS!;`(H?iRy;p2=(Uc<)uN)X~1nIWfW_^Rv$D8*8MEl&1E0&J}R*cw@$x
zFnz`u&!rZ9dnK=0f8~q6ad~k;vo*&}qk?9^mrTY59Us454%^WD?m|ys1(%-_lZHc+
z-z4F!U+&#n&1!eiRj}yhlr9$CA4`JQ?ucv=4xBN0|AiL&mF=@7Z?9eIHErj8^Nn2_
z&h%7uFmW`5zG_^0TEkK00#nro_P`BYRTo+`EF8A<u=w>hmEPz%UC<tMVO`?fgY^Yl
z;!L};V!Z1!Si?`8G)Zo?dL6oElhpg515Y3AIHc^eUv;^}Gru_{%ig4RdN0<`_3F`?
zwthN;?rEQ%Y{umDmtls_c9y7~J^$zO|1!hmlBoR)s>998t{=M<RC`FhK1pYdfZ5jv
zzH2P5ul&5BlVM_gsK)wY!KWn?kGn>+#?E=MWJSBj?$sJyUO9?I*0UTvZFcdV?<-}V
zAR1|I!2I>>bUg_U<Ba=e9V?w)Iw-B?+al3AJ<8~q_v~;s<0B`84L7(lrM8|CVBT~=
z@{#Atg_6vhG@NcT1g!ocys74bljD@2rw?8^82vSs+Bh{}y^8SRwJYNu99{X<YXeKq
z+PAI2rjtAtv^rh9u~Ww@u*5dANljwX!9`nIm4#PF-QaQPeC}l?a_DnM(3Mq7MR*r~
z;kY;3ZI^D6Rd?dKA4hhW?r3u6GIy9BcB4s`LFn)=!T(ZQ0yl0BbeR{Ey>hCesCC~u
zQN6C921}QTVlVF~zH}DpoT4VSy-O)ldD`SJ`$`@v)f^3yGQ1bR`L5NJWXpBQbw(?=
z7&Ywq?$tzY`_ggg=jr|4)|ZX0O4}-N{a<`<m&Cm2iI+RHofh3(8er4)N$32huTR#O
zwMKuqY-G@^r_roe&>U3IY;p8e`;DHdj<Zfmuo`mk99Y5Us>!V}L0D$ul9*+zar1b*
zRUdvenzW(kb!fs<p=C2y&2d#Zk;)RH8s5uvrQm|sR-W~?2g8zDH%mM`wV-vAz{C4d
zr$0|vmBA#n?83Uo9&hE>@MIdDmYa8>@!A2^eGVVG^lVtyZ!huZTV?#e+j7aJEqn_L
z1Q*<93Yzh#Yv!`R2+NlT!m@*dr)K0C$X(ADy>8^Tyj7U<r_-8GH}axCibly!H7)Bm
z>r3X*&ab#CV|_09mg}75!5z&^=YF*1pR`(2!+4}gBgIH6LUPyf*0=d>M+M#2tT>(3
z{G@P8PgCID;M-LNtP{RQ_<c(emWbrnY|g7$n5iTd5%?i!&6Klk44bZQjxu=YA<<kK
z!5SzL#aMBHJ?&|{%arra9xRZ&rL%(1!BV9D1?!d`CSP8+`UUO3*RUT6EA(a)`^Z?x
zaH}U&=c<l}blDC`U5^WK3)%w@2nkH*5q4ql)tI7iU+%Emr;7{PJM3(K8#>K8CI5e=
z=WPw|;={UUlv(cF{bsBrrfBi{Lckoy{5{5fTUp!EGNKl5HImpU#Q5d+zW01Db_okk
zcw8yE)M0PfgqZHOFDLK5kYKudQFPzLV_znvO@Dr)<YV_lKlZQ9y&V$@=8FDa-CnXa
zq~C)vq@(|f$G-!<I#=`O)Y-RPFg0nK9(JFBSGeK+SIhYOObtx062-qAXS&?|XLfq=
zPNSpCGO|6Nr&?Z|bn?d3domA|SM|+)9KaVLn%k;(?lWuoXVLF%2`}Oq{I|Q>E^=AA
zdc*mRe|1$SdU?mTFLXUvccSUvAD%N3tb%nHN^hhpPn>Ksk;f<W;q`gXH(e;&!XmaP
z@Z-Xx=g%lC{2$43`|qDy4fa>}d$^tbQ2w9gp~R9M)~uq}*6monYF%2ZH_sM_pE3Rx
z>?ZG)+_p_!c)yO{tYXD4jVEG^!IQph6_Vtb=ikBG@Z#wT38@gvX<N+n+^QD$2Tdw?
zxlEn)O<n(|2#eQyWoK`X`<F1iM`hcZ{Z(tfSG{&M>lb+c{e<YE&-|aC`F_53sl8&?
zymj2O<YaBHskd7AGJbgcW1iL8?2a1AC%^R!#R3f{Y}>fz)!dhBR_y!O=d1CdjeF9+
z-ar@2z;|odCu%TX4}87RtWj@5!R~zKXorlyAyahuS2Z>i2A;SeU;h2s16{+8c`Mei
z8!ni)B{1#Zm-CJDvwe3+GIpH$Uv}Q_?~~d?%lcd_`#es3iWjMSXD}&n#!Aiqr=mof
zOAY=xP5iW5b;{Wt)))Pk?Ma;yWU@@xn3wPPswF2Q8tpQy-81;leikocjb^wR_Tsay
zmY>6Y4yX54zu1rHX)yjdUVroE|3B;<La{8<9t3?*kTeS3HYMxoxrNTrUvErM3JzqD
z?2$4qx)bqnv1>PzT_2C<q$?*T%{j3q=j10<bw=IPOFfyJ)qEM5td{n4P70A^bi6$!
z%ehPGXxN%bda9{cG-moTZSz^{HFedwHBsA(-o6TDn7$$5XxmdGGwo2l=rde1HYF~+
zt?AhODo6Rtx^p`#K2B2%ZaaN{P0mic(pi(vYW$zlA*wF6;YMMynP%yto~%f%Fvg%#
zue&m#>2A?$U9JlSbjPj^-1_Qga;*P>?78b=rfLNr>s=ka?o5fH@Ri=V7Z<<&S@hOF
z{Y_m|GWUYzteq>af6Z`gT+}&LgE4SRM%31Ia@oECi>Bn=optp`Ak(j{VfXs)uF{BH
zlr6;Iy1cS1c|uC$QsZuhS2Lf_lJLlI<j;AgsLo)$xFcv@_SX&0+}00xS19C2ZrT<T
zcGB>BV)&WD9S^D$Cas8R*PO&LTc2sdH80&YSDd`G{X(CJi>Ex<@i^X!?U-0><WvQf
ziB`OoOD3>!7EEyT5?dlH9yCEQMtAYnWNz)0o<m-)vs<2|PX8a@6?xIiZ>pxScSh0{
zp(zPFGsWiQUt*D%WB<=$c|v4i(6M>7mt2(>m~u&JEG+XqdQ>D=u_|Lxw&^a7h)y@I
zrAsm*Z>cSd&s-I}Xzr}k6$}fSq@Ju`x{@X>sdC`t!Bv)OQ<61RbEky5Ex1^<dhO0E
zkp<I|bC%s)CbhNjsq@<_pC;73`4Z%B|M$zLd13!Nw#?H1b|=V-J$6O{TVDANr3US_
zH-cj}pEkSAznN(UcdqU%g$bUS?_C>RL}q3vp3-_V<#e3=Z`XzT*=IKivx+1eM4!((
z{`K*k#$}ugj3P-rn@(v&IZQmNadw+_gW5+c?GF;GR!V=^WVQA-_kN94+70&q&u;3G
z>3V4wz3$A*Lt>k&&FWX&*{n8Oqro<^clG5R-);nL(bbc>>$Ss0Fw`w~>rYoUC!bAG
zPLk}UiBGP1=lv~ZXDhwfcs)@#CE!|;yk*(ssM%-B9M?3N9uyNeSh3XBbk-Jg5AjB~
zG!Kc`Ba<Ta*G5{X$J|)5!>30*FIe30qsU~xDbKeAhuOSbV(F`KG(<XJ!S;(>!Jb=_
zMCNDt9(9ab)BMy*d;`1mnuyNLORFL>4|ZM^i_V#FH9D77|9WiI(@>jFj$E$K=f;a3
z>q=&F(3eb}Dm!gzij}8XZPMZhx2e--CQc2I)F>6ZHFfb!wa3$rb!wDsSvTdP^k1i)
zh0>b;g|=>A==ZMBwO@nh2;Yq>N<xnlS#8f8mV4aOl(43NcUM-RtnZg(*`ycernxD$
zb*J*!NKH9^RB5AX$<{V8rUk7f1}bw|cB;8(ox0Ezwqwz{1>M?Ov+w@i@kudSg4=D)
ziiv?=woQ_KxuT=wli-}TBciH#t({&WlO}d;QPH$A^mI!)J$YG$E2Bpu_mUMt60Oc%
zz7nk>4bh#`uN@IIi|y-;Ul}_4k&3(d(Tl!@jA!P3keFfpsI$Ljjm&(n#Kn$+liUmo
zMHUHJvbySJP3#wn50kn&^-M<Vq|Kkbmfu<=Y~JF<u=Ybrn8nMf{`0y#eS)S6`Wnsb
zi<zWdP_wiuiBso)<f`M^L5kNl$`q|J+>jLiyeW)#^CRy1nA6O0294qzL2Qe*JW6;`
zwd~xEOWc7D3PNfw*ENo9?z`$TL6$#+uS#LU#(v#1Nl6RXzc5T`^;r|lbK6FImj=t;
zmWh{B-!-O({kd+R@ab~>zm{uf8E*MEZd>7MrMZHkE%L%b#<fun%(pHoz2Y+8(zM85
zTXn9_%_ZA2eHr3EzX;owCN{C@%VIU%Oi$KHx=lPwmZn#~bhxD!+`DR(!YZyf?KjW3
z7Ju!{myx`)#!BhR;jJ!(E!tY@O3N*lI=xf#d&~9QLVV4Nsr*?_wR<jiHBDL_v2=CJ
z))ejiO75Fh<msmM2G6cjx7oq+KVRSR(&^}l?Th!bs0BxDoG59(a>kSmN=Cgev%fBR
zZhK<Fq3Vab&n(^bNK!m7vvzC7{lah8_dm&aDwelEKEL8|Tad>R3uZ30jOm-Xs(!Yc
zeu+4G)a2oUz#FTRb)wtF7BTE)<V+CVR?2VdQz6AI+q3P&&I_G3Uk=UZG~c1crN%S)
zwdkXL8g5>aA|VA+PAOmN>^mjRBQkf&UFBSHFMk)Iisce<Dz1fl9@(z1Jh^(Zu1|yT
zqLV8#?paUn-67Z)wNd}=Cl*!fvm196ZMiHvoBh&^l1BfcRWZKnyjD0)JQ&#ZNu)*o
z%nHuptg@MsJfbG1GuEdRP0TyXwbDW7(U!liq5l&M4aEayP6+sWBy9bWtLu{fJc_CG
zx7|O<OV`|V&K#}>ii}dMr%tg3%w4%=#w{b}Nw*(#E&h2}{@Et=GmGx*FLmIlI&0Oh
zvE{%y9}k{ZU29&h-|puozV%N06BX`|e{Ic;IXx$|jRH@)O;f5Dd7gTPQ|cm5R)0K8
z?s>Q0n`P62FLup4wW^_WrPtAK+qPGlT+~RN?S5G7<wUQl9IK5bOY~!3dZn{oPR`c6
z84|T^Q)AS{Wqu~Bb_!(eII4HlD(cD0qAIJk|F(qHG2fZ>WsBRP&~;_08|Mb?6nJw?
z*RN=Lch<8>do5G*j}}=m#)dkxZ@p-G!#BL}u9(FIjT<NRf~Bwh7cOIER^L=0t#i}t
zdsXf6m7E7!??x-g#n|WA?Ag*-dH);Nf^Fh^UwZa$xOjW@w_j`XkHjoHIKNRP&1(~@
zZf0lu$5Ne?yXDU7p73r?t8muL+nC;Y*<gO;O4faB8FRi~aCW*=a=*~zij?@V0O!Q7
zTt%JMG2&j!ExxUAEw0Z{z7pEfVY6v_-E<Gv_usyI7OzRZI9YCu)++P8FKVkUpO<!>
zTj0R8_CnU0h@YRr+IH_R`m`(ZPlo#j35z$$>}QthFBd)i_N0D8m6VdKX5!Avv6JJs
zPEonOyL4^I+`g#Bg0!|vCwFz+&@0|}Ha=ph7N1T(i-clQcc4<TM0M&h!*g9a{|oPU
z#cW%A{J0a(sd;~VjH>^Lt(|dyTY>SeM=X_REhcX|^JLGw_^LlM^=?16|1No<zOQc6
zuRC6Ig}r2~F7M2!DqAtX@!0bYsoayovR_;U`;~jeRH9}6X58{$_1C@l|78W)m9y=n
z)F-E{{{2k-zei+M?bKacuIPt;^?A7d;_9Dsba&{*1mv5B9NrnB?6!68qd&74)I=E{
zwQT&zvPn_lnZ|;UTrXxP=>U(}Pw&XGmR-G9=o&fYXhh0`$ekzO2ONFBVk*NP*4<MV
zEID%Z<B|vMjIkSTa7=gL=lK<yV9S=kInD1%!a-G;^qiwfjmOlkFr7QdaYXS#x~O>3
z&1UTcv41cA^RqmiELe4iyVryFq(FvbBHz(FXZA*l9hzW!rTH*t`<lIXdws-(6a{ZC
zyc(8py}VLfx{0-X??azVL9w$B#dJM|E-sWc@|2Zpk)0-f<?q7FtdA~-%JoFvV|_Gz
z-2_+hhP&5)&JCL|rJmKT@haz!g^AaDWED~uIh~1BE?uNjyNH!dR^{d0H!aaJx-3x^
zmv#rb#?($)k<uDFhwoA3*;Njlk7v64T+;F+Qchn?Y2B3@I*i@RBF^>QJ*)op>KjY$
z%^L1!kMQIy@C!WPVXNbPdcn-uZ|BYa8n^3+%=sNRPk!<LpejDkG-RIO*-x6jmu@s!
zaY?!6P7C$y%@sKkKcn~0$^Y_qCWKt)a9KX_?9YWE-%_KNH1aOZlv*nKWC`Pv?~b#U
ze|+qeHOo0|){2ett0Hf?th=>n?yOZe7v4E??s2QDn?j$vTHpWJc^ykVpMFr<z_`GR
zE8&JEpKpoN6NA}L6>e|3s`BiHitjy@XIGBR`!IJ~VC<qJd@6-cmgYow-#eG^qEFZ9
z<PO7mQGfh`{>kp1_&{Vs^q$z%Pj`;Sgobl7O+3IElbq7>^^NHM%=6O(rX@<o{77j{
zpK&xS)2&(7yXNPG<X$JH-Z@33M=y08(_bJX68Av$ug``j0^K`0#J^1_W<7Of%EG-D
zMK3H&6c*(aIl0iScJ4(}CFP_*nb>Et&panb{8y8$TNe9LT==Kr<;;r;OpC58e5NX?
zlrT*xGGSRx+w2=>1?7)=He5}Uw_J3q(d|xVP<tq=M$y!JjaOOI(v}oljb5`zM1@P8
z^{6_h+=`l@l5Gcdf_x{h=qmd)$HOD;|ITIW?o8Kb<EZ@S)wg%z)F+cRZecdsB{wDU
z@sXu-Y<JEp^gRCh?PN#J`;NTvj)L=@dQTUM^nUsq>9RK2`rNG0g{R^}?z<h0ci$@F
z)#+=WC*SI~qBU&A@}G|vPnG|btMNln#Uqu~`Q75{E8>59ru^2Gx_v>p`;h$Vlm&em
z4|bf)$l*KI%{f*2?2Fkc3jKTv!IBTWUp?7msPIhY0n@%`OrQRDe&S$w?0f0n%N3Dp
zT;HuY+Ecb5(jYFa<pD=pE9a~QT#FvGw|3<koX<EGk+{Hx_tJyJ6A54LJeU>MHvP!U
z^a)DQzZUQx67P{o?&)E)u9+rc^)RKhBV}sGJHb^8)ux}xR4n@8^ee$rf=MvIWBU28
z9?3<jGnOS@NWCsvJKbkzqTD7$d9z!~tFpv6+b(`g6l={|p5(Rik)rI@EZKcpa$HJl
zS1qf`?0S0hMa85g+HbXjE_wD=ExUhr_6DW9a|0H+PnB&@)t0(<dy?L%s0%OG^(2LQ
zyjCrptE?JSwCYia$#bU)ZS7fJGwNpO98;cq;l6Is5-m?T*B4&;yIxoDO*1(6KYN47
z;;L_=x^r%tnkm2ed8Y2&a|6?m2Cm+Dy7T87&euJ2?DJ0LP+wkMBaI_nQ&zA(`JSn9
zKUC+N>7wn8tjjmDE^eH4-Q<>P-<xH&Dko><oK#YAUbe<5Z<gaWosOU-3=t|P7OimY
zi*gA{S-tmMB+u%)qtABx#VK(e^J;Uw$hKg^O+G&b3BOmK6YX@HeEBx7jSWAu<npy+
zaXGRZyITFVRL`$!4W4&9xatObQ5VNH*Xd!ZA6G00J0co;Wp;Aas)~QU0<uTLKS_Vd
zI2p%yXWv)p?;_W}*Yq!4lj_>w-~3QB^MptIMy7(+Fm1tSMOP#X8F<erE<5Ka9&lr}
zpk>;fvj47`e0Q&$eAsCDNcg1I%c|SY{qClO2`*Dwe%&-My-qD^Zq}Pi@8sfiR^M9n
z&MNlBH?2Lt^b)=$xqr!s(0QcvOJ}3e+njUDCOs0o`Bcz1r#J84#BdhZwIK>yrJkQY
z8Mv)Xf6*Ext)_%xuY)Wm%L9J6sh@2vZ3+qyIa)f6p)Butr{Kf~FOnX-xxb9#{kjvA
z!alqYU$VU4l0`emNq-^Fr@!e3jHWc`Z8*3p`A}Q8YhdWYUd^v(r+6QX?`XTynz!Lc
zV$P4+4IZAi+~sb>TxmNpPlIc-2z$E0&%_MR!WZ23PtMe>_`3A&88a1^Wg08q8hD4M
z+&mC}W5cO#A^m=%RJOeT2{}r0KW)<!G&k(mdh5%h>?g2tqSgXmrI%qxx&pp<YL*?{
z;i);*FLO)NT6QDN4>>QRw@g(lTzRW1^IDfg_yyMIYWeHl$Y?%TyFai$fi;X*DS3Zp
z|MVqKdp{<`Rvc4%pedpGPGSvH!A6<xh<C-Eia&K^N&{cXd|uYJF|YjO+cMQa=```!
zgm4Y^*xUEc|IIm?>vP3r=fPsLlYb5<e(jvz{A}TyS$eD6(q78VJ%7(gbMmuy|B5PX
zla}>8IVQIG!l$gBnTuxEr1W`c8uJ$~W9c;dFRprSql(lG6{FXZS;n3ZJ-+Jk-(2dN
z6sUc@a#_2fP>jn-5uHgtr@gnzZxr(R&-k-1Tl=2t!jo67E;$wS>Fj&C;tS^@dVg|N
zZo4!^rgx34i(e&wqt$%Zvr%8BHrXumn!Mn(i+0orz9&c0yx1pSIi~f5J^Yd5HXV=m
z;*uUx*MeeIzqPJ>z?S!fHDSf3(3P96t&m?Cy2K$!=S<s{)2rS%y!vwb-4>7EC486H
zoPDI@Dp$IqFeSp`=?+KZPw$LEl$B(!zPK1x`cZ4`6}#D&Kea@9=!U;{@k!h27o{r}
z+_q9<rH9$MK%=Ew9X(&`JPn#H9n$q^=>nzQjLZE>G=HvpD_x&?dy|4<UrXSFj|b*H
zJ#aU7_P4i5OP(HDo0@)LY4kGB9}0Q%rm;zSIR2c#U+(w+RdN>V>rHAezAap)^x7o-
zl~kR<yG^+=yBjsPd~s>q7EsYG{loQGYggoyCJjyo@8!MHX;bozJtq{T<u^ZT<Sf@c
z{%-c1Rl#cJDRtY@nLmq6Wn^y&52(>o?lUMm-?gsqL;17!9!chrFEy44EqeMT*3@0;
zGfPBi=iD#L7UVurQ`>eiCO}+GUOjEqqVJoo{wU)5X|`O*$HAt1hM2PdnicPLPEGl<
zOyWuD^S|#_RXn=BJ5BBxtGAnsw_fF?mpO-6wn#N!db~1i#j;#^kN$Y&Fb(BLkKNrh
zWH*{jzV|TID`(Y`PsPVhxi~6CY$)lOcF@J~%g%jY(iX1b`(EI*@58s7|2LePrPG<G
zaiiwese1*JC-d5$ThlRV*4K|8mT!!o+8uR+#W3*ICc*D-1kb+^RQ?!J{I;Tdwy?DB
zN4s;Ey0n`_AB5j$5tUyl%AFaR-O94Kv_iS?olMtGUGvARRg&{{!&#Q!TDLK7`umS(
zqWA1HPbfZmXyXjm7tsx85>h@~U79?vT&r!)vCO&e9;9t)R8-z15VdW}joZ)N?q(h<
zT9q|@<+p{~bv19ky{?|F6}z)>J5P3k#(81Qo4PEl2~!U0&N#GYcjt!F=lfLFNEa+!
zd8er~O+@!_kxbsiH;1&=+&0=){w!_UHpA?+Dof6idc89D>}FBEoxgR;olUo^Zw_vk
zc2EAFu$$jF^YeAH^y0>&N2=$qe>R2D|IY4!H@g~pGvm*f-*HP7*WS3TPQPydZR_{t
z*8k0a{`+2k*;wYb@wG3mzM2+p7W4WxbZMRSvj1PaC(F}d*7Bsxqgyx<CGslbORk)5
zi$7jv;NkqpZJ+0v^9ENod~xBdQ8mg4=vkp#bTDuCPn%^$`SL9nx8KTR-X8yE#~qva
zM?ZgTzwy{B_NQ%F?c~=bXP8W{=i8pD`s?Ur>2UAEggZIslJ*9wmrgF**><#g`|iEF
z@)v}zw!AAm`|`e>vk#<%%dfxJ{PmvnOHu0w*`Iz}JDr{P-7>K2?X+1(W74@ILcNyX
zDGl{<40}`hbDDmb-v6qj`A;><TU~AXlakMev&BTV{kw0p?SWO<tO-un)-EmB)1f}G
zkwxL+%1z7CCM{ddv`l#FLG2puyxVMh!q;ma=SoPq(^uZD`R3Pt^JkkcUfp>3ZNiks
z?H12+bgC1k9Mb05k#TBq*~NEFrzXA)tA5+}zo5JRXZwEL`m(jxvP$=#l$b3&HR`3&
z`Tu49al5=tWm<1@-2U<Qi;)9|z=6~R$3_-GF`pd)4-Xh|oG#G1ywpd%QIs{{$rQs!
zhdcPph3;q=ENPgi#=1!4!o<_gMl8X{rf3R3oH8pW$M5E(;^hsRcIG}am0TO9EOhPm
z3ovr+yu_)!@SKkF0^Tb@+TIgRMm7EC=G2SXyYI<?R-Tocm^?Ujyxkb0x8^?Fm-<@l
z-JL0Acl(4EOn-lGrt!^ftI}7;u0GW2>8tR2s^RrE9!2|UJU3t8I4R4rqfm9hl;h`8
zym#)Dx-~VUc^2!Gb3R$GTrbVBuF*X__1hc0S$7LMW0#ezyup0@Q`zQ837O$?z1?QT
zSIau}&dp74`8TKPi=Jd~*}<N_yS}}<!J4yQFTncen@5Jr)z{ZmSA2hav;6&o+I^pY
zeqVY0J725=C!@?iFHW^)4jBWrmK_cs7O=1<892_6u<<y+CcoywA#UL(EslP>Zk%wQ
zV#jlF37@0LfknN3CV@v8B6BVtWw7qv*mKTgPXV{yw*L$Q2YCMpH@M5OG8rA1-|^+-
za?WKAL7tjw370&V>PhLX(u@|qu(%~nDT8~<ErnK5eRt39seuuK4ZQ|ueKOB%x#8g5
zV06=^*w@%xaq&6p;9D+gwkt197PdZ_aa7Eab8V}Zrch1Cw&p*=C-%Fk_LL}iBxilm
z_DpTMVt+O-^XSxzF@d^UqpL2z*<n<1dd<?6R@#TBU5a|dw0+vI?yHw1Jxf>gSexx$
zq19-zTX{=ttYPxi1oPs<y({NszX`fEr`9Cum`v+QL(Y|{BHkwr<`&#$JXpGAf1&)^
z&|Af4RkvTeRwuP0b`IaIOLr?cl^(n8I5gEHp6TVS>-Ra8|B5|eTKm66!zf2qsY9t%
z-fM<(oB5>|52s4GO;zaD?6Xjtw(d;PgV}eEaxR}+Ae6<uxWuSSV}6+3s+H5{&bq|Y
zcxy$`s<5nMt5l8lNOiO{Gt{hHqj%m&d)?xfUcc5Yy`NPjvi*mM!Ygss6-VbqMHy}p
z_4k~Z&}|kK7_xf(vC|v}8n-=u^k!YwU&$R)P9=Ef&N!99x_$MtWxZP-p7QW!i1t!k
zJmu^y1&u9NgC3tTx)v4W8yNo1pxkVe;p49D=4RT-J5s$b#F+Bsy)LzQkv4nPmusI+
z`zL-0o1XvmYr^>#+drk$?=8ETygb(Wc4>O-f}bh=clZ7KroHder#aEqYEM`4eTc8M
z|KIm-ft2B%4afM*Sqh!mtQ99ZH|=uYxZp$R77vBb;Xd-UUuNguQ<BW=S>eR+ed_h&
zQ*Py6KcO^PWz$L3`AQATJRF&rdnTP-RwCGSbX6i-S7)@xMXsQbr9QbymuFupImnf?
z!B_f90^9ts<+==~>Z(><mbh?<tFgl%pjm3l!VTBBG&%aT!n1Ddn6+JVcIPqPGVP=5
zmt9&trRG@MQiafmH<GwRw;mMTeqe^fjAEClqi+=#o;vGn@|mkEV;#@2RbhvH8h4s&
z6odvy3ak=Kvn;TD7Bo3%y7a=O0%oTKjf!15WYt5DpSUxz?z+~&n>>p&iwYOSO!F4c
z<yfhAnRAIpbkYAM3^JX{9$rO8Q>R2btCXB%{>UAF!*RkPo<|4r9P|69`gk0%6Vjeg
z!7_Di%2OAsMS>xPK1!QR1a#Mjcun+eWVUP+*Y8vjQRbbT+EXJLJcnuX869J_AfX1X
znZ~C#Hl`ZJt>lTklq8n>s5P_HG0Obqrvux*sAg3a1}5BclsKH@+Z30)NczMFAuX;2
z>1(o_)Bgp_T#RrtH7U81v24>$3C8TA!aF{iI+JH!Jk!PO*wX#DNki(P%yaE!4{n!w
z1-IR}vD_+lj_Q_0%wBCV?1~3|``z@@l6$x2i}kZi&5DBKK~tagSsV9izP<B#_xB|(
z_O_yOwGB}_<+iS`{9kpU>A!}HaFgs*r>n(U;#nb!R&<?kzBhSQmy3yTch1xmv9*GK
z(nPMYJYC^#pEdE{kL~-FZC`paB~B9Jif~{(C#tn9I_zxD4&9kIy1kctkY-~Iax|H%
z>6bKVvX9=0Wmc^M0optP(mz-<oOW)=IzQ!**t#HRxiC%Ry_IRPlT8JTZX|ZjyCfsI
z;>HfYj|Q7}?mV%oqV(vIo!dI+-;79qpAdM$wlHZ{^bE;MGlWbxZ8YWXd@Osm^$wG#
zcK?ecWslnx#|tdGqpzB-EBTxe+aUUB!ue0@iuJF~lHU7T{jse_<-^o_m21V_Id)E0
z<7Bh_Db}*&Quf5a>9gWP^>UI{-Fff+L;JsB&%!KamUA~6d3!d#_GoF`n6+MCQD+M8
zf;TB@&XHbWVv}b$dNQ2skz{&VwK>vHFVFV1*WNlclb{WXv(s;`HlO|Ntb}&1e!%Hq
zuQNB54koM>YfXI4vM)?(_sW2UG8Qs1QLoLd6{dt~CidR#IXrU*XUHje-*p8#VNG$C
z?5100-Q3g^%w=4#(QIz?lv)k<op;Xmon7$f+^eYa;K_^Hmt}AJ{I=@2wb^?2^X2bk
z{VRW6e*AOF;=|JOf}V5hJv&-H`E#zx;?JTBmpOfP3~pWJxlMhNQ0I|V>Xs*e1u?z)
z+&b$@oaxpn)|Wds{Oc6{lC1JC;qR+|YrlHPsx4Tnnvl)-Uo3#JAoNhKCAayC1=|kl
zCV5X+@o-#KFyUs?-3Oa5v*dCcm9;2^oSybU<lweDD_dJv-MXeX`{p6n)H^FDO0QzO
zDt77FGu4H;-R?H!tAaENBTGX?3Ij}6Uu$`mbb41}oM`HXH)&elO(NG1mR9Np&l2jt
zvMaAl_Fv2!x0**n?K_TcRz2b5$Gxk*<z0x(Wsj}X#3!k$|2+BFb^Y->+D6mY=pTBu
zwLr(#|Cy<Ms-stuLBn;UX$<dnx`Z8i(7&K7X_>NtcHzHM(+n!6EB`*(8X7ib@sB6h
zL&Q(CC4PIoOv6EQXIzS8(J`5GYgpDNZ&tg#=h392YmVmMDfw;wR!Mxv^{4+cYhRwW
zbWfBh%{zPCu`*p#sPw|Fy=6L~h0ak`j|CZTJPhFy;VW!Vo$%z0noY0Zff;{qta)}Q
zWpQuiIW^&+`;#wCUusla{2@$A?&qG8FKf^Hy^Nh7H2vNen`bLG&k60H_wfMxidM6@
z(uVi*zs=cI`(x?O!1-MZUuSkWUrGKKy0R*C?XoAAm#Y2yn$8p^d3D8fW%-}~CtbPr
zzchkjr&;~asTZ>2!ex(y6xlFGewnndY^9Hu&g4z3O3S81T<~fBy8qwf7M>Mn{g$vD
z=bE5(Y0<4+y_>eUsjq8V6n3a<#iIooY_pP@?y<DaoROKotRw7|@6>Cpd9!?`WH|MT
zoDb4@n*6`3d-spz$5wfdUc7jq^z_WS)sH7}?=pM8V~^|pqzR86c|YFQ{n+OH<3&#O
zZ;o>WAF4Z(?R_G9(kz3N1snF>dbC%;S7C)gOkAHs(xRLkzR4CVRrnS)9+{9>;y0CT
zO;X!BWuY~u*ESqr;K{RM+S;Ob=!?2qm|DY$_pi3-&E#6~I?j!0$GN{DFVfZ=DmkHG
zaltRQVVUk0O|=N0cNsp%R3;vKmB;ht$-6&EYJN$=EX@X0J#(WTru_5!Se5Z3q&feJ
zTd|5igHG3rXU#7{*w3a+T;k+)?!lSKS?OY-tCol<nOQOP-a7N0&9^jXqr`un?-K-<
zzTi1uqkeu;p=7_IRH)a5{~GQWSaek_61=at#ckTCFy%w;Bmo)zl#47bQ~t0zotiks
zXvw7;g|c@GU2IaSH+iY1Fx9dhxxC{1uN&;Y_jvtId*Ya;`)A6M!;_9&o43fbh?l?E
zS3ykZZ`IoCOe!}XEW7bY|No2pqqB5xnQUsl<f33PX@iwei$_Nb*M*ixnyq0T43kgX
zPRiik#_(~Shv-t5u!z*gPg;p#6T$)lc2oqh@vd9V^DZL9P;K8Ck+h^8I{NJLd79N5
zG!qN@OEx@O^6AmCOYEDM2Aybq%;CwgQ1w9y)7qro8JDtjdRsVaol=gyK3KAnTlw)*
zj|)GvUT*VJdM>7?lGE}0L(t1B8<f&IP5<}hz4UQc`d5_P;`c%*g{!Gav&WO4f6^bm
zqSw3p7$12tN(31_%$k{1q_rx^m`yNrL8{IgPQ5xc$F^lhd!D_WqV{f&rcm(ZcYl<Q
zYi;7abn>{vMD8UCZU@%bYv_x6o)VWlY@Yu7t5?=Hr~U+&{x324s!q;lmNhAUJGR+{
z(|(z=MDuy;1pj4e`rp<WTPNmAW%hqvVEA=|@Uo&Vn-#%UtBb7)z0}^QtKBk|@qeqk
zb;3n47AHH+pEJ%o&B=GR^8FR_L?vm_uOEdD{}0J$syI$rqHz4ZrqHH8GE)@O7rkP6
zp?K5(hSQt78b%I%x=mZ&{xjh4b29q>>wG@jY2Rlm8~)!**~q}^zw8;qYA4m#Px94X
z&kV`n7GZvHi=(|H<gr|CZ<edW3~#NpBK7p7%@KuOuUw7mSc}3~cW9|{#2a`mUHvd<
zLg0p{F*@hjG6TFma<qT(>Z|*rwN^wsc7pcFXRP@eItM>py)k7*-v*tBB6~Yn;uc*{
zyLHI#?b5i2cU*;mL8sbS1bugVtmHZz@ZeuJPwuA|I%(5bgn8YSc%@52FHLAvQL5t;
zp1JAVVJn^241w!Zj2R`27)2b6)*kwsvFwywlFkmLj07hAEh~MmF&$GAoRe`i^V1=Q
z+3mvVn~!H~kv);n#y4?SS-Yg`sT_erM$gim+Csn79gY&&c)CH^{eSR<%}QAj(&-k*
zoQ&7GoWAF<xMTV9eOIL}zLEUd6O(!V`~t(+-)AH)i^d-P`n|>Q?6RcXU4glEhvFI^
z$L;=Tzhz?FZ_$h`fex=qFMfNPd|{)bMvDF2iI-fO?4_q&HsO``JoHOtnw)l-Jm(hq
z!>8n<*VapkIHf)^V%1Pp+44%`%eEzl?iralsI5p{s;g`-l=i*<rp+f`whOlkzUf#v
zwlX};O!?Bv;?%k$fjQ@BScJ<%F0s66NAKG_)Aunj-f@E0&OdTarb0n;lG5^Ik*tn9
z#!o>?<|ao!G35oCosTx?^ST}sJ?B%->PfjPKjj+x#cr75-5Q{C^^5)qqqQlyPyWx!
z^Hbz8ittlpDqq6U!u>v*{d&=^8%0L%g7l@gzK|=Q)|FS3?s=8r@&ON~X(~>9@{3Y-
z@EJWkWyE^GNL7JRMS;`Gl#_)aBs4t4b?dr>a+7(S>$$S>jxjj|^$4DRBv@K*T5E3l
zoX_;QCqulkaIA}d*;j*Y&%!+uX34BmjLTD79BOeTEx)c~UeeQapY*(fV#{zQbG7&4
zOP53*5-Y6<{4|Ht{KR$9dj~`oHCHTiw!czrwNLo#mdRh^^sRkgS-mf>e|q@atLp`f
zJFK65kLCVR!?^Ov8UZ`+cN66@8x?1`-rMZ{HhY)Mk(Fw>iQgx$`epp<4*TjahRLos
z9DcK@|7Y@e?IzY5YS9`Q@mk?h$ldif7_OewyYz3)8|Qopm+wj&Sr)mZKk`Yq634N@
z^=C`Qj<@N?0@)W1scqY;?)oxIG+(u4scymHb!%p{yQGFsJIU9yB76V#fbH88%^pV_
z`_Wd=tgFC&e;Q}b$?1>t3Zkkuc&qZoK6tVF-i)2gwrK4Y@ZJ?t#noAJc4=>>^MsY%
zT<^B|UcZtjD0%JJr|B{`{a%KoEc&qbhsniH-wW8k{s=LceEdS<83EqVTwcZLNlume
z(pP!8&!|Y0PK`dKb}y^MTYMU8lTkWT2rE+vvjbzCcq5x2UwZS630IF?V{5v=6job)
zEF;u3a&vpGkABn^*Z;3nDyzSS?`#m+lN9QBA&uv_$pQ{@<5zS21jB+<-L_0Mu{o@;
zXotuG<+El>BchzMybowpv((%@prLtuG22=x%Tslw=VPW8+Wz}4$sG0V&(7G%8^0xR
z#fi?k(DOV<Xw~mHUH4O~f2&AMQuwv#>(<y^`O=3cF5b!_BdVKlX`{mhLB(&y3L;{c
z<-(Rm{&H6Pwp6z)!S9irW>|~rr`2X=%k2-h#O_!wA5?moe}yQ+3RVTqs<+Ejg&xGU
zIIFJk2{1`p=zDWPmGaD95sB{G8`i4)>=NGI%X2J2wK1$yIGL;9i_)TPmg}b(PdeIh
zsEudM3e_Hs&2ictHqX0ey~)2IwdlW4TacG?Bwtcw&(We?88#C(MV$!COAqLmW6l+i
z-Z=Ma-y)&ByJecX@7&8vv>mxNOFM6#Xyhwg>G9#)gtyx!{@=FcwzjU~PVT=VlUExQ
z1~w_axU^MqapBvw%3+J9Xik3-7P&p}sQ$&IX;#g})BBc(o2%yRtSzdXndbIJY-Q-S
zCc||<kF>DO?qU<F|Gw+~xm_O@z3UP;i*(UvSt?gCb?z+(E;9v_HXhN|!&T`|HuJGt
zx-{#)amWaE-WIlKVbI~(mXA;Ksmv30FTS(gD&=t?SC`e2<gY&Z2ZLYL{k&c$J?Eg}
zpMyt>zwFq2=-}dpTK&5EF8c|lGV+{HrB+Ixz0~+$bVsB2AN^OQ{h=`lS}M)`t7J8G
z?N2scbz73SnBO6mrS=H>ard>yUk0|AUX%A^bf41XRA04XVoYgoRmpT4#t`wg+-GX(
z72Yx1oUcC7pCh$l%b(`7MLx$=IwHR?BsXR3`jvlaWk5)WNQqM90lisl?Js!VN9}wt
z_rxLH4IXn&H-v2H(2{!2_onN_qDRb~eD_a%=)U#h^NDYf0<&3lr85gJcs;l9$rSLA
zzPo$Y>P!9H?3%YPP4s@qvCfBmW}bZ?i+%JK;h$F|f9m`VyBb-`srvKk{cf(VtCPCd
zHwj+9W*n(#^)tZo5U+lkzN%?Zc%|x}P5A+=9X+vu)`wK<&rbVaS~)GaX_~va+N5eN
zAI0<oeW7kXy&<83_tW&Ew$120Hlwk8X0p$mNypN6ONZ~D%`<8G+*=Fx>^#u!bLfMW
zc4&D=(vM}EndUB9wItA_i~EejJo9CyXL=+UXYOm7Y<;(<{&!{MbDjm<Qn9N0%Gq`;
z`%>PJ>m_A+@7#*dj~mx7Kk;m#tfq#}u01~c7HwB}s=4c_snz1oq83bI&ups_a_p*n
zejPQnxEwX>`F#H%EwdM&1(#`0TUI;g$F22$Q<_>bmw3t*g<S4Gnf~)+ch#%e?Kie;
zbDfaewC+#p-g&F`&uhvnTissKF|+yhf!nPi9J6936@)z%o55VLO2zzz&HV*0s{iXB
zQ_O4T?f#}wTI93$%g#9x9D?sOo=)4f#;9M4<B`NWO=H=Qrm>eF@5t4>ug>myrPUzx
zJl6^F%KV?tCx-5QnU~`4y>j2aFL&HMw~B{vOMRzTm^XFG-oQxatwq*1bMK!GzZsNU
z%;Z*jOY`&hTKgZ(Wj94ScAh;GnHMg2SZQ7uk8*No#X|Yr8)xs1otzcrvFG+f;X*dU
zg>U992`@2PUR<)Q*k*GHM-2PP>t7~r{!((Xp83Y##`iXl;;nZYA4=x`r+Vj~y3Awm
z^M@E0bvHD|dWqO%7$_cU<q|G>dBD}-NCz`}-c2Ld14ny;4YEWg1TwVuMB7c<k|C6G
zYLdEdQO5tsNsEqj38(X|v7Fd_Zmv!Bt0!B8U--{WX%TyR$kXA{Qm@%YzEKx5kNSx&
zbj!8+u=2{9;Pq91xy;tw2ueLMOSgNP$aGENDH<LN6taXE9oJo38^N|dj<ND|!N=F(
zYc82BiJF)cD)`aiNDhle*_<6pnkV|@t5c1;e4hEHSXA#j70Hlj#AH>TclA`*tFWz6
zoEohgeKo_^-kGw?wJbr>nCbs7z8c{l2ZTgB`Str^78UB9o_&AL$vul^@44u0kRBJn
z@bT?+S+y&10=rA!om%ZLvwp79udlCePxe1=TUY+$!+L)E`*Odo&U*R2`72+1#pBNF
zv5nWBs?G>kjNO{h#BCSx|6+4?qsc=N(RD|Uib;5-D7IU0USJg!{OT6$%EYM_+@)%F
ze37r-svC>)^_|3L^q5_fh>)~oT9)i$RkSEs(&5#K>-~<CoR;LZOx>yy=`qQPEt>0y
z=yit>wuDsILd6r)8P+|$>NO#i=LDm8O2NT{zL}F4RW);b)DFyEVrmwUTpZ>il^OUn
zGHQO+H%o!}Reeef3tRX~Rz##Mx#p+)cEPh$>EN7(6wS0LkJ_|_99aTqDQI11by4Wj
za$skgDDkSp_}Gq>T{Ct~(kw_#eRZr;K`KRS-STHruN?RMd!_9tE0FSN!(t)rH|vgQ
z{np%c)~;G>%ltO2oUO;>ymhx7OUwSRz4Zv=>g*ldAHANr=<{xIbd{d4Wu4pGyo>7?
z-tSv2wCBwvlSwYm`KAcYQheH-=ED@QP=B?~vKhi9Hk_?f{Jbo6t{fIDyu=zT>|B!;
zDph=SQK(Vn9>XxDXxGptwb#=gL~1P#Jr$L^S2UbS^rVfzG>z{|#n09JRTGX_B5<s0
z_5{;oH!iq|_m<3Y*N**?vaHh0&2xTX*rkQJ0kyZXR#j$B6tTMXtW7dcW%=FkW3i=W
zA(_=jTP0I{%<qQh?`AY#vn#c#TQci+Swd;*Z_|VatW0_Ge%D@x?|5Lw+hOsjT|Mqc
zZU6M^6;FNy=G`y0oqph8*@x0Rg%^#@em_nwn=kiYv_)oRilV#tw!VdJvS}W>+orz0
z-g@|V>(TjjfnRNcgXF*2{ds%T|IUsNPE2en@eco3RU~UAS1M1sr0kh8Vb?;{T6OnD
ztrMAqI2JfCJPnqddN#LX+4bqUFFe{xRSFF4Im73kR7+$!pt)lHzwi79{&NNd%rp#H
z$fvRW_tIr@eObI&Q5#!1Bi6GdS9Pq?`>~SWRW(XRhyU?Erq$s$mxUfV88R_WRq()#
z+$dhl6Knmh9+LQWO5w6mAg}d})zYs7+O6Lx^4ITh4t`k>c}a`UxliM;#3vW$=o{1B
zpBxqJ_gSSJBIB{|$5+vsi7Be`7n8i|zKcG{%u^To*eTiEB+~ic)<-*M%BhrlJM4S6
zPSjErW{`E?Hg$CbYiW&-!0CS#VtzReT=^TlLo%+ZPSM<?zhBVXg^T&*0^K&FiHf&E
zucgmC;bWw=M&rR{E9C~?mLTH=J;It^fl{j$x|y!Z^u7M4Xql9*d+y(zC!gf(-29Y5
ztyqNTw!@hRUDhnm_0t5>5)xGRUV7ph{7P_w@pff}&n$d4@+#+R_}t}lHn|sVP4Tb!
zm8|f+<g(43IWZ++Pp7^=@_Z(X(47XR%aiO*O?DJqr7jlR?$7mfva|WjyKi!~D?U=?
zaC;Fl(Sh~Us%EM6ZpJT}`qVZp?(jJ8e)@Ibgf^8WN{XH)o`yn^23#teUby_{6f%(i
znG+^ByHRsutP=O1QzrKQ6A#v<YYET3;iA6u?o0p1Px1csulCPjGw|>9nsQPswYyB^
zs_EJccWI>#+x=(w13Es7%+fo)RH1i*<>@(mE<8*=0)-E9&s@3q@Ko2Sxs_S1ESZVx
z%sAFbo(k=dxic+xa_id6T<`Vg9)<CCzPK)Yb84jJlWVaHQ`ZFj%-Z-Uh+}8X)7jHs
z90~r|6TVyRsQ&3Tl{nkv>8{V79#yNl82_v5<jxBz26vw>QRm5C>!-!G@rKg-q|LlD
zLv^+(S+}j}Q+O;PcQojD<vgAVM_5mZB(P0WND0l`EpqPfqEE&iewQZANSRft+j)0s
zs<`2j|1GnwEHd*7?3z?tDYW!j#N%0)DtsC89^8%-@|4os_&s;VDuugo$_<<R^sQWk
zG9_aQyXP`KGh-?{DxDJ2W_{<oUBhGVvb@&$nOuDZ{i{DbRq9DRF6)0-%51r%;rEB!
z{*#MRt<O~~dU>wE$Zv;^*qK*xXRNO6^_#!t_j4D9TNddHHMX;uuJ~BD=b-|7{)x@0
zD?ZgRUo!I0uS(qzH+k#wCu-kMo(x<&Wx@^xR?Y*g-e%qo3@O|8?P>XG`ri0;=_~jB
zooD##FRm4yQuFKCLcW4?MRHddd!?q%`F2I@TgHOvQ4hI-JQ_r{b*2_%84Cy=>YAB*
zVG76OC81jvY1#;!JpKQM#*Fv_p$7Z6X*O#bJ)Q7pUt*+D&BDSbUd~5<6<l@5F<l-1
z_Ep5;Uy)Z_7OhQvG2Q5GOQ*S=r^xb-i6yMFyFT`KdDqoWoUF3D+t7{EOG(LK+uatW
z^fliOO<;<?lf_yYHPa_(kM2~<SKs>LJh?(W(sn)Ab|XbQ`}@@826l`!mG>9}URHlL
zx)`g~#nWQgldZMb^F&V2gK)+l2LvyB2IV}P{Alfoiu+GyEniz?x-KL?GA5AiX3yIh
zLWYS(`3o0^-B{@E>b0RE-MMJqT+zHYf4!SL=Up=oP&bI}h`xF4<=HnYjAxsNZ#3Dg
z=~vG5*Y+Mul_mER`Ni6`vrl}KF}eHy<T;W0Bd!TC=JJXyN2^~Izg~OU!{M*x#G9QR
z$CnEyJDyy(v&pW+O(3_Lt#Er;yVI7%4JW@(n(Fy|&-bN^y&Szg=Kr~<;hS*!>E}&n
zru|+zNn3K+m)$34olQ{-;o@DMKW(y~ujGMw=T?3y`4*fe5GrU>-)iC;yNW?3Rf6-_
ztI*sZ-nm|mQn#mQhNW~&JsoN>H!^XK<p!<8;oMHwHhz5SkQU%^(sAot!?#&KJ6PAm
z&5XaXH)?fs^Hs6L|5it&!|XRyzRlkFLF#YHLoS=cheEA)2RrV+a4u2OE4(c)D6)5h
z>p$TM^&yv@PMx;rwDPwiad(H+f<Kj_mYNDL3w1Me`k$M6Br3xtI!nZmpTkh;vq#WJ
z!(E|`vX@2owdOGgy31-vyh<yrc$lZPSWtg5%hzcPr@qF8Z}h&wB9@aR@wd!VbYtwb
z&{)%sg6pi3OSTk5e=u2WBJfzWaOD%#HAg}k7qL_+*)C1sR&v!AWYn%Nly*ttmUIuR
zOLS8DBL4Ym!qUygJZzT#63e!+*i3Y8lbM#-W^5z3%u)V{;~&GsuLs+zi@h&2$|;!1
zpSoBQYm|8XVoaTevZ6-Emno%M3rqFe6gMbacUB~C-JGbFTsG08)WNOODNOiKu%*d$
z5%Z=_&xZl$H``nnPW9Oq=wDX8Br!GcsY)%QeqN$o{4u-NkN-=pcoNhdZn1M(TFy56
z{7Vwk(kn}^RUVj8$)-|O`YC4Z0f+ui8Vzb0F%H#RoTL;(*w&}Zs(le(79hCpfHeCi
z=_?;Yx3M^{6w2E4tY%YM-{!KummPhtSM;4TS29#lyJQ>~+#029$e43bY|k+l@rjD!
z3k3}s^*>G5d)-w3iOKcHbp_K#!NbRdj<~tLE)rxqQUBGbRQ_V}zU17LB@>r2ip(hx
zk?nNLRhjhgn#l8M?s6wYpNNT-vowWYoYWj4sQkk{=7W*rgVeZ*p=+1thfY_w<M90F
zR>0=&*>5O0EkwhKvtaI(*t?=$4m;FaCweU?iQB5cu<e0r&;oTUBk4Cw|F_QB*6PX>
z$nmM2L#QMDamkJ+zR#D)J^12V{-e0t*)KyfF~`&Ix=2ZRWr=%x@!LSQhZCnid}4iM
zqWGRbm!1$c<%5jE2N<~w7<)ABf|<l8Nam+sS4v<MHi+zqI9_Uet+e}vq7Y+%tdkVW
z);#f<Wy?A(QcfnDET5I9u3R+TDet(bN_kgfVYz*fisv-(`spd&%j(wcP>tDaw@6Us
z=_36!qu`|rh3%Cq_erErJ#Jr{q&|0fr5AIEOKSQu=PIwQJ-$<`&PlZH4e8#;UZY-S
z@o0vw%ahv6+iQ;=a&FjK`;l}0XG^W?;<_JhI<5+fFe{mF7NYfZN0vxn=0V2=T`d1A
z+a(=;+z`_@R?TA+=Uc@fptX=ofmLi{zt5Ape>dv%ngqX^3&?3P$lX$ynmOTEn*QNu
z^_sWpnLVN|EYts+WWd!dyzVIL22+tVB@we%4HCwNlFo+D+T2458}~m|lJ;2Q)-`EY
z>&#sb^J)Y=nq9;L72|guH0}x#n!sZ6Cb`GcId(!}^DWUSZw|-Kb1_}~CDzeH>UD5j
zYhugVlBr7)+5a6ie^)9~8kNM{+BP-AMzO7ZQnF=^MSIqa<<lf))OVFEuavR6tZHQ}
zxU+dimnCDb=E_IR;w|AT4T@JzU6nd}7DE)9acI*DDU;5h;etoEE&0ll+`*CDeIk2`
zhti(r|7DhHs~1&GY{^<Zdq&{atDTcBs=FVv+5c+xL9NtE$JG0uRrXs7be6Ylm9Wn|
zR?#zE{o>K?JfZZ18EX$|IFz?Ja5`0eT_APuWQg>Gp2SDRHLLnoE$VxzrTgmCx;HP@
z{e2nGeM<aOhkf{0q1Q$I!V^+ej;`mLz#{l+J>&27%-*X1Ux~4FZ(zvYz$Lx#!}E@l
z+j0b=C(dYH&lXw#l4;SUG{G66LIN)*$nBmG%pxSb({(zh*y^bhZ5B3|+%hn~WuSag
zL@jfY&%{X;BAXwp8I`jv(fKK=U%9IDn9+L2m@Gk|(_3vry#(tQsfW*6$}V2uDY?wg
zx?th*0-kU$K5>nJ$p3L3u7#m5RmGA5Z7wJHiBI1>+0rDa?fu1oJkJu_@I+^GziU&b
zxtedklrr5!QU2Bx`S>5L`yQ?co-jj3fl=6CN1tUy{==1XSFJR<yz)u2N7SJmObd3*
z@!C0g^^T6{l|RabOqV$(gs+;f9itXKagntDs?9Th9N9VlXXzHJDmA4}scydtsl}Ry
z0=T};+B(~5-i*MlFUwTgXK&eQv&c~L2v_Q&2rlnM!tad*HYrLw7KNIYE;w~^wY{;x
zm0gwBUk5Xp%&nP}9`aPwJTzqC@wKf7=kjmqiEfyBVrQChs@hUU_ZU^J^W2r`pDMC1
zsV$wTCgKwQC$skM@%;`<=9g>@Z`t~P|GS9!H?AkAbSj)=@eVhQ3|_MSmloq+Ej#wv
z44wjv5_4F@&m3eDU=;m$Fxo(}Sdk$}fR%Y6>+FVokA*j|pWdKnvyl7tjxC!xw;t}-
z+SC7k#zH}*MVyls9x|(!)tVq5Sg+9)sleEv{Y#Hwv(l>8s8yj#JQX5(-i!G%ghi7Z
zrIeRM#Vnbf;U2N)sMLw09#Te}o7|fi)w@)h>YpsB59zY}+<tXsrJcqe!-#^UtY+uT
zSj!Ag%ZS(-8?Ge}V(lfQB4@`1p44#jm@2SD(d(OC?t(2_61mp}wCYAIIC(@o?}NI}
z45=w|>NK7e*X^<}50^>IQ0_eCU95Ggzr*+3)&HtL6hlKAW;95zZ2LW9YWL}huP5hh
z;EZxO6!hV=gbeG8+g$24Tt2F-$p^UB9^l$GA$6KIqi4hEOFU<%M=hM)ZLyhI_7;zg
zj^VD|mz5S}&*T$Q6je%I!EL=^@h*#H>MJ5kADk)sz~g4;wqk}x$~T{sov)5gpRv1E
zvg<UnZD3gHo+{f@A!{-&9!qg#OTQ$nw6Ni_ny}O1?(1F`Z+JP}^6Ke&r26#sT!EIU
zYaXt>cZw_LP&(g}wMz}`uWi)eYmqv2B7K{|rO-(JGjF8UUUz!^Yn>&p%L^_si^D5;
za@TY1TF>xW%=PSsC(T!w1@ns!UJ-3P7}U@+>mt{i1+0f_Bo;6D&$`BdEzp7OjRM10
z1Gd=#+!4AAr+6+iCGe*xoMp+rVsPh>%iTk9>>|HTuv*;d7kYhGwQ!^2sq2!zcOEsH
z@R?mG*ueF*(+yXi96rfSK`#0~jCSu+UMzic)8yJ*d&87lZj;QpH_wcnH23UgqwdXe
zD<&m8+^nlMdBegj`vgrl9_~;SwEq1#zy0Hu&AM9yvv0fXEZAx5H9!4?ON7+fJ6rim
z1SdOAT@t_@=DBy<1+}o1lj|MXwkceC*(!C+ck1ysYfs+d5?*Ric=~RYbk^e|32#`G
z)>tY%vAy@4amP#9m6LDH5IQhp;>(p&tn8wmT?tn-E-t)3Nt?mfR>0SnYnuRT;DZ0G
z+YGpp4K6PZ;BG(2r=ECb@<LXRfU9c^E-%``x-;it_vw|gjP<jU73cjd{g#p=x=K(f
zZ`ES1UH9MQh-P+Z=j~ecs#Mlaae-#wHse_vt=dx4A6w1~_*kPX5hAR@WV=!G$$X1@
z0UMP)4FfB-m+yR8F8(z|b<X<lP3r}8SVd>A&*1XB@w)rc$)`nTbK~Es&vAFSxpS>?
z#)+r5FRgH73p&8YbA&H;ZV30?6oU-@JO{38iNQUS?5}o*yj`Gv`b7GyHT(5g)~{!q
zFA<^@FS*`)^NMq}_WU+ag?tV&{9aI?EAD4I^XuLFVNQqsX)&;$e)(JXm6Q+L*AEW@
zFYt9d<bQMEKi3*L)-?-4W(zQOD0(f~z`EuD+pG&AxfAZTs2mpNx&Alm<y5&AlbXZF
z?>(IL`YadIBju<GtelHFbl-k)uMY`1;`L|3ccF&AMNz&!xttb)I^K(e_ble!vD%yQ
zn6c2Yh@MV0&0`C4mwfyu>Son(BvtG}&FQ&O2jA^oP%~HX>v2|l&Z(z&o5XiXu=O90
zo9KCUsbna(X)v?b<E6{$E}ii@Dz*H$)NBP-j=YcFscLqdcf%yT&unB}<M^Qak?Mn|
zYwZt9wO%}_JEL9cS^JVT+Z%7b&~$0rJZHum-TM-CJDCLX3mp$Fns?vT-q@x8i?h8z
zzqUm2M$T;kS0@=<i@u;SyWzh?paN^(qtiJ`tdX@6k<(aCx}6pG+bOzcJ?l=PIkV3(
zZao{X_((NIFyCKzUG1aFv#zJ>j8;k|XYc>E#>;xUv(&aEtM%OH`&TUAzAN$6=GBgJ
zPtF~-*~Qr9b1cO+bj_j&)&IBd|C`0|^X!!cT;Clnq=b4abCWI>t4Zb;&ArK0b=0D2
zc0g6{rHk2qQr=5Lz6$)>YH->7V`$o?lbKokg;%(?Hx#}x6)1ZwzMpA<>=k3XyW4F(
z3VlB%_M7XCTkK1>yH_spe7U*)%B}fViXOh4Ehlzg|JCeuEP)SSUf+LTau2I<ti+lP
ztcm>vS-Olt4f{l-&hmP1m};l``~AJo`~t$u{^#5{)W|Jql<KuL^Xajvht1~f$a$nx
zc-loze0$EB9jRXKjEdViBqWQP8YZjzwz-_#q;hgva1@t_MXJ}SY1-yIa~@q(KR=hL
z{N0xw!p}VPbSAg?oI0u*ygYEJ-%_oMD?=H>wDq1&xtblmA^vEas&$xy`g+CFi&Cqn
zrABX2e7tGO?`iL1_f&qpr@Qn*{{0|D;g*mI4vh_mxR~B7UERX7GHin9#w1V6C8>-v
zOv9henSSNzIVQWwH(hR(WKLV=>?C#}EyTl!DQD}fSZ{~5VG~tr@~&qIuMXQ6WxMUF
zUh(Rnyi-#i9=fwow?#2}>di}T4YQZ!dQA1aF~_^}{X^#_{*(Xr)C7ONzq}x#zW5Le
zr?AM0=?fg3ntpxfel1=wf9c-|lLQ#-898MpEIs%kh{=(SHQDn3gZvZ&r#%XHw7HDj
z6qC7iCrvoy%>Vmv{*m8*4*PR*F1g{*)q5gC&&=pn#$mo&F*DDKh39QFJT9B7n0lDk
z_|c=q-F8Mr9xA;{yw&#1xYB)6ab{7c>!cNT4tdRBSrD{jPn^&JPlNq`j7}-{>ug%t
z&+@{lVd;&O%(D|7q~x62eQQgwrp?kTm(SYnO*XbQpUw5f{|ndO&8ziJy~wh3{`;oS
z+I4zxNW%?_iB~S|`w*ni$-?t^%Vjoxo=o$%H+}>%1Sfk>y%e12@-@Uvqc$}3e`NGV
zozS4L-m6!6l7$~^ozQUfwABrd%*Asb6=g5o{&I=JVKK(2#1mrWJBp9WG3IPJ&KEx8
z^@5#7cV4`<cJr*5XrsrYwXj?0otAUM7oMW56|<Lk1Tu+Cz2Sdf#mrZMlauX9Ny5dD
zN99ht%tR9$8TtDH;&&<7^lWHjZ3t*;6Lr&h%)zQ+(Z%7!6!~NVtHjMmonLJO8xJqJ
z5VWFchk~Nw0hywX=sCFuZ%&<CzVD~vyxmV*cseyUWi0C5cH_wM-YTgK&s`xx#!C!#
zUU{uDd&9oWtZ9CqPQ2DNQ+TPioK?$f*@la0{b#pc-g_c<+szoS3(MAZd1<fisN>S^
zyppt1`~TK&Vm6xl?!H;lz3u|j9qprUSFP6DGAZj${;A7FuRrd77B$Cj7sKz<jz@T$
zbH41=nWAKLrT47%wX<uoZH;djF77tEp6p&K5@e#2+kKhE;qCR~9lpM1DbX8w%`Zme
zxbC_cm3c1W>g^AGmalHNeD{0(Go`<q^TzE5dw(7}-|r&CsJ6j{yT0YaVUb)$ncBd|
z%e5WPJzqaxZ~FWB#{VZwnbF{?w6nuH*TYXc<|3EW!;WaKI~QeR1cj_Q64H6zL|%}1
z(bfD$&m&zccv9Pwi5eSS)FOm~udWC?w0nn7Yj~1k_XZOoiL{m80h2v>y<Z&U6Y}(E
z?(&`eWXf`@r>fOGl8^o`c#-C!SoO5S+ePW&gd3uV0{OcpXPpz#ojO5#TB3DC%1N=U
z5p4C9IR?o~?$6n#F!T5ft+3s*rmm@Y>SnQv%dBw8W9j<R<JUQ|_4^~k3bPXD+}X02
zH|T+blgd$<4>P!9A1I!mB~)rF?%B$m`CES9houZt6N`dpzqx+QY$>azivp)$$012~
zC9jY!AMRNzShW`^G#F-hcz;bedcZ);plf4_<C4`EVr|qG#v7fPzLH1M&^FA7QDL&)
z>X=j=<0D<Z6ILnmaz?T4ES=7tc3JUek+X+fk?exP*XfU0W=Sto7A(nYRT7Vkx>Wbc
zZ}x<@^%gVLuNF8>pO})XW_)wjl_mepoEUjhbCef$+B81eGIv9IZpaKN_FpRIt87#k
zMNQ<g+I)#seVXgy$dxBt4>no!&r&VyDdvvx^13v!Z%g6q*_UI!EEhk|nwGRE_)G#{
z@@0k_U-X1-IHg1!oOQOrZN&!Ztbop{kW(tA5sNi@-+4UV&GU0z#OdZ$Yw}X|+?MDU
zu~|E>@|#9OSz#j=7wh~QwFdd$6Y}jB9ajA-)N)$o%N4N&K22f~5BY;U6uCC0w5okM
z#r5FIVY4*`eAZbDS#NLPPY`30J*0Ew#vX^V<<*5ZwZCQ=1r%lQ%sz3~KR2|y>WA`{
zg<sq4-(2jve3c`eX$Lct>W^F2d(4_Pm%hzWjkx_k>)EvEwR0ahx0MO><_NTVHmP5Y
zS;ivmr&zBxXO??U_8o0a^Idu(C#P*wIXu1MWXnm_;PT5`oK-fyRLXiJTp1s=-`X}L
zfIl_Aa-;hSwO!@`FBh&~IBoL^&%5G5-KV<Z1CD7n-enD`@SLzYYT>c-Pq@OkmxVUp
zKH(O7(kq;&Y_ZOJ37x$h?+sHH_}UhIc28aTLLnd}+L33LeC@`b_{2Zg_is`0>VEmf
zpkVF|edZ|)npY0VujoCXmQ@(#D=<y8P4=k9E5kO0>(1PYiR{|16dB59@Ff4|x^cuq
z-G?E1P0IQdPeXS%FN1PR{WDiwbbj7SIn1?9ZtshB{a=Am=7<0PJiD~E*yGkl#-N_d
zv+q86HZStf^VdIip4Z`&vOF4@5x!%Qh|A^K7hA6QEj8@bNcnwh`Z9qHUPZcDiW8Y8
zS1CD7DcKe0^U7rT!7W?8!gnojl$`Cb?@5;M<R!VehffDT(O$aj!P`7P$2$)!v@9(?
z=H@Hd$~<hF8?;@=ulQuzw<jIFfyZOd6`krcdw%bzcxzALvUW`$$@fW5jhFSUs$M7?
z6wPdXZQ-f1r**QaHyW&$_boA*z2cS3+RZj*V$a<o*Sf|%e{BEpW_xOzVnwH0i9!P_
z$MnSy3_A)u6@UH=;(oJ%U3am=ji?RDJjWcHL?QyYmE9G&+^)4SWc-|^uziv9|E{H{
z56hpMF@5usO@SxYh)uW?*;a3JcAv!BjM{(CtnNQO>Cboag0CJ!{f|SDHqHv?Q+}7+
zcDu_PI9*}>TZ7WOVZyJiC(m2duw>#xDb5zZ?8K7&EiwP^s_vW>wwr6KN$gXtaPR#3
zh2DKh-!0efZ98l$EPCkl5sSIu^*%OA>rQK{3JKfob5|*R#Q(Lz-qKJp?hTjN(i^S@
zd}re4XfNIz(fELK^T*!B>yux4<{!6-vp8JxQ*hI*kH<5frcL<o^i(@@-WlfJz%vW1
zBQ?!0a;bVB44Js_hMN0vPrpYMkxLe&+Ik44hHY}@x+s76<lHr_fhlv0qJH>lxN~j_
zd?YvZA^-nQqe!PT<Nl-*&TP6F)q(uxzm;B=d}F`x`aS=yEj#6N-W`wq_3{6N4C84p
z>?hyu;CiLT!n1{i#f0sT@m_~5&YNdCMQ>T?scsgz+#_46;#0?(m&^Zc5ZC-6?|plD
zRh6mQ%;f?i!Xk5auRGZ}Ex@!{OJm`KPJP3!h||j_%vch=*;H=L?2catS8mYQ#IREF
zkA|OvjFO7k6f4ciPG*y{xYbh*O%~eoTa&d(xPjrpHs(N{Lmd{~oaQGb%=Z{*8S(TQ
z$*ek`(7eou_iiHRl?RKjRX9I0lzaTsBJ2VaPoVu*<<^rD%({+DL6evqdsZ7UFV%Q(
zxbE=bu#8P*AJ}~p>@NO~ko&73$h29)FPSykV^dVY%&HqrlP+6M?pzvQxKv{TlS=f?
zAeM<54FY<>s<SrlG`j43fJry{#m;%!YcH(Y{7d@4V&wyit2g~q-nB=-ZI6Om#H7_(
zNu3;rx>H-MQfrRgn6-An!?ky^bY(g@#5si~3QRJRw4TqPu)}%X=E+QjbEKXf+O5A(
zPrPe)$rYiOyYwWqOgC?yrN(Hw>f<`YmkMS-_4YF_`m$^5=U?uPUyf%>pZFSO;}E&%
zSChVdqDaL_{m(D@+MleizqyA+a)af>hWpkVViKov28(ia8%$^sj7a3<m~3GFOORVw
z%ytns@9&9xzYWqHb$4g6R&Qtx>`=Y^e@2h4#gVdv);Vu@m1bLLe4hA9x_NJqk#B@t
z^i8I-%@W#|o!Awd%LH5&^fVZCw{Ro~?)PGvsJwLj7q1CFH%*wpqPt<|9GTWohBf`c
zo4;4vTXvsb!oBO?YH!9H`x%bx-~FbQLB%2di1+`;rzZ&b95`dYB$-(=gW2V^abeMc
z97U5|4>q&!;#(Ev#=)b$ps-b5Qc77$e@4Wtc^4eKq)q*}CA^&k120QW_BiNud6G!r
zy8jb9ud_G?F>5qsnNE439&+0^eDOi)Ii@okW|vHqUeVAMyIK0l9j+%&Rva)i+hDP+
zn`zsT#%+gZ`n|N_eid@)=tjRcJx)`*wxuq1@?E&{|9=6|fQ?2oA1j{};C)zla?c9c
z3oB%oHu;wrZ!bSBd-=iK%4V&EPTplxrrtT&yV^-E>_$`HX~VDyOez<est$NwkYWzn
z!0x+2`w#OzYZl)76E0|6U~z8=u=~=w)=K{FYgNXbwiAUJR3eUQFdPx_4V?UN)0s;x
zO3bGs?^s6f*&nmVepd?TV&h$le!D9Cy(s@SP(jp9$@Y@cSsvYrX|fBgZZG1t&R~9;
z(QGtB*GQq!s6os;qTj6J*qY7DUS{;=woG!+oUC8DT&r`pfXi;q0H$w|0{+emtJWx%
zE(x}pt0%T<l6K@oo3BDv7kAh2So1UOcI33_ky@nWbwy>>Ru!f_3jc%lG^^PBnzhJn
zq0;&XN<M!Vc?r%CtX+9`gJ{gp6J}Pnku#Mojch}XDu>qkUwgd4o@t|llde{^t>0$W
zuos3Iip^CQv}dz;T|F`3(T3GwRTriR_wruY<9BLlitVzEHGzx{`?s*@M0srf)@=Xx
z^2J}>#te5{cfYx|`^ixT8Hc?lJ_TR*@A+_gzs~hNAKHCYn9rmf*ta9dXoIAl0!Q<W
zh9w28LK8Z3lK7AQk(_^0clJ#_?kmFfM;$jQOE*lK)fy?``)R&Z(7I|*NA<$90(&BE
zl<fY~=_~HTdG|}iW=oC58U6Z^(v$W&8XlDEWI81OMp}i3d%xkK7d$I_wVY<D9#Z{3
zr~BB=n{!lGs!wtHsB?a{Q_mc(&Z7pm0u5ygfAbV;YZ=!>E$@xGxL}IulI@kRMJ<v}
znJ+QFwczj#Wljc<291Wj>3>h|DB2gefpx-7mT1;Bdp1oyp};!pv5xKw&%i^@&8Kf4
zTO^?QgZJxY%L|`&&Z{=Q^lN7k@3DMSxA-O8$tFPyZ*w0Py|j38%rRNF6SB>^3lx?&
zi+?K+tS@Ed<Ct;wE|axFGv@;)_D#)IwPM$b)?O)Xy&kKO`BQh&1Gn3E-9?fnNhAqv
zncc+dIPqV^8o%yIL91`<nXtR;j_S_hg`2!D`%QMQs1XXdy}SC3b?u+sCKr!=<GN)j
zdE(orxSw5D)c!}EI1%m9zDM8dqgKb8DcyTKtR#C1e%ri|Jjw3t$u4~I*V6mJPemhz
zTzQhOCKZPAAJ$5}$)dHG*Y9#~hGJ*Xj-{6rTD1>u)ShXSvex75=OvG|{de8)lAXN&
z(yskGkJy{vKD{et^Ijf@Lq3Vq0v_g{<vw_3zj$9_f$kacZ>^$k52fuM9z2q`>qBec
zjfQ2?%fmigzr9H0%?t*k4a^yw8kv#&%d5qOW}KP*ahbEViR<TGixoM~*K``(l)90_
zdHGF$%}tfMi40plcHB(4aj7Ow$jJBBn(0$d%o6w$aVN)DYR}_|vXKvZq?Z@0kn7><
zxMZsL!Zca?NxbzFWs{VBjoV!Q&%5ci&M)or?73IZy<3wqkJtb7!?-vnfBh*E{4ZA;
zJdwHikmun{|Ke=_8-nwG_iVqX8CsRidt!nrhuDR6M|p4GU}88h<M;#l<$J{W1p_aC
znQ}-%^pAjGQlEVO<?bC>?e#BqD*6Vgl-|i`OUwFpF^lig$+OSW>z-xKORM$>kV#xH
zp@5mQV8w(0g$m8|2T8)w8O+}bI;<<iFD*@4zFBSA<ujf)k8hB+Hkui{McKV3a7{Fm
zgsR5w=W{PN=kb`Dz7Pw#{J&Gt{_O5MXOBCaRaBJJ|6sMJfknwG&}MJLl^;ent#jtN
z1um+YH^pb|9#7A-zlH`Ef(`yo-4Lef8FV+^;=j>V3$MM?_Hc)YB}C{x5Q|>(=(YGF
zx0$^syJL&i{12M3=-<*7>z=hatiPCd?fzq5vi8+c+w1vXUHAEf=ktbdcy@FRU-sS#
z`>+W1+b>S9O5>E~dtKsc6`#P$nX${cTdZ+{&m7Ixxie-OEm)Rk)tM{Az$Xx)xz9)H
zqS9W*MU#G*Y~8%2yY#Vv-P!i1OGG-iJWqA^V?HQisdyqSdHPCmku~bGI5akKb~Rhg
zmRhs0#VRt2dC%0`n~y`zt+jBvwe048Pp4achaA`C&YkKsFSdK~C%^mGQg&HHu}j{%
z^z)EG%=u-x?~F_GR?LkueKU1+?y9Qadoy18uUpHzEJn_VgJJLgv$iQ|0(UQO^AnZ*
z`7kAZlZ)=o)iQUV<(RphnmhT#(X?!}puA^+fAx0NMR@xPFjpDu{C%1G=e4vYw_T1d
zQQ+j7tuk#v3FmZ9f!Q|>39=V8H$9lEdm!j+uiLd%%a(0!-S>iP!)3)y$rF_lcMBC=
zu6LVwR4SwArbx}tV`51MrL^1yoz|UByI1<gMCN>ORnH4yuA+5;Lfv-4wRS}x@7`PB
zThtq--%z9Pxi;jt5&xG`k4~5SEeZG8Ca%$wnd*Ci?`=iT{6+DBy2Zak&5fpKvfth?
zHKlXzw7#&vJ1#3dxTC<T6>YfUO2QYvurF$JQ@0+mUB9tK(4e0?arM1KSvDVr|9|Yo
z!b%P;;TF5cEx9f$$|*aJW$lC$JF?ztM;Ux!%=sez@5`dw4Tb@a=5uzMB^10<=*amP
zKG)HtFp`gR&FlPCkIH#JuAQMMCi6*f%8h>lk1y|8QWwa?{ZCwTM$3|!EH5XuGzcC^
z+b(W6^TzcpomXn!h<uT{x8||+yR$ay7H$=M^We(Chi{S}`JC(O%1sP@l3+Y1o!RWr
z$#WAY&6$&1?mf-y-5i<B;+2+(xAtB*l=y7vkDW7A0?uDid{;bsZgKYZ-5sKTt}OVw
zB`@q^Um^3UjGe184)R*>Uz)@xI<0JVz(#qkFB1X}1nzzFOl9hwU(48;QrH(UyD^`C
zZ_53lY5}Y7e}UG}hSOCq*tgbfQd-P@@l=mt1go6x&%bs*bB>AK3)E%l(|x1Bc<&)o
z`wmu3*9TX#+&Dg|MbFr^YWCW-p9^Z9opJoDP!MCi^>^(3Sc!(b!bR79_zJo-ACtG-
zTj*GGye;i<(>`m3=@|}N;~Z@#IYv%i(y`=WX3?SamG5J1j+om_bgMbKS?L6`$no~+
z>k?l6`WeC+z4&y!!7R7WvzIBG%~~9~GIMU2?Yw>5MjSc<-P7)EC|+b{=eXU_yX4i4
z^!GQX*WM_9C8fO7a07cz>=&_rwfU)E7A3bbd_P?k!CLmBU0J%-Rs5fi_!_m~pSL60
zLs!&oS;nqXY$Ut1QoVcW|NpD^M_p(Mb+{foC$T6t=BF~VuEO%;UyL)?{5n~4CP!&Y
z?k~03h4lrG#F&!KJx;Z5xS^78X_{uR3a5vL!=Xd1UMe;{D-=#1>GBkc@lkR-CDt^1
z8^;8ppi^#b!EV#07&)Fgsi_se?v90H;-Q(+%W?z)3YgB#W6dvmp}8Ra!UC&at_Kq)
zbUQ8c-s#YLDkLk!FLY<bS*?c2PU}o|Hr>@~&|I#UcyN|zb(n+t)}-5qYEPHt-PvXG
z{?Xah=7|j~Od?;N?lxcOrn%Nb)=sA(#3_tR*1S%~(kPX2g0BCun3;yN15<4Bm4vNK
zG8pHHZj+iiq4z-8wwUd!uCAKe5XPD3{oiAO!^6Bid*A<)n`5w}U{;XX&(+V@$A8-M
zwTrtVu78?Z6xZb6$>FQx>;qR%v6nxZ6}tAt;;^+zPv^$%4_tiXYV^`u^E1m%Z;W22
z*EOx^+1ESn>J9Z=w(a4K<+kqfno(Pdy0yz}Zgj~q`B*%Z6$<J|Y!!L7HPK19PUC>A
zOx%lw4C^=o*utYNOj*Sh=BiBc(BmqZ<Y&HZNsEY;+Q!8qwpk)eSVUMYFHQ(5FgYqF
z!#j0?kQv|6NqsVbJC}F|+x%3S%-MJ(wJ@ORq^oxlpX;)2y-hD3MPyAf3=7WJW1TxE
z?^vYz+&sU~wz+<HEk)-gPkp$20aJ-$n0Vu|6-kRK{~Lv}Ez&G{x`JW5<`eO_Y4^6W
zYUV1qaY@gcbm`ghMQvIKRxF!#s*Pbu*REA7*R4ABYSqeJx1Ocv&RRAht8!~$`0U#!
z!|vs5-H<eqX_uy9X8tbKBWpID_4~4BOZ~^Nw*}R%3I`;<-MQ|s|Ci%Nx6=bIi*-95
z+Og|dC#`YjwqA2VSx@_Qj(WWA-wjIhQdj*oKeczuoA2|>Uq3m0s$|iILvtAJtC_c{
zC0}{WUHtEh5!Vq;-STzYIdf(7jz8DynI)2Y<I%C^bvqUYoKj!cQ{<$#`A^Yl<I^$4
zPA2m!BqW#Qd=!l1s8zY3=JS<D$irw-i;5?|F7L#NUJ)%9U4ra)x?YU_zvxKNWl#RC
zK|Y@6uPpToZt?wgBlYu@Qt!;kwU@-)3K`AW6+TLSkMQHn5)G+JE)|XRt1P`98T%*p
zM_53vYsI6KrB}tm=bHQrd(!WIFU)D$dCT;K>5rzmI0@xUj95PNyH@)8>6T@`R<G1v
zqNTZ@Q>@x);n{fs>$kX?-OgUGs-iP-=hLvvXT?cfn!fYo^&8Z_e-&5oz4X=DU-yf~
zmvt43ZNj_No<IFP&;H%T&wn-EbQa0!zFT|gZuytF>iZhLToJajeK%G7Zqc-p*ZC_>
zUOmLucvjZK>Kyk3hRmRD-3bT9*6lca((H)?U+arTp%#U9wG?i?*b7aZ3w^a?9J_*g
z{vXRyP;9=sN=0a4t4gcd7NJR1M;6P=YI+!4>B#F45nA1O((Bm9OH-yjan|&8@OBIC
zxN(GS?lhB3pL8K%;prXA#Cf&4Dpub9z~=0#>1F5_@n^EgPA4&k%D}*QnZ*$brYp77
zT)3<sUs-;uLo0OW$B;Z9kJSZFwXCjb^>4ehST1z)>Vs4K6jKshwQmJ>mlYJ4t(BOx
zaly5ndn)uMF-=hUB;w@h?bfZ-l6Yc*(gYUWFU#f6NT&Y3qbeV^!P)NdXLj3T23%2R
z&c}b1Xl8g9%KiR|v+egwPwPz>^<Uq~7G1PsI>VmJ{hxc2WM3?7-5#`)ucz0@N?|v@
zuM4MM(L>hR|8G5saH?6zsb}&bh0Af`*#xzQnF~ASs7Ecg5-mD-_o`CFW<?c6&yTwn
zv98USwBwf8x0(K)9*hgTq%*t}9+~ty^m+uI?pPYKNJVv{ikF_|3vUgnZPJyR>gE?S
zZs#2m?*GQ35o~bE<8IkvZ6&{ekjR(!{#@JMGKEzqqR{&RTl=&bvs9N`)XKy@ITSqa
zma1Ncj$Z;p@=6At^y1@_ot6s~i(h&A)X1bUD|N-HnX9dq8E;qeeUq?V>(;|QwGB6A
z6|)z!+)+`eIQE-2dabMO{g9870(XdiKDCw0@Q1hEtQ&l#KMr^8)oK?_j&QFI6SQM|
zq94zcFu(T3LaFkFo8(_a{@3}pMw6lZWq;+O7PZ<IVTb1pV*1}C4vUCzD6%zqD5@>;
z%+fuwFj|9U+k2U8ZHv^~zKU)0+H-g^K6rNZhiK0CiCxIKe#v5SF3p`iYaZs!daW{j
zLqKcvi-p%VY+0<}w6eqH&m^Y<Q(TN2uXhH72#dtJx>}27tvDokW!{4;>Nbj5-9DBg
zi#bH^|2E?a>^t>dvi17?|5EL|Yk!1DM|{)_TcUL^_tBI!Pdv0E9}4!*kMfl-{CeE}
zp6Z$koh3Oe4JU5usI27-Ww<iu>6-29M5DC!UE8wg>$UAl{K`QY?MJ358!`r}C%zSI
zF50rSD{f8mwEIjml3OI_ey(jl@pr{h?f<Tx{T~m!j^4Y1{jgY>Z(HxBWAcxmO%q<U
zQD2)SCE=UV@eP@?cs(@c2;B0Hh*f5|ch$^{IgNGhO^0;m%Rggptvp_~tmmoJ^|o^h
zB%Lp~Tt4qytdep3<3l@3r3<RLES_w>FP8A%I`K?RyI1;WjO%2z$!l|6c4t^iuPnbL
zylvxKHRp-9Js&+&TIA@e+3{HEe%9OVhYJtt1Y9(IoG>eH@58FXJ?Auo6^>s{+Ra&X
zblw~%gXeG3oOgE}G@Mu8ZsO_N>1W8rnNql;S2ENwv~*(N+~iHWByRMwg>MPwNEUdk
zRede3eB;WqIZGM@A1n>^Q~&n<!-~0kG`1YFzftL@{qyVp@OLHZR@;5}Cmm*A(A1i5
zC3t>K$t>j$One_Ua|$XP=k!qMFaNVy;{O`^s6>fyjethws^wXk*3y2mY|ngCSvNj<
z+5K1ZxcuE$YYsWYaR)tly*=4u%fiOrhZn1JzV!Zd(YZn7u%hGKHXcJi)rssjEly8%
zMRGkVooV8t&wH_>`J<lD)Eh@OFX!yiF?uii^vLo@d+zQv>8!4PdfIj41jC*=98)BY
zc6{H@zOGB?#*xHFH{NmX?+Phs5lv>XTy6E~Vqf)~nzed15}_AU(p$H%D!sIwt*bFb
ziF50dSiQ=V`#p>*CvG{RVmQ+#=tWG@wHPBk)(v{Uj~qYsx0t8i$SqatzrNXNuIO`V
z?>G7CN!Q1m(!c&`vi^d{*75pZFED66(VZQ3;qtj#`;F(`WnFXs_aXU**MHWQ%uCHX
zC-?Wn{k`dKjpx5-J8{jr;ACf7m{A|+duiS@4kz)dmno^YR6NhV?~rqqoZ^);-R*9N
z?~Gn|&I4|?SKK*WJw;t_%Cg?Q<LkBYO2`_Kxl@kL{&36t<t@L9Q~U(`{4P%MKYP@F
zZGu1N)z}-XQkqkpM5e~698^e+ejIu%B(*K{XF}SC1g8SG-=(2~TtX2CyDAR2ZMYEa
zv0!SHAba=Th-gj+h7D7hST4p!c!@l66_Mcie24#^3;Vto{KvmY)Ny#kp6oof@_M7q
z`TxgrWn}m+NK0|}erb|26-~Y%b9Qg!kspdbpURZ+#GlU%-lgD?Q90-9hC^xsQj$01
z3k_#onkg{Jr}1)R(iKTg*B{cpI^6qYy{|aNUT&RrJuyTxYx?vjt>vmmW=`q6sT+7}
z?a9i!omB_tR-b$%%IGEb*K^?+@tUjR;t#GbQt?{yqEn2kJ>G*Mi6>%t!n~%1Ud^^Q
zjMeU~4v1JC^4R2E&su}FrysjQwsN)~b+f(@ZJp!DU@+;0XLqMAcl6qOP8~;FH5ykv
zWO)^Or2F92q@MZPV}gqsxN8F@A5ZX^-rA$l8+l^FO~<kx$9t_aS|g{csyH((akN|F
zkagy4W}}Yh&HqQP%&@z6eyZ*K9RewF4ojSjRCY8hadLCtCooG}^z5IVDGr$jmF1)y
zIj*>9o?Q}pq$BmnQpV%10>>DbQeRk}b<UYSJw}9CDRt%6-jyqR{Z`z%y~8&^fv@`k
zFMHa9yIW7LW4s-v#1}5r7tR(PsdGDGQ|lv%+c65!4{UBfEpd&$^e8TZOI^}UPVZPW
zSHt#=eEeO~X)KdoFNl6q(kVOhnEAbl=@-H(OKynWiH>?I-B7qF`eKWBL8FJw1F<K^
z*do-DSBdTyWRrb+zTrrsOxv8}ozIS!#$+_bWX(&<nx-b*6?1H&S~lN>!y+6~VvA%h
zJ(M~1FzZuV&L_#H-lhw+E&s(7W?gc;!vC{FRjZMQ(=*CbmRDAJpXc-9IN6ekNjL6x
z+*tVNhU&%g&g<Gn7w5kTY`K}Ld#h9I?%caaJH;92&3+*(aZuf$s<lhx@dpjZ7aIgu
zY?)ZMZ=%V(Q*w!Mat|*)nacH4GGoP$z;$n0w`b1Fu9=~<mFdM(l}^j>{3UU(m(FYD
zbM<?3tVFXx#3xN{qC|J*r4BVmmK-NLq4_gc%759J@!7S%ROEPfT5#M3jZdM%J-0GK
zGM)4<xSMF+H=Wu&LFna#R~g-GD<_I+`reu26?Mk_#(eLrMin2;NjkEVIHoLl`Gg_r
zh~>OwTefz8Ki?OU+%vTfvPAf5o;xy&tKq-zmY2-?G(A4Huy`%-m?&~*P6%6GLP(72
zd_UIv4?M$O%vygp<k6ccoX0{qH!XO)BVq9-)wqOn0dJ3~O^Wjnhzq{j#%DLR;7>y6
z*9Xh9MBh|QWw>xo?&7hgR<_)mPLVH<2pcVn3s`6?6gYjsOBNM|S<jB;v7AqExg_?5
zU22zF&bq{g3(W=}+yCxKlU=mzi=vYBvuB&@WVACH|L4S{i?&Ofz0OP1&f2%^tHmP?
zgGD}GjbZ_vQ~9QF|2@I8a&?jC-PK~}H)?Pv*DcCBk-gW6rL1YLrr6SbW#RkRB~@&F
zbW=A_XVcwEtGLQ-;fJ0jRT=3R{7W*hT4V5T`jKnP>q95~7h4r*>l?A6p=*tTgh|ty
z`a@igO(UMDvY0<|mCm?vuHr`an`=2+m?b$<H_f?W6~|?*)-X$@LCAphMZv+y&Uq6f
zysx?*h?{wTJHryQOrD)KD)SlR^)E1|Z1vXbWr~~1KJ8|rvz&@&P|9_;6vtWJ*Tm#^
zX7HyJq<qv=F>CCZux{-&sZ7VdCmwOf&-(fwjPRLqEahHb_B1{b&o3S#E)F6dj1Rch
zJ@}QYW;H$DN9)zyV>7~Bcv)3DbEBqr_?8I<GAHTHbxZJE8WpllYF^O6h!e9yuicHf
z5%q9FV^~D%`r4(BR8~byQ+;f}_w>pXCy`T;`w||w-j2=C3iMR;HDLL_Eio%r=}lA8
z)!0v`_SGDFeT0weqhM~s?9~zTpKC5lGgB)mNOVeA<`nQO#Y=r@*ebCNO$t$~Glc>q
z{o0d#j(=U#^xxu|w&r9FpGd>p;|Hd`&)Bx?C!6;6SNZ=xy+8IawBgI06A#lE`Hy8M
zz5ahF?S!Crmd>KXJ&c8Y+JzHeKge<vi80uj*QkG%?LpN0`n}nkYu<l9x0)s4;oJw=
zny*^R9<kosnslH}U3l$Wy+wj|&pw)cBbLv6P4%Y_5*L@(99zDKBkqO7jJh(NdZl>}
zr><F(viPLg$BJ1wg)6)iOf{Z2&QLn3b9KYwwz9PJxD{%1AMX`pYn}8`;`W73FGt;u
z`~Nj^!;BYRs+i)w?~eD;HBP#|xrVjJ4|NJz_c;Ez6SqC$TCY+3m!}yYuf6zG*{yLQ
zNQy~wvRVAkUAgafBu?bgoD}v+L7e^Wr<q@OOgj8z?UbfX+LC9=T-wDZWOfFfeq<&-
zu|QLM@=C8IFSWE-W^c;$yHGTH+u1quqC5f<yd{p_PFOJiPVc;TnfKnzVqbOC=is^p
zT~9fLq`AzB7a6@W`93A;-MXr)3m#otpvv3%?Bdg9r#eGiuP(33N@>XwESl<2Gxdq(
zygji39TS^Y|63P*RIB93(m5`ZM0(aA(DX@Md+2A!L9GkNC2p`Mbu#UkGAXmesiNml
zQ3k(pX8-3E4;8fx{%_G+&G`D6YxWnd3?{R?=Nl7(8D8>Vx#ryHac!d>&!Lc*^O_(2
zajnkTw@mTv!?~U8XW!mb%yVM7puMlrK=j=g|MS_L!OJdI$X=VHw13-%{re2Gj~R%z
zJP%$!_Yhawvl|<NHT+&aTEzE%kv?1DJOSnBJZ?s(VvOJQH0wsJtXWfUwK(!VYxMCh
zx!lF;UbLP{lS|&xe7Z~dOxnlyQ)Zl9wmANMwD|eYDusq0mR<aj<}>Xg^X?AriBB)p
z864Mg;F&BbHo4PrkH)4Voox>1g$_kT9;<lkQkR?2A9;L6S!T0O+y-WiUha(Vepfaz
zdl}EH-f^iq=eS14p;(zOHy3TYed_K1zPz<@Kbo9)%;QALLtGuTS8tl}>6oj-I_>Ez
zFFa%M4S824`;>WBuj2*Hvc*x}0S?|as;fKS8GPqluxrY_54Rp32npX*{*XsH;*j3E
zS*n{4hQzIT8l$kLJGaHQlkcO{rno86pV%y3FW?hgb~|%}aiUg*uC8LU{;}^x&+41M
zCU`hH|Icci^0wGwnnQlY7mnb{2WGlQ^6&OFnNDmk@#(iW^UoJ5=-zL_l)ZY>uNbij
zMj39$>*rjm)CtqpHTj@;?R$rl<92uX;xKRJq&zEaH}wKH{s~*Go~M~{??1ZqYu|>i
zyUv!TrN_y975utdn@i``RPm>Vt&Ltkbl+{byD|O$tnUW*e$Jg0KJml5^f}F^vMRjl
zwG4%}`!$Mb$m_Z_h0lA*A(U+KLUjL}y`rAWET?aY(P;}x?938MTpwpGR3=`1Nn%F)
zc9-K4EN9}E@=xw{teV4KZ4;RjeExlHx5i)P35%>sI5x#?vFbB=J2A;9HTTYzZQrJb
z=I*&+X#CYF@B55TS3brYeu_W&;?34VX?~+#wfknVMw6_5-_~mRRxxSgh2*XIA2r>}
zoffVP+?VOQ=cR6QwO;v^9iLrT!yZ)J-ykMx>^(Q{{uYsi3(Sp^k5<oru&_=rz%*-#
zmqolrUYtYhiwmjmn?4_4e|Vt$)yHS2!!-Y_P+G{ht<$>s#`&Al|NWWW;|dbLo%0m&
zy7cKoar#oBLmp+$??0BMOsn5wC=qsX((Wy;#yd|P-;s5zNk8^@!<qAr`7wV~Slpd&
zR_@T(y?cJwmBJ5>pSJBYIlcbR>HUd%TeYV&8>rvX4!Hi>MWA$FootZ&R>75zg4)Gr
zOctNLXZoR2kLKJk2>Z`@^R)PpMSqtx=*LYsXe1_=r+F{dO1U}GZRLg9sLOk!trxev
z-P!cZHg3(DEQuhCzGP{MUoB$(OBuHLhrg<~{dH8#?9-1`KV5})SmYjcIMa2BecH<0
znLoFA)YWZrPO+ESevD)LR=!$u)fU?uOs}sW%63?BYL(94mon2jPwU3|Y;5w^zjO6}
z<sajz7w@HNx18zj&Q?$9J(l}|-{_83{c~m&>y}3?|5P43>DfBH`#SmKl@6Eb5%0?U
z$|tw%e%2HcQ1ULYM(?i1(Tel4X0D!MxPSNV{oBJ%)vwE4`udzX_cik+@7BHJQrykh
z!qB*cMNDVLfd!77OzvBio^A+r>J-)tn{vX9VRetR=C+$2X~HW56&$&4xGYncIgy{W
zLZl<eK|6YF*2N`}UQSJOtXY-bWD34Gzkq47h)z~O_xiej)2?a12uZMwTK(&*@P#ic
z8I}iZEqnUQ^!2qhfyc{KtFPs_Zj8>|y87rV-6QR5qArSR2(Mr38~w`g>veN2ZQrfy
zE(xy*Y)}iU{XgAoscP0$t+2&m7eAl2&<$^%bvkd}2}#|I1%7|8PTCZ_ykQPg%_h4a
zQR#2yaHYKVS)_1)g)w@`hi`kOe@uID*qleyV5vgU)0k=I>HliDr@vrYQoOM4?yj%X
zk2PoczmTmt_~y~Z$<Isv?)mzK>DqSv_4BIzHB&q`uB^XbSNG@p%je<^f~x<wT|U2$
z!PAlR;C~h;p9M_(A_|U=W12SD&AgSVs>Gu|MXB+&O-zRwYuBz`jm2@&fsBh>L>gSg
zV!upq6%3y9QA2ENLeI)o>o&bw8Mh|ujC|v-m#i9c+`kHyl!YH%6mM?wS#eU0b;EJ-
z>YR<6=FQ5=S}dx!_su3zbC!bthefSV>0B1GzT^|E5FR&aky!nVPl4`L$0S<CmfsL;
z)~Y%p;d=ZS%c@q{up1|hkFR3d+ZDLF!$s(ln`-ZrAkWq<C6`ne-+JWCpDOa;a)7OD
zs9VtF7X{o8xi0xCJYl)S8^)M4H}smO+6K|<(fN0ix_Fm8co6=yPq>4*@z7#DYrQSO
zVv(scOSj$pJ});c(r?P!rpbCc4ovILS^HHi`r8rP{yT+7d(T@mMG0#hE$zJZXkmqN
zSHEMk#G363sz2WiR|@mUOcrVKkb0@owB7ln;tUpr70YIBnN!|<E6jPib$PQ?V1q_h
zX4lFycE2N(F1qM(uTk*)d`f(K#-eV$%IB;8doP%oGVP@3%)2?Cr<uLiIKjtwCMJSA
z?BA!;3T0snd5l)Q+PF&WKpWrBRSVZLX`4*4-n{G8n#+qk)&weT$lZEPr);hIt+ej5
zIX9U+9Jm;7-FusRk44BY^uaT|74P=M%5J-wxIE#V#ex5BdRF1{7r)DYwM;qr<LjQM
z>$;ni#50Pzj#dBp^xZ_hzxd55i~b%XD}LLB8y!P)C+~U_{P+H^D5VSq=G6c14>D!W
zzgxzTA1<KWoU`0NaltN=6^Ekk>@rx!Yb_GUq?(Z&{^oj%((J7wtM#j^x(&oSWG!pw
ze0En_FBH%4>Fj=!ed@xCF3r<iKj-IZvwy6Uy$w&AT%Klf@z4L8VvSW|3w&~aMKUm2
zI2d=XOwXObB>iWh$Oq0P;*paUERssz_rWv8<3)3V<GJmQ{Cur7Y%ddjT0|WB$f?;R
znBpv=a$%*UsEZ3nq~Hp_sSJ`gFNAOv#4)IBXyx5=R;gP1-0BonhWLYA;qG&o*5{SZ
zQTxVa8N(n}YM~(WULqy(w37d`J;6IqM7Bp?Et<A<o2%P51MTQ*FI{$=da7H@`NXC`
z$3y14YJ;E1lRl=C$xcoo6UDYouq!I$ILa|opx5Vwl2jn4rr#xxNjZm9Zg+AV*d{F8
zezV0O`oY4SP2nE?(^j|yX0Y~6TV%SSqg7{x&k4=iK>q40k90TY%v3N``QI?}?xwVz
zED!YZ*$&M8nxD?Zxs0JMazf(2{u!(@Pl}m66n8A-4Zg6YBV)<WlNT$N%71ymYRloe
zV&#P<+5C%6LS;^VEiU<0NzS}km!?hT+j-V}??m;xZO;Bx_ju<E30|{fy?*{6qhI08
znSRgLG*4=BdNKL<FTP5-_MK;KOKjdX^4C04ZmihaC@OKW{}0EO`FD2aIc&Hb`%d81
z?pp`nC9qdbD6Dc<G>TZECNU}Vh=;3l*;bC_ZVM-V^$1xdv65rO{G!0l7Pckwo}LUI
zUyn|n5;Z}Cv!(X^=}A71!@Kkq3+jJgmN%-5bTF^ox;lY1d-kEIu#KNCiCx<0zDR7T
z=9m93U#y+dq`g=#R8!@uX|Pyfh^^9%Em0Sk=QP@0NyrqOD#Eu?e%}ojmJ^YegFcqH
zT29H~zM1B~-~!j?t(QXAEvV2=`1#5r_|74bjVA)$Sae5Tsk)ZMc`C}N=yde;qGai_
zuF<#8ZQS-~m$Iz(rfn}-6O@8hbjOH=dDQ4;DE)GJSEMOqz#*EGpi-%Ks7qRKiV2VY
z);qmN&Sm<(SDT)^lx1f4yh%p78+|4eEB2^<cG~>!=A!9rg-6d-x!q5>c0K+3)0=gF
zkKh0EP5i+BzxmoH7O+N3e4Q1=&2=@*GLB6@H}lSuXA2`fp5<x}p8e&8d;B!9xb$zU
zH?An;wYnK<wx#g;|D5fN4J^0R{42KoFgM(GyXSM;_Puv+<p1ND^RQyh+#`}#w#5E<
z=OU72*%9Pk;&IMzlI8IyMjMWmtiQ>0I&AvIYd6yqB__&y{aN~y{pV^&^`moVh+bJ>
zB>tqFr}3nQ&ZSGO6CYL_JAI)w<CM+wpDoKpBd>F-J0*3m_26{tWYC<pE5y=wrRV$2
zlYCc6?Kk-O#wW{Y@{-5V0!%^zml(}n>)lpS^HzVdI!1MMjpg5$anqc*-xeq&`R`B`
z*|+OzvFiMZN7%!aQw^?Zo|)9bd-b(5<1{&2C!HU2ZW_v6ta5+ynN>SVL-~1Qk*md?
z%&#7;ho4riaMmfiy^WR0MR^_99gqJa+taLfyTr({Z&_S=Rb**~Yv&rX6CbB>OgO(n
z&Lpeb_Q$`lz2`mjYTk=&FA~|2_xi-`1q|kj+%J8ry0d*<n4UgtpVQ$kB5BZMdr5&K
ziBr|PYkGTRc+y-GzbQL~m89qKDJHLQ*X|Xc9(jseZ(Gj#vk$oMZ;-k$?dRM0^JRJu
z{(aq^?s$Ivh0b}!3#9id9p2Ntd}3sYImh)CucWz5cW$1RQzop}9>uDA&9QsN|Bk~)
z`+mhP=i3zL(YUexO`g?KciVUCigWemE$@Elw*B6f$;T>wZ?paW(tN*p`thqjZ#&3K
zDZY=pxMgYWcQHx(!1HU>7k!-RKmAqm)4sTSi+BHC@xSqqOxj--{<FK8<2-Nje4HZF
zp{8|Opv_}q?(@6PzFIF`_~s|i;dxc3+Bh%#eOGy2YK~DpSKqeyC0n&#{+lcMbGBUR
z%E|R%o6P3_oZa^C%i&$EffBQtyo$7ASwAg_HqSg<JC!SelRGDh#rj6g*?l)w3oY^3
zDfLffOZKu8c5TP(%D(A7Qa+}#+eTF-;i#W8SKEo(8cLhfS~r`$4x7>uUc(yUl6_3-
zRG}aD)=8{e`*=?5<LFaUk2VXiy`ix^&zq&#yPwamt%lp2<x&46hPX@FaXTLGut?IF
z;?4Hf)scm5SJ8)rq(wUnnnXAj?KNmIIKhyUl&tZOCxj`%?b@-nfd2vyE10-Kla<UA
z^t0O1bl!?Not^fK`_h}!`<T|eR8!(HJ-eSLX$Ftel>-yfcNFlj^6-}Cai3stH&C`y
z$eH~r@z9=ghd7ohnCQH*;W0kJr@Jci#EB-6B?smlI`($W^Aazfs&i~Y$&cqg@ypXN
z%zO84-jB3n-`c__9Txt*ET192pzLvhSD}?kK%&EXf!&KgnjJsMaq8rqKBcM-?+EtA
zRnP6D&N-U-fA;fIc=BBQWRH05@ueJ#zOX!yP<JZ7=i~fDr&8w8ty%4Y&P&>oI=+{@
z4_WcNwM4^pNucSJ2mL=Xltdi3?i>(%_uR+gZNoq92AkY9cP_5B<Gu7CP)RHIXUzYD
zmp$If{XOpcMO|LqmuLR^hGQy<66$_WTr?N4wJN+>|4(bf3I7c&3V%y_Z?wH#m&L1`
z$a_6!?X?MOuX?;tHvZJi#`^XhziWU?&LUsc)gR5)tlfCS?RLY9cO4yS-7C~eT{oY|
z-8{p?a$ZON4PK4Aypbva3lAw<pI8x{!reRV(!E)e?w?r^!@&38iG~)p-h*F0N+&!`
zCn-#bNfXt4_E2NdqY$NC3$#@$Ht;IEw7+rYu~Cb`Ed{~g$5jQ%(%eak3k*!(1u1!~
zD^W>eI;FsU_6Z*o$ILgYdHIW92z`EWXu|U;p3gtM=K1dZLTw%Ik`+94o!nE;&FDFK
zd79r0mIq=U{}WIBQ=0wGV2(!5>pgjUPq80b;`XLT_u#%DGxMb!>TY5;->Kg_XT~t$
z=#m9TxA7ipD$X+yeBX6=)2lS4f;I2|y`0bG{(<9_Rl}JNMt)}9n?x51Tid5=?E1Lq
zUu&7!sl|tcKbxE?-SEcx@MNpshfCe^t$g3B2Ap4?<+;q`;JFeHhIwx{#;w13!0-ao
zTEYLt()~$Ni5(8YPF$xNez1A`5}WdCPQRmgiI0=6tmj!#uLf?j1SXG!)t5Fpo?fN;
zZ1JCS{@Q%6O&eVp_MBJf^L1@~axJ37JuD<J?u*By6t|DB{=UdsSFxmV&#{eJUd!|J
z-Bh&OPIxlhZ42cpP>NXEVens@(bBuKhILC5tHm{64fPGh2JBH<1yNf<m_PgWY$>^)
zrp<D^gk`Cab|LfJj*z+z?xh@>Y%g2mQzkeHsIZ^>q;k$kiQ|y6OVi%8LkdxiyH~U;
zdFW{TFw!`oVDOy#^1T4w{Q(DVeVFlw+tB{9!kq-Jd2cSCI&k@vLdm-YymuDxSTu06
z6gw%0D%~oW>6Vz`(V3C8<sh42cHGtEgi^EP-b#NOMJGMTb~#hRc15sQU#Ogkp~i`G
znZvvuQ48(Bya#OsZLQ`U?pzju@9hGG^L>ub*et@XAI*QtN_B!LC+A0*RVNb)PqJK9
z&q^`Z=r4^pc*=<B^sOyBzBPZYDm-)Of6kd5J!gbmMAg?VQCK25ak;2oi%3|2l;9!z
zrOzvuxjRHBCvFs)@=v%;WC72)i9Z(UEB;FlR1I}{+s@;4Ku(iM;VpyDi6gu_9)x))
zD4kj&YL>u#q=lvF{oXqXTW%O6|14C<Epy6E<UW?LwkXBVBuK^FD5HO^aw4ya`J{}-
zu*~AzOv_8I(Y>-^2Aj&b+E;w6TH*U$Zne~ZeTK+0*^9Rxol=rNWlGLrU3F1`@QO>F
zQ<U#E7<DxiX><nVJ>O>YO1I~bhUV!G&GR08S2AM0=*GO`usfHm^*Tdr>!x;zbzv7>
zAKg0h=#-G;%g3rxi&VY(C*3kymb6H<vb50EFChNEvg;0oz_$l@yWTw)`dq@B^!(HU
z9)Xv97NsGl6PV2s8rVLU+<g%2+|DD=!LzD^+3bMwsRSmA1|E;C>1GYwDPpO`&-kJq
zmLE+qV86^$+4k~}_6!akO@TGVw#-@EuFq;(qH_IYEXUTj=R9lfYI4>TD^B=oQTN%r
zSHoiBVT;bgk&TNZ+LA1p`J*?QE;x1g!=bMqI(!<Aem$X-uc7u{>}YV~abf!lK8@D?
zUur&<g{-pGO>$~z*ITBQkh)6r--*g!m+f+2m1L}6&N0O;?^O%q76)D@MvDhN7d|@5
zZt-n6s@(KWCc50G&NXoE8DG6A+#Uzo`?srlHMYevMx2>=X~q9!<`W5>z5Zz{3k;mM
zInO;W=JC)i?CiCet*$%Q2Nv+UHz&D1e&wDZ;vQ|{mMrowC%0SJo8eePt54B#x2LWv
za{E?K3wiV|d%u`P^CO1oA?mZQgi9GjEc1x?l=<<=H>S-V-5ahj1#)Z3?~LBSTJYC5
z-~Ga*?(V)@s@h8rq&{PfOE=j0$b>C(0$XSLF1@^6F{>2RG?g0)(*Auh$$D&-epBJt
zv4GRp71R`5w<TN>DGi$V&R}T=_pJt=TMKy3xRjh}VNq-FIhDZW<X#~dVtVU<?}WoM
zR2@Rfo0<D=wljQQ&32jBtbtkOfHIpgckA{!v5MTvpQle;Suc{2Y^UQI`(LytdFwoT
z&);9P%>2_t1SW=y7RrSGP`H(89`3_g6Kt7sDYxfqWdGDi`CpNWQNr0hQB#YL>;0N9
z;J$!sRiTKMsOs@8TI>z&OdofgD4w<^rrC*Wk4oH|=EWRLOI)~?ZSN^-`WTn}t2A$x
zc>Qw+w<HalM~mH0N?KMhJH5I2?sMfko+^Q9iOYF!ec)L&T`DBVg-gVN(X8ReyL4mr
zOT33vu0%0*crOq+a%ASj=`1xGJ{NBB`D_z?lfbREV8+ZHsV5)kvw!YrE$2S9z}w`i
z*fh!1=661)61FHMt&Vb>rTn%ef7#0|7p|l^F}Eyhlv2)gzSOd9#_v4e>q;+Nsukz#
z`EQY7zdO@$cBZ?v+PxW-b?dVP7ilbc`(#tu*8CMAn|v8Q->BskxYLnwryywiWRCEL
zr=6G9=HCC7yI8sV#kIwcdlx7~=Fb(<nm%dLTMib^=?{4Nl#~+8tttx-Z&Lknl}$NM
z_w5uN_eEb=+Sua_bbn7#J2l&xDby@~rczJ}*9ny_&KJSa)446S8l9I4ZjCH)xytus
zSNfuv2WH$9J=HL$w?eUG`m9t|$I>$9zM1>$E0Rt->_4h9y^O8=XJFO!f^-w(12%V_
zZ)xC-C>8zrN;@M_M9ulA|CV0YhaOhPsx_C%S-93@N@^@}PimWLUmI-RbM<&MPh|9)
z_Bfw9L&K>3|JO!NTpXdJZTa7?etvSj`L6ooi<`8Qmh>_=Y~0+??!*+-{#9bf;!QdB
zM`G+x$SgT`+5Xa>B_>=8zf83jc<I<8m5`dnbfVLK@5UoRJM?4&Wqes~h0PEw-nmwJ
zt-_qhkn>ecZINZiQ&NvVk-7NG!Xsf{*tS$(vzZTm?eA8u=#MPx-oAeO0^YnCELPjr
z{+(#RId{59h*N%ES7qnS4g6-N9Pa$wO?<4ieQbtpDWXRh4o5hOgsc;HbL3%I{P)t2
z%+ovErmFPp|B?CJOQELGeYUTIEo%~Y$kyhN?Bo^M>9cQt-*G!fdi!4q#*1Z}r@Yua
z#d(W@LDx19@7G~AJ(IRB`mgG}l6jK3!8Sd~`}a>M2s}`bc)e7DZ>inQ9d}IpxE~e1
z7MUu1L|KVR`M{dJ9L)w%iqljb7+rYGb8V`v3Ihr)&xbP0F1_x{Amj9JlTV9tuyM#-
z)`Xdb-#cO#@;=ygKvhiMac}$T7i(5l{xGmH7TtPS{Y$zL?~8i5pwj`#zh8@ed&Ixg
z@9>F7*Nq-!RJ6#sHJ+WbNIuI^*v(OdqggY<QJ5>+kxf~d?Xi18&F^<VclRujD++yU
zVZPw!oK{~RQ>!IMFPTU2OP#piw(y|YqVNAse4m&9%kKTGJ)v8YMY!cpEliHSyEl4m
ziIUdw$zuN_e#8d!A6jIP;(Pn*sW_|CasS(GPRFoq*AXw2I#X1o@Mh~&-}$!nyW*K&
zR;0XgifTOe^YW^tRx5?CePg|1FUoM`?ZcNlE}n~h{Pl;;1<?d1uZ7dM{(2f4!C)0T
zv-y0-=UY$Nqtk@{u93gP=k#^PM?qhYX$qRTJi;LjBD41!<OZFLZ@Bg$jd{g^M=4ip
zT(qw-_}8U~x-8tV<DWl=@{QvUT&6e49%E3ru~sT{^?duPq_C^2j(bgEVBCA7Gx7TB
z`3-dv`roQ<<S%yXle2DO$qamQe6m__o5%q_FCkV>6^+0N4hPT7;%C*U=m>UTV)U$e
zCvr+Stx1KGNo9gk<I2m+0+#wrRBCwTtSMZSR=XqA;rjnI3Cmk#wG+B;XiA^sa<ER#
zy|qc^7FUOMgLjNx+3ROVXB(^E-&MEMM#N`^aNMC%-p8Rj4I7j9^|M;59bRepz;Bx2
zcTolv^(Ky>sEI<lQ4A~3u3+8bd3M&<tSjrIH>VlDGMVX{;&!)g#_XPkJN(fz!}fYQ
zXh$FKm)<{1p^@eJx!?c(Xzn(Z4BD2H+xRbrDeKkBaxIO=r~0&`^UiQ<Oup9piZdnV
z>RpFtev*8`k0xmrFAtS8TD`WcU3>c!_XXO__Y@nN_<1wj{mOQ1Pw*@E`0>D5(2OH7
zizn||Vkx)R4@J%B4UhUAHHuWaBo<u=>bGBQapaQ5EDi%Ndmhe!3;&%Bza2iOv3gEK
zzgjYbC71S>IT77zYpxVu>{M^kcseEc*JJMBJHF57`dc=t$7Ve!Y!_j;=ee+jB`{#>
z)S02ap;u=Id2V!^6Me+;c}VHDtp}r0zwUf4R?9SNR!q&d)E7}PJ`-Oo>eAZ9D&Buh
zR5NJ`Q(J_11Mf89W&DM=rYEdu+%<E>lBldKzqzcUSDZbk+~Cq)zwMxx^pXt=rL?zg
z3Vgk8*EcPP^?TS>ug*Mh>s9ndOHuDr8@K-3_9p*AlZ15s;;q5n+H1EY$83u}Qu1tj
zE$`CH0gtBbz8P>MH*vC$nQcm5w)MosdwZU8J~$>O&2gQJ{nM>WY~r~$lp3`2wmfqC
zU%Bm!_6KK9^*<9DzwoV@S0mzYQ{X2OKWAg3RQaBb#}(?&6dqfZd$7%3J?RO*`)Yxk
zN%mU|Zu;vk>Tx>nu+`&8-=sxrocMJY_c#XHF5VL1?;3C|IK|Z?z2xF$zvv?quc*el
zl)42v_kO)5<N7n&N555f-VObiB2%V2JYAv{V6e6KN^pVuRgpO*%fC$xi+gH&EmHWy
z+w!P<U%~4Sav8NU<}a$NTD7$PW9f~=DqFuBN$rwq4&q&fi4m!D67J1-J||l2*NaVx
z3J2BJPcpmZynNv`tEsa$2F-o5;-Xx&-?oR}X20A1vutku7Ov;g+RHaK$rNpUvEp%=
zVc~aG|5!KvzyHfqZzyGz-}jt<IpE37CWYgxrha_zgWYO}LZfh;M#FcmRq74DKb-z<
z*Yas~{{J?KNA~sq{@Clx{#Cp4zHZdU)<B1o3Vu~5H2Pe)12;T!7hmaq`jumU+m<t~
z{}PT$RIx3QQq2@$@|eWHm!Yg;%i_iF`5-OihPYVR@zza&3nxb?w(8indp|s>n3Zuz
z_4$OATAmY{M79Kou41{Ul^4Yw`r^oPo2jl^{yH)ub0d}?mhJQBSDdnF(+mw4$yMF`
zlTI&EQI)tqr7$Q&W9qVNGgg`%^7G0{31teJDy%tyF?#KVtE=0Na@`kLU3y({T2|dr
zt2D;fR=bj}ISc(<ea)~aqW6EwwD<F#9)GoF*`(_yr|I5mboblV#H0T+Y<G#1tlT}9
zDjS>Y{u`9F@*iH9!X<FsdCG#xyEJm*R!sCi^h$MOdu&F^{y(Y5YhIr2y4pWOV%7XV
zCSv>JtR_#F=2kOW_*3C+&q29;4uNL+hqz{%8n`7_T(mkeL4`BRiAQ0|f{Qv%+>C1$
z$vLcUoywD>+99K>F6!uM7aBBa!jEDVjgy{UEW3ncJsH*BaH$50-J0B1mZ@Ry=oK)f
zKxF4p-n_g`r)GaRCffJv?c!SH(<0|4P20EgXh7rQ?$}#N;tod*3v)u|EYn)L^H2z%
z{*=ZZKh@2bkA2x<G-)EQ&=P}Li@4G)@_knQ4+>VczBR$ge8<J!km6)%0X_B#e#5vT
zma;3pCd;iaWb)oxQsQnDz+QbT`E0<B4|`dD`q%TEK6%K@B>Y;*i*+K4CUksSwQ%1%
zeHY$atK%J8t{<!m;@;*k@w!ul3gdxbkwq7dI3=|?GH&@iS1dE=Qk5hBaVs+~&8Srw
z?yqmW^;l48K4)IW;V*L*2Q2krQds1@XY!FAzoP*~i-bkqt1#UAbgiKARd6<+>m4UU
zuAc6yD@)W;wPtO*7(7o&x8uqU$*`rm8VfU9*Ce@eMPBR*+g0UT-f9}>GATT=TO>@m
z_^Qq~xk-~QZhDetl&Nczw_0{VOP9v><MHK3FXg=B>K9@7Zz@u7)#w0+-F1;S8@$=9
zKd(P2qjsVw$l$xnni)dEktfV@eE6y(4j*{)q0(z%YuJLG7Gq&oi(;WI=ZsW5RU<jB
z_)R&hvDPPR_DU(YUy+kme~maE_PFEPF0~ZJ@UtfZKDRQ|7<OG&Dp5--o*fW2bz8@(
zUhmaK(M${*gu9(o)qHL`&3&=W<=P!y9R=2v_no|gwjEK`ZE5CSc;iRVejc`vN!2k=
zzU2g;vRzVK&o=jk$#scSHGL&(J((mEShW{#%JVuh`CZ29SJwBAZl82W*kQ@`TU}37
z&zmIp>dx3At`;=gs(RI3pM5vN*Yn+6$eXL-Ed6BZnQu>@h;4ty%f7>5-+vax45z=z
zy?^^6Ws~-cHXPe`BmQCg#tVBm8UvF$-ppwctnzq!Q{99kJH<m)V%xFX5Bz+0L?-u#
zEihY??E7l>gh>nzfv(#uE#Hb{R9f`j+U*;7#$0r{h3ms#(%lUCQ^eWzJ{U|(?MYT~
zUh*R6xnHi=rK^pZ@d634Djo?YJ=3Nx`zdiRHn%3c@yJz$QdgV!n^_Yls<@s!p1pdi
z@u&VRvzEMHQu1j|%yQo2AD1xLaGql6C{j086u%J6c{*s)YW*!fyLNk>EMa7kJs4~u
z(R}y2fO$b8b96<q=CX6A_X+syUc2G&KD7;u4SV@(&QD3ZwcOHhYw+fzAJxBaJ@<16
z`thGBRraO2LtxmGhG{9OQV$KU%yXG^Ksrvf_V`DR-%ZI!!`Wwl+u&-Ob+OMSuStzF
zwrWR6^kg0JrTf#oPx<e8{8YnXsoqh|&wJlK3pCjsrxVxs*W-8CCCwQ}mt+abc!!>1
zoL-b*u>X9>-0LrP9Jw>4=)`wT|LN;%<`=zNzG)(pwp8q_%X{9H3QX6}S(Uu{&R4dU
zptY|uTZ(60_4!{h_nP3(-Ir}-z8S0y@-n}*tau&s>lCIxmi^LfDW5lq_wWc`V0+G(
z9rHo{T_Sg?15@;Z0vQFny{4>8NeX$Q5p|&v4XiohH^fDQ8k<TR#Vk~}EnrRgkfSlB
zRxP8+Z&FiVD);|40o)skR2&y`mM&t=WHMVQ7N-8g%l?X3$U?@AtS;-fHZ(D5>?n=)
zPHCwUQWsv(@@JX)KQ@hn*TthkG>%=>IG)-X{UoNNQEIA4;Y!8Ae3e2E1%_k^VTtdt
zn=i#?t|-(gkIS`?$y=dQbVH|}#q<4F>yN6^mmlfOYSF#*sC~f_2A_|*MiUeMi>5wS
zlDqvm{(Vw?*N=_`;i60)`mX}Hnw{Cg+Ut{+H|)Mv&1{@);m*|LZ16*<OrW?-NX%C$
z*-+)N;S`Oe-0fZeE_WUOCLNu?T~Y4fXI$a%EqPCe=^lxmy+2Iym!vQ;R8DfQT>r^>
zwnAE|l6gXNK*C|ORZN}#lbMCX(^3PM^=38qt`cHc?Nr6&P-SY;$GlDFom-l;sp`7v
zX*u8ea>Xn(LoIlpyOpt7{?M@OncP3wS+gtAZU4vqX`d}^j#{=I<7~SorJ5FUt=*b$
zX7kPPqFo!Ur(SFARgp1G*S_9lV;RW3<bo`hLe}T<tassAE5asyc&_*1VAivR3b_sX
z(G1zC3XD%KvPCv3qy&`x4P;}jD2=`#|3k6#{^Fc<Y)Yq(gbT2RuPL0eVxir-Z9bop
zxm7k9UwBs0cfDNqid66O$Q6ZCEvE@<SyWy#s?`!Vi8f$!HgRzbY2GaAVkRN{m}9y{
ziSw7~c|l6v6K{w!Y!C_A$eQ`7`G1svaDAn8=jN)=LUrb2Ey0G2UK2$3AI)#OoS!yP
zIQWEXe96o>7m3Ijt(hVHg)9>{I2Sq;iX<<H?KxI>w0V~N_E|d~d8~XnYnh_wjr8{O
zPP2D>w*D|Rq;X2|+!lA!C%PMt=s7(vak*ZyWJ%?<(m9tWd#!Jg3UZKJZ()#X&{Y<|
z_92m-;e`A@qb^kgR+R@_xejw<803X6B@~#KUuZGR+dePn`n(q>=kZ@;yHuW}HMu<O
za`)|&`OHtJdZ|`UYIi@qi8J{^PbX)u@5dCs^3*LZsb14dZdeEle_demQ7`dQhuEoj
z_lXRS8~UDkR25WOZC6YyUR?D$BftD|U<_mRe?`&i>fq`$#s0QK)w`cp?^`ZjUbgsb
zX+L9UFw@D!cBfb^kJe1QSTpsY)bhnvznW?n%xqq*Shqh=I_PN9<Mxol8z;VO*IgFq
zyu3B+Lh|&R&zCZ86OMVHaKBjb%|=%Kq{;VJE_*(4@}6gtS6`HSFthZlpc3DuM(K`5
zu^r1lZC>v2a=EC7qPwL7Yl%a*s`Q_eQ&$$I%nRk;7r=kQdD?bEwkU&U=P7E-o=;nT
zbz0`+InhGX`4uOrc-GHxpL9tzuvE$A1Di(1BcY9wnoULt;Tv1JSI&q?T)pL$=<cWL
zwV$fPO<Iq(wuV>4WKL<dv?}&q72+^~AvweShLQHB%d_rXjs3sMaczb~+@)7-A0^xG
zShbhWXkWXq(9B$F&r1E9m$>$83cp%avNc4{{ds)y=E{nfdh?cdJp4F!{SSpa1&c$4
z%ibw=?cbiw_*wc-5Zk6UgP`XI7bSf+&nSD&Ih9*_<A#c|-0xv~P3&G5Pg8G8I=CZc
zWnzz9cF#od$|OyxY1>&@v|QM-5;(2AEgik}UT@-^wZPcgo3*W%p{*mgxnu6D-YXTc
zU%FDm!+NVD(>{t#e{`ZJX<462){NN~`vO**XCD(Y*d<ot*itH3WAfCZQq1ztiuBYe
zi~pWTcaYNMkepE0GQqV=%3}j7Pi2VLuZ$w^rPrrR748n%c3IrlbmCrS-TyOOJR~YZ
zZ#UV@ZCkqRsO;yiNh_yKdiULCcH$%(6IPv{+2t453JVRsH+BAH>`E;#NXg3Hcg?Q$
zS2n|w(#22h<e8=lhILCAN9;E<;A)bIS-@&KO-YX>CGlm%z6(wIC%6oEuuj!-$>B81
zeJQj~Yr1@u+T#^_=HFU*E_>4Z@KvRp^$Aw0oxf!n6kLKYPA?F0;a6;_y)C|xOWiv{
zv?p_)*M=D{t5<uzXqh_G_5SaDN3OaadZodUzGlIuf?Srile;82CTMXsSlwzZTy=BS
zEHfFlIR|cd&CXFMni#fLce{?lajQ2|<4oIUpWss3Y_#s!M$d>{GAtp1=|Yp1mm~zV
z&H3*lzM1pdW}kGCvmX!XztNLsFnqZ(c9)7Fm$m`NX9XT7Uy;}I?EcJi;F<5pGj+!_
z`!7L>{~Xy;9!xG?#Hyd9TC-hp!t$QC0>z6PHfd;^rymkYcvce4v{~2sSW?U}{pw>L
z-#3{{FEo>0m?yKv>b0ss>B8V`hth-fvL~}J2DlVY7IrbIVrC1~uwLM6D5@N_s5Hbq
zx=}R9TkHcz|MvEvR%h|zjn&4tEGnMO<PEP$J95&NYl-cz^aVx}W+zL|o~SkR$b^tv
z`_*m~UHB9d@^#7DVCmC_OP41uy<Vhy|Fz7D&{e@klm0AaU{sLvGCXs*W{ZL*qoO6_
z7o$d*rmp9g75<BSo^t6-qsR_P$w!JpiyFli*)P2<^)%67=>xV+Y)c~B#e@tPxfYyT
zs5WD50RK0Jm3v<AUZSwzpm+08?>)b2qUPS(!+q+!p`i#vtuv$SD(gbC)uvgS824VS
zUd6<~;5uc7i;K(MORIJpvvGGwZ#y2iJ!_xWj(uBB?Nge&y86++;zp?hUag;__j9Ok
zTV_-&c2at!;N?HuN^=9-#HU+H+pIO%-nRSDWxKnD_in9qscjdUb8z!S_qyA$Cmkzq
zH|gFs?0Bsu{q$0$LCsZzH+9y#^;%BsNm|43v&egybh+qnpI4q66{aO-$tGo=^%3*7
z7Y;Va4d6-*;PSmwttx81$fe`IgZZ%=ta~=TN=-KkG>N`poPDf4_eQJk=9HdeYZZ^Z
zmAV;yciq=jo7tzO&OL6n<Vo+G*j^6b)Kx)&-Yn4zxuyp-NHcZaQoOarXLaA^SAD_1
zPlQM>p6t`#_PDd@VURi3w!bI(Q>?d5Idrmtal)R!C5E~TBH~h4u1_en-hS41JO9a3
z)zK688%^v_)1E1|<A&3Y>q<6eDtZ&5g}oLEKi1WI)L|R*Ye$dts(IRt5*7A;jdRv=
zpFQhx|J0UU!Yg*Y7CtAsP5$@77?}y@Udl{jS$F2-9My%tR9~1~_?XuGxZ}(MsfT}L
z9)8<i>6oY&GR-z)v+#j4J9yXLo8)$UP2iThN-qB!oLo5dSOXh2akXFETkN{gD*wf8
zwJj~83cQzsr|jFy>iR)?|CFDPXJ{5U*{vy^*gA1h!Qt{uv+Yk}R)nO!*mh^D`}&i!
zSl7&w+b+p{CoZd^=qyv5h)vNfH_x4%>oyjLT&;4Ks|l%i5_(|e9^tO5&zVx+Y}(Tv
z;q71Ur8?)(ztZ~X>j_pqNAk??JN3+yC_eJ3dFQRv4Ji&xx}rfof|L67<|Nh}n{Zdp
z_~8ProMUFMH?_Q7&~o?Xk(Ub$q7znLTwuQYWs1(DY@a!iVP>0HZrxoWto~*PyV~&u
z*E?)~??`zh;`Bv8LvQ+S;}Zd*i^2^TwdX8apSy3@u0?0oElL+W`9Hg{I=}E-(X_3`
zS~auI6qwygpDet@a_iy?L2lWzH6Q;Ia8O{3Zdk%=xxKb<`-aEwE<HW9(I@ob^9fI$
zPngnPxATwmo}RD^)mu3A!n&mE56a9JvX~;VqVd<{T^p>g%|1Bgw{F>!H*CKH8~UBf
zA~v2=?A)>N+k>;*4`0~!o%>wJ;k(KyJC-LlRqOp}r`ex$-hbk{yJNA{=kC*|`fqD>
za>gvZJN-eowB*`?i7QWR{&jI{)EDKws}GdA25~&m^4)(ptM$~<*1FSA%<hYKeOt4U
zY3;qOZI@=X&1!2~X1Ml&sLm##xYwaqQr^#!uu=80^~{-2eCo%QB?^avd2ePdb)FRa
z;eSQudeMEt(e8&2sU%5HlUnL+*I{@4|HbOP+grSMtUHw>d@%F7srSp|Gg4FgzAHGr
z3d_I2@3nc>{TnmmelAYctI6-JtnXc}e{=IavzfAH4%`XzwwFA+D^sAnk4b=$f8Eiu
z5~2GiA6@)bWbNIH?^dL3I-Az;I4vh}>(Q@!|AlGAzTO(cedpr+n4{qm44!xTy+b^*
zWM1>#J(s)vBJcZ2+qN%vls;}W@h-dWt-Ug`F+1*_eZTP7LygQ&ecYc7{I5Rwxbw7|
z{Z$<XhQiZR8Jd_`Zxp32pE}zp@5GdcPHEijn$jvhJ61R}9b=KU<(e>|;ep2l6<;p~
z%}EPRO<;+hR&p}v{}L}olia>NmYbjX&9$xk_hn}B@}~JJnNoLm8m;tP>ND4_)=T77
z$nqf8h@)0PS(8{8SUebayK<e>_1xkjsa<$R%xJdCwpwk6HzKO>FYD$mNa>ogr&`_P
z<LYp|2M2?n$f;%ASeeAwCR<!7Bl#(XamvXMv4WYWPn>13PFAb1{NizOrsw=+Rz@eo
z&DKP1%yRZ(&^FtYd0Wh3)!DZ<w^rZ&b~f5Vx@mj)Z!rfOv*0H?o8#Bj?kvfiv?y`1
zitDneYpd4o;r?*1E2K$9Xv&@thmg)IU$sPcwD(V5aCOO-rJMZUPwAG8E))7D{(x`A
zpU>xeTcb1|D;inM$mr%=_T%*8&;L!L`VUQfm6Wx2mx)M%ZIgisr;FhFS04^>Zw&Hp
zxVmmv{?YGNAqiV|#Y{1DlgK{u(M{I%$a`mL+m?&&Yuo1ZdnkB@<l4=OvGLlsY?_zC
z>RC&Aic~cx?#VKp7^dRSv|))urp|P?OG%S;&)iAj+3u#;>>IRA;>_vwdwM45H@!KM
zy@d0NNWyvczbza5Ikcs=Y-E?deZa6wD|$+|+st1&LFd%gzA-+#<3W?KkN(_q#%Fz`
zdsSFM?lg&f^qZ<1IyptT<LZ^LQm^gd>AHHBX8BXso{cGHUVlAm@8;CqVH~Sm8-lE~
z)-PVr#dPZA(g};Msw@&xI4s)uZo@Uv9KoQDg=YWd9yPUes%^+%Vm<TKL%G!6sdHu7
zLRU?d%Bj64HF=^hz4Tl-?dkQEsq<f5SI_Xx`sBH6md=l>Ey8RTQSIrh3XGaZ7D*Ux
z**B?MxA$REx9>K-+mqMr`*EpT|A5c8?hU+Zc0A{Jqug|!9dj#Mv+?AzhBIk%%J21b
z>sB4($yl(|xYz2uT%Bo?@1EOlx85kboolutG1)bXaly8?nRgSEmu_1T`PJ&`tv!D0
zvNtDlTjy>*c|y-RV0qS3-b=ggt(zNlOVZ3HNJZoQ9^PgZp`%K^M~wC#^4xRDj&)mx
z^1C1*mBSZxh1Lc;ab7m=cM({AYtIoL&m{hmCBfc%w4#>!9CB;=A0tv!klA@;Qpx3R
z6|L2`g57Jr-p*fcd-~SIBY$nB3+ESYy|Pv2-Z72d@Vk3hT%=xmH*Ja78>@T8=ChCK
z?UdPXb@#CwS6LoD9^td>&l}^~FaKqx{ACyR6}hOX5w#`FqMuEw-skiE`u_{GMWdEl
zJ>7q{MyuYMv58^P1*wp^t3s6vLs=9p4zh;{ge~6uz+=@hHnY@Yx~{Vo3fok-C-%BT
zY}H7-ck1xOUYpgCzlyH;CS(|_+H%_e)hq{h6DHwl9!pz&D>*&mj9g}3@z&&7eCkk!
zs<Xz?3!TO_i6WUE&W(oLUCh#Rg$#GNE>>ww3R*BvV6wt8-8HGY1%=*H4})g>-!<Wl
zzH+m2;<uz3J4K!ul+2NId^df@%(p)2^-pe|F9@6I7<4S1_0P;qomMHmb$u#P1->ea
zkD0FYoG|Gyqn}#+X3u*wa&vqh`mXMqthLQm-8j#9mS;?}?AwCl)<-|}X@9%6{Y)r_
z+xN`aHwo&ye%@)fTPHl1sZV8TqLgNdU66mBMak6tOEsqJOQk*UVo}L@q&`z`SDOCZ
z6pMqYo}SiC&-xEVE>7RJZq}O1fp0FnJiA)w<?LOXgRHaD=a<=SwDXzruzSIaa|{)n
zI&q8Uu_d)ldb4n)@9ookv(K>D>6@BVN$kGt@nMNw*FVpIhFii<mu+Zu5?p22J*y*8
zg<Wcr$NyIuQ<)Z|SBVKSS}hG;SQX7=t<-vbL09;yLz=AmH%{oDTAQ`&)ENzr#?_q9
zdNmJmi8+2Mo0r16F`oJMblZ<FbyghVR#WYqE*87veA?61t3M?^5?tyPT0e1XB71hy
zftVMeTis08*tBNH%sn*qV9iC9<Rdr4wtTQP`F2(J{_f=DDGbLa{Z&nEe|z~Pk9NcK
zhlTxWD<e~ij~SnmGMd;Pl9JZR7QC}Bh5O#ZS(jJ2ZC#moHY@S)tUL3LZQYpbn;Vh6
z_D<2_s0}ZVP0oFN@AiTs*%v9tt;-HeR~$NQ{`tzK;sY(R=3ARi$NkeQ*v&Gx`1PSX
z`(Fy}<o~T|@VR)Y+q83~|8H)6_wd<fzxVsrm2VTyud4rP{Y@^fd}na%qYm*sT1Quu
zeT~hlz8QW0t<t;7CzG2TRxZC@?#bwTD_S^4WR`7PTYk;c<`+M;tP*>+^#^=V7Ww-@
z<C3SKKrq9b`R&XS2mE<I)N9LnvK+JMVtW?SBvxa%K4Hz}<wqo}H?9(8Pd(hm|7u6h
zCdGxEo<)a5n%ueGPrSb6#Hvu8S)!S)J|}0SeO<S<C~$TDT&07@PF>Y;zB=#XEUoFQ
zd!A~U?>rrLZ<Adsd&35wGc~KFy(PX^*=}1A*n6M(&EF{u)Al@A+FPr3GtNw5#?fmR
zFSIO-T%^o=Vz%U_3H@q?O8&pLg;?dduF3rGe)&<Gv~QSh_FO){6^?UHuD5g#J7QRx
zJwf@^iw9xb{WKGAo;c7Ibwk<a&{w{~ziD!*MU!{k@y#omX>C!ZlsWT(>$jr));V#v
zg7%&Y%Qs3tzNjRr)$?i@zu7^}Nnc}j-@c`z_Mj)%w@_Md*Q&%k^S8ejs$AV{>l1i4
zXp*jV*7EDB;xnC?rYKBX&k~|HJ7B?2uBr>Bx%mrkc291W-&N50u2X@}|Ndd-)WC0<
zQx><s%2;gDcWH84M6mp`iVF^YnQqg5)U!A``Fe#02R9x+YTjDOX5H-=R^q$DW$wnn
zZmBg1qTJIGva;NFNIlz9_jRSOi<ke0(2|XATVF=m?F#=tzt3QU*Ts_|?%GlPde_!C
zyS@rf-!)CT^T>*dyIuV6FYi=akjPiSsFcrtUtFF$(Khb=dYQ*JBof$`@9}t;-+Yz(
z0l##{l!ZMTjhipu5xwg^*L2q=i{Cd>^{XF+mxpCOTzD&I$+XH#Mf<iKjeJ{lYTB*M
zftR*rY}UOQcY9UX^KF;ZI;B_C+~#`lOxGq=bKU*E?6{<CR_8B2Yd1VR>-}NY_8Xtq
znSYuX5|@x%erwN@+Lv>Fzj?X6{Kx&)vQ1yP<KHbV|9$oQSDV_KF_V1`{CWP|;g{^u
zR-q{i|Gw8h9Bi>b&^(*5r`onB)O16*$;y)tZdmWoFI>UMV!&Z6ZyhPmwfX;suGcGE
zE*i|x)SSq@k#orbVd<Uo9<oeUR$!Ev$`Y_KGTTs6*irhjX86vHlFp8TOQy~eZs23m
z;?v?)_$)d}g7w3Too`=E4Py`%3)-Y%z4^+IUGhFkOPn|BCT-H!UcF9vm-cNVC6&#a
zi=0&UC~2x3TIC?I$YA={$lX5+t+#78>;G|T6KH7buxB)AG&DJ^r@_dm!D#x0=ko;X
z$rjz4CAhn8&akTPwp=}<<CVbv)7|zgd+b&ASS(ULv_jeTlg_aTYg|7scG#rj@=DTE
zdFAv~HY+rGMJF-@-!xgaQfkr;mAHp{9x%8DK2{0(vL&cw?P|lmfULFe995?a88vnE
zMy~$f`Yxk&H)ns?0b7HMds8NDE1GTguygJM=3}qD_a&I9DL3v*71{P$$~>NBPL`2c
z-eIoqJNUm}==!ljJ*#?qUWjp4Prvd`?E>ZPS(AjGUQ$b+GVSB)?S_{7w?^;Zrp@@R
zcz;#S{^b#eYh*ZUW1O2_9G|Gc%=uxGzLbG)!(^t@5)3L9Z+_16t&rT<A>OL37dJ<E
z^2y2FYgTk#KJi_0N_TOOGpD9oqwr)S%~QgLQ?)GrT39kVG;u7@d~YBcn%Fg`N?~S*
z<0^xLGjA`L&AdviWa=uzRYH?^m+U$!bEm0tqPVKbE(tFYy$a(MzqKTiv{n90*O0MV
zeZ))cjg^9r)5g{Roi@xqv{Kr5L-e7Q4Lj@}Hf&Ai{`T3)K*hPOfRWLF*?7unp$zBU
zdrtFNIPW%T)H7%lGHB$BIBcxqv&+W0{Q-l1k#U`JXLsl3PEBQ-HJi7tkg!-duic2>
z{?29x5u1bBHiwP&w6{3eb?F?-b~$dXa>9Da$;$_3DvMrcSQ~M9&pB<~TYLCcFz&sx
zW=UYz+RN6n@7K)Ya<UD-vu5RkwZXf3!d9@y9N8MCz#MgH@5@=T#nNVr4ou-q=++ca
zi~DVx_W9h(gnh4C^wLAjA6AP#l|G-|Y<_L!KE=%QZ;NM~b=>zw!2Z`3|6f;Bdn%4^
z%ihhm!2QSW<3=f5bt#9LR4*`_{=dNT_qb`#em2>|x)WwtTv_b9L2JqlZI<8rnNQoV
zndA|7#<qQt^%if3?~xANhd1;uQf6Cd^LOC^0cOMhGq<>E`7L<q!1-g%jGEpV%~R*}
zY?RY=l=^*9BxS1K6H$pRqD>B4SI<1?Cv!@A3v0ZkSL#ni^@W>OZs_DJ5SaKwPL5|$
zyP|db&!Ba8POUq$NUi12CXvlc9`B88+@$HrxoE?x_Q?&r0p3C{4sF+NHe+x$6lvHI
z<81HiywBt?>)+E(qCN+%cpLj%*{joRtkQAl&FRWH$G>=cTb<?RWN2uN@tItq!TM<R
zi6gs@rp(kW@Y(ap`m71RL&()bN+t&_wrB?M^m+Z?a%9Gus8kpCZWoU;YdjYpxndxt
z$#8H-<JqSlZ0@``;%Rg=@Qq2*Ud=#h)w9xD?`P}{PwBeUu{A8%Ph^+$!>sn_I(pnc
zU89YTr%vuanYe8AgRSv4$3ChoJEG`TIA?CMvHy$LrWrEZQ~v1X__+VN5MIW${G;@;
zriJPizV2r|xBrvs?0zBqC3^q&VCQO)3lj^jFnvAFvi3&z(i=Uo0SzoCj!Pb9JvilL
zqd;8G#OIz33LFg`j7$s(H$TmgFkX4smP^>^)qazWkv5Yx_*4(H-P-z@WB&dF9vzE!
z2&FpoFnjh&pFCV3()YTDlVw3ax55k|-f2xM&jlFjDzru&xHx0Z{{_=O-x54xc%5bG
zlLw*;cJ<7ea+Y1mYrb$pl*Oq9mzS>3UOjgL)AB1_OSV|)U*5GfNo?ybuf3|Lmp<90
zamnk@+Fd(+C00hre@No&a`M)j<KB^Bzx|Jsct(faRPQsoR~%J+4t()G%yM@xPpolH
z$feqlT~lJ*ZywU|>DS<7VC86F>4;_UxVdga?D_<*z)%|_K?dCg#(TUc+b}C{VBidv
zRgU|zoByk{(;uHh5jq!r_Waa5>zcFYc(w_*j>4qo)zjB#o|~<EUV5$PEt{0LHjyDm
zFWi>8qC7k8?yQ?u(r+^MdbQZ@6zqLd&|9^k^}#9E{JW-Q$=l!a=+BCrX?Su@EYtr7
zk7r+JR<o<p)<4fF%b;gnnzHRnPQtsz+kU4AK0S1(_3rtKKmHx2jYe~Bbh!pFsV4S@
zUSKJ`AlldQccILrr(HLFZ%#?QDYY$eny&&QL&Mao59N-<PH2eSdujE(soh+Ew>L34
z%rTwcaVL`fb)>*kVZpN=LQ9igvaXytaSC(P%730KdG9>p`@P{!YS$F+Cc_RDb|L-^
zqL&mxADgK;&YaB^rpP+`w&6mhXknG0txqjwM7);h?9-EVn!pe?(e{)I^OKFjw^f;f
z=WzwQnJt>oaCz|##m|f9TjzQkt+`_~!`ngE$Np=K^WB(zC5QLbgs`h}Fp4HN1vr~d
znX|=a|Fzctu{Yj!-CBCLV`b_@4hEyruE3qzoUTW@J69O)2-uS?`DiNB<G+(0oA%T<
zowL+hGkK=+F((t3?4F3wBS)=$uU>gJPg7-*)=Ae#I&M~+V%+^J1j8bE?kCH3t_Zy3
zt2O)H3eg80=^;y=2On9IseA5~!NTO+**SZUmOTi||9XM5^O&4sc=B$KNlPbi9oyFy
zcHVSpLNV9*gN56wmS!X>ov--1RVdM&_ujfcKNgn1NvLeO@on~v$z}oHZ@YJgzGT&P
z77j}^$q6XPw4Ps?827mAxntz5*vNV766MaNHvM>bMdDSg+KlN-B^sKAx&Cq)D0a+p
z-EgG1OKsV!`ENtEOn$Ub^?zVQtw6s^pqrp38{^K6ZidVLCP}dJu3GDIv7<rb!jZMD
zJJ;9lH1#`p@uX+yuEz^*ZB%OATehv&_S??Ho6@pb3*((GZ@R>}x?uHY={qaeDD8ZE
zdqee`^&xMTsu<78*mu$M!11ZatyvE<Y`wEn<nB4u+;gV6hu$=~|Bac#_t4m+*{CMh
zGxfBW?eyZp*vNTzZ|{vY_D+d==HMq9s&p>$)kEpMM#{YtUp3#^tnpy>#*BBLIOo^j
z^0}}i?`YAznVd5ZSFd@wHttG^%bbNwdK~Zn|D3W|^W62>T$&e-yc4}&a&OPauW{em
z?%!2edns6VUE|u-p7DRfq#v;?TrMH@!h`R6#{c#|fBW7mFsq-lJ@hc+Ez|ku&NobL
zAM`D|UR!IIH01;P{k<G~`;wexKAuhxbL5;DXU+BEd^_uf4%@_9nF}I%iQ@VfMCA^P
zrA?Uq&TKMogXF%4vilNc&pnit>y$C;T=<P^nrN2t|A$iV9xBUo-ZA~88ZWDOEV6;w
zf^Y7HCZ`?CMHca`eZ^kASy1(s!SfUAr@hkQdo1SqS#O`ilq>Qo6Bu+~+%t(&ouM2g
zzckxC?eXqopF=n9G*jf`R9a~BOmtDrrS}qVRE(_^E=4Obd8-RM-D`95aG&*P@uilf
zQ_EilwMXyPJu>A)>25ua!)(3YO&#7IUt{*!Tsh!U>Rfw;FW~=MLz#zXdPBnIUHNIa
z|JJmI>$P_~CMHHNOVudvS~O4e{<pVR6uwQgC~NxgcJYO~?p6+aeqDKgarY77$;o|k
zDQwDHEF?E9;Lfw3sjD$FBTn+L#P`F=Jdsnov#0UA{+pNAx8)7%^YDdzh8<^%!(2<I
z9o;Rq_om9VYa660%F<I?uT_~H4J$GA+H%cLM(lpXx%y|<%sgcISvk{pEp6JjV&e*d
zIOk<eV&}Wf)Jj#=sxz;r)wuQBuj{&3k*PDOm*Hc1<?;R~`~N<LljB|n$@z32{VAIN
za@w;S!P8!P7f$*1arUI%hcf9m8D3;A`TtTW|I@PYvXt^)_r+c%EU#MT@oB~W|5dBg
zxfC>n70rZSt^CAzMncD2c%~(%<GW9Mr8_!>qQ2hr;C}q7-{<0nWxx4<&s_HOk?t`O
zU(f7<xX+7Ub}YSl%Vgcj9cF*5_Z!Y$vvEUiGViNdcMe#ta@eY7yKtLm&XOyN_fB`R
zPvmG%Ubtql(HiSJ2Xtc=HSg1E`m#+uVa+bBPVdv(x@G+5$N2iCUYu5XF7&YfHShgC
zA!aJ6yKL@WuFbu!clU~J<CobCTsumqT`#?JEjRidSAu`aJgu@Q&$7u9qAUStY+YO?
zDcqa3@XX;IXEXvPK9gJgcq&8g|4A?IeS0Rm=jf?-wl|ea%+K0=%Dd9DCq^}Y(k-O~
z){d~X5xhD7RTuDTGja*bg;Y#XbYkR;RP6EixbSF?@NJurjtRUgyQJltrkvQg$SEi#
zWRr^Wfu)CM=(9?+XfmX%F57nN;u6gbQ>U<asqpB&3g|pDgG<J%({suSCsyytesi@P
zvaT!%pPIIGRjAsfwS24VwywIW6}BdPso#xLVGmn&h%7635VYN8x=`wuPrAv65;+er
zG5zPwiTIG%+AhwYCliq<$vE+Jf#Tt(M(w)N%X}s{g*Z-PVRY%{V>n=vrLa;|je&{5
zgNczL<uqFZ1H-$UTMI9*^Q|s>cYjyqag}?wcjVoV7gf`LXJA>v!Y7<_bJNOQrHr$;
zcu#n=cXz|9>&NpW*KM%=Uv>V~<<0r8?9y)kV0u)%&i<g?zu(_keQwIkJ1p|>qdJ2)
zpJ;_-6Q@?if+jYtl9|mMR#T3ME!mQ!=x8!g=|G!U*@=ZNe7Q3wb*N6-F`-jo-U_8|
z<!vjjc5rSqJk@5r>quMIVvdqWZEJ5lTHJ3}C^3bF(_=vcOZ-tahl!Unx2nXfnai#^
z<=IIS#;JZrnX1#7bY?E8OgN>I7F2ll_(7KPLdQtS++8o-#M3w)I3=q+QEiy3Iwir0
zz3S7;baAemni=dZeBZ7wZ2Tp<!a4QHQCD^uhK^?}9!io8Of#}(F)+-jx}{mL#`OA9
zpG%r4s~A=+I;Fa5@uX9$oL3#QdcAVXrT^PnMWQnrS2r;AJe(kQuXX!Ti5m~2;wrg3
zC$?0d;o7jK{6f`omn~Pk3|obQpH#iwdeiG-WWx@&9S0k@d<7NiU*riY?2g&;d)@Aj
zd4l?l!7>^8jX&*POz{0{wPJ#A6i3U2KF+N|f$qGOPbRpj<$l@VDXpfm(Rs~=g905A
z(K>~m(%vjW9@?EY!9iwkZ(Q(V+7j%oz4e1ifOW3N!iIELS<7h*UOuf?f)XFTF%I&5
zx#q%!SW(?+=aWpAhD^#3JR5q>&%O8am2B6oQ$o_tiigiCb3DZwS37gV^mxO!t<P^J
z3#WZcPPe}{BdL#*Epjn`)U=yvb8DwLFfHsAW5{w`^4}~vn~zcP>8h3A(z1Ov8Kq}G
z>Q`sITRe64^H-0~dFly2o1s48^}6e-tM6{U%Q<`Xi)&Mtw@!V#bKktGxwf~jz2SRl
zE}y>R^>q2VKR>s#HvDJJ5N~9C75!)5N%{OgpZ-6a%_7Mnn6j`<q3*?_uh+8Y+kV$P
z_T}Ms&0P|89ST=H9{YwJ5)sn2{}<8CprgH|b)v^L{b@dtc{4*M`i6O?1+7}5s-JnX
zo<-&Aq>SxzR5S~icwB;GcQI@;-MP{?PvbD_%P-=*?>PNsO&r--TbB#|V+~XmS;%pG
znSk+>vpoG;jO<np(@l=LGEO}evdn{dm4BgD@%mp2rz)`@ImWxu+ezpDqoO^<YhE2;
z34hmkZ3D}O)p3TJoJ<}~EHheNb-lEs;zIh%w=C5ysO^q^FzbB7iPkkmGqWAuJnG;5
zAY;AdvF^Sfj}Gm5)~Y}KM|a5INe9&GlmvP!S(15Xdvs_p8M<1nnJ6+*!d*#C$kXnV
z@*8VUXB|zWsS|Q6nRj0F)mbvR?2X3+$Bi=sCrvZ{<M>u1Dlj#2#j>sIY~E@)L~~y|
z@o3sworLU|*SBI%EEC`AdQ7p^F*^Q-lKC->$um!{xt*fYtibSeZQ6pDv-x~4OI_x8
zP;gJ@+(Qxdxi@EM&0)NA_Q|oE(${NJEusoVt>wj6-MJuaapdKlf=@elJ`Tv6|9%Q*
z>;Ds*Wm|LD%RCSHy_$E`N=<Uh!nc7hE-g}#|Mh0wq8~TUq^&r)?!n%Pj&f(OyDPTr
znds!<^m`Z27nY`l+^rflm$fhcIHDf>;Ur77QS3xt|07&MH(sc{IDE0qqG*%g*Aul4
zR+4|`C28(>vEI`sYt=t3(WNW(+#5a4$k&`>cGSAk>YJLdkad3)oA{Kq2Gy^aIR1Y*
zXlvW%-_~^IMo5H@=%P@?%F2aHGCUZi<G9-EA3x+PuLxC<3)Jolj#(?zeIqDR)6&Uh
z>c4eWH^L(I)OM~qmnv(tD<;xVZTG3ribV_7oO9aPedg1-%%ussSG6WaC*8Rwv0hZ}
z<}~$~m$U9m7fb(lx4sq_w{P#`Ju-(@Jk#v2JT|$ZmCsG!so18SvWa<z1ll%CD_g^>
z{3^m*xpR?w+lgguTlCt^PP*4+K22e`ec@EpG2vMgj5OvsUaCI0Y4hBjle9O)s9t>0
zeJpa-QQe@){raz!R`NCY&ZtwI%_%%*8>_XkT7L4gn@YknGQ4&emA&Lo?^?vL@8?>R
zxm-;7eCD$&CheI2n#a&m@%RlE1xG6;mC(e$JLeo$bY$>&Azd89JXiMgjE%N!eiptu
zPfpEqx49uXxq6dE>y_B{F26#KDv4jaG^cMY6S_M0T}ArE(zon~4=D>Y7&r$pOP=3x
z&bYX6{#Rjs-jwRRcXD&?>!i>AU)Jwp*t^ojDacOXVMZcbr|0~e=^;O^Hi>@lUbQsO
zOw`cuS7_MQyZ^SaJYcO;^(~p1FgxPJA?~ySKb4r(b1%I*C^SjHzf{$9CF|8go2Di-
z8O#ac)ecqJ|4Lwk?U6eP#cQ)adn~`+<|`_&M1WP-BZ+60!eNbwg`9qOm$f^d`I;EN
zXU>tYv#fTn;(Hyjoz-a1apOg<*ES|49o5zIJ{i$-NO=Ea*X+{gTQ}C4OXXVYX{|fm
z`|aFXnd;Ng)lWY<9GBMroOJ2<^c=?tO)i3mw$6@qae6Xg@}0cHR*Oz}7EPbnbA-b`
z`qCsnwWYl5#T<S$E~lpdO%Zkq)S9v6*{OND?f>^j?4ETwPHujDSuYE#Uz(xoi!*bT
zpUt$Hzagc`Px4GcM7|ZbZ$?u<_JLjthWj!bAMRRt<MxW+jhhTZ?T%+k-wK&)IkB1d
z#&njllcw#*&q;<Bn256mZD^9sUEX^8X5rE67i{N$-?IGaqm?o-*?gA0ybn@C%>ynq
z8ZP2g6`d;lU(O@Ga#7SXSvPa7O2(<j&-}A{`B5(9l2P{+hoFe!*?V6<n_k!PMe=v?
z%fw$wbF%usu2X0AueTCzz1*VMmLor*uI0)vS<h7&Hs{w#$Eta+?(`MelQ8>)l!W+`
zr7SWc4(u*-uQ^qISlhfP`kbcB_ORmmHkk*nSxlxLzF5-rXWRdGAJ?%<Kh|><3p3~o
z6}gv}v3<D<_kyf%>#qMld@@V*!_}Xm6+4c+)bZXlg*_vaF*7nyL_>0yWp}3Q>C8lL
ziAwHmd!%zXId)yRmfPCC>p{27)7$&LPLq@MC}?G$_4oG7v*-N<&J=r2KbR5j;xN5L
zvxUd0g|~|3sE&(DWX!QI3ddVoj(xn?!E@&%kBjGvma2;inz>A8WjL!Wd@s5rhCaCz
zq~lOOlk+-<#EmDdjVmYk=p4IgaU;AXQDRds!wJuX8IGDh$E4PsYz@79FXnOv$GP@Q
z;f~%Jc{|$79XoGk_VL^DF};W|f9Q7os+XM9qoj}Xr+9egct}tGaQDdtE`k3p47WEj
z-`P6%V1|>zGq(wXGu8jyu+Kd=q14ksaf*-1%ygH)$s75mt8_lv+xMoXdy|QP)ZISU
z$dG#*PS<6)?P$5#`m)<&?gE2l5BB9aE);EP-{9f3A;jp|Ok<<Pb8g<8Q+YnTz@_(1
zWWd*!GS*p68+rmok9dEO`|<bQ42Q_2jse#L+*Y@SeyY8?eCmbBpPbRE5no*A8%~|Y
z$vT%I<^tDDsnimNa9;*i9fzfNE-af7pHO>Bewn<pnTmktY~_DZK_|Qv)S`7GlAP8Y
zI<jR(!qH>Nq8Af=m-PP)cn~{VM3mbl*D>An;-!>}qB~V~77Du9a&H!K-CDsexmQeb
zuHer9XKwFV8@TsjGzZ_(eLrUh7R9JD-tn2hD9LllUqMju<PpAI2}-8ohdW&k``mGz
z!E&rqT%C80dyPzAL(B2EEvGLm@cMc-eUHplkC2#*#2XSb;yl_o8x@mt=AS>=D<Ri5
zn`3V4L*Dd=V4)KW+gTef&27svk(K+ExaN!S)u~;1HqRdA%=Pe75%@C8vas(F%W<1r
z*}@-DeMeogE1o26o&0pk+%&hOH5pFYM`CYZRM2#3kh^yI?$yaf9ZJPFf}hV>)@isb
zm2=7g!;lRr6U9_QTv|^~+mqycR%J<G$h5nU9%o)z;&V?bGVZg?(r2RKo-a>)QCRG<
z;mVN%&&s`cDt;~a_J7ZmxsH#!CY*nDbxP-(5a|W|re-c?o6?;muJ(U%v9MD7;d_8>
z$w{#rb3Sf-yhy<9^HWdupPuvIcrG;1SU5%C>&~e+uk<E2`0`x5@Le>8Dd)zIN-n2+
zx7J!N>F`qNJ*g(Bt0cfF=)!lQYmr8x=fs6aSG?WoBw`@FqUTYPXS7Gt5*N7%(_I)P
zV{S`qcqEa-wB1pJVeQ2holIu4?#YX;`Ien#=@MkUG;Qi4Zt1z)d6q|HY7?fnJ>a;;
z&B>QMOVvO4-;tugr^ijhdA}Xu-L}%v!KFXK#q*COUs&qVl^hkUi9wUZqaJk#icPJO
zJaS+{6AQ!nv!#h9s;%>Q+!mY72>UP5c6$eJ>#N1mX>1K8$5>=!6OCe-E<9YmXogeD
zG6vB{>Rb*CDz0fk=hUi_T;_T8=Bf#~G^#}$m*_bpmeC}#<wk1Pjp7x;8;WB4yVAOU
zzF6&-mijQG*x~$%tut>=Y!Z1Pk#5&+{cu^<GPO-ntNW|^oLw3mQrR7+_KC1`JLfb>
z?d{UG>vmq7U4BGK_mJzm8%pozoUVPc==~R^eNHF#2>3sns%fxDvSosk?uMCP92Oh*
z-7}F}b4cyY?826whK>N!7R9CfIaSH0W7asH;E_z25OHwvak-57EjgSbUMKjxzAU_Y
za?yoVqQ@3bVPxMTWhJ&GL?&geYRZZpjdRa#rT$lHT-&!=g@v0zjqz%u*SV;Z8|P&x
zt&#|ya7eA^P<kfguGloEuHZ|n;?H@tZ~P;_aZgLN#YL}kVVinxi|$<_rg_sx&2viK
zM9H<fd*%j4-v|>v+7i%|5co+<^nu{LrYZM!xio%w>m#xBplxc9NzBoVqer|{51R%Z
zQeAf>RW;0y<-~`V2X?j`zv`kf_35cKJ^nE#SPB>o&!)zyxy2phefes!3FnMj4z)}T
z*`EuKWoo1;JI$Tmb2qhWQMX_~*GIRT6JI^L^UgN(ZQ4Gs;u*n060^lJ4r#1$zP+}|
zDdUiYi=%>DzWTFgHo8kD9cX(rLG6+lyGoGuEmd~2yAi7%9LxECEA7_IxpxcCy??gs
z*QYcMwYSeX;>vjBw3<%7V3?@AY4ty+D=t@)_Sngl+&f-!>-LL;<@chVKNaO~w&^$^
zw#KVhouz&G%ZbaK8<ro;<M)2K+{Ylfa@W0A2i1?XZKw_7(R}cH{y~3j4+hQ!qRoEu
zzq+ni#1U&ht6}v`*SM28*E;UU-(0g<bnWk-8-6J^ZuE5A6scq#vxLDy_Y>FJT#jp3
z49`vGnULZ2@oLape(P&1>yGx#n|yU+hLGUtZMnG{Cmbv-S-W|Iii7Uy49;lrwbP{C
zGg-^Jcb~nsBST?YmL_}N%iR}r^Iu+UW)a`}L|2k!`u$lC?rX6~v*gXWbcAQq%31&0
zPRufz&SP|@<4wYRO`j6QAl2O`j&xl2=1Huqjrk&@czOrV!bw7!J_p1e+-VM7_3i4>
z)F+REH5l(FT$Z_|XwIo*z17XKRG-Olqki}~_g{(M<&wm>Kd%rnT$_}1r}VLg&P=fv
zH`I1<S#F%HB$fRw=%euB_)TS>x4shoownNIagKqpyJjCdgVah9)tREEH~%hL{cn}D
z&MtT6zxuAWa(iDX?f!9G|CZ}hUcs5qyg9{l4j!AyGCRHM(v?n$9Jb;Utd}JZGd(<P
z_vVm-(czE{3!bI_xzMs#MuPkAQTOW~R$mn0dtGprDeUPKV}at67uS_ccR8qWWd7}q
zOm{9aY5TZ{{FjjS+Wlq2w=dtOJ@fb^74UhB;8`zauY(Hx8?7dP+$`g*b?ohpJi(ij
za<@B*Y+tc<_r_ewNCp3(IJxZ;1Kx$*6??kpj&38vC&uMl12-{hY8IVS3i1({bTsnF
zA<Ii8xh}^lH|_5Ue|k>N|EGuT2A+lgj<Dr8N*X*hxyoK8b9eb9wg)z4dY6xxC$7KJ
zdn0+;<=i=5DNI`pvt2Fj>UAaE?J4C<eVkY9(D^tZElntLtJ?8N2ThEVl{Um2&-tch
zADEvTFilMO!dtdQAN1S}Ugzb$lTB~>eD|&Jt-H_6gID*w&AIP#fN4?RTC<r>f4nxZ
zbZ?MdwMiuKpVo8d(&+zI=?wqv<J>N`=KlR0s8rChqO^&p#IfYL_tEXFlhte8Kh!uE
zd|mR~sEp&)*0}1V^EHPS__(BcMXYhV{6NH`f#t-lwo5rbE4!vIi0vpc4t$>D%|G#9
z<;RsOjG70&2x{y5stftBy9yi6zQnfGEIBme`wKDu+pAxgeRx-vdaNQf&{N3eK=0(+
z+g^BW>@k#3FAaKQk+sm!m&<`!YNyocC&HSiFKwAN`LyAb;~ynExu+LpvUh!ztJj=n
z&#{ZesNk!vg0PXo{FMjqnkozH%{ladd*8}gL8>vqscQFEKKYfY&U?mpi__d^Et;FB
zJFHl;yyaNVoka7QT|dP9EIRLA*IaGxY}oaG->WZuoBRI-tW+{y{VQMtUm#PC(}pA4
z?s&GSiKK54TBNXS@`k+60iVsxCVLr%hjst|@;jE#g5N4fA+}0p{kjwSR@O(>Gc1^T
zCo9;UyCNoZ*Ru9qqQ7@V&D7~KXL!s${q5%uEdKVz4WEv<9I!mU-{FMD3pvOA-&-r5
zJEr?rTnhQ-@%WHgvP*GCsc*?Im6pR*L2m4v3n%Vy3ftk`thP72Lcyt6rRR`V%p>3b
zZ%PYP_+5YP75~%yODdz9lj*^ZEf%Kz&4m-SN=tup9sO)|K&Ny*6VHvJFqg`;6K6~N
zxP5)(IWaAl+hy;yDS2PFSDc=5`r1yV(|2Bp?a8prpZj&j|5aO~`W>UC?>s0wS)O^c
zB(~(}9G6F*9`HKJJbCu@SZ}!e+%v~aQgyF5`>p6v$aTC?>h(6PY<*~cmy~Ju_V@C)
z%r@MV$W?G;Iigi?T;krzBU$2KSElSOiuSyHo=s4C{b#-_Vh>hb_@<?(^gW+L+wy0T
z!lY+Q>>s7YXW#jKEKjc8!u@@(Vl2yw<7?ad9!%^@tNi+cVd><rRY!wsCQMqL9>5_i
ztG+9*bK~s3f4j0JTfZ0U{f;S#{J?O&@^lMh`0o8m9bz11MN`vj7eB8({{3a3Pi^>l
z7UzOjp2{x^3+A0YVH##Kog?Sn&CJsm&R>zT=_=pYeaY+Nq(d(pDt^UQ{QLjYL|{Sm
z5o`TVS-TFH7cSmkAu^l&hqKu4*;c8gb9>VTndMGzRV|h9w%PQ?${|N`vulF`BQvXt
z$_j%-=XQR@NuE1A7d7}sFG;vF$)(9zlXIyGqh^qU2cxHt*pyEb7dlN-^<Fn;Q_v|9
zRb%x#Ybt{ew9V35ZZbzG<Ul~HOLq~cm22jurM@$Ta<v*xA6mq@JWRwPjaz(GRK(S+
ztII;Q8FM3SWoDP<9NNZoZ(i!{W4gCm%AOuNy4yVW-k$9D-=1nWh)M?KczBx33DA!1
zOnJG<vtmc=>Z)y8hu7Z8Qf6S_Y*1iOVCYD{wAg>K--Hb%&MX3n$Nf5@4!pR&U3#70
zivPDwC$u>z<s=<mb#=up4zJ5@e{U(~-{hZZza=)>;mggdE93j~{_b>E;Iv4)qN2)`
zasI^l-EDj4T77@cIblzn?*84*-!G*0hxti!)_i!NIzj1xuAWvwk1wCk2gTJgNgEch
z>t+=wu`4dxptx2@DtR^U)*l`YlccIv+RbnfTA<ux6{YDQn#py+Ra5Ol0n>!EHHBQ#
z>N^6D%T!(|<d*HerLaQRf}@L@kK+^9YQfacUR|4_6#Y(xc_^OI(O#|6C!)o=LTH`V
z_A{GK8}3ypKC|x(i?EsL(G+9fU6&^5=<a%#)#0~wRnAc{+pjIzi&*TquUM*a>R!3<
zb;6CLDH_>{t^Y+>7<#{KcGg^bCFuQ(&$(t78P?`s@U~^0y0^W?S%GPa(Wwk>BaNu3
zi^4rCr-pWztoeGV+3kvfb4>14(eMDrOD_(yMNVo^N%r>=Th`B!_3dC<aBt@_iNZ^r
zQ4;l^GEc0YqnfcoX?>Mf%<>aUleF|!L<GgIy7Bc?>?#!rR~?OoEY}uVzwg!64SREI
zyQ0bWqp$DR-L}x#*eqJDk!W=4mVlwzsa4CTm9Z??k!Uo@fzQ&xHQQt7&5Y>tTQ5#}
zA+o#0rCN9I%RcL!X1=Cv8eN7rGM7G^#meCO_(IIS_d8!i9P{7xdxo=a%KLTBk}uy}
zx$PHxd4+XV>h#HJzAvvj9Jc!ZeC^b>j?}+4*0=Vn_{AD3Giyp6-LO^A&nI`gXmIpf
zUez@l6jzJgTC%X4@6zr3Ccd74_nkAYoH9rZY?%<mzy8w%HL>`f7>8qtA`9G3>)sFi
zEN3}?k%wY8%k+iiQ&Y<C-tbti)v5dCPS9zMomYgsw{PmX`^_l3Joww){P}ag-%01c
ze6i(O3vYafhhX-3+w8*C(*jj`uiOi0{2S7su_eJo=Ml&0hzpB(x7PjosHs|2^YQBI
zUy2NYS3MLO6J%R^13f0r|9AMEcthP|p-21cj<G$T5V@5_kv-CbasJXBX^I~jDw=zl
zqk|TPWHPF9AN1%9+9Wh#gNt(cZfS2Tm9;bfpEwd_HS?s8fzgurDlD3Anx}$Jh)$Jo
zdKwcnk*jspBfVuCUVDCRh_ycPNPWA>6R)a`mt$Ch#SbbN8dP~M5}wT=5vIV_m45Ql
zVF5PjlQzqAg(_#H889zBo6r;dQIhw(!?!tqPq+myDLirUjJWi#IW0V<*~gFMq;2M4
zV6dIMsBc+CvaRE)=gCW|l(QG8nK}RDv3BfUe0<gASuYm!+ZC70tp63f=>yLRhoWTZ
z+fq%t1B}>wx*MZ!pZKsX#HcM#_|@%u4A0FEy1oeC?-bx%BI<DH>ZEC#E;el_+1x5}
z@R{n-FAAHNENBdw**kOc<d`0>B1^f{b;Y()UVU9nXEixi{#STl6*NgIXxF8y>2uU>
z9hsM4uBsX7IrmMA?8l_?WhSio^Ku?dnyqH+Wj=9HLhPi=ky1)CtsGmtYog{c%(}kx
z#m|W~yt0BqOBkj-X>FCg9<A|9W>#QtUWxgA&xVaBE-UsboCsYr>BRq%$o-Q~t@RO@
zIsacm*8cx*{p<g~7W$u(tT@+h!t%1undNSc7JVhpRV%zh*>|@doRsjSQ2QjOUzQT<
zd=705X^(_ZZYJloZQh!(vf@0Kx9ToA&{h%~=%M}W>IQAmiwpf*I`&WVkl6VvbzRA$
z=?Cq0E^Li`k#uBr#}SQflA9BBvOhW|Un-b<<4dCY1f}97Ckm41o7w3qYW%;+qM4`G
zc%h~}eZk6*<1Wkg{kn79^1<!1m%L_gdiA2c_gkdyjY|?IH!@}vMg2%^<d|}5hb_zU
z16i52yHmU#PW$BaXZ2NG-AM{8y5T#|roJm?Zol(f?fcZ-H5{%i3YYjxRn+(e13MBX
zg=E<J6+JCYd%DE)?Af*3-R+s?KHv6gLeV_EIVnp|Wh`OXSh-WrD@<w8^s4D%4?gnE
zV}2H7xBtoRYPO1lOA4k${xa%%w6Jq6-^t6L)~QU>^L$V(@l?ZoazZq3p;myr*~fNw
z)`|U#SgPGar+TKADn_1_IHndVT^6kAb#lv<gDnSTwR;{v_<got#7WO+V&SYO?;=zi
zeD2-;KUw)kiiY;Oyspc$yLbLRcq7A6^}?r3><7Jtm%Tf(QmrS8t8vmZ5uvB<P7hy&
zXG(|^-xSc+T~j>we(Jd}4l%EirVRp&yb_MLQ!f;<_QZ48&e*Rqt26jWwwPhcay_dk
zqiLMUku0-W&Nr}cS&`YxFkyn6<)LTEt2I?F*R{P``;m>o;B&9Xxf7cA8_sW8&MGQY
z7p(EU=9c0PzmD~S3^|^ks?2BpIj^R?ijztAcc|2*uA;B*3V&JiMA)NL%f5zB+_`bm
z?X^8$I$UP@n}w>}+1+HcHB-{!HJ{(zw`u0P`CYlb=ejV7|IG83KeC2Jsn}t1^PNg&
z7T+&HlOJ47c^|ap|H-q;lAX+7YR}cITqLbKfobj?@8@qGG%dTcw|E`bv2@$_oH3@I
zyS)|Jh3Bw*IB$LPtD?A>hLgH-E8qMX&+^v%JZta0^=^a7BKO+)p5Cjxj|nUZ6?@2a
zp*QZ*g3VfCKIxj?!EzT2IZd8ge4VuT*Q11H%dEN>c_ZJ0`xI)N1KtR#{Jwe6=V=X>
zwyK_C_4M6u9xc(!j`S6sAT&qns<78r^?R>4x37EjY;BgPRz6?&E!h`qW!`<rt63C&
z*r!LWouS8oY4#uIAF4Cl*ZZ!Y8}1XywR!!+Er;Z0tdxj3bBRkcr)TrT)ZQ#d?%xwl
zk7vahe_dHEb~5Jrz0EP%w@&`xnE3zXu6x1ltfp`8KmL8p=&Xs`;<Gs?&35YEndWfq
z+Y)uzpxnKZ>ANm`wcFJz{4ca>_vbZtUdQQf^?z{aSoo`b@9y=@Uik9Us$EUR<t4I{
z6dor;zWH@+dxbDZ)Ryi&4C@%CtL`m$@_@(B;~sa+CDmAkzMvXkrbP{jIgJ))4wcG0
zI<Vr27tb5!+UGtJJS_F@PyY2(aHV?laIJl?QYDS=d4d7k>f^x`e#%=C6xbFWi(@rw
z$(zpBF|F=Gen|mG>eFK-uQ#phKf%#l_vDoK5B3us&$sVWdA-5!P|F&w{x`2vlpB`h
z++Ehgy}#hpDxSBG@|0avl*L$h1>N6W>T@x9QTQS6zxp@6)^ls#|9aON(V?;8z_|x&
z_}}-mCN!V(QFqZ{E(_u}yV2C~<kbbSi|3_2cC30KxH!XrZ|!^y-l>bub(<Az2<n^v
z;E_>4{`mzdU+oU}c&G;3wOP9@?ft|&F~vD`SF>4Ak0sCRA9ZJIo;|)Ub?ADEvx?g3
zYiVbDf2`=u3Y;9*a&t;{Zph2WoAO<%w4HYFFWbg1bx}9BXI=f4w|}-N8Ey+S+#zUa
z^6~Z-k5_*(jaXLQaeH@iQQ+?<@9!Mgu<K6lt$S-_U*}JsqpMtZa;DUwRm=SEn=DYs
zdF}YGkefqDVd2vIvtHhhQ<){@G%L?#FV8ynZvnG48XoFQ(vo_o*w^ykbB_{Jz_NP_
zSeSonKi2b`o3ottZsy!L_X7rfK0bL<j=WX<nfXNRjiSuOL#LRG`BqgPae5}^#nxz~
zQT<lML5nNLV7ph>(hti|%rP{JI#n;@#F5&0Y{evw_75+M-n`tk?$slgb$?o4r96JJ
zh41yYKJDq7>hI-imz`o)?9e`NM)$s+-c`QoF>iMFoI3rakGIn^YKa1$>H+<=uJ0GU
zidymX{hX)KZ0|n^6&s3v*4m>K%OI%qh_&B<_mjxP1sqEcaj?d#ypNxCqU(U*r5%EQ
zgVrl8)6ms>+^3`Vb;$vH3+?OE9#3dtvr3w5VsR|nk6SGHuw{+zgkQ`PiXK~s>89^n
z{NKE_e^OGeM5dsPpAyre135E%w+FmT$nxV#<WicV^CwAbS;^~PTRvL52d_UAEFJ6h
z>VVNJm%Mo=ma|u9%5kReoOC*Wk@sDJAcsw_mFr$Q?%-kAt!lqKe@~Q(!`asihQ}3!
z4U|fs6i;J{>1cVQ(`p^&9(T%#Rr;uo))p4S5*FKaoXzge-T@WcjvNp8P+=pi{=22}
z&LN#`3l3-LviZ7d@IJko_2K9xwW)GGuU9;A)6Dl#Yt~)&?v+4~d#i}<$8?XKxdk~o
zp4nTpGS6_#-C{je;KZsF?Z2$7uNQX|m2ho8!qv#bJ^hl^{f)MVFL0mSGJVb)%QcJi
z99>i;m{xu&`p+wQWJbmzwG1V`+$6r>RJARtXFq+=xT9l~{$lkGt^m^@Ed?beqozkJ
z-W$w582xSO6!Yy|)p1cxFz9bjkp4x31A%4=%XD}a$A3HX<xG+>*P<RP*6SI*KccRP
z?LKTCVdyd;b@PM6wiO%0SYKYXSsgZ&&Ggsd8xwS`_|;5SvKXFmGCF6hxI!=Tq@eB0
zuMxgJCoc#|FAcRS&DIWCDZO#(XS34i{TCCD2}Yh2luaz+UK%L3{&EawyKKB??8-v9
zn}T+Wi+G;s$_u+Qm^dm<Ijius^q0#id+wrNY?tn{T-(Fs=cdE%CMV=r{A89up5~5D
z551y?29q>gmhZdP${p7HU&W?Kbz8$D2mQGY*B-fS+aL0&%*w#ys!Q3WjlO0doZoXk
z4LRr6cEY7}lfOxiafFDQEK{I)lDSvUVjosjC!NqW{Sk+kF!zgH;Cd>s@rwTpvkU*0
z-qF$Vt9m1|_FGs;mQPsM%Y;_Xx$m;Ss~=yL=k@f}{G>AXx!3$(Fdf>m&U?qX;{uO!
zPjB=|bDeg@$Vu#w?%QvUU)1tD{Tc=I3-0$R1TTHt^tAboh~HHK-h-=mhcUfp2v%b&
zE?Srs{fti_k)v38vZ3Yo4@t=mliJVgd=AoDW31ZmGM_p3y_SjP`GPN8%&JOWZ<I{^
z{f)B{xi>BQ8kb*wH9h24^2VsN>tz%Fhq^0s*$CO?d@i5({c6rR%ZQIR=1iVk^g7|f
zA%V5qQ#w5!Uo%^`Z)@aYzsS;WmQ~t$`+{xueyrJT*xj`HPt-}pp3=zUR}XzzsW{It
zO0(5AIr;aqV7rCgBF7s3Z#u2?EH|E6Kz{a7``KLy+#3>-moy57C0_UTC{igAyHYIm
z#c=|Y5w9v!%~@sFQ_0ziF4gWGMpIa%*E|-|X>}D~IlSb_mUr`04wo!uIV|+b>zG3N
z$#mb)vqw_4)P^19N^Eyp_{7P3%VVp}3r&JE-C4Y=PrEy>^pa0HzQ}Lsv@aZMe|WBz
zc(o|x&@_Q{T|YV!mA#8gbk=_`cu;miLCNQ8h0oC~|9y6En6A>I(DZ6Wz#(O!?L7CV
z6pD7JW-x_1E%Nhx(Q?Me|NVxun@;&NUOi_rouB#rhi5(vXPy=>)h~J78t3sv_n(fQ
z)Wr*7>+}>h3eEKB5?GvIvNf7DLtC$H{x78vO*KW2Qeh?4<gU-=N&4nVzGm`FE^f}@
z<uk)39u|}Cw(K=r#PKP2&32=Z$y24<LIU4^ahe(A_|rIWN2<4o_rZ#;Q`ds#Y_&UL
zD82k^RCKw?QOVk^U!~7R3eRV*IUC_|bgSfV_PZB~r0z%7zuXyfb?U7Tl03_UZY_E5
z{B5(szp42gLCPwP@-I!}e|~igN=Xp^u_vjeQU1pRl?tx9l%~d%|Fhd#99@skws2@x
zG*EuZt<<;lxyVu-J&}}$nek3#OC19;8GRndM9fbyTK1Idsar{!v!?p(hYRLR<0;s&
zFz#!B*VB-HKaMs;bQtQa{OM8|7N9dnYV972RlPepvezA&CD!%Pm20WQ!5IeHdv8si
zVBmT9+pmx=&WSx<n{RgKMx}9f^~f$w*zvBl`_1-cLHVri`RpgwwSLI|Yo>H-5|4aJ
zpJ~zxCzh2TA_@i78D9Fi{;?>U`e5aTrA55PjZ7c?&A)H5HZM*+oU_Hj;@yf#H?=w^
z?^tLdEXdtfYU`hv5T1KEX?f`7mG3#K>Mon+uG?1mF*Nq-w#4MkS<xyrza_X^{^!c^
z&-}h7N$r_p$Yw>R*E^VBf6;mNW!AUMrO{htx2B#=%I!S9?gm%S?&R6_YiiEt&$;}4
z@{!wrZkcz@6uoms#pA7b)cLD>lOIdoFRY)-ear0O^nes!_KF3fzcn~-6wdN&JiEG)
zJA{#cosy<R%!Adlk8}lDMA(M4FIk%4eSz~((D_F{y7vbJEcKN*U{mAleWNvWyIsfy
zv8=}B>sxsCJ+8LXVeHs-bXot)J~N#QSu3YYnF(<{%W*s&IiWJ=&hf@Q$D~yi+hi6B
zP28P)W8o~Z<@1@n?Q5n?Dd=8Sp*?L%*ZK(0wKt-cPvMNb<hA^lS9MO$(Hfoo?@r8P
z_PlS%akJuo@98f-SMv(}TV1R&R`OZ*Kd-r;b*Nu@b_-MF`zt&J|F2Atj-1FI`$6jF
z`!il<0;Ud5FBA?b&3wtae7emAy{nUZUtBVjoMk@KV8e<jhYf|zS#;ejUB5orJbU>L
zPsM4;Gp&3!Ud=2FG3nGepH{`2yu;VZSf<HHb}DP<9>t<R*K6-ynR`5`uR7%KlbsW5
zOH}8c)ZaPT`%A@|oi34#v$b1a#caR5Zf@SQXEg^S_Z*%%=lo7#`@4}>f65%cYPIq6
z-$^|4&X&rq-gx)(Ou5Tv^HmxY%nwGKITknDb;11BxR+P&P4g|ZOIpafhavk(BWK<s
z$+AW2R<mWkDX}IjRy0;N7y7R(d(O&Y+LH5GD}6UT4*yk}mG0#up&CAk`Jl>^lx-DP
zm=1c?EdPBZGg6VUZjz_PqGy)dmc+ELoLY4}{9a|vUh9fDhO(L$t@~bd*1ZT1Sebuq
zrC`_fxpgiV#GbE-%Dr5+pl-F#j19{+6s<Wut7~`N>(wqf|6gh^uUcDJRdGON-PL<N
zPrupr=B>Y9_hM4syYdft-S6@~%IQ56y1-huz`iB<A)mhBY+l~mk-uha{QXH^*xrEi
zXOZp`MGo7|hPR7%I~RRxODx$tX?6AWw@+s=Ki*<~KKlEq;Lyl*G1W;Glahk=rlifT
zto>ajVY`FZt;a=hW}v-sQ2&m<Ed2%Y&vZimcPcX7=={?s(Dft#|Bra<ov}9q7FB;%
zt-aLEvHNAxuj`!wd!!>`#P{s2o#QHe=Opjned2XDt?PPC?%tkVw_SMolSrQZ8+QIW
zwo}K6i)B|zmELZ)U-9!(3;$34^~d-=gHr>GCNsB?O@_h3DV*G_J_Qy}AGUOes7I}_
zXk|Le8*ITf<4|H(gRU~`44Y1)R4*1LAr6LuNlOo?1TD$fsChKSk8yU?)=f_@tFI1Y
z%9^MY#r^ccLid?m1zHP4S1u1)=_i|YY2~$5e*Ke@N`-R5rY4<C$vt{1On1RHAEDk^
zDvm;&ChlH0H?^Ko3yP~{TKjdg=!0#Zi^cS+K5ck(L`&NGf1S>PwVo#@YHv@K{+XV3
zpiOxH(>L2bL{Hv&#HlXF>g&s6ivzcXSzb-axyf{cV@8(rca6I#ZZF^dz5QM7!Qs}~
zsgHMm%X=E`zEX~RZfVZSHG9I+pZ<Aq`9@8p*}W%sRVqHJA8)tMx2yg6Rn2o!ef*?9
zv3poD-`y$tuHyOU&--8R`4_~#;|!S0@XfREL8I3dlUAYNAKw+~ls+z>?;*c1<zcH@
z>WPP*vUw}+dxp%IawLoGt;z(?s0oEr{ND;Z4Gvkq^QxD0-o|esA!cn$7f0kagep(4
zw>!$FnLFW7(xk1N1u~PjnwF;Yxol1Lw)GUe*2nNx#gj|gd9TKe?zdMaJ)QBtv(a;z
ze`edtWj&c<t<Sm(_Dxlt6SXWP-Ro;cr0UGtpiI>Tb!<`G3)N<NM$OYmwM-6)I=16h
z#-w)<i-RUU>X?+<H*=}ba=)^wNz3P`rH4)0r+89jmDjpeS*wpPj9Ri>D|6SAT=lbU
zuag(g*`i&sm&@Rk#J*2jIveh>TCZ7uREaxhBa_bSHJhUwws*2r<Zg0}VedU6zfC!@
zdb--5%)-lCqB9cnYRWtv6rA2^rCe}k@nV_iA`xpM=q_BlCEs1RO+-mthR0*l0j<JD
z2PU19B|)8<*Ghtqu(&BYid1a0n33SB+ZuLUMn6?MST_Fav=d6zu2UCpTXZM2Z`;ak
zizEKu`5AgX)!feQrl-{MD;ejll1~|5SgpHil5WVnlA9N2Fh14jX|vST%<S;2ot1T-
zx3~0y#@iW-J!39=Nf&K=`%L=gfvLV-1)1NsZM&J$y;ded|2fCBUIS~htKn7Rk+S!7
zPxoFbxH8Fc(gPd8x*rc(tIh8`>Z%U6c=SL}CaHR=aEEwZ$<;0Kb#s}2#J^a;-x1`y
z>~hL{7H+MDznXb$Id+RkSIoQ5_A0Amx5uvd9o6smRR5@Mi<o~yVs{kJ^(5`0Qxa27
z*{^=GCEH@wvMUo^kF8oYdDAHk?TChNoa!^9Ce3^{BgxA5-}#s8^VNPonQdP;^U-1c
z|5r4h#uTTRzTa6t^?&U<{f57)PivXi-1j_Ha$wEfHJRs|?`1AhX^>xdG@I*B$O<iv
zRY95g%}a|ySM9jsC{S0>taejw?E$vg$Icv6*`1`gKJLXKiDgS#jeR;RnZoB8PH2zn
z*&O!lk<s)yf6`T6Xr0{AXEDF{?Q)ejt2p=EV|8u2klpotmC`<krlb7~Ii1r^OZkYE
z3C&n>#O}vUPmz|2V>?wEV{|@sADh%PWm&;etw6<@vhHb9ZSqv}uXLWhx<cG>?%yMF
zGZGUP>peZ-lBJt*(O8AqFk<FOh1^MjJA2Y*Y@2zi<C(#^$s+2ytS8sCewtu2+t2IR
zwK<P<+If$1_Gw4oJQb?Oo_8bjT+X`x%TBz$^3ke_nLFNOtJATK?)tAED%M;n_W8Ep
zdC@b)xePkb<T+IqBt;2K@Uv6jl@rwAWOdTB$?y3QR?mfwf>O(4)t*@1$vx=wnnkX|
z>_}_*vvr2bTu+VXb}SOH-JGbtllxQCrp?EH^k}&-UOK-)Y3Xal%!O)}3n%A%UaWMq
zC#`Oi@ZV?4)qW|m2Fi%WFAx$vV14stfWqGubM3N%CO$i3G*NNMzn*W)|Nqr&c$Ca@
z_09K_EPf{|Bdk@H?oji3B|E7~<IWXU?}`xau2)TpGK#Bb{ko#Qca`U6D^7#_*3erH
zX%Tu)R&Ct#YxM@b*AW+ju2q~mH90BtwO-VM?wA{@4F5}1KStkmN_zHZrwi|quL?hE
z9tlett?#(9Ixg%8$NrM%6LlGU62zi(*nNcjPt0RUWO8uwig{$fJab8>h2vrojW8wC
ztZyn$PkEQ!TV%LIA<c097l}|iwW&9r>1G>E<~_?LDY3xScjA(m2fa07Tiw2yHLB>Y
z^Z04vIa||whi~$&JjTr@uim*d@0-jDuFb#9wr|bNt#+LKU|YA@vySq-Crf9C9&TE_
z@9kUdty%9M9&z?jbA4CXmL2oROTzNI2Vdc*y)sXCaairTuyodKkqKt}F%K?XeKLFX
zWTw|gBzIrrOepg%QTtHS{_sJj#o>iM9m$I}oL6yiIHD`)z@ibRP^9^Pv)LMnNtrF(
zQ4PmDG`LbHI;|C-t>n72H~DAWKPRpI3j6izT=r#5`{rVz;;gp(+u;YPpPujk=V^H=
z@Y{;tVbv>U$!%Y5-+W8MU+JI&(^o@2*OT82Rva`bzOYiRXVt8$Z)PlF3tqj;Xc|wT
zle5iwxAxwfq4Vad`pD%eDmtWGci`G1zmv)1;*C>xgm`&f_q$zGoU-k@sD7_tUxi9{
z<Q5rm!xKW&I|7*&t38$!?@`z#apHn{r^D*_w-Soe_=^AEJS4Hjpl#0rZT*?2zs62I
z5f=Ym{rdLGZkhTZ<Fgi*PM?X}ws|)1+jQ^Sr$winxu~M)`7|nRd;Q5J7Ya<$J(!9z
z{vR&oURT7bb9cpT&kaxX3h%8oD_bv}R~)RMUm1B}TIqAMU`hVv6Ly|S;j($a!ftWE
zopHeqpU*wlWMrEJ@0_38AbeFMT(<d`r3<IzMyIzwFW5Z2x%<Sscl*D6`C$AoYLP}y
z#Z#4{t*c`n2~9roXOsHOoiQ;+NpqMDpDVFFI$L{h(he5cXF(4pO;2!9b&@-K%Kci=
z<=02POse|gr?K?cm42VElizT@R=Zin^tMoFI#<I4k9C)B%;&j0i!<wOi|5HCtsA$^
zzKCtLk~UVVyfvdxd#hpkzmTE{Tx){na;?hMvza3uoK|wX>zvlY#E5u~u#AlbYtmx*
zH|zC><upE5;4k{Gee_#ZMxo$D`$tLHJ0<qYd%l>eGH2?S_*M(ngHbITes%qITU8OE
zYZj)`W7?TKSMvPKaI+qL&3$J?V>X`8D$|SbZaD3$Qyd;2_N!0O&PyoaX3Ihr4yFXo
z@4lb@PWL#dU9*yFQ=ztGmi44S3!!&&%eel3KRy5NmvWKcwbol~UjA>m^{zfmZF6-0
z6Av@jSXK_n-js69iPGDf%y>_j@!g1Bw<PwVYyP>7Dh<pQ*ROiVZZf|e81lh2{zyu^
zjY(6FvgO0Vrhm^YeuP^75tW})EbcHRYf*9nUwZRP7on>uR*zZ~{9J^2w+Zky6^d>v
zj4iR2os{UgBXNIn151*?@Bd4R7JjX{9-d@yy(PRtC@`T!w@f4CT4@w}Tl@+kuQnmC
zw6;WvprjdX*&b~fGfe%>B!ku3)3%qSABs|Gb%>~tYIo6QOI227_FDe2@yge<Y`2Pl
z)Q-*w$A@A*eeBMqWl{}o&YFsqhZcoR^WaKNv9f2+oU^&m{DgepGhOc~!t2B{*%e)<
zx<$-5)+MshPexEEx<s+dwWU|l?bvjo#ce`-&RMfvg%#YC)}#qtaj!kxA>`3etSZJh
zEmi2u451SlNoRKSSRAkA4DLO2yjCwUl0kuSc7tp#i^{*nIXgvjFMf-@bga(ru*SJk
z^@5ao*MoiBE?&RWV}7|yexDID{r?if8AtlrHB5Zmq9sHccABK`Q;ECJ(Rk!x-m!xd
zst!uC@ARrt^YS_v_x+mpHlq&nmL|64J~EaC4CV@FMG7y5WT;Lm<Y(`aysj7g!B^yY
zg0Q=vx@Qr~VL#KCeo|~j=9!7U6-n#D42vcft1c6|w!Hm(h1ATa2GI>F`wT_&lDNMG
z1S&o)&0W!!+R+wrQzBxyhU-fE^mL&V@%D7~RHh?kYF|C`+k>x&mS1qzc4+n1oGA76
zn?!ATNW=H?I`xWWkJ8$<mp5FG3wKTLx?XYIDeUB@(Dus8*3KChgDTGlRbDAla=2cl
zI#GB&vtG0U=ih^t(|2-Cl`LGQk@;y+*XIA%yTZ5l{hbuSpv(~cBw~-b0h2)Fc8%Gl
z?lmfxY8P+qvGY_o!yc~otY%+j&D5z3w=!Dqw$~nCZgAXk&du~<HAdlw$7{F0kkJ0*
zuger{AE@s3x$k3m{hgh2KYyQlmvi1d%L(6H`lH+W|6TXwT~ZhGa^9uO8db|fg?}`B
z%`_8~@P2UA%zKfv#EUo$!)VzRakY;nzKX{Go*Mr>wCUGY3mz`Ze@C1CE|vE@AkU>P
zZo6WVrlr-Fi3u!D3Xf6>O&J&Qs`>E<D$IAY^!^f|xv)jID)IeHp`#J42398LUi5hE
ztcg~TaaWfp(wLg_V{!Zm`vi+6PQMmASGBoTEveen7WV&VTdH}x*Q-*CtASdH4$+zu
zsw>LtHRcyyYpLBH(%R9HJgcK*rc-*B&gO~f{TdUenVZN}c3xQ&=C#st+2gQDEB)o7
zX5LcidUh#ert{3@9J9DD3WS8rY8CC?b$FKWBERX)-K!PdI2nW&h0S(b>9)7C_=Lxt
zi!+1@5B8Xe*Qp2<pQz})`a@#-i=NMhY;h9}TTA+$Xym@z5xi-Z#LMq>w=7rRnkf>w
zNo>!q)Qa-p-CYv<cGd6h>c1i8Ips^e`Q{)7&&4IzmUchsfA&o5^h8aGO6jc26E?8q
z^Sx}?WRjn~D)eVr{-0;w2bWBgi!6{mxz0^vlKf4*e~k*0yw)2qGEDm4)&1?1|DKg@
zwi4_AhKTETX4zCI=xMe{wQrDL?kD%$+VbUw9iP`2-jZ-x*>09KrSD+(btg98l{Wq}
z%Oc%PLM1nbzijghD~r!8OSoBDViDYE6x@3-<=n4amC(Ei6B~p*-GwbXM7M-A9(S_!
z*2*dC@S7TvwyVNjdxq(i(6gJi%m}aSyg8#MQz&RRXI5zj`^VxUMuzj2D-9W0qE8#>
ze01&a657teV&2{}Z`M}3<J<DL3NO*DZk?(xGO_xfkx=)m>UAfp#f2h|pYXp|KHK<t
zkFj{2YEtc<ipayBvyG-l8o6%_sSvuT$uOzg^um%U8(Zh*Oq~1Z<j#jLcRsP4xA6b%
zogbGA8U9}V)g$J_(|R-JHLoK#FBDnJc4MB+rS#0ndG}An?))8_T5R@<bKT=sb8X`V
zy2a9xnN6RMHmS^LYQMZdaFw1h=R(s6`AZ?|e|Rna-nIAY?7g?6_j(5Sxnx+Ye3$Up
z(V}>LgL3!2$j3d-%8XBxSaSlD{XEOk+Lw4o?f2wb5+t?6J8Q|0?)~{UB))j3GA!92
zklxM`ArbLv-g<4@M7P~9cb8|1ZApq+mc6aR`EkeO3}@C3pAc)^?F&2YDvHy+4^7S%
z;z_PaSaPUWaNAvW7LE^x^m!Nzcn)(a95&-&Q1MysxqzY7YF5$g%(B^n5iAC~omp4i
zT;cq4htYrY*-n*_mrfqqS~+L>)ZPWJj_l4j)NtD1SJ$B(5j*}|6mM*de%&tdD&p8-
zuKra=SNmLE{lQ~R9?$Vyo#QjE^sW6Z)&239N~px~S!=Y9_UAhu5B@C0*x^0ND0a2r
z?l+!!%fE=6PMomiqZjv2FRqh$dsoY>+LEV#EN=Tk@BJ>)yNyI2-CFmWYyHwSdhey=
z7fjA7F5Gte_nJH0r*B2?y=>C*(@~+vYqI&t4SNo=N?bcF|MB#p!&7^?54xZ9PcAvb
zTqrCpE#k=@tYqD`ym9K~I}%r9&VEyo)SA8N-{q+un@e@2OMTVOMIJuT&C-$3c8>p!
zXVO8gH44)sW~48kG~LHZtNFkA!LA$A4@%F7yqezfb4%CD8D~Fj33|PS>8!B)oX+<_
zlfL}gn#6MS+)kmC+ecHm>k@8Xu&7xtzV>4DY=#vPnT6G^8_fk3FY2%Osb9rCd+p3i
ztj&>)(np;u&-l%WOlT2G*s^bX<>l2E58aGd>75gKD?{POLe|7RCbNI<T=aUUN%Zlr
z%dfcY4fb6d%-b?8Y}T&lGh=j?=JKl8_9=?3op_?)=e#qa0sV$%dyGWCcFy0qdhNYa
z*S`GR-Mqql&Y8xew@tHO<u@;G;=dI?dGek=Pn#}exA3hJ4%be|QaP=xeR`wJX~qjy
z7C$;8ocHOSN<8XwlPyCag(1Myc>nj)OMb21pY^|@ZGrEmMZ9Nw8O!1&%QPmQJ!LwL
zsb}iZ&|u~%seKC%Y_j%>P;|_%j7$Bt)W`Mwfxv?zZ>BFl*zxC0`ms+HlTV~Gsdolf
zhrM!aI%*_Z6Dpt2>N5HCh2+(D!((catPcz7Zrk3;VEkvRVa`R%oWmw-4yQ}IOxxDw
zP$Ss6s`|68(AO`s+%y;%87}+C2rn00IdRt!fxAKzql$g!2raCd&1Sdq$19-=JJ%Oo
z)ZI8$EUBjN`}Ao+n~&w|9DloFwamJQySN@n@*U?=S)&?v++y*XRkik-apw|z7hjB;
zw@z`!iYsv$ch}z8wf3%+cYg2M3lp_Bm26qKXq~g&DFw?@%t=<F|E=aQnax@EXN}Rq
z8=Y(Say{595Y)20S#j5zr@MsD?Ac>ImG{||RI3lS&&aL5=^J%6v+k_Vu4jE8Z;6)N
zN)|6$&AT-5X6uwKWhPfkqrOQPd0#zNs?4>=M$1_9$M$*LPv(cVrrB4A)IWcbUv|DO
zb4%CsEn0~)&Inz2-xxMiZ|j_YGcL)lXEt1)W4%-6?^Z$HqaxlLh2I`p_U_e_+Ix~`
zFTUlyBvCl)pON79xa})DFYR3(v73GN37I)NGFoT4?l^K>$g;QF?C$+Fd^^rc9OZFd
zd2M>_;qBcCIo!UDt8Z~$_3yoM@7$FKoL3*~y?dJXa9MTmitMGwSDENc&7HvL@%Wkt
z>;E5DnV#nCI_tEeWq0=7yki@8rzg+9y|kfwy7_@Eo3&k<PD;zGbKE#Dy_fI4Yx0CA
zhC2?gN?ZSR*1o`XAHU3ESU-7T->j#Wnm4^Ao~gz=3Kj^=kQJFFoBI3o&9C0K9Hc%k
z)ctJw|5;(*=ap;MtkIpCcKvK*q=Sax)xL|5F5mKG{L${;81VLwjO33s38!9&+}V8T
z==q0bcaG=IICjLML2Y^L{4mb=trv9fmTDc!eLd$@?xCr-wsPIO&vfsue&W3ie)k@)
zz4vnN#iEOcS7u~7+H5c0o!J?6WHX0>d(EZryqZJq4|3;sAAWa9&+&4%{R1O)VaB>c
z`msHo{BNiFD=mF$IP3px<1MdNXU#d5vFD+Z-kOOYAFh-?euwW-MDio4x?QfDKX;b&
zKULF=k9}Xc{M;VZfMrue7wz$~usdn#_h+IF*OzOLUpzaxU**)Yo>MD)ie5=gdaGzv
zIB{=Rbj>fWgVOsy_T)b0oPAn(<7tIYPY)^=$z9rh#r3D!{g(e1m2w}PVVJTetMAza
z$IsvW|A$LFpQuuHi$~xq3y+G$1u@5HrrWA(uI42iZraKvH04LZBiA0*XwHHef{)!8
z6%03B^4K7%ZN}uWOvLk3l2^Z^`$S2Lt_NpknXo>%vvRV!KciLiF&&St1s5AV)bnm>
zHoRP6=s)*ekJW}%4Xb9aOIyn|b>+47k^fmG?y5MR644Z1;^8rg!J%oJCkK-N0}Dg!
z?&8jOXFayRyUS8`pNo$t!hlJE<K8h-YsUjEntj_{<{3$DdaB8!k=?eY()pRLj8U<m
zZo`)smzMi-cU+jB&~$wxyV3;R2ECj++gP8>^PN!kzHtNlrN4V`D+~1$u`9*<O{ipf
z8ok6L-}Tj!wXIt>+`jc<L$ycWgT3wl?F#ldcRoM5-u(Q$njhbu-d`;~)6n)<--oaI
zE*cLW)c<F)>CoS2wkltL`mvx14vSojE~qp`1g=^T*rT-AX^zLtm4_CXU7XP4>%cN`
zih!-oM<>DbPX&khE$?)SEcIyARGZEAD@#Fu^Olpl9Q*%U7mv%a{s{C~enR4tN7Ia1
zd7dwC6utJm)o`I#Z@<JAqh;GS)dYF#ZLcw6;GE*%y-W5qPtl1^BWVp|Gt=VDz83fQ
z6!}@b-ZI%ybIXi#yTYO(SIlb>5^_**Vpm-e(8%WWCBRKI_ey}fa%;&Y7xh&pYSkR4
zN*4I)OI?`fASx?x!Ark)#oR8I2^m)#qdY_RTYGB>9BRBJ(V-INx69PAC#|PcX<^Nx
z%o}s+_G%nkoR_sqEmeP);OY$ry0UI%%oXa^+%(JP^_s><P2M}2-b^wwWXxc0h&j@x
z&B0K-I;C7e^YmJyxF!zfL%K(Jf_3g6oaeL1@RWkEMZzh&<}(j8T~7Y?*>b6j<?PuD
zS4whY_g)coVCst0Ua>>#;L#In8>4<(?C7yA`>r%$r3qimGZv2~A5Fhi>%`tNT+LH{
zd!acX{r!gEtIy}uKZ^|QI&}HmqJ_<2QZE*LzOB7~$@kM42N-x~eLm3t?U2<WZQ&<d
zL!%B>2PiKwGxAcIQRh|oRv^@6WA3*5(`LIerhO^Zk<6Vm(W!sNCPU8y|HbBbC~&LS
ztm>XNCF!K*lnDz?Hyv1e>4xRrH=l}53kyt+HaZ_7?Az)5T<M&q+w{#}%$S)N6o0a?
ziZC!S=rAxaFf=eZFfwrbXJFkS=x{;wU^7d!tVzd)2M2?gqNWN4Ec4;iV>-ccpkqRa
zTQ9G3m&i^}hm#ZJtj+FRSvGHN85jSgOG*t_Js4N#aY-!Sy!`xx*DNfuySi2cE_9v9
zCt78&;#?s6QCShwty+%r?CY&kcj;wbFLVuW{3|7v9$uJK%q8G1s&#d1i0ejCQ?Dtp
zJ9FQ1NvQH^#jJY#hfPgFBr9NED`%IhmDZaZp}i-);{J$gK5DBoe1B)Rq?Z4F1>tK-
z8Iu#9t+99R=UMwpA^W-@zgAAJu!?5yQsreL8>AESrbc>h6uhaHcxHxR_x%u$RTZ;u
z&)MP8Zx_3(w2ysb^8b}GrcND->RkUboR^&`n<zEg{CV#O6DK20$Gnf{>Uz#h=Z)NY
zX-k%<F5~}++q!($@NitySYxtGA%(}R?W*5|+-I?V0hMjS>0+x&7lw$&{dnM!yH?d9
zMtPdYqfRxsD-T<Io{EI{&A+Iw+-7|3#LZx(vW$g&b5$#pd*3bmam^!X&Xoza+j=fV
zyS-UyAye>0G5ztBkg!bEs^|$5pO#*mcu>_*qScdihEivHRQZ=s7HPLMBhL8AVbivB
zW(DYW3V7>nHI{PRy1`jk+wZB$i&m+)%*R0;T1zt)MOS`b=+Q5AOC&s|_3G87)7o0E
zEnB^yRXDo(uIj55<yTY1<4Zd|riM80I<65M5%_<a=CUnOtEQ}7A6F{izbEgTLYUdv
z>jBnGSClie{SS!+Wpk)%D{o$L(<ElorAgQ3Ze~x^4)zG@OPO4i>*2yBer4k6rKOXU
zQl2@mY^@9^|IikGukymG=NAhuEL$z4xlLw^y7lj^8VY`_YB%N|<S;W^&}66QBjwLF
zscb^OVk+C!hA!D9506RZhph<J?Ap{dL1BB;qD;y76<*GILN49GTX?iYkBKj;5k9W@
z|H$G@XTFzr&)G+<QJTQ=`9;AY`&U=4ov2><bozpz8(Y$yfBs8JcfR{-is`jYrE@3j
zONfu;5|8o<H@o>F&qdJvS)=7LhFBlRfPJ?&YESSeGcGaT&NSt}QJ|~1;Yl+`x!Blb
z&zW!2eq4BaBdCAMnx4i7R%X1jjL(_2+g=KDkKLuQ^zBrhqBQ<p%PoUjbhR?VC$8Q1
z>!^D3sqpCHiB+#+HZzAQ`R{V?`t?3q@!8c6bA{JuygK16@ns#aY1NZ!!C6;M<(>+P
z{d+y0-(33o`$Bo?Z};23%=UOZUsJhg?F7Ntoo|-QPAZd`pt;{K!|T?!%3|{n3rVka
z9SgJMBwO9KxXAZ&besP@b-vDRW8Z&)6>4l*6IjGu9667@a>zV+C#mG=JienR>TMof
z@b?fCUpDcKZdd2jkb*^B{k1+S`cktvs*^0GInSmA8{Pe6w?jPa!vrUn)f4*v|4|oO
zX3CaobLohL*W?2&H&oQ>6pw1ODIfj*;zLQqnMF%HCN9-2IN1ICQIPnDb^)U>Ud1YB
zjxSX~6FVKU8;ZOA`S_DmrrcSg(tBCh>4xIY|2(VOESSALYQ)XX)vhaEkQ}$Z<f!n<
zFIf{$U4FDA>#M|uhSuQuP8}1zB<=RQ)EcUEX=-`gQH@6ryMj8s?)8|cnAUSH==5A2
zv?*=o+Ndm-Xr=|JVMWbzQr1qad7<p9{bjNIp_LcSZ#T^=n#C;mVup6l^^=Qs-}<_)
z*{4_~Rpb1hfKPMGI1HwLX<W>}SK-i7kQDtk^HGM|=J|$41k|~cC!CNnK0oo~=1lL?
z>UN^V|DBf!s<?<sM>(WkF&3WrVS%d83%lRWqHTvZFZ3za@VmE2Z1UYT$<u{T^7=39
z6v+Le#(I`ZU}BqsjMbYZb1X9tYa9!@Y@w?%L2{8t&(_YnAM(BS^Thl0pEURU^+hep
zYWDOc!E6TGa&xb!c!=LS#=0}tCBpsdCC52Za@kQ+74ACvl`8~w3F(~n-Bx*ds(`xi
z?IWN3m@;C1WErvfGOW_c5>#o4>Dnama6-un&DaoyqY_I5+;n3PW+X67UUhJtYH01x
zlvOVl?Ve!PGC`s?Cp0PL<Q9t;NzH=Y@nI5^_8egiSt8`T!mo7tT#;`nyREgI_FUT2
zSNU%1G4FYwi?-dG{y(j!h-m|VQI?}lQe&~#J{{lfOO7x3^HP7!X~B2-Yt!$1`go-;
z<dUlPq@{DhEYf3USEx;2v3T3OBl;@$b(NhCWwTw0DV}w6PrUz$4|A&Te(=2iM@#Zm
z#@0vgSndC9>Qp=8u#ofC<aysjW;QU(Y!dmn?f{eC2Y=JZsXKJcQaEHMatit+3jKQ0
z-*QSusCh*vS80j9?AwC&%qzkoh887G-iEzwR~IgJ6<TItx#%>*6^F75&9Yk;xZA#R
z+&JMzQ`J(#pt+~Kx|BXYi_4DPFiACX&VnYx9_fSe;x$S-g-R)Q5ptdPE@=oo@i~&8
zRTNyfIWp|`tq<;BzH)|KId!gii|K!z>1!`&JhQoBdAssl$-&KYZyTk@mmWJmMK&;&
zZSD=n<Cpa9m}i}pc<p(+^Pyh9*wqD+)-T!3rx{N<SF-fCPQU`z&O?iI*TyQ@u5zDz
zD`e&}L6zDhkDx`LuC7!Ou+A{duDWpO+LsfB`mZ*=uKV%p`o3@Mwpm8O&rfYS&3jPQ
zaY}E_hs5ltxaf7;m-xO(<!&!{`kHrE&?lLDTNa9(Hw@poz{gFy^}6-f)vGGD77P0w
zXy^4}dM=s0bNbI&DPG$;PFm#}ap~%09O!Y)h;=jj&>(&1(xTtXL(NVVNM)5|EdEh&
z<hIbGz!WDD*}Xert!8C5S^v=G;1@mrPAlMQT+brA|JB_`!u>u@d;j#2q<Lg(4cn%x
z-<~X%iq>rU6Pz4%y;<%rLzQjQz5c2n$uIs|R9&2!_hq4Z?Mr9#Sly_H++iO4CdNS?
ze%rTQUZa0A;N81Lr7INX8cv(Ll6~f;+sn9?Y3+_xf4*_{u1!W0OD<<Ov>J!Z`g(nG
z*q7C!4bwf_-`cd<i0{*D7eBiA^`?rW#@jwlIB%S(?i+QY-{a$S{;doy-U8?D*J#a<
zH$8rIL;TVA;lfcCwXb|*t1fN6cV$A~-xc!y)yLBReN+Bqb8>2!=-J)(zAQfX=S6gV
zb;#^#8?5dGp3HRH8F~HJwXG}MzjWWrPWZl9eE+(A-zM&>jnBV-gK5UB|5^2)q+i*F
z_qI)r^IE1pY08Eq^Fy&RB}-NQmo#R_#L0Z0zU%b=*>ip?Jal#Yzg+nG5jQDE{kP#V
z9@{LEj!J$E^m~0&?#YyV#X#=AYDThd;!0wLG8#z|?S;bPN%9ee%Fhz5f|^u*G#Z#U
zE!xnecujbg0)wH2l9EJHQbFO&f@a+n&6YDbXI*Hv?r63-(X1QMqIaUnRH8{)qDfJs
zS>cKRtEjlCMUzWKi{6T6IgcduZv_HhjJ7;0b`WoKWiL&BQQ{Ncmhqw_b4OcRh7QvM
zex?ijEC>0bid|(3rMrsj;<vX?y2O^K9&+J}8IwW>hhxWziK#`#@_t{<*#qOQ{5KIe
z$|xwjP$1q-F!zdWPEuz|LRk1E#p-mQsLT3~rV9oJRP29VwY<G+$@Z!h%d1xY=vtp%
zwTh#AokjN=jpiK^&Bimj_uS}Oy`pO7gsLcko+B1L$0B-;c2pg_(RJuV&$*25^B&!+
zJOq}P3+&41z2VWjXlw7K8C|ol_ikXXZk{H&<7?{<vD&NYwXa@ed!*OC{n3|kqHn*O
zz`iztV@3V!58I9^wX-t{e~ZX{QKEWYP3Xe5oE3sGiU--sCQ0vHIN_p^q0vI`SBi!a
zrgCDDY_Au`KG~Ka5TRNxtR^@^=-egg&t~ST7bluO6jV)gXnZcc_CmgbTb0E1Fp2W!
zb<>-F|99;%-%(}NIXU9TWP|j{mX<x%H=A8nPG)kL!mb!)s1fD0a?0%+ligRcMlN8T
z=rA>GC2Qb>snL?Gkr$>$&72xqIrX5&G}p>$R-V%i|7bQpF>R(ov-yq6{Fk!%Kc`8m
zGgwHpRB1M;lo`fFB-^`}WX&k?I9}3rvn{it&8d7wfVj{f38BMc?MfeK?0YD_L8R_u
zs&drzoG>Au2Z!53FR{g{g!sFaU2+V~UtFf>7#ia0cWmn{CIi-|ZBi#5nopCkNO53F
zTr^wyq3Q`yb<gCm`5pq%56Ul1myez~<=l;)GbejapO|z0=bY1)a}P<(y;?b!MX=|l
zr*Os1xwmf4y{*X*`M;s(@C??82J_-A=LI@Udnw8JI&)s&gQ@Q-=Y8p%_k1Sj<H&hO
zX3YO_a^CNs^N+8Xf4F1%ou6}WW%iu9(KE+9sjE<P%>h1su3D>O*&ixu4;-t#dAwHi
zR_*oceUn%A=k3V;=^?OJK}gk0zQm(l@nLkr;Rz28*O?}&8~h4Awzcj6qb1|a_5~ah
zLX%?SADcg(H1VpT?9GX?%oej5I;3P0I>Hyl6)d;A%cia>ZnfZq{kLbba*eV_PcC|R
zJU=L)>!8K7h{&l)yQYR`E=%N`mVRn#(#d7nzm_F(F3+%9p0{c$lK^YE7DG1Iw78p7
zU*23^FSVk9Yej&<indeBbF5axM=me<U$yL+<noZ4mG@t?FmEf4>qxdzYs<P(;(mRl
z|MrzR6>ST;`hRK6EUH+wtV(!FrbfY!8OM}Vj~tR&QYa<&(aoH(Zo$me<`3H)MLS{~
z%94fFuq&=%QS4~z5R7!2v+-nE(d9KP6V15;eMKW>;=*Kv3k4ewvx*Da?|M1=U0A&l
zljz~d`eU4SYXn$>CM-ymT=2ST-lvmOKW0w-_-bAMs%hbs>+@DEuevqw+0OM}cFz09
zHUEFt^3OZxeazgzK6?Y#?hQP*H}K8gz{<UmEqcS(%K0x=_AXVobY;{KoRz)PY{qMe
zO-k0A-dXf3O9?Hl+B9Y6CYFW$>#8={AKcU*S>kDIs{dbe!uBXT=AD~aPR>-mSZXUO
zed{pS*Tn*Nmsz;~&P^3qA|>g&;3Q+n)g>_j4R>5@Kd`BaaE25)x!XE&rEZw?_{yZ}
zMe@dt%R8#JMeSTsyJ~9us;M7$Z7Z0)EOXVilGn@Ar017+EoWJ{y>i!f_J!L$61FvJ
z&kGP>ZLQuBD6pe-^^VTlJ9?ydOnAMc?e>mt>z(bhSzBLEZOz&?$t%jxRN>t8<R;A-
z?Jrl(JJr^Cvu*k95@pBDg%%}BjO{FnyYnjqwsZ?@souRYd-wl{-J8ATRZD%i659_h
zY`1D$ZO^!6`>ys}1N9VUxAf1eB?Qe6uL@Z?bJo#YA!dhW*&R&fn3xy$f2yyF0>f=B
zd+*(y-m~={-^%-*$RGS%I!J(#<>5RQ>G>aaZkQ;*$i902$CLX5AMF1(djq4){4cln
z|E@mpclP$m-}@`I_jBIaKlAm0|I!D4uihb0utT7MC4hlda?e58GY35c7(E&ox}+Ir
zc5k2PuwZF<I{&W)mS(k=+84g+sJ*v*p=r%w*{;Jfw*(Zl+SR)@8JDWs>u5UXC{M4{
zc^F)`K5%jBw^{Q&Z7(cdZ1Zbo^pY)ajh8$LT5?+?j#+~tRwqvBVMqYCnR}|~`H6Cp
zg7RX)_2FGokyQfG0(!4pETSf`+<CdZ>(+9f*(`wvj(<G4qjmRzrkV}qQYU(=mp5zg
zZ2W)vc=w$hJ#Tg}3GAG-=EM}6lhbr|a!h1hVqU<>tg%IH*Q(Pi7v=1lAJx_qd8*^&
zE_TDsc{^5pnX!uF;VE9l)1gM2Z5>bVt``2fflHIoyWe<D=_ToR?CVS)#XN1^vr9^7
zk5<_Q71={sYfhiq%c5w0=+zo_m9wX<&ffIt?5z+eQfFWj)DJ5ad~Cgs#c<!vl&~W@
z%CEKS`!1((9pw6`J+JD|1}^Rc|4$$I)qS4r&iU_q_6KY@&oYtaled6c&V>L42FAG;
z7-bL1+_@mBb4b$WqDR3&;k_4FCbB--+0(yl<3z?<Lz~*yFZf?YEKHrzC;lt@t;D9i
z(K-g)iyZ%)vO2g)@Bds;6I&sZ+RMgCI#XUA{<c;9*rBBdw#r<&rt@W6%np`{sC4xm
zE#5o4xZY0}c)d{IR&lOz*2KF{wo0z7f48|k;P=(QXw~5ECBHt)Ts@?7<hPvqO<}i1
zlY}@bHn=E6HL#SgxhUx?(AcxHY4`Pho}FB^C;EI&c1xe^^uFOCaFV_72Ft=59eZ!A
zV4OCqXL9~DgDnTRYL#|%OP*S?XV>yOH&@ooSZh7gY|(D7X}j}0Zq>{%^;{%r*0kr;
z>OJ<qCq$~u+Q+%qUv2ieGi9wBLdi~NZ6D6MC3ELykMF(1!MO>7i9+^quD%}+cfL9;
znUL+%DJ+zHK>yusF0<8~pMFoR+r7Q&|L+4#3I|zb@39-+W4ArXQ7gdt_J%^vA^AB6
zxp?o(^jzm!yZqbfi+$JS*@Lq$vLC)Ry+qdQ(t%?OS@I5B);zG^v&oKU_iv4vrjvH_
z1|C_oH$g~`!C7WeNYF!G!I?Ix{Abcs&!sNj_Ug#}$AXu)E*7|HTeCbk!dhAGp?P$!
z?3=|$r9U3^<cd?;D0}FWYk`kIvdp#MBsur3cXoSm2{^hc7fPKI%4as2qdalxC&lCI
zT;_DGy%?r9FJj%rid9cLbavF&-e`V%J)!SLd+amYi5CjBZ+@PluqNT=n-9f%C)}KE
zd#c3m)UrP}OW*D0TeoWUo#(5&Rw*~$YMOW2zVTM1;R~bxwx@;Hy@+f)y`Q^%?*vI5
zFP)tWGu+eEU(eZNH0k96&)fcGddpVsJ#W*o#Ba^j+qW;;_^n*GMxf*DL*>o~K7zH%
zudlGkP4f`kd6M<9_mi`}W*YzQxaII?UY%rgVt<Iyy|2FapUR%7(3$srXH}nhy7cQB
zh2B1G-%B^_HfgWDWNtHKFWV-Ix&7<jY|_8`-0I<}eHV5s{c3YlXV`b4<Rbe^!Mul_
z>X*gmy$G9h+3@TO8=lR&f8R%T3%p)pc}rOEXv!6i|H51^C%ETMu-9IEa+$QCr^V?n
zD&NzO8u9u5HCHwJciV1Vhcv?z<D|6;oFOa+IqyCZ50BpS*wOa$)nj%Kw_5*yRsQg~
zbn<~4m2;O@`R+*DeJ^ssjhgPAQnJ$~?md~^r;sN%qbTlq%gLLw{(edMvC>`q^y)jO
z*804NPk&MP<Jr%zyA>H9p7c>&ZmT-+vcTexLblHy*?G^-p1*a^lt**lxfbP{ZO@gx
zlilGXb<B3%nws^mYWoF>KA$}z`1-+Nfrj8utTW1gHI@3kdBT#&eW>p)3#0Jt28K^B
z@7}fDUp{yKJ`Ju(%5N9EK6L5(My1#XRyhxr^*^{H-hTc1C7yE+mT~<yVtOH|C-hRT
zc3;5zqu2QUJ>cKRApGqC_qPeV_u2nBX#XdQ|3l(>VbOID^Ez}FOW0mxUVMq^>a*i9
zKAUV+?x`I6pR+jLIp&D#QKL(~26}mt{!60g{IfcEl!4=p&AIO~1!oWYs(x;gdXu{K
zW0`Ws+)3Jj@eCVX4>&Zg&Ed@n^>)xLTRefILrI~#!AIz5hLoz-<5huctTrV@bjRse
z{eAV*dZy4*uc^9$vub7vE%TnG=F4}abMyRjRhG5?Sau1o@SF3GMZ(moWWj~S4wHYS
zoDxxA8Rpl1YqRUA(2H~4I!Qg5CA>0dQ|j4yroT<nLzc?bhHYIw;fjCI8jp#R7Spf9
z?I|t}TiZ2FH0n?*ue?}58uv`y7#Z`tKM@U&HJMg$tf-0HoO;e#pMQ;vl<~P!8Q%Ol
zt(T|Ho|DRwll$2C>j9(G|0kwAaBA{CCuH%FOX2D<{YQru7f7A=-Zs^i@hyYC%+jTg
zt-{`9Y+N0xU%kHS&hOgy&-rbBR6WU^_CGFe@6Ru9a{I(L1lRri@#Xn^aR>YQoAYej
zf8Ez#^WjNwwqlh?7r#2g?_e*jMO~|wiw4ygu9_dmvg3nO{Dwytrj?!2SSq@{ud7>D
zFuU-aB|~wP;0iJRYD0(lVj&xj&CgIdy+7`T<m2UK9md_fb8aZOdMLHZaOf*e6=^ux
zIL(B|^F9m5n>DMFRtFtgS=8lg@$kfhmTrT?C2M?+H>mh2G!+C*_Pu0b!Rved)D2$U
zpi>{Rcc`5I#k)hzT#(_yd=^m#%Lk_vME);qE|BtJ2s2sPt~O0=>K3(`dP_@I%zhPp
z%6{9lDg(!WdaZ(x6;6Lw%D-6?eN;K>ldx+~s8?=R$m*I!&(~`Crf!)!Zx7eTxvr0u
zj*3<PvlP=_HD!^gZ~FPHX$CB{XHG7*ySQdkpu*O*VjE)%e|?KQx+6lz$Y`0yW}{Oc
zpI>f1cW-U5W}}O7Z|s6?S-k4kSLvjlGb)*#v*4aZ#-<0(Uu(YYaALlD*<X96w}yJq
zF^iK<U-oRiq0+NcSbsNf?#guMKEo$h9odDCy>!|V;>NSo(CDJlWSvDPlqPB9U29pW
zu^=MVwZ6wHQ**J%t5yfbjL7t;`WHK2gvP%rY!zwslRNQWo>P2j5W||yQF#jSY+1{#
zS|@c0K5Z;XnHV0jQm^~zoI7^a59hz(6!*}S-k0sQW!=@=>U&iF+)iL@$#IQtxzE%U
z&6p&-V2YU+<K!5_(<;U0P0Vr{af}uP-_`vZe#xlwU6H9>aOwITo8_mr-DrqQf4O9`
z`KC`fMI!roO!KB>zs?d$bo|NYrLg~m0bjrdwxEuM?wc2Hshpm5^kug1>0cosua8=N
zUAg>*ThFU1qedo9T?3(3<*;inSBhyY5O}&gDoVC>n%m#R)Sfx5uhO6Sd`V{Q)VXo+
zWWUJw$zdHY^w<74?)f6DtNO-a@i<Qp>lw|q(V@CJX}3Ka7r8pVXk7K*!RO+<w^B<d
zxUw$eZTr+Y?^BCYiwa+i!^unBXU&|K8n_ir*r_OfWvTPIlr)p&f@d?!lqXbd^(_5n
z)sxM(X{y%Qlj?pxK9U8^(oGW`81}y8jXSa7sP)AQ>Mcx<r%h2(;<sHc_LV!bMrX=`
z*<aGt{cm45o|DKPej~AOn?ab&-;btK7G&9L|LW_FnX*9aZHuFX*6f~BO(INJoq7Hx
zANS=DnOBv<$eF}3i{XZ1o5z(mX^{d)S7%J*<mh<h<lW6Aq+rm+)5pa0eX+pxb8Oq!
zr?RR?M4piA4eAQovDotF%3E^RGn7{z(U=$~(sg@}yV$gYZ}mA86?qqKS)Tc8ZGP?l
zLu!dVPbTO7b14iuG-+N7^PJ<Z=M1MBPky#1LacF1FvB|&r8x$dceNZmH_QIuX^W#L
zG?re-E38yzULX`Y?ZJXRFLuWZ33DcI77z8;-TI-`F)ES0dB-6!w#F3<OT@QiWaiJ6
z)17(iT9D9-5O;<}4Nf96!i7J)%~xN1f=T4a2f?sAI$Z0MSVjJc&;NJSgnPBb35f{?
zT|Q~K)eE)m1W$S;&zZ}rDWm(!YN=KzgPfj2^TniVDrM6cPYD#Vd=X20)utx>XoAtp
zSv>r<&6CePREc|Kk}><1h`NegplMoSO!hMuHtk8S-G@Ib6;)j^*picEDeiiqzv2kr
z#|cL@*e5RjFMIbvm-(7wyi@1u$iB_cNfzaByk&Tybk$+5Dv6^ryPupYcbIkNm%`hc
zC9)@4<(9Taq&`X$i9YIl?V;@3kTWX%XPh#%HTPZO%a|f?qQz;8!sU58f|LU<G?jT+
zb{)I+SnRC=Yw?O_{QR5=P1`swvWR9*;r)47PW;gQkbD!5Ek`=4w_AAyo%pagzqhpQ
zz-q<B50^p@WKQ)ueDU(#3zxc%PgV5kT`6OAcUQti(;w;uvRvX`=0{FW$>aXC#~|QE
z=cXGsJ)Jhr<d*!xG-oGArhy9+U)TG{J)6H|9Ld%aGZ9ZxYAX_`-nwRm?wy2oqaBW2
zu^$T4)U^WFa5l)TzR;oO@}F_DM@D0sKbsc2BbV*gn&VcB4dk^K3C&eM?_>2((p78o
zg3iz_Tn(n84Pq<_`#<j0($r{W(tUMtzed1z)skiJtdf3d86KIL8L-07XJ+|zC1zK}
zGlr+O|NFRj^H)LZx;dZN&O2@WKJV1AY0pkYG#r}K_-M%q9lj0o>n3lnO<TJ!b>-~Z
ztKHKIA6~d>Q*tJ)!9wc7AA<^>I`!Q#?%{_HJTqlVI6HNd*zB7whFv>;c8bOtXSMro
zy&S#im4n$uwnR&H^={UF|9x(X?;izjo)X(Dc_)F*_{uWwt%l8-YSuT+Ty*Rf_c*Cr
zce1d!BrRL~!eiYfGp&g!%f((xG;16a*#Cd=m&IZRxtq>d?d)1~)k(x$n7Ncet~LCT
z2$!h-j*1LHR+blACeIXbRuSI9rNNOP`!9l(g~3+g>N74*+gGg`aVNQEPn{+k$Y3e5
zNPFJgKZ|6SX1G_@r}fXzU17QAgi4wCqPez4S6yzr?2}^6^l|k1I%C?i<ja~{zS>%C
zcr71%P3%=!sY0r;dYbqh2fsHfHhG+UdylIxp6&C)`npWvNzTh>hW!;Tf3{?<*uFV}
zFFn`qk!$0YmFp6!R-YsE+jV}Cbl0?hznA#3Ofg_RTN<}(b=kITr*oo3Tyol3?B-MK
zPNY>d?>buhPHzU|3EOE>;S<y*RUIf<@$20A3m?`B1|Ildrm*OOlghL=PoCae6UhHu
z()v~5)9|?^=^K`&_$^+*85%ih#pFfpVn?oTp7bE@t)QZoZcx_qB@#ADSIqxUR$dt8
zyTnLowp51>cL=}RZRKN6L=~L(WXDYrsd`_XucP&tOJ+0cgalU3i2`O9WHmMRw62gA
zlj7iL+4ICeIy8B=*f0D02NX*(dH7mPw=!{YX!GnWnjH0;Gfui|PZw{zbl24g8O>X~
zoZdZBUDk)9mft_g=`6`-prl+Lyv8u0mElZhe_^Y2M6a^U_CjOE!f2W38v@}98+LQ_
zF*)^bHCX47Fh43`XWItJ85?zH3R$*0G*}(xt4e6~Sz&I;tjECB|9}2MmAAp0zdLX~
zu$tiE!0_d<MDt~qCzqV&UG8t);QVSeyM@Tk8%3(~UUt?oXmRXV(sgr3-(rbPC;N96
z^9iqXiT}BHd1wFI6B68#0`EHdFI#o5UTiffP~h#wrHhnzt$(b?{z_oY2959xbMqI|
zqqgwgzQAkskU?*OlHQDl3lkTvI<Q_+$bj+kwB#+z)?J#mN@32y!YOWvZd`vj8I_mY
zAKYD{HTm>Onad|TtuDG92~aP!l#N=fel2n??}1Lv2OACxJ1)4ut+-Qm!3Nobf*S?0
z&4M)MC4Q0F&mgi_Ly?71;c>^RIR$g%wD-!rHY^n0Ui-5(CG$jglhcGq&Uq1>WdX+j
z0|oTzZ!CGcc<&2=`3*ix1233QnPB%gg7xiejy{3T={IKj1n5q6u!}me&->Jg1%(~|
zGv_tV@H9Lzb+XUq&==;njW^jPNvW*ZY&XmJ#jL4fE7|!I`CcXpPW!Rgu4}f}MTIMy
z6ao^v-|kX~R@7&=QD#}vwL_U%Pvi7v6YsJX*{vzwHzqdeCa|tq*~(rrLtL6Qt7hZ#
z3*P?^9pEvUz3G%<iQt)CNz=D|Qru;<DB9ZbkO_~@M^3dYnVSW@LOXajAL+bjv3t*f
z$)S@SWOKI7VVIJot?(prb5rpIUO_&ALgwHK8Tsf#sxlpB1y1W1Pnh1&Z&uYGb%BM&
zW8D`C=DA6S|J&VicA2uTw4%Rm0lW7r!O%)cMi0#wQmt<sPCg2nStp?7`>NIU%~{>%
z28|0w-y=Hio?(gFAke)>va)%{qe=X7o%0=xIA^N#-(ceZn4$CYgn++H|9g#De~$Qn
z&sl$?%d6Lm(^e_qR*mbqO)J6*n3*;oty{?(_oRuh;{xZ@3tX)iG+#0CTn&gj6Tp`{
zw`}&rz1nW+HM?_fsjX4EC}JBZnrbbax_js2<w4rZnYAaMu$WwS$GYOw*0Nhomo6NW
zo2xGThx4d)_l<)WMVK}QNzbW#BWCz&y~9SE=1;p<W;CA{>^QGEEyt2?N%k}*t>&WQ
z<{JjA*Nq$XBs$CwtXN(s_kYF?xw#^({VttV2M#?kIvml-(q&-j?zwbcXJ^!o;8)gN
zpAy*nf1G`3z2mv|s;G>W9y>jced?T}H6?8&`=rZE?y*u)9-Df8>{*o5={vJCD3JfT
z<Ea@Z`gu0V`F|C$Uo}g7p=f{Af}K34nPpDvdwOrtIlcAB)om>SRx=mH?!6E{cagT0
z4daXk-wECu5+xQmOv(`M+4N%7!iK;D5n?QV*6%HuR(I3kq@a($onv`N=WMRfKDuOU
zVXf?(gkHt3>kq8pO*!2w`9_;@gOZ@Ip;GAjMy0Ldzm?1crnNd;GhMuHa^;#<55eS}
ztw9Ei`WBbNSLj$J$<^POp~WNG-rVHz;L`txv+PWz=La6(^Iy{Hp`rTs4ZlSQr@2Yj
z{Xg<|=S1|L?Q`!mx3hK8jppt?!(!(uYTTpJEyy1rBGt_pFyU>ae+;(^Q{>+e<u67m
z3m5Ru-KyZ5vulwxXP8dF2IFb0t%r_l8bqrw$Fu}+9_3y4mT5ifZJxhTe5}!%Om7SQ
zxTWnC%@`4FEZec=Xm`^KuCG<r=c*?)HBP#5VtG@d%$d)(^&-WT5|8alU_GQ6qh%T6
ze(*HY;)~qfJr@gka_<~HbF=sSN4Na1omQNa1?F7hwmyFTZ|A|_<8q<L^Huqd7;_sX
z*r=*@y9Fv5GEF%$N4D+b@dF<74!+><`^x!`duDPa>;G-4mm3S3tr@l@z3k`O*#G~;
z+WD&lf`sPZ6W-~W$o^V;RoxBesjJ$<SGG^I5a`&k@<YkF8HJm4POw~Z>I~%C$!|65
z(y57^BB4_|PAzb})yZ>frT5g9l}n>O2t-T`WjPb&x8drRGgleEh2D7(T4wTq$t@vn
z&my*CKJ0A?I=jwr9`n&W!O0TH@nnZ|&C_LynSpgp7X|#TCrW#76zSuieN`wdabwCu
zp=aKkIVR^$O_XzT+%&`V=3B+0RG;}P61SYtuKCEB)P1H-@LJ4~&ho2^_H8(L;1H*i
zAs-)GSMAzKJaVmnT~y{?@c*!2Lep1<7SRU269I+AbIKm9HRI}9YH{R$s0aU~=B5`i
z0#jZ%H!CmwoyfJYi|KX7{+7>6X0Da^d%5MKWOL~R<Ev4-9I{L<`|yjO^fxWy|NCKz
z)|V+C3|uD}u3Zp#`ChLw^VyV@N4adGcv({~Fs<%ow@u(SP34k}-tsgm;pv5SzNs5s
zqtBRJj(pd!oi%ye-lzJ<I-}0TaLRT>$KAg3;pol|#b*-^Y<fMLEvMHyG0mnhE$y4|
zUV)7neQ6IBgtDK>Dn(lFevlT{5V%^kRi=sO1n+eTle;U6mTOAvDLrRe!V^>y;@+e<
z{kmcL<`s_ifd{U!a;3`6IaO_@zU<OYk15?u51n?}`TP!H&0^io8mGr_`N|?WH^Wo^
z>#r^`U#fd6n*H$)8^#J1-Guv(J0xc=VXX-^GE!bOD`D}p6MGwDLRb9}`60?Z^9TRL
zB(cpNi<=$Rw0Jz!&1#MkeK23`L5%N%g>et!c*8i|UU7tFHO@>}_OFxOtoxqArPM20
z48>C_LmX<(yi(2Gvvc;fT@i^v8m&@uxEcP*Y!++SoF}p6SkOa>*J5JcyC(?joENtA
zLYVBK*9YGu=Gz7di3IJwlz8|+V%1XK1svOMtl+)tuua)-(mpA}v|2IegO05ekInfZ
zlUTU@fWo}Ej%nQ&HR?5P=)GuCHEBMg(Hd2-pvzJ>aF3-%hs#`<)~*hYR_>ikBid(P
za+>BT_l4uY|DUmob<Rb8(3s`-@#c<`5@tbHr&J2Qyt(*?(5@AwN88^_h={qsy7yLe
zi(~Aas7<2L24SfY@ABS-rGET(>)p1epN~C_4SOGNcj;?fbnLgz@5i=oY0X;i#IyR^
znIjy1b&Y9h$GB}At^fSuWtirsxg>3`$AOJko*h(-IrOnROoR30PHVoFJ3Q9uY|6r0
zez>u(Hs#D-e(mMOqLSyD8nHt4>cZ{mXU<OF6mfd9fZW3f>l5F&g>S1fT%LRC@Wa|Y
z-E}?tpU>FnRGGV}VgL0ETSg1!E3Y>uH)g&t4E;V)GVD{og=%Ka<da(?J}s}h@5vbw
z*ec+8+JVWZgUKuLw!`TK*RKBGs`83CteAD44@X`x=d`)M%3dw|#%Y=VVEM9Q>+QvI
zZtnjl2MJy7-n$@8=&}-%dh6zMtDNpnW<1WBAj9SUUj0GC>kVzM*KAxUc`dt4%W*-$
z+^E@GkBa2Xzwtp;sNvE*E`_o)wVg^5S9-OYt}oP>#4{zQwK8X^fY{p&cUNC?inCPV
z-MFq*F1IlGY}kskE3B`YnI3&}H^Tq#k);*=t4^BvMfU%*@pDgm;;N^nW1nK`o4f9|
zm2clW=D#Vgw;c_PyS`cXeMH{Vu(s5){?re3<y%stE9a+f+giT;Y--#$&f4wYxBq?m
z_1<@@kEu&9e6KnF{nNTsgRqJpb@`Rkm}*rt{%0h|)|`r|-B_;Pv`F$xr{9eqTi7N$
zu79Q^_*0oNTQPHcgV1xX>37(<XH+d)c=ce99B*Oiye0v0hVp$sOH{6%Y~WGI{J2SC
zYetCm9ZMcgNxqKh+ZxX8aZt?I&HAbTm{)Z{tCD`Hxk21_*Ke*~oqi`jxlO+RBlo2?
z|5UHpMw<_0a&(0X@9SQ}`pVe;b8++o=69hx^%Km`2Q2k_wParL&4Zu!mQS<&|2y|c
zqJrLQXGso*e{)Z5R@-v=Ober%hy5RsSzk^r@8a8&Eh%<A@5ol=(&-gHzZ^K0xIgaK
zp#|Ye*N(F7n{)rfs*GjFBxjXI{6C|X*Y`Jdi{~EsEeGt=%Vhp*or&yVz3}_(lmMSw
z-HLUK?soP*<otH($g&f=p0R(ou?p~C_b%Yw^fh~h%R=7py^Z=*SN@*u{RY|h70dJM
z)$1$O>+9}6{dSDAt#8BE|D1LF;adX!FHFcdyE64o$8GVpg0A>Ss%#v3F{d~*GCn9Q
z*JI}udUGf<QN4-DOY+T@4@n2xCAlXlrcMZv4$AS^nq<1kceUA^9XXFS9evuSsb}dU
zxS`eAQID-OPjEr-a{mPqrb$OmEPOs~hFNmnOrwL#f*HM7eL8j>d~kU+Q+JVVL|Vf2
zR>j*K38jTo+!-@iOKdW>CERMxaG!WH#rTb5f7$nKS*i;{bs4wb>q~sj|2Xk~&$pVL
zMWLsBwWGJzT>Ts#>0sa7Zk(<+BjP~%>1q01`~FDkXI<IGa-)l9hrZ_og_Bbrx}+&g
zzrCxl{_WY{<(zj8?&o_R^vB|c5EF-%{*9OepI>m#7oS>p(W<2R%2wke9kshZzgxMw
z`#6V5uCx(%?q*f55}TSU?;ppjgnE^&zVY+>WTnqRY7H?A7IFTpq8S0crlLz0Fz^^j
z$Omz|N<0i@=M0^o9W(Li^`Z-s?bE9YFS_5?$l^>0cT_zj8kX{v<=Hf~tzRxjI)0V8
zGPC#V7bS+&Z95_&?f0JPnAyuNy7R(B7rw|8{i#=-C8yZfom_0cPbchYQe;%-Q~hrx
znccSk6S{6jc%^O&^peb&7RV-fkk!4Zuy9vqjJWERo3897yCRp*D}R@{e6H+|l>!Ty
z1Xj(Nt1NZpU`LbKlNAg-Qe2vGlk`$nE^V_r<+^Ml7i-8eF4wN*E1HaMX)d$Qy>j2_
zmFUw|t1N$Rx!`ZfshZ_9OU!G63wvP6L|4{8lQ%9$cd0kHpH%986Tvhma=N#H@5ZCP
zrguvM*WNt0>V+%or7H<O5B&r$2R{ueUANQ!>6c5bA3k2o+MY5&a8l&jHAjza`q8GE
z<oKhiMPE|u)b=-y(O<6Ka!uY~G%KybGICbNjNXc<2$AY*8}!#)y%eR}u<=lQoz~rN
zSM!wrKa)PCzWR-H(f_@rD`bi<T-7vQbdxda!kGuD<#TsGEc|VJ!SV6c)!|O!S5~i>
zB^Mi4Gr#TXJLf>-)$*Q8w|u%(&y=Mbyq!(9V!>s0+Zzf^sromrUXNS7w^iiYwzXNi
zCG@UnuRplam1&PmZnnT0x8hc%@IUJ$BCVN3@&vSJC0u*rYGS76FxlAco@DRB#G0j3
z|0TTSo)%vvsrF*q4kfFU33DseW@k@xkd`c9o1VU)<WOJweAOurpS`U*wa<D%iIw^0
zO9$hlvH}a2|9Y{=Q9ZV1%@(u##LtP=@3((={B^mY<c(sd61P{ms;eu{PrrV_d2WZ-
zMAr<ll#4auf1}>GedPWyVbh^Dk=(8SFP%G*(5TnCUuauFkeQy$Qic0b9SoB<Em&@%
zsvQ`3eJ2;w!8I(dfm;~6&WQ<0Z<=v5{;Off0k@CS7A;w-_mA^*;}@q5&4n2TtdfVd
zB-|EDiakx7UKn%u!z9D%4a;O@nteT-uE|$EUDlx*Eh!YKHb+&cz4K=0%=2Yxu5SX~
zr%Up5pIdigvqsuQ>wAI^97~SvoqzGI(%Z%U_h($2nw*%e$W?jb)3fxwzn>hpkKUc!
zu%>kSmQq#6l+TMVTyk5u%`=gy#U#aVhuN{#AFbhgvM&1<G?^Q4ZKw;?@#<umwy1vt
zhmY*z3-NCj?d*QCLLqR|nQ3Ormxgv}`fL-Js~weS_21-Tk?+37xpOR3)$RUf1~hz{
z{I2Tz>DiL2EXyy2{@9|bom;ErANx||)1s528E#rS%U3-!4Gi^N8asLRE*lT?$;;1n
zr*MVbj!6sD5j{R<i+=UHoc^kNH_dx3bn`AGC+C)Jozt=8yuyp-4u)TvYmx+C*S<Ga
zK3HZXyyoG1jeoD@Kfjo4X7Befnrji~PPq=fH91>)|1Iv%-z9WjcZ#E#+(+-d7bc6D
zWo%MdATvjedD%8!F_wZ;Omhw@HBU6`X-X7V_T_UttbA?9v&qsbuCKa0jPA_PN%iwp
z5-!f%&VO`G&!v?&#g;b<zWewpBQHW&Y_ITky|_(l(rjNlUaFXARQ7*q_$BRImv$7n
zeR!KYH*ZSvr8k+1Mo$F$qkf-{omVEvKGQd_sYd$tvH%ZNjWQ-LgC*C#sZHzNa%f_n
z60_fd6MOq6JXOkCIxjX#%iy3Si$SZkOuLz5+quSb#mQMcFSVjiH$Hi%Ez=>aSA5#~
z7N>Ffp(m;Z>FO(QKRjb4-qmMWo)o>I;j+`7NlUdA4y(<nNma`0c<0#q{EWA%|E#Ob
z@s>y3uW#?#p7}6ax$69GbEYzms+*T*r~g!&_jaY>x;t9QxtBuP>e#Yv8K*8@|A+I!
z8@VO>r7xcJy5TC^XfdVXjnMq<J&)FMdcWuw?ec1t_~Fny%e#qnVw0nT&Qz|ZDLelc
zD=&=yV0viBhiny<n2von7Tscg=-QfTIm0pT3fKRV90P^Ly}3OaPwv`mO)bn)n$*80
zWw-Cg)H@FsiNAl{c6`b~o|v8QrtHsR51-(wlsdyGr`7G~1~)<ds3+HE9CkW+rC_Rd
zPU+_RC5r^+nk;dS-D~Xh*dwmxQ>D$;B=7j&r!F{(i%m$ej#l`dF?qY>rj0$ZZ$#u)
zw=8oBW5}B@+fncCy{881(H50{HsLN&o2O;yD?E9!g8R|QeLs%sFP`Bb*j#+u^6AQJ
z2k$)9XZ?En=2yS#2dq0{|KE{GJJ)rC*>Up5hhcAwT(;kgkkjA1{LJb!VV3;sq7Iwi
zZrk(LZ2Fu>PyVknSy^e5Q&)Xw(xJ<i6T|(AC*M{-&MdmT>w6~SgD2-XCO(#Yu59}4
z%=ysbx$|OpCrx-OShN4P?lf1SfAOuxHHxgND-LTb6@GhIUB|T3;@HjB!k)!$6B>1+
zn`R_UE$rQKSlV<agTtCnQ=~7bo4EN-|2iwI+2r+-Y{hPOr=l#zBR>>u)m@`rI8C4R
zt~+7fsh3$XULi&vjVH96&&`=+vwUU!inOrjd|^9_o*A8A%M)E!9C50Ey?LLKbM{{6
zlEUU(>kUUQ)$J7hKBGr!X8fdg8KzO;54rT44y=0r`1kJG#H;JhU%Q!F>EC;|vG@AX
z`_a+-8`T3AslM1B6z9lfW%BGl=jpY~S5uffYPTdi>t`~0{OXgaTxqY%EwZPQRpTSe
z$-CusN+~<L;#dAVoyjjG6mhv~`=O(2&z#e(ndAA&>B=0@U8fba{@N`QS;OvCdT{%)
zN5SO^H<s#M%sCn7{P4-m=!1FN7oQem650{*$lO`-*jdrtk5oTyJnnJn(fYWV$Ioud
zW#?jG<WcJRad+Q^;P?Mu-krPf<hK?44o7)^FFk+oukeN*_P;SQ>N*mPCs`PC?|eVw
z<!=$O@aB>21@m_G$S_N~w7Ljzt~z~6)A5ui1DD<j*1qGct%BhvygepNdobZzRE<E)
zg`+(SFSu`MT=}=vPxMIlhE^6G=}=oip}ON0IsawdV#IT7k9DpUeb5<OQY$&N@}{g#
zkmj93;TNafOZ2!^>nh!IZq^j1OH-Ya?=Z&N&Ya;oGb~k<r;lAi_K1v?yX-a@xgypt
zt_y1(&aC<n+rH%D!-qF7$;f?4;dXi0Ggo(!%Z7=cZ$t-LD9OCMzgP2K#mS4UKSCQ)
ze6R7GxS?{ZMdp%X<GmNQ)0ig2HM>lk8XMI%;poy*u|I3hy^>*heKP#*-uW>SEDw6b
z;yOIzHDonC9EuWRcP$LwbaVE{#_MLO-t%0dbsI(R-3XG*l4q}(xq0WIEs`ypr!p<R
zamlsh;MS=Ja}#GX%xJo!+kW$=n8-TLX^F}rc~ZYGvhNP{+Fkqqws9hhiRxq9UD19P
z!W%`U9fFb~|K66G7Owiqi}6*si%G)Ig4RPj&)3_|aqRWD5+marso)kC;jhx+ntH7{
zAYj)7g{(OXxTdLa*CpLp85?nB-nqLE_HyuPA8F<I7;}5&(ftxE-Ybt@6rJmF<%!w`
zM**oLSsNS$lO~81-nEqT{hJ%5m?s*n$1WAN(7$G?ed5hzkEKgP6=xJihV5dUYa3nJ
z8Jo(;I<3%ACUK@q%yrorGt3St>+MsGnKwgG@A&E1B!zhrYgByyzg)O5;9$kYh)V*G
zE(xeyIC0-u;7FpCzxWlm_AT9)edhTTD9NSHbD7DT6uV5(GVD~R+{?^m3?BavXtHtC
zSWNOiackLKH`T4KG3VsEXP)SKkf1#$Z1usTEEA@D=Lp#2I&tZPXMPXT%r{N{TiH5m
z4QKwMWNW_`mnlq=F-x}?s<-*f^j~t)WddXS4d2agkGvL;w^DkZwCj*`h1Zt5=ce1L
zmmTaY3yGZj(A7pPCH%yErAzI<|N3n3?f?5X@}HpEg&MK=Td9ExDxY75|14?rn{oaZ
zM~1^q(PX{j&I>x3?Cv-$b^96m<f!S5+(?DiC~tNx*0#h5&ZK)Ru`L}N@3b#nAhcou
z(-e&k-V+(CRN_p$yIB3Q?j-1Ki;ZnLD)HsPg{fhe3@3?95Gk=*D&nQ6SSLC0o7c3N
zj8R4Z&xcQa8r~wg?QWB!qL1W+b0<@lYOi=At-xDY(7r5kmgIuD;p<LH?`WL*ljWE0
z%cNNmg%KCCq|VNIsu*`ivF_}t^qpSYUL1>MN}A2PC|Yi@<B9oB4{l04RCmqtN(y3e
zj9^JRcj~cZiqT(}1(}bo{qcD|vG=*DnBu}D&9Y~jvyudtDP2$T(6;J!oWW#SbSvbb
zr*4(lo2{8EGXtCoUA0>{SO0a5yrMQOQEczTXx*l(>t0Q_&}-LO>A%JCdZ}-8anw=<
z-T1aChrY*3OFCq8C(SOAl`dMBoX6E586iJYDR_YwOW9t(xh5=&uK2zEacS<E$CjzV
z<%QGD_O9Oj>+Hv)|Br{<4QpjvvPeLyI&5O~lW<+l7ncOM0wgj%usr+aqH@IYMWm;O
zTiTq4k`&k8?juc4P8MGA*fi(V)Ra!GgPrp-nq>l}FMFco*-^zJsQ%QQ{jQwG#-^?(
zDUoYLqMvA}NgnB)c0r9*Rb#7D|Exr-Us>%snp0z?a&vAj6=aFn#WY2(clDB!x*1Q!
zy5#ScIJQ+h%{9=J(qqrPdnlsqaZ!x4_=;3<4HpY3-AQt(elGDPNfQbZbg!01UHN!v
zt;n<JW6L8;Md$8ii55A*`Bb+iac$7Qx4}`5?(Ccx|1GtiNiIrv)g!Jat2v&`ewn()
zW8RWyfmsFj+ON7S*1eWycP**%|FZN|362eEN?X07rmeJ}nq7EJC3?rH^)H@hZCkeH
zSAg}IWsNxpCAK{-UE1ytkvdCeb)lAe^o~Q*R<ch!8KkVp{cazBVI<e)m5Or$UKhE%
z{%Yync4N{q%h?unk=hUVmM^@$L_==n5x)fjUF#LS3IylBS&|)iL-aROLc|*NtTXD5
zriR<z^xFIM{+B1W19s$xJIMWKn&B*S_Uj$V#z$uk{L}pMMZ8&UP06(Qa37WJ33pmw
z^#?O}gkEsr5Ie$Ax1wFwrK3%Q%Zu?v(VJuQ9(9Jf2KVIDA4_*{@rm>ayU-)xe#133
zOd@82YwxZreR3`5@){+#F`4<f%!r*X`-SWLrvIKZ1oM-R_$JHvN}A-}%9?+^Az7kK
zHP`cY__~w+OZuWUQ`bvGUbA!$;k|8nj*};kU3inG%%((%yNR545^D~8j5wM3Wak}!
zpN^EbM{*ymUS^Y5UJ|;(b;>5r)&+Wx4rO(@?wPo0@AJB6hV@p@j=HW=5kJ%6>ajFw
z-KorH%cech402U(R?C>DwA3&uTIfhYRfqor-}R?Xzgo5I^}V}#MM2>?ht_OkYcJ?{
zes|sGqSqT_ULUno-hA_5pKSESOKi&}b*H~<xhd*mkrlntPx4*O)Vpgu&mDOC`i0Q?
zm!5@;*YywH|KaDgl0j>!$SI*mi!Iw{+iuI=-gw(`v#~ATJO7yfGV<%6NnXrVIp<#|
z81qd*MOpmRr#BA|DaIW7C+;G|9mP7+^%9Glua0xLtA|rS#F{xSn`OB2)^u_?98W4?
z-OYXLl=#V0ZhI44r0*m|oiK1aB60fI_2YYQXfXK1y(!2H<kk$x%*(kMVpf=MV|Yz%
zrRhDU+!+t8PYK=qvtr>6p0yID*SuJ|*X2%fy+7+2%jBJ%WgVwNZ}!|Qk`0S5tv(b#
zdsq6Kk6hPXuC=;!ta*{kcF#5Mh3=QNrVmyXR7b@<6gxY6=hm8{uVMa+YIWb4^BFPy
zXNmV+*AQ@P%atRyV_q%t4B+UNOBXxBCD-zzZW*U`Ly9@a^&9+WIlEuYT;~7af6E??
z_PjIQXIE{BY`y06A?eMnLvL=aYV>F{z9+W%FRPL8l<VK}!kyOonaCGCJGbWNG(FW*
z)p47Dd<whzQEb;+DXWWL<Gvd+tzR9^e0I;i_Z3^-pa1dx&oSfblg5uvvH#<gDZf5<
zF_TX1&-n`#9vs}(&1j^MHASA~uz$^q_g+3LSj|gVt+#kIWpo6ZbS_)qIOmaTdWm{4
ztJk^$cXPc>Puv}6m}K=xy*YLCo|*;Uasi*}vmz51rz#cC5Ng?I*LF)#Gx`HpaV?j`
zf0v-D6Vtj1mrt#jy!6=2I?3OfinTj-gvsa4KBOvJ%piJ4FyX4f4TdL?*H86rPXF@N
zOToQAcJlvC3BP#b%D-2=UH@l^_>I5(i^Zf9_{)3}R!wb=aLvw2U-y8eq-gs4@)ueT
zf^9`Y8V*bnC!D6N&DzcwQG4|ByH~CuL2HgZU9I;c^1}vx#>C7*#%%u8vrX>Lzrwrs
z>o&bqu{_yd28G53Q<t@#h+TW+Vv^AG&ucip8Fhs&dRnwP+RDD``yw}srIW4Nt*okV
zTU744w)L~M{Y<^wIm+jMsEW!QWQ?dYRH|LO+=c6l^QP}r)*tTf{qAJ_rH0+Qwp-&^
z(i@KKH#4Sdxasfa)US4&lN7q3t$q70cWdr9ttI;(JUMeI#7C|o?oCWDlb_E1W${-s
zHobWv5XZYlb!t(gtB30UuTk62+|-l2cGOcuGrl54wj$!;q)m!ZyBakERHsUINta|R
zuU>vft^A-^<r~xTL)+sdPaNrWR}1*_!%Mi*@tW7evV+GqfBnJuZf(w<6W!Yr^n{AO
z`|PeLjSIE@F2lWEJ87|0asvN15g+f{S8O!3%97=eoNM2C-gz5~N<#|!-kI7C40AGg
z^@{~o_wC&tD_h2r{!u<=)2TU^6C4+?WL?*C_c*xb?WH$={NA&Ef3N9&J%74b<xEqy
z=O&ShB`uz(NiYA}C0{zVNispTM0?MJXMfMzm@B{jn|14QO;mYn)`KcnxqzO<YaYDc
zcBJ>w`S-Po?_%n&JH=^Ne~)bRymQd*|B7|fL$lYlY^dMoY%iAld^?A+S$*-F^hN*b
z*W1nDb`bJ#Xs8zJTEFDvJW<UVTYRn>aN6$ZJY(`IQCgfs`Lx4V120+Wce~GDYL`w~
z_<!x?&4E^-dc2dl4&1QZ7`5+@)b6UD%L~7CPgE#x@6A?vcF>*o^O55-_s?IYSIe?$
zB75FGwab&9->jEZlRx%0&OGZ3^BjkW4`;2#9DF7kIdVES1&c_CY$#}IYGsnKi_vgo
zaP1Zl*6>)QkbI0mYB`6-=Yl7P!jxR6DF!!jb27@Pc&RwIg-8Z*pFDD9=j3$%`PTXO
zzU*A-s~yeccWTO^L{;%=vSLLq54bj5Ucv1>Mf0P;|1`;PWy>jrkF(ayHd5VLFhT3<
z(?grncNe?}+IHo}L|LDmnxUnu8g?yPRaJWVn(oc%A?qhq?!JEI+FF~1UM*5T6;n^!
z8>U~ADc;0zYJztAwV#$Ina8b{`#s$>%XGS~wczm{-NJ8ax9zfDKlxHnerDo+%fDih
zH9Ky%R!_9wb2FOZv!DHUx&5}W#w)MNEb}?BFLw9AM%Jkw9P;}X3AI00T-HD1U;3o8
z`-{J;Ty@h}JtHUSR=h@tQhm@b)(z{Y{$~<-uxt*?jVJ00{gj*!F39oJQZq7hc)8`c
z+@S(R^D`1_mQ1<#!>eIa-R}#%dgqKz>ux-8Dtk##Orh(NO<^jTE|NR{zgW~QaC%1W
znge^ZRF4X}OYx|_%6Xlnt}32nq^7Rg^l9OZB}aJ8UafN3blhm!gGHzHwtooPHa9MM
zg%9I78{=)J%X5sij09c_F2A$OVGXy|tRCT|{Ij=Q-0^+FAJ3C_?PM~yzX(X`Q+KVL
z{Xvn1w@1tR1J9x>L9J?C9ZHO5t1^^XRbGTNGO&DDaV60EsLzS*9y89WF5t;|d~xmv
z?vTxaLNy+aXM3DhiE^HfSfzP#$0r@#iLZn{-7soVS$%WktpFdFgRhicYm3d)T`an{
zE@aab0dcRjC;3iQIIFBmx$<i5nw~S<?u=_>a}0PF-_h}&_ipRgZ8fEj!sF+z@ai_@
zJM`a2`^>6ybBxZexUuHc*;Pf+x@vwZ%PJJz*6yj;<s@r*L2dsG)}I&781!*ooGkD3
zCH}U?v%E{3`;<KxKWB*X?`^#{Swrc&*rS=3=hq&~IsKzz7XQjWVc$A?0%h)GUXPJq
z)Zb(5*(spGx$Qym^35uX3XkPR?^M0J@sjRSjfqFhE{R$f8NE4glou#=@4&gM8>U3Z
zeVM>*{$tt+SMwbcHuD&>@z=cOF<<=s{0x!wKLzu*=$XB&PmEeo)M@u6=(B#Z*AMmc
z%NM`e+90cDyYF(ZuwJRXdi>0t*8^{LKi)f~#!ua)d~w$HIYs$?4EO5mSDc=~?X{w!
zNyXrvQlR;%6SmLK|L1N}4)I(Y#}M@V-pbz6*;Szo(+}|5UAq?jJYQo;->ReGOPZc8
zP*a@zcWu<J6Dz`0Gv>G6?+x3g;vKd3$tPv8i)(p`4+e5B+B|jR6b3Iv<?gUu6Q*vq
zaMKP<e2_dv#Ie5fn6Zq?G=C4{xyKsYYGiq)vj?u2e0<GP(>0p1xpHjImo49DnZDt?
zcp}_rQh}1Xs$yS(SJMhVmW%omC-oikIyUFYf;SzidwiqRROMz~%t;G1>G!XBCa25P
z8_anjiD|?4$-1d&S&t`QD0#!Iz##ci<=@Y{m&KUepUzO}V>&REA#0+@SKT9`OKuzx
z>pG+x{=iqIY=Ug>o=>|M?O<{=%=&-$Ucr>fJ48>4_RE~y>h-Zp@Tx{^%nN}DJt_=5
zy@m#12T$#M@<ilF&Qf>nm`mEGkJs~Zbxqq?b3*6$6mP}{L1xM~?>u7>n6#vk&o`y2
zV^)%d+QbK$(+f{soG8bB_Jrj}vzAYD7Yels-Tvv9ysPlayRxPgSNFWBc*r=F>4Iis
z--1&il93(~szPp+zkHQ8d2#w>Xb8?Y)1qav(0}cU%GH+N+?BQ-5?$Am)E|E}m`ggN
zRolx@IOA@H%Gci$|K>e0kete-_W6;Mbjla|(%z#yewv+KK2_8BG(5FKPRg_`+cb4*
zTIw>XJqy^vtGwClG#9AtIm;2eW7DzL3(Nht#60-VQZ~1F=d&5pCZBjApuDSQmZlc>
z;#IuT&kVYKdD?9cN*otkZq>>0(jwJjN@O>ObHm#uDYsLSqN=$J-UUot7q%uzZ1<0&
z^Hdc$VpT$ge*7|?f5OWzMTDXDjV40_(-rk+v(L%Sz7_Fn0@J4G4N*Qhj$OJDVNPkz
zeY!O(b$kE4(<!^5vWFpj+JVleI@Q+GJ-n>8Ow8a|nP<3EGxPAIuyvOTA_AAXG*q41
zp~P2|HFw#CoMOu}8s;<CfAtO7|HV6z&)k?>I@U1V!ukH2BLBO**___ju6AxTExRdm
zvUtfBY4w+VyI*~k{V?msZoNRRkk>KK-9J?CZ9j0?tflqYyeO~#&eIRL>~!?hDa{q0
zGGF0}*w+=V)?!r=nmLKlCuT|S`eL*_Xv1r5yH(x0)Lw|p=?a?aFIecayCG@Xn~>(+
zDJQwtJ5A;JrLtrD&Yra$UcQT-NtoKoo%OeC4%xav&Bs-5*^_lqXRgP5*SfAcmuESb
z!N*&#R+|ZFA8+coru^3RpwP{;(^)^r7-$Qw+wOgHrl60szVF3#A$Ly9-pO%8;e%*T
z<J{tvObO{{u57dVVtG=slIiAIhFJ@IColGsS-rqo?@P}%jiX{q0z}*ozZQFbYu=uy
zZm}OZw`Oj3)zZ%2ofOKqR!e_fVgdW{9d|B#oA#|KbxP??tH^Cu*S5U#oc8FkO6-5-
z@LdOdK0lW`rlxjy$A+WVgd>*z%vMQ#>D+9jU$R4T<zAJI9glN3tlUj)|D;*(|Lk+s
zN$A6bf@W*AJnvKATzG}wSL&<V_V6WN=Fzl1_c8OC^#)HVnTN@K4{E2d++p?CN!n^4
zaFo@4qc8Km;J{bA9vJ9wZr*WK?BEgYBFnEYdutD!K6zEb*!o+czeS(QkuRz%1E$<4
zX+0T}8n(13axQPM@nSJ^&zz#mF5CPz)m+<LWvO<4W>N9OU)Q$tZBNZ9+w8W9_s0Hf
zk{g!3T77xr_b&&W{0!~ypKmBpT5y)(#XatqyOP`;yPQuSKks9FdHp%FFc0TBu2tHl
z2THt?{NHc;KUHkY11;B13!Yg=b<%EhO8R_k+osIDh4o3a#^J8=hiT8A9Vv||OzeE)
zWa#<nVoc<;YuT?=0~cP84f>Y5=Cy(P;TdgyQk>qqvp=6#<}AD0^R4`NSDLhCSn`4H
zN3}cW{{GWt9s2N_Ekn)!w~IYjh_<LYub%TtXllrp^4PrZub%g2D1UQtbjv*6TwglL
zET{3lV)WH6@wm(4>s4d+Pu;k7cLLjCmo_isqG_@7eAbHC8bmj8*Cd-Aek`Bu_J1a4
z<)w~S|GT(O1Slu8<;;m`UKy9Xt10(n+G3v>tIV8|+S=BdF||EQ@Qsgp|5Z%Ax$Z^#
zERPq*yi;q~%gT~xi#@XZKTYFB3jaY1#<VxD&Q8cX_oV0CyElrzy7rqqo@lfo<4?|l
zvZUT+=QncjYRqtRJjNGR#2T)#Cj6TRLthim5(dAN*O%Y*HF+G;`L~|u-GTT2UMqS$
z(Yfc~+H*L2M}dCU>KLDc?~imXzxImvuE4KH0`Xs1ug9#7zvJR*b^g@?{+S!NcC$3h
ze6_@rWunTwm!~4!`Bv)O6l34FD3|%}iw${?)`ltD*f@PR5Uj3y|NQ_j!-=<L9IKZ`
zE!FmT^rPwc{5012A%Qg!g2oSW72^aq$*HKFc`n+!NN&@6#i&GK6@`L6|EVE9Q?yR}
zeYW0xmr{b8(@!mrzkWybLsX9bIKk`l(ER_LrA0~}4EGL*C^i3^vnVQq!<prR*{^&V
z&JQhTCK@++9sjaapYO5NffR11B*wW}a><LH>Meh|D9h{3#6*>(T?z)P7&iHC>v~@E
zXm!=%j}=U7S~%9Wt!O!-Gbbhg1=A_3We3vcw7)#yq5LZ6+$SB5J!j24-l`WpVt=Wz
zQOKJk_vrz?EMvDuUXO*2Vr#?29&w&@U{c7tQkA{i?FnDqV?Mr@OI|6do_VB`=ji%N
z&-p@Yn}?Rx6Bc(a4YzGeVltlZ_+T)7*~&te4?DiJT-&fI&PD0-svzf8Dl)q7o2Q(d
zzDQeFbfVRS%rEoaw7zNC-JtPRpoc$lnfeA#H6>N`J01G}8z+3bryaDTfoE9)mtNs*
zfz9VW@%)f5W|RnKu(>F_Tia@zyQ&LsvD~6ifh8gzobSm!5*0kt%dlB=E&p$&9&yiy
z?!P#-qqJ2|oYFhNu;kRCNCmw`cBkrD7W3FKFPGCRkn_;v;?R1NR3Mk3?c^fmnPT;e
z-}8jfL8c3xMLAE{mYq1_A}6k-VxvE!kJE6)=Y*_`U45=!6rTA6@ad^O)1AD`d{+W<
z^Yc@Dm()EBSiLWl2Ta)Gw;}M!XUh-)rKbg1DbJLj^{|;=VmtU^?mbWT%BBq$Caa%l
z5N2};3kVV~cXE33SnWZs$$Mqa%#HKR5;J|A)KsVNOgW(KaGLj41J8eBo=Ah0AI>cM
z!FSr=j6t5uBfb=uSrfQEu;l+UYte~$#rH$QeWRhiRm;|Q6Lx34^~ldip62`6OnG$$
zQ)$i%&$~~)ToRfgQJ`yhnE&%ivl*RjJ-thL*abJ5tW#i}z2(@f51sZBsV{c0C>j-p
zyyE#GHd$CTSUqb>ZO|j5{Q`~^Co-m_71<c8v78kxUv%)(3r=1arl8M%U!0HJaiH#5
zyKu_n#U~Ev?fGcdZWJ5w(mX7uA#aJKyOVUQp|%T?)}}{CgVwB`wpr%n^8amGE0mlc
zTjiZ!cO=5gePv8C^8uIei%XoIZL~StV*OR{WQd+k@7AYHy=@tJk)J;b);rlow*UOU
zg-x;biJfVi)~jGEGl9}qmo4QEt(~&r%$yGHf0NHy6*-(2{b93d|0^|b(U+?IY!3h4
z7^@3~i?qxTIBRFKh4ZqXn#LiG9c^w&M?9hqPWY2_>C}M<RZBEPIJLwVUDA6Q`@@aj
zxA#56vUgs*nucZSK5wPVd7peq{UmnYqw(}dFR|x7*R*$kQvb1Gb=_hw%XFWws$Vu~
z^Y@?Ky)Phb{?t7=hXpU2R48rK5j|siLDTSaj-T{rLuFy3moNOaHmmj>t^RUpehZVy
z*GaqBww^9IWSqTFj)mb+(L|9d!GCiMrQ$bFN?M$Ig)PPJbd*a|Y{_NaJg(62oMSm$
zO&(kQqk63VhnWS~_z8x8l{qW)SY@%9)9S4qI*f-?xE^erAu+vh{br}Q#ZEryE2nj}
zZ3wphYj|vx(~Ki7tL|NT8pEz$wb)U*<Z@A%l3dZ+1b6%B4S{aQqGQc1Ym}moZe6b7
z#$|CRMs#wyWe~%)sUOd&21E)vK4Xqk4l!0TsWhL$Yh`kQ?Uh<oV`kE(_{DCV7uZ!7
zuucj%B+AvlwdzPx;HoLhmTahSx^f_Y`YXQLwM;UF+6(N=0#dGqz4^fZw%mI^cd2(1
z*M`OQ>I~LG0lMo$(r24&`mo)5=E)m5t3Po_cxRRFKEOHu!j?d{M`{yz1DQ(C_FOZ3
zJf(TESytY2^Airw_OYw}|JG^HcU{%Axx%oywbV)MaOg&^t8#py;z!H4>r@uIaTx?H
zni{ownUAOxC!d7tnk(x9mj|sm`c14;_uaG;h5y+Ej$fPDY>;Q^?zi^FL|NHoOP$tq
zd-GYBh^8gD+2pXzJ)_L2lEC%vgy{pr@W&U!8$+#Alesm@Hw!#lDHk!n!&HkS&1Uvv
z8*Mh`o2wNH-`R?@e`-51K~e3N%BIdenzr%R&v<nxZPjJ4x-{u0*JRsG%JoW7PM;>o
zKQ;Yh!jT&BdQYF#hs7M4+TkCjiX{s9s=QUa;$9o&$h9NE?p{+(VCM8vZuM%(D|J8L
zo%>#UI4JdM*3&*o<7d;gCZ74UAmr=+4qx@}Dn2`txc+Qc+#q>-(_5jp-g76hr&+CU
zjAk$C=MlKR#P3*!-prW|D}9X<mYXT<s8C+owB6e`T2Orx^ZqLHOxqjHvOA5UPhB~5
zTy<W4&4-i5u@(!$itLxIPcFGAm&2IT@5b=&gb^#-gB#QH`1A5?4==NswfmS+-O-()
z|6NPu-|2`bUuG*_%E{KV*_lg)VcBahwuS6fDFw>sZgiPtm+TUjtNm4IYQExP3u9$e
zA@i}Kj}pEf>}p!m6MCvl7pzK~J0<1W<*(~h=6+{k$!CsPnrzRx>A6Kf>ESnL9$ijp
zJ8eIep~v9nq?=W1nHd<E7|ta$Pbp(^Jk_GYy(II0J<|oBnG2I6Ha^LCb7r}t3X=<W
z^`TXL?<Q!?y}kX&x{pm_@0>opGufcD<f4D;9;1LC1$F<I2IPsY4xIjUO(m0=f})Rg
z<&{q5J(i3?v$w`pZ;iIzX6>|XuVk8OdeO3~uP)|Zy}tF5g-cd@R;Wu<91zk^=U*q%
zV;_^OY}Yh@ZZy{+=dfD~M7FV~FuDB<`smwtxMKO?te0Q?Swq7)Le--f7ezA$MmvUQ
zGWiub8M{|3jdoRa%1}M0x^Lk%iO64FhSFW09yjlNm)R`QuH^A?#->I0oD7ScPAstM
z(KfP4xs`VBOt*>91`V5K&Z<lrGKcqSgqY1aG?DMzHjNCH1q=Uo<!ruFp;$3>QD;<d
zdlggLu2rvotf<^IHTh8S?L(&DCO%!Dwd&`_a<QG$9$omdsMNym{Z)}5zf-%`EI%Br
zI+Jxahx7lZtY?!a-?o~3)s*pAa`um$?BI{8?lae`|Jso8G-J>14XHcq-)x<un#6NN
zI_|q^wQAF}s}&owmCaV}*3;5aY`^o`u1D+O#g9ieZQ3bxVPcT+EaUeLGOk7(YZi1c
zdqgvBlH3|!eQEDbg#@YINa@SYM`tfi$aHe$R@329T5$RQVjrERt=dXgRUav(E$dC(
z)>8EQ^Q|BoU-m68i}V_2ui}(6URV=WQ`7h7lK$T)gXJyUXC7x*?=UouJ+r)e*Z)u^
zzbi_&;x!J|r<?aY`x&$246~}3`IhUY8|9R&G$$o(?s|0omg1t9hs6|r-8mbr=5cU-
zSO2cwN8$JWs$Rb<cBhfOc(3mf@7>#e=sK-_wIDVjrmE-oVI})LXVZ?Q9dlOR(jXeu
zC>qfyvPFT@P4<ei1J@RZE9QyZW{skY9Jp^R6ur?XYSPH_s*xw+pqN?Woey2=8^Rg3
z9A2?9_+;E4tEUm+l~a#?tT>iCPraj4Yep4!+rJh122AIL^%~E8HJf1jvB0)b`Bm`u
z=#QH_V#4iqORcFqRQA1?b>kFavj;bBSG}=+)Fb+MgTNm58@sJ<uk|_gqT=~hj!M^f
zE8(j4zZw6F4plykQ8&xJr~Kso!+9bxm&CL<XC<jP#T<O!dr0;d$J@H4B0t#eKHX?E
zRlZite7a1sL+Wq6dIFQe-7V|a?z_~v^|kjU-~C@S|88BXu-RsxSOnu0<$Yov4^@vV
zaAh@S-b&>0SSU8hfy*S3C#>#!bmNR`i+{)`Jv-=9yvMk&SMy|F<@U*6>K4YIom^US
zg2BJ&$V@Sd&-K#Aj+OTx3(T(hw!QY{>{W5m(o$=+1pS^)X|`wTy%?c;ZT8ZuGx}cF
z_rBb5<fx38Ir9RBrk;jI7H%n<iU5T}tvu}8Hl?o)UE8#Fn}?#a7lWqRTcs=&<q09H
ztMXV1Bu=#YXh(m&F(K=J@cOW0eQQnnwp=|Iy4ozpiswyG(*pnI)Fv*C%>^0L7cyBD
zNWKtxbzaDyHNaMC3r~<y)ae3M=dC>Ad@(z6ZW^RIMS)HR4)_$-D72GH=)&CY>3KqX
z3nzd1YkfUGp~;iwNAB^g1<RrqM_f%l+Gn`BEag^i<eeR{Eh^Dw8&|4NI3vdB#iOQB
z@#}%e<~8OU#Wtj#I&gy}b^ctT>O%*%Y)BQpqj2C&#^wX_nc8h{Cw$y|(4Ad;!=IFb
zm&t+CL=U{Iy1D%3YN01z{{9X&dLXUjVkM9%>Gz6#T2#G2t>zmZ>r%;xdk<=wKE33B
z<08<&dUpGK239eR1(nRHOBPh~7ApL2W%6VZa9|P%;}DM%S|BRTo}GIjtV68PVv)#}
zPL-f$X4NtQX_l>y0luaOUkEtm^f;Z;NXc0=p-FR+p;yl~3zLg3qKxWNZgRPdlNe@S
zNl}oDk=WX*ZTRwpmy5Mm3b$t5wn<4<p|P%0N?0dmUI@Ooa^^k%)H~1Byppy?PHUI(
zdGL^h>&43DEL=Y{8)G?HxL(X}mh*bCpiXg>#=>r|4=*y8i<${6Z4*q3&|c%wctq^$
zghO5S3;H&>iYWUoX<gRcc+1aCXVRwDrL0TMCNMGIRAm)kox4ywd(+QlNkyz1J>L}P
zm~B~BdC$}$YBP(i>7C8Tx|Yg`U)|O|kHMDh^#9!L_l%~**J*vzn6TqkR!N?lN9vX2
zZBN;bw?;E+1SIY)a$I#F_E*}Eu>E;|Z(n26-<uMUabN4jBL_}>krYQs+a5jVwIwPH
zQvxO_va&03CiF|EOjxQbtu1T%bjq!DE;qdeCN7%9ux(4=sR_AjQo4`3>CR3I(Qqxz
zag2>zJ$0*-_sN4|a*LLRMgM(v^vga!bJ-a!RUb{KGfa?Db6gsJIMsA%c=Xx)r7RPJ
zy49j4tUW35^~4TOmX(u~4!X`hvNw6n_g$vbW4GO^x^<7~kEYW4rp?D*=bSUMmdXoW
zpvsVcK=!&tQM>)~IhBhY?dB985%a5fe*0kbn^NAk?tt^G8;}3@6o13Hz^dxmqw4~5
z9v#{l(8T#KU9N#es^h{tUeStzm%$EMi61Lf6*Cr;UW}gpsmDpGJX+$SMDhn-$4Emb
z$&Ck-9cK%C`P(Ei_jS-wImwjMV!{!-S<fnL><>J;mZLZBxAUf$Q@U$AuC{5%PIKhy
zpR$NI#xU~blF}(GNvES!Huuf_Fhz{>L#F>vqma*RsTvN669Ob_;+Duv)nGKyGHu*-
zWr<6N)aMggiiU}<NlOk$%0(X04{A{S;x<Er?^C9;TGFJgCoZ&2=P4-I{YrF0L#V+0
zkaa~Gg7apDeU`~nUELYdljN-RTvo@<+v}iUx5Kk52Uvdf*r#lnCwcydFw6gCD|?b`
z#4A{g<cyxV$9_3^ZG+Tu8-@su_@-OjpM)kf*s~c;5K>b4^o?nHBX3L9om*l~CZUVP
z1YHz)lR27&j3%gGOi^MzIK%DK1FpXl6TDI%Flbn_O`CUTIk$?&Vc8f*-SCPvQ*$|H
z9ou5$wQ7Q<j>pf1;d_j{7IE_FW^~Rr{P}dc^P|(cbu2wCb2Mk9vn|taIVp8*i`q6(
zF_+og8<$<(pxL0|;=u6O$AN#U+1WiuQv-^B3dh{oWT7^(ktIelLHmw^clZs(`M>X^
zPv3Sx+mB%i_iP3Yr-IGWHyG5-cV9f~q?hih#&uvJk0oC*+om}$OrEQhO7UeMdlYEe
z`R>2flIFRd;!V0{w?1k-&}@BmtHhxqk=>6$>*XAw76ryTedT;!7hdyh3TEdN?VTJl
zUr1+*e$yM?31u^<FPQfH$}?X^A7id*-6>ZT-snzJ&`X-=)}-Rxq%%>$@t0s9$0NbD
zx0-l_ZX7z3$LRfw^8oMK0$0YPNmCZznOaw~G%TK1V=enL&n6L_5I#ZI!x4=ETnUqy
z_~cn5-9v+OqO29yW|X>lXO<}#9J#V~Lz}azPeVI{Xm5DJvo@m(SH*tCbZW<)jH-WC
zysfMtH`n#@PL*qFMLaX-xCAmCjbml+Ix#8iMNeytu@!gJ3Ez{2t0!HWB2>DIOKWBQ
z0auY0iz0>F|5Z=a+!EdR`q;9S0b50vvUNHg_TXx5wn_Qj^x}k^?wg*EbDubBMV?I9
z+L*NV#rva1VO%wzT&6Es(iL2?^Hk%6rP@_*PuU!2dbmhYS6A{9x5CnCXB_g>n9kfO
z5VDbJ)NnY~X4=!la_b`3@n3;X*Y}8+ZGPr^zHaTT)|DR})@ZW&T$MQT<d1>n(Vxk3
z?G{G@JDSA)ZDm}%bjdv9!|HawEpMw|Hk<FatZ2&mJ^u3w6s^}4@~+=!KI;RciP^cp
zIe8PrEbj(<37_QLw?SZ2z(yx_%cC=kH_ke7@gQ%Wb&cOT$<v$VS{u|$Q(3;e3X`(A
z&FZndn{C2k$=U^MjOTYxS@!O~;|_zyWnXg@M4e8gTF#xNzVyWE^gSt#d%RvXebm^W
z66@jHuw}#Nt1G;7zpjoyeCYay3w<drvzz8B-B9O#_Il;X>U#@jsI;Ela#d_`!$Ogs
zoJT4djqYz(w062JT<N=Uk<v+)TrTd-_j0SQ9er|7C-r3RJek$fx7_bFi5=UJ71pyz
zY|RQ+pCtxuQ+J51+qg7W``6F+*O+P^FzNqrc%#73vU{P(j?^H*=F`twc10Gc&H9m7
zyLk1(1es2jlZCU)`0iK>U+*fP@_F`hkDU)XmUUlf&6D2DTI_v#j^b`J|Fhyk?(ZAa
zpII+IUvtCM>_$WiQ+460SeBWGilm<?t!W4>SN*?oLw@3sgc%|XtDCPb?&R&9JlRBW
zcf-%a2~ArvJf-7WKPH`UlAAQ4taH}w`8R!z-D(znWHQx0Dlw(Zp-nWc+b89fA={=#
zE)&KpE?GM&^I9S<2jxk6+&yf0PLaLxle1pY-K`1Rw)7_PtZv}ZeUbP+?#f~LzaL!f
z|2*uEn{!V-`NAQ)r99O-pNqfwn|XM$D0nyBsayB^^|@)$#uu8}W}fNHzEfDCEPc1&
z&shQ4)m_J;p7;Da)>YaRe$k3+*PGK@tgjvP3$4qFpRsSxjw|cAeB*C%{@BBAW7_(%
zrs-De@-6=rbieMb)5@LT?N_zp_x2~7OIj1Xt_m<ZNNm}}s<G(5LZ_qcXIFuhZK0+Y
zLzyOInNCu+-lWX?N%^6&K<3d>sf%L%A00fL9cvevpLWi%ba9rrY*YO)Lg7n9kbu@#
z11`@6X8$hmr3afO3rH?b;GQMGy7&S2?1cP;Aa0u={*8wsAC(DAxfIECp~e41OW=t}
z&ks50ujRep#`)=))0edTe+E_YFIp2Vs+P02rM}=_vPme9A@%Q)w%~34JK77_8w-ke
zRI?<qt1_zQ9pLm_l)f*?WlyNUzSN@Kj%i)N9lg%d+=VW^i?w@pNcI1SQQ7WxP)PTP
zP`%D|xm=-Q<8<2x;=ybmgXgVKc+#Y|q{6dgi@?o;Vt!Lgymkl(J_uX$;J*@2MEo`7
zcr787sVs)e#7bEWjh&U0Z!AtavbpT&in6y?<7Q9n30z<n<<L6IK&q`o(=1J3!lA@I
zL&@lX{IAP6*B;;tRA`*fRKc{sEZHFc%>uy>VJ#Eg{od|KwB45Y=u2X0c;(deO1Ej1
z&ocU|zn3q(&cDjNim|eaS#!cN_o}}J6F532Fj!8|yk?(m5iS@hB)qa(G*V!9c)Qq3
zqqPolPaFf}Z>FA5tVv#GF|oNuxOGz2vgi|rtQ@L^8Z!e#rpjzHOrI>AJ|S43a$0&;
zlF6PRF}szrfnRDl5@UaCl>5F>bj4=5j$>BmH+nrQm-`>c)~6;Pd@`e7EaZCVl<sf;
z>-Idh?*3+d*_DgILUD#_1Cv9;{I9-^ErJO`o^vt;qNdt*A5nhzSY_=uj~799(w26y
zk8(mXlx&wo^iJ}*@Hw&JWuDLl%^PA`Q44xwUPQ12I^8cxesrw4dt-B7VxOx8mxdbW
z1Vx570sONUu-;(P3Npw&H={-Fd5fP%Ylwv38?hPF4cTTbX#M)F{4YaWio}Fujkd%S
zvv>cTz3=60_KW<Yi&;|)R0=cXww4O4ZxL8GVLIPI;l^VsP0nl+erQWBcbRilC&+;H
zRhq!2hlOW9b#!uca5zeL9iHoUrQ`6{=&Y6^o{ehS$62Rqm?SNlzh{|44$Jiam2MWz
zB43ZkJ}m#Q$6e@IXc8lQAS7$0l8}Kx!l#mh#tTE+-E23-TLf~y74Tf4DRgyN`DZgO
z7cthL31+htSR+sHrA#QlcS0g+!psKq{51`N7b~XCdsdMWz<+t$!Wq}OG^A8stl(5k
z>x&Zb`y7$-(ZbKOs{cnv<*yqORoDB!CRFicPFQL_foJ8?$Wu#~Y+o8zRmF6HoB3mZ
ziq$fvfMv{y+$S@+&u}gf+gZ2&c&hx*)GdJlsz0^)wu)uD325D%lognk`b^qmOO0M<
zfJ~9}IrrK#j7`^`uehujH1SpK9mgpfodvcyThHzg*raS_ceu{rdQjL;Io6HAZ`0+c
zpQ`Usb9+B=>QBK;=cE6-wTilWGqkrR32a@}Qs^9dUuk;&&DBf?8umsASP6x(D+X#7
zDLWi*WOB$CiqNub5?;E<%t%pqX(soY3Cc!|z1u7+QV&bGJ>zByn5B0`b(vwF_j1;f
z_F2&lEUK$m-)5zF3b20CTJMn{;hNU!rpCGDrbO??-q?&bM(a7u+8bD3^D`A};0WYr
z+OUE5;s&-v{^Sj;&P~<ahAQO_0ut5S8DFD{U6r>7czY<$Wo~R26wJ_6bmo0NPw;v4
zE3tVGe@>Ep-jdqZ(e@&(Ijbn=*t}aW=hc;^PpWX3d^qMvQZ|oB?7yGO)R!uJ&XikZ
z(Rn>|0qd(R%PST<`k480sRBF4e>F#ut*p#zE{m?=NiNlTq;%WJ$Sp`fON~`SjFoeP
z#5Bc4eG6HaZ8Dqa$d%k6@q7nsN1<Txca__2k|!<;EOiu^sNU~v)~^zkGUNM_ClX6L
z6Z8L_n82htfunK)SLe=|r<cyC-pSs%jLUNwTj8AJmIBFMyI2Z$u@>%Pf6cvk17l@|
zKweh6`$EyrEH0Z1g*dLW?MtiP<<9k5aryr1smX48Do#af-4xTREX)!s%u1Z3ra5`{
z#YvAPrDcrLCpZTlI2c%#CUo|z<$dWXp%+(v5ts9d-uJI?%8JWYzBApvHyS@OpPG7L
zl~wB09m)F_bO_i@-M?qC^^|Saf8Ar3T*$0B{6F&<BSY@i)!LUDxaVxq-;~9bI9=#-
zXtkuE$ZLb>28PIB?zJD*MQt#f`$^)nnZRNNt!EZovkTVt3Hl{(=w0hDi`$5;z9o-i
zqtlzRMT;-2j|=5z>aJRR{qWwOv-fBoIq-8f$8Le;zc;W>JhEc-29Alld~M2u0;<^`
zvc)(sd4&n(HmKV@QB*Bd-yZHIJYgbpA@}QWHqG1Zdlqf({4sYk<GcqeS!B`$^JM1j
z-#V$=LfZYxJZov`_cvV*sm6r5%>SgR8*-rNH>cajg+U&-5A^cLegAkO=ZoHw6(@c!
zT+pCWx;DUW<1NpK=O@Lg4AOQll$P?067Ue(Ak?SID)`VWHt4_PxeBwtPgOLnN^_ZK
z99!aZD#Vmmi8aSy=_>WA73mXJm`@1*we<gur6+in&9&Y&KYQ8vm|Z?Ob2ti@E&8o{
zRc6+D?cLoaVljbUc~`kI7>~UcmJwbckhj2&W8%^M*3)hpE<fZRkaA6Wqv75Y+r=JA
z+H#go^82w|&vKrHwSed1^NSf*WG1e-S?wT|Dkc*uJK@#dN8A_hl?O#z)){QI+RH4T
zG^OtQ?u%2sM66lso<EmcbW8s4YlZcvf;Dq9=ccXJ)!h-)pj0$d!l+kZ<+IQm!rS)E
zsJ|67T_PfsN2w+K^qQp71D+S+Jzl6@Vw1L9=A)LB^mdky;Dx6Cqq;A>j<9wAzsgd*
zfphf{mW><Y-(G#T@d&%&wT!H5{x$-sz1Pxg1^y-ErY!L1bQjp(kQrLY{{A9c*#--b
zg9Snh<mzO-m>igdK1@(qz3jERs#wL`x)m;Wf6f*AxT#=Hrpe7?`7y^IX{JwAK5ibp
zxwdEd>z^*#*;eeroeQ0H|E}!Jy>f#6@s_o<`+{D(y$sswdAh`4({xtBZJRUX?uIVp
zUD*9XXtkfapv6Oh8=E;>Qj}h@lxH2kqy6lT1lwKhZ|2VudwX2ZOqo4_>*SeftM9tq
zIdlK)kq5H7&dQwiui3S{`)s<ez@qEiF}X)_eFfh23OqcuVXk5O=4hti0Hz4F8@mkE
zeuc2Te*B-!*@^AhnK>ey@zp=hDOjd`Ub9*E=W@j)pNeGP%d2Z%g_WH=E~a6*SNN&)
zncsWbW-RA5t(7>sH)qpcC0E&72Omwcy0y|nzIoa{@5+lS&hEQ8Rc?Lr%0SQk>=UPM
zc)V(rS7z6@{c)L37|!VG%$;_(<>cnZ2A(^*`?K7vib|7zd&DU*#7JKevk?&9$lA53
zNnGUFv9gHvQpe7w)jPZn&vE5)*faat#%rm21&Zz+xm<IEgRyG$g6GrBq;4>)mIpkD
zQHWld%APVIC67TlaGOQS1H<y58?1rH7)p-SWQg5-slCTFO|41#QRYoA1=dBX62{Uq
zrq|vSW4tLgeO~j6o1gCdf7mBcG{GXqCtI#9#`XJ3wS<<}Zl@hS?7p?PFLucfw{Nai
zd)6v-lqc6~3sld%9F!i)d+Bznx1g{xgN)Vm_<iD2l}z)?OR9xUwZlwH3<Y-UyyGg8
zk~zk9WK9(}=h-B#=f>^#*dJc|(y(E<_kH$g{^Oq9i#KpB+Q3z!R~mJ}{zY;P<Fu!>
z`*zDH%>84clI0}!ck;yT-{q2DN)>AblvZtyHsDl~ym`_2Vf|#q$~$5|c;<b#3Cv%!
zoOR)H_53sey$dfTSG2^WUrIF5aFkVGPCV%vbl-f6+SHZZF#@LBvzJR*zj2p;>Hfv#
zzJOExV+q?!E4wav_{C=V%@qIm#7O=B)8dl?or#UyhHT}Qhi*Jp&27k4zWj7@iH3RE
zw@iJfYb9G-9xVtzEs%DzP{pR|itD{gGP`ol3S`UfsyfHD@-*KT1A#9GxK<z7z~sRH
z``mZl!h1eE+P@Wd@0ONPd5|3+B9=Yl!?~H){~2#8b5Ir(bcuVCI@h@9WZ=ySyvKtT
zqUQ05x2}A-BL3C1xv#8lr<v@2xn|w@MZB-|-|4o+be=U;FDkt?bH^>e)1B8u--xVr
z<2`ti-^yV7+SvHrZ+86MddR9PP34Z$rVScxwu=i+UFLnq>$HP6=?(A0<Yb34<@<lE
zT7B=h<+7OC4O<o73+><&X8h0gu=V}?Yg+_<d|?pS`{IA^C6l&Tdq&R8&Lq`@#KSF2
zZoMm?>n}RmAt;#UlTny_BrNFEj-tbX3@43LPfW=ao%qDteBB$44a&i4XJ@N=tCd_{
z#iMC7XGf0IuGGnaizT=F2pKG0$)Fj$$Zx8`(p4;~5uzG_6I2v0trTA5H&M%U;SEkf
zRgQ?=*D|9vMK3!s*I?<p*yUxyem5s<f6vAk!&1Ouv%?@!FGL_;QD#Q>gX0tBt;KST
z6qk3*&`*E1Vkh@<BPP4{cS;IdQZL4MxxWcn_BESveeC>aUuGI--C;60E*7xrd;a}{
zjnCEcEFGO#WhQHTOo(Z^-VziwF>9vaqJ~#8o5i2|&FN`)&t&E*)B3-v%Gu1peQQ=K
zcZ!SA*?xDQyiNNWe*fHEyk9<W^8S7!t^#|;Uy`lUsw|>fr?R+9w$AH#C77XHmXX*#
zH87%a!n{yEnbd=IRyzI4VJ<6|I&-ix7%YrpQ@OAtYTJWDduQG&e##dvbZFx;+0?95
zHHF+qAD8vK$v8SS=+=Xl9^Vra0*`6#XfRr&JFhEGUvpc)A|2HwH#VJKGiicC<JKp)
zl2#cn^)g<yY{`w)D|Oe3Jn-4MZ`NsFt;J`$yLWz>(tZ9Em$yd1yH$U*7CD4Aujx4@
z;oEWXf>NvgqRT$jAA@{$NBf%G)j6tT^Fhs*<H~!Etv(yPlxO)jgj#wjc!w?f-N*ai
zXZ>jt<)A=mL-Q!dqwA)Y<z%WWeq&L1y7Q?|Rw%=Zkko5N>z`$Bz1gzhwf<s_Eh{`4
z;*?%(+ns0VwL;F&?WOub7S<I{6&StNxZW)d>-_G(7<SR+u#j}p(|c83F-zw~xkN6N
zkJppRSQJ?GaryMRKZ))<Weg(I59BL<N)_fSI%BwKQdZZ=$8&OSX(~;~KEiu)L6g)f
zjYWM{To=vNPoCCVwP}-<_PR+ov>4a_nzBl3<&j@EeP`9Tartk$v`BmXmJh2o={f&y
ziDEdwvU=950}`iq8S3qLw&2tWuGKk43y<wvv+1tJ)ipu4Jhazs^1b<PiTU*;ZRNKu
zW_LwQw+VmJUHt#SDS^Da{mQ>)?>@Xc`dj3-1-FuRz3s?e!TT=fXm8jRDWzl)HQn1^
z_qKQlF6i~WQQUvD<HiNcg-7#UdvA$v-N3q#Mex^&Z%n4A_Rg<+v69{Co@=3iGw*9z
z)kmSn0yZA5HxyBIX=Yh*+>!b8(o|7y;i#uS+w_y#6<GeRto;>r>1u7vqci^Bt)n~c
zSS*~Cm1TcoR)BG2!$jk(_Z~r~GJ@O|IdBH}-E?~XPGsUrpNHPU43~?SD5!DR9WRVr
zv|yHy#By`ZfM)-dlNMSy9oS>z>d)NLb0el*N{d6GRW)VdVjWdYiJ&awyM|XeJDazk
zeX};$_|C%AqOU9DW-V!BnZ)#8GQv^&%-$2>b}5fmxLlRJ=z39QN_5iCtc!g~t?w+e
z8^u;z2<g5%pl6;G<ITddK$k&rNz$d)tOs5M+C{Btb`)E9v|VNk-`h+lMW)s4y6>*?
z+iy~GnKeONTkMK^;Dtr<duMPc^>5d6{gy176ye%!cZs{KWJxt=;*BFZk3A=|bcjg*
z5M1^tedffw{c21bSgRW@87<ktqq#!ze8>`q&8zS5Xh%KtIwzvQx?-YgV~A$d>~)uR
zY&UV$_Z4=C{Z%Bk&!sqX852*UK(a)(+%ZF$hMp8PVKc{tY{rkT@Mby``yITSYo_1h
zAs!Meaa*OOV9!CH>?bFst3N%P_j%538{hv*G7Wt07G^5vY9<-aXIRsf9=)LN#SUT1
zCb9P72a@M2gf?lnt1aQZA}OmWm9(p4kG7Jh1)sQdy8P)FZp$wldptWIi>`NB=sIm7
z`$w500%32u9k(i^G|f<-$R3$e{aR?o;jUeK1OKcO`%-YD{mAw%rmq`gC6ipGZf*?e
zGg+fFdrwDqOgOjZmV@qAw{FM=1c&`?e8=}TQT61FTV9c-p6i?y56?X_MSVp|i+9|k
z?CDA?j13>U`-m(NPF%ccDVG|HkDucL_OGj!FZ=M)U%5x<4BJ#q{;B|jgxSFfbH1!_
zYvt-#yfwtd*IVJt3xRyqHB<N+cibx2waP;1#t}D7ckTaY=Z0<B_4I1q%>(li5<i9?
zoAf~9utm_5RjR>pD_<S55ZYjPVX^q+Gby&m67;GZdR%LCCMrp;+|$78!?5T?Z2PmN
zCxk1%_p(H*9N6}%yR9%(d2URKQ_gm!(;{oEf2}>v<`KC6?~EL^Z5xg@aNn61b!Cw#
zR~~b9)Wx<rRVIg$Z?LrX9@?~IL7S$L@-yA?mEQ5cCM}6`yd0>(v1QT2OIoK|yj?Ca
zO<iKRP1Aez<nF^WHD*-i2E=pom}?rZ*vOe??(}fs3WsE=3saotdTd)`U1~h@%8V4#
zxf^(=zK=Y=`=xRI&cIK17^2MYzfmh(p6ln8>fP2h?^+S(#u%Z_*16v^{^u>a^E1ZH
zv1`|~El!-A3c;@n76(cHiew1PV128k($MUgAHOAFV)K&Y6S@*IME`ln?Ojl%x1y_G
z^mW8V>7~(v0m&(=FA2x5ycqhbXHHeLq~QE7I?4MUEm*CYa$|;;|MhEOyVS4$&VIVy
zW8=k_4707)1bh%(`n;RNIFpA##g(z&Xu4-1qrc2X)fs!99N52ZskYkQ)I{#=gL{>}
z1yn~SOjFpcerno>O|Q#h*{x4Uip{zZbiXrkRYmptb9dA_cAN}#^PD|VvCN%`*Y#1}
z+#N10#ZTI&@17!3=<NFT%Hr#{HS=7iTsWF_(o9+0+;g)<pLBL=i^#qM`5vv?bvxFc
zaQXjN=!RB|U9e@G#^gPZ!@gIjZ~Sp@kAvBB%l9R9#j|c--?!cRe)o~56ALAuPTA$Q
zq3I@bBFE;39l_@{wY9#uai+>XkN$pqm%))G5lwefk4$7?%72*{^C(9<HqknvGE7v{
zhe>m?#qO@vlLSPJH|tja?7p(-r}!F1HN^%`?_{3htw%jh&Q46O$=PZAr0CkS6R&d4
zK1q#`+FQu=dDi)k9F0`}>f)C9OtO9tkL5(!dvm|fnaJnvyD6>4yQFyTW)1g0yg7_t
zx+1pMO>bIwD1Gv)_U#RNe3}ktCsyV3Kbuf?LuTo8&#F5%SGH<z4dcF&wbm<V^$qvZ
zuda6|-njg;@J8W(SM56=vcKFeyRlWFtoqf%?tt|8fA{9^J5rRj#&?3#I-g0~X5KMr
zeSd@L_$lornO!OmRx`IOV4t+$z^RTr<L#y&w;eq-sW(|+%?v~PlMW|0t&!fWc-Be8
zC_+S#MOWdK^d+vnAG{A1aPgj5G39;6!CGdE$EPf`Dh<AH^Dj7I`>0^^-w36qPup8A
z*#6}dv(P@I%b@dU^TLBd+|v*7PWNi!kYKGZ(DU=+;J7*M6QjVdUHjg7F8;Ad-yme-
zg6JhbE-iVqVn)}_B_eAi!cH))KE1MZqtbUKfgiJ+IzP5s%{iq1#nO)D@Q+i@j1zXA
z-Dt40OSSxQM^_hz=Z>YC6I!-@Fqr-S=At<}5A+q!NnhM89Be4L*@AO|+U^g#mkQeS
z8SHUTJgU25*1DU98$KGADobzD<WcHAcx{rRLf7(31=80$mb)}cSFYORzR>8x2lb1;
zdG(n$U;e<Gs3Vj5r#JP=+O$fU>xX$eZuL)FFlX8W$KFePvNEFQR@(A!(g+fe?at!7
znAm(NQQM}Q?}6|Zalvlmg_C##m>AE9bY++_DsC+?-1=^{eW<aiamd7Km-%-B<~zUU
zi@43deiNVWid8=r+t=ot_%(a~TVu{h?+M%=SNtiK{HxsDbVA5;#&OG)lQK8+EO$7-
zZgsM|d1dC|$%&uL4sTju8mPcH;kcdQWV?xL1Sgpb{|_?ff1@MDb?`ugh+cz8nUkaF
z9#NSt$36~=R^w@xx>hJ<wSHM3dg|a*^-W%rfAGokOl{gdwTJ0YN5c-qH+$dSXkvKL
zKV^=m{x6Gb2YC~$T*D?ZWmZq!`Gile>5#9&PNvAkb00g;dVF}Fiu3K0I~P6nUZ}h)
zPjbSMCwzyG^d5V|H%~}KKSiSLp|k#%({m2{9JsUd?n935hW%j)tU)i_7Av?c(Xfu1
zrOG&gW#x+9F}JL@_Vn$V?Yq0#VbMx1wUCVge~#Rh*^~CmH|-Qh;2O2V0ejRR&N_PA
zQ)6>aAy-$?t{$Do9hWM4qD^{F6-)E&QSZx`z5I%_{^Q;TDUCh>{{#Ou_j+eeeY~T8
z>x_QpoE0xEPWe2XlXryoqNL!R+cF<BHbiTCKTz&dapBz=(am|I$#7zWZiQiA#-a4a
zKHZ3|{-@m6F{ws+n@(P__;`!|y;FLcckKA4N*EsDSaf-9)Q1%XQ?$9HnSBKg=>G7~
z_2~MTz?J@b%|w-1%wJZ|JR)o`bHV1*+IyP&@?K9$oOH5#^VySHK_`lAWRlI7ZRz2c
z5m_MPCD=1%;gX}vk6iFB)V<>o^zw~}*cXweS^Sz4tj;91zHhL|+0tXYHE8)04y{k-
z6HaY@W#KgMX4hW<<*hb-8V|X4PHBptWVS`8kK;m9RD^{1OO?EjVy83C^hap5tY{7V
zFVI|?U}D<4H%_3{P{FzI;au+2t=$S5A0wxqmGr&*=fq-#s~r<mLyijVxU70~m-m*>
z8#F#_IF@)mtkWf7lXbuo-+MhB51)kcZIQ~Y>N)yhR_Gfw#cEeYY0<M076FEirw?4}
zSz&aY$xWAe&I=Q_hX;Bt8_VRy9?RV;`r^hSm4p@FxAd&qV}4_`o6yb`=L|bPt=L=k
za79rJe@TeA#pYJs2zMq!O;+BMhBJkYCU`TYx?j1_)N!(x@!;I*n#HR(IXO?-vgFYO
zpNLkGlUk=gtZJ4y(Nwb4`{0eX1v{HnJQ_>3a@KB*KDjbb!Z$VhV*i&DhfZlPyWye}
zxL|UNW}dZ_=KmAZax`TtYFA2!GO>LUnk{o{w$827UFP$qIQsnb)Z=h;E4-z-)?91u
z6>-l~+PTZsH*HT*;kz}fZ>7z`b$h(-746V8vUsC#NJnbwRY$$`38%$R3av}v-nW74
zK-XE;)H&N4oHQk_OsZbD=JhlSoeA74+v|SFS^g4e*mz}imBfqYkR{wp?<~Bv=rr%&
zBzE0|RSba<cQ$aGV?A+>)&7Di-?_O?4T_gLExJT}?~2}K^1aa1vSP*6yBCk$Txzh>
z!LXBwv*T*SRHkN$D`6c?Uv&E7c5%&q*?sp2cY2iCX#v;!U(V(T^(0Q6c+~jXn>UI@
zO3P12^xm4VP<8dWGu>fpOgnG>|6o(IY4&>=nKYU6U+!MZ*X?V0<ruCsA#+b(X35+(
z0ohxddwyNf=sToq_Sm*GvtKu%uTIoXJx8-yYjeHr%Ik?Tnk%%f?3ln)D6k@`o#&#Q
zaAC($S=K-Sj#rk`t1C^OYwv2A5_95A)2D+XJ+{+3K5U8J=)rSwf8dG@BDsNwt0GyN
zxTdzOd?K}Fj%ioe3Xb|8?PUdb4R(36t>LoTxO$dNP;jTY>ffu6JQkj2nyp?cqA~T7
z#INluO%r`)9=-A?==Ig@TNER<F6ayL=znB!{nuN^>jzhKFnFsJ1#gkn`=Fq??SaMA
zthuQlEuKa2&7JA;=R}85v@J_lgT@N2A5QxW|3_HPE#hdqFz*G|5%UrOj=~9D3)r8&
zoc4)BzUh*FgAYf5Td&Z%Lw6#Siy~W_0?&kSPq16W&b_czWTmp(=8l?=K1>Il8zv^O
zI|g36+H@^9R+Ax3a{`NQ#F5R#3vQ)$Yj|+DTx;S=W^a9|V)G{QX@cgH1!Bv89+7z6
zy=#L|V9uqbEi+eLluBI6v*D*&xX-m@-Axy*HZ3qvf75bK<FUvynPb|IdumRpt2*~4
z+1$?yy<cy&a9hUS6PDL~9A8ZC<@hq={QryhFEkooTqx83%Cy>-BU*u#Y2Vbkqq{=|
zC)@27(VTFo-(z~!196vPld>Ip(brl&-JDnDA>ivUK}a!%`Txwtm7RBlPenYr9WhVu
z>B+DC^&4;W?vUe|m};iHghy~*<HaYjiR_JveT<rsTzj?pizYO!NSv;-X0x@!iVLyV
z?=tB+Y}Q$0U!Bm(w&ilqf@!}dxO1dDTKIO&;<pFxL|nS_!(mBl)S@Z37W;VZS9@Ho
zHsR8Y**hNcE>`I~9%wJ^waA$Fwn^?H@kWESES+ULl<s?XlqoRVtm{wPc=_D69dgI|
zySs1y`;fEeG#8`86T=PdCZ}XS?p(*jIb(^`de6F$%(!;L2myA>j#-Bet+f(wdwAvi
zUeSO{nG4vm0>s2cVtrbD?b;?hSh#GRTkRF|O;<`d*Z2PDIQKT@?Yoek8HWo0M{02&
zobdYB(*(9OU5RI|Ik9~b%t@0coMGAJ&~o-~#MF4NdlJrZ%NYIM?T~u3M=d*v=cTXT
zv1q@^JL1kcELYY(sC?U|{MMe^H>6L`?oCLU&Gp|e(MBeP=f$_9Q#PkO=+d5>ai;g@
zUH)Htr*vk8-_hv*^y2;$UeWG%wxyP0Mj!p38l^NT*~#B+*{9m?w{K<f%Ej%nGgxj;
zD6TxwcVpVC&q;T^<d$(Qb?_4RapH(N(b6Wc>iNp)I@9NQGiOQf%1fG^5Og9d{@kmF
zf#MxU*7v@!+NGPYu9xqB_h&UWi(`+3moCjdew$y0Sz4!T;d0YnP1}QdeJc+4?KQY(
z<hVUM^?&E7?3B%s4>{y&a)jAVTb$$g+p<|Fi`j<Ht|@KquK<nEpXWOa7GI5faw1lu
zEP|DD-SHK_#O7Z<bf)F&-Ys)=k6bXlu5a^(WA72?b9YPQH?+rZXYFL3QFddxBkzgO
z28nwISG_o`*VrlE9<d~4`ZFfaZ|dO(1JCWUm|_{JH$N})%<(Pj9GAK5{%dS_vMXk7
zw?ws6UqW1b?ma%2J-I<g<rd%DRJp0F@8BV^l1snh-kms_FeNPa?54V9vkzpt-4WQ_
zz;^wr@^!)R%b6iB6iPBqOYpY~$rfF4{Fb=7JhQXsU-;IvPd%5L<d(~Z+!jBiYT9|l
zuJ+8n(9L#SZ*Tnlkvadr-i##WZ4r`YCpxDFMw}3B-0JX1c7@XV7s~5P7qq{)tu=2^
zgv;%D5A1a3J<IHx@~T_4f8rSt)*B7m)o$(H^m*=?ORtWKRB$_Pa*uMH7`1O&%!%Uk
zvfahLm4eK<2B#)goH*w7j$!KYvYgFpzqD5?B^2^c<T`E3tz-4{;np;n^p!{0#O~>O
zb>#fay}QvT;-1)o$?qfTzn;+F^@#6cfLUkU|C&kMwF%qVqZMj9iw$<xZ(#Qpy82&Y
zriKAi_ELkd9Id6_9?YA^)3$2nEz6z_i}^a2h(G+(u;pvtkJ#x&x8F@@+IKmB&jukD
zUA_Yfeq0CCUWM;@>3hybzi;dNDQx%eo&SGe&)dCg&NMH4t@Hn8z@`sR!g6DS?{PI4
zIBhPP8h`pkpYZ#>HymfpyE8=9`R;s<Z%|NP5T?z@%)V)rtCtf4C!4&J$qNIeqwT`l
zNlPpYI~n@e7GGI4edGN7RsTG?L;^l7+25hdy5x*Tv6}A;{?|XV!nGAwhS}Et%DH;#
z+J!~iVgzkhiM$M0=CVDfigRQ9{!k{ji%IU@S2UUzyIoA~3}Of^+xp_5O{8(!*=;G>
z--=IPdoXPlm)yOp8mXu6Y1VB2@%D6T*TI9W@<k_S{4hvn>`uLwd#2SVNiS6^-fTvs
z!_1z!ruKi%+|+$}X<2Gn<&0fxS%Pw_HZ6*{kl@&uBX{ioKHk9X4ELBK?tZh~Ec)Rd
zKkKYI-i>Di7+G8Grc6@kWKos0^;%OgF)glxQz1`Wz_DRk*}g~LUVja+ja;Q(eN^<o
zzb%gOY)gM_UVkb4^q;@1Hy(G}ZsW<|cXTqGz94(8!E-n5yd6pIErPd{9?FUaX)J6L
z&by%)6rGdE@A~56=2IH0*Bs)q4QX`|t;t&H5?Z6b?`l{~kgsceujyI#iFJ#6u3vPP
zyL&Y@&a345M3!Km2jY|c%s3+^OADwuq%h@t2$;&`^Wnl&8HKhRVhKeDS=pnfR4z-F
z2?<-qFlPtna>t^5H?L(0c9uP#D^PlkRlT<8nx|xa(Xmd3r94#`l1u+<WC<+p3A%DJ
zvrS4RbE&DNN~4J5#6^pj^Kbolp;&UGA(vz<XTzNE@QIf)SKD%4Sn0@g^HQJgqGN7d
zB~Ep>UY9btTs#$Fcy{R|PbI<Vz>WMz&4vA}RF7~K&S6u{b#jwzowChax3qoRJ-MfA
zcihj?UdXVn!n;jmn~8O(=ieDFllCOJn!Mk0+U)kaO=k{Wec$m%YKzXM58s^qJr)<V
z#JYI~O*tY`{grjb5kEJ#8AqpDPBT0vU0szLTbk=5nL53+Y^H;1zuK&fecN1DuJ7OI
zb5YF6uwquhvX5oOtCnmC+`P(SIoGPSR%g{5teHIAV_hl>Q|9_`{(e=m>{{WGQ=Y&6
zulyo#EoJMLueJZa9G`#VAE(B*hOYvy0Zlx+!cN`dEjp3dT3j7bey1$kW1{x+Pq`Cx
zw@nF}7h*1zvcBg<;Mez_?^N@WeSf6#{;Z$8Uc5m6YfDL3*qV&1mbz@4A6=;6nZC?r
z@^bsmU9Ut-Z-z~aZ1PN-nK3JJ+1u^ecOqQ%KF2LTEnpiqbKa`LN9mGHLRK0JG#1ya
z_&mW*e(&;WTn@Hh*q(l!%ODhBZoFom7;oT3*W+rzKSWEtj;SqQ^j><+!eeR^0=8dE
z3iuh#ATTd;$?-n{B5P)_D4%ZP=*xR#zQxsB@p6-Iz=o+J%N+_dw|CqQRFlvvip+4j
zcWQww%LYe&xBn-anO-b#w5n)t-;o&1+TyV%%xRLuT7!i7m#*`?zR2A?vsHA<g@;-h
z7o9g>c-+S7?Emnaa%+1=zmnS_Zr3RWtp_}Gl0;TdN|bRv+QFjlGwI-oshcV!CIqI1
zb>HMUX^}j0itKWwYA#-B$IJUW{=8Exv^gWi8+2S{f*)5(LbOYY>AnS1*DF~m_ZA3T
zkq+W`lD>M@$?Q6_DP<1qO3^d?N|l;qTXi=oFA1JCiKoa@S<{E%+QdUH9tY?DYSEqg
zn=_y>Cy@V{LWd(y(QF+>mW6zYC-`=rd3v|1^7M|m-7U%`t<pT3%#-b&%&6_|Fw!m(
z;hnHZ)mCfKo(z+-=4mY1|1CBtJ}WrUC9<)(c>eF0XRjQc8(g2*BuG08HKhosst77^
zc|0ng&?2iDpy}m5>5}mLDXPsmD@$Wmd59lY+1=c9#&?_cnWRTstSxs1GL#;9<n!x`
z`V-mr#&aWhKS@2caZi0#CAe_j*E>$OA0k{bqZzo&{Ef9{PVowR-Ru$W=GZms(lc|m
z%&82MoG!2wtY5lhr;6JN0f*TNtHW4sp7dM6D&3@VwDI<=RV{r%p(Q($n(kd%#dgy4
zAYZS+8aB&^x-J@Lgr^m#ZT}?gwk08C$-C7ZMoT<=&dpS2R4VPTxRV(6<JRGp3ybBR
zxlHp-adN5NxTIAj(&PV<4;%LT2pTYL_<tis+qzY!h$lI^#5>6$ZRs7~G(#ottCBO@
z-7a&5y6>GW>~dd&HG)eaWaFZa0+CFCdn%Jc_*dp$$a_8K)6H$VYdEGVPkklhsn#&%
zcCeGG;-iTR^3Lyco~7rq=CMg!sY|t|2SfSp)eJUyj=7s^Zdb^wn8z+MciwyCc9r^+
zGvCdOw=UbrD!HxTRNc+5iiUgo>o$vte~r;A{vwfXVQZeyEP1BQ_~P}4eA{=hIp45l
zdf>0UsKGWq)2no)_gx28t$<k#Cl%J8zA<x$f!gs4eC6D+8ZOTw)URDjF>PV${irg_
z#%bxYl#`QBdTJ}JoMfj}<v7oiNkXkrMJ;iu-ORP?|DSk1?ej^M+SksZ^5viR1wJ`u
zd3I&|-?Xa&Ke-loB=QwA>|5Mm)IYOc>0KxD>`xAPrcYNrIHbk?GUqw(CCL^wo`;;R
zoX;*zC>4pen8x@#O21yRV1v1hx9*lhhh@!v={T#r*{?O}hWBEY%X&}pUDzsAds2$j
zgc~GMym<poOFi2ZvS*RF@wUjBnHR6_eC(5Q;zX-V|FOQ@GhgT1PF26k5Pkj7xfA6o
z8#q}`o~k>^H!DY5D8_ZuB8z2?d`i{a6I1k~Yp#{`X)g@k;v4X3*SwyN(p|w%-nLcl
zzWevwuI0(kS`61J`2`<{jawbZ|5wPV;+VMsYrvV}zq4nYFiAUbzx%&alYEStib#da
zwOu}`$I2hD{VdN}nqYeQNc-$r(i_6~Y|gQ4_pqBfIeYIaP5ongJ>1?`D!P60J}`ap
zsv8T}aFlBX6+iQOSYrA8`Q&*PH|u}hQDC}z$v%~Lna<1~3;GO$oh-L~F}IweR2*`1
zUHCD#)xj;?#ZO+O3$O;cUu8KVFKg{m)#K6`pw5}{qcdRPpOm1RTeOW`=WjcGJa~Rk
z3y;Ipcb6{3JY$$+$=Klkc}BL;XTLP9UBViN_uZ~OSa{W!A@!}oQ}-8bcC$@Cr>yme
z<PEweeD77uqW3SI?c^s$Z#l@D`eF}9YSEO9b}C<NbC0sQZ9LS-D%S3E)r<c+<4e*1
zjPh}&mTMxq`Xj2}dZ;nGn7IBq$o1;T=U^_0ukjvlRdge_Z3;VLmyz0g^IiIp&GXK#
zIdQIH)8$1QukS4~JRQ}%z#;8-u;psc0>z|gt>g0MM?~Xw`@~K*6*%p%Wx2XVdN=2R
za+!JR6ZwiGr)$(sXSyNG;*?Mtu%L9RWA#r-qqk?-R>hyW(Ih>Ak==)J#)8oGf6j{a
zG#)FAiLboV*xFJhb8VsM#b(=xmbrIM%Fa5i8}Vr;i|>`@7be$MWZn>9y3)SW_2SV!
zhB=;{3r}9UdrsKuu8+hi5vMEj1Guhg_PZ>Uzv>#Y=)<gN2mej1Hz!P#b-QpmRwlB<
z$9r;U%k7It{>wkT;<xnFl&eR6=QRDux#@W3)IHgR^(m+B$=;ObQp)s^7jL-Au9y^&
z7?;`N|9q#S?*_Sa5#iTUAFRFMxMIq|)SLq>DGHe-N!lyCmnhEou<=4uW)$~ArDbbb
zKQ9c<`r0J+;{5WE<7^Y6j>ue`w&GR_?=j)fi+bl~eLpI7#$(cSmdELlY-v7il9_Hs
zWe?YtDxCHSaeWY2su`)hp>g$*g@qC>(o_5$@3h@Eot=NS%WRUeZ%lIDOt;*Qe(lVg
zTLUlogdCr^Kyi}Jb(?d^Weu~+c@(}qVdX6FIOKZdkfV!wo50J!BWrCsA{<!`CmxB4
z?U3DaXlBW=)(keu9S7Nq82@wrn7u#uAlsHEhgZD^16$@#Y!rLIQgz0C_nil)Hl|d~
zm=ks6xaCU)-<ZiKwuabFa1_bt-y!=T)@8oqpCwnG+&O1@Qh4FHl&$BcU2(Zs8tA@A
z!f7ezi4~mPM?FK;j%8hB5&v{+Zp|^#psx8L{);$HU9SvT9`n@siTq*@E=ebT&x6XB
zdYl4m7Oy`dGBwa8TqpX+2a%A4{PJ1HzcD1098K~rZC7+Vv7dAD;gfPMK5{Qh1CnRO
zMQ&x3yx}B0;XqQ(yyu+)DO~~_(;8iij<Mt%5NqjrQFxF|g-vonn$g4qEL$9p@TR!V
z=r$}B2;9Qxv7r6q&Ix%+_kF}>@$9?!sj}<;I+X=|CcL^~_v0ftQw`I<#a{cSnfN_*
z+M2n?O4_C+_k^X-kzcn^Y~C4F8QUp79LcUP0?)Q|<*{7KyBMFTF=>^?C54H{3UZ^h
zzs$e*LiKygaaWV+Hi5jCe>~q|%lqGeXK$f{<FPZTC$2^9baX3P;@;)eY~jT@;b3af
zf%GW{!~!(L4ca-TIk@aNIAaSFr;nrbnyIoA4n8`&jB{I)n9F5ew;Rz%5_l%vIsNr~
z-Ny&9sl3Y>mwR1nYUNBjlOhqA7cn^_e&V!a9kQ_{lTLOnJt=JUaLvw>zxH;D`%N)O
zTlgs=@pDaLbjMu@FMb!629vt!lWNarvIHsYjIDZfY}To^|0`Trr0*<@b!+iu^>y5G
zP;Eh`To?bY+UJ=$jjdnKXP!D=q`=PZ<6x+(!P&!^*`~%h!BNu3p>>zzl4%E<7A@;s
zl&114dWl<>w$Fu~4zF0JCFGnsvf|m4=+2gX9O?zr7CjD$a#P`s{j1jZC}UsAEeV(D
zLWY9b5k~^-rX@?ZB<MWXTP3Bx?v{9DV5-gY)mL2{Mcu?h+$1y~8|o>ShsAEV%5v>w
zxCzq=x12-S0_}Mrw<jIFwkdG--<;WgoGzPLgDjS5`(C+Ua3<X&Bl4mSi@V1`n~QgR
zO4$8Rxd?sDc71Y`ZO3)TWAl7&TymbmuyoVO^cg4J=cQ>TXi08yl<Zmd|CKL8QAftn
zzGta%Y3<t%iu_o-y=SWSmb7Efc++zjBm?9QA5`;QBN{SiQvNKq8xgav&0#xxFT?H1
zlbSyr))$ZX_etA!O>*H`dQS4`grIBA5{j+{Pn}yLjw#4ap5>X?67kkXbB@WCP(8LV
zzqwM4x~>Tir>$$de$r*p4Bf>NTwhi^bq?rf{B){fLgSVv_hVLNnq9h<v(o?XQ`K!3
z-HJ1$AAMaJ+aswim9fKPUgXY3*(r`k{gRI6Io3`*@G4R#(dUIo2^*)!f$V9GEHMe$
zU1w5F-#uD*I<xE;$FFELqd3V1Psxg`m!WG6YcDKh(0XktV|*|!=U$g?&Y=Z2U-*jk
z{1;H{<~Kd};95(g%fSS$w*iW0SNwf-Tt_|DhjD7k{gshEu@gE=n&fLE(j@{{OU|4&
zEo=41q{+V-+0EQoc7Mw(ICAD+r<ZoZx&MFJN@~0IPnzv=Bmdz`eS5zH?shDkwGS4z
zJ(zKYFH3AP$FwsXLc)m)xZmb7a+Gy;E_&0M^+LxmZ+c9Mq>m%#jzg(O`r6z)BrUY=
z%S>+n;#*a^sIO(wS+n^1ml=$Le%=OAebY{KF5s9pQ`owL(b~a}JNd}_XBq3SG>eHW
zw=+BA^I(pE*4se6wV5KalibpibY4!HHKU7B*)1UPeSo5G#x$A6rD8!W;h$!mdK2+=
z#>VKlQ&Cpu|MxR81Z>;rsK9V(jpAt=rF+{J&fccEYnqqKorMar*F$ZNY<&9A??T(I
ziSG|Jw6i#HxZYuWtv8Run}Kslocpv!*`5W`Q?7PeonXspXz|fv-{SOqp11S_Jx%kR
zB%2drJKpK8Sl4Efr<#0q#|$lY+m;uS8C=uZG;<Cf6%EthJmEv><>)KSx;;)_GcIfO
zG-{MY+Fxl>TXU$3XR{hZfbn$yHNQUPM;Wg@r==~`BKAW=DSn2}3Q0Rlu_HDYk3UfQ
zE}5zSt5vk|OmSJ^>el!f4T2|b+%lSWW^JzK{FrQGw<`~h83uS=Dv0E+y2Pf<apdl+
z5I@mNJ}g4{DUy4FT<&Jg{-1k;f3Mpl*Ee(bac#+r5ufxxSbfFmPp5=Sc^|NN<TxCg
z#?<~SX-nFQ08TY=;S0GAOIN9huef(g?D8Uq6DekI-18pfWO20FJh^)(kMm>Dn^3vh
zfIDVcjM7_NszT23NV6DTKIG7INMeoi^tFfiq>ntEaYCZU>F<Hl;a8YEFEmT^Fzx9m
z^{rdZcP!hE>n&5)q-7~LgzP=PS``}ly}hD!ca}%$>>S-Ib5(nK^IiYk+K{<4tnH&H
z+wF)dk=wHhZ*w_FN0+6BZ`*is>93u!>F!6o7pCTaEu7le75>64hH0G~+iUB66NTPf
z)ZOBAd0Xp@qhdiCi`XrV?)x;@PrG{R-l6}Q$F4DWbG$H`s~ecZarIqG#?&VZxr-;i
znS0bIY-#ApO`@(F?v-BOl*5*yozwVs?&Qv%tsWPgjSipI$~Y8$gjLt0*ek%;dd3Oo
znxk4*n7TTcbZ1=dTyRwJI*WG3!TFXK1Js_UJFZW0eV<~eKGErULd5cKyI8-}#b-P%
zZpmL5E%WVxQX>O{ql?QyIl<%ECeiK3!(4Z0oAp-wKBxZYTfAE`r+VjSyIZUsJg)op
zU1nG{bKRz#n%V^SZHHtp$obuFVk<6Kwc#X(dHJC@m8y%q)pw%<jCbChv*C#2&YGJ}
zu{W4|Ot`f(nxf)M^*fxnxE#Xkk4E%=*Vxd`x7pdg#Pq-Qjq{c3nV--3>bb_L_P9I$
z_XYf~Pgwc}>t;B9aZJc&e<_eRBTiej*1_<StKn6xkVWo_J$q*)O*!_y<Kxu0yW)N6
zu3Oed&x$Q!V0C@{UEbX#M<D!PDc4*shIv-)E9d<BBjXZm*<Nx%xbr@jXK+Qa@v>CE
zgG$kE%-bEM-^~91NK)xj<c9Z>y64w9aS7|~Ze8iO-D`K;sx{n#Ck3o~nlz65d_JSx
zBcU>EhKFuP6T_43mG#2Y>^C!b{MI_6;=P7>>+~aL(ow6+ZY@tSUU$xHza8(T53GrI
zsuNgZCoVHiT64>~U*5qraq0QbVv5RTUDqV0vL#z5v+px3%k?_@=h*+VKPEaoF6`i6
zWnmR9<!G|mmBq>^>B0i_Kid|s$YBVO4HUfY@}-bP^Ufg^%>xN#CGu^XlPfmeyK`x6
zfCr~yc`MK5wm*}0CY&fyE_Xci>|*$l(_K%T)M}$|9CF&?6!KVS`+6ta4NTM49^w=4
z3A-OKr@PU}xOn>d!&+N*YOG;=7Erxs$(@?ZP3JbPs@Zx(;|2c}iH*??ypvQ4+Y344
zK3iwn7#hm8#p~a_>i0Bc*Q`*dyK@9=7H^NZw{)WY`6J@Ra~C?4J+)_9tn-g?l4k9d
zLf?wA=oYVEOT2tfa{Vji+R=9?H)iiDX_o01S+lvP+7>j%Pjj<RoBFI#V8@G_2DMrL
zOJ;kQ)UV>uRoc8g?xXQ!PQ$-1Po@QDt0^1*KM*)CfaSF1)#gx+`03eBx4%2z`t97A
z^3%%VzQl~*8*^MeeoXjy*L_dPuHE)7e`O~AO1+UX)mU$zeS(PWH^b#AvGLaWKNc~!
zFf=lAiisR}n&8wDvrETELttTp5!0>!6Ai(~M>?6zlb%!vrg%(LX7{>rr7KjMG1p_O
zpoiB|h9INZV=<na6lOMB6>>d@oN)HsJl5EMA`>>ntqn6w&hu1q>~vsdT`uMmFm(ab
znxM%XH!h`hHLY-79yWK^(cZP)>)vcARXw14XIn1oInf`}6ykO`|NQpVIwV(HcS()p
zo9o$AB&z=Xf3;CaH7#U))oPY2ua3{SsuA@!Yol<1<d&5BzaIVH+y%b8ylnfsH%9k?
z>8tLIyw`3k6}`E@qN2((N8!Qu7R^JQ{(=cIKR@ngHRs6a+hEDgcxh+O%5$v0552i5
z%I9(-Q=_8!k$#bq^1ek3elK<F-W>9Le4tO3y=+ze|3~jM45H7@*(u{NAxA00VS$_B
zDTl@z7M=}Zd?F3KfzviEof@U(?a2_m%|n5avy3}HMbqX@uF9HiLJEw0Q9E53to3#s
zoofAV(nZ(i6qAmlMglUg+ywrvGE`CE3_2~Qd2EF{g9>Ms&y=$gi(NNY{aLtNOjCdL
za+Ah&CcInDyZv6XK$|`2|E1$9$7MQwj5w`wg@yF3E-F|!YiV;nR-15W$L$_rhV!=E
zpDz11FWb@E?Z`hXNzsMB_l4rbyyg?u>-m(D7aB<{ZCc#%?DCX_jW=fd?+ZBnhMTid
zrJ+;QEG>z3$%4KjPsb($k&dei0)Cwg<>;3>u`G_gX{+PHKfAUr2%8k8wS4&!jTNgI
z>_rr`R7EPj-ArRW@XC4Zl37Y?8)FKa9b`QpG_macw98}d)MXDE_VF-&E?ZK<JxN?;
z@0Ug8D#hC`9GZTkN^pf-)Dn>e;+Occ-#$3c_0MR_X^y;#E!RZadbdpE;{K4tXe4)H
zeqG}49m!X7R<`qG?|yY5YG06?$F0@7qW)hqf6ySE_w2zWVU2)Ee{%d5xJ$RbQ_x#t
z-NEd%G5CFk-z2{(g-<7bzg7>hpH-#uZH?H9$zL`d`?ZK;+l%!F0*@ZcRGWL^hD$@B
z$ItxSnIbJGm&dB7tlD(S>`g&V^Snd5*KcCEwLnBnU#q!UQ|D;{Q^MPV!>$Ke3z~Vn
zZzs&@=2cTz7`yL7)Sl&X-*;)h7h~U)Iz^Xl--^>$Legh=>#=*uo(a6_GPz78_x&2N
zFeB|+9zoVBvy#S10SpW)-h}X3YF3o3TX0*Bb&KV5Ufn4HuClxC>Aw+Qw8dIaN3`R2
z&yo2a%YUpmXHzlB>ucu9(7XFIu17p(@6r%xG|h0I)1~!)+uQ?_{u-Dwyz0Fl!6wRR
z;_a8WsEcXIg|_WCycJYuIPFx}-e`2^VRXuah6|@2u!T=a<W{-xK*Z|-*9Mbzz2{!@
z9X^<@65DlJEN6m~u3zv*)6$p5Y?l`v(O9P1x3{l-{p_>+te>Qm#ROCS1@s<wx~3m}
zW3kuPjN@zF4BLF)WOFMCT@2o#(KtmjA(g4~)G4MLlNoH5DjZySv*1RDjd{}w%ZrKg
zN>(J#U69M(e=Bi{$DC^>_FOEs4-EMgOqt}neU>P_Stzx5otl7a6_<+81OcBv!V5O6
z4G7b=yr99-VAfbrr11ad5g(Z)lg&I<h-PiP5Ezog9aO+Enc>zc77>I0jYbDJJdBu5
zv*bi}8Qu649sPho@|J>0%a(<_vTN>$#ke$aE^&3UWeU>MS6Z^rXY;J<M^5TSEMyJ4
z@T@J{N|rzUxpT4Dk~xXLy5F))+I=UrO?0+Iv)HB;teP2$TtOa9cFGrA>#GD89GE=0
zlg%h#A_GH|u*7MRDHG;sdMI}D@8wwFWMtbN#$P=_ifgfelg7Ks^+9t^g(%;CEI4_Q
z{xlAOd+kdc+Lmr<Qq30@Slu;s)pUs?YQK|@)`ujm-qENZ!Bp6}IwEn~et~X>zpKwQ
zcuixt|8%(v^G30)OM=!m#a&fk)M?huS~5M%BG|P|;V{byg|O(f6TGjh=3G!(ey#cc
zjZb_rqMbWGwXWOR@S;^Qvw31xhQ#3yAL7|Kbq=yk+SseOvBM|ijnLbN+|@lkU4L)B
zyt%EwwIeH#DVpU)@RFmE22M{m&t=sWU;4?~U|N|<s*2}Fy-(71D+4W<cg`$wU@^IE
z#BY5?aMr~y924K`@J<k{oaIq>Xx@s>Q%mYQIXrorT66<8*>Bv#;j3SC(tB4+XvzZ>
z4elUMj)T7HT8^Bpw@zvYRb(g}n5^g5;2tJ%)m1<B27~ymOKtaS0+>=eCf3Y6*~)n#
zeBb#a>p9j|UeCQGT4L*HV8+g*ckybfg2SQ<5+)y)mMkl_%-qFco%@`-Y)`Ss$0Y@}
z?`NK`EeiZUKhEc)eW2N*y{sa~s-yGkgc4-T{~q(JDB_#9q#=1G=aH@U)28HXZ&=|d
zX&ET_md~$d+1_`NT6TeA^Bn`M9#5FVvdmnzW37jw{nRThqNX=L&1mIPn0`lJz5Zr=
z&|mQ@W-5v1Y%-m57G9NJp%IX;;F;j)v816$G$OWO=VATmD_y)bPgl47ejxHJddmi`
zgc#K`(Q)^eKG=Qh^wg*gm-Y*M+o;gx(&-W6yTX}G`nXeB*P9tRqU%?1R(mmSRtj=T
zdvB8+eO>DM_HA1uDr9vt{{D)(#I-GC(I)B13ppIrPjj^=qzT+$T7I0bH2K^=q2nz!
z=~j0S^X0$4p)rXg$hPi3OSxa0b<C5dSGFBG9!l0%C%D9^aHcF+GPU{ms=ss3DhM)Z
zt@e_<60&SkxW422_Y16&o5GH6jX99e_;fepE*}Ti?F!7C*PiYbO%<3~mcbf(=aJ<c
zgB=PDe4WjlOc@iV-&wbPgRAtGgl_-q&wLoVf6kmVY08urc_ND^EEjvz>908>V!osO
z#tY$lZd(8UJfErIW1f|X=rZNsFIoO~&+%x|U3z-k!V|l5=2iGiczROojfM87LZvE=
z9zCwNd^<bN&yvyXFIrU-)3N)CpV#{d3W|*WnVhTcxeJ&~mvs2w8R{SI)b-(5?}vRx
zuQnXI#<g_ar|=+;hP9DlAF_LoM1{HQ{@-9(zjm@Ox2sG7H@Ew9wxW$oMN>6eWv_f%
z8`hAm7t=fOO{Bi~&U3HbXZ_}Ga(})v-<EIL@2vJj;oUA*IS$Bekd$1NpZY`llJT`I
z*Vd$^pS@{)de*hg$Lzi>71iDNr%ESO&@9XP^wck}ZYnUTdPwkdykEoNuTsub)W7&`
zRF}|`e@c<}aut_G|CjE)-Ncru#I)lO?}P*HHg!&n?oIDr@Tqvbn#khBb-?Yzo4lN?
zqv{qXC*L`?JHkno@4beG%T%^SV&}T^uVnB2<HpF8z#9GGlS4jt4x@gRi|#F^<`fpU
z0s}V|1@`14?9~@s#rT?77@B#UJY;%`qgXgS*Ug&oU*J#-i-*kd!mJNp)t2*gb@Hrf
zzF5VRx{q`2q$A=Rjy!viDeT7htLog55T2vgF1+wt!`sdGlFg54l@p(R%7Y2L4_0XX
zJ$IgolWAd<PFmmdh3`@XN_9>&<t&<z&MdR)1efZ`UF<^cO>b3>o_umnNONh;vGh~F
zKKv14;NW|a9k7TuV(IAz&y;qpiC}qeU-a}$UX#j&HKKq0G{o{X+yd{h=}%p@!a?Qy
zW+$fe4QG9MI2tWlB)Tt3#Io_<e;;L*z<xbRPws?Xg#NdA6Q$lSTHV(4eUo6EN4`hG
ziC8I~AGZ!nH(62C*8DHaSNjOlbu-0H0f&BSX?G|1{_5jfP@;SN-T#YnrM!Qv`ac&P
zmiO0g;K=&CBJYo$zRIx^S~~nHSG+le6d39zDliyYywKma<gC)+b?R$)6<@9BTi0xD
z_bKB_UjGk|JLe8F9L~LC@@l!kBO9|r6Rs>)5@@=cv?;i+O+h7r+v~$5i;25D4ymnJ
zonzy9-)W`et!K>5s;Vv%!lF)U|7cVAazWw#dDid(^A(?1FY5bobxl)A;Cge2qt<}q
z@ga8cJOvxRLp=tw&wV<0P3TETpQ{WzSNwY}W%qa2Qkl#gnA8;XMHBe8Z<_XFlK$*G
z7bf8a32Tqk^}V<gVEAwYZ*QM|{F_%kNBKT3<NGP_YSE{ci}WM|4IFs1w5yr?Zv6MR
z<oIYI)pcC1Yhgo<j>s$jO)o6W5+d1D&15gWnRe#3QmR4?TX4uFA^u0z4qCx>P3n8t
zcYac@z2v-<PviZG1+s1*H6jXCp1C{3T-?KQNUf<!J?B;3itc5bPOEEeP!DiYD?0ko
z?ZW$I37-@;X+3-+k=Q9=%kR3v@71E9mrGjp-+n$PlE7@%CvESp=i>fENXKFOMum#?
zI-ZBV9tm{USXv<V(eGUfpW|!0txitbTYl?=$bOzExAXn4cN2dd@c*+f|Bt|?KVBv-
zM_;%sWL5A@StWAGh<B3l{7bH!PFsI`>}EQ!QsL6MlxL1-Ct75&rT=45{@h<##kRe`
zdq&f;jH3VE)*Me2r#;)g?{U9}5|gi1{gPvM_!dpHTB@*wVV6bVStlk9`<3sv_^9<w
z`Y<JlCD2LL%%$;66YKg*?r(aGtdIDWhxkoB*Ua*WZ_Wgc=@a}N5BZ;+r0f2EZ$*Iu
z-&NN;3q&Syo@(j5`0kVbUn3{R%|ZHCgE)hO*(duw)IHX-?&z)zBh|?vQ}v9!OO5r5
z`9B*dE)>#AEeSnnqF9_3ndi(k#bM#3<pS&*-5eL)EJzcS7CgCzUC2F1sC#Smq2nu-
z^h8eP-pR4()Q?3Wam#q8DV<?iug=)N{LmSN4WVaJ1Y^$TsQx@;$=e@Z6u2iR$yvi+
z(v?Zut^0bD?v_)FTwecwqZJyUrM5t%%i!Q2;gT&ARXUDHb06YO$dZvP;g*^5TSLU&
z^h)AILwVOJ4P2|-7B9W#utodNME#&CcGuq<KWACJD2TP4{h!yh*oXP4`#S!*7)yF4
znb%znQ7ZZ;H${oD@YaEMN}X3;_gzu63i90fR$26rTE4OO!L@fzF{_+On_!ZaQJt+a
z`N7Ub+%~DAE-s!bNm>zn{UQ;Lnrkn|-r1u6A|z)$pOX90Mj7^}R_cM*%>tTT^=>!l
zt!`%iRHUET8KfI(z$$E@X_~KWk{>@sD{YIg{^yXDs|wh^7YI);(Dye^4KbFUo>jY1
z<(^7Pweunt2ksv(Jt4xTzQ*Cz|9Qe2lTBND-XvW%?e&Q`-)B}iS>+aA(7ki!u3kzV
zmxKDaSFB)*nlHLj;X}Dy;L8nf-mef2+r;ys<GN^XxJuom<*T$OZwd>S$hRq7YV&$o
zWziNl^(}UiW@*gkVV1!P`bl5qP8@8z5YPGLJ6D*aXJSm(F~{W!POd3VkHwwUORmdZ
zyIzx9{C}d`(yn^%Z(AM+H{AA5S*!YoD=W30x6$ZZ{r~U=X^BP~v-3V;Zo0;9nloJ1
zDb;3$+*Au^GQV!Vr6aBWi^ZGs|1wUuFg~sPrXMs<*|V5isV!~$F5k>WB@F&W8lTb`
zT^A`^aG861F+J7YrL^7Wm`IJq%1K%RzNP<NeYIPvY9A>+5ALuIQz{eCyLm0v<Q%V6
zho0k#0I3pv!>Pev#d<zo%Rg;l^fb88e`;u=XhHR%P|XUJxUIdv7d0)k=wfs^-ou=>
zQ1y64X?UUOYk|N36Y<i1ozjUsk?n^nCxupLZ8tX!JoRF&_5!g`7s1YWrpP+g6)P@>
zvo%E@-5h;*s(!{s^;4xQ!kA*?vlPq}-@owba^04E+0f5JCAGxF?^M&i7pyTCw(}fG
z-QK}m_xZAua>=zv+po9$bSgdTl6JPa(bvw=IOVpecD!UfXGf}YS!4W!6!$Q#^;t>#
zZ>D%oiws&_e|qMFDQm-{%lg!Rmd~@8G~;GSl5R`x|7mGjpVHE^RZNQdHodia$I-vb
z_ifH(g<ly9a&Ii)|De>Pv>?|gBT!}YjW2!GO3}_YSA2M@>hki#mTOj*S)H#m@ysz=
z^f`H-n(K^roAORp1ph7z4*V7L-z-dgQ^@-d<$4mU<jhy`_!V5`E-JTDdG%xFTRn-p
z60S9B9@aA!emYV-`Bs0c&%bRON>1`wa<MAuc%AGCRjIh)Y5OYh5POK#p|WtJDDI?o
z#gh|%m~%M^F7wcgnQQzY?xOxOv->M1D}+2PD`bj1@cHAdg}X0pUSQH+omi>h+&)8c
z+J>jw6OFfQFf^GJ`ucaArPk>c3R_t>FiL#!nP$8`a#rj&C6j9#H~v>L@6&Bs95m;s
zq}ytyW@c-}ja^%kUv@82VlXi`a$a=e*+-e|u$J2^db@75r2h6$d}`V9thM&FS=x>5
zSr*>HQafK;mGd52RVfh^!o|2pB&Vf9{lAjNsb#s(szjSvt<;y~DeU6$yRfl!hgNh|
z&%qt}$=0EZ{Tlhodv*N_cbE4uvuvK;Z2Wr1^2ZSc&bRdA$`p*s=O`DycDUZpXc;hV
z=}jMTOEJM!`HKs*E}E^~RarDyNO|Jw$IBvTi%wqBz`Le2+{1IvzHSZYoy(j!r~Ku!
zQJ32NKz(O>6YsjKQxsID8s8L+^K8tYRkYb@)9QEP)uLO<e6=T4@`alGvDm%Q_W$Sg
zm5-H9P19M{SG2lvQ-4g{Rq1I5EjPG}#e1JVHTh4n<F`*?&s+?C*Xie<S)O#Jy?w5J
zQvcpVGE4cLa;3LxUz)el_y3IJw_a!{T20!%O<9Qlm&)n4HcYV*PKS1KABvDv&3fLu
zpZUDPm1mXRM{{!)WqTGZd%F8{q0MG<{{5VnL$-&%x>dbTR%-wA;66{;zW85PuHIzY
ze|`P^h!x>4mcKox!_mZgU8b0qX;I4N;=eNvEY$V*ex$^Ad;7}eW^;X1Ip+$s2P^J7
zuAZ{JEZtmKy5K_b4t1YJQPYzul=p_7=y7N=uUO7~=Fn%gqN5h9E}`$*JC~lY3VpQ4
zC`>EKwrJ-68p~~msyD9M^wKtEzN^CaM-8)Q_AcDydN@+@^35#|@*|6wPp*`ly;<vi
zu~nb%?%2q`ch&zzojLSKGj5vkzD=Bu%&M5@?|B!P@cHcBIZCf3xBE*dJ3F=3{#J;x
zXxrJmBY9qCL6iElGS&UF6zUGGvR<SVqIK`qav$-(`VXp(@V|f0aL!3bMQz@pmqFI~
z4`MgxTJ00RekJ!--|y%b*R!u|-e=7C+ePhL)SI%P>rYoYF{Le1Ft_eHUUc<9^;}cI
z8ifS0s-n*)GA`e!UbJ{Z=-Vlw?@SDDm2UU(W4*oiT6U=6WX|(PGFMGjw4HLnCv-{i
z?%1WOW;^dVE3HjFx-`deedzzl2Y3GZK5*Iebg!Lq=V{wo?FW{#e&4!h8~o@$(^uQt
zS;o?@k}V6i@A+!E<6Ch3XQj9bm3j9c^7?6gdlYzjv-rMQ<uyT-o<Geqn!Or$ycX~*
zkpFekflGwp#(KVkOBI;Bn9@p=y4~bjG$*<H$K7f>bnD%m`J#uc)+qOHc)nx#mj1qK
zcZpdHoCYdN2~5FFFLusTImM>HGC@!wLF8aF2kRCghgS;Pj9a5_dRYWE2yuF<Xn361
zxVV9Z-Ms3F#;3=Q6O@DJ#dr!VJvB|o_s|^8&C9%7%tN0odAaGy>G=-zyJVz{S6p1I
z@7)!$V2VQUaxZm_308?r?5l$<3N$xw<q7}4K6lTwLgj5|!;EB@d<?Bx&rEA%T9Px-
z$n~U%rtq6bTeTCM?oSVX#>Ju2sxbWk6JN2XP3p!+)1&#^%xVfgrfg?9sUvn^YJz0r
zoSg@r)_mCfym|5N1#Nd$iLSmTcT*;(IAEI-BR}f}wH?vs2gCLT>us4^z0mm{XXbWA
z(}FMO!#3S~5+)JoVBIGnXy^LNsL}tO)_&!?k-`fn^|@r0`D@y#RxZ|?y5)Vvyo6dt
zPp5e?t1ho#`FhK8>HG#3t^*;#>{2Td;zJ{kwuZ?T1UiYUP6>S2DwfvquuUrKhHLeI
zPJzfex!g@hJA*tXJn9l&@?(0uu+PKm2`1~diaDAWZE5_U!v5CdW?xLsLB}M1t7Au#
z=E!79OqyL-G+{D}<I9T~%NmbeOzR9?rs^k@^JrEfZ=?FN8Tu#PPH7ZOOp115*1d8&
zPsmZ_xzk#Yg%19POWoCi-tKL`H?N*))`a;@Vz&f4oBvtPU-&EcN{AC@Ub;f4aL$CQ
zVk$XHS4Jd-zL?56H!9*#tLhAg#FZjt28o=qGqMh?5Ywxgz!IqPMa-$?YG`=e?pvYp
zeTA&i5s7AUD%VA(<z19W>Pzi>>Oav`dhO<OS!Nf)m((uP&D6L2K3ODX%eD*7$L^-R
zb9}0FgiZ6M(3e@ePw<w%^H}r7Y;Jy@){<pY-58I{I0;=97R=e4RwDm@3FE&d7u}?D
z3ZL{zw^{^M_c@)4Z{&>bnd{Cjf5q_F8{QQK72K0FvfUMxtR@{{*|;X*C~KFx{GmsI
zcV3*f&|VPO%$v1TwB4ax;>!hDsjZGeb-db(M4~trw4T!AFL@NB74~)!i`Ri4N%2Nv
zufIAT*k|%&!dbbMOKxTacTbp-ZU53tJ>|gJGWQ5A&q${jl?|-BYsyR3dnFoiWk}Di
z5c!e6LscZuwQKo`O$()ER<<0rd!$mah~bmS3g61Sc_Ghb1huX_t!?Tv@Kmzjb@x@R
z@6)PKnH1HB%VG<o+{DwSFlMzb`&74zbJe0>S|1jF)2ms@G((|*TSP9!Ewimw)%8D9
zYj23HQ&ffZjhIr;iCfoKK5c#c^{Rj4jGqVD(jF&XG`f9bT4b3?<ex9y&m__oR;op1
zgnnBuG;Q(4xR|KRZ7=`yDnId(aXch4=l-5f<s}&kj@McyFbS=Rd75-|KZ}I)ix1tr
zYj>V26-k<-n{vc%&Er1rB^w$O3m;Y)?w-SQ-FTV%RCm>W?noZXNh}lZELZvdGx1f;
z4Z*Ipd74TOBj417sVuS#%m_Flu)X2YexnJE+EJX7mPWOhvT7(YG6-C(pCzUwv8_j;
zRrr9!1&ghsp>LJG%@n*IC9+t8_gLT2Ll?cZwo7c9yVk=cX_5k~!qT0+Dh;c53jLWk
zb%M%-ho-#$E${gASvVVhZR44ka>9{sy8aW56s4^*CNtz1+S)|=`Twh2$Py5F<aFn{
zNsk+sF3f4$6{Yt=vv1L<kWEcfm8Uuc@BiZUuITwnqa6y)yPZW>hb=u5R+?CG-e%tF
zDUnP~4O}5{g<_l)CmA_keDdSsaIiS&;m5#oV2-}xM6pEyGoIb>($frT`d;yMZ|ud4
zn!=slzb;MJNzr|6n7UYgi;K(J2CFvhSvRLSEw@<evdo2Pv4D6%QSy4WOE1mDUFWC?
zZ(4V^;F9^lg-i0)I2>;l1X!Gw<mJd#G$>SjlA-l$&hfNa0@F4YaIHQv>qv~7{FQ@e
z9NbvE&bD>U?br25=JwM1?|*bl$c(dRrm0Sh2uW9M_;2ybA>!<-MzswL^KuV3+Aj_e
zIsWU&0>-7YWG+toYq6nK_gFBar-wrf^MnS5IiKe+oLC+A{nT{>xh0t_j5-N|n_5kT
z$}&S&ud3BKbYtzsjoGDtq5|4jRN7UKC5Cv+zRC1+vXjuK&GSkpDSk{-aT8META$gf
zqPp`y--fVh9*ND4ViIa>i?kJwbxw9ud}J}_t>&tB&M>BFTpG4}H`#9#S@p7K^`S*u
zgi`E$SDoN+5@pk!SeCRiN%Gc(R{zrQ@+8xF74`1-9=R6JTIY4oc&>WG^G?<ocWUOA
zAMp&|HA|Qw!6v=#<LUcFzqaqMRMiMb_`id5#{vzb)`PRYm;BoMfp=NWVW;~E9L6)m
zq>k10Ro=fFWVC;m;Fidr+-J9LbCPnI*QTs?Oxn@pfZl_OmY8ijGuW=T-o6*GK{;FL
zoLaf}rHMHk;<s=naAaTX>%MfXMQ8PSzoS<Syd%AokF>5W*;#D#dBV#b-A_FPoP##M
z%zC~mVcrz;-M2QYMR}}|xi$CQiBr$G9Q{@QS@Sx{Kk?T)b=<nx*FoSN7t7vnmh0=;
z92T(8z0UII=7maEhxx1z!sU)0b&txpm7u!q*qIIv*UqgT9p?^bzB(tUKjCduufk$g
z`&qXPa-QZ)vYE3)qx8rGCYj!L^D`%OYck`4+Sbneu~hH>oj#F-1tLpzw@qLOF7n&S
zWOd%Ht^28TU#HEJg->6)iSK&;??{@xYrwg@vW+o*tlIX>&%$=yp2{_GsnbqxwfA02
zt;;rZ?R^_2_o3?fnXFURm2Y*MtJA-qRt-B^x|-*?#oN{K4FA7BVDq<FY&?AeL(S>T
zLQki}y{6);PJUMU`Ou{KZRK>a(m8r@ua_@n6W+}u9g@3b-ZjU}^0hfTZO*lqXgHk^
z&}+I|X7)L&SEcxM_H`?lIQ#YC*_u`**K>+u)oi=6*o7X4ZP^m*VWxb%YSoR2F-z>!
ze`}U3uQ~SO)Rwku+lsDkt}$8@csD8Gm{!34y9zfA4bSHGS-0CXg)v_FFRS=>chbb9
zaChswk>{W8(O&S8E5rV|DMRDE>_@!kGZbWweALV=aZgxb!ZO)Puk&f<a#`tDCvF*P
zSS!um-15X=ORm@vt&cgkkELu+cNLkmv#<8itp>HY=*+D<Qe!kYjdFGt?F~45HS$BF
zeA)9;^GdFqO+A)3^AD%TfuNkLr^2(stvAk4+;f8G{rjf&xX`VuLJtM6-*a8N;hoxf
z!@tiCgs%JK`-`t;{|A4IL(k90Hz_<&=q)?>_ig|GK97C>wE`txxhUlYX0og41Tect
z9M-PiD7HsQ`G}H4$3|Q8PZG_B*-WJoTfQaEnkX?>Flqk8q);c-ZWj9jMkkgQ2hNKE
z|G5P%ct4u+d=dD*Nyhe>zU?w;rK5u8#{}Mqi0HL>3Y(<3mg%JwIEg80wjL6R4M{zI
zy`@G-sHnsx`*^5Vh>EO(Vuo6Il!mKpV0r0v`=twXXC<2DU3D!C&G8H<X#bwxvAwO0
zouO-5Mhjbey;wWbhWhdffr^fHj*A&}FX|a)2-Jy6eqPLZ@R0LoW63B7)8y|$r6%qR
z&9bx(x-&7TO!ZJ%CZ@aA++)@E$ec@pp^pS5UL@8mj^doy?3dxmVHj<4P4Mlc=xrCc
z7K(UrCvwjY$kBEbv3w|OC=@d-+56(N)crSNu5K%3oRk~9tkf{j@XU+oORUQGx4CRO
zEPC)t?Em|zTH7~kt(v6u<(bwwll=2n7@wxc9iD7+VzJG&5Gk=k`TR+u|J6hvxWxYt
zE%*}N&)LrKzg(o6L!(BcgQ-m*DBL#4fNi@|2h&UzgDaU|L;QJfh%1<9nHd$mT3p0B
zwb3zAlGQZ*#-@N!kzx%t1L4x*omZuIe(gH>G3ulugSAEp!{SNLH_2EY({q|$@|Q`<
z_j=IPMk%)+hGEx@bhfb8HTJ}Qub#Ed`Ludj@6;)Zjiqj`sU9l~CvGZ>a@7pwP>hTS
zi;ggHJUDUTL*rc5#<Ym=Qq_vOW9j)C8R?eoHCr>77%G(~_Sd`!uU{ebb4P*0#SE{-
zCcerN+b&HH7Z&+%aH(?Q40nf(%F|xBr#&{9B@v*n7`5Q*jKxbmCjFFNHp6o*tK&)y
zHGait?FfNAf@%qi0(YsZFHGTHvcW55lg8O+f<6}B*QU8_uv7?A?1^&Lj2F$j{5>v7
zG3@vg>!XKjZ-38w;BIr@z3)+bK6_)@i|c)Fo?Cs?n8x7N|H*vr=koTt*7n|Q{eQO4
ztvoT8<6^(e!nt=jH3G{zTDG!IHPe}<&MNubaQ3l=U6&emDk)neCH^Z(6hAiO>t`W3
zF<sden#yHT%GZjRT2xmhW|lt=%DtecpXgw1p~vOetoy=~XH~P?M*;h1&5mMf8aMd1
z8MOQiYKhy@;@(lpRb0yXQ0ae@P;kQ|VVlmJ?Nhu{!lnuprA2U-M|hSm_>neilIw)0
z<prTRMXA$%9BnJ^$Z-AD<{{Ny;vsPJq-^QUcE`Z!jB0A3x7wRM1fDDwYBSaOA2^RQ
zaCVbqWP-9;6q5>nQ>S-Ol(&;d*<{tZJF5Qtlv$QxF;i3H;K43gN9pa;9Ojn@Y_afK
z&MEX$amwW|L-B>i1`|yU7IvGn2%WMJm0X;A>6^ypn^sYW1$Vh<UKdroaziv_V%{&d
zxYI6i4}Ro5?1)>rYt{NmeNWkKSPE<RPm%chtna=_{CoGgyR!NdQl{UR6!}|GfAH1n
zN17Uktd?>XuCY)o<rHL{<krx5bo!f<^ZwslI3ex$gt~_fCX?1yO;P^0MCVQLL<tL_
z&L4@7c1={-7O?S};KpY`5>LB!r^>t+(ziVxDEK96L1**JMQYP_CUb9TK7Wvl*>RGb
zR(DR3Q-aqNlcP1dpBJTn7xGf;30PSsqP^IQCCt>zB&%$3z>xrs!tx7?1FFhZazfia
z35FMnvix4!R@EBr&`{APwz-y#rLC>Mkb}jiYg(_kn~iiv*~)O4iT!=%Tl&9GbCs?v
z(e&=UruyHbqkV=^t7`zqv<<R~%jT&^wpMl~2L~)U9=Uva<J=cB7C!B?Kj;y2K#${N
z)RdM<b&rcTd==QD?!YT3Gx5ms{R^v?Sgw##R10<6`k%{Nq58JfCdt_nn+3N@_HO#@
zeUN3<eOIx#gEsp-S6yP6bJ%(3(nqx~r-rTCI7h~T^(<%qXVq!nMC<qLTFszlaWt#u
z<f`_@9qKnc)dIT&I*M#3iAnlSPPnQp!6aG8$>=T?C~k2`{OBd8XU38znv{iJbU*B>
zTCCBfx})fq@B&pgb5)I*ZtWtL77H0JcbS*V+{)IU+u3Ydrmz0AOIk|yk&$3Yl1#2Z
zc(}T8aB8X9mnqeUCVQ}mo(&f@YTD?yIPG7z4@1vl$wL8g6$0@m7Kf#n*-K3;ncC)=
z<<}n8F4)uWv1^HtOhR|qX7exgHrDe(q&kW-xA<{&IDFitEVJeR29?U@qk=Un8wI{k
z_bA*NuHE_l$RX8;$mP$r)*n$>cT8~NPW27xB01H|<t8ql{l&bDTfac4n`L8l)2v{I
zMe1fpW}W*{%2~XUd(+B}?%A`GtZ!^dJ1U(2u`Di$$M<Q*u@jy7JA>;sY305C-e>i=
z?vur?WzoByK3VPZNBz{S_O3Dkj|nWE0*sa^YqS{W*%z*9o3n_+knO5*g!-pLQMcv=
zmzUm(IvKNgt$^mllS=C})YWEPVx1+lm%+-FnbCposH2;?tc1(V<Bi!Lxz>xa=x=?f
z$78jgJ0<Y{Fa5tt8-8C>dn+L5mm`~-px*l<dS&zez{=1QJZU!0jPtV7nD%V^A9=D>
z{&uV4@1nHBVYTAZjxO8u=V*Gz^)^?n&1T)3pT(Ses?*+NcId~93`Vu--`Te)FbbKy
zUM(wASzmEJ=+>^>PKI-<IFk*ymN;;&Il#B3z;@&7X}@chR_biE547NzmT)I<_5_b*
zEtMkC(PqoGAD-*8t?1X`qAr2+N|PHF3tcum!u2G1EzgmC51Y8BI7m)izoT*c<_NCK
zn~h&88cyGJx#jfbqA!<CzlvY^TJuh$HulYq>~A}6rN{Zy?wtDj%B7Mx_E$_NlYJJy
z(%JfJXT;OGAFjvCVvhehvrF}kVCNaxN0HameXcneaIFd8UVrggzAal$=Wzv}Ye}^t
z8>Q!!&lULJy!8;%0Y3qCrGkyS-~7Bj``Nq%<vk^e^KWV;9<Oe+ShVL%&_wC5Z898t
z7s@0}?G<4yshO*LdT#8(?~{_*U6PrytHq>F%eLwjP6~{g#qfT1s{0I~BY!S$=yVCb
z7|O$B?c37&*E{Upv9xtr<!mu`)Q{=f{!(!06fe8vT)8d%2kYIcuna+;yCOZC`Ew3R
z$hdWtt+`WsJ)QU7-@<x@hvD7Rc4f*QQrvT%yPaWHz=aRF7v2PLFTSw)Pt6v&jrYFW
z3T5owrTp-EL#S<_f&Gl|GY*McMRl{p-wIl8l4xJKOl7x9_swN%pIyug){v0exBP_c
zl$NfkTe4*)>TTocKKAg)|BW6w+!|N3{4R1#T-kS3++}ZF48y5|PmU%v`fj*%WtEl9
z(nEOxy>ZVZRv$|#7dh0If2WQ~-7n$q)&H*7dGEG-KC`QO4QqifH{0E7ZyMyE<XRp6
zA=I`;{Y9^(&P4gj%7{tJmNv^qY!zaiWgh(X;M#(HHymOU1uK=UohF{)mh+1?liF#o
zoH6mwCJD8$(|1o7YrZ|TY0o|uWyj5h`s;cvdgfW|T~o4$$6&vX`g0>zt_MmHJ>K*B
zLTwJ;&Nn~1<e!j&b~@9<vjQ{3<3yH}Y)${ldd}qcCReV_H*`1euzQ`mv*6jC^EH27
zAL`n?Kd;_n!|TKM=1vg5Ry6Y($HX^k8{7Yfa&2jvasAC&q1v{n$ai}S7hVYaC4A-H
zn-g7b{H~$@PpolI+}ad*c!7JsLUo3e<j7e^FRpl=Rm^-z?A?s)&rMnr9UU(|{C7%Y
z#SFC*jgtgZ^wu0WGDA=Dpt}3KXPFxwD|Fo!&0P??Fzc##P~7C-v#0)^eW-KQ+Ffa<
zoPFH#R-cpH^_6w558t(KF`f_K&VAIot98y62SL`w0o<!U2y79sn3Zrydwxt3Q*N2B
zz~n%QZjY>%+9>u$4ff=+=7ZuqE8KNA*&cbMeA-F!*y{O@UmaTfW4`>4#)dqR`LXL4
z-F+`(r+?>kQ@fnBd!WPVg)+L!_I=*OX|>qpd`)ZkiH#NZ2mUYg5LiA>U|FG9<d><s
zmtNJY-*t4jd&SpJaLtdVWq0{=+V|zXQHhyey{G@KuYg7L8xv_(K>^ml2A0~MpK=%P
zANDw}vgiJ<x%Ym*J)UgfByoi=xq;>WzF*7Z-x~HGx+m%OjBkS6M%H*QwHJ98O=Zk%
zA7%WWdAMOq<mzY3Jc=GRT{bV%@|5cnxUBEJ<)eUlslIaEb~&yU0vEYbKFD7+_gU<v
z^(;Q_tD4Q%ZU2tE?0p=ediL{5*#&jN4vg<RRu}vD#qE7^`Q;M^MTZ9tO{~0fE;BAD
z2rvd7%c%NMFhxw$L4ae&ih##%eX`bRIx`%Cbs5)sO%gKjYM92tAjHGsDe(V^ceA1C
zC6R{U2i|k6xZ_wV1DhN)nYVJ>3EHrtVF}yf92u>Grb(JiY92F{o@QNL8_Mpt!YbAI
z#zv-{92T?1-bQVeJFgacy0s@}7t=|eJ>DPI9NL}AGE3*>#H$Ama<Gf%h0geJ%&cCP
zJBO!Y!IMS}&!=B+tP+);!lbl)^VPm3&koF1o1)nmRho5eb?kOO-2?fDgxLSydp2n{
zvp8SyI<J653Z6`ljxcX=;d7|`#NxofklA-YwdB<$!H6Sue}8>`Az>Kxg-_&9@kd8i
zoAsOr_As<@UKH<P2%J3a=hv+{mzr1_4*X>|UM0U`Kf`%pj$L#3Lf$R<%OY|?`5&Lq
zhP{9P3*AWR{<&m=QYS}S@jAzEY#}F{*{!qIFA(zfS+ufwTZERDW{8N!DnZ^^2eek#
z92V4JJXBP`%vfX7-MqEsNmG|nv9plpmZIctWk0s6O^Y?Gg>#OztWfbVY}ydxqoB(h
zw8mJES^A91WF_enVh22wI(RrwX&i3R^U^rj!sqqkfS=XZGoQ~{JzwL?Ak0#rc<y#e
z@@f`6s|O9uoN{v=7&%1(6dJs=Q_rdT8b2)w_Mfc$!BN{?b?RabGfCyHK)1x!(4dH!
z>kkRlf8t*jlzI1LSYwWm>6O`6n8S8WTN2z69<z7%)l)N;NG#USO}OK9bg@9xE6v6!
z9a@}{tN~67H2<%-C3r__`HUvlqV=~X9xd7XsOfGI`>kx<%>qG&_qJb9VqRDCHY;mg
zW5Sz`o@0EId^YKOyS{wT6qWh-^kKfBH5+!{kl4uErvJG|q}$op*w;@@;V{e9H7%Fd
z8J)P{YcA6mtYGn|i6ddxjM>qA9W$+$RPLHp9=6{3+mYAnZIi2Y!@>o)g$fns^q7XJ
zw<%0Y;VFxlx$D)zEfT4k@srP*DnFmVYMr@``Axy@dBQE3*BY;gnW}9pH?6YW@N`<{
zg4~Hp9&VhI1QLYztS${WCbjg`#sb;OD<_UuHvSP@tK0G9t&ZP}kfNfK*JstdX^Qal
zX*hW7jnP5_#cOv?>9dP9JTm(4t;DN((rNQk1@ng{3;i6kUo!<Vo|&<hM^B`5i@CdU
z@0TF&{`V$7{im~+9=tTa-A<8Zl7e$+SXN@YSXjo)S)yT!ZZs`@ar}NpMvsZ;wadFW
zAFT}AVf|_47nVs9hZm&xZu+gDxoF4l1$o?3(?d5TDX$Y)62pF{DdOH#haD!K`qF`c
zEHgJvV>@<KVMb@T)3yf5)-~%?x)z>EaZ*2Ypy8xry3!0!lQu)XtR<x)-9Fk$>1vla
z8bnN+E-x(DJo{cls+NwnA5)(&gA!L^+Nqg*MGGdosAaWVv7F+on6pFs&4lOH4l|=G
zHvO<`s(G6cwdQu|ju|G&69Y9DZ<u%W#>1umO~q6>d6EQLPGzj_-gCt2x#OaCmoH}8
zw#WOUKMUO6_U-7R!a%h~msyUFEF61%UTZQuj_Os(+;QmbiAi!F1s^L+SMiH7XlCyW
za&)S6YE(H>BE=}{v|`GsQ)fE76;(PsHNp%cX0`BeC0t@@Ti)Q|`JiHkRiAg0rpr^&
z$(83G&-&>lYX2!xlQTob^?SxfzW15=s<qXN{in?I={FEo%zu5dV@BD80;wf)j(<{f
z+%mQ0Z>A!b$jbSF3sfg87Ez62-57Rno#}){I<B$SQVRXM7Oi7`w<4Cg@cN!Ntom*L
zPA~l2v|^7Qhv(ie+;M-MWZgx|CiLV@DVeYBDZS&1s+Q&d9%GT>2}fP3riW$tw5iH;
zXnQEiJQOhIFxz|c?1bXr=^wUOdaaS%sIy64ze+HN<>Q1q6|8c4=a$U&YJRn1Qr4EQ
z4_Da=9_UwFQ6>;{A)VXf#<J#nW}&WGE&a!tIvZYGJ38w~?;N>I0U;rU>kl)$xr8bL
zTfgjT-TUgQg2ArWroszM83nF+HE+JCQtRRAUFE9yHET}Zv?Gi=0>pH88+JdLVRGtH
z;kxD(IjNfKQx0D+6D}1=R!y*Z?Z|i_PhqhpXJKEaPcq-8ncI)%zSK)nTyrtWP}g^=
zS5(lxJI}gy+>V(QeS6mG5~Y+mtd30#*KVE@4>gZUak$E6K5@NaiumsT$v2$J4@&zp
z_H5tN6T^A`%W?ljAB3In%}NMbr4-L}%R}p)lsA*~?E9KqRyXm@5fqh}C2~UV%X&W5
z5BpC9IQLe3NsQyVd)SICvB@^An3Z|5N4QplnAVcg-p<5~tc!dHwj6E~T5!%kc+R7j
z9ABNi-}p)VZg%}_`87Qw^>z8GsClhA*SEQ|3fwu;VKTji!-bb?(*>4Ei>Br<c`x2@
z!8xN%N9DXn2Ah*s;{=V+%~dWFEX@R4%ABtM%;i`wu}L8Q>YeNF*4;c?TWcK8>2z`7
zm6`c<&iN}<X5Nt2E!)_;H^sfhWr=RV!V7bss5G2>9(k#+%WK1v6I$*TnOxO=itAPW
zFHBGHnmK)wromc?b2$@Vu67kGcN4!kf91|&XP&n$*&{m1f0pUQtsD%ecd%=znCIsv
zf3I*^8R+|D?M?I8*ZW>XO7~1FJ2-oNM&tKg$DiqKnk^fbqCZ=?#7xKW*U5Wzv$Xed
z><x=J;ySfRbm@ZU*Yu-gv;&#Tm)vDH^KcU0l+fZ(Wz=DK=JOlh%=3XfOF4g<r-b|v
zpMGnR+?wvCEHOfNnbzvAW%bIge_&x3_Wy{(<=ai)H@$tljB)l%{f`%p_s`Pl-XI*h
zSU`V@Ws6zyy(5qH&liPuMqi8V<WOJgck9ooGxj{5j@;*)j8}cR^=!fCsV}nacHIv!
z75H?hT6y+`+jIYa-cz_XGdJo*sZLVujdU5-s=FM5m-Z-_m+dgf^qeW`@XYPVO%boC
zEmQwZ^0e8d8ag{*i_TS(wx@9uqV|L+s7U&Xv_+Q(dmpUcns8KNgU^p$w=b%`sYw@E
z)L`dpXjk>@;G?<?5ffBg9HZZFSy#5vy6WQef4fh){e9`Fe~<UkQibw^JZ~&*s@X&5
zYF>Dy?piXx`t>651jae1YIi2R+cc%x{pZ(f^|vnxGCp40cs@VFQ6=Ji?eR$^*Y<B;
zDx3CF=#h2n{MsImw@e>eW7phYa=|BqCE*dLx5K)8Qx)eKyCzBszjc&-KWox;wph&#
ztE$)DXqmss{>ST$dO|y+xz&vi{Z}@eyyGu7|ML!|PXT@F4_eN)-Xxtoe~y8`rdguv
zFZQv!tYR|}oO@Yd-r^2(>E=KQThTk~PT$a5D7=2r>(%=fIxe&BShm|ydd+;<Esl)`
z9QS03NlLDf+GCY#-BERF^PipiM^~xHIm)!2P+^oXIc~T`C34c11u~sdvo$XE>}+1C
zvr#Cl%KF{yIhQ_hOcAj6-E7cwV#WRrS^wYM2d!791g;OXbZA*H|Mf!|Tbb7H*?Om?
zCq9YNOVCtHxNSH+S}lHZC-1^_!Zm^ys}EkhttU3+h*!@M50kwggg9&_@8iC;?z(e_
zzs&wX5!YLfJ9?E3<|lK7hD^VAcy<N1c%imJk>dXqQD3@BKCgdKy`8<vwJceMFKK$S
zg+`|mv*}EOmf*wsArkLbZ{dpGGWk^xAG2$B=E~g{3?!{~gf$!v3)r#qk?AjCWBJ*F
z6K->~bubwvw1;J|zh5{jOrrJK4*`Que4D&FSk}0&;MzIyc1PD{zB!uG&MY3S7RwWN
z@2tFXe8LLtJ%_aKGF$CaYCav{C|=`X)Zx)J=S0_$6Wv=p_<}iuWv0q+ne=NX_XHLF
zJHiZ-JJo$JG-NMQ|MT+bOKy|3-a5ys4y~`&h?m)Y^3b8VKD*C0O-h-lcp*}Dp^h^L
zv!=nGEf=MncL~qEBH3|y)rRJe){G6#ni4h+HFBCaSi%%ok2wFg(f4FZI_Pysd(+yV
zhj%$Mxc#wl&)McZ#U&`<U|@rH*Bo#4?$f)Dc-u6eUV4S&X13zZ$K2OHdk0T(TX%9>
zh|8IK-Uh7?xC%M9JXW?-chPv7J?ZJ?4wlUdGkiAu+_dEA9EH*~`y@MOn4Ym){Kw?L
z3yEt=ieVBGXCo93=ycT2mN>sd!Yo1JmWZLy;vQWO!<gtjirid(SBrj`u!yx~)$0w)
zy#iv3JR8byc{Lgs203ipz_o4dOi}BoJ;hQd<fS)hEjD_}Ah_|*M#<n!|1-2=Hcz!N
z(%W}YV0YAswid^Qn@{Yxa^m!c^PPLn|2cBrE!e9!;Gmoi$0wl+GA0}o=1g36`oGlf
ziB^9mE?Df~#6DNy_^Qp5pBSrsF+S-t=ftH8zD_2aFCV(3*eY4Xqn~puSn9NZm0MHg
z;TcL@t7W`@GB)t;<X@N4x<SXgQ0jEpVO_^7?kWjKG#reRrI~djw!Lj~@pcK?ePrFv
zDc(D`oZeeAHK@yWe;}vdlk-h4PV{!zUf1T(UV1QS&A~%+CN5D}vyfp1OR}KJU$bL8
z=BpXRf30#YzP$clleksyir77#C$bjwPS}3@kGN!#eTq&;-z%Z&$zJI(5-xwYv}vB=
zb>=ve!E<}fF`-F{)f$`@DP8JK+D~mf*fZzz{8_f8aM80P6EqidZo6pX_g3orLaq3p
zrZby4+dltyh<dp7iH?*|55v5}6NGuJ7Ja_DaI*iD*)H8JCw?8d)Og}jisv;ZQ;sR0
zJ<^wYr0iALb6~TkZAYI-uzb(@1411fH(bv9bS?Jin%Z(w;-vh+pIlzQHmmD!aP9D{
zu9Vr#vHQf%L-YQqY}4-Wza-BVthz2s^+KU_(8og;b55CUF{qyH<-fUAx8S1qRT+tB
zx%SeC)U5NZJhqn(M<}1YQFAlma<@x=&NYwQ9SyfTQm5VsJjl=X*5&zU+x`hhe@)Wg
zI3f6l()0}re7HYx*t+W1_FkCR8tgr(!B=AAtLS4|niu(BO}f^z^n%IR44aPn-{xm;
zY`C_DwOZm9!_x_~euprv`M*QYTHvF~0ShMej_yTUvv&T`UanuP*C68XZRUPOW4#Tt
zq(8(+$Cw^;ciHvhjb8Va+doWp?Gs)9+37;^R-4JborhI9{N@}vFz4E_T3dZlj>D`M
z_EfB2T5;s|?`;Kt%{X#BRptbgYz;VeNQKw8yW(wttM*(G&Wi^QTsdH{bg`=MC&q|l
z38zHXZogs35zTQx+BWjV&nfrzithRtnB;xg-RIKd($i0#Mr>aq-v8pnxq!|<g~%m3
z9D0GWO(%k0KJ{pt5qCmWv`a}|V&&nfxi|X?)=Y2JFE-p)&S+9<++Ch2Zd1Ce?D0*@
z=!rJ0yv?1l=M^IA&TcvLqGR!zd#7#~Wd8YYE>P@y{$da7>RaBmdmlPX6ONq!P2qS@
zrs=CEJ9LtFY_*O4QX{`pw1M{-$1H(Fb`$<D3s!UN(4IfJ#eDYpUn<um&UyS=;*wDs
zmezYs?#sOsU&GhLTzm6&UE__!_O%myOC~N<STp~0jH+12)SO3+1xZ|2r4D}VwR^nl
zTPR2Q-r&nmc`HQ?R4sS&e?8jq$aS~n@y6R6jLYJ}v{qhHy0~ahP;&;4z^UE7k59L_
z-nwh+{Xll2<&)%drza@z&R44S7E8J0yzlhx8poi3xTC#tC(b@>>9}_|RaEZZ!zl^P
zZ<eMUd75%^?ZMw~IX>j78EIVZHTH3MEGW;(y+(lhPmvR+(tn{VTZPW(aGYGCKl_TA
zk-|n3Gu{4A@s|{PnXYkMS?Jc&8FE&D_byLQnF2H4#m*Ox?lN0m-Sw1X1Fv$(*9UQ(
z&z#R3*w(u0+e6M56SNEeY3}UOYn!dz&OD)K#p<>V5ldEk{GF2iXUX%*Y3cubmU=8Z
zf1-8o|2rpo-<-Ffb}e;nnBLtZ`y>N0pC~NNUMR1Vp_peC#1LOt>)HPyhD)}~cJ~GD
zqmNoT4k;`=q;0cBD@5qD&$ONc0+Tah>uL>H<)jT|CYt+Q`9GKE&R3xW60K3UlN+C9
ze!DqAzVwKaYHRq!)8U4%rakgn;%2{iS>PhMS78Uer)Xqdsk*mHPu2I?!#k(`bE`8)
zHlBDgt<HMqnO9*SU!}Nmu5#1NNj>N<_vrYpwTl0iAM!esveT-VdHR>sr%_*~?R}wN
z8EyCT>Vnx@;+G3QU63j)GWTWAAp>shTc4VFUVY`5lC-dK!r2=ozD5sZj91>;clb)0
zU;27mEoRLVj8o=q;Y{Q_u{@_Pw5da{O(St>=ejkr&jtQXxIS&&&xz||TsMDM<p29g
zPx}+E2iwj+P;0$^&i}F6yT4CPr#uvzIw32Q)zx!aUjL0tfhs=D>)x$9cJJ`gON^l$
z%zM`^sW?&bD9lak(KL|;F}1FLiH{D<dbEDV$=<Z<y&TrZPkP>6p%Z6f(t06$UhSmm
zH)bzMn0#IJ|6#+HmknEaj6(`e^gX`lBz)=3obW@jxl2x6n{+OB+6|Tj!C*C;GNm}B
ztLv8Ay;_>LQb_%yu=&R&X|F`{UmfbWbXX&AncmWd%TGMFJ@L)^a7i*rPjd20U)jm6
zw_co0NpbDmJ;R$XIU}$vyE)^vp0C))T(!omHH^2o7@eM4eqF!sn@XgdJdbWis|m-l
zTDhj)rN`o;q!-P|k-c*Ip7XVp?F+B+97t$37xJ6Ep}0Rd`iqMnzuf(w2Uj0FY$H4Q
zxPjID#*W;VlhzrZUM+IO|C~pU`~Aa59-L}>$g{5Hm{<QtZ~eUGU$T~eG52Htdb{c6
zykmI>d-7Umr1<;YX)bu87XSaNVCF8RDeHb`a2;ZjICs^Iop;Uo*7p}|W$bM^j%Lcp
z&pf0xM`y{6d;_tDXOE@abhvDih0l3P7(U<Lr*JCT@P@ui6leI-yf>K)FXXN6%obL;
zBo^|~-Q7OZy<9lGT%`QF(D^MFLVUe%TQB)GM?Ai#U2^40HC=i2`R&)@mc8+i($Cfx
zd|luCO1^!==6Bx{WA8M#$#n%4eQH%{IJo-b#_ZzHR*Z{WEqS|j&L918_2{L>HIkBD
zi!)lAlh^M3ykURCmbbxIsveu4Za8aJ={0@3|Je;^&n_!?C(9@JO)Br0ses#wiaYkV
z^&S?>>1FgCf7|f(_O{jYi#--EbGzG8e8J`aGmH05E6=`Z^W<5d-WthOz2kTHMt`1-
z;d3{BpZEIf_Zv%cUIdn|>-_zC+q*a0s+;$Py;yco%a`+mSW?<sj{W!ZZZ__UUGMC-
zW>@CJYe&nIzTYcV-J-7@cfD8q*u?qQH*{a;?-ftG&&|&(Gwlw`<P9$cCBI%$D!6RI
zA$L68tmiSix?-Nb;-n3&-1lDwZhY&oW>1^9_X+Rdw^rr3EOn+YtxXl{9z4&<J@ZB=
zyxdK>)PQkritep#+kZA_-rUm>7smBzlN`^s?Iun9Yt@dfn=--W;5MP~>oWTvet4nD
z^<V<msta7+uekmtDX?7nbV}s$qolIt9Y55vd#<r^Ffy?V>HN?5pl~RRDS|WSXut#K
z7A6s)l%s7T(u^r8LMfM5iLUI;jq1$_T_3hPN`&3Zk->#wQllZOlBdJvWe#(gN?uKw
zsgQPdfqkNoYg59r11g-8JSJQ!Ijy!L#c9&7jIDRChG}jry83Es_VxaVqkb|wf)-xw
z<h#15Xh&I6G-K|qOR?V5AKq@|Fy=b*qWnR8UlV)3P|Oa4gNK;}_@C{#_#nBL**s|8
zor;eRN2gonN!^&aIlVENtFO$)tBFCENkt`5)1fQ7F(`z4$=s;2T+I!EpY1Fh54`6L
zZV{fZbim@%V-9nUgmVjbHatJSlVicT)NY11(d&v|=I#CE@M<=@mxF>r=l{GL>m_eR
z-3(aY5XJjczURcne-3|`JbTSg+^>yH<2)emxOPjqfMyWGh2Z+xj0%EGf+~s&#R78{
z9+j9q!H`Qk^lGYtw$@hlByE*h83~;l^DGuIY&&vYDQfEj)yFO-T1Nt0ET3I}kicN(
zrjW*6Xd#!(^x^1ZM=7OkiEe*PH7{uV+1PYria_AZ_$h8dGgYNy_%tR=O37HEIKyL>
zra;;BrINF%m=<n(Hk0k3S*uX7mf{5#FNx{P3!^;@kBY=7sJ{?v^b1_E;C?6jJcsUY
z?nfQ64@qh+Wp{N92u_i>;g=jC`s1FX$j;@;F02nlv*Q1>s?2nlxYh8&YB|d-7u`*7
z{a2Q+W$~QIAn@#|iGZKs-Yt_W*It|=vYu&Y$c2``Gisutd7WPcy`n;+A4SCZcI$3U
zUZ@hVF74v<OAhyEExi!Iv_yKr@~lRyi>p685?vo-_3Hby+HbOzktsGtVM`Kca_)&t
z`5SfZ)xkAdIRz7DXqtJoEtLEcnaSjGC*o*flucJzrkVn42y>?6Y_T(Y-DZnOooF-4
zTEFmG+UeE2A~FSsZ|cY_IF#Ekr;usp8I6iFf#$N}=R?!KEV^$h6}Obtw<}X<Tc+Ys
zi6vXg7aaTbcgNyu7E&(nSEqA&2u|SLDWTvPt&?$K!(T;3#q3=_Esu#wmvIPf)$Hqd
zbhT#r3ZVy8zyH5o)csJd#LV%nKwQFuhUw84`zB=D>~#EXmL`8-KIb*b=S(?HUe7y4
zZ60XEhVe-xbjW7znz&N`nyTjF9EsgCq^tanUG9B4&)R9v_vLeQ_66xZ&nlPFZxE_s
z`kr3gZq@(db7Rx;7ay*!UnZBYFgcTZ`G&CdTNB<fb_lw0&DRU?3_1Ec(6&|mUYFQO
zxoVw&=Qr=?1V_0nUHB_E@@weU14U~Y7O@n>C@L&J!Md(V<b&%2rSNqHUl$efE~yr^
z>0rIDysiA|DTSn9k3A6|R|y_k9&)Yw@_v?Y8`@@eCQe$(b0|h)gQ%HDar4DFysApO
z#id%3Z~ea7%9S+R?Cvs!+)Mw@W~Yd4o*}4WVP3tcV2ZK?mqWXi)9phg+k)r5OE_*F
zy(F$#NWo^AfT4Nt#Pc^pQU#>1HaRFb-cERPb^fM{-3p1nMGo+=tcqEw%g8U4e8+)p
z)2`ZeTA@E@cM0v>?#h+tT9bY8AzRZ9Pp?grm=-=!)zv&HGWDqK7fr#HEW)P(<{1<%
z+jWL7vcsUhGS6`9<w;ZLOj)XF$?3hdXX&&hM^yD%HGNWlJ$+J>E|BPz!tFRmifLAn
z)2#*1P0NK&7{9!-+}^c6pzYP8h0BhcG+SiZ?0zgL964!|@0CD3wv7pF++QYa-!M5u
zx@5W~yOV;1*pX1)pC>}HLKl5o7|ihXLrn1hd7C;7PxRINmtL@4=+@fSqSXtZO;T`F
ziLmgS>ryCMyTE>bXkc{Cmc_3gX?QuzI>^D-7W_hs+sgcgqlH`x?*r!eIVmS5ihTQ^
zU=q4C!EO7gR#l(n9Em>Z|4KJ@IQA%5vOIGznikT;XsmUUab`{q(^M1Bvu0Y=XG3qB
z7~R~{V-siUQFth8bK{qu*|*p5iEZ!LvQ;n3_t8tg%X~?~Y@7;Ae!F|~ma$Fk?^%(<
z<it^T#X?Sj^_l|D!f>~mBLe2L-o~a`rabti7P)`oiu|dvb7mSudX*NYOiEW=`Os|k
z)?Wp@rJqy+rUcFOx-f;aeJxL{cJK?;{p-$cS^N65|4-xp);DYWwkKsxUBc&dW=p48
zZot)E-vw1pH#`>lc}-~eIK{lto>M5Gbywb(Y16hmDoWYW#JM7P%Ldl{LF=;rZuI#1
z!gIoj6JE;gOTvE3oH(#n#6b1(Ew8W_yG2Vq1s3(ce$S%FnJ|ItiOSpG%N_r%e9)*P
zaq!=nC#teF=eWNGE4=hMal>P`<a8|+Wdqj^pTLY@(MMB?^w#wFGqWviWAsVZJ+?B4
z+ih9X4+G!blV<jQJS8|q)zEf#;3Ag{wQG+&J{n!RxU78J;iLLl#WNm%3}t!2wcL9z
zn{Cq_r$jH7#jSfb>OQ$z;P^)?qO<1bw))!sDK4c7^0AXPb&1SLd%VWe^1t(p-5yag
z*EX2RNzF9ylV2929e2n1)Q4+6t@jqbUGPQa#?ws~dzPgxRy_Mc{pz{x4pV2$RV}*m
zh9lfIIC?@+`|&$+(yMneO?*~3<GK24wdYq_R{2FXUOOAKt<>1S*h7Tr)tbK(AD*wB
zH@#Bz(6eqHkG<ayK3B|bwK!)~5PWS?KFe2+pozQsZ++aTA!R))_m{$q#q1Lc91rZB
znR>dsxBqZL|Do@%*;sEI)$@Jy5A}K6c-DHk?cVQx=^6%IfjKKZjXHu{op(0cl`@~%
z+R5S-CVeZTNmavnW|iC9DJr{Ou4WZ95;-lE-8AKD#skG$QPQm@ZiNg%e`mh-V=>S%
z<Wl(WwEZAo@yZ%GO#@cjlrO>|9v|lm&;8_<_)o}j-x@b2*>B!#%mOzZQWbv2%vjo_
zSYAJCUS^5^4#SFR9vREeDO-NqsoW`fBRTB$sm`_Ai|16INk6nt@K$POS>HFIJpcBn
zZI}I&lMl^J*mL`CeKt?$`pnMQulM9Tt;H)Z%xw}`mMzn)EcfyFP06x_>9Q|1eP0)H
zJ$ERIE#C3eYR`pPT)$SGi(I*Qm6CX1$Hx1U1$`I(+xyYdV6%9;zy+%fxAi}nsVJWb
zyz_pa>&LIlw<qfVVShfa(ck`Gn#cSd+)TUNAFS5xcy0N0cAf3AA7=82GGP;Zv(Id0
zVc)TT|M|?DXO3xl{og6~{?v|5U%&r2?#*LVkfnA_$@@wAnL@vk3t4kjt2chyEZ5^}
z<KgJius$JfDrdtpN8W6mgtM$d=VB!lulgxo6w<U|nGmzU=H%t1)CYBQZh9R(>Qc~u
zIZ}=@&h4`1{0(!bbMd*&jBQ&#_tMWhVv9QOR(zSHtvO5o>ruUzlXN^!PWQRRC*`T1
z%W-S!gIH6KsugW44N~ViXJucI$(u84p3PO0$m?@FCN41XI{)K-<V??kxlu8(Y{3OC
z*)EN-bJ_1`v`Xo*O0Rp^vSJ}ihFrx<x8MtjRU9JcJP$EH>ks~MV`?UUS%ZsRWUKCt
zW)7v6X4NDQ%|-7GXDGW(a(ocC<A3dKWg+>Gt=vslo0>9}c63N@p36DA@lpKVU>6M*
zwkF97hAln^rT2!$@7*Y|mow_i+bRA_BYfA+&=#AT@n!ar6dwJkGmb2Gc-LP4dg9nm
z&B-}4&Ioi0hr2F3tr8d>Did+^{sh(I$7GL`>Gob+a`j@y^~k^S?>YMYIL<rFS?Cbs
z#U#bOtU@%~WoC<uWVVU})1JpRTCFy>l%?-2wmFu}WRM{JDcSB{i#+EO+f8ma?gnz4
zI_?!R!)3vPPlZP~gJK;mo_sJ!ag2#|sqK0)<wC7Q;>(#gwX@vz70yq8aixcK>MdWl
ztetn0z9>xDk>b5?$(t>)w|YG<*G!)L^vO3~|B(MH9{X`52#4_(U+pxW%U<%3vvln&
z5reog-cw>hau+I=99!`C@1EpSA6uliEeuOj4bNK|UZxtcNzroW^_VLiuD(f)jPsk`
z#x>hUM6X+_UZtcGw<KiNY4t_D;d5@f%sR>K)$;VgwRpKBFE%`yc}L`H=9$iglfKES
zP1+G=YxyjI_ex-ITkq9{4l>F@E9bk|$mHEwc!sA<I5b)0E4!%d%?mX#iw{f{uYCMp
zw1t6bftA|{38e&!qdk_tj@Vpd@w?%|a^*<Ni?n<_*Mfh`3Vtnf&}z-&QepbgC84>z
zSm|H|>#db75A0hPh0MC0nliUcj^%RTxlOh%*K*sgOI(xL7~TATMPKgSzNL<A2X#1k
z)7re55_OM0&xxGHl60$d(MjL1E|C*+f*2fFVrB}LrDb2~+<1X=)(R=p#ObD~ifbw#
zUvkN)<8qT$W0i_i^N&eA73kOYud7{&CFlj$i-Si!HY|-!OIHd@+WvR4q7X~B*P^$!
zO6pG0YSX;C=D9@WB{fARshvvA@QkgnOL2bY>V9mZ=F|)GS<@0WUGh1qH0esxp`IIE
zO9fJ-rXF~xK8;EFhiO}It*7x*mb9ja4n9qXO&=T2dwJyI!}N8T!hy-hG>;efTr}rm
z5RGj=-aF^y!KrnNrWVAu`LZ;&GI`r>YID5t%;wtFh4<7paA^6Iyzt?8;`jf`ks_h)
zQ@lsK_bhYqIH<OPY37=vm+q#1YJJtxrQ}lNGT|Eg{N7a>bJMkxr1*9UH15fox#HTb
zlTY%Fs(7#Ro3`Z%-^D)PEDl4?xc512556uG(o46$=eZ*AT)-t65j$}c&A8$QpDzUt
zWl!Z>PR&}C#}*zoCw8ICsoty(slclyE&nqXspPHRmGpdP;qu6UC9c0Dou&xqMJ~90
zR_WW+Nr`!{xlS&c8h0%4EBEsdpCcb`YjfUlW4V&GV&Uu+?PVowWe#s{h)&FMxpQAJ
zFkd2Jc30Z!-uBFA^XDo>{`c+Aa#QX9a%8dIDw|tV7d<<Ycdy0wic9QYFP{%C{(D+%
zq*`qL?>kat$FYp-!1-D13?2@5cTArZn!a2@etFLF37fLr^d?;1rF})}{*_5nO1X}$
zY+7C^^|Ip2Q@2Ua57>C_yBRyJZ*AtEYgudWd2jQp`IjJIbW7VaK~QPxwWof3u5ZL#
ze9ImrJ03YZ-S1og-x1@ZJge5;TBx()Ge=h;N7qSChsSA)w|zPx-X$jeu4SWyLTo61
zP|xc>CE)><E}dNRS8g_|ib;5;hU^YXikg+zwM(P9@b>F<di|?1R43)_o|k16xFjxb
zt((<cj-}}ra+lPuIW=kCg#G&-J-LzOk>nO2*kWL{et)KBCX4ccs$&&9&*UVmaIiU1
zZ=^1;;o<kTrP6F7!T(P@TzV$0QORF8Rc0cmTV!sySZv^lnK@beLT9~Lxk*RMa$C#-
zJ*^EsT8{g2P961W&&zmT)bhdR#L_S=$3JTvdDa$%w%(kR6}wK`>BYJ<&z9?DZ%h9@
zJ<PFY^|U9;4qo%zbt!SpU7t&LZ#~PpdvR|5+_}$u-X(BoKS)zMyS3*$-;<^{8~DB(
z@c-Yy&u%E7@6l*>Z_6{D1wIGQ$+R~YDP~_<Z~ip>S>9@^qCA00kL@ChSiif+tXg+W
zXZAaz`N^hlKQx^8x^j)}d3K@fdiSm+3pkYza=E2c`N>>1yld8*s+90FOp9Shi+9C(
z`R|46IgFSDisYFYl-M_Ut9(*qW)S$VP{h!pKkvl-*>TG5bJWzD-^>a)7(G?z$+l&V
z9OwV>9m{_7Cg$lHMqw7M&qpNL7BK3opJ&qjzfVo~w2|W$UICj{w}uDHW}JL!*7i{2
zh(7!0G>*^i-}Kse`LFbD)c>5vy-XoN^_cF-2l+SWellabmAW~9Wy;!9>bI?Cx!q9b
zES`S1ZDCy=<6Rrkkf(2%lseBdtra-_S%5!<K}Ielbm2mSLbJWIgx%lGJn^AD?UqRJ
zxo2i>*((<N|LS~R<MM(nKDi<K^`dugPU-dene-phbG`hougJwETEa=^^OdGo@3`D9
z|GZPN<M=)omzx_7IalZ=Dy^TQr>{3nLOQ=KC)G{%zxVqaw(DiW^=B;M)cml}!Q(_w
z&Uz;P^UB<t7}UNhX&We+zbSDw{T-Kl{OrmL3E3Ar=CoLG{kwL{anG{KBRN{)M<jmg
zJMp+`^{=ix(xPEp;P_yZ?s=pCZAQyDg08NRG`t--;gsstZ>75`C(La$agWjMw=rHR
zmszzgWy+~lM^4<xzI*QGS*?XD?|Lm0TmMch&G}?4Q}(^DEB*K$zjxvNb|q%9)rJtY
zhDWM$rjBu@xr&oYABr9SR-s_xe&wOs5?7Cjdhz}4;`t^~|F*sUwJnEvo5bgL9~vf1
z3{mLcV>0oCbaGeN{;XS5H(itQewJJ>f6!GeAXz_u#Y4$&A02*hRs6rc={#40?)!YD
z-D!OBstg(pS>C0lRXQiKKAF9n|F|ehywo)3^e2tuTdn!G#wZ>=#hM#q|4Ke|mA3c>
z3%#tf+hQ8Dwy?A(xC9xnnsdyK-Lc}G>Lh`(w6!Ab<{gRU3D;};bJyOh@{-@5!Y8=s
ziNLZA(^_J61sA57zqeT3neP2gFzk$&%fTx<I`bE`@#SaR{_mU@GW|1ym)Nt<D@!NZ
zcTYRcQ@-ryl641;<xXySe^8^Ky3i$mr`z;ARkrP}<tF8+7uwxrdHC3FYze7a_hq7x
zM?&0Y-|ibe_T|O?nVByZeL3`T=9wG&(@vCqPdr_*-J#~a^6Zli@)qs}CxT2^YySI}
zM*i)IHthYARcb!#jd+}UR^ECBmID#1jaq(uclfi$cZzAVNret~g(HuNqlraO%htbF
zk`MpleC@lzBcJWEpI__e6V`J*_iS7JMl{}W%1+kFmvrB?XXW~>zLK&@|F`EqF|Vw$
z)ZGSY^9we+f2-@uy2rNEwr#t7<ObX2WskR-iHFHenQr~`hFP9=U!MII@n*~E=_Y&H
zE2k?zntuIvf=+~|;%71AmUh+h_5y|KLzRhKJ@FR>bfn%V9s8J}vFFR8SlQbF2h)s$
z)^^G+J1NC<TA9DQQfJ3%@l6-ZKCEc}b!&#%vQHbH8dW{={hFcYF1qzk$(iycB}Ni2
z%U2Y$F#iAK!PIWv@ZG&Wd)M-QcE2g?4fd=#5|dh-XJ6@Cu(N;5rR<8ems35wThcjy
zU;5oX^KJK}JA4`MRh%~b=&G+>$oWt!w|ME)wNH<3xcoviWU71dEtl`J%*t0<nRrf~
z`1p=#Z17~qjWa90{`gvZb@GBkMkO5*i{8F_nzwm(!R_7CPwUMv-}lJcZtwHmt+f_c
zqo0?sRnJn^jIW8%SowCgd*)=Z>TjaI_h~Jgd-+Gw6BB=->CcyN{!d}QSYBQ{$@I?u
zn&dCvEzH=K?>1k&dXJF#S@m#1bH4v4THV#T8Ps2_oL6ME{I|_!nIk*wID!_)EO_=l
zYW14GH~t+v&GJ8Q|MKVizW;w+6C1luftMp{LB~4vziaJtFT3ykeR}$a_f^hbDhAHB
z#{0YLpIRkN54!k~+krzsBjNIaCRTQ-kQo6CGn-jBt$KbGh|KH~WsOs54P<a{<WT2Q
z*_^82*`^-2EM#ZU12Ikg*hy0uR2Um(F$r-j$ef(&Ki}S8ZQ`Pot|^P8m*mV`6rwp@
z({CZy1EtajAq#AMxlU9~%?@85zu#-ZqNRze&6wWa+@!uh?)p~s+gv@T%O1q+F8z4w
ztao^R{J#44zvjN)Dt0~UZ4|E$&osl;jeU`u6idZaH5+rJjN&Y_6rT6YwQhQ5Bef;1
zZHZ;`9}BgI8P}NNSF0R0FncqJNte4q^M6>g2&ctl6^=Tg4h9Ae7AAu{o97NskFzSo
z%gwd={Pb)gU(UJUgwJz1r*zC!NIf*=)AO_bf(z;%wB2U1<mj-U`cI5c?REH7t*P99
z|NVJ6nQM2;6k(5Dua<QyEM?~Iao9PB>x<^07Cx3&%cf2e@MxMdsUvf8m`?W5X=N-L
zIfh4fC|%sr!uENU>;4Frm#$MnVpeWgo%8eM5z)v~x~eBw7&f_|Hc&R&d{bxES@j7a
z(Hg2M=AK5Ux|U5@u~2Jc%ZtU|C2n~cPdZ@J*}WuADY)xFj+gL3&dQwN1q><*U1v0>
zdA(Y>^qq?4DhAd~U0UlmeKOY8KX~fGS!YS%*XBlNigf<3-Oko@<*a1ztcyVlG+a#<
zN(Zl+xKP$Kw$n(lUg_(mMg!qY1=Z?LI#({({XTg$(pfuW%8e{(jq9p`w^X!^iiESb
z8?kbnyYY-imV?7A{;wA2$1}?MwqIW4aH`rbFY;ggW82%w(e_IZonQ6;$K`hpZ}@!7
zBAEiC$`^129y%A>IN=1J<&!D9%N0u5SA{P~S=xQz!kieT6{06{JlE(3)Ia*7<)`R2
z(XefH>Y5EQtcP}}ciHbfWf)w^VtMtvMbX!V&1<$8DTYPf%|ET}yeahLr0BGuMH+%q
zou|8^m5i57pXa+^*}|JV30Ct=gOm2mf0@E{e%ZW5T+x5`-N_GFvN130zs^#LT-A+b
zS1#q~c}8;aY}k7y&pUU|H8GyIn}jb#D74%9c3ut>5Z1_-VqayQ-6*D<n!;G)vNBaJ
z)3(!OlC!sr&Xp|H44xLX_7^E_i(<Z>ILkSo_l(g|t*q_I{k|)AJQt`~XU)j;U}fp!
zp!s<YZzJ?Cym*r1e!*c51IL5NPfJ|=E57+%3S;<wXLgp|&lR)sy9>W_Z?G@$KlSp!
z>-(Y_Om8+9^Ql&phif)A%ygab`|a*z<K?$SB|I8=tfw$VtY;Mve6!W#&#t;DGIA^G
zFIzkl-h5Oe#Prkc`*M>71PYfHFlR3CYG5&H+;GUa`LvHq)ukoVGdr7ox;UmzDpAof
zYYkA8+j5OFcG{uUbC0yFDdhC&Nm`b^G&92~`mX1x_5<_w@VEIId(Hc#GIhn*&aC<*
zp9Nop1sD{6vapIUFfr&bFfcGQFgY+XaQtUr<q)}%wP1E*E}K@#(hUn&w=kdN&<J*0
z<l34lm38Mt;S`2`DMvmL%?Vc|f|%Z14Z5-Js+!@OjNFu?ehRA_%~{u+KDlxE`T0`E
z8D(D;-*s5bH1U<qsVOTi&wIqyHAzch)uEM-LV86uyn1{s^s!BotZLrXUF*`qr`paA
zS<O%yTY5~?D)d3fF7NkTD|Ux7+?ycnX%@Tc;);h$+sijeFV;-AW_+7<b8~3X>&x?U
zShhUstV~YpotfVz_F?Ad^h>;!&!3!>R8Q}eiJTc$F*!7oajXCQHjND-l0h*|@y|{;
zDo=mJYndvuZbJE?Af~%I|4+Im9D5n(Irp2^1QpGL4(6Qy8*~%6XEt#~-%K+13J#w6
z;rhi}JG1Oohpu_?eE)uX&G5}kM^z^TEMD_yqDPafs!-US%A?`sp_Qcx&PhLyUJZ)n
zY@Ht%TiEfi-HX$8N~rj)tK!A}3tN@C`14GzcGn)%NbXsEpyH~7il4^=haZ_YM70-p
zFkbT7DiY$ay?8^DP>aAMrrRw(+r(~ng@k23ohmgo@tznHr}#yO%M-5$ls?XNc{cr8
z<~89e=bew(N;0`r(&j`gVtMNDsL_1F<Rx2z_)}+Xy)=oz;AV?(UdOezr;B>NEfI0=
zH|t6YnKJK$U{q?MYG1_6RZm|=&UIVr*Rt3xRitypBa4U-r>$8J8<;!)SEMF&n5K$p
z&*}-B%;3V6z&+dB>(%A5m1j*vHf|Bse(JyY?McDeysXk|{9jBw*g5?W-%}YUY2We*
zZUS2y?>eMUV!u5{%6IjIT~eLma=X@ES+u0u<81qk+D{kL=QDgO`ym`(dQ~FfKyYP9
zK@3OKwFL*cwbulMi*~W9g#QdYlX$d4Dofa7O^1bHo(!|hf=-q6)r%UK)>$N<RGO_(
z)TqyWCbYZZq58%nQkT*eRKHxB(p|CS$&z&E<+W=Hk7Z?T(OK+meYIqp!^DlO;xSE`
zTm4gf3<ItPU5?DVDru-P;Re%$yT$<*!b~RKN||zDHbcyU?nMdC`7M{!CapCINMXwO
zfAIZ6cYEHiZkxp9UO)6PTQOmcm#JOEl4kvP8(Pa7w`i`Jy!g%3sHDz{PZn8%YP$k!
zmUdM(*SyTE2=KFfcmK8W{Uxm0RoU12bic{uF3n(8?4ISefpzz-_w7p7dwjh&rLB3_
zVEr{xdzau7>krrGy}Ug0ntf}G^W$GfqKlavzV9tbm12MAvUmAuw%w{%&kA0MKCG>;
zIr9$7{rlf?Pt33Tm3hOkb;aZf;UXf7na*XTvgs&HReLgVPUVTx^+I#qBuX5-6%C4J
zNk>l2NIS@PPNz}Vdg1|#j@gAD*zKivO=!Q+w0icJQ)1ht{V%XEZF%sMMYXGI4qthP
z(n`-3pPRd-?)(=C+WLv1KZfaHkYM+Ty-Q4&C|tPZp{a1mw$<Z_RhU!fT$TrV3`I}!
z>X@cD{CH-5(3GV}NaWfpr5CH%+%nvq5~nWCa@FSeBoJ0{QzXpoqmtdbr+!?D-bYw8
zPV~o|3s#+^{QAj@Rn}fw_Q?XSj$gh>R4-_+)fSrb$>FffJdw4Pr?YNl*e#aTeUxG{
z=i;M}GnUW1AK`7`ShB=n-}SXoK`X6dBKXwaZd-T1^yBmjC*gS~oFz}4=oX#%V4_2c
z)5h7RpJ)D1^kQ}P@LlC&P{esT!P#ktLUvWb?1)ndon9(klQoN*^@I`+I4K#-2s}FX
z-Jb_bvnCrwEx0hx$g<O^i{<~}eg#g|7ptT_O*l_AZpfVMF?p%eZWYrUPM2(^7fhn-
z-1R!ja+T6oNr@{|*)`5+jAoHMSK>8cuhE19Ra;^N?;c}VsK>?Dv(L+4*JEL;is-Y#
zcIz`Qtu{@SovEQS^>%W=UnQ<JSDqHtZJhI4%zQe-+@s3rf?a_wN~^Q}ZP9mF+a*77
z)$vs#S6Az6&9FG~VdDCCt!r+(tUh$&l5F9Yj$$T7!SY8IGaQuGsfUElSo3L?#YCGf
zHIs#`wk*-yRU+HOzdGdTx)sjy^jlW5`wwU3l#)|lPO4SSdE@y>sXFqs*p#W9xpIN~
zrca*Y_athLgpzyG&Re}pu4#s{+)q97bV|&BE(OIR;Rdnm>dU{leyiQpE1zDc%22jJ
ziF>uj%_l2doU~tNuG9#6=fvzB%cl2zmT|YY<@W4zt~Wgz6?pO+O*Sq3d8S>iZK=Ec
zmDTsEz8v2Bp?1Cf9qopnem(iFUY*`ew`ET~I_8&iLM>yt_XF3dNm;?KFBPmg?xMms
z(Z@@s<C;a(HuZaD%J)|lI6g?ZHE+W-%VjMWiYs?^8VIFKT65u0^MOaxCySma(Erxe
zAz;E~cPm1~eBH-h{vV-|UuOh7oVzS?<?ZYH404?A&lNn}6}3VBaK{?EZ-$WzV;r>n
zB~M)PQNR1>maNS76QR910n*pA7B)UoQ)~A9oGASIUfqtL|IZfmeRg<d_}pASEu3lP
z(YJER;m>^5m#4{`Q+J(x_<~k$<;&YV`FfTwk{nsAmR6|hg+@PE9<uT9i_>ps7+q?+
zpH<fV(6di;dWet4HYKjANqWWaN)&vz?7GT&*=3EZ;%lD^zpkyDwsl>mEm!pOs%!iI
zb*yPL68-*QM!Je*S#`zQH)+h(HxC{Ax@NKL+XVgJ*KLFkY}jfmB6GX7-&CllONiZX
zTIJ@17{m27HyZDK-L-q~=dbSe&wj`CeQ{R!B6?!)*S!;KuP+vxox#ha>D;#9ZmvZu
z-xfE%T@y9<c3iT(Gs#)>P-~}_#UjzEACB&1cj1x_%;t5we=|q(w(`2%+*$vZ1$FG)
zF;#EhdWB;$$+|ZdvifR-ua!xBnd+k0miWC!%U--g?SfUw!tXJ+-Q{<wWz8$uWOUco
z)!pjK)v7PO4`N<0pWJqBzFz6d;@FoS$9G+9jhnt=_S_d4^3^wX=1$*LAdpb1tS$eP
zPkH{U6#ERdHHo+HbX)A+`M!St_Sx)BH!O{__9ZKL>i$bPdCEZjDQ9Z$d#BA_lOL;8
zPUp{jU>B|9W;=^#_5-IZ`!f@-bEQ<@&VKvSQm9Dwm{D~@j>N?h?z?L@+C(hY_~CZr
zVwcW}85(QC?wBq7rxfd4`QoHXh>>w)*fah8Ka(o<ePMl2^UD4Hugm3L#YgP~Htk~n
zb$R{1|3B8RuX&RkFLG7?|BuP*1wIAe*9%CmzQ+9Wm0cW1;>&Z>6J8&@SHA!I|E~`D
zpWV_+%mjX46JWm~{AGdQzYoIfGa49w2v)So?YqEr|D$2b1YgE&dOfMZ?QDe#79w^g
zrUw~aRy|MLHr++~Vio^Y<ANqT>9Rz{sRC)n-eE35OpZY*4r-?vEBzC_<Ti3zeHWQM
zK{CljMRS_=?dx@Sw+q%O3)BS{2kt1T4Q>t0Xbt6P4ck!@exo&d1?x<K)<g|~<Q1)-
z#Ov=I7ksn5=Kb-uY>u{q7j0L>+e;+c$|Bm6JlaDvT5C6#*c7+gK1?;8CT6`*qRBNP
z=UI8zk2LAPhUxz&HZU=Cx)e6_bu|3l5HTmvbE~uJlaJ{x2Zi~rSN{L%VZ2cGf2h=+
zjZ!(2Bn1|_u3wU{<hZlU2G*ko88=$UNQ#-JOcJ>NSoZiN3q5fkro^g88##BW2Y(b|
ztutmjv%DtjYfIz`!8hgt7dYB$h1(`FwBLMDJyC(-&Wqlt3%&PO^gNR2yS0M#{*1n7
z9(_y)`rcUx1YYQqp4clR*w2*M|5l>!r$qm|A1xne^gda^`eX(xv*d&u8GSb^Ca`m+
zB+Mu^4^uvw8u!Ma_|NmakD?8^Y7L)F1-M^yyxZE)Zy_Kp*{QHI?pN3(1x*HxlljsU
zCvil?e>LD*vZ3=;p$g-2y@UU6q)RskaFwerd}6}=)VUzpHugawQ(__0fk3yD2~y#9
zt3qAeb{6oa2QWLDt(_3GW@=RrlbqG|R9?oa6anYoUnG8XC|%wmnEj*f?hK!gC#GF-
zpOC+ET7e{I<b~dno6|})8OkN6mvT<exj8*BVEQ)+&Sp)4rj-JzD+L2T^mnhE-mN*K
z#B)ZY=Zt$EGbT#1PQ2OIBsqi0ar(U%y)!4Uav7G`g!`o@b_nk*f8QFBe=V*1hVayk
ziis~oCUKZ<y~+Jotn=>$ZqHBoM&H$I%3=>I>T*1v!qphLdZX8-jg@ABl~dKD{0|CG
zG&5anuJ&ZnRF}rNQ<e#=WS@HI5u06xY~lan#66!)yeBm>O_aP}DEY`yZcafDlVgnT
zLe`)Qy+3~Rf9ja=VTa)No%8?fnE#z)!OxX5ziZ55v05-EbH=?H3+87oxW~czcjb(?
z91BIHSOsRy7tvw}Sg=qmYhfe<tNgEpa#0IKq!uaMTF@q0yXRP}XOR4xX|otK;w2>W
zB_i`TW;SfrtdPFgAQd=U(W2ABDt^lYJ=aC%^-9k66IJ9mRBeAQo@&;#<4RRjv#saK
z!mEZ$`O{pqZZzqf5DZ(X#(Ge|bcUe*4KB%LW=sxDnYRS8eo2=UsV)eMwN30fcrx91
zJL|_6a(7n<l&uu3bDqE_wW9ZCUw7n+wyYH`RVzBHR`mWqx#Ew-ir%gT(>E?;a$G63
zYN6nUg|k*I47jjz_O6v5GgdB`$(p#LB&i{pHK`+OTF1Jb9X%EKa+(Y*8mqUg5;T{Z
z<kzLd+EHO?=(&Beb@-ypBbSz(OU^iTN%DB7^x?%CIUD92YjXBf&Wzj@u)@82iv_Dp
z!_wX+y`8Di8=qFZiqz5A*tk_(P~)KLF^?>kz$LRL%y41sZPZ%uOLBpX)xtj=EZ?iv
zfA3)VeQW*O6YGDRS}$I;a;g+7gEj-Zw!oiN8>BaGu-~}w)QX-+f!YUQe!5=V!YBDe
zIrBfbF8<Osk;!n9ZP#M1gNwhaPO^QtSpBuITD7EJ<s==;*>(#zDMxSq@&6jXr_h|r
zXWUB~oOe#Ie$p0x@{#a@6P3#C0UAPGN33K5yfUS)PFc?$=)XE@`;4ZjE>%`Ty|o);
zY&CqWIm{ncO2$ZTQ4-WCx#iEiFx~mZa^{KCze_B17G#uG-2T>MyWPVL%oA5KC$5~Y
zB_I&6l4JHt0fi0C+>8MqwomC|srOzl5Vc~7*Q(`Jt1=tqkC{&Vf2_R5Jb$-j!|s*&
z0vZCX;+@-P%{E@WdZ(1-_qGVPWb0o`Cr@|OIs3?EM`PrE4Rv!vji)Co&sA;ler>z=
zV^><yT;|3qzO=bFce?NzvPv^FZ96J0?O_x4WS8yfd4E{9>9K2kytK}*qqX72g2v4L
zg8!cDZ53B>SPSrVZ+|v>A5XWcg4D)@8^t|=$$uQJrEbR=z1nDOwOIRggG#j|?~P5r
zxesuCY&he%**fKb!@^D9pYgwOwBBRnc4Vr`j&zONij&!y*6#hS6D+B_yxUahd4R`C
zQ;(B^9-ULJ8V1G)O%*&Q!+Nk=>9}srr?pH0GFef35_e~L-V%NvIeFGbp|uydbf(VW
zdnNq$=lpkC>npigOSD&sRjr)0YCDqvW7p~(OaVJW=WOrxT2<gNt8;ox!_-;b+bh<r
z?D(XXzDaZSeCy4bI-7IW2yC<xslDD&l(X62aF=XO=Zq6VlN8<N2d=wlt?I%(S4L6L
zD=5QRyX%xywf+A@=~KVvybKgrklqx!+S$vb`{u4aZYC#P6c0-m>YZy+nm=)}?Lk%c
z^qwY0SBCN<3|bQcIQIR?Sg@*QWx;BJn%5!<xArkb7Jtm%_`PvsYiPvx+53NZFIM&5
zl^dhV^k%>HEe(!68fSHM^fKd@>#X)z6m8B})%RucEOl?mJ13txZjBY18o0Ai{l?NW
zfo#jZX4n-9h%^cw3fU8_DG+;es*d=U$BxUl-a7nN;K-x}3;x}iDY$F-quT--eNMMU
zZ|~Ne*&n^?gEd!YdwKhg@^0~p)IVvp;TLoN)T>l4-um;5ZR2bg#!KOkxWAblm#aN4
zJNE$Z3hm!4a{`2Q?9Xt`K4>%f|MN(%%d7n!xhy=w)nc<{xAqC{@0Fa}=K8Ikdq;AQ
zpJ366jjYlJi8FGl!knU=jtI_in|JJG#hw<a*Pbm`vwJEw3wEorMyzQ4aBCa)uI+7+
zr{{C6c=x*c%yjK_VG82W`w!GiJgjp@(Brtps>N1Pn>7|4IDSU(<e7%Q*#~Cd*uRwf
zhMD()y4XyXJqP|T<II^9?|y6Za>o-qTVxBXj{E;Qn__g1N#IcEttQ<Ug}1CkgtwiP
zet0M#C2?)8Y~EwDEG@yTsM}U52}*6IZkMztC<+#7&pPl{<lx=uOMjJgUOzfTI^w13
zv9+&OHJexHue|uvX7x6!1GQ%-Y5o<N5^*E_&zX<^uQl}CEjM0n{x5;sIFo<hfwUc-
zm-by~_+~IkomV8kMz!qDjc;d<Nk;N_hFdo^SznD&EmD?p{Cv6lQRlqIjP^NewktL-
zy}&h5tV-m0l|$nm&6<<JR)rcx0y}RApEjI#^|y$Oa7?XE5AQ*jYf22uwzC?j2QN<O
zpL4TSX!hx-3&pnVd0tx5kLSjH4Om>EyV%I;Kx=FR+nl>BV)r$SiW_-vdc?&~J0jU!
zdti?30mg`A{+xjQPABHEop9h4vEjV3u;RqsGh6QSY?EXcx%s$IHD{~C!YRI$TQ@~d
zwNG5eG$AqXa&%mFGt0M_=ZunUOVigVupZOpijC=LWIZO-yL0XTpXD9*b_VKQl6m;z
zn9L^A>e-=9vw3{D4&2J$H{s>vix1u<aQ8en)-`;!)<AM*-V;^Xr#v_K*90W>-4N=3
z>e{3hbvfw)f4knCP@4{+-5#rTXBo~_vk-CnD{|tt@Qs;=LQkpA`XfB&&(z&Nrru3_
zt<X46De>IFlx;Cj3bq{P`t8fVH(|eQ_w~avFB@auO*nDAdhR9-?i(-8-O%^E(H$%Q
z%1^|9o{+Js+jZSn%z<1>9k|vROk`RpRC|f<n?t(pMBxW>7q5s`b+MEAx8c0ElZxp}
zAu$bg_Mn4v4lX%%>vdbw2?yC{0m@SZwh1cTdA7v%mPY(7&HYWD-cz?6k>+8Pjr=cd
zr@v<zQ{Z8d^1~rUYzHN(<12;ty*&JOg77Tvid~%PTW00As55l5&h|V1C7@n%W990t
zp6T!SmW0mV^uBfD&qRe{^Ab}<uD}aQ{k}Ikt~M;>zLY7}`8ky9t;40?+w#R@`IlVa
z`^@UR^a0=Od-v8g2)+1wZ?fT+{)g{E&VM)-^P{TvexA>!diE{4zA=Y3>CAN8eYog>
zgW&F%KB-NC8LS7pyqQi4*sWzrdn>qXkNCbfwi_QhA8hv16x?<$Ug6`fuQ75TX1XlY
z(3j(QVp#q4w9d=U_$O`mWg9C`E3WF9sSpv>z>*a@TWxn-YwD&g3S4U(_-#({B?+)v
ze!N%Pv`OcGCjU1Ft_AP-CqMkI`1^2;S^T#T62_VWdY!K#9{tm1y1Dk;bH%UfvoCDE
zekMX;-h*hlpNL~SDkhu}Gh*MVBI1y`z@?d=HA=@L@W8ajU<I)+85176H?3vcrF3JN
z0w<#k*EGQcK?%;2OnD~>H55OPXk5(l<4xxTAMNO`If6OIvqW_3|0-DNzF0j^ziDgV
ztxeBQ%ZJvpR!go)4^{|SpCmX*x5F%Pf8AmQg^QcIM6IK~WVNon?l0Wrvz6mou$9-9
z8B4$ZxhO2084|&`*Tu2pP_To2RN$gDudkku-y61ci=Wzwu&mW(?zK0yFZ(^nWL&zg
zD6M?@lBmr|{}<NDR{YM-on86+XOg+vwk*aa-T$7<-D5Vp`3$d7y6FT_?H88>-SpQ9
zS36E-_MLX>WNNo-k2rgg=6UJA4%3<B`eZod_c``Qn{6;(KWBmT41L8(fu0J_E110$
z!b*g)C(mIC2w32|OZb6CV^(5n&szuM)8eZ;9$lCgwc^pM#p_t40vm+9ZM3&H2VCht
zBFZSN&=qL5_@aUEYf08log0!FqJ|4jC<*E6G$)Bv{SrKCw4d)X6Q`<g%u%B#7Na#H
z9Q;N-Q^F2yIxerv{%QH#AdkXUjl;XH^lxj4ka?}V{9ee%mCaXjgjZEnT}-jwpQY)`
zVC1oLgTgGXq`oPEtGf06UvRPK+T!Y-9j&pq<pH;6gIDgPj!W)>PMr!f0)k$iYCIr(
zfS2*q3;$&^!lu4dG-A7ZWdRd6&#T3T1)G*n3;UzAc4}Zmp_abZMw1hLjdu!IZ(7G}
zY>tfHVAZR|cu0xcPq=+)*wL)zM-A3qxZotXN>K9kHo2&z<aE)%Edoo$$`&~!iJZN3
z>5Q(%qSsnn8b><T&Wmz8nH0a<q;pa1DWy)O$xYcW(?vD5ELg6&GDdUhv}LD$^@v1o
z+qB&H@}G^ZQ-basJSE?^^X@eT)0ZtA+Y^P0EKS^lmu=EK^haoc?4_)iGp>mxuFZ5<
z$ey(BMFe-(x~XogA_=>+J)b5`{`$%K)_<L!W~(x}__SR8gP#7{%(;x=uMLM|QwEdQ
z$`?02oP6H@FD77tuFs?kOJb)+u_~FdE_k&fjn!it&%X&<7az9cUbI|p_lrf@Q$pP&
zU4;Xm=w{?M`OPUds&YE5efOkkqfGiKoo=V@_Qem(bu@lO?a>p`EWWh7YSxvexyv$S
zSnq{0Fi#S#FS_0$))M|v-P4!N$mdMhzm<E^=63#GE;{+2rSX%+GuB)E-ePvWM(17T
zw41-pj;`P7^yb@>2|a;6Mj|ho?p!;W%gi5D_44(a+4l8s*`r?AcyF9`+ubB+<?5|3
zf|6$PFaMS-5HxocOOsN^^)1VVzu#;Tso&*&x9FL6*o4>r6`~S*8U9^c**N1MPpU@a
z1C|eM;X0>zcW+|<<TkNWVIm*bgp~`EY)*)YSuS-cKNy+Uw3yRSAW%d$=hXg90<Ah3
zNgkZ1ZzpbQV9|JTQt!))qi-fKG<sYKaO*V`v{e#VUw1``OF+O?jCIn?4GIpzn;e@Y
z3Rqe*E!UL!o|5i#ShjRcjCT~%M)wmwNt-5o^kIm7C852I#icCfMDwlan>j~5c^HHS
zMMXc14lCNq`K*M!P3Ttqw-wGNttPznzMJPqGP!VloUmNseWOS9oyYdgTiS&=Zc1I<
z7BHVzCO~1&9u9UP39t5sT!qa_9)J2I=e9j)$z{#FutXw6U|tTBb^HI_vxBk@X*Wuo
zj#wttAo(M?O>e^ImS$~HZIvn013MIjn<Sg}E!e<y?CPWHPU|4Yf+Jm}8=5+no<5{e
z@kGCAAzz8#m6i>YO#I6<)`_vqPLE>}V~vtf)33b9BJ$vqaNq^u%D)zi)s{%CKXuhv
zq@Ytmn`s+Y>sby%>0T$f<xl(jX8O*b9eB$pd)4aQ7fjMcxE6IEn!b1M)Q+@?Umk9a
zxgvVnRn;jh@u*nFQf<Z;r)F7X-frl7Imh|wlYmU)rH<Q-uLTQz&X}mwq5rtqv?rrQ
z;qVbD_k^FP|FeF`D3d;M!^i2+dj`Ha%NLzwKEG{M<h)aBKCKRQ((~X_s9G-j|JL3A
z5et|^tM42Vs~0{X>-F_|(9T1f=V~%*286N&Id<qe>{%gEB4W5LgU#Uet0c7}8~FBr
zNtXT4)ii;HDX`<_H{I@+?s0FDB;<~sHkhTkm|;gm;MKH;)#9?vlW$CP-B!?7R({Jn
zz9{&Z&xh8;+>6s0lpMLwZb<GDIB{gkzDbe~CM+`f_vw1FytK!zJ(<Q#5*cZE-=+VR
zsir;X4&J$Cd)z+J2rY#(r^5V}d9E?Bn6S3uA@}V7u1nJzdBoHXi#!XHX_RmenJp#d
z)Uzzq>Ein-DNoOq+-Q(}*7H{BaivEOH&gg#wF%16H=a5=T{<1dRXk0|gxg1}LV%ll
ziM>N*+Jy=KdwhN#oT_NyctPzJgW?{uOSvf<kH&e29{OAJN_=W*tFDHQ!1Waw#s_UT
zYb;=3{luqY!EZf}p-!Z=CCF{@_bv(RQg&sl@F{K~QLTN~4(bYjZ59ogGJV$KfZMa4
zzKdJuIx*4Y;ga1On7B&BrX_w#yL0x6vM`gw%PkXFvJ7peFWY)S$a=|f_5C*%$(qQ9
z_rB{EUpT8Yc*g3|@Lz19Tw9&Imt-7fjXx@&r_;g{qv5=9qHMCB$3pAdFNN>S67$`x
z5N7%Hb6@9KDb1~2E)DLhL$(WM2+Zg>b@cZ_1D{h1w5CaXR!YfOu=Z%vf}?qyQ>05|
zRQ0cMU7D4sutKZDqH~hxf2Tv`o^gU<1<bOna-1!4Taw+gpK~x)USy7ovd9xM4m`TM
zM@eYH#U{058@g`wF>y^~xF+TMMNg%N(fv|uaGH*9;Z)`GX_xN$yi-`fS+z;YlWE0W
zDc;@<Eq=>-UPm2GQJ?I3WtNYY?@z8Pj-RJ`^ywdHNt@}IRJL}-XJ2C(=MR}OHf4OC
zJ4A)oeP}&-pfPpxUgnqr>pMFGnDj~<7*t<0-DREdJ-E=3DgM=m>hrSNBA@%G=qeRH
zeC#$MWZ4s&$2(IEZU!It%&=2o&WBk#oRhp6a~>S~znR}<nL%TB$MOxo9{T56ESz*u
zy@l7zmE+Cw&I!IA(%D<(PwZXnytA3*|0eEKjpfTvWHKr5|8PL;@8^K~DH4;F8Leyf
z1=>q-UJSbK96w9pblRfm-Dm$~xB2&I+VcH9y5Gg!?O(^F&)VTJ^KZT^|KOyQU}6|(
z<))PXPTc<F!ID7JzdKi^-4x4a<YiRhT((P$dCk_VD^~3el#2hnHNn~}S=%hB`#@^2
z`Qa*s83Jo|OLy>oIgm8Dsf|H({|DBv432eDQxv(mqB8ev;?l0X%)jMThvede9EWCk
zNiNtbv{zC^ZHj_o(*y0}%saSOb5=AvwrntCZ8WMrZMMI_a_we~c<!~24saC-G3YDU
z9pc=azLDK}<<8uXyx!VEtE5?35>}iz+2oTT9(F+KzwV_b>61O%8hAcUvZ(&Fyk#bL
z$Le*fF0q_o5Q}PHVi90{GP5VDgT?$1@0JAasDeeM4;I$FX#F$0dw0N*Yh1l&1a@sq
z=y_1YwMzQHbjei#G91%SD#aQLvT|Gd{_1hyTEfA(B;8Yt!`hllSa<7U>ut%_yQ+7`
zuCZp^u$!IfnBD>FJ<=Teiuw1xKHBntNl1bDeDTJgT&p&(>RoCmb&T1NQDLLlY3=V5
zq;@V^s%+CCeR)sgi>0x<_!~8jYwqAr{A4Jp-EqyE<J_v=3hkv$4ih3f<((xtvV^%*
z=cwH|ZPxf;VqnLyh|TTs4<_jEW)1$ZU07>s>dxk{0@kn}i)X!%`Csv|^DM^})5}dd
z3O&0P2%bH`qJDB;6~~qarHMVp;-?Sn3tcdG=?uHtPNi?btE3Y(XHJ@F{CM8XAkW2I
z8}^>u<<DZkc;n=)9W!RBbgaH1%JygL_2vWNfo7aO+mgMvac}YBQ{hN><FMw*sWl?p
zyR?~0ugsWq!iX(WdXuBHkTplPaQC6ZX0roqHa;-R{A9NK68{dN?P3$BTr-|3FT!D-
zp<r~wL3y)dfe**FGoAJeXY<XTz;#kgwA0GNX5Hn2X1@;%Iws7*0_u!2HiUI7>{oCt
z7GyBbT3dQyh0cPt)*A)=-n2hEwfxc)PsRZ8Dy!+C3KG9x&rx@hQ0v<4)H3I#l<WWN
zNn03Nj#MfhI4K||zHst_HJoQ2cCG!vyWw@0UDLGiBVFQJlR2GtUt~JHYqs^y-LCIz
ztYbwu5_SCfT{up!?$k5r7J1Yl@I{4hb1T<DH+RV?JJ#?utvL28Me6U&zMm)kLxfff
zdoTN^!78~#D%D5fYsmSg4;#<jHk>VC%TUC>vzep9q{A%0(b3WDTnop|YUz|m$NM=1
z`WQ|eo;lx~Yr_L26RDR%;SHR{ub6}r{0{~$f3{NC`1Tx&Q=8*%SaezGEh>?H7QI>1
z)6*elMrq<A)vHZD3C>ar*`o^P#CPuS-J{XGLeMgB=93SetBt#tO_~<<Kr`&c)_7w_
zqX#WqOT7Mb$C!QJaq8e5uf~SU0kbaO^4P|CV*{rlXT2orZRx<RG90sort()GbmbJ$
z;`Z3A)fyzxzb9t(`PHYl25I{m`utB&;Jdu(T>vNdWv@dus*MLO{QWp{+s}@#hFwhs
z;twSRH*E+CoXu5V!F>A2nZ4TES4p|*&z&4~g5}FEy;A`aH)mZ^)HL1JF-i8PMwLXf
z@~7z<xl8Icb)U>yn{oGWhUyW<B}-Zq#EUFfSzOfLR~afW$&bZaWn=dQj>e-#6E_4o
zSV#Zyi~VvgQRLXp-PYgcgjZ}0=TmW8v*h|-V~$<U9eg*!*`@|J-3b47=X&=A>-aZJ
z(@$)e_=3-_(DccPc?%^s{{OWm;+cix7L}#lYtO%Bxbpl2-?hi)XBHcFFEC7!;*<||
z=ha?NtkdyZ!8S2Vwfu{S&|dHIZs{zaWqg-8#g#)lI+odqc2ui0zg=MzuX*CkOic;X
zP3AB9qb*K&9SYT1%YKz>SNErGMafAfGOX@8`e7cs_CIiWqI_F#t;*)LyYhC<S^RT<
z<BQoGkzVX9W{JVUsk6Pz_ImLiIbd=0`aank``@1OsukIly-m}&({^JQw^rZ$sW-k$
zt`0oV>Dt(_#85TfTHv(l<@m|BPqT{fWjnh3<v8=1!+q(V?M&wP3O42Itl7^ryV$tX
z|D)Fo3HR?3#{+7oREuz&{Lr~TSDG!e`Tt?XzK~vvXCl2-J6eN2G)U@BHQI2u|3r6T
zg82RgdLB7*>l`J$*9hNVpl1+ddCOx*&Q#$uFL%X!4iiq~4VmMYX?Y?nXM;g;tERIx
z*J6V*UjOo=yFWhJ{oD5Zp%Ax?2Rhh9yPHL0gdQ|;8c4r9$Q55XrOm?esO*GLt}6-A
z0vDOgqNY69$*ar8X7;+FFOW-QfuVU`Pe1$G)&I`+-^}go(&(Qyb#~k;?ez~iri*f1
z+TFkIVaEyYzAKwKQ<KjeQ|0*C7{g>Lux1q}>q=XOgPju_q#vD_xojnCph3`9SB}F8
zQAJG48*c{%#_X%pI5lO8DZf#4;FetrW4hmP28T;u{%=%!fV=CI*}79^Zv)Dm{ST}S
zetYm{%Z$gfR$R8S>#)u1Kj>nn`Jl<|q><5vWuh*9>n|NwHJ*|_>-O2X(dS|xd(OLU
zc#_kL&ur(_6u#dx-F*8doA&=&5HRJd{ErV(!JMK#rfZsSgos3mPIpNBe&YgX)*V4#
z4$W?XM;3R=HcTk|IBn(bi4z#@E@k!V&WsA^5zhL1L|Zr6>36E>L6_B=S1kV=7R+)z
z<KhzMU0h)~@g;MQ{*cv85k6D)*RMhL*iPdJ%ca*3>fC4+c+e%#vE8_%W$N=Lh9?zv
z99921TAp72<Z`Uy-~G=Mo_|qzQS&rCzK2t<AhGd+<V(w|w-#*YJlFaE!Yn7QWf@$4
zJ{rx^M^h8*BXv64HYF6F>Qim?dv&APfOD!r%H`inBepp*xZQP+buU#pE~3V9X?7x`
zg6*Z(Qj*P?Vf&Ih9xz=qR_)O>+#zynp(;o8zeMM_J^k+z)&EVMRj}V-k-;||*6<$_
zOm^;CC-?Zt0?C-)9ox?u#ZJ7ve9vvJJ-4^!T@HQsc<q~z%BPQ$-dwl)_ww(|?1Nj=
z4;v+L2C6K&Fq8eDmV4b^*2X)N+BR|WZ8MC_nRDQwiKz`^!<*C6YZ?zcbiWpJcI)Mj
z9;*xfa0IM7c%dw${eq+LHt(=!PXax2!`Y;n$~bSfDkL#o-1_D&@4oHFx%wQPPo2?p
z{lB<kUd9&AAQP_(7J|n<tPid^mof32^RG>>WURlp-jDRL4taSl=8Sd8vfaN$-9DFO
zzvGL!`Zm0$;(482M$JFB^^)(uu;f>y<?mIC*p=3wG`p$2ptbGX%uOM${;<95d6s|F
z_smH*Z@(A)Cnk&dEeMxk>p$Tf+$k3oUv10ozLX)>X42h`R<(}eDGM6JI&L0KT;I5R
z3a|Nu12505?wqkUU|aE&XIrnx^IZ}1Uvo5D#6dIjMCz+rpH7u|oO50X|EYT<_C!X!
z)l}M3aDm7vQPaoHr=A*KeX?_H^wwjyC75TM-;KArmR!Ix+s3WKiuYCjflp6Py#Ds*
z@pJ)xO^wM$CpY~!ZZ}eV{KBL)$2l$Gj&m?u-sh9Y5+|+rTq!+SwcH`puEXSO%F~9B
zT|BC@d>2UB?VWUDThvi)U2dNl37xzlQxDzx^k(K;ar=&2&KEf!rY=~ax8=YQ^Or7J
zp)M^7N6lYaeKXqdv`>+_XJOd=XXzVTBU<81eYY3*ZF|FZzNqf{9^;6&Y&Ytbsg|kb
ze%qJcGVl4~i|Gw|`N`J{nl|jNcgx6#FKe(XDPNb~*tfA!Pbbm*+jg}FJhL0tOq*9Y
zeV&;An-kY=99+a{b-aUdeo5mCmXm%R75kT(_`cbBmUHTZh<UlHGTw<ZOZkJ>eG9lG
zGHY7m+;zP(Wq&9*bIfTi;**yCfAXqXu$A|`-o6B7)|;E3-reecTe;)LjI0(8@smOa
z7KprF+|ptHsr7i0gX^!Y%cmTeu|0avU-QFN<yyxp*X^mCoB!?5m+O1iWq%j@_2y-W
z<IJWDSvi&8g&E?%i@1Du>@3<`m2>*s_cQnIT$#t|FZZ_pK*uH9L%$|4^%q3y&B(vZ
zE8ueXQ%SMH{{Yd*TR#^wK1vQ+v#{XVqZ=h{53fD@xbXf5<tKgyPs8$_m$^o4Z|C^^
zENZzyiu1zvXO6}kkN<w*{@&Zk%ECIcbkFOYf8MB(zv1Q2%bPp0uh&fa@ZoD*#g{UX
z%jNGs{cvrmTX20_htUC>zsnYIXB9Us_`JfrgPC3WKjS>9Rr`xSWGtvqH(n*)5&i4M
z(Iy9nMAnH@->fK%oSy5>@apfQ_22ma=g9SFU$8ztp`pK7opbt&%%#tHlW!HVy}Q6z
zE6=!adxxD*Elbot&78g$!Ls5VyFS_p3UUXB957BlWy;23mU5y%>2T{Av1TrQBb~!X
z6D1i1)xFk4I58aSmNewLaATtKi3v*XlXN7F7B%Rl26TPNC`vv(Q7(4hlAn`Q+Gkt2
zJ`*{an&Lmf%<$VDi_d9}J#!o;-He>HG<cG|s~)Rm=z-9QOS~p#Ij_2UeZyLo9@i78
zURS1P7_ReuJ?$~uj-X{8GX*~B#7)dKn07N-Fh!HG@#iNIho;Ag|JUS;C#$F)`1r^-
z<<`xt@b2WbWhrh~KRrLaIlX$H%WdI=^VeQpX3E)_^mbLK^o^yb&dl>YmdLKRvv~T2
zx7FM87-gOK{8w(|KCXAvQf&XMuX<meOVkG)x3bYV@l-o=vr6ExUhNN$n8VX+igzYG
zRP8@BWuBas+Y|kZ{^#TBt^R&LG_hQ=s^mwhknh!yU=>HvpvJsV)k!t=nyLqa<`$?Z
zdT=v|O==Nw`XSNYSNb(kdg+_gYhr4?Pv=!CEDU|*Q8-m1Jf=+d7;Ep<g+C@qbM6zl
z=pey*jLUsZPm`duGZ(`Wk2MZz4HI(;Sj05F^A25{qQq<YbkdRu8#}cXW6N$z{C|`1
z<aE~78<)IvGrB~lIdktkIl+-{>2aaCin6Iw=E^^`6!A29D0#I}W#+?EvJYF?gDR3g
zie_|uF#V;mWa6<S%bQDu$~2q&RBtTkn`9?(u}N&-_7F$8rqWd%(FF@vdFmKl3tOVP
zOY1-k(-GI#EM6y0h{fdwG79C$>};I4NP<^d$vsBqnUG|suluotd9zYf+{447CrwMf
z+I%sUY0KrVl&w<>r6u=%J;o^hV$tl|yAE#r9azX~rZKfv^Q(xA+v%t5lZwt9Ex)nj
zq}lZQA>tR+;wqLVT)*fadrfT~gUl4BIO(T$Q}jG}t)}Z7|Nip&#w~tlrHz`^E>4*|
zhp)^_!tuWh*M_2HdA&9dSID?}Z0=Wd-M#s&($+VM?n;w3y;>!`M&rO5h25`Sb;=Y@
zFo~8q=yj>v<M7tWB1)36kJoqaJh&p*TWRIx8_D{fOSW3~9%@V0$$P2mwyhwATQl0{
z<FYyYwIUM_R=@r!c=FP^FQy&|{Uz-i9<D73lI(B}Ii<VqhwBQ47Wdt9?r#$Lgda2-
zZ~NdU{rX9IRGV{`jOz@m(0kqs`&3?e$7~Jak_`S*wfL%v@7v05nUgcN3Nf+tU;MDW
zJ!H*BrXNddE_5e7S>wMa`LyCz3EoIs&y+1ek*P9UFCP<;7U4*!QGU33Ug^G<)1{9u
zdi}ca$cx|FL3`9H*$V^yGb}p6=*9Q4cEx6o3marEYAEF`h;3|3)d_#1Yx(T<$$bhL
zO8+Z*xXljAKEIVdi9_gk{OQP>j&4p1mI%B^RWea@^WxY%p`%N9rLbp$iObYUF@6kE
zrvf}Zjw*gy)5W}PvgoCG%eHLa@jN3tQlh6uy&-n5N_J$hU|h)2U2}Q78?6s{I5m~1
z^oejPJAG2pY+w;fYAx(m5KNx3Xp&i9MyBx5Nz&V+63zGh^(}Tfc}LV&ZSuL953TEt
zgv^cO5Z!xsVO#xAKDmud^P*jzC{9?tP+57plaWhagn(dxq)#EYv)B5`^GZ(5cS~A1
zd5@~2wu<V0i=<tj98bA*RrXc0rR@G6`OS+Vw^7P{Qk!?oKGu$i8l~=6n~psEnKDi7
zruTH0OI=z<twHlyPEP7EVSC-mQZ2D^(l?e3x{NW)MDN{cie<~RyAb)bam}9@mzO0^
zDJf)3Vww=M`H}mSt~=bRTPJetx>xDYQk!e>;G#!?%C~bJPl5|0EPEWkc*-%Cy;SUV
znO<?~!kp;|0?p=~sV7|8{U#o5G0>0}aQBhCB)qC`3Ew5pP`5TANv>t3W;%C*l5Sk$
z+Pf~D|CP&Wi@?RYKPI@@ow-{w?U-=vvrX#JJFbQv5*Lbj*VU<1xafm{65}?X17#*j
zJ_~%7Z{KL~vd3C;vFDT{DnSv(DxpFLBu-q|CX)C6VQ=x|$06?9-Cw$<OcvPb`%q<*
z|B8m^rg<5z(=!gw+AqSTojPM?BIi|)TOLCF+L}{TZ%fMU{rFUI)|<!1zE0=PrcGJ&
z@=P$Bb@(Me*%z(Ld%F6TIZX`Bo3gBTR+4ALa;>OKWh(287H#pGA{>9~)U-o7oQ6x?
zyo0V~>a!;$D8Fs@@^{-j%V(FT+3SnCJvH2OJ~qd(KVKO;_sNpopPQ%usS=WK*8CK3
z>+$l1L244WwkX=9eDK!fnz7+-$OPsSmLj`OEwGp}b)h*=?8!L37xz!87B4t;FvHWr
zV^Zh2(&7bISFk#7U;miH)y|nSAS?IOp;?PQq^f+`;9<^UJ;f<Q=|6XprbmFzBnQ@I
zM~!t)@@B6Jo_lRYd+>*a7vq#d6h4XQdbuu2N?LMci-Tx|P=Hg+@_?&zVy-SyRE!A`
zV+lJ^^RXhWL~`2RA1VuCFK@d(sfS_LEw}B*CWbXzXS%v&Dg-X7RH|$F<k?o*ow{y`
zNV-?QY}h4(C%1Nr=3ct(rF6Zmd*`EojkRwUN}P6(uYEhIb!|-M{<Ndl(zabvl=f<L
zy0OfwbcUvy>{HF)YYJZ%{cQBx$C+gQ>=YYon&!GW=iV+$Fuya`N^AR8seH?z=?|WH
zsd%2__Uy5qlwiC5N?YT@Pj;1ipRe1vMv-+=$xct50?!7K2MaxHmroXEY+^}RHRpeC
z65q5H%ta}8UiQXw{yHqwwxvAc!X{_ug<t*FznCoY^GM25F|7u3mC4o>KINSIiZ#5;
zRFv9Jo%Ldx_u+@m@tEgJrbi3>nHS~#XOTkl4?i2HfFGW(w{S%jdL`{j<cwXamC&6j
z>7u7Nv&-eE$4klH+Eq_Jyya7I>bVuTzD;m<2(QYztW}=#EQB;=A}hESta`hmVEvW@
zKh-!RZ{FOpV(QB5qlSJ;tNSb`T{9_Ku~ykrSaL}~+gnziCsl7xE1o<n-j#SHk!{Wl
zmnh@e?OPU2Xzflh%0E2IvH7?db7jY!@FKPSFRq_w7MWlzeZ9oTZ^^~Wg(rOUuFbv>
z^iTKP9Zt*t(xM_o*Q_+cPue6_eAy(h>_Urg;z>>0cbONf|7hvUbhc&57WV0~T5k34
zE>5Zyi>?X^^PkfoJXh-K1dFfNw{f40n-rTjL7+xi?x0>+h`-Ts3+Y{JR<xGf{-}~=
zt(K^0rQmo!d&Nfyl|?^&CB%=No_p|wbzsWH^de=m14d8G?{Aq|euU>8+W}>t`GRcx
zVN(tq`7)hV_gw53wtN<L2TOsXnAVyN{$EP^p4~b5#&yBxztQtz;^z0T9J~{_cSSo}
z-;Iu`bJ})ZYqRO7oWS*JN5Hw37B5eamL)Q6QyBRswwzwdvGR&U-`yFjp3J(sri1@$
zOg?Ai5{IcCEY~lFipB-}Z+vaZnthhjqlalykJB_6&nu>E69XbeCJA1*o$fIw^wy8o
z<Q;-(9?~~F8^6uzj2D&iW|=l;OPkA$$a_-{&XK%kt=gmA!6E*pRkx!<e(S+Ti;yQf
zB(vYdyWE(oe6wlZLYIUu{m#Bej)q)&?fKw-#e!x|*Q0_Hmqtp>Q<&yCVb=1uEU%`v
zXd0e9x^bGi-Kn>QEs`6qG~SuFVntMyK<}E1*WO)?xt18H=gE7CA@~CWf5F}(7ZY0z
zryhw&nd-8}$4QY*EN{_N4hD%?Em1R%P0yLh5_9yhijT{I=-!CviV011vH^c(Cq(7C
zC+G+>i*hKelb`d|-P-Y%g;vkOkG}PW|Jzx2F5Y=3`L9bvPV5}ZT7{*Bb2|ShADcSa
z;oD-?v_)rjD4enpe&ZqUSobJrjg0q+<DHRrs&@8y76_ex)AD-BX`NhNkADi6?woqJ
z<J4N6ixVccT;P;-V^F?0)kR26cJj%%mjTy)mxj)Wof#f@*Skb;>l{&!IiXP#k50;I
z{B+b;yCpXG%fduQK4#BpyDcZlo^g}h&^G(!<WD9^2SjE${}fG-yb>?+Krt{(#faxg
zDqpG30%Nc90T<>67u<_3n3m?MawK(@=H6M$=gxcCJB91zk<b?&*Pnzg$VhpRaU#g2
zrRB!F#Vvi7FXtIPTy(j$rT-^i2&;d=+c|z74ni__ugUzEcbmcJ!SLvJVaq4S`J4-n
zgs40<4xR6r!Ks<TCX+co(Ca|Zj&Mi8J6oTgNoiI7^yli|3F1p;o_>_$e#~cHLBs^6
zUCCZOF7-Ccy-##=7YY0<IQLZXd1b+Z<uMW&fsfeAru=yjSJg6UdCv^zTOqGv{9KC~
zYf_GKmmYN86~X<==fnn)-nS1Tzg!8tnk00y^@DDkV9}8eGLns44l$eu43uPZ?j8}A
zO4-QJ>8EslTH{5JfQ5lV+~TD-w>TsR|2R4&=a6WTaL$5<-z4O6MWT{IpEBNbe9Cdc
z`sP7HxeE(ogO0Aae$=6*WW&<Zuq36#2duXyI&NDU%a-yWL&T58?~=uTcBd(E$Cva>
zPP-CS;CS8Q$coTgr~W?nKa%9~&!tIXrLLpehPR9Kx{h3&uyWmwC6zLBSyUQ?n$mV^
zUU<qfBh;vi!IFEIgUU${VN=eh9tpuYZnsOWGHLoabUt_{_h|Y(28A7*0=rn|?EJ!h
z*kskOFWsd!bF?fcSQ&a0>)b6jU0(Eoy=uuUd%h*CY?^LNYSWfwt?bj97!;GJ@PzZ&
zs>C$07YF#fj)<H&({uVHyLYC?mOBbO4;Qpu_3*zq=dMKW-5ai6`&eg5y=ahIIE&A}
zsmUc&Q&GQ{CG(w2X9P!h>Wic)XFU?;dNw2m_XvuuonY?i_)AtOUQ502QXqp@Tz}U8
zgcCe`H-5A~xtY=*^(ZOkx!R>vkDPnyQ(0aZ9>4fzQWD42Ref<Urv@DUraF0F`xoD&
zHIsO9?s6nAyL8BJj{Uq2U9rV_yEr^Abu~=ONn54juJYofoQ9i*L%Q05iDz$pj12s=
zce2w>8DlTk&<)ErRX!|keQ|Te)rb(c?KRw*IS0h1#O~qiDSGD3*0$V#1OL7kOZ{Ye
zG!v%%;Y@H*WzTn5e!VPr*VEG>AKc2Dp8E!P9lqLf%de~4N@7CY@)*-)6V6T8qQmh!
z_Eo6qbIZowuB#8c8CEa1a65Bx&P55|t&86HF38->Cx2Qg<J7(u76Cm`!~WnD!8WGu
zi9NmIo6bI+_Mc@!#yhdR6`Wfew>Hf5SmLSe!MHl%!MqPE_BY*m`c-hs+OXSu8xIKf
zIqwzTFL=yKHq7qdx}r(Xd)Azr;B;d`l+J3t^~HUePwRRPIVGmL<}Z(3byHAzO=#HE
zPkvhSg0*(>typlABaG$sj!QXnThH7&D|Y47g`dw)-eWn<Hl4+1HG@h66HCK~6Kh;$
zz0b)-A7ML_%hTqUc1@$_;HK7fkxMUxuQ*mR_as|~?3H`12hv=$oL1dyQ~sOC6UK71
zLc)7`R<33Y%TXDYJ74=RFN<ckNxwdC1Lx<*--_Ni?pn1)<H_O|t51mJtS*^U`ABEU
zmNQ>o@hBg<;l1>RM?uOGg=If4{P&n0wtCXWCF?GF#T-rX*z*4ID;K64lZ31uW_YrC
zd#rd~kQeE?`0JF{5=y*h=6Gsri0+@p^04IKBSyc=>tdyqyO=oVcsQK#Qn=3Gxir}|
z!Y<8a-=Vcj0t70|R?OYOe&AKX0j5@^+b0#BkAJrNr1vS}(7s7*oqWspv>adPH*4!s
zj#q2di)GwAJ7%foANtmH(Wi0JSy%6AIv*{MhM2xH2;1nv;xOCJeWQrP=@^!*eM0f)
z))wDwnbvdYLeTW>HJ3#+-g+<aS@c%i-cHH>*xl<N`WsSTZ4C^Qyprjzamq!*&w`I_
zf9JBo3f%)Ak6W~_{j)^(<%GVQT$-$}GH!Eyy3F){$^Mv@@|zp<mnU>?jSO1Sn(Wfe
z_mqe4XV&LCn}g=G3dx&zh|CO8yy<b`eY6VW)^l_Ge4<|DY&0x75mEEbgP}9IhczW4
z!|nZ&^V6nIEO{X@d!5lq-%H<w3+Ep3|G|2srnhjWQ`MiVpB^qrG(DrOd7)S6hK%^7
zM?O4vYR}HQ^LR1S`okBqyFBLbgkMpYeD7DWDDbILKw?Z7hg$T~30ZTaPVJbNC3#yT
zn@e+cOUXJV)%7!%Mrpk+(L8=8Xx)NO(faQ^j;s&qbV=88c)Tq~^mLKOjrBqcio$-l
zZdBl(S25eOGW(1N|5lxETWb=vuQcEB3D@4@%=~)#Y{@faK1ZA~|0`GahDS<q&t1_S
zD4t+Cy+KnXsLpfBfs=Xp*A{d8$=TJdwR@F3!D;KAvPo=WJuAz4f+q7Ubvs*aQnd13
zhu^}4HKz(D+|ru8@r;M-o3<+#f-Ksm&uVe_y~$DH%Ng0c5XX%|D>9wEPw6lE<RWZz
zxX*>b*sUXN;f`rXE_E1A^;tZ<V)EJvjtk#jm3cqkC1mNG3GHXql5+1HD(N_GedyMc
zkBcSmFnTdO=-SEHEPbo@QA*UY?3Tq(jd#1|$5%LJJz8z&#d3Obdvpl9$Fmfp_qv^$
z0gM-axarIjdRC<GmNYG$Q|flJ#e=q*_dVfj7Mrm4ww#Z7aVq}zj>l~-kBgQuefV!0
zKfC4ct@j=b^E-@%y`PwE>5y~WttM^ADeo0+sGoXGMXJ%(bIy+GjJoGGoDhE~F{__L
z;dsr4{3W*+XjpKXU*gn1=Pwf>zh1T8<oTTY-)@>kNLjzNJzPA0d+y>7oGPy$Z57(m
zrrS|+M@zvm*xQwRdhQH?`wHr3b$eyE^naiA*y@X~gNf+!903mh(6=n}j9A3mTO=0#
z`d#|vX2a@fnUUgmT|AQ)-p+V8J?Feotg717Cha?J`9%k_HQbzUepEITyK_*wZu^Qj
z+c^K{TMn9RKK9X4wQbU~2Hqlh{S!CJ+Va<gR$t@pFOB;scXh^&^D`~<Vp}!C0-WY9
zt=idJlJZ}F^`!52go;*I{<e2J-S@m}AM;OxeMWzij>KJmzdX5R5_5QT#Ko&t5m{ed
zf^^Pzd3zNbmK8JiN;__wp3!z==Jg{67mi6k-w?yn?bT!L`t)E;d#F_w^Q7Fxmnw^x
zVp`jGtB11}AG@#ZE5}&G?v?O7Pd??ulDAgk@7W|%^4*UX__TIJyLQ^J9|{XN{^V^)
z%c;xi+%F#R)lAyzx8=;a6AK<BRiEbl=&(OxX<-w8=+%i&yO>JK%O|%i=RR;N+|cE7
z>5BI)e@|`QFokcHF8{s!FJg1Q8(RJf*mp!@N4ZCA;;fisEE60R@2$IcKx2;9F(=6f
zk9Z>d!o?i+u3cc?_2%FIz*(AcpHEioHIrD-JgMOIp9A+;UQA(lAzQmt)mC-)CI1a|
zSHoPGx2f`$7X>f*e!ZG!F-zLE+m}RAb}>Ei`r~)?qVMZl)u%#APi}Z%<eDG8u~6%V
z@-DxCc<BH(kDBgj^TQhj7aWpIXIytVaQ)L8>r%A>H%RM0IC0?Gik*#`4fp+z#Lv#J
zG5O!}=kk|ZU1A!ZyPvo0liSJaP$FP3VL^isD{Fz|4p)Z5ja-6mDjbH1j$NY4aZ`3o
zOm^=Vw4C~|rQpedNvRxNA^}|rE2pWjPI+=-(z7%D=7n`nj1s$8RJrQdLN^$8HY{Qh
z=J1%+kSV~(SS0thBS=ZIDU5y6L#NRH4$>`p#as!_6GRVgWIf2GQT^@liuJ4~m#9Xp
zo~o-Sv~A1HUF8Wg_o|vmO{pk&;CQHsTiE8z#vbV@u|nb|IU6==AI~%3esJrDpX8K0
zooulO$9OW##Z=jjbxc_Fs&}pWvU8l)i`{RnNwz<;*Xq%f4f`AQ@2tI$=zMdQW)SDT
zIhMsIYQwwaI)k2-H0g-irSG{_b*&|4nNNU{V`B>o*9Cpnd0Tc~zmVrusn@(!;d-MA
z%Mn-4`TvTx?SJ}e#r{UKji;t{+){Mr;yb~>W6&t{$icixVUNe5t0F-i+Ik$1KP==b
z);?^s`ox3{Eo`S}tYKLvrNg?VYX#HU^+qv0>o<mF{I9vFa6N}P&#=F)=fwsEOQlZ+
z9NErml;wh@PHca~)g+|Yt|!8$u2nmG0(Zx@ITOsAR^_bQw}FYJZ|R0}J$`3QIUlju
zd~E3YtiJMwc88|*Tq%w79XBSwQa)aHyG!TUjbHDTr{7V!u$bxYB<*z_FIKhk>`D9M
zee%AI@{3Jpj<w`1^0M8{y!n{T?NdBw1DJT0p8I2@yycfqrE2hHx6Nye!Wwz<JdUz8
zXIwu~#_}y(wKJt}k=8_3<3+riQ!d}^kVxv-(Y55uB!#!fw!FWYTRcfqX?4t^{-di|
zIJV#UbB^`<F0+fWnO(fcy!{(LGX*ect;h@NSFTYm%D=mh@6dV$A%*`!5=!fLu&(oA
zyyG=R+2!Pl)$^h_vX;&@T5@B7BcJK10@L&7Y+f7+docNei8N=Npz2Srt^Dq-I^93F
z7%6BUJ*dZE+{-Pb^?9|DUdg)!n}lMf{Jc7Q!W5l{OB-X?-8i^lnO4-}1-x61XBBRq
z_Od^0n#YoL$=;_`r>{ybU*K1?YUQ%VOLu;$`Ke8CI%m<ns#@WzP{S$Bx}Uuo4g3Df
zTu@%Cq?NrmOhxPThd_~fZ>6AfMHL?!FKG#^VP;=+Lo7%oBg{Z4JnTfT^@$mRu^0Aa
zH9zs$`|0$SQw+L?`Hr}6{@52Lpm%htkMO6R>&+kCJ~h=^E^23wwU^;y!{9ptC8^=}
z{$IFu`gXwOs$X|kosatcCf@qpX_fu_|COd3D4u=xQCIO=`=@KZ%jSH3IDgOM?Ezlh
zix?uEW-p!iNILU$#aq+$3`~5ArBPfA4b8h7Ti6oWUiKf1aBedeJ*8tQD6=bXlGsT>
zm!c+VR>@n5-DLqi3U4!<k3A_`IM<fNzAkY>k4R9LnFy~P(}c(x53%`+&a#&-e(A*%
zn5unt&N1HenvCZxSR}(;mK$+0_DYHbNzL@kcRef^<kzDjsKTq=?fdwu=<LlBSATT%
z%$;<CZIOte$-N^^VXr)9tP)k5d}O`y$AxpeE?wGIzw%@?*T-q@pSHViInpnZ|LbJs
zqzUG%DnXj8C;taCH90s~&DyB2W%Z1RBPlyNCOA#DP;5SIc5s8g7)vqJjXP}1KG-qZ
z95<e>7%%xsNh(~yuiX2IlXOfX`;!HH8QZ>`d6&U*X3dYIqU){Z)yO=a&bPC%az>%x
zDu$&q^=2>{34EL@xl~;$y|d7)^<(IF0j3!(ox(NJAMdDKZ?Suv@nnvS;>3(io#)d3
z3aZ=+zAp3ELtu-mitoS2k42U>%&X$P5?ABIRqXdzG-@7OXlQ1Tcg7Wm9VaF?Tl_rF
zo~+qWmJ!0RWAY&ukA+s3BiPbD2x|77P*}WW!a=RP3|8+%4W%CfzHYNNPVHC}yjUn=
znTX7bRM(u+BP}w8uC7PL{QmFKU{F!*U6fOG%V9>><P}9rn9nz!5IdzlM>FB9V)TT^
z4i7vgtkiw7N@il%k4NQ>6Af0*VY;zA`eD_!9mkeLm-tS;C1-YW?wQPq6W(MCyi#ef
z<$V$-y>haO>D8%orX+HQInGhltvcz)m!+JOc$CfDLZIEWOC+Ikii_FyR?QtD#>QJ%
z`&?Rftle^mPr+KCH!qOukZ7^?(u!RD+3ntKa_@aUES&l}ifyLenT#DW!B^wgcuN0P
zF?ZQ6ncP3g#cxZMzxp%lRbRtS?p?;_J~!&{t*vW1C!C#-z5Chj1)={e6sqQ}=E?oB
zSZ?Qr){g=u3(BQA82Eljv&0oO3SCum+QRhTahvJ!fID1EodugNHckmzoLeFO_Jn}Y
zyi-#|^zY0K&B;8Ra&6UA6<x;9K7|4oR-6(s(e~*II<oM1NTEXP3kRXitLIi|`5MQZ
z5tTR`8Jm>(@z}YlcavH~Bo0kc^m%*dgw$^938q{Nyu9O-zux#E(7xen&O__Y>lTNW
z{QVecZuC;U?ZvC(dS?U8Qyv@1A7wkSbf@Nmjx*6r5;q0+eazl(o~YsX!%5)(H*clt
zXCBOdeI_t-CtPU0rt?VP+d<!mNiEBAj*2Y_I5TtV!!4g@#!3g>;tHSQ$FTi|s=uj@
zMo8v~lD)#4mhRcH?C=i#J@a;*tgv=E_q6R}x8=8Gw-f%}|Ng&r>mKjf*L?4-JQbJU
zeTw1Or%f_hveOG%b7CemvAoFE-*a{6xy)dll<Yqju16fz7T57;ygTh??+^1NrWrRJ
z%x-nL`!ATru{(1u+arn4#CGPxt{W34Rcv1PXw6ZMi|0z$i+)(<SE}rJS8w8?QyW|M
z{J3WQspphUu$7+2iRCgC;S~nwg!v4f%n7hs?Y`&MUG9~|SwTjYTQ_Xq`qeD+*u{tQ
z6%s#P+rCcb8EfQhmrGNQdx{w?=;S#to3UG;=~M59TJAR?eY#Rc7A_0V9Q@{RAl71Q
zM|9w<uYS_UrcDjBxqESfTc>tLQukd>uFH0(jaTR;Jw76Gx<l?%-u}ZU!=5kO^MCF2
z!|(dHyO$dzhwtu@)U#ncKee|wfvI(V0~4Qp<L4>uAD=m%m+G~7b6>@>Hu4?!lp@`q
zYgeYl+*{~Te#i5AXON!u!-el>e_SoX5_VgeV~=s~=?3`&HB*=JKm7J&pX$UFlglk-
z*Kj>Di<(fjB}rJAqm$2c%@vh>5i5Q!JF%ZxFQzeEbXBmTh0COGzA~39j&HeFw|3|H
zZwX6(T+<BdY2*7;aCXANR_!ZYM*~j^&--yi{l!XqCDs`&f0SJ~qaW;;bXMzRUeZaf
zq?40a*Sz6-x+U$=u9cRXI<?+(^jj?CR$zMYN=xV3>(gvG%rlb{*-k1wc#>bIH_6RF
zy(Le15vTh9Z-;MlEqb|6*FtJ}qm=uFyd>dMN*7NgT;zD<Q1M~u6o;k<hec;Bzc!ES
zj|lTGy;Osq7k^fqu8CUfp?BoL43-i*rf4I5#bpN`ap->(SpV^yGUFL#p8W-JQk}ME
zl9k((R-5Q2vIv~n(ZICO`Np*mxwAIRdo$ttnGU5_?Xl-x>+WEPTk`gKo!(cLgs(P_
z;@X%B0}i=*v?+XWPB@|Jv%{fjLErbX{A`<oI}AM{S&2zzT0deMi?=bk#&jHK@!KWm
zw96(<bag)8>{SuZ+?@kdgP$xe$y)ztTLN>D;p2I|>%a7C3VHr-5<J&2apJT`8V`Du
z{~Xa!OE5jArMRk9aSo63{|#PBUCAdOd7oO=l66h<X_v2$9P>OUCFK=Kdv`2(Z{wtP
z;n?YGsVZ)d*5;jiR<vr51*3rL2RX)cO}5p=HB2u*=-8*_Y|{Iv_UFu|6?z9PPO#oh
z;@<IYi-#ju6K|GIzSAA`4>~~)7V(@o!Y#X*{XN$qrr%CGnHFc?OW(82g<Cy=S-_u(
zuixg#g^vde)XrbvVP{fWnxLTaKtoA^bIAiWRVOAJCfiwsmixRvX`S!2YN~y3LG<DC
z6*g>EYYxvhX{x!!+Ptc0@;af{`yQ*mVrBTJ#u)Ve#k?kkZCN&bO@i!9iFp^bCbx(L
zBq<#1wHN47)_4)KMo+2c{h|{MqB?B{|F`uQyB*Sfth^+pyU68Kng&bkG(Ekd0DBL`
zLyw$FukgOv*Z$zh5)ON%-%3rtj`?TG<!Q%d+CMw2f3N>!nc=@F41O0LPk6+_^W23Y
zE|`1E0(DL$CN?I{9Aopv4%`ur+$;{<Tq(jz3S1`|WTP5IYJ!!fG$>5l>^DV0np^3U
zOSa;nX44`;smmM;S2@myXgpoUbLxPgvbFw8gGY=vpDCSUt2q<+C5un--7D1sr+yE9
z^<!){VxBCX7aHa)IyWhS>xmnKSf=uZM|1C;wBEF6qspU=S}TNhuG7)cyl_RS&@Aaf
z*W$z&C+)jV?8e8Zur%oTooGAqZi`ch@vB3kd`lYtb2Rg$d6xQRPji`4<}^j5>!esz
z-=rB6l-z@EXNk-f^S&;Z^=OlUYUC#F*iYQ?L26%Jm6kM#<TUC>Hi|AvusEI|yiI{?
zf-%#U1)@t9h?zL*MkR`5DNcE{FvhH1VOk(VgOCPy64R=Qns?Kpl-AqayyQF6&D8Bt
z)4U=*^{ks|%{)H(0tbb3xC;X~mGZ7^(Z4h4@|mp1%hJz?m1I8(@;T8f{I;>B>Ai)c
z)<TbkqOTrqX$ZKh;qJuZ6#B-orSEc5-{m{AL|WJuiJm&7s&~lfvAX~ABC$v|1>>bA
z+I`}@p&Bht|67<GmL^$F7xsOvre1tWW71a*l?Q6K9k~}J@ce(3$aAGhHe#XNqeWs%
z5_oPna@|Vw%5@YLa^TvMAX2kXcHR{&rtXNu&9U8~49gg-7N@M9lM;7*igc^;zAs-T
zH+DT)V7%{whvF)ixT~FE+p|-TeaznXSn<GT?<bREZwTgIn`nD;qr$z5>(23LTNY&g
zDPfxQK>Pj21@D%L_cpCjyO_K|i(8*>stdPGlv&*B)J+#S3$0EotEB4)q)*xQu3$@v
zx7cOXxVEKfYj|Rkewa1t+N~5myhX?5ic2Jy%Kwe;zUA%l2%5F(n@V$t>dpz=-61^B
zS=G{o)OlYl=t|02`Y7jY2$S1Eu|*8$vL0?aCgM~5*)y_HWWoc_;}4=2HzfXdOI$2*
zJaNWUg%ix`?2jzF7e_vtBE4~0qG`YT4DXj8CitCI*GfKIc+pkyN-DpnzaFoN)xQt{
z=BxQj5^Na0*vtz3XQyRj_v&Vtd0v2csiabA!FjERn^rzrq~E$;)T`0g^Rnt2M}3Yq
zlh=zx6qaz;bn$pi6@7F_?2V!b?~|oJgxq~j%I#=+xbFDT2kZK;=--xm7xtXl{$Bp0
zvci8spKqQz9L}2I_iDYuBnRKy<qG%0BUz_MKgi*ElgK?mfvcxj*rg-tqypEr1;JC3
zf)#`A3b}A?OUT)_ActR6u}w+SXxVD9BrC^tR@;sDEm^9u_Q=kC-mQ8`r*2JHo1yn*
z!r80;W6lO1ZgN*DIBoUN@<HLcG`{C5O3d=^ZH>v?N0?tYOjBU!E@3vA*k{slTl{0t
z=8nIUeU+ICO*n7wc*i<<$KoH|Iu@LPMT;D)UKw(#=vHiSzRKIUJxMrgq1c<G726J}
zroH^*a!BmZjV{}(<%a4lQKgrkY^aTRxy$SA{(Z?z0YNs~wtYOs?781e?ev5KZ!^z}
zW=iKmBpVk;zHJma#^7V_$n|8Kn4A%(ohO&jf&WVsI2CdmCx%YADBvK>-#DKo?&PPo
zuYH>0<?o+NVf`Ceq&v-5;jv$(zw!?gVRn_*F5Phf`T^gyBhM#qc#|g4DtPXhn#QcD
ziae<^RYfOjT~z+hv?Nh0Ti9}Hl9cb=08PbP2gGs~ilp$Tw`os5=c3SeT{Q3E(x6jf
zv)o<M(oIq|n5KG}M;N+#sXxk^ShLNgM%V9O$n<~54(q&JX4;;lG3APnw^^Owty=!l
zh0_HixED3PPSa5MwSfDO+0@eD_$=-TrK?SDEo>CEW$CcJz^on|E?(Qk;`MxPX!}N!
zM^99otUg}-e`1xdw37Jzz`{sL#*bU2pU(76S(I46D=RkF{cF1Nnc{U;i&p&1lrx>s
zFZYf4rpmq@M)?k6S}%@i9J%DjAQI2CqQqHBe}iw@;YSL#ha6AtTBN0sZfx@EO77~g
z*Wx;v`W<0!-o4oo<E8&>@tgl{hKgGj1wFc;v|h|(>xac1Q$D`o=lTCbRk6q>BPd0B
zWzNZK3knx4^qy9$I(rfSPvxSb$rD9=Cx&RA3s_{%&?von;|Ye>epznbU&MU2Y>iFy
z7TTWG&|LVnZ(H#rjibx$4rMzlZ7)ugv~Y8>u3-7HM#=wp%RH4=PubqoGl=eIx%iQ*
zdx7;Mzc;V*gjO~jPf~C{KXd2r+XtF>BND%#OG;#WyGFh^_~DL8a=Ba@6W#1p?%(P@
zVbt>T&bb|C&s<V`O;nz!IP3lU^>hAg(Q{=%2bv4jdH3WhZt`Y5VD@^uoQcK6n=Zbd
z+c#<2oO=|`rDNsA?sT}_*qmF*>-KkX)`rsml}uCptd^!{EP11l_U3?CSj|yK=jlpK
z9bS&{tNpeu4rFMXq|6ky>}e=NmCVTrK5MFKs`hwA7TpTm>7sn8c=t}V_clv&A6%Hf
z<@qivM^&$brxe5HpYwUP%16%fL!$9SyXZr9?jN*mcS)!=E!b?e;hC#fo`F-eXxC*U
zr|fdwCo)~8nL4kg6`h;qAj93wnd`u(5_G4kt;uI*mFhJir~c+G{U;x1?=_e(`zud(
zQuO2mb<L_HlO3Ww6**IidD4Qp-)&!M^m~H&PPP6+>h5zCB4Qa9IdGLtj@ceM`S-6a
z`A@c_xnGa2K6>=sQBJ4nY3XHFUdNu>s6P6toK(G>ciWBsJkylts-B!TsX)Q}`m-O~
zjfGC#49+sWrYEZ&n{?>r0$l;sT^gSPR~IWyUYqp&nyvZ)-puDFyBAtn<b^j_u38**
z@MeTpLeKFq=S`2qk{<ureW^al^m6$`7OA>KV+UzXCNW*zEenJ{nsb>X@|ZYsO;`}a
zen{NSk^9C%F*`ZSjZ>8`nQT^{qoGlAII;Gr_?*Myd!DK$<u7)4Rvs+8Wx*r&M4qrl
zQM<aNR|~~o9Tb=Oa<q8mi&oo>TumJ{9P01ZRC1Il|5+b(tMgOO<7J#lQWHPDT03z~
z-{ScCXRl{IJEfB(^Elv5K2z4-KYY`Zlh}3S%Fo%=>M?xbue%!K%dEZnKYJ6;!|fKc
zB-n&SX04CV`)6~Z_`-JUDczcB$CaCyS0wcB{_ug%dV=QPerZl$-T4zZn-;W3GQ2q`
zZj$)SbD`Xyg^$%G18wR>UNwrEB>I@H65Db>%<bSJ(R+Dm_D>s|ujU(lkB+@oDc>L%
z!L2h-M50~v-ZtUCj$d5&%lF-vpJM;>@4}#^3mV)0-VjXcR66vlQ(fuZfnVH@#JD)U
zy4GrJUS;~?>JIfyalbX@&bpwuVy{hXvinBc+x9`XL!9+ayX}ojTAH*dnN{#1qf%oN
z11F1!kCK9-Q!A&mRmlvOL(9U}$VKh(SeAIKMOwZrrTC-L$_Xllld@C~gdL7zirT95
zUpTOfWu}*kMxcP9XOoJB@Q*8;CC|^b2=cfkns%*lY~gm+3b<t8+2XtJ&DBdwT4UE%
zt#*m+Ra$V(gHdqNl_c|Jach?>-m+!&ijamgEkRP}L<N>=+*_3T^O@@HfP9e+&07<$
znxus!g}sr;zx!(a{PlJJ{`jtwvHbL;Wv|e;Y>l1s*n-NuX4%DVIwPjXxt-(26RRiF
zZg8rbY<W9jTOOmd-Xpb})`=h5_wIb-XQ8^Vknhw?^PaQ25=+|81xD*!7N|a?cg;8Z
z{Sn)R-+~^qZg_WPLr4|li>dZ1{c*L`jC1tPh39XqJG7DML%F`Ckht)N5@CNU)`<=R
zOd18V{5d(V{J-Fn8`Cn$S=5YWdZuKUNurZ@)R9vjo|aoWU6Xr_g0zzxw**>inO^bI
zo&7__k;(L4#N=M~yBQN0gj|jYB$Vb}^wajt+34A^#Wv)EtNkvwi%~&+Mqai|3KPRR
zzA0Smi`ly8D3`YT#GvU6+*38L&s^at*eaImwXsz!S;f+8#?}o?k+%O{oJ>!vjY?1T
z{Q6@NtE5J4Mz{90n-7n<c10~&p}noZ(XCTV_4=$FCY_f8v&1*DWqHil^*mt3l7H!U
z3!mlgdgXc}^Si*B73aLPvo2dMiC-(hX%OVT^wM>Kda=}y`#F|c3IPl}P7$yDq9<HZ
z^i*gyS<7%W?6rW$jVSK_51eyMO_DrPdYUeE$EscNbBj8#%`+)^g3`0*6xV}kEI|@%
z)5J>qHo4A@+W45;I&$sOMJy`}Z%kI(Heu2<1}5`moZ4YotylMpx+-=ucxDDoTi)`J
z)hJO(I&H@8)dp_dw*6Nxcc)C*JLl%<zX56o(#@{-Y))Tb`z12n>b}fPi71sT8H=w;
z>M~qfsu~h?ng8ZTA@yZ;+n?kx_1p@`-{1L5CZ9id?$+yDzM9=BSUyYluGksL^gFB8
zwVralbLr(TZMQioC9l`M^ZmZ6Zj+ixd~K6?OWeAhuXrsUcC9{MQP2Cd>z>D-pYG)z
zb8-w@eNU*(U~uFRz7jB5z)QfviAPH1|G`O;aX%JjuAM$*p{rIFPoDEMzS}`g%99jz
zyvw`3z6smcVi{;%+;{Pr=aL-5Q}36&&`z~=jyL@f$k}=#q}(Ip;ibTg$n(pNE#9!R
zcJ>z~L06%d%j4A&qYiMYC(WJ^q&~0W+e-D+pYLbRWcb%2qA`i(g9-DVj;UX^?3ps<
zNoje&0%xfeldSigQ_Nd1h2g}8S?w<E!Qu}^o=v!Hs^`O4taRndT7%F-VQ0@OUCvBg
zYqMg`i$7--WgD;ZXty7hxzrocx7y#mV#b7D3GVi}65j0<65EpnSxOW$gFJ;6u-wYw
zZV%qGz)dR1HUFw#lvHDwcASV9|G^Zslnuuvt{mrz{%^8OF6Ba#?ybZb`n`>^+Z=ir
zWF{{ub5Y=2o6*j4<5E}Eg=g-8nJ0Wj9_jQ6F4JjFREhX7Nqiz_LrO=W&s?d+qFXnt
zk=MO^=CsvG&smm7(wSbINw1q`Ja6SCrN0^vGGnfBX}V~uc<<)8By_3GXvaaZrHYJ?
zass)G?&vCR@9t&a9$*$&SkQ7}Chx7H%UgXsC#xv2%r0zNysgcGy-@p!+TENPa|`m8
zD?U_w&Lztve?0Pt{R>O~)w>Qzecchj@L2P3O+o3z7~gf)Q;d6`m?^Z*6Ii0@yQQP~
zj7sb6#10ivN6#%u?MK^eMBKQ2Pi#5n)OGu=Ud$06LCrb~A@Rx;|5cMFG6<<G;H)pu
z;S_p(eQw7QE}fOFy2%HEkGwKeufF1`n_A@QI;TQ#!M_*ouY);ex2a9HILoQ2*cD_u
z@09(YkN#7vbWUonUS-_<Vx?$n5T}G}QoGp>$LVrf%;Fv!X6a5@C^Gq$gImnQHqD#`
zJiH<Y7N6NAcs<3yDx&$B^O6SXv{hWiEpwf?o+eFSt+{RH$;>s}dpkp~U0}<T>EtWV
z>$tkg>6+C}u9GEJuLTo+zM5n5MAY^w=Ng4s?c82Acbz$YxoA#k6uqu=(p}3uYLdq}
ztDUk(TlRIW>v@yH^R2;?#VznS$C5(fq>a-VUI{J>%V-k$`eJd<zZHvm&ZL;F`(OCh
zm7{7`b!d^{jF_4FY_2m;EfF_1x_$Gk&(6&)fBg5U?92)aemP<Fy`HK6cAuyT<}|6l
zv1qlJp_I7U0<*1K7c~h#aFRUb(4<+gKs0p*6DLaod-#L|rW=J)>su;U3r+3Vz2w#F
z+!sR9!qzFwXWsO7g}#~fWXCe|D%C!QI_Ko|q3Vj~JYTOen;m)ek(c$H2YmOJ9A;IX
zU)J_L@XiB=-xKsW*3EfS7-Ro;v0Uvs^%H7B5Bsf->$IDkOmHj}n_k!<lDJS!dGqbN
z9ubcOY<gK31#dLzn{9MAir`dSQz52#MZHzP)}PCLSGW+zUuCEFh5c;Gn|tTIn6SlV
zYW>uk^Z(CdkcekAJ2LhB-i}&Z-dD!2d;F7c9O~BEVv<sHkay=CC&`#VHqJf2#8Mv|
zU^&+Hfai82kMs^vgVP1(UWEZ0wbG_KFH>NZ%wMH9FUvRC{)a%uYqsDEVSXEJqgPi>
zW107e@l)ne<yQsIKeRtg;pH#>y(2&RV*5cQ=Vwz~%{zTEoFp4pxbB>DbV9)vl^efa
zh=)H?oMN$BG`fRRa>)dSnMYF8Eix?vsva$FH-DnJJiE)U$;G49XQ|QjpEJ*=T@whZ
zyJb9QABU!U=Y5~zV^8iXK0W#R_XNMbB<A(ax31oK(8XiCmWNl5S<*rwF?@^TItj5|
zE%HGwiCmMFFNb_H$^ZXn>&}F8W-~rrxiXXU(9x{gkcj-k8(}iOqMO)e-<~5ay2~?4
zvHSPt;?@1uSO2}$iC3TZ?M@%}d(LzJYSg#BziE_pC1>u&5MeDJD_LRrLkr{=-d@N*
zb3toS`{_HWE4C@@TD&!2OMds143V5`veI@^LF?D0sF+!m9LX1%=zjf#&uaf#rxSM;
z{Qr}|!>PG*NB;Uy#uaB)%{F_m>~A7}n8(9unvJhn%nTGgcRu2Cvj2BGDyciHz=274
zm$Rf<!)%wAfAf^h$~b*{cBE+^G;3)Mc6JMWoEZMw@`~NIZS|&GS6!Q7ny$`iv9mev
z&VTk(SI(s0dMFpg$5>IL-f(bIp0eS8*6U?bf`vl-fx3&Q3cgsTyCo<(*rZ|iBVEab
zLh+XbTsMhWBpO&GS_!*1@jF(`3>6Mq$hp-Z{@2y`@2>HGo(lXoZC<*?Yt@q`y&v%o
zmjynv#;!JCi+dPnu{qv(NBpFNd_oF3y^|6h%oUejWJ|fgreVmMJE53!V)6Cutm14X
zvTcb>6H0<EYyCPH+sx{;&5-fzHqBrIMotg$Xk(|vOcA9QBf^Cvo<DC}GbuTvLNH}<
zJJSJ~tAZVqL$&U;Io}BMlNB`I-c;V1EZFNI^g^-orJCOkr_{+7oz0h1dKcH<V+m1y
zAz-X%y(KAsSCHb8hOWB@D<_02Ef9C(59HQh{Liq#y_=t@+ApwaMPWDF*D7}*hw!i6
zcMXdS)SJy;WJWy{VBeA%{ZVp@0#}j)|2}i=X^PPiHv}w?XNTTkb2bkzQwUEj;O1SJ
z&F`kUx-Co7k?VjP+o5IVHD62SGC2h^h(6xl7R_L9C|0(mv2NK%$I=^hGY<K*FLm1G
z)UjNN;m)$W9gFL^BefT_nr{s1tY2*Iyd+P&LyFm4QDb8Lu1Wb^l0ssVQJbISZwqkH
zYb@X|?7mst#K$Pgeo=YrCILl>!f%I~3@uugD;AynA!)tBQ}2ds*db4*2@+cbH1-}4
zeKSE|^#vRK!(6i*xR)9Bh+UMdPBP`2SX{8Q^~}RC$+Z7ryGwd^v4w|6c<=ZUSUy?%
zZn~ysT2A*=8Me>TDbsQ`Ow3`>EX|1!(l0aBG|RC_OHNLgJ8?xSrCcjmSoX6@8_SD{
zmF-eq+hpf7bT}UI4Z4<Fb=-GqX8CnRzrQVhb;l<xFSIgS6!72Bajmg8_tk)D?z$R6
zLJN0t&J7owf83g>Krcb0HkyIq;j`E+pOhm7Q}zn<e!I%OIzjc^k0~?Dyw*OMy>^Pg
zmH_oboUyYExK<a;`Rkx+x|nnI1Id2}xRM&?Dhf{B8(w|>DSsHVo3FBF(o*k}(=|#}
z`yyr}@|sMGcaf97I?HsKPj;oIdYHI<Q_i=H{@zfvZ<4vPN3^mc|F`o`lu?@E)aIeR
zuCT12c>>?Z84s?_Tz+tY<&z2Xi(Gxo{q`=3=JeF&OxIXz9Ho=wD)zA<%q8#FbU`Ig
zHl?GDFNC}uSff<~7pE<9=5Q8k43p&l82=$v(1b(kwE^3g5BAoav2LgMk~Z-1I`VHh
zz~^$4|H}q0b>^1U7bIf?1!Fgg#@uTD8^FD21J{=WOXe{yORAd8u+3OM+<0nXL4s;u
z_{w6>&#ivWH7>$J2ScXC2+Rs??vI!(v+ZIAr(yrYR?8FLsv25l8dt5zz0uZvv^{+K
z^fO1=RUft=+NjmIB6;3bv+fnz&B87&)BO88W!Dr&E?H_mmm%kyinc$Sj%(Qelm=tx
zb*}_wWTbA7TKGwcVX3pw2_ezLWl~FDRIFeMJS3#IaHZfArnXWiuAB=t4BxF;%4Z8s
z>R!IFdx3}0^;gPbg&9j%XjfmFvUP#c#R&et1@>PRxI{1Vhwjo}v%u)OC;w|Jt~UYv
z7k2WUnJMy*p~d6p(k%-(nHF${Kjr^oz_sYZ+!aoeDM4HhX9%1*ZnA=H<MkUG#1o6<
zglZEz9CMw8o-Pm1njsSuAbi^@q`=7OS^J6~lDSzg=CfucHE2yJHJ)Jc#OZ5#*n=?V
zsS-jnrmnVJlFMJ2^222!gT$6atdlt(NUbr*7wL!=EX`Lcl+x{T{dX-h@uI4pp|W7O
zN7A+bvVK!+qoq~Sg2Y50a%-_Wm<RJeH8u8@nj3Oy$yWoe#|cwH9?f?4T>8?BKY0V6
zt}?@-2}Z1|s~<%f`)V4_S!kbKF!$G0?lm8#u-}%Q8ps_Kz;-}2>*8^1g>9TQQKqrR
zoRhm&bY*#Lk5KS5m7X~{r}%{C2KQ3+Z7XtTBrO(-&{~$eeUnP+bGcQE%-b_9ZL$SJ
zzI7aEiz-?k<!a{tH6`*+h<x}nf9umK+?zzCC4~-G+46PF5`VZ>(A8eN(0a@DS*N;0
z=C^khJab6A$dw#mW!_a$^0{+YmXzQ`k@*XII1Sfr*}xUhB@%keD=}*6nhEQ&7}tL~
z#rIcX*;6g9#UK8!yXeXN`oTJep1J=H@Lhb#|9XPR)`kOaUK?J{>S5;I@$R<0X1eVB
zW$IfuNIm`8x;#+o%}s&44JQ3b;hY-{UhQC&wDvV+lUB{{7k8}qJyUp@uxlHu{q?rG
z)KAj?JU3mq7XB|Y!d`51cKZ>dl={s@3wNHLFkz|r^JNQ{JLSo((&jl)@!zoV=e3E8
z*tVLq2(gB!gs)z!lDN+?%^+1VQ~!oQ>_)EF8@Mj46kW{FJ71A)$urRuD=y>5Jwk_<
z#yk|YIJ(_`mZxKcarQy3IWGKb3=Wzuo}=U3qTlT0G%Ma^#<J-)#^v99O1x@Zlf6rR
zZQx7f-YQUg^XjbWs~w9UYp)CZzbkj=+BOx2mgfaR22PtF?p*r2YGco&+g_!G94n8P
znI1OGsnhIe;P9<BtuLB<<afDL_42Yqr)8gRS+&Z`zwxs`>t}%lt1K?=vDmR{&z2x7
zrim;HKLxI8XRX{JKhxb>Dp7Oof?2L6jvGDLE=LL7sycQhOK_*B5Nn}8(1-PFzSu81
zz{l*ZTFtcKeCUR?Px!kW`CnKqduGM;`oe{`-m0v3Pb67!X&yeH(|F<EV*812)K7U!
z9yMjXWgaxaXrq?lG_EJ1|EF1>TCB99$MMv}usc&vO}!*_My%g&lCY8G>8ID{*SqGN
zZj({3=+C|x5s(-WasAAqZxNF#9gRw~yCly^aQ&b7QGBz5%a+}*wK{&vTPGSwoz!)l
zwrKnHN|}fG(Gvw^B$Sr~2$(I@_EQNFS8tLimkJPM%{1I*ykni;Nl#`UN$*p92Q}F|
zJU0})-j)=#)b|!Y`<xSB7`z_clAOzO^1t!ag<Fbd%Vz1QPt|=cnXJvYR$#u)U)Mb)
zw=Rm;ZhRq>dw@&iV`btYwnV4f{e^l{gEY^pN6wZODplkRVz^TrsHT|h!6$yba0lz;
z3-f0xix(@lU-7Cddnr&eqhSrV%<3SSxMacGF1r)MXWZjy2o&=R+7@=fr{RoG>T&Iv
zKM%^8ZHW||HM8`1=Ze;|zZ<Jxo?gv2vEo2f<<#p|J=?8T1pe2($RzYE$6D6$+Mdf+
z`;4tEZpr@q6Q%eoO31)OZ-x5m152tei5eG+3T<BJwQ`E_;R9b67<sI;|5WM~oO<%)
ziPXa<c1+ia?~m5gtJToGc<WTKi)Ecq#U!D46AgR2%x??kdNkhNxbX7ifWzWXr1tFH
zR8nZ_#jr`?b~oR?s`HHMJN9h)_anST{YZV4v@=iL!)?;%(<dyQD!9IBbNJcRW6}+V
z+);OyZ01#pUhD86;l!3~H*KK`sW&yQOEWd1FS?5{UEg?T>+9DKu33ruPC6!WU2Xca
z>-9WELYvh!jg3!4u2ax`q#!QoT48Lra@o-rFZKAcEIXPRme~h6=xtg0YP0SC+goIp
zYAyEEW%e=@HdZiiPIt7QYNWpIqTrlK(UQV<nyprBwmHq0u2#sa{$>;Fq<~Um#yh)p
z&IGfUUY2c7c-%fGN&0xrnKc`I=l;5~BFW;xwpFjpR$s1}@a&1e!@#SnPQ6<cC4bGQ
z!T5^B=ef(X&Z)S03$1b9`@1hxQAp`>(VFNZg70%Q&%LV9JGke<xzk%S1bNo8EKWM@
zD_5PICdO3j@F%!>V&fwT|7{PyJ=zhzE+`;a$8nNMW!#L!%)g62KFpN;&#b@VhThtj
z0v>)o%R-etG)K7DvbM85<uZJ;XHLlX895(1B3i=KJojGl6m4(&ab%X!jN;@c_sTYZ
zJGW@6#s4U$EvNQ)FN|8f&&TMwuXbMOlRc~%f-|RIlWJ7|cH!FdBK6?B#VZ5_i?*^V
zS~vwX`uJX)6zji&gVkWolI=@bisrp5+9mXA#i3=7zDYh;Wn0AHHGSbtvk2p5B2y;L
z_nIM_^{+rovB>7lCc#+((vu9LoR!2+>2k6$-&qzfZYsFTrqRQ1-kX0j4ND%rDSkLD
zJ7T3z?%jlBLCZ3!?8B^gYGuyI$nNM~SbJIi^<AwPhNz&)DO<H>t~zyA>d29*=>mU`
z)c;7i&ogtMaJh;{x!_XC%Ec1t+@X*A&Hqhaer}nm&X(hqvnNbUK487ZK#+H1<D+QB
z?z;0DoGNQl1!5Qkf>i#i+%R%&WMJeJ&`Y@y;4~%duhg~dh>eSmbc<<n9q1@ne6&YV
zx-7-x(jw1(De-Tg&AoUI_sa)z{ovT7<`MrUVxrR4mFkjhv$k>E*tAk3ASj4MBlOmU
zl|q_cDk_2#Tn-&t9lBX*$|hGX$;B>5i$o2!uDqeiw9F%P(Y6FvJ>JJnOQNSg3}~rf
zJ@Rz7c%meu10(ac=o=prkMLNkZreRW@zMb%4^D<X6?&=1#Mb6zrOx1ee4fKupGE6d
z@!deCpyg`13w)i8)<y;991r+*bxnx&t97@eQ-oN<w);%okjh=~xU|yPM<{XA{;mtg
z-?BJr)lNh$4$Gb5@%KY%pYq}9GW!4L)f|7P&H7{VjFs1_emszjTCRO}QAz#Ql1<8{
zS!b3RHeJeI=qonqh?1s|kAV3$mW<a2Tc_>L;$GNu$*Fs(k&8#uGPMI)ijVIJB{a8s
zs7zca>>hH_MfCBb1N!@RJaAesp>LgID8u?-Nq1|-o!3TkyC1L^$en(%q(_nUL!rA~
z%@LlH2L+1WtZ$nm^5BHuI-SK+T6at-TBsxVsOZcYBkqQU%|cEZzAZu?3Z0kUyy8&|
z)0wr|r{&$2jwy5Bc&$AUcq1X#FX+aDoQ2=TCh2mvzPi=5(A{39gQt;0N~=?nH+aRx
z_E$?@ssy-7b<V4bs9Zlk^w@=#sk1+ne7)NFC()Jre_6uTP<6XnrVS2^Om~=^Jtuj2
zF~o@{om$PaFsf_Swi8)qPWM0EO6pm+>T6ba`cjY6j-0n%WpN4ayz$LI`s0$qh5IyS
zcW-FDvGwcSBWql?`z#Pkz8<?@AUou&nd#()Dc5h!@M$>jrmb`Cj6mJgt;b`Ud#8lG
ziCkgEnCG3KWjcNHi<@_s1s$+(oBH6;iiaiLQ`S9m`*>7u@`cBJa$#YKt@Gl%uI9}R
zb-CFm_Wy+<f7}A$*UQCx!xnn)GfMx&n)c6)@wNSiFP9eGNHXQTpvkC`xM~s0&MYs@
zwVjt-x;L6CHc!oVeo~;xXZs|qf%CxJLvM}ryS0xBZ?CwhzqidR-0i>Y^I4Y`S-gMy
z@k1Ni{!1r!ta;;dLi_KmPapO5i_e_UV^#Qkde<JFTkG7UZ9N*h&u<bAvlU%C<3Zc2
zH*fr$elOPE+NfULdtq)+{rYKPSrZ?fUBK@2Eot?Ql$=$owHK|J7xZ=U(s^}mAE(H^
zWl2aEUF<$l&g$)-mFmyyG=!FlFL|vMZ#(gv|4XI!M+H<vKA*~twC%cZxbJ*m-K`W6
z-%a1N4j5)QhB65xG@3Dr@fxON^Cd0VBymG@kwXB(-_7m%CW2G5BUeXdf4tE0<e0$j
zr`|13nWAF;xUmVjb$ZQDPIO(i#G+-w1wF3rh0hE&I|&x~#d}J4z52i+(kj`b=VWr`
z|A7!@p#^)M849d+*VUM|JnR9RqD=F(eN%#`K3KzPaln&De7O*#@0TXqrwcvLg>B^f
zel>?@v4Xs=%fYUN7n{ZGqC@Q?7IH>?VGd1hh+F@{QNCq`zwPJKC;qTh&U5(XEOK&o
zXYCu|6CX;Jh<!QWvGT^niSBwVR=*Nk>;EKA*(BHzA*^z(fot&s9z$10r_3&~h{yc(
z46No%3!k!VQDu!zbInuo6714xG8a{BanY}u*j=Qdyq|HMf&P_=v9qqMQ`bBh+9rKw
zV$2Lnqe?v=-jiwE(vuh->$)hibH*0NJzaVA;>(hXS3Yx{4jZ%`5Z0Oek|!XlP^0Ze
zljx?6N8juG-z3^F&9m#*qw66)I|aHP2v*%%yhOw8qkC1xdHF99M>Vo8o(%laESntB
z`rYFuf0W4DuzM<+9ybnm`J7;{s*UN82|u`bhN_pf-R0zqQLfX~p6S;!Y?`G~*DB5|
z)ZL~K#NDziLuY-)!YJ`1*R>Hr{E?=uTEV3OO?!-3W3&>vf(61IkBA=5J-MMxcZ)RF
z|4r*oJlL=#`{?CVF80+0YASQ`yUu32IlEk2`8l-0wMS5r&A;Ww=fur7WsgQ(?Bk6H
zH11C25MApu-EAh1N>FB>TGqsqbzhQ1lYT6AaS7~xUt5;3D5CiX3&UpDXH!C@Vh_t}
ze!5g%>CSM)^Za=+r|AEO&$OuMB({tF>#*xD*|bPzZL7-~wT5M3TY45PI9m0~u<fI4
zuA;W{30Lc#$NmR+>}LP8Qe{*0!qxvSmvDugy5Tm_{gqcr#-sq9B{vQ!@=2yMKiG7_
z@uiZp^h@2gYVB~2nV!?PuPR-WI-}d@;FHC<|1^qhyG~C`@R*@;v%qkk^9eIsb5$<=
zBOUXz-@oA4<abzOqoUoE$**1<65b!y!c#B0#+LD!X0(ErS#X9vcXrQ=l^Kk4XR*9o
z_3MXQeaM$Rx1?R|6Q<27RWPqH`8cm+!LKVn6wlV(_mq-swBDj2Qm#}|C9kS|>taWj
zQ@>Pc#)Q&iDbssQ>b8kGlo_15Jpb4K5*^l8_nj^&v93}&qP6SbW|2h>3zx;+tWLfl
zk*{)OYSW(77!J<YdyYtL5%`v>=g_0J@5U>iZX2r%zowo;I>F0WJQ~+?OMc~l8?<51
zGAqLrh8qd23Ek>pYecImMMNcw9v_vOWjLkRK_uei3B&epJ;E)?`rnL?KiafBEjf+T
z?aK}!hjZUfeQ;h<rOdfX^Dw*O6<3auJ$vQie<<JCx1znz(0WUpZKTrCx2@h8-}YLy
zUCF58$(;KpIXAn0_wCKPqT1_T&)+oBm)q{GeEHG^Ti2E9a&ZL-+&q@B>Q&Ej`F#mz
z9ZoCxF)s1cD(aT*v{^DeZrTYwXUR0@F3Tw?d5&)X+4fy@O3;|0QYbZxgY(iE$0TJ@
ztz%DTp1$-f=Y%SE`nuK!Z^Dmt`t9PlDXwDOCVIL1#kpNR%N&B1t7}erF}dwXgxS^`
zPQA>pG$Q7%d68CVHF@@lzN&!ri~Upkx8I!i)_Xs<jbY?X*SGu5YcOoPop`L{OjzLN
z(3B*R_tyF@?DlD89ggpNZg$*V%3cuK+xQ^*!%DWPRX;T}zSfJa?RQUz?ACBmI4}K6
z;AQU2jJB4o7tQB~b4JTY>=Bru=w}pjCpTXFmbtLO-`r)c1$!miINlhSmEH*OoxEI8
zO3lwf(=;n;yUU?%H#fX}Va2Q;C%Dnv#k}z9=CCh+O1ov2T+f;Czpwnt>Ng8L*Wcus
zsQdeRP3Ehv-M1fJd*Y<kF@fvWVX?0_>ZPL|{x?ftW%y&yG=-Pz;kKz6=I5IF{Dhdx
zUa*>E8Be>k_-s_G=c$>`v{a5b98hbLoY}aif!o4UV8^D18exsrUlruvADpl%fTJgp
z!^~4S=FyHe55eDDmWfq+cc1Lu^H6_hhV8+VJW4wik6cvDJhWH2X3o)oLq`;Qf(oYd
z7&|T8tk`0(NTWskqG!kY9kx%hI25bqUS)JX`Jl6vVa?Ty8C6kpH#>Gn+@5>yrB%#h
zr~9k9G#S>+ZrC)#LF32-qX$XGwk}I{S?u1O-CC>V;xuP+*oIaYAD8eAt!_C-T>i_n
z>MmIMb47bqvzGUljZr6BLL=;-JBdEET0eW!+P)ntfBkfRy;1Mw&H1sCtn(+xb^I1t
z__1BLgW=f*?le!6U#q%HB0Kh0?k%^R9(SZ?M~6_dij$>Dk0$5Bjb3xEH!J#h_n6;g
z>d52}T+Oj3%TDE!(=|_}n+;P{ZW)GIuaVf?8^+Mpd_ZFHA=fF4)^BbbvtQg1C(RtO
zY17n-ZFWn{a&I5^>FUc{?OynKTcPmwKezY)eJp#E!=Y8hfkA}9T!6v4fYo|KYv&q|
zt|=#qL>R&^oamV2;k9#3b+wjpRQu~)OWyz7?RaVNnWU9lFKw?{A$g(eq*~R<+BMzZ
zyj3DQnZ*BJk=Qk%L376XpNnRPGcKF-`)GX6!l|bfBYv(w(6vF<(#btoZO-ILb8cH2
zK3sNe#v$z*adX36T?)%DY4&WA_7+a=k-cqD`bx!Kkau~Y^Y%09H($sGbsV0jv(mRy
zfOEy>tyW8Z1<S=}u;1Ic!ug4CN25e=p{%RV8CRb(y)GTSIX>169w&Rw^ocZBSDcAf
zXw^;FrT%%|dqIX9k2w0Xrm8*mUg*dZe8t_PNo86Bt4;Co*WOKQcTIB*=HB4NlU6fz
zQ`Gd+FJ?Dy+?mp2U75(E?Zu#aqv_m@4JuxydxB<cn%r|nK%zQ{HAGw3R`PiG<b9ov
z$1m;P`rxJChUyMo30M37DI9r|*Z%N6pTD`k>CFBEf9x4W9NJPmI;Na(TjKHOjYn6?
z;noJ9wv01JW}Gm)acHV1|Ch_kqH}boNvQlhxt48hcik4FJBK=JTvn7WZufLDo58tP
zL&5g_k<$}qHt5b+FRe4VIMXdziA&xl@W4YBU+u}aoZY$`HYmny5dAb~?Hk)&2Le|w
zaY|paLDQz^qG!{`UAD`-=Vr%nG`<vmVWZT-Fhx1*w8@&~vXAF>t!zEH;PmZaomPb)
z-4CpHtL(3BIo_$T{eaDxFpV=NvB5`T&K#|AaW6UYug9aM<BU}-hxO6mlPMiGPlIh;
zJEAjK^)8&ACb{^lA<xVe0WYJ@mPg9X`p+S+Y;5#Aku@rTJ<33|>_Y!#uK5g=XLsdv
zJZ2H+>~dSGqAA=Y7;=>7Aji4elNB>+0yC#HsknATmP*)Up1UTn=)A&_S0z&~N*s6T
z+1gdfE3UJ5+k=xC8@FcsT>h@cF(Z)a;^n5<QT_$d=L>}y;x#<}t!8a$InS8mVZOot
z@0JsNQ#>YA`1Ja?^=x6)n{X{l`SO>*D;%Xu_O<kL99{BZh2)GW%9fi#>epVebke8|
z^-N3Vk56FjcIMG!ICZN<R7Qj$=~MS)>F%1wP}x%xv$dR3=j>Inbd$~HQ2(=2J~wE=
z%bg0j*BCx*co4AbeC0Wz)g0xg<hQtRERW=M%-s0s|7}Cpl*2R6_PpViT7AN3f`)6$
zuhWiq)DOp8zE>E0v?ti@jlaEXhxH7flXIe0uDxY@_SU3bXN)E^^{6a1NaeX)W8bwv
zW!A|MuiBMc_J;gj!M>Sy{-vJVY1zA{RA}9g4D(pooxgKhSH#|+ugiqju0P<Utu@h7
zWywy-)T{C-;&K~rJlwPO8<U-0=GFsQyq6EGwlcKaC1B_!Y-px2rK>P<0>hdsE03*r
z?I=^Y_FCJj?&F2m%JU90?9Ypw+`^#lRw%qWp*3eK>)*}o-~U|u`DMSA==JV39zAa^
zcQFKnSDffdxL4zJZ&$;;%TN7}Eb!^lVb#rOXS#bhQ@bZexpV5QB|QJ%UZ0V<Y8h8V
zKZ7nyu9@hWHLR?iv)8Qc)ZTtIRapDd)=d#Nc4po<%%HYohWNiv2X`~L?P_p4QgQLl
z?{4)Nr<=hWWFiyfuUeko*v-AULuPHa#F-tN8ia)&AG)~WYMaN#E)B6+xrQ$}d4fEq
zwCz}NjWsHRbAJzaL!p(`BJYls7PC!#qa>1}JbHqUFA4s=Cs^@X@an6VPxNfGS$d{*
z#x13F9V*j&RIWuS=W(dkC0S4Ph~I3cB`i8MV%~?BtUOCYyhR^-v)+hWsNyr__O@ty
z-FusK+qU_>2(YoV<p|ocjZ12ZaQA_^TaK*tI@&7!XVs+rTO-!KxEZW06BKNE<ixSY
z|C(Emu0FKcP@=)}NY@XyqpP;w%$=Y)>n?w(u>Zf5fUq?W0vJxc5(~c9RG=GPq<im)
zZ}@Q@spo$mAKq|YH$XCB^+{LXgM3#4jP2s<)~=gg65o`{-(<Rk{p{197E;qrcz!VE
zQSFe<h}+UG>(zOed#bb75}9Lba@>wKy7?(I=Pk2zO5^ZO?qFG*FmZ3fVT<SMZ6=#V
z&Yq_0E%Z8}+hO?()tePXK^qD-K20(Gmvh*x_^grS+$m=*7(5sxr#+O=3%j#p#-@T7
zEv}P21LLgD-dfpv=Jb&lYg*$DXC&>O5qH~zXTmYz9={`*l2N@4AwdT(7%poqO^7zE
z>;Kg_Z+rCPOaEKKuiB(dd6wjx#VVE(=6+FOPsEn>x48fPJ(!l#W9#;cElqs$rkxA5
z(j(7#RR+!}T-TAwV)*1%Xj{Rq-AtX0f`%7;pR7H%dcT&lN*1f9n){WxGZ<?pPu_Vq
zVBhKWUzAii`tm%FZ!35qR`-z2NWFavw`$*0dsCj5tenqllN9AL4~0!JQ(!jl*f`Pc
z&7nRIqZ>@3byuFQoM~zF_}sP-%Si`Pi{rm&=Nk3KXIlH}?elC&c<T|QAFm-6-91Ie
zD(uIl`#U7Frmf|epsdwWF;PTQO)nxaC(PjJj#pc}-pO6HJQd+|;-up70-;;84ax&$
zoVQ(Ez<jAI;W@`YHHkeELjS|=RK7bTkk-L#?R;$p&$@p-XM7W_XPPn`u#lKCrF-T3
z#tDl$+SXY9QD=R?;FZ>)r1wZG?$MgfK8!n9nb|WZT+3jwe{^VH@RP3@EbbiU67S}#
z^l+X(w_@#^|65qkZgJScdSm*jD>tido4h--^Xrk7X(2Y2_SFikDR1`$-(tv4G`YRz
z>|(ZsA+@6SeO}c>?mKb9a^<;*Fw+FbIJHN2oK6ZT36zPRP32HzDpJ~&_c_jwH@P`M
zc4OENS?AN)+YV%NREn*7aj9&H*2$)6ZzoUD?06$OUrp$P;`37G0|l+RijA5YjVJwh
zdSouFt~{9#cR%Tv`$vxGIcZnif<l7UglGLv{kYIJ;*Z!0$vOJBS!RjcbW94eOo_7e
zES@S;X`tG<v#j&ok+mJib%o#U^O|2QT7K+R&tpZc>8YDUM2u`qHMd%s%1E5d+iW&_
zt6BLJmKzObA2Ol^%fd4rnOS^0bfUn1YjfC+W-|pw&Ib)fCoY@KU^;YR{pmfeMmHFq
zEbBfUawKY7rc>8^m!~h2)xOMidp2iZ-+Au668*8+H(h168&BA<KJR(1Yz@zzidS77
z7Zu-^c2CILy@E$_ZR|6St%r6@J|D~Bzw)Yv`;*7jqRK`oI@6z+F`VhjXxG)hcIZds
z;fn8fZ%kkhWQaUl%;kUYSR9A>jLAxQ%z7EjR{M)z)tdJHuk$e1Xl;LT>)|sdp#omL
zj@d>TXHGw9*iz8SE78@x=9_qC5`!+!)2klQGp2>j49Gcs{AXeAD*f|w+QcUZ9IQIf
zE%AKct!rI>d!@H-aJ;`PV&M@Jp_vmIzI`mz+^{3zl%&+xrsdqqj*~vI`Dr%J$$$4-
zHAhV@wd3IRKpyLx1zzrQOcmEF?QSca>6%e-L!nprUyRV>xw;+s4aY5tW;Gb?XkE5H
zW7XVBHuD0m`AmE}j#+CkU)#@Y^q}liO0Awo^X-QXx*CjoXEwCwG;=C2UVH!dKl4A%
za@O4E4b#8>QPpGSyTS0}X@=^$j&8kg#eXi9@Xy+q`%x^ZGjM}zgeOz<|3!tjayqu0
zC|puzB6h39F79gR*9fIQ-_KrG44<l|eBAro0>9lJmlhrhop9mmq)k>02}j)+92u9K
z*rK&!L+EN>PA)Hxpa4!sPA?S)6%WBhF0FF5UOobW4#&IXbhRb~9lLgXnjYt-6wL#D
z+I-P>c4Qe}o+fRSo3nN1p+*Hsz8H_3hZjzrox;2H4#y0KV|>#&nNCf(X~4SN*WBps
zjZKqFyDvA{oK&k>`As=uTGXqRU#|!0Iqs7CeYdvr`ntPYvp@b)<@k{r&-m}x)~-*2
z$qB7qY92EUTsJjn_DbgqOsL%4u+VvDiP(YEZiY)Se!?uZk-E#L&pB1_N@(|jtcxtk
zkN@ixmzF;EWA*0zz<b9$zll>s!d*{+v6wT+W2vs6K~>50tI=`cvo~DT2wK8b_ua$I
zVRkc9RHc2sz%GWFYdEfDnZ{ZDy~;1zrBbZ@;a~snyHhT@Nt)br3=8t~4L!ydedA)A
zFK>~7t90&!#)-|s6;In(Qg#HjvB>;*GFiLK#5BRNe8(fU$Y{}$ZtX>mS7*qwo_%q#
z!_qbIw0rYU*6>=U9~u!8n*LrD>euBx8EyY>%i>lSRn7wuQ?d=e%@B!>iHypemKz!A
z7&FOD$}y?RB`x>sOwZ_<EH95e^Ui(uEl^zd;LpW#_2yPaGI?fvoaj(@F8qSSV_ug4
z>3S`bfQ798w|<h#@8a5!;kel6m5^kIP)UHc)wN50j&Du2sC!<zyZug->cXf1&!u@w
z8kkaAo?et?-FdOAfnm}CS4n-Y39AF$9=aS7QJHdyEy|~Gf>NZV?=e=%m<ErlENd6+
zP-qa&33PKxsR)&1&uwxQZm?R|aaq>pV#FLDuA7&-*xp|(eKIlY>}`Mj@~h6=*1tA6
zou9O|XOma$XRd<xeU~KP^y|-ZF<2UPN>k+0ihf_QnW?^lCXtRaQ*~l<7t~I>GQ;`i
zvI#R*6i!mFEjVQP+<oiDm<f4}S|;}$*S4l#*vz_>O~%n~y=zPYzo4$r%<_FpCdc12
ze08y!)AH6`_bEZT2Q(W0>z!h&<|@>RU+J-(MK#}rdqD$B^cQvQEzv2dmp9A!Y{-@l
z+v;?cE#^inTUW}FyIgn5e)J|jxb>|ixHY$DM`Mt&QmV)!Jtm3AKI<0D__l^=M^fc3
zma8}7b{TIndEHaKC8=91N5%5Pgufe{xNW)CDsgGA-XQcr>)#b&>w^i(dzaYuHLX1&
z5}qchHa$+RzQ%ijVgBq><;zyysxY}NmN9ov^7&%#wY5k7-tg>6eC-|Sy<GUKp61<4
zKflJ#+INmqHPT8me)pD%o=h<*b5?!#R$=u#;prKw?8)S|Bt?)@qalTBXJCuRfi=8W
z)`|!}aPogtwr+vSYL?)L&WJUOlsI<%XN)QhzR=i~)NZ7sr=I$uYe&$LNz$cxy84eq
z-{);<_5Jb4fb}3ZzikkAJ?8?kHx~qJ+!$?k_Be{HZaAG7rpQ+I<#60$f#&_oepoRI
ztZ8S@_DSq`GX3z6W?5m*1@SN31-c#Ej82`7b8MR=d0|GIPjtZ9%vs;}y!*JUvhL*B
z>>y!{t6u`mn?gFiM<_R3tH`&w^|`k|go)vgO_0j$gvahPo(VnBWwD;QDQYQ)5tnDI
zZt#7_{_+x$$-Z0N?N|h*Z%jJC8g%1PhV$o+o7XIMS{Eo9FKL#3cE^BGXv-p=Lg53t
z0Ug^nCM-2NxXGt0%ws3R&lHC#vlE}P1X!Lrn^OG0u~YGVMpCzL4x`@JC|2DgnOj3I
z9G3UH!lKLL+#lGX)YYDu{w5-@zcPy{?n^|gT7wEhnN~`<waf}}+eubBam(7jt!Vup
z5h?#r@VwD6>0r6#3ht&$+w9~oPujd>`7EP3nrZrzB|SAWgIagInaJYk;K(q6K~v#~
z@-xK~4vQY!F$#1A_FX#ZaL;(c7hNr@W~KA0icbZ-DlY}qZ<2Jfa&>8H(6Ukf8Mu-|
zV`9*aLvoJ;`~C)f6$&rpQ43h;$*C65wAq5SAok<z6Ef2$FP3q2F>#F9QRcK@{e!8x
zn!h-BuRj%%T6b*y9TslMEdkvXiX!Dn7mf%mKH(qrrCDs2!2jkqODx)|-3n!y7O-o(
zFh<s2Xx_9YqWwU`#w8cB%odg@%{cg}$u<5;G7sP8`&si^?4Nt6^-kWXdgD-)$)+A1
zQ<diAEQiTPk5{dkkQbGA(&my;&oRwiua!@!aU0#8x0Tc4)Y2@Wl*&t!lY~~ft8(2o
z^VAI5kSOBQecrcHU5HiZgqY>({$dWJ)&ElTv|}6sLtdQPW;UVES9{B|-T;wZXCf~b
zbIqLk!iIfU&brq{NB+!lx4Lf089l>o(rGnw@i$yQ^B5;vziHQ)q+w9M#<_EWXzQC4
zcJYR$GLgkEj@!9j?AGXN3i@@cd+(Hr3-85_%l}ho70Jm~f8vl<A+2%G{lCFC)uxQ6
z-kVhN+$yIp>#57JY2G6@VdM7L{+gW%9s&nuOrFr->9jVmO(?`$>}uTCSheMK6Q?*G
z;Z9$Wb!xtxV}Cr~4I!hdCrVy79Q;mA6FgSJa-mJ{_7;^M%fiBiEXrKZdXGs>^KQ>}
znK4iCnbq0rdgYQjYb|2qzfH2fd2)u_mVl)G;|sEGu65>l-SKFZrl_^ynn%Yru0A?R
z=ytzphm(8rNhPO)Y5}@Z$K<yL^c+_CCTP6viPFTG+oXI?NuJwr(QH1aZ>F2F)Ri}Z
z)yjK&ckpMvdv`FiYIorGlEZ5S7HGO;uuS~!9<pnBS#y=H(z+*=mz>l6@|pq`ns_*!
za{kZtwQ5~#Y<8=2<?}gZcRiVe8o1h)pG;t}m)h{>Z232<nE8@-7l}qM2$@jvg1=(c
zwQC>VxW6+A>_7PAvcc*J8~0@ye!d;)dt`a-D#z|Gi<|kP6?c4oH@}-XP)D<W`}Ca|
zQ)hQCYg*#zqma7J$t6l)y5;sIGXDhySaxjnj(TOYW<gv@gkgNso^%^?Rr#NGpYpA@
zKgn8S5Wj5JExvi~r6D;>I2flZv3A5R3+0;OplNhxp43Hw4XlASK3Pw$Pg|q?X~O+o
zk!taY(MLXQ`hMv2UcSR!b+@_|CY$y!%(FOuX>!f8Jz_=r8Q1&ew@p(pfA=Ug>Aq0>
zi&WEZuiEw9KAL9y_ioviD6K4|+h(Kx@!a+cth~ovo|P0#=SeNku~M9_uD&+<ud4at
zb&d&TU;7!<{pU{TjS>)G{Qvg1a{jVCYK`La8m`~WO4)I%@^(R-S&ov`>n?xoDGWc}
z^s-zuJyxqAFlPsc`O>p>XHM-p<I%R!rR}YI?-37Xp(AY@TiXiHb!JSKjZkQ;JtFI)
z&}(q7RWM?=h}d~fwhYyaiC+#|U3J>GK{)MZjAiS@ejU&CFDH0&i2P=~68q!u`!zzf
zf95->M7m_QO`G7EmcuyRATc%LsH^GuZ7&6H%s6>&Ps?|~BXbgBbI+Vq__&~?N77N@
z;Jh5isED8?3l{qAx%o>l^mZl7|GBD1_CFK}yx4wA^N8aHC+#cDTt+STCJHW?>6;L7
z<iA9uH%}ttR9@E^zPnGdgk163zQWa~Lds*~L-DlpJU5**GaO49j=g5>NxZ>reZc$V
zMg@&qto;w?#jwauREXw(r{J_U>7Yoo<Ab=^JM$Qx1n^0==~tYZ!V++pRW2{^cAl)f
zYV2*ns<v5ICd#rNn{L~0yoyz_plMgC;<vL4#oeZR@8H=R%jV5^;O2sa>2nhcbK3W-
zie6<o^{4gtwF>q_bA<#ATSPie2`KSy+3B)!h9Fb#@y9o(H*mBD6dc^8lF)j!m%-rZ
z-4`ymezYE+Ipw%vig!=nS`WuEos*}3a!Aa)^MAv|I7gO~j*PO7tjDx>xt!d~67wd}
z)*$9^1Md-q3oJ5o|H~+TI=RG2Lsq<?(|1;zyU+bgj?)=R4!W;oyB2!aug>LLWYSMb
zcaaH>)*+9ityCpz+$%h|sygqMoOw8V>D_r}7c%@gdHW}8$UPCKEr;&vy2wX`I$n4@
zcM8jq2aBT=Ca_#_Tcb12H^x<x!?o$Em$Z&+(^keU4yU;^Pp}j)?D@!Nf6L>k>!K2l
zX(#5+O}Ha5am6EtliVJ=THczrr9MiEy|U;~#xmVoi(*c++Wu&=z2Rg#C2jG7<9~V?
zEd<ZI2(&PjT>Z!LFo!RG%7Va<j59u`mR$PDdbx0RZUN60j_dzVUJwjS>%UrfGc`f>
z=h5h^q4AQc4)bM}MlF%Q!F%K&r^A8H@T(6*f(qLgD#+AKS!z=^-S(UiqnbmEo<Il>
zYlBUP<GNm7jkAF-*do`o)EPct@$zGrd*<^Zoh>MVt;mmKRqI33W9^v-Q+=QJcPyF5
z$eDAMN&5<u<`q84j6)I`jba{+ZCQ>YB9SgN8U|kOQGD}6c=%HsB(G$ioY8Rds1M_e
zlrt`t{{C4aLetLty257q?%0_%u0?t)B&v=`WHrW#wR8lk{t{scP}Q8hmhJ0_hy6Xr
zOJlm^J}o>k_u}CTY5z1>7|yD>6^MHHOueVu_CJ@^zU=}F$1|=@pL<ul{)c~XkxX!H
zS;NA%QU0-F@7$zCx^ongz6xw=TsHBH(q54xZujoZm(p_Vd6uy+VYx}ywKdOp6FWsb
z8jAz87-L(OKH~qUaD18CqU^es%_pBl1+;8g8W=Jo%WzX*L}GMC!<p?1kBLYL$xVz-
zNVr?`W#Y8hg|l90?8sES@#dsN$nyCg5`HnBFDrOj9&%QtR_M>k_+>{FTX#J!De=|K
zX{?NTblu}XwPV1op1w^D9o-)$`AvCQ=fN7<aAM{wc237uC#NN5Yuk=n&SFdAf7viY
zLNr9jOvAAz#Zj}xy+~!kmqP*@o!(A3q}J1@meVMf)7V~>AtZA1a8ziA(xOF`E_Okk
z{}t1Y{B`l{<m10ImCdIr{L_lo%!nzMO{W*`Ix<y|H6UoNJI7MfZEyP9dRcrfrA}$z
z=bCx0CwBi%Sv5(~aI@aCe{v>IV!A8nDp%#D@7I1pm9bOICDc}TrOVoXdt4YdcDT2+
zYIB@kH|J&W1lER)E_Ze`g$caW*W`9md$xSz#6%_Dh%Jp`8<-ZdwZ&dZN}F_V;Xh5i
zTdbMy6jy9u4ml^kB&(^|;}xT?7Ne}57t8(WVzCJv@gG8S5-LI$JPdP;XblZvdlq+Q
zvS<6J!1*V6rZZLE-rjh#eCh0Kjc1Fr!qp2LXY33S+2FW=>81tSvCAA@Aw@!t8rteN
zCCgjfE5d^Q>qNc|RART|o4lr{BW(g>y)VNf7go**jRuU9=eeEO?66+f<Z{afdFM~+
z>pf<4-<a-nq+@Yv--$r3jfwrd?;akIn%JZIqRq{*-OKT?Q%*<S3(p6~uXyRcU-ERh
zk(5u=Q7OT>Gc@=a?H?X}^2F^Xr^vY%yDSaNPQ961GWW34@#=)GOJ`zB8FgjfiO8;E
z_cc8`>+H=ao0m~8D=Qq1%Q;=Wly`Me?NmiM_i16AQL|W-^x71?4*0(ajkKH)k<(yu
zHsfx{t=@ujOIh3Iu45D0?h=2~)2G2zS7k-Uy|)YMo{4Dl#HwiBH+pqIj>Tf*vbLbC
zM_f@ZKO&b|wK4qQ_epyBKU*zobL*<Z7t&<rUs*Ei)zf)ezOU3S?Fsbxw=C;SAk*cH
z9g0HIA2q)2$tl!bzUCJz!<_qU?gsf1uggWc=1z1x8@GJ(BE^G=Ld%~lWMH}}pmm_O
zWfK$20gdTJbsMyw-C>;Lb(!;GOZS2k$Buq7+50+m!h(h{1-<H7&QWsL^jf4vxJ{-q
zMtV5jWplG-$!YP+cq^mRrkB%h<@wj@jS2q*r;w9J4t@0AqIz?}+0AR8beQE$v1$8I
zqvKl3a$JR<aW;?dvy?2KMke#N{JbewHm+el{zm!t6Yr9_-lfyu?2?@P%k|tDkq>X8
z7h8Q|yL|J)vB-;gs(y-Z9~-hr&XCj&T=Ty+XXQ*ocenHshLkNU?;LlFVBr)v5?3nq
zbdNwpNz}DZo8~@vDYV;UZO<0(+f6NVPd|E^s+n-~wu$RehhvO_0q)bDEpnTh)ihZx
zBjceP<5I6^$=lPM1CQM2yf2x3WV+RrCtWYRO>&C@a&tembzHkHd%N&io}OH}Pvr4p
z>7{Q9*1mXgi%ng^_2sEmUn9c%FMaKoDY|xbw#XG7yW5^s1?lRnQymkG%4&qA3$--W
z9W;x#Ec_e4&1##+k41$Co;Cy}?76gMch=?9;BEC<ERPEP9^DjQpOByKFX7mbpXq<&
zsm{^h11mSHxCVP1VBxu!vw!K$8`~H)Q_CfE4jo$mC+PG4{Pq>z5*Ow_O^(hh{Wopm
zzkAB@lU=sWD($*<+;6g5M4-r;e;zXmLdv|)n-+baW?1gML1gCM5TAvyzg`?E-4f1L
zyi`hyWm$tkwK`X&MVHg=#6pLqQ#QF<Y;)!g-%=#w?vYW_kZqD|zIN#$H7|~+Wm1w%
znORI1*)!yqs;uK)@8Bx^z{KzO(y$CyDc@JFJ5IIl=JMLn<yN5?rL-<+@<)mORZX)C
zlQj;6Hh<2xI+l1jE$sh!Elu`SEt#R+D`hSpcNVe8sdI^VY?xp6PH>&p$v+!c&&pY|
ztL>Z5<27M7KhN>peb#fqjvl7y?~2Cl>h+43PrPI2<_ZceI_u)~{+#0fsueT0umx6|
zJexM{tfuldZtL<X4%?F5%e)<=?X1dmH@=Np^TqS+@df_JmEH2coSPWnaggoEx17^u
z-uJfUR+r4Z{xgcfQZic9EZC~*m40;9mX!+I=V^RjUT=PESH+ZqFBaSeR~R>V|LAw!
z`-|cJ6YFSsf3sNkwQp?2RbO4=oRE0;s{$hv!@-#!BWq2T%1nw?^Hz0u-E-sbl#+&2
z=j>j;H}@+&RE}``T=X+uiD|<BnP<20FOru3zM_BkLd6MscRLfEb~ODgYPz|_@fMe#
z_|aQWu8C-rc2w`~Xy3hjTSd?A4HJIPzrph7-<pc#5|?e;D?*x&tStD-eg4wXBGdl%
zU;mv0TD*9Fu3RUW-??V({579BpQkEZ_gs7a*N==Wo9o{vl&{QQTDqfU?#YMSRvbT2
z#H)2O{K_JAv*0-M>ApFKSz2Qy)-N$K{OI+ZXG!LEkA>Wuvu~>`KK*U6dsXh^%BAhQ
zLMB%|d3AnX^RBD6_gwSeTkXl-v%2PJS~k;)5Qf>WJ}fYFOnB$MqHwa}d7&!rlTqfE
z7&cX$**g0{*2&XvSiBp~HMvhLN?o{PzPHMkN!O;#O-X&=Tz&DEPW6eFOL4#5bF3dQ
z-s744BRj0K=tA<-(>s#4sBB%)I@fOAleOu4o`1QO^JB~2)P2<!UJggLeXlMIDdByb
z|NOtlsx5z|E`P}O+xMUQw%r3$(>dRNq%tJuFzvj)yCqb$;n4Msx66BuT|QpY-Zn}5
z=d3NCQ(I>4Ox^D-BDDLZ$`{@9{$a^`_lDf?cE9o@@7MfFwW}gkXRinQu9%VYPgyyj
zXUS3LO)LwG{qE|xHZZVEH;OsobJHNPxrLKU%q!r*g@-LH0<0b@W;i&w^-5bN?a?@(
ze7slDwM!*ZDD_mIrf9>H8$qc))0I=NZK=Ghc4oSTv04Dbfuv)aUduQ%qC%duO!8>u
zn!Cx>%T3Q?dEUhnPqofZ3R}wcz^m=`fu2;0Nf$H1Ub}5F+kGU)+NkqtnCi`rl+fvq
zV|Q0Xo;11bmMk44`H?H*@^oeCpwj=(m&vqF;?X>i_TNh^!8>L3PR>bpY-R?kd(B;(
z*!E{lU}sBEdCtv~9Y429_vS@y)eLRpk<^pn@tdjO%JiY#ON2*Xud%{SRc3m3j-K(B
zV;8KfxjXozQcpbB*uHwo2g%Bl@3iJm*N?1CJwIvr#rG_dyY_ngxM6zid`3LOuWwwF
z3jAgTaxFFav{rM<j3lk4yk3XA9M?<TX=+|4cEpQ8Z^I*%wVO?@yjsh*`P2$sfo!HM
zVMc+|0!KxHXK@=a$_XbJh^N*VF5dS{ipL=6R~EZjZ%$^b_OB3`Alp6}rZatEcQlsr
z8nH2TZ*LLHda<xwZP%yW?@hQex9IK-Ilo1JFVFwU9-=3oaPBqU9PNL`*k4=toQ3fu
z28NzBT3(tq=27c*KV1~G(tP!*Q(W3y+CG5+>-p{mIG<uM=w9QIU!{Fmz;e?TWz7>A
zI*Y~3lb#v~Tjoxj8YcETWMl8EPazu*b1C;I2N^qgFW!~&RC7~U#KO>xJ99GI7nem%
ze6(v;(<Z;g$<KZK7}C~j`>8x>s#>xA{u1A{J90!7&RM+N!MxToQ~Or&xk-JY*Y3>j
zvNezWX|~EQ^3Som9CwS%wak`1d7%Bx>d&$4ZOeZ6b*(q9yuAAS(sLe8dW{T0zk*kG
zunKHgxtQ%tz`-8Yj0uZ5be}i{@HK^SI$Ii>gfvVNf9jFSDB$s*fm1Wd#PX<c;3JWx
z`#vlx6fvqe7RccDylc(0o+wSJOoi4x*Kdn_Oby<rv-V8UqW&P8w`ZqZ$q7DVx_;Ts
z><#T}#3N^)5L>%s`P;eEbGt2)U->V7WY(qJ<=S*=g~Ij)U0SZHF_Nxp_WkO*=&CBa
zTE$EK>I+?OL*JGAS2U~*TpY&bJ#~4g>88rhrvwk*ap_HaX7gf;61S<r>8dq*rusSs
zh2^f@o%btY)}7*8)<wQ;n%}0(dtF;Lbwk>$oim)A-tScL^0vOeVAZ@%<v;WMPgXwa
ztr2(aITY&G$YQYRc1*`>y$_FOTkuVCJg~jOz-gk6k7U7*>_uBu{9g2TUOWEZ?f*($
z29_1ZdP%|yd~&XB?){P@qP|J6$IHD-B5|3RyKsYcRHIlcuYq&q=GeDkO%vBPa#l%*
zD6)5p7|i0$Z_zx`$ufn#!G=>o%VXl905yjxogY`K1pX90ZuVh{L#9i9yOzkJh8a^`
z0z1O`tmZI?EO(ly;>gs0aAqeTvxNW4tgiK5u`dKJFPf>^CbEmEC+SAOswhRa^aVl<
zVt<V!PUouhtZg{^#N@KoxdVDpSAtL0t!p>i{IsR=T%`2gE3VcLp1!k@ObVBqx!Gfm
zgpTVEM}O}}pY=tQ-l&|;<m8(k=JBRLo6#%al+t51nFl^OH6_j$CagKMaoraG{P)sI
zUQuEH?WYB{38yT)`rEE#(~nFRM!^X;MdostiRGpUe_t$<d@W+NK@p3YsEz0JmXcXX
znH$t@@Z4rAe558SyJ<$juLWNZtvp`vsjoaU*@L0P$b9{rNo|Z-eMiiiw)fhowkhvE
z72MLaFYelr`JK*5ku!d19Fr9(H2WCfq(618O2z}1c%E3#fajV)8#m6J7u5MIOC@j@
zi?6<oWKY~4ju}@a&x$Vjz+dq1>e&ZVgUjCClD!h4q&BzUWQB>T?^mxI?tY8fvQ@uy
zdRB1Rnl7DO_G;F&l4)8?F3dSwb!E<h5J4kWzBg^$LJK1_7c=PxFY2DRIwsYpBEp4r
z>3Qw9w|&Fc9P0f3|EiGdDHq8UlO#>9r<<n#5@rxkQ4pWem8-`Za<#iCWEZQ|hKw1J
zb4@-M@3zP^Uat1@b&6*6%~L_e-gmP-zpoJ%Y%@q<lHqtOqGKX(*{YLAwLnPxN3+n1
z4I3g&mpt?@T`@6e*^ze1o3lg9w2$z7yq5oePUvDTt(Ds^tm;g7+!HCa_&{gTM1yHB
zWu7tx?|hXtWmlMqPZH1PqlbC|wYw{mJ)hp_TePuHWUcA-ibKI>Q)8Czy-+E(h2y0D
z(>>c`H5TQEeD?PEujZ{*bH;z~f*bM0n{p3mDHZGuQuer0nz`9Gnn7!&RMeWdtt_z`
zj*@0~68ZE4m8JEZ3zr=b>5e$^Uu*sES36YL>_U@rk`2QGT5QxBq=ZjxH!_@ZDl2pU
zqy<Y;MP{w`c%ZdNSJi<_RcUT&_O-hYt3sTQ&+%SwQ51LD@mjyz(#&bcJI`jbUEQ0&
zyxe<c%(Hx}P&+|p-TWIm0_M{h&oMQAo?{%t#5kkjFjGMz!<!r?-up8Ts~Rv$nmw4f
z(JPVLb3&8Q5(8(q?n`a|AD-ZO&v|&y(cbyZx8(hps#Yl2bvgNKWe5pI1x%GV70|2A
zxopCom7<B}8JoH;J~c855>6CRPfK*|*~D>a%MlNQxk0hEMvH^)9@Vsv?%<l{+7Ud{
z>e>X)+RTlYz68CtlunqmHf`^oJ*j!|&*T66cUo1N<Q`w#-nJ@i>7h`=KZ*SNt%KBd
zbndwu#H5$7M5Flf!=3xGyt*3w9mJI5eJ!r-bDmpT8x*L0!fdW$>a1A{>ZOakWUf!%
z;c)YeB-fjjF)sUAy4IDrtu9S6ofL7_LETmO22;cVrQC<9GS8-@mie=;h@9W@a&xd$
zZsvWR=a$nM&!_z;aC;NOWS#xvg_^*Zx$o~x<I8yR(r0<O)6?GPMn3QJmT&P`yl_*-
z%`T&<44X=F-%BppJZah%%YRF~8dm$)N{Iv=ZLL_xz5Ug$d$yc{8WT^P{c$Iv_0O)Y
zeR?N0d&ja}^ZJm@v$)uEm0Qf}BfV0h@7J>!ST5#eoAf^S<^RsLTyv(LKdbmaW%jLQ
zm)-x$T~cf;;m@?{oU~wz$wc4V=PqAhy*g{dy*R;ReeX_(CGxG2UM>5;Y3|+FKP&T}
z{4n_(uw(<5(La%+N>Kq<L<D-}f)5#3++6<QTm9#x#^oKy_Wivfm>u+_Ct<~LDb1UD
zfwKFR&KD^&lx!^)o<B)-_D)d~qZb9KDR;H)HmmsMJxe*HtdyjrvOpn8t7T2@X`7RK
z4Y=p*(|I<JSKXj{&Ij(ZW=UQmPE1uD<`w6*>`FNI<ji@qGv`;mJHP7PiwWn?r93L^
z@n&$D@KWo@Js0mS%a&Nw97=rEqP)a^yVP2%B?=7-nttu;o2T~7f5rMsUS}>H`F~E^
zNO#Ar_iAeI0uMcUd!#9FQnqs0%j56f<?FC^J>q!sPM<-&t2gHv&rOa~Vu!D-%2r;s
zW@e6((TN4`cV!3XG~O_3>Pu78O5mTMm1N<R$-T|NTWx~JF&*7L_A`1feU>a=x!|m2
zNmA4~UXvxK%yoDUxA`CL;=gw)^Cw4=`L*Nb5nPr(7AH!nF)%2IGCGJxeysnazT=eZ
z{HDV?Y!iQN(Z8Lbk(6{I@0gd$x#b;kE1j-s)wR5NyQS&M9rqnu7S%T~e<)yjtoFh6
zMnn386B{0!2-kCFShGg-*W;#nyn>F4+eIeKX48=2HJnrMDzl9%Dj|tk{lKa<Tq3@w
zpWE@AoALhyOYnsARts2uZseY#ci2ba)g*o8K7O{-4lib`;4wLK_D0UCB0qLf1qPu7
zep-{1^f!sBb}Lm1sIJn_n-jQw!6A-7&GuvGF6kU=kK*KdsIS}P5m4f+GKJ&O7J+)N
zUbXEi4Lxl$0uC+OrfvL+{{suNzMEILi&J-jZ*PL@lBBgWm6~~<9ol1YC`3wI?ZmNf
zT1l2Jfj7_Ss%=WmT=!{zR$rwA5A)$OPtO#6|8!c5vBys7jP_R7?>dVPJ(zfR#{y<E
z!Fx72??O1(N)J0#Fw3oNwfJ?oT5XeP_(W0RV$uKX97akEQpU;(!733SeFb<ORTQ{P
z7~V0sM9x!ouoITs`=4L^RMxSaHBQ?a9=3I;1~(nQ;$Ih(^k~He)2W5t0=%-mZRsUx
zQh`hi+duM}H$USj5UgK3^-gB<t~WZ#$7k6j-%@&AcFxdQVa+nR4IU+ZXSX%5RK0ut
z;0*60T^Fx_S59Zn{V23yeem*!q4J`23Z_iwMT&GEoVmw#=w6n=7P(D|H#qGLjYKtz
z?2RYM%$%&$Yb?R)@N0<zR};5p8<VA5=I@q6b3bu*@VNy1So!cqp2KskD^9up%}Thv
z*E-MKsBYr=#4fd1#b4D)bxBQ-t2xs(j}Yw>0-s(?2o`zF_c`E69m`XZ&S<ZXW{;Od
z{Mlf9{*Cs}?mJqB%4!aZ%U%h+{$FHgS@=StOF3#oP(aep867<9PB~S17;HaUZK2Gd
z_Q^X`kRw9SP+nNf^iZOVux~OGqsb$m^HY=~Hd>hki<vqy@R=wtI-vBz-N9VR!OhS_
zyLXM4$}tDGwH8g<_qwL%WY%q(5STKVVY`A<b%+{McCVcCf1dNbf8NckQps#fRoA&-
zY0FV{u8?8B+ii~y551K#<{DNWTU+iFCY9y=*mBu(+vLs<KH-gdFQ@piUcY?)j4@m5
z*TQ-09yz5bzU^fzUu;(xu=&$w(Vv%<v_I81UGBVfMY%yqbq$koq6bHF3eT;j2R}VA
z_Wd;PnBk5rAJfhi+FY(z`m&AfMZ#@N3cvjK)B3wk!*fdUt`$6=7a6i0-mUUVFRizz
zVXgioBi(sLMp<ilpDmI**Il~8$dcXDs*Ou`>!NSVKI>?jS*@O=o1dQ)|LD$xOS;ic
zdMi({ZPe1A8=|sgio(MqI~*q7bt&}zt;`XyQB*iMJ@#UK)i%S(KqF=2e+St7?v@x$
zJiz`pWP6WGu$y8GXNXOaph=*S>$y`VlI!NR2?nKqHtp<Lmz881*u*{Qz|~j%|FW3e
zPIm^gJ^uJ>k-BTT`fCp^r%VsAB*v{xq34S%bDIQb8;5RnT9vm>K)LCVo3`-FLhI89
zcujf=nT=S#PCVbXm38Vl-8NA+<#N0B^Q&e}Ra6p+`u$(HWTmLsaZTr2AuCF_W(1il
z`-rPDG?aR8GUgNM+QOjQ^ZDy7?`=7gl|N)=o=y*7dA?oYvGxiB$?uc?TO>XDq7~`(
zJnpNhs@tnj6|W4vQyCGdL6KXoT~Kn(ZxTr{DUCVnR(s@AgqH4-15Qm>HgOrWUbsG4
z<D<vrUM7tO&$(9)-}<cP`%IX<>8{&Gk=jM}2TJUZi)5TE@sTuEz86yZ_)3QQXElKv
zYH{XjUDJ%duVdftz}>`TA=2xduVlb_Rjpy8w%ijfo^$zPp*eDX+KZCZ*G~@$T_nYM
zp4U25dh#-{U0jh%CeKz<a+r};k;io=>T6!*q4u>?L#7*BRlHx;^<On<(ii4zQ<jIq
z=l<!i9o+n4(K?n3#kLdItC;n?VBvkCq^c-tWLKVCY|fyh8l+?x)Vus^{!zB3*2_`m
zO{MoMe0H=c>54ckT5yqNfy<<@H^-bgvse2YGfAA(&*{`yAj4WHGIK}9<qC6&j#n3U
zWwp!@_?BkjZoIF<^;JN|qXJiN)kS+cx_p+KJ@7L!Is7R?{h~vdN6_MfMqwU7{Wg0J
z^S@+OJIA{3kgwg2AgkMkbJcW`0>bLv7X5g!{YTeJh16+1^^4nn92Q~xVNe{FzPdD{
zdz*^P4CSV8vrL_4i|$a_WvP0=(mAH^QbGo|*+GZgZ2!}ds!?x#2P}EOCF)f2Kl=pV
z&Fu61X44H_Ln<cy&D(Wh+nHsX)KXFxRed(y7TEVp%qsoR(I_Q{4VlkkdOTE^O4mM%
z=qu>tUv&17@wpwv=l=M-IIx*zdf#~#qoVVh6#{43y0+-tTEr9cpu2Nj*xR+tX5#Go
znR^d@QNQ-1Z*8R7vmGi8QK}KsOD;|<jTBt9#))IEX~Vp2%BB-E(lpxqF1;~fie28c
z_KW0#9u0}o!%Gz<*O(_g+F_tpef<Lm$A_%P36_g^7kYnOk+F6=cj<=jF4<4Qc9aLV
zEUS)ODE3BCxhqXCYN2eHq_*X?F9tVsvLieGyKdZFWcB^D?DI*?77jn&6{UVW!*cvl
zYO0f0;r~VYsoSC=yf*(cXKZRPVzppo`Oy%o<a@NGFTT1l_|Wo~J4!=awjHgU?aDN#
zUX0;_^ZYFe9*dML=6#b0R$8-a+jNJv+9T6alMekkHOq8s61U%}e^EOsvvaQ}v=qNO
z)aY`q+k&-8;Feg|Z~L{!R&Z^ckZ^Tc{f@b&i;miD%L@uKe-z9j7*zPy`bEP#`RV6g
zEUQqcE>(WR^Ww;4<r7W^e0}Y*uPWx>D)ujox;|r;`wHKUrj3nKO`3m9I;?bCb~TF?
z918rdV!*^`5u~yxNkDY%>T5gavHdh=-l@i;<h1tDv1@s&Z;KZ2ZQ>4PYQ39u++S~9
zV94$b-k%sA$6Kpr6#Q4)kl!-%*o8%G+j?eYtEW08@}3F{__@TSna7A<h<}YwpTwbQ
zUeR45kxiAWG#G?UKW>^IqG~0b_Vk^nQ=qr&ZE63D24_uX1z$`Ix}E+#Yis!JlEjat
zs*~qPTKvni-o_cLa&L!W;qLQ4?`&d;VDeJQTaqvES7e)pg;?)y$NQ4Uyo3Xfo^Q#2
z^p_)4(r`+k<<p!<3$>8HYfh_d*ezXsc9EWeq;#!|v}y95<{2vSdCSzES!q05^l*2<
zO&>YSTRg9#U$pQoD-M<K&0FGIWuqzHZJKm`**ezPs`Ff{bR~-Ha)0%F-h57ePr7o<
z^~|=zflb?rZ>L&3x|3?%cyX?8?*C{th81%bGwoQdd@(~%ebFz!2_M*do06uStrb{u
zIcv_b#G)%6vX#L}$L+Tys#~5aOlo9Gj-T&c#m~R^gvX*i+){hRb$6NfeA*!PVukD`
zncp(IzZ!7NHRFz|@s~5?(cgcA#Xe{Ac8eo<`%<;d)4qy3{g`g?=evmSZoh!&t=tiZ
zZ|}&s9IJgx_F_8=<Aet<l@+*_#BrM_GD+-xdEd6FIm%U|_)_lcIg8abDo*#C{`KHY
z@vvZ9o20XLQ<cPQ>G?BzuCA?iY5Y`GxXj!(v@hdnM{r+ra??D4Lj`kp%xGO_?JTvw
z@@x2eotI{ZBC?f&%yP~P2`TQJ%j}g<od02S(f?Zqt>f>Uock`j>F~^dv2KcuB`F_2
z=`&s1d-18oy-M2-&;1?O=dP%Y3Y;>_*g84o1Dk(w;FKl1eupTvom%zwN}orFFt=>$
z#8S<=8S!zu59w@Z3S;^`U$S4_U-FvYyR><MzkgTqbYJRO#<8d9?UR(+-Zg*BwLL0X
zLPCP?t&EU68+GWr9?#j{ncd;DUKGUnO^P&kVN)>knW^B~B*f||q{1_yL9u~_fystN
zsX@t=QB;`YLWhIH@qRhRk|Q003@5u(8VmMx2&#B>>l((fSbkdKJ6k7z-kFt?pL;Se
zG<YN)?TQVY<hJ<5pPgG)US1wBpUHPoad(4nn7`VDBGKe)|5r$;bIdSsb-1}XVQ!wl
zfyTAbdSPpe1P#Qs)F+kaKie89zW&@k-{_@#J+nRphgoqayX5>>l5ntxb%J8#j9!K#
zxdvQzo>`k2qUW35)|%tc)U;vw@nzE%1U9s6Ss#3V&VfWOc0JV{Qxt>U%4go&?^~i7
z=vKkWsAe_A!fuaCk??Wr83#1xTAkFpZn#UsImSWqy&lUHiN4w1?6-wAG-hnDYhe1$
zBkMHh|AxBk$-SSmuSOPUp1l_Mb-H|=8Rv?r9-azflDbS+4t0sinWiqN6ME&M5N!VN
z`vuRKM3%&`XpN~Z?JBiTudsMcS$bI{x@1RdmwwezB{$=wE8+>wwMSY1OP2b!T`X7s
z>7nT6bnD8gw0XRl5tCFDm(5`4+PE+xO=4@Hg!Z(%DUs8BIWr|C*K9d-Dy3=S#Yo$&
zCKW-}DIN<Q9L+kv&Wf;Fy6nK*L|=#5xiWXkqzfC0)HBNOGge+Ml<-X#F3*&FwSqyV
zKv9!DnxkcrBbUm80B24X#Xx5vRsog((TD6UD>;6&O!8&%lbG!1am6HP&8i@aptS+M
zFS0@$R1LVA)kUwKQrm5HRm*de(rmGnd*V*4VNgkGnVKyjdgQ8Fc;wZEK@yr9r$m%2
za$Q`PeN)3>XUrDY>AqHzOs>aOYu=PdnSOHijlBu3%a$aBc%@C==U?<pGDBji<*om2
z>swug=3ddVX=IUpb2B~fz)!Up$=H|}nZ?Hxcg<lqtk@pQ-Z`m8_~ebdA#IWlNpqs}
zwiYgUUd6TH+r8>Jx0Zjd`S$J2=kurbNhSK|xkh}cW0*JPm4w9F!wR}wn}o72IESy*
zzUr0iE7ah*+*E$MS?!La9ZVS$TUde&-iB^rs^}IBz31gPq07%#aYAIp^d0MBUTaBS
zzgP0n;%50|-?GK~9VdnCc;t{>7PP-l_nSqsk5%21$4eaEMLbpEJf<2kIaN(%#@a2;
z25xUTE_kJFYFw?JyXEUly}agYav6^kL|@(QdwI(?z>z(3SHL-e+N>a#<+DlxuQPmO
zce%*Xk@EHbg-e@nZQX1kEC1la@=qmNp5jt#w_ey`{MF|6*KLRV4oOTqzq<7plfXew
zfvT;n+tla24NZQ)*CKZJTb6h9Gou4LKk&MH@9bv!snk&Mv(@QWmKV!GA%jyUp6bGD
z#98Mqnh~g@nvf_Y;px{t@#2YAv!_3K3wB&?eUR|nTJiM^9(S|3PdFM<EUtK-a7b%%
z`YMp3P%W9aLcy5tf)CS&gEOnk!sHn<^lRSzp1FF0LaRoGBe!DLp)0$}g!S@N?89G6
zip=F`Sh(b*<8030_P%RVlJ72w$d!q|dThtSUlU!plP?@fc@-|Os=<Rnu_-J#$Yk1{
z!V@~WPAYdpj<7Op)3)#bf93YOIYBDNLMKh?JN$sPUhE)O(i5-$mx2<RzF9J6@M)!4
z3C&a~R=cpIQ{`<&vxr&3?pt$R0-HT01q>IRcAd!Lv1f5eE=SW!6@`}Id9ys$Hz-UD
zI(fuCB>QUXu4{t&&sJr{-kD`Kw?ml4C6Q<Lg(kHln&%8%8hd*^uSq|(aI?_kX7H3?
z4qRZu6;x3m{5g7B@Rdk~S}E=tkxGtV9!6`ka-T%iZ|=OE7VNR9<@AKMg|m14auX_5
z7185$Nf4QT$Z|!+CQbRfFMQ>ay=J$VB!s;3anHTjC1JVT@=&9nkLx!XXT^3aGm-vN
zEzCyKr?RJQS$LC0sKEHgh36}*9;f~HbUAh9AcF$eEQMAT1rx5E-e94{6Pir#9zVZE
zkaxE4$D}U;4J^7Yid>!@3__C{<V%{S&ObYOvGw9bg3qR2bSal{R@!=Cs!GB~72Q>j
z6mBnD)HsQyL1&Zn?rTc}54&87sb8qk=hnXC)T%QEf^S~z|JAC(qomn8@p1Tb3vs6n
z1)5Wnl{%-eI3K#Qc3qIe<=BNK#=8^wR4Qg)no{#b_(GS<`I)Of>0V0<+PgI7gu~^S
zERStgVH+F_HFO&vc_<mOEsmHj`dWaYR8=r0?BnzKa+Prvk5|MjomH;x_ckuz+Hcz*
zu7cr`O!KQxdQ4p^GDRih^(nCrRuToh+zn5xuJ2O&|4Pr(@LC+-5&=(X;UM3<4+jH2
z@+94OaPV}<kHT3C!_|7f%nZs&WSVd}^;pHv!#$E+Hu}!}XKoyx=OY^0>9Mrm<N2)+
zO}B*xpVl1J(hKHZtf0oV*n_Ees)?d`p8h_MXUp6;Ox!Z2eNT@*`%K!L<-~^2nbB$0
zy%U5a*YasLx;%=K=v{U}ao5ofy`D9n{wZ)zJEpgE;=4mC6B)c!{!LrBcw%QyTB!I#
zZm)>u0IsEvZ8TMT3Qig=ukq5d-yLPL($je3nh;IB-^$Wo-Db%Do6E#!v?ry>Yv!rs
zu7+o~LQ@^5XC>8g$^5L`8rv$kM)cw#wLG3RlQ$^)3SMC8Tom*8zv97(CqLC1h3FM6
zoS7ry;KRo7`$p;htglmDA}2N_&KA<!bYgYv(<1KJE2h(TRao0|E_*SFW$v^SHk!@6
zX<C!LGGCmIIaxgSbV75~#S`}f*4VhtiR|vo3Q;?@<K$kx9S7<sG5EH07_L5`=aw;X
zilL8m`Fgoc7anPC32dI4UG!w@k%JOj+K&j=OC(BJ%seK#&+9;c_Tx=kc(qO#Ih*7&
zEDPM`f9Wmr@18u<^x~+NC4q8{t)BIKVhdX1E-ft2n_Q!3<+*0d6f+Hud(Dw0yVr(T
zH7(UUJljFq(DLlc9}Ql13)?zB2_;N?Ciz?W(bgTFU8+KF58g6xC=`{g4~kju^?&Kx
z8=u!*I%Sp3*{8%cJ0bdBZ}g;I{uA?``~Gpu4tk@)zFYOYbCZ)&fm!xP(}{kXE4C~Z
zVB8q`^P$y8{d-CGX5>V>Hr$#vWj1@?bo=*`O&;pIF516LD9WC6NO+yWZvE*8!&h}K
zO?go6oUli+<3;SpHSt{QzkbfrG;2HMkawZa@DSgMnZ_q14pw{%)jgiQF8Ia+@%Y~h
zm_EF%|NpkE{k8FVGbL@Who6FOycgTBqt)i!_M9-mo%c=z>_2dAy2?j(kC=@`O;5}o
zIeD~C_2{~qmZg^0dpO?Av1-Y8Z|g_0`_zNpY!h)zcl4R$_-j&zcc6vVRm;_)QhgWs
z4lfcs8xr^5E>TM}G38}T&6C5m&!!4rQL$U&X1e-Yn7Fv2h<n|H=?oEqwPpwTk`*|U
z7jT}QE+&><e&AU}dw86ySnKiQI^s%|oD*9!n$4mU3VyeWCfvxeT4)wnXn)H|_vhAl
zw(AWG*c13o8!d&>pKfAh5sRE4nwPJpzxIRI`t-c^MEy@@`rhW%%@P@&=27Mc+fzQ&
zpEOT4O04LeA+OCCnk>M%_<`f%16&b>x}TZdy<cP(E%i0u;l~sd9<P#mXM0)bWhoB@
zhBxZDx<cI_!|P`+2v^?Dni9~R91)q);q~>q=zDjw2GRN>9Evqt(|A41d5R;LJ}RZV
zr9UgOs0&WD2on3x79u+7h=b*IY2Knz8PV#RXOdFK5-ZC(CT^CPdO7fsgp=frJafg;
z>A_7ZQxaP*hIwzac_bdU{g4~eqOOv1+o$Z!Cz@CU8$>-A3c`L=1iz?tQ;z#4WOshC
z&@2YOSPj2eMbUs8{%eA=Bxa_3c$#vjxvXfq(8^7kQ5lsV#oM$v{ls^Q2v|t$^$_xz
zT>qoEN4d#9oulfXYSsVGMIug&%x>cKo2z`LCRA(m{d_3tKPhipx?@YDe%lU_ozMGZ
z()!E`t<tY>EjiFJvBR3nkZtjTsX5G@W!I-}G7{vPSkH98B+a;M^$Ouz?ut_E)9)@<
zS$nWfoxT3^_UZdqgztPY{r^|>$tPY+GL$TumMW38xLZz$^>kQj{beQjAhVk*l=2Q6
zJZ!40RPJq9T->~Hrnw_)aaqKIVDURIWILy+c&sd$bJe2JBFJi^Rf~sb>qRcRCe^u#
zR{d>OQJRczTWhR2#OA*+y%*{m<iPc&KynR(nM`=$!XKdl5_3KXPvlgnjQA04Qju~$
zblU9{iNc$)s)_wqr?d(lnDMdQHG-v0B(nRJrb=t3@Y|DBg^Wd)j?IV@oZ0+SWLl(0
zT4L_wN%HN@dfi87+B;7PNaFT$%JY`Y;A$%IIVrjE=9I^2lEPvcLRB3z6{Do5b+{jz
z8hg_?Rdm`z53aH+3j<wJtv<`&_FVM8nNdh$Is?Z70d@B7BRk?BxUlWm!MSrrH?P)=
zs?-^s#lo$fi{I^>=5oupTBRYVp{VNfjJ}&9sf|igI2+Om&4UA$URa^ToKoyu)SFl~
zU;k&{G#9pe8XZo{)7P!^oWop_KC5rlRaLW%89v9o+>=U~CRk~Ba0z{gQ7o)GS!8l|
zbIOksi;laO2UT=QeDqyfD(SY6bNitQ&97Vpmp2!;#PR(UHgU|667m&%)So|HYkg46
zx;8~2c9A7ct8$kLsdt(^N@+`&v3Riq+j^<h8>&1d3KPZEX8ig%&$Vmy?o}$6nH1Mg
zYzq^dS)V+=No36r)wFyK{p^WLnYXY?OfBv`A^Klhqx#dOSq<A3`x!b4YgwvH>^x}H
zx1nX~=WARtTqaN3tr-{^lNPPkmg>~VsxzCIRpT|i_wYpNK&iLK!>cM6TXrpGnB9H<
zmT}69bqeZB9JF*<R5vJuFX`I3n)USRtm4I*=1PCQic0(zNngmaC_TXRka(t8Z{aeP
zJh4sjvqa5S2@5%JZ4>a`l)kK8L(QDA$x3kf)3l(#%Dx*>UTKT0A|I)QnlCHWQrqJx
zw06eaQ%+fS!l}ZI40T^ut(t6=cxi*$!R)&~W6y>w&Uv+B%_5<1w^sTWHt+gmJI`u@
z+RoOen)Pd~xJ*xNvyfVCRk+$LakWn7YVpV!+h48Q`%A>fSn|K1Hbc(qc}2Uow^iCR
zMVr-5(G6Y3*nPXf+tIvt=bE2}fyIXHcdnVUJ!{~dA~ugx)YE}&twBH6uXg`PNA_Qi
zOMWkCOcZ`=(IJuLHE+t)wF~BGX-)q&k>&eguDJ_$r$p}Fw|e4>rRzgg7T08U|9iE*
z{ih0#cYtoE?zz)@%r9=(|Js<}+N@VoKls2#?xSY=gvI+@dZWA)*;y<mhKPkRCS2!`
zu$vz5WUb=yv$0WQUEMOr3C=;>Qfd)O43ZwJZFgHONsx@PTyB1mZ|eoFN-YNW-y&0I
z)&0HFlBg|wU0G4kGmJ|yd*_KQ&zG0~<5+d`<%DC~{7*cxP3;!eyeRDFy6XScmR5tV
zN!2SSt*NsA#lEBU@wVdV!!sndTg0qx*nQZ_W{2YQZB@T_D~tQFm73iss*lrJy8fVO
zage#4rNqs)Dv5)+GjDg8pA7O@y4Hd_(LYdfdsm+CB8exftkyDAUFM!+oxGx#ac<nt
zt)CwGaR@9{Hd?$nV~=Ik;<}%E_WoL$-(}CU+oJXJwsp5e?ao*%ylKI{TKJOp4gu|r
z7r8wq9~Av}-8eAIgG*zUqIO=e!vej`_C=hc0zZ~!O72?0Z27cenN-B)1tA?~j+?&<
z?VmMqzxTn-n`cVqKG^Rb#GO1#&Hd2vNnEy{Pc^?kb>Q~#isc$YCyVPI9u~a4`OM!b
zCc(^N|F5!SKPr{KAY=E}ac+9HZtRUi`fKJHoLSs{^Q4I9lG4>?t#?GqRv&(I=CJ?7
zbLNIS-f~W}yLG}o=6wC`X`*J6r>3eM?rMu)$l9}7Uggl_KQ|?WRNKvpc1AV^&bmFb
z^o7VTD@pcZ;pN(nmaBJ3o^_0UGsQ2^OYn;Rd}pqG&%K^z9ZSD6QS)q-h~l1-oa2n$
zi%)RxbvS(7ZqkYVm=jY<Pxx*;q223caWlM^+2PJ?bDrd-d4~Hg|JcEp<<WiRO4vrt
zutfgV-KVZqm4yGTatdGDQ6U+<Ncd0m>5pZnw*~MkU*(HbtdzeqM>MEVK1TZj^E^=_
zMG2jiXEN*JcGsp{U-bV+$AP+9;YsZF*~^6@JSsDoyR|tt<XW!|IDKQosl!#$N8YWR
zr}TY;^`3L@ctk(!IXZLhhWk+)Ui+MH;$E^)_SWth8``{e!zRWDCc3MtGxVH3^6O@R
z?M(Tb+r-YU%C!nQ%J_ZJ(v7U*8Nz%*1+@{?ic7D`=-zp1waEVN9f9p4>n#><iCEmU
zhjWPoSF_A{=Gzi>yuED?k8^2D-uSet$!Fo#S*nX0?#<tF^O5YVyK7H;_<O=<>T!mQ
zz0##8|5geYJZtD@J(W3eqo1H>E#qRH*S${BmVb+8t-opfN!xPD<z)#{Dw{JqLfj9e
z`3mj16J)?B%4uVAu|`Pt-tuQ(vk$)U`~T8vz3jVCdA~!;XB=ppdr;zH*V-AYG*7Nd
z$W*iFyK8lC^WqeV#@9E!8*kdho_Nu7?p@E#H*@7Z=gEJ5bHZWX`JX&DAKg`~Rbiah
zF35b>AnKgm#N7t}KA-fzBXW~XqU6L(-oHyLBrYB}Sr^uLRF0!>%k#AhpS1hW)L)!^
z@%f7VUxOBQvRTemywjh^{r>Hpmb^pjU({{XxNJ4+-jW5o*G%Z_t-EXfXTPo7^H+QB
zp8PKQK(?vTO!8aG%lYRl7TTqA?z+syU9_*}*p3zELJqsX1&J+FPwOZVlD9s}uD15_
z57ESFuYBKjMPGd8*Ldf))SFqNj*T4>?uWQ$UfdOuskVL1)c*@sPu;G=kdd@JrRSRY
z#%mM8yGk+@rb?~9F-73nRjD}dvUegkmc6)cbgs+o_Pg^_-mZMK(s<vYm>t_J>)u<h
zJNl{U@LL|S-)|0k)*b#cuVzNA&=2KX%IW8x?K$ta?)>+h^L+0=*voz7UM;~KQ8~$S
zQCW(+w~mKI;~Kut@t-b<?X2A?IyFE!deKeu;|r|&_le(eVBW02y0GHh;bU^g4B{QD
zS4eI(cp)oQ|J-X4XTptQoBHlp|G9!M8RvXCaqmT#=JWn{3m!?l4CK!UjF;G+^=kS(
zV~f9wAMd?8b>hW#v8mpXAHoGsq)Oa3dw6+I-F)+pOO5_2UUCyX<EuC&bKjlaJ^#1x
zJ>xxb(*NT9y!Sga@4KcicAu}f)m-{b=%nY}4@)Fxo$_CA^{XgKV4cOx$NLu=H|>_$
z+HEmC*sR&9I7^l1#^smMYK&h@E_QLR`Ni?ZHAQ{>wamFH3tOLPX1A?=9JDEFo2Htd
zx*Efp=OLDFqJn;APF(P-FmZ{%5j&fEZgoQ5@9fh2KYmlX`L#wU;NRo5Rqo4Ia5DQU
z#ZFsu;geM9{Q#x>+g!0b0}|72{t(HM+ji)1xURm0Pvxn=o1O)A&mY`K;P&dP^83%A
z=-?n7#Kg;`a`n{$hG?d&t(&%HK6L34)s5T3;lj|sA~QMTVn?8gYcB_j%EYFiQ(~IQ
zW)qc;YW;tBCd?$0D_}*zlXI<B)sI{}4Aq=wt`XSuP;<f=jb@+nyn`Bn3tUz*sq`r+
zH;7ydjAkuK<@)ye+M2k7dS6$CIz+A!UG3MxHT_X^bD{e!->L%cnZ7|R>wJ_H*PqZ7
zQ|Fko$??E6%^p^*wo|Da9Zxpv^0#$`J2a_qF7cQ!>1x>S)5^VECRMYNUYRvVYr5>3
z%)8QTy=Q7@M6`n~;}XxuIqqtm2UtXW#pY*RTH?rXgttg+N6lv!6($p<NIn+FVwDrD
zOdtDfmK^*zcd|<dQ&jgd-PtGZ`2F}*YW?%e)BXQ4H;Asj^M`dq($d~5|BOAhwkB=n
zaAaYP+OVZ{&Mz&e#{UX?bk=a_9=YJitG*~;fg#I}3rhtYSuU;=^w$w|6?K<9>>?EX
z<-*a<m{VQ`j7vHaEOgaw6)Np7`=Wh9TGQ%{0V9{Tk24=jM39mC+gVATjXMf1EjlHW
z<G0|Q&;$i8O{e6eV)JWmtz59=j7ou$iMMjF?iDVTrDCSKO&wD>%=U0Fa<0-oyL;k<
z6NXFmZ%2DAoAs^Q_e{H3tH$#7f>lQkn5c*PUfOdkC3ycrHK`Ad{=89}l{l>DoKo1u
zSCp^&{njro&COg!T5GrH#avz)HTzW7!eH;qJpQq^CK;NHiv%*d57fv8F{tu+72Vw5
z_a>C#<f6SoH<Mm3;)&j=_sD1O)c*><eD`-oX$L426(0Jsz>L?bJa$e<@%CVrVq1e1
zVNu<>ikB*kJ<f&QN=)7{VZ!RXgjWI*?=71J>@*u2H@mDeel{=LZ+D}q)%nv?&2BQS
zp7CS$`>j!LesA|J7gzFC{LONS)4rs4*Y_=dPaV~`+sfeLb%~8XsK6k|P-^O;1-!pD
zXf9s6cEid=o^vK%Sz22=D`REj-wFDSTRyzL(VQdVz}#H*CyHU6zLwBN&Pg_#l#Z%J
z-CnSD`-DliPOiJV?N*n`<0C3|!VD%{Yc|R?ix}QD;k|N|<4uUhp~*%8ii@kR?OX&}
z_p2EF#-(bZ&gV-V-cutM6^b|uC$$E;N>AOaWcfebe1+oQu>91omz7-id}Gu-TeE1+
zk*6U=oRJ$#b+_|znQ8^E-Sq29%+4xP=e12@uXQ%gi=TJz@ylnSx9>bH>c4Sx(TR&u
z`_%%HxK#O^Z!7E<%FfBlyuUu^)LZ@TX*)h2`Iob_>5ZdQ+^oi39*>xo=oLSGYP)r%
z=d6;$`^%R2DSXX1TCnAbOjpS9CZ-(<^LY-bx7E5^satYpux;%;u$MjXi{r!nZr|45
z`=VC*@YHE;ULn2vJx5X)uJrFdceIW1-*JZ>k58U6GT-7~spTv-YksQ6788-c-b@9h
zhuuLcuA;tz5x%o0R+}j@>TY<VBKSh^hMtm&0&ivxw`TOMOXd>)kLq~V`A*`Vx+=vy
zUQwO#!p~dBSd4Zoc)${->mB^6Mo3E`ilzMMZjPXuAWfI3Q>CR=%9n%$G&B=(xvLBL
zeZAD>4NHy6RwUm{jd9<3EzrfUYf}evsN{kFvP&kEJc>Qrb-ZnU;*l!8s_v^HU#GtE
zJrTP2N2<YYrG^_U?=$$dyoILDJQ@=l#8n=n_Dk&1QI(rZ>rS@_xvujR{4DdtJ7$X<
z&&jLHyky@6>CBFN&sHS0gkz)Q`-7g@CniY!yplY{IOnLb>5lVloGBJ}Hu@blVw?3%
zfXhymJMzsNV<YibeM-D<raL599G1Knyoi79LoLHB<IbWG`*xG-+PvS@m^l9zDyi-D
z(DlBos3cUVxc$yN?=+W*D~?WCCL1N1v1U>0xgSp>>p$M|KdROs%gL5wC;Q6fQpb`m
z2~P&Ut=)GYsV&V4)z*vI@NCMqmv)Z@(kBVEbg6#5p>as`NeXx7l7|92!zNnv%}`AZ
zkLkDEDHhZx>an%MGeBs@)SMhm7RxK@{?AYP8L3R=bA00{wJGB8BA<=ob5FQv9=c(>
z>UH3Jktb(Qh5Ai=^m2`u<kie&+Z7Wz-&aI!IpuPrN$gpxYuLJ6@6VGm_XNeJtUkS`
zqx#PypVtpW{j+s=el;DDc(m@K!l%W`d8S8ZvQ+phEC}>htNt7rFS@B+)RK3fj??;^
z8yM~XM`rrwW+n1>uQSM2c=DPf`iq?0wdE-qF^SAiPh0g|P~XIKF(c{g>jMX;l<zm)
zvm)iSl=rJoEDrs%gX02biBJ92=PW)cpk=D{!el?*F29sV-4QHD1zbP6NatPbGU`!!
zJ#m7j^{yvrDl6>dHbxkv$YiNpv}oPDbgfBOrt;)vA(C3I$7)ZCp3GG|Ca7m=-miRw
zQ=aq3{&Ptz6Kx8oM+XK^2sqj~r8<XIa*Mz&=cDNxntEfCi>Jq@aL!Pln(Q&-<8Fb_
zX~l<{8RJ^3zo(=`_J<zWbLwJ?sn3!(Kg0Ov3HYonjeZ+hVUTh|*L$Ak<GCFZk3I{Y
zW+J$G#@ASuT!(xA1Ge<+p5PnK(jR?TzT|7WNSoCZrCpL8F~Pm(CiHoytxLHO?VmNF
zMzmnwe@)5!AEr{Lbw&2=lPhp#@p105dSbrxo$u3(2=2PjBVxPtmbSfW%#rw1?wn?K
zVyB){EX(tC2OTDVc8s!0oMu!QZsJ=0tjkTS;#X_3*0+@^RW0G`_ExNj*!;b?^UIax
zmv&svD!%Qcp&+0Ya8)Vt?lq^1o)0fHa;(aqL`CE;*{GkW8@{4@Zq}t=YOhzdu2nu6
zsr|bnY_7+)4Pv$(TlJfY{zj$UILc>srjs}FxL3&(b8%xn!|sS*N#Rd?-NMo*dfdLk
zcKU@^p;u>pYSg+9MpbDGMRWc~T2He+-})k7?u@t4TX6=1T_$HYO%zd6Xw{rhTPvNx
zGs&cQsaB$F^_fQ}mupY556o{A*?91qQE^a0)cU9I?wd}oIH{dBE!||3R{x8(AE))F
z`itBWKKL^)X8G&m@6?{!JpXwnv2RORXN>{(x2awiOI$ek|K9eDSrlSab5uR-ql?@!
zrS_#}O6sq)3L=!XLM$%3inYJZa*<s%;g6EUdhwI@v-a@xmEKt*SNyfM`QVo$EJjbB
zJN!{<Jh+d=Zo9b2MAOI5SnStun@{sB<nn#)zsEyqe(`qw;LqWCJKvo+6L{ov%el&z
z7M|rr?=A(nYOXxhb*XE^G%dAx3pX%N+3-5^zy0l9+)H;%+&77VVaMdtAJ(=`Q1Gx8
z+O*gtW9Al>nPQol^EFS3r6%n-ePA({%c=t&{QF)_a(rl45jAtl%ylJ&k~Nc7M{q9v
z6s$XIXK%=E(~`{AwAmeLv$q|~YHthRxZb!ho!d;(ibL_p+C8^9)!t11ewtg-WOJ$H
zWG3cak%uSGx7^FJ#K3@OyFQQk%&J98R%r4@9%gUsTDrnqw0nvELXO2;`<57SNNUgC
zywY&lA&xDMdW-=)Yr3Q-dg!gWwees^-&sw*j7@fXRyZ`D*thr6zOHCapGVy{KUjD)
z&OEIoaE9sVnNR(96_5UzFh^?7`j9m{Lv1)TeY&Ox9}UyleCYIlOP`M`SWR5_c^j~>
zY*|$7aJ5O`>B_aqQd3_|l-o2xE>3Em&t<iF3MLZ^_&5CG`QRY6Wg<hm;5yz9D|ZC!
z-tu{)W!2{NGu%6Gcgh&Le@@U7eIpdO`ACzJVqfL)r4!uyEBmWj`WaHVR%P^Gi4tAC
zv-kZZ$&e?O$C{nmG)_#h_E40O%ue2BTI`X$n}a>dKB~p0d%>a31leDk4)I2+bX_>K
z{m@*!D-#Nf=P|Yz6bc_Ux+CyB^6=H}y?n)cr~3$s3ko{VwiSBVy+ODqPm*)P#_jWE
zI&D){)k|}lSIPgJxlHg(_u5Ge_KJOq%@+F^_br*>z<5INZ8yVt%j2t;9P#w|&-s@5
z^j;y6jVh<T=D2KL<D$93^h(8)U(QnDclPX9VmYzGa{H3gr)ICY7PxuZZr7s+k19#2
zExhTP!srk(QGk(UnZsvQ79F>vF}s%L9BaAEeY#^>puw~=5`21^y^EtdrWkOfYwkX@
zK&?5zT0rFBiXg8gJB79=o+_Lo(cR(Fyu!=Bny=II_-&WtcaHczd&PJ6i|@QC%NZ_C
zZ#y%m*+ezC$u(@vnl7Df)xUbbCi^w(n5HCe4rk%o(X;f!@6$#pnn_>I6(;*kWig*J
zM_{^0=QJJOFE+FIE1mOiofMrt`}~K!_ly^(9CWs|v9rI^=`Cy_ctmcursLbrZu6iS
zXJ_u~E%?8~?awN96)|ODE$%0?tXFz93371S^7C%;V(DP$nRtQIiD920!(@%l2BC`r
zM;EaCxS-Ood{u{VtASrnm+Q_uelaO)4&9zBxs}IUc&fSJ7TG^X@BF$LbJCdarE03l
zCF|E0W2>x%T*Q2wwC26ySf6bo=B)KLc-jW;vnLC9@^j?3RqxY38Yq0`tl~%R_m+!q
z{_*D7+VSp2i{0<VduvYg9O0NIt-71%@}32Uy6>#FmEF*J$gls(&aMrTGXtjo{%p2b
z)ANJP$xFe@7G~RWOB>#--YdXpICDwJXOlCTO03B<&42IgsJuPVNNV4&DM8H!cGATH
zPmR44ZZl~uJhQ~$LTllFK9SxA{;$G{7tOhO=6{ypGyOU_$hKqEoC890J3dPAD{bEG
zt1I5R;qsCVW@o+qB10r4k2-bDy4Jnx+Kldt#-|<U%-;WI3y1UE4HlPmCi6JQOgdf{
z!@VXTd~(W}r;hVv|9TfL;c&Tp^6Z7lmmIY2e>&5;!|h^BXI{)DPvr=MyYtp6c`ZM}
z@odwnF9ICwTThuhUB>l^hrjaj%QYNII=z|Ahm?+-*c0RTWAWv=2@94a98&&rBO)g(
z({_1;m3WiHN*R*{i-kF-GwB}W@pru%`L0G_dWzC=&+VLxC+ePZZhx$0^+(h5hwZ{u
zK|K!+Yc@oMoaNTH?bs11{clMW-vj^ud;WKvy(#lDqwnzzg-0*?_*q4sWJF0{oYlNT
zndhuv(Vq)XA~;Ofc0??lAyUn$RXru`cW~^UVD`?ibmKKrrlt#CoVK276j^OBTlKtB
zP*7~puGm#wt2qu#j|z`jD0kfE#)%vU_Kjy$4xAA<y7-#R;tdm-(pQ@2yjgwbg8$pZ
zRgN(ZRk7=?PdU5)t9ti~l}%N_l3VZIt2MI?m9?9CboZUk!?xP{=gMF9_I#0e#YOb|
ztk{t5j*#o04Q@nFyj2~-zQlG>;?(^Qr0(!|2R5$@P`>(`=hUB)387avc3rrYv#^Qv
z%%qz!eSJF?TYu!!?v0qY;O@<k%ly2z*L8*Nb`2Eo>GkdUzrTHt&CI!{cT5qs@7<Z1
zD=euKcf8B!>78p6SVLIX?LYifzfe~@yK{P+&zWYApm__r>z3R(9eVwelh3{h$4~n%
z%9wb!rMuhk2KSP`LX9^$em(7wd-hQNTcXlq=?-K5x5|$`=*siT_@4UUurl=2i{^b!
zxBXh*#=cr{!+Q~D>mmJP)e6?Kk&F-I*$vO>-xKIPd!yCh@!bX69z5mb^y`}Tku&#m
z!rPdY&)0U8T60JQ-uv(*x!m`F>ZFOS8O}>2pTsZZel8@jeys$<3JI30eO)*Cc#=}u
z7se}aF!V7rc`iF9ExCKSZ`az|`Hjia>-KWUt>)nqFp8Ypv3{<^Udh|Hul}FpwkEbU
zc4O4MzP8hoCvJ$|#@ggKv1wrk|LNb+PmW&f?K#I|c1LDck7b*&bJo)Q)vAuKFP{8*
zaY|G8*{!>@&T;oKuC`f{^z@g^=H%E%mx6wHo=sJX&HX02>*(FihQ9L&ToVM-N|&Wq
zdp&*?V{9b6%_nkGF1JnEHnB^}dku5$HHF2wZ%uC5c5=nSy&9XkX6#wSn`|qwFf!q!
zXe#gSn?6h%OfSCW=ui48+`MqX*%MFNA8Y&jJ#P@^2#m9nJ-u<4!PERRbNaXOh=isJ
z?|nMw+0#0vg@UCM_`NRFq;|#_Ce*q;u$yWrVajo@z^1$4ad*Y5FWwK#!uDqfUrE-B
zi8%G2yCF=y=R%;elFx#Rs!tNHi~C(Kd8+y^`PkphJSzn*>3)#AqkU`tpYYAjS1;+L
z<|v)ryG;6rNmSK5j_9^jfv(*Te+YFlNVai3)^&S)CD3DnZO75RoGFvvh)L~fU6Ipx
zA}7W3v3P3MM1xnX_c(tWZFPE-b@|gBE4}k}VHx)KA`6ZEvyOE*9h<zOvCHDh_WU@G
zS#q3OdVLHZ)GsvLvMWjI%8<Td)3M#q;-pB!_pkmT(;m%U*U_TbzkeCW`m?Kd-py@I
zeBQi*Z^5@vo>_fodV;)mZ%ud>eaA|B$C?NG&suk}<V>6}#f#JD#nYqze!Kn;el6t3
zRT%R6oG({(Y{!~E{})O6Hf+?kKJnq$Ne1_zzW40&=a)}oSb3q>zV!p=Vx4cv$;r`?
z_l_>C^NX3lyx@rAURT!_wq?1^-~6t*U9mCSvR%UP_0&b3&+fh5%W-Slf!>5CufzD}
zOnawecwquZQM(`qtAO=J=~D+Igl??mkn{G+{J3$JTR{1`0AZQTvw^9c6Zkgv_VMn`
zleZ7pCi;HeRy(T@jxT=ub7b#L<cPU=?V9=a552pbvOVWsDx7fnoA*yM-l;-!j~#X0
zl6HNIndm>e9XV_tPVSi&dy{+7LT8cFXP#erYtGHkq_q23guLF1pzn@(&1dK4etOLv
zzpQ-Q?gO*VB}+S<zVRmHx<KgtdDZ9sU!G?C>~zP2vx^UQT`&vQx!%RFS5*G|`U8)z
z`W064ezUKWx|w&yH0S)Kr3?J!dvwijd93%n;E>r+&C|7E_O^%F&$$EG<Ku;|Dx}A!
zM&3<rWXLO8&hdT%-}{WdM+~pt8(cicslmTDGyGoO@0)8cSQP(hIK=nApdd7%p8ra{
z_>a2z7nP@!-h23+-9B9CpIO^$QAO1+oh!O(L;?=1TXOF)N4-<<F2AHKr`?Yxz3gmA
z3Q+$!cR~S|zU;jlZzlEU%!~NBYTcUZs*4>(hh}ttRoOH%wqAQK`<eGoN{!Ckd~x(<
z(>_ljKhbN&A7_0LxY~Th{j5RN!Q68Dj&lY9PYePkC;WffoKY^%UbKz1p~yb-i)`n%
zOzFiU9GlO-o;baz^<mLmvyP(UMceg@cJznZ-~2K6MwQ*e_}9B9?wF9C#CLh7`j3eM
zHK!Z$d$xSObl`LMjK|?zYfTdGO*JW*XOP`BKd`+&@9V?s^-ck0+wZD46nKX(KH(w1
z>g&{8_p4V^ewtf5)~sKCOs~cy?(5r}ZPT~EnwtD7U3qiXX1^6-Su&4)z7aV&+qY|4
z)b_{eg^Btne{b_Ny)*Uyy{R={7Ok}5V4SvOomA#>o(2YkPk!Ibw^+VAGhc1hq)A5t
zGIte*`fVs%G9~nVgyD|=WiPcaaLt<WXu&&9sh954pSth=WXR6J$Rx}m^50>@0tOW}
zVGR)hMo*T;pluu~SGg3A^+=g>?U`|LF~bCf>`al!3r~(WsRS40y!@2v7;VUUqNMVp
zn$sNFL@|}4pVJ+~m|D0}HW!K<3v{zo65Zjce6%Of&`;N+?a}2m949UAdOXv-up#L%
z(}4~LhnpL->e(h56+W7xTlk#o$)>Wkv5mo8@Akg_uAi`xbCQR^gK4h=`oCFR<9_pF
z<NaACudn9ZiFoj|SDQ6Q?}p!j^u~pX-KkrzEcd@8IXf%m?5xKUjq6wy*82Y5mV47P
zKUwCml#bilSmV5|+!%+PbsZDA7tFZxQcSaz)vrHDpyQZ|L?Z)3(}CR$o0;}6*X>wP
z^Z&;OhI8}WZ`;`(Vqkir9-!oSszviB>zCZf&;x8tH$)$}$@m;R8W&u+F~PC&;1<>@
zUC)957iPth36WyEROH-5ltdOf@P^(HYUk(_KN!jBl5p3>f8(XDfVDr2W;x7lO%YBp
z>sqRup4@mvqF3KjEz;ZOnnqX_lM1J#hpwRMm24*4%qx@h&t4NMIPllSZOW>(OH&=A
z|AY&)$xnK@G@j98!4Z-Cld1x)9C=R^QuwSaLhJu7+^IgFIrI7)k(4`Ej%CbGy|$R4
zU2MwCMK-2g7Z>ZhuDZCmCoiRYsgz*liisWnzNWaDG?}Q*@{T+i;<)ap+wE10x7=FA
z;9ojZYwfPY|C$$8={H)&tl1HCG$v%Rf?L;&{G^bGnH;7m6Bz2hot#{A_TKdSbydA5
z1?S2L&d_x#+jfM}Ep@7B?&kEgCu=s#G@S^ne67?n*Ws3+O8!nsRqHrMURCQEiP001
zytNIQo~?|UZCWXGNat(C#Dl6oOF#J4vvN+JsMg38F;%?%>yx9)K7~zWkh^^<dQxcH
z%jpGs%ced(Szxz&;*6l2mNmPd?OijC$;(H<b$0$c(diAmY_*%?{(N*bxzx{Fc3*6^
zg4l;6OghspE}5ZQwe_l}z4qk^(^P*6dCo4~C3d`AGStjp>3>Re-qt#s*MVEkRjKA3
zEL@Vhjw6=)^@-Uz3tJOgeOFwc!yu$mnD2dhU-q)LJDe=}-cMz(=sB{uN*}rO<Kg$s
zQ4gx(1a{oLc9d0As?dAU&WxpX6CScIHC5WJ>ss*s4fE6)fh=$NLhilg>zsa|li78G
z%jFc6pO0K;I?gGGm|3?saKbrj{*a{+yG6c!GC9i<JlSoL?8YenS!+!Fcek-FTO+km
zNAwgcqkG_)i<yd>FY0;EW_ZN)z<Xbm=MK@kB6VWw3o_3}zv1R!Q2fclD#F0Tpu@nx
zz|g?tz{tSypMjM_BqD3UK_fQyZC)A$3mMGU-kIX0v7yDGi|Ir{%88AO9fO!WCVsRi
z5I#IH#baw``s&2BWotb;BVDr<v>RVbtdrU?dCB?tvd0-kElw@Ju+U?&mamqdgJ0w0
zE}_&4lf>ZWMJyqrGfG!Ag!;}`nraoW=HQ%^=~Y*w%wDc4&1dboRTS*7x@_yKo2$ab
zm)x4j#`s&6Gh$()8K=0IoTTxG1BZA;f92+se|XZ;VQuw>V^iqt6LamkMfOH2XzI<m
zQuOddrm6Gw{;QMjSj;Lrel^8TIVbt%V-2(B>3N)yMoSM&+ZuH>>8Mw5Y`>|s+w-4S
z6Rs{UEzgtPVVfP^7dgA?zqrA^KL<X#9zCt9|KQcCd2c7q`f4e^>do&T)0fFwZ&>mF
z>wnQu(a77J1(zGyyjmAHtU0ihRr{?*inq3IYe;zDl7=hdg_5Zn4rz8%r`_*RnYKej
zTzTy=)*MOB3r8I`Y_t$?GqP$Ea9hVQVX>q2s})l^Cd8HmN;=NkAtE_@%R=!uriVwn
zBo_ND4=!~J`x)Gr6S;D!dwA-}xsHoVT(4<_zg<2xB=DyCm039(Up||jbE|yXbj!2h
z_iM}fw)!<k_dF8t(YYBsYo_zUmXL|ejTsXbWf^wO=-^S?vM`p-Ei2gleCnx-#e7pQ
z-pY+udwQ&gxz{RcB~S0IwuS2tzId4@r}$q>K>9ytijebmt_iQ*HA8Pq&^6CwlK1@Z
z_Q<IjR<-W2^=JA%$y7XdJw0{H8{h0)r)Q;GySClo3dsq--1u~D^({NooE0xV9T!;G
zvhMW6naAr&7v`H^Em71q$uwfq-pVmyPOR{$qg@`ePn-%YXq(8AIh9-eYQZkqsX~hx
zgsYFRiZaFWJa%1oFzi?X)09S~l}=nKTqpT?Rid0McU^MR)Nel{z`0s+$;2~18%6F&
zWKVQ4kkoNyyFB?^Rmr>Bh^Xhm)9q`Yn?y%)e*Lo5-8m;@YeJB#$mML-7t$d?ox*DF
zv2ov3JzLby>UgFpFMS!kW`l8C@mG#7L2>)m#b!9MN&mmXrQLH+Wk#}k;4cl$9zCC#
z_Z8JlZ*127@0;noXsXk(<!;w93f)#Jrp`WN{ww9f**#O0F1c6Do+f|dXrsuNmeX4o
zaYmjm+HzUz@tu8ZuBp^sUw2hD_4K`p*fPyMRl&^bZEksJ-qvZTQs&kVT>T?_Z`kKk
zKSgyH_c&g261lcnpy2h+*<U}sSbkqnQel0@g>I&U{vZ9<U9>prux7U#oA~_gZ<PE0
zKR%Jh^+>pDLZZt2-TaYdq5GN@GgbEO^47hvko7sk`JBS6$!v!Y%dI@RTr)}7b;dEZ
zLmzs4Z54a?*=IjG$g}CJw^y+6wH78P(NzU4*Sq4jPEi#NOj#Nmd^qv^{~gnW7j~rD
zMR+axC2@!^YQti&yUFc$KWtheD9PnAYnEtQLy&b_PFrxyohh%>)K~W3adBCc*!sxI
zS#x57pTgYeq?0yDT8wLQQnzIuelAeD`L4p5n)i=5Uh^28xc0L<=v3h1eO=ph^E11v
z>~o%IteR$*@-@fhq<7qh2KBvcRco_X9$w;K_kH)uD_V;GI>Y^6ZB=KykR<CjN5S^R
zMgFK;o8-TKaKGzzQJ?eXVKJ-ZR?ibca}+X9D0g{BBu)8gU>Vif@W!~~cT9-=S;;4L
zl8<j>Z;+Ij(E2Wf=VhR5b&KF_0nYTA#S!<XF-;1pdXmm`qnkm=WXsOLlas5uZ)vb>
z`G4N{<>&eTa?anlWTIxcQQhaBV5`KFOQQ06CO+Jy8qW1E4tJh%l@a-y$l<PWNW^XF
zV#&QL>J7Aly2VtN2&87Xh~7FN%cZdBV$({;uw4P4u6$X#!bkS<F|`Sg0=cyp?pYEr
zH%LtVSg+U@hpf0P$J13`7RmkXSQWDT7C)oQ*Olj9sV>M#<d6D!bng3{iwAg?NgO`<
zEWguxd74Pj^s^r8jJSjsxQA`ixMH|YK}j>k{n3xf=3bZ8*FDU37xI`CxU+G?`j3kp
zdM+!q8LX<QSK(<dX<bq&r#^A(g%+`8sq+FV-)=6tc!b|oIx4W^+m>a%x>@1YQC}SR
zw=6rNp5bn|`}+TDW!nxbudPp89(^s#iD94bT8Fyq3HOD*PuU>4G$&{$SIu6dX>V2>
z*PY$v9s6k0uAOV716D=dU3Avia$83G0t=V6zY1^qV@sA*y+6!(!bjMC-9&kJgR=cs
zuS`#RYU0x`7BrzXAw|)$sKdqOv+sr0H**>s*W~zY;+(hjN+T<Gyz>X8?HX@BANnnl
z*yd?r!)LF%@P?F}_vTF(gdT_R%+e_Air<y=V9}4Gntg^n@ij?vxQ-k#IBHm3QuA2q
z{Eg!dW*>V~d7jMN^kbR1EJtChjey+kAIp`><Ysz!JQZ5@)P8kKs_<kl{$<aet_<Nd
z4rN~y%Q)qxLh8Cz4y&TfSA2W?KVtSv@o;UC$ORj^jN`6cdSrLxxs7#LL+;kqOW(}U
zNL#f|G}VGv=K5@bgIcZ!B{Y2`T{h1+DrB{3qAX{U$HQ|@Mh42W1q3%lZR|Yj^>o_`
zwXVWU?wGLY*LJM6(+%&KeanX-{N$nyrp_gEUpIb;QkW^wyw-Jfb|Ulk>qqRqZJ2oX
zwU0sd%?r=E*3OlEn=)Db)`75Z8{2Qc&d$Dl`+*q4`subYh1$EdkH1~Js{HltzJ#AU
zeS6;NZk7&>{k2$r*Osh(>`mUyZ+_pGUC8cyWsX4g<z&%j#`ucphdTe)7_Z~;eBUwa
zWX6)5MQ#$sf(f?Q4fBjzX8-Z#h-E$@aB<C<4Qu}kF`P9@nphkqY~P@`l-XbXd@Dm1
z&%_C*B{pqJlG^>_#H`*z)<Byt3xww$oo!piYQDRK(PZvRzy4ootW00Ix@NzeS+VPi
z>Flp-vlqS&*uU%YLBDS+r`NtoJ6?6w^mWNL`?+rqq*~p1ZufPkG0Rc=@{NVrUU#x%
zE9d7oN?Z5cO0D{`NIt=B&y2;p(%#xlJu^YvLEpmZgHY_%m5Uj87G!<&?o@9Lbi7rZ
zqGW90W#~4uj{D0C;o_NXTjqUbm-d+Z)N10jJ#*@09Jq3d^iT9GJaJJ)>ZiZc^cNEh
zrfuX@Y<+NSh4vC2b`^!>``%GZ)=zsDHQ6YC?AyM+R!F@1)}eoE|8H!b%jxSA{U+*!
z)781_>b_+w3cXC<_3NR1y{^l{zu%7k|NC(N|38=G|NjWD|G-}FXWqbYqMrRlJ=cx;
zx62#8hBv%!FSQNJOEa(A7Z&_PJ@42x`9&LauO8-l{2=DF1HZpX?DeN<*QUm%ZOdm{
zn3n!5-{T>FhFL+@!PuLnI_1LtAHR631^Sp7N=O7rWQG>HoDj_Yp!xo)pyYIcS(ELh
zl?A>#$1@qod}l8D-7H|)B*4&S6yf2<sK&5VL2#LqpqQe7%7UU82V<EHQU}`Q-?B&^
z{*tgqNk}M7yy7BXN<n>|dTEVBBeP*?;Bmp{=>pHgJD!<!L{2DftEihep`-8rjgH<K
zoP8F#kqVt%Gdh?UI;VDY&brar|D$uBMAv)`hDe95c^+NMBDz*}bWPO|Sbd|btwS^^
zA$9T=_3BNjanC~Xwp9cl71Y-d*q;$PLA-~@vB%l4lJj8fK8?^KW!Hb3LXTBwIBXP9
z-fl3@Ttp;Y;N{b>Hnr+fRn4i%;`55CUmUL9XKo+$h>f3JxhYVk?qk94h2fkNwU3Hq
z#j&{l46py}-u3=?zvcFRhRk}Vg8D#%32c!QI4UP_&78nvDZu-4LS(}PLCXnCJti^<
zvWj+26px&^B4c7C1FOu+E(Omn<(Ctses-y^oG52GN!WAJx{iqkon4QgIm_KBIH#Jw
z^?ysORcHRrmV9Q#X3oI0^kYrN8KF$+Q?kEJ_L@0`>0|z_r&D}?PI2;VW-g!7d!1kG
ziv9yp6R}GX6=so~?Nu))`P}HR%iX|gw8)TYV+*rle2%4n9Q*X*Bdmpz(<Lw4%Uo2p
zJtQcUD6rF9=vhS@+mZy2g~kpaGnpnxnWXbiE*4)Sz{>h^!lPrIGcr1#m33NP?_ibW
zoG`Ov?#Y?6C1=jb>|%X6bAsh84$hhEoU?dN&SGCVi)ZF6KF!(Uk~1e-PGIF^-M(}7
zW=)1&nlpE33hd9Ez{FS|IlWYKS)zPJw?}Kp*9F}{Pt)43cTY_h=vS}2s3Cn~hQMu2
z>8mHF-02j#`+uf4(*s*!we-iPkzR%ADhD;sE43^WpI=vOHRWU2S4+uU*ZJoSCF@nH
z&Zrkn46OFdP<=8(;7*J0yUT4q9A~f#7R@&l;&8T`S?;yW(4fe$T4tlv1^3A44U$*c
zT@QH9QCcOk{zR8r)FS?!6Sy>4**jSR7tGd^TC6X{Y8172pXXwxfW;PFES9$x+x!Bp
zgl3jpY`ba^-^&IKF*Vbh3KvvkEj{yZrshA<Y<AG>@$@VRjaq8pA#~qUDQcBiyp@vK
z3;yFwZX%lmmRyjS#xhN@aiMEOp}}Nrt)@QaM#HjM0^T#0ms+(l6)a%d&=Q(q_}f@W
zdvdGYC&>rSnjKticK?M0Sp(Tzg4(o>n`<>WxG}YyW%pR&++KZAU+4;74#Uj-tdj*J
zyCz7^=8T*zF1biWkWuF0DjvqwoQkUj8(7v!F$O%Cy<^vEn^~(p7#R1@TD@=8>b+cR
zc5tn+<XUs;)MB>6Re=JX@#ZC`Ug|WrcYmEAt}ZdRJALUsiOMO*^Y6?`yB9SjZCTvS
zpTe=Ldcu}gFfA6S_~@;oq<yea;<|=a=S8lJs}`A0EAMdFX-!<#A|}*T<R;!|S-51u
z0q0hOO-d^QRohLAI-W}H5%ky?7~cAD+00utwh|KGG6Xg-3!OP7Ad@K|6FKY1%+)HH
zn^sp%RFvMt^l%gN#Z89Rn|TyB>;KoD<EglYd1k+8N|e_tohUO$zgr<eQCkAN=J|52
zJ0QHw`%5>&^erj}x5zGB8n$xF&8_}XmCIsWvm_6(Npp)xXir=HUHDFgrplz=O@RU#
zn#y;UESEQr_c~}*rzLRMP)5;R;81vxin+kL1zJoOik2LzPAO<(o47qoaND*+Qpeo6
zVxF46J-{_dVf8wRMKd@IHC_vv&0c)`)SA<?*6i0}5I7*Pt8&#;jkTLOD$a(k^=1v-
zRgvHReAjLb2BU~j2hPw#H<qeink%z#cmA#2uWxbRE%wki^nbW?o~NKnPoY)hlHPpP
z$a^Qax~r<zsqZatb(<Qr_vCZoiAUofM=ZGcW$*0&$^y@RgxeL`M{i)<_r03$^n#Yh
z0`GqaPYf05xG8x4_;SgM3z-8IPcd$CHrymzINRd1z|NbCr)R4kidywuJm=S?Ml(<0
zPi$%K*;~G;?T(tXl=Zn_eAL`i9lJ$zw#s~5D%T^gSaUGUso(}{4EIEtqGjTuRaVER
z)F?E$?%N%gTTm$Sxv1Q0-|Wp&K^`7#x0W}k8c8@h7;TnZ_<6-OZGpv1D>)o@Fc-G9
zyxzM>FtX5M{}Dx@p6IlOOET-SRxv-^?|E=Glg><ro=xkdb}ih!X~)f(Cp6dIQ?A%p
zvUb;xU3)Cj4riq4MXYnaxb*p|<8?kl&u`_2%<Or5Ywfu!$Gsm`-2A^)|GA{bJ8kK5
zv8t>kZZe98o0qwEeAKw`TF~&1V1Lp!yT%nA$x`zRg=an#f2<L~k!>1%fUz-aACL6{
z)kFKjJu=#6i!45nDR#YPi9n|8?EQZZaYZ+<G<a-^_Yq?7nK=FSYUYKT&;FVyIJ2I!
zC#iU|f9ffH)f?O{(g$rKx7_ENC&M!@c((jQpF<)xB4Rqy$~h{p_o$}3?cNz;5;<FD
zpMgQ<EBVCVhwLx9W>{}ibDE}HR9I{!lJ9BnVYuDr#PafN@nud&G&gT&x~S|D!<MeG
z@~O(v_UxLpM+qNpYk#}oU3TJVqs0Wv_A~a1v*yq2@SZ-$y50J{pn2ir)L6CS)Bl@?
zuG0>E92t7O^Z1RE0(-P8PX(U~+@0nid2X*tvq2*NzJ|1yHUjp0<D6ofGs;5q-NXzw
zHvc^%_+h8!lfz*Oo36h6;@ZS!ae9iydh_{pX)SDw#@pKU9={gO*PR}GfU!*b^v^)|
zUFCkdi!|ykUNsDld+)>*c1GaqZ;9v+l2^nP589jwNna!zSjsZ9K5D{}&4&;CI3(i~
zU7@aVQ1H${yTn7=Pp^I8xpZFcK?Rv+@0C-0_b!$1Ik+-c;PgfPe;+O`IWQw2``nVZ
z2Yq(mUVBzxPwnm1cW*zt(-bz<`^^KL6ANY1cL{vEvn*UjMQqmTny~dVj1Fhco*`>&
z|76Btxx(dl{;TwFYE<H$d(CN*^5P<+hzNIyL_xnb*GqP}>~B7r_Dow}(Rxv3X>Lx*
zq6fD5b9S{%@4gvzysW1ux9oV>>f`pc6^mlmUF|$!SD0q)dnaa>2-i9Bw-XN9?z~d@
zko(&Q{<i^K?+k9s=rQzgt}kA8W!v5d4S&wwnYGTtvEZP@<2Ii=J?iJrTilP7)vo=f
zxOd`_w=2V<K1EocyOSa!{D9f6^V)_Ux9i`mTlbi=zK}lMY57#`&^~_-KQBjZ!}cA)
z7n?;G{f`U!A3S4x?Trqfjbnyg{+!$`dVGd~dsUTfc0Z3%4dDu9*nQhsQ)9^ku5yms
znX^1*Up(ij_(HV2IcVY3kN;VO&pi@vvK9MPb27=gH)+bX)NVai#S52Grq@TQ&YCAu
z{A$4@xr;T{0>TH6)cxEmVrUfFwDS4lz1`lA6m%W$9^Q3hRmBGBx$AcCE}mzXJ5MjF
zYVE(GwZ?x#jyrMRkE~RibI@*Oe%LMkeF}C@FYI{mtD@-=pYlDfr2$+*Qv~L)Pigq{
zW@q+Y*@c&nojsu!==#5MkHbFIscwZ+|GkUj6sah!HWa$<koZ1p;f9-6uC3P5dQ+|Z
zD_ArBtUzhmDRHYCoPqmSPpph)aQs-ZOt?fx*yn}w*;|j+Jd(S(bV=>)Ew6=l&Ak<x
zb?B_B)2~UTuhJ817S6W4cxAt>fKTYsuMTTB|9{ILzW3vS8TnZu{Qp0A-_m3F|KM{E
zr;x1Omjhb_cD{Wr_y3kx{cW-NAGLOHo@a~`Ns`KVb?08i(c~ox*Q1}bsVOD6+!Ixg
z5EgE&n04Wt%01<W7haSI^khkU$FPY_tbWF^Lw8eOa89Ob;0(=+Tsm)7R&4ZNqus;$
zuW8qhH!^iU_P0EGQ1|h`k4KI(<&Es#$pxARy7EVTSbfa0^!HxAzX$ks+!FTCKCX~!
zeowV0c>-thg~@+6aGlS3^X~!wx&W@N3eDTJ1tve|{&zt_Lrz$?@wcl!gLiIP-F<Jq
zeX99!>sss|%RWpye=efDP&}GD%)Zd|W1MD(kcIU+*ZOULo&Vpn*!2Bm>+Koy8!s+A
zWcA?glj6DiUS@pno+^EFwS%0~4Y`9333gvkBq}Tl_`|vJ%Dw<@hQ!rV*GBDQk#U(3
z@X)D^SJtjYLU7Uks=p?Vork(&_g0D6URx}x+O?wPt6T4>C92&LzH6-dP6@S6W2pa|
z<>;uiIeBlW$CeAi7cRzXm+_SU$_`(3*S`8+)YdHZ1uJ(4EzR4y`s%7M>uPpuFBa)(
z3+*?iDu1iCWJ?TVWRBjbcvLGdGKeXn*XZ|#(0h9;|H?f%qN*sa7krA#uH%NHXiKz!
zRNRXS#gx`Q*<#Jd8+s4#V~W`+$gr!W;S9T{iAc}REB8Ap#Pggub}MQcE%_gHb9Jjg
zmTp+qNx46NC**`jm&HiB*WH<>TYaXMb$$DW`28F9N(9As#4ap*z_jN7v3uIT9N%5K
zcx_R8S&x5}Y4*2Q;hW|6>UbQPuzabemx}J9V=vm1rc`BJWMT5n>g<>0D6%Tp#pF8k
z-1Mp(&F9-5@cru3xy|xn=}z9i7XvgIZxy~f)Vd~lt>U|5!5v3MPAm5-e$2{pog8-T
z_Nr;m1d>$sjs7JyFO)whWwdY?->Z)cl%qX2O;vttwZgc=Dws!+?~{^_VcWVI4#T6X
za&nlpgHKPIEFzq>_(X?6w)e^AE}2iBtq+niM1o=_pD?uayt&Auk#AM2Y?#Z*vqvN(
zH>v(td~(O&UEgE2RjL;^FA+}IS9NJ|=cRWijJ~*5>4kR84Ru?XDq^_kf|CQ|qK2ep
zts1Y?g*7%cte>Z|K%imTT%T3`2PQ0%Xk8Te<}^!0{fmoCohsIwj(fJxNDW$5-<I@L
zJ}p=AsA%;X*3~wChd-GGKNR44cRSx)G<?y670%s@3_o1ST2K{r(?yTb?dQhh3h!Nm
z8Z1s8n5?L|@Qt~boTS$$pH@Mm%lBJZjC~A`9SAyOcuc~$XH)CeCy$Pw<%@c|pkGuf
zVxiFqiMI-RtPzn62Ig)n^elg_)qc=gx8|*xC+mrg<_q|IIre!8-imCn<|{hfYjnoo
zac}rN3qQ+eM-ET;FB`=Yn|{U6CDF5c%C$lcH<8RoeQToFmb8miopa+*2{t~dq#C4c
z%DLEM(}k9$p|28IIXPS|zR;WA<mAnxvMD29R%1a#YtD~flB`WVuOj`s8BGNpE@*7p
zU^PkOM&Ww-=v#?L<v2Aby|b+Lcs^mxa-a6Un$H9d^}34t_q_P_R;bBPV?k}a$M=Qu
zUFTfiW_@Cfzq?`9h9}0&J{F5=MZ4^dPUN3ex3D(y=IZ19(LW=MW+-P>8ZaJ7uRPsj
zd828A5o?Bp>xFgdy$dhh5J^a5++=U`%Qji<=)RQ26Br-4nk{|EB5C$eol&LB+|)@R
zqKi%Hg2ED>SOaFZ(+sB8TMqr_TQecVHEdyP<XJ9Gi<|w|BZO?<Oj#W?W7Scuh=W{T
zR9Zx5U0_-jFtt-IutjvXbZ5Pbk!5|45?6aPYt*f+UCNuhR17apbq~9`s_u)5a_(Xd
zT?3`AWYuiZHzyw6Y*Xb@c2jJ+uH<^r>k)tTjwX?B3Dcx=3#YK%YIAv$5VY`Ex!RSB
zDK@nw3k&|u(9}$tm>(^1LuTXD{jW0LE12DWJcUKfvyJ7Mf>g1jQ=D2ur{<E$@l72+
zG@4H~FX&gCRv4KnU}O|#^~6{$_{6ClEfS$_m(sScc{1B<;pUcvR@r2RyAf(C4bdwM
zP4WsHGh!~b1x;BvGqzxv@!bF}vn`Ko&HhUs-5s&9&rrutc-JDoyrv@4-W^<ut54j%
zog~!i)3eZa!r>Ep-3+x1HzZrxN_mPn7EScn@ua<eHebOXK_M0nhuKX^{527btQ;)a
zvH}f<Hr+Ygr`)$t`L)2wl{)2XqBp*>n<~`5)kARVx+zo2_9?{L`kCv$Gvs$smI}<v
zddMIDB`-OdTk3towRIaOoL<iK_Cj$})FnPc^}uZftU+_Pw5o1p);g1TSYy-k#UI~#
z``H)CeX~$D_-nXC<V*31)2G}Gbt65EVho;&rJda86Bp#8=dAtbUb)&*SvBLWU#`b|
zIxY}&MRnEpQ{fE1wZv?Buf=oQTngOU<nH9?60~lE{QpZ)h22YLIWb%~lAw8q!O8y6
zD~&fVuKEcWaRsI*wi^l_)>xJ-IOjpGsL>UEA-9S1d>j;ne*N)SvtUA}SM3t9i8pc=
zRJoq$V>oK}ws4}mMpw(;t4=DjEadJN39ac$&{L18>10^e?OJ`~a32$=NIp|2|K+KD
zks^|^GPajQzU|YB6?_@8ysF%0{;ci^-+Mb8H*Jc4ce70-cLA^Di3=O;JQwQm$#8U5
ziHP57c<K@B>k%f%Hqk9;lj5{BXOcyj3>gzXCC$p=>F~apDCDo^x-#=boWg8@3u|`x
z&WiPw44G-fI4A9J)~&^7emq_L-)zJ5My6%UE?ioe+4PK;-H>_G#Q!njH#{}8&n3w{
zWQtn(@`{o$hlRk@Vqw<ib&K7T4s>Mw-YCcMLC4Naur-&xed08c)>9%Y`z4MU%a%WF
zkqt8oU<tp??W>?5{OC#S)Wrb-AA6L}q&s`}D|~o<_0pqLJ<+<2D%X|Ej;6;|^mu)2
zDl*#?uvY#<A-~g_!|N6<WOY_$6n824F3Wdxd8yOVmjz3drpCWHvBL8&&yQI<Ur(Oe
z$gTRLbVK7^M!k{++{e@y#nm3n*lWY8z2l}t-&v!pO`2ya&vki!byAz?8?|FCcVf2F
zG6gk0pT^pE)sk_igTgy`cJS~r8^oX8^k5yIf#QaVXHSSk%x;#M+GfIXC&K6Yng2#t
zLZ`BqAC8-KXib#-6Z!YC%A5Lb>nXc^+1z&8Nl5z~m-qD+vCCiCt$Qj~>#T7Rd5|Jn
zvX@CaaD!Lw$zPp{?MqqBw|6KR=&xJ&xNMS9T9TT4Vv=pGHCNc4$p!9_7Se(0HXS!i
za=2D=7DU}}YF!|*=g#Z(lW$FMVA{Pqd4^~2`uAP?f61o2Nu9wItSYMCxFKn}dzbLm
zui?|VUZ*Atg-NuX++`4$RdHm^&rAIwD<5sLPBC5U@yRixN5J%^#G%K<y-`WVKd-;|
z<mUSMsYJv^z4r#(2U1s@4v$`FB3!YMC9U9a`n2Fn-={>mMd?jAzg$>${(@f7#{!4%
zMjaB?aQV-WGuQr_V6)Rihlyd_6F0oodeK-|z3yAYnfLCq{=b>}G9vxbos}hDa=R5i
z-Z4-s+&$ew`_oIttB<XpOn<-prfJqEi&&+Hm9<VW83F>OEB-B8tof!V>-DoOheUU!
zsY!OLopG3%@VFt#dRx4;8KXucYt8|_gafR5*0fhR9GcOjRN1w8W9u&A?&FJ9GKFWj
zb%`}aas-{|dF{Ay|8Jp<NgcjYt0NZmY*tcjTfn1d;ULg0e=Wg0FL>#MncM<*Hoj?G
zvSQcd04~EIjaK%>y+@QL9$m${h+D4TV^vT=YmkH1sZ$Ghnr2ylQ1CG1m>V>4?h2U;
z4?8p_%n21}eSDe8w_@%8hle#%6{j1SOenMLI-RH<C87AcU~_V0tFBXb;>o7_(sui1
zieBF#a8Zc$>FKrBw>x|e@>xx?@30nrtHJ-vXkzh1m-Yj^A1-nIFA`E;*~=(=bp7XD
zo0WOZc51V^bR7N7z43Na>g@w{lcz^D^fCkq?c5-)Q8`KJ;Q@|p;pZ=1g}#`zUa)Ro
z;h3DlaqPuzZ=sDIyLhH+PMTO~m>JWdq}&^|X=D3I`O;=d+1=YOPBK4so5f6FqI@>{
zS?PHanbYeEjHYgCkJe!2f59Z|V9QuA>q>Li#!8pP5xR^H4LvjVrbL<;GR)Z;xwbk<
z<LaV)RR=5?8M^vr*gF57Qy0NL>2k~ewHKR06{oyh;UTxE@8jdcJ5ToAOy1vYb7Gt2
z);AYK-ZxDAd8zNCgml>A{o$wWBdYmkX{>sd(f9PkR<jFRGnASqGn&O)J0uyOO5!%-
z$T=|OWz*7}Q_FJ>aK2$`n{kR`%Bj;`UW^9YnhmzCb8#^8*uAfzYk3!+hY@$&90T{L
z4zAVRHJ;vF(WhVB+?MXc5iE6#C;OOgLr-zGJmU?96CS<a3gv$&`a}pxc6uBwOBUX9
zdv(hX?N^$P9TE!F-P@ZT)(Czyzqez0%fcg8G5Yldt=y-ac!Q+OJ(M@z=$v=LqJIa+
zGb5|@5_T6R>Arl?(QTkyCeZ4mpliaiS*Kw_m%!BjyDu)Rj5@9rB>nR@m!nbFeaZQL
znSwtT?|S!wuX)EV_D}i@!W#lrn(Y{^*LyGhC1}kZ<r*4d^C`qN?8~-=TO2iCaKFo3
zP{cArY0c4=hYq`pPw!9gk)EZ}tl6C;!X045y?e_Bt%KTmIjZ8>tdhYTrVl5{zm#BP
zSZ!-`nyIRzw)(ur#jfH#{$4D0^$qU)AD9G88<*)Y&wry>Ftg9DWy{M2tPiX#cdcaA
zm$uhFY{b)eB5#pp)B`Es6HIr7n@cO^7;}qkW!&<p=Hx5p!(t2hYFq3}zx1hWo!h1{
zJz}^0oFn#8mpvCtb;Nk@$jDl@K!^FO%C<yruhljjoHpB*yL9|t_2ks57Ke2?9A>^8
zmaQFm#T_hPOgSe^zMC;^cZtN`lG9nM4bl~7c0^2S^N<MoFv*lL;7UZt4@2QyJR5iM
zY!|QT;k1z7-?Dngp$#%V9C;}nvMn9IAG-Z&@jmX-p`hus;f1iD!J0o0x3d*-pKUqh
zut+5Mbw~6H1xAbJr6<=k&p7E}GG*=y>109O9i2?m95~k(b|xDxy2;encR<Q&4ZEAk
z)~TP*{Hof&p};Ti?@FH-j=x>g)pz^dQaJjK#d@Rj(O(NUmdT!PbvQ55dA(WKf5Qq@
zJ;~mMQ<#iAgtirMtE`-PY>k0wAur#Xp2m~@_Pvq{U%Dkl<PR3~NbWf;64U!XGiGV`
z+n#W(JyMS&F2A_J&U&@=;>MFa-Ku{k&b`Td>EL7y+bJ9`*(*%Us!EuRKX^Wyz%TNa
zy{n0-J4k7^0K+r^hBJ!0KF;ocd}iNOrCXC*1bvftmE2j`E+94EN$~UUuICEev-bo|
zK6QyLi!c1g{+8Kl`fs-~sa)QqyUkda+bWi0Yi;n_m;>AHo?2NFEaJ?=Q4?&L8*Onm
zdaccEF&~cITOIX&oIaajHcxVy?JPNqM4JwdNykb%S{9mh?wA*J<6Qd;ReM*}!#A4t
z+sxNqDRIDs!|-6pzB5ABu0CSiH`@gbHa7=s)QGq~W8U71tH0;$saef^++)EmjpM5)
zs~5cN_+MSX$}{WKvW-rcHMw6c;FuiIdjIp9<egJ~ewf?k;n%9b&6n!;(#Jh|>ZN?q
z?mXX)ix2hH3MZ-5m@|If8su}m)Yqr1HvF^B`R}FityAN<j`V2GXfSy+FHM<)amCRY
zo<e)8I)3vUtFyeZ!Q)0&cE_n+9;q+4+XVtHf9>wdJmwP{lXpa0oV)jef<;))g`>3r
zKXVP*Hkh|A;C<>CxhCM|OsVOgce5VJK6Sx>b$(69Jx9S$tG8@(yrt>v*?M5*)Hi(v
zFCY4Oiz~8)J@|2G)*8=6kL;H?X>DM2Xew4S3_W1zdw_k)?HzL+wp@LrbMJQAn&_>6
zw{?CEHYhvrRfTzDPuG8zXo)RXR++V(saHHG$9cuflVeTRhQ^7z6+HuP++f<Bb0xoe
zrp`?R@fHrf&Vwf|NcgCl@7Q&4)1>)V{)Sw7&~lbxgG=q{n;kR%o|LzdeAs$mx^Kl*
zyNSGwKZK>MI={WpoPB$u&J~?=x9<IjvXr;G_gX?K`fJ=PUG7&`y6S%)`S0Y!Xnj2}
z<a&ro+UJt+AEw<Exg4o^_mjgqDtWoN?IusxWaJAt&+DKv@8YwI8(H^M2+1ZJ6o|O_
zu9FL1JC8{(=HHTJuT2iEmTn<J=3IJ)GZ-gNZ+zakGUEIN-def6>vtWyc;Z;jmmBu7
z$7U`JaOOSQc;T6b=fh7DOTI~Os#NU#zw1Jmy6?o>7au-cC^hwuhxAc_aBEqCgZ2x9
zl2)EL`8WH~#n+y3lQ%99F<WBem2f)z;DqRvF*XM0xOcd6>_`nZH=S=>dwJJYuQraX
zxqOco{EK$Kr($w8+9<9=uPl1+TZ!Ly&C+KtKAyrmCo*R3wX7;rYvC!9Ue}(;7si~}
zo1)=4!^~sSj#a1ggrDpV(N4Q^rL|)9PB8`_-=2dll3Z4?o24gR?YLm&7utSfa_z!c
zbI+^04w$o^;yt>F`<_Ir?VoGTG3(<!v>1P|+LzkL9J}}MVjSN`L%%nv0W4|jFZ*rC
zUY(S7_iEm=k15YgdAmzy-@oa1KH)Kw<_FK2N!F?pWB>2oAnYrau2iGa_0N?}`fZV~
z6N7i2%9kV0{v9c`6W%cAdD{#}j&-LwpY661o!?&Dai(F-#E<WORtnc!^6uGW7!?$e
zZ{vN-V}r{{hkccSqBr@zBp+Yayla*5qlwqvpDF88J+`&w^GSw3L6Y8Pi$V@C>qN8r
zTwb1bS%06-V!14f+E=CzPuU-PxA!;q>t$~_pBF56cAI%lgUlW7XBn-B*99E8cTAOK
z@7}0GxgkPxGB*TFnR%;YyBzl#OSya}(PL)=6c~Lw3T|)UcJKKS$m8Gpd{>S3@3U8S
z9m{UFFyAZpX}>_IT+ZZEIubX0Ij$erxNFC}hSCQ&0u~(GVzcO_+&zv>|2rMFF7tgl
zZ&PaLf^!}RyJGF~-tauL;y<5sOgp{o8Tb7U>1iF;^|&40f97cq4;8W2Owe-Z6lzRp
zjt!c0@RjxkIUQrh=kD$b>--cBBuE^Td+$^Bj@L5bccy%&E_cxL4T&W^yPiqp&Y6)L
z{JiE|&pwXZ2CkCr7Ww_2?`4%0I%a%~{Ns8llhYtZ@Y9aDk2bEKDYGgi-zf6-t!VE|
zhV++{H9t(33!Y(jd(9iKEuojc>7Jk6_I5>B$FgZ37H+$3`LuYETlrq!@}mc$^S*yD
z@O!oRUgpwgk96arTUTV}uK&(`|5PRW9p*a>ng>=dUv=G!J-u%Cs<qz|+5=V1WQ9Cb
zng4saS1|rqC0Bf>yZ-cAZw^Do$&pSx^7kbIg@l8%J~eCn6qE7cW$)IEpM1$;(nL<T
zhi2I-2bZ=l@HzS{o>i(avi8lr44Fp<7b(umY5#GEv-7k1X{-2ie(&y_yZmiyh?DEr
zub-WcpSpJK^R;(n_UErVKL52Ye8$@OFF)UGi2bxbsG{qCHmh-9wMFKPj(Xvf7hd_<
z3wx+0GDyEQlf2>{`g8y1Ut6+xG%IFkt`G`XrNC-?rd032Y{?hBXTKa+wj}@A{S6Td
zc&_%YE!cgQJM)+GBfgU&f3wS?A74@zbC6`-ou%)!L}nYup4(UbzZbA`FT0mHU+#Oy
zj@$X?lXN~N{k;48|Nk$)Zu`$~k?1|$8lm@NRhp3__pdaW>K{+pc|xlyST=oce!<c*
z<6p~yYTx`q)tfyEH@V-jzpL$A-7%HJFGJF(FvMN#Lh2Xm;{i`R_hzfNYgb%9c=>D(
zV{rODqtG>S>C#3o0+=|AX0uF4G7e}s5XQu^Wz*ITp}JwoZa<?n7A$gQ+^2IZE1g@(
zC2XxnZ;taSo|#5-P829^KC0?HEp^A4HJS&~X3l2vdU57N@x$|TCEJB`v%Y9tTHw*h
zrF-z`%8PTnm+MXS`uZwJ&($))bk)~a;j0q+wa$86R+8)vvW!f*{Z;qQ%B3DNWmbMz
zcQJ@*Tfxkx(r(xNE{x25+|vyL|7!{x`Mtbdt|7`ek#$GW<_%M%_j0X$mXlh!FvQ5l
zljA{DSVP#}w(w{fi>w#fi<WRq%8Xbq=G^u+s#odp#jEb-Yi~_aidz;kbF${n2^FVW
znWyueU+n)rV2iUjW2{%s`*nY-zrE#NEG(;icV+1nv6HLy^Z(W!`*36H`S(Wg4z(X{
z?7V(Ie}8rT&Ap-_RX=2>&THTn?FemhUTS_VfOpe2g+N{*jfKtEr<Mte%dTR7nByF|
zH6ThwspXne9`916Ztqg1hOVl;D{cjDUTY#9%d}<TVuqYAKa%s72&hE_US50rRzOGO
zk;eGew^J`BwUxe5oFqB*;js=E&!tx*t>0P9?*A{=YH?-8#?&KSu8B-P#iTbMeKyI7
zulC3U-8B;wS$%Z(E=l*%?VT#Tz>0O&lmMT(VIh%oL`_#Lur<{Rm5h|SC9-s~Rq4xM
zU(0q)|Msw-nkyD|d97H!+OulPq6G(8ABL>i<|^P=HgSsd%oLxEtP_L8rmlANt6LR1
z*~9c&gxln;=@-+zBgHb8xHxWo;*{LC>z1SGq{e0L{G1p3q-XmC+%0C)%18)!+GQ=;
zcE5MJV3T{_anY;~yMBc%Iq_5^PjX&jSY&G0&FvFv{r?`H@5(Vx+~D9J&K%i;9?PtC
zSFG#vTIkC8@Ac(?PjRltysW0~Sh2hBnEBMMBHpi$oHP#qc-8rTOY4j~ef+%b7u2Lb
zREdc1Q##ss?seLeCt}h%Cao7-bi=kV1WKv02Ktn7+;s}rJWVGk`q`GnPA&^qehrFF
zcj3}Z^V{0#kZ|0m_!^VNkpNvT!?zhL108EZS1C2A&5@p}=pHP&?o0PlY5w9}naiH$
z?YjJM1#jNEX<3$b=1&5}CAIcQPcjX;Gts&#D@HZ=t3golRo6>vm~xapp7$4WRY`EU
zr=ykB<Y{>Hw%fyBHv-p1OxgDCR#NWGK8;>8k<2AqkMDf8cCRa!<dQ2%{<?2Z*j!3V
zEv*W9>XR9haq#KX*_Zc!h^$xLKQlM;>Ao|s`tJBvv7ZtxV?4J1a{cG*<8u%F|8tc;
z;V*YHpF!A}=BdtqRbmqjA3QNRBJx3}K!<gW1dHJj;dLQ-YVIHJ1lpB3b|zkkjV-v^
z({<_K>4+9LP3?!LW|S*Oyj*IrLX+pxnMYH1e9<wODjfCgL6G$^pHA=NMxhLW#ip{o
znl8$b9kb4{tnS&6D6;uxlw+LownZV@QORB&fp$g9w<xSHT*W0K@pjoN!+T43$}_~$
zGZr12ednM^!{IYGjhx!{sGc^;%kY$Z^I)cStf%_3M<S2(Ts0kpiY9b=%P9#*E<H1G
zqgUzW1g3+VW{Le>uY1<`WK7gT@qH3(1_!LVCdaieiP3c9^eF7n*t=s&*aDZ7xXey%
zQ*ovL9$NlpNm=){T~lj#5tz??Z1JgYeTUanJ#~q5oG8w^-|0B7Ay49!NVSYF+YYpb
zuqj5JRBm@WqR%*GQS2A%2;<+M&$jj&*Z)6tf0an2=V6nMq=iWl$1I*0E9B{Pq-9>@
z3_fypaf`uxZ^?zHcT}!lz_~2LF?#ysLqgM}&IK?0VD-Xj$~LKGhG(brEObcj36Z+u
z%j#VseO_R{+Omnq3Ol_sg|&}`99^_0O`)VP_@+hD#BEz0xvsm2o={>vTQ~7z$efdl
zwG~>I&ivz|y>3Tq+`c6CyMH^mE1h}0Z!OsEG52WG6YpzZIVvp<b8ZT5E#?d0J1OP1
z?bJNyhZ;)79incL|2b3EG^+gIX|~yN&Ooo&QT5%lMVoK9@;TMbJiw>BsD(2kBr{k-
z?A4wrv(J9e$ep3Nc=IF9P&rMbDA$S8wt5uJE_=|XxHqRc{X~FGx6cKKsk_rsqb^R`
z!`fea{fLidMbVr+Ph0w`6z5BBn>G2AkGod2a&PdjZC7R%Ea=ZI(D2<RIlthYfBnbD
z_lxItZ}`RdA?A+YRp#kIA&UciY+gxS=igl|^^HSd&y40Um5D){)-!BL)kunr*|KO;
z&x{Sa{}*5E4q0aOXWuK=oS*txMc2gD-nM$~Fw!`0$8u`Zt}bn%tBGzZ34x4vx*}{{
zgsw)pwh6slx<px&L3>U{q`>4u+5cTKA2~VIOvwt}SQ*#s8F=d&*X(JJl<i9IEO-`i
z_VEs<_>w7HCcUesMSt<LHjPX!dcR@&OdHXYIwxmq+ily&oBphC;-}fE|8`u}EuUHt
z%ljluTl<-z|K{Z$t8Ry6SW514)3bgXdoi+~#ryb&)QG2^cN3VHwc<9en&Le3!z-EQ
zsY>ZfW<|2`7RYbcn{~y!(!yx2=lVl7lTuDA?bbZXwv8d`^VM#XLpfqj5#GKJbz1sb
z^d6hdUo6G_joa!%VOQoZhOJ@JruheToeRD{y?A`ntv;q7g^f*b_qXx-I6Ql^&br<s
zO)_kS2FDZ?YsHQe{R@BYeU&T9@i1ES=Q_9jVuAl})>og{y!>Y>S9fKUlk_fuwpWq1
z{B;E}OB50rBSM&-om+A2#vI3u92b?WjS8=CU~-+ju0&DCEH*01&1}=jD~d+b4vA&m
z>5_AnJhAulTHSALscz?A>eL1E=P%}xNfVrzl{H28eus{Wu~F#qn|oIMKDTJ{?Pt?g
zJdcf8y5qRT+-&i{;>+KoMWm)*7V0uhe3_)PQYd`Mn#ng8S8d)UU8%$W&~$1=+uV6m
z#SZr{L@nat=vnae+tH(ZPq#&$%JMSlD2a61wKOF0o<>w)f1k_w&1*Nz3~Al0e0x&j
zRk^AxFPp<5r*<@|Zr#U|8@)(8TxNUz%!geDb3f1XvHf~|-HF`7|D5q{pLssVHQAgG
zchH~9VPsIl#P#i^`?Ceh=5y<@tkK(j;<4Wk1=X7o3nkpIMuqK2m@qAL(#}Ae>Gutb
zoE+vph_PoixbL~HN7(ku1(6vo&Sm0i?NJLK1Z=YlT<j6S#Ps7Ai-7dZ6F)X+ORkgh
zSZncU3+DkJCAW_OeHKYS+9RS&a|$~zI++EX=u}>tadhp6`F%e)rxY?~u!c(JIPSZ{
z)GqR%@CH-xQWlnbfkzj*+*1hkyW?njw?#JLWH^iXeTKf!HG&><=Uo1I=iJ`76kTcV
zZ3&BI_}w!D5`WC=Tp+(|&CDnX?#r6<H+6PhC~fpQar2bUtR_xbClj~1fj4aa{*OD6
zaFkC_EXbpQsX|^$kySgktvyg|-CXfKIf>ONinkR#KW}kb`u0-Z7N&Jix>Ft6rS2Ws
zCv$y`=sc-ce3DP*X8n;6f8)J!>8!_hy(M$+9cmD@yx4wV1NVV5az}2=d#2K{dP5xl
zzE)+Xxz%@OawoY~uVv->d6|>*zCaXb^qvltv`4r1c768coNU0kr_?pEr8P0O^=f5&
zi{`=WmGM7#ZwT#cbx&mVPGOOlbTTMmn*J`%&@CZ{&R+WW^pelt`(k^JxSPm{*U0#q
zw8ULWblxy2IMmIHA+$d;&iBj3fRz(jr?#Js^`F@}lPk_osnAcu%(u#QlHk79cMs2m
z2XJoD{6A~a9IjVAN3HDUa&#)4JQ9AjlPmm%&(j=6*Nf3pJ#M@e@P45&YyV%x%v_J+
zKSAv(cQz$X6UjZZ#$wKwKl8IZ8y~Obe>l~tc*ZQ(FWj6H{Czk0FVQ&j;vswKo?F&(
z0?d*en<QJ6%g!&EA@2W5rS6GJ%nsqm#2&?2;!13@{rOT~Zw<CNs=T5^puzF(a+6z!
zceL(b!Mc6xg5C+yKU(KM`QfuJW9}=Jvv(M7UDrJlc`y2BW-DLa6bZxIJ{n?i9h?WY
z$_q70tnHc6^|j;o&82#i_&UF^8Pu^z<uOS9=#~95YgL1?eB-T&w#y=IPWwFy;c|Pz
zHBbBqL+b2_k-I`J*<Jge99DDc_=N@4J6l|R-2A+j#>`}Vvoq3q-h!>3KJhYQv#$n3
zn0UVBVB77e7<yolYlVES(G#uOw)740I!j$`b_vWpc-N`tpi{~c-G#1wFN7rj-50y*
z&1rD@i7d~b8AqEXcn<KWR^ELQSm9pbar?@`<>wh&SuE$K9gEyt=wGRKw7P8B=DX@|
z9;Un%WVH)bs$shkwNPF8k$TXZaQnEUpJH3PeuT%Dr0$dynZm(viGerfgI5A?Yra%#
zv)x%;rK3L=cFoU`(APVks_5Fu__#|^M$qw=VOsYblOt~@a6}|7;|@zZ+VCLt-JPs0
z>M?h2glarpzbCD#>_vUj`3o|g>c0Q^R`)#XTp${4n%QEb#PIig`oGKDYu)S9SfbCI
zUal~0u5QFG*D0<ZS6Y*fMhdm&dr8Diz5DJ(i<8d1^c=;^g!rcuj_K`sSagWXE$@ul
z8xQw4eWhNE-way=iu@a8E(g`|uQ=#k7IXDz3BSeM;EGx5)qe9fFNrf=neIPfam_E4
zz_KjMwpTkf9yK~-t&~bVZ^^^GFLG*_YgdWKv^nXeg$EC-Xw04z;3km~Z>n+e%c=_+
zzFI-7>pye}rDO!oS$Xk}bU4S1pHEpbzp+@@J!!A)j63AzbyxMk&l5RIe7zcZMXyfk
zJTd?Di%Z?xm?V815^Xa39)+a56p>f$4%u-?cESGz#ZwNJ)hPHfEDWodzVV{U<&Dp3
z?zPlzYE#~q{z>A^yS;p`INCD#SSJe|-5k{V-t)w*P|ia_nbW4k3u@iaE$f*6jdk}!
zK5eF13{x`06;?B-Fl^FtDA?t7OXB(Zo8H<NUu2xSG~;0ho9LW)Hz~<4ZMUu*cAoQa
zZoyG@$(iC^r;kdUoWDcRzgE=$g4j<Me}A9QvbgY#cl|d$JYQdP-G=u?Xx!vJso6&b
zG;*FTSK692C9XAXm#}SC>+>DD?qAhIUY%lnnyy*G$QIbSU+z9<;q2{eyL&z?IO%ix
zw$<6aH+yCrYu(xVawAv6M!h4C(_DlCB6PhpVxA;LS9(o<;-=sCzsn%3d-58Q!!unv
zwye<G+N(R^(Api@QyZn86?CRdo8|sdFk{6<qu80Ri!N)kTxFj$?Xp;3Ma|3=WvzQx
zT?&!u>ffh2!7AnEBc4eM-cH=t_k~L_+DtKM3F|j8*QgV#rWmb%w!%ka*V;{ui&p2Y
z6nJ!IgT#%s8Vg-nrfE*p30kn`@wxm!pLM6wTo|^z3D}^Ped&b2+FKnev%W>CaN9+>
z`AiHn;6H3BvtXJI`<rgbDNT|erue>NS;u&4DFfF^Svk%nM~h$ZuKDJD|IOrkG5!rR
zCSNOxJ6w^~R-?rz`S6q4rG*MgtNy(RnvgZ8^k73&e(Kq^tDY?n-6r1l!nNnagHNXa
zXVp(rT3_}w<ci<*zspwmyh~WQ^l8k|_PF&&B-Ce_T-+wcntvy?Yw5{%3DeHW6*x`t
zJfG&0{#Rb}#+m+&7jvH`r7duq+^8Ts*Y(lWce3}7$UJx9UODG=RVX7bU!bVcn_AZA
zJ!^0FNlsS_I%w1}VO#svQvHeRq65G9d|EZH%%>>6wToXiR-G;WQtZ>Yl{zQPj=f1a
za{Gf?cFBDwl?K-XMnM{l|HN*k8^~?aNOPOf#!wMeSun9$a}wk2rRNrD+*#VH%b>g9
znuxg19Fvb?4|}5|MWzS)JPBUnzji9ql@+trZau$gMw8!H?tdn(<x4hHYWeVPk6U$V
zMU=+T&7qH)c>e3~mAfhjyMAruu{G1LKO`RYvp^&JHec7>32Jkzu3T>tW9?m`_GF5Z
z);;MxGcUEe=^Z)4Cz|Z~yz7Mblru3(p0RvKx9*y&Dty$TXWe$CJJ~y4=$W=m_;K|3
zQx=wMVrE?K7Hr~=EaZG&M3}Ej3D~;ss>M{Bt0prpZkcwpprKDt;8t?r{?wzl9%w3Z
zUNkigvv?XU(792dGb|wEr{T+QRr#t;`bW;BsLVG?^gWnx@O_BOyU0`PU0DtUoG5Z}
zlu3F!`J>acz=EUtzSmqg%Xay-GASFU>uwg|@ReKdUwHSEZ0W<w9Ff!)Ecv#%zVF4R
ziCkSerMTkV>PK_Zm(@&f`SJgC`Kb%){}n?DJSBHb)V%TG!>@GzW!tRcS$XBZE(l7h
zs`_g8`tp{R*60Us1=h`de#1lk*YpRU7C3ER@L6z5^S1n!i|Z3-ls>*RyE$uqV8Xgf
zb2+X}S)seI^ZEkL-<=W%S~+gKG4-BXI78!u(+i>7Q<@yTrt0R*2#!^h-`ymu!@ja{
zR&hsyqH*_`i4P(p`&0RX&iNko*X_Q$qfzq0_61W8O7A!%EmK-EQMc-p*%OELA^Z7u
z>$*g*Qj&Ddk>p@|G3V6LYYU!jaP=z@c$K1aeI3t>E0T#zcNCq!{avy(N}4tLywTeu
z8m7G(&PRHd{J9ic-;<?s(X-KOrUpOL?H@}N{}<1$dLrh-pvm*i*TkK3N)wwF-)5O;
z%Ru(5o_R-?95;zxw!cBWHlkCq;$f)Ss;C*W-0y0yDwN9Ixp3uQmUVj{GbnQMy)Wn#
z&V04=fr@mZ-C;9_cC|T{`oZ$7MJiv`{`;AKdD}wekDDsjG&?QSQQGdJeDuk(HBXLx
z3e-53Sompu>BWlFPNx-|XM$Wk8#`Si%0&W?dWa{#<YLMDB0qP=b`}|SrzL^fU(aoB
z*`}~_gOa0ghf}LGOP?f9g<0b<=`~VmCZ_{T)t=~ljc!Svs5d*L+pATblRw@_HG7lN
z;%)Wq&t}hjubBNl@+Ox!&orhbKf0r>{&6|<sH^C%Umy75f8V}_Td~ncVz(dh%-HRk
z@?=82@A+$PUrMIsZE>8`eXK|A+pOz5tL|7XWfEv{VHe<=%r4LU{ne7Jl;wUg!85Yg
zh9(>ju5>Sw{btI<#q=-Y%_HmV`v#MaoDDzF9e>3*gn!FP?vk2ktPY>Qaq`Ls-q2c?
zWi5V#CsDIO$F6GXoTt0k8yS3lU5swIsNT4@GS%JF>u$tZt+Vs>qoz5Pd=Z;&Rq}34
z;Fn`+GtL$#1^%gJ5%*wFiRE8Wm=(5jmDa&6(pwtcIh8UrDkN)_v-dehU(}08Ww#Li
zz2UIQf*BX5idNr~u_*a%c_`b+_V`h2ZMzjZcQ^U&K9Vt^P5Sh#OpDmk{}wYFE7VtY
zoPN#n+*~EPhG+5gts;D^m)8GYF(*|>X4<(5tG@XDzWYZqgg(r2Srho<-sX2Ni{Gue
zafIjGs$ScaBmZ|EQmcsf6PaM{%F!5d;#<wq){4&y%(?HMnI;`6b5B7bwINE+@0-8R
z)QO>OYkmfBw(ntZDX=)NXQTfwYkR%T>Tl2IEU<H%Team+R>;abVVZ}2UEBOz%Yoy8
z(t^njjO>=OAv-=KXtwbyx=pb#RMON7Rs)?T+~n59G^s&Ez$LMPMc!G?$1<p45@)a-
zS3&Tzgw7c$Y+M;j3)EN7u)pb%;@WjqBWSTgUgD0<o(8ieQ4@s@l$<`-yp3<$rmJ25
zdz#u!>W!WxZC0CZlq}km8nJAOc6gc0v_j`)J~OS?@$D;on)V=$vGnJ(o422?@!6_+
zW>IX!_p|YOc_O=RYG0oseW?EbDc03yO3$~<S6?N{TlnSRr5T6I7S6c*!0{Tlr9q42
zw6eFIUg~@@9;#cMuFlP67e8m9=yZjvyS{Jrgs<-(FV9r-w<zEepDrWn;j!q!lT)|U
zC7CK-e0|_HBiCTD`+Y8>D#nx(TpjZ9ijqc4ZTPS2Z!HU84DwWTUR{>;aOK=EvzZMd
z<{zwIm<!20TGV{~gw*LZLAMU9=MmDIa)vjUW!46kogaR6^PG&T?$>2wy}@n2d6&e-
zqee+4I-89I{%bEh)@*0uYj`p3l)v5I4xLLLomLGRo*O6Cyj^f}#$;ax4f~=^r<9w$
z-z|JJN7%u;L#SwlqQ0co3q`||Ho*)g8%-un?zXy>q1@{9$xmrg2DcKU&GRK4%a$Gb
zbVlo;$Hc6FunA2kmapJinlZWcM8%XT%lW!W0_H_M^y(DbxM<}TqdUgl#&YFG*_*@^
zg>^i3$LYK=+;c`~!R5gC)-%qRgG?Xu&D!<jf|2<d4xNp={>|#Nl(J=gyVET5^(2qy
zbLMhHb*x#^;gz<r^sQHGn)O-7OPhGj)wZtceYe1DUy0W))6V2~i#SC(!W$WO{N}WI
z$YkNc)cWUAr#o-vrjyHVi2WCCNceY0_`ve^Kf((h_DkhS1SsESp1>27v+?-?Cga>z
zVNQ?2?4x1-bdEZUpSHL-)oh2tjFg!Sa+~81mu%@&6RV!<6WW|vlzL>c>f(*AjdjN^
zW-#s&Fm!uf^eW+2e!ESEXYNEE?R_i7XGLXqZCKW!BO^M?UvW+S;`ipumzGW2HDkvO
zg+p^z*>Y^xD14lh_pD;^tp(=&KHDE~T=zPWf9uQ=4WG^nJ~B%GZXEsQ+hfu-XHT=q
zq2PuZex;M=ZW+u!aij3*yo!b1e{H8PYMtBk;$d@#(W}MoGbfp>T6tnc*TSf%gL}2-
z<{0kOjcAu!A+jT5(_zDqohL4c-dD6fA?kno{{l_J#=O6;`FL;E`l#)alm4oINQbAQ
z>biiipU1-&+_O)7X5Kn0q3zyxW3R%O@h491d>rRt8u$AB{%y<HoP<|$2v}zS?01^B
zdCp-^)}lKzukXny*1!EkJ24}vKl6%^G{;pI)0hWL>NkoeT>Y7^*kkRnNTlh|_bDIj
z#aFx9a!r})!|A|enaLb`qgh`3;S%18z?lwVUu^#H2)f#OPd(WaX1qbc&A>!z;(<vL
ziy4euOfOmm{qmUM+LmCb7Q^FvdY+1E=Z0f>4Z_^cQ=E<&zjEu&bn0WbHnuQuTvEXO
zarR`Hpb3X8T?*|Jg*RJF6tv7$=j5EQfJNO&=rkw8#6-jY(_S3oaA}*Mr?g~pm%q;o
zgEkLV+qPf)kLA8j2ng%awDO#?R7B`--&RLXrbAt|D#eE*;!cQ_YBuYB`=r(Qr9f@F
zl9-TTvZDOlr3Rk@7bjhKB&p-Jx+XehnP@As?|mQP=u0iP3O=!KO^IF7`)g;k`OWLT
zN~$d}UjwJ@yrC}IF0!2G)siXf8$UZ#)h;RK7d-Fy?&O53zISP-6FFKGzS!}_zVTnR
zm_wy~@=4)k21=sY3q#%nD)enVB)Vk60kMD-EjHGP?&pm~_uDN|Hh!>_CrzWdW6AQx
zDl4xzW_aJ)&bDdGtT+0up)OHAala<4dwHc!_~uH66UTyAwz6hh3QDv8mp77Mrf8ZK
z@_W|Ve<9O_E~FY~8w;@)IB&SV=Y8!>5%0TIUefD>J};ZP>!i!8jI*sGE#ADE&Q4s1
zW*xp-*q8k2{_ZIg7$%EcQsLfs=b6T)^Zj=W=J2eHJyho8>1*VpxT4$Rh)C1Cg(6Qi
z3^kt~P5VAicLBS0gyw~?C`DGijH8BfLRV*1S?=A{&_83HW?FWimx_MVX_ME%*PPO3
zhkP~My5YdexBf-Dr)ep@u6BBKK*YkJ@rhr<POgfR5l1CumikPOlJ^WN`MX^&*VI*u
zBX(ir?rH~*Nx^%b9r0j#wlsYGoz)Y6T<MRBNs;=&k<h#|)9aASjvMd4@jLK@DlU-|
z`meA;AfsgFtcD<uZ^}z2x^^*1cl+jYU0I$e?tO4+LByI6#pXBXr-l3#me{z9u_wWW
zOGv}e-J~f_G(RQY@~7I=JKI?AJn&~Qc{VM3pX4=<S2OC0j+}{peQJV@Sb<W!k&)ws
zJrnG>W-8S$kry{w6t4L^dm?w{lBXZGsDB9gsdzS9-cN2(#I2a3$=%7Zi`9%y1-fWy
zi)p9MFjj5VEqK56(3BH7J2MhraFy2yZCn(k{3*SI<Cl8Y)<shL!=wFU+l-SP*A>mw
zW~?~q&BQsup)4dO^WB^7i9Kt=8B_x1a$kR6H}%wQNq1R2huN3cY@2!G2*-*iQvM8k
z)n=+!d|Elb>ObEp{TCZ0B^6R8^EO>HjoH!vQet*|s&tt>4=;Be>m9D@3N!6>8Lb`n
z4=XlKJa5P%9Q$+0cClY98)k~Nb@WZz^=j42X)EP5wRYPkFXarKZW8#{aKnA(N4gym
zM}###c0P9TaMoHdfk}Odhs4fF6LMBro8Bw#y1*gbdgQ%&!I~diw;bJm>xYH8eB<iA
zgH7`f*Zxt+>AfV+?S4Ibk*1}eT|&d*M@7dRBbB%DHF%2My!Sz8NiipbPJb1*#@&kV
zigu2%hXTx<<}Jy)`7bjzHGfmu3XjyY`F=~+7@4;6@^o)I{*x<HfY-k2e{T2o)MvTP
z!gsEh&3%*GzTv_&)d>ws;{TN=mA$*GzxJh1^e>g!UQEkH*R9`w`GWOtqdiVOb6)*h
zy73s-$&dM`UzHx^Zh!2jWaYA>d*+8t7mP1$^sDk^Vu;~Zn7?xx$1{-zb)L>nT}iLV
zV2wPt@+7-me{D6MEmYEGEO_Iv_C}M4o*SDVhy9QDl1XhwB1*rcZr@-#T^_vPzg*Ck
z&2wccemuFkO{#grvBTTG{A*wP{-8krjr+2llb>)%Hma=nuk!Z$IZq{nE<*)7E<phY
zmM42S{j^=4a|s%j9`(pC$WcsN)AQt556h#EE1pJeJMc)M@5-(PKee8k+O$4CajG{b
z`GgnW$uArye{lR3Ii$3V;Y;4S=T>jyKB=<)zw3R@rb#Jp#no@8Ke8+;Q*%4xm2TmY
z9$2v`KqN~^X{lzKy2BbiZ8JCRlr!aP&fcnO*Dc7@F5tAC^mLcQQOzQb#yJ;6vo!4w
zt&6<zcGj2nH5|NGqYfL3<(fpDR(SDlhuVR6YP!}2x^Y^%e4#yxQJx&z6}Vg$`7L4K
zpXi`@EJ0+E16SK3Q8!0!vxQ<FjDlMoRP7!uiaBGpF8|6Wb@fvTW=euAS9L#~YW0ab
zt1v6c>Qs&in@6PGNBaxSJO)j1PV4_lto)+jWHV_&*_OkN8Hc_~sTeb<xXx)w_@S2g
zq5G<e)615L-&&kYbl9|BHQp*w3a-;n-u9@~D9!dxqW1qA3BNoP%}y;B$yxY4s?lLf
z(ybj$qB#rY+7|IfHKxjKc=PYjom;uv7qR6`I<;bsOVtvukf3A1E4nO`od105J#vcw
z=o6KrY@8oBCJ8DlsU&2GL@bnzSolw5lE^j(;Rr|JH~Bn#fda1+xy>3y<yvKK9~AqN
zulOcKb&128MMrWECFK<`t0p%kh+L>sns_7cz2ctL+e-vZUU8fV>FX0|TIb`(Y$(v^
zB%pL5Mcv-%nbU?cmX$2b3a8(6K4(f`VmkRgCOhKYb0xh;JKm&eyRJQ^c1(Qwxsnor
zn_slGC6a9OF1%bN_-dMyVuJdwwBs5~E;9M;lG_u4PCU>oI4;)a@AH3=lGd!I89NS$
z-FX=E<FP(BixwM8`Hn>!_PltL<Lq(nxsif?^R@>&u1w&)Yw-TZxyB#sE@xyYF`ZYC
zeDr?F0Wk(cPLGAMED0i6i6T)8<vbciBN7Gq7s(xC(7LA2wXH$4#_`p=W7cZv$~!n&
zUO!iTd`!*p0@L9p%U<RKEFYe#Jo4U_z_jh~H6e#DM-G3HTV!SSRN>Q8)gNz-q&LX_
z?)`J2*k<Y@@x8A<)h*I&a(dZjq;sqIK*t1A>o%T-^EWykDSuwe%Vk_XX`^V<*&A$6
zl5Qp3Vsg=b^)Ttif@GsKy;~En)V=w=r(bW^+SI7FKT96SGk<cv*#BS)^FtQ@cnK%#
z|9tCJMHaE{4|WYb^n6K!;%k55xX%h1#ade$4yY(_#ZBh9buh>+k(;kYRKTUl?Xv=(
zj#^w0Lxho)YLnLu4F>4~#_jyO=cvR+Jg)ik%)89U(rQAN*#_qQS9<LXl!O+s%cg72
zymU&*iIXj<=$8KbcY4ZdXI1B%R<#oG|9kd+-TQrc7tCb315}cxh%I{6AmDJLK~t=a
zV>6R>)(W0yK4CIR3KohSnq3@coiuX}im~KqM=aFZqV!!vEx6!h@TyeqdL=H&-Ux+$
zmP-MbX07J6EWZ4$FU+x6@tc0Q=jZ1yT3q@=9&Sli+{(cDEdSq|Mv;F7+>;V`UNNdH
zSrBNjTHya=2kvbNJYJh7zH(*Q^GLOeB~<^-$75Z!Ek|s7FV~z<xwGY{=Q0kqizj<e
zbhrNEo+sz=C2HaqkwssYEjYsWNaa)O1Gbm(+<6Mh$B&%xKlks-f@y1Q8#jtFO*P~A
zYHOaTKE*J&?u+$%<>Y@$G`$pm9Qjz;)5#L`P%KNc@(rV&URkOdtNu5ogGsON_N>%d
za_aCMt^40D+WZxKV3K5QqGWBO^x(x31@Fm`6{Sj5Wq;icibXthj9hp`B-mcm;oO(6
z@_{AR7rd49-l`rc_FDNYaGHryKoDc<8?z3V)oWNp-oEfSbWUxv=F^HI=GJy~%|aip
z<N4Fx<U}4nbEV~z*@B|~d4B6zx>Xny7_35~?GI_oc|?cnTwWpO>RmKRN?WbZPqArH
zqHMY1wNLxylJc&tDOvI^Nv_H8<by@GR=Sok=?Ha~-&v%kdRXiYbF$EyuQT4ut#vW7
z=Fmxcs$ekr-L(z8n!J}EO=$bE?vmdI6UO!7p1~@&UwDK%aZP%lSNn)>>48g25(EQ9
zDjt7$|L&;vk_6^$<>$^Q>H9Cyy>YmC3X|Hdu-~UjqnUg^T~QLgZ0@bHNLbL|^Mc8_
zcK)JEd#t}Mb#0!e_+ZL^u5-^>c09ON-1@-SqRmh{;-QFrQ{e>>HpNt7i*8f?Mc;2Z
za9Jb-En6#MvBWg&dq_{FOAe>5z~birI~{IM^0Z6YwqVWr*4%A>{^>sW6PoV6MdkYX
z2^ks1-P`?m-oIyi|G|nWoHIN;wnXv7;rBsbDlB`I`@SnGG4P8z{ayC0quIjR>(l*B
z*M7ZO{Po6T-90z>zCG&RxItvilf<JR)c19o?tA%hR!H6{k8I%pw#6p0vnQWeR&3Si
zWAD@V_{bI?iwUtcK^n2^o;DqRTIa;FwORAKX1q{Cr)1M<r{ba;Uy8gAO;9<|Bdnzq
z&|c(RpfP9dN4Y}*rA@DHIlNk@G}Y<rB8%7k`^*zv9+w?Y=UMsGNIv(H$>#E>2Gdtg
zo7l45Z<*4TPXX3x9+iqoD&3a+z2Qo2)6Ar%E6$2!_-_?*Zdn5Ns>9X6YR-2Sh`w>q
zj(I4yL$asvB_GRyiW8M{w%(khc-6M}jnKxGv;BIHtEutqeOjBH+}C?}_aVMXX?b?7
zoA#gcW^~d2B9|nl9;x`}t0KeeI4;f~X&&}!Zb@DolTLh{{IXa?&7!Sjk!*x!bLf(l
z4IWNDEUR8II~6dkKIVVqOn&-fmc1-N3c`FUDh}M=j{e)i5c%=LGEpTio{6Hb9Jwos
z&D<YG=02PmZeqc<zTUCPCT!6^zNVF|4J+s0_;=;K!khk8@=nWD9l1>&^8cS;X6nFo
z=Aw*0ThGZ`VkfMW($bWA((@H2S{l4FJ+STAp>LjZr<lEZR^z+L_kZ1D;av^hr#lrE
zEfxNBL*db?^T)1O^cZQp^-_|F&}eto{LJ;7rCaO!)OiYv=J_=}@cL9@Ia^8OG@Ix0
zM4@FGw}PfHya{6AQ7SsV$iUBImI|+W0(0KYa&zOt*6Wu9#7*2wDh<_EET1a!XQ61#
zt^k3f%Oe|QcU=^Z+AQ}dC}R3gg~^pBGrud$nz>@-Z{?Dx<y#YYOde_ov?gXRlzp>M
z?oZ>Mv$y3_-pmf27SETo_V~@ybKKS1*o?ed_I^xCRxVv+_@-y6cFYGUbG`MJ7y9P$
z9`7sl+1)Ww$*ljT*qlZ;fiIgr$9;31<aoJYwuGt>!<w~DiT8~UPSyK$OXbI(V21w@
zlclY<FrF;`v*FtjyDJYk@)~O2P1@?*AX%vTW4r1eC+iREMgJ@m)0pIz_`*zZso0Ii
z$m0rJJuwV5jj}!OGE5Kt@why3;&p|q{;OW|uX_Lf-!=Wy#lKcAc35uexcX?qYSBq%
zHy^gftq}ipQ2f_H`8#{gKD;CUEAia(IcHBga0?v^ds40VKw9a}?OjupEDd(<=RcIo
z6Obd{q?)eN`*26^>MQe(w$5GADXjFyLTST!4)v3_QrG^oP&#!b_U6$EAH^nI*~0RD
z>L%4B3tlOkVwtbsxYl>KOqRdyR$VYfZrc>2)l#<M-#Y@dy%eh+PuNym`MyN@qr^uQ
zvx8z{g;7fzD*t!R@IB~kJ#XutY(*|N#<s3KD;NH_Wtw=KwLwO-Xmz3RtmehLPX6s$
zXeqEcn)^x4?%lr?4@U1fzx(XT+4r8_kyl$3DZc35-$s!w4$|U__PML;yqZ1fO~c2A
zjgM6h?RtGH+VjPJ^#H|tH@9=lQM&Vej=|kI?=m6<(tIX4$K~=asFYNCq^k6*UoojB
zNJLFXA^WBRTjJNpK>{J)-P{*TuKeeu#8vj?`<huUJOBJE*>`hc#lMOwZk0yfjxWKS
z=Uo()(&x|MvDqXon5xwkr*zwq+aObY$L}5ew_kWV3K!ivbu{jD^}o}zqgHR7wOVx2
z$@4O8uN==_jX8Jo&e`X6XMaCb_+Rx<;g{lj)`a(>P49opDLuP$HrtUaud}7<c{79R
zlpPHp#hJX$eUIY#TN61+Rb8QYznM}@(CV@^rl+oXZSwTc`^PQ2WbKJ31#*wnU#?hh
zTFX2?c%$jur*CEyy4*FBYiD8Yp5&bLUu(CCW=;Ik?~U4?TlvdMJ)W1G{iJ!r==hBu
zO|x=!ub&J3ergphH)9pl>L_vaOk7#-A89J2^u0j9-jqvXn%T#>k@1(mA3Ahtk^`69
zL2<K2kx7DYR?ardZWKM15G7h9-1p}+qf@ttuIC~b!3KlGLmEMRZh|{pPtWjO;<j<q
ziB)2|qrUzr{qw>w#l2O9^@?_<(NeD##q8Xh|C6*A&YhCyu}$bf*3v^WIBs)j9QIU{
zoU5@^!1aWy!=)+SQ`LNS9nHMFIArytIb9pNuda<@WqJEc<Z1ZSWYKA%mQz9wZRWT-
zP4GdO!^-(_bBi7sr8+UNR!`P&kYGq;+Mp_|(phEvG{B5gQd#C##PbU$ITa_D+zDHH
z>lA10AywxtAMJUZ(KDwhHHxgR+G>`0F=Vx@cA4|>4RWS#-`p6ZZfrcIap2nw6}96T
zzwWNOd*PtgRE3ZUM?Q#!MJ?))OfB%9!oja9`Gl`-cSXw`Ub&jJtmemVZhcYsCUkwc
zef{Uxm-pMpH`p`&{qg(#Q+cPqj5C7jnYb7hH2z}s2x|KE_3{7HYSCMagq=hug-mWW
z;w*URBgysPSd2{5im(jL+@^pK&XpPsUBVn27`=ig98v1lT&HoM$86h<Nxg<jM;b%?
zU8i0Nib-G<o20^ev@<M<H<V30fA7;J3{k05FHF%c-4x~&Q>V!=J#C^&+KgB&l|?gi
zX3o4BlPS~|lo9MP!GXz3qaeNH$kH$3^SM~n7~+Li%#iS0vmj)JQ~RNn0-8$_mR;6h
z$}tMjp1pNzq|dfHCQ{nb3JQ)QYp*D?3Po)wi?O}_bFpXDzdz4rt1<m4o5ZlS=hH0P
z9cR6|bf>e;x|HDQVUQ!l>lzr~=%FCZD#Ro+C7?9QW64PkvEEBwZqkh`|1Z_ZK6MXT
zt7f<=VW)=XiU~VjsF}>SP4QT$>14@S(5`KBHak%|Z^}hC$-FH^-r9lJO4uc3bW*)F
z9j>Z2r1o6uxX|PNaO#93UUqAOdlNOcT#0t6>?u5<bXf9AfMcVLNSMO@rD2-6JGL4~
z$~~Xjz{2IRL43jam0OnkRdrpPe1dOr%gj_~^(A6ms-CB22B_V-)aP{aZpsXnH8+%8
znF2FJ8UvTy2$Plm8sl<IZCh-Lw{6sfpl*hfoC`Q5*F<PBZs4>G7rJ*u>4`x8+N`3x
zrJFY%k8=O_!|49QPGvv&$Gk$J{W~=ocf@;6oxEjTO-vx4qPBFvilF@}9w!quWpWDt
zSG_XN-RP)Uw&T(<5yx*|v=3{o?D*Zs<=Jzof6wGEj|1&D@fijgsF_$F-!8rX$;_}u
zpDR;U0(Wkl>ZoaRZI-X<*EI2X-8-G%i)LDe&*Yk#S?83wPjgdLj8DPyxusenED4-1
zbH2~7*9=H!3exDE;KF&pIB)h;U#noLMT!OuF2UTFD@v4G7&aub&Y2Q*;(*DeoK<3L
zZ!j?^O=t<sWaJD=XlLjsp5%LDkyX%z6TZh1rtIu(65(N-Beq4EHJNikp1Kc5aloND
z|2I0B-de@J-KFUW>r0myWp7U5(oO58+8D<E+u*LdM8RQu?kb*O3n9*B4s5k2*YOx@
zHnVtL;4=E}vw(Nq4p(UlgI1oaeogMHPra`T-#F<;M|njFSMaQkNlKyXGfOTCDfDkz
z=B#8iA$HT5XcoztCyuD<CEt=0e>Y__3&X4Sgw8&Zwnvk#jM##h6jU_@i@aPX9Wh?W
zc{W4nlG)`gNy4G8L$574z!v23V6xDqrRv)i6jeifxh^#JCGsTuFcuW4o(p8!xa$~K
z{e$#bUlXS{DwOz^O-!+Wy=%@BgYc=RHB;=LTzS^$wMd~*>xkXg3kTL~J>k-O^rSSu
zuvz=T2{-E%6S?yOlqKK1Z2MlxlPa<$_G8I(fyH+W`^}Rsc8fG!Sg}E)f@@)hQ_>b&
zMi$OVGS(8ugfcHoap3>2dByqEM1ez`&hzqXv{QwCiYVDlsXCg>r5zk_pk;#5TL0Qr
zXL!`jCaC+WEvRGSIABzMP5r<deFM%0zHY9mN4dPdPFPhb;v~o6z~I#tlo#xqI%S!r
zU+$^qBUPLm%RDD+ezzf_bdrO>f?)H1iY<Pnj~@n3T6Jcpnj?3&SI3Skspl5EOmqx>
z^KxfEP{h=bi}FztT$yp@?2JAQ+}k&Z?~O@h*UL%ZPI++X<&7I`oD&xIew=c^qrWMj
zMM70o$a}54_a*nIh9#<!Z@aP;yC!Hn4O6{jye8lI*$JJ!+f?!`XBOy}u2hPhq<^M1
z;;Nd+)pbjZrfM$Cd;3g6(qra|T(19}XLb4|HTHLu8yt1vxL2muV9mJN|5Yf*ym!a$
zJ)aXk!}9^3^qY-BOo6Ih&2P^qT(ykYaA3LE>lfOgS5E51ugGfpwq!Yr$AN`&b{v{#
zd8^@!SK@i=ElJGAvo<)Kj+nIOk;f6MnGEu!>Kkvk2{h@%opdgoGU?%2K5Z?lOfLtm
z%WJ=sO}NJGt$rYU`rNvwUNMj4{PU$QKU>;T`_h@?_l{2~XDe9yDulEGOP6h4cFW(O
z>7}=9{-x^%;ZFlvKXEQ!7Ik3uo*otckP~6723$N*t9X5~538~4IB3Gu;MekV;e2N|
z9nCE>W13P0j%|t*D*SmYYEo2q5_9*2{GZJt!c{r{rB5Gt$TRhkN7;drmgkxJ&5eSk
zf6n-H6fW6)J8Vt2M&QMskeuT(FF$JSJ9;YU)sAG1ztaN6g}IeGlT3E}*rjq~t%`WS
zWKWCG+=dntRYm1s_k`j!!y|2*x0W6{b^gwi&Si>614WJE=gu)#eeA}0@6>Of*B-$e
zAKG|Xo?aP0O@Tx8=^K7sH^**P-VQm=BzE0j2L%~B4w*fj^4;s(;{|mp>v)3{6r32m
z+IT%y@)W2z`UOlEXz+_j$UnHDz_V%5nxe)dJLcvdiF)#?!#DA%&MsxS%H@Wk%10I*
z)1K5j<ut3-haZVIg*C;4(p0BRdw5)Rqoz;lsaCDllgTdUx%$@sFS)$b{AO89(dBiP
zX6lJ-%lxDCR`LIDGM3*K%5~IFA$-wlVZ+Ud7lc!am+m<=qic(99LLQmjipWDQ*K<{
zTfsjqPE}!b>8eR`ZXyS)GCpQZ{i1C5<$=Gwn<87_liT)hHgwqEP~`eslOj6HA-qBG
z^tuUK7HzcI$(i;xw9RP3F5bm64NmyP>z{}++oV_|=XpZmqW!uQuAm1idcNFn?>$+z
zBS=ALt!ta;+~}9r_Wb>Eb-w?!+xENlmQSrydHLwchBYya^&h#Ns+LX2*AC{iR(f!^
z+UxGR0O?GJZ1!@&OMKpqw)vG$H~p=CA@kw;1g)oL`Z4lGu2VLPvKX1E9Z?Wx{I9TD
zfotoAyw{iHqzt9mHu0@CX#5ezKgXP_H(Y+Z5m)kJ&Yx=Xe>d<YU8wkyCVFFw)3iY0
z^_LTb&GdCE6y+BQ7|p2NE-s=XE~+9XsxK71Zd<cfyZh>?MH^C;j~tFNFpH6Dwec`a
zR#{jRdolaQ^5jFRwq|Cc=35Oi52<=g6WIIJHfN(j#-ZS(i01uUCFhzluD&XIA+RQ<
zP=l{sgG<CD;hIFRQjSndi&wB<>SbYL6NmCPq1jHR?OV#V7uG)%&xlIk{&!U4{w4lJ
z8@MKl3B9{0xFgYNd!gW(3l*;l1(P>${SBy`bcm1Z2LG1?U7e4*xy6E#8A5Ugx#VAD
zgf9HAAe+da%plC<AhjTk`-hQ|wxRIO8N$mx3MU^7t+`=f#2$TrdR(+X&Hn94s@tty
z0}ZW(TKOMWpJ^7l%3gc>MzK?%?Zq}BwPur&LQU1hB3Cmy{xO@Bc6hz<@R&BSPL;7Q
zC@rh&V9uNExw(g9mj!Z_ey#aCJx=wbPqnE~&k0|zEeg|}{aS;$?-_9|I>5!cP4I7l
z;QwP>ldj7Pc=B(VP^sQHajOECEQdMM1L@ZZLVFDgwl>L&YqCxJp|HS$?M*-d$I1!+
z7!$q<D$f60pe5L>&?IECLcr27KxQ*%uwc<-rsNG`hPl&~war<h7qkg3R&v`I<)@i#
zrIf8dO(;<Czp~CG5ywe3@s)j?j;t{crnaXfJ3bfnD(|}#9Hjcu_Qm3qM5mOUUz1;6
zR<V4lnrvYgxUDyLqqu5fNJC-B-_6Py+l8KA7iB8Z$X+g#dpva7B#9*oGk0`!cPe(=
zTP{!*+MznJJmlldhvD_<TTKI8`2TGce3PKiT{dy;j|qzdxK}6WPEW6#x?N$WJNIgZ
z*-9CeUmUu&-IV*PFj@RZqpF4w*NNCk%Vzm)5p6rn6*NIuTqrUmkv*VQ;H#6+x(UKc
zPMYqOJ^g`Gq8$v~4V5ibrt%3-jlP*xqmr$rKJ{vIUszzY>P1#@W!r<_rk{U4?f7xY
zTbqj&r>dovi`0Ge+~4|t{!)>?cNwBr*d+Ulgaa1~d{lS%IbGmLVBRlxnP1oAo*XL|
zGKl}5Zu(0wpTnq8^hKw*BHQZ+g4Yv;xPIgp9B$aFp!xN{#5EWAO}!>=+B9)n(c+oc
z`S)(%N`4@>E3#4j<?Pi4mG)eEuUiT>?GW;+suX%Kz2H-~Oj6Yv6Up+00vd-nQ;mx}
zI2|T`unw9qk27#e?)IYX#_A@M#BGuSqdHr5nA#X^YtcP6fA2LBi)ljE#|&yt%@1#~
z)p1qx6ST8*4NiQaYG_(g<QUu}m6~m|wCl^v*y|y_r4rGNW!EaxG!-36JgnUnH75y4
zi8ykxgv?Yu=(u)*>8TavKQ(oDFaGE1+a}L{Q*O?7DW>gQD=u{{v*7wGu;lR1uCEUK
ztC`aGzM4IAJC|T&*RGd>8&?+m+c5EVk$mSfLHEFj&UVYt64$;~p)!UkQbAR_SGZ1(
zET5C4xkR8Sb@4i(mUWXA=WUoa&uc;3?v!bVu4maCnpdsD5E2=v`zfn_X0`tcVXd#z
zKkm$a<{`>HOY+tY(NN}&dl5n_S^7DoHePZSZFY0y6PJ=aC>5mKV;U^fu3_|0Y*Q7h
zlb{3Fz6ZWs?2}m38Ilwx_iu0ba$(Y^z{PSqxTGx?UlWsO{U*q9P>xGn{^!RvT+`)$
zBo<6q=KDXjiJe{X>ju`vA0m`5DTd6}*cQM&yXpVdUmD&w53%k4D4pxZ7IjL=UARbh
z`+BaxWiL%x6~AZs&lD(SUcUKQ5vSqyT9fTs8v}P7wQfq;VD&KQ=8ckMA+Z4S6_#Z?
zwr!tY5h-qUb^6at(I1D)e{Xj74P7u{cG*&q^k^ydi7qoaUTK&+mQP=~>hzVFDuT1h
zL}%4)5=x)SvgCt{ydl?I_JWxQ<o|t`@TFn*q@sMw>dIX+J9lNf9NJx>_)vGdw!*8H
zgc;hMvKv{G6J{C)&P@~zi@L(K?gE$o;=M~x?Y$YqEomtAHGz9Sqj;5vpvIxBPZjG`
z53bM37TNK++WOPJ+!;LrtlKxY^-6qhG23L+ViNTE#GcrP%N2tE_de(l<vK0;NL|9}
ziEsf&%sY+)Tn{CfR_(jRD#dhxZ;7F)?39I~QWI>IH*p#2a5k!c{+_<Z`_NG##z{(k
ziX4@)j+^IuOx*Bu@xF)o{7mv($1VRFuH|5qfAvJ+#g)8$s`6hP*`f@zVrJ{c{nl7B
z!8@T^<MtuJ_l>@nj#zXp7QD7?r+800>r#PDN{SqcoPnOpv>Me9v`i7$vh8)GP}I+?
zb*+KlxB?F^7TUU1q{?c8?J|R1;qxON*?1O;S8A=WT(%;)QMB5|u8(_qO{L~UEu+a&
zI;NYC1qrun2Wq4&6Y8>@*=>}ev#6c-al83Kv268SoPxqKC9Yg|(sOlKy(0fhsJrM~
zkk*r%IGf9T(oyZsdqHa_ecSxfNN?h|>DRfcI-i9rI&!@<OrJ7S@W4c~%*6>FD!!AZ
zO*nE!YL%h${1~$rx5S=vFB4D+T+4Ir_pEu(G|y$OnWwWT;LGmn$C=q%k12az+-~i@
zfzik4T=I#o?gQ7=gN~c_#R|$kQSW)1(SLdAg3H^51m8GtHf~&SMMM0~#-)i;T1*qP
zH~QE<_6ZhHPPb9_=bZ5Pn)=t!_zi)U|58u)xnFvav}L;Rk+x|9r(Yj_Gi}Rble5z{
z`x@k4c8%ta+#vO#@aQEM-^;Ij{~hdLFY^7u*jTtst}jWl@v~`M&y-mkSSKxD&Ah#D
z_0#@h*Z<3M!;h`HIxk1cFfh{mjD(GtXy9>XmGFtx?nd*!`Uv?MrMQ-ta1}<m1$k&*
zEz#SmR(Vpb^Usbd&-2mGW?HjoOgU};zql-|J&k{|MswRnr;VKY#li~~Ub&d9J^iH+
z*PdM}Yi?Fvtu~&-p!Q8^L9Xc0O{~E#@?P4mhh_9a8`u6^Iq9<_f7Ax9CoFo0bdLN^
z6}o?$%lJuqLEE}TQTs$Mp1pT7;Lk0eKV5f@PT$ArTCcn4-1@S(Q@+vvHH{95$L!vE
zzP?tZZr6fCSBsUWiafZ{J9kN6rihhQ;|vMm{*1!@+=EA@bQYeTC?w`3B-^vAauGw&
z21XXHsXuwA+L|1ia{K?n+ZUbMizD=V9~3z8Z||C5wDzuA#F4*&y8jFo`FM-i{IU4c
zXyLxM-PmMR8rR+_f}$68vo<IzP7=79;<HRni}l<SnH7rbHO@^5)@7K)P_w>Gs*HQP
z$#S8BSt5mhlwZu4rrmhFu1BaLOT4=3iT%gqoQua@-D0k9zTUF?df>L;^Xw;^0@eLP
zPqK#y6=clqDJ-AW<KUdKOk-iFgo(>UTdte~p{40FWm34eh!`~o8gnLuH6OGx{VWvn
zaaDDy)BNeTTzLIV|8~u)@L0R{))~*UzY}G2bSozA6MgYWP{h(je4}Kalb}e5ppKX3
ztOaLZo>KnryUa~-{qf#czq_oH6aGKm{`t<NuaC~W*zWmbdrwl0_s02C4sPJLo9^oO
z<l6C<u}mjCSL_p!(K-C-26u@_kD<xS?e{_rGgKGtlj4?I+H;UG<(~V}h0mVNWcj6+
zt`qraWt;HaIJsXwK?1&~qbEzO6JWAP_;^tMpIMif<KhYJ`DT$X+!Y&4?!Ie(dP(=}
zB^w+0)dgCdb$%Kf-!q)w;`(<nx7@bOg>qXR9k&Xu%8<Jn6Z>lEx$32Q=l*mZ)3~JN
zcJWnd^VIUwJ)+xeZ2l!xdtcX_WFz-=yM@~UwW+-~%y)cUDeB2sdLh}aRI((ma=LzM
zX@XE`__;mzuc=oD8hdW&b4i>VGVxx@I*q-xS@B!{OU&mMcRsRqdCtux6Uvuoi<WGP
ztZ}Q;-L#NdDx<4X=x?BQce?;%y7qhxhMkpyXSmWQnRm&bkP`4bvqvvt%N*AC?6vZN
z+}HL<)>#NnHsZRg`BpW_Z2r7}`#e(vRw_TR`FS+F=B}k?ia<ro+E)*bHOHnHmj1~u
zcocPJ`rSHz(S35Y3_K<6M?6n|`||ee%2Row=i@GLb}w+9tSOi!b=puzWR<YZ7N4K(
z>PLbO-7~i@-|FmjK=$GjZVkr-!N`f5n<g>|Ea7;R|Mgy@(XGa%7F@3x6duZ7Uga;+
zD}4Knz+Z-C!j7>DAL{hZ{`#<ocW<Mx%=h^J8bVULnU)3e3Ph{?_Y@46u%MxlX^Y3D
zMVY6UwX+`YSrxi|DnroXEglae3cMW|g?RE5Ig6M*rf3Eic~%A;5aW|e=1S<?q;|H+
zJoKH<P1Ysnn;clDFnWrlicg!{P_*mi=AIxwg?zWSQzm82T(NW>m&Q@ilq*dM!DYG;
zQ=dm}$qdyBn7Lp<lM2_8ErHSDs&{uQ=&wpqUGV<s{yI-iiK#`C8xD7JPm(nK@#uls
zk`o*oST}8I2;0Uc?voV0w&fh_-CbXKCYfY#w!UtB6Tad83d7e~Yo(MA$gehB{pV`(
z($28grCY+*UJB{ozM^ieE9Z*q{#UfBmM*;=yLOZ5tygEbf4#o0Exh{m8rCg*{r@+W
z7FT_J@ip4FUv&3JrW?JlUPnF6tA7)5F@H~H&$2)h$DAp?Y|5#VPjWGPb{ncVzFe%k
zq@#dKQ={jDlkc$v0f(>;tJAkfd93tZsyW5N<B(=fRHl{K(nFf+Q`D4Ctq{xJa&gt%
zl3SiSQ`MX^4dhv8=xE92`)D?*Y|~&}Kes8QlXt>`79%&tWmC?1X;$9qS$^`?ihRQ$
zy<I-*%U*anaB)uY@H~6LAt-pMIcq@ixu#1#+I}{U-3ncXZ$2L1b64%v()m&IRxiI0
z`s<`WPg#{@sz_B<Wc$UsYm&|L?B!*rOnts+v+mkYc7HD)3k=q}YB$AeuU{ae+Nb9s
zA+Hy<%Kndf@p9>cpKg(^*8}UELoU{3ROT(eR<rS}d_3RV)rL3zSo17Bl#ngD^stKH
z2?a*JQ(=m2CYdYDbYJ`yxbZRWYpaOnmWG0^eitE6hO+Ws9&0B@tO{?K@u@aYRqt16
z;Mx^OgtAvHZh4Y&Y<|`vLyea`M&1mKir-#6+{985p|h!H%fv0)oxD67SyX0BU%%y>
z*U6M#f3pJhqpL(dJ?*}skeX)Xv2)GTD6UgE-skW6O0K;;X@b*%rd5yE$uroi<V^8#
zeR<UO{)<Rgk*I4kXNg5>{+ckitL%09-uZ{;@!4mdPfET1x6Id%f$^fU-t)j|zbDMw
z;jDTsd)-W1g99i3DLV`Q4`N`yW6$oWq;V)G_)J*0#ukkeioOwy{V@)2S1wiV)6+R|
zWcnl5JDZhqWo9m|JRR_3P3zIH+OJ2?am`hi^?V%BaMN*8cTZTry3?nc_NC1I-c<3-
z$ZJB@vTUEZ#nTVz&cDwqsq#FqE6(Jl{kI*-^KN(<H>tceoM!rNnb-l_K)034r^jBm
zl-?DczIN))=*iM0d!8it-u&nPe)a2XP5a!-tS2bEHk-BL=;||{x7-R@BLAl0-25X8
zX1`nG#IsGRRpQ+BSzJ|Ip$ZS@=Qf0L|J{(Rlx)${AoN*KcZz^wAM;5EgXwy0CD}?5
z(pTnNRcOCcSsknBI9Wu^VM23`f@|;$hyT5C>I~B#aJy+mJrcY+$*QweRU<*rOJuFa
z(XN$Cv_fp=iW;|x$u&Odww6?WeQlD8@gh|Nt&6daFM{SDTfpkrtSOLY5TtrCC(UQ;
zjhl|QrnsGZ(~`V+Ww4OSgB>4wPRqah*v+tP+MM;0?Dhv9`Kg|D)|rsvu6y%j-`z)t
zcU^5MGJk&Q(DZeOWusi)IiHp6FJusz^RTDMcJI%#c~!yO{)<vfmxo-@%2wRzp4;Sb
z@svO!OUl~mOT3*<?Gb2xak$TOnTkzU)m61cAKWLjq;Um5XnOSK(UB>R94gA*ojq+#
zC$6?mjGxE3I3VMRu+rad^{tOP6@#yEsRgby@hKAec;Ua1WJ6YtQBFeZ)GW1SlOGmW
z&i;IJ7R!a17J>ch`I|03yHgT)+t7rw`cl)Sh~WM2=J+?wShR$pV8^N-Pl9GyAC!;s
z`l>$TnLop^Uz7jWFkYU0?~0k_*Lk)%f&9^1l*@AusB6D)V$&?~2<=Kb&U}3Gw$Kn2
z2d0SQKe9af+4;C?g;^Gq21(8LVp=WU{)pGW{l=R1oP&yYvpk}Wzxc#N1ua!bbryN0
z!musDpHsL)+<#KUgvozO=O|nh_fA`MWO~6vWu2Xz7L0DHEk9q+VO8w%*NGOHvqM;Q
z?t!kOGTSDv3R$YnFa1{ZXouevmIY<JuAWR2uK3T}vQaO7(%LAWvWfr0&iGW=1PV;~
zmZUIqp-TF;1lM;bI&ZyPpvJXh$}<(mDH>CizSqsX(zmHjQYoyZQ{?8^+;?wgNPft3
zbiDOab&-LnW}wdn`NG3j^mcAG`E|r-SH{Do?PbCJd)~H-9(tmm#PnU`uT${EEJgmh
zh=YgDZCUB-b%SZQ$n~?T&phTOS?7He`DUfz6@J)yp<tktHbcd?CAwLO+>=xebk>+{
znr|rV(6!mbW^2OAqes6kkE#gU+$1qY<hXZ+wpf(y7L)!(7o0s^GSg?B*J!Mc%5e9#
zwFvhqGu6HC-RZkM`RUwaU+S4&y!5Z1wR_>a3Qy}}=`-&tFj(I>#JA4l^Q_xv(kyQO
zPvTDgu!22z=bWz|y;*-3q%tU0uHL-SshQDaWyOU{2aCQQ7Ee;u=14c4^Zm*yCDS*2
zt3#?*@n<P9Nlu+NE1+c>*YYJDg4ZT`Crp|=^F>CSyjgDSJx}4%qQrz*x;Mi3%%(It
zp3#WX^vJBXeR%oS(jGBm<r69jt#6gSEBPGK^{6Q~vObmLJv)%6N#w~4Yd5yzZ&#c-
z`(sBdgH6d{hspC47Y9V!ce9@|yQESY%5|kIX8Vp?Z}Vn!amPhJxGlD=z;)lxTFGz@
zg$EgICugzdwTDe`<nCshzv#~4)v;HzCcf8>iqQ4E%Ug8m!u%%|l|irWgx=b|L`FX<
zw#t$#N9#txe_p>%^TxGG$Nkbv^iLUhcNopo>f7yV$iCR7R7GULW)4sJ$Ze;u2ra4O
zD^WKxdvW?e%hIMz9LEFMg#(+Ty316~A3wAE-q)U(Cv$6J{SwQA<AN70*YIc%SzlCC
zk))NX+uX8DUv#P;>x|%(pI4SU9F3gW`Kx^Y_D5&((l*cH_OECTS9{PcZsEbUZ?eVY
zMT-qvpUhWI)%Ms{yYEutr&U`2e{QY(^mfst%thG?6|SxGWU3WCu;TnAmmBY0;(s;8
z?su}C+fgWS<o7AdfE@{uembm)Q$jt%y}YB=uejW;vuH-i7V&p;pO~g>zkbs%Mf<kC
z?e1S$SGdzks;8w-d>FYe_rIarCyki3`HT!!;oQg1#Lh2U%~$``KurFr?4qjeN%xWt
zm8boBTVG#o6TYMM=<eo5V+U)225SWiW}Ob!ngdfj9GLHHH=EEjXJ>z0)%w^d%_m*<
zleKxj{g}1B&?JmuVSG|+ZxA=z<Oxj`8>V}6&tu-osocy}EEMM4&68}u@ucO{2M22n
zcW?G+x_)T+y-ceqA128}TdB_JQ#iS9^9(b+n>=-scuu|8=sRPs#X&Wt!}G45T-|H5
zByfSzlm#ni?lNLHq^bLRo*uJSH-oW0<DUJ(4CV!`mUqm}jrW+Zb}~t}vDD#UY}q5-
zxjF1Y!`ABcA&Qp4nkKt8bMJCC3DC54`v16dqtxoC9X8)N7kVrfjg?w`cgA$Z+nn-6
z2G@f*UqvtH2<DvrV?$^%=du?&r8up&2<*GNSz^J1HF3AMhOO>SeB9a1xOU48*B8;<
z6Dp@|@H8}kWjF7FT?)@U>#Uj9MY}Ajdc=P3)B3Wlh1*=$WN}W;<eQGYDiZ5|d^pgv
zS}(q1yO7Shzl*n5ALgk)yuGga__5~A&Ds`C#uil}oSZ(K94?)#J_lGz99Vk}SX^c@
z`ruIbg{RkhaX?a^;1j8^g8A(+tWgynWf8p_llY%6Qt|pFP~gLv6QwpmLaCxr@L1Hu
zU>@n?UUT^_@0D}WdzvlylzZ;<oQXRoFx@efe2^))<i9uf14}I%t7CK4a5rSJ${wD(
zz=_jf_n`>}M$1&JZ6vKlYs6Jota#-(iQ@*#p2bG-7p(P*o%bsnACTU2IM{g89%tL0
z<Hs&LzHHcRs=RP^(WZ@--L5_^n%TTwTU@GUO#ObsrEB&UZZ7Q!Og<hty)wIcx4gER
z7h$_kt3&CN+CitIb3GO*ap|a3O*j&@rgkQuY~s;rIg8HBS#<hUQ}aew&gwM{lHCcv
zXU6JuFeGqqW}d0c*c)UrdrN@Zk~^JminqNxt$X-|uNKRsjN2`5tTfC{$r^E(D_n8B
zdPw`K7XvE;!}VKCVJEi7U)q1?kzVy>`?5FtYq=d7iucs>cyOi|v;J4{XZhmXkTREb
zi(uVu){cl19!tFEeNf@&nHs=#(Bq+0RHes1lY>nW{S!_*^!BLzI%2`|rM>JzQ_h`3
zDjNi*R2YbdbW8UfoHE05-i)(TB2LO3)Y)poEick_;p4$A5dp%=R_T)s)0jEcI8UXR
z?2=-h*QnGg{#$T@#Cqx5Jad*fJvrEON3r$h&0|kQ%*{NxpNlwMHwscI)K!_Nu_uU$
zb%xjMFV>bR-ZpE3s!MEiz8*IA?mXUoU@nLM@!LWAp`3<iFKu3O_+`W%6X#V-pS1oq
z?*3~z-+7};6z9UIg{_NbZt;4|=f%8I>eqpnD-`QDurhM;PCT)C`5kSgq`d*&izfcB
zymIhU$Eh05;NrP+R&*+z+^hbJ(=S){k+R+L69KCtx@K|gTXREjf`i@DY&#t#GhHv=
zDG|OZhgVMsWV&%{!|h9FlX-gfP1x2~VwJXM@tr1~&(16InDyl&FXU|Eo=|x9TFBa`
zCf1x4a^;+y&Rk0-cs3|~@~imbT>a-7v&xPp4u1pQ6LmhF(F=~V`dn|45H8=!Q=!{Y
z5_`QNC7f|XI7iI+T@wC}CY{q*-TjQ?M%$isf7i_FJR>Q{bE9XAMgQ$Rn+tAC3hw0J
z$gUoE?d-)l6H6?&Xq=oBBRlQGvdf$rCuXhTTRU-PiK}?#@|ZcxvrT7;7#~yh;S_s%
za=KvT;-ZUF{@++T=k26N6M5!zZeQaqC(qmaBqvOnRZ`VA>bljY?H%09Tf0pfqxNiC
zVP1L3bYi!Wgz&M~LB^(wi(==V*0?O~xjF0xlaaxKojvlFYdgI59I?K7#7An%spK2K
z99=r5^6jX(VVlZ+&C7NQhwF)yGtvvs9IoO#{PxEAnl&8U5`uU4D$JZ1?L6b$EZ0eA
z9jEk$t~c0s^{3K{;D&@bve&K5)?bh)QI_1Y!|wg%$O#rb8J5TPF6hs^GWV2(*0QZ~
z=S#b9PhF8|YL-<yPuh}~bq0gFVz|SvTbm8;tqbigDP`7sz@X=F?~=fg4vkrH4`%(D
z<MHr7+&V)|hObhLrFtG$_^$r<Jh9p0#LqA6rxiO`efRojJI=lkF_}l2d#bp<tL!9K
z&eOU#w|sDva*SzH+_z%EL<L(RHtuV4Lpdc&B{do&=WP%Y`>k$MB&4xr<4J>=vu5%{
zZ(v{YCQxCH@b#YVr7cnFrYh>OUV1!1-Dl-kJ%oiAI5!@3pL*kBnQQ0zy*<Uc9lD{$
zF9o7GuOv3kcAoX1LGuB_I^W9&gE#AHsP5|#VZ3nJPE<Q#WwiOz1@){8cF#S3@QsCw
z&5_BM7G}44t!~wLIgv^6kv0#jtoY999<j+>u{Ztau8Dpeua&f@>1X^U72kOQ`_@@R
zzMOFN#p0{WQXT*H>WVMzzQd(ARpHnp!7$PPqCCmDkJP^D6)GF;Pu-t6l|j$r`YDBb
zUwBR!-gKXEA)G0MvwG`w*{k7|teus6!%dnyjXt=WXg8Y~q%3l<FwbBXn!#YWHC*am
zP{_TY`ak=P6gn7EZ)~{r$Wbibc+rx;4~qp;eZ=O7Ph8W@pZ$1?#kxsbE*!UA{_wWt
z%SVrAUYWS!L+2`m=XaB@R#^$(S;)<1*Iia8B*uAartPlTtipE$8`f|~$;t8vxp9h=
zX;=kzvL0U1z3SFF=Mc^S2EGZ2#zHoyv>GM)u1}c}Ykol8Oq*Hp5yN5+2ca2FYjT@4
z6&epeZsI%<e_p_a_k}>%!_GaS+<!7(x|gi%&5oGX<$vh^l4W5Vv^O|&yT+cmX3V+3
zY?4Zl&e30zzNMyfH|Y3$<(Zr|@v6)7mnT-AYu%SAYt?)5?7g2e(;uJJ-niIo#o{mt
z*2yo;<Z@XRMRuq7y||kELhR>pBZcPPYm!D4_re}9#HOW2mpwEpVBYbp$#{3Gkpklk
z4W-C?O}rmof7;V(CXlr6S;NT!R%V^%NolFlk(;A4Si?QEKKpVu-|0Lz?Z%8}Z>}od
zXNq2`>+x`%_Y3)?o$O~jIehP25}7M5)5Wd4Z>~?;lqt*kUAuEdBvnEJHA4c!UI<;C
zn;skc<dVTVjWrME&FzW0@PPN9P?JM;{j)8XPPlDW=Dm@kp<3ZH&tYEC|GmwpJvIwz
zy!;qu9J$O{?*_xB3}&N(IH3$yYlY^+YK>p-F)7?@lH`2Z!2Pmn&kKcP?>9|(Gdm$;
z+c9@j?b`-6q7j|)u07|Q3XW|1Jtd$q(|yh(-ctoXbM#zqxA;o$)(^An`lKiymAtjU
z*8S^FPg~O)(?ZUhOp;)^Xd|}x{Hv`?9lE^##>YwD-6GK4{_|sqF7I=lOEapUPcis(
zd8?i3bh9UJ0<LXB$2lf{Qtolr<FNZ=xZ^;sO7uffxvjqIoT_|N4f`F;H@q<_xM%jD
zX?xseqXYTj8qAy-%vLf<k$w{+#o|uHz27^J;kl{o^R3K1b#eAC2fm8kGj4uk$uCs%
zpEtF*Hu3mxbGDl~y6*#AH?oV&Nja}jSSaQBy)}Nu>|Eu>E5>Zs#4_)AmM1HkFY&!_
zZvK?W@bI|`6AdRGy?JTDp_)&TZ!e@9%w{m;Te5QPyVblID|mLV_`7RStk>0Py3Ffe
z7}<V)_LRwz{pGKFO-i%gPuf;yqTI|GQ2a}8f3E*#qYoM30*j1xa71on&Agv&x|Q|W
z-3Fr<?iR|cHs=Uxh3;T#5^bOB-Qr>rxzhO8B)$ICDnA^~?6TN)G(_gT*`%Anix%B0
znKfaJwB@Vy7q&jQy|?b;^d;vy^TN(<xuFz)Mrmq<A7jV0YfC~)-E>oRVsF*B9eOZR
zTH4-Psb}AhS9|{-n5nhrll<(GzK2Eb`<o-*W&i&6SyQ3eY)A9gwr|g-Ixn8#Y%Y<s
zRwFY!gZW`Dt3~-6vk6bTAADxy`Eu$3!{292LJG`g=C^y@zG%vyH@H5-`i5lv(Y~2=
zE?<7E2)(fWhg5%5$DEGPG@WhTOsu(`rs7|>cDF^T70pWX<DaX%%dt1m^put?&*K0=
z>yyFZ+!0HbU(326YaA=~dy>diNiCa0(+>*GQ{2DC&aKudVM0J`Pr^Hy(~HA0w(t9`
zJ=I(Jw0oz$moTRWYi;aiT~#Gh^_RK|T8h((D+-%St`?g&aJqc|wfg@zhw|p28x6V|
zjYbu(jVITcui)9B?mtVTIq3fk=X>HO>aQf4&wgma8*HL{+Dt>0S!sjsnfZ0w`J*IS
zO%+z&?U>!PVitQ6zu^Sc1_6Zvg@a8@)si9?^c4@aaEgn)xxRY-{<>%`FM$~@2~E8$
zQ%j~Oq&7|AWm@jzIV0#nOQX0fmx<;CZ^xNTDVOGGK6rYr(K1o$Ny-GDL-XC(%N)6`
zs5>rXI>qsXCA1-=ap@@z-K`=kg4-EbYIv;$9GX~^wl8YEsik{+E7SFDilGIQ8+=_@
zCrz~W%9CcyW9^H|UY)QuZ0!<_4=2oWnHYmke26&{*ucce_3NI}i&uV<e8P8jWEnTM
ztd5Q`o3L?ntMkn0b*FqX9(TFqgy#iq`(LQs*saZ&aw<p9IgPnfH}%#H%Y8e)7hNxB
zyH#j9V}UH=Ro1xe?=JKncz$;$cZFWvg724>M(0YhZjReu#r*nWlKKKU?Z7RsehNEy
z3+pn9{_QpDQd@l?IM9AkPf&tA`#JRnjnzBUXSN7)xd_%3X)e1FCOuWfHAQo+ibIF1
zq3gpI@rzR~$NF<R2#2^hnua>s&y08+W-?7fk%>v+pt7TEZlZ)X+t;tx9BLISQ=D=*
z4lHT$`Sx^4KC`E$q^66g=7p)w+k(BV^CjEjgd{m#mN{~1EV$_4wMH{wme7WsuF~Ib
zY;sLv+K?GAk7a4+@>HQU9CLN^ayB|@&+$FLnxXpNC-6{q@PrFqlB>5Sv5O0@*%l>v
zb>f6sx$ABudY3HNsrV{knOpnZ(!Dotta4m<&9cq*r{ENM-Ng<&5_A{!1p4c*?oi~?
z{&7+Hn3nL*;PVoTFDU7_{#~P#W4m>O@-ZuJw`gytpBJM7wzD{zT-bikP9(5(-;ZE#
zkyNQA4;`esOdLCuw`EQCREtd!W>>w`lGL41dNnj8Wa15xgKXTPSAAj({si}go;)!v
zER6F)Cqq<X%*JCZkv1P4<oZp+m>4QvX!$rxeGzxmTD(<EyGXU<x`xY1SCLb%es1MF
z%_JIjT8#5e=B7x_3s<^y%fFrsOP;DJu)^V4rP^!;;evt}1u_5cD86)!{!`YTo4TVg
zMmjFs{oCb>m20!gZdRsmx_#I4I+tXWkGfvnn;yf2My4Jmuj(}!m%eY{bBYMm{lgh>
zzvkbDN#^(0tl6}{{=lL+0WFhScVB(VXFGe1nDjyG2{TzHS}to364=7JH+t=nqaqWZ
z{wV7-+Gi1PWaDkQ&2M&dPq-3fTjv@e#A)+QTsugtvq*c5z*e!z*>_ou<LUw}1-|$u
z=?hE_T2-^u`I2}3gzxnm3sp7Ox~y0mAGfci>Vlfq!UmxNtpg`@OEj<DNcsKPv+B==
z?o(#VZ%)y+Sg_(?gKoo;O@0i{Tg?`1@L~`YVW{AmG`I0eZj<PfxvWc_xpW`=Umh*=
z^7y)@sSDzIoOr&xaNm0EqI}qeWS&JUT3xOs>M}ma6*0MURU%A-A?0hbz4Iv<=bS4O
ziUgl@ocPqP|3Zk>XyeIPzQoBw9+xIHu36`1_V8rjm#rdgXEhX=7I*dZDGA(q_^yY!
zaF$eUxWIi?pWdP?Tzxui%8d~@PAQ%YZ!AtY?wFu7p;(bav*6hD&p+MtI&7vgumwwe
z;c)0OlbXdM<iJ+FK~SUj<`RZ8iyCT0Q!Ep^{nAukNClgzE6!Z$ch_s?gy&OytvC2^
z+>Z0~{`5d7k44hIyhw81*)5BBg97I7&~WPCcU8&Z4TFrXhlK*8ig8qOg8Qbhg`E4=
zDmnaL{NlKw>q3FXKkafiRTjysZV~9ZvAAgBoB*L^0YcN~uykt``iQY^(l?&9rr$PD
zusg|Q>HHlV=WN=YEOC>G#V*L7!BvTMN?*$+9ZnTb!O)<o*LJ9!3<+U)Sj;hP*^-NU
zI2Oom`!UJkR-%Mai=N`I#Xj?GjCM7i@MnzP!Kyjwn%XHJ!(|z>4zY!cYH`{HG&!Z7
zbLh~zu-J<&WZMQ;?JW*mB~RUpgajKTYPNQzO0Kp|{-MYjyn-`q!iCxUqE1*X`>D+l
z>Y>Oz>sru`R<WXA?J<v)9$8wuU%I5~GO26keUC?x=c`mx;=hY9XjE=u)%(%Cz~SKX
zq(&z9rZpOxe9ix7w0zpJ@${}P9Z!j7tBG1O&q=w=eK1)_^2H<RIgb7lb&8i<G}{<-
zlULVcW(=3Q_GAX<y<AqDD{m*oyo}`BnyY>A?^4|*pJ$xa?X#Pik!Js^%=cT*qIOZo
zg;jqJ_JwYEWX~P>&`Iz4ymZBj{ROUhQ5RowU-2w5^i()07Vtg%n&gqvx+jbLPfVC>
z^kfP5+Q~hK>#9RoT3FIuZ{2ifEwa&_XjaV`n(?*a()61~6^mGJuK2;VXHVk{{@Nok
z@zx!S7&P}x(fz6I@n=<sRr`yp_44-~uuKm!zo)iD#I@<;<!M3(#gh$=o0@U-#<FU?
z+>yjR+d+JX*cLwBfX!_?W_i^7&#;KC@>!zz{0#r52~z8nrBY%ZNOkBRc`C1#R1}-+
z^f73KgnrYWaHr@qg1#w18jWk*)LBk)YWy$V(9~0ykXgtjrD-DC`jxLgHbdw5t_M>Z
z4youUE5|)*a7y@ZbxXhN$?cY{OFU*M8tLnw7E+!W#r@}FUdH4%OH$PgHa<MGiRaM4
zd{*Pna}L@ZzuL>=pPr=L5d6z%``$aMKDVZsHGc`Q-p%K~p<P?xxkoxrw8Oj2uQqb2
zO9vMnUDM&TVsENP_jUPO)8eXRg_2Zv2*>k9cD{0HIlA`X<#}a(%B?SED_P(7>our)
zyzA-<|D5ey9ClJ16L`)Tscc-Ne{Oe%G!x(d$vUYo^&?MZw5&e<VQq=X;mXWb4^J+&
zDMnHnA1`0_zMt~!=hG7HkE?q`6m~rp%WdF3I@3Y&-G#^7*UV~Anc=kP)`z0-yD}>;
za*9fC3E<8>9=e<RV&2w^mvnXWSFe4tDm3Qc$)^jNg<s^QKDu&0n0uREznWR?{kC7}
zkF=XQn!oMhOq^NdU-I4CsPfonhih^Mt0bh(2Y7Nxr9N?U;ZC{WxwAUp)!gqZ9?0s*
zFgCDkxsXva`;6|ZiQKUg`2uD)R2NRa_Axp~ar$+oI_;%-y^dzab95Wn%#5;_#oyd+
zbDo;`?1yPWk%dBIMf>q-Y=&j)Q{x40J(g=)qH<KsiaW09e}(y;hf@El*_>*cCu~<Z
z$j%kR`e<IJZ?~v^vTMm!12)YJMXt8#OfNjxgSR<MRFSqQe3h{1{LG$?eMQqI{Q6&d
z|F;^;xBB^Ve<P<<3bXR-aCR_B%B@maU~<0Ex8KUJ-|1k_Ny&(2Q<+1R{!RfB?mu0h
zrrcOlc`~{rLcQ_Eyc<_q66F4GX#MkmbCv^Z#Rjiy6>W@L9VM5<l!&-5FK}2~<16WL
zN+hIr=?0m%Gd;RL&gj1B(XBbvU?Zcj=tV~!7N?$WA<6sZxkm*{XF8fJWH4D6ZFyIO
ztHJgM(_Ei!PtW^Pa}G)5I8|pdc62Ue>G702<muQG>Ga0)LdgP$n!EpZoV?t!=H{=h
zH}^kqnfukH`@xCZC;OxwBYHiq&9R&&r^+BS;r`#HKG6~j+<04moN)bV(XV9ax!iH`
zMH>Yrv6g>-IK_7a-Mn$k*QYUO$3d0}ju{h@3JMrr>}3;Qb1!{Q+uDb-0_MiBHXfI~
zlEgWwUHU`N8{X&w(P-UQ*WNswbp4K#wgsPB(237y*|%P4`qInv^(n7JR=h}o{P&3q
zP1P99W_fm$O8<E2xnrihSnFNCCGj>p?_2KTeX}RpQNY!D(}_I?=lxN99QI<i!;9Jb
z6d6BmIJnqz$~1*JlXPU7Z6%sZr&yPTxUTIunR~v)WzIC8n4@cZ5)%}5SH%CXneS@l
z@;_;Y=cOYd^BNKyk4%w{Vw3xIWx*OYgItCezP#dpLKd<Fd)+uDHY2J?rOW1Aa_^ft
zB{>H-$;w_yjlAqAd+F;E2N%VOzOuVw5=C!0iRt<OoHA{~*6AWW3nkC_hhE|rtxXB5
zW0|ujq=VBxu2D5Ua-w5TvgO-u&kIKqezGL3oGY<0_C9Ot<ux2{-pcM}eRQwT>A}a?
zZiQJkS=WAw9$CyO()gC=%#?mNT@lF#JdOq@CoGxfmV2cn;*nyW!Zy{jUV)dyVlJ$@
z<0xIhmf^$uHGo&Lz+J2DEcdoWIT2j1cY3{JWaZOq&CY7Op292XaqzUR{QD*TUwq@m
z%M`yQ%9{oy|2Lbq(5x@gO=0<qf><@9<+G*;@7OCGwdAJff)gdPP6Y00sXZC^=jpk!
zTTgt&Ts|-`GSwWdP-<j(!IQDHr%GtjWRC=A+w)V7w6^FjU=mVjHR+GGc&fyjX3Zh|
zan3^aPqV!mm(JF4?Rpr(ac^2$f%^)9u7x)owcm8-JP7Yzps-?!6X%~}Q5siEpTzLJ
zo7U^=!SYjKO-+EcO`<c$Ly>hFS6Nj9q7=pSo_^ZnlUbQqy>+_B3yt`~rN1wncgwls
zTdCT)^JLMY8x!hMci#zd>T!wR*kXC?;kzq|J_jT<9&*&KoqKS@V%J>tl){znH|3*W
zsP2_KD#s_4CfJfTk*T_A`TvuDk4Ej_X%n1u#4stwM$LI5lj^V5r@4*{DjG&fX>J-0
zOlu;Z-))jUBa;!qCE0WK`W8pc1-+F5kM;k~a?W|S@n)Q($ZY2q3v|_3Ds}`FCTcC)
z6lCDaI9spP=vuS>24~}Khi0AYQR`_GYf1bTnq2ruOVaI<SyGF0hEheBUzEdg=Z@$p
zm+m;^9C+Pu!u5*e!mX!vUp&!eakiZ4b=ZXG;cqw?GN!S}ESQ#X<M7^er$5Q|N=qEX
zjvUr}kd)BLpf)LiWm)2%Gm1e^T;GX`gg$eZSTWn=nA;TB{l8Z4=X=HS=FGXROc^_5
z$`*Jn|9hqV(F}(#H`@2{s@^(u(CCuG4#)onv8<9e!nOM5v-B_pg~^IUCEDsNovzcL
zbnk^)%fgU%8j?k+p;fYOyB<#3$1i%Y#rvVQUv7fOED!T(H?2BTGsIkOOkMisM$4{^
zs&n2>zMsn~UzMr6N_&P>i}YG$@heXct(ot5<H@3v*;_ZYCbLb7<8j??#;;r_a%PT~
zwg}t&Pkin{ZVW083>pk~POY84FL&mQR)!FvnKN2hzJx64V%KAvx?$@w{jF!0&g)G{
zY0HmvKPY!Rf6BcqpMz{32iP<Yu*B$He)BfXGTzus`KxG<N!1CHx^ru1y(@f``$|oH
z_nLM0`fi;0bbfEu+BaEqew40tW=NAcb$x$J$L3GsDNon^FIRdL$2RL4Q)>m2=bTgL
z8bU*21Xw>+_#bo)VLr9|idIC_ljYxh7AQyd*B^WB_H50Wdz`5}{*gTX>Qh<oygL6w
z?bw2U8h=CF_OazGeIfQp>FVK4M@x3Z+<dYAsMeLxDNLLNjK?p@bc7!JBV#b@h{(de
znEx}*`8|6f;rI0St(+gXaynZT!p`NiwJdPIF@^no=lMmfZ{KA(_4G~rr}Ji-XMF6$
z3)578Mm`reWR2Q4v-OqB@24%2shLq<#IK04Y*l0VD8wT0S^DA3Y1g%q;v23BMqimQ
zCHITWvo)S+i3+>{OHST7mwQL=?EFV>?>);c+NKv(F)8QGqJl^PpO_12ZzBHd><qfS
z<B~Lg%bG;@gU{MlZG4pI`_99I;aSqV`8Tf3ulbmg@T_50tEABK)5{(PeR$RI$zOV#
zVzP{e+N2F9c24d+bNb?;;H160Th>Y2muAjTGW2=V@t{{dQslgY#GQwdoXN{rH96)S
zR7^PR`Z3Jq<kw9{cNs0yHlK9#XKUh)Y3zrm>|gZ$l8WQ+uvGVX4_QBV7Ekj$t9Ew&
zAH}DY*N)6Pb@12wqjs-erFMy?Dugfc?2JwIs7o<g&pGMX8>wX*&Mf=v+N5F-zp-lX
zX`|ztC;fYQZD-eRmx;{ti}!5Uz&$(qB9oXSk64mv&wh)R_xqmC743QNm0jL+@ryyv
z`wJY7w$uL4Tl(^ZM4X2Hjj#hN6+&k@8_kG`;ZIql%PQ4-=Y(-lU7>>hr@%^~5Q99+
zgC8H=yS2tO<Hi=oCr-z>gWj>c+4!t#l6>$IHNK)(|1tv)yUZ#GQoe88X!JEWd&~BX
z#;p--^Cq+;dmMYc<ND9Ne%&Qst*&v@?saMXXAotwC0#K8!OAP)PZmF1$+N4@WMW9f
zVZoL!w~A`qt~BmF6D=bV^ZMLN-}6V#-kLp2M6=~c*e=eQcU~8*3%R7>b}8e5&3evR
zZ`w@MkEnYIaJFff@<*p7uPMmMStw%Q`RC+`gZh7@j&O=+Iyr3X5Y2G&nt1yb2fMLx
zQ<P!0{?WbLu7uTmnB+I{ztpC@3tKkqv}EDYKX&6~%j}0NoqJd$3Z49R$bPGP89Du{
z`tj*OXXO8ty8I8#)+sJoY^iGfi05p=`LBCc%{@2kYrsc0AEQTwlhmgwtv{JmR+F(p
zEJw42w=P$Q*U!N}eVsh>jzH(qYT+XFgGZFT*_yaBdNp>eHmL}ymfo@VY==m;oKbFs
z{@ar0efD1^m#j36dh&RN!ldSvmk$|U`Fe;a=8ycTA6uO_G#+#<>^|P3Eq~+lE2I3G
zQxYZ0UgV@q_~e=TtHN@1!G!K7-}iFPig7u-?E9q6r;hGw?1??f&~0X3y*V{tTj;el
z6TW4wzN7u)UCyL^n{R*XbbqDe6I!rumcjqL#S2reZ=U%)d;7!BXT83~t-XBPW0HH5
zj3bZvEMDD&nJo!_Pv1N%+0`JpMQGM#zB7FK$7Dm_I{$8o(%$93;J~t=FM&yWRnYm>
zMvop%6^kob?P0v=^n|8uAFg>yn<-20Nca7u)|=v+ek|dR$&D4a@>96jUlhv6=LN8r
z{s|LO?(b9fKBjVCJLo@$lHHHLFSpHdo)!M>*Nm+n3g$<6UpU<{C!%`c{KSp7OW$3N
z*%zNum~P4Wu|+0Ch{ZEX;X!ow>zL!Laq1763_ZVCoSk4267#=QTW3i^@p}i6GS2ha
zvlstN+A}Tb_qUmELL?_N9h45x3aVr>w#gFynftu#Nb{2ao0eAvEv}jrR>QIUsQanS
zNA%O%-fVN+qIlx+YMcFG^A#*+uK#@Cvhl=oI?T+x0dID0w9~#BS#k4cEZ=38Mcya6
z6iWWu=BkVH?3J|;n8LHndB<0eTRPR*OKmqUS!k}D>7BU6(NOtg_4~EYvTCor&wb;!
zC&)bgsPPYWHQSj}|K3bbQte%D#m(~Sg!WI_MXyv=+x`8M6u@Z3Ra^R9SVQ^o+b<sd
zJpBJXm$<a`N9SBwBFj0COYMN-v;T8nD<$>EUN?2_`r<I{yj|aw-ANw%7qd>vTd+5Y
z<@(;wDfPy)rfi=1QrGG==MjIK*PpkU&+|NTx~yr^&o{rz+>h0{F|p=e-}!%z@#aZo
z@gL4t+NAF^TK(buezRlswT9DMTaJ8KG(9$I(c{FG^B3!f-A=5RF?F>rIuu#{fBGir
zh-^(6#aW9|WiC}uTe`hx39FE_PF!q?(2tZ~E4&TaB-xTW_8jRpJi58KnYo3Dk(r%K
zr9vRlsg+w&%|zotB2z2ByxEot!Q^AzqNZhEUR+ReZBjD!3dq>>z`aRGNP|Pc@uUc&
z@D!do4j-O5&SYA`F=Nxx=YCDrI~DFYZCG)t*^|{k)FDmtQe)~42i?yhE03<`+?^5H
z(V#GiCFsT$p@eA((qWRfd2+6PQ@FL6>DDI28(krX!m`<)$({&b$FQ&IHs3jqwvfo(
z0sjTJ8{S@b?O<62tBPPiu|itEil7(EPszp4nip|axpnMRa&K8BJS*-+)Z)w=oTm0F
zYlTveUtx~+dy;!$>p_(Ro>6&wCm6rJbHd#`rmiT(OO;DR#b92s+POD&#ayy4{yw_c
z_-HoQ4TE{izZzpMO&34ERAc#?h~<1wwuBxCTORoO`}y@7RyQ(<BphgA*wDA<r$~%|
z;^s41%~LoyPkB47V9R=aaHoRFX)#8to;B0Lq9!JpvH9xwHVE-I`I=3QuogTfVL$1G
zYfH~-pJOtma|~Ul3k5XU%#wKZQbFnH<d@6m*9a}JZRxt@e^C9jcNdQq=dM$${+FFu
z<?veNM#{Pa7oz@mi*P)Q>NvxB^Fz170g<T7dOL)yJA9iST)Cip<V3<zofQf#tDG5c
z?eso!ieuMWw{C-KJ=Lo+u^bm~2U+o&Tq|qNSmH9(Yu(PDc{VSCo7%eHDNL4ld&MnC
zl1uc8h-lK)&<Mv>onb!4!6quvrK*`tni0ERsJ+=0{prZ(bJ|+AdfPJcyVV~~-lX_b
zZ&Db8pM*2Vs=o_jUhK<QovP8uymUjs>HVj21Dwp<Bs+KvHl8}IQ^7jhn<2<TMZ+lC
zWr|a9sOi%;3+e-H`rPjo<#IeI7M-{LexqOR4VMndvJ(og)I<*~;S`zS(8Rm!!;I!t
zI~H`U;uR9$Iyqw#^ZONZJ7oTAIW7rM>tC5`xa#Kut(&bv-mM>$r{5`NcWK(LvSdTJ
zehZKE3B4+PAMRHNbS7o5?=v_vS;Xi<l+NqpS(0DYoQ%4e6w+=UTKcd*LVKf&H+xz}
z?tQ7O=yx6Jwp%wnlHgsZ7<oU=_h6m2pWb6lwHXRej_lM}$a73KU#95T${>al3)v<)
z-FVCMzwgF5Lm{r`U#G>cw_W~V^|un%#;}vmw69AS9Qey1@W6pZ<wfSpE!$19&sM3&
z%#F2?b*WmwXDw3bay{hA+@GDZ>*bU?7oOi})u?q%;&<Qj2MNC??$u}h^WkXu<5fmJ
zFPc7n_I`1Q#pqPPgPl_jKJNTCUD#>U=l-cW|6gfnF8te(7St(O`!Co|*O0}Yn{$?1
zg8KwL)x?J2(|21QDKVJwwCVdMyIy{gq~2Cg5GUI+nRVl~1E!h23|~8X7MxR3+H~=K
zLaFwNlCZ^6Gj`-EPvW}4r^!0Y=VyV*rOTrH56$*^u^K9FXcFbM|2$!n;qt&4o9?X>
z5?XUuab2}y^&K(gbcd-HeLOc-$Cu3$UNdt+j^~fyfT~GHmwXaf68YDAuG<P%-QN*M
z`aC&ho!`7zvg}MC1BVBLz>)`g%BM~^?n^$bcEX^mEHWcjOr~|uwIs1Hg|ih*C#G|U
zZnkL&?BNdPNcB@{GLy_=X<zqCMc}#E4(C_TrWSeo9Zh5UzduT@uR7|_y%R^m**-lz
z&Jtb7_=_chiE+w;8O$#k8v82uB@404+>jh2z3jj&O^-x%uN@~%TbpF(Rk*8}{_LFC
zv1z~gJYnT0oC};vrp;ecA*lGnQ^B#p-J!^|z?LuOYJ|=aH6AVAmH=z+7_%4CjaMGc
z<TljcT%9Jk+ryV(V#os4`%R*@wrT;hL~e8ImrJrPe5M(*Lss(H_dP*+I(b_@UcVoa
zWiY*Ib)NaAg=tyNF3lNkW=xB3c<Gx>KAbwyw)=I`FNO3ojS^EGPvo2kI}pyLr{ci$
zS$n1A&d+lbD_u06pG>#gYPD;b$;&w(4<<Xk`f1$y=A#npBrm)4as`h6$Aa85RtnD*
zC|cH>bHce&)}!OtGzNbyS0?GhF-&S3T6O=3uImhalcetI{W>w9N&0|Njn{_LaZY?Q
zUMQz=PRVK#nsa{hF_By&&P_gFbT`k4=1i+Q6JYltRBT(w1BOQi;j)pksoYx@d3q*R
z_;%;`Pr3hce&;ny)yHjD8a~hIw~19>@Jw!liDyT{Zkgop)&;DUx;`E=`HUpqX{9u&
zq)5%yaB<`>bMdcPba~!4s}+u&D`ZnP^!)wccEm66U0cB2LzQx$HUzH=Y7gA<o<Fq8
z>y)NS5XZNN%UhCzgI)@>iwQ5<uTy6DI`oDb_YS2;O{<PB6cgQi%C;j&(CJaX%J%;a
zO5SrG8ko<0siMfp*ip#95Rlv{b5uolveTJUI|NM6J-5mA5vys7zSI9@qTRJc%jFlU
z%(u(qKFKbaA{1~#{Fs*Ox=K!s-C|nn-z5nt{#wOaDIo3K^HJ?MU)<9)A5Vi&xBle~
zuUF2TAO5VzWru>$xf{n+CY*jZv1fYT3#K1Uo-Y>J1=q4TYNapKY~Y(Q<H%F)W!z`N
z*}R(bwR)?A*tfOs=*`t((EB*Chtc72!OUG(m@Neus)UReJc!^7I*_(j$VsK(P>;vL
z82-RRw*-~B7u2U{ytC#!5wmHX>Adr)Ty<BZVmZ&48tmQe;d*n@mi-atEu1%vwAVeG
zxNhsx`5OOo+}-x8DgB(r6}m!(Gvw@L;oq@0J0`7q>3uckNcUFWq?I!tJ;+gNOpxq5
zEcfiuW3C+nOAdAHOf87_czD>h|KgK2gSL;24GxbP7#1EAKW}l2or&dM#h%Azb*~*v
z7>=L2%lGuw#?EJ-u1~Zzt681R`egf&u8FK>`kD`=yqzBYo#^t-Z;}5@ZD(OGo4r|^
zM7J3%NsTH=4|Ob!`F*$W(5&?5ir-|G`qzI>Hg7vRVd8;JsTpe*#%q^--nf3oa@Mzt
zr|z^nFE{mwti_K|yXg5JLZ@j>W!-(4|9fU&d(^Qd*M6=#Tsm>TQo{%Cn}!pc7M-1u
zz-E1~(A?=t#k_~lHkV3qKKNfScao||{gO3r&TurHVGaBobzAqQANTgpht%^fx9X>5
z1(XN(3smoo(~aKNF>emrLhfx&tx}>>c0RxDtl`A6WN8qSPr@g`DIIE4nZIe|TChH6
zI{h<dnUhC?@Fy8g?;o#>KYnks-ed8lcE*GA-s^r_?THt2zq6i`qv@!3Z2hn3Iggnb
zcK!?9`~T*}cmD;F{&OrkmX&i{DCKy=+oAxz;w2A$+He&|^li6W)H}yDvah@E+2dzo
zNzq3BSO0i1^(pPykaz1`lL*`6s6UHNi@b{Cb2`&@?$?SH3K2>h>Yn~s;TOK<UCuR~
zU$@?zeR1q%-wV@<6AGKur;Dk-IdS}g(d!i*|1~f6dCk0+r!awYD~HR<T}rbg+8xy1
zzMHUNQ(E@64a-%dls>S%zY@lKmFYq5tat3XOge0jc$qsD9tAuRW7_^Kn=9$9`ZWL9
z0!d7Fa=9*OXb2sdW5Iv(66ehYty-Iw%o96wLxO3oi+<-1&CESIx7{x2zR_yF_r&Gg
zlZAGVZogRUW55-EWEs!Aj6=_q4|oK)>8Y|kY`g!b;ILg<WYej_bN$^5vh94%Xn$JF
z<CQ3Q^YDg<9yO5!iv*nyAM_t{2{0I}yJB(9M1bMRqX|;aFGxKMP-GHNwpn+qL4mQ)
z>DmJg#!9Da36C_+E&l1UID8LhZ=JT_-Y(luFKz4IDlE$RuJd0>X!9Yfs-)J4KvlUV
zd#35=pLi<#p6iT}XM9ehh{uz&5uR)-JCxU~`xR$+&g{|tH@v^^q;)Mi5vX#=ctd)!
zRWI|l1zrrjDR<OsxH7h$QGe^8J!>AP+VqoKWO`=?bYGmbdaE99?Vb&n9X3@l=pQ??
z;UC-kr6mEXk1n`$1S!Vpa-F!ycsy0(%0+cGfAuX|o=hj|6OIVH<&5Uyn^%-}a-C|$
zG%klV$<MVkz6xpTw(&U$d3}4d$R|p_#f4?zvnQDxkM{^QJIQ4=d!5Mq!ZKOm@ZP9}
zvbP%ItDNptoigLfW}b0^o&U^%8#(8gju|d}ue3a0`L%9>47bw0{sU%f!bSe4*Rrka
z7HbhVPWroI(Zes#`5*F2X7)&OUy$UUP{uYvA?LD#z@oJr0jgU99p@BQ9w=0m`Jj16
zRXyv_-D@9@hdh$GyTViNL!`ll=%jaj6LOesCn(I}QC!p`d`GJ(go!PgQ(vuPkA{cd
zlO?RCC!M1fiXFPFyu{)99QNN~D!FQxl)5h%<t{aETAIAaFU5nchDo>P%JVl9G%xma
zsq%LmS#g17Re&0Q(xn^iEAPBqZI#RXhwp<(!-uv>&Y5d8`aelM6_9opaTi)7<r$(m
zq3G(pTpO`|^(_Z76gnmzS)4jS*i+44jV(~U%vVl0P_sa-q-t5ZR8x%A2f5>uW(KKn
z?)$$a;KCEVTThzq7=2C3&phP)ecfvXlaA(HYn3-GYyP!TxBr}@mKWm+p{aQ%!h!<b
zGhVuc6`Ff2I%JZ@Si?2FX%+LCev`}V75$90<u=xY@a%}19RDZl5!=Zt4aH`bdVDtx
zq&5dSGF>q5T_MSMz=(%aW_yD0k;j_uTs}sjjQo%8R~;>iaM#+RCAa^TLfzWFzTVKk
z8j2f)m6xqK_2;7Ul}$FSecwzp40`kypL0@_oW!fUh^<*v$s_Z1O|!Dbq1iK-*=LF>
zEq$SUM#Ds%;f+txg?D)ub~!~)JjoOFY4Zj3cMAeyJhivRu&ei<tWM}=E?W7%Mo5D-
z-<kXUsRK?*6aIg4=`d9m`|2R^h)@4`hQ}ng#@vrPI$hfjCD$kk82VdHW!iq{@D7(_
zpGz*<^En-vP{OQz&0B1hBi|`qpZDv8l5|f@&V1MO_0ShR=Af_FTFM;+wPuMJa2f}&
zZ8qSatSGS5vSah_jwSn-_y@nrZc`IZelW%NJFn^u6Kgex!|Oau3I+cfbXQKwzTlE(
zTHX3H?u<^V4o8VzcIp|)uj|(u@a5!B{{QEEp!6ZL!eGS{+a6bJR#cj*c&E&c>w~wQ
z=#6a-+yWoUSQZ4o-InM$wOB1cdCMuw*-UZ^h3Ecw5&h<3=yqdkX>&!9=?vw1z7Ar0
zbPn0hbk}pbR&Hj#Ku`1Qh5wU9tFG$a`fS~wuNv3<`x=jO&9y^M+>+0<g_yW0#C@N(
zr;qEyyXrH1H7AyDZSvf5AhdQ$*TrQUw#ck?%r4^IY%nWqHFs3D)&zq>GahHxw+;_N
zH8PKD_-}C$JQQc&#CJN(+*xGZq1K?c1;?Lxb1gcca!es1p7%yll1y)s`|PIcL0<**
zA3cqks`YBIN3F@T8H?(Z-&u7()3Etgbo<CRo$>`)>wTYaHAX-5`t5Y<4wJCxA@750
z^$&%VOl)^KBchzYirqKG@`{OX_lDPe?MzI5D|Tqi=+LSaS$nY~yiM}agPLP8H}z8<
zJbuX2&fhE~nf^x7Jnym2p~uT6R^MM%ZTO!l?`}YizEaTi2<1Csx%x~Nye}1uOdSLA
z@?U>bi%XOJUlwA_RC31IHsEUF{|+npayujbFl%R~Ee+4Nb%%*7Ip0W~<d?5|P9^R|
zQ^hge^-e7%oZQO|k}ALH{^7HXeC=jBO|;DMD@WmWo^@NdR$YHnGhNlWyLOuR%o%SV
z{Fst7`!o*|%eEa0m^XAJr%bmp+QRd=^lHZOjxE;{#6*|*gt%l^a%>dTnY6=Yi~01Y
z7nGBk-38WvNIG`cB=p*n1wp~33DOg`2Ruude57vkCFMF_n{`XQ*eohGsp{nV%|Frg
zML<F~w7>EAPkp%~lgz@uo;mWBXGhkFBaOSWQ~3{jpZ)LqBIKI$ozuE%i<>1~f_lW1
zxBDMhad^U7j_Dc-DOL@ckrqd?W>iT2a+&0vb$7k#+HdWixstY99vSV@SSXltH%v-g
zsWY}RS8;33)`?55mnr@^AGcJH@yH`(zbfhCC;z!F7?~W@Fev4oRTNy!<ipdQ7oHa}
zZL@NZD`WGS-YKjr7jIS$3)K{vZaw!%hS@@S?de6Ut+fAe&JkDX|EsiMgOtwer3=4h
zl^l2DW&CBx-&U4+J<MqN#MLh{{t8c0z4?ZvcZFN`=BDkA_w!~-h_oFobvV_2iR(n=
z-Iv_gzZU%QUGV>$d&EqQ|K~NNm6n7iO`ko(g}I>ez@bfx9k|PUgpd7C5cWtEc@S~x
zhFQ(SSG?Q?Oj*6^JleH)-^L$zV7x1?!7;5^<Glh)_}nQ^>T7))SG3&t;q0h&E%Ya6
z&buJD1-x_33zH9L>zjPcJY=P;_v7oa)w*U8E@lUfxwge<_3ZwcbJ*qc;iw&#8ax$k
zEZY(%wVl1H&C2km<VHb-gxaI`?Sfa2?JDcK`$kD=_C2SPrT3?8Npz96^%IV0l;yZ{
z;O0V6H^=?DhZgT=s@a@mHu*uG@SzPoKQ+Hj%dfTaC~1;yzxCwOjuJ0-h7}36-<XSf
zUr0Z%uoKGEmFiq)S2ibqeZ{{Sw$Y6|DvRdNeV-xe6!B+sRB?Eb0{dkt_je7`YkmH|
ze*1y5QaEWcYqj<q^DOPv%-6Q6FaI9fJZY+H)ag(em*|%kQd1cO^SK4)h$=OUp4xL@
z@`CA#)-LL^U8*CebM90OKC<@BhrC|5YgfK&W|VwPlb+4BZHe8p;Ip2yo*x!i*b?(#
zrN)-ZMGe&#H#&KIR%W_)HhJlkrj4gFm+iik`Cci>zv<_WxF4JJc5O+`ozwsEm2=8A
z(GAkmq`3dcF`0x~_<INMI6d1jX!BpWbBvl+8q5j|&)z$1Dl6-%$oZN-!A<(y(+kIh
z4oL@EZBpKSFg@xjhXBKis~xhUUEAkuVJ_&JEy&0vcJSuGx_8pAS4A+m%vRWNBS*!d
zcDqH>lF${Q|D6ghKaN=9)Su?(KF>#SUR2072fe(|IcJwD{#&WY_oE>@)X8qzq*pAq
zo{6ihuN|}1Tw8f3QAIoDlU^+Qmp>dL>koe2QyKV<<q4N;N1;Y9_tp#CsoaIPOgDx~
zo^IXrVbkAJ|G3U<K9uXL%&@=wR&nsvFkP-|sUp{!#QZj#S(<QA-09iwKOLEtMd|E!
z^^Y&V%iCAo6cd*_^Yet+T}RY8CMp!Yt-C$rP;H3Klapz;cFf9^s1S14ve9?<B5Um*
zEt59A)iuh<T*3S8&`+gbnZ5#9x~rw*qd4@_t|u|API~rb_uDJnv%|Q5TWO!UZU1kI
zmQXtPwTba-#ImdJUTL_S-Tpr|CwR%tnQ{!?zYfMeec`xH@@M|EYU7vNYrn)xJ)WR2
z=||In=tkLN44i#(HHo_pUf%X~@(eRhrn|hSgJs=nqt_lfez9(a);(jU?eE3c{CHeq
z!T6?>z3JPHRZ|x=Y5#a&pQKxIUhh*x=C3BBKR=wa{v12+Q+;5A*vCGeCYuw{KcXey
zUQF4<GQlRs`QM6w*Kc3&O)=$F@G1~Ik?3@glS@F$V}XLh;Wkbt6$XQiO0En{23$8z
zY+QW2U%|Oc#&eQ#Q?JJ&m5B{0k55igjX&gbVB$0PS!UT>2RMV$+~+U|1Y8O6X3*|i
z8?jYN^kDM2IkKsCS9c1o@?=y`&DzT)+z|esQ+_MQg-x#AQ#La#QJJ_Y^|9Etg5@3)
ztz8{rCzdYH+4(xm;r2EemV$>C63<Q?%w-o>3$9q4bZ~<8q=yzedl*hnvyWO=vvV^8
z=WM5Kmb)gF%)TTW-Dsn`Ks395{X{MW(V)`f7u}L4clplheb~E1ZTqIXY37<&x7}Kr
zcfIS|0VCA~-u-^IpWnY$F^qq9sPXp{&ihL}0v0KFGJR&eq|PDKu*jvyfr&xTpk8?b
z%jegZT>|X2!`8_MvMg|y%59k#D)BQV!fRH}#Rtt%Dh~r|^+YokG)YRHxY!uO$jH#D
z7}+7~5m&1*Eo+<PQ7%cLTkJv|DhDkl^(dExENWS}_2|X_VzXapxJC)NC_41liJlC1
z7rqj4iLHfGV1|3N$uZaD>$YJ+>I#WJqlHqI+z`x_Z`7PLV@WUPq#5yAKQ9IzZ4FV=
z&L}FmQz#>P^;SWyhl87>UP_vixayRICWgi$l??V4u7($jWE6J^E>bO6GB-7LM#9V_
zmcW7wEL@r&COE$f{W8y0GIHs&l`HyERyFeNQ*BU5v$W_E^G*CQ-<u(Csa7aU-iy~O
z)qPtWgqHLeol;BASmY`q7*@J^*6MXqSM#?`^O`7<?6C8wM9NRmR*}Glqk_?OXGO2a
z$o`F37h7#vwj|m08*6@V(6Q4u+`e-;wDnE$<hr!<li#%)DRPZz>Hl+ARy|R5cb|Ck
zR<5k<*JXig_nJioN&iffkltMxc_(w*mB~@s>n0d6PUXxGaZA%)9m6KB;&bvS+r!@F
z4MKuz!v(8FDsOBK|FTv~X&vLgUpY<ZZ+R?o|Mz#w!!1gAp$Zo?G7V3;E#APauD#i7
zl1{y*_=*Kvyys^vJb!rJbOxb6ytc^~rTQ{n-?7b&ShPcS>(#|#319aHWw`uWv5vv>
z(NdMkpPkx%i23{76jKj2yE$>1ruNCDS%r(jem+}1sch?<4JX}h&5^#T)bP^vrqLW0
zm5A>z+*B>Ns>QpcZb_E(2_A8D6j6Edd$~Yw)kIg}!n{8pz5JHAW~|*9%&omO!}rvG
ziNFAVlTX$krG)%Xrq7=k@PK{lmf(BY{Fg$`)rI7)WB&B&YUM{U&ue0K%S?Af7wlcS
ze`isxiECV0;`F#j?Gh&zyfOdtX!_Jg)3+ZH^of_VOqgP`NVcZYO-1N1Z<mDHB$3>L
zX6eWo*&)Hpt{iDoUYzJVCq|i}xARHkk5>}=B0g>mQ%Sd6t8w%s&tyrxJvwQ@j}DaY
zbC&z8;*cvO829lJ^L+k`>I^=`f?>a$6%}5*vd=4?ec7pKk`5P#)^|<$TDL{6i&=Ap
zqb9VtZ#m|sn~}t$+xcj{j*4AuWJiw7!oZF=l}RgS3R;J*36!*VQtnMy;ljB?-o46Y
zNw3F){Jj4!#rE4xQu)xJ-tc}^U;>lzjAN1pMY9**>CjO3(&K!PuwPU0#UU4=(4>6j
z<uiGX8<|O4zI~RcmE+g-#$?XJIcs?GcKD=uF=)KY$YK(CFlD*T(+O$Maw;yUTvdpl
zX_poCSxv-|Npr(dzO$04B5NETtF?rjowVju*mIre^=?k)U$?$jbBmVWaPGTw@05;4
z>4#!}?|o9<dgQhK&)3hRjeaWcU7_8`>@t%{)qE05(~}0}!dTCeF9)U{yfE=h^$m}h
z&bdZwFDMHge&C^VO)Biiu~t>Z!+BC>GavmpR-*j*smL-H;}1RQHtsHq&S@Ol{FULX
z^S6WNzj7TBp6RoBR)xsp|JGBxLgE&#uen$?EBGbnd6v8Ey|W^OrQHQNU!4@Tj=#J3
z<~q-?jdt0&&pR7h_HCWmW0XH%`sMN+OxqV17A~38!mP0Bm8*om5ZfYY!Kcpk$~VNT
zR5+6l$aq*pO*%7ms)~Y&=WIE%UC%fc$S!MssoAdF+FJ8!=8@)S$?o2_PThQB=OMwo
z#!b+6!X~H4*%@7?i=)o2FM4kuR;gvpIOWRQgG)01Gv>*$ZV*ai-J<j5^zuDrEa7$Q
zSiD?s-H&IIm~@D<A+0(yoJH~TlCz>4bVF9WE?REwxqm^9(Tlfdn0G2Pep=U=B(Hcb
z(B_$M@3OXL#Vx$N-m|2Z-Vk>^ta&yw;6Ime-$TuoqmJh^r*0N}wjqgY;i>Zy4~^Yy
zmlfB#w7&YWWS8F#Uk$fqr}*0vO6D*8TGsXBqi5|F4*7oy2eO|;DPNzqy>(i|MrD<E
zi!a5oY!iGrvEQ!mV!_f1*;P*k*FN)Xic(dO+h)=+(Wc8txt(RQ+{=!W{W<R=maT5w
zcxP%sg5I3C;BB4<eGH2DzUq0hbT2)mworE#>&ZhAoHCOvdd?XX>iBdCFBd+f*%H5L
z%^b^Y-Ic#KZaK!S?U`|Ur|{$zN>K~$NgUwxTdd(Nyu{;>sHm;d!mGUV3XSELpWl@+
zX@T(y?MUx`Cye&j?v#p*z7f9dM8lf?9PfA$QHO1Roc8~Z{r_U;Ha&5N<+~^PICbim
zYd$Dqv<_@Dx_#p$AEzsKa^OkTPn$k$&F^KB*%cCGc6(KcPHxO%k;&UWm}})SKl2No
zbMnR^Iq8Q_3rdcdO$k^#H}>aaG1pHaZns&KvoDskTVA=+lzk&oX_2Sc$@C8ltVf-{
zS1RX3vqrBkTGt(Z@}%h0miO!PqUOf9G~Mc|7E|;7`J_jN`GMK?*L^;gy7L>KxE&F)
zulvZleuhuzUV-DX6-y<jOwiQvY3ZBmGU<>QukmDa^|0q5FMoy<PwVCOe|*Yv>ARBU
zcXG5vIy2tfRN-1O<6iG>Ki@}pmsib_Xg$JuVS&ZFYjZU(ul+fzDtey9|ERuK!K<Ub
z)!p&P=f1W@?(qHSe-o!k&r{ajud#22`)gM}w-1vOHz`Mp<Rl$muO}SP<o=~dWuMBi
zhKZJ;vlNBz9r?Oc?+NdN=g+6!JL^!O9;DD*B(Zu!SlO0uswur3+549<HoJVB!WgWR
zd1GUgfyC`InXP?1l`j^&J<=~B81U<mgy^em4||^L=IynrtOc8rH4{oaC7MmYf77fv
z&bvITed;rJ@#w&l>6YCSUfHJbufIHRp<cRY#*zP=J1Y)L3f8J7c_>X!d44b@uce4<
zMxesOxZ{WQZ#q?6yz1errg!qHSW;u7!F9okWE-DDDyeC4os1=C#Jpq<rZFv0Ww;Rc
z-(z|%$LBcj^z?4i=C7&M-KuJfA4JrPsqK0qxXf5)>w|`r@6vs4(ifQ3naa!exrY67
z6TPUevuJ9q_Ya-LV#2SMx43;4S$VAdl4$nq4_RBp+Ttv-D=&++T=$SO%-*WjsPZtZ
z`-=Vr(d;kIdjC{}7MF|9a*r)Ms@E)>s~A|dkX@L&RJQJilZdIOxGUT9h04nW4JW8d
zXBnjjZqI$@mhj_{pq`2MjLShw78rgoGkSYKf7_*+7wvWW6It9@8-B0}%{suvbB*i1
zJJ+V8Ar<L<-mYe^FXu0PCYt+L)FHHW{x*dt<+cYbLSzq`u~szrZfi&};L^I@Gx=K2
z3sbJk84~}UID~RmWFJqi5aCeCS*ZN9v|aY2?9pZIS(jp^FIv2akf<@Woa$T>crfMo
zLGP_o)V@V{mw%HkJQV2vU7CBb*S?Q|8%)%>mPKwg$iMz2`Ixid>;|u=*SMybx1@yk
zEZ!<A(Zo=-JnPqIw>XKm_#YD!Z-mJ+)+jc{#s8>MnHa}Er9xoIq>k^CZfw+jP$8Bb
zs61nOyY!67ygOq11Dno%t8Dlo(ZHd<LOrh8Rnq>W`LXiq@2yUsL#qS7Pbq)YwBxwc
zGqxbv#7=#aJk3Wz+Z_8(y0dvaaM_Vs`+l(?SGnLa4Z9siJ&&1V^aGP(LzMI{y5ze_
z-@k6-X=t%hqwdSc(Elv%LNbr0Z4XU68?M_nMY~he|8HY5M^kbcv&p1Nf4zx4+8cXP
z7*pdtdS_eqp4DKOzc?m8LP(vn=4M4j>+;@3o?-gWJswVqF@B!<CnzAkEj%f$&&x<c
z<bjH;qVa<75g#J7^4m%$tjPUUF~>k7$n-|ZM^{7djb5u;3=dpvmTk0J8^Fz@F>z&(
zpo0M8)y<slA8n%9%Wq|NKbaZgyQ02mbB4su*5g+vDoD=XKXI;7r)=+XeeHA!^~z4=
zj}G58CBM9!@3B)^k+b-b;KY^;@yZ)wE5ZYAEp2zd)Mt28<+?_$*ChSa1!fzsPmyYs
zW-4&{_|#EZaqhp4sgo-Vr*?GyXMQCpb6NGQqk(pk^ZX8xwHNqw7Ad&7+O4ce+`1#A
zsZ(Bl(ZXm4t~Cd^Ow%N!w}%8hUVPJe#?$RBr6R7$i-n&&pXQyVI89O|LXCCfq?tC0
ziW0ai1U-bb*k^@2>si>iY(eC*U2gMpyebyH6r3Em@W+yv{_lc?yDHiW1BGVj28xBN
zUJo<*F4?wO@LH<p(~KOYjkBc%r(Az3R+kDoW_ka^$;OTHlf<W{FPd{xP3`B261j&f
ze;i%;O;p;lBClcED*nsOGmi?cQqFi9X=!_SZlC~bbi;y;vsPbL64HG*@6y-a1gkdL
z&iN`g9W*lad$uoPIyGNo=Nfs(`MYLKimMR&pBvb&%;eB|*}v?dO0}bLo0Nm+r%LWE
z;R}w7H$+TH7Rylzlu%Y@bPpHW^1W%5SELGSP}onYSjnJ?N1e7TnL7Q2w3g}AIV<v*
z68PDJ7au;Tl*cT3X}W8fd1(Df(HzMb;oYmB6i5A;m}I`pwNb@YtISrutoutydC-xh
z<*MaoN;8jJ6rGZ&&sjK=+d?JzmY~oI<?ItSf*Q**xrD+mZ<cwudEw0E5tD=FO0I3{
zn7{8?udLyUjW@-wwW?H>dUj<=$e)N!JI1Q)v3#+spu5SORmpRdjXF&q1<vLY)>Jb5
zE8(=HQK{sUAbTYDri?9fE>AC-t#!3X$Ya99E5CX-iOv6?Br)lK<oth`m8&!-n?7o5
zJG$m*SGru|j*w;RE||uh;c7p-Yf;XRiA)JGM~;?WZ=B5MHG8Ius{5j?s)`F$4676$
zPL|S=Y5vyn*{fM5N@y4RZnZB9_itY$bC~V0pv;toQf#XY%(M!AKXZRBp0#qulHEbk
zGG*>tXUtGIogeZ`Fz<PpYGN{zLXn1{o9!y?y|cm|9$K37aOv!yJsDiRk5}tEN$1?;
zs4;lbX0dy-wRc$N>muci`(!^_p1ZZ9>vdT2tj*3v^V2?h1U0nD?5LC}EKZnKY}#b8
zuzl+*4~b@p4#AebW$e1v6B%Z9^>1~lmYG^&m?5#?Rfp;-spXE+Gd0rxb6wtcq|)lT
zCzou*b_1Ihb-{w_?wAjr*+-iW&++6s)2w#EYu7R7z>{97j8XIZtB>?qD_zjqet1`7
ze~A{ukpt{HM_)C?Ui><Hws7C_-IGOYBG@IQ-ZCs|Q{2`d7WwsPuE6Bn^wS0^NAo(X
z^BAQQVvVGCX)}BkNMCzUCU8RHxk=IYKDXR-HhZ};#PL;P$?1^4?xF@wd*vMWW*6_B
z;IsGt@ov3IDfc=X^nWj%D=9N)P0&0UnQ-P)Uw>>?_LRJ`V-e@!iW8IOr&(=rmfoVm
zxMj(kQ=4s0ZRy$Rw&#e=>g7KyXZ1_(m^`~JcU4g{m!AHf_G4VdwUZAxx6FQiVzzCh
z!~e%!85x!G`P(?-xz(nX&Q+Xa@Uuep{-0eZPgW}~Ika?h<hBjIhI`IUbv}G7RP6kY
znMZ}nVw}%gdMx>VhRs$w@8#;#UT;o&HSOq{zN}&ODYcyyYLypO%-Oe+>4-bG@VYen
zt6i&A8*>y5x1KcHb#7U)3S%H&#!Bl%xtx#dlFsC2#LW5OB{ONOrJ|~k`W(Z9j!rDb
zPOQ-)az|wB51qKOTJX>D<H^^xe=GJdtWdkWdT+8z{gvt~vTn+EqhoxIEuE#)JNL{<
zU8}4AUX`#sUOB&WUsw0?8xMD6MlIvHdn$$Z^p>1!Dbx3#+I4NQ<>p%}iX?9M3l`Wn
z1+Pd9%bA`sS^BTa|DMOi?vf?-dt>Dsy{1%&elK+KNGNh&9<ee}{Wr(lMOufHT&w$Q
z`!@xyzk4jcv8}Z5*jXRxwT#ElNl(mD6W=#sJ7?l@Z<d|6R=su1*ktN*%F&ub*fCmX
zLFVmkGD4m_7Ze!RHe9aI7QeGcXF<ZGHHDjY?9#opcd!1B&E+d5t&88bmf0ottI^px
z%y%DWw1@LVc%~ICFRtl78kjCEt>ooY{wO}&c9P-WMNDQg<#!SkTP{mA?p>x-tJXM8
zDR8=jS?&Mn9_xba@_z1-xhUirYp(aLK>y<EBc|7{obs4IXZPF+nIqFhmM@grm-z5`
zS&-zuhc9iFwtRdjC3ABp=nDJO|5u;6y=l&)RN2#8WrURK_8-nYvW4d{bJeNe)^}Fb
z95I^bzuG5zk~%|Hs#vpeRaeQ1GjGm>oE8k_*1u+}`Y`qGbrHt)wdK;knmr8jB6bG)
zSUyViZr;YI7S8Jw=QLOQj?}8h7g9~-vwo(D^xAwDEZF#|{OpWdYu1Vg`8^9%U$p+W
z<1L=}jj?KHFP>c-d|r3T`K_`-8FRImeAgT~HS?0kg$aDKdY`}OshdC9aQ~NTt$%Cw
zdlhAW|0yO|bY{O}09($K7=c_to=c1KUdb3vk@l6mzQXLjlb-n#wWF1?*}J8t^~NWy
zsm{AIKi-M$={JX0*SNkmcwO(je9%a~XHV?}nV$d5`)aRoKRDC3@!A>Nn@ggNlteDx
zyBn~V^~#Bp;*)DQtYx0q$$WmgaC3`P_UzZ{F-!Dw^V}U1^v^C*R(zzNcRDh6*(0}W
zGwYTm?R}IUdpg7Rn(@8)v-9S^DBFGT&TXyjXCl+W`<@6kpAv6wF20f5%QLmF&+*{Y
zT_5EO9a?Hd|0xzp@a(%e>FnZNug{!3xNz6gGoI(<9ZMhENOP5)OIufS?aDKS@C?Ui
zt82LZSU7F3M}A>nX_cS-MRDtwkWSgBf8$qkO%145YrfYh@_7C7uJo#DAI$Hao-upP
z%c~!bdr9uz`}qI4ue+o!UXi-B;?Mzq?UgI~j;`_x<6FA*p5NLf8>aj}k$mjfO>Yez
z>HJ64noIRx`~Gk{m}~I0>x`fj%ew9fIkktUUb!3nKtD-XN!<N#hajs*jPCDE7d@67
zzv#_zTIOVXjZ$IFsgGY6!ldN4toqlws9oPsm1Wi5##PdO-dRek6!!hTu+vh@?n9jH
zAEx($Z0{#c*`3Xk@ke*({L}B(OtU+ZEPwuL)q;x8H|PIVuGoC$e%0*Y-yef&BqF45
z91pg!zBn)X!?`oe+hxA=X9=zM<rm&rC%d`&X<+#mt9qNPnOe~qt82P%F*vT8dLV3T
z#MZ3T)d`1Mxw1JMBmx&Tv~SbkD%i0h%rT6~!pTHpW3qdP3hRtHGlG&<cjzeY`f?-a
z|5NWE<G@Kuoxu*iW{b;~t&uF8d|{?myOWKU!pjT3DJ&B*e{FfGvBGbm(;BT%hN~<5
zwx@lK3R%;jn|yYjtG1!e%~jD$JT$cxJEv?BTJE!_I{XpKuHaKnPacIkJeV2tN>1*#
z_{SqGo+=y@K7@5mTlg*QP|wVrOFX8|NZsdi(@n{1=Gq%FdCpxthnDN#E?M(^L+aYH
zwQXvG3@k}Z3_Fjjc`lIN{((hcBF76}7Dg9^y{t0r9CtQOIT39ZslGTWA-CmDbm4N_
zjdy)+iaWaL=<kmG^-<?Gcg1<xI`8Rg<Er=UkNodF$sl^8)m!`JR#v==E_)aoEMiLM
zdZpeW#iFIe@n1EhbM;y^uT$zBCVL7FcUZ7|bQZK#nRtl(aFq9=C1TFnu4|N>COU6g
zr_8OVDb(Sl!>hK4dy$-CcLvjLmJ8gvii)=sln$#rYH(GbI;W%ik&4%XQ^#Iha@wSO
zD(S?L8EPh-TV_eU;W(r2&y}lm-az=Wakov-iPcZf>3D5_`fL`rp6;<5Eh3F9Rx1=8
zMNhpryNE~kyppS7S4N<_q*91-7uPP0y%R6+Og2`xW#%>c@4}^}9(L?b<`pK-$PI_N
zQ+o_l#kXd4tz4b7ch%NeS*5SSBtol7*N7?y_4p+U?>==*T0VK>2_@DAtB><O_~dPH
zbec}rt;F@Bt2Z7~=vou4`G1a9ez32p@yp9oUz`?E40Ey8y&8MfY3kNDWlkNcF2`5v
zYTj8e+r#{sl9!&+9kxCnZr=@;*WG)!FL{Sq&0*Q@;Io&U&+qZluhV$^^;&bro<8NS
z^(GVJe@#)HY-{DwaQ$DihCPdDOQmBu|LhG1Ipt%{EEfpA6?s)Xvh}0JhL*68uB)R?
zpL(%k%CWDpYeS@*8SEBsJGVAh`BKuF<J;JbR2BQrbRShw%-ZCcwR1s|=iwC+YjSf!
zX9?}>5LgkkPELJuVfQSdNh{K3UAV)ae#E0%@?c*?m89|XKQbp&`yG=ydY=CA&0eRm
z>5fzPs(*j@7do-tJ)zWff8L1;GkIKPmA(GU2GuT}9Aj^z(iO*76Lah6B#Whu|DL@G
z(hp?46}~7a>Xqx#J+HJXS5JAKRcSQqI#<<W&-9Pi51-J;eksMukhf93I)q6{@qJPE
z3EfXzYjh4B+cfQ5!sRG4-CWg0>+0TUtkPZd$Kl$W<moQX+zi&D8>byS6Z$K-*FDNm
zf9ldRC9ErZ{U<aqJS^Z>5<D?kY|94LRS%vhuz%#MI1s$?=Ypq&rJ3%pG}Nqmjx1|k
zSFGq^GtE#aD#z|wm*>AD!Sd67s@rR4d)B)tPh{>*P2>4=X>#7QY4c1AT~@u4T)*P`
zlJK>z&cU~jNIcl2${3TWbRy{FcO|A{dwmj_5~IWZC<HC}zj3FoQnK}Jmm6#oGgqqg
zm26R-BH_GLPwjzW*U3pK4~(Yhsh%s{s(dvqZ~KvbvCHZ+v>v%MF*!7qxGCnGm>j+B
z%bq|}KV=@ySm&&Yc~h3GYw2$4cv;}SSH!F<aAl0=F%?(mn*w<;&LY=N`YqcX;Fu8-
zH#6Wv*QO&{r%uf{p3D{2vhB=6Szq0ZO8Jk=D%R>csAfN1eJOF%v&9`RRBM<XaBn^(
zzIKYlu{pWc&rO1be62jyRqwqDGxcOVAS~yWBxD`)&cjWJvFoVTs-FtdEQ=hfeOt`7
zWF9_b7N+0c+MfKpafY5er(uVjm+GpPR5!&Se#U|+i*-Yu&bHB#7QgiWQ<CbF`O3DY
zM!!1N6e}l6K44Pc+$m$I+fy_9+`Jb@gkFZs&{djsabDQ!$BJiHc)s?^?h)5s=%tb+
zF-t0TU9s}<gtM#e)W}~?`($doO3*OX`MKB*hqaro2pXhk-FJC5WfF_R!S0GL6QACD
zB(}cfq_Wv$&AA1$#AcUe<ukixE><v&=;As(WxdOrM8i``wQ*ObO8GAJwk!AOKk-a?
zv$glKH$NrMy?eKgIfQc+i~5>Jic43xOV2Xezv;|#-nCm#YIl7-cgXMNvQ*tSBHpHr
ztl|%zyf8@eIVrM4A?EYfR*!~@-oJ0myrwRCa!pDgXI}Rr-8%-}svnm0J$v?ZV?U>$
z-TyU}DW?Ul`2T&`x99wvc@A@%CRE;R7Zsg&;&GZpyx-N$!4FT#zOu{O)n68}xI5OT
zUU!z4_$?)~$C@uSJb$eG9@EzTR5ZWEy4yQ!!jn`JTkg2miyqE-d*!h0-w(o5t&+aX
zTN-|)TWWrv`&`|?t<i!vCq~w72`yOnaYtt5y2J-JRLs`}>m;-CM6w>>%wGD{k10Ys
zx#6+<ny)AF`VLNx*>q%iVCc8W_J)`1vXUn4eO9`ZRfgMM{>ZHM8{(V#bz`F#dL)b=
zUpP}TX}ycg)QHKNOEyiu<&kK3b+>J|clpD)CswSoiJI5>LPKUAXV;9ASC(GPnz=pc
znaj1H+i_RJ=9V1#FBkIS(KP8;UftVU-o?HsQqlbIR4=(ZEb?>3aqq`BJg?YtPf-<M
z%@1sA=HXd0%WCuMDJ?JUYv;z5o%nF@O$pzguNFF$Su39RcDe3XQr%E5-~HxN;Ozq&
zLax+ZTywu8Li*<YePx>s%P#dUopRk!b0*VyU8(6y-l&M0dGF>qp~R<r%Q2lZa9^#O
z(%dGABZq3@HZ+^rER)>&V7ct7%d0x3Bvr09OgJO8;$?blUg=tmgk(Y1dm0QIX89^b
zKPa?X>~eO;N)@JmT@F*134fH?+xlq2#L_cM3wG){u{zG*;`*i}?&jS3CHARSbJC=a
zsLiPiy0v!Rv)Sic4l}AxJ2!DH$I|}_=F*c+=v@eZZyHswL+#unqb+OwmW9kwFWa<R
ztZK(4xl32~iM+}ErOWW)OQK6{a)(A^&EZzPyVqCjn>c@JRoBFI{gEuMr@J57F4*89
zV$gB0-FtcTmTmVBt~0JoEDyMG%klEPlq@3!p<9_NBlVw!vN<wb-@M<@X76U!KW4e^
zyKmiX+q<J*Z{p?l+XqyR?mg<Z{F2PK&j+;`7gSfimw#Vn>1a|NJ)Qe)zD8Tc&&$EP
zBmHb_>>|~bCRJLL9@|m7d4i#b^QTvl5x!>ozgH-gs%-MIIN*1{d5QUHO}?kAyqBsh
z`D3Uf-jJ{U@7#>jtj+)0jWjt9F!VId6jnKNayHAA|4UfT2(iQ*VCCWVQri3?d2_M#
zbo*+d&cI!{$s!?Dohwdnj=W^M{<G}k(+<;<Cd}Dk&gd~U-g3*#4f7%cxvnN{S(RY2
z&_V9YrY)}wyX2NQSQT+?pUlK?KtU{1?DeFke50)oXKnm!IDcaDI{7cF4osFW_u}03
zLHGUTRUHe~=}qC&XF04da@auSu;Cvk&4Bflr<S{I+{VXZ9&TjdzR)c5vDpM6^P@?J
zgPb;NZn7=-y{(UHd!Lc!<XQa|(sm+Ow%R+(l&_Zcm}RaWxyU8zpo-AO(2XwqA6E)=
zE7ea@x~gTp%xRXJwDlFHqgRa99;ukMy`rbx`OxcP*Uraw-M71?tpBh3V9ecXtu41%
zIaJtEtn<*L1+zOG_AFW;eSEjp+L@g{jr$%s*rb)LX3214>|oeF$t`P+Te?o)woO9Y
z7fJ6}Eu`SlkjHcU!Hs2nM!R@*xJvdM*q+Fl7Ja;U;*3K@?v{@oawf02I&l~Op98CQ
z%L=*ltj(U<xIp%)weU@)S&@ahZNHCf)LxZ+(~-xsyM0b))~;T*CEPoL-L|}#_-fY1
zd?CG08x0S7$bX*Pp|OVhTX3hQimaiGoBHgI#5arN=XCV=aMl%SKUmPsa;X1m<X)4@
zhl4h)cer!7Dre!1-DamfIIB0bF4(hT-kR3GUbaVy_TF|jn548+P>1XLg`Q)-_Oea-
zZ@`n|a`u+N6X~^8!eaAoZC~VIJ+o!s#u+Bd9mL)yyBhowG}yI2;)~t-m#ca%D}P<A
zY*8ew@Y;HZ(aBHVoliZCc1H5OIN`Q~`S>JZ?%iuna?F^VR^!&$aqM}=$<rckoe#v%
z_i)_x;Ja7B6Skmh_pAfuDW|Wd2s)Q+<C}7-b%Rl)m5S|Yk&S!AI;EC5KIYt*!1dJl
zgk+3Ki_J2rDWdm#bekr7T$^ID-Gj-OiLYl(_nJ&Go0%OSHd?FM%zw)xZGOr1S&Fy*
z5(|SSpARP84>Ni;HFQmV$;TMr)jxCd{4Xben)v6uJe>Xe@WfR!98LIZ49z!O*){V7
z|Fn%87A??BF#gZIM@e+^mpwugJA7I$JbS%;%bX*E(WisIxU})OiYQty^ISEvLTja<
zb<mBbS6w}KSFBRrVwg8;V}uCzx=Yq#nUhoud*17)D;V==>F^{>>^S1meQ$-(^9Q|!
zE!N>{cHCtUN^`mN;((9X#ok8+eY+IS3fdmuVz8NQw!wv0yNa`qyUba!c27t3CdDHW
zI>Ec#S6n`R^TzS$q~JR|0lKlAvv+a|Z`|G#a!yi5>S?4$>zsX0*POq>bx7*v7NyvW
zTLc{5S%&O6;pcm)d$+}G&4Q3!)$6ur+Hc%&bf(OncROcAI`7$&v-Nxr$84UfGkwlS
z2==xt>NO8MmD_wi?C<&|I>E-p|GhdKyuQnHM;*P|5fK`_bZ=+H`S4y_ChyQI-2JJ+
z2KuG^ACwd>JW#)-xNA~}ikM4})nBXqnHM^LXl|Jiu$kkWeQ(E#LawhiT{@0p$xoIY
zpJBM8VV!UHQ8Cf^d;i#N(U|mY<Hac)J16z1>1l8)Rm&@BpE(-Rss48Q&WDQ&*9fUJ
zcz){fOgnSpQG!pv+HQ@nfjcL16u30Jx}dwbnu9llv-Hcgt9K4qem-FBY<_x{`^;Ta
zgwCjPbPMn*3uK-Oekf=Yb5ic%l!K3^IJSzMJNhE*+7b`$N4lI0?mLukJ&*{>zalnu
zGWX60^Om+6aUbe^Bcd0w+c0*?G&8H^hyFz5bpD_HDaJQYl>6m|t-cBV(^p(w`}c;?
z*TZ&GIKJJm{b<3xELrnt;?2n!<`u!5JeRdSPigXZU+~FMS)n;=dEwOLO&7{DO>9E9
z9G2h`x@!NH+n1rh=j75Dh6M@?8GWbD-gz(~X0fcZYRh$X&2BR-^#jK0p~gOPs<Ja$
zPEP6$(qp-#*y@(GMZwO>=YEH$3OAq1=iVRIJ)Lir6gnFAJd{}*;%mWbV(8o1YpUVE
zeZg$=0o_<0n;H?5t(%>nxKAhuHt&tz!ok(}O1jCVhi7N-Wvg>@Tc<9OjM<VZ_=GcL
zJA=bUhPciby}zFFKG_k!D3oDk==CU<b+W#(S+ABc>h1rS{6BshOX$l9O|O4+w`>w$
z5@Y;5#w$KFbn1!F2-j}zXIFWzJ>a_*8awwv_^+#rw)9x~tYNa=uzJs_$-I(}-^Tqt
z=(~(r;O~wNnv(<*oYu2_>|Dq&aXHt#oT-sEQ#}{`3S(>ZmRDS?G9&&w^9;3Pv1)CP
z?7DdNSuB=tb=?uvxslQGz=fmLp_BGs2;UUBPSLybCv#9|!YvKAOFw5v-0|3S*OYIn
zKqyPY$;P>zPb*^Y`F1uubh27EasP@X%wkVATP&Q=barB|s?Fr+*%_CGmk6`$4O1+W
z+PWfg&eoKaLn+&S?{J)EcjE2+<)OQ647Y0DU?}p^D^c=%_T=cE$0>=P54P<1@0nKF
z{Wa_S=G>>zxm(kjS8mlkm0GRQSLnX9xpRSa+?v-iQ#&~s7Rh?}1?a54uupdjXK%ok
zgxwo9J2pnIT9$cYp;ecNaNzPuQUPBk9_EhgXs|!(!gq>M;`GvpXA!Hf^ER;rFlq6{
zs(Vh7l(pR4x!AK;Pi$tyded&3zNF_1e45rCn_=@@!#26GO<J!cqj;H6)|DlXrrwir
zy4DqU{8Izxv!e?qKMjg~Y$cp^Z<+U8cBX?5H;cSY4X&TF?6#%oD@|R+zLzOojuLye
zc%`P!7PIm4yRvD9&~b*0SDXx-5m)xzICs$E>7+NOGZ#5AAG;D=CNyhKXYNY*<q<PK
zZs>`f`oAOg=vA&|SNZ+CxW!)26M4PCESpa+n{!#{Q6U4%7X2SuiXD}=e{6a&!B4pK
zaDOe6anHQO^~ulDzr}H`oGQS3QSy$9T-l@<`{b-9>MoFEUd16ceX^UHZ}-+$QO$g}
zHP<~?y!p~9Yq8m?>$0qmYeHXbe$=A+mDfg0;!Ildz1E1cEibk@`AoG)|H*y!a6{16
zN3jjPeOit-zEYcyHD%@R4Vs|&($;6YXXfqXvsSKAFL!B(pH*0Ipm-vl@saPnOo@|n
zEqC5|=W)q?J@|RDr_j@R*{{x+EIaTw_RxL>^^j|~9=5&LT$IImgJm+~Lf$mN=zp&^
z-+Q&ijQg)17sGr`#_9i!SM)er`)n%=?oaYPzDS|hRZd}=#nm$}j!*gV+O6w|xASRv
zzbzixcb2_!;fcL3P_D(jbD!MHDAi**zd}2&hd%F2c%I|K#Hz71=(XP#k4ZA(3>W%3
zE^NCMSF-qfU&NJbk2fb|H0H&&ruF<@>v?w`XQ%0lrfJ9CG3Hfynm-FYBF^?wm;J(%
z-nb`kFBQ*xJyRoP5zoqMlbfYScy4ao#dV+G-2LAA<O!Dr;-4J3FmvDK4Z$%-)hyn$
zop=*wZFFMRbC*YYT5g;bZ^LHuZEv_TNu;MJD(idadwHLdBPaHq*zl_Pfd&(M0spi7
z4c|ilvwi*d@5^7lubZD`+2|A`ri!rD|JPU}6a3HfUFf0g^AGMlTW0R%vPk&h)OlY8
zj=1Hp`}!S{6|<k3spT_KC$RI~!pS=#ll101ms+3n@mNyW<&AR7m1CalVB+&R{2@H=
z{qq?;mws-3Tkm*jfnK?U{8fv#7jms{%htuKyS=!r*Z0%=%dX0Ej*F(V&oB8Fdi-(Q
znkQwl(<O>G?<jpGY4Xyz%<g{X`(tuGZBy&|k0v<Y^2*=3P%lee#Pj<yhY!;#o-KK`
zvq*n)Y>%IRw*T?>f#H=K6<#xne_eKF8?*SSr_cOtwe#+E7tCo-++*sfdZ{$M?}_ox
z+g@+J%KW*uK<nr%zHPER*9-33TQ0ki9{#vh)$Imv!;1_5-(-q+n@Qd6T^$&rmutoK
zc%#s#bMvw)@AGn2JQuU?HcxxerdL#6U$m_~N$t7Zs%J_En(kSY$Gy7wV&eq&sTvij
z);yu&YfePviT<0i%9-CHJ#+Jg<t_PB=PD%CXvcf*tFf_nU6fRjrO!3zTBg{=z^l`K
zd2inB+PZF{$j(jfJ<n$?o}>1lfBMPx?2Y>qT>fnH+Gy}Ii2uWM|H@$XufgK~_|l@6
z=IpicStucO`QY-e=N^dsR+zb~H}PCYm{Qe|1^2)5Z@+LU$ndfht0?zGS)H83nI;_-
zIriTx#CfBwD!(n@YfzYxBz@ib!u!+N|NqExZ>kmbQFE-{#Mq(9#G)ecKjVW!Q#0FS
z3716$3l6tC@Tk>91S&dri17N!%qV<(phebEPe;?iX%Y*klEIWsu3ZdaYdttus4kf2
zG@HMiE8}SK@^kZTn%QKtE<8WHh;1rgiq@Bx2g<B(-!zKY`pRLAckQ-SSHG@4yTQA1
zQ<K!?HBRf;Zhu&!S#rhEXm-n_hnC`R{fwen&g@-1eSJ)GH9Oz2>>nGPGz~=EZgM&t
zIMURiZtuIOqHBsK6RXw}r30TGE=&<DmdpKR^6Cn|@Dh;^Mcik@HnH9~+<m#^ZR5^N
z={tLWFL;0NQ0aRfzdv6>8BgZl+Ex2m;bK|RT&@XHS92m)c(hJ))iSoe-I#h)LdE}|
z$^SczF9omHR!v`j=xf3C=X<S>z4_btll4t~#eVyGrY+B}`O6!M2)c1FPGDf@w>1-J
z7LR2~Y-Zv7pdi30x?~{(Baes+lb4UeQI_B-9oN{b3l$D@v3Gu($q-TyI3ry@NbRA6
zV9b{Xj#^JOZnko%9#M`Hl=&j)WcesV(cNEgXRsIR&Mnt-tXQ|s^}Y6Y$y0}Oj;^u(
zX_YglMf%-*A{?0Lv|M1iP3F?jurS5c({o~Zx7?lUJ~uSDEaBgjyYp;v+n3L>oyerI
zz+@}?)oS*=*H=`t*=lJnW%gZl%dypV$x5g9xgRejt*caFT(<O|NCS&k!na8bvs;(V
zT<Krv6|rXPt^ZM4*{rou({t8(v<K&2OZ<AfU{7hJ%qISy&(t>_*SxZ3<9RXenEb>0
zINubVx_d1~H|Xl+++fSz)M-(bv#bLuvbG7`+m#|~vcj>!&+UQp7bzF71g2kG*VpxN
z8Jv<RKB!u7;Qt}>4F{R#v#m_v{T_Y2q2=t-h!yT^njvo9v#cKa$#E@cVNsdIp}m&b
zcY|<mK<-5)aosXE*9_*uPr-?^j6wt^h&g8joU?gfA~;FqtIn0l{E8eVQ|wh6CV8hE
zxH{FRWf5!sj?n8{CSOVPYE506#$*~bJ8<txnVTnjWuJxHRxX`9w?a`$KlMY9tnl7c
z-6+i(CaIF``}r$h6~r|E{ja*<LG}5n)dBN&ExF+rWDv3?xg&U4!>XA(pG0LE?9!<0
zX(;|@`NC^Ci}1d@uTjyt$y}^!3omZ-Tf2Gf>9{v<Ha`9uU0j$H>R-HD`s$RDKtt83
z(-L1-uDg@Ly=s{c(~|0U&d(&>6dJz-vMDU(IGsM>>$FE}FD$CzzjCjJ=|T3(xab_F
zRU(S*ud-H6Vr85eDPP*+&?+9rQrRl}?EGa9HzO^f&NZQJo`OuGlNQY=T-a#PeA7A0
zMe?LegS*3{i_H=yl3d?9oUD~O{oB4xUX&T2Xvfv*rLuu@`B@d6)PtQLTPAHajXdk$
z`s?o1EQ@tApC)8qxScqQFUV%a0Z+xW)n1c*Uxci16kO`xQ7OH%@4Nc`o3E=U?GCwj
z=FkS6u>PR=L3gsnj%ziytTO1E?7-mlLtu)5X36A02cfX3Gta&`>SnWXan^?!nyoA>
z42nNlSVb6^7<3pI7#JFu92glm{xh)d5OUZc>d2VO$EPx5!ULxkc4e!Q;I5>ZMnc|G
zA1F2SFz89BcF9;yn!*qy$JwW7;q^#uVoJcQob=TRYs=QUbxu6t#jv*7mXl4@bIXdr
zrq_IGy1S-iTwWZo*sJ$e$yLWdq3e;N26@@nn^&!qyQ9UonxW9;tW@r*cM4HkGd%ky
zuL#TaUZwn8N_JP4?CQYecU%WNFUOr**}=NO@Ks@kOLH&3RE*DrO)jSem6>HNuPnLJ
zHoY~^?ar#u*+Gg+iv&)lZgufvPvrKH-6#@rfk*L~*n_t6nPyw~m;H%oQ<m(JILYy?
zX2Q;sflFB2YfoNO5jUEBV#6la%~RKgZH?IY|J0FJD}$QsFAJC6DapM5Ihr-XH>)(y
zd&{NsTqooW*9C43OB9t4dhU8PEXjAF=={)ss?Q!Y2dp#=jb%zmT^J$Wb!2|1OqRw)
zi!~8TPim~W6T<EIR^pn3m%iYU7?(?5pGdTrT+4Xe>o)OHSD!{M<7LN#8_j~UPDrNC
z?p?RzV%MYtXTw4ykIT9*_B`Kv<<eBuqb;%8U$2-%XuoY)epJieRb?@Q@uL-L!OAnY
zKb!40clwn%?4_;O!dY}J#G)QeaGh3H^U7U4szl8-qqX7M%+R>s2WLjKKkZ%U>92k-
zrQ0@cUx#uUdtp?FKik2VD;X9}S|#DXtScqbIpm$1U`8M7|Eq18%NB}E%wc-4OD$*9
z(reO=j<1g0cHH6m#7*+}u7?*3oaESroVK~Sa%tuyNQTzkOm**?{-E(VpZ4TmB>^)t
zn6~oi)+vf!30m{ws>ZB0Os3OAbuVh)cWipvy?${Hm%-^Bd6$x|>EyjC7is?fX|+MD
z+sEe%S}K!{#?M;((M2iY(%cMhTg#1~WHyV=?p)=uR4?b#N#(aIj2!LCt{8d77MR=?
z2|CHMCdK0Zn+Z-f;wqCS<*9T|$UQN2!X-C9ugqj0b>8hki*@(ij=4O0gJZDml36FZ
zBvkUuM2-fD%1pe$>h$#fO(vDA(-yMJYBbzVS-eMMF3Z`aq25uws`GbOz5G8@eG$vd
zP%+2aYq{GZ8jZ7dw6nzBeim5sa_yDsSv?2+923hTS1k)Fe04Ep<)X~&l}q`qa%p!v
zUR#-QsP*3r7lr+GA{j@g{mztDTUeLoxMO-)+qwY8cw@)CMSjUns_MU^H?T;C%S_oJ
zykyQ7gD|($trsmT=6FUY94{%q-+H}E?fuV^bvx`&&d95LbviumP}jBPD<10JR4;8V
z@?Nr_F6<z0{H^bc_!)ke>u+f}+E8^n{+|r%#uW}dPJFKd4qdt;x3opWW$^{38Cp;0
z>TNxuqW|-Or^h0h$vs((8ZE)ILar@-xZ{IDEJKh>j+@Zz!Y7e(p1qDa$pV*7UPxCF
z?r`|u#AdR{a;1lnwx9c{hFO}u0^a(aNsDFNRNr$R@NVJczkY(%Oe%Cj9)nJZ#BmSx
z-Cv*fr1hNE`k+4Fx$|Rh0OR8uPgoq74i@@lJUONH?9)`!UxhMXlO9LPux&KuTH#p#
zia-3>@e_-8tWJ0rBvJkA&^{hMRiE%J?pvo_n;2}XQO0K!doC>8$Ukpc$O+@iFE2cC
z4mUlb5TCp>)60?V;nRNM{oPy>>pUjilUeO~R{A9G^cPERi9T(fQh4LEl8~uDR%Ueg
zn`u8xCYpWSnlD;k7{9ma(e~Px-KxKtmRl?}o0P}b5&!(pY%B2<>YVpZ&lP;R?2v-R
zBsQZp8UoU+?Rwk(bJb4j@VRkhSv-T3QdB~)%`5{>cJ^fMup>v3ZwPYUSP-n&HPxt9
z!nY?q!lUioR-P&Rix!?aQ84%B6q61s;RemBXDZ<{6Fc@8a(w0pYPwm`(#BWhcbV_g
zmT4}!_G%AXrtMK~h-zS0k5&%cwd&>$gMIGWX-96$T{a6{5fr-W*O}WP-;XV{V=BsT
zTys7Ay2J^am4PxzzS`l}tK62n`PdWrRz2pnRruPft*c{swIjaoI<CENYoO5Y>_={f
zJB~<xo0rx3r1FiYP`j(k#%|?PTIW`XuJzfrX^Jkts`lxw{Ef|G^Ow5ozL<EndBzDj
zb*Y8lg04wkj9^O<Q001&<9@s2zeK;K<IWWxJ>s+GOv(zg7CK<G@RF3`#$%>J2Qwur
zRxgX!JFTI>$SEl#G)1EHQ+8<5I*%Hcxlg&SR~!HI`{8AxloPimcInhv$%`*7{I)OH
zZqb8)nuoi)e%#u=@5kQtbx&X4`?v3V{<|;w_4m8unWT3d_%k=ZPJGS%z#HH7_*r7C
zc=yC{JUs68eadRP$FKc){jVMV_ousqoh$0XwI<oG3*7Br7*4AC(JULQ?cuxQ@x}*f
z>6Q<tWVn_8QIp&1w8HLJM(;fBGiRoRw6kwKJ2Ap)Li>_sC!J$1U5Lus;wLweCt_dV
z>UB@H2uv!@()q+8=6i+DbK%Feu`j#qF8{e^dRX}X+6bwm8ppKKEtxl+D-?6}b~*T*
z>*dY!HGEqZu*+t+q@K7C>=H6Zc<&1v>xzp)$tDSmf1h)^{<>6uOlTQ9w}p#)%f`8V
z>i^z7nY#U%Z=mPO`757tT{ZTbc6Qp?HSvEw-MnzgY0K9YuC`T;Tk8alUf(C?VE-$y
zTSDBlnD?oQhvJgkD;l_zHcZK#wJ3c`fUk5^|HQydzsIriH-%D-4nI9qvvW_gjrPv-
z%${3JcAeznR>%qz=Ih>(dZq1}lW^1vgLc6+_qjgaKKS+P_IdI?AAT{_JXkC-ao)<E
zAGlQKs9w3lZ)&Q@)n*^_h;RFjW5(w`HkSXn$jkrpu<O3UN%b}b%K!a;EEZi?G}-)5
zY{W&2`hPM7Kiqr_gx$W>ecxOBC$C}?tL^ojHw<ryte>H9a9W2w-y{AtoCz;k8h>5p
zXUY-3{#bwNB*SH94-L)k?6@g(F!$6FrZTzNJ?^}mo4(l1a*3;ZqM#eB^rlPb?t>F1
zCA;?i-q2zpcyEE4N~oW&@WPZ--S;7?5B_9M%Xzvqd3}*fDPPYVSw0Ed@OuZ7C3Z-d
zoUdfhzxR2r_))(fHdPn76VzAO?|l`l&*r<}-q&UKZC<(h7hayc@7vP+zi(6ft8Z+d
z_x-?f+YjOL)wjj#)kRP3oj)<Z=GpI`T4|ki^Z(0a9kdBJpDJlF|D;sBLs|T}Qr2Jp
z*LeH%NB2*=E+}bssrhdAlHgjEckQP(R3GF!y>yzv^hq7NpUX|W{!VPl;zyb*9$P6L
zwo)!u-k2gXWnu2-BQ`reH5jMl2}$@I4z}rAWOLjpCjYz6%0ed*(Xz`;5+BuUZ$4Bu
zNfdstku%Dnp!ZmYvQi;uc%hK7lF5o96OCqbk3uVps+kIGGZnaOBNT0aG&^&&*k!c1
zRJ6G6Xz{wy;`yU_`S%tNi&h_vRv(QP_ZL;xKbqHtE5>#-8@*^1Rc8=%Z%ba$oMus&
zTp?<8B3)O!nM1stXIp|-YQPI8evM;ETE~)<H<r{{l-6sM1}b*=u#5U`m!A?8!gf(r
zZ&F;>|HkA+ldVNl%cfX}>V&XmeK*o?4t8}k7IKt6?9!MoUhb*Ju+30_%}`)DN3O1_
zj;)J;SfSv_MnTiiV3ma6qUCC#N<wkl;xrUPTv{#74cP((B(xQ)S5NO*v%Kfjiq<ve
zg=^j0E?E>_Ucq|BqnE|7@Y;^vAJ=;Cc(g?+Fhot@eBjYH)1dEmM&FB${O2!v-*oi7
z-O=}{qVLOz-VZx^AHC@NcB41tM%A?+)z>uyFTCiv%Q1mXvx-BrdfN7^EpBy-mu9~Z
zuUBHs5m1_V$ZVp@!xEu~*?X^*+zbvmUZ&a3sCa&9v~8i&-qMCm#!}iQWx<I)(;B_>
zE;-vea_uOVI`jWiF1JYD&L<itnsgLcyUrWOUffi_%~kipLZ5<i0p%NlI!OXw!jyCa
zo2@i7MV~kCzuvqqz0dAO|C1e4BQ+VEHKxT{PJ1H3nWV{(Q8_Jh=d>(MhR6le9&1cb
zx12t40c&C9^iofOD$VINoqgp$r$3jN5p`jDOXZA*5i>e9`|4-1c4_ui{hZO|*`HY{
zF!^Qk<Bpl3D>z*}3S({**@XwB9OC=3p`9sVmiWwBf$SZA<sHH5=0W1K*Pbl#n_lAk
zod1!c*V+f%2U#T-H%zjc)-cs#PX9q=H#VJl5{<$d3?U4$i#TG>CyN{l5pnt)eWFRq
zt5{&QV~|XNY|MwbyDBP}{$B|GeaYwSLe=mVQ}<A*B?7EhSM+|c?0w47cT1xGo8|nO
z6Ij2jnE!YNYt_s7e}B$rx;6ii#SF%(1qG1{{z*>fo;APGb2<ap^m0i~?yQ-dQi9xG
z3x!TCkebCBxL|(1XCLRS1u9+(`Bu$Wk?N27(a&L-{`ZL8^PsvN)1o9gOWwB|?KKdx
z)Dp5@rS|!G_AxfK>6a=@nj{?)twf9)R)^?Kn`qU_K1ZZEy8EG_=;S5AiyIa#R9@O(
zeOgR<&W|$J#u!1xWr7<!1s}#1EV42EY-434J<rhgN|CL>r-}Yw^4t!_hOd&5Rb6sX
zMDC~>Yt76BPb8)bHcnGYTv4YvvuEdwp8va6^xax9L90(*iZ#k%<+P}kGqP67?OG|%
z#X9HL%2`$`7tCS}P+(YeYvnSlRqr^b&%3p1i4^O)S1adwt&)pcIlF55lvyoY-xrm7
zlvGyCGRs=rA|ZJA*DNn}(W72MM`x`$ajV@mu~aTuNn=vI%A(*^u3WJuOONbatA5bw
z(ATvFDqRa_#IR4Cb9jo&<6ld4COfZvVZ8K`G;^aMf1xp3qm4+R&L$5wQ&GX~H`ZUj
zxOAVXAB&=^Z{<7<!JZ{f61I)~ot1r@Q8O5BEo_rom@hd)#!Eo7Ya{!{RYDtAb!4rQ
zmS%mYxoMf!CN=9#s@a<qtT(AtZ_;|bNtb(*Ame6rZdSGb*_$n@HwOxETHjvqdPn6<
z_Bvk8s70y_d#){BHM2x4bE4qFi7(3+=Wp5)@O#U_t}VW|lY}GrKX2!fE3BUuR2Ik>
zt$f+l=Ap!#OQlvq+ABWgtP2%M+HSSxFxQ$xY?D?jjoGzU=ArU~NN)yi#VjsS716on
z)}oq?Q{ux!PEQfIx0zM-v8?JL%ZDBU6Hm{W<h_&S<HAW%J7>+_S+H}}l3A;ko?4|4
zy=uj)c}sS0WEb2eB)DsXmC!0}ArA!xvu>6^huzzvckig)y~TR>2Cmf$X3aFMh+ABl
z?Y?}L(!n)>2e;hmGTOXz*2{KN-9wJj!NNYpLae7H%Qsr*Gnw5A?F<o$IrRUd(Mid5
z>=);-B<?f(qGFM1eE6!(MM=S{KUIzx3FSJtoZh_Nc#Bkk6Vv}!TBbp+Hl}`drAse4
zb0u$({WigrOPW<MVP}h$V5{Y3<%1h#xJ5;Kc1`a-xJi4Hpu;X+p4IYa4w#)jASk#y
zfPrzhG~>2zhHc%4G|n8-+p}B1;E<rgCbeJvQ-jmzUX9$IR=2ZEz~eT*_wqWoY>`vh
zM^0LAJ-yo?to(rBLc?(Ll-Z7w(yB%`PDT4_i3EG;vIa`6T9kZA$Y_q?F{>bL1=dDa
zH-Qtb%MMT8FO%4q`D^l#sd*9;1x`*8`uQ;8CbR7I*Fu$1O#wHTKe)J8Y}bt0w>Pev
zy=(D*?cLdn4lym*y;J(+1e=q4tq;wZ#k%%;#`CAE7ndIC5S%StIZ<TinosT`EfV$f
za+DX?bew&)=fn4sC~u{=4gzkjTBZ}N<(t>)6)iP(+7|He^y`gBmK;)+WnK3~%IDRu
zvS+8~Bv^HwopZ*R#b|{^+1exJ*J1?g$_3q;_p>})FMT8VRcl43o5=YshBKJeYy}zl
zs|5IEM0tG#-s>=Yv^j4qeg1>ad3M9|0SgXk^&I-!!=le~!Gqy2)7wM8Z4TL1PiNKS
zYIOA6`s}bZS5BSA5fAkaKkZZdmmLX@Ui`lNVu<u9*EJUvdL>rsT=L4k#QyN&vFgQ#
zzb)>|&}>_%)}1J+{(t3ZjZ`mp$L(uawr^4On#!1)*(4cbV<etB+04_sV)Zdy=6SM>
z-c~;qp0N3-2nv;c*SRuHvRK^uyyJ5BiL!c=WEO8wHF~z=;l~q`tFOJu**W?22^PWK
zLNSL{ckkZgeR47n>y$a0r+1y)%3ko_g<Ew}`$zXv>vV1$^IBtCbm`!%i=Db>4&C9`
zV@vK+j9dN0@IZ|1*3jr%tlN6tZaX-&)9qpB%V?3)IWe6d_bGoAxEgcTNx1t~jllL5
z$Ic$!-^X;|mpNNp+Povd=T>Z;_m)AAH)f@jw~(B-(0iRjGCqe-@UAvqbH}3l&?(>D
zYOm*8pYC;-9crVqc$eE^Y0grST9L5-)}qceHx$-xWm$MheCFcM&y)U6SRB-Tqhs&d
z`@SL{ZSPB3<{UK>QM;r%)%V<-k4kYeT1&q;El9Rnd^u|E=1KOoH`zEuxjnYuzUE@~
zX{pdWiNh?%&wibJ%INr|(yNCULuLl94dF2j_$XxeX_=}A+r5d8>TU^p2&`<gImi*a
zdFq|ryS1-RJ+rcFCF{&J1xjves!se;oTnNy)|}XV$u;na{8{~ZYqscx-P_)I#4Kx~
zM_k8o?Rzq@Yv%Ply)esgW=PIeYul|~8<Y3kXW4Nx;N#vU4V|)~X9PE%vA8-%X5*Q~
zch8)>xi4Vh_BXqXk4_Sh*qE|2P)qx$wCAR!Cnid5daJVOKkwOnSGD%cNLCI!Clx8R
zcP*#snUzY~cjZrQHrf4BefLVOTU9f~CEg@w8%sTX?>_O6+QdFzfse5h*Y~_~(P7Y9
zJ8{duQ%}!cT4|TF?*ach$)~z+Z|vUp`c9$JzYX`Ic4vRx@Y=0!i=X(B2yLNfxv#tC
zZVCE$@1R+Zewrn_pj!Ci9HBt3^%f!uhc#DjzQ6RywoRdWXOD3$oxJob*TW-|10Qq-
z&$}_#{*zkPt)(Xd%X4Q5%yT@EINkRhTYglo>oO)e)k9p#7alE~wUV!X<&;-5s&qAr
zu03Xby4tOLO|$RQ*0mphUcT3s@x=U5ov+-J;AsB3zfb16t_l18ByFDkzOs(l|1a#=
zb*-XZ@LF)<l*LoZ^&+3yeA^o>wNKg6_^jr0mWlgZ&!1V6yG+S&U+~?ovUcN7QMvn`
zaBYfU`|(;Trl8wv$qW8XsjEcKv98uNEY#fUBVieM_@&2Pp1l)P9!1*5yxQa^q&ipP
z3$OYLnM<$#mK=+F6PW(SEpYY|S;0MGIZt};x!u0B?*QN01pa*+Ccf4^vNMf;PaXHt
z1c7}Ie)Q=JUiSVO{O^-!=9^>#;rL0O%8^Tw&1DYEoTO5`#5zB9UHlGiyDKegF7tLs
z#MO%!9lqkDD`;ZOD*R!NX6Zvo4$+dHT<6$|h}Z+1eN$q5MO5Uzsn-cSZ2mCq$z%0t
zf3?&;dH&xsYjw^kne!J{?7I{sck2DxHM;Zv`Pa>k*!;KC_Fp&eX`T9xh{s7QT)8;!
zB>i*v`RzSF!^2ggn$c^swtiMW5VkvPEz1_+1FNPnbcyP6c~k@@AMKI0&3f}aFvVk{
zs`j_l&+|nz!$j2Wwp{P(X=qGw3srI5sOr~cTpV>LbJHT{E|*q0*I6OTE5nxF%2C?<
zR7spMWM#~rS)m6)yJEI;%?P@<+I5xG;jSsCQqv?Eb8bfE6kkk?=`HNsmHBkpnR`M^
zza|MV%wU;*FqC(DXq3_O^WC#q?(F$Fk!R&erZ+bi-F%m@HZb`W=Zok1DK9QAa&KOD
z-M{oz-CjHH53V0gkH&TWPmi8c`&-HLg8IZy+c^HHcwN<qy}l~ETXt>7mtPx&l~<V^
zeYCrK@}70GstT_y`g>Ko)nM_fxFxk$i!1JhY8uUYowzM@Rj76Jx0=|rjQ(u}H>FFz
z`|IrJt+4Jrz}Ua5Z|VuPTW>Zo*v{(g?N*s$uwq);CJ(2%dWQ;>x!YHHIL~spbHRC+
z64yk{uv-~Vb-OGo4UTQ%nf+0TZ`qB8MN%gpEIBNpujQ=2*GH?lTd9A^o8toENoNdu
z=ZG+Pb!#-QTG5m>u|qV><t49j-@Ya1yr=9_{=BG9=hcfddQ(%3S57$L<0lema#5*E
z=S_=J*RvxIPTW-wzVw+g266G48igjO|5vhzyqGd|%A1YtBH?>@iUaEPluyj7H(2}r
zQvIx~^kDyFCD)5){;sAgr@V?hsmiDpwPLC-+b_cvi+etCgbM}NPS6Z`aZ72{yuMFe
zU3w-`s~2r%nb5T;ZK}~FkDU%B6Bl`|y?a$u(OYw2&%U;&42B0z%vk9bBEI?Mq7!o6
zo0K%wH?I_}&MNFsJnJF3c*(^m<twJ~zjnU2JCoF%Yu>#~xXYeVz^+2Mt=c*_;J?~4
zag*zR64kj>7A**#Y-lCjDq2+~$lBzSrMb>#N<?Iz+=YG7sUlHP+`V&kloDH?MVT}v
zh+I84d;PwLKYTsx;{P!2n)~^WW>iD<t8UYa=Qmh?_<y{8^|P&uRx8F?Yzef=;n>+!
z!?VFzP}q96(k5M_Rd44<Z4BKwDI!sMdDtzDr)$&BiZ)Cx(EN4#SQpdxj_&lIZe8!U
z){7XO$g=O6wsyy<A1m|9uExE5U8)&1UBoPJ(xtn{X1v`kvTDJqO$)T6lhRMG)t<_G
zVM^A=rD9jLwIo)y?2|4&8^ZmC_gr__RM~S;pRTxUcQsF2>f`oWM1f1!L~@I~@1he+
z=7qXD9(r=c;2aZYn4GACUGwojJuBUFCw>X6_u_o-_3OZrD+&EqmU#HKCW>$PZ_4X3
zdFGC3%kI^*iv}&4u6xgUVmxPqh<RiWv+RkkIXzC!6K+me*R@G|TdvFh=16az;3t=r
zyFH#po)9`6zGc#y4LaJPnF~9jmu+?Ip5mmEC(s~0XOnn-tc!t_6%U8xigjHH1;t{G
z)01UP=Ks9H6t8*HH)~19q|YHAjkaxQ*50(kwJU+G{M*knBFksUzWtH!u-=g~cNxd5
zs~PR4M--Ja*FBcHs^QMixyb#;5eJQq-cDJ;2B&jEMr|4eIijy6T-6o{aJpYq60%*b
zeE!pu&2<};{NDudUcMI0eeud-GcE0qU;`oPKqX)MB^~lPGme?8eb`q2|6{CY!b*dv
zOUG0;Gu|uv^@6{u$5Bb-B2Vp`kCtX<Rh*bSX1AOuTgb_oP`I5-ko|s}Qk1Tnlgt0a
zE`|@+*p40W=n!?h<F>=#oWisTbEG)a8>HHu7rLCDdB;WRxkk|T`7Ua?u2U9xEC^9$
zztrj_k=6Pt;yRb_0v;80#e1uA5_gB?Y~;-=I4s){G$H)xvWlQ5vbL`r`JY}%l4a0&
zkX^A|%VaHA?V^Q#PV=3+LoXb%T%8nJ66UC`k&-C-_XLZG(Z$_e6HNRU3Giy0Ju~iE
z@~kRg!O`>y1@1jpGes^Xw|j~G+&g8*V$nA*ggk?i`Ri|F2hTU^vcD6!FFK=Z^?jGs
zo*$k|-w@PLoqeOzMMfxK=7Nk|9mz#Dsw)%}>n^O%_&RlsNBN<Az0<9tm12pmi^R72
zye>4{bK?J2okfSuEY|7wi;9ODv_z`98W{adx^7qbX_3mQ?nU2%&Pn_;k+J%uoe+NJ
zk!%r1VEBxMJZT24(lLeY;W-yYPYC)}stIil>~P{KId$Z0-m=#IX&bYjR4Ql2Y?i&O
zvb=P4f@IMN=l+>m;S7-%T!MZ(^6Qr9b0slI7X1)u4HUSZ+#<DJE?Ak1WzHFS@z3tM
zMvV<klf=1xZdx)wY~k|b4vsuIf5K}zoJ?jJ_SVi3iSYZfc!FT*E2EQ*sZpj)8*W`O
zO8TE%knQTTqQ3K)-E^h2K&dB3&F%(!ra4dSHlMVl|A_8-i$>{tU&;jYr?qIycwBSm
z2%mP~Z?9HMsLq3||3_}Jo9Y#YE|iHin!Zk2-_@xnG5lEz)6}$yO6ma%&-kx<B2Xsq
zMRkXy=1hqw0c)2lJiW%vt~Z1(?%BX(EE&oE<pT?=O2hUYhRk{&yS&tTt(KR*ohrG|
z;M*1JtD4+Q9$Lqa1qAzQWQdk`9s056`18OWJr#={6pLhO+4{!lm|C7~%08vdHl24?
zXl>Uk-$f4Gfi8?I-dtc32@~Y}7WO`nd8fwAq)U@FAL>2&zo6aWkeXnb&yG4qpEK{K
zFa=GJGpS$4SLSrGW&HvPl?4KltZU~;3LZ0fmOaP2@XoHY2V-wmxJdo$SU=B5^irhC
ziBq>a!cH)W2^<nI)Z3KEuB#OJzuo7RuGJMq{=b_Fy8cXX%j{j?D%0$^S81YG;Xxi@
z)5Cp+38nEtTNdeVSm7GJCoxE%_VeuMDck*~EaXnrXyuE%$k$RiB~>q$RYc*yQp0ae
za$LJQPn`NJYv|&{y=aH<GTuMStEQN98(DwRnxov{nZ9u6#1@BTzYp%!NSNBYYU7i0
z`R4+At~L8dO3YMy)opaSrEOo;tK;Vs7If;c1er2lt6X+dV^;KqBX;%eQAKYq@2V15
zBdfDpOMU-h2R6sRX~LH|^lX{-ScJ_>v-jq_WKwT(ZuY0GOuNsd#w2p*-tDyw7Pwf$
zmdahe{Oe4OS?MOGkCG&3KgreodD1LVWvbla|F=~dS5M?VUU{zT%RXPfe<z|Ps#S8s
zTGo3Dc6PkzeA;W%J%9f#Tgd}|1R9ev)6aNnE?}~%(%p5Cuc++s?ASu?=sQZ=x7Svl
zc`H+Cl*+@|rzO1hHOt;vT}>7<8yf5$PkOpy>(Wbdf!=bi%2OUUGW{>!`r_e?wU=ki
z;oM((VMgc;4x>t~C99V79yI&(N_g37*1*@BR|N6;J}}npSk;{{KXtMA``evS8K&I@
z+l*%#Fg2N}dMaP>Sh-DW)#OXt6sE|sMs<j%v^Gj~1Z5~Bb4^p5)8uPmJl%s&{br}K
z5vTT&h2_n*OAEH^*-Sl<C6<`ErpRfkr01r?MbnmO2%VU;`G1#%pm?P-<ArH5S%TYt
zOxu*uU7#t%w_>8Or8xT{qYI0yDqqRW(OCO^#dNhtJDw%3J+zr!=92E@V&>ouV@3mJ
z;Q;$x4pOZS{GB^^?`5og(b&|K#2Xc`#Otv}l|-v&5wBe*!~C7=uT|_`BE<YevbEGh
zYnw0w#|2aWCVhb~!eJdv^ImFRUBDXnc#_jbky)I(Lx1#r5^5;DpjQ>KzVyWWx*1!x
zDszVI5DqlZxO0Pt$C)p9=eD9F9HxbPT0ZvjJr;jabi9nmQ0VlYbKDZazgjOeOHEYX
z8z#_^dunz)OOGjw661@8UX4b*2Thtcm=<pkXZ<B8ae6{;#on8@IbL4w_`l<j5l`jp
zqT+qoS>5r9t-23vA9<@Q3aj@T*zGekDtOe8_H(wb!W?l%Ma7vO1y({vGh|Y39ZG!Y
znX<@^ZP9|`6$^AP@7pJ7Zm^oCb;0UBF3DfbO4B#%Dpqd3T_m&T2W!^_{>__Kq;qS=
zY3T<h?DB0knS5xgUACOa9Mf>&RdK&}`kXLjY-oB}s2SxjZ!#BWNOj+bNO7m%r+9og
ze;+>Jcxv)zuXT2ctGZq+^*P|s{b2>qW!Ya7w{3mJ!T(rl`>AcGW2Sx%kUe`sNac%B
zk>vIj2FnVICtlL&6zUM;o50kbFd_eN&&dFR*(W&XH`typRN(Gf#;-Y1H%I#UXNCJM
z=Km|3CmlJpT_H-W)>~1*Nb$%jJJpZdxBl?Wxw&BS<iiUaT;#MEjb3!WjoEQfp;=W}
z>Aa9^*X)fdvpB_W_qbl+Tcybrv|x_Wj@f%OScP9O`7GhE7TCF8n|~eW<Owrod0aa7
z!>NgR_u{|>!i$AlXFjxAugzEXz+}<{M^;U%HxqY#`Xu~*g=Ftd<-fCbckbw$8)=<=
zSwDI;FS8J<Zoz4{nFs$JFvxx^U(48eC1Qr%rah;%`yMFuzW*_OaiUb!AL|RvQWuqN
z_aq3)o}Rr|V|x4IE>+u}{FNO|UpUlV&*r~u+V_It*d&JCnq1$1ERgq9<9F<uwqSz6
zUC#CoC%#<~xHFsk|AaN>Mqg~}wbc(xo;WDjFjdF1KVeh%j&rAkIG<Vz`E_tt1aaL;
zo^&g_XHkwq+QogR7HD7lY!_HyK1pEjSAm`J1+CjHrghAk(lE=O#gns2fHmTGpWqyo
z?Ppp7C(N84#A&zK!LF%^ZO@7-3)g<32F9aH&PT9v$F{dT5Vmie8ya%(<Q@*znKO%G
z0uxQ$WuABypE>y82S2kX$Cr~Tx!k;pXMD~YbSU%aOkg>CLFH^cbGT%aj-=A{t^;dq
zVlA%dFl)}}yymlDS61tZ3nzLHoYnkjd+*Gw1E16qD+NqiWC{*#H~BMB_?KUQhx1Dp
zp018=OO+t&H=PYnn3T174op(`|EXq@B(uwk1O2+T9sP|Lcbwi6t-;Fwfk}A7kxA98
zcatV=y=8DsL6W1RW%|nY4-aS5%<K?Kb(`DG6U;r|@ufz*j6S#Q%$T#2eJu0>lh?hJ
zXw~^}toudlHP0!3EmwT(2n+wkd(CU+l){^9dDm6OsLFk{sovsVBNBAkIA}()UH69I
z-h!a+jN4rqx03}qcl!45y<oDt)1ceY-N_-%o;Z2_l)Vxw1*9)ey7(%n?DB-E!dLdq
zoG97pDOnh7#Iszfc$$pioM|jK)@05xTj432xNir?j*7=J$8K=;ty!?pgj3J@ioxqe
zJCjcBsGOXxJTJO{dFRseD|m#OR>(iJ;I)Ws{oh-7>}S>0kD8MWPhR-lJ@>rCF`tT_
zzn57IJ=X15vc!6priiWH#Dly(LJGYJkpiohG8{Vdp_V`AsL+m&KBpwle(}BW)%E(~
zjQhO~;p(Cu+f9X9uf&E&Y8?C|Q?0a}*HZexU8lQq7S7!hzAdrqZt}UOOFG1icqE+8
zt>);i)?T<JDd@-y&gRQb(wh`dzEa4&I88Q*`{&zp2NHD5c+Pjpp4Ztur<=prh+~D|
z&7Q;3t$+U*v?%r02}JU*@YlCEk(Vg=BtfE;bMd1H_P89*WR?TZIsyVuh$qXqz06n|
zm15d2)!b)kwQ{x3{JE+pCeQGCcz4YzKC53=Mvrd2W03gFFysH(+k50yqgQC!?pd%#
z`RT$<4MA#8o3wlHbX$0I-SE?!aH)24NAkkxw%x(@awp`LCGU`Z()FULY0o9;!iT!N
zQXLtSFD^N8gPCit50{`|LgQ+W`A?P|&5fC~<blQA*?U)Z>1uSRP2@Vg_Q4}((I>Aj
zXZ-X#nYnMVh4jC<%bjewuDo16=Y`;k4xXzg!_FOW`Y$oLx}a6hfH}>K+4up|o)^2{
z^#(HElF*yT8@0gt<w+)YZH=ygqdpv6JsFb8Tjy_O(oE@9l}~wI`ivu6H2$p15%D|M
zP9|`xwVW&Dy<V#qKHWI6r1m~j-=&P344MV|SZ2*%vPXO3iOwCCcld-JTw^}>Ke6{*
zzk?IYHI7FyYbM)tsc=t_vYgWt8$Z$F&Ylwsm6yh!VCpH{eR=;;_3jruPDdh6S(ta7
z>S>Q?mOM6V!VU=;xto)GEyW)02`jqTy<w(@#iA=oNr4fqE{yK$Y9v;cUete<X_n+J
zc>2~lQ^U$Pf$NNxu6t%vz2mro>R!dO+?%I9Hj?O3ay+~0fwPU-3D<Ajt6nH><4tMg
z6VTYpvG-qx+q(NYsXl%+FC+vL#3#S#=$U<Rt-y2bU~bRL0cx$A45q7OzwCcu=(j>}
z)r-9{iPMhTrj}T3+A=A%&3U=<M6ulmB3|tFSnx$g|FBc<vvB>_oLY@jm-<XO7jdy)
zK`u4z*?&EYW^?UU>&^2Y?&FBb*~<UhAeHt0|Hkci8=pUK6IP1x%9Z8FJCvR;>wEc6
zx`Ja@v1#~^zj?xbQ@6jpyhNkh?io|r+79t2S59_>mtOSUQ}wcR_G&@PJBQiU%=+-+
z*_n)IF%O;`$$qw{E8MMzm+$@8t=uil4|lD*d|*Y#L5-eG9}d^JCU(u(A^NRDtS)ii
zR&iECja7wutKYT9XH3*$%@gB%)XAy5E$N+T`r;!BeTLiiyinp3GVVXbEaAT9_<80%
zW@0<d&1|=vl`i-4wLW*pX4`9(`O$W2xoYa~9qK+YOw6uY+~N2vyLU#B&T@|3x|299
z7VhgkBzF0vq3Y*C>EQo{vY#cSucR4uEx4lgO{4QnoB67Oh0}68k5o>RdFaf$`qa0z
zsarpAp8Uy~W^_gB@l(-lcIA#Cj<HvCSG+Yees-tqx#7kBdwr*V^L|+5wwSro=f$^@
zmu61qpWI12`7Zxz`el*u%*^{b=f8wK7cHIIq^r=8w^(C`)OOxX&O6)*&$CZmJ*SYx
z^v#U@^5O&Uzv(ppwd->F_TKF4b@kbo3pR0?*Gy=?G)>{*w5*w&wkpi;RX%L7Qo0~G
zw<}qp;^(&Zj73!mQw-|onp$#vzs93~Q)=bCmv&jYkKew$rnOP#{;PFt*=FndHy(5U
z)SSFUkJ;z}cjm%R-3xAiZqwQMS1f;F^#7gr9&7a%hi|V~((%ddTep$M3y*!7zZANh
z+9w?7%}C#LmXpcren^h#@sz2@uU8~ZG%D!vJ#fd$wJhEL$4Z4Y9c%9>XgyJRz46pO
z#p64_TKvE4ur>TeGtaBtn=}_LPmcR`!LageF-KCGK-h~NCs)r}Bl0eLb)vpwy4dxq
z=VI|6S-+l}vi;_s^!%%N-+CuDgmH-rCiZ4jFI{k_yZ-A=yKlTRriTAqs<gzZ>$ePR
z;Dk9p=CCW(6fItG^t&k6r!&XzhaEq^>|mXMipa0-fP_9doipd9l%LN&vtE!nom+ZS
zYVU^!Er-vk<$q8TeEHyvdd;RApO)}gSNBTz)l9tMAG50O|HTg~9XF%iHLNp9-mo<C
zv21#>oce1|+t(IuxA%5Tn;LgXyES|7z9O~z9G&Gqv}(Vtn(;%;G`}G0^|9Wf9koFv
zUQ=JGzx|lMd|z5h?}q5_>L>IRm|Ih3d%lZ#(Y@f${GMG8|6gO{Fk8YhNl`UHVPzAO
zI9I~u4G#~ruVD$86ESfSL$`!1m&Vm&3mEhy+=HHMII41TidOihB^r|+iA~gJ%@N$t
zxa{m4k^H_VL7P=CEMU4bMe$Z!>dbkmJ5-itOh|T{5a8}K_s5hgN5YnDE0SR?5uWMG
zew<6exa-WcFs7WXNztoAb;A~$FuiTRsCRF7mFBIl>I?Gs)_wiMtLO7$!^8haI%=;b
zzuq=AY^^-^Bu#CF%+#)_vyG14_+s>8$$a-Zz00kOGP_j$xOEjpzP`S(DgE-g+~3<y
z3B{BYPrP(0NtZD{S5oC_*5$CBC*5_`O{f1X4vScxJvHvGMd_RC%E{aNLK~uXNnBkf
zQXi20Y(a3tKc3FQwDaF3ZZo&OzxQ;?te9m7mhZnbRa#`K&;o{aD_9-`FihLHAltK0
zp&@%k;Xy|6lszU=dD@LC4dKeYMr{#sr5-_rQ#K}fTW@Q)?x*{9!zNDczh6F0?`gbx
z<9x5htsAEkOn+VGPu}<A@C<wBTgprH-KBnZ+io-Yxny$APNoFyyeUD)riM@2sKBXb
zD{*Q1|8I#`7+qs0%uEbeGjWO9tgKC%^0NwqN|ptboVppro<FT}*-WM<p3mkBlsPhq
z3rny^T2Hu|<gcw<H8r4fqMVE5lF3R-1eQ#mxJc7+w$rVZB|Mf#xU?5XEfHKP;aU<P
z$=Uh!@^qEl4f8xUY&+69L-DN2GjHR+TV9LVB&y{2$NV@Fv~Hv5)|`MvhF51sxd*D`
zdPz@OdT9lRrMPrNvaD!WMApg|V%CMdhL7Tzez*!IwQycrm-JCcEJasipQw0w)YOBX
zsTp6hZ|+}u_hwLwO5zHsfcc_#-W<p{X7%ClR@azGUc%E8K2B%dz0I&imRo^UJolif
zQFPiuc3EkS)o)@>{a<*lHDKSoV}Aly+^7}#Ty}rr_o%|PKdxV@dL5`@;^uO=CHS!3
z-SYT4-)_%VhJbFBk|6h$s#0!Ol2TLyU1E<J9`!8sd~wM$-1=IFA{XmRu_ZTx{0nqf
zg`U>P{_=u*g7d4nD)-%;es0%Y$MVi?s$A;M2SMJgOJ?UOmL6bTknHqt(y`@{dJQwD
zPw%vRR#vw0mPfyRoY@?0-JXjtD`xSUJ=M8sxy*aL@)^rF*{9~cnYr^-XU>ah&pRj1
zF8Oy;eL<Svy4!0eX!1uUAMI7uTr$N=zIN%k=>Im~uGMc5SR&jx>!NdLZi;|*l%lP$
zzv<TRpL30Nrmv|Fw$TvUxKTFy+qM6xTM7*_^&)5NV6Zb<+gs6L*88ZdBScB8tHD+0
z8gqEmHK&OaXJ(6hm=&HZ<$Und)bnYaj1q2_HcOswX={mSKFjGic~@U%PpjS&iN_zV
z8~i)Dq0MZHFi-r7!yLH>112P%<`>@DIe8{)<CH|cuPmFU_Ujl1&ynEXKktNomrEeS
zpGlGBTNcZHIO3-LJ-PGZtHna!?t0pPmb@kQ!&vCQLvNW%X6v=Sqy=kvpL%t?zSNY|
zdYtp1d$#B9du<nvDozVn?|LvWZjswWv26nDc-x#~B&Adu^a6W)5?7sl=eB5>$09YS
z`QDu$!#-OCa%D|fSabQ^q*7<YbwcKwK3{&7a(Vgxb5l1>Te9cr#~(MOKj^F%)yz8j
zrODGaani%Cqc&=y#?7lY`4k5E*F0Eu>e0&P3VxfVo0hF)VUpakBqhvaam34trKc9Z
z)$)n|73z0OdCf1*hKxUj_b(k=xn|wP2`cBcm?hUNTUA?>;e3tB>+H6p2Mn{)xMy}Q
zi#+5yshTmE>#vbUMuPIiFs7xgnw=N&jBZ(Q*mf66g?1hYI<$X+l#a8uj0A7;(o0>8
zsY^9y&Il-adTHVogLd)FEBV}w8a2NJr5su4BQjC*iYDLI__I8xgjZ&qubs3=B;tsq
zcxaN0k=WX~PLc7vcbD&5RykQIDC2qEJh81Ci!M5_d3k9X6kVHG@Sk_m3SoyeG0qy?
z6W6t!^S-ShyL8fKhgD()i#--8ICF8vu-UErJ?E*8mGr}{d^swLQ}Q@mS48~PD_-Q1
zGf&TY&((ryD(Y9C%yEdmc=jq^+4}0IZ*FWk6#i^oiT2E=*^SRVr=BRDmbWPPg~RNf
z4?k|3GUM!u7ezOBtn^(VGNEvZvSPDjn%v|SXKV6OcQ}Nkd0p>{ov=vMvnlpXnbPhC
zn+}J*Pqy!5T^io4?A&lpa(V1T6}4@jPX00Yq)@i($lQdrinhwTgwFlZZ}{ii88xp-
z?=wqK=)y%E3c*TZ8*IF+1Z=vxG!6wAc_g$5Kb7|@wy>U7QF3_YZuO4LQx_%ja?JlL
zUwh%Mx$2SV<{4(DcNKeNcMBQ$EDR0PS;nwy>Ro}R5Am}CH;Ql<$?3Yabw4@im%4)M
zSpKo}43@J2YuzTzU(c1x70>G_q3OhWZA+m_-5>w!Z%!!ubn4)#UJ}sV&8)=NyRbpW
zbA@A&iDP_LDpT}=CoNH@0=Zrun^I$?t7AIt`I0FI*78YtP8CR*Wk0>he7VYG4cXjN
zH%}BN>{^_$G&?!s59|49wY#1!SKR*mP~4>B=WI%yEH=i-$cN6{ew1UKOOv){nWM_w
zgvlP4W`*=D*{V8q(z9<p#l9zFG8^?%<*l9yXHVsL+a1^BxhBtP|E_f^8S<N7StriQ
zT=3+(sZZno7lCSzj%P8c9{k&KB5dBDvuxA8>+t&YcS+o0oAA<HV^^eCr}?S~9{X-3
z7AAp)D<NG6c{4QT3KsHk&P@`o{<3-f%IB`d!qV;gr!2j<&B8N5OS<>=`$dx7AERZx
z3#YxQx~{6CxpZZ2QN(5&Rqf@Ag7e){9ydIH%5wCTPsg&9+Z(=T$Z$kSyp_H-v1F!7
zvC2Xj>!3qcQJH5Xq;9KdZ+YAH*SIlkexZ}w#V$2Y^$puJTIXJNo~XC;nPX9_>ij^S
zm;QP@4_qF8?mTU9oOLIo!%~AVj*Tp8?#<^m)t%Q#viu!=Zlz}UY_}ap?%sHEr_Jn9
zhSOTF7a<$&INs}>IpwbUe_ei?L&g8z@BV7Cz(w}X(hbLIPu{xTZ_0dkQu_C&%jSPA
zUBCPB&G==zm0#+;n|d#MQpxw3H;nJNH|<{;w^)3g!AH(dy{i6~_xp#xI?P$SsCV-0
zGv7_!W;0BEuD*3zjb-9pv6bmgIZyccj8tW<6x=v$gjl$ysKzB4RPcU!C+t~ZF=3&3
z|MaKsJ(G4U{wLm`dB-tI^k}4}pIc))L)Eb<0v_6Tm@eyfbSCog)P@|c<vYAIqC0f~
zTTn<_W6Z$?b2c?h@lN3Vzb8t2ZrfSQ)-w-%s#7AX_b%Z5dZ(J@te)W6sh%^XPO>#J
zxK0&0IyHt(a?V|k63*Cz$3t$2dhU^ToAE!IBeZMI$Dl=14zWpGX$bJVd-ZO$k66sd
zz=bc}ZW;O=oERtCbaGzdK~bsscWxfM=zBrZiZ9JHYDsLE%&)nR9)@}O-1YLgyR`8V
z+qFdRH+%}K_*SfyRlId+0q?Ap6OZignBcX9{Z*u!_n9N55ASYzdy9YCxl?cFH88Ng
z{u@<!P*C$ni#hA*Di-c7Tiw^Mxft}a-TSGeuS<{B+>4VY9$~4t>eqSXa7pCFy=_}A
zOunwr`rqbs;E#*;TU*?I1piWu*u{J0qKUF^i)-c7j=qyOleaX9962(3Z~LQ|V@*7+
zfdPK|B|BD}yesRF;GyEDdFPPSHpSo^PPZG3uSyvA?Q}V8`d_T8Q-*zCX!pS@KD{pX
zNilMhB9Ar(A5%PYtl)rSnZm-E0-P7w+H}M@qmC8bb>k{hdj8;{$B{XOQ<!>Nr*7SQ
z?DNdt^$9n_BBxHio8YB!u$Hw|^GbAM&vEZtv6po_I6TF~9|%fb2u&-Qe^Vra@nz^%
znLuq5wr&=&lqG?ZD_pg2-0+y-s63I8UEslW2@lO97apI8y{mh)`AV$xofOHqL`fTt
zuw726XGG;ySpy5ip2YY)*r@t2H0-`%+@s!W*&%YCH<otf#$_7DJv%t5`=xSw<E1AP
z!=9I(lP<fayHjB1g$ovvQ(tKcuXcS_b}_C-Bi<undV0xpwImmHhdXay{+CPVYIxhJ
zcDgc&q43a}87y9NTrPObuGW<QEZXhUanjR7UhmN{_hm=??k<{UDxddb(e|m0j!%Vu
z#I*dhIO?*1>&FRKZw{6U)d253^CNj8`wm97aJHZL!6QE9(3B0W-h43(oh{l^lCL@>
zG|Mco(DJjI^?36HN2^_j%(gA$U$j{FjY7{#w%d2p<=?5-Uvp!BChe}PlJ1i7N;65x
z=uy98N_r1l&5Q;4vwR<(Z1>pH`P<{7dxc8dvjyCG_s(tguWMnM`uB(k3$NEJwfY>5
z$Uv4RkrRBg8oSs$lsny8X58HTaB}3l$7%N_d{($IOYw%#GM3aFCuy69T`P{(sXU$X
z;Q!>Zld=^%mP@%^6LSi4vppewVEM+HHtjam-~i678*?L~7TzwrbSc3@?Wo(#dx^(S
zcB*u#1k70}pCu>%Nimyek)qQr#V%Dhr$vpc<R*kY32Hr}a%$F_(r3c4kGx9MKAgC`
zu2=1~W~k`Y)=z(BYIaUH_jT0udv$MTOW4GR+I-#1Pbt(IN_$3Vd0kU-eKV!z#iQJk
z+v%3x^{%ILXRvR}aW$+9uHJClWN-AGqqk37aPbxJi%e<Lwn*@}B3qO6kTFeUU+ZIS
zk+4RU2LWFeWM8=}+wzk0r$XSB%Uqdfj_1zUdv<AVk4(a|rRi%~j_*CTQ%ogy&g-KR
zDlG~sJ+3J(dH;iy>e_A?s<<?@^}p1#(T!X8Lo-#&*CFghGh6G%IT>y5j)#7nllv<z
zJMQt6SZS|C8g&mA)?N~P$a3Oe$zzp?H*e&0rtW&8bycfm+Y!O7jDe?mKYd}7oX{R-
z)*7~~{jQ;(<P1j@*9Un54@8+{&ZMOtcM@ur5IeUhl7&lW>&$7dr@Veq(ZdrsXGc}G
z!!^E|J$!bjW^fDoygj2Ced^)z+F2hXgH(c~l+DCXGp@O~EU8pCOETlxz8<&tooT^6
z%Yv33$#`~eQJdP`NoqN6SxZix7um7Q^MKly(pR}+d46|}_#Rml@MKwdt&i?CrNp_f
z+V`<s3CYVnu;QP?oW1uZ*T4NQ_j8T>i%YKUG7nuh%-<<_yGeG+&j!{dG0Xq{Jj2$~
z!@{{*JI1qBrNio-LWJD3vvu+Yr?Q=2UH5qCKkrn)$3-V+?Q5I8EAxb*40r5;&2x^K
znWdg;jBje2pZ3LT^HJehLhrw5^=hf+%brs^G;RGMxwW!EvweK@{Cb}Fc{EN<bm_dL
z<f_rYpy4Q`>U}YD(H0(s3Ynumb-u^@V*LX+V*gI>GrGF>-;3F`T`Zmv2@@NgLmvhv
za7L896it(RRIq9V?-FgJwb{AaKUTTrtn18PdgT2N&-1!#rDfO0{nLy)c1?a$(BUPQ
zxF?IvoiwR+#^sx8XPM6D9^hNE#?*CM$&q&p{?EIkclGu&IaifJ?<X(bcnanOb!_xl
zwR9J|zt4&4r!73s=IE?)w@!?gn7&HHMbGVxTKyF@w>OWnF9b{f*}&;}m+^2`fK0N{
zJl;@+iwcs`VTuwae!-C&)-h~3T*1SxzQ$d83scQEoil1Dj!x3{V%2YcH~qu8rAZ(C
z=5%a${Pnp{=_Us$dB;zSvlO1>P59*NxPG%%(x#K_GSV6!g7Q^XUEFqcjc|DW!7TA9
zd)YtD%;zhM*>7~eZb_ij*2`=)SO4sq>=oo<^g86?oEO<LbN}yK(e%cjljR{pvdi=U
z4$*iIwy@ZrwV!_lb~)eZ4&2tvQhZ=q#Aa{Z6KAGPYx7;ZSn>av7YictHu9a_HPu6;
zRWoLxW=_cCpG%YeE&OmH_0qMas{Pqd7`MC%(B1Ja^~6+duHT+XrMgQrx6F!CNsBYd
z5?OUGwsTeLGEvpmX1j~G)4sf$Thiw7@m|=+`%$m%7a5xQMSoRX=Xauw+l%qrQxiXK
zh9_JNlCP)y`LXQjs|4u^U+wQM^}NbHVMkufi~nVK+<6Jh%QTjfDbb~NT$LNvO-O9{
zdy`4h=OKsSi+~-D+jcE!Gt9N!c41RqocTO~U=h|xevjaWrDu1&P;p;$xH0*c(VL^1
z`YroD$jvfHeC4_C?&GGoj=F;28%&RTR3C|}Kl&-P_T#({&qTB`Kb_a_&kQK}ucGg+
zyY=UBv4EQ$eHj`3EB<V2S^4RU(Q32A#QPVv9+A>B?N;M>q_e>{s!edyjo8M|TRDQ3
zgdDr=6Ob^qQDlKJ=LHQ5-;SQ7&C`8W&iFC2VT#23NQJPlg@M5wcXf+Oiw`u+Xo}XI
z7oBy<;V@_1u7d0(p0mF#50F`Q{gXjGQ_lH=$Bjyl+~V7IdS=4f&SwiIJyTmIDsp|T
z*A=71)82~g{-PWA(ND)$e)f_np;ITXWuH1TOY-xVmgM{XzfE*kO{`8{BF(O!%XooB
zG(G3aE3b<4C0{Og=<Ha>HDO(lOUpza&76Z`Dt;G(KFyL?uy3JXdW_<^d(r16J&$1L
z;MAHWx$A#Z_ow1$-;mvlzV7JiR5L$wnJukt$?HcymGkBlMcI4Txkpa1Ts%oJ<7H1k
z)sxS>s~_n;-sE;ZuG~;RZ|2*X;cqgkQ&Q($uQcV|790K8=!b~T=dAf!{{MeI-n5O$
zqd`cZ>fff83Hww2^SEwF)AkKI(`?g`vh=gd(#ZVh5&<5{1>&koXK%AjI25(o_($B+
z29d&0x4He|+gQV&wmx~2b!BdMQTg4U6(MO)r<uu`r<FJGEbG{~!TEe=^S)zQs#~-f
zU7y(8Ti<Np^f>(^hn!~PTArOh*Pdug^$O>@>t3-~Q+<P*TYAdrkLEl0&#e2ZE48?A
z1=HEN&OanxZ)yE+nK<+R$K9J-Say`ucy0^O=?>i9vYBB*P_v%tM3z+%-eD6g!gG$k
zOgh4})P0xX3@6VSCw4gX_%|x=Z1l-dX1E)@m^riN(3_a<#pxF+&N^iua}x=EQ*1s@
zkwN!_bb*fh()B#6zUQzc#hahmCR%2jWc2b+OOpJDTA2;$w<;pHDAZ*r%wB&)_+k0W
zIcx^rE7tXV^Lx6Uo69$2dvIpBWPp!@(eknd_w}#2=NY-L`>S@aW8XHf45h}3Bi+9e
z7HY0MxFgB0X!BBO7p8@JijiCe|3&ArUbw%lNp{Pj=`$kjgu60yJXY4Tow~ov@j%W&
zSB}-<QmqSWMD<v23M}X@YngiGm(uV5I-J*+&Rb&eJap%V<8C~wT`n%x&pTn`{U=N&
z@1g9eTNaFKws0Ta5&X@kqwV#?I^pNvx80ug#&o0CjXN1m8*|>=YTF>UPQv2Tdo_vG
zzeS&$v~D+RTW+>zf1A*Pk6UKidHz_G-V_m39QgfXN`T4?haEi2mLFN$)V+Mdq20NT
z1s`4&xY#_D$&T7v>N5AC<hS>!UuyrokDFS+=&+?wwV|7#nf-KuX5h1_tBu(1?hp)E
z)^nsYFJ|Y;h*yfPz4H8RI)w#G4o+h7Tc(m3@X%|rqTrz^nwL~h&C-g0W^y7??fgvJ
zbg@60mzOszQmmc!Ai(KiP^*8UpXH8Jwa{rS8>5a!1^j<;p*dw=o9PWz715b7dzZ=H
zGF4IE%6@m1s<qjRTU&y6_{=m&<&NL4Xzn4X5c<Jim7Sk^#)26iSyY8ZEaG$=J6Tw#
zu*Ucm6>MfWRr`69?G3L34$VuQyh?&s@g6l&J<+3*So|%gDWzpw<?h+#9zh1~oE&)t
z7H94+InkqZ`Bd26sJBtB`Pb7<wS|eXCgd=OHRK5`4V%t*-)qmo7Jk+O8ILyRigte4
zS@ky>Q)l*kH|2WKWcbj<SoGrIS2~Q|uNJlJ{Ng=f4J-ef1joO8HU;LtQmiypxT>p9
zJKZz*lX>{AUCYUX9K3H9HBamN<++9D^{y^g(T9sx^Y~qO{K#?FiT{tBR_JjRonGt5
z=frE;+`z+Ws?b^!cyb%d3zqGBzI6qy*A$=Kx#6JL>>fi-o7tV-&HH#X4H@~gC;R9u
zy|rq4zm}BNfzLY*_?lnhNlMkYRJYmK|Nfp$m(PWHC@$SG!OU_+0OJ9#bAERxDJKhC
z@n$U*d**f1sVj_4YXwiV#jTPPmu@{sGVMHdphPJ!_Tt&8A)8O1PzZnbLL=<60MmtD
zJ*!zB*JFy8PQ5-mDkxKBSqHD!(&ZholeUW*yBMgX^EX{{ow39>U|CkUjM&Xw)-xNl
z)3`TvS-Z4KnID)?V7+3s$?j8zYEe_)e!E%0UQ&K*M}iiI&X#3QwYF$7yyJeLx#Yh`
z@rFY@d@H?<%B}86)RW!p({$FvT7!{M&rIX-8MO&aCVSp(IdS!B)Z&aKMki*Rb<(rw
zn5=CSA>7cr<=E2i3m&%^n>>Fs#oEK1ae}j7!!w>Ly<K73W+d3$c~jin(>_Pf)aJX*
zsS~!nUORs6o9un?<dKaAQv{YxJ-RK-S5LzvP;o2wmG?*3cx%?W2wg5Sh?<>yX!flw
ztFBhB5V|k+;7EJaJg>WluRcm~t(lrK=csvg@E-R{LH<ic%IR90JohJ0IqA-~f@klk
zLwuLI4QC$uQ*`F)$&dv;tUN(Zjb~Gq*RJ2UthW1X>iN%+442+a$xK_{X0P$1xr^N@
z;Gu`5h}v4S|AkX{f^Plbz4&6g!r5sa>RVYeHE$gdSrcTj^u*NHUXfQ9b*OFH>RVCT
zdOv^JIbZFC|9$jM97vw5TJ}HbxA|m&N)1;H%?Avu6%OYf`f462@^YH7NoDPfqY6=+
zo;x-vO)k9J6Iu2#^v|DX7Gj+nia46?gx>kIR%~u+LZ9LcZh2#gYX?=l=k4@K)hgR~
z_Ju$)(~X4;O&6SJ+OX*>Xm{kvHA#8=%?X+7bknoKXtu_#p0JbA5Bv+#_D<+Y3^2Al
z_hniz_wzsRGIBknGz})Xh0kzQ6g}CIeJ|B)+Q-$=Z$Hid|L)n;X#y%uY)Rb}F_({B
zN>H)o?i9P@VX|iD)Duy%yZ$elWbnjeeT%LxM-6wgnB<N_u>tN`RZrBd%N<>gPHZ;z
zoa9);B79}X)8OTv=d~SQiq7%OV_vj)Qo@n|XT4TU-L$IMH)-N$R1WCPm?`;e%913<
zt#97x`g8}Evus$Aafl_z^7A8;*Ct)}9k>G4D6lS=rY`(kRx9vSvPh9!>4YdA-=MV$
z%|i3qj<j<u4OdvB-Vx?xzoMtNUc5!hbqB+84!I+|xf`6<c_p^}O-ac;{7RkSFUQ51
zA>58<Ufp&vJf+nANw>dxcbMGPs}6Z9U1y4#%<&A2IzE5N>kzg?P01r$mAT3UT1<LV
z^L=KlUzj7XxFlp@-<L}UCrx}3le=VR{MTdK-Fq@6Su(gGnEz8qbBT^g#LUY+Zr_ew
zvzVgJ?CWv*lIVpnsUuzUk3@>68?Bqqnp<@?a{0v>hdK3Xr>fdj`o80<oUp5RsY=d5
z{}!oRZvXe{x_>d|RXlycXv)VHd*RCD-^bJiZDw&4G(<}$9!)#Br*nncB$=*s+t(Lo
z&w8sqH8;SaS4e2>nr}<aJIAPhRux(JpnF^9#+6Zd+QMJv2&w%DQq+p~bUM>}&7$C9
z;PFLQ49>i1D{X)2W7ue6E$?N{vB9{bw6kKjDWiw3Y++<Wo!eRWd7%vFetUjdbbW6A
zBH?AXg4K)ftZbOY%c>q%c&*KeOZ4xn&E?NCC%ea7`7a-SF#Q0lr@XVTk_dyD>SjjQ
z3o%YhB02B7G>Ba5OiK(puu#PHFxRt_+q$+2KUx*W5VDXfs_u0Z|2r>7<-Y0BMOJH7
zUcE9;_{qD2Z@LrX3yYW-jTr&TpCw)^sTm7ptP;5DbmmNo8iUudyaNtlzB9HQ<h{-)
z;OZhHG<C0zx!yF+b524shKpaHzAVkGd1Qy5kyE=!e2_uxk5$nVZCY7Y6dewZdUnBz
zS)ea9!&%egfg8uZHl7xd*AJtTUKq`I#JfyF#`T!i_C0MHlj4p=FHLRA<b9hqvE;Kd
zyT*$s;b1>)O}~vxJtOos+6e3Hb?O!4H98%@S6*!Y^Gby7U+JU&S+AB(DZIO$w^+^C
zwe+N&jfVhlQ;vd^^v!j*xet`EFSu5*)UBY(qhe~wp1h>bX_x;5FG#Qz{%mzhAvb`}
zs{i}7vT2T-D{Q^Jn6ysp7CLe({Fzt%pEuh5G0Q_EUUrlpE$$BHd~)s!pT?REbKDqb
zJPi52#M4V6M0j)Tm&M<n^12$Yx_tM}w}Um&1sgTaEehxkQd_9^)hp6^LC^eKmrhl6
zM8~bu-u|KFnYEm%?MtEQQY$$V9Ljo_^d=iBN9&w)4s>8x*IlA^_*|>PXRiH+nU6RB
zP)RMVP1||-m(sj_s(085?=86UYwCiVhvf2G`HFA(h+Z~OJzuQSk+gfWoQtEt|5;2w
z@80!hG`8N6WM;42d{@zTRl-@p(|T_Y?0lA-9<z62=8sC{wtS|CT*`V|rk0%CW6Tn@
z;p~!6agqx%Zyqz+yrGpj?BRw<Ek#q-MW}8+m1XPm>gLIH{H{ABFDb3N-}L)l2H%SP
z)9$`+=b3y?V)hND0IT%0Ni98^_Z1H7L>y{6q?WV##I2YacHLdm|85C6KRf4wTGUZq
zF|#$>1EXi${I>1--gh2}mfIZs-}CTI?Aq~=Z?2MP#J%G7H+x>4V>0CZX|YHouTh!n
zkZ4|`^@EnMcM7XLj*6HZQdU|da_@otnP%Tt%l8@RJb9#{@=9Ho&m+od3G?1Xhf7vR
z@o8wzV`}=Jqp~&1Pc7)&<W-A~*mdbV({yy%z<Rl<CCjfl=UlPd%4b>pItgc<Rk3)r
zoVhOKuxe7S+LT<iCViC&dR)ElU-Ya`zLICc_h`eKGp0$86|xR<uhj5MTDGP`ySr?e
z`2miza~8|8Jb1&9%Czmkg*Z<8PkA2ye5}{#E3a^WXVkh;rYVc>oXr}w1yZN&KIl5i
z2-u%`@ZpDZaE!9UpI2W?nAnp&FHd1<6w+LDXI-DqnlGE)H2L{I)^Zn$%_*AEqTrJJ
zZHq^D&`MGB^fewDo+bH5EEvt}_<p3g`$VPxsF?U8&50|O=_Zo`=QpKWpB}S4Jeqo9
z!M96?{L-FPD6y*jW10Qm<Dlf~i6SQg()T?ISe0INLt$@M+uUm_d6zVBM<p_79^!kW
zAL{bd>c}Zki$x}Wk8RD4D)OD<`@k=+eyCdSQLn+Fzj}*46f9+uPg1C6eQx){S0#|y
zy)ZH?BTi%E7Avj!0ZzhSJr75%TN3h0eZl*XjDRg3hDx(Fb-t+TB&=gE&5B9b*tf%>
ztU-@+@`jVsw4D3SEA+ir6Li-0NuHqOYHGzAxJyf|;?e3|N@*Rd*BsKdIM#V)!?~qR
z%S4ZHtjk$=cA0~c)6=3j&e{k4wg(=Eo>Nvn(VWF~_M@D^LKy*p$TgcyT8qrO)wEn5
zsq`uS<edLP=$xL*`b&$PpB>73CglF#t!0y+_U5?c52qGzbG4^79P--bs_fZ+BxXa=
zlvYX4bMq98>mCVac&(gTc6y2qyP~1G*lyixPLqpEdYP9TxZbw1tRr(q*ZQAdj;%;*
z^8IwU?$O%vb?@K3N>SUi(R^Lu9~M>J6-+WG`S%`3s_t?Nsym*{aG}O1HB^D=;HiZ7
zO-ICHmde{Jr3EZYIriwU-67F?57;IKM!q_&YtH@Y$@0e*x{DRwb?sPG-*Z&uj|b~h
zW7e0#%C4UbKXSQwytYgBH1ZefPIy1{mHY7sCa(rRwf;V??p(+Ei$7$xs7Y@*nZ{(d
z%17CY?Q_Oc?$<p_|E-yGLhE|NGlgS3e-e~<Z(k7mf8#^>0%yZz4G-oW%aU53b;U#Z
z)tbz#4{ontxcD8ppJrn7^Rv2DztB?ETs1x6-jY1FVns2f2@hDzquSIRmQVY(D0N>d
z_o^=LHCvi?F_qYT)bLxgHRQ#Ux)AphA`Xq(Q}-}^&+4C=#NzvNGEYVBj5bY9^ToSf
zh5VQyz?J(+T+zfVmv#G|?o7E(m*pG%4)F5wvj!Ll%djTQ-IE>^buih6HBjqHZI<iy
zq^0xfSc^=qD%@Qgt5#^FeM&KOU9?l4;$v3ZwXP94P7@zY?tf%#7c3C|$alV10+$1O
ziCRLqva-=6gBK0w?B0K7ja;V_q2#OQrB{Dt+mWOxF{`$x=(+t*(sSsU6m@u!KA(x>
z>^1(T$Fz5Z`fq$ybZLch0&^<QAv2Z-+`>-o2OgP;d;IS5p5qeY^8B>sa(1n=dRA(x
z1`M3;N=w#7DCK$`;4xU#ub1Lz)$;V)@y{PV=_dMbHtC(jb7-?pipSeeN%5gBR(z8c
zmMBd%$rI5O{+yl?tafk3l+;zu`b95I@--MMG|FPkW=y)Es?K0IeTm`bGiim>mUW40
z3KbhUPl_yGGg)Few~YJo31ZrluIWZ>jCS~Z`V!OkG{LOElgIn?7@HIrnG_gj@OSj(
z_+NhXgMlHtUsrdA=eZiydkI>5mhc3nwV9L|tIP@1t@`{s>B8$%+LjG&f&a}F1G+x%
z-IdO5rqFk+SWRTrZjU5azZ1SH=a%WXM7VP6y`E%qe6#I$PG<ksx|FXr8C!qd;0rG-
zWine3u6WpJUe}BTStpwG8GGK8Bo-x3n_6tOLc7MK%-gi+o|35dm1hbUpS5gI-TF0a
zyI2f=Q}q5&1_p*o;e?!ecIyyjYqO-)dkU(uAE!Q2+VenE?a{S-w=X>9uQYETv{?A~
z4U2=pgx2~OJ=`J3k{Ku0_I|0kd8ur6>*72XC#$!0{Zr~R9%X(LPyJKgSF+^Gk||zV
z&Zg7X%t&j`Zr544$H}kt<#oNq(QhW}28mhN3trrC=!sd;7MBytC5}7GcRKJhc_z7t
z{SW*kvbRU%lTst|g4T`Se(14frx)FvpwqTEAUpm?MXK?Azm9qDbQk>NGyj@aU0nQU
zkLR5<CrhEP-BS+x9op`<=&;zNWo9og-<q>EH1bi3-uAG^XKk9o%HmTNYYEvoYn_wP
zI@opemeuy+l@+m$<;)eLMqc478~BoAzORfj=iHXy*Sn@aGCaAhXZoV9(!@=r%FcDV
z5uR(B+@GY|3+^=f+L2}!am?pa*XyrcUz4)8i?+|)?zlI!@_1R!L_hOYJyW6|^37er
zllFMh{dcdjHj1ptsMfEXHd|D)@$s}wwz+q>*Uf9@xxsH4(JOMwA>)vG%_^PqyAqsF
zmPlS^ogth$`~SZ+_HVkAq`#@VY={yI43%g3V&m)NaIKcTIOVtaf`Db`m@2rv^u7Fw
zeO}$@%T7<)@y&aRKBrQ{yyrXT?F{33(=u<V1NV{z^Oibv?o3(Pw*F^bZkoS%_{U(j
z$L8@nrm!mBQZETxRDAtwO@K(I`D=>{Q#+P0o3fjfrxfj4v@L^ohlN#K@%vdRE4SCJ
zYu#?f5hHOk?T32Yt=8RTO3TjHE_t7MW!lw4AI!ET?AxKK>v1@Dr*B?rn%W;0>9=M}
zBebiU9?$MLobSo__lTh7&Lbu>4x5~4;5l)wdPT9qnhNVMW!qIb)n{HP9=OGzFZ^u-
zYwmHiM@C}Bj#>R1b~xlr?f!pt$FC5BhSkb_)ukurygD(Z>qUe)r;0*#1Y^Ll1M^fB
zxRxkzZc*S`lwka2XMowmwO&1<SqoPk=H1ADsQ=@R3G$ImQbtcBc5cc4?pkKxoU`V3
z+->&O2@2aB%FHUWtt~67l^M1j2r^4HK4IcF?^$Nc&rFUFb66$j3QqP}J7wyo={%hh
zm2R0akHWf->73c4#9PYjs@z($^nl&9o;Oadyl)g|9m}t}^{)I*R?Y1VqKCH2Epumh
z#TanNeU^Ln?9bw!&SiPqTDkvDUp_yjuOu|pP}y*eHm9+&cyoSR47-{P+q}sMJZ~C#
zOdgs`MRVV16ur^d?cvP*%8@&wQPd=H?*By&!aF!zBN?rhB>bGv@7L^E7ID3U<Iu~D
z>H>WaWy4utqg>nzY??JDy2-R(H4CsWHpp6gbE;3p)VRzn-RIYG+zOX8w8mCdWLl{(
zIJ{(VU<#3#aU#X5Tf^jv*xcZ2CfS@NzYd*QYqDax%c(Sf-F<I6I-R8(oA1;;w>WpZ
zs%P`uWtWn-Ia}PhEw(F4#`5=+>|*X)3*-zouUdGxa>K;Uho=X<^9)tIv{+_U{lDjz
zOsDUc=~;hUBGob>+;w`%)Er0tKL;eYv<2F>b*_uHW!|{;fS8#g(*%X1tPQeRi6Zlo
zd7NafH>Mqa-6%3afosbG-h5k2F6J%Hi%K~q`#sZ-{tvNUI*ar8%%x?5k&)kzPRiwT
zOq1Rrx9DuSd27S>ZPUZAP7K%8J2t)d?8*>#CO_lpd7XEy+3txdPk(0XWu{oODSB6V
z@}Ca(`)jQ`FE_1qOWZ9zwdc<6nw<=mCmMKPHOlX7;O(;NdcAP{$>rAX_U>t2X1S`J
z*J+OEtpj3LHfo$oyClh3zh8306q5xUXH$Bl{rt?r@7MYW-06!h_Dq=Us&#mt?VWa^
z1mPUU>#rC7UX;KsHBY2!pO~5A(f6go#~x(VL^jGwEEL>jwCb@!PE5hteNAlQ-rnp6
z4AV3=)v5cfX(;)o9-?1YsAHcQsjD~hkwxRgrPqo)q?E!Zd3tz={&%lv{M2L=b7N`x
zk5jx(+bx;BStZq&e)b11_ddW<cIrs$4gWp!{5t9%NxnZ)zH9x;cBai=!&aT{UbXgD
z;KX9)Z3}psW{GZV|FV0R^yYTU0JQ}?vKNoPtC3@UD|WkjQN*c9O8L{6x`lo0yq6eQ
z#x7{k@?w-WdbMMvkFCm_HP6<+$ZlQHFKEqWwYw(T(Z9~_$W{f;eVIHH9JoCiwRUjm
zgvBublHAA{dGqg%e*T{~eRm&U7+Wtfo&Bc$)v{B1m+Gd>nwH0&%#{>W!Kl>Oz{0>H
zB9b7m(1|f<M~<LE;z9;hVO6ex1ViOxy<RFMcdocG%<NGS@7rQwq^ild=GKO!|H=$%
zPBXc~^<-2T)Sc!^dQSZKGPpsvb?vRKSw{<+&j)RLv-P19Th?kbyXc#nLfgDKSNP<%
z>LkAoTRp!l##Z#gsa|dKrCuBbw;$$;Z;$pqvp0H?&rH3l(-S_<Hq~a=Yq89^X~6oC
zMN?>r!j~NZjzX+n8qxbABo$rH2x@BI*<+yeL`B%p?plu3=A&2UYgUW!1}3s}EXk2g
z-P-zP*?fLsk4T-Hi<?-4)cpMnVw#$s_KN#^=p=kNZ!|}z$kF2ILZ`PJJN54<JlJ)(
zmG5->{G!+c(>}evu0Ll}Y|^huUtXS{?=NJ3$Vln({r&SN)?K~2JV28v#QZ@cD_6n;
zCpNVk{}mdA{4(ydc&P|Dv`Mi!Brxz=ZWG9q6l9<1z%^6tqT30<=|?;>0y6})g_K-_
zw3lpf>hiJ4brO=anUwU{XYrqw3jrCjsoa{;2h^u%JG+Lu<%UI;PIV3Y>B1JxHFt`T
zc%<XY)EU*R$EMFn$~$;aB(uoiAPd)ym&=Os_)I3SFsW{km@CHFGO@Gd*Uk%btN6Tx
z7H~;wCAx?mRC5qIn)z{pNRWls%DK~o+E*@>_1wt1JbJ=MP08FTMysTKZ>4o>Z=d4C
zHkoOMmtup#Ij_~_syk=N)vLVJn&h<S5?9WKEn&xBum9RxGPnEB+K@F(%Bm5unyEP(
zj}|c|tYBb}m{svV!0EPOm{5keZ`*|V9a>Xdm|j}_QmQrC`D2~a$0*PFzLF1*2zxsG
zw73+oRVFe-I!NQHTLkYeS0~A=ofE@K@_$CLaVGLTn|>r^g5t)wXht)`#2ATb&x*TE
z)nt}V^1PN&_{wcUi`eWHb68Fb{t!93Q)JaP0avT-%Zjt?oTtnz5ZD-fCsWh?;3gK2
z<9oK8Q4_vCkxAwMv>Re72|8LUdG%A;onn<k!XmSlRtc4Fiz=Dn!r!=je%0-*vh$oC
zYaU^|%TZgi{Z6s4nz>UcxA($s;eS=J>bu-D8rIDbk<PtyA?WymCVsbd^BV<~_9-}h
zyEdir$*k+?`sy(r3lD5{Oqucjz{@X&Y_mGJ45Jn~$z-)$*vfh5-VG<klRE^0JT7h#
z_#kvrqvrjr#~z}Z*2mff`geb{IQ4$^1g0wi*R}_7e_N}gIcb{ENA|RYv@ZX;8G&n8
zJ`*X}ck-Vrv+DHBWu7HbbCyn3Q*_Op<hHPNiF=cUVzF#i!$$1|bH%>yVp+7>Vsetg
zQmKyp>~<6CgDyBRnLN<cNbODJ@eZ9R^5~#i?F5Ys&S-6)6|384^g29ZjPZQ%>`?29
z4f&5=GS#^qnxFPoz0KxLVy%$KnyEc59U@VkFSLXkE(xY|vc3!xR$rtsdndoENYwOQ
z+b&F+B$6NYVb%pNv+0u!o~>55S8NV_^--VkzeK9menr=?SBpf~PDo6yzj!+SP{@qK
zDsHv=6JuRjoTL~REi;-Ld0otHA?vc-gBl)-A1qONFyqBlXYN1&2J47}T)I7n#DW+^
zc6t>nT{*(0yuL8Ry+KV%@M?dMXLi(?8DfoYr+lXM_Avc7S;*?!lPI$LNQ&--6I{Fx
zp0qIih%)Tw+}1PWl+tOJG{ftUx;D*8wY!y>Sas{OY-_`E>%3(C>MO<bzgav{yq*|+
zXWz>WA9|i!r7so`5dGxWeMC6#&%@i_7o^VZ`NQS-;mnDrA}s<Oo=1wfULN89`_d_>
zAd&0vO@TIx61@{U)~j?WC1|KN$=vo#5H|Uw_+EtRnAF`9|KpN#<~g+;SzOkc;$dbu
z>08d_m7k{ev%KVvoPEQb@yVA=i_$|PqANQlJpFUBPDr8E-l((X&)?)pfge=0k6pM@
z7I8^FO2O6E;AMYc#MZfDrh%@B%-4Rez0{b}`(VkrF8hCh1>8{*hrhTzyi)&@N%89h
z58FQv`Qtqn#tR#X_V%1KZ7~s6RPT8f^+s~_zP(;y8Jqi)9#1pp+B$h-&)chfg)>%*
z|0>wLd6vKUtIs!2I9=Pcw@cl@zk9;Toq;N^ydQ1d^7U5H)pY^=CwaQ^v~LSW3N~>4
ze38aFU2tYgh*C07ret$aciErU8m>3zF_>*+wC(FDJ>=-OQ)RJUsj1fP_5Y_F&?r*Y
z&}>^ap;xilw#KoGBjZv0e<@enV+%OL_Dom(Gkd+R-pRh$TS>x9fj2MwdTlpD^lYH7
z_6Hq)A8$<;$Do~Op6XrSz!kRP@pO@Os~MhkO^DNAIV|qxVPAETe@~wy_nfmGH+Zty
znI9gve(kAgeV93RD`&!N?n1+@hbC_;eDovFa^17!t251ucIj^FID6v$qnoTX(~h`)
zzL@Cdq|Ko5-+jyPkTtxl2>~q<U#u>9z0LG}y`V1YNZgEWMRlDJt1B!07#uY`XNy~y
zED(AXptWmm%8tj;LYJN|mYvJ#{jX=jga{TjvrQ8}shHSKWYAQL`pW5EJ$>(+-h+W3
zt*`uF`s%onHLLn%omCG4a(1@*K3Nv1%;mex{ULYQl_X&yu>;laf!s@S56NwCTcTED
z$n|{_)4D}@$0eLhu5nBgZ8|bfLG}0Xupg5c*R9EDT@~s(zmCJ<*1=t`W2dG}Z8f<p
zD)i8-_00yKg4+kR)~!%C-#B^x=ZMcXmCX;6HKQLLdp4QtxyuQU1$!o(ue)~l<GJT8
zKHO;u2iICTv&?mg`LgCi4nwC`&0C@5$$|?{M($~hn$EOLAtRCTdaO|6$z%H~E*Yd9
zac#OcJ>ZpA=wgZ1{ZrQ{EGgN^Ic@EmsjsrmED;iD4ajHA{5W@m?Fog}MH?2$DmzYj
z&Gg5M@kTPwk{_ztX8#kp<F+MVKY2Ap_r}GppM8vzJ3{(J=CGAT9W$D?Eo6h(xpm%g
zMzck~tk|&nuOh>a$D!{E*IVBeWGc_P&i<}qe%YPh54r2NIB`j`uQ6uVW(d~&H{<k|
zZB7eTPRslxF~?vV-_|^yd7^1?dB%G-_FY{(S6W%+^P=y6`i|bLl=>E}v`o~WySH?1
zTCG_-W4qupp03|LDkm1Od@Xvf>XargC3se<N~8Urj^L#&ktT7mC$?so3s@^JcU|2+
zu_vcRSxPtmiRN1k={6DDg&HcC)-YI25>QeOEq?Vn@P?{nk)CppgXrYQbl2e5hF!h~
zV!4IcE}a%ADq(&S+P8>v%Qn_G3IF-iw`Vh5@V-Aagd@;#+msOQHkLpEN3n{qtk#^|
zrJ-lSxG%INiajeATUf5-A*7Vi#I>-z%3c1NT1Auto7*GP7FIJ4LElzUGxHy&ZK@0!
zVG-K`QdAx$zipFyvn+41YSf2k5sQvG-wBM2OcacD$o~~m8~VcOSd-A=r+KG~3eGWE
zwluoj{uHI*ETq0!-QZBH;Kag)&8{jJ+vaZ;W-^G0T4I^nVw>R-aq=qH;stRLPxxmq
zNLZV|FSIR-|D%3#gPX9K=kylguNzpuUf}Xj4y`U#p726KsVVmHBjfcdtQLWuOaWzE
zQoK3Ly8Z}rMor-4T~?OU;Mw(}OjfM?pTPg5y5r^LO9g7on}iNFZC}x3!;(~S)YEW9
z?)fERvB5nTrZlUoMO;*2$eo~i;fdq?O#%)oayFj?G?&WV7O6VZoc!d8tjRXnTN-`V
zYVsLP$+v!(GdyZ3e=PqlEK;~B|Len)FOC*mL6%&XI2TUs-*+fA;+R6HSimmzwAJ5T
zxm6{z#2DTzid779Xg0EzNYyypXkE>!Q){MevawCakt<|lgIa09xy?C69U_W>ZvEQ?
zCYp(P9570G&>@-;GHZj!+Jfxa0<4Q4@V{vg*t#KZcUwco!|Z@|>GRW6mXx)vznm#P
zgGFq`l(em3ncK51ib}US@JodW?apWwo7g1BrlYsxe@S$LipDdQbp`C%AAMFnVmmQO
zH1`5qc>r6ppjnIoW6T7tOIKQ*pPAXE`MwBjdC4eWbu;hlHK%&X84a1O!7BvXBo)GE
zMs%HYV)iU3(vV-Cs=v@VinCmH?bQN}WTjf65W^JL&!3e(GQ~)n7um8D6-ESoTPC0y
z?q<}~o>e09f0I$b@uE+n9dAE$OkP$}{X;lvL5#zrj#&p{o=z`5FD$5b$*MO=a?KHw
zAO%joWo1Hc9T_3~A(`bBspWe*y2B#Nw``g#shE4mQt{Et^1lK}4dz_Z7t*;WS1b)M
zZ+2u-cVRp4$aHa`TJS=(wota8nr2+4z2}dz&K6j}e9G5&$$z;GpXBnI7g(if&ps||
zQf4-9d+(Ervi}PGz8%RoIFxTW#bVzgfrUp|gUq5lkIDaNv;4Cp<zIyC&h)6rZ3_RN
zq;5SP8PB36)@0SXMBv1CJJlqq^Wjs@nakW%VobjrC_goQ_SL`})0GQ7<TXr%JREIL
zKU4NlY}WAP;xLeYX3D1bl6%{dx~&J=Jqwh#D%5{Aw#`*#*yxfpc~k6?!V*!{hSe&p
zvlYT0uN2VmP<&#k7-^|pb82plSNP>;{LLDyMiEX)k!&3o*{Tc~Ki*^$5}3cO(K+`3
zqtF7eoS!nuGqnO{$Y~~OSDaMg-m1c_Dp+N?%GIOAQ)1P+P17fyY%#d>->*ZHp>3wz
z#|)v69nL)~SF<eAS<%{ZKs+NMWhVQqK*wJb>MzYKo)&%TRO~AwwzVJH_C(E!JW)7-
zv;DkiQK_VG=?md?o+4*mxym0+)!fRuL_lz%*^<7C@edw({++7#pS7X?q16nD1X)4j
zpaWdx?()|%!^IA+7i#S0-q@tOV)=v0dH+{U-SVls;W(d=gU{(3rc+PJ#|CRH6^P9>
z5REoq<YJzln!vv80b6HcMBW48_bOI%%6%7<_uXLZd%w}_?#BgXp7M_^Quq@6Ow#;b
z&gh%Ps%-RYVf3SgJAc^*E;C=VT|x3ys@|&9@G}4ZGomImTYkQ1k!rTMs5z~rMbRbm
z|CDy8LPt>#p_kG(Rs?x^DSAvyf61Pq-?Xh|s`3?Qp&Ku3T#i`lm@XA+oaB+Hv?o;l
zbC;f^V1vDuhVx;Em7K~Ax7N3D>ujx9F8Q$OQW!t$?e!5q%l-v$w>&p(OXZf5ZoVTn
zZGDqOW|3@aAe&GC+bSVht_y5_O1g1I3U*4#Q5~{Nsut*f)7pA_dZp*8+Q{86mhE=4
znDJUH|FxQSK-ePLQwu*w)Y@_DoRXek8zi($v}O;J0^dRwVFSr^M`sGF>YRNgA;!6-
z?wa$(U27FsXa0D!r}5ZYd&k6}qcLVo3GZ%g{kTf<tD;`*Y^9bbf*VcPZdh)JIysjy
zdg@cn11dBAcXKaH)c%pUQ=_{geO|qK_g)A7)e2l&E_4eEZD<fr`nrKJ>2`CLWAM`K
zo}~#|nMrO5hkJxKuuANn|7zz-rGuO)4GNd5^PJNb1ScHkS2WV|K76M|;={(u+0%U&
zn;$lIJ7WLFpuYHsQOb0W)6NDaGkyy$QvaR$uf&2qh^=k2fS2}`-HTE=o~G_QK4a10
zfR{zO%QmlxwvN8#?YgZvUGAu}*RFP@_O)uPwtp`5WH5^OyDxogu3GVXyPL9L!72v<
z_Y)#Y8vIK*C$2tG;ia(cM_K)@9gSJ#dKKY{*UCymPxh~w`fmf*>IG$+gPO`;@t2n-
zI~TH_KQgUjVMU68MBah_tkMe8O)~PiT@R%iOk4NBO}|EVx1fS^#cuXnnv1KZzcAaq
zde`oWM_29C-tC#uvR<p;&WrpL)vH(STz&P-(JfB1{GQ3LZPiW939O42n!hvf9NRu2
zr&*Vx*0LNrxa8<8hlg2uS7SUDdgZOs_^frJ*iG=clE<lAQe7=ODt~p~wak6yd4d1U
z39gMB7<E_3m(6>o*}cZ#Ak&5oTMqCkvMpQlAu&URtAoQrO0B!molWJM@<mgLi$~@C
zABu@3_SBa)2QnT0z;bxMw%~&qvagt{&VSSrey}OixXQHljPo>EmP42CMEH5nm~n27
zoaU(``@00Ttx7$<d(nhsi;IV}e}q}EfBl~l6dmCAy8jwa4foPa#|^qxQVu0%wNFiz
zB@1io-7>1w^y{w$N_yIuJMH*3Ehva9!{H#?8nZg43ASgxu8YcU{Vj81?lRVG3_HI*
zY*IN<K3#`F(2akS#i>cYC-cKk?mfV_WkJ^00RBY^Y{GAv{I_!T>$1IZxG8k!fcl9Y
zWu`Z0vWT4vQD|IfprFFAJZHsL-MU#`(<U&xG(DC(aC^FU)8RRjWcv<A-&(la=aKBG
zv#Yjg@7Z_yifWncCN05$XIJ9RMxKe5|6F)Z$Z6*FK-pKpYdoJg&A++#{_kr~ZO)3%
zy!Tbft!T~Kw||72X1HaSh)kPgd?0AQ*vI2ijr+Y07)2%g@4h2BugA8V+xFDz4Sei5
z^Gdh!{WrR`wP0S5CwIR#!<G$vYc}w4={-!a;+p;?eaiyQ)d`8uZgTGtIk-sSA-@e*
zlEBSFLh}^ecDAr@`g+mSe4RjtctkSuQg)`p`fIBW{}C|VtD<z+-8<>9Wv}e{z{3{3
z3r_AebKL8sKil_#WAgRJBbuw7s+srbiTEGoUi7xYUq)-ok>3k7v;-rkrDWc_^5@GH
zy{v%U&n<7%nrFBjYgI}=A8yrOXnkbi_7Xo4<*&6L)(RAy*8h2kO<7bZ^w#mYE`fsX
zibZY;?J@R&6WkcAyQkQm>bJSsr``Qmfs4t3f6+buQdfS_e>ddc@qd}{zi#mcF44K!
zi}&*NpFO#*u5_^i*CjPBn~4YK>K;`2SEjR&QzYi#0-Kv#H^{|z3MB7*bAOw_8H+rX
z->p}4%r_{>?Op8@WWc)LaQEx$nnqXd7_GizQ!RHwH)ZwnyjzT|pCq=(a=rL_Oq+v!
z%aKd(&c0rK-Ro^agq6e9`}gmxS@3diOo;rW+rm$>1y!%Q9(PrFtI+(>Cad+&;?LeU
zPF~;MFlEiPtLMFh8I2B=^Ve+Pm7T{O{YoV6<YIw%X})a5J^|r7`~tf19FC6!e7SeM
z%VIKk^uzS?)(x*CJ-O%gPUd#xPyX;(rE!DG!3+BqaC*o@Y)R5OcA|%8v4xuA(M8)1
zv(Nf}*>vsYb&HB4_?4{II@PYc9NsUGxZdvU-`=vDn{GZ4sCQ=dI)6sw)vBOB^Uw9P
zzRUYwv^n2hHZ^c=waBi1Utjrp`z_3z%EcOYg9A*29xgtoZavYkc8-vaWamEjD4`?9
zHo4u(l7+p+I!k{#*`8Fc`}Zj*T44W4DV=(s8!lHvR4s1yN6(#UJ5_QYm(V$(#k?0+
z8@$rByY;5w_1}ics|zM4Z{WJ5##SJjc=_05&Uvpb6}i4%C==CvlOuj|i@~41-BVQ^
zPcCj)pp-K4VVc$%i)rTXa#Jrh-wEU5d?b5flU$wT?i+lKQT=ig{=en?wd-Tzf=`T3
z_wK%8_~lL`r$8*re;yrW2Ze);oT7RuH#R6PZ|76>TcXj>?PA1h%(o|E;*~>9^6XAB
znhzdN;S^O@^EhPGC}Ons#)>l*#Sf0PSQOrxazZ%WX(Fe)?~$8n3Ytb9uD85yoXT9S
z%c-If7#XT~wb4^3D|UrLL-+<J6_rkhX;bXCF#Y=a`TF{6D*|6f^a?3YpWCqT>w{~r
z+`XqXRIPEjn6%rEXK&P^u$`fI!VQzEwy^w2U0QZmqU!I9wNhKNrluU&a^tIzL)8NN
zt`(BA;!Z?8ytQp#L~h2?3%<_g>)7sXd%Ws<{{4gX+c#ZZb=G|evw7i`8#UdBo-;4c
zanwxcYM8>cJb1aC<EbgHFE5Su=lm~cQ~5^Yto9uRg<oG=-?E<)zwl3qeQJ!J-SejT
zJFj+Gb*)fl6ct*b&>|qSLYUD@u)(p_L#SbLvx1Mup-D=61RVIRuW0OQ6k6fo!ph{K
z&?w~3Qn=t&#F7=Roe6I?hz3O{E}gotEyz!!^~4R9<+pAGtP%~Ik+sTVh07X`rj{9N
zPMvIBkmKQ^dR%Jtrj{G4j?3>gS-D&;$|P~AJmbGr%f$>&Uh$q%cTJGhJIrKdtK1Hj
z1;rvp&u5)BwvOD%y}_RGa?hEzSDq&V_}2yruZ>z_;KgvCWruRmgLeW67ST1zewTf^
zjW?Msw^Pzo4%&VrgIjOe4vo;)L9vEjuS2FLb^niCJ2^?|l|++&`0FCsd__iXpAbbY
zA+E57q}kFh6c|NPPTWjfJaw94V)#{!<(zAGtx)J#km{kbj)kM#W%{HE4Ex+!4@_Ft
zRuOdKZp(wz6U!EeT-tcNqBtl^L76pUsqFF-4^DZmbPDk})w1VSmICXPfEy3jOF3C6
zwx3na*0_4>RqEP>FFv-*8HPFLidw#wbe<BW7R=-0>z0^4b!v5FtEltMiEHK?D=$8-
zcy-y=O4DsD7aFGc6u&;3;g`Fy*&^B|b)#6r+@x0CuvN=uc}(7PFOb2=LeOlXXl82B
z^_3zQvUb1Q7rox<L(r={#SdmTO5c4~+r+;+p8Zok_y1qN^%wq6(foDmaJPoe9_97D
zu?7o+1l}y+>5yuF@$snet3}G`tSX$JT69V@G%iT~k6cv6yg`d$Lx7E8XSj9D^o1p<
zIV-lEic)d)kp6h7cEO7i7cQ3Hk5oymFmvPVTKOnTu}@gl-F`>H)LHx&PpKS>=<wF_
ze70K7H%u~Ps^Pn;$c^UZ%i5=wW+bYnZrxP!-;|Ft)}!vH?5g{Rb+=zpTILsf@=APt
z<tkBc<x7PbDF?o;T`ziSv#V|mqeGC6y86Y3PVWO!CqBQE70$D;OK64)qh>*(@VZq~
z8S+Y3w>D_lTq`_Z5q87r*9Z5)bEnVqhz3nyHtlY+-on);wP<IeY`3E1Pmlj@3PC$E
zwz_l*-EhgR+cZJ_ly_9nl<>VJ<@)PXm+<O(c?Gdt;8S+hO_0gq2veEFvRRX-H#;O^
z3R~IKl>z!pvtPJI`F-AdI3-_p$x2r18`C#WS-32{Y0tesDo*>I&6cEph+e`VwQ_>H
z)u!5wFON$++2XJ=GB<1Xm&x;=g>)NVT`d0fgKHGWOON<1$raL$r~2J)%vtyIg=ML9
zzevz0$Bh9?RX5){BDUhQZ1hpC8BbU8>8Tsc{aldV`s$=#?T2G>M;|`7JGH5!E@g(m
zf1lIsB3WYUr!MShsW@G5rfNddnU}t@b|)OB1}}dq7Q)6IdBTxr=5wbtC!E_CbSQdd
zEoSifKV^cjrh`^rSD@3i(tw&@tKGI6-?J@m7nf)BWH=hFdQrq_4(q+*q@>IscXlPl
zl_4jacP<fKtKqb?KPrd2BZ}9WW2)qD(<gPzGc^;|9J<sJ^rS9Y>9WR_g2mFR8^riT
z4lv7WoaDQCSjDU>!DZ(+<%qP4?m9LLIW@)R@%0_eIi-`ynY~q8l#gWwXXAODKn|ts
zzq$)qcgaNNa!e|CX0p85`--ssCzn<R!Ke~hafiRNrWWNMwBVkYw2PtTdS{Zi>OyG_
zv3#e9b#J{H{{EUCd~1VCr{tNQ-#WL~Y}?StmzTh;aHm1!*UJfAy{b$>D~>983VXI)
zxuPhY&8_@YEBybqFUo#zCO-+B7!~sUV3<<A;!%w`S`6WK>=lu7mPTwh=+r#p#ih`b
zC$o8yTb$c;t6OuKN|$Sgsa3jLU6u4GJF)v<n$zZ{^5E4BzowlP*=HIp{8MT(SJC~6
zcgkmrUgF={a#TV7&{gM;cP<1r6h+Tp>bKx|*R(exLA!4<UU0Q}*x6fje8O`Nft?3#
zANi0G5~DbAp;1E0#NZskDf_B8mZ;U7J95N$>GME;?(&r@bNEw(TOXKs-*A57Qerbf
zetZ8ibGf34p_h^s=N}7LGox@4Z>q*mR>?XCwNDONfy$3dC03od+oU39bSUmu&@Qzr
zdfdw=gsy!tdG)2xhcoV0&Am49zi8_o=Gisw?bpP(x|?p?5^H+u6`slZwB!1(Ip<G(
z6*_pTFC@Tcjfi&B4aK_G2YR1QOj5Vndcv=C(_4*?`+JlY^>R5>3B61>Y>^bD`on9>
zVn5|@DKYm=9t%Y`xEjq2a8|n^uu3=kVQF6FU!xgeM@%_8pKNHq<8fc>Nn>xywh6`E
z%TBp$y1CJpcf#|am<NjKJbUl7e0_0d_fs8ruE%}vb{zj3$iHr)>hoJn+voqj^LoZ_
z>jNwgZ%kV`Vak&0S~vH2R6qE=D)j60S@y~oS|3z?D3N+nAUUg}>4fDZnKC~mv938*
zvQGl+|4&V1Hk3VXe$L=(uPYbhylbyDghVD^?r{5Gyz*2bqeRk4InBlYN=~>r>x!xR
zCU6T+=+dtgPv0NZSruepImLf*c5MF1iBYR`{dcx(?yr5BtP<Voy?D;Yi(6SulM8bE
zn46z+T;pElx@NlmSugFkOrOsLd2AQaNz3Bl(tfLBXFkVVgJBhe@QX4ziw1|5^k*)|
z=LjW=E!cQ&;g%wfjrY7NLJiglt(TEf)8z{CU(y=k@!UI`d*}7{n=B^GESV_cws4Ka
zQifM+a=vU?WEXM7$AeAe)t%`v7PVcHN!wPsC~&R$(4L=p`gJtB?)8S7H#B~~@ZA2P
z(|EN5&yQ;dZy)Y|_kFKK>(^<qQe7L4-8i!Dejs!3)>q~w3b*b4d$rHJeMoP4eb}T4
z<{Za!RX47g==nBpJHrk8cfZST{=Hu-y>IK{{I&)C-+n0`{keO;;~b{d9IS$i+FmrB
zcsS=-fdgZqL+LG<wq!m|#^r38%UcyJTNBot?cUe&Vn^!&p+=z<duLDN*uA2)LGbXU
zMXg%%+D~;Z{2}=GlHjeKb88h1pC&2<O<=mb(Uw8d^wXif_Yt<e&YIm58FVgLEkDh3
zVuSF-2SQO2tO2W>q8<nzP>?*?u+6w~vI4jB+@L8=6IXsu<aoo$<EtPtO~`p!u+`QF
zlbl!Twpj>oc)j7_O%~ZmzQ?C#8GbOD|5|I8u(oP~^>*VUDh0bYdUV7k8M`kL-S&UC
z^;>~Cx3kuXxNr!T$W1cd^K%nd-a@rwyBD6V-Xj>f$oIygl8K94XNw<Q+!4B_L*}L6
z6mHw43VW9obd_Z^-}=46<ziQ%;p{1kB;rzL#QT`v>25L%=#5dHQrf}&e&eQNAK1$T
zn9CL%I<bIz>k7MmLrI&5$FBccYn`=Z)+DonDJs&zW}Q1s-w896&DiLEN_TCv)!JtF
z%8>4=242P)!X8{aWeKa=es$zVZ+Y;uqab@-%LEo(gYDP6HP;%6$1SpNF){a;vu+g=
ze{@twn$Urk1^j&_`Ufq={s~w}O`4FfQLJ%;{-hlplXE!o7tToy<Y1e)oVVqq-<bnG
zB^*LN5?B8J<Px$GT<Os?qd>6DVCSj`arW$ktrZ7vNy@dqI;=fu#UsJJ-Aa95HCuI0
zoSYrPv2e45=7pBpHES0Zth9IJ-SJ?O{VMkNFWr47@Ei>2t_$cn*415n#*2Y-R@4cm
z3lHq%4$HYe($r37y*R<@{EB0`EB84tZqmLiY?H#uxoO^p%SH`%HtB9y6rO0k$NGqq
z&k>g=syAM*TDNQJq@vYnTw<O!4$m3)xEHJMVwuqHEz)7INaxkG9bNmJjm7H^3N3D!
ze&MvZgr!%Q&)NGet~HVziv{-fyqK<E#ig6j*?Ll^ZHMKx-CTu@LMnH5+)0)VIyLFe
zCb@kt+V^<$1P1IfyrF#VzeCUI2==lgJ-S;CoqW+VH`2G$pto*<|F1K;@hjL3C0bKH
zoBot+J@vwGS3u8c37)%NleY@=WH@Wpt&zGPxs*?u^_Mf>+KuyGXn6P<oM`-DBs|HS
zPu5&_M?jvphwcvh%GaB|1T|^CSlS!HX>-}*d&PRjC1P!#4|HbuDhgR}sHn4TnR9mG
z#i}BK^Oie3o6ontI4R7+AW(I%^TXM;ii5WTycj*iIjT2E^c-C~)3I|^&tng+=m(BI
zt60N2_!Ree3Pw8VKH#2xO1|FU%$rDO#s^MYmq_k^(6e&^YhcmJMh-KVNt=wfY~0B*
z!OTQu)+0m449PjO+~2?H(P`nH(<yBI|L=^2FWAd0ByS|J2cF>mZYdlVVfpyvrVW={
z{8d>4t{Ta6dhlj1%rDmKoU*Rv4wLSUZ5&?qx(h@FOOF^Iwr&WP$$scE|7QQQ7aT?p
zntBfC|Jmg!mD-W~)`izGjFDMPTFS=fk4>w@WuXwIX*ZgDuiChEn{aL3bB5XW#3T;w
zqnDlph)2Cy<onvB;U-JKZJj2;=|)+8tt<SR3r#yeDyV93>l#eia=`Rh2lqjX9=n;6
zvp>$Lv6<{N%V@R2aqC_4qohOXTirY7x_7;`-Ll~ZdlIwn3guakOv~1A$94DU&OB!C
z$@6uM-8`?J&;TnZr{2fX^ZcIf(rpNsHTR66LMzw*ud?~t9-c3Zerucmc^uHfaxqu?
zR^X56J2xz8kul%6&_m$UPSLsiA-($bpHE~?>UgFl5I6T?P9sN25eH+VL!x4sY?LR<
zOqoJ2ht?I#6=XZQqkU_&E(zY*!TQ;gbBD{4n4spYF3~9lrI|V{hbHPj7L>lTL8t1H
zykpnm+F3O<UHMCWuif_I>|lBk-BGnca_gEqi4QN&S+#X<iPeSDLneQO3~Ia8F8k?q
z3LhwlEZgC3(#U;q#!NHTrEdgTHJy9wj%>=#mOOD~Q{amYfxKPzORUdNS+_-d<-Dsb
z-nW>Z%Cc5}Xs>rT^RV*B8Jqrn%(rjfx|ra~F?WWq($QT3Az_*SMUS>}h-dK%X3yQT
z_|3J4K00DG(=~X*HFnM2tLn$R*yK;I$#2thCxbW)ldcyOUeA!+THCa-W2c{HMte$e
zho`Lf{G0o9CZ6YzGm81W*J6(U?>!Om3EbVT-9f(+sz0z+Z)mMjNPN0c=dnTGMejs2
z1*@ww>?X-B-6_BvD4{Bq#yT;!J8p*C7aQT+nBL1f&OiQq{=@BUW!h1?#!<gsFP)cr
zM9<h<Xzzahxsgjng@dXdF5P|p+!vjXn^b=k1bU_P_0<I4Yt(-+LviAs18v;uK@DOo
zJ3Fd^wA(-5%Xlilx$|*HK{S8o<II<y6QT{SeH2(!GqFjEL$}vJYUaVaf}PU;dE^*Y
z@hBdgc*|klvCw(F7V&F>owPff!d7U`nRI3A3ZwPbOHBU6T&z88BJzm!))ik?t*N&|
ze6;!)X6dY&6L~t9+a&UCwFqne?knGGc+bD!Icjl6<+S15OTzwd&1^%S`MxmQ5%BEM
z+!M|++ch*+a);iVw(-#I)b79n>zJu}%XEEz-nw=1V`#c=M^A@^ELZ#wpRo1?7uyd%
z-cxlgQFD6n%|%@SM~}R^cf5$>+f=Q6r6&D1IKnjimUeKi_^DK1*%cVEUE?6v3!UIS
z!p9n4Ea>!`(G_Q#Y%;yZmfL!g%0nlWnAuDEmOkL#_n_<QuI{f1yvHJ@vn57LB>FhH
ziZ08X^*@fo>s)5*k6yN0O(`j6ccs#z40xtgUOwm%;m~wA%lLF1i~Tz1q!rg#B6kI}
zr0mxayT$)EVBO)wcGtUg8Lb)_^QvaA6JKqdammAApR~cVj_w6ae+4{?{0`(=MF%-9
zUw3o$cCVAi7JIgJcwUT@k!L#S^6)X=OLd_)JFYNlPV2~aUKh`2=yklTW2wx%HjB{Z
zvpA1zooJS^=%24-bi>o$1>q;A#w>UfGgC-qnRo6R$L<TNOVzdShHaY|9%k9YrSNFM
z(`7qb|1C|q`)+#O%U<gP8b%kISnZ~4UvRn9Vak~YcQSXz$js~xd~o-LUX(!S{?0w`
z-0!rkdU*Z9+#7!`GR6OI36KB7w@c`GW#2`GqHAe_0tdUb=QLc~`|qAh=GoW1?|<ZR
zc4a76%h~QP75}9=Y32%po`BBMm)F#jf)+i9p3JfFWRj+#66>=qTun12n3i$eZkE^+
zyw>-_-j)Uavu;UjWhjgZcH@8cASfwQr2eIV+(RM1#M4rHk3~MzEn=4V__k$3;e5tE
z;oIQ~ZSwZHyw11wO;p=9@8^=bmhM-<ajF0AKE2u^cs5JD|I_lg`PFCE-j}v}^IOF?
zLvDS7kl@9g83&`&m6E%EnC}Zt{!%CQ-Zok$ieq!DC&!xuZ3;P^3yLelIA>l-UY#Mg
z=1<oI?<Ofm0ae`;$7j(2GgtJ#uzB)7>dUc72i^NFNeiyIyY7ig+x0oCK5&`O_#xeG
z$Z^H$O>l77+iQM@?V56@7C&83#C3aFO5MR+$wV&e2#&k+IG*@)Snud<mEenQ*=OIY
z>G)K@aF4q^({7z+?ugJ&Y1@|EUn_OX)P1W1k3EymY{oO^Bjn2@T0cBfh@TkvW~w&N
z-HYFn#D0}=l$H51saQ-E;5gPNm%a6@`qcY}Z(Xa+n_d~e#q2~_quSg>7F~L)c>Nx4
zS?-Z?yG2O-V%OQEz59%Q+&<j3A;Yxo;!?f)C7K-!>s5-T?)xz<EmJ`3;f!}bmT!0|
z!d5wF-j^`_pI2r0bRXD03{o-PqueU{>6>KcjO7s(|IbKnd(-p5`IGlryQZbBCtkRp
z`C7TqohMLe>7|C__6JqtYHk_&bJS+@3oPvKIii++h(Gg_$5+4N&IK18PydpBw5<8X
z#iXZIrN27lMP)bhA7m_^SHH-z|MO>IMoocLTrB6Kr&@SPG`?~1D{;F%SGQT~KI7CQ
zcXK2@zG-k@yhcZ2=kcBVL7Jv--vynM4`mBhviy3pE%nL46=&|(+}Stf-_#8e>%a1K
ze$D9o>iM&GO262%bB2z#PXqW%XF2hFxGk)d)_VSfvc?ff<;5MkFZXty=W4T97Gsz%
z&Gfw}?7h(enWyd@h4~Ivb^m%6=O63NX8SVj*~ED#vda%jmHe2hbo2kkvpQO97};%o
zHZS+O9M5U}qS=?jFo{)@Q~brnQz}<>#Ju{{l__z@d+ik=Q;|&029F62Ub^-Bcy4`8
zJs)Np{r1+DrW<Wjj&z9K<v5{nVA|ZCZ6<YJIE0ejCP|sk^I3WE$;mz~(_L9Vi=KIh
zm&M#H+L}6<q5f}G)}NI@8fT~ft@&nkOJ6f+>E9_vXCv0f?~VGB5u4gRb=tMnU$R2i
zUe{i#-}^6Pp6RJA_h(EIy5n~<$YklAol=$=Pl}4X;uuT+rd^pmX|mg)GS-rdGb|oI
zn$jew9p<?sMDj4xm!hheNQS4A`;DgwIdm1Dx}fFD{bGsW)~7R9@vr772y$d%^<?q-
z|9~?(dG_71>ditIyc|CC9PZ^-_v<>l;-XY#p3>psDGh-sE7bE-=T7<dSSouf(|xb5
zx6?Tj_WRCvIP~i))200(3J#78zc`uhoR6>myCUjN<kmxT8`id%$7pYp`985pLsw(6
zx2ElnK=*PEhlidbWhM(5_={FtY!+K4GKpdP5|&4;0=tTiH%9jao{!Pvdpfzt{i^Ze
zUayl9QyBhUxO_T%&4o=~6Sk~ro*Me~iwA!w(}5F9^4%Og7u%o9oZ{lem2*%$zWC+U
zr<46psa{W?KJn)CxK7ouHioRao6BbJ>y?a{oqVh^f@%9*HC9Q%x7Va4rT+e09$WwI
z7~29SiL4B_l>a*F2j+L4Yl$t@;?#1;Uu3vz`Q*18?h1`U5tWmsdfiZCT<Liu?A6Lo
zQ>S05WI8CddQJH<sZ|VKGqPUIQ|a8Xpd+72`i-YyuEv`U<~I|$C8I^AT=YM0^?Qo{
zrAI$Pw%Vm$@s0>9yt+2*&4x=;LvKyeJ`j<<kUL~o=1T4>(_JiIzj0jaBIxMmwMXSB
zTVdvwgWmt@UasF?@$l$TF}C<)u5;t<uFZ;#GYVUFM15oC_3UFwK9?)sa~8~F*>pDi
ze$DU3U(;tFUE1==^WNDdo7^;{J-D=cmzmruxz4)kc7gHsjVB8q=015@AzgfCUe$-W
zCG%>(K0LnV(vM7)`+09A&$c#M{LigO*eGe65q@3r_qVStW=ky=Z7iQq%dHu`At|{x
z=+N8qx0x<9r|7<k2<4vO#C7u2-N#xtmxSnZt_z&(a?m>bk+Yy)vTamE=IZF68GBU4
zqs2m>A1eF1wK6t^BW$CWsCk&?jpg=lBvLl0vK1be6!a}|tGG9_A?Du3MK8STmYtdH
z)+SUHR5<<85+m&cD_x#?PO@5cvGDZ7LmClVQ?B$~5Ehu*vQqLU+w0toWi_|oXnMvi
zt$Y^0vSFplnN%UqbsMG#Zq(X)-n8Y@d6E4-3Wc`&+f(iz6wA|amaa3My#A&?gW<+b
z-CGM<xBX<&@}JnXZQCbZ?!2ekiHUQ(jw$JsrT*vEO*6diZ8Kxi)*nsk3#+`{9G-5}
z^l6BXH@cblp-58t)lxmK$(Iwjm}h%U?Q5U?IB_P2@iM;+nkJJgJ=Iy3+RTc1qq1V7
zgz$Wq*+HrYOAac<xy5{we!e#){X+GlMJt-;dP#BwGAB)7IG`-2dE%W{AamHyH$v8n
zFBW8reiHCBDzTqB<AigS&jb0uB>OE-Tbxv!o-pN4R0_8Ua!tK)h;7>kQ|_Pq6RP)c
zDn)N(^?kyd@1Qm5{)aYmRn~K!E2^(+r3We3SA0|rUNC1GOT*56Cr*0zOFrUZd@Om@
z^u0<{j;gOrm`L(Bf#hkKGx9T*?|qW8#60H9+;>``XPrOh1^(B+c-KentMz@C^W`(T
z?q%Ex<o4q{J6U_q<Fyed3>iJn&D|_G_2(nYeJ2X}!<Pg#@OG|rn0~`W_48?!e<#=i
zQg$xlO)X${XX;whf8e}m>H{ZEjgv;^wpYZWFEHKPrYg2^PfP3@rBfaYOV$aBv9@$M
zXL^O4a+Hnic6+&UvD*{@=cyb$Y7D{${QM_fp7ZKx+g#zE8TUk%d;MK$lqEDbaoVnB
z%e34KdT;uQgePCU(9-4=%re`-M94jCYrC1y1g7$RPv&<2GPYoxb<=yzV(FJQSt1(B
ztgdbkWnXosyB2;~K6lS8L(UcNryW#^b~8BX$-s4hH`hVBUGAjkPs3{)`@j4@&357e
z+i|W1ELmC1>^nY0@cB5@`**GBI-$VI?8KSe+AW?VwtevxIkv*K=9PE;D6&?}bD8)3
z7Sq#jPA)gzT=K1dDHAL9)pAydPf5*FZ6oC}p+g$wiPf!hjsITd3j6bNO-IwSvR>iS
zxn<?$wOU-CTVC#YIAL<&Dv^e-Hd|Y7?P*{3e$M-tr03x`7UrvQ1<d#DUu@sBEoH$8
zuN>`r({-$;H1d3S%)}n`F@Tlr!$H=x1+q?uS%pMq@ce$%FiR?`O>=_6LWQ&ksb(9l
zE?OX_#;(>XcI#o#w0SCp>q^&h`f`{_@Ag)<Xgj;}*Te3ro`YdOW_60>S+&l~Qt3Y$
zdUD2phHb1ezb;L;_L1P%y&U6}+9G`NYDbZKyNmAnb;7YbHl<GZ5pyx8%_#ibiKDBk
zq*gZ_^Zl(kb<)mk@hx_lvkrNCo>6lO@11keDqZBP=LyLL_jFG5vQ)h<+W%$M{&nt5
zDuvDRe<LCs54AXTwxmc}?e_3UOW~GKY-G9h;RNHlmC6z`H8__Av~5aFnIyA8N8Iee
zO+jC?Ej)V6w>(1HI74_EO&A4RtL{2zKG=Ej6x;2<KZZ+hKia0TcdfT>qo>;Szn3Bo
zn_f<J5PxmjXua}5*{a#Q_T<R#%3QT_rqtUnN7Ns^ycuL7`D|g@p|e%(<`ok@`7YI)
zdar2SWs7-A%|D_v{-=ng@jv|8>o4oL^+U}wFGk@Hd=0!&0WE1A%Di)!*p;dx&rLaK
zo#G(PIWOk*62Am)frH&EzlXL3ZCL+UKJC7JZcx{`ut^KeBQLX5a7u4d^x$4xQ08_?
z^zoDv-(DDdo@Qw_T03ptxfjYc9|LB^c55tHexm7ZxYvXjkp)}}5}F)m@G%=d-KlW-
zl&he~4C&@vzn;pK3HN$71O$h!Sr^?^sn0tn)adSYP6xB!J1>i!nr&!P|6O%sPh^3J
ztL$9{sq|^Lj$|+$Sz%_Pu3Du$!R>$DyP12qHZ-W-OT3s?byCY<lG)BHsqZgjCd{t6
zn(g)C4XXj8kidjVEOQ!~@(YX1|NAVM#90>NZlUn%zhxG)P{3s+;oXeSm|suS+SDp`
zV#BTLbthd-tWNf3-JZO7&y}egaxZ<Y{k-PZtz_;`p(k0YR!_`IpY6TlNJN2s_;fuM
z^N3jwKRqvB{O<6Y`QZ;eSzdS+SR8jrIkK;F@*YvwJ(<EEH4*~v+~&9RFHGqw4&bk@
z^s@fb$>ea`n`OajmeX->!`Jo5xaW8{FFc!?$ih91@ttb-;u10T9Z^YvK?{CxyM2gr
z|1w8*g5#-)2U-gmU7jq^Ju^r9f<hbbgjqWdihYS;yxA{3fwAr4`3)^+k7nMH$Z^^{
zA%OkpjYy88K?l11T_#-qc_S#~X!wlNfg28MU2y)NJyjq_Gd3sFh2`gAjT@)eZn<z`
z#gPcfo1Pjs<11wS&)!eBiCpn8z~1%djl^RDYbB4S+?>|S_rF$JrSDwIhR#nqZL<TX
zx)nU?oZJ3@(?KAE<-~<92NlP;bN%*gTvVZW{LBW%*j}b$ky~Y~TnjT<Tu+=S+TgTE
z)I<Bi><wFvb2v#QIL5!)$=K=><k}dP{x{)shJ?7rwe1rfGHxD{eKGIt-%e={hvQov
z&GkYuwlvBX#7N(9lDu(imX7zyLdMxU9IdA@b*8k~%<A>B5X#}WeM8}r#s=O?iQStM
zytlsb=(FJL<6sCo<DuJeNbAH^4TC2U8=BNEIQ>7ExOuA+$Gn#RzP`8fp2RCQ@fM1P
z&ar9R5vo*lR^Iho%dJ$&TW4A9t_Hr%=sz#g{?6e+*~@b*yOO*rCW~EM;&&^hA~3-u
z^~nB8mgOZ}jRJR$Wt=~_G5DKH@0Az9s~fs%a?VdEJYRDn(0b+hWQi#ZCtZV{-r2B4
zN+fEAY{5a65Dp<fMwcaXm`cOkw@7hLa7>RCi{%oK4C#>;nc-n_;(2MRw1?yEo+(=w
zc>8o74bGU*v*lr*Mk_<cA&CpkJr|lKz8nu)(=4&ZS$_j_+Ahz?6(U>qOvw(FmArO1
zSFv}?*89GKNp3ug@)d3#kVy)Nb(=Avlkex8e3N4#mOR@+mBe2xE4F>4yvpT&=*na2
zpXTPPFfgb*pF6?v(N@n*Z`+E~j_YZ0P2F?#@!dtjvG?ZBWoc??*PSqNgT&dY9nl8A
z=EkpW|D|#u`Nd*}4gQQb53&?AST0(eJ~zgMakh&m!@Ivx&5ZHQYdhm6ov>D$;aSqs
z!6M|f>PRA|`@^5fy<0AR|H*8-=8%@p3#|?&ZI?9r3r#LoM>Tp5Iqmbc*GWm)Hc@R&
zQ^bOdpS5y73+Fp#9Go1`_xopSLSo<32N{8yH*-(j&##o;(RlVhW2P6!%BCzn+f){z
zB`rmdI3r`jnfR6l#9cN?y?*+M%KMbd=8|68b)NgrE?Ky+rBQ%`{aL4+)veltlUY{8
zX#a8gUomgF*9Y-wTU?xD*)}XZyU#`VcxzPZRG9|1<dcC5m%d5uc;jDwjpctrR9ECe
zj|cvXwFDNO%iNT4MdC*DZXa*^4NV#j%QkC1(b8#U=xFTlTOm>LDsa!ju#^|UR}O1s
zIPJJ_NFzsULg%5rmmNRfvRyqY5&WZh!J5lCJzlfC4o2LZr2BS?!oSlSH>FJyTy)T4
zW`JX)$E>*wE0%0o7rRBwHSe1A49n{i4zbNvxHuyx^j4+PuAQAlvtB!Vk)M99HOg0A
z($(F~rF}u=<wYDGI$bS`8q>8UdqoNwwH?xHJL8yg7*5Z)H%sS!TZ(deWKdEe<BOI#
zpCfL0PdL#iCZxzZ>Ho6CTXJhzuJ1U1td7&oLd+n`F>=O2`!&rP3pA`_R%rZjaS3a0
zyVk5>(CJ^2VSB~d<<g<RmKPd4oh^s7tTk4|KIkY})iU+PX}4z=ukmKPedyMktMZqz
zO|P-<aNlk33DVoy*3RzG_GM^v(|Gwtg+V|-u0f{X;^WD^nqD{S0-2Yw_(cf^$hDb9
zDp)Gssubi%_%}E7*zvlvEbB7PXY#V!Z4AEeHiyZF(ff@5@{OHNh9RCST)Mt`UE3BE
z-n3k7Pte6kZe5)xRy~Z?Uk+(?G)~G<yVsy$-H@Scb5vtO)`S+VS7jLmBF`dSQgqHn
zY-mW^U2tFgm$U8x!P>K}@>zHPUrIXT_f+Iy%iWJkJ}VS^4q2>?jgXYgxVc5h?Tgmb
zl#reUCOHWw=FVcdxb2*4Vc+Y5b+_-<p4`dQAmVUO>#z2F+mlg|rxxrzvO8&>!OKNe
zl9QG<JZEp}y?EnZW5N?3mNzUjlzlHK+iJb?=9oQOWyP_5o=orEHa>K5adBj0O6k4e
z-0_XsIc$YXm}{F~t8>;NjXe)p(=+ZjX!Q3mp62s&+oE`J-SpTP`ED`RK8BeO(q!`!
z&lTi)UJ9B$r=yd{lc!&dwJ>$|tc;5;wX;={ay`BDHq@+jV_<mGkvos2KVX~VI~le;
zwZgY&sTVrD6t6CvZpGfnz;9{zD$tHa=C0U(--}b8A3XQXqfvWDla257lA|j{Ox`_y
zcg&%&g;~C>Bjk?U;Uh5&H+))L+FUlU^|aZ4i3s}A{HiODD|y+SfM?mU%QSqJJX_>r
zzu~admyWP07eW-~%DlRfcPEcyq0zUw^Bn`+MH;&Do{I1LlbdyD{)b)5Hnl}vFPs~%
zvWTDE?Ph3X|FqkQCHV%Do6}#k&O9oW7;;tM;@tc>tKa5atT@=lyy?Bmf-|KIvl#ZQ
zej<6Qz~IywkMpN?2x@-0^8cKGq{rFG0li12GTQc?O_bg6w}jX4z^#~?6FvgnogLFM
z1g|jJze>Er=s8F3Yoz@5iyW&L>FM-3sx4W;(9y(U?cK_>_J8=279R7B7x<pMOS(H<
z;w@XwlS^rx8#q4ZT}ab&emo)X+{}o#Lg`;-m*ig8dMTpLAf)2Z^Ug<)OLdBuFteGL
z|NJAqr)CE*Z`~ux<(u<zaz&<O{5>~`RP8O|_x2shw0ru4XWjZ|*Q{GlSuJPT_uy2W
zr=#Q_u4ASm+(sW3ONIEDglzIq{LQdM?thH%@~?g}E_)gjUu<d)2*`+&Uoz>y6^-ZX
zydGrzINRzy_hI(j_@wgrn+><!IpJFM;_czP+#mQ>U)txnQAjE7$7wy;NBYsPkBA6w
zEqmhA!)cWI-bZ4i|Fk7LL-n&aM<$xIZJ#xJ^_*;>o`b)BwQ<YGMIKG6NSpq@*h6lw
zU(4S*m$=5|T0u){moC4^amsP(S?h0^b*toBZW`YT_-<Kr;L}D=JEaeKPmC2Cc{eV7
z>U-yqT8`txmWA=j_Y)JgbRSa8-pjcq=5&oUU$VZ7+`%Gmt^*f8xn6qqX-1B$fUw~P
zo*j!@&UqIwigKLJ+5g2`!NIL0cT$+A?|n}0jXnEWOQ#qu(B&&F;^;aR^0AG3s~3-Z
z(B+ng7qgk<j<fQ$I|rO+^V)j&u6Ggh2fK*}6jwiJvOMMTRmOSkg93>VZ5GATJDpZ9
zonyK_NP)R~k=>L`Q5VltmXq5~rk`u&6l|Sp?H$CEl9Zg6Y&~06s%3h;W%7Rs=Q+Y5
z#~1xC$!UynIrmM3b!Cc5>f!mRzmIMEdo8Q1Ps!-f&d@HQX5SJ~SL4arzBBZ9y((*7
z9@y|lvZP?M8UL*eg{d(Ug};iFH!7SEYw&4ay!=HjqwPGori%)UI~n5>T(nnoetfv}
z^-ovnjEE%3H-=X0>@_RB{uGxpUsAryq-`VcFyq%nO|#l~?_LgTotM1YCQmdn81pk5
zMKT;kJRDazNLyWQxwuW<@9GPW_XSfJ9~D{Z9$B)j@>k`0X>XDFE<T5!hzWRz|2($W
z_i{)6^+5Xv=B0*rwuw&L>mzwwX_4g9z-@=r?sOhxG?-KLwI`c}+w=E<_reL9NfRzV
zNEKMPLUiZqpZ@<_b+1mG+VuM4ydAqavus7)Z*ojLbZzSOSBVC(4t@fC7SWqecFfrP
z$5EqYW;DNchw++O-)FJE&Sic5=7hz8)h;Q`_q-o?O*DM_`=-&2KUcEk*2n9-znOb(
z%e`gK6h!|!>VH2L|4oZ$cF$d*?aHcrnzPT`%`R3F(N|~Sc)f6c$s*;hC3nu09uxf=
zHLtjZ<sZ{xX>+N&!DmIT7;h?xN;-SKU_q+3h0w~Gx7w#3<kp+d@Nr(o-XqcRv(yA9
zZO}D%Je^}1;{^4OM;`X<aa+yO{jci#-rn0Mdi7@nZ9IK7S<QO0@$Sv)jOVgj{!Q!W
z^{zg6`d{chuiuR(UcbY7m$3-!kN+PS`u8%|=kz}sLdm&TR=Z5hvdNjSdy>|2Ut`T(
z?@KHSFEX|*|0Hn!c9yV3`9*W{^P<Y_CcPj1{|QX}aFX@gG0VL_W_<H$jectNd(W{2
znqgfEZ!f#d(Q4RlVIF<`*3tI6QA{nXd86;#5IT_RBxxROwxzIn!fDw?VT&z71%k<L
zjBKW5Uw(X4c4u7YP$r@ov{*L0Ea&E}O-JYL@BW(-_r~<|vWB|7zg~X1DZIk3{$I^S
zm8>h*0~UEr=KGrZe&yP#t+%!+9qkgC=^MSxYhuyI*Ws#6OFSgBzb$G}$-G`B6CKpR
zvZI)_C)CLCR7-cT!KbIYr>{HCE5ROrr)lD8cRiW^dTMqx28Jn&b8LdxJlT>Zdtwce
z_PvN~c-Fnhnf;qYr1C4hIVQ#XUhEP+c4L}1lShJd#}waXmEZqm6s|jJ7VgO@vGn7Q
zLdF^0n=fWe<<)$^l(ltLoa~l{ZTog!o~o_b$igb5vg2TddxOgt6_qWKJ5xW%>Phd|
zqIfv5d+t+_?GtV;_&Lq!l!EjA?mR>P#nu9URyZ)`DX!ezt#HO;QTGMo4|`Z*D+3lq
zM=2_EO?k%EwM^)*q2N*Bzbn&^iY-rCE*2Ic**eAd*o72s%`F{H400zwd30@SV(~ul
zcANIg6CBLDwb!p_(eP<FzU;;!{lnj^dQKgdc&%Z0CE)ejjT%Sa{QtE1sn=nj%_iQ#
zlg^q|IzLdlUEuA&WwcCqL2q}&&I}92uEQ_<Yz$8cU+Ccwy|S}=j`fs_j;2vt%#Ai(
zGWPdee0a^YDu&$F;2<H7OS(NG0SwAJSrkH*KL(sVq4!oK;Lw_IQ4ghS8z;>u4dG;p
z^<=O<F{6Q}(_@B{vH82lO)N%YC5uJP)kF%N?kzg{LM6*qC^Kr(mfV6RJt4oYO|{NR
z?2HM|$#f0xsS<K<6`mf{rCHJ0bV_dJimX;q&*L||x@GGBobq_f62~3Vd+*eVD?3Eq
zX*}E#)w}vZw@`xcwLJ{GZ}=XZcc;SO*tR=LMtUxG9Zz;<EPktW+JO1(>0?pn{%52b
zd%o`2Jky*xbL({zp|eJ_ttWlHY+x)mYu!toO|Q;(_uX#k3A}N`_d-wCW-je+i%sjM
z7Tb3JxN<<~pWdDg2c@moeQ>djZe$ShP<R{oS!CtG_A|Q=FAU4usSz;Y%WebPeyKmR
z^}hUAW^lOs+q>I`3Pj%*6dsj-Qm`~l^pqA?m`{qK>zUXX!($TZDMydhmG(Gw%O7Mi
zTHK%?B)hI@=d*PWgHqFP=<fOOYqj#7Qua)*Nr!npKRwXk?sMj;lJ?^YJ%`VdD$hF4
zb!>Stl_kONw%JU*^ESe&PXl&*yCS^V?yJw13vUg7#Z}vWOlz%i^_~1>bC}lIE)g@G
ztmR^i7P<eIu6mSVYoWZl>vVLI>46x{hoP)G3y%10HQTzDw@JX~(2*3;c}sjVqB??w
z*ydl??kRX}^Ej%L<76s#;mRj1+J&r}XP7QveAvz$yx}f`szcW@9aXKb6S($mSTwaZ
z)6qTgXJGV|Nz-n|KG9vFby`sHo7MdpSxG|8ix_Sz9q4D6py%_6<6Y79xepJf8cK3T
zp0H}#)Yrh)Ti}`OzGS+r`PntXros$44n@+|Y6tyfuPk=&Yg2!;LDBMrXH>C~5!Y%S
z^?>KA{Iv6)3U25<n{z)f;*Flus%@+fCQW-I6?4c~j@5!+?~YSI^N(c;JqjK93Rn0w
zxt|Lj+gs$^XPl^`n)JVW>XbvgsT=q|d2C?i^GRS9x^S9B$Ma0Eq;Xetf~)R2BQB!{
z9wpXALT<|<&ngLWD*N&H99*Q)koWGQ^}a-*I6bDN2RYVO@O<Q+dg`XN-k(%&74Eov
z8Rhdv8w1m_FR5Gn>=b-;RI|&$p&>Uzb?&2r_x`y*=FQ&}zW;d&ORcA{!*8`$8Z%sZ
z7mJkgxjt9YJij5SZu+9WWgoWL2Z>B<+Yt0}g@MsVj(sA_xC%7gHgp{E>N~!S<I55j
zA!VieKT4FZ3$5aE4^rq`QxxOrx{z~blv9D>YTl(10+M2j{fef!PS$ai5O&=<EmNpR
z&q?Ibp`cRNZ8KJN$unw4-VF(E{9n3!ws3TJj9}Qb%(Nw5RWd%uSq{w5-+IQoD#bS;
z;Be=RyPH*F3Vcu6CCxl~N^r_OmRqsEZ~AvLeDXhav8!-ao8)tqT>mh^+yMQcxy4(Z
zxHjKPVeoG~d2QXPuJqubnB&VB&fHn6yfkHU9_w}UgDV4+G<u#sZj<(&x?`b8uE2y(
ztE7CA_=H1`t$C7_{94fWQfcYUD|ux*e}}za7n-o-Qd;V5my?^M1o!5wohwpwL|pTs
zY59vc-pT)*mL2=>tXzUoF4TRojrKv;svl3DC)o=<=s$3B{(L6~AwFk~pOIez<sPn>
z$#P&{-yhCu3Bi@~4Vv{`6L%~U{>Sm>gPDDQl*|8i)wlD@&b;;wIU2f6s<F^L?Tzn*
z7a>e6nkRSY9ZJ4b)e_!znuF7|;?Od=ovO!6T|-U>a0%(K*%>X-$w|!?YuouXA!6UA
z!zVs%oup&9S4vWUc9cx=<(kP|au<u-ncR-3O-!9peeraS;jyDT9W*zzeoRe%6ltVa
zv$45Rw%1*0_xZ~O>{A?=8(0>&#c`yo<S%=v9A>a+j@NY0MPA7(N=z1c_<1(w-YYFh
zu+o&{lgWw{i@H!5P&x0|L7}3JeRHH`HpLh<^e9#eE@OI>^GqULH22`LPcge*?t0tx
z>&D90&&?;>zOwBr?r!1ivD$dWdjGBBdHWQFi|rrBzp&hG-unL?hX4x$Ll1)@qld@$
z<)11hHvf95%yo8h`&aQt9PZ0i?tVT|pwMb=vr*}i>c@<QI;Jb5ZX^f)zJ1`xJFR&)
zJcSZ1T;H4fsyc*ZhAOXL)xB!jhAh5iF0Fo{uTIS0btue4vQlP7|CE!J9|QN=oLO${
zJ2QFRvq=tr_GJ04pT4_U>zZPx-kEvXu^Cy9`zD-Pd)~O4<<O;U$*|1woLkz-`&l;q
zmOHkuz%aSxPQ|gBd%-COi*rtG`g^1QMxfsm*7A!gYwd$lW^F#5809)qIm)lQC3rdO
z*QmC=QHu;`H}<`GQvXgY<|t>l?NQ%9+Y45D8!7bj@GvejU{$zl$*#6*W`Dr{(&vXN
z%}O`F&whV)Rfe46=V^7TnNFXMUG!v$rUj?hKdZY__FdY&@5J<td8f8r*tO2Tkoku4
zUjBKF_DOjvQ=*etmQMcnB{u)vg!Mla_7=Eby1uXGW&T3Zv-_mfUD;k%n`nREE%o1G
zYwOmOo5~I5RvuwX|Gf%|S<W@VNqUOZx+Tj#>^M6~OL9fmeBKQ*XS|w1m@b_1TD0uQ
z;p=P~jLmv~#RC2wd-HJD>Gv6Lj$T^Nlzu$tn*WyvOn-A;b<OB|IVbPj8y%+)Im!Qg
z&TmucDOz8#DfdXzdgFu(ui1LpzMf<?Z93rNe6cBw`RfU#4NUi^wQZjAc=L;MKLt)@
ztzgZLO60!se?7am5@W+t$8)SZr>y3ka(ueV`JE=5pVw$VuJgSi_JQ-^s-?Tq&Kwiq
zJ!%m0?oq-j&Qd$4x$j=@%M<W9*1~$0#n;Q(xQd^*o_~*;$K8ylRm+&)nY9L8VPE_s
z<+qHNvV+#|OX|vt9tSuu3soiu{o(y%)4b$Mdi{(Gt8bkDysu!FmS$PPyAZu~FZw#{
z{_*RdSe(6uE3|@7MWA5onbTdXy1B1ri?{}QX0Ym|Y%JY&`t&~LuxU$POk&w%bHQW-
z$L>=Tj%|7U{F?t?wY0NWcv!UgrRV1D^Dr<tlbO6lW#fjm=6-2LTl@@7n2e8QWJIa4
zGje52YPigl{QE$r+|9nM3;zq>S)Jxk?6sIA5a5^1?PMrYoh)UU$;ABrlbPo(zMy`G
z^NivKUoL%+ET3r2HfgKTvoAYNnHLE1<oA6Q3i5olx?;sT-&ZFq?40*?oWH=<bHV6w
zTAkY&AqGaf)=6?_ep)F7igi{0(GW6zb<u%anagmYTI%9&6Mt(ITD~~`n<YUjq$l{q
z10M6BehZ_JzH=rzLA;tplF4fnnE7umxL~r)`7Z;9#LYKK;>S4W8lG9SY0@RF<V!u`
zS2rbX<4}3&!>}Vzxq$t<kTUC2gYRx_!VJNlKMdGRlU@sSr*^2zFgstI;iqK0tpC>W
znN!*(d}Enla%jPyiB{i^i<%vLzT&l^@&BfqD|)mZ2kF`~X**Az-?EM|^fPDh<~3KG
z^wbvc{%@U?_t|q<%G@_EUv;$eJx{vbwfy_H4?Z!MO!xa9Q}eLg-VkiHNKV-BvdqQW
zbIT9iQLi@P36Kl4UVqv0gX<ByM?(Dt77vzdvM$j2#;)~rz0UJRB4=M|trfBpTqfZg
z%#gc|`=PMe`^z`#3K=i1(k&N!{^v5o-Vc)7A15|l)Hyq8-I1b%Qy=)HuXL^0_=3-*
z(~Y&^(VFWIxb-WW4cU|p*S$?%)n{(_$bjEC^^_r7=Vkr+&x=ajj2x6+`-Lc{T{07B
zxVUO_GuOpUch*_lI&kS3--JW#_Y8`+xID75T577iMEm7`CHsd-3f`YRH&3#=`Q9RX
zjc4gZ#Xw_|#T{Q3aoAjH@;<B<ek0B1gosVbE9+yAl}(e*-MR3s?6cnI(_Tp;`#YW+
z=B>S*72v9Q=6lp8w(5R^YC&bDkHIgN+$|8f_bnxB)_IPT&#hl7EnM=9_32wRMbk8n
zrFBMEZ9cxUxjeaH(mRXWOH3*sdr4keJ14`_y<chJib=<U#1bb@a`M$qFBJ=K58Z3T
zobBUucro+qLz5o8f7%op5ueA@wKZaX9`mAZUq<CuHHTS@7wJb9RQQ+}Ogrvh?q+!L
zq6K@l;d2g^hBP%voe*UeBh$4LRHiPTAfWu|_~jWVJ&%_%Ulxk{?h*I=|J5&QCzmRC
z#vg6VJ#%I9JntFH6xu@iR(UmWPjELW;Zzi2@Va%y%v8<qztXomohmnvsI(dz{ov;R
z8qi<of1tzHQaHi<+wp)qzBLA+HD1&FU*yTHT`#1)Ch!CAp=W0g$%RJ$&AsfQQ}5Lk
zeCt7&l-?YpcMm7=eSLL#;uJyCd_(WY#tj;p+YI=Wb~iVOv4z}yWV};byf?#k^*8P7
z>%;k%9oyvXl>6#W(8R56qPZ8p?d%DC8uE^haSh9<1uV-NLJHm~xqZ-kyNShrLjKq6
zo(=DED-XNa9^SC)Y6_pO@`*s@tS^0br{b6&XWUy?$E0av+wI16)W54FcDg~P+tut-
z|J!m*COD+_SO;xY4xO60xJ>+Yski%dU$3h#8ZLStJ~sPYh|ke2J{P~lSN1BglyT4F
zN#qY>DDrgZ2(P-bgw6W^%PxgHO9ifpFSOfUu1o2!T7SedML)k*J^H1m@&-Qt|F5eB
zHyeb1N_%-I`K3|{w^n}7v4aNUfvT%&yB-S)f2?MDS5r3WC!b)jwU&*c=$|T{fOE$F
zyH;knhx^8_OL|qroLHV=6E1l>|E6E@?vz(I?r1TqzIzk*=4js3(^Cp{j(uAe*XMJ&
zETVU+l325={vj5Fq*ms|ECxz>$uoFN-gO8SInNeVN#a+wx?&I#!1`CmkSW-Wx0sFl
zx_;y3B-{Uo-WVnMU3;ee=6j*FQqkgMx2p9VC)q8oh&J?3nK>urMOt^xnMdbWXtgfB
zQF_)Q`CD<uo+*u=w&q;e694_`!?NuW$uG_~d{5vHbNKe7^{j$#+z+KVi3yC6%5gim
zeHJi3JHayL`vl3#iM|VX6eK6zJ|H?%R672=*-<WYU-hK_?+wmnx+tqyFuwKM-{;G;
zwK7g9I$Ept@RFYm&i9^#FMP-HKxJ#)Hvc-2(!eGBflJlYcQ2n7J}uBu$;Q@4^FPb`
zZ_9ZdeBT=AZ(Eb2Bk;&mX>o=yo0<33M$7Y==VydXpZ3DT(5^P{%#te(>N~PkG?{px
z&F<`T%Is>nKQp5BWyJ0O)3~h`^wc^}U{#o(9Fg~X=KRNH{mfAgKl+8$96Z!cDsQ<s
zX=m~)JIn75Y!~W+3q7~E9ygZ_H|ack!syIdvukV&N(|p$32GcSEi!(asT1xY?qy*l
zJl(&Wli9)Z>ZFiE&7T&vxJH_L9uD&=oHbu5NU8Ly!s%sY>0Bkn*H@gsvSNOBx%q-}
zSG5X3Gm|ej`dOE-JmF~j`qR|Qp;GwfME08#m48k+C^acoVbW=<ww(tOc(*ZIJeb~d
zb@C;*zl+SPwa@cDeqA;9$=oL<1<#o-OndCtf865u_6q?!7gev9J+D`La)E5fH6^x{
z(^r3HUoLF3QrNmZB+W25Ra>ie!H+`|{(lM*;A;5z%qCEq@m!Ob`T6jPeoV_6xc4<J
zzbLUaxLjQEhIZ|)zAHh?_XaVR?`GiUnCE$;<>k$J&pRd<FKSh8nZO#E_x0xd*|S%%
zR`tujj|d1jm6F<&V8x-Q#BgTqofYX9w<SmjORYblxqesRev?H8VITJY^WjsMx_B(S
zXPfzk%Euc{uI-#Hx$WnPw?Wq~8ARW|$2alY64`afws<@=+*DR_cYWFEDKB?CXg>R6
z>*w3%MYoTomG@=8K2+XyeOgq4|Bscs6&5VtwT4w?!p16J21|C~pA)PTDg}2Q-5oXQ
z^sWArxB7X-DtW$5w@A3I){tw!xhye?p~2?F<SloE98P@xzv<JYzV2hI^lc)m*T3!9
zdxCGjM%uaS<{pnX%uMf8RZDrjq~Bo;Z-V5d6~gs`u5~|EF8@1HQ(<|?E+<e$t9r@#
zG;OJPmDc<3-YL25)|z!8Blx^^#P63PKXldyDa<$f`t0(f9Y1H@)w16xbTIWP_x@#}
z`^;KABSPc+Y_6H@2)q$(`e=u7+WgtS=ex@8%H=(kv_e!+BZWC(cfAZlpUsJrYgJO_
zcIGlGn<(UqS|sKiPx@5m{P2+UU#>-4%}?=f-xxQ6C*WCX(2S_`D}C464vM|9{JeX~
zcSHHk0uT1qrSlJ8-o|ikeme6Kg*%g{D##ZdvVVVuC!$oc$g|NZH0@Q{_y3}O#x=Rc
zGLfGoR&{<i^_a8MT2^I;)db1N)zic0PdCrwjazg20P76#HK!XYFIE|9G%1<=@u{sj
zp)dE?_|fg9f_egHc1Qbg{WuhI?Mq(l`s(%D_qiDD+sXX>U$%dK!G?#?&$0_nN2}H5
ziwJ(L4*aWo`H0PC@38xAJ5`DzXXs8865aaXo?65AbYZ)tr<a(m*c*B1_ngo-#a??K
z2j}W^7Vt#ezuBVh@aaXRp55G5U*(;iwO1H-&0k!->QJ9(s>jl6Zq_|_PQ=>!{qfXo
zjrl9!!|-F?`|RebBU5#xwpF}2b!t}My072PZ7V<j&1yr7gllDTZ`jWH7Ex-~dHXG*
z4=MjQ&Sdi3{NSvm(`kRBVCB9qF~=B|@uw~8jCr-&a=EI6j(SkZ<6=*5<3okJc{;7i
zT!Z@_#~wDWJ^Xs^Rz<nQ?-4s!SedxoJ>nh1V0U+d{LYD-^Nudw&+|uO(%SvMo{Amm
z+`qHTH&o}}iF%vI-r{%9pS#xh#PE$1*NuR6|C#pZ`z!zQ@oZDK=>B{=L}+7(Q04C}
zDtyuklT-v|xG=2@V-n&@*cy<yI&94?jX5_iC?0NEqOvWKwQ!MJ+Y+rH8O;V?msTO+
zIv>wXs*+PUyeBCYZ<xAv27hs$*M&)om|BHcJOYGF7B<ay@)Ub}vgFaFWlR&4dZS9T
zPA(4UjpEG;?F{_C-cz(m^ug4O$W0l-O;Za(7;kSYcq}(pA+hN$r_86Kmj_f=-=C+-
z!lD=0w&CHS4t{yP!!vv&^(O4hxoME<l&Z>Qz|Av3!GVE+li7#m@Afb83Mzs9D!dC!
z88~OV<p~-@FBIY2ewJ&(qzjueC&{p|y!YN?_V)hLc&-OaVjS*2JrOMwZV}7S#KIt?
zazIxht)xqQfA{i^4WcW%jfMNXPNZG&Hsjs9-O^+JpR-YGPfXC$P5kZ8TH<RUR}}nl
z%7n(qNK*w@HoMjbE_sZqQ|kY23|$y4vWs;=8{;EWR`wX1ur>xIt^~GZwQr`6x|m7=
zof0^EUp#VAoiRb6NA1cD*Z+NLZzA0K)uudjl2qqAIfGNND3D29=0R$Bth?$02Zmrj
z%ZV8aHCvW4L@V7`>hxQ2!Gsx{6Mx=qu2-B8E2MiiEIQBGGhl)clb@q>ep!%;LMrFi
z)H}uJT)#heNx5?EPKU0d*Q59bv+J(G4NO`G9HyvkY%S+h6bW+T*=jW3V+PlTl`E9K
zP6_gO*gHkbTlnDdxbl@wSMRuSUNo8SvP~nz+nWEX(<z7Dwo^j`gmN6+<|XN_c+<e;
zWO^;?!atKMftRy1#AC{DepQ@&r|-LlrvcZp3kjW5PfH~BERF8!nCO{ylO;%FLd%}H
zoB}Nw(a%;t-K4wKY|>2OnVcIotz}GH@LwcKWr?R#^bt0Zzyhr&x62QH)jYt)lk5|H
zlqdN5^WyV`heC>5s?O+#EJ$4*D-@J7@%2dq<?wk7LPw*FG$lJ*KG!^Z`m0F$$3yqF
zjJSn2-z*8#TylVIkF3TTrwqRqwWiySM}@Ddxbm7_3d*+?x+Ws1J*7=7DZ(sTILG5U
zn_Sb5B^@u1$tEW~+P5Nry?na{!#%w+h9&o{ul;!3Z^u@dG$D1zj1+eHM_Uu?Lz8x<
zPMb2Hr_7)E_??g|3@dI1pLUw6x;&zc>qfLw@Q0bdL^iXUN0%CSTM2gS^euVuhHK&+
zYtOEG>n>~$5}wi^lCs5l)ykQ2{(&C%DsNqKm3n$d@W020nb$9PUUT+0{-(ZjmZ<N{
zV=lr!%s2zQ%PnUvFWsd4_sP}zTvyh$fnM6CVSm58e-?7-kMjzXh?$A;fgOv)mUQqn
z%qa9`diZJDjs?69<qsJ|9ga-rW9U~hle&<=60CQ$#2~NmF_Ywuqm8GYw8hOn$TuxW
zt>NziH&d?0hb{>_=C@oi&^A#y$vacY`N@|Jn(H!W{j1pAswBm3ey4Co!?X<bl$mbH
zhjvCRXk(sv%i~yzuV7@!mK(x1dyjPMUhF$1bZv5A@2Rv|(gBs{oHyUuxm;z*BoB{0
z$Ju^G<SRLB?p&@m-E{v9KYyo?X&dyeZ{>X0pDk*n5+LVp#&gy+SKxn2^kj!4ddKFx
zaLH2;TGVt!Y4_b1lPj5*-nIImq#&s0VXM@m8N70bs^L0MkpStbX^~6k2^1B_Ek41w
z^VsPZ3K`Q|Q!Z>`nKt*$7M7CT6PtvV956CpsHf<3WO5<P1a*7O7On`EWd98}rwAVF
zP}(K!X(1k}tuyUR66el@4{Sn=_bS+Xl`n>rdzt!&YO<)jkreeZDiWG{W3tNC=H=cA
zLE1rA1jE02oLDE9)E&R^rP(9Sl&E>n#8+x$*DYKe#w2~iVTaBF-J47kZ_AY)<oR}F
zNdTw#PMZ@!M;bQpD7Wyis};}8{Fq|=?UL%#rh)}_-?sRiW=hMfyRm-ZKkpThjtu_=
zqMY57-YrYKyr(zV+;H|K-y@v?j$$Ie7j1M5OgEd;;8Su;Q+tN$1FP#BL!_?1nm%)8
zyP|%lXpO`p+m;Q7`j%MCKmVy>a@d_OT6Qnx|E_B@VUg|K5aFDox7h9CU)v2iTsQby
z|4cU9J*8mTl#rl~OQnliO12r??-A_z@a=5DL^Jnt=cRo<54InA@uDEs<Ibd*n>$Wf
z6-|ptdYx1(X)yoOqz&<h=NLpMsjXGrHa&S$U(sKut)^R7cJQaB1#p>8yy+0Axbkq`
zycqEn`*_>~Y%Av$mptD+{e@pyN~}m_dAfJgfn)QzJv*bFmrj~xvt*xAqDRHK9|B>j
z{lW8<IK3bIPoC84ain_Ftq|!s=KFeX9I1HtSXA@LRiQ4qwVDwdSGfMYF3RY3I_ml)
z@6^p(bp?7=VmLPnda-Uu+MlywsR7%=Nhw~z6~5laoDWu}+ax>JJpHzOb)sU%ktqz#
zn%hsLR6T6&@CsSBVMD89+^UKhYYu-jiJZRH(;=tA+3l9dMNQUD-_W&Pu^$t7k`MlT
zSFo+X^+u<AS;CV?E{}YKEVWyNwGN+iI3b`_t99hO#Lnjx3~P^7E^=3V-T2r$>)J8<
z+0QSE>^$#$L*$Ov|H2)7pQ7r8Hf1W9ugE%*lyd8WnkwJQwYjs_rbrr{oZ&NR1&i^n
ziSN~#wo7p?bE`XiQ8Qe7$^V0rc~@IAmrj~<gmJ4U=k}0CLh_q8?fZD=lK#xwYiDg(
zyez9+Z>MXiQS@xykT(*=D!R^U`FV%c%#y!aT)ER^vi{VDtT5w%6P;6zXBlTtS`u#a
zU98~)SJ4DUL95CYQF1lGq3+J#TSQsglgrJR1X;>2FL)^2uvQ_jyJc2?xTT|%%gmz0
z%RH@@<@pLba;1av7S2#n^Q{aHiP}Ex_>UO<)E<tRTUvz<amP(uIz8jvRDpTn@4OE4
zR&5Di9CJyb@x;!aZ3i~w&YB|1!{Fd>;NnDwl^QM#oGi}gF4fLR?flfXD*jo?o;2av
zYO~fEov_*-|77iiS&NNli)`GSw{;%Fs{Q|u%~cV6ap!02gr6}g7vIiImiate(3<N&
z`m@u?+6KmAdF-~Dy{&(4oHBdmW>;`z(h=`T_nuu6t^KCoe<(14$z4f~m2YaZ_9V4x
zm1xzcOSegP&6zrF@AtjdaW5V7*Uj3ppfj+iN%ZwX|3hv6DlRijpB`;~b_=7zMa6xY
zcb+aw%hcH1diCnFySG@@&z_we_ti`0-h(@LEBF5|bA8#zGgYdCWsCWTH7-l$&zPj|
z`GWiYPi=_<@Aofvllr^YccWts$Mb~ej|J8?6#kkfmibY>$6f4Fkr<ywk&=6*;1S)L
zgCe?ZGWy>f<yeXpT-Xecs$`fm?m8%J_e}Drij-DJMD$^){|QM7$CjjPYxsDvs@-Hu
zU3{>~Mp2DvvNLC4>a8^Ah;1QrCueLauzeH2J<FlSFeTOSu+-XvY8xjSY-J3+Afglh
zJoQp}*`@6|nb%_Fn|<@9i57_)-@M+GbVT;o*Yp#KZk6HHe2N*&Y!%JdHJ>C_a80dn
zRA#lgm|1f}c;UB*MQK^eW}c?gRrHT$IepSFDk`crj%qp>wKOEV`mpknBiZ}QI`tM=
z)Lc|47p&g$BsuSgte1!2NpYXOtSKkkL(Dw75-wRX7)Wp3z~$<}y(XZ0O+kr6NNU>A
zXq7~@$gtJ}87@<{v9dbW<+kUAp0G+T%!@f-b^J%!eMRdpVQr6HjQ(FvG5Y$=m1|>u
z^Z{*+MB^Es{CpkjXF6$boYZzkIANVrL0wzHhDYMhrpo<q7cfbb(@*67wm?AQyTEL=
zz{`^Y6^;ew7FG(+5EolwZg{xJ@LG`m*TyM{l7)*{ZWT6Ix2xKyOa0!cy4WMeWQU#8
zjV9d?=e}?$=H?O)_7Ia5mj2=%ze-!)2r!=hDs8=GV!%h|11(al!8*q;rn)V$cVcYi
zTFB~sBrhvnO6G7VPq5+jir(U~^o2J>t~-`;ESUUaMcW)Ff8CGyVLQw7t_l==sckwg
z@NcT-mxbZ&?d|*v+g%bXn^u?!IY>@6VV&^NZ1zzJ=TDXTm&}_Pl~o<QYB!tz@2$vQ
zlUil-D0^1PbiJjKsvoN6yy%?xIobcGwBCyJC5DsEJj=;ER3h^;c%1~};sU!Z8~C<*
zaKE|06||uFP-)lno2KX6Vtfis{ViP1g}VeQFl_bAOE&NgbQN54dG_^E8TXg1+>d-}
zKMC!<?0d{2;HpGjz~Nqt#PAB2cEgPc+{?ATe4n#$l9sHaAg8#0_)gp3=F@Bq?Nk{X
zN{^&(O(=XZSy<X3QXr&A>8kjvDHBQ;H_mO>>tzg9Rtd^zcbRce*lJmhD5sR;f(Z6c
z&c{n@j4jmcw=CFoCC5v2VW9a!uhtNkQfH$hQmQLsyk0b0xt2I^sCl=t6iyX*bAW5<
zNxuKd4W;@mGOH}4mVXpqm#AQ)$q=%Y(^r)-%ze&g!zob_Qvw)W1Fp1+T~0eR({-6)
zTfo94S0<)&TrgtpaI0#cYU|jSeRb*=jl#tOLP8reLbS9dOcSU+8W63(7#yIXlF)IW
zO{vR5W&N^_D&r~*F^NeUoy)^JSBF@vR?nGj!I%?Vxrx1Ng5*q<pW+)lXEHBj<xpk~
zawy&NLwf0p=!w6Y<#%=Iy5v4f?%sMpa7_UB;s%9%GrBcSc4fQz+)!-wHEvBY$lH9-
z=fsay<%ebqf1EA$IP^if)U7AJb&mNb43*9vEPE-=dX!l%M^x_jRe>|bHunx|{WBN*
ze!YOpe4dbc|Nrly{VxN#f8LP&eS=Fzz(eh5hXjk>V<Ww9I~(V+D;aIIQpyPal{WF+
zE`94#%i!jj<}I_9Wvo!_ObI-0G|PaEdod^P<Mjrb405wVrfHRIWiJg_&bsH2pn8|_
zk`1g8=5o73HX5z!w#{(%cFhzCbP1X26ILNJnQ^tfl&rItpjV{fwJTFvPR)r7PPxS>
z&1@kUd`iUhb(`^F<A>tYaz(WIujT7EPg}czRXBhx_k)shl6m97D3e3xlV6xO1kVVX
zUDbTK%DX7C?BR?P*EYqR&N-@58ohusN`Q0mh3X`2*5nPGdmJa5XwD3{*<EmY7H1<%
z*DBV)13rNbEICQ*3s0B)pZsdp>YcNWx-8y0(I6_pW$RAqvn-<1zxi-A8eabqdtrr+
z4`*H6*VUIx@|9ft-aM6g>26c8WnPop=C3IQb2iPrc(aa))jzM*_V#80fsOSaX2^YY
z5q{n&@YSH<<z)G@?#hco*FAd_)F&vZ`elOp5jBfe{ke^@)4inT3rgv)XktrBS@0`b
zt76iX)iY0vmb!~gT3oe3QhM^jUEV@m+bp!WOiuGHR^W_Uu<fDuh7Zz{S$?OOZINHw
zIO|9B265>{52T$vx3w6QEY$Q|;q+8W_;-)luhhq8R(zKY?OyFTRg|WC&^L95YmZR*
zGZU#>h5`XrLQy}Z)3;2$`0?OE?#xFw<^G>sJeNtJCNayd{djnc0i&@~XYI`8tDa=f
zw1}>JsIpCTMf0t#4s&MO8t!0Nz`FQAHPZpUEgQJF1G&|uSUq~Y7k5>!pINdfYUPve
zjjJMKqHGQ@I~|=pS+>(#s9l?(>(q*g?V+!x<-K_(^S(_dXZz}c>uOI`{Mv$cUiu~R
zvbFAz@|t@;;^l0F9#s|y9v3-zs!dnX|J@7!3$F_F9Cup=_ODqWb5n~!V!QTguHx>(
z`A1f-Q&&;V37Tl1zCYi(E5TVxedXR+m$yB-y?yzM7D2C2Ze}_4Rf{GXuz7x16ls_e
zJ^572D$|6RmCS2SPgymKQMqN^#}WpE4c99c$EvAY{oMF}`C}J{uGP7R3{ONHtUI`5
z*~is9i_&jz<P6j*IPbjXzsQl#SISQ*SHx7=h=1J7(OBV_ESzG%sHl_^A?-c2G^gLQ
z`o*8~Ia7_Uu(2c`*!OHRAJc&&i*Bx5tiWm^b!5Bv<l~{4mwvC1@mjU+&Z(l?!5NF1
z*LAE|myvrdtUNF1Sn>gbQ;h~)*~ff7x~DI<3VzVzs~K0IcD%%APF%)n)48if9$NQ*
zk~w&4O4zTqk10zZt<d^6^@_&ED<4w@|32;Olc{?WX?wY9H&X(?{$Xv2aJC>*-OrQu
zOqp!->(Iq(w`VyX+t2dbdmZDVLw~kq++FY~+d1aV>64+(Vz-4vx)x7z5d6QjXHl{M
zYyH~=r$e3p-(K+V&}nt&Gr78x<K|2%vppiZ>dZ--o9h;al!j|sbxHT!N);(woG-NL
z_>INN7hRZ{Tup2>!bFarGHSc!BN#J5HZW3WTf!Qv%~Q9$74O)iAm6j}(~OMV4NL#H
zU-_^`AUR8Ti2~!vZ-*IjH;C-L<1{DaCL;^)WP{Bbv%F%uqb}@A{&O@)=K9e)H!^td
zN$frP+xo`o@_X^#M+~9`bT+Sewe1*Z;4$ICOPdYZV#{`{J(a3+_~zohtsA2xRTkcQ
zaQ&7_<MGIiQ=U(m!}V~r<Q#!J88#0;&T+XbwB^uTGwZ&Oj%z+mIU#iM#9wy#8ZlvZ
z!;>--cmL<y*#Gtdn+*5*JzM1Wn`}R}=G1v!(W74{`lQdif4a;2_$g_tqjeW8rC+ai
z>sjPdw`i6EtNWV$GgscQy!Yg<_o+YJPgnoAac`~niJGV1yPw{4EM*a73~X@OtIeQ(
z*U%^N0k1P#blb75nrCmC3OZ}nUd_4ec~kwynN8bY-TL_L==8m^74El>OPzT7=X|VW
z#@bC+HX7aew8nj((6e*8lRLB?_sYr2`(~zmsovLa5YpNmbLY<CC)?-o<u28^DE{`M
zlI$hpRrjL#&c6M7PuJNkB~>PV+UyN_&o-_T*lGD}cERhAsVrsMtnu?iQ>OJ?dEUm+
z7usO-`ck3H>}aEB<_~_g-TXiI*W7pRa#!q@3d-$z5cqJ-BiTP+>)5Xe-*cYVU?cxu
zeHq`vtIH(!^Ht@_)h_6`+m#fw-neB};-wq?x+(v>7yiCw!2J8AWUSg9ZUZA1HKm{X
z#AP3M^q$g`EN!smN<3Y?$NVn4jq^uu!Bv8}lWRYO{(BNT=ghtAm5e+sQ*1w}>{pwi
z8`^bCD0!aSxt8PRTxnLFJ*7$vdVhTi=ZR*Rxt@FIb9Yhnn%;wxcOSf|$a+QcMdZP^
zxg0OH`o4(QI&|`JSk4u>_lgCd+68sjgx4J9>XCctz;fwrN%+fiUrqi=dnmB}&DvJK
z_wM6^r;^`4USN0hrEl)hDXH?dJ3Is!bvD2HzjofXQlDpEbf2mHP1O&)FRT6RMcRXH
ze4@vge9KlZUfQUW-u{F2-h=G+1DgsTv@{)m*{U&PFXzLED~qEaKGDf&FgtPXm$>2^
z8#z6{H?F&1D4%=~%)hh9&hntPZ?Xa_*UMV#6koR*8@WZ1e5D)*j22(uT6&V}aQfbo
zx0(B<HW$jS%6tEQc4y7mNX?x2vs}|<gRbR@)n5N=z2Cv^(}_7}oGNa5Hhhm;cT@NC
zM)lfDX9IVB=s$3xujjM9+`P5+Gv|u($E2Ou7|NjNpfFumhK);R2kS#8JthmqnjHa)
z8rp=#-L}jSRBT}J6jJG!;IeRS=Ngt7CNmA6diLtZD#iS~^gl_P(L}IsSLP(%(`Gu+
zH$@y)O$}gF@w}Fmx>_`-)o89B%ZHFG@4is$NkReM8mq%5EvsXdzLv?<lESJaFxyNg
zYFnt_4%QFfa+y}k2&y;;1vv?625tLs*T}8m!TKgvZ4(=b#79S%PU>v)h&YgZ(vLI0
z%7^XqwEav=Km65leI(Kn?4utNF>z5;H}6W15Q%7p%qvW@(o}*!Zam$$cj=Z)b;D)e
zv+W~yz2OYD3<%2&n`NRW@ZwNd%<StT8$PQ_2JPOt?a{fmDaDL$J*MSfeV4x4H|;c+
z0e6aG6r;l(?hfsby8rq@SeZJqlv$n#Cmdk>IGdk?xv?WbIIJnTU3A_58#kFaSH8R4
zv9FV9PXA%1|JK}EEV_>ZocU~31Q?iJJ7>5Gy1Ts4<+q>0ZU0-O#otlJk!6Fse0|8p
z<2!A<oYwFAb1S7=#ay!4vvH2dfs<;&&S%VK<{iG|b(7_m&_bP^UyOWC?>Ta2^FAd3
z5AK^=95R~B)RlU-ZQ(I`p`iCJq}S9o`d5dCrkN$b>V~%29IL1COy#)PDi9iC`DTf5
zhsc^Tr;z271z5c_7KsIMt&ED;nd=|8f0Gy2;|-6zR`{|Uns7ew;H83vVavNr-KJ;F
z%?Rs!uqb=oj*6{a>ot~0T+=_mBBdNMeG2R5NbPv8sT&R`6t6KmdL(9r(R7u`W;fOT
zCr@_jR#B<Q;!<TPDmJ_I<ch|5+gNSiwaZS-_^zoW+>x}z{d=#T@+TIhl7&y=O1CU~
zwd(S@rFT6%C9D@lWiqd>>NB#i{T8wJ>q^O!vkLe1IP(~4{d!asX!Z4$qB3VsVJ(01
z^oPzDY-8<tvePD*FOqyFY3kyg!V>xIhmJy(%&f(W*jWr37mGgW@Zelh5!GEV(M9K^
z`f9gpukzV1@5*jj(`WRir+{hoDV^=-zTIp+(Dvr^iD@wz`bGP_!WGJvt}5PYwev>w
zqU#!29Lg(ytm9Qa7{BQK;=XzJ!j~@4$r5?+SRiT2rzYbrn~R+pv-+ky-}JcCs!>U9
z)$-3>QCeT`{^v-GRpZl@$(izMOV^^(+xh2DJX_JQboKRJQ~qVFKmY4jHn*whWwFS#
z$w#l{KA*k4Bl1o(e@^#Z7h#8{Tk}`#-nJ}bcP_KGbWynOC&4P;$nvKlVO3X8>7RZX
z+Wh;~=jp##wH|fx-rHj_ZJxD6xWFZY&NY43tABHrE2+sf&N-L#Dsrl8j+y||#;5i+
zzt6Lnr>y2>&9Ijd;Sl<;l{KixF;(~VjMEl}q-UI6Q*N+XfzADh(fo(Y(|<NjT<i8t
zo<a7kyYpqi1|FA9_bhm`9wtq`WuzpnYIxDi`<K|{B^FCVeFM7!vxLM}&p4_bR#+A9
za!=TEg}}uOf2QNb*Y4iAviJY)PS=_eClA*<toGMD^0j0bJ3TxWifm-bQJr~O=9yl)
z<w?($Vs6Q!OVqwwc*JIDI2d}pn&QN^ps}#(%f$1Xx;s5PUu2c<V^V$Jv&|yWb8SUd
zm+vQ~h_#&t>#MdU-b;A%!!*in<+P;3PqmbHSoP}U=bkD)sa0<|^{$V1<(Ea%88&>M
zsd7riPyb+?cb@Zc)ngaSF3p}U`(BmRh~>B%^W~1eD<1DZ)@ZDI`R9Glx=oW`akMmC
z<?>(X<u1bh&&4+7w1;ZE(&q0WLe2kJ9-fVGag3R&lWsDnqgJCriECF%2KQNhXU8k6
zR_NsFc?&<f=%V4IZ0GBF&M8cPrK(`=%>TXFp{t|B+z+}=-BuE+UAxkomErQtRUt}0
zR9<=<T_$;)WAfDNk8Xum%)ESfis#HA=gH|VUF%wHgmsFFuHXA);@ckaq~O{V4)ssR
zCQOT&s$0Q1FXO__2@S4U6Z`~2o^pwb&6;s!&f%AyUPht276lv+e$XhZ{&iRQ?YHa7
z_cQ0be>0ikq3gjZ$7d~d+j4cr`h;K~wvY~S_06Ya&#m4R_-Li>(v#tpso(S;&5GX1
za>4etPa=1b(b~HrNfSN?Uat9Q)p1wnke#Ym%PiL^2mZt*opVfNpL_VHuG2nEtACsy
z=5j>Ntq?0)wcy2h8}7jK{sL3O|LVBdS!_zM(^Wb=r**}G|5A4n)DyoM38`xAW;uJl
zu_R?d+o6DkMU0P{CWsx8Tg9=)ulvyD4)&wkS!Ox0JhwG8ikGe|=*|vV@~A#t>H+KH
zCrgDoS2~1gC3ZiUGF|(qx?i=`#50eI3?=haZ-!0Z6PJ`0dgFFCo5~@zizhm_EEbyo
zL#RpjK;bpjz`z*yFRYUn-dMNhkm`~G9Vd+vz8v-Q2SpB;q)vVw=A*gf!jj&lXKo(&
zcdERe_1=yjNjm2;j-A&1sQGmo*X$Q}CvDpL=EKattG$gk>zzreG%iw4+H2(L%ivn&
zm1OKBT^_hK=f&Zy+~+bc9Q{*gtEh11UyR!D=t4_f0;kIUIiDxb<2lpNn)LqxgZ#4n
zyRJD|i7B(rno?#goGYI0w^H#OpUJ&ob`xc`)Iir_iwhB#ckTUMc(HDRfw-X5+eLfi
zF4T745Z7J7ndjHJC1it;BXfGok>spYpQW*v*~6<3US5_NpuQ);wlw10+MY8Wo<3}X
z%`>gJ5+~o8HPe=HP4)TAi667Xl#=b4cSVXG<ansIDb)6AK;h#@X+bBlCU4=|se1Tl
z;U?YX1q%IBRo%rHmdmTobKmr+#B^u2_9C0qe@iAjRykBOb>kTYU15HMS&FR<g;gGI
zE9cxkxMXR{^<U4OR==&Bd~#W`PWMf%v}w!Lo>g9G<onXU|J3PYpS{1C*@>q4D&8|K
zfBW|TEstkc*0`B+&9qwG^FzYFUeHSF;hmBorgQhhmaDVKXgwA1nEcUWyPA0ay8x$b
zF+r&-wuU7W-+hTxSaw&^e)r^r@=}-A``)ie;c~rD`dawS*H;`B-yUk@O<b|WkfXxr
z{<PvWz5J7InKzVXuHECN!=E_KX!hA{EFYT-nUc8H?>o4@ymp84skdhY{yx7KW)R>k
zZ=F&Zr{yZzQ@8lRyo|0k-<%+ijGH^uKW+_UDBZ;?%Hi~4@1@qza>X~(ZYvZhZoaF^
zlfLAjw0KI$-TZjll_EYoss~Q$NGdLO?g}zF_fazRsmJEJsqwCIHzpO&Jon(fjQhvE
zADCyjbcw7f*rBldzYPP+;>)L(^3K-b@G&+C-W-0>ID4a|)MU=Zo8+v5)r^>U?<7vI
z67JrwXq+C%yCQ1U9SM$08C{V(#rGZPDSKh1P%N@>_rZ-`eJn9c92ahpo5S(;MW36a
z!zIlw_h_!K3ap$F6JxFSEWaW6de*|Q#r%R?ilQfV)m)@HqHH2>>m7Z}p&l|Tp4&O`
z_9o+?Nz6$SEGB$ke{9+4+&Pb-YheY?tcV%j%Gw(&^a3w#$cgUQqO6&dY>@42%dy)m
zYqch)at~w8&P9(6TqgCbs_fY((6QG;xKly=Wu)>=q3-X343d}4w`X!3pS0bnS-x(w
zZe*uq<UzZ)hV$M&UOZi3`6ucBeP5i-SWWCploYlbO{rgK;XAX#?`P-I*V_Xx%3XiC
zV?yw;z*qYp{j^W;nY(+}>XSbWZ)+M&_vri@Jf)4%(2Zqfs^(^<t~LL6?VV?Qd_ngC
z4izhYW9=EmM|y7XFdFDwl$gH!)@EJ<-Q`_tgQjfycGAJ(A+PeM?z~+)CRK3B^;kHb
zJY2?dsBWdi>l>Z6F(U7NY*~=d{J!C6l|#o1r>>oWGY>yjpVR5|)k=WdQgPZrm1yRP
zM}uvS@17XHd!ka*oRi6u6j=^^y5TWLq{F>=it{JWhqDgHt>NVCv}ky_u_M{xf~Q31
zoUR+$E_V`bZ|v^8zIel`DPF5X)@<F$bC!Ff>zn_!%*i?jB{r;N(MqotnzzVhrz3}$
zwPRWGs;A7|70t?x32S?HOy<6$Vsof_!UU<6ip@(t*tPQPUvDiE$msBlLDt#OL3OjE
zXXTRb&K*B~cV=YGt=h9V^@z`I4$h=Pw<*Q;rIR`@zZ7}t!1eIeuBAT=3x!1GY`mIV
zELc;Td3GCYsu5+k*~{tUUX)_Icos)~%JJKs#!R!<9M_nq!NIv9pvf%DaL*T)YX?u)
zvIqxV=}>YK{(GtA_y*Q_bGTYecGOpLwc1E@Nba96bjWA1%;v@FDr;D!a#&k6JP)nr
ze|>xFiW&ZlE6&TF)Yks8TY$^VaL#!{nWZO~E&8K`OrQ95U-*A+`HF*5EJpcRDz7AF
z$o>(3zlEdBi*u38g20bwqf<_7oOz0Uv4Ox+Ro=~Jv%0)o9&Ni@eY`VbDd**$Q`(F7
zec))}SUW9X-?A#h=>Z3uJGL*seLV1pnB^gdjFYPtEEH@JSZL$au~M?xHcR`d$By^f
z5+9gVnEsqpZ{4xO>C{C@i-aj0iECI-iHIJ!ZJ+AGuvcK(qk}zx39YKa3jV@pcSUjd
zCkJ${xSZfJm+jNZOFeFvMAQ$a%(ay9=ilpjG26u<#fq<6XTJ6Eh1%{5t3%erT-@NG
zc0Qn^zksXUSiJYd@y;2GJ^vnGsj+CsNA92%tyMcNum7=Zs>2xuoh45#`o0?c@9U`P
zR7_FtkmT?2;R?BX^2>>Hx(!>uB~4V{v+my%{S(Y`j92xOgPreN1w6{Ike1o+eCx!z
z#Ru2y2%9^F(^2%wOaVQ{NzR$NSC+EKUUL+=pL`&%X6K5My=7Cq%D<lFG3MdEz2`<E
zPj#qE4e!cR!51IYaGo&`QT@rY$-z7|%bfR2N6E$GD<_<O{@8u{N<(k2t@9G6s@y%Z
z)kSF52G)Zv7Bf9g-IMa)#2DgyQp7xW!3mZ#Q5EM>bb`}F<n*F1cU50oA93NtmdNED
z!G0S%FN(NbRynizmT-Hr?<Cf!&0W52w*$I2oHdI!UUpZL-6b%9wR`dJ18*1^40Aho
zN8Z@&`QK`Jht)%oEgROVy(sCbHoRfdD5d+udxP|q@UPdV{h0ouV25WmYgtj}QWYn&
zh8gB_B`&3$=W-PI&tjIM>zumO^PK_TH{&J6vw~+Y<@B$<bE4$%*Ds;wQ#g8g_(IO!
z`6|L;wfE-Sm>Bld+iqOzT2$68jNy2j(0Y9e$1)KW-lb=+RG$)(JtFwkOYp7lCYLMa
zPgGAn>d}?3oBEMUD%NoNjUzoB+ht-o>X=<a)>!iJ%(<As+dgB}Dkp(oe*`8nY9G2J
zq;05r^W^1Io>LWP%dGhkak6vEj0K#or-Ql`WPMi#CvIu|rg7QW_wwZ832Qs<bbh#V
zt~S7$^?_!0@ZuG#KkxV-WM}J}r4!P$@Y<plnes^nogq`qtBqgE1aXEKyM-FK+q$1+
zbYJ=RFkg#mDpP}$>}D^M2=m1078f^9-Rf||Dlt%1Xz3)5BxU9>kI+t!fSi`-7USg~
zTqM}@?skfJbx*k(yR<LcSMucT^{2S`8M7XQ<eKEa4SUmYhvC>GsWX0ao}9FvJLAEP
zhph)<{x7|p9qWIN^{%Yja<;U`m*;M=(79C<>UCpk$np=iWxcVtTmmbuKA!z*@6Fd|
z?>?QD>nx{naqpZ%&J}m|sx|RnNDvm??NXl2*}8VIzl@2v%zTM^lf`W`=KSnX;az)P
zs5fzAukHka^LsBZEx5XJg&Qm1O*hZ~e2Z)kbUlvW^y$p;GY`+_-ibZO75nX}KG);v
zCnCEaM24Cr%uh`{_BF!s=f)c)%a8uyx*rpD_Rv}OoLhIkgl{ZJT)J0v`3jvX&29T9
zF#Vm<_4ekKZBMjpf8Pq+X(f@TRX(jV(Drs^Tu|$c&X8j#=gZ#RvVqTX&9Ns+hu+TK
zvgm?G@0`9lwinp0U9;Si^yNjW+T|Mw@1AbBk+ju<|MP{I=nHp(*Iu5*8~a8eHgjrp
zW1z32YWTa7yUSR*t<792&t7+Fd6_#o^HX(a;pDx!iM)cdH)vS#oXbclGJM!NA!WPF
z+VXR|Ha>N~;&3##&+607Rc1=uTXyW8zi_5b_p*I&x^G?l|0?4bCzD!dS<&rQ<pb<#
zC%y#VKiRmW<)O)fA4jjTbpM;$8FTI7oSF+-=Q!TYiJVzsGEe4N^0w6(A*ual9#>QD
z{5ctLYU32MRIl@|V=Dtgc<ybe*s_;TZgcK4i<B<j!gs<-u^gHf(<>IPjf_h7n>26n
zz6p}c(w82J6593f%LGn)rTf!3cGpdN+!L@a?nvm~siBM;*e&jc70%`86noS4EGc(y
zWSr~Mc(HDochQV9W;9)Uq5JIWxwBzXcV5?u&7Cij*}`$Xgz3tavlph+aoYJsS+6r#
z^(A@roXk0W+qNvU<$C9JB4N#alhZeoCj~OxyZ<*++t6+iUskt8+~vAXbFKeJS58`N
z?HFM@Q>{OFDuXA(%07nQwyVlk%X+DZNrkB;U+i&rI=ekM^@7bQIqCa?;)T~l4z%`6
znO}NlEkj~P?#D#FV{=WP?fNJ3zxS<leyZE-o$6EP<di&nqw#)u!~5)Q&y?05Fy}5Z
zOb+yV>(JjcYuU+=tv`8oAG~#O#V+o$b2m4+hXo&fIp?BOnA)+%S#J#e8Z|zfJmg^%
z?%FXq)GWULg%iJfvBxD_i3O_HU4EYSJRS2$ASceRyUB&4SA@gyc`BRgm)CZO*cQ$2
zV*ApmP-2;z*AwU8>z2GDE?{1c^Dee`?t=br6mm8N#a@f;v{s){lKYZBHhewL#gO%n
z?ehMAbdlMh;bphKU>1K_m*xxx#_4gJu1@x^<y1T?+Ic}`ex#t!+YSzeH7gI!dwa4&
zOPqJ|<DL}7?V|N6e|@KZnBsOIu0K0|Q(pZUcGnoO@X$MjJ7mqiH@jR|?(k)2+Zng(
zH{CVQ#=ZM|{^u9gxI4S$RJ<j#OLNy0xTMc@-dwmxhi}==`v1p#Z}q(X$<2A=&B__l
zsz2VppULZATk0nwS{bL-e0Kexs{a3R9ZQsojjfcD7SB@6RMJ%1KIMI%>D8Xe<&~44
zuiyH%Pk9|{b-}x74_<U$`)O%gva9U7`+eb^*Mu*=x&1=##q8ztpXeo<nB|EdfA^=7
zkBj$^<u&2ucT{Y`{^x(0#wp6X_zPRg(o`Ov;?i5EPc3rt_FD5*OI`WhMBUhW&gKOS
zEgjYVdq1&kf5Y3AHQiD_th3-p$A<?iKcvS8b$_T><d)d7!N=#w=DjQYF8tux`H)+e
zyYr;J(BzaF-Q+n73?f&a$UawpduLr(XI#v}tsNnvRVU7LB*g`^`E^R&uWYWlTGgL?
z!(ZLC>Y3}$@N$zQs_LII&92myckOOJK5atT83E4Zi3}Rz=g!~r&o+F!Qutpv^NHtQ
zzfbezuAJsx$@1zg=h{Qpg<lnMewfd}c(uv&k9QY0-|X#`Pv>>XUh}BlVX@%<rN`Si
zem>9b_4sw^Jcr};7i>%&RZMKxHvf0Mkg&FF+bh+W83hjxL<y+6wP-Amb?I5F$8{qk
z&|!9?oOM*rVXjA>9m>%glbA0qb)CVKQn$u)mD;&EmhKOqL|#@u;>*<VNyICpA+W<C
z|5eVbFPc|orRZ?ATChEu=9{u5Pf)?NNkr3ANM)kJfj17C^1GLnS~(uLy~5YL=E;RA
z-0a(86pCgr6|QgCSARX~2~z~a?OB{GI@@kUFf{ac3vxJoh*;}-#`H9wkJ16-j1Etw
z0JgadCl4I&6?730c)0N~$BOJl%VK_)Fr5za+a!6lOT@cNBD-%&_8*H+Pfi&d$y@AM
z=Ge@<n0rI$o@wrHgtkBT(R{F^{QYTNvF$rT{`~j(?A)Fw>u-^j)E&E(YpMN)zYYsN
zZQS*~zDBz)%Hv`v6T^h1k$$lomrfI!@A*J7X2#LyR}%k4Ry-_BsZ@0kNn7|-%4KaL
zhiHrZ*>8d^DHBy0HFG97v1zY8VKKp0`Buf^)=Qz=A3ND{sRWcV@g9$LI#l>%j-R%x
zX^;o=%p;e|)m&YLi`?I~EDm=6xFvO(*Io9&>R+s>7ehTJz6_aZv9$7;#Ftc4(TK#_
zZI9Y!{WMJs5&rRXdVR^;)akJ$Ymc(dJFeMrBSFAb?CK)<FtwQs4hGk*w6C@82+mk}
zvNbA~X+c<0@T8z)Gg8^q9FJ*4|1FE1WHB}U;cBgiE)&;meD;4yir@K{S`2GFzH!OA
zSow+PXm62<e&eF~RqCGmnYakKiu<)!Zihr%3%TlkBg^`k!|kZu@xHA|voG(kIO~2)
z>_yX7*N9}vjEv@4Q?Du{WqmxUz9=xXBXOa>>M^k#3%zGAwrg}vlM&KVQe><(*ru3l
zz4gk8*Dhk5A<7fvZQU<S6iECt(e}`_DIfb3;w?5Z@TBsj=sI|be>%$T7!t67NArk)
zBY$X2V6pE_Ep1N$Ps<4vJN8|9b1|Uv=9NnphDUS#Z?2m@f9A8TKQDGC#6=qHJht~_
zNQ98fLZx)e+6=L#xq>p&tA)9=4&2UHKQ({fpNCi1FB8*!wQ^Dayd5`^s;B?wa5=yn
z`D<c_@6;1gD@;#*(<ovS%8L}r&)cL`rBt}L@}TthH<ptQe%i6v<yv5f&Zb*|JfT}$
z`nO*^FEDY^Pyg6k5gPMe6s@){+EM&YWEX>*pxEwob;myqCpYij^C|J<&xu08JRLXY
zneM$iHNN`o>7NTL{?7}3bf3TE(GP)%^KV5Y^X@n)cHm@$&Z$TFa{G@<oRV46Z*(~)
z=g6LK_p^_bm1?h8b0kACF~}=JU06%EUEP23efJ$#Oc+DDv*c%T)h%1*xaz@@n1IO1
zvtC_RyKDL0QOfIn*QrlaMVpqk>gae2i4;r|^>g6Lapv2{q}s4IfI%t$ww$Oy>WsIw
zEggRY|DRnkPic|LqbH91T=$nqeBROMvC5@i^J0d@S3%F0Yc+Rj3PeuZ&U)r8kCu?;
zdCAzP^KOLp^q%+`x-RUx@!XCzO3POBZK_zdGjgh+`njr8=iM$@WuEkFUR<%D$fiQZ
zwQ6-%-yDUz_mWq5sClSA^%4)ivdQMW$hv}KoD9h-0()O_CiutRyuP1nnnX>b_oREx
z3T+!Ij%-x%sa%`ET<}Ys;g=@6vdZds-+*JMr@Uk-kv)9jsLm&&nYB6|r$i<>x0$F|
zN}W!3C_dF=bxCEyAHxh!Z+}0DjETz?jntoB;Z&LV%wv`6(Wan{)0DQSP2}=h>pShV
z*T*^KPX1c#Q|1RY@cv(|E*{3`mRBQrBJsmCi32xS7d9C7FjyFF-}m&a;lz!+E|YGo
zJU)lRwEN|i&K);qO{`jG@}gTQ<%gT=XGfQi+Q&(5g-Sd1l^h+SFC1Fs;W#n=(5B_$
zLJth5T3X4z^eDY`QrR@y$m6<hsO7d@oWT)=e&s;|?)$i>ha7SBoL|N3z&)v{;Yg8Y
z<67>(_D5PF=T6BtEfRH~_i{x{P>a)3Ms+u7$+YOjhl1xmD0R`(KGFZKU7-7qeuEUZ
zR^_^#?)kU4yj-;T-d=hvRH*iRPVmzhtM`jltAa`u!e586+h$L_SocyheB*PCV+Rhc
z?`fO1Qc=cB^z({iI$A~z+)WHq0{$Nkih5yCvawsR{KqLL*6W(puakUdyx5`g&UYvC
z&D)E(Qg#SbPRXA#@rrwC(cLM0E;}!*<;fFD2-chs*q>RlD!A^cWt8OE42dRXzn+!u
z%dbp~Sn7ODR{7Dr)Qx<q*J9oz)i2(8&Gv%o^inSy{VOL}4y$k3dpqm8-i}VgC8sOj
z2x;@I-Ktk@`gCeU)Y6?wFCXqHnwrDO@VepEEw0!_IjP2z<KuNjk|$q0k$gr)?Rd+S
zO_v;lj~AKb%bOmW_-V`GUS>7Viln1m(_XY}GAr!&U-eZfhq3iwT-Nid4KbnrBYtXz
z-iw{0{zh$WhLT_W%bvD5hx|KO&dh(w+&xdH=&;9sDUpTpFBg7&(<7{8R-&N%>`<Y-
zaP-tgU-Bn&m8A16o8`=XYf0xmjVFB5CO0Hrcd^*%dq}8Y!4ut|=GLl)D|DA=FXA)Z
zsC3ZJJ-~NkmQ`!y#J}&|6U@&!HEnhcU**N7Xz_?`v1oIQqI1Ne1A4kE*5*V69nerr
zI<@oduH|R>Oj&$=!l&8fSXP^!+gfOJCa2%$w#CiYJ(kmw|NhiSHJY8~!nN8sXH{Ua
zmzG~;%HO%FzrW9(?8>KPT>U^W_1auJ#)oX{o91)XP1_uIYZJpImJfMzr`3KbeAs?M
zsBxo<;4-m?i;4y3GTdNOXk&c3HuzPKu#J5ySIe#9j*U{H8h+FNpB1a}&61kx^88VX
z;nZ~+XXkqdzBD?vINqi(qCjeK?eQ#Sr)zgE?&XYs^rzy)Ca+x=C100hpZc!+AvFK8
zy;IggKlc3|p<B&Y{(SZBN_7K6+M72^ROO=ISI@QEx=Cs4uY|`jMio&XyFN(>&*Tx;
zVZ4<5T14#B)eydxwW|NZ9^X-ZTY6Gsx=g0wmYrUkrtQ5kbK3en&P_`;pN-zz-=gYy
z$@i^R|D8)phCw=$E1f+GHpzLLpUerp_9b<7;!N+kZ~gbrF1po!qxek9g+pye%M7}n
zw>+B{v)O(sgV;T(ICVDBkQx?K)))KJtINGZ;<A`7&M2GyTi0GNbxL%QW4Or0JO6F>
z>o8Bhq*C<hPc^g5!rk|4cldc-o}G5xH%#^2iRyX(=XdL`%$03eDaEK3dh_DB8wY3Y
zshk`#bKcsSm*4143Kp2>7m(26I`1xH$LAYU(r;Wl^0!aJvwxE)Te#``|8o~4p9%dK
zbMAM+*|@^ng$=V~4k#4;3H^70r_a=V;*b7<HMb|lTxL2Hy|!SoyG~@v-P=Vry>EB2
zE5~{SzUiA4(K=N_Ci7#QXqQ-mg^c6`r*MZmUNhM~243-c=<*|0rfB0#uA`EE9dlg&
z1PIECB+j|SxAS<S#KfPj(_7cvTrr`|PiB7MhldL!``GtIe#%++FR*8VExT0Vxl2c9
z)k~bcTIgc>Kau6y!xjd&cJ~PP>vPX-)Ckkn@MhAw%HK7w;pg=7FK)LNK49H8X?wto
z4&O^V6NGk^uzu=jJso=GnM&iA4cwn=`yX6o**9Z>R@1C4H?OGK^z`Ti_r9H9bRi+`
z-~;Y&r{Y8Jm3)zS9?7ygaPo5v&lg*}JaR6Zt&MYUVV~OaXtFJ<Wo|%-PR!Lcoq-~E
zI(iQ7ca;ncP@JtI^E+U|jNFG}7Z%O5Je9QMeEO3ZdAau4j3IM;8TV~a4Eb`CLurch
zKc!PoTMSNx6-|72(B%40U$!MGQhN82d<?_?`gSd3<#Au?mdxPC<a4KO>v_d_vHm;7
z1-{KUw@UIgNs>`)ck1wC_-_<jmeS6+i!ZoSxk|Ekb?i08NY5z88!9X3T#4!SKdGd4
zmGhHF!>^h5->W$A-u2wavpC#Mjw_TeBKP278_(Vr>E5MJx1D5-GEz2hY&D3yy&)vI
zgkjpw*xP*X9=J+y1}M(GvN6!CGT6e+Tf=Im-;zfa0XGk7w%fdlJCxbm8si#Pam{Yo
zDYc9PS-SB*&$M;!lwo3UkbKp9RFOeIMextw`KJ|C+IO})o>X*_<=Q85Jvc@Fn+ezK
z%q7`1)Bn`A<Z8Mz{SfxDbt_qz<}2_#_(Mz{PYa8L2h$dnf*t%yZLz-Ru6O7xn%Eh)
ztyH;`Dbh7$?u#=^ueGMXWLbKp@lnnHnTsuxf-MiabUOsSt6kc9#QlSZ>b-?CKAZ66
zo$_aBXyy2HGW6`ic{e?rY@a+lsJ?8%*<Be&e{X3rwpv_$MZBl>_L+ctuU7b&1}XDd
zT|OI=ESb9avZ~SxQP~Sq=9zKwRxM%i-qFJJ)qS6?nxDiRQLAMcXJ5Q^_3gEL7P5r%
z;nU>4tfLxtgzx=LZhs*&@!jR#8S}F==JPhr@zZ(YQp@tiG^Bua;#|*=hHI=pWR@>+
zo%M6${O>Pg9&s`h%v<R3NX8?oMYMHEvO(Lj-eV0*y8WI`ba<e`xKv#J=aZrfD_^`_
zIrrI<#8$oyJlrkM1YZeo7L^DW9l5>6R>2~YGs!Ad_5a7HxW7_MKk<Dw=oeWN8U6OT
zhGvHYZ-#cEuYr_Cgwl$zEhn0iSpIH#>b@f)c+Jy)dl$645;Z-RTExM9_RfR9cb>Ss
z2)dT&68}wH^Ic}o2B*m?0k0M3B-U~FNC{mrxU)?owa4z6t<uq}J~vojDmR$)9M<JH
zGWCMP1F^i^V{-)jGQ-#eSZ<}O32J1%D!wY_(zCj`CdPGFE5n344h+}pIIc|o#wS-a
zfANfE4+T;RA1yji*TVEdE&tIXb-^{i?>;(`IqOEwLd8RUM>NEa-SRMdm09GXbh$Ps
zAZe~kP7gy-?8yM1y0v}+tB$C<xl~QNTQo%}(MmvT75mD4i{^hkepfA5`+Sl1E`~WG
z9{is?&M%elNJ&VId#SW{!h;oA@_h}fRAk%YSy&hpf3mQOFfcLbFfcGMG%z_ZGI0E7
zVC9gp*l^}R81r2LnHOCP4>#tDO?sfTz(>+Zlr<^l#Kz>~fpXT9xHJz;Ylz~G%#1pk
zm*CB~CV{WVbAsB**%rlIGS??QKi6ni$|dXd<=L)&6-TZREr*vD%l*W=UTb72Twb8i
z|H-E6>MDkLj+{xXU0;^pT<;uSdc~<I<>Caxn><H4CM4gT!u^zMinQ=z-<B%pQ+t1Z
zOH1f){JJSA`Es(R(VQI~dOik49?i_wu1jJJ7CW3_(Jxza!|;(?lY;w8U)u|Bqy3o7
zI2&{qh#m^l2;`D6?wS_Ww?CO{dStPh&-^tNeO`WIrI(+xtS(xkHCN;E>$t=FY*z*U
zj(i$t+5LaJoSaqa8|R7A;^#hZXZZdu|CjKKDh26f$Bw_ge}4adccbWnjjaqZbKGvl
z<jZ<oN@Fs)60f<SbXj0_%7mg@b;%-bUAo_1lnMFF>N&`#t<UIwr}JOovse%9WiOPv
zGa9E()>yMpT*UwCUzNvwhLcnmM{qw>ZwPbR`XauQyHeswx6i&(ai`v=stzszMMq9e
zW-@h~5gl<&%|TPd&FrQFyVeeQH$kr*0?nC4OC~t*gt<(dDYw(it5)fu$pVLJJr!y9
z=q-*ne3CnyPIm_KZQ;@m|Jx?#xY{7}s>W)M@R!At?}fznJ$?8^!*iOFde$_rwMi=O
z!M^EnljqpJT2(h^+5aUXi`N`Ue;PSK$dxOx{#l87edi{vHx&%CBH8A$6gslboG)l{
z)Zx6+iHVLJTVLMXTG)2Q%SDWH!^MsFlH889{^pEK_t<?+*}<A+>!!|*4+p0vh2Jlo
z@@f@x?yUFwyiMERZ!nn0D%2Ohu>AdAf74Rs5H90A0f%gx!u8#s+1<&Ed|#T9U00E+
zDbk+q#W^|tUX|yCM43p9%L%I74{jxGkLoN6?BopH?9f~%HgkrHqPMiW496E^#{{m_
zW4>&W-fTN8CSG!@H4nB?4BXqta(n*nSuxzclGkR<xo%|2cF$s>+pQ@p&nF(2DrMU6
zw(RyFCLhr|`OQ;a-YrR<_5F4Z_x~^3?&$4%b!T~@uxZSOXSs6>9cO8rTxY9SEfXEs
z!JKDVU%?fU(7-Y&Ol|Y5@18Ta@=m=sv6*)s-^A$x%Q`Q<=7?HR^kCPuOAnpiWgYa}
z^XKD>-^Dh*U#$;5>T#O;aj}U1<4-Fu&iM8zQ#|JC4Z-g#lE2#U@z{Jgw%D}b`(lOl
zHlKVA_2+)vc<h#b*17Wdy6^s><*z1W_}U1Ri*dKud)oigwmfbkDQKN>urF$Zva{UE
zI~7G~5|wY4ds#5{)T-T(n$Egf)MRD1Z<3J^$J-pA>8k_k?>!OB*7aa8KHV8zcXeXi
z%abWGFQt|S1)E9vy$YK0>Gshp%MZ&PI@@j&Gr#h`R+!<b75?$h56*b7X#0_+8A+Wj
zyE#P)4{7)&=BQbIx@E<o!BBRgS?()?vh6R!V^u#E%l$QYTx2uJxah5W<MeK2KZX#U
z6h^ON*_Tc3i&Psj&U!i(GV#vdGevi|&XSC?zs_XdIkD=@)0Jj#J_W^^Eo9yF^g~WE
zcl^P9;&vUjt&<-*$NibrWD}UeQTtTbSu0$VD{#f8sq0HpZ>Mmcj$9=)_1NxI?bGHf
zV<+*~p3YV^D3(0)NtA2)rZZ2qrk*?#do5_@)-_%_@tO>w+s@9qcyQVMB9oQXjG0|l
zii?hAY~p*!GeK_40(ae4f!#*mn0o(Kq}xBTJYRWev-sZ+?)C+0|INcztnrRalC<Y^
zT6b*6v5B2dGBOWmOm;fP;?qBoC+CgFl!cWl8B7N?a&)Y8y274ivPqsyjF7PWwLrjM
zZ%^md0(SxaUPYlb0<KHbw1QRsZE$;faHZeuPe)#KItkm$vIewEg)EZ#y4<7oQQ%aq
zE6YA@S!rAP%5Ui`vt?1GL1C>~f!kMwtWy(R<@9tFLz~sLO<9LJgOYvsFS3}Tz;`Zr
z>!PD#;f@CkCVgC&YLhH={zSO(uhR7yA=7sqW1jTui)&v8Ys^NbFYZjGM_T&tE@p6<
z?s3bd#q0W`=ef&{Se*-c?Y&^FM~lH^lQhdq_9+^RJfawPHs44SiQ;%_=<9j==6|X6
z0h`ZG$os3t7XEA7_EkC}1+gnRABcV5k>s$hX!7YhPp!hIp6z{Cyqr7sRaE${8?||+
z-MjC;suJIMmsP)b_ilj~VVjk=Z|T}M<NH2OP1fbVq=h;;W}GY8Ry_CXne}zAInV!n
z_j&(snKcY7J_(IqnpD=_^-qwH6#wXwX~8^K!I@uavg5^+X3m(Kt){Py^ggWFSk#i!
zHSO4mzK|6YE_n&2vMSA<v#>=ZP~xy~=9JK7pGwvwzFpN<eCF{&HSv`*6Y|vRwAvJ;
zBN^PT9h$g$CZ~#c<(fqqMw5?6r6>mPJbCo!r^%ano~lV(o(c^6G;MXw6Xjyb6;a2E
zrtChGX2SgQ|JjVTO%4sG)^9#{Gr!DlmGH){YjYO0g_Yb*V!WAn-DsKMhR;`C$Dhr3
zabRAc!snM9wp~J*eJ-1pPJFDODY5F-k*Zx9C0Fh=o_cWnqD8jC>8Or|4?8A^uFor7
z;VkPY)%2(}VhPJhElJlC4os8NwuSz>=_0L{JA0{%;-0ftoeDc=$E2G7-f`}l%a-N3
zZ?m+2-?}7rVCz=hw>}$OPu~BwZQF6)yxi*Dx3Alo@3?sOUFq)VyC3|_cRaqESGYg=
z-q&N__in!X#?rrfcTVLt(|+%#n~S!`b1V^DdTnD@5!Z!%Un<n=Uha&qyj46;Y`cZy
zz03zp`cDqXFjm%Ixth69r{ti{|Fiv$|DSIX+B_%uXs2ZQqbDXo9>@CTCzralHU%Dk
z#xhY~=xo58Yn#3<oxq#7v;TjAlVG>e1kp7XWz&vx-&Hi;dE)7@UE8<Y<rkzMe4X(9
z+jZrItsAoyxuO)TZ(W`DZS(fpw|VKi?>zP^-*Nlz+lu+W?|z;4eb@Ke_a)}L_ikU;
zd9d78xV64$!rq`$9*j}zSaQ_k1m9lY_igH@OqmVOSOZ@E=Q{A2>Ej=}_>h+uqN^09
zzw5BqZ#(3_d!IpRNzOUvL&rm^T^zTGbI8v>+G;X&Ta?JNFb}=g%TgZhy1qb>nIqnQ
zg6|fm11yTIB5TgR&7b`H*R}Ke9-jT1yUG0KZ>A2G|C2T={CnR%|IdqlmtWV{|NFN2
z|KE50_J5ws|JOM2|KFGM^=cRP|Nobs`2W@O`oHZdEC<DM4@*D1T(IUr!PljJUqASX
zJ&zaK5kFr%L6)(W*|6$YW25UrJ+TfYA0g9%AZhVs4m>KwRU0$oHYHASu`EB>#AhPl
zs;r>!B(Upgwr^7N%%@`O8U-~~L^MrH)Dl&Xx0SiQD083D;-SIdcA_Qyd5b4UYtW4r
z*BPv?Ct7?mS|cM`0~1=~I9TIuw7N>LCaq|V)nJHZXp7}wJwB~1Wky?MLtD;{wp@$$
zJdO5JiS`1G_Nt8b9FDfE9qoxbS|6$lxQc}sZIwCg7|yEVHs$~FjH;>rjinjg-!kf0
zGno!HN_k|?3#$@UaaDd;)OJPhOn4XbK~9EA*;d!xnHIXQSY}xhBC<|RMCn-Mbj2uD
z2dTZ+vd=cvJT8jzF%mkUk@IF!m7X#yQ(?~U@|;)4dr~Uu&Pnt>{N6M1LtDy?)~gl0
z*Jt$J@@Nyh(0h-gFHnHd$D;50kKP9!eJ@w^7I*aBI?+-r!J1<saGitIM}qO=jlQ2R
zdW$3aZfXc5ZOd7>MTUR5U&5D$zoGs=miV{36@D!7|NcZ!?s%d6b-~#)8kbcF%iQp=
zFt#>$F81$;hst4bRpUgJjjHn{n%6WW%`Ox-J8s!}v3R?=$VLvy34&46{y#1fR1A`v
zC?$PRWJyEG+3>*4hEt?339c>3IoZ(~?jhi{Q@~$xYHmksVMS}0Cc}w{mU@l;m`;|M
zom1l^rv)lZOR=1m7C9}Wa#~jBG^PX7;&x80_vp`^&U!M{rf<7OlUikOTIJO1(`!yn
zpJ5?C??k6eU}oD*!RC{d9Xn^VS_*VZ&fuOnLn@+ZQgFn@>GGQ=O_c2HGFYi_iph#;
zLiKtLfkh|P7as0XxmdkdJhC&Tn|q<amT6X-jtTD3U@(d-vAC)7K~!i{VvqVkF18;%
zVK;i8ddxW=F~|F7&nt_*ryhMwj(ry_1uk;V4G>`TaG0Ch*?(>4+^dmuuUF2!yK?Tg
z{~as>ALc&gWPQRp@7m32PjAliU|<bkV0@@4@aTE%xil|s^Z4gm{DjRLg*YaD4Qc#r
zA|SnUBFn~!;wJ^!v_v>}RdR1!z$H~B;3X&baQ5Sm%`=Oe4!nrknwZF5sCw&?VEe)*
zgN>33uLN~}n#642ocA(wk9pSqwul*yCB7RMavz**Y1qw@8YI0)+Brfx=))8%#U)=(
zw0IZ@xMwXnIb%uk&Lw_Q({d%322?Ex=~@zYO3+hkDR<*Mro{eeuGZA>6yK>Gg{B#0
zPc!OfF6-}DHsggL@2Z&vr&Kzf7X2!86fx^kkXSx-V^&JgtQ9L2cC&Y{l<c0&s=Kyr
zk&K|aMw;%G|3Oh(b_z&56<ahzNqJ$^Vh1*k#GV75Q@twZutoH-Je<R=xN6ZX;ka3I
zKUOYT+13A4lVSC&Ri8K+nF8i<7_5GIa@D2G{+m1U_8yhyvT*o)(obNeKX<185v~RN
zorQvxjlW$cI)7Xyv{G0}y{Ib7@>LSQdWJE#qxf^70JUjGf^5Q0JJ&wwYF@Kwk-^Tc
z1)M9Fxe0C0T(K(5X!0ugT!Z!IQi5!%Q@$1|dZq`>p3r60s3ZMo_F91@<{wvj&)lFg
zacStPRY6e_OTGHPX|&~;rx-u0*n5zRO~mcflx6iVH`Zy&N!?h`^m2yc#2Ni7XSCm3
zrj@-(XZNPApPTe%3u|c08~^X_RFR&kF1hZyqj2_;l0OsIYAs#9(Oqy+<rYQ7S?7%u
ze5?g(omR+foR!H`BY06@X{uGn5d{uLwbF~a5(ingcIs`_klQ5KVt#S<>qRQJrm<ZA
zA-2eC33K4w*jKB#CoWaIxbpUo`tOxKKPA_^W@+R+IpOD)_`jx|f2Vdbe~ZtU(s|+4
zg7aPr#9r>Wti`~-ZNe1miL6{Zr%&D~cx$Jk;lk#t3pg5u>}M%$FH}9sCM+Pf=)bVI
zf@f33FY&eELHaw?49b^VGz-XGTU2yqOUk7ZS52Md=ym41#E(i#m>C+^E?Ru)llry=
z(q2VnPEzeMhEtg)wq3ow^hWin^oqU0*{vS`C6*~}vXS5}Wm_`6A+>VSkBx7#-3)(M
zN}Lo_mfmT_xY;6bvr6KA^XRqf!*?+?uHBoo?M2fR$z!|z%ob*N9kJDX*18wPEQ!^1
zud>cWuV4+5T6tJ-<1>M+-&R<+XP-@4ul!8r0hbU*Ab+zFo0sR_h!=wAPR`-F-FEgy
z>D6P>`%X2yc+&8rb%KQXj-Rdmr+ytiDdjJ;a)<cM9ipAWXKx8!k=pORN8q=K{JszS
zSOPO&o3z|Gr8lu`#=Zyp`c^M@J0mZ>($VRNywlFaz{BEeHVCTDXwqUg4w$ZbFln*b
zExj3mT;}PPMlZ9@8?j0s&{=;t@ZUibr;oezw@ax92NnC6Zu_6m!?BU`xTk>DnnN;*
zX;BKvWh*-hmu_s?HeKQNzM7v$*D#%GS-IKh^d_c-+qA8f{=YW5Sv5n``e@LZBP<g)
z_eSn_`grR7>`l%V5r-zuJX(FAQ#8xGT*0F{%PV`uHuHl%?P_a05Ad`I@Gz;}byASi
zVb~FNR+{n9qk~~`joYSQ<I`w76q%5F_1fx%Pb!#9;w^O62)vlUyE2}^s&h*B4!P$G
z1ZFJ|&Z?SMy<o!chW^#(6&E(LWpT5n@HfBQr<QnhT>zK!3;ulx7rsrn@b!Qq|KE%c
z+WY_XT$K8|KwLva*lPdpJB3A85{eursmWUEZLfOWD1LB~lT-FVWsT<eO}kh9j}UpU
z)!oXq^5v~rmMerGZ<@T>`dF~C`M=W@&#W_~yO+(obFy}4xZjy03bhwq8c(+_Y+Pzy
z)o+=(MCSD3KgT$}WeCP{Ej_@+vQ}EQNHwF^QO@b=0UJ3nuXW5k&T|BHB(Ck+E`27t
zKr!I7PUEWsD=LNAKW%B4ds%McLF?0k9V}`w6S^IP&sw+7_EC~v+~A;dtV%W2X!cXX
z*ClJ_$R3ewU&Fr4UvA@(>B^@R9dEJEl76$qPkEColLPlU6SuC;TMKpff1a4JFnz|q
z1a32{i*FkQwj~@=e#rOw3;$L}uC!eK8(nvDeXE!RjxIl=^2Eu=`jjA(>LQVrfPypP
zF-*os{yR#P{yn{Qqh$Za%eKmLw#OOI+&=#5l*-pt+RGkJp6+SkV5!JyBf_=mgoh*l
zyfas)2wvUnv)|<QreEBrRk=@FI`Xfckebfkp0I-RuL0Mm3Js&zot=^VZ#HzkNwAUo
zbV)Yz4$}j^_)q)-x%^8WaD8)lr1Fk`zb_Z(Gr=6+)6u;TjO{k>ICE|3buLa@0p}T(
zC$se)cNwYOyrZ}=VzS_kg+H%1dI?sa6|7!+X7NFfa>bg>Vv8jw-k8m}jY(h|^TQjH
ztn?o?Iw&h1*dcr;OImoVK*HHwJ36yBv2Q!d|MvnXujMy^qf?`=nk!zA>D?i@_j#S3
z{9gy|i46?=dFOb|6W06}nEzTJbJ7jJiRKY|3mjHIP~5tJGthub&WOun0c+G>{<jYv
z#p)zXf5yK(LAvxF|NaYnsSo(JUx?cJ;kLTquAe%mx14)kzi#LDyw^<2o-4kc@$Z4&
zk{d^VZS>%HbV;UpkK^B>Rm<O~FIre&)AYobO(*K|-r2KGt5@q^Vza7PFEwegU$jW`
zr^Sm8@QK}B5SF_s@XwxY4boc~?lRR)%z1lf>z=FgG}vbQz4OX-dsC3JS|LGt9&4gR
z^w$9XH4mn~6D(zuubQTJ)!OfI&XMQa4sh}&a))2yJ9swS<(|rvyC#)h%cuWzlL_8!
zv^=7+*L|s5^@7T`zdk;Zanx(`6}hbxRrg<3NW^im{-hxGRJnDI(lHAj7x4=3HC(WG
z?TkR3=idsrw<WNzTks`s!x!&-?g0Ob`tO!W&hY<rals1R%?f*WupHq^R^j3h;LMC$
zBNombRBr!CB6^|{+e87z-eBL^@Aj?!&>r<5Jc@yJ@qwev58o|}V|!~LzcP-iM33QH
z0M~A%M{}NENc+;bMfZV+%!8o4mCJg+I&C~sQ(tBIbFE6CG4D1<gBijFBA=x$&SZ?9
zwIor}X`{NiuN3p9<F@6$)+n8DVcfDZG-%!GOXk0t%R|&xatK9V;L6`CBerip(?jm9
z8{Tbu@Fg#R%{Ts4*@DM*{EsCq+!9P2_|i{J<X-!{?!o*2-XDrY-AoOWbMjlWHgb7P
zVD0vMecvqe%>q^r1*O`lF9Hj~-b`Tia1fsM&dcM%`!so0hK;A@Gc+-Ii0W7rEMTbq
z#<HU*;)2+ru(d3IHh*7#&#8rzwSeVjpo;rMW&3$5D+HdNnxJW`^hRAl?PQ-x^j7W2
z3BHa-vYx$4#-|h<gV_1YQl<uH1}*iSYu6ez>5Q0O$WD<xvnG2T*JRoy^hYZp``U`s
zrDa<@0vl#Gtz+3z`6J9BYGNwO6b0=Dmvi&d{%YMdnX=+?*tS=SYfQ^U*4FLiI9BuK
z#&m{wzPC3wrXHUv9iD$}mS*s=o|We7z4qUHvntG`X^mU2kgD6{X+Dwvm2UP4WnO)F
zSV{4;4#&-SpGQouE=C>KvfO>-!1c_XvYOkcm7aBXbP#gA{5q7WYHhcY!|B&I7R~zp
zTAj;r$&uBYHcR*2i{Bgg{x|2<kkxV1G`8H}>^QQYgXzy%xo)*tIYy74X|I@jl1*#r
z+)k+_-HOq-GWm9Fsxn@;=Kj1#0t*GWr8X)b_fR=u*!t?@QTh0)%^GekeZO9>TvFwg
zr9QDlN5h?QhSMoc1??-E>I%;%c{c=2QArlo`?Si?t7+Sdjy1X#3nrafcc@87(Qj7M
zY5g5<j8^jmoyg%@vDr(=OGWFh%Zl|)FTB=RUT1mrcKd{yl}D$AnQT7HYksQWFpv2z
zjm7`DjYUkEx0**4K0Xm;a*~Pb-zEp9E)BPvc_Lx|v?r=mRYa$X26~;{AQ!l$!-?BF
zT9nJycTq<YkH5R1b8ui%lIvxkFSGe=qZT+Yb$Txrc)Y*q&gn<{0~a(f-DtTc$k4gu
z+wCpkj4wXAi@bd$+#JoPduZkMX?(X{ELJ_8dg4Hs*UA-QVNna+y{g~4b}U;Km2`4m
zXP;7$K@*d6kY-RtPOIBernntC3jIgISLrU((pa;9SI8@0MiVy;o&Loo+RF_l&)J;9
zctIsY-!k0J>s-gwqMw>gyt7t_%@O#tWgb8GsyAZw#@@acoO}(M70kUfnKSh!`R(fM
zYO?N|>z|#d{+~<b$c4!CFRxan%HD93Y@b?^Evee=lVNPl6+d|wKa)nwC;x`RjGq}Y
z+j}RkpJ!+yyY%&kV>dFg>tAsCbC><Oyk7hZM-+Qh*sny^PL2n^H-u{(3!EbwW!&i`
zay0wok+<7L<6js^{dK*&;iQ&~M^;(X!&~!z=3LX?_%QC<+9mwnB`X)#u?0=s)E^_W
zZ<AQ>_T`)6gT8M!2%nR;cnYt@G20W%B-A?G_jx@sd?sJo5|OaPO>f1Nl?Gx8`g|Ll
zG*+uy|Jt<RE9a^1sugB4guH*PUK8hK^dL$|TI8DGf-Rl~hRH_{vON<I3V7okT_NQ(
z<CAc$=aI^71<nh&wmP=lc=TU<-G)}p8%(nnOSY*<J0{v5THIN;;DY=ciKDqus)r3V
zR@>D?yW1{VwoT`SyU4L9ckbuLE>#(foKXVef=3Hnp0&7(@F)d}C`{l~+_gbr>xpw3
zaWhyMdmL5HH#Obp36@{|_Lzm|!9Iz=BIEFc6{4TASha-~3mz^xAr}yH!R_6PLnSkx
z80t5kIv2BO(!Gr=6GU~Mv}S!gR(ZBtgfrWjd8b30r(aG|&<!TdtTwSZo$D3celAIH
zX|kPMxT#*ydj8)%hxRXv$Vpzm<WktKMW_CKb@5X!_CCRNNNj7*8NC#(#<&*G2?@WR
zOv!0F#gLvdgD29xN2ckFZj_<axeck!Y5xzXInGJr7FX@he!HWy?!xEUVtt$C+*}?#
zdNEO3cg-TPFAjc7d9ItOFyvVre(5LD#384_@gRgrAl=@fz2Q&I3p<vDu7a!$rZ)3F
zcQk6J7jkS5nA_`=`!GxK=9*&{`Zq8K2R58{f2G(K*1DxR#yRtCgv-qPnZAnqn^!&(
zG54G0<~c#C$W#B)*F`L<>t`STtobSD;GTuI4vDU72+>}2++@kbg*-727{sj<LRa~1
zWscpT6|zu^;k{tXl<%Je9Sc+%=Gv_BSo<>2v*F0%4}Vuo$>LOXJo9|X>MLJo_+E^d
zw!vxIPv1_3+FqfbTuL|k7U*a#;kX>Q<@(7L5*s!aa-I1f_$+h-&rY_9-wIe4acsRQ
zsMV!VmU;A3+YCXwun88kHiWdzP@OQXsW0O0%d5-Ub_n*XrHH1k7Fm15psT9qOUE6h
z1siV!^80cq2&+2fiWoha?Ijzomby7a&102W;DO^Y8w6eR#Y-$+_qp_Mo}26CBq}5L
z_u`%a$0v$cqdnSxWKP^#ky3NAK+P|pNbB3&Czn3`4*B<L`~LklGwWFm8g+eED6=i|
zX8M?VZN-J72Qx)aPADjkUvtJtdy9=)+rdX7eMfsX$o?z{j@c%@Zqf`M!`H7g!=Ih(
zJL5C?TZEpY+eWr~ohFlfi{~-yN?_7gU@>;x`}mqz_yl29m;6Jaa~MqiH!?6@cx13+
z(Ug+^!KNB(l+^Yz&Ye`cEO^z-Q=xN}qqFT?x-a~Fb>i$IPRXgReluVA+I-F8u?{no
ze)&wt&gEJZcQDK4nFm~!A6AS_HZe)zeR{Lp;_G8iITf1)X1iveW-bc6`><_^TmH(0
zg{$foTU}l8D0P+U0WEQdzb^y&f9b_)aIAXIWE^aCElyyvaKD(gDqHvqj|a^gqS;@E
zdKEPa*KEDMe(|K~t5Vh|MBg|N{$cvQCXV0wisw!SOLl}cUwGlMIH8$QmU~I_i;T4P
zr8DBSRrI%9dg6T6dDZ#6LZcA#Q=;Vst+HH)m9$@Fs<Lj>5Sg=Qt*V=%@V!e-YF<}w
z|G)Y5bEj00hLAYR<%3s`7yNE?Qi%v$uXcXbDd~^X_PK8A^4c2O6sE?(_1m$3;{DD2
zit<Umetqj_+bwO<YTWtZ&hKMgN6aTZQ+BTX+#T=r@A`iG`YeODP16_7&$)UerquAg
z*ERXZA~#P<j^_Jb607w0DzDTzsbjq;e1BQb&G61t$8DcP!tb5d3vVp*D~Y_Z`^i%E
zYQvm4Yu??EvYk}E#30#|$@WN|-YI5QLy`1F(<N-%ZCv-<+_9|bh3wak<H27zR@A&W
z=(<MHO;~^<;L3+GnZSg3hFdmN9Eg1S{w`0u=q!(E3QM<rnjpM<nMted!e-};dtaBG
z=s0(B-j@i;n~ML>e|PyR&SB2HGDv6ZN=Nb1(5e4lKIz}Zy`ar*)zbCJue|p&v(D6V
zTzB4%F=FfQptp5DE^n`0y#1D9$@lwL@6LMEady_xwT)$8W__{y61i7J>fh^W7H$83
zHCt$<ChTARd%p3+jr(^@-26fA&*w?ov-nm@woY@l6ffckcIa?5WOUMG-R7acG=g<&
z;NlY>)fKZe6^;1xtk&5q?ElU%SLXzip~Y;C2mO~`uf1(G+oyuJ>2YtIf~6yuzJt*E
zL(M$`J2g3Sx)g0x<a#<9cL>;TY<2UJ@VK<|_o^lHZtE}D?38J2weU5^QtoAXIvn~e
zhZhSww*>6!xXHq()yY_7#o5&LfBmJcO^t_Hte5?+64ukO{&sM~zt7#)d&HxYtd}}<
z@Ehwa5!5~qHCc6|eabA!FTxG3N8|!-I?VmZcWOhsvms+sg`G>IIM2$B%P!3E6<prB
zY0A+R;$;%dfeF&>r<l5`d9qHj>?-8F$+5#Qf?Yj&&b20+>AN=l*x|auk$-Mhm;LM`
zGcIT<t>Nsin2`5sul28$?#{9v-m(uDZ&{qE^-qcUnX+k)$kw38$6jkYf1JIqPr<(B
zgk7Nux9t_@!ZVz9HIm<6Yeib?d>5W%az-NlFxP>i`I8J>_IJ!ru3ow++KiFoMC$A&
z&KvVfj>zixESng(%>0i^zqfX!NaxItixxR>{MR}t7iQ3(9MZ+9BCK!G+_a&&><8;L
zW6nt{bORUcI2g$)*~Awn$gI6uBe23@VgjF?*75_K>n}MitTFWDzc{zL(sHISgKN?f
z^({M%Z|>Ai@mjQc^`d4k!zWHle;k`W^VEtJr@Bo%HaA!;+Tg))c$=AL7yIW`E4`0w
zTEM0IuxoAe($JvO-&W4pd_;TV4z67V+}k#EOkBvlyMUV^i`j=;O-R_;ev0+Z9G9w`
z6PBD4w<NG0(A>ip*mcT6<bd_8un)@$we<NkCwej(uW;nMv0-X1qucux?K5ZXz0qvD
zw($(#MB5{;k9^je_#o0`UeUx`fA-$*Hi<S^l^L??k@Ktz9K8SkUOM(-^D3X7eUGQu
z1+2Dvb@<5Z;&W>X{67A+lU3ed7-IVB&irRfPER_Z^@e+Y;TOLpkNy0(BsxtTmI-&x
zF!X<2=%Lxjy<D&%{<QYtIj1!rG_2p@zy82{-5E@OJk6R8$p1ab+46;(=_%)q0_SQ@
zhj8T%{h5+VlBdHoR;>8Y-c!J8QpxPZxpsaJSJZ^XlUFa&^4e)5v}B%+$&3@ipR}Ch
zbvWx7J6jiYp4`xCx6txO(-Py8CR$elmwmC+`LlGX<*w(y4=qu?Wbo$DnjK!NmiQlT
z@tDssKX<j&eudLvkKK)KOxk2{p}T~utc8o!XTFum>DXk6SgYM@eCF4>xcvTY_J3m)
z7xOKa(>;ca2CkDA1%y9n(Q^<9?vT4Nv;X$x-E%I?QVO1U?1Zr=58v?(+vjl1`+Q1V
z%=ehP?>vuOlPe!*dI;MsU+A_ZaMpzb-2UGCa_$^kEwwd(_o~kxtygDUo~azyv%LCF
zq_gnMB~{r=D;WIW1#`dt(S2mj@#PK2WgWZQrmT@Pl+=s3Tz%7h#}s+py<B>0XROsZ
zF{wg6t^0g(R&zwH3diD!W}b#_B0PDeUCk4OwJt8vo;CT8z@g<8^J2AFmjnrjmR_H3
z9Oz`#$-8I!SF2?w8w?k>EdH@8a9;E&xm+&=)m@?8r&N3|Ejw|lMVH&y_e87hwdB~z
zCW&0kK@$HNFJ0w0ZEB|D?H^;Mz13MWP&NEh&}!-K9aFejTqo`6;oco{bF$*(eF0X^
zZ@Q;V49!2=X}(v|YUbrKSB?aoiQ+3c%zgx%aS=Ql(&cw&=B?~2GiG18J>`l=@0R&D
z&w73KUB>C>FMD;hQ0N+?bL(9FWQ+GbJ#r-E^1crt$HR3y*Yx;3+v1n8)W2<NSlFwM
z*FLwGRh%tLIUoMHdx=A&+QP24Q$ja+o3Z|hJ{;n))Ir!-vM1-XT%Z<r(1#sKstd$~
zmlpc2UUGrO^i-$N+JN2%dnZ4ht|57<HuK`786B!RJzWJV8a9UWX3wa98u&lRl)+nM
z$!stEEq9fso?;Z`+`)V2|NXb~mv?Sl@q1f$uY~2D8%qyNf3%n@?dqj<+5xK<FD)rC
z_|&=k)9*E#ojYDX)oS*z)|}{}@3icDw##W9&fqIb`yH+vlTdeAD=02H@1FL&1yxZQ
zI$C#S?7TK_d92O4NZWL+)K!LOQO}$A1-*3&4ffmZkkGOq`c?C}XEh01wqA`8?POz%
zTOAy|B4Yjy?+Y4{=jTi~&UWaA#g&MTMt{E>OnZaOp4)n-Fn6#0;?5v@;rq+uOO-Fo
z>riR?Xpw59y(8n&vK1#9HpVnKb)2!ie&?m^JqeEF5=$}D&dW=8S~#7OK7CR1ioO<0
zM1!xq?vqE6ckjA<i8Wet``*__FV3Fh-e&(_&)~#_h284f9j04P9OcsfqSMLe7x(+O
zsaV|c&|CAjo7@bZ<^SU8{EfogxkkIp6y_Uda$B%$*!`mWYvlX`Z|=SQ8++1MnDId8
z?3aT2SrZ!{M6Q_F`9Sd4lo#zcm&7k$+jGZ6>+A#H$H}vNUSD|{ye~rS0pqu**WRJw
zPh(%!biHD6f4kX^O)t9W%(<<5{npLB9VL6%=-3g1yskYOYTR;Zab;Z@a_2Q1O>Ija
zTh2+@xuJVvq~W*b8S;LIlZv}C{_f)GKK<_vN3+R^uZ7Q>8Xl%Dejt^ZvV4cC*4c=6
zo~u_Bdw3t)?(k*x38M|>I_s{t#8<Y)$4?NDid9+SGvnIYL;q#ZKDxoWK~Hbj9n(vx
zV#!HPmx^sv)7>6VWQ+Tub81e2u$-a}t5Juh>{5eN3G2KWeFpOn)pCV0y<Bl&M!(_X
z<jG88@0gfMot<<xP707reEW=lovK*!5#uAB=AX2>H%xsz^S-C-st`ZfyI0PJS!mr`
zb8M+*bA;TvJd+1`CZTVS_-zag&7AhE%8WZx>>1a*2gymVv)yjAn)*!q_vmFv7iXbi
zjc3f_WT}H{Zw|`k?0CW1JtK1ct{Yr|s}6;3GZtfD%$;vj6R~`T{5-dgl;*^y7jNE{
zYztl$W@IKIzbv`(N!VjkGmD3vOfruq33ds5e6q;r?)Rc4GG(iTqrK{nCjIaB+qY2o
zhK@+y$2HFkpJlAL@cQ-MxYnZyn-9!5Ew`=u;Eb35_HQ!Kk<@(QJ42y4;0_y;>SBY*
z%^dlQqbJSLZe4nDr;*3|uVMMSJ%V>MIoJJ(m|Lg1f6=`Di;wp&koz-}NpnWS{JO5S
z64pVlq9@(lx<gd$U`p`1M{zs1M4Bl!``o(`HiPTM#gNsmc7BUj`7$T0`gCQ@qSWxE
z(FQ(OQ@^D?@=FgjPTMAXdn51hovP2`_KByOb#LR{pTuOpLf}%$x`e&kvUl`-+`$%|
zR(5;wW^LvFtMl|aw`LbC{V_>>{<U9|KF#Sa(rPc#Ob_1uqH~$%zR97hKX6W-buZ^#
z^`tp{GyebFBh(!!|6gW<+561Z)^}38y3G=`>&mz`w%lQ@iZf`ucYPX1kVm5CltW$L
zb~f%!H#vBB@4FX@wl^MsD}1b$snq|)^Z(uIII{}Fu9m=jE%*0xvW3#B?}pD>@@3tj
zFB7*dukrfz@uJ^zmald1H4@|gJu!&B`+36B(we~2<{$H$Zj|nfy=EM@+jZYN(}UgT
zvvT8Yy%$z|SjNG_so7~Xfr(FnS;%9b-{%kSuG|tcHw`r{R#|@fr{GrOd#gSg-~Lya
zy{Wfrn_GFP?~z^8UO!6v?%pq<P~TN}U+eAFIIl<V*jDM+sZZAK_pko5Rb9LyKKT7=
zi;Bg{-N|NFp~|n=md^hlCZ6N8W<o;h8~;gP`!2|rxIH&tnDdkI@|x(}^$m&5JSQfJ
zb82$jv6%1Oy<M(V@%ik1#ap^0nXT_GQfW<Le%BLLa#*+HNN!Qmt>bR{vqh$VagG17
z!2Q=N$zQR3(adkll5L+yw{yws<{Y{g_vP98Fm|iZ^If6;FK*!9utT|9YFX8{hgD14
zCDXnJg`diw5WAH3N9O;qrlu9uYZK<Lzo0se_nXODfu*;(S;aYzEGk*ZaLsb&gyq-N
z%KjaTTB=nPfBXHFA19a9EKc8cVMoEK|4Vs%dN(Zn{&w-Z_soA!eE+IpS*Y48kumK}
z$ZfmWbDgV~eBZhCSy4HcRDJnJp6|MBng3Vq>YBG<weImx>fgU;=70S)>veO={FlLX
zYmVQW^n!)4;@^w#NiXAtHrN07F0)2)CYRRCdCJqT{gS@<>)ftCQ{>Cq(i=-YFAJ!Q
z6W{If<?p5+^P3ia{&|05;DxJP8~-!q1UHt=3}oPlW?~Z(NmzYlnr148o=(Ifg(DrT
z;z1%8Hh4R?iLp;Be0=Kh+@L(Qdznww7B{r>t8U79`APN645Rc*OKvVzJ2%HFGwjXI
zi>Z=&Of5y6R$CGSd+m(V_UsH^7SQcvshh)Ws^zoBpWUzWR%pYN#t3x{M(YV{8rJJ9
zc9T81ajEt+)h&|*9h`L78B1U1UA6Q&ps}xt{a>r6m&5A+Q9`$UZXTGv^6VaG)nd(y
zAC;fAPExNb(KcFlFQ`msl4kHK=ckQ+zc<CLEjG=$CKxNcWNp-<T*jT{kKfr=r@iZl
z`NSp~)i$~CN!3*D2UlVWKCd{pbAqE^?F;|4Ia|AU^Db6uPGPndRPncaR>;_{J8jdg
zWq%KZz1o?RxYJSh+Dd`f{Q4bR_c6$<FufoqU6A@9lIe<hBLl}Fk!udAjxU7RLrxrv
z^{$zy6z`bBae&M5wWOM$rUKs)uSAoz<q?T8UQ;e4tM60{l6<r=McI9BX~-#UdCw<-
zF^;+=-kQveHv?kQ1h!0a@v%~ocJa$%c{)YlVaAjK<6O6=Q9_9~E=~Jy)cO2!wB=KF
z;SzsK^=Z>tYHwaD<QB~ob$RH^xWGG>cZGsu#k-SI^H@ADPHpB9OIg7n{AcDg?*)P)
zPZl#-oC@Tuu@!mdFll0w@^lMXH6`!0H-h?;^tJ^lgsLT8i=0@-`eM~&-oRrM`(;#A
zH7A$PGz%(Zo%$@JdflFfT$5#gJWq2u#+R~Y6N_Q>?R?F^m$AM<Vpnv0P6csa%)90#
zJ#l-H?#oLv(_bB94ZIP0oVDn!+4ptD+54t0i+B;%sZ$<!cguS>-kItS6aOFcp0H=F
zP)dP|>pIV4%VzV1q-d@!D2jLauIf^_z>3r3!hANV6AC9fyJRl6czFa|+bDKu=jGY|
z_X(XSaM3=d{-{c>?Zl+cX3h<qdj)fYCWK~gdz9qJ(VBMJJ@Dqk$1avpS0?r@yZgwi
zFv4;B1o!O;k*9Q}cTOq^$lbBXbW=cSmD#P#)puV8{r$%hc)phNTG7nx>34$yLpU9t
zACA#Eb!lGg#k=D3>a;i%qg2y1C!S(5snd^P-Wu5E74za$z$`v~k@Dr5Zcl!Ccj|`8
zYbGg&xn=S*zj4#%p7`OWQ(DcHqf1wv<TEH-(CO~v%*|<fsVB8ZS5%zsvfYo3-O1_O
zy>I7n3+G?lwCCGr;jiK$aWA;8widm9wdc?69J3|cy^Hs|%nqJgz;Ez-m#9j9?uY5&
zlMlVx8+uIaKc}e3v!n4Bucx{En<mC$x=gi0NWmpQt&!tyueN`u)TdKh6g!{vsEM|4
zNJm^*KBI@pCFh5vqs0-$qZ}<f!4k7{pD@c`;7H9``|8ZspyZ|287vhk3&fYmg$b?M
zc}R?x(bH{$xZbpcu1dF%xK9Vt&5IlLn{p<d`g<pA``Nj=%FkAFhpqa)Znnin-%r&_
z0ePwG=e{`E@%7~?mP1AhH!SE;m1OR6>Dn@3M}%vi{mj#`QzA4!tZgsO6c&8J#-t_0
zxg_NMYz9r6L;DsTakbOsiHq?{)Q(=bOm$Mq`DmSBwp%UkDitrA-dKH}7<4AJQu|=Q
z<{nd*@0Q)(H>9p_isD@NUrn}9VfC7m+S@0|Cuj0ozf|hl)8wi5A(TZa`OQ=PUP<9k
z4n7P|I;RIWNuDj2)Jb5;o^O1=bl$Ti>Ap9#JEC4~ZryW-J-Mc+)8g<GbM}qJDONKD
zm{~VCus5zR>_5%9{DImuvxcsGLtzIt_CsxfGRM*waswTiEQNzAI6|T&q`Xfku(oj?
z;l1*ZiRbI5t&*&oS`k^Zf{dzWPF#?!BCR_~B3otgqJ)m%*fpv?$y&inbDpT}ljaew
zzBWb6F|F%_5_3aR^VAxrOw;DaeS12DM2z-?C}lY+`0m);>Nq!;t8G)(qFXPn%-z!D
z_rdYZL8D2>|HZtWwNzTgAyLYFOWS`BLzP($+;1;%32m96d`0ups&(O9#sPi|>sjQ|
zbQVqO^H|NYCHq87K(J->+18C41J7zsjgY>7<cW9GlS`@cr*%)onXQ`4nk=O1BvY*_
zJmbKTlhZr2A`L~C2=i`S_UnuD%`ZLab%H;WWTZD~u8LURIZ0C>T;%9RA4$GBEG{C8
zI9w(?xzy6j8kyku;-<sgEeHGcE_2`Kk($UoFC*W8ajui&iAi5O#0@_^oZw)g)VfIF
z?4k(a2G`d~QcGSgE6>}JKY3D-kfLj<UIbTHz4qn!jVu)%t73B{4NARdJU9`}yDrIl
zcZ%5@k%^u9f|G8yY!aWtmv&g{-@Veq--SL$&QSaBS?d|hUX&#Ixn*g}8<B=9Cl+W%
zZ0(Jz<qT0gK1pfjAuSKijkE21j?T<$71_4IU01<MAbdg+Q?!6wQb?1*u6^64NCk?C
zZ|Iq&n;zK~8|LZNH$8&svQWNGrr3<cLn?=lZ3<qgQ#>VZlK0En)27_tR=YeqGfPjn
zb<4h(x4#3r*IzXgt$Udq|JS1MMXq1!tlMke?(b@Q(Hv-%_~v%5(B;x;ni^fqkN@00
z&XKf$wNodBQz_EhN6nv6qv0yAj-g+GxUkLCJ_WT`2~n(F0q@p5Z@SZaLOOX9r>b|#
z-jsy7)s~;)8GG0cZxh*|`!_g&XYEeapjVmu4U#T13-7!gZTSCH$wUr^xgjkY(HB1K
zbX@aTuwD9s^p(@L_Pdjxtxa40sd;Pc-L)|{ZAukFbo0U{S0_(fu%_Fk`^qHIm-3I+
z+^&1I?x_Tq(!Ec+mWaNscrCW#&irFsJ6Pt)oSM-d<ax{V{n<4=W~Y7|Pq8UF!t?MT
zhq7zsq*JG6tDak&<7iZEHn&{ssNhdEqnXMjTYt(NoWy5REO(dn<}sGU8zoh_v5&q4
z6}-35H7-rwrxa_hxBc~svUSCrMVrF5s~k7|`Jz?#(6yOQa^8x_oH*a)`|&*Irw58t
zoN9kmp9*@%@kBeh;MPy8&(ddC7|ZNz*7QqiJ9lo%?R)p)cYl<<wcGIGVo%3s^Zzea
z@|$&H``wax9~zf8e_MNEJx`iK|KoTepMUc<D;!*Q^4Y}`D@>YLb(!7s98dhav)SCH
z#$%^<+@U3_rLW!El^LAjct`F~mCF&)r2!%hi<{C;@EtGOaA}h3jIi{!6LYfGFS^>(
z^Mp&@{gR2}ONO@+`sp1uucIpC_WL|r^W5V6rH=J;{_9Mi=X34BHt$*YIP(lIn`%ml
zA2?Prr8C<!D%*FKw~bWi9+o3B<kdskBbG!r8FVywgs*7jDXxuW;%qoMrFE6#^aT$m
zKmR+Ye6G0TkI8GlMkTU{xIF3UuIOMtc`W`$Y)uR6R?XIK!D+5<rr#C`-aqA_Y(@Ot
z4-;H@T|8X=w<w9Wez<Y*u*AKSdyk!a7;`pOaZAFqJ)Ut-0_9}xPPs48`oHI)w+LVM
z!CUnWVS5gWRLnTYs5UvRF?xX_|7*vVo2&t^eH&X;_#zj$+8kuz+%}6vh2bWzjIR%u
zv&*!pd${@}<_rEk6R!~OvgP2>SQ(c)?$2}CPG7v5a^U)p)cMjA4yN4+P`k>Z9Ld7|
z#v^4a6YtOKE^Ay=1ShMV>~vq_Vi!40)U5y3gZX-OaXt@~GIV3V>72h{JI|YAvdfRt
zpF-~WOlf40>e*wV(CvBG+ajUjgP3^0!H67zW{Zxf2XWpSe9WGVB{yd%Xf?j$b+L+D
zl=EXo=SN@OQ!E{SZuxxpf9CaBmWvYvl1g0f>pl{h%5?P#$EA?#Pi}<wN%*`zFr8n>
z#jW(<bq!C)9jxv*94p@RYI>dOwYWI<;6xvb8$LNH%jWnB&h?e>bM)MHPjahV2Cp!e
zQ~&PDDIJ=@=Xs;Tj<o#xI6a`?z=8*Ny(^L(TVy%r&9Y{Ev`Y5^(?nTCqb~0cXXkJ*
zxCBM@>U4g!WjVc*Gnn_LkE@c4$0C<4O|~ZYSNb&G=sCiB@hb0C4%ZMBw`;D-rEaee
z@M)YA)LPaueJj7~lL@+#J^Xe-2@@xUMckbHpylMlnch6heoF?<%5~jp+p4wEbN<)k
z|IfPozj`FaZ=UZyUgN$vqibR&X_JdR{y)AY(UTp>y6Z?Z`=yo^mZco?c(NBgUbS`B
zyMt-o8Voj$N2D&ru;d=(jbtrn73#WkYpo}rZ^JoTHTjEwdM1{p)=punz1UIx!y{8C
zQBsEGRZm+r!;(1`UG5)vgBYGo`g&S=2Gg~gJ2E>PHEn$R1g51uYx(uoMdnm=^{p7u
zpiVPBSIM`xeSY*y>h&rAi;+4MeePzj%(f#9*HXlm$$jK$4SgsZYB?cNBQ=aQ)n|f2
zt5XY$3gb;xMi-R^29^VvX7gPPe4X-+%nn(4YN}JcQ)k)1yTJjA7`8Om><}&E;9q~>
zT>6Kj?i-d~pWyaNWL}7aq^7Npme-Oz32v7(w^Wt?^WJ9!O_b?RDrNj7d2Nnv+Ki`<
z64MeJwz24^o!(Lr-MHwq!Kx)bPDi>nJ~Yh@tNb#laoe@*nQWUZ9+>4$&hTQ%I5+#%
zn}|6+XRHNVIK&ps-!&^|l`^OtUAx@n#C4X^>s@`*eKh1HkHkv4u>`bs=td+}d2;n|
zv`pymlAO_3xWmb*X;n;<zj5S}LxP!)CeDdnd1cb1n}HjqO!d7z^-J9Hh9@FSM^kH0
zP7j$evqDi~W_Mu5V$XM+2O1~K?&I*D_h9xO&L%roaoJGeT(!yzZmC6+_g;vQS|}K*
z+Sjtr#Y*aFHq)yGn`X7`l5%8mY~MDEA(VyN$;I)Ks=5%%hrR#3Lay9eB=z|0jw5P2
zR;acKh<h+jF}#|*h3l2W%oshEc8PoLhQ9GvIcCZFPb|AR`5fy}A&D)Cn%arnfgwRY
z72S!?j!ZjsMb|ge&*G$h<IDMbrS}CTUgbS1(HY@i5qvP=@&TV@v%8uTT@wy3IDKtn
zi_}WB9jDS3IJx9pYH7QsR(e#$@x>LEi;LIVu3hPIq^pVJA`9DfK{gkk32iHmFXd}-
z*uf-zAhc>?<Bc1Q0b7`OD_LBf+Ilop+V-xMm>P7vwmZT_{#2y2PVTZzI~m<}Eb;ju
z(|Bn4RJZ<_p8nO#BwS}KsddtLbF}6E4$g)ZVtsqw$WBsWm>08mqUNT8iS~W}r8B>H
z$-BKMT6m_tYc5mF3I!o0MYaZ}HR~95ymMe-a5<6prtG!157*oqvB^76EnLNJzx3qZ
zS1uotazCA9`M}qrx~s!wVq?u3uO`l;ae*vHyJj8<n?I#&PQyx}9H04@zFqxvr9;=K
zW#fex?kY1vDq;k8wN@<XPdt{s>5R*Xb!*ZUPaFK>cd3|lY7+Od6?3+W#{OlI@@7d`
zYJ2zQnUxBRJy(CucDW;AdG6T`E{B~?296mTtj;ZMMHxZ`FC26(G>D|%I<mO@AxnqG
z{q{etw$I)>g|2cbV2Y7-y~`Exg>lB}LmPN5>Q24)_U6eGPHrhb%iQiqxd@h>?Jr%o
zPgD0n>HmdOGbW}fX{<l@q@Qo~{S9vWU0oh8S>jgX+4b!ab8lw+6gjihhXEJ00s@Zi
zY?Za(TIC|6x^vq)rUlt=PwDMCwN9p7L)M+;W-5<t<J8a(ToG4!|Ht`-z6y5X(Vxx|
zB$m%4y``B;`Ta8wM)nDev4J_M1rt>^OG>PleDF&|xK||R6;JP}iHAh{4Hw4$-FV+y
zq<edz{@ozXljmIO814q-$TdD#uX8$N>$;aS-pO~HUQWD{<L$Ee*{vL($Y6C<gZZn}
zIM~+iJmkU>Y|yUez{c*#<gwXg|2k9u&2QKFx@}P4OUpeMGhyWvk9Bv?I54{T_B7Qj
zV9`90;kbiI(qiW3tN#m<>`rc7xcuHd<KRCA&P}}xKE9Vt5_<(sexKGTu>Q$eTUWL7
z>Anof51DQ~<z4;wnffNaBe`-@|F7^72@bo+`T3TG`{t@IyWWZ%o0KN$bMXFzxhxZ6
z+k{%)6luKmI^uoY#Ao$lp({q)Lzkp|SSMJ|Wq<PhIhjJojtNse8k1kR%B{>2E4=U9
z*}Cok-~T+e!Wrv-cCz$dco9^;X|w0-4Kb6V3l2uq9h0_D-EKQ=+k{Qi4&B!}E%di^
zjsIf<?^ewRJkx!?g*iKa^-|L|D(edVqhI!==(6ZWfAP<!xw0-Eb79$C$aKWSjQv1T
z_6F1C+Lz28HyBFX+{dD*f93W<gF3ta3;s$(?pK@d>EGwyC{pwAx?ok3#;wmAU7xPA
z;a%->*5kv~;I8|hTDDdQWJ>OF>Sa*rFZKP=!X3CKvi_p)=C01)(-daidlh>+Wv8P_
z!u+$fU74GtUW<7JMi-luJ3KUfdEBX@@Tbn(-BZ|h{o&;F+hFDPLFW1q<M_|X><*Rf
z2EL+~ogyL?cKdyrc`UVBnp1!6ikfXt1h#tiXaw5}Mjbkr_q_GSj?6>7_t(#1VZX6L
zY2#B5gKH|g`zu&H*{^hDbKH9q(Uv&1VZ#(gP2qP@w_lwKTB%k2#(mN8f-BGev)mTR
zIb&N@5PjTR;Np~Lr~36DzcG9IwKO=YBFF27-e;eX{|1h0wlnPb_-0edr0a$z`T;Kw
zbqZ&8a2;w{-|81`H79JBOV=OQMboYytku<2ana7W&+oQ$Vn@rWj``^o@iX(!xyjBJ
z{C1&y*NplFb03~OamF)Il83uam37(&uPh7B+#LVX|7)i%;&sjro2K^N??>N;yf0x7
z6_>iddK6?bZI#c$`=9&E^D-5=Q|5S<>noM_RK-qM7H08n!Kp9aD}J>x9dJ-#ShHV`
zp<o5mo{|nFnNyBNE=N{ZM6^0`y49UOvY}XD+V#T+w>mZ&q#s&$YR-8N=@~558bUE#
zzh2Mj^YoZhwQ6}^)22yF?zhjL;+}JfUrg-joZU%*e!pxr^-s-@ivPc|=&<#o`9@36
z<o}sx+I7%qafIRHJl2MkEhW2)bQ3uar5K(&<&w@<95J=&zDp%*vVj@@kwoW~N!JSn
zF1$N>JdZ)camsy<$BU+3e&(}#+mRFcuj<(LlsBHz*id{?rR7G$+gB$#9gNc}3?|8b
zaCJR;^k&fVup1MUOkJuORcbg}B|qezlDwvtIAhJF4_-3;JiGaSas1g6ueSff)AX}i
zJ2WDU+-{^#br8IHY|~FZ5&P|ezJ~J|Vp7a{7v2t8UacoSU*T-d5i{9iPDg6*Ru^T?
zFxhET`PPkPoqNlw^Sc-hSUo)cYmIh8n5W7iz71FF=X=k&`tg)V__3B1EML2|ezOVu
zUy~r<dth0Ti07#uo=g!o&dC}&U-(+yv^-DPD4FB)zP<VPzCCqkBaM=`)oy*FJN3ld
z)&GLR_sAU(&UQ=oI5wxoXR78-6?X&m{S2$m_5J@e^Vyo8ygOOFH%!dlFe!YG&xJgn
z%WBUj*49pVTNfU4t?%>W`#$w<O6?5+u@4p~zxCzc8c<c}``Uo@^`Dl|jNZTt9uH%m
zxZbjK;LuQYI3OLIx`X3EP}0H%AwF3>pBo<%U7G|rr@aW;=zXk@#XgQj^Hb7^2}-_`
zWIPQXdQDOfJvPO1a@xrmnvp^}#exg{^sEKnJuynnxH!vwqFSg_F}I_UneZVSt<ucU
z)n3tBPdcakpWHClgk4CbbwalgCnL)O)fHiBOj8;34#i42I7v6H)qAuzW4o_p8&7Fn
zt(Vh*%iT+FO-KqZ{up|EiAP`1ty9x%r>-^gRY*OaEor{a<CpA<SDR;t+3r`;d077N
z*frU>?QA}?7HIaaW<BucM_}^1c5&glr8gfc6*e97RI>kN{WJ8L(oE(0x`{u(%!u50
z|I4ZkJg?vJ?5}=$$noJ1P3dJm3a422Ht0$x9h!gBWO?!frI*@oR~fy&y5>-*QI9YK
z=Pi9TW^R#{_N=TgvK*%#+LXVt(Mz*=%0iX`ZRahMJQ~*Ws4mguSX$+k#cgD9(VHRc
zlh6UZCMoWX%K}f_c(UkE+W)G+HM`TQveeCaikUSO8<iffp5LUsR<ZTK<c#HN_0d9}
zSAVVc(|m1kDoEwwsRdlBQ`el)SSB>R*mG(B4wXXfxlXtHe9bIpG&OTRn{&XARqa49
zBj>jxlULZwPYJv*W7&?6`~JxBlw5F?<SlVL%-6%YY5A^OnF<fJWDNFh?Ys10?GBa+
ztJN56S_2p^Y}0P!Ipm_$vuF9dtW1RyEFBXA9Tg|{s6=xn9i3K{6?oE3`Q#68rR3gL
zUb9{=?c|$xi@L8ZxxL`-iCg#b&W7#GnXdHVtl{2Yx~9tYO4E7tF1Su_{(i~ZpNn%!
zfWkc%kp<K5sXRGr9=BQKRugyq_7m%EzkB?@^~mPIi_`BGITfmS2Ift!+R*d2<q^}h
z7M`9D8Z6me2FZ;MZ9i5SX+23|QoZ!)!r>|IDcMH^Ra+u`m(Eghb(PFt7OEx|dByws
z%p4VuO@UvyX1PynDqUruveQ}URL7wy9zNZ9mxUHR*SK|JsY7AaNwrzKzHL(5sOnOD
z_HDy2&bM2eh1N4$KD_tfoXzJao8wr&ZTbF=MP<YG{5}809GDodXm@E$6m(y8Vdhq^
zJB4hoi-U_i_VT{f?&TLw&?yPrw<4s;<*$I|Mc1m0qL%~90~v&t=)GCAJg#(ZipS)x
ziQ%&pmfqEx8ydcH(`k>H3pDjOlINDTMzmhJ^*E}0@6yNN|3&+4W?u?fS+u|F^gq=n
z+eC|Z%-SWcP&04CzJBimLBgx7-yL!8C@nE(s9@53)-HaJ<()6XeZG(H3+n%>Zm?(N
zD5z(8A>+uF;qdV9>4g82{6n8yp3<^X#onT7MyO1v=&G0k-{gl8l^#YD*132j%XbE-
zOR;R?KldcLLwRBA4uw{hb*;LlB|R@D1x$T;;8f|9&5`e;wrt*e#l_WXSC5a>GNB*|
zL8Is$lAb3N80C^OEoJU_ef4NkW$<K;zBoyyWX%iHc@x$~rz;BtsHn>b{n7L*T0DE+
zk1OVupU&o&Oj#_l&Ea?ztAR74%p7@EagQb*KNT(knFV~`Ejmnd3YjwgKk+%S!ozWG
zfC^hx#DaCxoo2r9vq_oZbYDP2Xh%nv+L9hm2DTLqW(sblE0R+s7l~Up`J68mk4##d
zbjZ`w%d1$+@R4g4qmJCeRuQ2Ul9I=qxKa*f>8veq*6cB!SZr84>!gdI(d`Rdxptci
zAK!keCoH5L(RVrKvElPwkCx2)H8VB-*D3LRQy31Ednz*hyL{i)TBU)*rIBZGN5g|H
z3s|e%5_wKtVRE!{7n-uLr_O~@f|*lnL%k!Py?0<p9hc~_oJgaftt=144Oh$;;9_W1
z-Rr^a{>!&<LXL*|<qPc>gf4G66QVkgiy`KN#^DOp#)u79F8cRg(KlkqS<$9-rt^PA
zxydA(j~YI6ISsY7wscGMhD|Vhq_#z~Axy=W;b0c4oaaNXQ$1f@mu_g)&U?tU+h&Q!
zd6V9Vt(n|Aw>9yktmx6zc+3@E!l)7Z`_h6<LMMOkVDZ(}6qLGn!tNoDs$c1)%PcQi
zx1acTwy&~V`M%EV2HhW7s~2Bz;+w1xpy`sx9Vl=>?N~$T_YI3g*IaeVj?CuPR54Lw
z{xkid$*Q1PELxo{MPB*R%Wp@Au2P#)6v0)#$?1`hiqlEiHM*x044k<ROt?I?Q`3HG
zOVETQ?jMh&*)Or)OI@|r;JMTD>TMhJzHd1aKj*<Ap{boBw_{WGX|2o;nlwvUHLPv*
zHj^~%hNJ&)RXqq*SjM<P*P~JQ+E&MPUo%?I%AAR+R)}DDdq7(|!ezBVZfDmO#@mH+
zOVtiMX)(S1v$MP`N#*^F;MkYB*R3A5ZV$PX9rwN_z328!=O&fL2{zy6sO22w(#>fS
zeRH8n%^-%mtY@KYa)*<)n_*j3Mx*T7jwTVajjasV7s>i=%=4A0y8Zn`hutT~>#lMc
z8k}+NYEny6wsBk$`YzUy&EmMwN5jl%QrwesQ7?~p8cbgxYiM2^##!RU-#kJ0NcM4$
zgLx}17&QhRxv)#<27}zguqnbnv$@!IZ&_|#J1;=`>2#eao^F>S>lnF@9TkOJ!&*(1
z89k0=BuQqOO`ImK`M>92xP)43_|>$9!CM`3_bayQ-n!VOn<9MhuSi4l##4*RGOmg-
z2%YOVedO6cnX;apFNLQ6nB{iT^qY-PMe@9F)AFp>Rrl3BiRQbzDA~@;>Yg~Gu49~b
zgNlnI_cnzhTZ5ebK#`kiQ5{XTCKqpMC?s-ou4mWm&|#f)A*>+iz}kY%S8p#fI5V-C
zH>bdGU*K;WpY&6%Tsuz*dCiJT<xZ`bzWht-{z|WgI^*q@XA9##|IB6Fr*tw_Z|WzJ
zP;J%l&F`MBOL}xg^k_ft<ve!-Q+K&*6B~_|{y6=L;}PFlrOigU4_K}yt_=;_&}cMm
zVaDQy%d%U`m(O`-`@B25FS-1l_W!AGx9r}Nq?14KL$;dLweJe4MGl%<7Rd&2<R(`2
zp1HK?klfaeCYD!$i?0dQaR;4v)@;t;&m6~+dhO_A=gBuVaP3-t`5j|k=(Rr{6Y4KJ
z@>v(p7W*ADX-}j|`?7#%yCzkOwb`i6l=9kp$LZtD3=d9W<t5Lb?Kp0w75}s7$wnbR
zx8ffwJYIS^ehq525WaV{YSlKGEgohc53kb`U2|d`YkG8}%DUC1o=GcvHl4oE6+flv
z^|ykr5uA^@)&~S8g{R#9f16G5#9F^IZ#SmA$^XPwpq9<3|GMAi=<zRae^l|f{CRip
zkDrR(1~mm|?uvC{d|s<_+=SQZM7&z*w#0!eYsv2aOfOiN)t&dfQOe-k9?~D^=fAX}
zn|p<WT2jA##d<9Z4V5|C-_(>^+de$EaGPFsq}e8uNzaMVrM*O>LwMPPhBs&L<@D6Q
z)2#o;K98;AS;f19lbC|uoqisb@}jTlchaNjCd<X7`c<+Pin1*ied9kffOnci6Hm-R
zF_|Ww35$6yoD%hD;JNca?9K!AIV+v^EE4f)_B!IKy&=8UugCgGW~k1yIX75sUp<e#
zvGU!!XYc0mzc15z|7PuR6$56~mp|?#6&!O|?YI0<#X7e?FJdd+JpI&q?A<el(swq0
zmI_Iph~M<aZPp`k|Lh8>%;0B_9nBQ$o;_~NI2kzW&|a}4PeKkk{I}C+?mHxv$|Pvo
z$?<*7*>60_TFbxh61*62>YK_DCmYTwDT`9$yi|k^xUDcyW6+$k<}~MO@BBBe^&eLJ
zN@!78$FHCC<W~xleg)g_f}=0;+E(v7)%oGo!7N4oxyRP!EEEWBR-M;*qoymSX_1iN
zqnI6Q3x%|AuqgQXEbHZ(;IFXy?u}eA^8<YqIsO%I?p`Srkv=Q?|6FiOznafP6|D_i
z*+Gv!ICI&ZzT0t0^(n`%B@DLzjw+u_nCf%b<io2qA6D@)v?^q6V1K(<f;W{-bGg<b
zLG7K7mh@={WW3MGNNPQ%WLs3E#<buQ+jG|JOKR4vF7G&5qZ187&hkuWnt7uk?0=n~
zTL|v~o=uv60@l50Fj>QwWpg+(&bg*fRjJ8IIWbk4lex@oQtXwN&&skIK6uu@dTDH(
za{YxSr`_cF9}d00@v`5++w`Z$y{btneg1Ju271~PM4oO`b1a_!FVp<%OQlyUxxc3r
zNO9RdV&7Bqs$@>5Nz6lonZ=fKKO1;Eul@COThiJ1uzrP0YvQv4Bz*gul(be^>1nUK
zWO!=Rq>2yrXM)v!^|1FR*{1iOKXPdapYH{yXN6wJmfEP>XLp6LsLQ##FwLH@i-|#9
z&`Xcqgmd2qB^IOKzb0PY(;xnc`$fYujavuYW@trZDK5UXK(qOk$E+o}GHJ>upPXUa
z!0=j)@xN;K&mYHhQiS@GF7;RNY*~@VGwA_~Hvj1_AKt&~y;rg6{XahD+D-1EPRARB
zG`m6`O_*~32e0Ssq!q8U3^$$R`}E4-yw=)tulVL|7M(TOa%+htYlCqfW2{;3qBWOR
zaH$m?*rX=cX4iFUF~{qjPE4GaMKpJ{H7E5;9i7B;lf_7h#fYt?c-d=C{YTErCaBmw
zVQPM_rIH~0!Kkpr#m{ttS|0nMN$WhrmkV(|whrO3;c<?2`*hSU^O@PicR3fuc771{
zoE&KGXuefp2A>y~$-;+=9-Lz9lg&vKS+d~NjfG;1*2wMh*3|p(!)1{OYs0*<HS<^$
zxc+(GpB5VV^<-q}=g9x%m$$6b5n_6E;#h(R8{grKM!BYyVv{y{+b4TZ6}FmN9CLcA
z%AE&F*QP353+2zxKDX^+Mo!9?9l~}ewmjxY5;kaw|1N6(V)6D(lhe}PJD!?+Hce@J
zSd)~!Q(Uum;I2jI|LG{Ih}rQTv3<EYiL=-4`Q{60i~QBwn9H&>(=t5Tr|x<p<nkrh
znd!w|mq%Bt+)9(Csl9o0HRRDRKixNByf&_n(%ly7&wZ3GWTd|>fk~~&M}#R)=JlPR
zkbW(1)rDnuH8vbSaX_xdk!5nj$3Kb-+#dd^2<6^Y_JF}p_z&aZhE(A{x?FL>n%Aaz
z+?}St!=I=B$m4mKC+n(*LEWO;LQnp0Iw}6@p!`3x0><)!w~Zo89JJ;ZD{cSceLd7$
z{}A6T(W0$hMR&KlUg=u#>`I}K(^oUSqB5nNEl!EX-)(PBF4>qQ&8y<Xd_?`!gUKu<
zPhHYoUbF@$Eh@e?NlonQ!IJjyFGeq3WNX&Bm9k%Y`Nc^;<IANYzUSKvxIcCpzwI>n
z?3r)9SzXtFDLi=og>4^A9J%MMliMU3vsfW#st(hV2Y;5V5i4>%ed2&f?qr2&Q%#nd
z)|CcFzAT?VskH99=YkEUr?yQM-6I<DYN7bQZSsE>HgNuE`1`QoH)A9J4RMb|<|CmD
zw?dmVBUCM#)hr%2-3-;)abR!Bc{hg&iS^0}dnYUWcUtRHtdzj<%zAB+je6D>6_=8W
zSJYn}-jKIQ^;M3Vfti}msdL8ylfUz{253p|T<Y}V{bZlOl2zLCZw0Dqos~6JS@@5Q
z)icqt@R+>h)ASa#l{XGJaWXl7WUgOewt3ODp9vxN{U)4@QrvP#KjDF|YT72XlLrpo
za?G#%8s>g5T$1lWoiF#52;nI{#Ssf@c(+G97xQ@hX~I)8Pu_QVe=8n*n<n}ztl<B)
z2KI^q&KZ-~zvr_akmpz=ueV69=&YWJgg57*DXvT=0j_HA6#fUyRQPbpv;UM&;13NG
zKb5qAlp@LLGn-V30>16xh;b`_l67e3i{iGm%fAaLFV$I6y1XIo|AGk$k2dOtY@3z;
z-7U}4J%l%CqF<(zNcY4;Zd-0l7n_p7(r}=`lOf~FiERv9e@qu$#4!I=X4uK?5C3uI
z+)NPh^5%{>eW)cTJa^fQte^<P3%W`P!WX{R?96uE)g{Z45S|<w{`K3537H}$jWxV^
zQKnH*uczPNX1>5gG5Ys(F$RknEIS%%8yo&AF6G`K&Ua&}u*9<8j>}?aDO4${<=t8q
zx=7B<p~5+($@_S7;FBd!a(s_Z=FHwXZ8no#-pur=RS9fqtyf=LNvj8%ywYATIKz``
zbs$G_kVf#U%iaIZv}<3pEx(ccN#vr+6T8QrY6<}=7nssamc*&FJo~UHO;lvg|Lml+
ziGHr@`dzooRQq%|W6O-PK3_GP)!ESp#SMBgUL}eIbl+FlId|_gkptJVZzgi@Dyvqt
z<cjDP^-$cOcfIq$%&l|YY)P0Z!saK;^Zvx7s~^%tw|&pO-!(Pr>-Bz9$LP<NC(E|V
z{VAK=Xg#?xdNS*DHJ8XKCwcfinL>*~R2!tG-CZy>|Dn=3u4$hGX6mE~S9qLln;BU4
z)2e-fV-xd|GcHTRJlcf6X?+M`cU)FvrvCJSmF}u;m1V0_Qd_$ASC)SMtyR)Kdu>yh
z+e6o7y}zzauUS3K?eKN+8;zpJ!cO+-%;@R~Kb~+<x97yIh4Un2a(;6j*yA&6V$&AA
zD9`Vnx&NO>MtW{u?CZ_F#&n5ul}K2kXjhL|)$JwH5le-C922!TE?m)AS<}eMFs1&_
z<mNvbHa|5SBu}_1ecG1LG=H{&(GPEVb6(kSFHZ%XF9k_$zncBCPk#@!@!c^^dgb=0
zq(Ymwr>keG?Asl1B_=*IGUddl=mk%`7co7NV_W+%=lr#5X;-G!y#6)y6*A`E!qQTc
z>hhnOoqzwRYS{xJ&4j$;OC0`uyQNjKv^=%xQniZ;-|z0jc_Km#oGK2&X}U}k8btPd
zKRGSu;KT<tij}Tej3>F67rHl!%=6=l&|ZCO*Q|af&MRdPlXknYE|^#t9sPCM_U6nT
z`F|!&ynDlP=ZVQSQ~uAbUGB`Xv-4c%vfC&14^6q<Fq>t;RP#?UUe11JPPwWy*lf5W
zTKDL+kkjsT%as#foSDef7OHjTb`IObX^U81`|Pq_eOb6vJ^i}?vr!q-gY8H5-eX<I
zDb-atSFLDL&B=57-!3lKz56SJeYMV;v=2$O!Uwj<t#fueC}yI_$R}NUa{33S*`gE9
z9Q=?Ztm0ra#q+7}`cGfdKQF1=wW1)hRp?6eoukTomM~g0Mpi8iowYRm*V0bAWwn2n
ziCZjZb$D56qc$T}W!}A)`K|gzPv%@{(=u<UIFqdY=t#+;S^uVVN+lniw)27b@&#We
zteCh%cxC0yGp9uYJsveq_?3{=WS5b->VKx@TK;c4yv$YfPKKNbZt+-j{y@})T?$FN
z8TNj^lWZ6pP?OK~YU0Htr?e+R^<RWjL;1D{eM#jFTK|^&TF4Qexn1QCYUjj1e^K`~
z&D5Pqpy8Q-PwvSDCo8;euG-<2xwliLVT;8FmgFlRI*inW?IzYwc~v)iNAvF+Q*B>)
zI^B9c<!3h2$@p(BDTjEU9C3eig!dMMt=(h36Al$`4x}_au`PRKa$&CY)xBCDPTcj+
z|8gzo>^{%0BG=v@+@$l}PN#q8V!!yz3zPQvuPS9sYGPpJQsEE~c(CA5GpCqU4adcW
zExx%d8w4HL79Z;7*Dt!m@iF<xL=|s25lb%z?dZ2rvL*jD7x+$U7V?-lQ?s>a3QLfg
z%ES%D3xqiBC#!fYI*@U3iO+N;S&70G44hsfeV?A5kGEeNv%Tu=_fQA>jY$V5$#zfE
zIk{2guVzX0gtZJi_+HADbeHGF?k&IX)bjfLG&Q4ja<>$X8@r{$%c7<VK4|N1Fv|0i
zRe89ohee3X(!MNta`3T>?3S{3J{3%MzQVN3W9BAS)7KL>^XCOv{Qj2Nv1a+4Z@d!^
zJ3e6AR^VXU%3L`4jCavQp1V5_9cN!Gza;L~!l%u1Matfvi(UBA?*?mwd4HYFug~{q
z?^e&;86x>JTFs+%M!~|SKYY?gSI!4GJ4|#u<-BIkaseNXX;Xq+So@~@50qw|lyur@
z&5jgqO>0RN*Fa7cg@f%1O017Ul-oL#I<%xj7BOh;+Myu%^~us{3}2_HOpoC{S^C)7
zQtQfN4=cTj0Nt~^lHwE0`T}RmUiNgjp}X$Jqf@f$Qc7glW2Rg@rJI_=a7XvTU8VTO
zWiyw}a+3Tgc2O)SgniEJ-B-8Wntk-9rh1->(e^n`lXP8V3eLNJS?-+86_6obt#@ig
z^)lO(Tgl60cLlgj<`U3ccHP!$?yMQpKHjNS`6+(U=gieBr<z44x@daHwr)vi4zpBw
z?YL&kDK~2!&58i&e65Khp<EoMYnTowvFdDeI>>R=@u-@%LnyoF?g@?pxk27hF5jB|
zFX?@M@oAWT%|nifG5a)*oYG3ZqV;Z3UhksY&P#m^TBWz`V7yt{DZ6EcY-{LsH_bz~
zYKfIg*Uo$<mZ>$R;1H9~hi7Ufaw-?HL}!~w=N#5t@-Zjo%JK<$!7t1<&f^p15<FFB
z7m~3^K(k8Vj6mRZ&NFW+w+o!@h?{=*^qreFLd$!VcFbk?RJiHj>}8WyUD&|z&0Q$a
zVd<{tjrx{HR(lCV?oBw!KJk%|<ZK&{u89Iuja(V78Lbsqndf}uTYZsrik4?Ii{+Q<
zyRT%$wcoze&C^}HVUg$B>Qkxx-jVkwzR&gBX2>d@yTg!?JvTyAb$^V^_ZVqy%d)i9
zi_favcA9kc+y7Z}>-U|kdbza!$S<cQ&wCf-9|=78>6Gr}eala6`MuBB(}~^xPcpyN
zk(VFYWwSI_#N6OCf9sWVv8!cjR7-8KTx!3<=RN8>vajv8;+}fBRKH}O(@Xh~^1D~`
zLPWQgW+#5zQKq@Y{9o1G_W`zU_xRWSov7OIH{rO%wLts&3(cZmKDgV48FCdis;d4M
zPKeJ4S<JxIAj?!SVX~csuwG6=@iEQj$sP-4)PFoJ+Wk^IIU-{+r*6JtMv{bh$Yz5+
zO_n4s?YVCmwhA;Vq_?lte3aK0AtP(XZdLnBZsIP6JtwC;GqeikxKOoB^WjM!8w=rv
zMcYnjel1v);ito?dCGxH@4t_t=0)kw`ZZTg+IX^+xK2e$&fcum{iHzURIz8{Edws5
zP>xswMb@k+7Qy9~E))H&MVq!AaGd-{!;^odJEQC+{)2Zr#1<_$+V`d1zG_0N+?EK}
zUB5OSIJ7p&eNlv~ZPh}~)n_xN${d+pU6;}y^~9x>J26r6mIK>)&rYwfqDc%P7E87E
zZZKRO<SW6tq4T1kRwrMjZ|JrK$J(E)^!1%H_l?iCSqoWDskGl(<h&tygY~M<)vZp?
z&-h$w^Y(2MPMvXx<<H_1+t2#W=a-xyFRn6al8n{@{>-Z$DXA9kG+n~qR(MEno6zvx
zFnbYaL6q~g>u$SN`cLec(zJDn1NT3}RsZE*OE~v`*yIqn$(d)O2!rNL$8PD3$)c|l
zj%pkW@aEWbsA2I6R@-Y=*R_^RlGADRwzu)@{-e7>j?4I&^R%B%H9xuI`6p`WuiPat
zTcS(YJjM4=fN`hY&Ni(Je{LL$3O4N2J>uuv(-GBsL#>@-xrNGNUe9MzN1rVT?&5#R
z6#iF5($7IKxY=s*#R=&qZ+OhCpG$JC*s(*vNbZ~Bso>`|Cl0Xw7U5o8n0!v`0NX?x
zsS}E#fpZvklr31Zvt_}Ow+lRZ*l(>q8OG2o(lBw`f^9q37W+pfv@(mgsvP>cMQ#yG
z_tmV0oin~Ai=Ihs)!=jKj<~c~Zta{r9e$&(zzvJ#7ymC<8vE+uozRX&@{URF<@KvW
zp50h%ab<U_lSI1Difi8#{9-vcq>BZ@CoB?SmAZYXAVOhdps=aN;o$5k3G*MN6t8{P
z824tyO7;AAWy!fmWj}thN|d@McsO!PlII!81SP}sl|rpjO)48$&u<K?e)qHEd)!e4
zmbFXd3<58x^$F?DN+=0ox?$L~#A%8gw*%KcTXBZtr4!;FtQC86W938-t+s=y<+6IR
z%@K9G`k3Sdb}vdW^4qG!5Ev54;E~F{{mASqt$)IVlNNgGhE067|Ho5B2^-FoicFz3
zO&9iYb-rIU#hvXUhg9MP9TPsyH3AC)jw;yZoXr*!3gb|&N{&-b`X5smtno1Aa@i@%
z6NPhDY;ufrH<4JV{_%{Yv0lzb+b1vkQl)0QYV8xYy(xLGQe}4Hk136-g904V1wXxR
zS5e_)>GF^jo|64tAw+S(U*%2WkA##bE_6|AOx`NQD`*iV?HDA{>Jj_ua?qtkC-3&E
zsL!5#r^;oz=(IEM%Az%=g{|6Mux7zgtE8@&vfs_q?u2^yrl-W+zScU0_sT@({d?7?
zPdT)=rjFyWjGZduor8Vs8(p=7T)Q4W49H1Rb~W)YUSY3$yv*72Q1*!r&O2r;S$iv`
zpe-`bMK8i)y^(Y2rpdWdt~1p(oH1OHeE!&**wujw{)tmqECMIKja+eb!>omuW&dw_
zbVX;?v6Y-=E-F^SKQAuPQdkrgTG#Y7x4wku^;N;jqr1Cib}=pABGD@G;$W<{bpg+!
z7p#HT7mE5FD~Wx=nq|A@%}L(A*7qrLu0b<(8cNp`>6ERXKkN5FU9WG4#b1{4hxLTC
z{ruGJ;a56I*UnzmKCQ+#DmAWNlFhf3Q{fWV#HQEHs;#;OrHPXnl05HiVSS+Eu5e`M
zjF$|mn<^SVD;nI7dADK5rpN#9JTqI|m+GS&8NtoJkzMoF>fIi-Z+(6^W~dxKqV;P6
z|I2A=mMe2IWDjj!IP;n#=hCa!C(J$Gt#RU`#KLXL;#VW(x5o+Z|C>@K?ZJO@Ynq-Z
z*JD%1u(1F98&f>Ds4!?UmfyYV;M*2!xYcp%1!b?Yf@vno#*ai3!o;H!z2|QUxpjn9
zbW!DT6A$f=`XNDuOb;rzEEUkqC|uc8nE9wMUfB2F(ZY>^)#<9z{bsDH-_^`piq<YK
z5;QE@(Cor8Epf+VY3Gu#-G>v+(}fNj*IKbh%7)2CosikRxp>|oWB=<mxydC7i$jw*
zN~Ht^1CML1TjZzbn9Msh)Z&?)+A@K-<1!zM+yAQQcAM$4B?e_Via(mFyH$Z}v4JxC
z#@N*df<m`(JxWS_xykG1Auf&*fiDVrd?_k-n#7|G#p43S5(T>|ABs&?Pp>U>R59c#
zN%9fhB(yDoed~w+mD?1${{?ezy}-5O2G^Pc-8l!i_MPaqR;^kh;L#XTB_ZgX^jUqC
zA!qc3o{4NS>z)TX8`{pk5PD&<%f%hiw=^<0CWd=#5jdn+b7XVtspl?<OEjK-)aY88
zbyQex>ohrw!*Wj<v*#SjdLbwuvP0qXMi0rWX2m9=SDNeIO{|Oloa5pYuxXR~iLeP#
z!h+wf`@h?m>$imU{!)Q0U-TZQMqfV~YxUGC`XHa()8K@{g5-mZb0a31J!+J@k=7ZZ
zw_r*Rds36OWAyywLi3eEYZMesILyQkaqZwx*=L~q?SaF}q~IeKiu{WV<enGGiu>O9
z(Y$UWtCs?6NTi5=hp+NOsRtSVtKD7{?PMzsOG#8&81bvZJ))yVAhC6>(=_$NIc-K-
zJtlQ;irZ5HZ0}5VUv$)3pH1=pW1*rK0!s`g3JHn|o9e!LEN}lXu*_&ijRr&D0#?Ne
z0S{I&-!kP_VFg}k%5yj>Wq*h+$Pk-zvhzUdG^PSQ=_Rb&zIGox!M5Z=_rVt_{~p^j
z9q9b$P#JkpblXnRZJcp$8m1ic=vg6{#F3oZx{TAGqqqBIRrfR<TM@%ehMf7ErcP30
z6(|n#YUx!83|BOkxqZ^e*QE6fqw(X7*;g5B?WVWJEX}G<l`|6;G~KSA%u;gF%`Ggw
zT}C)Mc)2O>%L%Fzr~mcrX?-TArx-PR!hch@MHaudYi!)^yg{%!o?Y-|vevrkx-&$?
zr7H>wKSoRJYSijnsFLVC$<vcVNM+d$G3AH+TLoB`Z|!!7<ac`MVBj^yc#^1W#Uz<i
zqQ!>Fx({b9J}|}Z)nZu_GpAEiEPwdsH7!}^J~hlUELtH!^`lk%OMxST^U6aq^CFin
z?NrGASzX;Ed!Sji{gZ^NaMH8w>Ol`IJ=wJxRju-NwB@}hX1iLK_N#7ciW!&EjNpsy
zDiiHg)fhKyWWDyNS3|9)Dq;rH0@gPRL_L_qdUvf@DAAR+P&YkLd4)#0W9mxxw3RiJ
zS1!-6|GI#6^@Q26j;rS16z!D~yY*;A<i*+UFBeOe{}0v_bYA?uu<e@jqz$Z-9!OoX
z5bgF92~se$xZ*rppx5roYX22Q8$V|(EtVE)llo}XxAtOY$0zkjE!kzlvM~qd9r`R$
zuN0yE*)>i?a9Nv;Q83#Ev04Ge+R00o7B0!<W~_VSEHdA<@ND4vau+r(PQltGYWzio
z$I?VnrwErV5?0)lz#+uCc7wU(s!4)^DoU@E1*{h--B_oxkaIOdQ%vEc`57TDNt1R+
zPTF&0@tO&R>p~USoi@3?n!?nudZS>q{%a{mjb62K0ZR=tRb%O!H>cJrC01VEyo_1q
z-u1TK!5eozvpwG;dtTjDOIh<|d&G;M0&g$-vvt|Ve4by{wso7+f1#We0lwRFHhBq&
zCI&QUMzbkSZ~nQCKhaLOQE7#ta9(w;pktVvE7$krMO%JOvRfpOdpvK(Esf<b1sY~2
zCMj@5ZWM4-6?c5P-7`#RtLBuo7gE;EESzl3wj^NZRPP=c@ioaCcFtPBx<iZe(u*qR
zXMsz!Stm`X5;|OUmLqZTZS|$rDcp=}RSkD<E6ltuWpZlz+N%=<n;-Sf`lO+e-1;VB
zos^m6I=5`uZ$9s5?0FrvrRLX+qt~MNO6$DMqea3ea0NxhS_^RpD=a>sy(A#2D_P~~
zY9YtO8NE(9Y-#fMuFp^rWPS5YcWTJWuieUD0=Vrp#mv9P2x+UR1}^eCr19_d|FUf+
zE0u5Q{#(6Ra>o7;V~cf_QD?rb@^~39w_PuJ0q4$HOFXrD!#HiNZxmfzw0YXHrAG{1
zd?Namd@MO1EfnLWa`^Cc-`(Ms!9vD9DyE8xS*w;Y9*qj9w$+)obu&}+v9h*Ot|=GV
zlukF5T7DDo{<F>Qq5MX{6(<fIvAov)T$wR^<A%C}_A@F{r=D;yOA0;{+VJLDS&@0V
ze&+VfHI5$3?4395(0C=VWdi5c1NI!t)0nOt%Xv6+){PWR>z!tdOFV8mI_%Q>sbw~2
zH>)JiuKC)#`rlOT*u~lPW@oR=ZU@E0_rHDl1@+?&Z{Cv}p%){2>zB-=#C1-|elI=b
zjFoFm9{;atpW4sexQ?qad#UF1Cx>;|6ctL9Swk+b`?Is;yVN%Biwff5(HG11J$mjU
zT_CSuq$41?_0?v9MhU?m*=Yu)%5BR#nkQGBQMI^rwc(BqgTNEr#SU?0h1+%B2z<E^
zZ9aXY!i@vUvxS;vmQ0=<YZrBJwgH#W=S|XoHZ4E6VCBh8zkS$p#1fZ8`^Hx8t~B<0
z&=Gn1u+-W|eymH+Nj;nQPH|bz%C@YR8ZU$w^6!*Ndtn<cebB32Va?I4?-J{hx=r~R
zg`)#XZnou0nbsXw(uoz<Q4JHCrJ&dSyZroCzLwP~fo5W_)1qZu_*WOio)Zmv)YP$e
zLact&Md!)+_seFrt^U8AKe3T(ql4bfnL9Wu9~|x8XL<hTui5FbJu|IO9=zo&dFI6S
zQ=8^%Gt67f*<KTzxA(-F!wU+dH;2F4a{iYbuWfiQlg+a#Bb)EWUqXy8tE~HYB4XB~
zJuyE7%!6yU{Mqw!d9sGunerp1tDY{K=d^qk<2C^s=Y|r+zbj4GG6a6xss2sfbd7Gx
z>cW})g0UhqVn06fN`A&~JoPqHMsP)W)S?ZZIoE<jtUZ+z51g30{fwbXj$?U6&Lk@*
zudiknYZ%0?95wy9O<DEifhT1OMpqgbcX8ggHT&^qO4k}zDVgdWzs|D<T3F7AH4AO_
zkYc^rDRm)v?fuC`Oa{|}*ZLjUyysow|HBiHhUaOX;+`b@pJlq%85!|L?K_h%t#5UA
z{UhWU<SyVER<Yu;)!+8uhYzcNJp2%RR9;V1(7|3%@Cs|Wf|y0;Hr4!dNAn&o%{y-@
zW_;CQd1~kVO?M_eIVKp`Rb3eHSvzPci({s_*tw;#Q+Mduv4%{SNVUFneD}!|>!+*2
z+)f;+*lFG>m~FczckhY$t659-o)DK=?RRXIkJy|g+^spE<b=#_6qFvC|IONdP1`cH
zY{A0ktbT_p)+Msn)n0#oU%AnvL&ZAHL+A##;A~mJo6=E7Gu*f*y0d(8H;GiyUwG@(
zoBPi%pA(pve^ZA`W{X#=dIPtK#p)-~+gcZ!cO7qx`!BR6V4^bPV}qx8%eU&qJS<}`
z>Nt9pzdhr?9pA;XPIL!76L?zN{CMs8$vwMr+E$luIvCY(@V5_F>*e76=N9tLadvhQ
zt+P$kIc~Fc;(gJ-Z#6d?Pudv1cGEP5d;PX$r<>*_S-o5@wQ&0N&w_&Q4$nU;vni>L
zug^5)^#t=o9o2;kUh-Ws_7z(5IJ$6;mr3iv%xftIsnNZsbm!bm%UCEn^Ty0A$~&z2
z1jEmMEN>Jo{`hg*O!0N^CKcCgj92v1+s9Tg{S*5}uX#_zbkFTbQk)f6IQyS--s?%R
zuP5zv=&D*3crsp(Gx6U}uI7WUHw8x*&W^XO4AOhHkx`~zE3@Qj<W|A|H8DF~{Imn+
z31!r#x*m*PJ7Lk<rz@UM2y%_Qv2^F@S3i%l{`|Q|Z(`I+-ZpM$^|#NXWDaV7E=?|e
z=`!Q2%-w4nJ&k5ZJ<EN6=C0YQjRk?9ye7q++$*qo?wzx1KQ_rv**3LX@if=R>I2JF
z*{n{bY$;G+tcbd6`u5)et|Wz3div*j=KoO7T%4r)=JTJuvS02=tPxmIEwID>`p2_*
z5>ijs+OwH(U&!wJ8uG(={pQ^d&%JV~PJA@)TmMDb50b`y+LCt-9|&bGcqA?SVP2d6
z^CaH9_tJX>wAt>h6O9t-yVY=$>z|!~>qCJyt+PsXD@+`xd)Le;@l08&AwHKwtch7*
z&9DEQFP^Wzf3TT}MdHiy3l9%8a!G6TytwelrHRilYRZYiMfUN$j$J%E4W4*b?|XA|
zRr>0%wPCqQ=cYt%o;o`$%1!dkjSbUQ1b+Q<G1)YWXK&qVk!!2UFFf#P{8DvMT6sa(
z(H;>&E8QDY6Zgk`jo6sH^uh{;pv70XBvc(P&+v^l$?g0drfa=y@2;Y!x37jD3>2EA
z;c@8Vhr}aX>Kr!?+$el}Bu2=8o5s#f&U({WC)_-<(&v%hjGZmhWURIvz074P`|inx
zt%t7jd2`O#yQ}ogjVYz?{bUX-%xB~hQVH;gX<+GYUOGkbXmyeHsZ?HXaR*teTd&1t
zN9SbE5J_*%k=dlU_5a+dx<{7uSf}5YD@<ezDpUKWzGY=V>(@KQM|RD(b^FWmVd>Jc
zti+=S!Xh$Tr?5)ru&PBzbzTq)S6dU{q<f5GgL3PXA`czk3_%B%s6AaB(*nah3q_Wh
zZOTws$knyct}SiL`2&l(yt=3QR`_`92AKqQPi20)`lE-iy4Hz#QGMP{1{(d757~Eb
zTky$1dv(YQ<5r8yZ%%7ldg=NY-Szf0IxFIp>wEZ`w?pIMYpW+sG2145evz4(u!8^L
zPb~R1egC$21oVAMadve0EBxNCeUoud&!H&=9CqIyh%{XKt}*$_2909{7Z=9GnXK5~
z%~Ens{ICJjO7+PuS)r;^b0RxW{twIAy7F9f`liET)-gA?UXQPR8XA#MxiytXch(XQ
z<*K$6_l1kwq?+fh-STSrJT?AHr#EyQxV2h)FAvxD`TPGczH}?gNKBs^8WGt(wd~}i
zQ_;o8w~EIEiQZD`WSJMzw82|-hLQI-j<7QekFv~Q+N5dlctv9E$v=~XPacRqyXdN(
z*%=c?yVr-z81?LQw*{Sw$lYcndnqEhyP#Ol{O*%ER}?j6O3$U)<lFJ2KR<ALiOqW%
zCFetBI!))qf+l!ZW*#y5`NcrWd`-sTlL;kxS=|AxCmmQMzgRhLP!+lD#C3Yzh3lK&
zP7uiDdSBqmu;uQ)R1w`5jyW1pw_;nxsw!U14*4(TE7I_)UG5jxN#$H0n^y<;r*AA=
zua=!5%-B(F#He97f6^wCna|c1w_g;gDmi~MIx@AL*ZQu(h2Z6zzJ8l>gK5{>A37$6
zZj!D&7e0J>v-k9SmJL(A?004_mHlFo#U8}?&-a|A*W&qCf_!v^&L}nX2WxQYa^#;6
z4G+vM{WH-|$}?<QOw&r<up7s>PKmke@H;%Yo3*U&PxaFmx&I!1`@g~LijQ)uw5XHB
zrU^|S+Lhu@hOgXpRP?y3fv5141lG+;4I2!Wiha3}5UMmm=8%R1CucT4r|u>RE{+D~
z&W8<C71S7-b#;w1gyy~oP;M}kZS$V{U{Ye9vh*#Mk0qNGCI1UlKHg-&xV&hZ5noZq
zVv|-CM=qW${__HdH@}H!Hwt^gmz3k)_O`*W(rvZtrLN#zMhD%wq9l~1bc6WUCVD$^
z{gP^a#<6hYT3x3C<JmW>u5R1GdWnyzAbR~fg$75tpotwNCltPyb4#3iZqQ#HQ&&;s
zxXd-6aEi^Ho;`~!7Am-ERV23V$|zh=vhb1vpQ4|(w)zI$JsWHdOBNs6^<;ukL1TI2
zs#N{k%+~Es0u*O?@=m|!B02MDdIp!_l@b{to7)pT%dID$Gr4(4&aJS`-_`ic6tkm(
z-mzR&#>PHh_j%6LebKHWq<PZg1pDf?B`K;blRDSla-Tf&gn!+^N6$CB`~Us?AI&GZ
zOr<9r{;M~bwPkWAZ)ne(w`_KoiAwy!TO1J%t@?$vrxl(uc?bVm=veikH$Pc1bC=U8
z#~jnt#(PfZ*=M)!eXR4Qbhn~niUpGoV}obuo|{Wk-6jY3eVJh9#43E+A}Qov22ZME
z3-7C0R+2L;kNa}%l;hcZOirh0iOxkIo{}6k*>zs&BDX)zjhx5n_LiepWZQ#ft`pQ)
z@2gxczS_O0cu@d%XVnt}mPfZ_q*IQwJS=odww7-F_hR){*|7R`6QmU-D^L8^Oqtjo
z;-MtV<k=nY&EQvYM;gB;n{G#c^#90d)l1Agn+)FB1shJ#^q%Pz{D@Vv>gp0T&B-nQ
zHGMp%2|DxW-o0U<{nO0QRwV1xgje3*B|?pk+vYvJq1@=Zg{h8jqindP^NKs13A1kd
z@|11$wmMvrZq%-H<)X->xn(Q{jFT2!F+cxOK-J^H-j2fcmsGZDHpp%+IwI(|U**D_
z$c-#&+YY7w`Skq!uc!V8=jg8Z&+{wciPdp259ydJ_7e+RFQ^}`oO-8?T|%oh<--ys
zRqxKln;dzsIqN@8THwaJLzFpblGVXNjyWzCjk+w?4Z}9QOetGE;n<81%Zy9rRu@QZ
zx*Nc=V(CkX>ZHkrj5d;`3fb$5r$sGGw))s}Yw<lJ<u$IY4Lhy&ESyuJ$?jm?TWY){
zVA_W@k0$7{{AcJov9*xl)f@)v+dGd%vptT_KDyOrQf}_{3(2>l5<MJ^u7$V6lqgE(
zM!NHfY!ciTDgCZ$X;+HWR{l&b+gAq<?rr@wW$jaipgrw7{FU_FB=qiN+_#bZdCA9|
zML17i@!Rs)JkC8GntDsiFYh|HR;~Ta(L{sWB7Q~hrYrw%?KAxvVzcgF-u2d58@Ci6
z6Wp{+-74m+a5TR%tL57Eb&q`b>VJyPDgM5C>K`e--Ji2H{~QayC-mTu-Rt&*-?|kS
zSxcE6S#P~>em8p}7vm0XhI{jxtyLB-&2D8|!u*JF*NM<6fri_>WN&Zm5LsGTnBd0!
zqOj`X>H>lGicIsr$8^q$TmI*|5%V(Pc&n(@ljH4{MrW6=&7M=e>_+N?g&9uQSWox#
zc!$q3a*uM(`R+1R*=72w#0kn9tai3{2bF)WJgIUoM|+7)rPg_g+dcCv&P&dz-e%bQ
zaZLlmhv~;spK~mm*JruITj3qoga)P^0TJ7t)^40LcT2kb>l-JFQYQ92;drRKdbVdY
z%VwEKn}?>eZyr{O<2o0XP^Wb#^V*_MQ?^7LUvcx}%MBqCjiO5)u;|}#IN!N_&ib64
zXHA2%&ztse?RxpJv{qJaiTu@-9$~y0J-Ru`@tNEutM0y?!Kv}?-6#LETRhAvU;a5f
z!SuD;YhkC`QA=4o63=~<_4~ei|NTEjPxhT{{y(|KwW6-F`@)}3bt}J#ENakXIDMbb
z?#0`?9rMDbH1gy{SWJy^oSu7f!@-jZ-QBvuQX6}N4SNkH?$cPXw&1{JZJCaa1Dqy1
z#T#e&zHZO5?)g!~)1AR{<FiBG3&SHbnD}Pw)Xvnq%{}vc#=*Xemf|XUhML<{7V}JZ
zRQfH<w=LStYvx>onY-?u;8R%A_w0e})x*MCr_{_IcbIYSKI~xD-mqx<WNWRS-H#8<
z(fGp8`k*09Vvl>V&dH*_<F|VaOZty;A3hpwWB!0iXhs9?i>(Hho1;9ITHdj_Ijbvs
zA;;ejDwmq)crW6)vU$&{!am-Qa!+sevKTD9HhZg*^5m%Mg*yLFZe4Il_4j9)2eW(5
zNA&u9*}LsB?=<D+n-lELX>i9BZ_Yfm%C(RqjALGi3(wueqaV8L^p#{eS6X%7XsL7L
z`E1Si{HAOC6cL>q`@RX<ry_Q0T38(NwmAG?>e;MgM=};SJ0Cmg&~K!`IOpMh$E@a{
z1D!@1t-?<@q66H%D>Q3*uDmc=>iACa3Ebl6H7x@Q`O5b=d|SclE6||fVfH-HQlYp{
zpk=PW7gfn8d*?C9PL^EOEopVlf%oJLyHgRXmL76kmb_}|;z<vg4{2}o=sR%Y%>&s}
zmd3f-^ZvaSIwNWRUHjx;1-DZPyE!$QjaYU#N-c?&(D0BtY`jOOHF-&N2J_7S5qpe8
z8ccL-=I1aAO<*t*SY<Y&$)=E5_>6G)j@GV~p6vq1k1*_r*5D3%aX92Pm(L>(=QT$z
zZJc#?!g}6}oo6d;`xb1yziH{+A3Wy@4sxvCbbjN`#6^>g4R;w#Y|vf7`DC%4!%yKC
z);y=A=LUUQcW<K0-3`l_HoCSSaGm_-!1rK|?~e{dbf2jZ-v9mg^pCHXnQ@%dp4e&k
zu*pWVS@VI=7Y*O;7v{NdwmY9v<x4m`;{lV5Hgj`UtJtAV-Um#-1bu}JRv2ZnYAP@n
zS8fd4Y49k;^ihqJue8hK7{9$m{GGxR(i?qNX{k3dI_m5^w<DxuhIP-3)t09mR_V`F
zl-{Lj`(JUBz2*tE?EX91e7y%9&j&8rD9rHV&2+ou6H9M89K9p?caN%>h12^ReJXDz
zzR%k1&gr(^=cKR_vxkz6QQ)b!N7fj1T!>a^zI|kA%%lrO84HbWG~IeLV`|TZsXd*X
zH+<)XDC%i6b7n9fyEW<d8_)j@D!XoQIGY@);XHCmd$J_&WHzM>Ck+B3tuKf-9cfSO
zxBqy~-SWao1~r33qdC>{c4@g;-JJYh%J^-Q+P<1)udMA>Pj*S~Hal`)*WDAl*gg6G
zbMX~&Pd-s|>F<OlAqfpOTb>4KWy`5L;XjVeOi;V9rFC)3Y||WHp$8Xc8#umDXy&|O
ze?!OX*ao2&E6o4B+-~~+4ui0W1^d&pY@hpEvsSDYJTST0;^L`UhhK!KeC6NBJ;Qa0
zhsEPwg{yuYnTnt0oQ{`PyP?SXrR3_IMH2I0?LGaX#Zb}X^oAKyN3LpE_dE`=`st&l
z+Qi?;I4j_+|EuDo+bj;vf4S<F1nW9!*7b+CO^msAl0#juVU4J(^K26ykD}&9I?Z*P
zn{{tA=qfPt9X)H5!DE}NXni7JiAaOcga)IGX5lCH<{BZQN?t}8%{5;|h2NN(-)K0N
zu%Yho*>g@jUUPZEAMl*Z>0hUE#O6xa)0TObvnK`pJ$}jaYDeP?Ro(LsviT3&1SntK
zn{d0oOQkhz&!u;-PHSGBoT=q{xZ>us|Cc9E_T~Arr!VT@woj{lOy}xL&F=A?sQ-J{
znLypBz8hx`a~z+o(JFM}I!~!@>yOLVLc&E=85tkknscim_ensr@a@HW&I&0o3mJs#
z-DuD&X}vusTqd|<^IP+EPa7T;u{Iuc^E4JaKa=;xfv$fuEPPqz`MC`bYaA%y6c7;<
z<jeLut}9wO<-pn+sh%BuzoVj-`tm6Lc4${{JQC59AJU=upy}=<!@C}l39PG5o#2Ve
zb-AQrrX;*<3)ieGyw?_c_sCai?QY>^(71N5RByR;-_4Ew%LF&4oju_e=&Ga2{C|(y
z+HAg04BCvT&Atx;Z7PK{jzlcGI{n{f<{8o9e6qe<{vW+=s~K`^QGm%!2G&wm5nbk6
zcbK+5Y~Vc5DSFA%+~9hLXhZQ`2E9G4S$BBcdQbkg?O7HRc-;8G-wQg2KA#QSaIq^^
z?ET92y*CA_B3J`YOcbe{mL|z6eRuWbWPWMZ(<#QD>OVIIJ>~5$RkkqUvADuB`|7>R
zrF^G<#4<WC-IZvbq0lNhxzF+F-1nO#KQbJet~%va)6K&fakp}(L~USSf2nV|@-5x$
zex1opEsaSvCOy0vA@$0QnjdbJq@D~rU|1B}V|~I?bkp^zPegD0=_%REY4qSwU!-E#
z-p(u=lRbMo6E|+3{$)9Px6Xn+p7$P{nWMmZ@l}800ghz>9Hp5;YyV5K3UV5j2{dz0
z>xi@H&Nbu`J)Cm5<M@#m38x;Ul)t(1O=$UIUcTcti4u!?W;#30v^uSLm3PM-*-MFc
zqgSO$<*fVo#P(XP(T5J+N3E&SCO!9L(-;-flGvKmyVl+P5VYpj`M)<#oc(aZSD`tx
zFYI}1-=cz!z#owXYqTfvOjcoV<x-x^S#k1Q)$FZruQfip{yD2z(AMP3+yIZ%v&ki`
zLN}U<bz5D!IzKZlnK4EB-_Pev%YxlkuW{?LSy^+#TEP9Rk_qF93G3dpaxUWtj9~pM
z!RpB;P{z>3pV-<Z#u|0QIqX7<*hcBV7Xmy@507kM|Kr1RCa|gP<DI)53q{KKMgGf(
zc$J*~8<FT`tv9zSv~R=1ds`C~3OJ;<b==>w=*@<8M;@#?FvaY>P`}c)r}sn}j+AsP
zGF1L8q&P$9*b#x<|Ff8@Gnf~-h6(kVwp2#!lnsoJOx~K=)0}<w+LZY=ii}%$BeqPO
zf2wMB^<?`7qdm2+<+pw_{o)|@I+mkqLP~dDN1#Bfnq;P@*u?o6-19vIIm>cdJN(K6
zyM<o1mkKbH9q6c8c;~&PAMfOj*PmY<mf-2TaVO-S`#l9k;Yq4(8&$;**-ouY6u)-!
zUh+)S#F=7OEN@PI+8JV{b=&st--UOc^0H(^PCe#vDueq}Cg0WFea6|WsoOftvRPFf
zPEK*;KlA_YwbaR{&Q?T!yE}Os-wo>pkN4zyxdfTGIGb!Q+~V{r?6{H5R5PAi3ol(>
z%^oPg?7Jaj+l(i@dE68B<@EXq)+exU-7!tgQ*htTY0FpeIscnrurSc!k6F=W*Y1j!
zv-|iDe^@)~9>3yIo{6IC<Zk7zy7%(WwcbT#94s3;B(6`7QsiIzX!4`tqkC=EXL(IA
zEH*orAd=R1RF`Y<=cuGPnXT%6J#Vf?w9J~6AGbGEn{~UUMU(ca#-NMSRm7rqFtaqz
zuJ@Ard0U=o-Ics~_T2_6{z)sF)<+q)Z?L`CwAy6nuZ-?R8(vJ*>z*(0fjh7Jf>=W+
z$A{7YF2jiS07+)CgVR_17hvY|<MDfEuuO36stsp640}E=7P`KzLwr(C3un;mSB4W(
zug-raBUvV^=P5kzneE-+SXF0PnJb}37$k4}n2~U|zu;f0`KeVpzW1_E@7lQNU3wPZ
z>uXjIclN0XIu&#m8LgZu*sl36DDX?g<@#@)uFbQ5rkOZ6@o(pJJ|wNI-(pjsZgKd-
z?!xIN7W2JMP4JxGv*ee>qre9XL>)_xiM3`gWL>}^*c8yc_h7f;OZEjnW_x>eO~@<k
z%g~6)pB&zl8S+#&TutJNuEf@ww=sNkbzjU84T){)=h?ZXLwtQA<BK0p63jHu-h4IT
z)t3kBW-slTb+>n}YtJ{q%a0WU@@6%t|L>gXHE~K%0juVNSH(=(Ld#YeRa|a*q+>5u
zwe{KSU&Wt$^d<aHb~(#!7FyXF%22?&bJ_ll<%SWhD<zoMzVC{9VY7My*U3^Yo)ka!
zT^B<%e}{1ke_gGr7~OMhgP%ao(>Be*{{rXTMEsSaW4%g3{T}orz54xSj^o|tO+A*p
zhnIi7ryO=~h2)EUwhAndj4r43z38(u?tXhCE&04h!RxGLCk=NdSKllNH2CxQZp4Ms
z=e(E9c^2wVo>#J`Ijmy_i*CSKj<N~gn-9F0BKK0QlFR)Xf7U}4JCmKE-7$~jt2;J*
zIH}>nc&90aOZ`l3W}idqy)4_yu0c+go0L=|O(TD4|CfBVU|FF;*qK|I^785Pm;QVB
zF`+fY=IHxX!XFp(rB(eo!ZEMOzHISL2Fuev%eE#p2q+XJG&OQ@Xay{Ku;73S8@rW^
zM&JS#y>hjrlFO@nCC!CmdAB?bY|!mV-8MzA!0?H~1c}(B9?gwQd>T!%@2$BxX{kdH
z)2owfR2xF=>;F}4T+ABcu(oa=%a7OBSFdPT`H$sGv)0xXS7!WO;q^u|c+I&@%<Ju1
zXN5UL)#ons^Enid;Ml__tLCBMxM)gaklK+sMg`732hv!J1Q;s16E?@N6j=2bZc3Ru
zMLnoaCtx$fxfqS|nQLYWPd`4#$@w2oWSaO{P1Y%^bW**w!=@&0`@d?=?{B&Dnaq5@
zc%ItMFh7^2PiV&O(A9I8qkUI;Pv8E8MQF+WdD;!ltEXzQ-_vqV;%euW-f@G`{^O##
zW>S}T&rJIFV{Jdv-qT0&=T}8qE1$i`blD--<Kh&bzgLPy!vbBu1~*6jWEKe*PCYum
zRs0l7i0!^B$^P0YE=QsruW2euBr<(b4Ni_uxRI33T&U)dv7Xn(t=EWe%A`KaT{n{Z
zr*kqiFnI82G6*mWO0r6xx#<?;bKLVt8$%G6%F?O+o1C7ygx|_oJk6@sQ&!W=>x!4P
z@Y`*!acQj@2jZAC5--da_7r&T%zRU`p-klB&Q@8bNh%jARC8SJIpj^5_@Yq!|I~x5
z?0rQK-85w;gm=lFJIS)5fA*skZ|$fXV#3V{g{7j+IR{HaTMIn5yvpU;B=tHneQ)Zj
zz?G|>PP5&1<#4v{+beFzG?H^RwTeZ17*1!(6%v^o<8@}`<<32lY7B`DzSe6QdN~g~
zN}C<W_9W|(SII1gIWIcaF{G?poj3h?+cU`?gCaqxEas!!`l%*<S3FCjCS2;8tjkxE
zxWakr)7u({_@>AdytdO3(B5Xd=T4rJqRxa_LT^v5?B>c@@X|H+ggV0_rphbF#AMdk
zI7+VFanN;9Ud~3wnbJR2u8jP3SA25$%dbAN=i8^Ibc>|ueVy#ET&p+m(%NZTSFQfz
zs~d2Y`G0Oi|Fx|>hrF)O?pb(LA}GVLOe9F9v5dvj<A7P2#97I=C0sremUl^&TTj@2
zNA8(!Y0dMyGw+*oUD4QWqI53#p`3ACSnL_Oli|`kWo?!o3Fy=O=n%;@VfK{Wyk9C`
zEDq1y>9e5SGO%b@>#tPXtp?_TlG%qUm(TmN*Ejav?SJi?PNw)6Fmg&?og*zS9kJ`K
zh~`|K3nHo=TZ+oJCrW9!T|M1Wo2tjV_M@2pTGz19Kb`Vnk+}=^Z_!`2ZvN`1Z$CD@
zetp2thihYWM$k5qAP<37mOq<yBN=m|=S~Y<zU}I}GNH<t7zL)~8A~}_g(PFT{~S{~
zT+rRrkRAAl<&a)k=>Ik&g*#U=*p$}QepKU(7Vvw_7rb-Eo&dGuoL4nOp3U~1kXah=
z$}>-`-BvGVd8_TEb1V++6CY1hbWoYmAbrbWrT1ON&aw&~$)syDt+zBW3*UI?7kBAs
z=K+^=Cmt!kZ5vvb^FGN+zS7w?<(pO6m82w7mf30(mbPx5tDBp1Ir;X5YsYPcCiLe$
z4(@kr5#;hJ=$$Ia({1)+#_=Ub+^io>SJI9YW8q<8bXs&-t?GvG%q4yd$2l)g^=cQ?
zjqRMWZ1xF0XUhpqo=bRVeolCPPDEHh)nGdFO>XB~U({NCmK(`#mYy%7&}tTPUQ|WE
zfiuR?b-HgMbE=1bQlGQtk(}c*|EC;~R8?POd$MM7hRw!#7T1@u_N){Pdo|~(i_cTL
z@<YY7A0JO$|3ZDuZ_)1hUp5L`7O;BM%T37ryC(SUCl((Y)eXrxMbqL~gbs3W6&jY`
zPKZoOaXK@hQ#dwAJjE!>sWPxI)|4w;#gSR%+>$58KfA*Beo$@5o9Auhsld#w8#%N2
zXn~TYZXb{Fvdee2FekGsyKb418az`X)G&2Lzl)yGQO*dLC8-6hQ+8}PGX18bO|$~D
z<SmCb`_!wvcceSY+pg{1G$%A8HE|Wgt77i?8y?9EIWBgXwCvcXEfY1@bS%Ce!MH7X
zrZ0Du%WQeZ%o`3$ViWSBbX=D_j$o9yG{JP{|KOn1B-vdrSr$nwT;ZdWp?Ln#3Pa^7
zoIZ(FR}9_Gc`$iMYu>tG)!CU>eCpuJ9QSZ(-HDyW%g)TnUE#tNa_HJ}F|KP`#g8?6
z9vDS>Y?aM_vh7wHr;0!8!PA?#%7tgeEMYlx`I+}a<$3F@He0U~crBa581~u9cuvY(
zPalJ+*Ip++*_)Qi*?5CNJ7lG6?VD!xm<Q_i(G1-pYb1g<e%iVJzhcA!HkKC-qDlcw
zRuX<ZodqqT8Hw!g@|3vFs&0P6baSu8^@U=y!nUdeC1nRbHxjeuNw>3*oY^xiX3|da
zUhB)3L{CSUZMDpO+@Q_SK5=Ejf@JaH4enQ1KSe082RbnPpLXZTZOsJ=F)Pk{#T0d$
zZCNPdlY4YN%Y!b4yAMxaX5L&eH!5vYm!X=n=IdoQUOk&}<v5=~vA^b?tR1=@3Bun?
zXGzxwaa)|an4P?0@l6ix6Ni+l`deLdVw+wpb93IcaT1?eiTbA-hH=|;f^Im(2VIdk
zyGTO0^QGVenV(8}If_g4{E8+8-xj#_W`^kEKTDoEc`gn3daY!QgY3(&m&vEkMk&SZ
zx=<LC6BEsKU>b|p3q8rz4jHB_{(D0{l_*L-Wl6BUee2k^t-O-z))O77U#N0M9FBgc
z$IY<jo9AZnhpQ(Wd|I9n_idBgrWm$$)0U+L&p5E}=k*7=H=G%EPrAQ8>frxA#`~Ki
zemQ@zlRk64-F(M_oqx6+>0Ft((;?DdB0enWsnA5#>>H1E<p^#OUU?$xWVTYr8lO}B
zKa`>-DaNQRnb`Xzv{-WXYs2qMFZb+Ioml<Rjcabo*5g&XH)yVXcl;=0;w9IrIZDP)
zEp01<+Z0y|x7o}*$(d2az;!5Pg0@Ls|Fj0Cb-@!Ye&<xxa~WS)=I&;;IbFeFHRtTd
zyW@6qKM~uOu-f+$@8&+0E*|xTB1_d28ab!T&Fh?hAXU)6j7cuHFIoGx?9vIxUDV4i
z@~{<d7JQ}l(oB0(*#0d`lY>eZ)!)cdH?yp_IBjwLc*;?wr-i+?Z6}pvbCtJE?Z{JQ
z@{a#st{1YvSZUo-J=NPRJH7Tv8oAA?+f{0M*Jqy5B8{aJ7|t58+&=xuYWrs1n<^Rd
zf3{5auon+<5RzTeFyDOtB+CF>pR<3Ru21Eu+fe;|Z~YNvDNSe4!g?W#Q(_-(vwZNs
zsoBCf>pr7cz)!;jMoSh4#sCM)1uSabCncvm54~3GI=w~Gr9JmXqRQ1DsR>(dax%Si
z=_r}O=H=3{HDGJNh0ZlT=O3PR>5)2pGU4<_ZZA)^2|Z$c94r@(vh+?omd<+S{X)4?
zgLof_hr28|kMXnw+;Gxd!Fe;WWpZO|YQv1ujD<IK#kiAP9DNS@9+~g+r7!qy?C!I-
zUH05`-4l6lW6S?Jid}OfMJs&{u*uAOWE=PB=|j1=YjV@(Zl1Y-GpJ>e?6f4yTZ(SC
z7W>Y+dYA3x#Cw8bLXJ02b~5%Q{&DRpkYe4)(QPBNoP$l7weJn*?Kh5(To@kaG8}xn
zGm1&ZQ$6t9T21ysj`{EJN<I5COS3a*{nT^1pY9bF2<D%dv~eQKrjM>n5i-YJr>p2C
zI2FusvRE7!5aYvhdES|mEDbH^Uiet76!)AWZvIWyw<C1#SNVM#d6pZ-$j!R%B%yF*
zqw>+(+l-dcshRF3eREt4x~;D%A2+yuTy{!psQ59>nR9u9xhw_TI_9(o&a(AuxvS!m
zutDzej)WC^@2J1x-EqQ&GqUaf>AlB1kH)!Qb4hz}LeuK>q$x@LH`qRLx*WMM>rju9
zE@Q+@juw$DMdfu5ZroH+?7bM{;ZW7H(BaJ8J6GpQt#Yru!xC*8D8nTuKH=c06USwh
z<~@Fx7~=Fa<kiAbjYpxEQj{+}d1sNd)bqGw#oe&JIM1+!CfhokdJ|q<?fu*l;mXyb
zx-Kp{Yf)=~cdQr7s$4eFfQi`?lK8z^GC#WW-}BDgIVF<epju(j8<ue2Cn8^2rzc-)
znR27!Im`71PM3|IJ@ImEg|VGqwl)dJE&JLS<r^0fxn!xm(R3e?K>v5{!8cvL_e?*T
z<MRJ+{Jva={kHR%UZpWPDd&F+_KjIEujI^srx{N<!yX>DO<=3L!?x^M(MzGIS1lZB
zDXDYh6m1hy7bN%`@r*SQ$xOSJ8z9o1(H;4N_svTe{<JHrGDRMqVBz1iT;o%_N0N(3
z#IsDtW!--kZBT8`|0CslQ7EI#Z$XYDLxo}_U&mC%jC+xi6&9E8-<<hisZ`~k)?l}#
z@=+_~UL}gpI-1-$FQxaX@->%^f~Dbkr#k8uZk{o(aT~|&oQSPQp5`7%XSF=$eNjVK
z@w(L7TV1QJ#oA2g5`9=}$Nv04lHRUq*_D1zzg$>%v_<>ttSM)tr2mL*-0Cz#=D4@q
zB*VTthSSteWZqzZq>>pmhb7~txmSzZmFNe$sg`j{R;!Z#Gd)YQ{&e}s(Um?jOKfhP
z%bb=PS){pHXZfV5%OWdQo{(L_^(%AU5`|(Vk@<@>`hR)MedWpdsg<QPiu<Q8Z{F$H
zV=InN@IJHh`Sv9-DRVBI&UEwso8Yte&Pu;&A{xuO;y5OUb$-6;k^NER+ldr`nHipw
z95s(DJfo3%`o_cHu&0l|Cd%u{Z48oit;l>CyL9nGzs*Tf3+sFen-&-ZYPYFmbHv?<
z(mLj_XGK@RBbS0>r#>D_t~ebe=68B-*0O)8J_kbhnU?vTNRM>k%e^zDVCK;Yj}{d@
zV9Sr>OLfZ3-I?WUaD0QNdw}a*mmH>Puaf7<9ebzp`p}`?q&1pMK|01kIsX}BbL^R(
z7dtJVU-f2d-ix_iIa?I7PTcb>lUm~QXVP>g6^E=FC%6Q;-C7hDDKKZMF4@QVKjy@i
zlmOSV>_`T`zMXS=i$Zqhq_FQ?-uvz8vOTLOdN>}~kfmza^QMQd!Rg_y6sf~bN=s(m
z6`MA9W1TMdyOS|BOFLTP!j`SL=B7JI^U1AAs;REe+jC#%3a};L$bBGlRc%*Hx29k3
zuC+c3Iiz!r=ol^2chmG|U9s=ov4^LQ<Xdu1W|}_r#H=aXg7l&?3$7epaP5e}-$`4R
z<`_*|WA`aIzb^V$ZJa~Zn`cRibC0k*Uz%5zvyy4)f)iUgPk&os*EE;?(#q#6ACykI
zf5Pz9|6`|G-kVKuYV(*QljUf3;iH$Evy(LUtmlbGQj>bpw;$=amzLtgxpt-AxxOb0
zJf13~uTgg^T)*;|_NsXgxHx@MbrhE~tW3#xrv6u{@CQo+*X+=Lnzy1Z-JTP-MP<=H
zvpy%wHCr~ly&o2}b>ExUv(k#6<sX0cKDQ(GNZj2+Vi`;P4xXC$+TqKaWhRrN_r)we
zAhA2lt#6hq`_BiQ_t|cjT;jest;nHM@b0bW7SBF-#^zQECFh-c?vqz1ckIcH6MUZX
zhH=j(hHhZ=H+XEbX0zCPC!-C_*-M$)_3CA`8TGHUX0H42D&x7V%bXjTuk<3FKe2N6
zrTK<<f0bIn(fi(Y)yDsxBFpZt%8op9<+oJfp%r|O{M3XWJ<`9|60n8wwvYOeO}9GA
zGHz+DzIpDW;^H_VW515rhsTXrrhj@T>dpAz%38MIta}~VeaGGl&Rc(qt!K@_AfC4p
z@AqEX&)4%fPVIR^U$*A`R6owgO6w;3y;I|Uv!>Fsn4x>0-`$r{CmavGonAKYmD9#Z
zD?<@yBj)y%CUTFJ{k6sZ>z#R}B3is)e%_3QUd9aVd0Sm?-{x4=Dtd%<nyzhLp<eIj
z8)-78ljlBU_5WbwWnJ93ttf7bi~e1|7Rmcd{Zh1Q)}?!FRnhzv<GbLs{r(w`e;oa_
zO<!q#(2F&FX`8xpE!qnw<}jIWc8TWr-|=swBJ+o}+a@q5o0MvNVxF&hhRNti)^Smx
z=aX!ovIxuz{g|~_<&^hyM)wIk(RtHcezk7%+2nk4>5hMAT@K9KndP(W(kI=itqI#U
z`rMIQc<7d=Qr?t|b4kyX+&&mQ+atvDvM92nZ}O-12SuF@5^Men6}nG;Dq8F%-dy6s
zZJeBV++Et`o9IT5{8P%Ow~8BY7I&5`)H-Ixzrf4ci*rh2Jad`kj05q)WeL*X_Ofn%
z?UxlD<i0&=g6;P^(z9KqW(T;L=AK^YmU-tv;B^)?!Oqe(8zQ{hbW}_By?S)8D){*q
z){?O228Y*W9R4P?(R6j0;SJlT4_BI07C3Tl+4A7vmj4OU8zujkN-~s*PjL{R(NH-3
zK*DSXc9n8Yjt1@ugZh|7@q3;wHS4>7q$9<BEoZXJyQP~RdHJk)+cs%h(Ic^kmp$iB
z{B<v9?)No?f_|1O^Lb2Xh2%Z|(3L!mspID-_wJap91lF>U*|X%n%C<Id$%1sW*Q^@
zo?UE(y#JiyFhk#GCu3SVT5tQ^%v%uXSfs2yA#Strl9Gl(@v=_&Z=LTs>X*1Uu8BG4
z^~yPpW19zC`F*xj{al|DcN~_lj%0Y4{PV@5YhgLbECoMYF8^$syhotHX$y<$`j-AJ
zNlRvlOgVO5Y{TUE&+gm5P5sQ?I9+M_X){)Bxkq=O<pz0W?kHJwg6IE(vpxmP+vDxG
zCrtiP%<WJ#`#@2-1E)hdyTRn1E0()=IV>u1W2ljG{pK3$_h{ka9q}Jb>NCvClFcPM
zlzKNm-=*ibbpD1nrA6t6?3u|d!P$Jl`={+q`r7cKWwOoOysz`L4{GIY*d<V`-C}%h
znd)hg2?r(0O(KQ2y!)Wnwmgr^%(AKbY3H)MlT)`$x@wvwWo@dy`*@Jz-8a(l_FVak
ze{9)a*?F3!B$;K~drRr^_ufa@)=y&-cX5w@<7=?Eqx&=4_9IV|69RlcY)R?YS}LCF
z#`$&0ZG|9b7D*G%`Zpr0`PltckFT1%%kPehwhDJdn#}a~CsR9rM=KXZZugkRUHSix
zUJbMG<^nYvugko7(|?LFEbv~wvp|_^k;<tAb*_ZRI}7jcEV{h2`25czW%Ck#wc^(u
z@v|Kta(wrm+F1BGi~WM)2Bmw~OJ$RiK5cu$Z_BpaIWB9lcU9zzx4X-CUy@o;<K=Kb
zr(I}O`LQ=_Pj?j6JWro?K(spg<fqnO+KYvB)(hq5963;vX6x{bjbqAjn@`Ig-kcTi
z*mQR5VeaCSw#FO2|J8f=i|KjhrQ5#GgKo8c*84g0^QQU5hwNFVH(6{cKG+dl{d9K4
z<+u|nd#41kJ;)VWz?^sY8Y=_Cgnut4|9d%kA47)2y)UjYTH8vub4<0~)%xVdcP^8M
zPb_vkvT!)x{Xb1*>+BU-ep}x=?tc4f&H4k=IKTNcDnz|@eRo?v$b^|$tJ%LqLEa$p
z)=TzNrWY>z=DF`aQ*v-7clk+ivrc*QdG_XK7Tce`zw?*=c83J(;`2Wpc((QYD1G>$
zhP|LcEZ5-!Lkklpmk5i-3I>M;22LTS1c?KQN7{vyxi(Z>d?dygy~IP%z_EcvRl;-H
z77JG8Q<J28xB1+h#O-4wp1!Lj@|lufw`u;RFOfm%-nvVoHVQeMT5@rcN3W4>)TXnp
zW@3@9r7O2&1a${4Pdbw&tTQ{zI<qhLdXcvfr?_ZCsdvyxHa%&UoEb(Ra^v<^|NM4#
zwnKh&cata6Hj7OviAS^lvuc#xneiauAd92l2Af9a>1SkjOnDKsh$rJ>s!xkrZdJ+4
zi%d<EbRL>(_}!ef^U5~shwFq`J%x05m^PZbbT9}pDasw#arg<tH0F&R2X-EP!DN=H
zcyf|Ta(B>{@2s3+zdgHU{!DvPxld*Jf*0P$-n@=;n0|<9lE+P>Z8dd&YkJFMTPNC1
z{;)Y?O<0!V%Bi|6#%m0xOXPGq9+)!2VZk<z+x<(pZ{G5E=CS>AL517+PJj!y<{Ss6
zI~E6Wx^H<sSkxo2yW?W3gy-y!Cc;(*4>!y1zA({U*o&j+guHc!zk9cjK!PWuSH;>b
z-A^WeJZfZhnaB9F$bx`I7QU#Ag&hsq|2q~MX&MDHFz9rB&}Et6kgd#U6Lng7Lr4_o
zk?sx6=as$}%;HqH`#!r<eSyQ2rk=2ULJlj<B7SB$xOoRnxwuhA*5FdNQYe=e<C;%X
zwA^l1gfy(Yp^&=D>9Z8K=6aSD%cn7!`g%86eO6j=m{&E}!Cdv4Q0Dq~UY|^lB*pFu
zFgWr+D2ySsRQ6iL+Zj`HHl$1M5i{aeKD^cOpkK@bqca(&&uqRJ<b2lT>V?#`ORj&J
z+#9RC_DKRa-?9U_O>x2xczqsrWWTsy->$4@`Qo3iaH6SU&<>S0!&&ckS(x5ZvgxTi
zyIGHsMU-J_-_J)!R43Fwa=F_3S!kxp<ljd+4ytUC`2X}kchu1t&m8r$ny1WM6l8iS
zEJyL>0T$JY54wvEN>wdb$ZAlPncUAh@nuYLu4eeM$bA~G(nXaw-d<;aE2xRrV7Aic
zGf#96^T=H8o9KSw*1n5x<?}^^Jj&<qIdNBQ<`qq^ow{0&jU}v=r8kRg|8hyeKs{>C
znPbOYf)m*UcVv8eeqhxeU(4845lmb4KFxS+ZP>N)(X7yUnrflZ%-!mfefb3JrW$rR
zaBYl{nh+ee>r`yR##yIAf*6(Ga)tERZO!-|9C>l?_fHYJA%PLTO3IvZmMqz?W^L6!
ze{;{Wy```FgIK<A+GPFvjd97@(D}97@06Zha^sO`L(eUf*^jnymU{nBh;L>3!JEKh
z?RQ{O^PGB9ci%ISlRXWN8=TR7qr{`!7q5~i%b1bKy{OrHv%#0S78hHS6?Ep#3n~-5
zdGWov?n9S1K0EfF+<Q{V^9EC3TAQF>E`NKTrRb-o>(*;_JV_V7=`wxRwgdYgv~g=N
zo)Ov@X1ww(yMfZGX(F@TTvsJ(8^*nk>fCg7hH#^Aj<(V5sYxkp(pOLPEKQ95bt+Uv
z;cJIS;h&2WbUsN{Ec8`R+wyp7T*`XU+K=9>FVDLCIci}nYt$H?oW!T!ZkXoj*eo-9
zhPU62=^nGx)!&>tT@_(&JNbaRhV<kEY8t8w0!+_qFPU_6<%h*yK@m$snU&@(-ty?b
zX3*Qbsb`ht>Z1Y=Kl40qleJM)On;H=QIACC=o8`JIwF1CUY&}(-+awPw{7AF-?Z)>
zS{FMHsp=JNG>m;1bKfF-`+VzT-P~VS_wFrF7MfV1$;j~K`T?e6bE>!m=5x8+a8-CF
z-ro3hz8L2VyPrw2+_O?t<W^oTZ;O=M)eyk&MDby}L)gWOUo(@ZzTrL<5a#l0!a~i2
zor)1PKhNyh@W@`TvrE5P%!Os5Q|52aS-xx1Ce6B~QsZ|;v{mPkV@~C2=czwurT+17
zJ9zX2uUQdu-@R*BEVuP~T5DaJJn4(n&pq$+eXY*kXZpTKyzp6nl$3|UUw4n=oiBZQ
zvpEl|xbNin|EkqjHSrenU9Af?vLVHi8_&miUJB?tnpwT+voP1mrjy?)K5b}KU309-
zb;`tflT_S3-gWKybYs$)O`BpfS<eMzPL7?{60BN#`tzPkU1dksY`(I2Ufv%kfrLoo
z^%7a$Ykh7-Wre<0ez;4V^^Ch;>B`gFml>+NJz2i7GBb^P`@_C{KaNX3Wnkwxq^ZF4
zIa7ww<ipysvx)YcT#j$c%>8F4aK^C;v`%`UW>a_fG{^DJC&joL7FQ-dJY~0J)gG7S
zVO?3Nxj(fx|6dZqpgGs8zD6S9q|gba6;mU9w(jtl{UqB?zR>&Hrih30i+Z~^OjYtP
zoV{?{9FIdTs<Q)gTg;X{+W6ljQ|0W0FrDM(Oq#14lP>ibce<|&d+C*QMX0sy+fqj_
z6`u}~B3GNNBTCn&s+7!?c{Z&kRwcP*-QTmpLZ4O%buo%5AAZ8&wZYg^pmOHMFQIE{
z#l2S;EEjM{&+}|}QY`B#D{*jc?$bp%5t=8IJhawYU0uG+>GZ^~r6E=8jA!}(n(`=e
ziQL}PQ?lASb-Ev@Ix8!RZMOOnmF@QI(AA{et#wOv4wk4dzx76G+1?#X^S+7Ac@k3k
z+ovgX?;PQRu&Cum@3-E$%X6vzvr2HW5JOAccPXobY)u>HUq0_r7v5C!a2><6weuUd
zrQAHv#ywTN@`CRa@ohp6imH-Ni}N3IStRlQ=Hxi5S@PF+n%k;rZa?}?IH<crO!>pT
z{#3(jT#V9IOJ`(to@ZgXJe%#b3eUd@Z8J_joL2B7+C^4+XF|v$BNfq=6>9?TtoS8q
z?)UXkY0cS*dXtvy)$~+1teSZ2*9=t$rE|v}{a#OKonv`=ug%Guy+6xm{z=*Hm$x8L
zZ3VNWMru!lI#<*R;av4s(-QB!oK%yjpK$tToY$i0vsH@5o+k>#s-7?3WOJozB}dBC
zlR-<TM4R7RlA)Qu{mGfOAC}(fw-qw^OZjL2d1|n7Z>XJC4qNT>qS+SyoA>>E`LuSO
z*_1sQhgqZi9k}-F+M3^@eEy}-^6L})`8Ibl&z<xj#%fx*(f=vFlis{_^m$%0vFxza
zwVc%HeS0JX)8^H9O?rPv(?(`N)3HOf6MxwKwUsy=!{ip(z4~`d^^xSm8d0t_lU$x^
zdwksGEy{EGaD(bP@okSTFWYg{$NX}$Q|J!~PX1X&i!9vzb2lFKb~aaw4E6BX<T3lI
zyuLWcfgZ7oiywIXJFY9JQmC2!^OPo2+=I`qCnxAIIG=wS)5IWt;-H0pwNVGpp~YXH
z9BEeLT%_i?IEU#aLyJEPYrvd7on`A<&*+>zapnvQYoMC<CbOmORm%c5I88N4y`b}S
z<DZ^QO0O$sxlj9)tW)u9i&5(4B?^qYmRGMh|7)76%DRp&wPeGh?iKI)xUK#xyua~$
z`;Bzd1CLB&d_Q<FUAvSS^X_R}RqDsOo(YRiU$61CtjpzT%)N1};AG9&lQVo0<ut6A
z1QxiRS7Ynqo2bCJ$bo;UgYXsyVWk6K7OnmK%~{|#(;1ydx4aI8P1Cb)Ipt^fTsQCh
zw+Gy>j-1iex}Xx%CX&ObnRaYb-T4buFaF0c9WXfb<6iIEoD)m;G>I%zVEDy!Y28ys
zXQmZZr=s<k9&THrx2YxW)$<&$Gd&?^a;+9TTH&?o(F=*mf%%v6z3%v%-SdC)<Hb9J
z$N$t0cn9be*1akG!9Vd*O6Q9Ld%g`BDSR@YmAE__+qw>Y&q@@|@_K&iqw|rEx*xol
z&b(H#`mf|SiBC;QP1(Rjex=`Jh9G9a)oDu(Y{|Ny|8Gq@AI~|JrHYKcL0+r0<5mW)
zNNT^3r(eSMc8$~2YkEO70g(dEv+f+asBl=UdR2`{GxI*p9oN*k?k>Cf!A<Qoht8&g
zxTqFx!S{22sLqq(w_sSIKHu9?CBJD+_oqh}loD2)G-%0`JEHOR(C0t=4Ei0i5ev1}
zU($+fJY~k8{K7!X@yM-BIhkS`*Y8?uqs6vJp-UxA|CQTDo0JaGn)51W`s}k#>SlY~
zk6BtC_E9Nkp~$v|??qhV?3<19KJGb^{=1LM?UMVGoF%y(f_*(lj@GFBEqmf|qhIae
z)BFhKe`ziK@6ug0{x6XF*uUn$1nHmtT6ZVy|Mxb_M~N${QMRUEEF#hD>7{3S@6Bwr
zekm;J{MIb!-L&n{V{>mN)g&gnwqkvc^s_gPT1aLq%;uZ?kEOMODK5zUtlP>dpMt`y
zPKRz2o*H$5^?lBL2kmoJ%Y{YHZuB{-UvwcjX{BtPTG+bB?7LH=C7h!tT<9)Yy?x09
z`|z`B>n`YSDwz9W`L(j5_=2-OOSmo1OmLig*l}j>O)Yg^P5q{2&Ac-gKf98`o89N^
ztz`A@Wus01^BF6OB+{%NT-0w+V2Vmz(s0}<b)5~{sV@d87Rm|Q_U*TAo-E8+6qlec
zbTDb(6<?3bXO#kurR03EbNcA)^xr!z?ZWf6^J)=C)gl`81$(%e5_F3;E<d8)Bi481
zN?Y%x<+%oa?mZLpIto25Y{_FuaJS0)`)gVLqsOZqCQJFVNLMo}e)IR&W^%n-WWJDp
z9h;zeVxGRr7yAswim-!9ZVxq+MO?D_z8rY8An3dbpW*r<ll3f5<sFyQl{KlJW48R=
z5@08)AiP?2!{-b4m_*pmsz#i;VE1}c-Z8ZoK}#8WwY$>adL(shdlI*2CFgD~uI8z8
zm?j<jw0Osx<8hy!&z;46z3!5FtI;Qh0|{4BKJ{GM)$vTbx^LZ{Pl;V;eebl)H~SJ;
zevTo1i;Y64a6@|t!(#vE20^~BZfU6r<{oWY^FLYmh0>%euXT^{D#!7+7+tV6OS>y^
zk~#XaoA6T4=tnl8mz~+pcz)E;Ts1{a<$(CqR}T;Pn<#zmJu}5f*VwZz<=7d0&mxw8
zPm~KJvQ;_RCw$VE_AmCAR5FzmyY(WO+a!@EXB+?aAi2LH0m2hh+LCm#6wO!<eBV-Z
z$fN1`v`dafmz^07FVH*Gbj)Glk+nh5Op_E=zW61WEV0q@D34uK`^JFdmg3zzY*t$S
zbYXI1tcX&2y~*#~wQo~oj-JoRw8?L~Cva)w5kLK7Nm^B$2Xhp|ALlZiNVxik&+4?Z
z!+ZCgJX6#~4=KuQzb<C7>(mFHm7<9!Cfuyzbz%*O>gUPa_@6s^{pm!t6}#KcF7r@g
zD!9fUE}ninYwZG~#Hz{K6T<@TKXUbMI(4B?<V1q-F^4@h7cBd9)7%ShE<0?sL0NFC
zS(}Q&*DIHt(=IB_2~RfoC{&nVZKWRgU|L$n*ThIuMRC!}ERTCx0$cf5ww`ccx^}|n
z@|V0R8-2AW`#S2`-AHxY)#Vo+_CF@9{$ywx<5vA83(6NK@VGH94ankV`N5xlUH{*e
zB_;11Cpd6TP~iFy5T(qtn?uyi_=j_Z@v}FNn{=GA>?M@n1h&L78M<+taSGkHsjTG2
zChm|!A95Bk+LkWjI%>-`EvMSQx^RVPnMZj=du7$}*DezSUzpkSGqHUAucVb_pp~O|
zaat17jt75#Iej>>fagR5%NB!3?vi@GDIaB+G}O(bj!P7;y=pLT(sQLLGTGOQ%EX%Q
z9_lqeUn0D~N+nTb!zZoNuhZ1qo6RFmt$N*{wLYyxua7g5;ja>t&Y`%C%vI78ne|V#
zG)+m==HygT;Mz3Zb7#5!tum3RkM7Mdv8i1t#w6~b#^B{tmVYUK^|kY>Efz5Qy!%(B
zBQ5G)|2@CrxA>1a&RoY7xHx8r9!soTQ)G6!TO{vFeR^K?tWPDqIlfvm6a@}6*<NRu
z{;hZEPE%!`nW6>LcduFh!7Dy~(RvG~N<~Y5lNWwBKCRbkQ%$l+jdDqL3e9ot)VT4#
zOfh#t$LC9m{WBccSQK}DPMeUlc~<-82KF%b?$_HERWeLhIoBkzNaETCiAc^6_0nc@
zsdAsXO+F_KinPU@WJPB62lX1RKUv_{<o0V<+Ly(5y(W}T%P2b^E3N6G&%~J(#`Vc!
zlUrj8r@`}GQ)6Z}$DaM_?9*Se`Krs$P61yfzMbBmr|_<yG`()lEcap#l@o_Q++kK{
zKK;@sEy1_6?1lQ9eba5GGp#XfDKA(a8pbs>Ze^QDPX=ogqgiu-g~GH*F~`t%slV69
z7Dj%T>eM%R*swMz=o;guC1EnM6*IqA1?-&UxXD!i)30gkB{yZ8uWQMCwJm{X!L+QO
zfr75f{dE3^R)+pqe<XR1CsW%&%g$ddr~Rt8=w7t)U-8{0wf*zYH#vtoVy902DztEJ
zNyk*B&Qgw$f;6SsKRP1=nZ%mZ5AX1qARX17tFdxx(FBH|`%b?9ba%~s<EyuY;jzW0
z|4R1rZd)yBkns>~SbCr&lkM}m$lcX?`u7U;r_9Q<ZRywE6_(}7ZNSld<n^f)%zk=J
zEvvUpJiYj<1IJuf>C<vQb<g$cYjvKQ#E|$ha!sx07L&?qxz+Ywf(!X|w7t64?me{g
zUDXM>==IKX(k3x1XZB&Z(HNM)c+}JJg_|PhrE1gJCl30|ntA)fgqNq%|BDzrxo~Xi
zwE5qb7B5lYvS>Q-_VGT=|83>R4{WNwzA5rhxxRFZ=ud(DlBYwQxCHXIdM^6*>(L~w
zW9}baly~&Bcud`R$MWo&&D-UBr+DnNsZ<YrJI(rc<ow;jNw4ozq%8|As0easozKcp
z@X<Le!jb9Rgq;@)ip%df-sYO+zN=VL_8Cud)47UbR<Ef1?Ynt)es)@4tjv+t?P1xY
zQ(|g+x$+jDu9J}7?^KT@ku45FPmUx=WtMC`T{Jgm*KD_<2C0>fDhsAnrQN>m%NH`^
z*=@Tc-8Z%`xY2+2#$?%>-G6nsZQ{>f(syXmJD|JBWslHPE>Gj5)9=K13V+jjv&kkb
z^y<euHhaSl=O1U6eYAVc-x-$Y-EQu^Tl`=6l5oOwZH{#Jh5T<V?#9kqc4fA{gv89n
z(TtmK&Ntc7@nM7AtK{|D4(yw_=j7xXZmA0oW#dlPmMc%GS#*q{IpD=6X@OJI_w3nM
zZGDHydYQsPhusGZTdFv1HV5BR{SxDMrBr*z-6>X2%ib6*S?8y?*Ib!Dw==Ny{<Scs
zdmGj*kUn{7$DXB^HfG;B!S-R7$wZzd3YpAy0W<&2jJl=RcDKB@wt-*b#x%LU)a#|3
zarOqWjlHuL2r)eTesq%cBIPNuH@DZ_e7LW?vvz+h(<x7-_sid=NB3`DxL5y*Q2n~O
z`&2G^oP6uNtw<y1;N4xaQ<PV?tHjtJk^OWy>fEmB88>Y*YW@q{Gkh|sQ^|47&d=YO
zF5PuGAvsT3>Bj^Y`E~}m2bnRmBBdTS*F1c`n^AFIx$V4@^J`9O*L@c@e9x^~S^a(G
z*`9!o|27M%tnS9Wuv>C^>z$`N@4UPsDHIvE`Tk6uHO6PxY<rco``O>hQ2V1Dsiv%q
zd#7f+e67S9dSY9MpvnSQ){rd58GDkrCrnzl!~5d&w9D6R6Rl(w1UMKJ7A!c>#KtMc
z(lkNwP_u_hPSVwNqPxSGSkxR&TueCDr{G+6$D=6maI>Olod~B;!>Q>U4M%($RMLDH
z_yYM-Is;OV&#^WaQ_=jK>@&f=^4F7_6VjSg{8c?>?s7dn%{P2=p48^8+Ts7##&7Hr
z-PyYGgdxX8u@LPBUH5Hyf?0x=;)ypm2|0Kfo(^YVpBTd={^Zhj1Mfo$mZdH|TNbde
z2yMGzbY`+>aFC*IxLe201!<=g4U>=DVM}?wZ|f2c)?5qD>}%|ka}=+3g}%L&JZ;~-
z+=%qV`^>@&0(}WT3hpV(i^ZLtXnE+WzWpkn1_dV}PR}JN(>Jy5o@N-feZozxd4GNw
zKHWLbUw@x&@RpQq2ku8ru$$TU`Dz|l<Tkq(@0fNRlWOSpHN25{LigCDMwzeb4b3fI
zkGD4RDIJkW{3u!y?;f?mNYZJ3p-7UWL}Q4wlSHCPQoMH2k+yWsrY}j-iwlprX>Yp`
zob=y4nn9I8V3%aukpLatt+yunShi0PNr?%$m_A$M<fSYP#^o`=vMXDJ9GO@-XH7L{
zED%l&@;Vtfo%QOHJ8n#omCuR;j!HU;1ZO5rjY#Me4U4FZ6Ah0nSgNYw(YVfZMRl8+
z)M5`o5fkx7CLe*IIB%(A3jQG-M-DC(ZTfb7(b+;3M-l0+8IkVO=C(z48*HnX<*}aY
zKvv=^v5=@PnWI_`x{4ovEtt80?e=t$*;|}uGgwc0G-1OTGwD^E_b7S>i(GeI$Xb5q
z<np%_9}7><+x~j4y2Hw{uch~QK3>-SZr95u7d@x<i?a2COD%uhS}N^RC|=KGAeP+1
z&*iZ2fY>LGg8z=wo3^lu3whnh-_PBwa<NaN&Bf4Le0@Yx@3KdyCU%`zR8@F_Y0r;!
zktW+5PxUTU%=WgGHeZqEak_{_WHyiH7C}FIvFnoyT-d^=I67E7ztkVcHG!dgaYUGs
zCg+u$Y593}%fC3Oay||4wCy(e;*}uJyUlM`=h;d9+M=pcyxp}`o?VnW`t!=1pup1Q
zVHM?5x6hB&j1pMDxcKVIMXkHSD;8v^>aBRxA|Lf5(&p;w5)sDf(HGb*h?!M9oAUE?
z@*3s$Ww+AjzwFXVS6FsV{H5&P4DFMwGy5*IZ{tkxo-*h8yh#PuVpX-ZnO^Z}ZMfLx
z^5?3YVQFmHwWQm3#qQ^2&-?%7#nEuv+Ln8*aXUYmzW83JGj*5Z52<VFcERf9QxbQ}
zOgvw=NA_*1L}Ty?e$8fash&B_sj6%2n`9PVXklx%=eG3GUoPx$T4cpc)=4@g8=6I5
zxv5z96dq}uqItHL<-$y}4UgO`x$a7|ZJE3vwNpQonRAlQ51l7FC-{aJclvg<tzD6;
zX?apwnyXq!$Y3LjYU-w2vu`~)bp1k#TH&OwkZlV%Up)1+t?E*`zpTUd&!7CdFca<?
z0ntfc1hrzLuUyc0R5ah9FS{oph4X=hlKsg>m!R3+6P_D{TTU?WaV*oh(fCE<crQ!8
z@>_?wnJKO(!h9z*cbNFFMy5;)iOiX>;s5R#>t7YyJTyIRap>feLY<SIH)1-KJvQ)_
zExEAgd8@na{aY^ELn7ye#`(v!eoWe6_I#h;u@l|(Cfv)ljxRV{=@-E!6=87N^vwQ`
zsR@5J%VaoxpDEj#=dR3ku`}Y)fr(q!h?+13W~OLP%IVXa{Y`*VvS_Bt6V<mZ%!((t
zCWde6v0zcR)=s>2&P-Uv<*l#Q9~FU(VFGtudb7C|l>)DwJlv<!s^FP?yZDXfat)8C
zQI*Y$T=q6C?<zVPP*{2T5zq0!{Tum<<i2LOaT{?NP5H9;#b#CS8%~-<QpNiJlEqkA
z3@*njJqez<d1azf!Gt|KK9&gR_;jbSDqqmK6t;<b;{UWJFD0*e!AlQC&9+&1YmG>o
zk@aoEr`n+>LnjxlULBj8TJ<g1d)^A6xD?kDk*AlRvG{a)M!{X*$jjz(TlY=XPvt!s
zb$jZyyLVjc(j*f+N>+Kd90}<?_2gvY$MPF}C)nJT8r6cmo;YS~T$i-rGLLlMEy4Uv
z2aV)&Ty;yXxK0<lvOv>OSI|~Sr8MfShm%f{cipt5n~qM<4D}LrQTVj@$~hljgC$Y3
z<_U>f`L<doDKcyB*>z`<S@_N~y?LG&GlG8fEUJkPnNxbzDOPXR#-7Y4%Ql~2-&Ya)
zz4p1NLzVL(v5#}NuhZZ?S-n1E&ylht4wJIpG*~$u=X&@hf${Ro`qTdxNm&2tKH&3H
zm6N$eAn2xFlZ>})T9W8lr$v2-0=~{zH^YmI(`M=_1wKO$iA^@EQeLlCQ$1SZxzR{$
z{f;HG6<EVlZR8}6_w6bB%I}rPsk{AHX|cn#3H`1%-uv%`a|tuNNoY0qVSia3&#uIC
z;8(c&G6T&ylYJ+oNM<Y&-1;af{-1<)-quY&$^;efsT`i+9Nk^zBc%A2;htkw+mbua
zKA*FwNzA#PsX8e~Xsff#$#%}H^G@8U9`)}!-P)$G?7G^Se<60=5@p|)p3%E5b<EjN
zWtkD=5+`na`D@k+_uRY1XQ!`y-#mSl;HJ2@ldkGMkUqS!REI&P-9I+%p2_c&v#<Vd
zU9x@8Q&;VLw^tJHwweZ=h!c^Oce~NR8+k-9;6#YQCq)ml(<RfNo_^u-XxD^^Y%1QS
zff-9aBr46ec2q5k(~dsFb!*4%h_`Lqci%qS6}Vx?z3<!1HD8@|tZANgbe;A3u4><)
zCgY{+Wlyd?bUt@l`t9wLZ=W{K{~~?e*6+>+iSQXllFW+S6B!Mqi!7Yh1?$KPUOgmV
zJ@=K?=@6l<GMjZO17Ga=d*Z3m6Su{+pU*B7+IIO?WqWU7pPNOH?!+0H2f5i&^WSQk
zE>Tu_87%6skoif5b*$>HEt198%(Hc#mfgL3DtCL)Zr{(dqj!gG6i!hxzZ>IMzw5F}
z>--OV+wb*V`G5D?oW|W1yz_#}*IHd~e{^E^r$s5gS6Ch9P70Z58hWFzV1W)p^$Z3P
z=Y+XdbGR*Bg)G+xabL`1d)v|@ntr|R)1p;<>Nytq+w)o9Ejx6_L@w=z$F9XG0>&zg
zi@*4No%QWq`R!%jk8#<3Xp}!VVP(d>6U)tahadZW<@4{$EAM|PU2b2%cWv&qx7HPX
zk(0vTXZ|^UyzTX`l=%zH-tiykDle0refFfnWYvVDS7J^kUn!cf@TT$9tq)$t?q9s=
zv&z{Y?Sfwggc?;?Jqv}b77MJN+7Prg{f~&2%E15w6;V}3)8mC1?bl>{#btOz6#^Am
zqb9^mDwB@?mMR+9q%osOBSZRsc9^trQknh_C8G!<6OXdAaDjizJ!Wnz+h~-%E3JI5
zS?sH&<uViFu77V%+gMPuuw4DB_?~C6pH1ynu*ElBFMkuP_#{OC_a^b`iQ-d&1h%%z
z)El-je$jj*p5$~<qdr)NU#v>wxX{w{#AU@HeJ+w0zqchlt4ed^dg4}hmXV=1$g|cl
zY3FyFrw5Y`9_QMcAid~n$F>I@y*oPM)jFfw+9!%yp9?YEAylKdSn{`+wclmQw_zcv
z#~fF-OO|WIpD1imx>md9VV6Q<dG{kbFSU*nf|U$n$|B$GPpSy@l*YFyvPv1$e`go^
zd0lYojjDMQn?)w;%rGrB>BzcsEo6;@-v33W4QDyp3`(LF7^<AUl(A0Hv}&63Q9;vn
zTcZ<{OHCd&URPxGR%wbWZ(185_%?uBJ){3~NB<(#{_heVn%6@brzoj!O55})lJ#WM
z_D_x{7EO@&sI+FmglU0gmNS|cE-bTIsHLuK{Z6&l-J*8Yj24~37PIH=xs2T&8BR*-
z46z(-T}MSuhY56<vl^asuQ(#?{H(6Iv~RDfKs`%fv_meZQI(Ql`?BLwlcsjAZ`X<r
z$lJ3~=g|#uhh~v$3uViKllPf-T1=YSf1@&5vNQ7J)Oq2#3KuI=UQX@y@K~_5zI#*W
zvFp=pI+}kk%UtDDEcbGvOv<F2$E7V^)_yV7Yc{U?|NeXJrtR`ucSyes*JNw%e*Vn<
zut@#KbO|pNxwi@2|2lfISGJ$JVI=xY>(U7k=`BXd%N$M^HB9YP+M!&!VyBSbcY#M5
zs&>i#UeinLKKJ?yD)5)8Pn_2D#iD=D&e?x%%w`Ihx%8yqo}~Vxk^NtHXg6+|Gwo)G
zf^eEjkhSWirYn(BKgB01CN`USNEYpEUcfF@k(u}AQ1ZvEVF%0Xf?vp$h*qpJ3tGKV
zUrIro$v`}&Lm=*i+xm%hGd@qMFLwVo)&4)DR+dC`{b7}Dk9+2cJLZQab{7@7Jni&b
z+@rr;d(DO^b0%l>7KSdlB76OC>HB9>V<i_&;Fz{&)6`bQX}J{({_hi-rmPh;t8m)X
z8M!V$`;VsUs3>*KWz5eMuSz|Z)OxV1uA^LS=k#0Ct>0~Cof0fylP>XfllV(<_g_Wf
z7dCb87IEM{<m7hT&D(`5_gj7clcKClG10<Kk!DAU6fw5Q-g7sX-n_vTeWAhbvVYKY
zr}-j6Oh)P<it^`NoAzWjO}^1zAJz1|L%?cz{p2f^tbvPmZ(6a>s6jSx1*_nMgPd~{
zpBWmo`|=*q^7v`ZC)sQMVx{QL#o5QDXFTk3s+`0cE+IZOZs{t;hnh*(&E>bU*~V)a
z{%4NZ+GWsIIHMu4O=qI!-$RA0jloTdm30?2yA;>#e-agQDLQ=e>b4g~;>K0~HwQVc
zPD;$o)Visq?5UZTbvz{Hz?7OsmRo@x_ZNDFb1u@=TBNdSDnHxGt7eiKrvyKr68bo6
z(Z^W}CVH&6)io_~RcdbMlv!+DOC!2gN?9|Bud4mHD)d`H>6PxO%rjJ0PS<?7;Zw3b
zuk}XVm8)gCWl9%wW~KH2KP7&FvwfxT!m5r<%G!P(jHKRQ;#!ia6gbIb|Ivg8KLRd1
zj){4`eBaI4KW=RPA~EM6=gRg|>s_Pg^!!@qRlUB;YmVlu6+K;ZoVDltdOnSDc00?>
zrlT)rF31WIvs^WO;YJ<r){Sjp&p(H5Ez_%Bx~;lAtx8mKy<pqfL-Y9`=^vQ2agIyC
zO9`Q4Pp$t)Mov7>S<G`v=(bzE%|^qsM}$)xRL>iFEiLJgkDB)8*VHMe*C__B|IQ^e
zvwQvKW|`Tmcj`pV@!Pz};8)wtFp2Y5)>{7RTs%WNUb;I;TWC{79*?%r*6&kv0;~Tl
zPWe00puMzh+N8pr8cPpVYwpfi9$&H1>xjyhM!^v2l}DeiJ$uua_lU@>n=@~$&N2+<
zpJJK!u0nm2S=lzm6~8rBG`?E?+hWV!L&E=76i-g<e}8(X>Z`PEk5}j$@9(wR&$@Bv
z$5$(u9;7Xe+!FeF&&6&f<>r-I?Me@?iHlTDG-Rr}f3kdqR&D5ta96LKC+6`PjylTA
zCuvE}Z@gKNFCujMTA|Fx`2PpFC0?6X9^<O)Z+BN+IJquFAXsyLf`@t_gY9?5X>)fp
zFY@Y5_+5EJE3x)qmBFT!dc`|;U)s}u<4DxcBMC2$1pV1E_w;@R!<n)d_lEOm9o~5~
z&SyQp+WJ3Pi&A$kP_I6=Y?E}(PG#;THGji4$p6&3o}OIgxkvQ*hI+}}3l7SD+bMla
z+kOVKCX>T<larq9TP=^D7T;>A`R>Mv@OH^FoJUT*SU=svC^NHOG`QW;BRqcDGCq-$
z0kc<3?wn)yd-loBEmq}Qmgj8gmfF8?&x-%M_kXjtz7xHlBk;8T<^BJ)r`_p3{dM-~
z9d}M|(h>6DPCu3-;WM>U;-TZ^E5~BAt?sSd@PD~h^@pdfJf}mSGJC%`>6%t(Clnwa
z^})SD(BNyNMCTcy8NtDIiE9o^M?9bIzI>_lMYEz(NA=G;RnBa7iQW*RR;3k}o$Hi3
z^U@Q^&gfZ5KaZ$&E#g{xBy7)qFQ1gyJL0^54OM3?P+?lU)LMJ_ZSmP$X{)2B*z;W2
zq<!!i+miA*mZGO?wLBH%rma@^80_Vyn5Yra#JOk9!Mb)HUCs$hbRGu0yU`cM7|ZJ0
zp!#}GqS>;ljw5T0&xY<WjJ>rlV$Fh6Cxwr3p8oQCUj)zoFTeNyh!%)`yPyB8$k$s-
zBmZ3Z_F?B-?qg2e2iD)Y*3P|b>YM}IEsmF4nx~zRy4H1_GhOfh!Dp+2Z(LXX*{pU}
zSmMsv3q|sFrMgT(VhS=sMb?Kj?}!H*1e<c6KN@AJU$dq{M<{BOu=Djb>l&*)1I}yT
zJg7P?FVs+O{>7=Oo*sH#$EHPCEnc;$`0m<8k%kjai6`k?ym0%r)9fq9?+Bj%efx;6
z&{WHXhm?+;(#@N;OR|mS(v*$Inxqe^%$y{!`uOgLQlFG%f(}Y0TBOWNzU-s6=gg+d
zdb~nYHro^zZ$AF@#6-0_N7Yvzp1Esk*4~Rhs=qrLHmaQYW_^0oowSd$Py4^#|99{0
zT#JT3yl1}edXPBhnqkvUzH_HD{-&Ljyd#(^Ca~npqLasVUY=ViduiI0Z8y2gomV9P
z-*n8Pa%EWh??B%bcHTQUOFv(hdam%b=-xS_;Frf&GZ`ne?$LOZTXI<OR<)tU&5MFJ
z3ZI--_FA5s6dEYHo+bI`wOtqg94_4<uYGM!N3@}M-`&^JI??B(GJ1PN|E7r)Pdhtl
zm+Z%j7jH{mxhvj$)^l_BB{3PD2t{eHU4nQ2N_iD54gB@^Owt~{-4P0jQ8j|<HK&Da
z(q)gRn)mE`aa7fd@9AYjyO#<#PF#0->80}W_}74U5;=Zn_r}Z3N%T9KwsGeNE|1@@
z`~O7G{9AjPsUdY8-|KCQU#*gRsKj_p|MP97*$V0}mYyi`{po&v+uG|2df^GbGw=2!
z-}h`iD!2RYbA|sY5Bh4)O#1OsXSc@*g~qq+BH=ufZgy-vcX*eYwV|xumME8?!%aFu
zCug@YCCCa+V0g#Z$I0-%w~fK%<2%6w{}1cl|Fkpzljr}9@56r+8CT~IR~%zFA1Ii#
z3+_BCv@lPC|GcFjztlpxtGx4eO5MAortO$zvo~|!#oEBy>a$PRcgSpA9WGt<oZ&;j
zl)!2>p%cf_3RTu$_O^N<BiQlOM5K4$=e~2F?e2ee&^ND6EdI&<AmOgyW}Wr_&hAt!
z{IV-}{hTFt4&7-J^}iE6@8hhD1L3F7M4w*E>$5}r+qB{zyBv9vK74J-Z*&utd7d-d
z_0Tcb*O3d3J(2ky`L@Pz@}Z{S|2O*V-yE+K<&OUMMaC-m{SNoHZHglGPh@`D3H`nI
zZo|zV6ZAj0CcmG|Z}K_L<TD>b5JT29%a1N%R`cX9&g-A))O{z*_toadJ}L7~YG+S5
z+^3V}_pR;YvuP?zd?sGnuv;VVs+^Xk=&c=s$Dbao39R-KsGs}j`30pl3l1u@oRjL-
z`_nVY@@)SL*)>Z-GlEXn^zQ6?&38`7ul`qdti+~&is`m<9)9`jZvQW{_8M#8KUu}s
z91H)g=ldFW_rtn=9mB^T5A+#&FP*@k=<vXyF^5CW#v^NiQwy)Ok<W{Q2P@jy%{gV9
zKP)=XEyCXSkz-PVTaS?MJeQM(OAiDJE%FeYq4M9+d4>?%A(b7h3x%43mvQ{q`DuY)
zV+>zjMn_3P0G|u1#o5Ry3j!B2&*odx<<Jn;7{w^_$7=F|u(cj6XIv&Y=nBnNKRR#g
z%5S+Xo3qbfvh^0ozc(|VeVHo9l7$bNh4k{2m^V&c8^si5nxh`_Ddp5u-Qd1E7pA2#
z&Stk0`ysLA@iFyCUK?M7uc^o9D{|H3PMYG*D9wM#Ez|my*Io7M&1`ai5`;K|O4NR6
zEm+GiXHq5C6d{wPelJ5=BgFG0v&t57rktAcVtVyc746bVl2_BdRD64Tm-}wc>U#&O
z=BZ9|n-RI_#rd^AyXABbT)UN!y2XsosAB<(YO{t}lZ?m$#|QrlCciLm_YhK0<~2T3
zAj0D<)L_PKDwQ#rKh&$Ur}5n6BumZ%$$kp6r)B$X;%sr&a%aqY;=r(!<%QsGO^XTV
zG~a~fbgX(UBc!jof0A{F_LC>4xE1&BlRmLVZS@*&g{?_K4hEZd2|du(U^UvTxKZW8
zqHP@fT{*VjzAP3tV*DViVPvCvN%-6~nM?jgmd=~c*!^bN#Ja`hx3Iw$+u*3&t!+|s
zMGm!iM1HXK4HU_<_LD6-6>QEbbakQc5|2f`j48jkI>h7GY*bv76zip=w}hF?bLE|4
zFRS>}T#Xf<B5n&Qez+Hvce(l4oSYk>rI%KUUdURRzwaEAEJt)T+pV&Fx&JvgZ8NI2
z`TcD{(S7Og1x1&AJMW%5q`G3R^ZPyg7p^^e!yn(yWOHF-ox1Pa`##~53T(JMm#i`G
zvXV+(EbJXpwyE7-DwTW2l8no{8yh~Xd;ES+cf*eNF_$E4SSJNMuxu<?E2{Xi&gRpx
zg;h2$J%U&l&E%f7^ifOqsxr0g$Bj%9FV1Lecbc`~;hr_#d!F7CI(_rg0>6ex)2VL3
z8tYP8Jk^iB=rvE;czhbq?LQo|z6UTyYxd@D;{1DlKl}X_-QYW1b=LWNYo#BnKA!RW
zk~hOLnLP!E_|0uTPOh%fI;lGK&h6A#Wq$onpXRU{7q{Qw$u+6eWRJ~1zh-Lxd3W8#
zC2!CF53w(}(8#uqC%@&xwh)WkZYG{8e9kH!uAO|_&U|lfdB4|HGpi|3L4hGbj)R54
zA>rr2x$SYkSk`Q>E0`K7=Jjl0O-#}A_bZvFZPnc-m!aR;<2GMm<B`i=YY*-h(N9^%
zx+$LF>&&a4s+Ra>&7b<=`keJISKI#<V%R2^yp82nr^33^!FvTZsk54WDACxdkj-N{
zVT-1h-%9I_B~8wnPOZm+mK+et&S2iN{oy)Wy)8b6XDGL7MDT>3lsX^YVK(<f;rc=j
z!=Ten<}RHHc?zB@PKr4k-u8=Qi5<_Bp6v~r^*Rr9+fClMNW@o2uVP|%G}8tpc29A~
z4W>_;O0%YH`Sy9{|N5;T{qsY31;rl;7#+Bf$T9ih9kZi{+9F;g90@-6F=^6%1Ib>W
zH3t<P>z+(;O>$MTpEc*KK#{0|oXaF$&IkWbZSgcZ%AXk(I4hj3X_|5sCugnT8J{2J
z6LinAWY4*x;>a^|dZO!t{E}jxWGM~zxU5xva~wm`Eq)w`>R2{o{>}$k3QA53SM=Ez
zxA1BQb$Dk4ZEl@>+E-t5aqFi9pA?}PCv`S2Zjoj^+J4LCgnwPBUkRIg*3vyb(c2u@
z{y78&KeytvyLwfiHY`x%@1CB8&y3yLuPl;HpTom>e}^2is0Rc4FL9?37HhV%f=-*1
zALW~=2;E)LQBfn(@PQ#y$YI|9<*K?;Pd9FAQ(N!AKaJxd*Kw8&hi7OsgiX3Qv9Bym
zWJw3V;<brSR0B6Tf3;MZ>?SJnZ(G`lq~1veCp4qt_9>kT%QKo2dCFVs@R6m_b!%G+
za(t5&IOZIm!)KpnsyzRc#6jDWo@=tdUcTwK>0p9YO5h<Ij|L+Pxz;FQMUh00w^B@(
z%Idr{#5M)2^x4F!V&=!xEWOmC_4199S1v1@Sp8PCyqNGRXWNFib0#K79b{|0!an~m
z%ZqeDp<~sPR+oIOdb>xD*{ND?gJ@pJivun07rP?V%o=PtTLrk4I(tHl*k?zy)@+vU
zlHpfZte&*2Nr9oL$bYkT_me=jPs;CqUYYoR_bjgpaqnf3`%-uPnw@&VX{B1lKUan`
zT(cr?vETQ-w7vGDZKry6rFW)gieXB~+thETPCFcRpRuV`I`fg6WZI@<4*Xud35m`V
zceCC&WLbGKILc)IHixia#t6PGI+jVBG_R?=*;DZ1)*@E3JJ$@O=J?o0zMgPp9n03>
zKkD|{#>rd%ZJJpbdpBJEmwG$bl7n1}dYn|&MD-S}I>}_1_x_#6%+D;FEaJT4g6?u}
z4cxaSRBfiG()Mbh<!?khyqj+NPv0VRhHt57XzwepzEw~EJC-ihOpKo%Xq5i<?34Ux
zhP?RKPp>u3`Kq$~cY@^otQVWatk$%%X**u9a^&T^ysql#e{Bwb<>kuTm-F%@J~NSJ
z+9SH`MwyV{RWZ8~!RK8+HU_Aki&U|m_xA1^562S!+4ZH5zL{K&Sf+QzA>@kw<xTC!
z7g(3=-tqKktywg~W2F^olX4|%7JghU$m=VuYI^Ooj-}G()oR`g=J>YU@mn<eQm>zy
z*i*jTMDC-4Oa4W)3Y`4)D?^C0!S}SKrQ5Xk?b@I3b?vF1YxZ6(+#$&;;L?|=pE8y{
zjE$P=(p2_*<BsxdNA2#T_g9}>ZR9)kZS7N^Wye%QikFD;eDulHHcq(07`?VwRxNh_
zuQMLbTGmXbLULSu4W`Vf2$}A=wN<j(W|6@1g6OHHHz)i^kC8n3T(AAt?f*_TQWA?*
zZkp~(@0tJCHQoE!v9uheMa$N%bm9JaaPdo%{N}|^3jXJ9Xm7o-qV~DsL$4C$QW^79
z7p*rFqAc&p3-<_47H(J`cJgq{s~=v!oj$#qrSd?ia!()2qA9u|u5)L~a)=t87yTM5
zqd0fJdBLRQ=Mm4!4Z^(qSZwOWG*#O*ym)$bzc{bFd0nPNE538#Th;43Zpo=~tjkWn
z!V$(;8QtG>>wLCK>&if#n`TS+HgMm}-S=46*6ow>-=LFhNmKvse*RcfXl8iQ`xW_G
zTh<14)J0_XU+R%njCiklE%5Z4wX40KWzIhTyev|(TCXr5Q-#C4N})|)#`cK<6K?Ik
z^YFiQPs75K9tZpb&ELP?ttC9~(FO;{S2Kcxq|!5|MsoLko+Nlen&bUPxrLwPPBDv#
zB=@dW<hklBmELH)+DR(>)~4w~vz)DF^$7E)CyFUwn$u~_IlG|i;p*079O6#J8$1QM
z?lx;rez0vy#i6MQ3U|Dvy=U)i_2PUS&B@rQ@$~l2eW!O9MX&Qxa$fM+{M1Ip=es1r
z6lE;EBv(9`*OcAAT-*NYt67E#tQ(_vz8_XfE#|O0Vx^bW@%6D)@kPTK!q&{p`>Z4T
z-4E_%o+N$Ma5rONk8<#CogC{M7N&=dI`1E_HgJ)6Jj*U%&&<NlD?|<Fap!Q)d@!Tj
zNa=a!=BE$4_f-6sTk-KgGe<+j=|1U7n>P!Xlq@aI&+1XGly8@?HTB&4z@zVYke1cz
z9*!NZlRFqTW(a=?P+s-9Rl|leH)ZXg;ssL@7Ptv_@CY5BT;am?Yp<@02!E8a;^)=(
zoH!kor}({I{NNFvl}W#`41f1UrpE>Al&iKbR^GqIvuVD-#%ANLSZV2)-{zKAMC8{v
zzmQ-KQfOAP*7&j7-Z_WA=gRETLfzlR+lnVzwk%lf9&%ipWs}e&@v4yB-dCJ_uN(=w
zJ@;Ycfo2cA;Ef$Se(&_<*tKWziCHyV8J&EOE$1!&DZyX8{J;a_Z#Nd^&1yI;eQ-sH
z@f#toqe)z+ojB(UDqUJ_F<Ik3e_J*OpGtpQf>`zDSqmLcTSu$Tl3+EHXbtn=y<nj5
z#F^vz#bX;LZrQTR`$2-GM+&djk=4xs(*G^g%0;9t51M7}>b=)_qE^JKduG$Z7N0pg
z6wPd`mQ^lg2zG2w>HVO6^q9-?BOJV6wU5|`h-rDPU2o0d^k$b#w(pM$ac1s)`7(+g
zAsT+d9n1??c$aX@J?*k1xbtwq<cCbo#lNSpf00%7?%CTcR=#`2A|Kc1T74&Cj`ndj
zX+E5BpxdH-!91hPNy{f4HIVf9xMce`jf2L!mqZ_4cIrWsT42M)#VZ>V-1I!B>7G>4
zIC%c|PWcPk{xUsoVW*b(TJSb&@P<XWxL<ajz5B`(Ph<BTK><GkL^eoj?_G3UrR4Zz
zX5a0?+j(U<bT`c4|LQKFx+^Su>uiCU-(6(ZsaTeH&6EgqR6OK<>V{7z2M2@VPZm}Y
z1||j_1_lO(1||na29Ey>tQ;aQvKBZt=kjrt7;gx5Ze=>bp|fy<$dPUd<E%3`-aV3F
zT*G!LN`1n#hEV1}sgRu;mwNZ8h;~U!1UsCaZc+T|j79JQ4aUcAZ?9P7B?LCxvi=d7
zQL?gO8S`W=m6s(i0vVk|ukEQcT;i~P)`^mk8-9zfFO<63aOFin!s^X&_oYH^Z&TG?
zU#>V$wbd+nbqf>Y|EI4%tZ@iR*-@}_M?^w$Yk$0wSj4I`2X{5FMx3cJG=APJq~E4?
zINTw#b+WJg8LNWHPa30HZb?~GuML<KvwT`i`lct2({$>cQ_h;Ky0rXw?Kj`LKg%!A
z51O7IGX2tyqR_KTv&(<hBwU@ore5g3yvCLqw+HHh>w+UEt9pjXM_(0<IDbuJ>yk}v
zCN+hBzWrxl>kw8?Ir8jABU9_PIjPfRk90b+a=s91W;4}D^y9m_ctXINh3)ek4#uts
z&SMuX4J}k!mNBLDc&9N}30tZvpO=>9@}QuLqUM+Tm;%g&JuT&0X8USqeHH3t>RZIB
zy;a6Ux_;Hl9f7e~nkMp5GebEi#zZ^)oEbNL=Cw9grYVkG;<H_rDM`+ma#1Z+*y+nt
zM_x6P^mKj4%8A+1bFW>9SUq=1pwH?LW_}&2S}dyG|75?&mg|=;zvI@Tx7EvkwT0`W
z@^hB2=1w{7d1{VhzfpMBN{LjiN1jf9mt1OBTKWIVY=$ihwmhlX{L_gme$9tYQNJoF
zkvG#8h-Hhrh%L>0;E=_2fpv56foT_$Q(e*$T?DuS#M7A1R=#yek#d=sb|dP9c>Kd}
zS6BG0y;3HiJ?nth`?B!4JA%vA(?ZwpJEI!9e&c_?EG;+N-tAK?*cVp`9rEcsb+1t6
z=In__8uL6Z9~I7T(U_6kbBSvK@7!(g+@iFWq)uJ4LUdA7UE=R`ivpf4)i`5uS2KOP
zBJZ-bX?a?0ftz(*k1=j8nUu85@BY4&&yHEYj&E{U(>v$lrj!W}XElaTTHd<!b?Ej}
z3m0F|&<LE*m9wU6QQVRbH}^Z$mfe~+$tvcyMd#kaqb1!cA?G&j{D1Y7hPM1S(X2HL
z=e|AcaNoize%kHZ8aD@F-Q1mAwi*tOk&B-OXy&TeUf9}ctGh5eX#eYV8)o%1J@IB_
zR#kBmIm|J)vPfHXrG~d<>*H?!*eI3aHD_HuzG91IRbIJR<js}PpyFAwOIW1EmpG^i
zt-SrUGj40uS2o_xy#>EFDl=Gbxe#3UbL*AKc42EWGA6XGnc#LWAnW2*;}|L4*Vo((
z`H#e(e1GZwKc8l!Jk2G%OC1xW&5o<hmsBZYN}4?TRDz)W&K5<@M@^|em=5&3I3Irc
z)6}Od28ZTq9<hs#_FO(EecLA9HLaGNv$%g7Z$A>Lqr9}aLyc{d^O`ROBC`bi1^y>W
zC_i-ZQqE=$Tv=pc7PclyCEy9Kb(Uj))rH0K;cMFMUtQ$a-Q}<FFT#8BFSUlMKe{T*
zANoJLk$Tu{*AfT5DRZ9h63}gwSi!@V6e-Iaq|g|Up&)cH!R?UhiMdl27`A^|+-27!
zrdAus;M+7^qDx#kuryumyyv|_6*rl-T$N6flG#p2m?wH_rA)GST*~0H^HdOD%hXje
zsooJcr@uS$$zrkymu~L#-Cj(s<zbDSz7YrclIHcxy<D6mUVZRrok1|e-;Oki#g-G?
z!-{K?XDF$>Ozz&q@*+hd(KXX)kr87<a2hAmlc-Nmgi5xhWQrGZBv~*zw(|I@e@~g^
z@FMcR+Tw-WAy=C^Uidr~VD0SdwB9(&aov-3x+#zHqy%sCDsMg9)yA&0V9mqjb1pjv
zm2O$Ih+~n5rB*{1*P38e+n0Z`%vbG6JwB;0_>{YIl^o;3RR!vatochaTy=h#bf?}~
z6rgyizfnZAhn@4WYt}=<2~#hc>=EL6$@1dZs(<W#f|=8vzlu~_Wa!6mvCB6{Z1KT$
zyLuEJD22x=wM;av(witYu_Kjj*IZwNAccLGPPnICUb3xZ$>LDOrH%=mr^Eb;rn6sc
z(e%}t(3j-GEzr2eLBTt+)uPOD`dvRE`%{O{MTseAI=|7jJ$g!%@5wstz<%EzPrJ{G
zo8`{*EXzs#J!$cOyQ>P@L_PA1Hyw@booIH;DZ`E7hU1<+OPZhAaGYX09+O+iy7Q5Y
z%&BKr#OyaF+A}dGNPj!jDl|`beqBy5>*6Vnj10c3J5Oup7@5z$!4SOE?&ZQm^CojD
zaV^-lR%dETaF*GhcjfKvN7p-UQAqeN6`_!HV7;1=;mN;B5sLqGQyZCU1dkX+C<x2x
zHgVf{{pmV%n0498hW^E?UNyYv3|hl#T{X*fewE-XuUdJxJL1WA;+}MWxpDHIk48h^
zmPaNfyy`PboI+RsYj^05J|idn_&|0bqf%wf8M*HqGqY5An{%#qo_&-R*F1s2q@8n9
zUtqN$w^WPB+KML<rV|hKY>@n~%U5=?<Gu9+pA^Q!CG$4Vd3`3`M*sP_lD2}mjBB6I
zO>x@Pe(&>aTh*>W3#**hk?nrHFQd|y>Cf6b<82=62cNrKOT^8NomjTeGvsZ{#)8as
z-nwrVN*Ql1)O#Irr6~FC)E|y4JKvhDbBkHKCG5+Y-9k+79a?-39SykP>vn8f)2&IO
z&0HtKmt-_&|44BxHxxQjIWMQhS2MwZ`@rpcVIC@MM(;Fuj@;4ZEZ^j4n%B4>n&Fw=
zb_S(|t=9e07uu(0dN%%8<g=EkKx5f)k@X%0^Dm`$vvn2*it|mBl0UMI@BX<J?zzeJ
z7m`JnNF>Le*}?h8XNpLi;JaPDXIL$YN~0$If4#|R8Bf^yD9K3XgLfRcoMMY2RbqJ`
z9h$!1M$kaodq+*8X@B&_PL2mN4>x@)-Zr(bq_H7d;qAQdbFc25SRVi6ZiM}Iw@vc|
z*LkbIn%Hq@$8otAr?MP3&XRCmtQf1|6>M0$@#>cx!E|5$+jFl-&a7A|T6D$dJCka1
zW_MeZ;{K;og7vTWu6q!7?asqfR@b`rpMIF`_WDWMcY_AQ9>!YV^XzkNTz|&&upH$M
zn4@#L{>jnK86Q_oypfV?mh5oo;k<c*4$cQ(ZvS8QPP8dcV}>>N?(ciw?adR&kCs*6
z+3)mey8VQ^_S5e1oOAK`&1=zcIq7I<j-hzGjX=Bn*NI}+bN}}(wEWjO-Tr38lWIS%
zaPH-|4vW=I@;F)XSM-rc<&&dYm6LXUNJ$T@RQOviu-x3T(@^05cZV~die9w{7Eco3
zOO(tuQ9qOx@MXJ9&lb)SQ=<&E684RpN#z28?1KBv1s=5J9e<uD;8+^9Tv%O$K~SKE
zDX~O~v00pvRbPW4fPs-|Vzc#%W(y9%L~+*0gmj)|5!u_hRy+7i39~zW#iuwuI{t`j
zd3d?=!g4OgtZ)v&h>rR<6MXlZiYtX@8!$U$PchPOj@hiH^RL;y=Xy-lq3j*sEaxsS
zn&6yM(jh$YqJ@e|_`ekAdBLVqp=qt#Woj12WxKF>2&C&J)~Ni*Rr(*%tf9f+slc%5
zdef%z&L8Ch0TWoABv@I>%91BoE&7n}S5g~tBq{catJ6|frUeC3iP0N6%H2%6H_j;E
zyrOQ$kAmvu-8&_+Hf1F9Jn!}pb=%8e!FN%QIg!65T#C!oZ$-I4-6pr089HJM>+L>E
zFlwX<Y;;djWnk72NW5CawlSr1qRF(+x?+W@*<S@33k4oFc`$uQ4Sj5>yuI;oVo5N2
z>CX)3bK<PW)vLFwS3hrK5nz~LRWX67U;>M!fZdD<wjL7#1SXz(-v4g~>qLW=bal2x
z3(ESgm91^H+winy)-u6&n_DZ~eYdR;-fEG&{f3ap_U=PBChhw%Nq1sqw208_|IX5D
z(i{^{*q_&k-TrKHykPsIB}Q!vvztGQeGoBj`k1X!(tdv{tIP$9V{Qu1C*_<zmYlC5
z`|4;_&1T<r!%lX`>gewslPsE<6(=w_HQS%4-oCwa-iuCGkJ_a(l5RXta=OZ&_=Ml&
z=OhzOt$NGeqmTH@ABt`clztmvkzvA>;vkcjmKMH9edc2IkkWcH7ln(9By_gRGVjot
zt*I*&62?`SD!NG5J6QbXA_4!+Re2)0Ar@Rl9QkjLbtpZmv42q=SkSD;k<Yep+TKnE
zTaJ9Q7uDjP`8jDVvX3VopW<`&h)+%C<O?jVuB=&&FNLmF3Pfj2I#x8d>7|g*QLcCF
zLTmo76l8wt7dA;aZJ~HYV%v)?^QK&n5v_2@dpIS_$SEeFz3hiXS%m;&guC<soo+{k
z)F8?KC5nu!3P%c!Y;Q_22X-Vf8G9<QS~5nKFYAasKKu5?&N(Yszf5CIQt0|Ot;OG@
z;OmCk)fORw;tF>v1+{N>Z<?{l@L3m+c+Zlw);$%A4J*qne|7KP5ncDwXn&~m8a3x<
zB~rULRou-Cy|zM)y+UAqrn~O-UTyV0``|v^hXR~}dbbbt#dI}D8_v^IGR-t|nBc-C
z_@Gn#X2<(wO%p2wBGdE!t?0DiSS}Jd(PhR&X-h6y<4pOVQH9UvR9Q~iuF-m{bMY39
z$-8bW(*Njc^509|W|iWVEd@b4rq5ZnqWt@Gvt5ESyH;3nP1g0CypKWtmWqIH68Bez
zWtS$3^D#P9G5HxgPjP0eEL_+YKhd&G$^Y)v)cN8$Nk<uj4NSToI@DB{Xl+_8G*Ljt
zWl7@-)vn_U)|dynx=3_A>U^ojAYwJGw`1W4wi?k_oskX+@={%W%VsG5D!aA3D{jhk
zE3ei$U4oijE4N-<XEAY+wUxk|2VI_7p<5Slt|?HJZIoDiKzzllDSCqQH+ro;USy`G
z&hS@Ua(1MD|H71eGm7SMrV2-g%?tL>e`vW<li^SpTZ)06*%1%6+ibEALzWmceSTJQ
zldDwuN40om#_^RCrEX62{2$>{duq;%Yje2UC(XN-85Fd_W>vKID<P-dg4+%>?$NMg
zdceKrn&6scTlO`Cc-<Cvy)Et@-O3rxFaNQXX<^TqjwQN{F)@=is5MK)H%^JZwcvcl
zye)2_^8^JCuuT=%IA8O)gt(w?@Q3-&o*F+tA<meU5<Nj6M_@w^w;<bYVWpBCj8~ez
zv?X$WTlmMUYjH*`^KzeC<%`xzte<YR#Z*ht;MGbGRRv*o)!kQBWL^j?pDg@-mF(<U
zlk`sRv{qwy_lVnGEM{q-lxnht?&S62n>O57;m>|)NtLx_2-i~IQxajXEcp%!%w9ZI
zU}7r&L_Hhv{%hI-->2+|u@uR5XkEk5RrO!WY{Ij$^{nN$%_nWSx`O$mPvDQ0Z%?gT
zRJKBoalf(TT(dOa+Uxs_+*)hP3(~Ie?Q59ySDPVF+HLy<zNHWNQnzGny2vO0`M}!%
z>7T2WnD4CIP@R1J$)vvxSr1q4Wn%Ix^oW_ZJVvEP@XW86>=~wU8y%KST+MMweD^AW
zsvX<+ORYKNrBE!bVECt(iN!%~k8n+*@0$x;e`dKxx2&6TYSFvfJ7;AHJXS9ZVrre2
zR`9r3Z0%|R!&|jcb9z`@_uEd}Vtb8ij)Hu`m%0qGgBgFex|eR<_gm<Vz=o|2+NXAK
zwlapEjdGdFu5((k_x(&I?&|FZIeXYX_6lnsZe7!$uef*b|J%ij3ilf7$gUHc?^L*H
z$uH5HOXg1N65RK|F6YiMubNuFFZo&D7e*OyX&via^T5Y5b^n$X)7@C3-zRtRCvyK&
z*b=;Y*Zg2E&)HkpeGdG4z#sgZ|DA)+msck*1%}*z8Nw7Orl9U+rFdw%_}K(A9e-!P
zGpQ13ZZl6`bokmb|M0i76AHJVb~^FUBCKw;2XnVc{~r6$q*xCxuAB>JcAs7ld$Uo1
zF?YdLyTCU)#qTU`U$t*>_2M79PQ9PZdFptl=&gmT9r)KMR4*<#>Q_^{;YY?HCf2Be
z^}i2y9h#w-x%48(9j^ZuY6WGEM7~*M@p0F@?n_a>MQ2}7(R;h%oCZTpkm|4h56!N&
z$NoNiyw{|tHl*m5Mc*IED-UiI9jz8@*&}G@t9U<1;N_16IgWa1rUD`#Hwp<#ec@23
z5t7={EV$0`B)cKo%0C|W!+o!03OJ>6*SNZUGuZ0BMliyowP)4}UB;Dcjjh7vTXu)?
zM>VhnGFYtrS{CKPAK1W>?2!D`fx9ZJWsO1I8i#y|+`{I$tZxEB{uyv_Ea2RFfq&lv
z{twqxW^T*;_ke4O!|8tudUo-+EqcrK{yN{=>1i*84{TRe)Uds+D|!8Mx9_bd0!z8}
z)&|-$3${-$7t5#!yVR2{XR|s?D3>jei+9T5W0!i9S*7>g+WvKCyJpc<vC}(RJk0ko
zIcfhlmD%>8RqL2tT+ia#RHtnXJ;Bl!j;>khue1K0)-FxYql+F?Zz%|=$qf0spk~&B
z5OMd3Nd|d4uk*i|;Qj6}mtd#pJ7qQx1FrZdeEEWxbYC5KD{$#w!6rpr1_8abZw*4e
zW}P`5$Y1B{mZ)a*C?$>YwLH_7ZIwDHDYrx!rlvf)q0=IB_>RU2eZh@aC!LdCcw(t$
zK;Duy8K-B+KD7IMkk7&Ug75FGOo{B<64=)r5GXeg2!6-5E`j}{b@DT9S*O+Iu~$?s
z@7$@i?Zz4fu1V4Ss}Ee4*c%<a<$BZu*Tn_Z%huS%zNlX8@ZjH!Q-2M(0u#~$4{-ep
z;GW6A`nuu%2loT#{#ObFXLCQfz?D9?(CaC8s^G!w+!bDPT0g86_%eIzjD50S71pv%
z+)~ecas~U@%#XL<x}^Mcp7-TTWfqg;_n-2Yd6ypiH6?xPIi;pGr<U%vxOgTh>^#>)
zsl~PP8iMb!J>=dx;a)%<J5QqG(gOB%22Ys<4{na&e{<kMm!#Rs#S#2RPySzX)Qk1j
z#gGC4NA4Yu7ydc5C1N6Lr^fW?%lxYo@)mDMT4Ny8qs1j4z^dxSEvU&AYa;MPfvZ!)
zZk7P+tbmXt2G-Wg&#&>-Z29297r6EL%!lV(KYp8_XL#+Y=HIS4v!8B~=r%d^%xj%^
zgYc5PMAOF06)WBeS>27%%@2)ux`gfji4Df_S4t!{>WVkG?-fqjy646xiAbqprU!d$
z)b{3Hw2W)hb!OgV_C!0n*V*=3a-6n^+E0(yx+Q*fmJ(0cV)EI<g4n|L2{3JNnCX_a
zC}HB96*Zi3@$G4+7<xnUYipk-PkJ*!bN-5mRy~0?3s{{rHs^9?ar80N#_s#;z#Z7Y
zFgs$-q`&Xg7e4#9flpw;hgE+<UN5Lw9q=R3U5G8#)mp@Cc7^L4HHQBe_?9H_FXeSR
zz~eT_@4zO*BiUygU#+w2)U*?w^E5U8pyk2+)yL<)&=Q#a=l6R516wz6KFAb!Jz46O
zVdcKLeinvzR=#U{$0?M_B5`Jh+P&7-w+*MZzFG49f5>aL!oQ47cf}8KmIYmw%W;f*
znDv$C&8Cm7+Z61M_H8-&PJs2|*ZIL}X&)>T_`hX*^I+KMHRV9);w>B}Hm#D|UG?SH
zRbhvGMcuN$EF2ql<W12CTjb(;k}2qlE906i3MUtNE?D8S=GD*S@}SeJ>V$6gI$Cn2
ziKryK__<l*fc)OcfhM^+5$oeF1O|FtOKw#;rB$zDc&&-aHRNhwz?PbxPZLB>Xatzp
z@?@<S(~K5ayKk%NfoXfYvvaquy83R>{<3WzQ$;>p%DuY1l~>+vPDJ6O1^JxZu{sCB
zwWHT;>sb}LBX)J*T9!Ye23kfBE{aD^-Bz`0Ll)z_!2N2TY24Ghga5N!XzR89BsOVV
zw)C^PGtBe#QVXW<^xedpb9mjFNfq65ettQ;<k*BgenBOP7uGUwFUpN_SbI#RmUD;d
zgFbERb$236i$DH7y8D6sWS0AS#gFU$R_R`4Y4ph0xMKdQ3`y4>pG_SqOG7=PR9CaH
zdj&9T&RUr+veND4dqp1IDArDn8;q~^cWiQS;_gstbeJ61^h$MukHCqRJ^z1kuh9xy
zz_C`mY>w1n-5!=7f}FG3E;YHUG<{KZ)(~)La^B*h<hsA=)+ufM^M4v&sdm4*7_`5x
zX5)$Z(ti(6-1Uj8d!mZO-Iy~DC+}FD7vJ^9(x1V`^ViXNJN}g%75g4lxOl3Om*tiJ
z7Dh{MFzN4zQf#)mazo+l1`pxIBEdHztj}x?I`N=|`)<Z%UPC@N?ND>RCgyj#w<|XH
zc8BH*hcfu7=&tSc_GVsd9p-Uzii)4UQ10$SWlQsf8O=7j>aVtuP8Ct)W?s*0eMR^H
z?@JG-byK3>{ZYLTG%efgyrO=V&tV-85wmU`Bh3s^-&0bp^DfMp^>X1tryi%}%UA@I
zH*EiPk)=z+pysu*M#f6_WtsXysz>MZC98K|U^sJ1b<zz%W!Ey+h~tkgaCB!LKK_M;
zN59PD(t26z)mD`o4$0Y-Cmg<ZXSK<{2QF`IcFo$mT5J#KTHr;u6<T?g?o?=g8I<qb
zw>>Q4<eUHJyFCuh5>0k=F58{po}7E*YJuwS8#lSCuNvGJSbWIf)}?=I-#qcS`%UTY
zA?q_cyXRcKZSC@QTUK<+&Z*L-$(^A*@2?z--YjPwtN1j)*hg@|lv9t54{C*f`f^;}
z=HQy=M`v$sR^SS2TBItx*6z)6u|Nd@hlR4N0<C)%yl@QUF>vS9?DE(WkomMt_1UFW
z0*-1k+-|RxVr^o)th0WaWcHhOF<+JrmXx)V-v}O$v&+&vJKMl}@)C(grOm6pJU#Nl
zjWKBp$G?jOZaycb>+RZq@_<D<&$I0>CUwt$vdP?>XBBUUmNV07hc=-TUEI7m3G6Hi
zjz=CWV_|AI$I`<lUcKev|ALg{v)5T1GzH^Yf||L*INdWO1XmqvJ(T3i6S;X#_k={@
zty(h0^VAxGW0$mVeR$xO%hxT@jC$=G8+)_uJm$u`x<yV5oTD2iHcQj2y)CTLU;Lhv
z<nf#DQ%s%3Qx+LbUG-(LW<Y_b{f&iN`%)c6vRp1|$vj%7<Deky9hmHX<iQjX$2B@q
z+{p*UrYTOD;E|TR#Oth-u%WJmkW%#(CJz<^wplxt+JB8;J<Rc0bwSHSm2Z=;h_7|B
zKQ`fs;Ubqa^S>BxStPnvWa49&%RR>ClVsc(Zm*inpjR<5_SL27;&+ycoLFG7(;-V_
zd1CXXf16y6<P?}3aq+qFAz32BYZXuW|HEhbCI`>hCh9i3J#lL2wKl0R(c{w^X7eW%
z{akbD!Lm&E7GC3>#=eK=q}jJw_NPZBE{)#O)Rb0mSo__E?cFAwQ(8B2SSv8F&az=q
z|0~4Nyl)d%*@Wd{fej9C4@LBw`K@tCh;l!0ZK@SZhLW*Ept66-^ZsWu*VR}jzL>6e
zY1#L+PCQR`%eqEv<ZJ7ESDQC!@*kGA{R?zj)19^W&#rO0JZJ9tDftiO=dEHoV9!~g
z*CZ6OkgN27%f!S+UK2ldISuXjPeo1Q7Z;gxA8+L}ITblm@J9SD=JVTm)&(<lgqZF&
z4u4j1JUEpj)KW0`$cdWIx@)-3@w0B&R<Q4AW0`Tz|CDQy7Ke=1&zyMkR*A#rxwY4d
zMM7PA^lv5h+wNS%{n79Y+vy0Ys1GlC1s+d4;o>M*8{48`yHPxaC169d?$J1>U(>yh
zJ(`lS(MdS6p-JKz;}ludC*5BKj%rM+mgu&4QWy3x=;IWx@Lvmqg3by>>8@}ySv-*`
z(<?iwYVJ1QCtJQxORe?Ruh!K3DVN42op74L{$`hMK-c>3j(5sJ4o6l6O<wn5&ojPU
zW500SX)3wd(z&}d8N!z49^4!G`c7Ht_U#et3~G0GJhGW1@#Wfw11oF0Z3LZXer&m6
z9JN`{#VK8MvDgavzh9a{cO@`gscN6%7#TdVBwb2li$<?()GD<%|2-BIf6SI$H~WQ^
z(yOUm4r|JJPr2+ZXgL{Za8uX(kn*_~+wc0zT{Tdfb<LT@On1gpex-jOSm!L4dfvqJ
zL7aJu>%7>9U13`mp6_s)63*(&#9q{}Mrg?c{oMgfx@J)fnRl0k9;uq>IsfX$zPqX7
zLb(zi)3<Uv%A_T}b87qVmmZ<1(;F))eE2Bq>no=!r}6H*a6?!*&Ei1d8<D0Rj9i}!
z%$&D`ZdfZ6=_&YI>0!?lF7e0rxAWXE@V_F`aG3e})XZMdd%jOIt*?sRh`FM(oqLAZ
zx?VwpwM^@_e7>O>-WU^h;j!byo+rCCPRUnYXj(36s8z-q?A3B?rNN3jH}}t<W&FQM
zoUP%@L2m7n-6pY1%end1KXvVCKK)}7%Z&X;c7*EXU5ip@v&>z~KhJpanbg*08Mh42
zJUk+H_Q={@2U4ZQ%_Qf{n5o@%ZLeKfTc`2q<jJpePZ#oUW!WGjsQhvE1<sh`+>>Q*
zJPFS@Z>1(7`kA}@&*y2JUy60#C|)^H!0D=$B^;=?sMD~+$<gnmkAB5+P9{?|<DxhI
zy06aiMOm<%T>NH2eU?t?EF)7R_OHJ+RX0yx5p^`Cc9m8b`_`+4jK2aBo;F1&ukiLL
zpQSR*p68kV+Ao(>s`g22>8(9<S<Gm~jPQ=Y#JG>EBtq5Og8!#(E&3(1>hOcKFj1GQ
ziTTC971aKxTS$NCO?vF6ed|KpjoJFQMco%yooPGvv!mH=U61Iwb9(QLJ6(Epx9Bn$
z?^(#x`s--v#4d?{DN0=BcXk)@D>eAIh&!tUcWE4*Z+vJ%<V1Jg9ZA=&NaTE9GsBU&
z|C5#e`XZ^Xk1W4i_59N6Iq&UoG~qy3v_tn~hdyhiCl44jKNzphn08*l^!yHygpF$s
zCo*4DV7(R~lPSc%@bR9*I~LyGVafJ+XZH+))jLd&6m8&ItUcd*>denP3qJSlHsV?G
zSaFholS-81#R{cm&X$+EyNgfroQv4GaF*tuoyJ!_HJ$uu9Bm-{XXDI2zbEwtbhQ8I
z7h2iAD_Qo%#9hBCyWb|Q`rkZjWBugGp9ADiF5tFwp4q)a_}2mBoeBKRS$oz6E!Zfu
z+HuZ;Ew{FCRxNNf?s0Z8nRLU%!+6c|i;H9zEy!PWq^!&2z(t$fO$yr{G3~j?Uv07T
z>SPh2D8*3jY1bC>rAAv`Q)rFU=6U#f$B|!q=Wz4I9kz^7=?s-rN{&8w=mGBqi(U<p
zX-Ou$iCemYym>Nqb4>bu@alzG|BQLwI<I}dS>~g)Mx*oe-!JuyrDgq{ctR!}J1pUT
zctg8(r0U06v%-wlpHOs&j9ULfVb0MVJE}sof6Q7Ff93ea2~2tgyFO^l^0nyQpRp+2
z$514Dr%233CS|u)7v7s6j{00toV3F-#&hcbj?2B=%A5j|-Jb{S>;B=PsN=-r<+kME
z9@SHPsUq79PpqFZ`Ow7;?cZLtoMP;+p2fFcdg=bnlDTUn4_fp5+t_8>$!)u5mBU8X
zwZDwl1u2}EY~#DwhHKTH?H6ohyDe_a@bcTeXU(1CtLF5v|Jf7FxQ|DLljkFMP&03+
z$-)x>X3AUkrLA5rkbOFQjqA}HijgM!j%rN3cbh5ZPLup4*VR9~qYSrfS6CCMIj1J%
z^m&bGhcXlywR_GaIbIR)es_WM(`UEv4Khz2EYd%;E$7P2{5KZ!7xuIyyDy1U`0C^v
zU8Mgcz~M)g!!NDglTK%jeDFG~!5F#bEQbkC<d$W+GZ>6k@cic$ndN(6mQUcx72b|h
zYj!>@@pz^q-LYn?_#anUk%n!O%RdzGos#73(_wz`!Echu#<e=L_XlWQykWjHYtE_@
zlgw*o{FS!9!eKFg(&2;ILe>|Bf7@8>xZ!;C@~WVMW_y<fMiC~CJ$p_BYr9^tcvf-t
z*5ujFJ^?ohE+j@@I8-z{cb0y9&%q}%T#RNk`PX#s%V6JPW9XIXp=Q*h{mJ;nmagEP
zduv-n=bW~TFzMkGV2GP-DkJ1C)ZG)N(;HY~ce#1dt_@wLLR<>Yy}vc*nM|=d|7QCU
z4fkIwSQ&KAp6HZW9ysNz7GK`UAeIsb&O1BUn+_Q8%AA?KhV$%2<Fks~IlccEuAH@6
z(KOwfM>N>)diCieE7YEE=?a?BJL|OVjTIiAf@?ZnFZ0{nyKN@_TM>p}kL9g0yX;sE
zOlo<9PV_9_tn>S{=iv<@pD(WU)zvt3c(V1$$^LsejvU_X5NYFFXtQ36nL{hUE&Isn
z*9|N7xO>iaaGB!OX|jl;HAMN=)wO5NMn7WWG?=wrs=rFYRnWCpsCH|j<F!czis3EZ
ze;C$C&$YaFXIY2Lp(8JB<*KG0SYi<Om{)2mSE1^@M~}U7d`+)JD69_H_WzWK!rZ<G
z1+J+pmtE$tIju8cUI~w1ZSUtD-3ui3%XDTLdCX!mxe+^aN9CI!E|<&KOfFxG>A5*Y
zI$Kxf|Ggy<I|bENEu6OhLC>Q%d{0VtX1onO%%KyrbL*UD>ATiSelpu$9Wi*lr$g|!
z-HtcYGB?RwU$D^QVbgn=lglGd_bEFbRnS>HQ|!8e{}d&2JxlrHf?Uh)&iCIc{AY{5
z&6ZiKOhcDES;Y{0b)(T1N0R^-Z|$E>0q)J~bGt5ZuiDc4IMn0Fk)GrWt#3_?YcH(U
z;n{Wd><-t}KGhedtqtuyVjFu)kwd6q+b@rniP}2_7lzol2E`}_|G9Z1dg0NCzNZrc
z<sV&+bH09Q$JTH`yVxtPMX?v-4+ow*9dqFU&#^DFjv7q6a#S?ca9UNWOhHZ1=Ly{!
zz80l17N4~av?ohFnqcw2b!r4xOT@{b8+&=<@)R#m488YsZl6(t$hLsqRfb`YWE5ZN
zdWbqXGBeMX)8$RO8$A24f&JPGn=*CuUIgs(?aMQYXTKcB_~3-V?-Pe3uAj&Vdue^5
za?!H(i~Oex#B3F>uD-P1;pJw=bGKN3t$OssWF70B?$irBPeZSqxU(yiXLs%G{jPRP
zlW#_*#x2!uG`hi%*ret$Gx!nf$^DLj5zHsA8lPO~Ep<TQ;)&X@XVw>Q9`4<D$CcyT
z!lOGZW!3~v`Vk``8@|1A&5;?7F%x~gZ!S7+!(+kAt2RyK`cskAhf|8PS#@tL|De^n
zSX2LilkdG+^{X}Ouj<^6ntJ<s#R27ivH#n)`m&0izkS8G@=LVm#v6B+JdTjzop<~8
z-3;yX8HYYz;1+taIyUf*NSgYh<cE`$Tf=|xxcWUQvyp0^GCN@BojJ}oV{_Mti(com
zz0cmggZne@^NRj`npS&$dOVTvY`No}wkO`Yl>4^<Z(Q*Db-DqZzZN>X+}^d--Rp5E
z^V$nRAHD9Z*y3Kj<(JlD{j<s2uRb+jcHGph*PCU|i6Dpgx(5&U@}?gPHQm?CdG>)v
z_d(C@gX~vAIbC*X&S(%+=8P_2KfvXG#Uf^>(v9TAJ026CAFf^h)PZkFVzW?A%yyTa
zpdA-)U)Y*+_v-Ig54#y+xnyIX?NyRLyD;!a_b=Z^lZ0>n*PX<);^rltT!+oS_Qc3N
zzOVcE{?Z+f?>?BQ_HxoOZ=M`hyUWsI%BC+@=uFxnnh~6Oto3>6z9|u1ygRFwh8aDw
z{kzPYEomLg@(;b&Kdj9tjMDjhv{%w+HK+EY-k2-=((y;G@IGnC*wu7By^&*&pj4bQ
z-wK}RY^$Ee{#&?do3!E`)lHj{O--M!{+(fRG%Ppi_K^h-I<F)wT;}c3`{1aBXyv}%
z6P-8Rm%bF-#<f50dF0wG^Rq8cDc)$!d$^(T+QP*T_v+j~``A~8_ln2w%iB48!Yp5G
zw|KQ=n&{WNXO}<Qvn=w(i_JQoC$nbO*>#D$@yzADuiLx5jkl|tFZW{Ze;<YarMF)1
z3+0*U))Oc6Zt7c$&1E5bWa1n3q<7Z7Japh?tmCxj%lHoO;dr%g*7*l7H{VH-%`@22
zc`lB-Z}GnzUp-Nd>c|^Al#2H~+x}vql2%^DsjO$2%zJO}*xQ*UDjkv*i_>4bgF7j(
z=kd+I`>g(#@b6V*WY$;Eaq0`?e4g@c%bG0;oadsq9D4RpPut+>4$knj2gkg)-oNL5
z|KX`yJm*)n_PGvwo~GsLC9kg2@BghIdL@!OFaO94&lh~#mh7B*c<Y?P%*0?5$Ineu
zbZ<0$cFki~D?A&kbMJ}idDYJI+w^z83{Y|J`H_2jSvGIVl1Y~ee2&a`xxHaq(BsuF
z?w&p|DdPVfy`J~MpS4ddWqn`xJ=CsPtJ!Qv^AEF^*2lAL^k1$v<FU5qd1JTl)1~D{
zZ?q`YCrB;vxaq)itZDg2yKR~qdN~7_7!}@on&(S~^yK|IAs1t)ZT`BnOXS->*JaYo
zX-Ug`6PnlBneV=sKZ(I)omXC7Ci8QLt1P}-j0*aK6imLy1{ObJVD^74b>A!J?h%{*
zo?UidQuaT~)hkR+FH6=h3$E;q>6)@DGx0#-%R^>mKlfg(6Kj~cMW#UAXU4MwbL<or
zC9`_XlzMCuAGP=>*QPLO8M`ft;Tq;Ij&S4_3W**y%H1jWE@qY51Xbxnc0JyK%NFKr
z6!?7hkwUXmX74wLivO<)TCE+X=<H?9_59pe#C)fs-#S4mVXo?ThTyC)^-YNhPi>QJ
zpXX1tzkEm}sz{#K@jKs9Mb?~8c`xTY>EwOIx#C6Fyf2>Xd6&MkxITd|TFUcq-j(@x
z6E42Ezw4EIP{C(s!$aG`7JTzs?((={=F!J9L=VZGDizah%{Jt7lJ@zo(VS5BAR#V7
zczXaZd)n);-`wHRJM;6>rail;P}%$M{gDSxh3X6b$6fTeEp~Uul>YT)DtX^{t9fso
za6i3s>T&yh-7`)~oA;dL-?q^7bfKix+3$RQ3Pm4}ZMZ2=n;71+#ZxD5FGt+w@+%y<
z-t7CLHog;m`~A>{PvZP}t+H-YwLAY`l|8oe*ZRYiFVbvmPiJ{neLBMPq$b~S!T%$R
zO1iSnZe)Gwv7j-lS(fQ(zy5+3(|0^7XbtyRuu-URn)v^Z_CCsfMYFFRUw*N7dfAio
zCl}7L9SgHmdU>bm+VYY~3{_HBo@}47VJl;YCKDT1hU*7~LoKN+69fX<d^nqU+4XF0
zs3{)nkuuMcG4^86>|-|P%t-fk&}~UsG9~kA+Je<)F>zKR4_+nAoW0b-LMOPXyFriH
zddfo1F9|C<*u8h`I(#8fvn$4;Ao9hrtB3hwl*$Au12UX8#2wEQnf-Bv+ZM^QlR~{O
zymi>Y^kmZ9?m)e$T^#*dwbm0ZI~dhz3W?0QSP=97h-mgP)!h|_iA@cX=5Aa&g^!(N
zGT~QQx+7%v3bySz0y~~QU}KD4<}<_k>lTK2(v}=H9#)EP8rKHDK6WE&u?S<33Qvep
zBA2+<CiWo38+-Ih`a0M*3H}sJKCy8P-z1+Ku?(I;Y`pa{r9!{O^w#BZWjHVMSvfh3
zb&h`V@+*h-NtA}2krdy=puf@mSZJOE+kEZ3Jt4=!N?wY`rV4m<2sB8CdfaSLdg^+i
z=yd9p#5P4P16fD$t{+olEv80HkJI6MHnrkW;UU-D3v<gBXE9x=c;v8L_jt6E7l%TM
z6W0rm<ZcdE*9)${BcEJt_5Boay;NB0N}r>cC*zZjtf@g)9sldkEDQ<f|Kt)ZnHl#~
zfZg>j%atU~dru}axR^$=h1v@q5p(CiXByz*^|AEA0{x^X>2uq^t$Y!1L9r{ttx@QT
z_)?}2t(QU5l$OZHw;tOjn4YY-EMj`o#1qTImxPI2oiuyru0>AFiETICws@7jc2u0|
zbWCh--h{Y*_KVY`#9z*36N%c%y8E%i%8y@k99RAoi!C@Ps@^Chw4v?Ja!s`-i6uc!
z6?e-xHyWKPd%A6UtoF^+BVJ`|CVZJ2F7BHy=M*{Vl6FnY(n{{+k1IVUO3vm}3DXR}
zIZ@4z`|I;#P6_XhCM=!lS7~@mGJDU*78O00M-ggCE?NuSZOxR9*L!tbOZo3%B>g7h
z)TbXsZZ1VFGTY6wU4o}EZRrx&p>^`vyAzt-KPGx=C|)@>ZONV|$NW4v*(B>kL^Y;F
z#;$!aIVwglib;~UQg+I<S!P@rF;V3U=SO%7-4b(KktI>yA+6SOy}QhL`jR`<?^JV+
z3pCYiU!leO<@0Km(xA39hr&&(7-lH0o4NMF#Dkpd$1Y7v-om``+T!+|Gu<xaDs}ql
zI=Tp%FPV5;_BiVsiQKsI6zP*TsuwkczXZOMp4xt)RFhZ5rgoL@#eD6*8`}k&CtjQx
z_F1Yk$K;`iQGKmbhhKi}n!D9sU$kp6^CnJm?7mX|Rik^)-xa2et^4lB?K~@(;M6LW
z`1t>#W$ngamvK*+<*;n)#7xc$9zhO5kyT{^tU@l0{IfjVJT9*P*SW&#r;vlKK$GgF
z2@m~vBd@w}h=|Y2SnKE&d3(v3G|i0*3^dxDLT;Y=w&<;DmcbmajxdH)bBRf;L5fWB
z{aVYMJ|ubt+kFvhb1Aat&=rjIu{m*tNlkKN$dkf`g)y#L;+FFto(nZ@k`2h4VtMrg
zi-*I*lYxH5thdhaX{=<N#Ne8;wlN`F!7b~q)b$f{_Rr0zQz{hS$x)=YX+yX5<%QSZ
z&2wSf($iGoWyRZkiA8eD4o>IV$oY%pw#-oGTjL_|dkL4O#;L$%{Zmp6=d|XDi}mTW
zxiojj24o!;`!DReroOK@G5OC7@5Vhk=DW^xXbC$xv|M<jAY?F!q2sa9+_&v&%!1QB
zq&_*m6Fkx+v02T1RmjrbEss==7<)+jO`f~)VT!Nk!wYw(1#es{^G3n&O{WU)wMk5>
z9o+91a@JjXBDQl&Z`o3lkT#2#x_jrcF1Yw}sblz0m*7*WoqQL4LiXJ_cUI=;!k*w4
z4^LcLrOI^7_}d?)H)h*gulZ<~Z2B3KKkKL^<B^NnSB=yh!<M&L3mqxFGl|zpRpmmK
zpGT~)(-X&ahuc2zsCchcxaZkmWYOxO<Lvs7<I0@?gJ6+0iDfT1orI2TcoBJG3LE##
z@SiWHK0R@?MnS6Mr&Ff%#<2g#rZW7MSX0vOBKI^unS0??*J3TZgo_^T2D7uz*oivy
zt?gTCF?)W79^3V$ZKWy-H#y9@U!83-nV=HWpuqRqL{xd&3Z?YW6HU89CI3Hoo*wh^
zP}3VtDbs@~zG<IBqApCc-6~e7Jcs2`c(L?KkyF>SjbcLbn^y~ZzC5X)d3J)H?5mLW
z7g@&BZwKhg9^+i|_h@MMWEt_}O*>jLHtw3cP(vbl``T$rElYGvqY}BjgHxowm2Sx9
z3z)}zW8<Hq0!70PfA&ph8=q`nvGdXU<m189-y{@s^SqIiR1t8!BXneDUV&G*G{?q9
z9|`sRgfnwAO4!>(Po#WI>502PZKJ^Ce6Rl*jq=C695&zCV4>JO#mmX`P(+;M7Va#^
zk4K}fvaVa!zkq!%_y0dqiV0aMT-UmkD_w&knl^-U26Gf0lem3S)m3bh%_^b5{lD**
zTTZeTDu@c%5nv|pwcTK4tG~;G&l?v>JXtw$PVmhlrj5pyERnp1UFXl;kqF<jP$cWK
zm;HZ9Mfv<Ct~}4)P7wBbH02;`FRQP~;-l$1d+I`_XU+7H+^{?Fm*A4ncfU*=9p1*C
zY$#l!A)4zpY0d=8bymu5QAU%6jasknIqLkWm#;tZigB~Sg;RoC{L&jv304_?uu%?b
zsP~yR@u&og#?l?GhrBlKSsk)OlUws>+tNn{T^6aK+Y|mrbTXt(t=gu(>YJhKx`ob%
zcX(w~zbl)#;Q8(sA4O*wZ2sJJra(n@=ONR3y|Ud!?zIyF_7?5?UM$(mR1wj$V&fkD
ztvB6ntQL&8Z^*eTvLd#kcVd_?dy|03nZRwo-@oW-6uFn<dOpXoOJ85(7N56(<gG1g
z%dfJ`*L}Fi@5Pw~SM;LQE7uiE`Zx-FoAEjD{cL?ls{_k#8+onW{C2HC>8g$Q=lLp!
ztWV7I*LXDb;M<FaZRZ~EY7sjUQ)M{)o>JNR4uLnfR)s3bdwWzpcQxqSK0~0%qVQ8)
z+zhwp7OM-D{M5JPwR9|RZ2Gfx8vmiTz$bl;e@k<mzI97a(cfTj!tR69f1^3V)*NmJ
zKHdy%h!Ij)V{uz7v+7$i!<R{R<u?@b8Mg1=5jFKD_x2+qM`rIA)qVNnWd8T2e%{R?
zHy<53<69;0%Bk09*|dZ1>KA4TYBxN1BEHnGejT6dBoVa)8Cl1fC8na6CT5?S$mt=o
zjLRU1iDOb7qlN;5v(o~%1(6ffzE(||UmGo5HFu&GyPxtc8E@T`2@e}tP8*7+s$TQ!
zxNz>O<UCcMPbndXBF@e>XjPpnXOqc$o#$v+i6ftFzp&-ps|Tf{U(Q=>GWTfDo$G%m
zpE%)aU^-2HtI)EmtS+h3OG{@>Drh-zQudMUwG#!6zAL5N&ItZE7*%z2ir}}q+jd;!
zX#Ib6UFFfoe?{ClqDv=s9}_&{xusEPLSsYkyv98)laCzVI&<FnIqh2}-uCT~vq_vP
z`Qmb{h&b1yDE`XW`DbR!+>%?hP&n%0!|gZjMm=o%{_=MBfwn1um#?iA`drwPrlPb?
zf#slJyaxwc!k?f+lI%wmTh0n_GHw(%5>0e^GWY1l_+PDy7&;dHwv;@U*z)h|ncKW9
zX@d6^CQW*B)%7Ing8G9;PUSX=TeY0u=q|0o!Mu$<K*!(zM4QS~2ex&*)|K25EMD9T
zIoxtP;%a7Gah%I(cGqv|8Plc*dA4`2_wdJFNKB3mJaRzj)Pyr`8@j}E?}#mYyua~Q
z&lOn#u7?vY2>!nM|8YyrB8CTxkJc{ot$n~Q)%&N=rS9wX|1ZPeeUNwbP&~Y`k?HL$
zk&MLUj%N>B^hkTm|Kxg!TWFElqg(0+=Q*$Gj9$QH|1&P<YV@9m_c)S7`g&PcEx6(N
z^Rm~^haDVBF+AsjY}<Q(dSC33iS~$p>o75Pi+{q0I46d-gg5=)d>d>wg{X>g$}Wum
z>Dcl6=oHOE@p}U#d~}YP6uP|USQ@UN@-bDog5!WjkK?1csZLv(7=-Sz<TTEDI{CW9
zWB(}(?8|tqCJMhdU3^r>%|EDPR>)b0lQ&Q7INADl=CL~$+|tC%=P4vB-7-;A^FG$U
z_UXd?2iaoIcmy^j$Ig)5y>QaI2mjswxhU~Ii)-tUdf0kPX&cv@4adHnO)gY8QXs<a
zdvjvT4IiO3@=h8JA~uoPEh0-CJ5F%8Iea=HoOiQjhqox-5zj46A`U?@Da!paNzHp*
zLO7B;?;LmLaclguEV(T0t?I?3E`=5c^%#%kEi&RxTNXSwnDe71LDy3GQtbKG8S0ty
zJhXRm-E3W&-o=u%AuVXzGq)9fMvaS@BIanw9SL!AXVQ3j|E&1tn=aKn3rfD+e%SC-
zd!M4mid5aSlZ6~@*Ahb(J(&<wk>dC_*+Xe!{e%}WcOvKZruq2<)E#*kk@EOzr-zx`
ztlYm7A{Dx8{wCKbDS4*39AJDJFzZyhU6{v;|FN>W?((Kx+20zOnGr9Wdb;7&-5fV|
zJ3bcy1(#`D4h$L%g*n_C8Mu#2xIKI+AGK^HLk@eyJdXD~Pb12NwemV`9or>8c-A{a
z_?~$l?sFq}r(DAg_Jm!Bv>8rp=3UizqvhpV&d{|>A{IWmc2;^xS*+K$d++bg_F1<4
z`%S;<qI2DWZgp&J`m$$=by_|PuB=Gni5KJjDk^fKGHu18Gbs}q>!sX7R$X^^kgVaB
z+LXkS_h^}_og~vli5$lkw-5JQlg^YMc~mFnVm-x0_h0`zfe_A*E}I-rJDoV@>(gtp
zm;LY*2bQ`eb8fISi}-vFaS1)>nH3w<U~#5(<1}9j)*~1GrzY2__c<-IX1rc-Qo}h@
z;g$@m{4)-PM~M=J>GwEfmOh)}TldVhl_fys<@}aY6O1%jPu`t$;9OyY=C(bz1iM_O
z-&1p9c<orC`8)OgnwNr>hZK8Kjubc@uU(tj;`T72%3UdUa^eJ+^_9WJVn?;4<Xd;J
zSF89)Uucmon3oW6`$ezEg+$S#K3<X?(mpBDelId#zZBlJ>~_)FjM$DP8XGShob^Pu
z$m77hxV{&yA6EM4W~N1OvaP6Gdi&M#@OQ$!Do@q;G;GhQ<$Ze<A+y5vn(_=YjVP0c
zt_v3blXTNob8(0Yo*l=tdD2mhjEtydZu)&5+mB7&>UR4{*z(2+j$3a2KN8y7Hk0wL
zp4E|yx0X(JIU4-e#c$0*29DTEcKs1E9CH&hlJ918Jh>p0x5`a|JCY~fXVRp&E1Du_
zvz%t6-7!qcx}^}*^Gtf*;%|+aA$dpavW`sSb7|T&|6WwST~_yfPwls1^XhZiXC=9m
zuJGRUPUXGCZKmESCk@%oi?Pg#nR-6vs?&{w%t5z(j<~#>Di)Tow2|ZS!YEf)n-kI|
zM`yozndERrDvZPE((47KS+%Qj6_-6&98~a8(Pe?I3;TylCwgD`{hO+gbpHFc*>#Jw
zR8EPmlvsMOGHv6TDNG&?Oa*IXcg{bNr5~xnz>sj{YgTvYvZWpimb=Nd{B%0;sLJ*6
zssEyp+YWBF(@s6g>b;{$b_x^chEpuPCuaVP3G~e~_{AexGQn?0nD>qMe~KQ?d%H3=
zXSTnI^6eb~+<}66S6zFf9<%%oyOij1^M_~bsW7&tj1rOQ(n2B!{`IDoo%4Fgp>WA3
zcS)!p<Hznh{o#wWJZ8K+prWJ`IwiJ+>-2)O%p@N5oC7|!MImJ?Crnr~eck-uq56?3
zjlL_@A4Tzrvpo6n?aU*)xWaR5biXT#)VfsMU=m*boMl2|jmxpn9ZllG#y%_@0b35~
z?l`nPb)Nn+`LbyTLw7n&3sGqeJ)#?OL+-z?(XuSXq__N<((V=$PtM7zh}w`d)0K0A
zsJp?Y3B3j!)Bmqs7RI0cO2wvFYDI^?M-0d9dtO|puVzh}&vc~V*UQ{*c`m%&E_~ss
z=k7eX?Rh!)%h892UfJ?TYwHwFv&q(8!;*PWKTPLjP0O)oTbWo&Z-;H^^4pPUXV?=l
z<Iv?Pjxxee_dBM$E??Doi`CI{QhKjnEYE$3hefrM*7odly>WIb-=oc|U%0*t$cZ?s
ztMNR`NyXu-m#I_EOU?8nA#9A_UggfPn`u6K(hn=esToFd%G!QKDT;eAh{q&0Y<WFP
zqD^)UyU66|=;+V&XXd?eoGPuectIxrbt#>M|1TU0TRy9HE}h6~ZT}%d^Y+O<DLFZe
z%dVA_Tyc}UGb{Q2@@~&<6Po_3uLyfpuX{vU=(5O}E7R*%hBvO<U=@5iOI2k*Z);ka
zlgP)56P9|oOcR>U!n~HnWu9>G!QfdE$^IWC=gFM0X5@<3Z}nH%et(B+^7WSMb^fMq
z$7MfH>%6eVxj^XMCLw{Z?S7BjAMSE{mf|;M)3amy3bs3JYmqr^(=M@4_mPac-+`m6
zqFpx~n7J~sV@^g9o2uw~-g60oTKb<aiS}-ZZ&YjeJu%0Ln?WnvblNMmwXQcK<z}>O
z(qWzXv{b^SY)QU;^h4ckjM9%x-`c7>X`Fa{QK0Lmz~_ytznhpZ-Ew00-z5tZ{Xe`4
z$Tu}tyV`es!$p>y29^o(KV>f)&b?oCbi;4i{}a;Af4<+-m-=3C|K5)iC$}m+;uliq
zOWg7{vrb$y=E$9CWnbJ%?|VIUG+6P;%k$sTCr&bX(}d3|b2sGd)^w98blSw|7Th87
zp+&k$W0}hLm|4ZLc9nY{Wgq*pw7YTk;XUr+w=ZX3XnW0byddTD&$A&y4)4E2-O<>s
zX84yYYH!cap3IFkr*Cd6Pq-X9f$dnJ#L5!I_lepQd2gMXFiB{=O+JH!#fCghEoBY2
zE597p$ZN4Pd~3`6Z@QYl+eKIY-Ue&E*7jSG^IJ~+<@$V4J;AnXvDXHTg-a4ITdaxc
zh}v*b>T5;6c0>RE58T2gJeMEukXBs7*uKTyGKWR$&aa)P%>HMuPIBMecEqFbr{nEk
z4o?c)UhHQ0V7*_yWwPnYNj5D0dy?*Ked9J~{iBKN-#Z+4m$-T<$5UL|?YBrx+>?MC
zS6KWaZB~lg_|>dPdS+Oo_U!YFvu4V(f~M>dSh{DP&3Q9<*<x$|h2bqLXPipwG|x8J
z#bWa7oOnj#{bzRT{BoWpvk16cd~S7MxnG{tWsw5UyL0?*RK1UX_~~X++R{9+PfBaH
z&v;z3`e)#q@0s_4{^<BQwMe>euxgNA-tmM_-R6C+Uusys50i`4YRi?bJ9a$u==`^`
zd&(BeGeO5&F6%rgWoekGD!Jvo<DBFBWX{i<#dUVKqjtj4?%-pG8`>(1CUsi;f3@&O
zMxp8MFAc2S=UG$MU+gRX-QR3IL7jo8zxtP}m6M0VG&cvK2?y2I_=$Kp3W?ONU*i(H
z=N99}PZL)M9dzPM-~GEfKX$dWU&iVUQ?uI+7ddXbVSP!M<(txx{q8ao58K(Q3BG8$
zap;_c`;Wc%`@18`gs1Tw5i;f#UvMXIrpqM}7XA1wx;6^h``j}Z%1CcG<a(W{P214n
zN|t)U0)}QbZ<&xC6B3=<`4x?Hlp9VT?BY@nn-kHtn4wSJe$q^jHYIVsXbYd42B}Qm
znnH0}6BoEJ`p#k!G4NRw;3UMk<i-T0#U~S37zGz`bSOPtA*Q={N><|0F4>tH$u~A;
zUGMs@d9H&kI`8JA4NB8Bna;`GQAk*GaHINuGoR{$oZGG9kDsajo^aWti*4tWm#0rY
zaA{7_dvtWg1Ow-zOjfcwTQ__>bh64YJ<Q|On=>qocG5f>3>Y&RmrC}@={5!*oy9b1
zik|mIo|$G!tatQ+FTT6eta4K-M|nYLOxUcin-X`tR48s=+$F72y5X8*G+Xy{q3qRD
zb%QRlPS7k}9jg7}uC?5odw+K`M6b!Z`pI<f)b*e4d%yLIc(gIOTkv=E3T_7O*ZJEI
z*=U_Hxbtrj$5PFpTMmt!Dl-zCSXDe67<o)Z9GIGgeta-#c1c;WWv*X{L)YsQ3Qw&x
z7bQI1?RzZ2m78(Z|KEAt@3%-KPce*InL6dwwigo?3LTU>ZT@6N&L+j(B3u{j`<Jks
zJE^@<B;mkKl`f6N|JUr%*<-dW_{6D~9JBkD@;XZAx?I^;6s*W<QJBiDS!0#q#PRyk
zp(P($j552YhB;;HI<co_3r{+be922~YBq<K+T!dfJABO)mhvo&*|NO1E6TB_=TX!}
z{tIDmGF?}^yOp`o>T{RyiD_kb1QxU~E*9CKcv$R_=px4558m6yx!>a0e(R9(!X=aa
zvND@yB&_m`nk`~*NnJHT*wCLb<r1f7dy=q2SJ>o^1P|YcmFYaqZZo|as<vEkIT04o
zsnA)`t(+X_8*uoVLcRDTC!Otl{~synYs}xnq1t}zQ}DuVBFz#HTY2`aPCCZ7YK@|H
z<<?IZSw0)BF>N;4w|e0z4)4f@KSy+KDi|I;^J#&R{G%i$RXcA5jw?+S91P}9rv!KU
zCRY2NpRHN*?Itfvm?sz4!Kzs+R~}8R;?~pIYI3Qq?=<(G$8E1?E%X#G(zAHxCfS$9
zXtvXD;p=s`le+Gj<t~+rKEZHK&wHbCUr%7pN_US(fAl8DeUAy$QS3N%VCs$LnpT-`
zT}}3Di<v(@YO>#~wB~~ox9S6f!#t~H?;gHv|3_?FbJ?WIg;B3uox00B5|7V+d_&Ti
zt1-*7>zkr`W_j$5Y{jJ~TBGI%b8e2wzxT6k$N!X$CB5_V@1^ehm)v2Xz4VW7!%L4<
zId5l5&&spX+m;Z_X#VhwPJ`^u37VaAu7zjR%~>&3vp!GO%XUxDnk94A=B}-(RFvK=
zVKk?;WVxLGEQfnL4z%r_+QbqQaBW3aAit5Ow>O*fROuJN79q9kdDlJY(|(iK9;CzP
z-S)D>!{LyEo<bXQlUL{FIHAdzZ$Ff*{ygEI#}$G9tP@L}zg*bHw@6pztxDx*Z#m5@
z#a51-MPg@GvP<9E+NoKk&J>hzSj|pE#Ar{8ZuARJzdcC;zj{45rFYzNVtY32_?n{%
zB_ao+P8n@%e^IDcw$c0hn(y=96(&_Z%<MVLkUZfer&{$S%l|W#qtzIux~dsX?(A_?
z3f?qB)Ie$N#6!P|%w{Q^;B;YET$&-GFvb5;OXu6)TvA^{P6-$my3e_ChLN?wo8?_l
z4i{tp?LF-?jMk)^t7I~3?^wL5|HmW?VZ(*G4o}T*a~u%8BGUP6nafU=oflerugJ6<
z*xAS#(J^N}_rw=AB1%_MI$U1c2wHnfo)^?)@jKbvq0M{A<=WfN=3f^~pOE0vWt6MP
z>7%$Js48hakLm&w&W;0CS3MM0`!d!RE$-I6*c5oiV<mUj#~I)Gm@RvgGK=_TawIB=
z+Zz8>tvNKaZ~Y0DO&=zl_t(!9)?sC6m?P)1e37B*_ASM3yeps0<7*Vo`Y$1+w}s<_
z!rmMXMrEc@RwK{pf-m_Vo>Ds*c=M3h_LYjJJF~c}11|F#O8Z4x8ch7N$H#saw~wR+
z#~fBk?Z}5eg@4_ta9(+^W&N*U2kmkvH^EYF^;MhG(>QhuT&->Gap`gD>{&GB#Iu#%
zzl6N~H!VGU=|ak8OV8NWkR~DSt|cP58LiSW58K}DSi~b7>=PwY7PwWwXIt9hh^Pfy
zRBpxTSnRBd@RT`xyS?U3IKSq~xW3&+o?6otr^o0es%`e{@F-V&Jh|_+g)wVDU(l2)
zPKNAd2Ttub`%-DS@~QI=v8y{Kyqd;6^Ge9eN%tmfJ7Ig_3cqAb;Ji;hYDFPNJ_$T3
z|AX2mJa9B$GvT>IyW&Mtk<Dy^62WgO<1RnA+<w)1jo*)(QyUszD{M|#uC0E><jcfF
z?yZ^5k|C#Mt!G>i)!ydXy;_sOI^eQ@sbxo=`>H_Iuix}%I$wBPB=h85Esy?f=FFdI
zT>D;~`m#$!`LKwJK*QI@OA1zRUpliq6INdN%;oH4hLSbW4v&mZ?)q@luXI&#EN8<h
z-qaO}^&SWI9@F(Jjx`AL;4^k+yz)V%rRe15G^5F~R)S{VjJ(9{MQ*BZeZ0Wilk35c
ztA`IWeT={3G-<zB^bv07ZHn!B*8?4TMO@8QG?%))xa<_|EHUldmX)PnyApLDw_Q1<
zu%*mj^TPExd;e#sUFZ2+sKn?T$Zp$toRvpF!E)Ks*NHxgS5$90&FS8F+(q2zbnVO{
z^GrK;CzUgPO-ARK1o9Za_%JL{+3qw?b!|bQMQ>rz(vCel4t$kL&+#jLbt{KkaD%Li
z<;%0vnU=9X(BrIFs`>B$i$%eMBDEPMQbiT*y&S?q>-F3&HeM8)rnF9O;vxCj6Ar1p
zxY{MR^7V!{ZgTeieNWpmW^8;UG=1^W(&f5eGK7s5T`w$~5^ueQ>6*^IqmJ(sUe7&W
z@tk*}e)&9^H|-B3j`K{;oiMR@0ZZZfiQSD=Un@+d_tcmy5q<c}$*%3vd(G=7@@5_g
zSUIOb+t_8f*MugYC3hZHe(L`}PjKf3$1M(A#^MDHS8rSQ1%0&&@>p=>p`xy5hDMaV
zwr~5R(nr}z!j%H18(r@Ix*+i;kbBXFrE>oyj*6^IlGEgQY4J~@dCx6H$L1SJB7**l
zn%Se3BC@nM1+Mt(IEm@iq9=7%*gBowEf!c`@$-ax$U-kR;}`vPKNiXV`_Qic#&MG9
z?&H(8{kyVm?mIDSyUYZ$w9}CS*S^@sZPC><JP}@-{g!Rz{}WRt${qaNG<}V{%FgVy
zWl35=GyjAvQ!*(2zu7SJ_KfV45{qs>Y7)+Sw*HWBx`GnJx}zC&rxXjcOrNRDzu>|3
z#G7G3m*Tc#L32(i%z5>`s54he>Au^+T^^01|F<bH7bdAbId<TbW=YD0hpU#BsQ5;!
zy?a;k;mMW*@+|6_F-zq7nmriU-E&m9y3$2h*v~X95#@TN{U!6T43DvlFE`6F_o^ka
z`<U$>EdIRb0{g)YKYu+`Ja($W<0xNN>+@g7uJI@xlRLzF`-8%&%#<&z6V~-j7Fd?Z
z;;AUMa^ed0o$mtD>duyWG^HJTUg&kqKf~#iL-M-{uihu@y*Juj7~EFQa7k!zVpeWa
zF}pBr6_?hU)xxX^YeW{wb{)U`MwKz@xW<Q*^KR(=*mmffNg~gI4n>2t?FE6{EbbhI
zEwXL`aVhQ`wS}6Sn7-Duh?q3S{aOBP#z&0`{qG*WiZc#r{%^?_yPc|aqfzXIVsq8P
z&qpr&JP^pkpTr}7NM7ua_@74YTL;8$1Tfs1sF;&`?7>CZmxW)W7$=tWI!GMqyT!C^
z9b3}8JjEL2ex^^a-dxzeV}YUCBJP-F{DLeuX1uw@aKT8%Gj$#Bt_{9_SDd}+rnTE)
zrLWwhD6KW==XjM0wBDWK?Ol`l<;H8i$O{Ue)-rfrcp=6#V~&7Yil0u|677Fkr87LZ
z6nHdry&2{;#r<iNo%hH#;kjD{)8Qo-tvwtIiqf^7w2IU$kxQE>`>#_Z&n5JRS{$F(
zMkOY<Eeb!@O%VHk?Ae3GvP|qedkuNc7Al_j7%P>e_jZ%s)nx7y53HKbDg5v9JNAcH
z=}F+VCm(*9sTqIK_{%g=@z0BwJ6v9gHF3!a{Sy;@rKVmbw|J?-ChxeeiRV~!&%IMF
z_u9B+O`V@-?>j%^B?%!jywZ-nVHS5{F-)$pQavluB&x#j<i@4>El#l}3w4waNIh=l
zjc^QI)X;T9v3Z`8$FW6XOPa(Z9l0EIOh4vnFL_{bOu;QeQM0dE)a>E+Ee&oajBa)7
zZM@%pt2xhg^|SffLY}aPKeiQ$PwoGC=;F^A{tUksihnS+=S|YHJSl$1nft@)hBpVq
z*e;7&99B#+TFuwi$EIq?{;^*y>37=eKU`_oTe|;#aZ<G843BCyuxgyR#q+<<GGVJM
zqyMKDOuE9aJoD{;HiI)f6K?Y*bLXi|@$-x-P%@L+sF-l_$*JB_uf^(iPxfqbyEmg#
zQRnEHBg-sY0*#F()?9Mi7p16aqNug}%i}+Z+7*V)EDT>$j(pBq^5qt*L}`mg+Y;Gh
z4SPOuKD}mQvoDxold<L&1<nJXzfV|j<;4`g*mXKTx45qf)Xi43+<QTM-v?dAMU7gQ
zUp8bmUf7@*vru1j(b0x=qJBCrPB^<<J#Tre$51#lP0IL{Qqcpk&kF9V1bmMP2)p=9
z=`pBC=4#VQzqamR-51|i=iDDFnMW5dv)B;)IqCgA<<C5e6j&I{-H%v%C3C-N6xo%;
zy~RlL-LmYqbdmoj6n-_dN9HC5Zfnx*Z};uYP<*pggO5pb3e)==!8|Ste{2x^`D&r`
z?}yTNP4!q6^lpDu`0`or_h!#O!Fu9O_Gbm-ew)ULCW)9i@X9U<yS4GGU6Rp_iz-QL
zc+M#uJMrZ8w-^0K);s;bsCz(P=~u@MktENYGrzCBubj5lLE)446aOVOOH<}#r>F;|
z@+;kvVoF>2_LfCZ+JzKuF~d7gT<%^8VM@+ql3e8OpB|jMEq_*uV{*v#MTgBcDr-(k
z;NF&`ZnE(4vJ>)?3auv=MDQ;1lIED{_E59!s?8}O-FPJzg{ICW7j?ELh?y!@EK>?F
z?g+RW^z%@G%tYn*_on)9LgW8`HZ%TU7Pt7aXjD)BorE@>4sM<@raK)6?wn9Q!nswZ
zoICA}Sj(amwYCiNuH?r}Arl`5JC`jL(KP<0t>-p9eB$BPcGq7Cxzre?ExW{G@Zg=X
zh{N)<m28<)Jy%?I-WH%ZOQg8eRIg*3=gH-Y-}tl|zIsep(7hwsWAXdUw~0Lal(~c)
zgl{acv2o$5NqP2x<>I0T>-S8*P;jw*-_r=+Lh;G(f6hAp^U`-aRtNi!!Etw|D+-h<
zTD@0z@HPLeVf_DTijTHe-Ed^8;N#0an<jLmHzb?sj^S382aR_c+`JQy-Fcw1u8U{g
zaZ91%zfag2k2g8J&baaFkfZDuZHpg@GA7RdALkp-ZV6{7Eb-oc(_rh)tYx<}ic^D1
zoTV3~u@{&bAJefbGEvAc_1k>+ipI3Iim-4$v;9mqlR~oBXRop;P&B)cBNn2z@X;!<
z4(*UYu6>JR_IPlsC}=J@z!kiF!3m3K8L!i~PMVk`3m=-8|8lyXXi~jz_+le}F0T&7
zcMaTT4ctj<dDI^3cr9Fg%7N+B0a1^J3G!P-h4{7#xff@0E2Z#lU!Y#Pcjct@d@0h0
zij>$-R9xk=G;a%(DDgd}80WRqDs89Uqmyb6mIW-Sd$MkYmAS~?<-)b*OEzfh%RRMj
zF^%A4yJxpV&ER$2p6g8iS*Kmo$Y4CCxZ&u$2}#_q#2x;>FXeeE5hvR;OZm_&-bZzB
zrzt+!p?FQy-uQ4le^bEr1Yx$z`m7Bsavnh|GmrB41#}$dJCVTjqasmVVS+QOp{$bb
zp+n|U#@EZ%C>%T#eDb~ixkLVTl}lXG{?+v;uE<VhEl`@vou<3#)}MT3>rd75dP?$=
z_V_JM-_@-y;We*m*}FQ)_i;vnUtRTAby-dmb8k9wFvPI;wy&z|lNCyb!Yp+vw<HAa
zyg9p3O6gfqU1!zo`!_tVm5NT0nmzM~%u%O?)tx+dGLAK!<Gpo&FDH%Lk7w&Ug{=a4
z%0lZ}cqV7;)%z2sw>U{cU!bj$hiTg^y@iI4o9wr?$|e=b6!o!g4FA6;U7%RCXxZW|
zIX}xIojCK<PR0CGY%nuc$(zQ!aa!sZg|6*EB26BX%(=pk=Un~aF==PCJX7=yU-5;O
zdb@rue`?;R%#=SnUuJ)$;++1tLYI}dx5+t4dmqa<%2yU}D&goY2Gt1ehY2|b>P?YM
zhg%q4lrvi^sj>Yz=cFpLbFFAn%kPPLA2_@OD*Em$N)mgfufX<S;!sPfTdQlA;{KL4
zY3A@Z{;5hQ7Zx6MVJ^`B#GID6?dG~Z;V`EQWm`Opt{<Fa<~@bkjF)MZoq(3!j48X>
zW=f<j^L5eJ=sEQ1_q($6E$+7!-(9j-jyc4?sPRUkNLZtcbcbBV!j)l)^{*2}|9|`J
z_3XOanjPT+d2I?U+&Kr&{)h<rxyj(xn&-C`7@lgeOq!H&_ST}RY`qrWr4jFrT{v_+
zMC-Ugv61Ydjm#5M`9DmY_UFXDcgbEe)7-W#bNalmG|E`ZTSMaa-+#XiX71Vh<xz5!
zV?_B(o(o|eITPRQ;VG@+(8x`X+ka@*(wlX+x63`dB=azgAtN!!ZFcO+!-rokln!sy
zxUn#<&_(3KW-*sW(P{c|;f_2n4$3?(*VDhd^|x8v;un3dFZxVLouOdfcs}w4&z0#e
z&r-r>9lxbyIDhG!i&e?{7a2O5wcNhdXR_l}9k)Tz+Y^RfF56p#ZAugsoo=x9ggT2?
zd!?l~Z@KwDewtF@jcJ=s8#@?fRGJ)geY|2`S?-_PtCW)VFPw44@%qx<DY-eN8;yL^
zuEl7+Io4~)9QfOD<;z6H7mZqZ={(mM<6bw)ow#^-^TybkjdB?eFR~_xE>Jq`>Uds%
zj_4EB!_Q?{o-S?Fx})$cTv3W6?px3cu|?1G9tLtw{<Wd&MbEeDKS^r7cE5jgC~0JI
zy*px9kbPqLg`L;+Pj2R1r!Lv58a#W;5?#BrGaql6e*aZ|+~!T&_x%?Rg}N5Mld{n>
zYxJ2=v165ch3KK2Ys;$SS*_0Tys1&DG?NzJ+WPL2tdiFIuM$q(H^se=O%Qc?c=)&D
z!{3UQzY`xQ#xmSE_<;AH#ecSW@9h7?%EU5kE4=hmR?Eu&e(jM5*^M%_jj~4+E~Xxn
zVZE~a>!K^NkF1o+c1St%H~LPRy=jhi64$g8p(xqL&YDKWomUI@DlWTisJ*P3;ib|K
z{!^ji0={0)^@S%(Xr}!-?P9xgV!*T8%1XB;F7^>fW)=%fcWY)`D?9D*Q|Asd=HMvX
z9cp)GdREwqzo|Ryy5g{v2jju|dGi+u>A&BTzjUYkwu!PQ3T0gu#+v-Gc#*jBrDDaO
zhkx6bysHj8TwA~)I8W?aqo`TU5k{xrwRKEfVkxF>!mF$P@mv#fT|LjSlTABH$K%5y
z2BYAdotqYaR65b5%sPdoa`_@{#<jON6#gGQsiYpnc2eX{&_r&(-ry-7f*VqWr)x&n
zh0Qs%bU|QqFpG}v4U?>cOB9!f?d>vM&Y;CKDQN4DPzQ0w)Dvm8-C@Z$wnQe!l|)~D
zaI4Ae<-Q|cn-lJ>irJwND8TUHs!9&;q^zymKe(E8iN5#EnJD_WYwa!04KV`7l80C=
zwF4%JeTn4Eu#{%8JvF@{PAdEA>Z5u}YfD$(5L{!LH8rn;DI+t=b^SFh#)GbFt8(|u
zdms2F;^i`R2NCV2H%B%I8+bA7j=J#r>*{^0rX6g%6?suPe?$2GHc8!W`Sn-DSM#p*
ziYrMLs842G7j<>*j~NWJ6Ru7P@|(yuHGo-c2Wud+Q|SMTeoQ_JtbSr{5f|B*G9o6l
zNqJQWhseZC5%H^id*ymv=)t8IBUq<SUF4c7`a~&m&J@#R*QFc`;XZSA+~{<440{sR
z%f!;T#MS!UiloVHl4etwJXHi3rgF9{Ip7)|wZwr*$m60{=o$%=XO4bm-xT6*gnpUN
zu;xb6#ENYbgbvP>Qp`wqOb#d!)V}oc+ij~YYdU3%|6bYLrSbKG@hOdeUxK^U1bN*B
zmi9QgoG+i+cZq+=q;H3BX)U(6{?hSz>=wECm2oNlx<OkWa?KH4nsUKQ>~5mAr})++
z?)5sVZo(-g54UvshU9L%9PFiP$QpFwUy4YFx9E$I4yFL_wK12ow*J=&53A(9=4u&d
z`fk-TtCykS(krV&eGE>eicJ(_-EmPwm_=*`182_8rLKR&wz|&HS-X42+{v5;1s=I#
zQ)CafbnPri-P;u&DBaR2Ce-BlpflZ7DB;e`<E(#gZ*&vPF}mQoca0L8?cNK<$24BK
zZog3Q>g#5|)BL?kg0@L(F7uzhuvWRD{Ce-Mb52hURRo<^U%kwI<u&Jpmy1OYE*H%1
z+8Z8d8$BV(eI47}C4x7%ov8YDVC}Z18L@1pe}p7WY%RjK$!RMd6id;G2x(>tvsMgf
zN`CQOL?%mRVT)%W$0Enx=vKuiqP;tm_M}ghz7*~-?O;;q+!90a=S)*3DZkJc6Zx+a
z;~eJtey8Hwt;*uUe^O^ThGtDk4V}#qBk?|bYUkA%2gIN3J{-v!ec_;4V9v!ks_Ub6
zcg@LN_{wG9MW$66uG!OP+aK=r^k3A@r_PYOXouO0g<IAHWfi=+@_38AUvRFe<Lkt`
zTimWDFA==<=fD5fk3XEaQhs=Z9Vz2xJkb@Nn5B?(t6&C)+>*q~#@iYS#^R}xE?g~&
z3VvEkbW(5fYLq#jdT?l}-u0ENaq%-xNAyfte$KNmI(;U0KhGsg>2nhjY*K{7a-4(}
zqEZq~UUJ{#5whJTut8NU@l3r~n4r3qX5#8ai@b9hCA)Q8O@us>LMQ#$*p%VJFz4kW
z3EfY7H2%9daBp?+cU^V3uZ<^NVdAX5ilWV2j)BK*w%qVrmA7qHP~h6EcZMDRbso9(
zny5?^b>zRU*CcoKMJBhLW`}Op#ns+%lT^26xT@bi&=s)B(P&A^(b8?7XN7d7`=m5T
z`)d3+bwX6dcap&b32O<F1v!aAOb?#2GhSDi<KxIBtdRI<!6L=ovVAU03+{+%TtAa0
zC+upa<I{g6RqTx9lh#*<1YZ?Yh)K$Iy2l3GwP@b5<l@TU*5^t`146beRjKhcJnDS<
z!x525CZcK`ypylGdMtD-+tg~X_41+8bzuTg4lYWo4f_B5`k_&OAvy4qGQV?>tK6TL
z{tIMOT@SFSHTX|r`rjmw!>WC2nM2&wL*ic(GF@kGJJDp)BJw@Owam=0ZTf+;O8uKT
zwt6({^jR!jZZ4Vl_+F=laM)$f<2ivZbeIlbE(~e4<eAR@cG47?%+0Y<ot?f%R<7^*
z!>ud7wbOf2x`39Lk73qYujq@HmIZEEC@Z|#Lwd;sh6xK>li42L@ZYngl~=$uKAHEV
z|0DrM%?(YqVxCK{EpgjAllN0#`;}W+0+o8Fy^2;nPHYu9CUELmj=IG`&!>xP3io&H
zQ(qi4JIXPkSbXn<MTv&ho@b6ZbeUZ(5{|x+>u^$omDSL-+xKQ_$QxIugRd?r9o-}5
zH|^%sWpUhv7oMzq`pIOm^wHPuZ2vW#gDeD<XTI5>qo;J&EP98NbZ?&9OPv#@Ul$3g
z6_$mDze*Bc_+m*_mdpNHrC>W#qXm05Bnq!lVg1a<Bf?;_-cxka1d$yJ7+7LgZjgF;
zJ<0h|)Njko`qqb+70-V;!I0J}8Wo}A6WlG>Emc(Ay3$oc+i2C1p3d@JbM^e!mEKs$
z#j0U1ndtF{ZQ*i}>n)$Rc6DxBBv|0M^~4*N*QM)!9@)Xr(f&g;DUvBfPkpm1(~0aJ
zUx~}twtL-N`@cpg?8>RtWi2bkB9FXw-4h|W_t?&;pc9kYc2%UCs412!zD{6SRFKGM
z@JXlHt$!h_Zj#ICEfXZdE-o;yKH=EETuk*_`+pVVMT!4fEtYlXb)*_|MF(~KQcXU!
z@{WfiUtQSJX=dxZ3KrfF@r+mzd71T9g7&4D6)eThVX5IJ+(CDbymVPGH+iDDzEJ2!
zol2e57cJ(oN0c)1`R1}seE%`&bjr4v&hv@79t=k+td~z{+@E&p+qzAQ&exlrQTw;&
zgxH=ZRw=QZ>YM=#jIU!QG%dJ}Xdm;kcQag3H~V(X*N&M>Zf5dZY<y&J)ACG%+f#$U
z*OS#O?>=x{;CdmE?Wo!bEiaE1tWOqmtaJ33dM9Z166ebkT~FTd`F3FAqff^&il4nY
z^gj9w&%uh+)<eN&9y1Qixt5)7ec<3?CFz=@drmX7KF>cFk^bTT5@GAfOe^M0VAylh
zh^^X0YT*i@isYS-XT9%WRBGBXIrVmOWSw}*=KN&C+kfxcH}9!^%a?gh`eupaT&DbO
zW*a^Inw>(rGa9#N22E3`dSh`f?ns;q=bEh{U(Ref^K@_ILd*VBp~}jEtR7Ew?}pBa
z6}S`_9hcD<yEQ&;#x2hTu3x(MdXyi>p9)!8(X(^8+osLI-4E>aD+@Wb>nlo&FV61q
zRBYZOs?6$}QgU=AUqT1>BfBVv<>re|@h%fNC9~RerLD{}t2?`&eYHzXOuK9E{dsS{
zX<5t3W!%Tl<)l|nk9hUDsam)}jN$pKg<ee}%uEj-O%*D#X%+D*jb$ng*>~)}^SzRs
zIZIQoO%db0QT#GgG^{N*FIhb%z~Ak<gj7dZL1<cLVb~N#^{-nvITNJfN>t-Q+_oAs
z22W6p{G=NDMKUeKjme=|j9ujOWYd7pQmX}}FN90~6|if%q`^AT_{|5S8-iMmON=5v
z*Y=b}PH4$!KPD!5kT0qrus1|hUaYRW&Ge3<fr%@h@D}ky##}|h=GRgM_?eW}s4+}3
zs7P)Ic38;y{kvn(1`$PfAGV84Cq5-}WC&cCB69ADh58rKM)#_S!lGM8EoU5OHJucD
z{8;ScExvh%%AS|v9z9ilwZ%%Xql!r&@82O+rUU8WLNW6z)OK!<XJd3UnOLG2Dts{g
zzh0Pnm-TYNpagEkWis_jc1#L(^NeKgACz6QK;YCz%~mDB-Xg6P8G2hMu--^kZx4wI
zZIm<%ZT!rnQ0*AqD=6~Eob6wGvr3BKB0~|gmbleQu`v#8I__d2fntGajZsH?v`dQK
zsVd)X6^*{CB66&REiwObn4#nKMvlwQ>C2SN%GKsw;qy$FToMrR*Te1n=JeXDC0-(K
zfeo$A%L0yH<X&vRHuKR$A7i#R%^FQiazbB?`yZ(@JN6_`DvK~IpMS#R>jKu_-zL5L
zn7Q7OzxIjLde#ETjgxMr3+k{&vpVu|boN#yW{Ddr_+|>sQec?1fi<8rxYB&8#X{B!
z3EfEw|3z618BYYoT)3*(u*110ICfKuQfpx94~<Y=Az{Z)v6@MKoiq9ux_6p~R4q5}
z@0zIkStVd?dia_hUFnXFJi=<Z423-l;~!mQ3um90&NQ(=jWuwBk@!T7!Y67>jT(wq
zHBT-KxU_xZ`oNyNAZgyjVB4Lhh3s_^hg+YP3(73xv=Odbe=%USLapm#^+k`=?;hq`
z^gv^((WFUkg7*cZKcw5NaYzuqQcxW(9H6QAJzQ{>LVGoHy?BX){zsh)k&1^yW1<^E
zzDndbO?C?WCU4m^rJB>vI;H<iNTtw>R8~osjE^&tz62(*l_YHoll`i4$f$bj1+AHh
z5^5FR9#<rUfBu&g-<XiD!ceqKq%fq6Q&o1sB;%QrjMXHK^_8`r21`dx*7P?vUhpU?
zA~55z<D!g@nW|ljK0f5<yyzB{G3!!d`QidO(Oa53Hrnp9H03{N7qX(YW>UF$WAqMH
zm-pseE=St}I7LDu1(#iMbLkW))nF})(5<*s*s7?gY^2oeG0p9o=)+bq%PZvt2U#bs
zD8ADo=C;XdLHqPauDO}3re|m6zK9g9IhMO*1<Uiuyu3%nQj9hYm3j5ErZ;mYcUu~9
zYAt7C$hb5^W8V(0`JeK4F-qHV+CP6NE2%i8{gu_?hE_#E&EtypUkzlBtuX0WG;@lR
zK+mVy>578vTbJxe@2$%EA6&^e)gwzV&tf%`QS@?4`&B{QCP$<F6<HU2OmI<ae6AGR
ze?2xaaNdknPA9{}{KE8}?^qUoQ2+DJ`QBa=eym&`*i!XJbB36pnyrQYl0-N0z^>9N
zz1Gr+`igUGB4uAc$eg-Lj(4F{&(#*0rICIbt-&3O@>D#(J>cdH=ie5<pQ6C(TP0ZV
zBy;IkBg>`IyI#r)zsgjUoIL3X{}V?!L(U~jCIoW^7J67sl~SC$?w8beVWGgs+;1+J
z&)&co^g%VgQ|FXwk<>y(-If>|SIes+6#^Z;)2}bJ{k{2=o3h>%k#lN(*&CG&H!9D)
z5&H4#vKd@%2|*J3%Eg|Pi7KwJj$HlUe^0R5ZfBcE2W{3er2BlEVZYFXRZ);-;<o-E
z{!4+jGRM>lZ%1@Kn<f1;v22n(Qv(0`jk4;cL4qw3jow^S+qQ`Yy5GAPu#Z#rmayEm
zhbH?H<&J#p4xPA7x>T%kV&3*utel2i8!ffg2$)Vkm9+S_K=g#EbsQGWOD&kE_oe*o
zd%-!c%dxUkLdEn`?7p@*&p>4-7T0T1im%(l9Jb5f%4$?h460wbUF5O#ldR?i5BKam
z+@G>!0Z&%5IHM8AqHyntQibjUVn<Ec%M0c%%V1itF+4G|`w6$<)Ag*38#vWB%)cV^
z@iyBhZH6BY+2j+s%O>pqd0W29ID^%ki&@aLv+2KWua$sc>o&8m8|*XJ&p4W?cyRku
z;lk`o8!!DVD5{8F+~g&0ow26SBSnC<<JTtPOBLc1vn8E2U!1aZf0*DsC(%p0<P%)u
zbeZ%Gj^yOb^o>cK{&I?yd5g_ViRCMf>u+mbm+>VqC$ajrRrRw&^Y5Lq-e=C0qg=gQ
zQT2@0Ubc<u_tmGwbaWNI=H@=hrI!}eB)ud4w1eD4&yAaevSYMaCpuUt?qXHcx+y5V
zPttu+)6B)uCbtu%6;B1GUeZ|Xl#usn=ej`qh=Yj?qPAWabX1RvcV*~VZCRtZu)izt
z<crMRtQF4Zx(+LToUol^g7=ykQWwJ_ywk#6Cu}?_v8<*y^x=P@T%mP-;p@sS>8sBW
zyZBjnR=_@n5CQX(V$-%|7){;xH%s8H_l9i~3g)To*A3+Q{8~s+!tCOn{lBEmHZKy&
zjS)yn=C)$o5b>CQ%|n^&39@?3T8&RKA6G}!<p}P3b6TuhZq31kt4q8tJ(pVOy>HJP
zv40P=N{#xY8K$jG6Euxpv&U?j!^F*5Gla7bZl2C;`E<$2xqD(~2ru`MULodqv3#f1
zC($G2E0tfoSiA1<sdY7CX5S0`2ioL5*s8#~P3K`+SWe*;<LxVd&RS(TdzGPKV$bE|
z8VPk*o0%oYvSK`Qa<o%|1vDo*><GHT<e<$|aD_{Sp?qq$u;cdjB8{-%oso?Hw+Go?
zHa>f_gQvCi_qGsU(={7d<Cd@-oTB?uX!knfdBPs4X@!SpWI1!&u1%kqGw096NJU1?
zzgJe?ajCBr{IPUx{Il!KFV!llL?<{3XEO)sIIfev9A1A=jAvI9OO#;3lRc~(-E@B?
z_LOWq)U)qZwBY})Gsn-Eoqlu1=ZMgE?Xy?T>{o5Pk~8P5%vEiv4G}Zk&OFmzv{TmM
z;px3vP68j!G~Z#IwNX&|a=WY`!%bGhX5McxY!4MRV=nA5Q)0?3DhgEM@6~%^B$|Cu
zeAn#5kCptS_WC`(lKOP-Ma759Y;&epDlsn15Qq+1zHpCx*y<yzi`v957VO#{7I-Fp
zZ9{l+_x~<d(TsGV?Rsqw>ZEtroE5U0do1&eyr7<$XRnY~?!#}10^Yf2l_rX<V^Ayg
z&iFA$U18nTptrL39@=ei<W9_)oyb$`6meDjDgU<%T!98|;+NYq?`A)%P&;mAk@BD~
zuXb5`*9~VsWs{mGxtd|3GYzeMZF&Wp6np2b=UEr|<%z?_bPb97nGbg>HX0t96`{MT
zwdnXh#^_rodG?>P31TYVzvl0){Jw{R^UnU-DF1eu)22<VD<9r|87QiJOX7e}d9Rph
zAaj{@=AC6C1sfvHEBw;<d`t7ynV={G)oZugqYDmYGRAf<7POSknPy!1a;E6yJK2Wi
z&P#$-Zm0Bn{XWr^`#&xF)w26r=Dk=eC|Ro)&*ar5cxhSA{To@QJY)83E~(C8DC9TR
z;@xtoZ?k^5PC%?)*AfMLoi<lay<-j4bF<5i*~?xL>bs&g@!_3)Z#5S_OkOPb;N081
z90A{0!NRzAp&PGorf^4ZP(PHp^ZBAcU*^Xbk8{;-dGt#zXlBtP{iRXuHFdEPQv)3m
zMV&<Kmg(87a}uAl`%dl2)kfEu-&(ovPRU;TCf(4#v(P|l$EoCLV*C8|te-4uSuV&O
z>Hl7BqDRYKp=0S!s`loqss2uCGMWFzME<_8q`|p6XM9-B$}@ghxAtAG%|pFh^^^l{
zc~5S)Upw&V^_Q21cb0_S>8J}xIQ_rb{PPP&`+dyyo1UF!Q}pS&s_n*kU*Y0fiT2>T
zuau^#GvyeTIM2Qzab{WUY9&AWJEvsMMxNml)tEZL@6qd=z!`VE#7(|l+4m`m?_*w7
zy=aM7*LG%!5F;bAi2OLAnq(Kx>wU-eC|%i}puM9>h;53{{QNIkCERkM5Bw`Uif$V_
z-QeEXc=`9eN4E~j1`0oBynnUzcInRbtMZgy3Ox&ETrqFX)?HE$V{;?*rkVeW*V=N6
z_g>V;c^+5#g-_nOz4FN6%OC4xpSJp*`cNts_C!TM#4zlf+Qnok{>c2v^WL<G{(L6&
zZN8#vBFoJhRtuR4`#0M@+&M|RtWWIgeVY<K0VbCJTpNrxJaA~?6qVazdSR+`Bd@a7
zk_bbkBVj=iKZP&wFI?Zam#s<U#_G8JRZOq0u5$05>m9Z_V&iO2Zw1?FYQMj}zItBX
ze?F7PyCoVy3y*in2r9|0*z&@Iahr+jf*B46S~R7XMLd}`;q-y^(jr9<13s>1&`Z}W
zdSx}?%mE{j$4YMwg*k|~<XEmeYJDM3a@*2flNM>r*l@^?Sy+3^?0|yk#$HA44XXl9
zEeo5hyKUi#%1zH3*C_0-wT?77dWJJN-zy+;VG{#WqI!pH^#Z2VoFW_(4%{w(-?Hwu
znxw_j#Dh)cEO&$!#4IR!E;5mSipK@3&o3_vy3M~4R{6E<zUa21;Qwls;qNX@=e}X|
z>iK~w9|OM~I=SkacbH!0+9J&cx!NgzR=OXU&-LYn!&w0@O-D9$>DC=xQ*I=6=?dK9
zTHB%};-R}$YRQU2T>H5M54U&;H<&O!nswS)EIMTMM)Bw)*@hdk?rd;nT%w?C+IU0Y
zz;QWtrb}xt)`<ui$+sU7Txb?y7`PzpmY{;M+-*f=PF4kx72EHo92W9s+%(0Z(~vdb
z&>~~aS4KXKTWWHd!lz9sKDT05)(J856Au!&_@Xu{pR>%pnbb42V3E81tT?L>hG|7i
zS?bK(A_p{=&2jQj@ir~a3bs<u^->5^VGve)u;hT3PGf+q$JG$lgwm^xrzV9q{NHpb
zsoz%5<c*2Z{=ATnE(wu;OckXvU0xV4PN?-^I4BjpO)0tDb;YK8G2RNDx)Y{~ExgR~
z;@j<9R;wNZRpC{6i=sRxI#>i9PA%}|yi@R0q4|z)hh^i7uN!)dq7H74YrONIC*GDd
zgE_!bTFJR9?8Zm^1C!ZgE-de4ty;f`HS|^2%54_QG#+=&V5(GDX5)5Ze$=7`r<63+
zu7oyltG^EL<I`4pyvC@2&&;*DdfSTiQa^<n){5pIdA)i^hg{W;E|nWgdJoT=bIBvW
z{fUmTyrcA`SISGTPSZZO#Lw4DgLT)XMW)(Cn^&n`7Rg|;-yO}7(4uWMeV3Jd&<U|B
z$weHA|HCb2ENE%*deCHSzU@K6y~>j_8hWf5u20wgprqv5&G`4@(dBcSqPmnOTUTWn
zEso-`eZuzoVv~jL<vYGj)AVa9{X#_dUtAO0`0mvjzCym7qk8+aQuPk02rrmCyPZwS
zpz)bdPFmx<w^OgbsLh++legBtWAhtcIS!KmQ7KkMBQwd2KPQcpcZ<EcBH9uAR`=2+
zQ&zrzv*Vur+NO8kikU5bfms2gQtFHbhF9O#*K_v7<vnmZ$7aU3b_!#_b`56sZ3*pM
zMF+(iw3=B|A|}>veJa+F)hv>wxT&|6r{rU@qLUk=Ot+6hnXuNm2#*ry7Z)?jLNeyL
zc8Tqmd1l~vY;)rO=oYcHsuLH9-g@LCyz-f?*i>%CpqfYPW~ppj5s<0yHskQETUYcL
zd%P^No5WpRrV4Nz4CrN^$e^jVUByOI&+n=xGv}Wt7Dh5lME`7x*Y-HjYG3p~dz%LH
z{8u|f!$l^sd{(;8_QSN5|JX_%=?90TBd<&d%V-ptDb(R^ame%HL_I~Bk588OC3&4_
z_*BgMh{Gdd;U@h}nF=YBn!*G=tqR|ptH7EwHR4p$gs8?0KbICBbp~Pa1rL|%Z+o>+
zfVIh`yXNi8F3WokOO{W_E?Z@^Hc|QTlB3Ek4aO$1FXMg)CE2k}RLokkg0Da4;XbiP
z9!<|Z1(;S|?%L#Vgm>)**86<_pLdygH1HH&x-8~lz^XTay(u-}!Q*!uT6rflHcIb!
zpt5tq{zetPG?@%%CiM+FVsARRD|-ugnwrh*nQ7v<{Bn<%=B2)?rLLUP9kEVHV&|hp
z3l%m$mV8%bG4tueDSo@8J&bI(v|iEqs2jkX5Ps;{UX@bbuArx0I~UCo$P9b3>V7xZ
zO(7*#lXFF$pI>wRIS}Ew^DV#4<nEZtTMlxJCR21$I=!{IGTOreMAi%6I26sQ?zs2X
zB3|K$B@a$HF&WiN@bBsIa=f*nl`|wrX`=8F>37!_wS}Az|ED3_?d2}iy~1e9ghWkm
z=@g@SkJMDLM%gzRGlJQk3WaTW+O@94bo2jX0-Lpq6S;f89XiwU;oI_KtGG9JJQsh#
zyT0n&ldjDxI?q1emmhU0lq2ZvSNn?z<*R%?T?=sxp0&m;x8{X#_?1&LXO=gGu8@;j
z@$b{q<z`dZo7=RT(>XF6xo4eiURvPHD5=$W^lP~nqwDFj9tSQk{aVc{wuM<-Lu-Su
zXba24T))=Y>JH&87epuip04`XN-}yvA~*A*$(awG=STffx7#8yzw>&xK=lSCP6L?}
zx6*DIaUXnBW-h|v-+1Ejd?C#foWCA+_x><esBhgVdNELY)))KMJxASxvV(%BYo)AA
zTRZ37tW8;)R!k5IJLVf(6z#O@ij_efr%~#%!WnuMH-!WLPht@;eG$DZpvgt=8egBV
zg1B0O9OD#*NtJ&;%=@M(#<{C;f8%T?=|>U|nmCK*RZn@5`u4!~U9wB^3Um^6Lw<Um
zEtwW*;mBco%%CfDL6ykz$aaYv2Uq5{@=cj@C1yp0_^~aepF=jcPR$WESsmT@V!COz
zVuALj!&_Fkx*mIz>DqB~Mn>x9?cTy?Hf_2&^#;R(+FQ$fgqBRWTF`lU4kx2{$^}bT
z@kJ}#y=*3$<R0{&@$^{SvPr>AW(s+Kzj+@>U7*l>(&1$EgO`gtzdUIBJLmYCf@j*f
zJDNLOVy3kfFW@<qdHBflr))p-Zdgt1_AAkO`LJ<GdLBpOIWe<El2cYZvHS0lc~Zk_
zc2?j9e}%0VTI&Be@-t>MiRxao>=YK-HEHRGDQniWie%)3HE!WKZWp`0|9#VDKEbOw
zu3G~)hlivnU+&SqBgVkQ*3{DI!D#YeN?OQn)>*p6;%B1md_&r&lof7G&Azv}HM~-V
zb;<K6#W<ei-pyN&7^`Vq58m)*srQ#1O>a(K?2L|)7CXJ>#I1!QHNF>{nZ>VLeB7=m
z`h5C)sZ9@0ChgkD#5gC-%(u!q@JUR+Dzm0&k^HyLC1R_eozyt8fvf*ZlIXg$pAs&H
zU4bX|$R)ox;eVsB`j^0d^+hRlPA+#Z8M-VqO25?WbS_1!>h+Hmq3b8;E8cpX7{2%T
zgv9^r0?YR(2{ZERS@XS%eIhh-f!gYfU2C72+-CCq72I+=nc;iIA=cEhH;i%>^}7^W
zONAP~&YUGTd3$c)qBf1q<(1NHtHqCNS$r#4VtqjUj5hD48%&j+bL^|QD|fd3NK)Zb
z-YeK8^M}(??1{WMk3@?FN5_mF$u%5RJEe^-v~+uRgnroDxS;*x!;a7sEy15TX5F@I
zT)`grfuB)=b&*7?;e-Crh}IdmJMVwqxrTeK+7auYD-9)YHAxC}tlc4yv{-ljkB#3B
z@6!1)L3Ot9dgi9}-Oefnv%(TsR}?Cp-*8yVXwv_QvoscKt(-J#=VT+XLLvRbDusuv
zIZ~8aBb1FB6^;Hc=Gaiwdf>HnW5L{Y0Sp_PjhUQQH)}E5D0c5yxkD&H!|j)F$dlO}
zA2wK6?ltuh2$I-1-$9^pL940^Yt#eIK!N7TH*6np+f@lLtKaE-ytuv2!mjKCN0>mX
z#+QC!p+j0J6E7T`66mzFZ{v)xhNX8;2$?j>uF2qz+r@QOfP3SL3Gq%Gw>iyt0=bu8
z=1DW!wqU2?*Uwz`G5b#z%!|9&{jzi2k&J1vo=j!cM+>FQ8Sh9cTohBa<S|oVy%?~3
z?@djH4CD1D1uZLOERq*AD>^nd^3`X$_MP6qCA_m`gUUn^&&DkV&4vpvB(gt<cAK5R
z>T97Gb%7=7fMsKZQ<Z?gjQ=9MGq&t}9L>t~agH2|*SZ43h0#J+75SDpFV<hPva3V#
zE7ziVJJ<3UN-IVgZS~kcd5YKuX^zcVi<e0m6f#b_d|;M3Q}6OwyHXg99oDScW;1Qu
z5BYN&SSR0Ny1wGb%UL}!lbHhUFzl=5-RQ)z&*tdFkfYwo+E<)8!d*H}9h`VHW#Orl
zOTuo<>C>1S6>(^a0Bcyr)^KjO?-Cq_0<AG~_G+y0ycp3sO@Q^z=_yeU_|M-w)qkPo
z<!7P1O?GR3xbc77Iyb9(y5;dzF5PBl=9V~d%-!g<HOGwSVavXNbL$hjXT4-u8LXG~
zaRSE$rkc%i3?UplCR-m;IG|=C$eePfC1L)V|6UB8cSMZ@R(`(1!&<PS_qKfZ3chNg
zj{27t4W1p=GY<Z#;cHoOkmcj~9xik13mj1`{h=3FrffJG_+n?>1(qvMI_BNsn8tFd
zD{;!S3ezjA0&dt$+wU@2^MIYKw6j>Ck%`L09TnU{p=Nppf~w9M)pr6cUwU;-2xOFK
zUh+unyaKDn<-?A5=Dj_%iDjV@|4zS|-0F-8y%CqJZ*usY@L=lVvXPRWdOJXvQGwAy
z*gtwlv-VNxKYwgaRQO1Qa)f8_U)VIG@5bIaA`H<vP7@a#5)SH|A>mv0f^+$T%bu0K
z0mXY4O6#sWArkR<ifyau*+llhzy{YP9mau8xA&}!RdkR0&%G?+lxV5UM8UJimnQD7
z30Wr8*(3aPW|fNkS%dv)lda6BG=02z&OlUt%`ByoTT&%O9$g9YMh55i+?iTBbF%J@
zOEsPP4v)Kj`Uo=45}w2(KB3z2n2d-{3xD*IR@OxMw7r(oZcOR&yyo{wax&N49>Y^D
zA6PXbW-i^bkDt}~t#r6yioKDDz3eJ0K|^WYmm%v6CN7W?d*XAI<>Ri#4c9jq3$FX=
z%sOY6ilayKpTLsW7Zn~(%YC@KYDZw<&S_GMJNRNRMFe$xU7~j?;gU(DOc;+!hm*G6
zQ4Qu*2ak8@9QwH7*qix3uJFXB3bgFhF}lFy|C(djht?kkUCci@t#i4S|JS&zdNHiK
zBeGkOGr;)r0@e`54eaq7+I1E9eBM}Z&x|tOc~(aI`Z^Bx+g==@g5B#i7Q}EaGybE?
z#(8e8q2Ssrl8;K<eMCF<HTWrU$+Z^9J#*y9D7um3DK2yJ+&Y0E{>0|c;Es1OS63?X
zynERddGs{z-CoWUd_Pna>UQeaD|Y-!TJbeOfw{Vyi)lk=^ZCCmj_Vi#&bIQLTibDB
zA;-C^^P@YCPG+9_eaAJ?LxR&c&HE8xS+;^TETJ{@c8lQ!CWAQ<?Q@s%|8?rFSi9+{
ztWc~_qR=u~Yi^OdGySH9SIH`MS#Vp3a9Fe)j#lP7)3E4`;%S?Z-RCqs<YgmX-`o`T
zKViO->wy$OVY4oQnWs!udoQuvWC&+ju||RUT*Za}5fyGF8Ma3aK@~dghZdap9Q4s=
zHE-+YtCu-!mx^38G-SPYIm_ho`Jcg8vbxS1sn%~0nP3sMYfWp}io>ls^L1~u+%dIU
zJljzCnOkJg?3Nw-Tv?CbDG`m^JuzNr*~7P8cV69UIkDfKN3Qg?<fA9D^E|t&Otcn7
zscqSMe8=7^4-d#mJe~NOB`DOlqcvvHfxlOlEDBvOaw4W%{9>uW=~hXek_|SqxGOy;
zGCXQ%dcEG&=%kp;Nmb3T?uO*=QGH<<{ogruojh=D+N-W9kCJ;MJEtbJswT1CJ{qfe
zp-Ce`R@302>whs(_jw!d{|IXd2)Y+)RWfDnlGTb^EW*9+&h=W_@kDWwe2}w7UfhWd
z*I&qTtb5F>s?_sZ<#cn4Q0t$0k#XC1KGxfvE3`+JBju(24j%Pyg3C(^Q`SvLDUWp7
z;o0B(gX;w698L>~wHww5|J?ldtK1P~lXD6;3p{Un8->04!Bu_Z;JmK_7qp(w7mW%j
z;#$0web<`Zj%ux+7!#LfuN5jwF$!eLJM!Fnj;iX?Cz__hE?<vna=6cS;%Iuub$5^8
zn%2Z`9Q#+D?XJ~%n)dsOxaw1579$zM{YPG`-M*BYEm0!m>r8dM*<BZArAW$G$j$dH
zI1%_`))fiX3mTU`&7BnOeO2%OM)`AZ6@E9auxZk$^Ino3w;_F<a^o!Vj#~^J8=9s5
z+|h}7)%xLl(~s%XmY&JYj1}$TzcKsnEm>yY8%)k_Ph9g-@{J!FWNnUq%A2)unr@=_
z)?<RIKQ44t%$$*U#n>mXJo36pO<?f5nJf*?8(w?8{*<~gr`t-#qlj7htb|aP+$A?d
z<$D!3bQfOQ)E3cEDy@0-Zp(y*pdZ2kZZ^>~Hn?grA60C&G*;ko;yz-v!Mj-dSdn=6
zleG4Zw?}+Be)x*%X0&(zyxheQ<u7^5|71d0TqobWyHofc>Rw>cRCstzX|kkui1!yE
zW6#-3d4&ybx|umX%$1qyJa^q&G3_^xRR#Yq%@&)ucKr&`D93Ky2_cVVmpzDe|Lz@E
z_f&F^!!q&ONuoyul*;V4mD+DfHGAfpv~B{!>#RqO5-&B@brhcLF}u-nS%S5dS6$O%
z_O`l>rw;VqouH6vHe-=ncSlhlo7Y5M?vJ&<g!{7jdb20=2lKan*ztQ#vbcPQz$Vt3
zgTif<zILjTqK`VRv!>i!>wHU=`L^v7=RPJ)g-;v#vOW58&TQk@I&C|r`LiyK#V6Kr
zNdCGIKJoRfTTlG1X>NGRwMT4YSI7Rn_rwf^8#EM{HGfLU&gD5c?a^VrNo!~AdXrgv
zwjgt=bpJgK;eE}YI$|fA+<K>z$?LeBC-2d+?w~U@6IFEoPZd9$&F6DD=+y<jbqpeA
zZXeC_k|)=2e7d1r@vO-#YpIRv($c)=55rbX6?<42mKU`tec9B!n={_2w)f~sE`B-h
z>$1T6wQd|szq%D(Ntq$|e#bJ7PiZeoeB8gdg?~F_bWdv&pXBX?o%u5>?{z(U-)lFo
z?-~!wg(X|>oSINJ;e*P7H;tJmPINrqHa(Yz;f6<f$@bnZ*Vi`Bd08biCIz3lVER!Y
zE1B(Qj6p89_phtf@f@EXtpDYubMzz^$D{_%jPq_jOhz{fr%t#jm+ql)rR6rO^9Hjb
zmG(<#{O)VIzt*`Y>iO(*?8>6AT;E<zx}`jMn$)pPvlqs!mM*sPeERz7(+&Thtde;e
zH>s@n$TaKnjh`oa^6s}eZ&X!o6y)V-U*(kU@amk9b11`d|0_m{!WH5L=hksJtgI>y
zv|amt=A^yC&n>3&zq$V~GLT*RMdPu3hLXb0QyG?*|GIr);uN-`_1Bo5hJD>qp7-2O
zVBcpYuYac_-gmu<O>B6V$M*5q8{xny%Vj0l%!C#u_FU_cR<nP=a+OJ2aCgDafO)%8
zZcmV5nl)2V>sjt4MtjeYx8FV~<}}`53IFEjX4$)F@)JGB?*^O<t7r6z#EIXDm~OND
zYWegxB5_lN!waXZ`68BBxb8){M7z+QBJHN<Om-_(Tk>66)2r|1AK4S4T$cD`W8IUJ
zWk+KE>rd`j8~QTqQ~z<z`fI1_%{F_^YuFxV)%H$gRg93F({XNTBl&4BwngNyDTa2-
zX*W#RuySfc6YFgWmy8Vw+O2EEoOC?2l#aAB-SF9yA&~57CdC>NbE9pldzYl9-W-op
z3mdAovh0c3q1wozDj=-k5n#}GuE~>2>xZU8qYx`omef>%jmsI9arZJUwK#g#DOzKF
z(%G(0elF2vT(@?ye#>?=Uw3PYR&W=OxL&lG&rAc?CMM3H7fNfROBSAOs@>}JhP!nA
z!ToKl8i8JcD;SPN$ughg`nu}T32SEeH9eM#d>59fb1>*8oXR-euwCL#!h~*yt4smn
zDY+94J87|I9{;a*;$(@75nJu|e{-*vM^A|j`>Yqi_VnGFz#A<-dUs1o?y35V$y~jt
z;K;bx#xCdTsjMSW(M@J&Yo<KA`0euj<$0>x9>}gr<-MgYeKFMK?`PIO-{rl+ZeHw?
z&D{~`CK)qfBdd6Dj>1tEA)f^ftrb^8r&phRd1axaT-A;cM<IhyrJ8A$rHi_p*K*wO
z7t=Cq6=6Da<5sqL9IImAlC2VwSsQd&<}e)R+&O2W2<J`Z{L{Yd3=D#b7iWm6uVtC&
zBqW=ns+~I_Xc9wsTIk)-grFy>{a+UxaTOABcs4zW%g0Q~B|l7Td8${Y*%Xo73F(sc
z5q~o$gf<tcHfWdaNSY?i`+un`8)yDe)rYZ?hay)7Hr<?(8B*r9RkM3;otC7_WI@ru
zm9~>!1t$yq{c_zYSBgbD%xr1Lq}8n(XNAt5-x?;+zf?1Gan_cL$7bc&b16s{Ei+X=
zkat#(<?W`3pVOXX9aYVaaXDLh@MNLD$#1zSWm}tKH#}!@TNZY=w`fuY)0R&`x{7n#
z^Qt}+S~*HGF)h3A^hfN=RN18}I~1Z*d=4IDkq+=kY*Tr3RIXF`o5{y2kz||LwdR2-
zA|3LZG9IOVo5Z8#HD_H%QEG|rf+H-ZVk%ND`wyA1PB314OlrcZecOw-F^Op2o+xO@
zIdLniEbo@oM_s!EF0<M4N-(XG+QRCn_}|oJqSG@mm1m(zRh<)NiWlCU>yrF=+C`58
zJ6oR23hBSKF?GYetVqdPK9Bj1pH-{EqOuDQGA=3Dd&g)_rJ!l#ydtBz&^h%B*C?GT
zc^qZrr_|LI`b1fs{pzuT+fkKkicZOF4%;91RI1N>)v?XzH|%s3%${>X?QN)ydrNg5
zn`rEpEKhCeuM;l)?3*OAct-4|Pf@u_4xevHio`kH+1O<6<FR?>z2~Q``HaLFq(c@+
zb4<I)Cds|)C}Rs>{0{?PsYjVJ&n(sd@-g-H>@$B?UwXJdD`mqs{fOCzudj(KQ<{DA
zO}gson+K;XlUdBEp2@Eh`96$=h0#XSD^TmiyiH*f`Ro2`wlo~<b2lhj`oz~?JENIj
zv*pR`ur16=hAdG+O6{9=DYhxClX`ISUWy3cmZZQ8&E~~S+gW$+;n8=!*ez6CAtClr
zK(+g|+=?)_i8ihm6@p`A65A$e>ODIj+#A@l#^w9|P(!z13z0yn(i4kJ_bm^#%;8`E
zB3<Cpf;_8Lfw3DmJ)L$#(OY|4bH^5oiCeNX&zg(gk=^EFZ0(f9UG8(*i)q^ep}&cr
zw1jsGIYlK3HZ^Rr{^Zq}7Lexk)U_{BG)w--E2WLSj!R2KjXPwkr&@|M>KH|yNn!9W
zQ-Ar!D8%1Uod0E$1TzzNT7+jpj?jwTt9-l7ZM-MtNSQc3j0tY~e|VwCM=qn2*;c)?
zBWKQ9Wwvl#>A~YUaZTsS_?#y7yBx{BypdzZg)i$Au1;uJwkSnUmsx#INucff#>L{S
zS(aTnJ}ENU6BD)sDIHZQG2}R#^29*Z>X5*dUQ?#tj7OU$#6LCE-kQQIUtDaQq4s*>
z9!9^-t4?-4-Jm+_;+tg*UEj{+r@2-LdNN5qndv#n@w|dc$Za>FAloSum=>{dKKtZ1
zbN@*L)>R6|fgQhYo=!~SGYJ2y8NU0}RZX#FsS7w?hHh(cUo*!f;@GFfO2RK4)_-za
zE^sj+&7;w&HO4oV|E7@b&snPN6Ef#U8E3hit6U_O^`zBU=S|P6ZJUn#C|)@$`oC6)
z$r8Dvu^SggOx0P=aq5cpzaYPasX6O51*xB#!qx6ya`A;lvfJqluh74{r)C)zOWfKe
zSdu^IaYvepV)UZy{+q8JZ%hgon6J4yviVMIfxyd{rJCLz=VLr1_iWXf*O|S`IVjKN
zgr~B|x)aOec(Ve;gW}foKCcq+ToqRF<L0@<qSeuB=ZC)LzV4G3nD=|n`!vsM)1&0d
z-OhXoJv+hGSXI+I-KAMUDSFF^S$ewKCU+BD)~GRP9qd+m&*C{liFsMilnIAr4;IXP
zkSXOK$GXSFtHEgJ^uT{vQ)ldqeX%#=&b@1!vKlrNW%b&#9#|cax4f`NXGKuyV<l;q
zpv5mU-2XQ|O*!y~%lu~1S+Rn}v2WFmPl%Yg&TOxz;6qmatZAJ>zHCg%C6O-H7nE5!
zgEbDV`>LrXv~kA&UzN|SA6uTU;(M0yeYL+vSk`>8O%qtm3>d!_E<DP;a^kCs7I%fG
zfd{i~1&jWkh*{$J!^7n4k`HFJqM8#9=0qP7TDkM^(igP>dz@TN!wzgq3@d5RmP>2R
zKf<>Ds94IzAP<MTeykD}XEkq3KI%2URr1)eOxA9TCsWq@zP=Hk{In>%F(<)FdzTU4
z?Cq!7iZTmBk3}1%8LghXCa|Yx5~r`i-F7}1ajOHZLQ@#;efoAb)=WyO-)^UC(7n5_
zb!=sCGcKRCX}8$4?Kb~EWIWuW;9oUWBxP;7=){GUVOm*UEvu3`&pdJAGVKldoAd7F
zp6fT}R9l2Ew$i&GHRZUxx$fqfdqd{1I21O=r7jdUYu%>P(P63IEZLLNls{GB;A)n2
z>jQpW+#+Thc`igDzvi3tlFxbv=dr0zI#QWwSY3JW;<64Aj-ss1vAT!37pfms+;m8&
zE#=CrHyx`^*Cj1&jAhuykS|@f)+b@R$F0w*JM|wnJLg=MnD*D?X{I3aRI^Ff!cwkm
zTqksXhU{zePtVLY?Ye%pv24Hfy;;-CC4CYE&WkT$u?uS0>wEo4|Eyy#MGAc%q$pi}
z8l+v$?mvxtE#Er*WZCIfo`)p#PJaLX-)4!*+1o4yJC2>~l_^zReCCHic;BS+C!abp
z$o+i3dj9jfw@?2p6#f$MK%`hqviO;pRmq`ibBpyfH2&Je@8^7^%W!eJ!<YB(4}9=q
z_{Dc({=e9cdd`Mmro|o|%QSB7TzKS?=$upq?n|686H=ti7~Q6`p1vF~LyXs@v?2D>
zjMH(2K~pybyOgj`=eRbNLu~ch>mCP=-%^=&<6+d*53`OpUVE0}AARRe-iG#vveH{~
z<Lh1L@yXrhE#;9|7?zRC!C!Xq$<NlgM>_*#=FRtwe)REZwydJC)5NfXDZ2^}h$hKB
z&s_MdQdICNOU4sUch%T^d)afhoIAheXq4mK(1PiCCe!~jKRdQWc7e~{<5zAZvbZ$H
z&TRP_*|v3uKwzlMrI7Pnn-cmz9y_M!wn&pz@{ViR%sbZ#Pq-BNop23FQBaZ(n$Msr
zcQ5o>z?w@xG<$x<#9f#>|CgpmlgN?t2k-ymY|u)Z>v&>5!-aV5Q}H@YapGI#-|Mhw
z?u(8-&?$1@Sp8G!22P3d7Q(O2EWD@~t1#6uN|L8V)%T)<f|tj1y?u)^uXM4@X}Rv>
zazmHphUB!{zUPm2N`=i4zGZR8T<a|7*28yB-gKOE{O;ZxJ$t)?at?@HVaZQ?#OHO~
z<`D0xFXxqCDcf5qu2>T`$6%@;Q~1+D2GN*#yQd!bmn$dkC+l@&QCG-?|4k~VpK|gE
z-E|FAab#j~U}9)Yy5f;Kg>T=)sUBzDOhRuil?+$T^-hryk~!w~Hc_hBHN2&Cx}$<4
zONlIZ==|#{XPr9~Tu$875M+_OqLBPXw$`yF()U2{r^`D7p4tUn5iv^KS$p5<Ld>pK
zrC(nkozwKtD_Z#XY`n{{`xB-(u>YB(Qxz8@c0^}gQncBF3jwW?91U@NC!0zdoaLf+
z@y=p85|gOLz##K1k&PjV$$>%Tpr>!+=_4JP3;MF2@LXQm@ya(UnwQrlWJy?!n*XBE
z8@4V}UHY#tZ1JD^jJ<J&i^7qQshu4QS+eFWaBGopym2UL=2^8Z3$k6?vLuh%N?rIb
zRjPFAsRBP!n`q~pOo_{dc8MEAXG@AM7il}SLo`KV@5zlveV#_fiTSBKZ;aC}o4V}k
zm7g1C|2n}Mq~XZa^2EdJS+$jCja8?k&K+L{WnEsyx@9VoS*@=`TwYt8D2%)%W#nFP
z<R0_Vh5JQgq&F!lIxclI@wZA^n)$=c&7t3YPRj(Fmaj8-gRjl_W-6QGks4;ZqUPpI
zl{i<`TQ@rG<~X%=eU`lWl;^J2#QPduPtSe5qxnlldzI4Pi}F#k8vdS5(mJ<P=aA~f
zCy$sv-if<*<zM1F4Y^wjJRK5+mU(EYCF`a6Us+zlqma49CuJ6&mrKW+o%8&b#hDxp
z@#Q=l?KtN@vlh26r)Sy2c^_M%+NM6fQG4NN;{~TR0Y!JF-t;^kbV!&d@~E{KOa7rA
z`>3`hS6G64R!qtXHMlA^N$^RxRs7Sy10n$p&Q;IF4oSDv%IS1vxx}b^jClNWgKKn%
zWXl=XBoS8sE$K`wj?<2%GcX*SeB;$*h6AiuBWkO}*<4*}zqyFy9E?80<GN%?%#5~I
zKYFIVjh=Z*rB#i+*~i<+Ycj*1SB1HY+Y*<qn%gpKY2=5_#S2eK2DizJ?tSs#?vW!Q
zp(kwbFW%N6IBj;A-@>^ueD}_3v`mPP{*WYD(%AGR=KS8J702TH+vdG6^{sF^GU3wF
z7rO0-zs2~@JXqL~dZBgE|AaV>Gm$H+<1&+8Wp#SFZ&SO?@M!tV1%3H{)%y>yW_d(5
z9X#&2Qm)8jafyw1X`|Ncb1y%r+;MVgnAv%^bHy$7(z8`_@4EKPni=3}Q6*th$>Q`x
ztzx2+|C<Y?a;$uZ6!@4}?>S1>Xr!l!y8ZE)czR}6uwu$CwRxpo7mF@U5>?~bUE1JM
zm6h(~va&6%e}cN;#?}W5(qkG@0&?CY=`E|_(%YaVvtIAWdc8Z-vRrGm)*ZT-z9yql
ztmal)!Ij9p$2V{|oSrphe<a(xjGJ$F=%k%{QgSA7#Z+$2L#x9YZnSPYeSTV$ql8u-
zldt|w&*MuomhO8!UpBI5;tSocvU`(4|F?=P|M2Zpi^MDqs}sHVRy6(5K404%Q+3bv
z)cwDE<6r#LxVueh@|OABJ_3`!BylpRv2Wn_%UDx=>D5oObzQSE)RI;^cIZ6(_Nx6(
zy6`^Uq=<EAubeoyCgseeypU&4V)unkoWsEqrR`!O)0Noyg=ejKQ|FhG#U34x%X(Tk
zl|(&WXtHR(oiQV#?JARET*RLn31N4<V%DhGDyfI%Ntf1n6;AR>JvymXhrxN*+wXPi
zm2FzAFCR^u;2^b%|KW}-zYWW$UVF8c>)kp#P9}+W?J9=9_q}gk<nq|+T{XuMrvS;v
zN2a+?bK?wVnec{3Jo=`S%8chyEN?p3z3>xOn<%{VMdp9DWqJ#gc)xI*y!Peg*AtP_
zd;IU5$-ecgHQFG4=`QW=Wj)_~rYh!L+hY~l>cZxJH6~)(qkpFFmzTvo?@XU?O!W|3
zrq-@hZ9Yw}3#xxJqmGmvJn|`tead}xp=X-?OMdQCUn{ik@0Rz6_vQ<yvo=0TPd&lW
z+oHMpn$MD{g_k)Sa%B?IA3X`>Fj(gMIMip7Uv4AYuA8@(K4WoiQ7>(u5%A%qTgyk^
zm>e&5>52dj%e7YLB1MAMCjTv{vvYN@ROJ4yEb>YyGgCLSHD0bf{@S_CFOPm;32-Q1
zmCEaEw6gD2sK%O=Qk=~n3|6ZNda!M*7Ave_TD?xMFva0xiOm)UpOXJB?A!)fPLm#f
zQq$?%tGW8ATiYuR@r^Rmb5vd|No6hP4PyL$A}>Lb$>V^S#46+MADN|#ukKw}RoGsY
zC|{Gqf98*2fWqYPt3jH)lO^(&YxF(+#FituAW7x8YX81?ouI7~>*VgvI-sNWGED#Y
zeyf+Face$Q$!l>N{;JZc)f5Tlc>P$crF^Gp*S^x|V3!!@79ID(*yv-EZya6j=@`GE
zQ^dnDWf7l;W`h3r{8<6A66r^cTlH9EH4<DHC65H$d6>y+>|*l1q~gI<zj9vd#IxD^
zZeCJzXG=D#c3qySniZkBDl_xB)$hsN$9;4P)gv}V-`OU=r#0Q-y6(hZ#YHJ8|0|1B
z#0C6TUt{rC*|Nrp(^E+AvE91$=Srp1FD1_l`MiMb)Va=<2?b*FUtZ;v`M=5J`-U%9
z0wgqVIC*$z#B7aSUF$ME=W=TyXReZJxz5ERsZG7UJrhlr-@K;%?%9H!oN~_=q<n01
z>($%3|J8~Ezhb_+z5f!m>7V4w%NL&NF>RRG@#*(hCH8=ZLr;o-FDRO7wCuQ?+Mj!0
zJ8zgOJ}wOnydO5zLVx0$-Wx}HPF&1jF)O&*)6mCuTIqOs_>GPGM13sFSR}uhKJ2?=
zwZOMFN@jU^2EUkdN{eayjOy<8__td)V~trpXR<8s?mF6iLzSN+`(m+~bbsPh?$tg5
zZ3Vm7X8m7Ypp+whgz3l!+vU=%DHXZb3x41H$@~9`!(x9ap>yuPbp4lX@>sN9(E8+B
zv!iQ2t&-I{!dB|>$jF2(vc%EHq-68fE}xB0iUKCR{W@KouPl4JSaPdmnqHFmHifft
z<=;7y;{%0!I(2<qZY^B*Y0me=r3p5LtLLfl_e>6Z+3=$C3HQ^ZK_}aeai8uCJfqPn
zP@%c#{;#hwO%D0mj}}EgTJ%@eeWlSGCyh1V%1l>id0eQhe!#L!^Y9%_bG74j7Rg;d
z9KXJc-M_v#TIu+f-^GU|m~7O%Ld<)89RHsBA@OPXB~7zc^N-20a^@^q*?8|>iOlEy
z^-*iqMrAF!73a!-tHk*KcBegSlUb`LZ%ZvGP84im-|zN!U0L`>%YA-sPcvW5aZZ_g
zTKAh*h0BYwuSu?MT~D9xY;{-j^Z0aR&9b6tLWZvL8yChu(&nyxm-MJ>_OT_wESL7X
z7j2($MDe!&m5!c}y#3j%6Kqy*KJ?_pwn8D71If3pt_iL>s#-I(At-Fip>>Xn>Z}d-
zGW~uhRsB(^+PTcJ(LDdRVuf$$r>5)kze=qPtDf1c{qihd?|!!@O%3@G{LgksFWcGt
z<G7tx?E1YioAa6es0evDiexYvY}?VfKi?tij>TTKlSYr0X}iW>J|TAIM?(6?Q%)>}
ztUqTi+H&bf#m5b8W^b2dmZZHEo_&7W-v41P?ap^5XYMJV8t-~JsqFCo^{E0b*=Z89
zayIVUZJchnaC&f<?dJ&5CRMRzHWhB4&UU4%WzMQBTH^TI@?8F!<9cdrjO&)PcRW75
zRhH?%l{3#5%Wv$+&wQl!J=Q#LiGATE<`#xVW>yBti~<EF-z9Dvi%#^3x@a;m2yCek
ze0;D+T0iW`k3fehx{~Ic6OsZ?I%o=qam`VBpn9TTnx$aJj@Hv2dNMnB-e?9hsQ1{`
zM~Qd^WCZrP>k8#s1vD`jt&NIRVZEZH-4nI3>}%J=9)@-DtNC=iH%vRA%O%7U@*wO%
z%XX$G$Bt?w>K)`{y61LPy8I9eUvR~=2SLw1JTy}KziUe5ouw@d$0n+~zf({!I>@KC
zXP1WZgW`<yX$HlHIWw9QIE$WYaXgSJYhc;LsIsE=_BMw8-2LCg8W@y6bRS~<@a^r@
z$8L`pCQK+gv1;1k#%KnC_Z2d`-?OM2_T4NBI(1ToWxxMJzGD4jGqPAUa_9fGt!(Vz
z+@=0wW<tat=EGcX^r!A$*f6PKO4K=_7g|5~?s%-;d`#$tBcqs#hNIa7;fCJLCL#;w
z_H6S=(o+?gp}oG}<4pFt10gCs%=4@o*7OJ^&AHfe!9w_jTX)eJp3RLFDjLUS-b-t2
zIZ|;dvq@7;An=Gvnzup+Ll$$>B9;|vI!--m5a8puw@E}}XU`h$|H6zd2ma_T<YwKZ
zXJMMUDQ_|dyVOxH^S5t&yC-d#;itNvWr70d{%toFENzU7@>iR6L->U4>|4bpfsAfW
zulyJ+f1Tg1qBLO<?<ylH?{i+RM_*|9xIbLc8>l9uvG%3clI#Uv{<VZE*k6wJH8|ws
zeP)A^!tbeu!d^3SHtYH9$T2)=_WQ;0pH)1(TQYVVFmH+t6xK0djq+A7yFKp+Z_Hgc
z+37Lny~cSKj9VJ|M58u&F<F%8PWrBGc3d>mzwz6h;w2vwxJ1;Zv&8i;(_LopW6?ad
zS}p5V{a@!beVAKrXm`HeY~nCg$i#brAG_4*RiXM?t9d!>)lN7yy4+jQ+Z*ZmKeAyG
z>y00)S7preVyJd{7m#Y?ATv>a$GxCH1FZ@z9yiIU^ESrJT;ST%J$Km+2Tx5E0mH^M
z^LZMwwU>&BpKjvHTCMwVf&#~mEf-bJc|E$~wqVh#y2-b#T27tmTV!Hrboq=itJ8_a
z`!+s`Ufi=yM5C+2XwoC^{YO>JeSRoK>{xOtTUF!2WVcmTCmRI+cB#46PU^q7!kvAk
z#_yh}xfkz+sCzJJh8@!inUL{sMcMt7dtA;F!qk}cBblYwY&a}sy|3k%ocF&QLT#cM
zn@{Ulhc%qmYpz@Hx_Qr<?#5lm^?zRWIXsOyck9z{rZ?l59lY}MFZbSCc6ZD7f~}34
zt}Ff79rZtN!HMfvq<tkjot%Bt8eVL1;@)=gzPp%*-e1K8?w^k?i=8N#H06v}vHn9p
zAH5lZdJ0*ID?j^kd^TRWsHBl=?QH)*HP^OQ))gAR4o2i}yPz=P&AOPVhg^KlM@47E
zK538LD0wdA8dvp}Q%a&bXL_coDPQ2|jAVK_Ph{`&y=k#2_VX1P#ZM@huG;t{D<`mF
za*D9(H&0IQcy1=g6W4Z}n7Y>Y`|pnb3@45;%;PPLx^P6oY$L0$2V;N2qQ`TkDwaPn
zNauN@a3XSBLyIGmYe9Tv3d8?tdW|{HTUj(uR<dkZ={eWMvxIZ$!d*&M-!-zh`)-{)
zYIH~>%<OHc<irNCH1+@X(OfKPi8mucy~?KEl;!B$rf`&P$Avbd32nxQl%_HCY+chE
z!>=4H;=#Eto$b#LuNql}Gb|bjcAN{=M{ix&`@5*vySRwOBJjcyjc+Qfx*mxti#raf
zo<BM9$Hv}9iJN_V(wjO1-w4M2OP+lDS*XnVBl@vFq*r)uF<@)@psh1UhF?R!X~jz3
z9F+-0UA{>_C;FUHpQd}nizj`?k>wMvC`CtHZA@9Va8t`ENA5|Qr<Fn!XH~9p`f!m;
z^H7kHQO47bwGNMJmOMC8XxrOuee>jkRbt21Of+Ql+vR(YCn$7dX{N%lN@<4oX%qKc
z^fgaxdi?*yE%V>8DwC40sxvI!z^VJ+#fh6&@~D=dC?}&wQ+>wo=<wgm!#4#Pt<Si7
zJHb+D=dF@Uo_<bEQ#FiEzIG7GmsptWI%z}eB%O2qOg9g3f1fnZ`QULilb4+C!InaE
zW!y{GiCoaOwb<2n>Rj0#XMvR|)pKqid>$gUab}))CU^IbZDNPtbf&H{_suBsk8<;s
zGoG0_gRj7G_R7RwCY1?x*K4?Uzc0?Q-FaJ7bB8GxPoamdh$ch(smVE!XZ;jwoiFVF
zHFeUiM(H~Zo9<fGsnmTB(YG~j^5pdL(v6<6+`%Y)QbT2*g2-1-pF5l9^%bQg%7^xI
zNwQ6uUvn(!y`nRh;Wl~CtDm;Taej2TwZS>QsOP`aR^E@zO&-%`ehtwIc`0z7;}V-@
zfr6XeArCRu4M+8g8dfG8i3*vYa*ToDVRpC{$Ml}bGJju9?fa9gUnj3V*}UuY-iXd{
zhVK>*9T7)jqb9U4$Y^f&()_Y-8&5-utE+p<GRu8A(+;HSKf18uhbrfN)(>J<-V@G$
zV++#vQEe4x)pTEJmUy>SUw60A!hqI_qtDJPjqrLUe{a@NvDHefes;;eX-UQ5vIbGx
zmPyZk-E_J(A^6Lx^Olo#6fNDueDg^8;(~~`M?PBoj9snDeJZ|LY?H32=lU6+rk`K0
zuw_%UL}$5-)=Dlf11*ltC%j$KHC9Q!X<lA>YKM%NX8T;Tpe6r5^NTNadgs!<RQ=MH
z8D5(g-JTX?Ro*@20&n1*7ZZ>E)=u0QP@j<_D1DjbMEn-1$sIB*O4AL^-yEJa{nWOl
zzgBK<<bVIZ?zOmQQ$WjtK(iID8@&Ag-t1Kno4&^VScnB<hSQmy%B~{E1Xv$DIa?T4
z7=7oQm;L1SJu!V-^AowPZ<==Q@!^bEGkJ?b;?_Co^JZBW%onUtY&f2-FoCJy@{Fbo
z#+B!6Hwxd3Fk{QSRkWnt_jYct=H8&D4TrWzu5{JhmJqu=dAmhnu<7cT>z0}S{`D^7
zR+GvSgRZDG%uYMk9+gkuTO5D;<N9DX^>yc24;=fy^<-`Bo7pS=TzB@Wd+gES@PFQC
z9{=<8?l(2JZn?BHAhOZdTA4lBpqW4BA$xSo;eh(R?u=(#7yB<)my{D~y~H<V!ozp3
z-#+U&tx@}}agM}&Bh8OzZXD>EG<A*6-!RX-YdfSTOuo6-!(sFNO-p`Q?4PbIyWeh(
z;Qqqo_2PCPL_F8%DzP>klF{g2Zu6u(H+7AK+<!UF$^R4HJzMBr`Z-CxOsAXSx5rH#
zcZC-|lS3Zuf8eRRYVB7K&I@ktfnl7lQqJ41IcILNVJ}Nk@C}DwL2r~M^_QJWJ(8xT
zypE~eZQ1t_H-?r>rB`_&XS~>+sxwM?F%)Iri)fs?q=9GAf#|FcH+Wv1Z9Dtth3EBG
zuYY$4Fq-{e|7D2+*FDZJ4>GxD<O<JN%e6T`VNbI^!;+f@kF39?F0y!}+HzcVUGJ17
z?`+fZmBrM)u{`7b`J`Ac?e?d|cLJU>_3K~mV-C@I#1cCp)nHMo*&~OjgFih!C>y0r
ztVmm7<-+x3sh=6s<9+@GJcq6cH9dKjw))w*)u&#q(n;Fb5D@di^~n)6MVY?G9Ixw@
z`0Bs;@B8&!NVwtLw-;Q2dHr%53-#7bda(YK8o$rBuJtVjYYhCwFKWFI(q>qDY;M=O
z{#h3{ozc`W(mpT77MQ|%@Yx6LoI`;HNxZHn=B(1wFjJbgXp#ND4cmJ1w;fn#F85Z3
zyJPvKkJ?rGy3YcGc>E<q|MPQhJ7762K%y}BR7>)_ZyB%F1+1z$`+Jt!`%~-xJURbd
z=6MX$($9yIxs*GUG&Q(SKH&Pgl$-O=LaU-zXHIiFUuvD@%b41`@Q^YW+lT!|NuGUr
zOv(X=ZUiVK^|w4=zO(PpVWVTrpHs`tShQC$336W4i8@n$<A98AlZL%p(jtCklY`#h
z+@Ak>wZ<W7VFm9#hr*mC2|N)CMQ<c3Nxh%=<h=NK{y!U<eHjwGjgI%<TlwX}r>A`d
zF*jUvPEK0=jN|hRosBF4^GqCXa&2I+O;OnQf!i_3fcdjB*JrgI%mG)H{$*)<$#%@#
z#bc_;Dc;ZyX5E~STTK0GPc?3DY}liu;rc(oF=^QkhQRGnAKt83EPwUMZXQD`A0@?b
z;rul%bKY#6mn6)2Hec9$owpD}z`PHiS4`-S;ZW|pY+(7xN#L@QO%eBWVI8?sSA7nB
zIT4_y<n!&C0sr1Znq5k*yCzS$raU?C(h@Tv>(?KYUNQb?(NMNYyVsSG;wGSMv($w3
zqeYOKW7I;~oQ1M#DZ(e7ZP>!4cQq^j$0<*i*XDfBpPMZbF?5&Ty-EA3l6dFDvpI)F
zZ!dY4)OY>KyN$>C*E2kK5MQrTnU{M>CHKuUd+R)xaKYzO7JL8GFHUmj;{Tv;d`Wr#
z1*@ZH&8GxgzCL3xQ&`1b%SPzU<bRJ0XX_cUHZXk(P}}+cwDe~KrU?z!PQ}VrpS3&u
zWhIxChZG!<a!Co;{NvaGvsz`Je&s@sg93pU8h!<vy<V^V?9#kfpFT*eH~+*p{}`Ln
z!T_#^D_?B7U~RC;-#92p?ddl*Q)Q1!Dwa*>ye|0~d}_7ow_5(7DdLgp#*fO=w`|?u
z++ugCb<<=UPG8+HwiLZ4TiegJ`b$KEj$M2spz&{UN`hOOQvk!S;z=LYPwMh@T`RQV
ziHL>D0!5`IvL_sVKNER$Lcge~^pO~cx%nf_=>?73`1Nf0K5H@MnQXBT%YLDvpgQ%U
z{_KhUI^G5wliW8e$2@d)PgRm)Px?IXqskVA%?U^K-CllaSfsZ2TyXvWLnRgmOY};W
zKTK8DDOFzLpxt)rwa>G%54~RgY}QLcwuiC4O$gOGR^qGgsq*MZ$_thx<!$mG4H>NS
zmDW#aGHbMO4AXMiW+~5>{Cefh9nJ^XCQQ~^^_9!*>xzl~+cpRN^I^&lyJMtTy6c6q
zt7~HTgya$r9XHbuan6kBhj%?bYbPwz(pe(<E?6O9t9_tIuGFXcRVjZK@Q7Yd^X7T2
z%e`E5b@IjsD;1*-rXNaitp2EQZ;j8YO*dwlPWr;1dr8di;*y8=ybJcd5)nM$s;m&L
zI!SW-)c&N>nCfc=oT4gu{-OQCD&ApV9yovJ`|K{d`GAx`SO%+6wQ|WlW~TeeapC{Z
z#y{Nbb^W9Ank)RZ@0E(Oot0EHc{v15Hstw#(|UX^fB)h+DcfVXG*91H!D-3wZ{P24
zKh0wGhL~T@jJ#hhH!=r{T-DaoE^2I2`K|1s?_OB=E#VK3fu>Fg@AN}SLOj+}1I<FU
zQpBvYI^VqFz5lDA<St8^PD=TlX(Dl7m7288d#=`%Zoi%*dOgJCbeZ^RzK|D3Tz{A4
zoqlv>ea}<{Ce21er^ct7Zd~bF@Z{)+AJb06l}K(?C|ExsT6?*{^taRIO)r$?YB6r&
zDe_L*XR6~8!5)4i<#~%T*K{5;-j@;6lovI<%rOi6;ZhRdIrUX?P=%As>R^Q>>-aYE
zDGT^EM*lyW(9fS37*M<Mi`Ug9DM~3_&Hf5m2NOLNBMYZ@zvisp^vrzmhtE=86{mMt
zcxSbg78PEf@orUk*<#^<dn>iY+SHaY1p2HDazEN@ebwr&vJG=u|K3H@ijM#E2`*vX
zX>xpBTGeu$GhbM5C$11s5xU;ulXbQ3m)eaEgD1Dl8U#!l&g4wkk@BdGS@DjE#fobV
zf2W_}^KWE-l(V7BOq_eF@{WAnh<>gD3wOoo-aA?<uh_dOwmM#NGhY-_yv4jsWQL$=
zjPV=p|H_BwRR*p-cEBuERW(E3j;nq1Mzwp%(kF|npSoLYV$)vdl$e`s=l^{|<#DHV
zO_}Ze%1S#$|19l({(ooYt|xwt)BD?mQyMGh8yd04S}@ux1?u)LnA}n6ncl)*rOf=(
zLZzotQp)|{^~E_MsaJ)xb+VK-zRqS>;!}R=mYS5VXw~)o@{e!DWz)`2c&$5kcEmlN
z$h0R%W*Qy}^~~A!=sWAh%cb^eOh$sgPvqD0dfoWuefrNtdFGc>!y^^8=ug^oKIhD_
zsc+X$wLIjy*h`;#!t}<O%QslH&6JRHbXv%%wEb8^D|1x}<F4tfPSY)&*zYbW;I^{9
zc5AJ;L8&3ry5q(UVr&6_n3_|SJl6!O$>fx%T@Ln4H`o`h7MfkXU1!3i1vfb?f1H{r
z+sGs+&hpFX(bBrpiH|G0UrLsr|9{NfavE26@%+d}k)BCg{C96*S4#C;y6*qXt^XtA
z48xXK{&uq9<dTdkvGLYT`nF^jx6SvSYWquDn2se*yOOAXWMk}vloy*rb~a6)A~tdU
z`LtC|_S}IwDh+JQFa5d1EmS&r#mx@3={_qwC+=Sp`l90S9&zoJCwXF1e(pJ+woz+Q
z<6*Y`MP3Xhs?mlwq8+C#|7G3QYt(jpQLXk<mePVF$EKcZ___7st)wXHN=cbD{9D(`
z-kEOhlr-mN<<%XRB+PsJ-&=o4T_lzk&!Vv1P(0ncGmcj`j&XKt%LI#Khs`f;;Xk|G
ze9yXNSvJ-=x4ydsntA(|U41rBS!25AL7(nFnf?FoOsroY^0-*u-nhIjW6j2<L;I7q
z$zMOEJS%2Z%(}NKI$~@#dRAZKI=*gMo4IQHQNgN%Do<x`ZQ#8y`IztKIj+L@r%883
zZ(oq8qwRG}>{Vi@ib77st-8A^t0qb*-!N@_n|dMp>pYF@NnbN<K13_64p{woy7tOL
zmv}E;%JuLI5KT3Bl6vl^v9;{<C(92nO*-Yc<4W>Zr;?wmQw^5sI!)i1?ElA`nZ;?A
z{-HKsn|4+uPuo!5_@FxsyVdSAr|BzcHvK3m`l|G-^I}WpJw7pwGaHTb({D9$T$w#5
z)abWht3m#E6{eh*IZZZOLXVzkJ#~kd<!-B4sPo<$$IY)hYPN3v-&|z&X`$+_tExVy
z4HsRttlm+(C)Iut593ds_1_P4MX<G7FEU^L)g_8)+XCLc+xKqlZD8AIdwcVQ^N|_u
zAAXw5QJ%N^XIk4X#hWJ43n#KvOqG_H{NHV@(nRwok$X25?fvd$-p04G{cIGctmE!)
zPu|a7HT~kIB#Sn09{ct>`|i4X3UO*rf}M6;+Bx;xql<-C7Vj%}VZC*F=~dCKYxhQ}
zO|oP^{OIawtIlOI+iu<{bgihe&J5E#G>cj7dg<&SM<?m+mY%`w(vnY$Q$J+i?$^$F
zp{x|CcE<3H%I{ze-Hv(EzQ#(EvJ-C~S3R$nvbp{+r>oKy^TreXGFE47FY~B;j{dLs
zIsSvPmAvk}8z-+W$q3!`#^J7(z!NPEn^RloZOhGEpx;#K9dzmVfw>%KCRZ(1-T1v~
zbMDrhmv57;<J4W}*(;TuaCqXo-?>KU>vWmS(@rlJ#?>~O37W>F<((EQIi_@}$x0()
z)%>cxyo;tj{k_+B-hr;Q3jA~baPly0-75Ki_a)uCOEyZ_Ph1gthEMs=Kc!!}P9`i0
zw>Ts;3KXQnm{}EkR!m4d)XFApRdOP2k!$-By(pQf6COMCGdX0*XeL}fVyMR2<GG_q
zNwQ5-IITr;lj_OY7R6R7%9oz1aq>*|n6OE8AzN>dke|SYkOLyi#b<Nfn6Tl610SCz
z&z+`F2J!#B@$7A1XKh{`!8k{!=wZTEZFLqV1`8hrhIbB&%1@@`s3tsf?yeL3CCgE`
z<o<z%)PJQN8y+1!#>39P#<Yk#cJ2(xQ?ffE)$gbXPV!J(#kw>uY)RC{pxD(@*M_Z)
z-@a_lWih|o+YE!3`Tod%c*a8UpB$&<6YpL_{ixC%GCMv$7n|)o-!InTl18)x_k_CQ
zu1TG)+kGPK#9rTc^+K}ke%?pTszpmG#C^_}r~ExC%>8Mt{)NJ{eN0<}<l}d%9=R>%
zq0r#s#G&y*AjaXw)`Vtmz9|n{nL=g+wD9mvabfc0P?#bTm9g=tTGdO{2VHNsTzb^a
zv}N(dTBZxv`0{rjOiTFRYx8N^<bI<#m&FXuY9ULU4tc6vEO=?UbK+zbRrgr`1D=`(
z3d5|XO!7T$xkbo-v8pO#MuqBD(P|z>)+-L)z8|j4J}>HeH9R)A@Lq6yYw5k>_@$wD
zQ<-nF-t)0rcP-kjO>3F*lpa47NzZK&*H*ZCw5Yj?L<zJ$(pa%^+pO$0XV_+ETRk&d
z>9>q8MXOKvu6WiY^Mj{e<oiguq`Irzt$MZYMd9*^>#ep<xl&q^o0#TTV3)e4f~U|>
zE>isM%@;-cCQYAE&6hYkprbKu8>jRJR?mR7ViRw#a$bAp_zbr-Ay4F;-&<W_i|;T>
z>9)1|WWlz*;o8%i4x9>%Ni2aADjxlpSiEA#qHd;Zmy&i&e*0sw^BTV=1xJ{!2qzzt
z&emCcTxNUn?V~nEAs<~H3GKWv@y(<5Ck{HFR|&hDJmy;JYhHSCg0IDCo4He~bhGca
z2d(kEw9#$%%V(}3zE@0Uab`Z98p#(Jc=ak%LBra4mz++!M&$UOjlIEdwdA2gpuv(>
zhMBKgKQ5LCRF8<aNM$j1b=#J+-fg1W1mT#eT&<d!%X%+)Wi9NJy1il(=Y&<x%HA=X
z&L5bWwezva_3(t9`*zx_+!@|CWzsp$iGG$>W*lXC^U`-x)AUjy1_P<ITV4y_lodF6
zi_P8j+ei1c3UBEmhixZLh`*R|RHtDVho=4fm&SVhoBy8_`Rw7=A=SnaC2}agX+s?Q
z#uFd9`KI50{H0RZv$k<fo?Yd>U4~0OI()uvWPkk2eU|?v99K_$$~y5nXpdj1b}Zw1
zPpiis(@Iiz&8T2W4|x?Czb!#&i|A_0Y;CT93C|{&J>|0qnR)r$8<UCapS}#adgx5r
zuT<G>vff-(Coj5w+it(>>ocXY)qHojif2rG(k3!<(F2KX4sE_ATBb{)<_H<K&phsX
zq|@igZN{Z5w0>VP2(g?v)o+W15HnXo0}o@TM6TIUqj{ND`ww?+(5MjKwMWnRoo82d
z*CQPpLv_X2<{&BAOQyNKEv;8BGIc#VJfUPEe;TuwymQKyO^r+5c)0%0y6*5!Y+~b{
zlsL03(`{FzcUN`j$M9B1@T$7omb~??eQ!J=TgYfrf8-LmEj#5Gr_2<Xcrt&AY08r4
zNl)bs?incVGg<6*?1jt}9+xIzMo*tjDrc^obji53u=`?<nP84;uO{<HWxq3@L{|nn
zX06_QOY(iiG~>G7{)C-f+nz8j+_6KrK*clK$*p*ipJz|l>q$JS>MJGceibVnd?UX^
z?59igHMY>pf%jI5c+CyXJW|^lC0;e5eR6qci@Nf`3}bUE*F_>rGp1H4Z`x34y`rX|
zT;pkmX{D=gZ^`-G?}l=IBDZXgY4k7p6m`!h#CWE!+r=bnrb)lkJWkDzI#TC9NB)0a
zgy;T{yYE+WE==uR(sd=oF0k&$>eM4{l5-}m;GV&gDekv$`K3*DPgHbvb};U(5KeiQ
zW4$<Hn&%P)U*olJ*KtPPc8I#Awk$)~BBD|%^u#f_)o)j3uU_1@z=KI+zwBwnPa+dL
zUTCtm$m&hppeTMPW0qu4j+s&W5?*Kj%qv>6wrxoho#g7)J+CJAV~W=-4nKM4Nsl5}
zs#4bSMbABQrsd1GhSj&5HpF!>2&jr><p(uwlFPniYv!G%DWf^1rD>*yK#qo#hVLO2
z<=`g2^~=OQo(~VK6#l=5C1z&IQh!+{Be|!?Og9{vbJ%L7R==g`Dx>D4u*jL4PCA&@
zZ&-7`e)a#V&_8xxly+TNs>#o}zEwtLjljAL$!e#Eo=?BcSXrv=tG1H$B1eKx>D|?C
zUw$PnO}M<oyz*Ajie)mZIyXmVUWwwIAMIzdafUG;=f$MIuG2TG9*rpfc){bC$ue)&
zi5;^)yh#<FzCBQIrP$foy$+H(6BXaK@7bifK4anK_$h0{Ca$?Fv3OUHT;ar>RUcl*
z|L&?f@bd__sm0B;+bh1_zGAmLL*>_om`Me9rg}N-e6uj@9bb0(gF<iRcE67M3~CLH
zoOv3~3!kg`xx94zq~f}SL0(fXM*q-(nWFkT3ts4DdVIQfV);{VNs;d5f=iR~R{mJ}
zbSCc^>AzxD$sSi{=(KX}`QPR6<e*TSpZ}N2><1FX3iobvcI_#N?p7#PwS6u5GR$bo
z-ntiu`)#k?nqe)-zi5V4qwk3X_M{ud*LIwb_<hyoh7d#Vn;phwktWVc(T}G6Gh}q&
zJUioy?vy%jN0UXG5eGd?3X)kTy0|#*5M%lG{$fhkrY|~Nnzs%JI*CTB|K>L4Eq769
zKhd$eXg1Hz43_Akvt`rIB|0CP7o2VxA7d+aGN>fbTqwCqO;i4s@%D*E6W-?>FKet5
zmfakBY@<xMWxcBBqKlhLUNm&*`Tw7AWwGDamC<`&MNGH4D_6RH_L^Ht@%5_57D;^j
zlN+>3Eb_g}1-l2oBu~6$W6QduZvHJs>;K7^uY!L1*+;oNr<tqk&5v}C68f=OC|JF;
zZl|hu&a&+A+lmf~AJ0iH+M;QrtMzCbN4Ng|snX5t{V)5(Jdb!zFT1SDk(#{4b;(Q?
zq3165q!?2cC*EJ%(y-#<bWJ_yh6w^?WgN%3Cpo!(bDHt}SI)J#Fq`!|x{j{hw9`DM
z#c#&<l{H1Hs+BtV_ZDCHt8Mq#hS4ftG3}oX>uteQo0C@MPG^GM-UY9FDZa4wtLlc#
zLs#5Wvhqc4t=_+yL0EL#frG9K9w%6c`V_heO$u~a`Bcy~#D(W#L)O=@jHf~;w^T@M
ztB5sXb6uQq=Tk*ak^8r6j_RyVGmlqFC7SM=<gT)?=|AViJPSd_h9b!g*CplI>bsT+
z3Kp0ZI|?-^RdY^Y*wdb@Jkj>~R?hE?MKV+MH?tKnSd=Xdu`N2TEN^ZY>+03osC@Ce
zW!pmm?dvM7&szh9iZ2JrUSVN)V6K*46mwFQ!ONkJ^Ly^OqL}1^O$Q8%P1=ML6{}wu
z8aSyjd`b_q`ce7sh<I#!joqUxPt)=rrDc2*>xC!Q|6VREwK=YDMMv1y>@Us<6FIyc
zA4Q$tZWuYkZXHw3M$tsx#D)fTRojWRqGDDlPc-;Ca`+>xguiv=n_8za)m49uxU|vn
zvx#8P0u%LVlEpjfPF*w!c~JSxtUavIug)mhG)=7JqU49P>i_=^2?VG|I<bkT&0wf=
z_BEQ3Q70^ASs}?Cq`6g`v1wEC)^tI~7Xo|So42VKMo%bT*;MmqdcXVEPWP<>9vNL%
z3{?ac>RC?dx>0Ur{JidFdYJikvyI8QObY(*rkgmViz;{cmAtTSEYnRDZ7y4;*l6K@
zAxwYyw)kHW8NNTtTudd+5A}yebZtx$*xJ;^-XYn(U0YAQd!boYc$3n}M%9!fVf<4j
zm%Fe<Kj>Cs($Fo{3ERl(I<ds7aN<=(r#H*RViI(fCr%Vs?B`(=f4xFD=B3{?iF(DG
z?UIKSy45tEztI1v!La9e-}M<)6$@G2Eo8kl()2uq43~y}G;S?DUikm&=IK{1$4_}U
zU0ZT;i(ugXBZ2M3&UJyt>L)df#M`D$lsq=6;QzAjIUkz?uIIb370j2Mbt9}U;Gn>H
zrH;Lat*TALZZJ)~kl1X#uzfevZ08@{|DRY)h=^x8&=Ik<jMK3*Q+&?V?Q#EGvzLU-
z`PnS||AxTvWcf|5UB*9M4K(MP&WyV-QE=^vR@0ddVlTQ<L(&&J7uPOiyuvz>nURG-
zLHq?L1ILDW3>)UXojK1zLflKT=im>upv0!&iFr|yvK9+{&uQppH<t9c_3l+?c>AsI
zri9YX8+j|P_sum_l)Mz7%4Q&{ly509-A_q_?UZDi66-gKnW<anSS_^a+9?t6ZPNeX
zFj>za38h3m&d;r7&$PL3*4=3qS2(6&&hGmru;=f}SufS5D!r0X+*y)cDVTpFbLY3B
zsTG1#B|7_G7zeN}_3xOz`IyQNm*{zAMZJp}CKpY%TsT>6VnwV{mhcL}3!7ONFKE16
zlAC^VUg<%_o@?`j9hWmaXzDdn@_szar`Xj0a9xC>>63-(D`(2ZT#%f8QNN<A*LsC8
zW3pgbqRD4VLGDY*sh_7My%b2?IhChSr?y?V+)_owD|PXe!t;r%Y=n%Z#asO?R_(f2
zQ*wQgSaOlZam&LjiH99)onJ*K&5$#>wVFS|B-2^x)<eJ9uVfe>v^l7iXfCxrdO};%
zL#Srg>i>41t(q^Enj3a5$eMK7ap~2IOOnh5{H44%9*J997Nz*z`|v^=lTT@f9VcJ8
zHMhCk!E~38MA{TN3D!$0Epu*7HD9<qc!P<IrQf3C0>5sGM9j>)Qz&=-lz`DRQ~hPV
z8>b5ht&)_QwUV`JBbV34cO@A!Zt1>{*eJ6*S&7@ffRn-PGTT3i_3T?#1<a6By>4mh
z6c}kWH`-;csDzv3(V{C4>&}JNN-8qmFqB%L*qpXvv2a4&qk}pN%AJn7Ezr^SDVr)~
z>==LVmCOUFrTaA3O<N&2^~ctSTr!%O>%zUuqda6kGOgeKYN?{4u~eJz>1z!^L5nj(
zi_eK!xqcLqTT!|@y!ZbM#?rZhEL;ug)ma<F1f70sI;AIy>6A?ke<}KGo3=K~baTOJ
zF)L@j`cW+&*!J$(wBBu1;;*N&ziOXWt;6)FP{lD#wtZD{_pXo5t>+xm$|YOZWo=uV
z71}*>QTIyo83&ifWGpjRT<rX+j%Bx~^)E$^1^K;YYc5Y*!)eZX=GS7|)wMTW_Ad1-
z3A|l*+fn~>P{*BJvs=Tq^}dif8o94f-TTYbIdbXtD#k)(?$OI%hd!Jt`OI~Fq4E^*
z6++^Ps`;;FQnpUeJ}5h<VZAhmL0*>Pnb#!@-MYUxi%qZP9@`-s`%(0)M|;OrxmQOr
zbXdgSUdt1m#L)DquV-7yTa{gZxWxaL9IU>1!l}rxcmC}@Jsyp>OXkgNWN-^s**aU&
zGG?iY<079QHF~=bUr>|@xVG!Z!`;uFqKyL!`xg1Aer(w8v|S~!{p$0uLsBl5<<*Kz
zESo&rPJL}_4qL%)vi4?nm{3B?{G{w(GqyfBy)~+OU3B(3=Fs@>Tl?MKc<c@mc(!Pp
z)fNA<D(SHiR?$J*&y)#Qo|e;WN_R9{`QgR1MYHF{ONy)2%<Fm8q|CIJ$y#WRdIVqN
zj_`x-YFlc<4b4tFvUX$%c5}&2S-tX5%fg8=;#ZEuvOn9${cP8^GpnQ;`|p*WuBp^&
zyLtFX<Dw&u((5Nldc>UG9(Alu=M;PAvQR^*iBpgMU$uPZ@<}XD!|Pr@)H(2Z^U-X#
zEu1!~k+Qxg#n~IzKDPEgd*qmS;=Wg>&&5g~-==N3_>__RqHV2$J~ex^ZJbZ!t`<Ds
zHhIshxhr;7NG~`c$8>0VVTQ?`cD{>w&9k?(S|42yo*dt}!FRKmi0n@0k3CW!H<*8v
zoIksFRk)dMPVZHXU3X5LlKQh#y?ViiExl>Q8x>A&)GO7gYg~1H;^Fc+mw$Jh_TxBW
z8|!wubE(;E*LuOUzk4=MsjR&geyq5)TT11u-dkhqs=B2ww;Wh{MqQIJ`h(u9nsd2x
zLPJiKsW5KKKO?Yn=GsXT+a??iS+=#_HzV!Ra~XybE!NEL3lGC|)3^UW$z%I>cC>4w
zG4r2dxrpYTJtZ$*>c2d>exCKN9@|SCr`nt+o;<hlq_6eG?ZT%Hd`!~c<~L!lBv-*s
zS*vM%R@3S=re&@aoW{M9;qX+6S8jDat9HtqkK)}d9lZO0#F0+PGsfMiOC`FNzluD6
za9h`Q_aoKI{P%FiG|aSDSK8=&w#Uoz<gcTrjaYB#9J`RQLdZAIlCeHE$L+qaWtA)2
zt}UL=Y6W+Gyc*RzCr5S;)3vNAf|kBb0_kRPt3oB_|8nInJY%={{LgB^7^m(vlcQsW
z)a0u!@MuLg&Ny&2@uYL($;-P;ul+t5JbCI3-L)G%FWdh()aQFht9F`&nz+;@$#yo4
z|6FhVlx#NrOWLH(_efE9V}YiBvV#ZLLzlm;R~&1NG%MvT{#~|{JFho8pm);JGwPdN
z?;h_o^bCqo?cMa`!bF+roF(~6B55Zc9Qb^6NoO5jx8HK-gQ3B<wT@m@Y>dnJyEpI6
z)>!XryTr<0^$H*OdF<Vt*)Ig+S*F#PUU@7Os%O}If4TCWxJ}D`Nnbdw)|M=F;nS=0
zTW+YH-Z<HMW?SN;?q2pfPG6zvr!UDm?d<GLmOWD~`{%;<%Dpo;iYS?=tFV|Z&C|WK
zJFo4j=we%eZmw!0k6ZOg)rp$btIs@~d8^rCA`72~$8JUCZwrnsUV56{Eb8snwVr2A
zpPSi|Xu3_?YxDoDw<33O#ab2K*)&5qGU&y(S5r9IQyx86JYTy-`1cJXVTLJRbFW(a
zUE`DCJuNVItysdjpxI5S-!q<loF|-Odp!4E+}hojBc}zk81BEAw_h?*sQmGLi$a;Y
zjXvFVri^!TY8QUUXcT+BQK!O8%A@naXU|7d{N4pTYI0;yn#{X*DsPqVVbQtY4?KEl
zT3aM6S1Z?kY~gLETLI#XoIc{3yq8wrxs=8^UGcTPaqcvwT0u>x)XRcTKE08dFy*n&
z|Hn3OzWiTkzj;pTS@zX?e!mG&y7T?tk?ZHa#_bpNzWw$vQ}OAK`9cSdhVE-tuX{aZ
zF++r&`vN-wr75po6gsWl`89Zp)%7Dk&poN$SJxk!yYAZCKi`XT-ly9MrS4Ai)4%SN
zSO4&8`JY~A&pw&9d#?Mg?w>s0xqPqe<Y!YJf85j0R}wVwog-tCVt-YmcI5p%lCuR{
zmOhu+e(valy7^Q7$~0PCTwdoM|1qh=UU0hhXD5zBi&wXQT$WL=^7TIco8pC!4lw?4
zTfA(^0nJaU&sSAhIn_9D9Th*qC%dcqNvlZ19;3Ir51lnz^0oZQPciwY_xyg{nB;c<
zVm_w;!z8_T;)3_irkveYJZnjPlkiy$hJ&sL92%K<xn?9y=yGgiWMokM$-*kaz{H@#
zz`(%Jz~sQl!114fb&8<Fh6N1atb$T86$TFvg{3M@(+u3eBgrT(nRVyH#Rn@4S&hqL
zB2LGx4dZt2OjJ*}tHrp6WlxRggk@(YS`-V(EDvTlKUb=rOCUTT#BrhRWUZqs*R8m`
zjOD1LtW{`MNOS0NDG_t7*G}`IrW^5C3$2V;YF3*i8JfLNw|cFIph#tzrt}P^cgIpr
zc<K4<=l=dq)uKZ2?g2LL>rbo2Hy#X1wJ4maGgCo)j*Lm#J&(4_M~#-8nj*w7b27sL
z(O_X2j}5+A*SHh8Gm1ZcN^RTfzxc|HsSj2hV|n^bY{SY2)8;Ibe5ZCJjA3`+tfWm|
z-ZP$knW4W?Kj!OYwX=b%JNW+Z_qw1FxhdInk@)JY6%31AZktbCFSv@~x69sV<^KC?
zE5CQMY9us7m8Qxyyq7&H?N{CF9P2piz?EsbYi}qu=oe1hG>2jF1fwpGl#;Kr9qV!}
z9&~Cfn;|S2U2ySH%C^X`fS&D!B{FVpoJ$t_EuOhZ&}*@->O?owNr@^Qnqu1pvrb)A
zcCeOSb%am*wLsbAu5`|&Ps2GrxrhYDiLoqeN=wX4Yx?~(J;70FUwYi^eT5p&HQ0?<
zt~q6DMJ#o+`TA0Lc75I!_37zKW^N^KCB9tCY!_V>;&Z@IWMxFohDG6y2Q;?@O8ZPY
z5xBhA(`_b$;i(l$8FM?sCd8bas~YXN?&yCvwrr28SJz!<&f9ijiOi(HJH3{S%a$;R
z>Z!|WUgX`yRd%{*yN>_rHAx~Bte(-AL#~}N(e-_ud3Eh#nYGvF1^(kSShv$}?(>kU
z2isQP+bt`);!1_-*Cnxui`RsDFJilT`IN>4u_Y^KGUtbE*!^J9u|?fFlcL;u*?bdc
z9$7Z2F#7P4up_M=>(6-vOyD(qa?E4nxoJ`<JO|y_IC&a(O_`+7xz$tqo51auML+ed
zUnE5{JvDBMwlw9NYbR*CCD8f(mD!;ou3Jk&_uj31UT^<ie9PR1e{bH-_i%r^^>U!E
z_NkEM-H~!tC4SkVvmX~uExi=8e)fa<wA{bN=ZpT?yeQy5_5VuweF3|ha~RCcwXPkg
zEo{^DnWj{>GOkzqYUZPZy-$qRb5E7g%!=vY+UPhxdfQE>#nL9fQd1^;ip*$;z4ppO
zSvth%Bvb3Xiw-56$F)}+)(cB?JiF1^ODyEnN#Tm@yxIq!IF0;G4_(OLBKIZd{LD*V
z3zYuk?m7G1r1qZqMVHf7TR05tbl+S#<Y)V3!={s-ERh-wlUGfClz3XMP$PVun%jH(
zY0}w&-8#mSZ8y#-v<V;R)jQ;%$Fn4PnH2Adf=l~*Pq39Le!MZW%Dipemp*0VVxj#3
zLESZh-i{6*FMs^^M14U|rbVpqQH>p+)Ff|B@Ke%?zI7*L`OcpKl{P#|eE;(+ocRO`
zJ3Q7T3LT%eO-V?%BRM%vWZ{p)tu@|Vc7OOI=gl}Q`RhVfqf??^$c_n)Yd6F(HFH{L
zIP#lq+9>vAg{y5<V1M10#o}dW7TdotZ2R(2yY5>8v!<E)f(A*JVo%3A34fYSm_&J6
zE}0>l#3MP!^J{A`f2Ys$gpkF|%QQmgFz{xSEVc44Vrdd!iS^qy!Ro`;C5H01TGsa@
z?`-0EUEY}JD&jWLrb0E<;=mcHA61Jv1w|)2&zcctG*Po5MNio3S>-chVHV%lM_<mm
z*1XW*da!MkWbn)fE@>94cP@KgyME?FlM`A>Dn3=cNAkj^NZZV3-;n0}yyj)JyzQ@z
z|Ib(bSlsuI<%NCWrT%qq-0HS|aQ*wwR9#J0M@_>ptby;_iT_JA8VoKeIBOhZ?aRv1
zlll50#w^VGX2+2!8zecL0)7hgrKl(|ZH&-nuUz!zDZje$X7*OLaw**tPn3<Gqz2S}
z7UUD1>X)S9A2e~*m1TXpE8ly*3KmvrOEnW-^=6Njqe)=sS~Fp%h`C(htQSHz1zm0T
zT&oqrJL#&{tj!A9rW|2UcZKdwO<J22`#R=f*P?YgQ)l@|_g(1`J7W-*DSIh0S^BYr
zd&8-VeHCYt?O6<NHid*Sl+4kQl{D!J>j_}U@AOtI4si^7vs6^%?TN=Smj$OgUh)x`
z*z-Ijb5-mA8|#d=>CQbKAnJAEPFC`I<H@cjlV+cJ*~PmzbBXral?QfBaH?CoW2WA5
zg~K^=!d5r-<{56+?s{8i>h$34x{95zeAad-3NLcVSK}y(V_Z|dH>Ebe`u(rC@0-N;
zJ!{pjIiD@E;*$B^&%7IIuHL@)W7qZrpMCYK?=QZ`d|iA$Z`*sPZ{8Dr&ANIpt}s2C
z+xpB?y%pN4E#!aRaO|(0wL_kX_l14<8-K<h&6}pR8anEECrez9P(Gi?>d$$ptNv(9
z<KCU|?QSU-m8`iIGpmG7H1hCnV(31(WYUd7jcYONRsjkJb*DN`TP%?DHbZ5xrn7X&
zw2e`TDvhRl<@|0>SYa=8?EmuiC~c1`USXf&MEAt9zAyj2hEw-TBtzY|l?xbsUpPwd
zQfOD}@Ruxl;U@m;Vqag$LSw}YFXmg9q7IlW`F|(FS6k}h1hp^A?cY96V~aY!V%h7a
z@Z4+r*By#n?Vyt+`uo^1uW#2XWjZHIY|8GlSb3d=@ta$fqea8c$~uG0)ZH0XYQlF_
zgU?*!P|B3rkTtu^tmx#o*qFmB&88d*pZ#XZ;u34|#wl*$%RcsKGJiQBD!3%{UC!FX
zuFsBhDhfhBdHZNDdXT;~ODT<M#^%Pyv#<TVXS&bMH*4AX*#%yQ%`Y=`xGdYMQ0V3E
zbWw=C<d~-3i|+Ks$71h)9MNC*v5)(~W0n8UEl&By74;_1c`B1%c_uyXqu1@4$1PzN
zEpmMEzA`lrXMU(W`1sVaq%Nl@<CV)hE*`h5TXf^)-yh3m*<U@14d2%HWk#X@kCmmp
zDRVc)yBE6ac~1~O&QkaHmw1uwoGDL3KIp78-4qeA#h~M=#!};5f$p<;X75d3vr{oK
z)w6n%mwxnx4{IZRXGXfMIup@z=l^Q+!W(;nn>RRf2O5;FaMyixPXE@W#nU#t;8seU
zq<`eg3Uv=L54l#46h-0H^?zU8wXVJ~x#pYt{=M%q>kDrm-}imv`M<A~=dnKg9ruH!
z@b5|gchxtH?Z2Iy{`d28|LXSL<rlrSCbB2XW{0hrc<t|*H~(cnOWcgnPuYC%TE~XV
z>qU}`kNjmya+j}+;*Z|4IB=aT=TjNu8UNIe74<4gNIk5t2^3i5s`K-Ey;P&r#!Fmt
z1@pEo4U}Q5TxKk^VzJdi5vgT?UUMI694)mz_)to5ljW2pX-OYbb~4B2F0@;#z$&o4
z$iSnJ=|H}?d$X}d^UUyOLGxzo8I^V?n(aLVW~#Ta8n)PYv^Yi7L<+EaooMl>Xfckc
z39KmcvuO3!X!WaT4O!6=aG@pUN2~hv)@X~isE9V76D>&*ZES`KcU{%jf6MyrlGO9G
zgrPM_$1FhWT9T<*WXyJfY7c?<?Sd)d$vlP~sp1OLzb2_ZiTubMmHR)?{<LdKp;BkG
zTKTjN;SzNR0cTd<Z2||1WEQLt3=!*MQB374PHjD&tTwq~>Wwa|OZEmwSw#*C^n_b-
zR*1=O;Y?Pj<O!5{n@}%(h-(IWAx~l9u^Y`t(tE;RR9WumS-ZU|P@s3BK<@>K-fI=T
zH)izSvS_}Q(R=4b^Ti*%_iyw*+R=NjqVMjEzUv&VOTJhARpYvH)cY=bUD20_;t=;8
zX?3g@v;XYq-?gn&DzRgqn%L|19GS+X<ORtGn4<~`yFDjruUes9y)b6?CJWK25-T=Z
zvl!;CeId>w(WG{n&G2LHY=*{vhZ+6V80JlET(rbWLQ$*LD8@1<&*bH#?CIj~{%;fb
za;?*Q(WHiQsh?>ATNPLzMu-RPs96;*;GWqsb2@9lhi2cOQyg~G*mbakO0xQAPL0%L
zh~7EXxPv7|k~JZ7YO?0E#FbOiZca_}oW^uuT3qI|_(*{S%NB2rqR^*p?(7E2%j|rw
z`v)yg3f!KpuhzkBIAc#(S(E0BmJ@X?H%pDeG(yq^+GYyNG6nJrDisIHENRf+YSQrQ
z`ApV|iPJ5FxQ!j^)EShv1_}oYh%_6f2n2CErb;w+g+H6D*cc?WFo<vB?Byy9TRYMi
zC)v7gZIF7{{cU@4)C3kg)0WE?Ep8_STqWnS39<?vXufLMd|i{_s$}oYn{!zw&dv1{
z5d6<Djm2@^<C9Y#M$UWTDG(qqFMxqD;pe=MkpiDiPW`~i`c;$RLuJ(&$?Do;p^FsQ
zjGhQ>{^s^o!M#GIuHp&*&Sm|7PYAv!7yHmIENmsjq^ZC(bK$p3k$GW4JxmfGmkH~g
zRIuet<vhYAv{Yt+rw)^c!~&u6jh}LLnI>=fT5jdo$l5q*%NACvRVjN^rPe5F$yTWG
zPZn7`#a>~Tp!vxd&6y(Sk0xuc$PoN6$BJ>PD@V&)%jUP1(;l8&8aHcREZ4FNlb6a~
zZ2faAzdW_1Kq5(Tng8Ak{$(@T%U>k1zic;4TRv%};AGF`-3u4CN=#^2B`nT0LB^3^
zc2n}_#buK@G?e};rJPx+RH!~{v8DK$C7sfWvls1HEM=H#-??&4N-Dod%(6?ViDE`^
zY|`sC%B(XKa1;_$Rp@37?A{Tace6~5;kxkAKt=YfinSY9!#pz{NVG^jocm~}z=M<X
zj_(q9!nrJF)x2}N<_QD{{9ZZl>@I--0Y?9yQ+zn8r?4%!y<I1Gl3j0V|IdnS=Ew=%
znG;T^<?zoE-fgy!wKCZ&aKe{Wb>FIHB#W<j|6MWnSwCNgy~ZZR2EmnVo28qlOsou)
zm|$c%{c_Z8Hi`Rc(X2~S-!gT{90;7U(aPORi+@L6ci<8$N49l}vKC5<<P5zhsaM7-
ziXSlI5_#mYbphwaj@miLr*QvIYzf-1CM2sRW!IY1S!xlI)6#deJTIMol*NEyaq=Ut
z_HvEojHv=i?vY{3MXF!Tc$+k%&T3_v(TX((#MFX}Wfv*#Jvfs`aM3ys$N8RfDhyW|
zeN~cMEv(6@v+Rc80<Kk-8x?tfD6?r;vrUoQ{3BH?Si@|o$g!Jz)@E#;2`w|+TdsS~
zWpY}Uk|mJ5bN)}swO=~t-ty|btF?gXR4uF3ItEvFo?Qzbo9_xVPYz!mS-M0irF|V!
z(u`1P1<_SoH6C$)nr@fZD43S5FZ#2r>EVQhk~-dw8+@KMsCq7zN-ejv5)%2*$+d-X
zS%r*Kl>MS9d6Ny7XeoKt|JGbrD1M;Z*Zluhuc^u2y&1+W&H}RymYS@d;&F1HpXQp^
zo%3v8&nsFf_%?Id;Tf!MuiHGbqiQOX{H&L6)L!w6Yip9>ipc5?p2n?Wzh+csZG9<q
z@aM1PEItP<4Ojerm3n%j#+nIM_Rj>Ju5JBgyk|?b+R}iPdYd|#4s;40p0#o1%IRIR
zjP?l5|7Gx2P34&AM5#t2mW_LlnC?5UQ{;Qurfr7=_cxcZ7_vRw(PCyj_t>ww_c?cl
zubQ&%`r7w9Y9D;u`L&|oKYZP@<LyS@vj1`91X{1Ry|}Ku^Vq+L8H}D~`p>u<Z;I{v
zkp0fw{f+r<*@wG+>=LMYvySo8f)CfX{@x^$AIN?G^lq+>$xZ*aOuV#h5!+==k;p)m
z)e^NE#p^DzZDeiCpRFWjoqAz)ilnrccBi$_?vq{@*_PGFwC~aCJj1q0rSe>~<uXH^
z=ntDz47oy13B=r(;{KxL=k9|4%a(mD@%Q3(_nDo<bY^*f<gsqe^=xluoVz2uvHM^}
z)xk?X0z0I0tganr6YNmEBb*q1K=2KJ?CJ?M7bCm5^yDoK@~;cO?NO3QEuYaPtn;SR
z=4R*TGaE~}9hD}os@|>1eKB>dK^oJ?RqL|^w!WyiIjt*w<L(oVI@>m^W?OhfFuf({
z#M<S@b~+yQzW2O#>$am4TKk_~Kl<VNuI!u%YHtq+t+~X~c>Kzl<1I4|XgK8@|9|pQ
zXuHrhgBcdFW&0ciOg~>{H9Rm~<FbIw!Jf#RH5b;~<#H=WiMKgh{=I%h^5T`%Mpu6v
z7ngf^V)YuCStij!9gAfRt(IP1VzrBPzK-(c-;%1)lFQbJD0FuoylAg<NPEf*d*?gC
zMIW_o!~z3PDb8%0UNB?Ji^X=wdd@JeNjk-Ito}#4fX<crRX57HZi?NRkyE2|C1&f}
z2es9^ZvI;my!ayD<VfST*>-CLrku{Xd@<&VS?|J@s$1<9WoktWw5=uX?cTOJ=(<&p
z>XFy>j9Uzh4|NJPuRO5#kY0}RH=XUqF?Y74Y+rxKLGp0!N<~lRjjNXL5}05pxc%Yb
zr(#E#A6=0CZz$uwXTll*+3zzVOJ5wV@yRLWxqS7{nUtC<+HZGd^+a;SoMrMnzW18j
zk1Th=KbIyy<^Ol!3R8eFD`UNFZvDs0d`lN723+D}kK?k?y{LC=!L+Hjn0FrkyXe-N
zy$`3%efa$Qy442td4=_2NAFtvoX~wRXN|%njgK38ABwAGF1BU7R&YBexL~8zDNl<_
zk}D5+O17%DrduqY5~DhM@{CQBtuCr<xs%-C9`Z(D=G(oO+<b3d{&V)#cDJ)ix;I#p
zyfxe(%@OGd?c8WRBj6I(|A6|X6P|s$aMR~5*VhmH-x!`3Jd_E@d)itQX?Xex+eD33
zmlY)pb?%)#rM>HR=Itl#kFV(eH#)q0rSMmu!?Rc232{peOEWa%NtIY~`288dr5jeQ
zaZKwn7vI{YdQU}cSCw(gLe4D#k@|P_ZQpWRJ-xi{9XE5&!8<w%6*>xACU8dVe0}z3
z)#?EL#R*gW@0BL#>}u#(aNw}_!!<{e0{Crf`8$*NMcy4-pep|D!sR87#ti>dmOkM7
zv^e;m6W`~<_sibt{WoD-azU)+&ID7=a$`${J(mt1EEHtVxV?O7eMoN1I>EeC+Ezk8
z-Y3>7+urI-v+dHV+<USxNHW+#E9nu(6REm|)@5f`?0uwp_~MdpF_wD|Ke^~?p>vR{
z_Q-+rGRe&^xVIJvZJQ7<b?w#(yvHWr6?`*+)#LvJ)<q7g|90E`YdLMem~&v?)0`0D
zKn1QZ1~Yc>Omp^+^jpc9bbZ3IzxVBA#s4Y1<@mwP`7y>SbqCkZXZmV4*V=xHyMI7`
z{VlP19s4%U>^hYEyI5*7@4oNN;*%#WT60y9eez3=;BphA?M6C=z4PuYy&-(!?)PK0
zGSbdLn`cQIHVNrG?NSY1wdUh?ox?^7Oon{()0Co4G%^~rRlF82QnX6tE_=W=U;fvE
zeD3@AF25^ZxN6;{>7nb{-t1aw|5PTBdu9W}%mD7d>HTXy%v*fm{?`RZ&fl05bwKD%
z!_?IVkq7!%I0RUa>N9+KFy*Ly<o*Q@-YJyMNIk%%_f^08t^PH>MgJd0=Gocv{5rb2
z_R^a357pm3e8zvjw9!5GbO&3Y@y~q{=X5niPo8j;Ia&DLBa`LohE3uO7ne<8U`(+n
zbUfkO;MB?@Yu6$XsB}b^Nh?gH<D#NCAItKRm<|D@<-Lm1el9a7B}h)?KPjMcwe7)*
zX+oK9GGBuh9qXJelb3nbt091KE5|0Ot*7QU)CtXY`;vY>TswO08;%W^pS`-ea9h^T
zrXL{(f~I^?xz-%@dEt%Dl$lO%j;<54jauZEndIs(BBcA~j#J8X){`1rswSogx^~B{
z4UC$=5uu(CGDYL-th7aQRK3>LZC&7!=z49QZqT}4e`_<=i)luGoi-^?cIuXfvTy(2
z{B?dkVQtjCz1L^et`^$yzVpqo39DSEB)krr9l@F4JUL|bY#~p<`noAU0@`l|YJLA_
zuqWRrddCd?%s?g4dvA>M=07s9b8MA3T~PTmNO7hnkE&L_=gy!AtqvAOo+%vmGdM0~
z?!I?SfzNWwEyWWWOPahnbr{_?ALR?bQ}C3p>5Rg}1?F*uCq%7tHnJ|-`7EhbHtbqt
z>XbkiLFV;$LtZ^M>AbUQsoY|PRqUFE1v0Ior>kUCn>4m`sBA8*l4Nc2m@=nZtnA1t
zRbdvBhdEQvFSA+e&d4I{e8NZNbGzKx2tn1Zz+aK<O^bfLY*nZ}k#SUh(-EcBbHi>i
zzMfxpB18SZR#Vi>b8_~wtJM9ksGTxa+E%o<WbKr)BNrU~IHS4@T@)8>5H(s?d|@Hm
zO%0AkA32h)s)ouYZ50iZy22US;c+r_QFx?dsK!hd4c7%md_tOgTOEWNB0KrAJe7Oq
z)N1V0HJkC?hH-+a#-bE)uhK=oYn;FCWK-Q8(D3?%#M507=}uc?XVuK=P;Xzi_`Al!
zj#W>0a;v`$Vc4*0)hRx67LCKbExR=qa~liUMYuBF;@}PPH97srSY?p~57*f(GcGG`
zpQiM4;n5o_(=D2AM7jza8eLSH9C)KJL)Ory&xQNc1Oe6EffE`Sx<Ui)ZSqjDyKMP*
z-<@08nZ9<bk8f>v`Kj~YBkac{K`vdf>Y};xk58y_u3pD&t#tcD-CC3DdBP0acwcXH
zb=!YPs{37MS+tkV-qOdrHTS>eyAiaH^VV<0D-kzUG~RJIWeb~mD5l1hE;io2Yo?me
ztE=x=F2sB;u&>W>_<Vu+yKU1Q;RL%5zbX&qtMUF@mGs^vd)gPAf1I>0k+pzTX=YV@
zMmYoL6DG%Y0h^bqv5xak6y5ve>Sx-qBKoM2FYC5+XEEC+cYB-p&72$B*<UT;ZWAtZ
zT@d!><#D5`cUGVD7=J{$8y>aD{_k3)6<ay=mwRQTSi-h0#f{F~a>X;%SSED7`l)OX
z(BLy6Ya`b|jt04ZD};m&TwA=%V#5FAz!NTg#|*g6?Qs<eTjXkctY8K=+j9A&CZE}S
z7Ht-ti^RW5v|dwUY_v38Dz|TgySCRwrGQj_f9XwA^`{BBY+_yMG-vW`2BoXYJ|CY2
z9eEi4V9APUaW<|&Ou{qlr&t_&Gj*eDy2??p2*V|dwyX_3(|k_)DTj9FhF38KB^_&(
zKJBR9*u}eMf~RzVYn!i1i^`hA$$PI93r?7DPWhbIiETn5Le-8f4tqAn`o<}pn7-$A
zEMK9V%R7nHHCz3Z1SZ|7{`OUX`|8`^tt+_RZ+gULtihA0xvNz($8&<7z-r#L45x!%
zT(qm_=ro%7BG9lql3OO#F+J)6i>}0X<5Lcw-TMA_G6nM}H-rjImJ7_DEFt$;{Q8bG
z(VHqfx^l){NqK<`TYmJY9f&+PDYn;la%0mZ1tZCGrm3>>E>6t6I}BAcJWY;hu?B7u
z-(JV7sgdY>TI%LSDG$xfIW68PZkL%>EoWGxnXuB;`d6y9o8c<n#S+aTHx8_C|Ij7k
zv2$_xI|;$_OIIy3xw>MN+jjw%RnJTK9Ao_hgHA3yrSAANl4G60B<pUCCzE#XP6<(D
zvPlxmSSGkL%a8Ms@afdA_QJQ)`B(3l6V~veT3c{Fr_3ksXpdBruM<wnM;jj1dDSM@
zck@<oSBvCKMgilAJ5Fi{6iDpc!9Uw%v6M(!+KzK?1blK{w*OZX5a`@lpp$ZK^MZse
z3R`72GIKG$^-)upz?ms@u0bwa(b!&_VgG|nBk9?<#lwXchiym>*gADlV9z9VhleMt
zrv8$CY&Tu{#Jr;gOh1$&{;;?hX%#v<<h+)=oyH#?vVG0kAE}aTCJIJV8hKP7FjZIz
z#Px9`FkNo+h+erkbj`Iur8w7<zKyNHuMJ)(^!adZeX)4WBD2$R*MnlW>TGKKtHlyI
z>5hJ_g~INhK5t#aolDD=mJ83iqY`G>cvAhTYJ=|qbEO}%?mhZ-)Y4+7>W)=2V-JPA
znP6mdMse$pqx`HN_^bacl3n}psH@?XexDFs^A9~u8ckb#7##vUzlpROasDsz%j0^P
zBzL+cR^)Kh@eZfs_NSRnEn;BwebaW_ZeLvB0)|$-Vm;%3i(QlX4xhVoMae6&AStjw
zrR$rk%5kxy#g?|Jn+{qoJuh)L<F3Qe=Z^L+&bh*w+HO{|ZqhA0KN?Fa<^H~C-Eid6
zcAbN|tO~8`HcWB!oRnO0OIwmZTV09amQ2}=Xt_H}pQte2JL#YDR?sHyh|=V^*qUF)
z@~hmQOk!6);lOf*jVbk3#bP;?YSrM$PYY*EO~`%8w)%$aEtzfwCdn?zT?tKbLN2bl
zr|wO%@HlGk9XKbl?iA<!o65U-l#+{H9kt(?sn&EMsog1$>$J&2Rz?@cY}bi8nU)f(
z{x`65t`}c>d}sSl0iT_n`r<m1G)x}9*Kl#-i8cy0%-q-|Qno?EX_0x+{X+*Q&R(gx
z=dDQLf19waQ-XIUiQM2{y2(YsuH3}D|6TjK>@{zzzS}+0m2*E~eUFc0!3+MAC6{&w
zG{u?OD&6>{nyavMU2x+&%epCx!|i|WoONu~%{ziA3f_qmS~x;)UW!`cI^$>v!~Z33
zIO68|i`V4di@6lNG<loPovA{nOZ&nsIaBKu9|^D8v!k~l>waI(Ve7=(heWp*Z2WQW
z0oyu}xPlx;zQ_-r-OKJw)i=5Mee#D*eJc(IZ+xrL@%$*m$E0t`hP{o4jz%&%R=#?+
zwdLdS%Vy0tXa4v<TU7RGamrSQ^hc*OJY|+IKQYO^^dZx_0_%65KQqsrG;txzk_{my
z+Fnw^)4inZqi2*XiB!Mpt$bI})4u*!?cP>S6+NY<Pn=J<gnrD-sSTX-_^j{8`9go&
zyRSZLl<l?PTx4WDRY5Bvxck=%lO6{x1|Qj7)f^#{JMKzOGh)yWOcIFvY4vT9ILE|}
zAS+WY!71X(tQ%Hpmw#$MCm^(ABYW;=hL0a*U4<>1EDkmZPS%KMU-m+XrD4IGhjS7S
zO}@N<{j`#K!(*dAj@?EJg*0{u$=ou^OOouoJd68PyWxi!iv;^?Z%>l{X~B1BN@286
z`JolkTzZErcTPLeE5);A=>>uRGo-nn2`!#^dqTWK@5>^a<HgOdFRinEqi3YhxRBZ9
zqfmoSamUifOBcU3+jCm;A46BihF0gmZILf|-+z$tGBFLm>2P0i!tX4O{}EdG7xgn%
za`q{#>Ny}bU&<ghxZgv_by9!<Yj;1B3LkrRXN+Liiyu}eBw2qS?o3l&5|pjUcbW5$
z@}fm2_`)WrPrW_ysWI#BVhPd35?4NK@~hNsp48LZsdi0!cGZXNdaD<SzUr>p(EYes
zc+V&9%_o|A4oa{!SzKE+amB_dWpg;TXw3HCxah^_%?8YK-Pd$YykzJ0Smh(Py;pXJ
zxA6(b5c>`Vm#GmRpS?MEPwwEI(6m5+%S>tO|JNKgl`}PYK2QJgX=+IKk!c#u7ZbIj
zs`*)KIIcWgxn=V-_7>URCX>k9rqgRUrU`I$#HeyS=}THUH+i@9j2p7wRvDx#_Z?~6
z=D3peam0>E-5qxrl+0f+@qb|2s<q2VqdA^ylGW+`hOZT`ISDIFQPPzZI)8d~`wF28
z3)&gFSQv#C1TuIlPMjFpF>y@|*ZzZRcKwm|S?I2mIPcC0=?(_&cnx6-nZ*`sEVp;*
z@fF&H-`JV-a_9S5vU)EVPA)lfa*of3pj~_~n9hb+Efa2VDd|`we0J&={>9EJ*DS3r
zt8|<UaoO{^zio%lH|O>38QOt|7yp?tb?xO{7PC0cK2(nSdE)=it^>&jWh2jMZh6d^
zUO88H1{0I^x$VlnO*3})J2({1blA1nb!x>HBL~sgmma^E<_1maDHZHb65&^TymxnH
zN16(Uz5>VAQ-^OzSjtIuACTZV7uCuv#1I(4y1QbMT-O>62H|_FyBA(OSen4v=REVN
zboXTsJ7H~e2j)4`9BgY<OdKEcPB>*ZiOD1G<vI^$`;UudTV~1%*`8>132dqH=*;kF
z<MC{oz+_jc)Se_hKX7`<%d`JK@N(YSFL%{`|7y-jYtGI8!n>$Qb}`rV_sJ9du9z;9
z3W`tOm_3>E%_c8yoq6oSOT<|Zp7!Jn$z<kQ$^O*il;Mr}JfaQZFIZN}O#I(jxLQPo
zulljk<m9!65tFyZO#U@dD(XP++AXK0jSkDbT(tB<@G(ovX^-bS&fVv`a-U<h<)JCg
zM_hcYOIU4mR?mC9?)Qu{za{h+FzcW5ak-Eac+bP-af#`_t|Rl9)^jp&aV1)m&Xmy$
zHGOo$DsIZu*uc(kL9v#HN75=?^A$tySzPMMIQhL{>$5MR%-$1rKJHL2o$$t|)0gR7
zXT-d;%TuQtosPc~)U`u=riJ*m2jcG~0yslC!xK2SRxvZ4I4ogp@>L|fDnl}xMf9}7
ztSNz;Dp(~ld1e^t9CkW<VSh@vU3AD)hVE9cEl(Z<OnxNV*s`Ti^F(yh2@%zffTz5j
zb36XGws>gt-VhJ%e3rtS+~aUOaC+<<POfCLGZ!y<p9%c-$R*UoGgs7LOLk;y!cF#^
zn;&#K!xx`aT&gu~#-&b0G4Dy7?|(U`8BXYqoiL$9?!fBx9XDFc8qC@a`mUXF?_p93
zf55WRq%q1%quSw0*j?`_66QUhW^Z_UdYOpxqZ!*&eAdaoYCj;dczfzPBdaq{4_J!L
zymIV`<@|1+(@W0ysp?<Mv0CPAf3Va_bm^HX31?ZNeUIK<AGkNt)yC@6uA6N(ou_>J
zul(_O<Z<Ur?ljE_fnigXqIXQaJx6)!g3u`+mfUj?4U-fLIv~q<rz8HaO!VeylOHg=
zIwJN=r6V|oV^YtO|5ZEhvhI$3#(C?Ktf}6OxqDyl%4p@|-W~R9_SV4W&ZH}A7kcni
zZ8^Sj*AjCHjxY|j4Xp87ABgyI-l*r@+`uYfz1whAfQf6eiQC27p4=BdBu8x76z}VO
zLd13dv=2U_p&q@g57JE6>lU!Sz09}e1ZU6&-;$g56M2tutm*v6?D?Ld|9s-snI?00
zoKE=I!!cv&O@6j(odFzEL;7d_<-7m0&-=4ylY!g>4>Mg=4*owIe!VgfWSOwy?V>la
z-R?_vwFqh22wuN;;o-C&N_Bg$ThttWV|ZqEv)-J;mf!v^-Lp63#FLPP!bkqy<vZ=`
zkSe%JZ=TB0sdx4LL@&hL>F_vf%;)=mibA6%0~f>2jX$mLPGC7UO~8-Cjw^1a#C2KO
z2^#v$DnY@e9Xn4aWGil1zPUqlMuW3o%F8AE69ru7N!>f=dbA+3*?{d7TiMm`M&3Us
zUpu9^BqBMAZ6(Lc2Z{Z)e0u_Ry4)3zQl26IXwA<r7Ijyr>|OYjqqXD9+RnzK``UgT
ze``IpF(ksP_(ozHXGiJ9jtMtX=H2*{`mARv$KOqUhGCqGpN4)*;^dog;#(;P*F(_&
zYw^mDRx=vzalQ-uSTOadWxB<#2a|Y$R9L60*Pav)bqF<i*lsby;Zx*nmrh2>BR-Sv
z`LI0*ne=ehhfZnXmF)(6Pq=n&y)m({_(kQB345=c{=ahaEbH!_GP{kp7wK)kCGfUT
z`DN6Z*QXBedFy<O(`vtKW=QO=1H3V9I}Xh9xHIFO<{8mMy=UuB)N;)6d)(S_XI|Q!
z<Fy@UdCp$%vDP=+wLtH&ZeK^7Rv_mCmJn_sw~MPn8#}HnU3@P=^gheE#LQMBjU~Dl
zII=X)UP_U><?H%P#UMv<LzhR&J=F~^de##s9DTUd?^)rsX%6~wb2}{Do|pW5lH=N$
z_mwMb*PQi7pZYG8u0E3bWpBWtcgK1Ex+i2d|C}4&y7zwbp7@lsv{P4COfNgZ!1ne+
z7^mg~rY9TvR05-(?VWz%-^F_j4rP5}_U|6N5ZzFAFX5lll3Teq84j=e|6BOv!VOx?
zhH{TTK3{!q#mq-qQ%+o~*}2icHJ!uJQYmlWHU?!g9%c7jJ$1I1o>zC;+&S#6%KG8b
zmA$;{^=^bLe0Jx2*|Yg!CuWxA8=GbEPJ2I9<n1rNso&c=&OX&-J~++D!F0#L$Ek;v
zSFg+BxwCk2$J93r6B14?-l{dN^+%RlpEcWSmyac`k<}?X4|h226LYnDa@RGbLt#T-
z!@1ssX&!wkyVXMPoOv>PRo}uVGCejQ{FS|YA6*Nyn;LXaM*5=RE4`VMUiLQLd}{jY
zsoP_g1h-GEs?Su-pQ-(i|GjOA#@?LH1<#V(lry$H(V6zlq3)U9@}iq5KDT`Oek3<P
z*(US<lgJ0}n&}@m9O?VxCx6u5yF}n$LD21aDYJSyUiG90Of!w%Z2Ie;X42mW4BV%<
zdM+*aqT14UCd%fPC7-xNe`3Vcuyo@LGovm+DdljnGdu3SINY+X|KE$<vYBtHd=~cl
zEE2Zb_w~)<vLc;(nO9TZw@%4C-*UEXLg~E)FOQxr?Y!`QiJAR1mC(6*$tRcaC$Ogk
zyZW{<e4FPMdTr-{%Q0VXZ_NC5uk@Ef?$L=>&1vGPub1@9y88Nxo)c5{%Q^kLKaQ~c
zp0+8>+RQ9$t60PKvJRKHCs#XPcUpYxU(n>?_T?VO<1Pi}nLDhqrzT9<daYcri?ez|
zl&k0~Q|}FqwpS!1{(tlByL!X!hwZ2LlkfNFU!1COLTK%$uh-(ma?}2oewwl%Ld-Q(
zBmC#o>l)J(407W6^449v;#d5=X{yhZVvFbd&kIhwUKgbNv^e-yZuzs~Cp8X<wq+d)
zo`>B0?l|SnrBpHZ^DFp{bk17PC3r^JDT{gc*~1gdeC9Ww`8dxf#Qp1{*CC6Y^OrnZ
z+V%9)7qz}0YLA1a{tgrgowmLD+pVuz>2v!eJUM6hwg;^DQQ=#0=<zK_iI9!E8#|O&
zxpiip&0<w=^{$HDZnl&={7Aei=Yss*e9I%Z)qSsE|J$zc@xJf1pB{z1TaR94Te_I{
z=8Rt(UQbv#hvCgJ_f=e%KTf;E^x<;t|M$#kTcj=~9gCkJ{&RxKtx4Z|AKmYpZ2t4%
z|Hn`N%Ve<4)d}wwj(<Pvg1%7zlVq;)!WCR=<5z#@TFN$&^JqKAr}Lq^PM@4}NH*<h
z@6z)ci|?Cs_VDkRR@QmrbB9EIr$k5hR+G65970SM92yY;3XY6RxF#9C_@H>CLrA_)
zD4<Zut()oA&8+a(iPD-(JEjO+IQZo7Bz12q8_S8OPtIVAOl#5jpy1UU!&1`daar}q
zTw{5rrz=0N@LZ@IEwr{XNG-S{AlU28%8-@LGbE~|N-9fNGR#VtDE8&orUelTGnZaF
z(q*Q6uuXW|mdvLYrrL%tIiXWxHE}Y-fjK7_*6o}SGV^~s?@f+`GdrF=-k0k&@n%wG
z^FiL|4ev_23%%VgurBAgF=@h=DZDGaxo2GXReJkuOj^I`jDiKZTCBH!^uCZxI(&+Y
zdy`g0VDgbiM`gw8WhS2a=)0G7!o+*ulMnW9ot8gy(v9>RA5@Q~{rPv{Pjt@<^-D4K
ze{KK#`1Jhp`uzWPbsa(5CS1C`YDz5YR7Kg+OND`VgBH9Ek;s1D`D{W#+t$Vz3}vi8
zE)+4P%y{TL+xH-ArhqL+V29w;h)2z;ZD|*p7vw6kX6kb(9A)tmVM$77eOb2HQNHh5
zvg10NkS>NjjUg9WQ>SbQc4p!znVckIHgkS_vX6SK>)IPZiXKW6Mece0->%vc8u;;U
zSej%eSHiuN$vYpOnw@WVM09HDsqeF0DsJsmpZ~e{=kv<0wVqt!(JbzFop_v%bk5^b
z-8Q-M{>z=y9fSGOFSPhhdNik4;O5cjg{v4XFEo4gatN|7y1Pt-U0tjtGIN99!>IBt
zQcrHVC~+>hl_OKr8to|aW=6bZuhR}!$t5v6WHe99USCtl_x0%ojmt(M0*!v5TdsF(
zjnT+0e>e4e+>%#EIn^h!PJP6d|MH&Vlj)Z`t@B+TO`0wjDsgd3{`~5DX0ofdef_pR
zU$B#N;tnP&)29o4|NWY<@slQ(qNZ?^g<yME?Tg1QdVN|BJ@U_=Z#=%lmqEGL?A8v$
zzW+iC**2Zv`keM;R*c%W+?_(4OI~jMdu`=Xh8tTyWoD(=bV?uObL3p=qPpa!*PNv@
z+=bawk4Oe`rifVIE>A6a5GHQVdvx-R<cU{=!ymgond+0~%OX10^5|Ky$lQsgQ*Wn!
zlnA+P{%o%B%<9*%v;E>{*KU94H&s%+v{zbX#^PGRXW@OU+~+(G2^o5*Buo-dS?eTh
z751>GXX`;P>Ddu%v(l>1ixs-)ZsIV`TAS1+={TeDnbi9Yo=dd*=Z5O#xEz+6@^go&
zujNAL92s}rNej%y0+&xs>%RDzBW%mj3pW)bFLY0vcF`><OKZ)CD?*XH%Xh8Y(Eon+
zx+N!FC7rx@12zAD5`S&{|L^br|Nox~SN9HG)gUnQcCDJ)j`LfcBKN-ZJDoILUDUUw
z>Ah!QX2O*PQ|7Kw5KfRhp7ltJk>jP_@x_vd&NNS*>#1FG>GRF65<-X1JErigxp8Ux
z22;jgYq_%ulH?s{T$^%Gmf!EQa^eOar3prBvVCI|I;8WytS>$3ukFUDFf~El!s@SE
zcLa;Vo4QoFHpvY=K2cnW8OaX%Y^OA>UmkImP&q!2*DdyBmhel7$qRQ)NaUKXcxT3@
zM*=q)U%NAN-poj8U466heVBw`r$>&-tT@Y*2vwtbwH!YrjV3gv&0Tfv`I7H8y*-B_
z*%+6)3Oo_NpmW6Q_Un7j+m5jQ=PbBX%QWFu`JE5apCfbQwihHhKN8Vz-gA;~PRMoL
zD8&O!CpzoCzB;>CZIXemn{dN7uUnB#pIo)pvR8h+xa9GnNcm*V2?cyFW0;F?EPPjz
z?yA=uIX`B}LEZ&x+?WDp+Nw)B{Zwi55P#e|VUmm81{vP?{@r30zfHV;Y;dxE`{SCL
zT7cnqPhNv>$%~{f9c9$&bMdou?38@tF3@B3F+QwV$+*{J!lV-`dG-l$-+DIX?52jZ
zsxdDa<_cI`j(2p`NL5NQ6K0>bRp&y{%Z-oRWu4~DJ9eZi$A#NdYx|UT$s<}1L%0|>
za~aNk;HeRGHaH=vYF`PfnsR}PP!und`w7qg-TK>ocs^TAnI*85OM98-D}{7+sa;Bw
z$`p3r%@(^U?Qw7OhT^rabNs#vJetUz$jN%{c|gX59@7o2A}1wy@6Hs|;Y<+TP;$ck
zghK1CoRgxAMf3IUmvqM*>Q;3xxgr~&)sQ5|p|UO__vfh(r~k6P<XeANt)N*u=HaA`
z>B5@pEMA9&ZmRff7jL^_i_Wp7(cc7{o{0pyxmm~_J*Ve3;fQE&l(Oe_AI6r7O$y6w
zQdLU#UESg5_0HF3``N&VlY99q&$u3Q;EKL-IJ$7kiaAquP7mYgoE(@lM{v4J@b;_u
zzV|a1HBB{{FgeY{_pA(m%VMkfdjHOM@4xu+u!YT?f3|@M|4&Rk(K)};t;fbxX|4uy
z!U?Y9tfh&Lse9vHo>kmAW>}cU`miO->*<8ZM*YyH<f+|JR}~|8BPZ??exuBrwAiER
zO>@iwCx!MuhF?x~_m*Fo5bqE^G04r^nR{!;)*ZhW{F*gM@BXh7(_CI(yD8Vf{V{pc
zggXKIwXUgj?ebZvr)#tC^TZE_CS2OAx7zZNm8O(U-!gtP)lI8*2THBa3-OmO=Q(#g
ziDi!U#;kJ>E<f6&$F*xoUB{+h#*=z$-CF9__P=@IDHVCS<CNCt2W!tr%YWm$m9~53
z%}LY3H2tobm7IKgf9Cz*?te}#B0>w|9XuJ{YKV37h&NsLx_pIo!I~pB_A&o2UzgH(
zILYAS+PGlvh(%GptGSBugD=kZZM-F_%XM{mzO>T5R;&ExxTP28Jrv(}>f6QnkEC_p
zv&jVcIZb%J@3?WEWk{IC<D-YB@EfFy9*FYSz1)7_;~WWl(@5QQAJvwYa<4jgrg*pV
zs<SJ0I<9f#eHT`&b><7-DL=(EDykcdw38zpvwS$FeU_46Tls9Gu-Ei25g+&Li#bwx
zDpI3eZ;|Iar<5~CI0ed@wyCMgoDeGz;>w+>q+&g#k*B@PS>(3z=2?HI)kSEE-8rr%
zw?=o@yE6WDToaXMwN)4%J@)m>#_8TKc0d2&R`;anj6u_nzL%lRMp<Xexep!u=oQ4k
zCBb|8{{!Wl(OxNOUXQH(rnt$PPWxD&yly?iv4ql20gwG#H?-F;Rb@<J-4daq(JDJf
zLnQo3oo5#F_we|tcdCWIFQ;(czTu_w{@~HI8pouzIy=~$^OE<QT{daoSN+;2O;cxd
zCtTckS0Ve(p6IB!>`fLuhUcdGU5RjL%G~e#;KoM_DN*G~k6Dr)-dr$Q_@@!qtGqQ5
zLN;&DtuweZ`DMGKfY`aS9V{X(LNybz6s{dqiL2Stcuj{VqrfS3FZ*TDmd|s7xL*o>
zNto_p!mS>9L~R4(g@&-!3qq1R7_T(k-6e9HyYBA3j0iV|n0-9qCSM=SGr3ix;<JSJ
zOaW&rL#6y1pOcbn|3^GtB2g50_2Y)vvn4l5Gy9%C;8>k2{CdN&xX$Z6BG=Z2#yy?L
zA$F*J$B!AYF(S?^7o8nEokZBA_Xzw~iC%8&qP^v?TMf774mal|o$r^<e!uqoEE!+Z
zhn-#)Q;VJ^%<{P5nQ(<)HX{7O-1rx_KSj=Oij>;R*i*BESHSOh5O2(G-l?x^CPsHe
z<R_f_S$MDiV0>srjOE`Mj!Q%%r#^5@m07%pPh3s#|4o_Gy<C%D`X&54cUJX*xZVPx
z7jw?V&SN>_^(^-0RNv4I36mt<dX|Rp%t#f}@@>&bbZIm3+-BpkX)9B@;>l?hT_R`P
z3KCkgZ!%8YBUbt2!TX=C`jS%`QvQqFl8unu)4kY2_~F;Z_NNvHa=3P^5UhGLHy|fU
zG$^V?gv+@nVB?n~MwUl()KqG}967S#cKj9Jg$*75Bo)5o&V16?zs~0%^Q~D^DlbUb
zwJ=3QiXC$23_Vfh`%rhH{|3!Kkz=gto}CL6qP0@5rOmzcy7Kh)Er-}ePqQp@2}!&>
z(<N!f8qPk=_&rk`J&ty2GKh+PyflmBq|&c@cM|T0-R#tCi%_zPSBje${31kSS!#Yo
zWM}9(ZL!3MtX{vCUey&l^2;^e@yz`{o==W@o^_cvYnO}53C$y~4NiI3Ogz5jn1fit
zij=m~Peb4HBzt%@o)dLRm2xrSa$v3!Fb)cG()%A*`M~p#g+QUpku7u7zR0v+ofx^<
zwVlz<!`UKWa>Fq$sUz-NnnZFKcI|Ds#T)1QM0tiw%bXI&SwF5Bl^&e^#$WS>(=46k
zyBrn;c|Nj#w_x#&;{xBfMe^qR<f&XO<%qbtFsLW+v4Ns<-;rjIWS3dD+^5ZQ%~Ll?
zd+sM0CmD5N#Z;arqL0P>Bqbh6)r%c@`g3{eE0<@!?D+?#I>wx6eK6zVmW7N*m$BcI
zmF>E$vBk;u$wOxmRmnY%`L;FwNm;6Qw@pAdFf6G3@{5}jQl>Ozss?mKI~Qa;{*n4C
zM9DSi(O$))s#0^J)LQ4d2p@D^-gZfO!lw2+Ki!nI<Xu0VubT8f>ZgF`B9>!03wXt^
zMED$x@72A0bg5`Uos5pz?1WiSi}v&dow=&SC^nPpaS1~|lhFgib(%&}N1pD@T(|K_
zxN2tUqKLg;j<)%ncRI3=-K+nzzy<bMk_8^e&9Ak@x_GosNzM#AQFmssPg_)&UQ*Gn
zqn??)ozobt#kxC_S`9w3Y%P3seeFqIn^bpB&QGO}?#@j5akX>Rw{8YkfzE&h7hF9Y
zojMj6Bx=|#&QWEXYa(776Z1i4<%*}B?FWKS{9#jMTwuHOjCj>T=A<KOS<mXD=CNoT
zcsf;F%x>xqg%?JgtpcV;&K*gMDZJ$ubTUhcox`am{c5xB7N=`7#cT^Rq&fb(I(f`!
zDCwP`<H8wx_U&Fzc`*qg+rS$S6x?Pq-N}6F!OIf0af<et?1iE&nlEn3UyFCEm>B*n
z<E!ADV~?IPhUV04$@=+MEq%_ZBPGvQ-Ev=C!>+~EnwZw-SIF8|*taq=G2xd=eBR5I
zh8#20uCdn!DDU-hxiQCi!AsAeXH#;Y8D=dL(pWs(EVqAy%Bj0kJ5R9h<nr3>bM4Cy
z*TM$Aw3%s3XXs>4xS5mHu_(*Eb;+@!X-%_du;1c5A-g4c*`!9<1D?(dYkpsu7IsT=
zKI?MFmMlA|HFIaW|KnIRYs=e@D;`g9d${=O%HRd7;w|!)SspFb3#(hQrh$v~WXOfN
zA_9_z|7A?dmgMJM{ycTHLgkE_wi6;P*Z&`tI#VL>Wa5)LuE$&aw9~X4nD4Di6LS&T
zlgTuPhvjXsTTR#c(#x|pv`lmon=EszWLl(4g_xXONV|t)^Vw_OW)p9kM%-1pp(}f0
z_J?E3%_JsvTv5^!FJe2nWLksZP8a*13&SjWf42#ST=Wh4>egSy*2^@V&+Yjmuk?S@
z*7^D9i3V|M$*H~UWtlol^|+r(^jFpA6R*Z!Yd>@7k*=86`CIO@)ZV`BQ=F5x;c2SN
zdAWOsTw^w!QeJM@voJ%UEr#ow5~s$^qfRdp=Y{#l=Qv%xrC4fng4dR%?2OkfG4I#_
z_vMosGgaO0mFe>v{!jlqVeX2si3XgP_kIbw!+9d~z}@|-?>+_PoM=)N`|SCXN8`r6
zBS}UrEB7p`Wo*wVc;fNqMAd>+vCR8o?aLoky?Ht<yke@jb*g~ZIgj2Qd3K&ITce)M
zex$ec-MV*7D;QK7m^2u-ZCetolA$BT8L7MC#nTzlUtPa;%-{L#*vz%ryZ2>l@8S41
z>0s!&X~Lb-Lapl?VwT<$n-w<A!P?6AquKqNHD`N+-g*2`*zu7!c+-vJQd7eZrMpc~
zeY!@q#^T*{iKQ2&K2MfP5aV7gW4$@@(8iy8r=F|3<L6><{Op?P#}-}xrShY9Ue=E{
zGoPt5Yw2E=Dq>%3{MN}u$l?h5dzJ-P{<G&hidEL!Y`*uoR(1uK{;K#8?-sijZP9iu
zO_!Tlch$WQ)QNJ6@6gzCchbD4^Ok8%o_jXz`iIAV0zT@lK3e6Jx7Q(E;2Yasg)_#2
zQ6BXd1M8lLuM+r>mA58KlCz05Omb?hq4BRTizCwH7v7YRZshR1C7j97U$-p9&1H4!
zyF4b2hCpqHAn_xC)-DVj4h%2uC>`$GF51b<EG@3kazLfm)G5Y(UEJqQjP5p9js~=U
zd8fVWPR0>0UQZ3r-}8FkXCA$N#?-){#bsLR1D!=bKX8c7bX;6;TlBh1zmu@o!OfM=
z7F}T7X7+f|W1bHcRWS)doxN9&Fn>NDcTBM3%bEcD|3{dD(%v1)tl3g*ZG9voTFv;=
zskA>jzTG-0nOB!y@VS0z0(X4!W=-~ao8r<1Ga4n1)ICfLu$et?rRVLyjZ40)?>`l5
zQ&gfqGehQfvdF>HANO$QeO&u8?1|X*mhTTu;)3OCn$}pB8m>7sD>!08#j`Jx{3)Fh
z3w>6d?`XXxY*!rn`{I+iOXB}Le=GDUP3cqc>+QidTkC(lYp`NH%dt+&Y^lIiMZ5S<
zeGiL2?{S~pb4jhc)ICMk*@KaN!a?tYEgyfLW!!jV+s)L-bNPD88?r53uFl~Wx_U-v
zq4&;(A-qPDC-QwW-ab3vZpgZK`gH;^{AuZbg^G8*<?o8P^@Qnz!Q=m5O0_0P{gnN;
zV$IFDQcE*r{yyYqDKBn}zLt2-%hRnTeR^@)y3fv2%7V>Hg;Wj%_V0W*X{V#jS7y0+
zQ$2NS&um_%cskeGy-mX+Wl1>azv8Q$TS_(rIVQ79o|(7aENs3>aFS2?QlD3|B%bGJ
zPR_f_@BTSg=JRC_VXGRArTzkz`sq;>+VfcUmL&CeOw#_mV?unw_Q%ibQXlIa@zt-8
zf4gz|^3BeFHf}Lnz@$(V>|8qQ^{#q*m-Ws^W~IM&xp2nioA6JSW0PE>Ho9J$%zWLx
zdF!q`7QTl)t<t^`tJceV9oaRdDNKJy^UJO6pR(?U-SEp$ozkE$l9{@&Fhp|-LsZuP
zHGkLfCe{lk{O0f9HFr{KUQ*UIaq~N0QZ<YJMr>M<6KTk?`uNh7rF;(y51v#=zQm@{
z&}8P;VHTu*U`BtXfJ`Y9i$n1Ds`c;ndS&J{E%`QA#9O_|O<mR9*X6~vPi0pnS$Nmm
z<bQTE)q3?^qh&$7LiOgi*O#h){Wo*5f%ziK8FP9zNzY)sz$~@oh}azMDoKTn#c5ac
zcQO`+eLt2fQGULn!<|K1eCMj^zZ6Zmsv4h_Ri@|m2uN`s=rgH|bb57HF+7REt+GA%
zPW@iV)Nfw5I%==$1!+Id&^}Pdyl=g;%Q|yYh94j6SavLw(>Uw1rt3Zj_e2?&0<N!D
zj;p?|+-3K_rKF(cn&l}=C*R%188=nm%z84f<Cv20)3{ib^obYNI-fotdhCr?t%ZLH
zL%}8&?aQ2VB;K4a@13>h;7-Ab2A4DXcSr9QoXW>Fr}jk!-y)M^e64}st|p#7_lVU;
zXzjLs=goK5f1fL~C)#71MDv`rX-A^lB5Nju#;1I{v`*9IjGnc0OGbXp(c&7%f*Jvd
zBY(w>GWC;ZFP3rOxS`OH;MmO0z3pMsgoO-7ES_7nPIxsOX%jc(T5w{Sf_w84x70g9
z1xwr*bDV@^Lc0^Rr?Gna&UkccYC|jA=_#5sF8fNFo1WaEd3D*+)$QwYUWz2F4)t4>
zx?>CDughvz=BMQ;{ka+Z-(jY??al(#jAy=@jHbmxdu|G6aWdu^8-?65RgT%k^8C@A
zoyOV6cGf>WcJ|hE#l#l#(pOK6jus>w?-%S-iJ7EwOpjIn@m}tk#p(N*JZ_2{IQcN+
zVwP+A*=mC=j)!7)Xv|#T(!|8cly_fkg+eD2;~rKO0f8k64Lc4^WcFZSNNiFGUY3)1
z=0>sR(ijg8g~T5Z4_{0@GwD-Y!Jmrr&js6jDmPAD+x4Dv+r*%YeyiW6M(sSAFS!5T
z=3k{u8$=jb3l?kZeGopdW|o&?vcoQ(mQ_4#-XA=dpV?*f!9IL}(1D(LRt_6@jEl4u
zv+$Vi<~iDCm9WC~OG3}bJ>sc0Cv;a>{J+t}EvvaYkl|F7&DrChPbl!3Zrx?QiAR|`
z>5`|SY;w^FMa#$>Gu9~|-fmQ3d9X?0NfWmM<0B5G&kDx9lXJOdie&{e@F@hJQ24CG
z+^cauC`nay3fHaA3SBxzD~%VOx!t1KtfS?+Ypqz-gcVEXY~njOOKNMXgV*^@t=;W)
zS<OAaY_9SLZC250RWm<iG-1POohcu5S=c8};NJZ}D(BRp0^6Vy$8t)0!p}<;>xOHH
zE`E2dt;uH1rbEj#Rzx<<JKNQh@Z9Fwt){BGYZMPGS+Y7hQ9R1n`*w@92jk;YwkLKp
zUt*oS;3;3yIt87D(Q6DBF-`T}D3o_fZ+F&7g~h#W{Qp0_Skfm@)^o1=%Io5uwkT_j
zC1nv+nu?lcH~M*kN|d^^f|e9@Fwf_+d2Ju$*0+IY<&9OH3gLCHHXLYrC-A^kIQ)&$
z5n+GsKn78Mt;NlAeyW~WcezhbBeN&tOJrm$_o9t2wo7a}ow>aHuccnp)H1_Yt9BpT
zcvF|{*Tzj7nX}FAr2f|Gj5@cfM`&Ryzm)&-)}?c%Dez1eSs=XhjFZNed|%~`lO2NT
z8XpcQxphp|-t2Q-LBIKRk<aPtq7FXIO($%M+n53zxL({)xxFk!KTCT<`>BqMMb|Z*
zG`=0+$?H^{aN<S_kAmbamPM1V2`>=qj1j6^DZeyBd4Frv#>2aUbpDsFTDd`mYlp)U
zmCyj4ZP!k5UNdSEYFJ?&b8ywQ>|Qak+P#<9!;EtTqjZ%m1e=PlCjLCs!IOM_{->MG
zD;fkdr!K2!6-l`35G-}c=85O(PKiu5;n~H$0Yyg(d1jnCD#Od=5GQkG&NCGS)|WS4
zE4V0j`E`iT-{P~D<=#oYIg?V<ndEY6w|euZ<U|NYUs){LIO&9#n!$o)G5HB0mRye8
ziWW?f?MPQUB)LD%L|gAoj-tWh0CB$~n?)rjD3};MxV>YBp!v(L{3vhkA41J@%?lOX
zt#3JazDVE3Y<1dL*)djP^-%%84S8*?*+NrnoOkp;2y6(R;P7ONi@+(>M4z`x|2qoL
zv`jd^a<zizgafmdZD@RP!cedCrE;ai$z8`xQ_40U_Q?3rGVyCq6VL0Niydz{blqr8
z5kBlJ;yLL$SGYn)2-mBwGdt85E(pld^A_$|)3xnrcTQmz`@soDo0}93%vff9XqHM_
z!|4&5>RrqJFzwH-2c8F(^fL!*sI#jXFtV<gJok^Xid)>vnRoY|H~v(eo-P(7clpL!
zg}YTJm}@<yHaoa83eQp5y>ChYKbw1;Xv_-7BPOhZe$Um)vpPz-KIwnrnP8(JoqEur
zbD~UKO5*E}cazUNo>Ud#qm+B2iNzsJ>By8u=AfP=-=9Yt^>UteUUBIV|IT(|r;YIJ
zYUWqx|F8RI+9GjT!RPHP)7jg$FXhy&E2_Mhn)m4FY?-sZcS2&LrtHu(%TshWns`!?
ze`N#9im7Y&Mrv=l6ER^^X<AOI^vN49(%o0MYR~&2y!hXylm1_2`)cO|bWN`NYWGp#
zhz`@&<VSVlH=|v1+=DW*S!E*yj7!foO?so?bVWth_nn48QBhBd&Mf`O5hi-dxgETP
z<w0!$$I54O_)153PVoJ*K}pnFPI8KY?o89KE6qN|8S`fN@4hj4hs4I`7E`xO|HkU8
zcj+!q(Z{A4ubh^-T3e-CEj{3OdqVfxY}5Qo?dN)Cf8Wo0G4YI}#%KFeUtH(2ER3#B
zzA_^(wR^E-<b#YI3jbL-*O-<_9!zky|9yeQKB)Lh7oY3Yc7guR_n}gzC%6SG;@dJR
zJPs!NG6|Z^Gu*I9OxyI{X%9t4medeyopqTqypN1C)7GwbiV`gTIQwyKpt<3L->Q<^
zjCB?JLnjM!tZ&Gitg5QsaI&XlQ}Kc|Ck_;xoYZr1VvOa=>l4p}8k}!3bc{Q>n`z%#
zkCM%^71~YDcF#yH6wEHJ7M~oZmhZQDO34|=?qr#ZCtbGmN#?a3<Pp9q)S|TXi$P7C
zxOi}xOZl8c=j^DT%J=djn5w+zmfrj>>3?Z6&&;3QWs-}|<dp@V-FNEQ3V+%CqTQ4A
zX3rCiJehTlGbds0)^%qau6Dg<`Trt{DMo90-n~)*Z@G4b*H;aDyll-`Zk6vo{_SFg
zn3GiMt0VI_&Usj&v`vjk-RQ=Lq!+t|?i^&TpKy?oV_}Kgi_=F9%Mu=JHgSBkBJgmH
zNxzEkjJ=$XqN2oHw*AQS>}0*bce~MzsXC^k!m0T~qvrbddB0Nq1mc#|D9M;eukQGi
zG*fozlojbm=dOyGrk1t#sB8MRAfHtWr%fxk$GhgiZ*l2F<7=DNah@zXCKNsO!5j{Y
z%IAkV8Jv|ATHF8pdCuB-NX%>@<DHOo*Lgl}-CX7(sdxQjWbsz3ZTDV3H90SSukZIK
z)4h?QRiA=2WsL9mKKiz=^#8@e+UVVm9M98coRR&nnE!u6@|i=e+J-`tUUWTiJ+?RD
z`3E-dv|58!=|?Mfs$|Y%yZG9jQB}i5{MWtBn^$E2e<E=AtEuXehAnyOz5@489d~pp
zbgizau3EO^xR~DWkB7W3b}gLX^fgWK@T=R-lSTgvW!zfz?DD3zdC%+8j(>S_J?YfW
z#*$OZuDyElwbC%cQeScJ)D(dYN}YNQKQrpNm)vj?Iwm0a-hH8{$~2CrMo#*gI&KVa
zO3h+j51kBX`|{VoY4gKBf6x2#6`8iQ`c89NeqdK_GslO821=aU6c{s|C#rDWO;ApV
zQfJ!jWRvjN_sxplAZ1gBWhMsxHyOAUtu82BQ-1hP&+b=;h}W5E|NrS8^;r0oYmuB5
zlZ}|3$ty=i4+V9mWuOaqD!SQY56R^;vfXVK)ljOo;5*)x@3BwwvIukUE~XPlULEb=
zncTy(cFLP)*H)Qtnebtci+P)_`7}Nr<pr#)2|PCrDw!nmxFzzKES#$3ARNWW$Lhd+
zt5MX1QT5mYxsD5B5eu&iao!L}Voq4BvSp1C+ah<flVJ@{BE<@~)v+AAqkm!9lYd<*
z`bm28cQtwDEn;eA;WRw7QKI`TSJPRa<bpYCy>z+{2r-;j;*2%xxWmE3(38x)CYfu;
zh8qSi557s|E^|Loa%7Hy+T^+vE4L1-DD?oZlZKyM+#+6HaN)X?v9kS+^Oyfzi$q(_
zEtYewNl+~>(MVWyNHJ<cl2VgsQ$g~b#iC`|ni_6O0?AL$D9kyKaqPp%St)AzhgM!)
zk{x#M$ea}4Jy%vPP*Et^mV5n7w*uQ4g?itK41uR)R(Zc!Ew_z<b4!EhnvG%_O|n^u
zswR%pwj_u=x*%q9&~i(H@SjFer)Hjg271e!|Gitee!^43swMl(crL$K|L=j`8=DW4
zIhq^oUaGuTQTKTKBu3e+K$n5{&}}ZKI}(eUcN`Gi=D?-nsUVxfy+NhLX$4pR87<Z~
zCwk7TIn@0urDLm6)8Pxri8bx_(u4w8{2n||R;wrs);c5?z}hC}#D4w7bdE)GGaf1C
z9F+Zk>p*$Q22q_p(Hjm*ZycSf-Z-sTt(e2QdPTP4tb)gToaZF)nN8|{w#wk7o;LUG
zQ;MRWjF+!5E^+2$IS^pxXyLI??$ZXb8;wd^94cN3{JE9L{VIU_M1$zA^<p-s_>ET=
zaX7WS%XoE!NA0PW(A!mWUUaNc+4%a|38sU2oM+eEJn&8_?~s+l#r!kLQxsk(oLMCN
zZ$<PypG^v@Ssp)nyM~3Kaq;FPL(TpoThW!JY75MZ^elF`_;M5-`0&~8>*niuy+6|y
zN2)DQGAno>aCVUxud<TA5}!z-Ur_n4wdE`cA~yXuZzKq72q{J!l5%YpUJ|(G!6H`H
z;_oLGl;$K>m?<9mFOb(B#pqIcd0JGT%YxM_9|`I&d1)22xcA3J`Rkm&PHA7~Dk{o4
zY}{HhaYu)lr=yam4|hb8NR2wzk_H*)AWv71KgSMCYRTg^Nx0GE?0w4BdrHIKoKI2z
z(iOQjn*3Oyc;eH%Nxu8jGI|&|Z^>mFx=u84Vz%0r8$0Q6bDZEjk4BBVuVNfLINh2&
zR`Q&ZSb37?tC;6!cbiwt7Z?BewAT1s(L1Sj<rhmtuNlQhIjWg0<GItQx$W@21;^z?
zl)4KFobntE?2n2Cq&m%6{v~Z0Z%UTF%;m5<Dux$NNH0HXw0whhHs8d5CESupiU%(!
zZWS@#H^D=F`J_vmCoxPk5#Orte}U+YJmnV#Q$;HR817AY7^K6=G{t^mC__dv_w}rp
z3tZ+qj3zI7d@3$z{ehmnD$0z)AC%fItG+%Gm%8MN+@jSMg-?1~cJ6xm@6Rz4kHgm2
z`%DDNM6Rx#|0m<jx|DO5j#}_}i3w*}3MKSU=rLd9W8tKs_adkGoU27mv$v}5XO`!N
zPRWj(XH)pqE|t74i&$Kh(aCdbvDE6o7J(&cy1X?p1unN)^z!_rw?6Nw*`{@5mE7(}
za+;s5?j%K>TPd%4;Ac+no_E6LXS~g&J~e5beAyCmO3Ep+eM?_NqVO??1CtV-Z#gij
z>}=L&V++1XW(I5))qx2fj`pvT_1B&etYwM+pWX8M)sc^zyp11-{ID_f;Z)Xmc-)uW
zBjMwt^B*4VcT#@8;?e>k&bTSe$}^NX%hc^|dL`XgWn}tdE_O0jN-0)DeFvK^%lT$r
zj^djkFB2VjSnN+K-7pF;Iq8_w81gl6aSKz|k`;vul@hisZvEHFlgDzRsjWB1>DdXl
z*sYhieqKIlGVRn3eXdRZ42uptnszn&aZ{Az<rslQ=3IxWcQ1Ld<w_MNQ!VGB7mE_I
zw~E$YJ~heXviULB-|LJfpWw1~E7ACLxa-XU(SPO*j6YIiR^|L!>&_v#Pogj`|4^=1
z)9E)adYJ@YH>_7IVyUxu`1;bj^#SYdX+`WZc;&&vt-MG0|Nlk4iTOdB-sovxnDnmZ
zWlG|X`P&#6!UfMidaBT{;K4V)WaY3J{ifTuCM&i$bJ+-eyz%&6nAu0K0!PgzX?c^f
zpSez!0p(fKq@tCqSk%6iy?01isyk_IcG>gYC10+6x$ORNv)$v%wc4F&m%o|sT*7*G
zTK0O@zXD8JuXiYH(>HjQ{QFk;??+~g&&{SyaSxfruE}s?!vE_jc`H@_UFWTE$v+(8
zb}Xb%BLDvu%}H<d-)u_SFSOm{1@GAxOS~=w`{-RSGEVxNk-WBvAs}~0QZuhnOP^?x
zhVI3~ZI`wr=5Z`Jb67di;7ddjb4y<DkEa#g%WP~`E)!F7FIsYfM>Bcz6aW9`y(5;M
zSJ=KFhntBnvax^Zg8sz{yoN=y-5W)tf`v{jkXuk>a&<=a)teF9r=9BZi+(us#hUQi
z`{DC?`Kmwe?B8h4wa9_n<ly|P0lb|Ter1H&*Eux^RyDkRpun?aVY-{M#G#;PUptx4
z#y;hUdz^2n@3fEUaM;D>!oNOVn~n-ehUIY_w&tFe_?GYgj24mP8wz1kMq<uuvu?EA
zJ!-+=V70^Gl--KroEhhinRnD)w%q0KDBPPmd55tbmt<jN(l3=XStlpiCK>KSNm-3K
z+bUL0`Mf*Bch89(d-lyzxOz(Au)&JfwW4npianbt+7Q5HwdqfkBe&Hi*U(^wSB}%Z
z6o~w<apbzUbMD_+efg|6ZcXpsx?sMm2Jd0ZIi5VquMdj9Iw&vqd*k1O@{fLf<N2vD
z%bh{&7T=u(b}1?I+CJyUe=@YFs{J!f)%(%if=R|);*<4fUa^YMmwK%7{iR=nhp4yZ
zog2YiOB~kR&k}7H-EehAk&G8pnfEl=+*+HQsWMwtOE#1hz7%NG&^8U`p4rpByga$t
z`Gu3vypy*S4wk*VHeKY;LB(G)lb!`sT)R*a@rc)Wl4!yP)9V-H_VFur>Ghu%`+aG8
z?c1BX|IV)MeP{ag*8HOhyrQ$k3?|8Wtr3^oEnl@zUhTHH!kvS^A1eG>C}!d~(X=M*
z5z`{xLx;JO7RGjQ|DWNq&Fo>{tfS>-nkAcaRD9+Ls77q@<uz5>m>c%{-><D#_El^R
z-IZ^pXuRaeTPd>z0b%_vg3CmgpO9x~V0Q5Nr&n`X>p9oeldpZU4V4(z2`cy8D${wt
zf>(`&H?1tqQqRmy$$?2VuUN{T=gbTp`@3@#E_bgvxHWsbCBLU4=aX4Hag%qnR^)}P
z61`%eY#5`wg@Mz}kw04a&!=1BZi*^5ChYtDOW{q#?+-gJ884d8Tg@#Xt=e%xN%zu5
zMe8fd)>jOf*b+A=h7@y0=}(M3|23{{A@`!IoQGK1&xtK-6<7VZ?@?!d^rSu}-rOMR
zDFzpPkDl9l{;{W>gm1F=6kp}0{}~qBZ*FcrR_d)ft;AJ)p$+pgrdGccBVWlT<&f=)
z8ndP;iv-VfV&z=q<=nCTZ>7pf1+E&sd9NjFmn!g@CeF4Fn)5eNWDA3`VUbAPx!64^
zDf~)IS(<ZN+eJTpNHi+qHZVFYXsf(fVNUetdp~XWD=(;hDS6=X^%pn(&KHbc5IuRL
z_@0A*6A$_Sd-8kXA>KuY!s8fX<esJ4J&TNc#@Y1LT`^fGa<j9Io7gP{v8a3|v4ZDY
zjox$!PX9b{bI;^WN|BGcjXP`&9BviPSbO8@BBq~T4II`g&NyPdx6W^8ae_YAevb*l
z4L@UM-6`hxs&k(<v8mZ2Dsn}u)#fD$uGiD2{Lkt5WOqo?_tnZ+^{Hn*D(DzF-QQ(f
zQJ|Vu%=ht*<I^hFi-qe>PP`v@VcVAguB?T8o{o1nhuyfeS8h$e>ZxsFW)IhP1_=L%
zWH5QC$vt;Iw>8(K17E-XG~dd9`ROcP)~JoPdpEk@ed7D?nen1$(RRvyu@_&(EoAqd
z_}<|f>w+|nrk~U01MeK3c((bvsnxd5Vvk>0lP0tJ{9dFlcK*iNrG_lpvpKwLGJfyB
zb>x?tho#5QsS~$qFkUr(p8DB(@e1+PTWt;}EZY3)vNJEEQ^SEq7S>IgH<nFpDB}|4
zQV~w=W(fOs%w^)Cb7H%rz8=eXq>&)(9`-dWQ|V~ml7{{_N-it^AN4!6qH}AMr$>;1
z`auoBB@=FnI8Hvw5w+#Ro{EqJ7L~vXz8(b!Z#CUnTRNnSv#x4|Nqbkly?TDWeSG5Y
ziK5<N3HptHuW@BqZ<x<eZ}s}r+V1UoyLp$qc_RA2JpI9*wYPQ@DIW;a)SGkS$z744
zB*&BWtOYk+A`gf(N~>{C*pa&V>G`Rzy`DME>~^?X5jkD$#I3EbjhJ`yl>D8wtsy8i
zq*&@~bb|VArZ#T<AJGrq2Qf_&ufO|8|MQ&7-jjtKx=v4-&B>&~A*8Tegw<lQia<e~
z!jC;%MOET6;u3y0M$5cPF%__{Yx>3gPQGG-!+*|-`3xyNpDxE6D{44Mv*~&KXVcuq
z6VlDV*E@+pNHFJ7znb^NCxN<adIHaDcuerP+_C105tn=Nge0NPwH_C_>eg<V#1g{v
zfaSi!M^1%S2l1)mtU^o%tWO-=M4w*NSiIrWldd;cg8TKq{wRJrH8It5x<jl`2!Bb^
zy-=x{OTRvvtX;=zHbI+dhvorYraKqAq*J#9w#{Yw+BQRXZNY|_vzWd-lgOOFIVCbq
zXy>$9Gptl!Wh}7_WRr3{!mE|JY*yZ>m5nzlcSV&j#V(g##$WsDM*b>aLsurQis!Nn
zOedsTL^$t!S;g@FZ-i9;(w`~w{aTy6a#9)7b}7Uei<+!qSiI$8-+Vqxm$pi#p1{Dq
z#W$4xyRv9^wJcQIw&k;ur2N&xUe;=v2fcdN_9!)q-N;SM7W({B<)FjQT_P%>A&IS`
z;W33r0`<Q(JoS&{)W73@xMZj5fv6(iKVe66UVc$^loU@{c2uss#_(uLmrj6fbWY;&
z#L|h5OyVj}G#NO9a}KbwN9Q<5OA2!|%#e=pad1mmGU1_&<BW^#(^DpuwrSLQO*Wg5
z;p^->%Vp8ZYctCN%f+NFFYaxNUcOXR_Eqk*LP4{Q)88F4bG-V{HAbEJq^|kR(+@*s
zZzer1y_+ShaWS_1;mPGJmDbC|TNpaIO)RRIJU+zrPkq|0q4|w>$M(gw{Z3rXA;MeU
zI82kS$Y>FL7V@84(>wPEf545ai(Q3GuerBPoV{arr-);pQQJFD55p(Ef_Jj?*Dsx@
zs5&)5Z~fCxiLrCSXQ=Y4*v4HguiCpr;jyV(t*}sudvkO_gT~LY0>{nL?YU*kS!8wy
z)kz9Xx!9HDV$UME#M9q5enS1r`8$0B7n)Wl`K~P5w2eWlppp6K*$qLh)7izISP7^&
z=O5U{%DP9ZVAHOtt64T2JjUq~rce^&R}(75nRCc-zLdvit|A>n&kG8IXFOl96dmRi
zTT;i<xUh24hSsu>Z0D2-O<avG>seGBncPes@$cO1^5q71E7L@Whb&E#qZ$3;nGBV*
zF9eBg*mG33KytC4LYnyQ|2h2%MU4_ieofrF!@xj&p8$jOmIR5pf7Xa-I{R?OTx`>v
z;NZLV#sLu_qejgY`2jDBT_t~oO+VeTBy+;8Gbh!KZkarn$0(R#!4;08S+{G9vSxMl
z-el5{7N2)1BwpG>vhH#x^D~#c%7;gm3hj`-dqOw2LRhv$AyU$#sEPY=<J9ss85W;i
z{1jf_6nxyX-AA66rB#sYnuE-j`Tt|y>EC|R@7L72z`1sv!W`+lrF=pQ3tzT~2<^C<
zzG%Yw*>_lMKlSum-}V$Z#cIH%#ToqV(F;R^!mQqz${9;G9p}=W^01nD=h68p8%`WK
zu%1Vn;nt6`Wy@wyb@a=daER@d`IciE>HjCCWna^<7ha<IAt*5PGf&Pl3C~L_(w15-
z)bz=prTN?<U}|c1YN|}J?EM4F3N{?NT6xHL$LAx<ZNDpVhz48iN<JDe)mY^k_tBCG
zEBRDPEYB>IO_b1OobzthwWL*hH=MX~d}XPE*M##?J5OBO|D&tRLee$r)QZUyU3C^z
zwx0XOV>SI!$x68`5y!P&Y}`2EWZ*{DkM6o6zif|i97z{`@YtcsjCH|OMgOcKR!Npf
z^{%{wylXoy#2<;A<!6^0G=Jh5PkGjr0aa(S7b?w9<0>+pZ4|J5hSvk_Ol9vE7kVxB
zYvwU8PL6qg;nR+yYH2^tNpm$d4urO5uQuAhSo@e@O5wZzE6?nmaf2(NjH`IoqdDOx
zU!Go}r#-hcu~mej!M|z8=C1loE#{L2OMOH6H{`^;k@`AIxPhxgDe@wV_K)AwUzMsD
zHWYgNdG?&kl64c$x(5v`JjLvekuNsS-x%qXlX&phAGWO%7hO-6cxr$4gP4`}v@2b%
z#Rro<hD5AebGA6YM`rb-)a8|p57Me<MXGu@Fixlrt$q_Z;pdX)D_IjirE@*jSZnoa
z)q2lq6~#*Tenn=qf4#Jv;npt;gX^tDdmc?>y7lW|=By1}Pfwm~t9i(8^3V0$h0of@
ze`rlVB9tP#c!9g)i$Lz37n()u>?CZDZ0KsaxO08!%EkI_9|I5m(M(HGIQ;)?R3f|4
zgERV*_Uhz&Ewr=V@@V3sPpX|0%>)Y5!nmRZ($icf@1IkVn?A=;BsW@e)?Cjg*S3px
zTsbt&?6%mFqxCCa<**cJ8c76LPEYbxzo0fjZ^bc-^NGH9N`kfo?ftZ4Rg$#I>f%&i
z$4^#Iww+<=@d){(;Ah)hT;$hdE}6Z_!2Zshj=isq@|$iPpT$43)5ULZXga5hVt`Y6
z(hu8-x(CFMPtxX?#4e;Fl&E>Id#`cyyG02zI5gU2Uss$Ady-zZ#z{E(r;xUt5<^r*
zlbDEbE5oPL8GBw_m3z09bq<HpE3+Lr0l6nONZ*>UsnOzy{`Q9X+WXwQ+|@W9EEn-z
zI_dwD{gWT69o2H3)mh8?aUPSv@#uI}$A<2e4~+I)(Z4I=tN3G9!RwG?3xh-rZ)c_m
z3aPA}%M`Y6f7Pm~j2*{J&TvjWdW>srR%Micz5l*hT+fWnCH0fm^-SjdKBfOf<TFuq
z5B+UP%j%Xak^3vrDj^WPDe8iAQnTO5uxCO2p*ANyr6buF9h~&MZ%c-(ko#W^7sX>;
zj6afVL_72&K3+Vyp)<Yx<(dzli$1gQ=7#K3nWkP`^iDIU$<#ku_u9|rUkc+kUElt$
z=epR_%h%5tS|<5F%}To!ocY~kM~Swd<TUF`kIxk4+)JBXT<DYc_D^o}ou_$~O5f$A
zzxXIzvB+donbYvUfMt{UlcT<Kn6-SS3v*p$^G|YLoFFgS)Hq*|A?}!_%ntLY1ALRi
zCFR`tx3jw!gc!YHa9^XqC3BHKYNDvF5`*$``9%)gFCG<rap1O$;AdkimVU0X*r3qj
zO1hFqrRBqjpb753)k-YbC4OyFV|G+$dMIA^Na3j)*F!adb!{4&*Ia`au&5_z#y)SY
zf7rU`iU!XVjh;)vDFzOELz9!$+v~O`uW=Cw{Th6+P<G~1tqqfezcb~3TAKFAk!|gS
z&`C<EA=5gNE`|QMX08)r|Mh}^+hn=k>nYz%JCfU6rwF@di*>|b6W(f2x%+y2fl5{G
z6O}gCMy3PlUoP;kTi*1>;6K}I24Uks?p4pbRE_wyYp^{&!1eloXtp8SY6Z9Qk6lR{
ziYksaHBPhqdO&>6H}Sb{0+&_@*FEG~$`EpfJ$7<QR;Lkb(1+eImD*JwO9B>W1s%=~
z*j&qGkj=70*!Zx0-z2j=OUzTlokP;<nv2X?F4moSCiUdGOy=g?gBQ)dF7LG}nK0KV
zkN=}_!(-XAPb?;EPiy+o9<zmYv9Ko3iizK!$zQ+H^4n0DLEZ1<C53Ng0%nsMQVimY
zIb`2VGO#*W`6YmV-HPVb4&1HBlvXXTVoFM|n$fIvyvU(bPi4F4B8Lozn+A&tBD4!5
zX1S{}DR4z58svOY>2OtJ|J=NH`BV>&|57L2TQ3OOu^1|b-Vk^=wPAvz*`pE-wP*H=
z8ij)zq^4eJ-7!^#<)dd}aNi5|;7kiYR>m@&&gB0uO4qiuTZ+|xF%OPe;K(>Zn|p~9
zpHgZ$hwR}^9qT`qKWBA5tzIH|qh)qNg=&bvw2V%_uc`HlDfv%*m@bC(UKfs^Vvshe
z%70On*GwV3aG}7Hm2VW-UKjL)?c~zcVAy?=zx~+kU7p=7PrDADoNaug>4i~q<ie)E
z0g5UUBT^j=;|^7qUW>mi8IiJpRWMZi`1h$z3xzLDYF%)+H~MO6tjMfcA4^hy)Z9Oz
zRl(Rl*D?Fejp@u2`&L{_G@oj=?IP<O1@G^{0@DxuuRCuTl&voF-rT%$q4XKXh2O#_
zM4c!&Y$C@gVDT_f-%FwVQiy<Tm2gTy{VkRD8J8{V!^O8>RxmG7W-<)pXJnYqG2`43
zWut}Ze;07ps)>7bs_xemPcjg<{#E4kQl~bI-_29i!>V}|yQ+DFlFUQ?);86{D{Tyx
zs~S`$FIm7EwN34hS<NIf#l%wyXW3Psxa2+CGVQ0QK+uP2oJyua!EJA53YVXpzjxVm
z?kBxgAKMI{EigVcN&k!VrYT+xQO>T5>w8Yl=ranpRAijCxh5#B!01SMj=RppoxK$o
z*&@@pzZrBc`aE-DQ)fx5_`5IxJwvu+!Gz?7>c1PB*`MjC1pVjUrpfTdfK71m?8Oe;
zZw$CzCm5_gz{e)s(==^$V_4788wTRvCy8#(IDJYp?}@7MCAMu2y`_oV+Z3{{zEa;7
zz-<~R{7$H14a2&JUCz;^^K@IBw{mIzd)%78llA1sUKLe_$qJ?m6$PF@%U(3mbIL~J
z@0`-+mo;DS^fFs&zDsezi7@G}6772eV=g?4*_XJ0$#KF7v)uS{^Rubz4wb|dbNHOv
zR(~pF7T<DV*(g^NCZ8XTzF($^-~8e$8!4$UGmGny&S$fH^FsmU6>>|o+I@~GR;h~n
zHAQUEY7s8fvtBWI@dbnF?%ZZ8n|?kuaI`8s8!Z0M#g=tryr_FuJ!8bx%m2Be6cme%
z1bDvMel_kicjB5U6_<8fPx}(rRsmgA$AsCRHD_P2eAbmUDPa1egUV~g`YQ#O2Oe%K
zk1pk5ud9ANJu$t#eWvD-Hm|Cd+)oxOEgj7^eJooyiRF>H+SiFI+dTw=RFnHYI%(WW
z4*e#@cS*;5kskB&mFZJwbuFKD&Sch46M?q}JHw{ve|*?z#V9INzJ_ICRpT+GE0!4s
z%bVBzu*v^8>-^;{B^#?%g45#<ZDv+XKO<eN93il60VnstwdO_24^OczVOW<Vu#Wk&
zDEnfAL`AXl!g04|wC>r=8KGG4JW9N*dV98(pW~ve2>10t3G3$@u8-Kho@s+|%wx$F
zivM#WKYK=AHZe7><NY$dAld7{RGC$R3tokbXFuP-;#eo3z3_Ya#=2cb(I3qJtkV8a
zrvKT{Rk5*vvt!fJg@;NL<=Gt_qkl-LomR*xu>KRKXrZ)trjzTMV_7p7v4$-YHafW_
z@{uZ+S&PW?gIp@Mdy}V4+bn-GiaYEP|F;J=;WA6Yylt0n)>$*5wcTmq)>GU4mRe7m
z5|ma^yKkAP#@BU=N|JwC<m7t<Cv88XcWlRm+dK4^O6ec*me}1pNqR?-X_{T5{E`Jb
zOKqHIygX!iQE$$!)RnIrEoO@wD+_)rGIHC(x<l!tm!N29Qcup8S&=nvr?S#HrfGH`
zb0{(7ifWk3S^QrtIdET=<0;F7YxLRIRW9#cc9hS-HR*cwUZ#(yQ)e&;ezcsR$obEh
zb5f$qmx;BrHX8q}&R%iQWLk7^+T^xF(~hTz@3ITcwN2IB@G$zgVeYGR^Xg49%qPuD
z=ZJH^l(Dw)%`BaG?MTDDlJd_!`Rs~~+C94-u=ogcIDZcF{kkw!@Q3rv!^$ZNDO?XE
z-fp>&)FiC0D9*4nAw;^k-ZUWi^jyoti`AF;-yN+iQrUWW@!UQqah^ii)06hKGpW8e
z+RmpTAU9Fv6StRxlkbA0?X}h?OMmv>I(RwedeD@T<0r3qb*(;EFlVE&n3-d$S@|oe
zkTAzpS8|g!vp%RkQL>_A>Z<?ATvK$qVkVuvbmEtq!l#tj4cV?aZt>~5X%-VtdWW1`
zlGu<JdTq6(Z`1|8Q>Uv}@W{r0QuT~hyxu7;UA#DI%|(gktwkkszbs%4dwIQdqoK;<
zLS{$tb0_7W7;Jx{U=`q$x96}m%fr(P6fc|j?l6xP*u|_e_lxK3jr$jF%&u=ddq*c{
z1)~|$#al_A=L<>avhI@F)@ZgXHo3lYL*eHIsR9d+PmG!GwD9oSa~D;7dv~Ti4fGX>
zlKOUM(?#9GFD%zB5Y}m{IjpzK`Pw1}uK<^yi==*Dy0CqBg%gudv5M{CNTb<*7rk1e
z`E>%TmSLIdMRT*F{0Sdbo}LQ5-g~+DWY(k$|8@SK-CyigYWwyqBg=+>Vu#aisY{!@
z>u%-nxJa)oS)LmqRlO&7f%mM9tMyY?*Yq{Y%h<J_-E{T+Y~h)|oYo##sjF3z6f0$2
zvsvha@w11*vzDCPze!o5FnpSLMCO;GCDT`%XPjR3LS^%>+3jBq+%+^S_G+H1I^s|j
zemnRyE929^Z25QA>-cWSU;n6d=DKC{c2*<Z{X&Yzm=d?U7hgNMM{@PGW2=3qPZRW7
zVxqaNb-L+Msm;2Y3$k;!A7tGlo4cVT=J6VtHHk4SCuL5nzTnvH_n}w(lx><<z`Cgu
z4@Ilx>#P?3+-7OArgaAAh3h64VwSm7tlGtp<D&g?q1DF=>!ki~J^b>O;AXx{$q{wS
zL|u1s%@Lofbt1iU+3HTQi><P@O}d&lnIo*NxZLg-GKCd}?!M98+Bm^PnA>v_^C8Qx
z+6?n;FE3cg$~kGi<EMEM+a)>PEpP5V&3k6jk!1@Go6WeoraU)&LrKT!!$Dg9bKlLp
zv|?s=`Q)e>lO*38Uwc=2@G6VrF6$@TCSU4Y%N=gPnD)8l!y0Y*xg87BODj11EtY1+
z3M#amloJ24VDf5%?!%l*N<PkZ)C}8lYmOh2*v6`ajahrjpA{}!Q>~X@7#&%;MCa42
z3*0Mi#S49Y+-sWNFil*O(Q3xs7ZXaJrSq+yu#hu)!x!KAl8X;~@m<V%`c77~!GBJN
zqP!`KgKn$W-D?-GT35Tn@ygYh+fxIzRyj)ZSKqEalygcj=0xm!p;ZU3JbWQyd8mJ~
z+@@&xD2=-^o-gMt5-^Jq__wF`{Utrq+j_AkuloF40~_K!Z3OmfR_XYw+P&akCEnDW
zHhInUO0RiUqQ;+2#eHA8kt<1{sd8hr5obnh;_1}|;xhVNyH^XpNm%x6nfyKLw?A#y
zJu~vmT&HqfPE^itJCmaRe*g9H716f}|9ty#j7LwB`ImT!&99_Q#{<@Cs=Rw2vE<cE
z4}k}<5(@XqXFirF7uUQY-?(POqr^k`(HmT*b!=|TxYxJHUvPr*><OQ1mQML;D$Hiw
zoxmut#pVCSRok-?90S+<+9-8n{ry#;U;kWej#xdPp@&H??a7Rd;qDU@UHeoxKPexN
znkACO6WM=FE9?yi*9vntkt2cThJ8}b@00ciCcXIix;&^sf0B*yB%uRccV_gY%qo&;
z70hz?mE4?k$E5AK#)dFfhpS2sO%1xUPH{~r3Ole&w~#eQ;-u>VmMt>dTkb>$HQe3b
zU|jTxJ4ox8k*x6aEvHv{Cm!h)FOG^?Rpfq1WoFVHo|~W3FQ}^aWQpD?nccHy*`}FM
zyG)K<mkgY|YFDSv{Ry*g=BORslBFFjnf=mt!jUf@FC4q*wdVT91<R_w{_2!c6?~(`
zXc=xBW7>69V}>y2ugezyOOow7Wz3b>cg{;L{;q3mcHCorYVE8jw~73oh6@>*GiK#Y
zSr`*^(jiK3RlBIKj#Q@dr;x)QhgPbr$y)Ws<0gypO1FzoR&CmLDXYgfgZZ_2=dGKb
z-J1#yXK<-*o6*!Qxtzh%;G4~6?&IghPJeWNV)R>EMbWY9%!vsslUErpTcYIS$=LP8
z!*Y_ygr=UbTQAlqUvqdB)NuGfkCw{HLq}eC_gxTKsb%Jwbk>J4D)acXvbgHpZk0P;
zPK<q1I8ORMRaqdU^7zjg4@T7=Z!-LQ*hCdNIkv25wf|?Z$h~LPH9=>Q?zlsI%IvHw
zmd=ZlF`BB;(-HN4c~98h_n}{{l0+B(|8eVcPY~mY&*|4=SWjeK{~_RYL4NuT$MXv8
z`nvu-Z%vZid6MJjPQBK5BXF(O{Y}!XD_0+D%69JCBvih3<Gx)JWV<_*UM-%!<&X#0
z<_le?-K32doz~}bI%TeWXWN4&uAUtaChk>IuYPl)kHziNgC3z`31!8lM^;Sf(RkNm
z(8HS0b#iILml?{OLJTi;CW_x}nsVdD;t9%J(R`hU&LxNyFX1-UD)ceSVP3cRQqIjq
zPx_nd#C6NGSYIxUs?f3ufB8#M>}rgn$jY#VtNF_87;mj?ooZ;(7-jpat!SEBeN&S0
z^1#<8wf0U?`sCyIJK~x}4Db3T?ZtUprBa1A?34bQ`9CaMwKaw@XUar3@zrfT?hWlv
zIQ)((|1W&dQzj#nc>1A=kJfIB^)=5l6(=W5zO*kz*X+s_6;8h^3e7PMW-2yo(pI@-
zEf%$iUU_`ll+9|+madLWyB0qXmA#_-?LlpMP=vqMD)~mCoT+=h_84iIta4s0vNC4p
z_v7NR2hX37i+{86XwB#Bm8z%1Zteg3_nY}erM+Kk1h>tPy0+0%k<EQ#=e(fUBMO|a
zj&I%;JSD8MCG4EHsQWcT#R*q<V)7L)MwxwCxPAjugP*g4+q&t3OAIvhgJw9yJPWnx
zd(f(E6O`!wZmPw^FRqDo6K_1uQxQ6^_{Om5Vbc_=z_y;7tw{pilm0X4Fi-LmDfE_&
zR1)-@<f7uGr=lu+WpP#UoO@f(8Ec>J$bNK0wfE{9r!6NvX0s$%^}XRy*><Dl!~}KM
z%Np+7YYb+2y!vP!z1Ka8^N7FvcZt<2TU)Zet!)3Bv(PRqXRYX(4AxTt$2_iXNvhqm
zSf#*AxHX{kh{Eaz{MWx7=3Bp_yP4^Of7!0ddRb2|T7K7jP@oXuruS}%cFp72<q;RT
zJ9~nr{>W?>TXI3X;%Y`KXS1@QF8hq@w#VF$>Ui>Id+e0r``qTTB<4T|%dS+RSH?mL
zt<$zF+10m)uRTSob5@eM`Th%hDl1Ojo%3qSru)493<pnlEba+jQDv8<av)nIMd`l+
ztCiE*zNx=%HUGG=+AGyx<vzE(^TFeL7bmdPZo4t}+O6YuKlb`F<Q1^GzfI`wZ@VI@
z__RZt<pI~dvm5&stm3lvI4RJ3#YrS3nR9iPBY#xRgsP@Bnrm(~pN`rgqINT|*L!x7
z+oDU4^Os#Zv1P^5)r)@|7MrEf$H0?%XzzvBam9s+Zr_fE_T@~CpWm!FL1+4szBdJJ
zeZmfAwNuXa1?Y4-%lHU#CL1a@v2~ph@jT&hX!#`1#ECUuz0WuWSuMGyr<}83ZirX&
z(g$lMig0=~#B)kZX})n^IGuG#jmuW+5>=r|y)i18iN>DZL6=0g3pD8ZWCisc_{JwW
zbFC~(!0rFn+!9p_7mJnca^ChrBx9jW%**3{8udyXCy1WTJ$1k6r|G?SK@8u1#Ijf@
zu^us8He<&|@Ay*z*A9hD-rTZO=Fl2W)jpTYff9_js{F5ctgPWI?f%MRag}BH??Y{K
ztu+_N-#E}!_UYWzz?+jrUTFF|T{KqC7m0HdO4u`T3v2!XtvqJ$Pqy6U`k{|b8?-Ht
zw(ANiseN}iL(`;7%A7-Ci300OkLWwM`VN>WZ`j)0V?8OUZ$}yT>>F3vBb~H@UT#_-
zv+>^XIqZRfkBVY<erdg2ro|xteU(b9j4R70lk?)a30y|Evbck1h-Y`D#7F<gie);s
zIAc@0d$wwDT0@4*XTSeE>-<uZj#~t8?0FNalDW*-{YBK)_;2U*H`?SRhAehHF+p(d
z)`w>#-s`AcozN&cQ8d&o$FXa|PS%5u&#F%~@t)kwCZuw)HniBMcgkALNfVbOv?_bQ
zp5?Xb+SYY*3wsTdZbzrHT;DFrke$vwDfaoTZ3hpQ#$1~0@LS+|=!D2^{(mL3gKlR<
zoqeZV_3NSV(o-#>TW*votrcZ`y-U^YkcwLRPVTfx8k^U?u9Ha8`+es8*T~%}joVF5
z%TBVdf94f=sW<Zo=PaYfyoqaW@>XBDVrP|V$f@c*WueF%L1)op{(|9$B2@%FIw#sx
zT$63K-FNN6yGEN;0#E1tEK~RF`DDHLzx3S~9<h5KNwNRls?PeYBG-{k)8PDqkA+vO
z7R)-;XA^l!QghvEtv^~ZOra|+j;`E#O>pvz3yUr-_Nuaf)2{9s`e(ZOoTiA2yh8e~
zy&2-4GL-LBaQ-N?hFj1(V$NOOBW26Kc9t!z=AC_G;@(%1=S5DJ_j!FQ=wWG4K3e(y
z?n93kj`IUmq=gpO96fT*^K*~!%zGxw-50egmRH!V*!MUi@@<yeG0m+jx3*?w<|Rk3
zp0RCJf2Bv$+o~JQ&rG*V$jCN}z1?1W?ADhR><rGog&ve!uUPV?)qlp~b<dwodXe9G
zXo1>?)4IoJ>u3Cx;Fk|yzxnsmwY=OX=jfilc5-UU?;x-L)lXG7v|gLoWE%A*w&vw;
zucL?Wdw&x5tji3mXDDtdDW0U_e(NHero#qbX^{h`)lXCyd<$x5y*;^g@fsUVmFgb`
zlNV3!d8NjEWrfJ+6=K4glbrY_aB<4)-gsey;rR`%0xS-<Z?8#OHFePhzQ9PUs2}qs
zdJe8Qtf|3l)UF{K`cOf(#7u*kvp#{fpj*6WLWAz>UE-YD=MQuny<WBFh4>Q>>uTYi
zx5eh#M>^yeEz?-pV{W}jDXZtO!5*ba<`WiZE1ujvN#L+Wl%o5u9*<8wmmVy-*U8KE
zagi~nGT+Pvdk;)o`l9!E7gx{%6SY$&-+s;bdZ^>k%;gLL4X!*5EEz6U|GS$0E9r(i
zOWx?%dNFuy>}{9W*DjMD9J&73#p|}I*Kbpa!yMNRaX4<-;>UB;<W|$)ifxku9PfNq
ze#xM4Z*l7arhTOkwk?qAd*s}|yizaKd1|JjY08mVRTrAHn=K?QWwuOmIvKh7YNYwO
zN1G2FI3}-XZ=B4Z_shA@VM|d`zwS?=@6N(syBEBPI6lc>`CP#*413g>TP#?wsB1CK
zvn_Nub<4f(&T-d3hX$W1HY;~@DYuGr@0?z286ACeCNrndAG<?Zdv|=WX#a39Vxbx9
zu8t+Um6y$SocY6J<zmiN)$?R^PImdMm>1-@Alh@90)PC1RSq-NHE%TNF5q6zY<*c_
z_ksT>?8GM?JO5yzfhSLCwZsl#hSJweDYL96Y}nnG$kmm&+vtq($`BcM6}d?}giKX9
zr!Ta>@nX@HhQsS)+)KICPbPQS#~huf**ZVbV4{bMUFGRjDJ~Nek0`k~oKilbl;h3l
z(O@)T<HRi2xWgQGeoego`^XLB-Vh$CI}@3HKh&Hwp)Y#Rw%9w{LO;0XO84E{yzO!E
ztmj>`LX);dp0VvrYE65yJpR|xg+IHi6ggj?o>+XjTdwiM3B?`@>(#qv8Ef^peaJRo
ze>^>}Nw%*;^7~=^U%$+`7}ce{CVzjd|19PBVJG2B5B4)2=3tm}`e3!l#E<T*8j=eb
zPgEvN=?*^d!Gzo4|4Jp6)$Tf*SM6W1QQ!-=O!bLb-5bC4=uVt*KBoJ;%xpvN&1-L4
z^8L<Pdr)&p)X$FY9V=&_(q#F2Vur%fz=gY(U38rEr*pCPrd6Fr&cajLXE$+n?AstL
zWz@ZJ!A(0O4GDvjJS~dNORmf?l9-_v!((#gQ05_?n$_Amb50$4Va-@^agu@Dc7Zi(
zOnS|aEYg3p=w^b+(OW!sH@IBu*y6=JwJ+kd(;JS{yH20#67N<%v_MnRNWsy3gRXw6
z?bY4q;~twPF<*W#W36$kVX(}$9*;ABQi5+Mt-W4+xu<|x??D6Si_<r~&V>F6O0+&|
zn9F=uN%WqxUfyiS{MUOE?)V%KTsrSRm(<jdInRE`X?8HM%x=1TrsL`!^OK%)d{sN&
zezv=LMevn2=l0dw^S^jbkYEn`p>~tuT&Yp7xn^g*^SNg-r~3kWi)!2_{E*afb@{aB
zcxUi==Yv7EE7du6FHb(o;hcC{SK~||Z?91X^N${#zASB_2Thz0n)*LU+`HJ|E73P$
z&te7c1;K`*VS>BnO%|OObkL)fr%8*)OS0KHq>1-#zz3;LzOPyvR>{=g<gT||rm!%u
z&q7Ggqv!fd8H2#9X-YS~6y4Y()Twf5cjrk#+0@gOP6x_29+6kvazkK?{wuE^%r}iC
zS&a^aPPoxyZ*zKzX79rfS2MnH>qV@cVBmTp!ga#`g3D$WCnhDVJ(8m9sKd-RgDGq7
zTI-BvQQwVbCxWLJGw;b@wK_X-lZRchiQDl%K4r37S6@|*3{fqgyD8wZgysq1%Ah9A
z4m)4o&6Uag8Y1=w1VUxIBKtM<lV9!Iwe_qf7pLY9Sre%p;aWS4lUSRYw@eolj{dv7
z@D<0Dr;?_L&2wk^tt(KTC^4^H`apl62<wcdyxz3}M@)rd7XLreksa!KM1k2nqjghr
z&&6IIA%*7Kij2HZo6HuN7@Y{VUUApx#+^-z!!lTTCvR#syutMB<pOyV&I=DLFP@1i
zs|}yO>B0nutr{zAgf3n1H|@Q4q~i$V4RzVo?Y&Ot3wECgknsD@;niU|>El-JWf?to
zF838rnx{A2(CE5vI43^$Nw=8}XW^1_F}yc7+=-ufRA9Z&;y%S@UPf`(*|T*Wb*@^>
zz5L0`XobzqDa%=IG|a8sYjW1V??!N3@>-)AP5Yj3SkDc!UUAZh;Z~+;tf(@xNNX(L
z2JSm2gN-C^UEa!_<?Ca<fP3NGhAYw!c{P|#N^PY!Z&6UY8!gfL(@^T!Tb-R#=cL*k
zDZj~drG@9Msrd)t^$pH<RK+;kwr)=hif`oYz1TVT3ZwF)HT$1lxqqa0|Dgj(lQLB1
zaq>B4D}Q>!(JS2P6nSmpOW~xa9Cnu6tf%kXTx$7k!4{(%*DlN6H8O||Z@7E&|JKXA
zJ1*;b@QCHLzF+jfYz7mbfnn~`u-vnE7aV<}snEDTM_1@X@SIeB2AQ~?IomeGu6UZQ
zAgZ%#b*|a`WPaZbM%%9Js$P39PPcPym2bmYy9o#O>3-ump0M5cwrAr^#Zv}e!e=>_
z@NE8dwWqJ~)X%GdQl~EN(&l+Q(PW>~o`0=8#~w{6Gr6hwS7~L0yiC&ropYC0#M*|)
zaGKA!cBggjW`)h13XON3Hp%T{ty5!;evqy=<9Ur@hTe`=*J%xv_Zm1IE*@EM&1gac
zzn`+vga%#(=9Yh3CDfQ_pB21kDY-h;bK*D0RSTc?E2?a0<(b0g<Z&_lMX}%;=H8@l
zJ#U^)*#3X*GOagmM^7ZZp1SX~=v?=x{be!d4sN|_?Za!PwtR_R%!SU$Q5R-S(d{#5
zk}6~4Rd3^AozUR$EGG1fgI6AF^RjCeii}39J&iUr2VK}?v>-*guvO?oLimdZLJ=uG
z&zf?2Cwo6*n0zSKI`_q8T}Jsl);$W1lccg;#Zqp6TCw0}li>r=<x-r_d3RN~@-wSW
z{T0Ew*r;ogoYmvdq;4I~cPrT)4@S65Y?!&^-hzjU;y1<RG_?;f^kh!EY0Mq}Zug;6
zmfC(Z6h(R`8E_uHyUe^>HDa5$(xDeD?jn2k9@+H9r2CYRP668_%P05OsYI^7<8^Ga
zWXUCopc^wTMe<(!9B=l2#ib|DUiM_X2%V#AuJQVo?rzbk4_)UmWF6gj<iHcN9n4qm
zY8X{Kv0f3Yw}ZLvUX$~>^jT{*I=wqBB7IA2-!&tTCs!74>EXNFXK=Ld+okp=M=WkK
z>1G%yJY0LOn7?BO=Ld<_hco&*EOuq&^64J9@g%I_d2^Da<HA?D(eu~M+WTUiqNmr&
zj@>(y^EEfwW!d!UX2^f_c7MtvY_iMwfb5+c2Cs~N1@>PQnzxd*T42jbsS`HW)DvP2
z(o7xf&k1u@pRT*4mfYLb_C{%IU$#$K>&;V_zuaRm-OhUBt+%Av!C5-J0cEXbH<$wV
zeH2q`ytLu<#9vtxFFBld^*G;f{;`Iw-hYcnT_Rf_m2!$N;S8%#c*xNeYS0yGG2zi>
z&TfeXpA9q?7-uf%zJA_;t4fzsXja_*1Kna-Uq5XVUw3M{+MbE~G>qqONi#aa@tO7h
z!4HBOk{26{d74(PmU?8JtCp9m_bypqDMg%t;Y3IJaSj#+$xD@a6E}RzejjhelklH6
za`Ms8$#*BP%<H--VCz+U(?T%cp?Qt2#)0`RF9P1|GTYJWbg#*}<F!q0YtXa1*1F8|
zE1#I{2o@EN$}4TI?Ays|emBhF#1-d^kdR`R?FCO7uJvs*aXh=^JIAW83t9s;L>mwA
zy;|6p{JU@FroGD~nEiHjEYOf@)|lmZYGTxpTT{iNAOCN7AknyQ%h7#RC$DSzynTDu
zw`Jz$Ip0d<rk_o}Z@%Am@l->1(|K3l&AY1nTqx+kBcm{#{WtXaGUZpu9l0cvzkYtq
zqy-N?|CDw4`D@SJPZbwm%RV!;$$heITeH)%u!U<2`2wDZo(((nO=E+fM~wFg`@N?7
zL*@kQ<+O(!4ZFZ_`}cx40oF9j&d`QwkA$1oM9f+@QzRy-WBQ9urn#CkFU;?2pVx9?
z-r09KPb6yVSF~<R5UFOF8FK%^p^1g6K8KF3l!<%y^!$U(^Yude1VSf9EW6&v^=)m0
zu1#95n37HRqK9c^KSViIE->|p&id0Dsvz4rxpvMG!^M{}m<tkhT3+ur`5#n!R8z5j
zozk}%OxJI1jaPmcox%LwFW)xyL-W~K({RI%`VW(hqQd@M4z_!GWJ_d2fvUp+*(RnO
zPMH+~3!PfG3z>;97%rOI929Z2+hW5#Eng<Ludm&`g;&?@b?Nom>32ZO@XeoAk*%xl
zc`|<ab5Yv0DB)zC#qowW%L_9O_qg;c`OeySZ-$D{B(4dY98XPa3>Jy&jVOI^ts{n|
zrd6vf=hFH#(MgW04Rmg9%e$YI!X2u|z~~{OCOE--{S>pcw<ct%Ht?Qr%<&ME^R_U&
zvqeglzmFsGp~i}}9<CxW>!&_wTdQJWyUO7}#$H};?g}Zx&L$P<H&X+ReywD>$^4n`
zzsuUdgKXhtYJ56}m-ZeAI#TbwB{_S+r~9YGwOTl89+d33w%$EPFX6OC{6CX#Sv&K3
z8lt`&VUbl{CAQn}b;QXpm2o1gKYpz4l`6ix+@JBs@hhL_26*slgf+TwmL|B|=lwEQ
ze{sXI+W|rzh6hE&luQ|8q&1E-GRU3!8rH#EXmlk~xr#N+IhJF>v~I1sr%LLnTW$z+
zWvvwrRM+A<kQB)@CE`(^3R7gtgk)7mw}crxk9r09Ww9(}2nzBPo3hxaO*PP`tnHfG
zoJ{qGgsmMbTUa;~+NUrD-I*jc>xS=xI|1c8f+o0>xP=P_Rmr8QyE=cIxom-ZXvt|Y
z1|x}!9$Wrj*nG;t=;CGnI{l50G^+kxIozeTI)qJC_p8pT2U8Y*QOa_0<K>#V^4kWX
z)-30jZla-)iK5q|w#Xg5)_d)wt6Rh&%`Kwhu`@kbqwTmHl2@HJ>Yl#Y;eo1Sg5Rb|
z5nEebXNF00Sz69?>gGLq!{Oec(3ugp{f<RiGp&&p%vi7W;aTY?E`vtx+!M=MB0lZ=
zd4**y%c=>|S6t0ExSY{!x#N)778&cDD8y=5l5EK{{YZvqV7L9=jFNj=_an8qbpK{V
zP4Z37V0ut*@-yVT)aoxz{MzO}#RENL%(E9bEpxllC$=V2MLLgZMx^38nXDxb9lYeP
zhK9#0>N;GJ<8%2Me&|1!bW$gC?%6e)Ot-ZvZQ7;PvZjA-f#9`{*y)RMG`UKPwsko>
ziLz?P^6k<MYVq8;eMY_TRo!V*XL=t~n=r8{GJ$E!#?Hj}N466Z)_OSRpYtp*JUTPe
zt6)xaZkEWh`3u~iq&suTuL(Tr5qkYsz~xp+4bA8clQNwDb|^VC1ZO$Tw|~>~X=A{f
zf?ND6m=35k1Tenqdl_i+yk@TbzMPFapGti+4Gq1`uj0Bk>8;g^@Q|GFx`@o$+H2h=
zb$9DF#ni~|{nLDJt9W=fGh0MbN3FGl_H8-0m^qz2ZW74{7BdvoHOxG7_l@GR6MMbm
zxhkh-vMH@!xHQd4_{wyaQsskf0sjrmg)~@SwLM}B-@%xWpUKL3B4NYNgA;VNWgK?>
z!#J5mvtRRdWZY7>w8>Fl+$)#<;`3oyGQmakM2K5U*V{6)DJp79J5PM>(~mkbi_h#y
zv&<5KwrQeU+XCJ?%YO0-^<!DQq~xsI&aM;JP8{eIXI<`Uyh^b5!H#KLR(80?9$0*{
zD#XlQDO;i9%VGwrg*whzPdwb4x>&XqNHQ8|wh6wJ-oC~}K$r7Cp_JyFomRm!k2Yp%
zOz~!5Q|**}B(s!9<pvYG+5twR33}5wvzh0gGH6^eQ%{qz+FLL9s(1UAChL>Z9*r*o
z`Tv=4%w%km`(n08AViRL*0L5Z&2u7$?%rv0`X9Z3E2!Y>-q?w}dj6MB?WikNKH+ez
z_Hs;m(4+|r8Bdq$ZC7-*W(Yo@A-GI*jYs#6wn+yHd0Ry8DfMjsqNe60v_bJ%6W1=A
z%Uxwx7RA1K<dxBOX#J9yD^2Gvb?i}jvhL!9#ObFq53_9xU=#CeWGNMDkUsTbdgI?^
zEHMg<;uZmH<`WW`gcTaOg3G5|b!Az>E4YeBxlE(_hf|2NO@L_UN`X1D9O>taxxdW1
zHQ}Y{ij^)lj{+uS@kd`#UbMzzj)VC|#s!sM%mg)~#90frB&a^^=9;*=&F09K6^g!k
zS=^5UF0ByQuhy+|ib<#cQ2E>q1xXVMw3{b{nM~NxsyZQH;{QV$YSWjUP&~RI)@ViA
zQ6<&Gt}Na&A~hb^%{(9GvBr4nWew~5lNgNFlqj(7boEyK;58?&fx9G&S-aX=;NCRF
z9a2Ik#LNQP+D}Z=;8eW(@|IIbQ{a<UPtM@kClp!})t1gYdud(iv_;l;zc$HkO<>h_
zJJ{vBAZK<|N0Z2$Ni7;G2WDLt?he@#ZnrMx=xk@ju)bMS?;qD#($|<8pYZrbR9V}5
zy$hF4#Qsu}P}e*c#qdO4<>#~uOm3U%zvpo5RJ!bMz3r*dy{Z*Yc8T~bubE)N70A$?
zA3TeHTF_eUX&*CeW<C0Ar>>OypvkW?%bQPNI`i6$%slpMZ4W-pSh`N(pveDI8)U9K
zem~&6bB6zNA1+;!^PS;aXMWoC=F08NNFCoRt9U1-KVvgp(zR~|dxY{Tf1@Yq!qFEF
znQ5k^8*e<ecdepSmW@>MqP0`I&mMYRSA67pNaox1FJ!#hS#QRRE>(3aYVon)Jh|BY
z_{5DYX~_yN)x=aTyqjp~@?wfs=h`6m%t;eiX1kf3lxUW7D_X-bIbm9Gx9*wnQ%Q`U
z^rjzcG~3e29KC|0J#XV2&76ej;2&Mm>n^of7btGud+o%Q%U@1ssejka?Qqgv^3tcb
z!2C>t9QT)38=rl=)+F-O^p3;}nbxn{%<sKp^<{DTti=_jw|J)~SC(2p{H6<DT;UJ(
z19`Oni#2^WuM-Tka4<bo5bOS4FDte{&2q6h*LGH~Ur#nfne{32Z{DT&GsV<?{pElL
zyDMTsZ+Rpew&k}zv1&8VJ0a$G<MpIn2gT05>y2*oQJ*v2$vIEsDBH3NZIYWVwXSKl
z*c`QxHAtmYAxQPqmG-32wMC_?JH4jt+|PBftS8hndBSJ4B?)Dk9t$$wpDJ$owCE_;
zKGn4;fld2oiV6Phb6vOXlC}2fqykg3JG$L`HbHwz*SLF&eQ>|I`0&X|DjdeL*8KNU
zoQ-5I=&AZCt~h(f(QG1v$<aA4S9m&X;y$$8>?Y&Q)E&;+W(JMI&!#N1oZ2D1?%C=G
z+g2`B*Xek<<Z#Ke|G(^%PS|>sZE4W>DSBeJcjEbq)7D&HE=6*ENXr+x#Vz$ZeE%Zh
zSK?)QGt^Xl9^Mby{XSrsQ02NM9!84x{{w{P8?$JzFbat?*eq?0)acZnRIALBuAKSv
z;Gg*?Cbl|89zU5GXMKYy{ekak#weF5$L42F%vy4TIWCxO>tqHMhVHeV=LA{VCFY(J
z4cETW87Xso^36H<DlKzVjuxjLKVrgt*x`85i3dezW|s4`UYmLIi$LH^&W2fvqO-Ri
zcHS$g*>iW!RmocmWxLkOW-#39eJVLW_d=e{!Y<p^ES}!mYp=RxCOZGQ>98Tx`N7=i
z7k6W(^0$AuD)wfoq+*mahs09HS^pPv3PtZZAu})V)|`$=o=cGzIBsq@-|>SZQMFa1
zW7fKfw@gYqCvmt*pNVpr;JGVu@|3TVnJf;yzKc}!&K92K?A_w@E;otmnrDtC%UlET
z1rr|4i-^`(^vK~v(Ci6*Zy0<uw_NF)${6~lZ~EL*#xKw9-01XKmG8^S>0dUAe@(nE
zbSs+eu5`!?=@$oOtWVAr|26w^MoV3)tArW1iPf#JA6}1lDwdf@#e5MwlIxg|FpoiK
zfs=s3lqrw%Sr<H8JO4y1xAYg^lZ`iaPM9-8AwhuYT*8jv=!*NNU0v9A2|Lu?a}ZKt
z+vk~mrY%RKfkk3&{+Ui)4W9!aPP%MsE&C$1E%x|-eb%cEa{{C#+;2~4TwjrJ@v7`C
z(cWVkE#alJXRnkNoOD<JR;19fySrCR75jSg%)z^&zZT!wIj2k3Rb0q5%JgyIh1k{z
z*R~0*cWrNNQ1HDUd+Tb#tw4rFYZ)XW{@h)(GWb`lRA5bVeJQWHL0q(nP@3z!DHERN
zue{}6b0p<YynX7uDGS70rXF_+nQS!gSj3eXU78LIDhE_PEsfVpuHk&99QH6)>j2A|
zd5#$^^D~~z@tA&K4_~BDyUxl-MS@%hwm9lcYAl$*s2$tE;&UKT?@7X>Wu~F`)uxDR
zE|{*iBHZBA?XNq<zX|%X>M3U@CgdJ+HF~6W<YvT=h4=ICT^I3L{6AVy{io=iFfOj|
zS2|=lXB;Y3=Ghrva?U?ir86OAMnVMljtun^vKR70Pg`*<C>0Qi;W*Y&8>{=kS9-_f
z+DH*6p(oXROPF6h6+OFTX3j;HM1egj+*2K05ARq|<ofJTM=Oue{X-3WN0v-*pXTUs
zEu=8^MC*(tEILb^uDRtdk<IdW;FA`dwWF){oQl{j{=ka!9d4{4FWB;29x*>-i<;;*
z?`>rFtTw+>Goo+!_wup&z3RTbH1=-e!aoHZ?FuqKTRkRhn)st6dGShL*<%myA5467
z_G0*pi}I|GCT*A>Jt4JCS0rsm)U-bCe>wLKZ)s9KCccJYng6bnIXy3x`B-w59{<nt
zV-TO<mpn^pt13_Ahm$Wj7_^^he!O{b!p<vcMULrxDj7}BoTOH@L`}}i>9a4oZ=d&Y
zR@4I)(<3#3SqV|Emb{8mNt$fWrW)z7VE<GNOVQ^MAyP8y6c=~8h6x>gAbKOPBx0#y
z<}8JSvn?WA5@PIqIoE~FTWQ*2`|^g}(`4s|ypshMI2=4`f0bj#1Lb=md|kV|_r<KJ
z-E^Z>Pi6TtwV)q!ie1@*{s<*owKQ^Ra4ihb)>%=&A?+E)am)9`7Of|(RSVfQS^Uy2
z1{{$PUUuxv+!tN1R_}TCx+m#XkEbGARu<bKF87rh0bOk3MOuAqvjY28iMz72zd5x~
zRI1-6^}qEs<uV(YHEvI%OQh~7xC*sARtVFy-o`53!<4pzBXez3Z_TwwEN(L<c&>Fl
zI%ArSS?{q8jP2@q+KxW%>wA-4Y!K1;w{oY_ZQV|_MRKn$IjxG033+i>EzsfZ{!h2}
zJxE(R?M=Y0*51ZdwI{EJYCJtGw_Ix7LP^%F1$+^!h3*B|%qiV)!bYj}_gij(1M{5@
z1Rwp&s;RNaN#WRCfeTw#L`8SGuHjv0x#@95mh=XxHrrJ)X`y}J69nVZF6QiralOdh
z_RcNtS4#er3nhZ`jkgZzmp*YXO7nGm;ggl{YT3G$eG#?GV)uwWn>EXGiIOYF#+6;m
z+V(zd6J6)lla+Vq-qQbZ58ko<(%zf(a6WJE9;y6~L2Jbi&HbD<aluaw$x~N1Oz984
zCUUm(X!yE|#|!63>q`ANwlH+ssz-MN`yM3zWSbFjgSX<I&iY@yQE@`2q}n$A(o%9`
zn<>D$&NJhn(cGs;JYQ%%3+zdnzsoUADN=HQqPAdGMqAn<rw<GFb>5xomZ8_{6!76v
z#@bxJw|P+_ZWCMgx%LPAT0fI*eMH8pnyA$gAC3m*2?r_k)=j-AGG$`eu2$0~SF=q~
zxAP7b>&e`nbkr;^>NeBHM%ANhiXL9J6}tCA>;9j9`^=W#RWkQP7P%>C-%(@Tab^9c
zsIUzs1+&hCM``Bo?Raolgva}@Yr({W|93jA*sqwiJ4<0}*a8;4H-3CiU$^PxDdk+T
z%jdAy`gWq<U2x6iZ-$@#J-KstMTCg&iZ<QIo`;1^7bP-TuBu!MkI@O)@G5@gWf@kv
z)vNMdPQ3kPB__DLTVdDx-(q^28*&`B=7@ZL>$s)+pOE~zg_|<=gj{6PlDZevs`55N
zOLx(wcU<yEzv{eN^yy<n!rM(X9}Vj_WdAJWS^9x(`UlQt0R{2Yunp<vqIbm<q<Lqb
zkX*|7z2`(E%e+H^5}LDvG`1i8)4D=A_l~gq(WzlK)&-s1HX&7b{bZ$C%hzVItULS8
z@6nRHDQzbHpG}r)Z(XG)b9gnU`$F-sL<jC-lgX{uyg2^P-Y4@(ZcW^Wb=`^m*IubF
zXwB&9D)DeuFuuJaYGG&3H@#)gT)mEG%kPhsyUg9{(t7vr_4=hJ`?)k%PCBaeS2Xa-
zMVW>ET`sed)~;H+z~hI@OLKRZyFu3?uYHQJ@RUvna+Wza&4DfPM3M6ZPRS>;*^5JX
zo=PR^ecm6nWwwcKl(K8&Y6E%o;*jcBF4GMh&AVN=KSv7pwgn`IGrZhXS^P{uGjtzo
zeedL3i*hZ+H?B5w3tF-HmfWX5o{W+a>d!y6FebK5S@ZbbyR74xavRshPZxNne7ASw
zuL8!u{F6V#$0SXAo*K}-w$wOZ_W)P1SlH`76ATt#GyG!qp!ChhD?wZSv+}0eulrJ9
z{KcU8>!)W=Hoe(+D60LIWbdU4Vbe*9K_RV2lwvwAc!h=(P2`xjdf!E9uURX(3foP~
z)>zH@eK4M>=Z)rJ(?Ef?Q%z^J%Ei)r%9yIyqIP_FA^ZJjXF>FBRpb3;?D4ue+qeJO
z@YU+>wzyd@^Ub~EWR~6Fb2Z-`dU58fz$Olh<aUeu*B5z8buB24p0aI9Q&H0MLKhc?
zCog=TILtfQ#hDairjn?xt$ps%qSRjLbtiVNS9--Cb71kdjC;@4mNES)jz0fwnc3QF
zehC*mI@qidn&z{`O>T4Fxh-*9Dc5|1*!SO_{8~Mg+354PLdVN1Hm@*Ikp8qsY`x#E
zE&rohOMFkMA7(2Nl}f$w$fWFr$x){CZ%I26>W!`|uv$!f|3j%%ZrS`mei!+t4|_w>
zziim9axp(OBS`I{1%uk7`!8Nq^m5O7Q66zSZQ0@tG1<Eov+am(-jM6{wy?NzqPRub
zoT|dj^JC+WPkFsGTRtt&t)+Ia!8(W3kc7>P3w2_?cqJCcJl?3ds_fJ8jjPnVEkbt8
z@3|%a_`zhiKdeD1+cwXZiu|>zGn7+1{3FAwwU$hkc1}W?JGGnGjb?1VEYWP0-(HY;
z=K6fu9|ij5n-^5C*u5)%w-wv#E4o_?58Rzw>f&3v`otBV@A*;hV?^ZoTKeNO<JQ}!
zZoZNml5=O#@?ZaZ^_PDSyOc6Lz48I?v!y?C_my7rt2xnav)N+j&fv<pW8RmpbTd^f
zU|-hj!|8q|{~*hSyS=HOS$B(G{&72$L9=c4_HT>gdE5PqSKW`e@bI8`^}_BS*Gp2v
zvyL*{HDBdfo7!xB>}`dW8H>&T`y0IaEYl(wKK>}rDx7g{>Dq0_+?TUWQ{S}PpoHUM
z>B;q?cQ*g(y=VR5)boI>_z#C>OkV%<qmp^uF%8-C88f?ovfsavA@k4s_ha^@FM9s!
zv0dlRcvhWP=-^xE$XVjZIi>Mg@xN!)`@($~GOnHe^J0r%cIyU4t0JMJs?mF-XReYh
zdvoJf!Ngk~3qGGXyu8i2e@_VivHuSWWYWF__xM*$H+dAZttoiL-dO?_SKF*iPn+2v
zDyyITtM2{3JJYL{OaJ6_-uWrro>O`i^X^L`A$DgD^Z&VXCGB2F!QGH&(;GQ+_A$(G
zc&6>bd0;=o2@&=e-L-G4jzk?*W<Daq$h3pwhLJ#`Q!BTyoJ_<7#e<Eh3VbTfA08bt
z;xXoW5mET~P#>qgo6C)(PaIUZmQKi=8pNQ<xF$_$ie>}vnOQvXx6Wi1J~-R5rsa@~
zWr2uKSZ-ci&gD~Kd#Bs7-cV9i>|{~3+G#Y$Aoa8uCzD8tsr808hqj~(t0nB%y88CI
zY~f3;8sQ9alQTc6J)C{*{Q;JS_kCxpr*BAT@%%5QZgQq#VO&f0+eLG0B6TCr%o6nz
zFkt*5!X<1`pc%V}C+lje&yupan^+Frh}HB9cr<hMfySVcynq+gitX)(1g)MhZB&Zx
zPt!JQpZUkq=>`8%ZU=>gr(acqJOV@i$<|bKFFv6n8(Ly{cDhmgr8QSKPh(iMRwyC0
zBYe}%4Xh_SME}blU|^M*;4t}<b^ycS6JD!zPFVyyFmZ@}$YJ6%Q^@Ul?De2&k<l&<
z<|9UD5`0~R&I%l05ea;=sO3gLQD6ffi-YG;33j7}Eh2?`CT`_lej$r%2g`%p)k?mz
zl9(G0oC;X<@<_3V(kq2m8H>A}4sk5hK4|S_DV3hWp?X4K(*H$M!{!BLtZcmG;$hpG
z;<Z^ctSC!aW7&(GX0@QYpvmXL@)i{@u>0(zvE1ovOyAnC4M~fqM&0VnS6m}v;>o%D
z)T}Q?tP|FJS>V+g<L%L^<z2AQOH!&d<ms`Z0ES@Q>Z#gcmcqu0Q3t)fPxwcNp3sdx
z&-FFroJn?p@h%M(-blVFPZu%kg;jquW!ZVFBZ_f_ZC^L1#AGMOJ4%;)847lO>WyJ+
zxZPx-zVdRX@MMdZFIF@?$qiC}E>t9R{}qSVy7jud7JFJYEaeq=u*AlwKfvXw@B;sq
zQ&aeZ8lq2Xh0gdOb#?8pX~7R1diFb)Zr=6I`1OHF>~@ll6XHW1u5qf)`Jdp#W9d-d
z#b?PQ=wj|<(s=Yrgv-U7qTUk<7s#aA++taj>teV>c4cYfOG$T?jKjQ^3cM!MRV8#9
z7$#Y0c;pKU?Y6jgz{|tDbSm@poy|KG4&Tc%HCnAQG2v6t+tS4%4Pl&SotKx*y7$NP
zL2KTv&wi%6PjzaTF{eu`U%>XMPegCGlu1#?(x@AWp0TZ8*90zoki^yJ>?T<<rBODc
z)Ju0?_U3~k(jGp4%(7$4Ht{4!>auC`eY=|MD7|*e<(TF&)9k?T7pJmLYaD$x)ARo+
zGuA&|N{tMg*c*PoV}4QRao!-c!+cA%GlxazIn_Kx4eqRRg&(C&;>#xVT`teB_C57K
zTJiZhXAh=81=eTN!W~)|UL`Om9<Kb-GOzU_OJnM$*25OZb&sX=gwGV3=I3@?mqSvZ
zMNvfYRL9qu=Y1YH$;>#EX>r9#pf|&1ZQ#9+5h{!IKJ=tJ|Gt!yWU@u%u7?>%p%fph
z!B+l_D_XVoNS&E+!jv_`znbC4JrTwU4O&bJMjSJpejd8Et7S^Q$3w46j@t^%GAB-q
zyH+*((3T||ekQ(DDlAcUUsXJ@Yrzq3Vb4yn2*KqX-zFA|_;@_pp)|#u^;BZ}El%lw
zEISWfbn%;by7Ot!p%pVuTvTOR!M?<5^3(}`^^8j;Wp6ZmcJ%stuG{M1)DuxxH{IL2
zNpF7g(iJ8r{-4^);-)z7&efMe?%t;@R<s;`beGdH*yggq>hud)zMp&!Im>aKsJy1c
ze5Y^nlJy=ATwDQ8OdHzwuJaU}bNBN+h7BAH3LFd^|D{h@DhstXcl3w2b6vN&8MLm6
z<GIBf;}iSdnJy@Q=wJV1b+{C#mHrF`N2cf%t<19KhZrxWG;-e*eq>?ZHT?^x9oNcM
zks|ipHxC|3-aa#+)}(BP&bLhs>{i`gvb!!#{5h%I$7Z2W_zNBJwhR}2pJ(TbW+;mt
zb#prxbHcdOY__zUaDV9Lmsh9lV3^_hqBD?nx#&u(t3~rnn&o{vtkw%NYC3p&dR;i>
zuxIO0-5GD!1V>D|eP)OM-v1j}y|`ZX%PC)S+jUB`e#(-?_n5dYbWB(%TbHGwEU$iI
zu1&xq9#d|n(od65ot>!7GABm&O`zKQ8LRD9vRp{enyM9WkIPtAR>0diazWy;ETc72
zX)Pyatw}Cxa6PFaxJBucbF0l~`?J*?u3m>*bp!NgUq2nk)a2m$uGW9fU1lfCo1Nn8
z_FY->T84vZ!wLb0lveWu7KPU$My>1TsMfN7?9D&FRABqguXUUg=Lar29NySepvo!5
zs-ApE;OI6HHNHKZhg8Dv&)9H$gOJIa2|He1e6g%$i*(Do143&iv(=x`Szt2r?Ve9F
zIXxd}``l%^V|HcX+|Czm8j6B^`4+Pmo^AV|u`Tx2mNllP*@}hEvJ04kH?wVC;dZer
z<j*|OlQ)wE<UHo;bgmYgI^pcb2Wq<$P43&Z-#O1H8PXRTBo`o*6yEys>$;FAKlcwB
z6K&q*Dh6$M5~NUh({UC{0sEAbvt)NKpYedR(Z})0Y{@bWhnJa}?YD%dDNj<bn3_46
z=cZFw*zyG*=f2z0!FEsQb<j)~eW?_k%~P7pHMCn=x*VtXam3b`uVLNz>ckP1vxdFx
zdme9OYuUB*TKk5jPT57PgYL?_GoL8!SNJPp*UoM4oK~%V9p|mh6lL++sm3^LhmQl-
zebMWYEx~#<W=Ga0JnQ;%?ezISzUuL@jehaUO``RYM}2?%mzAiTu;_Z5ubyInAVY2W
z=LgLFy3Tt%3!3bACol)k(EgETqVKp$X2!&;Eo*%*o#}a*ICrmsOWun}zR)L!r8pP3
zI_`Pv(3hnv8McR8?^{5RZ?mb|CLM;LwXgct{@UCBYPRC5H>K<6&lZ|`Us}yEE`2h)
zlEB7IdEL=G*EfhO>3!cDZqGD#wUqiMH!bhzeAS1WtM8whYU7r`y*8tL4(l~Jlb_{_
znv*+bdpdfwYkDcRe13mbGh#)<&y$)jwz(^GKc1G;=GVRNZq#1!BNLo2&%Lzi$@lng
ze5a?Z$eEGtc<q6EY}QS^$u7Zh9bKybp1!x>kACv?UH_3?XITWwyBsan{kNE|?OOJ4
z)?Ch?rcQ450`ETV`l7A=_tS<Sp`Tx}%}#bQ+q9MA-hF`-?w8)-|Gz9f{~=}e_p)TZ
zaHR=TRZ>3fYZKmH^diK<MN_DVVczE_YBSz7<}5s*;aU*4tt^8h`+sTi#_#&Jz28s7
za48;7desws<Dt=u92Y%}Ku(uAO7hbx+^p6-oIYjQmMJQ7N36~)Ta(thDeavTAEW!b
zE{}7^4Nfd&S@M+YNJ@pS;}-28r_}BY{@ha;id>Hr=B;(8%R4=Vee$$sdzC|dJ|9lG
z8JzymWfhPV$Hl}p(T{1@V)ZJf3CsH5NW4~C=A`)Mi1j&M#XCA09RZDP0d_3UHVU<B
z>itie#Nf-o<h@RUS?}7R-!gA6RxGbE%6;E(G-S)6OJ3cVUin4HtmU!i)vaj19K{`(
zb!vvsTGNyhr`TAwKTC`E(EYxL``UpIpY-^<N_2Lf=~Q&`|B}!o;QL5GdC`}GwF2%h
zviY32yOQVYTxcmszIo=5`7GV~3nv!*TP$opaY2w%?l#|B9v2OlEa+!B{c~EsgBllS
zev?Rwk~%}Wu!z&$M~lT6j)`sXU;QUHWZR)XKXe`Mywy3u_CU$C%x1BUMv8)gPg>3?
z=S3g?mM!~NaZ2i|{v$S~%q@nBbs7J^cpT|sJ2ofvn9aFVi)Y6~9(1kXI=*fNBkvl)
z^4whlN=%}L&ldd878L)elalw_@7z}wU#=+*=6TKvyiGzcJ4<eG-I~RxmhtYF&LPDu
zD(e;~v(C<YF-@=jkCxIW59S;Jf2mU&SlA|UWvhs~7b<jaa4R}{Aw|sixQfJS)$>em
z71E2$UT(V8$iwzn;pU?)8i#-X%DWuE{YPu<<r4)TG!l+YW8D_U{BhPLomT~q_ARN~
zl(O9;_tS*4Rcq3Bbe;aZL!Hrzf9?);Mj^quN0jCFD>MC^DEgVVA#0+!(nrOes%KU_
z|GrAO_FB7@Oc2A0TwP`r?JWv^EUWdEm45D9tjV`xA49sxHx<8C{)#z`+!kpy9f5{6
zYDy}H8Gq|aBsNKQb2^)}I2-+6=Mtsd=C-*bOeHfd#XUgjXimn_R{`xB?^KeS|E+3c
zwpze2{e_J12HD~q*}E4P&gqt0>#TT%^M21{3w>cZULW1>hEI1eG44}S;!CpbP^r-n
zw3zaQ`Qmb`AMLtH0xyFMSf3v)>RZI(nUwF<WW&vM=GX!sm1Buvf$C>^)Lsh*cwJ&Y
z?_<zkp!OlfPGLFc`8-bBMLxF_xm*wF_o!^W^g&ldWk%i0XH_fJBikLz(#ub+j}S^?
zbm^_=TWRLcYbv0xlp*{`Wu3zt_4zv<Ts>fLEv@&-vK61tc<xLo*(sznw{7ySly~(U
zg54U%a=Vqja1^?S=Nn8_X57c+rDP~%{y#COZ|$=eo|f-|7;dn$oIGLeI??v)vRhUY
z6o0tazj2-WPmNu8kwJHeWs;!xKW#hK2R4_^8VVoM+V<@3qnA$_E=}**DBdn8Is4<q
zKEdpe72Gi|I!d@)3Rbu$EnZ&qB135%vxMkTfkn)FE=r5)t7KhXmeO;~?%lC8U&hs1
z$zLu$Z4-8k=<}F7S<v;X-1Ci=fuXvNrmE*AF}=R>h3AsWlTDJ{Z{*|OE0!pGE)#aR
zwd}>aQ){)1d_QXSJ$j|Fgva*tF^gj=lf6D|2yzO1_E<Sy$dH@cw&_FE{&&hxS$?HV
z5}k7)F=x`<69@FSWE(TGa7%_Tm`&mJZ2mjzsk8GHQ<wiC9TH6+yPj!EI&J4@+Wtx4
z<_1mejn6;r5c<6C#f}4;++I%%4cw|buUPJ>V0Tn#RN0i-N(x*?VO%PX+>0JqCrs6j
z36*pI*8PY}Pko{)gWt^q%C%fcdy3Y-bO}(hP|R=0*L~V#edDZ;{3^S@-8@p9cPkDV
zzI?6q>aim~^ZQGtf2VD8GI}Fvy(XP&s?<w=_ZSt^@9#5`R7^L2d+5^?Ws=8`znQ`Q
z+%bU<oi<<1SHB7=|7QQ8Da5%{ZfSzB$H7JaRK<@oFia4(S(hBD?4<i&EJ^!<N=08;
zlq&P{lS0Q&#C+?lJ+RqflC$m0qlMj5>!v(r@xO2~qKTL1i^}3sJIVjYe4c!cl2=pv
zvts|B&HMkHw*Q#(>qX0keaUL-EBkL96k9T(_*TN~RU4-Hy>@E!y!7hbMYEC*77c&>
z9)EDti4tmZX+Pz%Fi3q-r_v|Rgv8v0_H4$3nhXsJ)!W6p&uPxh`=WieGru!fAU`Dk
z<;CjXVY{5xe`#8?OX8co!!$Y7M$sD!>sT9P1(HPe6*;(wrS4lMmU%90e~0ArmqNRl
zyi7J)S{N3$^NT1ad8dadD(w*7`6%1NF<soL#PQOEDN6eVSZrr7iLLIl%b(cvXv-2N
zzt-m^In%;Y|BJ*$ay%?b%i=E$+-#QJm~}Mh(P=X!ndI)r1;OSsPvk7@ki8g|`}hC#
zhAZLf54KxKZ_`#P5G<bN8<to!>48<v6}j}QI#ZRy&t_l9>bub7;yEuwsC?_o9lVYA
zPb<DUrhH4HPjS<Hy&`T6qaU|+Xe%WM&r`Dh_?eBdW#N{-w-QOdZB2`W+$`e`Y09k@
zJ@J)&Gp9u2Dvd<9irCdA(>_(a6g73*^07the~R#w|M_h!4J=W)pSCpd@66OLKa(I@
zCiq`{-;N!}XWU4X72`Z+l*Alfdo{=+vofE#+|Wa@-nB?!%L2F5nYp@ZI&zz4%-$F_
zCFk3;HQG_ztX7!l%QIQtdtRrsqfvMJ-hIl9N(U5vWvEW|cVN2NpwQ)$%hY63{w5+Z
zLwJ#h+J6Oxh}FiI_}jdjl@?rcc5C@(;}ZB;KPYg?il828#dH7XTwlO`Xnv&bu_>uJ
z<;B6h7P*bZxsz{B_f=Xv{dlLbn{Z8mO0=VJ#hC;5#H^05cUYBiLt`S(k^@WU{#<%k
z^=8bD9@USHtSr6X*uC0xmM>-LXZP|7DCpaAB;RBE!UBT{bMl+7K4yDqd`_Z?U6MQD
zUGc=Ot=b}zMS4!JvdR-z96Hf<-8uF26%l<!8Go(0?<&(fu7rHivTaR{`7*mkc<v_g
zxj(L3U7BuDY_dK#Wc}=juRo+XR1ViqRAA(=o+imubK&JvudN}aCi&-^bpHQZbXL3m
ztv2Tri<!3{b+?~kI{jaL)<P4T6+drYdRnJt;n1*N_@~i|aOoEkkAEx()iASAog$zz
z-Cyjw<u3*Gvs~_X*W@?Voqk~VB%uBFKc%gK{nbZzhy{HORowNa-Mo^)*E#ac#vFC4
z<5ipI7=508R)k}b1J^MJ;RDq?M%Q$%h(9g5nZK|*DY;Z}k%GZ)Da}O-cHOzA^F+L5
ziGq;zZ>3Z1<|_ismpH_*9Lf=yULq#+a?9JE?`cX0UCrC_BGz6}S;G>jwW!ZoaFfho
zHqK+s{FxeD+?=fGicfkHQ>-UWZ`MqkSax;|?}R0B!HWXhSMi<QRq4d!vTD=z8&(O?
zA=Q%>WN&JpTJDo*?B;Pj(>;FYzS94Zi-S#{HciRjk|2Cw_Y6}<2D67LmQ|Jy&1`y#
z8@A2inK$RuL<fPJzuhIJ6;Bu`DM`<~zg>WFM;LE!$Tp2D3qoQpvBw;Gz4ub;_X*0X
zPo8^8ezHEytUaOsjghs@3l`fVCk?%9l?RpeY78zbzO^-K?415PZhwoL*}A_oTA5~?
zTB$P2eBuxDlXEWkvoTI!*t69^=*ARLH_(xydK)cH*Yhl`-no0PHRq&;4b0a0&#eno
z4|lRYIk@S@*Nnb`?MD?EYUaJUqj=6u|L+|gz8{_k+fFW8+jsGV*TWw+mFKt5an3bb
zw4y#>rC_&z+U>-IPlbE<CVn#J)bE<{G-cK2oJm6emx^A>6I<9+yz-D}gLVAPq^lLL
zXM~wt+j41k&7Z2npQ=yaS?8NK?R!;?xRS1<-SsDj&Rmw>+r7E=vZuVdnc{>4CqwVn
zeZF4T?jpR6;Y@bq75{y^)_GZ+mPrlKb4wQ4d|3I}`5FQCi}?bJ#6**VB$7S(t~W3)
zNxj%Q^Zkbd^`dGT5w~|Uady3r<lVf@ancda;)I?L>`xb~TQ!MF*eY#eT~HelRQtnw
z(QIb3S|+!{#~jY>(w=o(rTU8ViRq0Fd0QQXUriUg=2)1KDEuhythQa(`a2FYgg1Js
zeTekxcAxfk>Yv1)b&8V`cvkJPIL5I5w7a%Nfw@@G&0Ag1e#so1`rrB1(pZsxQH>c_
zRrbZ+WtX*Ca8>1ogKA{a*MohpBs0&SVOkv<ZWk1jUe8^%Os8|h(ODY)+P^mJ*s^@8
zuV3|U@n(mzz3s(c?kFvGfB#T+o~~RS)8<!q86NCbW~_fYSu`kCxZr0>hhTww>CZV4
zDc)(qGN1nZyqz4I&&(jxlV#<_==Fk?clOM4X{MJNUNwJ|R(<6tqrk!;;*hoAU^6?n
zn8*%+g@@Y(1UE@VOyFJF#lpWW@kE+}bl6t5HXh9hJxqE`7CsLR5}TN&3o&U-kW_H=
zW#BA$b!MjVqVw||nz>}XwqykA&RWv5YOCJ?Ek<9~mMcoBJOV6k^=c<hi9WpczaxS-
z@2^(F8-E7&vvNGMuc=?#E;u>orqQzpVvH#(xOQl}z7t^#;`t|Pk@$WJZ}7W&qB|^}
zI!=lG&X;p1;=%ph&8#tX9&QTL&&?Hd5;KVUa`1wX*;c0=zY<noV@mM0u_#>R5SE*B
zxNWas;`<vs(KnanZs1+nx3G2JwjG6!v!f5B=6g+O*>!df0|UziO$Lv0Mn{$fd$j~O
zB3bU8&gTs%Q;2EI5nP~aZEqd@gXz(8`|kUSmCYG<)k7i}I5Z|i-B+9<*TBssa#VtU
z6LX+QbVgHLz0=w66UuqzEG9%sWpOO1XD*cp>`>uSNlJMcyJC^ER9tIIsi%uTD@)Lm
z6|MipqH``K_X~I`HaZ5KS$dNtNW?54%`MkJfSt)A@|IZ4oufymE=iS?3|K0AgwrLy
zsx3f>=>+Sg0{(?BC(PQ+dUb+hKG)GS$J`#J0Qd4orWdle-#o^pz3s#{bxF<-TZGFs
z@>Z^B6fm@!vbc+lRnz5Ju9Abunhlq`T7o=Rbc#)TbS2K^x#bbDCWnPu(^m4W{I#&!
zZfBRkV#Y4>32iB&T8=`Nd4W&X7xykaDiJ-wF?{23DeZt*wyRH9J48*8oZ)cUNb`oH
ztCdOAbf>^)6Sns_Zke&E$}NN~<vCZ#F^AA;ChHsoGcI~r*YF)(=D7CA&&zu^_=Zea
zz{`7GK=YXB*Qo#LKb7wAm8cw?d7$2-<?H<w4r^}2F4|_d<!aZj$`_(*1Vm>P9{pVz
z^C^ni%xlUTJ4S{Uk=Y)Kopae$pC%uXdG<=|#LBc8o0fb&n*MQ#_W6jr7cCz}JQTC}
z^`fLd+2ZYlrH-Q72PQaA%X|_aEMj&rAwGLrXmYUAPu<;2Ehm@yXSr)VTg{Z=*rmH>
zi`etMzl65uyXF1A5;$iy?{=wqtT(p>9#*aliM`*W-6bYeyYbGCCk_kQUx`f$v1n7<
z_b+1Z(JA7tCKvfjV=Py*U!Bpy68MGd&Bj1s?_+95j`>Y@oaJ+Njbn)Plo-#z48;qt
zZYmww`Y~GNneg^#t+KKm*TU2!|8JUPyKL>W8_FuIl3(ss>t|o&4Uu+PU+aG{|B?Na
z=)waJJwv3|N~FiQpFbL=?`)|k@UdfYm&byLMV|u7wmI$kw1mT%&G16cszY6d3Jb%S
zCh<FMGIpGjxT1lbF}CvE;e{+_8V2^CCORZdWBIb@xhumZg-Vra3w|zPcb)XRvs3Qq
z{^lodGOi@b_Emk|BG)G1k#{?GbIqj1jT26KG&oMWzV4`~y5~v5x0@!V%n-NQV#}f$
z(sM#>fwIOrmNV*=-ombz%B9;|1ZxjyDofsH(YU#W&1y|VjOg=@UDH>xsVwerV0d}L
zVs7J7oqL{4p*5mwRbI3N*2##v2zgq%ZR<&2{$GegNNIyr+}=FBV#8@~GTd!%a5_Yk
z3n~fmD%rGOd0K8iQ7*2lVtTA@T9nTmpM*80Gj!4lE5p{z+o5)NW&!6?V+}=TUG`0S
z?70g?w@%>w(_=JIkLy6Sy&#{<AJ=1FIS(kYF!nkJXbRW~`uKld<!UYECdjG7!8!4k
z#GQ(14p$;K=-%hNr(rgArA5)3l+8)Y?Ic+>OOpyTZtm=Ic>7rP|Jq{jtcwp$ip)CC
zRrNH(Y}3TPS>K#*S?XpnEx7AYrtUI-MbLEqmoGN8B`sItTjEg@`DntSG|7#eM?LeU
zIeb>0Xv|%zJKd~TQ(m@d@pX&I`A&fxfh#QCzyD2B&QhB5znG<9N1j@q>Dmd&q9?Zm
zGrBl%Em<*p|0h+}wy>|OyI5X7F-cK)uhKB7Hu0`P_mRbk7h5N9UOm^%JxY9H-evWZ
zC$o35ExJDGP3Za+Z_cRIiq6=_aIAt!<4pV%5!J&A&$@O#>Du16edd{*XNtW$b-t}=
z32qKxR1%tTJ5T0Q@S`baW?5WwvOR7~-&tX(bVAU}{XzPiCkOfy?~2T~WNq*h78FR`
zA1lgw)p?#o7rXX?*U#0AT+iFv8gmu=l($nhGTc(L)bUuW;F^dX^M3~j=pN2h=L%eL
zf#cEUU23bY9u5gv@T<mr52ryCU%pDK<GG{lV!VlNhSq!k@fCWP@p?5>{{I=jCx<cn
zF!QA?A-66)E@br*DD&}M&JnaRj$h-&o+zPX8)g~sT~&XUecAY|$D`}Nmc(a%37^B%
zIw57764%YG8v?f(__t>qTx}FL(|WUz=j63oC5?Bc_BFHy1&DF4E7&_XRxEhtu5WxX
z91qjp|Hztfs^!_@<qKy`sa<~5&?lSw$Zw77Qx43CRLl3Y+Ip{tbB(t~_{_7D*Bb0z
z<2oyfZyEQEELYn#Gu>`(@SK&Il9gBbctTU*(-miZJaQEu*?q74Jaez)@{n&P0>y2r
zhxeGyed$$hH21FXajoJ=rY@5%AI9TO#vwOZjgMu`Um~Tq^;OrJSBqn}id@+JW9Ipb
zr<OSXR}|&kAJ}}$)%;bUh5dPTX1%W1>3grQv&fFxeduCb!mfma8?G#MY|Q*7v-xdD
ziDy^-x21Dh5ABK06Wp4@qBTchl6hnA4|TbH(VZVvUvzwVHhZ<}vn30)S|y&}&|ugg
z*vq+zHQ@j62lh8hRtqVlB$(z-J~k`%h~>?TyXrnY64N(u7fdqUuVhp5*l1<jMjbPC
zOaGkU4PsN4bO>$!+WX`YpVt?)!>p%&ZNFWe$=a<v>4i~We~p@4VfKaV`kAZ5?(X!p
za=l|Na%<8eW9h`^bFU}*T+et=`1Z;dw(CI`Sc=kJw59l)joe;`?QVDewL5EzkHobJ
zE2}2P9Gp>6>wUMq;s5r%H&0r3rK!X}(tNv5k!PE$@vJw6;uRXL?t5ZeCRHpj`FGaj
zTj;f{kUd$kx*M-ADh^cASyp!R&tn(PAKxZ;p1sso;hUV|@oKr{`-d$z?jCy>{wqOP
z<AuZ$G2xfn4?3<FohRA#U4Hsi744T+%OwAw5W0Wl$kvx;I`Nen`xoDOyjo0hhWzaP
zN=v4xDj!lg`?#iW=1+|&_NQ#LZr)V9xBr&dEr<0>76-n3|IPSMv(k#o|K}X>&;QnG
zeMe?lU+^Wb?X16(omHK}Zf{NgEf}*QsX>2ItnD-V=PKE%&6!+H4s0Qf?~aHq2-cRV
zC`)xxp3<W1RU9bh%C=acaIt`?nnmsZDaN(Z<pL(f0>RIOrKT2|9QIW<F=aMvc4=&u
zoND^k&}&AzK>T7AzAzE9RI}&=GZR+HdDqjJ4l?GYxlXuf*0ww?YEojxR@TNObIFUv
zE0$Mf6;-Vg7nExgNOZBkZX%GOqT#J7a8g90(^w$tg7AUz_6iGyjHB8I9~Wp<q%2ub
zGh3izvv3E`!RE>t9S7X4zit)3lvKOrO2W*ks+OUyX5TCCHMw@D3(UF^l=LZ*U(q~N
zT;zgDz2|k;b*`DKO_RQd+r2!@C3~>M&M;b5O(3w{Cqz7>$Iwu<$Vu!**2PkzPmKy}
z;yoV(mEspVvZxsC&k*1}6w7R=%%Rx3XQ|=;XmR=2>#;_k8cm(zZCFgzf0R7(V0&!Q
zXD-nkqQ;=!A#P|9&YoDPbEA3L7jJ&U@OK^+6F)Y+x9FE+7qnZU?jVs{eX)b-p-PXa
znY)LGh(h9RHpvAi(z;GG^aOVE7e;k`a&<adyqdkl>wDX-3W2TV0xBC>%T{!#dAf55
zH0@FH*uTkhNkG=(0>QNclOshZFLe{BZ0<-7&aqtKYF$w)@N=^5Neln99GU6K8$?(I
z3tBg}<n|l-q+B+C`MmSKd!qVwantRpw=dT%5tXjF=K4uJ>A~kHu}p!b(*=K;i`-Kw
zofyU)a-GlXxr3gvCdZDfGR+?L9gW*Bm8o8uo)qr<f6^3%C`0F!WeO36CQ3iD?S=ZD
zc=U;Vobjk*n)FM77cYd%CQb7XpZW4ej)^hbEKWhgA2ZoL&a{$D6<FA>6gsQ#yTC%t
zDMk_@Ev7aN3mLC4wRE*sa&)HIEzG^Otuwo=c>5ROm<L6Ujebl`p;D41-?!&w%n+E|
ztR|~y@!{FT(B=70Ok{VfhqYEts+~FM2)j;GM$;MxuAf2ko>b0z##wJTBb3{49*g0;
z-ea>~S<Ii6*>UHl#kKPJOF}YrG>e-YB@Zv@Jb2PwWqIY4t=_>WCEiaL{q{Xieq|TW
z!E`V7&_}}EE{df$TcU!VPg~3(^VWgw(hTkMH)1uC<7}6Cbbb;0&$`LVfGsX-W88Zc
zL-9}Y;f8WXyA;f3Exvf7cdwXJZ>7wOPU)yrnL~{s*)!C|W~tW+YdV#-)qeFXQ<^p9
zWa{V6aOER`HWt>uTLqerFSWmE9raE4(s65LQvpp0$&-%BDGIDxDiw}yuJCS2<Fpj<
zzcO3qdCOz71^XjKL{^w)e$kKj@be25iT@IHC1i!eLA7^o3sax?Y*}8cbkJJ0v%N@>
z)w{AFqLGF7xQUSXl0zS*c6?u18a$2du@v`2{ku~_6K4qax=jvk?fCg}mUZQn|6;4-
zqWY^0ZM+@}Ut<dP2`UasSv`$o%JoB$ev*-Pg`t+>C1+kOh)%LNH8by{nBael9X|fz
z_WzcrGcinGp6Jo2>3RL?^y-@dl3&)cUtIperO|RxgY_*f<)B#BONuFvdiQ@@yq?ki
zGt;UspQL_Q2^3UDt^F!p;gliz%V)z2!70UUlPkloc;)=9sF>WmltZw;X34DODn(43
z65A{Vyi;0k98)RFlH%qRT&!r@ZK%pJZK|eW-L=gUT8>gjGzGGR)zU(H)mfch9-HH~
zaKpmnC9gg&XcTKd=fXAlmc?|gfVhqBF%wrR-e{k>q-I{1Pf&DqkhIXY<5DMG{MNZ!
zE>4}c#$39ExmlpGBSm}lI?GmT)irlsDsOP9+_)<u<yu7gGMm84N$wR3+$E-Ft*(!8
zT=hR?XV<=iD+H=!YPbcjag_!fU3fdhNyK=ufz~25=EdI~)y{Vq<~)=&RtnH7%DmvA
zHZNOQdBw6w(^WrDt$#FS{T;3y{EX9AN(;tz>u>$Ia=W5$>Eo@NE0*l2p0p%W#AZg}
z*B`T_Q#DhoSM$H#u=n)_@#$U+m6dzkCKNUo&1g@Mo4r7QQPp2XO|N}9lh>NRrBk<X
zwa#ZRPh@OKy|E^5$>x(U*R1Z^r@C#0xS-vt=RSoRlQueX2~C*G@I2o^FlE{kZLNr`
z7oJs4)(0lsvMB1%@7TO`+v<I1xfZ5yB=v;NN?0|&N4f6K=}ix<^DZr}xmhIcow?fU
zLhancrVq*z7l_aKKfS}Ysw(0~QPfh?l5a`f7fy+;`OtszV9B5MwRc1KZ=aG1Oo+Xl
zx>??M9m@=P9#u!T92fZ)z5L>R@`1aSpI-lqt3CGM+9t1~+r?Kp9G_Zy%x-nm?$@U+
zf~;mmuhK}i+Bo}XzhGma=89RB6?^8*TrEFCcf*m0D~C2ZpD<Ot9T75f@0(4m23uP)
z(+-AMis=0o*C;%(EYawJ=#ix=)8c=3O}?4;SxmCQE5GTrRCxE%B^N^3zt8;_t;6tV
zW$5bS;OtY@?E)7zp5ohha;=TPIvs&EXHLmjwr}1zhsVZpZKc2_jcu*f^AEi`rsg?e
znpM^Y7QupQ$;~Yi#nBV?T<I)vIr0Al&)$+RyShFd(B-NBZfwW>v42y?!k^Qn7i94}
zaK}C?^oyUE_$#<q<C2o4u~x^+Y}-$ZAH4})QN3$T_t8y`LThIq{b!TOoOm=+@j`@8
z{VMPLom-{kBt`b_QB6KmohNhAYQ~-vuRSNTH>TIj;&<el`&jwv$7Q#QE4SS=E%p+*
z=H2<nbWR4h$h}84oibDIeDmY`cu?lM+Y<3(X)m|i%szeg<${t$c@BpASDoSdzFH&w
z=%TsI9gI5dE7lY%B%V^aD78^XB;5KcTi{hY!>bX!SO2f+=Bz0&aSq|w*yo^lM&!&~
zm2e#!Cf!)oT!)F<H$Mrg+I7fv<E4#L4*5!o^E>YQUvfhE(>19*54TUeBkC75QSz7c
zty|~6D6x9HQCd?r<Fn|l0PntlJ?p35HCH)!IBd=?Ze;=P&(oL=9A3TfW=yQe65b2W
z7tafD-C)a2x}hV&KUdVaVi{A|#jHD%ol-P6_O|czEwH$q@6(wp>wT%@R%D~hiOW7y
zRkNpDRc;Nm=$!Xs6Vt~DhFtm$g=fm|DA!Lqm85dRQ9a-3Y2KMtYVJ$BS+ctxT@HQy
zbZ+fKtN+pSqExqpSl?$#xT?S@7#(~6*V<FRG&O#|QJYv)&+9vr`CQ*w_RG6n_N3|N
zao0^c=B^vfcxjJS%jMupnQqt9&x(3E8(fQVZD33K{P}d--1>%C?*HEsxEXe++di4G
zMEmG}b-zbj)`rX2&D6>4WPV(G`0(%8`&N0}R&uxO<eqGp)4fSYz@G1d&7H>rjgJ{+
zCF@R?-2Z&GcJpncYY)P$4vMTTJRW^aM7Ccdwe5V8=3h4(oipM(XZGyyymKX6V9(8n
zIpUY!IL#@2J^RoquXA(L91FMbU9>v5n01P-sEE|=R%N@?mx&ddtKM3=Z4A%<l74@q
z<AKj>^R1u0STXnE%jlO%D-Q^~F-tx%E6Zljw0+A@{oWg!XxgjD%Kc^AuE-Nw+Idq~
zXiPXU`8bzIh^VGggwBs+Q&(1(R5367y3FUpv+(V)GO}!?!iIjHd8Z3zJaSuj%eLpq
ze?vV{vvW7?^iI!`y~SMk<fffS*xI*l``$)Ad>d=K@SEE^?wSj2zd}RK?N0r5TJr3%
zUmO>O|28cac-An{`9`h4%)$v-cCXL<c^_VJXYsrD57Pwn)bF01E6LB<#l+-TqiJ)<
zSogxSomo$xH|uS+<GpfG>t3gobgoBcfNJ%*z6-I2kDVUAnNwS!@m^TD|8mmPyD2&X
zQ!}4_lYAw4PRGYt_ph7mp{f=&_d~l{9-7%(242{s?9u07J#BB@yN8R_98T5nH6G?n
zw(qZdWVY|IePeXEt;nN)kG=Mta*un<vG8$T<b~*<Tfwp4_!ZyHc(>l+;r(+hZroi0
zVo?t^ygeOSRq&s;?`izGr$=|kAF{r9AUiy2SHH?ND*->74JG1RBqg+4kH-}+@I7*z
zol!OXlAgY=)Z&Q?F4;(wzY8+F-SfmtrF_rzsYk42dwsblZav#l?XdbK*X$Rseq3J9
z+s<n9MT~8F)2&Z2iTBn1M_KEtI~pFFEBMFG@>)}WLG|q9Y_679u`BHbZ42j01T(C;
zz3mzMx!M=oUrk7Tom`~3GFVOVMfP;#o5v09Hhf%fyGQEP{q^teOWom;{x~ywOGLEO
zuC=$kp0G{a75D!QgQ7!%Z6hnMoXd=WhmO%Kl2RoR+>0DKS(U9s8VnPUw#wRWJALBf
z;sar<-hDQm8_ph_!f~^wMKbvRGoM*ZW}Gs~AC@)DVJ-i*MskaqWUDo6o}hwj7mH@}
zBo%=JAq&L%g}MA@G=(0RvR1l9>`Bpu)eX8bEN!MIzAm`AIpbuNi&jJPt>%)8N*W#F
z`Nvh57W(xn1vtgeZ01#$)L|_Y()FF4b(5)*d!}!STE3EL*BOnVB`g(ex)Y{41kE-M
zb~_tsoX)7v+9P*$(^TCFOxfH!=2|D7?&r*xzVbKp$A|UXSR~y2CjEH*)X{eLyNoTB
zcUrG>i*4zP{eAJ--B!1WpTd|LSUjg3Thd+P9lY-C&K0uS3j4}`ecjI{<{^>T(iCv|
zf4!Ci!|85?P9}aA9j?P7F{>Px$%O<QIQn0%q+sO@FJXnHQLh&IDm*bcrJ?*-M@Udv
zAlYc+g8Kqe6V)cZ>DJuYY*J;oVv5(J-0eKVQof2W5=0WFC^XLT+|_q%N2aifsFIeV
zvhb^yC!6Q7oLVqvf|A#&KMO7eE%>mfM@w<_hD%-R7VP<SYR%6BLJi#JSF5=dx4X?M
z-fhPEsG(~cgFwgr=6|QW6*)i6K6$>G>&*#&6(J9WCRg5=Ef+mxQ>O&62r(!%dY`;)
z!uZMJ>lPa$o6{GJ!?I@jE|Tedle;W5TZAEOirK9|#YpR{iz>0)tCk**o;iEv@n<cm
zlRPvd*Ph$=Mj)z8ZTYpur-B*JG<WK4T{7d)t+nTFg)R8<|HhMT)~}A~o;b7MOO77n
zAFfy0T+Y8;%+yqbW^G?FwIhmiJFl~r#`SmKTr-X~W+<&N>tk5AVqP=5*a^Ary4Vx(
z{PMGA9GO*c=j!5VpRWE`AkUv=bg!v}<!408sz09Ua|EPPJ?FXT6{XIsoYdFD##*hl
zN?-KW%>{<63dXlrrxdNuXg&4FOi%B^+iwff(vyBE*nM0T&S1y*)mzbxeX-~MzExre
z_HX#y-4NI)P+-EeWpfw@$LghWvvq=Q-DtTn$*;vSKl-cKkJX`H%`!}qS|<mcTF$uq
zWx)~Fx7vG_OD&zYyKl!N2hMe73HjW`*LNA~oOe|V|9JH9wSI$t_hkQSYyR(0S|utY
z&9!mazD(b_79Fk9g75GD>+Ag9S|@X7@r1e3l^%^PE^+yj1&v;*FP@Zs&#Q4>xzO@w
z#Z0H3YH{zHv}WlfrMai)9hRQ5%8_04&8jum)@$y2vuDvdzs);t96Eb$Yx8F1qSRSc
zDw*fMosMJKFg-w(r8KfwWUoNe)_(%08tV#YGYB41Z+@WK6m`HS)u&Y}dc~?TF2>IH
z5r?#euCat|KRv@orA(%6j!Tk~3a?L7`NkQ&DV4!aa{^t`C&=D98EMzp88k~t;FHY;
z1zYXwVN*72*0flbDQ75k#%24?Wfxgv<5u%82snPAB9u3W%cMi(n8PZwo@KII89EPr
zxVHZPOb(tz=GW34K`jF9r7X&&b0#_prERp?6sVyrB*7yZG)bT<r@~1w^kd(0kA8{H
zl-@ZreU)Y!y=Kj<mY*Ug5briuk2NDI=-?vv3+yJ}Dqm-m-HLbIBQ$NJ0<ZtWTYT(~
z4vFpD!Kzt2>$LXA6jz2<ssVo|CoEtIGTq~l8aJIwdcl{=8qF@J6zUD5??{|*ZQ0S=
zlrPQ6Fze7RmwRi&X2i&F?KR#ldT5F2qc`VxSU)Xx7gJ4HQW2Dva%qk9yr6o!BkC5y
zpLCA5a_KLX=KRTV+2!oP^|ssh+<*Q`#kFRe`+P3TgnQ3S0~u>JTFrLodL6S}@rLac
z>xwJU-C-Jw9}E0v$=8g$_$EhWNArry%JR*gN|DQK9xgbeb~9S#`=Vl_shkZ)h9~Du
z$P=?`PVsVLkm6|&dEgnumaCF2<Wcj;`LyLzew}a4sr;(q%XjUWG%NCT{C_PAhnXg;
zW^l@+-MO;s+Ga=L;Ka|Dm)>!6+4Sk;I&MZ0h8aRL&(Am_#wV=e<Pssoua%ytypuOE
zkx8h8O()Fcp>IO;7j20}Ui}V5n%>r*g8QEpTb{kN>hib9y?rJfT3r`IoZkJ?WU0{#
zS@-o))h5yXnhGI`+^Uzh3oKfDPU+ktrWvpK?TaTFvn^NP^Iq)pVTD24(Q`YMBNw|r
z<Cy$1?E5C2Pm7afR~t;-_A&qep2C$rtw+1$4qjPqnyK!4Df+y}tB}cNwUO0dr``LZ
zr=lLYD^ld2^Ys=6uZ9TjCWa{vYMN6rRnBf&8qsGYrnf=0Yw=wz{@W_2W5ram_NrvF
zoO~6(DQbhlUv52y&dLc}J9*>kEY-xDBSUhwMJQeuQHc9u<zjS4!=>@<+5$ffVVT7n
zb}NgVHTRB~eCF7ZJsg(4{;E5L&mC&nYsi_KVJ$6wlta*PQc+<5Q-EdDrgY|aGY>V}
zc8RoGnH;aW+%L|5X?j4BJJalf*ftBUOUCC}@=NmGxdgc{R&)+?cl6`bwn!70vFh^)
z<HmcALZ0!zZEmJc>RCUDugP}SPZh7gNB>vWJp8ojtV-&X->iBCtQ-tnU0dyE3OO)*
zIy{9_WXBZ$(4|eRNmEyBI_dk3n>)i+J8|cdOA3Lz)fk%pK77a%-IS%Qee>C`Ya22x
zb{wuNte1|Mu=m##QFYI$6VC1wI2x4VZtdog`1*~~jK96gYdumXb}C<fKF8ejv6!A(
zX|G665VvOA$J622NtQX59~YEADy?<$c763h?A{5c7tFVP-Yz*HdAm@iPv)nEu+8_2
zk1pn}D-}Ex^CqmPRqj2<fwsf5o#aw3IEwK~U+K(yD)YzctE}g@c{)Zjm6ulEP(E5U
zLr9LnQQ($bVgkdqn1t};R=s!KrE5!PmMl1V^5B<jZ>Rre(@$?*(fW#MM?vi5y`}L^
zPqTREh%@iMVZVN|>Dg%WG{;q9Nd|dykDh1Q?YPRfExS6+O!Yt44*p9Kw+^z*t(reI
z;j!q=y%$}UIayXZS{28soxL-!bN2Gn;z9XK*SK?~d1nbuPnJ8Laqe@XUxrfw_s?Ge
zfuH1D<uzljw}mYTWDw2gyJt`*a6{eYo6$^LUzb<qw(YymZwnOcS+Mw1*cXkD3eNAH
zBz)(48`)J^ug(9weYtJF$$bTvjeY5DR=RAHtd1=?rgodTbt|Lz->ck>TF>KldNtqq
zRrmc#-@F}6>PC+}3ysRG?+9#q^ymM}S<CE17=#Y+DVzN}Y`?Z~cjoN>yC-ef&39>=
zc%<lKFWG(FoE%?xXZ|qk^6a?#c$>(sbrS>3AHFa--fSXcWvlVJZ_Z<z4~Zgnjy<kF
zEf?M}s*C17dz0(7kZpnHUX@#Gg;^%u_i|WxW2f&4@#}#r?`*cUb2@Y**y+w}i}wp0
z`aX9Y%W8l8c;davl`|stt*=l~%$zr~a!ZqiiUG?c0~?`rr}}<>nwRz1?(#zIkJ6m~
zeyij}%1b}dYM!ArBV%b;lJHVbgOfY0RzF_5`nM_1;#nWPJDiW$^IIQq5n1-<H8=lZ
z1KwoAcq^9`8q2z*r2ie3GUMspec4R*xAFd$yAN?64Yt`n@we^7gNCgf2U`W^G)e3W
zRQNCUOUl``de5GTMh7famUHl*&UW0PwCMj!KGhWmZ#f;4uGw$>rEksWgXuDyx+^z6
zuG;(j)!N&qE!58(`_*8pwS|9r&~~$cy{$W3f4^{AxMANR%bBk?Dtg`4nZY63tkJ7m
zA<+CmMQ78pnHszKKl2|kus8b2wK`(*PsSZT7%eKc*vs7!`?=YmV57@lWy2W<>}$0-
zQiQuCijOe<nf!0Tf=vf!?r7M2aYpZfVAq4fd@|8fr?3cd=`5f2+4i@^o&^c5x>tCF
zLN@W=a{b|a;?PCQ>gq24z}0SF_|91$J73Lj^|4Pg$RlG*f2%NG=8<hri`Lz|Y|*{i
z*1F1RjYyxh)Z8bZ{`Y;Hq}XR^BBOnLj#r0n$V|PK^+^|1w#|_%X_=&7<ISPHw(5!H
zryq+tyBr^KFtHlgFPzD=GRf-lWYci29cD9CmuquGRVk&#bO?S{<!Lr#k~uKH+lT4T
z@}>soZI6X{S4$;S9cd4-sOW6a=;+oxu#ImfC)b_bJ(W{~WH#+@K0Eo0;k6}-XN^uj
zmXw>l<n){rua*EV)`i>Vs~qFp)X$R8rFVJG;wLiADN~z%`z}l2TIF!|(P7>jGi_I%
z*{qTyk+nqm-JDZvc+Tg__-ChxwoKg7YN7M?6UT-c*Q3^3O1<{ES{=V*a&TgXQL}@$
zLelZqlec^?TJOotW%PjMO|d|=&VL2gMGC@~?LQ<-H!?K&gqSiWZ~c0)Ydr^d(;u0r
zB<^3<CuV9G%hnil`)s>%>BNEu`~O#82;_2_<#Eyd$)bZ6XDzJxIuBUi%XAIuIj7k(
z@lDc1nYB}c1*gsU<25O<`OB)4n{vEWANxvtH1#-j^tCnD&8|5I9C&VYdHib<)LZN<
zZDM@HQ&r1o$G6J|d%kEa@0q@LRnNqdxpxfDx2`yKue(z;(#eAL{KOrn%-8De6SkCF
z+WCFT(J9SG9YXbIMCd=>xMcnY*O>u9Pj;W5o#WkeYpX<*vtm*p<ID@D%4QZZeGDIt
zhjB|qS#t!so(g+)L1xdYi&+M(8?2T&a<38cJ#*#%5k?uq6&&jlx~{UPT6^Bvu>IhP
z?hxy;L^0k)fp<72^5mSo&9W&!%Kt$0(c`NQEO=m@BFYtSslHcoj`CFZvl^0(8t(UV
zE-m<QcGit1DUn4#mAM~XvGHq_JeR#yN~QaNOK7Z*QnBJ0*|l@8YW8OT-ez;x?^8%e
z*WtyDxlW(foN9jCXLZ*?xG5}as$#;+j_DbkO1ZobZ}`8^7Q0qy_wUHcW(CRe7NPQD
zyZlrezXyE9O9jMwyHq$BLS~EgZ&-Hv$cDZHC;m&#X3?-;zUz8*l09Fn4`;T4?N6Da
zsgo}rzA9we!797@m%)Kq71INi%@|!)Gu~nf$_%cP6^ytzAy@i8XNRET?I*X^2dw-W
z(|2-G*qN-AO*1@OPMklfth=`|s3&$_(bJ}L!b+!aPt)JI>H4EZd%A8_%si+kJZqhl
zt&*qevPQp%BDWe*$y(WUw+$x7JKm_jD&5L@-fC`4{_CjIywmJMZ|_?aGM%TZn^nne
zwf{EJ!!v&P=c(R2pE1eh(1gIs*f+}EMMpJ`-`L3(*yZTB$Y04%frCMuXKUY!GkbIT
zIRA!lhjIt1hVEK2pRaU-(N@!fNR1dTpO>{qgy)`#GrG8T^BLbgH(O3zSsJRXljZW~
z_Psye7tO2rf>v;Z9q`+`apQ#Uv%iiyDa{P~&2Xuu;ck}dl{vT03f1h{!78*<=f9(@
z(RG%0<~LuNf5`OFkL20MdMfLZl_={RmWu1QxcYB(nHH~{eCV&vx?h+2B4Uo!p37S$
zYThc?^!MC5mx)camm9Y7tC&bk7CPTI!C~f~I|VANW(#gyb2V}CHDSINQR)+W;qB@R
z9KA0C`){W{P!n3jB`I>sQmM^ZUe#lj!htCdYPGk2d^FwlazOV%AJco=8oNHSHZX8(
zsK0Zw)u8+5O8Fy4=36^zgddHY_%V7`Z<K?g+lr~Ta)Q>nL~bnqX}MxT({u^Z-cthk
zQG#oAPo}oIbyNzrmx^9*ah=F;<s7ffwSr!iS{0?VsV<im-qH=<9s1yF!GptFCVvR%
zIT+BuFhM#YQ}gkXzqcdVgr1c2++4(IsLU7Fx1mzBqk(}z@h1zb2m=#?4g&)NLj#io
zBLl~O2G$)y4jUFQn6n9M#b^X9m>rhqHC1!+1f_$WDLO`Xc1)PU5R}5z_i)Dqo|R!=
zb0l+)XI<7PTO)K##Uj|@>@1aHqnPalcbw+2G;)2A+_K=pVynqov9knL9$dWo2yd)w
z!5xj(Fzz^2kJ7Bbl^#pAM2=2Tj%-Rd7s}nWVYT0Omi3QB0{Zo4x0at4+H*VW-LWv%
z-&!m_0S0FeHrjG2?1^BAKguR~I_B20g4Cwmq;0!+B8m^4mD{fJ#ATAOc<VCGNn!#<
zDUv5x{MGlwY`EgqC6#;GY{zY1$@|j&?N5T#7B}3L^W4~^`ylI6Kdbr6DSorI9QW<I
zy>iRdbz-Xx7iDn%*Y)WBv+I*N_c5RSb+W5N*VG&`-*N4E=x^uQ&+o5_*Q#$y%G^3J
zB=4>|L#T~n#R3LSqaTw41*DcF1abu}^>h0Fa_Pf1nMp_FBQs}iy%@T9&8Dk9|JN>J
z)qbmz(yiOI(a_5yM<YGi$5e0mVqe*BFCO<h=w0JdUz;l^<hWL}<cV9#iMH7c{(ajf
z<goCXOmGT&_4JhU?!YZ7Q-p7drNqQbO-Xc8yPDMR#ae3KCGloT`HeZrT2H4m`#*fC
z;BWWw$@6&)ec{g+xL&<}N9(c6(W^~=0-uJ3^vP*lUBd18G<ZH!%Zr2tSHmsWf|~fE
zCW|dLNL$rk{`Bee_<Gsv6018B|8EmWm|A)*Vy(@*Q>#+yIJd<mvz@)dma;Y@B%st`
z-M1*oQ@6g|$@0ADChf!$SRLRv{pT~cnIU|~Z%sa2^h8|K=+*KGB`Y`+VoTzgu630!
z&TyJg@nY`tc?^#>KAKl!KG#D;`=1;~K{50B<?@|O$EI%BDgEhshGW4?_V}hpx*i`F
zi8Ez<Z0asM^YM5~+nt9qy_QbBR{zq_rYn9{&%rL;wHqQMdOd`eNUWKEP%ScHv)0zl
zCmf1Z1U-*+iQRVPp88NFn^iJ<jgypX?^~yEvyfEBFeQ<TE&<nNCe4sl3x9Tqr?KSH
zEbXnI)O>6%bs9&{eO0+Uplw~VMsxY?yXp68;{T_X2dF)Y4GphhlKMWkQp=>grSZ|*
z=UV;md4DuUDprMtOz(bd@N~Mj4PWFme!HCyyc(wlG|V^hi}z<;E2`DJ=Fq8g9&4^J
zOU(Y}@ICnr&&R0|iN=y;S&nBIe@Cv@{O|KThr96D?6fw{fKwCQ^jUJ9LZ+$7Oq}x|
zZFcUG#<f>2`=6dQ;ac3Qk|!Gy^rufKGOM2=e$sN{p*6dha*CFeF%>+Q)%-N;M)em{
zhwmH(soy&L>pmPl(7ZAyaW;39pT$?hmcUjA0bvF<!w&iX850t#I;U_LORRLA8qieb
zC~>H$XxWc5M>O|G@z;bocAH=H&)pq4(<kSPpzJ~aN)4VxC;lf)Ub4|}ZMH|S$K(lO
zOMm35ct<Rf^4t{G)#326%S*?vaMF#%|0N2Q6>hp{UkK|yTH?+Sl_4GU?}*!BOOD|B
zThWW2b!@j(dev8UMp^E?#>z96UNRGymMn3TIAMG1?Su*5DoWxnEnKaF^85EFt9+fI
z;TW{Ec=qWf%%K_!cp?QRENBT)eeZVAX0KmiQSsu}cP!-Gb&V#yi&W80Sftr>WvQs=
zqYJTFLbhMmtV-ujY)!Xm);`P9pMErkM|)DH@Ub%~Gf%zgHo2ax73Q<qpoMdK)(3UR
z$yc65Jh;-mmB+}hb<4t<;9HCS_66HanvgBBL_u=bffZ9X9`%{=>CyB<7CZlM5^b8G
z9nE>Ck!jY!zRp8|?6v_;;TtbBTZt?b2+eq|RCUOG;)i6huQOW2Oq_(xrYw>V)o{{R
z?^-)0WRXT{hKul|fPPmWO~E4qEapWQgsz4xH8U0TV@y+O;4MfsiJ7@K<6vLZhP7h4
zMjW?9n)T*5JX_hCx%{IEi_1LD!T^_$OP9EkOpF2^D&2g%h@oeh$@PRl#l*WETP2cp
zZ-w+Kh2Hkyj55BiTlBCyRWSOmRg%z@6Fv-YGgvh*tq9&d^~8own>BuGgOrZ7oh#~<
zbxpB!k32o~ik7;lYnr(=mq4uRX0^X>JdIUv1!nXGPCfc2IW&6b<s`kWGnFr9dRs@I
z{2w<>XX)7~JKa`CU*h7}KIhQdJbUg1r?+0)G5z=KaG59O`y6?KzZ9k>E$_bZYTct<
zWx@|CABm_}U1*Zqy24$O^HO(9&%zZIg*>aK%$%N}$)bMHdtUWa&OL`FESauS8NJ1(
zS$Ad8=lyoK?oEn2EWp(56YcM<9ieHoW~Rf`n2*6{CfXEDt<-yJ^6N9lf3e51VJVLS
zWd6QzcKvlhX!VsNs-+p};d7G2cJEj+D`MlGBprQ;{Wp%D&D_{0`RkGJ=ZK>Y`wY9R
ze<v$SM;<piR=6kZj*OK2%q@JT8j6-(ffEc_TxNbU>?%C<RH$<2^5+X52HXtfui`jx
zCc~`5o@K)`v&}#MpUqagaqiYb?y!o<3O9BJ7S-%nz517=$D`;)IYl!kI~t#jSnj!n
z;hmRE^MkK7zTPJs7>os4e0C{Lo%zLCpy-@-(viyvOt%)Fc54fo7|7u9l`m2C^sBJ@
zH@3NpZAv?;z!i0V*R>t{OxKpjW;4FIb^XA<t?P>KX2(g-zHwY`+uFjj*{RvnZk}k{
zwmE;UW>k3etsVQmZ7}}&CcSv~ZPtTE(|$(YDZIaNqt)L{yGpBK-#GO(-1(cQdw~01
z)s*Rb*j~CjemVQTR^R15%ggqCroq#jZXA-^`XRslTCwoGvrofiHcox_E6FEO&aGl+
z)3PI`d1s?_pQ)VM<8wUvq28qX?^oUY|9alvIjpiD87q8VSg&5J|7h(54I6dl#rNHo
zIJ-PM!lM?*z(2V?FIn1rrpv4YxtC_^a7k(JefhDe>eBLiC4ni6UWNNtU0d7tbzSz}
zSHbc}uN{9^x~~53n>6#?5-0k~wk@}PQ!u^is`1v64d?H^3prnXrCV<Oy;+>PE`A%Q
zvFAV1@17a=eBt+*zZ0Le{#Y#kZ^iM)kN-StkY4g2{Tato?n8XC-||)E1>a08Sz*)J
zzw%4HT!?1W!h~xYJF2{1y^8I36S$$j%X3}kmPw1Q+N}P%FRNwYv}~^i6(NUC9u<Rg
zg0(7UbG3aG6LXr&UR_#rK&$QA1Hop7P48bPiCf=bx!Cgm8>dm-$KSX9Fgfi1#jUjd
z!}9-sXI(hd^K^dQ&)fF@eysoh=evH}pWpNU{co>lSiw3op`KNup53E?ZASw~Mg3p(
zdVY<%AL5N77L8&z>i<r!yJcSYaC_q!wYrOC*>hd9=d#MFDMfZJiM_9y|6po9BYOc;
zdgVVghVS2$8IB8XHQ>q#sAZjK{cEZ4%1?0{lWZmh+8iqOIj~vUg~KOra{M<@AJ$`n
z7mQlj4l=Sjs&Y+hUBf7JS6NVw(d-nHN$Q16tA!>z673B?2^uE}d{}IK!7;H)Q0nJn
z-#^U8_bze;PN+*ZD3#D?^u1nRKclokqpqo<bmoS#u<d0`1s(FoI=cV==;-~?F;Rgb
zQh{Nzhrq-ItWzTzSZ8$3@o1Rvqht1p&h8bRTozrC3s`4bbj{7^TD_ucSwzFiA6={+
zZu3?&%%9P{qoX_9y`keoN9T+3C+QvgBYKX^C{GZtcp2vK#VM_KdL<WQC8K!d%H;yH
zw+k8?#(9Nu&lV8&o}_FplxP^z`y@i_k#JaqM%arT8K%Xp?=l2>(gd$h7EV|!$h@#c
zDlxOSP&LL#Fx8=kjd22#15?Za#<k`~b3?MM3T1OA2(54vxGN}JX_5Sfy<SYR;rsIX
z@5dXsZZz!rF>%!l)`<#}cHihysGPK82J0M;Niz>j+F&thp2j32%SpPClZ`Sb8~k56
z$trWQuIJ>b9G#gV0vbCf?Yhw*bRyendBi)x{4Zj0raJ`9C8o+YRq9Ffswhr1IwA1y
z03XwVsa%2li>HZ*T{jD;kVux4{9r2CaWLNGqtCZgAEoa;heT|Zrx&&aPVZ%u_*^le
z`4DT!q9QgnznYWMGK+NMl}tUaWE?g!3ktRQa76yIiT_n)S)1n>N0Yf_!v*X;J7>)l
znC>Cq*y+Y9IBSXJtPBy>%qg>$ubj2AbJpsevsOsXj#8MtC9-pK<?O%-tQ%L(-h6WQ
z&YiOl+?;)2CTqZg*+(+x9GyA)<jvVE6WduP&N=pT_CCwG2RmoYnb8sHp7QsL+!v?(
z)MFJ<mCdaG3wx)8&r>Pvl~J7MwQ!!yCGNWx>G>VXc_#!kH@EoT&~iQ@5E|O&BUJ4X
zWb0>?5gTgDa4J+<Q7YwXpU>qguME3&j~1vbG<};<b0oc0B(bLBrgGKN1#_<m$_ol@
zbC}xHIBV((PNkh4_BSVSF6>%hIceLCMU@(p45c_LS9BRyEuQgWj>)XWl^%<2rC85#
zE}5pWgh^qE!>o39t0f*%tN{y_xN0r2t6E}zYl+LPIa4c^m~gQMPFNb<wKQ(lY?E1&
zIV(C8IOLKGA~tVn`f3&zqRC*;F_lMg-kZp&MjFfiPn((-CFUO~P~DXuwleLjfzT}v
z0j-b5=F6wCUX1%LDcSg6F#f<|tyMwsR>`41Jf~fCk>oM#pDCqM7by_DA*0Td&E-=o
zTbSAl7L@~n)51Lj8$L?;x!SyEUj2TGy;tDO3b*##jlQ!U%uxyK+@?8e&B>W(H0SJ!
zTypf+>|<R^Tz9Pr(PB7zYsrPIHNL;*oQqn^X1Mm+thE<TExGBnmZgy;kYU{;sdZ1d
z)<vya`nYPH&#5KHJ7>+?QRW>M{L3lCB{XgChaS#{X-{Sf{9Tqd)qLLnu6h4ot!GJ`
z$I88def9<`#SJ{KD>$TemOPkh^q9M8sl<ejt87FPyI(2)KH2tXrErg#s*-p0yBQfx
zW?^qS!hDl-R=v{NqLg`Taew&a%<hi@|NeWhF=e)@9$-E8y^r^lqR_PP;uH3%Q?lMD
zNbNf{)AO{zo?BZu53W8Nxs*+CNuc-Epspo?4ojW5SOo=GBX4hwJiR3T^w#H6+n5*_
zuW~V_a<k@eGkQGOR`7dUJ~wOD=_RGB*Cno+#HYcP{wwdUY5v2fQwpWh|FSJF<|^>#
zY!1D-+_P{;`LE@Fe(jjZy|bihd8&yxyZQY8EG@4@b$Fid^!rkfk~z)qiVtu2uA-H4
zmJhYr7HYlwq%AA$5;kMgqFp-iD;6}rR6Y>cc4Xm<-bnkWm;APcS8wrhxfJNu(l~SN
zhSi)KXU_H%IA^^!pqoYd;T8>-y*HxQ-RfouP*`_Ydfom1r}w_}T6ZaPZh-oF?ZaGd
zrujYP%}+TC*nV%|+AVV3VmZs~_43+AOw!^(SGjyccloIl+MBGjH?j$CZISop+Gai@
z-nBY5L159u-Se^*Y+kNd*lN0NlWO?pgM5LibH4SnUfk^4TqF8X`TWD0FDyca&b0@O
zg#<UO@rf4DUn3rnEnxg-jy}&eqdRk$8usSR-fG{&!qLDIX??`OX1nv7ZJs{c?|5&I
zs#;>r*{B+-^np>>^=ebe!SyY>k3QvCE}OJ}QuI#a6~YNNYI_;{bGatwbsXYcxMJ?_
z=C7vewGWS(9TZn-7hZZ@`^(N1_8!ve%f+ux?o;V*ePOY?t};9(f=%fEMk#L*6GQc4
z9!Hs!2JO7de$~r1in<BCaAI4e*uJJvv@}BE&WkPQxeuS-wUyIxuW#30ZpXbc6Zbxr
zI(>WhUO|P^uYS$m?zv2D$J|%1d%oYw=l!kxd$-`%>Uoo1r+wz$aKdK8sWWG|r1yWU
z-hZ%X|Fs_*en)MIzbR8ws8l63-{z6slvfLXb~|;InJm9};7!Fj){mjDWWug~Udb}c
zZ`bmbGLD=1y7~>WxlCVc#Y`4B;$CB`eb9+pQnYMY%4PxMO^0rEgeWKO<t{vYb@t(N
zk*AZbw`EChm(blV$$MnG&eE#rWg&02e^QGxiMW)k(_1-fYHE&v>5Lr#KQB$6p5JkL
zdC32j`FjKSnXYp)9na72T3+|+vIFns$-5Q%ym$7m&YxHwSG~vDsPVwDEjFvTuNG_i
zEV_MVUy5(~&0R~Reb)%A3_dzPWE1P$RVo=j1YZ_cvzxUo@3Kv+aJ^*`Z^dY^q+rFx
zz&&48+}=-ZHs#r?;<(P0Yn?;Zy3IPLTYT4TubI74qtW}=>DS);-?h$Od$ZxH&ACa@
z$0nK2o4!^g)90pDV8P{@{fyg<JY82X1*8}I`pY<8uiCTGQ*iejU4|Dq0vpfvZPq<u
zm8jyzbkZ}>^uYJcA<nu-AN5;mGgw!*vfSL{y|5rr(@a}N@~r_^ob*<ks;zO=OJcc~
zB);D2Kj-dE-X($mZ5nG<Psx(nQ8Oz)d)eid(}F#x1$uVhGt9l@XnRcNQ-%57{eJ`Q
zZxg9}_;lIRsh3L&?_2D>WcBb~_|F|I^90!E2{V7cs%5xBdeMrHrqg?G#Z9QPskwOd
z%hZR{s}6kcjE_Gnuur$OIrH|my@K<#7-SMm4jq)byV>_m#~tZM(u#QkVh?v8*(_6j
z&H3o#ginW$1r^;od+_91fm3D+uWyaqccOBRzs%enD;uIN)Zcm;Y!VW3?#G#evZw!c
zt$((1efYVjVe2YiN8X$?_fpEad6Tx^<UF|S-vy<(NP*JAXA{lOcz(P&(e~yQpEF*J
zXQy7TEWW6;H9+#7#4gE2=>qH~>tg#9|7V}u&}-Yh?%WF>RrPh^f_pZlo-h&jnXxG-
zllP&Xq>SLoRf0^F3uUa-)Ypix__Ce7E8+2ViQTHnb~78?CGTnMnDS-UvTY(eTW2qS
zb#r}+&XqTr`!n7>JEV6xP_o&wv2up_a@OnjC#c8G^nLa;HKgGo|Gxz8ZHfH*CRl%+
z@b+KB+k1X*BV+DW8D9Pr`|jn>%PH#~Jj;7y8DY(~@qU4i(C#I1k$<nOIxUwH`64ND
z+A8nD`KMogyDEN;S%qn>z~SAD8!xWBdQhkS%%g`U8oRq)K6tR1g-aez+W0N?q|rke
z?&L(y#MO>^ZhY(ZEnd*2b1m4xHKh0N{ff8^8h1b4d~>Gizv6T5>Zt{F&;AyCF4Vh!
zFz%Ug$K{Zj&3y9&3i9~>J>YA4`pi0ii-&9P`r6QJHUsB;>!p|43h%!EHuLL>>koZ)
z?E3yynRUJh>nXv*)zzYFZfjb*o?zMRwc=p(V^s;ML&|f$IXYe7SbI|b)}0o^Cw8q_
zDu44|*51!i`|(Bg!JFE5^=c2Ey<5hpSJ40bN6UTw8*@IJ+3swczx-?N8xx5KPem)2
zTu}Nw`)y7T|GEVGyBB4@ec*2Vs`n_gHp}TV>%ARGa`zPvPtC1+{g&bJ%cWOKd9HT;
z`n`Am1W~_fC2Jo_teY<@eS*b))AGVcY%BE^R=FrBww8X9&h8D%t(Cm2zk0*}EaR?S
zM(<s?<ov%~FkIlgPxc*?ew=5e$J}@M&-PiZe_d&HCR*<1={3(c+<(d!ea7WIKZE0X
zIqRs^EIm=GSyR-)zGkFeJmh!4h4J63$RiQ?Np4-@ws~u=_sh95eq-696lu6rmr<v7
z>+BOj2acAljrzIi`}ukMCohTk^u%(b{LzJ)!ECl!Ay3_wdQ6;D<8^Um$kKq|YqySY
zrJe5F$}(ZrjjSm_L2FJm+5Vm?>N-W`_OhC2(={=>O5ffy^-ANGoD{?IV=l)IJyFJ%
z-n@k!KQ_c1i8)cAsjN`+WL3@EZ3{ap7o{=I2%Kj1(yeHw5R+H2f$zpI@q5<xF0!>K
zJfn6!<$u)8OCB4=&IE4hTK4B|{)!KAUu(X8PQ4?SH|J!xxV+!@?=HPN8<$_Pn7bn8
z^+9W`t$DEy`>T=`xpbDq)lSh_l(n{tY3ag(hyPmIm*npc+Z*@z{pP54v4y`Rm+t*n
zkhN5H&JOPj;*52f%SFmHHXWZj?M-H$Wtr5g_tuN|2zzP1b@~(FBJ5q#tMc)Lv@^qc
zk0i}?BA!Xk4N}*4O<c59?N(reT;Z2Mcg0E>BacZ{R|?%#4&T}+8WvFH?Z#*^v&c>R
zuaM4$DFU+}>FaG$X`XDLySm$0+f(Q)uhG**JVyIGB8As)I>B^W<Cfa4&)QwD{&e;D
z-b?CS8g(WqUF)UICXN4=@mZW#EL@Hocg?%MXx2#$uk&lNT~<1M>agklb&`ck!%ebv
zYG_8{a<Q^kg&tQn3fvV?42(!@w_ywube`PD(&IFFp^gx9=f(yrPn*R)n<l7)`>f*F
z()DVi>eaHG%;!Nq>+eokxvOXIv@5rJvRZpLvUGg**!614B;C^IN#B>0J`B=|t=u?w
z#T^~ZUFB{^gT8ogG?MY|+|((e9o~@g-s;9w6Hn#jty4S%7?Q4?&^R1fHC1YRj9u@d
zpa#POo1Pz?ToL{H(!yWL=K`arZ9dKy{$|6`zG+F@j*8b8baCyfD|r@a!DzLI&&Id;
z_D|<e9gElR_*nJwxZST+(yW~tTz7Z=&-Bne#l&f7aaDu+o6*Nz*SBVLwycPA>ty0&
zymnE4Z|S<c)+yqSI-XiAfnBSXm3@o$y8JePB}&h?n0wEPL+V_6wDt>%+nfk9$yzNo
z`PG__QFhbU*esQfJh`WMsg3z1Pc>t9^Eef4cTT^$muJoI@3Rm%Q@YD)UxkvnS~jmy
z`Pmj-53^?>*-8)23zaAD&QM-@F7V5vGt*Ao)VzA>ZJq0si?^dTGFQKy`+VQDCmx?f
z0zdZ4Wmb7CmhQW}SaGhV@^7728jBa(SBZu8MqG4JWO}^#>7r%fS6En#9OVO#q>HS+
z6`;0Afq~I+sa&!{lh^`<2B(e6+yNO6Tmv?V3VEvj*ZZg88l&VQTmPn|P9rIvYw2W>
zms<^5uSIc%E@>Be|0j)SvB1pe9+QbLzP>2Wz8N_88q?D9lN}vzy{=Ennyz&9X18gH
zp-<uzA=f<$Y4P2KJ?|tY&*+rNiF4Z+kYe(f?ZT9WJiOf!t}4q^Z|Y9;f9%!yvBh1;
z=tqj>uA9+cf@YZfQ($*K-Ffr0&JqJ-kxRVmDh_MEQ(%~}GbMe_Dpp&i&Q)92G=(e@
zXx6AH-m<Wz-R}F(lt8=4{=XBJJI`O@?qp=y7JRW?^ySGGv80)uhF6+J*L6KnvH#?t
zlF}&mFTwp*mgkx%mH^=zYYt7(A7=eGg}C<B%(qW6SzxE-JFDpb?GA0PjYe}U){EBb
zJZx&Z=D1X)uTbvLl!r#8a|5<rp1UtXj(z3hb*fC6N=rV+dc9dWMU^pdzMaz~p-96c
z@`@MLG#1_!3KyTDZyU%E{NRAtzAFJ*s$2AO-}rQ$DzLtIBh_eEcC*ZP6Zc}teBG1T
zyh#(nEjQ#e`8OuDu8J_^n%tQmc<1mu=@c~v;~fjRQ(rWt+<DMuy=RST>mokg1K-YY
zdiAc16K1}B<oT43oGGD?cTZJ2GJ!MD;7R0Hf!6C=1UPkd47gM$cU&=X<o5h~sVS(L
zqk7YY16@5%A-@w_ukb8dw8!nBLUL!*s!bDG7r8Wwv0k~l+3V4g`V~iNg=bFq@1hno
zIqb!fmTeplx9<wlZe5m9sPRVO?y_XlpT1Y8%@&AU%-6Ie@Xv?&DUwRIsoB%@&u!~?
zziRG*4@*U^Czg1Z-n_nVSBi*^@00jrtURgTrs_8q9@|+I7?JLgJgdmnEos`t-1OvR
z8;h`~ex*7m^E4(M-@9yyh1ubgx=b%OE#x}Dt9N6C=9!3%LVvD^wC6Z$?-3FyO9|rP
z;9PKxL+Pa5B=zaFGXA>P5(UCKoTf#dI1({q+kTA<#>ulbe5iP3*v0W9bkdg$1C~Fx
z_@i7H8E1a*4Vco&pb=p)d9}v;f@?-w<JK^7rGJ&Y+%R+7-Yg|9uEs;xx;9;T=XF`_
z!JeCblm5%B&Hv>*%U@|JLyp_pyIY!MIdAF5*-2aEEz~&vZ{`Q(;5L)rBFoSG^X{8`
zxyINmUBh`>!cnnlTc+NBI61CtTVQNfi_itpDIVKg7YU^<nR))rHZ8HavaSNP;(9J7
zsvDc{PBF*|yg%<{+0NhMX*y|Q(_VG@FDb~Cv9#q1DiPn2weYUmi6?u`o|~en^=*po
zj_YjJ9m|%h++3lZx6r`&U&-Pn72F#gicV|vJgpa7$l9hIlPD9`A{#X!N~CD(1RV|s
z?lm_yZm5$K;{Ba08>y{d%Cy6w$|0KPDYNGWB_qw18=8*u%vqpuD1G8Ssb~i7JsUL?
zmOAWhI;|eGE_zE=<D~x+zGhe&_P%uSIi!=j(V&gDU6I#UWUhc~yZX!7+cWPjFP!=^
ze80aHcl@<UQ8T{e`8(w3@*6)tFDDzg?d7G!-5IN6cZEc0hNbuxw$7gPFiESsUvNfh
zll13`z$4vk$=s4BE#gmG@FWR+_C6?*_38SA*1n?MkGnrv=mmbeSlM24SmADfhq2F+
z3AqKWx)qz|%xbWDC~GvWpoz0J>Q}qm;tm%v6GJh>DZ0XIUbyQT9eiYXq(V0Hw$x39
z2g;tNy>@}KZyo(x?9H)e@eHOpv+G?H9d`ApYpC?GFg?+2NC@0u*T|osBITi=vEhfi
zTG-#foXV%OVrG8Zx+8MR@593EEB{BT?o}6@eqJ)F{MYfh3weq^{a?Pq%>4h^Z$DU1
zhqYb)6SiMObhf^@$At{W1#S95H=LSYXj*Q((x<O<{<GcxJ<bY0E8CwLopSj3QgYIV
zlS#rWHO`*D5-`uYYEkGW?gJ0@zgfg>%sIn*cXz#|@)-`zoi`+Gl;=JX)O@eRd6vO6
zbTP+_Nt)Hc6NR!^_up6{u*XF3&WfmxeXIv3<O^;SRb2XGf_Z;LYg|<82Tenb6D?H>
z*aKgvZd=$__hI9KPZK|9Y~(n+uv>zgM~kb+!bT;cmG`ykl2;uPHmg^CXrFjt*2EX<
zIW8?JO<-l6#6M{Pd+Fpgoe`}*9OjZ;3x%32OiZTyFZngwHmLK%L}wREvp#Q;#~*e$
z7EapHIonQ=d9TI(y%TtfBwg+toZJ40DMv^$zGHTfWcQR{rH7qIBv-E0*r_}7wa(sx
z755HJ(_YjuPnpv{hNnJruHH+_z#}}|FIV(znj5xdZivjm&Vm`nN}X?4JN$jTuxce&
z@ZtScjN7cay7wlwi&b_#tmw2lv@ghF$tL4QJ%h&RW=VF&edm=H7G0iYCuGAhVf&)Y
z4L>?MPCFZRY4AyR?>P|7p0(SspV3fGX>(nNqeevg=ADKIn!8E`rpis+GnrFCM4GEg
zqIr*ZfBL1Cg6vr#QiuQF?Dx)e_WCIO)?wx%Nn^vw9>&#6jDt7-@Afb<Jv>3>wc5*M
zN%IWW2|3L{5}wPNC0=v!TzSE;GlKUL$Eqt6j!u~~Bl)6mPT>CY3pp;F++2OiQ*PxP
zwUzcNN!^|?D^_L5E{)i-YKPL52aEO<XzUK)(Gxhj%D`dg1C5o3w;qUaNKn?4Go1I(
z(sI(Txk275n;llJZ<xF-!SwGffxsJk+l@QJE~w4kz#40;{FrOE^$xk|5nLt3I--oC
z(pNb0PtRHr#mdv&;<HHfQ^zbFiNjqD{69a;_g~naV$C0Q(NTATR=0r77mm)V*9uW5
z%y%7_AE+TeFJQyWs593hPAn_fk-czU?_^)=FC9+~Xxg%v1#zr2`{B%EWAF5Y`+uY0
zIVT>8r3we8MD*&;VDJ-|!lR-({nv7N#(BT8Egd%d%{Y8?*A9C|iCvv9c0N+lyZ6ZN
z+JT+74({A5aOCkCE^bfP+fqxWB%Hs%;c2WS+~wJOsljSZiE`J2qq`d>$4}`zDK#aL
zNq&)o@45~<;}=Z)0uAXb)~y%ys}0l-YWBxTaS9o4OkcEg`$Mz;60Cx^C%&1nG4R0>
z{~(+172DFKrW^G%?cdG8-PB|(!cZ@BqH@uWnI{%rS6N;x)i>Eu{>#M;9e-40nU}7r
zlz-w7^rgtI@6Y0wlkFn8Pfk!d`Fgiq*F$a{O-8*FmvmcCF8Flm7t`hGYxE=EG#Qv)
zHqs5g>=69lsC32*l}1etd&Y*&y^Wn!4Ei}MC(3-B({#vK>7#m`k@(NYf(pL`r>yW=
zer3`dlT*RB*R~1q-10m%=du0nh@)#t^uKF0n+2R-Q`2+ygofp;13Nn=OIe=Y9iX`{
zquD)$nWv|7n!}DTi)AV$zFa?B<BY8z2U+D*TU`Gl;=5zx-%W>f6fU0J(~+*YO{scy
zU_r-T<->=2`ehsCzEv#aieBC4z_IG0*#0eDy$jsTwg_D;(z#VK-|CLVoduivA3FzJ
z_BH(J?D*&G2@bzUCVG!dTq<?_T;JN=UNmhM&(&L=ixwR|(y;g3?gu=+F;}uX=gvMX
z_Q7V|wFf<qMS6eq=rtEM|2Jbe`BRIzBQyBXCI8k}=dXR3oPX>3#K5VCe%tJ_5UBp3
z$@qbZ-{ErC$@StdjYL;<)HN(vxv=AKci<m|{;O}=cPby7UpUdHp&>b{edDR7FDo`g
z{`3xwYz<9d^%PpVQgmY8+k>BP_O;6d{aN6l^fszzi)8N=k1w}8=5*Mpmj+D`IjK2!
z{p-m*Mh#ZZlcTQ+TwX9sZ*fGgi73Nf*K_BjE<c*WkTWTGy@Gw$2A+V!*S}2ewmYnT
z(UNn4%*s`_{q#MrEPvzL7r5oj(K`Vu=R0pOoa1#}_GiC`YNy^zuPR4}{z*J~2?xGU
zK5~8S$ten3e`(L_5A8C1&~0aO$12jBtLf^0onV=7H#vVk($o(KFutL2w#YXr(6HWN
zw*H<S8?p=o15`67>l_rmHpN4AnJDK>E75aZT+c+83MVd?*c7G3bNyyWeBb7?uO)Ys
z@UGmU*{r5?<l&s_zn5OGz8deHD*Si}$M3uM|CjP=Zs;vdbTN9d&dP{eVUlt<^A(Sp
z?pI5=-y9LUAhGMR?D@JxUaeO#yli(rhO9XDqx=3AlO@ib-+RuV4Y>XD){RRAR<TFI
zZ)~_*yi)lL$AyDmC361G6m*n)`azTbLsPoTJ+m7gzbg)(FWR1DJwHyuZ3E-%z22fz
z9j3}e&5`=mCntNq>u{8g+|sf|9z82Oo<szxsy?=V<*~(g_kRstPp#BlI<=?v$DW*#
z5^UP*DZVb+VCn7sccZ&Jg01g8?tBm}zVH0>JJI&fZmXY)w)0EbHP!X4a0740<&Tr*
zM7_8!c~iZn$o@z2ekD!~CtIPP#*<4tZ=5<B9(ZJut^@CWkJL`Bb=4bJXct}Sj-26p
z^uGO<D?3(BW_zZtcuA0R&76N{CQr$5xD{Y}?`c<~_OhrAQoCQYKkVgUt!;kNyRWU<
zbXS9%uh-#^5u%=+hXof(e~DdqA!@n5;Ps+?hl^|DCv?xA@XNV^_1vGi*Q=(cGaie7
zsFG6Ucc<pr&h;7RCU0nMJk7P5`}8UWFRwi=6`FH*KWNdJ>E);L^k6{5eCI3wt9FV9
zZ0Qz}JMz_M^6TB+lM=j)H1*P7UcNQ+;f)nt(}W`VZq1fyQ+%VL82*AqzrfA-#Co;E
zQ#e&;O=~+WttvI|VU%5$M_0_|g?hIa&D*3_8x$FQJ51u_q?CtOw|4hV?b*Nd)h^Y`
ztCu}q^DlYU?&r5AX3b#9ZvXkJ^8o|XwB;YWVto!8pNZT*_s&Uq$(I*2Go0tH^_6()
zvUG*#-aAt~y!O_axSzhh??%)4uifXr3eC2T$a$!7cTwtz2P|xJgg1BG;owo)C2@Mv
zHsL9Ycnu%keZ6o&REON$KKXD4tqeWZ@Ex7|Lr&!69nM<ztaR5!Cf<qN#_6|wHun|F
z-HA!R`^Wjeq2!VKJ=dLE(izXKdu)<d!<O*x>e?rV(y#8F)>doskw@a0S0|@{>m7z%
z+r}f;T_?SKGO^H5seGkwqiX)&*XQm(N?5MrFL2EDLkaJ>4HJaVt!QQxpE##yz1_(U
z$2Wh^NL<g*+?R3g?u}l<1KVz_6ZxFgS(cDKyFiddemT=p3k6pxy}DUHGu@SZjh`6s
z@v29g8z=YAv5RcJur>CvaZ?^+#7VvfOrrCn*R0Fh6WS$yKFjvq?X_}Gtk|CHx%yG&
z`9}j)PDi~I6WPnv%L>KAdzJ4e8?ru`@*->Tj3(X*Lem@`+&X#r`le4!T}SL?CpFw!
zxI=?=<&oEcW%;c_JVzr^|F2{z?lQ=!Nle{0ja%zMgT^oWD`&5;@bHAF9@)~je}Ra%
zow)d9v4(_Y9SM03HjL{tA9BfSE|a?`;uUG}ZXwsjd!dEPN-SdTe_#3b+u7W|r_SCz
z*7NJn+a9@hRlU!j{kyMbpI009HS{?5{uM9FexAQ_Q$p3tvUuTxYa3h^`n=d3F?prU
z``z<iB>X=jS(u@FFrhn)t2NVqRm=HSP2MPxugfn_vY&Iu+PG_Dfb733-&Pra$&7qC
zS)$+!->k`JqdY1-=JRF!7tAtYj^1FJrREy_(%YIt_oLQcZymu{wdap(mp|TpHA{7V
z*6K1(@$FfQGm502m)DwCELC{Yc6?g%^>zQ%^o!Q2b(&0N@S7l1tzmETH09{sVyTxW
zErcSDPwfuVV7}#$x|HX<$ihj6`z@EHd^VlW@a@~3!{4|sg<iShTJ-4Jsk@dhPq}r+
zm~ox^A?qMsV)M}7e&U<=Msia+G;MeBdEUP$yY#L1`nSc~<WkO^z1t#upY2`wy7X$_
zys+&;QRmM6ja#&F`nwl#mEKdCESrsY+%PWEoci)gk?rxdyA&3En8!VRcj?X#uXU~R
z&+QfGIXCx3Uh3($4-&8cUUIp#v~%6k#pf@dxNd4>vU$J6s@D+{{ZH4GO)T57&_Mg*
z<HwQnO84|Ayy8p0^t}7~E|<4U^$LHvJUMlCk_FpqdHo8>|H=P1_I;dt|L^ZFf1Cb)
zO!)q>)x6H~-D6YHf9t+oYFYmH+56-!|01>J9~F;Bug?4T-2ao}^~Xjx8Z=jU^)A?D
zd~msK>1(ayPx{L3okEo+sj!-)7jNS8-x4p|8Fa(cysLxjkNI?^Q>+|%Oe|9*Gd?IB
zYGJu664DW%Fx^Oyb&KJRt}6~<60%$g$rGj??UnT0D5-it#hr0B-?oE0QWa+QZsXgw
z%9LwqLmgY`uhWs|@1I}b(Ed$Y>&uG3_^&y-Ya=ILIl5xWtpX+GhBIQTqd28>UToFA
zzCLLuUyj8F@5l|B*Ei*AGvqq$$a{WiZT5G)dwV#x`$T9ixOTtsYgBHMGnev-|1F(@
zO)NLP)*m`*@_Eusr3a}_XP9)PxBR-W;6Tel)}Cu?A|JlE$jrL0rNY28g>k7CS4GZ+
zvv1d>mQ8=)<;1|Lx1YPH!E!=b&Nel}M=}~lPt{g$yCs<uzFKzX)PGZc-Tqg+d+)n_
zZ+<e_*ZtA2-gaemdFtt@v%@~#KBE`f<|7%jS5-{CcEjOF2EHq^9Sa0C95{<)mpo`-
zn<{Xtnai+aL5%1*)5K`8c~fJ&B#yN$Y?oP9^1zYB$RaT{+Qv=Rcb@8Y3GL3tCl`B|
z8d9dZNtUKO4()P%CQy5GZdpL~!j;eD)!o;%&6>2}t9y!<w1mpjtaY+emN?23s3ke^
zY<wE(@_(VNo3z`K10I)()qlRY=+HLtQcU!0nPrn0mZX$l@G-k(!WQJNnR-6x?TsK|
z?X@CYS3}AkT~`i^w)7HPk~Z_`!X;UXCM%cng`P@{nj5!DVA(u55v?V;rj=S*oD*N&
zU+U#G<<%rj)0G#*f+V86leRL1scCCWHCenqt<1!$x9@7%8;5?)Y{C9*icfNiv~yox
zm=F+kTFd?FrKPh{gSVJX*v1t2aN6c|5-oFfay`6uEHCxTH@%4cKc6ct=bE~5nRBLG
zh??YD1J;O5Z?|mpl-{;S{C>^1o9*`(En)hR(#*`L@t{dfEMif!h}sXa)hhec48=v<
zdlXu@a(gbElu_PcbnHK8C6E3d<>Wt~nwYazY&yjmwddNo{M?v_=WRFNIePBB){4(9
zn${T#msyo+KX&qpsW67Hb9xERshE0G=z6qzRp4B?i#rzIurVtO=rLP1H^5aq^IJ-N
z%!%6TYZgd`sO$(b^Hg=Oiq?GX`1shatmxDYvxR0X5e*6SX5;L>FsD@3?UF;J@|8J>
zC4EK98H{<Y0(6(;OcY(bG}<&QD!wvm<#E1)a|+MsT#x?oX6x-b>vuc37r5--W)arC
zKU^^SXX5@@mR=uE$j93(3bMR!^W~g8_gnWfYT33<UzUAY>lE=e_4~Jm<6N`0vsYTT
zX3e^Kc4DOH%-eOjI}}sqhyPBUBgXzeW{1a*H^0`EuAj?tbWy`rM=luwPjQwh2Y3wy
z6h(s+Sa=s5n8~@oV(-th&7Zz6;44UMmhV2@>hwZPz%xT{@y?mf0XwD%yj<7H+4&$^
zX@&yB*Jbu8GaPcG79N(pr5vmDP+Z`(h_K?Vl^vz0`g0~^%s0|BwEwbFxJd4?pyo9;
zx2xVB+I!g7?e0udnDgVokDetedNa}#3bMKHi!|Bk6)+fY2vq)Yqe-Sza$SLdA@|o;
zDf(MfpA@nsPJX&>tI$(MBWs4&F`KdiyL1cM>MJb+UdMe_=yd5w5!#tD<J1LK_pVbZ
z*@~j->DyNZMP^M|xFlNUe#Zp$m6LQ|Klad0ZQJz!_r@G9VTD%hKTA(LZcXCu{jH?w
z6}as1k(L={tGq(jHTf{AOr9B)`Aob~^UQPIZr#c?K|gLR()IhB+np)H@k>U@V)qQ~
z*yCF2uDe_WH&19x`&6}*TXwm7yUz0RuRHh>N^Z32<}8@2{kr!=M&=@qq7RM-A1+`O
z3<`MYl4AZ(rSah_W&0m3T}cyJd{y<;b~l)1Ecl<2@W#sOboZ$~7th71=S-GtX5;f+
zV)y%nce2-(%@rc)#=V-So?LRAEZEz^xadNCnA_ocqmsCo8BUT>j`Npli8eUx>6+`a
z{7kx?mqOUatp1+GqRXc-b*0=6>^Zk=@z05cp0}5D{|b6JfBKI97kh)NvZF16ujM6e
zZi?hOJNcQ1xPI*>W8uJQ8xEu`er}<*;kVH1gxe{T9JEp_t9tsn1S$nwjf70r2l1+;
zUf*;zVbkUXvpS@`8m0GLVozi$Xx4t0aIRw0wyo=Gb<f**1-0`S&P?f0Gfr%h)V1m=
zId<~2xV7@x7ooX&UURQ~KgRBMXDZtqj)!MNb}gNCvB$We!&-b33&Z@D9P=%^6+8a2
zxwMr(^RM3ZSlmx$_p}*b-eq0A`JBx|P{wG(#uZMXCnQzfHr-ngR`-puuel|-?O$1`
z!hhYyun(8xcWp3mRw_y0bkwk$bJgdNz%}pY#Mckzr){(Nqv$btQPhOn&z^7lZ}8Bi
zvw3#3eihr%IG)gg-3uNymK;*hTDP&-M)>L`pQWoiXDK}`F<Ex2gKz(x)vr^pbgfmH
z{Om-|ncL~_l<w+o5j=UNH@S#S=9G0&V`2N=N7J9~)IYBJ%wXDQZl|zKlUz6ph0mS3
z951KL`fJ~-Evu?I8m%f;-Tks%ar=*VD|TGCzk1O(KHitSwOe0zwOI&FSupFgrsvI*
zp10SXTU_<!&4Wzw04>eUGh2JQVtXb?_7~|IZ`<hPH&@_}(kj(J!>KVVk3NZ1{=O#0
z>7C;e2EQ+@!HQQ?x$`bW3jQo)wan$v-Cd!a+03H8y6WiM|1Xl%>u)}uVEKeQWBcmZ
zI~f8NRmc7(oG6<Wz?ZjMVS(-kho)eGD?-axck>lUK4^Z&zpiAnCG)Drh%lq+E5D>^
z<z{*Y-g>#rFd&6xd-=Y(Z9!+xeOmhU+;;b5v8_u_PHFle<rf|!EESv<I`f#Mp~>ln
zX#HnpPwJbySMdftUbk+Jvg4ZbL90EK!!~Gk=lm7Be@kQgPR;POVOrV~-Gb*ldg04(
ze~$O|uFap#SGg6~Jhz+>bV$H_MtJCI&k`%v*G0!4v}{NS)7kf8iolt>ueB!no>*|g
z;CYY?kA3Qn)}4n!rnSv#T|do~_0H)awyQ$b@3@z3X84zLGic8t(X@^!ZNlq+9KEi(
zWPVg^{93)rBf(KF3*Agt|6e&X_JWLM^vSi;m+>00YCa4w*}LtX!?B;Qn|-EyWsc0X
z+WmUn>UYQYOrOe|eetxIY__P1_eQD2yq#~uVm@u*I$X={`npSqC*n4n7Dvs6qP$Cg
z)GYP}zE!z<_HEm@?c4(1i7mNOb)uU>7K_{cKUk~v@6+b_hn}tf@blq$v9G7+D|7cO
z%y}F2JYd&}BeMP{{*@g2Vd*kS#xuM&uy?6UALjw4vh$8p4$TPZO?+`7YlrCC7kADz
z`v0prHo-<Rv*!NR8&O^w2h%uQ?rh)^?+Ko~CN9>%MLJ}z^Pf;F&vQ0sFW+$R>*HaW
zCCf5PSImvaXZGHijy2~*Y>q8>5+u#RQp53o7Q+oj#|a0U6wlq8dFam0LqcNm+a9>P
zESa{`mBY1`bD8bEO|iG06nZYn^vY?uH)&_s@`L9NE$oxH)VlcYP02}fWVgAVTIw=0
zV_wb{$DcbciLpLPYnXS};F8%(sjNBk;v{BE8}$9Dm|4>y_ICzr)6&@m20WPu65dUb
zjpMm!%j(8AvFXFf+o2*YvNmkHZXOqJi$2}sCi%pF&%$fXA&VLpiiK;$JLfFa&GWeO
zWwu9U)M3V{23oytOMMRYh8)=G^Otk+YnjEKCnv?82{X7SeYSunO67vjUOtD1JlAGk
zH@Ycik$38;uV`NH<F19bralont?0sRc7&-+)aH@guHHG1dHxI99Gi2yf_tj!%?rKv
zM79XB{W^Z1=gHk2ZI?Jx-u_7DTBeZcAp0XF!TC|^w3njRjjob8jB=^hU8h7Q_Rj3T
zvS7m(uN!X{oaPh@lTdV8a`%?(8R1P&9L~68?2T6IO}sO6(YDOU@EeRZJDOxW#3D0p
zhI=IcF_a68iU>T{YBuSp%bl5OyIL17<%>v($eyWeVCk<J;&y&*>o1d~oVG0M65CQI
zJcx3+9(b0Ed)4L53D;luUdy?{-_#@CP#Gb)=W^keY46@fY&m;5Y->|BuZwle5tafc
z=@ZgAQt{3=*mj3r@SDq{Vj5rXD(`sYk+z%K{Ik*FHMdNb-Tysven{U1hN%BD^tZ(~
zubjXCq)%p7+7I2w=Pz7fx_TkhCQ`h`?dsS0441A=z3?pmZlcw$`*+qnuJ?^n_u&d%
z=gMq#nR#1d(8;qeas!`-F1S>gFnR4%Nlqmvk=WQA&+Qg1ObwIo`%GW2di+YEkQ!rL
z?6l|oXM!JFw2L-A*|$K_^VI?`8MdO7c8?0b=NebN@}|2)Ec{b4bALv3%DzVFFLU!+
zo<Cfu7{`&S*{UcRB=^=L^@(X5&mpfVtWvFd;?q(y!fsu47j605b2NO8U&NOa9laW|
zbzB{jRxlK$t~%>?>gmGATbwkfsIJSL;ZVS)#O2WUDylDQdEYeE6o(|MhsPpv!Z+>Y
zcK`FAt@dPZsLgG)V{SfMnyPpv?C)5lKT*l?%@P-u;|XnIol_bmXS6m2oC(m(keTGB
z`qGE}>y`68XIy`Z7#b=6Fb%jgXNu7-*EwFwXKrN1v4ogj(!6qjn`6__l?O8xHqGhO
zot$@%l}RnI;Ny9wYZ0f9D)&AOV_GG{{7lS^LHJWH!><ijQyqND(mW?K%08&Me{myg
z8DEmqmlrH8mpS{mV|)8KtXigj@YmBi=Mv*TS0v0`NOHZEd$MZxhABs8OSDv95?gd-
zu@jFMW2~#ltw<rU`B{!nO9juZdU12f%bPmuj;WlKJuLgwXqWtgZS!{&aqW4MwKC_<
zu|&7dIH^3V7cXD@=kJ)>y1PnUXXlG_fhUt@EWVS&*efTx`rNFxXReaDZpD7y{aKTh
zp1JPaBKLY@^zI6-&MVWq71(FnWX3)7wsmri=Q<D{rOCPDuHy;s6t~qU<&I3A!I3S*
zF=^T<9ig*HUP6<!qPKs$Id|IYU!fWj6VnXamKi?F@md?uV!Jr)V!Yv@`<Yf3cnXhl
z1|5sp@FH*CVrLO`rc<p2X|D>Ro?V%xe0`p>k#6kNje^mYTFMV6I0T%uv%B8*$+!2=
z#GgJ5>uw6U?|HR))&h%*UU54XTq&BC&Lb(`E82hJGXK>C`KKInRI;DWIl^z$?p1N@
zYDFKP=Rxl&QIRcTjk7e<*Yd0J{%?_dGkYHIiY(3*j~wKix)weS@{%x%TDC+nWr3Ht
z?+o>`cbgX+-MwzbzYDJ&b692Wy<2*BUD%0d(z&ktt}To{5_EkRYZ%L$+t;|GOTAMg
zj~(ig`zqBE!Pml)!?yltTbyL{8^s>ym1|-{<6SD;Bx`cHJ6V#Sa!d9&Y5%#W$a{({
zL4{}6o83Cg3hsozIlwbRZ`QU;IU>`TRB|WnnY+GBm&KKF*4Yoz%>2pHyEHEZ#rS1L
zsAT5lZ1FR@7m~9_!|L5SOTV?zF>y9!O+7awBvlfeW9D5=%PN$b;ke{cjLYRqx_65w
z&bxTm;M2a)Rfpax_;ug;w}Ih7z)Q^|tAF(}ocRB0&C;oHcd{h+xj6=$PnJ0S^~b9O
z&qC2=$?LsO-Z8G`n{+Vg%8}Y_P1|3rbFlmv`tj!9HzKNcFW!55%thwJYMqA<C2y^G
z6C3_g>C4l(&+pxAJ1BOk>DkV*Wv|!xJfAf;PxjOQy%U!vu5jI!m$zxvw*w14u4{Ez
za8!J1ljIeN*HN)@QKARxia6FON?h$ta}c*sd?e!Y-jV0yyEU7H7&PN{_`i{Cu`Ct;
z>v1c;;>Oj#6JwqD9(GASamkGTw3M$>OS0mab>x&Cnu^<=tYdqq|GDs}klvA$eXcsw
z4zcZvZYo$=XD0tnbi<y=E0<O+lxll+&MH}Q_sc^wpQW|Ez9_bqW&Z#BYc$TX2|bWH
z^kNm`0^8@G7rD)--D2=_Vz$$P4cyCF>}KCOmw3`)#@pLoo!j1QDBk)i?S!t=icO9w
z@3juU$~w|3WwrF~S5dLNxad0%<azzQ6y6o@QJ;H)(>0-GR;A&ITk0QnG;L>i>bLOa
zGcCX8TTaivd%V2x{>g2pWkqhd3m=h|h;y0Z<iA_#PHd3omy|t{vs_$Wt4!{C=4X^s
z`lVHD&FV9H9lWceYPnrXl5-cW==8e!PEY*WC*$QcfsQ&4*q>)lE-08<v+2M-(Fv<2
zW!vP(#=QS3)gr)bCir<)7(Yv6)y&FM?vV`1yBH-_Tdd^HUGtD_^K8~ricf`<4gH*b
z{|D{vS;H93TJzwzpOl%S#{Bx0GM%<F$KPa076jB^y<Yd>Nv=+mxPZjDMQZK|X{^<y
zomVos-<IvocxT-A^6drF8{0BCZcD_lsCN}My`40tczN1Vzg0G62Gi%<ay#>(cVCv1
z!<UFK6M@Z}7*g~rceje{(ONO-VvK3=L9e?Fo<+wzS!ydIUH@bqd-fpWrT6+JC&jh-
zV<TFx<@WvG(xlyUuC^^KMQfq7$(4Zl?{zQm-`T5>wUbFCE>Y5<bkEJ>K}FN+w|tEL
z5IW!W^_9+R<+-2UvTQm3DrfSN9S+=|F7>_ktksTq|H=8owY<>N|1>}S-^wMvSM5(?
z>0TKVj?L!oG5<vq-NOaWzYVmMOuy3gjqgbO@*^xiO10TPI;uE0c{niK$vu3@Rrhp%
z()D8-I6u5hm^1UROXb^+?vyu<8-7OZ*m+vArRr^<{8oKwp3m{yn8ZCCb+oQ#3f>P6
z-qAGKaeC7=!^cVCg7fv#G`HWJdBWnE-?49NjpmpCoyK%O#PNn++AOZttYvdHeKz>?
zGL6^3WS{ZrJ!@7ne_ww7K9m27BsCY?@(qQLj}~m=KYm4#OY}KoQkGZE+w13kt#~UF
ze?#c5Q`_pk?NgKAyXsd*Yc$@i)0(>BSlhM*4_mn8<720vRd7#u^Wf8`m>HbKOTKLQ
z=22y*=jM_(=O&Z##75nWiamk<e=z#H?4G|~`@;EZm4!Jsn7({falUgf`AzID7wy*%
zm#j56)QSAi{LC!h_{Gk!3_fEc8Rzu=;)@e%x+9dA^q1d>=y<XCg10uO(L0Iq82wdY
z%sT5`uTRNuyD1f07G)CM@%^30fikVeGs#<+5=xvj7$eJmMpfwFo=~=1;U%l(w;z)X
zQ<ksS=gyA1<(XK*SGu8tZ)L2r$}HVow`)r@7r#H|CnQmy`&azkvEtdYW87ERuY7qq
z=J92Xnw92%6gJnks&W)&$eh2$x8bsDN%x!+Q}3Hfec^r_w{(}mFQ?gZA5WjFTy=Mr
ze|)-S+ysTG#{VAvkD26Y9k*ja<m89z{P^7?{$Dh;JmO}@{b|j5JN6#}yopMZvYLC^
z6SJeUH17xKvjk55;qI^VB=L3ABF!DEywvt?+};$nvf)qqyET_)-PfKgkS?}AQ0l+s
zcH`(f@0At@Nf=7zZ(M!wy7t%4({As!+Lq829lTcK^0br7KTT2oIXS=Wv@**%ewH)*
zEYnuoe_OZTxy3&5yz6ZTt|FU{yRN(u3qJB}d#BR@O9u`G<%R^uMm9k%g{1)pG}`#Y
zrFuLz@Hz&im`%(~U7euIxHf9y#6=f-S5L}2C7_eM$>+3?7WcHu&JEs9j7Gv`GcyfT
z56&`Wo$^3w%8GLfEWBUnBp<ut%V@Q8i_gzvSI^C5TVJGDA^KnA`r26TsJC2Z4@0LK
zSKl*PZT9A7lWdON-!6gY*EXa)R%)3Y(h$}hZ6WdHwYi@9Au~(GusH^W2M<?X=E|63
zX5_TLQ+T@8<Q>6|XJ>w%_G!-al1#=$oQuRRSZ+<>3syT2X4JSqgfTcze2Z>FSq$fr
zsUkdj4vqz?T$4B~{616^sR}J~Q9se@{a~Nj_8D#+mr70<MJ=uq37DubyXgt*hD&|{
zwLkZ)QOncPn6L2nC--d~pE-qz)_pSBH#<ab-I7?bbjCIB7aK2KOEOqx6j0=RaGKww
z3!+oE6g}pf?6sO_*;gr_0GF0ELWKuki)`W4IPK80((2MCWnS~9BpnUw59|Ij_xA{M
zOLZ)eb$2m1p~xNWv#Ci)dWGTfoJ(0JS$57y;?`7~k-I~)&6wkKOI5eFWz#+v4UXBz
zOgtK2Y@5;KK3i|iim4yZ2_IMzcIUzy2O$BY00VhJqy7_$>Rs6j9R#B^l>guQ*r71N
z<xvBVO{Ug~70X*DWgp$##`Vg1myGtSrAxHFcCMYwv5G6$I6}*L%Y;Q$o3Ffolr=$L
zD3|Nw`RL?a?(i7<w~M%CJeA*UG<B^J@{H*<?U;Oa&*IZ3FE5xnMd3`Cbg#m-1){#U
zV++^%nQ`{aZt&9Le)VZP%Vrh7SjI^K4LiyXmP}vpDCu@W*MS`ik_;HLdIW7RDcEc<
z`LOT5ceY|Ucb31B<I&&<+ZNHRei0|`q7!OHZh`!^Modp%EROQHx!_oo=~08F{<pOj
zFPoPpbxMCxQ`f1~h<q;oWz)>wtX7Iv%6=(!afijrgVA$#9X@<X&o#<5)Ag+8!6mb&
zX}{VL)?#&O#p<-Etd)WP&h6UawoTAW?f)z86SK^h{>(92;i0iqwEtOH-i&EJI!jyo
z#Lng^uD4sUbzjQrO<qr1lrsvRO|jmtz$qjUc(0VXV$Tkf#VTg&xxJ@;P+mDf=t1_>
z6C3mvnZ-N}cClx5ko|f|tx03O!%687<>*(6CzL1Ln$Xm|;>3)TD}*LAt!fDh&|3I?
zfgJDpAL5-wH-F47``^DZGGr(Bs_rQ}yiYC>pZjP6^IJ}>ms=*=d^z=N(~7iPfA%rD
zX{&v|Tcq`G(q}Ef+N~!hY5tGg?!_6=6xXb@QTCb3A)zx9mzC;Vy5if&QIgQ4^;mhY
ztG1$DR9Loz#(tK6y-hAbyoZH03FvXWvp5_W<K(wvo`I^0kG6;d(*j{>J%%)^YY*dY
zcnB}t@iE0J<-^6<|E6xRIIC;BB;l}zM%etH916;hXGYW-s4@JPaB&FAVm`erS#)OT
zD<`wbH_MNFU$8G=g=*BxMXTN{*cm<Jp{B+{AITL-ZS0c0Y34hdHMyQ~g*$lkue)(U
z_l|<%DU}m-w>Dn=YmqvA(}SlIT(10Ikh$yF-q?o|;*)0ksW!5z=zTcp{@-}P7Ol;P
z_o$qnc*;slz52)mR=Y_#yd^w2A_^9j7nganF^W6-NiZ<J*ksZ>Timhaq<48|cM+3{
zuj_@+%cfk&v|%@z7a7E~()ZT}5yzNVP049T95?py1czL4zcq39fh#L34TQNk<(6nU
zGQF)9bzJPJSF*4%XFX3zR;!i1iqzzih0LZ0FMehD&=$nunxQc{c<!-s#dDkuArn+2
z%@j{*cl_gX-?1?!#%88+ubYYE3`?H08C!LNa@?l2N~t>8bzI6RG;v(g5j3?+__D&z
z3ks=14Zf9{%bHXa&5T?E_ovNJc=y9&O8gTRqyI-2&+XWfv{E=SD|5y#m!^m}r4r1K
zLJYt8y5=qs&AhZ&I;Bkb?8!BLf=Qm9w-VgUCoEK{&T-bvDfHiwrIw+V_*(LGLAUL#
zHxp;9zAPPeiPd~YWABPx$qJ_G`x{)3WwD){by;Dz>1tau{obomsfM?gERPM#p7m+g
z9SzA_VO%W<b%D`6Cjyi&&k{4LNzR-wg=OttRn4hOOrEM1u8w?lMvajxpn;{1d)eGa
zMK1lb#Th;c?PPITUL-py*wG+;f>j}(Kl9Hy!52<$yv$N)d)=p_{CCR4NpVL?D?8^p
zrKR(0-s4k=zQGXVTonG|*+xIn!o#z}LeIYxdOt7Furp1^>3>_$4aVX^UFK-XDB(w~
zpHy^idMTV{n9iIS*qoodH1+jz?Oz{V>wnDlj`d1Qlv~z&ShX(4UQ}}Wsf?w1y3gbT
z0&1qq+!RnWyyf7S5wmSy$29}CH-=6|KbEH~Z8d1#BbgzzNQz;-ny!PPScb#ml<dHO
zQ!;l%OP2{;ek#&uqO0S<RO2Zyd!4}<HqQe&35(UF{+;Yy7HrsCvtvcmp3ZZ(0%thB
zn=|i<B8#273Tx=w<hUS%t%@tx9u;9!Y1pxBC2v(ooTSHb!O)Lytb4_lto~9l*Lc&_
z&_dmS==@EBd@n_-8_#q~DzrR&w`W^ez+}<pOXgb}ck(XIW9CZMo%-nf|4mi4=7*NJ
zg<9W|zj>s7&%LR+{7vQx>XUA+R_VzRxp4cKY2PyQr(u4)LYArle!kO_Ula#B&HK%@
zNpI2-;U!fkyuMGlbXLvx(NQOh-BmTgUAYDN_561xoBy3|Gh5|e+{W#;LVtd*RaZRh
zB_-n)+In8J;jP2wtiNfUQqfnpY_l+{pOigA#aCUH{p$N9v$>M2S7(_0jOh_eahN*&
z&kb4INx7b@@{XN|vJQ=VxvhscWyZ9CxVW^1yC)dmdBSx0%x)FFsR^5tKjb`I^e}nF
zff;Y-`hGIi<7mE7-F3q1o6nlkWd~0<+)|j|ox_wU7hK}8<U&UDkMK)Ne@HsooOS19
zJo#UATSsxEqKl24<B42BqvngpZkU|9Gi&N1AHhtmdy(god_PV+tMgcKzv=O5s=*q}
zqJQ^2)_h)T_}t`>pEhqtl&tiV9G&}DI;_v@nmb1}3!8Lgh2=}?$*32d`fHS_GJV3F
zq-cf{s=ppBIq;S>v(K61;e?h3_nGcbuNBX35iUP0t@5F2`)kf$;+n?}a7C{OXLyr)
zR87s`fvwRMQC9z}`~5G)+jQ#J8U0kbDG*n4XiZQ4lt}M(=Q|#2Y(&03XD(EJxcugw
z^i;MBnlJhp_yx2z-e^>{YNR-6^&D4Km{4Ap;$EcVv5vbzB*lPlaeT{K53`J$Y3sJO
z@u~%I?$tTu{C`Q$SF48^RY^_LdYW_cJah8$7O}*LC=`Wh+~QK=DDr<H#B!TSfpdYf
zZ(h%n7OqRT)_MjdxgR>KwysafOZohb<PM=@9TPfM@f@;~S;T!a;li>mk!=e^1=jvP
z=f`kqou~o3(=CVgszs_DT}M5Zt_bV(;5olANGJT-W4@10kBod&k2%$vocWZtOknb&
zgnb%<i=UWUBrvHdOj8itk)`?7=P)z>iO+m5{yt%Pq}3xdacN`5>ZYK~8)8YE&IX2J
zN>(+izZDsHM1A~Tb^gU8r|CitZM)tY2&__2QQ7#;TdU8VnUg8V>ZQRM=hC43*+PrB
zJ@e1kJyF`SzU<i=*%?_(|J_y<zd5wP!{yy9o$E$=k1IU19~=|by!fGPMOMS|dON-l
zIf05RhCi#?%UcdHtzK+(NBeA*7gwI9#iZt2XO?VFJMy_eMPYkFiiYR$Aisut>t8;5
zmeJ*-FpYJ|HRTf?oHq;nmWrt@wa9AebG{|yFnv#sPt1vBCFj*D`Yn?Vb4*>Qu$}pm
z1?wY@WbPV{Uu)joy>|H3vz&7~)+O!86gl8mcH*2G(-I!HrU@=AIvWJF<I?_Zb5iup
z|CF&#bLulqdH<}CXWBaFtE4{EEMn3ManlG{matA;(D;0`L?M$IgOL2i?R76(8}ecW
zoC+JfnR1*}D}A>Z2x>iElv}X6PvFwl|3?ZvrWtGtQWBY)l%J8SXy?3bLF#su22t-z
zJ}sTmYVN*yf{rsZ)!wasz{a(^V2yFH%K?w%Uk`Hc&e&+;Td26gOY*F@(w6Su&(4T>
zJT&LynL97@bAx)-js?tZP8D^>|K2FnIK?p6$Y1;CB7VXA%mP+Dv6Bqq{nDyO>(pE|
zZv|Q{{rn*1kd^X=p0wgu77OH#y2c!MI_(~-<sH^DPgu`PT2OdvlSK#j^e+=*Z?Ia$
zP4dWVG1&LauJ){g@QLGIeO4ykEdE~BO36wFpO*9-ea1D#B&*<*&=!*r=h=&nY@e`L
zdF|WQEsM{{884rp&0z7x%<t0PvNyM5yxZS6y;4$E_<#A3?}H`hM3$X%3hoeU;tE#h
z_HO1rdpJ}rjfeZt8>S_qDhD`u3%d_#b+?r02yWEhGwE~GvY@zULVq{>U+|3Ydh=rw
zzfWA80+U4qH;3q?rnQ~=BzVw7=<u5DEDeR{SldJ%96J+o>)MpUYm0sVPqNcpIZ0=t
zr&x>d(hwPSBR}s6UlJC_X03l=AQC=>+j5!04-H|PV{2!1wC}(7YAs)m`=-~ft6nQ;
zUEq3rNJQ8r*f(pHRFla8J%6#Jd(U`OPKl`LH;H^&%(OhSEXwh-MH`QWVpw|V5~iFB
zFLdlRj!K(%cW8Kr?D|-Htx&ewShmslrv*Eoa;e<z&DL*K&5jh?{;%ea>vGmpRp3%l
zu)Ji<sgw|Xb7}O}1jQYpTw4^lloYtOG^jJ@>K@LqII-D?|NIQDpooBThKJlDI9}LC
zW_ix(pPUwYS1sAdA;8*daf16|$L&iU<FmOFR>d|gP_EVFs95Q9O6PXZyTr|JeRd=W
zZ(}*@Cp7s}FZVhQRkyIn<$7BK#MBlusj)VtKFx8N_bP10s`EBm)3jG~&ge?MkszG%
zMLA0G$GHvfVwNU&wO-0v#^Z5NRN|`Wl@Go-jAmQCFN*X!pVAC&bJ6VIoSBxIZOSTL
z&FZ|zNJlVGYsVMs6^nJw8~I#ay1mHXmV1$nu1MRlf(v?0(UTpxmpE`uO5px~#+2)r
z12>BU_oc0(5evn(B=AHn6x-6T*Q@PYuI%?;Fz1<x;@7WgUy9mKO!RMdHB8Dk=zp3Q
z^}aE0k@^o8`(H`liVR9CGjG+bSBMI?KDR{FtC443v*m>+Vy@3}%ug;|<YZjaAvagl
z;7<3;w&_;e7*6^)o9x$i+~dvdtK4C+Ozh1;<!xStstLkN!V9Ap%022*<eFZ6d!fFH
zPw3l&VoN?&bDCFkCf!?NILBjQklNEPKhz#;C?8XLs$R3)LRvtx=JMYdChN7It!pyM
z&t0=lWR~CTqI>s}!X}kiAr-EOgL20lgl{Qxt;piJ(J1<7o9vbb*-c);m!>MzG|Fx(
zSNJb<;B-vKavo=IAtn1>%_411*B<ePq$k<mmgc0*u(A-4_<YEoz3jxC)9yK<2_3oi
zvKnlz+l?~}OsqN_=USX}YY{eUc&_B06n~<tQ-xuQgL0m8kxkgyq)aoF196w0n5L~L
zTGr6BG=W)QYHx4A*&i&wR&XfZb4i@!dA{j{>sKG!KRg$5oVeK^`5Lr|m@)3D3gHsC
zF5Y!QjA=>d5{<&##4^q{ku%#x-5Pl;o01=OiRx7<9}4H~+>&1x8h>zNP?3`7VKJTK
z)4W@g1TRN09VpSgIZ5G8nXUMb+(Onivx8zw9>j2#^E}HH$!e69X%g+R<hqnBVj;oF
zGF|l5!{7fS6fXq5@Oc_kte#o)(xlDwl>Q6<y1h4N`EN9tpgV`XDd3c){vDr|wDpFe
z)2GG1D0NNYW_ufOScl<2>&{(gEjXTA&SRDryb<+Ra>2z3(;CW>WnOascQG~IYBIa9
z`s}-jk&fKQHk3W<t!7^;_Uhma!&|)f#BFXZs0l9p{ps=Ih1Wvt4~Z?(&K5F=uwA$G
z^2Mc$7WetyJ=ifh^74;{gTb*WTDP|3#7WIK{jGAozsG!Dr)a@LD+Jknwk?o5))4Kv
zP|WRM&CLeU8wc`tmJ2IASZUhGvs-hKX;Z=W1+ll5Zl9&R>=tW=yT9<(NP}IvkA8jn
zoaMVVI*(;~VA$-X>UuBI{)=33%zxLha!u#;#Y!7rvh^Qkmk*o1;YL!bq2h$sO$8sk
z8~9!(CF?e>wmn%g?d%qx(^q1fl4E`xQCnSLE7x{Duc%L4RXz8b!U`@^{$omI)BLw^
zsWkc93c3DXQg|Utz3{K|zX{(mcXLg0mQvBUX}^QZ_RP!jMZ(_Ojizo7ni?r-@Q&3?
z#&BB2V%^TeI|_bIzY`YFXrOoO%eOn0idtrhOAd&+IdYpYuG+ewz_?B1)YTbV891Lv
z=Bu~yoGHy`p3ae&v43%I;jil35i=ALB(@YR^?Ukjt-V36oA8`M)oo#!l7gFgzojYl
zEG@}bK7H$^Q*z+O&rV!3vbeLv6|(}l{?E`Zl$v4Fs(tjK@$SwIWs41dZDHQyFnb}Z
zYS$d|sSCcZTz;J=clqiq56bioO;+yBbbOTQebDOJOEH$4d^wFa|FU<#NHM-Sk+J`)
z;{;P{?q}t?0TpkbMl#QuZ(5@(8?jmV5&QO@GxsN125~4l)ttWdC+35o&x1eao4%Sl
zI@x|R3q2ea=cQru^XLBm(hFXvR(SiXraw8B^=6WQYz|+tTh0kP{gaD??ry8O;d%YA
zNaK&$v3_+40VSc%ouZRH9BeoIl<LZ4Syp0@<XLoBT$<rnfXas3FXlOWvDFlIy^dWa
z$(5_r|L{p_>D}x5xlQxM6n7a+U}fq&*5F?uqM|Zu_5T^KG8azc`Ly_ibJLestNjI2
zrf!YR5jOO^IL%9;FgHHK_xRjfOOkgVm0ff!`taQ^y0x5o*Y93C|J0k!$a)DwTZh5@
z+amW>3wxXS)wVx4!1ZM946%1RR`C=lKjdC?O|s_8DK+b+w?c(2!oSx)-0H0Kb?c#r
zy=q6b^UHEX<2RmpV5w(zP;{F@bn@J3ot*^*JD=w$-Mjg;W%FVswnGxVg_VW(^1rQ-
z@Jyb~(VCPs&$RINn-^<VU;2>AsUt4#b!zQ#RkQORmg&o;hsw>pDwnH#uKsb$=F(ex
zLK${_jZsYyIrQuGPSNU(x2|!rZxUAWUgbOg{vMxfbDPe9&6}s5d%o}8|MzzHIhC09
z?8`s@_5EA1r?*?@v)cA@)-Ll@U&+yQ^6dfzsrrws4F@h|DVT>x-1?QJE%>>(FKS*-
zNAHtu%X-#XJDY4^jTSRGm6eybWX3lIv5otyRoVqij?X!(etTNRPn)&UVy&EK5>6am
z*wrWe<ml=?k&JJ_zRc<RNj5T(SzmPuw>xkzsa5c4&tTn=v-W<@;%eqILb_{zvwyE<
z`eGgZI`;la`~1Ys(evl$Z(ksL;=%6aZ=W8tJ|moR_VS%~AMK*Qiv9l_oBz|Ei^0iD
zL^p_o%VSe2*L2;K9a9u9FPkzujEUt&QR&5L4bkhg?j6*4kbHEaie}Q2L#d}crm93v
zs`+m@iTg}fblSEpox*9pvy{uZCU6F;P3=l;m{#eTmJ&3>n^op4r|GIIEA=foUoe@j
zyt*uADwl%lf)MEqq9?gFREgxQ+?=Rg6u3yip^2eT_y!9@Fw;t-0@fp<0u0NTRIS<f
z`xsryb3JM%tm3J#RPAV2?utn%#=@UN&ZeF2d&3jF`BLE8SD9-}H-+ui4LUI)>&FI`
znMSEz7Anq%6sO<Ou*~^rn|O*vQ;5Yw;a<2S6KglGez?=5pN~|kZf0}}s4{8Bui21&
z^N7@MiLf<qI6I7%DOrcj%4%E|Cs<YU(yNPGAx>}~%imwUzwhVE*9fXK-fdtNnjzbC
z!*$|ahglg5|DTjt>~Z2$)tf6p{S0ez5~t~w^?XTmRXq3QVTaPA6n0lT&J9YT(jTT=
z7ISC}zo^C?x`okOKkUhEfAfD&F8kRpGMVfX7Fx0-#8b`ka<E_IHx(a0yQ?StT_QtO
zCo_d)3Qvzdw<XLqaof|#nE|(sczT^u6*v-;^Qa}<HE7DlPNoh;HHKvN&?OBb#y7uD
zDlt|~5MSu|BU93KGUtL99#ccNWTf^6nO$0xGr{SUhH`F`@bbkfS5IY2?K|e>Dis>#
z?aT8rOoY?O?;ztEVV?yIYqxwld83HaV}o|#ZZRvNc>!Oux^1@_tY~F8w&=K*_L?h;
zPHKGJ5YDZgyd{`lQ=;$E|26fUp+~fLbhW<D^}8ZlwNB*ay4~}4F>x+A>JZhl&~w?E
z36FAg#kn4E`~2nju-<;FN8?O4soIh(ciGS_8yL9f|C-dPvNK3B!0_gipo!^uI-%Wp
z8!avc81?2no$M|<^~y=L>Z66H^`38035`-?6+3I(_*CVX@P(tB=cyd^6jl#55Q~`C
z_)E2+#`E^cjmZ-xZd}1IqtorH$0FCMAwl8ptkagMG^Sn7{;@6D+gf$2Q(N5$&k1ub
zw%UD*m{(xUTh20T%cjm`A(w;C>%Atc#P5Fl#oA3I;pOV1Rm`zJ)~E?`1e{ir;5zX3
z?pu}$frevqrE@m0n*FGJvB13JO|ziiiT}El;@6rJ^}l|3?f-`9hjI1WE#W1*-+s+`
z__bpncfcZsyagOKeusA_PW7Kt<Ta_sHZt?0w~lN4r=wT;-_(4Oi~qCn^p(RhA!pp5
z?fG;v%BpnV8Rg2CPou(mEk$qW-#&7brQoEf!^O&NFU@?T4~Uu{eWA}`q3KsRgIQy-
zuA4#It$Tm8f>upBqms2asd?ERHHK$_?~JD~i$>~rv~fuJ1g=yPS?_mazVJUThuv1;
zyU#6cO}V<b;AP@bjV%SsZ4a_U9P4b#(qcJsY^QqM)g)Kz6{d#+uPBv9De%_u?673m
z$Q`R+a@~O~Xu^pz+EI6u&TbAfpTslwWxS~1WA7s``gQ;JEEHvrp5dXS=D4cnh1gPw
zOUL$ocd#k*Nxr)}?n{@`l!**RUIB{tnfXr|t}#eei+Y%#Jn_ktjV(_DU;Os+xfdkP
z@Zm!``$MKhEK4O8xIWMDU3^YW%lYKb+_g%pr_8!~Xq(8-i7HK^ip-fl%sa1aQO>>p
zQbF^W16P>GK?e>Y50)(f!fy&*#6>&sTSpmMF!n5zJ9kLh^G%Zc{JWl9=PwDoHaXaJ
zSM#FN7pcR>HydZFdA+b<y&3&NWQhWsD`VWU$pWo69J{Y3cJwIkjD5G@lmDDQ-SdrK
zwwJa|7jD_-ve2Ti^?i(En`rI%M#(uR#mW|GO-L+R9J65JQL#(PO022>)ny8}13a#|
z&sgHQG+pcPPaaQ?8D1}Ib536Jh|}^g>|G+_-+15q7)yx3+b?Ol%17DWP2a!sMZSvT
zgacwXz6mkyb1^*B^k@OgSKa*rt6F_0CM?$ASUEjl!lZ<&6CxyAHF+knG@o8A8KtPf
zaD9f>^q#0B;pmH8^;5Ehmu^&)WLu;g^kJQn_mc^2IS2bb%4E-4bHQn=i3wZc)8lH4
zMoKMjxXv{-6gxfNnY3ro1eQuA#x2ZE;$obeS}Su8wLMuWmc?QvR3~Jz&DbzGB*#0t
z{$(lmQiBMOw4(M$wJ(&)4xBg~BN)@J_-4C6b==oQO2YGBeC+Eu(pj?b`SItgbS7N=
zUz_mjLd%J@5tr1=ls&_f+JrQZEY&L#d|JDWVgCLJLYbEiY8`(NCgZ)=$Yf?>O8rtM
zvB<vZ`{N6JQ)hLDZ{bv6T02v*aogs3nUO8BzFUrNF*>r6Lq9NHaDvO$tVdGOTbQ~y
zU7{`>*!tjviAEf^RBZRtmkJ!64@<HmTw607`FRpM$~H`veOs5a`_8_UZFkqodRBcD
zis$g*PR+>sAQN^+gZuo#bp_5-SLQqw+PR1)dt+-y^P#=ZlX%?DI%gg;=}LRecrr;m
zRWP7#^RrnCww$o|`NAi^YH3#jtM)VFC*{SHjP);uERVbNa>gz9^cfSBRDTx+|MPH?
zwSRarJy4Y4KjYHPb6ELK>$@5&E)om9=5l_M&i73Z*t8d2J6*%IHsN4TjBb*kP!<<w
z-KL{zY$E5<_L)U{Dlos^rLmS*{i1E{RVUMnYC0!_zQyotO4^eY;TO2!Tjcpg?2}j%
z-0Gh7op5gJ%4yZfTCsplBY)0f=jexRW+Ky-^Jets-D-%LX>*iUSVPnOmO}c3R?DTL
zmc>4QYSpZjjXV<1WyF+U3Z0{2wCo|@_d~kLn~k<SJg?&Et#b8{Sa+kV#p#PF)vOE+
zc5l;FynD|DO-_hsePy6ta@$E;<L0Tef38op?A+Nf(aAeR{)g(;fDJ}(Cz`#z;Up|n
zaZMwKQS7wjM4l*vB-76S`#QZhKC?J%*^#uMS>2H{%R$IOwl$<ZS+;v){AT6d_9yx#
z#}(W@!SH(dZC0<{Tgzvw6wT+FRJ*ZL)^*LPln=h1o(y|_-CedMwSbjf<%7s!|HS9!
zZ(n@9az}XY<P@gAaxcAfx&rI0zpGx-T{*Qy<-^NOtdoK~Ey^Y9es#HPuBhkJ@~N=0
zo4c>>i^atG{>4SV8D}r8^C}1y=ql5*&b@T$x{2ZXpu9)NVv57%W{WJI_x@^c>#e}d
zsi)L7vHcCa;#0S_pr0XQEo*eoA^(&J-}S88TAEv)1$QL%&S$$~>2X|MYhvBn{kQyt
z?#KoGzkAl~-$C7!`))$t6e0{9V`4V^XN-Bkw#AUKHJnX|A%A-So8_fE;RCEviR`9<
zY$YkGb1o@~9Ca^MYbrBK=e=(8%S=?O*+}g~lYVkxM6%~oRnH~v)!YjiW=>&XFenOM
zT4dZ<lfKA&?Z+Y==VI3g(=}V-gC1rTPHmYclAzCM=IQ9Uj?wh&QH`|+*a91TSu13M
z+cng;Sst9EZO|z3RZu>GBlOru^Mi|6C6%mI9IdZEE4y{9EKsnW-=&=~R50ZVf9(tY
zZ1wgXpUZY$<-fC$aq`E2jhEP#Ut(LE!2bCfS1N-xQ-Gw5Lu%>;scje7s;&g8d`vsP
zSm@9}B?j{}E5XWTJCwv-3%Ea~nKuckC|B{O{}+8(tfDGm<{T>c=CkU%VzIUm)%`o{
zc3XsmZW23sqp7z&q$sh?&p6U%hx)0uOwH+;i&LC93>}YG7%!ccWm#C8|I}-Jn!(ab
zwM#y<p8wHvX`=H<v!at`_0tYIAO9HjC&lfLdV|HmdgG7nArI?=KDPf0;Q#-_=$4s5
z|0RP%%-V^c1h*^LFl}IrexN0=aDt2hV^|q?%7qDu8%4^V8EO_bT~cPy3ha|-Zjw?i
z(3ObYx<xqlgYA>aMl0MsR<-w*T=NLNq>|%km*H65^-S!@!XEa8GWHhn{7cLyIcgS!
zrmPhyo>y4>EVP9+P4I0({i=&>K@5_r*ZSC&^i3AE_&M>vL&SHXYz_6dhqyAtMem4p
z6q^fWO}8$W2uR=AUh=)2<tl%n#q`!2{J%DFT{@=!XGYoP$%;!7TKPUt{16tC8^9K3
zHbZ5hAjb*TsTQoQ=7qiK%Gy^u9hpS5E-KB~DKy1BO@5_&v_j>%trHh;s@OiPj47Bk
zXS(oxVdLOJ72k&{V$OCB7diF5swQ0u-o{xSy=b=oK~8>k4}H^&luyA&W>)V!Zur5O
zE!aTfTeA8gcMW~Unw%tw&&le)FN%mfsYw*A4HVS8P#(6hNMCx1<LwCV{G+--hik7c
zVs$v&mtrA&>RL~Gq9mILt7UogR@U|(GZc#>`Tv*nC$?Ds<23l^@SoqJt^VJQ{;wDK
zzbG*L*uwSt%d|V+a_@*)^L!LyPo5ZUAei?-dao&?bO0Aur-;H<*4`C@yp5C2rFP4|
zoH*5-^>*<j`NI>>Pc_^yBSW$@mZe?TL~^oDXv&t3bl*fn&h1m;4?DU~R7p@&{(8~r
z)6z_3MUAkXhCja6th_SCZN}2z$xB(TE#1Cj%A!eL>kqn#CYnvS#1#^$XCY{j>LFzP
zW2*5&lk6YnzXcg*Eo%SGDO`E0ozc0xrX!{C*7BMc?X_G6tx+r1C~&QQu%Oz5do_cB
zw}@k{1=l5|j%>-4>0SN*1Om-AHt@bIPcg7@Ewt&16g>Vo+U*c)=Al)6>B=c-{|odV
zs;`<MpvNV8d$ZCK2~pM1Ap1|1H<nhqrWtk}Q3*c8xlJ;7BfIEYPSpb%)wvI=?5o72
z8)QnHGlO1<?-lR;w{dN4aMHZAum_Q0m#@bjT%ss$A|%n`nDWpu!BFf0NA|LyhD%<e
z)yY+iv+8e@xv@oYf0@v~*_mH7X~F6ZT->+%>t6QrcBK^V<Xf|WYYT&cpcdDb1h+2?
zEB_wgTg<RBb;|TocGZ6h5h5%?T$wX8b_ssu63TyKAmzAdePOM{t(8k>+AjPtacfo}
zm*8T>pTb;$m8E93yLRQv6vx(z*m9eD1b=e>($#AmDOyoBSzj^O?4+aGQ|&jQHI0i_
zzwi7nT;M##?Wa&gRB^~fuVvp`mVWO|4XjH|u4lTy_akiTr|l9UT%m_9IccX0f7+Q-
zev7+GqdiZPf&2FILUUc==;`fMishQQ|2Q{t+~VJIVZ-YUTz3vFn9%Jn^*j4AOMB;5
zIj<?x1iRMDTTe~M)>3?=sKO+ASlM%q16$@H%{9AMv0YCycHC?t7^`K-IcKF~@PhI=
znhbkZmoJpGmHez^v{8BPbFr0{Ta;7_oV=PXm!_ZlJ;{=F&UtCEU7f=2ize;)Y3Da-
z@6-^_6P5|bJG+lq+9z(B^X+kUQ)AKYM?xpqr&K11mkCLBDy^HEw12@fjc>1)J~|mT
zw`iV-qUNie+g4uu&lSDEzM{K#$~O5KtE2x%PXFuS#`S9c-x(WMFVJ7{uwN=^!PHm%
zW-1IU)?5;+H!=xqSag9~a<$6@Z7zvR8<{M)E=|kbvVe1o!@|32<&r@MmYIiJ%dFp^
zW~l#=iz||i=i?^rs)D?2o3#@+7h16jhOUl3v01J%ZL{~5EU|9g;@AonksI4%Z+H|g
zQj1d&Jmz!4)a9w9|LG;p$y;qD<ilk|{ghh_z6(VkUK&>=>B}K%_Fd?<Mr)Y!GBGou
zwC3>2#cdj9Lb<cV-`|{CdsrrT*VKY2d6r9Z`8(PLY*zd|V72*jJNqidc5m)Q23)+^
zJNFu#nikF7{c3@r7Q_4)t$!Q-^CcH7SWu(LerkbWR__1pE=dRY7*28hv+7`EP5Ewa
z?d;CB{L^9XpGNB+?lSwd%6Jp2)D&TzM0u`*nhPyhwH`KZ<66D=g(08FEE6r|n}>yq
zo^`o=>bhqt996W&Zd3Y&>@7PbdXCJPtp9M0?`)N9#q+Ifq7S6=_8gBZF6{FXkhyNv
zd%Y#IW|1cc<Jrfn_J6gu7g^F9^w3n!#ru1cRG3DRcjEysYdwyOD+*35_#LFxe=6!<
z*2cvjR!q6PK)>gtk&K^gG=HV%A^kc2OdI%krFPb892Qu;!h9Ro|DsbpT*6;JT>SZJ
z<1|BO>x*vJIcz$1Xz;1V9DQt%S5O=3%XYWo|JBID0*1e|?k?UO#2_1Vpljko^~=n{
z4DLr2CDugjsq7aIcU$NlxOz*v(cY9E5zE%e51y*HoodM|oj31?m{5cLH&w>KJAKWY
zoF|qYpW@=ZC|h`uqqk>gOZ{D=sL!UoC$?qZ=3BVPT;FkNewVuPbK``Wp#hFER~|+%
zUTyDOswv5Rc+DODt|;!}lYB`I{Qoy`{kL?s|9fitUcPOA4<>KmQ}Nj-Kj*f}9=10D
zf_>475<Hf9E0%95y(0aXdvQU>aUY?>!7%}4sYj=#hL;J3O}lvgIOq8}j_2<LDzM4D
zerq_npz^z-`rH|tz3=27Y;3YUy;$t4(EUA1%U|AC$_%>0{9i=->Xx(FlB%9Uu>!_(
z7kKQ}JoZ{Rqg*WH*Pk^WS!x%xva&YG<OZ443hj23T$;Bz^!G#I_|pB?JN75K3O>4V
zL5pvmK#JJPYy0OYX&D%biIhljxusuURik*WZK+cKL@vMoo|bB74^G^D`m4fWojr#(
zp1G{O=alxHcC%x;_Px2M{^ox9QJ=bGeyXPEpA`PgTY~!^3-WBf%T+AM(HwJZs-T3U
zpq@8ZSngGqIgU|hrDiu7PMmf%<?lWHSI^H2@5!$9V40pjPeV1_P&ALnZrw^dws$W+
z-w_py^Vqe0jr_|wHpR`HiH0`+jxBqlV>M@r`;Vo8iZ?x~mel@{3|PEAp!5H-B?YCP
z%o;`3LfvQ9rbo0jTsompC&P1$Ys#JtUliD0Pv{SlyAsH^a!bO_DZ;1Le4W3gCpSBq
zf8FAhO+Tj1k~&k`@wUuE)czP>)x*?4x74x+sgL!NwgqrMJGU+TN70jyQuCXvzHQ-5
z-f-`e-uoNnSHJiv8$VPJDoA?~TWNPt==z@{*Y+GSJt$<lu-r~?mi{Dxs^hDtd7e{W
z`r>6zdVus^gO3HLqB7KjXGbhj*~2OPb&l|YgXfRWP&>mZenD_?uxjt4MH10V85j;E
zSj`dUiH+4hr|}|d+d?DBq~I{6w)sw$q3R~0v!}55E}38TlHVgv(DH9KU-trr;@dn~
z!px!n1B!W78JneY4m>*+s;Tr$C%I5l!GtR%-|@-8&+%8dZf)l*th<`WZ{*E*mPvtA
zK2O+dfu8u*Icpf3{<+<^ns`lkzx&*o4;(JCIzJRC_<Jtmy;1z12NT~-@~M20A~@4T
zLzq)tX6vHOI@cu%9;>f-;<YsBReQ^n-=#}@ESIf1_o`*<%>+jIrF$=0Z!sxrEDd<7
zzPEhql&j9kACem_bl)_jh{w2Hh<mF)ad};If9Uxi`8%ywAAPqi#aSq5rO+m>3AZ<h
zH6{O?^7lu{f;;p5FUr4vnD6~X&o}h%8Bf90qM?G59Ol2i`@GJGQ_pepl<yBK&4rKd
zu9Q7l`G4;J1U3OCi~og!0T&)Pwy>TQc@pC0Ftd%{Qc)&jL-)~6CJs5B%PUVG@nd3j
z(s^O9@MI8&vslc`3#q4?3}VkMiM*_;+04AXV8)@;Zbw!#UNw&ZgT^Knp(Qdoshcx{
zm+8&tnh{l?CEXmbIY{*=*K-ZVgv?1wTQ?*~Gp5|kxs>JIa64-4EHTjuE-TJ1;!^Xt
zImzVYG+pUsfA&-xbc$^Fo1tZQ;>Lo<Rcy-QB|ef1lh5?WhKUKxEKWSlY0n)HbMwBM
z?&64*X;*iNt&Vz=bbOuqY+2i~EqTx5md_5$zqccK^DUQEKh%tlpH;HgOPKt*HTs8_
z(1+rc2bTuD@p`x^^K`-orggXHDE(hhbMWKU^UZ1VKRowu`e=IYe#M7JPVGPCrA%(#
ztN-2Ol5}Dz19vpXaz=iufCg6a5{~8nxSRr({4#m<(*Cf}gKp6+e1}#ktg$okaA9Q?
znCQ|Hz;p4)tb!uLwKK(D1vnXbtUNj|YDO08@<mf7EKpEwa`ND0Y+^a3x4T0mfr-)V
zhbAZIwiyhkbn9$Rc&YFz1$A*Ol5l3IKVYKabM%nU37^?s%DHFiF1Q$PzF=Zq>|^zC
ziLkSYXR<1H*o%cn{q2iCZ{F@ud-`pFv#{%z3xRHuhcEgb(>NE%=J|6{PoS8W-zD8W
zmsu1&mk4~d(w?c^Veo32R{o?<r)DP@w9K&K*ziA6P^rV9#lpZNe16R${e@hAxeFGx
zbzIfo&&0j!#FCC_q7Rg2MOm$w%#rQ&GKz7-R8KXY-CH@AH}0_D(kN1u2yb9`q4jFT
z;d#5gSFaGd#1+@_qSt(Npqj`jomE<2S7~nN+vKXJDYW7A?dD0BIJ`7ZOmJdil@Z&d
zw(HfT2f;1k+9%|XS$u1sTjzGVBIW;|MAfY~>Vhj}tE!T3KAl$I{WEoHWpw2Bho5ZE
zS<K^RTD_xbUa`29=aS8you77j^BFAWx5~F|KIOfBL4xPHa|$u9xyu$Ct+MO1pT27A
zEZdCdpZo9s6O)>Da6@d+?}cV+SEj#B*(<gxso7LzvFMsvhqB!MhsAK!yxAQ$MZj%4
z>x@9fIk!%3To83DQ}si0u2)jh^hqHH*7>S5Oq#68I%~nBSyQGcaOo<2J}!Qts&n)F
zxLZ2Ck8fD=-IT977NP1s@#v9)E%$_z_HMY5eKBQ<bEIco;QfZ+xQoxld*jT5WO74v
zHth>Jk^Y$H)R$L&tOp`&mal&ua!G%-+I^dkODatNL~v{MZ{R9id(}4f?6Fx{O@~;w
z?fo}pt-YWYtLH5?-K<7mhAT?bY9pK+nhUg}lMPR_Y;(2ve<oLBhBrs7S-Z&Z-Zfg$
zD^GGPe<ZqoLwmgM!IM$9K8Z0NSt`n^?eSD#nMk{)ib1dfGkb2wgheuL+*==<`JZAF
z#$T@>*g554M0#JuvMP-<3#J#Ywh@JULR)<NvVYx8Gf|s$`o<EuC;``;v@*9DO9Rug
zmPnp%N|t`~fhWU8*qfE#X#4k=+*)=?|DqENGZy*w*4$sosc0DC@gdgNj?Gb8ng6Fp
zb4IU&Nu~6Dv5-Y?Q-kW;AFDV$Ni%JBUCFz4#>GbwEQ<t}UUFU;bFnLIr<cghz-G>g
zf=*dix~g3CiozPN_pd1tP!)AyJZN%4zqV6*A48fZ`{y~n)0%dv#yu<Br0gpcd)=j~
z;EnleiKAK`oB0gaG>N{JIBLb>(C=n@PA&1&Wvio4mX+V@RQtdpC(5e8_UvhJ;O{HT
z7cBT+zV{WYNFkeK%u)l<Y40kZE6iWir?N#rD=@I#^8L>%#R|WaqZYb{c}wvmToXPN
z!`Tq7bk##{`vq;iC11kJn0+^OsR}T99Z^g*j;`gg5Z?GUz3JAbPM<W+lRb028I~A(
z1sJ%C?4Pi3fy$E3y}w!y9@x65&raJX_|Y`6imgiS%(E2(J-1!_CLrn+sPJf})3Ls5
z7npRuE@;(Qbb~)|jZ#;_$H{6D2g7_XIQiWB;=cIBLDvOr(~MSkxO^@sHa0)?rdQW`
zjdXI^>?2I->8k}_v>f;}wUNzl0oTPN)zcQSe(FuvJY(2b;!{0$Y0C#2FM}lsf|E>H
zJ%buKBo}1op5FHVt?v=VUA}jG>ax7uFKsf2TkEZH)5+`TMkZ}<RgG>R%gGT<&s=o7
z1T`-xEsEZzoM5=bOMBag!@CO#R&X$`_ugZ5rs;<fmmo)swez+<!yP>H|0c9cY`WQR
z*x}T@NAR%5FGqgEHH$k_LwYp6bg5lWG^j{Ca@lIrgFR3GdNMT4GWf!=bap7ib8{vG
z1KVSn8eu()1D6OC?n_%_95$g@CsEVm*O!V33q@NNKKy(|%xZOKpr~#}&~5qON|K&2
ztZ9>dtwR6pNO&TCS>KVVEAX~~reJDfN+a7H4{w`Ef|Eo(C@9SAXHlIo!9VKtwsM}V
zxJj2b95}Z+Ub#mrc=n`2HNO8v6%$TQ$@=9wZHMR;-Rf&u@mUooKAme>Q8Lj<<jqyJ
zl0K!*V~bhiR=*72F>kY6VecZ1Tc>@?I2uInTzM@n`6qMXmCc56Pkb^n1Y2Kj-I&TG
z(ax}^QaV4&No1x#>*|YAoW2a{qJbPAB>6-6uO7K+F}?J<dibBqm$=R<%>N~l(Qkg-
zqjZ}7;-IyEBBUa2g!u;lw!AiD@6<Id4YN<)KHS-yF=^7RETiS^yPr7xoxFX&#yk7V
zuBlEjmD68Ndf~c1b+%?c=i%70>*wu^e4cfw_$aB`>}55VopK^nMO#q#k!J^kW=*1+
z@z=#-g|1&D9KzzuLJCDvMGh=3`Me_PO0wvG#r}mk9*>u+PV4kb`SfTGAJh5Xr7P94
zn(n?iX2DhWFHL0D1t+yF3p%HAD+ot+tk&2h&^a|-QFyh)lq<`ppBIStfBjDFxP(B+
z-ov(D)sDq|#T<|4N@T~rap`XnDcQs7IfZ?(B1`55H>;o@bG9srntXn1@u%;MysIZ}
zz3M;3TWDgI>xomga}#`&GA2w|!lE8mw`cp|XrBcv3L5)mv;r5Yh?t%}eb$H7Hh6KC
zh5xyo(*j<0l?r~c*!kn;r`C_DLLP>aEi3jM-m)Qk>F%3htNbf2Z(ZPi>io}{tmo&c
z?WmtCd|O0p_R>Ixi5=S7E`=pgJ<hLo{ak(J<;IdgnX;FQSNv~ZRj)fcA^6&n-`BpV
zvAI7Nugw(U5|wv4w!L$q`6<(ba?K)J400DcN`1B9xG81S!nsL4S!G@10XHqqr^0L8
z9_e!a*<G?&SN-?y?H{M@aopUTpjX$qZgT;rfQX}@%LM+1=21$m)u-3JJter)fV;Dp
zGkKMwwxZt0Rou<q9V(p-&rb*+3$VPpLjQl)!bhvywV0G`KAJ4_FsN4IJr*E*ddA$V
znUk-6WY^xaLwn01tuL)rH<)xU^zwVpTbeA%Z@hee)ULt^)<A(~ABTB?9V|7IHh4(1
zI&xZVaavvaVA+~%5y57`jSB4Z12qo4Skvmn@uzso<H+{snj&QnmKiGSa`+#_#n~|R
z`XVXC&r(e{8iFq#d1YXvvUomI3djCn^ELtL2`i?}C~}xpwV-pyf|di*X6{&EkZlw7
z$Ean4Yjg^~h|2nf606P%sw|(tb-`favss<yhu5^MP%e+`Fmo2L7G`+MWU=-Z%cH6T
zEuQODB&?@*C|&AU9hkkZIbok^qf>yk`?n80#}m|Vew<MnvPi{CN$<?W>lM8x1sQZN
zG?lJkHM`X05g=zOvx`TV)#Cz7l(W^e!d*T;CVM7ZnFQ&)+uVCts{Lw*vbC|y!kJn<
zH(R7XA9CL#_nrIDtpbm-6}@v_?w>z{Nol9Rs*CdjWuzx$ZOYqxEH85QA%X2PKS<qE
zGPnN!Xu4n3c9jd#PpmkC7i|{_T)N0*>5D^MQkl%pKCPV<(NUG662vg0?83y>0EVE0
zdz!V?TA91n#aOg3pO!e{nDA*1$Ay-t6T)Rq98VJu%{F!qVqRmukViMlefO6cy(gXa
z-f`OZg|+&Fcl3((15>&KwS0oC&IA@UhZQh~7tC1wTR6<YdEw<2jT738E6%Lmt+ed%
zvPFxntXT|SI%;?7Y+S2tX(c@W!%B_kC)T@1vUa{vx9;hC!!bW>W#_L0qJA}o^H0d{
z4c1hR_M9bPKI;r;hlbx=E58R%%<>K!O{?gfyUKNH#hz&eM|Yn%>Mo?!>>=Lrz-Cd$
z_MVGf%l33P6=|*h|H5;j1NZ)mCZaJ~H&(aCPVr>?Fh}=?_79%p`c_JRa)ebjIhC(r
zukPTv(a^gxrBzllfN5@5n8b|j5)yXY5<w0;##M_$8(0DhW(IaJ1!lNtthjK@N+<I7
zg{j*6rylH`S?F=nYRXBk{T9l6&sWS7v_5k6!eQZDIjK@f=gQv8llbBnc22W7Ag|oB
z&eOr@ffRR%gYw$*yB~N?j5yk<A+5f(v-`x!zYY`lHZGa9LgtN(mc~t%Ee_nOvgbe8
zsLpyZ``ZtJr&X&W|EyeU6MW3E`|TrDqaP<{YUsBLHc0PrlYO#hrop6L9SoWt-U}t>
z&1{f8<ze6?d%US*ZsD07dygE_`hVt7_h;_|B4?^5a36Xi5TwO=IK;=?qoZMG^INW~
zEgP7lGnh3cn74ge?wsgsq-CWm(SBcm**kfj<4ifXFTK&d8|tLg<7CeUdN_B!Stj&2
zG_>^$$LXumS6Q59A9*)})Avhfy6Pq$la4!2Ty8{pvdrO}{iUBPX1mhWP3spPJv2dV
zy25qU?xUx6A2qb@TC(X})l0D|$%WMu#YKJGtgSeNlNFCf8fM(tEa7#5lXJ_>Xz`pY
zO$R*fPOjK_=k}uK$?ok}&zFZN#TUphsa{yQ=f=sIb9OA?V)YGZ76`4^<*;s0^cQWg
z7HEj>;M!87e00;kiKjR4@$${EoWhs7xXr|V;(w98iBEd3JqRtF<Z)v|FGsD1aPQ3G
zQzp{{8TLxCwrh8U-;`qBvNl>UXj>u2p98u-cm(FK*mP~^e5f*$eP+<M%JXM9mo8t#
zS#RvMsAlP+996Nz=1m^!1Z~bQ{k6Se<*9dvxOg*{wQwvu#i@2+E$8A(UP>DmN`46r
zo8g@mbw@4gc=;qJ_uf0Yf4r+X&K#Jccw~#-Yo{|ubt3C|BPZ3WM1OF);Ubcvd-uFc
z>{Z{p_MRe3Z9AM8r*H)yv58^59orkv)x}wHQhfLP)hEwRmC^KG)Axhp=z}>+4xHx9
z2wcgcaxTclZ?;1xYty-BPc$P$!k?Geyxn@Nn6qJ_k@(UB3oiV(j{c;wckPXdVu!A8
zu{g-8c=@RocWtKsCfSH)3m!}6<5^6?9e3}Sx7?_jaien04cVX58K)j+spVpARow1z
z^StZL3$?+s8y@T~XkDYg9O69XL|5dIRW}!Hy*cx4!n9r;sil$AFGo%~v*pEN*4v@I
zJGi#pj%~fKu&>U*Nw<T+?h(_9ohP5k&dk2LL{Yc1_s7ErP51sg1|5oYJ#2k0LC|l_
z*@Jw)PVRSDw_?loI>XCbMT5C6nH)T*yYFC<!RmEY&GF`I#hW}C_HX56Js`AagN)8Z
z2Ia!!^M%Rwsmw;1A-YrUbZ>4wwKit$qnN{Yt}I*d;DRd$XWfH|rcYV~p0Jm32L9LW
za1)Dc-q6gh+FI)r<{dWE>P$+^(G<UZ8<KRVwEWPvSJb$5)*yJ>UD2?4u74&gZ|Q7(
zu%~C1Xtu8(Q_j6iT@L>xsn29L<=S3XUVGq`3MZS_BRSsAuqpRm-;K-Nd%bbdx#ogr
zvlICFowPVRHeRY;w^?-WCGGpkN4Pe4cUw&|w!S@4V9mrj)yRgue2ul+&&+iE;LBCf
zmwb3d+MhS)D|!|(=mzx6^?JuOgSqTUKW~Od+k?ra-IH}Yn#^u`ls%Z}d$XzF#qNd|
z%@#K&w`OdYh>R|1HS1t9nsC$fS4QaPgbDj1C!OnEta?%89dF^QWny)SqStz>7PSAL
zm8hB$q5Hp1cj}g@D_T`1UplL{J1T$f-nFimIyYQe@ayE#3S%BmuM0m8C~o6==43lt
zLX}mLbybLZhDj*zMWN{dLSMWlebKs;DD_%FGkeoN1#45TC6g}id3kRBhSsPR%=YuL
z-NF)_+-5kLB{dhYS{JbL-e9?Qb-DL9b%&Q*ngpH}d%bz0`Sg}+s;kPIDXW#P?^K(f
zvHTFn<nGgg(qhX}vey3PRD7^B?Bcv@va0uTZok}}p^zN8iRt2$o`6e}6V4SKjWc|_
z@@;AOtD_4FXFBo)F}(lEeUJCpCz-r$B5se$R(+f@;rkrD{Tnw+1up;gW!>(A=RbP<
zTN&eD`*GYi(|Y^m?H-TUasN|h?6?@Q`bW%)gbUk0M0`1x!Csfr!q(d|@5Q$p>zw6M
z@(Wp`FEr_TJQR?-D4-T-F2rDT$3VLEVP4Wp@p~pd^D<|$t<l|dzU6=|Cwp(z1viDx
zRBpj8Ib9x4CQU`YOKNIKd(X{q)d~{yciqML(P!0+Q=BHv2TyI*T~XzG{<V=;uGnSC
zLM@TNOMEkriUqEci}(7yEkx?kD}}~M=4O*NzH@ELeSP+1_k5e_rN<5~4QS%bXm!?m
z5^%PtaT(|Le-V1+Pkz1o<bLhbUa?4H_6*bdZo~eR@E7|-|E1jLcv^f(+u+RMW9JU%
zt-9-~vTw@f$r^jOx3`@BKf$<-u_wz`^#6Yjv252(zZ37R`gyD1?xq~i*_{fklMj>>
za9`?LQKDCUw8XbF#q-%4+wCPsComk$RWMq0`lDY(ZTPnb&&t1YnHg-%5aaea$XWX}
zh~?w;wMndpBAy?-n|JoIjkW1FXRFPUCtv(%TiC2t%$SkUYS&@r-!WzF3qAji^!wj-
z=Y5VYXtjQjJ$*t(t6E0OiWh+sn(i)~x!Cs2lzS$oPt<Hh)TjOvW?TMJ@Oyt$2a~NR
zzmrw}zQ=()P5iD&o)4Lx)rfaJe3RNe<KBX8oxI{(bZgfw3lZF^>X@_6Tz^?onww+n
z@pWgnwyru7-Dvz+dt>ktrLK=wGtT~wUZ-|vt%y*Qvi_O>A1Ai^SF}8M?Ie~Gl=?ZW
zfI}!CW#Nfz6Z6o1VLKZc^-B8>JkhCD6LJm+ZCw$5!*T0Ni^|ihzc1SPhUbD+$JyCT
zzC6F>Rv&S(=ZHP&8)vE6wEXG+0?pPAjs??l*Ra0&ZgTxqjo<CC>*>qhPW1db^TWMY
za$M(woA!R3_Gxcna?Nzn+0iWy{$iTub!{>dleoF%3wJfl=*jQTa_{){?^}9&apTOg
z)^+)5?_(yimWS^sD=?2VQfNGIqv`DN3DFs?yc<?e-FBCCdS;$a#2?oiO*1|mlHh$1
zAEENcq;Soo>=fO77cynje*fpl+rCk{&hg{y<CP0nILzL(Hf+1liUZGN{-5tMzS<=f
zFL<DEoiT5)(4YQI3Ky)SIBd;2R!!l~66so|Ymr!0sqm{m!7F**n*4?WWrv2<jcn{<
zHY+al>9#US811<na6q=5S>4Mf<M`Agz2z|zZ!EeHCe4_av-4rlF_qK2YmMSeECLUl
znaQqhCKG&K$-nh))i<FB{tf!gj`e(hR&IHIZ~<R4U#=Fz%0nwcXKJOC26Q@V3NP!h
z*}=Hlna^f>kLZD;YoeNCZmp`d4p`^vm%={Da7Ot9FUB&~H>Nu_t#8m{_MW=2yCOk-
zKa-W9U(fga_#^eQ)+(tRKPEOWwP`N<vGeoY-CUDACOB=I>Uxbma(d0(sD|w3t>N7N
z6O6mAXteIFm%nnSYs#}pN3t*a+}jYgdatLKbAMfUQ_~(%_OLx)1FXuzO*MCI+I%Tg
zT8}Beuw?09<*IKlFLsv)rzM}{jZWg;@pamnwMNoA);#B87oL+oq0BHb{bDKG-HWFr
zqC}3)Z4nNNPz-ysB5k5bw2z`gd)`*p>oJN^H-aOWI#e!2mlRIC?xn}H>_UR-rHbpR
z>~}K~`&88?xG;I;T=dFWvq90p)77da#lKN><r4=ltIU8Yp3l@2C7syHQ+40wWJYVx
z{-SZ9n91Y7WT8~90=4kuWiKbNoZsk~HYdfiV3~(mWQUNXN=y2L3YjzR$5>KwwoH^>
z%i%JAV)g%N57{cSITOO9mlhbXa&F(LI7vIzCosshtiX^l(dJ?2RNa(}jgw2lB15G@
zLnfMvgsw?))sFQ0YBHIL>5bI{mf#$QX$+a3=F>J#RNDQ>A$mipfcW`I5z2iA(+qDp
zv}vY@t5>p$%}kgwby;Nk{E6BQJD*NU%gcD>_B}HBo!pcLSH=_C4ou&@Hk@pUnp$={
zhwrDgK=$FxWwUqKbVjS>EY7%kOk=C!^#JX&yjNCl5ud!WA(CD1iF&i#?&-0P!4{|H
z>6T4+xI(&lRXCeubW2E^ws!FXp$+_n8y!!j{3&NYUli*;Z^fBM+gGwIjye$_9jGu}
z<QUgVEr%=q)@Mc5T>5`=@^qF@*Q_sut5;bhl_>dDDBVzK3vG+>kUvu9e}n6R*;W6;
zwi9AYlkX&Fg>Q3G$=&udR=Td~cG%O&^~#>7rnG$zQK`E$ankkNiQAL9ZMl4Xysqc+
z9a(lXrLfhoR4AuV$hWM5N6=k;z25{Et@AQ*=PC;eSIVyaQ(pa!H8+qgTXt8{ea|_k
zVl+8Z3K)zlIiyTJvE(N{xgW=*dc0=k)nq-!{X67JO0PPox`qb+`{Hc(Q}Ekm@1Ny`
z?<c#i<+@-O?0(c<V858=$E(-HHd(ycbYxR&PP*{ym@=1Xd7Q_T8aa6)<4gpCJlLAb
zEF<oSD9=1EQQ$PEv2Ux_#+~OS|35L|+7T|qrm^<;*C%xhzh@u#cTDQ;1}EbKFV8w;
z{BRJH@?<c~k@D42XLx3jo@aXN7Tc{23Rkwe+dj-Zs(VL4esYb9s!9j9;|YbvS!-LE
zJ7zJkryV=0n<S|FtW!kz?_t&}e~<I3Y97|T6|mCi2wQ2Pv;f!R$@0vaOO&{kx^<Tv
z=vo>0BI{}Nd`GsI{Z%v8*L+#PYTFW_Tz|!hOVHr5h?&U*{!U&!l_0UJK`lzH6Lj+y
z9qD#`^6W%#lG@biClz|XZ0qb?_`$>SiQnWeNs1bQFTDB}$LN+_;$!?{RGetUF^f%T
zmzAVS30Gj#(Fv74Mv|AqqgO_7IC~h`=x${Bza`k4^O{ks(3u?F0-mdqtgag2Qx-U{
z^_aJK()aC}A**>GC#rkRFf}%u)WEI!!29g253<vGSk#m!22{jsYz_RuRA=khr8qgs
zu6e?G@eUI%YnB6jp=+8%zi@Ermb|cQzwt_onKSb{*U!NAD~DV5ed5}Xw@v4NXG;XL
z;<b%@Nz-O4y`CKXz^QA&R8Or*ty3@EJiX1uQ)1CMjg^b11g2dpHK@ql(A@u$bC1j-
z-FYkr6e49$JQHGQVDV^dEa$ut!M**^=0_J4mUJx46Io;-?bX@JzgqFgD)WsQaXO6A
zx3&At4ji-L>Saz@abEItfacFX&&4!;CQ0`#lba{ll~JDXKTPu71~I)YjXWt4ofetS
zY&%b=Z@#j7v5e@Kjg194{SlgPefMl`)qTS<fminBuQSV6IbT~e(evd|jj)B$E7v*+
zOS$^x>uNQG-cikJkU3Gb^s`oXPST{LL-Or&zAg;Nx}<nx>BZ<npCb3I$z7Reywv$b
z_o>f*9DY^Gt@{HkCL9dyT+{wqdeWa|qQ^sD)@|LBAM@Kx@uX*G(YB5#gZ9f?%|kaA
zM=&a`|Dq)nv?SH<O2~(n@~KkwSx-{ZQaF}-6>LgW<f`Bmiqlj}5Smcp<)B$M)8WpB
z<+n^vsc8IB|CW)HJTu}+mDMy($t4YQI2^ag@0}8AZ)GJKWssx(Z$YcZ|FA@lvJOY#
z#WOPO-4xkexFSV@-S~?3w#O}wU<uFZIApX`lVMubiyWiSj+;D}bOaZsMl$jw_&;IN
z^%I$(>AqDp{>`@McDFmGFm>Ev`R<hA5ZxBbmD$5uc*Nc{UqCzF)gjA9OZVqLZf)kn
zb0&5KJKy6t(Dv(6&+>NW*;~(TRb2IMVYW)lBBPs!ME3`1mIxVWTwT<j*8g^<Hp^2D
z$B(WdY7djTtB#A`=Vj@v+@l<9VJ4g&-IB~c(JU-z_nYVmPRydE9`>vDaFy{Km0v8u
zYWrs+S0KmsJ6|MPHR_5y!a5c$uAee1Od_&Bv}d>c_iHUJd)}M~%V^Y*X7X+~nE5}p
z`%C_A(@#s(4_hy0l`?H?h<i1?Qt3+0Rn~_oj7<y{b^<CK3{Mo;o%**OSZ}ssL%wR^
zdPlECQ}-;Gquy=Jp~vsOR5Ngby1{v&>|2T5@q9~U56^rzqid#`iDodPhyRC*C%3Q~
zrT8j(zxP~!PUKoptxLtW2UkQKpN8LhVm#kSy5p2UsbJ$hmg>Geax07GtTy{mxTMJb
ziNueyeR-Rmgk|5pkf@68*Lt0z6|=5=Pb*Vu;F^Wo-&mZoI`ZN6$}cVp)l^RhJvk?=
z!0pN~YsQj`Q$5v8e&;VUFg-ELHNWbA?^E5S_ccXBk6hlNVPh43c4F*{MQg=HMK8<=
zShbx;>B_$U234!d4)c7dzBJ?UYV99lzqA-Uj?4ONpP;#Q=Dx^6kH3!h#n&zT*1I6n
z_*KC5w5w+_Tl-gRyRQ4<xRcj?C+CRn&EAqHQ&&_kI=<}eo8POqJ$jyd)NyIrgxg(B
z(`3W#H#q12JiM#Z$<~0!`bn(r6n{o9Uq-crUozJp%g(&#${U*U;Q!0{Q9Gp!IS;Tf
zbRPW3B0gckvA-cVC0u56y4^a$q0xBVv1973FLCT;3QRR~92;a9TIMj+hzfn_(93i^
z@WWeo>x4r$!cYI~J!{BUcQPhsXZSf?&nQ<u%`1nVYOoaF=w7c8d`Va0@<%VB9EU4~
zeCul#N?zlPzH;Qx-4>Zd+5fj6N|Ya1==^5J*B!F5Ztc1`2PFhpx6hT<-Edf94b%O<
zcQszP=qgN~8X97MAo~8p*%DveY<n6tzBJiyIJIkSGw-$Y4kyk#@~~=8Ir%KYC5z>>
z_smD?bCou3P<XlZ=&OxKJ4>#|d^xWb+7VGYpZTYp9@DXS38^EzQcSVkjtQsPTxX^9
z2&HJWeUNks={<E+^GJB()Ng?tUv>sWrAnQaRV?i2-tp$x!3z@#HiYM|5v_hGQPeYE
zq~p=i1G7ZDj{myKvBi-8c))Du9rCtnEdE!Nk3ErhTEOK$<F-VKlJ*?w-7lk={aO{d
zu7_ku+P;Xke=vLU*J-Rq)Awg8`QLDs_q_g}z3S|NP;t(A3IBUIJi<8EYIYyqo6w^X
z>bgoLtl^Gh!JOl&3<6$^(mp)eNACQ+)Bi}~<Vo8LkIu-1{ZW}(aQxg#(fb7_9XVv*
z*|@)`=@XX<zS=n9Vyl8?0b`cO@k^QxFH6t3IUM(CY1dtF*TL|pOjl~;GFiD>iBV?#
zx>KB1rNlj(=<lvLo#BG>T}wH}u7lgYIw`DccB!4c`zXJy0(*#qNXdjLtra(meO-6l
zZ1zyNG3SV9R*h?G#?luvSs8ZDjCwPrOJ>f|pK@Vm<}G+C*Z1Y<yUyuHTw`v}P<w9}
zn&RM=6(+DKHKw&9^yJD3t#{J=CtS_@cRK3o1jmk90d30v&D-WX&UwtfHpTysm))*D
zp<0(r$LEd(N6K6-#+^}iO*m?o)#7TPUh*c{&X1+i;H;z1*&RRSZRgxhSa;~kAKBu$
zA;p1DzE4>0S|M5E#2%XPuq2VS)Tcqj=OAa>(bC2fl?Roy7A?BvGiTnyw4zlh-cc+k
z7q;y?bE(Bnu{Ka6Qa7<~Ys}Y|0TYi1M5+2^GDHM!3b*|*;lP8wJt-WQBYnD?6falK
zbIeF-mAuUI!`rbYHLW)6?2d!F6-V0V`Z<bnc|GAt3uL)AcWLCYhp|@@d++p#tq}Om
zp;q}RK)s89##A?Tp)}9GEb9!W1P43_S<yd3>sit-XQxP|S@TxfZ(;gx<#bt0OZ|nb
zd#aYptHzGG%UCy`Smg0Q^5%n^AMUp5Yq1z~O<G_ipsytmcHpc**}-_7Rk04ob|>6P
z(plVc<(W?uhgsUJ92fbL9c-&wZyn4@F0i@q<cIfVpPt!98o?TKS$f1?RkVmF3W{$G
zth(VStq@po!B3KhjXg$ip~VqfqmB(_?6N|y|N7qE828*gqGc1Gi1j+HV>+!tDnVP7
zE^U1zW_^W8yXIuqEVufy4t2>EcOzHNvb5GCNuG6LTXizF)XfYil;O8};ZQKAU!_rO
z!eYLK6Y6jJFPA&=cR`HKUDp7u7D2wLrjr;g|IKk!c`ByD*!(c`@<*?$trxC%3ja4s
zJ#)o#sjJ2TmWb~8FFd6d&al7fsQF}a<QkS!9M`;du$;2#zxD9ODntKCYA?$lob2s=
zI_c3uxjd)9GhT_mlxvyZMkl$>bUV7m>E0H*DXPmfFI(k0{XFD)?Qlb`=luzdig|~!
za~nlMqC_HcdaPQm`aa{dVwpQ{(Tk%Zn`Ru1-t$P^w)?fFjBeD_c!xQ<$2gcKHf9EO
z-@7%5d99213Ku5HV-LC}CH`3(xK{0x;jy$C;=u;-OXmvZm%aL0I4kGFbk{d*2^S_z
zs_9%Px;*!VV$m*bLrsR$3wdUK$XM~DJ#l7x;>;UozT9Nl(kSjL=&y6)?L4tEHR+Nk
zkIS#kFS&E*zeY}@i_^SnDg7%EF2y&OU;W{-RcexA;_A*>8rmC}4(A+_NLeeA(fG}+
zQKaPkOh+|;5#b}(Iyu_3yJ|e#`&PShbRRkzm-)s2R?GASX764snsi)}g`e+8xU1uY
z7uk7HEDXm*g{%@|CSKz9)p+<|QmfD!=bntIDmmISGA?lXSYK$pxbsEJ%*5UlgMS_k
zcW1F!8nSS9bm>koI63R`{0WZcFMHlO_^&f)ExV=f`D2EY>aq((kDeSnH6x7or$fkP
zrjN6ht+aJ{+weE$J`cNOO76G3w=*q2YP@?sdlKvH1#H(>vF+US=w;?oExruxc}klL
z(mYNr(wgd)c`d(kq1q*xtp8ls5_mtcE-`zR921bzaC+;W{C`zbMLbsg{nqVR^J2l@
zXS^#z`8KvKe0aR*Pr%b0#v?`<*EQxctvTZJV&a7lCw2ey=$_%Ls^E~B=+HIMGww#C
z#N6W>KYjcg`@Bkm<3HzyEoyI*?AGVX2I*XH)-T}PZp7o1w^q!hQLkG22j7t{uY)sK
zHurox(&Oa*uVVEMF7F#_9=PXSy0c<+%oW|7X`dvodoUeY?dYHpadFB1583Tf27bRj
zPB?N&%iH~|tPYp!hVy@eto?Z;e{?+eoy&A#v-6eBbzW;`Uf>L0aq`8U0_j<TzFQiX
zN_2V@O!H=FtDW;kVyDEPvun=jtvtIccJjR&|5rU+<5ZI>-)L|tNZY-A_T{E8m*>57
z^~${}<oP6Jty<1nkri&<^CgWUm(G@%Jj)|TJnzlyL!5@6kIa#o9A0r$)$sG2dq*Qw
z&fiiyoyPY<_Iz%}E#9EHDmPyw-ScDH9#Fh+p31R8r^RNkdSXsz>E-q9S!SfSZpmHl
zwF`wzqdH8)I0K8adw7ojU2*HcjYD#`H!F(<&-@S?_j>9jRfaN2E^*0;^)0T_4Ik4~
zqZglHms=QCHY=>@h(USP^W!frzE=~hirA)mMV4LeeeqtSFNZ#D%{#g6n1FW7d$}zA
z?^Wx)cbkL-7#cqRlKuLN0OuXHAIn`s#An9}rMDS!&0F1BQutpdZtI^<EvEe2x9l*i
z$`w6XJ9Qt=Liyj??L`{<Yc?}j_mym)vx8^u--;FYO+GBR#I}FU)<a2t)=yfQ3f>xT
zT;W)9wkll5cw%Su_0Cqq$(MGThnwEm+cSZwhU066YX2;k9@Bn@2YS9IHaPSA*i_j5
zUg}4r6HCM<Kld9=NusCDaGcD3s(yJ@W?Qm_p!eGbMME#`LcP^yvkKnoPqr|cT4CgR
zJl46;uv<U;u8Ezsi|F_Cfc5#A2}0ukL*^v9Ha_YIdL6U-mE3KGtW3E8&7~V39P?E_
zVmP^fYsCD*Ts20?8O28zZZ(~EYk}oSjhNivsV<6_*)+E_@}FfXzQSa^?D7BZ$EGu1
zgwB3HHBPgI>4G=IiAtsmmUCx6<haQ#XegMfUFLsc@uo>dTRtAic>2!nRKW?0<wB8V
zjyIVWO>xQ)jaI*0R==!G*CO{z+UsrmOsB8@(zd`XnB((`*Ix|W-LubE<h=iKrzEbo
zeO@Em3gO9bcNSeb)Ow=aP;{Z3c~!6Yr09-44<7vcq#&ShApB@a?urZ3Z$I(V`dux`
zS6B11&8lVDw6=p?8m&xi6~;O`2Um)G=JGjiZfv_|$NR3M1-*Okc<)+mU1#skq;X(o
zOThXoC)dyLyUqG~65p<scNEV|Dw^Np(#-tlq{WVfXM8k$PW3H1D)OS=d-`tIh=mvC
zZV^h(|1Z4S%y2cE{dUe9c821sepJWj+?W=5B<qi9u*b!pUw)nY<?-wM?2U3;3YN|7
zd~)a1pLDaWp@pY=1GY*oPFZy8#3Hd{?^3sBO1%q`%!_C@4USp$H*Tl%lO=w6@z0F@
z-uLLsmt$FVi)qU~%ed>_-RE}BnQp!J$xlZEqq!;$f|>5`jlLyYn>D?DS0-~LCEdMF
zr)u%#y;TZaiw;~&vpp?R&=Br%fMtet$q`QP&qgylt^P&0WG}8_pQO*Ri?d|IkH_V#
zx%1aMEqU)KF^7{aqv6;du0D@j72y>p7fkSOH~k-(s8ZQ<&11uFg}qD;oPV|RyyEke
zKGYPNKTueB;7(nMN8kVK7W3>WUe*U>BUJzGnk=yU@%Gy8Kix`uZM_>8ein2o=dq1*
zDG_Rs5p6l(Y|-Lv@p{VLB~utRulRgo>0a7x`#Jcy<gp*cODeh^iykX$y-~jKfxJu4
z;_r?-qGlcUmJE^zW@uqyWM<V+tPn_a;tUe=^Y{^PK%_%JmG6$zgww9w(mF*uPOvU?
zV-&Y6WIUPq$i;}|^pZI@iWGdNbM6*cWBFkj!%X(*Yi~}j^jSW~n4QUk!J)}xk@rk9
z)mvR6+Kf3Y6N)mo_BMnug>6#WTC_lyaVy8IO|uSUMr~!exNR=O<-Ud;%<p$7W|S-3
zvu1g{Y_IBrzSYs|nE3U4ZcNn;FZ&<0Q}c40kECA8vYZ(k9Gjfmna!2o*f=(xzQFEo
z?{nzO7oRQ0!OA&%#lHD8GN&I`i`e~uMN`N!F2qWx;z-9n{xBBKOJ+wNbB3?W**Q7&
z%M<VUuT!{e%@`ORSR{O%mWf_j%M_8C8$9p9&#)KlL9+H6Y*v34IIn&t>;J-^E3-~c
z$TF^5YUaV|ufurDFktnXu-P6=ss++3wy3ZKEik#nF-hPM7kkkMg=Urjhb|*7o$ezW
zSPEJdL_+7Bc8PvdAb3pt{f5+I;%_6)h?+-bE}mj^LZO+v-)x4Hd6U~>gTq_0qJ<QB
zS*tD`ZsO=FI^1qi^U1VzS+UkBy`^7@7U&!m`JZr5>$Hf%>E?5H3Kq6?37<3T<C_%B
z!1wCTAwO$=CqIVC21#5?TPzr}16pQ1ex<nKm(7ZRmLD=5=N$wmiKzPR+Hgf}LCK{l
z3;Z@XpU_}fa%0k!DbK23*aoeUTB^lpW_dzu)skCTOV=*S+9<l|%&nw@m#*nd)!-GL
z(9p9#;!d~TJ^`+t$b$mAk4`(t6TN2R@lz9cJovOu`!t>td3Dv}oPyNZ+ZVh(e_a^g
z=*pqXyS#Y%!Gg@_@CNqhq28UGQzm3|2RciwSoBaMXx){Q6SSQdPdhNHXF+Rak>|yk
zcD5SJJC|<OvsBsTt<cHxCpJraRmUVPd&YOl{R@72#N{hY+p6?`rp+psv{l!B-jE1j
z*my!|#ay!`8PmO{i~ZSp=3-kK_sbTWB?U<m*-H%F#Kkus?UjjrxUnZ_)7_*KoE?88
z4)M!NoxRsNA*9Nya;eXXt(O+5+&Z<RQ>!}HXyzP_wK@;mG%|KXzqVLwa&gWH$F?OG
zcoLdaxGWMDxd?HpSjztwF7lAi?Yhvxe(B22Ho>DFfiqpUcb3fg*J5HdpJj)qh97%Y
zY=hqiGp)w^FZRlYe)15!sTCUO%AvjX+v2OM=5Q*lo)%UiDqVi#)Rczq%%%0yqO5Nf
zC>!YsJWJx@<2&!ekbhX+qvqTL)$ex-&W2j&RLqw5xcmKS`o5F5-|WA>a?{Iq{{!x8
z9+)>-L24yWf!21J1i|AC4ptt|t;~0Mo;|NB=FQEtE!fehG)MTMrKnRFoAJsG>~_q7
z{4Q22<o|5pX%U!tGw9c6`I)|{4r?U@&rkWT%{fOmj??l|)SagatOX42Rz=SJbs{P@
zwZ7)&DRQj^D;m_#9&!G)JAQ9WklA-u@65%OXLEQqZJBaJUsXmb=0cLPMe~Bzo-9k=
z8EoSA2Q}Y%woAvad{H9U_hO}g#U_pdhffn-WAlnLPu_WMV=i!Z=@RwTo98Uha~1Zv
z$}(xjqs})PQw82mIBA{IaMQ<(^Wc^{I$`^T4`_59cYJtumDW+=(;Ybr_ZVD!-<A0J
zN$!@^|1)_@v}-mVS(JW))v;iy%k(c?&vZIh9@cz!e#VdN3H!R9tgm}3=y!N!$6L8W
z>JEBGXC36c>#VY3ntyH9Y{Q4oj_%@|=$*&oEHmfk$$+#?0!~Y&ywB;k2|mX4eZi^b
zwVR&T8Kh4Q3s`Dt;e7Aotw86tB^RAes+_v`pj;s3+@xOFB>8=pzMX1YmD(<zI^FsB
zwLqh<8y3muc0IT!ajD}L+rlDe-`P(h_8r;TGHdeV6HmXS9LduAG)ur(Bw%0DMq9@x
z6YQp*42TIbbX5~t={{9<g<0B^z|u`KmfY$JSs`@3Q6zuvq215V@@?8YXKNErLeHvM
zSGMi@$~d{s*z<nG%`EAS|E*GIc&(CmmtUnCRGJ;X_|j^IKcUC#pHAtIc6zXAN0EVZ
z<Z`R0U!Ij7^IXNE(3mVP>0iO8BrnwWK_N16W{q9M{FVtla~HUtKN+^e`pcRL8h;ZH
z*RXt<FDa*CGy8#r!9!718MTsiyGxWrGmDq*RBXxH8p`1$zh%GnwXX^<eeyh}SMR@8
zq3O+5>1*Ks>(u_Ex%p>QgQa&y2W^pkYf^bT&)NG|%q<zeK90zkA`2(=D_2dF>tj-a
zwN9>ZmhG~X_;}exW$HACSguJNCq0$J#WlFZmaY`45DmFKOJ>E2F5dm!8x*^zPmNlb
zvuf3!RqGCD-TS;Wdr|zBv+dJ`!Xvtl{hzFsR}l5~)NRqj$GTE$J6Ptq@2mOQU3k#*
zgb2$D3lGygtLfGfMJFa09A*pM_{inS(H|yPx@~<<I;oy$WL;orq<A3tfg*=iblWE1
zO~Q5w^PaQW-+roVzobHE^#<#!pU$`aOxJn)VO8UTCbcK4T-WgQ_KJvhT@+j$a$d;y
zgud&mDJ(B8WoU&qT@<p-(7U*LN!2;M$eSE$YR|G09MZg=28bV+bxPC2-Zj>fNpr;?
zHJ8M7*0S8(FGUuaxmw>kSR|xp^j!RSnzmxLsH4lwO%Hb(U$>C%2%lU&|H7QMBO3w@
zgYQW_iniZfdC&KBRHV-7*j*AzhRd|K*_UMGYd^UC@c%wVW8O`(_i{D(6|LgwzZg4F
z@u%R1*?c;CK38k7dqpz*ja6_Gsm`xA7rSql8h%`5x2WNBy)6q%Z&t?~J2<&>rmdVw
zf4$%9DgPRd9|~tZ&l4T?{K^&6rLR^`*m<t>+R~1Rw|{uQIB_^4W)tK089t6BVy35N
zaVx51JY5?tC4Tp4!mHeZl-RC{_?-)`bzJYUzP|3pj2k7(&1;!v&TQCbA)ul+LBYc%
z>ixCFD_<_XvC%Iv-p}nr$DE4YOM=>-cr833uwsFq`r95Q14osWCu)Q(CUG7tQhQmM
zmSOLzz3SH7mbI_^>XUw~($(^=@Kb)P`*PDwsi`HGR?UshijO;Xt?keMwbS3`%&a!q
zen2W}@%GoboGcAaCVqbx&b^^~-9O>g-OKmhs9qL2e&b&hlLq6&f}L?0eJLTm!Cbq4
z?|8NOKX;I0Xu=|vc~NuqZkAl!{@Xi7Nv+RQp^cw0S2O+T&GzN&e}r!?Iib{dXz7Bo
z>%t$?!x(;vTArS{KyJV64$%!SrnJw~tT|>MGgIi(jR_T>-#uTO)On%BZ<<p69wCGC
z8tc8JHsraydl}T<nb6O)yPbK}x>pK^SS~ej?^%{(cKCIimZC-P2D3-s7i8{#;m0~R
z|Nbt|z_<@5lLCIRttnyYDM?yuZN|o^q`;-~`u#SRDSX|lqFgsOoJ}ckOYci}m*L(1
z>3@2}lmu?A$6Y>aK7V`9zTRNfgQaF(&Br#K^*Ge~@4$wy$JRzK>0@5j_bubRRo;8c
zGYfcp7N~kXP<$ew_#)tYiM~fc=lX9S6waMhs_41<?ZuCMd4XInN<MT?_@}Enf#<;_
z<&zg$lD90FQq;Ri<MbJcH*;;eZKU|@EH0kj;_q-Gx#f)WmJ9i6Wygj33m(5xRE$vb
z?b7~#BZX^1v#?o$sninA%+=>@@?UbKi@!g1^c~N$0=^}uPMA1p?JB!?A@7{`t`*`l
z^_XV&@n{P08h^O@#GP;RM?P<b?{Xg$VzkUYsr1_%^7kmvy|FG(%I&@YE1S1~lFG_S
z*9`B=J=&jDprG;J<C+iyqlyFfmh<nAyk|e|9Qq@(ysUT1xtwMX$MC4O!%>MLGWy%=
zR86KdROfXV&FSUR%>Ojy;wOjv7^D6=j<sB^CzWR>nHIcI$q&2~;(sUL{e^&ho_PIl
ztJZzH@*%3=ebkPNRyB)Qb~&iCH}QEp@L4zTWiMhe3(#l`;@i!@=i8up|AVG)!?z_1
z{(WiX7goC4<iA&K`Q<x;N;@vePS5`-vh3=;cS#a~Zv);Yt@|jYc}(u=lH^GVJaKLn
zQEtM~ZL5t6C$j{##5mdRdowr0?YGZFjeSX+2b0R5tdV=RX2BAtonA{Fr)6><5Iic<
zb?eG1m7o2}zLRS2EX|cTyO{4kZ=Hv~uZO>h@P(r*Rxk3<lUmrvwDKL(+PrCR0;G3d
zG&|iOUHnNcd($hQMcF*d9fcGa=Dm=%RW5Y;py0CT%>~||tgik?tNr`@fBtAr^Jrx|
ztI61)z}WUQbc29dQSbb9T`5($X9S!a?j<WuKCb95Y--MB{ILDlwubCwXRTjuw)0-9
zJ3-C<wi3@vezp7zsq~<4|5m82KgyaH#8({jtM8IJ=OI4x3vA7k_<SE|+~43ddlPq2
z(Kjyzk7az{b{w@ARbUWGa=G|KX^oK5zvmXhNpJQ9-HW=CAk}f7gY(opeJi2WKd%JM
z5nAaczAAZ%!|A>a`t2f$CwLUjZr)^O_{i*k_Gv%Y&5s`2a0ILFJH(#oaXO+<fp5uz
zZI2#TT#PMS?YXMRXzz(zAz7VTmvfAYy}$G?p0aTlSCYzMkC$^cE{?kVa)sbG0|Av=
zLekp<zZVDx6c?M9UsmyH%HFrkQDi}KlY@o32?N7}KyQ(QLQd9yQ|nEbEn^q$J>=)V
zjlt4pt82Q_+ed{H#9joR`P4W~>D`ZP)~|WV+a_Oo@KS#A73*eRVdnndLhJP=eyAvN
zv#VoMQgl+<emZ!m0++e+%>r#dQ+^fZ3tWZSm+q-YwN5qj>HnJ1!^1q~)s6+smo})L
zaZo?S5L4`5dZcrOmBYk01qvNSn(qZP<_bAK7S*br<ox>o)@mCO69;9Nz<v#ttMQut
z)>E7~mo)VJc|OxgU;~?BLl1vRPtN^IMuO42D^?lnt}Fh<=6CRo@_Z+bv}LR23b|~V
zw#aUpf$pp1`Ab6nhG{=um9&+o<;E7L%BHW%>P&mZ4sm<0iA-PPQ(TyS`O5S=tr^#{
zB3RB!C!X|+UmqP8Z1((vqC$yPhOt^+uz9hO^T`F~o8OrWi-z4+;*j||iQz%EX8);k
zou^iuX*zQ);%$33$D@KR>oi}e%+5MxdA2VvxHL)en_X^z^$zCK3ff#vSL8pJDY0yt
z{K;tb8|NFx4pnSfB)05<%2Z$Ft0LT!r;5)kSa`)`_qmh^vv4NHDPKSR4^>;4$8E92
zKg&PH|2pqo0lw19W|x^_XBIi>i)hUbim5en-r3-6-N1Bi0#m~IuP!1LvR@TcwpN~*
zrn$_a$KY(eh&tEo=|BFdgd{7^?%)YraJVYeiEXp_sWYNg{ioRMRrQuS+&Gc5>6MVD
zf19VLn3wfNVe1ZFyA^Z)32nCUwhs`SXY$fEFKufSqc^{j%{B$!OYRLn)fU^eaLZ1}
zKl4S>{`}&IbK-MzbDygAs(tdlbWHN)l{I~3jFYBGeci(RV~WBYGqVL|(#%&|SVg1P
z_<8JG_D>|#LfVv}#hjh>L7?XpO{NBGHl_B&<o2D+9KQm+3j){gQJdAFbazIv!khmp
z%8ynDJoOHF9(e!K#CHK-JH2~D6O-R>yPS6NO<3@vZhNt&JlF7Y-LseYE4C%Ml)1HS
za|pbCs95{Gk>rhZ>q(IpSEcKRf4U=lq-7oV>(q>mS3B30GEFq|PwjW+4q>>lAuc;a
z?LsJnhKNS|^;qu%6BU0>lnjZRdx`h8X!*(qJm&>U{svn(E4Q&bRdh~QP$_l%XR0f9
zq~d*G#w0h_n_)~ri;Y)!CUmXWo~tywtWrmkEq~_wD^i9@?UG7NJ#6Jo$wCTT+Zb#X
zuBzG1zvYxpitn*aeByO~um4^)Q-SOJnORJt;n(aBEtb0^k#^D{t#pOrG6#v)!n9g3
zpY}qD$Nz86Ro&tA;>wb$)!X~Zm^3Fa?Q;LNOH}DfsJnZGG-GL&pp-f9vS|}CE<XCy
z>U}f*yyn#JGurr*+6{jxYDDQ8%~G(b3VOI)ek()Acdi_V4IPSy*4OpFnI&9ma>(B!
zHz=cU#%;j}k!xN1H9{<oMdUv0X5B8@dOZ2_rJmB>FyZp4QOCq8SbIM#`!eI5^OMQk
zO>D7tLUCLMtG;cv{VXv<PKAe2a?QkRMq7^h_az7To2#jCWHny$uMT2}yUt?85cl*K
z@9U^i<16av>zUt)mZ(2ab^WqPzsf~Wz+U#XVqB$?00U!zrQ$i?hI_e*yDv3dTl7PK
z!$X2&W7-xggV{QT|4ZDBe{Y`qCb42l2~&@5-4q@^l@;N?XB%_Rw<xqdu;!r^_pF^a
zXC<rIaDP4&_9*S&VlgpeZ)?7}AD92@$y^(d=hbyPSEQoPQ+4Sq!})>{?It_sC%&Km
z^-EH^ahCs&HS3Qi7%64TTQN$QGW0pMG0$$RzM*PyYfcB#lFpZVi*F>eFmy!w#Iz^}
z2Rmtg$xxYL!5S*s^-}PyTGFQCTX~YH2d*Bmesx>b?3$(o&!L%*mfKFVJCLs+yj|-;
z#bJq0N-WE#2QA_@{k~7NTxpHbg*y@H$Deurn{jkzMg)gtrv1$5vr*MXvw1IDm6m;4
zyV!Z%Lb2Fn^L3gcai<?R7;fSD^naG~$~-2G2VejFn!-GL>P72Qf)h4yKALl)Z14T$
zIX9;r6g*VGa4ceL5DWK5#WRI!9s$!;mDL)47T?&jKryLgS+Q=<t1aS-QfEI3+3{w|
z1)dbw6IwT#AHFkEy?4F7-byiK+JtZ0WzSgJT=+R}YRa_3>X!=5a`r^^%@*(}xY+Qv
zgh_0{W$PQS?o?@9+r^jZRL!5X*4JVo|4beWljv)@opMq_r++K#nx<PMG4(pr#`LDW
z=@KiJ>-c<|tq}LrbkDKqPtu)}YgJU-^Wv-aiMh&2_jD!t$}68fSYNBgvu-X&{=5~h
z&gVW|bx(Qa<l{OA-bP+kk8!avxuwde`F+*x;5AAAUv`G*t1UM?#(PtZ^#JegjnZcy
zsCh(um#I|Jid-ijxo+{Nl7}DGsVblRI5j#?qe4;XdMBrD+1*9buflcgRJYZ<n3H~~
z^7XL_<xS$xm8Kb2<vvzEHc581`R2q6_xN7V&J)?<y&-0E^siTcLL*jh{X2bY_mzXz
zo!$4p>%CsA@VLA}t50*q7h4vV`bKHTbC#T48#88d^evn1D^t)kHFVJj%S92!Oy(Bx
ziR-IvDvz07nEYbG?m&y`h9l-%^#ZNatTX>=>|$oLxTV38kmG!7lHIO%_P5@Dou_H@
z>lFW@>l<Yi8x)dv|55s<zIQ_G(#e1O3=Rd`A9}NA)4pdHyPkghwEBMp>*5~=f*(!l
zYf5I={{GaNFNudQ1lB85t*vl7*j(j)+gL{N$_}wTm3=N}ihfq;c0F74E8^>)2qw)X
z>upP-v)`BP|5_6JV4~;kq->{2vLO%JjWqQuPpp<-Z)thb_NVjLsZIgse_xz^^N84u
zWM)Re-5<XG@Tsc(5dCw*?fapLlK*sHDgH9#<PbYk>wj6zeR?U+@}KewEIEP-3(g#9
zia8+=b0UC2yeU=6X-dQeMW-$%5uutB21>_TWo)@TCMk5X1g(v#O<6HXQJYa;bz97j
zBDI4v3^+F(?AWa4JIk7LTjEu&=cn4NZ>PkZP)oIr<_+E!xah%(tE-t-30;{Ldh*);
zR-rvYBC8v8LRm8|FPnRyjHy*q=26kh3$2Nri`B%QsaA+<AKuIKYnrCvj-JFrZKq2#
zo!gkj`GVD4rU`{Ho@C){QsGcgV0g|l*IAfl@1!jZmpS@osc|fLn6-a{Ark}B0<+xP
z8;iG3<2aI^e@{p_=hqHTCfAcp%OXPk3Ya|@&spj{*)Z`}aXY`I=l9c{N#E~0NMt^s
z<H)7G_~GT%-&Y4cJw5lm``-8!Ar7bJFa_wfa9!P}8?>BL;hIzEUNb3Iwy0DYM{cf!
zIV@Zr6Bwcd!d4t$5%)XL*cMW7(3LaBWy*|>Hqj*!oTY+STsg~^R)nxK9pFe7SKE@|
zBxyb?u<bv0>Whp+NzJW49wtj|DpiaUW?R6JK4<Oolq^55EwdQ{f_4gIgvv?COm$Fc
zaPSOrS=cHPBr`QCYkg{EY7XbN&IME116|x^Ci^NLWht?8zaSQTWao0%+pe517D%dI
znc(IvxMYHppkmk54sJuQz{R|sA{w5a84rW2-U^yb_BFmL9$dYcOW~D2%S5A)<vJUU
zruc2t44LATJHe@~F5v2=zU6DTD7G?9Ni<0aOP;#*YQu@7kf~8;)|EbF%FX-{v7n0c
z*TvWyY+({f%2H}el4Wu>9!!}!QRG0{0#*SRhR2K-RI(Qw&3@Xr{-kxP^V+-G4*OUX
zKg?#>?|X86CTHAN{r}GF`cn*YJ#!4XG#5X(&ZQ|Bv2^`b5!F3Ghf3<Ew73WhERiYR
zX(=vSa{2AaiS3sHU*C1Q&naN+Y;69d=&W{Wc=6fWwVf`ND;|9ptTJciO}MyIuvYY<
z9pj0%8W!6ec^`N479s7woB`bH8TN6DW%7Dy1U4tjcZ#*8%Xcb9Pq1W5l5|oNOL8f_
zs@u?A8@NTu(csqBfOzwN5*ED{>vk-fppduY;Vlonmllrit9lK?I9KglI*oDKv+$|Y
z%tM||3Dmg~cXgq^UdHoHmK7n*!Wxdx#8Tbb0_SrdeVJaNYwU8&G3Cj;4A<+OEZ-N^
z8D(X-@GI&EdQ3^)b#&btF45rswemNQ2)mjF_QwajTlh;W$W4^>wU;C3fwom&W_nFw
zh|JfWD#FR5t<Zc@Xm`$LE}aScZ{-|xZA_Len90DfSzuzxk_HYNuWcKqO5~Nkj*f2N
znRhC6=2?z}*g28coo2DLlwalMWSxE8>5J=z+X81wlu~-L*OYFo?VX_D5y4>{;Bk0$
z%FA?~i%xH!U729bsgQD#^HE?<xoqc=mx}Bk1Ev&&9llYs(wlQagV;8;Ew8_-WL(dF
z6d1B-$`6kvH6|N7>sUUBe5+g~w&cd4EIF2G%8ng{1tr~W4~$fH+??oAz56!PcQ*&|
z*rjfjj4FL>FBcz`(JL%46nZSPT;*hDV1VNPnv}ddB?_jcdD&|sSe>&BWzO%_G}S%A
z@k+zUYxRPa$!>QJ9Y3bTpsB&$q2bLp|KAq@#%E~?J8I53WJS8%|LRxHX*gxkv_~r<
zbXkKY1YOMV5Ms4In-aW8c$V*$^u29Ymd`RO<TTrvv1)b=e^2<%P_K+syO^!c;fp?p
zMt;~Xma$b;kKs!)llD__=A@Ly+pn%M+p^3o^qrt3ZycWS>F7G<joxveex0az@O9a>
z3D4pgR$StY?O5NzbTB^1>u%nrC5CNV3Km7YToE6&_^^)GCuLdDmtIO2PPT3dDdiUA
z7Vw-V_bTAI;<b}c#imHesr?d3b5P|FHvcqnzR=ay8IBGAMYdg#U)t)P89t>Wc;j@F
zPM0$^mWAgp9?{s>8gS8WZNOt`)dOZ$`*ib4jL-CI1x**zb(z=jcs6H>$l_h+G~AlB
zV-y2flw9=ua(iQf(swMH=P@tDi!&oUNw|BN##EzXKNkPz%f1NtEL)Y_Z@VU?aRaN;
zzm>^e9NlxJoPrhEUUfX&c{eauO`!A5xvi(Qmn|{)G$H=qyM|ADggloxw8gp>cLy@+
znG{~K&35}5dX4#lN$k<Cxeb$RR(Y-Sy2XCYEGLmke$k1z0u{lDJ3K1#YqR4&&G0`|
z@OqEV>+}>SZ_x`JA<g17*YlbTRL{vS-KQDyf?Kj_X{(7?@%oi#dH?<wW%U$D;hBBG
zNkps7vS+F1W)V?Y>!^c#VLlayf=;XnwUY9>pW@TX6Syk$#TKuo7M^JlAJ@1_EK*m=
zJCdq>x;Job(3u5I>T#SC4r$DqHE*R;h3Z}A^pN~TlNR@0+#b2vReOg)qadSxba}Nl
zqv56PJGs^|hl|Pd?n-ELwo{wf&6A_-`uVZZYPCg%S6Udh7%Xe-aa1te@^JQ<J(JG$
zHj1p7aQ?8FA)D@mwF|S2*0`A5d0^TbakeN+HNwMtVx-aW@)n^f-RgIvy!8FcrC-i<
zn{iwqtMFI(iI_sqqC3%XuAl9KABTR~b#ubXX{V|de!7ysc#-WA+wU5Wa$J9&Gmrg$
ze^Q{<^|c=Z|D166{A_b{_sp;ze@@1o4*3u>@8g+WcS_fvXL-@W`66h6qV>gQN6kgj
ztRJTdoyu*>_<dOJt$4n`g$*p(Nj!qmM>R|WFA2Fv+}Rr0c_3ug&U;%X+<&Cl;VI@L
zBoHU^V1wh2KntIVVrD&xbyCc8)%dJ!&Wb2k{qi`rve@YLhe@fEuG~C5r)Bd;6$K{G
z2~3*5ntkQJ|Ii5FjXI-gWT^c6$+PIU&yJ`ao3N%gn2GJo?br=Di-n$iebM&jb&SwS
z#sfx08+>jl%w2Am-mS{O$$YU%O~;3O?ZFh0sQXMp|8_gA`n~;O2LHyXZ@TXvTIIH>
zPVwt}v8D$xeg9o9mT2kfW~O^b|6;u~_aBF#`rlhV$}E8=Kkxo9<N5Kx&)2ReuomSx
zs7}s)^nBK|{YmRn{t7)4JG!%OXQ=eMVEHeB*X`CRu$FDl&PkSE)+jJVA(i3Rn;&It
z`*!rtxuO{UZOi<*MQvunnzi~H?sb29^Oe;}<Un6WSB4VTY=@^+K9>_WMW1<d%`2eY
zcvW7ZcYOH<=I(H|N6t?;>J%mY{}<c5D~uI0OgHPB`6pNBr2bsBE^ED6msUNSH0MlA
zhF-hVof6mBE50u-+FE}XN}ZCj)`898x^io=lJh0TkOdYOlFX9SBKlftWmdRyZm0=N
zNaWqfHTSv?+qK4rjv3k|YXA4C2Ohp&yH`CUba~l=6xIA~tg27dg93sUhN`UyiF#~Q
zu5H#*$1c6Xw4!;F_gYu)Sz<m48#A>X&DWUv2tO3;3(vaQVzT*|(7{&UHEP)_+Ok$&
z)10qlrCZpxt|@fG#Ow`0+Lxb}a$n>VR%WzT)q7%8Jxx&R;x)-7f-K@!5_UJ{wtdsz
z|1thnnm&(uo_u2fk7yDXdlF}9$AqWVFP~ZdVqyu}z!`X;p7$VMkOJrS&9w?)UgC@O
z=Qww8Z;yY$X0TDQ>)}S}pJ{o&!lbuoBy6cD-H{>W9vH-QJ*xbYVU@U?*;4_*FG=#@
z&c<dT&O4&&o(IG^wQma$-ufV`?qmIbHPaNm=gFpx>DQ(RFMY20Vsr750Pgryx2+en
zKQ3zA?VcuG5n&XOG1;Z`{fodiYzm(~hE}o(ueo4wzr5z-kN)lzUAjw}vX*xo5uc#P
z(Y$qH1oz5_$B9Ck+e5sUvTi80-gH&q*u_XCW9v;SM(mEktEMK2FjnjhafuG7Fb(q#
zs1THI)F@yUoMV>7tK81mA$tFER^*G;$WYhd&-E@BW3nnlw;EMBJ@kDO%+KQJy0*bB
zcaya5B1s`er_k$`SxiA;57i<sNf~YlkTUCEaItJ2OY@A&mPQ>6+*?^^YY2)i<D9ue
zLGWv}MoMkL%BFQY92uJpf-X7zS5svymfmV1{hyoDX-1IH(?GfC1!;<$(uWNTmrqpR
zuc2`*QBe0<o%UoGSB`q8(g=&mAx{hY7=8$cx5zEBH2L{)W>iQ_cxiSLOB>e-`Lrb|
zYZLM{Dy-fb%{HA9Q6j|oDJ@a>FxQuDoY4h=+blbcCCU?bmYvX?&UJM{_tOb2k`qNO
zCo(&-&2^dU{xL}JY1Dz8A!|bnG}Me93#p$HoOxVAb=tA0?T?duJVK7YXqmXFr>faO
zuSqjCsBcr*{Nx`#-+%hVMYJt&5=mS!IcY{#{Ejw_!nUVZvO~6L7kO|AJ;*)KIQ`x-
z!vLkuFHUoKzgs?QvsznNz|p`WD8M?=WuZt^7dLB7R8ijlipu;oiCsK`h8<<ivo^3E
zn8}hG-ckHA@J~^Gil&nehg|i_e1>9up6}D$c1?(F(i9S4ed?%YGG&JDPS*M|fxi|;
z4-<`;40@P08riVUH!(?`UMbos%9y%nzQ1c|Q0p?MU{lkTtp`I*=iCso+%BMD*cbOS
zR8w*G;ZDJcEP3JP0&4_VS#~auU}Q*gVtYSr#ph+LN~!Tlnl<XI(^yx=-+jJ(+O0VW
zOmpx2OzR4o8+~A<$W4LTXK6f)bG#p>t!hmWX`IUvXjT5bcD;(d@@DV;Z7tiRmOXcm
z?hTu?XYxEf$qJpDg7c3FcvUU<F?0U$s5Xg>zF~=#H=iW0XPUBoTlRmSg)v$O+13fg
zZflrQFhgkV1%;`LyCz$$RB6$F%9e02e1hyNeM>VomxFvgvnIS0O!)jNVqKz;^@(Y$
z;px{NtvGJEjy-e2k7v`jKh$rW*{u;i<Bn#F#KD!kj7ybP%H_M6$#0&)vq?ZhtY;_J
z{5R}{(i2^#Pn)T9Ti~<0;3)}Jl|}{)bGEe;O#gkKxp-y0!FS;~8NJ5d#Xb{-9~|-h
zVQIDN(X2lch5mLJmd>nwyfEbO#mcn@x?8fWrcH_WJstmYmD9P%t*R3@%<5{k@Mt}v
z!jS)L1G`(fsY*`R$FyrF)4G>UbiSgV^IP5Pgm<9Qyxm%YOP2<J*y+8$B}%0-BxptS
ze}~Q5xm}z89$ynvv4(|f!9@23^POy+1GBX^YMgSl-BUK@g^NJZ4Xz-CIR+=zNlPt!
z^(|h|RcO|4r@L%HJU<2hH8GrR5i(AZe`CN_xXj?r_ojbKcdD<N9u%Fya&w`9htyBC
zl_%Ugb!K(VD^c5PxJ$Bh$<`aHo?4T+UN3o_Sbou4kdJ$#JL9%RU0#-Hs|~(7OJ$2j
zbS<6xOwqbk_PzMl<3CmTm#174+L69XXpTj;f3q;dA`u(+1rIXjJ9h7Uk?r+Qv?=hY
z=LW+Sf2-$wzOgtkY2}%f5i=MkCSFS8me!o@uq{q{Rd!i|&+QFvQ|1aCT*dSuNUG49
zy>YLQ<01W}&i}1K)Fg9UHY&Ruh_ZhET3nGcxjsB|>bIyvPXtsZyQl~Y9kvpByK9Pv
z^AuZc!7Ga=Z+^DVopE`h%G%CvuGg}(%{K~fOW^m++_~)nAG@Q_CW)H0hF+nab7mPI
zJ9Iqg5zCGg7q+UX&V2R_PN$X!a8KXRC^h@?QP<sz`ZN+)vz?N^PY6G~#Je>^Yg&}^
zmkrmF_Nx97RC%+th-oj&#toreqAHUMcP>^xTCrVzca;1@*@>J&eS(KyHyRr*+2?nw
z$msi~zt<}*85LH^?9aMYANAstkMU*>4MBJ2EpL>!u*yi!T*SHLz^RNc>AQPs!!)I5
ztcp-5DbiF|6I!|Ty`_(c%(nlR{><a+PKZB!;(E>5%Q0t@W}moWF0=SU+Sg?nA{!4&
zds<&QJulefgp)+ied`kjG8(5Cw<}y^jY%-y={iGOy)tG;)@HZ1=nDBGrlu|0hO1&?
zUmM%)5Zfs^aqZ&mYq?ixYYNT@`W?W&$G}-@;q=edhkh<rdJ^z`R%glquB;mUavLZ0
zu7!?K7qdS_T>XCR=<#D~ZY=u!X!@3n4PsNfTRbHNWz=uoIV=5oqmzfrojqqCY&@}T
zr9iz~I}f9``1if;jY}<Ddkk`VCYqgDyzor)g-v?TPvxyz<~3*8wln8#I8z=bFSotE
zNw@n%#N2f=AFaxjoo=~H|7_;8<34hcx6kZ+c<}$yQwJ~o37R7{X}(waOB=QwEZY=s
zdM2#CQG55`YOAyIe{R%VyDsb_(A{}xRr8563!}C;d%L_)S)tmJ*tOkj#=Hf01Y#Xn
zPnE1+a%}CgviVn9L_K7ruP&K9ce+Sjp`GtSp-q=!bWg91U#cC=God+6)$s16M_#HM
zZy%HDy=bwj%yo6pFQMJ1_X?bjtucME{^;E44Vi&eH5-~{Zm{uPydz~*F#FOQ)^qQ?
z-r7@}5FUCZ?)SZLRfhMxS3W$ucTeX21mE(ndj&qrp3|4A%l>9qc_>42w~T7$Nz2k(
z-aBOXT|T)jWYdS4vUes*?cW#?7_#g#$L5HVQ(=a)8nZWVR@)gJ@#w$sX44&OQnntr
zR(SW3PRW_F-ZK%ut-MWdXuK5nc>BnzPjDxTeU7bw<Z<<#A@kH#Z)~$VY>>0`xQCI-
z;#;$7)(aVMP34Y><cRradHeqzA+O0onHAX*idD1I<$G5PecfQ1#ut`lbNLC|@qVkz
zyp4MfXXm|Nu}E~*ZVu%R<uAFyU#D$)y|b6gS&_wSLq_1r>cEzJs!cmDr#MY~ayI?*
z+pDT~?v+e?9I^4l_qF$L=&btRdq2YPzQw;q8y}uXnN`ZPfKftPHNvXrw&D#h5se*I
zrHvI*ud{D=aZ7*eoSc+9Ye(Pfv|W?^ShbS3$VXk?RI~T-F+Tn2t0J_x7xCRWqaJzl
zztqE*8cz<&&ACx`G3ds->u+SFyXL-~_V)TcxtsUx-W~b4WAOv^AG758uif1KYe{JC
ziAA}Gc><%>*NN=j${Hd3EHLmwYxL`|#;36yn-?s<AW>+SGC?SaN0>kNXu%58w-cTP
z{S>@9_s$*h^sJ&gQ&o<CSKXuW)JfPrV`E);@NNC5&Lw5F1xAvKo-QmE$vk^$pOAjN
z=z}-+AI+1#Xk$4&ZkykyPt*P#nd>KL^Z(P8Oqo49nza+_!UWa*<}H0+eo~up)AfUb
za|HCxA2c?Sk(T*r_-?CDO3??M=kI0Lwchx#Nh$7{{J~Rg*77d`8#bgZU*{>HWyi&n
zHe20x&d;Ljr{=BvFW|HIiYFUCv+3U&lW-QpFT93Q6F)rp_x5O?VfTEw-t#|RfBc~#
za!~Z}BkvnGYo0HCzE|Ls;`YY$r8eh&&XSi=TgV!5LbJ*H!VhVaxl9Y9)-8)!ClaTp
ze(~sS4*w~sc{}$oePsIZQTe~xL#t!Qy92j;EIZY?aQQ;R&a1y~UQ0N<e_bq7p8TrI
zKeA@bH9Z(qqy6;pjt7OG>jd+^#J)T)V>)5)OUcB459ROA(YkLbZ+fTZ{ye_?>GRI^
z+!6TXd;jzLe=qj`za(>x!O*Lr!H|hvPNq0OQL>4{GgnX{F+r%6OEK)ojKIeSc-bvE
zepnbRan)n?o@TN$@ad^ujmRXH0)zi)O&p#}GBU5eQqX3!GTij(XVLNiMw`e_JgY8b
z1Tp4xrTIur4GD;5&)&MpRf{QbU92h71OeAY;hVIQnLHZ8a&NJ4WVPK@N<A%RH2cH^
zp$75y4vYQGzg^|{vEFeX`^QfyGXfv1*qbWLedCM7rY9#R8HcUwn7K4<At$TH#G-&l
zuIqvooSr28THt+cn&~9H*Fg`bG%raN_MM<|@T_m{E#92P7t*gZT6W8u#7)x-Ie4f!
zyTr=imqSoWQP}&NUA^b3wrX9=a&KGou9<^P>14-|Jq%y8uJW@?GPJ8<G}QLGut>GD
zw)L~&y89>gu-r=KZ=7+(nVW;9;e@U^=aPzoZr}eZd%Ah|H2v~j{ID;3odN&K%N+~(
zRF!#@54nBJS=_{T=*^KvA>|cEno~kLkFCpDqv3k}L{y=|IW{R)C0(|zB=bvFvve0V
zX58-ZWYn<W@MO%B)?9KsXjezKwt3T9B|oiQ8A~*SSFPCkNy|y8k>$ayMN`YN9<J24
zY!6zs$l~P}?ekXkhl-WwPJ6xbJb(2pufuDO-P*W_WzQ#nlM6c5-D|gAa8fYW%$elf
zy~|_fs>@+LpYj&0`okl7#3huA>%x2~VTFx>%T{w74>q2?_1dPoEu{-~ut)?gx)3qx
z(`H?t;-WK?!%kmSTC_{#)V8j!yBgciZ+Y?dtZcBWo^?;h?XVpS{}(ZJiAHG8I=6E7
zErULu#TP^u%xm;2-OStdEP9Ir!x`p)g<mghULVIOCeonv>reOk1V)iR;tw0yY<?*G
zzo0FVz^3lk;CNTB&cL2)^=^ZJ)oiCe#7=Wc(_H!dgzq$+$aLR8cX4Sc4aJ%9Vhdc_
zs-{>To6*=)>e|f3^ebJYzHO%498n{VW3wVZy}5ot?60X8qr$8Oi&n8Vl&NZ3PYUvh
z4b<wqIem_8nYYpLBPZUhXX{#UNu}+|BHuM7LT}DF+8+CTMerq0m-_N&Ht#G~&RF$o
z)tW`yL^(A-iFSSLQkuJEE#v(|CEUJmXKBTR2HW{5hW-DPRlIIibk4P@)1S>OKJ{NO
zSyuT>T>8eZCu@#$nEbU%?%by7lq_Vbb~S9a&PGq+-&=b&&HJ@cMPTVP#Sc4JMQ-P8
zeQQ%SVQ=lqsSjL^udpgeblEOe?Nz&dZdl#d?TJ0lPH%tk<!AJNg^r6R6ZZaMGbwQR
zJ9!1?mYrO?x{tdYw_PE!L0jc?v@1u&rgJY>YOeCW+hg|0Qu3i|hR~JBK0S$pyhglD
zF&71$I<`&FRq#^k2=ctTZkhE)%a>Uvy*VdsOHkE5IDz|I5a*2}Vm7*|EbXfWCoqV$
z&U3xGx}N=FlgO1+?ztw<inTI(?#@#>YU^w0U%pEE+dQLnJsisdxFkixA{uw_YpE1v
zdtUyewY>Ge;e}L@H78_3jULHbaV_XmS{)kdrplm}rmgwp@#O3cqEjXxGz`;H=2~~k
zWA&|{1*PjA-%&dz(ApidrowItH}}ObwPf8O)f2xLY?0Z@8g_%Jahl?TFHBpv+dp+t
z_j{;x>Kv;c_f|#?%a!gATZ-*||9!0>8Xe&yaYgR0pt0|VNPz?ImMV0cW>h9k>{f9%
z^xl)oVbgF%+N}J^x^*qDd=;&iYMng96=qPNY|!*{rQh?N!L0@95^pXl7ONbZ!FkqL
zqwC_uS3l43ywjPt^tHlSzM$zFGaj~xN(#AeC=(EE5{h7Y;c+u9Vz+eOrA0A66TQxq
z1onj;Da@MGaDeU6{=gUiReAz6Mdws4y0m!Dlg`*2g$E)lT%8zC6|CK%5_nF-a(nBQ
z`l5y1%lGY=XgkeycD2o`HNSos7WM@PxJ;Crz!1DBLMu&`>8Qf1Ew4OpYO;jKzs*uI
z7n-@kQ_<YA^vQ+|ZL@CvF{tQwT>a6&iFMJQP(|KHR~;Rc8}Hqi=zLd*fobP<#RF;^
zR^AdmaPrjDiDn_sr!o5EXlG6IwiA0^xlqyHjc<zmSDxZ}zDZ4NkJ@%dNtspG?Yj46
zhh^mDZHre(R<2!<uv(xz^0T$iQcbrrHz!Xv6fjsNblq*$s$G|6awz}ukv+$>>wIO@
z)YV!}9GC7{a%IW3D5b=l6e(FQsQFf4(f?hO6nW2IJE1A~KF3yN^7RR>_k0(n1Ul+G
ziq&pq5PA01c*54B_kK9-vn=&krtr#Jg!l5=yXz*+eZ8n})w6A^Cp^1!d7^@!F1g@l
zH04Tgo$ggRQ=cg}o@vdQw}(rP`LV<CKPy)XnRL{BR*GGCgQILqvr138`Lzv2p%d3l
zJngt`(uL_3nGso~FNzONu~BEWj9etMI{3h2DbeWljca+rtwNc+-`8q2Y|mj$6)o~r
zjN{oNIO*BWhEK95ABj!)U}YG-W2K-&o`rJux}4n`Rz9`3EqG3P<26US9&=&mtI1m1
zdb<6dyC~*=-7$6N^04@C=Oq=zcOE~x;Pj?hf^8ox5)R$^&(R;cW6ANFBbOBxu5k0+
z6u4#I)wPSK^*p&3HhUr)kBJr62d{~Tf|#w`mU;gSn>E|&ionHG;es62_57C}I*H#l
zTpydY_J7CisV~f@u3a!Oa8g=~*y+o+%np@>ZT5Ye(r9|DXUWI30(L<TMuy!^FYitX
znY=A#;;|DMr9!dAJMOJhh+N+o$=7z{Yf4C?&Z-5@Ia*5}-<ZP^P`&0upWOR7BGD!3
z=TkXv&f0mipeG|(L#k}L?WEG@iOp*Y_{9z9{jZ&HFU)Yl;i-PzyFUf3xg?x6F|OoH
zyh%W^@X?IR$2{uRtX{oS;AyKwVNkiP)B8y!XH`tkOxmqF`ODM>U$Z{z%liNCmPx5J
zsc|vw_;tmjS#<T6vt8xJqCN-jEw}UD>fg@j*TniyA}N{WiPS2~z<cZ5b-!62sbh^z
z{I^=BcS=Xetvb)s>1%dLAK$Cf$+*f#?#a~40^N6|P5)d9K0E0`imiRJ0_*N?Z=SI5
z6)ju&YswVOzX$f@96!}wn5w5*C(xH)*_V=-JI7M)ciG-Z)x0{-+xquD241vOzP9Vx
zZEO4BhW^+cM|8JbTx4zhY!P3^$)nFNwLRPHHeDmb*#6x%aU(8+_*qwVA|E=nnt$n^
zvczk#Tee$xQI^il4~0%Sx4n{#?$6RtSr~Y_D$-f_%0q*T>i$>Ma)JzFPIGG+t=+?E
zHecZXz7vzJ_w3ef<xqd<y|U{EZ}(x2JHk`X{xqMq>45gD?bB8XXF6L>TOj#X+wp++
z%tOrP>xHzZ?$~%rLa0NcyEuA($>!DHvbKNyZ6_fmEfFNO{zg~W>W<Z`ZSDkithqg7
zcY$#DFW$K~jQ>9}erUi|thlo)frC4-qv`bwqwclWq8&Cx>}}+5{uH_PTK3+)&s=w2
zZ&<|Te6z^;cG2Dm%?BJ9EsqFUZ~Q!aN~C&k!ScEy%hm)F?yP-$tG4ZU?d*^vsw(KD
z{N$j@!!4^mZj_y(EyE)gyHatdlm7Gv4LcT@FTE^k^iwF_YyPi*PT@mSChpp`=5j~z
z<R%S|MLTbZ1Z|kWa^?S=s6QeWCp!fuuZg|l8pFc1(CdKH&;6poT#rBWZJmAi3fG#c
z9S&?yWC}iQe*B?x_v-@!HCpR`tT$h@Dob0>b#b5V6hpBohN550(nAhTm1sU8psCo^
zA*0;=_SK3_FIS{U%AK?jJH26hwYTfGlN|~o4E1-q)>celXXO1asmtkjf-~?$iO=!f
z1t&6RsqGFp(c*Jr*M<`gmeb!X*xPCCY`fd`R?=RJD;<3{X35TGhB>>ff2s99>Dm<B
zY504Wrt;oRFWlZf)_S^m)vPtkr(K*m>&r}+$%1ozWF>w&F67`f+GNt!<i7sKtoOf6
zK5MU^u)wM{V0M*Am$u{j#)$sFGaXeE|1+!)5%%$1)O=!pwKRwDDvkyp;aL{TW-X9u
zd?IhkwQ^_3V(*WdJ3|&XEIhrlr$c22$FUD0>MGs(EywCMJFv`hI8tMq@rCbLj9coO
z9kP}uc}#ReH}2TWXtMBt(iG2~(vFV#PZpfta#F0Mr%2?uq_un9DvpVhxXWF(`~Puy
zuyDK6=2O#V_?~_suWWN_<B44x4sg`#EH1rc*?WOkbHmaj8tU^G?8uGkG@G;c`5KE!
zA%`0;SQxS_zA3~|Eg|~$h+pf49cDe7?-y8@ZRVO_EYGW=GI_Ib>yO#Z5(gQrMY)Rf
zDwDQcQ0chRvmnLNWsbMYo8q1&Hm92}=zaXXL^fphhfV*FeDhxK5yIhj;#ALr25mv%
z<-!dCECN3y{ocP;jSuQjis9U{X1(`m-#VTXTb|5!UC1HAdqU)HfYL4#k-e)~ZZtW0
z@&s{g%nIpDOF57==fH_C7hM!3cNHH#RCFLsx@)hsu0ta0{@I?vj2ihmdO<$~HP>!D
zsLWkda_l%m_rym>RDX(YeWX&k$MIaCm9p?A4h5$*)+_Fx33_y=SNx<+uMYR5J?G4x
z8bmvqeORMBwdRc7Ux!%jCHBhdFDe#%f3ur2%FxN#Zt83GMV?z~xV&DjwiD@D?)2rn
zlqiGhiVlStN4$USoGxHBal)*=3a>4S^SUOTt=@9>qY&4+D=Y8wC@FFNzYzKN_`VR4
zUWRj>C$8Qv3GB@fDT|r0WkN^TfjKNs`q)Z#-!DFJEaqD3n~Q0p8`Gb1$;5Ut#GFyu
z>yTFMkbBlTOLgO&ApgI&oQ41R<jvXQUd4HN&!roUzS?{FTR9pSYDGLvPPg6?_4VT2
zw#e)7W0y-4%>A?$-@9VCPRMWFjv3;D9B&^wy^1+^>-V{?iW{%a__f<|ygPI5?UIO|
z2{(GT?ma%?hQ;HE-lGw1uFl<SH+Oy5`%b`_`%758^VCVo*5}S}OgXIOs(Nhou1Lo(
zGEbY93#Wt});_gq#`SbT-Zr5I|Ile=cdvi`r6>2K$$QU=Ij3AWO!dRI-fF$M({*O&
z3g!Q<D}p(g5|5Z(z1o>DyQ%i#)Phhq!E2XS1SW3@UbNwwJnItc>TA19H&53%qbPc5
z@>ZW5(;$^r>$5dl$2MH~x29uy(e;AhJL2AVw&zaU{b_1bkG5|p(=CO0_h-7-soqky
zp6Yt@s;d+~@877N31Yh?SffuJz5nCJ)QH$uXF4X#jp(htdvj`RPloM{ujjg27j$m+
zd)?y9!z~+n%G~}H=L{2v?^}4Be#`7_IPb!ET%hX8Yiq%BDJj?0A#EJD>{PsZH*}ZB
zo^8}Q@qst~L+u1kgW#zTqym0#`NX_+<-!Y^D>c4J1<S1I;PB<-x!UoaDQxDA__a2|
z>z{fHp1mOWReaWk|ED+{`n^7e{^XbtpVfVIG56Njm${Z4NL%`VJMAQASeV122PtbF
zOsnnTJC|^Lj&r8$rQ5%inI0WD<8o=@V=glW_t}#Z^Rg$p1YR#t@jc_Px}}NtPw12h
zMH57LHCa8HI43kQ#`cyn-#DXs*@bi7meM=D2GLB0(X9eAE=?7<H%X`5@bHb+h-<Qu
z`gwPIC6cFUBwxFF_g3qT8&=5^Ts%#lT(gLC6aOdj$b0vU-j!-|J)NiW{M-_m<vo#W
z@fC@_hsS>iE|pX~{5Igo$6L{@*Bb+xjz&*XzIkwB!=CqAyB5x#<on0&X2Pz@;LiF|
zq3=n@6_ecmB?SIcnD9d>mbdickAVM^Yi@SkVK~QB*U3_6zjf*Ql`{=E*%EJ5@0Axl
znYb_7Qsh}e+qD$2uAQsb-1uZ;DQEcAC#W}Jstt=xvqJ0ilWFe1%+?(^e@xX^y18f7
z)!um3ZjO^GAA_%M|9V8D?3m`c_%4aHZ*L~)U+XYDmVQh0vB|x3(_^tl+tO?Ort!YE
zy}NmtjkBuKTBrFM%d=zFOx=BR1Gn{&QeG!H7tdIpaKV&$E930<gp}mybv|>S@uEMl
zWqw_xoTfp%O4zh7p&CJ~TYdDdewwYg?bf8;Loa2DI4fsg*!^qizubiXp`C7i^S30p
zD;d4|%pt+TmerJ)u!4EQiZt%Hc^CH>@Ck}}?w0*8nd%#sdgUV5yQ~dj+5Bp+d9HD+
z(d$l$-d4Wx%$c5J`)j*$+7i3;7CXgt=0Cevs>;1o@|KHd)7{KHHD~YqjJ@i1#qpa}
z{L-Vhj&&w&m^%OX1Tl`EQ$Iv*uK&h$BlmIdg3C2!u~QG^)ZL3PW6M3u=V>D1FKm|B
zFSFpKr|A7C|9ujAcJ~B-rl?PMezK=wqSsgd1xt6f+=%0HR8svJv%cec!_8jC&_}Cc
zANiQg?!5IxN3&DEkTb0_o+<R*(xvf#uipBb`qVpFhQagcd^3I~PNffN$Kzu~BJN%I
zS1{vvSk}^S(^t3&Pgs-{Z)UdkUI%B%^qs!XcJO-sJC^^4@Ab`F|D&zn%}H)~rtJH|
z@#3);6OJB^?a4iNzWC@vF88xqrO&4;Fu!3sAsZ=NekA75xetQ%w@>s*FZbBn@a~O%
z+{g2Ig>`Z#FNJ-SJbzc}dTj6MoTXW57TGrv+%jyO*IwTB_=wTIb5FRQ=H9d4*0bO)
z$Lr>uvo2=v?UFn=x#w-HOY4uAghv5e9$c7j$DwMF#57f^HK?QGP>Sz7A+3aX?aF{f
z4bwK<OxYDwT$x?$&XzUpL*n`i@e9|9M<0v&8kb}JJ(K6C__oM?jt8H1WC*Vo^Dz;Y
z6jJ*bta#>6*|odepOWR$4ovOQc{Pty{a{gV$=-KdyA_lhIS*;+ahI?e1ty!T@B4o?
z>gLOUdd?8_=;J>&b#usH-}+{HLE-Tov9<TUR?EE=k1v$|?#DC5A|-xPmTfj;6z8Es
z!hX}!rY*=mpR#<qqSwC|sW%Qc?w0X-URLPYA*sE@^3Jn4fh@X9o<4hGSS&BJ-Z$;5
z9M@5~CtOzbkFNg7Ow>w!cduNmSO3esfMYN7jFnuO6ie0}y%i|3(xkBa^{$SU4##p;
z{gdClPdaw(uiMSri=uOnR!m^|(DOY%?d-+ZM-K4zRAj&5nto=t?7`Q&*IknCvkp4`
zeUWV5)0?uTZqsJlO)WboYB{|$Sy=VLhQ2o^EKKyibv})d`Tyu@XjO7~uUW{igy}uT
zM|L#X9p?P<U(H_h(<H@vizKehG~JUG{^Zcr+!wo_Y21IY>3z>r?OBEIxc()ivfV8I
znkm26CP{6ji;?)}_pgM$*>XO;CL}oDC2(_j?ZfYlORp~Y_Gx0Dn7m%e)DOkh?T@~7
z?EjuOn?=Eib0uqBv@zrLsuL!!FR=f;`2WwP|F1Tw{k<&yx8Lz~*8vB+AivK3ip+8|
zjadpe=bb;)x?o!Q!F3vPMlUS)?h^UO`{mqS*(dvMh25VRwD@{}T}Qp>pZV|aJQZZ%
z4On#K*P><1KPJ9^{#Kl8W%%Pw58k|cUwZwy$F<_km;GL-otgJ#&#~*%B0K(-s0*dc
zRrPXFcB$xEcX+Xh7JG*w6Pu9Ee+`EP3_)zHIf4p`oeUigVoDJiA0HiJU|#BBaHHU%
zdmD?h-V=if-JZ=F{#*~7E~I)l$#@9ONxG<dZnDA79ZyUIk`A&kF$jogi7dL%=*g=m
z>b2!%K$8b+$eO4r4}w=kP2>=WDpUwx7<16;tyaRD$fo38wGM{BQ`6ekL~T@xIG>=~
zQ}y|k?uqaN_xCfu=Mr<c@!)Q2=Wo7k6+0dz?{45^Th_Bt*71y3uHch1l@FdbuI1Ra
zDl}?K{ISy4d?{ylJjiI~;+~WgTyW*}Z83AfCwnh!XLxX+SXe)A&c|iCL2PCr_bxs-
z^7tsTN8{-W%Y1t*XL{Ga-IQ3y7_w`BkLN#||G&OG7gnj{D=%3cHsdYlg7^#j9RK`!
z{aM~a{(tRX!7uk4{=8*hkoZDzMO@y(O0$RzPO&ACy0aohW;2NCwJ6r~B;Gg{!?fk&
ztlafyrwMd1&#idmwBzZsBu5TO6Skf`2Q4mWq&%<?=uJ5=W3i*vE7d2C5{gfxB@YXV
zxJKCrx`;Bk2#U-NVrJZU#mSTLz&w^9AAx(>(gG^WX7DZz3Gm?Uy&mIpTkyLeyXC<n
z5=@*PCvOxp*M7Y<*I!8FV7Q>jmh`DFdsQ_YTliQss+kU44X9anu{|<wiqsOtW--xe
zn#-R=X1rYf{ASa`6lTt{TTWYdEW5ty=vJfKS!^2%7Z+|R`|mZcV6W%#*^~B96ue#}
zZ7C(_R(sEC0*jZAA?xhEOIt3?DPAQWv_-5j@}9$E(IvT(5{#ue(;nPA6~|JrSN&EE
zlhL;M^<ug=t|z^n_x;`O*J2v$8@^a^%uJcfd!<kJYe<Hc+w6bZ4DHh=?RwZDu9_3Q
znt5u4V7EkQ&+RTdJ{1*B7p@13j*FeFnAoHI{)dv|;u$@H6Flv#CkRD!c29H@)KoaS
zi76)g?Tn{Ox3tdSSsL=z^%B>C&Mj`-2EyBHHl_%uyUz?=8hGT@RuPx3Als#3)r&%(
zZuhP|JH<69`RA>Lo*L7iWGp_WzHr&JrCkv#D~|daWc3`q{XV;0cUyUKtLuNU9d}Bl
zYYZM8E8S6$Qz;uaA$y}%pMm4*zM$(zw03!Ptvz`5kLB{&e@@AIUY(n}X6k}+Q^uGx
zNz09sz3%^-TX}4!$-LRCUh`E6ae5fK9bXe7As^48=kk6p+p5(y4acAFuGxEO<8P@?
zmuzP|xWAKkjrg(+6WuEBahvk&jGOuTp5-sidYMg*4xc<GIJ^<x`O$T*@=4E02lwuo
zDAGI0eafN$@u`<}T{zb?%U1d%O}b-xHuHAjt;!U4ndLfJno+Z7IqFqR<mURIB<m9r
z|Lw{_78M6>CIx2A4)z%ruN)SirWEnrC}pCA*z?x;hp)sYt#E#Hp}@%OPS1WlqgCB3
zX^B%h|Ho8Gq)!Mt+m-q%`{KGXVOKUq&yq#AoBHCW&3(1yd34pq2WNIIkzp}w;kJH!
z?sC@hd0A68s<B+0CbiLYLf+eTeUDznZU~q$GwjY$ImT$Yeap;kt<~0BH<g5c;(IyY
zo~vOZcV_?}@8-RGV_1wAZ+z0WkA2^%*cQ<>hDym=%fx%O^1Di0zHw4dX}|iu_sJZA
zl6`TOOZqzf6!&zU7Mr9j)T5Jb8K=Xoc>0>0{#-$2N8tm8I~h#oE(z54ntXTWITg8d
zz9Z^^g_FY9JzBCLV(skeRUKEjuAa^RTQpnYiI?l4plxasduK-`@yASZ;R@<GWzwpg
z6wkBl{GrfiN)64@`~QmsZ$HKnQo!tc!S3TVajAFN@e-Lak9m%3F?zhSSl|=*tl;v_
zrYBhnvS074y>vxf`=6GlYOi<MtgGTrJsdfMGW-SNink?bDR2gFIOkwCL3S?R5tZCK
z@gB)RK2uLjY&>YARdu6z!mSK9yEDvBKF_(X_e60;RfkDjoyZiA6<(ro3ph3Izj51e
zJk`xgTy?6tvYH{|41Zhuq@`|G-B<6p<`KU4SA<i*ig4{`McikEJe`|TC-yzb?C$dl
za}g4_+*mWg&|$^Rxl*T>uf5qb<9TMsj43P6%$RbqajWvJGfbiy+<Ghl3L*@P0#46Z
z;-R3qEOkv#)-us6b(fVA-f+E2I~6ZdQ0A#vmGdMf>S6ccgrdCTk!Rv3XwR)r%TGIV
z_PK59jp-bcDe*Oe^9~0kYi%v->MTxGeeo{KdvaW1iG!(M@v3HHJv$eMO}c$~{j(UZ
zJzb-}B|ELkL3;OW36?%aMh3;7EUY37Obj{<3=9knOb(0;9RC?uIb<v@h#qWW7uJgL
z*uZkwh)3DW$HL3OwJjxzb47S|f_52`i))POge4A>g@af5So$s8(WuYbB_dJ0^z3Z&
zP)`v}zvT>t#?F^`tb|_#%v#*!S97x{=|VTttSA``Q7y+60ZWBq+(VVFPkJ5EDRp?=
z;)b%Vw{EUdpK#W}$U-tp#3EcVrfrSTHl?jb`A+)_Rd<OV$og<Ftddb(=jtj^Nxw$^
zB1wx#25~(Zo#bt2GCw`u&2&blWk-a=?8YVR^On87nCGRpnw4jrMc4cIn-TeWf(OcZ
z@{P7dbFVPX67ibc5V=sqVf7P>NvrociT}9sBXkC{>!&k%A=c$H*{7EM4}T&1=kzCg
zW$qC3346~?f6sb^-9KQ@uiro3cW>94vgp8(7S<p4OPOMic*V_nk~+=p>kTD_g35^p
z<(=D}rax?zxOYTcb7knVfUMX6wMR~qr>>C9V0K&<pviSL^txBx29JO&rh}=;l1BH$
zV^Sn@k3?y|?U|(D!6Y;NVXf{}mjGW~)tUFo^j&Apo8%t0Q>Dyi<s#W!r^3#u8uuE1
z-gSNMD;ZklJ8xyESA6NzWnMu?C2y3hS+lq`H1lTC`R>An8PTz#T+=6YGk%&Wp0qsk
z$@HW~xg%aVT$<k&CG|aW4Uk@Hqjs}(YV5O@%jYp2pY64BshIQ<?Z#&dGuH@RdEsL1
z^*`jI8vDnsp4w@gPZSE1*DSi@G==ZT{8DyF_uE>=irEuAU#PlVDW7N?t+3_GTA|6C
zOy-8jZ+Ud{`u*+Vei^AQJAJ+5W1jv}UFW1GX~CsEtLLECI;NkSrZcR)@JuYehA$<0
z!(%p`0+$9;H?A(hP`Ae&%&i&$snHc;hT;WlwKN>rIHzo6kltPR@wjQ<w*?MMj#_=#
zWtx?8-qA73Xzn%x{Vi!uMM2l6?hLj(o!e-m`ZYMvuG{&U>*F7Hg2K$*!#>Y6dmJ;_
zL)Vd4c&3tSXzBF(VYwl$QQ{d_&U*x=i-!29`Cd4~zVPlc->8Kw8$4ILh|itC#dW2R
z;hV*a{FEg-{<lhGH+Zv%YpZcbygbnO^XRdK%(h+=4o+LTWD&#Ud@rT6c|{Cb;&UCt
zESx*HY3*vA$N9z5wd<qlYp>*vU9%mczjVLWC^y^QRd{vnq}`?4=Zie6c)mzI_gaUw
zs9gDe``LdgPPfng{Q0R}zr`tT-uvQ>!JVg5AFsYpRPCB^F|}KF?VT*?cm^Ssf?r3s
z$Jc1jS@COvlU!BCVg^~abFN3GeeD16xark(FBS{sKe|uq8#(>HBwSVACa^>{xggTz
z$&_VFWqUIvSp2V4h)r4&;VUA~B0OV_n2=%Th8AU!$?Tic1qF5)WGqhT3TRO~{PxBN
zb-U_?e6~BB1-RC>-=6LD`Tq=em#2m;?H+#=oo{_nn>Hg?!6(Ec^bDIq)P}=dTNb#B
zWgnba^`%L9wS!=kRLS<=2UG)?Qs#Jxw65RB`y{mb_xTtGwq0U^N6d0L_+(ixwKY98
zEP8wU&a;Z(w#&VKrd*bxPBzcleyh6Ft(_H6#B*FAYVDE~rX5owznIC~3hj+NBHX%Z
z&Gya3C%q!>Je(z<yQp*3)=5(xdsMXww@qL@_o>37`jqZ^G28G<&!#%f;xuS~c`C_^
zziRuQlOI;J3y1CSwk$oQWc|Xizb@qStiLPT?O$E&ugfu>efv(D+WAYf*iNQ7b68K&
zy?gvz<5Ydc)QY)JPX$P}Gk!``i4N`jar}R)=9~BPwL5)R1~hxvDQyy6esBuo$C-;8
zd{3C4tVxLr`{q#LxWyvivx1v@pou_{u4ZyU;?-l1F7ovqGFS2Gl8D+7WUw`aLBIE<
zHtPp3!!sco`mFxmu7a0^-hEka$GghA^;Ga8rcf1k-B$q<yDtB`Q=(v=x@rd7rQl_4
zDN8j)uel~gth10@8zG?+<*FyKc|**{uBavK;yZ7g)i3+J_U{YUnEP7cjb9!s+@3Ps
z;moB4RRRW~%Ck8ZzDP1++;?IETW<PV4&xcle>WC=kMT$s+BCD-XPUzFlV_fpEZ00?
zQj)-3{KV;thxC4p4Z4g?$u4EZw{E;_wTQZKq;$^_C&m9a!-Th=)y*r{R+;rJY};C%
zbs8<+F>h0xcxDPXsJTqM^H}fu&cnU$w3kZXeJb{S*A-E{lI_v4&#JEPyqmkUbg%Zk
zH$mll?##_AKjj@W>zCP%y=!%~W_GSqax9kJav(o)L!@usFY`I~t}JUfd~zBS_Yrxf
z9jlwzH$B*$cSQbumCVl?M~llB*cw@a*R++YFdTD8<4&oV+i8<>{=}opF8v}G?f$>?
z7Z7%Fm9p%0eD%oCZlPE5k5>hZ(n3Z&`aSuAp(hXicywUrmS)jO7n&HHq=F15lq?On
z*5bQd=c!ik&r@M+dcL`@;(U*MI~~XOc{-~>n(FMEr&Gi>&pOZZ?8c=n|IcRk8P9pH
z^W5O}m9q!-8W$v=PSyRr^K>oSm-(CbT+%)K<pNXVr-kvi*3P`}to7ETuE@<EnOiSC
z?#`RLdYSp#1a|i$jqD#y9A#$sre9O_uih1q^XJ=g`;AV@&1Vf`{~U7LvnI{G^3Q9p
zvM<|hXQ}DGI4qK>5N+d_$7j*H<w7ri(}7>2AC4~$x}V|Mc*BbCNm%_#R_!eUth0Wl
z?J$~{>#?Nfj<fRCJ12Zg)TZ>r+*JC0cjkqb;;jbni!R?j^SbZI-mi1tS5Mw~|25wa
zufM+U8*l%9z$VVT|6^}{qsX88JkQGy{qMcg#Jy*ysJq3{$FX@&tM=?X<@>!j(<|=T
zq?$b+{)?8M(Rp+xV_oyCYdw|q`z0Ub)de5mXDeOdxwg2eLwLRIbPkJ~AN8uX9ukw4
zJi6Dcao>YCFKVRws+cN2`&DwKaPcp0a@e(?wC2(J;D#eso2{SvpPsUJ{{JWUqpuiW
zI8wEQXNe+Xs;)Dei1**VMVs7pUllzunJ#&@-rRU5`}!DF>l%e7bDM|PbtgPAt5fLv
zzJp=Yg*!SfPCKU^|MS#%V&%E;y3eibJ6~w_|2&`l@5`M0Jx?{~vtHdT_j&Sno4uO$
zzZ)*cecQ3$_O0T*JJsu@&*q2i45%sIz2j=9XvW<n@!O|-{j#GJ<3&95*^K_~=!jBU
zlGb7W%TXq*z=dI(gZKY?&lcV4xbr@Lli&UX#%Evt@BH{~;v?nugyRC~#ytu%Jw5)t
zUl|#Y`7XHpVI<RIL-T{2r`;@aK9p>kBL3Sco>|dfPRP2ZNxMdnHEKbAWI?`UN8|Uj
z#$V4G@02wv{%BNs(a4n0q@lqu{d%sBg@CSxfUZQdx<#{5M6*dnv*8R@O9@u1iYBWc
z%{CG(YuQ_zHCmiFSY0w&98TnVb2RyBG_7^7U}}ukE3y8r>ihLT{7n|W|E39XKN6T5
z6Qmy|+))us>IhGq(Z-(Emb0VHE}Z|Ii9mUANwpCd=MQP|gKS$T3S4EYue;%vyMa^j
zn1GU-QgL`alS0zJEgc!*WgP#v7&19Ftl4D9w$X5*u(VDP*CY)AzKKaG4T&;lf*Qst
z6^D&JT*}q55LmFiYh^{{x)n_`#W`(%bZK*RuiH^IQGj*RjPAe%-Fr1y_nc_jYteHg
zqUYF+?t>LQ`!aeCRy3Wc=s7r}Tm44QDURO5C%UIy@7`h261*cdIZ>w2I5T5wSlpGG
zbI)tUg-Un|Yb_4;nI#r`yQx&O)}=0yTePTd$`!W_+mx5FO4KF0GpHzhpOkF=t?qrg
zl=LQnUDw=dS^}6DC69=C%-qU4>vR3|gPg5Xa_WC5lpoap|EOV^aIQt5qK~;iRkQY*
z4XmC9%?2-;*c-c;1Y0MY3mEMbFyfqS@&Dx{rU@)7I9TsaWA*qj*|D<8S(4SRa*F-S
z$u5~w0tKeny`18Ea!P>Z)S$?z{+Uw)E2ny_oXV1z7WpA>cWUeVOSZp7i~k*MjoDGl
zP*!X|UEt=EX%h?E?5?+&D^CBu(f8|ulCLX__zvk`*)0D2u>Q{q74DrA`<=O-8h1=l
zEMvM*8uMUA-^m#he;OoRXZ^HAdU9i_;NgxQi3n~{$<uAgOo3cW7`tRV%jdVs^P37z
zGtsL0?yz<Nr=UX5UA5kuCnlZaXx{sxNASSxeK))Ja?aT=$-1|)Nw9%sbL5;OoXw{q
z=P(t_Ic6y!$iT{EIQOdO+<=6+=TA0W<75?RU}1VN_rZV7c@KZieb~w1!NB?`bBch&
zTm#PLHRUljPXohF@SO<f&I;|DsTN+awYF$$ZL~yqnQ9haNG;RETF%4;c8i2w?GWJ2
z3TJuPH@8flYgPiISUYoL*7CqQiGw*mW+WD>O!5<7)Se>TI#a04P=cd1poTG0@Mn_P
z;(B4hM&X%}|F%e)J+frx$ki;As`;oTDwL?>zDW3^75nFgoD2H5%-FOw1-q_SnME|a
z?wsVvIWJU;Av|ko#IB`Lx0c4-n!@~WDGTE~CdXybQOjbgmL+NldUUkNWdsXO5ef5{
z_BCKytVcq{t5%k6ZDo>zWjB{+t*B*USkbUcKxmafOV<kbMZQP>8_J}L)crFM_hv7x
z`O*I7b9*(TeRD=B^TpEMn=2QrDxGIFle2MVrv}5En;jX`1&*W%2pn8ow9!N5aB@*N
zTZ%xG^iIXK6U;a4<lMV5#b}4XIZuIeH)kLHHRnXs+)I(m1ShZ@=bU>&a_xn#rS~i6
z+^<|4*0uIV*Sz~D*Lply`{L!CV;<FsW^C$T^(LofKB?#{Ug|r0nLuiVz^fk%IN7qk
zyz0|*T;DcR=&p;*+6NAqjplPqv=nY8w<lT^FA;3qwBg4KF;S-Mi)Ir#Ce;bdUissZ
zh1oat-pv{{8acXNmT$gjPBAo;F)&>4NkQgP`>Tze=8l^C(uCJ?imug^R{W_@{J(IL
zch<5UKPPi-oXw-i%5-swwWWZQwB%teLHBBb+*Qp=68by7>Lq8o{<z6)T)v!f*|e&z
zVxD6Ozg7wSkXj+w)h1ln*1Bu`<4wNR8i{W=^oun*v=}ea&)&v#nXAW9Tj85k*Uy!l
zie;-Gtz<bk<44M>`qN3PUL`H_6vz{pC0`{mp-B9<xM0_8Mb_>aRmaPZ6jnUiwdVW7
zH5s>dUd>!~Pm@7FVb>yVRt|?<Oc&Qad^y+7bIu8mb$)5<o^{onSY9*Z*}DId;l->A
z&W6`s7hiwi_f}CU!5>;#sju7GPc6{CxV2#CR;`a)%^epAut~`Va#u1-Xh|EfFp4v4
zPEt5Lq2>R>tcgYK44Q%~iWYHSjF_>hWSXkr^o0Vy#rJ=?tU2{!z08HA@2@o{ewL7u
zG-#jgAyVXiNJyJkLvT*i=FS_+?1|hhi#MCi6nMXLnPaxFr?<e3$|l8(#^X=76iIDO
zO<Z0nbtqEWuQ_UZ*R4GzJKM^?E>~S6bo}-q1DokJQd#OcC50ak2|w)P*mmfv1NVc=
zQk9?eI!;Q98!ozdY+;eK!AvRpa&5syua3-LRT^O~Sto6|yYT3WRjXFMnpssSFrh<W
z(adBi!{kXBtN9)Z6g`&;TjY>pu=%*kfhpD-ei(7>jht+|dh(8yTbLit6L_<0S4QRc
zwDd18wZBD*#U62;xhym5KWF$ej_}210v~^!VCPz&*STPF&!Kr`;cRRRxy7<(zL|DU
zg<)R+|GotNeH(;WW*vE@xX>(|KV_o!ingd%8#ipcB(ZUp(2|A*XP<~Qg&N9T?jonP
zz9$`->?AScQ-s26yD!;Ee<$x(I&HcB*P;p5dK(h;{%J^PU7VE{vvS@`g(uDuYYe97
z-kj7WKCfD4vW-WY&WwYKD_BFW9lT(opBlL})AEpV&fz1kx2DZrVd*1~%DAo3YuhRr
zgPtgJS;0C!CdFFe&5FM)S(a>iv3tGTwC(&#{teoK;<+7h<^Izru9_QZ$g*?->%mQ|
zr5@`XE-?kH*8Drux=m4BQ8D4@tle72xBgEsdSBVSDLp;tTklap*{!p~UQ31VeeJvJ
zc9_fF`FyJq4$MA(j(e|I(1Hu)idC1F&-*5Lt=qiBdd2ddgyn0letW?GZ^KoSJ6Bf7
zY~vJ@;4|CvTH-3F)r!9Y?F_5MR!$R;k&>x<q_w<xuTbKVAA;K%&$uuB;66v|?00j4
z{_GoVvWtH&w&%}S{5yElw|4uR-NL;h5{BR0&K&7q=FIhI)7i%%g4-r}rZ{LyCC<P5
zV``{aP|`}7pxyCb3zmP=IizyuyhhKpUop4MV-D-<T*$kj^vUBe$1{icvl2pz$C-K)
zcFP`C<yqlpyRGEpopaR-IF^MUX3XBVVH=Mnm#pW$m(~}U{x{E%FI!b@EfAr8w2FHb
z)5jw(dsn^jtzUZUh8%AI2eaUEMnU6ahHPJsE)+f{U@b6rhQa2n8LOC8c28QkWC3eR
z=X|cXzWl2v{@(OWzqz|Cb-~5$CofJHn60zkd|}_o>T9_-?r<D)O}cUChfdbt1nw!`
zS8qR8SlVO0?tsL@!;k8=ZZp}p`kzDKpHo*PFWph$Te$ezdZ#Y^N9xBd3hmRkDD!nc
zG3k!ts}@)#DqX+TooC1CiHjsw3*MMKSMi+MCM6+5k<3lMev7sA%K!J4G?}EgLFvG9
zmID{fZtY^K+aNZ3Yr)p=*;}=Bwpl-Kx7~NG&T8ScDXm)zT-PM{ewZZwr{}rR|2Nkr
z)I6^K$e*k$P<5T}{{-`;0bENGxKlS8W$#`cKi52@@9@3cLpD7N<rb<xc9FQ%d!Sro
z2IHqq>+V@Rece8*>u5~PfiK1Pq<bUTc<#5{p4sgiu=3+ki){u{iCPEWlxCc{q-3lY
zDJQdilKEE)nKuq%|7Kp99;5$2{ZwW0sU(h9H@^2>v=z+SE0ASd`|ki>Ra^Jo6|7Mr
ztO5pHqPq?+ymMQ5pP*7+_*MhHWC#A{R|>nWPNmg`XWxFg?SRCx82-t8g8Kxn%{?c*
zCV}6^@1%e4`DEFTaku$J*aFKI3qCYgaQ)}jc(rc(!;kw5H^j<VFz(!O{jyeF<C#fI
zFHSprvGc#JQh)c%Zd*x}?)}1(lR7@%Hxpd4spr5>oiF)1f<EPTO%WDvE^sY!&^M@g
zaigz<uj$Sy-3#UOUbOSw4S4^shVR0JJzS2GtOZ)p4>EimEET^nv^VChe}CvblOz9P
z0oKJ2z6o6GR(ki+MelC46W8_x{>k@VOz3+QJpXOA_=Zf6ujv)fb@u$YxcB+JeL^j#
z1aE9pke+y9=Nc>JS7JOXo=ul}HBENn^v-*&wMi2fm&#4L_;#(ptd&PImY1=8S+zD%
zAiz#QC;APG>Ki7-@=P0n%o_2D)rzTcIyno(4o_To`5fPE@psaJwF|ZxRVH(nJ>a^S
z&;HohZduN$E8jD@UR|mF|8BvAJ3k)Cx82>s_i88CubU_KF5u)?z`4Q4diM-gy<p!r
z2XYrbShq!hlY@cv+5Xza4J-@`m(JT?_wUujtmo(FGqmy6e*LVvV5)8Wx4NyfH4fZA
zYV<ZIF(-WWJpEQ))-T30gPwX#(~aHNvQtRad5W>L*c$V+DJP@k-fr!BIjN^%p)2bK
z&QlXaotN<3Hp$Tvd=)a;qcLpitqGwaOEsf)W208_u3VDZTol<k?ewj-=vzfI4<&L%
zPoBeaBRArhj?dm|)-Sr&U2iV6EtzI^_hpfA*yNNG+p;;fgsu%-lXfd*=Y~Yx=(RaJ
zHz^<JTYYYu1$W1Zfa1MpculjvNZ(d*yt?kc@1@mS`@;4{c`VA1=_p=!FwkSm2J1-e
z;3*o5-PT@Oa@4}Qe({X!mv(qHyw|z$JF{!iO8M;SZyr0F<`ylmjb4(Ineu*K&F>(G
z>;IE;qrcVt`zy^S(zD@gU1sWvuoFTJqAZe81*!*?nm1^31`A!V=MK+eUC-_7)Xl|s
zA&J9f^PO2v4Lki>y1R8lQdaN?e68|Yx_n)hh8tr}$jT}EPboCFOsU8z)X<Z;U7)&2
zNH{@VCDds`kMD^O9GlMGl3aY~j9bzPImR5xv%HM&3Xb*~-6?p=8-76Mt!?z1N?+SI
zSqt06$|4feMXNHpc#UUDylP!ol@ZzQ7nLXNENtj<QR)9A4Worm{Z!mmrtkS!^>X>8
zIJZUxq0{?LPnlMy@wR=gPS0aqVGXyHOXr6!xWHr+ShdofTVsp$%0*!{QE%<*>%30L
z$KTRCD%<w&;-$UST1gs<%%TtJ?4O($rD<=w*!R@3%*aD$Znf4V<_d=`$zj#_*6}D`
zqb(!l#DS?d6jE2Lm?y<`Pug?lMz^(6SG3Gzjw+pAJIy7hL%6{?#xr3mi=2h3nB=E$
zhV9!Xbg!FYqO(a+KUw+GQq_;XFFiJkDQ9mCG)h}x+@`U)Xxo>7$8WT(_mnuF5!z*t
z>|^e>`LOZ%ORIw?8ya0y>RPb5Ly22s(+Q^o55wZBZl*e?F5PtM#>SQZ#az1Wc3D1Q
zS)t0M(c^94SoQ0p)ud1tNmt>(4ATzL`)(iEEmc?~R^-)(btb*5im<$8^dT%zkNW_t
z$1X)P&xIR(KmBp6+4uQ@^0ui#_msaqS({zG_T|(#=ggOTbBd~-&RP{4nR<0aiN9B8
z&dW7MK@*g^@3O_mPndG!upYO2wXf6mM@w!UI33d_o44XHe|P>07ry@Q7b*m$(^j}V
z4WDG^D&ns*uSc={Ot$Km3&Ea7E7SJuST7`HwcquA^rIaMRimXOjqdqm9pCx?#XI9g
zU00q6{pmiN&@tnb!2zQw2|kHguF_E-CrrMeBpk_bi+SRNl`3nM0?Kz?^LfxB8`YEa
zpC>z+y-dQ1+t(q0o4=<;V!~(j;M641VyV_On>O@)^ibz|ydq>NkCH5hfx3{umNTA@
zrfm=X(wx6sDt2j>ASXvhl1-9B6W3+|PXC9#+>0Va%H*1Kz6TuXKJn>M_`S`mnfESN
zV#{Q;&YS9H)fvFBfy-;#2bQ%{zGU_p?iAbR6J(ycFil}aV8{9`PxzyIda`*KPr6^(
zK7ZkhqZ8{jk1DHjp7fliIB7~xikdFxV(&gyZ6>P;o@pr(Y<pkp|BbruE#zi1dCiKc
zN3;dMaFvNqXE^4zJ*V?S(xTH-8&s!jc5+EixiaDC0hX!C)jTa~&ATtnY$<hM^qp}g
zz>aasBeBH)zqrq4n5hNrxTvr-!QJ-Qw!XJFPAaeoI?TDywk^=)l)vhk7Ku$0G#0n2
zhkngiIOo+tzNj@yL3=Y6dYn7S_cG(5%B%(_5vA61jS3}U4`pu4y?$u=vEx;snkI*>
zn<-n>5#KQRw;Hw!cFa}fWl>?e5af00bAQ3cV@u*!9u2zoxPL;$lm3+<o^pws)}$|0
zDPC7NtuA2K(vu<?+$O7718j;;F;8t2{+M-LQ~KwX{S&<7{4WZI<%gVId#GczU#W`v
zp{SFqUaj-^dXT$AO3l0Jt*cAEZ(Pulbfp~^LMs(+#VH3Tg{^<1=OnJ&9Czuv(7v#q
z>H3iYf9_2d(%O68^o^$Qf3yGU48}7U8rEer>wasWSXRgrX367seZhxSGF%O^Nk0-<
zW*T>GJet)sGr@Jv1mRY-WyRJ>3)q)(T(W%P@K9yasw-A|*j7GrdFZ$9<#CA`=dDdo
z3KTbJWh^miWYGE+63Qvdx?)W;i$;di_BSVPubjs!_-(@DXj{+6QWGEZ7X*22=Pk^4
zthpK!z^b~UZs#@MnQc<N96^erikuDmPslOVKQqqDU~RgsFwKK)o5(HY!(wI|mKN_k
zq&y>*Gr{D}Ue`(9>t881M@T9qZ4hcblGLlxb?}zn#z}`JPT#U*=OU>o%XCy@R>Z`0
zSTyX}w#mgs#9HpqyG>qOH%+P#=w2Gr_rKggfbphNu0@XS@#B-&4Oi+MtvRWd6E4A+
zFmuKGgMOzjUG8E$=HnB}s_?!pkw0neP7~|mPJ5FsmOvSE$4i2(sV@%B3cUK$I-r3`
zyGi-^BZ)2Y5h+vEbTkg_<y>GXF*m;accQEYPnOwjQRBOR8ZT&cw1}NaymI5+ldV2?
zPAdG+Nx#cy>+1bGGu$(D$?DiHzsUKQ{Yzf+`_G;kJpbMG;tfAbH#r$ehSfZEV3*>S
zoH92&ZTE?rF$IF+OBPH|R`%0UEL-*M09WqDqT4s`-YeU2(Dq&a^xL<ei*4UIca`#i
zFY3;_R3>&lva)G1^}lClR}}qa@ATJ^`H2GORx=!UZu`G!HjC|E;R6YdF`<EvLXth3
zI3?d4zOiEGM*qhvJG5iwoMfH9lUq%oL3-KF8^&5H?U#$f?58w0_!vlaMZL&a`cI`n
zjw@<s<c^r7VT_Fqt7~MFBKr8ZZs04Jc6s@i6SpP4O$#VmHYZ=eA!tdS(aWboEus$u
zqe^Bh=5)M%q+`*871s}HzG$uZJLj{i$1VHNl^^#Rwu{#%`@ghspS$YK-aD7#>zUtZ
z@8hoUt=He69W(I(OV*}?dRmO)RtH%53KTzf*YA0(T5osCc#cMPW?_fv`$zJjQ`h+I
zJst5@>Duy1a<`YTYrhJOZ}V%|`gMK!Up38_l^%LOG|iHjwM*8EuAcCpwQxnvjKjq>
zdqVlnPu);5V^XrEVN}qX4ZpOw4;bA&-7bGr|IMpSJIfYXolIa4+B&n>`U3Nku$W+m
zTXUYt+8mx4`TBd*;lj6te^@en-%J)=<dD01%VYjPhFw?L7PfA;T<$Sb*d<YP9p?($
zOMTBeq!-WT-B~$*CWF8t#oZo}t$`ituAi6B4lvDR<hymDW7lERw40Ov{?PnzTVEt(
za$@(&Eaz=yiQ70XZ+lz0jla^&y~&KPm@{MX!R3z*UI{iY`h8GwkE4>wAtjeX%3sW`
zL?06B?qwF7^mMVga<ca3hSi!ToiA4&+IdLs4X2&l7RS?z1*Ih|q#M;v|DIU(dIRsn
zuKzP6n3J=3P6WuhtlE*v$(Og0^_~Q0^^Vq{7Y%I{&EW=<F9s+~+OT-~29cIk>*^e=
zR8BBeTxvQcF<T~TszyZnnZhY^FDO1=#2$H2cMoU(jaRHy8~7M6w1h-1x;4w{X$MoF
zhv*cGBTr@sUyHDA`mtuRwEpv|#U7EnGZU?MUDmX{xOIMzSo}rnE`cQsn>(12`LAtc
zWeGN5yS*gH!$!Q(=4kOAy&Y~RUNfD#?AG?<*i6Ya2Y(&rkF*iLxiRB6_bR5nV$p^<
z(i`_#7*>|7(tEH?Pr<gPL`(0)@wzqc8QHcQxje32KJJmVSLF7R<{sOY7Q?>Ff?S53
zOC9F4zcC7X#PEN6!k#Wcqb9={wX-)&nXn<^7Q;lHiSo(ZyB)aIgE{vu@YvVE%Q$nN
zN~C~cv*)xAMoFjkwHZwJT`=+JBdM?l8d)<}&fL}7{d0$&z+`idoq4ww&t26X`hj=m
zs&<_Q1}o+{eVb;#`7mqggae(EOsW!EOA9nNl$d<HF?)ga!hRvvs|v0M6Q%A6oStT(
zen!F}`Zj;|OzX=Nd1JlB7J02&vTEDL2ZG{_8+2zJl=N{rxXAWsN=Ko~{3SOwc>QuL
zn0-dU#p#rfkE)N)85eU!pY7*LoRXUky_mRtr;_jGGuyR}Y?tfi)%#+ur?QX#;=B(x
zS5Ms`u~NdKEZV5;gP^=;r^f0T|Gyb>cP^HGv!knQg~g8|%?F>X@^-d{X}IeP95;&C
z5WQ2rCy4W{Q-gzb=W-DSjRgyzX`Ooz!S!)h&kyej>|G0V6l54bO!;`C*U55<_@XKI
zFF8+Un0o2glw~?=tzWJDy~}i`^ue@=ow=GE_P^FYWZ00B;&#B=fF){!uhfDAO3g<$
zUpRT%t?<n;UfcbAZ<+Xgxwdxr3aSPQCG3$%>CQ1WJgaT15VKdj^kP{_V6BLR-vRf!
zHSYB-fj63Mo97&pH{By|yEOC4TnR@Rqv)Oo#&*iLCp0q7c&<6G{U+0-IhQ7;T#^Ys
zIjzEOIfudt7v~@*u0sWD)=lu}Na;!p>gf|~_%Hd`^WMac8LN1vZ<zK~!lKQ9QE$b}
zIUi3hxiNdYB){eB{mv7GPYJO4N{I9etzX&Q7npElMI+zy1;U&@yFE^N%~~;=vD9>7
zlZnU?zqrR&rk|KI^MUu_kKRefX15A?Qkg9dJ#jo@<8=P@v7D@q2;2RqiqD*yY&7}D
zo}*8$p1*u1{_L3q)@zBbhx7`}l0{FP=P_TG)R`R3eWgZK=ShEP(Ant)XJ5^nCouD@
z_`wO&8F*TvIUl|hOgN-@e{yfx#(rDZb7fo?EMB)iH)-xSZm!go?E2vU$A>fGjjqSd
zwbq=wG^0eWdGQr3niV?7<-w`*zjp2Lf9Ps)%hirCz*l%xV$bgX!Y#acKlS$p2c*OV
zFr1#W|MsMG6`R}LDom@6=^0!+F<Ub4i5u_SnNfc)@SAeZ4BgQdv4=1AaFGbZ+^aXY
z+}OzfQLto>%@z*BMW!<r#a=AUG0IIgROmT=k=ZtHvt!Fm_q^zC>8-ccoV|D>*ibb1
z#Cl&3h1e4tLk)eRHkkGX@O~6m6>jKG;n=QfB=>vz-6Pste?&5FxHRR*i3uhXC$70P
ziD$-So0CdA&n^f!yx@UEP*CvX9iCd&Ys_?Ko@F@Y^IGPH1pkz%HF_4z#sOZ|3C6cM
zw$*$Ue%j#@yRv<Gk5m}LoIeH~r|&99XUFdDjM!Nm+x2(r1=*`NJb1aPH;d_*{jW0Q
zkzm>JF!Yc?jmq(yGsk`IJuTgF_^iRL;Q8)d_u^75lD5Vr9=)gh<jh$e=9IT_siAw<
zBuy}Fb;{_~zR|NiW9cDnUAvn*uI>7D=*Aa;#@DOU&UUq3m>8sSr|pHW$&;{$-kq~W
z&K*<Wt}a+?+I4x;>3-)uJP|)x%U|4MFk)S~p?%kB<sV0Q?JajszP<MTtMfL37X;V%
zZ+|Sj>&g1uixH_I4;hOuq}do8DsEOVjpW%IF#Ya@>AjHxtP^L2cFykQl=tJDHT5PR
z>x}8F?rkp`X6U*ff4y<*4LKp%TT)MtFPfTk**i+ycCSR}-V3kY)!iiOp19X+Ili{_
zqQcXQH<)k#R|rksnCs!aYMD{O#w$Mi9<09hw;@UJYIOU}ruLak``<pcpU0r-BB!2q
zmYeb1mJLfSdS`JSv`Z6OHpOw0VQ8o7#eLfx#HAA3pZ^HSk`!-S5dL#Q@bnX2dwT9J
zpLy5fmW*#`>eLN4_L=ma`*HeywPRRpsMMdUFGCN1kxYv6-O-YqG}+-^T;7?MhYv$;
zZFqTgb;`5Qy|+T8_r$%HeVl&oUJ|Qza;w?<t!byfY^$6&$H?P;HS_bWx6CK_aPNG|
z{~~anW^4SDgC50kC#P#1mj5_$7H`x;$p^e2ZDXB+kDQVy&26>lRqj~Pp64boQQ%;8
zX{Ym_HA^biiR_8kdwF-`hxPwcQZAV8Y-LmBWpUadzw`ok>dl;#NWQ&^ivPHk{T@A)
zeI&mvQs85vg4o0<Yzgw-$9t~GEe`Eod^B@*Sd_4H_o}mIdDeSZ#&Xa9m(+4F%HUd*
zG^@vT>1bcC#~W+2T1#GS_<GUIZ)0`x?XRgek*7RbJ304!n!ZbDV#zh9wwX;1awi@B
z-R@k--PxJxqISJwjotJBfe!Dk3Q4sSa}Tp9vU*<go^oqyw$lciGqFM<ZWmKi16a3(
zxE)OG<X~<~4W63%U}>tz(R*=WKF36!oz;7mRQG!7hn)Qdaj{$XcqXO2y4ro&?%Dnc
zY04#6Gu^gl^O+~yjobJs?!u9`IkykZ5P0%G{8U(ho{*iMZ^1Tm!MXA4lRB*S-q-WZ
zd-kuJi#50K>811ExQl&P7C)R77|}fCw0F-b#k`Z}{jM%vwnsVe?_#@~(<Uy+U`)=}
zFEZ|_dzkU(BIB-SCw9AWzdgnynmOa@g~+AvrWiar7x(f8%ZG=%53ScrPu%C^lvdEp
z_*l#B6}x*B=lM*vGiH~#H*p-dQGfPfkzAIv?=@qy#~W;u1^-`M9{1R+%y3VDkw)kx
zCB?vP64}#dTyog<ar3c{TOJhLFKl)`XHeY9C4E@-s43_ExOb+q=ak+XnJp0f*eRvl
zKh6H?`fDGq#Cg9t7WT9w;@~BXZPL|_p<8njbaQ2o+*EM8_dkL+x7_a042ie@&7Q4#
z_42Xj+f_l&EVmaI-3-0qcPK+HORppC&DpdB-jb|sZxdv{JepfLRp9x842i6}{aa@>
zY0kY=xj35XcwUyaPPA7in{nqq-vS0dojEO^KIYD`wN+Wlo#>Ji-tPAB|Dzk>@efUJ
z&Ogxo{!sS?#zk%!%FV}CC+4!ejE}!L<LcAtx(l9aepAr<EMxt0ir(@3H;L00+?<(u
zFJ*q1>Ga-jQ(y8=EM$NFZT7cs{3kQ#iIp$8+RcC6Z1J}byy4|bj@>Fq_E>RGC1Z6~
z`fHZjHy4*J%QB1eU<^2MMtV{&%M7>w)92^Ls~=sx+qARar<1clLS}#A7QWa2A9;Fi
z|HiMO{FZm6=d_H&yOtH!8mq`$G;+Chd1--ChNg8}T8?%($Gxu{N!bVQdz*=<-b+||
zFJW8d)6&oXBHqqP?=*YA;e?Hx;>QgO&AY767r)>Yw0(ZiuK$_+^)pWQ?_FE>?8fQb
zGcoR^*;ifE)4!_vwhKSkozSHDBl(dS1OLqUXBUHfuXi^xhkvd-BP0<%J>&59d+F0Q
zn9XPDic(-*+RbrSs?$E}y=dNhF|!R5x0d>H>B#xq6yE#(&$`4dClV8<|2fw3vTJ3{
zG><(C1PZ1aB%KNW@Mz`Dq<1&39=R%e@68im`DtvqHrKw%_7||<FU!AQQ1JcHt@*V}
z?zvyuT>GEz`QLlt<x;7y1l{)vxj%k1zkKDja^FKLwZ{u`@`Ks}xjQ;L+sl8*EH{_T
zf6aR{$Y{CU&UpgIJSH4rSh}Dig*kG9dSUR=9lPuf6-~bIDd_9n13BWU#%B~){*UtW
zxi(MbeEPlF>>P|t>|8pBT@yH)xWt8Y4qs3_+##eMv?k-D(y<;1Su2|psiz!`qMiCg
z9NZ2Z>|>Idm$LJds{0K6K&dq+kE%_d9kc7jnam66esgVdh4$WD>a%>2SE-(A)s+lM
zztnktYdycNxMCEzH7|9Q@KeVrOxt>7J<1-6Y-Ya9m9jeUp_pdoW51c3Tu+K<rr&K7
z)!wkaA&lu3N5Rvyg{%KJcCv0$@~KcbBBahf>BsH~XK(i`VVQJOrJ(0|>tcyKwX3V<
z=^kUso~U&B+%(%8+hWd^T|QMdJ80+W2@hi*sY&w1)N!lZ&GBx3yx+UzrGi4D6JzWm
z?Vi4x2k&PxGb*RNh<Q-XD7!0$=Y6>1&*v<cy;J7TvvT}#hjo|y3A;ys-_F!NSD!tB
zIYRDfi;%jUsK<gvPNj|nKlY-+gb+>9ipwqhY@vy*j*&MO_zO=9JQb$UwIfvBSyeSO
z$b4=`h_8a$(?{((eHO_*3hP{y`=Se--0FE4H3ZmG3I$c1^ZtHaTqiw6^+F_*kHH0&
zU>D1!Q-bE1K4__zJ#w;6?4Y|~+JA|uY6sFpHU=$YSh)1$g-G?ShPN`*IIr|Mt=*!;
zD!F8fBUe7##~<qA4NSX29JTMN1p8dk?24T(S^HAnx!>&6izQQ?zFk-%pxg3fX=_zW
zkmvk)Uot(C=3Wt6wDMZZitxH^Qdj(~5B|~$3<|s<vUX>l>(#aHs!y{+_rJ4Rvo=<8
zwQzru=}8q=x40{dorF)Qwr;8PZ3vG#7i4&08?#{9jI5?it$?H!&M>YWb9_T)#`j15
z(sM4Hq&Y+P?G>W{-8DHISo@DEe$OvDZCfs1a{1zS)#C1X*TuI?GwqRcV)Nd9p+e#*
z`+L_(Su&Yv(yy1_JO1gY`+Qd(Zt;)?&Bc2@HT)N2JMr+e-szYHArf&Fo7-fb-C;dz
zbxk7qtje#RBwuZ{(k(sq&v$IO<lBGZd0)&scd^Lg!U?()<fFf+OpWzlxP;-ZkjaBe
zmYv;!yCfKoopQ=Jc9TmpXa3Ha2|1H9ziU)1-ucWaYu1$SY4duW+U~O4Oj7q)mBlW*
zxTpO2j;hCABEil^qEQz^dcE$+9-A3^$MVHOYnM3j&iW^To=OY-g1nMee%bZ9sBfLD
zuPtY4^_rdMyR_f$dgT==nYJgxEi5`yRWvO0Zi!9!QE~s(sTpfSQx>1&GI+c7Qk-?G
zkU!UhX*ZJfr)+cTo?35~&ai3qI)|Xe^1B>T{mdrr$-Ug2pYVU}8|zPpZhM}ZaFXrQ
zV#PqUs;HfJyuB7o4a;~{|9T@ogXE{3|7zYfa<6>gGDFztPsEz$_p8pkaqKW+pBUh1
zY@Em`UgGrfhEKY5RU)^!$imqT9o|a?7^d$inIM$gsm}C3u{-3-q9umCZu&)1Ld7A*
z9~ocjgfm}?*|%L_LuRkS>7)}=RxEYZ(rw8Rti11f$H2a;qlG7V&WU^Xf~3rPj^#54
zUY5ABRh{ujVL{H~$qYf0_p2UUA3UR&dDp2A`;2xRG?@KZ{=~0}6Rt^FO6^A^lz-ni
zx_!d)66KvsRChL<3~2SzSz<KxjoPx6f&Z44vpwYWZ%FcTmTXa5nzHn8)^(Bpi5EAi
zolp+jkhAi1oaF0iM}LLLH{P7K+2di=)mL&x+m{O-tl2oR(XuBcLw&;6(7;Jto{JqG
zs5JcIJ(9b&!gnr<G3z~_V1_F?3!JL9*sXiP=5o;SxfhS}d8ciO9_EofVVj&+zIqef
zykzkWJK3WfTPx0ZOB5<puxSalsBKc}<L)lq#=Kx^<)&0iVF9<$YexRdKD0#ddco~D
z%XmYGo|;-MXJy)V=gCDYh3#cL1Sd-@mOg2+O!<=H>}BSy>bp{x3w@Mx+cd*AtCT<c
zZ}&XYr&|P$^R$Q-CM~|RL$kpkb@S|;=2Z+_<^n6-T4olUy6nVvy-PQsP$5}GRa5YU
zrIXbEWuj#UJtnM0?BQ2FJ8xd-srhDukW=65iJpJ86PU9#`x{bJTF)+dDdc_QXw$0F
zrAZN{UyT<8bDap#oT-z-xAfX}gEdnFey;gcGIP(xN0Ta-na&Fh`h9v+;v0qVp3`4h
z8pIc+>Cg0<@JNMK$garK#USBHy5k`w%l&CBtMWUO_ox_k9l!E8t|yaK{6ooNr=aQU
zm>4EFUn@V?8t}QHb;_KAGp80C?Rd0SB|%$3wD>?zgFw<w-AEQqNy8?sk|gni3MbVr
z#c}W2zMt!7r<>_XBjIgl9Hsw0Ig|AB5JPCkK@DZi7DaFChwlm$=6LY)e@#|q+>xQ+
z8F^B$-$F_7`j#XAeNHPn&bzT>{nk`PUnVWLZ8vN+Jsdbq`-Fm2*G#xy<y~C2gd==w
z=h@adnsqU%9&^s^@ZIoeqQcf2Op&)PbnU)(L3I0$g=Lp^#i_AP-&1~|$T_NzyGD*>
zUG4(jjtkx~2bBuWPDuHjrhNL=Nm~`2sWv+ESm&f4)+?L5{Ikjech%+peAEK!cg_(1
zC_V9?K~(mgo0Ed_G<7_~k5<M!6wX|FX?9X!)%2;4M5b^A`80}5UU6r2Zm#j6D^9A<
z44M*`C0*KmzEn%}oJ91y>^G*%54e=qRsRY|<P%ml_S?>*>M&hfOmyzif(I%EYk5l4
z9=G*bJlUPIHr;Q9^vvdK)9t_f{a^OnBqQLQsfcygJN50RMKV+7yjjF}ZNe#r-URp4
zw--sYntN<j@i{POOOc3Up?AENz;v!BZSLA4?m1bHS}$v4rS6uRG;fK@I?I)hBs{%W
zZgr2A7XRAmaks<u&Yi0{mwe+2FJ;Y)`e<f!v|k~o`St|$BB6-MoNIU*--&rloN<`5
z`jokjYcAWT)cqXMw)-v5$Q}q{Gn~zDv{b^_Wrss-k?LNx7h6rQJuH2<CVAEQj;qEu
zex6C6@?_?l)yAH$`l7OBH_JBF+j^*~YKvWRdOGd2^V+5Uv&*ks+pFPqPII$}QPw+2
zVf`JlrmJ#y@12)+q{c*^V~u&vIghznO2=BSKmE_ybVqq|8&A{9<5?2UZH0a}4ea^N
z=4U+a`nv1jyajHZOwVskH1e0;eSPXKhD+5NpP$(oDqS*jzU(RJYHqb(?50)0V(~xU
zKWzV*SFT@Uw>`e=Mam*)sgqyXd;a#Eo1K`LX}yxq=e^8v^L5-i3tC&P3tByRi&@+)
zI&+VF_<5Nli&-gq1t(L8gnr7ErvA4*cO`lY=P<o!4J>rrUJ<ygAl5wcp2iUwXCF4r
z35UXS-6r$4=uSA~F=wXkiTN`cJFe_J^vaQ~u=Rni!;yBLSVxX%j=Gy$SZ9~rIGDSI
z@uvpQ+=<f8qEq(YO*rv0Fly#~{;oSq_S~y0Veyc8_)=k^klKGm;jUP<pOML~3s=lJ
z{^!e4`<D`*3U40Hc=*F{f_`X}Ugpsyru^Y5tzk<f@4XGuzj?$Zz&CP<>(&5n$2($Q
zRqnB99N6l?AeY6GsM6+aGjG4Gn`l(a6G4|}jML06_GiBkbTpVCQs6lCYVhX3@PZJ@
z9X$#qIekp5w_l!R*(BM*G%;LJ@XDtXM`~VNu49pR{p0`9GGu#dfS6wwAKR3sr)R!p
zp3t3lch6Ia6yAt^ogrsB<m-E;v_72RvL{}B(|jkD$LASuFIAEG>#E$R!p+FZ@;}mL
z<<YSJS0j2g85p%1%^%GOWWA<%wtuz(OXx}8u%o`gsiLP7LvLrWNEQTV|M<_Voc3U!
zz_F<hr)qN?6U#UneQ<%pjz-Bn3T0E2Hod*7-N^kV$6focgv+T&9kyv-JSVuuO}G`a
zB;l^4u#)0@-G@Sbcjwp6y*c&Cd`_vKOP9FhC?*`dd*|%jtceTnolKE3I=}4Vl4Y{q
zE-BoyLi1woh&>8yk-as8$-{xE;&I)K{>P5<oagvP^jwrYJI5j5s$y8`(t>$TG4q~t
zGQ8kyC^l^qDL9aJAhcrUyaJuO4t?Fy0!yk{&Xg3mmT9zKG(BGU$HP_3rKE>jOYhca
z&lx*ziYKsLiS6>)ncA~6GeyiPWS8pvT330eik7deo_;aD0Xgy>De)qGt=~OGzD3TT
zR`Y+^p2lUj&Ps~^bDcKlS=E<TUPadS(oXMA4`+=zIbMs?BOkl|h+lbihJ!}~!-r-0
zO^?m1k}Wn#$WL6R5*s*O=8?-BPU+NRGkE$~+orPRHBPz7#AVg;=WSfdi<V;r&lk!}
zTGBB`Ky8v%(`}te(n_ITDl_BVY%aatslKFSNlnv}np4l+uchDQj9AckNxidQ)XqC?
zi+ubl6}O;UlADB^<X-s96Pusm+WfCaa+Bf|5n-c~(^~I|wcT3Q#>H~{Zlau6s*uKk
zK&d!ajipIEZiy^zYxXKJH=PSJyFTZJr?kgGwht%0Cmi_XBE6w^!T%Wdn;K{AKZaM=
z^0`YKDPMBLNAkbgj-$tq{!!Ikl-_NZDPNj?#$(z1*p+H4T_2P#G+6m$FYk;wzAd|L
zI}F9-UFO{5-8#Qv+p}!F<m^S6*<v!e=awCk<#0|(HcQelzZH>xX`w?!vN_Z8f~LhW
zG79l~jwVm&S2k=3+qwAk&c%1%_%1O^aN&`B{`72!;1RV0Qp($$G;<DGWQyr<9+}Cz
z#Q384Q<Z>EYa_P`1}Q9Du#NMfOVhdfq$jnu_rFNSyWV;FQI|#Yq56ij)9y9rH^?na
zJLl>ymwlsm$;%~kz1uYFH>OPYSp6Z=tx6@scUdCu#V5^nD?$#O*p`<aaUjvil2bPB
zw9AW!d5;tWy&O8ko(8^q=sM+pcb6NB%M$J<Q-aGj-n^fC)hxCpjH}JIB`#qLW4GtA
zUlGUqb8c8qz5HYY_oTS%PZVyKuM{l}%#obHFs+bfi^j7bfB89XDkv>nbX;%Blq{al
zx-XM%CGa_2S-53(O3^_!k!e$-gz{EOOpKYo=}peJt+8J;9_&64;P;`YWp0{wqe8pw
zrCEM0vkr;)oYlOdC794NE01qM4x^UEx0Os9M;4T=T_AS#dxP&)pDsr;E`@jOENNP-
zK`qh~PtM3`oZE7hSMuaynRe#@c9$<~w+>u=a_3-_(50xjE=h?=VHboG7$z-Ji?29z
zx-w1j#MVd`x6G$3*&DnBYo%U46<b}av_OyP|E=SW7sP_J8-z5b`}u8Kr1&aHTgpx2
z^{RDYe{L;oas8gNcJhUHvcJ^czFL^i<s$SVuS2aNuxrH!Erz0NC%rFduTjmPv(CjS
z#O;tC%SK<-YfE)qw;X$Jnq)2D);sBPgA<#AY_wwF@v^_I$8Nbf=0sYB&i={M?yPcb
zA%l?1jC)riPp$FFV=I(hxg^Lh;icN%h5vUcGKi~ZzPeE+V6a6m>fDwKZysK7{^qK^
z^*(Q|+bkoO-#e#x>#jA_b1}Y@c|}dIzu~Mo+o}St1J|Y*I9KR$g?B#XDwvX#Wwq(f
zVm+2SE7%<0@P_?+{a}H{>0B-6D@-Mxj?x^LHV25mwagPZ{Qs40W4H7k$9T>IvKg)%
z%X~#vw<H!EOwu_oy!!T%l*D!0+Sys3eYTvx?$N6Lz;!}Gt2Zn&lv;22EK6_GKEoRm
zWz#pfPJ1@(iNJ;zX-nL%ESupL(0@Ti>fKxgX5Q^}C-NFN=6`YX+rYZ(+p}x?R`1qg
z=<q7~FzLg4Ee8gUd^P!0h8X3!Kg0I68J%@ymkc?!Z(d+9UrW&{BPBCo$DF0Mce;yN
zkE!`E?7ARRHuY1S>YUs+8l{E-Hl2d(Pp-bQoULu>cXUUSq|fzNoZ-I@$?dXnpDL@r
zW5to;DQhMlT6EhoEpgVH=o!Zy8jPpRTE70-(-V)LYPGSP5xaIKE695C)tEE?-$b1F
zEM{}jaY^gBXT|5d0_~=^axflam@5A@^F+&r<NHb%UJ^_G&9TKPeZv9W_iW_`?Dwa<
zDO2W5*L^$bgTsY9p+A`&O^@BR9Xjg1Ofm3`4E6e;`242XDu)dvrz2fv-1!jR%akS}
zIr+$1CmB9ThK(GP90i2k9^T6qogevnVmP}{8xNOD(B-)Njs**syuVy}<WEqv<PL@C
zhgUU@-Q?f4d~NUPtx*@M)~VM-b~y*6JyS~T?p40FX6w^khSIwWYZtMIE;#r6*K_Yf
zvKRQKJr7%57<7OC!zHh^GI}`5UiEb-1e!$u$m5iM8W?Akd^s<fJ6|BdC^g-vbKa|u
zPWS#={eSP3w)AgQ9Pj=WlP_-44Hvhyyqy^0rtfderS{RgLr`)<S>l#SUagA~XE3>J
zX=3VqRPZIPzirXRzzHT--=~XjkH396<=y%Q$H+v-n-LAKA_P1c*bO%Od_Ons8AJM#
zeZ5!q$<Fu{%XdKX@r<66PSdv=&69U-h&-tJ*2Vivd(XYeGd6G<ySQK9&d{^P<GFcV
zyT;ZPA1tztRK%THt$E~hX4zt^sa8g8)>`>X;zHeWnr1KH<mYBl{`JAHc)`6>3J&)I
zM1FtC{o<y*W}ah@l4M81>$x*!O}@%*ak*({s(A5%lapa!%Cweu3X`k+B)%_o3-b~$
zHcT?wvi4W2;H-~3{Qmn#i~O9px=?!c#>Bm*^Zv5RM*H8G&~v`&<L%7vk1svBYkPjF
z<frC+C)RTc&0pX>>+h^Q57y{_Ps#a4YTOQL><zQ0f0!M<O|3g)>mKf3X}+mE6V|^p
z@L9Y#^6hcn?-Ay`2h)yh3U`vz^kL9E<K}xHbnD#{Ij6VHe0s0=jBx(coB0#M<Mi4$
z*8WPKwC2E>BXM=^cPfjW_qv~WbbYaebhlmJbAim&+pD%7dZW76c-`^M>&|~(w_9=Z
zm(3HNtl7X`y`fq&{cWkXlBi1x=d3OF(-I0U&nn+C-`vGIw>rN1a)DD#{uT$71GV+l
zLMjY%+N%Xr9CoNT?Cjp{vD$k6wbt(<|MM5OJeO!J@^1Hgcx1~oBbSUTU($Nm?##>8
zeq&PEyI}Ljw$9C=U6aEPZEuvlW8}Q<I%DloQ*S?~hDYVb%ck7Qu4Da?IBB^Kms-`m
zr(wY?$DO%S@2S^%`A0pz#<h9t>D^x=jTKis+5O?|qN#`dr#`QFS+~xarzqbh<CW02
z*VA@(JUOPw95Hu)<($Komv`@+z54eY_r2y`JKwh!CGl7!FE~F>{K5~feD3g;!mVtY
zSD3N{T&xfDvh*zd`Akl-Vg1X(8_WFnZ1g=|s&jmQ3j1CEp5&)L_-7@v6uKS}e`m9D
zd!hKhB`WTzdfU|;Z$63_Dls-+aj7^m!u+ZKzNhgMvp@bXb*s_5`DIajjeL2__AS<5
zoWehNM}F{L81d5j%WuaIvo~$8E;djvX!u`h@_t$M&gXSZQ>q!>?0f&(ZYO{G6Yomb
zjzHHL$@}A6q%yBEhlSb%=}%h9dAGpx!H$xIDUCu$w#o1HdRsE{NP8xqm*yHjx7D+^
zZTa~2g~0U*zrMbx`gp8+quYm9byu|9pWJ+E);)3ap6E3Z(b|>{1v8Zrx*VF=wo8aC
znDFpGYY2-=jK)N6*G_T6v@1I%CLiw=RSpxe6jV8GBy`e6XY-`fr;O&Dno=cM{LE{H
zk>}Bz$(yg7n~|3?jp<jgnz){qiVKf|LPOJ1Z&sCtSq=?M3_){~co&5JZ=BQ-ZLsT;
zMp%P1V`>nWf@0y@lUqejPWiOLP}ki|=)Kz+?S%I+`<EP>Ae68^Y<Krs56>G-)0n3V
z#VXn9J>22l-Z8=1ODEV#>G@_M!(^M4L61*P(mK7PXmM2O-7TwcPs!DFD7|n|!j^jj
z&#nCY`?Kn^d1?zQ-rW%|zb|)g<B~wV-nr^`jx2mIjhA_?_noG{yBJ>I*)R0v9AnN8
z-6wnV8LQ=Tj()o$z9?U=Ch6GEnfkZI51f4Q?}4i2M2!O-Y-`Tj|KT&)Al&<8*NJWH
znN0zQI5g)7s5efU<kg@ew4m$oT00+w4u)-9PFjMVJ`bEZm(4g}-QHCE*zA`|QtPpu
zIsbb!%=Wq|ri%2Qy3Jw8SaqkVH_T|&(y3-kUYy@}E{`Q|o#bDsv%S`K&2NpFPda#?
zzS^L5dX0wdqDjY|x+wdpo;h>p(%IV&ba>C2>06!L(!zBrc!4eVD~@A36J$IBKC+$t
ztl-FNbR}Rz0?(#B6BSfXIvD-6I;+HDygUDrx8bG`lNr0tWU2<9R=66n#mjZ-a<eL}
z=Bwe}noCcMt!(02ymnDn*6P(|wuP(XRVOH&=r~p~S%*=|<>ii!pjVsLb$H!ey>6ez
zR}X`OHLqXaih8wCRp~H`gU`k(9v4cKFB=@<@$u2E`er=++@wOK6K>jzmxdi;(Nw!7
z=B4YivnlHAy)4fEdN&?4i@!Rw^Tm!EE!`~&VdpP)EW6RS;GNd7bGv)b-mb9icl=jz
z!}sOqDJsu-TGlMlW8w+8-@~<M#er#mwF?e*<oOvKU@boP(~Vo3N9x5w!w@INRf}F3
za4`DBSZN8@+X}uCo}5s~)c)d5#&ILVF5Z)CtxpLmZ&dv?QCTHYQB{Al?JX7E9M*#9
z6Jaf-;iq~|ZAeztj23z@ZLg>G>2pR;bh`_T547-nyuHJ~+pYTgQVmlb_OP|86O*b~
zKV0yc$L-5#_h0wRfzPjVzx4R6|6;Uw)#J;h_NH7RN;fs<-7Rw|w7+8ABDJY@p`E-K
zr_u%9H!_L+O1}4$o@zge31WHuf9pxr8B5mNU7hFhm9u(o1yk2mql~j0FJpusZ2j~3
zthqgdM6&g_4c!OIj2CZzl{F!1Z_Vb}`n~VgPn#0O!f>E_$08A(EN5-E$5OvmE>WE(
z>@Z!%*y8E~?)aq-8UkV*+q%vyGfOCH-D_byi{sKU-agG%*_ekjpU-*HTr17B&nJ;>
z=Za@Brx*3}nlBcVt~zJFkn=8|_h<QP$Kx88IPa<Hv>o|6#od<g<+<`bpIzP>6f%2n
zNvhhQEU&w*J$a6iT;rF+dJ{e@SNfxPLBdgKI@8pR6;d;2Hay^Pvs62g&a>(9g_ddV
zHm8=j#;xqSJcDhj;8YFQW|zQDp#?3gW&KTRPyas=H{r;Mo(;K^TsA#nJ}f#>&emX(
zc%abTZ5Ozr8Q%E{aRy7iQxHEiw}pqvS(W$T6d|c~N5oQoN<WEl+^O!mOm)Q+@5dPu
z>!(e5B(_!1LHf;Bq2q<UCB-)rueDr0W4W=@w>&#oa@9_?ONqky`w~@;#4KOl^!H@>
zqm{7>OO~&2>1+>d*>%FhNL90<&SjF+O#aVLULLJ!o9Jzv$+;qH(yD-!T?@1)b~?ri
zY1{cJFmkQna=##cakY<#wpW87_uUqi?}<U}f9}X{{vdRzs;p(wwJg`|2A8_RVwUro
z^mMZ~e!9dtX#$U~Y4`qJEV?t_dF#Y3O3XPUBz{Zfv&sz3gI@n57E7L9lcwmN`pRpU
zf#f+E(GKOUuR{0k+|z3Mri0)6ypZF!sXe*rwxW|GPF$!8o48Va?JdQKCHFLi7SEWN
zs`0x2^dBMPo;Ny<=C2~Bd|Tmu(657A*g#4_|C7kF4v(yTUV*_&1Y}bFv9DIyDk$x>
zYoh$gT#u~D(UT6Tt_ix~q_;LF;7OI2H%rk$(Im;YUMB)hr*I`qS-H`5`l$qNmS<wq
zm@eFjymw&7&12dsr)*9?ZTo%hWXY+7`Nom2uC(lGI(wph&r9FpQw-;>ZPl|$QA&5)
zmwm28t!vK>#|}%$AUPwORlQS}>e#*a+*5H**ip@!^K05A)j#*1RNeag-gis+|GjVg
z7KZVacKy(p^+5Qemga}kDQ?qPmRiLw(GzWqp7QO|{%((KJ=LX8{<;+g<yQ)M25op5
zSydFJ;%YQgPj+p@kxK`HKY35fI}jcjaOO(K5sQ|klh34S1zlOmb-?(7=vlsrcch;-
zq?teGT2!)dj#|IY^^>O~`O3tP$-mriH8CMD>CUNUm-jhI%<)rC2p>Lk#6*9hwbW@P
zarRGhI%_AUe0v(=@vO^@De+iOZL3<-Jq!2Qo+{}zdr!m{rnzVEPP^E6w<+=X_K8kx
z544n5m2^!KSbX1Nl5$G*2SeLS`~Bina%Ci)Z|c3&lb_g_qkQfOn_Kr1spJhUW=D0o
zeqZg1{{NstRf8vub86=Dliw!o$bZ4aQChL{nb&6JE1{c~?}*%$vHbL|PqH(noHts&
zd*T_N#fN3YuSwe!l;xznD^<6;SesW?wKe!n>*}2+zr6e9wQASrl;jL8nSh3QSqm0>
zz3}>TdNQN=7EP~%9ZiCDb^+CAr!2?|$hoj-+E$&{x(dB1UMr?1R$T57S(|aZ<KWUu
zx*iAPja|YdO?GWxBDK!&Yt~Q4aHY`WXZ9EdAJyoSPo7kMSJ$NQ){p#kCqK_WzH$Xy
z5$CEC3mo-6wA=IDW|=Yn-<OV>o5`Z9ts>XjZTwPmbxls_>Z2}g*WTT_Y+lqlGv?yH
z4W4^@lsnY6NM|pbxc`&S|91_!U+!{dN3`F|(hQk+E7`Lp^Vf^>=8OA+)Q_rVF~(dw
zEG5aBrracPFku=`6JxjwLq_-8B`lu3hsAXiEtW0j)|*=9^zKY(hVzq6DTgE%76p5&
z$L|lhUh>Y=Xxq%gIX?r7XV-2PE-;yK_bXpL_v^2IQHRb=Ec&jtXNz$DLWd^5uCIBD
z{Y-IA2AR)q{NJNIt!v7uj_-dvkE~SE)co($clLg5G5>?rr{?|6{A93L(ngu{`PGXm
z0`|Q}+|<rQb+cJM@SWsmJn5+Zvex=dsmc%3Hf!)GpGXjPdGje_wd@H8t`iQ|R&Z@g
z@q3olcfyRz$U(rOM~h`K*NLv4TT|Yg{J-Jt`3j9wHGXxg-X_?w|Ite}>EpHNJNKod
z!)jeJ(-MUU6$j334KKeXzdW(v<)dYZ2hN<{v1ZMw9P4=j42uHJF6&z}%Oh!*9{1uE
zXU(3Sy~E{jg3sQB^^d~Qo@svPy99b}tWdhb5vj6>H?b|4$(P~Ix;~*^yA7=Gzg#fm
z^5&dyEKKRqB?bis9~I7J30zGE9JPxJYLb5jWqq{MRkB&DY@(-J=JeCRDeq02h}L_h
zn6qMBrzh&A8%<cd(!r^frTv7(Ba6Bd`bB!*GV;Hz$-cFyn>D$R<?>OsPW4q!QpD!E
z*Tp^i-ctBnsmaPBfoVlDlS%@Uhy#;IgXG8cno?O(|09pY$rb##v+T!~z&A!pKl_gV
zyre60o_F7lmfv1(^WWsoS7Tc5(JULBX1_sm{}V%XwpDV$M?-_os24p-6nSufA&`NA
zLE*G}gg`R)1SL+HQy-K%Lpzo_YOQ;uv?%7^c}BqvLW0ZmB|bzM9m;cC$ZPj5T4j-_
z)&!Ol4Jv_5lVh4Ie4LGDE$>~Sa-yO+F5?MnWWnvacg1!7YMzh3&&n)_ITUHa!SM8r
zRn-UfdI8NIHbu)6%}^J{X_qw3U6hP`Id?P&TR!CI-;g%N;MJqF<ay^_AJck$=u`5U
z$6Uv~k~Xa4-ne4zo{eu(+LSKktqy35eKE-`qb==)-o+Pr`Y$)U`+q`Z1A}pf#la0G
zhr+fgO>w!DF{wy!e(Q%vE2GY&37I}Rl=bApha?fJkIJ?aMPgEvr?skwevE0FD6u?8
zIY?b>?@JTS)^lqzpZmNw?ao%5m(0~u$TVf5WZ`6~(#w*n3XD~s%_lB9$fBUBvLH)o
zQI^qL?KmdwyU(;XU;MFc<DD(*y7dkTZau8n#wv5#(|T$DFNsBa4_uac>1qF=ljopu
z(4&<70j&EUoDaQa{J?7|?~lUw{|qye7KzjeC`zB=73YhR>6yx+z@WTI*+bFn3%`(N
z;F3R1heb9hNAN0JeR_P$v4N?BNq2&<p~*`v{g-?zoIM}vo_g@#;$x#*iL>XHPycPU
z#Dq+m8MeW6yC-Y$iYeRrU0+|CSoEs#4U5LXx0Y?2Tzf9D_aD>zn6Iha;L&NqrKH5f
zqQJB$fyboLbD;_6j1Z<R4%~|xw7P<|mKseo`=Yh>6UX#*9E;a+UUfdBv&?mqA6I+|
z*ZxCX>z&swd*C}egZsKr*BcQ#rYYTS$6`LL+2`<x=R8NH)gs}DW$PcjoBPNk{a4Y2
zcN_HCzw%0^@P7^Xcg)>+P8wHL5_eO8$7JCvKZNTpF*!JT)nDs=sI=b5`jG>FNVd}?
zBTud+6)p@0CCWLCB3F{tOT4!yDZk9o36DMa!Ru*b+|p$(nuiV@4R?R36j3O)I6KmL
zv!1W<tp7{0r$2jrF6Y8?horq(#*))NO3wZmUB6kfwm7D-IOdA-rU}K)T_w(o6u7o5
zaG&JBZPqC2mdHJ+K_tR)>4^ow{lVVL`$Q}*<{o%!Ip6rrlFQah61ZAem2X{I%TuPj
zBlzx-K>rUeDh=)Tk~VDeX*2!WX`_2c_WfD=8y?1AvTb7@g$SIw|6!t<hho^9ry73(
z6_|yXJQKNA9Wxiaq|qd#R<tPHojICGK;@jW!&6Qbp7(mcjw_k#ToGV<EE8bNd)Q!F
zi$ZRrf!#W}TM9?s92ARa<Wc!@Yr;{nR|^en`l5Fn6+7UXBksTS$>fxT4YOZ7KHtP=
zvh;~tA$Mrm6^#?k?Eh;`1Sf9NZVEB9+^VhDWYK=vqBGdboG~}DQ8p`4WJU>(+d)s&
z?XfSn7H(~j)tfH5ZGoJ?VgnxYLg^l7y)E7wC%x<u;5`0CTb9*#{V~qtmy*wX@;w~n
zw^l%x)5+Grt;?=$j#=5he^)$HidUxyNA1tBcqi0%@5I{=oAiHf3N$~&XRBne^1CX}
zrT2^2j`B<j4t3#L_sptkxmA<Uokvs5PEWU5V32%<?;cP8mAI+OH7V*wm#-T68yrZ=
zaX;u*vij4WrJ`;N#TX<et1af;Hcj*rTkhZMkM|sHU|k@0XsOt@v|DKxbf2G9dFL#V
zZy-8(wZ!>mGxY@KJyRtsH!nQ0RsMgq@c)`(3+1)uhDw*UOtp;9wpvdYdGxh@OM~dG
zg@y9p3t1avRkn+nJS@Du)$6Zg@ki0v#0uw4Te32yS<by-xo~s$435O>%8AO}T`O;_
zJ+`iN`{wTK=iSG(%eB6h-^)#X=VS9ybKjKXHXl1<FSu5DU8}m+|99K+zW8g}i`W<p
zw(@-1Y|*?$`;QpMUUA>`$`R|1vGd=o7Tw9-eLmu1gi_2^$Cpk{hKpopq;T^j=cr#P
zu79FDktNqZ$*AR&oI!U(_E)CWVPQ8F+*e)WJEL#0;>e0wnSXwDPIck%o9Ary(`@2j
zvx3^4S(A!_^_et(m0GAOdGm)e)EOvEi=5f|S?Pb53Dc4VVjhXHvK2YZE9AC4s6VXA
zWmV3@6Cs=@F-QJ-gK)T~ssh)dNUy!kqFzT!)*H>6d3p7e4DK^uT(>b#y!@p!`pK&U
zlgfTZ`mt8oW_<6Q$Kn6*v8BQzyI+rOJP+{{>sy@gND*GbAZp4dy_N6prGV#)s+cF+
zFHz9;NZ~q_Ap9pnbMB;=&|NX%x6CIll8nE#L8OeM^@zSzroPSETH{Q8y+UO-w^uSg
zVb+1lS&hX@A54C&xN_&WL;sdXUr<s>dOU^I*;8S{why5~_B$t67#hWFQ8xN<!{E4`
zOR3@nL%-dk<_Alqnzt-<57)YV%em-itEtkZ43)N}4WbMhT>t-t3ro!qHE9%GRU-0-
zv1s|s^&fA!-I==n_0@Hf)y_*4EH5t;v1k-ou;T}ZSr>oV6O-FrmT8+$PSV|6@n+up
zGL<r!uPwVzUCyoV^S6Cfd4exRQ)>Oaj$W^8axXv1Wh$laU*31S!M^8NxYV&Ua}^Kn
zbu-$fm(DcW$u2%a_#)F=C&$U(78@zWFO|-$f9!6gt@`LWli_=%TMI?EC`?h=xRP;&
z=(cZ-5eu`<HW{~^61#O^d7h)n(?ff+Pl*PcIQD9(m)+yM{&SIQCc5`7>gp2WU=1%2
z+;o8PW`9d?Nw1~Syq&>{G2TB-wagFoPl;60v9X-I$m8Y$xmQYD|DuW+Z^!<3Kj<~}
z%8{i9j$DnLJH6!SLKChf2Vz%;2p^C%H2dQHIMQdQm({j)YxoT3_iifPxNc3`p0eYc
zUY}c4(`lr0cja7xty|l}-!_G>b|`Q-rcrW3qw@ZyO4%pDKR=zG`@KTNbc07Dw^Nm-
z#X$wu1KLx9T$LWRs_H7uIdovvq4#Z9tRy{+w1q!aJ>JY<ro?hMCq`FkM;ODF1&yg!
z8%n*yq<37%ez>iI^J$)ASXAcLN5V}XroD){Jdwd&<lu3M=p$y%H=-k}T26<y-o3Ts
zzKzyHsdb^Y;c>RtG;6NuCmk|;v~i_~;h$NtcTG1<P~e*MU}K{#Q{CR!$q5}3P4;X}
z;CW_R|NrBy!q*4IUmcWxb5NeaEMcz1ov!?ZjTR{_Pj4@G;9TOs(VUVfAGUerj-(b=
z>#f2)sp{olTDZQo^f<CtJeeDu_kQBj-Z$4O5>5!lo_dk8yj|_a<i2w21HTq)pV?#4
zxmlB|FMfXIRqI90Do(CK4=yO5yev_o{H2}OO~jG^Sf-|r6Z_xYSDmNiC_eq!D$JSQ
zbkEvo*Mg@FIjU;sSj-PcdI*J{TcjfIaqdL&$=V4A8U-vvmso_hc$z-2SZv8<av;n!
z=32kytGP30O26rnoio#PQSjQj*Q?`{CYUHk#4tq6@jUwPX!4(<Gq=RfyyCSwggc|A
zXl`u#>xJUa>Tdl$DE|M~LHR$7@4wD_|F?0LulVuc?<FrIlO|{+*L)CQtz?=}u;$ns
z&fk``pDmMfwx3nAuYI|tx!dsEiNy;KZ}yk1x_9A)|M?X?@4C*r-4B>)9%$aQUgBzC
z;(v3SSr;9CO%ONUDfp~r(g8zN;arC}A=5){IVxsfY$kWv3%5(vOUSOs^j4aZ!lO4O
zG-{#7ucu$qSavS>I@hX#*+q7(>x{K0uL@iiU6)+JZN=xAD06J-g0Mu}b+1oeOTT+f
zUi$S+OUt$Yl-|WF$<2P-+IoyJNqA-8nux1O;j8DZulgrp^yc~nd6#xU&8VJ?^<tVq
zb54}J+0l7M%t+CFnuw;t(gt0v?H+~yUlw%<X?iVNlCslC-Ft?9{JJlkmzQ}>HCvt|
zsKt<Zaf#3Jf<msZE3fDVE#`Vq6zVX2jq~oRzg%W(ZfuO*SY^6eNc+~52+P#c+uzpS
z*^$CBL96+@UYuX*i76L<i|5Y_YT|tUHoEQfWlk>@6@d>H4NVOFlPA2*s$9b|HHYQO
z*ISZro}boy-6tnC>y5_x(%|WNste-QmN8{)&)(|Eq-m7mRp9BfXnVo=j=$HQIypHM
z@kN`N%-@uFibYdsh5Ez=E{;sBo;#}Z@7ySw#1XjdiI_tjkG4@D%a;84m!7}T2#VO4
zZ0hB`H|(u3_lNqs0gqzM61)Ok5*B!|?`v16<kS1{KcShsO0|_CDrQBe#NwWfiDJnL
zD$%xkZJzjdsyekSa@n_}BgtNGpGIJh&bk``+PmL=o9NDzqtp;Rr{Lq`ew$A_S}UzZ
zO<7C5yb>;}x!Y!jObPI_be$S3CYf3i>NjEO6tmoeF@7Eozf6saNnJW4GHY4qvZV0B
zjiO=8W`31akDakK)T?~elQ4DLPb(+Q*Zlc1eSW==mcznMy{;AEO_P=hB~IdVd%0v<
zSl3F1DU&{llyPw!xLeQ3;uyx@T+1c7N-K9+kWk8oNkTbs2e~rWCaq1&_S&$?@l;>z
z$t-W}6c5L?nF)=Stn3$jH#9N{TdZzixu~YFHu+jm_uH+`p0oc8t=8T6h+i{GVIiyb
z{|$>=YnN=96fd%N%j5vvr`v)j=+EAw^t%366Klfiq)?BC;SA-fthG!cYg|Rd`xbJx
z=Ln=saO8>&;cr(9lDX(&7`MdGRjPhYp{rTZlZo!BZ&ZR#X}m9)9H_tZjIgh#Smk6t
z`?oAtPP+?TEjq8M%NiQw&nX*v(WTgNYH)bq)M;Lo^LB=<3z}Z47Lm57_1fid^|rLj
zY11Z(-e~x^ReWg*dzhDW`O>{*(lvL#E*25a`5xglsdV=$A<dPq^dlLUU7493Twi)C
zqG|HATLHVb8qE&h;rSqC;(6cp*r(SuCpwgSrsy4f_PEk-9>Y(^Z{JHFRlfco+8ngy
zQNRA`%ph;KSsQ{rYOo(Qa}!CvA{?#Dq_Sk5ZjeI71}|p!gRdIdgzOVPFI>EQPxEQ%
z8Sh%7O!g=qS#v>neFx80zKflzmu~R)O1ItT?t1CHrOy84?i+P~^iQ4N_e<ka1i!ZV
zv@<W=9jcDBPT)8h?v~j(InJo_rAMQcY2hU1@<%F*WmLUlth`t^hAepzvBBHD^PHDW
z$C}j|-Yq&0oo4u)n6}K(w^dt)E2`wl%q?4v>MUglS$k)akW86td+n>Hg>qjtWbIak
zP1jOd)}_(lee|rg*2$Q4ugX@1ayl@j{Nl<<aMor#pcNn4GnIjF`6O*F2D7=1zS3JF
zR>%E+=#|^Bd%9PSp-5utYpaX9{I;z~J$|l~+o8(mkay0;&Zb;P?&uHxUcXcq{P$=S
z+2?TL&5e(GjCa&I10`Jl)r6mnI-^&=dG>U%p6QaBEk#p4DstC~EHlv5;i>+i>7mPW
zi7UEbjgq4B#PEvt(`_4^gnul4s#EWIy6@_x=}Nad7(B~_7$>Qusz!D`$-6dr#uXPX
z&reg&Ry3W>Sm&alGyBr{Sxn~^g{IDVwB^**b6>VB=gpfNmaKT~&daU#M^op0`V-7&
znkxS2!%_Y8+vy2^m}cx|5HaiuJh1#nn`Bv;s%a~83{T)^*K;l+J$$7ZSsJfIV;Yxf
zr!@Gw)lHvtdxFw`Mz+mz-^?y6yzum3;=Xi9O-PI}>EJ23!%0VTZyC0^_+8YlYL;93
zK$U^-YLe-j2NP7bHVBkG@DrYN<<Zxo=W^d&GMqS9WmWr}GT;+EVz>3dyzgQ{%EGOo
zZYjQ=_D71YZJu>?&A)_pu@@Va#66$5mTZlWeSd4wo^zq=pK4#M`?h_*%u^0`Rq06+
z=a_UZvFV(}9dapAK6{F%L#9>uqDySL+*_vhOi+#I5<Y!z<{^<~p1Rpq>R~k!QER7Y
zX4!Fd3ox_Zme*UgbyH}_Vlh)w<yYT2_B?IzR@=OOokECpph&~YU58e&cr>~v=ByJJ
z{dH$!$;YtGUMJn$Cazz|(k-&?|D+q*+)3O{dI9B%n&)?UOtcQaq|LQOc&@$MM%Ie8
z-KzbKiz3z;b48o@m_<pr2A*bJUG}wHWR^#SNPs4Tip&lt*5nVG!eO?$DN_{$tUa6#
zyxN#pm7}vqxi@O}r-yY{l!Ui1NE-O=*(kDD;;5M9%!Jbmd>9Yy*l;>$nct$$$E=T1
zRLqY_nk8NnnxWa6ZZ92mp)co&inVHHR^luVzuQhPymtD8*m+;~=~i3!akF-Ux981$
zF>Ljdd~+)ljc+q8o-wcO=)R!f2A`8gcQxx(P4+g4NP0Y+b$7!eQQ1CaO{D`NK|dZR
zyxqti{#ki8ui_DfF9xgLzB(urc0s-V$g9x*wznAMLtdVpz?5-DMD&{8G?^O*R@fL+
z`n;XTy0zWrdUe*fZCs05mgLtykdFUx!d;p1hm-2wh}B+pHX;fAORqQcm4!<!Y7L#z
zQTJ~1{H`Bfc9K_{)}J{x|DV~R{gZgVox8*5xjyQ(!%@Gaw8v>WljWo>l><MhngoAO
zUHDm5;MDHTPLc~bU&lJ}?eT2Q>oUp+eSJvs+=W)#qRD*Ti61tWDEW6p?O0f4vBLSL
z@OEMON2z7qh4+MxZ4Yu-;bl{I|Iss*@SCBYbB<41f9^om)2ugK#Yc@c=_h^fWomdV
z5&dxI-HA>I7fXB*lgaE3ba52Uy{e=;W0h;z4QJuC3C9&`{~zR<GvScf-X^8ltZwpN
zax*4H$!~29x@5qazi6u9YdOQnn4Aq?D&NMx+&rgOHm5!#NmN34;$7`a{8c?2wX0{8
zIcF$#eaoCNqyDIwy4Od+pq3Qj4)xAQvipA=x8|I6^N5;v$CG54^3BKX6pJkE`Av%Y
zCZ7yRyk)svX!-VV->)IUxwgHfPdi^<&~CTnv1e7jP(SPE{LY6=tJd+w%9VaxB&h5A
zk+-~NV}i;23098--pTB~X_D8YEPGyUg<p`-73IW@H=S%xOkkPD^Yz@v+SKne>!cH>
z9ydPYn(_3?n(bV&7OR*v48FNJ3Y1+qv|2eTa7$#0lFlTdqhWUY#l)Jg{m)LASa!&)
z;kX(D$BzcBiDhwxVVUa6EYHdfoWj34aQ|}W*E0%PZNR1UBL1HPf6@l7qN99C2l&cE
z?Ms#_)uk%mVT&j~#%j7FSHV$e-y!ABP{m0G#tt7H+Dt3EQ&rBHI9^h8oc2{F>5$Cq
zqaunS#wi=s{w!0=cZynRUg&ICoqRDWfzd@MT|+QDTEtvL%-uz5d7I?(=$nNxC%(2F
z2siKBXujlfz#8$IMF(Tjr^FgR7d_ZiEGSU-^KkLL0E6J=tj?+(>yzqUFRus+m5aP?
zIXR;=+f3wdYNEQCkHEuhg=Io{=?x+s?Q7TtSAPhb{eoYmLtgntn9L>quNU}#T>sy+
z$e~<lq370$@{HpG-yBL+4s*%`axongKeVD^QoE07n~7eb@T}#PYKlUW7FS+hVtn7F
z_u-Nhm*(E5Eha&QqF#r4S57rutkD;K%=Gz_s^bq+U#{r$yrSH6P+a<XU+OaPG`F<!
zg%YBlG{c|EPqh&1UZTz4T6FSz*=cpLB@b$@m8QQ>DppKYp1U~4?2~--1?`8^rCXj@
zUN!6#Updjw#C6(4*;yBZHMUwMmD=-9v{RVY@a}QvPNT-@7qio+HVFL?)!muRr7Gwi
z&i&89zIR&FyI_G$j@|Eq`ByLC+!DaQSb?i9+;9<tec9EVub*>Nlek$0Bg{RT?_3i!
z)o@_k@?ZJc!RFYVLhLPh{jI7~LR+F8BE4ofempi!-X(w5*J<C?oPHk@`*kh<|F`^w
zUzH0VYUEE;F9<B0cu7=jdqMM|0&A1#o=>8&8e*nXg!*Q>WNXayI>P#cL+n8L%wx*w
zF<-RRuFt%<)Lv_0ptGaeYDa-9-zAgXixXRlIURIGo7}HWat{`nUF@N|<)+Jpr|#<Q
z-pz*_v^nQ&ofMX+n6u~e>_gvXXCD<%Uy&nu%u_N#(B&l8>H}Ph9IW4~a=kGKUu3}b
zWkJ%4it=v(%DWW>CZ^AmYxFs`qvE_q<dow*)6^|p7x~^@;u}9P@_T0QtsN=Bjj1c!
z`?ejL(JZ`RYg6h<*Z&1JOZoyQ_stEE=$Kf|-&M`mC1J0WK2v$3ruqcF(sohP^h806
zokp{g9l16pb8k5~Yt1EDO+h)X4-@Y$GM^V_X}vU4uS;_FLtS&OdX1xT>=KP}h8c&h
zP2#$-*x5=%Ok;AA;1Z*o++Pypd3S{A&RpX9%5KpHwwkM9IvUFFH*;z1sOK!4JljOc
z<EC9vYq-IUoP8Ip{v~j~DVVyrVQRisgv!CG&WD=gPAX6OFty~AGIvVE^)0@a6IH!6
zR%Fi<IhC}+bz@|r(}Gy#sJ~_+Q5GVmDTN%rTH_aLtS|N7#v08#eTLxjDoOTLvyDWi
zG{yv7amo26F_&FT)x2-+PQk5*{<j-$(Jq`K)S$sydbs~2tK{xkLSK)r-rqHI+aq^H
zuJV1JU5kE*9=OJ>Y*8L$Cg`!VbNdT{jTuvRr%cg2*0{w(U{OHQgP(KD!<8OI^6$;?
zImRLMa%Y9*#E4Io1_p}rFT4=q%#^j-mi%T~WlE5%bC}5Wj+EcK=HGkKtIe`f|H`VO
zw$%45sa1=F5|q{BFQ&d(p_y>a|C7YR%+F^0-ckp-Bm*Z{NI%`kts1BvuFax3Yx+h#
z)9>1A13b1e6-~=nm9Q}FgevRq<~6Z`lT!j44@!1fNsBqpY>J&Ipn0)Td6IzA3+oG^
zcG0T@&Ryfby3=;cf{eWZA-7I$iM_hi$9h@B>Hk|yE?I3l$j9%@miKDelbgBLJ2)9r
zmPb#B=$N|v*+s5B8T^ZiWcySZ&K0(ZwX_JC<ma8-t|;Q9)*{@sDnF*tl&focHK$6N
z;ert7g|Ba<HdMB4c)D_ucOi$>&XmA(ar2q8xo5t4GD}&Vfju&I3zt~rkyU3|wHI}}
z#V-uhpDv_dKFjbr8<SvALCWqQMXM!$ix%zJ9e*&>IZ!gFq2$QU&4(j5A8Qj}a#%aJ
zBJ9}9&J7;?Umf^2ZuC}up>(Qx$`+??l}&tiC8bYtX&&?Bp1EMkK9i8OE?d=ptbNUG
zr0Jx@py9Lo1^*+fsm2lWCWI#!MXBhwFV`2e?7PmjKY{<h!T$qG3RX|C==@x195*rY
ze#`vtC)fXx-0(a_%5clV*Gm>`n{<#x+;qcs(FxY#zR5e!d{^IgyjmdZQ2E5!Tw7Ul
z4sgvaY)>dTC~F;f!d&bKmstB|y~n?-)vVks80FSH%ABiqWNw?e7USa8kK`412D|PF
zW_c{PTX-p7#1dz#47D9xUZRc7Y4Yi&Y^yIc#Jc9lc<^u8z!#~rMY-bG>kFYCp4>t=
zwk+OIFTW|<{N&m<Q^JHBLl`rbDJxZc@D7ubI8mLt%*bikr^of%#gsxcx24Wrm$z^^
zV~Ekl6n>^0?t*Bxn+q-H9+u_&qk4X$umsDg*hh{Z%2qs3VQ9YXmzvlYdg8xY_+vHc
zKQn3;`NeJAQ7c*4@u`4SdgYweg?ij;7JOP={a9<Jc5Hm&uKe;{DS>W+u5PQNxT6nD
z++!oQugXJcLdVgr%@1~aG8ya@5Zt?A$KHE)yp~sXa~zy|*}Gd_qv=6Yec^Qb+f`fm
zJ@}P7wrsq}|8!FKy<JIv-kkr<b3sjM-j{BDX~&2skzD6q^t=w@T6^Kb^QC5ojLfeW
z>E60%X|-(C&sUYsVJZJ+^?GgVWw@PEu~JCf=WNJ{6|sjmFf5M}i8++aI5Xj!v{3X$
ziQA%oEY8F!GUk0eB;>klp>^NNg}YuK(f+Y%!oFtx2iG_4uf80+(EQui>V3aOKVH}6
zI&2*r^MB7CF(t2+ifc9;JEh(9ax<5YCPT9C-bD@8iytU8q)$monEQPr->K-imOrz-
zt=JYhY~H1oq|DJ3Y%C~qD5r7l)_|W=T6J#(M&5{7eM7Bj+5ZbxQ_pm>&NVW9AwKC6
z+uDMgc^A0K9&nW<aL>tQk6s|F^GkN-W7)eGC5kS~Oj_*dd!yyUwe8ANPMI<8P%)Wa
zUfKFU!oPIUbhhWWYb0mdZBnoPsopK!)}|T#eTw$6IokVjG~X=Ix|(=d<-6!cuE75u
ztA9SK(On+6cC}E}73~cdwbMh=-zMzN-@++XeJL@};4+)o`WyUj0{E4R<kuAJz3iFj
zc9Z`O=e29U&oy70`+M&H;C!BAzhk@K7;wEV2wOd|{NA0gALUzA4)I-yT(W_q>m=ts
z#cy#zG8Y<n8s1x<`2A*|;=X;tkNLL-aQAFZf1avrR&?_q)8lms+(vqjH*UGPG34>V
zB=#bgi{&b_uPtH;brJi;Rpq{T{+*6gQ5%a*I(-|CAN;3wXtlanlCbm6ax+HJg~4KV
zXYMYP_nu&&GeQ1W&B^0WRrUmI`6T%|>7L!4!;1?S*Dqe=vF5U~;G~(2i*GwU4_IT6
zV<)0$@$gQi{OS*iniGSSgXV4&yr$Q5{paDezNgpjNuImuqxD~dYnRLRDU}^ty@4y|
zjoo*{y#+Os4wmf=sgVp{vqxi-x8=|JG_U{5nsrVvXoO{2*=2Su+se0nlZv-ypMLHE
zuIX=I&$!DqYwgVie{ased-E9It+{L8oILktzAxLQqFWc&SzhE5sl4s@K}_~k&?)!8
z6&<%CXHP%KQ9XTQx|%4D!#qJ1+fUQGUpl}27d2<s=`E>kuiuI;)Y9DT)xLA)`@LN;
zUqfZL78NOR$3BXECiyfV#f0l&u0Z@kK}+BF2UcHxXEgcinrm}HlGx5Q`mhRetjqcF
ze6GlQ{=2K^D%}rL(y-bf^y&IZ>wB3GE6b1aKU{qL=icCF{A(JP{=M^Xg~g+f()y{}
z)3xt$Iq-XH@;x?-`(iBj*xK%~^|?25_ulm5zvX}a%eo6}vd{lNE>F2w(rmThwy4Zm
z5xzec<zz)Y4~zLsj#B&kLC2*a>{%7ZZPUe!W_g0&f2e%_`DH_)zJ#%CnqZf>udZon
zv;L-6mnEwt6TUpPi~X>*C_Q=0gv1iJSGHG+&RzLaB$-vFpb<ZrX@cFp;s+5?)|Kr|
z>m%l>hSh&F{PmY%t7;f`?Q^BP<92CVl{6JA_N48Zd!xa8_vTwYVYBV6w?F&6&Fu9-
z|JVI*xeIPv|6qEvZ=Tr!y*IOa-<<b<bFu!7um9iEd4I3IznQn-jr0cDn-~8sUVHN5
zIu_vrx6W;0e0F}?^>f=d+-)g(`A_{we$L1I*3YMyt}7o6DyVp5;@2$N$|=Ca^1n#%
z!pd%k<~dwtfj2xGoLZPzr%dt)R5;QuWSaNn#>K@4f>NFOL?VlxxKH6?Ss}6_h|8-p
z*KX4!kIkX(Gnq20?r1)E<{y;9($}Opaf*MZlc#KMdiLVAi<u^iEY;e&s$pfAshWU>
zBG>9FnZFmMO;^R+N6pHZE4ra;-u}?lZX2asy&LSy*4((76#6=EZ(aHCpR3((d`LLR
zbcikPj$uwjV<U6+ls$!?j-Q^csjU|CZNvV93mv8PVs3?7@2iR4nB}_a>#M6<qfTzS
zy6RhQ__m_=*K*UhzrVM?^*7tQt{opAo@kbTf46H#uAY#P$8x*7Ri&?jwsG9JxA%oe
z=Yyl%67m1%*-Y1bado-Be#3+6@8Q-$YrN*Ks{JGMc5%!6_{6#?r)GFCFlq_~EMVf)
zX`XC!LZf+#ky1;5-4vk-O}$|)PM*$;7GDfj@|g>D9p(*G>e4c*5<0PU8B3P?qOz1t
z0ZeOEf}E76O$m75Hf2+ir;?QFgN>^dE2k)Qa=3jwD&IImWWoG6x5%XiRbh$Gb)s&(
ze7dG8YvJ-~o1M6WjV_!^+9ZDPOv}lWeI~CvJk3^j1uL1UOXo~6eDl%OzdC2-asQhs
zlUMeJahQfM%)H~!XA%7-V@u#;1>3w!uG&k#1bG`DExGJ-I^e-&Sy2zCkRW%{Qg^BC
z0ig}NJ7y<!hcCaOaP5C&_R`R>n3Ing_UM*9eRuuO4uP_Wq|RP`JEKRZJ@$9299p5_
z$9KzPVH?+`ZBgx;mKNO3_52jJq<6=njan~-Jd$=V%DWZjnr9ldYO%1Y)x^VQ&x9I!
zR#>c3y<ZlnRmA-^ZqbJmh6<NCw#412oV1~pTR7L?gx%*859+kH%}{QSoY*9KE#|ew
zwtYFmlID|gPoK4X8naXR^3>Yw$kZos(=5|w#jcY~n_ax_3*VIP-Aku%t`nBp{d!VG
zs6t8oTMnDX8EX1AU(N~ZXMMZ&IEnk~hP?isZ){25k~VyA=CoUX(r%k_E6Wj%Ma-67
z(MpUvJ3bubD1N1(y{2nYf^g#%UcsFI2P_|zJBPgyYY=goduv62rjUeC{<>9%qPuuQ
zu1+laz!dh<L)!W}$4TD{Th{m-oRhNJ^T>sg;F8ntZgrGiv+c6|P`3SuH*?Um$tC4(
z{lQZ%d#he8sd&(7E_eC9_|~Jnx=x`E9;`wfr6IDD8{#^3fBf4T8oKj*X%}bS)4Z@x
zC*DsDkKgGrpLxwy{cRE7TT-{#Z?slyzV&0#()HX;i+VYf{^_zP^6Z?p_1|^Dc9ZV1
z(2XZte3;hd=JMrxc}#TYNjIFe;dH*t%agtyDN2kzk2GFMxk`I9>ND=~Q1WZ+I>2B)
zS2{P{ifQJ#RV#!n*NePS776A_Qz?>^i#y{x@n_<HLCZ-q%~S<d|E@UN+H!f$0~f>8
zqn77Rm2BP={eUM;e|OT-T_@(~9$M3_y?K$?>IxhEZB0DR$;zc`%;!#Dpq&|F=r21z
zOm20ku5HBOS=GO0tVz)guww|i=EiYy;=dG^9H~mNMz$pf7*#bKPg-@g+zK;%y|YbA
z{>lkf`xA-||JKHS{KX+1@$ul1g>K3{n=iPveGybFbny^VY8DYuU|sk2Mb^aVPKUfN
zUW^(KJ%lc}=srt1qOSPa=y=K9IlngYZ7?}mda32I+p3p>Dw|%;l22S_X8owUDrJ4e
zV;5!1vypx!vnGoO{RsK$_4QnEB!A|;kN4Ijq-s|`Iu*7{=={1r|4(Wsob|7JZ+vaT
zqxLn`8+lcG_ndH)P+FVukUyIB%C#;QPP48#OSG9DEOoEDB5IMB-7Nc(bN8yGODhUC
zO);FvvCmCQU|-I{#R;nuFQ_z_s`?yg`(!!YQ(Qu<>I<{cVgXj&7lnaK79ElQcjJhd
zkDKN^TlaM1%aMgERvlfP_BAbP?yIovw~WKLI5N2?ufDM8x>m1>?^?g)?20>9raQcR
zv%-V><vA;_FR5D_U*Wr?mZ@@mYU-0o+^bHw-7340dp1Pt_RfUeHCh5w!&EmpPZW2b
zRO)DSqpf{yhU?>on-dEIpHAB<a@BX|g%I&DS1!Q`P4QhtJ{Mfpcz-d-O2{%4|6g{3
zRcFhA$<uVqj`W}5Q#}+r+5W@=R&$Q?<ukqR6=_bjd9i5o&X*?lN@|m1pZ;uPUeGk}
z%bm!*$}?Y2yR-43?a9}BKOEHB`#{+L$BD=HVsa1Gyzaj5@S-@}PvS~W%ISP(eWm?Q
zm)z_56(2MR{}60n!<$>V)Qfrbq6mevf-C#~cHI55Ofl$?c8hAPpf;O2gV-jgLngA<
zOP3h1auhFF?0(6~U+eSX1zZj#TWqszUnqs_y5OX@I^mMU87Ka*4km*w0<GER#Ya+<
zc4|2K6iuGE;)&XQ*CR_OD2bLEZ`Ml_>`rQ#Ic;yoG40DaJu%m`#1F<?ogHXsm7DU}
zVC!e?_X_{tT9+>|mis!xef8Y4IrV?Fo5fGAb(yAnAe}ov#kfeIeZ|>{`PGVg6Pa$-
zHEvR_6}#v*$y6!ul)CGTt-S8HE^^JxQ<=8l>(bsL5uYH=%X5PROATUm8O~+8TwRme
zI(biI_l_g3B5NGH1LwD>AGokh&P6Xm*JWeZ!41wl<%!27M6UT{h#wU%X*#)O>MED4
z@Zj4%SDYlL%s!WI?RBSh(L2ktvG4rWrd<7yvfbw8%T+axK8Nk@`Y!VCr2o4O{O6jl
zy?1zcJ-ksVB#KF5ha1PIfGkl9Wr>w57$tpniyWS{Sm^r+F*}|4)AN=ziz>Ia-<*&r
zc-C8K?TsU;?h`-WnxM$=zllfp#v}#S3Eod`<R`XI`Cue|h-GK{>J_eet{3??38da|
zeS2x${J$TT__Y~CD>xU#J==UYWusY{hO_jk2W?>~8+oELnzi#Ta(z+hUmfu^EBvhA
zx6SFHDbg9mvnLxg)}D@h=k?eldbXSCp8GqqqN>l!|9)Y5AbQtQ5Bb}FzWBV@{C8{p
zxuPWwY<2}llsc9%9WLq2y0pDN#>z$Hm`&&>(a#6tf5e$L_T2Y4cY504No_)ncUC7h
zZ9l*BpIe|rGs7C+<y9RAb$sraMwvKHj$4z&6K_>Eoh9J+ZNE&Nuve}T3r;3HI<@YW
z*Vm(JI|TAPwl?ypOqg95b#ULQ#{E2H3jYtYoG6IZ^I0HrFxh@r<8kZnyPGQKzk2uO
z4S$^M(~v$P!}V3MTjPFk9r*TW>d!3s+Ti)WtZe>Ti6x%YXDIIfw@W_JQ|xEqf%X*+
zjNA)YiY>M$+ISR+^Z)6n*=g~!U_-azRKa9}bI~#`j$D3L6PNy2P;*lu$iQd@lcDJ;
zZI0%>eG!JQFSck@PpX@-bK{2|ElsVkh*l-;?$~J7T`Su40%siL(CXMIoTA0`fN9;T
zij66cobooYuZ!e+n!zkIgUQIkIi`s3r`7%qvl#d^L^u_i?`kk}YA_pBFS6buY4mBA
zp0l<6o>h-#&NRxf-c+Tm_>phxs}7NxS<6;#-g(J-*Z(a3U7J=PVb<YN*<)|Lq~Bx7
z{|l>+Dyz7hmJhV)@T-~4yn4polRLU+>huO^o)zZsy1eDkMw3gAJFXds_i_luOHB{w
zT6@KLYdDvv!bFZ~2c+ee*q(@37AM_W8gM9Z!K57qT2TcvqE@u)Zj?PD(R(Rym%@={
zDqK8illar3Chb-@zO-qw>QC078w?w)`iy2US;x#|^VAcXz?8qaIc$dLn$>geGfL@Y
ztSji=Uo3sXbz#Tz$n6zgTZ1Mtyq&QAZKHX|mZ{a7w!i0`U%zX6LgIn%3L1Si^M7!t
za=n?K_;P*oW2W{jvCc&+4y+PuvywTqx{IM>&eYec6PgUB9at*LBNO_6kIh_(ou?0P
zIQ5EyZ)fM?gl6A>qdb)-yCyQ}eqc3kl#qUHsrrPcgOPWY4bQm)5=&b+b@v?V^*Cl9
z$*cLoN^=+UPp#(l#m%c18yU?IH42cgFyV_lY4x+%d6$g!9-ZCxS59v{?0&a^Su;U>
zQRlpaGPBIPjdyvwY|nBzHbwhe#gR2DS9{Jm;>}~T|JJG1Cr<pevYAq`r_X|;sbluB
z!%M;p_j)bfz@H?(@Ph(ZWbZr?$0;6S7f<UjZeWjAVU2mxA^-3o*F+AE*_!uUww}M(
zRJCKTyi{+>Y@y{klh^&>cznd{XT`aq9HH#nYPp-u>^=%RJY3`S($86Q$|q&k=bKoC
z{;zP3&S2j0q+!D(QFhJNb(h!e*wP&I;JEwCxgMGOJsa%{cl-al<NyDRp2;QNZO*2z
zf3ACxX&$1ue&=iZ4?p)uFdkskkzW=$|80Og^P2VSU(6?6S?$oo(Zr!Lk>?`!m-S4m
zxLBK)Oq<fr9DQ-Mf#ZyhIr5y7TwPWjbLyHNB*<Siebo#{mqOm_Hx$Gkt&!sNTf*gc
z^b^<hDlZGm?!}k98XqVsyjplR!%x0xllo+iW=+2<FSzwud9VElx+&0cTSD^j0oC{1
zebE}trxg2w9yD;?IL`T@S7*{?BaOqW6X&nHy~*g*Y1=!8^>5A8KfKA<SS~tZ^Cqj1
zg^ovRX9v#6SZ($Hkg?sO)pkr45nEgwe{EiL;L6X44SYHldzD-R{~S5+YI<>^=803D
z{T)YrnRI-zo9C{u;W{Ypx7tnmkI9`U2iFMr%#aXQ`Z_1E$@axc!Ne{m&43veJ9$hl
z9$S9KEH5dn|ASh_XZGS4p*MGgqhCzkAR_YqPMFid<M&qj7l>|m6J)p_5$qi~@5^H*
zC(Xm=DRN~;L{@7zhk58Vo$>!8;!!Y}GsMw@<*a?1OJ`K4M@*o8?JV=Im@5sN&NRRB
zV7T4S&$@;)M9h!Db4rXwcdqB3%xSzfT~m&n>Fw~mdRTB}hn(hxd8=-mJa>6V-wBIz
zuXpfu?iBvHdZ7&KO&iO8fuL*uKS%UG;F9TLnsRov_TEeCM=xb^Ptp5(>H3392C|a3
zH?*$5%)6qLm2pCp-JDyNq7738JEl!szuRTz)WT-tH=;Wi8vT^sZ~bPG{n^u79(!-Q
zdwBC_$;m5u`<y}!nXG=fVO~Sv71pQr{RJV%&se|fklL4Q<@v>W&!wBbraPR3Oj_4+
zowwOx#NoT+A&1i6vnDCN;h~x#O`Y=sG<PRX_qJZh)1`Bv+10mci^xj1+Dz7Xo^$=5
z&aQrOb&<fb>B+3cHWHl)vSus&?zy^W33c4Qx-9GM<fs+B?``hoZVmsphxy)54(oz@
zeII(ubDA6WtSjE?|MaW>-z)wMXYakc8~1xltNVY!cx#QgkGl7wI4)OZFzX)Zx+Enk
zkz~$1b@^UbsmP!9?5Yt>cSXNj$$z(SU|1Kx$6@2I^B^R&Ls)9h;g<ou4H}&mN2l@}
z6rMYc{aHwR^Mj5Gi|)%exj#vVh%!u8<ys_hF!a`*$)_#OO^Q7tx^;a9k5W{amaUQI
z)S&7YlNNss(qGEEOjYvUil`YgZ&?d8L?4Li<aqSlG)al?k%C-OV?~^yZ_-pv`z<^n
zHag1M+TJ!>jAlFDwy}xU-<D*zWs$7h%s#_~^M730?-XKsEV|)h^sdDM)=fwDPQG$-
z&K0Mi(C&*@j>&GZz2?%-;d=f_M?j2?>*g76HKvHYQJFT0GyMOaF8)K0`4V}%Cx&__
z_u5_T@K25nyS*Y_WLe@Jj$c!^%5<8BGd+!IF;$%ymfL+Tb86V_&9-VM&)L5e%3GVd
z=66`(*{6l9hMfy;#m&1F@3zV#lUY-t@zv4mRd3I~VG944oAzaIShUCen-A6mKbn!X
zEv-+mL;s~v_M7&nYuXhy$`!g@X!#QEnliOrC8CmBzH;`|>er&KU;Fl*y5PG<zva<p
z&($|LkICmgeBNDplJnS^6BFgSn(uZ_<|=MXx%E+#e_z+!4)J{pBM(Na3V!Nb9J6rZ
z<2A=36Z>a$&3kKXo;X=&YwO+}iT#behHX)kD{slMJ(E{^q^uS`W8O=(eO&7Q?{z5X
zaT%t`8`fBDYh5*8&HC;HD?5?<%SG=`mJE(g=ym!SXdD)G_LFdd67Q2WuMTctug{2=
zS`__yMf9N~a_=n89C>n^HB~C2an|nc)yHM-oS5R`X`;gV?1jxgj@A!ncE!b{?Nkvp
z&WN<SJN0V^%f8Ojr%a~4ogI9OWA{ZHGYQ2hy0P;`IA$IYkJ@^aHy|;t%cAecn&xX~
zRXiA8zI=23+Z&b}P1#$+a$Mt_9p}6|vg<zQ^<Pi?<ECY@1<&bU@V3j4Pwz&<za#rT
z9`XNhw%08)Dt#Gm)HL4UM>8HDOKZK!ooC0LbL805mPr>HHmMhS8Fe(BVATJyb^qHq
z^HVqNE&n&~ek<^vEj0bxEK$ZO7d|@5Gv>W-yV~!1X#K<^2X-7ga5Q_GeYQ%wfV5E0
zgH;C}^cP%Qz2l_2b)@K@j@br^y$a254Q>iPxj5@+WbcIDbBEsK8U{@aZJ5Es!0XSz
z_x;V)$6duAFJAv3aqEENQnyztuJTU*);mk?(HylT3GpPQXC1TGWj4+zytOU2GvQJ1
zhr<3HnYZQ@N|du&g-o&E`)CSdvS!yU9i^<Pa>3yludW#+3-iy+nD^?^hF9ljypuS_
zQ-481LGg$g^J_D=8zLb-;e|&m+I&1p`nT8JXb%l-4ZGv*yDGp%r~N}np5^o)gS~Sl
zrg+NP_&0T4NKMIGn6mhP!AkqUh5Hz{`igCB(%ZNvlJkOCuFv@|KF>7-)kPzMO%)S8
z+G@_<-FCk3L+Z+_9U;?S-{O;r-qCCnu>X7Z{<L+aA9eGx;(DJPOPjSIH2crj7tcN!
zCUWcX@La7FUUlG@S{Flv^z-1KIYn-wr!3PO9rD90^S^P)mmMoBw~KJGInWSwpjYO5
zo%Hv^+^R08^tn|m`Wh0Z{dqp+jzhPS4exi~LdoL$do&LuX5{^h>_2T#r`SAWZg*ej
zhg<hvOo`y+3$IxBqM$XsbLN?&l8g8vX8&kvi)Fphu<SH%<A+^JW_)W{e@^54IYY0q
zb4THf2Zd7@ih9;@O<O->y{Z4Swp|+k%WoOpn|`(8)e_&{s0&4hBedFAzFOeUvx*}s
zhgq*nvFBFu-d8K4P0cpG6Ua6X`@A{s^GA=@`&(DrOg^*r;bZHxuV+kb0;X`B5Z2ku
zKb_sX>0q2{*n%xrC-@#%6*%?MR}Zl+Dc3JT@5K8Z?@F2MJIc9L>R^d?{~XR+GO|Cv
z)cXEmx|>rU$=P-#dsRef|AO3;?v<+oDihXqg`~#`L`$yz-4Qzd&$R0uI=32LTi?&-
zo$_d3+O_j(Qmc<WEB1Q9*ZW-U*R@VNuEb^f@^b?dzD?7(Q1&?|=;ON9Ip6lG=07P9
z_)-_}(LnyWyZna+??lOjdhHc#bKW;LMQq>ro_kN7;Qw6DO7pHC4<fqXI(Q^q<l*}u
zeoJ~o_><XBITmp|4C~>VnczEX#YG3knxM<Bn^sjcOi)NrIM~F-ucWiW!GVc0h3}qF
zfqR2&7mK=^j6~rg-6mN>wJ#b%N(Xxsee*;TiWbP~rKlBThzdP)nN%Kg^On$M(bZu~
zYPS76x$4@5u&u9du6n+Dp8YbP-#?qROtr4o8Lv+|yUKJ)=!$<?u7)=X*05|!JH5zv
z!GeWTbaQWW-4H5Ycc;1N{js@94*K`^*8TqU)X3G~;lVae?gyJ%y&oN8me1?exoMbs
zj9u3}?M!Ep;~CD~94}NDw!ApWEbO7E9LyknZ3*8tMTy-Ha$2@V&;BpQFjFD#WQxbt
zx<A1V?oDr5w%nTJvXFt(OGKkkFSfJc1(TRij9tKQ$3|w>s{4Kyb~$pY6fKsYxiz`)
z>sqf9Ir;~BkNjNwV!FoRL@()|im{4U=4qGhaB34-v!(b-;NB~jz3ut!BtlyRoHQEj
znZ6_`I*J$VSlAv_TJg|9JS}2Eht#Z&N1>8Yt^xMmnHOV<JvAH__nJPcSlnmfrhb%#
z<*&pN2KzjNqbzPpl}jdhh*hRcbbS||5)<@liFjO0!px=9qVr}(CPkQi6G%>0JLWm9
zw~=jG9(&=XqiOkE49n*fUfOvxyX=x?RA%Jc$N<gosjdMGXE-Mu5nI1>dWO@Q{{ma2
zidZ&&x}_1``$<50vB9lVOZt_(WE{5&Y<Feqn$&fbCHjYX<SH%25W%8c7W05=!L>)U
ziq>%j9AinD;wUY?@#ex{ZS%VakFxE(wmj6AO|XJnGy2M>^TED>rn!!~Tmk$}xA=~%
zb-bb0qPru!w>n^_Vx{#u`+pgN2a8V670r*#K5Km~O4!5jXms93Z*j-cr=}Ogs#t9V
z4pwuv6(qD9u8~OW=3RC>i6wDc^h$B{mYL}y#Wn(I3v$_RIVgFXtzwwXBQkq2tEJRS
z$DVE1X8TMt+odMG<iZ{4oC%v26)%t2yKNJL{Z-~wj&CO<7M*0#JR9rir`&QkG^jvf
zwfnmNx0=3QWfwJ{D4lxYdA8)bErzU|3~eqCuIGtdXOomExR{+IofF99ShnR*W6au|
z1Fo)RHx_AcI{D0Ofo_gRV7L9g8DU8tebJ?ho=DVJ6h8SkK~SvR#H{3TaaL{L;wUq(
z=Z_tJ3SCKwv*)Wxc^zzGdGz)A_pYi_f2+1xi&w4gOud@G6d#!yF>B?_)T765?*1sN
zyk9c8x~uKE!#UwCH@{p>w!bj%TD#kY`OLNFC;V7$7%S(P(lUR=g~@CGYve33kP=$7
z_TV3lziW=x2k0&e^0O=x@?i`Y{n@dWCrG>H%{m^9OlL`v74jjxOL+3K56{2l(4;$I
zq1VR3!wD6ptVUb@ALJFb;CGBMoVaglZozg9=lfin)})F)X<#ipsyk`(%6|_H+9xJ*
zX)h@dsAM$M74m3e5oyq`JDAE`aFAR0FYAUzK^BAhz=*_EH#QvgIg=Py)p4{$Vp4A<
zPh&KrvXe*JEESJL3+L|Mb1IjQ9iQ>6x9gXrW1I5)$6FpLUU=Ep!?TEC<=K?}D+|w+
zo^U?#p~Zz!IwRik)?qn+)zua(x4Q4VOXfEJa7r}qf+x#Mx2C&O5Ahyjx;<e>!%6w6
zjXbVJ&Dvj%nN5qi6cHV<#N8so^yHc@rEMopN?dDI6n&Z&b7R(>&Y%+A1SJ)hXw78|
zO6?O)UtJViyXRcrs+o(kFP(7zzf4%joK;}AYU5<j4bsaeAKN+o?}E!-#V^lH-?3!r
z>6(>uKc)1{XkwQ#idfcNvg_WQUy;k}<yflj3Tyn>lOEuhd$c`{$Md&D=Y&7JXBB2E
zF`r^H>RRwHM?vzce4)oghNZeDwn7seZum6`Os-kWvn-@j$J2myhhjUku)+kNyA!r^
z8Web_6x?Qs*y$;>Wf6~1Lp1a43tb6?3wU<y;$)A`m}vdvqLG2gS>G&=Hi?BAvD1n#
z8gF*pS=_pMHS<fa;A0=wHE~>wa}42*oU*{Bb|>pP@0dqcuepvk-7W2E3zUfGXEtni
z_32J}HRHpBj2jcOQj=4xI8I7s&kAJZ3F55$-?2C)ZMLaniEm#PtG3*v{Smw_eUl=2
z^V}|WmDyafsG4#};IzBlT82v|8+AK`)D%p%C>~$7GUb8#<fw^8drHcKHJl$LEIJvO
zSH#Tx=~?Zbldb-f7;=tjvHEWRV7iEHD$g1gJ$5-I&rO-%^CF$4*RdIeu4OnUCb`AE
z{g9Qw@uzK$TNkkFzL>ah*@h-r*2erWy(1PqqQ!-?om2!Z&5vyD;Ira*Hm9lch4W?3
zgxU8@F1=EdQ7PQT=efoBxz5@pyZQCg%0J2&S|84Mbf~J6`z*8YcZ<gJPcoM&Ot`6W
zF6oSfRZ?3ur((u;76T0i4h9BJflm$%XD0LuvFKTpXq^6kL3pK~;;KbOIlVqB_AG0;
z^2k)d^-YS1!KE7l7eaqYK78kPLVJdcsM^vUuF~H=Z{=O_WYX>;R?fO7T(Mgl&)H`+
z%T|@9bN2AAOx@wERr*O=IBU`r$%P9!xhuNeb{A-?t>om*z1XJlq{!&rFA?hlTe|`i
zvP4!3u<E|L$RF6#Bo|}W8=kc=VB-?gYuQs9%)TylQ86>T{{Po>k?OzV+Dz6PG~+Jk
zz3_aaE$uRYwcEyHo0f|zb2Bh9IXF25Y@YpKLsRc2PR<P03vcK8x@~a~W^l<byJfwv
zOR>f0gwvkGvJ<#mvP`=+2;KG-SlOz6pG9*K%cbWNHyu5~r?1HD`}6Vtu6-NVDhYEu
z^;W&q#rfl{s@K*2pwDR;_opfCD&Mgr=(xAS0nP&jPgUPLPTM^-bm=LvwG$k)ubJ)?
ztX$#Zx><io7~}oGjzp2~Te~zO5?@vI>=OISu`9y)kWbp7J2Ot~3D~(Hh=0wmi?ht7
zWX(kHDA%}J9J;%wz+$)Gxk9_lIf0)FglF%)afac%QOu_qw~lWqOO}>a)Z2IOi-NKG
zuK$<k*S@+p|I3l@a?aKtSQ^Wily>Lnhpk$~+<D)1qBECE?iGavn_8z8?_dnNvB=@&
z$4yKf+!Js3G)~Q(QM~a%4*Lz>UEz#-R(fSurO(NlbiOApT6z_ig3E02i@~<<?^d0k
z)crs7$;E9J0Zv`VxQ<_Kx+3B9RqGg6*59ZOxds19T5c%5S{9MYBPGTBW&hEu8y^*1
zj@il;%o8u5aOt$p%jMIXAO8~Dc`tR{$NER>7XNtN8n`jBQPk|t^($|LCr{iYJC83=
zdP`vRMArjqTNoFo++OV9_NO2$Uqd?eX#eSo=qIhy&Tcry%GYw?OvITC5yv|hc=z@s
z{$<JHY`DndV!fqNXn~uD#@U?Se#Z=V$t_Gz&nhffbKAk?_%VhPs%I~4u#ri=;WSq;
zTsmgX_Si`bF4t%Eh}YKKY3R69<Qw#m=fV9Qhs1oiE`2?u^+ZP3Wxm83r^w7h5^I{9
zH#nW|4VAjYWZl8|-+GPrr$TvcllxXC!Jka;JASxc&HJGH2eY<|BZ~@4Z;RgxgB!&+
zPPj+PuFbsr^3Mr}73V|(W=wj++Pd=G>m%n5+~{k6*;?@PT*u$}vkasY9oxKHJU@r@
z&$@9}>sl<wy$G$mo(voR<cSy4-rNo2h!o;bcxo`y@k`R=NChXKQyMW`k^+x^TF6QA
z99_O7!TsrjqdlzREPi=2d<1P<JZo4QzD!Z;o!~XavBXxY)*#@6;;qGjQK5o;VLnlw
zF)r3x-kz~6Hs_9bJvbuP)A%FO!~TMkT1S(H%+Zx=nSvW;t>|^KkBI;5*t+7Pv%O7x
z@R}wy121W_$@U)cj(iKkCWQXi&S1PUXa1VXCy^?RYoDIp=W%eY<weg&t{y*b9b$<V
zO%$9Na%N-flg>m|$%%nZFC@Zb7O?Db^H*>=W0K(5kTA7%f#Z==dPS1bEU6sZ{H?b%
ziKralt83@6^Yu_z#I%5Gk%G_yjTV*@u33zJ^Lu%NPo6r~8WSOP?tSeY2akvK5-g`!
zJr<?h^5O6}@%B<^BiH5LsrEY(S4_R7@gv!OgQD%4Lt0A`9j1kzU(1}J^h~RV(Kf)<
z?%N@aJIU3$moG`qb$pX(yt5<xMw1%HqcFZBrh#{_gvu&9u`4^>n6Tm5HQtjdYMgCH
z({9WaR{zCqJy+KEQQC(qmlLjDzUiwl+d}%kUy<9rrzh<?dmQdQ3f-fS^mkE;=ds70
zUKytI0>Zok*;<X%T6Dh%EOby_b%G@{%atkQtjivOcQ5C;TnO;+ajE<{&sB!yz3LI}
zor@hWEU&z}yyj9%?WGHwpYU%J@jZW)S$ff|)-}%O1D~v5eAaxWNlhe8dfCGB8=KS`
z6df&Gbt9I1FFpOe@~Yzl<&_4lwhF1m3*z@Ik&oIU?-|wk=O}Be>BIv9K5?>V;$F(f
zJN6v+@nQ27Gdib|)Hr2}Y|EC+_J_5z^}n5QOu3S#c;w8V6i%hZJ2G1Xw_e<7y139L
zwb<@j#e%EF9*x>loU->$b6hh!#H*31C2sYZRL339ot9?)?`Jv}ys0TQ$-}lms+oh~
zT;K}3u0vWDcURahvtM&qI*eO;&EcAjOlNsp^gUct{!Di?S+&UIRsGlI=06K}=SJ>n
z<ZBXLv`kN8`8N+`vn2DEtk?IdO+1mNR&?WS$)bBt`trooZ7SO|vwGINYMUu?<nCKG
z$%y{Dviv%IJV(zYZ8+vBe&&(mkK3UN4J-?M(!RcM{IbaBO23QeYc@XLb(5O7%z}z|
zy#DuaMbC&&>FHA1+Qjp;jb~@0jo5|SP0vGLq_14i!g%w7@lWPej%l_H4<&jO6Z@D#
zFHAps^>9NN>sE`od(5=m%vx6`9(C8siGF!naH5Lv&KJ2>7Y~XY@rnxk|IfBZj8VN`
zOlLad!3Vz4C&XMt9FAq3VA*!mi{)36_LrpXSJ_U*T`N{&k>27|FoEf;*zrOWuKzXc
zP9DeiF|G}jn-TRRM`>!i$d}_a=h|63C%CNP%3CTXbj#QG%F?SDuCvv+@^W3PZZ7U}
z;XBV68*+uox+VAGlf^AQtm$rQmkKj3iN-qZkulo2w&Ta7)*q9e=o}SuY0L;ZzTyqX
zRmOLjpE|PCW@by}<+KIgxTw~?G0UsyOw!W1NjvWzE4sLm_geCdMMvLgIo+6>eBsf0
zzSU_n0v*222vcy7O3V$rFtPaB#q0$<hIw7*LezV0@0F>!^L475`ZX}Ay2wq8^cCd$
zzxKwd+7B}i<Q#anD0|7n^qP0iKMD2ia(Z8_rN8;gaorWyS9fxMd#f5UNB2jee!5so
zSmv#;nv9RJt=ku@XSmU<pVOr9N$~QLtG`x;?!B<SdGFQVd*i#;z3AGPaqz{}zc(`u
z{ka;i>ely6qbu=fN14thrkjjXEJ7yBgY3Nih-A9tKIkb`*jyCG{^=N-5f_{CW47{w
zycNd+7b$GBda5>~C*<FRY}?0<1?=317fjnEcF8Qru^_xymfJfed>(I$#zEdY5>vfp
zPtQ7(9=1hWs9MmkX5}`QqgzETFImUon3I+_$76As#}c6<^;h3MD(KKZ^7hv)=O7u@
z^ulR|eFxQc%>Ew{;(op?YuTZ~UKIzWpu5YRZkVlo_OK{6=aOH$Z)fhiz8IFG_YB$$
zcI*CM%sQTP=IN!oJug=!-qd1{>VFYblp&DKl6!1p-}CKmZD(Fx$l9^4hgZ3@uD~(J
zD<I+Z@+TKF^a{A!U$0woq$Z$eih;m_z$5Pp965R1-o9)r`N|toIoWMPa@B`8L-YKw
z69TpOTC`mpwYD2Q&G0r!@77o0(B9y*V(CYTj7BjRMssgQDbX1j=Z>tI_*vkh)V>o_
zJ(MO|nF`LicwaYi&A-%~!y+3thQ714-<)~pi9>+&gf^Y6mIeYE+B*vi*rblU(o1m6
zo-$Q(%kP|4rrB*q_uBGKu>80Cb!6JtH5T2Uf5#o)o6D;3m4kDM*L+vb8!t{dzu<CZ
z=(wpeD{sN&29|}9(_&7fu66vQ;FPhpD#0~t<x=eo2F)vLcNDl9NxIxoV-Y(%$Eb2+
zTg}FfABQBwdxN8l&*dBv%V>)BaOAx_%d*{AkT*!$o&8F}mLGp|uTE7nb4|UL8N6a!
zT6D4Sqfdr6W(v2@eez`6<(VxPZ{FN$Tiur5b~66{=R0rRBy;q{m(HD?&^f!|SW<E0
zYzdY*4XgExUT^Z<mg(_)smm3$3q2f8agHpDH#@Pp>xI0tOLi_e$T(AgJNKY(&%GND
zBza%9$L;CWJo-(AG3tg+H$!ggE<gQz`QraVpXC~P<b#^79<5pNV{S)~i)o+>mv)Ym
znKz?(wXwzS(*m`|;mO}kT-){?T>tB=e)Prld#{A8eV8Bf@_ps(?J+VwDQ7hpPJd=d
zf5Z4+KGmVb&YNvQ(Cy<w7yqoua}rs}5&a=8@WaGs{inn#&bEjxy0YEzLQc*Tt*n{4
zrzafRGgsb8<7`B7YesU#l>!I9*OPRYmfim%d1xZ9cX7Dn3!^!LQg?gKrs#y$`mN`y
ztl(rkCzin|@&3A4@Wwv%SB^TR)ekE!vF`k3XDVr}`X}J4SV(DDwrRHI3`^_EJ(=v8
z+c(OGR?LXAbk>M?ke4O%n&aWUkII{#Iqdl=7yRgr!pZ;gt9SZw7_H{p6x+W@QGQxa
zA)Ca*;;ltF={`nD=4G=(K4mO9&K<U`*7yp?>`!x*s@~*9%O*yi)nlJ3p5y$YwSDu<
z*(W30x3jEX#~HG6O5PI@-jzNcD(_b@n)iHuuG@1^Y>IxBPoIhVX5Em_rjwac<4Q~1
z8%6F|SwxpN#%^CX`+Rvw@`l|XEEc<4p7s{15wV*7#N*zrP3t$Cs|Fs4V@*)u>D~4u
zVNZ(f@A*DLR;Fz%b>d#JKT@)foXTNn_g}?RAo28Um301>@1Y?Di#$r$SG%=)S<Fy7
za=vS2xr|Wm!mIA{kI5@WFx{BuE6iSdgeBx4?}Hc9SY}Q*F6mc&tLcB_FZWlEOnlD^
z9J_83JuRR?guOwfFX-`)eq+;M<8#6KtrO=LK8oBLol`nxb682s*OG?Pskd!-(vO$>
zRJx0ws`#+BIOFBCue!W_%btHYadE!s^F0ru>?Kw(d9K`L8l$}~CSS>m@Ak|!vYiK?
zY-kl!W>--E(zULwcAdiQHM`~}nVT)p{+6+Y^Ij?|15e5J@=cFjPf6#V%HHDkR>)-W
z_dM-6Q?KOg_&=qeiRFNkz`v-Pb`}Z4U`^4*nJ$)#csDxS`n-)R^zFCMJ*(YrFs;q8
zTkUe>{+)J~iHvrV9V{CT*&mo=-1^=`TD~~`N^$l<3773_AOBicU%dYNF3Zy#6&2P0
zw|DiwPqsYP!md!e{ZE^D)4|%x{w2Scn!lP+-f~gC$7}z=9pziOKiGVaXFK|2&hwpj
zX6xBAXJ)N&bMkhZc+BmT-b?>y{48Qec4xbCDo1B)bm&iF+V{EULwCEoI9J&WpB+pt
zlUSHUR3tJkBphsHnxY|aAfS<n(@RBT#)VVooWuE9Z)|$}s;{A6(X#5vj}MC*detMh
zo#~jQ>^V`F^~4g(&kN5^+vau8>g6fl)n-dih&(-%c)Cqfc!|N>A10hL+rQP^+#I?-
zY<Jj_69sp-JY6lK8>BKp)%#;kB&Vl}O~|2-Z_iHV^;gTh8I~8<#4pa}(eOS0o(iA*
zqpPpO9S^f`{b$n;o3lgV$x)`sIfk#dygYn{Up-^qrjP?H7v@`Qi|$Q*DZ;oiTw8QP
z$X7AGRsENqop`GF?#}i~?gvU=)$&#zs^w<T+url>@X^xZM~9Yne{Mgwu=vr5mEG^2
zGBxBVZk4U6`1ojlaY}KKh&JPM`E8b-tFE36TYGQcR_`Ahod2>gtGxIh-^lPG+q-E$
zkB)#NBZrNr66c+4&*gJ6CTSjOzmsut`P;n6mrHn9KWsS6pSfwVXmnKJ;vTaT8jsCy
zF1z8-#HliCqY_KzoQvx>dE89l@!fVIsas=-fDxzW4XsZe%GZN7?K|Wbwc@~`Urn3V
zPg}*3@71^NN%kH+ev=IfEc;#l2RLZ57zB4|F1exLYjQQ^@+OP;*~Q(K@~eXNuYP%w
zbYAnvpUeLC+aDwaJY8b7<+O{auLuL9RPL9P?F}9d-R)6<{ZrT@liu5U`v>LkeSanS
zsbdi9jw#B){+CLwx@IahPyHch)!*~uO=jl4tr<Z&iY+y+FBTqbx!S)dX@1s<8+)ge
zX4^NZXrx-s`!s2qYSzlToy)SnZen#=&>*yGiqVcLKC_E+jHW4d%4MHesP?$2dtvf?
zrPm&eZ?YyWC^xm9e1F=Psi*bo%T053);~|;h}-^v*CKJ+o1`C}$5h<JHXW1`He=j<
z&7?ct<o8pZEfcJb!x=6I8Jn9f4XFz}f3<7%e_`_(m4|!N-p*LObI09|&u=XrtO<Un
zXk=kIWwKElo7Br$Q_tPp{@kKV&w9nd?o(G+t+fknFgqojAj0VJ@QaGsE<e-P!4F?^
z<n}$A+_W)OUTDHn)}uZX@>rEaw<vvIvGzk2r=iHj<vcrCWpafAOOlU@{Y)utl1QJj
z@!0;n6wdv7WTajcvAjN;?Y%$al+o#Zhm~h6KJ`_p*7^!3)8wyAhhyHX|97Uwm+SNu
zn_%DD+5VHi-N<^*;cwkq{&V3p=Gb2UC%>26^nbdgdfu`pP49xL7+2Z9u?SaWWon$v
zFd;!W^7J|H%Vq|9rLMH^I5cb7+St~sd*?;3WYzA=vfMgV_eRox0i{ym*&0d=Tco{z
zHO*(5#@EPiIG0Ivg6+kptt#6(61F!9O)+Ze=JB|BKy1b%k^i*<4gVgWaFKGGaYSW;
zYT?22oS$0_(hPMHa)W0aEW2v(@d0a6o1^SutMbWmpYj7I2|hZiq&(rhYHE3^<z%6Y
zS0pDCvYDtj&&+f>ETz5cnCrfoy<(}?w%o0mn0X<)letxF%Y4oU89^)W6<+(X<=%&7
z+;LO<N@i`I!{YKVhyRLl<(^{`13PlVm;(5h-w~d_w&T6Z)*16f#5NtURr6BfoZ#h=
zSE$%qkfJJmC_wFC@{XV_3wd;un!C*m7$r_}6ir-nu~%%$bPe7M?eliuelEB2|MYWz
zH%!wlOwrr=WS8OJOA3XbW~7ybpSgQ-rQb=Jdx<^IH#snJ&1%*XDQr6|<nmEywh3cX
z+Nor_k0-TiUqxSCVdJ{3z2Q{N)7v8M5mQaW3c7gXjYV%9VAlS4u}gI8hU0(zjO737
zPE<SFHTw|DvIXTAkG8T_d(Ev_w03?7ckZ=ADXjLEdO}x@#bo|`@Ly??PSd0d3&M@(
zz7mlL5Hae|`<=K@+1=|xi`<)pA6^fO6qYTWeo83Lh&RlM|J8=`o@bI=S+oL#H&|O#
z@~refbLNDL*wM+Bp(fhVXT_FpIBIRE-s8JEGH+U&@$5pE-cz1^7dp(YEsRv}b$4FO
zy!S=u-Ty9YGXs9F;@6&f*1hd=PIJxG8%HAjHY;wLzwL(8F^%-_?KeZF%V-Gt+OQ~1
zoU?+rL;G;oHUlrdBPR+)JyW%EuXr7@Fw9$cd0|HSO7FvRY|}e~=Dv!sa=LDnX|uNV
zZBbnK?Q4IZR&UvT`Axxfwd7xVW`}~$hIU5V^vUq=ydZLAeoyd<tfq-;J-Ge9t`pz8
z`s>na6VF2uw-WuPakS2zc1fhVWs=>Vb?vpwotCe@wNv(o<SC1dtK5}7F1Md_FQrvu
zjZ9q1>|L5W5(|WF1GSwjBZW@1HG3;^bKDd?ByjG_R>jh2f9|LoZ(GhSuUweuyEEiM
z?7b9&+dISC5`P<n%TD=kV6umkH?(I)sq3>O$<q?Ew=7__$ZYIPO*}Po&JM|mp@k^{
zwOQF;wj8~canUTcWY*<<JD$vWnU?PQxT|#EWVa2gu2#t`TsJLJ$?esfw!W*DJWL15
zqjFBYvW}HzEx6=t_U(vQ?#ruTx@^`He$V)MsBR{Mc*qeYp^#9f<y%fH2)`M5a;wpy
z^){TID^o5vD~K(R*>$OX<H<yu@Z7Cxd%x(%yD!^!Vr6nrhS%&XOVpM<=_{9BaCwf!
zgz0JDJS#7%h@74C`^mh6-!FCDdS*89z@#{D#+(JaL=qxD&0YU>s`AVze#_{h9LGy6
zA!{tmRvim}`>{mT&Q_WA?~j}QMl5Fx{{LUSA;!oyT6W(wrQ($yE;sJxrP#X0ayW~7
z<X7Ez`^8*oXP$%;JD0(Zmg)sR*Ov3fdL_0*-a4Y!s@byj^oMf8GqI*~&Kzm!+Q`tL
z@8@8(Eb>ax%$tUyPn_l{o$j!?nx=JmwQ+P&_}o3A*-?^#E}w6D3Z!1kdZc5S$Gf@W
z)s@`6JN)k#89v$k*F*cutF7}3)wcb8HRaZ%W83mqg{bOGRaHJ$b8TXD)zjLy9GQRE
z{u;!rPpucP&Gecay?^`G++#YMZ}%CxTv>ikVpXHJbt)VGdv(cm##66<cjTSEbFoNr
z;X2LOS1*_Dy4{%^w<%kHU(Tc5&%E>FPip@xo!1rmJp1JTN@M?hn{;KXPv!a7L>tD(
z)b1|Q<y%yzzqYFSbZZ;aa@{2#Kd;+7J#2TnzWE_bro$`mmK+J+TyjB{Ir;HnJ;vyO
zx;azgck7nK9G83kvA;Q{c3LT$>7vNqZ=Q7`Gqzf-j;!SqU$NQbjpxiY38DYL+?fCL
zTk}qf!Xv$ZSOw-D4&Z&m-_w8X&`<Y$v((Soux?nfRLo<c0-N4{t24z0JcoU{i}$=+
zA{4+>zAFC+Ux!XkWE9tfcS^6gvRm^`PZwffn8sgG_T0zi?3G(C@4^hOhP{24_vq>`
zy`9stXK|co+p4@vEkJMEN<FT_wL1CMx3p^;&drTFWP7f8f>Y}}mX#q}{yz)q+o1X{
zzvrCWZLjxrHS0u+0`8jd=;!EPp4Zfvv50$qo<7T~r9o#(A~fEZ<bJdGurG!Gn^?!u
z9F3z@>l5A_RrE-2e;1H=>*(@-sXyKroG;7bb6?{rag0AzfB&tucWaLRoW}1ljsKTL
zU-GPuUn}0PT-5c%Y+}W<%nwu2{%p_^DHM3tvsmPG#$P|C)9akm)1H|v@G)0tZTP|Z
zrKG^-mwwK%XExiO%~tDI<ubVR<k6c422975w_3Ho30T3{9cYu+KkdvJ$@v$>cuzCU
zG;~imK3(NON7w4NF1h7iZ*^xmaeAHIUU9rEZ0(MSr1w4Qm3_y}8P4y_%KfnAgW@(F
z{k;Eg|L*Yl_~lVz)AG=L-7hLMZ|!;+{AGpqf)5<$+Zc{&OXQwV)OFYIm^gja#Vi@k
zZxQFc_N`Ma%&MQ3YN!;%SiM12V4_m<xvw$?%)VZN!Wm3ampUyTXzWN}p7dT@Y2r_v
zWTE1mE6ey(R`H+kVl&yKZ?@sYiDxMv?r~H(6<T(=ty%V1rNBV+<HSs%^MA`GD4)pT
zUXd#md$iW$VfH`%kW0+x@)j-hdSuP^NJaE%Lx7RZqC(~ELV6orq(oC4ju;$rb62(f
z@YTy}fe*)29VaG>1g5mdmR=3q5iEwzotv&bVx9a_`nJoJs%)lXg;#2p`-Qw^J?Q`b
zhaTJX=kL?jvMu~SiDw2+=9`UL4}-WGlI;XCUqmG7Tz$4Drua)i>gQW0ZEta&oguLI
z#A5Ni$GBJ<=I`R(v+YR2s)-I?mU2(Ntb6V(<L6$U>mQazcsD-zU^!Qq*>RHYM&WOA
zL3(e_++UD)l!GtvPl3w%VjIB>`+JM@PYWpfC#A$}QcVo9+qr;c*8-MP59~e*+kX$X
z|GJrnE9GY$i_WxuA?tO?GgheUUsU&XTeV8dY=V<skl*TK`6A(Y>--8u<5fkUYyH`E
zraNt-{)Z)E=K6Z6!V~|!w`R~hXR=hR@#4!C!Ao%=jH*VeCpYZx6J+Ww)H`@sclVKr
zUP3P3A=aAe45tL9CZ!mx?y}?mKiQM}jRCjLlrsnHP9I?T5X|6nz|Nb&+uOl=`4yHF
zqn~xp{a&3ix}ajT`=igc2BV2D%cpcon{TctNtT`b{C!a~TW7MIpuc=%lh(})f8Wn4
zKAW#cF>z`v{XFSh@V2yU2i6?1QnQ2F`qKjR3WOQ9E6=;<uUNQLR3h`?8aF1kr<P}y
zw3}_<o$hRPHuz!Nmi$X^zh0Z9;N$Uz;js6&wS2BUyoKl83<H#;kE;eJ+x?zw-~83?
zcbolA2gYRzGxs_DoDzCPt<Yh%|8WbZEi<~hR(<V~n_||(<J2$IqcSB^$W`mGsi-(B
z!;L<U))LDj?zdjNv|glbcW$eW{2>>MP}}DJCgJmgWlt{9WIL)`{UxkeB$m0Xe~B^w
z#o)_%PxPZVP2ni^6c+Y!erb2M!G325!;TVf?+1!L3z%Q|Gnjqz@;{VpZI*0#h-J$Z
zuaA>;8(RFb+_{f6#A~S<on5l&Rm$TDCs{9URGoElhaaCwI`3+({%31WuUjD$@Sf?@
zC2o$xuXnfc-u(0;n4vqg!7p^qS@G}sdsV*5?Ps3&Xr<__gJQQ7wXdWFtMi8b<o!0|
zQOmXiExS_q4hqKyZqnO)Sxoos!wgM<-?{qlx4Q9$DcxqWdpwz|E;NGuyNSQHCEs>#
zx5<?^-f2sf|J>)~sPVCICqt2ao2rwau<>=rm8VkgOi^?AztL%H*J`^d&o-P%|1i<{
zyvN#*@0#^(e?BdWY3#k(wYHX}jN`1E)~%P;S55|9IsE)e8h4k-B&7=u?^>FTirm(2
zw9O1rep?!*7{oX+FhT!<@^azE2Wk!?D?HeB6DNzP32yd0&EUFd%Yi!yUay6d^}~{N
z%aRShDOx!sYlo$Hep7tpQPp;}Y6*MN^err_vJ7^Lmx(NhNe*|A|0vk%!mwq5$C(r^
zH_s1M&u6^ht5Ex~L+i$lM<!-3zWCb-uZ~mpk8aBN9uo8Vqra(%KHFEn53kB1l`^B7
z_N09N<dkYwzS(<=k<BS)&dqFgVy_i{oSdJsMRL09MXd?${^@si80+f(Z?_8kq+I+&
zFL%oRV+CJ7C0AM=R<%?pNZ!Q!qPU<@wEp9pg3UgK*@sQeIVrL%vS)DUWoF>GBEfhi
zqIY{{Bx^`y{Sw_-Uj@w<YP)gzS3K{GS!UwU?704`Y0Tl@^L%>jKKAAv^8b8s?;;`1
zOeZn_ML|i$RiBfzt(I!L<(JRbwY$PRanCpL&+NK4CwCoQta-<1YgkbKAJNJ{QN53A
zORg0QR+pM3Y%-X=g<I}hit0A6i`VTQZs#_+#-bJGeSiCm$KR9PukAXt<AA|5SDwkt
z8V{yL7hPatnf>3HWxFH;Q$mBQK;`MJMhz~HSwemaMd-XaW~6bwF*9t2R4_x-S^32O
z6O5DJ&YaiT9NZ_jvqwu((#FTQC0~gvW?Ol!Ap`%d-RDjQ-)IQ5?B~!Ao#Vk8EM=N=
z%sKn~W3z|OQ)+Jnv2}0fdF8(PM(eFZDGuJD+m$<iet7w1Y1r2=rL~{ccIZjoo|K;*
zrT5@-`1$SK-!~UzuiQ1K(zT~4;FLteg&X#N4$q#vYxYBlzS}nfWm1DGKB!2Y54>XW
zQ?A@W`nu1YQsEQ*JPO<2>V-S9TBb%ko7SBDKFUxtyl?fKoSrNB^N%vGjd;}aMQQh$
zmiaoC3m5Xf(W!j-U}b+rcx~3h^5^Q@t=HXSJ*P4;hw^{8cS|^Yu6W)|!`D;FyS`7G
zn!QC&IVnZw|Do!Y3Ei8O4Ud~oDVyoF_llkUwWReT$>wG$&eKw&ch4|3vwZVi@xykW
zH<LYohZ|TXF#a<#Vo~Yc8_CjHy>jQzb_sz?QtuTQ98!5BejZaWGEFX<@_zNt9lKUX
zSA5cwy?(LjuU_T7iIpEpCbmxV-@Rm__vStFOs!jw`0VO-`J3f(Tse!CGdpw9-bZit
zNk(Q124%~>3NcGmJfou5bGb<FP5gbae1FMlk}|W!Jd{c}OCwelm4&9Pvrub{w6Fh}
z^1gW6^ux8HsizNBn79Wwi|{n4DX(O9Sb2R%@9vu`cP<VTYGBWdVO0OAr5br$o{906
z1sBVLzUDb;s^yXUXB9pC|77=a)08z+fA2WIYRlGcft}IUr8C`b%<}nhO7^DnZg;WY
zA7duIofH{S`|bCONoy5cf2#Pj@Oa(a(lA56JS1OoPUZaXjb{Yh_WLdMnQ8m^tfJ7h
z&H0jB%71$nODFlxPWGQYqgm41Tdels^^22#syNwP`e`}k)x`R2XM@J*9n35%`+v{2
z{ku}!V75WeDn<n+xkW)tTT<&)=Jr1Exqb7klEgeyVVgbGl{LyHxd!^bcD8I!KC=4K
z%<m?73hT|EAI`eQT|Y}V)M(dy9mz!#Cj{#5;yv)%ao^$WgJSzw9kf~IaK~r2URe~}
zctUBVc-8jc{-9lljvxLOWOUyCb-t#8TH=49+H$D~e&<VTkIj56yLj1x6@pDir_Wl+
zZ1*6uraI-1N=3n>rL)w$Vyi741u`&jZejSOkl&|mUUd3V;r=Rz@2*kSa!(rGtPEq*
zRM?U_Z&P#rv(+A#LxZNPJ*!$bSLm6uQSzf(d+N?TD-95zpS$PY<MQes8-#D`w%uB%
zuxHapoz1d)UVY-7>h$>h)GG!Dr`I@hdM}mRdh>eH;}u(7x79wM{yH-Hbj*X4uY$EZ
z7cj55IV)ImHtUTY*XtTOe=90Hy*Op|@#30y@s+C{#^_1;zrSCnz){70OyTPB1wVBb
z2i-cfD?Kd!vdQk|?%mU0?%pfi{I+Dz(c6_P?#z#7(z?CA_tt;?H5GgHR34m8pP^<~
zxxQLhAv1WPXz6;-FLRG1PgV@wyDXN$=e@c|;|3KT#$b&o{o7MNZi^56#iSl{C*0$K
zg|&&n9Og;(hhN%8{7(B=W_x-`yXRBmgBP|b_E>Rr{64z3viISP-oiQ7QBA94=lwap
z`1nzKn*;yV&c!oS#q=7k^P3`iV)pqwD?Xid);hl;PX3=^(jcUu#&N?a^^gc7E5F~v
zf&d0*#uy8Z28)eKE_%X-X<JTgTztG=!MSTqhl7KsiqMiRS*Dwoo}O-y+&9Z3Z5e|Z
zQv_#*a>EpcX)?8SIj+x=nHb$g3v8kSotTz0P4P>S+PWxYNyN&iJ-bSqF8yC0svak+
zJt@aygY4O^wN;@Dq8ZsNeG)v{ygBt!Pwmi5Zd)HOTzQ&7EZ~sKbfFIZ@);RPh91K5
z+}sR2JJ&btX@8qFP5b)la8^c#W<P<35+;V_!Hd}x8cMIt*kC>F%?i_RZ<Z=~pL6R?
zS-_yXZ)?v@BezpWbeVeO?>n{qba@=K%|+fqc4z5*eNm-5cCFsu-^XmL;P<yHySPZ~
zJh#LCeM-N-%bbpxryp-u-R-%J;qS@)_Q(GUf4m|ez)+{6;BF&&BiP-d^-7!H#;q5*
z{GU(UGzoMwe4w@PRwpke&j=MqZ)LZoS2}`XRb4w{jxKZX5{g-R#6$Jal50^GZK)Ig
zM|1l;cpM{Gc=2Fj>Xd*d6W!D*pVVwLjl7@gTBxYX9-R|#q038SLd&#etcg(!32F>1
z3=Av|YBQ6W8W{wHcV3${+c)Zo+HA#xlF#RsZM&(Kbu3q@VSYp2M$Up`y)z~)=*Zi2
zbkV8SFVh_JR8y8ZvP5=Xa6VxgBIq_JPfJK$rjhHFn{e-~Np7MuvnH-kwr!o$WLc>a
zvPRfZ;_@1?hfx#P?mWbzExq{0s^Dhpr>hpa3kQ{o2AfV(4RBYzrsl9&MJ@H}TxCtw
z3*np@s~fi46)lUDtavCI5nuh4b;iUt#cMNmw@*DVV^8-<Yr(xOceV4yMXx?TG`S_U
zES#PD?9zt+>37@0qB2*{O-sx2*r-;R9USU%HCKG{6T#3+$4YMJ%GdVfPEPSkxGQl$
zP<>8Gki&GB(`QZFCzf3D6`U)=e)WxXk#OdzV)t2WXXlnP_o~O0zwW$#&;3JT%Pa3!
z7oXg_@=bE<7mp1|u@~$1<mFCm*yhLau+j0Ym7w2|J{LC4oHZeV(m9(${N1<x5Lwi%
zR;O~XM<H<b+P=EtkmLtev%Wrl=ung(KFNKZ_Kitj9j{G%qI^<)(o>&v?oCr-A}lqh
z#i(6;dN_Ge*e-_U(JU=9nC`4yenNEj6m^dc!Ec04Y)kCkJuknF<HdZ@Tq8GE-ToOG
zi+a?Szua69%JR~ImD51;{}(pJl+0xUwjzQ>w}VZJeEgHWpBFEYS^fI^i{<x){MS6t
zO86-%ds{J}ruj&rx7XeKkKBKYpS!kI@nYAuS<k2K)Jch{c<B`PYm)fk6|V~?7&<2I
zQaD&?c+60AP7j-EyCrYjF{9}lWp=V1V?6icMZ$fr8yOPIW{I8S(|)!2#JdEK!@o}+
zQ?Z!i@wVi!U{CIuBnOxHw^~h7j6Bafju<Xlv`o2GMWWxL+w0!hNzR*R$3E}c5Xf=y
z0JB_&#@gVyuM+l0HCg^}d7I^56%@E?n~3IyM&~%@lXW}NCjOnUHvDhmQCa3MlbIqt
z`ty@ny-%>6i1zGo5>{+&VVR)Vy=8Hp(EozD?!U}Ey_f`Q69T(KxFqzrG}bY92Mg_%
zTfXE(MN5&gxAOC4Qzmd~B<3&tJ!vKrLrdf`&eB||`Ln7lb{h&B`ft9e>K>;t>Eqs%
z@?$KXM#&P(jXfWhU+}r-qNK^_8fn}WoFn2c=jy<u)}qKivuUZ=B;)%FUd@@;<om;C
z>8g;W#vx1z%dK^V8oP21sy~11aec;2|GIa<Dr+}*`fu7cN%v@9!K>Hzw_kdsyL!jd
zgo2-f7RQ%56icpbj13T<c<U--pP3s&cOj35_N9xRC0F9a7<;(Yg4o0Z0upAuY*2q*
zbm+WBPOILTmq+%cJNIYyn90qY%O9O|M)KW<<(5kx{^zlGI&<TuUJ?h##8YgS<~*FR
z(8^DSL!@1O?vnt)^edXn&%d!0TA2B9&73Tcii|^>e-|8C8`Q1sF{{jUSCa};)Q!dK
zlNL?3%zNwj=c1@Li}nJ}(i2J<!kingxv_0$)yQL-F`MB@dUu#hQOdMAX~DCSrnUNI
zPwx1<<dKNF*RkG99?RZvcm;9^pSjx3^F%85`MNFJ_o^<4UKBk;*VA3?ieiJQhd|`9
zIZ~Sx9=WxDTP$n*ROBj$izD}}hIbNfhHV-r3RF&qJo8R_E;Zq4rw`|ko9Ed!aq=b{
z@4OxoB4TbJyWZ;kUZ$H2HPzqNv?wg8dNQZuo5~~G<+|co`j%W-pZ*80miYI1zr1CE
zlK2_HIa}Uu5N(v-6t&Q|PoreghKI|dRRq?x7Cg{<qSeeKub#xt<z}eb5#A-J%)nK)
zg;`_j91l)q{htd=dW9Juawt0mpVvrWyua(^p^{azmMiTDJ^fZQz`ba;F2|?-H3{yz
z5r%A558r9HKU{dVW43hf$AHtGd)GBYK3=|7C|x*3L8M4&=Zc3pjREmrwazU$aPM)#
z^K*t5`{wn!ZTZ*6uN?SEWd9-eRfeZR*Ja5lOTT=%obl>J)#4KA2C<b7n9hFe+rZSN
zP`=0`W@BPlYtKo={6k!GJXVNJs{NA6xIOZ$o2uJ%6B$Oq<Ba}nE4O$pIy1@U(5?Rx
ztBke@W~#n$nI3)lN>OR5QEdK8m1U+2^L8*?C@P=)>#4zNKfn7^g^ThvoD=<2cE3pO
zon<1J%D7>1*qel-8nbHODjS#XKFrckx=?b@e97k;$sYNluVWtOmIWWa$h_RiV5_ms
z@1JwU6t~oyFZ$qAzGUv2B+)RoI}zpWE(dw!Bm)Yb3irHrSs=EmL-*13t|*U(F6VA9
zIko-L9$Cx&WlMY)bcnwYH_kDMd;dy<^`q;0o=F#%PX8%BIZP&a#^Mjgj$&sg1Rg%r
zCi-!u@75izRkt=i+O=f4anG;PQVs@9j#jbN3tAa64swOLFqS*NUFhDH5i@D6ZPG(u
z<Iu@B=1h_Nug)2DBkSJO%b_n`<m@RI%a7C+GFc?6tDKdqz3^ePU{NmD+@eta*te_9
z8;^OezhR`)Tx0ZdqpX-y=Iqzecf&;ZMOm-Pu5{Hl3z$9iFzXh(MGfutl3vFw&lRrx
zs^A{*Q%I)0qMB#sfekxDW+}9A`Ye0Ac8CAtkZY{sD-XI@^S$35GMOdZckfBX9f^;Y
zt+KCwb>Vuf`=Yrmzvo!1zhU5;p__MQdw+cC-nkx&|F>x9nw+Y*-jgIc^}EcopZk?Q
zT{XT~9M5IwH|>!CyP+|=dg-gn(i{^Q0tJd#uL=GYNmL4K<ar^lKha-ok;2rgTsEII
zl$na3G9|SeCbdlz*EVzfuezXE{%ewvnB(NYpvl3>Ka&I*ip8x?FtQw$be~@0lqTh~
zJXrFF?bWba#*mhG*M)X9O3gM%6=p0;3FrR2yzIgBvIoag@1(U}X$ukk9Q`9L?Pr<G
zm5r>CS7icSWDC<=Yr+c})P<_tD(-BQjaZ`jp2bas(Jg3FJBPVx5VJx4({|B^E)pyJ
zrCwyDDZ3XwkuNi`7u%TGaa7*oV#kKWtk;PVxtF<CZd24g$Qbm&)1@i9_ppU)N3y_1
z%dXFs8(eZWZIN1XxF$!aTXdmXSewYE4rkTCQrDx_cehC&U7C9ONsgxgYoG#apaA1d
zbt8`x-4Df$c6WH+G*vfzD8<0Q@;{ovz4QoQih}2kBMo<0rBj3$Lo)<_rpbJ}ZgsW9
zCg(=E^1{S|i(dDq#NRR$;tKCq`k2lw;s4=F!7mFR&ILk|!s;>^jlvv43*08?9@Lg=
zDhzkl7ThMUY%VC3#2szGm~w%w?xX&jrs9uGLAn+}ima1VN}DGg*8goJFR3d3XOg7A
z#b&l<121(cKlc{@<t;oN3?2-zixq5U8MFlkl;(bSzAYvkay@msaBkp(@CjyIU$>OT
zyqFRoIpsxjYD$ohjj_zP7P+8=`kZMlu}j->GK}l5$2Gc#$tjxHCQmEcQIL5v{`Nz`
z3Jv2jH`CIq?Xn+@gjRHPT8QXP%;bI){$HTXY`(Yxm!$gSi0NAs_+=k*t$onB^SR)k
zM?&jVojz@}Tq9Dmeu`>zP|mUC(Wyb+kGAFbJj~tvvDV~?THtXhhn1z*j!C`Up7KbI
z!7@|$fFaw(tCMFLbiZnDJ;pj^`{P!ZEmj$3T+s_Sk7mmFN~YZu?UfVkdH=1>Uo!u{
zh2UXkn@`7$b(=*FEsSeA6qmou=EBE%;e$5z%JEE&0)>h3kFE&tKNM&X7nEhJ^u1Id
z#~7Y(F0J{{Y?qRb_>SqE--LKNbz_CA_?I=x{t(=DfK|9bP$ETNG(=z7RB&QuhL2K`
zm4*tdGox*jli^7xU-!vol8YGAT=qNWEeR-lWybYCFk!Zb+M@lOA<G%tPF%~m$TsK1
z(X#j%G0`21ud#*f7nKbyO*`z&*0rcDsMYm`X1Zyy@l%zG8$#_*zX;BqR3N@k@QI>a
ze9O{oQ`2;pa0$bOKNJ~-zRj;~)e#WOoUqN5CCH31a~a=D`G!=vKY=3rmm=0C@P9J4
z4;K}#T$thLq&okx*L|m&cqgeP#Vby4x9ZYTi`ZUfS`{0vHdX9!j@vhH^IJXrt8#T2
z)i-CxKAo1gAS(9g##ZN_v(ygGx@cy%Lee>-NNwMe($kV5XDwG>@C<QeWmp&7+7u*m
zqH#`CtH#BH8nx;*@2(2|XD^r_)c^3yG{ctu;9dO}qO_DIGW_QdU}Tsn9PmSsy;A*l
zl$l~+N8YQoW`zo^q5-8(1cg^Fh<U)K^Hsk#u~<whShA|bafN}Cgqq)VsWaPG>D^j?
zgNea+=9JXw!mk_FE=i~lomO^wMyu_q7G<j`o;PZEPcJ@Q5h7r{)?355#%M{&ahXHG
zZGJ2=sh&38QJWG<Ws5C0DK<{i;x>^D6z^!S$Shr&GgYqXMOb6m^rTf)|4#bMFgLOi
zG*fnFbL^H|Q&3g$Oa5&^XtuLnDdUPQQ&o4ES{?mv5I3o2V`lf_ms0zVbRWLjx@oat
z<TWAL&9j6j){5`0OV}OzBs@0XLh$9Z*|Qr`AC+&M6}A0h=62rS@glP~{x`ej>~KLx
ze)sC$YoYsuU0r{r-TNuStH}E3X3<>}+w;OTg&hsQp03ybEtC_qR?s3M?X!Puv;QWe
zU92a4eot98QDa8`jt=If3c{XThL1%3E-q}fYH@wyG_`BzuGx*B%rtgt>Lj_BT{Y#p
zvtu{^>)@?{xf>qNu9mLLiZ<kIU=f@UYVdiZOP5^6H<|1-7n|lK7Z~TMdWS`bNT->$
zYrKvRKBXBEWfHk7%uCrc_Lf|B$n-?3W%?JF3A1G~eDY@%+QP@#`Tt|3{cD9{t}5+7
ze!)ohTm`EY4^_LhtnPoydF1AOfn(!Viz#Q0TYoHfmJgo1OHydp66qyB>((6<QLUN(
zKTgeO<|^mg(|6mpupGbXzeXUoID3j<j`S6d(mz~6mrnGoec1i>TCeMi*iDYz3Pnf!
zt{6qnEdS-UNWM{H|3xFeEgE-rYE*BpzkfUb*el<Oljc3XGVf(_f2-QDzY)hiO$iYR
zFS5V5*8Q7*tHgrN6$)mJ!gIba>sz{vgKPI{#!{Cn$B+F!erC%CY2ySY0oG^(uB{5G
z&&%>m*3{LX-mv@U36HN^l`q!TpPubkD!q<#j>KaZ{m(J4HDoHEZwzB~4Gk5%>RGPR
z)pjMZXaC2!jnj`lI9VTGxzF#_rX4v2Uo|U&7KQDbBl!Au*dd<%(=Uect~9S>n$c0Q
z?77s84F~6YdFcGNaNJ@USom7e!!z3b?FFuDrgj;#o$uSs^6NaQr*lGa;YoYfZsQ|+
zPH70u_F3gLbNiOXYC#FJqZo?p3eTsV@JahUCnn=u_UzIFB0Xn*9=h0l#6U>)PNwfq
zH4R=~->aNLg0>e|#4MRrsu7YS+s!oX&5ym=nzkvT@yr?Vp_7c?daCBV6nrZp^vQF6
zp^(yc*96WtYvnGQS}E>w<=Dk`a)N-+u5z2mkB7OGxK9LEZi_O|IO~1IcaqdK(b(vn
zQzs<|-(Pcu!)^QbVCl`u=jP?~d~-KmY}4lCb;{}05w|I`7nQqBd@gsly1X)7@ZRl<
zmk#D{iz#?w-Cp)W^Zcs<sYcuX2lp(MZruFowp^-<wARMX!lcYJXLn(Z%-v3Azq)rd
zdU{w2R;{cS4!+Fg9<7t3;K3nd|9P=*)JLb1-6=L}&RHy(Rj^4d^t)5pr~YIGkH<F-
zi6pBZQ#n@`y()K;we{2E<=Te=RX1IEsy?ZqXV=@q&ug~tQLf$nwnu;0wjC+z%)Lj>
zO%>ApoOXP1{xue15m}+bm+z*$y?bS)i;I_yCZkQ|^=p?dUfs0j=)bl#91?!&f49AG
zJ{BS+q#Y}GnOo_5NF?_q!PHxVr(#-iWRBZ%2e8-)8tXD_Q@9e6dzi<2?fI6Q!ERi^
zcOQ8B3h_3x_=>Q4OgQKgyY1tel={;mvj2?!>nFP|$_dR-yXIoGL`CXi@l2P8wWnMU
zZWOV-*LcE2cH-WXj54a1_NHd|N8Tyly2f|=9N9Bk_n)x{?XRiuyO)0BqRh<dj22(E
zUXug=Y9AOj?w|R6^OWsojjqCCpFCtYo_%w+n_;fH0MB{xbAmq=J)VmPoUz_^=Jc~u
zU-z7yawv4=xlO$Z6QwqK9zA#2J?HZ53+;z495SBsGfX)7$0L2l$17xBK74z30^dEE
zz`0jcUy4n9;kirc#Kavh*0SE)sgw}9M!|O8`>E5OTVA>@e4pdmrB{b_q7H1nb8GW9
zBW>omUDce&zyDq)T%lm^8>x`-&{v6-`FF>f2`mi0Z&GI8K2xXuKbrTHNpYIt=113G
zZ@l{NY2(~$nYNXuQ^U_R-V5qUf4uvi*Q++=;_ExtG<eP1l(}<XUhg~J%rh-)N=c{g
zzgT>Rab8E2=<+{-9!swkW=9E5@%`{s)5GDQaKUA+lzGc1n{Kas^l<&dkX4n3gCBaY
zmc2E7)%M5JS=P(Wkef7TV{&RO-${j@RIMKCt0#K)rX0K|o@Tpd`_=Yybv0_oYSlg-
z&siP(=$jntmAzAsW%~Bm&AC4Nh3UH&?)sJb^Ip39U!3Omaw6Z?hqf>8+X_xwe)qNG
zoc)Hnmy={pG~P=+(YEZL;JY7U^PK%Y<X(E4_i&ZvUe}2a7-fWX4!-{D*~sy8e)rrP
z|Cc-v=KJ>k?4R#qh7%9}dt=2_Ua_5%b(Zj*n$i}3XM@1!A7!TQm@9qxw#(IASDn7J
zS@B;Uul;%Ov-;fgKWE$jT5vzZLeA#K>$f5Mem&0k^)B((TKh`r+#S(31s^>v-x2@h
zxZ||!^MX37-{nN!FY2%rm}p#^Ha}>diD~s2VMVLfy=OKaQvCDURA$nL)zuvv4?R7{
zbGH6!f|8?R=^aJx`!}N(E3SQ@e(Z_b#8b2Ejx1iHd*Z=VHHA+;wMV&5IewC<zbzK{
z@aO$!42lYk3{9-OaxOC_C@{40N~<|cQs{Ky4ARQFxG8nATc5Of??;YF4<r~>j`t{O
zGAv~<(vF@~WBKa8s`nJ5>|;4IFF*B~su?>?heJ5Szsbg3)THZ*dQg+@Ofg?g2j!W*
zEL%AyI20%!)Qw(}cT*v>i$OE`nAgln1x^RIupC}@*C2H*!%mjV+nzRDU(LW*ZK?9a
zX~N`&19O<Hbt)tbosP1y^NGxGIHVT6R!m+c)WG=Q*%|zscFEjQ$T-KLD{K=x;q1Z7
zOKywQtleNFd2ws*ZMO{)4onK1W$H6$J^5Df=xDF-`C5Sv2Tso=E*FEg<s8!uIT>=l
z<gQiOtu2eC-kb}nsd)W}J1xC)pN;0%7oM`d_qQnI{^WQme>`5G+DkKPQi|(|)q6Rt
zx^<YjMG~4fElK*I-^{jWjqa}h0>Tpv*raD)(Ay?a6JXEH|2V1pfYXr+h7VXa996vF
zA>HTN6qBR6^&(5bMg_^-kc_Rr^^{fJ%&cB-oGaqH+DoyG#c9%l<C9V@Ex0pDnBi1Q
zUrdI%7VC~p3)E%1H<>F4yjXkk>5h`3Ge)vaKBrrk^lUnNf$bEpkum?NOP;1TRbHHY
zS~Yvc`7W(nPLumgm{wkx(8RRr&_q5XPL0W7RxefNs%`BI^42~YqCRzw6_?%AH^C=@
z1LyzRq7me7DjGOp#jaBa{;zt~<;b|yIJ=AKxKoa9ENeq{*M=8d+UxhEsGd0Zr!SUg
z6N^BH1D8?M%GRkF+^aqwZCT~o<C<d~&2ziuf1V4+u}cBU9w(Zz9CO`MuAe>cWplIj
zyP3xQPz~R%Ig3q|b&oyh+x=$KV?XUuR*g+7mEJGrw`~^h>tD94?-ZZia;1ABNq*{O
zTA@vjPj~ESlwQ<z==T|>4T_t&4t-78a>UC!kS~6luhUGy!y9z<qRxF-uy<WHms69|
zpPdW;Z^>9Zx9yfEZwzCSi^jG;bK-U>ibYrYX11R=(UWzfNoLa4Rfj|vI-4d`Oh2Xb
zw$wOR<8{vD?G`6pI&zvNoSGJz9QO0bQMteLx}x!OndpVO_v@<PZ4tjT$-h&{L#at)
zNyDBs6YNg)iM$TzideD2MEw2kbAn&{)Mk`P#a^22cm8D2%>N<c4y9{?+iSnxcsV)K
z?)K~A8#Z?jPOh-?7yc!&XOhx4t|HDw6CC0hSyDe}Z59c+8OF>#t2>;(dhTky1kppQ
ze<ytpdb8o60gwH{gECg}4PW?$`xbmt)mm+Iq+!ndC@JYj-!`{=l0WzI>Z-M2+IzN|
z7%}L@vAZNNiE1<)`>W1*(a<}5&FA&s16pmxO8gJH2=6HsJ=47Wk<apbizf;8EoxC0
zS-9L}heJhdx=z}}rsAll0vvZwg$b&Nh4X1BHs48gGQK<^yufS$qvkRvJLk*Ho{f59
zt32u-cFjs%a>67mB)5Sx@@)SMwk2URycs;K!_(HasB8W8_4HsmkS7qd{Qvg5ybLKK
zbDVEXn{eoCX59A5<SF-lEVX&|(%WZO^c1B_>c?gIizHrhywTZy;<}+x{mhf|R}?u3
z-?C**+<N-PQI)opU7>G+Hm{!Y+Ucf;>ps58W;a7tow&TRy|!!WvRf(Ykw3kb9eA#?
zMCxAcx=Esa6(_XMO>Syh(yrb9CM!JuA$Q!5pb3ebfm*d61BAPU)sN*iifm)x-LmGW
zy_%Nq>?d7OOzSHaUp>R2_{Vu-rpHCQ%x#zF{`6#cEb(3DgsXt{5(QRmpBer&&x+@-
zd7+;9VW+>)q4ww%o6d4=Es|?%Tb97lnxPxJQpl`i)glp7MJK_-jw!VThVlpbPW=fs
zl6v+3dDvE=ZobYPdJ1MoIxG)IUa&eMq;f4yDM-TEdd3Tv#ga#|;(cx<)k*nf3yQkC
zh4Oyzy4+P1)s}7hRa0};<n2uo2L1~rgB9La2<%nN^G-ZG`OXfmWo<4k3Q4zvo!0zv
zXViJ_5zxfJEZ)B7AIB@@P0>&KH?Re)c_})VYvKJJQOhNlr99JU%Z*j5zrL|&s#b+y
z=+ngH=@BQxu1%S8ROfY3xYoIM@5;7D>J(N?l8cFCsoa03Tg*g3RQJOrrHo45Q>)Y$
zPTjRsFZzO9M9GRX8EVIzte!M&tT{cSS!9_@$IVIilmupWec_n>U0Z9@zbtz(-*VS)
z$!E83d2KIUx!UOe+JxuUlT{nnb(LG}EIe1A!Fp-hgUzD0P4BCpO5c|g{JM)pQ$3yG
zmQ_RCcXx)p8k(m(PxmMtTg)Dq;ViT#p>x@W1^jviZrWXk*!8a{a<1cTn;1H`>CGiG
zE#>aD3#L1N&RWrK@b;}&cGjcQ0Y@Ug-q2+7JZ#cFhb?PaVlTtTZdaYOwY*C*n6+6X
z7K<#*UjBaSfufeh$Bg4bEi-?uay%pDZgtn!DtK2`%wxCE);F1Pk)qOyFK-EYa^5)O
zyiQ5`hC+&|ey>U77T!6ZXWxH#fq!me?o&sBZ#R#=v9$U0vM)ul^ZF5|NmF`c_+3s0
zPq8iDR@PVhWihW%=Fw$EfnR_A7e6)2IcTn6cgpsl6YKesd|tOc4G;d|9Y5=&;=^eY
zhK)9HB}T?ewByoNY+D(amem`(a^~dvfUlYrvAxUgx+HQ37N~EQ;W)S{Bwd7I?)wSV
zDo-*$OP#-U^nCTfom*<$c;_>TzL;bB>P1V-w#_?O79LBhyjUupdT4RN_uV&p^I8mU
zSV(R2J+!WP>I<K#PHJ1bOY(Lo$co9Ny<X(x6n5wK%vn7T`2zDlomdcDyiL*c2GjMm
zwNiK6?)->axp1>{&7Q4#x0r6M$YgC;^J@D6%O$JBboKKOtUC5|($_U#tF}&h9wn%G
zW9vjirm71Q#TGBD|5!KeWZHzxEoo9mmz}k~_hZ}th1NHg=%t-oI8$%Ji^En&W-Wgm
zZSV13=cMHFg1v8|?RPI^oqJPOGb(wqU-^yGchh&Q{<N;TF=o;>C5E1gTS_IJZzETQ
z-=3~0JoEPCt`){CzWW+Zd_4Tr>1PwS#A7Zu#kFZ3&jgM9Us_DJ*uAXk#m3OsznR{O
z=DQxHEl^wAv?RmB(5Z~+aQ6(WpK22pIqE-VP~AB#!*Ji)6+XHR_B9X1<J~gc_DzUi
zQkc2ySmrIg-Uim)_RX_)zk9yTpy~NMEiQM7*7t1&t*`H1c%)+!7gG5!LgjP5Yh#t8
zd(F(By@#ETY}w+hmu>NPa_iLVm(-52e^HZ}xG39lRY6tngu}8QKKwU~l{R{?x?TU)
z%2Nyi8(*%{{48zIX{5s`t^2umQ`98Yjh5=h91lAzPR`_F6gFD)dQp^v)Z8pbF_T@3
zt@bUDa#*>0{_8^rSDaqAEYL14yJPig<MT7@4hdPdW|&M+w4SzM&I|1=YB5Wks%7n7
z&$XB4U%JU;`|d+Kl1=XJ<oDF<=$g<G(P^I~+1+I^!B~gm=;C=n$%nTJi0#fWoh7-B
zpSkC8fSB{7KEoYqOHWG4tehB?Y?o4~#mKQUXVJuq${p(}_l7B`RcxGed&lfM9Gjo5
zP@mH!k?~qE<KmH14~{(K-uHMzPX?#!Q{kf*Gk9MJ_hv^=|GrCKevV#gx3%b{LvNM;
zn=@%^R8HOyqhcOWvX#Yn{{%tq|2y{j&7Rt@daAbdj#-oTw_VtOzi7t=4W-uV9V(Z$
zY&yZyHQTDi!t%6*<z?Yj`8p1_J9;PXJ~+wRl2Ksa-V4&zk(Mn7CYPyj7{3%|E1Khc
zs^io_sqWW{BoZC#Y7WjzT2$O~uvx$(TH9f@<E|Ayom)RxIOT9iUYz#-q2Y?%yrDXW
z6!*+k_0ichd5y}Q6a6(#n^&)iSYrOr)1kO|4a=W3wzIp!I{ALwwDye9-M3oc`l^n^
zRdUS%&f7Q0-8nVyO{C4OUo$gwEci4g9$R6<Z^$FCaIe|LJ<Sz8EHe~2K5;#GyhHYj
z<3$GB!Zo~CjsGvtU)kywqM&lYwr1n<X${*%-W=I_=SaR4hpUhJb?@aaIeT}Na40(;
zFFxt594#0aGf|QIc(IaG#YQQQs)=VAdMDiG5$2jYX^&lV#R`E{OW!T_srYQiF1`QE
zm3`H>4H8ti*Gn8@FzlGjWGWQxHf@qyv$D;f=H)k(JGveCCnryL5IL~skC(Phhx|qN
zj~%?HD;$~&4(guJY7yw0zFJF_)0l0M`>sWOr!FdqmK+rQ<IZtrophq(9u*FWGpGMB
zY+GX4yI{5B0`4=nk9Zx)=v}GYd*qMfqYWO4CI0=1hbGAA{50`79CPxwjngg8Er0Jg
zxnDkid*YJcF(O}XcdY)=zc=8&;lAVv_MCezD44$SJe(3`+N?4Eg2lSWLcX7!r4I`>
za5ywXNc)~RILk%Fc8!Z`>y1;3mHXxwP5h#AzIjFKWfecKB~z<^**;G8e)#*yZXb>v
zOT1e?T-+|xH&^M>qFG`kBGcE@xIX64`XlMn<I;7;;lObhp4W?AE1H*n?=X+u;-kCO
zwPnL4mP6*OlSO}Zh;}x(O|k55eB3of@^D4wv0D*m>o@oQcj!2AvB5WjS3>86`l5|$
zZ!K;F%(^JNK(jQ6v4VH`YbC}C2MOjAVGR>1zf8WaH05gK=GvBN3tu09amO!b50CO^
zr3Mip31`8euYn1Zy<30Sx4c+ZcP6C%Pw*n9{~_%a`_4ND{x%G0x4H88fbm*ko!&22
zTRwaAs)TKk_6*&%@4~Mc#}~P(9GUAqt1FOUg0F)9N|y^WMH;TIS~$CAZJNkdMnjvo
zg4cIWxM+DK;L#+_Lk#oV7EcyymKU;CnPbJ}E3!{a(IMOMh~&p(xy*-}e{9T-yn5|6
zOI4N`_pXcEQoQ}<oPPL1Dn-y^Mb(X_7nfT!g7)-mY%MXCE;%3ObFquZ_Q;h`gV-}i
zBZ785wqxIY=7V$O$J=LmFP<?B?I_yY`=&y1v%}@n8T*Z01j=kJSU)X1IwSa^O5beN
z3Hmj<oHv+kRXlEUILNjva^U5?b-{O8<+R_1Qq>+;i!vQvKKXy^#g~vP3?6&W_;rci
zej9Vc<&Ur1Qt#*$?AuCj2c<@Dbv=H64eKu5+yAA76b+@sTO2xg)!+Z|lX6}q&U(Ch
zub15EtKa|Jh)s2BesT3zr~jc<=YJm2(K~Wh)^di+<MSa|Iy!&mrfyX|Re17Q0}n@|
zJ%4h5vB+r-SB}HUhZ7u4Exsr&^a<y(<(s*6GGFTCidnWRJ8b{2I3lRZ`}3h0)0-Q~
znvoNDdt0`ik~fWR=D76YZQ#X>h>DN5A8TCNXnRT7G)Cek&n=$bg{Q2}2^bvf5uYN`
z8-6QDbL$HI+zta*h3ewSxr)uR=5nk&5@~9BY)a;l1#|7K&dwALJ}aX9VBy>UcbhG4
z{aYa;W9+|R(ZMzo4vuraYm_^fw+PPlitf-g@{^5ryLwLPZo;<Sje&nJZgcH<W!tgk
z@7W!5Z;MR3EgqNH>=FH7N<?RFXkcq-i_H0!2?+&B7cW=un&8@Tq?F@F>79dLLMMoX
zsmfgwbX}wVE@o0mm_hEfTMV&HnVd^hXRPsHNV>!nSiO+%FxRFR*O=@$0$0yFyjkwZ
z@0*DeH5aVBYZ=2Muu}J!G{^3Pv(=ito)z%1Wggk+q3>m4rn{g^K2l2g_p&83#rpOh
zTw`<ZY(#Ij^Og5U!;j3kvGnS(mK`a7I&S!UOYt%DV6#ivyXD6ElqUyS*M#ag{R=#Q
zT4L&b&Hs^S9i|EuEj`7upX2cw!>REnd^%#!^}p%Zt2={(A>wYjO7h&$m6<Zz9(M%K
zR*Gts;C@oL!0A>0lG;N>k|*|_efH*7%JRKEHA`+?*%7s4%Gu4X9g<<U-CU#HQXeWB
zJ`|sKTB7XUQHR7b&WP@%J6_+3miXf*oj7a3?T3#TLY8knEA#Qj(OGv+wq82^_v+t;
zcUsCGxmJhW4ixWP=k##plJ2@x2WQpuQl0e=ckbVPbMFkMAD_F{7|dhq4G0eIHdWc$
z?67slya3B}hb_uhF0c%@t>bvs!E-G0a$Cn<-(m$R+ta=U`fM(}nUW3<PA_|#`)c{W
z=#a7usm`aVVI6*EyQ<~>pWrjQ{iZf9XZ6{(4_W)poRh1$mE*Hu{;!pP78)y;Ef<N3
z*SULiXBfu}Q-g~evKwC}Cms{>>yG@Z`gpSE$`q62)!*Xf7d6fC6SLoWxw*hMMYvBR
zL8fzt_T9x-PDYBlzgq3eWDsloMC+32trtDYB@@d$LR?~Jt!53{?UF7j#w%LJ@$zrr
z8dZ*4^WG|M%&l&S+ij8_=F0PS$+-u0S08RW7}=`yq4&Z5MZQYUCWe@tRB6jQ%dopA
zCMltGO^R>)u}7QVrd>O~u<Pk7^TvebJsY^Ef7~#|?69Bl4wJqMbGCHHE7}Mr+i=y|
z9J26gT@jG+h^Z}MxvFJ?o@WEY_RBA3{FiI}kiBNcGqy>~7xFz}>sRzD={x?ipVPei
zMZ&fh9WtET3!)~@`E_CIikE3Q+p<zFzRI!okY0IXuSx2QONJ}c6rcD#E!=piuzGjN
zYM-e`KfZaKTJh2K@q?MY8S;tWdYS)cGd0SnCR{c=5}T5lwlMU)Uv|;$h&jjp9SUbI
z__*wi-N{e7*FWh!|D^x@Q>Vldo$p=O6`Cc3-yUtqZO?dnb=~ua7v2iKe^q=kec#jD
z!u&Ch-%a~qs&8HW!ao0n1OHx$|HU2qVi@l_C44b{_ipae1uCCc%w54DyU?IWZ(DGj
z(lkk{ZWh1y*K#wPZivqN!+j&QJ1uH!+MWxgGnJbn^h^c+cPK}SHFh-Ix0si#H<LO4
zt5*G4wnfJc*G;c5TFt@nt%IW=>3Q#Q0cn<oj#SIC)cd^~9n>Y<QyWd*HH3!l-4hpZ
z&#iG?qAbsmjb=fY9>guawY4xl@a3e>%Om2Q<}BUmc0X><te-cSu0MMHb@oCf^~ah4
zOec17ryNn79$6IrIo>*ClAgF!3&+`Vt@SMhd@ol=DPCS_G5vO1>_he|x!wKIo$4KC
z<s6;o-!}dDaqHOIiR*JGm{&fIC|>t3VR5BHj!#N>>4VGMFE-oRE3VsBo|JKB#k-eb
zkCJt*-!T7r8|FFniH26)Ze591#*@~K0ZhD++;^BNOnvXV{Rz7k`#9C;f5#h3Q~r(n
ztwP@1H|jZJ5&q3lsO%JPo55t0nYO9wWp|x>V$#3bo_pM+aIjg!uX^Y2*^g&$%bJ|>
zygJptY2w)%(g$B9Z!Ej0@$nPO>n-;Zw;wt8{OrM*<?l)>U7y|i7->>9dF|7wPycil
zt%zf*J-g@6pB48n1?^YvERx;TWmLkaI-_B;LFStBt@CR?9Y{E<_N%DLL2jpfA(zhT
zlQ%?6#2I?Uc^|$1(PHrLp?l90|GMUkf0NtAmDU#9vF0s)nO^iV-8Me$bor%KW&ZE&
zAGK{e?`ZCMdhH9B>T73sx3oX>_x+gRc2br1)Are_%UE=8t(o!5&wqmd{r_`WI2f5&
ziv<23NId1x!m+*JhSGtBhZ}{|gVtnBRBGy$)Ky!-;gHbO&m^SaF+t&wYo~^)ir|G!
zsa_KdRr9`B7(PDU%Q<O+r0Rj?4q;4DoGVN>-gQ~bG*zwFvqbI6ihzkhOR7RvIV_4|
z&#Tp%sCl{3sk`s&DN*fPTher=iJov{(A=8Q==9aH>xzb1+1E$4zl#-OI~$+!-3hs9
z@bGYS!Y-DX4NrYKg0^r>nB>wWG%aXJ&csatT}L|jg(r(OTu_?cxRQ0<JdIsn&dimy
zKD2JGSSQQj*(Z8(Td&-6c_7kyd!28L(#OX^uaBnaUD(-Bl`#MMqNu;xm%^?_3+8^m
z^4oQNz|rs=?W;BaKYl4toh0b7$~NZ5`@5&*f82Xsd%%#LyTSj){t$<)=4}p(x2QKd
zEZ}vy?ZB0{iM55lZ`uL|4ljkHt>R&3jy~%nO<W!Ma}_RRiq1Xe+PP3LBrrQ>$uZAt
z;Zqt6Ld;iPCNoH;nnd^Yo-A41%c+nk<Rzr2G1<lGn?``AX_U&PN#;jco-}#7w9H^|
z+WM%wO03oN!gTLYk!Jx3UaoRMDN!!>qEfw{h|WnDHC-N9I!iNbn#jhJ*IWx1ZoTSL
z_;YGdLfx&U@lhq3uja*8{C)BwwwtR(b!mI4r<A0GTE<KU1{H=D&sni$GhJu*o)TK+
z<#SLneR1crhf57LQ=(kg^ZjSh4(qvjEMbkYWn`fDe!UYn)sk(tE)O{s$a1%UN#k{n
z>v_BGI}Vq_j^t#P#f67hUoZW=mLV+CRHQt0ns#{Pi%63dwR^*!r2Bk0q<wME$8E3W
zs)e^czrXL-x9js8{(sWmkYIl>UEvUWu#1B0heXW)5#dIwkIO}a=eQh|V4ry4n1uR5
zh72yZqYGUOy;zhw*=H^@a#dU2qTOvE&aOIzTQB00r)k*~mx&&~Zb%3yT=Y>6G38I$
z;ult=Hfvh(xhb)(Cw5tvhE$iOuA80b_xx$Z)-6}P@*jIH(R2Ok=VErX!%yX9yvWDz
z*AsH~*JduAd`;_W^AanTNUwQyPs*1oa{hm}{Xu{0otX{-cdfRbQMosFM^?9W-Nbd7
z$A7Kfz??eo(uDTntGWf()Y{hWPVC~Fx`lb7oNUA+t*dJt9%MbgyYb;k)4VIw9?v$b
z`^gn@|J}!n?(^?xGDh6K{fE)wl*E#X8^-H{I$2m4<~6;_ObeRuX4VtY)(q_@+!s#;
zoSex0ASvX<{Qr-BG6x<^neySa#}*&!aFLd?5}Etz-JHM4*PagvvSeftsB`S@y6DCv
z%(1mwW3{*R%qu0!IK5gdg|1AVqsFAcn|2|y<ded@79}VBMAlqG!^FmSuCttMeR5f6
zb1ZovcG&IgT7#fWnXB6-tYm9sGYUC$>H%B%!gB(JZo8QO-**YRqIlP^P-*t;6ATM?
z?C(@LP#k%1ZcNAL1J790qcQ^dwmGq{eJ+xoeP@;MjssiO*5+)m2s%93<;K+`RWo}H
zJ_SA8!}e`=<BN~+)pwHj|6(<7e(7B)nt5KV%vj)*jHiOZL%}~fM)oW|YpNtK3#{E{
zuGC(T;WFDqUgP+3qqYwIcH_dx-$9wejem4)Idv8|D1A8GD{@jL;8Nsrw<19eS%c}G
zLJk-F9vLs2dNL}4CtP@C+SWUXAAHj@m0izhEL*zd^vqnZXWaKw<t>GlRcQWscI(Qs
zv)fvQXO}JR$a7QP)cK*;q(wPKD6G{&c~+5{>aO(fQCIc~J>=^2`g7Cg|DuRF>eJj7
zIP3|HiFi0WYR<Vi$tjP-f|+Dr2C)0A>yCYWie+AQ+S}e_XKt>CGt~2}uH@!M->c#>
zoB!iW-kcogIM=vS2Av9SCOlmZLUk6CUq()<n)ho)`ZSG+ZzEG&7F+h7+Hqi$z}oXQ
zuet-XCsY`98nhXjuIxOgbU;DF)xX|iRp9?e>B_%+pD4@jOm%Il+;Z%OepSE9)r=OW
zt|$uuEuk$BTs}lO3an!F6}41z|1P!LKGndlaM!bvEnhD#c+$DC{8W_Z=NY~d`!@86
zuUzQK&pO?i@o3<45r?~8vnF-evLt&%bUJSQbnX!E#?Y?rtM;*~i>tb}ZHryxyZY0U
z|AC#W)@@JulUq_FeJ8DRV$>;3y)6yK7aueg?l@VS;herm_eN#5=3}Frup^s&>>`b|
z>Ne_lCMHLoQcK&D%B&*KZp5}=&Gf`KygdrMKdpaqH8>c@K5_B*efp&#XNcvirBh0t
zT=r)@5UhB6Ub~eh1MB9rRnLOwIRD5z^2v0^p%=x0_Tet4XTB(z8Z>E%7n{=Qja5^R
zv{=55VhvIb%2o6CW%ipjA@jDRu-NYK-5n;jFSeTMhRxol*nj0zk<INLA#>c=(m&-0
z-x5n|JvpJcr>g12N`@`pw%%>#wNqDDuC=Q?<mtoFTkRUK^FbN+l8mD>)5V{eE^l2_
zdH343LrX<W{}*Q~Y<XOma8~Hj=95o#A6}K+^K4sO<DKp0lVi_&$68Fb`*m&!>#p}H
z*Jtj(vg%sZv^O`tC20$=t-fU9Hn%BoyFgZyB#WY#>#Od0vtC_YasD)OqNIv+ap<N~
zEtewRUi?#c_3C`bm1-WJmqt`>$#$P5;kn>qlgK%q>C1CIuB(Zi8`oOBd;g;u1`SKU
zC0cXcT%r1HV{d6;*sf#Y)d!DhuUYW8PDFO<cFV{h;}G+V<VR1Gt2&I_XZyB4-4vxI
ze{SnKz20Tw>8T;NCSS?1+u@_Jm{sR$Y02Kyb47~(3Z65CE1X<;typ*7$M&d{okyNp
zgq`U)_NqerT=qe|YoG2cz039B|M{R!?p^6kH@EQI>}T2BQTzJzo-glC&o*6Oxra63
z{V9j1!JoJ{xCq_QWC_VX;CVpBwZ}@pkK5&ya?;F$yiG1Lo2*RZ)_<5RJ?V>xey8JM
zx!vKFrl*tYBb0wm?(%bS64H|M<oXb};+SsH+ea=U4?_>n>RtTvhW^tTPsM%C=cUh^
zp``rt#FIHc_goeF*<IpxNyuV%Rap1l(As6EW0&Q6ZJJXW;8r?6;o7GA*XJ-?QWMIq
z^|zg^u2uB<fZw^BH~q>Z^tK%|3TDVv=sB&MbdTZY>ZSExlIvbv6`scR{)fx4pNjV0
zu3OC6lD9ov`Bmt^d*09^ub3xVse8I?YETY6B=$e-?5cx$6PNDm=hjeDXik&$j0{hy
zvv{>t@3>9<kC~!3><?|#a}D%0IKfcb(p129W-d4T(j(JNq(k=1n;RG%+9RDQBYef=
z^0rjzO(nNYET?(Y@?KpapnWE0($b?(rUqY_czM#z2^m`>P6$Z5eh8YLanCP@?a><H
zmanJQtl)TJ!E$isi5Caw<-R@gD208KL&EwmcaJJgm(Wd&J9^~kN>`SPo>E6|mR>#Q
z8qxZ0;<0(056`Cx*G_cY=yRlM;T7|r>`N9-SNM6-#&e=$Wc<+$lk{)S_juuPHb#`e
zK=yLs89#?u*Bt(KmPbvT4eCpKvRqtxJVJdoM5P2ANluujCpcmK|ATDnXUKjnoY8*r
z{!`0EE-!qPCo*c@I20x`Pv8)nkkGNKITI=-Bn3OL-c;}wU*vk`s_QPxTQL^%9@+YF
zeNy<oU^1^GM}|hD$lqK4V!Nhf&SzP1iu=;#sf>4Co)vpp+MjXel*pe2?*gJ`e0`WV
z;m~`@*8C|+Z&vmN?TBRc41RkxZtmH0J98e+{1cZs!~KKd#e^fR4G)fgV&wTeH?it!
z>){Ow8J%r%Yp*mjEN%^O@qWPHu$1>(=o$Z?3tbr=Daal9(L4W2V8o4|3c-;ozYcVK
z-!p6F&z?}<K$UH>dLQR^zKoyJAi81Y)bj$-Uj!c9jy$5=tD^bCGh%}0wxvhzeZ3d{
z-=~%3Q_7kLeQbHrvoqYagB0fLidi3Ovu5KzvofHOVXo_(gvmb_PYzr<sg-SYLz_@!
z8<T{u)2WA7E+jDdG&D8DO%GUnU~4~r*pc5WR4pz^#7tz8mgx@FNECm0uF&!NF;m53
zp?8X6yiWu^e0^d@qEKpCWypliDcV&JTs>5GE}imJhwIRb_Qp4h&$3Pm=3-D^b9~Pd
zHWwR?wv1EUZY{?dCNn%Z@Ao(8?}k{zDp7$q52rNpT=Z;JXp48NiZhPl%Vi0;6X?RA
zIoIPyn%5T|ozB!LPoh1JT$*f=#CXWX|3GKpnsY&?j%r3N-=Zm;bT7icYyr#L8No}A
zd=NceacPn6|7S`Ie<=GtJS*hLVIU;9y23*<^w@ERb6G|g4ddKu-l>_~I%d)*moblD
zvVub>u+j9Gs_;jiq=rXxT91}0W!$Qec^$z1wsiTd(snn^2TV$7(<k`aOk=B>G(E-Z
z+DF~{Ua4xeHcywdJQMR_tSLLzn&Z}!)%BCNg=x;RJF+5Izg&^aJ7aZJ%th^-h>uEV
z$P>j)ZjTMFW)-NUZ<Eq!J(0EG3BRLK@|?MUr%uhj<>P&$zl-toZP!2%UQebOH~kyV
z23ZJ9&roM86MrAu8t<DB5|hA^<m#9Af~`tAsLuDshLs;w-E>wRy;ka<arOvTkc&&t
z!Rlkn7u|bs+Qg$Zrp@zzPEYNjdxlbesUm_xPt&5L=GbkWZ=&aSz%fHO^y2%1#e!k!
zafgn*%n&^GcB$|#t$4lD%wFP&O080|Y2}&=u55^z8+RmnLFDv1f<BV7(#2i}|2opl
zv!r?x*YnK&be5~jPEMQZ+ah#lMUID-foMe5G`8F?lj@(|_I<L@;YeE7)5LFy{m#c)
z6!I8WtbDrV+>JYm6VA_QX@9!n!VHP<yV{d_B#KVO-~O4c-o%oZ!zi7TJaa=+;18#g
zkJBEwwzha_tYunuBq*k=ETQ}vU%Aq?87@cX{mrbr(KbIqY??~cq65+{IgMg_RxG-v
z7N&7zWlfK+Y_fLetG$AL3>pmvK_^_fTe23{$<?d5ff}ibKUr8s7?>Dz7#J8B8kigy
z894qkuyV*)Z16qUn8%?N<8fgsLpZ1Cq{N6-XI$DOjJagC7W5ozU6OTk^VPhB-BF8=
zY*>7>Zz@9+Q_LZe7n7bHpKT$&?)Q%11b;^RX1+f&H?O$3*sh)LiAI3Tg;2YNTvJvB
zys}vKc*(lxrzX!W7BKDO647p0*|3hMQ%+SoWckHei??ayEZv}XX|`$P#-!l#{4<NH
z#BcHb2{#DZ&BMzTqSIh_e?|v8TUq30-j!i%Z-pE=@iIBRErzX4Ey9gq`bAD&qcoij
zC7D;5ytd0lWG%>H+{(IR=h}%qua7Z#YKq9!D8>h+<>ztv#T-yGD&f48>h3l9(sbL*
zRQVOL)~P3*7P9?M-P(0Ee8uy_Tn_Qya;lf7Sx$2G7WfzRJ;3r*Jipn>hDg`5Pvj#W
zecT@0=owggy@^L}hK%1YUKOuuX+xK5P9dot3iVuE0YT0DW*Uz=RHmH>(4M|j<A7Mq
z3DpCx`o}D8JH^gWTg+g3?$~UH#kx->^fUjBnBqJ$@k_MiCe!ew4%b+<23@8rLQ5Ie
zF8OlFtG+fpso8p^X~Oh`M$NPte{(aJxh2<4of;fjxMQA0Kq`lNd(NuuQ)j(<`(^69
zls?Z1^SL-#)BJL$R)(Z7T~vKKv+btK1n+*$BjVCgLTs6dGvcaNI!tl;CLY^T`0Ay@
z+`g=v4qnehWK-8}xTKXfZQ}orGZ{9YJEgI9ht`o>j@^o10-d%>e!1<QT97!=L0FPy
z@ru()QZF6!vPxb#T&UzwojM_M`&@^+ds}3+UuUpS+3{4cL~e(}%<c1bJ)K+qF64VC
z$ND`YP7?a{pY=?<r$6Faa&E^eo-g_PZYx<#XiF>0P@4SF&}5+l|Ml65+C0`#5{D}G
zUU^t#!RhW+te}4C$-?c^au&rZ2G3OLO4)Jajg$W43}wfZp0GDLhVLzu3#^URQe601
zCAZ|L@`N~Lq>A+3KBr`K(y*l@)>^<TDJIj<Kcc=ec4q6Oqt`;J*9mP4j-Gh-YH;A&
zmGiE}<i)<YUAt|2XvpiehbP`Bl)DpG_d53f)$ez67wJ~qOsL%Zb-t@;*N+B?%9-yM
z+htmYpX`Y9y_`_9HLIv|%~g@u!sgT`L1{}9!rIoRf3uK!weab*%M-U9op&+s%Cjx2
z8J2|j&6#vUR5Z7E@r1<d&vi4EE^FOW+|O6}z^Gnf*Bj9j_affbiavbSW2I_acPIP(
zy}CDrDf2rkpLcIObmQ{vg2PwthXz+Y3%eS>KQ&bM!dBIr!SCI3dj3@YV&GAD7vSQs
zb4QQj_LCExoP?!V^}I`cPIpXbX%SJfWy#pMQufT5DXW%0ePX0MbCN@rBbV+JZCx+P
z$y|@8Ej!ext5dOC_}->X%Os1Qw&gxLrSZUIx%45;cGD#fga3!@=n@s$=<VwBC~Vpc
z7pYqpI{p46&iQj}J4fV3+psrW_s?xQEWPh!yO~S(EtaEO5AW2?ZnRo_@6nMXW?~aX
zkACK0(40K|kwv%S#7iM@UxJSNu&R|WkqTX;G5PrWD^DWlt-jNl@k#5_)<sHlSBg3Z
zq^m_rwi-GdR`F}wdVp1JW#q&p=T#g|UB`QJ6y$6~kGGib3`=dQEMI(MSJF!P$W80x
zw%e%gE1S7JZk@YGA=5Mc+DN;&SH8-UT!O#OmHQ-%1v{@6NHb}!G!G2<JnQhEr+0fY
z?bz?SPv=VP(n)sm{<$_?WVJ(j<rPoEh)L~YPXpu1ZzwLk_?N%v?L&#!mf-(83%Neb
zSAF%IA^3`s#8w|=PLZ?jS=UZ3PMH+$$UMnGh)M8n%7(I~j<<RxM2%P2dMc`1aq`)A
zNYH%I<RzXqnO5BvTqS8om!=9zKS=wveN%_BlklxXKUM{<Y^NS&Th^@VBCWo;LZwbI
zimyadqk@-(ZCvrk@s;mvFW3DqOIDbfuJZkQIcUDtk>yc)R|S+uC@yo@>Le-TEp++W
zA?uwJsvHGpFT5reESZ-o>J_52qw9oF)P=+A>(;v3vS?3;6Ide8wX)sm%kc?;k5~-5
zM1158Rd>y{$u3@MHSw9mOv$EGt&=Psy7a5f6lS|+SS#@)(9LbJR38U>#R@0U6LXGD
zSpJ_g-SS{@j`t=5wq<KX=H_udGimg^^H6I0juXAlG&*+Qc^36~=egK*3~QY424*bU
zd986>+2Y?ZPmR9sKKFH9#rD;AKk}9Dyth@a{JMAC8?)^?Z)^4ZpZ~u5v*@y~d*y}Z
zqF=W&7k%EpKR3N{n)>;v7Z%HXdBJYWvf%+M{}Fkn27?w>rH35)EwWNO)twJ;Uf`Sj
zVk?V^tK;?yYvk^#E##OQsjhfJpnb`<a1j}f*t}!`?tdqEmnQH`N)ex(JTXyedg_Ga
zUZbuRod#ZqcIJ6cKl4O2fBR~`H2w19AMVWi7Qz#DY|}K?jZgK8BN^h>-I})k%u}Pu
zGnYqfE1GuQ=9$s<oBvOz^lhG{z9r2hdgeZxx0>_b`#iU~9C^NGTEP6@Heot>5o@2%
zSy)x``_fc1sr41HNuR@#9vryjykT!Zj-NpFl(+&Z<!+^msp`U7i~*~zOxw&E@8mu?
zi0jgom3((s30~H;GwbrIUUE7u{%Dt3g4>bP6PL6LD=W<vOflcSIMR7}t4>?PB+X?e
zr(SA&+4x(v%d6z&+H&(x25;Cz_r20RU3zrw{KDvPk?U7kcAcO5uJm)r-S=XyY8Q0#
ztIkW``?jon&*QauHQTH2{5rRN|4W+>wcK;={ZBIwcxW5jWIgBpyLszME3H13c;B&l
zw5+1+s<zJ;x19R>w(ANna{5%ZX~%H>FWb;0dMLAg%b&{@jXyiQ7Txf5VGEI-cxOX-
z+@h(SHyTb9-DEW9m^Jq<kMhEqPR{FnC#1wzo9<hcv7RNUsaxZ%l9=JP&NiNfW>=b3
zlzW8QPA92qb^kmaBGxhEAWxd_n!9y5;lCZ;JTGg!QoE}}p5xe7huD4o*Wb@#*z@4G
z-`$r|`#$9H+dTZg@5h1kNAJs??|veEzT8mb-lwVi_dGK`|MN(B+?mef)sME$N#7y7
zBr7E@Z`SEOF<RD^oy={@3Hr7X?DEqt&!4u_p`>2qUUtj(?w}lBt5f!EF4t-&MVi`G
z_PN)ot=hc2YnnsL+|{i;VGFqg-#S<v5O~-B?3dc&*#DWjNdh|ro=<B~2~2MYe;dHv
zBD+=~bC+&^-Y@li|GsbA|L^KtPQw%@;k@tlf4^5Szo`GG&cM>qz%_%FTZ3Vu0t3&F
z3Z5Sgf-4$CRy6dNHwxWoke<OA70@Ur!72ZuL4F0R(hOF$h$gv?CMAics0U4Q63zNQ
zn*NzL8%8vnTQq!nR&Q;Q-=7}Ow6NLfMDyQmdDpMT{{QNyeq4xSTal)_fT&yI?smaF
z)2o>uR?8bE8XgyTXRcb%Xni_e=8SpUs!c|743fkd>sS^#xLsrwxS|}wpqphOJ%4fH
z4JK(O!DKc^@udyP9-EY1otc;%n3lQLa33mN-YR(TGT*iZnacl<a78JwwxyT1hj(_A
zS2U-0&hV(1q~2_FqseRr>jH`9MHbDv5nTo|n&x$MMLKjX*Ju*p=$d$;YmrCy<{6wk
zKbp35bZ?L7-r>=;|3&xOj)ub?4N9s(5;r=HS5%zNsEj_AQstUCtu3?9LZGcwd`?+!
z%8Kgi8P&{%x>t87REo5%P0HTZ=072|rdmXDVqo^9gISS9eKjIlONF%7JkpwZG4fns
z?SfFF<%v-@wwp*UEOifL%XOH*6{xfRpz($<kqc$ETn`<6m#bXeCa^YudzJ#L($+?C
z4FR4N0w**W6gzwPPY5Vm3aD!eoRDCZmt;MDVv^F!Nrp4K^&%&mR8BVgKXY>Afk~Dt
zS#4fUw&t9o$I0q=a<X;h6#L96_MTImUrrRTnBpoq)%4}$vlaPgr?z}5Nr>86#PhI-
zyQwv$L!_fs<DrNCvFXKAM5iUb6iTy5x?m*Wm0rvAuubraRN=`0qeDprE8FzTMEu%y
z*A<qYy%y2qG?7Jd;_|ODJsd%AA1F1Zm+w5$tu(b;HM2=hk~2_Y(u|ceXKHrM-8pmK
z$(c-zGXp2gocXe0xhBK%oimvNXRWXlSogDG`O8_$e$HB(IcuXN<BpxPS4px42(ZrJ
zWL;}HYl9}ke#tpYU(P--bIy{;?!_}I_b$yn9h|vEJnNPQ!(WC(gYeei9Rk-gd)to+
z%=-Uru9V}v8<Gxt!rPs`X>6XBHCb8mLua;6xv3+YME_L73B{6gTa~n%gtJ!Eu39R%
zc<FShhgON1Iv;1&acM=L36zl$o0uRW^EN$KE{J<#hwyQYNm8d8WL7QI@0{$hbMmT}
zlUHdj+B$Ql=Ym<sb}l+RbC#6i98E8QLz)7{rxv+4vII6Pwwkrrrfc#3pNsc(PSKpf
zI;}l!v4i2~5bie@T9S4OL`lq>eQm1y$0cHJOYhFKGdi9r5hlFwScFnhjoRm^EK3=d
zk~S4LfhtK|hM$7PH|8@b)~+tixv?mlOE9^zqP^5hwabyM(L$glLrTR!aLrcLLmwAN
zI<lF#3wXT}ocrHWK>nn_YEOZcm5a2zR!+G!`=saWEs-l%Niy!2WEGsivUKHariHTw
z6&U7atvXq`YVohd%dG@F66Odrun0D=Y_3|pZPx02oUHRt%~@+v8L=$)=|v$nwTPCb
znPIc$avz+_{jejfLM_2^S;!3GSczzN!@fl+ma{|WODI~dc~DdBB5;b`vW+Es;pWJe
zkM`|HJl0*x-ruJEcZS{d?W}j)rRqF$%mby`JdDK~+j%SJAG&D&RY<o(GFt9olzI^N
z><P0JXLbv$Sg6%G$yjTV;mSpaH3c?SZS;7!NdDkPu8oVWxL6f#FIMSZ9PGKoJ!8px
z7k}?xE&Sg$NiA$O;@)gwxH<5D)zV8^4*yiFmKdbCcTL;Q=5X~?jY64ZnwPPy&~zq&
zcBX?{Bp+^(P!wm`HC^YEpi_`k*$aVy>jIn)+v_ztm>9R2DX&PGAbqDxWc^kh)^5Qr
z&N7yZd_Aic^{?7EIZI?+=El98vuFNXxrS@i-jk~W3RcaFTD5W3YL0{*9o?%pSMAvM
za^>lhljL5MEpkZBUm{)~I+tl-#Lne&Pj$@=t7<i$*eem(d+}6eswTs7Yo&``0*_A?
zU+;`~W5AXyZrYXFZc!Nd_N8V*m*$JAY4sl!nNlUvwUz!n$^N4`@yhf5i%MHv*c~1{
zpKtOk!b@mR#|%NC%IK{dG_E`OFIMQ3ytP64Ki5X?z}ZsW!gAgMt=bIAyZ5QCp5x;=
zRpWHwOBTH?2e`ae?R@!pr`hQuv2U4CJ9fDS?utAiz#!9Vqh9Qsy;ks*hP}7qn;DyB
zpB3jxF5p_YrKnTTi+j1}^JSW9fg6)|3$zNZQ<hHXE;$^wSKmpz+0^u{fuM}__Gj4}
zCtJ-fx8Bj{z1m{Vs(^yk=5O}Rd^P8Q)lPNQ>Z4W>U7`H*%o5Mf66_G|V>q)*^5K!^
zH>(4zT05Q=`R`dHpfgWtQRcpYn%`aX_gPEs>lS|M;XX5I-Ln<DKS}iW3$44+q_s9_
z&-T#T#FLC)v;>o?v}2`@{pqsqNSYzZ=)sh|V$LPUt5p;4{(rsVVe}%=-zQ#W&yqa7
zc-70rHos<1d$Tz5!^#lLsk02&+$9oD9NzRpdhIXo%>f!qquwlW{kZe;ttI|Z{Of!U
zEZBY2`niygBOA}>*0hx#yR%Qr8VBeo*RpRF5UUX>=n@GQkop~!+?}E#p{OH#a5+<e
z$=6jmxmHRVHYcurD$Rd=NYzt7?bnW(R;yN@+Ws`ATTgQ5RP`gDz1M6`UlX$8$QK=k
zXEP&ORh6F2JbI?<)RvmwgqKD87DW8J&=Ihxm*H*Zjm(H0)6X+c++}gPTHx%FEpvpw
z$9R7GDKS}e{=VOq&5XzIOKnjps;MrN+#6<fXyO@(!n1CM?KgR*OD7x@7hJFQKWReU
z8RhLCb@m*-*!;%&T%jPxZqE72rYcMJDX*FmB)P~lvgsw;DP7eBorO!bPDlv0Ik4uA
z!ty%`e`1XOP1qF`!L3(#gvp>x!tiux&Vl{vOVe`I_H5t3uUyC`R=ibEY2oM72YR+C
zwzcI;O$azG;8lL6+E>8AU7#WMpw*=10mmg?RW6q-n;^E(<f_cg+AJyI#tCw2NrxUT
zWxB|>MuBtJ&(&|%R9;-^m+vw+$SGpxZT=nY5o`8VN8C7{xmK{x;<j7ju6rkUtyz2P
zY}CaS(euJJPAARX<#mhe>xZM)llb>N;NLf4?zad0C;r|MFuPD*tHw|}KmOPHCtjBd
z*A_>9>g)K=)L*N3Y5Hd^zob1&9oP<<37Gu#m!5Ox$oJ`dsS{?mTo%&OOzjc;Yn<Z$
z*im$+NSTLIbb^h$lz-sfB{yF7ns3~nHS_A;_BDcRcVh4I3+SExGh29_OrN%nX{ztT
z{}1>s2J(Mf5OMZz>9^{MZx`IWJl9VCkOHrb(NyL`pI=^-;k{QFYb0cI!(3OOb+3Tm
z?@Rwhl9w7f3Qja>-YJmqJ6YUyh3me4`6FKUa)jPa*|lr$nvT>6z3G`keVax0J{9Ag
z_kYQ)n>YC*@{Y`WmU;K$&gY#^x5n`ARDU-A*t5L8xtI1n;+mLwPPS&91J}|Ed}|AY
z{uOZ7)Ct8Of4;WhLMYEs-v34yc`ogeH@vvFA@R!;L4&-zO@b*((>fM^yK5}h_U@<P
z5kapTX_EaRXXZWZ=RSBzYwm=H>eAEJI-YFPSUka~^rB4m?LAl3uH63-nBdEmGFR~b
zY5^g)hwX7!SI1oSp0jjc-c_--tJ)j=xS!X{1ai;#sUWPk`M}=`v3IZX-{tRMOq{&V
zKHw4my9WXRMi>4yNUr_B&-8)++ktl)kCsNPn9CECnEiJuW6Wt8+b!Q+B;B?ASQ1ZU
zYTgs5c`Tb6<#+pfZS5Jq*$49_Z^#QyKjV}0-6h#b%p}@6P}FgPYS=cjd4dmGZZb7I
zs;OHtYgzT}&bdi(ch0}%R^Iq&*6Q8_%cnE{-+rN_bTKC8;@1a)0s*tOYg8^axN4at
zmT>Nw2hZ~r)>ngyj(#=Rf6am`Z~<p`5`RC7s=@P)U%^8E4!o<@6VQ3jwe{j1U4Nl9
z4$t}KK04edEMBWUd6M9Qx)VtmhWpRlofhPNPwV=9tK;j6?oBkw(Xw4%*k#KeDZ;Tz
z>h>D#1I2Y3b7cPN3OMDqUuo2sAN}=a9Cv&Sx0cO?_&wK-_q|sRdcVJR=_LKrH#NET
zSaCgc@{jJ^wQnV-kle#J8@NiZ2Yp?T8TnzwUkC0zU4^l>Y#s+hHuXpzko$8)SKybj
z%D({Vw;%Y`6CX+CeUs%6TKnPX-Kb+9JRWa1dwDwj?{=pTPXE=FtnG{?7hYO<Z;w>s
z^7oeFU5-yC_;y^6IdtA%AUWW*LZh|P#Xq4;%U0z+{oC;KTLL>n;^{tJ#&vHvEG#Ze
z(~bXEb5&R%aM6*vy)0i2-#9Pt*1@FX_QxXd$&or)cP|skjY)1j8s^i!gs-1=rfxNh
zOX79!<ozB0^sZfQa`E2VyVWJNbBXKZX+D8Q8z)H|UCdDbT0ue3;9ZiguYjBH7S~nN
zJm$&QPEyi15VyLFMS7RefoW@XtFu<d*~~Q3KQ+0w^4rnV;Y$-3s~FpLdOqdvkBay2
zW0jFKm_2!l=5`?mHwWp?8NY-2JZ2WVoG<jBHcf}&mCJ<y&#g&UpOtDGMgM0hnABmt
z)rC)a=bL|fqmw>#$mH}tSrYl<Q)k{Oo#&tCY)(JU<l(tTudwvV8zpnsYc{`rEqkZ5
zFpR&(DD6!9wqx5iJw4_pZN2W@f%f><3)gqL{9V7EeHp{c#uLl;mwoP3S=Djrg?%T(
zmv%YE>P79-LbD>%r|3*sl|EHtLc-JiQ~q3J>C#Ah<vvAYL4>OlldB*@*Tf_#SB;es
zAKT|jWLj>Jz54GZx6+wgjF;vIMXhA_V7#+2&+Ty4B%OoomuCm6%wMuc=-}=pb2_}c
zf>uA*KA|R+EqrB*^h&EQznZd4CZ1Dyt71NLaqy<FxWk*xp4M6WHQ23W$l+pS`I>9N
z*ztPv|8tW~s@DZ}KAUoa_vD=#y+uK8%8PlPdc5%7=_Sfqy7TqA&FZ?H$D&#fcIq6;
zu&=Js3{_}(&~&;-h^650RP(r<&Eer<tCp^tpOM%;wKPmJRV468#HyPDp;jA|r!Bjx
zv4}x?Qs=y=Ev*^Lqi#s1uJmD%czUxgqSK|pW77$jm4;eVyo95)u1u86*ndd#IR9~l
zEF}r$g-iEG%{nE(W%w#lwM*y4&P(!52DdrZ=dI+*_Sqr#<*0l4i>9mciNZDpA&2&f
z%=DY~;na&onYB`zx6Jgpz2=F(`lPo@f;LQk@vP&NaOd++ckc9wE(~wxGFuoL{5B)(
zUCz>}W_>!XJ?~4jXa2W*)X~k6J+aStT~@&SV7;wIi@GH*O+K|-@5Q{}-+oF@j|nFT
zn>`S5IBx3x>HPH82gV*j=S~$Y<~C+wztdy>CcDado>He%qGWQ5Q@75M6DcBPNkRsX
z=6gMwBO4WQQ|y4jaStu7Wp0T(=2i7|#Ry-L{;zd>!6Jdw+xuV4y!%mS(T#KR{Elzb
zA86!k+GY^u<I*o$HsMvr(ZEF)mMrafwd>pl?U^Z0=kJ=4b2Rzw^{iWB9k*7pe%Za?
zi`08PSHI5X6FmLiAJG1ry!5RhclCPDm6iK^7j64J{lpm`;ca(ZmhjB^A;PHMB)uxo
zm%+;WzKh#Ylc0=cW<s7*7BQGk;!O(pZ&b^)@CM&j8Bf<EIYN!x>0W_sj9d-DiPE<h
zibx!s>-i*a_sj&_OOGT&e0!AsHeA(zV>qE%<Vk_5Nwc)q!Xri#7V&7a%+a1Fwq{;Q
zhREs&cI}W9gR(O$u00FIwN;+%i|TRa3AgDwG$~2=*a{7ybeCxoS=WzgxGZx0QG7;%
z<$x%Q%OWxTPL6N|0n0#z*L_+GT}AdR($`qCOxdzaaDR<Z<0;p3Dt(W|UH@I=3%YSw
zp6{cIMr~5}(!E~oZxg!3YlHg3HY}dHM&bUH6-!ny@6Dg~j@zdpq-|nc%<AcT?s)2(
zKjQASXwg`Z!?9+{&s9NFSBXa7IOh^|@z9waXJ-li5A-Wix^&=9i*KXxnq{T;u4!if
zI~%mX($mEC5Zj6)&rAb1`!Y8lKI?Q<xIvw>@0weL#_fVc<~=8+c>N_bv)tOaneX1r
zI<@%R*4XE54V?a!Qywq2O?qbY{<pVn&BxVYSB~>d5=s!sf5;U+C8?k}c#*b7mtFSN
zB-e|Z!&)LP>a}`4)pR*1E0hqvE-=H9AyUG%Z&L%ym9^~p{}PS{be2ySS=%ADTDYt4
z*J5{FkFP3kI8KFI3GHP&+9GPTSNg5WqE3F^e3itB9sMV+cSZeJ5psLu@u-}OM_=Al
z4cVr0y2zkwlSU_pQ{CCwd^dM(S^Ch^?BU84x(3YhQel^ydVA0OU#<{gWOLWka@)Z*
z^G&2D3Zw-DH0-*?QGa;(<dhA5O^=ugWlSdtX{{1fWr<G9JGxSQ*9Y?y2KAJ`%U1+$
zSrnqAIbVf+^Q7o4hk2Hr2wlfA!Nqt+;?CJWr@9=u9=7l-i{ZttlL7p0iiQr1D^5kT
zMzU_;I#$BUwfMz!hN4xh)upOJ(|$j43rkd5y-Fo9?&Wa}1%?e1o_<$j<o5AkV`KE~
zNfj}QYWM3>V&F(oO5p5bVUp4mj7mw`&$zi`O4LlRPZRys{z-QG|Cp?*)2X7em&LRG
z&!Ov6cJ@bUGb=TB{5+YWBs}xX67N3muiD`|4rXezY+7bwe5=6df|-AI(&R(`HHGqD
zJ==IU=95d&Y^l7W&BEX3tYOp@Raf0EoavmME<H<Z>&lng{C?euy5_?zaaW`ysJ8I%
z<fF@ESdB!|bCix|?fVo^v}uvG=#uD-ZiV~j-*7F66<~P4#eUnI!H;48jCtRbjFKXc
z>2n91aA`=|#I<n3cE>F@VnyafE!NREC%?!b@Ib~!5#giWJ^w21wd~$vvq<TU+N&+4
zBKtmx>s?LeaoQ+u%QuJBcg-QSp2v58ww!g<%)B^JNN}3RqP22a?#H~9SE~NmWGkVu
zcKPZC)(JmyHeUO5u&t^6=-T9~wy6hXjyGkPZJsGR!;UMe;Os{~<8)Q06{{MB7p~bl
zQ%Kl$(*I~*m*AO-&eL8Vc^5d5dv>+h)O=r2kFXS>^=~b7G>^PVe$M4kkf-s`$;R@u
z$a7a7kBEpsBSmHXP0D*bR%X@d+r-RvNtD%E$gb_?Xu0wVua>}!$KL0bo>;Ic+4a?%
z_PzxQF+H!u6S$LF)ymqZ=yWh0D1YhN`X<%8XUdGF*N;`jDF=&9-N+Ta=gZQb)R;4U
zH!EhA>~vj|bMf)@e-bBkQW6)fzc$;sR3W?0{>s`@7g+riYtpCwGB#Trvz+Tuo5&GG
z=2=_Ip5`tKQ_+rDutI9rt#fOPmEPsJUeWq5?U$fCPkp9&QA6>HnoqF|HP4-{KbUK0
zKL3d2g-X7=?{Z|Vng1VKxa7zES;C>4J9?592^vlf<lud)dNZl>!^We!X989l?K#L>
z{$Rz*1NX#*W@;SSA>1J5V%Sy2C|dgMmB6X%jZ5#hzF04NiK+X=!OR6*9bywdyz|LO
zRFddjQn={EHs;8!p)1xeQJ$7-$+>6Mw$F(UHPQ3;H#rDq^8e8=KfH_Y&LuU=%bR#*
zEco7Z-!fjoROIktfv`(AhcFAzC1%IWQ<KjWIr6{iKJ#k9g;^cvZt0w;(qWt=64fbk
za>tBw3@cf7O>GL^b5r`D)yuW_PtDJlY(5<*sU^8APjJySqehiihP$|@WgGV#T_HH<
zMwib7gOADwHcpsV7tt>2DcZHe{?D%s|9e)nb155iywF?eA!e8`XZZrII~PqeBPWCk
zv<ke`-?u?7Pk6Ue$OM)Y!)ZIjrUe)%B<=Pu>hSPUORL(QcFJ(lDr*()O+J#F_Hp;|
zDQvppU?QU8+ElFgM1eVX*K)o+%g&tWE6j2&3fg1Ks4jA+cXp#f>+YlMSwiZUX1+{l
zy*8ou!el9D&ZEjU$7+Q*^&Uv2K4Ea&x&4deifgxebu$!$PW0IbGn52vnir_?Mx+1#
zN=MDxZk$TY|5nZplQ7X<X`=a)J$#0waL6LF1XeF$z6fCkfsJB8(yi-?Cv*$Q9dF>2
zRb16EV@`*JeM<3?6_=SVcQzSwZS2qxtIj;2p*iz^ZnyjoPafTm+c^Gce-Bs_C85KW
z#M~QgzFko|)yH$wLz{EmJW6voX4o8>d58DT23Kd!6%!o}Ryr@Za$s^$$1#Z|CpmtJ
zFcwW-JVkio3&R*ehJ)6f5?&@xe{APaYJT;D&rX`NH(FCJsa<VPN8DjE#)hfkFIf0L
zFbQXz(yLH1T&#5R#rpZL-B(4(>1lPYF`o8*x5*bF&P}Ui3nuPbk}*AyW5&AM%4>Je
zkJ@GS^5lt)&Ms~;>$)?JobXxizGP!SkXnF{l5BLx&d-MO)w6b=w%)|(>VHJNbH~Xo
zCcgdMo}CJZge-h7PdVA4vFE7qj;p(ub3N+Y?0HmJgwx#kpkIXJe~v}!2Tr(lF4(j6
zCC~Diy+%L!d=uQNcOA@)>bvr}cY)5@IW2xQ(Fap|Ca?Un=jjLIc5mL>8PboGrx%Mb
zbOo((_;r}|38zlu@uxy_o-)k|d(bZ$J^#Cnv&x1}mPq!HWUj!7u6C7qRS{h~=5Ry_
z_$<35*7au>d*FeEPxfhk^x*u`|1xRy#4QfDW*p7ZI>o?rapH}0GpBf#O%9kP<M(Qh
zkpK_h^fx`%Ppl~vIkoQxZyoa#zDwH$pKO=u-J|P#s^f>Ff^LW6(@pG23$La4=1)@W
zeBgHCRiEiWum73MvQ|@Vv)I?m@WlR_)R)1r$>g-w9QTbZ)*S|%^E591zP$Cz3CsV1
z0bG_=OB@!pzfM|dnR>=)^W}esmfF;A?9DiGNJ7oyNxx^zGEb33PFve|>~;x1aYP|{
zp--ZslJTK^1-=hwgiLHWdBJIm|B{139DO&oxL$wF%jCs(GWb$;(Wax?p|^Tm@9)sP
zYSZ~xhBM~LQJrRmh}gc_lA<qnO!0m=|C+N%%ASL|61s_dx7Xilk`-NXV#nUQE2d02
z(>Ez)MbSyGODFoQKN^+z^iDd$_eF;D!o+J|D+BC%&%0!s$Tyoz47_0YaZ=p`f$lki
zamC)fJ=d?yn9uch^)~^Xqd_`cyM#-E_tf)VT>bOHRilH8bZ!W|^_}H%>5Jh-!JHfW
z986|U>G7!L``<aWhgqxj)S<Hpcg=-*j@kVTwVJr6(;>|1<wbYTQzw`LHHA8F_Uviu
zww?Il+?T?hBp=}q4Z><IU0Ew9P29oC_<~`pN$c{DYp*!;IIHYj@|C-%=1jka^JZtm
zoQJ_%7jI0?+P%6~wB7Pbnp23}YoFcCi!R8_zQ}Ux{tKo9*<OO7Zu~K4J62q|rgKzy
zD$kaq=O(i9w`^6vT4eMp#%T9Z&+wz?IxigEClPbSIOZkGCPic2>!-JSh&oPuz{1DT
zx9;Sc39_B_NyqBDIygIe9{S8r?e&e+;cdI*_T)_1>kVEqlQw;d@ZaVnWp$UkJfqb)
zSV?|%fYTc@mnqR@Tfz@ib^M<pv3<go3w3)Pj&`q=ak;^NH#UpwuCQ)ksBC9@>lzlX
z`P#cZ_f6$_y>_yiR<ndCXPtAH<XLlxo^z`#-7a&*&q_Rch2xT9tuDWa!VJ?r%15p}
zy}@FX(LHg;rB@Y`CRcFWNKk)0siXfzyfcrtxbuUl1<nawGv;P+NC{s)EfeB(C+chr
z$F9$3<f|9)`kic)_SIW(`|i}93v;goPU*QGBC>1mNj-;4O_!vkMEE4~LZ!~`k>FcV
zc}nupo;`0w8y@TUJ=ExD-n2(LE#^#8jMOXP3>Bj@UQ_Ce9!cL}c+Ju?({$mzoMu5=
zk(V7aw*+nqd@x^q>VikcclSk1{{OR&E9vfktyZq}%sl;p+^I?KPIv4oRb7^;atb9+
zxW_u7=i>EcnNr+5%a5PZJ-J$BE6c@$+E3g%qYuRPJrhklq2yH1l=#deG+^bE{jP}}
zb8lP;iM;paiQuLi{5~e<e<^&L%W-;ciZtJyt!FQua5yNU#gQe#cP2x`y^e3+>iYo`
z@7V~sfA}Cc^99fJX!kou{k1kGiwmo*jO|?6duY|#D92K3naxMI{q!~*-R)+3>7vu^
zi#E4A4HCE7c5eTAph=rqh%a4A@Am%EJHpRW!v3Y?i_L7Zj=141fAv>1Yt-{A3Vl9P
z{SKL)3pWjl7UgSvb~M8D(!4nx={kJ(GLF58IU2m}PF>LE|Do4nKK)(J=@9njZ_I&-
z_jKG2#jNAepVtv5msxZ%?6!g3wWkYHA`fhExn*@Q+4kBB?z7h)UfsdvWzOq!Z*E%j
zffG;M)6RC+o?7_sWk6xdcHXQKJCVdzjqF`dUW#(oEfaqGwmTrq)_2*f!>OBjS?)W2
z%yMLW9We2cz@NBIKaq@-rw=mjn=NpQXj6zdHr4xQ_^nO4H_u2NVT!oR^z!bEg^Cw<
z+CS<SyFJG>B;kAKjG25_GUh$-y5iG!D#7<o!XDwZ{P&g%<fTiUN_Q)Z6BbDicFCF(
z@^Es+*^B$$TuFH-yyeXqF}~ior<cFnxg@ey>fS?b<%|XMt}<(-PUO_R6qWV=o>+r=
zUq{!wZjG|%713c2Eh5h}OfjpxcAb^y(owflU9*j6u0B>V{Yu4oT}Jc!t~o{CGIg$x
z`L=G?42}!SS?DU3#m91Rv0ucEXMPWVWZo`wxN7$_^zV}&Y@)~4-e|hER@C%r<Kei;
zuU^jF*!xA6^R>+D?{a;b7JO{|9GV}pTpPI;e{uBWJhbnrHIvucmebiat8ebp=vPx^
zJ|i*zXjc1vwkvDqJ~%Vwnatv6*C#3Q@@-mC;&t8oErZgb1z#;>zd!Nn&D!4k{>Qi5
z{vt<x%Tg80iw^J>T}&(bQTATY&wIb~7UgZe0gjF)hx2>49li89J@!k{S-Df9P73Ty
zS6Sra|5sb_nFzm|*zlplpy2U5NdX&$b2Ip!FH<xV$UOPtY1{3YhLPuOm0ArCOgzAJ
zZ`*<c_G)o=yj&gaJXy8ge=56IEXJoJmim*8r?@YDsbAI}*X%P@=Va#YUA{~>OX`Nz
z%Z$}KZ}e67&8{qQ)%+mrx|vP<otggGlr@hgt|;EIAT=)LD|i0Q`2D>ti<9`p{V$u^
z2~Ah)QRX#lezU<gRr>$#(6;BUW)HLSKb6_}`fR`A=9a$Y>bGsSPMdgR^VdsV@{2xy
zEv0kAn}t`@YY*|7ES^=iK5OEJQ1(qnAFs*fyejFn{9zpz-?B8f>ytActP}41CiUJX
zFYT?3xuvk}ZoX3)%m2SfJNSCV%)Q$d?oeFtQG+))aE<PSrjK&3g4Um%@ThF2%%S4r
z(Z1`2zpeAVdnmtb+m~ZSr;H9<?9<KDQxGwky-C|YKlMrZtUJ$V?%bq$$TgJxV&(6m
zS6|-#{H8ERKU?!gVAT5j%=sPPmd;k#`eF7>kty4=FIFv@m~rV?*Cl~;qm=r|_m3^R
zsghpF_Ptfr^}y?Avi`5~%FZgPG*4D%)O&EHH2i}{?5Fa7(bsxHbdG;h<9|D2f7$-&
zSNxa$S{Z(_O3v|3YVMo8M%TVQ|1vSOA?r^=RLr?mFHDyUGoQ~1O25|f|G>pxA}z0M
zKgjsMxA4ExD{a3}`sgF3bG|>nB>r6U@XG&<oo8RXNO;C^_v^hcUhb8b{^xv_`!|pM
z`?l#NNBkpZ$uHbc_J_IugTNb?ThZ|b^UF`n+HUk>$Lu5D<*#pAQ^)sT?n7AF0XFls
zX7ia=+IP-69%w!>Rq^Uq#kqXXS4`RMf2n0(?Q?d}Uc^aCm%SYJHZo;x&EA}P=ztKD
ziq@Kpk4i^7W4sC;ZYWGX(jk3%ie~7h9tI)Jol{C4pYmPV!{l&o%Z-PNoMu`W#_h@6
ztbSpxQ~R$qGXj^pF0nQiv$YCwnBEoWY_~S+s?zkXIqRbCbcH5_=|yI9X|xJ2b=VTe
zTC(fLHr?wxicU{^=;HQ5j4}GyGg*sokJ=g=AA1QJxH>%izo&NoBGC(O>t-AEpXYPg
zz0^>XajjAIvptats{<FXZcx%~_`KrG8k_lQN8KJQTR7AFZmzKOH^;_pY}c|}-A={s
zx0tFHdSa5Q=QOv2lS*DLyIS0svcqQERO`2Q-rQpKTYD?^;rGUe+b2(tiqkk{6mnuy
z{jM929?pDUSoZ(+kAGJ`K3~7?soj>1G0_WE7PvB;6TjYgC~}KqqLZmbK#jlcvn~b(
zE{BD!7c9df3shOc1tqth{W`1U7OVO#zsZ6wZ0Sr}f=@flb~F*@)R`EVENSki!obe_
zbw_ZB&djeut#)l8ms$-npSiW3$}GF&9{TsIxL*jLX&mR>OwD;smbOdphx~V~Joz;I
zUvB!+fDFFvkNa)nuBJ`@eKK>IQ}oOyS3}e|m#CLnA2gj1>GW4pBf2<9?OGF)?zW7C
zdR;Av8Uem93mAk0PC3@(R)svMw{R7^9>KmcBCwk)HSASz&H{^ortaQPw@R1fTs&3E
z9O>T1a`|aQVyjQzuEh=_7efR+*X;>8wtOqwY1YX;Tp_Za<$F`6ZEBn5E>kJSW<5RV
zhFE&2WC7RJ+*IE%(S=Ukz9(13rEV&}-p*wzmcLW%W%^s!?{Q1sIy5t#S{?Z#Hq@;t
zy!EJ_cyP`_1HUO@PYRZE1y;zmF^ld@X}#z*<zlSX)g!EuOaFAOU%5P1<<=?-(J7Hm
z|CyeuDXZ%WN-FoptO!wdH=cSxzm&<P?eRP#%a>eUF)Xjzip~joEu1C%a&_#q+Y_^=
z&L|4oyEgQm-t)Oma+A+o-BrqU;eJe_>>ROqO>r(6(Jk*HFV2sTPYqgW8+CI9L!+op
zNK5;|?i+Q=;Y%K+%;)<fo}RO%<CVkGsb&I?)wmbD%3@w>b0L?h!)<o5#-@!YA2YpC
z3&>Ap2tTGPawuEplI5b-n3Rl6?@PLC_xLT`e9Tl@#%nvvoL!rf?;c!#=VsAX)wvIz
zR!^MeKRbNSbfKveuij68ClY@sf!F2Oz1z}0YF{$9nTOnY!teU6A=5PXj%$1B`tU?9
zXPpZopEp}ysruh+uYKX@jo3S0(wCY=X7?UVo)dVn-J?XV%CWC8>|~E^!V&>V!_%q#
zn$Fs992N$#@O!!lH%Rs^oxm4jU}WobQZ{Czp^-;_J<~%5$yrCW_ZKV>$$xP_c+ob~
zw;w-LS{|JA>74Si<(f~k3NPOBo3(vU$DM93_D2r!3w}85Y~igs_xf_wnN9n)pFEoT
zaC1lGk1G?qY`g+%CQotV6W%Ww>cSNK!0F(TrLyss8@v>YZnk|hpK>~MDeugO$-8wv
zX<gd#BJ}DdA5Feuf#gDiwqvUTw9Ab5s&GEg`=fNSb;;p1(=%99jXeF{Uud$Pw7{js
zc#W5v-p=)*i<ZY7DSA5N++&gdI|bHz-QF0$`R&@?t}nR?uRqRCyj41*X(eaez0H?I
z*KQTtwA0(SJ}J$WgVooGm1o+$&8J;IC8-`|2v+3R;yH3*DNEyqi(2fNXVyy@8}_gk
zOtZNY8s?$6!|I%75x<SE_N$H3oMmaub1KdhFKc(?^~*P^YC74!<AJcQX<Lg%VduGk
zORaL2uNTOxFmCF2GUwPI)tT0YkG5`ja`98c6!qJhn%8$IPh6Q}I3ZqUzQck{?>RY3
z?UQz1H~RTG$ocA1y-3NPm<G*|1EF4A(<V7AWofvQkfmQQBz0<qovZ&hP4|L|Eq6k-
zKS>>5nPbXzC%H4K(dew@2J^KOm`>yzJUMg6|08?~%7^omirXDHLpmj@UPQ4@a57$U
zR8_OYuaRfE+^UGBOXF{?oHL>Q#%hoEIo3B*_0M)p*9z_LdcggtY*(;%d8fa^(_rb;
zsLr`FOnbR`Z#n(BaAP~$ReufPfJ52p0g)@BRLZ#+S9mTeJv1eQ#o9DU(97=0frr!5
z&1S9UPzZPJigF1k*_L&bMeniElnDz&Sk^wd6t{iziJ0DIDcftDzB5&W)Rt@51wQgQ
zx#;ln2^+jKoHAY8Ec6Z)v33SeJbC%S=_&IxHgcO+#_~VhGQ%g-ze{xGBBv)AH{S34
zq?vs3W&DcDi6<O`mv#51#jZ%7y5IL;&=Zqa4t-G}OZ2{K|KB|Kahj-!tZ3|Wh3gx$
z9zS6^G53zD`9{fBM?EKP&74M&HLFv?dP<LPVt8*?f7dgGr~DyXw))XeNl&kDS#~Oa
z;iTo<=gxI1yV@_?lz*~l(}r(Te%zevF{3+G@<i0FNp|PUBxkJjUF#6heM}&nMQQ!i
z1}24~r;RZM&OaPaCclVc>#R7d#3_2QY+sfC<Fs}WcFEWtzDd)0>Ne=;&79{hw`z@J
zeF(#IS$U)B8Zr|CR6HV2Mb>;|jn|J)y&Bfy#Vxq?`Oh~quO?X=__P}Np7?wE+H*ti
zrnXgXM<Q0;a5XLK^o&Vw3q7*3m~n#ivM8m!Jdu&@f3&)yYD8VR6^gkW{|EN|w|LuX
z6Mo6iW!fIS4jHdwMZO+A-m&T@Wz;uTE&2K~ICM_T>(l34?K6r_seIp?$R@r`|IpLz
z>w~+p|CfX)czsHmDZg&XjD451Yiw0DyVw?TY`j+YL?(TyLteV-6btL6SLVyf8W%QJ
z_y#8kp7s>_C&HGeG@mE1qm+kPLH6fT9x2s(PP2oQg1%4aeEig;!^5HNi;DW?d#xvT
zpIqYnQ!G~D8E@C}iO*lnWtgi`^m^Cyf2W;Uxg!)Dj&{l<YF*x9oNw=^#@%vVt=wRj
z>$B|H_aYuWe#qA>yi!SQqr)G&36rWPhb9YFbS=r9vpt&SSh%9=3ps6BnQS(*lwO&&
z|37_`PuY5KTS8}*`t(ijwwSn8T`F>xeiJaWJ?+huIfu$KX7i+8Sbk-0@V8IL1pd1w
zJX^YH_m_D|Z|B9eRupPzFW)^=^pnq)qnTU^;s-t_N`LCVZTZAlX(toQgqEdhj@vr)
z>(s<gs`xN<Zx`0<d@OGLc(U!RJ9^ni4=oG!)cCC7B^|iO%jQbk^bHeQkKK%%B;uLO
zaCSM*$(e%k4V>|Za%E4tv@&nIa6H6Svhe1?7x%8m-27z`DR%bm%u2x-nhB`|hax0I
zI~-d!&0P?*bipQDR+l+*9oDpOKHAD7$+I9-Ze<UfoUPwBSH3P6ImMM;p$50od4wcw
z_?EnF)6F>;$MOFR-&-!to%4Qibid&_?{gwLl_AVC;Nh#MGV5;~yu{h?-a&*<Gwj3E
z-W9y^hZ-M9#&zyVIr{$Qt?DgP9_h}J+Vs#hWXkhGg|aQZVJSY{H#)O9S#o$6u?Qrc
zn7e?<!-0vzK}l~>kmf<X$P-6>9<@E0`s+)F%bRIlEa9IxkIHbdZ1+7EyypI&P?xhW
zgRLLP{PA==6B_hSaPD!I$Sn(5C3C#~?&X{GrDt=;g$1g;E|zm<hg@>zywW-$UUZ*t
zMGyN`&y#fru3uG{yGJB&^-b;{AG|00X$k4Ld;E{<>@8OkavX2%<qwvWSY$C{)&|+5
zhKrf{ByX=sJl4x9t$O=Gq{|oH|1P$DmqlcbCAgfBuIhSYal<gr{nuL;#hHsVCpkIZ
zSm5v=Vx@!BpIipb8%_+G?Cw7P$%c1hLOW!i99wR{&7Rt$yX3g8Mfapk7O`oLZc&^U
zIZo;>a8XfZ-#zn6kb#)*Gxlo^XE(Q=|I!n3=<B_JFN;~{_4^fG`~Kk!uTaaItKDxJ
zl7sHvR8(6Oq^k5yR7rVSj1WhIveBZPx4aG)eNOERc)m5Ry`;mbh2zANi<52&`x#tV
zz2&go(nY8Luthz&_C-T>3PaSTnuo;}ryT>h4g2PwZ|P3H*do>{aOq9Ymz)V_mK-mi
znj&>gF(vCb^QTh}OC_s)JX3gCo1WfETFGf_`Ttr#%f*>%UAr6RXS$^|R<J&Nd&FX^
z+Lyh{vfbwAu(ok8lGqV4sf2gQlOr7baYrL!TQ_a!=#Fp^@^kEUZ9gBt{(oXil;Lwn
zkJGGMoxIO=dw)9RD0AF3#3$j}k)yFEVi>0#R^;Ag$o6Pz`kXszRb^tbQEIg(T$5{k
z?LS?b^CmiAQCeukj1w}wrWalJDtK9%Dk<|FQQ}*nl(h7lq=v2DnZ^&QQBg;tu5!7)
z@X(rhKeEV`sj@$y<x$`{nT{ML=^d&uuiUIpEIl9Ktu`~`(g!XdlVi&Sdf)tbWH60^
zJ@bx(#qo_N_$L1muS^v4y(9PdZpv=nBUgWjUvu@@XQ<%E^Z#YtuL(097Tj!by|pko
z@#hJHNVTaS9#6V@DdGdCdJp@L#;};01#1f%MAj_J=32xQvs7rwN~SfLLR&ItImOS~
zCFt_RHMfXWcCTC*Ph?wFYFA84m&OEn$t3{+^Lk5X7u@37d6P-phe7;_jCYE_&$YLU
z!{#Z6CTg~%L>>v%w3&3iW`cCfD(%D|agC#Ip9Wlw6q{^#Y|^oNli#wxy*U3x<igrV
zM_6Vhu6=hrR)=qWMQh%xmkE(aSW@mcD7|pCSj3dF)K$gtZ$awL4z8%a<s}s!rzJgo
zTRKj3JZfF})OCZZ<dP)YJ25d#$LHOd*gfgqzuGPiQ?=N=EpMkZN*csU>iiG6{qy*=
z46%?8rxK=S<xhR(_CuTD*{TD5-22@YR4?U@c;l+zdyp+<X4#SFbq&u#miWfZQM=2S
zr2a|KuO{=*RHs90(i%3soZYwPL{|KqeVWRLrX5~%N6YYn>c!bkKNS3aBy>B<1;x2!
zGAL?JQw?g0PHS1Qc0v=2&8p?=u3kQyQ4yE8p-W8U&^*t?=bE01MJHw(=B1|z%A{>f
z|9B<6TF>3p%Io8rHD?NQWurX(GWqL1Je(mIe`=*i>=ahXH96lnkKAE9aEDFfE?1J>
zvMkppif0RyW}k_7lDrU}v?7eRg{k$?!Jo_2n>hM*E-jwem%l)?#G#!dl~uEM`ph}k
z`+xk;*cZk6semP^%u({e>In`ZuX#I*XE^Oycd{n%Cdbm{t0#p_3}W5)N^|ZMt%+Hz
zGM>}C=XT$WjGMK9(;`W;+C%e3%JFIYay1uq&1gFRDYDhXF>Qf^%cf^<{Zt&8SPnMo
zE&5iZuiSK_@{d;Wg`TaNOS)S4e%6U@m%IKgK<?Ru&{GNvUrac}mg6X0qqyc#x5%$E
zKPoT1Qthl;B;YwAFJ;m7=i8(OPpTAdxf0THdYj2K5yv%BQu<kT@8?+XyZ4^GGiOpt
zg8Q+xagr%A$HR`?y_Czapy1ou56aIL&kE@1GT=OF<IeEZ!|jSk?}j<wP6$7edRLe6
zB8KO>W6SE_j1xp0{;L$`6!2RvsQQ}0WV~_ekB}=OIkgX_N-Sa(W=|_k4bohg$Hf2U
zz^io^4osPva(<Bp+iHW`brB!@6i>ZfmK-{z;O^AOg)@$_HGTYY=0vE`qtJIBT?^ia
zHmzqU&_DThZEoG7bw5@Hq#RSseW77?Yddec>kX$g5ufcc?S4<@@$`Ot-s`$2P~}Y8
z8D7RTZ=vq@Hs0Db0_?R<Jo@IVe|}Q=Dfd%Q@=wQSwKKU7ojR*`G(GLu(e??sYbPnL
zxS05R5xdpVo<$vN8J28tT4G?V-EjArWdF`f>kk;HuGMI$@DSR_@iVGri>Gwwmi$u{
ztE0E^yfir?IJs#<2%GDIqbnD-hzb6WikRZSHq~iaE34-gW3`v>FP0cKoKfXuR7>h}
zbxpl=sc{m+9!H;qcgc@VuROKt&AHA=U54v(PTiDBiK~c?KNA+8a!xHKlb<p4@T9=K
z-xlgjFR9Eq8qc6)Vi_Ddg<p1k?%h(gZ@)Hv<9e}?>xl36mtkq`$yO5GrW*UMtU06A
zvRziZi$^QZXVVLrBO*3!T{|vcw_+~~+nSpxs3z*Wc|(hI$`bCz3ES=n#7}k6v_9GW
zYmQi&k7UoMfFrMj{%Xkniq@&u+h6dh>fj{iX>$GxZvAa2l=`~${jG)Z_odja``Kpm
zc%`@{otJN{2<)2a%JKTB$Q*-jW}l7tTa4qk-CgxRC+>k?@28tQ2VHpmvm=FO{k$8v
zpiH}xVPg)*(e+OvcI@MrpL>ukWy0G#^V<tp&dHcey5YB=<{=yFgUu5d9XeY}1<y`6
zF`H8}<S?tRct1<XkC><m@sJxY+<41U?!A_f{~Fjl>(IHG_xB`T)+qi_@>0t0M(B^z
z^G|FEII`t{(P#PSRwfD2O6JA)?ZoE4iU<`k4gav+)xa-p!og=-<j>7)6<PAS(8sO7
zcanCFN<wKs^Y<R{sPxu3pVZDi*k$PZa#Cmf$9}P@(n&iHNk?>4Bpq*4?@y2yD?8mf
zt>*3fs>s^y{2x9E*dF}kee;KSF=u0jZi=zw!Vlg5lvEpfxMXbq-w!@uUc=pT_tT9N
z5u1<v=qh>Q@$cek@h;aB8@>h9Bxz=?-X>;zeo~j?l_pDW7iXK6&bU=SbnkANa<sH}
ze(Bd#HC5kjX+4Z#ZkivaK4dvBnk5pp+o^opy+d2Wr&V$H?O(;OC|l>^nmk3(S-aom
zEzg{kR*tfxzgC<)bmF;>%QcVF?%Thvdv<oF<X>KO4T)L4nwNKRfBrUWen`&B*^=_B
zI%V`F)dN1HYVUN~A*jnb;h;yfa{g>5vAm}dN0N>l>)rW2nZ^BVtV{kMwx-GQo_7xJ
zzSSfadOTLgSNcm&l(hH1RZ|a}{+PE!pmj;#;hBxOr6MuqVlQ@<?#s;O{~JAh?SFBm
z;6TnlC#q`O_Z;P3yEx5!=aq;T?bF)(XRYzCNcf<&a^}N(uF)R9_OH3gc<KJuV#d_l
zy8=?SP5R$!ntfX<z32bac-JFJ1)Y!fXR}W$Ze7HgUi>jwJEw^yrR~*^J4ZUd&8rAV
z{u8QOx2p7gXX*Vi&l!!c&aDneG2HWV_1#8K=lNEZcV4IF`m8LP^(Sync)-cr0>+Ka
zuFpbV#%rIE?wvKwP*i5&#x$u@zh;=TZ`Zn8d9iBsF4qm;_itIe-{skw$1VaIt6h)m
zZB*MC`e#epwFfLGXSim}4mgoh?tMJh#%*a?^s}$Fo%2?2nSQC*Zc_Qp#<1N@%f3ym
zc*gfJ<w~jk|H6H9tk>qfO}Z&QVVkM2EB}TM+6|#<p0~9U7C);w!u~x&s9c0c_?D4Z
zvwv}a+Ilwy;bs2E1o!xQ)~t<ZnZ(V&pwe*H=tX$=k<tRGt#K{K)7=*QFxQAX+WPHf
z-n}(n*PkeQDV$$&jAg>1r8jNne?PTR<D^?f_<=~-xfe956}Vqd`qBE<ZpztzmF90N
zR|M6b5!yd**5wcmhbb?DmicfRvG6KA32N(ZXk$`v6LDype574SG4J7~RPH0bvaCK6
zcQ7Wq^s35oDcqcp;x<J_Ax`9g<C8OeM#^C=D}<Le7zx&Y6N%b%C19~orPrGlq30Kt
zx-WE_daAQ~N{6EXTZe?g|HcM}nC2SahR{_E<|&JeR5lAFYxm@^Y{-({$Q!%3RB_kT
z6I~%kd^P@i$*6Aky&uSA_Ods^+hJ{3?ve?bPoJero=WF*;_3RhT>89|`Es4pKa*Z|
z_^P&TtJsyoc_miqkY1*hnwuVzt2GaA^tLPAdv{Oi(2!9oWIV#I=W}ktPp9W9CVmTd
zCLUr@6<#)FWrmeORF48{Lrq3$N#$o}>n8T|QeP{-Kb<VKmcwTIk$tl-ntgc}d?1*y
zwpxo>|7y)CpN61Y3%oklRCzHluq)PlaQJ5;d|>TaIpq$XW)-7zW?McS%Gm#1rD^pN
zLB=C5HXIHx5>yI0@#3^xv{d&Au^_eoHx4;4CS-T)yJfOC=lJ}p)f3DYI4qv9S(Vvk
zlatWG5+T(a9X)4HO=`0C^KSIgIB{C<po+&~BR-|%wOe;QXlm9qn~}t%snQV4$!YM)
zt3gF1AgNnK{ncb8#z)*d9uw3gvbxL;R%h&O&noIvo*b~MQ_bVNv##CkSI$``|Nls@
zQu^Q3q;Z9_TFUfPu%XgYMb<thZ=H2pCV4r$=WxkfyX#eukCt8kr0n$_5s4Qy5?NPx
z>uvFnTq+u?9A#!S#q4u|$-@<QKOHyfx}tOY`z>w0(1qLHp4^dj^6aG3sxzbWET2m#
zDW28Mc$Cn@D>7q2Tv(*2LVT^_rr?Hke|#O7WW)bY_PEe-!h_#q@`=aOF6Ap`iP&~%
z+L^|5RynU;bXLzu)bnq^r-~JeM80@wtdsMxH0``oF=;=~NfpLLf~u1$(-y|qbUmBx
zpY!qPMURt`Zb4Bi+ub^&6jPaHPnTR*xSsSZX>DP%_6v*ECzq5RZaq=(Yk6Mw+O3aN
zyoIC}7i@kaXqmNIj6>0Ix62EKrX6MFYfW$0eD>vh7Sa@ICF<eCbdYnqaVHDM^56zb
z<xl1XtV<p=g}0P5P3b-$6=Y&Foy%3{lG(zj^{PE?|6`Oq3wN0WuP88*J`w26*>$-i
z?Y-C{KXn)1l!b0bYu`-L>)rZML4e<85vyWW<jD>_?ySwHmi$lY>k)8H5qo*jX-}R0
z;htMZb_f3Ql2S4=`?ytBDzjvbsOC0{j@bEmk9YMp8x);YTyb-+uG+H~ervZcD>%Gl
zVa%J5ty~KIRwgp6%i3Z~vd=L;-afJ7tT)%UP1E<OC>S*SeH|~^;=%TK3#$dEUjh?@
zBIm+MEZ;KIi)}x~d0rEqs9BQ2^(OgjfKmD3_P^@_g=X_+SADTo{N<*x|D}Ik-J#Wj
zsr*_gT_^T2C<t=6E^G5-;A+}XvB1pcnc>l{xibaSOiwL5Y8}*lH>0f0_*8Gc#$@KG
zixP(=IxN&|3~xQ!bjo0h?yW^~0yF2|I$^R!f_IWg^qiT|=hFGjD{_okbpAh>vZQ6>
z=C!;^&wgD}y|1>+Xr-&~+&O)-?_AlS{z{v%@{qU-i#ltlmjYLLmxEB8q<4gyBX?2Y
zLbYuRIyqbtc?uH)Sz3&Cuyh0|rJM=jGZsHQf#ZSi!@y-8H8&2;>v*Da`iuu(tkC+5
zus1==JcFawl}x#*`Lx4D@v5b_>5)w<vb5EHdoeUASiW{>2@N_eKa;~~zZu5~OHS|L
zx?AD_M%(iBoHyRv-tlzjB#vy60#7OFup@2FR@wgBRvf(;bCFB$%2Bb?Sx5C_KH6;A
zvS_i`s|-(h<;DNE91=a%EOe{q(*)*IZ81?nx0%=<373B4Z+hD_z2r{F+$zK3_+MID
z?Ek+XUA0amlxg;h4Ba0>t>IaXs{(H{i76?thGiVo-P)3+Yi8J1cI2oy`{M*pmgzG@
zW|sEWM)CS(NbGd=<XPtwsCIN)=YhpL{N_HN$sfGrlJwOh&$ER!d*qd0ZaO78Nj)$}
zhGEvKGv-?}Gi6?+%=))dt9Qf8fH|?ZE?I063i{L1SzFs(UG#0yBH6W(u~)gRZxrma
zW73Fl-F&$A&e4mhf@|frK3J<8;mCHgYKn-N@T2~`S$DQ1Zc-@rJ|TXxM9sg-Y)bXy
z6M;Wn{W}%%3-87Xl&fCapfO>gT=E4Mmx_~oVH=WpiY~T9ur9dgYoNyTT((vBhGIii
zg*sQ;V&VJ$X8&)|owG6F4a*cKcGijKTzymuoz57$3r#$?(Njz2riXu&khACAtG0b>
zI8Lg4N%Z{YZswiovApZa?jo*M^}WVxtA8zOS$CvMJ9q`x{dK3;SH&)m3(64o-BF++
z&tJL8`CbFB^Q2>YA~N*$ZR(EZ;}TTYudHTbn7`qMSDR=nzZ+A7M#DMvhQ(VKiACKA
zj*AHFzGpT?v_|AWMCsJy$F?l0S^MGWg)g_cQm=Y2tlQA~d){+p?UNxIj}K)sZCt85
z@80J5S(`qxSST=Zztc|m$tAJ>&(gcWH#M1}KOC)`xZ&R6D@h^&vdvYI2fH+uRPlV3
zU=`Z3p)>GG^SLz%s~`DvcxnFkajXdl+!VMbIZ@hOc=nr>ob4a8&d$5>?&v2&{@gGd
zhihL_?N=DimQEL&a&g)dN5&PW*aHvUeXrM-zwFEHIZs5?*`r%+q;I&y#)#x^ms0ID
z>^i+Z__us*@@^N^pSjjgEYcZW+_oLKWF*6wXA*Sah(gB8jGmmt{zDTI-PS5F&)l8S
zJ;7-Aq>T?J?YvdyX<%9Ilj?N2<g?f*nfosbt=@I+=x?7`=*+T*`TaW4Ou2WHG7ieE
zQ(;|sb<Uc&ZMVd=SGB5Ln_a><XP4+22F`P_oZc(<U(yOJ(^mg?B&_p7z#+Ai`|%Dz
z!dZSiN=sLFE)o%Y_~8JHYDhr(M%L`z`Fd~umq}!P6m%>-cXwyR(X-P|^)N4J(0nk<
zMedC0I>G;I_I<Jr?b%+m|ErONgzwJCtq(Xff4qn`y?jEqT+KD!Yt`G1-LBjDwjb@C
zR(eUV>U8w;AibiJ%`)uI&YNb>QrN4u_Q9JttK;6TuQa4f!WKHjc0CP@5%IY@W9>An
zvr8@<TD4tjUSLMz%lD!VB5OQOt0erNdP833>f!}nf~S^>&Up~*dce+jjrQa~Baww3
zO=n9xCa{}2wmp2cuXvtZ)RDL8uj>UWkG(&a))D9IRq#lu<k5yhYM~~Jl_a#47#g&v
zXkKnO-|~me|JPFWNnBG^7EIXV<Kou*sex5L?0ou{v&w!3O8<pc`EP2^s5<m$$pYRg
zy&boF9vPgMTX071%>oTG7NwYjd|wXnMy<+@N}PD-sRNHo+JProLWjJcu+2YpWYasg
zxNQv9YqB3O=oES}JuA}N-LcX}>KS(`pXw0-zi03FTYPZMa@@P@{jYngc^fsko&>CV
zVZbyYVS1S-U%4l@NhAO91d)FRFK;-W{o>un@I>j;Vmp?j?G{OGGhPb*bW_oD{~n{1
zv}jSF-62H{9_3G6!iq`0b{lPGF)fQYle%xEaMn_80Rh!xno2P~AsNpPobhDnRbpg1
zua)&q<AQ){%k$1HP6<aQob+n?bB6iUtflHr&rZpyEp$s<!qCF2_WrCw)7JkviNY)N
zKYOg&-gaz}UzYnH4+kcJC<)&A5^KNJby=A?TzbcP*}|jwnP-9r#}=<9rv0sQua~Is
z^nZBM^5K*d-}d+NUkmt_Ch*PgUv)-1BIxbvwng$PhvYMI*G4ekeCL&PuYcwg2bDi3
z%@&*#?pz^sf3a|>21EVX_bhAGLzV}fNe&5WQUAm2;I+C)uYgM=p&-nUd2_ajz;b1g
zWEJ}fz4s2gPSJ7YW4>MH7J100PoR@$l^@631EQPQtNN7dPNX`UEa4PB<Rhi)`+;rB
zte3kPcq#>+Pkr%Z>N}RbM}*v72~0CNxIKwae6RMiV`p`XKkA;{u>0Hv?Y(&gOgy}T
z{Hp&OIQd^27%&#||Ng+s9JrTl<Mvkz#SIS0vmB6D5LT5CQj!T~_;pbH*1>STLZx3$
zuU!tWQBzc2mY{s{?6H4JCbmI0dzxmMDZLFznicX{Seo@_U(-#wM@k(}f+9MMOPtI?
z(lry(ozm1Te(R-V6lzGN=~X}4Z*<mg2a|G_76aQdl{N2dSACXP*c9QGrF4M(e83|&
zlZFLdE0!O-XzBG}`7&qO;`L1Vms(hO{2ml??@{OaGRcm$ecGC&)fFoxmp1%bHTiqX
zc{$}K`GdUj)lLO>iVK!Fv`U0H9`<j|&~)0`P<tlFUfAgDi$o*wAfvO)M!~)Q8jJ1)
zd}#Z$s!hy&>i<K@HYplGIet@wRR3oLix@8YAEhA}du(D~u$y(ByP<2ah2BPv>{OeH
zKUyA5>1pzKt-`72*YWAChRpe}uA_F}I;^G!6@5CPHT&2+ozt5%R`0r$>nIR%d6$6v
z#B}}3JX6EYir+|7us(FH%=2a6x~nJj4R19%6zlO6uF-Sn;4bV_yIde%{I;2;cp97W
zpBEGO{|X4OGaFv@IeR69`_pRmKL^E?nu25}2l=-e$_wvU#`C%+S@_m@p>PrIa^C+=
zOe#hbmF0Dm-IfRF^}KO2H8X5_yWp(GCf5yiIp)PqX5ZeG*BPr9iI{cstNto6`Jl=;
zA(YGJbJd(P5iK8?J3og1U!vo+cx&XOj~}a;YLt{}1lZ0W^Ez|jtJ^CN?}x0H?zq@a
zT~(;FR@SjY`N72x3zFoQE)Yw5^7GkMxhzMnj%6=BdiQKwmBDsc{L-5KOKTFp6*};5
zb1-cBz>)4Kx+PKZnj?FW)A}r)EeZmiYG==d>^BHiGMS?Id!hW62C-i!l9w(@(G}Km
zFjF$QBBwD`Uw~P6%i#k-kAhAxgyi~!Z08eS>fnFuok`7!c9)em>O$uJ;8OJy%52{1
z?mIQ}FE7jAQ+n-Rn<udEKJ}<T>2UaywfFPfTy|dbzvZPBwqZuqC)ZmKYWA5b-D>0!
z2)WCl&dm6%;GgTOLqV?=t=s33J3XS)@Bg2peuhp3x7Nr-JlylEL$0b*RB7SVCmz$i
z)?WFQVen{?+?E9qKR<oYDc50L5MVl;Pkq`duFw3}6GX3di@jJF__{I3@Ty_xc9k%3
z)l;txv?eQRPj*f(FYAAl?Cn%$xwUB0HO6Eo^^`{mE!SQLIL+LaVB#$rq|#ktwp`UJ
zFv#K3O|9cfo1Ru4@(e$@(xQ=FxodNEZ*it-(mbOJ4ysN}OzR7eh5Oih9B5!wbrDDs
zVLoN2uf=qD;i510Z{AvaI_gVLt95x1d*8J1S9wvnr(L9DL8WK%@dI5GLNvmB3j%c`
z`to?^=kvNZ%4IAxjLFp%wJ?m*^id9LkQH+hFG>3UadXn%bb~3b$^C4prdLwDnNr;k
z%ey3Av$(P)DlFmWp^zN2%)RU(;l?TxctRFChsK(yzR|at(34cKNG&9A;}@=){?qRa
zmS)6GWSs6ET(+X-ui9=Yi@3{Kk;_cA?tQQQ{#f;FGOObeyOz^uRg{c#Jp>-Ve(}YC
zQ()>ek*_~*d7rBAEQov<$k-?PBF7}b^UR@B8HZNM@);Sn_wiP1a@{*Ewcg)=*;s-5
zQt?p-k$)cyvKN+!el}EAGLqk%Y$~ekwKKW;QLFNzDW+TH*|twfoRJ!CnHm{6O>xn*
zcBkOKiRE*9)wP&j$8~U5C>b0$thG(SWb*Y8d!|`pImOo&&-`DyO?`sytQ&c1Zq6GH
zEPc0S=FwdWc2*0I1T`H{ahkL7XWN0BDvhBGI}cdBY0+W((jC|&_utKP;US5w$2IPS
zE3Hyj@^OjCFm^S$eDBgVw>kZ*BM!<(JXEOm)?4$jA+w<GMxuyR_}N>l#WFGl7M@?c
zfN$RSK+l_}r_b|`IK!><<ipQr7v%kSE%E=6Wa@ZYFGR8RkYTu(Qpn6@`4OqH=1M6N
z5l0mEXGtl?%rN@Bu&mu`%Ikw=#p0?lW><eC+y6Tfyfr!G-x8txY1{#q18yu=6H1%u
zarxPV8;Vk?o!ZL+@2s<RaO+xpsB7(vcMG=U1PMePGE6dBo3)sw(5gWF@V1=)vqdzX
zuT^=l$TRSfXMS7fEeYkUg|asm>|f(mU|cS8C1lNkt|rduq6~Qk3OB?Qrt|afp7!IA
z*sX=dJGvV`Zk({vznIf>>E?!McHx1+xqE%Ki<e0mg^I7#+-&H$v()+LvRtd_!Dd<#
zhfhibJO52w(HULp{B4KxHKXcAxsoV73E}O9(PgFKd$(<0AOA$L;!y9uM>kS3u1~ri
z{P$4tgmM#KCYue)u1B4%l%A=@B+b6Ic-C^IWn#<RcRcX$P<emM@>WD(=5(j(=B)uj
zWm)ILnX3aC8w~Y$PQ9O$R;Sn`xzUv2_5=M7IuEi~R&8P64B_1IZpl@#9}SAq*W|9B
z(u#`wZzi$IMyql1vgxOa<lL(jVq=^ltpwPW&k6|#ms_nYyzxFRT%)hb(0EIKm3N7I
zqG;H{$$t|?A9WjL&)B)xX~*A%vSKoxZckbl?3l5_M(dAx+QD11e7?+bX}X(a;$`vf
z`n6d}lge@gE{c9kvJZJ|x1qv(ZTagRr?WakV;@KQ&-<$Jph<nP%Lc~@8}248pZnBr
zOT)Dnmsh$eG8yFWyOlV-W%Wz8a&8t$w-x-;jV}6~nQ$(!X{yqKnxI};5A(hTnJJpa
zrGYcAEPb?manrU|r>TiEU8)!b>Tk(cHOhu1o`_r`9_Gk%t+6E1QDlNmbL~?333Cn=
z?^gGldF{>NqIuiw{{LEZ+D*u0w~WU>rO@MfGnEf%O}nb$bzI}Y?^(`9UEf!|vtxCi
zeMF;Sa@TV4wf@Q6Vn_V%GslN$oLR`y+j>ZMpH8&=oP#rYq*$w@a#L0dJovTX)C}Fc
zmmZFGH^pXHBys<_c+^}<{r$z)8a1i9cB!F1mp4~!|9X0wwcPP)=@s^N%aea<xY@QD
z))h>?o!nm|a&4jft41*w#pcYq6|WOTT@rVy&N2#5<hkZJEkEXoYLc7IlQZ5APqi|I
zTwbp;k9EPc&VOAI9lyKWZXd2X`?vj?tm@`@E_EX2pRHUE#Ci6GiJ1o0dZx^n7NYf}
z_k62T@4sEWRY|?wnQN-D4Hv|4{oj?8;L>sDgPcz!AM^Vq`Iilpewm1OcX_yT1bND?
z44rw^vd%d)Rx3ib+_G+muKg2rc~!5-vYQJp#Fq28C_cDyMt;WbE036NRC`aB-z?u^
zyDV^SsrqGc3F|%e#}pMd**JY(rKn`$suZf;)Zx43R)2W@s?2p8;v)9{jb4@Fe%W^a
z;zyzvJ@?m3DtCPDs1sc~dC`;$<;>fa;?J2kPY}KNsj}lslgP51oX(@Jw;ZDLuibdQ
zLUh}KjTRb@TzCKD+^-`&!^5=pa`@UOj7|*)8d<odY&r}U9%}w4#`PtfKS{TWPn&Pe
z(xqYc&Hr3_7ik=bx1aderC0K3mZ<K8)h1V^|DVjeqS4K1&b38-!c+z$){{9(ss)!F
z7Fe_92znKqa?tc<&%3FV+8wAWG|8iLS8Gp@W(-TgM6Cp_;Pob_xfV<kSQFN0BUon2
zZNBbK;~chqx)I@u7mNzMd7oMbG%_44=2V-wVP!9andt3rGAAY%CeCH;w0d$gYeACH
z%-51RTU8mIR9jgmY@4y<K_+9M5<{-V265lntf|slVt*^-HSK2o_WSOI>j&;L)%ND>
z@M>7`WGTCo$bog+4?NjY_kFeMg0Q`zi#Fto=r!E)e39*Pk!$7bWnUIC9am62p}O8-
z!(=wEEG^H|wZ2S?{H%5z&E!&<cp%Y7Q#G_n+QaZ*4X0-6{|h40JSGbR6M2_hi9O4E
z#nn+(?MZm-(i)EmHFcaJTnTlhrb;zTB}xr7d?$BY=#;ifx!KF55)c|bQ}N4Vr<tst
z0uz}u)Fw=HGn%IC_@2|HD|O~ZM=tHf3F#Ab7qA+HXR%dQE@lY~Xq%AEG*h!7ecr`o
zX^zq>Z%**oAlRZdE8R}yqMP!?!(7%ITtxyszJ75MwB52n`GC(B%M^9@S!<fqa=)mA
zpI>0~QbpL~aH~7F*Xo4I$IdKQFQr5=2^_r=;p^HeDn5J5rBnJ`A3dbPnR35|tl(Q%
zD$vQoxa@VWiPn-tCl#jGVkuK)P2M=QG+Kr?cKEJb<21?k*o<Tw%e4O&wlY<EPe?UZ
zJt915LTOll_1c`E1q>JDo}`9~&Q)`C<kI-4lgKjBEmTOb=Mi_#YfF>mg+<5AE-m=`
z<q+SB1Dp(ZJgy!6Is1^(OA|KR9f_5)UWa>)+!m)DeR*4A(U(md94Gf}@;ZHy)8Vw$
zCS9r9j$0=71UOgaSuJ0|w8i;?1XGWD*9B(VmkJHDa~g#%F#Y-DuN~y!Fm-RUq{fAC
zVTX*6jiO;k7#;TriIlB4s>PD*c;eL5vL4ZcH_A2?$uC)-Ak-li;rP`{p>^l{i!Ny^
z0;fe?*pb6~<&?(leOxM%F<f8X-6>QRELtHb={0fgnhlTqeWsk6sqXTZ=M<lALtgs-
zr&@2mOpYmMRlV--lk)9wm-oV7o4X`tNqrHrUA$YfvUqF4EDcS2#ZnQ?uZmCQG>x}D
z>bCv=<8%L)1*#$I8En)pCe#PjitcgF&-@+n<C%1c_=fD=FE0E_E8Lyf7(F3L@6Yw7
z-`gGPj<?!BY}3wOvTcT@$C5*{U3oZ`oygR$=5q0PqO_p1@u{1_?uj#ee>rjoI=o42
zImA~M5opM`W^t;NW&f*5ljlTj=#kTsX<e!iYS6f$S#(`BhxL($6}*z|VzU@ToemV+
zF|Ijorr9iUN`cMrhRkv?2WOFhv`=zD6CAsiWVl-AB=YNAV+vcF;ViyXVeOR;wYXc&
zbGAshOk4BcvL!pFTcC|6XxomO7rOg|;vKCRb%Y8V`_h#!ZnU_vW{2#OTr;ku4MGQ$
zLs|1Zm=aVROPti)f;tO2weJ0&)UqS+z$=Bd4f7Qrh6?NnP37*55`OVSW#e;|oRY#t
zC#EJmTVAy!rDU}Pt4jultkkt;IU~=>dYVb|LL*iNnW*qNX07Zz5bG}doKw{Hh`^4<
zE1JISf{~6&uEN=u-29b1lU}@e$Y0QPS?sMss5YDVQH`w&yS}V`q{f=Wy;H`GbLX!V
z#|xL-8h7YNoV|4Q<{dTheJ9%8`XZwz+f<%dF(s*fdL~cxiLEm}X+D`WVPU*~)I!;S
z!igepi%w**O|m^K(emHmN7J>vY@gQExhg2ze(q%zy>`&_YqP&@H2aaP#7U(al4Rvy
zw)@0g@_y5^sN}JP^Wg}wNz%KFl5cqiI~UxL7L}T+AYd5lEn0N-&YC0@k0mqDPl%Y)
z8na>b5zkaD=STfp*nHOh<CqbB`_?s`Ylp>(b5q3M-)d{s+aj9xAbT2%fPV$o0a2z8
z-Hd_CjWuE{I$}z*T`wG3Z87WVe5ME8soLE)P56owD$cBN`1Mn%p>3PXLeV9*Pn9HU
z1zi~Xg&i0;F5fyB<ICl-+QY=K^R}DVGOzBLoUZ+BcicXG3vqLjnIu&a#h&$Wn{<fh
z>!3@TFG~E>Zah5aGR1gTRY#NPrvD;GUv!=H=G~O+tL?m~GU3_E$Yq_WmMSNFnj`uH
zGcG8^nRc7jexA@WH)wx@)Li|BV_groHi<r$DQIM#GdGMS>|lfF0bMDX^Bm?s_||ni
zmz8?M#@HgFsCZVcpYPC(2fc5@IBupFm4;5YTJ&ClL-mA{;3b_4a!)pJDozlwFct_i
za8v)${i5@4y1*Hc7DL~&e2+X9M(w;15xz2Mo&CJZr9X^5Tb*7ww=#k0>U^h`RG!l_
za`v1Fb~$^&Sa-LKwu8~csVCghg<0YfGnkK9JoQ_)PjS)Q^0su<O{M1xN>|%5wYl!9
z5?J5zZP7ww=NtRE!Y+z!Y`*6(Yu1!?(@fqcKK|cjc*S-{&klttSJ$}c^Cs`QklJq;
zwDFe3m8pIuj&{p*-*g^MH+NA}JTk|}qmWxj#f?pK68l1*DFTn86=XdbtSXl$Y>$X}
zB4?oI?Y2@xZ{@No6=S8XJ*s8NoBHyME*qaQ2{B51w##oXi`A9@Prm!!_kG{;=zSMm
zZL-^|yJy46v{>Iok^Y<-9X}5F92PDR<@}Q$ksIjUt5VS{Zad+Cn9D<^odt(k6WGly
zY83jGeQ*$poyekBpcDIY%TJ+}31a(qS;|f-+-g_5>UhL5&jX*9FwK5@AXCs&{R#J{
zlezbFJMQVnm~7N=JmK)5Sw1q3>F0_pt2ySEIaI3utP=MO{XcoaRhNfR=Btl;224Bn
z`Nx)`tM5dXEHLJp%N6YFx=brV@{vq=@>Ub`#goPUOu4q>^Ut+)d?jMp8;?4_o_j&k
z&!Xk8-KXXoi+H<aIxg}SuRl_@bY6UOvZItrr2pzdg^Bzt)<wxCUu3Jx>-XN25wZK$
zfgJwh2X~c5)*Xp9`xB-RAai2JIZ4T5g_}fKCe3E7Y^k0l^X;Z(bYFVP^vP?T)|9-~
z`1JnVzMmhqPPPB_^!+~8OYa$kB_6PqsoRLok&MppJS3R5VX<W0txkTi!agyN$CLX6
zqHK>oZo4AzL#}Galgmq<Fd3M}oI4iK%5z0=`}6;aTtc2!VN$kc;`?0#gH8S?*oV7X
z1PVl#n0vPi{3@zooygL;Jn_dO7U>PF@r$FRwoCa4vMNpvZQmr=k|g~2nzU|cypT$L
z(MA4uAC0Syb5%cgt-V^lErDIAt$=HJfrL6kt-AO&hL&AHb?2BYnF}M%A2RRVDj@A4
zcGsA7CR?lMrOfo;O76t^IR!eu-E|h3Rk9~4*D;#23byI)P_%9e@{Y(h7gG5>Su(Fe
zz(p{J!%?J*v3l22*Ts`mf|!e!9%N10!WvkSEbJj*QYLYR-RAYOyxB&+sZPFo9;cZa
zN<V+hzu3TjgHr)dSOMpEWuEXD*B=Ge9)+xhiG6Hc0_I)Ib_j^8)v;gfVkzWa7x16W
z(@Z<FB6*8g#cVgR*p0ekiAB$oi}qWH%w|<R9Fepxt$K4>j(xabPob{T_2j)B0)K-&
zSDU;3Zf+<IRav`%@x<3apY)1{D-xL&G=?fNMi-cW{bVY;%wdTjYgB?-ag*^2!~S9?
z<H)J;$5P5#75g)n*Ze!c7IlHmDx-~6(!X^{iKwB%S0=k19s)h3bxeh=ldj90J;Yk`
z(f`|X#Ti@8)TRj}M@)FB9?dSuP%*h>!e-^?8aZqSW0t9_G&p;%3%58LCX)L>*dd5J
zLB%}gpopgfTkvFtm<18WZUR9ISRaV%<X`gD7fg72t?BF3JgbM|JY|WEQ!~WU#g<A;
zN-O+7)!t&t3wMEP<7__33IQg2j-?Z$U$z%s%;LP5Q_B%#&gSWPD8@N0+mfw5EpWQY
z#D<5HB<-fizTLoh)~#DCu`hZ;CSy@wN~XY<Nh}{;#Jzl|rg2$dilw2Gis9Z5LfnVF
zH6BjkGjw5QbSc=TDHa&NrBPO?yo+h$ERRe9*B!GsFV1RWuW$+%SSvZ(`9%G@MYi9a
zg18QdZcP<<emzszXyP07p45+Gl_?YBJtoRCDX&bcU)wezE7LSlQetzOU0;jE)@3U0
zMlpPkRBVn*>@>@%y5X@^Kx)?{DXtHbKYtX746KMx5=dg2k-#d*{@C{(i<NO{EcfG>
z&Lpcf+e1I6{hzf|vh#L`)Z4Vu9Y4CZTXwCvnXsy(fNSAY7QxwE3kx>fEMJo`o8=*!
zWRyhbmBKk{^_N!YlppM_tc-dSF1BE+SxZY~-Nb3TUkaE_YkzLp!0y;!Q&oImnt;ng
z-Ow)!dqt$qADS(-j6piB#k+mR%7rY^n-!Rif?71xf))sGQ0%|<QF_)-naWG*0?De@
zTe-e`NJ%PCo!e=&wPgN34F(61nOqmkS`BA)N60MRz^M09z^<b3@pCSvtht<q6^cjo
z#U@rne-Q1-6yPnYo0&Fe%5o7&r-_qG1*Tk`_~x<p+-q66%#%1L&UqGW{#av@pSr-N
zWh)kLn!Yo%;m#vNP4$>Ft^Z}WZJ5^<GQ&mFE3hDYk+RPrkyObivf3`zbCeh+1aaSQ
zvtHF~y;^()-$AaFl~d1a39NHS+@`Qr^jX5X4+)olEkB**v9o;z_rpc02P2Y$bxyBT
zWDQ+e7C1rfWhBSZbxcnqxhFDg?TYAJ7`-!9uTFW1U75h_!u6aN7u!Y&+FufLk@8<~
z$R*`M&MC3dgGQ$9jus47Ar}%uHYSH0OH?%vwK6mmexztDQIdM<W5-GMsivC+)=XHg
zAhmW)XY(Z~|9Y*Kbr0B#ydyT95Vwfnl2IrUjhemgWb@We0g30cSUzsP<vxF@SL5E!
zNj{F#{vWC9UF8-dysq%!x+zgv^S_$SS^9sg#wLN5;JNB)=1WhR8)};8uaK<a6!Cvj
zm9oIXuF7I2V+ot!>dm*-PIl%}`kZ2vD0@4U|LCibnVSStql9dqRwpbMo}47tdt&X6
zQuTimCU<@8y84PO)nKP_HTzNrwrB^Yj$N#&3Y&uuFfRR&kjJo#ed5{`8VhZtyH-Rm
z;WR8<{=0xbN^1LYbJbtAFMk!P%~}_{d+MiyyS1k!MRbUN{5<2ol-+rb4SC$+OAfGI
z3~>!t5pZBOnH%D^BD92$u}`72@ll3U&1e1Ws|^aw8tNMbGooTUqrGbnEjzN!%ISzx
zdQi%b4!N}J{hwAZh!5TLGc>;<OlF><sE23b?9+@f2LBIqU1SpqVyhH2n|GQqm|^F-
z3kRHzFYop?zrI>P?Q8wlB?1DP3_Z6FKIz!~`5@zRDV-Wew#vdoue`D(BI|X1+7=yK
zx$fK6c<Z9iQL8q%3b#8Rn%p@3XLGi#kH9G@k#*^kPnBG!FxG8rNV%h>HuZA$ZP%Rp
zW@}!q7MdB%{n#|`i<<e**#b9MI$x#<7cu!oepG(+oGZnEah|ll$^+%Z$<5mx*rWp%
zr36fkJ-F^p*VMkS89uM3P0Dg#=yWKhK-~JzVy?i&jwRyOuMZttR4J_~(84(V<@3q0
zEDo-P*$;$z|I8}B?6J-0r2xz6s<4(RZx09`ko5A9(3*6uJy@ARAml&y=O4?y3K-A$
z<dqpzrY7CF<KR(ym6-{lF8lZFKfEx#z_h&fTl&8S*O}Zct{%Ij1J;%)Y!*1q%6o||
z^}>PF1a7rIn^Fx}RTb9q8E!E#JotF~Cb5s31=6{NJ*Th*D$Lv#!QwcFU9d9saf_mi
zPM42P?7~*R)$2}2OU2)wws2CEjnbUr;_Tmxg<q!#bUCX1SaVWTb$bJ=$izD`Jk^Vr
zOj%I>TTrq$KD((#`S;ACg**OK=dTp8YWP%h)3EmW5w6fJjkykqqSIA)=OpGHV4ZoJ
zZIcG0vd#$yi$l{+vr0ek<WCcD(Pa>RKFjTI0oU_|9&dLsU%VpxEP-#YK*(j&zkmK`
zsd5FD->EY>Sa-DO#3AoU5!pubZ^~>w7RDZ1PdV95JXn0%Li}vDUWn2uKVyNn3nVI%
zgMM!kSpBir-9n=E*L6-$0YL>;zF>jkB8^Rq%gc6kNGt9yl$v?R<z}H!dA94B+O#tl
zg)=zanrp&0bB43kHcxpP&L(n=E#nMpS;NG<1<iQ^$M>vgWR8@2Y;%r7@Z6^qv5961
z2PEhF9c)Q8RD4&XEx6|Dq~znSh2pZC+>iVAEWUbziBY*Ou&pGc&3aN4OW>rXT2hNe
z#8>`W|7@o4(m<{EALSN1h#Yox<(OVO_fAtk=e^X8r{9;Uf3@AzGuuV+V$IK&HGkA<
z+1zR`ZU67OtF`v}@>-h}2|6+7ei<{$IP7duWy}|R$X7NyHGs|BmrYV_w>sO3fZi@c
z--m*cVv9Z|o|`G~-{nN?3{$B?+A2<0Pkvp=HPOBIPNvDDJIa}dukIEQ)eyBW^1B}-
z#^fPz(@0=O%oC1<$#&-AaV}4eu<n&=z3w%E;q%G8sfiau`vls2CA)=YO^!^ja&q}m
zx&LQx*v#IWCDUEY-Tmh8EjP{<;7jM0=G)RRH8Hh$iuk%l<~I)*dF*%QirIP3E)J}_
zZ56SMWAlZY2Yiu@HMVY6ayNKRMku=7dBc3@oTeh*qKGACD<k#_d|3SQfK%2S28n4G
zBme$UG>j8G>vTB5`hQhgM*Frf)hua|T*G#UIF&Qro-FRy+Lkzk84BN8ax~rinabSw
z6;tN_4CDWOZTm&_1V+0AW^<L9Cu-}&{rQ&_xYfR0CcRc>1EcGWuCfbkQ>_IyR9}(f
z>s}J^E;RF<h@ODk+jr|M-pSqT+L%#z)$9F@sI^zJ*l!%a`e~JIbnzWcIh|=~k4|^J
zTHqFSsrFH?%(Z{&j4rc^nW9C6<ckEaChhRc5#isEWh;{NVQ%k|$r_@K4SP#2B=t=c
z3sV1@5m3I@OMo$W`F&}ZH&cB-AN46OJbJTqYI!T$ty-(MF4oVBli3}0w_MRHX~;Xb
z_3iRjzQXo@%l+p!TUI;>w*Oku{y)&!DB<kwJ@)fTS~$E!J*4)}6$r?#i_=}H`fGRZ
z8P3qzb_E+>l`1Y~bG*CGs%XR4%IlqvrPKGAABsu2^xa{PfbqdSThCR!v=s>!xftd2
zbar~jp7i@a_WB;nyL5bA{?jk_&*jzbVvAr?PrUZ8eAVs(lZy}6zvjwSU|e^%OYGP?
z_x^XO9~T(${yzWbY_sU)Z+-8tNOfP?Ra<?-_01N)^KuC{(z&;8$loMoe?W7J^7pC7
zPy7+tBXBTV;6s$o!D?|6rjs9nW_((_YMb4?Q>TT`xITF;BFR*tYqDIjQCaex?UN~Q
zByv7HVOgk`aOBhi7V&VtAXQgW?wk5G$|gs71pfW`|5M1~!aV)zZ9-}}p{qk@w+kvd
zm0Vu6h@nT)H1Elci;vyI<t^JpZhrJz$(j0Y%9D)t#cTUiY<Gz~IJH!E_O=493olPi
zJ2&64dE2JY^9vi=*F<ehI(#ZKIPA@wYerF_i}e|q%zq`T8hIXH&m@!(qNLE-qA6{V
z7kE18@wA?TyJb%nZCrPEr_}BpcYkjTIp)V?!O5XBY087%`uERL@7yR9(kznQoHKd5
z(XlY5U0ru<yc*Jb=eZ`QOy0PKAt;5fe9P4=?}n3d$BTHn1DhB)Ig8$3tKGJkMVIC8
zRcX6B(*qf|-nbg%wQ6ed^EEPZXMIeJ{oj~uJbdIt?PFa=?SsiI@Bd%=eQN40)2j46
zK}*WbHCcO(zq9-E=eN|==ll8Z?fdZ|X6MnV)-_524lKDJPg=dxX<FC&;?@MeU2R#G
zehpOyXSB>1Z!u;Y-CZJ(z%@<fF!$o47AA$tS{6}jco=`((r`V__V~lG-jpXN+zuK2
z;&zicI;pAWjM?QECfiTE;5d0{!k%?1DyFY9OsAblc)F>l<IM`$>p~pfO`BK(4k)W0
zIOD;|I?1Eh*W`9gu&<%vFD*u^<EuWOQ`vK8!LCk~PsYkKnC|lJe%EC3Wob`}k;ro1
zj5*yZbCkAbUh-%ZdKI*IRd#Cd@)@7D1czlru3HuN>L8bPy_rwxRbInGT#G{tUoBny
zzb;{`>*U@;XI7Z47tsiA5WW2?qp7b$x6?(4wZOzf>UGhzn|<Fd1u`huYVmCCS@EUA
zy@!qa%`InRu7qtj4Wx>FHi&p6xK8WgDOWC!TqI<2$t;L>@j3;SRMT$;le@!v^KSE+
z-{lb7ai_>QC?UpBGVaG63-ejQOqvR(Iy4*%^K=7xIlo3I|BkrDzj%VN)X&9}{r7$B
z)3x9@x$Jk0lm3BOSyCGgFtUdPq(_&;ZP;q%F;U>q^R9Ut#JE{^7#xm>KCA6Kae>~+
zdA&mWdX-C3cjw+L|D0sLSZrl!_v>}1`vf;;?iBrf>*#^0ymwoerzv?|{$Lx#u<QN0
zaAlp1N2@*mGfug)Vyc?D@+Omm%=yQsm1Uh=A*Q$8!-31#m`OM`_UNQyozk~QHlMRO
z&SY_B^+~JC+p^q~a!XyJ&3Bj1D%x~-+Jw|s*Vo?koN<VkQS0c-<@a@cU#M44SaxdR
z(gjIfA_3V8kN<qU^jy{g-t634q0Z~md#0{7ULVC^!nkg&Z?d#^Si`5&$4}h&d@^l1
zv-oo*>5bRd>$dhBefC~(@&35;+MVlv<{N#i;?sV1bz<NIMw7rO1x=ala`Uz{@oYFb
z>yQ+iA&=XdoUNAOuQmnC{}J(0`Erqi(>Q3}dxlji_YxD|+leh;{K(g2zKc~-&7o~B
z=ZbW;TM_TgzD{6wP1oepJoUeD+Ombtp8gH8f){sBD%dBK$`W-UYSF)G6-Q5=SfSVa
z@j=SDO`S^<T%3Yd-M)8<W5=W;+TLlKgYGkZSK+tK6M3pNH{wdEdyh(j&va$(XRkV>
zEgSRoL_-=_gxZgu+>@icZFX$qm&?{SZ#I>yWXxP7mFyWE(&k|w!nMNTC|{IKf5@^^
z9+$VMukPF6HC1+|=o+t=CT|LtOpPwQrDd6<^xpSaxI|$029rhmCNWH&E;W<s&!a^;
z&V_6Be>y+A8E{2cD{o16PoVUFsmt~g54bfmU0C+aVv)P5j8L~*kaCdMrzJfXxRqbI
zoXwiCT#&uWrK!>7Ns7t>hC(Z~R7uT9Uy=VVMw35IwpaSX=~i>8t?Pn%`sYT;tqmrP
zv!A%8KFmDIwk?61%aA>D{iLW>Q#vO0b1evy;e6>a@rU5bnNw7Ql3%8n39?q#9r1bI
zyeN6ii5VMAg_UD>oc6VSbh)Z%g<fon@HM+fs+t+v(MfmCOx6mPEMa?>zw}^G_>7h*
zMe`=|u09z(X`M=F&xC`{z0xw8d%m~}Ol3(Z*7Tm4m*pvFC#0@AJ2o~}@$h$-Qz^C^
zwK)wQ%Y|=o^qeHp5TE^sP45k-vT$mT>^1|oRjE%xy(Z_R`bSQRJm$7k((q-<dZBfZ
zWmk1}i*&9#RH7H_XxLCGcrDBRk;Z~ot-@EMo^5&YT=4&Gv&rr@ht$lTKiVccl_hEM
z)&=HoC6pL<1@>6%I+u9k<((2awZdEm-qOXl6wHq3u8=z!wNr$_{{Iev1y2&^9?vV2
z{d}W^Cy(QeNJEQ)yIh7t*@1}0I}FNt8@BUiPULrg7trRW+QlLqu&z?_F_%%sLS@Cf
zdCi|Maj_m>)>J9NJ!64{<SvEDGta~%F)V-iVAIEp$mY&m_dQ!$l@wpDY!h1hkVRpk
zShtI3-W8!DlOXQR>q=**J7s&Sd{`>WzGg?t!~<e?6f~|%Zb_REROC@|h?Q^4p><!L
zI!L~0n6&H55~VjsJvmPVuxXSp=hfvl?(s5U6|$%-NS(nn?Onp*gOC1us4cm`bR}zR
zWcaaXo8)$`(bS%ueu}ATlkev@+Lnh_y?fB5yfcSq)*U4u#vh-QCvD@7c@UH|G32tc
zf!5)ir7JG;?>}8<C)AMdEvsTHCT_HDYP$QWpIPDQM?3<Lp6Q<2HPLrT@ybpAW;4o`
z?>3mldZg0x0NbqurwLP2)-f4ru&W(-%6v0%FVh3riqj6BMmdchSD!ga3!P@QR+)Ay
z<-vh>w-PQ!-|6Glwch4<`$D|TNu3Qr6$)-XrxIjZO`K<6{eD^OPyVDdm#l9m(vvrZ
zS(P?sa-WR);V<2sdF+ab{#CB88=ij8lzE!z!GF`^{)|tL`X>u|2mZ3S>la+K?%&y-
z|HgL8$+tMJT`_4{BT;y#xK8K9@f4Tc$G&>xZt;#{^<Z4D@cN`|*S*O{@0o58dnqff
zc7WCEQ2(i)rL61^g1+ibSty#L=BT}<F2!8LSmkJo)s>aCU;p1&#3Ec_b>h|mMt8%o
zn*wRB3j-^^AM-KW{P%;qea*r0>hE{o$<KWAz}{N?aJI^72mS88{gb@o@3vi%o>;LY
zIr!&~muAYd%6_<~eEBJPm(^osp!8Im*jIk)Y03xou63{SPPBe_aW=d7<y|s8s^)2>
z1wpq~G1?m^zu5Ho=;95pH0FKDKC{K?xY+cwY+jD?2kyOD7gon1ek!0%V%wQZrkc$x
zt9aJ%M(muCTkv0pU0{~j>&Z8Hb%Q<Awkf=I%boelPrs4<b^w!dvCAp-Gu^c>7Ar(c
zOj>c0+vfG|*AJ~9t!NK2bf{Xu`YgFeO>6ee8+H>PZ0Oyw<?W)0M;fPPuG*pfXwRe@
zy6KlXZvE(bYTT`KdrGeY>nbZ{^H;0Z38|}}T-DUgd$mJL+j3rSLf`D$9rlN0ZvPNX
zj5hh;Ir;VsgW{JQDr;ucZR`wFXxy|}gHJ$aWpej^4c2b0t;=W2u3Nq9ZeypJwDsEC
zM*4erw(Za^QZ`noJe0S4uDF(n`4x^GvpAz~G{p#Oo;&5-+abD-*(T!S;WaPVR~f8~
zi)as(Xmu80D4QY5{)WpXcxuRhBa<j0PW#)3qpZ|!C-exL96WKD<BYY)4?{IZfun*S
z<$anb?roST^@f9=g(v7R$F<!~L6cpxt+eJ|?l8GHiM^yd@-fHx9b1oh&QVWfeO!Im
zS8(EbheOggd0A)dnbaYDSE2cX*RCHN_D8rjtg1Hq{<!m#cC)9D07LcuBZm}=f;&xE
zjD<EB6-_=S&TKy8gPZo_{VB@pKi=ZlIh*^t1hcT=N;?zPM<=?fUbNRr^RYi!!=BV0
zv5=infc5#w34tGYswzC{A9L6oKHN~uvr5n|MrKp*7FSjUv6C4VyFA47ez-7lEeKt6
zAoh&XafXIi%hfT;bEY+mPYvE9c7(^&$g=o<kkAoHnWA4DT06IH5R$6S?C8HWMQ%&)
z+}oBn8RjcxGnW^*sh*j2IjBKTN#MKnCZib)`?<DrYOqRL39NSJ>fW`lNSiUMP(N!k
z$Bc?Y`loqj6l|)i>QCIvYIdQi>bCW&jHPu22cllEhY7TXHb@<pWL@@>wM?QpP=dKt
zq#>^1#J<<<M?(&r6zM#uy~m#0<m{e_XJ#1e_{s5JlH2|ERKFfx4{O1>T1TZDcksNn
zaTb&bVi4Lc$#dV!wzpz$R0<En3>oc6&*Kj23wh?03%TWFvKqYNcruY`8P6om8&l#N
zc5qE<5ZTzDRlTp^Nawr9TR0o6?>0{S)VyoWW7+N<eDD7|Y!mp^@oCC&_fF3LMcvT~
zt-2>x#R;&wm>f8o!hTkQIc8z|!-x|TSG6A~kqNu8y{TrNa||mFXTbfQBNkR7Dpz{@
zS;Ufpe0yiuT-&np$`y`~GZ$Dz6;uQo`2Pm<I5hM+FzH`pTDofz$IlMKBFCebrNxY8
zavfz}1}tjJXkC)RbJ<BeZs81Fg&hjUb1w-APhTl%sk7aT!Fl>dW`^d82b7qF6q-*N
z>{`g(?_jv=6W1h*IXvbq+S4<dcUCT)_=^3nNOS3n)_qfijte%oFKIt6!(6?Z)x{?`
zY=vl*0Q13^*3gI}YZBNE51jYtQC%*%#bK|DgY};K96_BYMYmfx%KW!HyVJn4=d{6%
ztRB9Y#U}%1XD8Zj+`0B3!@+(BhAzd$+L!i5ZI+(6g75MUXVpN3z6F!cXz(U2+4>~d
z&pq~-X2p5A<Q^qse({MxOEQ=h7Ot=IIq&?l|KtU})VCdzXL?UP$>VyPV_)={cA3_?
z1O6Q~C#pX1th~U&{)N4);zU#g`}u+cbpmIOopJWP!BaE4yX(#N?k^o@6SgP@yHzhW
zP!(9l@r0vOc{caoj&B#cdlu~A6IGc0q@!81;Vz3pRPDyT6|2r>bWM)d>ne@t_n33p
zgZJL;4vF6!R~Kyb%?#)*;B5$Wo2DTi>)FJ+q1~)t%c-AoMuL-hCvfg?3frvMx$}Px
z|KuAt*M02NU18kC;uFZRG&F)eO2Fro4D(`(BY{7HCnmA_YDC#iJy9jW+;*lTY*o8y
z1gpcHGkrUJA}U2+35Fc25OvUX>h;)q@`IehXO6_iD_>7?c%;fV)ap)s(sAe37MYu3
zl83M4zHIWp%aQLnSNBoCv>U!E6Pm1J0)oGCEKNFCEY*6rVO4Jf=kb-SPh*s#4NR+N
zHGSp~Z#@**s1-47LuB11R-*|`7AfopMHs3a0{*iE_*N`%jtQUW6wb>TzAm<vk=5eT
z-NPqdNvYU$pULDF>U5Yaa`dReQRkWRl9@;E9~D&4xY%P5$hhRj*%dv@cFayt?w#;@
z$4nDt(f^kt`WCD@_BG=92Jz<(eO0d~=tiuL6<wFBHCZ=e;*Yl-n|^rg$l+YDLi^(<
zj#<9DZXTSdt2>)%-JK&B82Bw!k2l_}ieL>CU~c^q{O1e%kvq475?Rfs>acF{;Cs1w
z&kf`1(}%4$d(>}Wx3Hb-AT`%D)#S32)2V9lq(+mVDOb8rb4$HkbUFFxq!kCVHk}P)
z32m|6_+oDN^9LOXDpx16Y8QX%u=+AdX&W!+gSqFoO0L<${K?wCp=<8!M2#a^%olSz
z|0Q3mkiL72V~R#b1Y^U4iHq2iWL(MwJbHil-{+mwwZ`Mv19#ga&W#eRnh~q3p9&O2
z>4qs!te+d;Kj(_eG&R2ejrZ=K749!Mxt@W6)57nP$Iku}_dZ<ZS(w7A%DFA#sQA1;
zadDd?jth7_kC6GR%q)E7NVJbr1mC5(i%*5OC7w2v{X3U4oMCgoohFs1Znqxt?QSwV
zGOJ<D>d4Mj%jcwWMqgmKpkOjF%OUeG$K<S8=MF_N9eE~bXVUd$jnK8n4k8>6B|4fE
zI_f{Pn>A=!?v#pKso*9pU)uO|Q$Tdjjc~4~0`b3&tk0Y_sX#>G=EJkEIU3TQTsyor
zqw#3dHQjkiJnr{8T7Bd1>ud-)I{U#N+nkA^OKyl42PH?}V6obm`jB^0b&+x1uGN8G
z+_nXDHt*8^)}^?4@-Cwd3pMw+Z#MM*{QpbG&Mld`0k<Y)T-T6$m~VEV&gH-(4~O^+
znU@K2VZw0?3WA=$rFKj>wbf$E=g5ikoYejVJ()j`TY6^Pda;I=M>(eRT-+SR%XzX#
zOgB}vPAlT^jh+b^+w~0I_Vym$!FS`%{T>PJu(DLuH_A$PI8xWhw0MV9nk^B(#3}vC
zEL>uP(FB#EW}}@GwrPFd#`_~Bb!Cvwti#O9yj5T4%=6fAW>$~aoM*Z7j#wT$EX3#H
zrSrTg<87f^$D)QgPds~lHyqZTz;s89A?1uqSzO4rL-D;F3{ztlNy@oCKGqRn^DZOk
zWJXk8A4AecPuJ?3iA|!Nynmn1WxBfWfuNsg=UtJB>;7BZ-?T9F<2utQY2KJ@)6?(n
z$=dWht(p7%AIIt|k9r<3yg#jPlrbfm!~NSSzpq`HLIye368n!DJx-1D{C$D%|Lp$2
z3rwwFGHt|m6x`IQ6UpL!Zr$i{;Gu=w@`QOu@3a_BVAAbq5)>D#{wMIWH}%$O?&(K1
zEdAE;`QQfEX#p}D5BPn(eI)aRhTn}>cST#zD!0ih_X&6QtrglXcE4lq)&%WOeLKYa
zd{f_7D9oJ0E6AC`drE5Z+Ek%6sSd|@*Ck&1aazS>Gpm_4<E_kA-6Mzo-0~{WKFnnI
z;s0Dtna0oNK4u1`a}ILn?wP$b{3W}I>s#Ar#)WBb9mStFYsf8rAoT42!d&YsO?Rd-
zl%_rBm>^uTQ08=DqWtxXdpIT+P0Zr6^wls@t9Q-g^Yq*&dvGC_-ohkK#djQD=eB<8
zd(_pp^~{RbYGJ8jMfsPuPPPbSSf{A-`F4JH)n+%%3FY3#Mmu`f-adDxAxB4Xp(T&d
znQt7QTRzV@%hS6@(Dq;2#9al<R|*!pWhJGs3S4iwE%#CAeh0^&$5na<kL?KM<4n1@
z#6e2`C_~Z?%@r%f`Nj1Dg#3i1W&dXiJ^dIyFOKEG+1Cx({Su4UHQbr_S8Jc<d!_at
z!D;W;X0e9m-7j0wb5d}V%9c+&Oqo@mavK#l8Cm{R`aENYN|^Fwm5t2yA9{?k9{L&m
z|6p{lusMh!S+uL{8_z1SF74_!bLUs+RKISwe!bhCrzAWpcUgK>Q^VV;1eu<AE>@}L
zTi2RypUV|&)D*t{wr9ihO#&0e*;E=8=YD*cJ;(dEF8^6qQQ3LPAzpsmr?qE4(tGzr
zPB3WBqW0N>(=N<!`dlt|Z)Fd|orZT}n>`s=IyS7$<0;L$S{n7}>p88Ss^2>AXI-+C
zICK1{ZKK`f#O>mXm(2cQ^!l*fnFHFbyJW4@A19j@FmVc`u=dZnxo}QqtKkKvCG6=!
zaiWa}K0H!*#nK}4Ou;RPp>U0Oq0skV>)!vW`uA&w>D;ioe?H{_eI`>ja(Id!JaVms
z=S;M@>$Uj}|NHMXvo?L$Zrd~CYlpeAIalhw%D`Z;o$Y%ieCPEXUQn^<pv0`zp&D!|
zqHU3eJ8K#Q6ap47F$zu52q-d8bZ+!yQ}a|%U_8noqNz3Gk=p{@#xk~Pnvai7lMR=C
z&-2G(L-O%Py?8zs&S2FuQw+0rEs4B5)q5t>y9p(!?MrkQsJ6?MUR@SCJ1AvI#!B@A
z@}cn|e<RmcU7c>bZriJ?S=YNnw1ZO2avil@yBxMzuxeFmK3(9jtN8ID*^cRX_x6>_
zI>|&7EMRG3R^=+|{Ju8nSmW(ocOG5JJ*z346_>V$QC*ymY3GA4Zz{Gl7|oBJB{pN%
z7Y8E)%ZY4~(rN098`7EodrWBIS$%U~J#+a@uLA{)OIS7?^n6tz{rJdS{`a~IWKU0B
zx<p}~#BPJ~W=@qFe*Sx@j;xGlX8RZXv+?WXG@G(h)xj?Lm;T$^^(QB9IX};|_S^TM
z2By0YPwB1`sYqzzkUFs-C~wL|Hp$ci$75nLJ{ucZW^#(S>V{l>?BAh$?U0}Dnhi!=
z+OvBWo)imkRdf&uviUNBVG3u0a&PX<8IP0gc3pX#VaxS1xxn2hY(}ED+Cl~fFFuu-
zDNMc}gQn``9+RBZ7h3r=Gd@pp*^GQe1&5jXj0MXW7#I`}7BYF72jnj?QD4rm#w0v_
z4%3!{UPUK(r^HC-OMOpYP|^2NV(!uZJJVvcuW$x*JG}L|&Q)gNwcvo`Y^^Jr&K$C3
z6RQ`+En#r=J;b`yd(|;jS7}{NhSzmV&b|s-tNnJ#{b-xBT2mZs1XD!<Jy=60Nvj!#
zdP%00Zad00?d;QQ2}~LWGni~A8Hq`zE!=vvJNvH0I!FGauIm`OWjpo!?p|FQu&VOt
z))_9Z9&Ndi#Cq{}P|VGrtdm__uc<6$VEWH0(;m66<Ywx;PBn+ayxa^?IScNZ-Od%-
z-(u*~8`cwiT(4A7Kwa_hmZ+Sawp-E{X`dIn>%GqO)ukdP74w)gwQgL&x@&U|zE4~g
zbULoeak{(i=|6q~^L^!X&&K;&nf6Y)V*l65)a%oK*@|x6$o$-d>+H8Ka<vErWhk~u
zdl)iausdw>c8jT#n7NM?Tjb@W0_JJv6RMf_n4k98HTT9OH)T!+zu;eHt<&rGr!Mq1
zY>p9r(z8FcXi*#YlQ^NEBP&07w{0rCGq3cVTb1@)_2X?NOLnLHnwUSyE<<~5iOR|X
zTk~m_D+N^x-cI^<Zdz>F&uyV>Yy6IzEm+T_v;U9(L9wl+U!s&-*GR9O;>adBH<jZp
zOEITd#qBE5KSCFt`mcTxYko_W@j$>le&LVWo8Ov0?PSuwpeDuf<jsL8`hE<Ny99hV
z)Ecx4845inNvMZCmeLF1@DaLvBVSEjfSa!+Wrt@2)BEQKr}O-8FJ)4^;=oZl=b^!I
zi@l8^9Lr7i^B-d~6v_zk%-S?@{+$2@bCwG^B3Bl$ruI5+6NuJ45w?&!HN%-HKqAq=
zSof2}+KlGvM>)TyX((AP%MWXLcz=b(zJrpQMzxkV9L$1MTSPxJcQf~98k{)FpVVi#
zOY!iVA4eoEN_r_&ZdzRUZN+iHXM%wSCza2}HEquEY@e_sGNdR*c$fRtbE-)%B|qL+
zI^*D%j$GEp`N?}E_FehVWoWch)Tb@gD>3wamG8-W--Ldwd9S)GYEw_rqDSnmUraaF
z%n6yaJ7=c%LEg@ypNFCo;wolr7s-s%*9<a>KeS#qBQb&Lbsvj{@BstIH>&?-7AWxC
zy)kQhL)&b<pEFK4wQbZ3>6v4}*dXF&sCK?^qeZ@@-x;Z&zL%a(6xmjN^4tl52J>yp
z)#k-0iNA3cp7mja1J_?g=YFMaW;+s{+>Z#bPdTyhV}-kdu*4a+#z_0VyvzklZYJf(
zDFyKted=zBewk6T^3vkJ?+Xqbym;TrLZBxy&%~1T#r+8zma%v=@_uURoF8*(_Qj7U
z{Wm0^3An#$`Gvn5EdzK(3S826xz;Y7GW!JI>ZWw_Mk_VzkPs$zfg^&8IaD=P=uGpR
zAUf&I&P&w~gjcFRG?$n@Ykge8+v|yIT<soYd|Z0WWSL`KrT0Fgwk0-mdspVEsEU6*
zA8iu)|Eg)@<g?p-UM6yXx$!t6dG`5&#O2JM>V75_mUnNIY@Cwq7da`}seIoOoq|a-
zy%S82xV_C@ziMqK%d!9=`Ck{roCKp=oKElM^>J|av3cfg&3R@~C;ytDCz<zO9iP#p
z5w>-nO=g_)uR9UzM7)|SFXf+D5qN3Ix8}*lyw&B>YV(f;D%efUXEFOIv#M)@<x$J5
z`F`TJR|I|aTebUgz~e8=>>s{7;5Id@pk-G4t*wX4j%FSWJG=evO@a4^&g!0E@mhFl
zg30TtQzoi*ayljLe^g|>N0n#qj@x(kKYhQeuTb;Dqpd!*e@&0?JpHKmx7mKVyJzcK
zYPs22g?t-tmY=vG!*S$4i<sZ3m7kZ*dmACmDqwuMpi$Uj>m~-17pW6vIaBxiT%YxD
zr)uS{V=YXQteihB4%Kok7yVa~RuD5sV{55N@tLC<^N(unu-Pm!V`JyL#Mnpa-`S4u
zGHUbm+WlL|I-hg>1iml%H-yeIg<bQOY4V)9L!tFl*JtzS0+~#g?iUZvlq)zibj<!^
zeaQOYt+Q1@O9I*dWS#F6Q0HkoS-|0$eeA-;v>K(>fc;F*BPKeT_XQdinAGgglq|lq
zPcmojVHT}@hSM)rEiUui+-om4|CjC?!G_w!k$KY^qZ&*bXG}Lby@~A<Thq40#W^06
z&&^$T?8@SkywPW5-Wkr^5V%;Oxyvi#|8k)=*04!ZNweqfeqng=>+WW26J1{8*i7SZ
z_vO#{mAOC8nR6v{*TbbHTMy2VSRktQ<krH>|3xi3zD&Dl=r1RH#?)_Th_x|i@Tctx
zjuryfT+;(C6l$Hz%Uyo;?6>EMi;|0DCad4_WNld@khzR&?WOGQKQRjPH|9+>$xLg{
z+;z)N=a}c@^$M%G6ZF`2_uc*{kZQCn=ID`a^O&<UQr^9dzjd*=XX8JiBR9(xW-pUD
zB*8XE!`)|jr@QZ-ojl8Cm7Fns@ni;@v+pt%n@sL)pUn#__TOmR&9ZI(IWfOiKcxM&
zqR#8ro!(*>HTlZr$}XFyc{lxeU*1~(>c0GYrQ|ISe6{}fKJyaU@+U2Q^HbfYyYG2i
z+P-kH1DluR)dw!M+`&`+xnJ_RVfaJm$;+VdgH~2^(++o5<u2z-aa?Tww<>i1S;5t%
zKWk0R();aC?!3MG%tN`JxGCalH~HWF{^))DXZQBK@r=*Tcg~%$X=h8Vr`v;#F^Q^P
zD>BcdwsIbmng7`IT#m~9i7PrEK4D|;^!u&qRU&YEmhZXQGVCv22EW|e#L&liG;q#U
zQI<EAbMiVL@D@!xwNcKXQ@kQK_T5vtxSb*I1s1N9cvzm{C6p&J&$n^8X2ViepQ#Fd
zQ+eja$ON1|$nvq*E$eJd`qt}HA1Wj;PF>e~EJ*bD>^XjO1-zcW2z=f#;s2|iRwdD?
zGYn+UiipZ`GW_H{@@7YW&)G>Q9Vc#%?EWDT_9x?paN-?@r`$^KuKqb0UvTGiw8f%{
z9e4XqdW-&boA7e(P6oC+nR9!0#_K3P(C~B6oVF;{=b%|`FndE==ETNj9TWfc9E@HN
zQ&7Oc+IRA}%S^E{vE+m^?@MobxA2^tFnN{&L)OMa8_x#&R?Mq54ZUPJuZi<Wz(l$A
zXZf5q%=L1e{yMkwgT<Z2E9aZ@EqL{pm$z<$!aAO~y%QJhY!|F**ZJDF?8svwtph@S
z46cQaoGoGETdupcv`FrmDyeaIS?sO<9&P7bJ9ns@aR`|<Z4D2r@5v`I3ZEacNaS^9
zevJ4p<l*qZvi+2d=$hIk1v^+?cs!OlHS6odwgp$BSR*C1PXu4)ZMT(~t-a+?h0TI;
zMVI#)GZl{ol=#jPe;U7N#|_WB@+W`Bp0u2mz_ZZ3_3G<~Gvc@0(C}KSId75HwnZ~$
zIK01lMC}Owj)aBzcOE&(2}t>J>Q7>0%T0J{!M0CRK6;nS%mYgq7!JzVCQC56uiF`X
zP}MzO;8MYy#8|(@B2(sC?2AkE=&=$~u<T<qpX3&BOZCXjm_H`B3`G-D!?-JAmI+-;
zE?WA?R!Y2ePiSU`O9mH%!?Xs6DF-sS8nV_s%iNUaTXHa4$<MJb%IR2d^n`;`Uo6wt
zn>VB4q^Ygs8IAk@liQ}ct$XO5b-yW6e(n*rMIy2bj@**FIm;_f-rG!e`o%!q7Y|DW
zj<aznSWP;<BC>6dD$Czqw(XvaDlg6Z#k=VL+Id?QPxi}-^=_4ozj=duYJ14M8MZ-7
zLpLe;q^OGTe3muMkyGSFw4&PLovkTzT+b%9J?M24KgRZ`^vnlOzVwrK9w$CYKRVAi
z^m1)wfU!|u{I<3itTO@w7r)$j<WOQe`$FD|0>`OG5@uZS?^TORDtmEv;$lm!XA4qg
z9*4|*b+Dgjr+chNi;dJvwYlx7=aOx$GBty)h`mUb{E(dSazaMevdoq<S##3FSzdUr
zNMm2KjG^OYW|>14mrK~JhUwD|{+}haf?H<A>}ScX4gNly2bP|F)GKo{#pk5jgj-@I
zo$O1ab-t;wbqO07&C6nOk6OoeocD;~tC(0dArn7kj)y+yeM4V=h~iXSm0U4D|3lk`
zo58v#o_O<2EZ^kc)0mt*>jql^qg&=;-c8T@*M(HuNd?@y_iwI$pb+2D(C!BuClg|#
zC+u8)^`LzB(;JMh)S14erw1I+%F}S*U6C!fqOq-I>%LG)4Q`e~{sndCom`GM7c60V
z`f{!iTXaQtZDBX#!imWblF}<Cuo|)L(R{_4x!_Z+q~D5@E(#4I1uBJko^vdO*1Szs
z>`dWYsj=?HA(0>FlFXL2{ypEr#GYuhY@?9m|EC``4!jDQ`0FT--;va)Wofq_Z+@Vh
zQTHs`(ZeHUW!9pX4ZB!oEqdnAk$IwICFhg|&Mhy`R=qsCE%SWaky$|<c4Z9g1q`89
znpJ-;+wX0;)OB$l+r23qo!fa@=KoX6U)MAFX6CV);6(vX?z^;zJV^BQiI{srIN?x>
z{LY!hGC9&t>Kh}kN^Qz*zB#dgYib!E%NN1pVte|JTvT#gGDpaVA!A4P&6jaEt|l<P
zQ#g6mSL4$Bop+Wq)+N>IWp535%CcZlXy38vT_>~qG`C8<crKT*ZQs<w!q;LE@3t$s
zxJ0-xsH|mRVPM#^Zq-TE*&1(Gz2x8785JdQJW?jWdEe0op$q@N_v_i4s;C|z@JN-*
z(v#(s#fmR0-+DLPd!D;$=D}$U2d>qbPJJoszb2<K`Pqu6$6ja`&JFHcd{j#>c$)sv
zy2t!Voxk2`ICeZ1Ry?M5S3d2O@`<2#lk?I})-9Xbq*lH1&9rT*J{#0#P0O4$YvtK>
zn&<QiW*<vyJh#%$t029o;h)ml*WX?oVe4U-@>(KhR<_6!w&<C@GnYnMv56|g`<CRM
zT%(%9m+qVKL`_Af<Coi<El$fm%~A@zUtADp*SMm?i%;n&PkzSpjjgQj*1Y<4DT{yI
zyzi%8-Po8_^J&Q&#-ux{tB(ere)1?Xv`S5T)3fPW?@oynJQvb<5x4q9+mUY$7yjRj
zXj?wV>(U`THg1;7M!JcwvVGsIzO%=bL%JpU%%W_DRo)?q1zWUz9w=qb_;A`|=3YL1
zW$wqbWQ1cbElp<lxWCdbRw(96uGS&GP?ej994EaOew@K0y88FE`5M37jy`*Fly8CT
z`@7oWV&ygM<tbjvQqq=PNUmS@X!6<2ZA>Yd3`=HZy*$0EWyYZv!~T`ir8aYx9ANsa
z!#uq)Q))VkaBlh~<x8{Ly$y6vKiQb?(IECD)1=XOy5t2156AhFW?dC~C9PVtXxsd}
z9VefC+;D#BTRE#^b%x5X1!g~Q>67A`EBm!`-jiFAlj44`-Y)Y~mYg7>_{!CJ%aq@h
zK`L_DkCvtX-<){ExnFjR>$+95Zfw$PV5%wEoRqg^pNoY0sV!mIFBWb5R*)UD@ZHuG
z?o2F<GW;#<i%v$bnf118v%?KNKjUM5yYDbPVo9x8v7Kkm)P+Y+rMN8+n&6;vA^*f{
zi{5J?=d*o3C>{U7#^k~fJtOx^&?eRJRX6!sI)&W7?Q7M|2|e~sL$7;^_UVgdD~ulR
zFx9cwnJc(b=k%tP4eZgIzlte~w>b!8d`dRyK9X>{jiFxhG}n=t``*mz%RIx?z_eL&
zj!}U)OEF8Z<3+Kzrr*!Knw0$d(np7uZ!+n#qq@s9nAs&i7_wbEVk!LU{#m!rk65^l
zE3KUQlxy?1z1OtaUP)(NH27ce#^CIVn=<bTJp8VHlfCn~>fE-T6jsGck5!UaUJJ7}
z;qrC;@*}6=<*aW%q@;w^4R4CBc33L1%R?>yssE|e!Zf4TZC4hjR;X=Zd+g9)SP<}J
z%PO{WN!r`ev^gcjrTrV0w`BWlUgvNmux4VJxz#bh^y7ZY4*vG58vBp!Ki0-HwKyl>
zz_b%(*6Lhryc?5md~vC1Pw6OKlldk`B&XQB-FsJLtlphM5l+!irKE_ws<T`B_HFub
zK<oIu9~z6zf~%4zzN$!!J*IcmNa&i8(6djrK?N3p9+?gt4o;QTGka?5H`XX`&bp<;
zWNmEnILE-fIHxVlG=JOVq-Rwp3^P7o`Cm4<ZPsA}pB_h-S+AoF9z-)dx3W)pbXF!K
zVoCbSg892QTR+~G!5DhOO+8TesQklX+kUt83m-1(S>nwnTC{(b?f<C9Hsy*36JPE;
zcI%|VmXATT^VT~wCavWP30QUdu~Jx_UUvNKNAcyyLZ_9aR10`bXL~WLP-=6K_u@z1
z>_1-Zd$;~{q2GQNzxkXkzfBsim-t%jcys8;rq5e8y}!2f=!xD3&))hq=JX#+6N!`h
z$;PUvAA0*|bWL@reb%!aKFu3Ws^-tbkGF*<-{%nYIwRA*&}YW!5G}tOhYDFN@2``H
zUGD3^*e<kj{fvOgjt7)Xk1qauZPP!$156Gt_54@5w7UF1)we8NGQ(h5Th^?w1|zeX
z>z2(<+R0qJlX*JB4DW(;f!w00NkWU<FP!)|^ZSqAhguA$?RH==&HGUC^VW_TQ#kwf
z%*pCDX5s4*`D1u+&zz4NeO81<<}7%6O>5KBm;}#W#e2t&bv$l4b>gy&lzDEiMeq*m
zWRpvrwtO!o4NG4a7F@}UbClXY#ay)h`>&5je;Xco(q<C6OzC3L!FslK&;3WI_Z6PG
zRpk|*UGF^MrPJwUE!lkv$9%4DX;gQdDWG=tcFoz@o-+(8d?kKAeQopfy|f&Adx{Ig
zR%@f3qD8(6Jnae9Em^wW%f97QOn$gTYRNm9&fu)h>2kSyUE?nmulZ2-zx#c~nqMyt
zSH9Udd(*tooP)*gLf=de-(Nf_|MHZ!unZwHgYDCk_OrTOdt=mI!(6_TIat&5_2RVk
zf7~P&&RiY#*Q4%fc7YsorOJMnUx%lhIecX4*OxDjTc-SY$#MCoXx7>`<`#xVW==5^
ziwQjqElj6naw-HK9cbfMmRjO5F+qrPHp>>N7ZVj$n~Adah-7S1T0L1cx>1B<Q>ye-
zMeB7fC(;!BCb6EJkag5+IfI!U>kXlR*U7uhf<!z7H4>LkYUEnLHNk3wmJ2J(>bSd>
z7qky<VA`%_P%ZYBiB<TTUx9Zb6T6=54vrU07h4+kRI*O{@#}j210z*7aW$VEK97#X
z{Ff-cVU&1^MN?>shQ}m@h6aY2%nSk?3|kx(*n~MWVs|~rxV+j{S)#*XYr~C=<yB@G
zzZKr?-}V2S*xlJ?E=ExctVADN`QgwP&Ej+Of%T)$&t+Hw+W961AAG~MIbS{^T;WH@
zy^Z`T_5!6%ES|@<IC0gc9sKce{dIeb%ZuHE_I-VSKeJmz(@VO0&FdSpUMXDR@>0<L
zvMNbHn}yY)TX(Zc1h?~ciw<o)ty2#k9d0`#ps-Q!Y{m;c5&l<;TKIkmop6<4)i}r^
zp1I{>i=NMdz~e?mCI{WcBA+Z`QCfLpLNn`)RTtJUh!hwZ|6a1k&#_6QWUZ18>w{he
zmK}47mTU1}`Y)uUdom`Mr*VdG0*@}s4VAM^i!Ayqjb=noR<QZZ@>$!K^H+d{UG;7b
z#g1#YUnopx>Fuy%{1(g^^pmASM5D2xD&)dnheIpX`?E5SYc$%$uF^d9Ci{Z<`gKO7
zhe9t_UDeqp7I10pjvW(LH}8J8s$c&_mFOFT!y?W(8~L9;QrL82#;P})&NOv=y4hGE
z9ITMVZ56DL!|Rk3sM*NCYsS#fpc|7J-~8hAgLT2-+s<<&g)8(uQq53oUUDI!yVc@W
z3b#G~tfU2;Y)lago+}vL*!P)b?eu&1B*YGwUiWfVYScIpP;#kf_uU`b{}sB<C(rhp
zrK!>5th;Ke?iy2*gRd7<{f}&zzH-*jL#oD41i}S6SvGbH)rvfDY5TM5^m<8FrKkmQ
ztQ{9K#FOt#bWv!uDSV}HT4K`)zGV(Zht=l)nPq&|LM}QWJKXKPG0TJw1y0RH7n~;P
zniOunb#+s##M@mzLq2S|`s-Tt!D7!zhx|1;!ZqUSuZkpaH+>iBPd>gsOJiZDN6(6T
zRl4_UPC1*Nl~9?<BXoCxTeg(OrAzivvB5oo|2r4VH{g4<-tVh)=89=Ozd{<y;)D#I
z&EV9zu<8-(BrndeUj1yRi1^TKrYO@rQ-5_$ky~;7buD*}TViJ@PmSJ*1#*!mH8!8w
z#L^@X{pQ>qacNTpp$YK{6=(Nt``6)q`~UxUI+>598Nz=FIkLTwX4s&fvPj95Wn0L8
zHmeCaDc*iAcaF%gUCQ<Un>5EkEkpjLNQ>3AmE2DjT%P-JhC9Pe#f1(c-{v;2x!@&K
z(Dj-n)yn-Yd#S}AZzfey>mU^t)iu@~@41q?r~Ks&Wow)%w6SU7tWsa+=*^)W4k2vj
zU(R%OZ9M%nqGj*ms~VXGw*q^;4k#~v+3N04)YyM&!!hBV4jz%aMcZ@C7&RJmwcm6v
zsNHh)cwXje@0~wSYq*38^#9?l|2tVoq57}~pO}hoSMlVBuS%A$muR<~bbE%Htdd}7
z1H1B9&ZqPFJ{f#baV)If<F`1-T;TtOzV-`0=PWH^|F6}&MLD(7=3a1Esp#{)USYX2
z=f!kwTWuisxrbF#s$>WInnNnjs=Fp~C!0;sIW_0K$qYe<E!$dVU#QW}FKbS?TX5Jz
zcj~j*tQXJT^ia0g!<eV=K}li8smyg2vMrXZJrAb;uvBYSS*VmUfr)K?)4^Ka6z=l}
zh1I#6)LRsN6qtlNN*zwfhQ3oNZe7LU;JtXFq!HH=dF4)C^E9go6PlLHknZrw`Z(|I
z!X9pcNjF>OMI4{4*v%|5;dFZ!%h|-Eg$k$dBn2d1S+rDkYNnsJRBz3ZkX2J3t_;@>
z;<RW8-k!kGpD64xMUQLQzKxq#%Q&aFm?+NwF~u!OWvBeqLnpTv{NJRb^z+sN#~G8C
z87`Tmz|+bRx6t^|PPSD_<(?gyvTmv&Qb$U27cBHLm=+<m?u)8pi_SZ*O_!#Hh^a1G
zJtaTqI@|w>ZRPsCchz)U7Dmne7Ga>%@r~uG&q7Y_nl8cnYlT*=`5d~;WcJ51zwAEm
z`?+&<)R*c9;TM(f6z=@2@QR5yi|fbQLQC_VA9JoVe-}4&JS3XG?P0H7@Mm$|4bG*S
zyxPYnMXXTyb#jvUT))yQE*J9-Ua)@j@m%GJ%@f4bJeZ<a98G?meE8>=WBac^)OU1S
zBy{qeyQ+(cilj(@Z_+Crmq?#8VxcY*Q@OV4d+Dx<w_@1JS8m~IspNCjXM?e29>;&Z
zoSgw-Ur${9*k;Pe$n)S<hD@^Y&*DAi>p05!C$;?F*Djj(Ymz)~kzeuN9bTNBllN%+
z2=6EeTK||;Ra8Pru0HUHU(KnECJymk%^us765^gZ3TX&=FVxd;{^%RPwpw#;%qr0Z
zK_3DOc6M^kn6!0XibwvgABz(HRz_IORufl9-gipF(UvLWn%Tmmp|@EWO+5;hEGX$Q
z-w^du|8bP`Yzg+|k3DW(%3s?*Lr+&V|IrQgHyN#~3Qu;0c`)s;R-3&mVe-<h58Wyf
zF_ZUPTjIFS&1KQq&Ih%zJ2nK>u`<3^oFlm3X{F4nBgtkxsxHgc9-dp_vWZiMb6c^K
zK+vA$8XNy78+3YIIJ$>pYNkxsv%tTXuE?`y9<e<o=>Od5`0B71eKDmUyMOY^>MJr#
z{WGaWY|Dn$9fc~Kli!sq-<G-_xFJbo$&Di#T<^^Gvz}Y4!@PQ%k(`BV*pqpCJ@Z}v
zuv~1FcG;?@l6cBm^3*}8Q%S{UDZXioB6}z1Em2$fbLQDQf)fnNN;_gZz3ZBUZ@q01
z={g#F$y7C}v7t+({M=_py*suJ$EWceTa;_6SiJA_6EV}7j16I5J2DUShF;lP_dNf|
z%5QqI)91aFd9dS#GK21vsR|<cujgF(xvpPXdPjIt_5J){`L5V!MhnWU|A)+6_d-n0
zO3|ES^2{@Hnz#=<Nm+gR|9WxPoT+i@4LrA)HpeIB+I5B}3(nzGWc}p4a7~}+g}!ff
zDgT6wJQ!wWUg?f&`l4L9`GmXLzN-`8^-imq{`bkr`^iib4BB?~9twW5fVCoWea4$v
zv2&`EcdR`fvh7~O*2mYUPul+})N#`HwuL(XxjC;-No^JA;eLC3^866%Qyn+tZQHqj
z)gIxQ$awzFmnBO?Qy%?4u|~OPooLocu`GUvcW+)DST3?6V^-Gc*L=CAJG!<@<SKmg
zow()cm98^$mD-}O`7yW&e9>yU=CJn4t(89C)S28Ixjwv8i16on;P;c~#V)0!OCsy+
zc4V{eXJYXAU^U~6Z^7I3Rvumo?Vr1P@4x!*YQwQQ&%*1bTmSxumF+?NUl-}Nha5Lw
z^;Wwescg^Mr7VXuoKhK76<)ts>rtR(WTe8lFU{#hA9MZ$KJmPmMF~6%uQUoWxF>Aj
zQ$BIRL~oA8y3_~*r8`|aSn|F1969#ogTBOybvp0=a2>J`IJSNltG`|#!xep-5Wc5Y
zol`!%FA`&!nw9@Q<Ix!}eNo1q17+)!kN9kS;`}^rQBdEz7cW#_vR(KZr6|t2K(6BX
z%dAF`ln#lROOJlZs4dIh@l5k~$l<pip80H0X^2?d*mK7C(Q@+~W;4HJv%1%F=LAOm
zQ_sJ~`DxdC;~T6TOA<sDE%?RLB%4&gJz;^VLI?ZdBPIv_8!Dwa?+Vx;!?}L<F;1l?
zoC@DJn(g8=;S*BiFOuf>zsYuJokcEB#CpdWIdOBCniu(aT|2$k?H$|8(^F0?o+shL
z<+n`vz}qKFe5ZymsXL{qy*O;|@kur6py2Zl{NhXH{(oTSPhvUFuqUEX>{?<_R-<eL
zBYz|NJhqK$o&r0a3N-sSF?%Yl?RagLlCzVcK`>S47~j&|MGccUo_{*lpd@|l9T(FA
zi%(VtMZ#~HbuVwaap(L=t|mhcry7qGy@#8MxS0N5Yqm`I^eo^bm+!07Eq<cfk2VG5
zGx$IM%9C{Z&YP(j`HaF#9diN%uj>hv7HGycALrXBrRZjQMD-e*)5n7UK65U3nw)%j
z!t3&CmPKn+8r9j8G}=G?lsI9xW~tHHz`0wvwLBj&-}Y13s$w$bli9wbTCJD8`GPF2
zGh3P}adjA4nw(Hj(O=-EIO$wJ2m3KAZlxQav=Zj|v?v_fYgOoF)4$n8m^0<YrX2=f
z{{$IazC3lEfbH%ATh~=z98N3jd#lj1LiC58=J8`%%M0y8HJMBjc|Lr2nAAUkElc>$
z6Se=w6Wwy({yNtGZ>{}7R_%>J%6eP0w?FvA^62Rahd7%l&ay&E|C^L%9@c#KN>`aF
z;hIa4PNBj+VXHHnl#VpJT`uA@<r4YYqodj&@*^m~=a6ztpq*f`XtKLO;-cG3TIxmr
z4eec;L~a~VewAooy=c!eZjR>9n&EDy7fu?p7+sq;F=oO6O{tdMPuydV9NBeloxtC`
ziEoaXDKYpxa}$2@?A^A8M{XM$c6giT2>dSi?A5Jfk#Tr-Mz8O4O-11%#mSR%Q%)3c
zoHKv6-e2IA!h&K2j;uJ9LTB?AZq<jM$Sk%yVw|`{g;6EVcEN_PH;P3g7m5Ys=lna*
z9lA-4?~GmK!Q?l_fl&wfC9ga#)6v#Frfz1cDSky^Q3AJ_<0CglKb>_-2l9Pu7hft{
zbSW=r_o`1?n~lEQIF`Gmp<vtAbuXSJUSe@kHB~yaX@iLH{2%WWm$wM(ui{Y<EmRS0
zJEAW9-6w|g|IrZBCQF}<oBRv|*De0vCBpEZsm$Y%*mqM#_C;b#L=yXzQXece^EaLl
zw0QmxmgqG_xn_xKHxhYvJl*l^qvl*A_VVI+Gp^c|B<D^mwiG_+ZF*VY==#*+&&g+9
zV|pg1wNL&zX|iSa)k)JX#6DiN@W|Kn<DrI;`5SHpGJGj6Ou1lb*k9Pwyii2_MnZ`K
zkJ@ClSHiklHsYx){>)RWwlD;W=J;<_GB|phCxAhDQNqjMz#P*%TqnhfrY5(PdNHlY
z=3FFZrl?-})#^!D{$Ecg)hkiY-L*Gg)m}LHQ_bUfD*Zv6OfjNN?%z2wUpucC$;#1L
z>Gqea_;Qf&)?mHk>B92=(>7c)Wn9w0^JQvI4u_hQcc4V~i-vPSb$y%8Og*#h?N=Kn
zFX3(F_ngmNYe`~p;yCRheVH%m!xz(er_Gt!y?>bewlHQiX%y+uNIrQ$aAsko>s$G&
znn_>2ykDnn%xUzF;p=+=&-%VM0jI@Hp0q1YR$$suv_NnDI=4oi2jO+HOVwE#CcHDR
z^DQmBpzZp5yZMRGX&g;?A6k?RPWbA4op?aS`erxh*RaH#<PgqB1*-~{<h}T^=B(;J
z@0{{Or7O>>MDuIbn56y+^GeDO<xZ0Dlt`1lBIkIaHS$K4yRg>Epku41=BY9`?+oR>
z^wp~8oAx2oLe~%lF>xik5`QHoSK0sJmV8Z0bDf&`eptQ8=eoA3$WmneBhw7!MQ)x<
zQ|~-}8KiH2?sG|uy2tV<`%guPimH}$oU>j3_(f8<#x2HjL2Z6^rH5fZDmLn0t7txv
za>GopAtLb9RhA@;#ShHoS5#e8m1cdJxU=7Ac2T&wo6o7~7X6<;TnlHqG5vwaa%a{>
zzw9bIMVm5APklA$Qh1VTC3(I5@K;}_DbBq&rcSMhOW8R!v$BEJ!A!jUQ<d&#p&JTf
z7K`GxtPjfT_I?o@AJO#var5oZw$5Db0Y##Qs?)XzBpfb__daCb(U<#nn+C&<{CWef
z4$)sBrVb%D=BzwwtYuQLerxWO;OO50&Xp1M>HlZmIx$t`-Ss+lrum)MW}mGTn#<F;
zc*f+0uU0G82zT!kuKt!4Yq{4(+(ubx|05lxPtE==?TfB%R@0N}N$meG^!f9yv&u%t
zZR<WxTe6*HZHmd7Ns^Tt9xs{n@m`+Sm9&z#Ayt=Oef;<6*c$GMA`{IPZFJ-mT>Hnj
zQd2Uf=ljk{p96MmY7%L0&rTP%+A@Fn%$d3~RynH`&->;4d!{<y^l61AG3$R#J-lo6
ze5LN=ml}*;SiN>msH$98%^V`_uON|OyX8Q3;EpEqiol4k>mE=3nz7xM{pQqf+)4~b
z=HCu6yS8{kx^dga^FCK1=L$1K-%eYwW9sFsnT3TW9qkdm9THX%|ND7w724a~R^SS2
ztv@@v{jlAy4@X6AwZ6ztzVoTjvpFJKG2}zC*M6na{MjmOISId~K36;XV$v3cBWvm+
z!xP1M7Hlg~5U=c43^SLPoOgiJc;af~%Ejj8O|f<5AEK5u+&Q&uX{tqc)E;J!%LQdd
z9O-#neAnM;*3JK{8(i7EOjyTs_C9~hY2t^t&t6;iaaLXOZqEMP^+%@VF=Scl+_p5|
za@~U|aYe+U53AAzcirE1c-NZj+$9aXIV<B$XRHb?;(mLJ>*rLJWi$6PEb$V)eMJA%
zM&|H3i4^VoI}2{F?<k%6^^fu+o1IEr)3*5UQ8M7r;+VCrrYzRb&@KMOKJ)G7O8<ZD
z_}Sa0A=N9md1});gW{SJt<^6+JuZ8@a6;Aca-XW~>EBI4cI91pC;7NN>O<j9<t7=>
zVGb)N{&m~2P4yDr?z#<vwpHdy_giDyCdZtboupT3$=hD=gVoPs(Jz~0?c7Q6({fHa
zSr*#n?(_e7@$IS0M>g4+|7rWSI7~lw|LJS1xSvMvzUA!vcje2QtKDs{*vhVUF4>T<
zQ&yK_?a`~X&Ce~;_S)HTJe`sJ+00N+e%A)|8Mf(<T4HsVGVGpJUcGAKyBmM=bC#FP
z?$&56mARFjJa_ru4+R#^<reYtq^4|~;L`eMk<0DLo)0~<YgFfNS@l;r<-*+~J8y+N
zEK?Uz|Eu)>#r0=f*J{qyy~}@pzY^yQsqM?JY+GLQe?kfO)rb^7_cICZ>uqP1i67e0
z?hz;x7*IT+>a)Z|^>03Z^Um5En{II3?Ozq`P`ry}1&d^1Oj?iNW%0?kw?`Z-npge!
z0{`7(wm&cGRtL#WS}iU<ZR(CGhPA7VYnyBDT{^vO%9dMp5t3FM+I}bc=S>&$H+isr
z83SkY+UZNTS+4k}utLrzY(?vl$*=D&HQ6>LTA(3!Tf($oru$p(&8**dc;eigc99n^
zoO@)CswN0I-1*Cv`sT>x%3B*Z-BOjFx1B-fy`=g)o5k1b)Rp+^Y$dr47ag1YchQ@h
zANi9$#J}I~A^p)aDOl9Ku;c$db1##kgDkHyYYW4qGx{Yy+nW~)OIayQ+~$31ySddK
zF8R{jM+TkJxz<ghuf7{(GoH@(RP&G8`E^I+$2qBy5w!<7*PXfJ_0~;Bc&2e`^ONl#
zR|w|Y2Gvbw->ZHrCnsv2|F*wY7PD{Nn*98D{pITB=ZrP0-``$+QGV4``!$_rAHUCD
zpwszTvs__9vqF>atNn*gJQY%4;gIpzu;5@bhp?7N$AJVU70yW>f(C90tDD3lId|+R
zSa7UQLA=h!qDa}TWeLlMM;a4Cbf>ZAPMjsNY1!FUK{d6^ip}bd44e&Iyj~?4LYzWN
zb0!+43D5Lh6145h%ZaN_tzwGoJ!Lha_u&7v%qIm}vWkO!H!~evWFt|kpt(NReQWN@
z!gUNQnQop?T@kkS>VET!Tmf7W52m%Wa~4VN*zw_jCbKYiM$CjF_fyj(v-M^$B(`WY
zv))oVA^7ZQ8OQA@HYX;fdR?xMtW?#w_TWw9+LYy2ZmbI3dy)0_)=5kU3Y`>~C#v|b
zn6u!L%3OA)PQ3=J^DQ<L6&O~@YD{=5DDT!&5Yok>8*SDr>H6$#_UxPwlcLUsK0JDh
z>Bv1Zz6jlw%b0$cPN)}^43UltR;`#G%_G&4(46<MU7${6(bp*sLT#!u+WK-^7o@N?
zu4rXY<|??sqQ<3g(05X*n<zW;K?x!88Jq`Brf~(P&Hc}CT~x(UMEM$<aQ*6w;R10|
zy<Ze2HGFjw&~z435%&Ef8WNM}{x#%IK#Y*--9S%W(_>;OJ%_laXfKsq+RVM|iCRmj
z-ju-bh^4NfZS0)e7KT={&bSa=GVkQna4A8p19e55858Cent25XDf}}PoI2GqYC_5k
zH#VV^HrbMhB?_vE6LeQZdI@N+x_WZz#Ox>$p}O2B+XJ=V=89zNeq%5X&^;8W6~J_&
z@yg8&OpYxwMT@k=bsTRievoD0{d(fD!!_I2Yc@YB6qhS`>bYP_@RBR9mN;(Z2+{IT
zR`m>6vyJ!2I)?ZAqVLyyDs@R*-1X1$ffK*q*R;J{L0^~oe@~LQ_WvN8y3XndalM{^
zc&_ykN>djIO?k=2xp2jVW6X&;0^)OKt(vh|kjF(Xm#a2}`@qvxQ4=QiZ8{R|wDwI}
zz-p!&m%NUKx?Py#AI)_kbvmblYHN_!#GNN+n#PB4pW&MN+T@CMDr@lm97*=CE9IX4
zz8X?hYH?wnX>09^*!+{MFPb>HW+ix~J=fe6nj*eMbb{tDsi0-mk54>uG7S>Ve83X8
zJ%A~2O~vNbW@jrW?~ytATJO5F>dULSzb|SAtP$UO@l?*<ziJMr)wV7=>9B}zi>!xz
zczew2)1tHQyt-@4GOsM&?aA8hFV|j|lYTr&Z0m>a&DSPAUO7){z2l~}u0ie(=dBg^
zKhOVO%U0j(o7dVNxScNhA(mZi&iq!6+n#|xMf-Xa?<)2NDcn@ZnlvNfMeMH#Un9LI
zg{gXBr#e%6PsT50nKV;c_n8}~reFHBqXLX3?uD9M0jHxr+q{wWtX_Rm?z6}u$L^S3
z<>yiz=PItPSSZ@6!Ku(MGHC(p(u{|iftgW`r(CV1`trj26lEPtB214xWUDF=ihO34
zvEUmkkM5EHk!zD?Ot6zpG0X0L6w9@A=N5y)ya=9vMjtbCp|p!?ikoNr-LqoC70D&M
zUloFUd%g&szo-+oQR<Q7qZ6VU4+~5dE5>S{Dbiuo;V$QU!pZ#ig@e<hb7hwsIb3#p
zmAtpI-S+eU$dj&%Qk-R{@tH5p6#Ce8+*2t|HCAO}%<d;A%z~tr$K9!x*st(K|E*|R
z!g{CsPDk9mc@2AVwlZ?p?Gc^Tcg0^pSxRu}5`km0-{>p;{rxCulBED=jJlPocFZd;
z7bU@a8zx(n+zcsrw(4#~&}_Rq;+{)B9h}iIA*}gL2WR^vCxfj8-1;qCQ{MbAnf79p
zvT2Nm-O13&`ASPdcQU5eW=%RZr7^`<e&+cIk#9%UmmYBr$my(kHOXtQ!ij9&zul2$
zTaM2Apu1S{ljr|m-viUMjucF93X)RS^vV%asZ@LE!D0SWeB+a4Ve1dh)V*?Pu0zrD
zvKvV$kM>QS_iao2Y?hgt|J|Yt9q)WNmAUlMREIyAhh?LR6%ScvcsNu}YBAB*XR((4
zy5`bBw}X?aGg6;U{2*f6>gh3^T~I!^;N|2#J1-__uQOsoi&=SHKCEA;xNI5Aru|~7
zPi9P4Oz`;UzGI@MS>ps#4xu`i2??_@xIbSEbr2R=syOv))0z+I9#LL<3a_%ZtWk7k
z^5YO;u}NvQkMat9K2v&m?AKX4)~+g?A{9LA?}c`cv&$TJz6#<iJ0BSEdri~27wVC+
zEQ=L8l|+_HYrMS4xvL}a{vwA}0!B>7Ht>F3_g5p>r*X=4ErBgJG;Gwb7A$(L7H!SB
zYF7HGvrpU2KWfJ6UaFWdRq<$almq{czW-Jy6<=PgST}2lU3$g372BG_rykL@SGRF4
zjx<$S+UWbZ>Qd09b3qANe{=RtyRl5D{{4|(LF*bfMB7b~>`7!lEzqOBWvXCq(v0t$
z=ck_CEzqZUO~IgPVyZ{RrP)0q29A;^rm`>{?BsOes?ZM3FEP7%^$e$)|K(MoDP>b1
z&+SrP6dRfAeB1xwJLBzlzHX`(w+Z!qb-F1tEo08xDce>YaMdb1XgrT!bwSO>{Bv9n
zSKM=Q@)Zc|IuZJ7MY7|ii2Z-Nv|fgu;P!lI8I<@nb(>P&lPfY8)*t)YnHd$(Sefnj
zC{=Wa`5WKr#=}=~+5)yHc4h_Lwh@R7zG27^y4gYW(7OLm@`_B8Ss#55`7?cs-u3Er
zEb-oljxYRkU~9+gV(r+MYBLv?UN@Nd=;o4h-2x{$Hr(dqygBiI(SegUmMB^;n%0-e
z^t6$|lZ)k=<mzne3sWU_A4y-U;QN|oazDqZKJ$mUtGC{{+N9iarQukI{ri`bA8^gq
z3aMG@@UKUp%lf=N|DwFv=UObcscJn8oz)S!M8J1`$9DtQzR4Bq@2;G7T+-y3qyG+-
zmnv5@EDr81y|a#e*2%s*%fd<vwd!k)g0E;y4SAb;v-xak8khOpv!d@h*R9I>64m@i
z%i!aOt87xsn>I<@ylQQxqwDb|!FjjBmDNkj0?i&b`l#62E>xQuX2QDn=l_ZgZha!#
zvnLhJj#>F6tm?~p0f9{219QCgS<RYZCOH4jo+lsod`v$ZV)V23$&@*}A9jS?Sn<Bj
z`lMX3n?+mj+|T!qy?oXu$vMr}zG{NNZsok-)loWqu3KAuyE2jmB6OlYuHL=p`y|F*
zhnDvjW$Y_2%6EUhSIkoMSwf|B_wQ+ecQ4O=#r^!?_59X?s?D`;umAbI+|l-_kI`(C
zr{<qBS@zCcZ1*}M`tG3*^}j*_gZrmly|3_H+~HsQy1G__Ki3we3oU<E9m{CE@p%k)
zVElS@qwiq>{zVFcpE4yDY4A=CoFtgZsu;0Xg<<vwP4<I%&qE_BjvG3Bi+DCI!gx}R
zT8F^@xf@e>zxX&A>uyT6_h*wj(k`60T&cOy<iiz_lm#ZKMQVYCAzNIdvw~WDjs4R%
zwRp^^jAbiub{FMyZ(u1;4B3&k@4A<ii>=Bv;h>A^5~{Jfj$YA*)!dHN4c7t>nP<p;
z)a+!>3*S-1y0NC$NYHGDl0<|+L%D!zbNe&Hkb6ZYjTI(A7o6HWI&@t+x;#3(Cw8>%
z;ABiS78h(aa|}IXYCeUdi1%7t(V?maH}_?mRL+Wa{V0oBm!7ooOW5VhNg3@8EE`=#
z1J!cO3y(f;<TTWj{g}+;(8Q#WB4(Vz<j~%bX5p=-%X-o2+(v=Q3;}^B#&aL`Fg-A4
zQqW3M(fWU7B8zId(M64l%No6N7s_xJNQej4$uL$iX2?9fl)#zjnQhcM{Y5mJV8e5c
z?6vBNA2KQ(OR7F48ZYx`D4v)uRGPk6A|u?Rn{N|q+LT7qhi&`a1?C*`DXI{Dezjb?
zL-~+|5a-F1YuCKHgjuV1v@<bIikLEyeImn$XZlPU41d|HUX}|?$;ejt==(dpp4qTu
z=8Af~$jMfjGS4Imwm$3Qy)GyxCe#)*A-^q$x5AO*a@ga?Vayv{SIrP8I@-0`B5b9x
zV3)IR!&JkHb^*oa2)W>(o-hH97G=Fpxm=BTmkI?=3F>ktT4+0Ua8=f@Pqez|GU<+l
z?2U}+S3hbz5%m49!D?-+VIo{8SaD3yT+`^CN57$@@%M-{bH*vzArfni%FG)BOPR&Q
zrz-e;QUAHsY|WQaH_zr*WmCClDwUc`lz2>3O71S+A<$)JZhy$T#-sJh41qhF6c!&~
zd+}VjbLQk9>~lB^C-2u3_|xvXBsueKxbURPD#@#ki9cdiU-3GAOiiORX|0BNe6py>
zAvuA^2IYadRl?J!s=9?XDedVHIFQk_=A^vdiIo1fAf~_`HbKT?J9=aptt&dGb2Rpz
zVzhqg;&DcaF?xX@dr;rpitP6u*)HV*drJG}WyteIN%TlAoN_Z@-c*6FtU-cPtAjt5
zW>mDMB;<NG&d<$Iy1k6$|I^Zm3fs$XPAk~Fvia*z*FQT2?leU*D(T%4S3k0IvPI?`
zJ1+s-TN;1UMNi$-ILlf8ea4b&rCpIL!mi%5%{5&btSL~Ynr@p|R<qDfdQ#j4^~FUy
zCr;X6c<j3H%ag1|lZu!`EKY``Gzy!{O?2~K=5Se}!aJ?E<%(q1&!yTTlVU<VE_U>`
zF1C7g&4J^ians5a){E1eCJS>WyKf7aW2aadklq@0Vg~!M1;5#Xo_?y|6{h+zw3S0>
zRrD#jpF8H3CC^Q`p;z9h!F^)_%THI?8CD8Pf{TP$H@QkEPF)?efMu^0V-Cma!m^@g
z+kD(2jg&IAu0~8Z%aTZI)bhDHNj7ovfA)=&>}I*$s9JlAYssBoi&-7l-mRL$6u2a0
z;vCE3b+&>wA9tj3eDj|dQq9aM^z)JY(i!<l;a!RCOW97Uvu;$}q~RSlOKip??d>;p
zJ5DrrYjiw&SQFA{xT`{-`=jRT%#@nzg2yD9?_>ywXA2y?sO+(@cXg}Pfgst7AA9e1
ztWcb};>1M(MlXTWmo{Gf;-u@?FS={N;+Gl%uOvdQ#XGEYUY}Lq;JvCit$JI#nb=e_
zUzY~OEG6YvWhNUpvNy_bhsbgXu!Kl6cmyuv3^z<M$&*!_Z_c=_Fl@2nN8>9Maxy1o
zZ)RHffJ^A_^=)R1+f84tY%pA+rCAzoK1JWOE>vmO|38lFuV;3#x-LEKHuYxZj&GOj
zqC-M543%E8Zaws9Q|Yd$W*hA@UT#&q+O7QA-ndD!*HNe0Yk7H<fNb~1mRTznJ(|{h
zYWj-9k~$s&owo!}e4OX-F=f)hB^-g9FVC7hO=HQo6)WG**sOH8pk#LT4((R{?Ouyt
zq@T{7d%JYXPb-CR7cTGU32BNo3QF7RH%2`dop{W#JtZaLUiIeGEdm<ak*!<i3l&e=
zd}-2?TWj~-UVBYyo#U#>H@TL4f4%n6tF<4yCr|Qfa4y<YsI(>YQzT!+`V}*#9BYi-
zwMD(|puw_Pi&+B&S&g{bmRqQ>?`{p=owd`darrJL0S1Q!49pYf$r?NSpEWyH-J`*v
zBPikbyxThjE?p7`pOnHRs3RD#OO4t3`b7b))hjpxd#(ryHW%&=e#(6yTlkRmEPktn
zLR>Ss7iyPSuZ&m{^S`RJy0ChWxcSCc5?8r}CK;|)-Xy}3xajt4?U@(ng>0M^%W3iC
zw?>2Gwl&SO^W51MvkCp*El~NU`u*=?ZY!5^mgfgr?KmTr%Mr13-Hrn}g%T1xj@pa%
zh6%E8E?}|UXg|%{A#;}|XW+DcRtK$!iLFu<ZL=8iDrFZ1$(&x*{#mkFGv(A;>#a<U
z%X_qTa$G#z`sA?DjMG<`iY=sqIl?!aiOZN?T)b=d+F!qSzn?DP?IRk@v&Hs$v?ovX
z|JIEQr~lmiC#a2~x$xOz&HNe(zk?f|DIPy2$nZ)s{a9B~-JEk_6IC;AbFF$PT^=oQ
z?bh+a*^{r#+W$fN{GZk5f9yX0*h=6_&Pt`tXDlA~vBcy|_YC;ab>Pa(9s1X%@HNZt
z+r0F=^wf$Zu_Xsi__s9*EIe)hao1I)ipWI*9ba=ZG}{HaB_3~+FwHrA$76n5lz?9D
zF2_IG#au^Pew9mkuV6eS{7rI_v8~RNkD7Tx0c#tMyqa?*f<q!GrvJj6G^ZD0>^vEB
zS9&$HwaKlJTR5@9cG8+-fxB5|TrxYj^%c)a^F+4^(Zb=J`R8+{Fvd>!!jt_)<G^b#
z$N5>pNre|e&v^ZB)#+;d-5?TsqVvts>~gzTlP{i2JT}c~wrt}xCP!AIO{_mBpDZ}R
z`q*3fj*X;()AElur%V2vx9-ltH8GQxWxE|&B(vhpO>V|pmy=HCN$$Q>BC)?jbVcA9
zF*B9ZGIO{TR~D_ledtZyF~ioQznv4Nd6vk|^nE2C>v3`As+kGtSL1)QCT?HV`thjL
zR}qe%We<G>1QV{cGKzQ}(B$8FD0=mkXSs_DPm1hkt;(M(@cGR7zKh3yyxD*C*V?dy
z_Y@2Lz110Z#tQ%4d&TtXl8ayOaIKjVbm{`9jp8Pj<cd!@XTM!ocio`ZX2wRJ%ND#s
z=Puo3ay)dj#ZWZ)5R+nx!X*0{YkmI5a{79U$#(Jj%AeJlnEKGU#v*LtWsBHNTdb7w
zgShv7xc@v?>v(PS(^K1bO7D&O>{`cN<egn8ZK`-b$B;)l{o+K4$Fb*PU3T_edt7I8
z$xBf*_rT+byeYo6<|`Dp?8sJW=$Y%6WxHG}DeTwWdTl?pK!t~w8ZA%U$P{Bp$bBlG
z8j<KJuq)#x%M=NfJb^hrPv`6So|!d$mF)^Wx#bpfC#pq1G<<hT=Jxces?*(GD@6KU
z9AB=t;$r=$=SL1l?zc_co_qWDx3x3Q<oKRFQ|gvEvqn)qZAQ-W6xSO$uQx>AP<Sd*
z?RwJx+H%vst`V;%C?~JU;*kgmSi>~oji~0e__zuGFK&6dh4;dtpXa~6**(RhlhOA6
zow&Cja^D^^EWJMEcF<$TcdM4X{B?xsUO=7i>v!|kM|E_`Ca&VuOKxHe=lM6~w(0YU
zN*z)?`xpvTc~}m6PxCvs$l-F~;bVd+v#NcBEzQrIlQihr*vGS3=bdoE>PLEVhYjyO
zlG}T#Dx;@ix|QI<iXa2ij-PM(9zN~2+{603z<0WB%DYgud9I(W%uAE+R(@($=8}BV
zHbKZiR&!n&!vY4Co8^zCg#_nI3Z9?$%I2e6=A@SwPw%~YPwTygE0@g9qE4Oo))Tc0
z|NXe5vF#4Os$WOUL5__(&U4#6*c85C(_LA`bG?Tis^<tOc|;UgDE|-k6KFkG?)(3J
zZRErA+ixv-`=Wo^t!DmRlcj%j^KZL;aE0-mTMb^iUW%#Dt@J!<)3(ih^?T3H<#}f^
zL-fo~Mo;Q0u=m>|eeKgUBV#*9v2)x)DaE$m?+M*M8MAl$w<hCo`xmn&Ok5qw{fTws
zCf3R~y_4RwJf0Qq->TNBSYcT3`Sj1F@6LbY`TH&V{!P8VbGRgCW^LYocb&t^$|~=(
z>+7dnd$Whv_I<&=fGs<o{FAbEyV_X7C;24vQ>3<*?(JQ*l0Uf&zf3>Z%OL1*;6NiM
zo1fy0fQL?P+=6^}E(JVt=@J$c@yQ5iU}+L$NKmN=Ty(5cUc4)$<I)m_$$}EQbaoj1
zSM-|29qV=TNSd%_*fy3cSx+}Vy3qAi;+m4yfft9Ca?Muq_1d6$F-&M7*9xnUjF8Sq
z?M;uQrX{Qn6geGs=ZL}E16z~C=ZP|GPrSV%(|qF2+2Z;$byGyV6gvvKryLBL<P*XW
z#NfHTmzCwgj0xRGPWGwVi3ograG_%=YsI&m$S)a}0)=Jdtgo%GWnAOEzfIQqO-S3@
zDO?+LzpZ++FyZD7Ra29Sh#jft`#dAeb`-UqnW?@+yFp~;%d0EjKi!f#`I~O=HlA}n
z0w2;iTb6oQ^vqHaXmjLPmVRdT0*59qsjPiARwf7k&X+8D)vx$9EXs5550S~48qGW^
z9Gq=gQW>g~{s+vOpvcbd^wOy{V6ubpE1^|O#iDy|y<Kt4!(^kz$}pEzn&ROqTt{0n
zq)(Wwf5N<Rk#wWiiPb`qOsfsoH;FLlC`nH8UbJnR$_m{-7oswDw9c54H)ZdaSFhKp
zcXnkM=r1+k*}LS*VMDE7Owt+$7$)DEq@g9kbaubFZT07~M(>XVFH+f}(9si?b+T9C
zdX%!sLdW!!j$;c&T&Jvbkq^DHLR{4Lis3?)*1+ReQml;@nsNNv;?ZQq>ZhdXKUFVO
zbN=KL7Qst*pWF4)V&|vb@06T67RRtUaBPmNoSM{?aL~pdbbjgqTZMB^wDz_M9Vt*z
zP^r;TRNzxFUc<?9V;VE#^gU0fGN1XBa$*UKNtc5%r-zckk|Iv0?>8Hj78Kl@`9}3b
zOr`RY6?a=LSQHpFPZ;Q}4&(cFQv7(`p4<f<y44j5ZPI@=U5i+6c&%AevumP;ruwgn
zMSa1dp$jLlH)&fPU!fAY%dA3aX>Xv%Oh4~^haRu;{=84d=Z5Aaj?1>jnpPhk)tEmP
zRqDDmhhwuT=b`g{ZExNzTGYV6p!k!8RfK_wL5G2XfuVuPfsujZKLaa=$c?N8s~cHa
z<z!B5Sa`TCS7Msrfu{=(1c@0(si-!H%nW2Y$?;@Gp~B2ErdKzUZsheW>v<EAIz@9r
z+R51##joz1+`RmJ+oOgk7R?DO7y@~k|B0;l^5Vd<$82t*7NHDR+F#d%#qRPtu&Q*8
zOK#%v4bvE+-+DX>slJ?cXcqI;O{FicEjl|Xzw})0iD!xNbJpDZw^eg;$m+0Muc<e0
zPYCurF1a&A#Gx(tNT)EXUyKIB<5Wgr!z>+%iwiy4C$le`=6yNi_{756JfWF0A1~Xk
z&3aABblGc%DzVMyWIQ4od}hbia-Vrsm2iF4g}tw8?g^w^S?BA0PRb)&GiJ`InN|Pg
z{o`zc-YcBvzVXUn=a<lHtS0Fe=gez#jm~mutk|Tya^I32W#XPk?}pxM<WPGNtlc&-
z=$KkePGOX@(#2)BT4%*ZTx?maxnhF!BHk&IsY;VLq$+zpy0|%hGr7X2J*y=3LHRMk
z=MQ@=o_WND-HZ$k>3<Yiay9f#;M3#@?(5XAIvmnmsqFvo>WbFvty_}2Z7YAOOlmZ~
zb>c#y@<ovu@zS*?1vHbyjvSj|Vih9btFkpHrm2y&Ekbj-quGRcJRffcyH0v)@~A>^
z>%oV<OZTRwb-A)eTvFqa)jE(Hxh?6mMv{POdT&W!Y4~I}x03>ar&`6QtoYQse4d-{
zO63b$>&*VUcuzibcG6X=&0JUJGHkcox@02H!e1)>tNE71ZRC?&IxleXnx-7*=Rc3F
zsa_uxx~kAujs3BP$;{Six<zIdnX~5<S;hNBPO^Tp%d&Lm{2kFPBI`ZX(=-IcLo7@(
zoL5Fpc@h2cU&yQ^@!p7IZ8KY6$hyn@eIi(|cc|mlve|Z=S)RUb$KM_E`?X?Y=yRhI
zxe2a;t|1eT*|iv*65H6bvE69X;~?#?GeW)3etWVlBz{uW%?r_@zwcbzRi7HN#g$>c
zW`J~?;<XRS)lR*^sZ5_$W=RWwi+Oa(`KBs^G|y)i&5XXAZMP%eUOChy@tigHSyjR{
zk<b&K=iVxNs1(l;4sx{ppDlQQ<*&8hA27+z-gJUra^1y{D`wv2wI-XGrZFT<SS0Vb
z?TWL2`(l|>g4$V}9zR}imEBZmW;!$b>$Qm+_x;+d_G)c`Gu!*!?;a}_F0u4B=o1v)
zz2%cu&H6VJWkN!2=ZSAR9r)BXu0-#C%lH0{$4hEHDWuL{HL1b+%AQjXCa+uPyuHno
z&0Dk4TdtkwwCA0KLh~o<E@iRW*6;i%@2gPQ^XW<5-1fw%WP=vvlPp~7%32|AcMK-D
zI36+Kx9m-q3z|_Bx?FvE;At_YZPL1XmS`+0>fJm|$%1udlEMz<g+F#(6JD&*9%<p^
zwUI@9ZN}6!ZMSzn@U2o6D4VfV?#l~y+y4=UU0N^OU8gh23Z8ATT6pzEopy|3^0z|^
zw1P6?#0+oj+4Vu7`Kz{y$WO_)Jf8&Ezoi6-9`b1w5ZRWpwv}hs<(Uj|OE?W3)>&S#
zjNGv7(JUda{!k;6X%=sGOb)a;UCAfN9<-~-=<Ux{R;M;`C@u`$;b6t0Sr&7(vTy0h
zLoX+4b?rX2iEsJg9vA&~y%N9KCyXZCHC!UU{*Al-3q!83TW7p<e?<juF`K8ys*@0?
z<M}_wNaB&fTZ7aOljpN+@~JC4pg8A;(wjpkSXUMDWN)0jY;VG20Ta=sriF}moj1Ig
zuyO0coXJ+wQCHL&d~_~OIw;bic5X4JvyiKZ(VXQrtPB48*KzT+sb1!A@RYm|VrkUQ
z{4>ExeAoFlv2zih-z;~LoHZ{<?239_jfM?#=E)~VmMc!rPW94Z@@}6J!ti%RyZsJT
zfhh~tx+Y|wjr(?eWyP%!^-vX8%}1{1GWlA%w=EXvRNHinJ6F+3`jvQkTKdFh7M;f5
zFO%Mw7%RoSI;$*`8R!&su`evgiMw8V>7%?Q0j*Xg^V6QHJ4;UrJGJxcB+E=~d-crY
zyR2N+Jz1IjNXjXExx%^pQ<M8OHi?T!TzZ?t$JobkX4C4nFNdsW6+d-Ic-dd4Q&=9l
zOUR(KNW-G@v-N!0_&X67m+sr~K6lB|2buSeEHip}$=fws=~Ca}El2)G1T*wyKJzc>
zmOA<2n)8h<t6aA&S<!apsovrj@hrD3G>y5Ah#7S{dD>ZU?U|ImhX16C=a$z3@A!`F
zKKAHc<sR;PZ|c79VNzT#v1K-AU`O~KjVY^R9{vufzjS>)&&%hm6N6&AltfyY_Hsww
zP>eikbVQ!tQ`3{ZDWTD1hq_j;XKd6%?&*3Pl~2}+yMI#JHM`0}*;3Wnf9uIz23#(s
zf^RhUt6Z7T%yLyxPj9ia#EY2PsSDyP<oG?+O7?~H>MBPpjm+b^bag$~g5y6$J0#9<
zg*BQ)GWfqcIeE5E9Lv=2ih+LmRfR#Ry341Z3jX)WVTI1qUsjRx5@L&-F4u$~6g2w3
zn0I21zg%`u+liYS?OwXtmpKccU8s`&^ry~+D$k?*(^GusOuB6Naml0h7oOYCU*YI{
zzjfjLDvupJ4-dI`JwDr45vF|R??$&Xo#$?<q^Q15IV!QJz)od*@$U6$?^jDQ#ckMk
z<XXTs4;2=LyRqSw*LPnt+Hl@FC-HL>!|r!qH%{(-lg#b1>%1Jtmf2_DWJFirWM<qZ
z)^v7)PP6FsdxqI+*G>zE9jjcjzGTB(!9NM5yQ3N2rhQ0xK6mouW!889tvKG=KVxTD
z!E1ZTe>z3V=`WhfRhF;x{t;5X=+PA(t^hT$10rTE6IRHnD6w+}Ub0v$ATa%ziSn+G
z#-0XerSw|^cl=L|&}Fclpyb|Q6F#rhIYqWOR(bcy!_Uf8HYo_q{cOW+{oe9Iqn^N`
z_ck|N#2L<a%Y8}vZgbmbx!D!TOC`Mp)e83EyRL1X(7Ciaa;??Psg2%A@8`Zbcdz`J
z=dJ5;@2kEnmfvTwN%wM_S))wvX0=V83pG6^a4jp-b$vZa==<YsrlF^-8B$!60=_@K
z)8%ipeX8Mglcb_Xg9+-Br_Il<J;dpG_LyF>$Dxf&t>0$E?^Ri<?mtcTQE6L6?5?%v
zPi#r#XT5bh?qz_6)t1cx)s0{FDenAUdvW8r2@kg4`*AS;+a=?lLV-Kl)tJkcHc1QR
z&*}Ml*IcvOe@lN5|J9zQW{=PR_nP9tbu8w_A~yNEF@Dbl#rR!HC%<&sCw0h`XQ`Ug
ztiX?%LB%28l?yc21thCHcU3tqF!i`_^~0>ip9J1smgjjGDyYDE(=2smdFtwR;mzg(
zLKdaGg;oB~1@4HkURYLqVOkZRV$;d#)w`$FsB7f%GqUPG7p^K5xpkQ9*|rEP$MlM4
z0#+-^m@YP}GPdw<w6^tVG5-|t(~M0a(73zRJAYx!L?eNO&B8H{Tud2VZ4`xP9~CH{
zF5KZ(@w?DR=3;Qao7_aT%=hUIS6N!0K4jCMCb;}#iz=hvpJqoU1&0YqSv(V^*Ep~(
z+Ab0)R&C>6wfK3{bv1^14F(Sbfko~eO#e4_X#NoB-qFG1*x6+vFl~myPV>&`7HKb4
zxn?bBxO7n9dU_et!P-(b?^P>=S4X6;v&dgDBVkj8LPmRn*^c(D54az-Sj|0L&XCsn
z(%j;WdXmyNfm<c@EK@S278RV)5D+#m<W6^!2^YG2LqH*j&80M1NnD6sSkq^#)AoZ!
zx?837#XF=Dn`{M}C0^t#p5FL&MMoe5s{lhM(}qp~1qQ(lolJ%Om)TgSRP-NG7x)>`
z-@c+$M@43f1HWQwx_W4>`NT3uj)|%V%MMyhR0`#<+9qB3F`YZJy)4}^SH*8-Xe-;L
z)?*tJi<F|hf(yl5d;(J^c?7o?EUZ|f;P5di)AGNNhUi9CnFEtW1br=b`r18gYg&}4
zz0-h4F>Aks(1axEPmJz-jIm25bj;h)w^L0ZXhKIwWzM4M9hbLFt!nN&5Z=F5yi4JT
z?rH~ayC(udo@JX)2&;>Bi)z|UIm)NwTK;>Y<?Uec?~|QMiq!rm7aTqj{K-flIAMlr
z;S9coGxBfr2u-&VG*av=s!U-CWfcs$G(*E&P2l{NnaqI=)?)rVm)ZC_1kN^=@Gk6o
z6(Ohdqlx9Gz#j{N)tU?pp0gQe_6tm??mJN`ry}4r!}-mH2>Xhz8|>ZdMe-dYYI7^|
z<qp*r?G!oxbAtJ!vg%{HJ{c}r4};V)C$&#<JoeP|yRiC~|C{ACJA-Ws8)_VdgFPm%
zZ4mUT6uvm6hq<sV{e*x_Lyx+efV4p&TT3Q;ldoms6qdrkD)x4Tg<|h5^_Ud1W(w-h
zJuIka>Ygy8>VDYNon?(NCp!YCuwGsk8+9T1T>G@$PGU<X{Q1ik@jskk-MHw;O2KnC
zYrkF-*eEgA+$r*_1Gh_{Y9MPy!Y9G`huexREfm!hBUlPQix-}V5On7%?7AuNanoc;
z!$Nn}%5o34={Fq$QyU^~aZT}*ysaTHFS0>lx$@b<0I`jnf(wmV1G!Zi7ur1Rd-I}W
z-}KpEEBYBYo3}{TY?jRSQsdg6Df4XE1h<YL6R-TXSBu(ib%|*Toc{mPt6Z&x=|jW@
zmX&8bSBgfiFkhJd&w+nmu$WdLzrjjjwM>PlKh5HlTg5_zm%Az_G+VVDoU|e;X1%Gf
z$w&L#0*Q)875(iY=Wa|EImkL!d`j{bVHZ`6XelfAn~Dy*yuCU@Ei44KeXtgiNLzDV
zAnHQD?ZN7E*Qz2nq?Dz2d?=Jz6Toff!oS%i^24#N{F8I+KhDw8%71@s(Us|os+w0i
z-tzi;K=;`(`F#O(??1)QS}w@U)bo9pu(OnA?>BY9<$|2rg(A=A^9UyM1t#Y*D5+19
z%-G^9cF4`j(R^;Rkk5_s>%Ro#Ud-BaEqg<G)vd3NImLFX9dz#&8k-xhJpOaq#{U^B
z?;WhQ*xj5bxnArce_^|D!SNNcD+O~oSGukcoRGEtHP_sUtMb<!44t)9X~nC##WR-P
znX)SBN_k$=Jo7*k5$)Fbn=?`tOzEFi`G?83Z<^HipLHUFnZX+v-B~pzmMgpL6rL8i
zblI#m`*tkIi}q7fbWmelc)nGrQL$;>kCeFZh4+sY-(g+!aF@&$gLRsME+1YAcimcK
zz%~8LEMb1J>2tlUW_zvUyB5mvX_4WnEoU_sPZmwoVO{&-8khXRZk0rl6IFq%-O3!t
zlU!Tp3r^Sn`N{Rb?w$ayr92liE<}VRg$S5@6Nt4E;5j0>uAs8nspy(E!#qy6s>jo3
zA7ER$bJ?5!0Sgn{J&!Po9b6e<ZanAi@?D?RB1^W->AAIec6R!Uw9TJ(Cv-^d|COD;
zszYq;1n(c)(o3H6|9il-v_O?<f_Gti!V0#0mN~mE828(EOuRZ%{;@`%{lY!5(F*(%
zRhJzVPnfZlRZ4B$Q~^^-^+%PhrbgRLXRZ$XT=CsV|M$wCTP&fE3%4^lE2vD=C~t4I
z{5Vhk<C^oPYPM$Uo@z0u&RQpMW?g&LPW#y_RG&`B51+HXWB-+QRhfgeeY5iaJ>c_w
z!aXaQ|6jtcV+$91zft_TEx$We^Xr3r9wp(N2h(?SD|i|&DBH8Os(6Fi#SPYlJ)c$z
zwy&Ha<+JofW|5e-P)xx575@d@?pG+Jt`gUtBsBdpmyxr`r&Eh2U7Zlfv)|iVZiV(1
zzv!S1OD24A;LdaRUcF$KP)!$y8&}es3I8_uePcMrHb+v6F(YQ`fq%2j_+$3_tl7Dr
zXL0bKL@m|wd0ct|*(TNAj(2YQ?z`dAY-VR=sJ!$bSF_``6c0I-wr$_O&#>|IJGn<B
z_ri=nMuJNe#bp?$s_2}JS3b^MXt$7gs^F`jnq|WCqlG@L+L_9J+IM!|g4z2&pX%Cs
zLFd&(*4xWiBM)$GX<<p4aOCfroj;$6zRc$T=D=07CGXpTWg>Tvu1mPcq<C!UgwxC)
z`D0y$zULf0aOa5TU*Y$&YQ5&HGmGNhZg{@?|5hRCZFMm(#AgX@h>)Hvu;#eboef@E
z4DMP2uD(|~I8Si4@0lQJ?Wf9cS)<qDp+l+z+pQUyH?4)9tVw3*7M5SV%-4cz@rQ`L
z0+IXQtSFo&t;@9KufWBX)w^E(>MANLd$VBe%nvC!!QNXHxCAC}9*PXws<1+1YJOIx
zSpIYVHwVnx_NLDiU|o6V+SdTCw--*gNUhAhEBVgh!1fLDTsizp3l1;|Y%z6?n(4DY
z{Y+OuIDc}2CG*6{Zw>P-4Hfk~3rtxL$=}>+8LM}ubt`9~xUie}+sx=)UD2<bwly@>
z-=DKhmGQRJ#XVMv3ua$G=znC=X)d9&QQMqq#JZ0L>7Lys{@*fX>jHVl-xsW!R!)C&
zq{oZ<<BHg6*0q^EESzr>?mv%<x)A3Y!vA#vXXFCTA9M2;H?RaMaJ8T1`?y0p>CY<X
z83#n?TK=10EVuT-(gJ~R398wZx40$+X|oDVZ_e73t#_(4<{xv<K9<$dGn6g226O!0
z&T;7M<CYmsHcN%Og~OwI1ODz|OBPyLD0Sv-E0@qo#oIxj0=ZY+K2XN6mpSlBc+-Kq
zvMU+qrWdZ=uUU9$xrMif%#El9mZ*SRvrh5-)DYG@%GEoUGg*P_x)kfI16;qvI0HBE
z<%e_6oG?4^0N>QDTmchUCtiK9LiuQd&ZYk!ZcUcGHA`8_Z1d63m>1e-pM3v6`@y}v
z`zx0PZ7|_7F?P4o>RWhAIA2iEHrPaIqRIN;rA^hVjMkm`6db)Nd)u7!o~|CnmG1;1
zN@m!3`s$oBoOgKp+e_#ACe4>JWD9t?;OL8>jD^ycDn~y{mG68Ql6Qfv>;aeSyy>!$
zWrd#nUmrNLuwAyhn0G0C%aS{_dFt|annVH(RR1pEEY*AB?7_X&oGbH8_N)Tqmb*6s
z3ylBYz0h&CdJzNbCq1q-yQ5mA5>kZ^Ci-1m@_<WA_1Ig1OSA7jXUM(x>eZXivju0)
zUa@4xCEk@;DhKm547q%ig_R70Hws#(g$StryUZKJ#_K)faP}3C+AHo=OFmv&Qgf!~
z#Oc)!|MOhEq`P+-yHxX|nUkE@jQbm2GaE+!QgXd=q~u!ygUm*r5T>*TZ0kO-F+KQl
zl25P1?5KXVNY=DCXY*?_s&{T!GL6%6vqtG+kB!CUwp@27<|j3<2ngilG(Y<<%X()M
z-@{$jvuCk-7_j9Xzg~AX;qL`m0R^tjC2Q9tM1JFw>5JvhwC9?|T+W(fnUr&L@>A}J
z7;cAd)djry|5o4qAXaW}eDCSOeVmt8Y<$9f^`v6vVxd|Kx#k!)#U{a}1_pCK3T)nd
z+o^M@Afs6@$04<x@_Qzewna@|>UU?capre($!LY`Rn3Pj86}UtlK&qSpduK2{{ON6
z8-=geN-J&^$};Bi{C{V{SAi`G`Rr3VH^wHgr9NOw3wX@2E@(?jO(4UAq<b}-C(plS
zlUeQH(mVASmndtbz<q{?>*cf=1C4rxOjjj{gnj+f+0^91W6j9)%QB=x#miMA(&g*s
z8|&lMG-my}cv)#;+|e#3cg2fN=kIGYZ*_5%e3~CC(q<FBvE;@&QO9WxtQI*}SIuRp
z{?DQz`r`b&^~=|GaV@9_eI2$gc5fZ)>knEdR%>scwDyL-f_AHG^v$N-E0ZMsQnpM=
zI=OPu-JPYZQxb2s2`B7o+52j3#}BO}qvpS2Y|=NgwS%Vo-1KC(>m~**re!&QW2T%A
zJ3BkF?%$qP@65G%X1mwjc(zJ=TEqYEML)V{@^D_>yZ=<Kc4g1iDyC_6xyQF%yB7Dg
z=I3tJ2YjbrG9^4OyShx|v|j4A@L0V~;?`jzC)1rKbY8ph=yh0cl576X1EI6F^axE}
znIxi-^dk0;CYSH2<ZO{Lm!#I^fmtt??~F2WY*<(qb}?gaT*RxT^I0wnC(Px$zS~2=
zFk#hmM-G$34U2ewOR_e3WPMEEso`>xi7QNMkERgEqKI{t^?x=lHLA{8nQ=mim0^-B
zTm6<+(WQM==cWs_E?QyHt+h#L?Yb>@PMun$m(r<YaM|dTwtlzOX6^cIQj@0Wt`TbR
zSe?^&+3d2%#>?kUvpD$pAMFZybAIiYOWf=4JhKY<&v5h3r&9rDT1iWLPemy<o8I3c
z_4bm_`dd?&&zf+*4GK)YEv#exP^3X)gGb<2jV3wM5bd2>Tz(B_SPs1Qoz}5th1j+;
zOqpk=brrXsc=664dG*_zokvB(PZgZy$+#c1dEWFbCtj`KJlf&H&A6q(iCe@_h@n%*
z<Y3pdo<E=6g*6sMoL^+>D#h95GcEh&A`QPQ23#spu9N2KO!%?#^vsGysS3SZd7bkF
zZ_09X6>Uy%Ul66#c_Mnjl0`k?9vfF3T__Y@bVBZN$COE`ll4W*)~tQTYn*1Zvt{P#
zz{o@44Rf@cN`;M<IEaR?-liQIa`?@j%FP#LnBVMPqP?icXt}%V+yAS5?Z5gkc<a07
zn(XchD+#&fHSw&OUWGQ-+b*N8?_RD;T3c!ybV7meRo5hG)*}XUnKy|T`y_UnExBv3
znD^G7kEyGK=3lDxmU+9h^<Gp)R|*efMq;|?B$LiL3xq$4W~yt1%`CXze8bZsqKZw<
z`ANcSNyBAd6*_j$?^YB@_1Pj&Q$BZc(4lXKy4@<vr*ONvFf~<Aobw?z@WdH|)MHyu
z1ynGuX%9+MIj(g2+3d@E?s@C3WIFvgHj-<Wsx#weZi72J8s>BFHdAo54*nbUM$1On
z^~#1o>z!N-tf3R%GHg)#{kg9!#aVK9VXG+1%J{!tfowZ@F674ToW)?)syS`SvH!vI
zE+ueWe<o_Jn`Qj3n@5~MS=w`NR_FvHkyUG)x*`fy^X#~#&zm)BDVd%OXETZ@`;k6F
zPsN*I>4T{!W?b5|{z|h`q{a;Uu)xKk5hvxpRB-w&d9l7pbh+GX!3$AFfs3U&yT!h&
zaPRfGWZS+&o9n9sr_iG|oBtCueXJfXa=u}>sF?GBo$N&yTZcu*n+)D`F`bZ}6>@CK
z>NP$gJ~cO4*M}_get0yXTk32^#O9NW8gjLB9xn(p)9?&T{WMu8@L@uc@z%30y=|g5
zr8KoK&GgOH3fb7>bJb+Utg|yVXzW{aHuz0Q)7F)}!8@3i>7;I*x+Augr(&b~xhY1L
z4UJ1IxBbsxPdp)}ZZT6KGOwUn`%OakkwDjod(x+^6b!7oBTmP#FJh1WRJ>05Vv~I?
z<H`3bj|6TXEtdJ&>7?S;=-QjY?Ya8)^3H%QyX!iXY^{?!?SC;IV_kW)^V>y#wPOpG
zEb3U|@?dka`+22H(^^U<O?FWaSCG88SfsTk{3mO8PGbAn2ggiQ6_^c8JIsElF4JE8
zrc2$h`IJ-aGozW0j<9iFZM|5sh+C9#0VAtI=sFM4hBbdqo2ee1>b}$~$a7@~yJc$d
zWrk3rnN9N7f-Y-D<UKG>d+TeYy7JJ5s}sT|F-n|&_iAe7+tl@WT(^SO9OC;i<%-5P
zhDpck7{wW9EbMUG!~8#YMe2=-&pvk_y*zFA0t2=ZP6wVv8vU<Mg=#xaQsG`89j0&6
zXi&B!Md_f@nw>>K{e}mQ=p2r6-N6$zWow60oQIH#lx3CZ1U22I=GO1hYK~6b#P~IP
zvDCB64_5hX@7*-*&dJm2FE`2V%sM*t<>grs>#Yx+FL^fY5$}7Ajcliv_$HdhZ4OP?
zrWAfF<7mrm9j=6)otpa;d}o$@J$q+C=DxB3VQ(Yuux6LC?lOyO%yA#m9^AUXC9yNd
zTjZvWef%OD!!?h3ZfuTK^D|L<lG-S@cY*t%7WUnJGAp*ecbcwtA|g<1?(6Me79H2R
z#&~fC@1!NS7q1I*&=R}nwJcS;_i}K||781hHd9s2o-!F)D>P`%FA6!LqULWJ(dP3i
zb47RNeE;o5lK&_D)L$jy>u>%w#O(K!fSOxeq2`<CzFn~?H|lT1l2_Ncb{()hkzn|g
z!~W;W?_3AgK7886x?d^$|1a_8kG7BE1UtpqR=sbyr`zSkP&0L37)!fa%3;3q?ke#c
zpB`7yoq4<`!X;zkqJ6UkJ#~L1PimOzEwbIU+xdke*Hn(visp?>&$yKI?3+$2D}8i-
zcHye7?~55aUsoJ2IGnQiu4|C?J?l@WPaHb(Ztk!1iOy1b`)%`zg3T|42_Fb{fBEF7
z!?BAmR{NyhI4v5`xpm5lE!%H9rD_+>erBaM<t)qnzbXGOPGDU9^xawGnL*u;eTqdw
zw-|hV^^v>UB8)*m%z^V1!;CPI@7@_gCk`A@@R+zoLumB`|HsE3Sl`Zcs+{QE-KI2=
zb?(OI&2?WMYguI7?B;l|O?j!xnyVsq9(|qK`nj1YNm;YXrz@NiP5i!?(OByLx|`;q
z%Zv^dc1i#Jm)czI#3TNp-9F^h`KFCk#tW*lv|YQ641HRwf?X{_r^)PD6~Ay*;+yZX
zp1wQId$MXpetbLILDqP!DC5J2FKcHxEALUtlduR^s&|Yq4Q8laJ2kdr6KBv5o!VyC
zB`H;TD$`~><mnZ4C~@guR+W0ltEnbVWXpuk$}~m0?96ZH`70G3O#1)SUahp|L!%DE
z67vm{!yOmqmuS_T^qKeb{=9VyKCTtt$;{ReIKS=_zl7=6x>nPb2ftVKTjsxNvNxH%
zsq6F3Pw$sc>DA9-_L(;Kcg;@Q&0<TA-#omx;u3Sbr%GqBf&LF^a|fko56u>MbKiI=
zHdBLd;)#vV9hb<Sp42}{r9awMCvx($1#3D^Z!=9)TJ>U~ig3fPU45@l^qzRkv_@G~
zRD}1<C9XHkySiR5lqDOBt>L&AVdrPK`q&5Ubq~5<oL-;6b?Ddz;Wx%hFJ>6qFW$X^
zYk@_vwAE~G7NyyoQVI$x=9{b5>t5)&x}t;Ez~sa3DesuL%>`B&I?oR|ICc37MXzkt
z|K6{(PEQuy9U;>CU{By`j;<dZZ&#X4{m6QtVvTvjwABtq>l$>=oK&B7iL3Kd?-MDP
zg}=8wjaa82Z1VMWm&_Rp_2_*vC7a4mPk#RT$g|&@igxoz>8#$_eK6Q(yTGh<rNvI`
zn|Z>$W>s-7UH4!K_bqe18|EhyR2!;oUTd(<iQ2brv*?yez1`eKr*C%ozSho|w2Vo(
z;hJOjw1<-(e`Ggbu*=Axm2<VhzsvLbtb}K9?#d2U$=jkf{ouT_43noMEn2F)@AJm>
z!fRZ%pB9n!aeDWn;p4932erARYaGS*a7j#2@i}AledAOgP3K@9*Tuh#wn+6#X-)P2
zqq)^s+frF!PxjIOt1T>+Ph53y#)7vm%$uBT);?f4=xpwIdx2B7qu!G#x*wdjXW3{d
zZ|}8UEi`HUUWwKSLlKLfo_-gV-7B`7TReO10q4cs-1QaHj%7)GetGQd3A10ys+KoR
zJSKa&7)phdc&*Yg_kJUv&gI&{V`sh5>7}&d(FGo=HCu|AJ@}+}M1}0vojAJa#dhD9
zJRA2Y>=4@TQ)&9)7MC#39FO1KMkkD>DKP#JGVv;z?R)v;q76&>g1zE1=XzdpP`oVm
zXf^xl8OvUCau($*I-AnJ#M>cPaK83OA&(>_r9}6&1=H3g%)isIiMMFiY7x=89-g2#
z$1ARIt(ko=<ctO51KuTpJ%O7R{NEZpD@4V|WcI1ZJ6vYbQWnWh8w}i5EV91eaA>tw
zkA2|gP1&bkIq5rHUhw&{Q_B~p^BFcS%Bt><Pm634Uew_=O+&{m@ceg?2J<^ze!o@S
z4=O%h!4ap};qYpkONh$G0<H}ndQ&%<{oCx2m!p!`bY#yb?S(-nCe7Ksy!*nz$3|;4
zm~Ty4x~1jpY6Z2!Z+gqEJ%T?T7i$SvU2U>A!94hoonz$15FS50?dG+dQ@8Kwikf2`
zv+3f)Df@IS^mi=ro;8K%W&!KI9=%OU$8HpO8F~2lWwZV<?u)viJoA9hxi3B?pX6pU
z$}b9;xlDwi%;s#@3GWi?*>@T|g^r#r-F$H6@8kcUJmI=|r%UL73(wIDix^IbTy<|a
z;$+s#v*z%o)e0wrzHn_Y)YkvZ`#Q>RlgyMCM?9tpEO@i(gnQ_fO;e1umT*5jbM))4
zU?JD-OD-Mf<MQ9}C*;%PK!G9~eo>Rv0?yvuJRvt%JxJ)?_sV4c5$<!+tVS84RtLGl
z940>e%vIoWps4uR(+l1LdpB*q6(Dqr%WscJhsjm%mzSSdpY(e=KQmOV)#R!&k0QgW
zT>@NwHP+@XQ!fi{4G^_5zkJ~0<rC(<J1>?jy|~lF&SUCA8L5MnyhnPqn-^X5`dR50
z8Fl{a8IxmIBo|(g47;&HSMY4M@RE+G$&U(D+@n_N&kcXA!L{6w=f8?hS558NqOC@6
z-*B1l^}3V7dGpGJzO%b(RRY)Q+??|Evi;pDd8NC9jFt-IT3_@yv}N^fkJX!9uXb(v
zvyJmri(Rmqw5@sEuX7)MU72>Li*p9k(o{Ql;Z>ru=B?nJ6`sTzwzJFYa#wg@hw9u=
z%Q;u&L>!wIY-klazUghRKuYJHs42FVQoD@Ixoa=)$lYhTwj+V1+sZcVVRhI)%kXt9
zPG!v=8x*fR4hiqR+Fh&b+MVjU%G>Y0j8YMA;Q9xi^QK&Tr)>Y}4ewr;OPzZovQ#g7
zC)@4V*>z^}R5ex86^T0x69Nuv?Yp@nYQK%6cT2GJodsI2ne1fsC%f8BX}!EFwe!Ev
zl<4&W9%~um?(g-r_KmzNYBELEzNKZ~*#<|kl(-EuA~#r^IB2`i-CFI{<^Zc=qm`FV
z?7Yl9ulLH5LMf}}o+zeO^RjmH%re*5tT~NmVSwuS4|mL$o(;(qwcDQBxo(mwTht|=
z&z?I%b_6aq&}EQ_KQNv3Xp|z$s>L7g>}ico4GIgGlkjBn1FmBUkDhYPz7-vGRw`J?
zd<xIe2f^Dni`-ul>1y%1@6u*1x2tC*#7^#OxET2AqJs8y%^xDN*Ak~Kxc?!kcc;jl
z6}4LzE$C%!itT(GbN}qYHlIiKo0n|=$>lw%_p<1<upfS1slMJ%F0Y+oB<kbX>l)W0
zlkE8U#_AKt|DQ;RTkz`c-6;>$7O$IVn=G4}^zs$&bl)q7f2{O*#AFuCqhaFqF4x+{
z)#6KOPhesEr4v(h5AdFKh&mY%wXAe@TBz^EhFjYYayLcXUGZ09b-}_5uT}R*w7LiG
zc>H3jh32gdcM{B>T}b&FBXViq>YQD|O$WJi?S!Z9F<+*q^Em#@*;!o+E?(%kBP$ZV
z;p@TPyY~gM9_}|_{$RT(E$;HphG#MVCQHR#KfTub<Ew|4Z!l>xq^T|ljNiwVAob|P
zgp+r#JV<Ba`QICJ^27bJ3bAUUj&m~E^(Hj^s6J@odrfQdg{Vs_=P*Sd_+nK(r~B#Y
z`_-GhYD1ph5s7=8`}Fpm9<^)1|F=s<y7YO3<=vZB8nW4CN7+2<(r7;q+s!MTq<W_%
z?o_^Cq}@D!u7Z%LpSH2K8=HTjUYBGlr{;nCYafOzH<_Jw_lc%jtLoiZfd``Icb<xR
z(BwZiuI2Noy)qX3v9CD(aq*Yhd0l&|aKtO{VQP~~uhiG9oV>UVwHI0rZx`N|(YH@B
z*eUS+glU;!PTafpt#b%0d@Ee!dp+l#f*fCHz8;tA42FVrnbQpP*K0f~EQ{^hd%c@A
zW8S304{mSKh!Zu-V2{@5Yg;U$snBRN=kbXLz0G0^T`kk26`NfpgG~S4?fS=ckN4))
zXAiE`y|}KnbVZ-9{=e93ZMnCebuC?X=bl04|4^S0?p1Nq*`~~%%Ht?F<(bX3z9rmp
zXJSLLPJcHvVLZV5>u=)P3rp8T-YWRk8xksF6e4BxfIB2mbPmhim90{%Mc?}{KV8=`
zd;M0?JqAx>6`MIXoV`_+`#;L=gtU^-TE4=pOg(OO#XVV~as>{&o7R2{Tzlg!%m36_
zW%oEmUkio3(Ef7m&V*HWwnkVVdsVVJvA-^o<9=F#Uhjn&O#9L}_lGsH#&20SL5FD#
z@7g=Pf|tVYfB3N5<M0U$srfVCtWMZ5Uo&Bq$ct<5n6CcwK9I=C^e0`WB>Lb)UM3Z0
zJp-8=H!O@AjC^&Tt#im-|G@oT@8#9XOO3AGe|EMgL?&5%^WFcNr>{@gvYq$S0!h7)
z)LnYtj=q|k7qWcgg10wB(!X)phKB?>#oI92@HWLKoD0zpdAjJsGs&gzg~M;Xny2^f
z9oPG9@4e355?!z2H~ah>(FanWW+>cj(A^?``bs_{JNI(t9_t&+3aX~LX*ZuJ;?*k_
z%l^V=wQ~KXZLhd@eO<|SZ~4UsH|)OH>oW7ZrAIr3uiugQ?bL#Po9%Jssk?N{w>-(n
zah-Cg+0XDdSL+t--4Wi_Y+8G-aqYWj&iW(!?Z2g&x^K?D_5L%<Ci&3hpo;Wg&2QGP
zxVn9YP+PXW=7XkRiBlAHS?_G<wy)K4zq~`RM1HqM?pDb&#}9wrwy^iKgjDPIH~;zl
z-x~7U-7E`Z_RqbuFIO^Uv)$68>)dv1uqpBK-_pIM>|E=WDP=ZK`ASaJrUdTH6AowM
zJn-^OkM@R!FQH<$rQHu-w9OBnc<Vud$odPO&ViSLu5~F~bb7mBvRS~wpdGBM8;Um6
z*oYQ2Td&wvAoaDkyNX-4Ijyi)J^btbif${$9>u9y%UcebiCr`h7wvLCr#*cspKRrX
z@(=4Q_U|;w?wgtzbm-}JeJ<rsT+3&!J9xD$yDh*kNMCkZQ-N9Gv>&fszw-)<94`Lf
zz4V^Qk*{6l_iNT|SgpGD>dAHOe);WY6PgytTnfu*zBn!7KtbUyB?lpk>YKLJ-ybqv
zzmWg0=<oj^37<B9=eqkw@mCi3#AH~ly=}<0>a1Ep&%TQ5Z3Ti}$@|(LubI5br?bL$
zZ}pq|FLphDD<ZY|)!*dqHQ%0m>$7{z^g@s8S8RdQ)9hdK7&HSIM5fp9S=a|jm0Ix`
z1WM-r`m*ITGY2Eni5{U7X_Fm{W=C=6Xj~9msT(7=ttfPvk7o2bmV&~^r_LTVD%Y{n
ziEQ$gWGvU?I#AKQ<Z!p#U5+;|PVvp`F0<RF^L0|%nKn)DZ7NYyUIZ-mnm)@U>QF||
zvY^%=QHju3msdt@O%t6Usw&RsvOFht*HMkg#+1uk3oc#IJ-)+qWt`{@*L4lMQ@@`p
zt(boKQfI9+)3?**S}Xq_2n#IIQjREyIl**3FJgB{7sDCVn_K~xDx03I<!Lm_DGt%}
zHe%X6Nnq71?wLW$_%Gj^bwgwJq!fv3!V#<IHiYGF%h6DMkgRE>!MejUPv&Rgtfbx|
z&54tqdp4f6T(Pg?Q)ThnrG7~|nT9W2PPXs9miK<%>)(6dNS7Hz&ClT6TRHz#_?r)J
zB0s)mjZBPkoHj4$<;vqC2@R!}CKMji{JNk)R4SCqO|r=FBAaNDnn1?P#uY6rUOo<C
z4Z&AKgJKmvZdC}}K5>$SHt$xWWarMiD;D?iZZzW3Wc6*Gz>urrwuE8!l_U|DXrGI2
zZqmFV$GmfEzE9TP?h<-^stspJcj158t&fzvQf~-O3Dn!PBfNgg!4)C1U0!ZgiYt&3
zRgcRLN>_~cUm7AFEg!Zb-8p8-(x_bau&LpTcwf4!$2Z({&G2O1)xLs3!tC3OsB}rI
znQ752ujX}nEjc0LJi+RTs&klaDAS_Ugsztkn@p!j&-cE&O?1(2zAIh2driO0&XiVZ
zlVOjUA$G55yMg<q5`)UnV9sSTzGr!*Z3wx$MYr_o)smDgWwYiyxtAU^=jYByjolvv
zuQD#>J$pUR)8N+bS$n=EimLDVVON^B$;H=PGI&B^!72y#aIcI#L6KS47gZH~+~6RR
z&b9C`Z=lA3W75x0iMP!%yK|wvu}veiQ%K;?#<KsbqC%&v@w%E3(Eo0u)2Xve95)3f
z@d_?6KD%A_NQ!of$3r1c@l2P=Ma%w%2)k*ooiRmv%f6oYb$Xlks09UBhEBWovG+|x
z<mJ7$gtkX-FBgkuJoH23dTCuwVNHXtOL^pOwPm;NF?mE~#---Wy5!;)bSx~xoAuR9
zhaUe66Ii%x&RxklC>j~%IkkGG{ywIy*P?c8yjU~iYR%g<qU-EFZ+nu@v~|--mXr%D
zLY${fuS?}zoN9R2rDWb;%fJ~*9Fh<9wM^GHoLGH!)2GwIsyA<PDr;OVVX*qYv1*d9
zeL!Nzqu2#4s;&1weicoNIMo&u`7Gd(5|=@xr<V4D7N#JdKBbfYKW55aEn*5?7qXh=
zk#^&s4h>^&zH=-=6E1&TqvB%3+s9&L63yeyuubFE#a)j#S1on%)-S#p`sd2L&5HWk
zu?(I;htd{qv~&$@7W65L+p+xl*$r`hi+JbF`n3G%R^58-;Q9QW+cl(|*2FwjR$APp
zVSbRq&zeuOM_MC=ZP5uer+}5+g;J9zf7z&O*YlgFQ;L26;U%jSmrM-uU@<V5zg0~&
zN%3&5)^w|-XEy4TiZ~>(?GO)|u+Z)6f(7}vF1GoWbi{q~>eAjfb;6n{ji&92cRY`|
z?x~x4kT>~|z?B1=53|no&htGewO8zl4dc^*N`sOUdqYap*M3>9?fh{HLx76pf04C<
z^=^_kq-?jkIV%a~ACvYu*pXy@C}B;Ko92aMhn}BSj`MJC(!6=nk8RNf564udz?r7f
z=KF<3g%aNd21ovSzK8jlhZcvU&;-5q<0`Bd6t^B&z_y-kt8Ce7GaXKztLuW6O*DBf
z#^czj{Z3%t4#UX{oJD5a#G0LITGX4vqwCGT*K+FHl^+%g_xgB#$(Xw0&v`|u&@NHW
zhY|;CwWisbtPI?jv8c`GfH9ZHO>f_&Q_NJB7MbaKNwO*_y7R3mvuj@LI&+c)+r3kZ
z3)ZySvRvp>-PtFiRs6z%$8!B{g;gzAcJ<xiD4+VF<h7MXir?D4&1*h{nylc7ec^n5
z*{s-0|BqD(`A&2>TXp8G=Q4vT&4A`pLUxaBP9zsy^*V9Nq02w<SX~>d;3<XlDL*wG
z7^Vbrd)*DbwjwLF!90s2@VcZjQ`R(%7bP4lKO-hA+<Cb)WrD*?u4zK%H}rDM3Yq7f
z3Yl_!(I%cqr8_<;&$+^<B$eqjuHwGBO~HzF`;I*<X}p0m?`(RqAvRt5W&CNO>y!F&
zre2)NRh4r>qUWob;ij9RiT~QS>tyv9NM@Z8Vfx}aZBxin-M>CUYhDI1syHxCs`j@&
zYAg`3HIQM_9R<mtYIjDSiUYw1uQK$vM6UcWQDp6p#~W@HDswVCn^%;?8y3ne?2{;z
z_Vnfn7qfKPyM+&|{yRTp;`I3=>pkt^B$<u69#$*wRMoBC`Sndosek32z@`hU7R>P~
zzMMKE^RM<QM(sHcTq}=7n=P5G+L-NoY0(CCt8I#mNvqY<c`U-DRwTW&?unn6?K3xS
zde=;$HOUUlQd4^aH(tNvCA96Qv*X95yep@6W%?YR&?lt2KYC?MZ_zRbZS#T)vjcpk
z%eHmiW>~H$p3^K+mdt+rR++NiMNKYsC*g*$v)h7_&rNd?QoCJcFj=lfH&@?C=t_ck
z&bn#pvwk_LE8f3gTewJg+QvEh_MMy0be)dp-#ur6<KFa{2YK$X9ZVPLtqZAnedgpc
zHKzS1E%oYrJeg`m1sFKD=oin3{vYPaq_MEuW@%YhOIE{G&xAWV`3*C5v-<k7_#Mou
z4yE7h$TCoom{72AJI}>}5c5L<^A79~QML0sFnN=kpLx(x*OyI_9OveIj5#Wr_G0dO
zW4-HoF8;k7RocN80xPus3!J)k`p1`dms*c)$W|(s`@D%YC1*3+G~u>WX=a&+cl_MA
z^OVrul_4IBy?lP|R56t*u%4a!<s6f5ld-G9`Lf3X4%-%qPV5u%Jtv%Wt!#t(HV(-d
z7DmFN@lwhUQUt?ToeCx{dD0SO#T9ZmEjjUafOf!DqlIBDVY=Cp{>9ICJ6HVma@yT~
zu6km|hKoWjudjW0FJ=F3zPXp7Mrzk5S1%vQ|2eN>4_dZye-rkoVO3~k;drq0M&25S
zIIpUU!b|sgdel74>MFQy+M@3Ce%g)|pJf*<;R|M4#@n>?)4K|5i9`E1`LxXn-ap$i
z<EZd+zC+G3;^8xny;0%pS6h7eW4}x4ku3G#Bc3-*BlMT<?9GXYe17$WyPxY{yS|ee
zviAS}bQmPad|dWP(d*OS`}Qa8Rk%)A`j`IXUG{<hc*@O?mYa-59miO&CHOj?Gxc*`
zafL;THF@oozgv#qlswVJa5qRq*79e^Mc&W}0`mfOZasCK>tG_qx_9!`ueYyv-p$Ls
zTXf@)YswK`yIFY~r@7DJ^!4Dq{Wt9Snbua>iz&ITTwA5qIR0<@Gxf+EnGmP5CxwlU
zI-Na!*H%uT_s)Mt$;CVhV!zyD9)ucliUod%e3lS5Z{_9k-YX(L4KFWFXGoBKxi(J5
z?1;=Y#h0uNOezkYHzw^``f!y1t8Y%XbC0+4pXrz09(^Agmd<-W&2k1SCquAjlJ*n%
zgFmA`M;<A@5n1x&$f|}T2R&unOJanMx|Xdx9=e7jwp8&+?LzHG0e{yj1v^A8m$-K;
zLS(s$$KjrE!>O+Eg2I6s58~(YOKw@nQpILe#um6B?xm&6oTE3NZ;hP0l52X5>&0B(
z-kOl}4PsYhJ>@+gIq&GZn(H_v=8U^eyR5;H85>%<_q5sS9=)QGxIM)`WX6BS9Kmpn
zr;igW9)4mG|GdCA<*2AJr%K9HnF$A5=Q7;WmhvwU<!JoTU#Rl%^b23fV-r0RJ)~F3
z$!wb^TeigE$wRrOHr-$R&(@sESrfmev2UjCQKeVM5>*x{uM7Uj>0oOxwPkD5WFFVm
zF)sUMf?^8h#NKItY?xSmht>9_8>dHy`<rmhI|si{44ucLn7d$!OVQ1+Clh)+m#%){
zUA^GRmK_rlKJbapIjZ*JR+zx?lnal<mYhiDa1&=-z$tgzT5IC{CI0O<@7zt9dGr9s
z&kL%1bIzPGyz+C(L9spJ+b;Uonz9({slBL8sWm+3;dhkHtF@@$apDrcS0z{Gbtark
z5py#5e?db*O5Rgok;$z5khvlrE^BHN%POX}PHgvBa%$DyuvnRecPr2S5S(Ja?UC}&
zSYE-@;0ec_F15)&x=^4bSadC*OjCL$=LLzr+3vAEnNtq+3Oo}tNcJ=lOO9F~cj`)H
z&N7>s$J{dfDoWygKKSR%ICeUAnYYz>mbD8ky1W`Bl@_m<BRoy%*u{XlNv9osU1pmE
ziSw|k8J_m+nc;fF?}P<QxU70}Qfu8OS>3*+MWJpwmM5#{%sD@`|69V8kS9rTZ$m3N
zk2Eu|Rjz%)^s6_JWAXZ{cdAv>oD*(uy_zVb!eH`_b-u6K>OvL;FINt;*xZ=Kmw9io
zJrxfy4CUT-Xws7Z9ydAy*6^}=YX}L=nppcTct%g`!jcK)XI%b=1Uc*sv$z^%)AVw}
zO*M%}viX<T)<=pwip_|b)hc+=TX>qk@F9<~(%#dVtrBH3wokqMG_hrBO!}+T<w<M~
z3#?q2UOe2E>Gq~|b|6c@+(+HX5=x6CTN@7E-Zk@Tz*a|VmeVR>bDy(DobeIqS;`{W
z;{EnOM5yJFY=a<;S;>-jT93YL-4(OQ-9p5>hUNIwLw-k&R$N%-n5){o;(o}ZWw&lD
zt32j=JMwt=iPjdS1t&FGTjzN-D6mA%x}vrwHOfk?RqyDOkMdnzH>YVnnO^tW@alx4
zIcHpWM0Q%W>@srQJ<UI%SM{&S|HVNU5?`Oqcwu<C%fKb?Nq9e(1IMal&$w4l?{G*n
zocy53$u-R-L#-{dX|b+_r@Mo?Xc415tN)pZTRB@BrAws3Zn}71XqmQ0%ObWjtn}s7
zu30>r!j|zY^2p$}(enxoTdFlTGh`RLFt7Bk62H<H?Y+Gsj%F*IRqlyhi>>8a%kxzA
zOylaJ&V>OJPyau1@o7e!cE_2j#N|N@4GUi-GEHbbU^iECZ%^OC8S0n1TUNN`{kT`X
zCcVU}MS0q}YO^CjTbnOVU@Y2lv1mz)KPT%!PWOhyMGSK~%Q7AnI$T%qyVJguMR_gT
zk;L}HnWDuJ+$$>M-{x{&GSWEC`XVNxD<S8<({m;T-MqGk6{};Uj&A<8SiqO-`;O!t
z?~ZmbEpFJ8eav#fiiEWc0$RZ=2i@5mnATj63kj)mIJN&<WbU0~eFv5}e|f#sOv1t~
zso)BexDSJPhRfH9k<YfWOt0hoI+L?Q_bFc;JL^`)4KXcuqh`77nZYn8YWJP26OUG{
zs$`Ft>2%Uh@XWM#MUC0_1hd!e5UBo;{F^c5-HyzQpIlRpxj&lg<#}@He=hOIovVv%
zm)MEACalYTI`f6(ua|8qecN<fuUWA$$gE=Gh-KH(_;&5Iw4%qZhZ801e7w`Hgq;!8
z+;DK&wI#g=E*Dy`yS!V~cJpS|)*~}G+K-sJH*94r37D@Q^gqDs&5`!lIp0LNo)*pD
zbWb?oUb?Md_Kbg8V)2D%?>(FSi1X~UMa^-Eah?Lv|JV<DxHZ^J+crrx<-k<2WgU}_
zibg8jJoVw+-&b#fBF;{aJ)!TZzLQB!UO!hq=%(}4InOhn_vEp@Q)_r9ckFH9gP<2P
z!x(NIP}9%T30S!D)r4~%a*S6G+p%aaQTwXLzN7X1536H&g|WZ3G0m88)-vhH_1G5s
znaccaZ%_Q=oE~vP<kT(KkFAV+Z6aMSfAzgA*w|^Xy`}EM`|zgLERB|zX&D#l9{%dc
z{`lkh<A=OUdHRzy*2=p-U$bCEVSvi+Y3}|B?_IYwo;|y|Oi0Umdds!44GcB^4MI8I
zxJ_{L>N$CJigfSKa9<y-;1zFQeDIX}*p)u<;4&fByIzdqJ8l$5oO6@m<~KfWKeJ8f
z`>b1z;w=p>bt&8xFFb9WxICN_!(Zh$pOV!+by}=d`|bim@rrdiGxMq}Tb{3amQui`
z6Z&GZU{CbS1e5&Yl!(vUo!pGwMb|#L+I-?2<3FXDD_3{vm3HVYVV2vpx9G{+N9!CS
zzQ#SsxU*z~^B?UAMzWHIa~FNgbl?|~6V(iKXi@$2z<okv=8;YQD;^)UI_AUEk}xNw
zec2J|h||yJM6B&qy8P(4-uI0=j;;K8X2!J_GydLNmK^eIp{JYSw@t}Q3ulLIyS0JU
zsmLYD<o{~tmFwcwj)*fYzj%-Pm}6V{LgUn(PcpkpLL!#TzZGLD$QowOVsqN~qtMZ3
zXRfg^25)-f=Jmvld-1vB0#of*_c7c!tir~=B-cTEiSpA`bEnqaaC_@sJUR5gQCx#n
zX28Uk8`9>~UOFyzJ*~XP_>GjX(~UXlKkqzASS@LPE~;+boi|&U4}N42UvlWpvKg$`
zyC#(Hw&6)JDXnJMq+tHMvq{9*^{dDNxt=`UmG4+3Tf!Gh9c7daU|72$L?(xmZDqgE
z$#qY7K5tsEcxuyR2hWo13v>A<CahukIdM|hCLe?O$4XBJoKa$}`+O;{Zu>>GCjzq=
z`)0dDn6yOXH2m3e<%r4ui;*HH79LF9>*M%viJE`<oK&CB^8&bT8(ns0n_Tt7yo>AN
zIt7l3kLn3i)~bEy-7UpwC13P%>*pMI3-f1Y$JgzMXvxaD-j$i4wmY(@Abe-B+KiwD
zYi~=Hc%FG%)O{tuxb?hU>xD`)@!*e(ws`jMZx=iLIIGVjxZpc`r0YwbGfxtZUG++s
zbMbuSk5iq&$1Nix8zhwf&$aSe%APv=LDU?p2A0_^QEY9G-h}n+3i+}$%*G>*X;YWq
zhc{^_5<DB<-g_6izdfv+H}TcaYb+N&>fJ~a<aB%^>GUD+Ji{c`hf5aB2w0*nVsw9_
zQTQ%H_2+i$e)QB&x@#!-#i1v#>&uom|Ap9H5?;BUo1J+ydC>}IHI};*4xU`<9@Wtm
z<PjomZ+!YjQdEg!gGqSr?nk#Tcz=|bY`sjZQj~wo#bcJsKeK5cRr>r+*}J49rSDpc
z_t`6Mjdv?^1-9=}WpP&6{y9umQP8wrOzB7tzi0BVr~8gGy)VlWe~|auu+Ya*acjTO
z8o7<p(JU2DnvQQX>XO?RG$SpN_s*hATl-u)HhlkWaV^dNuMXn^-|E%TVdv9ScqSO%
z40@xQ!7UhD^f_#T1}pdF;O>X7<=Tv7cWjXM+Q`oE{?T7{bEDlxu2=V*==u|PAd7W>
z5})%GkB8|y`X_x|@^AK5^>UTC2^Sm8W-Zb74GoFi@xPQsd7?&V_ThT#34669?x#0g
z`|-TL&Zg8i)vHx{n(GU}Cryia<dbH5Fdj4e)w5;&nJ-JiqkI}C^Y1zE-g9kye>6u|
z?Y6s*4|d$-|1rJI(zCtt+3td8p@N;K*^X`h9IAeR{j;7;^2D+&TvJY*in@8MFUshB
zpw=&C>1M$a_qT1jDpPwsZm4A(a5&?my@6?^Q<CnVK<x><4kdyPZVrx&xn7bs5fegY
zhB4i`nf0`7N`n!XuGF500_9_U5~flj5d|t96V>d~o=6C(I8IlMY1`wmj7PF5hAU5V
z$HE696NI?b3|F`?U0mR>HDjW{QH9`Tj>;~Q&RR}Fy0f==bSkv|=g~Bq-SE#uLy<F*
zbM~!}BOU<^>PB-6@`Mz^mov<@TK;2>k>jbD$#%uFwyZAjifxW&;J?=-tnNQShKGBa
zj^y)Or?^<B>1ghF?6HH@I`L1ALCL`u1?TC<R#X*bUth02T}yelh`1&fi;9MXf@1?i
z*>A1Dl^-^FZIgbVq?2c!da!-NLhc0BouZnr+1zJ_fX<-^yc~T}^=jJXV#aMe*RovO
zULP)F%JS3+-Z)P;Xl>L@t<4j?i(9u{D1I@$-D{#p!&5FNfyHP2r2HBt=}DiozwQ;V
zVlg*wQntc`9V@z-8D)$$*EE$ddzvp45z^w+^l)4+7IsV^VYT6;Nl85#i~iqener(|
zWPxZ{*2N?4VG$dT8i@F6X>;m%Xr@{}dX}WU>7dYSPJPW~6Bar5EHUUYeClJAbLvIZ
z>Pg*N3sr*8nlAi3X~{J!<4nsrCRZ*jI)6&y?A9)!tQ8{W78`lZ87r6%Y}ehSwR!<3
z%Z!9B-fazA7VvW`CGXs8z*X4o%VDIy>*bX%Q_gM`3+m`;f5W(PTA)hCMY%AyN)B5i
zrL6W7QLi$ad$@M5Ix5%cw{@wUna9ViyTabATq>JdDtk5Zy58eOPE+l6W!TR;q43aw
zRXsQ<;2{@NxkmFZtranhP1+nQ7H!K~DyO?7;c2XH#>Bq4fo;8yjvKy`Y@Qpn=t7b~
z^o{?SOXo@)wld$|v}lDFljfyIeY&={tGc;&iX7rno-1+o^}}sEm$xYNvK}beFi*mC
zg@FKXQ;zR}7_o++JoVL+4{zg(;Z48!;8DtA2kxfTn|ht3drg})*myH1`w1_|Z0)NP
zv%2+6xWxRv$c+@Hhf^-Bb>U<ab?D?&d2n*W6yBtgQ@p!+*6=ppW7Eu=ye#?ff_8_m
zA5C}1Pv(BSUH;W6y+8k~^KCV6WE`K@ej{>D&!?!5OBYmSZS-sq`qjN>S75=`R1q^1
z^>6N}tOcp>nwIr$W8pNMol|<!B>Tg`o3>m&>v<1(Np5#^KeEDwu|POHW+kh^>Z5Io
zBSP=JtdUf;?r{6>`OmtHMIcMKlC>jr@^ZD{Nk?CKc-~U*V_w~DxcAiYq8wF$y*fKt
zitFkhPwt$~Q+uPhZU4V?ec}E8mpVEbvhB>w6kqA#Uce-7o!ZeTs(6C&aTr_68Fl|K
zZw3aT19povybL^q&U`rXZSuD{eT_B79h`~D=0ajDtxYG-O^ym$#j}+uoXaqxbc?|1
z&vTZ`T~$0>QuSkf%vH|FbrK$RJr(ogdyk85{cv$T)AI#AOO8Zstrqq$o8+36%b}w9
zsC(C;P?3#a^{ieN^?q=gp=Y|RFxIE?dRLAqSA4*Fr9cJdr`z0ZeQn(PQ#2$rS=t2O
zS*mIUN**rJ`lMpLQ#GJ2_`lE3PfeQpY;w3}-w0Y<W3=of!!iTy&ED+-UslWu@S2>{
z6~?oD>iz{U&nwSQoRAW<cymP4lFU_=xBeAm&boQ#nNJIsgTTHQx;NIWcH#Qb>v*Vb
zqKd*Q1#Oeg6$_ZPK6k9L($P8b*?{SAV91encFq2TsDo|ael&SFJn7pI=E@az<8TRQ
zQHO5N3C?ha7ah!9GaPe1nw;6WjE6_7S#m?h!jQg8>N~YNm`-$@x&G(ff>TF4ZaCb~
zHCmg&J@57*jhuy@4n=t)K^;tMmP}{~%sI4}=|YR&JfYTo8a%O%f({3mmbI>I=oijv
z<UYoBSW|C8qT$4ssXI4KnyR^Gy+I0tPuPE<k}E}DFU6d^xa9YeHu0&dQ>TWWO`V-m
zeswmdt_oMmg9{Iy9pK3{T;sPmLGzY?Mn0nhW6+ESc4aB2$vj$&Crvz=c3t60cU#`d
z_VH8+1GlfZn|nx;x0qX&V^<jGgV6gc`{SO!<deU7NG?yKd7I89w&)uRMN~9eWn%&-
zo3)s!X$!7MF)Q+({2_%|Etuo_g{c?X`&hSal?~sKXRT+#b*yKXY+Bcr3w`njwX~R|
z538|=F*=H|aNRlVp~j`e7AV28f$OEKea*$gjVhahlvnd8>xaqA@l%>q9hn)ZmT=Nx
zqPcpywa+axwNnQiLQ@qt-497kyUK7+XTe?8;D0H?|F=vmS;Q^EJd0sb+N8x-FDa~I
zRcP8OCA(_kBOgwW1Bc75OYuuRnK5al_J<Fz%zAu2&%82`r{c>r-_|!7H8&peX}U4)
z;GLMn-Ew$&an!-7WhvrHLY}L8YoBmgUrli~DL%_oTr`uvaDhJCT<w4h%k&;fu$fEu
zZE%vvnxK>HvQ2r?+MO**thyG{6Sb1O6u2j~X2%JKyjjuCpprhRjw9h5(}#B3KZ<v{
zu6$XRsdWFbo@MA#rK@YbCNPETc-k*ZI-(x?>XPzL=|q#KJnA!de#(AUBJ`2-SUvY4
z$t@axGnc2jtVvpXrN(Wp0CUg<-I&P-S_4y}?{KA^-jdgU_f1<0<NqgBzT9tK&)Z}^
z^K$9(h_8}{nhICjIynyTuHC`GsaLE!lPBFiQbl``=F$m8H@_w(K1}H15}K;JL1}_d
zU`U_vhKI!)ORhbBmAgCsx0I21zRr=ufsR`gR2bJJiLx$awdHDB9k=JEJlo38DUW=f
z*ZnvwU%chqk$Z>4+}U~?>Snt5S#+vuiabr8c0}{!v+Wv(t~}K{*=gh<^!Q48?kpn~
zP3KgRC-W^jtrR)^IGQtOX3aVK=~DKhsN7`%BCHjsPfe=aI_<63io1ygGu6t@ZT@sk
z>h%Eum0K+S<z;`*HZ6FbXL+w_U1@EU&Z3u3F77laDB3LY?h30bU-u!`+Ksll2eSV2
ze9Usy?2h}w?`E;EtLso#*b#%rSyMFcYo6Huz-H&X^E>6rzqMMqw{7;_-J&OIuT}6R
z>4m0n!Npq7p2eb%R70l)`F1XJI=RBjx>@7I30w1%+M)V0t{s2ZWhs2W?8On;%SkFp
ztL6L`d{dedb$Z&mD6@SX7Y_Z7eIB$S`%qdo&ubUyO-W3po^M<DT&jwBQ)3_WeP`#5
z<PTHG=l<GJ&0=CGD&^{+cSU=L+MFFWJzSk<D(lbgx~ixAzFk`7^RxO{=XLILHQitG
z_&(D*xvE6bBM1I)Eqcha=z+r`aTR8!Xy+onzH{rkALcZQu47I<)HiY0>!e3ahT9tU
za%DX>%QO6MbWA~y@9~4~IzP3bH;1;q(zZ%y`x5bPe^~$5vVLx%4j-X68A+aIEdf^+
zw3$p0Fs<s#YUtOQbZ%kP<MKLo^AnHSS(5YQl(>U=^1RN>n&+n&utq^3(`d!HyfevG
zQ{K+p^KLc6;e9SksuM08R#~iLc2NFEzN?4g?pJ#M8d&}%xCp;GxN!-yt;fQ6pCbxi
z_*8EA`UV_Xd`xG8$-&BmrDvD;8%X#wNUfL8QWSpm@WDME6+1y?n~Mrgf+B_&RixH`
zoOYu9$)WTIPU=q{u~)Y(XYe&@%39i{<-IM%a9P5|ckHoESq9Ub>xwvQ{rC;`v3%2D
zULBJ^RV~>hg6~_Mfb)M9pDw9I_Drdd8`O2mUPqpKWT)cnT+rM2C)>HsSMglo(USh%
zQOzy8oWd?8du(xV6>-{mpyU6W!2iz-;^uhle4x6pgQ=zG(bAGdZ7omY-5TS!FmV1z
ztz>fhW6&jgOhM87V6lLp$`ozouln-)&#25$;-2HZOD)JMqOpo6>48dsh<t#7#)NZ!
z8s)Y$$VL^Zsy}9WdPXHph-qW?!#@vQEiSCT^u##lnD3u97T*m^`Q9e;9hoS@8K|~U
zPwC~}8J)=yZoi+cHd>{UHY0Fq*@jCyioV7yP8V5Y8liV^LEyAW0*7V^D$fWA$U0kI
za`<%3ahEBZe!g4w{L?e_>uX*aB+dT+rA^^P^Q#BU_Dd9+ceL2*O>&v#6noF_OjwJn
zl9478--LTco+=L9Z=4ypp2Z#E7jRd+|0l63;i&uxU0b(AZVyE_vBKXcmUyRlxi2~V
zA&c?jqctMY$s&sv3!5swx^_wEZ2<Gtm7XQ~{!uwoCp{~aWB>YWx!x`p&QDAiLMB{1
z!74E|`9i_UDKoS#J2**J8&6kKNzc%~@?+&e5v8?0ryJ&|>1wQSe&p5D;BKg%eEO2F
z`n_k@j|s~9KC<(F(GsC3TN&*8%d1n%@WCy{xH--{%6$LCtx8lnxFpUiAwiXEQ-kQV
zR@qOvW=~Z;p0q|pEHrqnyS?DFmepo458;10Cf5H2jPBoP@USyZ@beN5^SYPvM3Y58
z+3V7-Uy7>5A}SK+6ok(0ic(gMKg_3Y^zd1ZTI&`@Rb|hsD(>Qb!ksOOCHebxlFn;A
zJ+E|L-NQ}Dh?A8+S5M2?=NXg!Hz)7qLXWR3n<BILE0eO7K#T9^=Q@tf7W;!trm^Yu
zwIwmF`F3O-cR|zh5aodC$(-92*1aqA)p?`y<e190Ebf(^dD||u@jbHYGQJ`5sM0Fc
zdD-KTkk*2APH}%2cbX{fK9%XD`P`bJ?;Oj62^}gXA5(c2IK&-u@Mw7Q?nH`a)I;O)
z;xm{08G5ueCp<VK!o(?Hxa5>-0?(rz2|BOt=zI*hWEOi^?Z4xr2RTLOR$P5x@aU{q
zo`{QMbZBYR$s<wzf-$0pgzGQMo#<r#-4b#yNml!+u7&IG8OAG?F<2II)ykap{vf(C
z=DfP!`!5GnY^sI*c28#CyGq8=*+{BM%_5Q8>Qdx~S0QsU8xQq$PxCjpbg|`}q4hld
zwg3*9oqqWrlDQ`>h<&BFWXpq)R8fx!-fA})t1a3+?(r<PTIDLVNwz6fSfrsj;q~u^
zB<&k#|Ho;a{os55#)HZ{5j&yx?n|ethK9x*o$4OG=|S42%#~$K8?VYg^AWCE=)Qhi
z)N(Q9En?xn5+5GX4_$lh9E$_f>1{cZ=frbdy?QU}KlmE!t^MiAqet^D>HVKqwDiZ-
zB)7ESO^>>n$_->x^+g`9ymiI;X^Lc_r%t7xL5MPYyz(@mGhb%}7`@0kZMEi}7Yo1R
zBc9!pca^P8R(|i<H*NNnM;Vr1kJT&@;}Yi2b=01;+}ZS?+`h-QcQ`hb2`iZkn8v4=
z)rRu?G_@07R&mBhG3nHjdC#qvF@MxE5J@j8*?n35h9i$p&^d((a(z>Z!WcJ%HRiY`
zir(Yt|7RR!IPu}9HxXH@PNZGby-{d=r72kF<s-FT?-!z$`}qQ;dadkNu9Wj#E%dnV
z^_D{S?T@y2NfoBq7cl+yc|32)x`S6XD^7XI*}uN(o)PB<t)p^Evf5=OzTfrqzbbmD
z+y1|m$UPxJ`_=+Ek1(!jPv0I{?`qQcSK&~?1=f-dRc)0AdOHGRZB~@GIL&a<NYHBX
zSfL)e^gWZN0F&$ng*p=tasS-)KEma~F{XvGNhzW-tHs?ItFsv&EL$xf#`y0~;sdu$
zp2iy<YNRW;JyeUD$bD~n&|E>sD1omY&mPTqIa$h+C7)ePkU928lf=@thSCcSPoEgq
zOn6rI>|jV!(bQ$n&S<Tkly>0Crb(<*t1_EicAc-7?4o$$kQS$Ret^%04{1l1I9st4
zv#Os<lX+2-nZbRj*EP*tYt`Y!%a{XMo_Wi!)p~YmX^gVq%YabMyqM0L(TU7a`@fnx
z3C>(^nl=CE%>Ud>=5GUdHZ6|&?I`rSQ7*z!q##w)MUh#xNaRnV=%1siu8AT+rZX2u
zs*381zRxLK)!A6$tNY?+($5_}MT@*A7{+lbNt|a<Z~AzO?S_Ki`I?dm373y>Tz^vk
zirtUXsb)u4lHHA@r_-$M4mC-B)R9{hxR%r4d+{Q}X^UK=d^sK^^R~Udci56c>FysU
z*At7DzF52PklM_a=H?fMoNa!m975LdXoiM-$~r9?JvGF$G0Ld#%gm$G)CxD5PK#Js
zI;lI^MI?UGto~hMr^LGdv-V^zl+SP!S>~L5d`BD?UrzI{2Sx(BuUO2}HMNW`l->O(
z&-c*8FWZjV=x;MyXCTD;^}j%CJ?D0dPZn&>o~dGM7j8ewF8p)1O8ym}WFs5rg`$fx
z6)ZUtIS=Vguexcp>~$}fq!-u2%aa!~$oN`@TsxxIFJ^V?hWizzaMtwpf62mj+o$)3
z^L^JpttEU~;G@y}88dXNySV$rCRF-vnI>xD7-f2KwWyMCvy0H->*rn^tc;m7XY+L5
zEnm;c_@6&^K~+F9xy;3xW7<-gvpRd+4XZXzZpkv8-#THUV8Tx0h2_=t8EkKgo-K?|
zKJnm@lTsVct%jwRl}3lu?lv&lM|CV-zh=EtgptGX{qK~O1h~^vU8HMoNMC2^{pIAM
z`Qbs4wDJrQGd-DF<${$<XRY2iV|J%H^Z(kf!u;p>w%+L65@yWaH%m3Zy*g3WEm1b3
zQ8wsL<fAiD_r-Q6Rah;mQaHlAd$G!=4`M4-6em1!^2p%XygEJIl5fR~PW?+?YE3e^
zH@!7wDEqo@j*s6-cJ4J_)B4JeeBzkurl_%u<8(~=6_u&;muBwD@hy1lJ^$#;H<OO6
z6Zo`={a8%SS&Op9nirm0h8yFQ1rwVutWM{ferQ$etrhv3)bfijE)1(?J+OVg*{ri+
zM;|ip`tB~g+2P{p(mC!6?`x#jSQpkT-zx7jNi=L>rug3-(Xlh614Q3`aCJF)<;C>n
zEDZg=k1om`Px#k(_B7MAeP3QTJYKKrd%^3O_1+*yhhLKaeGg4MC}bA#&U?mR(+ED>
zjs92H=^UBme;_c^-nS#kUdoJX&+fLhydpBn5`VVF{<3|V^`b~>W|bJ%(MO5h>Irx9
ztFBJFx^Quh;l=wq%~xKxRG0K;dK<v7=a+DEwaB{Ft3&6k?v3`<`onlXRkTZrd1ZTY
zcw_ZWsoYSb!zY(K5L&l#eps#k-_-Rdt+iTZ5BCKN-4XlqQ2ftA-b<<5%dVZ2m6?3&
zv|fqQ<t>*bEEP^)eKIA^S)q&BU$ffRV&MTH>$4|obhLNxHNAewL5X2r!Lo3c<#Y75
z$d=Z>T6Ra@wED&)t4mG=mt^PPh<UZQ&5%7LON#%aN8!1MMmes$$EQno{eQ{xpy9~5
zR}143p56~jRr3E67iN1U_^ayN-F0Gq!oPg0@9RA9{CF|;z0hl)+F-+t787Jkm&}tt
zZZ+lUIi5pu{|<&N{&3aDo^w;nlPbNZQ;e-^PpQm`eP#Goo3$xPVAmo|aXlHI!!eH&
z>rU9rIeGiLu>Jg{$-VNjj;U`h-W8kXs(7oBcTqxfZuS1y#oG*%?wM=EncYzldUt5^
z>tnjH3@;w;5~y4KR$#-8#0P2{@9$%NWIj(!X7%dNYRns}M6W5TRvTTByb#H=Po5#E
zXIb4p773e<3kmyMcm(xwI1CrLv~vmmQhISd+&21aRAtU}H-{5uYgwk)WOzHAYF>NG
zBk=!%BMVqmxmXUYvAn#XNyS>#LvTatX_o~Y<`N=SBAS7WGCW0+uf9CEyqw+Ab4k~Q
zRR{f;)|9<EG$B}g4f|;smh1~_8rDY(7o|pUt-7<b<oPv`2`-tj6WMQW5IOKFA)!tE
zzZ6gAh9?gKnaou0xTqzjG}p0CP;%y4wDt_=w<@z4DeGG_nzgS97gT9DGV+HTuiM%+
zjiLTc#KzCt^F*}kzl8O!+M0KDRqL+GeyOYHziUJta+i!h`YU07-Eq^aMiJ-ZZNnDH
zEnD|{!q)wvNiS4+<o|ss|5AK7c+H6qzrTOKD8B4lI{*H)W!s{@>PO!2oH;QlqJFaH
z6-g&fj|~s}=i6%hKOn-+*?J{}slipOg(Gq4)VA<S4TqGOl2dLr79C{`YcFi<2oMs?
zxp*p}L)An%b}HwC@Dy{Uk_Wxk*F<mj<itFacX~VHNf48v<Q5fYp_XGQ&R!l91g83*
zb5V_payxlpN|4Ke2uJqL%&3wt7beBpZ{48WQ1Ir<C4TK$Yl3cTtiIyht+D3I<mdD5
zTs+w|_jad~xBXs@U&|}MTuFLS_~na{+WbX56K}chS$I-lX35XP%S$(NJ~+KRMej*m
ziOAGfk}H>`bp^P6`;o+-B@>%|BT;GVqc(<}To1U`YyF%xNw;i*QX`Y4spLgJb5D<j
zalDeE6I(779nc9BacZ5~%I+%mU$QN=P)IT?V`W8vdRFX@DWS64zJ~1#E!`TnCwHSq
z{!~VVl5pq9TPhCeJZckKIt5v81Z1*Z*S@a7E_UoxhxS63Q(~M-KJO<pNiUh}xaG#m
z+gf)v2Ay}hkgFauXYrcFvGcZmX_K)%K6B3;hU*v2PUSX#OP;5}wsF(!S-Z|IkG222
zc9m2J%f-d6nK!mx7g+V&cgd<1U%zacc*aP3u0$jI#jFXse?3!<dzQ(RbZh6n$-dsi
z>GM$MD2Hc;z#YGnx(?fJ@9HggmXz9(=Oi{ydKyFO6vHH!x7S!zLp(hkPI=V39xLdm
zefoA`-)*52_a{a-tGF`pZ94Iw*<SZ&3d`&-i~eh7Pv|;$ZdTHoW~uTFId=8ZDc?K=
ztdlmFostTc5BJwy+@r{#yOni)^gD(0GkmiqER%~{!Q&|}xI`#Yzu{|JR?w`=Jc;rX
zzbAD`YOUFJ<8Yu;g!~h>bw##oZn_-Ww#m`g@TkegCiagmm;JpIx3caMZB;B1esG)T
z5aSY)RE=ukm<MtddU|hobp<sKM{7j1?JHdB+-`lM>xrx79><kxTp0qI*W9O1x{^`E
z;i%{m!ZG8}R$qpm#GWS-MoSLGs7I`waKrJ_^O;Asu26L7IWc)#!K@XjKEjHx+cg$6
zOCPf+6!~(YFaOuUzQ=1k8Sbt;uC+^I#Y>$Q(TyvUPE6{U)$;#Dx{`DA`Q}TD=Ni7?
zQG8|TA-(Y0s+)7%innt3JY-{0I5eY0ar=zZH+YIX!expyy5=alsBPQ9CY!3MXw$J}
zjp)?nEvqN#Y`Wd1G)Xa6^@tYJX`Tlx9}bosJ*rci-67D<v_Utc*NF3Epss59;_kZ}
z6n8zD=#s-RQ9-fAW5vT+47V;GR^oc#`MdeVK93tq!i`*JaefS9*%HA0;^H!!LK)UM
z1=|fBbdRT(2>C56Fx$McH#2nQWbT=_Bn2y+QbPDX9@jY1;1;sgSb4#rb8FZ%1t+~(
z6~6aXl#?8ry`PfH#BCN&rzss0{^$EZqfdlk%d51D>4vU(PAkJ)6_va=)mZ*F^Dazq
zwpcN(Y$I#S+8Xh2s}lk<w7yLI+2r9X93-q+^Tk+0=S)D1#dK!gXGVP18rmJ-FU<5)
zadb}!uE|K8`|`%ht}h#Smy0FsX)QR_=cOv8&e^nF=RjxZt%>ve6^mq(#imSP_0@2?
zV|CeDBxKz-jiWAtP21d!IMkdDp7L`%a<Sz=FL%|)l<?O{3wa`sE(y{+tx^8+Kue8)
z;$+tj=jp$k+ba5abQ3)u%85uWFk)$)`ZLG5L2}VC&tvwptT~L#1CJc>^_dyE$@7ue
z_sN<$3yNJ`ZvXXCZ@nwgQniOsGCRxA^nKzRg&UL8SIsgLd=<&kf4EogQpCh-M|_s3
zu>7x`u;E5}K-+=PiI0xXmW_GHwj`x<+mA(@{w2S{n`1<j%6WC1rZ}+%)M;o*p9;{_
z^PW1*s6{K8@gZB_OFh1MUp8<kCo(<SJzbP7Xp4YlVw%ih@8%mH9C9UGJJTvJgiUDN
zTb&sY(DG>Va-W9WS-H<lCNFbaw`*eG>Y#NlUZL(%-cS0<mqggtD7G4IP;xtVMX&tm
z?rZUiDLI;vDoU&c?2IBtdxF}Ermp-Gw!*+tF)+<z`q~_Uos%9e6PTF0Ut)_sNAR0T
z)4xc&RsZzdXLw^nwWqk7XQm|otH^Voo2ENzy;kJRXykqUTaZuHc>07Ns?$phbKAmC
z+%!4$#^Z#F>;EP(@mW5*h19#5d;O*uFwXVT;FK2d5>iWeHA_ibG3Y~Q!BSz_s#_A*
z@_6`dZYocVHs*DjrQ~71ZA0f&7H(A@y=^WWS?xh0+4`rfO7AkPh;O;`Rl&D0jBzFF
z6qc%GjS_2TYbf6k*gS((rzdtp6Z7|=WPVn4&K;-s@;RmbZFD=EKGkEd{1T@tW-N!<
zA|tX+8gxxkx#A!aeL-x;h6Mr-r}5Y{F7=tDz*^frA$V`Z@ilkL3eTsci+DAz-lg~{
zb?Ggpz50s9aivKDTMdKHe${%=#&-MdiDPqa%hzAnd2XwxMfr<6LJQSG7}OY#|G1+6
zecwf{%#%v;6Fc+{Jqh%@_5ViKnUW}PyLK*Lxtv$L5vq(){IQz^9xXK#IU@gH@mJ}$
zhoyA75=C9l{pgVnDr-*Ae-}6PcW>>({;&*2L%I7?kI%hhdO&-o;_g$Q;wIe@=vTL!
z*imw{X8OyQMvtGW1wCNe|J7_^hmVOfcZt}YEoWO<FV!4a(qEi%cB*J|zy$87o3kU<
ze7Zm5SLHXRb(ObLHu-2Cm|4^%$P>>Z`L$!gv}ktDlL7C0msFJ=doXdML;&AD@s4b#
zOIMDU_AXs4>%pm;ye6TqpepdHrR7&g+hY|!KJ9BY5-B<r)@Stlp_SS9=$U`xo*8XC
zeCJ<h&CGvca+Zq9LaB@qsU@aWE?=B8#Qql_EAwP$udOdOzdNZxD6m?<s8K2@qUWk(
z+Cs0Az|zYqtP0O87+FPkvqzQ}XRv>C7rIz-yD9eIWP!lrJ{DWEE|(Pf1ZDX%32U1%
zzT2vAS|PAjjCG2U{{}Yo2j<41uKX6vvW-ke6O7o_9_I7!Fjik&>mMw*)X6{ZNzB%?
zvM&?eSr>->OW+O)VCzyBENE@pwp{UfC>!6SW@C#=p;FzM&vp9Q1)UPxgBx;v7!`R%
zyo6Xf#3xnpEfHJ$-QoSA^beEr3N|LRPYqW&qR%D3@Ubk--Z62Wqk)iPV&jtxj>5`+
zVnTuH1uYl3W-V(oS;2bARPDrNwZtc`K_T3;8vbj(4&Z;-uDR%fOO=q3e@NZbCnkB*
zHNP3KMJKdz9;|=Oo)*rQ>gr^v=x(@9A$)s6_`HL8#({x4<|2CLo|8=?JT`j%nJQp@
zJwm=+l)<boZ&{TfM@EN*ugbS#F9v1x9~KO$O}ra3tUpUm`0RafNtEApt^oB?PD8_=
zml{1;6;sq$qY9+{v>4VHg&uO&Y<JS!W5}klS@yL9_v;T<+=gsvCTu4H`HMnQzB2Wc
zJTF`Fh<kB@Nco4@B11DjGyBv7u_uo)ratIl-x$`BE?_lNz~-kUe{%)T)VQCMRD+MJ
z&J+!_YxF!{9_aSbekOaLV3KFkQGo^QiXtsjOE*@rZ2K?zbXrHVM&uih$ajm{RJYh$
zEe&B|cGi7T{o{qW=8gQ0WwA3aCbDiam0QU2&LVjI6onuG)}BM5EDxp44!eeYcD)?P
zze~fqVo`TMsb86qbjlZL^JR+rJ@l6}$*(bBO%C=r7h%>O$oBkd0LMYbm<wz#Oc^IC
zhUXeErX8$nn%-k>S@Gt3YCK!`pZ0o=hdsZabEP_T+PA0ea&Pzc^lUJlDi>O{Tg8j#
zS`h0EMOF<#sTXNRs=@O&M$R==o=|4)9^NmIn6Z0__kBx|8H|}r0=U0Ti0!<{rJml^
z)f_%CyxDc9u%Mx8-~!gQqMDnGWdmo}uKv)H9W-&TK&#wi>DLVZ>(&Iga(~olOETqI
z7+NZBCp@urzL4ql?J+YSW$!z{B^;1nW?-+fz#--V8}HKgT|sQp2CT~?vy`RGq!kt&
zt>_Y5=y57d=gNs@^{_bt7pF*Vb&yt_TGi3Z^Cb7)x7>+es{&t&+%u{!Dof~=SX?ww
zVbYJ;e>Vv`T8YnGVR!qh_=)9-CQrN<b|@Q`gl%~ws2@1NYirGKV<}}z>Aj8`DI%fQ
z8D&jWq*)!gyPCB2X0%rSl03lC!)@7Syvbwx1vV7}*3<<aDF;|n3b@rzv6iF;rc8)4
zjWjD;(C)52d(+C<g{1<F9dULVf-XB1JP#xVWUe|UX+HCMZ>FPYftp@%;s40x9qAsk
zJfGh%e_oV1W2t_hpv0O7;>JP}woDQ_2f2RzP*!zp5?WYNc}zg)dj4Mzr5P8wepJfJ
zZDe(uDUkjusl$1`ksxbNaBLBKSZ<Qdn#EReu4-4D%U@RstjZ|A{8;kVV%wTo?(;AC
z?@!>Dc3{)lnUWT~D7H;|+XFV$D7KUd4$CuFq$!49392Y%T>QN}pwnF3vn=lGW&N8~
z0wxRXdCP?hcJzLly7+;H$k)*LpqZ<_IEgAv6^ri_h&vgd#5gT3(SrA)o~VTn??=a#
zA)buMIeWKvdOuHOE3Ay$l~_MfGDo0*H;_AZmO|A_$x};wYBB}tGGn}*1;0+1F+=FT
ze52?3DK8gX__{T3$?`8B1(LVBXPTxcIVv3TtiKS!#i<gWDo_>ufNif5qi^7b(rWIM
z2a6W%WR(_JJ<H4Nm-&vTUsKo)*1azm*l<$2{DQ+RE~$e*HyUqkP+Q$(blmeoWnTiP
z;MVYl(?9i|vhAF;)gr36pVyJ=k)@#5_BCHUGQMy`%|EL9>}69**4l-Js_YYmC$?8h
zoZkJ1DNsYa{c4Ho;)e3%>+4omEPp>m;Kc5jbFOM7&s%v51;wpIlGSZrJZ`e6mUw3z
z@O-L+a@S-n!<9?|W@(0j$(LvCOWMbj9T*d^K~HPnRFgQVO+s?F+R6*<J1z@P_7)co
zZRecmaO>Is{Uwg+&Znv#&FIrTJ#`DasP5Axoe^^@F6V7^bxBe4<S6c(ks-vqN%_|@
z1AU2QjfTr!WiE~Vs^auQv7Du+mMeQ!(za~})YckIVp(MR(lgcb+st(V9?}=s{+Jh~
zPMEc}fL-a@ti76y>vjf|3FuXN@A%4obWX_ZfM;BP6We!j=W`rVm~Ad-ykuvrjBcUk
zk}Xl5@iv0&iw-nNcs)#O(EPreW0LrObBXU^65EcfF)j-GWf3L1EVj00bL=Va3%}M}
z{$+6WmNW15V+UUEX0jB&G$lMF$*SV{I>A}lXO;?l<`NV)x4Rkb!)rB-qfy&#t4B|D
zj@+qPrbYYLIWT?SdMeTRKhx`s9lQ@G`|sKjx0yB4$x1-bS3En~K#}o+xnlU6Da&UG
zd}Y;@ZFS&s$xX2lEIG!hHQP~tcl!HJXXdpU%;h+3{8fcBp+In22Cw7sSu0}mJDbj)
znzlE+=E$_zZ4aEcvAg()9ybyb%s!$ds%0w07;wx)$uLMr`Q*}P(i?>q?wE6vO|NTq
zP*^~202^;~>YT_ObF+H7e_ts1I7^6I@wIyZOZtS;jo}gL$2b=EE_>Bm6j>~vyZS{%
zx?qQZ@EgVJmTOFemfc^uOC{sDWLs0$GszhrFEy_bdK<vK_9`Ebxwzc#Oan#nU$4D(
zl>6RH*}LtUz=tVTzfEE`HrdVN5o8MeFJZH?f+O`}=|ycOYu43g_U+pg7PB`%==TNz
z!_&vvHVU2H5%<`XEs)zg`nXN9^MX|tr<mf)X9Ox_3tVQq!Je3VyvOrYk8a)L_TH-w
z8&7TOyb)xjy?Ld%!mS+f1Kfp6CR~$wb6h_}TVK|r@|lF9j89FVP@SN}s?+DTyixIF
zNG=JidpFTgy>Z*C%kFli`5z|l2$;BXp5dtwpNQzP9h{D*I&KRD>j)m3+v9B}zCD54
zIwv4LSi;OF<;KydO^aCPUu@P2uWX+?hd;^T=t)+C^r}_5;<H)u{2haqeV@}aC*<d$
zxOu`&MvpQMNC~c*EugHVk|-r)^>(%M!!;{gOIDix?<$%r!Sj4evEbHHLn}5x#-Ibs
z{B5P5DqolRc+)^usPy!O*;)HzZAE6ro-RAUD!t(vBd@63If1=**B=qn))KTasmcGF
zYjaHb`l}Y(OON^I6g^H6U}gGXnfK=Lo*MCm7xO$Lb!&N*RHa4dr^kn0Ts-^s-Smed
z3#`4gr1h;%cg*eH^|YpA{v7l3Dbeaa`suATnc0$0-YgZKD5+Yw<&0H@^_5QRXmRdC
zT>l-ep5HC#?sRxB*Qsb_k6hi0vVJ-9I5#$FU*Ijg(3!nml23H5<i$W9!S=oj$!mrF
z3ErK?sIYg}mTdvtmxB3!>=9?#crc(!_4Zi-|C#3ZYjul%c+UFrKTWkGBB`@7`rVU~
zhf^bWHcD->nEgO-ThZ11<=#aLGu9X=GbtOmuq)1NG>zIUpOPgsk<sA2<$NjaJv*y)
z_HtJ;8K@c>I_1yN@p52WXYlB4@b*c5k2dC1G>hAZOl#vYd?Z!GUY0uftMutKUb9E6
z*Nb(HcmI3T@G#`MqiKlny1z%X@<g;&D%;vGQ;R>e+a)&b?6fpa!!vbjE13)e-8L@f
z2zkQ)XikT2hqiT|#M7$FK6*1hI?lDYIr*)b=o_CmXV0D%G5Ejdc~PN3(Z=^~Zd}>(
z1Qy9I+Y)kXSEI6C2=^SpjEFk|@4s!?&vP#K@61ybZ8GnkAIW`JANOdU;_S{R|Jhz=
z&kmBWXUGkh7a4HoZ{6AlTnqPHW@wGQvGwHwBNJ}R^J(cb6Qkt|ugtT!D)2KXcyU!Y
z-yv2$p?I~s0*bw7N}0`1uDbLjcC~JGpJtLk+J<7?r0)v~Ge5GGbT&$;8>;NF^H`pJ
zdB>LXMncP$&RsfL@lERejE0TMEQfj)$OxVPw>!jWDNo{;uAD2vg3Xev8MqEG#>*DX
zJUjXAJIOuzr=RS5r}vU;Y2~9?8$Ax2Jud0(S3YCx)f{u;sNYMkz10UNF!77tnmmc=
z!`j$OJ`OGRA=fvG$_t7w6%pV1M7erXz5kh-ACil=>pi#@AKNJQIwCTRaeA-Bv`yij
zUYr6<EdTjbA}%~|Y~keHrgcI<QJQfH+oYKl6Q(wVEm7IlS+Q}hF5^}wt_`Y&jgFj5
zDsxOE8Wu3M8Vb)_q8V^Vq-i_LoryCiq@QoJZ{)g=xk2ONT*LP}iXI<ZFt;(Feb=td
zt=-Oy>sZ9DzT#5xu;$U{Qn}g|q8+yN(l(}--VM==UmtOOFu#&uzq?ZO+vo80VIrEI
zEF9NX2PHI372@EYV6<UlgD$hOyO_t$4*_#5w7Hj@aCLaz9IMwa_jcwMrx|6!DxGHn
z7eqF#<31JUVVD@nxP{e0gyZ73mmLz7lT$f@mK+Fsd&|S~Lq)=*DXa_md-5U@vQ)3L
zE>W8kmGJE?$7byt6aIfO@mLV$x%j-*guRYm%W`&VGVeU|`{KGM+%q2CQVhEnAvi&_
zj6-DQw5lpW2glY|LMIN+c-17})bzn9<4~i)mDS?=UR`ur@Y2uQsZnt0jD!WBSiCka
z5EZog=+a~%pm6lekIjJ$lAn!y8)R7@9WwEA5mxZ<tNGHytynFxmiuxGOW&Htu9{B<
zv)O_gRNwP>F>Lf*7LuXU!*<9^&0ykX52cxIFLiy)+;(>GcJ?W+m=?5+;efXd_oM`-
z<$aSpHCk>=UZr5q%{*n5(Htd@6-(b-cyFv{@!*k$1OKWm3gLN*$<4YKgf_3;&a!F3
zx+XW*jF4b%=}^7K2;CD`xss!{hWuyzvFVM%`=U?|!y^Z@Uu=vP=K5xGRzbO=J1PII
z;j&*WMMnM)=JlRDnYP_TgL4|!r*)P_5>CMl8;|jnZ|^=fiPy(0Lh_t~!SO9GHk;&o
zhnrd4Oz6J)I4ZN%@>R{&Y{QmKsvZg2g>w(YHqMaBz3^>?&<U}5b9P<P>xh1sz-@jd
zdR2?j6Yb`1Gl!z3y_zX=ZmgT1qd1ed+N0LMnTx+k`^s*%Su+ndC5dsQH&xi3NN-J>
zc6EB)jgBOR4~vv%C^T?;IWVe<7ziF+sH6QYf-zx^)9V(K#|N@or#+qMe)(0lnBvCc
zZQISxt9V79YE=0(Z9x#<tCR)1Sa(IgZ)i{AjrwoM@3h09?V6~^*~7WgOV4&2P0~;f
zdp2#e%Bfjwd(@jECpj?Q<O+SUV)2FQ5cRp!ex11}Df-oO@7ftki+lQ_SM~?H%1;ec
z=bN?n@FU&@Otu<J8}=`qsw*I`arN|Me>weq5u85jU-3N(Wvb_$^=3ih!nGTZy<Qs>
zUAgyxn#Z5UOLFpmShif4u6+HJtKRzi4PEw%9S^RY)vLa)+S=QHYohVa+O^9Ru3Gby
zaR|*lTEcE)&>2^;C3q>L*8;mWyL(P}Tw>Fk;HVv6$Si4bfop}&0+I3!tgoUTidfI+
zWwCfQL;IA#OpnGjJmT8U;ZH8N=@uLio#gbeX4{6wgMm8QpEe%-FLo?}pL3^AC(FeN
zLW(VW=Dk=GCTVp-xWD}9bki3h|30|u+NI23v+}LFB*}9{<-webdo;4MO*|_1r7f0v
zc*5wW@rk}^uKOB0(%jR89csTw%<B4*lFxqf_JjlOsxEn&oS%1}c_5>?<G{w%ip-JI
zXZ4s*=q@-CD>?D<oR0G4&hyfWzkZw(r{FflsK85M-(@aInNT)PC!L_8oMvv-3A@dF
z8klnnn)QUgIvd?-<d(L$?H6OwXyo%SXL|uFr`iQeXWo{XDHob9@Gz}j=rB>?#u~}Y
z1uJ|bd6&rj+i+ZBpYtpoQ3FN;CuPSMEB35j=xcmw(zJf3N`J-<w*IQdbK5WcH(9oG
zCa2|^1sx4%zDRc&v?vu+@v3lpd6u@WG+3g*tj*wZvD$Qr=xl+6WF7O#hD{o3H<cDJ
zZ&zl#J)vOg!RV(oLT*Cl43k+l`zRW{4`}0@@K9{&4F>T$35Gonr*Lb%_189Qn0&5f
z4a<q1E`~#&Op`vj*p=M~k@Q<8zJ7uechN+pwM}Z9XB=3%+K4gA@zg4<T|bl;_(-4d
zTva}4Sz@M(mxpK91&+l{Dg5iTyLjT*jatP7RpxE}GtcK+<i2K+%?eJhTx`8Bu5mP*
zmhZZ2_D?pW6Q4i&ZdP;kt?(;yYknlGJzJ*BSx>VeaKb@f_DY#qXM}dP_dZEHBCKg=
zsNV2@vPMGje$A)4vmRYB+tSD*bMx*^713#H4)QHm$#9icvDozdP*9NmnOJ?#2`dx1
z4)97pU~p~fKDEBqJkea0S$u)gE;}P0P3=&Q#XURc_#NYl5?keM|K!m02?dw8ZBk0t
zz4dbG5sd@uSs16K@Vq&9+LcZHy7QZfzkVcCS1p_O+jWh@gR+OsI;V}>kH!d@6imCr
z<6{2B%0v2;%Y=s#d`G{W4&H8&H2ZJu5&fjB<8G`S6H4-qTdQ6bJ8)>lgf}j_Mn5i|
z=P5K%);sfMg{hBP@m+&6eP>sQdUNJ$swLbDi<!A9JTFZ1(52u>Dw|o9jeD15%{=sO
zSK|p4O`bZQPjb;){yT9m`Ow-tt3tB=%VP00r;2s29qbBSFirH!g9!{Sj@(*t8nZ8a
zYS(|2I8(koL``MVtzHk~2f|a+UGsFpWO_?y*511~LC`LU?eoltOB3Gbwdy>%YjH<7
zfJ@;TQwIZs(g#;A?k7jMZa$f*V6^?HsEd-89=GNr?Fms^ns~~iZcVP5I=$qO3*&a?
zdqF1@7=@-NPT#rvD9>4G&ACqD%^P@T8g#yFi#AH-irEk=Z+wHzdJ50f4cYx=!c(Vo
zuH3%Yxp2LMM+Ud<1V_DJQ{SI{6Rll;p;?Uiys@s!L9VvT$vo}L+CA<(;V<h@yfFQu
z{=xP~T6+pMdN=)CQCbx=S6kJM^M9G^37P6oerH5fA_et&l8S3*%KBQUd03peU&4Ib
zGpWd8-N)x$@iIxbUB1L9-MgRn&vI5*+?MEV2d2-k@R~5`nT6uY##>^ig3J$oh)~p=
z$Z#y_^%bEa#+iM)B2IWLJFNDeW%)Xl!&7e`2=3YGG*x-^`I=J>Oh!`{SV&&}em3=y
zXmn2^cWK*%=wD}^R|#aCUsG_|;`c!=-wTW6U0xs8I`ZMe+1HgK$r8)$W%hN|U2zg_
z-1h9;g;!Mzw#0?RJULq;P%&ZB<iK_5e?;SSH!rH#_VEB4<C`=V?FlleV$U0OI#TDV
zY|B6RcB5F|rIy^jNJf`wCJYlCrt|R2+?x}2BjeY9v8VCfYJZO0G~2oVjFsm@uJwP)
zm)yG5knNCuTFR+)hV|V8EEBrVc%0lF@@V3xQ#0~jPrhu?b!c*D&@P^N5na&%HhHg%
z10z_=R!of(XbqHD9w@--yMv=gn%VRrcNK?8d-RTd5v+j%3{?}l4I^4bL|6-c?bCd;
zs5Y@@;RP$1+4Cd?JC6Nul=-t|)j?sl-zHrPIP4Z3jA3qSOmNz!#G&#)tn-$9*o7q?
zT!w9{4hdAx-?4LkQ-X@%?X9AhySuD*b-WN`vFwvN%n`K1aEWEdij}$?Ub7TtGiYWo
zSI=lzZn#tavM6K6EX{z;%LTOD6&wG}=#SJ|r?T68MZjFgi{{+_4{3c^sPKKokre@w
znk!hXPYYBDF!)L^mz^*cT_bu%_+aP<ZeAVR_SuV^A9J7Ita7s0)$oOjs+6dvly>us
znUZHDXMbdG_`RlBsQ-45f-#fR&(A`>!X19yf-}53lsHYRA5Gj`D0I<5Bcys#`00J8
zFCE)`&@RH8LFL1)YAz)Yrml?v!ksTBH5BOzIv?+ln3Iysy#18$!3gz57H*!^oGC#E
zS}He1J?b>eU`_kA^0dZM)|^$#9Zu|#n6lcaqjO`ck7BFt4vw5x?R5gJqA{!zF{}Y!
z*uyFghNiHKZt<v(XbrS*og8zpZb2)5%hY`zjwIjauHo9bPl9>U?m1Jq1!w+0;QofO
zZ>F~VSq9#73@WO7=JaPtsrxWk6^U#2NbBWvERUYtdSJ=cg(`y8lUf$3^mKLqIMvbj
z%Q>c#^SuJkrpZi38`^axCaRmPoMgTDSHQ}qfc0LP9t#}$n_0RJo%TMkbD{164vxSH
zHnS~UgqxWd_eQNaAUS)7;ez(*F^&b1?7f@y4JBCjHFTd%W)G0zikYJn=*V&Nu;hYF
zlg<vNO^f-yNp~&pl-iZGe5Ih>-UvZS$qn;N_Hpg*xxZoOCr#<*h}9W?4&74WxKkr`
zDoJNAL&Mb-D;Di>nwh|HH(>391l>&wY+{Yg%#=GUq`NQg;dybRDf$MJ>>FvVNcGzp
z2QL2?;QFR;{)x-X7D+jy3oXqD^aI}Pagc5e^w=?R%leC_Z9-(a3xycC-?X1vELwHL
zRq9E%o|V+og`G!!m|P3tnEu*VztHkjfkC>bKUcDo`V>A+hK)T5D-KNLy>Y{7lY#T<
zE8V{q&R-V1ah}Fu{S=OzOX8gxmYLR;lXv&2%wPzb(eTEnV?|)Y>rX;nTTXwO&85)G
z=_SLZG|6M#tO=)?_txz=;O{fxV)B_Q!VN!q_V_(f+T$T`&c`Hf%LUC_d)$LNj|%Kw
zX6e<eFj4oR?Yuo(OugKiJ?s>ZoXT)<lHcf-({iElz}eg@(sQM`qO*C5u5cT&%r`fb
zdoSGJ`(fQ`mp<SB7VZ^S#P1xMoW{(@I9tTKQgbcC!e$Oxr$WWgQR|xoy4-C#4k?GW
zx^T3Gc<fT)aKAa%>;ls&ZPA{~2X=h$n7n3cl*8V0Uv`{sP>TP$YTgTr!dXguGCj<@
z<}439b+KYO`&Di$r)gZS(gv;5ctknoB|FV?Zn!FNTvz5wo7CDn8p1^_9Jd%IFFV{N
zpvvK|$nnv+V@-_klN`yLhdRPK`i==$yUYr)JhIEY#Y`u#Gt+q4@{Cr|piQ!sQ(C4-
zOr0rc6kr@udg7BwsETgS=SiE(Ud-I`inUCD^SC$196{|1UxLnWk(|7g`{ra@9WU(-
zlR9^;*etEmJL!{XP9tBAp;*m-UZ({N`<f4Mt_yVY{(aK2^q3XbY#!@dhq^kqFiu)x
zYnppz@x=%G((dRN%w6$(Nk@KWYv6;96PkUmT>8Rx+<da~${vRv-GrsCfo3U9b2$^%
zMX#7~CqZ?&LbGm!;;Nv|J0Hz!X3Ypw<+#c?ulWMMrpA?XYp$J6lv9&Z-}xe<`&P^G
z6Mk#EV{9b_-w7Xju-Rn6hBbR;>}3DjE6aG;=A@gAA@4k64xU^m#*RZ8FT|de?0TM~
z@lBKCcV>uStyh_@Zj%JJ=u}Rl1H4TRhb4uDJyZ8z>e<rbu(OG=>xCkJd*+Na2Nv>5
z=q+ekd2WWD%Nes*MXT0us+k+#nXc5KEZec}f5VAKi8pnM7O8eE(sZ1vw^nKTSJvtU
zR;#?X^#te|-DuH05HHW;B>P45|4~~8z8hB~CeF#dyho#d?<2u?f`?Y>a_kVC9kiGu
zYbA%hk!j0;vzs-UKORgtoVDL_^MzHl(~UmbJ#U$GCqTG6O7}{UN^K%vt*K4pPEJRb
z`|7tFqP;{8WHA>PFO@uH{YyZP@4@6`rKMlqM1M2le;d2<uBKA46))4@9Z9-(3QjGW
z{<V9H0_&;W_YE&J=}utMbqILwuvqqO=X;;vBODDmnGfQBuC@`3S`eC$CF}M6EH{h6
z-A0YOjRxoVB|{b|_f}2RSkW_kjnA$&DUI7lIEsJvY}wGh?!SrFS<RJcr^Pn$x}Ed7
zmm<5M*-%!>PfzPL|K7thqB#`bS{e0qXf14t=(-xc*Wgjhj??oRxPC5(_Vu~XmT9Y3
zbbo4U{DKed^%*AB4e`+*=9hKMxwrTJpQ%hK_u@4>_;y7tc=sW^RU&!+LERZ14GbFA
z$2Af%F3u7OG%!mu&|A<Wb1`Q32FuGG@;uG6cfM>od@5l@u3h7g8HZk6vt;YKHg*1W
zU71G-4^Ig67_}wdnZ&?Z!1XPnYw3ZNQkz&cHaE*?F{ho9GS6sUzmQ+3LeHFGO+f0?
zKMY*ET>Mqebwy9`IJ8yw$lYbT0=Q2|vIgB~(S7$w|K9yQ9tux?rnf|NO8>9E_KCyJ
zq^svZQ1af0g!eP$FRV>p;K0DJ;&JnZYwx&)yFbMm22TGZt*h(OeM2|pRw#$S-d7^J
zJ@-^Eb~y?Bxgf5`An{(~Y|2``EqvY^OghA`9T)r+H|d5^c$Qjf(ZV*p#$8wW`MoxI
zY}DHS`#7&v$2(D_A1^qsS-v?C8tRkM8m%XK+D2|s<V}vs30BN$`&SrlS+L^b;ds5a
z%#^?J+te1no$&H~hna2Tv-;Y@M-^`GaJnYw_*j!eEZ45r_qC&_<A!&YLPhI1PJin8
zwZk{fand5+sG~{gyA951c!)1L$8A!0{QB0cwX)m-#vLizO4&0rOPqLE51d_kVyTqS
zvOE7Trev0It!gm$6TQE7M#P>kLf>DRPn4YZDR-*IGSP3oqQ6$L$0yv0I`HVqqc^%1
zQY>pHZD~`O>w2y3EoY`(+(sGR*=3!3w0cha%H_8uFJZlGFUvd4fr-U~Nq57FiEf5v
zQ%>D_%h@2LA$*Qwxyuvbt=;!>!&{BEJ&^54S=>`o+qZ4g$8(EPmUZ&o?K)huB<*u#
zS5ENC&)d$;U$CIA`?Swpb2axzHQqCVLQ`9}wk}%mN$67lM9tQxH<+q=mGqk86U5Rs
zzle3c%k<r)V}tTU`;gBoDxaB}bC{eGy5v@1apYsbq$iU(rk$9Pdq8vF!n-_$Z3_#{
zqAc0^QWt7yN@lhEpYcHB=fpr}u6MsD@W<yBtSr&!oPV{c!zW4DY|53^4e!_w32)w4
z=sE9-q$?+DK*@oN8FkEeSX8+FS}b&_oXfL`>wwWj+s3{ut(WY9ccKzvPE9^xIDu(8
zTLH6LFH3(W!#T&a`CDeKh<W|<^M0m7?B8Fe@jkz|#_ieSFZ*V0Z8msyRf4&wWd2=)
z#AXSr>s~1>A3ungbLfi{ij}{*XVKkX;gu@P^E~5q{Z&VHf0gUYGG!+-oR<h{O_+Po
zFXzMBbKZ&l?@}wmOTSK0RE$XL5}q{INMn7s<tp*WXY57`-cMkv&CGg!<$m<aLfQE2
z499z!c9jb>E?;!Gob4HWN2n+Izh=K}c-n4{gooTZY}39y^o#neY07)~&X$eAyz|_a
z`rhL9=U|x76xiXJH+@N>^_(V$Ic61C3^H}Qt*pOGSbAR*ZYnIu7Hg<m|73l9!K(m<
z^BS@p{!iy6d~J!D@-c`*>v)m*g`(Wmr7K<?6hFC<(}UM`&&Llk90wvEC)>MSaI5s5
zJE{Jg^X;gxO-q%p&Ee7PD6om#&dUA#{NJ<LG4DLv4n2D?|Hh82TeaoI`Rf)Kmm6wY
z#csaaaqDi2hsGL(6`NPEo9`|^)*&bVy}U7V+mZh@n^=$R@8t}ytJoW$zhcIW?WH2_
zoF{hwp7F^_>kMycMqu&e=2_<F>!yW&dpo`J$p3in8RjuFk4?*7#`Af5<?97ydq3Ek
zeJZxEPj1;UW0@=$_qq=iY5VTSN&6K%&y3Y+5Kt(1;GoCCucXqE;C-l#Pu{6V<70w0
z;~JI?f)-v3D;uTxw+RMJ?C}W7(Nf!T)v%L+RXF}o&5NJ~O}x2zyjvcpEzoW(<M?LN
zVf@TFdV9~VO_xvkNt>tMD$v~AdRl}}$jn2)!tp>@OXOBFpMn6LgERF{3dv?%d*eJU
zlYLjM_V))-TZ><EotVX->!iuET!!V$hj%dtnAo{ZG6E9B5A|^}KeQHTbWmXuP`RP9
zNi>}?*J_fJrGd(+3$l@&Zk<sIuev4Jw-w1=O3Y}IjX(as&b#eO4x`G=E8AK(gw9ss
zT%us|yYxZ0D)UsG_7{_u-d=MtI%&D((%uhkck8)c{QK6&FsD4{rsMllydQ&7PfU2Q
zDe)AG=2up)g1qfZnWpDR|C=6DcIIvS^ai7&Tke!{zGM$<=2sIDcjOgPnbyJ-Q6W%T
z;CdplmDyA6LaPp|gIG}%N5dnr4&D}-qVn9WSJ{}(oKSK$=W3AXX4h47=;tV07~y2^
z<TA-2r66$$L+X!XoQ}PgPnPuOt4?5%T(r0Jc5=(sB57%P&XU_2vJ*F6%<|ebEy!Jw
zk0ZFjaO2EyCgCZDR~(-F6qy*v^fogt$aCtW5b2aD6Q77k`!t01F#h)vcwQIFdF0t_
zcBzW6_Iy9~MeWf=v&5FoJ+(!gq0aP(KvA3EmqiTvT+@=BRvc^5T2-&QE!lCwBP($M
zCKbiTX-r!uU35P5s4c)fg71h|{sAfPfLXCd-a&cm4{o`=MUpW*P?!DcB|qsP8FhwM
z{odD_zDAx0moPYTwk)rB`myqgf7zOFFX?xzn(_&4uI8Eb&za81*QLIFo*#8VamoQ7
zaX*%Y9ZwFKzO3`~cq4H5{K?Xfhl^v^%x5~bam&#j-Mod8hjiCT39jKh%aIwbB%C4<
zI_H~5&;%xrlY)*Z1x}MDxvPmy+{NTz8IbIJn8jg7=CfO03eF0dEOTR;=(hGM$N!Zo
ztls%nuS^d9^Zl^d!Q$u2DbnABtU^N@V#7a9i|t>!^16f-+xKI|=R>c%RC%Y!#^q@8
zG#ug4JrNqO$^GTgiFLQUuIyVUw?t*4;nSXSmnm<n%unzv{if~8;`qBVYjI_mkmHg{
zy`UKzb-O2et`gol+m$I%I!gOJ(`~LDtETNPcghpKv&=hEa{-I^>zzgKt38Sw<ZFYT
zY`?Rgsd7$uBZJt9?MI9aqcT6_Ik|kiE39-Q;Gt}g{<^l=-`>60q33u)E$Z>NdjT=x
z*?$U|96c`l=;Ghmp<%da;zOp(Hph6E6z1=Ybgr2uu#vG>#phL^q{Ts%*_KN(CKq!s
z{W9Ne;P}5Kv)}PrjMrl0`zlYA;&*wtUYerzF~?gq+JXDSx^!o+rLIh-6AvtkOPaQB
z=bQGqo1z;#GIn-cDR8LyeNXWAl_Sk2tGp)9J|yO0s{PmJpk+|R%|xLJ0oGMH3c)jO
zPHDI{LGEgff<TSZrEBaBEea1U!e_<K3i^If_)=RIpQ7TMwCRE=ThA#S={mVFWBabM
z2kS)d-HI)dn&PrZ<Xw$<&FR@y;r7M8E&9fp^P~198Sv_I=zhN(-Tdg#$@T@Tc~aGj
zp01eOAf$S(dp5)40|C<nrt_vWa7f<lOLLuZRK>IR(3gqltIQ^tee%dTaYsmKi9=WJ
zMx*dWl207BEOELj*v<QYO~W&XZK2&^X9TSOd2-z8d11R#(@-r&IsJ}9>bZ<V#d)SO
z%Nt(2oZ#M+KZmp8@hLyi(|W7kE5rs~6<2hhG4-NQhIG&K<i5*o7i;-S&T-9k4Vf%`
zeZmH_%?C9&yvv<*=BlYQZ&|!?%S_+fob0pS-te)QyqINs?j%Q_IVm5mF<ri&dcN&P
ziITq{%bvMeec?{LT23-z0db6`O43y~TD>%bn=G%+S-nU8ZpS4<);F%98kf#=nIzjb
zE;yCn@%qXwfr?{|A@3BOiV`kNIT9wa?qk?-7e&W5wxwI#mUwZdzKGVkq#8CMWKq<n
zW!VMk+y!YG;pV+lN@`w295b6Tm$Ui*3G36>&F2J7Th{NX761N{1}n=0ZH-S#4}-#v
zE|+`a8k%!8PI-|sW5LA*e337_A5~3Xq#4RI`$f;@cPDfbzPd#y$#9s;K00*YZA-t>
zN-4#}a4FYYNlQ0mx@(&Ss(POLvV!xJW}sc`;j=<&4zqf$_=uF5f95kynC9dWbHQ7H
z)6{)aLDc<eA2e5<ImNm<^cF*s+ic|{0#h8~ITc(AG$%YOO-*fk$gsR8^VkZX%-+*E
z3zF~saBI@g<=Xl(XSt8Y#2lgNOYW;Ur&_8WIXTZ;XO8Rhc8fPj$JZDs-YE5WsB+Cr
z{al2j(LXJb_bUxH@$lz_S7tV_E^5{75p!SGq+9e~BSijyPM-qLH1C|uOo5F%za9<y
zdetROS!$Be^vn}X=N!yrmYshgsJOS2S6($~QddQ%`F7z3ug26^PSuG{cbadM8*&SC
zC2p_tzQe!3bamWnDXYDEjZdp8HpQ+=6G~oT(@`8@q_@#x`Nb*M^s?%&%d@^P&=;2Y
z$|7-wQLlu-OI3rJ(_!v}NsS+^gcJ;X*en)H%5DhV7@F;qx6P6BQ0O`vsb?EFHyz{-
zFIDiGe8}~((2aAGk~W<B>!Iz|xXkE2Tie4-*3SM2zn^vIb3Zvh?+Uzg)BNg<i@k5x
zO{<jg67bY#kT|reNW_4#dipM{7>$Edxm71Dv~2avQxK6fQ}{nkRx#b6*Z7Rs&PA>p
z_H_mCT_d)!JnZ&vABm}+5AGBsE;jCnT4LU);^z_*9Mi#2WO`QgtgBPAM^`|(u>9=8
z`L~4(-rp16Y~L$#Hnu13`zdojMy>A=N4OVt%@o?&_oevouHTvKCPdwrb@<xvWgs9d
zu#~5jOWbMKFaOtPtGL=1DSheUnHF0$QGv7VZDXbG#CY+Qj|3u@<w_O{aQi<q+VwSe
z_vJ_7=l^txNKcgF=oGzkrE>CZ!@4&S@0d4N%-VdU_WbR>x!+E_U0HtqSzc%5##M)_
z)n_nA&)76!7sH3<8IG~4FB-10USMpBsQt>W$$ZV%VTI*ab60Jxwe`a9rU(D$J@#ga
z?FtEXdvnW4=N_NR?U*^)^x2)2cS^Q#SGv#K`s^ypbdBRye>`&n`SX~%MdXuhgj5BV
zI8S5>`uTRt5>=;?b6Iw3Kb+?~>)i}d5puoky7bSNEt?V;=SW@TaCyziaE-Ht@lo&9
zy~o#B9?>}&ag+C$j_yns7fHsuQj7<$F&<>e{mKzHb>c?B+tFJuMmH*)vtdb{Ak37}
zxMic;raMPBSMqAcOqN`FAvfZ(aqIDdlG~3;!|MtdS7<Uw&bWJ_H#)^JY$=aa`$Fl<
zh7T7$4AA<?S2rhO(_e|Q6SKWfM0>L^L_d%YH#xs@t6U3bi{us2$scY{Ix_FB;E~s&
zym>qC{db7C`+9F2qgaBtM1tg!dE7JPS~L%cbnrU&Fh1JTA>z^~@<+~j0%L>d{2be3
z>)!S-rONOga#1-Hv4rD(k?5t&4|82M21suRaGG#OcH^OwbNgIMWL!=}x=D1N=4>e8
zblL8}&4293NsS(%57(o=+(~|t<iAnE@r}}+nbAx&tp;+6B65m}OC}X9IX~a`2vd#J
z%F?4NSs(N(B$&p{y)N*8;ot2^QyBLdoWJ4PFO<+$FUYc{SF}!IQ9KWuWP#|Bxv`F%
zF0AWf&&Wh%$KG*v2zVOsNJ_IamXl%AL)p|L6Q@eVFQ33Ew^Fk3CY$*|f&A2P>$$?`
zH4lhgV6Sk!^=xhQf5$3cm8YRqD<gCT&-C;1`!>k&hq|tu`rzeR(Mt)kjI$2N>|1EI
zNzP~0Lx&407bh&_exUgFt>XSt238Y()}C2=|8i$EiaD}0TvBmVkZb9ZX^R(mv`Xet
z?*`@Eha3*Fg3~gW9Q8QhwnehYmPNiZmAmVRtIPfSmhu|k0&^0)bN5Pml*ZdCxah7q
z;`}K7z!UcRT-LLKvWYL^7u`%1F1lqT7aF8{_utj)mv^$5Op_94Ss*RP!ekaGwQk8Z
zQLhOd_iC=V*S$C&m$J~|M0C)<$M#CI9T*xeX#||Cxx41<EX!$})=F*4N-e(<6&U2&
zUR>mHS#pv6(o`3TPWEXDnSO5nSu8GxPkO4B^(5YQ!b_FYxdyy^O^e<==rT|{&{c5o
z{lzqwXEGw9iY|63LVO<OWzPjY&-<8gdCXJqSCuX=Qhf1Ly17-+@lA_^MXQ;d%j>nX
z81AJc-&w+tl58fIAgkBOIU_Fl&F#h_2Ct^L=dEfPd`HrQI2r!*JN6!IS$1NTN~2;}
zAdA|fNn57bDz)hIp0wJ;k$cwl?#cKFl}5HZZG~novsou?KC*&aP2NL8<*TPi;2al^
z2?sT19q~H3^!QV^^*>S@UbIK>p3rv_KKE1U>B`yR0sVcE_b&HJg@>>*EQ%6m$Y|Ik
z#2BPzo)ue@@a#tJGw;Tph&PirDYCRIy8M5;qZot20oAA#t2U^u>^Z1*Kuy_>CCMjJ
zFERCcA(!eqwftjqZ?DvJRA^+M`83R~Ma^x>bO#nqr{@k!MA92hpS5h6?RBIgEv36M
zUElAPYu&3DiDaw0)2lrB4Xc8@Lo)aMJ&|O<DS7Vsq#R|Vr4Q?Gr8o&B)L)w?dnqpI
zLo{c>0dc;kdoLalJ2a_TYjupvy@&;At^cn7S9ui7<sg#MD)ysQj77=X@AY*<r4BEa
zwSO`%%vcfgVX?K~lTW<Uovp%MYE-9baKE{8EN4r`ONT3&6Oty*>3lt>g`q~?HN>Sf
zXZp`MJgQsc6NJ)SB7}Y~e3e|7cu4DRLQ=oC#bYKNhW{pV5?5`HOjBD>KW*VP!LDSh
zWhX57)8}dG)T+%^W4R~0Ds!8TrDF86gc)a&BpB9AVm>9svde4UIoGl#Q7J*zf@6FW
z7F?Y$C4oKfRbd93{9L8pS&J?|>w11dL2DIXf~>mxkqZSc{63yJS~+i(RpV`y!c~($
zCC%nidv?)f#b54^HKI3oSZaiNla9Q4W9lOPO8uh8vR5Y`I$ls|l$*?WYoV-M24{d{
zlbXhhukD8vUW*8|7AUcBty5pom-U2;g-J&`nD3Z`#nROoj%~|4w&lGRD|s!bDdDt0
z!zm!ic}boVkMGW;4v{ablPeZ+3$?4wiut6QrJ>}auGVrw_rA-U760>R9EoOoshxL>
z?L?BQ=YfSejpvnggfx|>c&*=U+p^&5o#vcNJ%ww3`}&9X%nI;IzIyPXv%@oIk7eun
z^xwz|PtJ&))T7hn*gx}?e<w@hsiYg5Zsj{$taEY5yL92rLM9akk4af^8tboUwtFRf
zSm?!i?6K0VXRA$FUVhEI$maV*NB!xP1uk0zH+@|ncTi#RqL>*w+_t##I!=(x>N&*G
z-Bu8?>RrnTo7P(kdnVg{UHasns_3nw^&WEZA5`NQWCPpg9slJa`9N&)js}Jaifgys
z`<eB4`mvPPAD=r;dB1<@JjsmAkOj9qmENUiKGa_Lvbm`srS_x!yH<vZQ1Q?g*`5yn
z+vFDIKX@5(Bxqu+>WVXkoo*7Z&xy81-Z6XTwSLaVPlb=tp71HW7cA2C&Tin#pYt>=
z=JY!$p%05LcnFB6|Is$Dy)33P@3uuyY)jOEKdNy$Cl2j%sS!N!MQ+V66DixXE<G;E
z(^lypKPLP;N=?$Bf8UkkEDp!LBROU<HMT}ZG+V8Rh`I6KujP!;X6HBgmkLg07R6q-
zRCpBOzD<oSyffQ}XM*0v#W}B@<*2qyUNZIdK9&Nel&NO~eQIW2t}EbZ^pfQjp23mt
z#L|B4+yyJy$7P3R*LJ)*GOIs<Y1y3>^H*P;RM~k|YjV_Q1A&VIhK?t+<m^~{^D-}%
zG^8+PT)YtXU-CubtF0<cV((Ub6j;GHQ_RifAKTs2PA*33EKk0z=bisWptmc(N6d2#
zhvyHu^#Y69vKr5a7`rrR`%GDJMnLQ0taTgairPIhT)VAx{f5<6_Px8!TBbjoXnJwt
zE;rMYJ735x7U<u1%yWy=$3tuuf|@=imn<)}2&JfBuzcw3vC`K3W#sa$4KF4bJKYTv
zdAC8TWm=rFuro{C;UeF2b9LI^tSn?ad5zsR@<i*^^$s${LQ72UC`d;w_!2wA#OclZ
zf6Kmb?sL`8?y+3dasQ0ewpoTg@9(WT(KWv>+gU||+4=~}_gQyXOIeS*T%5p?x+kzr
zg`sejpH+Kdk?C5_&1@cu-v5_gbMn4eys7bdWWV9lOT8&;CfkZeMDXyWPhPS&^C*kH
zn~?pH^^5ca->5Sseep|*UHD=4Low~|48}H7Uv2Mwo4TU>saxL4W0yBIZaY+(En1rA
z`~O~g^F*1~FOx5v3KjCS7n1({uf%80x~{B)g^hg?i;groPjOcA&iiq7OGt=w!j<(C
zBp+-%clF1!PhT^%Z!~}GI-<J!Xnnzw9Xzrg*Djq5xuj{oJ~?m=Q-qFovBSe%n_{lS
zrQ~s@X3fp-(D^g%<)3%w3;WeF?cQYsxiO^haYeCpT+o>vVwN=hi-W*t1M!rU`*Smy
z93M{cX^mU$5RrY8w<Yb&rHPhpJpuvq>mL69zxVFihR-exy2hRjVo%ue?6(FTKa#iK
z^>SV3<}2EJ({x=~%I6&L{k~#Lak78^Lw0^?ziO}T;#`Je&p-M_n6Lfyit(LkpvD4+
z7fU!lsI28%mptv((W2IrjxWMH%kQx+eq#6Yj?K@>-uF%^IqWK%_<~7d+q#Wy`%Y}p
zN?Nt0D_gK_2ivy*X;bsvKUpv5F3XcCR#IOsa_OpD%Z4oh`3KpGEssd0vV_G~2R-ML
z>^Oh$j_W}qO+V4p9`P+FxD@(stII02m=>+v#r-kbptJfzMVj6DgxN2(WsfB4uTDMh
z`u|(ey{WH~#VxDN-`zd<^3)=$DWBg9Dmv(E^RfsVm^4@Z&!6>CW--^21!esKjSfG5
zG4HR;6SICL)jP{+oxwjtxrdgYA9|gVZ<$ef#eKt}2aQiNU98<dOf)Za>X;E6G`Fhp
zOLui$qR*+d)A-K%@vYdU@ZeQoTC$JHeXa*W(+>!~Z}CeiX=KxGp1UNhVvF;Xs=VvG
z2WBsOs>UGX!XV`0DB`iSYI;qzxYfNid!O!$YOT^J`f^Hf-$v&Px2BviT=!&4HuLNg
z+Xe4<D#>-*Sb8>2yz(bP>3Ordcx&^yMNLoLGjk*DigLHId3K0ov1tC#oELF>^BO~U
zz2E2d>okPr9Gzmj?B^xL<M%8aAAEJO__#lES-*wa%wz9=O!SzbZr;Z6{D14-i?_Vn
z7Q1}=m*$r#Re#3q)H)esM;-e^yK~Ow-O=!{&)(pCW^(J)-8T#mzf`q)koNrOk=cbm
z{vY+TZ<)z|<k;r0+b3P4IM-gd6v{4E!Je+6d83KrqNPuX40qHyuFM@*nOYbcnLW2=
znF=gy2wNiH=kcS!iK$UYPHoAG1chTg!Xg$T2CdBvLQK1-NM<a1bU=?OEKKHPkOO<8
zn%pTL4`D?=MzvEjG;S_sna&qu;iIAD(8$uu%*+upODHR3Ww5H7Yrw}>S5}0w7R;Ts
zHQTk5lffhL#JXt>Th>Hu^<oOkj@envS|!`Tz;M)%`8$`)X4eN|n$eRyBz0UBBo8$F
z7v>VPa8*cR>|rlj=q<j9L2t^_Su!VX7R4Ab+j48@iWOQn2lB_c?Xc2vUXd%YE$oDw
zf^*v%fo++mqZ!`q$cgwVrne#2I+{Ovs;82}$&ju<!>lV7iCnB)G97JVKR%^hnYK>*
zl}qOapI=5QJGR_6NNp0~^ir{zK10RvvI56!JC=fSg-wfo`71Cni!?YW$W7v4WQ}d+
zUcI}4p}~P;XVZi|OsXsjoHrYs`($<1rf?tT@fFd~=9=%Fv0m5gWwue{q6rRNyc*0o
z!soVp(r|0sQ{BPTxWYqY^8^#?i(4<s{VG&Y<u3_fP!aM-;ttBPd~tY+-JP{cH(5?h
zTA+W_<v-s^1MMv3rGauzpS^F3MXjB?W!mJ)3t40yU0nE*<wCInr`n7J=5`0Cj&7r<
zN1C>mU4G?VoWAS8%Zu&Dr1=%Sd@D}wndkLLNrQ3Q?1m<L>rH$s|DCc4)!OjJX=>=k
zC6BUn7(aM9OuiPj@M&0B#?2$^LZemBhl?<JIB==|6>8}5Ws}nB{KdoUvvEa<i-)^f
zY%|ZMO#RJgj3)Q^uGn&EpKJe?DJmC|R)yJ>7%zMjC*5Iwn`Kh@z4iuc4&NJB&VJw8
zoMY{`r71%4t%4EPs^injgtC_Nsv0cHU}RNt(+FwY;oYhBq2!n8(up==9(#>;Jx=KU
z{%E#8<DW2@6ON3WJpaYKGz6<G*X-hI5j~jL{+6p@wO0+tn)&H&8m_#JZKoV&>$_;A
zt(+t$z)_-|)IQ~T$}@@AjT%orZ$3Gt*N5S!im*dus(3Wd#*JHVEl}Fl>~i9SboACR
z2EQ%8Cg!bFS`gIm^wgJ2aVr?6XMMO4G%5V7Ve{`Ib8}X|mD7SOgbu85lr<GuHC0L0
zW0|}4e6BsN<%g!Ic=(=<P+7#Hk;P=cx_H%<HL2INUL-b7&}dqnzWkNL>K97fjT$+J
zr*4n(2)u2|+|2eNd`DYsA&;)a?MLe`igiDVJh^D@wT)dYr*#(YS<V!H`*=-C&Sq6k
z)|<O8xP%pPe{{U2)ycsSyjH0)dh!43TMsQ{3XVIzu-D;+3d8*s+~y3cPZV|}%LHv$
zsau=ur_H7+?)9`qMby$a-6vUSazVQB4a=E_7by$;$#|=%EOg-Q4E5E*oM$ssvL1%c
zYg1BjaTJcaz;bhoLMw;DL;KUl7e8^duq<p`+NRgT-J&7FsK3#fAt0ppR^77QqQ_J_
zH9eEWWUnk>$zXKjJQ;Z7n~Czq(zn@@6Qz@vFzjpNX~}JDOm?y{YPFb^U;6ZN*WD}b
z)4zp2iQYJ=W6F;-o1?Za=CNKK|0l34yOWY3l5og(n$ixvMvWYWBCVrhY7g4J$h<06
zjy(BXAZhZ914pLJ;JWMM!zALj@}0(%$q}I<7yeKAKO?Jrr^&^LhK7dcdpx*aTw>Z)
zl;NWK>XXYpD<uJsi__BFii8*6U`YyjdPqp~bLd`gkIsZ0z6=?Py_}ubQl11$99h6^
z5OldKr9@@BaH6)`lVaY<Po_&0OwDfQ$PBYsxM_bCZ;I(hC53zzF6jm}mwGPA$-6|N
zqzxsP6tK$hZQ{B#i$yNSkIR_juvW*a3zZvqQUoN+os^t@TD-A5fAjnqCI`i}p&exg
zz8>eJGNvmiacHf+HS_F(MGIDc{WL@3RjSaj4V<!Sehk$YABs&VFqxKW*lKvBdsT!+
z$h6QSU4b88*7GQ{PE~dADsEg=?>FU$-mi9N{>U@k->!s7{r}pL6WcAAu&iN^@YRXT
zc{;*(U!7F(yO0}G^JS*PH#UiA1EHDyCW1-g7F;(KzD^WeBkru7)E!ok$kdrM#V0PP
zsmAFPkF95HK-VFw$ZKy(nGU2TMeg1`Pcf0lvgOV6h>3?~wtdhx`g1iRwpD44p5S)f
zka=d)pGD7O3!N=`Q!{4!hSn6_)2s7+UnMR1I>GGJR>LP3wVjMUE88lq_Pj4EuJBU5
z@%LBmg-jEA9r=&X@m4D-UlWkv)t7A0R<%+=NW>$G=MtCSE|!#eZxuFmY-*k4rRcBW
zd!Tle(&mj!g00hk_PeOrD6K!i<8-&=mEr3}hwclRJ4Nw$H?tUU$@;DRzfHBT!C{lv
zyr+>Gp)%e9dHz#fc#r>BX|&j<yGKfr^Nh#b%p(_*j3!I&KK8Uv{$od9*OlXUuNLn)
zHT#lS#KdW3x<Pv*?56&gsFwT}+G2i)Q)3Tb)5*Wz-fj}kXs)_pKIu}NiW0}lb6L^R
z>DJ1<I!B~jMIsoF>aT4!-;%(ty~uz~^UjZRsakp=EfJe@ug!EkQ+=;O{qYjrgnWY+
zbFxb}E;V~~O7P0`i?7QU9=fi~s#g2J^~CmwT)mQ+l8SvHFYGU`UK+JOJMML=_{?it
zvze_XUFzPPuic`2ze&Sg_j6Wj*1ob$lf#y1`btLyyUv*qp_{Sn+O(3Eg;j5vex7lN
z{$KNPv0#Yrt{3MjPdPJLT+P%EayhtS*JIs+)skDp+)dk@mCd{7UVG2@_xGxvedeti
zR}}Qkw0G-iWmyR}f7rpbB*$=)T+GQC3$1Ng=9P)XS3mh=q_y#o*!N3Mr$-(=ac#xP
zjUS?7rls><cx3y;>+z!EMgF2Xvrhles+MCF30D+|5||?DDbboD;A_3}LC#^1@-1f@
zt5;>8GdLvg7BElvfsW7j0D~US#S@Ne&iIugt~JN;Ug_1Y*pQRLT95y9*q%?9RDZfz
z>63Dp^eaPwt3N(3D9r42`TXu_<F=TMJEnbQV!z~Y&*sAi_K(q<iep8yq{Ck+_SwAG
zyPU!GJ}2z>9K-+Iv*lt=9G1B4y56k&VeO-Ko+#m<8%Mg<&xkp>e93Gt^^b<)z9uJ?
z1MlDbXJCHOSE<RJb4~Cpq249lA9w1^S)1eR^HH#AouMCFE~ldP(+$<`AEt`*Gv9pY
z@T20@5v$I;n1lQqJrW~#JqRxhwey(H?7wWs2Knnuo|S9(U!J(8@cT@7)Bm&OC;uNY
z>$t0uxSy?8w@l-n%lZFX)_#gR|5@Vfth}^{B}-nN(ahP@S9K!$s)z5|dE7aZ5@!5S
zn!Bny_mx)8Ce?R6{(44zJLQya99nL~ks9?N&m?MtUKi`NCy!2lcz2C~|AR;00+HkK
zkND$HcsHhPFw01e@5}WQa59znf5<`ZJ!AE&&ly~w1e7l2=zovW?>@uDILnJmD#u|(
z!TmG5J16Pg@8Mn{bm(B6fA+Q9o&zgaus8_Ew5(+6OHJedmX_Adr^5Ko_xl%v+bLdQ
zS}hKf+yvD9`8n53&ugF1cD`o@zhXj){vnMw8@hMd>AI;E3TrUMcAd|8q{Z@7EBD0X
zM@~w84>*MvH|6a*@A9Ti^z|~O2*b=Xr~fdyYkTCKKHxJk$B^T>>ZFQ-IF-{+vksmT
zc;r=J$Y`7r*0D<Ao|2^WgqKBLVGRykWgfyw-hA$gUoTD2+i}u#N}gVgs?^2;o>DbN
zfrR{T$5wA*H~5#NV3xS&m(vM{4-t0FY?}W!$}u~8&NynYVfEcbO3~Z+-$X3kJ!{Fl
znwEeI=YH=<U^_g~D{7&{V}GU8_P8A9C%^ixvnVxk<ePAH@Ma||);RCwN&47yVFC*i
z*JgoNHOps5UHZC#i=jb*$*fT~s!>+ufqjprRDXu5rO%F<O<cPtW-e37DLJwFK*08I
zybr>jNiWvpx94x=S~q=Ox|h;{o+^%C0!bp@4tc6f)YoE?4-C*-_B6XfC$eSb!zIUl
zvOJK#%2}hr?Ej2IY1gtot-{hdIeKw|it!oWcQv_9FjN)D{t&{axQE4O#}Xfg&#^ZS
z{S{esDlg#Xv5jjYURSO<a_ZKj3-bcYetp=%=52h~P-oBov&R1q%cuu%En;}Rq`hiO
z>gyYQdmafgM?TR$%_7IXsJ*U7xA?RC;v|vSfIgFeD@I3*wk%bsZBaaVI(k9Ue4%Bb
zDZJiGm>T3>Ezk;Dm~c3D(ISri3k^ldH^ub-S3J&)TDB}|S@)3-t3EJ$O*?JOsB%YU
zt%JdvvYZP_&Py2kmtB-P%lCJ~<5Q=WZaDFROC@vOI)3HcZx)m9Z|QhY)h4)lQ{=8L
zzTzjYQ&^uy^f^sreR?86#9)f{WUh4yF77s0s$Vp*DH}R5Ivbty=(C$>q}JE#5vb*%
z#`x7pk!7_K$H@)zK3iT^b!NMKO`y}Dop}?RI{R7`rSGQ|W_&U}%Eh?kzeC)tm8Z&#
zJvXlXd}89KjQ->UftNnLshsoFU>ifv0gvc3?ni7+KHfT7b5dLmPKwfMn(uHn;#nc{
zem8?XT|EtxJ#2C>iv*SJaPWWJ<R2^i=D|9pj22##F7|u@3zbt7fAA@nUKG3RuPm!|
zO+Z8O+3FzeEX59GmFR<Fuk;kxwS1N1KcukUmQio^ijw<HOIN0BGCR3OcH4sKVN=yV
z27eFX;4*YkZw->+f1<TPQ*Y5nu8oh@SQR@d1^ftU`@yCxJm3H7tWZx6MrY#|MIrU5
z=Zj1yd-?30<g4xEvwEvs{1awPCZY1+*Vk6$Zpl}ua8Y1Y=$mmS>B4ixlqsQ4Otn`0
z&-G$kEVllPyYyuDqbd_$tT$v73;Mp*se0mvSFO6&m=>&Ix%TIo$6^!bWiIU=3kz?F
zB<WvcWX}$3W0jqKH2G%XqeDdsx3{n=^C_A!?pF)ZzAu;@Il<`1#}m(#(&{h&KBXe+
zf1ySv*~36@UeD&V@LnyO3Bk`7#eY^$`1<t}i&nvx<MC|)%1fEddE6BF7Bx<}5PZ35
zf0QGWNXxBFO`rKnv_%3mB@Hy@2h9)h-+N4fQ!VFq-@5WuTA}9y4(0GOGH&4#7jzaf
zN&ldv*2o{Zev<1_r<DDhR<K?7d7kwDjL6DOrKuaI`g{)5)|pnQEEu`(m_Z%SoTQfd
z|1S0&NK4cHucTM$^81sQg{7O@*Go>Xl1=smU%SBX854e*#XU;))WovK7GjG$7*C1W
zZBJeLar>47lGWwqHKr@hY)qbebdHWe)vrfx2`SabmWDn)lyrPc(Z<P7#mp5?Z7E+Y
zZgB6EU+^N=qefQ@oVAUd->zcS+O>N3npPgZOD#$bo^Kf?svrBXIjIOY1ukBcargE7
zAJ0$EX=!wrv}j)ouSIB`{Z`omA=e$#j{PgnQ8m`6o|>7^-}pLs=c}!5OO;l$ojw+D
z>DvLXH6mXgw7u^s%eXA&Hz(z@qFG7wYID_7LH}MX^4Tb^b<Ck!^ZT<Wg00;_c2_+6
z=jj_C3RiG3chOFY)jnJ+_<zyw7p3|qokP}ccs1qdb++JZ-?n#6{1(XneuCeUQq~2D
z&zFi-gjXEjan57%t2KNncMY|cm6t^rPnY3ddE-ZtD6`=+5zpwSLc-lfZI?SPieCAo
z*2#Q?`82;#re9|H2J>(361t*_i@tHN9y7n?U}>{8ewzc2>J5kR6Sr&FKb46J{x{}*
zepd6dK=}I)dh49LW_e9*yn6B1XMWci+5K0wGFe{laVzA%?z3t8k*HqQ<xMF+J^S)2
z%Oy^?@s?|EpX6LvX?Anf^uLd0C~x#pEisvv?>|9Z<n+7h`!5u4JQMM}e&B?~_X`^@
zoa*1(cWusj9iMO0<DWiqz7nauiKSfokyqJ&@j0(&>KC4pvESxhs1j*0VPbOlOksl=
z;qTgR^II^q^QNs{FuyR$!r(PmvhT}9)zezjEZJ&ac}AZ)Zhh!z)Fg?z?}x0pXFhpX
zuJm7hV!cSh{xixiA1QNJC{A(N$0M;Y>tU$vjb**BTdXWKLyPPurZwH&+_e6M{g>kV
zd>^yq@?E5kES1_=XW+}cMBOPcb>VIPcKzn4<jV!g5v!Sh83}Ml{Y+|mnzV{B)MYZ4
z-2P`~iye9*#2!f`bNK}P+A=ft>8JF`I;vg5T0JSXeypY2nK*fv&G$_Hcy6cr#)@wC
z;|Aq|3u=4oi<QeI&FXU$XTSX^dG<${Px1WUmd^DX_h$TGJ5lr6s$eGHuN8X#Dzy&j
z_zH<@vE6#(*D_yuaka5Zf8fS!wck;HB-Z_yaplqSg@Q&C+Vh2PERZV_&z^VLa=940
ze$i8|^95nd%OV{U1=Mb?S+i`hO3v#;Hq-rbmYSHDJ}L9_sBD|cZ)oAvbZo|%YOd2~
zJCbHjNQ>k$Nia~jGVjusi3<;Hd7iZ9<)KRF-Gvi7|K|LNP%AW>JoA8SS&{7a$d4b_
zTxK&&Ye{py(0-Rw@koW%msuX~-jtlwQjExFTDshtm&u~bP-%U|hV2#xzBbdnqmHis
zp?p-O@KL0KSpK}+=;<dGt$yWm*lV)4u0dqT?Tm>ohQT}bHI|j>e9PYV-_;^x#|CEh
zEi<jXBBSN)o$Y45H_~Wk`8Xxn<Ls4LJ9bTIHM})_&HEjCua0NG=-Uz6_v8V0rtsBC
z+cs}ZKkLgArf0W_*FmO%|Bzh9uLZXfc@7D$ZeYzxW#jpu>{@ej&kGUNwXYPzrtY>s
z-%(`r<JEG<OqYj7B2nedfzb;aJa#?3GgDWBBPn#6!UscVX8w(zqNBt)%{~hr;Fi%@
zziZ~~sVA=ANYHqE+-72`;Nzca2iLNy=2krEds9`jGGAdcf84^R&RXG{JNs`Qziq5M
zS83L2YkyXsT|WX>S~(XqKUwp{ptE}3!awK!DRV{ihw4=S;8NzQu$d&(xnLbjwxW@8
z{r?MdE!*6e@48nTv*!C&U&pzkLd!EGJsUk1`l!yaIbiF#XUd#whS_=779HPI6Fob+
z&VB98BH;^9&a9lvzS6kBBPRCW(}eJMjUrt<N*<H4A2uj&l0BhN(JI7iyLsi27_W#7
zhitcy>eSHHPL9$2N5UlS8Nwgx{hg50yld}`cX__&ia#9h4GX^g)5ziPpA9($hphAh
zV$1%S>TQ{NV0D!CuBq1)*R61CR+^Tdv+k$c6Sp^uoNlb_@6c5<@DQJ~mx=R>%dLw(
z;!pSb{wlrq{QH)Z$qg0EE7!Qp`^|RWaq^wbZpSB=8_emp{`}g=#b$Ec;g5m0mTZoA
zpZD>I{(Vo0Lt761pZjWK?P|sDBizB=-aG|UY&>_q`zh6zy=Are*N$WCL8lA1^`FjL
zf7C{GM$p$24Z)IW1}~rPke<D9>f5t3o8M<_SgQEyoUh$6W6h`GX=}<ZtudLge3yq~
z+-mNA(@5>R2_~V2PmiA2XYZV_cFx1ozMsU8ewDlbKDur5u`hwC|D$C839I*rNnMLn
zb>2AXhYX*eyq-2IlY$w?4JB6xM@9}|EtwqxitVk!0(pTa5|y0vc#XB5v;-y}>Xc<|
z5xF@*n2AwcdYK3VOJa)(J8Q;7r?#t3vw5Q{uXGqE`Uf#JX#JTfxFUejqLfecK-%-b
z)+Je*xf$J4TINP<R?C?w>cI4WJ!hO(r%~}~F{8y16SKCOE!1s!dvBj>M0R5AF3$H{
z7fy#Ch*?wnG3#yS<PFyjb1zp)n^ii+$*5Ow(#<Uq$(LLECZ-*mDY@|BX%1`UB{4g<
zyqr5*t=~~2)qB$F<;KT)d}k*f*5oWqzk0SH<qb<<l2-=r7P(8ex`fr-cf=$XKev^P
zp2oLj>bYgPED`ys`pLV)JcE=1XD}#E_{4OK`Gm#3hgAydf<5k&6g>ZO1h!@0+@fl#
zX|^V!JR-J=^MYS)rDn;4B<?hmseb%fU*;6uQuK&U?@OF9A;UFw$wDWYs2vjmBiCM=
z(5aNf8tJUfwc%DuxS+~`Y%bp=t+~v;OO*dRGf6&4cC$?r37o+F@K|)02iJ#Cr`MV{
z1DJ$d5-+;XeqnOGWuB~fqVvVnBVM|mve#5YI8vYA37YHrG%$3t;)|)_938G<*-<`j
z$D$l+f2QWL?rOan&-Ha?Xo67CiwD8YKT9O0&AzL0cX3Y7rKCyisXHF}CCjA?Yx*~~
zg?3CybD0$;ywpwIQ9trV+H8-uBeV0xQ%zDmg0)z*b3HRu4n`Jk4L;o%t2RBec;>Su
zH>Qau-BilR`J<lDo%<rCKZfh-Wv|&gXKlYw<#8hXe(g`sg{@+L?^Z5v@VXo7-^i<{
zk<ltAx%}#GDbXG8_Hb6UsD`bbq1musdEUx^V6)1qTmJ(Wv~@+U=uqle<dmG!mUM(8
zFy>|F*4~XLd(%7<<(<sUj?8!2`pDwx#Hp?-45v5?1%*VX-d!cAzO76|s4C)NjocJ@
zO_ie(S^Gj0HXhgAdSe-rQp<{qXLO&uFABUlWm$;S&7!r{Oa~(8v#j^L8I`rz<hEBs
zwbB>y1(Jfd#P|N@+7r;cY~fplJzFKyB64!8cTPCi-&%1cm{VV7mb?6>O*hlLm9Jd6
zFV(v$@FbUfS(u}&`UHm6%SBr(`zC51*>uv?h-K0X&1=VJ<*Y27H@PhL%au7n0&4RD
zx+j?4p5kY!m>R2l_vGvg<w1A8&k}hw&HL2clTI`8ZgC1(l;;-QXWjlkbDyz>?e3|9
zYxAAXT&k70zcllN;tCZrm3<7;%JhCG3b$Kj{+h(eF!NZ()UBM?Elywg-1}tGzQsMW
zgC$*rlmr@WRLqwI&+VACWFm{gffC7`f=|r?*Bs(dGqibmr)81y)|NF-EyIlG+_)BG
zw*SY;98IN=1tJpWhLWokznq%H)4j~R)wVU`LGr^6yD0|^IaxI<W#;@WSur(mY8TJi
zmor!zHdsc=K2uO_mdUO7yfgCa8pWU~l1`gKr5>2Hgsz=<KknUD+qagds@pa_*e^6m
zwjuRsWU}+U{ZR+4cZROkE328eOD|)={kJZyyffuuHU#ZGA$dHu?BuC0lO}I@o7mbc
z_n%WKiRrqfT+k-7MH_D%Q8{+fn>VmWDe9t{#x$4ynSNX@Qoj9Xn5;SzT)TpJV#5;L
zCbjjrbZ?FE@Q6`kn5DZ!_2=nXlf#_m=d|i5Ds~_0QS+F(MuIIc>xu~PRwGe!H+PM$
z&Mv>C7u#A-thL<7wWQo<vboZ&iL<$0Pm=uE>ej!8-({YQ{6i_Ht^=M~hP{*0?Uq~a
zI5Xi;p$kh!rrLC6O^=53Rlg=`cSal)IeN*&;`W_YEDkfvgMtICcFvxb5iw&~1HaMM
zWFF~xQ$^xS-IZHyc$B_t`1M<?H+d1_A*XFD{`iO9()v4{3WA&G@TPI9PjV4jap}py
z1sv~Iys|#~|FF=ekjph0LEH;vH9M})Z_0UFqTu}Of0QOGqgP<VN*51C!NZJELWSxF
z&v3Db&k{|2mf^bA;&xox4DsEXS#2s@!k1s{y1pPFcR80%dh0Ki8!l$No~^pQ37T8|
z)+(%7&n(><peMQN^oI%a#au-0Ye-*EGEylG(d7_ON|@DdXHv92>!xVxOF`{TqK<;G
zJ<<MGBxmacSsEUl@+f1)jQMx^M2mc8#jGxqnDZ!PV$GE6mW<5;_vdagJN{K!a-xao
z0VQw6e_666HBKAcL)$i6vE(0^V&;FzVQbQdlEAck=^<X7Y;J2dYV{^4H*DBc<R#oW
z@mUs&3e&G}MXld=a{2xjAGp3MY&yro6CPnr6FwOjgsW$spFHWn{+p+kHF7L`?{v;c
z>dVTdnWr~$%0AZ;HTY;8`Bo*rA<KKKgx<H-^w6B1y_pNkpHAz@ys19($xlzY$<ya8
zS-CvecJukB!u5L{1=l;VcwLmd&8fC_q7#e2f`_LZ@`{%)og%WbV`^Ja*4n9xx|z~*
z1Pr+koCy><kkk2G`5&)~aCC!l{j)psttNNdr!{`=ePOdncMEGJ<D$pK&ZUoco|Ew0
zwj*<q_~+Fkrqkw@pY;k*m-O#l<9~AN#uZ_+vJ`KXS}5!9n(xT7YDrFLWWDWd4`D&0
zC5%}rN^A8#oXTM}zMm4AnCy1KOQPr6|0yCz&So83m8qogaE(WW%}uFdp}3-{G5Rd&
zr~NptXI<(tF`Z;q*(}g=+|ol<ZCd5+J0}acOCvlE8#F)E4Qo@YexdxkVg<9aQP6GO
zRad17jf|zX)-n`KjaO#dA8+zOFWfu--@7HsnO{8)*)B>9oBQh~Yxs$c=~EiC8jUV+
zTP|T(roE!{W{F;N+!>#rH}3SvY`^d|tK+z@+<Lop+!H;2neB=5x$e1tVn<J~M5?^n
z#-0t^kNddx9J_FR(ZiUht3KLx2$mnYrkQc>^qkj`4g7l&`xo?ZZ?gNbHsNSzf|0<j
zLr;_{g_OP}ME@|5nrGqE=lkZ1#Y)?>GV43%V<b=i{J-;7+OJEE0dgOYUXW?#(K>rV
z@bGtu?q?SlMN0j76Zvv(#I?hjpRT0-&YN{5$B{d3cE{5lVFHZS6JH+kGd(Dpp%Qd<
z*ZYKX22Ycn=S$1>i>$qU_myZyjLY+<%cT81N<>W_7un59bX}@GshIB%ccPAKU-taY
zAmdG|d6pksCdQDHlr*vPf!Z|ok9#77nO^@_bGf#MamT^MaaL1i$Y)z8CU{MAN!#^0
z%kXEkMbfQ%0_#i!IHMF;wS&Y(7)3XISoo>Du~pQ_yyASZV-0%)N0Q_0CehBtna7q3
zN?dA~$Yim3drCxMzMYdmm};#7qws$-ft+aqzg_DVi)+P*X-Bk({%3x!^7eVtlu23~
zg=GdO<l2{&O&1o?5YpWlUbJ*$HS?mPD~F4Ywkm98XtDDs(scKGVkTl{E^?sMU~_w>
zIJ4-yGKq|XcI}2*iwy*Sm}PxoO*{A9b%DCfrSw#}g+`CWo0uk)vWE$BOiPeGD9HJ+
zlyjqy#bJk(w)SVoU8`0Igf2@=3YXhCO^frm>!;<}k_WTDmCJ>w32J{*eaIGkrnQ0n
zhHp(!=fa={ld?vOX#%qqL>B(Yl>3zIbR;k8VyyNg15u00Err>9t@$F?q;@iion~oT
z{Y`LIv5(4&PL74Oo390ISmFG1V#i!#dCtb3qu*5WO3NF|TA~DtJUgoQbp-w2Ro<)1
z;(Xau;q9UZJrU3JBjFr}6rUQ$eJbl)mf_s1-Z<r&gm0^2fT{iBDOP{ovLX`&r#Pnb
z1UA_fO9=~xZ8c$y5vcPk$#`{LK<Gnz@^=+3!?eW4oQ55)UtH_YeN#CVCfPefFy^Bu
zt07ydkb?Y`=BpNyjK5VsD@*c7vWRICu|Fab;-ML-;qu-sZlcp936Cx%XEviljXNh5
zJZ1EW5uPG{qH_H~S1*I6T@$;fe6mbS)n2A<zgXBZHM~H`Z0d4G$G?GUR*t>gA7d`f
zD7>M;5a-z<XE=@b<5Z5OX^A_9Ju0NQDsrdqNH$HG9QAS1@`n<~Co9bf%jXu%Vt2Ip
z&pJ_CK3Ix}h2edoCD(_N^-r>b874$A_J}KH%RX$|w?#tzpsvrPnJHm%LfeGCiTliX
zsS-TXzW92==^3JX+Um@Hbg)gdX53W1EHOFZrh4wlX?NJBIn9`D=-w$_8lk_n!RVQj
z<&l(^p<N%I_)klk&i#3kkK3HN%H63Cm18EW#0fIgdsfb8?g^jnUDBx%`m#)jecsQq
zSr^!irYI&|SRokyb85`Za)*f7P3aLEGc(wgin$(ZUfg2N!PIyte2z1V{Zhx8r4Q#=
zGv?R6R9ax>73<Lyw9QqgQGnrqXzR|2euwI~3WYcY7@~^>?q08bd|e=Khwy}z2CXv%
zxjGZeJHq}~uUO=@so-^^@u%`dkrz9XzZrWa%XT00xEtBKVWlXWM};p(g*IoTr;wT1
zNyQk8{JG8xxETZblGP6?HnB4%Uim31w1DNhx%6QTrR<7+ogIt6q)jdLNQ_coTyn74
zVa2reh0CHC=TDjLo@b?T@MQ1}*Cp?!3HskCU-u%}@8XgdN0q#}3}uhi8#!i)SJrVb
zu`oUmJ<Qyr${Z&2a7F}UyWh{HA>}Kdt2Ya<FN|nh*zsy*X++P1Osh}s^A1eOH`1Ed
zE;vhYnabsKC5i8PClh71WM-^#UcDny^6k!PdrmpoOKF^*oLM|8uxO=ZWtD`J+GN*_
zapIpue5x8{+$@z}H2z=sNsV&>i#=17PDqsob5-=JwFy?$%T-v#mdjo^vFt-;@Wz8v
z3wKH6o?ORuMg86jkDJO7$Gg_J9<>mdF{kF2=8Tj%&sn_|l=!=tTgxpp{9>%Nq)o0V
zb0I^)%x~KTrk_-)+Ogv6#2IVaL>)i%Z!dJ%k<v6*m{IlA%2kyURJ47%D$*9*obX~I
z!?)=|d^;x?M73+|*5$pX@@D$x-C>JU76|8V%y_tK@n@+P?-Mo?!=%(sigisEGudL{
zCRI4|qR7_G%U55Y<7;NUK(#3<L7xAM)RUbKUk%rGcQ!2-FO^>{n7&%bMP*sU$E_*q
zN&+d%Qa`C4JGjizI{7N|awAngDfT7*)nCdghAodYG`lODzx%15>#n|ktJb{RqSXJi
zm`&RL{?Z-W7A}bQ5NNcRF?ZG?&FK2JpSm}!wr#(tymG5FU!!#9?^>~gtm}?4$}+C%
z)|eM_MX1NZXVK1;b9M`)ipw9mGBbK%fn4I$<u4u9X*G!nPAaO(yXxeTvU88;i#<v+
zMYn$6vsY^G1?L#;C)2jg(h!^E>m9hGN889tYJ;7b*0WZLle5-{eOj|rRWj0BVBJL1
z7Kz-6!E?_~cTiYuu(Vx}W#`iJ=-zNu#?nJFFJ}t8@tpSRRg7ff_Jo`3cCMPIvwWTE
z^8;T4f|WX3oW0jSedctuX#KLn%wsp#PjrlPNtH5ODfs{S(iCTPdu}a3w$!Oxe>8C{
zXgBU*5PZFoM}$GQP)=$U%i)U)xfKP1g%*h~RF*YlEr>8$DJ`vNn7VU!UDuCx;pk@0
z@<qFs&76IE=bmV(l`~iBNw}R63}?Dgv`cgE+0%PcWA+wi&G-DY`nqVJ+D;9TMS<R@
z=W~f{o%D24@A5eol}+KhXSkaPDx8?SE4umWFHs3@SJv>VoYbiQ-qWLlmzg}2EI)Xl
zUuMgN3I!Q;rCZw$E?ek#XX*U*-SbN&4|z<Q?pC_5L2D<IwWqJC*Qa8etxY>)zRbNF
zw3b7VMS4|02jfcSLWgJp6`2LKhu4TsUFq$+QXuE{&eJ=N9$4M6=k}`qD>UalRy=FE
zx@_a?GZ%f%dNs)lIZC|{ntsf1b>h!?={|dvCaufLSuK1?uWm`_e$6HOcG@(!#FjXl
zxT&>H>295>l;ZK~xS^Tk&(u9fl6&SRul0OTa&y;-)YV69o25_IoLt>rVzSW2TCk{f
z$^rg6%XoQW<>o3~X>yt>=;yV=YWc)1vNg*$er#A8v%r0+B9oQ>TA7_px0R}AE|^qk
zGdn=wvbpfjYXZEpcS|WAJrp9xI6=Vutcv*Q{(I9!B?=up-dIX*Jkq@}L9bDc{i)Dv
z_RZQ^Cproj`36mV_WVrxvG(+|O}s4Ec-c0`dK*oAc0lN?Q_h^=W+7RvFUNd@j~935
z{l9bgxO1Doeyp_5sg&SJ{Z^K$4W2fs-%HcKiq1YRWt6mX{R}}C%R2tAM*DW0`QkS3
zu(Gi2X&GzQ>zYcpjN7I)JhCoWy#42*&M8jCx0F50VhU7i`p(?Cq`f1+UrST@u`yd}
zKaWb{(}g=#vgE|9ZQ?>DmspqH|8Bvz@Q6!{kg}k6h}66@IvXVgXYmTMY|B2&>F$&K
zclYg{u-x=(8B<x(3iawXs$2eYdNSwOHp|`=%j@r|W?zV|-l=+jzK>$p#DjY~lWab2
z(V7(ZYPC{x?1mT3UF`4teO~SFI9OwKGE8Ri4Y|f^R$tUQb8hnV3Y_%OE}VOlQ^07#
zNA2D#*4%Sc{tKPCxYTyijm^P7YA*#J<XSJg;8{jy=jjTgm)Dm>1U@@d!+bY?v6=8?
z$&HeNS7O%)hM&pab6Z7squj1LHywLK(@ewo5);m>wR-XJh$i>q*g31_W*_xG^fbax
zE_{tj@;XtA)pcJtvQ9GM3U)O8`ha^+xzwvS_YA-3UA+2en|N}SYz@zk<XLG8lyojn
zw!Rtd5m<J}i;Zo+{vH9Tsz#2tC4Dkl>dh;68BUnwcvaReS$^K-{l>>;&Qn<?`~0kn
zbk)O`){UzCO~OCdT;e)nQE}7!g~f{+NrUy<A2ZGEPyONQXFgYb^%<S0{kIOS=+tzO
z=eoeOz{r=g?(E(-%L~s$v6THclG%MBX0z+O-LrZ-PR>|#ZI4h`<lDK{%YHvhTvMR>
zy6ZrO&FpKl&pn^>?nRx{oDQe`S(pCUF4xX9c=h^vn0MC%=Y4kS0smRL+W$S^@w*^*
z__f(?Ien|@YqJ=H7o5=QP*A>A!tup&TiCYwbs2Fbi%(rtP@gCII`e?)zGaH~(-isz
zvN|uO-kkczPx|Z6bCyqS3|pUH|9UJ>Bwwzk*j(=QKg~668y|~=rU!3eb=1#wIe*#C
z$x>UHb@#tR*<a=Ot0ZpmHXBE8=C9rq+P_I`m7q<{!(|t1Psx3ZUv@RSc60dOrYnA%
zQ|~6cSjb?My-B!p^FF(?(Ry#M&HWbeD1Bp(dvtt|_>N!qcD}OC+I*{V{hn=J8oR5)
z-pN?1xg|Mr9hckqJmZS0=}gf#)(fPY#2i#aaup)~KYXy*<inY@FKjI5dfGm`{<ga>
zckWH`(go+E<bMb3a%9``<pHmt00V>KPZm}Y1||j_1_lO(1||na29Ey>tQ;aQvKGv4
zOl8xGIk7?UP^-!bgDDyfQx13YxpQ97xS({rRbG74O7#Vyx~*@Hw1^zYdtlYW6@GAz
z=jLT+XIm7%lCkJpetx?AQO*eU-~&4rNwcpLG8K7#d>+%nCK(Np$6k%$y=G^2O<t&`
z%N4D;WXFX^N4irIXNewIwJf41fA^`-6U!23Hx^27+oioBG@3Ez7FXWZ=^t**j<LA#
z#Arf(GGiaBn#hYE3l5!}z?v{`=Cp_RyJSWGJ?)z86BV@WOIFm^A_d)ck5VPkJF`O1
z-{39ezIr0$;qm)SRr+hJK5tIA&J>sY=goy3PAiKSUskaT{^9fJ(9%iEc4moOS|oqn
z@PBzmep+nX^QEif&+n^^4_fQB^^r<~_uq3us(ErZmMSJ6V`AJ9;P|d{YkWG>RHj)B
z9B$h?{oZmoTyogD!T6ZQnk~~VSG+5I@vz<OtI8sOch{$veJ=!Vz2F^``69_#vyVf`
zZ%5*kdw%9j*Dh3^xhi_YK}=TdLgBNGM<-0MdS<%Bp;S{-s$-3STB`PMPKWtUOz);V
z_329#yO}(F<#D!2i8EENMe$xz5v*Nd8q%$H?`F&FHm0i)lZ%-nw+jSSUE6ZkL1v@U
zoPe@-H!s8+v2uxOFJUx)Tve~760@Y|S>*C1rQeP{bdZ@gBP7Cn>!;bSxiekU!u{-4
zWkvGNjY`vA;`4u7s|&-e;|fj;%T`WU%Xr9T0mHO~y8<LtRb1XUsH)tMb5DNrC80%7
z@5-8@JDOXwy41K_=7ii}-1TZZ*Tu?3fwv2#g}Y?4G(@*WrOv)w{(!IKR#BB^OUkN?
zxm?2Sl`pd|7cvN6b6aD~S+SIXNy;YR5M$oX3)8t+UtgF$dFw~{L&a-dx!R>%tpYZG
zva<-t;tG|xBk?Hdijs%x+Cz7O?{TTb%>QWgBqK=b?93~>tdIYB8D^@Vec8S0(x2y{
zy5>g(b{kvHRId8Qru+5E`-4^T>$h{cgf;VcZxcxVSsR_|yld0;Nvju_UCC){jpG(R
zD{lHq+DF;Qx7=CNKh-#F#<%|}=d>=QrfxYiO~}t8_;y@Mxo`LG>))0pul)5m-v52c
zs)+hro*B=)&BBG2FD$KFy#MDk6$8!G;JX*<`buA3NJ+8Wv%odEY@fisx<gwyPx7u@
zawX&Wt#y-+$JVl}e<drlf2x{fq;+!GDd|_nFRIFq3v8YCJ1yqRiSqwT(%YFhS6|(p
z_cmbNrzBg!2Bt}0wcTdyy+7d(x2m<y1c}*yi~S0HH3BwoJQ;mwkx|#;vu72Jjb)sV
zY_OkokhhWRj#sGFx6Oea%8h$&&UZReC>m%}kmqyq!C96=4iCP(Y--!QU~!d@h{VsH
z*vYfvc2D~}Y1)<+Ii>~2J^e26Gk!@@uody(|NmF9L9WS1;o>j3TL;?rw@!Gc<i93T
z<oBZ)CpDLJRb)9&S<$j<-G+Q6cHxGk4Jy_z9eRw?J%!&nEjQddmCYo^@TSHVg|6eC
z>=qM~=BagUb8kGLnRn*$0)}4|!tV~MOyEuIl9O)oSn#p8DS}hQeU0*Vcem%wc`Zv4
zQa15LZCLE|b%8s7bBAz{R!heco0AS6pJpC@(yBc5^E{nER{0+W8DWvTMQ_L*n$R74
zCL;2*koY!*1qqs&y(WPIn<N(buuV+O3GVh5y1A%RneF*3u9n%R)#d_hMy-uJF&@h7
zmJ?jHQs&gDJhU(s^n2Y>toXiXdH9hH=bWcqR($A^9{$Di|73|<E*dk|a9U?HIy0`=
zB)4_NagF8-ul67QiYl2I&hn=OLf0L4x5|1{dF_$@CBc(%xstLmJzw_Te#q)$@zLA!
zmcml4)<E^tJsb@SPR=vFb|iFV*HQ<Uw#m9nR*4n21lw-(3<-R(*`HC=Lrtt>;!>@z
z^WrXYDoOwJTQSLaRo$Dj0kfx1p5PJGHmzrYbh5KY&kh~4FALaRRgYY2`;xrY#b&9W
z)GHSiUt>L9rAtvfmZ$R{z6_0BbKW86s{Goy{tB@(oYt!t2giNP;u25pKXRdCqG6kF
zXy(TV?Fw}%%?SYvp>qWz^o*`HT7)tzw_WQLl6`gNoUr+8O8G<oTHI>=-($9I?bPQ^
zE`gzC=aw|wQ}hZnatUbSYH9!bV_}Twq8Rx}MQg1WuI4W`^4+;)oyCo}r$n}T+fK52
z;yxo<GttYjZF*8>BJabN!>2T?j}|B{d$&>3XO^Vt+!g%FDH_eL4$GEAO<A=q>4Zyz
zZ}%-mdr{>VYt}Z2PGV(rd88C5;nBisl*kwop(uLpLyw+MLXXLeL(D%5e6v;yY?yFk
z)wP?-b6!QbGe-35&RvtJ@muj!+mWpug0btO6N{1?zeF(Dr*)KgRz6lRR&n~aK_ix}
zXmN1v_Vp*E<zKyNK7C{2+dW24Ri`ntL{3?zYaEyC5z`d9G-Yaol8S)CtdEQL&wZ!z
z-&E}A91)w_0(Y*Ztn7%<%A5Wxq4CR2MJ2JqPVq^tJ~J$iw22+)+~0cJ-n!*HZ{e4P
zAAcG$#$<@jD3i2~C_F5CZL#=#y+tMIOm`J;v+NRJT~t+du<J^!xSHiEZx2iNowBX#
zi|6e2sJird=B-qbuKCfICLdOkn0AQE@mZ`WJA=yoSr@iw|NhCd#z@^}Ut6A{T`7m=
zmIH|b$q`vic_$WeuFaTbBBAqKtb98wLu)BN3l|fodE3Lc6F9l8gjB>nm|9;m3|rGA
zHrHgus@j-->3%8Y73(@D<|jRA^>){pGgUn7#DUAfGr3MlT<=ai(Qvv=!>oQ*Acv^H
z>kVtC-m&ZX->73KpC&%r<aTzKXLshMPjBXCsBEa&UFMt<IBy?M=Q;UvE(;_dWwi0!
zJ<MD9-B0V!^V0u6U%n|TxRlG(S#I8PZ=&1UYpR-E-oi^Ke^JPAQoYjuL~Dt><2U&d
z^^yX`&FT+bZFa|tZ@Lz4edOI&rs;mSH?KT&<oLTcttTZM^?Vu&8fJV*J#qTF^R%?Q
z^3FqAGZ}(5v#PvFIe4$FbB`#~doFcHhWXoTUO4JMY)z=0_My#5EJU7F_r=V4i&rSO
z8C*H(zbbSJm#)Qx(nkgI@9%xLH2e{B>tA&!YoPMgua$lZ^FL}_pY-^;Y^CqJ?v33a
zy&NXI{gm?F=%%)rO2n67kN>ld>o=q<txPl(zvXjFBxmE6MH7|6c2!N0Yizx;U(sX7
z|L3fe{$Jr%_%C*>@_Rdrz=3+Ezy^VWD!#ym@9aqv12WbvXMJ;}^x)CVqf4Dt1hY<=
zdht2dSUt2axf&y$n03}&c*&Lkm91fd>arC=K}v=>+mETZ9pb7#D)UZVV)HhIwT}d}
zL~~RHRo5istZ<fRpPVEADek>#MEmjzU2**rDmFfCf>(|Q^KIm8c_=XTxWK3D0wrk;
zObV>ijtfi+PfXM>jac3gcD?e>v#Jg1)w7>?sv9QnaCfnN)@bV3^g6L!^@XC>G#4dh
zu@r;qudONuJLJx^+te$j{$PxcRj6hC|GBV9)Th~0plfPvbdlQ(58=wfx~ys5JQD?~
z#bqZv5sP2O$(xb*RXlO|3Av1lQUQTB)*pGVr!{QUVBla#WGbkiD3B<m(U|1WxF)fY
zxwVlqw9NHN^=|d1qV37n3w!d+3*Ihl&vh-Y{-M~uF*fO<%Dl(1cW$^Ct`N3Y63^Wz
z9_SROAKJogToL_D;8jILQi`?=LuBW2IkCxxU$6FhKhwRkT|0SWJktiwZ6adZ4sazm
zusr^j%Nv%MaIv*SEKz1#>#K=XtEaQBU7iuiP`&!0>=yR+9pA%BB6^py^j=hzy>uf$
z{%6nI#FY65BZV)iq&?JMxI(V=Mom{lYWG%wOaH?oBm{%|R3lymcg9>uz4yGf{G+M}
zvwe(1oy2yR4;PEMH*$X6UMc%%0#9J~Dh-CM8Hs@ctdSoY+RPfoAC-!E^hiaNWgJf~
z-{Ez6#x$mhjogAwE7e@GZ-(VBomTX78ei%3R~yTC8Oz^3oX&Jn=*h+kGr`G=0xf4d
z<$mXIHLFlj5%vDwq%t`<=UIko@Mn3C6A~fQS`v;G`TS5ke`Th3lTVHT*Otef=RR3_
zIZf~^n7~_D@-(gVt5~VxF#)D9U+IT!|G(uoMe=jCPCC6J`O*&I6B@B@W^>F6o6el5
zR@f=LYD!JpSIte^ninPJo?R*2`J;y0k!{oD$$c+6xIPx1|9{PtJF!z++)!LmtVJ?S
zC|vl<Q^ngZ!XF%k-hS{3)MRKXOXQVcj96acl%5g)vwMGpL-vBq<VbGTrCI5f)3eg2
zn>WtkdDvt2us~?l^dsRZ+zVq0ONI6c%vpM@=aNPf&sJe!rG=uagk`H@M2Z|NC#tm{
z6c6{vnLAOu{Y3L6!M>kAnjU>E51AqSmZK=bsG_yqAVjSH>qf>;9fc>B%rGw%Zu~0v
z!-<V8a<-CLCfCP=C<8WavFf`I^S2z}THwOJx1n07sfVjv@KE`(oX+V7!fR@ea&38F
zZ+dDO_rrpU70Jpk<!qPfue_N3TD|%D%a*NACW|=EGvC?wlvDV(N2lZen=5>O)Xv)}
zu*^W1GtvHm(Q<{0Tz;Ab(t@lNZcef8jjS!1hcwFAzRp&TT3+e3yvj2x=jwty_1L^f
z;Tq4?7jBgO3+4XzfPbr((YFa&E>jk9T`XS4GNbxN*{R4iW{2b+e+;}Y(fa$7N#_Z!
zsfV+jg(QwAu03L=;;_QCUR~gYhUtuzGe7QV`5YnqsdSz747o+0<ZiJkiG38?bC@gY
zK&G0q;!bmyUFHR+Zmw$FCAYWT`PivdvOBy?ZiI2C^ZyT!dpBKbkMV@~Z|hfWYGVHq
zu39R5>DwBi)g8BQPUc=Xnd!q^amIN!!ZtB|jOe{pXs}V<%PE-Cu>V)7_Kd7z%ZqFm
z{|kw4O%UW)4OFre7SIw<j}lhP5_a4@N6V`0q~Z)`MK-VMGRJKTSWgQGe-h7qzTvCn
zgrM63vu|yVyuFo=F-zjQGm{F#1x@jkNi*gst-ZT-Tjg_?*ub?-hH5Q=ai$*^b0=<p
zow0(ksfcMoM07#LoS%xR2Uw?73198%x2h0Zy0IevW1Uyp7UsldtG*>q(w=@SvPL~K
zdDiY_CGOmd47lE;Wbj31h%$+-HrP-c>B6Zc@a2HUzt1i6i<3VVax*{V7TdTr;Kj1w
z;^oDx!eU={IA1c{y-itavepLejW!Y9X@RLc!oe!11yp|TnV7L@%8l)ZRTT8ab;<?B
z-8*$WEd{3iH#JlU@|3v9#=q1{>=0k!(Z<}HG4@Zm|4vvGoy~nnOL0M1*|`}73Kgr1
z16QAvEMN4Y>*<$n@k3lH9}^ZSG|d+)<GHwN<3;X86Id5th*`a$>WhQR{|kIe0^~}b
zatkk7$&|=l_>lj<gV4JR3-?^+n%^R5AKkn1SM{$;>%>#Dr(B)cwz9(UN4A$xU-BpM
zr9sw97WArK)vtTFb}DD;jLJ=Jx8}?*6!&+r)}L7{esOViu_cpwukEo7TnDW)rFNdZ
zzWz=lOP~Rlu#uwqF^8xJ_G`6wiK(%zzTmt?AaQX*&zlPgksG)QRk>EVd#x0#TPd=2
zTY{L<oA?vI70t^w#Dt2i{r|;aiNJchjKxXUww;|mcf+T7&ys~N-S&QATHEt#d-SBb
zDT?uaNh_yCu~jS=*0XZhRkm5gX~G+ZZ3cn+nG@MfCN5(CeyC90MZ`*6iEE3xhl~9c
zXVFD6uLW2G8bqA0O#7?A#Ua4C%7s5kAhE!R>+Kh*#SE-b1qX8&SwBoWtJ1=9_7b;a
z2%Be0``ZAvcMAl*gz_KKI_<Q(XYZT!Ig*Pm1#$~E>Fa+liYv0u4d9ym@H}VG6b4-e
zX5I@Q8zm1+5%@bt;A7yF@=a^zESeV`!sfQ(WPI~>1Hl~+b{o7oyuB`aor0#I@y+13
zmIC~bv=%kkZnde*Ph@9*D52ZCjJe&}>c2^V-ex|RZF8pHo)&K%Fk5KV>Itl?j|*){
zs9bbl^@c{4NQD`!D_Qf;XkNO*|0ZFViXqp$D0Qx@35y%9JY}r>Bz<V|@^egqhdg|R
zkMB%Q;5o*3@o3iBt$Dk`E`1dgsm^`HDRo=Ah-;F7*hGO!Ua6Yi>vJo`xh`3;@ZPAO
zd!hC3_1Bi-kCo+vIF@uqbt-mGo>03<?z(_|=3b|j!uvm6GM%Wi?2J~w(PlMUp_J8&
zbPU<v7-YTC5%7~NWPiw(rXaalSD@TM@moQ%gDKmducH4wCyFg&U2Pkiw|4hVp5wFC
z<a}p4FYuli_@QBr7r)XnA=Z`!dpa^%r(9aHh3nWEuD|~Uj!Rz*5S3xIID2=Z!19Y{
zGGjdhL|)!y{<wN+0{6cOi}H4^U+Zu`<u$jTrQA0Mqm!+>T(nJd)z5J$t`T&XShwJ4
zPq>|PMBLM*@qFhv1^u=Q#XS_vo7w*Hf#P(5UkfFdUO0c-n~V8!tjwY8qt}95ECfV#
z*Kq`EE393Y=p?*A=ayNjLr$Y&L*lFzo_i$@)+O?4u<g0FHJe){SRn60E!RbPcE$A%
zi}HSmm6lvtn!jf=S5~#oMiJ8_#h6OJ7usxWnp{&qP2(>+v#Z_9nO8_Y@=Ed>{yS&I
zL_&{CFAO+ySS04AP}h;Xdv9*B#B#m;!2cw1`o9I6qyNn7pL6=UXY!~2XD&OoEadY#
zI&twMkzPYhIfVl)adQlIG@qXPLNxNK`HtCqH)9^WoHfxvB=*&GxuQ20Z>)7l<v3h;
zu~K;Eo|EE?oqsZJGAZ7CZYj>wEBLJ3e)*lWJQuTzpLV}7;99?CmFzF>rw_Td=m_L1
z2yE45xY#H7_Ck!hV%e8~g8~M}PnJ#-y(GIwSNOX({~a@r_OOPOH~df5UHLCoruRAX
zs{-2%n>z(-(m7l6#MhL?$+0Hb1!%8{`J!<A?*Xo3HV3%_xhFO-ED9)NvG<s_%X$6Q
zMi#|_cRCEe)_O@-NjQdWbC901?$n_P-gm@aeLN!TA{sBu8YsY&t143XD$Gby@BJt7
z|BV;87RIgMTNk=XjQuj_<b$6%g<d}3ddU*Z_HT`XeB6v)M){CUkKXLK7%BGUmDZBI
zK}))(O0;vO_O97j^l;;)Ng{T2w;}|e9JBS^k|TLJ?nR~T<2M3{%V(e7a^RT!G*)hX
z#lV1uC<CrlOE`1-13paSWOpsd+IMwl4F4L1W6`n)0vYyH@vxq94F9{KYNkVS#-3fy
z8^4zdwIv%UZgI#roperQ)1AUf!GF!WRptwyT(hjL&x<STg+$^5`*$k223*SnPsTiW
zU=Sz#WajTpdg48WFBiW1y;N_OW~g_=i_TY@S12!!IBDO>Q~akmv9ndKE?uN9b%SJP
zVp^Hwtyd2XFZ^%1<$2NhTn7K$aJIC7+iDw*@_u~3BkDWno@!qHXI3rFAKJcuUBH^u
zb6I2&C&z>DAMfQcC?=e-Vr1oB*UZowYTw56igSUuYr@`8k3}0khksbu&?Wwx^F&5~
z!}_YE2`LMo_s?}P{364Zu>7gY?*6|oGBqt$O?NY7+O+0(#D#FjGS+O{I~rH+Us%X$
z{OgtYr3dF0C{6BKqZRZzY;7d3*`1T2OG27`TyF_qSd}0>amkTMf(+v882ngw7AOUW
z>&NaZ<(+nsYX^_!0oLbS8aiRw;_G_2t-h_^I=SKG)Sc6WG$lShGg^9MRqEp~hG@n$
zH<qMcZ3~$lmikSrO;*}Wym`z2-0ZtSxAopO1zS9M#yk5<+RdZA*`m5(y4s96ZlX4|
zHLr`C+fVjv3O%h6$hbHn_Ab|ro)aH?PVdmX@oIX>+JAXm0-HC+Ez_9!iuLVwuI1Af
z&064caowt}x+V_>UzQ0cw8H*Lx2~znYII@<{1d6_^lVF#>!Q#mrPT`m8FW1Co9CT&
zV$gM4sb$Y-wD9R_fxSi?nrjbe>m9w3kdm(>oh<FO`x4Kt4GPb0oXFj_eTSEh@$Ma5
zdhY6T_vl*Ok`ZPw_B-|9kg2Za!81F;Jbni08g+>*IB|HMNCH>$nV2GNqou*YXLV<Z
z91u0RsPfRyc5X<w_OqHRt5z!fu30*T&!y<-f4*p!MP7@gUteTt^r?yA>@s|@GF8^l
z<e+Cy;1=n}DpkKcv!-e+x)Gofbxm^d0-uP)<wvG%aq?JLC-$aqNmJP>RUr<SgN7~|
zn-bc!rt@5KnjHFqvs-(m(WO#tDOoPfm2D!cv!Zh5Z%Mznj3r{rIjf7(>vru}^7z08
z8`cL(cOO|SJZn0!qFZOf<7cbPJvY8+G2e0G)!s7;^j4c>7@hMH?AdV9##m$5l><3%
zO?SOWneM6ndQq=%`E%XLtIbc}aqzumCztDQa#BLiquC=ca^s76DwBI&6e+j%7#^CN
zCz{K<DR@cv1!G4B^9#ZcriHx;xVd_jQBstX$`*@h(>N9jDF2@<8h&x-ya{fNOkHB3
zTUR)`^R7CY<(nnxx+skMRk}!IY}88Y5BGLo(VXB?lsV=14ymgtH?3N7-5FWf=eRfL
zWU^n_zQbbj6Ls$&r{9}tf0KS7qiNRR)hB=2W{=MLKQbGNwlfL)Y&`L&B4zEBB}%?`
zxw8J6o#dS~x!aOgDQfkdo0r9lPH-zdJaaSI+fU_HSMX}NqxCU%QR%C{I9M)Rmlb9h
zsL;_O*7icvkSj1Oarv~iC96ItZF(f<T-n5OK`%IRYWZ#T)h`$CdLPAiB~!aBfmdwx
zL($dq{<}4at~Xg)AMM!9u+LO)W5zMF_NE04d|M6gp77HyN%AQ$ahKkiq55CHq_KNS
z(&2-}t;d_c2l0q+QW0woJl4u8Im07LWA@2}Wx}&0ebR4TP~Ezs%{29f%kh8dGFFLx
z9mS{4ia33n-4}U5VX?GQ&aW@Zj2m>+m8AkthlK61<~;9y=HRW3+cud@yt6o+i9@Qb
zAWfz|;E2+i6CU<1f~>7EN7Z;E+g<i0i&|wZl3RSh({YvB>aU$kvXTT?b%UDB*KfJ#
z+L(O%^BXZwud-IRC9|H~*mY<sbJ5a@<&BF@)`+_XO}muRz&L}$Fk(YjDvx)RmSoW(
z5%DVzkCpb_p8U8#a<*oHqwkdBAPrX~o3(3B8~id-&FNF$`=#TF|8=dti+oE}w&|>Q
zWBR}1s#8U&!fHlOk<CI3x-V8tUUFe^R;ls{^ZhU9R)y$CPo2VRBJb&0&Q&n^`kKy&
zeOu*@&Jk=o(KBav&kt+PqIZU}4}95IObIyd6Wk=CobhbgO_lPi-7N=V6b;@kJ=B@H
zCh4SN&r`c!j>W#~d_SCuiB%JxDaO~cG$wQ9ft08T(n`;~IGlUZy)Jfje0*BUl%N!_
zJ?Y5&U7h;d9-b9r$=cl%ut8x3x8!aXt0QItpZyM<2^8p!ny_^%tKMRX%OZ^-vjRH;
zCzWfTkUX!@V9w#o@Mq7(yGssrZSYK0KHSwL_Ni)H`-NoD&Lx4mVv@>PGuKQ$x#xLU
zTa(z4wHs4aTXp^?B@16vj0)Y<5xX?!)u9>FjndjgndeS3irN)+!Od@V@m>Aa9ctHY
zuO^G0_SOBn?3TsblYV7Erw>#(2P97mP|;Ub6)(QR6|KSV{N}a0e!`MF2YL0BFS#t1
zJaM&a;e;m}eqDA_D|sdw*29#&CLt_rhV`~JZ{E&2HglRRySPYsU3Q}-*Y$TvJE!Ui
zTv;fnnx$zPK0!$5N>a>^*vT5&hq&Ictqe>O&kvdTK;YA{P5VrH+}gWBjyW7#E-n|*
zmU!ya+L|(LLmi$&Ho`MwrwJY0E15NMTZ7B3)|{iOdwCr?D?+t`E_OLyO6qOrlwNzv
zZOxsRdTuw8xzFxgD3YDCuEg%YQp5pQ*X$oRJM8&7JWWLXbb?ND^{v}2s~B3Sd%R@P
z6OJC$Zq9}U>HF4rxb1wrHsY&+k(`o);r9DRn~e--?C!01UGbc&E6m%pSU1OuC;8f}
zFIzY3Zppdv=)8n?MX~nhqFF~b`RJ5c>PBU@mM<!jiNCU#r^oOxU-{8{4<joZk}XS<
zdKag%rU^B^^Hqr9PHGR^k<?{qb3O5e4#U~NZD$0R25sOjy6^se^*N=RTQ%yto<CMm
zGYqi`-p{pYJ-c{mQLKYz>V~rq#f8==D2bl-PCSt-DfMJo^rICEnQhlQN?YIl%=L<m
z=TS(R{{kPckhPkhv*%22Wjj8hY}x%ayITK;Z8@yKmhM;nP(!x*#$ukwlG!;{Q~W*d
zWu88~xxN3^^-Vh`9PDUV`7Pz@T%U$tvn<{iTIR6H>=Y<ceWH@zQ(|KklHI@O#&OQv
zY2RX_MZVvewlOw$_t%i<dCFbdWlft;#>II(JakNqlV!@G$-hsOoikQ`SajMg;mE7d
z_g*?8->saF-Ilv~K=kCJtXAnLN2xh^YAt_d%{HmNRcVhiaQ|WJx9R8W_>}%4Jr~~1
z3z`3CEGY{xeztFe2Y*dc=g}>9<mYMb>|dPNGWWNaME0xPmO9Jb$IDXo)@1E6*xst?
z^I%J_8DChp@TZn$5wkOer<30=3EdI%;`m*`*{h#tPGH&h|NEmAa*L0smX~esXwy~M
zsygZ0mh9sPVhvRnNH|RP`@GY-b3yAt)kZ_ENG;#}jT3uizE<YW*>ynj1FPBoyl>Ky
zbN3s0@65dWf6M;LWc#Vx*8cs}&v^OpL)m37GoLAWDbID?xGXTD`Tpasxt?29nwL9h
z_OW^?y!4o4v_aqabyMaoeXHVq`#)^uzB6Z6GcT77Upv#m#~1e1-ZsBE!}x~6!O6v2
z)GyjuzMhjg+wt3l1uWL9P8{Yrv!HkCB`)crg<_u!@|B%<rMDM4&yZeabNzt^x7MTu
zLP}M}Lf5mVn<r{<PByO;=Df^3lb3<-$ZPwT3e8I|3qJ@_J6h4Fo4`AfVaor&6SjU<
z+SRAj58f1x4(QfwvVHtv->b*YlbwwpdssGe9MNc!c{6*Cl9jm7ralX9dvO{2y_dPp
z9PVk)UMR9<x2mMg+uL1d7jYjvu~Uzu@AqQ+`O>bloHv*U>MqDOYx3+lf1}H^xp$|5
zvz-oKP{ppaPZOUDtX~=6@U+QclXdr6i`{8fhg2e6)&;EiXuRUagWcjH`%|BEIlAby
zRJl5CIhOTg{p{??rnhx>YHQ@w>^r-_+;`1RChqOO4s)md-ewfMu<VFqr(mb-ls$aO
zs)}>m)o&g=cj34+lSHTvU(t~xMwW(7r=8Xr*j`rNa>__iv#YnoxWh-d_pxN_|H*yN
z7xewNTgZ1)n>UO>jJ0Ok<BkK~zYX<1btkjf@4tB<+hns_$gv|6++JwAPYpR%Z(|na
zvW8)Ecl@WNcMW=!b{;ZZX`ir3%fn`;`D$ZjCj-AHHp+^t8XGr%`mFiAc)!c!zMwrv
zZk;fns9<|5QEhF4iR#aN873X~viN4_a6eLXm}szL&E&p<gEOOqS0A3-U79Jrd%_{9
zCF^r)TuXHhnk_t*A9HHINaubPpN(rAJ6;`23_kvFqs#u22b@G!y{$gBbkC^=1>5Fx
z?PO(Ete<ICc}B+Sr_Q++n^}|Q)QA|C%--J6b7r;AJ~bmP-6vc(bUK>9^sZqskGgWg
z(|K~x9FL@w!(|D4|5d!bk4iWn3!E#g<UDhUYi5Y?VI96_A%1(WaJ`UGs?uhBUE;Ta
zr>C*#P>YYpZO0AMvUhScb1$00D|6_yX380++iO%bS%WoLjc2%sSDA&GG@oSX`FX}t
z=i`oRGK+lXG$~DFGM&kwdBbbbZzJ2p?jpf??2A247I8C*aAzmYf3#XD`H3#$iOD51
z6%OoK|Er28UfSmK6^$7p_O}=K-B1YJ{KvE-=;Xd7G7@Lc-Ix#<*StGUd&TZ0XHM?f
zdCbP^>YM5OtGES(JL?&p^FBMCH1e(8vq$8Wgwe%4ET`?}i!?Nu@ZCzeXeK@PNrvRs
zg1zq~gx1Y)z1!0JWXIvP8LXNbs{aoxSmV%gRD|{T&Bn`*4Fa#-***V~gZ$Y`wu+lC
z3M;L?<yrPqbwSI9Q#L1O-f^Doz078^c>3h&JXLeudu6N~m!91*$!Nl<cQNPJa$R5u
z4c64SpcxQ6|8duyLf5O#LYq>IKS*Cz->9Sa%J61E_XHgd>p2Zd2M<WGo{<vSwr9z~
z<|t3TE!?59D;B&7J!%n_pL6c|%F8XM*4r;RaFoaEWro|$8>X4PmMPhe2PQbFIUStw
zMcOuYTlpLRGaD|HUf~XSb3|EX#)*i)lNToQTy__X*63QeK)pn`Qdm)YZP1M$OPNe<
z`2-|49}y3^8+=V@^%kGt!l^p;EXJHmF2CTp*6DlkzvW)x=^^{trrxk#87y+zYodnL
zS`$qzUjKw!rmJ`P9IyzpN{*PdrPtqk(n$fui9##SGx5ysJ@q%|rXZ8rC1u+icN+Xn
zIRZkNLRxy)Kk0Jay~R2^tc-C*m~F(`8!cfK9Y-#5>AyK&5Yr>$DIQwNqiLch#IS<P
zdtu*Hznec!+snwsK0Umz<c5VQ$H9=3#>%(3tVQy!x*7hy=;Jb3ZH{HrMi*t)87gNl
zUAFN(v$?mr)o^){R^3!Sov+7xwoVr)akJfdBDJXdX~PzqJ$uzH=NTV0VU9hjuGlNG
zh54<GaI^2S^1C;Sa_+9UDr_{FH`Zw__a)D{BAq`jTfeowIqS;y|EI2=IX6@2Xo2R@
zf-5?T%_~JVGrWy7{TdP|dy6xG>E+W?rH^Oc;8=Q?d0xaFt`%?N_H!;`59NzuogOw}
zyOG7@Qbp#_*6po(?y)tUnW-c-!Fg|C=*3yyCpW9^*|FMk=hk@P(C9svZ{_-2+dIcG
zfjK<>Zd9)5@tX}!tlo3wLqmd`v@*`jD0$2s^ZKC26Eo$OaMiVwDqYRhi^Oa0u4>t{
zzo{UOQ6luEtH<prGB@`MzrQQ|ac<vKMz7wSR(+M`pQ?v?b^DG;xNQ)T3*W&y@x`6z
zQ)IeBAMM<IF8s>HwH-;bo&?^nynAKW(p#}>`=2stNyN#&wTtuJEPCej?HN3Yo8AAP
z*d6JudnI|-q$jL%pNMc?e0|feR(PSWu<DGnFGHL9uXbIGyl~^i<U<{HAA);dZBBf+
z;?|cJ!a_>3<R3TLB*we<96!1C&fx>PMhCWP3LV_j92=B$TYY7i!;-*_D-2G|d30*-
zTw$9_w@r?g<s7=zk$Ut7cf-!J=cNwqG>%d8HMhDaas2GW`oAI7!VIbb4Yx!h+~y?8
z?%wjg^jVWf-@QA+W*wpZGnsarSbJ!L@EMP|$tBw67W=pSdN@<4XD7pRvm0p}6CT}v
z8d!R7d*HkT>$?|&HYFu^CF<^b9-`BC`>f~HH@8{#O_q*1_TcFZJq6>v#+SUz7FKl5
zxxlsd&26qav5^1sWv?!3j+;0mj#VLT!r6%r1MW@`?llz6RNtt>`E;XM#FN*p-4_{o
z7CB0Oczxi+V!K^)k_*$Wt1oh_&D^)=?}1|?4|XL#?F@Y`c{Tn|Z*+diK8CumC1GKQ
z*Z4om@Zq|YzM#rU_}uMXA*m82v8N`QojGk5-DmU7`qGNKsVxT|JYa}=e8S7LFpkrJ
zgVW&IrPsYF|59EC#C}tdG|cRNvq2)pm}7Bg_q>I?=UaGxPU^MR2)*Z!b2o73&eEKh
zA-7YPK6$B<HKjVIYi~}JPpau<%ab1;UoM<`&2sGzZoZFG?!=o${$yTens(|fxAFN|
zo>elwVP7^apJ#UH#T%yXOIMoC{<lth*dfN*J1_2y==9lfd*7~oebaD9`x-t?3*q9n
zw6`gjlZ@52Em&X_5V~aPwHto#-pzZ?I4$R9!_oP{JPW#GxULD;%@mI~c2MbQ!qiDG
zZ=85~Nc3Ud#O?8I!W%U%+}Gnso)=eh_>uj)jGGf4>s`JhpKYu7k<VZ2G*kN8N6s=6
z4@h3Ly?p0D&ozrVc8Oa)ZhI-TtNBR67PB4ADq@`hQO{kUMlPIY`TOjv6;~aMA2;cB
z*uOuMAG9`3Dt@xV+oX&wj?=EZQY}mstAE?o7_xD~@gHjzUbMCJ*=pmomG{}xQ(bab
zRVEePn8EPyp6b=oqqENmOJ3vR@=7sG4Q$?gE6SDUzvhFc#>TuO9DVa%3T2$-d3#K@
zY~~Y=-EIuQj?%eyOrP(^iXO=pO+NMRzQ(PWUqU`wUp>cIce0Hu@JPV*g;r}Xua4U_
z@3AC5--8!n6IngfO+&6Mlc?KhaB+`8>$3-?Dc67cZQfMYdp4}+lJwP^Q<%1^rKvV#
zZ-3|X+I8}(B>sn4Gp9V{5)$XWxFT(mAn(Nq+#LMhqGJz7|K2n$&&9l;b(>Y8<=QtU
z)zKz9%O)T63o@H|r|s^oPwrZ_E=v827W{i|;C*T%-;Bc3=g-bKzuCm=P=r$-Q{Sya
zkJV?Ea$d-o?AX6GnCDyU^s5)Rq`tjN%VXAj!0_->;(}e9GrdCi)XV=j6n$t~USTJ)
zZSJ?guE15F#cm2e;|e=p{ME!jRI5lK|7l=+xsuD*6`~JM884b}_fR5RC}Y9vrh@Dp
zVGPM)4dVOO>~{`UKWV5k?ODg<t7h4K&o>-R;J;vYVftr}SjLDquXgV{*(W3OH8xv3
z<K4da#|d^C7ll1|4D|P}saBi3a$TI!1y3{1Q`+~gh{qWCEDfoQEc$asRymrxi-o7n
z&isyB^(En}-T!i$pK#UAZ(zBR;VLZN+&V|^dcY&Om#Jn;%N#b@1zvu%(qaF{sOiD+
z8zaP9G<!N;&GfH!ah}d+pJTVE<f4Mz+k+-gb3!-&-Q_cpyXOyYn7~t0o?~D7^jG~~
z;kBqRU+eCD)iuvvn!FA@WBgUzr?2H-!VTSQQ7xfvyPCB(^4G4Yob~;E^OnMsOiyl9
zNG2RNc|767T8_)_3U&o?%)B@8hD&a3UA&382G5s*|F(&bChdNcxjpB-oa)B^DJ-Vd
zh3p)ROzd0_lp1^-o4CY{a$bB;bZKOhmohn_spxFZY|5u|wNPPYGm{M$Pv?guNya6o
zHpHA<$|J59ti%_hP|!4KHjk>xgas+wj^?>6JXcRlT;UYX6vgXva>9`VLcXG#7G{3U
z@@!LFyKPfwexkId^oc1B8XLHsw{UA)nXI0$<WMJ1@T{p?7czsFt+jY`^woF$2M3zi
zx!qQqrTYBu32!uGc{Sr<@_rc({wdj(509KlSAX<&QRD&FriG$;e>JKS)K9IFpQd&B
zm#NQ8;UydsjH;P@IhjhdLl!x7=G~DF6AnM|N^#|5-kEwiqMJ&bUof4`6UsRKaGGZA
z|5<@Kq3alSGrzsCg<0->;Qj~Q`Z^o`t&f`Zp7n@&g#4qLrq7Q*i)&o9^?9RvWQLP5
zXUYNw4lRodJ{&?R3w(r5hD)|Gahct(V%xZ6VK&nSlc?b6t&Ty`mvY&aJd~#%^pw<d
z?FjYrdLVYk!RD`-a3gQ-@+r=@Gfg8x`FDyg@enIJ8Xz5<v*Btz_f*EInaUH>r^lpr
zn2Xn`PEfg*;@PXh(Endqw)AOigi*+$TFsTN%N%51Tu%%6K6mMaey)?5>CSn2OP)uS
z#D%6yR+TA+O3qvB`k;VYb(cV&jD<$P^q#8a@{JCO5-&sF$CinzcYfU|5W})m?ZOJK
z-Y~5$vACy#IcrO;So1c2{1)!Vq%om|X&dK+#j8&19u+9|;9N3?!NtsF%|`Cf_BjlT
zHMXyDIKyf!5U_T~Beyzk$)l~(>sn?8c=_>$Uf32Uxpji}^@}N9&W|>}yuMp-ul4)c
zLbs&F6W<^0Rt#e}bu@eZD#oh`O;@5)C2Bbp*GTQ3AJ;K6>0{)PfECP3y)Nv~tX#6O
zOS@KMHp31<l}!!@G#AWfQCBw*Sofb(W5R(P2JI>L93F4mIw4w%bIO`f8}>gJCT`WR
zcGN6m`JECRGvny1^#!%o0!{^?GI29g3sNpzdG}E?aG}8eKc+Vl&r8OMmzr&9Jj2fQ
zEn+EK{n|{&&Q|NG%f)W%UT_t!EctkpcdC|I!rEPIpVl-yd^NjAarVnZ7qit~I;lQS
z?_AJe?hO)s678eia6xF>Lh(vp$+<y0uS+cviDkJdv}XGur!Q^J8;sB0k!AUrVRkjs
z@yh&!j92Rf_WpSL@As1LZbz&g|L>im`yh+|#Dd$b=4J1n@EYF#b2vw`|6*53*?+~y
zEWWi@I+Ulrxb<@bllZ4!T&jty8ML`o_OmeVsd>Eaf6wE06CQgeuL}4t_Goh(Pk{of
zijTIHxx%7EA5FPHFOM!YfgLlBYWg&K$6Gy7<c(a)9J@gBW60s<E2n5GU6-Dn#b7!?
zujhhRMdiasueKQc-%y|s+r^oDfZ4!>H-(w)SG;51i5a>sTK4uOH#qkPE4Jlu<;SlI
z|L9O`9&9*Karq&+qcMw*Udw4rX*wvnZpjA?9k&)uGv{fHl12x7pYTL+uaw!aXz8wI
zjfK2kjvbM=0@^ML9*t((DCRWdkdVRI92=&}TQlumsQ;g{GW<8QoXexDC$#d`PdXVr
zXSdoTIXAw1#iA@BpDUBhjN(`wLo{w2ShW3+=R(!aH!q|2O<S?|e@hRWp17iO_Z*i)
z%XmI1FO?Ad6}8xIn&FdL?uql6-iSBc>?u_6=T4|@X)!cyTU%7n({Vm0k-v4zVN0PK
zUJWuH9{mQ(r!SeEa`xOLNBh+7_?&~vs*0aHf-YTlYL*faneL(4$k3B;u3OzP<=D1A
zt-5NRGd(hFj_%px`7vqRltn-GmAD4^ZVahk{9J7730D1SISN~wPCQ)3X_mpqR&;$y
z+E<pMpr!=Br57)CJBV~lFpBvwp~v_X^V<hve6v_vrmXb;P}DAZ#NvtHiOusGR5&G`
z+&wwfHCAz9T+#Y%t!roUMhe(-s|a%Fo}0YdMX~zbl2yM?u)0bMoy^G66qs-K|6}L#
z->KO<Hnh$;@m~Mrk^Avx->0wT=`?V$df>FAcxqJ6=GiQYN6jsTmN`ycXs!C^rO?q;
z=Ra&+JzdzRL)+>2y6Ug&ZT(+$<{kL8{9@fTy#-eTgMDsajlI=pm(lfV$+iW~ojit0
z+_#P_y0^LYE*DGg?oW!2kC|Nb7%sMW{8=Tee#17p>3Nm7r*GuqmJpFcljb!&X7OWr
zFnj9HV>6gO=>6IE`!353e=e1^=^NKg(mA1dS~0JNqk65gi0TBzr6x1e543HHG~<2R
zm@B$Apj-Hgw_9+<l&AA*<z^c&2un=T>pAu5llaUBi>`0Gpe)g+aYucdL`vJnkVgqp
zZ80a~jwt+J+UT9ha$B))X^TizjkZCuQ}_Xm%WBVsI?ncOnp>$gb=GP|ov;cwu0vU_
zOwk^lS$kI{*evZkC>N*1V_Gt0?ZMzCo=T~@XlXOiq%8*Pj;43WZobW#zWj{PwGE+J
zv#$mQJ+u9g<MCB(+MR9FBo>u<_eT9qd(P$jWS8`Z-nB7oajh(g3XUEQLP7yAemAAA
zKX~bN=p%1o&*;7emX&wJZZHI&<9uMXc0xnUn@=ZK?(m+sW+}(MCyOS&ZCD<iw_={O
zg6ve5#TQmg*)vJ)b!)cYx?|V2o(oZ*Dz0GU-hY<wpjP#Sv)bz(>E!yc-rlUd+q?7a
zr@396p56R8uMV?q7B$-TztiD>Y+U;t(S|rC(JN9ss)ue|Kc1-Us=Y5MVuH_u6EQC~
zw!1|LuIPB4#pT%Hc*jZ1(rPJ-ALj+7%)c%(rW_P4*J#bMzP$MAw_jS5uBsjruDg}{
z{@jC*SxWm@7}oj*Oby)4y3VOWx~P?5ddG@wA}3NiUmZ|cwu@nI(!7-(7h7YdoNrP-
zIzQj_OuL8`L&v%gg*T*iGUkQ%u_#Wftt<^PYFT16ds@Es5{0Wx#l8%A8l}N`g*SU|
zync8~%;CkeNWG>7@7^#OHf!c3s<Kb+y`HeVT7<Fr{mZs(2etE+(xo;n`=FD5BVBIA
z#-D4<c;x3PScmdB>8<-Q`~03;FYiU{`oDbf(^pd&r>kCB+!c4ZE&uR&wUaU)3~Wpd
zKPBf*FxxycXqo7NIdAuBy*-vDCC`*{&`Cw*cF2iWug*@~70?^7?evVUxS-%x315XL
zjvmJ*tQT-(b*<T2JpJmlIi{Wdbti8;E;=O4dAqYWMk#MjM~>{I6>NglXLl!k=9Bg>
zZkL(jl=aX>JKSPo_i|Cwf9|3#U*<S-C{+EG5ibdMIpKei_1Ifem&+3`dZo5{amqO6
z%wkHtD>mV>mF$hhOIvPEJosWt<BhG1l1FYu#h&(wb#M3SttvTwO<_{|of)d0a$zQ-
z>wm`1?wxVtL2$^##!Cill0BCkmd?@g^`GU)=ByyybmITCMwW*q3=i-7-uoKjDSGzO
z?1G3NABDVX`r|ivWbfcuFjvlTiJbEl(LFY-k3B90Fi3`Okghgxx8^w^a$}<O8kW~O
ztdbLsSXu^0?`%Dm(Q<adakrLJY-&8wGabtpv|6?bEKZI2e)0YhTNYlismF6CB!##y
z-*f$QNPCWMx1-B3kr^`&I-C~MIkjj`<i8(bnrT-TyRfkzh<O?~@Agga<tHRFFC1}s
zD|znc{d1fR42M`5+Ik;MVG{Q^xOT=vC!MIzbHX=h#4edB?W;2HaY$d>%5IUA2SINZ
z`c7RSFWbUZ6}!ANYA%P+j2n!eF~^yU?yR(J34U^`d9U(^o~YKGARDFs)7LE&cKhMi
z@RED0p@fQ|jQRsXY1tMvvD?aW(ps$Hd`qM5CH8kb4L&0oK6h%T`<-@A0~V*46qdb5
zW7ixJw>r)^Yo5b|g$zsNJU=aT({P>fg<omy?SEU^oK?74U&XzPbY(H)_U&=f{xi>M
z3sb<2iRWCpWhb_?Dc<qW==Hzaw{3$gXXAxCGUp$8$knCZY&!eMDr?R^2DZ?$xf!L`
zvgclPWJ!s5wMbp>bY3CjNyD}iPdsL}Dz%)rwrQoLMpC=h-bYzicvB@s(o)<?46oc<
zbISdJr1KQ6%Pfx_wZwBKI2m%uzT!P{YK}^RNkW2YdwjsG_yAewn9EnLE@23W(q;J1
zs<}cz^23Rj9Y;Pryiu~EyZy_R&<9t73#Wa**tw4(PW`KFCa2rnn3E4Im9pQq^WE~E
z+~WV7^@J<Wk`oUFLoE|iEjV<%y5gtIiPu?p(1By~&G;h=ra$@l>|g*#yUjHB4M)Y;
zWL!%7Q&zG3s!)++kaBi-HkG$!jp&hGmU5CJ6GP>eF~mkmA9=VZ(Btq*k2Kv&+dsG-
z-r30IxpcMTlEXFgYx&&bFC;fM&YWiGm%X8d|B>pw8LbgrGP+?dQBp^KJ!FeAYMZnp
zby2~hMN95zO^R&W8L!3Ju-Ku?MI>D3RE(;jh^w!n%a{9mBd#QPg}Yd_oMk;`aH_>u
zK{=ISPJGV)qt)+xWK8*GIOmv_s+IXTaB`(Tb9H&aG9#?+RG8HD#>`39|Dqax-panU
z?A@7wf3CN`$++#9(Wn%q>63Fw#Nrg&nG-B?cyc{N&+({8PEgW1#ZnQoILSf1YOc`i
z#Dy_FNuMo5#kB6sUeF@4^oUEH%eIHD^8cn6Z1UQ^wau_i+;dZUu|;28r+ZCS^xP@S
zmkPRj+<7v!D`t<WyY_>LQ&Tt|E4-YuLg1ax)0J;$tUPuh#OBi6l9$2%w8B2AS?6iC
zUOKA&YXW~~kI0V}{|xc`HT+xD4w|oQIl#K4_tkN=1xsW0-fukPvS!Jg=(OYO7R>)L
z=VsG~X*-3kv>49oR-0E9^1m%E>P6-mN!Q922D@<Qo4w9|=j-deh)xkP^lRL~*<dtH
zWxlTFw>cuxIjwI!A2?lMlDs1QccW+RlV#KD!sgUmRlb-!{nq24y9p|Ztz}`Vk*-Ns
z3m7>K5}U5PEa+M7*CXxnp@pR-@OzJ2sl@U^v&1mn<;fir7yRSaGJA4<(anT8moAF2
zl;%yf=gP5B)oP7#`RA$iIN@PufS8`+s~eF^-TOL<mn?tTxaxI_mc>0+R@Wn)wk}rJ
zf;VkwtTSr$@nj59kbJvv+V{lxM#tPjjaAo{h2{A99Tm`B!*OKuvm=|iTDD#~b#NNb
zj?(K}?cBFba=aIicd9jm-;^WIhcUP0zmmPEUgHAQ=7Nwiol`S3Rg%}lFIqCm@sGfP
zI~RB(1;1*z9lr8t-raa7fd{9aCS5-#*}sWRZ0o6xmXt-DN0@?Sj;&a^K4$J^$0N^F
zWYhmiyZG=0PdH@SnpV>CXwpqq8_#EVeOqh4wkDSJP6|zJQsfC(dz>fq$esyP?eu2k
zP0M_B;KExD2F@c#=A2{6FAxcvvncDvo8J?hGz*wCdycsTv`4zM#r!?(6SMfjyw!hn
zCpswV7V2<>mrC5%o7i`0-A|_3Ek0S-?*#=UrF74jd~AbD=N8B1Ee}0Uoqs*?ZjYCK
z^@fMFMwtl`-lZ>=Gk6}9U=YxpadhH}v`KfBCNTs&dHes;G2NTTcwgT!DGU>97v|5-
z4qOo_5dNyIC-*{|n7g*K%o-7whVs4V7s$Fib!lYCXcC*Tk>QJhWX(tC8j(kL)-dHX
z6|+2jlJ(l5WOe7NJL`NJ<?pxU9#H+VAwy{0{aKm%yX&&G`ZAjKywH@?Vx9fQ>53AY
zF~4Halmx$`{#R$+KWHa@d*PTiC8<oSa$-2cgtyX7y{W%i(@sr18!$IZYl2rs<1>y1
zeS@Q=e>PN?NhxjB)jJ*NYLs;Jq3nH2ExsxaPB{sm9}3NTUi@qGd2FWVsId4?PGgb9
zaWMr)5toBv9*)n|7;}3XH?P=uVMCM13s>`t$DC(8JbUkr*n#NGCI7ql*&h1s%yF6#
zY3|=B^rE;-KzBVy;*BRKx-YC!{kZl@m!@u}G+VLGb+7lXYmRKZcINt_Vvl3}7bh;9
z#BrdYfT2+KgU+hRf|IVd|3%m)Cdkwsop7@`E=O2<f|K<PCu<fDg;;mp4NP~q9IPiC
zoEUPH^<2`4G{u^j%vg_&k2{ZO?^D`*$V;qvql1W_^NP*!d-}z094gFk(%o=a(*3l?
zmknYCeowbFi)}evXy9hc(R+cR^-n`Mi^RJv_i|V+v^|-2<Vk>@c!}5Yl1BnFZ?gQ@
z805VAxm?GI33u!>O%nKz#4)oLEKPL~{FbkqyGw53MW;aPhrCXjcO3O2)Nia7N&Ua+
z+)YXDe|qlUlpgS8`mr3;yx!)ab<Ux~wRE1wU)xzz_C2o_^8W83_1Vp3id({jmNXf*
zYhoFWx>HQGw;b}8KR)5=;nEw;7jk3+uQciYG5EHPNp^d)?TaE^gPic22~F!J?c-^Y
zblLd!&jKf#%}z_oB#-eMdK#ZiY8Bh!q`jwK?FaMo3(j)d?A95LYI6#8Cp3w;9F$-%
zcdYoF_1>j7Lfc}yxu(d5{eJAyn!XNsjFXtOezz_Xd%amig)vX#d2)oJkWdz@c#r)>
zHMZYtThFvJMnCWLK2dex(vc&_pX6+qSrIo;;d9Hh4N14WUORrx6FB1jc1N&wOW$XG
z-vwC-KmYSgd@N_aQf)%?W}V$vyt$7|mR9|??V!e#gJy>8R(h?~+U=HMsy_@93bu*q
zT@CNqC3DJhb+Gqm31gT3Zt;-IY>$}S)~+!vbY~RGaJ21Vw7z1Zy}>E`eTudV<Gk)-
z-^X7z$n28hc333rBH`{J(qNc-#jqrRJtIIbHhs~HO{=4S73Q8e_3MhM;HK0s9cNnR
ztM2|LkZfGkt@Y%v$a%*|jTP0FT?d4E>N~HrlpW~uTGwpkw)K=tME2~}ZN5i1Tuz#$
z<{Z`X)^yooBrAMWBW6jKmZf$GcTx6FU19EEp6|MkkM204^{wlUw(-$$51odovzk{;
zj<OJ&Ho<dl)QMA9{)-xI`?=dX&6h{Es(rTm*Le&Xjbc1A#d@4<W!$whoJ1nFE^=qo
zUa(QE=TMmNj=uLlCdDsS`?9jyy1nVVWWWc_mi#|V<|ovHD$LJM6{*?fU>;oBR`W$*
zZL%}VCd=mz@#^RH^d2?z5skX1Tlg^b?w@qaKKEU09!&0!OpeUDZ+-D_b*~J|Qq^i+
z<>Q5G#1mO2>@a6P*=c?C?UHjbI~T?uKj7JU{bA4>N7t$i4`zp66TTj9eQUM2ZOz$h
zyW=k$Td=ZrpL@&g?Ry=193#0~BK2At^p8YwyWGFtCh5&s_jtR-ddKul%(`!W-u>-t
z|HH}t22+@Kp2Mq6FS1r|7gK+h_n+}(>m!wdJag}c3(vaRn2xG6Y}wt{<5ZTB`_Szr
zpPAFH6}(3?9OcDxqRxD^a8;M+dRz2u|FhLeFV?+hQ_$aZe1g_jtNGvmhy(|#AGBs?
z%dyvY`NCi9Bim^4M#IPN$Tg>Yd;fy<e1g(dH}APtO?kf5%kc0<oyL@s%Jt?)u1~#Y
zYh%<k`_J{u7u;J|3O+a3A6dlxODdbmp+`_5K}0)jD<5Bo#)5lJjDO|qzJ#Bjzc*@0
zRH4$*Oa)CnKGq4GJ6m~9`o?T4n5mSov_V&I>7F@Csn1wc^<0}mc5GV8plP@LNXg3L
z2i}X^9lt%{{FEHbDDPV4Xf-wKYNMaHm+CLU|0{#ng~`@E(U_Dh$+(nd2CL?zD+kOn
z&Q8-Z-X0<yl(Nj{LZxt~&}t#Gtx8vyY0X|~DJx~hQ)GBGFhP9VX4M5^Cwzr(?V7b`
z!x4uq8SGsmKSN8>+tUQw_q-}<Zs4u_>^0Hg!*kz#-0A#N1X2}e2JZaJb>UJU&-=#h
z6P?3;Oy1UT`z$+K_z#&(mZ!nJ*%wXHuFMKubK_!A#LjU0_<t-5!e3-(t#1_g{nPk9
z|H6iCik1_1&aO|W^Naac{c4GC%&jD0=It|-9Oq?B^6i|uLt(;!=^P;^R<tgw=v}&S
z+m3>z*P^OdZsna?)Y;74`)Gmo+<i$CyZe`1^0M42_NZgAAItv-845DC)~AoR7CgD=
zeu0H!(+TDE-j`MxJ;@4Mx>%$@!>g6a!(pMO@P;5J?ZqCxo+spjH#s$laBZI2sP(02
z;VBnx->p}#1#uYaZw#8)e6#4X_xd{yvx*k*F8Q6wZCCo)(;_UV)2VxcS>zS5pc&3u
zmP^+3yz^${2<<rUliA&LdB&m%PF#&_S*=qWwPtfI@^<X%U-U!g&<Q)~;t&saqffV4
z!i`FNJD2yzacODHPn-4V;G%%4Pb(X%nm9EUhA~gu#=T#sP(f?T+eJ6mXQW(>wm5vr
z^H^(CO5x4*OMacw?7N_~I&5=ENVZ<<y-yQG)J{KG!qdNI)fA1FGLs(J9jf~O``QEj
z^H*L5a=yFlsb_Rbp}A$k4vUX2yy0D6mU*yt1U1h7^EG@|uj+)R-m+sHeLUSg${yVs
zi#R4L>dGZA7cD!Mv3%+@@mW_aH&k>hF}-9srtA5*N7z~9-45n{v8n)3*JsVQz5SQh
z{nGTei+{DURrdFr%lvQpvhL=G$NrZVIOr*KCuverYIk%4<MVm9S1^nJ($qKUy`^+y
z@5<7(8r=P_I=dP}YP3EU^#^^|+_)p_`x?EHH;)$k#jTmDAuIc6&vB{RBNIJj^w)cN
zC^asLTB!EX)z;I#lsm-WDyv6?;%&}-wuTdysTIGvb9#%dpK0)&;su;G>t3n%v2GA5
zW9<4Lcfu>|UuG(|%9f0y^UJoZ*f6!sZ|ifzus;_UZ3?cHTsk#OtTk@oi_TkD9xQ5=
zN}ha-#llpvT`DW%YGcGg&aWDrc3eN@1Nkx>nEhHBHwv9`D+=MN73|<ny}`or_wee?
zA6La3YVr(7`y@1FN{hDl$=U1&O?MoqJzXGeH9c#Ik<_iT1(DiJ(;rKS>zy-QlgRv1
z?1q5w^es2klYak7EU}onM0bvs=MQG@*%HpX!>%mK=G*DPGA&)hNA_sE)x(<w9#1Dv
zyXmIRa9BzDjUq$Y5g)}_AEeIA(2!B#Na1<2;%KtR#@1~ahiY|``Q~k#q4_%Es8#cm
zUa>940`H#i+pLiK?_pbGD8G6yyH0%e36;>L6BwIVY?<GrxM)N={5a!o_i5$%+9TJT
zV_MJJ#~93B$0u@n+s+M+#)4NvRYJD&olH@aUUWf^b-6enqqej5%|o3rmljPcNSJy3
zCzrd?cMHZN8M7Waaj9=PWO=@HUD)kAfiFzbwc9y7JxsQ6y^i_5?+(j@N-hqDsccNU
zRXh@XtOVsE-f=n<UpqUmcH;e<Z4CQcS-cqLNp*;^OtNd5bFxZEK}!7e6uTCMCn3oe
z9Qkg6+~$3U7BO8oq9L`(y<1Pq+-A1dmn91{9`dAED|9NWADrZ{>Vi7=%3IH!QmRbZ
z4!u%pYV_XYa7pZZW`Oc8qqGD6TdpqP&b$z=qQ!N5)+Hf<s?$+=cTR@(Owwsx@OoJ+
z?{(g@t)g?cu*TWlxg&74YJK%zAN8q1FG7|m2pxV>tmRiC9C)r}#!k62W!*R90~jaU
zADqNw(=GMHvC{dthf72Mn}nkpH!1|`=V*nl(KzSy=gEba8HK5LHoDr{R4*udvQ{BJ
zJ7NZ(_mnG5EE5+6Jkh%GAxO(e#r6$jXRhof&s7(lx#NR7s@F`q-1<Pw=>O$gH93j5
zHhw)ed)qCAS#Q)9CU`2fEXulYr$AJZg*BVWB=Ca%%&T`38js{R8@~@&HEWSb-qgLO
zCHmpnCq#3KTy={cc7>hD+}YTp-9AsV`Gw51|LgzW)0ng1b-&Xo%fo7sUS52hGo5x-
zs80W~Atbmm)BfYC2^RM*uM_tQo>BAI#f;^c;_Q?0(>(&YtLJ2PCQCQ}<T>Qt{KTKz
zC#T&_@YAO548@~N6(XPSw2PR3`e@iC)hagW!=&RL3o~4dRZ5aJt(YdH(r32!R$vSF
zH1ALAcg%`#ohG=#VO8g*DIy!&?XS6WnXZ1Sy5*C9qbH*`uiuGP>OCh*A9H!HRr{(H
z9N@X(-I`rOx~`gWE7SOnr*3%mY{Sk8n&FF1KNT@u_|?Q};l-t$JI>#jx6-gnvxmWD
z&1B276AV^bUY#cH)Wz{3O?aC`Yv~1#$g-S;a%(ycNvw*P+4n!ljlptttH_ps{;-TD
z(IpAa&JAw__Fp+xTch(LkV~_{$xV@)D^g^oo=!tt%p}|Aj87dpcFXzH1#LeV>bKL-
zk}dEA`|>btg<YMSPI@o;T|d8uzsBojf9Dy$6`!_FZ3w=rw(d!h&*T-iz3z!k*8C#m
zE2uPQLD|nq)~}a3H(#FcKxnx@n$qPNXI$<q-6QF{ao6|R3X0Ds-jO+Xo`=)(j*ewh
zcB$3+Al13dD<_@hW=&l5BeRunp>}MhdFImz6$)y4p#=qJyHt*>sj*qtDi*U=qyC3j
z=&#SmSxcWK2VLM%O!rEXbYx0#Rs6D;S&~)P?<il$c4^J*`Ifwvugc~Ir2JQWEzn=F
zq0FxJ&fg$`%1~zcrO$SGW=#G6G3M~IGpiR$?1?<>nHAtNRcF$VRry+s(}KQN?l`#B
zCDKX#=WW?ZFTB5gZ+%b{8+mHN;l|2kyeqyde(3vjZ83}3m0vbpv5~FSyTA1vT3P1N
zb-ChvSjM)itKLnrjVm&W+MGC#{i6KYecJ_?HTPKtWh`$gJ~v0Vz{F?Ef?wMY>b|?f
z6khsp8`saw4!MsJlA2}9&tD3ASo<y@@b1%(thPrs@P(!7+Won{UAN?!-iH-=7VQoe
zOjGZ%>=H3ZU{QFOwkXAFNy@SWhoVR7d`C2PWi##LR&hI`=(c1}8<X>@Lxob$wItki
z?D{?ayQN2ZJolLAsqXT8|BcncTb%S5-puA$vr}sA>J<LXv)ENOrF7RE`n>1qp`7O-
z7QJy(K4?riQ_gm%(`ym$r<F!gikxebxv!~nsvM{hQ#D|5<Jj83W74=J<W+4+pO(@Y
z#jKuf8ZRfxFr7K_Mq``LyF+VupTFZ>t+f1wz#kTdtWCa(E{URBdS$~BMSeBPxh0BR
zc^q<Wp}ZV(mfWG!ztorQJNy3B8l`^+<#HND<v6DPSp31HJ$!|$(5oEF39rsAQDBtu
zV*0U2MXTRAY0-0@XDh0dRzGRkE2FNJv*N&{C#>s}FW%CYIL)fp&>`OWs6(T-t)eHZ
z;610T)4HtllFJvB|Cd>G#Ubgm#44i?D(g}vuHCXkYgtRrl!gE~tv5AIH3poM|FJ9C
zoV@$N;GT)5(gR+dlI(^D1&Yr$hKe=a=y;TU!*P4nL&2Z!0!Ntza#zV6*HvU+G{r;H
z?#tS2nbo=$S;BAH=Ra~$uWQ=n5hygn*^JGLaT0@yt5J~E5+|XSNAr^Qh%mVr^hLc%
z;j($RV#PAexFu@KHJ^s@={C9Owy@0R>(LT=bD-&yd(^w94(t7wJejNK%9Zw9qw^8}
z;ts7RS;7_*HR^Ov)J-_J;gi*=jam$D38&u5&7G8$<*2ikY0I^@QkjmzT$dF6Tm?86
zDTF)nuxwOZ@$uBTAf<or&d6O7`+sT8cc({sdK}Xxt<;*eG-z9jw}^x2)(QGr@0!nb
zuUhBDGW(ULNxt~;PgXAP_fAudy|a2vUB{ORAJtiN^yaa@S+=NqN(0ZiCwf`#inBf@
z{1bR_U=i;>gWeJ+W&>4~e67wC7rRd!|8wtS^o9Jk6Z|_aJf3tUL8PmA*^zujH%Fej
z*V{kji(crL`}HvU%lR&gF1eE4wlG(_Eyv`QPEJv{RPN@hzU1`H4c-hT#`8*;RAjwW
zju&cpWiaguERa!h?s~)=+rg|CyjLWBFUOY-1$Ne>i`ZCw^iC|*=u9%+sd=zU=-IhU
z9e-htvjH6MgDhWgeN#AGR(0aA-Y2p76U>4GJ^x?2Fh^t~-})T>xsx@XP2iolNGCN!
z_Vq&fN9VUHOyqxCAp3a3wJi%Q<rk%0@!w#wN>_Z5c-TUL_F~1PDXus9rsxIUKer~^
zBJix6<JpR(x+48bnv+zxm;%qeQ4QMCEfmZxDa^#o`=ze+b3ve@!W7Lju70Ar64E8w
zO%vS&nxiWwa)>s0=btj*Q081}D3!ndTM54z|JH#2SG0Hzdq+K-bmP71lj9lxJUPlg
zYs_PwFvWf1A9i`s47pknIn_rKU+Bu-F9_wEsF3kxV%S4@)&@`0p6g#OYMH&vF1ymj
zC8#Hvr1WUTogKWV+b-)kYG&VSRy;k~W;dHscdy8f^X79zZT>4=6+4%sxQIbz`=|6D
zPAbluqV1QZ6fw`>aCdcLdF-;GRYvLYfu=8yxcxSqbo<cl<k+NMX*65IeNIfe2Im=%
zHY4NSW&Q%fi@tPdEhq}h&7Xg4$#((%D{{>XOOm*?R9zj{pUiL;ZZY<B>vVs-Uhb8n
zNCe~68@h7YjMk1xqANPYV!Ff6=7}yimb^Vj&%ZU|wzBZ1xAl8ITg8Nw3T4PjzEJ$y
z+o;i$tf-lxoiyo-?sEla(WzyIvabx!a>lAp<~BKR%y}h#KWn>&u8IBm1FM?W{8BRQ
zPZEC96c~Rw>uT}Nl0(kf?B>^<xVIK8h*+wvn4HYjb?wtd#lB+2j}ztozdEtw)k1~W
zi6Yky+J;?yry!^&Jkcm?tpba;jYhY1<hH&7?$k&{JHECwCo#<v`pXNt{}gC1PYL_;
z$I;_{!2NTFOX|OsoGdP7Rp>R@D7M1%x!#uK4DmyDZFZsun7R)+mo(Y0<#Jua70X!S
zu)R!6;(1)Q$}BVC2T6YF7VVC2OV}oEfBNfc&iZEmKUcI`{M8?EaHdZ^tgv|Aj_I7j
z3)t6&bLB9Ww3=?O>%OtT?Jb+r|GW~Bu!V~I)c(nB6lW65V)>XI^EyN*P~dmC+kZ0w
zSI5xb27>R+_$E4KZS$!We&PAFK!<xpeZ#cuti-N4>+OPk4I*?hWpnI8FKTl7GW<8u
zzm(A)yE=*e<(7)?+H0n@w)P*&-Z?Exf-RT(SfC@nfv9MZd$vNIg~^|!j*XM_(;g?^
z;OgvrWcf!$W<Q^bh-$}U0ox61O54_kGG@p|G|H(t-MtsYkiqC)=_|0ZLm~Q`F0WvZ
ztE0`a<BA&7--#@e3p;rFP_aOzNr||VfO)w330*nIsgo|~%JaX-_$wiI@^Y#D=@*|o
zxt3nwbvX1QM42&dy48bg(Nh`h1C<K&mM}*K#h(BAI6}2JXH)E1rps{_bH0VLt?n#(
zH#My1lSe^InBTdoohSTHZ7JKxA3Q@Su_Rq5bmhE?6$wjwr^%jCT-5Ma=1?D-(!V6r
z#Tnh3{<9R-|K1=_SR(Q-P$1eY?CJ~K7vfn0g`#S!1-LIwia5B{;ZfR79sRXTv-i8a
zD>CzXFKD}CSx&wGVT&EpMGuL7s#M5xoX@&oKKmoSeJR0Hriq&FT&?$6(;`25(exSa
z0)Ez?7pk6pDwdQ}|0z3i>CqQIL;jq)EOq#^#V_wo+t=x6A1-O<X?onb{E^AVty&hX
zla@BfDxKuZcFcOwC}z<mTgCXFZMy6chA_?TGqn#37V@SZzOmx7`>caEJhu7@Y;2Q@
zcxYWzx~)$sf9<2B(nVrM3&q^TLlv3twRqcFo!k~zZrspkAa!%TsGFt2O~sazT5?({
zQyuK~bE_(S*0}eJ`M>*;k69B7wAvgzKlttMif&kyl^n7E0+(*vMAqfy(>jYZR+qQs
zt<w@P=G0O(sQBT0`C3AMliA4@t#^wrd!6aBa$36HImy^0dZH4S)^<1H2*GS$;k+&K
zOds=aUaEU}UH%xmlF)QnuB2D{l9eWuD7<*sIFX6ZeQMhaA^yp?JwHxze{Pwm_Ue$?
zjV0ZFiatIIn4=U~s*JWoGq70gS-I$=!InL<pZkcW=y=cg+~d^t=t|{TvkmbPZ3nq#
z<rTJvxz9PHxh{(Fo0G1V=vS+xVlAWJ8XRgJlB;+-e&jx0X?K&$PTnTioh?&&kCsiU
zE2Gi7D=(H`d~CSlh2gxmp2@pT{nt3Wy=>QAkLiuvmR^_r8^4tE9C(v-^{}|WPC2#1
zn^_l3oVaN1+HhZoW7>t`oMsF&wD>AcdaKV^X__QdIP2xY#Y@#(BTbt)G}W$L^6{Hv
zk?z#XqLQVzDbsyYcvW|v<N~gk8B-6R5obEiY18g#Z5~klj74kdi3Yy#hYh*2#iWuv
z_ew5Le>3sJjf8tY6ZkUv4hzXkD}`!3p2v71q~&;-%r(V4#@)3(3~vrB{8?#oaM6O$
zK(04dEVi*1)GoKguK2v~RI6_7-X%%1kGxXeQl>aT@Ur3@t&V3V8!rTy`lxV~9bo=*
zS<A;#V~yw>jbAl-$EUybz9(azvN-yC6Jvdn$bY-<+aCn&m|J}-w|l}?UH;WdM@_|b
ztiJCwU#TY?dE>>-Ma3};GB+ox{@vBtbX-$b^GcPX&zU1JjCUW_u9c5ezq0G&lWVm~
zXC|dDzTCUVe!0?mbNwr<Aq-;2wUpjEoOe2qaY4g4efAztw?|vuSIwz<I(6UftWC?A
zEZlxsWfWKMF2BJovn99vuIIrm@y|_vs6?=-dKSF(-t?xvXZKD~jnhy3YS$P3cW4re
zh%TS-^X|$=yF2eGcGd0H-g}AN>E*#_MWxsUGw<=;QFtl3=+ge%TKi_d-1*=JJJVKE
zS^K5>#;Hxti&|YiPMPsE+A?5*&b-2%S$S;+Ne0tC7?ioUi#u)kpK7d?pR+~A?B)rM
z->-r@nbS_}Jn(CZf@4v*>Y^v+3(wZv_?|cM_s+<g+R7`d`;!_q3)3ooyn6ljo_}rI
zo+IgDw<cLWS)l7z5Fw;;Li8c$*~fjF3%&Qoi1n&1O!8~*a7$|8Ucou7IHf9>y~LzW
zXR49*=1Kl(;RzT1But(!DeJ7*U@7Iv$k3@|AL%;7>WEiRQp@2xPW$vHzcKLK!}fV&
z?A+NVeX8+|hXP*Q@B2JcPV?6Pw^QO6w;j1Lv+Q#JpLOXg)<-kxXU#v$*wo$7$iyyX
z(_ygiP%DqL(wj#Dj}92Iv5JWlOq}B6$0KUu^XS6@w=ho6X(lI*C1|&5JM-oIe`)m0
z!OVcYFX{5RFu$qHOI0Q$1#~vB7<H;maZ2lAQjOWBle>yR#hH`cF>BS*3)2tA8SONB
zbKT!zePHs9R@)b%4<lzXZ{_omHd-6qR`9#`W=i^k)vc9+x00kc=|4Eg^o{SI_Jh6#
zP9__^5}SY*j*HnG)jFl7zM7`XWKn)^)4_xnZ9S|$nzdCAL|Rs}SFPi)Ry^Lqsvq^_
zQfu(RAQ9FQts5)79XofimK}N#{pfJxaZckaFYY8Zw1sG^uaUfK>GOvDHiv=bsf3Ew
z3xexZlo?(h*#FAf|9srn1?dYMtc9;73GF+$IX(O0;Zt1J|27EAX@7a+ek1Ca<@T_O
zwSQXw&tTXtA(9x^<+~!HMNrQq(W#QtOrV30Q!S~3ZDPiRq%P64q&UHQpQ6e*Jq(hW
zf-A%xH#Mw1HZy%2r-PvDDa(?HeY~QQ(#{@cM+99qD~5DAp8c7&BztXz`D6x7(=Dk}
z`4*->Y39y#ycxvhX%<kL!L?u+Q&7P|k=f=&Y?A_l9Jd9q1Z{Y9-jVCq&E*VVTSUcl
z{}#Aj(poLy)vd9b!@;}a%a={(B)+s<eIC^)b>;ao_FSv$jth&tvZA?iUuk+X?RYpR
zkz;3-z?!&oyDr3So1iE;FT?BTh17ksZZ59nX?0tZ(Jg5mkRh{Xo0RD3U0YO&xC?`h
zO_nHZ)ZOOHl=(j(pNWHAu<(eY$y3LtS|U#!u3gePP(49YfFU)L@9G6LrYY?5<<C~$
z67ak?u`%EI=~MThYR-wC+Vg&S8hWkb(Dj<sAox3&Uwe^=mV^D9HARNDi+h4rsP#Dg
zQs4b+>X}JmjQt%Kx<u5(vfI_BzfkSqQvUMJ$yCcgIlz*O`%BYW%_GWlm?u75yv0lY
z$y>*SS!-0BWK8F(xG)9UPQ3J7EH&1{V`I}&=h(RI6Fk?fIdg5=W5t$l-t4(I{dC{T
zT)Cw7iSLMZYPt23z*3H%ZzrXsuy9>3U&NAOsQuR@YX!sWj==K{TQ(>&M1H+ubV_UD
zW#zVpIqRE>%8F0AMm%J0)%)L|8#Q6$!_{h^L|03%o95WGmTl*%g-$!wul%Yq`ugsI
z$kmr`B?9+6suf6^uBqr)%XZXa7Q;f<*#cRM3(ZV&jjm36lE9SPE#rPnH}}HiXzmXw
zyIxG^FD((0+3$M$?R)R-48QKZmhsG)kYwPLa>inRi_Ce2^2ecvm6nUS=e|p3J{2=R
zqENJIO_IpJN|A6iiD^|GO=8VLtqcz3R#A!u4f8YSo^iUuWxeB2YU%?g?$ZHndst2?
zyx^36&!c4gVCR|C1HwJ0`J6U4@9=T?AAO-`lEdOl3eVl&CQqx~<Sx(}@isbyM?rY8
znsjRbzamrQ^vP4$?EdC3s>Mppp2pHJZ_59YlWi9blc)8G$r|nAZ8Y6@+p%TSvSSSA
z6zYn3)_*#o7I)Q?p~0j>L`k8+@P@PESBb+KRS)~4ELO=+x)P%wrr7pBrB80+rYVnE
zR5d2G9JQaXb+t)Gts$h;OLt9*$9%VqjFG*b3`+wP+54_J_1st}BK)AK^@IVV(TWKz
ze2nva-hJ3&ed>wdey>9;O|28I|KU+Q>UGU=*?~q~o7G+(hs1uSMF@5)xLYtQKHqY!
zT&7RvdCieIT}~NaX0)m_YMd<btou@I-+aedWl6!}zdS<Q{#mYdd#lo;CL<=cDMN3f
z`05$#`;}KXuT^r`x*(>^twNwj%5yPiB)7ZqGqXF2PySD1%4n9}5ZPuOlj~IWfX7wM
zp^5$GAzmScW}Qs2Bf^5huFQ$br9UzzJo?z<mR5AqWU;Zy8$p(?+u9i=%Z|C`sUC?w
z^)gK~wrDeB*Ah|2WLK}KjpvQOZJBQp(qaF4ra-+NyFuY<_m(K9uGykN_SMN90dv$D
zbQJ_bk`j+g>`S(Ey!gnT!;|}^&sBjbGx?U}-|TXHX0&S!56e_O&XD<0hK*5LElMgM
zVs}}*<}#n6XtL&%+UiNAA=e$x&d|?35tq=k+?FkM+X-)n5TOFe`MNF&7qdA$C$b)0
zlJsPIR1fF(uH6$>l}(%z+;!>4jLrL6R`}N{E?gJfKgnNc9gB71|F7|V*NzEHzR|7t
ze|2A&Z`r*zkAq?>q(xk+8;z#)7PMqL-d*$NwE*i+rJ{&Lcio7O@kSm8tam>?#Kv`r
z&1gy^&vpsj{!K6ETJPB6Ec~FsKVF?-*53)KazQ8fY<mui>~vEsy(-C=+4AVul4)Uw
zo=)6wMAPq!hI-GIsPM8iw<hG?oKa(StI6|F*~2#*Huhxrobaf!T0K=~ldQo;R!xTK
zt0yd2If22YBlX;p2`UOf>;FCI*JM4w7nV?>yV#3E%k=Bh8DEqGT4w324EU<K_QxT)
zZ3Wsl8LF1P{Zcf8L-TM}nirdpkK(jEhO?`F^_&gsFw1y5`DxR)N2_iuGgmov`po~d
zf+H_pd=0BQwn=XDw9?Z#dN17XrATe%VVaobI#KaWihPM>(%nxsnaAV0rd#uVnpa_$
zZ*cQ(x^vaq^UWpUDoIx_ok~!;aPq?IT}GWwx*w_oXC4uER9yLf`Kyib(Lc;>RTL#6
zvYX~kz2W2>{NNCmU{SZ(olU&!W=wGKvTnTGwu9%^9VX6gD=zcBzU?i2@4IfrOQD-)
zYAbR!<@V-XnIL*RCF9W5r0iKmCavN#Z?-6w&fhH>y2{O5ZF*ow@`Rvm8`C;Z#5i&O
z?fQFc(!Jodw-{CZ-?@rvOunHwneEKkMGA~|J6qN(y68CFdBne9`lO9l)~%WI^3Ao=
zOONL==)PR#^gnw^>PcxY=^0F%7bd@Wc<(%KSDT4*gm>YMjTsNPv(lJ8WxSZM<XV+-
zT7^@`>b+gcnO<`ycobJZj`?!4)^W4j^!v}3Oa9d5xMA4n7jba6zzm0UksFoS#}B`o
zq`_monCs(;EuX!=ge)^EDt`Q}Uhojt2Cgr4MyFrK$A~=I^i54)J;zx(W`>XTmXyN>
zCQI_W+Dzb2u29_79_dt7U#;ks@q^`xU{^YeN5-TJoj02e`_<!Aqog_S>rLEOx1vck
zs6ss_=Z9fJ1fRR<G}EW|8MDiFa8)_3Ot&_2mkZeVXZwkkC$&r~Dp@9YbQxveoBI3y
z{(t^cz497M>L>c&UY=3)@qg*srPeJQt?vc0t-WluPDy*s(+u;J#=3<9qAQ|WKPpN-
zYZNicdvlR%>jf@Vj%Z%P@W`;tT}K7xEX=%8qSJFwA~d{o8DrL|bb$i#(AC#$*+1G|
zxi0Wp#dMvU_mAY*Kq1v<$trJBqJuxmRX<5?IWD$$n=DtMzxHCm90R#D5raDiE4Ug(
z*FLZ@XRgfo9GFv>Hn}x-;$@+~&2{^?$?0yi=890{ni$;B7@4v_!lJcW{)N!e<B4?|
zHN1(6M~+)>e^9z|a^mf&wYd&li`Wyk25?s#6|?*-VSU+{X`=G_Lcz-_h1>^2T!Q^V
z9yHrVw7D47T@ZH_cQn1<ob_(Hfa<sZ=E*HZ%z|MFUqjUr>-HTJNNWjGTj(~~T;Qo<
zMS)0;1gmflqomM;rcdS)n<V5fYB0?HP|vWU(s8>-Nf>uRc+hRrGT9{V2RHJU-w;qg
z%6Q;#uisOy+D3^t7I{nydQ07!R9?s$9OQ~C?SErnxwXLZD?|M6hhk-h9+xa899{0S
ztyGdDQZ`dv_~K`&g$K*#7TU6Z)LB|4FvlV2;wBZTlPXo|GP@gHue!Hfb5{-6+>^sp
z^yG!Wt?dGjQw5|LWnU!rfBW3p`CNEIglZ{k@7MIy<N*HIXGMz+#WOdG&NuJbZBfNL
z(V3^vsp&<v&&~Gc7gP4P3p8!i(VtZ1pIIGg`M<AuW1?_VN3wYPx&;+|H-s4268JC4
zO)PH9UYcWMBy!xGamosTgG#=kk6czA%viF*<WY#<dJc(j5uuBllAC_kEY+~vpjeyo
zL#8jo`7m3Z?{<OsquD>U2$ZliKPweD6H$No@Z>Wt#j|&+&1js=@<mwLqv6Q4lH7uZ
zq?-bFG#a-*7T7pbK<b6HRgyK+#177g*_%%`DjMe9*637@XiDFrdbiEucAMzl2mMSB
zxL32JFB6&jcY8h4aRKX_68jgic$Emu785qE&^c`+b>mAy`ANN-kES~mW%)k#WmT+|
z*_d3i-R^^D%V%c&k52@bXfgzC=+-Sy-y>0Ut})E*Sc!1>|H(RvtUWsw4ATpo8718k
zvv;{kDtzqde<8H(fUMEwDKbgz3fuC}U7XsuLm*XwF*v;==%ta=uPK{u<~>}|QSUfS
zKXcl*&mEbW+*=jWb0vkWq=dQB=gOH2u(c<7J?Px6ChxXNd0~fO%}N>OQ_T)51pKPz
zZ74Jio$Re9wxr=<vOsD#tJP$~h4xzPYTA$eQ&?mt-W2HT2%8(eAiBYYTQHjWcu(CI
z35AOboD)TMxQc8~6`58c5Ol#Nmtk?7km9yXqn9sKb0<W$bTn{eaF=}$FZtHzQmMLU
zh1-&X{_hn6e_n|GRVbO98riL?)n97W%^JC+D`NJiikgztvl$b<ZvQW0^H_~Nc>0M$
znb%T$1!tCdEuF6Xb;&VTf#anDE5(*xZ?e1hMMcU`x*}cR)#MqGOZ3iat^KUbIzJ-d
z&<=r)=k|M6*t1v7Ubk`bCRdS<kMx+Q&zIlmW3pu-x0A7VWn+_J`#Ua)@`Tx%x7wxC
z*o6Yvo=>X|<jfRUlyPHWt?94$Nk7Dkt`%!Z3V(AD`Y>hk@#8)BBz<?M&Ff*4uRO_G
zQz<C$(IjH=hTW-MiqdQIPl|aQl<j{hlgDDFCO$K*Wf>1+SjN}DwGWofsdSff6ycn(
zc}C-e4ZrFfQa76%pW$`Ds$4?2jmz})4uJ!gxl|9Zs{OLw|C~#D1M9(=a<@~P)EoZ`
z{L#?hkZ#Cpt88|RY-#keQD=~P(9_}~!F<K%th3ZMt&K(!%}fQ}r<}G|y-d2srR(D|
ze^+2kc*;aW#ra<<CvJIUE|id6@ksmQVZEP<tv6Q*+}b73*O7AFQRt52X1>*v_`URF
z4g?t`ZZQdN-~D`Horc7AhQ*3j0!D`!doG$t9TS_wI5}pk(B~b!=Q?wlH+tk9TWnJ`
zbNcnb#BPBvO9giS4*o18>=de;_j~y|$LU+FlEkwGbhm{Rq|6BZx=~4blZn`p6c0(E
z4V<|ba(iEtyfp9mv3TqAgWY$^mmLh-sivNCG|ZX#Vx@s2+v-!3Z!TPrsVyv5*thF&
zpVcMtZI}Kx21sj%oK$jTTFI_xZR^Um>F}(%p-df89Y&qG?MlMG(zh+@T9x!r;Mehk
zyoTEp8|75XQWX<zI1;11CP<cOZgh{b-DV}Q%WC6A?W~vu%}aN<+%3xb;K+6F(Y)kY
zJ|2_GFYGFObvfqi&HedRCjHR?j?boj=yprFwKS(nVD^KViRE#7j96cvs^*>{5O;^G
zyg+W%?5P@0*zy)gFdbvu#}XjCVSSl^lx`t+$_B=&43%;RF6jeY|0nFN|1;@ySf|6P
zLjvCNZCmC?FGw=oQ?u;h3D3thBF87*YmV7yAt7X7@Qdl_<j8$`6NQ4fs`RaQt_kq}
zewpk0ZPPMQ;o`2zDL4L`^DboA$RgkT$|TR>^xVYYXBiUOo19oqlyYxduf3?#;SpQR
zfw@(R8EYh^nl7@1T|aGg=}e80c(&l7YctZ)+$1i)iMYYDjU#c};#C6ePh`rBif>tl
zoX-~U`Z9q_aeGm;PO0~HFHvR7p01aYYqoDZ9QsN(B|zZbrt`8hU0bXt#Z3yWdUWLa
zucBwtLW?J~hOOwB?X`H8cVDR~qwJ4`ft3eVDhu$Wb>s%H2`iOWZ%kk+$ggx{I@ogf
z*DJ37k7RFLVJmC!ZM!JG_QJ8_mt;>L3f2wYrQmd#S9lNSX)*cKT+WBPw=m@%{=KS(
zS>iam{>|bOa~sbY<ZNsf>@rL;adrD&E6shPdFSDqj@{chj{f`-`eTyNZ<m>wT1OL`
z*Q%zwMFj0P`QmP|d7Bq+x9}{15JSd3waY)3UDn!iVBLds6$a;&19^wH_OY2A+dDB~
zZ;On`L$32F><6~6>Z)X|Qz*T0T2eDbpp|XGuZuxlBEnaD#Fm+ea~u`xE{(8nU*&jQ
zjC1O`4Lh{8VwP`rU1EE&pigFR@LCsvnF7tvq%Lmu6*<|QvvHzlk>Cr#BMsF@_AV4;
zn%MICva2)41=kr%ulkl9Ww%HU=~=$xB=f^bFS3PGHgNhk>#JO>l>U@*EmqakT;h{i
z-?j~XZ?|a~E)idUJu>wI+X)e=KQ-Q)lFzJR^7*jsf5L$jdy&M`WdVt69To+tT5n58
z;OEgUOGt>EnZU7Uk&z=?R6wtqvxbAEz`Lzi-5#BCEOqJlUDzRWb!(PN-t_Ia7N5T(
zbTW0tox^MQHY9F}dA0BETmh?nYi<6SUHqv3CHbgr=!Bb|(Hs|h+PmFPI+wlqD<PB+
zJDXRf@fd4(`QdFbOu~v8TQ8rHQMkk&c1C)GQ261S!WX3vObMD+n{l(Gbna2D+y|0M
zhf3M*Ol^HNTYZt>+XoNpioCckanBJ+`}Je8*~FZhf0ys&oV|K2uyzq^V8N4LiXPiV
z8S;K#;rw{$WzWk8nZ8FC={)IN;yy{Gb)^ZDtl!NhH&(_|ITyFrFaLjpqmhB_<Ev}B
z`kW06K^yK`U5f3$9lOSW>xCoRLqWOQzD3d2JwkgGxSHo~`a8|hEUG{zLuId)-cz=o
zi$dye*<Nj9tu%Zqv*3YhDz}Q1%H9yRHJjGwE?D2$csRF!n~Sg7;Yo19&kH;covrUK
z%sViDMf3Z4Opn+aEw>0btub(9$T#E`l@m)9aGe`5m3<G#LRP^I=Js5t<-(P%x=)<F
zExc2<Jb172`QodFA5EfUA6=Lv;4oqT2Dx*;F3opcwla-%fwuJK|0ZvYABKKo?$O?T
z&FG+8PKfZ77t1yF2najqu68_k^0mFm;?oX7K3XOBP5y4__j<O#@qTp}o9BJ5xuO3h
zR2TSKyt$EDV4rp-M{5b=k_+l9YQ>@s_NPCL&sOBx`oQ3ApyzUC$1M-||JhY8e|0u>
z_Q6S01^hSff3^11WWm)RojQXIOkU}9+2_l9wMZ?UsJy!IaNm06H64Z3`%R8N-G1x@
zXY#Ezdq1w>E0hs;oWwVgp-1}D8z-&{3kBac-upW7O;i-u<1m+V6E~ZEU8`Dp^KxFW
zmZ*;e<Ff>%>b18e7j4suc_3T%fK6r5O~XR=<yWiEZCn?(#(MQ-fyD<LT{r{(K5!H<
z5$I3)FqJvt-BUva=0_@y{Cy^eQYFOH7(dE7p4d2pfuZ5t)V?Ja9n<^XxI|wNIT!A~
zGVa$@9swqn|9mPM3?~n$Fe&F`xlRamHsia^)1xtgcV&ktYsQv{3(D^OOE@M88eDwr
zIYm`kY);1Ny9cH+Em<P7qG1`sY&lVnn}<${X|j6d@l9C}%p`8j63e+kHKDs9tUSv1
z#v?a}nVPCOI~5N*GO$l#ir$*}beYcCrM|QITz5`;8@+>hI+w)erPmMCv0UdAIJ#<H
zLsgR7(^%6DVfIl>5l_!*Z@4!jbnl<9QmP5Et3zk92FZQCE`RWRVP>0{gVYy?%S_VO
zXL%Yd4Cv=%J=Heb#c>kDCe<@fbbmczVLZUfbYI}l>%!$s%QVvaWK_i&FDw*pId@Wd
z`pe7fxn4Bc{yOlEY4;DgtpBkKDnI2+o{%{8-rwqrDQ281#$72ldPRcG?lLhvKRM@w
zx5EP6n9O8{9~LuCD;ue8IK``XDk5u*QJ$do%IyXsZ~GceR63j+9Yh>lgxG~X9-i!H
zQs~O4^?SA90hU#+lLLP(S}q^xw$Q!dkXz<@u_`yoqb6li(N8yjV@Y85XbKa2s<-1`
z6}MZ{iX*R=h;us$*v(?&Ik0Rw%dACPCr5FAUS`bcRlMRU$C7ovM%ym<?Kax6<@32l
z0f83_Y`*K9uxC^e=v+G4AxqVBn}96GMYD*L8usl8QB#(ka_K&?cE&7W1~bM|q07_4
zR0OiTo3=f<vRLy@!D<EbXI@&Gf`(gMUH{u;Z#7lb%o02x-p{l;XG-V`O`VNLZfWay
zYR(pMWvKd-=)Up1)9*DK()=H7RnX3F3TnvRzPIdp<38OLMz{Y7A23^(QLVpFMaW~3
z#>@wz9$jX=)!r7q-O2@A50_mC_i41qnA~i-^TVs1jn72R?tZaKx?_sbn%6BnUs#k{
z&o!R9@NQ3&mYBw)?@La63Hv0Xnjg@eDWti`G2SZkZ1nbwqoS%$M0J<*`Y1&*aN8~_
zO`q53mdQQ&qmb&2s%cMG+m;o5SuPhgWu3adZ&)JhiKuBy)ibA5RffHYI=N@3+72O$
zQiYA&O2%)WTzOJtxnpLvme0!_!VLBsKKph0Gk$q;+w?zEvevG}o{Cq`E6@Gfm;K~V
zOx4VgVrJFzX_~LL%+%PS;&D}X`?RQl1?Slx?OAXkGl+X#XrTJ~FH^QDJl>qceR=CN
zFY(h>Zbw}NofXQC1^6mNU0b<y-a~({-X}RHfBSGxIp^me%eZff&Aod!>i8`l{b8-x
zX~G$2@vv?0>fgzJuT)n0H6Ghvt+2<q?EmBM>t9za`1H`(q3I`^KY!{`ZmBJo#Mb82
z{z!CQs@F8B?Tb6pgw>*7YMPC|R333wE^<B3vS1-+U5ArQS7>|0lu7d!YPgvHjO4J7
zQ2Lc~Q2b<ctHiC#{dGNu${RXwIlOY@4!g5d{$v$D|N1ZUp0I8#WlDJP|D2Ej!}f(|
zl?@h$7rjbdUS<2y_e*22quDk4g^G-f4Ne~7{7S4wJwAb*pF0+tU6MPtu%(B^Qp7jn
zpnczmejDD2K|VPfIa%hOt<=<1%H8(lfbT?K1<S_pr+(oRtamKpNr_P5T$E(B>_(z)
z#6t(|HxIga#8f7$O;Hdtl~k{;c^&n4fvjVdBY*6}Jd=k~C${+o3)FkOZkJQ+)nQGN
zDqP6ocJj7|NLPotDoaME+{QS^52a@sAFQ@GP}#L_+k-RDWO6jZm-Lk#dbF`MaI%7+
z%8806<-)m}SUIC7ELQ&*B5BFTrNZ|xUW@gCU+hFh#_7iD(K$|R+>4g5#A<dvXZhdY
z89t|DVoJtT>3NonMr$tGh6TpF{N$-op0skF@Y+cXE52-5{&Ufs?LyB^-!hw^`(a6y
ziRQVP^B#%LII7z9(z^5KHSu%b4~XQ{8GHEtoh+~~bD^T}&CKeEg%?U^eGF{*z0zUQ
zl8L2SeTCDL9y_=lJka+@#bHaJgP)AXnH6G77c;)nkjzm$*>qzOkMIhWaDi2dLL3b&
zsTo&Y4m=X&-1=~4>Is#apy_hg7OIrAw{)?6DPR3~srrURozv`V1icU6IxKc1+g<n8
z$78>85{07^?&Yu*nU$3q>haB)%`jv7vU>;IXCBk++qX=?V7Z5|o`LNAl0D7yIfB;s
zm7L;>N!UI2e_!Eb>llfCbJxxDzU53=^wKjqNPe2d!(+wsTzj-G%({J1_r{`<B3?hq
zTNl_kJx=g0*F9-`vY_4Q3a69w1E%&phb}zawY){As8xH(hwx7vg<hM#DEdkyU%ce9
zTF!qn`$st^uCSgak=YW>Vp|kgbzKtUSUVXST8<rUYWSvm$2hp<^5#pE3lh05WX7|k
z&WgMEV&$@Fswe(>2nzSUII|;9N77L)bYd@;xZW#~(@E~%gv+O{4QBeeRAKgNg=Hs}
zs;E9#C+lU%xTI$)&!n8gnbV!xwoGbRGvmo2Hm^d@DJ2)uE27><TfFSj?Ma%VA-FVa
zS87`S7v<^Z51(oazU~gt`X9FOYliFGNJqAy6U#RnJ>2Z@&Erw=+$G1I8C90IFs+!B
z<;W}_s#NLrdRd+0<fY7kx=ZIT+At+za_!7ze!n&a%ig_o+$L6{)N#kj&X)y;w(sja
z<m^zh++6NLfnL~|C2B$mQ<fBYH~X#kT{dA#yHhK(v`3e;ngbJaYPn{vzzo?F4)>N6
z9B>Su$e?Wb^ObJ(Oih+``Bk5VPVadWerI3Cn|fxQ`AqF52eX5tSJqiic_#E^=aZYp
zjuo38sI^^DR=pc}=ZVVl=}FT53<8DoSq`|`sycEjapo<yn6$)A&?Q~i{7%3^Ce03U
z-j(gmUp>QwF1BrGnrAxmmBQJl?(2@Z-TSYy?PM7DCPtrmM;1tTcxjsjh|XTU#L}1T
z<<VHjCXQHv67`$e6N1aHPyP_9>5~4d$?CX*u8YXDEj+%nBP(Yek_&rXT$VF8dgAjy
zk7=9Or#WRj%zZJ9-MENVGkgJK?#(kroD*!{zX{fuE>|O%GBe&X_kJMv+$on>F5j2j
zb60}R{%dbat(;})T$%HRyGw;_-d$YT!z!maYs-5zHRc3IsY_fhay9Q+Z(FXQnjp-%
zV7h{s)%&mp<B98X8d>~TCU^>+3Tr*IbX~ztv*|*TiyDktA8z)E+pf)J-JKM2XKRM5
z?it?%-4)UYYs{(*+mkYM3p2lz&8ij>|MS;q=gPRo@c%DOnAg92*e0bXpjYU4%wXD%
zo{b9H-{w1rNEI#?I~Qis<`sO2Lw_NMfFI|CSm_R?FCzBVoC^bX-<D8WHL1a!OFR8B
z!{i(4O&S`@ySwlGUU2udO5=?svrjB|Ag#D;N9+11u0@h-W>;*sh}M~IA#!5Ynv+`A
zHQFkHozv|<cTD77!Xl!w)!0(OWID?YyZZ}xm3z3;Z#C&^bUD2;p5>r6g@JGVi|IU<
zMZRn7@tr;YyCFZHvuV?YKAXd8KO~($ywv?Zga7nyn^_&5W&)i54jNk~Id98gHF{|J
zBjJ#ujMJP8hhMyOe)&@4>g6R4DcuRh{m~LGynht#dn}1Quw+vPv-$rGdTxjM(r0xl
zuGG-_*!nbDD2!tRqcpQ`fH`Y7)9Y?-%^BOzD(&6Kd8ktP&~^`L&8#i;FV}b-7Uogr
z<onInCFFenLqFq9rSf3uJ)C-`Mq0mS+y6VbzkTA?sM-5fCOZGnm}MX(H$j-AVyD)m
z&BtEOl)J4lo$b$37LMbLnc{XayI*Ry)+@;_vzT}>lly9+Xn(|nEdj?DT66M92@21g
zD!gY&lT>$5gbLpZqq>LUnm2kUN;6wV>pZg-vAMkY#bNVp)e4TYPAJ4!cB^#AwCwI+
zk$LfQ-@*;f+dAx?&D1%Wc_`>h#~x!f#v9gGnxq>QCEkDJjoY)#PnhAe#rz+E`+xs0
zkpKLU<?Lf!8%Kskp30^z-EAD)=02T)9h}Qdcsvt(3rrj?AD+V+)WB?`#Ok0lXM)^H
zmjeq|2n#bg-anzT@rb*SiMP{|1>Q!RJaU>nRT%4MHt<ep;9TJC<TOFHXTu9;1D4<o
zZi_pWS$uvd9N6x$^N`4nJuI4<0ZmC+yqa7qBlb)-{%Fp7`sl(7dzucMS@cJG`3mDG
z<vCMs?C3sie`(8pBcrpE8~LXs?zpMR5qojE{1e0Nr|p!wPUWBBJpAO~&7SV}kM_0+
z87+LgpvJ_}^U?858LYjE%*~Z*O>-KWTDEsB@nFBK!<yqz%&GRm^5jdKsdFAo*{1C^
zg+Zl%%ZV-jXRa~P<f=8?J;lSbafWMkrN8H5&i6kzXq~Y+F?sFohfa4MHT`zz|JB(Y
zx|;VJga3~M#*8O6>&y|!kU2YfMXRoZz=~h|;a8-Me$3HhUMTD($isThE$4{KBt3Ht
z*6lv$bOmPYoa6n*vQu`>g>D}I9~KUwdpIXd@mue5G*qYK;_0&!Y{ZQ$g&#_+RJ+x{
zrELG>!}e{SdnO;?KdTZTXU#E3XiAj!Dh*T4pTRwyJ2<p#FC^Oph)Q?-e!QbI<M172
z{%;G8E%V~zywKatqH=Mw)*X{GtaqkN>z>0far)v3Qx%-KANU*>;5jF{lQ~+!FI;0q
z{h7^A?=&>dX<%KloT10h|NqS3Tpu49U0boGfdbY}A8k56_H-BCIdS(&Cu66NIpa!S
zjlR1DP8Xv*LvuZ!I;{QT5cJ_7-xuSpZ#xve>^yl?dcj)-&iteq2N(NSE1&w|y<SVS
z<EKZbnau?;N0G!*uAs(1zfwH~XK&t%MlMp0n|VU!abDpnaxrf>(x}C3`B8k=meUUv
zmfYXqQL|C<#LPgYO(9c1cQ9%$syw@oF)>g&_hRRo5Vg?2Vx5jtMZ&6cntG;ia(%W_
z`r5lZLs-vZKTGVT?+rJma9r5;((AxO;hBjS<~Lb<vT&Oq5xnt)#S$-}&b^(Ei6>TX
zY+qP8y=3DR+f`cYd5$%$knF#)IDYf-ivMe_336Vkl?kbjQR!_w`K-ooN5MR+)?VqF
zey^kZW*eM2P{Kd!?REds>)RLmIo%doJ7-M;udrgS@b#JVnq*d{G<BOu+BXI0PQK*2
zb#Lbe3!9iT=i4i7W-pPS&NP4ag|*s2`=?B3(4AnlvbXzcjl{v!fVrNFZ$>)ZoDwzb
z^tQ$i0pS~1Jwz^g_6Q5>UKdF{KfA$cO2N*)UE%(>LL|3FOlH1&aLbjjIp=D1ZC}g@
z;Jk23@saVy-JOPveP)T$Q>vt&-Pphze0%%S{_PCc{w3NwbT@qqJa$<|Bvpj-2Xmyp
zZs)VV9nX93rIqlrEbaehU}xkoea8*E@Hc*}Djc(K$~gQtJ!kggxU1C_7hR7fuGd!`
zI=g*B$gBjX#ec({c5}Xeqx2&2))WILkI*?gK3`eB_SU=8Yx=g_D(AKR&amV_u#=jO
z=f=I$H_EP^{$$?OP0`%j7JUee-g$JbZ}-|WPtLzD*z@0pvrD(5$(y0*Q}ooIx5Xqo
zlE2>V47eveS!CmaPT!0T7kuxOUro#qy?bRT=XK4%ohE&p3Qkp9-BxWCR^Ap-w|BCd
zTkM&{hl}=@pI>C%SSl-3a_+QI#D~oW?~D6eOSyQx74J_?nx$c}&yz#9LzzwW)Sb%~
z7d2v@v?@P8D%_{aX`Cuha!~69i&N0a&LWcoch-c+u8o|OsrPs9KKWx8-~Q)*q9>Rl
zE9NBsY@^dX5r#d-#P7y`P>HS7UDR+l%Je47gAKupLeDQeFnyEh+9rt^p4C^mw=G&Q
z`}Us_&FHyXdNsG^cFi}o6JK=E-gus2sx4=iaCex)uK5kqtM0~3dbqW0tMG}3F~9bn
z@cQ>ueOly!i?g%jx=v@VoA^47efJ!;Z8QD-_^x+0HC%OzXyJW6=dQxmM@nb!KFhcf
zR<`n9@1A(J`%7hf({zs|PP;c-z_(saCdebq#`4r3j&7?}4|o6ci#~Z>cUfnXs2T%H
z{HG0pAw6f`wq`KbbzA7(_}v?Rw@1d>m;dk45Qby-Z{|eyTnU->?}UG9aQEN);&z>`
zpVv(JpLB>l&R<DwZRpZ1`@D5O{SM_R(>)YD|7KNc;BC*N+K*4J_T&@mXmaM%Okhrw
zmHDx2Pmz#bc=qFdZqAf4{zFs3cW}IxElYEpeM0$KV!POD>9f*K?~=Rz-Oj1opyj7z
zWUyhE+H*T);e`q2)yf>3Te$;eA8+*yU^9HP+c)S)9rs>YDc8u(HjbFdKUas{xHOw1
z*n6hw_XE<$GMjDVj#e0-d8Q;<xiaL()fY3ihPW@g|9p{p=R3z+M>^N@9r9t#Oijy{
zZFxIM`DO37E87dM%k@3k9G7|H+oM3)S+fm1Ro+-?#!PP9xuu!$UW?!>!=9)5OH#Mx
zYVKfY>ay+leQ{;t|I~*nFKuRN#OUkJ3Q=k`(S5!1-&@aTi@%?HQT;CW`K$R`??u;N
zJ$Cra#SFcUf^Y9N)^%2H^<s+GV@Z@_xt<*}bGq&8>}Fowy2Y;dBQ&{|N%Oqh@H1?3
znlI;kEsh_*tc;j$F23~Un}=avH}@aynCTzS%(!rF%8NMnxB2SJJl^NtlBj=uB}}IO
z-P=iS9ag4K9`&8Dn-w|5?vNkrmKKh@->pfrGIoBe{lwW(q{)Ax-m|m#T0`-*-I^PA
z&#_CL$0zZw>14}P`G~U{jQS<sER#^(nsDaGp{DLvm6EMr@(fq1D)4Wg(R=IFWQA5k
zcg}LzT($J<)jjzYuGv@5rTAODJ^a5Xeg|Jc?lKL}ScSiQQ*5@Jx~%=kd;00iS{HY9
zPI&1dbIaialW>QjL3(2c({AskEjOEV<N4U-FU_;g+tzW$L@}rG+|{sY?_$&rFZFxy
zeOi-QM*AIa=hm<bM;ad=Z0$cKmyw#@Z8o8)`sow#^D@%){Q1*X{n=;3+90k}<8Wu&
zvuSBxrbry<<a1JOzi5AH<Fm?)it10_rWS27lF)n6a8HYIhXG?qXScE8iJ1Y1d>=IZ
zG)N7-Ywo38@i5_BL*vXB|6<Lq=S4pI+Fn`_bNy+TD#tsw#H8~DO%Cc3)@Mz!7GLvB
z`z~PnA+*VG;c}I0KdR++eoO22C@*19SDKPuDfeGa#dggq@9+ktzE9Jq*uK0VahzM9
zv7ph&pm{;PO|^c9&<lol8jVZx{ij@Dn)E}bN2lxBi}Ifx!mpSgMR7j1T|N8bg;J)Y
z^ZVTw8_&&2z4|$czpy)z*@bgmM?&10z979u+Z7d^Wesl{FEmZ*k91IbZo&EX(%FxN
z{3Vj%u}b?VGp2mc_WR?|ec63o)QtHdC-wih==qedUp2S?)8$Fu`5rVaumANgaR1C}
z&o+oQ_-1%asyS5R8@OJhkMrNWuz9t=64$nUkBzy)H$Udh+Yh=KzLyO?J)OKGYl=eM
z!ima#1`2Fn3zkS<-6zjw6v?+KP2q>Dz`^`a5B0KD{ri9W<z@Xp|JJiV`_H<vQ~fe~
zS}IefeR(>ogz3K5+3S^a)wMUh`XkiPr>DW3Z`Ax-zUp`}V}pRgf*uAvCa)<E6gIRt
zG{#EF<(vp`U~l1;oh2|~ql=_DlS+o5KvB0xV~U59NJG#85yqe;ITxMME}xmK7yRhV
z%Hrh*rm94Xv3M;@aBjBY{NyvMXhqN*&k8SvsZ&COS23;h+N#wMDm^D^_Od^#CiC8y
zvQ}!Is`iCxH#bKManxu9H2QH`zPuJ%RLm9E!TILXo9Jzt59YSMHnXwWzJ}>wF{5=^
zSGebsew&ACp*jqk8Fn&hIfpEYT<>>@$;bT7?TW9jIM=BcJP_J_<Zy?0VcP#EGph?u
zbqcBr=_t=hmNZJ)p)h?y%F~41LQ66(I!)qPIcv7rfisqh&)eF#wr|e)5v;&Ig{zax
zWL3!Cs%5VfiyXNo3J2d4c<Rh@Nx+dsbM28^<)IDC(@Mjf7q3vgpb?~bRn%oG2g5?o
zf<O<2bc;l`8?B6a9tpv+RT7Ijls_GtnJiiNWja&q#ib5RdS+?|ySRKD7PCnE9CYks
zJDMKimX@IsBe~D8C19dJse*&}?w@XfUi^k3ZJeqpCQlvGJTzm(w@f|g%DL&_x2Rs>
zJrWG*XTDCEEW+!$ahZ4Yj>;fk&Ard(w(CBWjGLvh@Q89?>7<U!0bhG}9E}XAy&5VV
z^S_dHVx#87rwUP)rKQuB2zbtV5XPiAOL(DgscTph-!^4Mm)Q*(6VkjEUYj1<v{39t
z!bZKe;M6Uj*rGaigq3ACRLx|2li65%Osbc=!;?{T<5{)9pxr0AE>3jkQ(01AtF-(v
zOR$eZYY~^q#D<xSED8=gpED^4FxeO`cl0}NyZxwIa*xwppNyuhcgybPE>EtA{-J%K
zviavsxoTcss|gNYI6r8*@F`D^tK}<J66mPU{S?1Ny4~Vq!<r3Cg(X+At&noj&b%Rz
zu96>f^OV-%m~5|Ir)*}N+*YN**im8HHo0fjFIK^xl@}wv%-HbuMyg}1P>ZUg+{}ji
zF8fq(Hcl1&uQ)Skk}ubk=xKU7nyKQcIWLz^w>*?qGLx%EJ78y?`Im2g9BXc0kl--y
z2^L}6GV_Rc!6KK-VqXu=T;}(qiFJyXCS$2VjjO7vd>yZEp3psu(0NltQY0!gU3mC*
zWH^VZt`O6nW!AFGVHVSi#3-f>2OmUk+cH^r6=RmzOqSpWH`cCcUE4L|eB&jVlE_)H
z6~?LW)7)mU?6|jeR!?u|NgZ+7l1AI>v6CZaPx1<^p0M@$v9_4^!8ZSEm|n9xCJHby
zPFS~-_vHMbzjLnYI{f3!O|o`T{K=yhJ%i8J?4bHv6BZk$7kv&A8U}fsD*l2h{pB8y
zEat|1u$yvw!Bdt=JCAeze^q2q%@s3ES@c4oPOGJ3qL<r6feSke)K}d;d7@<7Org6=
zQ&wtnpXLa$S^CARRdiFtnXZn7XaDdR=RCeSBg2GI=B0)i>(8Kq36BI7I9a^87GKJp
zQ!?T0(<h3aLdjm6QkphSed`_n$YI8;9kUoVoQ-+ZBrUsmtNq_dH-+--uKO`e!i=+A
z3d|!X&%WR?sgc#Au+mCl?!PJK{HgDn6^%sv<y056S*$2c&1B|gop5qPj_PE-O;30Z
zUTO<#$qG^wKC##%QR$G%QrG!`4>xqQ3db3;b$n~Qw?1p7ce3PLQ;`Y0hWp)5Z}Rc3
zxq2yR`X*L`yvlAPPlo9`j+Cz3VY)Q$|1ZH4X8K#s{^Deq^Ceh3|J}1?<!dt=c(ore
zaEmL{aXQ5F^F(+FOrJOB$I1lFl@d?Bbx!%NvPS366;C0H?V7(!HfeI&E;|u)X=-az
zPh0)VhpEp#9d_8uqa(gnL(qEH<k^ug+QbffyO}$S&pmn~!>?!2&1D%P{Q@n@%!Q5Y
zyVZ8K8XU>u`6zcyBWYrH#rkE5*2=k8IMmF{3YiLGUddD{o6miquUT%R<Zj!#eyYd8
z{Cb`_b3B|p4nK7%WC^+>w&Q}ulp9MDQ{O)KQ<Bv1RBIEN7UC5qvTNlvLl52lOQ&M=
zPp-Vus}Y=$$^KSGRUjy5ih9rsF%LzdJx5a7Rv9J#zsNV6Y1Z*`I;K;;F3nK5EpSOg
zA!1w9StgNyTd#CqSFP%m*)oaA?SpBGvXG!q;fVk{y*ckI7skdjH5K^IX$YR9^!I{A
z<Lt?QPwLlm2&^k8y=32b!NrE@W^~e?sdEiC1kcqARB#bFad*m>?()o_B_*t7=UX4D
zB&CWhJS@i<AgGqQ!0M&H_Ae`|wnyShhCZEdY$_xCqO_cN4Mn4GO<<7@D%rsl=yXJD
z%R-jxIxe-9sv*YOkqeg|IcV`yFC?upXlh<}@5Yx--lEq`7!x$Jys~t^ubCnw9Fkkj
zBDuJG^%F<&DT+svjiWgyJUg~^Znv@j(X={t<4~Jh>h<NxekK0}7q_lmXS}xgNT<^q
z%LO)`2C5+^o+}p@T|Rj*G)(Hn6AfNzum4G>*7q8%QTzWY?SENUaIW6z0Kb#>*RA;!
zxK^PxNoe(|qje@GJI_4qSkQhm@|j9liC`{YW2MRU2>$A0DuUg!C;2?+c^`H2kgiGT
zYL<Jy#Ol6w$M2fQ)xwaoS@UeGM{%&NLb!M&!#k_z^Z&EOGD=)%;N}S|msv4EV_VBa
z-;3&12bJy%FmkDLRP-HVYrg*b*7DLzePWtovxI`5rYur)-LkgG{l)#zuN4AjFVo{x
zPA6>I!fcZ?>k!MMHkoXL6+1asZCQ0qpqlyWGRcF5SI*tYms`xcQfS4+FVnpKr_AM<
zsm?h+dRupez49@sN8N`#OXes|P(S=KFV<F4@g#Scw))Y>iN5_2w+|m&y5Vc$yr{~^
zZ2`MG?tc$hJ==Io{0^(^e4(dnHLjkHti0hooi%*Rx}1hRM`wrb><V4AXsV94x2>UX
z1Jee@G^2x$W_wjjZgW|+XzPxGIiJ4h&3dpXZD-Bwb>F7eJ`}&_AL*tW+P8Mr+i#T?
z5p8^1D(>EDczdMJZ_jOkql!lJ1XCVVsC69ttTOwYW_qRa`R;kUzVts6a1_zV)T|U@
zjbK>vLB`MLDwh!7_IAm&7q=_d1oBp2S*-eXjk~|e&7CTunVVlu%+FZmD#E_}?%qg`
zNd=dFSO1SzSr&Wy<(|@<D=jV;O<kQ=9=tj){^_S@eXpxF+~F)qd0{M)nj7R<mSr?=
z;ceL`;s1W`4>|OdZ=2}+$n~1Pci)MdljC>ZW~WGlMgOyNYc0>No;_uWOMBYd34RS5
z9OHQx=w4|)|8oxSL!YxMvfFk2GhEL%tvuD`a-&JMZNmeu#eXMlw7J!Hr0v^{Hn)z1
z>{fnV(ax=JW_*3X(YK`E<mAMeEm2Gij4^w71&;JHlmwqSaEtfhd7exD#hl_yOQKeF
z3JV;Px$}|bAqUIagQvZ|^xqThyZf=_{@c^*99=4>%>S`e{E5o#jh6SSbmBHHbz%9z
ztJ!kq=}*zKjpw%b#)Yb|{y$g9^}?Xz?@e*PEr({k;aND*^YG1}mzH6ku^q*GJmduD
z_8ghl@s>lhlyQOy@1BP@sxHp1{UbMJj-bn%+s;?o0t*kQZ411p6ZrXI%V*iduZ~A1
zwo3C&?9gQEun_c`>>J~pGIcF4m-0?wmySDLD)WLm<~%HHv|V}JcVRr+LGj0X<#pc8
z`QOSRC3nT|O{kwi*PO3YjjX1ebM>&Tkb7NwZNX9&eO-kRi~i3hQtVupHx)2^Yf$(e
z5S6oY+GYm!LLSflJ7G6ohV7UlkhX`vaAlkzqmpLZ<T{0eJ3h8(X+;+<W%<@RImqC;
zM$^>0OO9NyJ?{NLD%eJG&JVt|g)<&l#{GW~*x(Sr$ebz}kf6w6JI{kp*<U5PI`)F>
z%z&qvac6s#WFEC_HT2k8IBBNrwDULTFZNg%v_|d-ui*ArF~wIs#WgG)C-2!FWtVbf
z5j%L>;l_-~f8_k0gm757boVY`_z*pfhyAUtx3prndeQAa8c!l71v@YuDAJwI6L<f1
zuj*IMQ^jvv?r-t@&UsEkL@q!i$abkfWv6J6mC&;p5~nf)gv{KJ-Dt1vWDULYaP~u&
z|0{3Ke7dA#j%(aTp>2^u%pZf&OBbK9IQBV1#K3Z5b^rrwje=T};N%UD-*tLit(-7n
zg5&fl{M!q-Q+GW0ZFALh;jB9WC(PD%JMyKpFX3LZ^Va`=HNv;=DxCSEX08|>`6j`c
zfsskUk;TPP#G^?p<B-^uZZ(VaV3YLnXD-Ej$5tJc{j-#1&z#G`k7ilbisoxvD3_4i
zz{w!`;-vPhC-H3$AMI@q<e5^X${KUUmxbei#1<#r15Y{*C^Go98t+>2@9q<>w=COu
zBU%g=>P=Pe;y6;Kacp%2+t)Qy-l#5e)LHcI?mQn&;k~}R`!ao9a(q?UIvr9}ZxkH)
zSgIOSsl=-nC_6E=)l<wtF1%sQ)5bS5|HR&_xOLu;Q>C^`fWdcq(I4MA9MUc~?&@}(
z&^y#_yLEbX!i8U)%Ys}E6uYV=vvJ72V&PU?_J8Kf=UY;yo1RGFI`Dt-l|EBHuUQOp
zS?Z3MKk8ZB`&ez(<7s<la4cFpWyRwqAK50hEXYrh%CQUH{^DA9i}y{=NI!v>6Q`xN
zNs7yB%~&dRzF=+lAB9G(U(4EE8bdcQSN_YG*thidM(&!{mY5mK<(77?J9x5Kqm{{0
zU{*tWg`%cMRKNq%c^w?eiE7IkCMY%(^{H;XDYfjS(WZyXB4(QiooF)A$lr2EtMhob
zfa-;W)u|T8dn6VK2Rih<>F};Suf((}c$J#Vj0``Qr;~5AUH@`>+LA!aMZByle7hxL
z`rfhh_bpxW$&IClsbvF`_7$fCd8`bQPbN;ws*CMdw=mv$ljoi#mpyN!I{P(>guMCh
z>v4UuU#3`IA_r^8y%{o&bK(=)B>hERBt46tqm^8HG*jA0WcISY16&d<MF-<;7PJ>C
zY}(WIRmUY@Z`i~@?`4YZIUoF%?|krU4tL(N)0d3Y9iJ_5&lKk4QsiS&KO?zth3ndX
z4bL4P9T9NKny7S8?&%Q;1;%}Hm+#(SWe5yk{p{|ejO$f?5B*MDUb~5<hKZ;1je28>
zK=6t1yEg4fTCJOpE#cCAZL(-(vXqC@)rGn_jj=L|*gtWIZjCNDam87|b+V7|wJ&U4
zWfxt}+}<sA^=pArZ(nEo-)RxE*4Cz6iDYtMYPfvVLuOL!s&<)~`-0}I<ayKhhIjw6
zJjYl5H&;#le`A$ry1`5NnJvfk6m^PXHtby__KGF)<<f_|&%_tqe<I;>a+PL?$I-|W
z@u%d{j9;}p+1MERMCHMul<J!|zZIQ|lu?M1;E-H%vii@Oor@m(y*c(bEY(k7l}}`M
zF7MjD4GS^?+-#T0PHdI>BXV)#H-S~Nw2C&(t&e;4g;Af6=>U&fV~Lleh>FAVytT47
z73`-yy57Xf?9FH$FmJ^sCy@{DIFdaC;?}P0oAqZ2H`ALZ5(XTeJGzs3@49^IjCFY8
zrxg&dr{eY~>*31y+EoF1Tr3}$G-|hMhTRE|-5Qy6ZO((f6C3+7XU}St<zm@Zc!=dl
zd&sM}=Vq*3|KdWv&A!o+_n&=&+N~XeGPP|Q-|{9hHi%>#(#SX@ac5&$R>A9Y8-vyu
zq$x0l^1P}4!>&D{N$kynw@t2ZqqI&I9P^8s8Fu#wV{gl>1un-u9oODE+2OEWS7~<q
zEctsiZ@w&R5wm!m6eV=})^)Ao^-W2WPA8o*e&(Y2^2kD-)<y}AmZ}LNXY?CST;t?p
zDg2|qJhyv!t{am{mZ|KWRu%Cx4o3XHKDI4*%BvgxYHwb>Spi4Cfl!7cPvmmB_9J&)
z4irnNGYM;&7rrg4`@r*i-L56YlXBbCCLGjAxGt7)$lHVQ!#acVy5$_JrI>EBONPjJ
zxiH>-6y@}t-)ln4nnSwPW-K;0btfH?`M>dQu;zV%4@R@a9&0yM<<Gb$su|(hsI*{t
zPJ`o<kjWuuuLeAwpVTRR;I&lmx=&|?WR45U{4ex-&*GKAq}}6mVA~;$9w(6m@zRIc
z{!Y1Xo*l9Or!4m|+lGCKnSa)@R~et!G@Ku?cx6nrzIjBfr*YDx=R$`#KWnvqo^)Tp
zyl>jGbOW)YQFB`yMUrPsWYTY6Kj$gSznPar)EA$+Y4nRXVV2&hA6yQ<r<F*1I6e@3
z<+>v|_wKZ93;G%E2khWD!npX|?UvJKx^uo5-U&&5^`_~Am9c@s_lzKp>wZ~rDi4g)
z984o}JpSGZ4B^}&d+_|kb=xv72i?757^ckg=!LPXKHvW}Qg@Fs88VqpD0jF}vrm1N
zM&TOXqq__WOMbDRzBNau+107?OOsa0(gl&HUwz`5yT#qXx3AN8`ihHcy9zJr98-Vw
zrRubo+~XrF<T{>JvQ;Nfu`4=}k$yQnJhXMqX|Im8AJpDA+I~55>oQ~SSGDMz&yzT&
zDVj%A_&ol2BI+f}jF&AUEA+*8pYsd27qho$w&(Y<>5`2Gt)W*UOF51()>$m~U=$Bw
zFUo1O;WF4!a!e*R`0s&Pe=I(S*sr}N6xg?UZkl^((2UQw))v1g&At6G#Wqvjf5!z4
zh82ESzWGg<^(IJn>T$ExoRv$XcJ?VRoR<7^YVo!e6KBaUKY8~}AkW2F|4S7$t3Gim
z&z+p9-fH7%vO}*=Kt6bGg<@h$R&6S4Y;lj&yR9eRNj{inVqEk^>!y?Dp4J|Z2H*MJ
z^DmY!ne4$7^E~WCPRzct4uefg>$E1GE>nJOZlCw6ZcE<E-FZwOj<dQqmaSxII4r-m
z)HFCobc){keEz}{b4x2tr!4q#^7EmsEte;J6T7CHyuI!G{n*<orMDCRx@HGmIj;G3
zW`ccMM6hm}zu78Ihu=Dd%cN4KsOLCvmn`oy``~2Yzh^_X@RRA$_unt`_V74Zm%EXn
zrzxxWbBBZc1oMJ_dp6{*{yZ_ia;cYAzSAn+zRK4|TaC0gR~~EenDNz$^S1YgyKDEZ
z`d?9U-0KHlh2VBky>n}=Cd~WMoujbvUfZ5is_Fkkt~AWj(YbHrY5l_ec6;-V2_E?$
zwfNl@neSQRZF;VnL#W5e<4<<WwL`^s_ng~YmcBfWWxs6Htg6@N9>nK-4fAC8a}jrI
zc6AmGdRSrl>0M`q!Vdxc@?e7>Czr7A6F9|s)iZMKReiSKyHj{hO>1r6z0mj3*BHa!
z@n21s#9U%1nv@`H_Q!EK!<4nFPbOTne(vv=b<HVvb?Gj@GYQN64E#$zm?>{~_&K4D
z;rcbUhXq1g)iMsMZD``$+#Iszq{Nx>4MuC*Z9W(J#jnu5&1Ym_F>`Z~=yNTxnuGZU
zhq!+fC&V0k*Ybb$@|O3zwSF-kb6g(mwV~2!`(-g7M-jdb8$&-4#xob{YlC`pzUpmt
zV9-!i6*>KXW0*n5cm2g%ZaK;F)voz!?61IBmghJ{{N^(Ge`zWQZQTz}PMpJdcKh$u
z7asAm<$QMg!@$DCz$DC|F@wRgMMY>)#>7oeA3EzsvxID^xTrMKS6nkprdcq>gE2)f
zPh`Q;BoR&~t3@t5Q+vF-H6o*0G#pM%nWd!v>Ivti<qZpDb$>m%w2^0eSIU$=6|XBw
z;)?^OPWvet<h8nNZA9!_tp&YDH-w*zOLbnAb9B8}^wPa2)Sg6a5xpI|=4cr=(*zGa
zwH;gslpb_fY5!WgV)_Qx!xI06`Tep3xI7sbi*>WaSf(WRG@X8R^vyK&bQaGgIa0}0
zS0sFcel(T6y`dy<$irl^Z;bTg^ka>T4DaqrGfue2A*`Y)r`RQ;$!DF{SoQVi>#mFC
z%eMUpWYFZ5Q1y737kt32YxbP><vUt!yfrOcgXSkpY;O4OWbSi+!Q@5fe@M*U^(^J%
z{;pq>`8v8pR(Ws*<y7YCcE3Bc!m<0Akb=_%ttA1w0=2SUFAB4GQgE2{_G6!2LcS&5
z3ohNt>YjSduSU@Ax{bF+r^W=21qUQbbp*}#*KB#Qt+B=C!~zwOSA2_fw@5tJoZvCZ
zhe6H$ke-Ux@*bnbvWAORE>@9ATCX%SW7q$zy<f|?I(Ka<i4s=+b56y3r3Guit3Afs
zB?Oc;T)UFG*E(GJ=-<P{ni;?tu_a@zll~LW^Ve-mrYw@-7iKWrWWoIC_yLo(9^1}p
z+|9joTSN8Z29<3yE(8RL8T(wC;8FTIv?t0oCrmQ_Ywu;R`b%H+!^?#`t{<`!HMNk*
zWPQ0krC$4*g>q<Ap3)~DW6wixPAoDyYreXq*K7Nl$(m=r9$S;4{4uMD@9yj~EN}Og
zJzd>#R@34@e{;^(JsjsRn29DVT*I?>h4vOc(Uq%R*E@Z^H-U>aQ)jDF($Q(H$2&|P
ziWdr>T+b2i^XSoPt92H}+e@O~Joay$E0QuH?Cx}h<`}*IeoU8?yo!`wnW&hnWW{eT
zUzvN*yE}beM$~rCc?+~AE1P)BaM{d1@uFkp(pgneng5!&I<qzj^+Yu6IuSJ^KtQ`@
z%JR1VcQ;;~r=6vIVW-8HU59<Ov^NAZ#+XQkFlw}gHcV^`@yYjNbV%X}+fo$4P^My_
z;ePiQQ;bR=>x$58^JVyDwom%|iRb791Ld!uVke8{Dcm&3-Da(_)b_2(JT;Z2w^v^}
z_abdV*FDX>-6o4TPTjEGwd(e4`%}Lby^mwcSJpf4T)w#7Y1R5gKkxPayIc8uHd9rF
z_AUN+2JLg=$rHkq9tUh-w41S<JKp{NgU?3~O}Mc6X4Mz}4(DCnJ(Gg{{@?W0jZkdW
zmWjIlk0ZiC;;%-qXyr{N?jn_$hVDuM3&SqxEHa*VMbFyYc?P4fS=Y_hCl#5Fw-_cL
z+bg~?)5`v_f5-u6^`*A$O>C#UY|n_>yPsU|Y;k15A_wN_Laj%0N^V}bsS#{7fn{~R
z+F{XqPD{7%&1DM?;EBom7BuJ5ghK^yZ#dmmxIR<G>s@Dm^i5X{wHXFYY8I2Fc{&B0
z&8PToG+)9z%`xwa&XW>%Zeh#m+w>mo)|mNBUD_qy^}?ooPLmj$Z5CcrntsSbr!j@+
z(CWKtQeO`S_MEm3y1;Pp(B|XQ4ftczcFl^ub98Uz)$V?o6h)5%=@V5HGK7_OoruX=
z_}`Et^ogU`r!Eh<EBofJIohtosBj=>qBnc*lb%&ylrlt&1Shrl&1BPwOgiRq^svDN
z1r7_o%oI0c)u%mPlNuLt`MaFlbbwX&;Fc3AON9?hEcN8#jl4di!c9bI*2%efiBVjO
zg{CZ1IqI2W=ymDRBm3wBz3TP90%G(`R!GIQ<nmZAk8XN#IFW0CoMND;WTn&W341RF
z?W?+TGG^*xlY3{Ut_gY^Y7i_vW!X_z{aTrg%YJNE^Yb#F#n>m4AL2eE)u>;6<z)HX
z?;fH-OQNm3S4Fc}p3V{!QeHBlCqz=z*F#MxR8})M;B4wTgPH8AFE3^VEGZX!ncA;(
zy*aLKo4Bm-f3A$F5}Xqlp53zD;ib;V<;eH5;AO}Pt^+eTvRAp~b53%)SG1wNHLzK$
zWcuzcOtZqKzDd!v)>3-Nk#;A9uQ+SkX0BC6R#GohA{Q+2<6#Y+yI6tMTKo5`J7<@y
zc%XHf<KU80fo11g^?YW{X4`dq;u2M+pbTd>%NqeM8(HhPqNStv35i{16R8z%o!x$G
z(W<bCJRZ!FqKR8pn`dbW?B2`iDPsA{a^4D)&S}c;UDs((^1FMicUgCDn$h>8Uyrdx
zE@htfQs6=G70u_0t7nIW7%o!em)cr>K>4&%bo=I$a$=$Wzt-M5p_<6u|0GSSGo{gX
zlE;JvY?Bfvy)F0Jblgf;|9|XCk<Q&-nhabS=Y4<rM8;MMF^W%2;}(pLDCG_1ZdBc_
zosyl~BXCVLnQx=$dYMm?X3E%w$9^j@4CXt##Kki!lS5a+WomnDvSZyWkMu>doT|^A
z*j5$YP&&9rKsv-)QLowUn8WT<XOEQ%`Yk)EZZ7CGGs(wKLU5`5+8*rymMhVW5v_+(
zq?S49HU&7^=1jMY$tWu1Sk%6ZRnI}hThTJ4UpY7<PdhYWomP?g$*WTqZ}=$TFLhb#
z@5kIdzc=|jvDszmb#h|Kk&VekCX%{eeAsR@np~Wz{^r*Gz>1Z8@1D$FyL9o_vqD}M
zzim=-j(IC&_*><);)|JwZV7%`;9b4-|J-{!<}bdcpZsL4J#+jnSD}pC8$uM9M@?{&
zm5O*NVp;ohpH~Fa*E^df`Bc7b=1$;U`b15BPs6^QI=xG5HtS}vOziM^UdU|674K$z
z?uyy+cb7iBNh@dFDr)#4%l>!bcAsb!jr9?8yJw|;eUQTC5v&&YZKu<$^PI{7J2!A_
zzrZES%ynrJpH)WFjEt_P)(N?mFOvW7-C<%>7!g?Rv((D%^6Xh5yT7D<b$2aV{N>x)
zFgvY)Y=NfxiM%al%dPvo<}FqE;!^Z{llHeh8@{WDwIpt;A1XV_qd6(#|F`!de0xqh
zPD-5I68z-2TC45pC37Y#ocDX~#J{V=DY|=#sPcQy)&Ciug^J8B`FBT?`F?|5+Kh0o
zL>@c#AIn`YaaF&baDcJ@OK_(C+-);>6Zx%*CJEX}EcZ6sbTf8`)1#9&FMHh4?Q~kF
zmmR<8h}u7OE%iVC3U<%ZxQkyf9A1>gdf?gf<U6J_On%rTa!)ecDHA97@^fEP(B~du
zsptAq@zz<2Df29>+B=we!|Q|sZ^+l~G&$8Ck*JgL!v9si>W`xlDPGH_tz>jve1%I{
z>D0O294|K&*5DK8t}-aU?Mi+-LAig9@+75~I_H`$O<3)y)B7mq@uN#?9?#<cz3Isf
zzBj+_oH6~z=f;w~iY>pa>Vs8{nwL$_iUt86p@KR4v?|XWwmhOWdC7nF&+C%7*0rmD
zdHna3(*+%Y<!_EnRcf$3@%;I^N0)5!bh+9#=pC6JbipZV;R!x2m1n&zZ8>2-p1js_
zdl=W|)}?<y;HA!$3H*%fbfOI0m|S!Iy!tgM`_Z$WCtUtFY_$C)POAKS_gm<7y-eHL
zYnp=Y+``^VxGq218rNc_vwrKm!>xB-a-CM%F>R6TzJiTYxW65nAbN0v(x0<0+D`0@
zaCzBrS}!WGyC?ParAO|5S$kC4^?xjK=u%00<F>R-aC2K~*o$Qme^y&B>#?!n&~18i
ziL2Xj*ZVDB&g!NZvL~+>;Pl>jiHZHEp3Fu!wzc~HT5d+CK1S|a;<xM6ktqgKZa8zz
z`F}xa({rU)k8OV#-f3a_GQ;O}P;dOTqul?`2VD7Zk>lyte;-)lk4U#?<nBAKbnio4
zkMm546@5~Eo_@_D3>*!A3{+Aw?@jA()+kaGGzxv>KEZBN)QsNKOZZLZJrO+qQP)7l
z>BI)d4ILBqtp8s3$kJeicMF@<Ip1ey8`W*ruc%A<!nQVZj!%nG(?6k0H6QvD&s=EB
zJE8N4Nx6&fWy;Aiy`$kfGL~4dD$hFl=9iPfymba91_c?-eI;*~J57+<*m3{TBdNrV
zUwwkUl|0ej!KCN#PCECKdi;|29GUFKi_|tdpILEs#XHxhOC02ntE|ylVRIsx>(t>&
zKM$_2hc7H@y6f`)P|GZ(yE_(`Sa#{}TWuQlNYi8CzpPH32!3hKXDlCmzg=SrWm45q
zN-+=LaIwOZyXn2_q|TlNI*dXM%7O16>hP)DexxV%WP!&gttG)$$J!peV-@%=bn;&N
zw|9EhN<wU$?}fa5ZffPoUQKI$cF0=M$LT`mlT(I?DU-J7DRKRH+G61JEQmw%>XQ|Z
z{54q{xUv@6-+00i%iN#C)L@hAI<1H2>w5i_>vK*Rh}!b{R(V}<<9(&(UcAoRjNx#}
zlTZ7%I7FQ}bLJMqgOC6v^CJs`O#XlQG^5N#;O8Wrcaxlat~eShJvk686RR$>*O;w3
zWa<s$Ll-tCWwGC`N!RnpP?h<gZ=>}7$fc!zN8WNCKO3v1RQ>Allxz7XR`hY}GA;G%
zpI3G6`MTh5G5xN!mv!HquRZrMJ7kit;UmXw4i@puv-^xK=Q2<H*UlfWGQEgXCtgJP
z-xM8|sjjJ)FZI0CPPx2c%cmT%Ufnxw>dbDoo(qK*o^Lp#X;|vs?2!C>-{jxdRzBep
z*49e$bZvTCloyhl)R1@FH0#{Duovsj9l0CfdGyeGC6j~Pyc2dYvF5#6)W(&1iAU9G
zTa#I5$^C0zX7uH7vn2NzY%2Pqt+&u{@`Yv>&8q=_T-?@+>c4!HTXiP;#su|WSCm+`
z>S>6$ZNBOneK03xqQ10?yX({jp3GG$E@zhf=e7Oo_ijP<!832p6&$kGYcewYczenP
z8?(<oS$^_{OjU~tte7U5+*LBkG*#Kr{%VolGP_ncji#q=2c4h!s=RldQ1VIf@MV>~
zM%HSDF^xtguLL$v*gRV<$?|K-jEMdK?Ig#HX+GztHj7P3zdME1e@Xh$W$$ZB0$I;D
zg`8FqS{-9F(f#eYD3;eL+?hhWIY!4mYdtXz309k7mt-6ED8x4<wEF!voBW)`Ay$5Z
zEgC1EuPOR3lPaKiiIus)`PiZCh<={9mCJ2D><?jo=h1lo$`|zj5jUB&dLR5<ZhTW>
zF<o3BD1P{>dX?$Igh1A2(~ei|I`_Wb^AHP}+hxn`;c@@}l*eg-?%()U&r7lub7J88
z^3iXNTHo`@bCm6k_2pBq%YRSy{dPF#`)60BgjqaG;u)@;^J~@S`D%LJo#!`~bYM}+
z)<EmOr#99li7ewuGkv7|{Y&NT0INS!P70J;F<(zRe)W>X+5p)Ur%ju6Sj=5TCpjsZ
zT3^}xyC-C-!PGM?x=NehDBt~@^>C_4ij$|<BE`B#*}+qFO3YQ#M8p1Ww&yYl^KLof
zCz{6C5wSHXao3@Xw+@B-ra0D`iR}#1llyeJWNO53FBi6@VtbzDGK4SmvQUk64m`Lu
zJLg;UPH|-k^TO7QdYv7WzCp3d=CzyOv(`(bn2LAsPd2=~F-}l8d+YxrR?C=mcO3Bv
zOY)S>x6;~tJI>t3EFo6sQbx*kujxhAUzk-gbhOg0PSIGTY1pLi;Tpu#{KI2~U9h<0
zlp|-XrahPwTzg=nWBO+opVKw_l3L<_w3Uh}r|f7@C|3y<vtnh?Q4g<+dCze2kmy<S
zI<YN@0pBw7jp{`|DxJDGu`V;vetPBF($f#VYGo*?ooO+dmKO4HWteKW)iJX#2`Y>a
zXQ)qOR%1Bf{33tX4#CLyjjrlmi;^zK<lb=Mxv9HGXrV}`O4c{mCBZHk5-P94RJg>O
zFJ7r}NYROKob9Ug#3+2@tGo#Z1XDUH?7n`D6XI-Mcci4deVXM;&$T~pn-ouZ^}o3(
zqUZB1ew$Y;)3`eflpHLi-)RMy$6RpsuRZ6ZdBn1{l+`uJOu4eM-D5&lhGUJQzP7eo
z{nhCgw+0-!>hf=jihN{yWoK;q`q~TSs-+z~f^EE$BD|lOs`zk+UTgoYRa)#h^FWjn
zqbavrpHL`gMh>sUxjCO)(kiEEZ<-dG-nVjf3*+|oorWhSbgQf8G{2l$S=nP<Tvp(G
zNK4vWrK6ko?ZH-UrZvY++xCYu9Gj*7=AhD*s5RGjwe9}3Ry4AsEi!OwWQUK`%b-vG
z2H}%91?O~|b<ABGwdsxT+tNcUoO7o$)$L?;jJ&#{_rvC0y)k=yS0Bl}&2UzU>B^c1
zs)>&OnZDduYj5!&(N@~-%n#G7#opSNDrNk;U7fggd2%b{iF`9qd99z6Gy5OUFH_l{
zjz&9<vt3?uFm1Zz)<uHrc5qs*JG-mha>>76r`ByW?7u9s^IlnP_Ju{KnwA>+lx#Np
z7PIN*yK|zMQClOlQ#b9_VE16^3tk?5TQ|vkR-Ki~DYK~@H<Ucyn2I%XfBqf0=%b=Z
z;-`SeKbCY!on6DS@k9aNo&&2x4=(LGxOysY*^l@FiGai>X{R6UT-&v_e2LsV$;E7z
zW)pY2O8)fm{uaM!=ImI3;=fil^ZPSwzfX#N+&rbi=vJY-?(C4ag|?3hy8?A$iyq0f
zCY9}uIHCVO_MWm`&VTU}&i!jGRv1}44h_0}tVZ1P%|<1jbEb=IN~h|i?FcxvX^)SG
zvSo!Y_Y+PfnWQt1Uv9l<bLL`=<%v7-w-(gAKP0|}%jMmoCh5r4l@ZPIZ!h&!y8qj<
z(D|05%FZRmQMIzqwBy+4YMfqrvqZ1#(r*pZjn(;0b6Do@Q9c#)Q$AL?`{>z?+G-jm
zx9yu}&0k});3QM7x5`%6q^sw@nezCZ7F-r!6tnKU)td8FGjHE2SnBLzbm#irU2Bi-
zy6*hwdh^|Dy|w|aiwd)+t=7F$?fz$$^f5Qqzix#kg0aq(u1Oo)*ZxeYDy}Rz*td4o
zB`2d(snc|Yy1wf$?p|eiu9-9OobIOo&ySv2JJt4fXr)un%Tp~rzjao=FnTuOs{i?^
zmlmzie>9cPPDNm!o4~$PzWIF4yecZK`jZq4p0>|i{PK`{&G}N5$MutEOpe{@9Cw7Z
zb+PTv>F3K^<z@G*y}JGok4Z@0#3Y+z&AtZaGoBoa-n6WD>B;0jJWEdP{^ROUc;e5=
z?UU}jP<<WA@}O70>2$7?idfUr^UL%yVne!RuTPY{u|Dqg+Iz2J56{$2I;`KcI(0@t
ztg>5u<xYzh27kH9jlbPP|EOeda7mjdlU;RYUJKhs7O!2`l=znHyM5DW`tumIBqcYG
zs*WVD=o~L!jkp%8#D8-mL)Y%x__`yeQ}KqiN>uHG-~a!)_}nczH%aBeqI0YV#2sW}
zFEh>jY@>Zr>8n3Wq4&OOcGLFR)ivU`6u-vE@bBHVkgK_7!=9^Zhtp2p*V-zyeXqg%
z!%cA~6*6X;Okwz}fBW2lmVLGtKTiLno>2RF>ynq7=YOs|^f{U3_k6qGd*2kS_-DGy
z?)6#wC%v(MZ_mGR_t4*saW{gmDJ{6U=CtSpeVK3ikD53o6{0u=F049xpoy7vOXAcC
z2~LeUECxL{CL|s1=JZUxV-b+--Y4PQC8N4&iDy4kKwXYUQNrnN-IFGFQZ_C0=`oLd
zCUd#ads1W2l9oR+16Hu~%-+UvV^iv(OPXHWIi@}eVYu2N?UcrAx%AbP|7*S2+tym9
zDzs0C*)b)gd*Wq>MNC3RL>aax-fby=YUZNEaL|FXoK@ziM54kZ4$d#rbncidG95D&
z{=`}^as9&`(NSD78QPnkpEu2JyjL@^#Q8G6d%H;Pq|y^*6Rpjs^iJGz;Oy1{?t-_w
zRGcp;<yzl)F;P^r+dz0<%(|GTXBy2jtER0CQZ1S$=eci+UPE=t`vZsLRg_Z;eokn3
zDW{ul^eU()cx_kh`?x)Szs0alN!4%QmfZH8^+rJA1Wt(u{v4N<CM1Yw9=z<j;b!r~
z7PUZ&L?^K{jfPgyyo!eoQcWET+oilD7PTua(@03)bXB>boB804D5pgq*={?SvuVWq
z&tjHTnc&FQDRQxNo~8N%H#e>H1s)#Dwk)Yg*)t`;&&%)UoP-y9zlu+DJZUZ;{C?`z
zR<XpX>LE^{oD-fo_@zF*5*c|=^FU<!%k<|F%pcpIPwu<wBI(wpd2K?xgkV*mP;=tU
zfCUu~r!oi>&n%gdsv7t$-D$en4pzzNoMT@3Oiw!k^J}JMY0cj=(?vR0@vX(HOr@o>
zF61j-j1taTTQTLe<K}fPrwW7SWlbs%S(Ng|J*Q`pP_YE7_8O0Bvn3*$c4x~q5^ks|
zwhB3VK3ywvzl}T3IlZfL+1B^Ar{x=}t}PL%W|(#4!R9Y=zuxTJwdzEohf!BaQp)dZ
zzn|=vSa&Pn|Nb?*IHGO0-tl-fdBZ|8nKhicR~D~bvhk3Y%QiKaCmXe=PI;29%+68d
zA;r4-Q+i03`6)l96DDk$mag2HLj9Mf8lU4(Ye-<}VAE}!=`Lz3T0g&%^;+1x!iFss
zrN?GZi0;k2Ibp#>EsG0F9b;6kFP##^H9ghc&~3_j=7qYGVkWqyX}HZiq%M}ZW~0~R
zyUaVEX=N?Cb=|yg!Qa?P4~(D5KDb|Y|5WmtirH5RE4Evf+}?Oh>G_)U>6cmLD#Ey4
ze)L$f#f9rR({-0!Tb|6VRJ*p6X+pzm2A<rh)yb~WRZiQU=c(`7{#`GwDe9i-`!^|<
zyed3?&$3wb>CEC6HJ#@tYwZ31NkVYV`;P1EJTl#e+kPZD{dKx=;c&x!^*OoEwM}H4
z`}s{&=1-q~CuNeaoTrS3T5MgVM@W(0<^P7&k!PlD;xwvI4~(jgy(K0p#IU8;sP5<F
zrA!PB3<|Geq5_4(%wCvS`zmI=Jal)iQ-uDt(6iaRJ_|knJQ6?cav(rY#Pc@GgG1Xy
zS11~@oLkCJpcnPEt!6v-{P%4;q#0%ORXP{X&{av>5!T!9_V%!cNTuu1_9w5+J(5nW
zpA@?3o~S|E1J9Yi=S*4iWJgN}<00XBK0lUR-)YDhEHQnH%Zha^Hn|E6lceNcb{w<&
zrpM)V(P`hD=kDh-)=%KI3~ZZpVrGj9PtB(PQN0y2c35;Ori3l~$*bz$w0~B~#>od3
z%4<wkKE@dK(lLF>^eHzM3Fb1gHZpb0ZurN-!m#6AQ@M*Ur|FK}aqmu+y?MF7V@28g
z%^M9>)I!cIt-Ex{ZKHuoV)L?;WjD_>J~2?az9lWW&3(dshCr1j;W+|_h0k4AQE`7^
zqVASba^&z_V>9u@C--=c?2EgpUL-mDzKdIsaO&IhD!Q-xDl`_&z0lfXz1%q^s6%OL
z+miOU4c<O7`?s9F^!B)4R-l)gbH;8P#ZQZuGCc_jxWxCqw4>^hcZ%2Y?R#D<+){Zx
zIP;U9k>3m<p~@TCDayXT_7=u1F@MXDWbM1@WY`WbC8^0B|AXwq1sg4&rv&iwKb`aC
z<Odf|&&3W;9V!-uPI~y#Mm%%&0wY!N{ZmTyI4Tn2vtC*$?D**87a8oiNl2aXNGA6o
z9nak?Wj+^9YznHC;#er;u!7@6P(c1_&qUtP8<K2nMy}C!%pGsJ&;Rvy?&Q|x9@BPM
zEGqk^wB^+e5%148V+}nSPI;+HEfv1-c!J^9)D+LD8r6bJRancPMy9<~l4{7CyV7po
zZjJZ9CJQk19^J8@**E_6QQt(@qh{MCOi0c2zB@@{(nOB#o<mB?AxE!1Q=Gl?iG!Q;
zg6ScpQxvV=XI}1(Q#bThN$%)un6@}{R={cRgkF)QUWZn&UTay~|7m87;r~s`O>&=V
zA70DJZdE1dw`A!KwMf0&+f2mwCA?k#+x3~r-_;5`D|Q@~@_TdZS(B+?cCJEH07qWJ
zVUMC!mpk6;$oHhq@sm1Uam+&QRrk@)zOvRq3~bW9fiKlgM(KV%u}aZ*f>o32g%|u=
z6|+ygC@>Oe={i#s@z%r1J}Y|uigyc(CLSu9E^_gPZ|lLm)w80^Rvmw}tI|{R+93^(
z$vbA<6R9#j`?>Xt&Sl%*KaDHaom5Q!{P@7$E>p+c<^7(oUE)|ZSE?yaS~-tv(xkoN
zr^HvvhkAcHpO$F4;51``->>uox}L1lI5TH2EE3b)nzz<pu!JY|PSxp!Pm2^vt}dAU
z?2G9ChixaDMNI{wADK?un8ek}naCrpxH#v)J}Ha8b5F-}Tdzpwo_1KIPx8LnHkD76
zyHD7&p51OgIr2dVo7vNS{@s;4uJ<dpb;%g!PPV<4$((DcxaO~tn_Jj>)rFA@W<Q?7
zut-5{!>euOc`OY~Qywf;WZFDE_>xA{J)gn{CR>7A=j9zW-*>`k{bsdpy(D47MHzX*
zDto-+ibRdo7(B%}O+Go#47qJBmln8Eb;X;kiX|dqzqw@gH+M!0{fp9KU_RKi>GT7c
zw=1@^T<PTEJ(*pY?H)5(ptwJDJ!gza@{0KZX|L7ye7ovV<8<`N(l?j*bhsARO1k}Z
z(JN5&x@2x!G<m`Q=o+(g)qVLf$uiBw$IbhCEWCwImajhb*y^v|e5dM^Ps^UIn3ZSw
z;t99xta`WNrTfagjLI+A6?LgUw=!M8GU05MYe;lD-#39;Tlcb$g^xGzcZ!)M8ctk&
zCBE)ivG?yYzwg$c_`aRL|JYGwWpUp6nng@n_iZQJRRq6LTwH3YZhQ9R<U}i@z`rN{
z7)*bfcK69irH?;z8<ixQlBM&q{7aM0|H|2N?9!|}9tS3gb>1=g7ycDqO_=*BM14=!
z%Km0ip$UQ-^-Wn!i^PQvXmmPCawbG%7HcwAG$?%({;^4W;^e%I-zAStZ|n#cnp3JJ
zaU(-5UHYh^pxj68fHL_9<%0ibr7QfsRv9WLv@1B3O+@sWx@gda7?&f_LJL?5k0ktm
zCRtr7`1X*zbBAE=vBo+Rn<xX5oC(Uej5GErw|YtVvu}=6x~TuMt#RQeDYsB5#~^|I
zfz_c)3}u2_RhWy-9@d`yq!2k(An%95Yy&nscjMU$7`u{0pE8QGyF2f5_7-_!yzN_+
z(Ft#HN5-UsCc8J+^S9|aS`>zxmz5oK2y-qge<(iTlh1U+va?GiCVWba`ji+UT%MVn
z^qgJZ=AtCiL9N#I@^v=^YQ?*kgcena7d>E>nkL$`;Gpe2Q{`O|{(b2}-f1#^i$XTC
zrY_nnd%0YG)@9j?f<m_vt<E>8vwbQ4zffG{uye`2$pzfcRrkBNxZUusbcs38+9qml
zV$&#NSyc0=F-U1q^}+9bdSBD`oBNfe$+HQkNNZH|d{#MIUevu=VCI#O3>B~6X3@8{
zXU=*ib4Rdum23Q!_I%A5LMaLDUV@R&J_bBg412%bL4I47TA}ZRM^>N89dn#I<Vq)H
zeo~V!33Q&(;_Z~H7O3D7sIV*}H&I+jL9;BJxsmx{!;5qwYmo^37U6GCrM5j#`gTBT
z_L0~J-&~%k<*^;=yZ^a7K(ZoYMuk#_RBL(2o`V%ihEc9dg?>*FI^H4T*ET`OGX5Qt
zlyOJ?RS)6Rpy(SCC03rDbtP5y9gbg(S_0e?{_`GWee+RXBeU<SD5Hs^(6WhB+D<k$
zht))KrrbVi;Jmb3@N=YabCKv{i;3HM4t;96v0NpoBr`f8)leh#e{#Dihni-mnxmpf
zLE+@6Nr^ARwHOn88)vX^6^J(6oaOU*dgUY0?iC4%8|wwyN_o{3Og+1j)01MaYsj#b
zeLCb&&^~=~L}&WfIe!*QK3v+^5Z--%XL%KS_v!DF-OCC?G8JC4<*$v%`)U$5zc7R|
zvuWDmxSC5%M{mr2`KegvMGxy1&c(sKvz4a5Wz=(A$RZ^v#PvaVyJxK6R0iP*Ro?0b
zJ6Yz4Z&lBpR@xgGD;vsv&3NjcreOKyN(VkJ?Dq^d54C+dOLb<C=1jddWd|3jZ8xSY
zKgczYDRbWEqCm6g<3c*zp5@D(jGdw;y8N2&6FBSLM-#V;1%)OiFCUi5zHGmI(=<9c
z%;L&|yrsU^T)Zb$2=eYS?H6Kj+%UUXT%qJq;Q#B^)-MCP4@FFgP^e$zaDJHt+l`4W
zp<P+a#XAnh2hLKp=#=Ae6<U%YwN@auuH9{dq_Xs_{%z%2@)_o_Cs%l|MST;Quj-*4
zy~SvitNx7dMyyX(+Ah*rk|41E5EsL$S+hUssR{P6q%A0%(#*Lc!mFYx_EWU@!>Vr^
z^?n8lH+8z_1hORy3NSDz{$yblVPIm=VPIfjXkc<+WZ?MEz{(+VBWr<nGqbQ(%!v)5
zGh6ssr!>r%z;n1;LU@wUhRlTHt<sXWHic(DTw(Y|Z<Cki1liR+2Fa)9cm^NXG0UR(
zmB@y?1se0sSpUeFzgoT{km=}Oo>?WDfsHQHxg2JFSQb*qywd1yRq%qawc^#YQg(DH
z-k2MFnQO(>bJHBES;hB>oM=moZ4_f~%Mmq7jA5+5X!hoIxWQpF7GWi^9T5+XHA&y*
z<*}*!lyr*abit{Ut<Mg3Fx$6>aa1KbnlF*$ezRvoaUA0kDfKzk*FN5v6P=)aruXLS
zTdk?0_tLh$GW*cDdP13KL|N$dP}}{>rp^mqRoZGkl|QC$PxZ~|4_I6Ni&t3haQb|1
zrv8+y9kxE-m^WT?Syft@+xq?UdA;3v>u#+5n`y-5J^8PZc&fDI^LtG^W-k<cq$dW=
z@{8pNNNg3>G7He;l??035#>z@^iWvTdc{5KYKXTjlZV+&jU<7NiA^tTcU&&Ile=6w
z#O&LS3%(9qstjRnoRwGn7hP~ysg2|fos!S<R72J0|6lb0|FB;kmt)@aEcDD*wY8Yu
z)bMs`U{ZbK4KD4DzfYG{G@NvijNiq1p=~DD)=Sbp)nzLJq?hhFb|KAJa8_XQf`!`x
zI+;|g1Qsz(b##+llW4LiIY6ysIzvvtR!#Q>&$a}*PiZvwQd^U9a!z6F#-iZD#WVgN
zzvI90lGR<UEsw0GuG^Tl>VB0;tonoy4_(#OVaNV$>00+_@2NMNce-gWS$mji>C>$e
zchzNrHu0&;t_fMC!l281u}Ekp)0NAoG`RZM)wg{(7A?GMbM6f;^`>=Vn$DteSMrxi
zXm(w6WD;C6;h@OWAgSz0ZdQw2%gtu!`!2qc)E^nO!b*6r&A%PW;qv7&Q;PiEQiD&1
z%$)N$Ad^+IsL%4@kB>z<%F<INyUjK23JU%hlhb8gee~{nt>rfq&e+t(q?SkX9oe{B
zH|*HT-h87Lb&s&wZo%`#UDpUmtB4(&aijX!()6S{*EzRc+pb;t<|EwH5s}Q4-VxDO
zb87YVyuzsebB|US8GFVTZC4A?*?d{8Z^_oG(<OHwP0-e>d@{wDMc(g<<lNBy>4#N*
zJezqgL}t#G%(|xzNfTFFS#K36m-l=lHBa<-@Aba5HzaF4G{m#judHuvE4?P!&M$go
zMS^^q%8MiXi*|fHs+YFZb-nRxi6>tjW9_-V-FO^x<-2E_|NI|Mx|8<(yz000%4CnG
z|4%gDsdL(&+;*wn=$!gt|9{_K9e-c{+cW6?&kNx@TSO)#UA7c*+aNBy<fev=_Ci+c
z3`c&hYfW<6uUu_^1@iO$TI_Aa75wE+uz=NBZnZB}o}7<M_82W(X>e>dPiIQU<a-yg
zeOU@UobIJrH#%m;bI*!M{_o>Dy|Lt#YUxbxc&kTyW|%xP;>l#!UgRK}$;nW7jA`1h
zHD8x}Ie9KT_@c{SGjlG+)`?+hA`OzKXY4d<;>$>@TrADH-OXZCpu(&Vr=*TbxY;hc
z*x+#bamk|<!m}#{Cul6`Dik>>6D5)^x+Q|$_C_GT*QLes)7P}yo2@wMm$ym&?~Nt)
zPLcd#7hh*_DYCxJIqo(0!8*T{Z)Pz>Y&P2ym}1Hh?UA``fqr8wi-~^<hk<E{cdKg)
z&(uj5nkE=&-FWhJD)&tnZM|z;+g_wr1ur}w{7n47d(qVz$rIu&9-5z<ydr<jB8kHr
zu0)C$J_!_g_s7$Q^?{$^h2S3hhl{P*7yb9I<vTL(+2-XAEL`X7xt1<qlofLnUv;64
zt92pcQFVuxnoByFMCSdQl5W4=_+l34lts*?D<-V^D$1I-QA<@>`NWo_dH;5#G>INr
z;A6VNG2k216pO8yevw!DTQUL^Ohp^?UmfJyu;*p4!A+6R5rwAfzOJ1r>G-NgM0lHx
zqhPu2k&cBGGXxq6ulkvtx{%&tKC5-YLc{q?-Y@5DGiMR%;_(sbkA5Q5YWU)P@t2#*
z(yT|W@BKA(eU{E*DfZLhfgK+=$ewLsFj8bY%5`Mpo5D9P6H6iv>#6onb<IiRy&Z8n
z>)Ph5p7gY<8IcP!oVHAvyCzd{I_J4{*S1_0a?NHvDR<_-P?+MjuQAmxoacCW1vGu#
zk)mdK!ZEOF(v)Mq*VXhdU7nY=WGB-H_rT{$PW1uLl;=08-CA%eR{HDG^35_Q-m?@b
zo-Zj?STj@FiKR)Zh0`H{O<1CxE2zzE%0d&dO5c4>hiuqQKPp{H3AFxq$x3(4<^`|T
zJhy!tdH%B1j)MYa8^lz69&$!U2(xY5z+klJAz$&2!*YCuZH9Xid1n7Oq+z$QE0`yV
zcX!26Rk@9gS1l5yjz=7ADNXM+T%BywX0oN?&x#3}ryg_({Z!iMRdgxF>!4clhGywA
zE2k`<6U=BKuBJ2V<kZbEm+Pjq7^%+Gj<PddR`&C$LwvXJjaMzsruJL^XRQ*6PMjdI
z+$7f5Y{i;|h3~H1fA`PRpj<gS`vUjuqX9=-zFs=5#mW#DX%wZO^vzp@Ws%?EFz>m{
zsX87nS7tWM+je=TK|%7?12+PSXI+WvU|AI^n-#X)>hP+21#9B(W`*SMxVB^2)^)LG
zUx)s-y1x0H>4y5b4+ErUUt4_5WF_mWH$nQ_HqLEbld?4S*1=V(ktb)qv#QBH`5<Z2
zj#(~xng@CF8h<z|%A~BDdU5UK;^dYUDVtk&&G4L8e$-F(*4NGZ5^stdDMrOJRh+Fj
zttFs!pe1o<ipSj&CZ#6V7E=`ok0a@xOnNWO<Nk+sn5_JndUAs^7t@DHJ+3wrB+DZd
z)c*Tdo(x|na9poUVPZw)8RLnqGq1-yvsnJ~3~R#YdHUa;b@lFas9E=UzOd{I5Bpyi
zMOS~;xp3!(-+#+1vG)>}PrmiaU*GH$FNgiY1No6Fy{1kM-=f4<COKvCnU}t~vz}@%
zKfN>3@4CYyzR7V1UN={s{M0qCOK_#1uCgDmjkB2URyo16oK>DR+Iy}E*lo^Lh*m9G
zc<!{9UO<=H#>Uh|ywmfyTW(es+~hig&FFz=vIW=rb55_~d#ggX$8oRkw|yO~+u(Wp
z&bO`4E8nS~{oUvjrLh0J?T6a^)sOi9|2X7d_hI_{*2n3JTxZwUewiL$^T7ZA&r9M5
zK5sn#TgM^s_l5R-|6f<GW4$VWb3=}Z>$aO)bFW2yx;=+wd*So<p8js-cf-}*IV^bn
z?Kk(5|Npnwzn&(*utMNnS^e{}29_7~`zNY1C2}wMAR+B2%DE|D?1#dCG2y9~SlQGi
zIG@L{3CA-P2q?{{NKh3x7Q&QbASf7KDBLd2JGs(sk*>r>&Rm5A&cK8f>YN7c#fIOT
zca~T0`rd3BUK}XEX{yfZA;IZ>qb2Y`OTdX1e~Z?Th}Ny)tq~Tj;T)~eH(Dcqw8s5t
zNjy>QuhANRqs6(rW!G{6aWTKLsWKM?9gU_s%DSYMWwi4Nl~<h*C}9`=X<lx!v7Ie~
zf5tR{Ns0WW56jiJ#QF(~U--X7q5L41N2ow)puqNvY11t#a<;~*eNj`Ll3tW#*7H5|
z<D>$kiJ|*V3~P^7Y}~9lcaxy!m-0soy*@koNF{PBn+OC6uqw2RU0hzv{IPq#N9}QT
z)+^6z_oVl1ZD-kYyyx_do(mDxrzBX<Wb|D1=nYKhJ*&ZRO@rY~2g{v~-n&0~A9D2F
zs_5IRE^wZ``m|ZJ>&CjhN39OI<{ek{IuTmGNTq>$VxIVp{!c}DYbJDmO_SDl5@>O(
z_*>D){>g0b<wm9h-eQj?a9(VTJs>5+;nLd_C#1<B`@@-Mku#HEq3Quv!RrRkk18^5
z_u&xn4g4&ww}{m<Az}aVmP`!>h40lC|0P)k3tA5>pKMdvYW=gt@#W-G+b3^LpW?D|
z@`Dvq+&Nj@Is2Fd7@uiOWpbQq^>Ru81LLz5Q}3LZ8o6_-2Lo%o=CmlyX>Kp4IJ?&^
z{Fd;ExlF&*v4AtxLNw2OYFUY-*Mcn#VcQ&;4o-j5CRk=J-1u|4QDB3(;f$ho!M0A}
z1tB3^50w^+3(qd{>Y0&d`Y~VZU`SD+;9PZy^&&zOf9UPnSh3`V+?v2ylfs3!i3m>*
z*HCkjUf-hEy+yb{m0`9c*P`tLTNkhj3e=cftUf!v@96YiX2!lJ8FTK<m~&$1oTE4A
zFfmR&byF^*QZQm?-wN~SrzWv$0=VB#_x$pG?vE9oJpUzxyLSq3oe+4QnP)Vyfx|-J
z%}jy!mh(&|&im9Uw#I;sD@cHOnxM&we2$y_-#=w&atib9oXAjKz?`^%^>du^4-LlP
ziaJ5{@1bnUl5xzlCY}$;n0rz|m_v9{Ab(Dvfa*rp73@8e(~67}rvzFGXziM;cWRNx
zf<>X0a{~kzBQh7?$yjXhYijJzIrlginHZKtSuXKVSi;me^`=K_*z<(4ucbcUQc~2X
zTbO!<aCrr8E31r5t@UisQR?W76z+H#!}_AE_Ju-6<g(7r8O(*tY9u-wt}bN?6yg$6
z`>!dSZ6;lKkW1S|a_S59(r-d~lRC4PEnm>7S-DC4ossjSM5nI*D@<7qR{Toql3eM1
zh{b$TmTAUPj~$a^zm^s5DplKlLSX9x&ZHA<I_$z%b_ndV6uxj%;c{oyis`Hu)D&JF
z*ITUMc;{vAJ;{0BIOhG7X!w7<>{mvEf#duSFXz4JTqB;ihLcm@rBsZS(OODSarX7F
zkDJt`xk7hLZd{nSAckS0xa366Qv%bYG<(|wyA$KtjaRN@6ckQeC@@j{r>SOR(Lyt(
zsvidhWG{+ce5{l#Fxf^UdvaRKx~v+mjZ>341s$vwNAFmYG{JA<CY_^<!GT`E58KLu
zZ}FF|+*nb$tl;~`+M6+TmYd9yW;|cE?2MLh#;#3m2RHRT=2rMJgU^YpcAIeM|6~`@
z7Y(zXdvzB{{8f{g=D68tVnJ@O=CX>FD<$>IgN2uT5Z`1eyhTEbr*NHIp!!ZpiLDzr
z_eu&YnAIGaS#zX)DzoA?PQ`8azt^r;U%jPE`zD*`?VojXzRdmpWA4{V-KQ@V*k^2#
zn7I8^)b@`n=RN<q{jH>MllBhF!c_JdTel0&WcF$dZq@a>y2em&LAJ4>P-fS|QxjL7
z^xm0L;4xc3Wk($EE&=w7aq1@)a2X2-e_SV|uGH5kVR~EnFQ>$(V>^-qR;j9&Xm(9A
z>Jp4FPkfgWp8sP>;Y>%F>h=?}rysCtC}D4?=2%|E*>U0b(n9vwx)swKCB44BjM=wf
z@1iIC%KtZdnk*G)Ub(EzaG6=tvR7ilI!z*l&y^<Lmeams-myxvEy^XkNZm)Y`*U>q
zckAwSHtFo5&3;omSr$4?$eR887^@6}@aoK_7g`x>B!vQm8y8thi^yy>UMPBHmB2(Z
zfvpByJ9Z14njye;twtuSI%)yy=2xm8jAJKUsS^+Jntx=q*_{5<Tk8)=tvMatP<30*
z-e%45S8`tj%D)}p$~(CI)y^ZO+B?l2@e6vW?N4x$<xv!GT<d4Ec^RYc%+vZ0ye7(?
zVCC|ve7s9QV5Q=eMu~T<=Au!8`Zo@o4`FjUt&r}x`3u{spSQ#mYy`HiZtLH=MEu)^
zL(<x>A57a}6|7LTRQ&(#jTN3TQq@a29XBy$?6Yt@`TDd%_sw80pLshhgz7EI*EAf-
z59E5Mz#v|nr^B>QJehxA0{_1U{LWXGF(sVjaa^`(jX+GAwfWc7Xe$AM2;ttSnax$=
zt%BXNIJTHJb{S2~->xXlVRKyaAm{wu!VJz~I}aZCcxjgL!~@rUxy3&@8)_}KE0FES
zE%PacT*BWDy<zecep0tIB-mW$yv3OrC$r{#?b-hQOl;7RefvI$eGA}Pm%#sjf`GU(
z!`wB;o%hW96uIN)o$cRrB<#;9tzV)y+hF#3liiUq)_1go+02j4nk|sdbBTGP^j_z!
zucdY^>0Vb@B=B8TP-W$=wc6`H6v<2fzr~ezaL1YroT=ieR#W#(;0dYLERW?XchcSa
zWT(R4?<ZR%8<MN?X6+I9>cDN|D$KBA`~I9%h8Nr$&YXI1oA0w}!q*GuciiA&4!c(P
zkUx>@3b*6w>x=enmpR$48N#SOq2~C)P7ki^w(FZR_8S*2FP#+Yxn=F_2+7J<olUu+
zH?yo`6l{Vo9FY8Y;H}q6lY^Y6&S>_SSo0`qsL6WoQ}m74EvD|s|K<CYV^Pb#RBpeq
zXnR%nEvCRD5?ACmG->bM#I@wvbh|h6&Mj?VIMKiC4wt6lGVZ`@Ob__B1#lhH6<C|Y
z?|Yl?zruxY4B{VWU*wro$R@j0apKy}-ZNI(!lhZ(lK)Rgr4$@v-6asKaF45S3xB8J
zG7HwZ2Q>~akzyD0ssARRUm^JOsl>+BN3YCP|0HHwdy;EW0QZ61a@()LwZBjOy>m)=
z;e$0X7orZ`I$yf)yF0hl_ZByf+L-}jiy!pV%&xyRTd#Wg-mL*L7p&Q48gSW^az7Q2
zV7;*=I`+=8Kd1ITxO(L6seDH1H4mm2+&!J~aFf>UYensPUng`-i+s|TrtWmR(}X4L
z=OJ;ig-`t7Eze<FC^>QMr`{(&b<c9l6717uh}ByllhAd2&b{l03<A0(xRj(93(SsG
z-WssFFg>|r$}zs>Z!d67e6Tfv+bov*@0<%hGDqI*JXtP%#5i%yn+dEQ|0l36x)A+F
zCgz{Qp#TTHi9Fgz469a8c*wbt>q>^)UjbHu1)PoA!pyc89yz`$EaI<S`8@I2HD2HO
z>+W5s%X=>A_~Kn<KAWs~ZJvaF;nPcR6{K|~<a95k=?E+88Z29F$|~3-DJlMKY11p;
zjE#ZodxMW^846j{u8w@w&wTOu^ECeb3EX85xE|WQd+<(-S1#nuvlw&LBMm1WTlbyb
z`e*6s-FvxiIA%}e<XFJT)VDG86xRa_t^#wew1~Fk0?*eUTDEU|$RWV_<Prau3pHyV
z#B5h!keT#4^A^8Yf75UC$Eg?i)&*Qsy|{Prk!ycXyt?xH+6i5`#SKT>6s5dhf0{hY
zM&v)g#w*v)Ws__gl%xbF?lQao*<+nl4WrpYnNyDpWhOgTR56|9T4O$Yt$uf{#MCN{
zep^kS+v@9AboO~#1y8;&b0)v1%q+q>_ROx6Y-^ua<*~oN%9fGOZnpRN-#sT6r{wLt
zdy99{_Tv8+E<P>W8RhuU+40*F)<tjkau&YGa^zn$;l-j4(Tf^h7+&IEqUF(@+7q~d
z(>9d*zWZ}I9%1<!0ls+>=l}9Qy7~IXqnPiR7s?){1^S6^IdQh5agqps*hOApg-5?+
zf^JUep7gV6i>z;)uxH%CeNQD47fgN2wQq~SN^e0`O$Nd8`=yiaFOL_Ntg5)$*Zlvi
z-l^GZYV-KF1-yIszmEM#KHCF*f&33#_ww0q=W|EdmhXQmbb1qCo(7k&nYh)y{-4jU
zuvFx!26|2O^m1kOV=(Lr+gry})!8NF>Ag%RaMK!L1uiF*Fs7`li(S_*XsFw2#Caj4
zKy-@6>#DDxudf%kX7o71u|QOl$)zfA(S}PRsZ$$D1&pq0MZBNSV8qnJwc`2v6%0$?
zvh0YC`m$ng*wdVwzn{Nf#Zaw2U2bjqmh0i0JbLr~+|tj9+>ms-PnCQ6+sIwTudfLm
z$jZIFIQ7&oQ?7`HgoBOj-&L(<gj_W07X6)jW=6<^qjRLB)m~5k_|Ryn759gnEupiQ
zPJX^=RmPPsm(2R!N<IBhr76B?&i`kZ{z>hAGi`Bj%KvS**~|+Vm#I&lcP8y!)^696
zAJlfgTv2j0cggI$t67?Hi<84Nx30c2ogsSdDei*g<Gr(u@^(#mnY3*0Z0mI_J!za_
zlf~LqCb~2>+5bBt(W?7lhSjPCaW7W3&s%@vpl6Sd%f%GYX<Sk43w1;`adrjDh$M)p
zc$``-R(9o<XM@I%Y!4oR-YQRScGE^Bp{4`QoyRovv@W<tEhyNiq<cfKuxIa%UuVTW
zJ^OWhZkX53EDgq658th3=KbTVsoQT=;JtXA%Li}0<06xFw(N4b;Jx@&;!=lAhm6*W
zEVfW=?lIDN+{9y*x%rHl4eJl%3r0Uwn!ByJuWtT-!I}52@C8TNr@;(%@<%_PbJP`W
zFnMG)_l)854C!+kTF<0PuLP*tp1Kke|2a4K3(JQ5l@a@AUA6nYh5O8Up{_%{EQ~d}
zCdT2hrNuWEd)(|=6ByyB?Z{a3@ws?)-o~SR;cH5nZOvm0oEd_xnsw}gH{~kt-upt*
zb&5c^AS;(j+nE<i>kSsAD=ayXoIbVmR!oMo?zMxf=a+8c2oN^>c3H)=jN{+-X;to>
zk7C)~8=1J2gI}>PEnN_BF`?Og<tF(~rJWu<<&&d|l~q_0j?Zshm>|v-ctwc$gpW;5
znP&B$jmO2SXFU43qO8I$;AB+Uy5-Z#q9Uc7${w7$8DX3n?fqYG<^HH_n~$#=#iHu}
zG@ALwdq^I(i8$HJwQ(uOY+<8wI*)TU*xj|`%G;l@c=l$$Pi$NAcd))#@_sM#^vP3>
z9r(ItikXP{TGO`$d*5rv1ZVrMTCMM18gYMtvDamv{!epEx8^JN@mzY6(>mp9Q1UvR
ztNpR7BDSq)u3hPSs`rl4U2c}o+R>*rKaC2%mh9h<$69fCYIGaBkDa+kX6saKPOnw-
zs&cqEm%hrpx!zvaWFhN8!++6%lLNo~KJPpE|5tU*D!otb^S(x}yEmU{qQ-=&U#2>}
zKCWT1G4A&*_Gwo_mc#~d+z%IMa+o5*!4>Jiv^j#!vG3xd$RMuEAt#De7EaapZ|V}=
zZM$mwlzWMKHZv{<MFbq0(p%Em?5^pr8x_bMHo=kWR$$Pppz`HsY+o5PGKMBL{SaR~
zXH`e4q0%Zp(>bbgsX@iY3$waDOP`Ys4DmC$R<O!rPFUr+;?0vnBR!Jk_n$bak@6ul
zu}?|VdS;+yAdBDftmD%b`*Ie&?wS>N>M>jRo`<2$h0U6M&)bSh&&qAfaL+z<wX3Q^
z)sZF8({_u3;f6IzLjR>%U5qA95b0!M)J{I``pdE3GwosRtOf^>V;}bQEL@@-*wCQS
zFlm9-Mja-m2-jm4Ng`D#jtog2U0x+Sy~|t}xGr(avzQ1P1$p^sZgo)An0kr3OYz{8
z2md2aKc01LOW20yOfRP{eJ&x-tqyNhGzHVW>OLRZxqwx-x=}r1L#JB23Tuw-;pK|8
zWukSvlq&KvTD9F4?>JJ*7C7etQ@+Q=Stl->G_#kHO>h#P(Y|SIe%9<247(a!7w3o+
z$nnkzJX0}c0#n}-^{K(0Z{03jxY;V2e<tMh2`1UQDSl@Bf4g#LWE{O?AsFKR@T}PL
zOF@r*d@#RY?UnUuk<LuLc8%E^?u2KUvt1L(FuJt!agbJ!)4DB>Y~M0!r}w2#iTb0Y
z8L_cFGB?mZedlDi$OnA0J&ZOdRJEu)j6VAB)$~PIa*P~idW5^NTsSST_NhaN?4_^&
z*q6n*I5Et;apFJsU-PbS8=0!ta6HxU`Z;&;9}kAehFR`g3^vb^-6W?Zq7Z$0iyC9v
z1)Vo4!BMwf>X_u7+$Oo_ZProed6MTQh+4(GNcv=Q=ebBZcZX3(QDEsEn~R!xHJeZS
zRc1N8yk*jPAW>x5y0lpVb616~PCC7!V%KFAuF{BYRleI|LYtH}O*beF>UDf*v1Zvc
z)yR0sqem?^O>f%rxS+E;M%?E?>*Ltd8ur|3$Bd3S9=>)sWm(7sm%2rh_VgT@9(m&F
zvA&x+Q3r}dB#kd>?P6CjYzY$P_|fSY!Qm_N_~`Vg2l>CZe25g#%+Ou@Mb%(p)8rPD
zBWg(z?#CVk^@r+o@8>IC5*_q^Q%d-v%Tw3pyw;n(Xxc6Y_HEnOKEGabZEJPj&k6C*
zo-3}u%GJPBQusj2S@J{xlir4#XJcM4%-JoFZdR<|Z??4Ow$5`iyXkDk7NvS=F{U#8
zaz_`(XncuzVsdHXrqqiI41LnY+IL@7-D<H`km2pK>aDNB3`6d-##XWSKEJM0(&d%g
zI(y14k*7XtrJ@)63S+LWT=VE%U&T$W=sLF5K@27p+YROgo|&-6`h(b<>{(G)#4-*Z
z^b6u%EU>XbsBBT)g{0Ieyq6ELZ1C#al0RkZh7IkXzMfd@z*C?2(?hK28o!j{g~(vB
z0~yM<`8gb;6~1N{Wba>MyH1aFTWH(%y!*LJ|KC{T`CxUdwo<Bx^MhsUc&nZA3Ma2#
z@gy$Gplw&heAcyYiMh8PAJF{bmaF_}GFRG?WgD;De{^D6K;gMP-h%JLlm2~LKV$l{
zurEoGX-n^Io|t+uXIgKuSX9@Fx;y7{R<6&e+qPx#l_rtJ3+`RHRW9hf>o_xu-BIbd
zi8c+_npRhuc*UiEI^(18VDn4`0d12@r+&C9x;$3dZNz*`FkFs>Y0r`y{|!!evGRFp
z@|Rt(j-1H3|NPNJhrsTp;G%O2Ou7%fs|h}^ROe~=sn5Q<kN&aUdS`R>zeS%D#6H`f
zw7Fn>JM(PYtm4@Z*Hl@Z-^r~yV>3g-{l?y7af{zv&0z0Z@%xa%{}R7+74E!(#lbpE
zRae>`X9x<Po#t2aaAE9@yhyL8OG^!Q<xPmw%&=j<d@6I6g~i>F^2O&_?;31e_I$qH
z@%@KBUzhl5>RLF5Az{_k^(u39gLYOf>tDKZ^|N={D+F8Xt92QF&CIvtGVb8ta7wJX
zTlWu>z+M9`!A_>gMeSFtCcI|sOl+E1Q7s=R!P<C3JaC3e;0lMTh*lki8ErQNxFr`T
zaUU?xoWLlxBTrgUFHkr6<l@rVHa8O-*sd%t>sa{ahwh6=jSl5?%1d^5AGFw>XdG+F
zb7%72#SKmxCcCtF?6gg|UL9T(%sFR*vyzYEs!JNXHZg2{S~NBA<o?qic>|ai{|`{+
z{$|}3;ko$;hiE+GzQqA^j%+x@G;ywFRc~0tF7}K1C!M<rwK%E^_!&Pi2~XJj{e*Mi
zq_vVJTwxDp1U7I_W0<rxm^+GNsp(7=S?>uGPjVP`wD20b`bsco1$PTHZwugBwz)xt
zb*2&T5l2yGzSWjoF;|SLA8m?bnXUJGQ#8Zg2ZgN<B{fWfoy@v7r^eW(*6dAh*_+XF
zEVIS7chj+_uQhJ{?s9tEWRk?)Ql+6MaD2VSl=-vtly*5ExnNPGvbArQ+ol<I9+RbX
zMKo<(R@x}9_<f=8)WJg^y4Nkf(5ri(SMx!G(H|MdWbSS&4~1JIKG9lX6?3h#WP%z@
z-^^I_-}=$v$kiPyjAp-W5Qr3VXfSqgwP`-^!E{QYXm0ba#SyKNzvaptJJPMWeNP-X
zxtKLDqLs@>iX)QMk7vOK1~2X_a^+5*+bR~mwA|sLvg7KlgSC@;KW=D^;&58dtY>5(
zqr7B?%9<UjXDp5!*n7SEkS5RRpF#GUB%I6M9KU|xSYo8ztq)G+b9fV#=j{7<NTF%b
zFX4vwMkk&~Pt#qsu1<t!UbFLVkIhC5yIx#A{J^N|r?J?v6z)~C7A;QZcH5-9d;>>V
z#L_s9lZzXsnZID+H;^+<=r=o{URpGDMn{M7LiSZgM|~ukOE)NdXz-27Q9YKu^fnLI
z#sI~g8>E^x96hnQ`@ix|p$(1`J|=Lj{dx59lRdh@U0bBw>W$nQoaP)dIrAjT)++nh
zmDzjKuN+Hf;md9D)`~fO{leb$8OAD?4wt!{{`$w=LDJ6V$uUbw_kBywI|(nkuCdZ>
z^Kl)OzKR;2rb%5Mj%VI0&s};z^H#>xSy5c9wq2YNT_$^^zGwHfrf|PA7JELSS4)X^
zwT-FED}}`o%T_R2FD_u+pWtH8q_O#xIoll$VIPTyDja1GT+UwEE<Hv4VTo?pEuUpR
zr_&o172fDdGU>I=+MyJBNZD8KdzX{C3D?G)OFI`hoze6@V|iL`Y0%1>D-L|*)|EN`
z=(gL|mMPYntcM@=?loD@SbJg5AFls<eNLN+$i9_0(`uru-otCX$0A`N@5&iL{jU%E
z-ehr@BjlJfb6Jb)hK6b38mwA6tPUaETRe<b?L2!j+ThL{SEm|Y#x1jC?nr(7aWpQ*
zuf}OoqRo;l$-SQ1D@=^{ng@qoE#7?H`h519W4DaABx?FIt_a=6aDM5I&}|<yKGwKL
z{=Kx5<M{pEPCre;JU7nyzU5enPFL@wb+4+zzM5!0xZtmQ#{cEvYnLqMeB2cDWy#8=
z9v4{tcJV5BX!7!P%(~#+EfZnAwXD>%YYul0&%~DueZ5nZQXcZ{InW)(;3=flt*v?S
zKn3#^$$r772L-DHQ*P@o6FDIC%3RJ?anlUB_yt1$?+XPfrD#|?9$M=XBp>9lNx=IX
zS7b?!%Y@p~sj*!et&43<{lEBJ|Kj8Q)%o($h|{x5Z{AnfIcM%wr68wA7nFY49AB{2
z>5f2;LCF5O-CI^@aGy5hJ^m(ak4pDRiB>ZPuYRAS8yjqPJE-_XUn!e0eUm}Nnv(}_
z9uhK?>^|e{_w38ev!9JJIX4?TR<_fcZtQJ*{=yCO-RI0MpUd21yZ^=2{i=NZtFFeH
zc18F4@0i0q+cn1N_RR#=zTKUR9&VW9v^waxXw){9Ynv|k|6}RPetKv9hJEK%=S<<A
zI)mBco`L&<+TM@B7iQmP2q_F$V6@9@Ma+`hc7BnjKcjEG(AdiR)cOBvn{A6P^oh%g
zt!0o~r|DI%8|~Q=5ng%YdcY)wo_UJA>s&8-t$lN2N8qKmQxCnXy>w(N_taje39%R6
z|K<9a%KasG|HXrP-+Xv@LT@e4^*{6X*d_-19NWkTD>Q5rnJ<er82^noZQXfq!^)YH
z<GFOZ*PQAx5x$vc6K^8L7i^u7ymRh`1(Vh<SXFq+)w#udo3L7sq1Qfx?)49RSuAg-
zzij`wV!7xuse~^^`oeB?zvezPbPINMvtH>JS{f=l?O4_ujm(@ooYQzeeto$5<)zQM
z++Vk@i!xpGTUJcn&Hv%%bvyo?zVET<>|TxPHP>>E%Dn#EWA}31|I`aVwe)X2nA4Kf
z{r`IC<04tH#b<kF>h`YbGWGV}YS|S2HX+=8GS~a+{g*mUz3p(|*kkx?>ZS!Pi^a5d
z+;8yu%n{UMD${$`OZjO;-&39*Zk`Q$_HI0rGVyQXk*iM@lwLX*aA%5W(t$alFRYbc
zthj4>D|U0r)8n#p8C#R@6fo<kbx%GUKRHNQ!;I(lhwh1!r_5u$e%&JZ=2j;TUS;!=
zV5L(U@e7?ys_w6E&|H3ZN~W9D@=e^!Erj+g&|0G!y*px|zM;;eJ2u<iZaQCmTT;(B
zd_$s4m@!-0-imp)d#AeHWq$Ng_vS~_Gm8~sZts3NSM+|JT1<4R<<(yAg|<njc^|d@
zOV`kS`eQElv{>o?%FmuoE^@D$;;wfrqo>S7ciWU6p$D4Zj^Cf0^fhPdeABM$wu?A#
zoc5i3;d;S|JEbe{&)CoOFnQ&y@Zjn`V<)lGTedJsrgW-4x#cB%=TqR~h$pSmPVLb>
zR+fhxJ9l{2iLB#!y6W3K-<Mb0%Q75;lWNpbKbhY5otW`Wm%HES!L+?GYvo=);ntKp
z^lDnJ_v}+a*;g(_UOg1s#^YRQX_}jvJZ-1%Zp&sF<xOI}^Q?Ppu8786>@_WZ>KB=+
zJw^HE1is}v9%wCGwL$Y-x@G#-qszD3s(cD~9;CYBrB7&>X>O{`+s`akM~*&eI~Lid
z_c}(^f303pU0d=8)r=crcUI3k^#5+&CFOl@ukO9MCCpA^-HX{xr@QR({;<Sq$Cwl-
z?tAcTQJKu+`w1@@<h1-IUt7xSymVEESNEAKGhLP|HP7z5^-=G==ArOSU2`2{yw0tO
zXp4UJV3qKSx>pZhMR=W)epeghXLf&LZqQ4i_=7W??tBQnruX_x#A%gDd27o)%u>#?
z63Lrq<^46<`=G}Q4Ku6XbHa=@TJsE<4{;Xi?E0|pXR3nHs#m|(JxmBTisRD!p*L%D
zzX8u-m2;QXw+1IPu0KD6J4=l(>zw|!7r`eTyR5TV&34QU)HS}eIl91culZ)<EAyUf
z_S-5POF4XJMXg*<K;H3#UvsY;$Lq6a%+kHHT_@?kv3jvTU%aNxoz%6VUu08ta${or
z3QYCG7AF^2ocheEUYxpj-}-{rCi`FLamZxf^?%OBdr3Ag<=mVGkz0Gyx^Kwdifo?K
zbz)}_GoRTD=g)DAc|)JhV@ruvVAZwIdp619$*Fw1!xrySgKD0AS}iB0q;_-5o2-?Q
zukWuYy?Nua@9D3mrWtR{p3Ss9eLODS+Wx^#vw8gIFCBVw=veQRgBF(6HXpyAu<-Er
zo~qWZ`C!gIv#uP+^(k%%4)z&$4qL2ru#=x2|65pU@y{-^2~2zq4?<N_Z*5NBmwV(^
zSVnqn?sdKHn5kEJ{-0xC9-1SQp>*!mhIi$UDoSteD1Cda*iV}8{|VcY-3E5gW?$p1
zd#5j#m%#3^dj`|317$0B<|VHz+@i*Jc?HMv*sa!aS|7jJYj$l7ypim)UG~B1rLS%`
zX+|9XSN3&dP3c$Jg2L|>4a@H<KD%nQJ!y$}h4zv|65qeR{q{OuKV#mZ4;@Zbi`E~S
zH0f%vW#PisM=yRq-P2ZO`(IBpA~<J0gX#*U7X`_`J1$?yWIjBF>(1&JPCx(Cg;Ckt
zxD}4C=ThbV>@lnCeiLs(j<w4Ems<XLGA};a{5kZt!lt{|^2&|Rn<qcqn0j3Pvrbg;
z-Fctyt92jToBH8RX>$GR)1EIT_wKuvwliJ)<(x@Xjqh#0i)eOgon|)G{QrmR&i+6D
zHD;Y_{`U2D8=spxx99)Q=XTDS&d0YzNOpI?gR<@1e*Fh3LwV1{a4qBf`v2<BrR!54
z1up(3G4JaSUZ(k_tM|`Y!2bGc)7Ovg70D~#Eq$KNbno@q#t(tsi=x9%v>iXR);+Rf
zO7;<k4nHQgBB21ch69bvyESYgE+~jIhKK~M$@r+`)*8*`cTyvvOQdHV%Z|Vs&mNr&
z<8tTI32pP?Y*o=Zr=omdSwoO|Vbz<LpVcokuFc_Gv2;V|YBOoivobeA4qWaQtevJA
ze5zO5TyJ4ePXt@G=Z1BB&sbhh;$~mZv~1nl8%qzn?htz#rgLzE(%l92$*)*vZCAg)
ztM<9pnd<NQ5C3<?Fm0B7^TW{bc+PU8J=z~0#_r`8RXd~c!T8Aqy;!>^hqfgr_eeYO
z^W3a*I^HMh&$q;OM$QGZC2X65cFgW=xX0wddH?R_?OqQLvgfU`c(tgo^?bjUiPeM^
zzIWK}%B;Ed!7O6Zx_2A4bd=^u>apifmFl><p!(XU+e~NQ7#siV{wDfv{`>g)jK<m}
z6LM;EpGZiu7ZomW-I2QNZux)PX^}3!pSn+U*dcpGvdwbl!G%t<J0%u5zFeyQFj-h5
z@IeF94Y!NkOe;br1#CUI{6YwOVS#Y;?w2C50XB@+ua_{{gsXUoZFCCJ)so$Ml0`6f
zs_4`x-X)1{jc+-^Lhs$Y=oRvxIh9qA-DaivGKSD?hr&}C^h_eyS8@t0PyWl9@I0}S
zPer_?@a}eXH}QKjXSkO6SxjgUF_Z{&%*nly;X3=Lie{JNNzaQN%G=BZCRi4_Epne7
zb}P7AuQZ5FJ!kE;sZOi8HYCUEM42=?$t+-d+^crBOweEOlULVdQO)*?6PjgLPIO-~
z;Sk$|-HNUgOU*qU-89#2D4C?IZ0a_x{Px$Gmvsf!rm}l>*M67tci#F*C@^a8l(Ze4
zNiC`YpSTj<ulRYd`{L5+eZO?~Rh>K)=(fqv<f8xNi_-=3a%59h><iL8^J?}Kw{7w1
ztZ%Q&O_TZTF1Si9YSoP+ay!E&3T>Gce$ny2^2HZPvs5&X6mlBGx-d!Jdb%fc{eRJF
z!AZ5cnf|?7w<t4AS+d3O(k{z>lUcjjd@BPgp4skAmei3IiQrs1p?c97RlB)zTP>8#
zmd_VleK-8Tf5$uGu`%DT%vgLP>d1pz%s-`X-FAqK39O#oIjcN=@zjf*nIZ>Y-9NzW
zIbjpS>b#ckQ^hY<Ki)0(mFLYmvAr30HgL6`n_XzjZaUZZJlkpk$>1%IPU-qge_|va
zvGMu5ZTF@s=1Z18iMk|Pw68r{d;1#g&Ds{c|8w0cgyPyhx3XQ`-g5D~tL?+ZLGx@=
zcsH{>yeC!~wn};~r;1iy<LnptcUg9y7rdt)y`+U<Lh=@?)BmSEcL`hXtJ@~*H$|->
zY34h%JxQ_tb!`*<LR}h@6<T#Wu61SHY!VSxI9h$}s$6)DZcgY)FHNt8X-5kj^`4!Y
z<(L-K6!xP}e|PrD(_SA$_kLaF^?j0XtW|;H&d4Kr9YGw|7pX7U{X-zHUDGF^%VXk>
zuP*KK)t7JWI&iq<;fg|wMG?u{%+2392HHPlzO(#@xSQUuBL#vF4`uDMn8Dd{#8fNj
zK-U%tsSQ6>tR1w%Hhf4IWc{UL#Tn#L%oeodu1mUkbVjq+7mv0~%_F=q3eIoWsH!e%
z$kqwYjNV!B+#`Q)Pt0ve&+Q(U=9v|!+jefc{&e3_n*$7n8q;S#5()Qo`+s6Z;WRtV
z02eDG!J8UAp#q2Hy4;>juABNsp_w_<)#1vdnYCHY&5vBB-`H+&gU2ORV&Z(K7v=)2
z5yIg+xWrnvY~E^Y_Cle*r{j`G;DJb^2}-uz3mE#`1i9zE^qz3$*7b8wc7HEesJZ*K
zf5M#$T@O1AvQ;}fc?4%JGkf!H((c=`K4(%-cLs%239h~%#;$anapLAhTU^poPH8mk
z4}3D?^cRf)o_32Z91CVUbn?2b(PU&i5SS{x_pJC$C$_sPpOm<A)Los9cP%h{uI^BD
zDbMU+*SxUfu0a9*+sq_At!;vAuI+LS-BZMFK4po&@t2S=X(#uu6Q=f6MaR!wcf#C!
z=l@pI+G_1+%_LjLlciPuIx}6fB&FCg1G_ntC)^9r+E}^sY*AXqlH)#I6ZRE!n<O~*
z=9oF`*IdyNHtlM+nSA;Fb5GBUs#|i#pV(lre8aj`^Q01(v`d#53Y1#hTf<@V(VdGk
zhq3kbq#(P(%TAqfo_gOryLntSoMbq^x3c}2zh3XTprVlAg0(5Di|1#|sCw%@?JHNv
zzlNlk2|rspcP71b&CA-pM=T>yx%z6tKegpiern4d19jY{L<Sx%Oxhx{@zEL`rTw$K
zUZ|vP?eJw<aeN-njDE?)!y9Wp&GCDiCc>4wDqxu_S9WetTHljvc9qc=ysx#cIV{@K
z67n<V#FPX3{%agkahvi&Ld5&T5lyi*u6*0B9k{l&+dz)T`2qKoNf+Eg<g{NdFiH{K
zpm?TB?`@}9=(a^&Q<l3ei19pg%xprbpXA;Yh1Sp+PtQq*FG=6J!rdl3{ot)K&mHDE
zFTx!<FC2LFbzS4u%lCFI30qyze!6U5@xC1a_vXzoSI(6a^?WTc(;?dBc!l1}3Bj#`
zGlE=Z_Z?1im0QlB`Jmh&>&gTM`Q|x`w@kKZ`s1sxRQc7sEekD~r%e>u7I5w4oE7Df
zoUixA{c(P0w!-n>U%u^I4qctz_;kZcL6Hr2rLOOe)4CxhJEgZeDJR8v_p#iwtNoLh
z&RhtIVyK<Qk-%;(&=R@5yZ+n%6<;%a1hqq-YzWs1b^8}4Fn5lZZI2kklKMv(zcd}0
z?)>r8l0I~S|9ytqhD$G}-VWf>`n@EAO|2sP_a!dQpbetZE7qQU_vH4BlIIIIRCVuA
zSKhAJEuF$vyR|LB=)&@)FBQDCQzc(qc^H|ld1vRU^EJD2_3wT3yJ`FKo6vXWxJg@|
zhE@Lk?l1K{x9j0o!#APYwN_uvoj=zsX`ePLRH!|2!GVwV|KBEWFw0dwuX*HvMBq73
zkBhegqFahnG%f|`UFw>!;rn5^@~5(AdMB>hd9Zxy?<JjG*C!P$U3u<gpXr0tx#kgl
zJ35?Yw$1gu)0*{oW>LMX&86kNL2j`wGfy4uIR0Owbi#`t5_ZcEZOTkJT)~?1`NR_O
zhO~t`YBN?YNj0e6&6s^Pl_AaQTZ)f`Th+D?2KLzs(leMMyLuPTY&pK*1+OrtQj@Sq
z+41+ypQ1QKkLd<XYUJ4?ciZEX^3UbP^>#XX5w7>{e(dQGe;M<H#hFj&!AgsqdHZ&D
zB(Ywv4N{Fd|F!vhs(f(i|Cqh%4ogq0Tp1g%;h@?a*$X+-wg$52Y;D_q<<{L;&mAGH
zvOTWN7fyti9G9KJbhU>wuc0+ufVHLN=)I|2)nDXe?_TBn7rS|<B-geC@tt?LH!a}O
zyZz1a_K&+~R~X!@=nVgP_QoI6*kiFroMq0Hb#(RXh%LX;Ryg7R?5Q!Wk~iaBWp0`+
z^l{Z-kP-FhIl_K;%8954+zf?Xc`iK@6aw6Mg8Xw1z4M*mbff98=fRU-g{$^@pL=rV
zOXaB<Gg!C!Jn)$_$G1T!=0<CSfp2+((3%5BD&BDSUh#Xn!R7Xn`)3{R?G*^@-Fmj{
zWj}M?LuRSzZCB<fSGAN{b_m^V?bhYfJT=dCO_vGpu?*j%8JZpLJaac!K62n-dHS(i
z;Lv@Eqv3kz!v6+_c`uAx!pOSUcNSlk-+$SFo}NAitxo^cmO>Nvi9H8bgx)V$(ZZ<5
zs(SY<+Y}~gqY%xBESj;MvIS?R?_t?_p=0ZmgO_$3VwYl(w7D5IV=B8;%ljq&*`1HN
zM=V)fR`6hH?2$8xEuFRT4;TfDcRt|S<jecdr6khDa8bzPfOEbp=f4%a!|!x^y``*k
zj+<iR#Pt=E4bS>aFK~JtGTHsi?8gBvLK+TC8w8)G#Gkq0=5>HQ^2(X12NW}B27A9@
z$$EN^vG>&fn{t=7FwHnQxhG?abQmYc#YpWNM=nWR@c5GG7odEm<m${D3mpp@<|NA5
zf4w{P!m-u7;o>DH?RN&ahpDW6dg^KEe2q<BT3J(e+!fKeH{(ib_~w@p3k>J#olE-8
z8>#tXrpPI2*EjLspY+5P_!a$eVc=uao*;XvXI6G$mnmDQS=bRVA4YR0m*B4BYEv1d
z_qAx7{O5h!(CvM|uiQmJ(r1CB%cRdQdBZzHo#$Nq)gv0k`y|q=pT|ymv+nKrKN56q
z-PtJH6}hk9|HMW4olh&*@-Do2b)zARn`(5wLwjY8msr=N>IwZGr`TTped;>ro&(E)
ze@9iyS-aD>x|A2NX4<;0Pl(jqa>zgLrko;2z!r6>Yt6a>s;@m5xdM~JC%RmgxWaaG
zp}KFc(zm;lS4K+8OxvU5=BzlSDNZnd@0{d@NBVc4*<XAb`Aqu%#8kC^-bJ!0CsQWo
zd`Z|Q7&}{+FM3CWNM%b6>#ZLwuE%;N*33MwSJW?=nQnGBV=<?L*Rk1Ue<LLe_!z`e
zCwyv~P$cI1Akekr|6M+-_WTHC6TagqTBle`lU)ifu)mDbyz)3RY7)n;?y7>s&?h`O
z^CsRdZPo0RIH#%dCE&=Jg9{#T%I&-IKxE@em3zxJtELoKxiy~Xnlr=GEg)r{)AY-_
zlbt03taH1<Z*pwE;ugB&h?vJTCl!Y!S}qI$tDIO4nz0`6nd(0I=$Q$9!mbMUeQ&f}
zn!;325o^p8vFe@5YOBT10*+{GIC$}j^Oh4Ql=d!GbG`4iqft|W<*4Vyrb!nHIJG%T
z7cNNftlY-glEUU+&>c~9^mrj>-a6k|PA{0RrA3~4q?+`4>$9`pe9np7nbP5R<(|h~
z0WXe>1UchIk6lK>^R8y}@wF^I`Tw=+k|%qNro|bmPWab8<LvRkyDr5h>@0~a>K+bn
zPe?{yP02a5>>iKy0g-zSpDr}dI>K7ly{`ATL+r`0!qv0yt#tghjH@Yn>4&5DOqRs;
zyiU%!(kPP7SlYNsF?Q=W*V!+Z&E2Hn)_VTwjW+3=6xTgx&b-N~$~m|<$Yp8SbJvP=
zw>><0bB=`FIpQPZ5`3e_<=GKW37PB#S&~L8e^(w+JCRj;ZPgYRmrtDG&3eb8Z88r3
z2sjh9;-_g##2)t31rO?@W_X9V&MweSE`6~rZS}r2y8i>uq)mwZz2m7`PGiR@&xfCG
zy52Zd5!yA!pnp-X$X%XS{x2fB+eDt8x_<fM^e&(OuN*vHIj}IkXj;dRkR|jZ*O$q)
zsbwKkSKq`}TodzRCtl+y(8x?&#lrZjO;@6Ia>VhRt6|9;Dq1VgemKA#RkTcXZTqn<
zPqs(7TMGT>HqH2QslDOLrOgwM-hH+#|J2o`Gv54p8|lxqaIUKQvMp=xORU=c%JoWv
z=yr~ns)B<T_}D7aTwXW6+LzPodn3h9k9)O{Kuy~DVjkADn|eeF7&KeNHfv5VU(z;Z
zLUi<m;D1gTM|oQ`*QjyLZB3fN!ST>X>0N*Mg!ZksmX$>+-41f)Y~RrFcI~!#AGEG6
z;`KVyRC4t6t@|P+Q_eK$Z(5-9^qFqr$0wPMf=joosF4k>&Up9d;(zfaQo75+w3fB4
zV-Q*UE3IJmTai+Y#s9BzbQ!3XK4>{`O}*qonAkIpb(v}Hc}uPDF7B?m8ouKoOOt`;
zk2PwQt}MGIzu(e0Wr~7JkLW&u%SCsW&3^P!=V7K~#rs<?Hn<1y@l9SBsitxyBW~By
zyp)imJB=<sh<LD*ac$-q?U$$AEkv(hp5!=#CE}b#XXDhO2|wJc?cA+hGIMRNI+*I7
z){&2@;N3Ef@mdet{oXkzdpWo6IX>maL+L3@>@BISM(_8{Xq*+ua`NnjR@JA9Tu+o&
z8F1|mTCgu^iBGG+%d>77AKfl=&67QG`jxMHj>`@=zm8pt+<18&FB0Drk#CkhB}L;u
z&-r%+7oVoapLo$*xK2pJK{(vSzWCFTU-uhwqvW<`xqWFXQc(+ta%sA#x5jN{>6An7
zBe`q$cx!~Rmd|PbTKFJf$<55vZpS0%^ll$?JHD9VhhNsQ<m3=v$vaMSf6DgyoRlu<
zK0QU~9N*o;-zLOecyN5(hHq_8W7lMU+vk}YA+ECa^1PE16^r?O3}^W%3bclNo~`)M
zzoCQsz1u>eY0Ao4I?^rmp{y?suCMPo;yI<g{6Y##unz;L%w`RNYf@WQ@g0e_e&(OD
zkl}h(kjFu`72ahEs*ws$umAaU<wVbk2u_Zvtycn4XWen+IC=AFK&PYu%dD2v_206b
zf4*{FdBeo}e=*nQ0*-LM%4zBui&NXX%rc9^)8k*1-#BS)o_4_RSfkCIrkws@>x>*9
zKc+{%&X2Y&RlNK7*+<?s|LUZ5D|kLupVjD*^o#duWKubJQA}54zn<5Y$Xwp<w|J&9
z9($hX#jd^GB{hoMS}Dj!+TX84O+3Nd<;7i_T}`?I&zv|JcFsBS!DdESOSf`puZU^T
z;g)YR&MotD=AXUfYJOgR*P)ni6MU0J(wVrPHwn*OWaIW&ps2f}xZ)-6A%;!+e5B7F
zTd@7AvEDwJMRqNcIj7E5ef{@O<IkjdOfKeb2UIxkpO8%C^=A;UdZ|*tpw(3odx0Tz
z{Td@LpM!DSB9RkzXqf(AYIV^_HunQht(e9_0m*iusVNJ1erA<2CK*QmT$5PQ@@?$~
z%lF;&{!@EH){D3(6gG-o_f0)jw!(JR){Q%#<Qd1`4WHloV$s)_ZM{matbNHxUbI)U
ztqiy_VImXzn_{M%O&8Byod0nO!!5R8-F7GT&_KDEKvixrafRZDcDdXmuLGv`1O#bH
z_=lFHdH&zZ;F2)))}c2yzLdqBY&DI4Kjn1W$!-zT^lA3%za4vUS1xW^gU+T*_rUF)
zUxk)9Zadl9;2UN1{5XTrQ(Kn?+a1w-Jp5H6J`*=P+06TrQ5i7def=BWP1d*F>y~g^
z7j0*nX~_EP_d8?XjTI%IBLYvA-`Meg8AolA&w}(fe?6|2$udQ*eWhiTVjv*DG<KOk
z!zBOC1FfyEZ7c1kI@<aEbPi=za7!w@$!fHpy~^ij^ArBIc^|mS%Z`2jAkqJgqg{C4
zNtKh`Hmh5{9eUxneZ}P^Hx@n2VzKQ1y!692wIdtc4WwKiiEr)a`u*`vjL@6qYowoU
zD=XB$mimPyL!!{`v#Pj3IfKNmg~6W<SH1dE)gpR6FXKU+&?~Qe2?Z%Lt3H|3PjjwW
zaP17gv86NTOY7XLt2zrzI?v2qGt*Dm>&rc>yk!ZltxN5tWKVC*?9Du}ZA!=HNj54I
z4~X30V!txYF^463LgnV*Wv9b;6#CiOI3JsrcBHlVfAnS7(r*VIiSLw&?(kZ$G_`c!
zoT(Y@d2@DmMYDA8pZDsm|33!VnlnH88QiZwp1GfSDFgpYF%BE&y)8VJdzzH4G(9t)
zo?&)mhS`QwwlSWiXa1gk(7f_pM`o1HiT8iyq&;Ubc|<WCxF|M-Z~y(4B@QW%?u9E|
z=DJr<{KTMQwTWKV4zC|SXTGs_o9}Yu;`eP5cRe2~2NyjucHnrS(vaZT%<e54vSUKx
z;ignC$0tgzjtuSMywe^wZG0fcm~yKiM>QekWRs%1k<ZNn)tSC2Hbye44I;i_Ea}(o
zTsf8E7%jhag5ZLU3gW%0W<{DCH@M6+ik(n&gkx)W!z!j}LT9?R{!i#!9d=M^jny`V
zo9k2L`&6qwymZ`>df4obSHZhOy(QB3TC=yEac**(Tr}fQV$%aoAvFyT9Yx0E<6JW4
z({vO%TV^nQ(G#gE=xR7UkF#vrrJwH)o|p0WetKi(R)$OKxPt;`gp}sq-kv{Sw`yaa
zQxoU!BBdMU8PfNU%;l56H+_nmo|lNgmK_(&J)81Ym9zRhQs~^BVpi02qfKOXKkHdz
zpZJ2$hrYfRJ#<{t^vS`nZEti{R1dBQSXletx_<!!Z_ugztsRHo>|rsEdZEo0+AN^o
zuJGox)3&x#0hXM95AiN$(cdzm$LItDzYX7x2MOK0%VsQa)l_K+<m617W5~&>^}qSu
zF<H&Y3Y={s7Yt3m**$9Ec_?zLrQ6qm`Rw9zY6=-AH0^e;F=aF}c(L{Nye|`!Rd|KE
zy3a@n`R!;tP@=K0@rKBXg>UZ2oH%Xyyv0N5#WH1$#@p8(d1wSxeJWDbch6kEa8{bo
zf_?pIw=ypnG#iEN^yH5^6~xlMYqit5vehM8YhSGCbq+LKx@GOcNzSD^`SyDkZs0U?
z;4wU^kiF*GiGtiW$B%tklCw~rlWCfQyl(bVwWb3;VGL?Zbqu!ZS_LmMHWW2^VPxdI
z_r=!hDatEu=N#k>-?}P8+gJbLDyQ#CY^MeEuD+VnG<m@@FU9pdLG!xbDLm#e$_r$C
z)w|+|i2Z-184B<0vif%<^X}?g@Kr`^y5DD>)TPt@h}=5w5p?3gjmJVKGnVqsi)s{j
zB4qimBYBFo+|ATB)xMn}eHJVecL=hx7^WNTu`c8mjV&>Blj^*7@i@z*4h7C7Q}e7c
z8I>j^@n|Y_C^4<zU9?M~cBz2}Q~RVBI@UGM@AAZ_8Q)IwY+WGg(8X!<DPyuu?CGNJ
z^Dpc)PVHKL!sn#l$Gou9H~0MUy43xIWrzR6Ro*cRO%9jJ6wZ|1WUwgc)7i-jCRoIG
z6wl#rS-JP3ty1bmck^`5hK{Sw=XYHW|KH)6HeFXEuq}JldK-7qRjwC3+_ycoyWh0`
z{Eub_rjuV9TDPkz8pZrKyeAeObMb=WQiEGB)-LJLHrH#t@M_;JJJX}zR&FZ@`>MQX
z$A`XdUKZt^=IxojIZV;TXRp51-E!m6@&CKSCdj)caxdw)?QNpisxhHa^q5!g1|Ft^
ziJA+O_X(bxz;Z?B(v)r!xtR@HL|kU?kY$Npwot}>&+_TJ=1eF$#L4d-_Cn0ukX^@A
zQFzjX%L>7<=lqhJ`bt<D78&+(+I&}<IM-s@BE!FL(p>|k(~hNBG0t$e;}h(vS!dyT
z-8QFN(niMJDM=&ffxErXjZWLFi+xd74vU%Z^tX**Tu~X>>XN&mUH4TW_q-<?Wq<tS
zlzw-E&%7urMDW#wuXal(%zAmDN=fMd#Lg)zl*Cwl-CUJtF*S*(NM#0bGQCJqXLZnN
zVhLbM&T-dfxw%!NCs8DNg4AS_gIy6^4%}~6xH7vNvRQB~=#78uky@9cY`lymjaBQ?
zM6cl8f=y{$OC+PxGsLBCm>luCH09Ye1EKxrPO#19(M;szZBk(}WGOl!#4u^yqeD#w
z?|gMi)ZHpN^aF%k425nqh%7kTx^HDJx7sYGZWiXX)lS)UKbK5e)KxjLW8pic#wd@4
z22C2XHav`wjTZ9{5t<VABdK)pilgrC7rUx{G>P(lan(K$Ji+kV;RsF5Oy7uCflXf~
zsyyJi8Yb~`;S&p1OEIshab}1759TB=eOo-0<$vDPE5|L=rc?)Yurxniy`pZ3Tylh}
z%;JDcVH=eDwoYKuytT1)cFrO3SshJR+#dE<PE_I&Z+x|3(u0X%1&<=qg3{g|`8;!X
z!ep=5>e)g|E^Jlbb#}%zt_53XiEPdjOWCN>a@zW?r`p@OpR4=#a?ETpUg@OuyeL<o
zpMUXW238X`E8|rvJ@?vfajuxQ)lf*vZt07KTDz8aKU#Fd@tQ_MnM{jl^bJqjlHJG5
zxKeu$t!1yA)g)=X&QIfqp_p_-chIDSFJeD8YPN<1ANsO1Yx&8SGlh0;SX93+W2@`o
zCu)X~w|uXi$X@(Kwa4w`#7Zv%F3uUTS0e(i+Hfrpdvk*+eCMqHp=B?Mc($mxHnJLU
zb$LC^WZEF*c#G@dYRig4VcD-e`bx$7mY!TxHZ|JOY@^Thl3cgB506Z;x*^#7hS&RA
z&b7M&uV-o0KAYdv-P<*rHDqPXcfDk*<^4B{mX%tW6>@4We3IAvq*-I8qR+Zu7mtmN
zeryT;q37nOta~%#lmlDKI>T%G*DRjk!28LTl~cqy*jzzhw%c>X=UDAsW^Rl(7R&DI
z(A0Vxcxm>INAX2@t{&!Hmke(ut89Dmc;40W6IrRMZNggI{<XivKe9dxZL`!$F*rNp
zqTbZ-eKo5X7JXhEz#YKgr~B2(XYKkn2Ja1D6<CE9DFnxFNETU~;3~4^0$=_A8Kr7b
z8%pc*_qI-u`?0pA_x(minTX_xP3s;W+1M1f==!1(f%%IWbJXJ(Ey%V!vR)!Z(LDdn
zjn__Lrmtka@06W=X2+ck6WdF?oEW$5JOAvo!YZ+Zl7beKB~Pm}|DJo&W23z3(}iue
z+k5+~rA~C2Y5Y;xT)V>gP{#Z}Z>}k<eR2P&n%Bn%2Llg&-QlXeX_H`d&7NfY!owY_
z!Z!u&I4GKz(JFmv!W|Y<DMv-AQymq@Sm*O@b$zn-V%7g#&xYzvd*iG1FB%p_G@U7a
zDHCY5a`U&zh7%TvEuL`5=SJcL!wpW{7J&=IOct_ohwF0j$DN2=x`1_ScGJ`~S2pWL
zTDjlZdF220D#vG?>EXKts+Y|?`z5<)*#Z@{MB(OGuaAzl!uLegubIkBnDkinSl-ER
zGh()ThFNL8PN@`iDe5dX+Id}6&~L{i<sa8%Z266D1?L}((9-lcA{Hgx{c+pF#R{x0
z+3Lcx-o&MH@)<kUxC>ak4qIElZ-y>|X^`o)L*g~tet!wz5)8LURZh)d{$9d2qo+jt
znT>1c_i0UwejHp?_x14opA%1AIr_irN;TKAklLqp0h|B-4Q{PxSy=b~ht}RDvoA2M
z*_yy}Xx-X`x9(g4TwEL8@HlaQ&}7Ixdrjw9lG`e|KaEV97v!}BnEyYV!m#wFhW3gY
zomqS;dotE2xE;Q=#DVj_jnhX7rG~a6w~w7Qlk(QNq}%M(@$63K<{ahNM^5Gkmi!WN
z(U@?Ar=fXM8&f-vl66Jay#Rm3H*YWR&`r_EvUNK2P|qpo%rdU69zkmal$F*!YIr@x
z$C>NRid{JmdA!{;qSih2a?{LVOZQ@UK4FpOCZ%V)l3Y$5u=Y4o8<WbsEn({^?yWkU
zGp4vPEqaiB#4l4rIqQT%VB68X6L|OWJ$l))^u`VT9KA(rO(q^vT9dWv*uf<ZLMPS<
z>*^djr6+WN@69bfrp!jUYb(TGF^c_NC||Qs-e7}R{zb9B3&l7WiL*Qqdz|<E2xn0b
zi}9Hh&p*p@%^sRAdNT9cv%?%pv&0r{{NL7c^u;ryJ-N;6+_g%Y{obq+?r0W%rc`(;
z^Q2n4!jv<g&rK|v@IvXsS+j3T8su6Rx+#4Vd#V<Z9-rj?V++T(jw6a5im4e%DlLob
z-Z5|gvAX2MV$oNsohFYI85jGj^ak;{Yj`|J=UVKVmTLGx{qlyK(;cq=p6GMkXn4)=
z!kurE!jtm~K5JDAo3Cnp6k7UfL&hbR+)b>L9U3{D-m+YHtFU4Jt3)Q_2~4>gHH<fC
z@Gp{QVc0QalR`tcg2n~8KaI?i%VIm6=2|SY`lEd7OXjS&GsbIjxmdW>uAIHG=hKZg
zl@%h~W(R!loLDQfeWG;l1aqBk_T-E^zntQ@UUnR5{`bF%)qKl=)GZy|B9FQ`iVaL3
z%f~L~&SttNqT1Y|&vR2vBd^cpk%6<GibL6|gd_Q`N3thraavt^ecz&Ua?R5E700r+
zDD<s+CVaN<ZBFNPudJ+3-rQQsLit5PoUdO>Bnj7EKG+Z}94(yPR%{`=Xl>BMw{DKy
zN{O!{7~jPeF6j|sURf-*_Oe)gpyqE!O-`q^yEZHA`7F-uWX<ZJS(_*xxymr~m4aQ`
z;l37bzcUImQn}R_qvkz%&y<n7$Z(@w^U*y<YHl4z^_uf`xE|dSsHo8Iw`aM6)N$V{
zDt?dFewHb55?$oGL`l|>-|@-OdvD%;2??_LqPp(Mip03}F3($L{FiAG<vJ=T$y9#A
z`RtiD>J8`B-yPEsVV%>G^e5@5f!O0q7P@~g=ueS&RIjmEl;iB7Pp8E$=`Hcf^32cd
zjd<z#aD|%6#eXyW)Sh?W7F(qD_tly^&(<6Wc~cQ&8M<Z3uk7A8I?Ox!<oZ5Y9p0=s
z?^BCjfmZY@MTO&Tn;)xwxuP0TFzeB=^~)ThmK{)aNsl&2y8Xu_&4_u$zNJxG+RlZ}
zt;ddTI^${HaA9`anK^Aves>ZqUMVZ_J@R_%Z`!s}^a=A>H8-t@5PJ!ypnyzG`Bt6}
zo%17FT-(^(cIYscxZeMDHgk#^W0R9`Td{wSo|>bs_u4j5tqTXYDD=NNczso#$bXA>
z(>5iDR2{rFZ<CnCGPys;b>8MTELn9ZrZ|T8(3>-*hc=|CAI^Jw$M1aC>B!VV@lrwY
zr=rZP4#F7+<4zab)NT}4eC^A+K>S(gguFvOR%;xt6if8G_++HN-{v4>#T=ESvSCtZ
zncW*x7VaZ{%18N>^1}`*D(JJxzEwE#QPGS^rm#bz_RPtBnNAm5J|A#DVc>MqOy4=Q
zUwNO~Ri2BBLfTz8mE@1VF?{~<d!CX?UXr%sn}8o1)ij>jY2>+0XAPgib=vOJm7b|a
z5<b&C9=b1CmR;s*I*DP|uWTL{#=a`WX*VAJ_h{6NWaOF_<ob8vwW3VnH;LRfyyhRT
z=}bLm{(nE;b*Ta#vxnavtq|2=dw1yy)5^;loke2brz-3y*8IM8Rhp8Y{B;Gb?+TAX
z6!pFr$u`+MpTca&6Y0h1vrR!i+sR_aQZ=z@^Cgtj{3d+))mg?PdQ;{q%idnUAE#on
z#DglQ>v6hUvN1V5P}wY@<mfa-^`(nka`(0b5yPDwT?)b73$}BM9ukx-PT^5qa^pqj
z&E~5ans3YwuHrZ(%zemIQRj`224~G%%i2UsUAJDh#6ulZJZ7I$a62fL<J-&TQzX1*
zT9%`5-L+}cCM)d7vW(6Y?)-eP%Q%*w$?{^k=={%?_d^q2Zne_gVO4t3+VXng@y%7o
z8?4Q5C|u!6U<_0I?;&Axw74aBheC~@M6`bLjbew4<%aw&?~|B5`Uq^jl9QXZJeTEw
ztIuNZ7?r$vD?SLV&SyLBd>~VK#Zj{ZA2;eu@l!E5S#_QB>l4}UOOyiM3hrzx2)^P{
zT_*o_@xp>bhO)&9%THVjIVRS0Ug3jylzW$Kk7}>PhS!(Qsb10ROW};tZ4=p&(6eEx
z=n{wiM;$yWC$20A$Tn5fVEs^En_K#C`x^n(H&?>np5nVcN#RhQ(Dgsprv5mtel~=u
zeUjDll0xz6&4H0=4?KjuX0#fGo9ca#zhqqUTtnt#mhI-;$Vo~`H`TJYIRtQi?L72a
zZx!z`pBH7$s^xp1>Pv+lpS8W~fB6KxtEnmF=86YWK2IpHP-wSM;BfbQs~?^#bd)Rf
zu!`YM4f)5+X`5acq;`8Wx$_)mx$bdDZq2G`7F=GYiM>*Vy`Ns!Z_m^GzkR>mMA>zD
zt6Pq{my~6HSbt?ngQ%Gz(;;y;HiN7!4kE^tZ{*gzS*5mNm!75KjcG9#Bhx3wX<IaR
zPZIyVFh2NJ;qR{sQBg^o-c1!(YT<5Lyy>&`>qeWUkCsOM5Wo0&S{Z-&wP1^uSu3(5
z?54X)x@RPP^mJf3Z&&|g!-BwL-;DLPwr=E5a@=~$Ol9kaLo2yANrs3%Hy2}1yI81r
z;pG~3r!{Mp+U)x8%z2fzb_eHke?_h+)38T@|NjV{X`42O+g0lMVFfAUlEWva@Ej`t
zV9q3Ppy$uSg_9NbOG#=)GIExjEU$Tf-DcIQJSB#5Z!V7QjeB0@9jx>|ucdxkPhCH%
z@z*rQ?G0~D7G67L!t^m?iuiSn+>cYlA5Be_D%MZhzOX1hQ)+6qqHxB;?WWtC>t}2J
zeyI7oQFh1V#}?(=<<0#Xy)|cEl3y}OoheA2MS7()Q)a%nboWo?A1k%HBWJASoq0HQ
zHivp!i9yif$jBv+{5GG~U-fvFq{)86L$i;%a@}xvAk$REZCrKwrUFmjfv4xXzAf#)
zvf|-1H-=*e<alPP<v2zh=lZGSAk5P{^@PIw*NwctPW*RkV*bDXn7xLQ`vkoZ)(jEr
zUya3_OsW!b&rO>3C764StR=rIL|NLD%e3yFtQhj+cxlZt(MOTt2Ne@GiaE_v$Ve28
zNE8iYjAm_+J)-cMsZ8`pgIv(fWk-ci?4Hw{Dad^J@}es@Z+6_`*r@uRBT0zGL5Sgs
zGLyDjPvrEOtQ|9#c;8E#e<i4V<F2xa7SS%(Ztkk~{Zg%G7$PhCXzlVFYpTC<-rsoT
z1MkGIu3JKaTu$y(ybu|avwjtSQt0zkl^Y3{z1Jz+lK82TuyOl>+K=7QlNqKSYbcob
z;8k9x%BLLx8aw;=4Ko`$4G&d@Y=7s|DpZ(1d&=6$8WU$LH%ZURch)q0c;dh0BKZY>
z7$Y4|hbQucIi7r9D%!x^u=BH8mg3Fnvb#6_UNY@0bMem^=W4z))hhm*axmjbUV_tn
zo5<W1E+)_B6d66%b}74S9O@{W5+tkCCU+#z{QW*tZ>J}&;i8AG=xkItlkxD|N7v-%
zA(01d4xDk`_oCq352b(7{m(Y;vso<5bJOSsgPYQ<Y3rVa^|Q^G&=%34=~CF@782ue
z_LKXRtH+8B3%_nHtgKsldh+R`t;-Lxz9?*79Ofv}GEZTZtkuS?J0cfqxI2n)IPthS
zim#oc*zs?Nbd%U~)BYQci)!-wCOq|*$d}@D65?_a;>*t8Gwps%w)ZraRd4^?vs6w|
zd~f>S%xtyt`M`&t=kT!lt(iGXQdW7>%(U1t#e2V6H2!inK3N&k+o-qKEJ1z!**Ati
zC(1kCy_~E5y&_-rlJhwqk)KNvHY#p(FWYN0$Ln;3>{ok{m`$s8&HZ#^^6u^Wj|$Bt
z?o5;|>YlT`|MqtNM<=$<*){#siS3)2rr(xdvuA&XqsW$61{cSuzoFkc_n5WGGB2If
zEA=A6L-D=K1hplCZ`Vgn2{BTsb<ue!d;V&*--mlGkC>cRJ#u2XzuNW*`)}LoSMhrP
zVq@>5nt#|`!RW-G&dA~|C^CWJ;lWlOX{#v?tEMuTu|#ey^q6qQv5VW1>w?DxWw(BX
z$kZH<R|#G{!s2~G|1X@I))2)Wy)|1^bzy^w&=eUTr31_T7chwkgeX1CIKRNH*-E$T
zVtU|8_N8w`98OJdsEgTNm8y9_-g#rv!D+nQ)1Kbk%6?wY=61K-+A4|POW$6dw!l8h
z_@dq$(=49YroTt738|)uYL^xNy&3D_e@0_ZzU(gF-!ndh)z7iKzB+V$=&9*TPEKIz
z44lN^8_kh*N$J3=t17<Ye4!4rmy2))v*bM5lzM0iFLyL|#@VL|)9>+2)_-zfj`OXT
z?Ce5Y{0u6^8Bd<%`J*SWy1|c$M^0yRfkI`|>($md{MY_ew0&54d%wTI0mr#aU*vO6
z-FMp6*>=A~O;9D!nUib711DagE&n;4mkXLmIp(^kI7rW4k}}JO<)VwQV-$meqZ6Z6
zic%Ms;F0+5-+Lnx+?tPiB=?$`xt!KmopES-?@U3Fn_3bxMIME%z7pI&;j2hke=^er
z<5L=2HXQEKC|+v%*vUTWXK4Djl5djQLK8P7I?tXUnAkE)#=K1}xo6Qt7MH{+B3-KY
zUS1Z9@z}^(Y`*O2@?!R%l^4pm7k*1$^o?`ZgoS*(EpdyqH>zYXsW-XauAe2e#Z_CJ
z>j0OloNk3e>C&W<Ko7o77v)A>!;I7EJlCcOyKU7idD+DiyNY4G#MzylVfC^uGZ~I@
zoxU`!OR8F66H`E>|HgkhnhpB%UA_Hvzi@2$&zGM0vpHwuajDfOrTSBsU7cNgb<;VC
zL{88Ch(lFosuNpHyCwIrbXHH;^j+wQcj8>%Xzr<My-r+n8d=p2%+A-l<~8}-mSqf~
zL3gVr%-NB<CORiNXH#q8v5hKo8R~?xHWjh=#w=LSW41h|h<UHiywW}^uFVV<`d8%2
z-kc5p?D+QT_BSOG&7ta!#asc;)z(R!2wKHs%W<t)Z6l+4ONZVTAx()*MqI9%H!t})
zo8(S(xU}n0Rs6ckJAEf4T*;lO>Kr|9cdqT0HJ8`lt^I$4-*)c>*YkaAa<Zr2TXdl`
zLbtW_=&7Db#V;>tEQ(nsKGmx45SMnJ>dmVW{nl+er`2Xp`@bS~mH+FfTefWo=GNZo
zqN4WZYsBk*+y5LN{Jp%FmnPjh^j)Or;*UT@hEE3j*Q_{ivTM$?&k}cS?w`A9$dXyX
z%Gofl=1H!*+#M4y$F}85I!;}>#=CIaR<~FCe+B+gH~bRhHTjl!a6Ip#+}pw2vbLg4
zA!igrZi<Ak+z`0w;vl@Wp!?Ok!_(pnp5?!>2(p)0CZ6`NHK8i`OhUv)z74MH#8wM9
zJHIe&3-t&vNL_h+_L8Og`!^hkTrA*x_tqu;3k}B%V+tpwzMDGpG{;4w<*UU6rGutj
z<mHK(`_WZ+<_nj^o62VFf~Wk~HLdK`bYu+j@H^3UTz0DAdj%PXxcp@dvK#(S$+1dK
z@mR8BxlyBpSJh|DkXkE2%{N!t`(7H(lNaQ0VBy-NB{Az<lSk;}Edg)c^c^{yjE?N&
z`TEqEN0GVnUxf1Z+=ZUUnlEYS1@o76994<ED5D*Av0o)JQ*`x)Nemf^UFx!J9yX7b
zg!>#i<)5bJRkAa&H)CR}l5C3G;?)y3ah04o^JA0Z%*uqw6PMPMU0Ec$F}rofyL6Kc
zOMDrHABd=l7a6q~Jn!Y*c+dR&$q8?HTB>rEOgvwAe9_0P3q{vBsjPNu@|$kKGF#g-
zR6EJk_t(pm?wsz6?;muURJb{C@s?Z@+&lGPNK;yTug%k`T02Dd<#x+1Zjg1qsn`(O
z<M_Yec9grm*9r5mSt6}!sV6*-CM-=8>Wt)_dEIG3YjD#A)+IWc*L>d}Uf=e^r1=+9
zINRw*E?YLVDtx=prTgY%Nc5FMV)ItLvRR*XHSESg(YzT>3X7Xu+g_wGRW0%}=X9Dg
zMUW-+flP?A*R^A>6I-~?X*O4h9nx@|xLoAPp1uo*ei-jJJ5uTLTRURI%=1jD3Y^gz
zDXZTV3wC;(b?mg^IHaJ`;1*T3c{5YAYnqektO<;aS>9`Y7%XBvZy3zb%qf$dY<_WO
zL0|ETHOCtzl({BfXmbC1LpMHRVdptch3!?zy5TN`OYX|)6?ng5FY>&({@{^r#f^Wx
z8aG|(*V)jNckTaLZ^6B*FaI{#+_5)BH+!vX=#8$-y^A(3Gv#!;`s3ea(QOOX*S`L3
zcl4TO?k3MQ&Q+JCo^hO-^KWi;<JRSyW-3P5aVLIol$oQ!Y2iC*)hmuaS}`X>Pv#%K
zd{Vzut$V>MwTv>(m6JuMwjH<>_E9tbXzyBsBvH*3-(nIs=5n~6+F80~l84nV9_wQV
z`s&s=^L*UYt+7S5t(3)&$5HN+Rf$J`-G=59Ns&iWoPG((vd>g-N_zI7<;VJs4h4#O
zS9GG}c0{Q>JF0sj#ye;sNBUY1!&Sj&wZh)t;WFQmw7P5q>-@hwFYM)yh%zZ`S@>`T
zgXrQPcB0-~1$~z%ToYigVwuSIzwp|OO)rjUtgD#1`rIRn8y{A%+vckjtl*Zu!F*YR
zAwIXmZ<0e>aHH$BZ;?yn&-w_SbQ5w^PUDciv@GkljxX1)lw}%jf=|y><wZw7e4(}M
z>75gA%NB(#bGPlNetY7ov%6;PTc5J3rK$(@;z|-H@wZpLGtusv-X*2~YM1qy&vD*+
zukKk^eN=_v=dthmw(V%GWq+IBbjOjQOyIx)Zo3aH!haO<76j(zO*K0v)qeT(r^%nB
zgu?mRtpm>`tIMo6cRMnT{mQZCjtmDQ#jcDN-tV`gR^GcU)VuuYytFV*w|Qbok-9BM
ze6$}ht}MQ_WlI!;V8<Z|ukdHpPp|1I{pBfq{GTa>Q|+hcnj@z*d}gVXeOV-57|D|n
za{TLtlB0%<%CZ~fgs;pyCZKF7%Uxi;V)`~?L!JC@-a(J&<*#s>I;-s6Z=0`sLZg3F
zo^O<^na{pgz*hX1U5lE>WAXQ0TmJQ{+VKY*-Ju~c@!|6|EsGf378MuvOulpZj5LSl
zy%%#9d<?BBTdVBIbCvV;#OHliKHh5%o!hW5-Y_}nz)z=)zv|KQE?MF`Z>3sRPvDrl
zeiOUmVU4K7>@XFUg{F(_dbpN!s(gFhwEBkFlzF+)zC3E%yCjcJ$|yP&{LSE0XYr(+
zv#<O+z9nGK7yl`q6<ql}9(;SZ2IN;OGc~-Mn*ZOW;cIHc_tO6v)oB^5>ILkE1=}Aq
zuroIDdNk@RED)$*S5B_|UZ^gA&`5f^jb?C|!u6U2$3g`U`T1eWH;pv~%!*cS)0x{;
zeCWHKT$`O?QL{;rV26`$^F)PIXD+wIq{fzjVkc=2Mp2o|A~T%?y4nP1PpMz~#A53K
z%d$X=58(##Mf%alMWfxtz9*Hre~qpDm?sfv^&nNbO)#eLSV-Vk$qVVBOB&KYA4%uB
zkzO8A(f(X<!twUL_9kveKM#p8_laT-g)PDdIGdCz=c>t`d{~*m8o}$R5hGAl?-V|L
zg)Hwy<NhGoSOb$U4?EvaY}$NWNN+_}mM{ZnqR7^R^1g<Np`RnRKKt)}Zo0y(rzU&D
zRMt*lah$FZyn(f5k#AC|K-7Z(_J@v16TNhw)UEk!F<Y_h=lAwyi}Nl|RGk!<|NUCr
zx3C6QkNE#r^1q1|a8|JQmKU&XFL)Z!$7@(%zNydf5?A*LA^zpQ-iA%m$HQ)ZY*dx#
z)k*YA-eUYZM1wU+fR9an_YO^I34dP?yJw$0q&{k9E);A}QC{OD)LvSw87{arfuFHd
z;PMm^@4!Oo3sMe@ZU-AR`6jYNG1UKFq<UJTzC797IW)MoxXsu_C%b*}x4<5^Ba<==
zxuO#UqmG8Qd{3{4n8N+BqMc2wf4WBh^Yrf*3wRDr;dz)YuhGG}RhxIC;l3IF`%R2k
z`z!jhO9iHCFcfUnkZ6=$-JWo#IDGE3tSQsP94`xO+upykUDNY&#G&ui&smZtZ53p3
zluBt5dTy*3#4L1PJc?(MwCiFPVFx?Ag$#w07*21jZhj#Ul)!DkDK?o|jDL!Sos;tE
zC+^#h25>~gw{8n%IoQA*-pDV}xb~&M>YEL3R}`$i*~e|z$HhKdv}1PXcDp(i^}nS}
z_eJ~?Z-%em-an^9{#i>E-$mO+Kiq{bXf9-`kw3<@;-TPo7W?SMMKWm<1C15g6|L<!
zY9$W~x4bCkZJc0pkh4gMan1%t8$lz7K!>$rtWg*EPP`DvP;<_8lfAIT+#tltTh;kw
z<bSa@8yKUWw)ak-#cMdV?uJ11j5(c3!pDEAx4-aBI3jfSi_zq;X)4asu1u8CXP%z8
zMYd$C><0D@dGm<ag7DcI5pAp{!l%rZmMVFE%<4{++Q+Wd^+ce1$@I;O6drX(e*0MC
zvP$u(WYpG2?p_XTy9M2fgw!uP&Sw{7z3STgQ7vw*sD%3_okc(0_dVg?s=zAV*tq7T
z!1hR!O__arJN>e}WWyF#M|BEjPFDYRacY-Pm8w)Ho7>_)9SL051ZrDluO+hC7#F=R
zT#;QoiT`1QxT5I`Wz+5wQ=ULA1-FTZkJq&C^cU~cSh8T^Wfh;b3m6X^6Ixzs@_r|y
zoul3HjjTnR{tNi+5C~LYeWReWP&tjkx$VkS-K$B_A&Y`;UGlmcKKY@Lvk_;n_02My
z%*nf~e4-abR%@)~`?x?yaUoyf+HVurUfH--V&arlncPPL)t$}NIRh)DFUNN^Wiox}
z;9Y2wy=9$cvu!h1R?LJBqsi_M10za~aK;9NU;fxR>xICw85)m-J@$srtzYFI@li-H
zXvrhT#Nf~VvO5JV6I%>9mO5UPl8sOZ6tKJy5qCu?wVzS<{2{MUA<J`Hvv-RHy_}|>
z7-bS|z_!kykuPww>23jw?9DuZ1=}+FOmFvD?Vg?MwPnjpF}KyT=Y^S+Uvkxu(8yA)
zQkLpXKC=GZMU8AvO&!Mn6S}-Mm@73e{1oZ*OZ`@6BCq2%tBOP(Cvo$S9>)vk{ZlTv
zywUafO4nX72Onm!zzLCjSDjv_r%JJ=EG|h|pJF}DSvc^L!{#Z@p&knyP13T`<XN<O
z&s2$}Fi2lET>C+_;BwX4yQ?;2K2*$Akj`qJ5P4+=-$g0q$;<f!*C+j4xHen4M0CO;
zx2E;aW%``5^1c|=s7&}*s#!MEWG|D$rIU+U#g<Gt$|e|8EHWvwx>9Iu!;HJ9mh9<}
zpRTkzQ;;=fgHNav_iZ6>zwa9pl)Y|m;<CIH_-}#oleE;!i&RRBeJ0QL`Odh^<PcYM
zLYEO^qq#PNz>UpZT#dN_Y^>$nxeouCVh${O^+KS)YfIj&WhO~mcKqzS@;hm_k;5me
zIdLxqa-SsDcnOHTinpHJc|x<Q#AU1Ub+_eD&3q^BeYV*neRg(2wcwf}MZury8twAe
zj@TJ*ET5RR+$_m(<FDe$?It0NBC{PN9a$uv9pCrZz#_a|d_#-(Vg@OV+x(%AEX234
zSwGb?*|^3&w2iNEO-{JWi6%Ysmup0-1=(}@@&dT>CZw}H=VCp>7-Mie^+0I!gas@Q
z*(&}h%-gMV_0`%NU4mr`8mCqU>`Gc^-zY6~KwaLVaD}%U4`YA8>s=qc3isSzzHpkD
zg3RKMTkb2vB0gQ*`g?{*(1Qq8wxqU{;&bh4`x^gC#W8Obov9$~?fIeAY^k?0hvq@v
z$NSc7Rld>`<1OUfl$?4#?2ILo);|Z;C*M?-2<vV$*?+7gp~R6b-O4m|1FOu0v(Xm{
z@(dVd9N6*#*i;M{UpTS}F(d{4oaMUv+~tJ_-rqU5Z{=}`hX-d~Y5B9Yzi!g5{g(DR
ztF}m&ADkVdaKG7;`zYHp<-<~KE3ZsEoDewC{1RKj?>%1~%)Dj^`rHt(_vw(dZQ<M4
zCGH$>Wg#o?69F4j)#yK7k!kvchT_eKMQyiOB}k;D?O1JM+ndbh>+&=$HIX~of$6r?
z@!*8xIe!Fs)hFd8aF+*g-BmgF>2Sl>J%tT5^4%?`oyzA)&H68Pgt7mwO63f;s>~L@
zn1n2*MmdYayE7}R>sn6=SM6O<ed>zf63%RSefK5l)k2evG!+Wfw!}ov-f+5p2kQ^_
zwDXIi1C_j<rR62LTN%qLfBc=duf*r#8-dT>;@ce9f;Y6U-g*4#!~E?N(p#=cJ@jEq
z2{^ZIf|g14EwLE~R`(uQd-mMAzX!N$1j;6?YgRep7_)rCtp3%neeavNMJsEZ{ke0l
z*|mZvHu)@9wJKI!QB1txF1V)Qy5hzfp@o8*!tUxCMzR~N%oMz6a_77m`wa8zT+zaY
zFGD>;yQY>(Z7=DP584=@(!09qVCi1b;OK{~Hk^THy4nsiiJ5&3$o=S(ANIeTB`bvK
zLP&t?#f6FKYZJ3CU1Zz#fGs89IFrLUX@}!63CGJeC>U+L!qzjT$w&6F$J#xTCj?Hd
zZ9T^NGV<W{_9S(oOVKK#v!1WwHaxkwJdCLzY~gmdGtPx`!m7U3G&LCsES{6qnzeQQ
z?}<;8i|RE5ea{K3yyU^zEPCTek;JsS)e$P&9Jq38{q}f>tbFQF_L#rOFg9wIPO``1
zy(zluS*<rE=Wlr&ciSWM#l?bOryESpu!^mdi8k#Me&L#{@KBQJ#KyUszZbH<+H=nF
z^ex%2nXEz{f4*jX+mV^5qG6QYDLm_rL58y0b|DeL!!uHvKN$OO%uyDM^*0PO;|*eS
zxA=eX>&^_{9T$BjUi6zdufW^Y=vZ3m!%e1tcbNE!onI0XD3b2>l$+fz{c=fY(T(yd
zwu(xLjFxTf`6>(lZ3%r5Xq54UD@*HK(HYj<1r7Btj;q_%NJ<~sdM?9;F~R?Zz@+lW
zyc;_hr}=M^zVpxL{S`-dwu^Elnu~LPx|dlVOt@HmyxXs;(q5?1^z&q)^hE{&)f!p{
zuL~V|D(sbGTIu)xv}*doXCiN(ZDzjv|9YJ7y8I^d_<6DM!X39*1pD4*unRpnxUAPL
z#(^m#giRoVO>W-Jf-}!=)@|uNcc5k73;XxCc+M!8?$g=(D8uB??F2!;y>G)NM%+0Z
zt8j&Rf>O+eos<4E)G*Z^*e+z?es^=xUEajI{(2W@b<Nd$sCjwglMC+m3>|&__jTKa
zw;b4<bkIUJ?BJ?o16Nil=c5r1tk;IPn>+{#d9#rzpr9iC)W7oiwe2n^f0Tci(sBIr
zy{w1tc^;-F{Ji(>$BLV4YvkVDSGj7cRPpzpQt|B!&+|gDH>KuG59`}wG__5}FzIDz
z_q4~3en!GME2e22HshN6e{q1m+_&Uyap$=r?ukd8+j!EmR``8hx}5UlV(Yyj+?Sp_
zd&l}biRn{O-mL%M-d1nhZ-4sSukKIx>}St8#Q0=JzQ@hyR{8}EI$Lxu{(C=P>}|i`
zui5*b^T)HSc)0F?=BL1`9?SwvEdTj@lpGJXHFApTrQEpi@USM+HjkJSt_`k6Os8y~
zytt6$K7q@!Rpv(P!jpbXK59OmK?|80gHCJ_a#$s*ZN^l@m7}~M{rG%6_B4?l#R;p!
zay>RGx`t#(hdCB+%SvB8mthI>Z4b{20iI4Xm?kxd{wPhjF(LkJSE_Wd!xY1Be|W#B
z7QBm%FK0a?8c?1eyZ>9w&CRYm=IJi+6x|+q=ajFu^*W{78(nV}rJkHByi4lD&Ej;%
zIpSSnUR4j=%ojV_o1ERa^}+?7T+SJ>)eF|%+L3vC)zwwy?`N+m;8H)=HJ71m>#MKV
z6dVp6-ne1ETHu0|Lq{5KuqyaiG%hLUyUps>WAQ)G`c1$1)#-aqoLuBO`K9za8Ou#m
z8m93DKNH^}|Np@6U(LP?84_Q6EqJkQL+2DmX_*euzKkdir^P33xF{^|iRy5ibs`~k
zTT8~KRuN7<kpnL5!8M+Wx_@4}FBJcID|<mz#+Ovf`w=@c-DOL^_&OcQDO5e7*xhq>
zYP8Gl!<$v#3OwLroYQj2vvGk)$0>EmO+gEFcgtj)zL|G&MNhPhmhlo@;X^@Z^>42!
zT40>n9Nb`(zj5iATTz^wM8hXU1+1K%mAF*QGI!;3xB6F}SLEG9zaAIY4SA!%$ayX4
z#hy1Rz8Xr62C^O8emf7W>;C*C`9z3sun&Le)GN`Ri~oBp>QnyDY1Zh)z`>d%u)3uz
zRZBUnY^|o^a+XCG)aO*3^*p?+?bg;CZEl$wt6SU_p3-s>zNN*eIZxPwQAD!Kk73K9
z+z+eQO?si#sIp~6pu$0p$=|mhlh}OSuttBgj&ZFe^F*(ccHX546Vz^WEWDni9O2jK
zaD{if4e!$F?q(gUZ&*BLiC8x;&f;BrV7y&j$dd`Fb1v!`Ez0m_C}H=SzGUl^fCUPg
zd$K14x2kQDeEw04%U>}pQkC;+Kc8G@x89xXj3pCVo~=00dR#X{LsmRbBfPE2NK7R|
z+~`$ydiF`PXv^&jLKKSE#&M`^V@|Mc+_0-l>4IZ*&e3I#nU{+5m0G{-KJj1GncL>H
zIk);9-_}ztM(6q*SUL-j2sat|>OJx)PB<ZaXtLj}A7wu7K@4vv1~^qZe+h8454-Dn
zExu&#5~+=D0YO_>v;vtlr9OQL-pcymX8Cl!YhOb9W?cJX+Zt3=`ZGe(t6GsWXVQmv
zjV-bPhg*tLx!$yFSie&Jy<?KH=4;8rVNQ(e9%((x7fy=TZ9Uc${coo5hW~Yc-r8^c
z>-9T)QGJk2n$e$CK8#<t&Qw+nP*~y_;Vg6AV$uyYMgBd#IXV(cbz9nv*6mElQTXO~
z??YSMftP`W9mR8cHJ2wVYTtY%v31wp6@Co*LL9t_ho^h!ro<>#&WlKEvu;#4lE!>c
z@_)IA(Gf+diHac-9Kxzkx3qEOs>W3GulzH4K9gvN{YmK)V$xR*3*F39IDc?%P}@~&
z_J#blJ11RpTC!E7m809yOOaV~%_3{Bw;L2QIJ<A3c<Ro|vOv3W&D>(1O6%uye0%gY
zne*CAW}mN6^9*$gnzlq_>EYN;#cZhyPvx8(8*gwH#Tm(HiaND%7F?P<XSMM2jLkc3
zZ!U6Qv^#nMi^4YPr<Ve|s}x@+SqMmTJ&84Lk#%lcIl*wj$LJ}GLf-kYPP%K!k?zAI
zy8M)Y??Y3K*{5naK8rcoY%%q4OuKTjMQqWr#T*P#ZVOLtx^#1E(X%D$=A8E><qt``
z*}70`drXJej{mB9zpv)kzrGazJK~%3OiTAP)`>U&i&+VH&uvY8bwo&A`GjDAmP#UT
zh+xN^0~h`BSUh-7`zBTtdq3FnD9J*xwLoi`;nO2u0+g90PkL+<kp8w<+*Ibo;ul5}
z=C-M<&{}owpUTz?&lOxcr=JR0K4X{hy_e_Gvx;YZIPg>Ov98K$rVEP#_T1RRR=GO(
z%44;hYcB*A8N6`1njGL=Yb0P6;T^vBAlJ)N%T9=@w)C{8Jd6ALYl58K6NeNDp+`o?
zm->cnNaX3foM?J{;^C+~g))u9rmgBXTh|mR<`q1f_hy2GLn~i?lh)nDfam->-vp(s
zHI9s2`>a38;&Rs3(~GyKaBunl)x~r>w{kzj3Qd70sVsBWy`8(V?Y5TaNfphA&G(#^
zTvD+3=910RsBYuvwC~-^m(#8Yolp}J2xR%WK1fIUeZ-kZYa=JJR>cWspHn()(3)bh
zc*Dd6&1sKka2_gee5JxDz9;!&>zVC}=M5FDdz3H7eOa8cH^tz!M5yyVmd%Q5_bxQ@
zop$)kD??p@rv^J$Om0y$Q#rz1B*k9$d{QW5uFr;BC(KwcJzBfU_}Dekj*q*v7W}C(
z4Q=h|^u5qwC;GEOF(o5erudLT*&{CPB?$ttE2SeIS*=%EH$%qV<a9*%In60ic6SnG
zkMA&>kifmtvqSongQR)R!-OACr%qebW1&9rzhmIHAIJ5|B@c552Or|Sx!Jjh%V(>s
z@CE*vr=zS7B{SE4SG?F$B;x+mg~xPP`MsQuw!<q0b{tjFK07yV{XX{a(ruL|Z5kK9
zaeBM8;+0CIL!C$QL$Aqcyi2s+D%^J1lYJs}63fMaeUdY$axOBHS+hUyhxo&sjb0k8
zvkg~j9u`qMz-koU#x1RJST-Os_e?>;nHg6^(zi{R&VJosg;nZ_fa+7}ZC`etuM6zx
zx$!rzZNu$5Wu6`JHMfE}6szhVlydAppS1tBS;|Vq#4S#KX9V^ByqOu&qP}Xzv!tb*
zDwzk?esnL=ne^<X#2TK9Kb+?o8?#Qlx^t4<a%TmX$BA1@!vC9OHpj3!h2NiZ$z@tz
z$t}Juj0*P;UYB7s@ZWY}vhK}B?v#vXStE@-HG3^?q)Plcc*>yhUlGHt)CUJxZUuNY
zE@@*~>+sZly~)JpnL$M?_nQ_-=ax<4Um+T`VCV7DUIkW$18I!uS+~ru3QoDbS2>`i
z&pT_XC`YSbc={T3&xpGdxMp5f-)OelmUWT34C73ZSjqK~UwU7EnY3D`>h2#2$xnqH
z0=E_Wgg5Me^5u)hPwktj3TB)&L6Sxuv9H4Q4~kf95WLN;a8^dcReG81Qp3oXH>7Sl
zaT<A~p5L)=;``FhhNTV~lCJUZxhLA~U0ZLoFhJtB@RdWymm3GxCr2s%k^Ap0#I&G=
z_d>0r$CDEW%DXq;TP9m6GJn5Q%$!9!Wr6=K*j9H2q$q8f!Fj`AtH?nqCK1#1lC80t
zn+{&?N?XXu@v>9Rde2RT=IJx{ZH=@yI=SS;!%dor#`7C?EqvT$w4k$vc^$6;v*FE#
zmBxnV3d~|YLRU1H-$_`R7Pp!iNE}gY<ead}^d`qof!%x}7U7qt*uCyhmNAjgTqaT}
zSbbu*YmvZ=2@DJY3|~)6PB<{LNqNWK!+aT;7T*pW=(AY&n5*-JrA<2rmno0w3XZOx
zg5~oRxWY~dv}ULW{GM7A#KGob^Qn-_Xu;N}HzsH<naekOMQcR&7DoLU87jW3gpL=o
z8vW016-wB9*<;7z!(7TrjhX@bG&7c@X)}gjaN$#6Hkz@`D5Kf;NspnYan^5*V>_CI
zZZt^SOkEu;e0)NWAD02oL+-NG+vf{$R41G7<JjJ!aBQKJ*3%!|@q4!K`*AcahNJSZ
zW?O<=^=v_=FVo~6ZU6APYdr_UH{pg^k{yD<f^Sdk-BrD8LdV9R7j(NO?~t0kql`s>
z@y4nt5}i{vaQqh#s6AuR$icwKz`*#yoxy{_k;!ARl(OanJ;NS>)&#xGK!>>*tEVJ$
zO$!#wzBoJTGiO}z#NIo~(*u_u7TWdCNp+>RtN0a;=FH|RJ8Z2t>^0lL`lZp)Xa}>@
zlP2pK+vX?lHU7W&pl-yZc^b?zJsi?+IIR_$|DDn~v0&4YKrc_r$zcMbsjt`_li8yp
zSi>Y(9~Q_4S{!KAaVyLea5;QTIB-kbiMiTSw0587Se{|{<khM6fa5JcI1~gL6bd?Z
z9l8$O=48Gyftf|tw1?|s=ZOgoTpD|%4pr}&A+$AWhE0sLvUjF_r_qW%ucpb~(sp*?
zK9M3R)SxndCR0F;8DB$Bwz1Wm-OSM%jIuoZN#2_;I2=7&;(JlT^<UGG$B(BPU*3Bv
z$u?#V&$%-~ms9qgWAI~%(h<4MRDEOLYisibK^-Y9t4bqSt2{bF1zJM|7@{KB4I^6n
zB$)#p&Xvt}+j?=I-^MK;{$E}^?IFiD%N@#60`DajeW*C!dSHrf&FMyidEYe-|6<Vd
z>)=n_wL|+CzrN9ldKnHz(Eyeo*6X%NnUr)MmTp}asa@f;{P^W{f3jNpM7Z)o1fpm7
zPgo(7w|lqI57v1-J<FL-_59F?o?&LBa8~RNlXV7j?(a1T#X<|UkDk(+EWTAp#c`iX
zq|oWgUai+V6`wd9u3R&31M9~t5>hK$AMaQixPW!RqH`?^+Cwj}ypZV#m0&GPm>Ve3
z`a*<3Kh`NzCYb%@=0vGwz7rUJW}3{i^lpD}f$^f#gPk1uB^*X2-kyR1jSftVGrSue
z7z(}!s6OU+W5{*$;=C74%5P3`6cyP7{Xg9?TZ4;z;o^lKdY|s-iVF<hsI}^*;RJ<C
zlj}D)Xex8^F)USGxyx(=tN8|7t!gX11v)Jn%o11j28b}+eB2b`+Zz=Us&%;M^pCS_
zLYG2zZaTAzcdw@(<BQhN9YOU0VTK)kWdh8vYNn?ssm0tp6&TT4-grrB5&Jt85srsp
zyI;8OzF=1r>XslAo?gOf79jbEOWZuvzfEIN--PRp7q)f(n7JTRs&B`B@z?xbi~Iln
z<}cmCvEre{4F@J&0i&abgGDoyrycg1yW_;o4Xclz-qUK?p(`-?jEza>ts5r{k485H
z-o3n6Q$c4|%IW^5N&UapoQRNF`X=!HkzI!)mwI~1O#1I&X0FVsxng<MiXdH4j;ISP
z8f#g71sc|hbm*-W%U#PFxS%!h&@~<3V``;0T31ZmD$KQ!X)n*4s4fq|w++FbhWx**
zc~?&rS(a&S=gVPtm8q3S@L)_=Oy*&^hn$C>biAzUa1Gs^c9Zk|2KEOl6nuH@V}cv{
zN;_0;FjQwWf4*dMGKbscHp8+l_AzUFPC4zH=DU}9QEzRf{)C_z-M92Fx+?b-wdzW+
zCbF!0CDLG+&|cSZ?$sLhcWb;}Op(1Q%jzWAx_s$E!v*Z3za;8{oUU7i_grfE_G{lc
zW0$9IJ7jlV5Rr6e%<zw%8XQ{4vE#}4b&aPNN=m$Sy<;V0$z$R$>Hm$X+?5_G*~&$e
zCYaB<(38cgIYY9&Ag*l!Q?y3wVP4J|9!qD<)ma)NBs=xyg^tNxQbC@POBuJ#V)pB3
zY7UA@XiZfW$z=`Jf7_mO_3jPRD1*ITOQw1y{*u`2?W7jMu_are`o`34(K8A(-OXh=
zG%a>CKeo>3^7d<-W@HmRX~WJohbH3|fne`|J!j`Fir5*#opj{%Ve68`uMBrutho}l
zBP8td6+^4U8#5SWd^DcMcAqw0!s+4FZMDrk=g3(GgCx-|L%n3CYa&J`6xWL|+<41g
zCea+|p;oZfZ|)&KUkTQ^QLWcP`>u;NgjP(vHg{p%msr-tb{|YqUU9pX{`GVDUng)*
z*uH6{cfpyyJWmeIg8h9z+&&6)eAqDaXUL9zkBt>FT?R|^`UGwmylF97!C&FS;T7D!
z_~P2W>W&s38=JK!`W8&EO%Y&Eb?^wimB1Lh#DS-~m-DA6m#RXuF2`b#i3}I5WfDsT
zjz3yv7$?ED@>W}b2*=LHtJa>Yd(nP%>Z#R#(yn>(xq8aJVvdt;>(%M=DPJe6X?D8O
z^|aezz9%amma=v3{1VI<al1~f!*Ax&*ha3C$sLW~IQL9hzMY9{?Tp)NEW2Vf&lD%w
z#AH6oVcf)<q1J!ozSnF?zfk?-wH-kMb{A@oWCZKnl$hh`!<!^J<A>9X3$8CGELd=l
zM^N+siK*9i-Hl5S(W<?*vm!|9)r_cuxh&ept_E&*_M4h0Bd~V^`<o7p&r%aUt8}mX
zz^6DX_<E4iZSJWdH?_2^Oxtd#tqu}=bo2bSEBp&P8gwIk+|Q;rt39>QvJ6U&f3`<;
zokbeYQ;&HDygECZ6K*pZN*~odxJKvnF1bG!f^J2!cRssk8T)a`sd7(|l_|2WjT;tk
z3p?|!Bhe^0d6D3Gv1RAQ9-e!)a;JqK+rph?->e%i1h+>d>g=&zuC<P@^!0>*>3*Wq
z?U&4Zn9=ca((+hNv1KQsceE`w=1M!L^3>yK*UqIGlOmFME>4s>7{WSl)^Sz;@?(#j
zo@_7cTOsQD=J;hZq5m4pay+~`A9txAbX@#|DWHV;f0GrT$<i#l$4PVF|6Tfi<Lm{x
z63j_H9W34ZoW!0Kg`P{`(n?z6ma=r;Qa`^(r5&#h+evncuRhVyzfJ1)I@x)ZeYyo|
z)uLU3&8ylTh_v30yK`&pqsopiFJ>+4Fw{$sswop-nZVTc*6U$iN6f=iTb9d78^t&Z
zFLcVL-87i(eL8(g@1(h>9jC4FoVL|v<>jlq9PeM~Hfr2#xV`)4q`Nw8H!S-Od|IQ-
zsk@(5cmF06zo1)X0()n4ORW^#xN`Ptl~!K~X5AA7+k{VVKYifJG?AN&tbH0E^5?Ho
z%Ubp221n<+x$jClce0$QmE)-E<2?3%#fw(9q|nE28`JcHqNg>t@uqc}+)(gQ-j?z5
zpGeyyW$m*08#X+e@ld%{LVs=U{P4T$Zza6vym3U}3B#==+sqmc+P~*)$=Ivu5qfq}
z=3lKTd{H{_*Ifk^U;fJKxTe5bzH-AG?^j0l+tckov@!4;-q3X<gNyxc3(xt2J1@)H
zFJ1Z4G`}Sx*uS)c_ra76uF5Z-a%@!HxRCkv#&!PXJe>8R9b){P4!%vg46?h0@3--C
zE|ci64Y_svv#8?l3A?6sSRFsGVM6JlzI(TuCrLc@w9fRLFR&}{jAQQ(R?)Sa4Ci0b
zka%@5Ea<|sm8)fcwlF*_o@`+J{f+a&t&W+R>{nZI|MxR#ZuqG(p{eP_&$;a7r@z>2
zxAVRpC-JsnOZSY#4SZiViv|>bOlRz9@O9uao+SG|^1^nvy!JX9CK0~1<xc{ouP{a?
zmF?p4={;Fksj=R)^km5Ish%7^?`|`-Gz-75?dlw+ldi7q^Q-S_x29L~v{am2E`2mz
zFX)J2@-MMN?G5kEoq2Q{CP;jAeeZI;dUnmzHGkNa8%$Q}DO=DLfAO2{4W`F$r{(>e
zfARY_FU?f}&(&UEQ*7w-fAizdjEwaQpMPk)kfr2wTtL3?uDaO556(N<ME?lPtFY%}
z_hjN{yx-6Mh^tar%K32OyDx2utve#0Z@9r={-J_(nOUcPWXPosq5rKen)jJf?^Vn{
z5UT6{@Ayl@sr9<k_X}PU;+v`7dVV^S?t=O`*DL3SU;gv_=l-gc1_6bFgr-I|ex;Cx
z4-1@HImNAFGy)HZF?#WY-6=45?4-wJ<Hw^}^iYH`ct?nc!?VSwrYXlC>d_2#I5>-`
zk;7q9Fw@GgH3Hi#Z?uU>Gdi+v6Y|JXSiOX4mYz)4gtI{{_Pu;IS_hV2n;9J~7JGBm
z+_h!ww<mfodhp1J(@SQbtHkw34%?Viw}@thFJaQm%i(0O`OwF}Cw%HgQSipAvf-*b
z3LXR<n>zQ%(pv$`cy5F}wQge8zvJ^`!Rm|idF!^BN^Q-&z~SF~FQ>pXt7~?R;{Pw|
z4(9LgGSz;)CVOe#+Nz`3TR(fYiOlqkp5xuA=XmPNE3Z3A`|AqdAMRlL*wzw#y;^AE
zQ*JJu{(!&Unrf^Wjs73^I=uSIy5qZ6!l8eyQszYs6YKP+E)-JN=G-!=iCfenp^=x*
zVxowhYs5pR@V!q$LYOyh>||KnqAD0@%Xy?rcdJCpBHbk(6BsI(TwCU~>YJ)GFmY-K
zIP_Vo72MF}z8mPqnQEfuAX&CU$sw)xug4ADZFe4dO<`K=HrpfYT*Sl4OtwE4PhW8J
zVwbk))z6a(n5WK6v;BJIke}|F4Mtp=vwIev6brf|IakEfL-6RlYOe$7LKT8q2fFj_
zrcUTF{4aDxqqiniQkX$WGjNhixl+g^cTLx<2}@OauedH>e5fVZ!!ze%R~u95)X=a{
z(XFdivu$lm?DUX4;<b9sqpH<`XLgDN2e?W;nGzYlQOlJnr&3+8yh!kgK%b|ILGIS;
z*S4&P<I*sQm0lita%O@@TXwsnCtpgQ$1ARer!K2EW(VwPj`hq>u;+WyCB1e>pqFII
z6vmrkGCqckr9lgJOt57VaF>~Va^t3$;^+XKlO>mAdt^(mzYG<Ry}gND@C5hDsd1Z`
zTHa`^yij)MlVdTb#C?aKye#voP4*v|?=YuGYr?tcKji}A^CxjFbaXS;5b%g?DEZ>O
zNa{+7PsO#MtEa;McZXaFI@08E%HK6H<7<$oY3MblB^tkzT20MF9GI*Q%~f`6*>z)W
zx1Q9k3H@*NOrGxk_Q`jtw5V9;g$L{pyMH)zdb33A&zo|wEoFl5#zq!ljRwnSLO1Rv
zEeP81h<o<hJ(B9$C0T{b#p<|z{amnix_O4j`AbhSy0uPs1iaJo-tyZ)-PlC<!{Sfj
zg6F>{%3fOTvN%Pc>14M`X$vPq;{o4m@+~e*K^rc5t!38z8XUT|LpCIQ$&CJop|Pf|
zb)9SZx<%K!3Tiw2J%4(M*1zb9s;eVUT{@S-bWm^`SHyw)E1&9e#&9!l-#qPu1g9Aj
zqfeUJq**sT49tEOopmsZn9UjV-_x{a^(2QcOV^+N(ao);a)X6yn_*sIUo_8?YtcKl
z3fSCTX(i1aQg&xi!zY8&sfK~K-#4)@R+1}JdgB>+>Qb=vw+ru-d}=&}HC=_hW%Ct9
zH%iu}E!o`2a#lIWIL>FuauefSF0DG1a?(@76#8vC94AR~TuN(c(4D|hRV5PG<R&Qe
zNmV0!t4BocB{o5sow|8ut0eE)l$-AoEL7b0n&WZ970q%_mBLvqlNfe|9J2nfns>tQ
zaLH{3CQXrk#WP=rJuA3kxm!_C&88-Hfl%p+Z8uUUP0qQdA{x58lPACbkV~Y;mIL!N
zznpP8*>iIFiUmiX@h-^>NcQx2W#QA}qgDO?hVcotTMF9mzrJ!3-Z(X^OUS-+%_}Vt
zF3%%Ptdomx<!i6iIBh=Fba&g83+=ZW6B+7vo?x^((pK}Zf&H$jz7}JinedTG9L&xV
z=T*+d<Xs5Y_RusC^E#L?=P)1d&zFI_8$wbHI9Cd=2f1pf723*Ki++fb5|TW4#VK^o
zU8hAmf*4s88Rt)$SySOPaY}(;x)Ep8g%`}L`Yk>yXUysRa^gTIo9FqgcP5iAEz%Us
z4lA9d?0HDyR=Kb3yHovjDW`58NMDqEHI2J`vd2QDpP{We4;Ma<;@HJkWYL_lDSx40
zgRGvgrQU?ZgsHO~w8D1=hRKyJX8-xfWy{jt&0i*!8+ZQK@tL;6{lUYfk$O{4rtue@
zp8jKt(!0pcYP+NIi#S^<qhvxo15OK>IB~3w=jCMbF%_75B}ZhHQ%740s~zJPmO1;c
zPW|1WE&8)FUu);8(7$t-M1Q?p9;R)m{{Kwcg#Ws)lTEFZm?v6@PpZqwP0;b>oN6+0
zmzt(&(b1qu8o%_7IC7Zvo~+<`J==fAMOSO>+CYQd!AIWyGp=BGm*226TJhG|D@(ZS
zKKLBqj#H0gsW6<{QGMcsZ%cvJ3g%x*^%B`ve^#^#U#O1DS*f*Z+0j6TISsQqST7}s
z9!_yOxpqQ`Ytm{zPg589qI+S-jw=d@rcPMDkb~il)|qo@r^I+!?@a%1_H(JhPRA(>
z%ri9Er(Ha~wuq_pvrN+NP~&(hF5B(g#f#*=26^sl*(9jtU*F5K^5&tP9j#Iv{@XX~
zpIdq{SZL;<PhqR)FOmCt&NF&u&g!2xnhgrMeSe<pJ-kw+_+G@0?YjyW%g#03#5Dc5
z(aNiq7tPN<ky#=-J7UT#)>n}er3BBc?EAp7gyj(DBW>f~x56c}Y6HcywnSffd9Btq
zYXbXW6{UMGpL@6JI@E7+Utl2IANRxBq1ocdrb<W2Mpf>~C)8%7>Tg`RtzK6!P5y{8
z^Gzp_9UNCCaFi;_uqD^~e43wHXW?Mfm>W8kD|so4&V=8>U-o*cGV1<dnNWBvs<%Am
z|I`f=r{*v@oSSe`prNsPX|(XxRt;~%QzsV|F$o{|^laCh2>tVw9ZB;F-sXJDnY6kx
zRdc(UVfFL#Pin=k-k(|4awkUd{!fwf3g0B6_j#CH7b|@p*;MX5jqBrDt^-w82iH0t
zTc8#DQ7m%z*SCQZwG&p&d9V3!&HdmqQ|-k9(_bITVlA2YDbtBB<qFfcJf9u8Y0TU%
z`*g2l@V>S_EO{^0`r*@viu*dBnf_0ImMdR#{L8waSzH@$#ckBwD8A-FQ_AkZI~R99
zl+71VxE7+q@7~e=;>b^1wpxW?g&S9U?g=?McN{sWbvRgEX_fgti(^Y1Cg%6&Ep$8m
zQXypL#v84BJ~aQIu<*pHwlbalu{9x+Wt7%`k(n91YQABAq{^--cBfTys%NU)|9Sm>
zosUw!z(Os>yGNcm?J9jIdY!8!lWF2wlb!aHIk!Hp=5kjLHK_f%HsfivKBt$}&-)n;
zUHtn~c?%zQnP0r@xW>crMoh!k=__UB&+ZI$IWlMS(px<)EOkpo8YO3+zR@~;swA_O
zpaW-|*W4+OeeZAPjd{HELBoMoXBFY)9kUx+jxe>gxUGr4Gc%qqD0umnc^6t*80sQg
zU$`(m2)t_HdT{6b)(r<E-p>0|bJSl$=CI-dF9*34+XZh8xww7_pNy10wX)}HW}*<^
z#8W3(c<W~B_C92vrO3)O@7vi2|0TP6mVOX9%OSe#N^5n61m7{q1-vZ`%jQY6DL8o?
zP<s=$wDXAII^j(pxYk-M{3m+)`o=CR!>9v-M?-pB%mZEjO?0(hAR9TMi7D*C#Y*-`
zv55|`iUvCs4Q3`bu3f}+Z_4I9k9>~seO$Vz&gb6g$b0u#&%0VoJ1)ZV^y&2DE-cTc
zid1ameV}_;Y^ma>3fB4zmltmrv>u7O@+P72jepn0z>B;`ItveW<}POFIAv0L&_r#n
zql<&kgvM10i5_bn=ii)n{q2k(4wp3|*BySy1&FjfYE?Yl+ajKpu*4!k=c|$~+YQb|
zkDPsuXsIn}-7$4y;555i^PI{~v9aAa92()zdE~!Dz!_f0z|78j^A^s3soC!%619Aa
z#2MD<|3c?pyW6#<$Jg>F*FVQc*AFgPQ*rmZ#3NO;*55KmZ8mZ;)h$|RBfrY?iRLjO
z*GWl2j!QOgQF^HP^u5c1(+_$T_ySZ`M(Vpp>ufx(cl42m$vkZ-d8cJkLF;5Aoo@F^
z%zQ4%VzO#sz_}9_Uhw>iJv(>nj9|v;@@bEMMS7TVy0DyxSa#6Gnkh;CSn#?QDeqg0
z^BLW=V^mk^K6QyYacb$K<i4&HH?P1j4(TE>86g$!dAH7VKDEE);jnVjF-f+AD*`mH
zUUKSj<T=&uS?jku(53Fl!V9d4({@bz((xoA>BeQnWlC~MZ&g|U|51qPv=9%zB3ERk
zn82iHB^ka^hCOo%6aU;rw=1P(+=L?BJRYk)b-8d(V_HZ>UV3=mZPBdfaW59fCd~W3
z_l%P3482!#okHYX(>UXF4%B7bVR1QDS9CC$=VU#XXT!9{n7-TPIZG76Ziq#2rRj-z
z|8dVQ?UIl^(s(8MvPt^h#$>ak#bKt`?g%Wha_dNlyI=BGEiB?}f!3lnfuNu*O>aIt
zzW;V{?-wPBvSrR5Dan4%0;QH_d`dghm{$9WJ+(``pDp5eW?x<3y#qde2d>PLyb>kZ
z6C^p~z-!-3P7a5u(-_&`_?@4?C~Ne1{+;%@ikIEveCKz%cq*~XnwPnJNzZ?gC6DtW
zXZ_lt99$6lBjaw*Q~CA_OFZ733hlV-QNo|#b=2l2^DRYI{?NE-8Z9g#=R{iOOvp>i
zuu_}za?%!;N$k)3!mcTOm5pmZ)N!_U`TdK_?|*!8^2zdJz4Pjv-0Imh8kkma+H`R8
zw5(BTnab6&_MhL+jnWsdo;eu8B5u*1ef61m>%@5vB^#qoc)Iz@Zk-qG$kF~yI`rTX
zGsQ-+Clfn9J@dY@GLpBY%^@*~FH$?ey~**_IfiQ@EMeQcvijI&@9JY|emAk7DWXCq
zEc4a!x>-T>dMi^{R!pA8FxBhj)G|lTDF>#^a$pb9JXCh@;Hg!dH&z|yVp%KJF}-)9
zThNjJ-ACA$xg4x@X_qxSe&NYXxv!CVLeE<Hp0#|L)>d&@Ns6`GZ|+}CS7o6Eg}3;x
zxIPM2S#ERdd=SHFpEpmMO8j;IEK%S)RhbsIEXN_Hz|lTZy=ULtaJk&-vfJ8C?Gx&j
ziYm@4zZb&rReZz2yGpzf9y$k@=B&zGm&N|&`GIY(Zt<P`n(J^%VD7qauZr(1Ek5Fs
z>v!b9hgB|nW=LMVZ5p*`%PtA0H7%x}*1Ei4vC@l*R6Npotfk}GlNA#V1<8azm0Na`
z^GMe<cm2p^r+0R+l)UVH@nV<XwQix8FPauaTzc^#wwJB$LBHFHlPi23F6zBXyIGgQ
zH_7c-vVrFQS&rf+FaJ+YTDe9@=cCw~6(Tw891T<T-b__`Gu3H5gU0*A>*UuoE&l4Y
zdY>H|J5NvV9Ur%EK4NEksxlboc{CoC<5>RWIJek}ItA^A?{XDfyJqnP7EBdg%9y@A
z?bfO*{{Mfy{BNgsI_>27Bv-3jO|}OzpS4LYih0Y?x9t0(mKSO+J5L2oIOY<>77*gK
zwzq-(K4<jRds7^GwR%`YIJ%$P_ZmEy`E+H_fft1iDJ2cJcCDJirt$64E5`?`9Pey!
zd{W>uCE~qX8`s&nA_f9mclZi0G<<macK5GU#Xkf!9fW-ztj!Udy3~Z<lCR@s;zya6
zOBedCaLAc_>7omZ3u8#aYwtOR)@ytIul=$n=ub=7m$2>Ugj-b^s)d%Y@%erUERg2v
zzx;0FZng6t68O0Cr$2k}Jab;lzG;jf>@ueuTD6~V%_qGzD#kir%ybUfy?2OM`RSY%
zfA)&WO^=m+tBaR7tdeTex_hGU1qTBQ1A|I~_F*HR53{QtxVApzbTc{o@yVwZAuY>X
z7XL|Ganh>eK~#sBgX=#(+3r{Cjl6FuGqHwUx$iQ=|Lnar=a^ij54$W#GJg3dg(=wO
zVqVsjvS(~xHq~@Kl-s?Id3WlHNqQelbXjWO8cNM@E{VALDV^v4>(!SG8u;$;OLIyd
zxL(I}wnj5RcK_MiTZ**RKROw2VPI&;eaIq^68y6>`v0GMtb7-Bi*M*gcd^`Cb<O9?
zTK-=J?g}ieNg0hj%3I`Em^>U-xxAa><+60|7mqV0XXK7Vc)A26c*wlgm;Jx7yvXnP
z&Aiuf9|G3<luM<gOPNOVmS(1%d(k^jNk3E4V&&vrjLQ7)4{)uSDx|M&{pO%ozQ-Md
zBd?ym=4_GVXpqd%PYF1nnJuHevZznSL}!1I_Wn)I*@@p0FE8nc-gQc0{Ws6!VJ#iG
zO$O_EvLd^@gx`5A&x{lL+%SRh$c>1$388+cPHlH<o%74fU+cOLi^7ei$+{w|@>fpf
z->TPlB_LKO!Fka;j|IjKUB;DF?VO(rbFaT+))%|^>^b*ggVlVT|JVF0<Go&>eqXDQ
zzgS$^Jmjyp0KduIwMQM<9i8;IIBnkK!tKB?gX?R?Lo-PiiN!M=7#vRRcqXyuNXd&i
zZs(8dwjU9fyt|t>xOml;1q<gTE3+_p9DF1C{4e7t3s!E)2U|ikH!nWBkx9KLYGb5h
zQ|^P-;z+I~tlK{a-T$~mDBh|+Ra5PUVOy@wRJFRihId}3lP_(UY-hx4p7UAi&0!<`
z2y1<(Hycck1~xz0{4MBU=04MBN4DPHV5H;yJ=WjL?B!RVg6%0+Tf}VmE0sI4!;ZXK
z;FB{g-{|MprFYb_b<Iqfj1{~Np0axWpY=z$1#6d)l2z$XBWqUa8>a;A5`Xcu{!cvp
z(Tv++=7Os`8jCmNDoiTXT^YsUva2{ySTlEb*@Hw2r#sJU=V{fyTgS6Gx54h4<C}tu
z0au$D)^i;dcl>{WU$%t7V|$7EMknhnD?dDSEUD_3b}=g~U*65KJURHBY>U_$v(hE!
zYD|v%pM2}$^U>u^n&j4r9?Rv|CVzBv;FX%Ycu^>C!mSb?_JY<cB|cu8ukjw;zV7m>
zHC(q=UhVNX$adsc-@aI`|65)~?Wp6~^i9;rw)*D8Z3YJm7(@+tJO1ab_03$nv*A{l
z<`*&jPewZHE>$0Sw(IhYuZQNP9I#=#b;;tH4#V~%y-OeOh<#oWJK0L_SVl|EWffLS
ziPb#cQl$S^@*g=ScVzyId@qqQqqXndJa%rBIvcbyF8Oea>B`FZx)sV+%Z=NkOv<zW
z?@L+Q%X?<mp2TfWY^qp86g-Y>Xv(;LX!B~9IlI?ORW690wmopO@k_0gs(C#39i?-d
z)rCrbv%L7WVEwClDXW6_#RoR+Q0qRjX!@SEq&Zx<2L)bV-s-YlJG!b=eIrA}cgGuI
zjz?@OjvqOkzwlJau9f^8f<b>ACzJ=iT$Rv&P-D+8wTu?m&9nKAJ@vS<%jJx@Psft`
zVjhkW{-TQ>``wB!TKcih_;|$Amp3wYvb$C{?Vi)vDXBei-`4ij*7KV$|GM7tT4TrR
zpjX9eeP7($pX%+L{ArTWrvH;SS%-SRJS6u%oV&iQYO6!arcXtmK2`lav`<c>e4~c{
zxo`KhwUj5tT>g6Sc+qn8(&aa7ztwIJFR`lpzBe~7=M76)txU$Qx>Mh8rG1LHq0erd
z%rES(QZPMu?&*E!r&SkQFuG_nGDq^t+(=vC(99H}8Kbcv)Ui3o!*-2E!x`sZ{%F@T
zC)5&GS8csj8snkX$)w7ynp1K!XlYZg&=QV_m4VAnH=0L!Wo8DaUud;t7gF67_$-jo
zlUFL&s^Cx)r?7_41BXWD>3p$XQ(pes+QYd%@id2q_LZfby{Z?abbe3smefnxk@D0a
zl__q%mxzkz19J~121W)EwHOJ82}hXzH?Rmim{G{Y$RHP#XD!g-czU{d-m^ajpP#pL
z-1;|1vte0-F0=8fdz(_7eJ@IxbFA1~c*=oq^G=QzYd>z!@N;T@&EtFdiR(fkd87Gj
z19uipzvLQU(y>9oX|h!G%_2*$PtHp-M12AhZY`L^`EdU36#IR9e=KJ@!!J9-VcE|=
zPnQR6wNlmIdFn;d0i#m|i+PL>y>fCC-XV0rajQ}ax6?MOfY%GT*<T%4py!jYntO9h
zMYhXkHmyeOwOg19R((3f?Z7CJ8j@hZz~ZgYAZ@<6XK!njNY|F%Ex!sED6kq7DJoBt
zxul@<*jUK6`Iy8D)9I5YD0I%+HX+HB(M)mi>DDmr|Hk`HA5!_eq`fKIw?Tm2id%W%
zgE^NMJZiesG^I~K=#*l+SdpLlbB&}E3Sr+w7VtV;%u>DB>M+Z9L64N|3b_swCr?Jr
z2fuw7e^}Z2F4kao>R-Rq!S$`>m!e-=G*<jt+|adVo0Fi<#$B6Cb(#%wvctJt7k@sx
zYllP`^PXiB95_>iBd)ENX;#o_OwSGJF*<6H?8Bfk#imGE`DXE@rK+Yv+t1`#U*@@c
z^F(aMDNWbjMa!?S-8ynwciD{~20im#zUn6G5*pi!nm29sF|+JuFx8!O+r#pi)}nK&
zu8*5{?Cs)<eR1KfobL(i-W)~OCKi>1RaYnZnntQk+@D)yw<Id<|0&HG@p(6w&iS`@
zrXA<9M+a8zeAK0VaDiyfgf*P1EFYblYwi{-jFnG%t-JY%l*hpY?-kw5^D52!_e&O^
zxp-_t=^UR2M&W#~+zVI>E}bkjPnw<Dczdr7C*z_C3S62yt|WOGgc^I8O~0Tsd-J_2
z;~lz3^qV)Y)!oMJwean&7n2t|apy8WvgPi&v%q<)@ChsL?|0rQ9aYS4ozl-C<k?Wh
z6SSgtN`+BJ$QGa9nU|MM(hA-3d5)81$gdZ#&+Dz-A+`G2qT;tx3(hT?w)#hs`VH@;
zsfBy&7!EOItZ_Real7!XTBE~(O@};`|NqHzo}Cljd@VlS_`}6pJpZ2QufG4U{n&#4
z;=1#h=5LQ<;+T=~!<F%&ZVNZF$br~XY3fa!2^P$q3M{-v2RV1Oa65J}WNzBh*|%Vw
z)to|!!#YY0Z4(VNgqlydIs^$`i0mrietE9MNli=fseX+`<Ei(roaJZ#EwI<!IWzR1
zu!3I9S%<WQ3yn=`VjFijFsN!YUUW)XsKMpnY*iRM`%cm#fxWF^?jnn?{YsuU@hC@c
z{BFS;$DYlUezYMb>vzxXXWwNk?O5_09UYj$j8z#k7V3sG&Dj^xJn<^e2DMEGue-5@
zFLV2~%)6sHVHSt^$(o6d3U!-A%Hx&``?}>!7D#O7az44Dm*<q?8i^~#N~#ULoC|{1
zReMOAeEq+F@=^72r@+v_qD>65Jzceq2#c*fQM7vFgHxGa!OuFcDNR{g`q7v9XJpup
z4Xt~8_#<mK&Wq1cS({P0-tgrkpI?(w^m+wWMm_J5-fp6_{}h8~w1)?S5Zj}3+aB=u
zeGOUe_~V$p;m2Tg-GwI~#9dZp<!a@!`Z=9}_tS~iH_LRzr>Ld-ggb8GT#z^Eq>yHo
zqd>FGnwIH`OFd36Jrq7~Qb2$8T^{yJ7yE845;2|QtCxRsRg7Lpg*o@L*iCImb_jIn
zxG2vORy?A}FDdFAy=H==#Sx2)Sub=s7DaV=9cbh`($x|3ddl^r65EZfzH@$xXg@r6
zZHZ2yr|@s(!x{{lDpvpdw&s{7PL^M}WU}_$Qt#D@ht-Z<P&HBwDkwO*F!-WV#9WWW
zA}L3YB;iXZkC~ZG?%O(LkxAvo2|BM7ST^b`d9>w5+m^#(s+_6&LO#vBeKW*nP102+
zmIJp|?3tz&VJH{TlX+(4)2^90m1zN!c3P~M)yh-ykRj*GB;S1|r&pg`dZy^^bOr?v
zwq19)g!+pjVv8o~doFaGl<E<D?X#_Ujdz#cO|d5bEf4q^CMB?1U%MSwwk1hqL1mxx
z*PDJDujMM&p1v;6ylQEC&Y}g(8<z(BXuaI{Vf8}2#tc=>7ebw0AB?~F91{Dq<z&dM
z^@|&w5{vD2C3b|SOx3T^ak1l^*k^m>zjgG4p0gIsm*u2>G#p``Ai=)mh>F=oXKt;=
zwlbRuOJYxSi5(JXFJV>E|F=yi-X^fiYQfYsyE1GueHIs$ipc%q+GblW5)rv|kD%^`
zMPh3b&ON*J=~_+MP4}P$+Pi;zzka||kt^!L^t%5m1>3t!6LKa!anq3$F_Sb~Aa^FE
z(_1srDSt<C{iZo;j90rH4c!$|+(goi#FlsH91&Apm%5y1kCs3Gk?HHSlvz`(wV%ie
zUyZ$ZB5!)7tFYzz?m*>(8+HWfybSD*y#873`=U^ZV-9V5uIi>wY-n*W%j%xX?09JE
z@z6KsyO!^H>QY?&Vf93wezv~1hZTi#G6d@RDneTRe|er|7C2i|<<kPG#*N+uT@PAi
zzBKZSy<vWN^Fm(;OF%DQX8Xd!`KMiv-(MoK!#mECD=2Q3h<NhR&e&TQ7&tyQSf0Ik
zZjDoGqT)nt2T||lqBjd<y*o~*ar|s+J9C9cbF)rB_1D$w%eJnb5V$%h>~Z+&GdVi<
zWu9N&ccx$EP+#9H%bOXQ(lLSo+jZC6R1jSoJ&k3}^<8QfJ}>36^NNa}H@2Mc^NS6h
z!?p7HO%I0)snTCs6;D63ugp85r@3*VTI`AQ94smY?Hn1g-w!t4NaCKUmHcs&)9oa!
z^SXOmUw=Mxs5LU(wd%OlyLgHD%5m3UT)*nXP%*b#w<hl9v+I-A{eRY*V%NJs{Q0qo
zJr%98Q_dv6{#?gbyXEupZMr>4rn(oSSN!32sA8;8eDJJixB4%E)$z;D&zZPA|J+m-
zpExO9k;^AMOnolZ6|=Q48&*7Z`@Ksz-iwKeYgVJ%k&n@~98V(lC{5THQPitpx-vNU
z)79F`pLQiI?b$cka_WAmH<#D%i;3*oEV4zxzd2^QXMxA_due8Ri%zfOdGT$f?u}&Q
zOKmr!oX^QUI{T@00*|NZvM1-5jy1BRHSl#>2)$8c;B9o<@LkWD$?aj`F`dma-5a@l
z)zw=4FEADFa8?TIR0wicH*52aTIv7d-L0tOqGpLqUJt`&y)@Eb*&g#Y^BzmYvi~df
z)~wv|!-;bnXZ3>C>V56I475BBoZk1!TU|)tcE{Q|cTV;mVC7U&Seu}~_J^17qK0|z
z@=y1ib)V&a`&3_m4@bMh`#EQ_m48f#zLRnGQFj@ePn;Rk_oVeX75oVTYrMsB4z}d4
zxv?bAf@AU_AB~t3_m{lq`}pCfTAFgmho35KNi2&3Jra|JS|73;u-?<mvnju8T34n;
z%ZVfXnPN%ZLGQM+ICS1Q|FolnY00ACZCR(%);;@hUgGqd{vTdKVa@_OQ-rrLFdq0I
zAzHw9)Zk`ZzsMyo?k7InJ2-F6>Xl+Pu)DHMRqK`FEiTg;dHY!wtateEl1C{)<U~P-
ze*KDf`E~#M&AXI;q$LSwJ-lG{C@Fzm!Exe78~4D32}L*h6nz#cuJUIWJ|3jOq_V|X
z)cer8kclE{4LmIGjXC@sTAu3V>CXO>6q)8%t?-`f;qp&z$|fBx+|5eH3=W>lT=zBQ
z)%@dEUn{u#U;oZ00UC=G_Fd}Q+qOwz%LB>jz6mzTyN*prJ#?7=@GD~mhk1V<?Oc&+
z*u<;-yxVxoYTJ25ie?Y%SMi!=oxlA~P~M*TSl(f|{z7?OPDZ5zKa_d{Tm*k51zcSA
zSZI3l11I+&3#JmOW!5R2zj;*tPI_}<l1@()-|B70T$?UtzgXYPRv@JwtS+Rmc9FxG
ztVY==uQS^UWsiCpiT^iH-@M7F?9<JC1#=HQjrnro=Bc!^|JcucO6C4>jEU1rW63+6
zW~WyrO?t0XW;Gpsd*#VW3+6X=9(sbTmw4W8n&!VzC~5Phb-ygUh2@iyS8bjww^ZTN
z5%z`2OdgI;>n^9az1CaCK3h!ReZuM4OI#V|u^R7Ns%m)nx)=W~zw>in1!xEqm@pbd
z-*Ge2NKT#BV#1)nb<9ENSc0*ApwOC&b$8hKmKK`qpJ;NFi!c4KvCiV#Hw3>kWQQEv
zq{Fyk-5O8RvS8&U>+Z@q3CeR><mp*7>2n@)IP#B||KnTX^pJ}Te!{6!EN4GoqTv3^
zZ<Sb`K(fdJbB_jBiOt#1`Xw^+oHYMW_I$`OVM&6BRhJk`g4c4BGf#4O_H6Q;f7sD~
z@|;5#)a@Vly$N=l+?SW){9SMHs}|vPY{yHQoL9=dFn{*ZJiJdnRIC0Aua$7FFmLM_
zE+@s%SCexDeo3f4l-eZf;LXtRC3y>{ZT=Fi#jXa|E`QRR`c*BT-+%HACK2`h+`-ZL
zbI(1qy_<gHL;9DX^}2UBJg@L*G&>3RY;vD6iSOVSyWa(-u}%BdxN2<Ka;8EwJaW><
zNep_q9KN?+)dw-lGQXC)9Aa{fH{n6Xj~_j9rA@*n9*1O(Tk8~Bd-X(r-s1ncy|to5
zSn|;?v9F~&Rz6d^>b7l4?Cq<niylZSA1o4US{wJ@l)Jgl`>}^iyY9_XTu#oLZ8ex3
z4yQ-9h5mn{v%b$S(Rh*RpOen_1%1U=%C;(*Pc1dGDM)D8>|5u&a*3%$r|?F-B34Vz
zK#g=ome0l_mlnG`w$yjkFl!SD=inC&ZCk?cy}>nl(*^1K>56YWl_vzfI5N>zz#%%X
z=u^kG-4Bj`)^X14<>lFJVj5hU`|5b2Xp;y-CeI{>thdMGuY1~SZ#I3sBJAz@np4^H
znx?8{IB}aST)=Q9cYDM7y#94Q$7Kw?7T8Uh$aKO~;+665qY4GD7KnMdq&weR6lzlC
zytv4N%j2N+gnnbcjwFVuhaE1yG1?fTpqA$Cp<w)Ga@0ZnJO4%7w-sbNZZBl;5%x$F
zi8%ORUZ9YNBbVCL66buM$-OaQ$29*ld4Ew`|LvLSH!h!VtJc?CKlg&`rHgWYk-m?b
zn7NsgbN1q-t5+0+R<R`W{#drvn=jqstw_NMhSbwfFH9;<Rpqukes%JR@AB+SDn9BP
zKe;GbEh=<hBdXHCZTQ}O-xi<M8+WZ)b8}Ai?u700UZg8ea^Ozz|NKR>|4{qQ5A8mj
z0m1glCJYIOt`u;`P0juJs<d|No_*8e`^6$EP3`VJ4|e_DsIF}67UB5Up!VD~{dG_B
zz8sT$DAxA4tiJ1d{>7xc7iUh(HL002MAqi#|1LAjI(&DB!onrr^Di7~()rJI`hz#q
zwg<J`zL!?;D>Yn+o~y61;qcTYi~SGe{Nh>~llr1GIoxjIN4q5sX9Cz08LmlWPj4*d
z;P{eO@Al9;?yB_Lj5%Mv_VP*8D}Iqa{~=*#i}yALpQi8L`%)5rp7P=9Y>Ra&l$lnU
z{JG!citJ)OImUF^j~^?wZ>siGSbTnCeyU-z>CcYW*Um*9lowtttEQ?jwYcW<a#sFH
zi*~$I3~_IBSgtJR8o2(f;*td|(iM&RUUNSP*MHq{ZxidAh=nRQw7G<mRli$gNk{N>
z|D5|TG=64f>%&R)$J0G$w`cM$y7#W*^qG}6no1*LE2NVv*4C7ZT!~zJ<6^sMQ=7Qu
zw7>s5>Wv=xh%v?eSgAJ2H2>o^U6G4^EY};BeSIl6u|O?D{`rU5$3Hc@Whmx^yba2T
zQ<rk*bp5F@t46)af4T5mpA(xbrduD9Sa)T{tz!+f_Fi1qLVKe9)ef`m4yjuDz0^a9
zL2oI;`fqI$E?0^Kl&=p6|MiD+#qM|BkB+=<nw*$Aa}q~-LBi#BY0hNbo%7!w?+}*K
zEm~}TZOf}wE6p38s!DvXdF!4ugTF>pPeUbWv25xzj~-#=L$&UY6#oQrYb>5IpL^L|
zu6W<|ExWUg5`Kk*AG3el#&60vQ~0Xo5{H;;U)SfqP<?+nn&Ea?{3iQSr_k=V-nPx*
zTYWuuZ1VV0kmdRREB|*ZyT_9=nnlb1s<&Ounj1WuC#ZdE`0m4xyaoK3w)OvNyF0UD
zW6s*1TRw|j7M%E`ZM|jTt|jfKTUN;*D)`Llxj}T1>Z4mag{j#O7ulYgGLwIb*Q&Py
zu2w2%rzomjzT&qeS?X*<!=XPeN{_5eJf%+8%{2Zp<Hfya>q^9z|2|pHZ8?|wrb5!~
z`AO0H-_PDJzinQA|CIl)lVx|FYiT+l{>byO&i5Os3Ks-r>(4JV`Wd-=JNJ_BV%9Cm
z;!0Dlruf=hRu``qee=+2$KurNvT3)(%T$=y%$qLQWgZoNlz3+2V%OuXQcGpC76wVY
zRJ*)%_Hnc48C#oUswbN$dMvS=arnQPMo-Mm5aSF{1I9p$J&R;y<VCd9`?rK!<i`KV
z-Jc>7Fgx+O<uuE}`*Ob)7tLO=+s5w~+gW3$vxN~17yg{Jzvk0eeJ;H+PU>XqL0(^(
zZ!VlK6cx0lu(L1flGf#@J;}c4mf6EjM%7)j^Y3ez*VuMhtzf>CDxjdecd6aWrCScv
zCs%3zPs;UtsI)C%w(Z?3;!Y9kTmCcbEa$yh&U*7|`z=}BX>m*Sx3NFEX8!0}?XCUE
zQobq8^Y2^cvzxoVp14pbs~|d=dG+)Ku}uC>-fMzt78(BaU%YwpB8dR?P3KIVZePEi
zu<NkgiU`wV=^JMY-CnS|WwCZ{-;+<P1V0D#$Itz56#dVMOQ)jbtyKNjTQe<ZoD7`$
z#pQ6z+x5mfR(me{q_4D~we-8`owByaVtHS)^WK~}7nwX)R&vR*;4{*5&r}9)?Y#B)
zvZV62=xHy%JuZyBZa+;Vta*N_|0G6@L#lJkavtAhQ3x}eW2DNJSYV_V%~iMf{7%PP
zR$8*UD}M7?e}5aa>erFG@{5JvUy%OK_QCDdk1LNq7;W*rci%cc!gj**-&c3v+o%3-
z`|+&h+D&l<29M)+??3(Y!0qI>i=#}h1Rl6kSNSo2RrZbi8zveWl~tumOj{Q4-jPa<
z&@8&P&XRNa5zf5Dlbd<|Y$+|+Q|iW}DituNY7e(^iR02c-1lpb-~T@${{43C+)|nM
zv-Xrs&B?m1&l7i*J?r$tvb6!XzdFmkykT?pbM`l0wd<4h_UX+^Qn32zXZIxbn-Not
zi3s}{(?x1Q%7@f<9uPkiDeA+(HtWXntAe~q7i_0RHHXEhG<!R(UM$jdrcbzSLfoay
zD@8K9ikh9={9mkz^(<LayOuZU<=+%h^M#j}-}|@yN`LY6t6LvDOq5kPkt5`h^<Y^O
z6Uzjz6B`yDZWmDYnj=wo*~N@g-0q4;p~%WW5$7%$&r1)*7=r@Ld}e-1ancp=_tW&;
z{Os^#3)V9}CtMx;rwjRidvkJvkB9D(oUPecPo<q*CcMxnN6S=kwb>e>ZKt$MwcY=B
zN(gTgys+%$p|FUXTUMp6zjnARWs-^y!^Aw7-Ar$o7J7?2Iw|mc=W<|}u`r&I;n_3U
zm>r4_J8RjkT^f!Qx-e8os^xXG=sTb95><ckXXnE-MhCw6*Y;Q*yr;rB$;Cs-(JNOs
zYFo}F+cXi$p#9Cv^8cb2e(c;=s~+OVwxr>iTd&%be|vT@JUg}CTs*;W@#Xh|;&oGB
z#HbW^FfHdV;n=&d_~mRBhc&iKRS&<rqZq|9>(g&GMj3-62Su2h(i2?fc(hEfiB35h
zpvxzxF6b`hctRjtEUNTDt4yDIP$<)muM>3FPM9%4m*=LF7t`w87mvCPj=c~O+Uz-@
zMI<O^;^Y5fOs2=X+?V|E@S3oAgVPeHM9u@ps$}<e%<nlZxiw%CPpzn;v?JGp2jMvt
zPv-ghTUExpTv;h16p&%W65?#__%mR3h{TR%o;-q~Z5}}rF7kEpZ8a8kt}_!^7!uNJ
zI&q=YBvD1@4zrf4&I)EBYKyNKvc?3*a0c`<cyCmno~dN|>_TR{Vqrw=B+IEc<ArAa
z3P_&HE0*oluX$s_3L(*Hk~w?sO}m}P{L^gC#QljsXHWF#b4i?f#*XEZ)A_vbF_o^N
z*0NRCth5)-7Vh1ivu$#mww~|9Hj{a|OefS2c>ZO5dLi$1o`s`R@E>vg+5D_CChX%A
zR2AB9dR8lXSI~r}mH%A?Hw9f?boSDe&_lfLEE}C8d#_G8%4a$&LpP*%M^g8lHl0m9
zCYPqQh*@SCvh}l1Y)aK;JI?d@3>&wDMmuL><kE>JcIm`L=+*DM<1RThJZ=WR;@gm!
zVw!I+Ot@!z_RV(7X7hC4aOsL<m&p4|CST`$s!}xD;p>%W&i~46r}(R$T)w<R(cSCf
zVP!+rX~AidTc@7QIjF0d)6}|cM*8ZmZ8w~k3%v_?sJ5Q#mdj?jb2C$yPkVAbW5)Nc
z<h}!3;cLztBwLEP@wm4LPUTVU{8cXJrjvQ)(KfqH-H(oTe39TSoqn(A@jkt`-Y<<*
z<onW$Sc2dEY`weH<+EDKdWQ4wPY3_sbNc+hfE2#7VvZu}Q~xawi|Slquq24vO7WaX
zvBk%2Pnz#f=!h@OS;SCt$-j$5AzEMPdyvRFdzU)RnId^pHW{*<_Am~faZqH(cK*O8
zX%AlCIFn;GRp+2iaKPz~@R(<F^bC)93oN|G?ta2LSJl@|>Sq_1vxM+pOJAiAQJx>W
zrY~z*@-ipU+hg~;ulqZHWGkgC_RM8rnC#oiq9wI^w$NR7$4-@roqSGHMD{+PaP{Vy
zpcIGhu&6|SKgU(|oGW=wd?}iKiQ_>qALpI5Z-Qo?GGXg&?~Yl=GJWP{m1n&hS56e!
zvVilhgSy9>gyust4oz7qaw)GYd(pIIJ6zP9E~N4O-zDj`t7%_b#6+b{+Vh>3O!mB?
z>0LilrvKdpcA2gR?-X}yUw5fm>@{V<>Q=Lyha$6%Pk3=qNNJl)=e`{b8(LIWsM)iK
zt`(TPs>S8N)VtXoclJ0R{i5pSBoi5LBDl;`cVcT(^1?-TtiCPi@ldR{mAU%p!}mqz
z*EF~^J||BU+acPwvsm{-%LHp@6TuEsV>k233QMKLRJc~=Il0e1@qR~}S8?i+DVk4~
z7fw5>D!$hsCH~=gm%i&xGksig6c0c2sZ$c{+jcU?rlf1vs_(Kpc~2>_KVD`!bA``F
zmQ7Yqy)L~laBcfIwN*E6k5cG?69N{Go-UuC=^OjxQLwSsI>mL4zW;-Qi?y#9EHR$@
z@Tz2Xv+_NTZ0(kquCVNjeCvv?s{Q;U_w&n7kDkP*Nl{VpqK7?A1rM#8tnw!D?LDt~
z1#!<~1tsOYJuYo2ooJ%8cgnooZ?t?CTe&d3m~w;pui3nhwny^s-kI>mfko+uW2?xg
z?lrRx{8Tz8$H6ElpkQVBTzMw%B$rRk>RGSZwqLVbp|!R&pv|M`{G^=ViCho8rp>Zk
zqIdA?y1b-G>U*0f=2yO$q^6=XSF7k$i;H7B3#*(<m{?Hz8w0n0T+3$1B#Bt~sW4TW
z#CtP;J@8)DXrfZ*BTwf~m#*8oESr{eG<4Y}P35i(%^(fYpxIutb{2@4C9V<ezVtuB
zlHo_HanRJW&;D4*ILQ_1wmr!TR)`kbUmCi3?N{NB6Sp-t=3SiFc4^VsB^^N>Iy^i7
zi*#<!_!Rv#?@7(wqS!4PR4>f>;%@h9mZw1S%dHYdOMG{>%$ubm^yGk-%7$BW<!;Fx
z;ykb7r}S{CM%OEG(X(~#YfT-ecx{*TObxoJGEY=;O_s|6-He6qM%oi+c^-LYdU^d)
zftB~CyH8nUK7Ym1l&2E+<sv8DlImFV`^tlYSn=?Ms`)z)@LqT3=X%`!=fu%GtGN1a
zUXx}fh_qY#d}?bDQm?xDGTZao#|fLanJo|4rTFCOHDT{N8+}rBvL<^O{)h-%wzXv9
zo}h=B608Q={||gAzwVxT^~s{DV{b#Z7B2|C5w`nO&iYHnitAd<Y;|?s9=~-{@oRB+
z>(;w(I+sK#ZCAZwq}h6nZJm_T3ETL$QFkWnbow><{W8O?37cA1KeSM_@ZqTYR=fWH
zliP}pd`2$+a>I9Vgc>KTUex@<WxG)H?^DMjyK}U^B(-O1%$m9Mv)CaUq5TZUQbO+v
zI$r9zWxD(4IpLX~w}##mn09>UIgR?wbAnwowr{v|RcGz;?ms=ZEBa5amE`F^yXngr
zQMK#O#n<Y2P1$hx^(Tpo|8E=mTGe{jbY6L>ag!-+es5M+jrF8Jk@X@IS4?#)tz0_I
zM|P<}kk|}K*VA7^)56UIwU_Vu-*vs+vmmGD=z}E9*_uLWW!idkg4<tPDmHeN+2-u4
z_?4UG?{+2ROnTjO{b0L~CJPN7$=*9sxHzP1^4<x}o0_a5om;ZlN+(8F|Il%I-Cir1
znd)1xT)AyYi`lmAzPqzJOuN2Jn%1y%SA$jVleQ15fB1Zxeo*zf{VstIFSEKPJX_71
zek)1HCaO&&)bH2ZvsO!tI5z~Ic&#hIBBe7=Eq?9O_<NtG9=@X~!zSYLXn*Cm(|gVZ
z*X`eO-?H}Usw)?zL`-y2GJcr_tUoTtx+4G4BAK~YJROW1DyEi{xw7ePREusd6-f-7
zu`FiJF@X^G(wn9!mP;~K7V3STsJ!Gt*|o#P{{;&3Qj*N8ie0w`@NfH2G1Z)_#$156
z%p=OQqVapE8k^w7W2FLumAsD4eakg#oSJ<sD&3~kPkCM!+}09&Lcr~ZU5k4~g>cK&
z_PA(6v512n2@A`#%uF|2&v+N+bbvMUO`5IG<~Aio`L7>M|C9;+F3kA4ti)-WzyWt*
z2IZRn#S+ZZY+Fs8ZB$qnf0Yi{SkSb|Jl#;VXQ}9AmAKRstt+>?ZZoLeT2QBYJ?&L_
z?kw@vx8lW%J94i*u2fd+5^7ZX`#dqjTu4Vk@bR-s&6X;U_OcV7x>#NatS;}~rEI1t
zD9zl`xLdu+#--qwb7{S*khNm<va%+jf;3HzuD6Rl{>NN&n8Ti)_C3xkDLU#<8ULjg
zCyvU??)5>AxwEfVOk%Ip*slJoJ$H5z|4;Qw&lv(ABbviB7-GM)XbNd~E$k26Q8(FL
z(D{b?oa-WQ1H?kjl2XMpER^GyUGJS*(Y7~3LDbUdbkW3h=5of@tZxQ|Yd$owNGr8T
zX+IS#q^J?WbI~SMS<vE$w~v@m%mv~7PXzWqv6$5CR+JJu*)62Qwa)Z#*B4>`DL;Ii
zuD88t^R;cai*t0G*xsdjv3Aa*qU|SigEG5TRa9JMo2tCne|dSEySZAHgpgFEF1MoU
zbct#IH}|A738xgq?r3imbPPQ4BB{Z`w)|&u6{q@tVb-mI(*L6iLIW6Errl^=H>3Ei
zm~6_98Bv==Cogd|{?Y&Ic>j_geLpk0yiW9cF}7+ipLsp3tHrectH)HA7fStGQY*@Q
zxr+sRI0Hjp%D#S9b?vj$b49^|lB|Go;ouED{|`>{z9w+w`E2P+QVPP1?S($aTm;o_
z2t1muZ2ie-##XU{Wm7NREEY`aj{oU1CB^sgMG=o5v)C3@vSy|}S(zwQICX)h(C5ul
zl30~IIHz&n%$u~r^FmViL8aLS+f;j&O?&WDX6tgH*@;a*#A0`9I=6&Q&$&FG=|fEW
zRDoH{GftH0G`6IE?40n)+-~{h-X-axlb0CiFP#u8S}wA%Xvs@~@Q#^fYW@G+H7Y-H
z2>xEtKk2$Fzv5!kgX$)$7A^6Zncp}g>1x-eABjqiN#)5wZ>LVtaT5ypJnw(-{2Ma`
z&X#8AAIrbys8{$SIjF!a(m7BfRhYv|=&Ey**E8XRM>y*m-IfGN%@mYAS`y)6Ds((W
zxu8lputE9d6r;3lMz)8HoIfc)b2qNzl8L({Bd@sP+7^wpgNpY&g<kAj{58CSD{9u2
zaN+J-b;iew9vw^1&`>^I62tyT@LZz+m!a34jn<{90UI@y4_O+iWhl>c6JGgi#XL5V
zMP=Pxlil|SN$=2-Ruf)c@M?P9%4&;G(|3KJzP~HtP}Q1k8#$u|n=h#Ksb<ZbuUXu^
zs_VZ=;bJ|m#n*1Rnn(#=vs(0Ceb$#B1s&m{lcUV*n^t~4>e8<yc#cEWOUuYt%E;?;
zzz0P^uLE-qI@=u05EMxjD)scPl@O9?2#h|!Soq2L#CN3=mOjE4rJ7!d^Xyu1I#YPf
zEZZf8J{5&N3#YmVF16hhXe2$GL1z*J#{(&)-5XV-H<c*LJ&#cObYiNa;ygA^^BGa=
z-n@)5+g&+JJkQT@t;u)WH62|B(Y_xgMMWOj9(9cP9@(~$W3@6P1DAr0Lb%Z1S4x#X
zW&bfX2~12$*rmuRu#s=q0_BJHzggu{ytgKQEQ>stqb9XZd)HbsDZw8di_N9Bb2@J4
z*|^rDeS7)q&HpD)-)tf+_#$&*jHiavttO3@B_8SBlAOMRKbFW#R+}c8h!n0b*fQsO
z=+=!}SoeAgW(Ir9X)v5E-l}T2f$0EeY_z()%jE0Kf}4XZ+dhUURl8NH+VE`LP}#V<
z>G6VH-tH?l?cTjjYhts=S_YN+O`6WX?TeH*KYF$Jz4N?Z%<JZ#GzejgJZYVAkHhZf
zb1RvjvPxSr^h3JSW-JMKG<&_5;C!Qv!ADC}FD`w~ttzi7w31sn<4J7W(=Ap@W^{<m
zRG+o|y4B+UuLY{AxBv4#V7hSoU9QCpXV$W=Il#B4OVg=2oHO#pkM!qNGX$5++B7-u
z!?V3P9$WrjHaQoe$3014TKD?@5;KG?H_m2e-z6-#AwQ``&{2ewTQ!+wH|Iu4!31}u
z)%&*p7TEty_FwR(`Hk9^HiAMMI3ojP)+pHP&#pW*b+fC7aL^vbI_oJ@S@I<2OcUkG
z>)E|SdSw9DMXS`8PRWO=y95~mXA4+-(`4@2udG<=X{LVXt9=>cwzapNH*cC8vskcA
zXc5yIdrq4JTwW(iZ}%5*uhlP_WulnWnz=-EV^!*(6^TJ!3U4OP&{2O`w4RqSVasY3
zCMlbbM#6?Eg44tfUr6r}ZcNdOEY*rUoNO(5L}VL(qucTzOTmI|a}vcFwryCvaapVJ
zZh^!jepfdL7Nm(C6zr_p`z)&0P`l-V&bt5St%a<k;}iqeF-fjfJLY@+c7fg{4fole
zpS{$aK8r3orP94j^<&pwL5`)X{@BlwX4v-g*rig1&(_wJmg?4p=M@*9eyHMJvqtU1
zCiCo%b6aGD*mVVUvkvgzIlyyQpykef)y4xiUhTgnB{b*fn#7}vRSur$*lE!HqDjaz
z;`JLfRbI8EOgTACS<xT6D=#klS>@!rSo&vR{$`gHfs5^omxQfl&5k}7+VDr#%15Z#
z;`E~w?c$4)WgE5k{tjN~x_j3miH5*UM|vzWzl2mW&X5jPGJCP7fG6`=<~);yUDH+_
z?fatsKI@wIO2IQW)+?_Zlu;3S$)l=Y(#~tWNvT^>>9+R&-Bpd^8L{sUouB>f+*@m-
ze=*y*vi3_A*zmf{P@Z@po>#SP&i0hGC)ky)skYs`8X+phtfX=6;BF1&$vMJLtgm-^
zCI{aVm{6^Iwb-5E!L1FwQtNeZ&tPO(pSvk=Dubhdbf9<8=^frC7ndf!(mx?Cx%tug
zPQ~+|d`=%cc4wO7>6B<mMis%!wUQj-g28_U9+V0tUzieecHRuH+Q*tZJ}<Wq<x#!S
zQ$Nku@dHclqq0t4?^_>vo!_l#3yNAF(<5Ux(R7(@gz3Y>Mw1vA7wGw^-!Mpeuu<jC
z#NK`DxCOuc7SLSCdMawK$&W<_N*DQjg(Uu65HY;T<8wg#?89&BvnKsK@&EcC`&@%w
zrC8H7pOs_`Z@mvY=5@LI(na}PrUy!emd%s3d{Q^fIC%2^G*hPStePhS4=#D|V50rc
zVy)A%f;<1jrq2>DQ?b}*<F=MjqQvn`Z=<$Fj*sOe!GMET_g%R5O=2Fm-#z<P^S(_l
zdYIe!j#GK!BEfrqOl1DpPZlbf79%)eTaR!0?#;|wFKpfF*Zm;f&U*Vs&a8`TTI-%!
zyjrV%DbYZvUT*Kvxw;xlm`{pv9%H*&HA#Eg6-C|oeYaZ0Uj6*|G=0l4vx`q-<s=vz
zs@LAqXP90v>)KY%=*gU3g0XS?H(L9wo@C#D`1KZTi?m(Kwp5?yXn%U}UC5!^4-T&0
zxGc{4|9#(Q&m-^kNFDw3JXgr@9(&Nr*RgxIr-m2q3kXqK`uNhis&x&)QEE&Tw{%z@
z%V@C9ne)co#>oEPr1x%3|J`16zf-HXe3)vuU5ansgu54R-nzgO^C7vmKK#uJrNVnF
zJq6QDdrg<cWcuFv5GmvlthTT5oar;6j|+tkzIZ-+;p4Eg8&)PIOmf_|WuvotqPMM~
zKI5g!w!R{#G=$zqyQ$@i?|6NF6`#ZPw*sei8CU=N>{5I9Sj*x4Zx*OebQPT5$+Q3c
z{f8eyVuVzF^%aE`yQ?T=2Y&3`6RrI%@8q1hvr9g%-zi^o(|N{3mJ=)9`xy$)b!!Y3
zH{$2}+P>Aj!%j5F|NH;4t1tA_)BR^`ex9*z)?dZvX?e=F6DPh|cSq@*AD5o#M-{pA
z3g5G@T-p2jXv!NwL56_r$4!1hORNu9z0}{PH@Qvmiyi;n<wa5zhTq+$O4Jo<-3z@t
z>0#ldi4uYPBzix--t+8|eT|!ipt0Fu^|;z@>3iF5+*_e8CwTb&=2yO58(3FfkT-Wa
ze(vp=b>YeHWG{LNmNxv8jairSifIAEN!hJJ7oKcZelS@qf4R&ur-k1iSh}27OyZvV
zP@C(xT--%Y8$og3i$DF-84|r49GaMT<XmP1JalT~72amWA-L#ZCx_xT!3$lY&SreJ
zdNN-FRaUmCxK^1|1|3+|tRved;*j?L;f!Vz{z+4k4ARfB@K%2lVK}0|z{tU;#_-|u
zg1|)_y;3H>L_C;Oh54ekW?j%~U(LBw?TJ!C_l@;T7Co-3gK{I91sl{Vv=}ljF6MqL
zmun@S$g-U8r`8|t(6z^R=L+}5t_)<*ZrsOnO-d&6Qwrl(v8ZJ_J7d$DnXIknWUNlo
zKEdr?d{1MO(%~sg9w(pt2`YNsw;?sXY^K3>=OgB<Z?4RIvbI5?DB0alz@bT~d2he<
zoJWN@C9TOfn`-CPZisuUuKGm8K&@2!%-yr|-S1zx=U6?DJx@;J|K~pkSNq@kze7sd
z`46XcP@881!{z7|ThHWd=2g5ASDayf-|QySp6358#u}WhT*?9~StkAPTeyaA*SW*5
zT#{C=+04QrxO11xrV9#7cf4My`8X!#m4c+;tCJe<+Ez_G#wDJ$ah*bQ%!~CZ0=JVE
zh2=E%JDfaKY_w2KGHcVRHZ84OMQvBD7slV^x?U{NIlDQ3e{)3QhXstnF&)09oLhRQ
zFp3xHooZe(p}|8_=*Gkq(r;zDRtYlZYUs8e^4jR<B<}m(dP~&I6|(1bqR%hz5^kDw
z!CP?GmdmX+Tc_AEUi&=dN}#{)*DC>{x327M;y86h`GnHmPZpg@Mw@%D7#K}n{G7>T
z!q%21)(M*#p8If^u?s43I@$0$<#@C|3-VHws9vJ&zVHQ?7sJ#42UZs|Gur(TSSWIH
z(YpmDm$?rd-Fa)Ow*CU9$7Sx>yvjVcmoYuwtji<ZB)-t*uT@9f##zlGo1(6DPYvwe
zrG0*rruOFIr2c(2C$6@+yk2A9>gRnvnem2!=GKX^#XlCOo)7-=VDhm^0f$uXc>P?w
zWYUBQ3tvac>D--m-M7QaD9GpLy;;eVle&FRDCj0POW9eydgCA)n%rhH`HDif<L`{5
z386xk%KiU@!?LCZadk$diHCepxM7jlEu1RYQr69(`s3{ao8Nygx-<UUGCAtm5@FAn
z{uSD%V-^Yj_h{^&5W(?j1z%IlhItCl`#er+iEDaP>%KYpne(HlL$~e;1=dT=|ILIm
zULOyV3trN8`u47GSM#q+l{4;HcBZst*A)IOQF)=Zi*Im$@!fR!gq!7!DMBt`2h5GT
z@8vd!$zAchd@y?Y;aq8^I}e|HFxwr#xFUk%;-~A&eYmciOmJ1Y6!bJCq<!VTjNsp)
zu3PJaeb>gXTCr?@Pa<1|M6$=5Lt?WpSM-FxahBW26SDV-s!HyO-Ep@|nh#FRapVrF
z5Gz=rJm<adX?M>^!B{o(>Ggk)_)ngE>i&u*^TWNb6-@sKur4T_G<Tu8zHi`xzAs9P
zcSz(aK6K^~^!&bE^_gm?w@H%oiE{NBNBkAGbj^LgsKlmgE=$&&hf3n_jU`uGu1!uq
z<3D4@(F6arCpzWwEL`BU;rYZnX95)qMNXcQN!h-Hae0ZOFdx$*4P_N0E`~m5j`K0!
zWEPmRyYOf>-|k>@^s-_pU6<)PnIXc{&uv+7stqU8VwQ<#JJyKi7|c9*#7&T6pP-_4
zaVo1P(`0Wc6;-yT2@wh=t-3xBt%Y9s`h81Nl+1h==kOz3(f`x>9VarJgHt?q%AFKA
z{87gxXXcX1t{01PFLh`vdVRZR$JGfZZ37jLi7;3k?G(7M^s=JRjTTRjhpP;{UfRTm
zaM&3>71X_DT>k&g+-X8eZuM=KG8dWnx^*4*atK`#zVS?tQv5Oj^VAEA^6o5DetYaV
zN9mUS%Ip&&Q#Nd0((-@3E2lu0QGc0-YJhtB%{7uf)|SCmOP6GRxw-r%M}^Ga+)wqt
zxf7gfehEfP7*ueCdfOlWv9%@P$ihG_&yc0%?h|~rS_>R{<@Q@*;eDTnkr#gklpN7&
zDA7%K=Hzf<XbGD0sO7SvR;+q|+b0o8M#rU$&M(_!f}Go41hglvOnjvBOQu3|!G;f)
zxC2x+EI#^8!$7d(bnr8#n<8!gokg<heID+SJO6Mi&N1%#opdPRghg9zq|@4zpx;*4
z$}Vmd?3U6v`(s06&#~my2ZdZNpSaL5MaQTj#QRI$O}5DseAJHH&V6a|>*Ac$^{Iz^
zoCHJzg8FZq-u!g;%?QU=9`Edyx3f)7^p<2|Vo?0a!YabR#Gu2#z`)SJ<iN<l@t=W}
zL*zwP!t6#4VZM^Z28k=pmI_T0j9`1{62xR+cISlPLk&iG*0wV@E{54MzK!BsbKL8)
zM%kKcianYWrZt2zU4Q7ZWAozk^BqL{q%A@cF18poYsH=_S-hiBnbmIUDPgV9)l5g;
zUa1Tb31sx#uQiA3((>@dz7zR)q{TE=H|9t8C5ErQdtfrti3yi93Lm(%*0b(ORsONo
z;cy$Pd5ENm;qfDVm;d$HOxVCMwT(Y++MEl8lKZ4Bc}31vCcCuoux5!(s7gL+6l3An
zqx;~2i`im_@-uVGMP~Zi9p`z{T^PJNY}*^YHAl00G@@ST6unt_<9Xo0E37=dtM)$p
zI-B>r+J6!GDLRL?$ri3+zi+%dsFZ2XEP0Q8TmCNix-<XFuflx?et8`YosuXoTRO#O
zP0yl}siikJ1-a!UJelnv#ry1{ThHC)58GtQc0_2VPG}2o<tSBb=wv!77VWCV*Ye1L
zb>)<YZ3@#I7de?v5{gKe(|crYp3KZCF}ekrM&iD_TiNHef0sRRvHXRhW~!6>#L!e{
z|4h;Q(_V9K@tWqfOmte~-CFad{!^ANKdMnGsOez4?Z)M<zRJS1n?B{|LhjD%+c+s=
zUYFpr=ZP)Le$H^}+t?V9(za;ljfI7>*WI+uxfC7+Z7B_T>0n;AOCZ>WYeJ?|?u1QI
z?tP7J$6921mq-`P_WA#6frE>X<*DL8C6&8c`K>ntn#>uyg&Q&jZ$I>S*U5S_Q1R;(
zjSBg%OYYAM?o+w9<+7jpC9UgKoYS4&3GSGd=((|JviF)Rhh{YIJG1iQDiz-Et39fM
zvfuCRu$rZ>S<JPeb^pS)s>j_-tzH`#(o9OSoY$_scB{v1+J?o>%#9K1?Tb0wgSK&9
zWlHzz*{Z^3{YrJ}8#ndmDq&H6tKLlY`_;N~@~I5X7P;te5AR$IFsj@t8*z`<Wa|3R
z%Gauq|0K_fUP#ND@w(E^_O3>%!N$ii+HbEc4VokQGPEG-yjht<YV)(#)7lzVx!ugl
zR9-U2iTkjs=3K8>D^+LqyEkqv6Px;9(7=7!vD;T#QvOJXB(!~=EfaI5cVXwwTY<-8
z16K#^Qz@S@eXI1oIpzN*uRpm=XKBBTWOl66WaoQd*6-<>5*^EYGt^&u_J&2u1xs9B
z2p^yK{av-wA2#_{*F|=|xO|W+H6p<A`^qm-OAj;mZ+sTPc`i`*8_&*}46`#fp3*Yf
zx@OwBmly8@Y<?WFZ}L>9{Jgi%%$M4WJbiim{=Gjowx-2lZ##o_P7mL7-rd7}{r?%M
z-jN4|7{oPZF0<|RjE!>lQ9HpG6*56?%L4bcZU?)n4&<qB%V>?$IlQ}%K}+w5;F&ob
z#>)Fzcn#-rifD%%I^=Y=#b|BA!y9Eg)L&lZ)-YOi?#_R=&(_TauY$QUeY1s_W_NdH
zDJ<3s?PIxM#&wcgWp!7aa|o-pkwUs9Q$k<UqsvQ|GH8_8Dc<sR5nTH>hNF?=iP))0
zCIS~OUd-^`^;qiT3E_`(T>G+h6gTc+Ix(fwPw~CB!q<Bdb~b9qA{QU<j9DZ+muu5v
zxi1HnFub_P@5j|5m%oc!_eRSc{kBQ*bq8f%Ss5)>+I7h}bC#l(?xHDr0gA3Z52dqv
zBKE|osW_`t#c?(m8S*l!C-<-jIYoWk$lsLWt0>v^|5e7$&xS@SIW8&N{&e{Mnsc=O
z3+HmiwUQ!CSu3JH9t`{XVzGRI)>6B4LH34Y2j~6|-0YLE@p;rY&Ho~=T!Ykwcvi5o
zcRBJ_3Fz-V;B7CkDS+|I;)T4j9riB_#ry9bRS}%!@#U7-47=PFQ(LN1-5-7Pc5BV?
zud(D?`sSD<-vgm8*DGi4$7QVyY86pjaIwRvmU+&C8C=RUozC6fl(yUB4%dXG+{?qK
zY57@br8c~dnV{yfaY0|m)QD$Grd!sATy^PtzB2WrmePK%t8Fo%E2D*uDsV?0w~-Ul
z@Q!s=Q9il)#hOD+Ulw?)S-w2k`i)ti<72xEt3pJh%NFG((HvWg8C!#9sVG-3yx{I3
zDDKwvWz)QDS53d<9S4rERoY2T^`B-gFk#ZFEp9wXlAqkXWv+V8{QlF!KYZuxu>UFg
z*@+^aVlhfwhCe>r-@hUeeEeld%a1UH<Dw1u(^ubVGzd33X_{9&SvvN4l=#leTk}d+
zR^M$g_`dV((RbyCU&lUbD__YFq#ZTW>W=QzP{o^jLL&~c+i|5CG;cM?a1uVXutRQ}
zVuP&ChE2=nSe*6jp5&?Mx1Uw~!i2ZWriSunC3{55<@Fz&9y!IqD8y`bky?9KVCKV)
z6G7)DhD=vs^%1VklIH5vR;+w<HhR;J&ig-HmRAO<7TpQap5bz^@Q<Rp$1*L3UemAk
zE_v%2Rz-z2W;pT8yl`Z)apN5g?dhk&*KL}%*=7Q(W98}iXGJsi+oT#KSDs1t+dOl%
z&()7Xq5nl@GAy5Tx$A(6`OT9a?}`{-7e8P8OMlY8-Px)Oo%#3uySky`qr0=H?u0{H
zM=v(M4QPrGJaj`=&3xzV+?4x^9&jdqyPW>{WaIzT9fGaX%oaTLSn2;;c<P*pLrfbY
z;yer{BzB!zDka*W$GXF#^5Z(*f?0Fjw=CdH%3ip8Zg$b+m3tJ@vKddGkK4pB!}hJJ
z`|g|9%C>D=>iaa0`^l|~VhlSL?#(TnzWWwa<F?83Yv1T?-+kkCn1|ZE+BdK2X7BM9
zYYv}x=eB+SqxoOg-m3i?-QW7HX#;2Fi?)BOr?kn%DDueX=vusRY`mdby+&eT&@Hw{
z5=JMt%wM-C&!YEX(3kA14*xX`S4eMq-Fv1;Q`Bv4FaLMRdc~B&o<u(FMFP26=?ib&
z^V`l)rzfyT{m6@@oktv|=m{+2KJYRie%F=R?}V5CR}p$E*CDoHUFoXoy)U>P{<_@1
z_iLH)-Zzh)Rb68jYRwmxJrz)AoWE1`x+k~Q`k!}~#G4C!JZNRLk(2R{o5J1uZ7z2%
z9{BZW&66XJ6Ax%V^J4sYO1)pPXTq!sPD8;im(vqPOItpltQK+9i`jGa-U?Ty4W)4>
z0tEMNsdIi`YU$on9Ia&7CK8zOyJ?PJZkD;_D^A7TJKFXC%%5y4WPhIN`qcWIXWD;%
z22ZcLvany^N&8&o1NBu-Yxb2a$!GcH<Nxo!!=ClEPtujRKhHDYKlk(d_pCJ!<UZdx
zDF4p7{r>9rZSo!UwTY%o+%6uKj_Mp{l6u?J9k=+jOtP+GiY|JbX?B#We38c56v;Lr
zZz~naH5b@AFLIf+7Oh+2Jx@t;MS50rifhZlU_mt@he^^lsl{A@SurLu+Y+j`JW{tQ
zZ(h2w%x+;bry#4Fds(h|iQ^2`fPfO`h!&>C7S2YN1a*eo@D{rjEuI<-0Uj+OGX#Pv
zS~(WB#B{Xgin9hdv_uNDCjZDuGDxjU)4P!9xV6BuZCL=<(s}{c%1p<KV57p^O?uxL
zBL5s}+L36s#J#;KqDlH1*P@MCsR3Err&MwnvdaE<G?=LrHD`L#oIo~J0qJ_jYy;5-
z$%lz@M!LHeN-sXZ_BKIiYmr3MhgQEEf={lOIId6>+#s_4N0;Y;GTRlcyc4TEB)a`L
z+L#`8d)_D$WMJK_!60y;h3P<dyaemS3Du|C>yCX(k^Jm$$XG9WJ)l9fP*60%?qDzH
z@qo6aB43V+{5UT1c!t<M2BDHfaZN>r!b@z6SLpXMHY$E~a!87r;OHtTR`vN&;Wx*i
zS%osv8yn|INR?g|h<Ok+|G1^Ln26jo^$F~vJRPo^zY50$6dQCDo8I6GRA9B4E@uBj
z_=JT(q(#U28v;HZEP)Fq9g%3WX)m2w(B^*BQ6|On=hpJ4|4&Gky$JYuUGUb4$pYVd
ztHt{qEh~7hPj0*}P~Tqg?||dl1$vvC3aSLfC$mmm`^`((vV$qHXpTXW+zf#gZCTwv
zgr|Pe*JY}jz0jQZpbgW3X&i}M8{L!+J7?E@cB%hZb$PLX#KlYwLG1z|x3?M$Zzgmd
zFPpSFLTb+s;h+^g%m-U?pU+^LFp1+}w=yTIzyyJf9Sl2e%s5`r6T-o$JG1%>NAJ-z
zmCHZsq(t1<O{cV}Mzp8a%Y2Nt{5jd-rvg(!pT|$(dq0HN-<-U~bISb{0=p|Cy37*f
zMbx(~P>wt)BJEK%B{Xr{V)czn)Mo^0v9Ay;o}^_vMR4N7#2+&xYb2eg{y*Ayx>A3E
zd*_C0BD_C^8y5@hYV2RSRNgFX;_E2_v%W>_JSwwRfmLp1soVE1m+(o_i8H)!%($Y|
zs=RWNj%8bFN2!}pxr|Ci&J543FNJxwPvP1)xqW&4w}<h~OQ*U%;eT_$F(A`RQ_yC{
z_3W;es7dOoiJObsk~+-NEi!g?Ow6!h-q@kvG)<pFByWS1=tZxZ!%>SCRr!?|R%hBu
zeTwtUa`l@fu=Itae`nXCl{MRbba#3#SgN?dtAnL;=8Rs+?s=Irk3=kezPv8og6;AN
zE~e#<UpB<Emd}>Ss%X8+Z>=ftrM#g0l*-|svx{d59G*G5{?u&ula_xC3YWa_xvA{*
z{{JNb>94vXGt~8?%q~_+uKKj({HH}%CQW_vQ<$q$z(y*HX<?Dx4v`lxr51A8EdRCS
zu%qNo$N91om#khkF>#9CtO?aN7InLR73Zq6ax7drNpmKX;Uo^hHQJnQ2@!2J(^=Cp
zRB~J<XT4}M?-Xc!<y9ReWvMB-f9BfeTWb%jT&wp>a7U?R%wg#rje=7{T@|<%{VD2L
z>J`jwIgQJ3TDqmc^y7l;8>M7cOe=Vtx$FX)N>S&QWt~h88`if@|31m-6pKvYD${SX
zSfZ@jHvMQ(vFs7!X5AC9Y*A+2#TEJIKH5GzwCqP|KtWW+Hc77oos%67uHC(Iwxeac
zr>4M5i_NUh!#Dls-2C<WLWVMN$Ak5vNBjPV^H&)*oRLhLK2cGs(8=?4=AR4+6BCW5
zMg2^Ijj4}?mkNnbDO7*)Q*16*mdpgrz8T(*AEgYoOG#Yh`gPGs`dUSp$$Yy+b5YA?
z!;;l@D^@3@m$>|B)t$NBS-qvDviZm}i@LD#a#!)HtDd#T*UA*GJ=j^m{e5}Q>V@^I
zc3g>yc=Le&t3$=@UF*_~3fBa5Nfrs6vP@QdIPY)7uBR(k&iS2aRJA_kKxK@;wwQ#)
zvo!<~PYZ6>7Fhais{4~^yV_i277B@N*4$#|v+Wpb{lyhO7G*9<>k3{alq|ry(W1@q
zMvq-bja|hu+0*%Nk2^Aj27LVgJYaR?6xm11@2ygNc3Y7B`NEE;eKTM4|1pht*<rdb
zfqzCc*SiP&T}LPLPVM6e-@L<f%HGI@FQXUUSfx0FTfZT+tH{aR%W=!?o06G9iaxs&
zq@VY%5IXpEkw$QJaBD^-r{k){uU5()<edL&CGWxAQk&-4KTMP;bh2A2xF^s+Yt8Oh
zW43xju9JdOw{GH;v&_vr75(2VQ)Y@@Rq5o~sbbHxXaC7w==W!B)2qqTr|Qj=o?YS?
z@hyRWNrKP}?##|<J41Q;o@gKI^*)w%W+7|Nl;oZQ?x*}`C+fXcVt6SLSuH3u)hjxD
zOXBL+)@tdM%3)I5RXQ!R*849J*x4<>e$imj|8Qx8Ndl~ihSOIFu>2MmZ`AjUn#Ysf
z)fkz=UF5hmi2sjhLK#P(8TV0U!2|WTHmx{Q?jc#&@b%P#R{>2|x%syST=iNZX=N>u
z$j|k>{GWh&(^Q$I0eppz4EH_Y|E7>*p1yzUnN!(!ET_Ca!})yYV=aN3i|1&bQv7*J
zFn95~mz+gy2M<m?=xuPVzxAP*^Okk(J?mf1Tvg}Dwp8hGM56UVrir^AZu_uUYUwsk
z;};5fhxim6xn@=`Tq|>QhUsj(#)W;}YwJw|XIyF9GGV8(6&LT_P1f8ZtylSETXt%A
zSH#pv&I#a}@=ffUwe!*!ed@0a*X=xheva7s&t~s5)VCH(>inP4^j<RZkHj9)J0iMV
zS&N@FEmqq7`i<}>uM;0;9a?9)P3O5lo$yJ=LaD{Z0;`VfSrc+*hpP%_5O-PDsVA?w
z+n;i)GwwItEyA%@;LGuSK|Xyo=3eIt+uoEd6I{#cp}@9aPA^;eDTmDET;&;6djqmF
zmhboB+NyBvhQ{W15`y0tuFX<q=#1t6e}OOc0pHix`=?pmSY5M^HRgbrudd|ddgh5T
zam7nEJzP=Scw#@dpy<Z4A2*%lo_uS`gj@U@r_K_*y-N0W;|c@m@@*nJ5*79czSC*i
zGEp;rw$lE+in`lf6-0#sUte;|oqwr$&#v8vrMEdQW;pgeQ7GZ=QP$pixmBCi{I5Kw
zS-tPUUctp1Ci$&kjZ~O?`3nCm2F95I{E2@&XHN;}G7EgR=#r9S&W~8`o#tFJZ~4|9
z;QKGo9#AFoKVkp622<v~GsQ*R-?WdH89GnDeI!PAr|#d$7h_f!T1RZz!1;=4D*vJe
zV>iX+iv(8}3J4You%Eb;UR0!aLfLt(z=H5AuDQD>ZCsVx8O5{EX77>8!!6U_2hE#g
zD6~pN@oY9fUv*|wZHD@7!Ri-%=Fj^k|9z4l)W>q~$)mqVes1r5EE{;#v*&!FP-$;(
zasx|b!OpTM{>5LeB^^*}uUfsx;dH$!n+F5qVuj~#CRDGvz<+$soNY@p9&_BNn|EV>
z3|I5ptK0r>h`c#Dnq%X>N8atTWoP@ZS^m|4?Pbu}_b(32+H_?$Q`4MJw|0HIwcwxB
z;<#59iv;I8zDkK^Q@HeMO{ms2PAS{J+LJHoZ|KrLYP(n_!Q%K?L0$8vH<bdbvJd`s
zs@kwgB-J4LSfW>}5!;so7ki}(_dQVKNM!E{=e`+xG~?_Ojnk7SuDjN8?CMs{Q-!zM
z{vP0qY+$*1_To(rp`GFvn~riFnZe4X#C22p;+h4M)->$=I-xqPZl_KoOBBP09$R(p
zbyIk29{gi?ux7)9Zx6WUOn4Bua_8jf550ehy=oHJH=*!gv*5=$;;}z>3kxbK%YRZ?
zpRIPDQ_<;@*7{H4H>c|V-=*?j{<E(BXWjju%=<rC{Fl=DzgudNkH)U7`ko0)0*|FW
zO<gLsZQi_jYnDh&vyj&4sjR;F+TO4sz%*KQi_cM;rC#d_o@ccC=dmx@>Ye+OZM}j(
zSps*ayn4dj(|l6{>Zitk4>grLy!Ipa9FhM4>9GY8F5(N;To+!xZ1n=pFfZ|$4Ggm`
zoY@+4^phOxYlqX;;*5(HT+Mglemj>_P<CdC?xH}0`~ZjR`wX}Q7H~%CMr>D@ko*6x
z#>eOTYh-%=s;xQqjf4H#y>oYyYz{bld}|Xe#qm^Zaf9jcn+J}D+)@hq!yA}%)Sqq6
z0oK@?f2Mu<bFN<VRJ_5l|F@3WubO-5)kd)ia{r%qc1{fHUVG?tyi`WzVi$d_1A3};
z&R<!R=T2C4(Edr$rlna2oVlVOm_|34ZvHeS#4~~+(Eor#6D#L6p@x74+3+%!9h%9@
zx@Q{Y-MN!_<5bw{sIPxGeKfMBXc*4i;$k^d)k{o+DWdjEdO^beCKJ{pEu2LPPEo5f
zHm-gis9+nW#_lGluqxR{*JF{x#YLV6G#J0k%91>qbww+1OUtgFrj*dVbu$;FL~>0E
zo1(Gk*T$ritK_s*7T@5^h|K2^Vw7XQwb}L5wD_4!Z>}n(S}fBJ`et)W_`<5t-Syx8
z-4%Jzy}@xdlht*dp3NK1%rLQ@EjM?T$-x<m18=8H-x{VpbM1c*Rkg#NJmO7zIHw6R
z?AiY6;-Q@k?sgx3<UeYywbnnp@=W1NrduY*?JjI<(7m%){LR0ZXCjOtJhRN#Th(TL
zy0M<W#PtrZ<1JQiPIYC6JqQ0V3v<8Fw`SyBbjqI5v}uiOYs{im^W(Tavl<=9Nj$OO
z=b45TJ$fuNPVgAV6ds;xX0d7c^txY>>fIijPPnMDOt|1Q#c<oM*De$Mz8qgx_v+;e
zv9d711kw36Rug3nl@h&=>aZ+$>au!4htuRxH%p-|pNPT$VU`6F%{u2<Ubu>I%yG(E
z8tAuV<unh44?XjACcH2@er}yowupwwO4l>SH#;Al@==OVb#Hp5xm33PP3Ot~)35Oe
zFPM9>X5q@YL3cU>0;6SekBc3M@j7vu(LUGDJs@=RN^$+d5&_c*Ig+itKDP^(PJMk#
zL3Mj$&Qq6$nC9D|7FUbznp($mb8lPV7i4`cg3&87tRaO}fpzEa8L4I)|KIEG+1D5$
zxuQ4d#EZO;!nx9ZYWH?LShV|Fiqy6ZW!%}x`=9-L9Hu?<hQ<bkn%Ue2yUolrJ&dBc
zR++EvPfL2MA1$&edEe_A-N(EF%CDRbKe(f@l(&rUTaT>#>xJo}%iSJUsD(ucdWwYI
zxOq`td9i@Aur6DZQCH}TkH@tn+<$gXTVrtByW8Pvl~z}ui=$F^uVa_z6OG7^ol{v?
zXL<eSsd^RYIeD7kHI`F-uTH9*nz{FtyYRLfUZ)MJJu;taH}kb_d+U6CS+4Sq*Be`>
zhAp|bW5f9kzpD=%u>bqUZLWj!4AX@tZnzj_obCwQUVQp$|EH!WYinZ8Y95c-`ND3!
zU(D?TrW!Gf*K{-X9x_w$_<DvnO8o1aXOlUjb27L0Sw=Q%FK#)>`RGAd)0)<>A7@if
zi9a~PAg6Kvipf+S#nw$$g$oWV^L*?JFEBsI!?<VrIZ<Es#iB(<UYXL0fd}6lf68&h
z%Tz7QByr=v&2B4~%10^P6w;3BFp{4BYNHs7%97f*${gAY7#fW&Pv|Q(AF`-2<l=By
z<Y`zr!TuT}qejBX|1Qn9I@J^=aISx&$|YTV!QoB|i{X{SE^9gt&Di9?FMLr+SpQNJ
z_v+`}CsfpenKRiPc~bksHuR`f1W#x(au%-7IF^4z@=iuc(GK-<DYdLW9|-=vILkG6
zVtdyrk%S{V1fzE(yG+uYU*C|d{p`-Fh`*ow8=hDQoIIIget}D1*Quq)V?MSCSG{3*
zof4$;$Y!<GB`xO7W!H6Q7HAvYH<Fb#ZnvzoZJ)%-Iq&JF&5SoCS)$)_^<^znRxnEF
zwg^#dRo?vA=8-@?cgl{lVl2z%Mt#WDR@LSZ)J|$Me{jxe>4sL_x<tw9KTb7L2hU_A
zIdN-rGN~nWUi!}>FgxvAvs}`D52=$Zm-?e-9L@ORAQiMNh<i?Dib&$b(~DMwP2Xve
zn!%cwCbDIvUsHz;w|=7g)re<;RXs^5?9A#yYqo3+b@0r7d%1IQ#^+Vuj&1F&69rZ*
zl91}s<1+2ME#voUq2%!^PL47crEdp$azzQUB;38o+c0DL&X*GH#rn2!KD?JKn{!@7
zG<`|-UY+dqsK?KAVoq$p$;&4{39zhqt+>jES7`n%ftJWr13u1a!2&`ICn`3Cyvm5s
zoczDXEx31;|K|-YQo^<w%AGOSYV<<d-c_#PeYDxVWt)b2<BKQxCJWsoeibdsyQf)i
zxYD)j$0I|*Nf#Q=7N0uiBz8b$WsV+q4Ez6C`<5(Y`qu3KVfPEJ=m^`UkbF@Yne47}
zMxuck3srwy4zx4g7~bNy($j3_BL}%(sy%iZ*4w6@bP;a9y|<}TA@YrPwu163wX|b9
z_+^s~996U<1GOinDm1V6S|D%x?0NaZE1`Lxn2%dZ`J3O@?Ni}oa<?v0bdYuuui6+b
zdyR4RgvCi=1&d5|^RHfa_|mv!-6ku&Sz7JZGd`<27O<Zxx6N_LG3MAObeOHlU`F(|
z!YAiNbQsw9TX<(H91dG<G^_sUbF0lh3%s`++IUXpl-T!~95wF(HuAc@QIfj(A}yqO
zvaqddwp`-z*=~;gQ9hjtNeAL>|Ez3XD)?lMhlhIPjsFX$7Jti34b@)0AWU%D#w&|v
zwXFQ;aCh5^ttkr|^`cwP1Wf8&TQE(*Vd5g6mzEPunb&R3U$@98UrFTbnPt<qZt2bS
zl8U`QfzM&R%=<c4+ZS(a*e#~5k;!M-nQ^X>ZwljT(RwbS-B)I`uXreQzI68Fki~o|
z2UcyA-M!$Iwo%B{zVE5d;Zvg$c^4a*YKtEDF(u=PsX*uU8Im_r*o=HSOU|?AUx@2q
z3*)Z#*k&nT8O)b0(aJO}MIkwYUAOM`la0T^8hLC2+192zb3DkD4QLSNjn!yKd7RT~
zDRO0w@9G?5VU-l;LkcrqeB8fMwWrhgWzrl+jmgr5YTKC)=8E~6uJ~VbdfSc@ralIy
zsqJA$R_(hS6;`y2dybD^p8dX=dS%>9bH2S}@7wcoU2)k`_gAk@U3hXTZ-KkA$NyO|
zYUd|;)=YF2m~r{eN*2qM0wcGzsh(|nZk&A5z+vg(QlGwT?z1iGv!80u6H<v|Zjw33
zb@;)$RYh?j2MZh%?!F9(nrf{q6_N68&y&hNwM9akSdZ>;pBR_qW8^0i5V=60bHR=`
zzJXD!+HC(MciqrYzkJ0ovcW}6fA7hxzY<4hZ(7i4*wWjo@rLzp-kNihU8i-O+#<Iy
z{g2rhZYy<}wMX6F$pl4hRkU}$zwSCO)5f#5R;Kpi7gSB<f1kQJ#rerA<>EXp*_S&1
z-Jjj+G%4K0e1%c8_FK;tzioE949lck!rIU6C|y0P@c$%}AIGo9XfqhUZs}R%ee7Ru
zqRG{4#u*E^RaZnDvzYX*(@$`=*W=Y3$`k!qqAqUok7S*7NyOP^ZIf$Yr{o@mO|Lq6
ze|5I3NMNm+pi^{8{>aVshZLutcAgXRk;l=n*Yq%_fz`VGJ6I10%zs+c{H$w7AambK
zE~~#U81y_2US|}YCe3kQlJDb;bv9lmr+)5Oe3{?T%Eb29+<&jc@4t|d`?L7^OD3mP
zmQKHI(<M2c%vkU8NzDGR-NV(6#e)619@9TK?Yr7M-Arof8x7S^Z@avOR(=}|nj;L1
zu2|hTE&lxFhW~dudp6t_6D(f#E4yz;(<+h20>Tv>5}Q}N5n$C&lvZ;T@{O2T6=CFi
zV`G0r_qGEQXD?_soS-fz-5PkIWs(5v)P=fdHmo_EIIBfb)c=eNyUZLj>)xgp+Zive
zIDdHdg~uFy0-CoQ)&(2OJu>Kfa=`A@WLMAGM>SOXo+p^xlJ4lV5W8MHJtlCy<1X9(
zulAnb*q5Klx1h<5QPFO3^z_V{V>&#hWtu#<cJxOZZk?~({O?CUSGVSk$Nf(-`I2^v
zy!_a2adB-^$Aq}gP477S1Or8aXRZixStERFVy@u<)tNIhiY86;Q2(})EAtc2q1POi
z+(&0_(7w9EwBdB02-A-14aWb?4tq2OOo<m@=u4Txxsg>MM=#%Tb&y8m3#O&r)dvqb
z?2nt$$+u&U_!RLN6^>a?1ZL=PNSRCzEIfFH!O8xTepHF){U3|}8F9?Cc2cdeoGsm<
z65|!;x#ZsiCO!qb#cM2lh1M@N_G-Gp@ZjO8#tm+K&AZkf**WQvNRbThmLq&Y2doxf
z-sRwIa7?hlu)_J)gN~L6qwNK=>y<f9Ea>2?Z0^cvji0cwTjpr!g{Ggkc}_b?Hf`Cp
zs$<cCO84tOCY|g#q_@a;Pv_RKE#9xYkD7b&UW-0tCM1yb<me%fy^%h9uY~Nq?9)3>
z$K-F)mUPE0uQypJx$Nvct@)YjbfZH5hCgTjU-sm^tmx;oxcBzOX+KW5{mN(#{5*Hn
zEIXqaj_XhJ2PGI>lQH4lt$+2xzEYJL79T}8ZIs@;WD)+rWNgqXsJLzKYEI5IopCeF
zgTKtK(C)alVs#IX=$jq>2AykXZ|IZ}6q$QsyWX!wF;erBxef|^nYL8g^o9t-oRE`$
z7wzLrJvmdRchTXK)2e0fp9r{o<luro7km~wl{|F3zrj`LsO2LCmHQd`0-?Rhm16Uv
zPc11qm9~*zbr1hb39IXOPWsI@xy(85!9$g^nY%u|I`phz4!^XVU*#E#IbBCMw3!2!
zzP_TRtj!Un$h_-HpZ7^I_bgUlg>JKE4Mqc2-j8xUh0I|Wc>Wvu_Uc8=Kg2TCEIGJm
zlTXN*GY181C)}ECqr5WaCI1_lsbz*|r-?8)r5?QVB=8M~cmd;12jhLc*1qgVPsz#h
z@vCxp_ImQ_o_+k_iZiSCsh4N{Ij=G*+FeZ9rz>;z-o$+mD)?S*(OMH^>6E?o(Mv7f
z>^W~guP~Noo@2S_7t2vyuAb*k@_iqsn0(ef+j%0nb&=1OwLx<h-Skr9ec~ba=)}j5
z2PXIg2=3t*;!&Ml6TrvocB$nG<3Y_CO$XV}hRi?Xd-L;^GGAWdFQ*<AdtUu;)cNbw
zb2l$ayy0EIGi}M4sY_grH7*ET7IJFlYnCS(UTbU)7FZq(JUD0F9hD7xyci?>{x5%Q
z%;;#K?#pW+BCw|UkXEmE1EWOP4UUb6PwVrpWBlk4o-nQKfnwFA<ENhN>3kB{d#R^2
z(%ts0V#E>NcU_^mGQJC@UcVqSwOE%U|E!ib@2!O;*H0N9cbUrR6M8aPlht^E@2%aw
zsXvdVJh^rC@s+@;zN{~S?-~4>Jggjd^WEtTdh^2Vbfd)QpvBkb*si+Z_b5X~WtLQ#
z1hdGgwe4SYd!xGA{!9s;a+A?l?MegB9}|uYSL+EZ6Q4>=WW5`<LW=L&$qUO`qs~S;
zzHzw4J+<@P(Nm{+ug~haUXptKh79i&@4F&@r|`aHs?+6>40USIjYz8$yTvN{{M5x4
z8kTar)Bmr_iMT3p$^YYp<=Vb1)f-k`*3!r|FwU9BBYnFm;c&Jm&!nxV=XUV;?pWZj
zcr-9j{reU3_`e;q@A_E3UO4F&e*l-|n;BZdzSp^y%G@ZqU-U8T!r7R!i|w8q*z)Z`
z)3*<swOJ-;CWJ3;IXnNdW1(ympJsDnOt_1!mCxH-x03r}82eJ+g!t<AWX?IZnbmD3
zhxh}I)myh-xo0{3!9t6$*q}*HzDt7+`|rG+cz8;tQ1kQ`p$9|5j&7ZI*`hb?EqDA>
z&dk5vEz*nm&s<B}$v<VwNshl4X6P`_dDFyW8nwxU<6`W|Hx9Sfs^0a!nsoI>6vw*j
z{)SOa4M|sy-eLI`bM?-n|JN%`6y|8!KWdVFx-;QnZ}QB?r!1${&$XO6X=f(m+;fsw
zRmIL-Tybin>0Dj|S<Pd8%Bp<)r*Aw{4SJqBr(ZUDSB}ioMOP=S<(C&^aJtiD`F2mC
z<OPY(?H7KpxcT66SjW28r%#5f-dZZV<)>-Kud|P4d<wg)Vo<K=yR_Hop~2h-R}%j3
z@_oHY>rCaX!kyP|xIFnjH{>hRsii#~DqC#bUN4^Z>I$#f?L!X|jOBX$j5bL1@(Z@D
zn|x-@s#k}uJh}0BaWvDxKHrRH`BmEuD>#)Fs{c;8tZ%hLL2hxjsOL%9tuHR^m@e}8
zVr~>i9-rp6c!z!W7prDmT@bJ)C*xe}i_0ea{(A{+5ehZ2dAG{?n*R4k8REWy61Ki}
z*Dn6m^7A>J!1%(e!!@&cM#GZJsmp&qfA)YSY9ViE(mjJSVk;kuY5eWqa7g@xY{%3~
zembTcK5B=f6kM!qwpA7Mcq=Neyf}OA)_rC-L@k-N#O9>-Za(@ZK{j}+#3R$Yo?8RT
zUOu{N_`1dJtWeI`M=DzXp2qtazihJ0-mvV7hRMF<60vwj`6WFsmu<PVnCH|quD*;v
zA=l?L1?XNX?2Uc0BV72MEU)Uk;(s^)HpM-<>9;m}b}7%IUlPng9TPTOex9>+eSp=Y
zf_YCp|6Xe>cqCf$C}GK?6;n@Y>0P-zFE!@Lg^hhLuB^+xTJiQjhumGh8@DdIzU4b+
zv9oXGgR@$ie$V*A`0QVuOh}v-ck+tz(Z_YXFFP&=3WoA}X}_7A*1vuVr}MH?Yd)p-
z92F~ad4F~%-?pA7kG>?tYPLoN%4Hb7YT<bG?!xAboe{B1vRCX)P1HQQCCxKt-roP$
zoX%QiFx^jk&*A5_`Pi*Bj$eh+W`3I5`t8Nz%xA32VrFtrtJ!Ot%yMzo>V!FqpBdeg
z@$HU#AMmm%W8ag<nh!EQ1~jeWY`?Nf=(y>m=(FC>12dl={GhShB~^2R)=Dlbtu)#D
z8*^`p33zTj>zr!&;NP9ci}&4gdp#>DqjF(Ufaj+Oy&fI;4^R6v7rQ*+%YU%0bp3y+
zyKlwUe(HGq*oXJty8_vgP89~Zjw%19Oj{?rP$kM+^!V%4d&f&Qge@{?7C5{{o2R5?
z*O7{)Op1$}1Nb<1JXriTe}(mjecMVNz3^l@{bg!#LKSPl!lzL`V?Vo?o{Cd@;^6l6
zFYmE0FPN@JK8dik<eB#M=5N`gBiUy=^K@-gJ~I_RZkzkw?zWO|j$6^nm2=+Kx^4~E
zKO7^&`{|a|x~Npfgy-Rmt0R~N6sP;$%XpNKlc6^u^=6-4uytAZ)w7KXpB}a8oytve
z{hXWMTI6qErpNxGvj018XzrsQWjET&tCwHdE%w^)_(6ldPbUAbtW1e$>kUtKf1&k%
zu7qgcoykl7$6UV<<LSg-e@a6n^WNQZN9VcI<le`ni)*q?Us3$TQJ(i!!0Bqk!+{-7
zE=H_;deN9?vzSrWr2{h>bZ7Lj+)usN`6Z~J*<Xl{JN(VU@DJ1F3IdLQkMaAp)U;rc
ze4eB8yZa6)@5Ih66!I%ojAonv^<?2)Rr%6Xm57A2yC!-ayp8W}<VL+*G2@>1JNc=c
zng$%IGa9aFT$1p&d)u(8Q+sRGbFD>oXJzF@dp(riP7u|6!0=L~yi%N@aQPQi9aFs-
z`;NvR+OE{`LxXi?k{xU6w9b<|)``E^B>yAA>_r@VC5wG>;m1!6>)$q%Jy5QHzy5i7
z^M)VX|K}#XT)*T>#?4UP|0d4^?al?3Kka=`d1Y$c$G6iucTVqQY<}q?_EUe)y=79p
z|6lAn&f>f(`R4+&m8!WLjwDo0%q=mt$#K2Lzo9n6L-dO6nwok3|JoMZ5h!d8OpZLr
z=P4|%s#BNRUR!m)t+w>;oR>@;dNC{lcbozaOljs47E0;(u;5@5m!h4>fuo9tyII)z
z?o@0z?b5I4*u|6iNyV#GmbKxDNnxtT3>{s$n3tP;B&TaSPc!VC94x74TfUFSQdP;T
zooR`{oP<kf4=!X{z?E_Na_H3+OkPtJoTn_fzLv>jqou^R)rU4`sOL$&o~E#J3(M_o
zo3Hvg><(l4vTfp_MiEJ)+%K<GJ0^3p|7TQXnGm3(V0cO|ZNs!zFON-~ZKM*$SCZ=P
z5MZ=ak6WSnvQqohR4)(F3yx_go4d1xbR2^g=1x{gw>IPaUFvo3P^$2r-bv=Z*XLwT
zk6AaTlJoh=?qg|wu}e!C+1Qoto?v|W?Y)Tn@s>40MsK!WbzK(DG3oRtKQ-^z<^C7;
zG0YWyy88Y9sdcH0hukMrs+OA1tMbvbIM`_Id6g~0W^1~TW-I4`u6oHy7sH~9Wv@JR
zT%5phO@k>X$y<AyPl~suv`R>uYy@vAcbe1JWtVbdT+AnGmN%FMNKW6GxY%j>&O@w{
zbMFR8x;t_{=+bVt{3%)_sdZ)6RLP{yu&Hf7w_Ka{Kl0JCr8C@QTP`|hsBKxs5VECm
znLG2|@1oPProNQU)vZ+(cGr0MT(yqtqq$&hkmuE)fZ%B>uLS*b<d_;4BD+L9j3sgD
z^-QtEoe>!Yy{>Et<%?35GpJ6>NbUM|<&vK+%hpTR*_Lob&P<pwRcKlo^He59za^y`
z7bh-JR1-;Cn{zB$a)xVX#Ds{#!Z)FL1xp{V-#_ch)80i{yLGqU4Z0aWovqP);trAC
z*SS6wi>|87e_xm=6WH7)qOt29TZEgs<H6}ST2j3pUI}6_FG-)Txz>fNMKN;4g+r75
zM8YGv9CKc^t2K5M#4J5pn$*R<b=Os~Xr5(@wpd(z(H0&t^H}SK|9k(vNR?cAXH!n5
zu7Al*S&vt%7EN|eO`Fm8cU5)l-c6joQ%c;^rzci<EuWp5%^P-QzVq`e_H>WDv?Bt0
zT%#+aXWUrpK6%dXnJV%A+Pq!y3on115-%C6;Uy9*X89$*N&V!G%ZI`kqlHuA6gk8F
z<K~JMX5OAu6?81W@#u!w3c;|eA~9QvZl>ohiTRXqPE9*EYr)yH6%$gIeEYfSMPa}(
z+0(P!q$E{Lc88v^IQln`-Oc?+^!}Z~PpS*|tFP7jV3%kelbg->-6ZeaU){MkF7wV`
z>v>drnMO)<N`Blofx-hX4m@cMd9cel{l-@pg+|{AHJ4s4m%q*a@cTc<=2Ndtz5ajS
z)iUFD#=@PKS(a^J%stzyk-Or8qk+lJ7EXpjo-IO>ZW|Y_pP;0dd&7mr_exh^YYKOe
zhI3cUMYe?#8r^1Tv7V3Eu9M@sv{ZRZ&+|=_G{WyJ;<cX~D0!k_=FF{mTKNv03lF%+
zY+*4}(J7e_wr9zaP7y;V?<4b99Aa}_)R3RWcTwtdn3Bb&%v8U-uO?skw?+6D?-5mn
zjq;9X76?=cy7bs8KCTmD6}cgi(h#;q+40pkQB@fZ?MbgXL;ma#-x%U%cz%{ga#6;_
z4HFg>1kd3(8j#8zx-o0Qt(gq!w-Q#LRXNSK*5lITI|Vy?b=2JRobP682+zIb%3iu8
zvNym@+55l4H<w!DN)^50m8XqzJ+ox@L^+)*RfvB#$t*SI)@~i9y??Is&&)k@;cwT@
z$<>CL3KcQjvz>y>IXN0;YffC`-zO&Kw?!#g+32JtXVbzGt9J^$*>hGNVp<fubuur<
zoCj)VE0mgQ)|dnd@b<<<unFIKVf5RECu4!o6s?^Hd2LO6&KoV(Sl@8D!^cPIGe_FK
zr3;^$IYo2+nzilXvw|)4Y}zlUZ@9Mhe`Mj3h{sK{^LC0Y&+J|%zqC_aTZ`c*!-|<M
z2m2Qm8L9SZv@R9C_}EiK!Jsw7OM7ONvaaqY^ByDN<bRHHRyaMKcFfmBckK#STNQJU
z@SKC1v$i;EnhBq}V*1~EVY|qrWH<J}Yb|dB_FaDI{;6tGs@?MST#@s>E)d?kx9Cep
zi52IzY3^IPm!;GmoP94O@{x68L^^k9@G=*(B=_c+u%78>b^aZZm}$GQUuBMguFr$*
z!n3@J<F+chR<BwiutY7C!Tao`Qym%-?Hhv?JBlAfNl1vTJbT%NrE!AI*PV+lX&f|B
z*UsQ^G~TSZHO;#scdf8`QiR&3Ym4stXeMv2j$Of7xP<kLUYYMCUyF?|H#eI;^Vv09
zbyio|)-yKGj6YWkIix+^k-bObuaEKZExw6dTwL)%QzlHhEySW#-F!-ZagylDG8G?Y
z#?Uhks?#;yE_rR;z}{tdjVpYD6Vrbu$)|le*Jrex@>ts=*%=o&X`aE>WfhLat6ns)
zxE@`${kUCRN7dVCy$g0hOTLAMS1-CLuliqN6_b!ukYtUJszvJz#Xx0Y-_N3Me(Sc2
z<UBa6V>z+I^MZ34-$}-=MPg4sUDo095VDw^%HEQ;a>|w0lX`cFEa_Fz@(3_w&8TXc
z^U`y@k6YZE%Td7(U&zk7c+Jve;m==pKc<~L73|zqomy?WyEivSA@@^hNb}`I9;yK$
zhNXX;nzpTY;-cfQ(&s7H9eGV*7p9sR?rFcdJ^o2*m8D&b@PDJZY_kquVfSKzsHni{
zdm=U+Ie%$lNJz5iiZ?GNtf;(m+lJ}%%{y%>hMWI$`2|gu_MVugwK{W}<cns>qZKB{
z{%JNRJw5XHN#%46QP=xZ{fyr0y^-#`GpVyMOlXs@u#C;@xg5@C4zsRQ{o^nv!lF^<
zZsz}z-`Xo;Ii=_R6#Z-PQ{k&_+S^q#-CG!Z4k~Y7E3&6kXsYwt-5k~&VdX-_TmEcw
zQJ54JqxGZ6sK>EcW{!vSJs*`<x`A4U0&mR||21V>;?r>1#Iv$@44izuO0=93?<V?q
ze#ukhUhI3O{NmoXxvPZ4u5|Ww)zsd(*UnRU`LxNMk`H@N_fOyApXTCu{Qcv?pFc~M
z*Sz;s=3hK<!i<`Oih-|OX5AA0Wm7O=d8c2Al=9CbQ`0o8m)p$vU+8(Br}3)ZUzu)}
zZ4(Zka5`VMaY|Oq*3%NxIs@F7^qksN{dCRE8EnS0E<WU1_*Z4+wM%dIv2Xn)(mSn@
z`+zIUucy_=Q@6Tbs&$BA;L3RM&u97$7rU$1cPx1Jx%%si{11$$Coz3s+nPLI<!p0|
zdZ63E(54mgs)qgDHkvEa_}-?@XL^yYt&$)WcW{|EU+;weIn$Rlxcb+)1(-~c<cU4K
z;oND7W3mRGXYMdLy`33-)K}9}YT@4L%Xxj-(r$aEUR-=tmc1|G!w<Q`E3bGSxw3TP
zX*ZXaOEpoPdGZ`~V(wdKF+7QqzQbai%H@}GhVf^()ZQq+iu;ZiZaHp<F;{Ik!1LdA
z-9qucMpvcYQ;rTD8`rY_6g+Y!F|H%Tr(R=rtWM8nPVZzJ|Dv3>4J|?qDyKA_PMC0o
z>y!>#D2HqM9!JeRZ4-9f%eisywMF+E2{y?MZYiyug<ocWI&maRWNzMsSe`oGT^A&T
zUd~8nINuP_;+S)YZN~%07^R81i>ejly=u<?C}H_I<BE&Nd^R;D<DD#fd0dWbO8$K@
z-D%IF`y2hM0x!E-v<53E3N2!p+S$o4i$yZ!(l6hZxDCOFe@qj-B_l2*^ShU|C^YG>
zuFvW%hwPVjyj9_v@MWeGC&T##6V^I(y53Rrvr+I{a8-X>&zT6<){SwQbFOB75ooz_
zyzJ^R4Xgi)S}GpQc(7Qht7TD1Lj4@^-csJ)i&KuAd~kq8I520T?$TZ#--(*@7S%6d
z_c-GpuF1k|8u#zQIsYk)&JPyl&TL`Pa9CS<*-@k6q~nc5SC6fNN4f+P5;D#(hCUR%
zcKX@M<L#CA;&KAaa~mf_aQO8EO*+u0<~w(7#nIFwJRvHaSFhfaX7dd{abDV`bC>7T
zw+A2Y%54oj=XL1AbY005_oER<E0=z}bFxw9=q*p*DxLdRx6HkMfNjr>psdcWufCUW
z{)oMo7=D7)Rj)}tZ(<TVE1O%_$(0q69tZBvyt?GbnfR41$+LHMiA3;qWt@{;q}=V2
zkjfLHt>UHW(y0-6JtXCS8mI2*^CuoU&NwK`5VbUd%{gewl0UOvvAW-!sI;V0q3q@q
zZ&N9+uam;VCUT`+evmmaTP||%l-?O<(n<x=ji)-TyZIpckHQVv>67kyD@(SV^lcD&
zktB2_iphuNQ7r5IGgmKnx=anYvUUQi!7LZiWiDc~T-r+?Zpsa?H}sJ1Iu_V>Oudlt
zYAIXkO}QX1)l7y^&XA`e5|jL2gv^*Q|4yw!1?MBXpji>0gqF`?xhtCTQM4u1@bNvv
zmXDG#jtxmaI@%Xc?eFQ8x-mCQr>nK)o2=B?ZqE}*x87dbThJ)@ZsN2ZGVwkw)8}69
z-NkigEvvM~<C%>eM-;@L+-W`j<f7P;{~l>&ZQ2jwCVyk;a#6@|;*4rko=~#P%jbrD
zoVTxQSdNp*t`pN2>O}G03t?8eHO1qizMY2X#<{m|Uhj5GF%<Rrl@W7$M@#sg$$KNO
zPu;>eIp<)hR>RuXB@8L*juNXFSQ;2|mNRrb5i%8RF$h@J;_7{hRavzsY;L;8nI~<b
zPsF^=MD;X@-;0mS3DC}I5DAd2+rpH6Ky-EDG5LANOp~re`?1&SD&@LJyIJ-J$Ms9z
z;CrH~apGrmJ%_f4<)S+aTCOOtmgg`YNqjUj;#%vy&SjA&ozvPkE^Cpv$bWpP(l<e&
z`xo77@2Zx)<Zl*@^ODe7`uEYJDF+RVc>W!b|10?asBNQ1SB;PT&L)Q3WpO=i!mORj
zg4#J%3ZE=EGSvj@Jv}A&I68AWFg#McbY{wtO_>*6R(J+zTwTx|sCdUM#id|fX8+H{
zIg1tvS8+=03|?^eNC3-J{xq3}DLgd`UN4ekSv;?0Nz4-eAH41b{+TkDypwpASFO6h
z_SSEYw$~QN*elI`PAtJQj>${4z3+(P`4#o(Mw(mgWPdx+;C-idO>}s^L};s}#?eih
z*$a-}JL~F_<K)hf>D6#tOhW!(DN9YonoXAkJCauXSkazwlhfymxBHAmleV6}`O7o*
z(xY#l%S*W$ge+DysWvWF(pvOQTyW#GpPD^01YIM2`1Z6N=}`FpY`LCVj6+9apX(Ey
z9^E}}51xA*wc?$iWzyj)^|_75WOZH!%{zAW?!ooPGDA|d*L}-#{^Sx7b;svJ-a=6y
zd!zObD^pH#33K=9HWglrnztxKAXjgxd+kThz%9~6=Q!&YzF7ZqWyR7E$sd{8N7i_<
zIC?p(^8Av$Kr+aCoy6SORh73mzj{r!QC$1n@HSiFLHoaou5bCoFTU(OdGyDjORkl!
zyK`NXUe04QUiy}UyXZ>Co2L^N__p?JU=wJ(Ui>b+@}b1E8_5;9AGY;eSDbWePafmn
zjk--9F1I(TPRL~c(a^qZ=ZEQhdSwUL`HEfKOB%oMw(6wCdj!mnJ-6s{(fw@3|EriR
z-j$`P3yW(-d911roMj-ET)*adR6@rdsc?@A!RKDGhD~hJ^;zsR=N;RLMB!Xd@swlS
zyn^r6e)Rs*W5(9~)*xq*#e+>pc+RhIn=<c)tj_}39My>`AKw4U6FjYX{V1oTi;I%>
zYp!OoV?}FbY`C%`=d94gMlP|a$&qVoL+9{Kb2%2u{$Qav^H#Nn3HfKNvKE(ZaCQ4w
z$#e0b#o~+0SiVLXZmV4PTl0v<2FI=|&bl>IBmZ2^c*4b&9Q|bC?U_NjZ(^4zt?aqp
zHR;Wi+wU@G-v1z3ziT0*aQBuO4`q+v+sRw3SiiYChqI%=;}+K(r;wyu(@ry7y%;#9
z`&QHuwUYnr&L!?A_T(DK-ud6KqT)k;yw(cd-zV??)4jh^>fx)o4`=l{R_hCx&nziA
z@W|<%qf1_W&jsP7M^1m#=#M&KDVX+5@Uh6E<-ar+bJ?F^Ja~Vl%ejg3*8O9>a$)V0
z$0crOJvw<G&Qsre$>QIMl&ZcncRmIm5wMKA!FZ~}@!=7_>Ps01C4@WY?tXdo?UaOI
z{!bs`#6PZLpZiI2z6GaGj#t^iy9Kd2#;?|y@ZK!C@N$mN*Z$D2?{D4px#RzRoA0W<
zt-BY^WE1b2|LXLkWt$djt@2E8eXQHFVCubY!=Sd*GY@lSihk4BCaM;7LRebo_Np1H
z>?b+~&T6Y|nq4Q?V#$|rrsDslH20G~Zh6&3v1P1TST|34uV9O&Olkn<iU~0v(!IDt
zOcdFL1Ao+Q5Wbfc_pr>hu|V&oNG$6{vBfNw=UK~r7!PR7-y(O^Qa{FB!_|f7&X%G#
z2DO4|iTMXD+A~ds{eQ~VuF<%cX)ZY5rM!Ln`&OC%Y9Ae07#A;k@2(%1eW)j~-*x)d
zYb=^aJf?FnZDeFS=qh2ZW<Kc<OToe7%uCF6p7M{6=(0pfm!(bE(^YqiW6F=0a@^k1
zN)~6YTndPZb+(<7#pmfHx}l&+(_;4AtuimxU;VyJw<tA}J%}?#)b`UNha>q`F4YXr
z=S*7D(JpoJl4jkFBlo^OC_H-9zh(9JRsWM>?eD~DH^fyxxPI}Rqe)wMhU~V~9`Wx_
zH<&7~kki$B{YmBEbA_+0DPi+8WLr3v_DFRF)d#g^#`sM!NKL)L#UYsx9eeWmm$roo
zk;08zBW7)VAIZJ^1dDo4utC)vhxwb!RJI2fINIp9q_SS%(tlmyawN_6`|MR$%33Ns
zIX)Tv%`Nj@a`Q)p)9dzIZ;s@y>FwG!P4`}@$j&>@)Gj=Y>o}}(`=@7K<OHS(ddVuX
z0&R0zV{hC)KIMj(P57A~6HVO|{Ei3ocq9ZY7CmRs=@{$v^yLM|4O?>ee`He&Y-;QD
zJ}xn{t)Q@__s-PbJA0l6aJ)WP=aP3LDE!)yw;Y?Qul!%>b>~&%8}qKs1z#!z3=i%&
zw&}X()ZTf&i?d7f=d3BM&~4AE?3(88v9RX0=JHK@E#<oYeU<1^fBH1jCrk0zb^R(f
z_3A}$bdS$ZzTdjC>8Dg(?(;3m8QcwwWo|t^anV1syH38^pTxF6^4N~4ZvvgSZm<$N
z_j%j(zc#tTMQqI*HW+2_Y-^jlQL65M*i+VDUpBTbI?a+UI-_v@Ooo}Nwx6pby;kpR
zIKO$5EPISm=0uT9)r{@;`fHZ#-JZ8y;PRqdn_oCpKNVbU<+$RzpNU+QcjD7W8oOka
z&-8vdR-X6x%cj4AEZr?UQ+Q7D@n$s1Kj;u`TXLgkS_)gsTI<l>xpwdVPrBij!}Dmi
z-$MTnsT<CTpZxE}VP?j5WG73|hPQR$F5&TOL<+7Yn|YMHZ;3kggZumsw<99OyRURK
zKjpi`66ct7ith{muAEs1!zSrl&iuaT#UDj`{)y#x+J9#L@eI)4XqxORl^a<xRrkXt
zefbuV90%=+OTlqZ9|ufPVm;0Jdd4~>(~$Gu-tWI{_sMd)jQhTaEtm7yYGRZea%LDg
zayfE(sc=|WG)`e?XWC&Sa^R(+Ylo1&-JMOY497W{mhrTl=v4M-j9%uUqbc-w3a{+)
zoFg|TKAzSg8()_5a>C)mvxRnkIb%`$%)i~-vb1nUiRSbUk@~2b8=Wf}j6_ySi8@UE
z|LW4Jc-b;m>(ZRa&3vb)c%63bY}#ymIz!gGEqHZX{_UF7uh$ae+H>3{N|sK$qG85=
z(&SFk>bbg$G;UAmjNW?Y@F|JYJ1nijUS1ap&7byCM{4b*a}vt4<g~?euU<~wTBLcf
z>RIMBuVnig-3KOl^LMSC75H|8_njRo+X`kZsyW=k$u*C=VD<y;uPp4WCM{l5xM%vN
z$V_9=GE8&V3_j<)W##5);U6E~{QPwGr9`h^pFTdnzP~Et-N&`&-nrU~yeeO;JI(Te
zdvp6OjSW-Sq93;CZWrNjSQI8#^g%IvjaD;{5X+A>%TKikG3@DNshBY3u-_4mJ%?qO
z)*ls9oh7(&ea3&8H+szJNgth#v1G72^AvJT>^Z-0mCwoUx+o#XQ)>=onD6+;{Mz%h
zn9l0$+n4ohIwc_gi|6FEQyQ0jPVOq!TA|CL<iVgR7j#IGb&|p=-S$Zn7M!yZJ+;}N
z(@ZnjebI|W-22X!T=G!=!c&wnx%~~Zrk!iKm&vkEB2ztsegrLYV({Nza#r0`@T$Mo
zDh)4FSCO4cs}IbKYBjwQIXR?TWZS3g=56ae6pPmEP5WeerEQOJ!iG&|H@OcBn`bT+
zJ)WlI%*C(%d5s3c;WJ`7D$cJbt<YZkM<~W%<&kYVEXRGmZ@PKx&o$l;dcl*rA5U8y
zzVNx-q3df`t(d*?{wujnehn%j|2-18gW^iRm<uoc)z$rb$ql7PpQjXj-uvyG(EGgs
z?~89FM>#Gl^?vj#d6M~NC8qvG>VKUIVzyfB?Bcc)EpU#Q9lmu9k5C?yVN|=6($e0r
z1I%K|68(HX-I6<R6<*krQFCox*sYbWi+qK=bd=|L-;m5O>eE@AEgb!L<{OE#(*gt5
zX@3@bQaJV4CbLCV&X-bmp7z<ua`KeR<Yv>`GAk4`RT_d99+vg$*wu3HiZPR&eAgqz
z6N1ZY;*SVUZ#s8qbN9v7=L4Ux*yzdmhFt#TWYkfh=@I{Op|9Dzm&#M(ZoN>MRdJ|e
z@thk^rUd<|<Enc(Q_5|l;#$A8E4bHgVv5x1Jo4Y`$Il&_!K<fmDqQ#Z-IBeg?Bqf|
z(GrinECOK+i>Dqv@n_F5^Iz)&xs+c;Fv?xs*c*5L-lA#gvROx8Ni0|0a%##U(cBB7
zQBse%tj~CxO-oeR>bFFj|0Yiq=d5t?BPL=yJXy>q2cFII`MjjVAizsi=G@71O$RUQ
zcxqM}^Kni$X5{lpVCK9TUZU47XzTgFitnZPV*z6>%@}pAD33#r&5YO%eHBptKdD2Q
zd7HE5jin+hm0DNHs48E~ZRB|`xv*A6&G3uQ!lX5iA1x_Kx0rfFy=+TY$PzCzr^#=-
zsuC6^pS0kx+tjm0Nqw<b>OYGG^Dl5HH=mjmDz#*_t^iNYhfV+Qt%`GPyHldBEckHI
zg?mj~_Ro2%Xt#<>alethvzl(!whivOEl+fK64%yzE9lY84dmL8dOT~}3ilYV3#>vD
z5;M*@skUWB^6%cVP;S3SgKpGDbN#)k&FPtHb(ccd@qIn(Y%^JfebVDejU4ZMnH)Ic
zpKMW2mSr=D>TqING(n(bmN8S%g+q(GPH<IjauQA!ST<qHhaK%JHfj~k@|^c4W%@bY
zTwzV&xh*Oo7u>HdQB&kR`GD*A&Xznboku@UNEt4h_iaxL_g?QQm!4Ei7VHaI(y^1X
z*D3gr-KtfgYcC0W;Rtb`8TDREIG{;{<x=4ClE6t<Cn*Ok`*5_c>|@<C=lkpbe|o%c
z@zwP@9?@Y<vySf1@NDGyBds>;QBrr*jZ0#wv%>%9Je<*HvpRh32UqPI2e_iGmWXBj
z%-a3QcwM>^r$vcr^Q1Em4v3e9SQ@WM+_ylvq5BheLd7J%Ip0psICP~+^w}9dZ8t-<
zupULO30t2_xCM4sxlI$@r{Jpl%8{*sjmO^F=g`47Mj9*U1Wvm5@MV=;*%a^AjmqYs
zOp?CsQXNyfwzU_XuTfyVvbaHOo4W8kpO25X?TppdiZ8r<DnR+{POBGng*TbX!ps|U
z^BR0_?a}Z)u=;MV%o@ihQ!+1F{4`#6Hf2if!zU6;!uI`_oh%}x(5m_U`I_IV%S69&
zOiH#f`7h1C_2AScD^}@l4NEZEz%+G%Yud~rP4mrb0uI$o@i0D~qL$?q<a;AW*yW*%
z#hJ4y@mXKg-TZXsbL{c-cz(r2*UfOomRm{&Ogq+yB)plRArdIOb_2V%+rchh8$YH=
z+T9Zki;fg~t&@$K+|eld%JNd7;zGB^SRb`Up-xdhuGNd@`z1VIaU-)!m1#$;dsO)R
zn~5#uB1`={y|x(Gx?L|`y6JK7x=D*ptF$U{CZ7tMX08y-`@-a+@~JOrhisQ#+*09k
za-sRs<jWq@Cn`wqjQ@XT)AT({{4euM%E$8w?|<L3R59nk{myL<bTWRPcL=GyHerHh
zz@~5J(~cE-q~AWs)%X9(VKKI_*Z=HZuQ}hg)ki`_{N#kYXY@E<96A^nbZ^ZehbLdI
zstTQ$o*=5wc-rw|yTnYsuJ4E6Hd?y$p1Z^(v_-+6@rUzWrVb{pw}CEY9F=>W(sYuK
zKTz*vxTwQ0)k$dA8n2)`t-|4dK2^t@KHAjvyf@M2{Ncja(VZ&{gB0Jr%{SJX|NFqi
zMf(LO6iv)dw>1Bi?5rtwBG=faUE|jT-RudSmyE7Asy5sfapG_aD>qQTu92C_sCvOi
zwnRKMwa!6$Mb+<c?r^S^H4c**)Z4_q98>YgT4C~3cao$dtNdlLiJ8t3S08n4XuHa{
zG+>#@*Uvdw%U<ToebaN{XFuY8zVpBSl-uV!fBqD;^nAB(R>y&E<>>ys3_ss*O*r1y
z$fF*%f!XG*Otab+g$7aeNTnAp&->1<WId5Dwz<39z*_IGDKr0EJ<Zb`?g`VMdM{ma
z`bB4km)zD1{b{T0EKg4H(djyNBde$P_fO4ofzp_L3M-6tWG{S)`z7ikZ~x>+<&)?-
zkKzp`*;CwEyMvm)rhK-w><%rQby1Ofo|((mCh<jCnr|k}*gEZN4PRa>OQTp=>{iD&
zZ2R=vl$1;sdb&lQ(Cc-+)9}Me#p}@_&0k(^c8A&+xOFNt3ZL;^+V!?bPN3-DQ^xr(
z6m>56*{z%*kt1@3vo2zx;)DeqUW+<j9bu7Y;{4C1#x;rIb5};vF$eBr3Xx4c2PJ$M
z-W=R7vq<hwBm3Wn@_sAB+0J!oG$|)N3)FaI?eg6E8ncSdNgI`CzQ<lHd%&_Ss7YBP
zZ~na}kNzcp`;z>)g<HwtsQIC#KO6FQhH$kmTD^0R6T{#0=BHe=BKW+e7V-Rf<QTTj
z=*!#rOZxR!cyVx^bT`qok4pTo>GAVdYhKKF_FF<_Z(ruiXREKz<C`|`sMskz-3LoW
z-zYft9WnB9G(NM;NJ%M9;G*}jw8LvS3$}HC-Nos*Z{_SiErkVMitm=pmU(m}DCf?n
zb@K&Yi>WUc|IS?0BB;D1flWSNZ9>kck~g2qdXAg$Zw^Zo6+8d=f7eR6Ukl}{4l4dy
zs8#b&v1+0GpM&x>3uoL|So|h`!Hx%dLW={0vP!sCDJQ6(2}tS*TV9a#NG)ekNx}2Q
z+cLSo7@j%Nd{%DJ*$yYmCU1rZK2Mtb&y_tjx{~xX;;B-|Q$==_%0o)aJ~Tg3QA)J9
z#1nt?-MK?fa?JHV4u$t+{hsyivfQG}1}p|#izB{VRDAY4!sqSElC?Eqy=zZB=Z`%o
zy5XGYo>fMEEz^JH^R_d^op8A|$+<1)(5*GT8x{yLvpOUk(dF8e(QeX|@o5Ru*#N~O
zxjGFRI`^H5loD8?H?X>I=;-obGv_|><Yd;aK!!iw?43!{>4)^5PSSXGN&Zcv{{L)8
z4qiRiNdlVA=}IxLt;O`Vy;!_mjm=iYZF$0KJC#;ZWwys!!YKzf1Rs0cx39(2BKVxy
z<tGjPKXy&%pVyQll;*x<(cY-$YKKqS$DJ6rE#QrCeBo6%xu$Ol%Yi)=>&~SV?LEMh
zW#IHuBH6>BMN}<;dxfj#qL(?xmfcio{{QIFtgJ_ec^-)hK4*ORq~lde(uJ4e@=n}E
zZ|7`D;5J!!TE)=+Ns4Wwpia@mI|nw4mQG}JUt@PL>FA7)*5>@1uQ|-Su~6&<<Cdy}
z`#)S#{KKfj7oxOnql0{tqpXtjUq+dOhBB5HGRpWCo(b%}@zG%an^P7|c5>@Z?^@%`
z-=}8OrTRZ*eR<HMMYArPQPB(J`Q%)CKG9?87X=~heNLezAs43byPitoTYmh)E}>t4
zmU{fpO!(2Q5nvRup<m;M!nY^OwdS4DUvlbl*`)4wX*2y?;uM-#UbQmMxum#m?S@$$
z8p}FF#55Hn60;c29owStdD7C&EDqd8+z}5}i@LpL>c6t#+ZLT4ZxxSBDohHTbC2c0
zl~2Cg9oU-Ju)SY9uVRAU>O+q0A<|EcWPWbC9&}o(<co`jlXX@KXYA_atfZtP9Z46O
zOLln}c58F#7bkmM?dAKV(Ea+mU2)LQ!~b_};I8L4sd%Cjwv6|Q0E1$|ok<P}%Z%(C
zjS9cKe_^#!aQ7#({Z5YmRhlgRE|<0O-OV@gZO)?Ic54dnEYhtyp)2C9$kA>szFe%7
zBVf0&&8vmdhW+gP`kU8Gl{M(z=(cc1+?oA14vM=i6uq(V#J?@JxrT*LoOKQ+9p86(
z?!76Wd74Et9@;Z+kz)&CkeMRtvQYd`Aj2j>okvA?bEYQkH<YP8D4(-XW<Q(Zh7JwB
zQ(HxzoqF+{>;996J9zx2OelHbuEwISav*09OZK*;c7@XAL3XKXElxkYyq?TSQktN2
z;oq7{E5-AZl$4Gv65hq&?k}uVpY(H@zgEGNC(FJi?^jCr?<w{+$#egq@H1s<Q@o<M
zGYw{}ny_R2%Puv8C0$q7EEC;y`W937mj4{tdL_R1w|&yxb@`y1BA44j_H0LPiPyfo
z`TFN{^=~%bbt>SN(Bb-*tHvW{&rzD-^d{i?r;kZzbaOX|ZOdZ+T%@M3)lu_X&5?w=
z99#4fx9g~wmi|nVmY=Hs*YSpylbZxvM$pj!vqlkt$;;)ICf8{9tT1pB@mnr7Ib>}C
zx4v<n(d3;zY&ph8_J0<)Y*UWt?RzdEs4UX!mtdUyAzh$WScii%YBf`4QHo$~UO>j$
ztdLCgsb}t9&|I{m_>+b@<B`mha_U?rjYX#3AAOd@NVeP71}eQeQvWZQK`NX*GV%Do
zVgs8G_NI-ZEbGHf8hLI#y#LQc@yItj)$>JjH_rQS#<=;?sn1Gb4&o)H=}L~aW@YkB
zZKmB~M$;S<cW4z==%{?vi<MA*6Xc{^l2RS8phk)F_X%yYsh$f=<hDBqZBsbyptO6>
znZT&!wn|4kSywA0H7z{z=4_9+V*mE_M=Z(%jy-nq@||=|W5V*AT%WV&_PYzey-*QS
zbwcTh!zHHr6SqIOFa_ymoMXye`{mjMuiWq=bKjJ@q)&<(5|>k+SjVk>IcuHRuAD1Q
z8$`_(ifw4K?)Kp>%2S^7U@q50(K{>s9Jx2!Wb1!g%Xog#rZrPVt2V@)Prvu%60`5l
zjXdTj%ozQY^pE6BEc)WYbtPzqtz}^P27`t*0=)cN<eK!#cWD2A?a0Hoy*<5k)>mek
z>%20R8xHUZ=5bGU=MxT1+@_`T?%}ouF7u=n*Oa-gGM_#3(>0SdBx&n9l{38dVo!=n
zO<wx&K7HXOrTgW;cKt5*ntoAxuQjWD*3LfrT{rHD_m$}yB2Qe5xP{h+Bt<MVWXU&Z
z$qZb&ApW^-q|Y~r=V$k8%<wZkJ&$W*-ItVz(~mZv3jUzdK4me(W;y>E&rLq5s7oq8
zpL0i7sH4(eS^LDZxAv<GWz54E+{Ba<B4p$jC7144`I%XN_eOb_L>?8B5Hm69t%fQ4
z56x0GP5FILTy2VBgR9`qZ>Lq9oSx-OR9(n*a_ea>c2$l=-S-27D$LH5Y*YDvq`5rD
z->!qZWJ9=@&LcIAO%4TLlrC3=F1hAjJw>TfIs0Oh_NuTxP3eu!ddz>n6fAS#$YfRZ
z{WO6$Q0dbnRq;r{htKuqX}Kxg%#&4`@3gb1oM-B}<KmNCKbSb`NxpVa<<@IFqd4I~
z9P5Lo<}>?0P5C6J{K={E__L6mvXAPcFWj~HE-4yX@;>*~F+=_5MLI7gmHCTzq)O<g
zZ{M9>E^4CaBz&dJFll$_bv@-RL0fO`KXbcXQ&dMnWbf9SL6yCmMcXQBLJZX$gH#qi
zPFs{LCA^^F#FtAq)gtl@o%MP|jY2&9l)e4z%O3Z2J<rmfz47Ovi0_LFR=uj@TkKg|
zyzqbZl1SAi^Zb@0q93){CT7VTwmAH_W7!J1f4<x;nq1oyw*1m#y3!?j?c<D^JB`&o
zAEc5+f_k%}9`3qgZ1>+&@?&v}VgBcNh9{N?Cb4dbJscG@Khm)>qV)Oi_MLxPQ!UO+
zjZz9z{B&#g8R;u~elu=im?HM#p>%cP66GyoE{&q=qRu{L*55xdX?<YI(Tz^hO)Do%
zs5`K@m*v6!hs7?j{y{<$3T`zesQgt>d0=<!wKKQV;%^(vVoc7Rif~H}v9}Rcx*~B?
z%_yBMh0pv`AG4H_@NE5kWk1!v`ir}(d|$i3zgwmA(61})RZ*X;!}b(=9bb7O?GN9X
zsI?}AKf}4+d2{{0(7AEf89BCfa$=c0|BCf{YDzp0PG;}pnXr4tXTLiOIF9E>?}$tk
z&3MT6c#_`UTUB@WiWw~w+b~lj<L90A6IVHhizKa)|MNO&Yo&<G!}B|>LpZjupWUth
zdlGZ->Q39|6F+`5ewfm}B3b+El7zAs+g1e_9_G0CnVIMCVT~7yxu5KO@?u4p<zt@n
z65&sEo_ai6!)x6+bE`fN=dBK>JF1T^G8~N*Qf4{$c%@CIs(b$t*<a=#lQMrzy)Gkh
z(=cV?txlddi3utVq94p&9rgdDq-G$;`e0qQ`2VHx_k?ZaeQngmR=(V=)fc7Ix9jfi
zIbm~dNq<&$eC^1s6dv?4lp*8bvHvDjJT8v+p5M7uohW*wLGH+{h&P!XE7eyYj9SAe
zTUhKX+bp%Y?p<eR;RcPremO71%-t`&bYavw=CXXjrTnsFn^TJ$l+=5^pY=L)`8khS
zACtLX@S(&sC9A`krgJu}T;#ATY14H7i+jpEj0<Xhvb6la&@nS)l3~(ip(izEwm&_0
zcS}ermmHaRUftk|SH!pF${jh}Hx|UGwoUhpD0rGIA^O8NOHt;a^qKokL9eUN@7-N?
zG5+oTJ0+Z3<n<zE8#c)$oq6z*@!Vg<V+>96_E&8&*{HOlGt9o8DI#;#)(!I;I)zpJ
z*0>ucyLB^J<h{{gczmeppWCm`-_Ot6AGX$G<E8%|X;)_WE-~S}q2e&fjWKwNhsTD=
zN&#K=sw$dQA)O4G(KFa;xlCV)%nMo0^&n{~cZiY5X_voR4VS~$TAysXx?(H0>sHCF
zvr<=IeC@YG_^91eudbwf3)2sJDO`P(>K2sKRHQp+;_J(Yy3Q-9U5z-fVxO?2zs%KD
zb9EVYBK=f~O*)x0MO=eJCb&3cEtL}02vlJB=ECWc?LTGh#TCa?qHfLAIa#3O+$P7$
z!oG8|a$)b$a=CcFo%u(erT6>G7;{+^F|tOwo;l%lW!1;UVH20Ar-klseRE{PWjR~B
zqhA%2ne2C7n7_Vi>#dDSuBS8%6U{ICq&RHj%-wbLTS(U0PPYGwS?fF%SX-rX6%U9>
zWbI&dm33-;7$R}%`ZTvPUlnojyuc+hBGjiTG4xpP^tjmhsnkiZ&-$fDQh$Kzf$0JM
z2i>GygSKp$oFKK;Y0;zyflE@RFl|s=&|vK=CNjCp@mQCxPgcrlHHTNzW;IVanD#8!
z?wyFMu1etcv^hm~SGl!UOKiU|OCr{c&9gelg}>~8r;CvF;+`OGYsJzl{*`aO1ou00
zMT*~@|4Fb*ra)!xl1W~FwocB_UGpW~+g7u2nRGx6tK}0HPK^l-ft%!1T~ms)Ql`3v
zB$x`WXIk;~+S)_sc0P(x`&T9Ens#$Vc(4D*scCL&f|eW+YY;S3P0{i5{4ba!WqI>$
z_`OLpuDd*S`YGwU_R8kS`gpE}SAoJ4kIiEEnkMqr<$I8H$i8pq%-*|1ZFqFLe9Bj&
z_y7+7C*FrH)owMY7c@OnFmLad%cr#-evSN4b|&=8M7MtyA>H=e+zkcxt9u&Lbsv9J
zpLdXFXU|=&$rD?QT-DQE3eO3@T~iQXtKYNHMa_Rs{K5UIe`o8jk#K#I>G<%O)c%co
z)ugTo$Cy5HiB13U@VZE;tdi@_AVWd+y{l}~)l+XA6;s!0a}CriaxGPgjMb4|%wX|I
zV_J{!#-oonv~3p?$@Nk6ytzAJ?-jQlK34Z8FiZ?Do4HoXw{0myL*%=i+v1ACx*3+<
zcKYA7X5F-F`l<h#M02wOWm05I3)o6`7k-uK4Sau*(Y^od-aV2+x=ey5r#>)SZmIU<
ztF95Vj@lD0v*giQr|y*3JSm^6m@{4cb-$RTYq~zQpK~WtDAHuYvbU43ymZ+Xvf@?v
zWv9}RHJ%w279qiZ{ipuZTwQ;rt1GgLRdl_GUgX|eVR0WW>8uwi@ZFl_Dt+#$^4Z+e
z(T|(LcPzMMSy&lhl#}7Q`_@Cr)2fP$o<GH#ue3;;Zp>5vJu52j!_w(XQ(J<pkIX$+
z=01P^MrXyX60J^p*A72O=m}KyoWgM9p&DaH63^=f4(BSx$yEu3y!SKtzb<eVl&kZx
zSZ3UD@Wc)7vi}>BWY<W@FZ5G8CfBq^zmVtgtZUZ=pFg~4Q8VX+{+o@h7w#<cS)~{_
zGsakI^9OxrrWXR@N6aS11hz0|%;vha;(*!DH61z}XE<WsO%qGGn5$H_I576grB=TS
zs>(LA9x$<b%w)LJA1XBADevBIJNJ2HD!mU9daA~xw;@1NE#&~G-@K-!)1E9<joH|m
zRyJu@#9jVr@rp~{+;G?veY3Ob+2W(sP3BzhTlrJ#7PTHbby!1lQ_|c;E7)yW7IZ!f
zTq4G~@Oa9c<MUfmg8h<C@SQuhL*q;4lbT|sjHTNa&-woLxLEX$&WB5WPF|bJGO7La
zRk51XX>3_jW?hU-jknTl*j;P%KT=L~!G0g(*}gFoX5Kk8$2(eT{`3q3tuD^Q0D~I>
zr$2m}`R-BA(<KKEty-(6l~y&^ElXnRs)=f*i$5IotWjg=dfDD1${6K&&6zt;=7_KO
zYj#_aOB$aIj-;*8;6HOrU|(s+!4jj!MGSWy_kVl9X|tH){E`y$w$PY0eJ-atWQ>(2
z*Ol-ru(dvS(t^q4XTTX{vB;#_dEcijw<_KA?L(_=PGEP?$4fEux&<ZUgjaVin6iQ?
zVQRC$OjEbMjEOQ_r`n|?Jv+Tp)pJf~L@qEgmp<Cdt#>Or;*p!VOlGTaOEPy)LCW^!
zH_FtTs!wTKM)@|#zL>tD=K5}>>)sbmio{v`pLxd6TH5No;Y!{af^#{2+vC=IZr*aC
zx4-$<lRe8f<oh?jIxes}VntwwlhNJP?)Bk{T<e)4YJx-<EZBnt9<MBx-I8$hgOJaj
zduyG9ll}(B{-3#TMw(K8y%1+e5&MM+XEXaNuiUNk3GWV@;uliI+0$)oWI2D@B8~QD
zjZ8h3yPH`Bmb?{xz$j^<zTRYe-nm^@<D-Oj^7%a}YUAW$H`h3{{)ULw#B3i<)7z5D
z8v~fU5`B3D!#x;9UQSqk^?IPO(iK)oO(y<>{F3ZJR{2sh#9mAAyL9c~GL73Iwn&22
z_R@=aach!Ac1Pcg)V}Vf(-q2)Tye!pO4si3PvwBgGtd2Zd2x^{?_WV*(WH3}pQapn
ze)Z$we%{6I^B&J>iwt9!zV}V|t%QSNI~=bG7qZUC)3rDEVVt;LFn5lU%6j)Sj~tP|
zhPjM8Sou6Y=^Wz=b9+~|eZiJV1#z0qTE<06a+$kN9JD=Me4Bg8ialOu7Fk}}$~XP4
z9B;70{Z=*UQ&)u!PC4y8`{y+=r`wO^avNt)JM~KHuyI%E5Az5AHq3nIVfNwc7iSxm
zoB8&eq+A0pB$dBe;l6~a$8hbWmfq6PewRbv{RHfv`fqD~Q}b$Kt7qq&E8?79?PrhA
zdFUdlywq90+wydm`0CTk*Ga^#E!S&^=2&n~YGX6Qn}Rr_HR4+F_dF-Y{pYyI;*-c7
zqd1#k`dq=)YcAdR**7`Y`J_&Idx@^Yw$rzpj1HRS9&CD;^r_VS#gn|L+B2pE7S)uT
zl03V`nL*<T&*TOv)>WR`ewTSp?elP!(w(wK#rpH34{Ue%?c|TFdsEnd>VnnFJ-4=B
zEPVRsK={VK2|mA6jbCqY5t6?8Ty&a}ZS>FcC1S^_z6595d9UbjJvZUxtL2-fr%rdA
zE|YD(;}Ea^#-p-EPG1(EUhjUyU3SfmHBvW4<jPh(EK+)VC~DK*Z(BD_eIa7EB}X<}
zWwLzkf;-oYVif$3<d)8i{#oxTw)238Hh0as2bb=ZZdfDR>@#h${z50qYg=`XmIOaI
zD*8C|zh%tCq_>meB%2sqjcVV1wPOBkE9K~a?;(GClFYX$vTaQo2bH<<9to5s8RtEU
z7q}?kvBmoF;Y@{RRd1Jawur?ung%dTRcGEPKjD)6qC-i6pOQ?)lLHh>5*$n1whKF3
z2p(BpBiz^`oLG{lz{qDBVz)`l#Eng4YBF<+nZZPz7{lP&#(Fz3k*ta4%q-#s2SpQC
z2s%Gg7QG_W`Z1+@qpodIYQc5KcnQ(aYr1M7{4*yynQu&8AE^IbN&5Sz5=lpCVI$L)
zj*!nnj6nyu@}>#Ccvx7LYLqtB?Qvo{_s7sbAJc2TG=;E=E*48txm4w7n7G#<)MknN
z?)1o|L0X#M{)=p2%RZuB^U<+n=Zt!(jf`9hAs4SJTvSx_G+-28s<`^7hisGL`m5qH
z6LS79&dGB2v`bRf-eewIpj5d~S=H5i*HYz+ZS~IIaxYygz3e3Iz-YQimG$|yZiWuM
zNe^4!E)#5-==1cE?iVqUWQM>iOIU&(?Avz;UsEx-$(Z#y(dGF?w;u5#MMh228+|gR
z)@vBTziyD{PAz^gso0V|S>$5z{+5<qFIt?%MUHP5cAJsE{X)QxZ=(L}6O5i{xd!T5
zOv{hXFgx{iLc+$FjD>ZJSVU8Ph$ac<eiNRMp=RnG=%lSGq0iVZ&mvmrp|ow0Ww2U%
zuOr)nLwdE}Rg;XO_DlS4v@p?^4p3aVku_na;tIyv<`eqnY7Bt}Z2LckJ2Y376d7z#
z4=6lYCD1&5{qam*XaAG~TpQSGwp!F2Ef+f}F4CqczwMfMSL7t?#u?7;Y+MD?mMYe^
zideTU5n1-N#%H5qoUz#kQPEATbs`UDMw;u`Pm6xHe6qiyn5Ab~ov_I9qq&bOCI@^h
zx#kq>KXG>9r}_gedPhtIXJ3$DNNTWCm7LnB@!V9<lwJS#k8JP4o{d4;NoAswc53}F
z)@SkTvb)lBKG;tCx%DSkN!9D(`^yERHn9H4m~+Bius?IsY4-W9>ho20wm5zlb^AW=
zQ)SD=g)_DXun8}iWP7RP(wCa~k^g6J7xg+{w!m@X<aH|r3lz&<E*JYYUAb6^QS0mU
z%0}JxgRRz#?KdXreR|mXu&G=`ajMk9_E$o#THo}R9B}&Y)X|xo=6X^}x<S!1Ve!@S
z#et8VbUmeVRtl*s6XZ}XzMAa5Xov8r;0bfK&5alAQaL2JZvkgY0)N?o8STYOxp%4U
zde;5l$z$I&<pXNbJuS(nG#H`}_blLye!}iBT|}O*ODyG}s8&<2>Q^(zO3@n@UUwu!
z>o+fW%qH!ykb&{RB%=^*sg<+${Or5GX@z5;>a1eD7RNa~!4iy{8a}tgdATPaGGbl9
zq`y8;`^#gi4=nPM&y<&MlM}yYxBZZyzn1zphyQFr3EZ~f^Zh5zw|1QW<)-qN%t^a`
ziSCmU-M&lo+R3<LrP>Xjx_79y`7bOr`MF^I$~A9)l%ANhz{YZI;YQ&KDRuW_T>T#<
zViv00+Z1b^sO@LDPHN*6@0-e+r@UVuDtCObNFYhyS3NH*I7+%f*!>{qsS})hqKg9;
zvd(UD2{>wePH^exiR#+dI%h90bX1w{Sv8%<d3wOcs_ab$Yc?>hEntt75|!Z=mHREC
z@LN>w_l$nemcqnZeHH!1r?k8T6@wOtp0LPC+f}>TQ}EvLo``boG{)SNa)+1O>#DkP
zMNh?SsMOwaGma}M?}4hrgvZKWnX2ZBTMt&w`S2{*Z_ym@N&oGGEY!Cq&6RRu3%)qj
zT9Gv>fjj-mT+xemb2v07r}@2J(j2waFv^wxYlENhjiNc%l%GX*eM;ots=%7-u=;?O
zj!SvX@#`%jjyqa@i*!nh_DGBPYphaZ)fQebE#+47IVp#ECJr&nmu?SL@`zTl)O6G+
zvdAuXoL6l5O+vncqdb9eYB-Zv)d~r%Z<>9=>s>E(Y?`Q++?KY5sqti0huhc1jMf^~
zf|=Z_-RhXC4o}Wp>g*cl>Uu6Z^Z3psE7iM}ONiQDR=y*>u_csW+JP%9Yw26fNs<R=
z$VBh6PtSHy-!z|FY~6+KP~)g#rX1I%)}=Fh%(#_i?A|QBR4i_zS3dK9ofA!7=c^8E
zIH|LUTQtj~HU8S9B)1jYHgfhb$L<cX+_zIW{?m%UG|TzLmXjxL^$D_CyJ72x(+VwD
z)UHR{{5F;2Zxl4!D8OTG+~t(`v!$_FO~CB4-0vyd8NL=-AKac`ctq(j_tqrt&98Qx
zD&O&}dk52ojZW=5))laCayuICwd2sP1wSN9Vm7Qy7d&Vm<h{WuCvwI9@QI=y4=#Tj
z5#+FXayrLa<x(+MQ)kDAZFf&aa&lOevHM7btZO=We7)N4N}KKvMS41gQyqVpXg|^a
zUL16&DP$sttg-XqIFazA51Ic{T^g0R4ll0!HzAa3V(97*>0-{3_inojtv(f?>b9c&
ze^rO7jMStfVXIbuZJWPt!@kX7Y^e%6$`ZJ953s5>oZc3|6db^|&0(M1?|tcxo7!KV
zv7WfkIh-wsLE9yA&9_?}*Z*ksypigDVtOKCb63eS^Vf<`&v=A6)=9f6oqIORapvTs
ztYUmJv!@(9;3qD>V`jbn#@YUlW_J`W^68een<}2$ARHNeXinjwzsB1Vq|!bcSsytp
z*tUtwcZvTOlfvFd>0ceTOTA6+b+YEYc=*{A+lt%vt~%<;8(3}KkA9men0J6R<p67H
z0N3^hT&V`E=S~RA9AGUEU^B~F`fn#=jKk##w|7jG7V*wm?e$}0Kp+=`)UkBO^-_)t
zF5jFvfA!j%UH=a)@GeVqm*mqrAHR{cp)=U$<wA|FbxKcm&6s@6F);PkC4twQns+Md
zU-<51cICppun@sD*Oy!knY3e0?YH!|8*RTf$j&U&P+DQwwMb*)cmJrAqBT+?zEbjQ
zcclb9U^^irk~$$L^tz&(Ib%$~zL*WBc?Q!CugR7vII5Dgsc_S+_=VXQ)YltpsS62d
z--|hxzGhj5)wvy)l20GyRC>C3kxR@g_j)()zOQSR>ds`jeAWK<o_U?C4)$yH`(6`@
zx+E|o@q#2{h<5j(crUiPO~Q+9B+btV@T%E~g$q7BS(NS=;AnI6&dUi)jtQ()U^U)*
z>GX59`ER*Q-m;}A)Z`j4rf&Fuf1LxHkir8M1I9z;+-nao3V*olc6WxLoP3mz;GwKZ
zKO@<^+dH;9Eohmw|Ick@v9B}FT3KA2bye-$&ZNK~JImYJo?I)r^Zkk%d^8(c6YoA+
zrR%p-V$E(#wOcxAwF(B)PQ2D+T{f}fVqyr-TtSbc;_c#$vkQcntBoE#R=crLK(ty$
z?qfL9(<O-}CZS*V&TOjAPm5gtqT0Lg%<J2t^WUD{9M-*d1Ecv_R*T-#i(=Wz8_vW`
z*qC*1W9`G!2dg%H?A+&Wkvz$=H7;}J6{A}1EoV6wh|Hd_|8&H;FmaKG)6E<wp40GK
zCslg2V)61rTiforZ3$?+laq1h;=;5EGrZqk{a-IxBNnh);=byfIjLKH9(k*M@%|#4
z*L*$M^WB9HtVye0t$LerN#SD%5A(eRMp|Bz#3gTXElpUw`PFI$KPFrA=jO3&Wd_q?
zF0iF8xN_wlm&x4+hprz9+<3%gt)R)?`E_=aL>{tDu0Ff3&ilO4u_cO||5cZUG%C*8
zJz4eM%b&;1*-RYFA|HK}QNJc+>B(kw@#3q-HS5;zR$2QYZ`#Id(r36F*XjL9w9>r7
z)p#V8yGcr#PxNMtw9d>Y?Ha5(+bX^&E_vc2JE_(^P_fv<>sjE{tfimYHKb||r)PJq
zT)p({^Ene2pLoNiVj#NBj5W96rkW?yNweqC3TIxtQ&zog-*xgoW5nUzyFV&B8tSf?
z$iUdJfBW008MhUqn-%YcPmZ4P(z@HBV3uIh=G@oMb&?n@T->~BmA~Jdmfy~yE3~2B
zM_Z@l`1y73?l69moMYU7uPOG9lElr}L*Lcjo4-^z=3R8i=uLuS(m_L+x+Lz?_nHG1
z70*_+-^sDbNxg(sk73I!!7>Ln%TuS5J=wNh__bi~CAF7qd)~aA;`p$kZ}qPCzr#Ns
zwHCbkiBW}9;3ZpB;0bw8#VXNrTwdz`>TYly$iE}L-kDA8$m8bsj{eVd**Q)WeqZ=>
zmdZTiMLhW@48Hj|&QNr#Gj-Y|y!pu2clBx8PD=Gh7|AL}xN8}ec;1v}6!^>XpF<?z
z!UM;ae_UE$o}ZrQ9Q{@4nvg@F!|LW-mMxi2mnl0nZ*yv5$!uS8&`3DyW?|~51+$G*
zb`;!jS~NwPQSNSsO678e^OGH0`CK`stk9n1!M{x;z^#i#RYA1DM`(-I)n+axl?g@*
zA55Dp-PC0JGW3ARMz-Y>o(OHpJ-D6y@iEgM)4F4KF<D7Ixx$*F$vB6FA#e9|jlhHL
zqWW$gtEMI&o2Z(ubK;f4%)ljX69o_KRFDi}zbSD<B1(C6*fs;PyOR_e8&*$F<*uzY
z=n@P2$8t^i<ZcDWmVM1!?Ej(@9pk_Lx%&BPUy^Q^nDgRgc^ehn-YDdYiaxmX;M;uV
z*1U4%{|kGy=d&Ly&wm`dBCz=-|N80m0d;?W+!ycOqo43wcgd7ZOQ-M(O<1ttR+?~v
zR!dwlFN5Y13l1GFRt4^b+@dO-U7M|9xE1xbWn{ZF+gvW#cEq7aM{mEMk&p8_i>#xY
zdOa?BC{J(`_BwvFLSz+hkel(ujb;&yXOD!bWVJ53dP`B6lXHPd$3bE4E?o^R-YzXI
zPSz7=<y$$d-feI7kTO<eloC3i$oOXF3Z+STIjzd(Oi%P|CO(MPTKTuFD3!_R&4$*c
z)~pvk8!Q!gDy*{L!Di#YWp|9Ph<RzpZW8S=k?Q3Qvq*LB)({OyTBYOPqua?aWx@h&
z9##YIhAkdlUqk<IjMMFC)+{j9IDFG>k9Wh)4@_k%#DZDgxQc(3%DLGb$FjzYcY6a*
z)PxM4EYpQ+Of|Ngce@$Z;LD<L)J4JYkxp<}i1v$}U1ps^3xih9eWRu67P)SF;~AR{
z#b<M(h5Wh$n$IZZY|Q<luwa|@jwMI;Og?Yz$z8-B5YBMqy+X(5;5SK(a&b9(n=S>v
z+TX=m;%Yi2o24lAcu@2vv5Aw4J47{vTf}dvh%h8v)m*Zppj+>$?5o3k{=X)0FA8N@
z>B=}m?AMV!Tv~?}mYp_Gc3mm3MN*M-+kucfO$ArJ88z43S@C9pMRIZsW6Pdxm$&Vd
z6;jD_T`hRB%;k{CZFSLxqAUL=Zxegvb#j5xidD-ODJ)hr5NEn9Zkk(o*>}cco7>8L
zECK<KzpK>!Dt0cFowCt6HCQY8tgZaXvtHRdOY(E9Ra-@GElyrOt*YR?t3`94bg)9%
zk`p^utq`*jHJij?f6t43VM^%6`JPI%n8Uf9UTe45UHPiArd4ds6g`WXQa3&<W;^-k
z-Nevs3SUiC=Dcp2qQHE4SL?BFR+Dsg|6kOcv8AW?OhvYE@oKUE7K;UqmT2dH-^{8R
z^I7i#?>YtcSqmN<P&HOvyIN^lVNdcRX7x*RQWEcSo~rV>A>8b8#$Ls8VM^2vrv%>x
zybM}*Qyji+d_TpfSmW@tUaN?8FI?13SrhsG=f6>8xHqTMaBs&1R&il{#()Ax)|*~J
zY6^-vTuYs9SaW_&bKbI5g;!;c;l&?m$;TGXIA8NA>T%)<mBwkJnmM_Pll%Na_D`O1
zHfHWa<%vB<w!MCG*7DEqGxs$j_N<A>YzlIU`@^zfsk@p`7l%+ov(N)`t=t1{#kWq(
zF-TGu>|4^tA?%+bv{WHrnV)m6!TA?WMH(KxDnU!uOnKt+XjgYai~F2ab5q$mt?&Fy
z^7RUN6wqZn#n44DgxhHei}GX!2A>Vuj>aBqBp!?Ag!zYOFFf?k;((%F){1I&&z{S(
zf&@NZO^s+wd~PB5NvCDWirP(_4l7rLXmNQZ=1-KIo%HK}=Y+qZYPrl`t|gYdTzYN^
zxAnEm1uQ%+Cl`85Y}#6=;lwmy>RO*pjbKL^e%_`j4MwY`$XH+G4K0^w*KJ)gai^;`
zL#m<BJQ4f9flqaF7#CdG=@UCA<z~-<VD9b{ZxlRTP8f=^$UHJ}<2F4V<*4@a3`31Z
zckCn4z(vay4Oe|O<}gfW*?Y@vr-|=Gg^e@MPI<6(y(Ejf^TUbDUSw=DR7zPJefL$*
zo-JlZVoMgd+!Rr;Ub%fso(d=DkLYQ7madB|Ix5?3eT=uNFmP%nGHZK2owvu!YhqJg
zz~U|E1MV2G3gw6?E?=DLn!|WyOV;Y?ObN$#6&>Hbpy#E}A(wOc4VSt9H(wA_+-1v|
z$Nea}`N|{BBf__`S1~O;`0I_Poi>N*;UM2g#UnZ{-EyUOnbxlB3e9>R{M^Rz)853r
z9<}#OIc!9qY(5@SW>l=2a+B$XE@KLtqK($syCSR9gDgavBt#u-mwRZj?me#+JjH45
z2j(eL#8PavOf-3Tk6M0x;cl(LGc&;M&?GkZ3LCbCCuRMf&1ZPJ()y-m2Y;=WOZ1bq
zVq2c_=A2w|Z~4_^?h~dR!m5*(&T$RqvNiS)D^Fb*)@s~zP1U=@<I1!*hkj(LESe_r
zj9q2J!n5vova(a2ZPVPQ)Y8zdDHP|ad?IQ~vT5<Dn=95itup>EuF}DBM_F^n9uKAe
zVs*kMCs@23S2WG}w76SKB;9qA*SRhe2SM*Cx(Q9%d~2SZmlEO>SJPhnud-8g?cTfD
z<?J!{q=HXwHq6@h$+RoA>(xw-#?PU8yH<rdhF#qd)nocP#&%=J(Qc+2_ttK*lHARp
zomH0@weGy;);%SQc=B&<a{OlGe^6!iJiomw=7>I+r6Cs{rFgd_V9nR_7k+5OM6qSa
z{(P}3nrn*auc_CA^KF)XR;VyESz5p)?KY`ns^dMmZxRg&N$kBxT-shZo(~A&m}KVB
zsH~C9C-}AX%E=uwE~~sOIsCCysCrA&N<S0xWQGHx3qJ1H$+c@$)0*zR{1X?Rc&$>o
zb+w83t(+@f|AlV2JX_<M+p^%8!vAV*C&wmTooMbx)m4HwyKF@iR_aa{<jg7N<Q5II
z-g?Y=qhZ)FDG@!-1g#|roz-7-W?!n@!LxS9;RSv#g_d=+tFo1`a;havUX|a%ooevK
zN8`4L11HycuApAFsEZa&FU&4GeayADiale}xTJ8a=&5`B%rR}ocea1+5Mq%$a^vwN
zt8=|e{zM21Foeu!?M^9t7Z4LG5bkfTyN4+`z(UaUiZHicm!w%VldxUdO5u4eik@9h
zG^O3jmc3AE+}U))+WlycoAHBBXBfO|CS14@9>eLgabAQXEA!E1FV(_#F`YQ@{#VRj
z75zQsM|mE#=}tK)nrrcYH{a(s58w9Ks;d?s_BnKaB5zItS89QC)vX1qf9E%<+9}#|
zp7Ec;mMv1diTy5vjl^q{SHV2345G`rdb+IVFjw*}Sh?!(iV3<*o6bKJE#T5N@p9ll
zGD+hRi%+8D?#9(8ZY&jEGP}vZanXa01;00m6wftmUb9$xp7}1xrHhSE25&ya%&A$_
zz3Q^@f(!f3FzrA2YS+5m96}n*))}q551Qs?99ntPYT5+OnG8nJ8O^#67%ERs{rLIN
zmh3Kd8}4o1PU<a(Hcak`Xc7%-khwd7xBd5l!xwZP7}(rf#cmX^Y|cg@w$44x0rQ?H
zc6Q(1xcr8NlStS3$$G9nN1_>b%13XS{J*37V`aA<!|Ia(+mf~QeQi3@Jtw5SUhm>0
z*Z5lD<cqy0G-mucIloD1aRFy<ZuIP&$Md&7;#t6D{`v;fnPv@36;8H?O_N`4XDK$g
z(%H{%yu0^>oppqD_zxYv4|cOOnsrqc@@h0&Yc#9xVb+_$pqJ4+a|T2Ig9gqU`$cAP
zZqZ<#v8g#`=EAyQf%!XFbtRae2JBdP$z(!+#lsE9Prov0b+%|tVAZWy5l~_#!PvlQ
z!}**`Y897JYfn$~<5k-mj%`?^z4Yc$lOo%b6T~*&6cc;1Enjov#e&uMyJw5OQMN0R
zJ~iP)-=1mWJhQK?XkR3}PU?)%@l%eL)*UR3#<OjDoc?c_U^UZPkjdKdq5V0H=Fc9-
zIBy)%t#F!lgW+#6>#~?r-?bY#Gu&bdo1+tsZJO<6q|sRHb?oG-T{evS7f;#Ur)jNG
zvh?Hxk%`T7;siSC9Bi5y^mcApyW<b<-pkWXChD?$*}}|X^Dt_e+pHxsWelEq>dn)1
zS+G%Vs{&_q<&m&T(Wpi}o*17f<DGjdy8V8L7FBh>%jCYr+#|B_NMTiPcaYA}E;F`|
zvo0K1?E8DUeBt_Cl9LK=YPZ~4{Fr;Qb7t?~MJE>AV5plcV!gn=@^t5n8%|jzty3m&
z7TEMZiQ#nTo}%m1`=X?oGr`N~0h17ii2G-L^A%p%G2IVSI;Sj<tNQ=j&S=GCp5_fF
zH?SJrXej&9UT1JXtI}EN6?@f&^K}7zbt_s!B{+UPIa?*u%obx6SF(8aMadIhXKudO
z!M$dZ0n?^cpSZoZ9En{L;3{M=NrKrYkyXQbwkVH~mbKm1pqW3pgccR}E@a?6na~{e
z!Dw0!2cy8|)~wz;yVV<)cy@LzvTxztKXcD487|ftO&T^!vuwO(PO#TgVbsrI*5AX-
zbdzbujRuE}tws#G;R`ygUQW=S+awfmEXbl?XvTiWn|_lQaPnnz*JU&Rir!*l$T>e|
z<;@dLwM@eMZ=Y_~=rcU1`|AyB)r!_1A{<o_tof25VF7$)0?cI$lmGA0Ijn6H@PD$9
ziS?YL2QJLnvq<}rqkQyM;g4Mr&Kxe6=W%8pc^$R)z0y^WHQi=jMhg}6i@3UcY<h03
znEoZnm*MQRZ8yyd1jWT#dbUdO`f!@hmS%2Pa#Cw%gXRM!))gAZ7z`pbS!bN;__VwA
zIg3C+PmtaQ&5|b#W(iAQ?`Ac+;I@8=n~?%@){?E8js4~;g!Av-E)vyQzS>|y!{z==
z*38E9<}Mb-m8XCAobUP|_V>%AKmpb}7ki^Fw1n>PuS#I`UC?^xVvAGkl_=egj_R(<
zJSW(DR;&%sQ?%}A{?m1inUm|!qBn-2=O(zU{k65Dmxo28W2z{F%7l=<+D!s)FHH7W
zKW{fj-v67HbGQ}HH%RA9R9=`cdw#X=!rdGad*?^pU^!H!a(EVx+2f`wKPDSh>nuz#
zKB48c^n-!n6DG%%mxLrPpJdwa!tJ-TM00({dCd&wXpKf)3;wwZil%e5m0AsyM7S!C
z&L}$^%^fKdU*P><RqO4am*XPX!zxY(3N(j)u#UI5;_~#msX(h?1gm{$hoMC4k1y<1
zEAHIA7*l7;!E(o%MQ7Q5YaYq&Cb_MScf92e?XsCyc;-mTy58Q{FyXF9=I&!t<^@jW
z<xX8*dPr)6(X<b@jvO{ve`MCS1-_9xg{GesxOnV_SkD5%Z3lfPDKxkLI>>N)>d%d=
zU7-RFn*OY(J6F2&fBXN0iC?*yS#tkMms2{i)<w4Gy9%4@iw*ib&sQAfY}s*p%N2g3
z3r$QmYGD%0fe8_3cyIg(xpC_2?x+(jp%>2A3AB1jw}xsQ2$X2Py|IU(&p%M2wchof
z(M^^Yo9{KOvFo+!l$X)rZj4=c;tChbT=v?4yxXySa-j+W4H>sCO7`_m+swnaRZuW?
zw%+c@iwmx9F1)zlhKI0;=dlL?ljg|Z@!TjG67^`;MZvtyhZUNYZ#gH;O_Fu#ym-)f
ze!{7_8z$)S`Uw?;pW$55EqO>njq{unmzN@|ZUCQ^ZAX*<tKp6ZbryF_EUyH<NWQc4
z@wPAQWfIN5Wu6>MWjztw>iOB9@qYws;0bSQR@t9E9X3Z)jxf1AZn}9?;z;PrD_ovi
z`{(xFp6a8=H8F1G-W|6#?U0CCSfKvR&}?6a`n}MeslH7tMy{`%U87kQ&VLZwIaA;1
z7E9QJY4NNJg$xB!)-7S0%ImkGRX5<8x)n!m%{7rl`$dg9rL&F)S#>6)m@k?Tp1(}2
zjN2mmM{vu6_6FA!l|`+scm3OWLmJN}R7LQ$&NXSZjc|0k^VDMU|K5b+I;mO{j>?{$
zOIK;k-WhSMVHpQ!<ndQGPCgN3o7#IRxtk~Qsp+)ot1==jq#U>W3}|3}B=|LB-e&1P
zJNc9?XYagdX{V&1XxDLE<JOf8>@i$=>`L>h|2Hjq*)6X9PP$>qq?eBk>YA_j6>guv
zY0}m>?T4*qo>+QsChP9v!`ovzqFy|%a!3h_U^k3te=5LydTsl0)`Td7n040@InK!j
zdfeH!^l>Q1>x0o23nlJu_Y;o3&{FOgo2tgexLe5IW_8OI2EAQZUIk8ww7j|KV65J(
zt<4{l!sB`_NGxBW$$I|4EWSH>Edpj!cH|tg>*2Vd{?s`x_)5nOk-&2btyho8he@>Z
zv2K6ayRTDdmTo~>Mwq4GJ*F?acL`bv{)y%Jz4XMF$xLQH(i_8WR2jSuEVxmXkx;WP
z%eNxM@!jLvb?i|Rtalc+{<B#owmYMF13Tkg->Iv2RIlP<_#eQ~tI8?hcEY7+>V(#5
z?5y2ul8oM@^-Q<rw0Bv(>7tm-MnTnk)1(i|3&c7~-SqfhwtC{0#pge)J{s^yF;b{=
z!o{l!tosdRcAJTwuu}1qdMF>tz?yMP!AJJThSqi7uOgDT=KT0DZAJ9L3;UcKmtL$=
zbv-yCupw*Tkypo-vRccv-Yrb|zvfjSNA|Tv9S56FpXf+g#9*_d?8>p<fvI-$UY>r!
z)~WPtk$~<ECe7=Cq1>rAPb|xEdJ~g*Lew(!XhJIM%$#=%pRSwenBjPl<JbJN5pl71
zx9yxYE%3tpK#jENdD1Cti5EV;TQGyE@QK8XfZ$Eh2As|vQ!M(6?`*lVPAL8Vy(V6<
z3@Pyke|tWgyu4iPqLg*;`eBAXiznX4KJ-R)G;yqmP_dub6B;mqV;1*Ci%A=bPR;RY
zQ{f5@+xwlZbDC+#k7*(5w*zDrn#F&;FiC~iB!2eA1#@Rgf9U;@FI@Y1kJi!n{T)#g
zUe-rW|D&bj^66#8#s%}2-O6_AcP&olU(A`nn7l<N>uR7$UW(WM-yas*z50DNy1f5-
zX)w>XJ55msb}u|MxB0``ZQt(HoY$>?UQ(yddF#@bhUcd}uL&5eY;Wr0I<{%!o@*(>
zeM{!v^who3w4R}x*;3%#<mt!e=q>u7WE#}7A~NUgsk^$yi&cWR-SlA6RA7DG5fLL7
zc=iC#lmDwLFU^>%8!`KGST64_zEH_9zIj}n4*p3~q&RO_y}A;{z<JI{fw}L%O5Gjj
zj9WRwA3D9a{pfr9?ZO%HzyI<F-Czo9@UNOvddy;(VdwFu8(O*QdM+F&o|O_NBU5VS
z-JqaxVq$0$|IZ07cbYo)>AZfV#(2YFlJIBW=EP$wN~gcwxPIZp2{E%b%$%0Zyial4
z)yWRUyBUj52L|5t7E-^ryzsa5O*<YNc7y6$8dERa3clpD{)W}X%>C+1IR7g@?kh0K
zvT@$2U1)dP)#)*x?Zrd-g{Q4`D|l~~JkXykKAE>IL8hZ%@*Tm=_DapZ6F&96pH%VD
z`8#7L!-tug24B~osNnv8vxPSxv2vpt-@~Gl0Y4+c)IU2`P6`NJ_~O7xKlNiCO@<0b
z`LCq*ZP~VsQ{lz+&;|hqFNT9nOi_Y&j=XxXqIsKun9B{Xt|k^C4lRQSgT`Zha`stg
zZcJ1@)|VS|S8&3n1W7(2izyoqr#jEHHa>YL(>2XWGCc3Z6vY#lpR@3-eYb;YWr^nH
zWqz~oWo=EJvcK-1-8bzSQy*Mg_H|3uAFqHl44Yyc)Ar7~7H_+4->a)xq1_^@15?a;
zFK$xp64Q*DGs#5blV&2LCkumsLMpe1W*6&(moIKS5INSWFa0FwLXrE~S)6P~b*f4(
zA7N^kU^B<%(26i_@v5B56Q|}(X37%&zeKlzSJP}a_ou!!g%2Eq4(0#XJE8Td@DPXc
zZ#8A{b$6Ny)z^eAa_~HOkd3)nob#UY_esiAy$)oa?sJT0I^`1(pQO>w$#m`T^G9vm
zXXZ=qxbcvSy=UQ;xS%D99#Yk^YfJ8nMc;UQJ~TZxO|+HOcg6z!e}5kL>wl}s@V4Ey
z<#CkuvD{6rT|qu(GZR^5mxc5&Wn7naY!ozk?EW-!NpinT?uHo*b9h(Ya1~nOF`?T_
zC{5mReyK_8<X{umC2j$Ft*MCtS(bC$=azIPh%DJsl+={o$0qH#c)}%t0v28u8J82D
zSGua%uPQmrb6eN8Y`$XSvs1e4Qdf?#sB(B6tKa_r3`;`Lx1EknEU8lzHQDoS9AWfr
z`gc<^bNhxRt@>hByF$XVIUj^Y7i_$AN~3h*CfB&=AEK=cfjTP#*QGy-Smnr|vf;)$
zFE7KRNz(#PN=#*u{4JOqHqEQm?X;8Rto7H$SQrHMOSQBlN=@JllxE&473i|-tmlM$
z)`?2DJ0A;OVas`~=OW{AWNq|>oP&;GGjdYCutly`mJ3mHiB2gGbeF6&pXFgXOWolx
zo96_<hS!2qrc|f?dEQnXJmEp`VwD`hB0rS^N2aBYJFj<1d+ywnP|bR?X$nJ&kXFMP
z@1u7D&zo15EO$4M-m;*2-Ho6wv9}%(Q{9#~P1Q>Dt(Q=WQT;zH^Nxm=R`!G}eY?^n
z?!v}TmHQhiudH_TQekLdnJReXrka~y+4SXpTpJu6t6raFoO)Wy<iY%jtt%q~&VJ1n
zc+}Dwwj(%dhnm2)bBW((Ig}mLlqqEH)xNlzT`*<2%fkEWu1t*+Z7q*7b?7p>uP<&C
z@{-yaD6)*HpkcCq>K4Z~X{Lr%7vJ!%TDR-0*4a9F$L%*BxrwaxxG~i+jdOzNmFU=`
zVft&XU3#^;EOS$9Tn%sZHh1G+KkmA(ye(1qk*UCCwfnS<KOQ_{V)!R<=ZqMO#g-+1
z{TX675@y`HuF3M`gh6sf>z*sA`n7vIi`X9e?ozNg*vI0-kkyrtIInrmg@gZH{iSX;
zxhibiu%+{ewWbb7zlZ{3$JF+m<;tFa-XtuTyK@om+K3D1OP(!fdz+(aq&!c+y_h}P
zK;MC}sQGkMU|Z0XMs6X6)|Q%&6W`=!I4cPS>UwRF5wX>1;Cyl}@!T>&<0QAl3>ME{
zCX2Q$-D22h;B_FeNni~-=L3CDr4E)Q1%m5DT@u+e<%79i>_4E{>g~5P-jQpDM^nKf
zZrPVDD%MJ#*^*1R=Vowa`=mI;pG<3>klArfj>D>DhjvN%D!+L-8(X_H-t@K#-geV^
zG?9Trfr0D7QoYoZjE<`B%M~&O1gC$WIH%HO22;Q?#~CZ9B&}l6I^cZI`Aq<q`3F9Y
z|9@s4VpCgS{@Z7PP09-Q_D$ja+m9@*F5P*0O_nNawCys3+&?MqlMd_?{&X=hnL9=3
zmYa6QU)6rr&Yon3^!fWAo>vaJbbE7G$f|X@2aUgc5#8`0eL}8gCvWPi=}#P%AFMcg
zdU2Lkxckko7cEwD`~9@a^z!|6urcYGC4Ua<!B;K<Mk+o@hEpUQ&t&*_{{5nCTr=18
z`Aq-TDjf$_M>WTmXPcLby`ABZ{VHUkR_GkT7F$Q16S{pBM^t~cGzL65@3TZj_4tp8
zrM97q*EKO8Q`z{!(0doh;akTqSL?Y>a1VVQ=f3$E%e>?6`6gOv?{@6p!I>ajyXOSU
zBo&V+At99t<Nq8M+=m+bEsTtNReS@iP78S`^-Y|k;GxQTq4nadZ?<z2xrDYI=(3$z
zw1^|BBZut}o4F=mQ>qZdGX<qx?Gy4mHk?^~B*S9@LqW>)(xck`AAe5d+`_1N;m?V%
z-UxvVq2d|7vR4*NNNP7r*jyf{a#-xoHpW)VldY3<Po%IHOH{aB(|pk!c*~4QDALGv
z{U%*e1`!3FMj;pO<ba+HA6B$9O-qm586~!M($fHwr}<Ils+S7;Cg1!k#nNbg>%=y`
z&8>H%^31+(;tf8tWa89E+S}wGo=uGrmb|m&kX+?0;THwF=Dxl1sl?Lj?8_s{a$>Kx
z@AR8FL*!P0bAQNm&8trzhW)ReaWz9uFvQF!DdIMRtI>iS2Wg9JyJne<2YK4uvsGrx
zFnTM0x=@~5^;f&H>CSSwN=f6EQf2A$=CfE@CK?t?r^G03ZavC2X}wR?sg#@A4Vo|4
zD%UBWNL|0>rY@&|*NVS9S>Crq+?rS>=S=DAG<>tX^~bYDjk~;>7U6pLe~9f^vxi?Y
zMm*iWzeT-StHUVU_jPo+%Z!`6QfGTQMWT<KN!|0<N#t+X{Lf+6JhQ65+D$zaV}Ew0
zX7ksn)92rvczoW<L?MTFNh?Gyoc;3AaP|eBdt7JtH;HKH^NS@fzrp+Z^n@eP5>JzN
zm}p4f*<liQa-)oPnAWNL8xouMM6TR-<;wrrWe45jwi^09n)fK_k;^jCw;3}J=G{7P
zclB$}dp6!<Q?dlsc$p?geNFW8irXw5_VZNCjb|+9ejWO@_CoEs)NSq=eiAvS=k!X7
zZ#g#g(6fyD+uakqH(&T@eZMLy#O;1zv;?=2`LQ2`Mp{f?UWv4Y`J2wFe|JqMSZi;j
zuk?}4=ApSaf};i2o4+&eis=$5Tb-1%_peV*)Ma;z=Yc1eMwJ{?ZCzbm>c43z-}?L-
zWl6TDA6E!1d2-R@gx3TPAL|`Fdv40)SMKx-U)Yht^w`qGQ!U|bZ*|uSZ$;^|h3ASd
ze9rQ4l3JLgvzfI?i?JZs;={t7j~A_8J+Dsi*qL|M!VVk$o6CH4QSWH8k6Csr<buLy
zrImY~Y9<suoZTpSg=3#C$MOeqDRG7Doj#@-GEEnMSc-=~4E}ICc-azxGu+C~$7|C*
ztv_}~JyDlA;Dmv~eqRT!XZI$?&NNEPxP0e`M&qF=Gbb=|&9ywrpmdk%(8LQL=PzfN
zJM-@JJ{5=5g0?$4*ODx*t-BzUmNR#kt;D-IoHs&mEs;EuA=tjm(KC}%AhXdWbK?Vd
zj|1(6Q$8#`nYV+pctVhSkDpfN1!tQhJ)-XSRp;_b9T6)L|8SyDDE9J`r99@9lUYK<
zn!a@1Z)AD!_5N0o1sM;*-OhCAL`JiGyZZ8P=ULOE2`@w$B)WfaT#4T!_rKBbm_!Hv
z+gO+RrfxQ}3*A%h&J2*4RdH9n^7M5LPRWXk?hZ#B6j=7Rp1bYn;;~g^PEU9KgXjY;
z=e{z!?=@)OUo+9sL#Fe{(K%ZhUp|>KKVe~Uqs!482hE5`k%ecbZH@WI`XDy3Wt(ZV
z?xJ|HI>}ib51IMqGxNzNe&~?-*`e@l(fPSIe-<8>Ivbww=8?Xba_^Dza-MC6_9X7O
zDii<kUaHSRIlsmSKOQQ#o_ti|nQ3}FJ;Gaj#a#Cci9b1vGY#$s{h2KoColabGU?^)
zeQ&S1Kb0-06uBSS^7H0($y-U~3`^W9<dQk#+$-*FosuNT)fUU^=D4WyS8Q~XPmkoO
z$Nx=}PH+1uF1K^xIu?<?DmQ}8%!{#%cwfm?*?ZSL;zn?S)V7Ll$DAg<yK^-x=O4(8
z-?t$mSYVFB6YsC47ryLW;x>izK*o`dEzUpgw!NGY@KMF#%*;nGR29-TNV+Ac=&$4F
zej4ts@j&12%$B+A%vHW0O&*6hJ;=Cnk!O_$N1x*r%VW&}bKQ(QYa~+?Ymdm;ot#;4
z@Gr;l*35?zQ@lT^df15ZukiG1Ju0^9FPqjzCV_Rcbblo4u-<LsmDH3{aPeE_Fs(&2
zXqj``lVfimJr7-0+_}tc#=Tc2XU;u+)UV=rIoE9>2WPTCNYuxRDJI^rT5kVJU7Sm~
zyqr8+Yp<mT`TWm5<5_5?bdU3yd#A`ZMV9Wr(t8^EinCmJ4{<xOoC%n9MEu~5Bb@<u
zT5Pg==d52782T+H_>X*e)>DUR>XM&AgUzHXyN+Enb&bqYtyz|?ueEq?=ghf^vhFo!
z6D96m_?aG9`7A<8W0vCd>|4oB3{uxGs1+`Zp4{+YVd3KCnQZZgLYCOx;{3JLRWac9
z)~<t<N51*0zi5<t=F+Aqy4>zrTXK>-C+kd?vS-edmW3=j`Rry)W}7Ng-K4&(XES4#
z7EI&nvr}bg33TASF~P+x{s`|{6~}cuI{wU&|92zq^<K54pIu&T^KVX^%X{kige_i0
z_hi{L)61Hsu)TcZZJXq87x3R-js0+9y2}=({6D81RhG`1(&6swxM4|OZPt?Nb<4^P
zqQf^WpFZ>6f<=)bek&{UQU&dnx;;A4ztulLc7>hn<E5uA7+*?n;)$z0tKsqHdC;AS
zCQVPWXFU}2;C$_J*L}f~;4QCi8M=B~&1ws~<*-0Z@{eK~!~E?LEKefRg9VleN<DK}
zvZC<Jl$}c^<sWIi@PyOpz>7B5C#kE>FV}h{dM#<oKL6!hFOM~<R@t;>aYbgeopq>-
zdsV=qXZv(<SkLp;$<Oxk%9&n~_If$N;Mi%^#}ZmE{!Q(QXS$#9B<rKZ{C#UybScVo
z?)Ch%H})$hOV7(T_m+@3VqW_ncF(O!I(pRiKa<MiUd!1THa-3Kp5K~s>gI;2k$s6L
zSM~M8Y4yq_oC<na`)`gum-6XF%TtYdm~QCo_H#HF<PbPvs$)(AQ_ICWR~9pDQSIKN
zdS!1^OzxrB3;h`%dS5-!TBFsv>||C}A>*w=O%E4`4LMFv1lAnZYA=|#Vqb0_b8kqm
zX<BK?G5facpPj3gN3G6$BpQ;o>{M2EqnM)WqqR1#)+#J~xRvYbnWYogFVK1YVp7!F
z)l3^!v*gU&Sm?-E;U*&3BH6-bCU~Uoi=g<MuDeIlT28HT)5&#NDJOSMEAxly5w4XY
z8yu}oCaAWp?TX}6(t5Lg-i@Fonl|^8=UB<SZSxEI_3nj=*#Bd7?@pyX4V>uWdS}M*
zb$*EtPJ|v)THKfBdM#JSQ*G<4H@6=p6|;tNr@oi|so5{}Sn}4JA5FpMKIyQpSuo#&
zVbX(%Zafm*S02urtCdo~l)vuwJp<RumkU!%JlmUCjx0-_c0}EJ*W+znZn>=YpU24O
zsjQxNqm$#JgyaU@y4<*66^GLk#l?9NcCLE9^^UH%Q~`%~bYtDpqNUkxDthOB^`=cZ
zFxe$AJuxVwB203M1GjJs?`;Fek_Lu11^m+;yj2b|*mkcu!IdHs(c<UU^CEoGFG=w&
ztJbp!Jo=LD=H_s{D>E-+LcV4?v+A6$g^%X=#4lVS;@Nm)Uyi`5W2fh^_54@4buv6+
zjdlT(tnA`@Wg8ZpYx!4rh;2{gN2m8o>)u!9scbV*Nj%e0ZF|&)RkP~e>+fa9f82V*
zr16gF#hahsa(I#=L^K^zdEPdCJG#AA!Lf%?(=eAq>{x1M=l>$vM2{(|=~3w(LB9?y
z2y$@{D!8>`>WU7__@)UNPah;WCiLa&xF|K9yQ#O@?MWxg?pq4U`rNupgeEk~N4h+4
zTkoOrX?qeIyT6Eo*gH3|b*~o~xUDEKFH2tH`K~%d=;^G3Mtj{QcQ`mc`M@pZ!Z*8t
zPrOBC?`G~P2W|GZb3QSa^!UKi5*Vk^7pKEH)#IQ|?GoFzbyupo^4XSium|nflGhnm
z_`lOio;gzflViHOX3kr?q&+Xz1y4D7z?G%EEbv9d^WLKu?!3x<<#CYhk7W6Z@N;t)
z<yfwdNqaBlYTR$P%DFGcBj9}xTckl3J7ZYBVB4#pkfq9-SF6m{|FtOJX7^_I7n>PE
zwyxh}oUz7u>5tE+awls~nwWj<@g0}D>ZhOhPCUersHr($I?Y6?zQil-?`zIy*Sv~i
zS(`JK@4c-j?6%M4*o`U{Eyn;u?uW&H7Qa66DD9B0gjS=l^0zHQ>w8Vkn7N-Zv!DIR
z$>i39FXamBcWJGA<dqm$^7ygay0j8U?r83%e+oE5f_ST4!qO+XXUyV#T`U=(XFppk
z+gb0)HTT*L+5cldPIC8Xl;MjgbY_cmp6lq+c<6wzM?urHNqWf(HnDFi-MVE#?CpdD
zbs+}*nK!hOIzs2qE;7mC*f#x++ngV>nJp(&OyBtBC)=v6g$ElpGNrKZxK#2@i__<M
zyv4&A83sj?dRixb#cXA@xLda-sJrdZgR6U$9_v~esmy*KJUNto&NhaWuc6{BZTC0K
ze^S7G`@^X<XW}M_{`2fC%W$_>3}<*Cq*uwppwgDP!LKW4(|kdRYOiD49VZEVKOd|j
z<#9(o^UT(^sPedbQ?B3jpRwoDsfFykQBgC6j$NO`7QZ|1hLDVsoVHkUp`mb#fYN8b
z)o(Hwwmf4ynauvA->EDu{r&$tw^p#Oe!jV~xNzx)M_r$z_itcW^7+ZCYjf*eGX8uB
zDR1CRG1abSu@i3LE@8Bv?I1qoU}mu+dx1m#^Ec`7_w=3yIc~7%KEC{~gLm|)^OL<4
zJYpWCUsyU{B%s}IyLVrkNP*M`F}9sE%<pZKaJxNwpTKb!WhHm%7R|5G&vI4<y$Ddz
zTlRiZhwsPOvIpPKhORy<w|n`Vv<05)d#$aH_w~-Quim_S&F0?qM_ek-IemRRNh~M&
zM@Tu#4t0*4!v;$$zMk6oNMz3iZdHaNR>o!8t$S7{&s1h}JYd13+7M?m*O4b;Lcy^W
z<wrC&zxtE&BK4(q`kY<vJU{=ZWh8F+WTnOSU$ex&h;8+TIu<5>f!HY<4ebjO@3+}*
ze7n4Bp6lL=&qEDTE`KyiUcGo)<tg=Ty*=(Hr?>3zGQKyhMs(Jm)AvslX#U7A{%!Go
z1B2tcMT@^VzqaJR|KWl;>t+60PK(;Cg=UMie`&i5=8LzA)S0<0`7txR>V+;VlWm90
zwXgLjuV!^TykxhLF_P`WLPJ~8n{K|#51GBmeNv?QOTlY;A+MWGx37-)ZZ?C-Qq}8c
zd~rOUyQNS(GG}A^@~yqkKmGe&8(qBh*Zts*H;dFti(UVv>WKez`;p<kb%8PK&iS3~
z?YpxYcZ6rpJe}EIzwh^T`Ed6GKU}x&2s&ezID?Jp%xpHc3IF4Nv<CaQO<LsPB<j}R
zY!z+qIw5}hq~!05EAO^``{f=I#ln(g&9r{jhjp*JG^UwbeqYU3cwyR;2W~4G{XW}H
znARQef@gv24f9}~CES%i(}TXvRsC@?UfOd{rICK#1#7)0Of3w0sXI6X5}X_)o4Lic
zY<6s5acL9gb(>OgLCLi_?bM7HLCd^3^^`1qG%Ot%y_&gLG&m+GG`RH|re51pIa#r(
zl~e517Rf~JnYz&|69f&~7K$v2;!zV6aAi8znzD>zLel0{A!|){2N=ExNOqj<#yjb5
zRoTLpz}$$;oTVGrGME*rPZB!N_4d$?T(M`VN83VXAIN<vH1CGiddJ;G|2?-p)oz#)
zxY^>if|_Z0;_9fcEL*C7i*0IA;hM50rLv5xT~~O@oP95&(n7j6i5|G75zUZyXWPYX
zOHS<m?s9!;?d)~B367^ZgEeM6IkYfAoKaE#+8GVu#Ddpcrwi)OwVr;ZDZIoZV=l|G
z1_kcsBRmWZif6wvWn6o*Qq=C~AEs~0H_m^spIsyLU?0o28&4b-sRgatbBCvkNquwA
zEhX*M61NhxH;Nq!SiHs};)Qc#6?3-H+igW&4IE1nmQD>5i+<tS{K9DBF)7vxo@2YE
zCTA;}u~c{)G&^J;oz|N5^W*VrLK7a?{Z6@+(8W9JN$07~SuA}E)e0{asXsJHI%%d?
zAN~Kt*58Y)iWhPg1U)#z5@BV$bc&CcFT+F@r5CGPEgoqse5v)SVPS{SCJ(DN2GbV-
zj0|rNwY-keXjT?h2z{}5;ymL_rYRvs#thSA9-Xw6-YykzC1|pg@`{CBQ5oE9^OL+7
zM3&xi(U>D*Qo3lB$(O7}O$RQma|u*cSbFrxJ0p(cooP8ABgB3S1?lho;U(soJkf^d
zL}yQL&egS1Jvs)8=BryH+gyzE&2>tZjdhzlqF0*e#79rxa{0;CSF!Bf(VdGfeG)wo
zzcwpc;hfLx#|#hOGp+U8{kqBdgwnfR!5lVMEhO^yd^)FW(5Nu0gx`+A=~@0lhTK*8
ziV8j^kqlG6-u~Z}Iy1)1gNem_VTX=p!qx9<X0tAs;AWcrNc8$_lav=PW?LNO<m6nE
zanxXT(w@$B;_+*w^m>=Mb?a?%YZT1BASS)Zqq)y!lc=Uh#HYBmGhzyuwz;bEnKSw7
z7H>1Tne+YamIGHp&RQ<q{CROd%Y)$auh>jAmbmcFJ@L`X)<|RbdX?HMJrmp>Eoi>t
zqxIwQm#%4^KD&9fVy&ln%TKuSV2epWXxjv(!=c>sC-EMdy+UY#Q}dKxl_g8Wm3Ce4
zY<es9Grsh?htte$85<2&Ew=5}yU@3Eh3J8Gwm#Llk1x%;{Wx`1kG_#u^5UaaIUnD+
zUoi7vx?53Q9Hq7}J8SpbUCq<~pWJ#Y>-3d`5Qhl%9Tym1eNj`MS}ehoYGl~zz-|-J
zB$Rz-al^$0?5Qfz9<?rFt2Jlx=^AQB`kN>n3R>u=yw<pbi`8HOOU6TobJ-WpZD|r&
z!r<@tExCg)QdN}eL`%4$P~gXJ;gcWFWaTuu?V@`tY5I$k8twsCPF?GY5Z-4asNP-4
zGqI;-$re?H)+MinoO2EfiOdpgN!b$p(k^1L`NuQntcS0(A5a#0n)^h%<8nudQn*H^
zOI~2tr^`;8j5S*XTvTKiIC%OAEnm@~84@~0<@GTOL9wohZqx5{1qGBkM{RtiCC0UM
zP0ewE&387GZ=0;*x@Xd<(0fn%xF<46p1S&fx}@gn$m3PsSKc|TJ#LX*ec@KO=>JDL
z2UK3`Pj{RbvM+UtZr^IzC!6m5K67~gqc7)Y{aMv1lycZNm+6gn#INZg2af1lrgB+b
zFnn-mi;!T;p3jPJOeQYa$25uEPTfm#^@C#^UxX%?1~lu6tl(-qpgJ*glGjhh17{eP
z2(~TDQE~`a$l3fkpgF>1fr*z>(~RZDa}Ne8FN|5{Xym5Mm)NT4#B=VF#GxQ1zKv?0
zmpf)oQeX|b(KOv`(ITHb&7-?~)~iKWi%GjBif4TdF}$D@ev_qOn|$uW+@~T9UO7q2
zE-ze~J7dl5Nir>JH%?3}*OK*^f98U^xWN+TjVX8JOoc36{$I^h>d|WEUh-IA;uMDl
zCX16yAxZ0N_XdR|ZV6J|dfCl8tjNCY>XNXC7s5+fS{1pnrZ#1D2`~Gxru~5NiM4T_
zHurQ@w6j=qJZ?UaTry8=#lg>ldXh!^3L?3UCha_#^|WcmqCFb5ud^q9-Ns=5qtJNu
zEzWm2$6iV`@#<JIW$>|zMIK-dI>Iw`wainFol}nnPFLc7;^)FK$?xNY<#FMr9&D0L
zfh!Lz-)QEQ<g2mkMvoMWorAzlC->QhL}vB0GGxp;Yq494w?)X;zf-{3Ao|qoFWc)>
z)^=)#YE4ycOV#4OIN=A6P~)<kMv+TCS$@H{A1?V6QWeha*HhCHl~Mcb|KaEqty?S!
z+$-j7+-LIntnyjT9o@Aj{FF6AA|($cx9S8;u#fXtq%=um!;eCxRNHJPn@Kx&7x^{J
zxFjOgEqqcUey5w&PJhRV0*$j4u`@)yo0L%aG;XB|W2oEChpRHKYe~*;)mgWt)5pE~
ziQ_ewEv8)u)XKPz?r3<{dF+r<;-5v{X%ADguUUWD!gtxu`TGl_hnKlFpE)l6W$Pk2
zoqb~Bb*J_2c_{3-7ju2afhYD)d>YmZFKA$xYd!b1AIm(mnR&f0A_dut8hP%|Tp$t=
ztiSwByPfW0&Pva3^L1YzX=RdCbd^|j;r|Yvy|zEkJ6c;9b!^+P;90Lmb+J;XVb7!6
z6aMeymYie||ATExs?Oza+QJFOl6_TMY?g4eSxoUd|LxdP-Cj+fez%9BVHJyA9;pVo
zvSzGJeS7%PvssRAiDJiJOi~I?KH4?=R?><?{il9aZw~qLJlyi$ji7R)n8>L$JFKTY
z2xysA>dUaz!v3FEkep`YGoen+r%PPCC-8i$*qC#_{`;{r=8<>9woG9O+wjQaLQ{{}
zA)kxi_wl^SGzp#DrX;e+XIWXeWa?e5=xJxO%r-?W=V<EYn797U?&munZvL8=^62&6
zxeqvwbLKv+^H%SgE$X&#w_8T{zE2OIy6I-QPYj5ecClb`9?!z)Lh+(Iml*=Co{d$1
z+@Rw>XHAS2pY)agYqh8Ob7`Gyddc|hoV)ydUcKKh?XGv7t`I8mk^A=il$-5^{6~iZ
zH#EsgU1VW;V*U2!^$dGI`Kj+Mw5)d<crgDubvGhU>3!#%5Umm+hFh2Grm;laIGvIp
zs9)Yye(l*L#lq=74yIoED-k>K;o7r4N2dCzTu|A2bV-8Hiqsyp_5XbRdQ!G_l)Mq}
z5Zn}?)&J?5mzoUQ$p=}A2FsN*)-`-NrSe0|@65?l9czDSZRpbARIkYAiD;3%)g+d{
z!uhA!o2UDXM~70|IX9IAZY#}0X0OYWT)8|nHGA9-#&o`M^LG5hyKK+8#t(0F+Llgp
zIHSg~cKL+Nzj-MfjJh5--d!;IuN-~Aot?L7+k$16x|&>#)~-CYRQ6PYiI*=U2ivC$
zFRgiAT-A9$HEZp>6A$LurCir~u_~_V`HRQ)_mZ^@PAw8ru#Z`>;9M_%<@;M=NqZOR
zF}LgQ{o$<6(B<usn)WUEt_BmM3gh>xE)gY%eIdtg#d-eNlC)T$r83HOiNmRwIqFMt
zTJBBhR$S$D&a3rY7K^Kj0$0aTR~5~7SMp259#8A=da1Ffb4u%@z9V8DjH+e_R8}nD
zW>c~<a#uX{>NwXT=B(cSe@m>yRIP=cGBoxC*)S!!DXsTQTD8RM`Kx9o-3uWT6d1X<
z7(QJ1>Z2mN`oyvYZExj{*iTv;BINz`TvMyie+RCJ0wsYK#||a7`#eq_D_wqZA9Pw7
zs`Z5Jd{ahP;V%jGEfPgbyi~Z7nZ*iKBJW*X&bGE=(n~#qX2Uuak6Y`MD_hvxC+I#)
zidr|}L)J>(B@N7{7O=eY*tKajkK{2m?Ixvd+KT&{xeeI%L>=}r;<(Y!p!Ut+TT#=C
ztn(}vb2rEYYWk}S&3NQ6!->l&U{OfhLY~uZQOoUT1${FJ_?E)NFsXsZ>O+-Hv+TA7
z-=(;I$ehx=8>DqAurp~z`-DZIiyjHfrzi;psVz|u-QJ|}#(n7~CZ&x&&PSMj&tt2)
z@ot)l(I1g_Z+D$v-m*b$&9TGxlHPA}eHoFG({e=YUcaK(|3q%KMapvRihou}-an=G
z$HVGX*U4Q9#rqBgNoWS|x>&MAOXjHH=@>UIlSC1d#Pp6No&!ECOa08by%*f#&5BrP
zn797nq6794uQjXl76&a=S$pX08YV3hrZBzJZG1k;9ZmY7g6c^x8P-pByJg7HZXmwC
z-LREK`Cxwat)%JK^s7#>8cxu<bm6kv3WJy)rpwpZ|5+{O4PsVHoODgVz+YfX(uYjT
zs@!A;RyDPf=S|ED`M7WDW!;_@v?u2f-_uPg8$Ri3Chc94AhJb4XkV|W+e7oM4#LMA
zgev^q(*0dBT^2q#tsbN1p1(-Y)X8;=1J@D-u01{sP8${)6*^x$v}oRcUcKT+GE)~F
z;8D6G(W056tY6&RHZO(E_h~!NqJw>(mqr!3?o*wy=*8QDMTTe7j1+~BtT^#@*|LWx
z-bc;q&}UeiW3u>BSBgpEB^IHjPxFkPMFlIo(>i6RQn2EUv_HR(c&PT01mT7fS9PX4
zixNZ@7@y%d9dfPkhg*ru!62(#CQr`-tDP;A1#+C~)Y<+|RNTe)Cg+KMv$Eo5C-(E3
z{R`9%aT)6xo!Qp+VY!h|<c1Tv5=wUI8!uiEG00e5=@s<*#v(4(#iDHMYj2!$NET98
zPfB|8!gAY#B)Km(P1--a)EqoC?@rp}G4azKzQRc}-bSDOsx5g)i<QBGH9_>uB&C@D
zB|H-v_NBd=Y7(l(cBDn6%h^0s=5ML+$v&P0jsr!Hy)G(E*_4#HJj>Z=b*OKk@(O>3
zPivo7C~<}?F<S5};?5TzTb&f)Jl~~m@3t|mH2G|(_|Dz#v+>oE&1wlr5(4WbIGI+S
znj;aK?#|7rRmz>$FPf1k8<8mbLfFDwk?Y(?neuIJQTjT}-!xj+$t?}jcKKSeC4t8-
z^G)1^?O9B1H+*bzm2!^fO-)&4V9Ii`z%(+QS<JmXs)5Tyi@Ey6<_PCahdFu;Q%<Wb
z+aP-8d9w9lkF3Vp2}^7$e3NxF!kThzz4Fr}$^{+^-nQD}wTUIhQ{8^ew@hXh9ezXp
zw`>cn{!f{7Bu6RF<d)ZqTesGlD+ekcQ{al2{+(%w=O<_PZV$o#0_t0M6aIeCneau;
zF5JC-I{$nzhgp1#6CAiL0=<;igsuwF7f(sdSMzf3X4aV&=-NE3;FaxrKXLyu?KAzp
zXQxEu21d_2y5d>L!#5%(5A(Ut3CgVRiJy6mse5|q)&!9(#qTT)V(+$#JqqR0D`ojA
zTF2@w*Be~Xz0C7d_JS#5{K5?~wb~w=jVtTb(l>q9dZorC_f2DllD98Qcp+1Ge_Qy0
zMLZYy4r(0Bot9D1DynW1)-^-bt~<2V{kgtJkoJc|ZHrsGqf#8q7g~6_?YmN{$^S!>
z@f*uUCLPBRZT|nKGbb83|6f~TQeNU~ATYaZ!Jf6H)*tG`n`EkrEk28Gs92;O@2^v@
zUvYoBXKH?I^Wy~}%Nm6l>J}(@l&V>r%sX3`rEx0RI`pK1*$p*=6r22|P7C-uR8}@u
zA3i<rxpCx=2Q9CbPw=yUTJq<O(~)iATDK0lpZea~sJXI(CF<%Ik89%m53jpy658h#
zFeP7!Gj2+dagXeb6{3D2+#ZUYizYYtSJuDVIL)(d(gp1lu?o+9JRTcAXTP4o#_EvG
zD12ZSpKFGO_sMs{qAa4kizaMM&SCGd*Iabp#A$8AqsZ&Usm-s#mp{+l$R6n>=E>uH
z@z76$SswY-DN7zqR%Cb_$@+hb?5*YV%ooaRUz3`|KiyrNwR!uV36u7Hc)id=QXq7R
zjQA_NY1=wxS1fSw@Hvyc(0EFS)pU1J!Jgn0;e;<0`*Y7`ESEWZX3sIxS9axD*^d^k
zdFv*BOI>ek+3y0+?z4BexNNg-WzJEXQONP;TF;E!Ma$J6Ew=5d{2F2Qi+l4T@$iZl
zNp`Pw_>3zgjI@#@t9hGO?e!O*`9d#g<8zhGf3_Xx=uGuH$(lB?g=fmOX^H1OlTvv)
zT)C=#tl_NK7;<cxS(NaDP_Kh)oNsQ8-`ul4)1||4>y+~w8(Y6LPG!mtH&8Upw&C<q
zJHfQ_`=|0MZ<94jq2G%Xk1d#48l`3OBy8D#OXnq#vwgpYt=JxTa)I2Pk5RwY_!NI^
zZC<{$`BlXBwr!`<L!X3oN3?kO&3>E3IA79gUyxewO`)~_ds77@UTi2@sN^7g>XYsk
zh1+a_4gu#=s!G;pyQF4uc@<mj;Qg^@LD<(miL{cP-z9GJDl_yohj1r!2ygzj{P*g;
z;V-k7C2=>H9I*SSHOb#Pt~81FP~9)@43FOy(S_V`Wq<bVv}K>Nns-n5-u~bJ#JaXU
z6T9ATc=@N^wdLa7ht`C5mK$?&)`?|h9n{;B(0Kn=NYA8Q_iyc;711bo;B?T{#cJIe
z8BaGJo~%?+Rk~j1wOER`x|0^Gg2mQF3Kq9R5Blju6j!SJ&#CBg*ID$T(xj(4w<Yi7
z&XlKO(S0ALuSm_kxH)1<=2kZSSvHGfKK$INI;%zOmO%NT8y&?~!8SVSY13{j6fNRE
zSN|+F+evWZ+oJ0ni%hSuG_PO!aZcykY<FX=WtLy(GgXx=^xdItz93^}QESBHXWC~X
z&M$jtu~#@{vEl1%v)Y6CkCs;_Z|t6_)F<gS?YpB%UVr-08IG$%yfz-R+1-_LO5LS7
zS^V<ydxtu=M6Es_erlr1b>Tf1-#pq{ZTjxs#RmswDP}3YWZR-~X5x{2C9X$*4s+%l
znJ*js`PZ$zpXPt9+M-p}C~LvhP_ilEMOfM$wKKI>Pkwv7WKo~<U4;eZ@|pi*Hcg83
z;Pp|fs@WXCdSZ3Q<|5uLOD)-53$1jny_sNRc1>?ao9vvm8r^ID%<c?zt4Vpk<icFd
zy3_y7wwLbMDmFvrr_Hx*)>(_KpM32Ji#W1{=RAM*6*upWUG7m@Y(E}*a>PzQyW{@p
z;P&d0=iY7)llR`a(VHK&q>ZVv|Iwz`r_=4`-hCWh)_2cM;7RtDhPpgvk@K~i^PUR7
zRi6+U>s?;3#UM=6lk>2U<vY2ZYLon`>%AsD4gOnZ=&NjLaq4N7PE5=frXI6DQzjp9
zomqTKwPE4fyYJOs-QMeC!d1QRx%A8JhwjL)Q8Ss#b0IceeYIwYsaBP`e)XPNcWpPR
zgoPCR{~F$6lbKMfxTt|=k>}Rz6#Yf)+us&<OX;3EIP<OC<C?VW&>5~bUQe0K`rX*1
zS>)4A3*Q&(FItuDb~zove{u2^?X7E;dCj{cvg`47mf*_&7w?)r`CX@7&Ak81W1+LU
zDK5{nYU`5E|JK^KZI<#5wT_v~&1WfD*#7l${hM9$S6Slk<=B7S^ZtDQ_+jS15au=M
zVSiVoaw=@|5OmnE;9xU{u-KDB6BO861r&?AJOUG$R9I9!GOw-@TOGy`$tmC|u*7kq
z3TuN(=!Dc0&3cJWJwGQY_%<os=Gd_^VEMtQtvQ@4N;iZ~VqwtIiaoVu<t3G%NgN)p
zwye6kIzmuwK}SPE(0>LGVU-RB(JT?hAeOwFMsBAiT5`6_OyM?HjP07u^(=Hoxrk)b
zVhy%?r>7KLTUsv4=4UOiHn}V68P@`Dw|94su`TzRmSb8NtjBt?!7k$G21m}>Ia?o}
zxVYftB#zq?ro<MMojSbUT}$-jq$~ILvoy-5EG~K)KY4A=)nLyl3A#Z-2b5lFF>sVB
zu&wmh^$=bCZQ0%3$_M%;Dtue=ly%ARx`Ne*K5*O&57)Y1bL9PPLAAT9WcMHX$grTD
zL5NdkYKxnc<d&mqMZJkLnpjPD+>937TPhw~ZJ9Bltuge<!c1$`FAEq%v_h3SSOm?w
z#DZL=1v&~TJV{ENm)SO<m&qsK!hbgQml_PBqP8s;GMEHrUU2c2>=bb0^iT*Dik^_<
z?JmFb*~C27txZ9>?02sTO*cK+BH*q0aoOyd5n@XwPfxVE-WPB#wKAk!<67qPxfRbi
z1tTs^e7Zy=aKj3*&MB>FD`KmHp3G!O>CsrmP<J!rLOa_<mzYIWM&BMe2x%<LaGV$^
zwvfR>EaX*M_o52%z6Fe&v(i`2%bF^Zyyv9Uvdno~L~@xflx44H-}+E`al2Z{9j{#z
z4^7NsT667|$3;~I`MF|Smj~3cC?1{b_%2fYK>3wt?h~t7rmE}iWcm3$xB5-#>D-;H
zOSgyYX4+AGp(2ivK|vr|G$Mf|Fk?xp7`L8<+y4X(=c<51Q{<*xR}*paNoo_Co?+Oj
z>2&4#5&_em)d^J(TQ3|`mM=*<*7DS~%R$FYOxbCMrOc<poF{jQ^m{G+W$5U2En1+o
z&g)~K_NhfKCdF=#m!z&s*DBev#83UmWG_XpQ);tCeuf0-?mU_~Wt(-o?zbDiT3cJ@
z2N-Uf*4%66B~WPM%JnkdkW;-l)av~nL)Ym{$1W|E4?5L$JjH#>iv`S?ic=XT9<@n!
z(Yv)b@}a<2n^lg};#;GnR|l>Q(cY7F*EEOa<EzRm%ePBPYv%A|2i)IScx0ECl4>~r
z^|Mn&-j;B-uAWl=b>-~4yIyu>Pk8@P@2jl9@5Jt!_na43$NhIaQ=c8X+mmH&t6H|?
zp3Z^?izO<KO9z!0IzFH5VR1;x&BbEb<MJ8%oYeJ0?Kc`sJUHVtr>El4)1`(o^J=E3
zEKgad`NE1fYTE*-MLq8ncCJ1#^_a`V840?odtU}!jbUCkp`k_8P?7ce1CuE_o{TwD
zyccruGl{XdI57BK>WI3tsnu;_XC}wXsQWiex(*il<Q1KkIOS4exyIs@vybHbBoVVo
z2YlYvDimE2WHr`cbXclc#Ci3J#ukmUJ`cM!Z4%NHl*A)F{S+IeT(#$XJz#w{O>@EH
zgc%jiGB-p|n=VMcyE*5&&V)T~+$>7{A8st3Tj!|K$f42^vSzXV-x~M0Q~y6-tEri$
z=OJ|7Mtq_e)2`%o%&co%S(+o->y#O+!&GXx9991~q)pf>%hsj6r|bBM1ukljRFia@
z6P*|;%zHT#x&#j1e8TcUK#sFh!OD8sjR_?-O)d>qhFzDYZ25BF*@j16@-wBplq4of
z2Tf4BqjSo;<dR!L?-3PEPd=9RBBu>Uf)xLCU;BK7ZQaV8r>#sJAyG=7_XaLV$=abM
zJY7#hX^o(JQJ(ijN##kKS`J*A*5&x3z+}a)>yCmm%1=ofaWB%`;*qM*9qjcaO=VT{
zjT31VN<tGpESqw1%0eY05k+aiOMNXRy(-?bllh){T%0Gir1X$d>Pv-7ZBsWiDsSD~
z`ro#|T)fV=eFEpgKGoHa<tr5@O*%5!D~RvNqWrVQl`)Mcl-aMWXa2k1mFL&I{1Yiw
zu?yej+8^c8j&PJ|R5~)HVpZw`jV~9LelZOUm?H1=Ygd5Uv=tj9qi2ele2}<oli|W*
z(qW+dXz`wfTZ*^>)xwr+n)+s`?<|j4scQw3tXAJqX)*rzlFge_&_1XjJHIJC;$G(b
z<_s6EcWZ(xgdQB)#BoY@wt{QE3%B5$X%n{j-Bb(~=#Ee_x}qi(dn4W@$-Utg=VIT3
z>w>JfR?OfOn3<fwee9k4?WzA(c!*erdaf_Jrm^;8c&OysEYH;wN`hMY*8QIIrtg=|
zQ<Kw;ESbl}CjD>Ff0VddiT&O=rRCy+-d?3u5{dyke<VNfb+U+_HzhIZ<c6?Sis^F;
zv%(&3Stek1B3!d{%j7uOQ-%jN+&Fr|^hA8<ourv-*DTimtmh=<W!xGwO>=VZ#;~4p
zsm-fWg?;u0xo>L|&RJC4B$yJM*W+zEZA!6u<f$U3RbPZlZ>4UVxog$jA4j4$O*mQ~
z&7fCs!sTq!gbiB3k2Gawt`Xr*YhCa)luP&aH$MfYZPVY^2*k22-Lc|DgrYcCenE_i
zo9)c+TK!>Ny;8i3|NEv1yRJQ&<+mo)Aa<vc>E8l&cl9;bJo(lyoqTs=$f6M6>|0xz
z_j!~}n)KjDMyu8jD=w#!+@t^h&p3YQ>#2L{OApv?EsE$f<t^x#>KlBjaF^y{-OIVN
zqNhqu*4Msf=DRI7(pyrWr*@%as^hEU3Qb$W85oq4lYcg86uVEqB(SOLY1J{GWd&kN
z^XHxsn<g4HPtP(ucgv?5^NiTm4=uYy3^^_K-r4HgVA;BRs-K8W<&$lit1}jRe+w$Q
zF4NmA;S$T8?#h+7T=(GCWB*cgwdZnQoWS{Ux}LwtN#?av0(r6y7-so+C?4N&a*jv$
zD>3OcZJik%8EU&Ye=dnTF=0-CJ$LZ@v|H@#w{|uAnUo#6)R|)u)Lms_;QdzNaNq{#
zZ$A^f?NZG-l&?Nsw<%daL)U0Vd_=hQ^-2HNZ7B=)+!QbT^>LkzQjK}XaqjEp!pn~w
z5p^sLmN@oyT6pw{3Hhn1e~z!*EHg`?|8=VK^&){pHc3U16M}}dJ$G38QY#FuJa@C~
z*?rAs`IZ!}bTe=6ADq{oWm@V!-@LhHna<wGWBm4K!uNeWWW0NKn{CLfO<pGot(;w+
zD201`S!-1L(mmz-_5U~CrR!}A&ad1m5NH#&E%vQs)JK1ok5|rFHrwseuu7lNRL<~l
z{Z6s7slwS$q-r+D?Dl)^mhGAvyyUWF`E+TuiKRb91*b2{I(jh6dugFUAb-+x{jMbm
zSBe`yF&S(R%{i!8Fi*JR?6DlShz5m+I<uasaT-?CwF~^e6H@E-u!-HUBEKcIaB6z}
zv+#-Ey-c6w-rHu^SMF7&C_Y=j*`TCS$++Io*<pTr99N^U{WbxPLLV2SR*r){U$<p{
zFB0%qa$X=L6|~4N_gd7dr;YPJtFn|9gr^ti8akDy3twC$f4|6|eG*I4A&vXV!ig0E
z``x1zJgSp8lng3F<C<!{4Wm9bN0@9BKjc`nWPyvGA=i=xu@hdHpKMj){TicS#P+*I
z%<ob*OQ9H-pyZ+##qZR$%!G|S($y`B6kSsy+b1fNZRB)cDz#{u#^3G1`Ukm|y6IdA
z3vu}@&S)X>O+iF`g_?Sb!uF|RLd7}Te+V;mbk|j+YW*lx3Jd<9zo<}4BdPv*&AAAH
z?UQ>s+cnjuYajV2)h=#k7a@9QdUIt{Y|dc|{>2uK%*M~$N({ai*)9{9I91?xgz+19
zvn!WG|9$jHIcOAdLgUmV8MT9j=7yzTOjBJ0YyX$U`zH3CJ6I5VDBar8WIbc8h+<Lo
zi9V&7YA2LBKAJ{Y2u4gk#J0}F%rPXX@M;f-W3Kh&t`9Ea|BDrN9q|?ww473?#e7l1
zEK|zgVoK*hR_9Iujz{*fKa?gPOk8u(awoHp%C)w6!aX6IGw(ldKeS2M<Y(}zX%qNX
z^lD}Z1$mmDz1TiCOyrCb!<k9ajX#7<ORe*!<s8*Wx__eS-1X9X8Vn^Xr~kjSUB>z2
zbnO@IkEW#a-4y;}D%26~wOuj3L$%Jv!|aK8@_XmF$J>&ZE4Fkww+IMMahFi^wy5{m
z-sLEiIb(684YOj*f>|@2bzeFbew$umw1hQa<ph-{?(zpyU!5?K{1UZm(u^GvPQgs-
z4<ZDPM~c=?7d>q`si;(<#bcrki{=^zZ@q=F8aEBxzf3Ltrt!vXQl_D#<;_mEhnfo|
z6d#JlEp04$9Xk1ySgdK2s9-^ukE6jpLzAD=RnJzY_gGpxyy!msNoT7;Pu=xt8ZXM_
zW;Ss-I_0a)sCzlb?_!O>#5oE|0cEZe1*9rDnUhb58qRPxKC#1_?P09R;pD?2^L!5~
z|6k}XpdBT!%2U99hQ^`Ex-U108ZAlgn(S|ISYiu{)1PCe&r^aoFBi&5OI@!%C%&nz
z;d-_6#gev}3meq^ervXIR*5R^Of|6R@DrTdEh)r%IaiZKW~ye2r>11(LbI=zB5fl3
zyFS$&XjFESFkA7ne_4}KlG*$X9)b%p(xxp>Ob#$n`xYmY*rXa*JEu)(t+LSZMtg}V
zI@>)3yiyn3u~>O}naJgg=}q4ka2Za&cSENB=gN7nR@Qn7Ebv;j*h^6Bgm3OjnexaY
zS(aS&_M|0^&F?-Z^}4UN-{GWxGB19Y@C->A*D1?)9rLz6uIaixaIMKK-KUEbJ}sIN
zEE+Sx;g;Zp|HqugVvky+Gj?%pT-LwPKP%ArFN>31RiR90;q@zmFS@L6Pf<M`zE0+4
z;?yANqbGCPBi9FLIr<nbH598AuTsxtS++S#^bwn>OqYw?A)$^Rb5o?|6$L8HJ2AP?
zs>^hW`HCG*sZrwM-2RnW0yl4E-)zofb=3QjqAT^o^W+Vq%!9T^!wfQyZqhrgpnZFj
zku}>*mL64$1>wJ%d><~DnWfHmYIE&Q8T;Q0oVx{FwHdx8tUTd4?YQT{Wr+*dXO$O+
zHa~RVy5my-L-qO{U7a$l-F``nMKk6(nj0HT7cbe!7<8fE@8Tkv0JrB$CA}tX3|HI~
z&uY@;Qu%MQ#l53zmAeK03o0bmb8kyHnsAQAS)SYB$;*xP*{&&tb@Nmi0}aC0I%vi`
z=3eNw((j|gEbnPkyUIV@+F8Wb!RaXCcU#2rbx&Zqbm7UR7E2@wlZBIKIZeu3y^-6V
zd0BB%VO@BkVhTgaGR69iK$r7T4iR6wzDNr0y0)El;T|V7M)B=gCmk2mX)a{@xOrjL
zs+n9{G#2d@IJkMyE|EK@_d1mY%WkRuduvAd(Ouf+Gv8#-ynoGWdz9i#Nu$LdR&!N{
zF*`Q0Zmj!YqLudApvbyPr%P%1rCB}?XBA{*n^aAxHxfu+IN9&u{vTzD>B-w)UEJ|v
zo5h)Eo889}ukM_<CScC#z+LjezUM0>{-0JXQd{Fxy<*x<q3KJmuD`k5YWv28J7<ZW
z4c$F~DS*FquH;l>pPBp3c3Irvl3bD0KSR{J_*B=LPVEV^6>k07s9)m$U5atj3!$|a
z`7ELZA9@RV?iPu!5r~Zub@JZgI9tG-drM-CsLC6+=$j@kt-`^pw|ajuw2Bec658KV
zn7rWS2BrlJ^)Zb~T}M*6i}SigPrVXg`<Rhyts=kq;I*c_&tKI4aGz9oZ7j4$L}<=Q
z_K67|0;*3trulj8z3XB5>(|82OUFuu((_hUuiv#x{ib36kKD;Coc<n}D=<rIO~tYd
zHs_tMXL2oMDePLa#Ob(TqRNgKb;BnAU217Fj6&z=DF2r!(>blQH(YVwu{jzy)`%?6
zUf=p-)$WBAuV%a4?8)^_-2Y{9fNMoGo0H(5ZsY&nXPj=#6H)YGJX5!8&#Yf(L|$H$
z7dhzpdXenLNPWXeJ5K-JD6y>7uTl1gqP)aI>qDCrrXHO5Jd1nD>dm2sQy+5l>hbjE
zx9;_M*ko`<ghy(rZ&}RaJ;oC%j<Z-7b+hk2$+J}3xqWh>NRnauI~n79s#@RIIQi&I
zVt8^UiP=-)qYC@N9_xqNvFz%pIqKOtqPKYj6K$?~F>djVy&5@jv&`~SK7|W&7MX5%
zJpcU{mnoN5SA9~PUZEsul-FGt-8S2y;mwq7N0+(F_>{y7WUXFv%;n5~o5emM9hy$I
zDKcvW&QGz44ZPylbg+4~$GeRQ?>A*P-aRS!(Mk7`*i$Q~>CxwuJx`S#I;DDc{nOjK
zL{pm=#olrfyKMR6@)4iP3_;O^&qkaJ7(5kj8!nlgvCJ%Uqge0;vDYWB8!lEk&@E~>
zOCYHvbmH4{-^{MGFYI~8YPIcg{H(WgoKIgZdvwY}Yjxhz^yuHJ2Q05$cRTGDxc{h;
zeS1)zWTWKUm}qXBvM?3hx4p6_I(pK#PWc#e#-K_xc*2RZQU~pn<pNW6pFFZwY?W_+
z>+{s`=nJz8=jJ3lcT9<OT>J8%O3aUS$|i>{uD!YZ&9wV(u10E^ZPl?m;I&)#Z_&}F
zTmNrf&8sp>SoCS@*H57pMt2e?Yppl(F{pd^hgaXiZ^yG`jG2KZ=N^_W`*kIzX1%BO
zlX9V}&ihuD7Ok+o>ZTbME|U0Y+17oPoBG_o+~ulH?znyC%w^fFYBwhKgfc3e@a@o8
zwX;kA)``#M>S8|6F7MJUo~!G??5j9iSGn7twJ-10Cb_n=P6hj94`28B?RfK{=1$dv
zI*t=Xm2)3oX;-?-v+~o^tIXxs1Qnm&P2Z{!xuJdH#+8Sr6bXvzA6D-<7j#@_Piyg=
z@3%rZR|=o{_+l|%CC{n7S9o`+?G(9Ys<YX8(zUeQbzj1>b@v-rTIv+;30lC?<Sn|w
zFC{MUIkybUJH;vg=N|7jPb%rkaR_)J^5oNl=~F~v9rcdw-G9M(apk+t$xAL=X_CEJ
zComzzL-F*Uxuy4t7^nJXU7XXsK>g3Fr*B?WW%_paDVg3$Ds$HCK5=)(i4IL6^J(iu
z>~hrKs<H+%9MqlnMuAE7u#K5}xq!<gt>Uitou544xeC>AUgAi6a;)b`r_-mT$qx(m
zh)fSE*jd-CReSh4Yu-*T#eIc);&&I%3VCk4OTP5o=X-Stg?E*I=rB~j6$rj?qEk`j
zK<!%%Aw9u{H!H+X2FkyC^yvQSd0&O%1w$@wXXm@H?Q5j8qV<|-H)YQE{hfC;&+PE?
zDVL&eDcXtTO?zp!yDD7gui=`rH~wd7iRsm;b6&Vyo}{erTJMt~6<z<ZY16B`m!)d;
z-vVA0Hm<m660x3rhQNl;GYY4Exa<3&Km2&<At7%6&k~D2d@TFCVTEK}P2-JoZz7(a
zc@gxH<6{58+BlYtL7C@7k_xN-&y~CDc;xgwS;jh#Q~UJZ>%E8weem+$`Tu)W9>ldU
zF;4Y-+NAa0aC4l+<@M@wYX9D!bN5Q+_rB{LNy5JplQT;U)mcAnU1VpL#P);b&5yIM
z?SDM4ikrSAu=mruc#Gcied=bpWkSLXhF%98npjyU7)1mm9BSm2v}=)=`0$V+tGVHx
z9V-_x^zxkKQQ7L+#nQ|<Nri_YMS-D>gJY8m!;$|>Pd9RM+*(r^%y4?H5Nl2YhsX;D
zK7nqo8BP<r{g(*}e|WR=Q>xP(%h|7FBo<~l&Qe(^*7C|&=juilWe>s14Wfs_w%y|0
zQX2X$W>@Lkdq-z)Pq@>xRc@2kjjja=%QzI7ww->qDDId7YmE%Y&F7DHM@vNYb?jW=
zaeBsN8(j^@)SJhAmI~O!?lKW=T4TPt&Gvgr&gCEzj)cprdKsd>Rb6J2i?KLd#ni>U
zqV&#s|9yw$-<fq(D9qkHN!{xBxst_>C#BtU)jeifAM%x2vPAC3PepHqC5^k=S6Dv0
zu$1{V_k{Qf&Ch<b^ozaGSGtrY^gv~n(xe9mR;39$I5tVxOjx@1KTp>LMHMNL2Tto)
zW?Wd+uHx|8Bq)k;&Dy9M$&L+gW>l?SDVA7W=*k!}Wul@KueFBRD;}xT<BdnOIoug#
zvNAl5XZ3KLkk8!x!sgEfA%~{(3pRPGs;pn5y|O7LIYU!-BTKNpfUfdl(L*(xk6SeL
z1sg9knV7P9fkpf5%j>L!qxu#+Q(7dr!;bftu=%nz*<BuTy&@+rv<ul@(Np{!bV>ch
z3E`94j9WT_Y&+J(bOyJv&Cyby&hQ{}wL*=Q)|u6-tUesz-QOjo?k)c+pn>DSsa0BY
zwk_f`S-$Dj)a4v9GAmX$I>~GjJDg(}*dP&F8@AqNxvGZ$^FLLmqn^)LsvXewf5~oc
zgO-{epR2n?=5}xT6StYe^z?<>N}QYwAEqiyjbYt<?G4kG{ssEHuap$^!q0X)7Qbf-
z<kmfr;l3g4LeA%eJ@=;USo!2ZQ*V^Cs@j%|4;oy)pO6vey6G{Av*^XeHig$23wi~V
z{2A2Q4{dRn^TF3&ONCkDMaZ(C-WT@GZ12n!7ryN}kib)#*Xi7xP-LaZX4>;hYuXgK
zj#sPmre=4ydahimdmwA$QVUV$uVQAKd3PD#>iT`C$iMTG&+q6xAuQ(ZPO8eTo<;|{
zb%J(k2*)TLn(N)Ia9+h@!M&z;wbq|nCjFX}aZ<nCqe=IKg6j6Ls5iOit)DL0yzb&>
ztGnm2|8o~b>%U|;ZLK)<b&`ovd*1$aW&PZymE}{a#eZs@xp25&L9_7Cugi@=UMC!9
zFPLPoYSqe;S?VkNCM7kk-tu1j#p=GWwc&HpdCfwWMbu7Mz?_~evwEqb(|v_?wkFvJ
znNHqlDK`GQYU$kjua4a**upyJ+E16{rHl#5|3ZxR8VH!RW}n(s!=!&jBQfsM>Z$vf
zG6d?5K8s(=I3fF;u))=gg685*hkrMcmNi~EWX|k)moxRnWbsR>4Rw<^v==O%{X(Ec
zbmQa>=454;2bDbK?cE$_?3g5f)(F^ISWZ1ZBgwXHMnTspV-JCfXTtvso@y|@T2QYt
zV}gNEuV&oFc*l9uMgMaoZgk+W<oqS!vbRIUzc=R1#g1E-x}I)$8ObSgCiae)^K+34
zdJR(o?+Lt=nd<mKt1l(|qhHt($G|kjB?{+-KARtFtoaa8R&t+*UCGh!+0rQ<j1Q-=
zIXMNg1#r#zed4)Y`kea*0+d=pL}wSwoM3&oGilqFr+Vw1PDt~yclany_b+%p`6lam
zg?=R^<HVmcdPL3}1@B1v=<!MQbisNBw!@vv*PYN*xRTY~zxldYW5K!NI79t=Eb+Ys
zLXANO+3QO7L_R(fzAuS4C5-1H*I&6yt7kjtE56+<%Cut}tIsCJoFw4}+nPZB=m}?&
zidTCZc5YTP{aUoRVpUk?jgoCQGIY1-nM9=oe^Qf`FZG#uZsM6dflF*|Pv0BPJ93(R
z?G2MgFF#JUgPF-X;nNqgu`nq9WMLIyU}Df=U|?WqU~*t&;P}tLIz`Z-K*7<Rg>{S9
z4Yh@b+XYU0=ma<M92AOSiBO3+^~kM5!C6g4HQ~w(U(P_G5Y3|C*;P!pt_lULp36`c
z<+d?uyVrqp^H{3i@yrrfabmFtXCL#<kjz8NRE4{wIfJHLS?;+x>+UasRiRB`)7w-%
zC%ielp6P<!R<B8Gb_9mqx0=f}^+Cu6&cjBcnb#6ym!vJ^^2@sNa9aD?9JQ3tY1&76
zm*{Nj@|p0-`BabUWUn1Jmp?tnWYFE_GokS1(G}j!?{aP#OLr`ZvrjpiX8iW@Huc|Y
z_TDbbn7==oJHvUE=xVbyQJVUGvn>ms9ywW1q;o0ZP~#0rE)CTU){gyB6UF|g$5y7K
zUs&j#tJ3-DL*RE!)*jB>?u6Adx8B%#$s}to!-6$y4&7)tz-fL?Yw?<*geGpLsSld@
z)+L3;WQJNSY?X+sP;|+98XDTB5Y#a}w%XG5Qc$jIYf`I9<_pDAgJT@Y?VdX;M0%O7
zTn$cC{Q3NfmW`9k0tT&#hF<CJZS8ScMrJRcOb)QJjMEMd^L#q>Xyz7H$>;*ccqd8A
zD-*PtCR7GEr75*tad}vK{n>2JS{79=9n&%qFSD(i1Dq^6jjq%;yo(5#(yW`B(bplj
zN@I~amqU1KzVFv-;frl<tqAURn;EjOv#cs3reG=S^%aYHq^7P|I?ZLJM$Xh%S<#cN
z{&THd<9)aF>calEk`?julwOIr)c@rQoZ$ZR$AqN7Q<3u+_NacDmBa67eRK1Rz|%K3
zPxU?hChOXyMQ`)h@w#&rF1+d{lfSWdcW%b6-rZ6j`}S(zNjesI(yi!+l>421O9M3T
zI$zFh6>wp;zY^ebUDU-*QnHm(Jw0!pmA>Qp$`rA=oVAkkoxLXo$-DgKd=THnH0cai
zi@+w;L{Dw?6`xM)^hYc_lX3OW!e#pB)@*Kb=-jlaTkCYrv|j7%)N2<*ZdSg!RC}~z
z@mW89?yb$9^Rq;nbe7+Um>X%z`*D-eHZ?O3;m0}KlHVWtX1e2B<V&+@ayyTwy2%$h
zvN}rgZn<#qUh4m;X|tH8Ce3BwJ3ccoWp?DXl&+Y~+qXGa+)@&D?8<yNxxQIbG%;Db
zY~3Rb^`?2rD;`hC`xUXWFRE%~(ZQpuUa#78wd$qn)^kZ~R`>Vqdbj8KyWcPO&X0S!
z;?W5|n>CxZ-^z*%XqWo^*<C#TclbuXt-GK1D*X-r7OK2<%Xe>P$tjr*j7Ryj-&}C=
z*Zndl=G$>E>t%1;XK|j4x^-yj*=_&k26Ddt?;aSlc>$|#`-f$gdp#AHB!$;AX+HNl
zUg%dCwQ>CkAN7Rv(|(S>)Fw5H_7!}1Bz;t7(vsAv8(LzxIlaxA9s6&#EBq8}P&+bV
z(-w~xZFZ+0o_AtV+LNQ9a?X3A!2iWYN0fv<cDP-cb#u=#lT|8$oqjEg1b!+X)zZ3{
z6L5HfbHg#`S%OPzSQZ|&%9v<Zb|ty?%L{j@%Mtx`TQ<u5Yw+}V^YPFs!|eL<J3fr(
zj~%!B6{sR`WRk-1%WQrHo!-8;ggt&7o4dF1h{)~C6AxUR*z_hOdUjWH91Cdj^wl~d
z_H{<5^RWZ-w>7+*vO-7nMTq69&_$o7tnXU-Iz^MqaEBL9=1SFwf|FV?zcxAT=UK*3
zmnj-^sCD|*Jx@PZy<Bx&b@I%;XO@0i@O^5AnDC4nC4&0xGkx>oCM)b<Nq0IlQ#>s7
z#GDP@>0e#M?cIK!>RM#|R4~wZ!Iz89wQp9tYTUl@zrXI+=lKl08TuwoI_(;6kyjY~
zbHD9a?ssou(p<#^m*ma5#erLzehZ23)SI=gCuz!JdES+~rJ4c@1zTd&bk9#{SUjOd
zwY}+_B_Av6_L+wh1iU8SQ8-j0wcO~6t_=t4bk`dW(>B?rsq$Zpur}n}-=5*bZ5J&1
zd4@UHza{HJLe^wO|2TYgZC0r8q6b+KtF6K|bbVcu=$ox}<y6?_X<yf-t7gZX)w;gr
zoanluv)ZxWyTZ4xySg^vZ+7f|D@9)p1!1E<U5T7YH;zr(s*=#E!}(TX((<P|-J2s@
zCCt71jpqgS=Jh0J+}DV*m^5t*i(#(QlNS@Ba-1eE(&-NL?r`7m|L@0bJDz%}J5HXn
zeec-`(M|o;y9`*HN)#1icg!v;JLNdx#aip76I5Jfgq8`d@Qn@lG;2=Ki<4y@=j^Jb
zoJBm0*iz+!MV^W_hhEM0{&uaX?L+E&o_5Z7-7DYsz1q6I?)~oje;$2b$*^sGJv&bV
z6Zeh-jBXzq#sA#@YjoqlhpCE<s(-Hd_(<&U30>b}d?ryc@8axTd(uoUH~A#1ZPv}{
z?5e-kX!m6$|NP%2$@)yU`Xzr)p1=3Oa_Qfe0Zm_&6)od*9h+o?W*E5nZ+7$wkX2Iu
z;?d?geIrL9n~-c_<Px4K2R}{qY@Dl89I4PGd}8`(fi$D!<!6%hHqScm^HhJk;{P-G
zYS*32n1egsEi?K4$#ZXCv;LESmzD;ycYN(V1o$kz_wyu6aID;&(d%Q`@1?~($wDpj
z>y~!ylpQ9%uPlUqpPDd5&P-eL@T$0=*`Yl3u3I&9=42(TE1i*@x>YS=ZL80fs%u-%
zm8uo&&E`IR;o5<HrW?|Gv*KRgx^Z|}>88@TuXPsux^d92Y~%F1*&*`RZmro@w&w83
zSE0$jZ+D#CwqvbqME>!rt7ThXi>j}_S$d&a_RVh{>GGUg7PofaTPC|!n)Se~mOnr4
zGci~k=>L@5E@JaQF|$=)O*=D9^`sJ?>g?nn*Cw20S!Zph%dEa3`9y2(pT|?L>^Na>
zSJYd){E77c=S-(8=gpm*pKhTqp4d}k?V&TZQT0%&$wJOWF_ZS)vEq&7m@3cY*<R3Y
zeNU0uZ4I}R-h=~_rqx80IUc&((k!I(XzQGM**OX8UWYTp^IP9Ip0{(;<iBsW+U497
z^cL7A|LyI=w%xZL%jKnApZl)(eD#*>xvd$8ZQmZa_WS<gyWbC`ANtU7fA^#7dF5&5
zVTV7fE;sbt#x<|!;yi<Mg?*W4e%8N|c*uU<Ykw<`;I+Ou72{*M0Sli7>g~3(e(<ql
z`u3XC-kdv4Qq%q3@t&;KSwH8r0{_2vwe{5#AJ?5(UtT+Bv31Z4=brP`*TUx?XT4ni
z(Qx@y&#pyNeFXeudURZ_F8+T+antcNGRKTA-#m0&M3W)9M~N#k!1_kBY-h~%``5mn
zZxZ{ozh3vn{QrRqSQ#Vg86{Ym3>ug;8rWwvu<mGJUD3cB(GV%nDDa{|prVn@qfxS>
zk?%!=a7Uy3jz*;yjr<Y~0UsJCerQzZsQ)v)iCLolH+%E<cJHljei~)YRu}V9T==uM
zB!52U|Lq$8Qqz{@qC&sT@;TC5zBtG(6jy%3AXI&cYxV?59&w?E=5;s2J!Hg-+Ljar
zs29b}$k3dgp=I8dSkaa;qb*}cTc$-@o<^H0d-2i*_5mKndCeZ}fiW$M+hY#2pSs%4
zxv<^+p!wt}YBs6zHyj0H8U*hjWNrJ;Cbxf~h2J3y7DIUk7NL8tmOg0>xzh!{x(iHv
z(AjR<xLBgAUA#*#qKj!l*F*tUwi{jRUv#b3=-x1cb%RB>&Wo;XFS-I5y7!*w-uj~Z
z5J%6E7hMM<diKxgJ~gA~<c{vs6+Jo?4d*htuT*rk9*+<zOHwPgo3Y)_Ff7^1E!m8%
z&*qpv|3$ZzrYXyx3)p@W@U=)$_}I6!T;wf>pzn#gzuQs@H}wl{)(v`=8mlHdU1h?Y
z1FZ7t6AqWx>{>qIP<!>>@Coda6Q$ZG2v|;3*<LtTp{S{NVp>^s__nme2W5G$$XsHV
z75*Y)qN*;jWl};?#!glt?#;!Dj%>UKWnHfJtNx#8w#Aieni_+bs?fwh^-~K4430=F
zR$yhB*vP(Pir394msU*i^z53W(d#)=Aegg0aOPCujE3mShD|r71~RZpYIG-BPD|CC
z#;!3fsgouB=CrJr(+XxzEBHArXXmuynXKv+O?(e?nwC0S?uh(wEz-Uszoj#OscF=o
zv;t3wf+CfIf7@o*J?opWa|XM5|33zy>7D{JEoV;D>~|03ep4>uX5=)*&EfP_*{LcM
z0>q2rPqZceXiHtuma8#)o#*ULnQa9Uvv+kC-BB#&3oJHakFonHaLRQ~Vv+fYnTC9c
zvK$XR_TMy53sAfCQ-C>8kj0USDIwzqr`mKAfnP6FbOKAI{~NQdW$-dy=&{CN>cSsg
zJ}0~1{haqfa{edF`Cm>>39FnRkU2kGlOf4+YQTZ{>=KOsEf+9&EnwDKz&dLIv(z-s
zSprvY#J^xw)VbFCYfELmOY*~tWCqr%9?z;p&lfIh7wL^miDnd5<67ioR;Bn#plHfW
z{}}=sMO8PXSF>o!vUyH4<C+*7Fp=M~Xjl0}HqQyWr!O|wnrP!S(Rx>rL?oN!%!%H&
zmiT0)olH+_at@w&B|U;mR^$p-gc^g+&q-HSib^dJy>ZmVGpOz&le)!r!-?MnqYb81
z8wqc7;PO~0`dhL-X6F>91Jl?~EDwCJ+^3VpQ*%Y_ujOsCSn9PHI<!{g{6D#(^VW)L
zsg*5KD_IUMpR{Xv|15!?u9dT+R!-qs@Iic)>5WjUkJB4Z+I?A;-&Ps5%Bw}bOJts8
ze}dI&)mH**KMJx=QBhy)-lifDTj8<#W$T*E*;^ym9I;w+Y}T61k+btXitbIcI3T%p
zqM~`iq4sNC=1m$M{DO0(kI3HPn#*}OxUpGQ`e28Q3!D7Ixq(H7Q#*uw9|t^H>2KIL
zFHnFnC}8=*9|D_pOc6Y=-t*>q7RC+yiW~Y2SF&+0=;2~);$r36y_`#Xfq;WRKm&uv
zf(;xB3}Vq6rGC#BU|<!9p8BD@VjEk0a)Fm_*{ZJ-l7EHt{bT6U-JSg8hQCEy@}!-W
zEB}8@b~)A;t=Ru{N8bjm)k;z`zj17tWVt13<4lL2bMu|-?}jgC&GhuoUYvS@HNbnT
zu;tcZ?xM)*t>M~Rt-ZFIcWsS1z1Z{BlE7ILo0_);&f1o?Y+K&#i9uRR*B+?(RWylz
zOEsrq#+F^8x0j1*T$S?JBq>lX;(lqe{mEFzpR>6ImvbhTm93ipedUVzw^mO4HGeXf
zD$8sErrA6BH#Tu*B+hd1TJYFz7HfW^WmN0TU8}ujFg0v(J-u-1&&>iGXR^ERp7L`B
zU*XJM+CrZsb~{z?-v4^{9Lqh2R?j##Ti~l;bV-QZ%rK$m*_lfcJZ4Xrby<4NO3gK=
zX6-$)YwxYoZTUA?6aGu=+p%-@hR)PtjkQfnRPA1>HqH<@`Ldm<z`SY3oWxmb7bWNF
zO_z0=ZlJC%DDko6*3SNwoQ1DeX}+<F^f_X9l3Ot7!urxx%RLV6H2gGQD0%@`^+Ac~
zjgm4OW!4;&Jw0FW!@Sz^gr}@lQ=cbgEb$kuTKLZ-S^spU$!q_YKivL=6=?6u|2*UH
z^NvO8Sz8n@Znl22Sz%+JOU!DkGmH8R7tLXFjwy6>&6*JR>xcm7HUZ78F2A<8{@NOS
zyET$$Til<caXm+Eb`=G1Ee);OmfL+Sr}|jVn`)D%#|q6H`Hw6Wzv3QJ-O>I`qIR~3
z$<HG1%@VIBE?e?5fb-0<=4^=qE<w)!KLn;R3(h&wS>Y)#F=i^C;Ytx5<x0t>s0FOA
zu6ix;=3X;Fetv1xs_0Wcr{%Bpp6PXG;Q{YGJAUul`0DU9&n?x_r>_VWE)FpCT`hQ7
zI%efg)f=Y`)^x796MbggP1fv=Gsj}iJXl?HHiqrIOxs1BqVqia&T+NB6I%Ofw`$D`
zf$OgXj-Je@P+s@&R!pYQEY^c1eK*Xfe(7*)pFH!4#V1C~D8=<2(*$^Vgg!p+jFIFN
zvY9$}w!ri^0{t?ZcCsa?MsHI0S!g(IQ^6B|-PebVejgIF-Tb$t&!W4}v}Uu+TG5R<
z%Ima+3RD=lpY#8Fz^%OZa8%$CcbP5Pe|OiZxXU-56Zrq|h?(p*i@#f<WVVILY)#79
z>TkU*#O6xK@5`>fSAtd-rmmjoQ+q6b_pxm0tNyZArN18w-BojHVPV{??J>M+3J)h4
z-OQ*tEn2N3*R{l5U1f*+Vf$%q9#M&cmk&yHpFYB}k=5hE38h5K-#iB=%hVfqlrgTT
zU-0_mvMB!L-lta2zOi(Uh!fB0^)-7Y=<bd*JpEPYrs7|Lb+2a}{C(<>^loPM-E;ol
zSoC-IbX_6F$<@=IoIX&q_mIq*(|7hBTXQDi_pNO^3u6=5Bs0%m(Al^1<n7&>0vlhh
z-e<1*chlL#pqQgm=X{dh&otrA*|T>}%q_U<sK%Vg#X4C)@3_ZZM@Jz=hG+j@35YUw
z&Rw|0U{jv9irh8_ch;@)ihHgzyj`Hlk)s@wAeeQC=}@o1wnH}8Dy2@{|KD=trNjg0
zIWt&}AMV__Xfdzq^7Q=E(n3EK*MBouzIMZ&nU<F{a`RQR9&ntyq;>b9;iVhhD;Ie>
zUJkT5YTtV_A?J#v?-lL3E5SU+5_ujQ-CKP0`Z2S4$I^O^1+U%aH}_bWZ(9DctLCl7
ze8tDlsxDKyXpp~FFT?i4@<pfp|4F-;&dNV)R(h7LcyH!ZiH3yZ(_g9XT6&6WjRW^A
z(NoJ$ySZeXa_QK$>gdhYe=bgZdvTJi@?Pm%o{Fb9<G9x*^8ag|S(kTu+nLQ8g0oI2
z&530y4f$_vzH;u1&1YFR*`CSvIMcVUE$sG-sdX>oB3^EhoqhSunc4H&=FXGVd9HU)
zQY~e|+QXG158v#IJ6l{)BXUjfHRs|E3C3&EEpuN;N$u-2e&?lf<sz3s;dzgvy+5Dy
zYHM$PdSa6y_vYE+n{CT3u1-B%v`+LZM@nJc+pFhp&W^n0aq#WNJA5e#!W-Z4pRVS9
z>L9RS$~`85hgFx}3IAIq=U3&R!|*w;WsA*oPCGC6zzMT2Jo>lp5d;4f9k~zgu^;~X
zU5VK}vEcSG>vxYs-hSZazZ!h^6~Ev$(fLm*7C(_p`S`&|z?@T^|9gRoNoMZZSo@ul
z8iHz(N%o5uJo|BN)z@b?=Ki;wIqR*;*4Ynk-+TN0T;JTIeD-1dR*li$0{HbrZvMN#
z_f6p8(g3bG7q~ub6ZjUueYEk*y;l8idZ)|oohs$K%(?ID+BGxmr_Fqk$X)%2YsmxE
zt~gI4zGG`_U(V;dT@Zg;D3PtHx$Uv^zGG4Qq}A`#^naIL_~e|gz(w01=Y4-f3YNIv
zlbz%;<-F;-u8O%ge+u|+dma4TVz1w_tqbmz_zLA;dGnv`t(sP{TGyq03H(duZ~F3X
zGwbyh`~Ro@s=f8lc@(4nKq0VyV-!C}k<ZuE?AHO@iv(C_2`pytd%$N`_4e~3nTvcH
zy#23E<S$w9hs{-B-vfT3ykFPi<=0%`J39N}$Nzr64Guj#eEx=<mND1CrRn!oHO@WC
zUjOe+{s+Un4=>;UGgx;ydCwB_xMNv5SFQH_H|M*`5V)%6fFJYe0xJ!NhfZz0vUV*R
z4c#t!!pcTE#TOGLyQRdZDIT40*4b>a$t)3rNjzTtOvgAlG#hx-8KYUY?0Pw2DnmWf
zudBf~@)urMz+bJkMfHIA`BwL6v0AS$FGE%Zux@A)V)dG)5v7!Q(!_1@p+JvK8;W;L
zh}*x_`R2u>;}fPWGc0VpB+SaX)OLo2tkBl<{wdn6-*P5yN<BaSQP|pBe>bzw%nv;^
zRag6q`}tMwE167Yf4QB-J^kY1of8aucPUNRT<u%DhGTYjPG8FZwBnaVvz3qE*`0s<
z*xzW2w^#S|zP{GG;mpCuXLp;p@B3|e==tX5(^vM*tt?~Qy}NvmN9w|g_F$7+IWs3J
z`hDg8tEX=m&Cvdi|745GRkdfS{fpzc5<1s(M=`FNdrV9!bxKv85|>7A&C28AhjTVA
z7PXGqczD_op~(d+msefnIMVS-Ft9<$J><oab8TJ|7uNM9sdK4pNmwpcmGh?cNYx)n
z)+UV%#jdr1CKsDLd?OfF%?nf#bT$@tZ)D;$lv$%HwAW?J@~J@zMW+nUywO}N8eg^P
z_|zaXlL=F!UPWG>FRECyVWEZOtm1`MqPr&V-@i)f@v@*3H&RUfSX4_)ow=VT{lCy9
z;wuv9CSPiD$yGJC#LU%LYq3J3S?W547MrsnyPHhjnuIj*D4B+agxhXi@Gc`L<3j5W
z1%9PpBD=OK@?BW%dA{3X(W>57H>K1mc7k`_YdQp4=PMfBY6xxcE&7zL7#1Puy2@OC
z)zN8Td)9tf#<kb?)Zu04?!Jjxe#z<kLecBF(@%HYU|J=nvEafJ_4=|W<?&$?I+CU=
z4!f9i%*W-VQnzII%XHbbWkDG{D(_ln$hJ-tY0#fj*3z+F_wk<!LwnW&*Rv|$?}{6j
zOAC2Li><wN?Wt{I-Oc5vVy3MU4GPij-Qw)~xpK=jKGj`Hw;MM1TD+Jhr)RZ%X4<)+
z74w%>{Wot}vhUrlaKpW`cD^^5%dw~RjB)?MP107@O5civ#4fkC9QBN7HM&*wuIJI>
z{+6FC7oy)M*S_UH&=z5&8QGroaP{1_2+3qe_V1z>9XYJ$d~h;1C^^b|Hnx)aGtcBx
zJJM_%w@ehPda$U~k8#qJ&kL;{$9ya_%ilB6FU&16Rd(a16D*#sTW@TfKW%B3pu+;4
zc^lo#ZfgF}VRUJ9m>k8jLUpo+$<Df}HJcVZUtzRz`_8XA2hYzixKPb?a&FJ@mJ4Td
zqmN(qyZ*dh%YW6QGs5+4HJRx*9lF^P7+5V1I6GzqF{Owwu)7NTl{7XobzED`VxaX*
z;YiccMG0)qOC9}6FZ@?no;7u)7KcUq4V4^G#fR>%jHmw5QBhHyc}gR$W74EGPxAR6
zhHmOJU7N3)QnfXKecQDo0%a@Dh~M2bq4iEeyV;I~JZUeQgsscjd1_a;8|=EPWunOC
z+o32N_2N{Rk)dt>0q<1ah3>ZZ4)s~=%okmJf=Qx#3s>0|9iga)UK0Zrb*_og?p~eX
zp}S>K=e#McZW^}*KTlBLdgba^V0g+&cSHK5Jx{V7xiZ}~SFs8n-MU!)|A{4!#ENDd
z`O_S)>5}T)HRIIb+(w?Yg0s#ioSb!)r+1pD<g6?|<}FtPp0w<ra$?P{1p;>jE-UTT
zNi%4aG)py`JTtWU*_EEjvRQfVR@eVe=`U|MIw5NA+S&Y`w}YREP3#HrR?vRp#Q9^J
zK&jS~HlsvU=VeC)CZ7o|$O?|J-LpjC*h>y2p<4p22FoqYXB=FR7ICh4UPhbwo<qF7
zGcRi?O>FJFC&<Z>FiGQxLRaXTr@lcOCpu+p>@vR;sJG|?m#gM2mHAhkL;^bwxs`7h
z-7Ufszoj6i@03!*nmdNeerQX4SS7wUYLAA-UjbGonVbBR3-U!hGGw$D6^I!+Y?Zfs
zbTYBy1wY3}mIZCftdc(`sRmgb6U)gu?su<TkbUlowX2LTo-)}KX8ScoQ>chlQ_6G}
zi&Pi0>u+sF?NA3V$;;E*U7s<sb8EJiBsK9a^!)E`w{^DPmt~ta@61_YaC%N~se7m#
z%UgEW?1QZ*vTSxFb=(kXninfrymckZv@B&IA<1P+wl!;uH6#_?ZhIpZprdnGE$@bN
z`xd9F@>e`Uxf@;1PCeM^C(@v~;P)IohmSiu5;<%mHuCM*5#qMGK;~55mcB_%*Ztg@
zl*2b|5Dip0Srw>|DJJ<+_>JYJYVN3ZhDquSz77mr9!@I#GCyq`HZAINewi=Q5EQCg
zvR={O);o=>Z9~J`(-R^EBb*D2BrUc!iT*F>XE+hce?sNx;jIC}!KX5&lzAT2)J?t^
zb(8gR$<ot3T4`SXSqra6ew#6EE#s&DX0dG-6}Fw|WKA=W{Xgl<m2+M**0mNnr`(>n
z!P9uwNkP%FGXWEx7g&Zw{@!%GuX)?Xi5GbJCDUgW2c3JT<F54aqqOyGpRR95f*trc
z16<55b{+9Oum5I+w}0Ixt5$|7#qD;*j{I3|trE-R)@87II9y`u;L`V)_KrR5OH-ym
z=fXuIy-sQeW=pm@DhR9HTqm)Bp)u!ZhMQdJ{iYaORUdO7H!dx2kqeh3*RZU)7PPYc
zNC~fk{6?pWkXQL?4Jr)nj*3iy2HRTne<?rIWSy*Nxmx#DAUAVv^byCDxpq^ptJOQ`
zIq9z|>wWx6V9l}hTIx|8d!nZ-;?PWW|Mo&4{NvvIQ@JM$);4UJ#%GlB{~%k~26l0s
zq`(N1c&RQWz7ho%RXOd>%SSeft*>A;lvs5#=);^vytyYF?QXUF=n7x28(n=Y^7-<L
z<*MhqI5!3{`QNpAt;D$L;g&a*MJ$>sD&{UclJzJ0!a9;V-fnW$y!9Z)e9yt1{^l#H
z>jWjM{~Qv%*YqVa*ll@jsulOKXpVzoY%XC5{9Fxh+Qa@y+th1Ja{GE<9>Wm>k-ERD
z4{az8*yr}pqsQ<_(y`Z8$2*vouHJBR;wd>UnTZP&9Q%*zWIXJABz;ED$n074?Y#DF
zhKDy^)n%L5^2~Gpvn^{wt(JVb8Twr^^rl{guBzs}Rj%4)YmEPHNK9XMXQ@u9{Aq>n
z|L0BH*t66nO8L(Q-ucsxh#xRm`R0j+zWu4gYfc%@{o|wUUu=F_^Mvqb`BhINj?cQc
zyl=*4&20r*PiAvz+Pg87pX)yIbME_|<@N9GObXnx(1A(dY3Qp<@BZel-8I{&S@$KA
z@C<451LjK~tP;N{XkMsoy;6wfg!udg>};>+B`3G~I54EHZeOrVpY6>=4})z{4azfS
ztq9&N7@j?6%Zm2(%yPcctOA^qUOiktZKq1x0WJo{iQFv>9ulmdF0rf=Xtq-7+S$>R
zqsS#F(_HGYeICO^nUjKSF6L`?t=cX$PiEyI)}30jH#(IU3-)iAf6e*8?n~z14sJIz
z=`r5Yy1JOvt9r$Mvj-wZ8q7u={Cz*P_dIOcZ)CAwt9MpGZ`Tb`dzo4Gdk$Yc;mnk^
zBRxsvS+G(1EK%3O1zi<KI#=j)pXTkDyyo~M7m*^JGn;fyJ<@4@!8h^qg7Z?A0#XYj
zUUKkhNQ(&B#c1x2erPGWswX$A^G`?TTnWzdN4gz1*8bHHlYTTcqNJm+Nd3qO`!<2r
zec2o}-CJH2tqa@19DIOJ_(hX>0LPpat=|OqN)~fe1<cSCTv>f`+Ifj){ZmbW4%~J@
z27fkMH%3mlY}o1ZV{Yh*NozG$EnLuAeRBH#U;&dy2lQe%>U@Nv99FJ;#TB?)pWS4p
zxUj7y=Z3n28?SAgKQmEMY;)KDc@e5THsZyVwl{-?!!AhPx-Dt6p?8u^?`;pM<uebL
zIPocd-aEbI<ct<id2YLxvpW(Gopf#4_oTXGZVJczoP95ok1W`;$v~T9&BkRPf=}r<
zd+BrU{}_EFL`W<tro(vmF;n3N-NMa6I~JC0vYf%-Z7n>(W^tEqMzh`L-G&BcS074n
zFe)*5_JqG^;cM8}wu_l@#{|Vy2R}S?(mLqyQjjbBG}Dw{T)P6~9-cHkvD&Pxfr&9e
z;$i33x(;rQ3<HO%d1itQI~y8Szi5Be=wM`Z=wDY$>SL!@lB|Xi?1F;xE?0I-J8nO>
zV(a2fTPk*&NEAy-lyLn0$jL6)rSwQb;O6xICm*hQxMWV9v-9B{yTUS@mp-2Iw82?V
zfpOX!hG}b@U%qL2snhuSP1Bn<46Qr-7+;)w7qUF2aIK;-uj7}QT?|rjPCLFW={Ov;
zso7Y8!%Ku?)e*KsJJ?U1@2b|}a_YG7X-$wwK%h)OUpGfwqS|J`hRrjZcr~O~g>Z0O
z?YNzwm^f3h<Hu~tPe)fTP-n25mJq{HeMvF>pkAwjGUH5PQ5VTouRD7ZOj<X1F%~on
zt(BBoEX?($v*wPEuE#N}TgUcBTTjS3!@;9j=Q;P-n)a%Q)-@eXh61hAn=}~?bMI`n
zu~t0M_vqMyj1##z9_zS;cCPZ6B^bOf#_($P-ZQVK_0QS+f72~VtHZWe3{G0?-L&&3
zUuhS=^;&+LucvH7m(84V(*A3(yz{=t*8B7pG|i52dF~zJ`o+^V*JyUkl}}SRUVq-_
zb(PP1?XsmT#=d)({pwh2w76$V@Iv2NT|piF3A+~V-^e>7gZ~Hf9O+fd1+I2Xm@&<-
zYj4Ay{UxhCq^3-W)@Xg_W4B9(vE`CV{j9FK3oJdmj&rg|sBSUQc(VMMOsnJM1NSek
z>s-WrsANgtX_G0Fj<1toRatrH%#!xF3s<$*Y}+oy#rDLT?T+`%jgoAYGrylavFw4?
zJc+|KrhfI_JRZ#^<|T*uqfYK$<oxPM)5|00nthkQj*0mA$M4;jbFB=gS{bhWf1kq9
zadh_gn)9EZL^yo8VQq8X!K9b<tMdV?BdykIu1huvPnvqP$;G|p=JcyaI&L31-X-Rl
z!*QJJ0)we*kjjPmvO5DiL)5vK>)vQ+wYa70vA?ZA`fGNFS%dQ`4ejiiJKH&~x8Deu
zKbbRmqk5abu^9<s^Do=jFx!eG%7!1{Yg^#fvWs)=WA9x|6OLPV{h748bVa*xjj~Zv
z>-yKnE2O!71C(_?DQ%EoW)-xpS~-!=W@fNbaK{plRSmcGdhV<&*qe9Lte@xPrY2{<
z375|;oi%liGh@K=-6w-BgE?$PV=w1&tS{lP^Ihw}dgVszT?Oka*H*-OOtpI&d{cGu
zrnD@h|BGZ!t$5<;-P^bB;3nUze14^(*+r+Gc1Ycuuy$3(?CldT3IrSX6vVxFGJ8W{
z=%g<@dkU_GD4pJLr6cO9nSFG}bc67Yj0sM{4DnwX^aUE4r8tgkbQ01$yJ@NV|0wVI
z337}arW+Uz?N}h!8{|~(EUWW}Q#w-EN4hKCJN#yf_Rma?HA!>t%-O!+g2(PnX5TJ`
z*R9_5*vC0P((huCUtb4*i|mn?f3!u*5?hy^?pk}}@?FoisU0svIzF)YcWvd6X){V-
z^`A56;T6&IdlQe?%0$i%c$DCj6qfSnyNiW}T-0n8okfL59u!30IdHz?vj6O_Nym%U
z?26<tyr$5v9k9SP`G0eTg6CYO(2%37r2!V}I@w}xCDwFUEk3$@hwJS}%Wp4I+_Xow
zjidASM#a_}mXTZeXEV!wU{pG|GhxTq_UMA<S3(Wu2F&WaSdTu~v+BmY&<OFjLamCG
zL6<$Z@0K<BQ|V#Oxo53**E;6liq@EGmpRN0FBe<JpJDYhYn`^F^77<0y^DBeU-XT&
zQH{M~`YbW*Zg(xmHQ%(Ww$Hw4*xj7^>}Id$quqDIWUe^N-n(0C#}#@d|68ovlzlxB
z`<9>NySqZoJC^V14!dU&>CgY3U-E_bVJ%mnYUtG6kNYEbS?_h8>7lM+Jx#1=`Fa=C
z$shQuH!c4d+rN3$v@niEfjf?`{1Ey-!BivI!kgpbRfE{=rK{PQyEAUS@nQ8dXV1H@
zIcK}h67vS1z-=O1t{Pvi*4<t+`Q@t5?JH;RI&va~_u_>1sB;aaarZrLygK67_Vip^
z?v1x+PA<vgc=0(=Aup@H;EGHghe8_1s<MvuyR&<yc0|wY=-YePf#r37sH;6s<fd<j
zo!_S0Ux`qkcOf||l6ln>k15%%3;pM`PQ7#Bh~c&DuIx2C6_4(>dwf97HOx_Q*Siij
z*ZrI`9&^rMTKbh^n)32`vE?g!J7VWtZHwrf)^T)0Yv;zN7v_3Q@HMktHi0XA!IO70
z0=Ktbznm>FC2?Z=74aQSlJ{6Hi(1>BdT`>*<h^&##r*%<;e6iIQ@h&ur$E|u-@BJf
zpFNJd`^4?tv#`9!ZF$ea-d)p;-G0$F>)5-@Y0sRmrrqIHyW47%5~Q|g8_&zfvDca2
zKd+s&F7TdC^>e+`Mo*?n<uBt&oa4nf;X>BJ*}Gq+w_lK+z1Vn`@adb2?EhHbpY!7J
zqAKA(6P^A{Hd(rQOVB*7dvnfSUuZ71WwK-QM(;=aD;|qKWGQ^4a?VKST<zoVCoNm%
zeN(W|VXpJLA(G}<r}yg3l8*|{JeQt3+0kp%QDE0{_u+>(g<Y)Yo1Z2sOwW=(pV+-L
zVp&|#s$;K|*seW|ex&(cR4e|`y1GZ#Uu7+-?0E6?*|#;1cEl-oKRK^=%;JCBg_~^x
zk9A*0h6%ng_kVmM$@P~-)X{atj0FM5*ye2eSk&I3(DC!J<g3Y(Di$U(dP~gOIsfv`
z_7_oYKi1A*kPXtE)}nH`;{$8B!Q}Q!Ru}EUb_Lt+XnlHWhaq3x-qWsz{IB1d{e2qG
zmlx0WEV->T#VzkjX<pj1XB{&(F@`_u-f+2|FZQc|)Ff5ux99R^?1{aV%FDPSHYw_;
z=DexNn>wWWzTJO&cfphpkGT<Fn`im7ra!-HSH|`|uRi~4+*RgSj-^{USM7OT(`QnW
z)bU?!+5bMy6LKFS?#^r}TiE_XtoGKD+np~&H~NZSbklZuIqim6MY7ZMV}^WvnL_M|
z4P1%;>-JieKX@6&5YZUH*EoaMh$E}H_2Z(xhpnZBO6R_HPJJl1@8mMKpE7-~IySu0
zOTSWa>f7pR5$&Sq{~h~z?N-yZSsb5lR7O-iUN4p{ES0_RedX4=M|0HG*2Qg_qARtd
zFH%?U&C2;5Y=<IupUZ9+NN&9`$>5*M{L;@ReZ{8hvRgM4Gj8~-nUG^Uzw6tI&1MS%
zY~_~6?AbZ5n(MS(xy&ca*&Lm^A9AXsSlO99ws}3RSbO4jSViT>jyrv}BG#p`%Zw*I
zxf46-@~h;N=BwXb(S7#ZEzNH0*Ofp1oWK6HgQNCk-@B;mf3C2XO&2LkD*v10U)K5J
z-Mh3SUOxLCoOs;%_VfRHU*ECtrWgH^T4EEoxFoLU`M$^b_g>F<zoM~Y$=V|S)_-3s
zZq2#iwXS7zX%I(+{j?Rl@nNSnXBy67^yK>TRQ*6t%%6$ZwNtiR&YW{W`{N^%!@|>l
zbnUOWxU2U?O5)2a|NGUmmaugQ#afjJ8U!>rws5hQM7-F*EZ!<8Y2|Wb!=t0^qMBY?
zRv0LsXl2zjW3d!mbhMMnuj@<1q@~Aa@TgDA@SLQ4bQW)R(G87+)8`jBv~%fNrKVo!
zaG0ue_fRW$`%+J9CeaI*)h|u;-)i^v)mLqCzl6hWZx5~H7T=u6THsW*?QQh7B5TeC
z-B%Uw29=(k^62$;{RjJ-n1$tXb`&K3-x$a0lkleEkjRO0b$uP@Wjv{jDFTzSmQI+$
zV8-pP%c5Il>=ecn(j0f9Ys)iVrj+!sh^c3mg>4mhw{GdIs}g1ZUMao_k8b$*X!2jp
zl)ei;-QwS7y-d4$swzDw*JI+#bFF8-e@Hztx!&rZ&Ge6=hSyW{57-`>^-1*H`kfc{
zG5q-+dMVzcIGOWLX~@Ke<B}B$P3+f>x7NJeyIe%v-BaU$h;V2{Vv9&xhhn>6)eXfC
z`L2qE3?V5yf?74)LN0Vg6}C?BGW@lDVvj+Y$iyCtdmU5ybd|0gt#nwXo+9ZoEznKf
zeb&K~6FtmMKAGtAjWty=D6CR$^7N(Q386tghDXKz=W3{^IpryYaLs(pdEt8XUtO0b
zr|_+sQcf#WBPPz}I%qbVVXd#*Y=*TrlKi#jX&yPHw&u#@IZnHIpLa9-yOHd@<lm2^
ze*I}lt6nZ`F0*>MWZjKRUeZe@N!>`T%{|H39AIb>uv&oCLVFF<$*KS+d%Y<a*U8t0
z2y{#Ix?E_`xM>l*QC)L(j@Mba*`Z;_oT49WVGgSf?~OPZa&g11XXO+8enz@32#}Ex
zy%-oh<M(B@m+QE%?tCA#W71CltK3tAKU|7lxGx~nTJS)M?a~<sQr4<mJE*O7>{=MN
z^&NvPWhYxF9O2&Ty3wi9@@DuX8ToI9NecC6mPGhzvPhnCzZ)sJ^}lp(Slj%CiBHqB
z7aw^eFqdQEHuY48iB4Xj_FFHVbK1Suj7|H3B*#kKw<f-G^Vg=Ro1S?*apTdek?rhq
zRk8V<aSiV}b@gf<9lJiiZt~0W&|BqO=ia>i<K)|A&c*X%1Me_g`n%14#>3i}d+F_S
z?_byQt@yD=C~rqaM{oa+9gn+~pJ?s0QEPb|cJJA%3m#iJZ!CTq-DbC|PdYL3$<&vD
zH?L1z^IJ4^^19%YOI}S2jje8U&wa3zA!*m9i8Bte-%^|X=HU(DIqB->XWrKip7!}n
z?W@kI3%<Qc%UCdb1LLa&cXu1Fl3up0QEk<<MsWqs7>6TS4FWsmSO5BWTk5~x-15Zx
zfB!3QwGDWF`(4z6Eq<4DXMXWgQwu!7P$(qSlhYC|&$wWH+#?}Yy9<%WCWWzTYB;j$
zajSD46x~tu=wPeB(iT_4#9e+F(xO67W3)pe18OF%-FRU`TXN!Qk8?s3cN}ol%U#GV
z@Osn4F3l%N#ezwbtkgD5Ff8m;UL6$fCnhPe&c}P}%X4Km%4(MnwDHDniOe|g!c}jT
zTKXbJ-n<_z6BaP8Qp(j4$!AG0d646nRd$6>{lMa_wn@3!t)HW=AGx|<<IS@<ZEMae
zem7g}a%{On&Enag*SOA?ytHD{#62o%58JkdwfdHDcs^7YZQ2oN<mWu8Sw_qGiO9)=
z8AVgI1z*gYs{8+UpqksOpUN+ac20`!>vLEk9W>z_^X|lb=Xe@Dg$y^BsNV4Pw2k~Y
zNkl4m=EG1IeTFU$$!Y2HbF!Ar=H3`Q&*<gZxhpr$u?gl_5MikCN@t1a>=_59T@qIa
zTdQC9r6kn$`A=o`JioR{Tc-W0Nj=CDx1pJJt`Ga-P-UKflVjw*$h+uvdS<1rZWNjN
zBIVWcz{0ObW@=A48?bG{y0X76OWqWP?}?GnU%8+pUhi#qh|Ht)YmS`I<~SU4kfSWP
z#W1IxH(Mg=iICu~KLSy1A)SJgyByj@+&X=0RteAc;L=!UlsZvlvb*Nuxz4iB)6QF~
zr+R!gUNAw;{Kn+utYZp{>pK7UIp${dD@bOZ_}wAw*tmN3y*q3t_h@yjpAs1z`?$s2
z@?!f2j;wp}H{EtbioR=IRP#ndSUoaql19<WJ!?AE%q`_o`J%R5k#X`=zjdj1+ZNHI
zKCgCd+jV2(H;)rFca4&*t#4mE=NJ4y*6GL-$uob>`BZ)6`F7z;*mfx~KjBPY3+8CS
z4PNi{s*_xGJ1+BCC_R~*H~0Kqi$n5tb$P1d3!k{GGYV$Z;a7CuCDYGzWzCPoBMZ3i
zUSl#%a%c>i(D-)i`WpFL+DW|y`^0=QPXBwYlaczNS)4JP<L6E{F*eoM=!}Q{*Gv^f
zj3hKPeHL;Gc3oa4kYgDC@J+bfv&(YtK6L!Iwb{u3(a2CV+$HIFk5<n@DWjmA)UIct
z97)E`w|4)~@nw=Yns$ccSXaTuw9A(zrB?4g^wi45EOlvKzK9bW>kZC{H<!M?pSNb&
zv!_37^BmSXpZbt$KIIT^+O*X^AD?-+{|erj9cJt>Rr|T+;@pjMa$Zl8WjZY48#ZO>
z8lA^_)m{$8E2dhkirJ8ox%Pb0>7uT%Eqb2PnM+iD9c!=&s_K~eQrk56vi~m=<KW*X
z7i8`%w^;UU<>>`+Mlr3kAHO`Ac4EU-D_QnA97^ixMw``x)&!e#O`OH@h*SAVZ@Awv
zF`d0_f9HKpQmcF%J6SqL@a>1hXGWK=ac}#|e9*%F^rG*%|Gzl1Xaq7yP1(nF*W<aK
zWP*>@js{D|g1-@_LJbo?s7qFT@c3MNiR1O{L%jF4?~2;RI*Cn2Q+sb%NAb#FzQ-94
z?P@FhTQ!t-rJU{%3f4Mir03EiQ|dcMtS2~G=hFe5p03zsK4I@aZ8}%5)oj(t(k73&
z=`$9q`K0%LnsPAs)ns*#GpTupr(gXt&)P#;aqSM*&!_$<Y>-_&p}aPwj5||d#pAE1
zmY*}4CE8i4qkLk+x#Z0c-u1{a$)}&6#G9JJuKQwZ`3cRbv$uFPxR&L9H9A{Rz9qAK
z%HlN)o5~y(#pcAn-nH+~UQbW0KNFX6?S9Xoe`du#!`EILB>g6ZD;7AN4f{XY`}qsm
z<zn9qtV%r<8|^-&);v@$-aN5ac>R3MTU-;jX>Z6DHT1q6n)#?<Ms$AMSF;Ou1lQ!3
zzdk*o<oESv&Jy!a!b={M<=<Y&?kW~0aqDfqe1yVhZt;)d8BX<&cP|lNHo;TfMxQJH
ze~X$bS8!U1v-BqW<pR!zUEV!Oeg4Z<2ys99J8{K-mA*ay15A~|O#1sb@}yq5Q?{h-
zjKs|>o|#S#=O=UqihJHvTOv65rH}PVwhV@&k-9F@9*h+|{n{SUGj}%0mIzOO>3eI(
zJoUSc&!!&IUBTsa$LYOGOM1tHEpw;8@r`xOo!2MJuKH0y;MUZ+hSRFn&You*rlI<9
zkH-InHv_MA%n%Z3>d-7asP)jJtAue;Y`560yLU1l@;|$K_o&c4Mqxe9$vKU;B->_9
z^$9h$<e15muwVzHOp<8Ymagk7Su!%%i&!7_e@&E@y?LfW?utf>ZpS^bDUR9!($cfS
zU8XqsmbAy;>DOeLn=rMJV_E;4p7tjXXUyEexO(sL_fwkAMarH3slX^Ip(WP+K2c<|
zLwwEO^9?hl7k3KR7xrmdg{;4EYWtn8H9qlSCHD_lM(m#}e{kXz5rI2D1Z51lln)5r
z+0`PnFU8OIrpIB!m_s~0hyI8h>5MqC)%9p8OQ2`Wu^#2Ww%tc(P8V9b`2X7EGgFdQ
zUhPy%V-xH3WWM%)abv~Rj0uc6F~0Lydn2|wl}~76`{kltk)YYKKzo6w%OAg0I}TN@
z?Ua4N`gW$LtIuq&j#JLtF6(Y#%9_ZybcbW(hoibD60d$-=vj1kPT<{B8|1w6T)g=n
z&Mlqfsl<27^`UUowA7UyPeW!2KDw(^xbRow!hkl#(<`pMyyEwI!NS)XaaUF1vO~|V
z{L7cd(SMQGUvdlMvX61T2Rh%D9QV8wsa+#?vE?4igr?V_OD`Qfv$gOb$FXG26|)-t
z_&u$hx#_Eme?W?4Nr=XyxGjlO<2<5e=PgQblGkawy<Idd(M~>TSyH_s$Igf0TPmXK
zRMVXPE^&Bz{_9=;)j12DZEiHR{=d@vaJEP5{fs8gM+Nbj^X~s>oo|#hBjAJkfdJ)*
zf>vKc8M8?ehaaT<SsG#aDn9V${C}Buj>=9v7TMXe_Hppqp0{_BXYHBe+;A*=hFfvx
z#f5J;CHA&uJ#d?MVh-!SS*)jyvHhDRvO#sJ#gmFHjv8qy;wu*1uejtC@#M8)?DVOP
zJXMb-3e2s#aBmg|L&Bmb;a!(q|1NL1=h-ZfxO&d<Ce@6z8FPj7Seo?~`dTe)@k{J@
zv827=_Ld{dlKZ0X1wPh$IjP<6)}>r2gFwzsvy;otP77YdIi<({Wp0Zp?|JDYw~qzE
zn{tJnRQ$6KEHpoRbKalC@+8;1ntAILTeP=0B^Ugc44J~TbVd8Z1?n?D#%*%+?%C%a
zWyJAjr@MNZ=F>u@m{lpeJm-HoJ9max+AK-6H5@avy_U@|i?7+6*~sht?d#=+&@`U|
zw^yI=dKsX2nBk~e#k2YA0z?0-G-y+s!H{fdq`c_a?Pk?1<3%C8rmZG<uKjO9Op}zC
zl_dvQo>}I^6LeJaXlL?@Z_f(sy#EFEB(~fsT=yzoBi-Nbxzmh`Stop49sG-1MKz~{
z@&vY}C&VoX@RE_7E0V(?-m)MqhpkF4I&_;8ht=g7iASY58fLCc@b$^Q6Szq2k%xNY
z+0~z>UYXgq-EGq4mA%)_PHo(9Z>N<%XT{w+5*o>=uh-2>5&rgHGFA1Q?<QGMKPlal
zvEpUfcMr<y3#xWpyW77+Y_f>lJC?Y~K37u>RD-$XdbgcC*%&!lrgKW3%ddoaGcx8i
zJW~GZAzrHJvU`G~k{t)ThOCE>n6n!Ho1aIbXF0xG$S*#DK{7(#_dtKxu_tM6E(z~$
z#OmBH-10JUosyng&~C}K7o`+GCaPVIl3%l-%l}xwg_dRML2DN-d$zDDK6BRng-L6#
z*U5ihIibxh@b|?Dk!xC__OzPKI(qZad;!LEi(_vEAKtfMn|SOl>*{B^Cw8oA%xTa2
z$-?ZUE|GXXTj7XAt6;_!rqZB<$&8Oaym-8xG269)Q?jLh)2HX$&t}LbNHNa(&!=Y^
zbn5Mm9lPGd@n$r=S@3*Do`BejbElq^3$!*BJ#Vzpczf%0i;z@n+_ctDGsX3;zBCKj
z`oQBrSJ;L-u^VLcm$C{={u7c~l_6Dk$1TDC<R|q$&v|yWlc$`$5^^!CKWpCFi?<pl
z2v2wwrG4e_3JVwQH++&)BzeC1g{1{+PkFG0V^UgGnDR;CQxE&M);yZ>t2;GDwyt-j
zf6Dv|QqMLEY^df`TX(}_)~-c!*S(w@B*heaA$6DZg?ULkZseq$)4wv!XZ|_11$<ed
zA-W6W-Wrza&d71vEuqCBzA7^=*Pu%NK=1v%S2lE~XxzLs<#+0<!v%B9_N}#;lqJ~t
zs;jh5Q1ZW_W!@a2Y@ZV=w2D5l-k+uYEv)m(kyT$#yo$ZC>bO!~WN!Mr#Dw)$H?utQ
zta<OqPwZ{3nJK%&N!sGX%(|sB6ve{&BHtCx^_sR$Wc$f$j0P{?eVqEY{VJ=`4nd7w
zzY3Mka<T7t-Pq;Vb~HQJ<Wtk7)hgZ^`@%d#*R`B%=)02|->~na|GT*dKAq70ILX|r
zphGXTlj%~<RlmcRraj!dX=PxlPWAGYd!wEz_MdKGoAqYltG4$SOn3_<xx_0H%yQLw
z&Uno%Ofjt2P8Z5eHR;?c!}-QlI%{M8XPF3btK#ikF8bxg@2?*F#MpgVsO9ps;=ivv
zG`F|rG+Ys5$?^=<<*NU`)@J{PjC~tyt91?f;&(=D+RJ*TJuvHL*^LMbpRR>7?mUyf
zdr9}$jko`J0}I01?zxrZKACZz?ZKMF*0p&o5tS1kCbf#QYyQe=o8Pz5KZns~?woB0
zv*ozwI@LsK%AMTub@>sgm$F+<%0>kLNO;<QZ=?D4n{_|l9o_Wk`rIc?8k-vL3E3y{
zzOgb2^U<C0W|gPo*FCFvG9RrAeZ0DL-(At5TiUtj=E$88-*#2w`K1r9I6h@<V$oke
zanAA785y7Jq-IOV%DkHvn)Gzd0=LSCiSredj+d=JedShc$H$C6C-ucwUYy8i=snYn
z_ja07vCMwX#uuutS>ozqb5$orx*u619sj>=;SRZ&NiUg$3t0YheYJhI(I8~$+sO@Q
zGMRt7Fott~YMS=Gs4uC@h3z_5-t^DY{!Gc~aLs3LwPybwz_QB2(XH_Q=8Ud1&glh3
z{b>efs}`v(R=sYp|I3jjQQPg4U-5DGEUU}1WV^azoy4)VH_Y3<uRND4v^v@4{oBN^
zTm3i4Fhv!*Ul*0~UGPTlh(t)LRHvxq<bVx-^><c9ri<TAiD}z1i}SqSw)+N4+;?u=
za&2C!$5WRdXJ;;)6Ytw*ohIvTefKP1$2QI)|JR{6QdbA+Z@Q}S)nBDF(8rC_^Pt#{
z9o^+>!HZ*m_<q{^NJVRAN7&-h?PZ%%U)>FO7W*;EPjm7A&w)~JmT%n@tYsOJB)a3$
zSLa(ADwXYPIz8RKYw#O|=VtU@_6a|sWOi~Er`}=Cnc;HbLKcmi{rsnN-B(G9Z+R3w
zZz<=j&Z?B?)C-kW9#5n7(q~p=bo{qmaNYMZ->$s!4^bVi{A-Qp|9RQW`Q87<=jkq1
zMb_J|9{X(TU*%DIpELNx-U<(w86G!pr5!9`y}gW;|7XB$u@YzYi}}Bnbuw-&SobSs
z&Gw$JF|Nh)iqCC#eo~da;n*4}J(hbt;*ReMzhqtbFw4KdX+y8qsfBE50gGJSqFwxM
zpT5&r<x>6DP$?<#$>#LmP9D))i=R~at)D0Cx1f1z`~QlI|K9Doe{{~e|I)9g{n(w?
zDaY#?bpLly{*izqKf1qH_pBETz0DTt?Gf61KXkQh-}`q9YZC>U!e8`!^_$dJ#c{Gd
zj_-4o)Uo4Y!TX-y4_g|;AO6duCGlm<v&t{=k58Vw&bH2yrTApq<rb3}|NWG@6nsuU
zP2K-<)`T^(8pc)I9G_Y}DX(yg-|=(x&d{1C^SoytlAglE*%6s8t+_Q#MRiehl)3i|
zA&~~wq8w=!d(-pb>KxgNkLbv;XP@8ZA>jRd#jU?9{zlK*eEa>k@*if0*BFbhDYkAd
zC~ZySbpKUY@0q;hSy6Bm*Lnln^&5)j-#3~kA{w-P?*+A+L93R2`oLejn2+<SkaFuC
zzW-NOr`xTG-}g*lVu77u(v4rDt(-OISFM>AXuIm)sgjRZ@>p_P3gwlmT}2+fXjPc$
zX`OZ?-+S}Ns$ctD%h)#?zpPBO_PCTf!*w--#@}Wi>Ar>6Zix2zq`941Ytpb`^~;*t
z-M^O?ecZBh`jXc6xqG_jc0L!pcB5QMv-DcSm9BjGp8Y>^zwX}0z31Td{h4f13qv<`
zq=-F@(QqgbR0wc5*u=ss=d)u%qH{YlFV_@~!X(#@Ic(D=t}tADR8NF8!R5qK9`#;T
z|5+-TlX$!)EjhKNMRP%KgAr5I%`26gr<|RwdSc20r$Y)Cn`Brfcxr73@ip_5cbjUp
z^??Rs*h;(D8>jw<?GCf#oi_1MYqq9QN|sp3<5x?x8JC{gQpCJ{iuCM^yZ1`86V~1p
zO8vu>wt7Q?q;bP(j=$NPkGSh~$$Pi(L^eF`n7T6U+Mb!O&K^p%Z1;N+S#qU6R%*H1
z-l)>-YcVGZ9yJAiQ)Ap#*nVYh_2p^zn6@#@-<0b0@saQn33<Dk?)*n*7n;ZatNB&(
z;=sc2`v$G2MHugFFW}Ul=w12w`NieaW9$CrG$}ckEvyYJKloF^`sKX&^854tG5zBf
zaA^J{yx{<&<SBi{U8fQ*9As0hnV@$r>5+%l(xmLxDRUw!HFTRj1Rb;mTzLf58J%P{
z9&WlJ(aN)?VfHFDqbBVRk!YF!O0G@11v$-2jB+#=A5_SH)y=JZebZvS@H+xQE9c$X
zz^!zELyLK9Yl~Iql3uq{Cl&~?1iU$YMqt*eGpC(?Ypl}q_TH>$D&ADAWqDtR<IG8=
z!@=iux2hxyOJ%a?UC{l*VzNUjxu=j@lP~qkMK^xYbqX$>v#;!ax+7uAZtvZ{*Io9t
zy}M<1psQ}E(mOV%soYv_N*sz-w_}8TCv0O`$=#QAcIx72)<;{7VoG;kRZ$GJTB$nC
zPbtjf64$Q_3${Ga>fl+_;?$+FWQx*8P2uII#I)M>@LpFaT;KK8jPce%!-aEhNoX$*
zyA@%yY%5E`q?18n+O5n77Uc$QJi-w6X3H_Q|4g43YP&X1?&jm#w8lW2w`uaZPUUG`
zCJ|@P?nvxSo_k~K6(MKcC;J`FebTv+vbdM~c2@UWv+0G|@3uKtN9Q%YU*>Sm;^Dm2
zH+Md)*_PMzbX&2W#r*9WRb6k7JZ7+2e)@4ksGfxZzvgE)FaN!to@nc<tiS#{YQcv0
zubme7o#}i3X3NEQ{K~)CLIfN;9_V?z+szsC;-H_>o9IJqnkpwAtn~Q#VWlXql;Og|
zu0AoFmwOa`^!JkH$~(LyXLVrghAhDYVnJ<Xr}fMitr0bxzvGRE_j#qu9varH*K<Bx
zT#~iP^O8;RV?JXcsX38-rN7oZdeB*(QPy+3hwHn3;$K^nro;bP)|StXT3>fwXSF`(
zg_GQ>Q+~e}_SFdTke%BU=*lY<f9cf<<#(6gJE?}&1;3nMfAu$e($%mt4b!hK`kyI!
z;6G!w@&`drPp)L4#HA^bTrMY>{J$t_zEn|n;tEcgP&7rxl*9S)t9a)<sR9x=J&eO1
zcG+fq6y31pwXTy=WRl8cb;*7248?0?-<p+8Ha0v`IV13L%q$bt6&%a-n+-emF1d1;
z-;1@<`w7pnStkTEu4($L+4$&qgt5_;H7aHsXZDH*m#zEx;h3f4;&V%j)LIvcF0b{d
z?kl-<eBTR~<7so4&ac=q&E59h#q7P0T5jH1ylLagmv;Xy_Sbw7-p;sFH=lje|LA){
zEGOIk^iHZvKG8l$T=;3FfNO#v!y_T4Q~r7?E^8zbHHD43L{#7TD~l|2&N{TVDaSAW
zVC3CtJYqAORQ0RMy6!zl3*Xavr)9>WFFum*6vV^t9IULp*s1$1XoB^Z7EkLrq5s!N
zx~dx;nI*YdZFbf6qgQO)<`_?TsJr!er_<k;)whj97HMo>;Wjg$%gbQCjp$QZ&$pi~
zXT9qC{YFBo;nw=PWnNcT3UCP>(h;1)o6EtV{z&_@he){STT!dlFUnIiR;yZNI!jM`
zeljETNK`zR*GI1JYuhJeD=M-ms_%d574CP`?4_26HP<0M&71}il?tBCJZvEk4WAjO
z%t)B?f6DZ=vr8U?+0-;gTb**=x#f{c{4P(wPfWI2+dOA;zI<iY&K`C-WLb)mw)V+0
zzb+rsd950&a#TF{*-`t%ly%R4_2#RYa=v=u!zJY5cdl)5_=2x*de6^vx%l9P+xr}+
zOy}%dmv=aCJ>k@w^>}5}l}%}KH&dqBzGu9he*9bHZv`WV9Obo{&x^P1eE4?a+Ss)!
zAKAnwhfH!bc-d(npQL#9#H7N#rFXO<w@l#cJ+b{@Z_M{g)9*ChXcN&c5)9i{q{LW~
z7Zenbl*c$ZI{d~$V{1pB$raDfw5)k5s>`SzziQ?b>laJ(UH0T<ta@_wkVejmlH03l
zgoW-uG(4@eY0{(5&szT5GS0s|XWiq;o&tA{?p(LIn)jTH^Re)KPi}k;oO(HZ>5XvN
zCcZ5TSiWvp5wX;sA!zAr{+E|#{;HPjK9H`Yv+R7Aagxw%k?vzWF(-YqHtk7keX1TJ
z?7&vdIeUqQ>QdP$(;`BDotkOoQfRr=)ix+&!knB$rfpl_&$QXReydJD%Vj5hL(j`1
zi~^~r%^Vl){bwY8A!WMq=_AvfZp2L3b7gf$*3B8t%YMY$e(pbSbLUm~N0~4O`-z>R
zyB{r#UVQZ01Q)dwh1SK2YyCWT`&{}JdGkzFvUHu8dvER6E%7%kBCOKX`&wm7OKn8u
zK2)sge=6xOr+?c0%a-<Qn?0Y@iA0{T|NqF{ZtuJEjZdZt>|u<a*f%A`G2^xQfv07P
zTTk&s>4mSmTeR$?PSv`i#YW0!?y8-d<2_&GmV)@gUpwQ<Gdv|P6<LMfahiGAr%QIm
zla!RlE6$2sTsx;ZeM#$@*)#l~cJLkTHuy8^MyIOVrnD`YADZu;Y<ntl<LjpKHAV+o
zgqJj&Y%)BT;N+U6_#|4ycB%p^*Zt2m(~Nd5d$LorVcnj?y8jED+*@aUZku}K`kc@%
z(N9_(7tBj{y{dFgowKHQ(dNbBz8+CpCnOqod{MX|Rm3eVE;>!EWoc;pOQ$u9GFPr&
zt^MMYN`Z<4S4Hs>)q;~6oE<l~-ic)zWM?)9+866v&1U_-H~*$lj@QO}^>;6*xu*st
z{YWa)yxF}%f6Ao={PQY~+WTM3+N;EK+;h`$pVb%px794!9g*pIdzV?6?euar$MYt$
z{F%HyXs@_x5%J(l+7-Lv+Qd#ax$n7+o3?I$lcGL*>*}ph*;}7EU3aqH_DiKGxB1D}
zkG!8=Cz~5x@Ak3F`MNB}D{I-R^L^VMTZK%1Xw|29HIAcjC;wCjUo9=|Uzdv(|1Qnc
zT*wn3I!SujtGq9_`tCayZJH2O*xl3TeShf=)|gN0<6{nrvOFqrd(6x7VC5Sp<vR-)
zS3j9ipvkn>WwqETpM><)X?Y7v^!C3wy<Z}!jD<bfXhL3tW|`dv#dl8(|64FT`gY`i
zACnqayNFlgBM-;9H%=bBbF%t^`pzesmTEanPUpU8@XY6X8swF0xv%-@ipNi5*40J0
zhO(?+e(23*S)gQMz?I0eh(n`+??rCRBkqZ=$8Ic*lRDHSbZB``(wQZy(P^uBJr4fv
z(K)G?!Y9q-HtkKM#G{s7xzo%P-(2hCKD1WtmczSuPjzOoDo)9aoAJa`EP&x!p69P~
zx^Gw+Z}ye9t$cro<%7z3zbzfYp3A1M%f8@u_=3m8Bqraa48IFC94p!K&YP*74-*ty
ze`1bYUYpQj)q=uDYJ8Ux1U{R*zs8iA+G1$l(mc;@?fRAgMxXPSoz7;|EvtRN(YJ=#
z;{Orm+C9n^O5Qe$(l({6mA>0-ThseFFIT4Y>;#F<8*OWET>0oQ<^0w|N;mUXx6e50
z{-W*W5q8%(uEs8$JC-!bM?Z1pae6JFsu=X@M1%6*1FnpZ4ypHe>Djcaok(DKX~<xk
zeD_Y1%BS@P6IfP1I4`>5lv-7i=eq?AXBnn{dOhvHky1V<Z;5w)0&c#2&lj2;esC}Q
z;{>0FNy}ALC-_%PQsrmWG)o9FI{(OP;iH@h2MydGv#IWsJIemr^NETA=b|Dl_jiG9
z&FKMr!CNLBc5}Tt<AXN8R(;jeQx+GPMc3)9G-xQ&<EmY%q_ax-!6tRjrtoR5S_clb
zwjDa2w`j=*SB3vv&$m4)c+InE={mk;Zw!s+DYZl_6nn)OTXf3p(7Crgr*v0%dOm8}
z{pOug`^u~rtb6&A4Le@y$Gk9Pb$EXz>p|S7^EpMPk(Zg)7pSstHfT^%ODm32X=2XE
z&tzEr#mk>jWP!P`gJiPHT)p?7%<|W@Y&6r$6;5+bGgG|sqsx5aM)ReUByDrK)e<w7
zC~ru3FS+#br6%W1Y=<(B34h;an6*spS`x3V#cS!t_t|fla{{s*uC<w6QtSETymiWX
zhXwq3pEldapH};k_RGjwb@x%Xd#wE1^N(Ax6n1I9QrM#4()mh_{kBAsu+I_KB|47F
zvty<-@Dv?-GbvMV!AV|*i^cz+oa6B>{=ee+G%jbot2&)o8C6Xur!Q-I=&?cP=eyZC
z>!xp5!gcbJyYeD8S*PhG?{r$1`;>(oeevwxo|aM%W}hjCGA$CA%p92192hs5c&jQ{
z)EhJI3Sn##_Bpx5+uJ{Yqg6P~U~a*M>Y@w&4>&DmZJOuBueJ1(-@+me?KSMbUvli+
z#JN8x<i;kG313W?3M^_8;ofPWSDLJxcIH?>lNt|`*NF@2RXV(#O2L!YC@fSm<P?r^
zd2FzH5np0v@O+(Co0VcEPds80l?s)PE96HuI2oQfpnsxs&5@Vx?ay27J|^fa^V3M{
z3NVa_6kO@{G2-W%PPd77eytI?-gCZ$FY>?Op(ZVtsH7&IgvFZ*imfL+{?f9+yf&Cq
z#6j!m)|lg=lIKl1k8L&g^^aXWS!(hki-Rw+X9QaoCR?6omG1rcW!dNG(^Ji-PmF(`
zX|ZY2l{L<3SDvfCU6Hlm8Ry=>>|2}rwUm3x-d+<}DRVl|;as4seCLgC#j^3;+st0-
zWlegu?EtT2l9TJ<-&^uSRzKP?ZMEVSy|`kgOJ)zXuPk+*?aFj+k={kmf~rp?9!=Au
zCS`;d>E&y%>IZ~#HhXlI%$|2Grs(9eiZ7<Cy}a!f&p2UZQ&h~Xmf*ehi~0T#rhOqk
zho|_Q-R5)kn$PJFt?S!zk8gFmv6Ahy!sC*H$48poKA9>0e^zqnO^Mcfk%GlRT$36K
zmM+lzDe`pFBq?UD0BaqckH-_=aQnH0YRSCfS{tDIJze+ir{^};y$rbSdVCJOe#Wd?
zJ^GS>b5ejNPdi`eDY><JJ61g6i(Mruby=IEh-q6wm~FDrgEa{aTBhP$eqLsx@7I`4
z{9aM`>Djd7>D_%7uNY@uS`j_@qxob7#-`xdgCQ|jr&g~FX1Fue>ge_AljX6er^lSV
zUZa}uHH_`^Jub`FSIxuR>gpeV?mNuBG+6RxY0dRgEBF4uS&PjjTpR9Zq=|@GEKoC-
z&u&!Z-n{6U&8^Gd?<6Z+du;nZW6Ql`6O@)sFcEX`$o>&=+TL!)#6Q9<{}osAu$efm
zXp)Qlt`ktE7&Wa><%)ush|^yFKW^sHY1<T?p7`szw+miUuv#v<L_t%BWk%06x6+fQ
z%a`xdn!pmMqibB=Io;%W))lVF-;Ov0sLd|ZH7uW@a4db{mY2&Gc+E{?+F+8ib4$+s
zX}NbT7+DzlB$YUMn)<j8<t`WU`B>3^c1r)(9sR7^3(g<+&oVIMlC0UYT<UOh-m@$7
zS|${*n-_c(@voiWZ?4BFc)gJM>(fOaD!AVT%rxnW<q53zYY=#GEI?BAk!oDuxy7d#
zcsUP62TeZ_^@2A(>(!B0e7pVmvKd=TwuuI5Xe1w6zTxWfNlh9hg)a{&1#7N9ov`!t
z|8=5fKGP!uKUTDEjaY3N8Gdo{_L*AKu2z_S7wYbuQ6LyC8CJFP>*Yxw{f?V5uA3gC
z{J`(7WX<!NeU-rs*S0H&FwCFoFrRzTeA%YCCru}CRxME7xnOY+Ls!Mlii>G?n=LCJ
z$EYn-=wYtET3P=$)avi6g1@KCbIN#(&RPcPTU~e_u&deTbVaa`gt>fKztQSzWw~Zg
znlA}yPO=lb-r;*=#`|SgH_AR|yQ!loAH8bT5mf`hEs7!66en4ktkX@KEA`syWpqxn
zc1y&mX>AJk_?G-wFT!Tv@yAEAfZe%orLyd%$kq_8Mc-JGRoBkk)@5DLb9Baxwlc5a
zUp<#4a!y|J`G0&{?|%uOr42k?Gy2|2)QGnh9NgZ=U%f#%X#=~o&&QknpCuOZS8Zav
zUcjByz@2(&M|<|lLIrWxh4$z2Z-*A#4t+B3QvUy4kL2F6na{RzYArOE3{WheI5mm+
zX@Oe4-uC%3n01wpTAAzJ-=zDt*I!E|>QU>A2TIY6Z?b0fDyC$1CYfucoKnmZUtXZK
ziz|tFn}XhgPQ%<r(Q}JL12|^9J;=9%)$Bp5Xw*UNeT(Ggy(xD1a`M`4?XGZ>Dvpl@
zw*wY;MHOD`$h|x#)4!^n&2oi!g5j+LtBsi!C2+eX^0*~(FH&G)aX6?t=a6U=*Oms6
zEec#)5=4AF7e+S9ny6U({}#z`<m-H4rTMH2jtC$6@=l`Qv87hB&7q}}j;S^r)3o6%
zsySY0bI3AAn8iU@=FZ~iV{vnX^S*Dr%n(-p|Et7LQL86kKiyc_)AuJp%eAVpIa6du
z<IX$Z<Zd)7Mkr<J+BN&<%v+z=)_?ugWIJi4mWl7}W4INKELucWoQ^R^*N5c?@6z5n
z@$rYi9V%PfqRX@-+^0y1T-cndurYU8MER=XX}gwXTX-E5+t(zTqu4s7Nn0*=+MHIN
zoQIdS=U$3Fc<HU9^PcFv<|?ZWDb2pP>FVs?3O~+Wy=c_M9dNwqcxcgPuk|81AFFa|
zYqi#H?|*Eu@L|M;?GBft8)YLDxti`w`G3rTE5ea`$$`VB3&mRIh(;s|XC(?pER@}F
zS1iYQ<5H(P?|1a6?%t$s&87D9*wTi>tPb2;5_no_L~bNFi(S^<qEIW<lf8A~L{X_H
z7i=edzEbeN%6Z#?M;td@b@H~x9|@TDDd>UwWYbCoG0S3I@$H$hf&Oofo-pEeOqwEX
z8Wh-cfHB*8i?n%Xa>kCWPfCM#FPt-#_jBpd^#<(1)02xs-Ui$%+qtVyDU>zA`q9yg
zyZ%m=Ia}r~tK*^Ad}z0J*FCv_7M?ZMmkUqrRh(3@Y39|?!t)cpR?JMDQ{I%~ua;F*
zeBk%g*qL)q^4^K}wK@9MQmZS1Q^|oVYoY8S1+HTY<o-MC65Fs>&gssrNJW=h2M=Z+
zJZ^kP{J?C{V-9yt%GPMd9G@9`T;1l7a{Y1H`G;!neA?<DY;^yi>>S<=b7iv{<s>E>
z)-IH<IVkUSUp_#7`6T5u`8OF!OX^q~4xF}I%>2zt_r11_)hX#&4~vs)*2ydlk6Jqa
z*;O6wq?61^%^$yOO#6If<Aeun$@UVmj?-jU&rjYczrs5)_wclw#tXlF`Msw$e4Cie
zTw}1x+<Ap^NouI4^ZBPvbrF{z?Os)^m6`Hr@{`#CI~HB4u5)8->RDU0)nj93syD;T
zz^!j?DqZz*(C%QWa}^12YGBZlvab5W5tQsUL78>R$B1Vsr;K95vYz}eDoXCum05P>
zjzMB`6Q{MP$BUoAPE8A(Yx!)oOja^*da|epTnI@HS>d<RF85bR*6|fV9-JLaVGNpK
zEUZ_AHXO~_YZ#cdQR--3h)5Wd+tw`8uB&RCOiL=}NL|wRI9R*b!%)VxTSPbNSjMW~
z(?nNCy~(=zx!iK|{<G76C*J#W^K)wTz9k-kMn~6A(QR4Y@Z`qyuOh0#Q|6>;IdTSH
z-nl(Tli9FhwQq{YM5(O?mW9nnW>3E9<n~xZZ{70n<$ZUnzP-J*yZB~P?c<VGY3I6@
z+I`AZU!Ga+3XlJ{_s@?n@zdTr-LIAWD--?dc}kEZm&(EhAug4G2Mx?>4GN8(Pt5)&
z<Oz8xHpVa6vQSX-;>_cT?Xnv`3U(;uzDSIHHgQFu_FIk%=Uinsip5HoUrqM5zIZdq
zUz+Wzav<jq%_m*DYh9kV*(Q0GbnB)nJei=qeaEJFPuag33{#~dBjel+XD$(x)T;Dk
zlZ@#&$ta$dXTUm3*kj|e7PePu3;gqUaVpMX(pVVSC?9x3q_w`)By@q?qnm<~Uo$4Q
zh<&|~%59z8b7<P4tr<d$+G{h6+Lpe_nJ7`jlp%ODTrflA=nAQsS`4d}sV(aio8dG|
z+<HxjTY&A}8&xyy_f;&@UcdX-<@4+JGX2iB-&-N3vnko;$AUbOO_~R0F)g`ykxj}r
z`eDnhSKt0Cs9T90Z*z3k3y;|;rBt4`^X$(yu~~0-ZDXBvf2a55l9($gYWu!7m9L-q
zJ@1OX_?#co_YZ{foe@05#K4iDo~Cz))qQTn%>r@JcpV3I?)03(W084l0uL)xubFsE
zv79C0r0mQ)ftqr1D;gLK57%62^yF2Y7_QjpA>L-&-m=h5iOXSnrG563%}ytBKP6q3
z7Znfe;ZnSEF;eR4quZ^?6N657X{xm>)O9*EEpP(=#EYkN*S>k)rp+~1WTN)cH{0W<
zMc3tA>Ni~aCFt&kiJ8HELLXc#0_-HGiaher)C&o1irYH%vAknk(o>nNNBt>Nb<?Ir
z&T^U9k{qNE7?ISbD)^sOXu<k@>PcO>7jLK?_k0$1{o9qt-Dx|SPFJVz>Av<mecx-j
zJ!uCfE4c~<@qB#{uwvuNStYVd!*1u^d34OC`RLR|rrHO4H+t?ku3I7_y|~AJj?<B)
zp?9}jE@Izb{^|I8xr*<R?*yxVJm<e({q5}bd7l_E?3kCFXE<Hp!0=E#HR__5(7Z{a
z2OKVF@~?RyrlGWDf=haH>m`K?^W5b4$~=`98whfK4;L4`dLm0)v03Dnf*IqMg<?*Z
zCNc;mnORRc<Q}T!ta)dH*^D#2Jd&QCoI3)T3}iR5Xa#akKlMO$az&=k-$xF6mTg#O
z7RcduGbxN|#?_-t7f(9WM{y;X91`dIFStrM+r<BijnH1ZuScC1u2K%!;$Y$-%ce6a
zz=!3PM59X2tNs?2ZpDup4dJ#~VOtior|JajT<{2*bmG@sRqf6-bqPWdv!0}=O)S;%
zEt?d+H9~H)h>L<=5_fb=OK=m<lLf4f?#^3MnwOdxvPI3xp1$jfz-6<8df$Arxjz)m
z_w9?EoO$&~*DtNf`{!}`W-y(xjoC0`rp`6Pwp^dtJg1+TxvOm|aPmo!(tdu*Chc6w
zqR(?Q-kdnYbYUWcjVt2^o=%o+1{2H5JksSgr7l{0nf9b)MT^`*kHTFpT+tC{v&^3y
zVpC|E(6KGiCuPIK)@3s$PCUS(rgl?9<fWPGu8r#dtxnHz@KrO|G4Df*hIzK;=XX0>
zpFL!oU$-#EL!mu*Rfei|IhUuj@^wYW#wW{bA9lI>P2aUdOjKt<)+!O1%_sKww1pkw
zmiw?yN~)fNAwu@0d-!H6o7GLBYZ8yHdY9C-cZ<@Cr3p2eXJ@1vGx4%A#9o|OsAe?r
z`3mls6Oyezr!2m{h2yf-``zdM@Oj&R+|ka(cxCy&H_VZIj0W}}I^U$Izn(9rRgnMg
z4|j9LsbGgnmRtp?TmJ%{EG%eT-TdP-m!o4+l%(4dSG}MPkwZ@ot2lk^P<xmwS#tY`
z-dvSVF2zQtZ>g@PiGo!nlO{_fo}A*~rN9--v!v8JR7>+pl1TmklBx0AHcYd7x@>vs
zm6dkV8TUTjnivxQvO|_Ja#hID@Q`=5PnO4R?Fu!xG;yoawDl8I*CfeuM=<~M&^Jk1
z(<c+b5a;@#p_la(hj7Fpfq5HRwR;ldr(AIK=?q{EoVg%knb6eum<i4MPI#}Fn8f|W
zDZK0Q*3S1F3^P6~3O8DBv6eH$Judvr^u4oQ>ht*+&*Ipec1Uj71{Gz=ld`jC966`C
zv7=Du6wi`L7N%{6rJH(EWw_p+@j9}g&t|Fi*1sMXtUej<I%3xBHu+)uFHbpsLto{W
z-Lv9FCtc{D^Hfpj!;1ObqQZU8_-8gbXFd$e5}B=-deJlU9_vYgRW7PiGCQMu{>zH*
zT%nP2O1JN!v4>Ole&a>qS5EE@bN8}!4_<9`wY5;|%`t|#dmOxkr)=~IJzMqqe2%D!
z%_oljDqV&I=G`v)_3lL!_GUlsy(+hM!b+cspWe9b4q1CC?Sj~jbi2nbt3{(U&iy!d
zJv_I0HqYzZ0l&}r+8<f<+xlT8f1TX#2$5s6>;E5?PGzo${Gm~h%(?o~iET!XwO)!b
z4?mr_;BmxJ@L1);>~{)T?koJm7KF9xdR>f+*>T!nre<9DmBQIkoQnEQ6L%@~3Cxp8
ziF0x-7q&?9R?2<I@6q2VyLN);GO<v}GLJ6B?2}(su5^-EyJn+ilT~H)>`(Wc_>A^7
zi=?cY_uo!yzrnH4{;Dau>0do!B8x<mZroLK{keOcQN&~pvEAX)XCA~(|9c>Vo%5DB
zPg&rp*vb5Isdu0EefQ{MGndiZdEfER_etytOnY_u`Xc^3I`ZFW#<7y?J7xEuNe+5m
z`#nj=WUl1dGrzMF<`@(m_d9krzs>H8@asJ<ZQaflCsoc}ef?8qs@S9%Gu<@|AD?DD
z{YJ2U-@UWb7I%l6eTkZ|XPT12FWc*@!c2pf1ae6y1baoEda>0{T61v*bJ&rZ;ERi%
z8d>!_Y3nhsEEDnlGnw=B5(8)MMHgp@I_j@qaaUqSz{Xz&s|;n6<8-rc`LdNCVU|lj
zaf&NyL&BbI)vOUpaiRaij+=#@n%0oGl=X^O?d9be!Y49Br^v81>K)$5TIVWL$C_6!
zn)kX{?6PxHTWG|~wx(MPBcu;TOij)2d#d>PSyQiT<kxR`|H@QLCr9c!it@8m%YUp}
zuuWvPfGHD$(#4Afyy*?zC!*w=1VqydL|r`i52;^M^$6-{jW_Z5q{=#Lfl~b!jg$uW
z2E}UoiV(pCtg*#n$qmK%f!wnN7$+_eOkvc@KEYQSC=*@K{^`5>7t_{7O|f3)@&4&q
zH4nAgJK}$thDN4mA2!lCxk-|#O(*lY;IT!;Up8n@c@dhEp1^gi<#}r&zgm}wnAbv&
zE`~7v3CGIi+>(^8CDw%}$(Q|4dNN5*Xgll9lE%(*y_gLZO|JU;cO;vp$)5d^d~}9B
zLv!VoEtRKV<Znnc;69x4Vxp1Z#mYc4)xfk~4dc{XFH+vP7~Ov%81qaplF|4^c$*Ys
zmT__PAC`c;X=(2=T4X1hC_GXU6pVIHZt;F>viPc_;6qXO83A#R1^FL}KiDjgqhP&8
zAb8m#n~7!hH?|6JRZ47eRt`GA_2fEN3uDxc?}E`6lzf~8GFKFrOceiWz_n#s@zx8P
z`45Y_KGuCM4{3jxwb9sGU}MLh@X%QsC8s{IKH%IiRU$|JV)ot%oNF4&4qX#GJZ;L<
z#Z&ejo8q`q(Ceh~fx`_csZ&or<3I5@Ou*uQZLDN17XyRz44K-ew#f@vUwqEZ*lE&e
z+H|Qn?^sFGYqjZb%j}IJEXyjV8%M~0Pjg^;lFu6Au;F2|Q-_mKf#Sy#X$ngUxXhL0
zTB119<<_;@*Inh)RuPm^tn<Ih$9phy@dtr^Casx=xMnD6o0%6!uxn;JnzSD>owleZ
zu&{4sO5DxR;`B;g)hn7VhsEl*u=Xc<)U${+t<ZYg+7YTA8WFC&T}fy1m(uA+tn;Rq
zEfSfV`dl!Ib;?0Cw#5fL<qOMX6X#Chn3o%F`RHZneb2dfZic3}m(M#guduzmDBbg4
zfLG1;u7>pajpg$>H}vg1A(O<Zy=#Tc9*d?XSD(v|{#Q&0mp$3ybHatyX`24D<&}cJ
z3{GdL%~7v<P_mF`$-))id)IFFTbb_nt3~1FbCFby>K0AbpI7=69{TI9@_%zfaKCxF
zqF_x@V*lae0qKF(Cnjq!J>Y&bou$pmWZMC*nMvF;j3mo8YP?RDEPcqIuyF3I0=r3x
z+|d)<UYt;jY4kpISi*xr%X6y4DrT`&%~q>EYj$pw>Rrews2RHTa(v>~`V(SON#axI
z@33E)x!j3!YUQgkt{V;EI~z_1<;Hn7ZuvgdLy+OSOJ4V@aH*o`?3<0<ue`gX@-97|
z)+#z(v!hAdW4eh(WX-Rpg|qViE^IE{DY#TBErLr(-?GoEpjxWw{|xUR1u{mfWdj#0
zFfLa3*is%Cy=;lbSusYPcALp=l7b63EtJF<w`y>PF6mAZ`?jHRtAOBIf#ByiB6F^Y
zdy9J<{3-fzDeJ@r_jO8ai#}*%AK>Z@;r?^AgYlU5p3fcpVLGv;Iy`LU1>egQ4I6%X
zEf;jpo>wt%-vvR2*;BR~HEcP+w<m~y^#(pU!v>z{hObp!;$~e!+zImxxppwdZdjtg
zv4~}3#e4(hr1&qc#@htgD)l<9hcj5MJn%%?<eMIwRF8r7=1e2muaA;{3Z=L_ljBe}
zIOnm@jji|QiiI~-QspnExO=CrF*k}96Y^PYR9O_)sKn}`(J#Lw?f(>$;*S4m8&_@p
zoottzX({+o%)P?dO_`M=K#}vHNR9`Wrvq1ZU|nn?fAnXKUM0ab3o-=??UsEE*>Kq=
zmaQee(Y?=*>*sOPEt9NwUaPm1w8>JR_x8%%{@u&{GA$q7oZ_;W^^3ri1J@G%7OZ;{
zz`s{aa`6VP#S`jE4cTU$PFPg1+V5w!w`arj>bVWSmd{=ydFEPJOr=QDv#HI$9JYt%
zJ=r9cz0>4ATf_y$rs-XI#V7YLq}ZGPSUIiL{-1DjRb+nSPebk}(^*198?r68U*)ng
zGD)#^>fc!~;pU8|)%(_bjZ&Sm`mf8r_{)|;7fowFicAt><y2TBw1G3f!|l{!(^|&h
zpauUKg9H?74vEO!7Lm$sbC6gwZQ-1^R-Owkt`p7nWI9lGNptQ)&UvZV=Sn5cyC*3m
z|3-0A^jtY3%`FaP3Xa^Zv*VX!YcEn@o3&eGk%QP81um^Mf~yVoE4?`=dqy!s{m4`c
zFV{bwZ#n1B-=e*9vdtQa3Wbjw)y2G>8@)qwHW^IPi(FG-{7m{yknE8X;e!&#CW=>H
z+33Tlyin-W=IlS4mtU`BpS;k;y2^K(;kxHl{=WquN%()h68TahWc&85Z<kc>VNd^U
zY*%tk&{(R5TTn2!z|N~!aBYI=<VG<DgV<-2ZK4=_VkS!PC%M(A$A){%zbIA^xhgp4
z2Y=$yoCT*&t-bSK@#}=SiZ*47HgJ7usIxJj_4R?|s@ME)6xdcL%w26zyy?xU(pw2z
zy3g$Tbn;^Oq8Am5lEPM8`X07VFrxd`(JEEz$+I?PbIN8u6kMk$JJ~(rvCg@tJrQ%P
z<SsfkziFKQyLtLA<5g2in?JuX|MD?%^)CBA*@2xaTk^xs_;{G^SkCHyP<)4@Bd5;M
z^%3fd<?5T<-FSQL!?V_Gy`cT&Rf0IL5btY6sk?qMz4I>2ytwK1#nskqb6?N>^5L}b
z>(k3#YgpdlpS+1{o+87Rgu}egF3P>#pzOOo-F)LRkBze(*_L0?;VN2d^w~jq)+(-?
zBu<_^I&5}{D_AvVSLkii%XqlSqV0dO$?Rmq?yE;SHfP!_luk6TtxmbRBqddLOXl6)
zt0xQ<l~eC>T=%LrQrxxW)(-n?R+fuf9n;v-BfGc$H`y0zeM0rq)@O$4f1ZdYN%tRB
zRFOLv5S?)H#R(C=iDC8LGag5XO0Qg1c~<ek(a`n3ca>&Y&f(s@RQFbu*3N~tJ7t7+
z{aSi!{hnL<jvc;sv&_JDDo>l_WF>~x4&1+&hptxOTByzN+TrxG&ME7nZyjH|T;XqM
z@Xm(1SyQK;op|^3%`3WRFQuKln{)OFoAOh(l|5cu=PsU=XFOMMbI<Ez5xO1*7rL9@
z?roVjOYWo2{XcH^Z=7yv%j{=!luNN*y>(Z?%is53NB>{F_VjfPCsUrIO2)^6A1@Uw
zYf{fQQg|XdJAg6C&DL1MT_Nb<1-ZsH5!*5co<lNDXB;FC-S?a)vnEt+UBce9LxGXI
zk`=f(<b;mo9D04B^toitfwKWyHt@ZiHJATbXx}99*B>;tCLEe1z4LdLoUY$o?Kks^
zpU=_HJs`R7;coFwqDPMIPnj^=V5@&@h2Apn*O!hSd2-Z1`dC_>&nc6hbqjkE-aZrc
zTKHIGi*0sQ(VE`UcgIS27RkTrtvaLlrR6&B9+9stZkoULO~1MI{}sg@->Y7H7Brr<
zx>v_qr9?dZv&qD{H?PH7F5PqUEsI3d0WRftr>1LPa?IIv$CID;7~k^Q|6eUu;4(SW
zvULKho1D;_1BYLFanCw%_^-q1W5PNA9(e4T`)c~f^9v-V9M@U?^}*Tcx{KH+-4tSA
z_@&vl;hW5reNrc^r!{=N`#5*yyUTB%)ZBY#DEDyTxof`-Uo@&-mKOSOX8LEvy?+bs
zf4l7!G`Ty&_U?}s#gDU<eq27ko4v*)OU~?xppVho(9N8Yi3fPs1)uwusUo$5H{Mck
z!lGxE4L<))%~>J1-HCg;BmdSlkE9egL|x5hisom_HjPO?e87!Ac>~w>pdAWtLT4>F
zWbt;rM9gQE^{j6kbmSS|ES(*n!l=``B=py<d7rr+&Fy&ntL2FWU%8y(hUcCS`1@=7
znMDO%{$JovIjJ-I>633(2d*R?6>N(<lN95#Am(j_)wZkAzOpuKDaU^pd}MrHpm$B=
z*5+b)Bk#6r>AZ%O|F6ye-n+8gsN#(v&y(jr_lQU}`oF12liT>C{_Ud9dDYRcuTP8@
zw>oTcPlq8{;KL#Hwdzd__oD9}T_?EUT<DbDQ`YCo>7IEcdA@jZ(iB-o{=Ex!?G2Ee
zbg-By;dN!=;l3lgtoqBAp61&-;nTIqB?n@buUdO&({27)4r0syzdlt~cHZ*PE-@jc
zxqqs<el4{-^rCV~z)3ErtBMuAGnBn<@=RihmASf6xvi)1Y=i2(+qFOCHtm^ZW&a@Y
z{~I^?e{0{ZU-<4Po4_2F|2!%h4c?B;Oe}ISGd?IdbqK0zO>kNkDjk%<vcYjh;3J2A
z7R#m{iAgCQy=vTjzdAPbGE82Zb9J@qg3#5CTRl_-6dD&asc^Be$$CwA;jqw+mBn|H
z$;*pPzO%hTgc29qGyeUxRpdgqb{ONEBO8PR&cAku`ts+Z^yWP6wSlYMdXujA&zljr
z<;K~yq4_+gt5&;Qe6`)nLAO5P;JPStH{F9#i^6hMIi_Vho@PGYVI_Sre*gLToK1Zm
zKZ~wiTFm+D*RPAy_g`m9IA0fgd!Bx^aqiaDrJo<U2HCM{NXGs7@=}F!lKOO`g41G*
zv$uInH>$Oo*>m$`&#k__yTACxEU}-z>HpF{Q+S_WE_S$I_gCcG$A`raH?*Fv{qvLe
z+4J@PAN>Dq@zdYIZ<j)VViSwZ3WqkHkQLl|JS71NT!JwH4SohTd<%Ivy%bh5R0t_F
z@J^o4Br=C9Np+EEbe7;kaq}q|!YvWA1eZ?_%SoO*JEy6Gciyr|g&s<Rlai*KNx9Rd
zR+hDpUDubTLwo%im&B#>>ej4TE_d?Otks^r>%O#$?s|1GMUOG;;%WJvH$00J0$pA%
zmuq@4@%33HPNp}?R-JE*{p}dvZeAZ1p}FNkSVShPtE=SOwdT=Lg-b0R)PJp9{)FL%
z6F1{F3Fq$TCvI#}@;6sJ9P%!}wlp+&yTI1V9m^Cv!v06?T=6w5x_GWqxK@#A#`Spq
zOD`6-uqjQ8n98AKc0O4)=v!p6@T)^N|F6&uJJP@5Xc&Xw(hTLJ)AhW|4i{?YW(M}U
zWi4GVqIg@Ale0rO=tTb{qj05%Y+Xq_YqlA!)!zKdbG2AmSYYtB1IuEbp4|OuQM*sn
ztB2l?)=8zM${xEUl{z=>SEF{)VI{GKc~LfEsl1z%l-6vH^Y~|ee)F*_EQXy{#%nh>
z72KO(dQ)Vw@3tFh%uz3H7p+cTb}#Q&*~|O;N*7OE{<gq1%Q|P_tye2IKXcHiuAbC!
zZ|SsKW+zxqq`lku;=(e&rjL6b>n+~C`}B#B4{M}X*fM^Ym&6qH^W5$C|M!-+=7;~P
zTAFXq_bqr&)SqvUoepN56qvw$@_q1$Pbc-o4>+7LY!`55Hx}{u!s0Ez!kJr8sAIK&
zZA-_IMwdHGM|n$I0vlw^g${fZb-!fm)>I{)DH0U1NYF|ucS_LtGFFAu1#-$+s*Icm
z_I=+`x$~0Cmc|!%yw(Mt_|q*G7IrbkJ1p&3k=~@kYtq;5SR&)9`;Aqgb)D))k)>-k
zJUZ0nqq*^sVdkpNEQingrZ+xItxr;Ll!_48xZ#nne)9?D_17J&QVY!6gF+SyZCx?j
z!AnhfODjXp!!F@v0<DqO3$L(D*eZUnG2)tX(Dg1STLzT{R&$u9E}u8K<#|h2-kM*1
z|5t=P)!K42E3ESL>~(WAcBorjv<ps|vvry2+9@&bmCtsCi7OwP&|<Qr>$H*h?P+Y2
z`wLi9RTVwNHhTzltP$+fwmj#Zbct)$mc^9^8?Og6B#B73<!Z$&4&T!wysc3~F`hY6
z#ib#G-)3pQ=--Yr68F@n?dmx+Y0=D?4F491Mn$YNSfnvC!HZ4&u*8}a@z)Xhv$ibR
zm*Q%|wS_0**F)}8Gu*V94+>Ar5>(^zdg8m`VQOkqnDng!%QChZMCH%;c4nEQZg%+U
zTe54~Ojr39XR#I@<=%VqY-2;QLhcE1-cA?&c^c}B3>TPsEkuvJ@NnQtX7DfHQS85S
z<0&gglm5y}{~w?K_MzAIrDwkbd%E_wJMGMW6EmED8t***u-#$O<U7q8CGwoQ8BQEp
z7hZ5oQk+l}vSh{S$xKn^i>B#pQaanr@gYjeW(MDprOs7xB?3;1C-p3fefcP`i_2p|
zn2LM1v#5&WmIvMTqDy@LarlI>EbHvpm}C&$#cjJuL_B2I%9^~Sb_V;){F`iE)^Z%?
z3U+OF*j)C0%`>l6%y**>?ljoyu`Dy2?ewE=HOW=Y3NC_OZWWU~mIkP4$h^^ITYbnn
zMR#MFz=0%bZH8l3-}olDawM})ikO{~;L#|;kj|~txXF|?<m^jD$t4P{m-qs=y0|=&
zSirnm^<Hk+wyZ~Mv!q@BYrH#Uci>a#);%X&b<Iw1=U`k{u-5e0;~Z`!p@fiK7hde1
z@WU@gDe-ZM?V>$RS45P`bWZGgb8+_5L){kN7d@<)cuDBBs{&`#3U)n3C+-BV%*i<q
zZ#am!^~Z7a$X0bm96$9ap7oTF$v=h<db<~$RtYpPHdE#}9b_ypSCPeR))L`}rf<fK
zSJD(t+erD=G*4VQu_@EP`<BVZ%A~Bk%bV5}IGe}5IaDCja@3G-xwL}O<Hj316v`|(
z(la%tc5s-M$w&DyHB4x@zUr*i)s@e0|Jx*6m85=Z@eIFPO+U`-{xL47e|`G_!}NK6
zs#EN~S<Z2i6<1)8<Sr5jo$&0=oSjU%|6deJS$U{CeYdopuUESI_GkCMYo4FhFfZfy
z*vYbGgGR@Ok~zDJJza{93*IxVQkaldvcxjTX13Uqt*18!?zj+PbHK}=>y+ZE*IY^>
zj>;}OeLI~_-m^0L&#9vw70EvBRMPZKEpBF8o@|Ze*YIZfbbcKxXWI9;yJ5-dv+G4|
zzD0*$S)MpOwYlo4w)U@C8iH)oTC_tFx5qX{N(4>YIIT7(MRE1xLgkcA%!=i^5AJ+d
zES|C{^~0M&3+WS26n0wuzISECkG)Y1@*hKIPvclSBjfg+M_~_ly<NNYS%%)@us|ij
zda<I_N7M73X7U{PSR!-#$4Afb?~k84B|F~nIDWanyyAbw8dvKYmkF;-KWTb>Tr7Xk
zO-3`sUxVeY(?Y}0E9*TM%2wR+I=SJ}#`WK3O=IWd&D?g$yXD2FgDWIGHZJd*c}+xE
z@#}@A_~~I?W(~HpbkC(-J=5o#8`Qh$e#-9dy`GP^AK!arxu)wrsZ6~pM%Q+x3iUPD
z%;p)MS=;f$=jzHPjk$hX*%z?zW<2J$vM}I3r|`8f<LPhb1s#hji|4G!Xtn8IFk#ZW
z&-2w~U4F<^b<WwM!02(Roh?^IZllJsNz3+jrPR+6-}|C6$3yO&{l3L7&emOV33aIQ
zc|AwhhoOVzyqMFX2KU5CY&Z7!Xno(&^k%0x=kMFoJ#3%tT$g+M$n7@K|8F?+7v3n(
zFW&Sl?SHcV?r*2k=HEHKk$LHx+LhC8cr#i|c`i2V@O$}3>PlQY6pKV}SUG&|SbHn#
zKuh+4n47wK4F?hoJK_ra9w}*cb&GXd>+#ieglKp0i!@A)66s55?ztd4NqLosi)G6N
zy%rCVsU1s88+m*ts<#*{zkI^-1xMHO0H%cxm_BQ9DgIGr4A9q1V4e3<eEo~*Z%zx(
zd&#w~z<|Sey84lsRWtfTjIBkx4Yv!c?>KB&$mJ~A%^~B=A-kJLU#G+P%0`nlhD{cS
zc?<UTXK+38*s5U2mCMPcs$lx^!IX~=cK@s93z=!;B;z6_+5h~4PJ<EuzQiM+Gmhx~
z?{+a*%pWOf+^V$sNT!T$i%ffjabn}<bBxOUmo{JIoNKpRAjHNs^oWJSOj+0LEw0_J
zGd^v(b5eHZp}Du5Ir<KCdK=GUX0d#CL3V;d>!Y93gidiTkTO-|Jox01n9&TT`4xN@
zZZ*tHKGxCEq`lK_`zD3mhwLs_NE|+}V1+a9uBxp~8x+zWyXBggt(m|#$7BD=S!<#z
z)>OvqFY4~7^x0n0-0|)4@yiY7Wskecf)6+Fc$|LJv|D>piwM`l6AS-;w(oedy;gWf
z*g_BGraqNs(Pz<0&4)$V|9G^nc6gF?DCy;aE@i#RQ!E@NHt6o)6rSSxa?!fU8kVz6
zcFuB9w!9?!;(wCkx&X@sS?zZZbe7$0vMcIx?Be>ndZ)Ar$D)u1O$)o<9}X(~QC^|Z
z!Qn6|?p2S9=IT=cy{lf&>$u6aQmR=^(mm<(DaA%B-5?&b3ryw~ZabAdcTVw+&S-wd
zame|Qw?m}5b1|RXEfL)nd`}XmYTRzo{LS~;!R1qe&9RdH1C=(%Cim}I+26Ebj`3!j
z(}^-Bm-n2ho|7rK@}jing)hdJD&_2~Z7(}=OcoNn_L}p0v`U31lTn7q{c6qId*mKI
z<dVMGCw{TRj6;5=1<yknj%bN}JG&h&TsqP0;V0S2WOTsJzEE}Vf}V9ztVW4FbqbRY
z&af`IBk;n8H|$1JP{sc0|B@$Ur6>JV?)Z|^`PD@HaM$6TM?{`}uovv{_;0P-p<@%i
z(&OKr6Dqg1u=v<KpD=azLfwNqbtjxz>-SRko|fkWuFmr+leoVuI%RW`_fNl(#H8HB
zb$ySyrgog1V9h6R^yI|N>pKb#HvJHrny~XhG2h3atx5;QG-oi#%XG|1=2CL$d$C|9
z&u6b$3poE=(qF%#wRvT0X+mq$LZOveg1%B61wK4mZFn{RbR_QT;W*uMeFdwoM)S%>
zZ^IVr=nIBBOq?IdIGfw_8kt<~=WsR)3C>eGAue>;&y&mbmbd#Qv33#1eV2n@O=$AH
zz@R(9=&(%4Vd*P}Y;1aXWZYvyTysMX{AZN;HpAx7LK|<L&Fu*?9gC)Md~`Y0xcU5<
z308+S0=}%c>~l8A*Y!%`(Z>NR!zBERH@Y^x2)Mo2)lK?hqsF}8lUv?K$cf+ajN|c3
z(OIDT!y%i;<QdD_8IvcjywUXCaoeE@+$(3vYZ~%IM7M5T(K+kG{8A42okdzJKj|m0
zUY!=a?R|<_Q7^|k<?!DU!HhGGH-((PTo8U@QfQ^Mb5n!&XBUpjT=QyK^AhEb=ndgz
z!Z*r98;lN!$=sRXut+YCk>O8`M_o5l$D8wiO_aJy`FeFYm{OH)I_+pSzR=2jvdy|9
zQ*p8GFQqAebu%m@r>;>7l@<EvVlvmok)L&Ee*ov4|9d^}?&&d0XccAkvK8{0aN+R7
z2U3b>EdIP+v!qv?RYKp{OJ)9!?HwHqx(@<d1i1S+d4i-=k972`F}O4_GHCKlmHG^x
ztqJYhEWFoF-mosGQ)4fW&B~>c4;#$>sAca87d~akn6c4Vr<q50q0xf|o?ib$q7A#W
zIriG{TG!a-{j|50)fJqw{J<S1U!#!IOV!WVx>USUKYnG?v6eIEWaoHH74?ak^Uq@Q
z=`%jZ6E|CCcQ8kCoZG2kuuI^Q^w|q4b1!vs%zGi#@>XQ#Uf<3Wdv8l?ML+d?p=mjd
z<J^3ubG{n04_yuLEfyA=DS2rDcYVVAtrNNrb#%X(!@avhyHSu;=l{$09Wz!=)3_eQ
zy{zPEhw!H3;yKJl8O=r;bc|N;3*BgHt>QGF!N8Z%dQxe5VWxj)hCfpu2cwur>Fwy>
z!4XOi?K{_;ujM{-dDDq5*G1hex;-uqOuQFZZ1^X*?wHW(!1+dZVoAVk+sLVFFLXxV
z<SE&ay2f02BS%lcJlEoNLN$6aW$Of5IYeY9iB91Vt##y&j+?Q-`R`^o>m1uX#$2m6
z+`4)|W&H%LLma9qQ&*WT=-j#CobuAi;Yq9ZSy-`DIcet_bZBflx8l&YrB>Uf8q8B%
z7N5|{o53nEd;fu>@p?BH^eRpZ&0r8Z;QB3tIedYe^zP;`ft1b_xAl92cUV6$`)?bp
zpOfP1)*)2H%BR35Up%)(R66*xK*i7foC>=2lkbEqT>etwsiSVHi;K;R02lXKm+-VJ
z?n|e7iWqyXO+8w8!Tarz-z#L=DmVh}TnX4SCr->YSa)vQ2IHV!4hcD~c?Cxoc5M;q
z4V7OQ9u;d6V>-3*z{=>Wdec^D_C`#+*>ZaRrq%}x2cI9E*LZ=6Gip-mYu0HT^DQPS
zFg9?j#0u+dT$Pq<6+LU|vgmDVjoll*32#&Cc>ifylj`{ca~s4ru^K5f^5`<#*x5_Y
zVldjk{6eGo<gNIFw*K{RUa)+7QRCWS=5xNV+C16wMD@Y#X=O2;Hft|e8F8DP;S#gh
zax!7|f3XXne?$oQU6@dMV9Jz$DYEN0O%r)bb|?il=pN{pE^<uRuJ_T@jyY#NCkM#A
zxuv(PO>V*0UNb8$`7_o_zho^ra9YY$WwEQua*HVG-Uel#p4eTIRSR10cCN2`A+h$t
z^SIkvV~l#Y>&7Jgnz_kp+5Tm{sadU@3asKn>d_j_nhK3;6b;3k`HeK1t?xC7DK~Oz
zG<*8Z;jv9vx8ip7$DDO5yfq(4J@UBoI&%5`iA|agx*b#1owctV>3O>=@Qh2U&A!Yt
zxA|oDoqYSfqTl1_eb1{d5B*{pPWqgAyr(NUmTTsObIQ$rTVr4F`ChTn&ka1gq99Fj
zrsl)ai8oK5y?OiL<Np`*?myuRO9*e`;CrQct+=g2d7~X?wY;_CBvsDdgxg!|&B714
z#&>MsVVh;<@=-x>HAjKc_UMKex(REH4#dC6VC9?8@JoT&C}U%^0&|wk_9urkekiYE
z`u2kHS=V<P{YfsS(o4_R3Oni=2&xH-UXLo24=PmQ675~X*Lx>HYP;y?9|>%K5?ZAb
z?>$<}^<|N$$<4<%4lI((QapaRKS3|<$kFKsIHZ@|Yzk1GC3|uCGQH_<I;OpNSeV5s
zyN&bS49Ap|<|hWZM@_RkJhsY5>orL*SD)ZA%rjV9F!TI`Lp$A0&ry7=Dbrz|{&?%#
z*GWQB3m>Y^duXskQZb`xnb7HFT>tAY?{(#0=<|V7;(W74>*|e$(uW@#Ehso;mOoL=
z+5GPp>E<UceknU_+~i(MNv+De$|9{O&GqnyJ?{b$q43xLjOS@F_ivf`wzi6|we4wm
z-qYicH#I87M$HQ`G8XgJ{d84!LTuXIvm$A&1$i0gWp4343wdIkRC&|#k52wVS%or_
zg85f(?>w4e*PF2A<Ws}IH<vi(T;M2_^ZVlV^6RrM--^i@pXY7&4X=Oa_h(;E_4oIC
znLh9<DK=d<%lBT$Iz4=c3iJF0UwEc6=;<<d{5KP+I-J<}LG1AtzUBLQH#XPbb+0|@
z-n78nSYI>sn=(VkF(Lgz;ZCM2MmF+Mg)`F%1OMN=)3f&_pWTi*?@!F*dpNOn+9c7P
zQ*=+B-r=y&?4hzf@0)39Q$KjlUQ^WXTt59mQOAm`*qqj--+X6s+<Ins?wnM%Kuy>3
z74tT;6|OhCwM?{n%eCOnR+Uo^PW{z0Tlw4T{D<t_>?ORXtDU|ZEbUZtnVu|cZGG^4
z(AL@Z5?*gKD^B~Bw+aa~v_@&n-q0MT!RTfF;K2EUMNTRH>qVkFrZ}y>tms)R5!=O^
z@Y41!%XPQ4r+m-Er2pOZaQTtwBge1g-n#$xX1~o(iMwiAd1toR_@!t(UAMybug}ZA
zyo+CKZa8*a`?zM>lB#$6d_Vtv_mEZ2jN@3$*|a!c6O*j(ds;b){>zy@n^V3{KuEVB
zzkVOj>fmkfey8*P`t+Yok^j=_oY%_^M=5@KY!+^@U!rK^kuLWJfduyhvW+af_Y{p6
zp3!Y)k`pR<x~xZ<F)!wBVJY`w?G8ECBAFinPmWJg^{jH?oTzeos-C=<%+DZ)!)@DG
zZcIulTF$^~Q}4v0;Glj{*LA6at5@1(30<abJvWuMz6@CzeZr$wE44dpU7TXx%ufby
zZfM3FStOcJ_BL{B`u$C&yPvLQ*u}ro;YjuMC2@?g4NU(!$_?)CX=APrQE2s99maIS
zMul~vh$LgKO7yWm0-p|_nj@br7O?Y4#=Hs98-*NxKGf(~%eeiQPD04)vTadU|Npw-
z=EbBcKUv`)uiy{oM@&7{E_@fu&$kC|x$>Dk;+J#%zE!^`t&2TS_U6*tpPSNG-}`ZU
z`|T}G_m@ri_2u2$8=D?;+5P?T`R(QX>xCZuX_qbkr|!_e7`5$!uRy5Ef|~LN*XP%K
z;;aaA|IB-0e(S$j3B|U5svZhS?6-s#F|55)60Q5r^cb6F&W25mMb?bd4>%i5b9wB}
z`Cx_%6NATwqij~0iBlM6=S*Y?63W?hMQk?DOwoxxb~~4Zy4aZ>WeN2&aE*w}Te);<
zeA>=~(_>dnT^c1b<0$K_m=%%JXBBTesit1I>*UcnMVma2%_+Tm^XS~NO_~ShmvgNW
zSC4&gv~xlGf8Adb7WHvm@LYVbQiWBsFR&{@S}pfgVAfoxFPbhhCT-HxUO&~XD`j@V
zwOblfE!kFQFkP7>y>@%w)CLygv|HLsPbh|Z+uK}qUc+$IuI0_feG}THwYP3)iqf7Q
zC#9RGl9WA9dt1(J-R-xWJQg-cnu=U@e5zC;;3F_IThD)O!3J07*Rxs#{n=N&7V!W4
z=*cCAGL;*b{Qp#iDEY0u(U^Paa@?_ktrzB1C};{ve1GI`mpY3j+_{iv2}_{D)}yNf
zDpNxzI8MsB8XUXv&(!d!bv%Jl#;c!(#S}cOF&1rk_%%HK;NLfanLXb^OJdzW_k6jy
z&#Q4>)^ta)ugP+2C1O&Y|2wyoW^Rg{cI%4sezAzGg$vWYt@}*b1oQW4E@UX?Pn_d&
z$A&Szu(*VIYKz!gkMh{EJE_mNmp&5ATU7m4@!dUF{$tlGYJ2Q&%x@KT^RRrnFYU#|
zcGbd-2cOD^?ofQTxVd9;hvKw~x1B3C-mubMfA}1q?w2b;{%`kwdF*Zf@5#REcmM7@
zxxDUh?>oU>qj$3y&O}=*-6SGpvp-@=pqJ_2DO&ZVsne6%IG25oY_eya_1ItW*~}!#
zz2S3SZ@&L;ekmW<i-pCP@|VxAyR~!0qTjFY+yDRX@5RWtVPbFH=2mY}e}=f<CzuSE
zI!x9vXcDM7v)bkGl1Y1>8E@QlW=&qOQ1piXN!&(S3m80R9EeL?qi&MLS~|^DJKb}2
z)MY2$EmFzza&o&}?p^ZOZf2y%C7R->!1>6L&B<k^LYlpNpi+X?XZ<qOfKIO?;`_6l
zgrn~?xv=j(dnn@Kh9KKFk$swBwlkjcSN~A9`sbni{??A&FW&T7=gTzryeV^fyf!D<
zMX({*gl*Gm6MqlKTU{X>nIfxmR42yW^pZQsB(^bRrH)bd>8PA(o421is#AI~McicP
zbpLa!jLUU+G%CZ`4oxv=HJLQqb3)0?(>`s+Ga8>V%Wqn_+OZ`&CisCKbMxu`ma`@^
zw0xy=npb<NKB$zdlwLZ+)22hi!QJ*^V4skIgG{I~<Nu332h^lKdwtvBUMO;k-|}O!
zABU`m$|Pag1T*$sXHrkhU0AK&-eShsvh%azk4rN3R$m-D9&o!TcFlOZ?&zZHvQtFa
zQ{CTYmFwJjnb#}0weNkwaqDhA?Zta5JnYT18v3eIL=J18yj|mbNaCZ`Qq_=iVOvy0
zmkVTaUud3}HRs44wb1ufmS?-Zt!YtZEb!Nje-!t#&1#wc!jnFaoaJY|5?b$Yb!Bb)
zL1FPPBFk+>PX#tAiG+V%zQW~aW?;p{DT`EE#ZS%I>ZU50nZCl*D`09$<F-E?$q%?D
z`i1%$?GNG24*4t@xpCoaS(n?+IbTI$SBkhz`P9j<_T~)+gRoXjng1`>X>8Lzb|=8$
z{;7v)J5}Y+`@QIVYt)h9=`CFU@^sAa77yon787RM2zDB4o|F}OGV#_WZbu=F=Dn*j
zgeI|+#Q*iWo4dzzZJVoi&(;j~wPz$`O?OS~-IuNGUitmvYrfUt|5mt)+?aLa#IgeQ
zhpi_*R2(_cucM~v^{^{g=;U!{txqW(F-uNM{hYe-$;7mbolnl@O*?t;jBDo9m)zIh
zNtk@q<elmz{oa2<yMyS*(kTnii0E~$yfByLbFARK9<hth4*1R}m~~w9VPVP=hMYwf
zM{md!7O`B|ChmSY_u`ZWA)V;QyIu5-%WaBzswO_8l1amYF>0-+#Fxia7q5v5KVWM4
zf8(Zp?IOKXib1he3D0-k*x0mioATm!yY}oht+U*9xi{I~h-;$JY?a;Dbvxbdi{>V0
zI$D;=+|^o}^n8vQtEbl~%~Lbfm(ES{X9%Cw<liZ^&$<45wTtcj;76@nzHL13TPk!W
zfys1+ZKP;W<Yx{B>$m&;-=A+0l32j7$66$!ZCQnoPA7BurdNWkUXANd83dT^aOAI3
zF;`0Y=<cEGaLj3&C68X|%F@&~r<A@aF1zQ+wBvxYqSQuv7n?=<Shh_L3-0RNx5-&L
zr+U)Nh}o)-e)s$OdB^1JTqK&upj@=Hb#3ZY!+Y6N^rR2!q`g{k^GrwJHNPoKbxfzb
zEUTEZ!M$z!f6?VOEDxhRPf5&pa<85H<lg2b>y@I#Hg``}ofNky<($sRY5((z3}$UP
zeM!xyV4lj_DXJ@P_-%XOdNSnf`iYXfX$pS?PfS^6H2acjn%Rxh{Q+sF4&`Te=e-n6
zmJ(A^*mv8xM%h>Vd&{w&+}TV^;$Cfwtvz-$@|<zy+Bw^D8SOmuI42lCKVQZ5;*FHo
z?#Z|JeN)=~Xv)W+-!}i7CtY~RYO9{*TqHc}!gb4%)2A1$a5?;DcSl*sefh5%9@pRe
z=-;yQtX!v!i-x~um`mawiPeh>CM6%*_0g-r_K@bw#mm#xeD_SKyT>&#qTbfcZ^2UA
zy^|L9&Qf(M*Z-bjIDgiqFClyXJDAq0noGs4nWf<@*~N3be99-*uhaicyjp+uxy$F^
zy~3yMmdw6pzcbI`=z)mpBOzikXIa~~JdCttGg`UwOs7p!gi*_kJ;&9`QoF_L=cyzx
zM@`Y)n#z4gsaZOE;k2y}Pye?);=<FCzqG}*z>%}#yyJ{+p@(7#1qWFsFc!`A_<eMC
z@zvXVduNu;lsr&)pJSI)ehiPvLWMgV(p**IeHE>TO#2yJ8hO|_C1s8VxX9GJT%hzZ
zq4wtj<=mdSt7q!IURQ05xg>Ib%^Oipqb~O)GD~%ioqjthW@D7z&q<<PLYWM%je%Vk
zY*{)*LT<UReNtgeWn?er@juFQ+heJ~v4pw*xeh578r%@s;%KLO!IwwUVJG9ZkC*+Y
zcsTBGd_JL3`oThmDUOl}T$}|5T{$>7C6W}kUA|G;CbHpz%hri|Yaf(^io2&y=Wuf2
ze%qq{sYRnLX;z|mKmhv(jXB1O$L8GiFuEJ*mwU)3_o&931%Vlk{HLyapPjzo=cB0&
z{R>p37Aek}G2?uYsnnc;MkBANrLLlLL!=gH@cO^;U*^lTyfMnmj?4dz7niJnyqAQi
z-I3LmU7iWj(jgaua%QBY2>jZq{QIrE%N&kBA8%>gz5hbTC7acKN9%mS$|H)?8Z!zU
z9R7)ks6KF(QIeEdBs+zX-GhOhr&l^+fvXP#XTcHO2?zhP6)^74xV`;km{=}@_7*{z
zhqq_WRM|gM#eL?H{Xe<9zIAzTdm{1S$Oh4V|2fxvdtFl%TE#LL!=60gb-ELHM@d<B
zlJdjpDz2BDLoS^um{Wi8XmDRlL!*$m(L!OlwxbH{pK=^;9k`RRp(#D&Y-)vD%8C;a
zukNg=VLMlQcG=3MdcJ4V5&{yI9B(l^R-}7f+=@+8g+aS?Zbc<ab7pWeW2^C2-^&lW
zWp~YOKX6*<*=fzKjk*>IoH8d|7P!0aV3eG|AbF%)nae>l=YW{TBesARCxe@blMb+K
zag^*>cH^d#bkFSFUleU;9`TK7Eis9E%;_+D<;*8DTb@2m^NhIeY;f;?URYcqS9<<0
zu6#N53$6~AQ%~9Q9oLdQKL78{2^Ow;Sq`rq{q>ys19O&(WE?p7^wF!Yj~4tCdB>@2
zpe8l5rZZ~Bc`=jF_l{GAneH!MIj5=j*y0?~Pnn{hGeZ|$kZ}o+Z}`N&zDK}C!M|`q
zV|3`!w#0M4LNDaBw0*rQy2jD%cVbuKR!2#-YaF{4#j7>2iap3l?UL>|zf9qbxLlG`
zMvCK<gDgk5*$rl9ElQJ|qgEB7;Be@LbBGFmElZ8x0oOC#S$PNcSKRjZd5~NCa1u*;
zv(?MVTn)BYn*4a?aJh-esIgs+ed)~6>bD_W&Xje6hyU>>Nykgh`1390i;Af|>GOX!
zSE8WR(wKsS=Tst2I<8EZ5LDoCps=;=io?olIkUuqR1+#5c9=|&=u4E`_wd@<glT57
z$Cg}J#CQJohov$`vRry>6KjuTJx#2d!lb;bX-12hc#fB3himqs7rr(L3V8>X?|atG
z_ee}<rc&B7xpT4;eHnWCCc7S3{^Z1D6|o*&4_2v4B~3PVjbB|FQrVt5uOBa6@>nr@
z;suq>Ur%wo?9%b#44!i9vr4Df1tr~8*C%>7>h(Q8;(1WSgVEq$Qq{8O24YhU#YBJd
zD9*e0{1VIjrLt!m9%|RItvYEqe>r0g!ylJVHZ4mPUit5tWl<-8s3~5T@!0A$r*b~H
zus&5?Gm+_k?jGq_$=h?2`g(Gf?9+MRvg6>1Fj2{lmb5N^SEZhQog`<!gDy4CbhjL2
z<7mn9Yy33vfXEa_>4FB=8%-{g9PfBCa0VPXcdmV^!7{IuXKr^KyZl_-zQi;9lF#0{
zOyour$Dx$XX}T^8RJ<<vZCoMg)AiP?DEFq3_D!~!;z<h4s*K_${XTZChc)j^=wUoz
z5Hzzd`(B@Vz%tFidD+T}sb@7(gKza5-+EN-?$HyjsX}rR$LCF83-OM6bu4Si)2t7%
zo?BF-k{&)YyBOUz<wnO+cayfa2I6Zp1&+2{zs>TjS3x65L_qt~t%^ih&Li!f6Bshe
z`hFA~eA~55G3$xzlQsX9pXEp%aqpX@!S2x#v7;$lD#`WALfMc6?UW<wVJ<aJDm6J=
z>8nm#2y$2+UAF#C8{39}WeVB_VRD;1-1RpwYp-F_Uc;<?;jl#)lP<@B*$<cHKU!b-
zPi^+g{C{O_cMRHZm&l$LZ7(s(O5$|&300q1k^M#I==NukEZy4WOegy%9KC-m&Oj)6
zw_a3rmBIT+{qQ}Vd%5Ck@3q$5()^$sA12DxU?ktLYeR%<SHOi03|BS;Oc7x6Xb|)1
zemqN=ed^-a+K0@&ZY@TgX)SKC7ui^E2DBZD(zcQ1)M}hCC*Vfoo7jve&L&qsc{yB|
z*1%%(Om;=u4bCLD35*hlPqUnG|Ns1iyMxxF;Fy*yy*{T;6HaVn-09N7U(0g&8TWxF
z*En}FsU8blc7^#s?b_d+On;nMY<mv<F}kAB!)VRHV0|Lx=P!etfR&yN)2EreT3W|)
zFs(J*<Y@4OgT+r$zPw2ZdXl8|SeNhH18woA{CPb==d=Z-@?s(`#k@Ha!y=KW^HB9Y
zi+}@*5C?;yu(3;zV{*gAn7?x}eR{7HoLw;QzCTCdvKGB`lQvruwr!pb>oz>vqULtT
zK&+tPP_mb?dbH-93ctyfM?U=130LsZ_Hno|QLSdr3fH=i4zpJM+u7K^z*knxMLOXb
z>$l^NUI=GDlgRAkZhs<dU9WFr8M$K8q1lS+|F0j)x4&{&tB2|DDSh|)D_R|m8f%(t
zx(;cq2-NL3Bz2+7qj{~niFSg<>&?$@Z!U88n<4RrE6s1Qc&Xp{buB$*v*H4r4Zd`&
z4*b&*k@8{JDs{tK_ks>)PdlL<zJUK?fN8B?)TJM1))=g-PgDFVy230@IlzN4H%H#U
zTqj`3V$YnjD{AKRK8;F#(NW}}o@C;y{pZk;Gp7GIZ#5U3W1HZ3xyWIH*O93fY&CJL
zJwo};T^ocwT_hbi>@+#tQ(RK|TlTu#W;!)(QrfmFVbP9aJz_nMB^4~27JSh<@_u87
zNp?r0+6AYL8=587FqNj1C~-5mUw6*3nBsPP8^f9=jXhufXDoi}en4eu%Ug*(92VmI
z{SU%FPjK}Abg?+ORivTkZdT)6ukU`hbZ6}|+I8&H{d?Dsa^~rptHwMr3hBBdJn!>d
zrG@|*V;2R6k2-G2CKtr@COlntjOk?C^%Wgg5_nC5*X^J7)VNyjB4eBQ#iuXyrnhTK
z$6KjS6S@$eq#5sZknP6dHm(A$k54xw_ONI){OfeFa!hl*=kk1t)1faq%C|JcKZNbj
z;j3$4vAh}kHjBgiMy%YVSo_=)cNtf^ulhQ9>XGd1Ls}C8m7lX|XEZTfD9c<d>9zS#
z?)mr5YnsJ6zG(Cuvij0g((@%#|M|an`UigM=k3~3c<;S>*zueI9+`Pp{xf`ks#QDB
z&-5hg&yq<TU*|MyYq|R$S@=4@L1=f@2LIgiftpi;8V!P1&RN2HKbGZeP1x2YiJ^<1
zr`NT8HG6i}{9dN%`<A8k@>b7cZY;Sa#P7M<#BlXYh0dFcZ{2Y*SD(OQu*l)YwBCPP
zT`kJo9|SuZ=<<uQY@g7*cTM-!ou?;0>rPW{TOZxx$+6p0fJZZ@QDwG^u8d{=@%-!y
z&Grpnv@SR)T|TUp!8Ge-nSRff{OioRES9zjCZ&zmx_`DsieKxUSY@@jnoG%Ty50BC
z31@;*HvN=5`Pt`tY5VG`uevpb<;tC`XU~1osb>imQmofaK6E|v@x7Hxrn4Awe?DyQ
z@}H$d*dna=tZT&6q(2SSdB*9AvEM4HW+yLn>$c@MaC6J7-pqNscmH1G^rPd+ku9fe
zTznifeZC2Cy1e~l>Sq%9<jag5Q|7*4`Fq>Z#Q)E$Uhcm+w}e!qifnrhx&>R>O6=Sj
z{#)yd^H%ds3@+8SJxyu>JKeeuT76-*?Kr5BAX0kautbFItlXbFtyg>93$zvCyu!#9
zwkvOCB>$8t$5PKK&HY_FxBR)BVCycU(ueEQ_b4sqkpK0|Wbsl(#eH3u8Z;k1ao@Li
zddrgepR29elfQg;p|NK5+|t8ZZ&oc;p6s%sIqlfdU0Z*P=ESYHPFONQWXWxo9sAU#
z-}-#$?v|xu>CD#u)5^{IKR-@mw(sQnFM8TVtn-Zhn--0XTKj;V8dLV#8*Iw|?wq$~
z+fL)%naNu|_cK|4VfMDacjIy2&AvVjzVDk4Ejf8Qgi*1aDYIwKWTOxRSCO1Ep*%<T
zdNcleTpyI<SjOBU!WguTLq#L?)WH_k<s1nw1Qs=P@JR=WTqxiY?~$@hyK^Xz!9!0`
zb&^h{P^#Bd?Z|B+4GUC!rg|=!kag6H!7qp@YH3RGrsWOu)lYJ$hJI0*Jy)k)>hI4j
zDHoUddiNDYZt7)N5*595mHL9%^9%SUYglB4B{6Nu@?PZnQtat8UN035!Gv(7IL6dS
zuL%*S`Wg-hv2#y*@hT;evEx7g_C3XuK88-4s5*U#&yGz`4&*KC`S^VO68+A<VVSG0
zuD`BjIP2Jz*XdK|?XTP7)|q&{``)S>f9<X>Dvxroy`cK{tB}I&9Sm1gC+oYv{F7xr
z_1w~aKfMRLT*3m+9P%~V{N(!-PoXI~`^EP*urQhD-c(ueSIy`>YewOXxUQBzzrVho
zeyD!`|3A0aFKARV$Y5d=nv>AQDlL-G%%+-R;Ka6l!UrWb=SK?4dKN!U9G>8&l&PX;
zA!w-8xkIC?Thuah;!#0&6U`;UzEv09jQpOsxHDEvIpH>4X~H6H#%ZT`PO3~j(#gBp
z?WPyQY89(V>WsG}T1A3py*V{)QkLdPG0k(<|NT8r3QcG@bLv#{YMx_n3Y3?c-_1z&
z>)dswr$=a$*2}Y#9!z=R&&b64Wr>5b^p*22jG-X`Zk&6&cy!ymJ-3#od`_Pl>Nc@&
zTIjEd?}DrBc>T<tPT4(W`>T1zThD~4{(hxWT_I$=^c~BFP^BHWf==pp{?W?s)|F98
z=1%HetMA&pXhKrttZA{|_Gc{B?YPyvD%C@=qvfu{?M^){&c*rWr#fQv>^_~iBjl;Y
z<rRK+FIUH;2@^89l~l79Pc@qs<fXL2>r#-$!iY@P2a~_#W}TnPwajQ~OZ3(4vFmd<
zwby-`Qs5mP``yW#am$ZWU$5;ExuBh*qWZweaLTO@7Y=KB?D&6X+HtO}!3{<hPZsc4
z>t*+(PbrH#vuQ!~znK{e8|AE2qdhJ<Sxn7JWVd+p>ZR&c|8{TDE#B^vGlEn?L$~^D
zhz&A*{kTR_@65Y5Li)C<mwh}t6CTd~ZTss*u7ATnhAXq*?`4$XC}0tlx#P&jJ73@<
zhw`4CizcP%Y&?2GH9YX{Dc6wcv1?7HXRJOKobuBB<dPdp#AZvBeg87KAS<(UN>n(f
zcjucvuTyH>w?2E$R>^wm$GGoLZTI&3p_f%3T9!W*v-7{z)I9OTjfC!lQH6_n%%gTL
zJ#BR2K~szFCavTq&TS3BY16o@<Cjh`T<ueP_|cg)yqb&rJhRsYtBQt%g*N>!y*lNH
zUuV5G=K}8^<_?Tq%f!}l#I5&wCe$0s#_(0Lc|+IMR!$*py=p@a-+zZ4H7h1GM|Ffb
zx<n|n&iSCozcAcWk3~<s@3oRtXQG>v)8UfLcZG*9FLY6wAvBYxh39Psv){2r!dDkQ
zQD37m!|TUV*{N@|OWjYNeEn;xkY=o^_FU-*Yc;iLD<(Ww$%*ESc339Yv-GHq9C!aQ
zyE3u%1D=N47R0r9Ojxg3a6q&y<7oSjh_<69>v@-UOpuNVY~q}9K&;Vn{_IQ3_k7*S
zKdCeGiSk8vPxnPGRZ$u387h<J{{HD7di^Bdbv4sde$$p%T@h5h-r+3cw@GN)@yw+R
zIf?)KqpmHI-4Vhi>0!wB=ZiaMb`<CB106z!(^O_W-Z}fuBV+Rv-fZr=m$yy+E;AQ=
zwOrws<%Bw^APZK{6;8nd>a7MQtsIsu&eB5HdSuuwj|cX-sJ=Y(TxR=X$*&p3=DXA<
zxQVDtQ%T_7C4HKYQ%h1Uwy@02%QIkuOV^qLPt|QVT$p;oFUYDTFy~dAk^NWFZ8pJy
z=|Z-%<c_7rLOu-fxic8p_g!Z*YRG3!^|;5jqo95M*_=R2*OQB*4qbLr)vVOiluul8
zfa_j_$jXD7m&0d7^@m+aN?4O{RKsmyYgXpMNTCIuLR+L7jSIipY&#UneD{VBo0RY@
z;gHoEIFxqp{r{sivhZv_`$AR5jK9lUu3gIBDzu3)yYjV4FZcBX>!1kfI+b}dAJ5tS
zXQ#7@N<!TIpXb!Nwpi6_mUzu*J(XI%NVUN>wKLSkGsC>xYE{O%*r_i&L%ftW#`G<f
zP0=`gRO-ipSkIQ+lwUJ)Pyd=0tC_%+=kdgI*Oud&TO9kI&ABHVn|QhV$HwZ^4L22Y
zFS0%Jb5rfOn0H~@?HTWw-tNBl^xegtX_G$OIwTq`aa8-I`RRXq%d_=9vR6A7PkOlL
zlK8h1>vdl>o}R5Sk!SIT)U7{`UEL%T!N?Kz;lzy<a!cl^b1*EGzFVUy*!V*$zjF1n
zwv{V=HDx%C$k$zX=+R?QWYqD$@X?})PeV#}o`~f;HtleXUGQvwdp)t^CRv_BClb~#
z`8T7>GA(zC$n7YNZ!Gb9f6MB6HZI#7X0kHCEjcPlD#YVfBe%4|VcnR6jhtsro<3B1
zf@PI?<BVO2+`=m|r7aW~gzi}Q+2qFm4s&|$nPg<#y0Mj&_s6P9A`=?w=ka;XkdldZ
z$W4e4+Mu@L>l*&LfYlStHZ0McoA^0qR!_>FTyLfet5{eZ@<dzG0yQ+&@Yq)=DD+&5
zb-(fIe~kBfZbuI8NR0?_ua*r@H$7Z8xBZ6L^B>3celGNx@WGk0XOZ^w)Kl+-%5G`#
zi6l7xxxSYvP379AZ|aefr-G)eU81vWg=ozGB{w2cqA&W)-`KNHS6R}2S!UnOFs|Ga
ziz_`1T7{lW<WAYgEO}*(t*P*)d#`eW&o24BeC|sfKlbz`s+^lYXgjw(d{!)++!fTl
z&a-8o#!idt7CQ^Szq&r{>yFfjh=Zqp?nvjJBoPq%W%7evJN_ia^!_}(zu0i^gdEkZ
zIYku{D&>+s)f_*h>~UAk#yZ&DahB9Hwf4nZKP^*sDPU8&({V7ely|D}T;1Cfx^Cr6
zlKD5wZC>=7>+vE}Wq)=QJ3eZW|6Mt?Vz%CmLj9ADo*$RgFzxoTl#6&@c09$zGUDBP
zuZDfB2Lh+0>|AE*oGEwGfobgo$Ml$Ga*E39V`t}yEVsXE^M6svt6;mry3Z{SYvvmj
za|h)#iptf8FMR#5J7yEps^GcjCDv%M3MPqOioTlluGs&j@zb|9_3ZoBpHO+~z5Hp<
z$@Oc!<W59$cHC+b?U_>B5<2Cs^4`c5+p@PzQPAD9P3qdi_hPK!mv$WJI=!LQH|GX(
z{Ztc1MPE;czrID6a@&h`-wO7cvG3cLo9Cx3>0EX;v~JO{4IRfSobK#fqSC_AbEIp{
z6wSMRoZAmLZ<1MKqrt^&d_m{b0`GmNMPs;je(|^`a#B<y|K1ygiJvrBb)5~Bl6P;(
zp4jt@_4sMlgWhbJ1$*APq;2V*u%?}-BPHn6vK*ENIR<(AIt-o^oZR2%bl}l{Wkr@2
z`xj2nws1X5$V<-XKKMiF;1MrwXSTbCI-YA4FxBTV`6u}-;CWt_qkm`d3%^5lGK(CH
za$Z{Xy!^FH!70aa*}Ee`?oL5(`lOov6umggw$U}9MIe=@JwRKqJ4gFe!ds6=$E37X
zRUEiq=$GE&_iWNppP*5G<ALp*C8`yg8WD~sYuLZddh@-H|3j4OPbLS(7()iTx1y;R
zMfYw{QE_0ra^_TvR=-Zap<ezyxy6${yqT!-vR3Jw#QHb>ZH209d?tD&eu~R-o$<WS
zu1C);(Sm_DENkKZMOmwAI=^|GxLDFJqq_F{j#HN|eViZikx92e_k!=`s_gbz{zg@<
zTs{9=n|fO9e0p^P&dSeydhy@l-+Yg*9>`z+t?OFYi);H>uV3JAJGHdU>&W%6o_R<3
zgjpYKR8iup($Oeu5|v?NnCX!kawzeGKZBun=daXw6Ze~CFW;#+-#*}bXU__iBPzUS
zIer!y*=T4T)H7h#)Dk=CtlRZ6IpejS)w8D}3-scSrE?^+W*Y=18BF@YqV@dExxQtr
zQ(e;6s886koX4g`idWtK#iOJtm!7>!;!Z0{nDODsGKJYcoS((KJFvn)FI(?mU(w}v
zpTbUT$o;V?<(9|ufJeFe7XP@##&ddue)^#wL60ua%g_4}X!qhV?_y5XX8|u6Sh$QQ
zJFH=~Z&7vBdtxmAKV{Cp53km}o3n*aXZtz<ZoxPE`b_x?-UwZE_fa{Xv03H88Nt{^
zYF_M{^)4-%al~UDi%5TqS4Z(XkBzdE+C+{#SySV^CgG##VqvT7i#n71W}RcPo~vUW
zY^b`8;j_%)<rSws1S#3X9tzl`t-401(yZC<)Pbe@CPn1AF{vqtzn*OOQdoShkLLEv
zCsn*<<j)?c$iJlacJ&(LAAD1O2$ej)v&^SQ-S<V{&wU@b{TEpp`WS3YU3(xf`iWa}
zh7%WKi1G5HzquwwJ@^o{%hf0+H-_uAvH-VWq>EtfXO)Hr?LrE1N*x}%n!0pWZ=Q8_
z;haUAdt5YEJaTV0Nm!<&>fya5;Qu0q*hgwgX=?0Q3MW{STRy34o?^PsbTCJc`@K@i
zwY1A#Y{?%u9{za8HjypUM!+_o%g1b$chzzhR@Hbf$Lej*oeyQsJ|-k?-FDDwnT}+S
zckd_0i_Toz3$`&_4cI<mO^0VMA5V(R1i5PNyIL3YBb}H+1r7OwRk?yai(W@}tjjyb
zP_V*ZR{OKj%L%gr@`QIRuyx{fQb{rK?wNDxCI8QkKT&In;)LhL@q}w$Dv~gC{ljri
z%U$TK!2%99uhc0E$_|yqed=7(E7Gu8WZ^6CZ3+xpSH$|ypKMrfRj>cDD?Ru!Z;Sxn
z<Y^x#oofHm78o1KR6XhVsa-Dpiv;af7BWBkUu-vZlilafKHq}lf0zC^=JBIysjQ`F
z^n9MnVTB2suQ<j&3Q|pWYSs->U38i4Y9c$+ua2ipK}qX$denk)o5SAPo19SKn&h%{
zeS}(*{PgF4q=c@gcwBc`q||Y=QX}Yume9(o7piIk3@Qzq3J!G`Y5$+s!CP16Ir)%R
z-wCfJ9SJMeYD@}EU9!S6-0AkTM`ac5NdZk^Rj%PZld>yc%a-e1meEi(OSo6SrFBx<
zPn@~$hKOq0DwgHTMPF@6)p~OH(_x*TU;ICdX>OdbCeKfl^=snqO$W<(bbB`SUU0Vg
zBDB_MYq(Tc@*_c}gUL*%9|v`wd|qZ$WV*FTxFKlnN|m(#=|+zWRDV4VVLnpAeq{Zf
z!!9K%5jK;RoP^5^51BB2G}-)a?tyn@k;SIbOWxR|r(NXpxWpGEaZPPnSmfka>Sam?
zn9dZ3HYLou<}o2yNn}E++P4br!(l276HN4L)%-Gc_$VEiKCSFH->uKK%O^U&DrNpO
zHGZ>j{8wZ9M*r-?U*q*n67MgQm7G#>V4Cd9?@m9z$4Yc;S25MkEnK4%CKtS@-N5XZ
z(Kp_gsoWPn>dEoGJQ3)?cIdaq3k98N3SF*teqpJin^f6OD@J~4%UhxGY-QT|@77A*
zOgWE~quaf{o!6N3^`F~mkCncg^#WD9r&$zRcz!tQQLl6-=i$m-n{DF$uTxc;mV0MW
zk?QoM;*FvwOcx#7UjBp6Z^tCRlLx|%3eETylK;0PC4x`?#Ehq(+OjX_u8|APIV#FC
zzrtqK6@%*@JiRNe`NR!xK2s}vrzR~n^Vtqn3(*(PZv4nP_g7^bZ)&>smIHQrXVyH~
zz-Jy@TKPmH&LwZhQm1_@j~wbO4Du0p%vp3tGhDFnSX$Zq5ARg2ELH7j$yh&qLBr(O
z7s+;hMy>^m9_?Y7e*8d{<mZXT&J}Dw!i?G`><eZnE9_c$(RL?;-TM<~-iX>47sfv{
zpY>7n1JkxO5*|McwmSa4KK1O9=~sJIRjw=&n6ZFu#&MtCM%U0(Wr^makDF_jCjS*|
z*8RU`E!U-_?pF7TBUgS*oF6_-F|;QpZG}sPueRE!*E7B-J!akh>gc~$>)$VU5f^5(
z$hBLv<A(|Z+pT$`olZg43(P!kED&Abq&X?)j$hKE3li$r-)4tzOgYD!Bfs59rGbY-
zbHc77*G*^UNG-|xIL*I$m(Jv%&d^6{^*l2^R@_|>5xoA;3h%21w#QaVmOD;ndERrp
z|DlCpp?8IrslJ4mYK-ARp;bnUA82_@pJftk|KPB#&pVfI9~~Vgo1}^>pL@w~zC_XK
z$m?62uAgR=Cq8ivONnh@aq~Xpc7s#(!5UA&hV<#v+EYbTj13cB@p-LzRMIoKI6>dC
zw%t>6`|R48PVfGIwYzO@e}B6D>+lK8KNBh~g9WF}ewt^<sg+pH6a80w&VGY!i+42K
zyuo<Y!pHcqP7`ahl!c;fQu09)2kjY=W%>ns^b5F-G?%tSoZ#QJUg^4((*ng$uGf#Q
z-(4B^F3>&lQCn%#{~jeZ)`BIyN}3B!#5a|xxK?>Bnx^`pbvlOt<8GBkwk#2g?RqzE
zY`bc(Qoyw5ue9o)u$=2ZwjJELJ?pm9)L*fEEBj2#C-^%ZFjp(GE?=#E$NT$_`u`O@
zLcyC&Z}Wr)uHL)F=#5kOzQYq8ukr?NkyD(>7q4sQ(<gE!HCjQ5-`sM!ep2DHsEAj;
z=7!9437cHxBst}}ezDI__iz7C?tSWCUiDc^O!|b;;{^q0JlodXGO$Wm@x463U)zdz
zhWIvpuWF&0{%4rCEfQI<Xw%O{+awzzqdXga9ynUl`S6eKk~InDyTcCc{5#J-#Bjxe
z?T4;7nc3dEncRCjc_QnDw>z7AUFYkD9Sy!VpIPcstXNuTOWQ7njZCxu2Hae9V_964
zQ`_p6P0`(Lr9U#v13i6KUs+vKZX>0-uA|egNOf|LdW}!!huen^mz`yeyslZ|c+&FR
zdz%>>rzgiGtuU>T4UFEp_}8|sd|kOaHz(E}mMmVX_<hytzt6k&8vO|B(KFv_eEo)W
z$iFvkcY;q(46orcoa_+yM?-1ypM>zq#k))%{{N-4sqbF2@Vtz&6TX$jU8NfYqm)7<
zR1=zxUo*Nf$L2$ev8vr(^|C{kYmaAcJ#={SnXRJJ9(|sE_991p`0vAS?@YaWXN&RT
zbB%i&X9*v?Y?#FVV|z{algg)zv;WGmDmZUtT61$}vFhtz6Rg)waBX2?FPJsk{oBU1
zd8IA!8Eva(E1zqf5_)KF^V`;!`fakau9$yRp1f_(%HID~S2pijd#Z6qXPbq`YG3sy
z)}Btk+$T1b8{9tb!MOip(}C~RqPB6{AJ0?G`_#DM?t#N1(GTY(6`#B9K4<mM>OxnW
z53+mu-Y&ZF^M?~>@a(UL`o+J^P?GcE{eP&u^$2%Fa>cev@zr-U|6fjtka6(Rh<Nf?
z<67;pokwS<eB~EaU!I>fH_!ZjeN7RM;E7O|w-b~~3Ow8%WvUj1ESwV~EUNaduKrz%
zlf`zYH;YXZstzrR&aRq!!s}b;xr2Wm8{heHXw@&%xNYILPwU@%YhG6&r}6%2%zNhP
zYYwgQ@tXH|Vv19{?bFN5$=p$M_o~`Ia9hpNa3PTQyO+}XcaieFtO`qV1PuxnFf_3X
zs?FIHz;IYIM$WB7Vxy7^qo`rt!$}h!JN8QmPxAEGwB+Pu4eu<)tE)n#gVYn{rf4##
zX)~I$Zm>L&_1w|SsBuwF=_cPpb8LdosW@jX37TX#eNs;9YPrBgoSvE<N{*cj|JSpN
zs)Q(*t+~XaK6%2F1fjQ%+nI!>v}!NVxw$dlbDHnY@0s@}7v29?d&}%?!rmNjHn9cQ
zQXcQ}f2lTQ&h`xtP8D*l5B(Lf`T3-3=f|erRVGg^rK+z8vu$M-=i{H7=d~kq30rIQ
zmKiMBK}}PRa4!8bZBgovX}-s2PFmR;v(TAQm?dY1P%YEfYu9C?$}MD-+Z)$Q?mNb@
zY4OkgCp!xs9h$c%nDPB%`6plGQpAt;ux{JsIl18fc?PzQNrCJW4Nv+1msMp5G*47@
z&<#-yIo*`+c_A`QB~VSFZB{17^fu`>jYl0SY!!)}>5LqA(%3g{U6jetb!>XvgwE8*
zj*K@<8~-z{I`~XkJhdQ@X@Uz|B$H&SL-=fl<OA&Yb9PNsnG+Ral$I(KRM46#BIKgj
z$`H@R{%l5k-o`sKz4H_&IGmGA5prg?{W8a;B<hHuc*q^eOSKAxS6IcrFI*8RlCq^F
z*e}8`L!^;&W21<F&!jIxjY~LOg|ziKLp6P-npsWupB=<?rFz~*$B=*zTa6yJL~IS7
z+^U~fF(V?Ykz;YZx~$rZwN0;H-RxYT`b8pN!#3@8UIS;g)CLv7YN7l%*KDc$YR}iN
zdl%jHh$>wjc;Qy|gJ~CBIa4QO=k0tk&-)Tf+}W}q#~`6+j|%j-9^Br0h;xf;k+131
zm=fktcG;5ak?D8;mt6b#V?uv)jkr+Ni<h24)grGmKUS(%UWq%>;%VrmJ%`OBxM_D+
z#brOEo!Od;-7GyqPV?D$Bx-y9=vuTd#$xuv6r1p62ilhw?)coZv+h~qd6(&HzIbk&
zc4f;RA9a?P%jGjow7s}k1g2dJbv~|Kvih$@%7m2Zf!lUu^v`~>Lw3dmN6y^+vfmw5
zeHJ#%WSh9rb+YElr7f*ta!Zzq#3flioxV$Sx!R1Ky@_eLX+m4n_T=sQ88e6XqUXeC
z0<lvj&Z{am5%O4K7h2Ug%hQofTi5ofLXhp>>z9{WxVBDuYs|RfrN?sn)GXgU@-9<D
z*UG1OYj2n*W^ygaTwcs2I%mVY|L0`%+`^MeZl1AOf66|0X4)=3wnaTv6YC?A=4RT5
zt>H51wY}u9c$&c0d*?I$WvupnFe_WO;9}#g=UfT9CY1;<aV5mCJ3e6ao^YUrrAtM|
zqrvfxYJ;yyCr`ZBiAN2hty?d=Zr&%@m#=j~|EbCHqAQ9KB?4xwVu#YKbyGYJA34mZ
zRl#l}qriVe?$}|w+9h@ppC9sBHZd?VFgiykc)pu<>`2g67CUyMP9L{R3tB8VoVHnb
zYS$fJBb1xSB{X5d-U~%Ke`?luDrzZ9T|aS5koTE><m%J0w~jBi`nJN9S6gVpt^hW>
zX{Y<u7@q9#67blua4j?Ul-Ud-Kesh21Tm=n_uydW*Jx0g*~=Zvd5?8|isi~s&w}tx
z0n=P&>OT}|FHE(%{P@c<wNr{qCaX7Io@2J%()y{Uzr@EF50zyz?|przDRjf-b<w3Z
zXWcG)_Qa4gEDoA1o?0P)Zh9JSW}1>!rxjwQ=(;Rr)7kiCPs)7@gBE55_D5|=p1Fa^
z`^UVa=QjSyRWJ>1`ZdMLPD=ND)>|GiyVTGv>nC(nT~X%Tu)uWxd6p}B--6|~Wu;8w
z)JZVC|7iXMHjM*aQ9`c#J||suPqG>X<Zyd>FtB<X32ozf;{A296i2w$oLbp*o~<9J
zdoMLMXp~%&U^u6bd*k8_%@%Kx9DMuvG}S^Dy%I=zYxwHFp5r75r;U9n%BQ<RoUa{Y
zxO+t7p!G?Oj88^focd0}Lid`MEO0w?;HiGb&FL|H%l5o#J*sQ}`sURg<|f;d)auI4
zY70NFJipCMaQ!mQ-rgIV7IS(eSaBtvzwv`(iO~$l3D1I}6>qCC?AUpBQ(^IKCG+1W
z&W3%P_{6G%^IF(R=5K`_&Yvd!d$Z%(8dU~{<C+)l%`;gxDaAA5h49k{e8-nfYs)NL
zeI#1@Cd>Nf1D||nE>3;^X3A;nmq8tVPIF5%En_crvZw|na0caka#?!fT+oB3o)vQ~
zcc)ZwX-@n!x53X<EA(daO!k=$Oby1@r}*&d3w6xf`)$(<hL5J1-&g`7S^sbGTUdC$
z>XCDXy?nCW?i;FxO<M!6J5Am%*UH@>qI)=lRl@V)yp%~?Yu2a;TX^=WwJdi0q){Dl
zW9qs;I*f{}4opk}3=Av`91I%@`xp(DO`P>)N8{_WULREt=_{%)%b2D4?9uE)noB0M
z-a5#~llWLx>QO<5+Qk{untuqsh$?S)b!o`|cKPnASu<x^6i)Kl#x<X1=b0`+t4@(^
zlXQ%_RunVeY)J1jdvM~%PJ`69pJ!g&5wJ-iB-)g9_MsT(ZMNSo<}5ayz2nr@1A0E+
zE=@S|M%gr4Xh-1pBkYFEjQ-wBuDtR$Ej20W3;TO3>~zHOb!A&US%mf&OHNl&|NCUB
z#Nz1xfmNFEfn36lGRw_F`L<78?^C8TS$lb!f6C6vPj}qPIt>q<NH3Wu>HKkBkWg}#
zu%_O{WfAtOM~z-D37m5&#MJuh<l^a-!5pVnrQgsNTD0ZxoV7-J;T@s-8|<><^zOxZ
zu2{bJ|F7-qm6{B?Ip+CS3tAkgI62Mfb<o3(U;Z8v+g30H1|>}@sd1KGo8lgCe3;Xt
zF_vpWx(BntW2x&F$Mx@TJX7A2%*gPdgZ*HoXT=;A1^E^WrVG6~+^@DM)^AuEW#Pv;
zQFt%UG4qR8557EDBz1UR(sApEzKElfqBeG>{#23n<`mkL-TmLeRIOR;!pR#kj*~Yi
zcG)?K%-Jjv#&z?5)0%xkCfA;wE?;t0RQ2QQ?t6xx%YR7zTUUQ=_5I(O5BBX7Tz5+Q
zO`<=q^f!}TIc}FWq#RFu>F9Q8)BLvu>4KiWCM$fh%jV8io!nA*W#6+at-1GC#hxyT
zxV`V#`lEaM?jL%5jM<}<^P%+QIfr-bZwxAE2zHw0mT8eVnRCY~nUfn;pP!00PdPb#
zcIxvC$+!J6DkXlOjHdfCxh!3=PWE!jvxPy2VyE0-DZeYXes}WqcS&wXg92tIl`wv}
z@@-PFW2rjB+YPK4ivCJL@zsv;E$z0NsRcdd_OlXeGRq4RA0{-0CjMg9`}^5mYI>so
z#6Ug~CAJh#k4>dI-^@QvVK8t@{?94EqPIQS{8^F3v}DtE28-*}X43^MB3#%Tn^g`r
zTc!)yi1}=B6jFU^!hTSCZlk`>59x#?8F#i|&u1k;3({|-8}Ln(jpA?)c5^mKaen;K
z;L-F_Ta!AQFLE9SBs`C3g@g*J7&p$FsF3Ym=bs{C&?qWzEE<_2w9TR6H>+gx^Xdlm
z4o$|0?sB*Jg%V|IlC^2+t;-d^Uluo05;qg7ZkuW#Sy5T@#C`I$%11_#>`%-VWMmdS
z&YZhcL{Xh#QJBMHCly%-RvCusm1^Rf!>f0G&)TJ)z4v&SreGQqLyN{Dv9pS?c5XF&
z7qcTa<uChEoV<wjh*->VGs`2XJ*V3&|Iaa2FZ&d8Wtn;nQ<=J=`~el8wnIkoQ$;e;
z;?x3tx+nQgG_szu$cOWz(8(*BDbuXqwD~UHR6nax^C3(6(-J>J<{-`X0<-4U-y+H`
z<^^n53+fgYoJ)?McTxOZvKY6h2{(h;PiIDv4fboNCyIqeF)=mG7c6o-BzOO#le)UU
z*77FZ<;nUS#m4T%Hf6=eo<dgcLUt8GmXa+>laliTd-euK&$`&+yTZb=JjHXF^ttVN
zrlqnvTY@`8omm`Hc|Vs%EDPDA;^G!67kAvbCr~58(dc$-+0$pSDxcc68*?c)TKgUJ
z<t`EqOmVHwaAmsCu9e|7&yZ2tFrBHaLUxkQ|CZwws?43sH!9zmEG3*C-1SYjJJf6u
zTjqqNmD5Z;x}RkB9f_Q3S;fg1#h>EP_^9glWvLb0loP)gZ@Heeb$RuwpIMeO!Yh_4
zI&kz@RR}rkXx>~Yuqji>K|)pKXwSjG=!i)zogd9lu83}VT$17>a8xbkQb*2F*WMGN
z>gUbo&bl1)uzYTcMcdw;&OwKTK719)yP5asrxjOGd+IT(6Cu-1y41gOn|>`NRN?Y8
z?n_+X9)t=vi=B{Eh%-!&P4;h-h-k``;#x6(qf(;>haF>>Dc33n@yJ4%4HL!F1ErTG
zzA%bX$P`j26XKr|s1TOKa>QL;P-Wf5r0W-xybeq<yf*2-p+>SpL$z^+i_J6!3-!eo
zCns-PIh$eA<h7BDomNaXpDAEoQPWZ=eM)J`vFk}OE-8L1N`5y@Ik8b~AEUbZbOE!e
zPVpkL`d<V%m8Tj%2{tJSsXa6`?q%<tlc|Yfj6oNKzMiOi#@4R3SU4w5;ZdZ}{hbQM
zk>#~NjD;F9{+~4deAO;YSa#p_4jxzK>q<%)ALCzd5jS(K6i`&@WAo^9UAV|BO3lGz
zNk!%o56|ThGbemmIp58Esi?*hwpp7bsv{?_(n;@LFP6Rb)U26;9U2e2H~thc|JBVI
zxO(2i+4C=YF8{20z{ui|m)D-3vn!Hw&RSZYVpBM~tX4!YH)ez2trNZfZ&$>=P|Sb2
zykC1`?){y0!HvQ!pEQn_h`jcY_#xh(!6x!;y29?waUU!55+eDx75F`NWBVv#!tho6
zyU>cCFY~<;8yLkKb($6&R4Tj~6mfS_<DbKovO5Y3P6Wz*mE|Zd+P`tq>RXd6%eu{{
zZFFa0Fsmq*&n|XnVaae@z2jB4^{!(3S(ELx6s)?0^mi*dOqj2_kyY)hq)TPV<jYG8
zSc88rVmxHF)PMPmh!alcOM`C+P7S#>HPxdn?zwZ!NpGiXRvDhw-e2;9Ql~{l8yS2o
z%X44uV#w9c6jqYSJ!e6armI9-r|J96>8&$5_$A_3UW{L)Xz+2Vea}kwH07*C&)EJ?
zmhzaU6*;wa<#aFe$ss$IPT!&G7}XP<x$ZjSqLYmLN0ql!%v%3y)w*4a4ZD?g-Rj=<
zOVUDO*Y=e=tvGibaIZVMgH@JAtf9H)Xs7C_bc_9&F{f9WuP})T-k@g0GIyG?_rB9P
z^ObYUjdCBf=icm`>pNQ@a0Bbw$#K$$VjVBm-7T|zHY4qk>fWc*gg>}Tyx2J{EQMb{
zsNrqGdM<~}EX%aNi%#TUxuQ-{IkdS^N;6O>YN6z{#JOfka%~I!4VxCYF)6EWyyCbq
zdZVcEwWf8)HgcR<WEWZNc4V=c=B}2;O*_423%`+^eS7kP=v`idj1HDHI-evIdZI<W
zr(7wYa`Irw|Ex#SYV2xFT3b5HmR@=|C0fJT+dDO};&9kX$y+Z=^ArWPeh7)|-ulW@
zDD?HRZ1K>*g+8}tr|A|h=L{@M+_Jo`wW9WE$40IhGYgelW=aWY*cU2_xpZ$|cX3Aj
zZPuw#$EK}H%yphQ`_{2}sWVj<&J<5Q#`0yyL&5Hi;j^|z9GHIl`1+}v@^-C`=$XAk
zYDP=ViPkwM>^%;89bE0t$igfrpXaoBW030MTQy0Ggidi<UhLR?>Q%Ak$K7Xl#heaZ
zyGU5*hFHkToKrSStd0inbq#bqb0l|$QJ!<6&{m&ys*Zhkw`$z~cIwx6$-mb%bRTLl
zyi`anSkHb_$3JmF2=jsk4=4P;5M#>QHBne#zhGCQ@UFAWM+%jWIjD-AyRVa^ad3ZX
zK$FbOrEeZ3X@?&$nKp|{=DfkHjSdOTMp@^TwGY}w3UNHxxc9|*8@JUK;<NXrH#70}
zFwZR(%4xRVu5DFykm1wfXVn6W8T9=(w|x6#?wqFoYf*{V7S5OtOMYxRr026GK)p2N
zxq!F#va7QWJ09I49(=-ci@d_&W#O;u9(Yd6n{heX`=W}mP#H%$lft$tkL9JFT!|@e
zp1l)pZ{GGvRI)g+@#;nf5r@i<1eM;e8MDGyPFq$vjmvXJ6jO^!)%4J+>7Ek3+zu16
zvMO{KSL`~zrfcW)+s%zVZgXcV8CdTU;5qTXW6$-v*n|1A#d8)pUYjKlb;0pqSI=qn
z-Hz?qTdRYz9!2k5t!89$BezLte|PO|^|klZt&UhPyKa>m-g&Ar`NpzKLg%ti-><s-
zyF$cCaoxk88tZdikA69|ar0h{k8&=ISHeR4R&Cm0`dRf=iyO0+yJ&>~_kji1&P<e9
zaW-wju_v3(%DZJg@HzKHCMknAOG0sxO1gn^%_6NCi}W;-AL}mWymP?j*nuvd8{U_9
zZoO5_*?VFSS4+Xd)eZuTferVg7I6NyWt$bSN%hkOsh-K!KQHL6ZhJOeh^4j0lgr6J
zLcs4;(BX-@wPe&bsjT4G=;Xicp}`f|*sjAKs>_nPLL$u`{kPf7x@NON+Vy2??mn`~
zz3jM0(NfS=Suph34~_VzM~*dL>9jh^;1*seT4}0!Oyq+7tQ(z=4qaU|W93Y?9VurW
zz9(M$u{+Z9kVEmFu2m6wOTSfB*mSRYb*@PFx+TZO(%5G^q!^RDxz;qyTYG?SjR31F
z%WRLwC*JVhD9?SqU%mE`%Z(k^T&LT3ooW|4*y=b}hjnu93zJaGGq2X%4nO7i^~Kd)
zrzYqz$ox5#bTl`q^)W+}=)+lc*{?JoXbFE%&3n1L@0Q4!d4^{m-r{;|uzuP-wiH)g
z*UR(U%I*}YB=T0B<$7%|`F!Ht9EV`NRr1UCuZW0ZaZyo^G|=)qAijIm4)N>%b#5K_
z`sO@O?PSK-^RI31+goj#e|vGtx(n*nn;7PvDDRtnBTne~JeD;Np1sjyc=O<0)Py##
z!UvVIC+-!u=q4@scVw4F$CBSsYI;vEo))}$c;cMIq)VYy!CdEqi$0Y`dgi8f3Aw0l
z<)6%wu=0`H-OGk@N7BRhn#C!_dJARkJpI^g+wsHgOC7>mJ7?4#UlF2pta3?b)Zd7U
zdn;$CUQ3O=)~!<YZFS__D2K&XtCni1Eb?S5a}nEOkv#46@x*hps$-wkM!lQ!Tja~x
z6NQFc!bQScHgH*P<P7t<?zM37=e5^U_`kjGUF_JlnXf&1-R>Jdm8P`xn%8`eZd<o|
ze$CBm?lu1xt%(ty=*^oVq|`b0a+`X2llR>hB0?Vw=hU4p%m{scEB8}G+}qCE^O7|0
zr3md_`$+jB*QZ~%CmE&31-?F(dv;ah*}s#`KEK{DHD$llD+k$MiJXaS3$_LRmv&Z5
zFJiuP?rH2j<BIb-QORyk#Gd<Ys?fX7+;@VrfrW?n9q*YxPwY;F`9II-U(M+IHDb=}
z34h*g2v+!%_bpBC;ATz7L+LH-T1yma^kWPU`TbDc{_vu{>ZPATOWFOGwyb^lP5yB7
zE8Wm-{}tkl7!JA~aA;y;;qsW!;<Gx8)swTr;zOcK2aBecPQ=7VuHBM`RV<Q5D%y-W
zZni4UT}mgL)<&eIq;~&L&}}l>t}<gmLQ@N;u!YaV14~(^8##5BX<l5)6SPEj(zF)}
z3s!P6sx2zp=w*<_zDjbj&>O1_Eg_7lr{;+olw`2;$<+8QSP*=$iG_>l#@5*i4fj|W
ze*OC@y<<HC0}FSz%bbXhN&OrW_H8neDNIdM1g*scmM&zNrV>4IQc=Z%RK^*`v)Xc^
zOix_xk<Gu!VziAxmqpb>aHflskmjz+-_x3IKb~e7eL_NBBelEe+3C63;u~5Qd}+KQ
z=r7n%%O(Ey=H}w>8UZ#3^->N?yV%FsRxt{B7`9k$IP1}*^PIck{7zoau-TD!|M%M;
zeJAbV&~$4@#{S<*N{=_P#d>uf2>X9)<I=fpYIE|K7M{{*bmq5a`RL4VE9GtUS-?1Z
z<)S98tY%HMia<`*6dxr<>F>9+H!%ERQesdLVsL6uXmn`sQ0_j$!*G0>&I<z;m*%1c
z>eG2PnP@N7@jcntv8KnMML>&(v#IHcuaM@V4hGIE0?ZC3NAH~B@YA_na>iG0>6%HN
zr|Z@fooQ66;9nAEGV!vDGe56p*O4U?8qE5qC~yRMlqQFC-qy?2-}Pc!1FxBp*-fYJ
zr+TR&U7Q8RQx|ZuL~IQWi*FTl<62hmaryj3yRTkoHo8^1;A4$1|6YzI8O;l)3%KSi
z>avR}+t{m8ePRh4chcfT+|o)<Z-i-mIkAlM|E(JrSeBd;5Z79LrnF0Y-Kt++4Q1R}
zU2B9+YVm6>ZYw&ymQ(D6Cu9A4rU-pj7G(*81AAmv1f+<&iE(I}NIFl}RhSlWQQzIU
z%=e_pyfxlE8+{}?bJ!QXbPC$`;q{gy7GXCGrVH0eCeHK?masg!#beR7o)T#ei}}^U
zddhzGw(G2(u_WYmJzLbgZr4k$LwO5U?45jT_gl6_Og7Dn?rr7P;u6Y;YF;5cyL-)c
zzG{W<pR!jjNS3VotvKn)I{n4%QHlbNf4)6l6V-h~VX5OrXQ9~(r`WA7bBc~yxN?En
zwiykFd%Rq-7M$`oIbC$r=y?;5UPQ|uSIx9J6W6PhUE%&J_x~@~B435uB0>)jznii%
zbw;eRN3x-$8t)Q?#TT3=Kj6IN!sV&<E;e<7q+Z6>Ll%6UCYMaqf`k~|B$-1rw<MTw
zzKBe=^SkYw#mn(}jkI&aqFiZ?OW!ybm2CIcO%-|h(&A~!<TuuL_gx9j55DCP_H-ie
zo>Q)?trnCem`sSS)ytCkAj)#{u=a#6T83hKd>6!h(n+;Ft$eEJM6$&vM`70%m8(*U
z=^UFRJ9+ffdOveWD!H%wlyOekZ(~gOE7J)}dt7xxPK5hES=6wLK}X-xDB>W8+U5_Y
z7dv|#osMK3?h4$X6n!Ij%DGA3UG(hOJeVAqCj2r|RqcK6CH^`h`M~x)|KIDnaq)V^
z{}N*rn*Vm0dD#|ukDkr$zZbr;tiLj)gk`1Xa!&)5yN*k)o^cac7kXjh0ip9^N|gfN
zuQavFZ9ISbU-OJ)K@P(j>kDeHnHL03Vw&=4qQ_R2?-J7){G3Fx+EpW8aGjW#7&p87
z!E+f2ZSRX3iZur>#@8KDPON(A5wpl=;;iYaM-4nx+Bf#hj#3u7oFk}Guqo|}(V|vg
zjV{qQKbA2ZIN7JGp_QTK+&j(alKRmnncl6L!Zs^s?mqS9q~{wS34uo|{p)5;G;BTe
z{B2gbuj!+gWxs?ybF(h5@GuPt5_+M!(jatpKxvln{8v*}^tG+BsX7uWUmNI=7^405
z$^TOm`Zp>hU1L6Z$R$y6dP<nEGE?xxf+NdhBhPRu8_jR8C{nmlC6K=DYG2;U{I*oX
z=72X}4=X%NVJg*Bifei)uA=oZA@EgF*w(B?z8Q`}YZ6>_Zx{+haV!v9^WdID(6&&M
zZH0zczeFT05f5)Ho0#?`CeHEME5ZE{>?+USWSp0Xa=-te(Bjz66@{}lM=a5G&phXO
zPT?<?sasfB)`N*puIw-~{UozQ!$#$V(nYf<uYK%xdK1sz<~m~IziOM3e&=ki`A40?
z82AcSyY$@uaJh@)s+hfq(Q0=yMMlSePwI_6J&kKRF#Xy_zEa1EcYS%w&+A5hUboRY
z*>UsgO*c(7mH*#liI<*eu<xT~+WPpWAfaOdorVp@Do1Z05f?pX_~B8W-~HIflIL%`
z)#r(>-ow1+!jc7z6L$DBd|WK=q8eq-*q~J}c2s`3nU}1l&)SAi=Lg-3go7SW6S3wq
z>0l}UYMG#zb!6YIt4`;n<OQ-=RLpY=xl(nUPP~{FslW5Wou*y#Geub1QW{o;^v`fn
z(x152-EC9Y;tiecouLe?U+|<K6KFMDp?Na=8%OG^?)aWe$@Q5wS4?itFrKaJ#mT7l
zMz;R)3Wn^57W1|I<bVHgf8jB0Lb;fy($_b0*B5R(B6(7V^=PrFp2sKu*(+EBMGw!v
zcFos;t!U2e5RS~iuA;3>i~hf?k!n&n;OKUFZNX9(38l&0mM$}v>-kIwlC>^NYV#<%
z6*y_blhnf&m%Y6&31!tinW)I=c_!{wlEci53%Y{RyW{>ndb?mtcKW7ZlfA*~KOCE-
zaiha<cTCZ<#5W}~6P-Q1Kb=`U`9<cvKULyooYg^RxRi3r?A>m^KNie%C}Ob+&ug=T
z59L}~!vr20e5rlko#EoG8t85+acDu2!9SC&jSSTyVQUtJdc?Y1ig_5Rq8WRp^XlXl
z=KRFyADS1^C8jz3nozLS@Tg{TfW?GBk6ZQvHy;FadvQr~ZrSg9$LQyVJ;igopSTBB
zoDhq+G~pO$|4Gfwc{^Y3`m(2P=kq<kKbidB`qR)PZ)X7K#>Yu>7q>nVd2SKs|K=B)
zk;NO)md7&UZ*IQjkn#SgX2H7t-0bfsR@&$--`JBnYp(F(NY^BHs~ab8rPZ2lT|f1y
zLaar{wH?MU;$0oMg>RLd46P`dxI;|E?rdh#qc(**T`t>s7-b%@F~vyawLPnSHF@gI
z6(wf5-}iMf>Zg^dS1a1;`@M3@zbd%wm+_?PnXglNOeeE9$b0$-MOmD5Sz^(pyW-{5
z4gJ~YK1VO#-jq^R*!ubykLi;5WBL|<tt);A?z3+(i|Ff-%)8Lk_Ki87>mYa9ilQaP
zbuHyt|C)K;Ft8Rp<V~+=P(0fg$+>amL_VR}3d}7$J)8e8;OO8B+R*xM#%%G)y1QrE
zbPMbhFxr`R(1GiWgQs%G)abQs71H|+m-6%IO|W+SvqEs*Zlik&tv3@6&h=)vGeP31
zg<MBxuR+vmi6@RnZWyI_H*DqZvv*o!ep&v=t6j=1#uF^|tEqGx<m5cSw8muhVq@-3
z+ey29XY#F$min{VV!{n$m#95&8Eihh;OqZk5I<4HIeO2TTT{(fwR7=s-V&O^RWsYC
zdF}jHD(6pdBug&sVGy`2)X8>n?U94CJAar&uV9ay#W_pZTyx^&dxeuToF-`pF4qp+
zx~zM$(oM6cCuQ%gSX8Fatg>P6)db1An{9pX9K3(qbk*T?Pd}>LGS2(N{9n}5W3tzi
z)uo^3?vWDOZp3rDTPQqPC~=FBmZzE4Lv8bqW;Tuum4|l)3ECS;ux3s<_GYEqU&j@n
zcON=pvH#?%m5!gx87GJoinz0SZ)nt>y1l#iSF-!>*HiRsL>Y|~m>+HEIL#N$sFcs6
zn-b{ItkA0)F^4;5T9krizrYE;C5A2^7w~FzYAxcQS}^<QOT+$%gK{=gelhL(&N1b}
z1`p?j`evGwOgfuXI+$c-7XPu*-@9p_%4EB@tJEGa%zGv<RcX(PSt?Sq=Ny}rv%z0-
z_l*sF5<NT`E$)w2@Xd9cwLN?PmSE$bh1M${YJ5F-XvGz$KOYxct=?>X`FPI*YujL(
zgSY=noo13=AG3$Et1mZkVVT9!0I#*X1RJ)LC@~-0Gx5ci1A+68XvpM9?!Lcjg3E8s
z&It|HiT2lb@4aTd;fC|Hmy-HRZnA`J5cqzor+D_H#K$vC1$pCB_8yFpsC(q1>?rau
zc~@o8)>B9H@6TTT?5AMW<OYo$y$=q`x9mPPWx=rnGu>{soSin=x1+=Npr-Z4i*9Y5
zi<=lk86}#hWlL>tmS?TDFJW1*AoKXI-TQwgTX1o6{=3b@d%BNx784^&7ysji^*?3a
zix>*tcAqMBp83p_+ZxkEoh|uR&7K*>$?Id;lVNzYV`HC$;@wly!Zy={zHAJ7eYoq5
z$FrMF-y06>dvb!o^}kWIQO`7&fVZ1?3_Tk-e)KBl^eJt*Fnf<@XNF~e#?Hwv0)lP?
z2u$s``QT)afY(|p$;T=7S~0uVOg6r=BjC{l9jo2edz*RIUEwp{BDLzE&Z>i_mMb21
z628<lp}}{Bz^p9kV+mVMzdp@msd{zN1fNU%yBc-<lv{5zaY}1<ZnHU(HM2h{&_yC@
z;r<fO_luU_^jwyt<96%fk?zc`mlnvL3OL3R;lr({wZW@T$#dSd0QJnE;Mmi4u}@5+
ze)}{D`Ls4}Pc*)AfA==q)NQ4!g>NjJ{Ho|!)#rU(S$;Q8$Yq~#dyuoC@x;8>f}uZ4
zHu5!{J+RRC)|_L!Z+(wU=o8KjZTj)ww@E^8Riu5<72d{RgQqQlQKI%7i>=$7d01}k
z|J%KT<rimH^#c3XN<E)<u&>!OLE!R@MSGf3c=((A(;Xcr9JoADlcRIbl1USr4rT<j
zHS(VC;FzKru9p<>;`FATi<T-A0v1d>*)=I}1GBZV%$&~)LvJc{@E$$z<Z?jg0*(gO
z2nm(IqZf|1igw<xSgx6|`)<bmLs7=}HaKx@m3RDQW%npzxAm+&)w7OY(2@0Baz<<8
ze(61Xy(2fdNEvNt|8Zcp4U5anuPWK8r)O`qky?DZNno~cD#zY~rz5(YU6U0xUu-?_
zXTAAKyQUd?rxxxqcI}V~?90jQ@U57X;U$sltbXLdf7@lg+wY#_*uZ*wt*Y<TBSNv#
z`}BX#TarBKkjL#etFLeboPB$-X;-h?uiIA|HS8bqoC|n*JK*TNQkIwlf5Z85#Q3~8
z@66%IJmNd8TXDge)AM2^dR}dx*C<~Y(hxIszL(<E_sna`YBsPdFKN3x<$v_lS4aHb
zESbe-w35k2GfmfjgP{MeNe#!7Wv(9z6Z)jkxM$^r75-_7=LK$QiW+qWe(hlWsOjx=
z@RIcP+0o%<jFU7bG`->MEvvovx_2(e)7}|(to&|<sHpI%^6c)OboCQ!=;zW|UsP2#
zDqk$^;W$xxN4vFyf9d_#f`NPAdhF-wJ7{HK`Z@l=4?D}pR+6p8lDz+y_^$L?{xDql
z<|)~~r|dTGI3&#Q{!5>!O+tSGf6!un-z0DUrj@pHOx$Oi>=Qh=^^VH+KmFRe2hS}Q
zb70ZxxiQW4O#j&%Oo^AH@~U^GcKHPV3BI-?@sHQC%Tvs4TQ8qD^6*~rvL=b!hKeC>
zrCUE|om99v_m|MN=-(4}ES;h_MWXCUh{fS!d*+6!-?~!zCnQJYDxYog<E7_=u5ui!
z?chs|X-IYQpZmDy#T5ywnQf1EcqV#1Y1z+k?0)H=t1Zd1CMaBEa=QDrWMcK5bJ3QE
zuP;>)W|ri9yJN-y{_k@fwhQk3cVNkp8UB_viv-sk@aEDMO;YrZl94~$|Lo{JHLjkp
zd;cx3I52RgdU1V?%Q+G-`^wXpZI6Y&Ubm9tvwao*Csh7KZ?8ZphgJ}$$gWFIj5!6q
zN?Z0hzWU%c?`3kA=%Hmk2`j!>IcoJSym{!@ufVTYy*51Ay~;9uEvL5uPruuz4)d<1
zGesF(Uq4uQ+C)!DVE@FMoJ&qSnK(Avxcrsa8x`Tgee;^Gh3x5G$<tT#j9JgV-DDbl
zIx37=x$D^SjlM@;N^@*XjhYbg$lzs$CSPXd=UdsEO((4j2|Br*efNq`+pE+4uHLy5
za4%&3-6_ehZk$yYIX8FO;+bNd2W_AE^WHHjyR)dThwqT@?UH1%C4EtI*PJ;TbJq79
z>tx<f#m8GF_dSaF|KQ)`Syw_8_DA1M?M<1+%kN>gky$xNv%$u1o3o1PojX$<rZn^9
zuqx_qy{TfVz4?!JQR~xv+fFFVeERmR`BRB_fvIj&Jvg<h(nQ<R__y8{aDA3I@7V<1
zSx0Vp&OLHJv!z2YDCeBP!nQ4o(%W)Q-rFg7i-|dSch}$e^P&%4C`d_uz2~^fOZu$W
z(GSKY^AhBrS~o68xZ`$n<G~%@Zm|4s;n94X;xNU;TUu2!HGA8#Jxqcw$9@FmBsPnr
zHA{$He)&N7fi-W#E}kW)O%M7^dVP84lqZ|aO>gVHl<gCn{B2!^>bn=ceeb{f1>aQ8
z^i~z^mO6LG<?56jYyX8Mnu*0R>i-WlU*<NwEOV}2%(U*sd#5hm%gWj8n{2YIz|>A;
ziCY$@JCD)4-T>B9Uo-<hpXLydZ*?>L_}!Cf#j&*aukS3^dF)ts{FYn?lTz6K;1vGX
z=d5?X{SbNJp5s2bWEr1b4u(%VydT8|oO5Wac{e-ugma=|=bX+5n_6cV#O>eqX@Y<-
zLs{PkRjH{LJhXrH7+m)-{LfQj`Z>q(bJzyo)P0LGJ_cM=%i>ml<~;GE|J={tRA<du
za%@&y0pAv-X&oJ@b;Tyrc-nZMInVD4;587A|CoKwYHnd6)3t~{no^vrFHVq6KW33$
zu}11>!-MrI*-afyjJ~Pr;io4TpH_8$mvPr&=g0p~cd=x2XiWPpaI3OyrQrv;ciLuq
zUfr7E8RaN?HfBoj)~_7Txo@9{a82e{zEbS^dUleLX9x4-+ttc%{<>YgpmyQH>a{uN
z%g%k!-LPs}?YHNR7cL$4-TCuNYNp%%&#YHlOz$l;i#T+(z|ua8Y5&>VGKG3oADSlg
zwVqnM`E*Q2*8YyCR}WNu^f@Y@sLJu`NXeDAuhy}>_Fc~B;dkI>D*vT~{VVV4Uhg{K
z-Iv{K@kz2<cgeAhvvzHn{q+Fz&9D%@&L68jsVsh?y<MpCD(^elgiMythUym#-t*7Z
zH{HJO@O*a5$(*0P{}r3C%jC3oERk=0lxithn)9%#XYD<=q;)y-Wd6q=^o%&%SH)Fw
zf@xy>(Xuz|+#?d*O6uCh{vPGsvN~1F*0}H2jrVtN<WI{zvFYN6n<Y$p9>`v;c9aU5
zxT7~cR-(^cHE*i(zQA{??OP^IxPMOjJ3osS2TO4EgVSoVM^l6+y*|9vbmkfT!!`ak
zw|_WVM+r}ky}ihwDgR#fvG3DN&982nGB3K@Ds*~6$F1!XZXN%7=RJS&%36*kwfEWo
zy~<O4p>|Jq?X<_{|8!N$GfrMR67W5u^xvViA6`CJ;wd*&Yl*kso_tcTn*Dxe;M)1c
zv!*a}SDd{XT4M6Ur}UiF@{jG_*ECn0i!4lWv)lAQjf2xy`_S=}Jt1FH89Tz5*o1Wc
zdvqyiGqQ7w`FL!&>eR|3ZZ+kEo5RtTT#jo}5)(t#hHbMD+TyCk$k3)0zA5MBr=<)m
z%u5ff$(*b(iNjMxWuj7Q_Y}j~CnhvyYzaLPrZiD6G^%^*<ra^nc1JrtygJYmvDQe$
z;=*c%#w7VRU2a1~?VyaERVl_^k5+V;hvq3ols|}VtNeaT)Zokdcq3<bPBrI<hZ@`4
zLeoN4PnZ%NG~xa}*@()#)4DSxv*u|!82fWFIlb4>UAl!uHE7Wf7OjS5(;8E@_1rY7
z?#?~CBeU+`&!689oSVH|K*jFD4u&U(xQiN39<BI1O|YV0^7N*wZfn-cq={eHyYx=u
zeeJxmiomD;E2lDfPCPhoL+t9Rtq+V<CbKwrF><mo-PB>=^fTg|;L^aXsQ93fO^rdJ
z!BBCcgAg~@g?JwpRfmVI%w8uZH2e~r;li@Ub78AkG!Ls}NWj76SGqo1Ze7HnyUIL1
z(%{gFDG|3@mo5$p+ImoSLfq1hi)G@Xk<%tJ#e@V_z1%Bu&s}??`vr+<KUEK;vZmg=
zlo~7aWNAH%tjV+KLcN}C)0jh7rp1M<Q<>tGy~#x)AgDv_Rs*xvhRH(J%fx2QwZ8fC
z@`BphZJnK+N~<In9Vm30P!R27c(8DaTDoT8iA*a2_G5(?B-vS9pU13VD%cob|J&4J
za(<E5%^=r+S(X<=Ia~if7pfFmyCg6&(r`-by7-$+(->qmTY_eoGAg>Zn>`H|@cNxN
zZE~hPYwB%{%-rp<9*Vrb-v&M4+dV1baGOZjcAurY^+MO!gkFvM7_~HXQ*z|-uqeKj
ztT*Ry2Y$O$q{q4J(t!ram>UVq!eZACEiuVa4g0<8s8<qC^2-ei`)`SSNXr#Ud7iOs
zp_-9@OB8oV`p&aq(x-Q^mVe6EW1pT=XqrAt=A7~BD|gNw$!weBU2*Bn>+tBOUOLXA
zif2#tM=;)I+n;gk%CV{xgSW98zg^g|I7g64G1rlkuSLPnRB*?t#{VnNCfqc%y)7@m
zvE<9chRdD~D>_{Szbg0XY`b)>rt$E9_6lpmvXq?l#(XY29`x$><vnC_%@ce2O>67o
z@PwP(Q_G#J>Qth=y6OTan@y~IeW5sM=db+<%luZmZ(j6HZH`%GPk8+7H7l(h7aa||
zFt2LbGxaR5JGC|o*FRmS@p;F?V;9(X0;OudObNQB$r_w6KXdthP4<_sS(dVt-C}ZC
zP%|@<bEV}gZpSa7Q}kNh_1|oGzDw}$lao(%J(*{!9Nc_pyHHB&O8aG<OsCkCXD(iH
zI>!5>Sn!#v_H8{WdO|YO8Dv%*<S;s_73>(k+n~u}@x%!kvy{1gST>0@`?wkT2|e)o
z5j0`HNSguY?uCgw#S>Joaw-2g6&@4y!b;+hN<!(w|GcZ&ik45DxUAE5<F)J<<%XLz
zZCyKm^7<w$Gj=cZQJrn&clLmWn;sKSjj7~BuUm#ft9;ig#AQZ3KXQ2PR1@FUN0l;;
z4=nUo3byoLkL164j(L975sgCQgXbL%OjBeC^k85-Gueyd%6xX)AXVN=oS%G5*bZ%2
zuAwV6r+Wg^sef}XOz=oP!J*!<W38mOT|=;{yBhoAMQ1-1bv_Ps{BpulX=1A9nh#68
z<0AIoEbQ!9C%vft(el+-T~``yExsE2BxHA!&1Y2?K|bGi%NFuGcBtQVIlExObB#NS
ztMt0PrX0JpQqpImaq!L0a?3jox4z?Sn71_PoGtgH^NlX<YDSe}jsM*q&%U2|us`tB
z3C&O^wfMPyKGB_X9aV%@w{@(`IH(jAu64sc$YbFo7ZU-s`5i{LbCMce3#ae0&{$WX
zc}`7gZGw~Wj!Dzg0(y;PIsNjZbe5zTEWLXxGt_R!p-Mr~#s7ASM_hUokm~ftt2XB3
zfmg+on*yh{HDum6toQ7~nm+-aH&;m<(^)p*_R`G?u{N(ICu*>&^1Sd<EPj;fb!_>y
zl!>#n7WbYB5Zk=>*Yj@6!#B?+2t~B<h^gJuIpOc%bfVwLtxv!6rNa$|nVLU0s=mG1
zchw-AyZXkXsYfOqYP)i55}T>ED#NdUj(28CzC}gRwbFO)Cvo3CRQgm~UBp8xa?=0X
z2D)Jkd#CpMFo&;6GVI%N=&sbxNzo?e(}c2<9?IN*aa`b1?y`la?p)eGYnQ5arRMI{
z*;%g^UAeGx!q>9PCd*AeZc+>C*<x(sd1zh}`~8O>U;01Pxx(}J(4Jzu^J`c-?>Vee
zXJCxZKM;Jw;k<>ajF!fY9usfRb^(Wd{8u{_HX2TJ(AwesNujUSb2n#52j|7Laa+^`
z9ap-AUJ5)DlD(wKFf(M%AxDl&5zAICUg^!+ku3dMWVy{$g<Tc5)(X^Zh)_Lt-T7*>
zQ<0@we9Xh?PA!tlK5cu_7O_fkTKl6=yG4Aujy-2m_Gzchn6S0Oyl|$7`{!%hjW$LF
zACS&rDM-}*A9Zofo{5r@`H!yKwomM_+sye>;*j)Kt|=Pzuh)cTc#FAeTlJaEOr9ib
ztN%(UXV<-heAP3aP52=aRJ1Ym+Nv;B4WW(GG;#t2`y76r+`qBNc!F#mYo@h)w2s+P
z)|a`Sc}dw#0ztj&mYDX&yo`RrSFW&Hr$a~IeZIuhpyGlnPk+fh5(xXVNj2!6R&s99
znJ=o{D^J^;C|fFd_7B&BS%p=?@AwvlWx96E{^Hv^zbAc<)78>8wM+VWKcrpR-Mtv5
zM5WuRKlQWgzBq4zlE=ztC;bCg9+~g>xwp%HbE~}UIn}Zwp)dCB^@)yjomu30V(V!p
zov9V8FP4=RdUh9GDs5f;|LFs-)T5hyUEH*{Psmj9SX{m~_<wBs(_=!L0zEbE?lcKs
zDPYS#r7->MZkMeJzUjLxe@0wAaN$B*c-joUgR7@@R!&V4zrD^%_Ran450{xun9{Ii
z4oA3oy5ZT1Sjjlmdu8_@{48~NZG4dN_ln)pWf%6ozU%MyP;_#Ah?WPR)n&ErWv5o9
zDCfKooYiqO@=&Sjwz_1QhY`L%p6<)rZWHfmTh^Q1rct%gaDIYg*=@n7-`koNsy}&M
zoBbwf=cj_BTP@$2uYdCF-X)z2vXcV6#fv@^So_|RHGVCzDfa_cz@E)YpMB4GtKR*1
zPXDzm^O2<s7#MTnD?a8LFPO0WEVuspIp_XY$B8rAJoqX;gXy2$hHsDV@0~B$axgmO
zP2H!(9*&|aW*a&Woz7`I_47titHeYZmg7%P{n^9sEz;`58Y_K-_1ca=mr9oZTUt)r
zUhsYq<5;lZOsxXX+Zk(PFZ90^+4Y0VU-I^ajYqhG6y6vta5~~`+$q7q*2eo(fX}vP
z%1tlstrNmKC5<lfL@u0~r{nIhvgKB<PfX{PJE0TvuH2k^H*#KY@Qf#O<98mJ|I+;x
z$Jygsjydj`b(w*aGtpfx*8S1~&nknsr;fa{OwKBtjPsU>_I@FEQZf3~*C_U~dtO%#
zy%3d_do=NE?}RmXrPhC(_4=sP>z}vQh|bpHU9?g4Hm7d%`u`bx5k2l(6pvJfNPF|h
z^Im)KNpb$mgxk!!JmY(qOpbcSrgW}m=-IXNdR<QZ_p{f{KDz8_m0u_`E%wgsMGtPL
z=P(F4_BXe3E!z=av_d|y_2Tjw6Kxe^N_xB;U1lg}Cbr8w%>J|JKkpI2Z!RJk2mNmx
z^L!|#tQX9h6;rzKcvwtK^2JM&W-eTP^?`3gBJU+dg`j|SZzCqK-rUI`yXMP<09Eg@
zH{COruwDv{<gpArD6xd+Dc@DyC819Ru0{Hk1@iFw@<ePsH*>9vIInN#8qWp2$Lbp9
z3mux7JwZNh>zU&&f_DQKGJKqSZ=*`_m&h3f3%wZ>7`RT|%?MJjJ%0A>lXd^Uq(lcy
zGu7mLQs{fFM677<q~|}|ZTZ5_1j@;sT<DuK&n7L}Z>!v!jd5%bqmIf>55DB_@PdqW
zP+#uVI}t9|ou~HIH{4aPy8f|HTtIEUSA(oi#B_~AM@%Np4xT&x>%^qEEsN3`#J;np
z>6{Y(XmWASN-<xKcFUJazjpR4_2}RKH#~(a;=9N-!=?r9qDOx4oX|fN>F|-wRBPgi
zHE!7%j=3Ujr(_kvSg!hi^<tL2IqB$?&RTEx1B<QiEjwvCeeaQHR(g?IzN)r$ah?@N
zSq&fUtT}o<v~}WD7w&J%q%G&WA4q#OWoFz7H@>u)&TiM%)!dxEb?KQ@mHmZ@nF@*?
zheSpGJI+Zv6DjES@XVLnVvWzeBNPw6JyUbeHRIphbORP&r%V4{9x+dxG`EApeAbDE
zS@Yi&OuAPR)U?dC$?Qeq2{(D)laDtlUapXOqxv*=)x4J{r5<uF`hP)e(Z?HJ8E3su
zC>HQaE3P_{e^PE8<7_3SbDf7Sy|GQ|y|Q9?D&KQUX{VL^PCn9(3sRZyrTMOT>RS;V
zv&*GwuJr7NMU$?vR{woe*WvbElX17;dGntqYcqrQ+WK35d?fbc(G6e41FGzXauVJ(
z6KyNG4z1N(zTu^pgG76VqUk-Ax_j#0E)82$9GG6P%ya6P_et~Q7ncR|*cNVTTR16A
z<V5I6l~)V7R)|=*P5VE`J70CmqQ;{kua*@{1)Z%8ak`l{qapR(Mi;-()B;8IqAJ(y
zMMAD8y)Ps_h$#wPtvC5f#>_aE(92&Z^7zI&$4%u+oO(s2jo()#)rBoJfhTs#M6UzS
zeSaJ|uF{ZEvMi&mZL?Nl_to2RNAJcj^6}DIeOFQ2`Nn((gBjjQ$9^6-@^i}SRs$FJ
zD39GgGk5P>z5AH<o^z}B?0Wsv!R1LTpL|m4tG9{;k7N$$oqKsu@g-;Gk1vWZL?6Ar
zD0TSSte1vM3nj&a>YfJf%DVVLj@53u!yk^Vxk;}CG|TR`xK^dla9D03^kk#f+_-7d
z@oJiy0_^z#D`y>4`hNEM_u6>j)XX{mA0Dln8~@8Q?Mu(fnQHDUFUbF$>E*a0^KYx?
zW|d1ZB`WjYw9Hz`vg@9ULujs&>w)VpR@S|FdEMj2HMi~-M~O`hxjrtZMMRHAD=gUF
zB6q1`6~o@Q%dTC^{d#ZjlZ<8ea&v9{+(rEEw<Y&IUHJb`1ecWhd(GuVzn(nYxnP;!
z5w6-JA9gC2-BWm~H*d-pg&C4MZe4dhy%-C1E;^LRJF6&WifF|yJfA+}Zr&aquQtu?
z=UQtnE%&-Hf9@}LCWZt@7ss7$9^M+4+!y-Z?wIzWZB~4Yi}XHr`Bg3*ZCZt#%U{gQ
z(B2xPxHjXVf=>Qd?Jl95o=;M3>t6U+>NvOD>)!QZ!+&0-kZrSizb#XFw8FJ@sl$&;
z+>9%qt>T+FE%{B$@mpPww3IY=oo&6~<LI!IZA*f3OU~taHIsGsN$MGKB_BJc_CqP^
z?)@b)><zUmXZB9~TiS0d#G;;K;OOGum6B#Mi{JZAdrK>i%1)MR(^kLFdRXsqy7|dV
z&cg*vXEwCha@@G~p+&$&xsXkBi{q(@tcnM9&kB6Jcg<z_r)N(J<Q_LIe04VQfYJI-
zT`%6rKKJL<;r=$^=%w5VeahYyJ(iKn7fx_m*JgOdQ00)qnW>R&Qf8bkQmbbKF26MO
z>IDTaA&%qUPA=hg&lS>J=)Xtu72lE8nREYVYR{6)Y5EyntYdU;kKWn;A_jMtWOGaF
ze2lu}<-r&j(y42zBUq__aM_A)YHPa09ywGPc`N9=x$2@ddy`hQ>g%I%n%if+J!`b$
zR-#b+8|NEB_mW&Lm*hAG+&W$qEpRAxW{J2V|L4_x*H8M*TsSA`Yp8rdLr+HAI<b9=
z#O}S>b8GsKqca<={1<1qUGRE#W<|V(^y9*c4}W#vx=%d6i;Lsuj?e7=xxPL#K3J{w
zUgOo0+Tru%xc%OhKP)DR@wBGZJluYxIU<ZN|6S4hhepoIC0sYxF1z<h&{)qkebs%j
zOB)%V2PHjtHFJS`IqM_2$YP^Hf4)%lbNO0+d5Kq+M!Q4`aDM)iv?Sw?YR$p|?+JOc
z{~Ko4#cBWA(I&R?3HvWm@d=IW_QwzHd|bCC+h@AJ?Ru7mKQ4<?+JnWCzIGWoOMKBt
z-}>#@w69#6TG3n8lS>tg#D3h8kmI_&-c<j{IgJNQYo085?-abK<Ryn&i1dlbTdz-~
zZmPJhzVB7X)Ab*prvBfwW|Gm7e`Z}yQ+#%52|9MX`?hiRo=b~-?_JNg*kEq{OrOoi
zLY&1q@WaeUvddkL+i^0e7kuz?Ij(%t<*gCRoLw#UtWxtf88|b%=M3lYVOXE`&LFtG
z_eakOiHyb+8<y7x*SeU0Og_8z=+m`D`X61=wR}`kbq}+wEGlGQyS4Ovvi+?id!L+&
z-OrM6X2RNOg-icScCA}x_|8TnF4`?ZQd_2e;{-mgj1}3pmfpSpup~y+|Nqi;r%fJi
zW7>L#)AW+jCyBG`gX$Ah&9{0xY<2p=dwAE8@aaaH?L}J76*|E`)ZHyw`tNDE|LEGZ
zskOhTeRs?A#{9^`3*F8Z9g*@Y*{vTLac+|9^%H+r^tiwCy7E4{ZH-~{lM_an;-~p6
zuWg*Q<V_scw;Y@76Kk@A!Z){HdA&Q&SfPCL&xP|vY@Q_tozM5_Z!Got5~61FaHewo
zgUuZq>YlXwWjy>b=lp!86Z7`9v|RZn_^!CM<)=M=%bd$U?U@;31zFuhOk7^Coo}wU
z;C)=NeyQO4Bd+pQBHNF+?N*rZKlHoq+Djhm6^<=S{wuIca&^fe6VYd%OgZX)W-ACV
zHuExgoL(`#YRe3(Rr9O1yvf{sFjSoB=<JtMr>9F7HN5`5=ZT16%iMKl3)Q*v{JR+%
zb~$;N_I8+lJ#{DFGk^X3O`*+2;nhW*>8U+CUU;~?*WUP5%TqPx@tq~R+hqh#Z@Ia(
z=Y46f@wDFWJB(hNCQ5%xsNOy&MtI7zZyezqObm>5S9ag=;ZVByd&TG9tq+dx|GxW<
zj>leo5A)QEYvU~)OUewFefC~pELv9Ze%}#6rYn1#7ax}0arw|@+ua*{7(6b2GV^Y+
z;lKTDj&)GB&%{eS-vd=m7Km>z(w=#S|JP~d@&*4F%Xwz}wo$WMx$H*XBCS<Y>c>*c
zD?HO3gKkDlEMGTCT2$8O%1)Q*Yqsy=e(8{ZP}q0#-1}*(W?KnYS*?A2_mJVH>-yEN
zGHe`E9^d>uoh9ec(~cIY&adso6`MD&IOO%C)!(A+a&1R;#i37s?_IAAe?9B)zD=Fd
z&yVCSJ#a}z;@aEWS?AtgEPcUH93}R-H~inkT}%6fKgV2n>N`j2?S?n+TJ}}BikxUr
zyK&-3>zB_rj#x5S#MoQU@n&GS<I=si{_J%h%Y2)(hPCk)@&4ya`ggQxJ3G|8dU4-)
zp3Q9j3I9TcuJ0<iHeL5d##;O5^H28g$rC<uTw72ok~gojCBnbT<^TT0XQd5qwtKoU
zuXB5T>z>)KLcK4Xs~*@L%hH&(zx;Lr@6GzSl61q?{_DnE4jdDdTn~sfda7^)%y39_
zZs%8Yo5Eqp*w7`e%QitF)yuJ8*s-nVMxdf6qnfYNlSLDHJf<s6-ZSN=&`F=^fo5Je
zuPjSkTgLRpbWQr?6-O4jPE-r6_@WrR+-adziq_VZ{?X32W4~%Hh+8|4f3Fq~_owBD
z=C4&)RNFCa>LIgS(QT<m-56HyELFY~x=Z}giq1MwCNYH@iYwdNM3v-p96lZ~RFt>B
zBT^Xr<Ye7a6%!4HL?1@Z2@Dw;QkqTeOIRncF&%jH>e?D%g(o)(HXrBQ=6%mj^1tEI
z!&4Qrou6DOsCad8zNLLiMwDXl<Fm?(*e)5UId_~By%u6|_e;q|Q(tTGT&b;Pj1Og_
z>LnOYx;3Xnt;{VAZMe0S_wDWJE9C24{&D?ZeSSaV<*FAtOtD-B2R81J%F@{^rRB}Q
z8N11qfm`&@28S(CD_z&JvsMV`?QxR{RO92W?l53fD_-Ghq;{LdZCX@{;G$^`vjQ6=
zQ+p=58~s`JQSpeI;?ea=u}@a7ILY?Oql;xtw4di9omE20tV?bvdG|NX4qDWA;Lh45
z9bv!Do;JStWs_%6R^rm$pRYEmYAjxqb;_?}*&{E1o9{YW=a(w|+U)PpUbQM<1s`ij
z;7qQi@BeH3+9$MHc74qvlOyvDdJP;Ie_qzN4~r_itS9_6YsHmCEY?LE4}|5Gx`suD
zhF&`wCp&3lv}pF3Wv5s=)?AO57P_?Hgn@U5)mJy=^@?xK8S|N@&UMzeVOUbi%XnRH
z(p$61W*IAP-`;gMfs5~$Xv3Ymr$igLN-xKFEadR;Zip<G71E7+q0$w!r{jt<ujzzE
zv1WU##FPB)m7l#msb?F{Z2c8~EJQcl?bMw2Lw&v8JONE!$4?U++dut|(3#*fXQk+o
zCssXEf|rXJEs;u06ZDX1oKopNUDj*sT|H5w6AEgapDa7OCjEYMzsvvdi&HAgP84}B
zkzLf(Z4jDu!bjh>IqLuG33qP4JUsuIOv&qwmqo7RbRKIG30hHCl(l-@DW<?D0@YRl
zi^`l>4V32HDdGtE^De79bz1DI(xXwub~=j=OJ9<_7A+h-Cn-@{saZtVG&m#UyhuRD
zmc^e#gSN12I2vX)IqP-McFnN(y|>$WlscRB&OUjR)oQUbw&9V^rx;7d9nN~Yn<g5s
z-_t(Bb(^B*^5PY{(|nUPD%xj>%T>JEaQ#m#;~j4wyXZI5DwI0D>RtHr`<%M`->-@-
z8a0!DzqSb4_5RwG+D?-#6Otz=|2q{VARVN%=g3o4jV_KP4)uxRM^CtM9h}*x_G!`l
zzbD*z-f*uCOI<R3Yqm%f*Q{q9(-@NfYxd`a871b-TBbVRF;_Lj`MNvH0_#m@3zDQS
z9_X`6;kW$yQZvMG%8hf!)D`b_c7#M;Sy*+#SygS7pIy!(Zz)lAl}))RlekVV(GZ!a
zBc|$5_E0*`%XK<Sn@f6XPUspb^_|DhcJ-!*#5^f+6;ZT37GC;?BgsIWLASvm`^s#N
z7aEUtHY{a}x{}Pxuz|IKL50zK`$b+J(MKMPciLheZF(|8=FsM~XJ1<i-4It_-@K>U
z@ktz0^+|=+cAeKA2~$_sNNGL_p8xl&e%+hJ3w;<o>^IyIO5Qca^Pkoh?~N@vb-A-T
z7;-%~wQp!LC~V<4Cne#VaqLW^$)sg2Qjd3ZW(56Lxs&riY>|UNt4CtSVsCzh?PX5d
zs)-6(Pc9wFa-Y4@bcwl<4i{&>rSta;Mmrg=B)jcu8Vyz~&Q>)c^EMosx?T55{n0t|
zkG%``{%P?<moX+S=G~(ShHiYCw?sP6>0MqWb=50;r;w)0n-i1Ly;$6*pY}<!GuqPF
zw4`K%<Sd~Q<Jt2rrcGiJ2!Czh>A%qKNEz40<poThUQY~K6dS(qT&!WA>-0-oDf01>
zsx@7<ZC@@{2|QfoylBHr-y2MN4^ChB`6Jx!qb7%Arth*JcW&7+X9})U<P^Bo^}@U3
zU8qy^5qS^0FN(V&Pc33GdBd>x$r+WqIuX5F6<BsxF6R98BK_ijo*p(yWeMM^;6B!m
z3*<zbCTrQMFc`0$e#vOlM3KDA*0)B%Z)`42EM0gmYuVPfKD(};UuvPH!oixfpuyTj
z*=eI+a7vO;Z@0z+t`fB@=SQJm*r$2&ehUh}(-reGN$K|<)%<qR1_^5|;anxZqTD%G
zAM+#!P2ABlOIVZdS;8#2ng1?U?o0OF{Nbp6kQ}e=%En$LMj3|g3~iN3Q&iYxE=hMy
zWQ#jq>AA;i<(19{!AuLf84jw<S)Z=UQnF@AUqo`{Lbi)V3(EKAMt4uz*mE;wnlPt#
zyUZ&AHbrA4)y<x#E?kLYHQP4JcHcAatgVsLBQ9+AE$U!7CAsveD*IgHko5nqA`4h_
zOI^IJeP!cQPModON{Mx!dm>!DCQ08_!FzFo|1Ad5ZH>Q5)y{~14zXy9X-aa{E_|&W
zr|NoL*t6wWp`?t5$nWVUdS)tZoSLhhq>Mkgy$jt}CAl%GYq777<T90;G9kOG=Df13
zwBB$qOdx5qV57~XXs4wruaBinDh<^w*ZOGWIz@fy+F3yi&pN*r7x;#D%wRwBWtCRo
z0+vbK-+kOROgTUIdvdtQYSVSqr5v+go|5117wySi^&&?3)f*1R%W4My?mK%=Gm=(L
zy`%N$V}GvFllBP<H*tp~9}Us)Yq#N4R?IZwozK&tVV=vprd1^<?5v*l;R_0k;RQTL
z|Ihi58?{bvt%bMlp|Tk3pJ6*r$*%KwVsqq)-^UK`Ke<Qm{t+;>=Gw(R_e9D5+}M=&
zmW~UArpB85iruGkORdd2dC}byfiaHn?sR$w7FKOz<v(;qVY_1ylY{MK-}~jJ$9IO+
z-Mt$B;$itKMs5ZH7M%w<xlbkvO-xadZc5+_U_7|h^~Vv3I}<oGB$Wgio=*C_d;N+e
zE3aK{V#RmfCHU~`tg6XBC&pvk^&^5qS3=5rxzIfCqwXmmPtLn>r)uU~Zqp@46*jW=
zy#8!>^swsmR@a<f$M;LWITfoduK#YLdHAxZ)!I`wvg%H2SV~*7CAM--FmjLhe{K<r
z%{({N%N}jNg#OofExEqgy-P$QPM-7ShDXQB|Hb+2ThV?&UAtg)QqR{c`+ZuscW1eB
z$DZPy=um&f%~9%%L_yv^IpNiRUc~#HxwQN9kJmPC(KEg*Y1J!PzA}t!`%NZKAtf^p
z_XCp_|M+v_-nM1SbeOKBBwvwgx}w&^<dUQsb#_G@Z~2+LawpHon%sR!POsQBZ)`cf
z`oX($Hnyow>Y93ICaWX}^DI`?IV#4te%hT=M=o@+6@I8Sal7@Z>sG~ukN3FFZ_4L*
zn)OsoX_pqiLtIbLp59$f{Vo%n?A;C?%ivR4m$Q7shx=<j-05o(d3OG5m(IN-0r!*+
zDV*q>6QHfsb3Ah1`fqG&GJ{_HpKIZ*;`3;0h(C9))`NNPe=@AMw@7KXT6eZddp%QN
zCx@&5j1GT`bC31b?O5P+RwhuOzx%dC{w}pAT^oFFb8&F|RQr>b^oMQj#x&NiC3$<_
zycc=TFIu~7t5lQ1vIkr{+pDKIFih6rC^##Ye29VdfSCE&gNs&`i!9nQr#<RHpG0x5
zWfi-%9|woi+W#j6l=eMkJbTpq+`6PkPv0)ex^Yce$&KUsmBmx<9ktpLczDrb8Rt%x
zwdxF88+cPxT#_{HY92bN9BIf|^!e7)<S?HZUjkL0rKu_8uXA9zrID|2{?+<Dhn%Jb
z%&=<tY-4y_>BSw7^%w4SJdaA@Y4PL!)c<u8i@NgsW$6VhVtc#h^znIJ(Nt-fxTWUe
Zk9j?rQxdkat<%seP}|nS&BVlD4FF+lJ%a!M

literal 0
HcmV?d00001

diff --git a/tensorflow/lite/g3doc/performance/images/iosdebug.png b/tensorflow/lite/g3doc/performance/images/iosdebug.png
new file mode 100644
index 0000000000000000000000000000000000000000..8cebbb84688b8129c149108ee4a47736a23dddff
GIT binary patch
literal 17762
zcmeAS@N?(olHy`uVBq!ia0y~yU@T`~VDRN&V_;y24O3}kVBnF@baoENOipHC@aUX6
zojoG>q*%lAX(}ozTO3%HwA^!f8Jd;B$g+gX%t^>YLs5lA(0bE~15p7R6I{bxbFvvz
zHYVzKC%ot9);P5*V#C5uYMcvJ->-gO{rvZ{d-mz)=2$+zvvaaF!;0etM;_&bFgM5q
z?o%-{nP91C+3H!vBH+LvvVhyiB4(w9f|{C;gXFIN!T0VqTnkut>vZkkJOATdCm)|)
z#ORQuu>9kaJ2EHO84ujuk>qK|&?2|OeD*Z`HlLP2rd&=QflWK-7*Bbl_}oO*T`T8N
za^!?y#(;%I7L#}wt~@C;<lZ59p+AZ>g<a5Jqv;OM&FR_?J}A_;*=VV&i*MRxn(}(f
z`#&jv4yWJR;LoR?JY(uk4Hs1p4bk-CW&h_t{c(TNWL@c5hk49aIS7eqpO1{&zUB^x
zWw7FW;iayw&exi1d`|@5IK9(bf!kYR%l`vYHzk~9U}l(d>M@I}gIPn4)~UWOnJd4S
zaXa6bkofjuMWUT`oTb=P&!S}Zzl&RIG7Ihq6svUYd}5ZmX-!Lm<9unBnr{=%FH>@?
znPlX*Wv#;u3D!v7Pqvw1F-I1h{j9{Oopr)Zse9^ip81w+msAxudP$$LakrWx{!CS1
z(_%JDqj&-NWis8q4~s-LzH~bG#9Q5k@94?L?5x_K4|V-q)N=60Mz6xUgf|QRh832b
zSzvqZ%z3`4uYDD_dvQN%JNE6_;iJpWh_gTXGovCkXGfWux{aG8a~`)6BR319+JuAC
z*oqw01h#!M`Enrs)I5O$j6xF_e;lX_Fg8*0yr|rC-6DYHU(*rxOAiYhyW;FL8h+^?
zj(U1kZi#jkw}|p42k|E|0*sp;=Fd!;l2NhE<pWDri~ZSitsB!0tG)m2c|1h^l3;VE
z;}L@-j*D#9kJlFF+X%(|O)NfSTX25ij(s<UcX7OJac_Cu!guKH%0o#f`6n75w79^S
z%p-Y5@9pn51>P^Nit-<+T~Pl*I{m?~19Ih!#;gzO3i85N{Iv_aS#a!rmgS4?Hx6zm
zl~4Rjc`dITusO6?_M6iBE8n*U9GhwuqkFC3e1GMJeXHg9?<v=9IU<(CzG3qX&NnvK
zdQ2JA%6PJm=q0*8vDsrbPvhFL3ppFRDmE>Uf9U_YknQ=DaNkz}&o8|FP`QEs$!G4*
zPxZOA!);=f&Imj)W`2-&Hhq)por{ZS`0@1>h);e0_te^_yv`e!EBA2UFR47v_(7-n
zR8PJY-<@W@az=;x@4o5Zr1X9-Wsu2yd`gAYOIC0HiT{CD9?o9G-t*vIDz9-7Q{DmT
zAI%8@e2W}yJlKL0=CT&d^*Uf;Abf^-=>i!YR_%jP8w7J0tq<gFkT+>aD`1+#DDEh|
zgYVs;t(Q3C8sa|ieNpCca?0S4z3AK{$bE#b(8)$1UPYqQC23-cis(rPmkyqpooC!n
zP4r69-6Xlvwa%SSS$)E159=c7%8nA{^%MOy+JXXeuLx{)Ile$HgF&kC*y70-c})0q
zoxCp`eUY_=?QZL}#p)NjUzBcPu5J95$Pj!~Vg_ege`SJ>j+FPoONN|g(%OetZIH_0
zzs55EOXwQr*9Yw$xy<P^F09@WclZO#obKljH-F@+(cQ<q{<#0c@Q>{k`v0anC`$;u
z;fP}~OJq`PHs}rTTA`GqETc51?Y*D+L)nQQDFSOd-hH_1t(GQ~E}-4<y0g0Tc<0+A
z$B)<vbuMw=r2KQjB2SqhyOmS6v~B4w=_zTw;`wXAqE$(29xV}CG-;7ikY?D*pqnd;
zLY{V=3Z7cGiS1_78`nOy&nGUQ^ga=N()Y>fCw@itu9K5oKQ25HY^3>FW3^_r#&XT?
z8u=lip}wKEp~qKqg+5=kF6{2|D!;D_U%9UmyE^UEBELnaR&=ktz4G|V?JM&$OlGN`
zoj=X#^m5&0S1<WplDlBH;^I}e)f=x?WEEdob@kQNbu0FUsIQJ*!M?&jP(NsWko|(X
z$i42(YZ6x;y!0UB%(WHQEY=hTFZQ3j?&S)}wKEq-F5DUXdCxOr-AU7yhHVh@U+t3J
zea&<2@sgfAxAcoUzx@2N_RHljez}XbO*M<Q4&EJ8y}fL8+4{a|ecgVG{1*CcJtsJq
zkL}oolns(MdsKV=_DJ{mEAzOs^gQT~FiAS!p{;!Hb>Ee**Pp(g_VV4;z3YB^_OBFQ
zGkxiHpY@LK_t~w>Uu_>(KkFac-^;&jmAATA^*`y;k=|MVM3(u=folra650-mHG8ww
zwz;(}Y)d+9+NRE1-6P@l&uylgc2A#Fsr2RE+TPY<+CBY}_UW$-m}lHPbM;K{nK)tQ
z$p*qRPo_-{n_L&HeVOHQ(&fF&zMs`dlS_M<rhO*rOx4-mv&UziH(77IKiP5HhYbP|
zYc^SAf4q0;l+nqe#?<Dg#@eyk+d?OX&b?}zJ!|Xy(s`x5UrnVK^|t#?J6ko|Ec(i|
zt6>Szxz#uMwia&|zO{CHw!EvvQ%f(CRcTjzt5?-F)Hc@6{d@Rg&&7$0yWP!u>MW1d
z_?()NntZPB=c@;=CcNEn_2Ih3(~@^2Z%m&2I9okheO>U@%ckkGiaS2bJa>B@u2-g4
zGo4p&d(6vqk7LAlY}-*+SX@4N$J@f!W#;AnrSZ?}E<4{=KDPMK<2mXFjV~1UrEh-j
zu5Z4+X8qr|`s&-&>6P!l)R|q&(y4!QZDW6}PpyUB>{_dunfq4mn_Xl5$LEhlooQYF
zpR<1||K0vGpX~-q3fmj53dt7&8k{yFKju9>I_s#FTnlR}+f>%AY}u^pEbDpe1d^mC
z2?mK@l71y+rTnErLQzL4C-R2xHpSVe*QI`M+ueSi%U;y6x1&2k`lH4rvml|HCVaBu
zc6mQ{R0oO$E?Uu+A#k=YPOjGen3kIUv;L1qJ{mq=eZ2gr)v<Fuw$kqXv!%8hzl*I&
z_%rFpZq93*n_8y1d_CgXxwZ5C<fchlldaTxW*TH4I-2q~C9x=(OH<V_HQMX`kq4dH
z>ScaKb3?3N9!pt$vUT!#|BLe{**-lH@^$5$mFz*BD?L|Q2iq4fvNWBu&!p_ut}StT
z-MhPfH$6Weeem_O-ygJp`d_ZSq<wjQ>W>ti^Do4HaIRvpb(&VN_Csh|#Is{(zDHh+
zyuF!i)8tLxpRIVZ<e7|EX6GiC<wd<k+{L$#mOTn8be|V~@6?_+&z6+gA`|_lnLW!s
z-2cYgDEp@VrgqkStXEm(m#O+$2ds$H(Ar~ix9r!Vm;T*OU3Ie?&p$YqF*DPs_}-cJ
z%}i;z^Lo$t&Aw}NJjd_koO3%D7MGt{eCG75@97(z3sXPd{^|L7wil20vuQT}RKHB$
z6|^<T^W|CAd)#JvN5r|rx9JtdZArU$-6-y6Yf<~BRiDm_-s$4rZ}o52uj`FRUv@`N
z-=TYR!;~E%TfQ7K+MjBzWvbn+C9f~PeNP>0vFpvQ?b^%N*~C8$+FB@@IyEQs_^T6F
zd$Z46*NOdHxi;bL*H;H#?S0L2O*iU$oo)W#sl5NBzvY<RiP-n?vf9IC(X*Y+uV0V2
z7Lu!&r+MSjR@1`QH}Bijt-iZ#|Js_{)88(7D{GxpK4(?w`n7j6HxzryMyaoUea|#A
z{Pw#AcYCYfe9yUe`bP9V{x_4~R{yJh_1^rO`<r~r2Zage9lM)f^IhbhdR6)B<=55H
zEWEP4Cwt$vALid~QfV&xEBTB3?b*JwR`<V<i7c8@wB^Z*0~;45*B(E5yxK2iPQ@Lg
ztxw;b<WyfjSE_#1*ODVIi_@LX&9RX#_;$l`e(~Aod(X-5dr-LW;iB6|e|fK)&iD7_
zujdQHwqN^o`rUNt+cvjn-8#Nae($65r$??XSe?FZ<NDY=a;1C!x*h6`Hdnv<X2;Rp
zU*D8{dmCQP_3q34xz+sz>)-4<_IDLK8-Lcn)X(ep%I3+s%dy$+`J(ae;@#tSj_c3!
zm|ycw=jEc(P+QZwsNZK^Mi!TQl;8f`dEPp{BmT?xDgT!KyuNGtHGTenZ~uQ^dXIDO
z-1RE!+2X?YT&&*wckSuh)Ar5S$Ns+P-q-#5zZZN@Jnr7#pKh;OTXoC+t?|pxuk_90
zXYF&V*=s(**g)Z#heek^BafMht!%^f+l&iT59ha=F=z-+O*d{;IQ4l+(}DV=r$2oa
z^B8izvnwR(F--g*oywdeHeuxlz5guz-`Q7e{#+m}lDTX1<V)hB^CCFp+1Tz$Fgau~
zJ?KtJSWy4>z=7#&4Q||KkU1YbkMpULeRH0a^73uxray51X`XSfQ0oO(&~63>28Cpg
zAYTTCDm4a%h86~fUknTk4KElNN(~qoUL`OvSj}Ky5HFasE6|34fq^Z_+ueoXKL{?^
zyL>VO0|RG)M`SSrgP1A^GkON8d|+T;U@!6Xb!C6VBf_Gfb~e}~nSsIlzNd?0NX4zU
zbIWIl9KF_l|J_N;Tt6L2R~8nQ;zKeAWDE<Bc2u3Uj4@W?TC^fY)KTEfq9g|g)}yOu
zba?yCE>AC0e{<yBy7&E&MiqZc_OAHy+PMFWUwd%q-TmwAN>}x%v<SR-BboB3;}+|L
zo`x;EdoNzR_;dXUw<Uk>Jvi~d{(aV+yW3v>;rDD4xZl@$EL1_dY`dwImDT>13kTjb
zD9bxsIvV?2Tu^Gsq6cRjUlrL%tl52ANmh1ltiq~&jk8=Aq}E;jw`<4yLJpO9_a*b3
z55Bo~_t=XUFa9tsDm$5fFI>D_R#tYdeM`LYoS(;e>s~%u$J`*$lw#y*HP<cl>fPMF
zzH{+HoF1!$?q9p@p(S)_{e`@C&kGmzFB=|jIeDQhX?^!@J|_;Pi5#m|C2hWGAocaM
zll$}S3Z4s866%(E1qUf?UiyFE)|~vi%cl#>NS&H&#o6~Z>zKy&-UF}8H}aiK5eg0c
zShe@WraimQUcC6zJK2bnXH%P_P;}M1rZu~^ulg1CpC#pzRLvHr#m)MM<5-`WZFJs0
zxzuB6P{i7>*c+uKw-^*1t|}~U+Vpa<XG#d?1OsW6OHvxM?U{TXyxo#qo3EvnZK*sT
zdv?i+wX4<V1~M4TSo7rBv!u;`F0C{Xt1+5;)R*m?<==`hotTD#sh2i;e>98MTq(A1
zw*CA|wd?_p%HI`CNwGOJYwBuF^-DjBxi#~{n~wgt&8jV#opSEXOp{f0eRd19<@f%z
z+v{)994E_td)KYH(7SVLk9pqc$+c%#tie^byLS5i)iXtUdlik=?yI<&a7_L~vg!L>
zzE?xI@BdF{Jo@{~(i`u8$X}b@ckZ3~TF(@diKRz9gq#{~G8`{4yPzAM_R6VstMBb^
zZ!XVrQMp=Xwq@sz@b&-tE}z@oH6`sx(iG3-roZG%`M%$?os+abSG;<q&K&#pm#=l_
zRDBZMW0)&cBD1A=M|x84o?l-M&ndb7L-vU6j(tt-?{`o3-yh$3{h@-4XYn7`>BYA{
zcz?L*vwx+I@$#O>^Pby1nWA{zXhE@QzJE3I<9qFYA3nMlv%T$Pe(llB=kpvFo<DNv
zXK_XT+}2>Wxj))Fe%H@l60`rG`nsJj<CmCRerER8@PWm;oxha2YOM1wm*^bXEj<6}
zir4Nh9<AH#uGp}JNz>J7+NS%yMm%pa7Swk(W<(Xd+vgVd&oMD}a>C?p+ocW5a*J;K
z`S*Lz*`v0TvlLfW{J1_@`DLVB!p}dFcPg((q|Gm3a_)QM=3A=#Z%>Sl8uLq+DZc{d
zG##>G<J~*2Iq1Pg$rBn}lP1;LUYsH#eSPg>%Y~0Nui`MuxZNG7cI=1X_m3|prODMk
zOZ`=*^<ekR^~Jwql{K_(cE3Jn&9X3c#l0Q#ekF*6E6guj+BzepGeg5eY4HR<myDdg
z$?-96iR=0!R&@36|1)n>ZU2*|&3((xyia&yy!hnH8<w{PbOW!gi*<K13XYlNuQXB7
z-rT|Fj*FDCWgS09@3CK2DXZDnANHutwcX^nNb!bGTxNAqtlBE`yaje52OpkYx@V2V
z;w2lu{I2Y&{l)6_?bW$J_Adz^F1)tXQ2lUY(_!O-yRx)azc#!XGiCbr!+E9?yV^5*
zR17mDvtCOV{`{O79&|yi*?4Bj9G6?CK3Km`d7otedV1a2$BW~h%suzJ;OkBH$(~17
z-ih$^_EO!yq9tZNw?IaccQmJ|b8e{WK@aYE4t7UbEl*5p2{68LHAFb4FR7eeLsRdB
z8xKcsfU~UWSI0Ge`bS+`BhIo3G%Y(8wd!?Kgp8b*ch|-lCFPUO&1YBKq&IWRtf?8z
zI)ypuGY{XAp0O(+HFEDU)!Lqz8@Eps{|{Wz!=d7-v^M3gW$saVqXk(el^NIf3Cmis
zY*7Cd{A5nY#SDe0%}u-Y<WH8K?@75|Ru}g7xpSVxcTxR)A6T;Y)pWmHZDdn&omZhz
z#k1j*w7$(rWq*IB)xC3M>^D9WSe_#uFS07;=ox)6<1hOP=TvyeGx*tj=MuTo(IDD#
zXt#cU>XG8f230ag4rw{B+td15U-roH|8<j=FRSg))^A@Hz%hHjZ1OyL_pir;EB`M(
z?>V#XiQVef_I&w6Er$Oey^x;w=t7y9+r3KW%XaaN_79FFnNQqMlJ?5G|I<_M^8pLD
zKA&gZrm$S8r|N;(o!{}NOpiY)emm27MTm+>clT{Ko8HW`K5}xlf|IYiZ&5sXA^P3Z
zro~pD{+uXW6T5vz_4_@npJ#{{tgo3~9<EaP;hy!1A5~W7_rGc!oZ@07xBPym_@@)G
z*Ms*zNY>j|Yf{!OtSPlDZe!tbXaAymhI?->@%2q(PknnTGyKu=H3yEb`JNuIdE=xI
ztuw3r?N9BF*I`@uQ1`_N-tG5K-mB~M{oi<KdH%jbKRn-0oaQ)VPfX3z-Z^#~*p2Vc
zJn(t)z2h$mg!-3Q=DxTZQ}Rl1&7W_<a`nGdqg13n78tkl?CQwYx9u(EwtM=~@bOCC
z<$|jothlWIu#4OOuk*c(l9D=Y&DI<Z6{hX1yI4||=?2Z*8s&WT!O!<0dT)~#?2S)L
z4LGcD`Prj;&tl~~Cv}>wfAI15`v+~W+l${FogNf_qp|Mp=|E%6MZCTmS=-E~d#Yw~
zxCWSbs4JPYAI^C-#e0)a!CQYD+i#(Ma%Jl8{p)96?tgW*ZPyLHe4U3Lwu;4y%vEK#
zliE)nd%)dZ^lIbZ74d(S{rjpqll6AYyuN(TIbCs?v)+s6J$YVQx@7U$mP=-NRSz_m
z^~iVIh8?}Vd78n8J&JQ`4hhQC{tE4>dcs!pM(FIqs!Ls`{5RiEJN10^j^ZcAJ~Iva
z{@xLqDxDSBsvz4b(5=$AX1DU4D?Ta{-Bw68ZRP!9@l8j~f8L3Wwp||DY<l}&3NG`V
zcgA%2_lJl3?Olqe7QU4|{PM<1{r(=Avj_g(67Dy>DAPHqi$&7>mFAk*{=`#T%qzcL
zjQ;uaFy|iIYQ<TL0>WjVUGdI;y475*I3``&_+I_&6loo9b;XIE$0N372y%=6JHnpt
z`?P)P*OJ%wrap~7|KRJ+)jMBpW0y2fV@bX8`PG&09xH;ft-VZgBTlu1-+S`m^SVbn
z3(p(<Xb4i<Ycg|H8;6sG<@Zg}lE%M0&)HVxr`6d_k1#s>e6#)Hqk-p``+FYDtB-3t
zt|7>M%_#E_i_YA>gf}~$@AxU-Ji|6u>HhjvnXgYrxIZ^nW?2&28Nzm{%f*1N|9t4o
zz=ab}q*Q;Ep3SPfFJ#76FXhEfT9f}~<XRkEa<=*%Q{%yu#mUpxm1va+7fAm2@%Uzq
zK{B5+pSIhiZB_F;E{V;Zwq#9)_w0RMSJuY+?b%@ce119C(w~0kc0XWBtGAK%y&bnv
zg=P0Q+3RcL{k#v}n^XSoq{uT%%b#~XSEm2snYMO4bHaXR{$~gN?n<8i=*x@S%^^z7
z7hfKITDGu~>B-NSB~6YFPLo8Y=|>#0tL1xiMETc4lS8ckcXXVW`>XOhVbYRo+xym@
z(%8iN?!zz3r&|t|<c0HOl)T@mpTFm~W^(<%rd3^BBGKk4%@ZBs_|m%P9y-M2v*}7z
zTxUiyuXLDs@yXuj7dI~H=wnqc{!sUR!-w6eU#umHtiC;5vY1cB{`#>4&c8oC`(zRo
zz2m(~XT_R6PyOEGUfiOq%?gfG9=0q^D7UNnW;;u%aNlg}$*E}^M@5}Vziv^PnlstU
zs5f-=%HFn{5gQv;ZWT4TY9`Fzn(3EWa<5uHt>&Al@bo(Gx35e&iql!xc9p(9_eK9+
z!kHE7FR$&jc9*TyDbn(Ab>(HtWX|`W?PVmTG_`N@zI8L3cZ+WQx^&Ir&$DklDn9>e
zhE8hjJ5Sf7%_W?T3riQ6F6B^BxzzPz^6u1@1MhU6_HgSh3%9nPa7<?J6o0$fCm;M1
zU$nya$mN@giN1k?PH%#$RU|#KGuGPHE?l|dRv**5uTLgUUiI+x`)LIZ)AziQzbdkM
zdA*C;dTCcS$%%o1Q?EVN;EM86IBDe6u;=GOCB>6$+jtVX=bllRW+>@<<gw?aRSb(=
zROB2pdsPILe7&_=xFlK~f~WYh=Zo%s6BzaC`lkoI<^sAePrp1a)21BdoEwl+>A`s;
zZmO!Mo6`b~wf*J33({C79c5T?+eU}CE#BPv-Sw?2+b##6^8Nm&yC}P5cm8hS<yYJ8
zrnm_``2OqVm6gJLlU*uz7sdR1TR3B>Zhm6J!50f-E7iS<-ekPJeert%f8N%K$K~s0
zxcu8EQ~RK9M)KAxW{s;nWeV;{+5}x{eP92h@&B25=cdQ&K6@v9?pTsZU`&+Xa=)W1
zX1Pt5t9^NM<pN&)1lhUw+XW^BEnTyNIZpac&5FkL*SPP+FI$)q@MhYMcM8m(uW>HZ
zzii-fGj89iZFjle)F1Ge;-Yr-z^A3%D<ih`RL-wG)AL_Pb^ijM<#+w80?+sGKi^Pw
zRp?&bbJ<yoJtu5xQe)Y&N_y6^iT4*;zq=t6H^*RN=<4uKpS&N>+`mn(a<`*Xyl1+f
z#U^q6{yqKM)6b<JZYw=A<Cl&3y(($%>!tyBR<iGNVb%AaablL~>X{Y~Ri`bpd2@Lo
zLukhB7vIyYPux=4_DiY7LD=ZGT&2S5x;dwV{rwZZsb8+FxU!$~XpqMw#`e2szJGrH
z@om58nc08s;&mgMRJ?B*<$g_@*yo<PqQCe=VYB6%20513W)nQpU2i9COZGGS@Oe_I
zO31~kJl^m;McG}?KK?P*|M>aS)tYbTRG!uDp0e-v``*Be$lKc^ey<MO_QRBeaZ<{W
z*I$Ef?GzWESIG4Fe)ZgZt*jfnug7_;l-RVrqwe=tg=-u7RwZ!UOmN`f^}T!|_j+s1
zbQhn=FE6i`4^wqJI!kIwNK;$Fq<ifAPZreO?QD7Fk!lhkT=VIh%f6V9lN+1QCtcY1
zH)GX0w%PrAcRjl-&~<rhxcsfBkt-tt8!x#T|0*fD)s%BSc0$mmhrI`P94VjL#9e(_
z=4PPU@rj~k+g%_1)^dz$ow7?aYT4U28K39vKKbZCUf#olx7Qs_GFl|*Q*!&e!Mfka
zZ$1%Ww<vxgX=C+IFm3O{Wh*SEZ}Ll(;59PyZ)mXZR5@Mo*Yd%pNBS=h@|wpSOin&#
z_<L@r$kuH4`L(-~jC0esT|BpR`#taAUv=;PJvY!%Q}#M`<be18lM{m3A8hx&Q+&GA
z#-Pe(`TmdjK79<=?-%j^7J0tu&{OsB8`1NNgyQlUP9`&_bTR#$F=yTzeKzBNO-XCm
znD6pgbpFm?n|RAQfAgc|HoLyvnSZLwVwGCqiJ9rfsqfD6?EaAB<>mdM^p5oW^L;s6
zCd`j-xI2gUbcy(u^b0qSfBMsLyYlqIr5fiQ3oGS>CPZIpef&*7?Td1`phm^S27Uhh
z>sdEnK7Muk{gZdM+n+d^KA-W>5l-j-Co|himVdH8t*LS9;0>z{8KspR6~?)nbS7TR
zQ8qUhpR5?|%`Ru&@o#?p={J_~#fMeZ3l4f_&ouhwxJ)YL&Cb+6$v-k9eAH?xStESe
zY+Uc|c(jST;@#BiKYle|H<%^MWA(p?%iFr>&_{Now@(9W?9O@~NME|Zpy6W5LY4g!
z%#Zmep6kivxOqm?x$VNrJ+{wd%T)XAE&HU~&mT=NxRm`M;nUu8_amKT=Wa_4-qN!s
z)w+BAKLrC1gPi)A&s@qrU1HuGI?X(Xv-OO|&F$B=Ic2@}@ySwitxXnKdiAN&>@zHX
z7;~nrS+Vk^O6b+ADJ?#Ed3-B-y6(Sr^gQB`VqzmFez8Czam&NP2!UBnwYjDjTf7wX
z#hYgxdh;Q{^zDxrFZHO1{fV|8mo#0LU9e?~K=Er8HAR!Aucj@BbS-S^xIPz6Q1x(`
z<(VZEx1OET(L$!}TJiRl*?}s?a@%DWu56h-d&ADZE?VLrFD)s*`M%@v%b;1O&uAuB
zX4cIxkFTAkw0-8Y1Y6#NX)*g4!ny@-rW{Q*yCA~z@WaoITW-zsR9&%drK?9;QIX9O
zW39M;_3UtO<Au+}_B7h~D7o6KefV+5M1do-ny#3&8AQwx5fw`?Jed;2IV;KYkDb0-
zq>^aAz<G{6a_7Yw`*R$ACrC`%ZhPs#aWnm&w=Z01IR5xx&b^;o(nCEj%j6w=`*jP~
z9tE*XwWX3Ds_$mZ6#4qKRK=L7-=2N`ZN^z%CPK#^eMm5r6mU|#a>2o3+JT4PCJ28>
zoS<T~l7*MIH)E3E{Ohi6qB~;sQcPw!_nNpy8QhL}@{}z~JXFZ#u%eW6qE8t6PD{%O
zpQ=ZKbM5)J1|E!@9J)$5>2ts$1|7E@IrYqcPpinyUA$?9nf9C6@6MM<wS8}t?l)!<
zdemOn@jl+g>Q~^BPmTZct#8HL`OcE`_4SVD^TR(sJ!Y}Aj{9|%a!lFZw=s3+PX1Y$
znkN28j)!ey)LILfUn$D6bMJfSiulc55g;mf;?guvPp1^tds!_WK^y)qnBl?w&cemu
zO-GLU>+fB=_Z8g8KQJ@=T=V(;r*4_P&h*s$@u@2#KI+GFv2XDQa?FG~T{c9mmEftr
z8ua4DA4`Fh$5%r`owl%L<tWWNSGVM?%(7K%jGh-ZFM0ds!qLCxN#(y@>|38-KKr6s
z>c-ohZi_?0!~ZX;c5`=sK7FF9hnm7ngGpYSDt|KGD0SJy*m!+`;Twk5drxzx{|XZS
zZaZnMng|11bK{&lF|(X}O-xKu4qKI`&+R++U*1!|U)o}Ma6`o}@q10tdrc>?#LZ6F
zU)1?8_U(+@1%EumzuOAxGQaeAJ$+5r>_(e@*8P_|mSw(pwQlj}+b=K1IXe9ez3sKB
zC&z%P(Bp90)9}fE=KT&3|86S~xShjpKYxnw?RcAG_6Exr_w}7Sck|{Nk&9cbq<{RA
z{u(R(-IlXi^5lI%!KmJjoI7`~fyTOoCNUOSH68tWyX?BN;&%lTR-xVn%gVQ_f`-x9
zJlUcddba)lzi!u#*C54zFKm{10b=l8JR0ln?*2T#L+_oHm6cV!b4lTK(BPc(qVC;&
zeSPQd3)r3msaSsY=YJN(e}=;BrG?iSH%Ts^bW-JIiBxFrVV%8JR@DMR6GBw?t=&E`
zq%L-qH*ECm%B;-`axWZ>WmjaG=)n>iy76X?MDPB0n<l7m*3N3_J13^XIfdoS`jAX7
zu1za*qv~&G?Oyv{inB2?%yQY}2RGKG_2f=53lcAvope%V)v8Bdt0q2;H|JTrB>2ku
zSns&seG<OQ?#8mn&b8-Nn|weaTS8^y(&zKeUf&pbb^9-Mg~`_*EEbiLt&#kul`&g&
z`&Y(L*`SpoVXMD}b6Yno%3Aa5ld+VR*!#TCr6vDDI41h;n;7VL^UFmQqpY%(xtkr8
z)~0@5kp>%NbH1H=;@;w;ph31(cTU{enr&XMl`R#p#lC;FqvuwG|DVsV;5k~h#s2D{
z>&14-$p+t~`}SP56nPrHJIQRaU-Cwah_4r;kIZ)7|MSkmTVMCsmU3-=_SeHA<n;5;
z`;_PTZ7z7Xe9y;!I+IS#U#|O2B`Zw->1=ti#o_<+mDhTvY{{AQW{cXyEZJjL%QiP^
z|C+5)c6v!j=ay@~GVkh5y0y*!fAx*UNB35JEuT|$%9F?HtI0X5&pziY{+g`Y@zqmG
z>3?_YR*|&24}6!?>;88~r$WZbbfV(7AI^Unp*;Dx-I+(crGMo9?XUZsqIUS!Ok=_L
z`onEUyI0P%Z;vnLP?dbRczgWQ=Ucz7ZOu-+^x6NzsoTbxekvSc`ssJP-(L{%-SlVq
z&zes^%T+>e?D(O5Z_Y<a&e+U<>1%%cO1?MqqhaRWQ~&RCD6Dnxn&`1wNK^IpZ221(
z9sUb3Z(@1=E_=xZ)rn_Pw)?K05ae+A3*%KY#mQNU3xDjWzq0s?#^=RZQfaHD*IORm
z#aqhqiK{Tu_sDE#6PI8SQPGA;33qRm9tqNNRegJcdq*_87t6w3Qf6V-6SwWo43BHO
zo1`SeHR+&7-+gAM#T)nW-F$U$k;%trPo{HN|Gv07z1pkj?~0Wc(%06j3;9osj&Le;
z<P&1@UE+Cs@tSoJUNI9j-(QfMewT^EM7Q<Ysx>oH=9IoJON+j}smzs?cOS#4M0cT#
zb;*C%@PGLJqI><4yZUcbGj42X$`n@H;IB2ktdZIC%KD^_heCNQ9v1aUT$uc#Ek961
z)OUu(uO9E4`mAM|2hYruH+s38WA5&f<oDlPXYdI5@@@^e&braUX38{?WW%?M<}0zB
z?JW(tZkn`dw}z(Pp)|ur0Tb5OE7o!+ZMdPSsd+HZbfTXNhwEgIsI~+Rrv(cnwzIC%
zT799%%*)$*!_3~Bf2GT}D9ZK=@TgC4nN?uo8TtF0glpTYRbGoe@-*&Y;wt*PyJwL=
zR@<#rUQH9lY#M92xQjepXLGntej@rL*~r6${k3PJn*IZ2!Q4fwR!!?M2ndL2x}WPF
ze96SD&ESMk??HvIaz0l}y{uKMJ5)`(yu3eJ<sO+e+arr<VxMqM*P~C*gJ(Qfx1P6r
zn_BM4ZFkqNeD9;>R~mZN?5RlBtYgR2bjpkm{$dm95t%yW(Uw^!RFYbfJkPK>I@by;
zKI`l0srbySN8+X|$HdZ;ORlcTzpLOab35|chHq@k^g<%T))#upD~o?K`*f#XL`#{;
zbCQR4WVv7TM)z*(hKo-Y)E?I>$$NkQ&vuW$UhnpO<GLkZsaYOp*Kx7@|75P6*8X>@
zIoI2j^6UH0=bz;kchk<s<Wu3d_hDx)<=Z`Ywp(tgq2dXZM{nvMx!P1+53TufC}LW+
z?2hMsiYHcbESB5BoBzN2#ms**`|aj>cYjQmJ5l=ShOW5r$Ijc8smGF-%x1P6Jv>+W
zn8Aj{6HB~X4EgVPd24uVQFVNutoNXkJ@3%%SvBF4RJNoDC(G|@=6wG?BRYCgR_3uu
zGy0C_9lP+DrKq<3)7!)MgDfBO&JmAqKe^}g!O)3$4-apTPky+|D$Kuvt$Z14;AheM
z>+hWSbp3zt)fI(B^74~dHaRYP)XhILly$lMovGLDXCB{mOhs9~epbD0h4`soKYJwp
zGHrcm;v_0l`b#yY@ZriEJ=R@^y5#MiZd5-m?A0{OZ`1jO%ZpB%ieKK5BF5ia)i|w3
zQdM&4>xavWkALW1`L|GPyWK}FvE9ufCQUczx<{^?+AIA!qw9#pyQb9=bq6)qM19ay
zKHhWxSlennyIx-VvyY~^$FXddoW5MP{7$uP{9ePp@6Xm9$(-gn^KMM#=UARMSL=>@
z1sT`;dZyho|BGCvWXs9ZORwKKvu_>SJIRmv=l(pm<$06w=-+O+C8sw!Tl`*7Bv<{w
zaoY8*30I6NxONv`|GRzS;;`cPTen|Si43@`IVs}CYH!J@(&hr3rh(h;bJ`TW7p?ht
z%-iSR7wI>A!WSdvY`J(k)co$j!rI*_ZzpcLD6hTm_mtz?wO@bxx>jY@OoMNN_vUZU
zDz;p|>uKIE(Ucwkf8Fl1mR<6Q^Uy<$YS}WW@>ugJ$L6OO{+=AX;r3e<p}<9ntp~;9
zPcBWD>wc;KF4#je#aU5shwcSc1>@X_r`BARTE%w0etA)*JoDSXM%v9%Y%|TozwfUQ
z{b#4*sdj14r1tlFTVJo+-M4!EzFFdN)$-MG>232YJHD&;J@1^z|M{B#)|gNI#Wz08
z|6ciVvg)1hPb%(JXDh|>nD$89Po0)AAzfa+b>ZQOd#v92&Z~S9rnP_e!A-ndH7-hQ
z^bk6+%l?q>S(D>NGNs>sf`|Lck6pE?IBjqyN@nqL=6>71I+Nq8JNGfp^k077c`DD5
z#hxF&KRLQG|Kt0>@=ZnGMDES`Y8>>MVTF$OADid0alcBUZ_lfJXd0CJA!=^cEsvD9
z<}R!1m*pLe-XUwB*td4^1-YYcx{C~^U%kA1<Fn7^o!9>PEcwNL>Ef*1*SFq0nX*}b
zQT}tGbGzQKowI-J;%EC>=~tQV!!xJzl5Rw*CrsM*WU;6EA`|m=&3n%i+UC@##8%6l
zi7mhDv(WUUw*LN}rI!VJWWURp|4$a!yxjN6?@&|g8*8_ie0s&LuO!O<pwlpU=l8aS
zSM>fgO${w|5z=Cpt=IGKsq2bfXVsp%*?#dR!>&~+9A;`3AOAeG`S`(hhIF{_qB+Oi
z`kzh`{$-qgF8s=st45WFT4Zy6B$mp~S|pI=S9HiS-0;!H-I^tm6YoB~Q}LbI=IfQk
zCEr(_nfsU5dUt!X^mK!VQ_U^^>{(rrcV{z?-GV9M$NUck-D_X`d-}&)JDyAYK2vPc
zVdH1@NnuKa@Z0Z}9qjV8O~>EuXI)#!`FVOs(=vgajFk1xekorr`(Idp<wU=B)18v{
z$#;q`AI<rZ$hSJXX-TxwnzeZkpLFM^ym>K^=R|mA<)Lho^i7NO6eo7J&e$?}Q(pZ2
zmnBS#({C4@oBdf{WMNQvXtUyaasFPJ!?xG?5^t)n|9o=0c7U;D*0uG?_np}d)4oKs
z@!Xnk9#=W@>D2wTJkQTJ2&CLKl=RRRI%r{I7oO_Y>ZUaL?bZAov7R46iPyAxd$?Bm
zd|SNrp~z(0UC;FdPPl3RT>3U+{$rK7fgh`D8Dz@7EuHi4r;S|gkI1~*ETi`C8p)re
z?JwRCd3xG^`RB=!Qc9V#UaeZO_Rfhm)3Y%}kAIfvX)X%B7`f-x_Y4_3ac!4xQ?reO
z%8uN5@#Rs`?h7`IpZ55<9CPT;ORf3#Fl3)VQ&;KNH&<O<IT{bkC%m|zSn~b?-zg@U
zJKt@m)x<XzMbv$pup*>pcLDotr>#?uOk(PO`+e^-v)l-;;G;2*IwjU^GCi_YWvjQW
z#Nzo1Ep0hnfq}v9zwRe{%(oNp%-CTq?-DuxP;TGjZ$AodocU2>Y4JvkM^2uvTXMdB
z!{Ic=9kE~Z8&2jNdR~~cdF7L*&nEEcCRbt|S<Efo7ZGgxdiCQufBS=9281+dShT8F
z%e7s7shITg$fTw74N@;@tIWzikrL8!VM5D+=o`JxzrQXzGjU1n@uHaDUly*}&=Ap|
z=W)NreZ{t}r>C<gZrhv~n`>Oha_`Yz%WoabnkP0c*7V%9RX^q5bNNqemu*_MtMi;}
z**)9r6*0cKCTEUyvHGlCyI|$EnH3s77OI8cS`Yuo`(f%cJM+QDjf*wEMtVx!4xF;N
zr~lrprfHrRMEuh)Z(=OlKE0XGx@FO%vXZ=Tzvkm-<6=*HSSqvV1b3(iaVWm1eQ;2w
zxIli6n?%hmg=hBG?FBlWEi6JuW=TcF1vdYQe|7uShl^`#FPc4lb1Bf7qw((R=UXoO
zFRTq@eELuLrFF8{>#*>R`R<MT*O)i^OL;i1n=I@ncjU~>&ntP_ZZ1D_K7OjM-S;;!
z0+CFU=e#|c-*;7jsaZvlWr;{*i%XQ?uIOcF1(>F<?m68S>wCWJcAU?uRSUE-Qz8X=
zP4f7H&vHMxyx5&TM(xJx+i?}$Ru4_285$RDu`6R;A!$1MU%-+~&(;9O#R3^!o36gx
ztgFkox_;K#((OG@8YcTcy|8cYikMvj8?Tmi-P$!@Ex=f6l1~1|<=zV2vbmo{N)Fs)
zQ+%Q3;I;I^mQss9p3}~m1qhuuJjIW>DNmL0)$i|WdHQNXftqEjZj?!dX$m+g+_3r)
z{^$AwhK;dpo|_MPcxGD8wGv&NB^6usWoi2ICfV{SDod_yyHjH2do>_9P;I|Ko4)p?
zITkN8batlI{n?qPZM|fz&XjGN!cV2GaX*w|60~P7?_O*1NvnQvPB52GJiKV9&7U6?
zRdRJEzu(5XeO<M5x6FZQDh3j^TeRQ1c=pMzt+oCo^K8+HboI+iCaArh-0vdHaAvmM
z%;(E$PZ}Pd?_s@vs;d3JJ$D(+|N1vvFuQ)xY;vYZ>1Y2{vm94k;yZ32`M<jM{gyei
z{gnjPes2@V`et|W$+gefE3(SG7iMz!{#g|{w?3_<A>_5gDwc}eg3k-t_C@Je)X!{N
z<hU{V(XM8mGtufZ_W2xENUi=k`_&@d8;kE6F}w_E=~DHM*i>Q2Ev~mD=_*@DXyUUc
zANTB6JDc2p@7Nm8^B>=ZpT6<@fnCN*j#<k){?%3st%*4|>s8dA)}<x4b~h9*KK}Xe
zrm1Ht^O>H{tu^GnX7Z>g>Sx_Qn|*&bHdSuE|9my8cU1K!wm)gYQ?sXU*rw>TX63{!
zuM#hGHopj*cf4&?R>;KKPup9hDihyu*X&PG-Mq~I+3Ei}3%@t)HOvw6PE2^uT>U;H
z+E;ed$M5$wH!W{ja@qgiz04%v${*JA+qNdxZ+R5g@#wF?v~NrAzkEKi{)XX^xo<t5
zzwbU7zw2-zj~ln!<i=3%=|X*~jGVr=!*(5=w(e}2;1;V`ldEAHi@sLryp!2gdUm;@
z$nxy<c3Vzw4%)bgDOG**ub6*KR$IRt<sQn`iQ74Cy?yQ6?Dcy)<-@OExFWD*O>lmn
z{U-$vmkzZ_7pxwKWhBjJf30TC!sL6o;Mb=ct55soOxTo|6a8MEpWQQU4UbF(&+jeg
zm+srw7+ZR-==81$?<aL+8rA;#dU(~56%nz`)A~N%@_rof>3jN%Nah)z4n>4rZR0Ry
z+4$?U#j_uePnKqwv>aM%)~vQoCG6fV9)$}n4jdDB+3Nm$;=cFyiTAa2F-Nk4?;j41
z>Nub3Dl9c?*~43|k*3X!tGe<l^6nJp<aAB$Xeo>G^PJ>iVs2P^$z<BPsMB*K-)~s3
zXx6#vsR<5?R=#pyw)d-E_Vx7-=U!hPxVAEL`t!f?cYfB(+?=;tVQI=?>E^=D_gbvt
z-=<k^TI|!Oto<gh`^fiDC27sgmY@OU8;h&osdWqQwmj3ko&R`}(V~@Hu6z9+eY}1C
z*;f9vqH{~ljlTSG_RNVn`8s%+;je|ScRYK;b?mO?oOE8cJ*lxfU!E)8cw=u&?d#Y(
zwYO_ys?N>y*fjO<dAU<|Z@FGp9GfXpU&(z~&t~?dOF}iW-;X`t%*@6nSI8p&@3r7N
z`Orgur_Vnm$b9eVx7TeC&gvd|sjFiDfbFJ3;qq@wguccYe_@J>N|LF$_jFEK1;5AD
zhx!`Fl1w(7f4IExYozt9YhuOjvaYcOCOZtJdTz+3U;MdYa&-MorBjo19>;BMy70)r
zGM4{PaLm5sTI;<s6)zh1800Fge_kb76qH}&?NxYdlkumXG-=DqIvzI8)wkR4Pk&oj
z{=&53(e~#CRZk@3WD4b-_4dyaTs-gTv4_Qm5<Op}EN0mB%Z3JSd#QU$j9)y)a897|
z{Ab^!*Et_^nE&|P68GtROZx5m3ibCNG`)UCZNlT_wVsA*%{>Pns%$sNX5Q3euFk&u
zIA6ofS1Q{N^k1EP`sd+wDMwZm-o7#6ees2i@|>IU8uwP)GgqBYyVAV<(U)fl#m^=_
zE?mgfxb3c#-M&S6F@-l-ZN8t%uK9Z~TjDIApKK9p?`koF{M4R>=6@&u?Piq@yL0~c
z{My4O-Q^Ca+xI-(_IjP7y17GKd)bY(YCLY8ckbA{dG_XQZh0l=0~fZG=9zQu#K?#V
zJ2F~UirG~c+vK-6-T0##Q-1g7j^2dQpC772&+y4V=il9ReEywnYgp^Vv+Lr{p7Gzm
z;Qhw*3%T$A@w~9j-d$uR$h3x!dr>Oy_j_f+A1j)i{r7hy|1M2C9B!ZV@yVvLgXwp^
z+>}02XpnNFz)JL2*>;mZd1kJsPegrl{Pl{TshQ>Qx9ORFDeC&R2GOaN%?ZK#RTekg
zYFjv=x`Z#(bn>EohC6P~RcbkqQTpinjf^a=$J|enOeH;zDVc0ta`G&%!s^Ubt5<iV
zv;4bL!zB0oS-^>i;0d?d1l+nOA6nFOt-#Q+ctc}bg0Wsa!|J0QIa?lWix!A#OK@gh
zb<i#EfcvVYGjHv3J5$CV*Ul=cthc<?!=mM6($bCl1YIRGG_(%o87i4fEhrWan=Zck
zWXCMez|Fz8>t?pKvDwJ+r+q%R+}8b&V%?n2I(pN)_p-$_oqc%kWm#v%oYdlFk4tZ8
zyr@<wd)^m$+jynQ!ewb|_{{T`Pt8#2x%lcxp5a0*%~^{z_sEH7Y56$`?uhkMQS4}$
z?Xkq<k7b<?TW5xaOR`4K$yX}&_S3)KKbQJz&ia!Qo8D<HTv=iL<=LNu#}(9PujFqt
zoR$0J+0!ILNwe#<HcKCU{OJ(5VfS5+tXT!#T<eebItsZcC8}&@o_@Axl9xc=^3D5C
z=A<?^-<cz$qsHuI<+vmDvf0gB{##lydCZcxSQJ|8EJ+GX+y9+!^{hoFHnBJ@kj#~-
zyEK#MMU=yaeSHNwjfdYVnm<sV+#%4r`p2I*9-|{lXP1~V$Mtt-1W$P8A+!8=^Gy!6
z&9butmF~zKd-g5KP}0Lic}Z)_?YB-VBoCbxyi;es-o8*i-rREM-(ROajwbEcQ^&L~
z=SPl}pz^{AYwM4kJ3Y(mf{gE@kCg_JPjXLZnk%Om&s;V&rvB>NIo0Pms>G~Laxho^
z{2AfPHtkGh@9p{j+XVaXv^o1FZP<0<Ja5jr?fbQ0!|{ts6Rd9YZjyMo=`ZgW{mB(2
zeWw=PTC*WvE7ywm@b3e?$0aknrz(BVD7=1B2s*^xH}_v$@YY7x`}GI+{p~yS{QSWb
zqrA-_jmt7Mx}3D8CLKK_pAqqc_bxl%qAkVS|Hn#dUEiGa*KLchd8@#UzpZaBZ&g-1
zoVWI#Y`|qtEiv6inltYeai4a(RerZ!Y5(KzGQa*e9_#6R{A1<Kle~xZ5B*|)nYkoS
zwBPQF$#S0ka~bDu|9W|SltHs$zRr_A>wlh&$3NL?eV6w~4Ed{Np68nv)stZAq@sDF
zuE22#Yq^Y)M48L&HM)ztck^A$m=dAm78fVScDT<=qW^fOTe8JmzB6gYZi@?Nb0{dX
z%=BpsvGU!c>Q?*I&8p;WTe7-s`CNPcl`h<wzQ<l}`0J?Mm41K0qKjf{_|(%LZ*#86
zQghXrc<^b_maYf)n`de8%v%s3+V(cyscf5*O3LHg%o#oQ?7zyd9gSVy8xT5m*DkA&
z(7&wT&#aIC>=e)SL*FUe<f8oDu>B>rf9|dmjci(Fz%wsliPfaP`?MYJw}9pe3a>J9
z9q!wbl(#tCH}*~`gGaiNN$aeRql+|F-3rSVfBDAj3)`L8BgRJZ`uqat|CCRh#;`ti
zL;Oyc|9WLP*S1C5MT>v8WnGxMnS<l2xS62Ft7WFHUrs+--_7DCeWJR=;7;CAh3@#5
zuO>}Mk&HBJxUy~H%c|G!HC3ED<ma^)Jk`%j`*V2DdaM7&K4weii96ogxBIrx+`Aom
z@2a_GiXHj2GqSK;yy=TNx8lW2=|_$MmoB`l_-nYgeR211Ip^1F<F2s(d=P!`iBtLB
zKYs<h)s)5d-V{r`WK~~SQh5LH1RuVFsZEjm!GEHYK7O1YKgV^s?~TXTr``OV5xd6w
zd4AWuZB~)+nS$M^CEFZBLxWac3Ddl`&2?I;j@W6x^i8i`y!caj;b?5Sv97M}n#j#<
z879BFAH6Hu`QpOD!+O^jUA%bl=HC~O)-k74zj=Lat@E^0tCANRPKzpCera+rVM6Ka
zYZuo>Z<nq5kfr9DY~Zr2bgA#n6txGRlMVlWTDojW)Q^w1Sw+g<|F(~@dF|g45b@`8
z{{x-G`F^+4HUtIh*4&RWKU$r1_r)Kz63~prqP45EWUWdL<ZaKNEmff1B49Ds@7kJ3
zXREo#(l+OQuKxRLzC_*U{T`WWXWVA?th%`N`azz(Uz~PLd;0d%pP3OI<!XJ8SG<<{
zJ-c)Fq43S=lMKWQuN<^3mtCk~RP*bhb;qRSGJ)@)C7^3d7kBSIx87B}TvkE*mRL9I
z?6cd>7VW#O>?ve6o45Purpv7Jr+`*29=p2z?zwxacB{hlQ~q%47yWPcwfXqtu1M9I
zl%PT_^RP>k!cOYzC|-Zg<k=A5p`NrRes9x`qy<}aW}i@=+;Qp3B7qOTH@%+OKHDS9
zh;cr<XNJkOut4#i2PaP+PMc;Zky~;1+gomr;ACCXQpYoV{7Y4j&PeTTyA|c*r*GFU
ztSY;F-$u@SKRdI<3$|^Vl%Ue5KEb`v(XZ^OlVV2Rn#H@;2~NK2>RBjg*4>#=EG*gc
zW3Qfo)WTzIDXXP7UpmuMzFe^IjraDvw;Eg(ftL$ctzOk}%$Vg9m&S>&%8CK4ZPPph
zgLlpgu<(lu)ZA&O?!lyJlrh`LX{BV=>awKSedoeJi)3fMo&M>r=p>aylXt%^8P0k$
zW$O0jd#?Cb{^k?;H8-aI^y4$}dm0bcexC8;3&*{`MTV0ue0Wy=Wd8R12k$XTs!Be-
zn(j8C&owjC^FrzY&pV}`pPKx&=q{YU_VK&f>yM^S|B>@*|4+qNPtMI=m-6%ue?`uh
z3ZYHR-FLmL78@0Om>N@et?~AaEj0&y-1;Q_b5(d(fmZh=-iUPn_-+6EC(lC^`S07#
zk-t|xGkjmf@!IeA&b`|C{?nl&<sPSn4n2ALNIU<*mc>;YKef-a`F5qE$+K~q-^ITZ
z4&E{Q!(=zxtnq6U=k?f1wIHoS-izfA-u+x+wXv!4zK&96%xQ`F({H!&KAT(L|6p(B
z>Vlc6L2BmV_iDu5ziJ&&(5(4)QAMn7mXb{N!SE|nqu=kEd~ROO?3C*TAv1nnb2;W;
zRo<N&e&@i0#=T!In9t3<u%3OceM9c#xanp0rmstlyOy7ma`I1qW%9eMb-TVY&XfHt
zvGmI4S6g1Hgm$KQbDg~0+wNw2mU(;Jxm`cs8z<ij5>fS4k;*=6tL1U*kV)%e73JPN
zW|>Oo_Ocmeosn2)|DWYh_BBE8CEGW;b((*A)iQl!iCxpi^L>w(Ed(!!vkyIL`E-tP
zr_TRL>GSFu!|%;IDR_Ktn|A2Q)57;2Z1}t`>CTRH37Z8|JoSAOokK+@|NpBLR9HVx
z!ggA9o`l!kMf#C}cN;EUa`HC08CBz`BBOR^QNat&W{Xc3{#v}4v3bYOu4049PG!@J
zAIJWGbW&Y@;coxbcXRR?m^C;jN%8FetvIjl5#yYVTlajs=Dx(mZ1)tE(<b>br%tcm
zvvY3l<dwO8Z{N9H|2cW#_dBKkD$G<<riq5>#d!vsoNL-?ymEs_ix8LZ*3vC1qHE;z
zm(R7Fp_h6z;K{=O`{r+`y6IK*ebR)K%+33woSdemM9q?3vChBnny7kk*bMiX|1J0|
zUwmFWDJQ+t)O15lr7mcZ*?zmJDJCvjvgxtD9nAdF2QC=17kxgge<}Zhy~XP#%q#vp
z)6H6xXcks;{>m(_S+8b!n7H5EP$(<CN_bA;%atV>S`H#}&a9~AKm4$8gBQn=wMNDs
zlUh_%dic!w+GUTvRD7_lS?TgxXTi_*|0m>|_L^SxEqOH|wl(<hDMfc#-~V#46B-V>
zzdt{vb#<nv%q-h^Di;HG&bswBr`x=h^Qe$W<$t@IJtcMayLP<(!{~8)P41?=pZ*~s
zZI)|n<hS~P)`HzwY3wJwMLWA)d%rE;<Smsa6B$ebLK{_1Hn@3v?=fXM-k{}TRyZ@q
zMW|_-3loE@r=a(djc+q})tV<>+Eip!s^t`Hy6es5>GS4tGxVmoIU1chvRGHec+u6j
zPZsXkqkNRnI_h`X7u!3|6YqBlFom8~d*_z2xF^JyLF;1(kEj7h<FXYwQ$O;na7spO
z&(Q0)v70xqUH)uc_4J_4RVTypEt!0Sqb6@^oE5k^Z`0aWL6dGA+-BFlOXE{+eAZlB
z*}J-nFF#AUq`M||YtvM1tr@n-THo!hq~fptlew^0Iku|A(d+1hy{%vFu4mua_BDjv
z`GgzuNso^0YQBjVySAR0{ZsFAeVq69+TD*fL~^c+I&}U1|1({#_YGgqW?jNizMx^<
zU9K6Q4t+i|>$hpJ+P*V$SXY-x2$cQ$^x~T8{9g&&cdNcFjGfopB`TV+sj-xM|AVXB
zpBH}IYOWMoe|Cwv{lVy4pp{r5Q$Z`S%uJ7M3cXsv)LQcDLFz5-^&2amM(z9erF7l?
zZ)$aM@`n<QH~%=!`e#du>HS*U?)T#B6<42Po1-QiDjl|YpP}0tOBbW$rz;lsEvv4S
z`&!zYBA+fa$;0URWAhoG-zrV}Ytf%+{NK59d7i~uw-8_F%O;?~fbDa0MOSA}_!??g
zy3P6h&ese%VlJVf^@dStm(8NzZ#H$?%-n8zl-u8~>!gNa3iIzb<xJx7HGGzQ%J&%6
z3;ub&uE;&(>|=c5M)7Hhs<SI+^s;@rJ$0eRS(}RMx<?)tR?RQrJI?o_%U1TmjlXtD
ze@;{vRu+HURjkRi!!RnAXXhiUOOMqJem<S9p{Kq4xSj098)Zz>4$3|FlNfIFY31`B
zg+F3=+=TbC@u?kOX7u$hzvj*VE_!<(<W%ht+VkO|yG;3!yD^q;nC8tjn7ZD6PS?pd
z2^T7-KjeK<c<||UgRCbJ?{<8wQsMbx`HpAT)F;7pXE<yA+~s(;`z>2haq)+;7frXb
znR#Rk9x&L|SJ@=T={);Y+@Ij^{-({(3%M41EpNo^d{QxoVF$D86mjo8R?h^b4hw#=
z{`#ota^KM;pGT*44`dlV(0p}F^uNXY=W2P`vkx!3f6D6iqV%L&!5v6Tdk(z0_blD_
z@`^QUj-+jV__eBh=bnaDt4z$zCwr)9XlmYcd=RSYt8rz|&AmwmksqpTXY@w*El&%&
za{c<`GZ__sEPveLyAZNyqJi7y)@K16tjs%iR^HHwyy$gI`HFeVgtKKedeW;V6x{SZ
z^Y1~a*|9Y;1`Fh)mdrXXCwr3d`JF>gm-F2?SLA3Sqa?wVyv6F|S>A+c3oS(sr4&Cd
zi3l*(bUG^-BPSm=*=NJTc?CL+m)|N1d`QeVn{r<$Z;_(Nv5ajKXHV}wn!>;`ahbxx
zXBuVO#aG|#$l3B>%S^$jwq*evzS|ysuk<*2*o1AiVdR`0Z_-RocAU;smFjKEQIgG8
zn|9{G>61s_Zb+T%<9;YYMoveK*=a?KlvG)gbcg&g#eMaHFJHJ%JjMR*-D<yd^F{XW
zN*i+YZOrxi+kRCkP4xKIf4kjv+c#z(HQ}9=mDX=}O82A{eg0JUF@|k&>!C-N-trhc
z`1Xr&!|n1pE0VYF*35o>@7;Q-Fv+w%Ke;wPKcga?c_8xA+{OI2venb47F06ruxZTw
z_v-Y?yT{huxubiZxssLTQ<+WQ;m5}xetiF-e7ipTcK!6@#;4W(SpJ;7|F(>rm+U;*
zZN)pIPCwrucq8q<Zt3J)vBP>fmoMII(tB4eBeeU}lZ4Gtnwpx5a{~o-M?||#N<E|Y
zxW4r6pZp1yAC45i+;Kng3j6GdK4xC0y*^s4*?rqUcJAFiy?50jLW`F2T|3V>xq0r|
zzq-uV{?9t*{wFhxclrd42`XHBBByimN(=DTi@pALoI_zt(P!yt=jQe5y{i^tQ@*0_
zd}e3;^qW=xvNnAP-~655-Lvsx)4X(lcTgW<HUG-zt^)ZdxOT7Ws*mqUQ;A=GH@Ed%
zo?HL4i{Q09pX!$@Tn_ztC1n8v1A}UbYeY#(Vo9o1a#3nxNh*VpfuV)2p^>hUL5QJ|
zm9dePv4yUIg_VK937IV?P&DM`r(~v8;?}U!k*9=#fk6XgLuPWaRdRkoWl?5&MhSy6
zjHTdMP>_?V;F*`Kr=tL}S>K1%k%56h9Hu8FwK%ybv!En1KaasBv$!B9u~J-m>JA15
O1_n=8KbLh*2~7Z=1f?SY

literal 0
HcmV?d00001

diff --git a/tensorflow/lite/g3doc/performance/images/iosmetal.png b/tensorflow/lite/g3doc/performance/images/iosmetal.png
new file mode 100644
index 0000000000000000000000000000000000000000..5e2b8bde8c1dac18ff66920f4f2a3f369f81bb3a
GIT binary patch
literal 36053
zcmeAS@N?(olHy`uVBq!ia0y~yVEoL$z;Km=je&u|xmWTE0|VFEOlRkS%;aPS29M6E
z)7T?|BgKxtuYTDpwrk3T>U_o}Epk=)W==v1QzV3&x_CtkS&lb#acDI0#1`l@@MtM@
zwK{H0oUhQ_WyL3!v*Hc&G>gr@-@e=azsCIg^vR#k*?!+wz3)AP!@s$z0x5@z7*alN
zZk_lqZp)7r7pZrK4U8-uClof^P-0ZJv5`=iHT(a%@7o3KUCecsCI9{X|9|KwnROk!
z3?h96M(#(>IsLFO`gM~j(Trh*&5?E6ZK4g%nsR7!O=J;#H{;or$anLfJ&Bxl=7)=W
zmGU1x2Ai2jg_bcn%!})ucG%DL(S%Hn=+Blmc84y%-nIF1giK4hLe9LYTT5o`7BKVs
z=RN1j{-ZwAj-_l2T57X<y5z&!3MEIi?5F=LW&TG;&V01f^veWCKVdc(rOmIt>`c|+
zon&NhX_C`dIY;upq@OVRku!$TPv%-2mzm~2;mgf47amypJ^X37v~bhRi^V!IQkuQz
zMSr@THT3>bt6S*C<Gf36x@Pap*<!Nh$1V5VxL_%k>cU$4{o}M-t&co1&uKc@zm(hT
zQh!stU-RpEziz$>?<Rh=HD9L@H;cLb^<(X=pOcr^HRb4M);Ol^xH5nCOZ~0gMT?f4
zPg9=$_V?z8mp*%@v$mZ#{J^+RLE`6y2Hg$p*Bba9HOO<Ef41(#yn7AbgARHr>YM7$
zKIWj+;hp^H=Co%f(b>{g441!J{N8o{-prN89+Lm>hJ`XZY>;L8zW?@`ZHxhL+ol}c
z_TH>HZON4hziU@b*A;tq=GKMF@e)f;r8X)BFh`xJHDVL<XzM#5TA?J#-ee%aD|8^~
z09&R4H^;%E4N_~^Sr>4;VW@VHT*1=rU~9qC*Hm_a^;{G81Fjuxd5ymvnA{F>8}Qw0
z_T*?P5MW*8lr_ODg<+=SHUZh9_L_wW8bV$TlNN+@@OLh}dckc4hgRFE1zIZ@e@R+#
z&TZ9mY`zeiAtBX#&tZO|h)<hnqVElXZA~u?nE9F(KXfh7+QW0M_4C8r0?im+^TW9x
zB7bnza8Em+|A14(VPe234cRHIU45V2S{DihYHBEE>8=t8ZCd5Fb%9l2?FwlP&R43h
zgr&NdEy~Q`-lA>AAlsa_c>2QYi?%O1ec|g1vl9JZY*iAregE1PCYTtcNr>|Z3-@~-
z)j2Mb;AOCNhVYEfGooh%o~hOmTis)N$SRR}1LuvBH(GDP-qhu2Z<Ds}kUi-4puoa(
zj^sIB@wViLn+jQX)b2>TBYCIvj>bFXc`etE-G21<LuZ9u9rHex`yKYZ`}^v<=Bu0#
z<l)e3F<@%nFlocw0OuDfF{(X6YMq~4lov@G&YIyD<FBK-P2jw+yukfQ9xAbuj5V8%
zC_kO(<>~Gr?s<5Le+ug+&6~6LOwduwSH3=R_sQ~;`I;hKibp3;@;JFHXR%4Zxj?x<
zyHf$FI;k^^WM-BcJvGWTx}F+$QYeKtMRHSjYH_M}O8?2YskNH!6O5;Po|b%a?NsgQ
z)|%!j=8FDPw@>7se16h>t@|4Fp8sYt`5g>67~Z&e!ukS_hjR{bh`L@~TorutO79h^
zSCy|;D1}}0n7DGrx`-te5rqzgF`kYm<21uETZ499(p2p<^12x?bLE_sdt!gC_>}wf
zyw)r&9?^BD#HNZ(eXS)Q@_c30s-jTt(CE<om33i@SN>d;y5em}c6jxQzA*F9{($&(
zbq;^`{5oCP^*QGG#^<8vtgYV9V_TEBN^s>yug4SI_8)6mHf`DdWe&yjpIM%r9(5{e
zUewE|-CO;#dozQ#-rZVxYxmak8TQLHFG_6jiPBs<-E7_Bg~AJzudTc!d12-i(@VV<
z?=HB#>UTi@3is>Y7jLh<zSw--`nB?l_DAja{6CNJU_;)7v<=r6oPW@~aAM+|hiQk)
z4yU`Va`Ranvsi60`!SJZ{>Q1ut{yY@-Jn)7nP>7}wd+2K!R(h!E*D*1owi}-V#)J4
z`)@9=Ja*^iocy?4yA;Q52@y{sOd@%rwW8Ug^ETvcuSs2+KQ&|P<yGfj&EDEvx~#PK
zYppcvY=_yrW^>K*b9{4eZhN}T{H;b=UD@k5R&U~N&%RmwmjA~6!Ua|>7CB`v3V*zM
z@kFB}r%<N&Qn^>rssgD++LsQ$jD9IT*YI52xz}@c&#~WWu`6Y_OzF?dR}YyU-g@!t
z&6>qO@BB3V?0%=@ZbV*ao>2L$>Pf#H%RAfq*iSFM^?2j*dFsbEU;C_ixMkwGjI=$U
zHhmMSn)~6`gSU_N9@~AJ{psWP!7q#-tbQQ<IQrS|6aLrrudY9tf9L+e`l<Eh3~CA*
z3F!v}f+wF$Gc!2OJeTP@%Qv=(&5N2&HOaBhX5((Qc78D7z=T*uekWe1p9|k6D)dNh
z(|(h<Bk;$j5AjMH1Rn}M74%oiRSI|i<{G$onv1F1-u`c0_vC8rk6F0MTo%@r-fgf=
z@tgA;|Cqu#D)+SNeE&>k`N|UK<gTP=XtvR;Fsbo<lP$NKxRs7hL`C()i;G0N^>oAH
z=j{A(Md+sL8r9>I3_J@xdp+YL64zv1b1^dBsZ=@V?b&;$>kfW=^5$7j;oTia?=08M
z(>$tqKIm>x`O?0nl}n~B+54pO^QnsUmzghVU$#$aNxhQllX@(5TiV0TlQ#K%68$0h
z=ibsN?{FPikr|v^ER$08PJh#p7H8wkcAbzhafNcUU}nv)qsJckK6)tgea@#j{il<s
z%SS8jZ`!}8e(B3oy4v%f{d)Ru3)kMRy{po<o_qCmb^CRLYawe}uc@tn8@YVDUj5yS
zdG&FBs~-L8`MN)E!@Yn#5APn^(L49&te=%Xm1qAkHx)f0!ol5UwmWa#AI87T@7o)7
z%c8&K{tP>zxhEziHYDgvs&@73r}DXVI)4kZqEBsWeA{=M|ILAS58m(CdpSA1c4JLp
z-p3Q`CdB6M&fLvizW?^-l=Dl4eos2J_sHHydna9baH+5M*YC17zk{ZQm94(Bde2s;
z#Yx^r+yBn_d!4^RE@S1DH9D*Q)J!P<;5W5CG<s`d=}E;m3+FidXC-C-s$2B`O2F#4
z=#bxA`{LDC_OE%LTacBJaV6u<Jr?uM*~eyyo5fqdDc-U8=%S^oLU(R`!C|3f_vz2`
zCG+#;d97|0_tdR<y5r!-F5xS}t+@fWPi_17ZA+P_yVLOpCl5@vox5(Xdfmo9OV@cX
zU;SswrIl}&cP{_z??3mXt?@<O7j>0pU*C9~pEU3H^GE0JzUh4-Yaw6q?-A>M?&rTG
zYSZ<0#lG3~r0C_7)L&1(Y4?TS`X&3z|Ie!ruR9r!H*Q<)7jC`IGXAy99Q%r2yN-X|
zY+jQWv*YBR%BNv3)*ifVxcxnkuFRx65ARf#nSXxwy4v!)+WY6W2W&KIG`??mGV$2r
zSNDDDvZq>j`SJed`EK7=Ra4($`{_~n#rN@UpXUe6n__dd?$(Dh-y*NixPN2obMATl
z`vmsPsEPd0{zqPPeNVjX|I44x|NHpk<HhTTt_SCD+ZS3L{r%j(*z+zwjx7Ej`Ci>#
zq%P!l#Q)BJE8iF2dA@w|>HQD(O{p{edF!X{i`&gB*R0fJ^gb~wlQrN(LvRp7MKC*q
zDkH-yQ-%)iwBJu99XH$l<7xbVWV7wp9vQ}rbIb~hq%*fpoPT`cO@WzD1)_JTGhCd<
zux<0?zIv}IN$J+o-B<dysHp}A=TEc~OlLZ9ec{TLC%>&+xqhq1rtb_d>g;bX_xI*|
znaelpmcg6FRrjwxZLGg&&mb$rxnX<tp%)Ac3<}8}LB0$ORcZ_j4J`}|zZe)88eT9k
zlo~KFyh>nTu$sZZAYL$MSD+080|Q%<x4R3&e-K=-cll%n1_sUokH}&M1~F9-X7mhF
z`M|)yz+U3%>&pI!M~K0IlTYbQHUmQggQtsQNX4x;cS~o8hRPrK`2Nh>8+&iZUb&>G
znz*-%ku9OTe0xIDf+WQ<_6~)F1x#H_5)|+9?#}ssZ}0ByY0vi5-&ramAlh>EbLG6+
z_&vtu&*m5xKQGTp6H#n&n55!4X^EP*?G?@mDxQ;6UcR4rFa{(x%R=&us^=t?m7S9@
z1^x(JJnVStv=(RfF}v&BzqDprJ~dYJndcenV14D}ztb)20wbf^*4ve=nEYw}lkeF%
zT3<F>t#8bont15g{QGVT6whqu&bkqCvE=+=dvAVrO-;RJ3l=mqH8FL0o#tpOd$zyg
zu#sE;Oplr71uG`+l2*QQpnQJ)p)>QtmGZ-X<r~PqV)+w3H81u|vw!dNd+KsaeWT9A
zU7!8?sr#NEA9MQb-wN?8e)&)5^me_+eb#LkOFxv}sXLlEgRivdTlX>k&HE1L`#x@2
zY_(YC)ssK2uB~5tW=H$h&F#CnMBv(j>h=t_?_UH`@4ByRNx0c&#gxYFZdNYvrTu4W
zsq8cfYhB@g($o7k?OA%K=zKCuDR)@6SA==S-KQGsc6|!Em7*HBEmimOva?HWfBfsw
zwN3ugmS<~coL`j7$TsH=fANX){+pL2zsz{JBTjbKiTA$0tp7>xKXG`17*{}GWal#J
zj7BT1y^J9X1LFc)wbdGXwx}?fF8#BifK@WLy6~Cg@rT=H3%bp?w8%*-t7b>CkcYqb
zu|;z465Tdb{ENAkcdw`XTaKQ^XO~MhW<0t+uCA(GOrfS*RhFa)%BL<~v}D31Fa3Ef
zDl%cBlW%%%Iql<GX*#c^A}38aaex1*D0|LMPL@|$AKFcvcuOmqtzJq>YADLH9(`)D
zDCwk!|G_m{9JW50iHxH6*la#}E?KU!K#ObNtgJt?+Maz6eEN0rgh_ryd)j}!Dq2?e
zkuxq_{noaVYNq#`4@CPq1{$`V^ow${x_HjHn(@DKU3^F3X04S*(L3MuxJ-)G(!N&t
z^U3n8J9*scWrYvFW^#lo3S3yh?>OhUQ;+k}`wKok-W2sphObCy?Gg9eJO4a!u_+8=
z*s{Sfv$Uhl;D}4KM~Bfuj>hZD7i(X*ut4yF=%mLwJ8~!bIqZ*Lyv5B~gUfTHSj@J=
zyNeBEvKA<&Jb$M6{j#2N+o4|Vir;U`E8lcx&8z>Gc<YO8C%bX=1*MbB9ILdwG&R>A
zNhzK*abn};JVut%t5ZcprY03vdwj6q;1=FEPa|uUmVpR&=~dAmrmbJj_}HvFVJXUI
z77!88aDKhu{5f7;PC8vo9L=jf1$CX)+PFLUgopo8p9!qS%Z%zWW!DznzM!T)^YnWe
zu5T4S4F)D&-CQeHd=wO&>=6|uBo=z`g3}Z+(V&|@W_x&h9#|st?z(WMe~N*9h)944
zYpChQ(w9DMcIHyE54S#Be_VQ7@#DMB8$@*zzWq7cqAM+ty6y9Y+VIUOzbfwi_~CuV
z&-x$RK}qqLvrE(G_k1;2SJhtowC4ZkAV1spQC!pN&mWo^ZLv6C`jwX1=M!t4HF&i(
z%*?Of{mU`4@v4pGC*41*xMyGXvshDnX{LGMIm5!9iWjBp<h~i?UQWKbDTVEPPvnsg
z%iA|4T>8Doypqpv`E09r*^_6BHolDWI<57wb<LfM_sVnXO7!m6TFu&iyvETZRA8Ef
z-u{Pv=WG^oZ&urPygm5YqrR?1=fc}{&AQv=s#_k(?@Rl4UYlcv`RAe-9`kuur(ch+
zn74MS9nadvmNfUzCz#DAKiMezO6lI}MTgivy?W4f@Z%QmtqcV*+ape$Zqm6GaGrB*
zj?c{4W2ds1MI8+?9(=j6<JA(Du5DK~Zc*?#vBdaZ)}u%B^7j4S*riu1{8A|W!_zxl
z&+<7F4Vw*SO4Q^qxi6djGpkI~`!sJ(TE++IB};bh`1Yj2=bwOlO}Jy>-o}O8Nsnb?
zYKv?>_8qof5Ur!Psc6>3$&&7Vo5hwK+r9Jgz4XWpTJx1BpQ?O%_NI-|o4lIY&p$o+
zod1|R`XW<liN!mecYA*Z{b|*fw5wr>)?(f<*G$K&W6it2JKo>o{_)|c`;6@i`Ig^2
z5FP8BHa)oF%RlK(!^)o@raqrxlP_vETXN0DnCjnq=UiEA`dZBA`OWZq%N;jf4hTMN
zm>zd7Q+DwTi@)q<;Vc#Jc7|7!+?eb0zldpx%;~Mqza|}9!e5#F<~hgexd{P@7gDbm
z-CXeB@bhQGJ?42@-!!C6?OIQ-+ud&L|Eg$3cg!ztzW4DL>s#cwg5$#6_ZP2wx^(*d
z<^IOp_s`C9JwKWK;`<qSU+g}{$jKD(x_xE4SNT8q7vG*q)7|!-31zPS{F43s=B((W
z3(p<06I~N|rY-2;1MYK6(%W^OouBJJ<Ag<3Xr<`d-+T{#KA->eL|U2MuMf_PZhd>)
zIz8^uwr;k7-Idy3{|UU1+NY&IOHT3)FFSKc*!4t-CWlyFUW=VS4<4_0a_;nv9WUHB
z?V5k?w7&iMiI!&1{(oRTTDkh?gTtOO)en33{QI?<=hfVkGs=$#MLk!4v3jXl%JWyd
z<(~aimp}II?y)nK*_O|?9F(c|3I8>(MDF{Y0!7u;`_f--W6P8KB;jxSPe69Id$2r@
z+^a`-m%oydP2V#0&5hVsZ&!J}HnCQm-)66}V*T|Gl3L<(W}I2eEVNMX#FkV$tL>f3
z-20~X?F@`5>S15*8E}1puo=s2^Pe*|E1r0HT&?K+OwE#>z#p%cOJ)6O`o$*OefreZ
z<;x4M78;6(U){ViDml8&_O&+eIZMUP*=#Bol<u|I-w|o+W7*ATlfLbh$Luu{pFW?y
zxW!=V)wq(_#oOYRzi;?lS$D<b-QJfrVQ1$Z+_tG`S*=;$o?jYrvL9{w@6~HpIrFB5
zmMR>G4(&a9xv71ARb%`6U6Yzr?{C_+VUDSN&EfRT*Nd*s?f>v<h4_ofxzGNV+b>(D
zcwkPXab`i<=NpsHf4CqUF1%Jk<>|Lak1qdre|^rq@c-ZP$#WkZdcF4ObN=Ai4YSJc
z&p$DdS^xRe@VJMo4%Yto_*-66Q#b9JdHm%)=M&~teyd!!vw(p^{{LJ_!T)+jGvwJ?
z1dc4<{`iO8T@SBZxB1Fe3879m?gZ|*qZgCM-zg^^$|}uSIm__HmO2%6wc<mW%qKh7
zr9a;G$$4&T?S&tO=e~#h65X~Y*0<In`RDo1C$+O30uKnv9{#94-@~uUNGZTMF>-=2
z+kC&UEFs%|+t$Uu%6iU-X1qDMpzujlpo62Mz|M2e%HFw6=hMx+xJ>uvNzankS6ioS
zN!yY+&GkaOx92@KuUA<&x0MM#KJK67?P<z;^=Qa%DTa*OTdnU}ZCw`mx@^n)Dc&d6
zCl<OsF0w4#ZF^+*+a;;nRT`!4)qPd9`~6JZ>YL51o|YBPu?K!HGZDLxpY&|qW&M^#
z7uHUl{cUBK(Ph~$nVHKMg*h9pw#eb)o9)#T>l!%4V<-2)X`2)}SNc_LD-e)Rep)~G
zF3W4@kYA-Dt_y3oO`c=Heb!heWoO7CE0rVBoraP%hU+I;zGm8TqG*y(sYJQv+un{p
zX`-q{a!pMRs>>_2*L*$NBW*k@e}BA-!J5UUKW^+=J0tEd+eKCL+0))u7y0d9T)2X%
zkzw;><=-;yd0|>_cguLb4q;EUJ+W)5^ogwd&2_cAC(fC6c)@12p!g~NnZ6C%-rjnA
zHvXu$a{QC&XH8{FW8{+-#T;?jz0q=xirTToGeR63Lqkq|HW%o~JNMwSzoM#T>t+3Y
z6Q9fNZJzQY;?}l|gIzl(SAKk2zT%qgmv<Lk%dVRR8qJVbYY7OHVxD?(%U>nIof$dH
zS8zF9zWDV-y3=*Tl7Op+zZ;l^h#1sgG!mLLSLa`)T&kh~@6R*wTco6#4}VjvcQsrr
z)l{`#p23$p=zV^M4#$O_Dk-N#2A;Dw_U4FOl-S|2L}a`E?R{NLX`9>U%n{p?bF(S0
z_E+KI-szL2ULVQWWAOgYjFlhvyQUa(NjPNPwzK*6W%8V=GlC&`k>`Z@`5!+wb#33(
z@m*8zm``o&1l`bwbLC%IWpgv%Sgd2QmW$&;dFQ)B3CC0R+wR-x7#I*}&~hVi&%~26
zxw;m;3Cj-%2oN=BQP8}4a_e%ni`$nr-#W$czBc&Wr9}qq_N}q$_Ia_}r<k&CO?j^3
z%H_c*_{s3w+UY-Po_Psf+MIFINP6?pmPJ}Ke6;?xRVmeaS($$Qnvt_UT~lGo+v<Z?
zek2Eovc_I7pR!-h?7$Kwk=A?abB|8$zw3Yf+Do1JrZW={$1OM^eJ60^kJwkfTu#?7
zY<cVMwxHFt<>tYe*Y6&_V7*Sw_Sl(l`Mu}3^ZgF2mw)wX+P|*D=3IYnn_hdTY_HF(
zSawfarR9d;if#X+89Z<17PNnw@!_TTlc&!=9$hYcg4^}7mG#Gf=V|M&>HeCR`%k0I
z&ZehbzHY|xcK-AahO#s4i`BN)?KP~QSi1O8@QK{nXO8FZJ$a_m{?o@7-)HVEP*w{)
znzK#tTexBQxBe%&vFFV%r$1QBzffcMbn%8n3G3vY0upVGNnv`P7r9Dzd2N@wTlm86
zTE5nyB`bE<*KdD2voKmm<VpUr1&i8L@7JF`KKc2@jqgv+DSsz9!};oz>p%EP1>Xlp
z?P?a2Ywax7j*JsnvPkVn59`%4cZC+YXN%`$FR4#{xXViHDeH|5OXvNR{<Gvr<c4UG
zoV2NzRv8?PvlZIos#0g_zcn!O&GX-;UqwP$ev}2u>h(`$Y2TGR_n2~nz(lUwvv^Bu
zHZPhNcJ}U^8#jABjaTGZ{84td2suBk_w~&Ub-J6E^*z71++FbUF7drqPu^Hu=h5{^
zjbx3~-BobvW36DF{yjE(wQ3Rfl2cEg=i8?pni5&_`)7pR?=rJlLN6x@9J;ago@Ps9
z<IJ+yTN`tauJ`#@CRaH-{PRh7;f}L8ZkukOF`wW2@V6-I9Fq?+Z1XNne)UMHrjFM=
z#ANHy)|^{&`ZLzNdDZ4vR2IF+>1U12!{BYta#J7eUcBS|#OWFuIw@~<zYl&;Ui|Lp
z^b4D7)s7x*ZP~ia;EUqlX>S58UCxBDcXFPLIIHI4XMR$%aE8yssI6yZeRFs;y1bln
zgjN|#F&<gA{o%3KdQZRds~6oapT8(#@s2OMW@k*6D_^mBvv^|I+S^XM=N-RjrOmop
z^1!J=^)HXFm!}0s$*O2-a;B#+t~z^o>Gk`5FW&M?{Qg(HB>m}yjn3RW3m<x&NqTTo
z{qwmc+ao{3ig13EGn&0T*=*&GMM)jzb5=BDZJID)M#p8bo6EGV+A5CvCb>ng+y6@G
zo`mf@`Q<&wlS&Og9Lg@dwPwenq)zd3$7ZedD)G_Dn!;>wad&wBvps*z*6*2qZt7p_
z;Jsgi+wRy(hYFThq#4v&r^vr@x~IPT^r@ffDj!bX(YiM~sOjhu0il976AQm&zjiR>
z2@sX(O87LVsK&ARh>E&;c@Z<0?54Ro^TH3S+g~k;T(4mhn6^f(HSO$Oo);2H&(2Rh
z6Lv7qy!cw7VZ|G+z&v5T>Ln(kp^B$c52Xcp^lS~<zNGK^zAuZJbfb5?oRw&M#dV9n
z*lO<SwkM*M=O0Q66I?a(TP%CQ&!^8Pzc30;DV#7xr1|RGhxN?AKL7eHyyNM#$Rp}U
zF6-FxPCj|+W3%4EC0Zx83)?B_na(YHYb6lC8xWEtEL>2nwE4Q|WKMr4rn1RiukZcc
z`DTr-PsM9XwUq*MC!FOikMqfy(*5O$h}vG`KFiB?jw{UA;;Ia9a_YUZ`l@o&baLSN
z{bz*z@4DyI*eKm{EAh(C+xd_yuCm@{``Ld?lTEvd^>+95Oi>9E%BlHtZnMvi3r$z$
zPP>NlDV<VF-)!#MRm1vf;lgaitSspdM=BinAFbto^wA|bNVJK!l%=GjuWQl5bKQ-t
zD>T>zR_+WBpAz``^W%bQ53dea+njt=!HVqc11ZY4-ro7Ra$)O=b!!hlEffCmv%uwe
z!sZzc{05(2cjm0#e);!_cM|uM+-&~Z)SllzJ)7;j6F;kluC}sH$fHL|7fqP`x6BTx
zm34Ghw6fw_*uHY@$46g|BpA>BaJ`bV^Mp~0UudP}Ry+H38T_0dH_i}Y-GAPD#mC=V
zO}lnkiA?6*e!0!yP)K-a;*2*(j=BhcxEd4~$*ZQ_>FH^aGtD{n#BWuB3!1C>gs-sX
zZJ&80gF8*MXTiDX`Qi^xeJF@ZXa6_hfN<)u)Lxe4M2<bYR!*TE+gAT*W)oZ8b#+O?
zizco4ZUMqO=B6!@)#_^a)zFs|R2U`NB5>t5@0x8V%9$AFzN(US4ecnLy)l~ada=3F
zb*GTfs|#LElw+BE)Xk@def7;NJ8V4G>UXGHaf$Yx`cx#<WHs+b#08T(h8NfT%rq;W
zE`H(ri?|~<c<riQO71ziw<769Da-4u03E(NeA0(5e5}geuv=+AyRyuZHG5?0Jq`=c
z_E>ZLs>P&~fZ$Na-tBU$mAjT4snAhlZ8;Df-D{sAFkQfV!$P?;+fHn2bu4wYG%e+|
zSZu3eHt%<v*rH8)Wa1Tkrl?Q*5y9X7_NmJ)b;&bJ4|g2okMB*rQa%4chrjX@U3IO+
zw$AN$oHwhSU+8yd|0IsrHAahh!lO)<mzT-Ddh$p6>H0b67yIvTk+&8*^y8{V-o9du
z<0YA8D<-S`?LLv9CmW{3e^hhPmQ8F|A1N4XPnOvJI6(b%{945Yi)URqyl?8A6i!Fq
zP|a4?)E7>Qn!1yp{E)cGG>uO~5;n56`6kXGE?k3P#UBjnTNi_d$~=Fk#qF*7y3nr~
z%&Tt};$V5=oDAZu<eco^Fd00u=H~}vuoZhwQt`}slwvR)B162unRYA2Nn7SFb9N3`
z@Ul8C&#dwCvIH~PS5v$usmR8q7)<Bm4lQkZKke0;bK)j%p5C?hliqSV$YNu6=MIgh
z0qT*HR9<dKlrqb?vB7z6M?HJv_C-^!t`3_VAJP+`>#{RegI)1i#zThG)X)f7KA%_i
z!dJMi>pfcBdHLj-+}_)HmK%OveE2%PG)>oh1-FAo^8>wlM&3V)jrs~VqS%}qX7h-a
z>NmSMU0<QK{y>0kgzm(&J#uT(ANt!nhA8DtKDOTeA4g!Pi}f0I75*v<kJMD%u-Sj*
zt=0a$QnH)>!ScwGNg|Jb-M%f{=2z*-C~cm%XwNFK)rUI|r&%uey>h+Yv*WJWo<4rZ
zvhM8<+{ZlE-}9IGq~vGpCvLUgFFN}3dC}Fc=7q=4nj5`X^SPq@kIbQCjrYSZwz0;P
z{k*$If0b><&x?Pz));>CotC4Mcjq~mn(wcsO;P_D?}s}FY_N(~U<hSQoo>%CdC_G%
z?+F6B1}zymd*A<^%JejPdf{u+@Q=^bFW-#&qZlt5-%~et>(S5*-m4Sq#Y1H_*i6Zb
zJyYu}ms~vad&BheWl;;J<=g%4cp805>1kN>sV|wAZ)|wW@WeUe%*;=M+wW8;_uKwd
z;pg4iZvDQx=jzW;mRG0vZq%*ny7uwe@A<)@;&vuKr!1-0ym_{c;mY*<v>SW*gC4&*
zZJnH_d{g(wlf4srDxTEcDw%cU;@is^D^@4HzLwjval+Ke&N)+rTl`!jHKo$Bv`-fv
zQ<*q%nn!4+BCBiajXN8kJV{C1{wD7EB`4k~Q$q!=)x`&MuuGk|y=$sf{fs4BcrHzP
zB;CAd!Ky7Ya$T23%)OZMJmuk*NCytB(_NZ6dPh^rCr_T-dpd7v#p^7)im#%5JWD1_
zu3&oGZ#`Fh)9(qhg_t)S|05CAvFTEgj+yY%qe~NtFYBnYx0l{;zB6%S(T(8hYX&^G
z(hmPTK10&p+@o^s%r=E;*>LuQkA4<M?yfX0-JD*0aHjgCXB!!r6c4%F(-!%;{o;?l
zg#{ZL4>d8>F!r`A37I6EoVjA%vbgq^mWURyC3_aln(@Qsjo8##r?_|~Pu>w5)fuJX
z;F|hWuD{dZ5NO;i@bO{U$qrRfM$?X6+r58XThk#W6P+c?(glQ{X6W3{>gw|1;1t%(
zlsw{Ut$F_bffj!ICBX(0*R(9|@XS9wp<P}@Z}-o6pC<}*H~EF8vbH5HT(^vA|FQ}5
zCUzcAy|3HLJC(KP|AXWx*<w5M*b*t;)Y+AZM~Z8{o|SpG=Tk|W-6w^@NL`NKdqb+4
zHI!UZLuEf|X1d&7uwj9~;WrA##?0-mOLuKz+HBmWv0T)dJ2)`1Q(aNAv2oqY1`8EX
z3ec83!{5N`y)Qkqv-kS3`_9+PBpL1$-2Qvyi0^^a&=qH*&rYa*pX%x;n&xyQwdC>2
z<?Wa4?jBl^xc$@DLrND)^mdeS{M~(P$N$EMPeQLOexY`5&o91rdp|14OpA5CJ8yTP
z^H<S5pRP3b%=^Ucw_Nn7*m-7uo6`+;ZzKPHzJBNTgUeHXvnJLvyU(wZ?6)n@R`p(9
zbo{UKht{_(ueB|U9$dAlJASmr=vVGDo!ciqJ#T+_;Mf5>KR28Iruut+SX_JLyZ`k0
z%gXaq4u>6kD<0=J^=hZX!W}E7e9qrBzk2<Cj@R>FFbgSN(lY09+;Ta=!CYH<8*9Yg
zP-oM(zo+p2KK1k8^QCjD?v;w@%s&kon4E1ZHgl22i$*QGnxBzs=c8NCrm<z~nBOsF
z`M&LE&(DP|$K|D+-rm}I=j)N^7~?0BelI1C-#%~eR`b1k$EP!K`)}WS|7~&m47*H|
z{QloUlP~w}cPu!@8eV*Jr+RSshCcCmm2FRNuQEs$n0<5JLjF7MlY-7o?zcU;Yg5%3
zHwpi&>UZ)*%$xU1oZqCoH0RfY&^v0z;`8pz>wMq$XhpXCLjP@#%YW{8*Z13EVKSHe
zz8@T)&8|E&k*-*sed0|KSO2|wX3lqWoq3`xTx#b!PE%%UzwGPPE@$)UOfgG=z&h>p
zL)MEep8az7Tp4p<!gIR=uSx`0tUf9<*Qcy$?w_WhtUY#qJAb(4N!0Kp$JO_JNnZD4
zP4evry7$ja=C?Up`Q6@i*(xTku0@Y@@BZk29z5ya+7$EOi~$q(Pd&3uHsiKU>GAZw
z8z$@ad@b9x%`4^6@AVt^+&HyjyIs<k==BMDamTXX*SSS#TnPyio_c3y`LVvoryTEA
zygoWnrOc}L{Qf_a(%a)sP1zp*aC#~0@2P!HH`_g4tm!TJ>r1xJuP4dBww8T(bi!Oy
zQ}<!-bN5FX|Ll*|8Fhxw-_@_X-!HW^PI2o#1NHgkyvfJv&lvaLIdVvU*@m5`9!$#4
zdw3)-cSq%m;F#ivoDv7wPOZ)|>=Bq>_u-e^XW>gGr@BvP)Lv1r`;j5@J?7f{%=xw6
zk-iE0FX=N(&U&y#N}`QXLgu#P(L~c9zr^ilZE#_FW+wt0Uu3(y*;(!L^K<G)=B>70
zwQ5t^KNGvzd#9~_SJM*izSm)Xj`91wpAElV(a|znyL7>xNmsk4Z>YS>CZ-?fn(At}
z+I-5zHKD)$&6U0#_W2(B{7dN%rtUF$mAX#q#%zE4iutBLRek=K$Y`Gmk*j-EIj{bc
z;v%P|f3{RGH&*Fe%KrLvB>4QtFB_^2wZcBWK9<EczuwZe$k^ijtlc{vd|4d$Md02n
zE~j1lj@Rt@|L%Cmwbl<|;mIGjJb1V~?!m*|>y{*(+`i_N{%zg4ft-?a>|RKu`Bt>9
zUcZ0h@%d6GPw9rLsH-mI;?4JEZQRf&#uXU(@c9<*{D(Uq_9v{q;WBm7*$(meXP+2t
zXWIW}UsdB+W2gVGV2O6@i^MD~?GNpTcFVB0I>f%UpXcxSYx|^we;firTMU$ZPu=JG
z6|p^4`tSLjQ<QI;FR%O^7gKd^qexzK>-qJGn|4`N9&p&THM=izb?VzqXQzngr+wM|
z-sto7(w4(<T-~d`DGN4T?VfgdRd&3e?LzZRN!=YEHg&JOv!}Q8^|i_K<4?OzkM}v<
zV^TG@LT~Z$zQ-NUGE<&!I~ugT*E~-4=<>_YE(&lcH{7^UvFJ`&lJKNLOC{lZA|fUK
z?|lw_e!Zis_wkg=eyX;XGZ+8*%in&tZ(;XekNPF5_ta$9z0OzD^;=%K$??6J*uK~o
zi<eyMI{L|7_`ONpoX@sX>uzi+GyTdqH`V^XLtNxvIW0ymE~)JBJcZS@{J*}wbt{VE
zYd$B{vEG3Hq2KaKA)V+wv$jTiy{Ny&(6xPC>eo$6GdAAba?a=0dYjOk+YTotA6{_z
z*Tgw9&TuZD>uBuHoP9kvF}aR$ufJ9IyWH8A*TkB)*?(th?tkr35~TK$Gi+I%S)b?R
z{Y!T#b|{K0vfXGQD`vCt<Bo0qcY;GgzSurrtTBI6mVDREqZ4cUl+CrLev9o};=8JO
z!n(I{Qex`q@wMjq_WxO!`(Ak+ou+D2^*^@fUy|CcMT<1Nk`=^`EZg>Qh1}fCYkO4B
zMm@Q(&bfS3Q1FWz3w7O`CpZLjui5WCQTx=p=~9i3K3U)3@y+0ID$DQStk5+#Lbg2K
zykLEkzS_Ryr`D=!Xt4aXX0QA@;a){y&zVm_((9|Z5)7j=udU*pYgM)+`hD`w?-vul
z<jnrKyH-nMKW6}!TBs}c>yBfI(aI?an^w&@qvjyMXTHp_VclzqgI00XX%>s^<9*V)
zua&&GA+`DHogG~c9F3dW&DILMQM7;6m-kHedhXRP?Yb}Tep+&1+9QRB1v9w&Hy4G)
zYtQET`)AX&18JN8KAg+QE6RE%eoM~}74h!+dG3)Ojcqp1^kaCtHXXmV(=+g5X|21h
z55F`==uyep4zs7}{;0eaVkV|Fr|_49PU`EKMyYmZqqnIk9^1BD=1-k`SJMTaFUPog
zFYuPghHUOy#C6EA_id=Gv$?~IGygv&-}~{RcunBPC)=_-w;XM|QRtK4&$@AW&4N`L
ztAs2!w`^M6<NNA;wbrZE-VqxrR3&qF7e4*E{llRZ(=Atqdp|q#zw=ka#@l+WH@@0W
ziG8(!%gN=G!*!ECU;oLf_xzjPwQkj+KPQv_f2{Gkw0|nkGj@m5dhQo3gAH<C6x2m+
zYFj(|+Q(xjmhX7~aQn(@G9|ArOnNyvKK<Ew@5pbTdG7tl;nTfcb}ZLik%xKl$3r5b
z^Sx&GMT$(la$#S#|0dyFo!DIxZ@>3?IX$$xwl?zUs;SzM`)VwsxxF$kFP45S_UFKl
z^XVHh?*`qg{p&kvcKYRQzS&V*yq>)c%-@`HmMzWyUxW9#{gYZ>bFb7}za(|{VvD~|
zk~u52SAH>NNZ544aHj5>m@Pe0@k?gMa5SEjb3Oh|_uGW|emip2c#@}gZ%a7dvia{q
zEv~Cy)$C?HJep?IQFb8Q{`W;u_Eo=*FMqY`=>5%S<&XaD=Xw0FUH)0;Lw|!GGx+`(
zWP?WobtkrPCMF~-%bmV0C;FrI<o$E~?>)O(^XVc>lJO?bKK}*T)352o8B9$`vXZ@3
zreN!7@bvi6EiW(i+V&o5bN-rhZ^yi^udi)PJ1h06>s06YtY+&e$6Nf>C(oaMTztNt
zg1&_B?rUc=Z*Q`+uQ#7($sN8a?PNdC&d+@NW^C<Q^(pI3;mtWW_S<{rIaZeHav03a
zopDypY<-mP+rH#Y{WU4iCJ4PgWKhOtuXsS<QMvn)eOc1_Ha|J?d;az8X;Rvjcgvx!
zjukX0rpwhP__I%7;-=e$9jzOSZ}&c3HN$5{>oNV$&o*i5_e<WdJ3LKFziZK(8*NRF
zu8~inIa2rmA49qm-{E<(2RHnde|*dP+|it{jTVo0K3;hI#Z|r8=L)ZHPJHZqr}nTY
z!<LtJ4X^cWr8jGJc>5`-sTnm#mZ}x}z3W`@qVZu&!L{oVIevZ;HJq7+7k9rspH%#K
z%heA@E<|sLJo93fTYs=#+K~&{JN~Zvy65v1+dXV5OSV3JsGhH4r`IU`zP6$Lef<>e
zbN1}68;ei<48OQFSNj#){DZ;D_aB|vtTt)hJh$7=ZbtZMsg=tdpHpGuzt5`c$`X}x
zR%WxNT(Jn*Jax1F-p1b9b@SzC2Ra0KartF0vA&a2I$?@P|IID$9b_suNniPOFl%eT
zqphEhg`3`3%jIfnwr1PhyRVy0?Al+?6*J>%_V!Ebo)>O8=cRb*V8YWK!oNPgw%)9<
z@}uzdIbDad+yZ@%s4Yt{Ub*Gn-T9}_-8$5=eO~G$u0YP4cPfsnR;+Q@eXb<q?yPrr
zcReoK_449ZkGh*{F6?yv_Wo1C#+?N>oA#J~($q;4Zd*9{_G=TlU2lu>cKr*A`<<Y)
zz0RETsMOJ=*YA3~{^GLf+@2X{f1dw*DEa&1Ra!T5tBOk2^G%RVd-mwMMAh%JDk?`N
zU(u1Cez)gU*TU}`Yb&z<NCXKb@Az@$+>y81^5t<(b8fr&I5Fk<r-W@!GJ5FgaUk*L
z-{tdX+-*FqZ_&$ttUvAFl#dd-+-kzC*I7RgnG-g3#t{)2G2yV~vzJV%@KIw8nL6W$
z%OabJpRZ??Pn;vvfA;Od|0|apKYCie<MX$84-cOup`Uf$qzR{8bxcaqS(Wtc#N*$F
z#V74&gX;637J*Hlsv53b519}i(4rRdNI2NT*DvAsXXmCm`J+cwGiG>t`W}6|q0wN)
z+O-E$N*|Wxz27E!B{V!BD6sdcpsQDr(YE_{W-V)Po)Q|e$n@TP)s_SAx|9_4#I`7?
z9zA-c-6ZL8a7TaXdx_Uyu7(5!#kJmirDm$jI@{G%BJq@$m(up1Dk@5c5@Y1}#Ws6c
zyjZ?w5!Z6-HjU{sqt?%#A;GT}dNa9>$JWO;)O6hpiySfIC<~9?>;5@w(&zIBT`c+W
z#l1K3bgzfE7e~{<0G0T%+kNY*9GBew^21Ln{j1@BwFTUJwwfkv5`0_s?#Go2P9fK?
zJ?!gJx)IQSR<!r*ng?sw9(?vGVe<@wLyo6To&MNvVzaUHNL7r6i`=3|YgZn8)N?Fz
zy8xfrgSGr`HJb1Cb&4%owTjL6zU7~Ut-CvVJKYX0*tMu9YQB8?ZRa4TH+>7IF4#YB
z)25=5qdrbaLV4|?Eep2oYMNzb|1N_+%y`Bczd2^y4-*<z&60X^E@RQ%vRD5!<PR-4
z_u6%%Re^QRM4QU&4ZD*zM3$`=<aA!VXv>UGtD-`>o2P6sNxH+=vM~7h+N00DB^Vv5
zc->rKb8&`9<Mrt7tkQ>N5f_{4_V6kuNqvrq4D*_6@8c6=qb7LlS<kUGFRol;{ZO&U
zKvQNCs1}*X^XyWhqusgCNh&YbPd*r9m2*OTt;ZylzoteryeFx61_`UDZ-z3OcY;Qv
zObg*M7y|PBAQhg!-=4q2@a?=UzcXl1s^P?B7Uo8Vg#jF~*Z==xQ+*k)wnTf4W}(2P
ze-<9v4R^Ip^n{=E*^*!;d*^^2$Vd*w4^?|#?CLxG@IcPA>1>|2rY>r@&VTwd@1DAk
z?fHd00udo|@9k{bm%C<Sxs;5`Cvk00p+*OW*z290oj2E;U-7(DKk=wez*T`&yG~12
zZQpN_)W7$=Rw8p<MD_$Z@nx>7O80;Nntn2Vt%@g0v*Vkx-8@ewZQ*CS(0gL>^722D
zT(j0OaEVR+G&_|^@qm2T34P;#@7{g)+;(Q^;U+o0pr9x>i#3W2o6q_MIVqXFmY=_f
zF`jkv{@;CEy3-t&e^);B@44UWL%iBIqH+sT`p?byevw=3!_^@1E1mt_&VSvXZI#gZ
z^UnKP$V`jpYc6k?&416&)c5eql6R#$<@kakW4%j!Y|65qJp6I!!pfDa4oQhNXHUO=
z#p}}kX$-#M7fJ+Y-dd)ip!B}?#oX*gk(*ALdz)~rK9Z;VeZ#T<`J6xf%_f?DVTYF<
zcDlGMUb5sE!#Rl(wdZq+&6?SHpB&pQmwffvBl|z^${*i*a8TWCqw0R9*#Do`OGVgz
z=RW2$$0MRq*E{|5$K~-4uh;FL%NoD$@ZNr#e)ED4ap?iUyJm*R%Q_csl{bj^cUorO
zuQIvD4+m>o56SOcv`{CX(>$i0_kFEd^IKP$FVl=(U96tDui&zy?`#!WJ(gbGJl#uD
z{MiW{7Dm@yj^_w>Eh;!Am?pGH)G3i8d*au_RxftVVNn0e-qQT_$KgVoUxzrXo*&ti
zdpBub&8x_~U0bJYOm>?nm#r?7k?{K3+8<3;nJ+C>)LkB~)_O8~C-=!W6*l7Y%8df~
zj2}8M|E#R6kZw|0mNsGDoQ8$%8ro}>^fU!$A9Xnq?_L?oocZ#kf=P+9vJ8j8|JR;-
zz88z;-Y#EOd%9%J?mdfiBi=9Fw2IC7&%QEQK9dRCOSMb?8F*$&E>S&de(elv=+dJ(
z+FD0n^{82^b1y&3<P;DY=95^cskVFeoH;TvW>c?rKl^w}JAd-Ub+&)2%2O^p-G0$r
zp<|Q#tGm3Vp*eSV^%{t6Dk|#eElSEzJmqzI;|-rFV$++>`fi>hBXf=UdzGcG%#w=R
zY4@rhms*u&oV_Spbl}2@m8%xqlh3nh;}yx%J|KRf!{1L;RZ+~9G2Yj-&e!?#?-ieZ
zd3t!NYu*!<OuKS;iilXi%eBXa=lfK79qKFJ>wW2=$7N+lDOR6VPV=v6-JL!AN7~AB
z(}J2CZO_iRclXQ^dDBx_tX+#1WG2Ymi7NvQthX?%zUkXNZReI8G56D9Yd!rto=wtz
z|Fk`Iw&m_{t@S+n=kxFT#mUpWWKEyV-$NXyp7}pmo&QY0ME{{@d#}3u-gzI_+Z=kK
zsohb$f+bf@uJ)PguBB`hFWR2(`1<tuiz%|l+v1mf`tbU5<=3_10ivv-rW3dSsg9Gj
zZ@qfdu8ptr(bDVl)XjCX+=WD2ZO<j}UJd_nDwyA>>Xg34+%S=78-AOQI?TVeq<=jh
zU;Mt=Klq(^%U@oZ+E=MI-wv#uqBHlD?Dzfy;@gDRtzUBeK=zL3U3GiDKAN3#b5m0y
z|DEQ)OSc!Uoqy&DZ=GEIle$&g*CoBE{l0P00xo04M@G9oe(6v@xnXMI@7C$TsijH*
z8hTn!|IB{({N8K*LsbiY-D^?MQk$Hy{NU`x_m&;q+%K?RL!+x{(F%!!S#~uqCHMWP
zQ!Se<nb`e!*X{RDm&mQ1Y5CFCt=?8VJVd0_PXF5uop<uzG-QpFXV|@Tz9(lqa{|{P
zp~Ir)caAQoi%prnup~lAN$7<0uBN5~S?z0FroKE~Qg=5YLE^(pe=FwdSswl2H^UMW
zf)tlD7EXBoZOVShmm3rXc9(uP%f3GM+)mA$)7r@|@90a|*P8bC*UUaSzfbLY$eGpo
zeaqj^6fdm$|G)6j1L?|NC#?%+9-d))R^sEQ>>k<ga(9J7_Wdieo3$`7JhJ=R4R;y7
zUn|#qGTXN4pg~E|u0Q>^^<{m|FJ?C=dh*!BzwCYZuIcB0_4F^(Nqup7z2VEr{V%WI
zI@qmzf60<9E#~v&I~V`_FY)`NN?h$LyKR50re3XVz4GhOoSJ*4J(B+o&)XCU@2@j#
zEwb&q-FC;RTkLM-r=8O+_GKIY`D%LQN>yOXgUkGJ>6II&2Ikiud9gk2V9cJL>-YZl
zOjs5e7vuJNG26$3?vHmo-}KMt_nYWbSxoF-Zdly$yi@QzcZRn3=YKy=vXnBghK5Gi
z=4;JA7j^h*Iq&1E)7>ta-#+vEea})g0VVLtt;e%=%Rg*gE_dwN-D7v;uLs8NXydn=
zbEHuG-U;dT`;RTSH?ybmy|PzUQ~R%wDNFWVGHETaFyec+w%u@ZX<bCF09W%m=g8iK
zFFC#O*QH-rO)NdUDOK<9a!C&7gdo#vx^a%BzOtW}`KL_`UAV`tW&IL{qf)LVB_c{%
z^N!reaJk`h{95FhIY#q#I&DbHmV6r}dErLaoxJ1w%Xh9llC^b$!1IGIxhBt<vt>Si
zrNElFEpx(Re;2*t+*>&N;i1!d&tA`V+njf|>eJ`!oqzheGiOe8>)ui<dH1i&n~R25
ztZr>8c)5Jf$5$GELRn2%oEAiNc>5ekE1ooKR#WP49cx#|&%dFgG)>C4&Dm!azqCC5
z>EWW#D?fJ&i%K8YTt3&Us>sOw>u>qwee-)hia+Cj^EpGp>ZGn!@ng~FpC5S3Uo<mj
zySil2f{l4k*)~~ySrUIgZ-=hIi#3gB>PnPSP3;(NmAv)a?zht-b3yI#Yc0)EqO7wu
zR<2IHeXF%cKhnF@llN=r){vi49^D&dUrqL(RF__T+D4`?#yN!b3|GrxuC^OHCvmbe
zwj5?^_4x2#`*Y<#9#E;&cIX{r`QuaZk-Iid37)-N=nk79=gBSaZ*Wd$y|pc~TYV`j
zpLfcqX#x{x?8$toleho3WuKhIjD2^bH|3t~@^~HX5V3xlqD1Cb$yw+6^xrxO{LEQ@
zc-#Kh+2IB&wW1>2O|I?fozQr4Rg%$c21koE`S&e$txINkUGaA2!<yYo8ok%cw3yyK
z+HXJAH7wKTb-wU2jfqd#{#yTj^<_feq7@U3f*o$&JHKpJnf7sJPYz*Q_Bm%Gy{CUV
z_gcS6?1@LHYp4iMY<l(aE;%!Y(7-Dmhsssl0|P=tTRmp^rt#mj)II4@kYsXt+3k&i
zOO9Io&-$eDa>3;-Cyto27Fa)+^w?MDw$hC4Mwiac6IfjE&roezu2{`}qf2Mk{bklI
zzIj$pR!)SY^>xJdG||=JYZGpzzb|?$%P)BSVf~l4mu8vWwO;r;+pzkQ#ir`?Rb1C5
zg$FgAyizec<gu`{RF)R!D>=3~RujHidOpe5c40pwX1&ls^768CX_t?M*8KYUJmk77
z({6!@`P>HtWu0$c%Uu(<S<?6a-}UR)?V58{JA6aYQLbtF@s63Px^JsX)9*IPOZ@-I
zW41Q>)b92BJ&Qi)+}f7u+Z}dtkL|16X=`pCcym&|;N;KnO$D`zSD#MZl2b0Se146N
zV!rCseQXSG%+#j^Se_1BtGL2FNVVlaO8=td-(o)V|5zE<$DP_VKmEh8^x}-0+l2FG
za~{62^qT&*^M3XXx{QoY5)<FfikVaYls9E(iJ8bX1%r#<I?8gQ_cffne@;37bC%uD
z7s5%##U7o7$r8Ky=W{3;@H8J;6EkVCXib~fb~)RE4A3$lw@|)2M{GT<CbTg4cC)1W
zhOby19x+$nV(V_N?=hnN)<P#@&04qKX4$pR_=j|n+3qj#2^)77-pZ1&`x2&;#=7~g
z-}c4lo?nYzuXHh0?c0olG1d31;;KG~xvkRbjLhBf{$O`N$kZ!8Qgr<InNGa9(Ehlp
z{@jJzdWvd>yyZDsCpQMVs6H)tmzw_Z%4~nb<RFvG%+4oU&K$`JDy(?a+26}@IqYt~
z;>nMe{5+LS+wEjr^RmA5nOG%HUUBP_;r5tv*CpH5&fNK77yHh?uQu=ayQ-A@ZhA;-
z%a$0wmvaw#``dbWo!aekR82@ex<BB-8}Iy-%g@?({BF}`$hcg~a6P`t%xrPPk|`o{
zuASI>FKu=6k|h;qmNAEin@7v4sHwf5areFF`KQ;S*DZh6YG|=0Evo$e`PB6Mv_rGc
z8wIw$Xpza=^IB-p8nKu_Llutj$9sQrU7QyYvb}f46|<Q7uSfTMd}vYkr%EPmb+@B<
zZ1-tJeeX*Ty&UuRob@qu5dQFXJyWP4&%3pcKD*0leZIGy?e?jTFA;*AFLsCS;CDN9
zME-7qmgj#4TbU(m776)Y7IaG3XxY)<e>^L!JbL+#6$14U0@G*s%sJlMv|!&hF<xys
z|0UJe-mhM`WDV15<yjKbWo3;oUiAAC&~Rvx(jg~RL&L^`2z9e#&Ntp4mN)xQu_&-*
z!LnVGs`mRO6+O!P&HuADB;@J>^M4wx4I2J78|Bp9EId354ZkY(-uNN%DEV%$>{F(A
zUsctop)=Oz1zir>5E;@UxNOZ5mUstcMMXhDzQ>;`Hp~e9^VKxu>J?$(`7N$2nH;9e
zcsYL6iil5Nkf^3=D!Tn}+rsvhD_0#86>dM9<i^du^5d`0-meL_&M&_`W%~5TA1gLQ
zegCt=GnP9rIJ#f`*yhEP5?)A|UfpW3Xi?z}9f^kusZUdXI9tuT;gV#lCvxEZXCWhz
zMT-{)2wb^Z8R%1X$4}bnMab0Y!LL6*?)o-CK)&1a-YhlQWm>NvG=4AD$|~oR<zx)w
zo^#KAsq?l&FC^KbyUOazsx?^BzX`;Y?V8#A^@0RwQN*H6n}l}X?K}O{>Wb&YC5OY7
zinr}q9UP}EHNRb8rlU4X#F>c;;@*qzpS^ieE6a45^n!Dp?C)>B*<|lMNo7r#cHDaL
z-FNpT%QdNZ?oWTw`}ltEVYl0?&J*pI7OMR&+r`MoQ?!{cZx?6*1n2V0p0TmAuCA`X
z-{ehS@?Xof(Sd=fj@!}x_gAhE(MIR1k`-IrZa@3hJlpQ(Sv^l72Mv)$yLL&<Ki|GA
zv)#&aQppX@mXNwQQGbP$Tfa|yKX<Qu(VV7jXG3>wDNWcee`mVj$pk&mOMFfoCccL)
zZkQkS3KC8Y`fi+}#kFeDqFFwFgv%RGC+LAzWcbdh$>;&~g7Ni}LA~5bD%4T&%y}+@
z=dG*X!V6yQs#`Hx$#c>c=|(4ooI^MI%X_uBW`wdXk3E_b^zR3E{bW#QUs;6f;pd+W
z0iicDRXi^}bY`q^Y+XMs>LaV@H+7rnvuv4$T(!5OJUVT6JPcziez{>mrc~T|@#&{q
z@8-D&^40l<O;UMzKHx8tZwtrEr2S2m0pXFM99O?4o_n`e;IWzPo-MMk4oy^H*nhwO
zYSz(*1t%QzJTE1lHgLG^XkPX=_(BQC|7XRA8A3xfRy6gx-g5Mv&hw0Y;==ZiSIrYR
z9Gkb#I<-pbTk?vh{H@KrQ>KM3dVBtWudpCr`^K9&?N*YLR6MIyZ%FPil`V;Nye@jk
zN~JQyDrHuHDC^pf#lH9Do`{~>dwY99!HI-_(v7!-?o}6x6pOF<fBUx9%{?>P+xNSL
z#@ar1w>?x^Zs)l6{*TMI1>4+QBMt59IU1JUuRB!vJnr19rd9jr9-FmE*XH*ZVIEnl
zDT>Z*A8uSeZ<v2k%ITN<+>)kOr?|9M*VbF}GVEISezWVlwI?3`{eD0F=(9z#^E3ZN
zE%TK+{YiHzy9)0l70=%)N^A4i8V0nNa%4@ASZlCg^~`_n5if3@Y7r2~;`wpw+t;nl
zw;yXX|IA+#YcN@npYP$(^5d5`Oplvb+rD$I_uqY5({$tg6_#B1x?-}$=Pi#nMwf~0
zG+;^Mmlk$XD2o*MH0{HcyVniZZfu%(Z5OwAOo`n|x$ANJH?M#DcGa#5=KOtM6{EdZ
zT{z@;O>f4*Q1LnEPn6rKE_t6Vt$45IALqR4zo+k>cIN15zK|Rcy|2|gUiM_i#P<PF
z;w!JTNN3Ib_VT#S?TBsBR=mmiqRaN`WdAw0fBAgpNlTg?8D_CPjQuCHNW|}L+Ok{h
z_jRtmn3VkJM2zsE8@0B7&u@SHPsEZjFWjx+{j#$vt6j~#rq!N*wZ~}Jc8{hF^H)w*
z4-hR1*imTNI+eHNwROYib5@Lj;ip>q?G8?qIK1iT=}dz)bL!qLmASuISA&a-OG9nZ
z;+P#xQfkI5wNHJd{u|osM)vD!O>=x6cPKD6TKK~>qry8CVry^o9DVs{OR@gL8+vDF
z8Wr-ypZ_y+D)0G3?)!%VWs^7DT)biSSq|F+b5_nbOugpcvhmXIAI1_U4<uY-HRtz8
z^79#IU(0dW?pFLSHT=?E-_3V7>9i=EntXaMzx`3cvyV7TuYCCDeYo;n>qMDapZH%-
zjv0P_%~@kyuIPBz>HA{8r++s0r5-(_>GLm3%I-Fok@4M_%6klE`v2$ktJgJew{zIc
zcX#KLMa7;So8%`rFJ6$a`DUbHJ(s-it21xId0YZTe3p4^7dvq5sl8F<E1fLqLn+&B
zn;MtTb#j_AH}!M+h9qyr##>&WPG_HdTcgtKXDhk=tc&YGH#PrXO}e6{D$ln(KjXN3
z^1KIjSJD@$c(yM-Hm!2U#)XeJXw4~VnfZ&M=3lp)ukDZKf*^6>r&~9qZ0=j7)w1=(
zkDmusqMz+vp%-y{kL&9lpD((<oR#<J_WL?khYZGjGk?f!zyH<XXq{+#(CG~ew;LtC
zTHo^Obmjk_%O$_v**#-X(#o^(M{Kv>J@rUe)Vir>(W;y{quJ}1op-rxa%|rx3;+FQ
zeJ`GfyyBW(d_8yjrezx@$?y9&<GbBVuOzqPEtgDfOj0bOPru%H<4uH)+8aJ=v(<A~
zP3o=rax(l!Q)1n(BAxAbzN#>0>lj|EYP(-*EMNQ2_~5T1yNX1X^}T;*&A7sI?Y~8@
zyPULZ=;3+q?u+HTSK45eyyL~D)i0)qr2L$f`(*vB<CD+t>wf5?a!@ku%H!|G=`Y1(
z!nnmYOHRJobv1A4zAWj*^CS;G(`pub*;(`NT=yRHyl;%wW=;}ERJJ@i)y?+l?hpQ%
zdn=5)c5SZF-mddxV`Bg2ZCe_Tu$TuQ-*@h`d`)9t!rP{udwv|!&RNALoND^x!PJR0
zpD#yk`BfBGRb*j2?de1T`P2HB)j#~no*y7TEw{vTQl0)Kqi~LP<-5-o8tB}~F!;zm
z_tPzp)qkgdWt;zRNpPID((5b%zUIOS@^clVpY4t?N@sgoWp#H~snV8By29zcOJ8mJ
zcSMtmNqc!*fvLZ3p*i2-_N5Pvmere2ooqF?^{nja(sJI%+dlUn58pIzwf?TAo%U-|
z0@6#5zvb>)wP=;rzE3s#=6+%hzw?u)Jg$a`_iAg+zn9Zzh#nL?KL2F#+P*`wvc8G0
zHKy5`b<S!yq`7?l+2@bdi_WxOFaBis_U4un>8>QFmRB>4YOU`2@1B!Bf0s0S-J(O+
zKioaD{^k9zHZg{rF_vEn^=wzNpWd~s==Npf4_Dheimkpy^7;8IWxn(bnSOmjkyp9f
zo)0>F<su7bIfbo_@JsBRKACNf&wV@LmA!UttIt`oD&IM<*13Igq|A*e0<Ej31#el(
z(99g1EW*&$nan=7f2G=d^Z2^no`u&JXlSj<+K_TChbeHa<i88fDaR~d%#u8MIr&*o
z$;Fi#T~530)C<4g{VuTn+N0Hb`v2_V>U!iIu|aRX_q&RYxLo6tUDqmV56NtsI^TKr
zmKV;oR$h}zX58F5J2p^3r0spU#EMl-tF$JXvi#X~&mr)@5~eDZ8K?8U)pu}I%{2OJ
z{fVo)^3%lMD{t+Ylsfx$<d$s>!ndmfbp38UD?5<L<de5+OXk68y}T`@%<ZQ=^Ipp^
z-1)dNW$TG0%PzGC)Y)5GKlmOfFR&$7{^h%}W6O#!UNvX1itnF&;In>hhqkKp_M85m
z&K#kE0xri^AI*0w{TeW#t;#;vs{Fu0CW{v{Za4k34-w|cjK~QSn*4~Nqs#lqrumzX
zzRvd2yb<PgJcYmYwYcG*i4U)|tw~WizBw;glUd3?%Gu@EAJGHRfhV%{ZBBmrBeiL>
zam4-_+r689$=^s;VG*mA5s$0Z?tU+I=y$Su;Cg9|LkkwI>U(mg<WaJc=+rlNVnqI`
zzhZI}5D<QTFfp9bI<T;Cg~pG)p)7yCo|$RX*=npS5!GS#HY#E3iRi~Fp0}7UTCmCW
z#VEfCdvVh(v3B8tJz@?AmpDCt#&xgiMu<+ploRDW_H#o1d_R%*&8<~hY;!`YV!5qF
zi?-h04r}>6^A}t1n)&T)d4|(bj%%AMK03WR#H1W>c+u1B4L4#V&T4(#YBx<MqMKor
z_oo{h(u@l^<)*~;$3EUKtH&ScH#NRKs+aM{sn0(jT)iH#v(ET_-9ORDx%z2mUVPv2
zch={RH_mb>mYw;;$9{Ns?uC@CkISqhH~k2&No?f!{Ui0J)SP(|IcCMl?{{XzRG**x
zR;i3lzdn{#e&2Vo>{`8DrKcXX>#GN6pM7-J@Nvbfht?SrWiIdVWfkU1)a_x|w)W3N
z%l%hAzA(=J^y-BA%gZZSon(HjT4BKRXw$92)28B^KI}gKBh#6+>!goY5>Ko2_JEjo
z%q*{`uVKg*k(;hOsamn@oNvw=3CrY1g>I^b9~Na-CjAh;Q+c|VN3Md)Gq7*}L?_Ql
z|JIh3ZFY}(v1?s{3%Brd`DCS2k~?|Vy01I6(|u+~nai$+TV?OQVc$McA^I5u$Fus+
zqSJbmcf7c>*`sq4(}UX^CM-X8KI!(+<w=_)&iwkNn-|(;z<z4x?gdkp$4T}aUb2Aw
zb;bLo$2H@uCqFW^56<yga(|=jf+EI=mX{_zNR(h(*W%-6|2$+~MWuvmcG^Sjh@;Q4
z#BJGmR1{Sw%Gb|3)S=EDvF|~3SJs&Y4M$%pDr(L%pI0~O%8JA-z9GtMOICLqv(G=s
zTW)iz^t!m9ukPmhbI&hMbS|9fc)#>m>w=A``>S2TMOt^=Dz*9g!FI_JD~(g}XS=q>
z<w<-ph@HJWXx==xr_c9)_{4oaXn!v2{v4eLJg$suww(7#`{bVgxoEo2zw1hz-7Du@
ze82BZO0n(y+V9-V>+Rk$$SF0hdq1b}-qV)Gm~BOz=X>XMuG22D{HOA0+1bYv+2xkh
z`ibm6^Lysvb9*P6^b7l5E}wlgPhz6F)sn4m4h6myuKYeVJ@W2e;n~yl4_;lPZ(H@%
z=8F0f70-6%g$pt#-aZ@Rx^SJ`v9s59eo7G*msz~2YW9M9^QrHran!}8C%jPLVf7Ve
zuVtE(d1FK3VZPFzGDS{{x14gHe(&##TXxm%<*r3hR%PA2({@_^F)CMk+i%s*Cu!ES
zs(p3-&hE>7@4Z$m>sT3Id;G!cbqA$BJoL;zJNG;9!;Wd03s&r~`B?I9Ligc3RXx-8
zJJZg-ys*+*H~5ao^l1|(+-$Xew|nxroin$*zQ8y8(b8pdev%b@>fc^E{?>cjQMCJH
z=0zS`83j}A>360{?yxN^$(rx&x4Nw4?S-yMlakJ?V_CQRZJwX?KJ7HugYSA4C0~Dc
zWT&!!XtAsMg`LUSIlJ%3EnlJLBgU=QF1z}y-{)`cMbB%UEgmnqH$$Yg=gzhxyVp*c
z8hUx<_mey}d*x+sl`E*~b**`Jw%|6;nv0q;#dkPMGUs|^W@>g?%I^Gf=U~aJ8xx<M
z4f^(aV>>S|?~S)*KFe(F-vtDQ&UwdTRhrX$I?XLmcT2|2SyPUPSjk-MT(c&v<n5il
zgjiADU7gA7bB?n%`yEL$Obqf}l4ia<bhA!~h{#o^vdzy@mM*`vGCMrSZNe!j`JJ&<
zg%?@VCy6LWKbxfTcb+|q@id7mzeQah?|GOdpQG2h?`=rc_Q0QI8xvmXvn|}vB*4`s
z_;OnQT&~!n&>QQuAI)FRq4?y=U3-N;p?B)uiZ1!Gw{zk3`CD#pdbMY1OxeG!cRu{8
z_*I@+w_@_Gl+cin2W7iA-sV5EHfNHGCwtUV@wR=-g9D`{x$pZioDC8+5)!G4O&8!X
z^=r!(;X7!#mv#D_tY-<V#W&CHlNVXty;*;^n0wUi`?|%)(hKh1b<UhVuVTgIn@eV%
zP5V=~pXb>m71?V&$Byq<wZHSE^!9^oxlExq|G8~F@t@1#B5%+m9=<RIcF8l}`%fOb
z_}B05W2M>Fjlt)oYE}f_cw#BT=jH8fIQ#65yLmr1iF!`r^WL>^!E2_C)4i{(bAIgP
z67}?Lkg3(yN$VY^{$tKDla}jm|MaOyOH1pURhgdWq!JAct}8!$W-t}rFfC0tuYU4u
zo^4T7Y1Gy$7T;BjPja%G-#KMi{Bnb0i-G{hg$oxPEM)%umY$;GY1P29@WO-bNgej@
zEJH*#MEV{%8db16N9K%P*#yX$A^%$Jg^m92eCPz~!h`x&Z$HdAuRNbY9n$XxdE0Z+
zl_Q|8`Xm)k-#Jtj5MS%jE~>~fYuUorzZUv+sd!$>W8rGtz?p0xH)okfLYU~BGSSU1
zAMBsw3hFcqa<zW=S;Mem)f<yZDlb1QWZ1XhP`LNnA5Ev~_FA9KGV|yTd~y4llh>Pr
zyxWatc(=3qUj9(EcSnrgn=<RJqKbWK0h5;e*So5J!KGt@-Z$+T?4?sfTLWU{pS`)q
zWteyQp!e;CdUN&)v6)U4y>jg*f2r3$6X)yy@7_IfyTA85|Dgp3!~G+S|6No2TW24(
zTD1MJBJ)la&vxCz46*EYZ+!K4ysY?V_IY0BkkEw-xXi>d*-c~Xcc!lUnf>$6;_nfA
zcJe5)sBP|jcYMx?lgGav%lw<YwcnwZH*UXQl9NqUt?J>tJxuQ3iz^<@J8cp0b>=1C
z(-PmW>iF4yH;P(%RQAFl9@8rZZ!drJnZJLAftIe-w+Dxc&np^h{%Uyle11^yw$|<U
zrM*LQZ5i4UFDyEJF{a?uM4sI*xn%C1f4uPZug|Y0`(OC}srtobtH1j1wPpAo7wzOL
zou=wJ>6_OE+X8M8)@|k6q#dHxcqMH)bjzuEg4oMx_wBaGU4wT0rL;j_F%9{A)IEQ5
zLa5xXg~7A-PqWV7HOu^%Z~FC}jE4j`Hp*{iZD@_(clyHd_yf`+Gjj5N&8?I>{?#gJ
zy9?LG=WjU`e>~nR;WP7xo%6fej?VahoBF>VnIETgh<DHTKg@T29>~@R^h{ZP?DNxh
z{ld3j`vW6HZ`jWLZCw3`|I_Z}sr5^9l<wdByy?{=-aS9BNqq8J#mU;t(#+^|&|uY7
z*Zl{d7ssquR){&j%#?fOy8IjKP2Tsed;L3SVzu_CivNGVM{eAiaXfw2{%M($k8P^)
zemVcvbQRA_@k&Z-^Fd1hS~u`H@(L{$Wh*hg{GWr{HhQK2lap13^*<4lw-3Hq{99{S
zy!!Ovqvm%Gi~j9*$l$dpK3{rf&PUzw_zUvU8|HLAKKAMJrmqpP8+mNr3-!Id*7oQt
z_sp$(rW{P?Igl9|Vewe|Qxjj+R?i1>`|Xn79kX&fvb^Z^+363yR2W3Zvg}>j^S?mG
z;>?=J&5U=Y&i<Sp>-pX6>;>nqKEF$BjO!wrtQDL6)=n}wnpWWUfAWSu%ib`Bveww=
z8^%^|d%XPp^1#BAohm!d{K)Q_`QajqOsAnv^qx6vhflqH9(U@a?X93qg@TjQ^$(|V
zZ7zALrL#^)Z|A3^Tc+MOb8oX|*FC!1pH}equzvCJQ1!&H2WuGJ3vbU=j@VQu{c>;A
z#<Ght$7(*QUj3?AWc~forS5YJlU~~Fi;qA3Q=I2mZureD8Dg8Kg%|w$*{m7!nOVMX
zwohp6jCbyH&#*q{Ik1-P=c~&bb5?R}&b{omOU}HbQGHA5rQ`COGk&J&_=zu{Tg`dA
zEWdl>_DzxnLc6y2bUwC!*6y0!(RKP!(`mh>*|X2wJs+*u(U*B}Z$Hm+Z@)i!+=na{
zE#4R*Ged;cHL&bTz;(S@hua^QFZ=nY_?K|%@87}3S*N#uc(z)3=FSS~^Y)f4Pc*b2
zwnZ(g{bgutGxx<6ol5svtpe3E9n=kaY;}a0oH;~7Z9PiUJSY9*Wm%9Za7}6RRmMrl
z&;b~3^Gun$*IYcBf37aw)^FFg)nohbgI~5EJ0t4Q;lXwIp}@lgj?GzaqK<Y|PYvDn
zeB_)r#iYhnd*|zGw||r-TRx2r`}A{5<d4Hl0&8YiPt{*`&LMbhL~fXlm@(gSW?`$%
z`+v{+>1`(XY3j#=+{zWXZ>}F%lD6UB@l2afH#D-E**@?4F=4lS1>a%G_76`BpU*8f
ztb3Jns5(PpU9`#cvtK=JKHgZ|vQ6jt_0XRTEPIyj`TyrKPxZT4pTLb*p6NfDcRMfj
z<25g>Php>5T-aTiekf;BkZ`0ywoLh*-z=&3F67<5w=2tV+Y`SzwjVUk^s-xSzunm*
z_h0aM*|~)&fBWy$z0O^u^=aa?M~R1|{Xaj^mJgO+_w=fGe3EONq`<S!8|4jNPcNO4
z8+*pK{NCYbT?`)sx!&9UW_Vv?B%GMx^FN2@@cc^V^?9mlR$Z0xy-)Xk4E8*e(HFL>
zt^J+!xoxM|p84N%zUec&ciror+P|K0vbK``NkU7r(~8f>@^4zY=U+K{#N{RtQIVjC
z2#30VW?5N(8;-4czc1<Eqh5_HmCaqNg0^Kcmf!z6+oXSfrGsAxE9>mD=em9d|H!yx
zQsNLO;-e;<^>}^O<4^LHiAM_Bc&n$BAFp+Eja}lNa`TU5jmb0TdzGmwhvjOg^xMtu
z(wCTdIUsPu#Bz7r^N0EE&uG8zS-jy;xx*!G&rACQl26;LarR)Iyb83!<`wgYpZZew
z*Bi~)7qUFvTX^kdUzWoY^}cObmLT&a;otSyK3z^X-lv|)_TTvCc8b@h?He*&CHd`&
z#k;mQybP{?@~V2?({0>+$K$6SFP$E9?n-92#Fc9znI}!zY(hf?ohDpbBm11!qR%MZ
z()@mj=*BX8r#CW_3y!H2JiTyy=a(gNI<IG)%2iH#w06~#+mf4K82^x(SNY5DO&fo7
zE$ix|a$aXS3~EnpU$Q03f4N_5#?0LGQ(N9!?0jCT(QsDuG};Kp`}zpa{UK9ZzL@by
z8#X<*N-sWU7`yY!t>2ORmUh(5EiTKs7a#6k6J}(Xcf9X>$mE$%dUzO~)flttZd2CZ
zZ_;(fg1t8CsNt8VFJnrdJub=5OFcIywZtpq==#&!i(cK`zA5ir_>?x`%2%(lPjnb3
zYMoAgR(^R$QUZU-<k_Fs?`eD99@`|Tq2Kj%=d{{FclY2MR$Zz6wO*;ImiOzv|K1d-
zd?0+@dGq+Q4_Ev%FFbnlc)%{Z=?lN}JimT5I<qY2cP!sVuiUFq?{@tQx>II6EA82#
z*K&{MN&C4Zrf17IgtBz+o9%Qn|B}t?-I+0^2T!_0|G(F%)#B*mq<zk2m+j?a^&Zd9
z$y)BJOj+EuDrje_Vn1)SO!POwk}R>)S2nJn9549bVf49V&_I;MGig7Y&pD6Qu*vT+
z5~_|dYpzoHcxm$Cn-9v1Q<lrr&9rOOb>+3O`{ETNcGGRTN@n@QX*-=PWHM*Ai$v_W
zm=N5#V3XdZ1AkNZI?k1U^@qtfI3R?h{h-B(;zj?%{x4s&Vo93G;@H&@bu<2%y8n5f
zbx!)qWQTu2HH?mVvt<}$HhQL{qy~gVwcQJ2O?)F!(^qmaFWsc?{l9&Yxf`QAeY$)i
zy1TrOByCM>ac23W7*&?r9m^VGE*rHy;q<io_4~JFFkepp?=e%if1&gCjWvJO{`}mm
zvEwP1O;^+1rwZFkRrbF75wa;NB`EggVGltSmF-JhtArR{TIy!X<u5-ce<Sy|pTEx2
z%GE1BnF+k^@OdKC<~Vmd*KX^sPn)uWHW_vF_p<zcmU~P1+j+&D-lhLP-j(=&TBpue
zZ(-2cCpYABXZ-%0q2;yh)4^!{<RF%VY}~ol+g$|Nx^8-Tc@;fAc=>1QyO4SHFP&pF
zZ|*o-=k-_d!l57E1I(g7i!jy6i>7X!^539$Rj>Vm*H)5qcL_e;^0w{~SM!S-VK$%c
zOx|+-%K~YE`E&S^{L`2Exn`byf9cM$V3o7=-;Z%JE|_6acH!rYtWTAnpI$#9QN`P}
zXwfP*?`LP!1g_3{$ZzYCHm7;r%s?C4G(A`Gxm}Aiw!AJDD!*UL|1nE<->Kgx&aYme
zv9E>o^{1Ur`r2PyHJiL9qxZV{nZ)RymRmftXH6DU^|Y#}yM0!U>+pANo{&(-Q0e7+
zIDeGxZCupg$@;~^!2ZVFH}$qNxEpW2d-lUoy!fZ)T+hu}Ha~9pOk2Bqn(`zYm)oIY
z|2Oshb2m92zN>ew^tMYotEJMG&e-(FN$d5i&dsw=Xltr2G3(C>HeJ|i!c+TRc%Krh
z)XPQYw`Ek0xqiI-=jN2S3Ym8Wn!(S{9iClv<^3@`@$<j?q}=uRdG<D|^!FP3Sxpl>
z9RH-<HQQqCTrcNbTW_&7Pp2r)3J#uhyv=mB<vR=hRYA^$_oTw|Zn>p7=~ccC+qm?Q
zf6wiGx4#wNDtFbdpX_fp|M1=G^SVM?eNXRy{?FaL;Ob%h$rERux%P9lrq0KUT33(G
zV&yKZSk%$kx9EE(b4B%?%RO3gzRwr?scPTz4JzV!Y58{N?}^JZr@l#eJnQKd!QQ)4
zFHah5o9=qx^cMg9Dyu&q-Sgv(=$hY|M%(Xy7F>1JF{nnw{@=EoOMM3)_9&T}_FY>Z
ze`@aZxh@N~^`vTFySS|IPUUsu>q%vojJBKR&VFl@mS^(8x~ucalpFV#-qc>SA)4=u
z>;xwcoo^O;d%lH<-8XIde%x<SoOwWtqid+_$(b*Y9Cy+3zPWR!#xMUXtW63EYUi2{
zuVX4(^ypDQ;D_qf96w&qG)@)bPLDF!B9MD)>jACkr;}9Pc6P9QDzGT(-4MMnUApya
zYtyGF2G+@3b7yN;yiN=*Ou1Gak-Wa}t;5e~-C}2so@0#-b?LE(b`%CzzMR@$k$+|F
zoT_gpZT_Az=(=`pLDl7;PbK`$zhoVY4zvbso3s1fj;UfF!<N;Vw)-_lGflW;GhNqP
z?Zb=Y#T6ANmTGi0DH#2}eWUb%qsuqlGk)iGeuzoC|4$%1KGLPh_P{D%*Da^Q%I62l
zuSxxM-8l39cM&#T8%gbku=9uBt9`yYJ@{w*yUZ7f^6&S4u{fHY9<=9KdY5nKoQj<4
zo~@NvLi3+|T)5p}advdw&pN)Bch4;--h4Ozuvh-&$*<Sh%vGPiub<a@*<_)$M)~h#
zWHQ|^>!}O7zmYuB_q%QO`n`R8x{GGoC-bSt7u#oU3_G(<Z%cmahhta1`bd0J;=1tl
zN#hZ}SqJ}qzvtbawo&I-sp4P#{j=j_H!&u={}b6fwdD#+-M>^W<#{>@Kbzibt}<L4
z5i_UuVC9)ze-GY0rDh&|aOwol{|D7S|G9lWXyZqBDKGP&C*_8(rYhIh_dNXGuNP9L
zx7g5h*SZU@r!*e%o3XEYmj3Q;J6kLH4Z>j(Y=x0;cRZcNy|enp<|*yDa}tau&yjF`
zxL#UQ@B2lin@4}CY9EklJ^qwID*bx?e(fb2zaN(r?l;!zY!r_@{eRcC1&b22XLr5w
z=qmbiHpFYvm;5c8XWI%bDV=;sP}ip<SMf)}GRM-n3&N$RHtr2ktH~GO;`XuLb(23&
z`JhS8jSY$QFK4gnI{I)~?sB~)M|N(sNIsD}u`g@e#cj^jyUMI5J@fSI@;dEt=H;o_
z?|0p&Pt)A-IOydCwdPY@T#JIE3Qwv&zqmBny(rp5Sx<BB5*G1Ud5gWj>s8$0i+b{G
z(p;(jyAPK^#*6qx3tiPOtW4JPTiza)Yc*l&M91^Vzb0m8E--oW=t{8P4a3>7-0UCN
zOMZQ|pSvc-Kl#_%B}>&GclD`SUp(5AdEfBIuIvqLH5Heli=Wpzzqq8-`>5CZM`dlt
zYCWswS*yIPzP>y^cYC~d)STk&j{-utWp{mDRvV^sn`_%ce(qm$EM_cp&QsDp*lMiS
zy14I+*MePBc5Ucnd>(uLs`&iI!%0k&d$(LkxmEh^sHXatV|j)TyL^|H_v>AmD*ljr
z^&E>ILV7YaMt-X$Ctr2T`^}fK^jKro(%R$SZiwicel0k4px9!gKmV=rO!pr3Y17u0
z>(8DvVRG>A`5w#9-96c}H7F#+$*IgeZ^uhDD--XVYu%j#L*~45srvGw<96P?QpT>6
zPN%Q!Oj)Luux;mw-20thdY*lGcd_!j3BMKV@j9FG1q&B4?!Mc1a?A75?9<ymT>WzN
zi*+f(NjY{+OWlB1pMDmebo({0M9sZC|J<JU`%;eWIl3nLpYr@#)9!uD=O0hGCmU&(
z&-SxUTV1x^M9ek$-<(K+tCJ@8y*{+<sr`qqXDUU+pM8FMgfnNYVey@dwi{=rzK)r+
z<nGUAF5~G^SJ)XA3h+3_&0OG~JvA(oORxRVO52b$9nmw<i3?sUxTN&hnu}U(53-E-
zK2!VA`FjF2Gk$*jTrO!<B(t1Hs$;g5o!n>6<*|PyzTY<~*x$XzDqnN@+<IN}J$Cc{
z#m;)aE$8JI{vD5hTK35OwwlTL`+8Eh$jft{b-p`u{%Jma`n1AEZp}5@^HV;DJuxgj
zX5=SPAk=wrq9R{>rCso{vrGSOTz)~(dSVmr>6u2kYEyX?qqe<u{6D!*?NnmoSO4N0
zVjBt$71kOq{`h#$&K2{-zPy^_ocq&kS*`8Vv#wKBQca#O3HDRewtN;K-mc==zHpUQ
z#RKMEulMJoFY_?o)a1H8XVE0N^&g(<vjx|gmF=$86=`z3%Q}5-nj72ay&S)^W?Izg
z?AMF$Pt@4*>eB4)bJpEIG;Yto;l^sbZ?1cL@~??=CAQw))Rmk4vo4ZR{i}7}_iLHL
z1q!F0%?}C;Y`p#USl;&ke|tSAsmQWSjg$B_&(C%3jkO!q8q!vBNpf=SUuVqGR#y8@
zy2ZJ`FaMzG@>2=_+IgJaIanvZ{7`%6&%#S0r<N^S#-_-!apT5zD^1W^iIA7Azwces
zT+jN|-0)_`oN0@-cDH9Uc1$>UHeF)Qjo%MUm&^(aE|w4Z`e1MGvX$9fsgDaRG<0>3
z=9oP$+x_%(n2M*B&dau@b=*hd^a6L^YgBVt5_(TBbaAluPp*)c_6O27Km1ssp|8Ju
z(W0dF!VgtEC-JemHcHLs3|8RVz3knawEM?*2hRzc8>VY|&i15F(V>I9+yBqXD_i+e
z*hzqcB_kt4L55HM)f10NDsPVoaPih@xBYlL&)Jo6_EnddHLTlDA3G?eeSf1pXgSd&
zmCX9TGbj8!&ntc^uM9NicPqdBdCk1T<yD(O3vN7ryRsF7gz+7UaxQK9KSQVN1$Bp)
zOYlCGS!e8|`f|UYLO`xu%-VBqTPyiWi<bOVY2NvzU+^K@tMGo+n2mBGH#Jo}+f@@2
zy8Mn$EM3I&%(sK**(G})$F0X7yelwVWjoFOI|G~NBtAJ-!EX<vEyV&-o<|BRH}Cwy
zFRT$}e(p}OfqSalt0Nwhws0SgcRajIQ)1yS=eZ&;{U_DVy`-oxLAS_r(h_EO4A=L~
z^qy2Q!_wxym#SyGF=!3!v2WjJwmT`7#M)muIgzh%!nzI1jyQVU?*Hz}T((-lI8N30
zV$)3(=G|ZJt2XZ}i4bsl@U`k<scie<#+y3+Xa7sNGOm}O_FTH+FH`FNq|FBwz4~%a
z^h<8?Q5EK0U*uKS&$yJRD8RAcwUwcv;fhsrPJN6|HRKL4ZCSO+cm}u9*$=kg_N{K=
z{gJrVu_|VU*vf0pb6GvDRQ`n?jCrBRwJmSqialJbjc2`?lXk?_nvuh-LUZ*Br_0LW
zEX!{@Cpqb?S|sr4_lIAHj~qR=$SlrbOF%=HmVO~8XT(|S=5B)$t<zo~S(`eJb$6f4
zQCBss-2V07!#R)FYNTGCR^zl{NovaEd6RZ1>-#zyNF2%Ae%j%mygI{O2H*RtGtB$C
zXNPW>aX|i->MCs}R@VK?Cd8**Q2HA1++h-5Z@sU<b*8vud;T%4-}_fn&ELkco>%j%
z)#nSzhhJRYD9OEi&-q^d{XHioo0Uq=l<`l$Ut)eerbJ4*ZswuWm#=;PvY1~Wxu0e7
zAzu69w{zcrx)ZH`(a^f=c%9Qdp|-mfa>myxJBw}^@XY<O<lLT5@4BBXlbz)*zb^KN
zE<aD@B;BiZZ_-*9oILw=yUm%W&t==6>K}2t8-F@{ey{rP6UG*cp4u0eUNPZaetGA2
zlh)>tkPyLZr7Q_X(VhL>htqoJ96BmJ?c?3^uPsv_yi{?${r}3oZu50%(Q_TI>&Y6t
zk75*((w?r%Sk5N<&&BdhY4s&};RP?bX6Y#@#soSv++^8X>}%p3_A{;1Eph6NwNK@)
ziM(p={jaEg)bqdN|5*uk-!Ffc`1fdejP(;kJ((-*zb>7%G5(S#SMx$rt+#$^!}eJ%
zVzw1oKa`I|9Iml_7#k<^&*X0P_hVvQ0g;j2iEr-Alj94B3iFIGFT34;;qaBK>^q)(
zOPo_3<6aVbJg02J#0LvDZRs#TIZH;c_2|+WZ}N(YHWl7n%Oah%@)Mf_e^6MUb5NC-
zAz$mBwUsZ@*v}Ttdbnk#Bk1H94Gpcs&-zr=)wzG?FDkD6nbvmqw}R=peS3_bnN7Oe
z<bO~4WQY5^<-W^3W$xPbvgNB*{%mdG5q=qIsd8io$IsU*KCx|>#naV&dV@vsiIXP}
zWHminyZYyYtKk+#6P(H(aqd{#^lFy^Cnx8OkQJ-f9m!vza?CY2GOX*{ZX5A;#%HUQ
zj_gY}nEq7v>9gJP${8~@&I&!WYk}|d#XDZD^3I$*)#c^0tv++Dn)8l-|9$yj&0qif
z2Bzm1I6uG7_SxX_@|wO$g^%WQJ*&Pu?@K$&#IT*qG;cq)P!sJtV|vl>wo9#mpXH%5
zPvT9xq?vaywUk~cjl6NG{`!8U8Ql)N@pYEHap^G!*Iws6wvK7#NgJzAuKQ+elv-Sv
zw#8a)Uf9mHouwKvWj|AWekPgC<~o(R|M>Je4UIQD`|^z<J<jJit`1vU=6(1>BH}C;
zjpe1Ai@r}#{|GsN<x|gUiGK&qKKc8L_xfFXq1m^4*Y6U_F5y~xY4MFcOQ&q=D$?8W
z)v+zEUNO7=*R0=ib+i6-Rf*g^GBf=C!M)~jEr&N5he}VYIloFsdVTNy_H}QK7<N3|
z_PeulcEjma0l{1PcIWMGzIxQo$8E-pD|vgq9OH1x-v9I$d;GDtMUVNf1q2B_I%O^Y
z;99relga$|lg@4G)v2$O;CnOY!ll!vCx;t9>TK^Q)$*zR&RVyo?DLmq^Us$yME}fk
zpVxk2|LWe~@)Cac>@NKYZ*k`R;arv{S7ra!C`t3qjyc=!RkgdzuYc6~TyEjpK=qFa
zY2{HWl@1)MPI+tStv>RuPtjIabobSk3x}tOi3dgow;Nl{>-i$0qsDqcu+1&i(=dAV
zs*XN?HC<Ik(F?q?TMIMXUL<;Paj*Dt?Qy5Snyo71){T`fR<BHm4%F;>Vc}FIq3Cd;
z{mx(2S$v7S0SA_7eaT#VY46U?{AsUpG)*OLTt4&i?pdE-fs5CKEzsbabu4dcq35sl
zKf_r*CphbF-5S?tw_U$X^wnzbh^=+Tvld?X{^H$^x6eGj$7(o~mWux1eRlp&UsFTp
z6&HSyKb@=BEn2`@<5QC_GP~Z#{c=fEiRCQr-`^K4+cay%lXE>Xh2m>B1x0?nY!2FT
z!(+A9{wMeTlouP$->A8|RO6w*(W7<m>i<dh<t9FRc6!|cqqj2mq|Li5<!#R1cv-xo
z=)(Vs*H5Q=yeg4eoVUNj_p|-;gX-}QZwU82-N9!)KQj(?98696>5GE1k4@cvkI&)Z
zthHNSRT*vPlj|_v&|Cd(PyhUB`42f%Pb~ZTdgrHo;yeB|CC~Wt(0k99Q{7*zA8P3B
z{1#UCuTXRE`pT~Cb9Rkii|rTPir$|5FYrLLZ|c@IMV70#KmR#nn{s?s<(G4-J>HZ^
z>AzDxTR-*aBNgrCl9n?IpO|F(WzCs=`+aR2>*`OIMl0up-<x%ILt^>ESo`VeaaGf+
zjygK$^1Acy>WS93ZMKZBIWaqV-;p<Emjk39ty!J-=+nXf&Y`|JF($Lw&DSj0wM^`=
z)bXv``5*Qyj&{*1@ltu<KmUtje*a7#57xxahh5U|9-XO+)wsW_>)~Pd4cit7ZMWu{
zy+U>4XU^Z2($Tl33QTm6yCf>Y5@wyEy2>Yh+2_}6%!eEN>n};(xa?B3=hQYX=E+=&
zygZfWi@(T!ThH<Ac5XiB)cnX}ul4tON2aQ>rs^H5{VJ%odFEE`w1V5c*B7<CzBy}q
z@8<P8=FZ!>a?R=c&IaM%_AYi<(0D6Kf7i0W`1i*Jm?mF6d-6E%lIY*l7)<i-_Nkt>
zf7%=F{&2n40!4wo5_N7zx9)RuDoXr~7XR6jcYoHCIQPpswxap{$`Z!;dZMnkBX-wt
zhnX4wx?9d!ahkJzuZURq!F7#+p@N5QWE&gUEoQx`=`Hd9-|GeJnqPkzmNX^`J=tPA
z%Uj=O?iCgD=dlHX!TV=<*17V&jZzbMx=ncYgr3uhc{~}rMS2&RuvwHn+ZkJ&wQX&}
z+f8S0oV-~+>;J2dcR$(-&fF=RIa$`ZF3$4qqeCHK)q+(WH;wl%zBl#oWB$9YZ`<M{
zC&?eWf1yFqC#K+RtW0f+bXwgXt;=ck=Vvbe%kSFwX_LV7dj;IbOFYv4WGhR2erVD6
zswO`<E_4-l*yn@olP%sox^40K2A_}gcdO)k(%$pm>CXK4(!#8IuFbzc1~yTW{kDG;
z{%y#a_CIviG66pKs5|$nB#fig1%xD&K5f=nE!Z<}ts={dDwU3Q@TRWqf3=%-o)vY@
z$>I5&R)0P<UCuLX`Qyr;XZNj{VLdUs-Os_?fus4%wb-36w^?^?Tj_FC?ZPHkV;Qvo
zk>*Rjt|8Tbk66YsTnP_VE)iC{y5;w|mIH4(9!-*d_xRI4_cwoT#P>GH-d&gRb<ejm
z!Zo|EoL?HwcWBnTN5|)%D&urJ)%RaqqeFIbrASK$i~r`MUYo;mUtBSizI)+i(UZcG
z{Mj3$izaQF6};h@bXbnoqx#rQ3>?3-W|);4t*xH#c5i$B=Qq;vhCl9bXH1nj`e}as
zy^ZC%D;8(=avGHw^c-8DE8t|pXVTyx@WFIJrd5JkPDgCgs!hu0xe^Tj7tZ3Jaco_V
zYwB@n>DHuWzPG>Cn3gN)?=_fY_V#vh&3@sJF1s^tF5|s_=2z#%)#3r0GL3g_6?JMz
ze6i(yvB=;4*R99b8N68Ic&6@)uvd1?M8ineg^Ln|RZF+#MDL#PQ8jy|-pYmhZh5=9
zT1Inmy{vt-^Gn8cnT%tR>fciHX7=dRAN@A1(n3bFJd|-W)58r-ie>9koE#lgUMv>i
zYhHJnYs#1VU)S&8_t;n|QE_bLAET$tZu_j}?O7ybwl?laxBgz|&F6AYc=fqmFSX&!
z{9?0-zxLOcWy!b0ETqz%)Kg5{wPR1LFu#}f_nFN#ou3l}&+l{ddY!gJ-R*B)y;Fy+
zc}we!!h?cQ9bKEeqPAIgUG}$|HevfDuXyKVVRNBYm&fHdQfopL1lkm|?51v$HPX~t
zxv=*7ycG3Y5xYtZ_tzSP;W(OQV)=?MtSfc&j!wSS)tEllbCR}CVPvSv2TiS?()*vU
z<#*ehc)7~1;(hF=Lxm>(eeR~w(34Z5;sth}_Pw|2=$Su-jH|`YKHe;O|MC68$LlvL
zpAY=Fa?8sZU)wYPojzW9{J%?1?DiJZ+gT^idAeolINS|*QTyqp&CegqG8e7pKHa=8
z<;G&=$nA-``u!DMsg<EA=J}3cMW6K4_ctwkowI(?i&>0G>F*DJn>+iA+@|8FOPf-Q
zcYS?(yw^O|FKdmkB8yu^^#`#RyA<zpemBpU<rLA)lF`knXL;|)#I>O&yiST8WtuTn
zO`UpYLz)y@H?Ymhnd$w%@PF4d%L5nF<IX8h-MZ)7IbolQ>rLxko4mf#oydQ;@i^DF
z7ZpN6Jl{&Bi>qee<Ei~nHm~w$?74q`m|K=jo^x|%BL6+FJ=x!<?M&JvG=G-gl-K6`
z%L76mO`6=%S?F?Yv-{=$XBut(K8l`GdCle0<;xkjZKS(BT}yX*dio#833D(NoIb_H
z$LGm5oz==+SH4up%u`vnzgmO;&VLs1IbTi1W=|Goi+75O`I5K0RPBkuo120>yZ;-m
z+xsNuOqyu7@jPp}s-C#qO&+oZY{JiM52~imcXg|>^4zIgXuYOmfr6gq++O>dldkjk
z`rZ&Ze74-)>vd0rkJ1rCb-DAC*Y_!@8aB?>vzd0VR?Xngyw@GR9_osEQ_n1k2yxBz
zsa83^%DD0Md$~i-{5<#5XZ>GQuw&j?>Fdd#k8j9*9CWYxxpPwV_eJ6DGE9|~6S*cn
zOaGPr_@-3P^D~amCq4S``Oyy1kgHb$;v^reF#dgUd$iX3vp;(hEG}^`UcseZH{<N`
zc=_X;vB5u6-l)5zZ&6Ua)|<#-z$3J|$g?l(gQV8osxJvo|48bb{#<y7(|zJ?rS^-)
zc8PK8(~h)s&;0Q=#wOO^{)&C4nkUBvspid#w60t|{Ho^Q%DbU|6eGeFEdEP~PWMfG
zYsMVQo~Y4c+NhYm+53ND*N?^3O|t|RwrlF0S-#~`_t9r6+Sao(!k?HeO>maox4<vQ
z=Zu{EGNl$rS5HOPrJ@>Y(k~@%9yz|NZvVW6!O_9JSDzlt-27%&!>wCVK4Ic{%h!Hf
zY>=v|`h!V|le4p<Kw|pw=0)r)KeBf8_8wfqk(Is5@aCNzbIlGNWXpQpeB{^>lY4Ao
zj-IZjVeTTDCr{q^Rx~@j)|a)C)iuLO)y}Z>%99okyH?Oq9jBt?=iOjglg2#z^5I*N
zi+4W$d2`RdW5G9EY{izhUwPu<)7vV{_e*p2ibF5E6wQvgF4#A1(IlfY-!vZC`W>>W
zShV=VwXpZo(gmu0H$Pmzpyt(4iG@`{{?gr+TG{TuPM<#gvBD-|ZFOB!_{U?ccR-i?
zm<Wq)ZVEVC@}f#+dVHOo?qy+(TOWk|_n1y^m)|k<k?VbfPiy;EUgtH4v`l=kG{$U$
z{`|ehlQ@mV9vAHD?BAN~`S#3*-}%X3w`{NY{dp(HQ72bd)@v_B^7dqN{I^!>Hjl4k
zG<_Pe&hoj(vs%+WbGtbnd)Xjc=T5US_aBYkF;RS8?JQMk?}93!j&i9bB{p5v`TN9<
z?_c!(_59#acK+ULE54ZBOS)IlU-^HY*8kUQ9>8|i{Y*_do^L7A)OeS5I-h~^d*lDF
zzkQXS%9sCtRf68z{iXK)&t_iami+gmdC8F*|NgOkKYX9_*)H%2DQiEPIZr<7<>SK>
zws~b?<%Fns0sj&aHCNDKz+1P&4+Cy3xZqz|$=a5rW~SD)L9e5)OG#!$_nEe9O(EB>
zfd+pS&7@off4q8iAm#AeEs8CuCoD_te)RWi+$rr<hPPvnxZUJ@Eg<@ITH;gxRco5r
z?_FQH;_lA?3B?0hjhBC~C^-=(VS6~i`{KLvdHk&p&d4>L_M3k2&-)1n_q_ady=1d@
zy<c~p`sbS4k30M9Z04n|U8VJ~pkn>OJ1N^H&6u*~`uo{u%pR46c!?xEn%vQOzgD*U
ztYg%%_vu$}%LKaYlWbKtOPsFC^=`#nXO^xcB|FX5AGRuoA9kso-dY@ce4*ZjWvdF?
zb}h@F?;};YXtq7W{CfQvNrG2?AAR>(l3^kDoU)%^%wO*JX8ZhKWAjnJp6BOnHQy#|
zidtK-H||ZDwc6yD@ACt)Hu20o@#A}bnhhJup14%ypkMrx55`#OSj_c%_3D*|me#HG
z^4(wFFVzs>a5#{!_#%D3+^)q+1uLwW`>$o({_|nw9Is3EzMy4m2}UzLJUkZc+BNIb
zry>(y>nom<_!=ENp3VsS$l|nYZ~2V(Z)(*~X=dDxS?V3l9@hQl;>3TuQw*l_Nr$bT
zx@3vUqeqVnp7<XAANlt7o9A<4TpdG0LMC_CpHwzhd8rR_(;AJgrlUzee@jnN@%+U<
z@oFTY@jBR;*pr+x&q+(_r`mts#ryNe?Rsv|h@jPj76BF}MyG`WRgX^>E@{@sey+^P
z1NWwY76o2nRBbu%`s<?CR($Qun<L}W{&99O#o6neH~;YKXq_#eIEz^5^73^H@}qCp
z)wx^AoQ$8T;>jMOAmDJlG$|=5Ak<R#`TP_^ZqR_(yH7d4-|A1ko&0{c*Nq1GYf>pk
zvQ4^euUJm_r}fi&vdYT_F1F?yZ_6Z4@H09-(Jh*_@AUkI3tn<9NHEmgAE_q5^e3@(
zkCAiV0`tUA*VSaxSWH7jr%VY?{Ig7TVe20eG0`A-4*3bTqInSq`MIYpe4Dy&g4n0v
zm>8dc+m_F?dFGzlq_i#X?xAVA(I-ydciU@!V~5Y;O>0=<6}H?iliG82%3n46Ll;)+
zXf8V^e#hG|Lnr!+v9_7W<Mnz^{)We;{ap6w{@Q1&brO!vng4<H`~RaRt=ZxqZ@rce
z2zqU=@bt&-qRU%q%htYrsIc_tQ4tZ54Qm%PEShB0v~ccA{d*gFE+(r5J>&iJvQ|7l
z=4_Lyc7451>#<Wy_irjn{@8f)-b1g3W#()-Yd_wPdh+?+PPKC{lNC=c7P>Z7*3oW_
zV?^)@evJ-$(@H<9IVTeIWaC^q%-!uy9Qd4<<~=iiyDRrT$<r&B&pT(%ZT6;EIC|Qd
zXOI2&ANtQ8{@`ly{)96}I$OSNy493_%;>=sMxWXH%Z{fn+!X{Gh+EDhaboB5d8ZFt
z_J6!;_WM-dI@O1}g68db9kuMwj=Jq{*Ub4V86HzA9(}s}*>a<L%kbDLNn`Q+d)}5U
zKWqMEEISw3dO!M<_++(BW!XR9hiTv5*7e-Icj2nM_o6a)<4=nx-%s&OR?xn+p~-Xd
zz6Ea-Sc?_E#lB0e^xpZgZS~H7$7b(%do6m$yIt9p1wXpioYiMuQ1UryncrN$%DeOZ
z@Bf;=@8M<5`+xq;=mL!koh!FrV{35R+J{pr`p&I$4!Jl_#*t{7bNk+ulZM4F4U!+M
zU7K|9PyS4^azXR?)j3ZtJD3U|?@zz_%jVVJiPQ2eE*!YFHu@uL>7`4XJ^eb|_^%r0
zpHjK^e`nXWx7L^UrLu{+F5jUr|B{yeyBRK<HecE}OGH3w*S_guxhIZakM~V{`l__%
z>sOv%^0(z?GdXcT{Q7I>lWmDH7uE}_R%TZeyg0n)%QcsIJ3ohHX`RaW;d|b3b6@O-
z>3dG?X?>kxw?LY~yMBhF=K<f}cYL-?n7>Ci`lA_(r_WghTUAC^!HivPa@RUfzd2y6
z->bB+Qk$WP*KS+R&91yni%nTBn$#Yz$VkmZ85<PIk3Gx2e&3;qjUq)t{tNyVF8}OQ
zCSMes#E^k;Vs&bzWLwtKWs9b;@op-fCHT1P&s${~KGO&MH+NRP_`BtF$I^nwMfa*+
z%gWX6=@Oq;-LlPn?vdu1cQYqGVN_h9rN3~;8lm0RZC|#8ya^L_;t!6BaY^-+J@=x*
z=u+^886pn-UnWcu*}QdC*NY!1mx6=>81G&P{OaMzY3t*e$T;<0mCeVKpiAt`_>6vM
zY2645=~A}pn;_@LA>gYr!#TO&8Uw2j-qUx4GWVXI%{R*oG{!%Hb@?LWx2MnV{P1OQ
zNAZjsyB>+xMeLe*#6_Kp%WcD~@V5Ks+2jARv^fTavCpe|$SPEGx}x@uNZ9lr5>_U&
z!sBXZg&NzX966m_aO6l(sMv#7)pC!Pb^G5-=knU7lm2m)l$>Olx^dds<lu)-x19dK
zkYE(;aX!!QvzTt?y+xMdb8LFgo3CH0SaGVaVZ-VpZ??_3uvgT0|Ifb9zsnDceKUGH
z`~Ax+zeS9aOXBMO=pFNk^_=6Zn(^5n=ZJ>ii+Ox5_msrHg?{?i-CulZuCnE_nUg2-
z-#b0y<^D$nJ3IS}AMvIp8m(OO`)T$&iC-rBHq0%kRdwgvGwb;K9W(sh_Me@TDjoUn
zkAch$i5tgHc0AwztW{k8c+{Rw#qak{cQt99Y3Vrq;#jah|MJ>dt%3dbPW4Q^YT+xj
zRdni;Z?@CL=kM%4qVf8^{UI^+NXxS)PWC7-k1P&)eCJNxdt;r5spltpKHc-_T2349
zZtnNLcG`TtwEK!SpHx=1Mb(AWJ5?_w&-{Gkd{5@HYTEYMZMOu{V@rjn^Z%1ym6h~(
z(M_8jiL6&mw@O^IShu&9BVFb$)6$JLj_JRa@K;u!NpaHR@a74>Qy`xH?X}a-YoR>y
zc1>GZr|!5Lc*8}vbFP1e(dIp^9W0+FtXrV4;g+IyVCpZUx$))_5^OcUCZ9VnPwmp<
z&~2d-GeS5rCr;aWrQ~E@*~<;d5@CK8Kb@K?wSOj_6mv*^^?dTlS0ys;Tm9Kx6K~Ez
zJLu<7c7BaZ(eeL<w^N&6R(@e{>#Ldb#)5lR-m_i1=RJD7F?r|DIe9hKwNj;t-zpp9
z3|>e3NXqVbGfDfzmJoyFdyiyKw-|i5dNt_z*Gc>9BsYucDC=2@&OW;3xuE;x$&>xw
z9$%(vziH<Y_nX(I>cj<~T)BKM%R)AtdCz((ewrTZtM1R^)Qb0?cYd9{ucArTs?!`V
ztxl)cAKdZ&+J^_l`5$loD1SLA;n4ffixUDK^A>%oyl#;8=eW)16TUislDPiozdzT!
zMCUzAU;X^=Z|j!7PhM87C(!0RgIVA(FIP(kOAFJ-4t9aZdXbZ(v~RxHmA!LWlF!Mk
zRK*8eCyuRH(Cu+Wg-P&`V1eVJMT}FNca|iqn>X{_%a=3fz1)8P`(68cg&o>LtLGeg
zdG_<lS?gxLo45V`;(O1(|C{&yyG`w?n<)<;?Vf43YEJ#0dd}|r-)w2=)*GLR`4}Je
zKELKsz}jP<OwVsDJ6r!~h3$z`%j4YK1^P`Vi)%gqxO`sn&q>vmAMN#vK7{!8%OtP<
zeX#82gojhU^MCy4Zn!CO?y?G@(<e4$AD7SgJniF$NymSDwAWQz;<Rh8?B$@Hr5|nA
z{5WcqH#hENUu?BUne62V=eC*K@1K1Y+Hah@qP_I|y5*KL9XT2WxIS)nuXuJd-uUab
z_nWJ(dOh>q(Nns=YThoFuC@AWpNCvJeE9!Ii+-CQ&Msl<Yo7;1_1!(X?)A6Y4=K`X
zZl`Ye-r?!Cee?UamZP^~Bk%n>aKvE4{ngExPfmz-mpB?3Pp^_aZL(v{pX@JZOFq3i
z9W8OaT*~^M#gREj|DW0S)t-HmjD5k3ibYeeZ~r@ud6vSpX^Nrs({!XRHYv!h_X|uE
zS$=v^))Nl1e`(U%d~cuK`I!7--KXxd*=e_>DoQ_>ec9vnujYk-tk<mj`S;GuHLsIh
zzG#B)_D`pm+9!T*tI0ZF*CM5nda^w4=-R0HyYHOZc=^TasN?hgvcJ5(MR;w#>SOn5
z6Q7$&#NF$cogJUBJ+<(Q53^{q-R=9D0vw42PwpP(k+tr$yzX=4c?)a#KiPb{U80v;
z7@vm9Sp3Kiv)?$U<a6XL>k=IW8NCDViv{~c9+q0~o}RuWSjIl~?3vl`5@N18Bq(&}
z%{pY9zu#YLuItsB7~@hiQRZKhzWG1wV*KyA{G##E_W8Ci%EB~797=kZ2=SfLUR6>0
z-7QQf)~zyB^zyoyeg+Sc3o0$wzpZKB>V9v@yC<4FZ;N!eeJ(G^dK7wQj@7&c-Ujd6
zD&JIPFn{e7ak??H^6|~dSGH~osC|<gx5C<OkDg<+jc0Ol&*k*<hKaw<_N<iQ;CayY
za`KJ46Rb5~tIyoI?)L3f90eQ~N(=WfT;3EQ7}2mUn&Zs_edEt})S7o*c^wh);DcfI
zx172UkJBrwu4Y;t4)u|ZKX~u?8!>lAr-@3d+WHT?Zt42O)pV4((0Vu51{Mz23->Cz
z=HD&nUpXo8%>FX2d*7m@r1k#&l;8VbEnhxA>A?%XZTq&#F5f;uIREYQ4=1bK#G6;`
zoN1aaSganJpvm`(SHdOJ#K-;e%=59!%x}$KIoa()d-yfM6LXU9T(Q>pWNrEJnps`N
z`|~@i-hK6vvYqj2ar@4Q*{e8ObFRI}o8jxc-8g1%@6`UkJ=gDLreB=<`NFS?o8R4>
zzyI5!v-{$|GyAqJFs}Xg=->&7GPZ68#a;Q*9E_{7`14<JF8Y12?AedqGHde^iy{RN
zt=e||sEE&uLgoH1l7|}0?>+Rlu0K9+@B5Dz{@zb}y4X8IU*_PK&Uk0<tAX<N`%^9(
z>{_*n)$JOKcS-5``Tq;93NER?nefZaW(M;w<E*TPNgLQtZ!-Am-`CKZIKwjh{@k;-
zTJxDc>`Z)UGPQZRZe0EOcWM1CEZcp;EGteWov$u4uMV63{Y>%PH)+$WR1ceI${Y7C
zT*zV{xo_G@<LAM%-XGlJ<Kw-w=#58D33p$D=3}>b$ImyDYeJ&?nRYNMN}UTS{&GX|
zR@7vZW4Ctxn*Di(QKE(K*Q-mut_wT8NzXt;{q!~8*&4fwq_o3rPhMJjI^ypUu4DWH
zf9h`AZJOq=!cTAgo=Js|j~my$$?#hIJpI;&>E#Qb-9NZ>bK0kW=X&>iTW*yWW6jvG
z%|&>w+l0vLXP2&8GogR=`H$0cU;Ruub!bz_s$+dme*C+B^TTY}nu;e3*<tGz8eZ(n
zd|`6c+-S!qt!HOuIH}!q30@`3aP_5AsYT`c1CBx~``$fWsb_z9HQUCw+gIP%`2W1j
zo&9sK>i?g3P&E8O@ydv0H)}5@eDs`Jb-u+cHvjRJPW~Ssnv>t$`NYJp8|hc^qT(38
z_F)s>-R;V=({7)BeWUn?j`6d*XYNQQcU-@AVUCAD_Nxb{R<BFEG(&6B*{>yOZ}zH*
z9DKNUcSYu9y>oM{4z)cu7kClvR(PX5Git^%&n@=_x|o`#tTf0vBXQ@Zi2R%tj-Od1
zjaKyRy>0&C*KT)<KhN~c^6vWHIx{!z(xzQ)OJ084u;rTB!HNC4>!jBjRCPFANM|p~
zpZ6eQYir^h{(nZs=W{_NqhxD<Mc1p(lb`E7c$R<nP-Zg!=JY<-&`TmB;({NtLV9=4
zdtdijL;K&`ylI+T7wew>InBI%Aw#pf|FR2zx%lsV4-T{a8T{<7Npoob-Sgh_1XNw)
zeob)=WZBUq$aU!L{8Of1Lxkn$txW&bvs;7f=G5C!iT2OV{Ii<fFRwCH(s{`e{<!I%
zjkEPadObYl|J>Wdofo<P)W#U=N}aF&IC~^QlI+cMl!Ddw9Qtb?;h_=Lr61!|Xp!)~
z{SU{x2I+-XC-wDI&o4BpITiCS@gQTE%^&5um!B`}&o^{t|LXdgZKh?tT)$5Bq@U(F
zOZ?{tX|a7fzEW$&z8W?E`Nx#~b(SyLFhP15i-z7w_ri=jJ6=@JvG}R*z3#m=!<7Dc
zM`OG5jx3CLA}%WQ>K40GseqQwk;f&UQY}sO=k4Lio~C#3r9ebibK>H4kNd3i6kmm|
zEZo&8)>^nL=8LA@v!kZ}&#u#zcDOoYcY&a?*6JBQ&vx$l@g?Zrj)U%wH(Wh48M_$H
zK3|>{QU5JB^7#8}W{X(6e*UxFF8t}~L;lvln#vZhQ#)V37vJ#f=GQm3x5%|WOTG8@
z{?W%}hROe!?%Dr*Z}aOxZAFtHm)B{HKDOeF6VnbD957%x)Ut*(`u?VXmZM*VMS?P`
zgJ#C(ny$=aaOLoM(~x)SjLmY+*Xwz?rY@07S|;6@8`&^3Q`zcwrOdxN@rvoL-b#(C
zpEy`;Q;ahwz0_IQAfG6?<;d)lpW6$v{do3HzBQ|ONon_=48fm{^Y6Zh66>rG&dqGj
zC={9acFDGFe8CwmN{&Xm95p0d{I^fKrrUJTp*%TvX5_5dvkHEH{`klBz_}&CDz{4y
zDcY~?WSM3DW99X?|4lPIkG#G9Y0mu3xrIIp1^IsMn(w<@=!&VyqMghwd-Ql@SeWD{
z>2kIl?!Et5MKJ%^Q~rlVRwv4qtz5bG=mEzq22J-C3UjaqoVXMa?!HRdTSe&TrP4hI
z75v(p?#*1baHC|ue#`yW3sh&SiEu5_YC2cab?cVZCJPNc&4bU+F`r(tRJCzc<;TiT
zySA;BoUY&Fy}N=>p!+BXTXV*(*46iVbK0IgW4XDmF81n5L9W&dGFjJO&)u@bij(cn
zibL~UH0I?v7ys7m5ql8AT9rO4VvS(11xNdz7j3Qw-n+lg*s8sG^V}oRw?42Payi7<
zvcY5Pwx(I>`TeOEWV$a-@_Nk6(kapsxO>;5%ekIOy@fJ*`ic9w?=tMaKV6t{?@Cd*
zidzddKK*m&%{10o!q0p{B(BA8DBFGV<t+d6cjuiv@Mq(Pw=WA{nd|=1e!k+0^vMLh
zSfhBI8QweBtz5}D{dDW~*MC=qKh5_G7W(&9vpJ&sYUulf>wFPn7P4gl`QFj<*C{e=
zlI(5`*3FxB$?5OvPcmm+H_eWnv_x)crqsz4q3NevSFMWw`s&YF!`^j?Y5$j>+F+dg
zzhj-(H-8DW9fp#Z{J(^FwtNu<?Oa_V_w-{$MsDuXWy_u|y8AZhybzO>)dJ<WON`Ho
z&wAl@@X%U`GkdMPpE~sK$)9%^bez>n_m!)>RD=}e`h~lXo=DsOWBLJ)ct3Zy#7|FI
zOH!_!-+#P*-*?&l#}_3nICO%`+;91TDeYS6ClmB)S*!B)Y`S8dIm<0EQSc|LppNnP
zV_DC&7(+s9Hty9G;BZ}_q4lZeOs&y5t57Be28JD;E{-8Hyr<8bdDLU>>tD`hpa0K4
zd3xKu*OMce&++Sv&yYPE(bi{^wbPxg_~oTd2QspkR6g%_C{$?Psq$5f$%lFIUcsy3
z8vbWxl$&>!ywu>6dC2(f{ij3mld4pYK6U>prhG-;-f=?7%M*pqE-g)-Ib){Erc-W{
zsw`byFES}Sxo{{(5povR(;atyvN3{211ITuKnDFz!48`4n+)0iTQmo<qj$>Vf8rg=
zGS8KrbjlbQ7*tDKBT7;dOH!?pi&7IyQW=a43@vmGjdTqyLJTadOf0MnEOiYmtPBhy
zRqFguH00)|WTsW()}T>&>=Oe6g9gZk%;aRN<otrlqRjM+5(Z}&OTn?AASYG9GcQ?B
zM*(EBz7MM-0|SFNOixN`adJ^+K}lwQ9)n9}aY0UErMUFe9SjT%44$rjF6*2UngF>D
B#-soM

literal 0
HcmV?d00001

diff --git a/tensorflow/lite/g3doc/performance/images/iosrelease.png b/tensorflow/lite/g3doc/performance/images/iosrelease.png
new file mode 100644
index 0000000000000000000000000000000000000000..a160c6700e60726d8d9775c4a1c28b3e34b1e930
GIT binary patch
literal 75374
zcmeAS@N?(olHy`uVBq!ia0y~yVBuh3U|7t-#=yY9&a0!sz`!G)>FgYknVihP;L$mC
zI(tO$NwJ3K(^OPcwm7gXX}Ra}GBhiLk!1;&nUj!*hN23Kp!KE|2ciNtCb)*X=43Ob
zY)sVePI%AHt#N8s#D;~R)HoNczF+;m`uXo?_w3Wp&9QubXXj*Vh84#Pjy%c<VQ!EK
z+^1q@GQm>OvemPSMZkeUWC6F2Ma)VI1vNDx2gzOkgYVsKxE8SP*6G^6cmBt_PCh=p
zh|wWQVfn`;cVtemGak6RBgxZ_p+#<m`Rr->Z9Xl5Ou3vq0-JWuF`n{9@wthryH?Jl
z<j4uZi~$RaEGF?VTzOJx$h|}ILVpx%3cH}cM$;Xho71%)d{C%wv(Zvl7vHqYH0AY{
z_kU9U98SNr!Jkh(dB)V88ZN3H8lvgN%l^-Q`s4nj$-2_B4)d6;au5>JJ|7vkea#&X
z%V5R%!b@FUov$_3_?`&9aeAk>0=Kusmj4H)Zb~@Iz|1h^)MFM`2eXD8ty6tnGFN^t
z<95C=A@S|SibOl>I7_joo<+&*e;2pbWER{JC|2p(`NS-B)0&nB$NADMHQy$jU#8?(
zGs(zr%UXvS60DKDpKLS3Vva00`&o%mJL`m-Quoy3Jo7EtE~zSR^pZYf<8C!Y{F$o2
zrp0WQM)3mj%VfHJ9~OyheCc%ViMP57-_etg*;%zeAL{zKsO8{~jb4Ry32zqs4J#}=
zv%vP+ne%*8U;8R<_u_ukcI?};!$+5$5odq&XGTS8&W<uQbsINH<~(jCMs5~HwFw8O
zu@yP032ggj^5sDMsd)ki7=<P<{y0z<U~Hn~c~QCPx<vrXzosMXmmU^2cE#CgH2l&(
z9QE|7+!F07ZV}~84&qN_1Q<6x%%7PwC8J`S%LkUM7W=d3S~sR0R(t>3^LU8-CBf!S
z$0G(w92eQHAFnOUw-Ji_n^=6vw&4829s6zy@8Wpd;@<MQh40YYm4}i}@=r8AXmNot
znMd-B-rL`A3cO!j73DutyP*Dsbozr|2jt2djaeVm7377l_-hw-v*6hMEXx<&ZyelC
zDxdh5@>*UwU~_1(>^G(LSH5ozI5yQRM)z95`Toic`&P^I-&3yJazreNeZ%G(oNsKd
z^_ViKmGNXB(MxoHVzb9=p2oFf7jiasRcu-y|Iq()A=~pQ;l8f|o?m$Tp>hNNlh53r
zpXzgKhug#|oe_9q%={qlZ2BhGI~Nzv@Z;+%5TE+~@2Ry<d7U>dSMK4yUs8FT@q<qD
zsh)f*zB|o)<%|yX-+j}+N$LGw${>^Z_>>B(m#p6Y6aNFRJe<9Vz30KbR9@pEro02v
zKbjK+_!c?Zc(4U0%w;W@>vh1yK==&v(giX)tl9^oHVEc0S|7;UAaBx;R=_leQQT2_
z2j9CxTQ70OHN<`3`=ZR@<dnf7d(pW^koyQ<p_7e3yoyApOVY#?715IpE*(5GJI}bE
zn&_3HyGe4TYn?lvvigM29@a(Dl^rF@>nHkav;_s`UJ=;pa(sbY27^@NvBi@w@|f`J
zI(c6>`XXx!+uhb{i`6f5zbM_pT-*39ks<h~#0<`~{>lU$9VzdFmkc@0q_q#P+8~v~
ze~o4Sm(VrLuMgTia+%X-Tv)v$?(heeIo;16ZvMzsqq~oJ{c-<?;UC*8^#4tDP?ivQ
z!x6_~mdK>oY|tCvwL&RJSw?A2+j~Fthq4nrQUum^y!&w1TP;l}T|m3zb!T<w@y@qL
zjvuiT>RjT!N%`l5MV>N2b}OfBY1`6W(o@oU#q-yKMXQq5JX#{OXwo93AkDCqK{r<x
zg*@##6+E?W6Wh(EH?DnZpHEyq>3t&lr0<i{PyCAPT_-2Geq4AY*hurU#%j%Kjpdr(
zHS$A3Lw!SSLyxcK3Vpt6UD(~_ReoO=zH(nBc6HjRMShD;t>|8Pd*$(!+gIjin9Nc=
zJAazf>E*i1u3qxHBzM7X#l@>`t2bV)$SS_F>gubj>sIUyQC}Usf_;U5pnlN$Ao~S%
zk$c^n*Cehyc<DjLnQJSqS*$4xUhF@4-OCk{YiBNwT(~p%^PXqMx|60Y4cj2*zuG0c
z`<mz4<0U<LZs`|ye);)j?U&16{Bjp<n`#zq9lSfHdVAUGvh{t_`nvrV`7QL@dQNaI
zAKS4FDH|kj_Ney!?UC;BSLShN>3Pr}VUl#dLtFXY>%J>ruRnb~?d7|xd)NK;>|ZIq
zX8O|YKI<Ld@3UK%zuG>oe%3#>zn6d6DsOeK>VMLwBfYc!i7fM#1J@L;CA1wBYxZWV
zZF6f|*p_tIv`wA2x<|t8pW93~?VdiVQt8XRwY{y!w0rs`?bBZwFweMo=IWW?GjYPq
zlMRGto=lq@Hn}cX`!dVrq|1AkeLt&_CYSayP5VsLnX0qBXOGW3Z?fKaf3oAY4;us`
z)@-uK{&?@wDWj7`jj7E~jkRO7w}nm&oqN?bd)C(ZrSnRAznV%f>TUO(cD8D^S@e}_
zSHlvbbE|LiZ7tp`d~5CYY<X9Sr<PtOtJ1FcR<EjUsBNsB`}gq0o{JL~ce|VS)L9;@
z@i{dkHThiM&sPs#O?bQE>ce%5rzP)5-k3c3akhH2`nuq)mrc`W6?c4=dG7W+T(3;8
zW;(Cl_L!IJ9><99*tVmtu(*8kj<<!c%goFDOXHu{U3R{$d~ETd$8*#V8eb^xOW*w5
zUEh3t&HBG__0_kl(<|S9sWZElrBnar+Q$A`pIQsM*|k<RGxx3BH@n9AkIx^AI@7xT
zKWG0`{=5BWKHCkJ6t*{96_PImG&pTUe$0D%bk<QTxfa$|wyCUJ*|J&HS=RH`2_#8P
z5)2Z*B>hUtO8HBLgrbg8PUH>WZHlu`uS@;jw!8g2m%XTCZ%228^hb?LW<f$XP55NR
z?ec!^s16hhT(qJsL*Q&*oLsH_F)cOyXZ;_Kd^CK#`gr+Kt7GSSY^B}%XG?82eivJl
z@MqGG-JI7rH?>T2`Fg~&b8F}O$xV~ACR?fX%rwY8bTs8}N@7tmm!_&=YP8q=BM&;Y
z)yw>f=7w0kJeIQhWb5Sf{uk#@vVD3Y<m<{gE7^lMS9-3r4z@2|WNA8QpGn!RU0dSx
zx_5W|ZhC$``rzwlzdvaI^uJttN&E8r)E_B2=U<5Z;9SLG>ol!k?T65`h-b&ne2=^s
zd3!V4rpcSWKU?u+$uk+T%+5_N%Zqx8xQlNeEqfGH=sqv}-l;uto-HY}MJD=9GkcbO
zxc`l}QT9#!P3^4vSg*3mFH`li4p<SXp|!{4ZrQIzFa5iny6R>(o_}yIV`ipN@x3$c
zo0-yb=k=cPn|;^lc#hx6Ip=mREG|E@_{`~9-_tib7p8u^{nPXFY%d<~XVYx{seYNh
zD`;zw=gYIK_qfgUj)-%KZ__J^+md$ix>4NC)}r=Lt3I6<z0<|L-|FA4U)LLtzU+>k
zzC-ushABHjwtP8ev_I8a%T&8tOI}}o`<^=1V%M8p+qIXkvx$Eiw6#z)b!txN@mD9V
z_GX{Ct`qyYa&5xfudfcg+WVU4nr_thI@|oeQ+fYMf6Fns6S42(WwnRPqGvmsU%wu4
zEhJYlPxHp5t)_*qZ{D}5TYY!g{<Sr?r@vkFR@OSHe9o%U^=t2DZYcJYjZ$Cz`krZK
z`0aNK?)Fx{`JQv{^o{6!{BI_|t^Qa2>b?0l_c!^L4+;~`J9am}=DWy0^{VpM%de}Y
zS$Ji8PxiiTKg_?~q|#jWSMnG6+p~RVt?qvz6InE+Xv>oq2R1HBu04MAc(q^3oQgX}
zTc5r=$*I16u2lW1uO&xb7N<L%n`0wg@a=}>{Nl6E_nwpA_n>g$!$r4`{_<Woo$v3<
zU(XkYZNK*G^t<WOw{33Ex^;Y;{N6|9Pmf$(usVI+#`Up#<VyGcbvx7>ZLWUz&5onH
zzrHE^_BOnn>)n_8bF2Fc*1y?z?C&aeHvX)Csh`*HmCciNmt(Wt^F`y`#k<Gv9M_-c
zF~8=W&dWump|+-VQNPc;j4UqqD8K!=^SpI@NBo!XQ~oXed41RPYx?~E-v0l-^d9Hl
zx$9Ndv&Dt)xmdmV@7mM1r|p}ukNthoy|4TAe=qo+c-+0eKiyunw(6GsTjQ6XU+J60
z&)Vl!v)6orv4O%f4~s5;MjkU0TiJ%|w;30x9?ow!W6%(unr_^xaO(4trUUg!Pk;I<
z<}u`aXIDtnW0?3sI+Zy`Y{JS9djDDazq7B{{JB6{By-p1$(O`M=S6VHv$5TiU~<S}
zdeEJcu%Q0$fdkXm8r-<eAag!=9_Ldf`{q0;<>lMXO@HA2(>&u|q1Fqopxq1%3<}8}
zLB0$ORcZ_j4J`}|zZe)88eT9klo~KFyh>nTu$sZZAYL$MSD+080|Q%<x4R3&e-K=-
zcll%n1_sUokH}&M1~F9-X7mhF`2gyKlz95OvOnSxVPO&ZlGEbKz|g?p>EaktaqG?B
z@&=i!Ggt82FJEP~{AA^liEft9dnPanvYJ`mOxSS!Sa<o}ZP%r1-v52~?(W~W_jW&g
zS5<pGTe_LkjLpTsQ(~f{!a^OzDVt3!%lx0OTlM+x#mj3Y&uS#_N;RHO?w_&Bs`Om+
z^3_$ZYR^j_dH2{>sMTqqg}2V>Me3K%_7q;-G3Qu8_u>Ez5w2FJg#j9E9JxDK;XII_
z;hz;x?Lm6Zd)z+vui#LlH%H^LmkLJ;x|c)M{a}7Ng^kHKrJk+ZUZ6pDv;BNFCypcP
zHJ;Ogl%A&BM>%mYIw`!E`#W)mp5mrH)u3C?|L&4pmi=$fC(n6H&!09fS*V;l`#b-(
zf(6En7hfM%RbArK9q8EnbMdK<m3t5CE(5tsgsb%t{{%-~mRSvJeVcN9lP*ZGNY98=
zof)ahF2Q;3YK20}f~6ld&hon3U91$ns;s$Qs?d+?T!FOOk|`zS+b-U_<e(&X{PEYI
zpg7+Z8=eTC{`xy<rrPoTH|`8qOOr2r^}m18g~M^j>gHw1?F=i{$hxlBP;p}F)rbFT
z&YK6_+F~NJc<oZg+je_5x?eW&(q5&xTE<Z2`>fpCN0K%&K7O^U@AzZIi5>}&*BB&g
z?Br*ay_@L2EqAhq%7+S@=U3)W5xQ(*$di0xp5x^+1sanU9uIYJKjp@-O!uYFl8DP!
zJi|>+h6cMm`1-Rg#ReP;Am8R#E^SPRp8VipFO%<=@NZl1l|Sj9waI?loBx+`n2ohm
zFKa)2pQ<9@bm2}x%d|G-kZV0oMOm!A9E}baN^Dj~goX6~**kUTLlYl|u*@6ZKZ?!j
zlRqY}bKvQ>3pekby4YO3a?Q%*H&13tv^}14%wo2sMO6Vm`}aDQl*t!+<?NpZhfZh}
zDA{7RvG(%sioJCVv!wWpbH62Zoz`m4d2^@s>Yaio!fRKAhxFSl|DCl%!0E!Jf*#rE
zYe6AFjfob&c5eQj-K{X$Z)5W3sAXrGx?b%n-ghrT=h$R_yUyI(m8^HS<(9-gf3<pj
z+SOH|5!0@|pRp|SPSxwREKH5XQn$C|URx7+c$R7Qj@RpUuUNVA=ic94Zi{RFd^{c!
z9v&PVY@BjTc=<zxq}?}#&P{k5_~?qvl2|r(qrycSC4#(a>i%2y!%}9z9NlIq4waj4
zT4y!fyt?;P-TQsIy-NQ#uUz>5bkVsPAFf*Ryo&wcxnhmv(Hkd?4UHxL9K6sVUaYBl
z(L>_!#L!u9i%Z(yXU|x%!g6zyO3Q+uKN26@;St`Mee?TAv0JnHjce}(adn@*P%3+R
z(~DObPamI^XD~>-b;tkgWCspMn`?(BGVh*YP|BxpV|y{TsHyVDUrh76n&ueu<=p-t
zH+Ab`3G+%dhHCqJhqm#)H>`c<>YSw_p?B_BvE!stTlrIlx1*#qv{oiuTjYJ=S6cy(
zL`;(R$!GlcmaN^Wm_5V#xaIa->D$|O{(7~VldYM-MM*GhZIr9pWXq=;jVrC@oZXRq
zZOy~$`~R6HAMdkGPj0^a(&c(-+1p#K0!^=8y;7`XQ<dUaeEr1CuX|2;bgpbH;7FR)
ze(jpxD&5(K0v2pPdUK-Ra?7G89CiEWKYEmO>-O!HD^@I6yS6vQ=;Vos%I72Nj;gE6
z@J;qxuBodln->TwDHaB3_?^uBdpwuZc$G^0Y;D80YZw!}m2)yDFqibcF+6zeF_Z6{
zthJxsO!PS8)8?Cfe?gUt?N{MtU!!wvHdI^ly7L=UKU_NJcZT|k6)TgE{pmixVhyKn
zvi6+KN~$djc5m)o_PBkfUB0I3)6knwQiB#{N~t~#^_jP|Blq?FrJKq`T92mApLM9_
z&4sv_JbqEr(uM~w4&Ifq-Z8x}IGz7zM6)EjiEg#^GagpwOqB^iL6O1kS355L{q{6P
zGDmuo*YBMgYtvuPeDs-JGSHrJ`DN3T`Rj9UOSCcja!B>dN?oZ|dafE6YRz@HV9w^I
zd*Z&AIart~l^doADZ1EwQ7Dh$n7IG``L9*?>;Ko*{e2y8oOVXy-M-&>P75zsetlWA
zbB0Z&k#*Udgy-kxGB_uxObAN2oe;UVOqw&#`LKb^p6ip+YJ(*%aq4Mx7ao5j#l_TI
zEfg{B=()MpE7z@CwqgZG!}EF7edhNnn)T~{o<8!pP_E{KW8II#@|)`ZR?V6<>&LI_
z`~9k`t>3<V>odb3(Zk6~B>mam@An>mczF0u{r|tubyIv6g3IeilZ&F58`Tnun-!9|
zm%RC^^TIdoglAQY$n$501x7($e+~*<ReqIeQzfuo-$CULtAjy^$K#95A^j_k%k$K3
z3!l-tVDD1K`V9)J|IA;q!1MO94Lf-1H#B^k#>VA+YP00owh$J_HR?WxCo=oYF`M3Z
zEsepgclM!mZ>3kPly25HDEVBI_1Ixsy{~cG9S?<;1uuU*+%sv8u>^z3*Q%y3ch7CL
zye`nZ`IFDYf<GVZc3Iz^VN<Lqw?6;Hr>GjH$2S9{Zuyytf2^2ee!pgNfyJ5G`TI_O
zy&m76&yn<Y>6FUH4z;zm@qb^1x92Rq_WG+)>ZvJ;ra~#b%uPKrzbafXjfhK~u%#qS
z{;Fq$*(K2wZ=F+5j>%2y`>Mni_~h_IgZ=-$uHX6Tly>B%6i!=P+coj~=RG|=eRIM=
zCO%m!7Zss{Cce{jqus3Lx^WzA<CTusUuRqYExq2&(UH-K!)PYY#|oL{mphL?wp{yX
zS*B3;Q3D?4_19<5@AgrfeE<FDp8`$^kz&g-m0A?0dOiGFb<w0ZLZ?k2>A+9*(*>V?
z?yP(K(PE|#+x+wGhYNpwSRJC}sx<My>#vVWtWKsF%~;lX`>h*?qt#qDug=p?4{gg`
zbWww$Zoj<m<q1KPyp|g9G`lD@o_>1hWyvHDl@lpOKg(wNsO58XAAel2_uihIlPQxT
zbf!&zIr+?XNOgMj=Of|N1;Qz%!Px?~hunC!-s=)*`o($q!!M?qEiY@ix;E+masT!{
z<<*;`A=mn(K24qR>~Z@|HIpS<{OA96$S{$&_o@84bLVR=1;PHuMb#HdYcCj<#vXtA
zZ9>O!{}6KtrzKNlY`2K_pZAZv%2Aecs?Ee_?%odN-{nPL9>}Tr%(hlZekvrCcmGS>
zJ&Oge`{MdVE-s&|oUd|Y-^<!G-5-Z6)%DIyn|N7Z@BI9C_fD=|@a?YM&a*p<)Xq=I
zP&wm1`(FC#UE60^rE@kW9GsxG-{xlZqWgun|AqAAY|FiT`SRrXhdor9Zof@?d1>jM
zzu#`3nQ6>^*`(7&>C2Zdf9~Dm-j@6MAiMmBACLQuv#zXAH066Dk)_<2+nniWHQBbl
zZ04~W6CM<>WbKiv-tFMUp1Ra^i^`!HX>9G6ude4my(waEmFaA=+(nx=3u~(C>gv{f
zx#<4o%a?$-IKR}?)PMW#@2NDNJ9qAxS*G37<LhQRxAT3x;LQK=Y<}J36r+=q{q0Up
zRCfPRVG~pT_v@J%hRFdpZ{NCA@#&<xWyJ>tzV_zT>-Wv-*55bd+O;sb`agwza<*OT
zUdL3w-KrD2Ys%qv{>@oewdVM#SATz(`s&Kckg%{%=d9mP@KM|O`JDAk+iJ7i++3Tl
zSAv;SENXrj=<oZXG%YnTQtbQPa(QX<yp*r6uBuG-eD&%TsCxY*C!)SN{k+_~x?h=x
z9vY;krrs$$F3Vu^@5f_Mg*;{I)Q>L~_ZuZ2<5}iA``D?e+BbKVYH!OmE`Jx3x9g?a
zq?0QAwqFEPRaHS@AR;af$}IP)-`h$_OYi*i>GaNbyIxnk-}~L7<b}Yr)Q#ou<Fc}{
zc6>f(t*NVfwq1%-(XVnXq~R%Y_-tmackaxysvo}YH2Z$W^6&ZU&u=}SQ=SmOXxi(s
zqq}KQ&@-KVJ(k`2ix;ez;;|#!BR2W->Z7KmCbIS=r&7vxiauQSxP8TnmDb<?FgF~3
ztvXd^##P())W{{1IBnwIPmpGB5pbzKWFl*QWW~nGg%)$3G%ez7pS)}1<V>r&85#Z+
z8B<afGpx!u)?c4*&SzpSlXklO{f5%NyK`dy>qXnuv_ES*{G#fb$AhoGHjDDLoslRp
z+ibDLSa<1}c5R+-kGWZ9?o5!>o8I1=9jAQhu3^ft&N<(YJlmSTG4<~4zjbEoYqj4S
z8cun!t8d-wHBno;rfP?8DthV_wl=Eu@WTZQ7BEadeboB>p2LUv?azQxi1zwDlkA^Q
z-jsZtukP#W_`uN6qkF&K^Nx(XIU&xwZ_dO6@4s&@e(rbc_U-!j`_{zqGi?_Tv1D2N
zr%5@nW66SZ>jjH4c9_U&mI_H4u1NH}UOG)L_SBJ1;h&Gh_n-Ll^K;~`5=~(>p9kIX
zf0RByJ9}C>wnf0i%&h3c14mE{J(=V^!>Uwk``t3>yzR5q=T$U$s2uvf@4M}@^YhOi
z=C}7rPk-KEn0x!#N%i?lHgBG6|M#-}Q||a5O#1u(7|F`YnwXgd#l)Ne>6oS)ee7|+
zy`R57|HX_c*W;>tfBmZ3_vfkp=TE2gH|O28dbjs`+^SWpD*pX^?zuF`=I4{i%qbl0
zhg}v2-m7}8o0XjnO4b)k=eD)6?fY?5|I^*~b<>lN_kDaeJD<<$ysfRR&4&ZbJ(9+4
z@9Y2nuCS2<Wtpz7t`EQe+Wh@;dC#|7*)rB;J)FX77p`7)Rhsyq|KE##P|~mX`}O+G
z?fLT4Qa9$^-SubVv14v>l}`lM?faEAO+S8~xr2$hIVh*8`OiD^_xpYS>T2urHlKa|
zpTD<>ZSS>%&!VCAX-%;kKa;e<+c(^AC2yZPpBJ-q%A`-L1VgU%CB1Hc&%m?mk+#X&
z`-QhRoy+|i>g9E6wPjU-eyr@;pE4}9$6x1Vt6dXL`|~8ySuRdt>5~_YcYpkG^g6Zr
zcWn2Ysb`<_7kxR9r*?M6|IB)x=bf8B9{yocW7_uE>-@i&A{~m{#-LK9_UqLYqnUNT
zW%<~T9z7aU`}Jx`X(=dpHl>^t`cr4$mN;kf>e9T{pO;fsoQjJQvuM<I&tRF`G;fPb
zxkIP-ge$7%eHUMPdEI)%blK|bTnVn$4?k<BOqufFYt`nd{8iiU=9TZ?w)1Ls>~&2I
zjRVg=2Zn{Y*~q1zo~GL)Ydx*>^|gzaFFO|(8}st=GKW2_uBy7Rx7s}Z|F7#SSFcV!
z(jj<ezWsa$g@)sg4{p-gmUGj|+namt+_@@~JsGmHvu9eD>&2~~-YKkZRQ$~6SdZl4
zL#^DEACHQ^eDfyd^|iG<lE%yC^BnJ&XHR%?Vq#3;QPCEsn*qyHPfy#JeqPQj|DKQ1
z#06{D=3e-jnUnM6KqIryJe!$Yv#%%JHnT2&r!v#0=yvY*kNf{!|DW{v+1WoI+U->Y
zI39eh+WBHpw?)ANhKB_+eAJw)tF1#rLs^&@#q{HRYHMwSm-{W<ym_*uaoQK-B8D?F
z44YZG#hz%Ft%=z=$wg`6{QvXj&2wB|y5VOSq~mOHzvh?gyYqs#w{f;{S#GLVm%d*w
z+wkt3S(bb6>u{&Hmlv&fnsC~}?#kbGF7Fj<Rz7}wR^FiU+M>Epo?W+R-Q4spmuI=~
z`u}%~HqGx5X!0mmKg;Vr!zx>_ZQ@zx0w=~v5^V=pY@GZv<1_n}J}LJN3tr6ORhqhW
zaYmNKnS1@)4e~zuXsvqv>Er*}S&5O7-|F_XR@ltqEtD{NeB$SK1%WvQ`sd!h{A*vJ
zbmYJ`-Df(Hb}EW&vzATtT)tuHl2v6Qp<5S!`grm09>c2D5-E3Y%yll>+v%bdf6Cyw
z=mhQJimRdFA5UqopWwBWqy6xn-|u#3%$lZO{NjJh(!b06=AMczzk4*f-}YEDzugH?
z5tW+ybV7`K{Gy8*^0i+CKUT~+Iaxh8DCkr5_te`0-i}txfy-6qG%?HGe6oIO-3rb_
z|M&_r5_bFDzkc%j$(s^}zvQ-8JNwkw#sAx1ey>vf>C>mpCvuY`#m?RP?Q(fb&7Z&d
z^}nO9t&2VV=jZ1_3z@%rf^6jc|JfPvFi+R64&1d)xy9n4r*Q`;$sB)K@@)U>*|TQ-
zcwYZ6-K|gN<co`o{b$!JOg{Ob#LAF|`S!Mz>(-_H`SG#RdCKv}igW#*Z#U1maR5{X
zuZi5u(dxAKe&>_avs;Coo~&0CKQwXn^_ZvJ3iB^7$p3NX<+@Nv0g$v~=K@}qATPc*
z_xv7TZ1$PE*JA#X4P{p>zP-QYW^?WEjLysLDQjZ3{oE({e0$vwzNewe8tbJP3vNAr
zE+uVmHhoI|-`zi&S$Eu#3p&ZNah^fjWzX<x?&`9fq(@&WZ_fJWyg<iW)3PW*{9u6M
z(WmN@7@ke}I%iQ}Ox#S<OENdkPKo3`UwI}V(eQGbe%kG}?+k1znk<dEv)}x!;PySv
z2kKWopHr`3zAg9mPnMRQ|BCeA*KYUEE7p4w^XIA6?(?EWPM*^*T)o(2A93KxOz(CT
z&xKPaUFwo5%lX&fE5D~P(PHHqNl(v8Q`xn?TTJARZaNbe$+B$b!p(bUY8s34UF51P
zGqn#tbzxiXtl6_GU$5Q%;@vy7$tORw>en>(N}Cs*RGpr1yX@YA55Ld6Kh(-C==9;m
z;{Ip7ETN&H&ue5Yi;gvBn4EfYa`O3=`Ht%w6*L_@{<J;lWj6cR!=k8Ba`;hHMz_-i
z&TWscxbm6$9^RHKB`v*CN4x(1GmD>c=l;Gcuz2y}#e&V7Cofv0bgW0xSwY~zDed)3
zwr-u8dwU!2EE^*M4uhm49N?-XE6bwpkHz|ZzqGdBud}u;f0y#<$;mrKr*-%I|Mz>w
zvPv&WzkA<)*8JQwMg4R^(eGo&+;;8S_2b_6eeN0}kM5S=w@v-mb?TItmzS4>X_iRw
z^K+@!*Tt5Ul`VR0rN8%!klp<L(|WsmK(!^PDtdEsGyDGg=X1B;b@TS-*5CW(Qp5Ys
z%V$?F%gnjEYpQMaw}8mVle^#V^DZqd-T(T-@4xA`_n~c}=d+EaI7|*$bh}nLGy8hj
zdG398S+PYyit}>87ly0KuU4*+b^UsxO5yh+vHq_Y3cfIiUM*#g>#D#1vFPSA?d;7b
z@676xJ@Z_R!LRyH%yDxbcHw1bK|SQRy1Xin3cI{cN!VA4#nm_1TAOYB{#cj6XO0>F
zT?_NuerAyg)@yx_&ooQt+?Dra{lh2O7fYLWZJc~_Um@?rcZDrn-sg1RH%$?l9%@wk
z&b3*BQzAj;$ClGFVb}VkN~F*2J(E~=C$Rba`YBKTg$J9Ld7PH<($rkNCiT=B*Dc#k
zE>ye`c(~52rv2NcyEpw-Y*4Wdd;ed?PdzwIZ^@cfWhaEhzUdZAbRWI3qcGWS%>ff%
zEiJ7b58I?aRM@yEO)UPzCgijs;b7CAi4PXG%Sjn29}JLUFNzSmAtu9n_*k;p!zZ08
zGjlU<T$_^noHzCMJg)7x&%U|2x$@u7=bBntNe7!)MMOlNg#VD`V{dJ3&6w48{k5vV
zi7LfjX>-1Y`G4P}*Zlu||M|4&yoU$b<)7sLdzSB^B6Mb^ar%X+m4^*>e7zR^^QeB^
z!TbN--WTdW9=Rn$aOu*eF-0d;Z9X0mHZd^?h=}mW&DEW?>|u%3=cz(10xeDpYko9e
zet9PUe$D5z&8IGWuW2w$K4y@1M&jSw`~P}Zt<t)aXU@vX+I-Hxu+VV(og!|3`@bfj
z7W>JRNw3%MpEqrqm|5PP6XE+liE3$S{g__=Q~cWc`16xgy>D#E6t4gOz1}}EQc_e@
zbjRI16W_%vR%pBreDMAEZ&g!WEv-}4ye&?O^J>3E-nxDJqDk-Xcf0vdRB5v?G0yQ*
zH@{zF%y0YUg5+~}J4U%o{=MV^-WO6fyF4F1^liF%b<)eJpL%|>nke49rE=qr!jD%n
zvv;SR+<Yq9V1c}E^6Ova7h8_O2hJ9Jy}fv%LEGVj`}(&hc+ZynJni4TQ!#ZhzUj}J
zQW<h0zng8#HB7!2)Nol&e$}s!mfzpku+*&Ilr~GUEI0Z*s2WRrm80e}yF8C;>avaL
zzuk6OMbF&-h0DbEeE<FJZ{qJ?-Td|aKCcxUS~xF1pJycfUw!Yj$){zRobzN(^nwN?
zr<_!0Pu4gk^X!=2<rzY4DSqtI(U(i@ZvC)p|5khE#4pYs3ERd5$BPZWpXb=BA&r1H
zORqZg&_FwU-IHgt^UodcH8(Z=_^4ZdQ_0I9&`6~2_B&1*B2ruazVDtg>Few3pI<EQ
z4~mRDS^NENx!*#DPM4;&(c6W3-DVo4o|@32w`4MB-M($@wI`FV$!)#(+hy*Y9ep{I
zZnP?!9NT66<JchvzwZxzN*c}Nk(HILWOh&yI{5e_s5<hUoMGanA)++VBjL7L-F|sZ
z)rlT06*hj6k!DL)E??fhZJU{m9DizRs)>oohQz~dd&1kNOuBU|>e>1E{zfyGY~3ol
z{kCsrX6C)t{orvrPft(InLcd^1|6x9M;;e89ZWD=qbYRoWyzzm-JZ)Ys|a=e*|^W>
z$*;3hna^xx>^S~-%eHNgzW@GMVUuDcxh?m$TXAvmzTQL$ov1AxU0qyOR#q?Gy<4_m
zLBpa&N|#=mY>en>YHG?+y5qs3$kOQ0ux_2+uHCye_4M308ZW;*@~}Xl(}jbD>BTNT
z9kJ8Z%s00ATF5NhuwlY+x$2&8-^xI({$0CvY3S*llTH=tWLcK^Y&-kzFv#$C*Tb|6
z=0c8UmJ4+ggPRi$f7~3nf@Ri;dC8}9c5N<R`0|IMLiFsvTb{BBf;xZap0a&@EN9&@
zCDJu_gHc<$Royxd&Fb)Jzmp~S)V@iv7M=RDlb6MB`s@sqGv9c#e;r-&JZ?krLJ`X<
zrMpw6@Xx(n7VqEmcJ@8>OIH~kY~*Bv9v>($4q3>x|NV_^sez_RCZ?sy9AYBRZS&er
ztti#<(<`2FT5?kH_IK59Vou3?bLBX=r}DFa(}#1`?*+P#o;&F<;bh8%(%KU}%$7Rb
zZmlf;0&gk!h3|dl$howxQPjRRp0B;RtE=lzSv+%hqQuXurR%TX&Y$PBFkt_-MhAuU
zdz{Sb_N`d4Vt%8-booQiKO1IUSy5M%G0QANX794hzsES)nt$G(>z97-cA|k~(e?68
z7f-zY8WbKr{r)-4d$p63q}nFNPu17cTef4zj{94m7IDnc2~JE@Y;A32UblGtbf^0T
z48E5?y)8brVe)e^i3ttG8?r+otyqaXwyz2oSaNcOa<YZ?CNih@>#`ba700pfZ+F<c
zlff^{@4Soa*AvC3SPNDa7XM{hZd<oc<iH#uQ<0OBXMJWRw|~1N{L*gqw|@>Dw*nut
zT}ieN-}I1A_GVC9O6;F@rRN6{9@ktdsyn>$4*%D!&(}SVv6SX-nm12wUd^YIb)7E7
zo?Pqq{d!fiuW{e;B2TV%r_4z~lYG<)cg8&bvc^O6bEjDQDv=bmsSp0|y?gdy1w-r?
zE6%68-c4Vk^t(3ddGCj}sjL<;`|DfTkN#Ag>63K3%s#mXI@p!o>B!46!Bi;Y(d6ev
z;!pN)oO4`}81#fqy4*pu`LfJ^9VL^uYc{*pMCE-bP}-toeIdQt&;I?K-o^vla+5aR
zU|8P}qxQo^$xuNt@!akQm)H3i-HDiXbYWXr-*&eNm+an_uTe9}I@Q#(O7G<H9oo>r
zaFO|9%S{9{M7WM7%;{}(C}dlrBJEIG<?`;n;-@(9SS-kOe(hDe!!<;@KKBZwaQkS%
z^osM%&ySCTWCv)gAidQHWEzqf8xs;oLnPtGw$gWdHAK4XC30?`n*M*Y?#E8;xsbRG
z&=Be37GAt{ZSs!P01c7(iYf^lvkO#w4qXWK^n#i4sJLn4{@NFdc>JbLj9$slb@VyA
z^W+|_YZ@G_1(rNtYY;vY=D94l%PIQOwvD;F@7pOlWLP;}{TQZU_u0cw;*W{1`{l0>
z#XKSHW2c1yJc7q2d{NqTtg$VlrDcM|-eU}%D)Y=fdV7{CbR2#v_orqn%gTTnHGw&|
zf<54xYPLyAF*7F2l)U8EY?!<%!#jfGz!ufciZ11~Z!<Rft#Vb|b$GD}56tbLu+%eX
zJL90R^hl<~i*2zvug*>QtXPo6XT;b%LH}4pe@eZ#%-M&3UEG)yS!OL$T<YT0DZq4l
zo8YXr2F66*uFuT^Qcf#=?hmjOyr%qezI(s~>m@=w@CfOWVdi0R>M(xBeM0jU^S8x}
zZrk`bnKkrGNnx?DZ3--vvr0Oi$$PwD=iY*@!WK1$c>!^^r203d9#;F4sw@#5fAC)I
z@tJnTQtfuPJiKoto!q3lBi3zyvB-QqCyu_i_g#xE&h|(K8<;#5D@s)KJd&(3S#pjX
zf82KMdX>pt!rm$uW%K^T@oU+3fIJFLB@L5#Se!U+-M)QufBk>W@K#gb<TEo2r%ai`
z6L4%P?=;;FYvS(4H{HHqz{CFDZ28X(noV=BX!^QJG#_~UvFOu}l@<K&_kXmyI{CR_
zp^xrL7yk8sekqp8a&CU}OFGlOuEBg><=l&37hC-J7*-~0K687DqVMl*o8MQMd+YC%
z*zSD3TIxxUY<kqwcelg5toJ{%dhsug+2dVBp4_LmIo9*T5(6attnP1n`)1$0zsrpd
ze3`ucdiA39;j9J$(<VP#n16p~ZSDHT$tf?gbxv(P{r3LuOx@@{ml+nRLVVLNuJ?Z$
zYOsf6X5!=Dt0T96Wl&Z-_jH%G{;_0^4VQDO-kaLp*fo99MdKZrM^%5!xiw)aW9PC3
z>Z<mIA`@Ad^cwGf<@67Mr+VjO4E#*(hY#+n{hg7OCH25lVd;`hn<jM%tMlEOse5pf
zLUP(vpV;Q7p=_Ss>0blqwxl*&x7>caT=}NsR&8muf<^VSrzN*nJl^?yrc8NI;2ZTj
zRa2OM_Zb$RG(9uhc<wjn=mXO-c1>0dzklM(l-6dR=xcip?~*-x<?8id^G#Ci3ol>2
z<`iA6uGc@m>(!|o&uzgP_hNH(w`?zeR8XzCe$A1`WlBqZ!>hGr-_CvEXa00*Xoanw
z&_<4n8BCe8TMh?ud7b{avouP2_Jm!NyH>5b_xb4pBb6k<s;WmGw^gKk11ytN*wohg
zo;@--ldH>HQ&ZQfb&~3aw>-JWr8&PJ;S>`Kjtlg%x#MdxZ^?=+BF@)eyvsN|#me<)
z>k*~9TTHhnc{ygfsxF)$uRVF$>do8>IS<W^tzc5@>UP><Za&##=9I8dL7s}2FW)KU
z-n{Wr<)Md2y`y1j>d`Mzf)gsQ23@*(t?Jt6wLeS*6;xNI%<)h;w9wW%bB=Md;nY@_
z-O)F9R_ouL$nh?MDf-gyNWQZt{Buoz-egzXRrW2G>%gXmv%7iT&9zSn^oN#kT&<6U
zFTKx=@$%FVNjf*j5;S-R8rtfW<zp{?c1BTzi+9zJo7)a-VR`8jwsmgoGD9;J3z<f#
z^~dh%O$Z7Ij%>Oqa@fqbWVvzmCV>>8K)Z^suDh%b=S2Qx(wnJraZmTQZGJ-U|M0a3
zC*Qhp=-xA_-PiXYckeY6J12AY8gFz*nR?~FiN|+*{g*G4f8*@t{Z(Blw~iU<s_E`+
zlFo~2JbrV3-fn>n*^jGjzQ3NT;=92p)6FOL#M}72Q;*%1uKcx8-Eb+h@zjJPUs`K^
z&a<}I@wIkd%^OReesS%oF5lZUICj0vd#@trZ&3QP^iI`fu5bB~lizOd=P|$g%at#u
za=yA;C7X;N6KiVC$9vW{cm3M_=En<zw&3GpdiyHAU!P%He5j#v+gg$Ld%wu!|9O{N
z5_|a2$)(%tW)|g7n%Zx3RQ%l!smZUJZ2m0%pK|4pc7#o}iT}SpjJK?kIWiyWuY9^K
zzgD>a?#c82Ub-*Ikv(g)aj9QuuI1acYgMWih4jok8T<YI(OIGAH{`uFi}7RDo~zc}
zo&A2_bk@(yo__ZDGW)RcneC7MEXtJXoX1!8*7nGYQ%-$v?|XZP%fz04)-UnuMbo?~
zOAK{1KR8`bE!x%<e2;n2_c`w>zU_xZ?!OHagC_YcFSMGwoVR*g?(DWUwo5NfN=r*$
z%-?wFbdb}fEt|Vd8=S>0z8deJY5IMJitou2PV8?Dg_rAb?ED?SJJoyEVl~w`=IMqT
z&TmoeT$WSfD^QR*CwobO-?Cj!#|!*dCviw@h*4j<WZ%Ad&FuV}vaheJTlaFYd;hY1
z`~FQ?eQ4U;uL=d+jn^9j>kdvdaJdxl{CY;G(;*f`Q=gT}OK+AZ$mH33hu=A{PPTnU
z(bcmtwMUb7%~Ic(e?4(oOeOdAn4K-hY+rAF%eq9!u<DSYo8QdSdydL~{`XBS@8>c3
zz!z=l^Xu5+zlyxuRm;k5w*FyjdbnZYF_ULLlFe^Q>&2Y}`b$2%I<>rTTHizM^7}rw
z5B695+`hi>!Hpo7Qz@q=Jv|d^GJmnP-GkPt`4>vwhDJzEj!-!`)%@Ps8CmlEe@^e8
zq&eg8^?$Es=fr#!k+!?z@p;kAHFe)^G8*~JG8MjS(&;X&v-@qBnEpK8GMVrVP5O0z
zy|PZ#toifO|HigN<ICwj3*JB0GwGe?TwVEi=~bWIpWO7KOy-u)|IM3k^V#P9pVj%s
zg-N{Y*VlB#ipmKY9XNFK^T`sqMQLY5)<tb$5z4jsHktj#&esN;kNtY_?oHaEqv6G0
zpK@3H6SseO;3D6<znkq<Z*6E^UjJ3deO8pGYEsG8uLY;&{~JDboBru`{GTaT&+j{N
z#4~wwrCN)^y~`(GrO&f{T%&!?o^Sc|hWig5%)7hAZ(Zv3cYAkmd2uW`ToZLp)~w*S
zPP28IWXVxg#`63Z2NWJlTK##$JbC_}e5H^|VQ|I&@4?GAZ=P&CF86p|^*coouDP4;
zw(&~;0QJ*1rJi1~cJ1FYk{njTV*ks1*_Y{FWRaMa@KQs5de65-^WBpFe~4&WCO3Kg
zdEEym-wI}N%hVJmEr_3;aqHP}`I1jxceeIumVL9{wqN<|<LJXCE<QZGVtpTq9#3JA
zYcD%xC@guhZ<@+wtvfq-3=^!?FUz0Rb_hSY*6uUUtMji*tY%sktMT&kx+n=UG#ymf
zoObrm%aVDA927R{6mTA#s%+Ba=jb1`$a=vMRgYJbq>sNSQB{@obUefK^2Pmr0jYfn
z9ETGh@QMg<{PH~REa6sWEx~g^<=0#Wra5~2cJsyWzFf9oM~BOCMJZ`(jSEo~=~uOc
zF6ITj6w%d<?O6Hue1FOIi<hQ8>5|r8oadr@b?VcenO5@CuX|6w{_MuX(mj74eQwzu
zwed~NpM6Kd15SRK8Lauq$up93YO<QargeSaeg)>-yX2D|E!^8Xx8;bZ(U!76)dS%X
zXZN^%*3i{fnc5K)8+`7d?&}*<{Vv~=JZK-pcVT&pz!rIXCmpfHKR?9Xdy%zdv$D^k
z<;@jtMK^o5Z}|WIf8X2Lx$piQ<X@qst&kKF7*WvBo4-zNVuzrA>V|52`@@e@550-J
za_wqhP^{~A_v9-xF1qZ=G+DkteVU8ciWw$TZ+=b}{IoN&=A^65*V)@Uk3QPlsm3z#
z-M2Se=Vk5_eJ<H~ST^C;6f3j5TSs^CUoYICdg#W>&o6K8pSaHd^mb>b6S@`e7aXnM
zll^J!(dGQI7E?bufSZ~}<>fkEntW%Q6&{x@f3la;#8+9a|8d#w!ksY@(~jQSnk{LX
zC2~EsT$azh?5WjMuZiGQT{~jL%aRtm982?$?y`>zaarh|UL8CsKJQJE%ZY@VmEA6G
zkIXhrU9Q6^slEGe>qR@GZ7$+X5x3N4PW=1JB=^J(0neK=%KnS5S)y5)k!*f6Ztnhw
zC?D@LPY&|l4PoQeTl+or`GqMeKGN%*Pgk3md`tOpvpx3DcW*Drzy}f=?uIX$qIq#^
zj{P>4{lTw;ZMl;&rCBOur82!A8Ex`*$@CWTzP*E|@w>?}uA|G(FUWrz<fWyhHNz;C
zEBpF7)`U}2G($o{=9n$JvrTbYOA6N_KTiu*b}JPpho$maO&j(z$GB~KtDu}d`GI0}
z?fmM$lQ+BjS(n*;jC*fCYniad>IG}IiQQhetmX^LvHq0@x3Np9ZaT)Rz5o0s-Jp*e
z>;FA7<TF*9S^Cjwj$Lmud-Sh)oI*J_XHN+DrlvkcK_H_e_>{oSYMx(frpw*h*!Wzo
zvT0+nSWu{W)3U|v7FR#56IdXb*R^_;rY=`dOr+;u8TF;Dj%Nx?toLp4I40zgV^{Mw
zHLkK$w^%>Utuj;aclo<_bN&6E2yQ4o#?iUu?H)!&`|aO4m-H^nR7_px=9;I*GHc!3
zt2Nr^^o2aXz9^Krxx0O>wAn-6P@_a2S?TY&KSGX+I;e0S-@TMe7SbpC*m=Q(SCOTi
zU;f$B>2ZhNmTk_yuE)j2_2ZcNzJmvw*(;yVEjQq4o*q}#IXiDxC#b3^dg=ul`_RyM
z@N9NIXg=X^t>*Hhk1xy!c(v$Get>Y%hJO`C>+jfa*w<a1CLZwQs^y)j&$iYYzMC0)
z;Mp-2*}`t=Wh|Ssm+oEvMBY^W=h<x>)28%jmQ6BZSL^kgW#((E*8SS%in#Sk{j#8k
z=AW*fDBHd?;%8t>!KGzq))znjBG4K)clkE=8&{`2YUGW*V`F;a{WrIpSEoIC{(AS1
zUw4@LT8+0H*V$*Hd0?}=d&knvk4?VMh%r2OzKb)Gzy1E(-cQH=Y1B<W*Ak>YmlNbd
z0pFW8+f03uEt*eX*il#1u+wzj!+#b35-gWix?MI|<}=f&veNSD)2BW&jaYrHx0<>$
zY@2E-P@?W|tI3?tBkcY5%F_K$e%_t5)_2l24VTH0%m)+rG};~>Nacv)lU=adx<_U9
z9)o<16H|@O$h|P-;kXmWKl7bgpR8Ti@pzBiP{EZMO&=#O3y6HO^ydf1oEP$^Ra6%q
zGRivLyr#ZVboci+j}M#2J#+v6TK;Fnf5Anuu~V*HGtPe)Gv}pBn$}!<eQ5)gm&zgQ
zf^%-G1q3+dhCXDFTyxLt$sWNL1-X~Uj7pd3MtIcbTCRMTSG4o7-?Z&p%1rF~8#A+g
z=GyUY_qkNFh1FOz`Rgflf!5?1^%qOx>YsV?$z`hCx!WtKk$7&>QIRU%M;9yV-fj3@
zq3Kbn6lPOzE984Q<KD)({P*GxH^$zseEzlk!?dY_AF8WYTisyc_F5sCvQM|o?pSQ$
zH;<X7KPsA3eI%mWry8!5vz>P;ss5UViv8WzYQ_vEQL6>uRQO0(xkW*!^TV^*`9a~~
z$3gSY{{H;>`~MgfKRc6nZ%-v?R8CYoY=Vc%kDur3{bFNf@Be*QE@_y=!f*3|;bKO~
z4*kblTqcS6*NZ%S=zI6i?^%4urdvpcKT?^w@#hLl&r_%Nt^BHzTefxCh8)l2{*wpK
zed(-J+O75fGjD8<cHyVDt*#a(j?OB^3hmApmTu13GX1z&`_JRsZ~we|Ps4Hhxeg|F
zTg}<?*S}Bl**IBVNB7w43qQVYGvM<rczcAQwAJt347sGFmEEGkn{Jn%;|L6Y@ucEM
zi;w*FB{y!*dv;Dzah=@z^r!BEx02PWZ*M-<o~_QdBfX5V`u62D*0UVEBKGQg1s{3x
zTYTz=){Aj{ZN|)d<VBwM2WR~W@@hJ$@cG%<<a>K6S(*<Tr!Co@yFusRL%ttx{bm=)
z_yi}UEzo7UJ!^sOth?#a&Aps*^%FFU(+eIP-d^##*EGRGQ%nE&+rre(B0Fqay0|Ws
z^7h+1Xo$ElD69J}v|!Nx`|5g$Dr@+?I%}hvDeeDW)_?wz?Em<~^q?pSg{fW>&!u>z
z8!fq%S+ae;cK&YR<u{M5|M!0XrdK!b8)n_gS>_{gyl1KS<)RF3`~Ua<pPBdftjN^q
z$uF$@`lknYD$ZJ`XwvHtC@8m|T~pPs{M(HE8(OsUWHp>!16SPtd1>3qxdwB06+Yeb
z;X?h!PZyUP*0}!oYiBf*=VghNe!kJvMLSl_)6Uy5&H44cC+Yt`^zZq3>vzbt-UVMD
z`0>eeZ~pUa_Ks)MqBTBqo2|Kf=7{BI%`1!k{QCcoJAePqzD1{s?#QXPJzTP32S>lQ
z^z<iRSv5IWFSa<C&k+$3eO#D&*z=B~xO9AVWBk9{{hZ71#uT5P&hhI`MGa%qL56z?
zg5me7ZXd|j<<mU>F;OwQ;#0+67G_41!zYhqZjLNFDL%uznk%op@4CE##D}!WmsapY
z27ivSdinbDetvd#@xFcYb{0Rsbnjl@<(Db~90ELSoGeTxW@bYD$3aV2SFT(4Y*zNV
zh0B+>A3JvJq1gYPCtniwdp|BHZ`+%jX`=W`oloSZtN4)vtS*WR6elUBdaau)vB2!f
zgH<2vPW@$G-I>XhH(%}%yLUI|PSdzMcMr$!yL6~;nuxEh8oSk{BbUwOWjqsZZrH5q
zs*@c2=4zp2lTrP*{jT$bZ%3^cZ&qWOl(XsR&#iK9;+Bynm%I<Gyf}Bu&lQ$OS17TI
zU#(oJT~ct=!oF^Kn{J{@WAnR~qwl}DX?~K5F}ybadse`ldcWvkAFq=$43j_YpU-)l
zZ9CuFz)rmdH(d9tK6w{mCYG9HsNye}v_MjC+Wi`H?)5tam*4I_{`%QF``>nLr<*fp
zh4jxnW_w?<-Tp!9HPvwAqgx7J23Tep-Tx(D-#l~U$5tl`?Po>z>sPA_y*pU2OH)fT
z@&7g6njLw{U#gruGKIC|&xUGG`+QdZU-_ZCwi0~etg1(M$gpwi?eF6(7MtuB{iDfo
zL#|Do{iX`edXq%Ce+~SKD&}{;FMq*jzAe?T?3(KppIQC(4;H6n)k|`9ZT@@hyiMW0
z+}k3dJx^BO|I4j(SCD79^!DSfMq;U7h4(MptX`%RxWr&a@yDk+yFOW|or&_WnRw(a
zcR)mlhnnEyfB)@z9UM7)IrjaVUf-)}`iIf}zLaHOg-+w`_s1`=wq5SMd|}6X{eME+
z&-bTt>;1g6HbN-I%Hp*9r?vZkaNE@Jv)?n;lr-*Kf3I5IXzn3lj=%_kL)&!HVvBf#
zb)~m2474wm`~G*kA@jP>sg^QwO9~9qKZ^RyR{!|xj>JO2n4*&$3^}(SeVYBwA&bw3
z&-W%fc(g1aX2Xt{b$i4&C-SV@HDg)lvdo`lvz9G<ZM7_O-dBsmju+MzpWdSI^xvD(
zzsK3<?%8C1>h9)W`kNxR*}VNdE3;9LWv3EHn_2-+kf36+=ONGTnW1uN;nf|>a{oPC
zTNZco^5M|O%oe?GDyK~eY3DgMz3qd9RA-NAf#XXNrk71MPg`O>9nImW)DzkBJ>vZ~
zT~+a`KRI@kn92BmxNuDG*mSF%`E?f~LrVjAY}0kD^z0nfyiPY=WMzuC>(1EU)bz9V
zDdR!+<%TaTW8}KLHW#V~c`+?)TElI#K-$lVb={k+y~o>EZ|8Lq;1GRw{^Y!Hmjw$f
z-v5ga2|W4a=H{2PH4h8x95XPndFS+J;i8%wO;dz?%{BB^zX(^4^WxmazWnrL{(O!W
zt`krF3wGF8+@0r`dg5)d#qW9ZkG|%A#M|;|)2=ye$;<py=e-Kf3d)!=Rg6K(HT9_o
z=fwBdpMHM)ZLYzSA5E;4mCmk5Te|$86rVVox?+RD{d=$ZpWNoYs}K^w5y8UMr70xU
zs=Bm7w(6f0Lrk3fyWlhB_shB-Iz0IFqo7*iLnOz-hukcUn~ncyLgH(Vf6;q&_5FwE
z{JA&j_PRA|j%>>{%e{5vP%HP%Et$d$qN1YUR@N5Phf8`-Z|GQ>)7NU;qgnRI%!J2x
z!)DXg`ztr@wRE+a8ykDJow-=))lQ>~2^&?Ur6%b}Jxu)jk%c$0xIymv)D1s&o8R7%
znKdCYR4yoNTa(j)rNN>m8Xu*4v`UQDUpsv6n#f#E?)G`UQ)ZatMlBXowPRj;zHH_B
z7rfC^D}R2{Vo#VHt1vI<-J|mAsvjRR<_NO##od-yowkwj;I8CC9qD(oLJwsBUi7Cd
zoxiiE_p@=>%txOON%FF!cMCLy+)8=AQO;$5y7~6PLk<@%uAS&Ld%2n2iGN|6f@>w`
zgzmQceJa~6C@|VQ!Q%Ed_APJkA8*+Ech)!g@(Ia7^S=E#cuUgHJUH#FWXQaI*XzxS
z%Wh>`zm>_>+5Jt;M>=NL0XxghPZE-~T0b9hoOkQi8XjiIP>h~SN$m0!D>!^F7g)^&
z&54(?t-tQAFd_EJr{8~f+|6U?>h9k8xX-%c+068cuUEr8mtO|03KbC*<^BG2&ZIfq
zfB(l`ZVVPrS@UMq-%?>_mb+4?kN>DBNEXk3(;xKU+DZ-yjswpxaO@~$xjBbPu6@CU
z#l^3;DEQZlJp1}L&})yC@aZ`c>a3He|KQP+e*Ao4?_Ha>uM{(HP5Dt0=JO%=%I)K)
zWCi2y+;u)QCB|N}K~3SwrI|7B9xZ=u{qx+Sq%7yYKi|s5#UGbvvRtZ7ef9N;K;+F5
zUxRH&PWA>j+3ZzzI<YqR*=5chTWTr}t>0ey`O`bOB*~MdlbKjzPFWblbj;sZ^y7{x
zE917*ix;o@hO3mhJ$2^jJ+xr@qeDz4vEhan#0`FaU|SlXRb^;$@<!0fKPR{DnDaNc
z$G*z^u>Ib|Pxq%;PdIRY{kad}(<N)tS3aKd_vyKxr@dwWyu75iITA8q@Pj8QQtV;D
zjHV_gozqHk{lYwKf=&}WR1AD4J197qNV#$}o}aFJUVZ73)vLP;EPOOX7H!z@LAcUs
z{`c2gRAo82H=FCvUYIWWaZdHVOz)^gYC%pD!_|`=FD#vCkgFrjq}MxdO7Ftd)ruaR
zxrUQlG&MiInql^1PLhMpe%*zpS(mOPKPu$Na^|Z!7xTlkL``RxkQl4n+@$__<$L^6
za&`%cxyH$EU-<Rrxi*p6dyQO!%=}^>n6YeY@i=wp{gsZTs?vSxEA-0*ieB!iS2<VR
z{`k+?)TI@&QnH2J=XSVWSQ~ij_0k0ra&6CYW123Mcy)DYXlmJPJ~V%|t450vU(5rI
z!wrk{YwsRAcSY|-+bkx>155Wk6a^=~IkJ}gOedZRakM)9OqM&}d8xc^|Ni@xP8?lb
zU3wYo8|$vG<UCcgJnSfsZ-DHZ%?B?1UVXjrkgmc5tB<<{0-o7#&RWj7f<q#Ng>9QC
zQ`cUrqxTwDf0t{Y_MJy!NA06n{TuFTz7CDi@At-?{Pmc#Zo9eHfu$F(Bp1EjlJKv}
z&vf0RKgl~}ZteWOC_v+1!i83sgt@yK8w!Go&Z;zPib^qVU$8U%!>q10zKbHO<e)px
zjy{*zAN<(5yX(gNiE8sUN(Meyp!LAwd!MLtpW$k|66lPZ&S@pl{yCp^u>DEfeDbt>
z^o7~dpF{lEnO=B$n)s<~4}?xg)cpHkX}T<YS9>(iQOO+&HXJ#(-}yFNFP98l^m^}8
zuO(eB0UCNNEew~ZEb-s?NWOq`l@n7ev-G(i4^G`H-oqE7(FB|SF+9Ul?PhR#lfh&`
zp3TB3>%vwZxVFbdSKZH_VfyW2zhn)Ou7wYMezNDyOf0zS@>kfM=LruhI1WHgN%|pC
z#@v`0wIgMQR<7-#;Hh^tOLS~qs;_vutF&(EE~u_Oth>r-p#^iD_>ZZJ)`L4GP74ES
z)FjK88>g6g2&WgQ^qo3gUEY+ddXcFxW+KbQk~SBcw%d;%mt6JQwAANGL+H89npf3L
zcp3LD<Xi5nU@|dYaq70eD_iY3Ca&eJN}Ji%mSfR=L@sAui^JtDZ@zk2F39nz%WHKI
zezfOg%{|NW*UR_4X9SP(fhvTL?WaGeD=4f!z^!<}Wt&Xfvc`u6kBfePtg;Q6+IC^B
z#K(y9np0~|`&_<u&rc_F-jjPeFFDIEUX(p(*Q%z!;pWbVwqbk!*=YvPyJZ{2kz;)H
z<6-t2`)aL!^T{brH2i#O(}gXHtG8>lEO>cA&Mg0atET;5(MUz#vfTS;);XVLUgyZ+
zsC4Iy_j$Y1_mXoxw4JM?wa=bCYhrGG`PRM0%$pgr+}7-#`TcsX+B`G;jw4@oR%fsH
zWi=n&3uiY}3p|-{>5$(RrMEl`hYFG<SS8vVwPqf9FSMn+e9{#Y8}(Lc4&TL(e;ohw
zcfZ|<J=%GX&t*?nTGA`f6yk5a|IbDH%a5eb$N&2n&RMnTt7)si-qlYPHdyS))_yG|
zqZ4Dn-+!60HtgimGta`!Ckw{ad`gUq-OauIwy)LH1$D=Q-tGMC#w8<pullv`DRteX
zPbY%b+*{H;>)P`ExW{hy7AUyLHy`Ib&#uTaOV47<Q|5cwn<VTrmGoj0zQ0tyzHWZc
z&t&yIGJdw}dZ(!W5}0<9{r$ea6%j{{{PJ3I_4+I&54T&D|L=Y;2=QHbA@70wzpv>_
zmNK{Nd8Fs4>F;Q<&fhgH;<Vztx_7p(j`gWOC{)bdJ>UJ%%9MB7(jH4{m6q53FMbnz
z?BMgT+r@vgB64DOw@umH(^{<N{9dbOU2J@Q=N>^vjusokviP4SzRGusCM=&HUHs_R
zlHT0;XVWh|zoGE>*}K*`<!)tr?o}_im~`g|r$x!sHNoIfK2S2MsftlxV`2=ORdQS`
zMUBfz;jEmDYB7hH&#`T}P74+CHcnoaVZQD5<kEKgLbEhA*Ys%R_xpd#O^>&k{%L`<
zB8z)je&XG0b47es9-OAImM3?uZi#DB1v6{Nl<AKy$m;C-!?tY6=8u=>zFm9ckb>At
zrA#N+RMCYE)0Z(vS3aHlHm35p{KlMX5q<BX&O7g~n#pVY%wV<6;Z^td9oP}lvFeZ|
z3$xvtlBUPKzDI)9T2-G;5o<qS_VU$>BTnjKFONO^n9R|r9^r7{wZkp5xw5hrHjV;j
zm$tpg%3h=>b1*!>$2HQhX^qg#j<(FKyn{y;>{&Evg8l3>hLgQ6T)N|<GjrjB9aAnn
za#}KV+2+mCiV89>7cST`>CuAOeb>@%?OxQ<6sL3BP5#JPC$HHFKPC2kJbr)4hANhe
z$<m%nO7njIk~dtYbfod^>z`kP^#hHQWQtcGm}YkI9pAT7lY|PUkkDy~8IPtIMW@YL
zcP_8QdGYGa!Vf<LMTR!szTn6qWP8`dbbC^!pVn%vkJT~im6JG@OkKEjY3I9)Ne+&N
zPnWtLIV`*9_b+XizmFwC!+j$Y1$eu!Eqh$zmh_{^H6(0GQiecp$z+e2DU%F?E-5@H
zO|rP#!o=vZUfe56>c-wvwK{i>{yWEJvb=xQDy})ZPMoqj@zqmv;x@|%`;1$btX`~K
zuW#{}HDrpYFmL&#JGc67%PS?%7Ek$Uuc^ASglmcEokgJX5L~KuDL!H{6!2(ycrbLQ
z&F<9Yo^o}Qo*Z<)c<gTbGwIgD4;LM{=(jzrD|42`+;fb3EoAkq`rao#KIgn9!bH+q
zH1+N+eot=Mc0Gr*ivL^1D_$KfH?YZ@R5;mte~I+<_>D7|XWvVX?LHfxzpw4N+)u;V
z{l`8?2)uo{u;`A{zbDc0ijzGOZZge3-+cUe`mse%|F~YYkjc}}-~YjYJ=!{>?xklF
zUvKt)gQ?<mUj^Rh-0VNRQn^>by#8sNSfBpWo$hsK+JEmZkgzHi*IW4c0H0S{+{y5D
z(N23F3)uhP{{Q%w>~{-2%q{k1+y4Jp|6Qp2=*ALz^V*+sdNb{3o_qZJ$5ZG1PZFY8
zWV;hGZNJyNm2(q&Y4iEU`O5s;fp_?LgTEKv-g`f#XywuV|KImtoH>nSdB60#e`==B
z1m!AHR0F*qe5~qpOV(KvWt0%tDv*+t7+3e|`kCEtqpxr@J}d|h4fc-p<-EF-HRoRf
z@9}=Q<gil3j~@)>e%$om5LB4N*}Xb{@9b>-x&GG`jPC#bTCHKY>&M6cKLrhSvlT$*
z?)aqk?$4v5BwzL)Pn-ES?_VjrVWpgP|N8sY(oYr7b1FBU*^u_N*yj6gQ;WNPi#UH=
z$}NA$^*-_RarToEXKkKwe_Ct3KJ~S={)SIaw_m)g7vwoRHT?bw-f-P#&EfN(-n?#q
zB++8U9GmG;i@ViUPsy+U>t}PM@?E&SdX!1;>U8l*9Wsv(&i}{$v7#sC)}uw*`H!x>
z)>FMz)DUl1aVfp%eYj+>=##rw&u{v)wA}wr`Sttfx{glvyTiRJ;p9~Ff|ET_a({j@
z%jCU1`1<RDXhW`JXZI=`GX0g5=JW0C-F7uSo5!tAA0u77Ix}ZUv>B!(212Xp`BS)$
zI9!-ta{Bg+<8qESG$&OHS1kKipV7xKYr>0N3bqkOjG7-?wIsuzr5|R0{o~M_PM7}o
zsmB}n#Lmt2%XUjjH;UMH?5cTf>&;;I&j-EtKfKFtzUj}SpDB|JtCGZmypkRo^d-pN
z|Mg7vi_E<v{_>Uc_VR`s)!yMr>y17vZ2#N*-paMfZ+>*1iQX{5Q}k)g!&Ukl|1Lh<
zv;UWNO!dngzM3Cg>(|>K(EWbzcu!#Ko)53%KfE#4H+<;-Ugetb!RVOd=KIRDl}~Y1
zK7Q$T@AjsN^G=Ij*pbifUd>{DHT>dDy_fI2ijU@QH_p$|)VKeqaNhnum$7>9lfBpD
z9tCVaw|(DdW~Y{)NB!$gb+6xd+SXj>2uE1)b@O$}A)S+t%m2`vZI+XKXR0t~dF#sP
z8q@7-ceE_H{7C%&ujo5lH~;uHJ^aGHeB1x;)%_1>h6HNrE}EDZB*|^km^-^=_wJWp
zxTL=4KM%;}=bk=gL2}}<q%#pU+~GPh=e76$WPQKyzv6VAIb~ck??3L-R=aTZ=GhfX
zuNNNgTAEXpt9pL_=DyX_bsrqQs$XdG?8!g-|JgDZlmEO~-v8;@@8BJ^M{PE!a5$<-
zJ!>w%w=nC~(mQ{b9<AAvDVu41bd73!!aeus&R=RDw%!h(@W$f6=JWf{ojIMKbad0q
zoXX#>Vf#MF1gHBbOzxOCMQ-1pt@%c&Z#eq$>$|t^ubkf2m#w+HRd#n@(aTu#N0+QT
zj_sDHy!iA^!A+)Zd%qj`@BPGNobLPR!R_Xl`>($LxHdOD;bztU-+6c4rk#~@*O|0i
zJTKWLb3*w1stL(~;@|84*ZoOjxBq*x-*a|MfdpUk#>K}T6~#=5SL8aH-dAFEcA~Po
zgk{l_1#AVN>Qh6c>+``4`|o;7JaEz#KAWJr(NsQa6XO~Y3#OYEZ|@niy>^%2dltW?
zVS3$fwQ4@|OZzf)tA0wT^_N`Uquy1pL&CDk<o){l=RW@~zfktY;*!L~-c0ePm|Uin
zJ7-!|>Lt5}34e|1(W#ocW#+-~h~wAyf36kFxqob3Z1s(OA7$@V{$}3yZ|-};d>6*N
z>5c{wE+?KpxzMN_(lhteqMesHc9l$v|Ho?f|BI-VlHr9rK65uVoJ@W_VOz?xJ3II6
zOH^g=GvAQ^`}mcf+2=YwIo;Bk9e6pd=;NPAkt$xj-G7A4{(eb~x^>Iq+Jz%qeuf*A
z#>h&Uoqp#3bgHfUhApQhVuO!;>wW)mV&<g_atqg;W791B`9~*W&x~EOyl(ZsU$RmA
ztm*c+V=HE9Wpz(8%s%_;a@y(l()#l|wtalTY_Is>(u;R*7Okn%J1ypOEbng9{tOXs
zv9rq-Eo!)Fylg}9*}_X7CEWYXjdK4@{hRXd?!CoFCMOsEW<C>X&_C;x*O|!65qnE{
zXI4yke4Kx>m(-ft*QPCV4{f=$WxHvjSL40k2c`A+zb<zB@N3!jN}YQye{XQ=%rTnS
zaR0#L!sMTSL`2zc?*D9%>2En*FWSd7a>7yZ_*2#I^BxsAvokDK^Go~I<N2ecz;BLc
z>+jNSZ*4qmf1muZYNlZ#OIWe?!9^E!_NPdOPHp`$J@(YB?`B009u)SOn;7T*d&+D#
zdCAh?@cK)&&m}m6(%x@fd9(UpPg%j9g`F=yuTT2b5}2`Rn)kEwNzTughig9m?#j`4
z`sJagGT+t<v_2|s6kv+IUbr*nlmleKBk9HtgM07%FV)GUiYJ!l9_W)jvSJtK+0YB?
z->#gZnt3~<XV#e=ueR>__laAj?|t&Q(D;Q5(ycih*Q}m+T&`w9$;SvWpV<v!`OEiK
zf7s`z@`HJ;d1lP%iOe$ZPZ<<H6rZvG3HKKH+o$*c`}_a6`|WbW>?2>|oSonQoYj^(
z*JDNcOtB!2mT7NeznsdxJ*jIK=auQZBR?OjE|JSfd&hhIq3NcRzjRKY+I=Z{TB}NQ
zn*WnuZ{~`e-XgHe@$o!gn;##%j-8D>@ne@(RNvd>{_mUwHeKhKAbMrf+Iw<uHtQ`_
zUwY)J=7Ue!>ylFn84f>Px@_CDfCHyDigl*#DD21ya*e&dxve#);~&G!i&gy-ZS%Z4
zMfP4*XkjR6IvAAYenlZ?cGsz>Rqp;SucNarK77~}{cd62Exm7V?j63ZZ+G(D;<+`?
z=VhOW-J|eZ((ClGZJVxVepd?+*;ymTv_yBK=?Bqi#>uZTw&g@R8^>PppY5C3*uHF8
z%%USX@|QMk%)MaJ_+sy}y7`^UgkNVpx)^Wcv24ltT5-FdahpHyzkQ9fxnb!kX~>`k
zXbtJ-BdmeXKBScV*N92p-yBwG`);Cn^3lGeqy6(g+GO6Am?zLJelGcS_5Q~ZRtNq>
z3LiM9^=f9woW6ptkH*i8vu{N_ny7ba+1yZ`-zjs>OS1h+-E^JBw|e`h$@TvQ*YDZd
zcWjwjpPZdXITu%Vci{^@9_Bu|8~*MpCubBc&b+;SuKB&#^AnqQSN>dj{lnDjK|&uA
zZ`T@q^zOVIeqzqd$u~cI`@H3pP+E`tMA_gBnS1*huiyLCaG3r3hF43k?(klvsjr<J
z603XLzGg1(?Xu0)x7p%;7g#I1_qut1pI{_=CVoRl;nt-syLWxI6Tf5^sCiIaC(&hQ
z_x1RgqwkvABY%I(&bd)0u#oe_@1QT2)V1r^UwF|r`=ju3gW_vGYW_0r7oMNfe!Qta
zX!h*zyHAr#?shO5^{<fby}tDOd|{Psx))>P-1_8PO~e-F9FaJ7{rZKHFB%t@s9d~z
zLU{jA-r3(wCUlj4x;^#$g;`$;j&_OqtX#in=~mIpVij}zW=Z!Kuh}{G(5s6TYs13U
z)-^A7@BeV@Rp^#kaj&wwK0iKv?>x)dgKKscXWZL5v3K|Ena0&teR0}uOCBreuZcRl
zP`SOb_N&&c%X-K9Bu}n5xGw)n#vCQr^ppv|*PY+n#&Y-R(!3Pi=<9or{mR|_v*L5q
zy&nn8nVNb#WLTh05w6xppO3dbbTr`DwVv&4sKcWZ#St6b*0^x5+jLXm6x)`m4G#-~
zBSL!gN{;!=KmFxow&uKTGgc|d>&EFHb)6L8X)3Ni>)dPof3wc6ujKe8Gi93C0>-@h
zhJXuBmra^nTPyzLDeT@QqoH$P>C2wkyIZ=y%RD)%F8?UBo%!+^gQ{mi({yLFT+TKq
zepfQD`fX~QCf|pnr}dugI^DPV*G>1zSZ4nXO}F3e>T}xP?GUIKw|{rPdds;ZGdF)&
z8*}u=%3w|14@usgmtQBB)S5=$ubx>R*SDaOYv<E-q9Q-`=<NIx)n{&hNZJ0cpn|5n
zX?ON?{bdb#%X*^Eub2BFC>JlVeESTh{o=9Z0TCfB$1@d{_8yKt&^0^n(L(p|Pw)T#
z@n>;X%sSR{?0IS8teJ0K)#*EBD3mz4y6%v*+VyPNh7AI~DdD?6r}9)!78er}O<C>Z
z(RujoLIV+p%d#;h9cQk`)(Ps}yym`b&Buis3_6xsNACI))n{gNNWbpS`WH?CW=Tw^
zvrl<^l}UTd{<>hgf7-Pfg;PX#RzAJyUisFQ`L5)HThe-p4?Y{@-tsZC`KPnL>Y3=i
z+UwJI{GIl?;?vKof8Kx1Z+Q3c*6jSn`xZ?&e2<}ZTKm0KQ#?3DONCeJe=oWsalB>X
z(;M4*^LDIQvsS&Tg#G`w^VuFzO{>@KpJ_NXHAFZ5;KpT>?I){>u(5iJ`#$fr`yLya
z$>D2XxZYERgWX#%{$S%{{*NF0|H*E*_gJuH)}$r|t@N4TE)^*G{MaJ6yZ<{+c15w$
z%p;R6{pP+m=-Z^KP`vs)Q>N;M%ZWyFZ+>duwS0<lXQ7JB)rvds>i$Ng&6<}XRDYm<
zA$QV6jai3#ufJ88+A-1S;QQ~%sgp`dN|ah14IUiba`64<#AT0`=zgfQJ+j#LLBZo<
z>xtW3O>Q-xmOk**z~HiHRCLsfPSt{+OjD*$e=ZtrJXuNR-NBb11wT}8JZmB=E4#!<
ztT926)iFMQ+CMp${Qnca=G@r&GPmyEi}yc1&HpRftrnVmds2y!->pd=#=a_2#~<?t
zh6psoeg9Qr<W`+7zU_*Jj^4|+{*yYsUbuR<`Qeub#nuyMH6DMfAocR#TmA<oU*7O=
zzj*mhMaK8RSN1|nlS`c}@_W8%9al0mnaEo_VOIA}Z~gR|rTmBA9zLKZ)ccs9#h@hj
zz;l7#<sX07zL<5uN4<0Tnm^wXjBbau&GRtPpPjnwW#@8^AAcowxmgJ=-n&?MrKE_6
z*yGO@8_)7g^bnbTzW?~s17^l0Wn~jQQa`XciL^CwY<kM7{Lo<AGZhh@1>BzD>DCu3
zxTatCFAP-NH9;jQ&}iF%_us$n`Z?P~=GMvcb)Og}>+POw9j3!{`&?YSQnT}9mqUpr
z%iAA4QtF9&-*j-DKT}%Ogx}|0D++c$d%yKfx{}ASsuwT%5-e<4YR=s|C3%ngSoeWV
zI#o({&q##ZrdV?582eUNv!~_%XkR8RA}02*%yPmu*Yb4hk0E-|ix0f!4;1(`cZRU$
z%dGr`3s;JAI|fFohCV&|qB$b$(L@8=PM)KxM{m5;kc}1m^~|=k>W_$c+a@*xW7o?c
zA8#phEvRG;xqeOH)2`nqwSQII;P73(V1>l`(}s8CCqFWFvzq(tKqK>&>(}{DI8NB8
zFlWk)`Hia<wecx#Kd_L0yGqx2?fFZeKdf0W&$89Y@JfsVTN7K>o9h?uh&Fp@Oh{u|
zD0yRLSzq>Pc2j|r*<E|2_ngq&{*$G}QoPbI!PM;ax%x%E`-}ZwiL*GJUT%D5*W39M
z6z-SDTNXX%SDU>oGpGK!YTeH(-92#?rr+<?@W!=IdmB8bK-YdJ|EJ@dH@^R*QdzBh
z`*&>e|JMl(H4I@^C;!=2cc@%e|CY_Rb@B0_NRboIH6_-pKWNu}XU++`IJV`JcIMar
z-Bz7?=I5jDyZtJY{;|(~E&RFAY`$mSJd<1fbw~F9f1<eMLdLt@@8wo;)iU3D_UuQF
z-Q?iA<?8pdm;e5wCt}CT?D+Yh@16g5|2w*?G+s`eVCb8szq9T4{;#}Q%!U6x>c4+}
zwcqMvmCOvy$%{^L-uZhrzwqm(l`qQLO4t9HCU5_P!`&)*M@#Qp{fF<?*Ui1XuYxsi
z@p?h;PrKv)Nb~R6F~9!T(ft=Tzcl~zVSc@*j5wd~;Y~hjy1VDq|5<H+G`DQSr;F?r
zUpm7*GTD};2HF=3*8fYd_nMjW$iM!B`IfS83A5Ck8uBD8YFhu?{QpON-K<!9a8w>!
z$-^?Y@Z{mwzG3e-zVQ6^Zco6P*4meUySR2d{PfW(er|felf440p)naZw$1%&Wy5V}
zeE#rH2ixNGCzJ0)bG1HVIV_i?z~EisVR>B9Kb2R^af9li$d=Sfzn|L|tQWfKzi?9R
z)F1y&-AKsw3p@5Je}0b0g8zmm8;bp}&(FPm<$TN`{$uBsEm$GZAN+dy{0AXMb1m!w
z#Ab45*`NJ0$II^Q!=yu7B%8Oqo`2*+3D@g}pRcx8TFvy|XnR6t-m5wHx5xgkKCTuL
zcYgc+&&+!onV9O%Yrb2%`M-}}>HWD;_qdNM6xbOUYtFj0d&age-8GpvL(1}MX2;up
z)UI(g{d3gT`(;-4qJ$?U`wEYLRf*>NI6L0<)c$`j(tT$CTc2_K+Q<KYAKMG-%bxyg
z|3feIoJ7Cf7l!};{Qo;_*fULfn$Jt~uvO|#(b>|=GAH}Fym-}Fbk;ULxyqMwTB^`0
ziO%C2Du3@h{&WBTuhMGzdmF3sBK;1p<NtZozy75CgQxTVeTv&xcHX?>$gaqrH=5tP
z{aE<_`aUPm3#VINJ$g9L@91-u({1h7XaBkWTkTo8YJ4UO-<I^fwfi<-X<T>jtmyKC
zn{<>WavV$$h_(MK^dtRKrG@;8^t=GOlY5s2X#6sM!^YHp*iq%#0@b+w-1+z7eJ+>S
z#IZY`e}205x-iE>-j*uShYzQ<ZC;o5q4HS4u9Yj+u;{J1`un`TP>H#4THKkb;W{Zd
z_h`!8*wbd){!P$dmyPl3<734~L#IDha^Ccl;s4h=c}f!v<oEt$bDzxi|1W#}*|Xi7
zjx`9TGVc4@`~K6j+4%)u7isSJz0aWYo|nSjX>QwZpT5+spZao1l#chJOsTR(bGOG=
zi#|VL`R8f;U&+a<oHrDxG0t!2=U2IShgE0)GbxX+8>-)G^~vf!J+*qjM`>gB4)vgq
zmW#9%BljLZwB_V;cS}+AdhfZXtrqF}hfMofYmghU_xPiAJK0q2wk}#}VLPp~Y$MbB
zCzU4@i~n~lEH!yiY9lMM`}>W6p6QeS`<!1?^^EO#&U@Wfr;GCw%>`cU@^cL2TzF`6
z@S&o259ezy`NJHo|1W0&5A$>3#skaLZ~2DrnCvW{`?w&x>6K@lOdFfig?kk}{MXm1
z-z{wjzn*&}*0ytk`|^#8W%Iv=JoA;A`+jZOmy~6Fx{p1o%~$pud24mFyiGu5kzn1w
z_5TlWv)cKbPC9mJ^^SjwI7?)*(_j5by%O58DVLZ3`kim?F`1DTGLg!udOa?23dd}Z
zKIgvts73m#WP$XA?qwTuJiaV@^J7W9Zfk?#(WVTps70T?hZR*#;;HP}kvU66h||RP
z@Z*m$2I~$l%e-{shDT(i<U7Yh^7fqDaxdS!>8T-ds{Z_(_t}gFJ;LkTH%9dMs5v`2
zGTyy=SE8-){(JlXn`)b$D}FDPDQG$^aOBdYif3<lxtBX%Utpu~qQmpk%EGF`7Y55*
z3buF)`Ocp`(SeCot}(cM+l!h15Aa+yo(|fm`8T+F->-ZA^F?!Rt4;KX3Ak^~b#(d3
z4bmJBidTKn7U|mT-jH{!^54Vyq{rQIyKY%2T1?bmyMt}M^8(J^e)BaC<@xJ<(r*_|
z3e&a!)4tw#(UIxAFFZ~Q|MWen^{#B)<KT<ZCl_W)wH`e>MU>r8xBJip{@yi<RNns6
zPo1?ab^mI%-}{2%6T1=}nB<uEZ9A}FpNn#8+SFgNXD2=1HoH>wX^HmAP3P;Jn?)y=
ze%t01cQ12c_WXNVZ`2oV+&Izve$C`>-^yA9u3W#qIp^l4``&?Db-E9?^9M&oc^OF9
zq~8?kWLdxepVjH<`pdU%`!?rS*YU?TpU)VtiP#7_1<*g&j{UHSFX#}W<BuP1I<0ql
z|McA3+j^HRQ_I`&knLX0XWvt&PDN}=IaxXN?PLYU>3r9HE=5MIe^|(|>(Qj=cAFyV
zue`hdE@bs*^-UUI^S<Bv@A1L%p})RS#S7=jvAROOuWeNhz8AS;c>BRdUbWrPssCn9
zO#k?1?w<RTKC~K2ZDnne-=AJ&=dm1oOb_VfmR-Ac`KR{m`+hn7>0NuV?vIze^*1IR
z<zi|)06L|#=2eUUXwTzE_CA*AVuRdoM`QLSDDf?y-g!CAaN#~F;~JUM+h%TkqgD4~
z?d^&$_W$blY%%jJ_0`PqTh;B=X}<qhyY;u=@{cAAeQ)ocbFYq`@#CnIN9o6NzoRwe
zO=og0_Ilyr;+ZX#akDYj&VGCC+x|$Ke@9O55PiQOB}H<5{>_62AIJJgMskEr3a|gI
zGMn$%&hOIY!IGc&zOT{zCt%V2J>9vTGgV5uKQhNA&3IG5<G!0#4_nUV+i$wSvP1Sk
zDVI>Dr_$wYhRHWY&o@L*IOUP_GN<jwar4L}J}=gW&9=R+*paSs@=Ro#9M8k*dKICE
zE0@n(v}@O_=)9e+DMlx`_4hc~$UR>+JMUEc(Wha1jc4+JmoNo+O**MEE%o8YiVLN4
zBXo|vy1F`YLjvRFWxl7kdoK-|v214k_3i6Rta_Ja?ugOzxvVnP>tWGO10Lr2=Z{a{
z_eGVdkzrcuM6acOKFtC<V)RUWoxMDjCOU95-hTUTxy;faO`X$+Ht8g7ys=@2TX{*z
zhRV-rbNtjlKRc@|a;nOI>E+BxCsXS5F2DRzIsNUCmU*tyoZ<eLem)AB+iquKc|~#O
zinzzsr)B=UT`g&sw@t9&JgZ;qg~_MFTm+hGR2gDZ=C97sdR;0|z&$DL+s%tD3J-)N
zq*!8ZK3AD_YM#8O=fC1!&vc)pjGO;-IPI1zOq{snmrZpWALDd|RIf)b@BQ7I{QK_B
zZ+gnV?|k!*uHh>#X45(S=)u9}D_5@YRM|{EIl*gb!ER7_k$XGENOF!}y4}xWtGUM>
z7A(k=T9yggVQe9@Bj%>So;!c1D9`nK{#%A`{mjh=x8*LnxPt3o=%0fHjuIR>w-0Ul
zWnbdNVd5LS&W7u#Jc|&wdhoOObGh5~tX^_>+&=c2-(XQ4)Asvy?D=c{&j0hP^Z5+R
zkGErLZa!U8^DVUQQ~kfjkk5BRejWPr>Y;naW$vGoPwO4-Gx;F@|3|q+*7L&GU6Ff!
z{bQS>oG#3><K?6YJx43P?z~^LLak-0U$9+~&Ggt!9p2N|KPa#NKL5|-|G(JZ?RaW*
zi~rq&hw(OEKb!O)mdrU}q`!17uX~^(TUX+dEvnVGRiyhpx0y%ztiSfbl66~4W~kKm
zb5}G!a2)bHx!!Eno1BRutgP`flLJIsrf{Cm*?yzsPTJ3kxyrF-@8u@=tUNI<`pxNe
zhyJA%mtFl*qxa|Sjg8ETEZ1X-dE@f08I(WnHQ!YIJx}L!(&c5o5z~%7I@-N4cXs)`
z%Jx&Iyg;XBE%TjyV@IL#wA8@p=+mmxV-B_PO5fOBu5W8=`(oF*Et!{Z>?+lMeSQ6L
z$VnexE_rjdI_>%WZnugM=dIhfJ(piTGuynsnVrAr%Zos_9*Kjd*JGM<Z-Wl>(}~{J
zlVNgdcK*K3%vllBT&?ClJ0_jKWX+l$ko~p4zcCur{wjHPet!S5Oc|Ss3Fh}In8Bz0
zs7*e}bNbY)b91dFt;=*?mdx5){oSbi-5thUy^?-^O~n^&+Q(ko>^%DDY^~X>hGn{9
zlEOB9YvgKAM)Yc{zp^MjxX^F*aq~!%?;d)x%Z(TMJTOXdS$IskT9+wrhP{*eq$z){
zv#VVU`Sve9YJKr*AIA&26{XvcuHLP?hi^yfu5-b+H|>u%k5o@Qw4|taUCWckt!nPO
zRYV;RzTI5nCdMT+^Udb%KWFq_=iGkYXhz)9ivrs|T5LLg{`vgr!L`TS`{ka#-F{z9
zsy8Si;>4$?rwez+fX<RTt-pWHp+io$Zr=u-zq8}ntZW0{$=BoS=T4g@7NaNbJzX#L
z&5ezJk6D(y5Xj&ARjkvcDL~`M)9LYkk&!nG?z=6YBi{OmzcumtKHbwwlRFm8{#&1-
z<)d@QPiv+`o}eh#(d9i43xeYU-456roSc85gv<N1CRgg+U&qexYPr2W>e$8p>l;4@
za9<4;|78E~czp14?~e!S{|W#9q5t1=cj$rmKVR<`9lgUT64LWfyFAAK`@ye&ZtwrE
zo;&43@A^9~JFg#j$sG_P@L+EEyl0@3R}RMin<Re!u7i?t#GXfC{~r18_j>Z6>fh7<
zfAZ@-KIy&^yQk^@x4roRZ%nIFFLQ`a4~mQI$y+g<{pbe;RSVvOSA|RW?L6^dzi#;h
zvj=bgit)+1iE)|9Z~xhT-MHkqs=2TE1(&eSRXkk(e(Fx%{_S>;#+hT*{d!_kezQ7?
z-`RfRLB{ka@%K($keC)@?t1*G#eOdD;9%j<(9p`q3=X-uy3=FJB>CE#uU!k9HG6jT
zbEiueE;y8zn_tYBa&>h$_+Z36AKgrRm4!MDil6y_PMvZUk3G`DDGWY*Xzw?v&(F?2
z1|`<z%iFi#tJ2;-tLn>(z;!zwadmZdRXplcm#{2S36CjcwJv+};KW4bpJkmcO{Y$I
zt=s+1YTf>SRaseC1v_IzL`64NeoiZUbK_u_sP>L8m%Jl)m+7|i%d1WF`0%J(KPWo-
zG_U!chD3=iTer^rUac|JYh(U>yJi0K&mHL$zPUB~db-iU*($ql#Xo55pS9^2qi@RO
zAGTZ1CS5k1{V`Q4eqQp2S9!DR>(!knny+4y&9thi;9^Ga_3yX#?7RNRU(cvOx@rEN
z`3rBT@5q_cmzv%yAnJHf|N87bKQ<nHySe1pG_Dd~gQY=-U!SOAk1_IlUa<Vh@%O15
zyEe_^oyAsu<`U<wP3Q0JI&Aj+oUz6S4n5h`#`Q~%F~9by+kQLE$Mf4vnY>3IE%f*O
zP`Y~c>YcjZZ+FD##Z|vGmEmK*m@&n+`WwTB?CW|$oh*LK!3n*>=3MUfyT_JzPOf-5
zHN4|!QYEvC+T@38x8GZ|VZ#LLcRQN9y11gEqS}53iE^=uH?TKm&Z?NxBq+M{=c9t4
zrqr2Rwv}GEcef9*@ae}C4Ozpf?EhZdfBfOh+-LbG_0P@w|79{O3K<SbKFPNKVV&2f
z${`}cXtjA-{Y3e*kv`tp*3)z~v~-uRSz<7Ig4gGQyW#te?(tr~*m{;iK!{grB`4F2
znvc1EijHypS|(MnYwr)41)T!sbzg7ClpN(PaeXY|$DcZNDvNDxk3?I4^}f&B@8o5R
z{M7fk?r~I}Z|9kpojprz6lR1j-k3H0=q+Wl%UZ15?pZ8%nihCJxb)Ct`q@o}QsMqr
zZr)1x{7=z({*yD)0)iDj?0)c5RxCI0WV8INAF)cW`)cLZpWd{_@${6ZVPE{>KAx$R
z+ZCgC>eMMvQ9RS9?dRv`!9hWj%G3E;oh;`1eS3f3KQc1X?oPtRMXsQtroyK0@WX^R
zH#UZZg&AdDQjwC9nqg6>#Mj=uYL%AKM2@Rhum1cf%g65O>MCPV(9qV#wr=mYs5^P)
zyUX7n+m_2GXVZ~;d&%WYK6$&I!w(ha`fbj*s07;4@%()H=Vxc1|Gk%)k+ER!UfcI<
zb#HDM-nw;b#flXc?>EiSp2gr|7MUKu<jlL-aZ`Pjeq5{T-okmlKINl3H!suP__y+=
zp4-fBn6$BOKQm#^*7Hf_t5d2M`!=dqzdwHI!zsP)+T2NtW5vAPvwc=yir9PnocbD3
zroM9TYjWE%yS?mevW!=GEWV&TBXI4;^}d@jmbfjrej^}UWk%G}jn5XGo%T5L;eFeT
z;Air6kBh6Tt$BHQuUx&lG5I)O&h2L(9v+UEc65p7WYBqn#_8vtJUH0wGuKM=vdPWe
z<@#H<ZUr4(e&lhXm6cUScJ}FZC5bjiS65ch@vWc(4(09tHSM^cb6ZWU`{`lM7J)Bc
zz9dA7+1lDx{#5)O`}p(EDcd(cDn7nJnq$cpL8k{LJN;IfzF)FNWKJIkY{}j4D|e(?
zmnjCQ{uAkX=wLo)%Cu>GRX!YZ0|M)(JNMgn9G8D{yQ67Q5LXx5duQW0A-xi=S3Q5v
z&9hH^7p!h6ysP=stcg?RZ)#K!&~jO-6U3wL<Zx@snp>G4pVdk8?kHMxuJ>}OncU`|
z&y<+g3vV;KvBY=6x*WA{<@~o?CaHaP%c{w3JUB0G_Wbp>6M1~Evn;D!lruZi*{1g2
z>UW?sVe9|b8O`*ubF+N@%|2pHN#4`j4+{!T>u%pr`@8JhySt}BN4-C6lV(%-Z|p04
z*x-}?Pri2M)0;NrO2=L=w3@s3vrd84+@C2+mj`iKSy@$lK5HJir^0aR)TuYN_2%BL
zd>YMh@K7uF%Ju8dKessXb?IxXw!}H#wXEdp{}^_<G@0M85k8@odV6d3b=HL&YK2$?
zyd9NzmAP1t@2b`=>{yfTW5U{|cOpXFOLW=VoICrk{cf6hV$KS$<XaOqZho6nskCx|
zu43-I8}m0FT*zPJ=sWXfamm}3gHG?iW_6@*TFAB0<WJUO&+L#z+#WO4?S60J{KhJ<
zFD+86E%C``zA4kD86_RznB%AZ_;`Q%>uYP@+}_^rqIB@a#^jAU+?uL++of-BE8KlI
zrtarc5mC{?CnqM>?4KT2r8#x#)HnC{&;R=Rdhtb9@r@Z57Zp$Va>r9eXwR2R-Y$zT
z?s<5AlMW~}em<T4`IwicYTf?%uh;G7oA0liq-7V+v%67Ywqzua)aNH9yFOIevN-eg
znQRg4zgbcHPqCj_=*F8nAMD%@NySI(V%V-2VzCi4SZ`;X;S})TY#W<JiA21b$C87o
z2U>~_p0_i&S#agqy`N2=9KPkhI5bge7hC@t^GLh6=iR|2Hhp(i*_gDc&T3#)HoYdZ
zDZ(IV@j8#k8++9}KmVx|^qFbY`t|ko&54KE($36qRGXam_0`ocU%nK4zgzzJ`|q23
ztIZ!4%!tr&b9QEae5_Yltb1eW>#$`$GY>sEIl1!3!*<KEHxkNjJqNbs-nxBzVvteM
z6OU~<HxFH0?5?S+oBI0N+9^|~KKxq6t|K)kOHgt7h2=q-pmXp`OHCKM^`4sUwtDT_
zrym|3J~PKsSX0%y?9G9vMS?tR8+E$x?k+b@Jtb21_SV$+RNMG3U%qU}zi*ef?`PVv
zUTO9N&(6-ixheH@Ll*1JN$bSlNly~EFn7Wl`>f*MK696CSYoUw8NaOXzfkl1efgU<
zAAL4yBb)GP*5kWXPan&Wc|FY~Sje~IPwm{#Ctm*ce4-`960>1T+~b7yC0Z<Bk2Zyu
zMOMCV-S_Fs-&l^0Wvah#soHu*Y(2f`jQ)(s#TVyJka@Lbrn%GOw|~ucv3+*WGI2fk
zzQ$X#^Jvn~B{jR&1xG}9I65*;ojTQLhC!pUd!In3%cjr<F7A~pRy?>G9-n$=M<GkI
zqtE3f%a*aNzka%jmHWlJcWL+c)$X|a?$6)JCsS^0O7*tWc|J3JUgH0Me|2JZ6hu7!
zYb(dE9k%Ahb4P(rmx`ZHr|*0=E1QF@`L7L!jokSSCszg3MDgrzd{A=eYm|2INtL?o
zZq?=Dp1y~TBX&6M*Z(T~&VHx9Wae%4Q`esK2nMBvyP>U04Vq{1U8G*%_b&5iuNH8x
zHNTzn-co(v6PNsFx9o~MJ8K#LFAN2pmDJj=tC;AOH-BmIRTtUC<)xo4@~ZU;o{avT
z^KM>t!O4iuOy_Kq-`mem`tUT8O<T=-4`0Q{<Zh3kOOwxpX}{?3mb;x%^xCIcTm4l9
z&yTZWldn{o@G(r8KE3#?>Gco4Ua#Mrd09=UlO=BbbWnDQy*|^bR7<}0iy&y*WN2up
zh`9LUqMZd6GA3qbn@V4Y$?&l+4Cv4dUX~!i_V{@J^S$5iDGP8I)cz_-J2T^8@p;?!
z!w(M_@Pw_2XgvJz!1K>fZ8>E4*c%-b{+HX9`*1iKNE|WX>65WMbZ>9<$`vaXY}_cg
z{PM|9Pfvq7=(@VPHlI!?duW2rCp<Mplcm{FZ|4)Cdv(7-TU}fXBvPK7nAqGFQWIrj
zYWnc|@0IJ<AAeiMY>;_L1$2;Zb+vWgcIoxky9JomU!R>{yfDeTqNw|%yspU)`EU!5
zlm!=*E0X6Q-R35Jx~-f)qUN_!^OTuu?|eKEHF*o`#w|y#&h|gHX`%7FuSzG{L_Lhk
z${1fpY(KK_Fu$Im#%FFji=5AalY6hv^nI3j^Jd*{VIz4vX@6(`ph-s6Poon)l!se>
zuwaqyzd6I~Pu}#iH-dWqS1xV7Cg&OLb4kp${(a}HWe-cNDj)ZnM{LiN4e|nQD*5?r
z_RcStyeogd-Cp_s?|0ATmt#sUy8irKa;Swf=k~VVPGNPXo8P~C>+Ng-ot)k-SJe@s
zb@a!_$A&!2(@(cvf9-m`RH&2XyxngdyZQY-YKK2PJ>70AxnNl)vzMfo=Ec&v4hjY;
zO9Mr_xMer~t~kFTYsE<yrumnX&U75Sd!wjpTjs2i((=nU<c`Z~NIkeU-Oq*dco2uR
zbGe0EVx(oAx{3MLgx7jntF=B@$leHR`*uximy+S0MkcLO-i0rk)=g78l(}N1*7D`M
zg`*{oD8;Jyu2``~=|QoxeCoYzMkQrsi?f^7y~~UF=X9#;&lXd2VgK7xLPLZ;?e26t
zq;mPDOV7nEzOJdNOOm&2DSPm#eT9y`>-9GKeLf2<qLGdx6yMStxbVlB34Y5h3m!0h
z{ra`Cb&I9a&X3bIrIc?@{S&OnB2wXE=e#@B+22dZmyv0XRR|l)w3c%_j$Dks<?E&K
ziL0wtz|$#u{X>NZVl$?x%scrl{E4VY)k14V*^S2gjcy37YHCV<R~RTLGi!f-kk_J%
z8gJje&6wpD85y~M>h1^~Hx(g80geT)tL#h^CZD`;^JZu6Z4oiCXH%mun{c)t-f=gt
z#!h})?&Z68eYK`8TChOjez}d@aRZ(?e(C#CW}koV%;BgbcKBhzg93|=qe&GuayqA9
zh4bmA+njhR_2ShlrUe@|82q>N*5qtIyd+4|X`#Th)Q4re&(EHy5Pr`vo#n(yk?x}f
zJ7aEa>rIWU*c<ogqs6Xu&kw7f69rwf@(Q$3rssO~w)4(mbp=uZvO*$NENcTjgI$uo
z-rKjUVN=6_<dpZn=iM`EjZ44w*FOB7!4pB3Ns)(V-S@OpQMeH;wdKZi(IrdWCvDvP
zx9!f&=iVJTm4^$Aey^#NNyxY`*(xx1;T3(4v<0fOWSM79U3f;nBVA$Eq!)WrMf{Hk
z*cR`6z9xUAU+i_xR;Qm+zhszn-G8qx!Sk*7VE@siAAkSVy?_3(V$QpFc`<t8tgNgr
zcFog&IrH;{m#<za&GZrCVf*;I{harY$#<gDm1Ouf7d`dL+xPQX%}@W<N1xAZ*nju*
zAN%IX(QnHwWTuFSUzRoXOgFOl{ipDf$=t{C^#}UpY8l&)JFCeZ{~#V5`C!S;rE}*+
zOll3ke`xypy)#Nz&rg2p{XU`Mj>==T>2)v7*6mL+WH;M;e3#hl_4jIc&)a@VHk0pt
zXsB@Ovh&j&t#|&PwbuA%U7dW*WYOH?pyjswi!L2FmvZ7k!j%(TQL|=OpN~9J{%(Ko
z*Y$SC^Y7nkIQ~{uu0JrwRA{;E;=P+^ryP4Qt@_>LOFxAtXnk)4H6ZHvm$aOV_<x7N
zvOMj?x)iy0lYahP;y*?6O^4^L<h|>z?ES68$@!>d`hkSpbLLJQOpON`rtX!oo1cHd
zBBA}TW982SI!bMc0WmRi?q4(MU9@Ns<J(_heg-_u`~SO~o3!uo({HbL*=5)>_$@CE
zh-?uM5fR}ze`J#Hvx-fJ^jlt(uWdeiywyi<`+<Xep6^tHUl;q|7MgX)Nc`!n{p*uo
z-uqmAyX07k-4%sf&m+ohe)q_{es_QS-RzdrU$0+%{=Gs?+beI6htQ#ulT(CmmbF{?
zKk<9<dH!dE{N?rMoEBbKz_n@J>*jT9s`uVw{NA(r{_ow_-dpaOuycZQwn*2)XH8za
zXL?32nXGowr7gs0wua`^6BE2rvYFDuS+_VQGo1BlE3i0u^!lBH^Xs46^{iX$;$nIK
z_v-)0wtA~={&?88q;2typvSLf?>iL#|7v{E(LEPyK0I_=<T=^L$jx78*4A0=hb@{+
zYhLfJKUiB{+qum4^^NN{?wwzi8Ljs4T%Ozcm{ak8zs@(_cqrleM1%7B*Sl*TYhS;(
zBVE}&dbfxAB9*1>*82+Wzt=vE>U($3?dX!Fpp(!Of~HRPoozljY#Vr+&778n&JT3<
z?sUz6wE52YlU%O%|EeE%%}$ySk?D8;nGegI$e*HJ3m;9k;h(U#{kNSkb07bwH9RLQ
z71->jRlBjh{{19<<^7+{Hr34Mo*(&Kw#jC>vu*KwxmnA2KlBA?*q!Iu&CbZcWTe@>
zyl&sLl1nNfPnJHdFrWPF%7v>^i+@&_hu^bm-c`8tfTod#x95TkmEz^Tiw@0Rwrtg$
z12azctXZpQmL{@3?_QH;sav1i+260XfBy4Nc8jcSpNZR-w56g?Hc5D#J$vP3v845;
zSu3t5rCeN{qSJlvi^)#L!k>T6%&?v<sJu!tFxIvG{P7tVl|6qxVu@XUVX}|n(v%}R
zI(?+<+d{nCK0QiO^PTUK>nplAYxRvwvPZY5`2GEJZEBdtIabYsTYPv{$3N+HTKL0O
zz=<cSc;5x(9W{H-HI^B_uKOc-@WO*nAFW#DMD3Pa7^lDf_cyXu?i~B8nbLC>a$WxT
zvnHneZmE{y!bkjV+=m?$RQaa+nanv*AQ4k`IHd0HW79WxO6FMqPvO+A-`B@~|IYnv
z1q;)kPa3wlT$*ihVcX|K#*gdN=2d)fymT}<(jb>9YR}wn?~K2w7B9_rS^qVB&X%rA
zvokiBhbhg!5b(`Z`1@V`iI?6L_c?vMd3c-7tl6^+OTHQ{3gBU#mda}&^g^npH16B^
zFRyIHY){@}7Jrm{{r7jB2j{f3;^(F>Sbfy&sN{*G+vlG>^(Orlc((T4mYPY`uYEFR
zE%TDp`nO<*0T0XGAI@@r7{w1d@n2Qu_B?W#g?sOm>mmX#yXJq3@%-P#uxUQiywG#T
zD<$SmG}){__xu{oRGZB&buTSrclqzdYI(Tf&F$#|F1w{F|4rR4X`ijR*qJGM-ioQ(
zzyI93o}r-1{O$Sqh5s$Mj;6DA9@%o{Uz<Uz)5q%M+us%#u5*n$I6vmegLOOEmVK=*
zshMZ^C1;XZdr@iF9M$<Qjlb8}@;_((dS<SasL+CdKXowyY)qA#4&{E=db7=DvGnH4
zkC(1k!r&{uAd`ig`{S>>9tR$N3W^AFSh1yTY4@^^zq<FQea@C^vSVMmlr^K_w!DJK
zmJ24m34w}o`=uvv_$miPbR18d@G3L?GW+WfVLpjDg`gvGZTy&3Dry+HmYTm#U0|)D
zqjltQ$%C(~np<LS{+-o)=#FK|dCg6GSGYzTO`pj9b=%Z!ccZU;IWoOXfXVlAyZ+%>
zPK9D~w5qo|w+qeRRW8aEaK0I|-R>4o^~}RAN{#mug8~Ci=>LD3%cN*-USF)}|L;1x
z%A?G8f4&LT{XW4~wQKU7&(BY1ynClK%aH8|<II;T7gO3A?)U2{NWFEmxbwqC?@UNj
zhLN*@fWs`!Th>++4}nf<@V@?|X2wd3OzxUE{*M{_7I%K!@mrEB*!%6lC$<6(PZgn~
zPd`64TA(g;^ez7bqY|Ny8j{fuzW<&u&2x)-*Kzp+e(IjLzudOJVfo5`?xW33sgWh6
z+k#(B7LV!l%e<YF+jnWg6pN~<y3t-i%Uq{kPRep+IL{h)IQnCIXYTDKL7K;UrKjuH
zFNbU{EZnzKfrrJe{p5>RMTcTP<}ACgpgY5&bIJqmT^;V1Z*22japCmAHnF5;2@ZwX
z(hAQP&5Z3lSt<9$R*g+Z>6sHpQ6>AY`Lo}yj^A$-F7<rL-!Jn2UE-O)^OyBGeT;nI
zWG0YuNi!~gqkyYviHdFu!{uou$AVp~Ztc(zNKf?T*dDDa6%-Sp!3?g*M7p>e6Zs7V
zn0ziPP4#>DDV2G}q?aOm`DS?<xyuSGSQbq7T5jOWt;i<4yI#Vne(CP|x(3Iug_^&g
z%Orf*ckbEueYewZPj^~qQNBdC@D<;!rQK&A`mEFA^gHW#D_P*%u>uR7Nu4Zt{j>dr
zG$!%}UzlJt+uKDLd>HApwe`8Hx7<4Hs8_fqxBlCH|D{uwT+aNu-4VQx?dS$W@Nx;y
z9lP60Z-1Nnkv00V$sRlVgG~nty4MD1=mn@qakz&Ss$?^r@aSCG_)JKmRqj@w{L%EI
zoIm!?Ips0Im@UP&al!7r6VJ}x_E>*?@}m_(-(<2<KP}lPbN8EK`(i8ccoX^U3zb{`
z#AvLZ&)Nby)ytze#3(VwcGr)b^VMpT6W`z4yZnsn0*l5eL6T2FdovgQ5CQEbUH4i<
zWU7#sj^*laH?Jyv{j?-NgU#KmF}E@G>ob)p56nz>4j-&On|aOLRpDvEF6}d-i%pvL
zO+Liia=`EVhkw=iL1rJ7)I!9QXBWm*yb83tBmVSP^r@G5)`!;5``xf#%5>7<-P<-U
zyLoWYQSKLZ8z##3YU)L8;h0{gB%&$4twimx<Na+ni`J^YfA4!}#@*Tc(@(YPc^0Kc
z`j^(lOa7WSKe|zz!}p8k=PI?ztu;Tcom?2Tv+$wIi?I7|C*OXu-tp9)@ME0Jy)TT@
z?w2(w-@gBFZ?akPvA%->dyVEit%=XDkl}Odm1;fwP$Dl-uOjF6vfaC*B{{*~mq=>6
z>%idkGF(H%xji?%ms!$y^_o{wI_r%*qO4Z;zf#eWe#y1b|FiL~-Mx|Xa%MQlX}-*{
zI;6e7c3R;|AuT3jk5c)k2ktqaiK>z=Uf!pE>yM%2<s;Lqy4US$^vo=nob>kA_R7Na
z9zo^TlA)nZhc9rHZ@YNyV((?)1)CP}v@5dl77H;c#lGFLt^8p@^~x0*$DbNFcpZv;
zyQREjp|x#D$dt!rRvzk$Taq(d<|@kc1qMX99bi4{t9xTtch@PdJ8gVTjwu&5J^#MM
zA+6)srI2eO$rBbX-Pu}U<m_0esoJW@rhi1sVb&IN(+i^B0va02)@<o0G4ZjOwrs}|
z31?xKUDhW&nin+*oQ#+;?OK^p(DE%?<^&vQPEC99?v>khme{+Aa~hpaasAz5YCieI
zjVWOv0$+A@xE@-P+i4P9(!TfycbBHF-iyC`d~&SQdm7(up7WktP?5!NdGJ#O3-1`c
z>jmB5O#(-s_i-Nr?-y`xKC<l~=oHXvFD!QZB<FY(STLE!_3n+0)Nk&(B(&>tpw8(d
z7mB!MG8mt`y?*bZ<L$RBe!NMpxRjZ4+-;kz#j$-af+c31-E@+bV<NA!#h$;p*Gvzl
z79aa`>Go{PJ2Rg&vdcd|zwfi}nP<8-A3g+cX`7sot+Vr$Ti(uhLU(q(SYF1hXtC*Y
zZQZ{QlUsrlmIn9CuD6We{nxuhKH{{$e3a**lEkG-S9JGG%D?wd_15>C2lM5s#I_%I
z-lg7jINBq$bM~@ld50!)oZeTWzCFHL^SE8yVSTxm`iFDlqx>$<<NY&#{#q9awzz!L
z`akONiKnyoCw^^r=iJ5O+1`J9-?!sB^CD~$RO4<)-i!as<Zriq)|3eu>zwj-J$~K8
zU-n_~eEYLgzyC4p&XWCeegEH<UES**pNcg7^JxFSZf03dnQg86Gj!_z%v~Rtu<cWP
z{rB@Fuk+6M%xWnJu;0|R?U~x5yNRck#~u2<ztlK7uCncZ-3HO^=UsKg)Ytyr`S|ty
zM~_y1{mCKdD=hQtM>FGfs~a`icJeEvo`Y8U2JqO5w=^EurmM4h|Lg=k#TiwM0U9Fn
zSyo?fKA_7I$L#i#)%}owr}NzS#j0GK!Vb!rSvR)be4fME*6zK2SKnOga^rWasv|e1
zSaqJ$DtmwH><;aBPj{(aewlY;TBt><!sITYmlLO4KmXv@)6bQe={@%Uu7ztBs{VYr
z`TPSJ)A{>%J~Wal-*Z~O{;y1Mx?l3O7e-6w>Q8QxeE+C*s`f>#^iSdCao#^pl*!dU
z6Yblk=jeaq%+4QDV!AWDZ_nHM=g&n`llYx7XN_LI^C&!BTYuz^*WCX<?*DfwKY8@e
zPyYR9w}sDtaI`l%a$kj%{vYt!oAYfaFG#&8>sR>qZu!QXZ!CAJxXwl&y}tjis@wEk
zKhFBspL({{I>KP1SmV_Bii#$?X7PVIvw6M2|6jXz=6sX5wx+7<dDy-K|9`x{pYZXF
z>7GA7gk_`>FKBk}sQ#C8=0n}8<7Xc3+AaRjbL*$e|9`Q{%jTqIu2s2n)AjGw?M085
z7FT?0{GV|6VBWt!`u}8BXG+`uJbGFtKiW@C?rB7JQ-0llyR<96D&C3z?@ym!J@e(`
zcjs@h7Zv}w6A8Npu=UXtp52WHHtC#65oy16-yz|q*~TrcmxT{3^_rw9(zV$!rI%Sx
zaYs>$+Xc-Rp57(wQ+BX;#uh!6n7eMt5?e(BU5lHoaU8t$3LV$qE>H+@Iq?4I36_sm
zvLY>9oUD6VihHa7-eMIx{BoJXi=&UHO%V}IER%h3Vqe&jjAvi(?{(6;=6lTR!kwGV
z60!5-!xcPStTHltil3gHZn$X^&*~D%deb~Hz4h}~6s!E^+jqKK-^P8}@?M`~r@~?<
zPI3sf{k+U~$*IYUvc7oi5ndh`9qX9%DB_LmZI10*3sd5`zujmwVwuRY`uN6_e>$hQ
zV^c19y7g)#e{Rp;EPHa}G=r^InIG?N6)5@t|EBf(A6l#4G}fm-`x<vy*h_(PZ~4oF
z!|QIGz7=b|?P_Rwecs0FTlKq=4)%EkPE@)yb={@th6ycKrtjJFFYVd)<J^jk3)bzD
zzVh>3x?%pkGkc%Uvo?NzulT&}@=YyWZ34f`a~^CyHTO#Bb>Vf~mj%Qv9$qQ_k#_g~
z%$*TW&h5YFJMFfM%XG(C)z;n5&bWjMU2%QvJ2TnG%=#}IpMtvj?7o?iEN{d<Sw9gn
zZ~yS>aZ8e6h-lT9x5qc^ySwov>oK`{nbr6H%}TJiCQ;0w^wspuy|Bhl%)hQ(fC3An
z+&l#jwaGJhnJOjg_*$+6g-&>?B0BYI2<xf<QGT`Vt!rJcUbyf>bb*FIm!l3JFWb~z
z5^q-cr7E4+;@HQ$Exg&0Y4xS3H^N&DAI#@E+w@BI$9qBM@21P_eJaJ4@V!*+{PHHo
zs>H)ROytw<FUyMO@ju;Q<iZm2WXYsm5ut%8tg3>E&vsw_QRUvV@i^~c5%Z0S&I@jq
zGOb!AGv&QQ!6D{zoDGhb9@@LK9kP+%7#lr%9ed-Zx7n5N?!UjBdcvSeEG0N}&*$pz
z@4T&~Ei2!iESMSi>C+3-6J7T+l_aNlaxFOF`O4Xl#WD46ljNS?FTMICE!%RY&zvi5
zW_I>EKZ|lm2x~xa8}m{_j)#n{Tnp5uUg>9M>YAh?I@>Q}A@^RDiQg7~`TJUBqK81Q
z+y3t|lTTg|c@p4vY{jZoiYu3V(c)kd+s(cF?x($+c2iwuR2thSL?1Y@^Z6XFyWc`<
zw&t-jANNzRKYshsomZ3I)_gk3wToZQb^bZacJXDWSh7}gxQfd7>6g8(my1^niaz!`
ze}BK>dh6wK_EL+#aBF-PpTKQ-Y!WN?jYw^yhes|}<eohp5V1c)kNbM=^BtGZFVy&c
zoHfaAR={M=Q{LVycvu8oWM9s_`D5=nc^^LM{1s^j)i)o!JJB$?@8AdTFVowOtrwfR
z%YpY`fz0yT?RV!?*|Rs7Ub))3!TOGctbM&=NN8x-`Z!*P{dIr0+{lx%?=~|RSg~pn
z*O?TPZ4wo$S2dd0OMLm-^2k%^p@7=vsqc5+=XcruFHUN^9><FI-0UBkG98cBYIr>^
z>xhqklEd43U#Lsz!Uc==O)puVy$|3}51L%NowMz!Q!2AX57WJ{I7TgnS3Dex8s|or
z3A~uq?8x96I%}<?{EQZJzl^{@L4zonEAQ_678Yu@-qgEQ?v?EC+i@qEyYtQcbBnId
zesN<jfA2}rS*JGb+LN-i*tqytj?vuZ0=GW%=GSj&`MiGTq+PeHHWmFY^!Yz;&5YlQ
zUnffRUVe4u_qvS>rhI(#;C|Plq_V1v>nG2@b5412W%K=mucbT9t_XWaNLUqU)FsQ#
z4+)z*d-|$XJNIa5z3Z{L^hS%faONRB*PQsxO}<Nczr2!txS9W++ROJxEo~F8-Oj(X
z^svgS)vpbAZ;~uJQ}AlL{&K5;`yneXJ=%NUN6mC;@s%rapY!)eoqXOMesO*C+?&SF
zAAAyjv_(QATttLb$B+MQ)YbC$+xDr}8?IWlYKC>W-b9Zl8xGE2wQ9!Mv}679{%L7y
z45HQh9{=+E9uz3-qVB1Z+IprbE~fBmX^(tJ$&t=^GHcUsK7YS?%aMaSgpX}{ditl$
zr%OKXPRbX|Xk3u>o$cS(FPrD2zqqifalyhY^ZMPTLZ{1FEIxmlJ+tKI%Rd)t3eTH+
z&$P=|{p&XW*ay3}E7$8jZtk}~a?ke24nt*wS@r*VJ}*B}s*z=MtSK_zC^>Wz<6$=W
zBn6L6S!a#z&Dz)VW?!;~kGbLRyVq|jrxzc*ss701|CWd0wO{4#)nr@Ebp1QSpv>ae
z|4T9og6p0hHjjUL)!crGmJDmC>h9!ny}f_Z)OudGezDzczqnL%&$so<1AomqGU4H2
z1CLuSu8+U>oZ56V|HFgs%a-3B&Hi~=bh+}8tjd26t!M7sGUfLB+Kyni^1nB?u8G{-
zrar$$sKIlx+Qs$p`?uxoIJSO<nYn<;&KKJzojj0XlDJW&z>4>c-dYu*AcN@KwNXc(
zq|Dj%OQ3AI@ZX7bJ6RedoOM_4`!{RXlBpAw*%FOk))lp6I|^vr-`aYKZE7BC<(fZ5
zjFa{YH~$h*l35twC^Gv_v#E(mU8M-C2+KE?kg&I*E4t-;mWE5Ve@$Kad5zGPtkc1v
zU((q3{{H%Wt)%>;a!bq7ob)@HHzRyjhA&%U{;<q?f(v8w)!xJN90Ec&asPT?Tz7P}
z{@#PS^}G6l&si^g;mp9$z~JfP7_x7|^yPa_GJjv|U^3%{S&ok0?9LRUn};68n%uB_
zI;Xp$^wevcZ|C<bbFfwQSk3#hCr9aU`=@X0n|Hq7cl1Y%+^0={Ubrw`%sBYCSjm%#
zL#b@L^yIS~D_-=b+1RnO&7M73J?_Bi;ALAU9L(JwkZtt%qov@|DT$_goy~uh#2=dS
zT>j9rFG-trf3!HGv8hK%zQ?ga=kb^A&-ZjL|9H)P{)ewkku!SCj?FEz+giOpF}1Lx
zSZ{xeTxP(w&zbuo_VpP$_a}aQW?u2*{r88TD+`}LOE7vNpvt)Yc(bSe!c_^ItO~f&
zSQuG6`%gbUm@;8S@WI!YH^13j{&UGPsnzO}z4h!Q+XHj1)C<kMdGw^Pe!?m3_U&K8
zCm(yj?&0mNY;8SzYxebRziOBJ&Gqp1w!RVCDe?4a>W<r`kLLvYEvdO(b+4wLZT;?I
zi95#ge{D)SG~>kuJ@&xPjn&ul46IL{{!;L3sk!lzgFe;!o7b`jxtPond6o5Tb8fVu
z)oO0lX&#ctuU=DcyuCm2+^78Q!J=LpKAb)pV^PQ?`R$5}RZ#rtYipxd@VR|VO$olR
zQrSxN(5j>JjeZ{GUm0Me`R2i~nG?_ZPR=_%=N_Z;qQ0}cPi?IHlal_!)9uyGt-Dzk
z-}TN5v-)~9HRs-@-i#m>^ZmxMn=T8s%Ik+4$t8YYXWNlxys?epVavpKcdZt?yt=Vz
z>CQDjX9x(eiVCjK)PMNeR{BEZT@lf;V@X$xpZzfQcyBZ1-`yh3`+H~d->aFKpM3Y_
z)zuw!cde2xaZY)A^W?Tz)ebi4x!l~I->n!tjw!#nvu$ea@70FZwIZUT4@_R)*j1;h
zzu#nL-eRp6Q2{RN&byqPC)DF7zjJ0?67N=#_qxZQo<8#2Xk+#BcZY8atxiAv>}<$&
zSI^U@78fu7CQ*9!xGz`yxf#a}`~R1&-YpknQONUgmF=F-A9Lh>9RHj0<Zt)CxpUs!
z3hX(Tq%65+>5@g$zLfnGDLLVHbV3koN}$TE0PmmcihTU!cSdGvbaEWsUA9=_P05MV
zPWoBLn0xPADKWSAsyyz^JX29FrQR#wxbKU%-JcY}nSakE1pKdgwd&N3EveEITO8u*
z7-frs-mmqj<&EB+w^&W$=-g?Z&;Rg6__i-Ucz07O_hfbdv=b8)8GPp1NX|a%mYA3r
zZ65bH-0QY9hX=3aq?0NeBi0-(bPen{nsnv*b!C;qF3OHBfi^k28x^`j-u~@NUv}mh
z|MmMG&wa|fWp1;FEmaA8`kK|avi4gzkCfrVgnRFg*{)%aX^Eb^p>p3x=I5M?4xHP1
zIwEG-1w;MbuiDcV>Q9iku$*J=1tazyTqiCnc#G9JR$X6MlKtHM+9&huj6aV~Z0ho4
z59QUG#1N;fYjcrVQPj0niT}jLYwCTg7Nmb^DSlaerc>gK*%vqE=dpI2b5Hsh%{($Y
z_DF7c*@3)^4`oV}-u_zo!sF4YH%H!<|8LERIr!9I;rB0(w(Y(ZD9oWWQ|Cvm+^Kz$
zXOG9PSG_6G5E?2fQyFWSc_tul@8`T#%wIdIjP2&L@AXT$a6(w~jbVpJ%)y5xi_|91
zSfj@BoZ;Pxx4m~)Ig6I~?D<f{vUg@mklN}Dh4L-$o9`}izrNCZ``xSiJlW<S?~M~O
z6z6%;cgOC=+A0y(gYV8hn<bsMTrHsgT*kpPBLV*QpO5x#zZol^SH-@{O8Kw5tpERu
z!Y^(V3mhyE=|0<d`>Bc~Z_)MnyB?QVO<>va?VJ6?0M|KX(%bLfZ2T3#vifYJU(m!a
zYCTqs+V6h|tId|R<ZSk{o4s#UgP-rB&nd-MW$O>U+Oko9=7QSobuon}Zys?{T;nI6
z*UoFH8EjF(^mekARZ5Z7_G{PT{bwDPcbcS9_5I!1dA8Ln)~(C?DE;_r)yuoP%_m*S
zk&R&pIQjb4Y~ypb^FQ%gp81(0Hz`i(V!+An`u#Iav-0D9UH_lR|Nd#Cr}?Bel{)@+
zN?Ezn_MUALvae<P>X-X9<BxClid#qH-~Z(H4$C|C?AxSw<pO=D*H(W&DL73xj%(UE
znd`R?e&W@R++L}Fod3k~g}Y{`zyHm?dH*lIxd$DBS~u(MZ+*O5V&<oVHIr7InX`S~
zk(ADl$1k^i{AT?Bkohm3OFegYex7FS`MRE~|Ns59Z(@IHn&17+v-sTZR_<e4o;TLt
zIltZdTk?gK4^BLu7XAL&lfc;eEin&Y?X{`7eKx1}w2E?K&0P1-7v}3f`CD&$>Wt`p
zvy)ZNmYNse-T7F+MRwsXTbqLWu`F6z9voGAJ0AMqDfs7^6J^r>+&<=n?b}-$(|&2M
z(aLj3)BXSFcKnaqi~ASf-Z<sAeD$<XD;j2XOpd<NUj45~K6aDO9^<E+|Nb7!5pi^P
z3VTs{Ch|_z&5e6L-8tQ|$n#o_W&8ePId|%Qa{Q}0zqdrb+U)z?f9~IMV~+XOS4~Mi
zzq?o4PupsZZvVZ{>~?kk1nr`}=}x-Wu!dt!@l)NH%6p!^(TtN8YKhhS<1}<utKa$h
zzeQ?B_>~lv{h?0})}MdUt$!`xiROccx77mIc)SoU)XG}DF{R|;K6bff`}IrrcQiS~
z-V}TH`*Q3G*2k644%!Rf`)O)^?~C{?Ui;bZ|7v8bbosU(d(V*nTP~*JUgV#j2g7H6
zJGT18_j=cWg5u-y=iKk@pLqNI?)mS_rYye~ea<Ft@7uV(x38bR-2Fc7+pV(_K5kA6
zH6H8lm}kA-PQ2Jy+R@pWIe3}RLg#kAn4*)a3^u>tYzD20s(!y$eD>LE*Un!5AzD--
z!NJn_=uwiEmevH7m6`mh1`<BJF>k+J*>~DS$W+of@Qm&zX$}V?!NuFm9D)Kn+?%ej
zE$&cRxpL*<M;|84e5V`}eDD|V?2|99G=EHHPc(cLFjZyh)6^?h+BaC)q@?}0$f<ic
zQgspUmDZ#es#9mpVz64=d*;lRW3>~wRw*ssZej8FlJ>y|PVHVwQp?t?nUfK8@U{Q(
z$6EqJqt2+ypIxBvOy%@l`S!DyuC2O$;m27;?F$E06RvvpmNklTvd->Xa+BrNslJ%L
zCFYMl{|<h>TC1#mvF3((DlZGIc{L9&zSwlIM8U7I;qJpj=N!u`yYIGMExeGCdH%$r
z@_Np#H=mq)^JSs9Lw@<B#|5Ubt5<1p2<63Fy-VA)auwGk!R-f2HpJ|n%zT@z<X7{3
zSuN+u2Se_EstOG?jCzw2CF5&lELBqS<{oprPGtd;hDXXy9Sx2hd+P4&Yj%s+d$R9%
z%>|9Dt=9an&Obf&uuOi=8uNLRt)~6Dov}GND{DSq`|Vy0jT@JYJh#s%+_hI??>w<+
z!7pdO+^>qh{j2X!s$~L4#Et?*uf-F8HatA=`m0>oJ`K;BuScKX3EDXMQta%N9o#V%
zNfKtlEr+&VQFGGtmkL#jiFP&;VG?yZs1RDJ_T)+~F9(OxnwjR!tnoFz-yfzIe~fgl
zsDH)XqWm$8m*uPI?#OLPwye&6&le>ZNB&-3zv;*6^%i*!cG=HrkFd|zUwXcCdf`h+
z=8Ei7pLJ$fc5iIXe?G;q`RDa5l{~gDYtC8a@l<VXOehs?Yg70)=bT82<<Fv{S>Hm#
zHdkG>`gZ-~mlyXM&)=_~s+#S!qv+E77xUzvO{}*&F=hL`<QogBKbYK}R-m%;82<_P
z0MSE@QL7H8>Fs<I^y~Zk(|t=fSG@nM`o;3$f;I7H{<Q=j<8u*lP&;#=aQcPI6CbQx
zJ}>dwb8n0PUy>&t<_!rCohZ`VWB1Nv-=9y~WfRYfhR3>3Q#^EdvifH0chAo4&Qse|
ztjTX@+xy)v?!eJx;~6$znb+*xyYuZb?H_s9cKiI#vb#Uer0w$gxd*O)xA!iapS<JO
zC*#6dS}WJBT&O6+=HPeg$~6{_*2fFBT$+|T-{|dRRUYftTIXbMPd({xdnS8weBzM~
z#V;qk@7iB%t2*^9<Z*_5gI%mOlfsKLJQ5egf4`Ow3fw)nLTP?MF0be@9ksg^wp+8)
zKONelAGz;E^shNHce#az?z#E1<W!+y_EWBJQ^EuHrl~GXW%PS{N&0r6A=A|hKjIY(
zza8o9`S-*-ZQoC$rd35}B(EFn%dfsy^F(jjx|^p%x8L#nzD0ol|7XXz%Gb5GtdAQc
zAJzI))&F=`w}0~IuA?ljofS5nk@X)QPE_gq`1ttc-R1Aqk`Fwc9)E6E>FX$|9#fNl
z%(wo0tx0&JcG|PVN|vp8VdmBQ2_C!$eAR@l=JL4)UYPmyh!7`_+vCL4r9sMGZ~o><
z`uompyqDwF=gO(hGv{;r=?kH`2~U5mRr&h6;xDgq{hlNx>FKqHsyB-(&Fnd{h-<N*
z&E(5{^7m4%in(^~m7ezbjPz#1s53_v2_AZ=ve@5lcBinL+0q+AS6I5Aopt-b|KQ9=
z*P0Kn_D62pDbl>8>EhX=^*cIrx$P6vT}_{EzISk;uto7950}u-N*6BtSSrB6XgM*Z
zI#b{y`=V>tuX9&SI{BXS$>pM=$`{)fG|#X9vFF+Gn3s_|j5qPGTxEFRq2M-o*-Y=C
z(>ruueXskTe9E`huEOVn<cas!9&gzDN|#A_o5tf4yDToH#O78ycE<&WibjgQsDHEa
zOBhSZ37K~lr4@7cf7Oj~b#;AnXXoVD-DQjurcZYd3=Dj&zIwv5uNQU}Gw!rg|9JlS
z$D_yPjTgPx6k;>4dD{FdVmpqVcY6GPma;}+*&9t~Lrcro{LhR37{+d3E#Hv!SbOca
zx6ii5Z``TkaK6c^OZ&|Wmx!NNi%w<vryXBc6w=*YtZy;-;a{sui=)k%4st~9+<9d8
zR_FW+6Yu9T#C|U3n3k6}>2k_<!%s5~o~g}Ki(mVz?X2{^!}oXE-@RQKu`ye_OCa^{
ztI9W%LXx`XRm4sxFu8fi;li$%(;D1!T2~t`3%Gyr@s3xgPH&W+v}ezQ+>p-;H@SI!
zkC-SbxqCBn`n*Cj*1oEJGybw^OD|HH^y%Q+$&(rnt-E{jtMtw3XVOkwP<*j^e`2<$
z?#3mD{@j#b?Dl=3xsySR-_Lh!Qc4AXIK>UpZWy$cy+5Y8mGMsI&6aPj2CsUzWZ$2+
z@GtkuJF`Bi2=afKJ>%7J=Gx_}bZ;A_KHSZ+nCqzYHACASb6*v3IQj;O^D*CJ-p=IF
zpmq6AXNc%h%^Rt&t_8g`PB0Mmx0`3XkK?jkl$(uQo*?(pjL(H<44r3w{JMKh<X>IW
z`*}xRoJ>D)K>5ykDXIN|I{UuOslOMw=Eplnk#t7!dBtjCrcQ+)%|sMW?FucvdGp6T
zFP2w=J!V~2zaoErWs+N`wcLIFIv&ZOj;|AbsHSb6S@`(aR*Pq!fAU!`TljJQg=cI{
z&$dLIG?MaE5mK4#sp6?L(??REWRi!<&%cwiEGH@ma1_2;DQ>*SUG4q8FFM})qfYXQ
z&pomr|GMxNvBINA(-*On8U1}+`Ni^^j-S-k_O0CMpYHYWzr3|s+xAn9??JZ7A0?*#
zJ}p^%Ep?8`B6IC$bq9_c+b@dWKab;4<H0Fj8dn-J`~_CrIeh5qY>Rh$R!c_BIM=0m
zy5l&j+x!Zt*ZXg}KI{}XObn4fYq4PAM{Z@$Dz~)jfpMFku`6HiZt`&3#p+q~&p-9g
zR^NnB*M)1Bn$9>Q@i2Vp_Wx>Wn`hSj{S_ECt*G(k_4V>;#jgW5h`KIY@LnKqMu~8n
zhwgV)ub%}+gSx)YXJgVh$>Hg$Ce|%5VRFrW<5dkRUK(9K211Mqw*;IfM!%fy(s8tV
zy5N+=+rN&!xv3!-=#o{wCi(3%+g`W-Dho4M0#>eHw`7f0i@EJ;!yn6TJy9&2aHVAH
z`w4<6J~NL#m#=a8IZcagel25UT(;JOjR#LJXgqx4Tht5t0)`0VjkA}?t6yJsx98=A
zDA@vwGZIpFD}r<LG_xy9C8po?(K2doS$9!l!DI&ynV#zxVv0gpMV1zJFX~vZ>PJzr
zq*;-OnBmEWM$;70ug3&#GBi0^+-hKIIsA3XJ8k#$&x$e}UAMmIaIA1J(ObMifGbQ{
z{ju-W)mf*vbuhi)VcW$oab&`}n>;VfCloLhNhQlO%vj^MspxOb-ZI<M`)+#O>xk3n
z6gWM7osz`G$@4BsHtTIIJ)U-Pv9P0nYR1tOOB60upYSNkxg7cDCc8^{?xNoZj9<*C
z+_Am8?O6U?y}xXiE+(qqTx@P@mEIiHs$o{2A1P^&p%Ah8#Hm*&mRP?_{20Fa=I&zg
z&&&6pT%@}D<)>fCB|i2$u38CoZYX&fRQ3H`@A>JcrUiwD9^F;?TEZ;nhG4z4iOIhH
z#fKaUUR-cmetF^b*McmJu7MopyUuw|^4PrK?)~URcW=)-SX(_|X5QlR=}cm8y%V<F
zFqrMDves;mocL|?w`zHQ>2FpfrYy0PFe}pO7GJxlBA(aw^B;EA;Ltglb0y5PBxYvv
zx%EgMza)A;FiiT$4KJCD)rY;?efYW4YaMQGW8&KVeUbNbnI#^oYK!^W5BDBUQ~SxC
z5_sXl1&a<1&n>45F6$ls+Wfwgd715&5C?br_6o5KhrB$!*j*)=zjlX873bweZf<)1
zPgRirP5Q-8UnAbm{aCrj!nTg-{<<eO)#V&l-9LNoR#lbUjxPfGf}w!{F1EH@+#6QD
z|9p+RKX~Up>C?~8sm-wXBf09{_PI;FuT8K|u=)P~r`w(|*=+mj{%ZN^PJQw9kG?EY
zT6606o^OAIuhd@eKc8Oju+^?#{QqAQtI3Rtl8GmK%3sXOPIagbHs+W1$iF|;(RTAp
zv+u!Y=IOM_8w)Hx>Y4a_Ud^5ozN4X~DK~_5W|&K7Kc6_K<k1g#_3MjLc}~jSnNnta
zwMy1@w^(A}vh5SLy4~+fpTDQSIWsU=<J3VPx2)t-ej%ccQzrCSF-7l7d^N?~;{TV4
zU-ZsAyxjYH#p?da9&eJr9sev@nrCL)_D$t_-m}Tymp>9`e)H#7@wB%qo~`%a<59m{
z&X(zZ&DIw4W#3a`)|*^9qZXzYd*U$v;d!Rr940EW+w@N<=DxhQl$-Cbj#t^6=Df%?
zaeEr}Dr!3)pL=DA^4zP(Z#?#U^m(fP#~**|lizPYF0}f_L3d-j!*^GTC%b=h4p|kP
zYa8&b@n7Z|&p^I!)9u%DJO2{@a4h}v#4lxel2$Ue{ZDAET=+5DU4WG()b)xd2UACy
zdMoSR9~CdtMZ&L7e(7OU|4!#kYx0YUi=XdIk3SfBVZNZU+lI`)TBjZ-K5<^8eR|WG
zJ3EUb_g0x+k1dyF0NqMtYHG?OXR~AN>@^>?pRO<vkg@t?{a~M3mr~kh$;fH5kIMWg
zJnXX2aQco_D>S+HU(g8*f6PBqfn&v{rb`xq;Zm1VWeYZousmEKy6>e8qsY>azrtoM
zS~WxVWlYuAuQ|7NOguZ={L{J9`a#0GCUG)<4r1eb_~9xmZ*RP```ebaAAeq&XV4oj
z{lBxlMa7dx#$v<E_t!4G>`h5a+qi#I3;(@J?$9YG-dJ^PR-fm8-EXePyeh@}qV}h!
z&p*G=`S-b!v-$3`j<>3GTI{X6v{+9i@IX@e#F;Z2k1kk#{03)C;YYP)XQK|S@n<S|
zf33S_Bg^cqPq*#=v^TnQ#@ilQ>n=H0*@+CFH$Hx0oG!P!Ok{HH$Iv?;j>qo#_pT;x
z)-KNUD6{$<Z<31khRN)8D}J+MYt7%YRnyi-I{*K3@yBK30?VM)cNX0|{Nm#AO<#5g
z8CCZ$Di1VU@c77Qz4Y+X39Os%r|smr*W+|f&QNT*=byl$w>P)WzV*T3kEEo<+`eVy
z9{q*FI}+dL-K$tXbItU4pSW4l?Z>^Vc6G(6H?254cb;F>rDm0`h_`x1Mwbj<si>@;
zzEOSt`E9fKBe(r9?z4Sbv2Mp#wO4YL^R~xVvXv;c*0vc|Rn!~Z-nsMPHR+1PQ__3>
z-tvCI<L77ho}cIStZvoB=S{->&zpb78>inZ>XR>3o*rW|ae4gjj`QjDPL7UtZ$GET
z*s(8K;{S4OWOP;U>;qq`yDJ~>ZMFIKBB_UuUs&H}cG=6bwkuiO=G4g5_o-BTJ^TH~
zgRkKMA&q=@+lyary?%N5;{Ff+c8R6FJ{xw)*tqakuJ2d1y?!o^t_BtsX&)T!{m+qH
zwuX;KCgQ@i{#k8qvlY77YF<jNunSLK6~HWfZugu=tE!%zuj+oP^}~qEviygHmAJ*r
zdwO&EpVgjtrLV#>O-FD4E2d9>bpP(jF_Q-E=k>Sw$P&EVk2T@?x>%oCCZ5~@D?UyZ
zG@H(HyJU`hr^^Ou5ut@rtFp5UBzR`{sOgAtH##&_*z_Gta9Dmh(QNhvAHSx;m93|H
zG`im0-7|&#-&Z^HyT8M4?Ce;6FY@r+`kiy{m`Hcun|o$aQ0we<`zCCkS3hIHT6Xi8
zO0%!3(m$Tt%PoC;fpd=4E_?0$KW5xKT~>H|@An5BqMHkzs=3W8&|2slUvNJ-ShDES
z&p9@$^yAO|Y&d^?O40TIcTfLQeZP5w%d=&x^;2%`pYL==;+!V)+&$*Sg3NBR-qTc%
z?f?JP<X_$L!kd}Q6|eH^kKX#rqGy+E!ngX_x5Xb{J5NhxGQGOQ+NC6_E}OSEo+WY`
zXZv9h5s?IgKSrll7a95a<}&<j*gi`kVbhJmTRKaGKAi{&pCZhoURb?nLeGL%K09R0
zq+T(*q%f|(?P+wyAy%X5@Y{na7bbSNu2{8-?PSaCR8@(b13AS>#`06F#rT9c+BRC<
z@jV{ux@_IHSv%Goeq5@o#i^yO#c}k}jX8NWa^a8Ts-}9ZXlyHBX+GI>H)+wjWo!pC
z4!+!=$j})v<CcSM)nw7>hwr6q=y!L1vwnq&X7lC8hu?15Y9Lm(Kc?!Ao-}XiR^3-j
z6NC#_MV)x%CKA$A%BWQMp(5smOY1HN-omR5$KNVQW-YLjIr#Ep(gjbAqNZKDtWsPL
zJS<AQ;Mu}t8gh-rA^pJgXUEgLB%?w?mu*|s<xtDfeA6vz7Qb_{)*%-iuULhS1?CTm
ztQ92}*Zi-Ww5sL$V<jQJ<V{u^W~wk<HFeK8UFWn>C9GX|{`KA~r@Go#sjR#c@8zq|
zqVVd&()34#OHTzpRcTDCRXV=-3d<|QXqg*2R!mn-SFjn&wNI6mzW(9dYX8s2dQy9A
z|C`C}d!BVB>WxY7eZ^ICmTcR_){}N;?VjyIMmz4<-C4){ey7INsZ(7ZbLy0d*R7X+
z$+6Nge0|(mVSgLO14}$7zqr0WKH5C|arl|QhmH#tJ^K37K|sKDqh`UR!w($-r>MPW
z+<x0vVNTA|stYfDci6^#nbeW6?ZL;LK|l8f8b;(=yy04;@TjChP$w)v^p2#gW%;9%
zd#sc7`0H%tRgBgBZKnIp%rn!AInwoX^$WW<7H66oF1|{+-?qNu&+AH;P-gYHrFL&G
z$6SeeYawg2XhK(9-`S3<Pmj-7^Zcs#Jg4Qt8-H6Fr&ZXW5wqUQ{^il(otuB2Nq)R$
zvRh^$he_Skj!7<VX;x~@?>Yh>iMn3=){~uKqGa-KpX|%s6*hi_g&*~!mM<}W@O^o0
zzmL$qFF7^7?tJUaZqChn!+vVxP0!6YXYac#eKsZSxk;_rOg{56Y2C6*y5FXK?G)0>
zH?CA+etGs(W|>;rYT3<SAAEi1d~2b3l!v#yjqrqnDXQk@Pfh$`ShO?x;>;!azF`Yn
z=Q$jD`Az;$%I3%U%gh5WJ^uD1YDz+B<%V50Yrpj#pWk=9{zmwVUbUK+%bu~HICnDp
zSH@<!oc|MV7q#SS>V8?ItZUrA<xJSV$tAj{u1WeVSt>ha<x6YRW79vU8y0^wtCBC~
znjTxQDNW|WkGo19RyLXK_LJSK@?Z9<Z7Qn!5y|@E_I7>q`!&Uv?@LM9w=X`_d|;LT
zrJb>An{<LBBX7RYO;D5Jo9v?&7&^75hsPx_<@P%xC(gTxj^<BN<|H_a%s$LlG^fo&
zX7|}P6|N(T6_Y$S++5@0%&XDFw_5e9v)Y1y=wM%Gi?)kNhhG;eb82-)Olb??X+M2*
z+f(C;UwgOSDLQ?1#!{9^2WM^C(Jj#3(s?pr^Q_JmrxlYJXW#Tt5?i)w*R&QUzv=Al
zXM0wyQV}+M^{PqYOxT-Q?2X6P3q(%iyvF__c0%vP3%hy)14UlFdKJZKv~1zW-!3Aq
zbNrqQ_Og7em?LXlHevq!|J{`W85vt_xlE`2UdemxL&fC+OWj-F^m%!Cdl%1He5m=s
zfwX58Hh!t8tgE=hx@QJx^eAxL$l2E1yh?e7vC3S=?YCz{Z4~j<{ZuV|X4^838`Ipf
z>w-V1y4<;EVso~ra<$eAp0qn|#!U)ITW*~^@!@g7PS5gfJkwGPqGzAlcI%DFQ$_Qr
z6DO=59IM=5H}_u4`TQ)!=C)(`yn!Nx7Bcqx6?l8^3%DEcW&ANoiOkCuG^u0J%eQ)W
z%y8PGuQS`)+KvVK&z9z3VVtzd$3jMNp2vgF%Z2-oce*XU@n+=%^RU^yZEneqLDK|V
zt{t@tiu?3Ta6|tT?mr(MXFWa=r?&mIK;9b*hS|RTH7~keOgxomQ&&-{H7nNP@%IB0
znU-f}WMuq_5pi9(Z(p3TPzD(MIkaVGqr-ah0CA2@Nq@~kmKu69Y~lH|{@UV2N?l4~
zt9jc`{|jFA>!e4O^0_oQPALe;ZGX0O$wCnLcz@wpUY1EJ4?q1V+*ND;s^+1?uY>;$
zXB9d)gTUh#j-P`rfmag!c$!(v%(&(5l6a5=0xKU4OjNjV;ldBw$){A~?hCL5u^nbM
zwCNF(mYXgt*t*NX^x=%A>(37_@Lnr(>ZC+ysHuwd=RMquS}wmmc58;WP3Rsk-&D!p
zr?dlDY%^cZT=M&FM^nL#uMCq?j=U}vTD4kH#Jfph$)~#2S_%opI|^@|J<)0Lbg62{
zwQI=+w@;irdo<DT-wCs9aA<%CnOYf9HYV4J6Q52y_@D1d&$UH%*%^P11q#=E__f*R
zZvvwhSJxT^PoXA;yKeIj{w=@Puy0`xd;aVrU6VM<|2^jHlfJ;&&}DnS@b8l96)XD}
zU4QMVHd*AW>@oTO6}+~eHamK5cyyWn=XLMko59k@8k$Z{J;ypd?x^MRp2b<`%VW%6
z%B^~>^x{iZi0iSZ?@!+UUneVX|6BZ7-DdevPbF=pNg9m_559QkJ?$6gdA9eY|EF8O
z;}u?iIr50p>bqr|b^gwo=XTDUzfypgxA(qMk&$0$sOhFf%akOzmCBY&@4ot_`TDzv
z?JeuCzfXB^gY(Jrirc$1rKF_nrH~f>tYKG>ShIMYUd!Ew$6o&UnRz}mH1+GNs|;UW
zU-$of=Gu=)P?DLJI_V<Ymn3iI#x)u(kBrzh%PxNT+}=V;O3Gcc>Bxe_$A!<oBs^~u
z^{GrZbM~`o=XBov(Bt&sHSKTj>n5N4r5j?Rq4Oijaxr_!A%+v35~-n#RlPR7m)*an
zJ#P!$^XJ#?6=wYX|5Jsg-q(Bjk6li6R@f2V@;hF$M4oKboLu>)vHfJ${Z=LKIh$57
z*!Iml;CZ-u;R&Nj7ky`)KR@HL{^Hj!s`mYR5}T%X``FFtaqi!1?BDJ#Sy=KqDz2uO
z?QQ$qqrA`OAK12)yYku5?LR+Ew!2gI^<_qyFE4Me^U6D)b+6wo(*M2sRiG%V<E6_h
zqwCe?{b%p9UdFyld7nkdzF3R-?r}4Z+WurVVQy?Yww`Nh>gM|o4vH@?esk3P^KxxJ
z_l~Qd{v0khD$dm0e*d??@}76UHfKj}PUH2r`>Db(S<QD*{QkP!y9K@RhA4~cRCq$L
zFRoj!aB=B*g<W47b6+y$pO0~wzAxG0>lDS6>#s_9i|HP`w9sDoQs}<8Z5@+V8NYn7
znL*1#jjOu<$&F<8$r9J5%idp?knE|tF{0-vx7Y^rRnPOw@0>i@9BhAJ?yc012h~p-
zs7<{h?QRr!Azeg$eu<Bkd->w;lk%TbIJj!uJ0YI!En)kxqD?f7c|+>WtVM-0=lIxH
zzP)vM?~9^cUuBx^O}%&R)RhYtEWA9`lBP_bo_O}fnH>+CS8Coo^_=_JhQegSZS}SX
zPu#0K@lnHid;Y>Dd!$xt_a0irG)K%<iHSq$Vn9Rh>+1pX!YA}K4!qxVDIoady}zLq
zmy10=-&nC$WAUC{^S<2qR*~ZdUUG3;k7e`ZIX?W2uNdBK$+={+ZChAp+p+fy6B`e^
zP5OAuByEp%@8o0s4)MQK<?A+eoT*DUSW$0R8k(AOYg4a>lY;*K20qJ|9ZW8-vTy7v
zl{P)7@otNc)${TLS(oI)4o}mKo?%mI<lMI8nD1nD|6^&J10y1CT<g95W9p@6Y)qk{
z0xXQp7jGy9H>{HK_}uIm@9%8ms3LPHW!oobIped;a;sKVg`~=7oV~kz?Gs^F!O+kj
zhF8t6r$@hZ5O85pocjA$hJ%P}K<JdkFaG_~U9f6p{jB@KyrR1!e_b{IB3~=IytH}h
z|L-!|`|kQ3zi7+I=;*nr{Iq$^-k^l%<*OtA=5ogRNws^1Xs%s%WXj#<d9AVmlLPlf
z?wzzMplROT>PO#S1|%t{DxaDE&1UIpJC}*6F*o<_{5OmH!>upAPAgZecy+K^e{HS!
zuhPupXTs;DJpI%E^5SB3=60FG8)K`o?=eeBNv)mCmF6^kpYx8F(>z~1;w`CsS0`8X
zkolQz#E}W_gn$0nnqE=%iic&ci;d>x^edHr{;u;DbYV1=d$jiXyXRNis#Y-v{^GHn
zx&Qy?GB0ue=i83oe>TUHzwm4(bL6*&(l(#2*{Iomkx8~b6#wq0{j_zrnyx-RIQ`k8
zMO&tE=l!w1KF4muTGK@fKkhzodmp>_ymGPnx(}ht_I}j*^*3Hn<j$Rs?Vl%p&wujm
z_q*r2WuHovXjy#T@pW6Z<+k<Hm%raP`)0OzvVkOL>!P0h531hn`4w~KYz_-AFE41u
zluy>G;H2twh7WHxpEpW9CGu|X_qZIhvjQtV&cFPOjVUC^@j%qclmr8bjS*)o=2Vrv
zU#%h&v>_tb{Z6<t2h%|Z(cj;uPIfsmdz$C2^Sw8o%)ZceHYDX0(^ePZPmYzYg3C`g
z2mashb5%v*_~A9N+vlBWl1tf@Yj5D_&pIKP#g^~)`Ol96LTB!}#c;9WQv=h6n{#ih
zUG{$N5AG(1pX+p|m}z`@5!T}OgS9$XK!Ek3y^#&asx!;-uQ!{{UBoxjeX8KA?I)&{
z#&6uXWZsuHt8ZbeS8QaPrW<u`=He<P4h1d4X*E{8yx#W~FIm!ZXC3p-M@^dzBt3Xf
zJm%6+aDV;d&0X=C-+d)RLlw{Mex*Oh{2uqBj-!tib#-N3{T_YTaiQ~merHL#=!gDx
zx%7K`e%5TyG|mpwPdoR<|3+Xbe|b#VufPvUS1(+!hzgWGt}^N3u@!}nD+_Pwta^Rw
z=H^UQecwvy@ZS%8cD>7)R&O%XJk2Nd-Hzl9T}qu1GTqB3WUrS#@1DQ!#36mVw0m8X
zHA0lHT)QUFx3n($smF|6v!p^)55B6L|LN$Vsb8cj?Zo|MPFSzEIelvrmr`d+^lC@v
z$K@?YZV0Oz{hF*l^XIeE9IH+lrpFxqvF~l==ZU)`cBScu-z`$Vd)dk6&w2LBr)$4w
zzLz=NBf9#;YumopdUm~MwZ()*R3Cj*N&9;+uua9YN8Wzky}i}ulFWMh{}f$Y8@+sK
z)72j*WlA64$Z!?i`r7P)ycxsXn=O9JmBo6Kj5|YEvpTz!o`r3b=3sjBE6*}GzJB_r
zUlwe#Dz3)rbE^1`_rK})v$?W+LE+auhEA2Mcot@Ex?VDGk%^=@U)=gd8d<?d&#w3P
zHDP7hSox&5!z|?UsmJ`AR;u+KUEv-Y%44Q39#?8xdu#b(lW0@hfc91YY-aLH1+fG*
zJ`8?-)S2;F?H4&`{gaBPlifqFE)%W#Dqxv(Fl(93SAnb7l=61J<G1ihnj9meba-Kr
zOjQ1weQn|MD;j!3HDy(VIb~kHwRpM3SmMfJVI}r;`I_AdTyJ7__qZ-KYK`r>;B6u~
zHR_9Na(zd8g^kJd)jRtHSemY?)@*ze-PR}7dOy!C@anbFx(!SLk92+)={X$>*j0AU
zaq85?=c1pQI~$#u?mg||1dmT&54~Iy&BYm-_&3~J^Ikn;nWMwRGjj?Ko}FW@)4TuQ
zOS3~&=637(S!8d@*+2ZWbIp%D>E}If4t}k#ou%k)mwIXMe<zb@!N7^G%7^`I`aRS6
z|6WX*ns@(1`R;GVpQdnI{#H^}ogG&Ed+YTLb$`uUl9oR-$vW6*Vq#J!5gO>|Te&p!
z&0V)#4UbKA;_~$-b5G5^+R^0TT52oHF!$)emEticI3HI%UOr`N_Pm7i*KA8pi3+-G
zd|B=-5X3l-)$-i6yQv9<h4-F5H>?XtZ<&<;BkOPUmY=IYl?qSydfVso%f+<!RSO)A
zJka`lPN!>cK+GOq*N#njS7XY&v<zMSl>+(`wgg_ec>dIS@ec)|mbSl2Qcp~9TJvmL
zu=(bjXLb}mj*|KOv!-44@`WG%7O!|%4!%8_Q};^s&fZ-~o1!+{etRRwY>uD0n0Lda
zRR&#2c@ytC2(U=4`u^$H>h*#f#R@;0rav-US@ZMd@0t(CvQwTuHC#|EWg03P5_Vll
z@=QufT9V<b&erBDtw|hSO%Aaf<xy5<%u}Ynb)VRil9rla_JVP$Ln*6h=vIj{Wt)sH
z8@^b2<?BI?6`oBCH6nLbzVO(vf6bf#lg87p)z3b6oSNFSkb}A3w*4`CiPJPu7ti+V
z?-y-bHfhGH!_UhaCoNgEifix9#?y&P0kJ=YQ_dY(BFu5++9I#2MZBdgi#+%I<GFK=
z`IxAs=})$RlyB3g3uJI4m;dze|Eys4YHP`JwrhEMYRmXHu1#V3Yn{Gel@@O)V^UYw
ztPY_S+=<1No}YILhWzaKs#P@US3*nU<MR)HG|pYQTJQ1anU`n&dGmS3`yF%l%{2;~
z)ajh^anh!GE%w+wtWhty1qC|Xr8IPPl{F<k{ESg~WTZEpd+(9NT_&$x%W~t+_w=Vf
zJeB9aU+-tcSvIAE45?2=Pb_LYv&JXq*|gj70v@@#9!G9$-#sJ2qR(Dgc>3K#{kJwR
zEOZH(B6e)kt4nWnBERt}`|qB!d!4oTWR~nS|Hto^-%C09s#hkSqf6^S!ScW>$r{h+
z*D`&#v77sHzV5?ipN|<Oy{Wi2*RK7!+`jo4FIiJ=IPEcho2%l@*32fNu<Gc9o=tTp
zW#anZ`><+mNiRC|aq)*J9*0%yLtS$(Zha~p{_%Xhjqmr3&x`)8RMxn;rRkDG6XVOB
zGrod*RgX__^#1pEc@^>DXFAKP$4e@U3?!3Rtm@lyC$C4!vgOGF#vUHG7k5o(ZxRx9
z4b13pVeDXAq~XaJ=+)~VFj+-N&|#sKlp3Exhs4*a6iY|fSlI;^1h^Jv*au5V<vLBT
zV}AK!!{j#?XUomq?6}rRgq4R)znP6s`F3{dfd!2TXO*;9D^Gs6qy1?7+O+rQV$aO~
zr`A2~&9c+4`!5>)ec+U-B9!>@($X*Q?n=);&+ZT!D*F1`+T<G>5@(!^T2S8me-3ER
zfbah$8=j>qNAle}!>;qFYOmb<>)s0#J#8i)NH8(8c3+Zm+Ugsd%DTUgcb1$z?-KfO
zN~d^Ik=~&d;x|5?HB$4ho!+rVU6TLu!uJyEcD>;FmLGF0I5>TQ`E1XY!!L5;jAGeN
zeGeDpag{83<f3O)Ds%L{&eN;r_m`W^ahV#*)?+#I^LdMTlXy>8WSweXQDqyJt~Ncs
zR51Jbp3~D`mnC27R#DY{9#>^ryXkAkBEI$d&3EqH35jxdbPy7mrZeO47uVZA9`~9r
zUVraE@%oxGE2qbJFWb?7a{WJ%ndhJFNoN1>@JjlRWhQMZDjls(H~z@(eDq5Dq?xS5
zJdPD!O-GBDeLlFR^~9nMtw@WfZ@MMz*RX3CUaQ&LRc70k&>8;W>#F=0mo>TP&PaGU
z=Vi^VKhbC0Iz3F>Vm1`sTN-Y(<-qwbE6;z(&Hg%z)g`dq-nRSO+fM<XK4naQ`g~%&
z&52J>vtQo4czl!P<;wp8k$uwlHgBBoD3Wme*U~*j9`SR^e-&~})VeqCQ~tDkt%FHh
z6AZjmmioNg@ixn;I?TB8RkiP~H#hQip3B^y9%=kMFgW<&?e+WH3SP>;oH^~u)068r
z>|MhaaN?cK1Z_R*=_Si|woY^qnyeFVG+Q^@Md|C;9Ywo79)GQ;emnQ74QCx2+vce~
zYj|YNU9nz}^E_=*n0|7ftb3lD6a(|I-o?+~>=eoW-ItP>Z)9|7Lv--_=D_3g4}Z|V
zwz=q{&?$8u_2tI}-tGMDH?QhtXk67h!;(ckXJ?;YTfeJMG1hpcmY%Zl*|Y0PxhsFp
zc3rWm$9OG&a~of-kpN_(Vxt7#KE2(&c766ax6j=+PXAcH`k8>gTBiM@6g{iwE<8~J
zOuW3k*D}5^+}Tj1*>68*N!EKYF`cMB?dANBkN2sZ4{_Rd`rN%wLQYyCInihTd{uq&
zMW#o=!AR!8-RbjGHZ0{|Emq@oCTDT+`Mw=f8Qg?){uFRM+V`}^GGEZs(Lwafm$#RN
zpU;;6d@7iKQ?@6gw)i>Ku9mc8S1$axuiU}Zv`Xn!(8@Wkn<pP?imTc4#kgzr>ysra
zGwsX8z8y0Ba^m9WJDb-((0OsbLs0p~mO9a=9}Bd6vy4+B)85_Lxh8J!EYQhQ4b$W6
zX3m-=#l|mp=bG;|aQ~#~umB57piAGej_cPJZjKa`{(j42Qj1EbMyK<)Zf>ouzgKTq
zwq05OaPy)QE3_nX>-;P~`>=>ytKB>C-eLcQ8xQKH<VPKwQZ5!)c4^}TkDzezr)M_n
zEia$emo8V^Q}x{-Z|CQ%YiBBFpWhdExX`)E;Pv8ugJ<6!uj_n#b9bF#@csF&=j&<<
zPkFwQTI6H#-^q>tO`l_r!mZ6aeyp1O@`15I?kDawQCs`uESo{ymOoEoy{;`y*|O8B
z;98{mrScCM({y8x-@bg$Mf^_l`T~a9T~9XT=6>{_{B!lyIR;zQ7w(cew5j{``n_Vi
zV~&14zxTlM&(n%dwuHPAb`)8md-s^T@w1Enk9?M}-!S3m=6gxc+Kpe{U2MF~#qzcJ
z`-T5shEEjy)3#lF?XpitCZ_BxIdYhHN@&Z@H*K$9UOsn@Jvi>c;eNY?ufIMQ<dRu<
zZ*tJrRb5SsR-EYSIy~KVy58HHVY}ylvc3Hy<C)E``4@tw+3mG5m+E49v@j~I?nk0_
z?M3#(_fmEKZJScOe$TXdm1TA{PhF>pUtgeMbTH$SK`dLp^wF8=a}Ug$o>ur#H2lK$
zVyXFl)s41_%-8RhiT`TbyXVKDY@LW(b3gk@aTHxn6ArQLJNV|=r)+)a6?YFB>+d<Y
zYwhn7vN0WNBsuDgcHV9K{OQTPh&O%=R{S=N)8pk@57!#3oo>CFyMO-&uCA0riKZXl
zKc9az$9wL3pZ7<c{r7rUu*t3FW10Bx_}pWcncFt*UnC_dxZZxQ$$D-_hWfnnxEX)4
zP9<FL(2)F4!58tIwQYX2<X(SE{^idX@VHf5iQlg<Kkl!y*i6DHebc0gGdm9_{dwq?
zm67pB@8Zkel$4afz}SOtB;}P~t>KX?XL~-UT(NoA5@jv9yggsp=G8ytomc<Iuw>E1
z#N_j74vE&+))Ze*w4eW{EKcmqEME@U^!PH<<lp6oWC{&Fwz$@8snFeQw|aSSnf0V&
z0%D8j>DBh9U+NXD`F7On)M{rDv8S7>e};*iby~3SqrJNb?~hBy0kwK6f(H(?eAZjO
zW7WK$o9`v<<l5KcbWYZ;?HsG$uI=8>n=9ov{o>|qR_#8$epiD{{pkhGX0v-w>+SAQ
zpI6beGI%)?=tx*K-&spoHLm<PDWl5PH0|B(lL^dEKZ{@9P|W}A4Ubi9q1~dm`z3j+
zFY7yIl=v=lja-@Wcf#Ds$Ir!{{#kPDc*|>Tk5iu-C%n92`&Qz@W^Ku77M77OZ&;Tu
zj?Lp+=qLN5=57_!)!#R_E|;=RJ@fa8@S=wd8!M(<I#o5N;qL9mWVY|hce51*jV~!$
z_vVM|KD(VVsW$ZMmi+0rk~KJ(_?CBXF|qGjlu%|N=(?e5^UR8Ol7BwW?^jMMdwTx*
zyoRd_l(p6SW$hiGS!Q3K=%EtmD*TtNe_48BwTo%{!@s*WZrU^@W0U`>dIy0Onzwy@
zS2TW#mHxS8)r?)Msuq3ydc}QFNM7vmd4CzXR)2HQ@o;?^w(s`7{x^(o14E}a#noCY
ztLK+->nwci_UUePydaOOW7gh#5sUcO>PD|<&@el7sbGgr=|YXB36so4)}%eZm&N0>
zd7p=i=GqmGw_=^b68YP{z3#iKVfgPxV9tzfLZPB3&t)F(n|nIyMbLzj8?k&PRkm{d
z;?1)X-rq|~3bZ}Fe@6F$BOa#;S81L0Sj7}%+!@;Q<%RVGy}6$}u0_9?^ITbO^2HyO
zFL=~0{aLo`yj!>I{-gIE$v93asJuMO?)Tq+&*GEHD>DmQ6&`$AwsprdhDqld?$`hL
z_-sZ_nUC4;hu2^AraarKzIpA`WeXR6Tzz2v^=5Ik{g+ZEh_J|(Eobi5e7d$}N%|Mf
z=a;H)STKE6<Z;vfXCm*m_sEChe-7`1&mGR&9vC0(?Iil)*Pj_YLK4P1-|6!kZd@dF
zGUITnu~45&VO7nWf6u(H<?Ai`eL#>$_QKTG{QV6Lao&Y17G7kF`1^K8{wM7OgT&lB
zeJ{?wJTmvY?U|*bp(-f_l{Gu&hOs=DRsHYW%&Ajdl089lbMv`ZPAx1+OjHbyD-{Jz
zH?IJl;-`Cb;j#0NGz<kyoLL>$AGXg?nD=k*k7xS*lgfW}&M^9Ozqq$S-L|*js8`qN
zRUX=B+(H$_R;4|>^faaX)w!;j`I}Z*9Q-h``Gb|}wl^!XcKvZMa10F$%;edzd`Z(~
z&a79}b5$nJo9F45XVo>yH}3ellI-{D`;v{FvVyG?HsqhyimUq?%f-&KbVIG!@9%l`
z8J8|dN*a7}IVW{h*VG{(bW)K}Xlsg1n*xW*WJ9Ho5?)Rz*LLS`Nb~=se!6FslKRtQ
zw}S6;J8ibCS{yMcMN!#!cFFYCHEVcwecAlT^i<8BEMBYK-J4{kHNz)c@CdCAUzeD2
z|J%wb&7T?<bSj+o3~^mxxGQ1xSGg4t0>O+^1VUb&`nIt!*wE7Xck-2!YfmI8o=sfl
zWqTuUZ$(9X$@>Y%mgNdv%$U-$Mu9`<VSz+sV8CO=wEf@RwExb(6d!z$siEcYLv?d~
zcE`XIC(fwuP8L6M>_IDgzNO&admO!mf-0U)pKo3P&1F~!*&kET<PegJw~~uL{a9@_
zbMWIi_oh_-n3pv9sAF8U`{nf2wqLhv>k98&I7#yT+DD%w(-vLSP@h*R_?TT}%T_BB
zle+2^Mkh8ZvoVS?^dFxV^~SqxcAs<!)6Sc{@7Y3IU0zx5<%pC#8^>0(<m)EZ7q=I8
z%wE4+s9~RNchFP$hcPP398qn|y~@R3BsmyPrW|?Tx09!v&9>{$-<K63p^D2lh|Ai{
z|L`q4KVeseQe5|eO_jzo_oi@%-<{#VVAr%Wcj|cL@AR*KR_9<LmA_|Os$O+y=xp17
zl?xog+Nb}CF<*L6V(zYQDk0mZx~?`0x>BF<YR<9HT9!tS+J+r#(l0z%xa8&Qhydw~
zsIwtYRF<k9&AFwrbdzj{mC>eRg&pnX_hx%eH&j=gmiuIFd0N5cRQ2HSVDGfN|NkS>
zwqBg<q;l@m)8d!6mBp9G9G$Lhej)Qn<cu#)!3q}nW$bstIyI`QzP&m*&Ghum7@>s<
z<>s%qzk9yq^ZLg}99K3TnYHu%=Tmb`r-bx6Pf^*nKKAGz*Z7Oe7f+6yU)!mxEZ=^$
zWr5tnxVarl>MJ#FpWgPiis@?VnYlj=mvX8<=XvKmYa!Ql#Z5W4q?2Z7EW5QvBSO~B
z>RFYB^&0)QfD3yk@Qdv`$}WF)nftuvna0mA3VdFC%9H7;>DpI8`34J*9Ll*J=(BEj
z$2VE)4Vk`{nU!-tcqBfn+sBa}COFYS)7U=N<GI~o$$$r}Ul$ZQ=DuRi&2DT1P3c{1
zR$tz+-|maPmzNi(t8he)Us2H}L+P*f`wMt^d#8oeweNbC>fsj5o%TQZmpNNOxPATR
zQt8%JP3KxJd@u5NeRbQeOBSmvxp-N$vNr!c@T&iW|N2#0Vg6#`@pY2x_y03HZ}ZQf
z%G2R$q{Y`c+%K-H%9s8qxHs3ZpIt7Z_i&=Rwyx}46{Q)v6*YP;o(zFU9+o9<oEgBl
z_hQrS=ZBtrD7ti{xbo@N@`^VHe+#lS9%@|dSsb-8Yge30sECpE_eIqQweHRMBa*h7
zx%p`K?KIa3B^GnT!p{Gn(ChtW!77QIynh8}x3ZY&D5iKle`I{lX~mCe>G4(bzgo8$
zWL(j>_w&W)|1oL%r9IV~G7e~H`ON>Nbg+P>{`Wo~33HykA_rf7Ot|2=!}ycs?;g%Q
z)}M{{#Tu~(8W}x%(#dk_Thhn!6o>CMJ#NYh7nc5a*c6_#pyGWU3yVyi^6BSw2ks;=
zvsFup#}pXpZRQKTvOR9SUf1Q!)Te*=f97rEv($Wi{_OKv(tgXc=ZOcF%BY=QA$<Nw
z(v{>bHw&#MNvzTHv->R*SNZm?Snb2j(HlzT5B*HPVo?84s4Ty5(!AUAUfnHpnLM>o
z+3nN0<Hgahf<y{`eR^?MbMDRe`&;kU$u91*p1ODG`cIcG_bYNTWmdZOocB}~(@DF!
zNp!}g-G&P^cxPX3O)xdhmECAPm;J7bsfme+$-mbpHf>w``gnqY^$Gm~v)ZuG(93h%
zj_LO*IBt+hye=-lKlzm2r~RT2t5*w4{WpvVu<~b!Uq9t|bDNts$87JW++A-ZJ6$%s
znp~1MopaTxfBMrmKb;x=qU`wA6$g&B&tLrEm$#?9B@0Vq?(WkZswHcey%k_-EHH4W
zbXDD?^{AxsV`XfIQ`Xmg+m_ABh&lLfL$gluwwsQfmjybLx8Iz=l99oo^ujmdtlOlG
z9K07<_;&Mpie#=>zjT{k$GL=KuYG+t2=_j}k!z+B=pdjV#1$m%F*n`xWuU99*yhHz
zUsg$-R}QaOrL{=oVS!bLyR*v0$6F#Dr>^9`q9~;s;k#?TO0mYQBa6I0-BIRGcwMBF
z-t*c^W72`A9}~Ar1g6Xka29Aj+UB_2*<snXU9%D_+Ky%&e_J-Gqpd?FWKLQ1WWkV7
zQK?t24QKeCnfXmAn>lUW&KEjD6Ai-Jx%)3S%B*|(vi*Jj-Z<C1OO2~maWr*YeRm+s
zY=OzTW850;CzUz{K<f^)%vOsA3h=$=UL5Cs^`7sUpO3Vy%3gc--9P$DKi%N_ZH4;1
z)k1Z%T}r<$+_h!lbeBaF*_s|ogv}Q2_Vl+ATC{2xTj-UCMb*KdRE4ixH&57d%RpMi
zX`#TXj00I_4XXsQG8~Me3?%!UX5ZHD(K*fXn)!>~iDN&kGuF2q``=VybFNR;`o#J3
z#}h8}^xC&9(ztT<YNl{3Xl(*>;Guv|tk;%)S3F^ulF!(6v;V{XZ9-d`x*8-_x~#g_
zr6tM9`&;|toQn%G-uw4nv$S&ies-R$|9m-~ns0R_y({_H?W*l3oxJjDXS?X)RS|FW
zKFPjYcAovtTh_osul?=c)TjvOHLuh(oqPA;lZM&*&LtenH8G!>T)bnWwe8tcxs`sa
zd<>#xXRJLZc4*bAuES3=L(`h?mIlZ<9(noE<kSKGNjFo4CMbWcxO8bl^_(A<j6Rqh
zQx6CgbrjiX9ou_-^@@%iwV4$x3s=Uvb@N_s{3RvGBG`VqS3~1QY}}tktlN*bduClR
z)#|u3@wMoYu#yF4lI%hA`KG&u2ETmYbN1Jkt@{k3-h{lpd2G|w`wup~Si$KgS$@lM
zN3L6lV{Yhlt%OZC3?tvH%w=BHbuUL%`}#EzfuPV(7SHz6kNxH{N92B6X6GcZ(Q=OO
zeb6F@j6daqu7d4{KYr#0tv&dmDxk6=Hzs6qkH_EjE?M^PUPLSvSJ9Zq{`sAwz=5o-
ziKRRH`EHu@dEL6>pujO<N~+UCk1V5A*BkG-1z%yTe)T@WViMOQ_0t@OCLXmjh_ty>
zzE&o8W9go!mhJWB6_T$KqtEmE?5U6W+b*B~@2C37H7h#UwA15iOq<Si%(3~ydLe3V
z)@i5cxo=jj>RQ9aWBZPWZ3R=6<HDbM6aH_xr~J~{HfE*n6U#3$p`jahZr<@}nYD(l
zwz|~x!X0~4mi?Z<^0oSP-d4S@zkk2~{mTcmaN_a)ew9qeZ6+oreR3gzKE0u=0wrlI
zdw)I``+550GULT&RV>|$Iuw{P-Uswv%PeMYjLTo3C-7wT*V{pjzxS~RifoL?wTSs9
zANlLOgrmpWqW{yww)Tp>PLyN&-(9tzU)12XOo`Qo>z^hh^g-5pKuPyysB1kO1)`SB
zzHDLi|7z&l_l68xo@V{}UO54@hyspfW}2A`G?=}-4r(`DxbP!aSv@%<^s0h<$8Keg
z8?t$C!Y+T03y<A2H}Z4Xr2Ewh$G@D*f1L(esN%lj3TqZe+hdEnc5j}AbX`o?(6?)s
zKv}!=^2g8fT?0a=UR@o|d|`dOeQw;+kDCQ<M=^lb`21JvjF|H2ZS=&+C+Ek{0M)+!
zOOno)$5elPI;ZyAO&^=rI_K<u%dFe~R_@vQyWjRke_Hh|=gCZUxzu~&ueUnvkdoSO
z6*BE?T{HJN+v)XtExT0|j%QW=duVO({?Y26MPJv&FPUbVeeJ`O$^M_7O!nWDc6Qdb
zye-GpgO(OK1g50yO#Q>?C?Mb}7<eT|0<^GVZiLCHa^w5UuWP11I<KFTT?m@DSbhS0
z{=kjR+?IJ)448gzGkQGvrj6v4i*Il4pZIW!#_he6b9Z|Q@S0hPi%oi0s=%;lf!W#@
zCptB(x97s=&(B%tx;ZRex2);y?eZY`zK7?}EG|C&P2#K4$%vM@*4qNN?U$N-*LnH5
zxgV``8C;~9a&B&#`D&L&W@Kf_yDM^w{iLfiGyWKL@W_}=>M>tt`1o$-9FsbhUzhZM
zewgf<^3A&0@$cuAtF@9ZPs?=?VygVfT5`hET|^-${_wT6w^wkw6)sP`WVxR?Z1TSi
zQlK;SpPZa5X_Ugj0Gl^o(OYR!CzYG0Fd=nj$`;1GH!e9SXmc&x(Q`28c!pKgq6X0L
z!7hWFZzT5GmDO0u9)D_{Y?A$6n3uQL%z+_5^ObeW>%2XTJG0b9qr*2>J(pwRO+2-K
z-s!*nw%(N?4!%cD>b^R>_y70ul1Gl=iLRZ-wXckAtV`LN{FLTRd1?GA=H2hhp#hs`
z?tZ_sKfi2B@|A9tkcr;w_BWqCCvp0CzRlUo>+75DB`64SU0(V8<DLHp-d3!5bhG|n
zZ~yfA_9f;D+Dy-5D!Iewl?$=y9@yD-OuuP+Ex(@4d!2uO9<27MejNJc_kJaf9XlWM
zKUaQ#|HRwxch8qwpE7(AQt+(sb?*0~oOpfz`?cnqujd_0FcRW&m8?Jb^-ks2oik?N
z<N)1!VwB1yYgO`KlD8h?gPZB|Q%_CNyi@&t?~R;o+w%4uTR*|fTtH>A=NY!fJv{d8
zhwiP^I#D#K>r~b%rqC5#?30h)y0!=ufTui88C9p9b=kgkhOO%TiT5_k%eu+@o_uQ6
z${VLYEscMCLTf>IqSgI8-NpIy*S-1k%6r>2&p8!;G<qg&d;Y%O_vky_DOXm1c@@{P
ztgd&ltfPSDy{*mq`yyP9Ub~ueNA}1yJMP#uvsO+<--uamKkN7r#%Uq9OahnWyw-}F
z^~r4Ax5G0_?%ve-pJ9}LrlPkp+1>C;uzTF`Jf7JFhCA1(@c(~n_e?kPXr^-YhLU$Y
zO4G{j6dvEo@;xspy|N?1=-8`P<C))H2K8BD*_iox+tytsCUwdlJ+_5Pi|v0-nwh_^
zXTd6&;$x;g?{CjDm*3gFH2Yi8Ro(JSwz@|S|F4-4ey?i&w{wPPA~tsIx%YU-yGNIA
z>`5|o3{;ZyPr4>QXP@V!BR@VqetCCy`DDWfpHAzi-`!Ojr4`XT-)FWoM@Xon01J=X
zW5vp6yi0x_T(Ew*;yeW*zC@$wp9^P5pJkSl%N2YZ%`I^$>{I8Wj#XNJX1tpHdh^;6
z1rDXA1N_SlUAd5PZkg_Fj_|OFYxsq>RNOTFEqjOcZeeOl?fcTAHS<{KpI2FZePhY#
z&(20?woIS*>@@G!m=agBtEsmuziq9wiJEZ!T50^ppLWd2$NHCLy$%X{-&^=*hT)7`
zf8PH3o$&kme53T~UeohjXP7cK9t-T#=*o%TJcqq}-SS0?nC7lzE{`o0G<$V3D9Br9
zr9u6-<0~%KzUURmx#1s_*YEJ^`{qR{(pz~QCw<-h<HzEcA@A0>iLiVNzxMd;#d+T>
z^UP{D&naEV!240fkEO(o`{qwqm$bEyR=UeAtp5J)i@Z$_&+d8pWp9=#t_s+bD0{j2
z_Da5Xfh#Jd4-FPba+sLZX%g6!r@8Woh;65XZPP`!KzDoX{c#5lhd%xC;__5QXReL2
zqWSdBq!~Xx)|*{;=J{vYG9Ty1_6y&(Z_v76vC_WeQwHYpjGi-^7|Sy{Zar(P4RKwj
z7vOE8C_g1!q;Qvw#Ml4c-18e`_&1l`tuo@Sn(%Vl-vcbMsuAK2f?q7&*q`rq4o!WP
z#q9F<Bs156wDP3wv-Xug(&u1$x_eu|tpyhrM+;g#6o2FE=pe#!_QMBh6`@s&Yo2@M
z?5z?!m3!P_`rNbq_s`BdZ-1!n;MM!tk?XZ?p8K8ev$R^LS?0UKqEPFbTN1gQ`*wE;
z{;dv(-!!d4sdLRMiRx>rU;3I|J3bs0pKp{KJK>L?tH_qq4*Hx|!a_If-z3$1)YGp_
ztxfse+fdGs@SuJBx%S?4D=QOIYvv1?>fo#TTYJ^Sp0rJyHycWSJ)-QG<!s1e>FAql
zd+&?R8#j%+;a!W2Cr+L_cbDAOP{~IV6->VdvY7hS8o1^CE_-o>W$Ev47X>^YczUbf
zeQmcWFf4RZ$eY{lr3O8hU#n}Mb>FgZnu9@~{oQ^)p%trFD)sTrKkmCm@xZ=Ie^o*{
zznsar!)NSe<mg+<J4q!VG`9KTL)FXK%lB-V5)h+kY|Xy>s>_OX(`GPvDLMJ(3hY+z
z<%#z)3-whvoi&-Se$PHrlYieey3RCQR4=?b^Kj*x{QAW38!pHBW=fQ#a;)moH=mxU
z?4Bq%nMWeRIV>?><<zWW28%QnEmGFsF1^@|Pth`iLAhD5s?_`W<nm>oPJGTc{Ig{9
zi^wZ8^TTwUU0pBSw)}m`^T+4u8_V+zPmAd$99*RO!@9&tzw-t9M!!9M+rp-{9n0^-
z*rLZJ!4t>EE9189`l$v-=7Y0b!@}Iv?iTWQ?~girZ|?3H_J4iFbZ`1jzb?Nx`PadZ
z6-bkx`csYety<Mp`udt-&JBYzx3+op&Ud&i&9TqlVlLlYKX)CeqiI4bxvmI_x}I5^
zV=lmQ<Llj`W&8f=J^fn9W98A6=4bI$rceH+{M?Ea<&KYE?Nm@OnW|{wJ8ky4;3>=G
zpEX}!t#S5}i0OVi(~BYkN3MKie(EZ{s`BYa_M0KX$CIusUZur0douU>UGwbTUJjg9
zX)UwDIkbM>8FpSxr^gOfrExYQ3HRLJ8-AH2v&p6`Bs4c_VL1OpWwAABr-Jzdb9EnW
zd;PEBu!`Bqql!i{TE`cDJ@zWQ;%g`Sjfkn29E~O(n0T*mvY&Ocqpg)<(Su*pHgPOE
z06M=WL@W5lQjWbYlLJ?}#+!U=ujn}LT)cDEYYs<+h}c8or}w44SP}T+ZJJ1EZtU~+
zyZKJ1_XS;1o7rb}e(HVg(&}j!S3j2d_U^Xp^+nz%Pn=5ec8vdaerDhL_XeAGeb4%P
z$YsV&`Mt+q=|0V=`zookQR3A<)tbG9;?l3<-1N2P+&JH?u!QaP?XYvty7w|I=X-zb
z_I#bx=+u^W`Rc~~cfYGoQq`__vhw7VYi@!f6F;W^ZPY$zFFE<$hsf{WzJ9L!adP^K
z-X@vuOSMlj?^K<2y^`~<>9>6S%z{HLGMpDG-$ed-Ig9(o|GP>5zWfL|BgXx47em$T
z{<4>8-`=fWxNO@rVf|mq*XL9VSzkZ*{9^syD^6{z^{iU=-q(HB*B<9~<>kg(w+g?<
zuS@^wuK(%bA=w>1OoGjGSLgjTPOjQw9pmoR^z-ulCrhu}sc1E2d@6b=pTD?0+y4Le
z3#;c?zRWwe?RxS5_;<@ce|%p2YyzK6*?;?+A%bq}Pgwko-TC`}-qG1QkLR1;_3TYi
z6YLij;wgDK#r@{Ky)3J<Pe>IAeZOC7dHR0!RP$$R59F8&=WqyKw*2vUZMosT^5w^#
z^SKCa+kWYq^2=!F$Mt=)J~eILedmj`QD3oN{Hb`k`q|s>&pjEx-n#Q1t7XluO!?BU
zg+iURXZ}^#Ek1Wval7z}uC7HIDJ3ub&?Ze&%B3ArSM+tPdGxi)^Vq8uyvIK<J$zN`
zQ~62r+v__%7Ih+})#V$iuf_>&a_QHLEqIY0uqj!-<aE;4N33snBPFfgE!lQ$THCRF
zK1UIO5Ya&OrFGlS_;GPpef)UoyvDk9>X#>`{JRr*gZuaeW%u*N=l3;jUSO~6){}7g
zSLl@EhuJSLF;hBiRQ&uLX!UYT`Q1{6J%2u(zH;S?LB)p$+wwLYTR&y-EeC}zrLx_!
zv(L6&et9H`GayRTEf9QKN|S^0*SNU)uck`Zl=AjH^SProRo}jLKL5Rn-Zgt{*h>5s
zFI&}g>Q$6V^UR(zK00o;8w)1v3gB_{6j;Ii(9ipLTEnMJe>EmeJhO)944duBt9!Y$
zR%IE^c$<>CEct@>%Gb?;0h3vr12+|YzA!PNKGJew2GjKG{<F?EYn^6s7G#sF<~z>+
zqD4o$V-nx``>wgZs)nKM=if4^dWMFIR&8dUe%NKfu21SG=5%_h3#?e{I`xZ-^eYb6
z$rXPkcg$t)(JspR#Jk|c8-0Dn3j)X9eVRD$nPbkC?3d?w*W8Wq@sUr8KFRW{{G3B*
zsMIT_BO;=vf~o=Hhn{_!^sHEU#V5hnp-)u49^_y?KL7AXX6xYa*kjL)mmB}Q`P}e+
zp>?fw>LL~4_7jg(x1L+QU{&wxKTK=Cv~qA9QRba8ZQ4W;o*y-OLWPpkPq)TBGKih^
zRcW```t)PRHy5OCUmicbW=$IN#Ds=TpSV&kI;4fEnH8UYtG~i%tLfGgO)e|+8kD3?
zp7|oYYVD$3dM&44AAeuC?d#*uyOra1w|j&dHl1`)5mO7ZXWp#bA;I-{%BjxqcgL^m
zPJhd7cj(8p;um_mIVRt1S#=^JX6K9C-rIa%#QbJfFlJw`bMD`q|L{&>b4SS2jdgdc
zs^(O3i_2I~c=c#~@G~7p1vy@!6~QO8q>db2-ur0nR^B_GpU*yN;98mRYV`~*_s?&R
zF5i&;HQnatpVvGRHWM6KC3RD#AHM2SoyisSE<tYZx0<Z&^=Ch?-*^7prN<MTxDNYU
zv~OJ89{leulU&uW;%{>+rIL&sOI=&E-|w26T%`JL_h0{z<?~N4_uoAoF8^otcCqyj
zz7#Ug{Pye0;S>pz9FBD<lV-ir5WicZEZp|>$eM1Yo1b>Pz1OXA#^B?lX8sK&|FYb^
zXB=O6`Sq+xx_hj?*1E;j2rrktJk|33z9ZTC`%mtl_e1z`+nL96#s8hX)@P*^YH!{r
z<F=_xxiYq`_+W3c;qg-yAO5*V?o5?GULLY?<(fqqk3wUrP8sg`b>jcV3U5<>n;#4(
zCyD-fe6jpSXr0x^fC+O`6aSU`jpaF2{^8c*`o~wV-%CvRzFS&(H@Aut<Gq^PSN)f^
zx4A_xxGe4IfBwMR4T>E~$?9{C-Z`p$b6Xv2wY1;kPq*_Hy_@*F^7qtq2lb#E=@QBA
zd<Q~y_kB6zBV(rhQnsq_@mli_f3)*I{CV8JP`H;zy4s9+SIMFiB{Hk8<t5%b{(kcH
zb;3V?%6`}-8*w-Di|n1r)0fpA5SG5CknrG3vEK6f=Q4Nh`aSuzy8P3zYVOK67mrHF
z%BjUwn7zHU^}YdbZ@g5X$pi264qtapaA6>6r)M+YtsFDG-EWNMRXpNc<~Nt?z`EGo
zGfXlki7g0ttbX|!8`IX8^RHQMn6>))>yMRtS)?W%5$5D^OWs`U;<ar}a=yc(TlVw3
zumAB05%rr{p{Vv;si&$@_P$Q?rz_>1C#oiWG?4w(_i$DBxrG16Qf*92*{U|rbg}3;
z_e}M2>Vlej(Srd~6xIEnEDAmz_~&d;T<z!Dtizrmk*Nn}Ot$>r^H}rdhMwiiYC2Y3
zPcBU>`WY&|vFzOLnw@#(YI{9`HWj^F8h%+lx?khfhhIh~&YtyMS}MY#;+S}{$<il&
zORxRC3gcg|cx}F)IzFfB<kpm~-!uNa@}KxK=bUvJgS=mDTfSGahf`C8M$6gz6Sddv
zo0K!zL2J&YhE*yNHtAOKXQDcEx>ihT4L-MT#>c}|4MNFhudmLyp3E_=f;E2quQt26
zn=W*6PCGaA-15`=kLFD2a7jMGD*ZgRO4XVBkI(#HJomSSt&emNDgAMIbH;r=epUVc
zvdB8QpU0bZ;`dFPCtIsC+j#DG{)ta_JhYYmE$PBowkH0RcKes3mCNPZ1y(+KtNB0e
z=Q1IkFmATRE<vZB^xa;kwk6$k^74Jxmz?hVd#vc($J&{u#d4;mqI>@S%I28<Z{oa6
z&$xMixNe`$VrYGHj(O*!O~sz&vx4229348_eiXdgAF)-v#dWy=*U^)&Z%hm*wJhaT
zEa~%ExAR|B*Crc{S?UM>_q?)=UUADn-c^*tbz887*0E#y&H~vB{;ZDF*wnL7(P!q#
zmWPuiU3>&T?J84{>U{LlYF1e-|1s4K)^l#?u-(y<59?-}d${v#%7M4vl1!ynWd6G(
zlu_gUvS4>kgpO9G9e?L5-JJUy_(L@wF6O_dF1AK-9#3G9s71~psa0#2l;ks~$CYut
zHM(@d+q%(x7yr?eVj+`fpYJ}i-1GHW_L0}8JdaH3jnjH?>h+-yb+#wA9?J;7!RfvJ
zU&`{|FR!iEK544GQd2+qcUw>j-_(VC;#%vz-IXv;5_l;m>}BNUcwA?i-kh`7SG>6s
z?tHwg_nt@li(RYM%-EE`{M&5pv%jCtUU|1(HRQ5Gtj~irpVucm{WN>Vx1C28``i0%
zJMG{z-+5cT@_fH~?$Ewq>At2#O{<bhcSu+zSx83i6nXY6@bQ+w8<w+YEY*J5X2Cm=
zW7Vol^)C#L@xAN~Z3;eT(X2Xo{}Mfs>`V1KwoQ5~{cYhNj@s;uEv*0c$o={K^!kO_
zYhEz;`-RKZrfE&w(RKOlktyc0+W%ecJ~fNs1<#L5e{T4=O`3VQ;^$Vm8M=HdE^hOS
z^!B#*DhJ3PO4#zq|8wFM?;lk?&pxJC2i_Mr`mR7p!!X6M|8Ca9C7;)&o!FQja9_YR
zf6Iht-;+%|8rqB3@0nb<JkCpF+L2cUk3R33Sf}w|a$DQ6`kt%z`+onkW$j5@^XrQ7
zGx+@dRF#y1owYk+%~&4H+Wr3=X#O6$w|4i%4HqUII^?u&&nGWXA;JJ!YO>Vg$`8<P
zss)yvJLVKE&Mx_EHml@_5+_4Xn(ZVNMNcKKM-%=&YVvyfPixJ~w<k`2SfF9Fipy>G
z{NQso!i=q@@%c->e-M^1+adpS_B6w%b6uy*Ezwb#bmgk6<>xi0U(Q|nc&5Aj;(N9|
z(eL)mGA+LMZ1ei-1$QRMo%!`bxo79!9odg6zO9QtbLD&b#m({YbBZSIYSR!tw|~M#
z=H1Bx${pWy^!B|<vHHQ|Gy4zI>aewo-OnjpNtvQ=UpakJsitJ^I=(rj*E+w*+RRGz
zzW(vs7W0d&-ydkpILz0x%!!Wvb2;T-mZ>_=ThA5i+BUp-^ecP)QN8ewe~R@#yxg-i
z#%N)@SMDN*+?Qr{vkd$Hsn0w9qp?+I-d8bW`#GNP<>qcwJr%jXlK-#&{;r+}=6m*L
zm`+viQ#rj<SuStE8l^L-<}v>B%gXF#d}5NX`@$K!-o8KJ0ms*QQ&xqCiio(nhK0!N
z)7e#$d4Aiizf<1dJmzlv{Nn$XX`8;hyt?z^)OZECRjJR~<+oiDuQhlRd+brZx8?U!
z*)M0NKR+BEmr_~C+odGjqsD(Z^}~+9IhEHq-&AIbiN_bR)><$B_;QQ*4Pm2#ySe2H
z^KYE)*57liY3W_d_e&ZJ66SASbtxdc|LEnJuh(02_j2ER-g`59=bLA<H4C(DKA-u#
z=H`ysi)7v(%r^P=dWH(;Y(MwVP}YA<Rj;%|m8QyBl@=_{HhC8@H*6R0+mq*}9=#D_
zo@CPJEwr(t{kXD+WDDQVKg<_>Lbeuq2QB73r}6S+*8PcxnsicJ7ka;UEO}k=BPCQI
zD6hR}lc8HDmsgNuU;B)2pSWvw8Xh{C-gjJR^M*9vua1_pcprtfs8xoFvWT*<@L0*t
zY(H^fakACKwf7w#m&rBkogMb!eEYHs*>5V?%JPr@%Z=`nI1(b@u)O$!;eO+raoS#W
z?reRVCciYByvBQyfTOP(2g_oH8q;Da{k_{eKB~|0sa`)-H~ae53rh<Uss%L{J8l+}
z&4~|UWStNcnHTc3Vc};5w!8h8L9GRampnf%U*eY&P?5>AKU@AgI#zCto`3i2k}BcM
z%w^5fIl*fz?kkFNF!^m4U;g_*W7_KWNpkT~V*9p)J-H<9asFIx_ScMzKhFeJWan);
zEbr8O(kt)Y!`sWhx(P>G+P}N`26gANa@Ayp&KYNR>+3x~yL|eGlUug>RAw;7pSyQr
zdH(vxQ{Bab_H5ns@rg2vj7ZC|+CRFdMO_71Ccc?t^W|ce-Q16c-r@NQnx}i214VZ)
zmu0RhQ#D_2YZ>{n<js}dvYnP9Wv?0}dWBcqJHO8M>XoZg48KqIkJz584_V~J@a7($
zjPVZnmv!c5v%C3jX68g~ogumT*@pG2<N`w%7T(`?_@$EWS^uagskr&Vfh?=dXLmlI
zcko>(_s!7!M{h3s9mu*c@vCsE*djxrPL)*|i|#+T_(dnjzi#?Hp7+;#Pn`-{<hfK+
zsmS+cUX~N@l=rtzsLwyMaLKB98MUlmCu;0ATbF$PAH%^_wo|UNuhh7Es{HnM)3=-T
zUtDcZ{`FnKSbcusV>hAIo|Og)TjTO|7Ru}s@ywc~(AB%iL`Kg$R-@~}4lfC=u+7Eq
z6>BGU1VsfNzIRtOMf6)j#{PFH&q`AMY+PJ$qi4k`k21gAhf6d)tYq}qt@g_GAMbYb
z-B6xyxG}@_^!^#AAFaPOdCKFE&}kDxqPf;}sqHFz?Y#V3Wyz}>eEC(@yF|;>w#Tv;
z?cqDr7!YkAy5IWIDvwYD$4dcTQSP#{!(STgw7#V9n&XHn6Q^t`r|izTR*!7oSuDvr
ztik)5BavzEM%Tzm=Q=c`Zv1*H_DVX5!FTny3A@s-9(%s4@(kzXv%aC8mbc5NBozF3
zam94<rc(;CyT2-&PJXc4SK4FSC!Pq~mrqJR8NS{6{l<=D^Z2v#&#j&O?$PZ>f3JjH
zop61l{Bb$s1{Y`D#sU#W#*n41e|kgq^fb>Gbv^j#<K>m!+D0bZoC5i_X$HkRt0bo0
zIk8bBrCLLGy~DDr?qczN?nZOCN)2bq?X6DixVACTU9(8(jCJsTU!`P$@HxgU+yDQy
zvDSXI;%<ER)ysVwkIQu*ms!4O*S3`@OC#rqyM{(r+|}s1^`lN8a%X{{{MqPZ?_{Mb
z-oNGlkn*eGHji)h=C$*-T)VwWw<QZS`}XIUAWP%FzrQc<um7*iR($=!kBu_2axAO5
z?rB^M5O-XD-SAzH+@7Cbgv{3IJ#3Qhd$_lL{e#u4w=?4(*dO}JH1CVaAyI8jz12!<
z4WvU`BUa36^`1WY?n-Ozw0$R6Jz_cR&UfhblpC8CuX_23`5;TEtLsw36LS`sM7J_y
zT{BjkwPD>7nZob6kw;Ij-{<|XVWG&j^GW~jy=C{6vQlPtn|C5|@vf!YBvxp7T6V5-
zGLmx^V3~AMVqyi4s-n^ByXFy_)qhS-T~_G2$uLBI)uJV8w|uL-|GH1oNl8hY{BF~P
zsZ$-N*M;qIo7o?})-k<2a9gtS(=4HO$E2^POnctkwwqcMU?*U*@n%<?7HrAbQMbdr
zUa>dr%<{C<(lhFNtage<>MXH56S9*1_F8fN_ucPgFWuPUt0dJsX@V$=qmm4tvNl(i
z&5<|X?`16gFWt;7rxC$0p}^wLlV?vAG`J*Z?o26v({$s-)k`%;!!ni?d$3Nlx*)$l
zWf5zyeEaeOj)^H8%^N2E^3uM0%<s_j=qX$tH+dWz_5`%_#_J!RdAobgzvBs8x8KQM
zy?K{p$(e$<iqED-HhFqBORevnDZ6pz*Ol*V#&+|iJ%tQz{@oE%$Z)Gv-l?rp(ckWq
z%>RlXd3rlOMafnFvCP~3kK1K&=f1~&M_+GHIL*F!=89EaDJ4Y*L$80nl6?Nd^XNP$
z0iG}Rg-WrXZVOtL$>{sl&X&%b<6O0C{+~TQC6hW*Ha_zAKiQ!0=+iF)%b<&0U){9#
zm6)sTp8i>X|Fq-tGBlgdXD+^#b8!AUx9buI^j9yCcm93$>BsteeTOIa9esK9`KDz(
zEGJXCtM5;}I5S&se}{Nnb-(N8dxuhrif*-D57=9*+P&SZ;Ha+oC41p-c|WZ;i>-O^
zr0}@m^V#V;9xszV;o*Jw_C{N=$$XANAAVk*Rb&;Nqi3^-l~F*IIc~pvx1&3QhrhYO
z|G(2K)^B3kZ9iQ)&qjVXi`#Dg>DrC=9zFS^z3%D9uv@D>-d^&v7<5d*#bq2q8eW$r
zq64M&-n+8+fo-YD>1Bl>pks5THd>y&@y1Oi&-musJ1&O`W%9URTI#*ZKJ327FI2RM
ztNP->&z$-Dzp*XrR)4gWLua=6<eT2x3a<A4WLcGc^32=g9}fQgk@b$fW%0*j+0pAF
zWFtg#{!EFQSyXZId}**$&&8G5{!XdUkz2BqUH`T(J7@WzN{7Sc74wl)+h#U?Rp0pI
zO}7}n9lIoA>6z(iINQu~Wqa_Ii4*5CAGF=``JT_N9!t)RIm-EajhkodS<l=xE6S+&
z_y6NY@)0jZIyV>njQREL#8J)gk55nWmPi$fsqMWm#q{9pyNugImAdB5+gr6M^=y&Z
zbQg^ichCM>``hy6pVt<J0gMdU&-R_yF1{8JWXcyf>Bhb?(a++wuJil+(rUk{ZtJO$
z2@`IrYN;w!>YB4;QI7uYZLc{)rW=+&J3d8i&K%<%{F!Z9G3_TdW#)GzE}D_Gc~L^?
zj~6puFMP9MpM;a+cPG1uA7b(~CbQ)#`O?nKad$q`ucgiza(&Ax@4W|Ggt#~7x*P6%
zr*y0Q&4Fd6PnSF`J*3<^^Y>4sIgzd|OiR!FDN&sJyKjNN^=-wQynOHZ&)QvJ<FL|N
z)m<pbm?-kImAP-msSA%9{@Tg6oqBsj`c>gizCvg5FFqE}z1ZZd&D6S?n{OVG&;KoL
zRh+eH=Dl_5-|xpYoL+U<_*IS-Z}0`?dvn&={5W#iWf7O$cFDc@3g7K7b*{Q)RiY-H
znQ7-0ru(?p^|WV)QtvdgIY0MWmLH1v_wmK!kZV^IC3SvWu8}BFQudqS@ttuo$MKYy
zJo7B0&ZBv3-#O%VWgF=2mb*}Vvvj-C3tfgSuT7iUt@w|nUREy6)NH>0{OAJV4JGm?
zon_k}Y?5Rs`0@G2<#_oetE?LCr5sK)6v)j;_;&h#TJ-|PKp#tkl~u-v{cR>Is?YO%
z*bsQd)MbO^oC@1Kp@~&;wLA3l%C#6S-{rUQi4iTdeq-|?BmGrVVh*2p)!eRg2?tdU
zzU8-Vx%rYQ;R<WGL5Sc5=BoPYy!sc4Z{*mIeq@>ZoA2?q+3Qq<c#_I1KkNv~zb3c%
z{IX&vSN<)pjT(N-SN@r>O-{D2NNT@M*P?*<`1O*UpmipEE}YIQ7_%Rriqw!%iwmzW
z@j1lG>1@%n)$l{K#uW#f0Pa(P3bT&|o(xP?$ar{YwOq{IP|>d))rtIbvcIMjetbIp
z<FD8GpU*tzUp$>9WNvHl`JD}Qv7Z9l`JdmF{9N&%@v4PP*``aEE_y08-E2Dk+-XWs
z<tCXtbFtZ+q8Frfchu_n@ae1j+Ds3Rt8PCow@2PlKtsl2mdWy|tCqcX4RKZ6T&%xK
ze9fcYlm3a@tPCRd?8psc@oc}Gnfmn)|IHmP{HwYSK6}Q0J;poXu#nTT5<R=eIxSuK
zmEV?rkKCWk(d^dyXl}pV|MRKec26u(>R{_xbm+@UeIXwA58L0LZYex`!CP>S96#IK
z7Hj4Co6pZUu-n-F@om*&0afK>zj+=Da`eUO-PuacAI~shVv7C6uTp)Kt?T<ccZUNH
z%{JA%V?V=dmat)Y;XzjQ$40LMTx(x{eYFU*it^n53Gc;TPnc}<?@rCSoexWPJ*yN9
zJ)XF!$Lb?!NZD<+>~iJKgRdSv*?j)t)a&<hFN)jkW{v)?IH%r?_p`Wu;&(Cimpk9C
zyeI2!5-Dksr1HMTX8Te-SHb1_{rBcxn)&x%^o_UI4PWgDz8U>XOW*dd#3Y3Y)54GZ
zm}8Q3Feywox>0klbgi{TVTBEQY2Ky>-*yK7RqU{c$y-w<EcHKH=%c|whY2e_vi#z)
z|J`w{kvXO8-Kk}7?=Q%UX+8P$aN&!6@y(rIzBK>M$e(k>DNBFj^{KNWyw$kdn3r=h
z@hz8jH*RNHeYyE+<%XPaKe^_1Z;i7|Y%^>nB~%#q9`>Dg_^uP5<CTZLr;k);Fslg1
z9d`UF)7SjzC6DfY>yWl<=NJtlWu8Rvz3VWODiQfA?@(n@cj2?gm5I-{GBU~4Tg^-B
zi*@T}Ui|bydbIJ(pz6i^*{pu=4<@V4ZdU75k=0u-9#EL}bW-Z}N1rM@zHiHEKbT^>
zqP`{U_NtG!okd(__>P~~_i%Q0emvb_!h&nlKADB4tU0(gB5Tb8E&Kg%=IhSx5V+H|
z;FN+_#9Fg8dfzW5?zv#01v%7hqp?tD!e#Z71v@LBUTnQFS4}O>wxxVtyqk*5^QPDR
z4;Qc0=44wGQ*~S86!TUSm691}pB<J~=4feVJf*EQF;sB*>F)EdokW6Gf8T0))#Wi)
zS-WhC<mR({Ua&)!9Gsm69P<xlnN8|(&FD}oxRL9z$iul-cWbF*37c_Tepb`m44dP5
zn-x1(*4&vGFyWTNE1T7x{aUX!%-}V-a$V4SiJV(#sGuun`(;NPDWy)i<!8B8`P44p
zyO?n>>(XD9na&qg#9}8sm#bumo-S;e924#|kwvecfA`T2rPD1^O%gh0JXW$yMq965
zWjSo96K0*aeX@&*i^{@BTW-&?=?)EDxNx6b^UWi3GW9o^B|MweeNIV`Gr4rf2MgH~
z52NPQ{rGq7b?6f{*%hn06gnc_@~ug8z8sgu_|za$MKEcifnns)w5>{9hZ1WOue@Gy
zYj>w*e2wezyxNl=cSrnv%JZyF`pNoo|3mqcSFE!OnQr!Fl84D92LqYIS;a|)FFW0x
zLPD=E=(39G4SYQ{rRmo%ww^F)wPqLR4-<Z?OgfNfe?xxJy_TaX8K>P}?{9;z!pyL!
z%IUd!b*fJE56)V)zC%yHC7MWGcv+(R?a|6rUPd!jq<V!qT_ScS@SXg7^o#vdn@!dw
zS+CVHWEvw(e1ZfwM)>>*D-S#NxUlTa4ad4VJGR65pYMXtK#F#|Ysz1GZmmFn!#RdU
zeezs+fA%{vsB)gPkT}C}+_vpPVoWsm(n68qke+v;2Xagc4)z{b=u&c<+-IyO-Lb0a
zXx@`8_B<|uB4@&wvmR8R3JldL6aU$(UBKSdzEX>0QOouDbvJ%`sNG=;4ldB&efT7!
zdF|YTJ-0LRw2nV3NG`9MHTCwVIUzF>wpkfO&o!ynT(PR^2G`TZ$85V_FR*FqxcoSC
zO5=Q$&8Jlq++TfMvNZ9##mA+o4|4BluV1Crq>)s<Yu2JotHCQcrJDIyt?Ig%bhu=}
zKDU-*R|^9JHtgQ>!<9R)pMP@6fwgh0fmTI(n{IURU76N+Eo!1`=)r_-NjcY+ul?G3
zF==_$d9!J2|FKPXb@w$mwd3-+gSp0wxUyF73SpdlwC`k6`>~=gY1)xHw=HX!uxgc(
zhgFPR`Kvkaclt+tbC<Ra*dM$m%(~g#=Rk_lqU*1N3iYBK1z1!(52YAgEPmnfDtq_i
z^Ouw=<j+d@iuKs<m*53~-v4roSbj0*l`#FcecaHXA;&SVRN>#$svnlU+$yV9E$Xr|
zFgc~}^*^&!YQOQVJIxAGy$K3#yM6L6uU#w0!BKwVKGTNXYi3!v`KW#NU;W{G(<&jp
zc7YPNGj4zN<7#w-)E06@=sugKUG@KI)qeiQDc{{PBP}oP=sEmW)%b3^%D=GdGPm0n
zUu|}9etf;>nd5?m3m1OWR*&h7`gym$G=IMW$B9mf>N4@*X!%3$K24ZCyV+CURM&@-
zNk!Oj)v7EHtp^2_1rfa~-@9@iyf3TsZ@%jBdM_ifWvkR0)?7GaESbY+?pJ;L+{;4;
zt(d;@>aq%}#>B}_Gn0S5>dMuSP0N>vgkG6Eacb}V$5uyX2dPGcd)GR%%stq2{`Ie$
z#q+o2nO5y>+~qJq^!4_oOy6ty7ilDIyj8H{js2!C_YIpGuBPQjef+o7FltSErNkfo
zSAXq74&)SHSh=8(`G?{`8PJ0DI{Ot@*t0m>k}YE5-b|ZvQBjaHC^WkNWY+PgC0ll-
zCLiCWTC=xKz4_Su9@AaZ%hT%|w<jJqTvi|0{W<BN*0Z`Vnr(b{C9T)+2WhvmzGl9{
z?i}%kFNW{8|1G5=hk1>U{gmu??-ZYP=$C78<c>V+<8$lyTzBr<7PhOc?bv(CZuy0G
zU;NJ!JsDy9-mkW?e!pdR3sZ-c+uTCV$?Nto%zt`di(lf$FPeA$pOH36c)Th4|FNmU
zzqY*;+k5wZ&B@;Sx(Tn^EEXv!bSZTuB|KVMUhsM9@$Cz9wBq-DHR9j*#jJ3y-c(ml
zk&Cx&?tJ@xe5S#I-0kzl@*|G@<+nR%e*3#3L+0D=9pAp~wpf?%c-fLy)OzRpdDW4>
z-s%}u%O2m%|9_^>@$=Cc8O6+vDxN!R>gK$Y|9)7HqiNdx&+d!<nX4bapUm)M{k7l9
zx9&{)3OZp-M)t&++UKtGYI602|5uos$5@ClMtu4h^6%f9-E(SQ%HAot>A9BYRLDNi
z2~*3?Jkn=vetBcVtp0no-E7Wg%D=@g-Z3-{i$6L0{9o?(h0hI>BINmsqk7Fhyqf&o
z@a<*M66s>K{yWy3+h5w3=H^a$J~i^rWcv>S%#M>nv!{K$_9ZptyR38KuFjk9_nnio
zoXvgCPH_3p<-CPYJZ$IJeRy_W&)wPi@%J9D*izQp<!_eQi5#9k)%*D$J_*&SmnPk1
zS)4HG(VO*rmlp=RztTQ6@l4pGgHormrB3L-X>#q@l=-==#Ek#$E|Emjml3~zy8n57
zbLEq-lBZeI`x4)sd3oml2U)f+k|#2icQFKPOtzi8e2-UTs%)U8#|Fis9VM%Lj_$kr
zMn_L^na7i|7cT8{N?e}ISBt-{w4}mz_qS=upMRV@`C)R;mCdVnuuQ((Zo8Lx`J6hn
zhckLKG}c7e^o!n~vS0>p`(M73of^Bl%Y%Y;$)8ME9PG&WGEV(sTU3|#x8#ix93k(I
zKRBauJwN&VzM?f@)~3I|Uz%{`^3oL@842>Y%cnm)m*XCIJV(p>dw_V@iPsn8RtGLx
zRH7eR8dzk#Ww~>|-JHt4E*;MmAFH07{nub?s?_H8z7g{VDc!wG7C*Eq%e#H<->j+}
zJ$u&HxrI?ZYfsGSyS`|#mB5ii<BiKJ_^aCc7x`7bxxI7hrCDsZ%C#2NPMcp5YpHMd
zNkiK#tl(zo^#?-Cdw%_@-edbVnN9R=(`lu~NeSPo*Bj*@trGH-Txryxv;3K3k<F=d
zZ=-KS$F-iF>X#VkwM_nS;K$X=eSd*+uZc;WXNic|7WKIO{YT?<pD*f6F5u=}^CL_3
z=~qvya$T1`wwrrWUn??8F)J}~Fm)}uwDG(|-rm#x?d$$dY^d1LmwS7gpx28JQj<Ft
zbtr7ho3d8x?!iO)VF97RM|Zi#Z>s#>%A<LP!F5^A{k&JRc^$0F-}W6=*qr=2?AzP*
z$KHGij0kDCo#o=<AmDO{*R*DHn(D$97so`wxaqBi$LBj%Jzx0c{WR~N@6OhIcy_u*
z`nBKsIM*+4uL*9Bb6>Ej=WO(LLEo+?&&roMmsu@hI9bs$_cq_P@Q{#(*Z*%@zqeSq
z>T{v}cfa?>x1~=r?bg}t-9A+<NJKM<{pdZ>&RH2IhK8=6yZ5RdZ0b_t5L=?5BqlKB
z-s+O+O1b4Xt@hrnuB$(@iFKMpZ@l5jT08l|XZM2dRiArvruyFJdEd9rTw$?gcciuJ
z7mYIupWR4onvr?kj6Jfqpy0pbh8-u)dOiJdaq|47EYkx6ww0T=pI*Ij=jB-<#)>N}
z>w5OypCZ`jo$~C-#dW!6Y;%_}XJ-5{4=jC<E%)oOa*nR{@stUjoHM20*G-tMXEnQA
z-S*%b)#?q=k2#99zT1EBerESu<ag47e%BRi)-BBq<m9WJv$y(GN5X?!yXBv}(>|a4
z@>FVz@6DLPr>Q%71kYK%SJK;O)xA!&Gr}uN=fE2C_~#C_LXVHfpW2eSIhwb1L+wo)
zxynbDeLqW2WU*MTl->XLa=heC-QIFF+k<c3Yu+#2|HpRvmk%Ev?B$<y?RV0=`X|?A
zqP90w|NFqyyl&5Q_WDB0avzV1g+H~vZz#OUWMo~M_+x^cE^F}OGxH5Q)7AI)=NJfa
zwr7L~or;&Q<|!;t{ciWeq-%+?z$fFBGr2vlr=Ml!w{1-+5bC@Z5WPMi=uYpBa{E;G
z)dn|ie|Vr=zpUo1wa?EA<NfuoT;0lj(>@vRPyQ&!e188Io7i8x3xoH!JU@Q#>-^0L
z-*k4(sg>Jn{Z!~{n8C+|_8&eji<LO5#?)wVc-_u!_V*I)OAopUe3gGb^|*Y}v+1`D
zq<mWr8G5<jKeCwr&Z&>G+B?6OtEcc8l$_};F=ns)xb~(+aPSIE`?jPlm)1rb{{H@E
z(W+I`=1j7)`S@jWgm&46oZF1Q=2n~5?iLS-j5`_o{a;&?U+jjNLWSGD<;I@dBFX*I
z{*{%^R_Vyv7?U}MoQBn}y7yRobynHiml3n@_=D9IPquHL^yo+4`=<G=jga#&ogW`x
zTKn9t<n@GO$8@DFLIS(jtWG$5EcS`Wx1}n#&n$75f3R0IT3U40y~X*DHnQh=$Q+&`
zls7Guz1-mA(dj!={v0mJN>2XeUH|B7r1k_JzD>rLEM9R>Dw1OU-?w$Og-p{!asKqL
zkFM%O?`^GXmF$RUD37b$<jH8d^uu%Yc~5Rn-8{qOt#XK`zfa)HTmOyZ{(fdjiFE4y
zQCa;c*@yE=@ayhN&;Pux_%zY_#?Bl;{qsL3*>A}`eatdAqW(btAvL+br|17Ty}4dJ
z+Varh`pP-$Piq;xVzns#`SovoLdySbd%hnl-edLm@v(oqcz4XPzh5s~d@nQW`Nsd>
z)9b!p_*?Vu(fpeWN>V@B%N-N6Ir8}XJ?A~If7<-t9bNhTs`|{W79Vw1X|}r0tC^LY
z{miiN6#H3`&4o6f9=u&K^>)RJ{`%DG53PKxrUu+Or!V(I__5pcf7jDqT)6Nf{z!bZ
zdtSQ`TTB7-&UqI8cApHmr>{+Y^YbqUYqQMyl@HfEyl(vIO_0@+n;)--1m~W8*cjiF
z{<+<_qW)=BTfFtW`#my8iv9mJhi01>9%g0!`1|jXD*~6}Yvtl=ZR6(8KQxc|M{%Oz
z%LCs`=NlzIlxmCpui(5}s)BpgxohS^%grQ~?>PT6#=gMdV{>-JkC~ZkVmJ0ayxT6w
zRa5^nY~8&BSD)W^Ox)J3AHM$iiH}-lYi>Ka*3Qwh^_rz?A-eBJh1%;f`Q$sb<$+V~
z8g1Py#3|f;@TYiB>V*aS*37i;&78j`$ve_AR@Kq>YoU(i+?`La#b4N*ZT{re+bbdA
z6JmqT{Ls!%IHp~n{&(5dp8r>wdA{X6f4}}r>gQ?AJnyfyez?M#b8E-kOIvKh*4=Zh
zdVl(a?{|~eyQh7>H}{y}(St1;r$074DPy~%pFe!<;@tkkKaYNS-`H9uX&32kB6U2;
z!zR??iMvIyWcA~lU*66+y+wRp%2oAljr~uxR9tfvi`VC=PO1GRCwe?vL#|TQJmq!8
zzkL0TY2}vtcmB59ra$A{oKWc-l?x^%9rgBU%HJ1vFn*ivq44<rMdrnqZTW+OuJEab
zoQ=-+TXya4y!~<Z)BNLqOF2H;{`<zgtv+>+W$#ozFSs8YzM<&dYM=S$Q(K+SKU}qN
z?*7l8wd>P6=QD(cb{tL8{nvf`@qNk3&@8sNbd9Fvm)F5w8-6Lw-uC|a?Ypxr0;HF0
z$&ZgoYkZY)_=fcH&p#H|FH5ghUu_@hQn_`1tNVPR?%>#1XM<G*Deu*m?c2{1T&4K^
z|G({D-o030C6lu?U-09moS#KKnsayjTlG5Qy~(Dmr$(!syp&W6o0+0&zNy}+e_uP}
z)#-}TQ@?L)vif*J*uUWGf_rc7ZD=vw&98XcXyXd6EI)g>A79tMoROM(_EP`N@-=Lx
z5({>%>e`W$bmD`z#;Hxor(Ye>-R`SkfAh$m^5vV79w)Bb@zd_#hXZqGtlDPy^zO}^
z+k0l63{KzvBL8~Z(KWZj&+b&#%guTI?Djhq`@60O;(gbeH)t^|6j-xXQG%nPVZr{F
zk#ak}$LZWNimUybdndQ*<A-?p$A|X)tg+0OieGd8bYt%6$nB-#weInUmHGcSa<_kX
zUCu3Ne_39Pm$z4b>XS9www(U9A^o=3Gu<CD^(ON^oKq^4xpsM*xbV9j%@3dRDV<Gq
z4h^lE^6lXzZ~aB_{pT!n_}b=JPYPV@7JTU%tCq<UR^><Q7k;)B|9mQYeZk`wn{Rx$
znq(KXrE|spNs^P7D_2_;9h|V4<Huf&ESdTHrzLFpxxHuJHd(HZH+a{?{4kuq{)SV`
z)CqoTcDMZ6`&4+<fkz&R7Y@|9y#BU1?XTIk{QGWMb@^3dPffqYR9CRSKO5DsJGNX;
z+VIJg#hF%<r#2m8`r@T4e}4AaX?uT9EU8jqR1m1rejZz+Y~H)O=iO1+3!Y31H(&m~
z=k$(@D+Rlrq$iheOFzOYZcz24a)(TqOU%*@)%ISuEN*Nm<&M6-ez}p(#rgvZG0T?S
z-#z#B^bH9o*>>?u9h@+aGoO9+W#{GJGIQd0&saWh_K|-R7A(kC+?M8ke$BZME-tHp
z>J0wP>F1N{TkR7C1eso{%-k(n>Y~p1#!q~DOcCc{eT(@r?Y-aT?kYaG;v$dq73tX%
zkJPMYdX)Ic%J{qeACvm(Kjlka?(wNE)^pEx%8A<3;6MFc!i^5^6W=60v>wv<C~)=b
zR~g$Kv!nL&?=P2^KQs62x4&0UOmaz``aS;FpLvD+Os;_)?SCbnOgqNl8p`P++9SrU
zrkf1jpWb%tyR*@ybO-sE_jSjX@oICv>=(Fh`SSjLk$&serlxOe?(Ug;bN6-=4U=`j
zv4PTk45uc!d|hbt$MIn6;dKutO;onHbnRGE>+2}rmr<1&>e*)R4o$ey`*$~Af|=xv
z&8525-=-G(xrZ-is(LQF{j%ZVgUkHY9`hdhF!8!U$pMC6<$4bn-Kv^&>GF@P)~17L
z=~s4mOB$-omz5UZ{QB|NmPHTUeyYr!c<9l&EuN-@D)W5z|NC3;>a|5hg8nLzw9{>_
zZ?1DHt!mtH{rJNCvPb;$@5IZ#yP}iRwRz>-x?0a^T7o9_T$+j}i#EqvUi=hkS3Os1
zKL7G^d-*zp_Q$`4x*}btPM?>1d*ZYw2_M&7*Svpo>Z3(|C8y3mUVirH;)M%8)^{IT
z{@(xbO3(L31z#d|m1-rtel_*%JMJ%L>So2>;rZR3?R*nj9N0=YQs(xY)KO%Zd$8eb
z-tm-;m+KEOe6kjK^!EDn>*;!j;~y{GRAaBbPj9hS&ZCFFj|qRc{_y+N&;PXjC(oRB
z?CoN2;Y+Pq6|a{buSoY~O+MbcaIx!#qNmf+E`8kl)VaaI{e0c?35GwF?Pi-!@46fM
zMYh^>rd+Y};lh10>}Q|-{d&i*e;Y+oclt~2oTa?G^3TNM8s^Ikzg-G^pq-u_=^M;A
zS8ny?tCt>ot#{7%UNT)|63Yn%m6W}Lp3?-K9(d)RK2W!J#-0S(_rJ@MOSYss%zt!D
z?a{)lw6k+O=RSG(&v5!fLscK~Z%ek_`8~TmVqckZ`?t5xyDg0)cKuOs&A0XbT_Vj;
zeO>Rt#^dvp)+Q({ZYx>%XtwSZ6aL=?o6azmY-+i>gn6M^kE#cgkDBD%IU31@Ru{dE
z-f~$7-`cD_EkdkSc2ne@#q$a-oL|-Z+A{2M>c`*fH|8CDEn{6aB_ZVDLU(ne#0WKy
zs-oLd3{nc}lWM&spYUE>YVkb&7t8!YW=9*j<4IFaob|{(^H7sT!)wl|$5%tHhALiB
zITLn!rdTLw%dGQb`>s8c_LaP!aP*bx=bM`Bp6B1Yo?o=>P26Y2e%o&*D}ORenr;zo
zT_#%MW-FwXSTD;Zz}4QVv9|hGZuH(B=Tf_#kG6Zvekwe4=$Np5+EZ=w!ZOE4YtoBZ
zL((rOpVT_IL0*)DZLf@N5zpeou1Su1i3YEP)|YnvFZNobKG7g-MV3-?@01nkCTf?T
zuVGA4kA7Jjrn$+OeR~aq#m@ajCEHSRmMtr5vA8(Zp|`zbHP=Ls4lC8Zy&E4-JGF1k
zWk$}UIUg3(9r-=Q=!EzZ@8v#4pR7J!aIQ}{ye(Gd#^zSZ_4`lkt7?7zuSCA=tk;5t
zALE<6TKD%&Q1bBeR1#ac+)U!3i&ObRt}DX#^)JjQ^$r!*R#c3-C%&pHAXH^>@8;jU
z5B{>Qu1J1#e}%-g-t}T<)H5~&2VCJRFA}<bw~BqA!=Z|(+6wW{cJZ!>)9p&m`Q}gZ
z>kJByZ8ezf;Z+>Gzt(nQL{CemgF?%%zfXmkLLI8&3~P6%8bAK*zd<Fn@SJ6>l=$r9
z-}9gDy~+Kd<i)wCYb*E}ToyU@u`);83A^K`EuHnMb5*bQkrtjT<JDU%Ud~#j)N(7}
zSIZox3zzF3Fx;E7cZPo0QHh$Dxgsg5=4P97uRhoLQDl7g{O-1ewzG}C$=u!ASMM{Y
zXLqvZ^Et(Q*4x)Cw~=zmRCB%%?X4hlY}TxnQ2E8{_K7J5`Sm>0H0o~qDALKaLcMPD
zth>RjUSYe1*h)@5->f?SQ-7!5nvYy!Ywx??-h6Dro3abp4+`FG+Tu7#r9~+5=p+9h
z%dVZVT)Owf`x#!_ZlB!#wEg4ifSu{g@i~@0Y0nw|UsC_kBhAqgFj0kb@yRPnzxD{Y
zT?#sw@!?VbF%DMd8M<csw5KJ0ZfLjP-tlcMYm!ayr<0p{1hkgF-uF{tUQH~=gw#&q
z`W5zv4k(zPwyjzgB<wY*M@98t!0i41%5Pf*dz>xX9>urviq6&7=byg^&0QQnK5LHa
z{e62QKArN>GAz2<vTw<59+|J#1UTAHdV3#Ql^v9^;NkXQ+l6~p^mZjy-4-&sYbI9r
z$IML2a!;=5@oeVhUskfLj@VVhYQgoR{`|kGi_A`jyt#cOU;c-(_VilUqpG!y)40;y
z;>3$&#Kg|b@>&&heBajjn>Vd#Yyba^X_8B`jk<`2Xy^)+eYd4V^c5$h$XZRFx9e{7
zOtVxg^VoV#SJ~Z$`A3(_tXQRfSTOnHqW(e=zNOb)_ns`AcPHMozA8`X=3JfLw?1=?
z=DuEMAzof2RqB2H<D<|1i$BYy#IK#cj`w`^CZoACug}cc-t7Feos(n5o!J3rHtqhB
zd^kPcsPftJFLvcB&v{;)|20*blQr4CGAe<WxA(s{Pmhe@q~7Rnn_e_WK6&<Z^8JOf
znsVZs{Ep3?<WyzB8S7jS6q&AKYRkOcdScbJM+K8-&YbAsul>4U`7&YOB1OeOt3N^S
zbl0bxy8XGb^pe{bHu39y``ZPXyEjDh+_C#7-Ww!yV@t={)TuoZ#jJmKKQr@8_fC8=
z<MoZ-pJtz#WjOWEx7^I>v)eBxI!{Wub8@A<hn-|&j$hJ=2cIMNz15g`m;LhI%bl{>
z2d`&`2Mcd5I?5X!c%$n(gF{vsqj|sd+1bU@jJ}@zo;mxe;`-j6qf3r2*gkRM%*Mls
zc{=%9RMuI(O`D(+bMj=$5^sqwMpk=ge(CXfad__w1_p`Co-U3dbNHm0GVbrRXRZ?2
z{HmdUp)<RB%uhx^?hRE}qt?8fyW`cVjXQjOmvvt;`&p7X``{bb;+q>>+Aq#Js=Gbj
zIWKE*=Zru14(}+qA+;}Rf5-R!r;kqRIPfh^nvhyDEjV?X<=vDe%ae^>Oek>+x@0V%
znU^BjW2P=C`SOO3)t7TT@BW<4=dk_DZ#U19IdsyC3%MG?@AfsO?4J3r+5KYuhZQc1
zH!>_MV0~rCe?88yiAVa*-`Be<e)hiH^XHDSM2ax;xxG!Bxr%>=Pbic9HuvKFr3D%-
zXID&^KfCbM-VBqj`|sWJ4GKP-C`_8fy83M0tseGCDi2q$esp4D(W4V8%Tp)Kd*tXg
z=c9c5LamC|S)9kGs8yy)PPgWq{P6d(+F#`p0v0)L%e&{4c3dx|>djgmq0pLpRV_J(
zGPj$>g+2;=wYP4Kn6mUVeb?!;{O8NHdi+186g){qNzvY&PxhF6-(xQ?HEm_t=D2?c
zyRToCG~c({@8>^TnXju$-rnh4)m&}yD)?p4CC<HH{yxdLca4AL|Hld(?M*F*4SU46
z!-|8WCP%2aD2TYF8zgU!+IRUynNioG2cNH9+?#Bs_E>VeVOGYUaDiSoO>OPP&h2~>
zz5nC>J(_%T^Dp6d^*<z=iiGz6Jsr&R`flH^u3DSVM;7bc-O#?ICnl!N;_E__d9%N0
z{C>9o<mUIfPs06umQ0R0rvJ}mVt3we<?X%ijx}wMPrvk}d_|w5r)PTBx3`O)G-?0*
zYrDPhkw*I2h3j{|pT~YdeExCc`aR;?Bh|!KJX&dg@BF3wraxah-6O4^vx>=m`ta#8
z@6TCNCAy!VPq`$0*)r&<bAE|o&YjXfpPrqLG=8f(Z}!fPYPsLi%2iuIz5K_2Tcs<L
z9(iBUnRoPvWq0L^eEr~A^A9b5zyIKpmBpRXu{M9s+&^i~{`0u~yyTCw);l^BxhZb<
z{ULhzyk5dd_xE$J+~0bq^l-!-wW<E;zL`1CCoZ4wzP;YG|NmQ7z7W^U>ERA{$~)F<
z)_Zm_{{D&C=WUOOyHE0~oHl*A&+<ifJ^LIDe?Pr@U1<G{lf~!l&;FX*e(@w*%*vm4
z%pS^}laE-!s(IzYk9;SNSFf`^{5otgM^DAHNlCI`w+zdk(iV}<l<nMebrh#qlwLLU
zNYl<+K6k_Yc5{J4FFg<COjxmct=rVI6*1?IJPWq*`gw8j74}0BZ}dP1EWh&S)fVq>
zTe<RkV(E?z^GjkJRv2cjTE%hv;L{=@PBy~`8H=}SHDCYFtJ&x=;g*#?+h*SFmoF|z
z=VH27RkKjTuUshX^2@KsBUF4sOHEI>%_-wP7{IanbaP<Pigllpw(i~#-J?7I;=4n6
zh7VS+R~4D);Z>&GDRI@bbjDh~Ic3}&qWQJD3s*Fqp4+n1v0c<@;l6EcE0i94eoDA-
za`}nkKR*f&U6crGpM52T-)iSm^D>J7|I)SMi?24i>I-sdS4^^9;I?Ar=dBYg-+X__
z-BzHgnmmDJ>s8Z)O%Dn!J6iu<scg2m;~63Kg!@Wp@s>qgp_e93n&!D_x9R$-izfMU
z;#Qtc9IdxrX}#LR*rTSsGIQnsPqRL!rz>!ThXwAgv|h>VHBmS1Po4cl50-ibhtODo
zW?e~d?c=;EoZssA&RUs&cF7&-qbu5X<Rm^h8g7&nkpJb=$$PBoG3-l^$v-{vZbwE9
zFK?dl*>~#e%x_!TudGWtZt?J%#>FijyjQ~wUanNX5h;8$%SfT+Kt_4t+qdNxmG$=?
zyt<_Fe&^lU_3qEao~tfZi)=D=uYCA2{YCxZy*xUnayKUHv3O*)>@-%}c>49jcemFa
zO!Jtt{m#y3!OPCX9`@ZHdtTf)?r?y?gxSY?UGzgT*3>_;jQUpj$$0L9Z8i4B%Y0L#
zBQ4W5PxP?x`(Er;a5YuAAd-he(X>K)%NLH!w>Mh^PH3!&-#e$nu(xinw?nA(X|-cs
zxf2u>-*fh}MyhX4x+XGBF7(G${k-JM?D-$A3byV(e!t0AUF!T!v-!W>Bvm+9&U}8j
zbK5l8&>g?eMQ?naX}I5>byCNsH{Jb_dw2HSSrnuvWysMT$7FhHmA+l#(`U9zw!A+#
z<D&CM$*1pA9WL{*NJ-h-{h!Xpq$;;`*)l%Gmr)w_%QZ{{S^X!mt3Fx2&+(`6oFx-D
zRozNcMFkn<wzp>-U{~YTTfX;Wxo4;Aigjxbr)+#!GG~Y0hw9gyEsJ*CkxNRf=VlcV
z@`}n0Z8l2YIC<K%ra=AeA#S}@fA+ap&s(D6V$reP@uJbI(iL7kzkXGTgdTcXmVD)P
zM;miUh@ASm25n8&{?qq9{Jv9Fe?arCL`gtwWS_po@ySzoJ_<PHAA7syvAyTb{F-TQ
zN6)xrMR_ew663oP92x2z!@6Cawg2`yNp{e2S|)X7f8K1VzHiQc<Lmd66}v=}w?!6O
z$lQ2a7NaK~mM!bPx@pBK*2ug~?>r{8?EfW@zt^&N!dl_|QuUT{n;)$9Pcy9kWwNQL
zSX0+y@v>Eu9$(%1qDRcg$av!{9+xJ&SImMcrf#vd4E|A8Wy(|JS36vtdE(5e14$+f
zO}k!sX|2vOjFREZp6fV$#e43?j>p_BC$lDZ&yx^6eu}dtdB@~nPj62r4X%9-9h=YX
zpCq{Sr$$%Tn#+^tyjr&HnT6m&^`np3O#MPbMcKNAw_o;&{rXW}p;#?#uS;yMu2Z_g
zoAv4feUs<RYCK+gXvZRj77q>8RZgxy6JPytlzAs&WE`ZCaetcsvig(Tr-}#MQIizp
z-<bH;s^pzd(WcEizZ|=L;DuYi!wRmcXJ4f+QC;l9eC1mE2Nl(Y(^*YJufEXrn^R`&
z7;k$}V#nL6XXodfdzSBf$K+qQ$TrWU=eyR=FiPe-nR&BEHm*yEui(cE!={NA^QM#p
zO^)?&da!=IvQT)3JJXe)hgFuQO?XxKX#Lu>my7&A+_t`?r$1Yyy>-_vD;3UHd+qM5
zj8wVF8zN`z(`(*vcBV_3Th?Jq4N;e7M-P*^iB5~A?wl^{m8jyUBcsqXan72OFTG-}
zqM;fg;WAG<?zbyzPTi97-G2#-nvU(n0)h3{Pyf4XZE<V2_0O4de@dsmxv{m~YQ1=|
z!jz5^b2n6j&J3Eq=(4n@uebHnT{oX8-F}u;Q-3HiD8GC{TKSv`_o%F_WJ%9-1y$qC
zANO9(vfFCk+_`1Tm&xnAZp_VlbFOWlubhnf^a&*@OP5Waa_iHso9_<f<ZI=Y@5rdh
zE}wAg&Ac_n4?Mjx1Mg>SyQsE(Yj5kd-`26#nlEpf#AX)hm+tWUtfTpI&J~x(JD!_K
zmxI<O@b=CZPQT-zAobrf;q+Oz^xRWX1xeGdB^XSYG{<MIy`PEL;S3{FZRO&M?`eEj
zU(P6aZqa3(qQnwo@-6Djx}`ICgi@o!Ki#@AvqgmYW5$Fs+xDX0rPBj9Cac!Zx8iR;
z>ZBv#lJBo}Id{pPMO^U;KKlcccfNb{*<wQ}YyP<zuDJ$%0W3?M^`E>t?N(6n+w0%I
zV=*lU8_rHWsk=S?__Fh9pRPV@p22HWIJdXZ{H|kv4)dlp>Xxd_X1TW(XlQu|S$U@Z
zd~|-s<`Vt#d!My=uHLT9Jk=gkbC|{Gsq&t0&-f?)<XUI7f6C6__=MLNIjl6@R0Iyp
zyEQNR{f^$8&z%qagq(UezS(SfX5N3*E*<5PFui@B+<u9FS3A6ELT~i9fIDRj_f{$A
z+%9gp<gI@sGvsH+#p50;m&TaxQup>>XaCM^SBgu;%dPx7pRJ3Zl>N>4V`ut_xAlhR
zy^?39%kMq@ZbEzE;oX})zRtJ&-^DrS?rMn^g~}VP9#+OX&bItDt+9B_yYJQ)Ube-2
z3(Jk3-I^_NWwWie{rto3@wo?*HVSdE8tgLKS$n9WFaKwk@9pn`m!})wefMb9!@Cu4
z^MzOTaj7is^<KAAviPakz1jO&=H7HsSz}aj<!OvrRzT9ndpWH7o(kRp+frGBqt5(w
zjVZnv5LRb4-Lg9@`GT~E{>$@s&0{-HUn+?6o@Tiv*;ZdpPg#U#!Hhe13VzR>6FTGX
z<0H90#3!cCoUne@-0E+a?Jgf)nEx#xP}*g=cSS(P?l36?P`7x@))YKJ=-nGH3#TpH
z<o4ce%kk(~dHLmqmnm0ncsAZwam;lU-S%bMlQflutGJvRPR={!Iz7Ix<<zsS)-UV|
zRG5E#TU7Jh>t02^=5e?A{2ML4eofSqFXq0x`OBUUXEs;ZGVgeQ(E7%Yo4advl_?+o
zU7GlhT|Ob}a`*jS*<%ZJPtUN*wu`O)dSvO5mEYTUK6sTaSrnGuQ^av|;?b|$<2`nZ
zDCh4t>-_rK{IcqDekHkQt67aL_gE?}zI(UNFxl<m=Jx_&>u#Lcoqyip{qJd04h2<-
zJl#-!xlvy2^DN$;H}|JH)bctWJsT^m_8?$Y#m}XlHJdV6&0{WT8|G|sd3$khjX}QP
zUia9e8#Z%An*Eo3c6O%wcl&#;>-B!FxwVDy{?FPQ`;(31EIa4TwQdODWT|-}W0=)f
zSM2q-|5=U8!PsQO)nDJ-itoD_=6Ca<^N+H(%eF1EzCY!y`pRf=mmPg84*CTq&UR7I
z32iuc<mO?0r)deVC+*aUiasW4-oL!sn*Vc}PujaTKTC9F{7qgQ7dI`sJNNNP1Gb9H
zN2k}^3pf_~eBSA2xzQhPTNQrZ!9Ahoeo%O{|GkF5p4}f}KFB}387_Y5@gKt;xjW6i
z@pVUje|~mpr+d1}&nHhextIi;b}xQD&*Z!O#kJw-Ns9J~i4Si@=PlpiEd55!van|1
z!ZMAlC#$sc7b&<t+uA7{xwlH#mixBC;t#=z22q7KzP`D=xBJv2vp09PvQM^Vb>AYV
zYqt=zJJG1?)8*Rf;B6oof7X96HWV<4Ug;t5{io*tB*}lZ_mfVZQ}};v^+evbW1H5l
z%T`&bnd-86o6yByiq`7WYk!A-Jh_LTvr4;Dn@jzZx5cDeLASQf;K`mcXHALBPt`5C
zlbsdz%g)cg=M-DY`g{M{V+Rby^dcNABHUG^p1mm4*E=xrPT%I=-;```dg&xBiY=46
zeN8_(+UDFuE&a+|R+Bk<%Xi&bvUKU9Ra#=pxg*)4i_DUb#~xeUZ`VF2^2~eno+D{j
zCQqI0eUpE}&&|HGJC8o%xaqTgpYw~T8SOen5;s=2lpgOF>ih2TgWp5Q^f+IT@Q2`i
z_3YfM_yXdes=oOeBp&plGE9>@e%4v@-~5{@KVRSS_Db)Y6m};Tk2!y*N3WG%^<D7t
z?GyRak3QUMJ+XId$(u{<9s4K6{{9veD89VjvpVIggY&K_la~iMO%wU{{`a(=CcQ8E
zZ@VrpNxt#Z`G!R%<JEgESKjPdl(b7kG(p*+&QZ@xrSE9k@hro|H{Pi1l)dts)ou3N
zeavhZ^wcNIp1v-$bxHBN2aG98EF}$I#7xtTJ>RHosxp7ed_7Mg#(2};e4)uxW}U6o
zIWzH8Rmy_cK+e6prYB5|PKyr+ox11Ixi4>mT26~BFJ8{ncSOm4qx${>31t`77R$Xh
zf0Occ*3~zwKLlM^%fB!GTJY+*+Mt^h`CUUp_r$GFQ@H>JKPLX1DZw#I(ZT42r)+_k
z=ic2Lzxtn_Q2kQljO5DyOZr@bs-zB{DwVMabB#(gb8=7#I6GZCY~tMeGq1{@P59d=
z!OZ?u@#fOx^2pHSufhjeTD1f`c-Koe-86l}@w(LN(q-e5nUnigrA@xQZ^ooOA&XLX
zPncj)IOF$;cVGAXd*;LC{X2NZbl0N`^Jn=;RS7R`b~8*X@O;$KC~#z|g_DDgl;b3o
zM;|#BX0Vv58P`86F{_jQf6z#o#VaQBjI373)}_&v%@I~}r}mu5xpX&d(<UR0SE;I3
z6OX-n;j%k~nOj_E$tMTX!}%{-Bl)w|+`Qqv_PW3Sr+_OPWu^Du@md?V^!tKti)}xR
zr|J9?Iy%=iu1bh4-geoH`LfpEF6-*^Mv8H$9=fH!pQT*vMac@U;8z-|>l9RWMqW7B
zX6T^cS@c-hl$CkPtbbF5G`CK0(9p_qewP;-8W<Y&Q7$9vpu&8s?{hz2y0J^!@Pd;l
zho{NOs{z+jE~)-<RXA;x^6|3u7jBlDZ>@_8&gYq}zuj<ks^{`QHrL+AexH1L5oo1g
zvf^nYzvZ8Ox<Ttm7cTr5fBgJ;2Zk4C)OzN*Fv#D`a0q?$f6E>LEz8!s8Pm5klrQ%^
zUh_9%(pn*H@!BKDPnQ?}wv}D6dY$TKUwyW3Z;#Gu-OSmxn71_g-5)N+S(YX$VpGl1
zjbv|CSiI_<5PxUtfsM90vHP3-_Wo8sW%A~gvS-VlQWsI?+F#$)&jy@-bZghK2d=Ax
zp1*j={N~O|Z=JAITgjW24}MSke(!(BqJ@hm_P>2F_xrPtr)TRg*|%)k8MD3nx1U*b
zaI%xysa>JmFK;g8)mz%N{#vg3|2c*;c!QEd{yuqAR*_?MNaDq<wc5MQ*Cjn)$naN1
za<Xmby}Q<iAJ6VwB0AUidefX`pFcY-;QG4d>u=@Bmp8UN*s^m<^sS;~PsMGo*4EfO
zbw0Q6i(5>Ya8)p0dg<BZhwSesy)z7+GyT-Pv)VJheXBXQ`<dRZ(l-b8=-WT&UYh-K
z>idU}e{yeJ`gFphEn81m-nC#;c@#K<*KMC=)r`ZUS0h4a*?6}os4iy?OV#7H``^^v
z{w3}1B-RkEFsmm^0{*VOH6>kI>X}{Zx7<k<WtZ-Vf7Nrn@MNRX!VQU6!{4kDh$%Wa
zLCi?eZ|>&#O~P)<-Nvnctd&z_Z+evdb~2j$w%78UsXP;C3^Jr-*L)+_4WL7y+m8Kb
zvv+20bkOiMWNTSfSI(=rK{d7Tudey$YtipN9V+%uyx_^PcWU?gJ#BHWvI5NPM#cA(
z<mzjf{gyYc*;4YwO7diqs^-z&HIp5t6;|0qyiH?CFo^uPWQU>kWJ%jHnc!u6k8k$3
zX;bkOnP=K!GQD@zv0p|`p(k}-xIOQCJ@enK`x|e}kyv}O;*FWgT)F9|kG1JP+myVM
zvE@L<_kzbiIIrZ!`yH5b?(6hk$5nTZbx%&;xO9ztEgS!{z5I6n{~0Z>y6`e}Lhp=}
zsg}X<2gP|N&bmIC`<!%xqxhkzQ(6MoAD(=%QCn|!!%X?wQ`?@-SA6ZmBUfq}Hd#(F
zrK@`CrNonLUBA95-b~CpQKOJtzVl$(k;(U}YP&uA=S=b4{)kVeTx8;-vPJruYU}@o
zb#;3icrhhzFnZPX`h)xO&+nx56HQ)q9%s4R9(!oR=G_l2xbFC{CqL36U8*}}gN3m~
z`Cr!^YuR+n)a9;TdZC<9yko*zwU%=p*Voxb{68m?cIM}l*KS`_EHx*3hlnzjyHy|S
z-8*6CXR%L5Zs|LuKRmSY@rw!Xj-?krC^|OL)8%ZL<ztE5JiR@Syq5XQzM1qcIJRs}
zn7Z82I}`4QKD{vSZO#pC3CkRTwB5qfjoa=B{JfA;oFHyiH>>$SXY$meS;t=nO#JHB
zwd<Rb!tN8gZ+dPnIGj~3P#z)tDKm@-+$(?(4!25gl)Fu5Ul_nKThY()wvRwl*ZQ!H
zzgsn@{7GYCy7BRfx5&Lx;g`+78dWxae_+UDXe-wJbjmgrPo)>D7w`FG(KVrP=VVXM
zGG*`1yT5PE$~qJ%+J2d_tLf-F2GcHkk)w&*7f(L${=1^&nHxH4k0uKIE8tmvxX*9e
z$Lj2rYrlhTtM>EL*^tbeDpO`{73nldLF7YV^mL7MV`+g+a;uMa+>iEGnP#D{>2m1p
zotH&IyzCvx2U52;27I`7ReAQB8)tsWs}w899ADM{`O@Y7WPPclsn11)IgS>nFMY6K
zR)A=0!~OP)S;tpgS6&zp9owrfaWJd+!N(mI4-54}Sm$4FKbUklDXWWFkcUOUO4+QN
z({s+ceF`tWFZ5!2W2U`I=<xH;$s4T=UX}J1i*d7ChQ&ErNF7frpEPY+m!rOV*~iC$
zt5&b>JR&0F$Dg;HSCQr6g84JF61U%;Ai%=JI6?RmhiTjPeR82@54X(Z{PB0j?C+gk
zxxS|V5?Cf*^(&ghK3AoqrO8F4@ot*3SYDN`caV0mzA%@`bjCxB_JXW)_^da21lMwM
zurq5owVzFUw%k#Gv%TXOOW17Dz3Vh}SFF-%VPb5_Ign+U_~6I&KoQpY+xy<hZZ?#x
zZ!>e_?fvht=%PkyYHG=g3yg~|RxCSu5nK^p_#yu^Jzb%rvi69x;FoPJj0$ob3pdX&
zu#&x>snBqw>GP4uf{PE_?*EA^7W%&A*ycCBseU{9=2!VoS~B@oZsfA9b$(yI#$Jz}
z{E+>mj<-gle*XTCTHl%$#@Ou<cM6Q&b;R7}_{qm*pJvRxRrmV3U-<s5>DD`IRXBf|
z6qoJV_W$OLxi$N@+C@(dpCpxRy`|#s-V}@4zt2+Zh5RNxykhw`@lysH6K}7){8@(u
z(E6zOK~MH~p|efTJcZb%3pcNRx$j`2eeKV%jNHbXf8w;mqis*u?W$hBP0{b8)c?A_
z^Q-p${4KAtS!)8Xncm)KVJgj%s&!dYo?rX*$FAuA>aYKAZZ_Th!~gvA?bp72udKJ)
zd1R&Vb)6~Br*kClN4JH&`u~KZ@YZGKbQd{D9s!ZP^G*ND?{VfozVoo)76t|e)e_f;
zl9a@fRIB8o)Wnih1|tJQ3tdAaT_b}KLnA8#6DuPFT>}d%1B1m&-;|?h$jwj5OsmAL
z;nI(g(+ms@8Xy}olasBI^9w4AGSf3k7@T1&1;>JdoKywRyktEc1(419KCF%m3=HBh
pJt?Wh$wiq3C7Jno3@(|)1v!b8;?h%hFfcGMc)I$ztaD0e0ss$)1D5~*

literal 0
HcmV?d00001

-- 
GitLab


From 5d7d5a76a339a7ee947aa2e7df812787346e01a5 Mon Sep 17 00:00:00 2001
From: "Wen-Heng (Jack) Chung" <whchung@gmail.com>
Date: Tue, 15 Jan 2019 22:18:50 +0000
Subject: [PATCH 0773/1095] [CUDA] Fix invalid nvcc configuration

---
 third_party/gpus/cuda_configure.bzl | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl
index af8876ba17..f8f3e90708 100644
--- a/third_party/gpus/cuda_configure.bzl
+++ b/third_party/gpus/cuda_configure.bzl
@@ -1409,9 +1409,11 @@ def _create_local_cuda_repository(repository_ctx):
     # bazel's header check failing.
     cuda_defines["%{extra_no_canonical_prefixes_flags}"] = (
         "flag: \"-fno-canonical-system-headers\"")
-    nvcc_path = "cuda/bin/nvcc"
-    if _is_windows(repository_ctx):
-      nvcc_path += ".exe"
+    nvcc_path = str(
+        repository_ctx.path("%s/bin/nvcc%s" % (
+            cuda_config.cuda_toolkit_path,
+            ".exe" if _is_windows(repository_ctx) else "",
+        )))
     _tpl(
         repository_ctx,
         "crosstool:BUILD",
-- 
GitLab


From b4aed7c931041d9d8406ad45fed49e56d7f394e9 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Tue, 15 Jan 2019 14:12:28 -0800
Subject: [PATCH 0774/1095] Make XlaCompiledCpuFunction constructor explicit;
 NFC

PiperOrigin-RevId: 229438156
---
 tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h b/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h
index e1da19cf1d..de2e485a47 100644
--- a/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h
+++ b/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h
@@ -112,7 +112,7 @@ class XlaCompiledCpuFunction {
     RESULTS_PROFILES_AND_TEMPS_ONLY,
   };
 
-  XlaCompiledCpuFunction(
+  explicit XlaCompiledCpuFunction(
       const StaticData& static_data,
       AllocMode alloc_mode = AllocMode::ARGS_RESULTS_PROFILES_AND_TEMPS);
   virtual ~XlaCompiledCpuFunction();
-- 
GitLab


From d25d54029fe407932155ad0d527510ebd9ec8e07 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Tue, 15 Jan 2019 14:30:57 -0800
Subject: [PATCH 0775/1095] [TF:XLA] Bump open source llvm revision to r351210

PiperOrigin-RevId: 229441372
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 9a699ae243..81de0fb45b 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -502,11 +502,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "llvm",
         build_file = clean_dep("//third_party/llvm:llvm.autogenerated.BUILD"),
-        sha256 = "066d58d687ea0a97d36135d4aecf6ea31b900cfc74582b2349cadf0c7e4a04fd",
-        strip_prefix = "llvm-06a887ac8a688a23aa09252f44e9bb95797d057f",
+        sha256 = "c6e6de0b5cd524acf276188cb6aae427568070cd1b9cb9c0e3aab8044f3ac278",
+        strip_prefix = "llvm-a42cde684126a3797f3eda9c894c379e7a8c66c1",
         urls = [
-            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/06a887ac8a688a23aa09252f44e9bb95797d057f.tar.gz",
-            "https://github.com/llvm-mirror/llvm/archive/06a887ac8a688a23aa09252f44e9bb95797d057f.tar.gz",
+            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/a42cde684126a3797f3eda9c894c379e7a8c66c1.tar.gz",
+            "https://github.com/llvm-mirror/llvm/archive/a42cde684126a3797f3eda9c894c379e7a8c66c1.tar.gz",
         ],
     )
 
-- 
GitLab


From 73b1e8ebbee9246a85acaebcce9b9a8672c22216 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 15 Jan 2019 14:31:41 -0800
Subject: [PATCH 0776/1095] Use default value if the explicit_padding attribute
 is not specified for Conv2D

PiperOrigin-RevId: 229441517
---
 tensorflow/core/framework/common_shape_fns.cc | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/framework/common_shape_fns.cc b/tensorflow/core/framework/common_shape_fns.cc
index 3d3711ffc5..876ac188ac 100644
--- a/tensorflow/core/framework/common_shape_fns.cc
+++ b/tensorflow/core/framework/common_shape_fns.cc
@@ -14,6 +14,7 @@ limitations under the License.
 ==============================================================================*/
 #include "tensorflow/core/framework/common_shape_fns.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
+#include "tensorflow/core/lib/core/errors.h"
 
 namespace tensorflow {
 
@@ -498,7 +499,12 @@ Status Conv2DShapeImpl(shape_inference::InferenceContext* c,
 
   std::vector<int64> explicit_paddings;
   if (supports_explicit_padding) {
-    TF_RETURN_IF_ERROR(c->GetAttr("explicit_paddings", &explicit_paddings));
+    Status s = c->GetAttr("explicit_paddings", &explicit_paddings);
+    // Use the default value, which is an empty list, if the attribute is not
+    // found. Otherwise return the error to the caller.
+    if (!s.ok() && !errors::IsNotFound(s)) {
+      return s;
+    }
     TF_RETURN_IF_ERROR(CheckValidPadding(padding, explicit_paddings,
                                          /*num_dims=*/4, data_format));
   } else {
-- 
GitLab


From 750b30d2aa9e41802028a1c6693f73fa6ec6be1b Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Tue, 15 Jan 2019 15:09:35 -0800
Subject: [PATCH 0777/1095] Make layer JSON config saved in checkpoints
 optional for restore

In general if a Python string is in the checkpoint but not used directly in the saving program, assert_consumed will pass even if the attribute is totally absent on restore.

This should fix checkpoint compatibility with saved_model.load() even without proper Keras revived types. There's no reason it should fail if those aren't available for some reason.

PiperOrigin-RevId: 229448783
---
 .../checkpointable_object_graph.proto         |  4 +++
 .../python/training/checkpointable/base.py    | 14 ++++++++--
 .../training/checkpointable/base_test.py      | 28 +++++++++++++++++++
 .../python/training/checkpointable/util.py    | 11 +++++++-
 .../python/training/saving/saveable_object.py |  5 ++++
 5 files changed, 58 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/protobuf/checkpointable_object_graph.proto b/tensorflow/core/protobuf/checkpointable_object_graph.proto
index 651f692f6d..f2956404b5 100644
--- a/tensorflow/core/protobuf/checkpointable_object_graph.proto
+++ b/tensorflow/core/protobuf/checkpointable_object_graph.proto
@@ -30,6 +30,10 @@ message CheckpointableObjectGraph {
       string full_name = 2;
       // The generated name of the Tensor in the checkpoint.
       string checkpoint_key = 3;
+      // Whether checkpoints should be considered as matching even without this
+      // value restored. Used for non-critical values which don't affect the
+      // TensorFlow graph, such as layer configurations.
+      bool optional_restore = 4;
     }
 
     message SlotVariableReference {
diff --git a/tensorflow/python/training/checkpointable/base.py b/tensorflow/python/training/checkpointable/base.py
index 1aaa85258c..f2797bc5ff 100644
--- a/tensorflow/python/training/checkpointable/base.py
+++ b/tensorflow/python/training/checkpointable/base.py
@@ -142,8 +142,10 @@ class PythonStringStateSaveable(PythonStateSaveable):
       state_callback: A function taking no arguments which returns a
         string. This function is run every time a checkpoint is written.
       restore_callback: A function taking a Python string, used to restore
-        state. Optional; defaults to doing nothing.
+        state. Optional; defaults to doing nothing, in which case it is ignored
+        by status assertions such as assert_consumed().
     """
+    self._has_trivial_state_callback = (restore_callback is None)
     def _state_callback_wrapper():
       with ops.init_scope():
         return state_callback()
@@ -156,6 +158,11 @@ class PythonStringStateSaveable(PythonStateSaveable):
     super(PythonStringStateSaveable, self).__init__(
         self._save_string, [spec], name)
 
+  @property
+  def optional_restore(self):
+    """For values with no restore, relaxes assert_consumed()."""
+    return self._has_trivial_state_callback
+
   def feed_dict_additions(self):
     """When running a graph, indicates fresh state to feed."""
     return {self._save_string: self._state_callback()}
@@ -351,8 +358,9 @@ class _CheckpointPosition(object):
           # added or deleted. Stores unused attributes so an exception can be
           # raised if the user decides to check that everything in the
           # checkpoint was loaded.
-          self._checkpoint.unused_attributes.setdefault(
-              self.checkpointable, []).append(serialized_tensor.name)
+          if not serialized_tensor.optional_restore:
+            self._checkpoint.unused_attributes.setdefault(
+                self.checkpointable, []).append(serialized_tensor.name)
           continue
         if callable(saveable_factory):
           saveable = saveable_factory(name=serialized_tensor.checkpoint_key)
diff --git a/tensorflow/python/training/checkpointable/base_test.py b/tensorflow/python/training/checkpointable/base_test.py
index fd935ac559..736cbee488 100644
--- a/tensorflow/python/training/checkpointable/base_test.py
+++ b/tensorflow/python/training/checkpointable/base_test.py
@@ -16,6 +16,9 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import functools
+import os
+
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.platform import test
@@ -57,5 +60,30 @@ class InterfaceTests(test.TestCase):
             name="v", shape=[], overwrite=False,
             getter=variable_scope.get_variable)
 
+  def testAssertConsumedWithUnusedPythonState(self):
+    has_config = base.CheckpointableBase()
+    has_config.get_config = lambda: {}
+    saved = util.Checkpoint(obj=has_config)
+    save_path = saved.save(os.path.join(self.get_temp_dir(), "ckpt"))
+    restored = util.Checkpoint(obj=base.CheckpointableBase())
+    restored.restore(save_path).assert_consumed()
+
+  def testAssertConsumedFailsWithUsedPythonState(self):
+    has_config = base.CheckpointableBase()
+    attributes = {
+        "foo_attr": functools.partial(
+            base.PythonStringStateSaveable,
+            state_callback=lambda: "",
+            restore_callback=lambda x: None)}
+    has_config._gather_saveables_for_checkpoint = lambda: attributes
+    saved = util.Checkpoint(obj=has_config)
+    save_path = saved.save(os.path.join(self.get_temp_dir(), "ckpt"))
+    restored = util.Checkpoint(obj=base.CheckpointableBase())
+    status = restored.restore(save_path)
+    with self.assertRaisesRegexp(AssertionError, "foo_attr"):
+      status.assert_consumed()
+
+
 if __name__ == "__main__":
+  ops.enable_eager_execution()
   test.main()
diff --git a/tensorflow/python/training/checkpointable/util.py b/tensorflow/python/training/checkpointable/util.py
index a45263f5c6..1ff0cd9f25 100644
--- a/tensorflow/python/training/checkpointable/util.py
+++ b/tensorflow/python/training/checkpointable/util.py
@@ -666,7 +666,13 @@ def _add_attributes_to_object_graph(
         if cached_attributes is not None:
           cached_attributes[name] = saveables
 
+      optional_restore = None
       for saveable in saveables:
+        if optional_restore is None:
+          optional_restore = saveable.optional_restore
+        else:
+          optional_restore = optional_restore and saveable.optional_restore
+
         if hasattr(saveable, "full_name"):
           attribute.full_name = saveable.full_name
         if isinstance(saveable, base.PythonStateSaveable):
@@ -688,6 +694,9 @@ def _add_attributes_to_object_graph(
                     % (checkpointable, new_feed_key))
             feed_additions.update(saveable_feed_dict)
         named_saveable_objects.append(saveable)
+      if optional_restore is None:
+        optional_restore = False
+      attribute.optional_restore = optional_restore
 
   return named_saveable_objects, feed_additions
 
@@ -1020,7 +1029,7 @@ class CheckpointLoadStatus(_LoadStatus):
       raise AssertionError(
           ("Unused attributes in these objects (the attributes exist in the "
            "checkpoint but not in the objects): %s") % (
-               self._checkpoint.unused_attributes.items(),))
+               list(self._checkpoint.unused_attributes.items()),))
     return self
 
   def assert_existing_objects_matched(self):
diff --git a/tensorflow/python/training/saving/saveable_object.py b/tensorflow/python/training/saving/saveable_object.py
index 4b19294b65..981d4580fc 100644
--- a/tensorflow/python/training/saving/saveable_object.py
+++ b/tensorflow/python/training/saving/saveable_object.py
@@ -66,6 +66,11 @@ class SaveableObject(object):
     self.name = name
     self._device = None
 
+  @property
+  def optional_restore(self):
+    """A hint to restore assertions that this object is optional."""
+    return False  # Default to required
+
   @property
   def device(self):
     """The device for SaveSpec Tensors."""
-- 
GitLab


From ee64c06de75664e937d3f6e7791dac9a9e6731ac Mon Sep 17 00:00:00 2001
From: Yanan Cao <ycao@google.com>
Date: Tue, 15 Jan 2019 15:15:19 -0800
Subject: [PATCH 0778/1095] Support nested output structure in xla.compile and
 tpu.rewrite

PiperOrigin-RevId: 229449848
---
 tensorflow/contrib/compiler/xla.py       | 227 +++++++++++++++------
 tensorflow/contrib/tpu/python/tpu/tpu.py | 239 ++++++++++++++++-------
 2 files changed, 335 insertions(+), 131 deletions(-)

diff --git a/tensorflow/contrib/compiler/xla.py b/tensorflow/contrib/compiler/xla.py
index 2aa5d77b10..0f1be500f4 100644
--- a/tensorflow/contrib/compiler/xla.py
+++ b/tensorflow/contrib/compiler/xla.py
@@ -85,7 +85,14 @@ def compile(computation, inputs=None):  # pylint: disable=redefined-builtin
       with `tf.convert_to_tensor`.
 
   Returns:
-    A list of output tensors.
+    Same data structure as if computation(*inputs) is called directly with some
+    exceptions for correctness. Exceptions include:
+      1) None output: a NoOp would be returned which control-depends on
+         computation.
+      2) Single value output: A tuple containing the value would be returned.
+      3) Operation-only outputs: a NoOp would be returned which
+         control-depends on computation.
+      TODO(b/121383831): Investigate into removing these special cases.
   """
   # pylint: disable=protected-access
   return _compile_internal(computation, inputs)
@@ -251,13 +258,21 @@ def _compile_internal(computation, inputs=None):
   Args:
     computation: A Python function that builds the computation to compile and
       execute.
-    inputs: A list of input tensors or `None` (equivalent to `[]`). Its order
-      should match ordering of computation arguments.
+    inputs: A list of inputs or `None` (equivalent to an empty list). Each input
+      can be a nested structure containing values that are convertible to
+      tensors. Note that passing an N-dimension list of compatible values will
+      result in a N-dimension list of scalar tensors rather than a single Rank-N
+      tensors. If you need different behavior, convert part of inputs to tensors
+      with `tf.convert_to_tensor`.
+
   Returns:
-    A list of output tensors from computation.
+    Same data structure as if computation(*inputs) is called directly with some
+    exceptions for correctness. Exceptions include: 1) None output 2) Single
+    value output 3) Operation-only outputs
   Raises:
     ValueError: If any element in computation outputs is neither an operations
       or a value that can be converted to tensor.
+    ValueError: If computation outputs is non-flat and contains any Operations.
     TypeError: If `inputs` is not a list or tuple.
   """
   if inputs is None:
@@ -300,66 +315,166 @@ def _compile_internal(computation, inputs=None):
     # Restore variable scope after computation.
     vscope.set_use_resource(saved_use_resource)
 
-    # If the computation returns `None`, make it an empty tuple.
-    if outputs is None:
-      outputs = tuple()
-    # If the computation only returned one value, make it a tuple.
-    if not isinstance(outputs, collections.Sequence):
-      outputs = (outputs,)
-
-    # Append `no_op` here so that return value of this function always contains
-    # at least one op that can trigger XlaLaunch node.
-    outputs += (control_flow_ops.no_op(),)
-    try:
-      outputs = [
-          o if isinstance(o, ops.Operation) else ops.convert_to_tensor(o)
-          for o in outputs
-      ]
-    except Exception as e:
-      raise ValueError(
-          'XLA computation function return values must all either be Operations'
-          ' or convertible to Tensors. Got error: "%s"' % str(e))
-
-    # Separates the returned Operations and Tensors.
-    output_operations = [o for o in outputs if isinstance(o, ops.Operation)]
-    output_tensors = [o for o in outputs if not isinstance(o, ops.Operation)]
-
-    if outputs != output_tensors + output_operations:
-      raise ValueError(
-          'XLA computation function must return zero or more Tensor values '
-          'followed by zero or more Operations.')
-    output_arity = len(output_tensors)
-
-    new_output_tensors = []
-    for t in output_tensors:
-      with ops.device(t.device if t.device else ''):
-        new_output_tensors.append(array_ops.identity(t))
+    outputs_is_flat = is_flat(outputs)
+    if outputs_is_flat:
+      output_tensors, control_deps = _postprocess_flat_outputs(outputs)
+    else:
+      output_tensors, control_deps = _postprocess_non_flat_outputs(outputs)
 
-    output_tensors = new_output_tensors
     context.ExitResult(output_tensors)
   finally:
     context.report_unsupported_operations()
     context.Exit()
 
-  outputs = [
-      xla_ops.xla_cluster_output(output_tensors[i], name='output{}'.format(i))
-      for i in xrange(output_arity)
+  # When XLA computation returns only operations and no tensors, a NoOp
+  # dependent on the operations in outputs is returned. Otherwise final
+  # outputs would be empty and there is no way to trigger returned
+  # operations.
+  if not output_tensors:
+    return control_flow_ops.group(control_deps, name='output_0')
+
+  output_tensors = [
+      xla_ops.xla_cluster_output(o, name='output{}'.format(i))
+      for i, o in enumerate(output_tensors)
   ]
 
-  with ops.control_dependencies(output_operations):
-    if output_arity == 0:
-      # When XLA computation returns only operations and no tensors, a NoOp
-      # dependent on the operations in outputs is returned. Otherwise final
-      # outputs would be empty and there is no way to trigger returned
-      # operations.
-      return control_flow_ops.no_op(name='output_0')
-    else:
-      # Wraps the outputs in identity operators that carries control
-      # dependencies.
-      return [
-          array_ops.identity(outputs[i], name='output_%d' % i)
-          for i in xrange(output_arity)
-      ]
+  with ops.control_dependencies(control_deps):
+    # Wraps the outputs in identity operators that carries control
+    # dependencies.
+    output_tensors = [
+        array_ops.identity(o, name='output_%d' % i)
+        for i, o in enumerate(output_tensors)
+    ]
+
+  # If `computation` returned non-flat output structure, pack output tensors
+  # back into same structure.
+  if not outputs_is_flat:
+    output_tensors = nest.pack_sequence_as(
+        structure=outputs, flat_sequence=output_tensors)
+
+  return output_tensors
+
+
+def is_flat(outputs):
+  """Checks if outputs is a flat structure.
+
+    Following structures and values are considered flat:
+    1) None
+    2) A single object
+    3) A list or tuple of Tensors/Operations
+
+    The only structures that this function understands are sequences and
+    dictionaries.  E.g. this means that if outputs contains a single
+    user-defined Object, it is considered to be flat. Errors are raised later on
+    if that Object cannot be converted to a Tensor.
+
+  Args:
+    outputs: Output from `computation` inside `xla.compile`.
+
+  Returns:
+    A boolean indicates whether outputs is flat.
+  """
+  # If outputs is a list or tuple, check if it has any nested structure. If
+  # there is, then outputs is non-flat.
+  if isinstance(outputs, collections.Sequence):
+    for o in outputs:
+      if isinstance(o, collections.Sequence) or isinstance(o, dict):
+        return False
+
+  # If outputs is a dict, it is non-flat.
+  if isinstance(outputs, dict):
+    return False
+
+  # Getting here means either outputs itself is a single non-structured value
+  # or it is a flat list of single non-structured values.
+  return True
+
+
+def _postprocess_flat_outputs(outputs):
+  """Validates flat outputs and adds back device assignments.
+
+  Args:
+    outputs: Output from `computation` inside `xla.compile`.
+
+  Returns:
+    Tensors and Operations extracted from outputs.
+  """
+  # Following code segment is to preserve legacy behavior. Previously we only
+  # supported flat outputs and thus for consistency it was nice to convert even
+  # single element into a tuple. But now that we support arbitrary output
+  # structure, this is no longer necessary.
+  # TODO(b/121383831): Migrate all legacy use cases and delete this special
+  # case.
+  # If the computation returns `None`, make it an empty tuple.
+  if outputs is None:
+    outputs = tuple()
+  # If the computation only returned one value, make it a tuple.
+  if not isinstance(outputs, collections.Sequence):
+    outputs = (outputs,)
+
+  # Append `no_op` here so that return value of this function always contains
+  # at least one op that can trigger XlaLaunch node.
+  outputs += (control_flow_ops.no_op(),)
+  try:
+    outputs = [
+        o if isinstance(o, ops.Operation) else ops.convert_to_tensor(o)
+        for o in outputs
+    ]
+  except Exception as e:
+    raise ValueError(
+        'XLA computation function return values must all either be Operations'
+        ' or convertible to Tensors. Got error: "%s"' % str(e))
+
+  # Separates the returned Operations and Tensors.
+  output_operations = [o for o in outputs if isinstance(o, ops.Operation)]
+  output_tensors = [o for o in outputs if not isinstance(o, ops.Operation)]
+
+  if outputs != output_tensors + output_operations:
+    raise ValueError(
+        'XLA computation function must return zero or more Tensor values '
+        'followed by zero or more Operations.')
+
+  new_output_tensors = []
+  for t in output_tensors:
+    with ops.device(t.device if t.device else ''):
+      new_output_tensors.append(array_ops.identity(t))
+
+  return new_output_tensors, output_operations
+
+
+def _postprocess_non_flat_outputs(outputs):
+  """Validates non-flat outputs and adds back device assignments.
+
+  Args:
+    outputs: Output from `computation` inside `xla.compile`.
+
+  Returns:
+    Tensors extracted from outputs and an empty list because Operations are not
+    allowed in non-flat outputs..
+  """
+  # Convert all non-Operation outputs to Tensors.
+  new_output_tensors = []
+  for o in nest.flatten(outputs):
+    if isinstance(o, ops.Operation):
+      raise ValueError(
+          'xla.compile does not support Operation as return value in non-flat '
+          'output structure. You can set returned Operations as control '
+          'dependencies of returned Tensors so Operations are triggered when '
+          'Tensors are evaluated. Operation found: "%s"' % o.name)
+
+    try:
+      o = ops.convert_to_tensor(o)
+    except Exception as e:
+      raise ValueError(
+          'XLA computation function return values must all either be '
+          'Operations or convertible to Tensors. Got error: "%s"' % str(e))
+
+    # Makes sure even pass-through inputs/outputs are touched in compile
+    # context by creating an Identity node inside compile context.
+    with ops.device(o.device if o.device else ''):
+      new_output_tensors.append(array_ops.identity(o))
+
+  return new_output_tensors, []
 
 
 @contextlib.contextmanager
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu.py b/tensorflow/contrib/tpu/python/tpu/tpu.py
index a267dd435c..b04baebfe6 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu.py
@@ -19,6 +19,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import collections
 from six.moves import xrange  # pylint: disable=redefined-builtin
 
 from tensorflow.contrib.compiler import xla
@@ -503,7 +504,17 @@ def replicate(computation,
       replicas is equal to the number of cores in the TPU system.
     name: (Deprecated) Does nothing.
   Returns:
-    A list of lists of output tensors, indexed by `[replica_num][output_num]`.
+    A list of outputs, indexed by `[replica_num]` each output can be a nested
+    structure same as what computation() returns with a few exceptions.
+
+    Exceptions include:
+      1) None output: a NoOp would be returned which control-depends on
+         computation.
+      2) Single value output: A tuple containing the value would be returned.
+      3) Operation-only outputs: a NoOp would be returned which
+         control-depends on computation.
+      TODO(b/121383831): Investigate into removing these special cases.
+
   Raises:
     ValueError: If all replicas do not have equal numbers of input tensors.
     ValueError: If the number of inputs per replica does not match
@@ -712,51 +723,12 @@ def split_compile_and_replicate(computation,
       vscope.set_use_resource(saved_use_resource)
       vscope.set_custom_getter(saved_custom_getter)
 
-    # If the computation returns `None`, make it an empty tuple.
-    if outputs is None:
-      outputs = tuple()
-    # If the computation only returned one value, makes it a tuple.
-    if not isinstance(outputs, (list, tuple)):
-      outputs = (outputs,)
-
-    # Append `no_op` here so that fetching any return value of this function
-    # will trigger TPUExecute node.
-    outputs += (control_flow_ops.no_op(),)
-    try:
-      with ops.device(core(0)):
-        outputs = [
-            o if isinstance(o, ops.Operation) else ops.convert_to_tensor(o)
-            for o in outputs
-        ]
-    except Exception as e:
-      raise ValueError(
-          "TPU function return values must all either be Operations or "
-          "convertible to Tensors. Got '%s'" % str(e))
-
-    # Separates the returned Operations and Tensors.
-    output_operations = [o for o in outputs if isinstance(o, ops.Operation)]
-    output_tensors = [o for o in outputs if not isinstance(o, ops.Operation)]
-
-    if outputs != output_tensors + output_operations:
-      raise ValueError(
-          "TPU functions must return zero-or more Tensor values followed by "
-          "zero or more Operations.")
-    output_arity = len(output_tensors)
+    outputs_is_flat = xla.is_flat(outputs)
+    if outputs_is_flat:
+      output_tensors, control_deps = _postprocess_flat_outputs(outputs)
+    else:
+      output_tensors, control_deps = _postprocess_non_flat_outputs(outputs)
 
-    # Wraps outputs in Identity ops. Otherwise a replicated input copied
-    # straight to an output would bypass the replicate(). This would be bad
-    # because the TPUReplicatedInput/TPUReplicatedOutput operator would not
-    # be rewritten away, leading to a runtime error.
-    # TODO(phawkins): extend the rewrite to elide these nodes instead.
-    new_output_tensors = []
-    for t in output_tensors:
-      with ops.device(t.device if t.device else core(0)):
-        o = array_ops.identity(t)
-        # pylint: disable=protected-access
-        o.op._set_attr("_tpu_output_identity", attr_value_pb2.AttrValue(b=True))
-        # pylint: enable=protected-access
-        new_output_tensors.append(o)
-    output_tensors = new_output_tensors
     context.ExitResult(output_tensors)
   finally:
     context.report_unsupported_operations()
@@ -768,11 +740,6 @@ def split_compile_and_replicate(computation,
     attr_value.list.s.extend([compat.as_bytes(x) for x in host_compute_core])
     metadata._set_attr("host_compute_core", attr_value)  # pylint: disable=protected-access
 
-  # Fan-out: Builds a TPUReplicatedOutput node for each output.
-  outputs = [tpu_ops.tpu_replicated_output(output_tensors[i], num_replicas,
-                                           name="output{}".format(i))
-             for i in xrange(output_arity)]
-
   with ops.control_dependencies([metadata]):
     if use_tpu:
       compile_status = tpu_ops.tpu_compilation_result()
@@ -782,28 +749,146 @@ def split_compile_and_replicate(computation,
     else:
       compile_status = control_flow_ops.no_op(name="compilation_status")
 
-  with ops.control_dependencies(output_operations):
-    if output_arity == 0:
-      # Returns a list of NoOps dependent on the replication Op, indexed by
-      # [replica_num].
-      return [
-          compile_status, [
-              control_flow_ops.no_op(name="shard_%d" % i)
-              for i in range(num_replicas)
-          ]
-      ]
-    else:
-      # Wraps the outputs in identity operators so the names of any possible
-      # `fetch` nodes are preserved by the replication rewrite.
-      return [
-          compile_status, [[
-              array_ops.identity(
-                  outputs[out][replica],
-                  name="output_%d_shard_%d" % (out, replica))
-              for out in xrange(output_arity)
-          ]
-                           for replica in xrange(num_replicas)]
+  if not output_tensors:
+    # Returns a list of NoOps dependent on the replication Op, indexed by
+    # [replica_num].
+    return [
+        compile_status,
+        [
+            control_flow_ops.group(control_deps, name="shard_%d" % i)
+            for i in range(num_replicas)
+        ]
+    ]
+
+  # Fan-out: Builds a TPUReplicatedOutput node for each output.
+  replicated_outputs = [[] for i in xrange(num_replicas)]
+  for i, t in enumerate(output_tensors):
+    # Fan-out: Builds a TPUReplicatedOutput node for each output.
+    ys = tpu_ops.tpu_replicated_output(
+        t, num_replicas, name="output{}".format(i))
+
+    # Wraps the outputs in identity operators so the names of any possible
+    # `fetch` nodes are preserved by the replication rewrite.
+    with ops.control_dependencies(control_deps):
+      for replica in xrange(num_replicas):
+        replicated_outputs[replica].append(
+            array_ops.identity(
+                ys[replica], name="output_%d_shard_%d" % (i, replica)))
+
+  if not outputs_is_flat:
+    replicated_outputs = [
+        nest.pack_sequence_as(outputs, replica_outs)
+        for replica_outs in replicated_outputs
+    ]
+
+  return [compile_status, replicated_outputs]
+
+
+def _postprocess_flat_outputs(outputs):
+  """Validates non-flat outputs, add backs device assignments and other attrs.
+
+  Args:
+    outputs: Output from `computation` inside `tpu.rewrite`.
+
+  Returns:
+    Tensors and Operations extracted from outputs.
+  """
+  # Following code segment is to preserve legacy behavior. Previously we only
+  # supported flat outputs and thus for consistency it was nice to convert even
+  # single element into a tuple. But now that we support arbitrary output
+  # structure, this is no longer necessary.
+  # TODO(b/121383831): Migrate all legacy use cases and delete this special
+  # case.
+  # If the computation returns `None`, make it an empty tuple.
+  if outputs is None:
+    outputs = tuple()
+  # If the computation only returned one value, makes it a tuple.
+  if not isinstance(outputs, collections.Sequence):
+    outputs = (outputs,)
+
+  # Append `no_op` here so that fetching any return value of this function
+  # will trigger TPUExecute node.
+  outputs += (control_flow_ops.no_op(),)
+  try:
+    with ops.device(core(0)):
+      outputs = [
+          o if isinstance(o, ops.Operation) else ops.convert_to_tensor(o)
+          for o in outputs
       ]
+  except Exception as e:
+    raise ValueError(
+        "TPU function return values must all either be Operations or "
+        "convertible to Tensors. Got '%s'" % str(e))
+
+  # Separates the returned Operations and Tensors.
+  output_operations = [o for o in outputs if isinstance(o, ops.Operation)]
+  output_tensors = [o for o in outputs if not isinstance(o, ops.Operation)]
+
+  if outputs != output_tensors + output_operations:
+    raise ValueError(
+        "TPU functions must return zero-or more Tensor values followed by "
+        "zero or more Operations.")
+
+  # Wraps outputs in Identity ops. Otherwise a replicated input copied
+  # straight to an output would bypass the replicate(). This would be bad
+  # because the TPUReplicatedInput/TPUReplicatedOutput operator would not
+  # be rewritten away, leading to a runtime error.
+  # TODO(phawkins): extend the rewrite to elide these nodes instead.
+  new_output_tensors = []
+  for t in output_tensors:
+    with ops.device(t.device if t.device else core(0)):
+      o = array_ops.identity(t)
+      # pylint: disable=protected-access
+      o.op._set_attr("_tpu_output_identity", attr_value_pb2.AttrValue(b=True))
+      # pylint: enable=protected-access
+      new_output_tensors.append(o)
+  return new_output_tensors, output_operations
+
+
+def _postprocess_non_flat_outputs(outputs):
+  """Validates non-flat outputs, add backs device assignments and other attrs.
+
+  Args:
+    outputs: Output from `computation` inside `tpu.rewrite`.
+
+  Returns:
+    Tensors extracted from outputs and an empty list because Operations are not
+    allowed in non-flat outputs..
+  """
+
+  # Flatten output items.
+  flat_outputs = nest.flatten(outputs)
+
+  # Convert all non-Operation outputs to Tensors.
+  for i, o in enumerate(flat_outputs):
+    if isinstance(o, ops.Operation):
+      raise ValueError(
+          "tpu.rewrite does not support Operation as return value in non-flat "
+          "output structure. You can set returned Operations as control "
+          "dependencies of returned Tensors so Operations are triggered when "
+          'Tensors are evaluated. Operation found: "%s"' % o.name)
+
+    try:
+      o = ops.convert_to_tensor(o)
+    except Exception as e:
+      raise ValueError(
+          "TPU function return values must all either be Operations or "
+          'convertible to Tensors. Got error: "%s"' % str(e))
+
+    # Wraps outputs in Identity ops. Otherwise a replicated input copied
+    # straight to an output would bypass the replicate(). This would be bad
+    # because the TPUReplicatedInput/TPUReplicatedOutput operator would not
+    # be rewritten away, leading to a runtime error.
+    # TODO(phawkins): extend the rewrite to elide these nodes instead.
+    with ops.device(core(0)):
+      o = array_ops.identity(o)
+      # pylint: disable=protected-access
+      o.op._set_attr("_tpu_output_identity", attr_value_pb2.AttrValue(b=True))
+      # pylint: enable=protected-access
+      flat_outputs[i] = array_ops.identity(o)
+
+  # All flat_outputs are Tensors, and no Operations.
+  return flat_outputs, []
 
 
 def split_compile_and_shard(computation,
@@ -1126,11 +1211,15 @@ def rewrite(computation,
       case the core attached to task 0, TPU device 0 is used.
     name: (Deprecated) Does nothing.
   Returns:
-    A list of output tensors.
+    Same data structure as if computation(*inputs) is called directly with some
+    exceptions for correctness. Exceptions include:
+      1) None output: a NoOp would be returned which control-depends on
+         computation.
+      2) Single value output: A tuple containing the value would be returned.
+      3) Operation-only outputs: a NoOp would be returned which
+         control-depends on computation.
+      TODO(b/121383831): Investigate into removing these special cases.
   """
-  if inputs is not None and not isinstance(inputs, (list, tuple)):
-    raise TypeError("tpu.rewrite() inputs must be a list or tuple")
-
   # TODO(b/36647078) remove disable when pylint bug is fixed.
   # pylint: disable=indexing-exception
   return replicate(
-- 
GitLab


From bea8a4a3a1fb0282b0c59a687d25c6ae160316fc Mon Sep 17 00:00:00 2001
From: Guangda Lai <31743510+aaroey@users.noreply.github.com>
Date: Tue, 15 Jan 2019 15:46:33 -0800
Subject: [PATCH 0779/1095] Fix formatting issues and broken build

---
 tensorflow/contrib/tensorrt/BUILD             |  2 +-
 .../tensorrt/python/trt_convert_test.py       |  8 ++--
 .../tensorrt/resources/trt_lru_cache.h        | 38 ++++++++++-----
 tensorflow/contrib/tensorrt/test/base_test.py | 14 +++---
 .../tensorrt/test/batch_matmul_test.py        |  2 +-
 .../tensorrt/test/biasadd_matmul_test.py      |  2 +-
 .../binary_tensor_weight_broadcast_test.py    |  2 +-
 .../tensorrt/test/concatenation_test.py       |  2 +-
 .../tensorrt/test/const_broadcast_test.py     |  2 +-
 .../contrib/tensorrt/test/conv2d_test.py      | 12 ++---
 .../test/dynamic_input_shapes_test.py         | 31 ++++++------
 .../tensorrt/test/identity_output_test.py     |  4 +-
 .../contrib/tensorrt/test/int32_test.py       |  4 +-
 .../contrib/tensorrt/test/lru_cache_test.py   | 48 ++++++++++++-------
 .../tensorrt/test/memory_alignment_test.py    |  2 +-
 .../multi_connection_neighbor_engine_test.py  |  2 +-
 .../tensorrt/test/neighboring_engine_test.py  |  2 +-
 .../tensorrt/test/quantization_test.py        |  2 +-
 .../contrib/tensorrt/test/rank_two_test.py    |  2 +-
 .../tensorrt/test/reshape_transpose_test.py   |  4 +-
 .../test/tf_trt_integration_test_base.py      | 46 +++++++++---------
 tensorflow/contrib/tensorrt/test/topk_test.py |  6 +--
 .../contrib/tensorrt/test/unary_test.py       |  2 +-
 .../tensorrt/test/vgg_block_nchw_test.py      |  2 +-
 .../contrib/tensorrt/test/vgg_block_test.py   |  2 +-
 25 files changed, 138 insertions(+), 105 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index 58c188b314..7ae3caaaa2 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -567,8 +567,8 @@ cc_library(
     hdrs = ["convert/utils.h"],
     copts = tf_copts(),
     deps = [
-        "//tensorflow/core:lib",
         "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
     ],
 )
 
diff --git a/tensorflow/contrib/tensorrt/python/trt_convert_test.py b/tensorflow/contrib/tensorrt/python/trt_convert_test.py
index 60c31580db..14ab4772c4 100644
--- a/tensorflow/contrib/tensorrt/python/trt_convert_test.py
+++ b/tensorflow/contrib/tensorrt/python/trt_convert_test.py
@@ -239,8 +239,8 @@ class TrtConvertTest(test_util.TensorFlowTestCase):
         # Run with batch size 2, a new engine is created and cached.
         self._TestRun(sess, 2, True)
         # Run with batch size 3, since the number of cached engines has reached
-        # the max, it should fall back to TF function.
-        self._TestRun(sess, 3, False)
+        # the max, it should evict an old engine and create a new one.
+        self._TestRun(sess, 3, True)
 
     # Test the output SavedModel
     with ops.Graph().as_default():
@@ -251,8 +251,8 @@ class TrtConvertTest(test_util.TensorFlowTestCase):
         # Run with batch size 2, a new engine is created and cached.
         self._TestRun(sess, 2, True)
         # Run with batch size 3, since the number of cached engines has reached
-        # the max, it should fall back to TF function.
-        self._TestRun(sess, 3, False)
+        # the max, it should evict an old engine and create a new one.
+        self._TestRun(sess, 3, True)
 
   def testCreateInferenceGraph_StaticOp(self):
     if not trt_convert.is_tensorrt_enabled():
diff --git a/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h b/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h
index ba455b9b35..afd1b83e74 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h
+++ b/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h
@@ -1,5 +1,20 @@
-#ifndef TENSORFLOW_CONTRIB_TENSORRT_TRT_LRU_CACHE_H_
-#define TENSORFLOW_CONTRIB_TENSORRT_TRT_LRU_CACHE_H_
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_LRU_CACHE_H_
+#define TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_LRU_CACHE_H_
 
 #include <list>
 #include <unordered_map>
@@ -9,9 +24,9 @@
 #include "tensorflow/core/framework/resource_mgr.h"
 #include "tensorflow/core/lib/core/errors.h"
 
-#if GOOGLE_CUDA
-#if GOOGLE_TENSORRT
+#if GOOGLE_CUDA && GOOGLE_TENSORRT
 #include "tensorrt/include/NvInfer.h"
+#endif  // GOOGLE_CUDA && GOOGLE_TENSORRT
 
 namespace tensorflow {
 namespace tensorrt {
@@ -40,9 +55,7 @@ class LRUCache {
 
   size_t count(const key_type& key) const { return objects_.count(key); }
 
-  value_type& at(const key_type& key) {
-    return Touch(key);
-  }
+  value_type& at(const key_type& key) { return Touch(key); }
 
   const_iterator begin() const { return objects_.begin(); }
   const_iterator end() const { return objects_.end(); }
@@ -111,6 +124,9 @@ struct VectorTensorShapeHasher {
   }
 };
 
+#if GOOGLE_CUDA
+#if GOOGLE_TENSORRT
+
 struct EngineContext {
   EngineContext() {}  // Creates an empty context.
   EngineContext(
@@ -167,10 +183,10 @@ class TRTEngineCacheResource : public tensorflow::ResourceBase {
       cache_;
 };
 
-}  // namespace tensorrt
-}  // namespace tensorflow
-
 #endif  // GOOGLE_TENSORRT
 #endif  // GOOGLE_CUDA
 
-#endif  // TENSORFLOW_CONTRIB_TENSORRT_TRT_LRU_CACHE_H_
+}  // namespace tensorrt
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_LRU_CACHE_H_
diff --git a/tensorflow/contrib/tensorrt/test/base_test.py b/tensorflow/contrib/tensorrt/test/base_test.py
index 0f6293b166..17e0b6f4d2 100644
--- a/tensorflow/contrib/tensorrt/test/base_test.py
+++ b/tensorflow/contrib/tensorrt/test/base_test.py
@@ -70,7 +70,7 @@ class SimpleSingleEngineTest(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name],
         input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[[(100, 6, 6, 6)]])
+        expected_output_dims=[[[100, 6, 6, 6]]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
@@ -127,7 +127,7 @@ class SimpleMultiEnginesTest(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name],
         input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[[(100, 12, 12, 6)]])
+        expected_output_dims=[[[100, 12, 12, 6]]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
@@ -185,7 +185,7 @@ class PartiallyConvertedTestA(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name],
         input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[[tuple(input_dims)]])
+        expected_output_dims=[[input_dims]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
@@ -255,7 +255,7 @@ class ConstInputTest(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name],
         input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[[tuple(input_dims)]])
+        expected_output_dims=[[input_dims]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
@@ -288,7 +288,7 @@ class ConstDataInputSingleEngineTest(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name],
         input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[[tuple(input_dims)]])
+        expected_output_dims=[[input_dims]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
@@ -322,7 +322,7 @@ class ConstDataInputMultipleEnginesTest(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name],
         input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[[tuple(input_dims)]])
+        expected_output_dims=[[input_dims]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
@@ -371,7 +371,7 @@ class ControlDependencyTest(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name],
         input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[[tuple(input_dims)]])
+        expected_output_dims=[[input_dims]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/batch_matmul_test.py b/tensorflow/contrib/tensorrt/test/batch_matmul_test.py
index 1b015dd369..46e3407d96 100644
--- a/tensorflow/contrib/tensorrt/test/batch_matmul_test.py
+++ b/tensorflow/contrib/tensorrt/test/batch_matmul_test.py
@@ -73,7 +73,7 @@ class BatchMatMulTest(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name, w1_name, w2_name],
         input_dims=[[input_dims, w1_dims, w2_dims]],
         output_names=[output_name],
-        expected_output_dims=[[(12, 5, 8, 7)]])
+        expected_output_dims=[[[12, 5, 8, 7]]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py b/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py
index 8a5f8ef5d5..ca23629efa 100644
--- a/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py
+++ b/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py
@@ -113,7 +113,7 @@ class BiasaddMatMulTest(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name],
         input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[[(4, 6680)]])
+        expected_output_dims=[[[4, 6680]]])
 
   def GetConversionParams(self, run_params):
     """Return a ConversionParams for test."""
diff --git a/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py b/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py
index 0083695a50..846fd009db 100644
--- a/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py
+++ b/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py
@@ -65,7 +65,7 @@ class BinaryTensorWeightBroadcastTest(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name],
         input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[[(5, 23040)]])
+        expected_output_dims=[[[5, 23040]]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/concatenation_test.py b/tensorflow/contrib/tensorrt/test/concatenation_test.py
index e6afe4e3f6..5d8742ae35 100644
--- a/tensorflow/contrib/tensorrt/test/concatenation_test.py
+++ b/tensorflow/contrib/tensorrt/test/concatenation_test.py
@@ -75,7 +75,7 @@ class ConcatenationTest(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name],
         input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[[(2, 126)]])
+        expected_output_dims=[[[2, 126]]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/const_broadcast_test.py b/tensorflow/contrib/tensorrt/test/const_broadcast_test.py
index 56a297afb5..9137d0072f 100644
--- a/tensorflow/contrib/tensorrt/test/const_broadcast_test.py
+++ b/tensorflow/contrib/tensorrt/test/const_broadcast_test.py
@@ -60,7 +60,7 @@ class ConstBroadcastTest(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name],
         input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[[(5, 12, 12, 1)]])
+        expected_output_dims=[[[5, 12, 12, 1]]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/conv2d_test.py b/tensorflow/contrib/tensorrt/test/conv2d_test.py
index d4cee3cce0..e7993b4620 100644
--- a/tensorflow/contrib/tensorrt/test/conv2d_test.py
+++ b/tensorflow/contrib/tensorrt/test/conv2d_test.py
@@ -103,9 +103,9 @@ class Conv2DNCHWTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=["input"],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=["output"],
-        expected_output_dims=[(13, 5, 7, 11)])
+        expected_output_dims=[[[13, 5, 7, 11]]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
@@ -128,9 +128,9 @@ class Conv2DNHWCTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=["input"],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=["output"],
-        expected_output_dims=[(13, 7, 11, 5)])
+        expected_output_dims=[[[13, 7, 11, 5]]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
@@ -178,9 +178,9 @@ class Conv2DStridedNCHWTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[(n, num_filters, h, w)])
+        expected_output_dims=[[[n, num_filters, h, w]]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/dynamic_input_shapes_test.py b/tensorflow/contrib/tensorrt/test/dynamic_input_shapes_test.py
index 1c6d411d11..90addecb14 100644
--- a/tensorflow/contrib/tensorrt/test/dynamic_input_shapes_test.py
+++ b/tensorflow/contrib/tensorrt/test/dynamic_input_shapes_test.py
@@ -18,17 +18,18 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
+import numpy as np
+
 # pylint: disable=unused-import
 from tensorflow.contrib.tensorrt.python.ops import trt_engine_op
 # pylint: enable=unused-import
+from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
+from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
-from tensorflow.python.framework import constant_op
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import nn
 from tensorflow.python.platform import test
-import numpy as np
 
 
 class DynamicInputShapesTest(trt_test.TfTrtIntegrationTestBase):
@@ -47,12 +48,10 @@ class DynamicInputShapesTest(trt_test.TfTrtIntegrationTestBase):
 
     g = ops.Graph()
     with g.as_default():
-      x = array_ops.placeholder(shape=(None, None, None, 1),
-                                dtype=dtypes.float32,
-                                name='input')
-      conv_filter1 = constant_op.constant(np.ones([3, 3, 1, 8]),
-                                          name="weights1",
-                                          dtype=dtypes.float32)
+      x = array_ops.placeholder(
+          shape=(None, None, None, 1), dtype=dtypes.float32, name="input")
+      conv_filter1 = constant_op.constant(
+          np.ones([3, 3, 1, 8]), name="weights1", dtype=dtypes.float32)
       bias1 = constant_op.constant(np.random.randn(8), dtype=dtypes.float32)
       x = nn.conv2d(
           input=x,
@@ -62,9 +61,8 @@ class DynamicInputShapesTest(trt_test.TfTrtIntegrationTestBase):
           name="conv")
       x = nn.bias_add(x, bias1)
       x = nn.relu(x)
-      conv_filter2 = constant_op.constant(np.ones([3, 3, 8, 1]),
-                                          name="weights2",
-                                          dtype=dtypes.float32)
+      conv_filter2 = constant_op.constant(
+          np.ones([3, 3, 8, 1]), name="weights2", dtype=dtypes.float32)
       bias2 = constant_op.constant(np.random.randn(1), dtype=dtypes.float32)
       x = nn.conv2d(
           input=x,
@@ -73,13 +71,13 @@ class DynamicInputShapesTest(trt_test.TfTrtIntegrationTestBase):
           padding="SAME",
           name="conv")
       x = nn.bias_add(x, bias2)
-      x = array_ops.identity(x, name='output')
+      x = array_ops.identity(x, name="output")
 
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
-        input_names=['input'],
+        input_names=["input"],
         input_dims=input_dims,
-        output_names=['output'],
+        output_names=["output"],
         expected_output_dims=expected_output_dims)
 
   def GetConversionParams(self, run_params):
@@ -107,5 +105,6 @@ class DynamicInputShapesTest(trt_test.TfTrtIntegrationTestBase):
     """The relative tolerance to compare floating point results."""
     return 1.e-03 if run_params.precision_mode == "FP32" else 1.e-01
 
-if __name__ == '__main__':
+
+if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/tensorrt/test/identity_output_test.py b/tensorflow/contrib/tensorrt/test/identity_output_test.py
index 196ccdbe9d..b568eeda94 100644
--- a/tensorflow/contrib/tensorrt/test/identity_output_test.py
+++ b/tensorflow/contrib/tensorrt/test/identity_output_test.py
@@ -61,9 +61,9 @@ class IdentityTest(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=['output1', 'output2', 'output3'],
-        expected_output_dims=[(100, 4), (100, 4), (100, 4)])
+        expected_output_dims=[[[100, 4]] * 3])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/int32_test.py b/tensorflow/contrib/tensorrt/test/int32_test.py
index 181229c8a7..8cf5387038 100644
--- a/tensorflow/contrib/tensorrt/test/int32_test.py
+++ b/tensorflow/contrib/tensorrt/test/int32_test.py
@@ -52,9 +52,9 @@ class ExcludeUnsupportedInt32Test(trt_test.TfTrtIntegrationTestBase):
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[(100, 10)])
+        expected_output_dims=[[[100, 10]]])
 
   def GetConversionParams(self, run_params):
     """Return a ConversionParams for test."""
diff --git a/tensorflow/contrib/tensorrt/test/lru_cache_test.py b/tensorflow/contrib/tensorrt/test/lru_cache_test.py
index 74b51932ac..7702413e6c 100644
--- a/tensorflow/contrib/tensorrt/test/lru_cache_test.py
+++ b/tensorflow/contrib/tensorrt/test/lru_cache_test.py
@@ -1,3 +1,19 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test LRUCache by running different input batch sizes on same network."""
+
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
@@ -17,29 +33,28 @@ from tensorflow.python.platform import test
 class LRUCacheTest(trt_test.TfTrtIntegrationTestBase):
 
   def GetParams(self):
-    """Test for running network multiple times with different batch size to
-    check that lru_cache works correctly"""
     dtype = dtypes.float32
     input_name = "input"
     input_dims = [[[1, 10, 10, 2]], [[2, 10, 10, 2]], [[4, 10, 10, 2]],
                   [[2, 10, 10, 2]]]
-    expected_output_dims = [[(1, 10, 10, 1)], [(2, 10, 10, 1)],
-                            [(4, 10, 10, 1)], [(2, 10, 10, 1)]]
+    expected_output_dims = [[[1, 10, 10, 1]], [[2, 10, 10, 1]], [[4, 10, 10,
+                                                                  1]],
+                            [[2, 10, 10, 1]]]
     output_name = "output"
     g = ops.Graph()
     with g.as_default():
-      x = array_ops.placeholder(dtype=dtype,
-                                shape=[None, 10, 10, 2],
-                                name=input_name)
-      conv_filter = constant_op.constant(np.random.randn(3, 3, 2, 1),
-                                         dtype=dtypes.float32)
-      x = nn.conv2d(input=x,
-                    filter=conv_filter,
-                    strides=[1, 1, 1, 1],
-                    padding="SAME",
-                    name="conv")
-      bias = constant_op.constant(np.random.randn(1, 10, 10, 1),
-                                  dtype=dtypes.float32)
+      x = array_ops.placeholder(
+          dtype=dtype, shape=[None, 10, 10, 2], name=input_name)
+      conv_filter = constant_op.constant(
+          np.random.randn(3, 3, 2, 1), dtype=dtypes.float32)
+      x = nn.conv2d(
+          input=x,
+          filter=conv_filter,
+          strides=[1, 1, 1, 1],
+          padding="SAME",
+          name="conv")
+      bias = constant_op.constant(
+          np.random.randn(1, 10, 10, 1), dtype=dtypes.float32)
       x = math_ops.add(x, bias)
       x = nn.relu(x)
       x = array_ops.identity(x, name="output")
@@ -58,5 +73,6 @@ class LRUCacheTest(trt_test.TfTrtIntegrationTestBase):
     return (run_params.dynamic_engine and
             not trt_test.IsQuantizationMode(run_params.precision_mode))
 
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/tensorrt/test/memory_alignment_test.py b/tensorflow/contrib/tensorrt/test/memory_alignment_test.py
index 6142e6d8e3..cc64329bbd 100644
--- a/tensorflow/contrib/tensorrt/test/memory_alignment_test.py
+++ b/tensorflow/contrib/tensorrt/test/memory_alignment_test.py
@@ -64,7 +64,7 @@ class MemoryAlignmentTest(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name],
         input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[[(2, 15, 15, 10)]])
+        expected_output_dims=[[[2, 15, 15, 10]]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py b/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py
index fcf89d3a74..a14bb0396e 100644
--- a/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py
+++ b/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py
@@ -77,7 +77,7 @@ class MultiConnectionNeighborEngineTest(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name],
         input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[[(2, 4, 5, 4)]])
+        expected_output_dims=[[[2, 4, 5, 4]]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/neighboring_engine_test.py b/tensorflow/contrib/tensorrt/test/neighboring_engine_test.py
index 840344aeba..06a86bbb8d 100644
--- a/tensorflow/contrib/tensorrt/test/neighboring_engine_test.py
+++ b/tensorflow/contrib/tensorrt/test/neighboring_engine_test.py
@@ -61,7 +61,7 @@ class NeighboringEngineTest(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name],
         input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[[(2, 4, 5, 4)]])
+        expected_output_dims=[[[2, 4, 5, 4]]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/quantization_test.py b/tensorflow/contrib/tensorrt/test/quantization_test.py
index fb09980552..ce1b25ebf3 100644
--- a/tensorflow/contrib/tensorrt/test/quantization_test.py
+++ b/tensorflow/contrib/tensorrt/test/quantization_test.py
@@ -62,7 +62,7 @@ def _GetParams(add_quantization_nodes, dtype=dtypes.float32):
       input_names=[input_name],
       input_dims=[[input_dims]],
       output_names=[output_name],
-      expected_output_dims=[[(8, 1)]])
+      expected_output_dims=[[[8, 1]]])
 
 
 class QuantizationMissingAllRangesTest(trt_test.TfTrtIntegrationTestBase):
diff --git a/tensorflow/contrib/tensorrt/test/rank_two_test.py b/tensorflow/contrib/tensorrt/test/rank_two_test.py
index 85ce89e151..97159bb008 100644
--- a/tensorflow/contrib/tensorrt/test/rank_two_test.py
+++ b/tensorflow/contrib/tensorrt/test/rank_two_test.py
@@ -65,7 +65,7 @@ class RankTwoTest(trt_test.TfTrtIntegrationTestBase):
         input_names=input_names,
         input_dims=[input_dims],
         output_names=[output_name],
-        expected_output_dims=[[tuple(input_dims[1])]])
+        expected_output_dims=[[input_dims[1]]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/reshape_transpose_test.py b/tensorflow/contrib/tensorrt/test/reshape_transpose_test.py
index 530c4992db..7fb2cbde07 100644
--- a/tensorflow/contrib/tensorrt/test/reshape_transpose_test.py
+++ b/tensorflow/contrib/tensorrt/test/reshape_transpose_test.py
@@ -74,7 +74,7 @@ class ReshapeTest(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name],
         input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[[tuple(input_dims)]])
+        expected_output_dims=[[input_dims]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
@@ -131,7 +131,7 @@ class TransposeTest(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name],
         input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[[(24, 100, 2, 24)]])
+        expected_output_dims=[[[24, 100, 2, 24]]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
index db6be3828c..f13c615cb1 100644
--- a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
+++ b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
@@ -39,17 +39,19 @@ from tensorflow.python.framework import test_util
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import tf_logging as logging
 
-TfTrtIntegrationTestParams = namedtuple("TfTrtIntegrationTestParams", [
-    "gdef",
-    # A list of names of the input placeholder nodes.
-    "input_names",
-    # A list of list of output shapes of the input placeholder nodes.
-    "input_dims",
-    # A list of names of the output identity nodes.
-    "output_names",
-    # A list of list of expected output shapes of the output identity nodes.
-    "expected_output_dims"
-])
+TfTrtIntegrationTestParams = namedtuple(
+    "TfTrtIntegrationTestParams",
+    [
+        "gdef",
+        # A list of names of the input placeholder nodes.
+        "input_names",
+        # A list of list of output shapes of the input placeholder nodes.
+        "input_dims",
+        # A list of names of the output identity nodes.
+        "output_names",
+        # A list of list of expected output shapes of the output identity nodes.
+        "expected_output_dims"
+    ])
 
 RunParams = namedtuple("RunParams", [
     "use_optimizer", "precision_mode", "dynamic_engine", "test_name",
@@ -167,10 +169,9 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
 
   def GetConversionParams(self, run_params):
     """Return a ConversionParams for test."""
-    batch_list
+    batch_list = []
     for dims_list in self._GetParamsCached().input_dims:
-      if not len(dims_list):
-        continue
+      assert dims_list
       # Each list of shapes should have same batch size.
       input_batches = [dims[0] for dims in dims_list]
       assert max(input_batches) == min(input_batches)
@@ -323,19 +324,20 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
       with self.test_session(
           graph=g, config=config, use_gpu=True, force_gpu=True) as sess:
         # Run for each input(s) shape
-        for ind in range(len(inputs_data)):
+        for shape_index in range(len(inputs_data)):
           val = None
           # Defaults to 2 runs to verify result across multiple runs is same.
           for _ in range(num_runs):
             self._PrepareRun(graph_state)
-            new_val = sess.run(
-                outputs,
-                {inputs[i]: inputs_data[ind][i] for i in range(len(inputs))})
-            output_len = len(params.expected_output_dims[ind])
+            new_val = sess.run(outputs, {
+                inputs[i]: inputs_data[shape_index][i]
+                for i in range(len(inputs))
+            })
+            output_len = len(params.expected_output_dims[shape_index])
             self.assertEqual(output_len, len(new_val))
             for i in range(output_len):
               self.assertEqual(
-                  list(params.expected_output_dims[ind][i]),
+                  list(params.expected_output_dims[shape_index][i]),
                   list(new_val[i].shape))
             if val is not None:
               self.assertAllClose(val, new_val, atol=1.e-06, rtol=1.e-06)
@@ -506,8 +508,8 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
       current_input_data = []
       for i in range(len(params.input_names)):
         dtype = input_dtypes[params.input_names[i]]
-        # Multiply the input by some constant to avoid all zeros input for integer
-        # types.
+        # Multiply the input by some constant to avoid all zeros input for
+        # integer types.
         scale = 10.0 if np.issubdtype(dtype, np.integer) else 1.0
         dims = inp[i]
         # TODO(laigd): add debug options. E.g. we can set the input data to be
diff --git a/tensorflow/contrib/tensorrt/test/topk_test.py b/tensorflow/contrib/tensorrt/test/topk_test.py
index 263ea0cfa1..633a51982b 100644
--- a/tensorflow/contrib/tensorrt/test/topk_test.py
+++ b/tensorflow/contrib/tensorrt/test/topk_test.py
@@ -38,16 +38,16 @@ class TopKTest(trt_test.TfTrtIntegrationTestBase):
     g = ops.Graph()
     with g.as_default():
       x = array_ops.placeholder(dtype=dtype, shape=input_dims, name=input_name)
-      k_tensor = constant_op.constant(k, dtype=tf.int32, name="Const")
+      k_tensor = constant_op.constant(k, dtype=dtypes.int32, name="Const")
       values, indices = nn_ops.top_k(x, k_tensor, name="TopK")
       values = array_ops.identity(values, name="output_values")
       indices = array_ops.identity(indices, name="output_indices")
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
         input_names=[input_name],
-        input_dims=[input_dims],
+        input_dims=[[input_dims]],
         output_names=["output_values", "output_indices"],
-        expected_output_dims=[(100, k), (100, k)])
+        expected_output_dims=[[[100, k], [100, k]]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/unary_test.py b/tensorflow/contrib/tensorrt/test/unary_test.py
index f630d695cf..497ea2848a 100644
--- a/tensorflow/contrib/tensorrt/test/unary_test.py
+++ b/tensorflow/contrib/tensorrt/test/unary_test.py
@@ -102,7 +102,7 @@ class UnaryTest(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name, input2_name],
         input_dims=[[input_dims, input2_dims]],
         output_names=[output_name],
-        expected_output_dims=[[(12, 5, 8, 12)]])
+        expected_output_dims=[[[12, 5, 8, 12]]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py b/tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py
index 1a4746e8d0..b5fed73d2d 100644
--- a/tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py
+++ b/tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py
@@ -72,7 +72,7 @@ class VGGBlockNCHWTest(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name],
         input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[[(5, 6, 2, 2)]])
+        expected_output_dims=[[[5, 6, 2, 2]]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
diff --git a/tensorflow/contrib/tensorrt/test/vgg_block_test.py b/tensorflow/contrib/tensorrt/test/vgg_block_test.py
index 051ced055a..307128f1a8 100644
--- a/tensorflow/contrib/tensorrt/test/vgg_block_test.py
+++ b/tensorflow/contrib/tensorrt/test/vgg_block_test.py
@@ -63,7 +63,7 @@ class VGGBlockTest(trt_test.TfTrtIntegrationTestBase):
         input_names=[input_name],
         input_dims=[[input_dims]],
         output_names=[output_name],
-        expected_output_dims=[[(5, 2, 2, 6)]])
+        expected_output_dims=[[[5, 2, 2, 6]]])
 
   def ExpectedEnginesToBuild(self, run_params):
     """Return the expected engines to build."""
-- 
GitLab


From f9b9cf52eb36a5d5e8bbf96fc1f2b6f584ce7867 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Tue, 15 Jan 2019 16:08:22 -0800
Subject: [PATCH 0780/1095] Checkpointable->AutoCheckpointable
 CheckpointableBase->Checkpointable

In preparation for adding public symbols.

PiperOrigin-RevId: 229459684
---
 tensorflow/contrib/checkpoint/__init__.py     |   4 +-
 .../contrib/checkpoint/python/containers.py   |   2 +-
 .../checkpoint/python/containers_test.py      |   4 +-
 .../contrib/checkpoint/python/python_state.py |   4 +-
 .../checkpoint/python/split_dependency.py     |   2 +-
 .../python/split_dependency_test.py           |   6 +-
 .../cudnn_rnn/python/ops/cudnn_rnn_ops.py     |   2 +-
 .../distribute/python/checkpointing_test.py   |   2 +-
 .../contrib/eager/python/metrics_impl.py      |   2 +-
 tensorflow/contrib/eager/python/tfe.py        |   2 +-
 .../optimizer_v2/checkpointable_utils_test.py |  10 +-
 tensorflow/python/data/ops/iterator_ops.py    |   4 +-
 tensorflow/python/distribute/values.py        |   8 +-
 .../feature_column/feature_column_v2.py       |   2 +-
 tensorflow/python/keras/engine/base_layer.py  |   2 +-
 tensorflow/python/keras/engine/network.py     |   2 +-
 tensorflow/python/keras/engine/training.py    |   2 +-
 tensorflow/python/keras/layers/recurrent.py   |   2 +-
 .../python/keras/optimizer_v2/optimizer_v2.py |   2 +-
 tensorflow/python/keras/optimizers.py         |   2 +-
 tensorflow/python/layers/base.py              |   2 +-
 tensorflow/python/ops/lookup_ops.py           |   4 +-
 tensorflow/python/ops/rnn_cell_impl.py        |   8 +-
 tensorflow/python/ops/stateful_random_ops.py  |   2 +-
 tensorflow/python/ops/template.py             |   2 +-
 tensorflow/python/ops/variables.py            |   2 +-
 tensorflow/python/saved_model/load.py         |   3 +-
 tensorflow/python/saved_model/load_test.py    |  58 +++++-----
 .../python/saved_model/revived_types_test.py  |   4 +-
 tensorflow/python/saved_model/save.py         |   2 +-
 tensorflow/python/saved_model/save_test.py    |  18 ++--
 .../python/training/checkpointable/base.py    |  12 +--
 .../training/checkpointable/base_test.py      |  16 +--
 .../checkpointable/data_structures.py         |  18 ++--
 .../checkpointable/data_structures_test.py    |  38 +++----
 .../training/checkpointable/tracking.py       |   8 +-
 .../training/checkpointable/tracking_test.py  |  46 ++++----
 .../python/training/checkpointable/util.py    |   4 +-
 .../training/checkpointable/util_test.py      | 100 +++++++++---------
 .../util_with_v1_optimizers_test.py           |  14 +--
 .../training/checkpointable/util_xla_test.py  |   2 +-
 tensorflow/python/training/optimizer.py       |   2 +-
 tensorflow/python/training/saver_test.py      |   6 +-
 .../training/saving/saveable_object_util.py   |   4 +-
 .../api/golden/v1/tensorflow.-variable.pbtxt  |   2 +-
 .../golden/v1/tensorflow.data.-iterator.pbtxt |   2 +-
 .../golden/v1/tensorflow.keras.-model.pbtxt   |   2 +-
 .../v1/tensorflow.keras.-sequential.pbtxt     |   2 +-
 ....experimental.-peephole-l-s-t-m-cell.pbtxt |   2 +-
 .../tensorflow.keras.layers.-activation.pbtxt |   2 +-
 ...eras.layers.-activity-regularization.pbtxt |   2 +-
 .../v1/tensorflow.keras.layers.-add.pbtxt     |   2 +-
 ...nsorflow.keras.layers.-alpha-dropout.pbtxt |   2 +-
 ...low.keras.layers.-average-pooling1-d.pbtxt |   2 +-
 ...low.keras.layers.-average-pooling2-d.pbtxt |   2 +-
 ...low.keras.layers.-average-pooling3-d.pbtxt |   2 +-
 .../v1/tensorflow.keras.layers.-average.pbtxt |   2 +-
 ...tensorflow.keras.layers.-avg-pool1-d.pbtxt |   2 +-
 ...tensorflow.keras.layers.-avg-pool2-d.pbtxt |   2 +-
 ...tensorflow.keras.layers.-avg-pool3-d.pbtxt |   2 +-
 ...ow.keras.layers.-batch-normalization.pbtxt |   2 +-
 ...nsorflow.keras.layers.-bidirectional.pbtxt |   2 +-
 ...tensorflow.keras.layers.-concatenate.pbtxt |   2 +-
 ...orflow.keras.layers.-conv-l-s-t-m2-d.pbtxt |   2 +-
 .../v1/tensorflow.keras.layers.-conv1-d.pbtxt |   2 +-
 ...flow.keras.layers.-conv2-d-transpose.pbtxt |   2 +-
 .../v1/tensorflow.keras.layers.-conv2-d.pbtxt |   2 +-
 ...flow.keras.layers.-conv3-d-transpose.pbtxt |   2 +-
 .../v1/tensorflow.keras.layers.-conv3-d.pbtxt |   2 +-
 ...sorflow.keras.layers.-convolution1-d.pbtxt |   2 +-
 ...ras.layers.-convolution2-d-transpose.pbtxt |   2 +-
 ...sorflow.keras.layers.-convolution2-d.pbtxt |   2 +-
 ...ras.layers.-convolution3-d-transpose.pbtxt |   2 +-
 ...sorflow.keras.layers.-convolution3-d.pbtxt |   2 +-
 ...tensorflow.keras.layers.-cropping1-d.pbtxt |   2 +-
 ...tensorflow.keras.layers.-cropping2-d.pbtxt |   2 +-
 ...tensorflow.keras.layers.-cropping3-d.pbtxt |   2 +-
 ...sorflow.keras.layers.-cu-d-n-n-g-r-u.pbtxt |   2 +-
 ...rflow.keras.layers.-cu-d-n-n-l-s-t-m.pbtxt |   2 +-
 .../v1/tensorflow.keras.layers.-dense.pbtxt   |   2 +-
 ...flow.keras.layers.-depthwise-conv2-d.pbtxt |   2 +-
 .../v1/tensorflow.keras.layers.-dot.pbtxt     |   2 +-
 .../v1/tensorflow.keras.layers.-dropout.pbtxt |   2 +-
 .../v1/tensorflow.keras.layers.-e-l-u.pbtxt   |   2 +-
 .../tensorflow.keras.layers.-embedding.pbtxt  |   2 +-
 .../v1/tensorflow.keras.layers.-flatten.pbtxt |   2 +-
 .../tensorflow.keras.layers.-g-r-u-cell.pbtxt |   2 +-
 .../v1/tensorflow.keras.layers.-g-r-u.pbtxt   |   2 +-
 ...rflow.keras.layers.-gaussian-dropout.pbtxt |   2 +-
 ...sorflow.keras.layers.-gaussian-noise.pbtxt |   2 +-
 ...as.layers.-global-average-pooling1-d.pbtxt |   2 +-
 ...as.layers.-global-average-pooling2-d.pbtxt |   2 +-
 ...as.layers.-global-average-pooling3-d.pbtxt |   2 +-
 ...low.keras.layers.-global-avg-pool1-d.pbtxt |   2 +-
 ...low.keras.layers.-global-avg-pool2-d.pbtxt |   2 +-
 ...low.keras.layers.-global-avg-pool3-d.pbtxt |   2 +-
 ...low.keras.layers.-global-max-pool1-d.pbtxt |   2 +-
 ...low.keras.layers.-global-max-pool2-d.pbtxt |   2 +-
 ...low.keras.layers.-global-max-pool3-d.pbtxt |   2 +-
 ....keras.layers.-global-max-pooling1-d.pbtxt |   2 +-
 ....keras.layers.-global-max-pooling2-d.pbtxt |   2 +-
 ....keras.layers.-global-max-pooling3-d.pbtxt |   2 +-
 ...tensorflow.keras.layers.-input-layer.pbtxt |   2 +-
 ...ensorflow.keras.layers.-l-s-t-m-cell.pbtxt |   2 +-
 .../v1/tensorflow.keras.layers.-l-s-t-m.pbtxt |   2 +-
 .../v1/tensorflow.keras.layers.-lambda.pbtxt  |   2 +-
 .../v1/tensorflow.keras.layers.-layer.pbtxt   |   2 +-
 ...ensorflow.keras.layers.-leaky-re-l-u.pbtxt |   2 +-
 ...w.keras.layers.-locally-connected1-d.pbtxt |   2 +-
 ...w.keras.layers.-locally-connected2-d.pbtxt |   2 +-
 .../v1/tensorflow.keras.layers.-masking.pbtxt |   2 +-
 ...tensorflow.keras.layers.-max-pool1-d.pbtxt |   2 +-
 ...tensorflow.keras.layers.-max-pool2-d.pbtxt |   2 +-
 ...tensorflow.keras.layers.-max-pool3-d.pbtxt |   2 +-
 ...sorflow.keras.layers.-max-pooling1-d.pbtxt |   2 +-
 ...sorflow.keras.layers.-max-pooling2-d.pbtxt |   2 +-
 ...sorflow.keras.layers.-max-pooling3-d.pbtxt |   2 +-
 .../v1/tensorflow.keras.layers.-maximum.pbtxt |   2 +-
 .../v1/tensorflow.keras.layers.-minimum.pbtxt |   2 +-
 .../tensorflow.keras.layers.-multiply.pbtxt   |   2 +-
 .../tensorflow.keras.layers.-p-re-l-u.pbtxt   |   2 +-
 .../v1/tensorflow.keras.layers.-permute.pbtxt |   2 +-
 .../v1/tensorflow.keras.layers.-r-n-n.pbtxt   |   2 +-
 .../v1/tensorflow.keras.layers.-re-l-u.pbtxt  |   2 +-
 ...nsorflow.keras.layers.-repeat-vector.pbtxt |   2 +-
 .../v1/tensorflow.keras.layers.-reshape.pbtxt |   2 +-
 ...flow.keras.layers.-separable-conv1-d.pbtxt |   2 +-
 ...flow.keras.layers.-separable-conv2-d.pbtxt |   2 +-
 ...ras.layers.-separable-convolution1-d.pbtxt |   2 +-
 ...ras.layers.-separable-convolution2-d.pbtxt |   2 +-
 ...flow.keras.layers.-simple-r-n-n-cell.pbtxt |   2 +-
 ...ensorflow.keras.layers.-simple-r-n-n.pbtxt |   2 +-
 .../v1/tensorflow.keras.layers.-softmax.pbtxt |   2 +-
 ...low.keras.layers.-spatial-dropout1-d.pbtxt |   2 +-
 ...low.keras.layers.-spatial-dropout2-d.pbtxt |   2 +-
 ...low.keras.layers.-spatial-dropout3-d.pbtxt |   2 +-
 ...ow.keras.layers.-stacked-r-n-n-cells.pbtxt |   2 +-
 .../tensorflow.keras.layers.-subtract.pbtxt   |   2 +-
 ...low.keras.layers.-thresholded-re-l-u.pbtxt |   2 +-
 ...rflow.keras.layers.-time-distributed.pbtxt |   2 +-
 ...sorflow.keras.layers.-up-sampling1-d.pbtxt |   2 +-
 ...sorflow.keras.layers.-up-sampling2-d.pbtxt |   2 +-
 ...sorflow.keras.layers.-up-sampling3-d.pbtxt |   2 +-
 .../v1/tensorflow.keras.layers.-wrapper.pbtxt |   2 +-
 ...orflow.keras.layers.-zero-padding1-d.pbtxt |   2 +-
 ...orflow.keras.layers.-zero-padding2-d.pbtxt |   2 +-
 ...orflow.keras.layers.-zero-padding3-d.pbtxt |   2 +-
 .../tensorflow.keras.metrics.-accuracy.pbtxt  |   2 +-
 ...rflow.keras.metrics.-binary-accuracy.pbtxt |   2 +-
 ....keras.metrics.-categorical-accuracy.pbtxt |   2 +-
 ...low.keras.metrics.-categorical-hinge.pbtxt |   2 +-
 ...flow.keras.metrics.-cosine-proximity.pbtxt |   2 +-
 ...rflow.keras.metrics.-false-negatives.pbtxt |   2 +-
 ...rflow.keras.metrics.-false-positives.pbtxt |   2 +-
 .../v1/tensorflow.keras.metrics.-hinge.pbtxt  |   2 +-
 ...w.keras.metrics.-mean-absolute-error.pbtxt |   2 +-
 ...rics.-mean-absolute-percentage-error.pbtxt |   2 +-
 ...ow.keras.metrics.-mean-squared-error.pbtxt |   2 +-
 ...rics.-mean-squared-logarithmic-error.pbtxt |   2 +-
 .../v1/tensorflow.keras.metrics.-mean.pbtxt   |   2 +-
 .../tensorflow.keras.metrics.-precision.pbtxt |   2 +-
 .../v1/tensorflow.keras.metrics.-recall.pbtxt |   2 +-
 ....metrics.-sensitivity-at-specificity.pbtxt |   2 +-
 ...metrics.-sparse-categorical-accuracy.pbtxt |   2 +-
 ....metrics.-specificity-at-sensitivity.pbtxt |   2 +-
 ...sorflow.keras.metrics.-squared-hinge.pbtxt |   2 +-
 ...orflow.keras.metrics.-true-negatives.pbtxt |   2 +-
 ...orflow.keras.metrics.-true-positives.pbtxt |   2 +-
 .../v1/tensorflow.keras.models.-model.pbtxt   |   2 +-
 .../tensorflow.keras.models.-sequential.pbtxt |   2 +-
 ...ensorflow.keras.optimizers.-adadelta.pbtxt |   2 +-
 ...tensorflow.keras.optimizers.-adagrad.pbtxt |   2 +-
 .../tensorflow.keras.optimizers.-adam.pbtxt   |   2 +-
 .../tensorflow.keras.optimizers.-adamax.pbtxt |   2 +-
 .../tensorflow.keras.optimizers.-nadam.pbtxt  |   2 +-
 ...nsorflow.keras.optimizers.-optimizer.pbtxt |   2 +-
 ...nsorflow.keras.optimizers.-r-m-sprop.pbtxt |   2 +-
 .../tensorflow.keras.optimizers.-s-g-d.pbtxt  |   2 +-
 ...ensorflow.layers.-average-pooling1-d.pbtxt |   2 +-
 ...ensorflow.layers.-average-pooling2-d.pbtxt |   2 +-
 ...ensorflow.layers.-average-pooling3-d.pbtxt |   2 +-
 ...nsorflow.layers.-batch-normalization.pbtxt |   2 +-
 .../v1/tensorflow.layers.-conv1-d.pbtxt       |   2 +-
 ...tensorflow.layers.-conv2-d-transpose.pbtxt |   2 +-
 .../v1/tensorflow.layers.-conv2-d.pbtxt       |   2 +-
 ...tensorflow.layers.-conv3-d-transpose.pbtxt |   2 +-
 .../v1/tensorflow.layers.-conv3-d.pbtxt       |   2 +-
 .../golden/v1/tensorflow.layers.-dense.pbtxt  |   2 +-
 .../v1/tensorflow.layers.-dropout.pbtxt       |   2 +-
 .../v1/tensorflow.layers.-flatten.pbtxt       |   2 +-
 .../golden/v1/tensorflow.layers.-layer.pbtxt  |   2 +-
 .../tensorflow.layers.-max-pooling1-d.pbtxt   |   2 +-
 .../tensorflow.layers.-max-pooling2-d.pbtxt   |   2 +-
 .../tensorflow.layers.-max-pooling3-d.pbtxt   |   2 +-
 ...tensorflow.layers.-separable-conv1-d.pbtxt |   2 +-
 ...tensorflow.layers.-separable-conv2-d.pbtxt |   2 +-
 ...flow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt |   2 +-
 ...orflow.nn.rnn_cell.-basic-r-n-n-cell.pbtxt |   2 +-
 ...nsorflow.nn.rnn_cell.-device-wrapper.pbtxt |   2 +-
 ...sorflow.nn.rnn_cell.-dropout-wrapper.pbtxt |   2 +-
 .../tensorflow.nn.rnn_cell.-g-r-u-cell.pbtxt  |   2 +-
 ...tensorflow.nn.rnn_cell.-l-s-t-m-cell.pbtxt |   2 +-
 ...orflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt |   2 +-
 .../tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt  |   2 +-
 ...orflow.nn.rnn_cell.-residual-wrapper.pbtxt |   2 +-
 ...tensorflow.train.-adadelta-optimizer.pbtxt |   2 +-
 ...sorflow.train.-adagrad-d-a-optimizer.pbtxt |   2 +-
 .../tensorflow.train.-adagrad-optimizer.pbtxt |   2 +-
 .../v1/tensorflow.train.-adam-optimizer.pbtxt |   2 +-
 .../v1/tensorflow.train.-checkpoint.pbtxt     |   4 +-
 .../v1/tensorflow.train.-ftrl-optimizer.pbtxt |   2 +-
 ...ow.train.-gradient-descent-optimizer.pbtxt |   2 +-
 ...tensorflow.train.-momentum-optimizer.pbtxt |   2 +-
 .../v1/tensorflow.train.-optimizer.pbtxt      |   2 +-
 ...ow.train.-proximal-adagrad-optimizer.pbtxt |   2 +-
 ...-proximal-gradient-descent-optimizer.pbtxt |   2 +-
 ...nsorflow.train.-r-m-s-prop-optimizer.pbtxt |   2 +-
 ...rflow.train.-sync-replicas-optimizer.pbtxt |   2 +-
 .../api/golden/v2/tensorflow.-variable.pbtxt  |   2 +-
 .../golden/v2/tensorflow.keras.-model.pbtxt   |   2 +-
 .../v2/tensorflow.keras.-sequential.pbtxt     |   2 +-
 ....experimental.-peephole-l-s-t-m-cell.pbtxt |   2 +-
 .../tensorflow.keras.layers.-activation.pbtxt |   2 +-
 ...eras.layers.-activity-regularization.pbtxt |   2 +-
 .../v2/tensorflow.keras.layers.-add.pbtxt     |   2 +-
 ...nsorflow.keras.layers.-alpha-dropout.pbtxt |   2 +-
 ...low.keras.layers.-average-pooling1-d.pbtxt |   2 +-
 ...low.keras.layers.-average-pooling2-d.pbtxt |   2 +-
 ...low.keras.layers.-average-pooling3-d.pbtxt |   2 +-
 .../v2/tensorflow.keras.layers.-average.pbtxt |   2 +-
 ...tensorflow.keras.layers.-avg-pool1-d.pbtxt |   2 +-
 ...tensorflow.keras.layers.-avg-pool2-d.pbtxt |   2 +-
 ...tensorflow.keras.layers.-avg-pool3-d.pbtxt |   2 +-
 ...ow.keras.layers.-batch-normalization.pbtxt |   2 +-
 ...nsorflow.keras.layers.-bidirectional.pbtxt |   2 +-
 ...tensorflow.keras.layers.-concatenate.pbtxt |   2 +-
 ...orflow.keras.layers.-conv-l-s-t-m2-d.pbtxt |   2 +-
 .../v2/tensorflow.keras.layers.-conv1-d.pbtxt |   2 +-
 ...flow.keras.layers.-conv2-d-transpose.pbtxt |   2 +-
 .../v2/tensorflow.keras.layers.-conv2-d.pbtxt |   2 +-
 ...flow.keras.layers.-conv3-d-transpose.pbtxt |   2 +-
 .../v2/tensorflow.keras.layers.-conv3-d.pbtxt |   2 +-
 ...sorflow.keras.layers.-convolution1-d.pbtxt |   2 +-
 ...ras.layers.-convolution2-d-transpose.pbtxt |   2 +-
 ...sorflow.keras.layers.-convolution2-d.pbtxt |   2 +-
 ...ras.layers.-convolution3-d-transpose.pbtxt |   2 +-
 ...sorflow.keras.layers.-convolution3-d.pbtxt |   2 +-
 ...tensorflow.keras.layers.-cropping1-d.pbtxt |   2 +-
 ...tensorflow.keras.layers.-cropping2-d.pbtxt |   2 +-
 ...tensorflow.keras.layers.-cropping3-d.pbtxt |   2 +-
 ...sorflow.keras.layers.-dense-features.pbtxt |   2 +-
 .../v2/tensorflow.keras.layers.-dense.pbtxt   |   2 +-
 ...flow.keras.layers.-depthwise-conv2-d.pbtxt |   2 +-
 .../v2/tensorflow.keras.layers.-dot.pbtxt     |   2 +-
 .../v2/tensorflow.keras.layers.-dropout.pbtxt |   2 +-
 .../v2/tensorflow.keras.layers.-e-l-u.pbtxt   |   2 +-
 .../tensorflow.keras.layers.-embedding.pbtxt  |   2 +-
 .../v2/tensorflow.keras.layers.-flatten.pbtxt |   2 +-
 .../tensorflow.keras.layers.-g-r-u-cell.pbtxt |   2 +-
 .../v2/tensorflow.keras.layers.-g-r-u.pbtxt   |   2 +-
 ...rflow.keras.layers.-gaussian-dropout.pbtxt |   2 +-
 ...sorflow.keras.layers.-gaussian-noise.pbtxt |   2 +-
 ...as.layers.-global-average-pooling1-d.pbtxt |   2 +-
 ...as.layers.-global-average-pooling2-d.pbtxt |   2 +-
 ...as.layers.-global-average-pooling3-d.pbtxt |   2 +-
 ...low.keras.layers.-global-avg-pool1-d.pbtxt |   2 +-
 ...low.keras.layers.-global-avg-pool2-d.pbtxt |   2 +-
 ...low.keras.layers.-global-avg-pool3-d.pbtxt |   2 +-
 ...low.keras.layers.-global-max-pool1-d.pbtxt |   2 +-
 ...low.keras.layers.-global-max-pool2-d.pbtxt |   2 +-
 ...low.keras.layers.-global-max-pool3-d.pbtxt |   2 +-
 ....keras.layers.-global-max-pooling1-d.pbtxt |   2 +-
 ....keras.layers.-global-max-pooling2-d.pbtxt |   2 +-
 ....keras.layers.-global-max-pooling3-d.pbtxt |   2 +-
 ...tensorflow.keras.layers.-input-layer.pbtxt |   2 +-
 ...ensorflow.keras.layers.-l-s-t-m-cell.pbtxt |   2 +-
 .../v2/tensorflow.keras.layers.-l-s-t-m.pbtxt |   2 +-
 .../v2/tensorflow.keras.layers.-lambda.pbtxt  |   2 +-
 .../v2/tensorflow.keras.layers.-layer.pbtxt   |   2 +-
 ...ensorflow.keras.layers.-leaky-re-l-u.pbtxt |   2 +-
 ...ensorflow.keras.layers.-linear-model.pbtxt |   2 +-
 ...w.keras.layers.-locally-connected1-d.pbtxt |   2 +-
 ...w.keras.layers.-locally-connected2-d.pbtxt |   2 +-
 .../v2/tensorflow.keras.layers.-masking.pbtxt |   2 +-
 ...tensorflow.keras.layers.-max-pool1-d.pbtxt |   2 +-
 ...tensorflow.keras.layers.-max-pool2-d.pbtxt |   2 +-
 ...tensorflow.keras.layers.-max-pool3-d.pbtxt |   2 +-
 ...sorflow.keras.layers.-max-pooling1-d.pbtxt |   2 +-
 ...sorflow.keras.layers.-max-pooling2-d.pbtxt |   2 +-
 ...sorflow.keras.layers.-max-pooling3-d.pbtxt |   2 +-
 .../v2/tensorflow.keras.layers.-maximum.pbtxt |   2 +-
 .../v2/tensorflow.keras.layers.-minimum.pbtxt |   2 +-
 .../tensorflow.keras.layers.-multiply.pbtxt   |   2 +-
 .../tensorflow.keras.layers.-p-re-l-u.pbtxt   |   2 +-
 .../v2/tensorflow.keras.layers.-permute.pbtxt |   2 +-
 .../v2/tensorflow.keras.layers.-r-n-n.pbtxt   |   2 +-
 .../v2/tensorflow.keras.layers.-re-l-u.pbtxt  |   2 +-
 ...nsorflow.keras.layers.-repeat-vector.pbtxt |   2 +-
 .../v2/tensorflow.keras.layers.-reshape.pbtxt |   2 +-
 ...flow.keras.layers.-separable-conv1-d.pbtxt |   2 +-
 ...flow.keras.layers.-separable-conv2-d.pbtxt |   2 +-
 ...ras.layers.-separable-convolution1-d.pbtxt |   2 +-
 ...ras.layers.-separable-convolution2-d.pbtxt |   2 +-
 ...flow.keras.layers.-simple-r-n-n-cell.pbtxt |   2 +-
 ...ensorflow.keras.layers.-simple-r-n-n.pbtxt |   2 +-
 .../v2/tensorflow.keras.layers.-softmax.pbtxt |   2 +-
 ...low.keras.layers.-spatial-dropout1-d.pbtxt |   2 +-
 ...low.keras.layers.-spatial-dropout2-d.pbtxt |   2 +-
 ...low.keras.layers.-spatial-dropout3-d.pbtxt |   2 +-
 ...ow.keras.layers.-stacked-r-n-n-cells.pbtxt |   2 +-
 .../tensorflow.keras.layers.-subtract.pbtxt   |   2 +-
 ...low.keras.layers.-thresholded-re-l-u.pbtxt |   2 +-
 ...rflow.keras.layers.-time-distributed.pbtxt |   2 +-
 ...sorflow.keras.layers.-up-sampling1-d.pbtxt |   2 +-
 ...sorflow.keras.layers.-up-sampling2-d.pbtxt |   2 +-
 ...sorflow.keras.layers.-up-sampling3-d.pbtxt |   2 +-
 .../v2/tensorflow.keras.layers.-wrapper.pbtxt |   2 +-
 ...orflow.keras.layers.-zero-padding1-d.pbtxt |   2 +-
 ...orflow.keras.layers.-zero-padding2-d.pbtxt |   2 +-
 ...orflow.keras.layers.-zero-padding3-d.pbtxt |   2 +-
 .../tensorflow.keras.metrics.-accuracy.pbtxt  |   2 +-
 ...rflow.keras.metrics.-binary-accuracy.pbtxt |   2 +-
 ....keras.metrics.-categorical-accuracy.pbtxt |   2 +-
 ...low.keras.metrics.-categorical-hinge.pbtxt |   2 +-
 ...flow.keras.metrics.-cosine-proximity.pbtxt |   2 +-
 ...rflow.keras.metrics.-false-negatives.pbtxt |   2 +-
 ...rflow.keras.metrics.-false-positives.pbtxt |   2 +-
 .../v2/tensorflow.keras.metrics.-hinge.pbtxt  |   2 +-
 ...w.keras.metrics.-mean-absolute-error.pbtxt |   2 +-
 ...rics.-mean-absolute-percentage-error.pbtxt |   2 +-
 ...ow.keras.metrics.-mean-squared-error.pbtxt |   2 +-
 ...rics.-mean-squared-logarithmic-error.pbtxt |   2 +-
 .../v2/tensorflow.keras.metrics.-mean.pbtxt   |   2 +-
 .../tensorflow.keras.metrics.-precision.pbtxt |   2 +-
 .../v2/tensorflow.keras.metrics.-recall.pbtxt |   2 +-
 ....metrics.-sensitivity-at-specificity.pbtxt |   2 +-
 ...metrics.-sparse-categorical-accuracy.pbtxt |   2 +-
 ....metrics.-specificity-at-sensitivity.pbtxt |   2 +-
 ...sorflow.keras.metrics.-squared-hinge.pbtxt |   2 +-
 ...orflow.keras.metrics.-true-negatives.pbtxt |   2 +-
 ...orflow.keras.metrics.-true-positives.pbtxt |   2 +-
 .../v2/tensorflow.keras.models.-model.pbtxt   |   2 +-
 .../tensorflow.keras.models.-sequential.pbtxt |   2 +-
 ...ensorflow.keras.optimizers.-adadelta.pbtxt |   2 +-
 ...tensorflow.keras.optimizers.-adagrad.pbtxt |   2 +-
 .../tensorflow.keras.optimizers.-adam.pbtxt   |   2 +-
 .../tensorflow.keras.optimizers.-adamax.pbtxt |   2 +-
 .../tensorflow.keras.optimizers.-nadam.pbtxt  |   2 +-
 ...nsorflow.keras.optimizers.-optimizer.pbtxt |   2 +-
 ...nsorflow.keras.optimizers.-r-m-sprop.pbtxt |   2 +-
 .../tensorflow.keras.optimizers.-s-g-d.pbtxt  |   2 +-
 ...nsorflow.nn.rnn_cell.-device-wrapper.pbtxt |   2 +-
 .../tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt  |   2 +-
 ...orflow.nn.rnn_cell.-residual-wrapper.pbtxt |   2 +-
 .../v2/tensorflow.rnn.-dropout-wrapper.pbtxt  |   2 +-
 .../v2/tensorflow.train.-checkpoint.pbtxt     |   4 +-
 ...-proximal-gradient-descent-optimizer.pbtxt |   2 +-
 357 files changed, 535 insertions(+), 536 deletions(-)

diff --git a/tensorflow/contrib/checkpoint/__init__.py b/tensorflow/contrib/checkpoint/__init__.py
index 94b7f4f867..99ed4959fa 100644
--- a/tensorflow/contrib/checkpoint/__init__.py
+++ b/tensorflow/contrib/checkpoint/__init__.py
@@ -51,11 +51,11 @@ from tensorflow.contrib.checkpoint.python.split_dependency import split_dependen
 from tensorflow.contrib.checkpoint.python.visualize import dot_graph_from_checkpoint
 from tensorflow.core.protobuf.checkpointable_object_graph_pb2 import CheckpointableObjectGraph
 from tensorflow.python.training.checkpoint_management import CheckpointManager
-from tensorflow.python.training.checkpointable.base import CheckpointableBase
+from tensorflow.python.training.checkpointable.base import Checkpointable as CheckpointableBase
 from tensorflow.python.training.checkpointable.data_structures import List
 from tensorflow.python.training.checkpointable.data_structures import Mapping
 from tensorflow.python.training.checkpointable.data_structures import NoDependency
-from tensorflow.python.training.checkpointable.tracking import Checkpointable
+from tensorflow.python.training.checkpointable.tracking import AutoCheckpointable as Checkpointable
 from tensorflow.python.training.checkpointable.util import capture_dependencies
 from tensorflow.python.training.checkpointable.util import list_objects
 from tensorflow.python.training.checkpointable.util import object_metadata
diff --git a/tensorflow/contrib/checkpoint/python/containers.py b/tensorflow/contrib/checkpoint/python/containers.py
index 5418e2605b..97936d9e9d 100644
--- a/tensorflow/contrib/checkpoint/python/containers.py
+++ b/tensorflow/contrib/checkpoint/python/containers.py
@@ -63,7 +63,7 @@ class UniqueNameTracker(data_structures.CheckpointableDataStructure):
       ValueError: If `checkpointable` is not a checkpointable object.
     """
 
-    if not isinstance(checkpointable, checkpointable_lib.CheckpointableBase):
+    if not isinstance(checkpointable, checkpointable_lib.Checkpointable):
       raise ValueError(
           ("Expected a checkpointable value, got %s which does not inherit "
            "from CheckpointableBase.") % (checkpointable,))
diff --git a/tensorflow/contrib/checkpoint/python/containers_test.py b/tensorflow/contrib/checkpoint/python/containers_test.py
index ac85c7be80..a2d453ec6e 100644
--- a/tensorflow/contrib/checkpoint/python/containers_test.py
+++ b/tensorflow/contrib/checkpoint/python/containers_test.py
@@ -52,7 +52,7 @@ class UniqueNameTrackerTests(test.TestCase):
     save_root = util.Checkpoint(slots=slots)
     save_path = save_root.save(checkpoint_prefix)
 
-    restore_slots = tracking.Checkpointable()
+    restore_slots = tracking.AutoCheckpointable()
     restore_root = util.Checkpoint(
         slots=restore_slots)
     status = restore_root.restore(save_path)
@@ -68,7 +68,7 @@ class UniqueNameTrackerTests(test.TestCase):
 
   @test_util.run_in_graph_and_eager_modes
   def testExample(self):
-    class SlotManager(tracking.Checkpointable):
+    class SlotManager(tracking.AutoCheckpointable):
 
       def __init__(self):
         self.slotdeps = containers.UniqueNameTracker()
diff --git a/tensorflow/contrib/checkpoint/python/python_state.py b/tensorflow/contrib/checkpoint/python/python_state.py
index 302d5cfb79..969c90c788 100644
--- a/tensorflow/contrib/checkpoint/python/python_state.py
+++ b/tensorflow/contrib/checkpoint/python/python_state.py
@@ -34,7 +34,7 @@ except ImportError:
 # pylint: enable=g-import-not-at-top
 
 
-class NumpyState(base.CheckpointableBase):
+class NumpyState(base.Checkpointable):
   """A checkpointable object whose NumPy array attributes are saved/restored.
 
   Example usage:
@@ -130,7 +130,7 @@ class NumpyState(base.CheckpointableBase):
 
 
 @six.add_metaclass(abc.ABCMeta)
-class PythonStateWrapper(base.CheckpointableBase):
+class PythonStateWrapper(base.Checkpointable):
   """Wraps a Python object for storage in an object-based checkpoint."""
 
   @abc.abstractmethod
diff --git a/tensorflow/contrib/checkpoint/python/split_dependency.py b/tensorflow/contrib/checkpoint/python/split_dependency.py
index 7e77453f3d..3e9700ad74 100644
--- a/tensorflow/contrib/checkpoint/python/split_dependency.py
+++ b/tensorflow/contrib/checkpoint/python/split_dependency.py
@@ -43,7 +43,7 @@ class _CallbackSaveable(saver_lib.BaseSaverBuilder.SaveableObject):
     return self._restore_callback(tensor)
 
 
-class _SplitDependency(checkpointable.CheckpointableBase):
+class _SplitDependency(checkpointable.Checkpointable):
   """Looks like a regular variable while synchronizing save/restores."""
 
   def __init__(self, save_buffer, restore_buffer, name, dtype, num_components,
diff --git a/tensorflow/contrib/checkpoint/python/split_dependency_test.py b/tensorflow/contrib/checkpoint/python/split_dependency_test.py
index 00a805af25..664a4e76ab 100644
--- a/tensorflow/contrib/checkpoint/python/split_dependency_test.py
+++ b/tensorflow/contrib/checkpoint/python/split_dependency_test.py
@@ -44,7 +44,7 @@ def _combine_variable_closure(variable):
   return _consume_restore_buffer_fn
 
 
-class SaveTensorSlicesAsDeps(base.CheckpointableBase):
+class SaveTensorSlicesAsDeps(base.Checkpointable):
 
   def __init__(self):
     self.combined = resource_variable_ops.ResourceVariable([0., 0., 0., 0.])
@@ -59,14 +59,14 @@ class SaveTensorSlicesAsDeps(base.CheckpointableBase):
       self._track_checkpointable(dep, name=name)
 
 
-class HasRegularDeps(tracking.Checkpointable):
+class HasRegularDeps(tracking.AutoCheckpointable):
 
   def __init__(self):
     self.first_half = resource_variable_ops.ResourceVariable([0., 0.])
     self.second_half = resource_variable_ops.ResourceVariable([0., 0.])
 
 
-class OnlyOneDep(tracking.Checkpointable):
+class OnlyOneDep(tracking.AutoCheckpointable):
 
   def __init__(self):
     self.first_half = resource_variable_ops.ResourceVariable([0., 0.])
diff --git a/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py b/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py
index 1facc83972..f36e8d5022 100644
--- a/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py
+++ b/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py
@@ -837,7 +837,7 @@ class CudnnLSTMSaveable(CudnnOpaqueParamsSaveable):
       checkpointable._track_checkpointable(bias, name="bias")  # pylint: disable=protected-access
     assert len(biases) == len(weights)
     for cell_index, (bias, kernel) in enumerate(zip(biases, weights)):
-      cell = checkpointable_lib.Checkpointable()
+      cell = checkpointable_lib.AutoCheckpointable()
       checkpointable._track_checkpointable(cell, name="cell-%d" % cell_index)  # pylint: disable=protected-access
       cell.bias = bias
       cell.kernel = kernel
diff --git a/tensorflow/contrib/distribute/python/checkpointing_test.py b/tensorflow/contrib/distribute/python/checkpointing_test.py
index f37026bc95..aa5b9f57b8 100644
--- a/tensorflow/contrib/distribute/python/checkpointing_test.py
+++ b/tensorflow/contrib/distribute/python/checkpointing_test.py
@@ -34,7 +34,7 @@ from tensorflow.python.training.checkpointable import tracking
 from tensorflow.python.training.checkpointable import util as checkpointable_utils
 
 
-class NonLayerCheckpointable(tracking.Checkpointable):
+class NonLayerCheckpointable(tracking.AutoCheckpointable):
 
   def __init__(self):
     super(NonLayerCheckpointable, self).__init__()
diff --git a/tensorflow/contrib/eager/python/metrics_impl.py b/tensorflow/contrib/eager/python/metrics_impl.py
index 566246de49..c8d9266672 100644
--- a/tensorflow/contrib/eager/python/metrics_impl.py
+++ b/tensorflow/contrib/eager/python/metrics_impl.py
@@ -37,7 +37,7 @@ from tensorflow.python.training.checkpointable import base as checkpointable
 _to_replace = re.compile("[^A-Za-z0-9.]")
 
 
-class Metric(checkpointable.CheckpointableBase):
+class Metric(checkpointable.Checkpointable):
   """A metric holds state for aggregating statistics over an evaluation run.
 
   Example use with eager execution:
diff --git a/tensorflow/contrib/eager/python/tfe.py b/tensorflow/contrib/eager/python/tfe.py
index 31481d7685..b82e1bb71b 100644
--- a/tensorflow/contrib/eager/python/tfe.py
+++ b/tensorflow/contrib/eager/python/tfe.py
@@ -138,7 +138,7 @@ from tensorflow.python.ops.resource_variable_ops import ResourceVariable as Vari
 from tensorflow.python.ops.variable_scope import EagerVariableStore
 from tensorflow.python.ops import script_ops
 from tensorflow.python.ops import template
-from tensorflow.python.training.checkpointable.tracking import Checkpointable
+from tensorflow.python.training.checkpointable.tracking import AutoCheckpointable as Checkpointable
 from tensorflow.python.training.checkpointable.util import CheckpointableSaver
 from tensorflow.python.training.checkpointable.util import Checkpoint
 from tensorflow.python.util.all_util import remove_undocumented
diff --git a/tensorflow/contrib/optimizer_v2/checkpointable_utils_test.py b/tensorflow/contrib/optimizer_v2/checkpointable_utils_test.py
index 72019b3154..0243927ce4 100644
--- a/tensorflow/contrib/optimizer_v2/checkpointable_utils_test.py
+++ b/tensorflow/contrib/optimizer_v2/checkpointable_utils_test.py
@@ -48,7 +48,7 @@ from tensorflow.python.training.checkpointable import tracking
 from tensorflow.python.training.checkpointable import util
 
 
-class NonLayerCheckpointable(tracking.Checkpointable):
+class NonLayerCheckpointable(tracking.AutoCheckpointable):
 
   def __init__(self):
     super(NonLayerCheckpointable, self).__init__()
@@ -440,7 +440,7 @@ class CheckpointingTests(test.TestCase):
   def testDeferredSlotRestoration(self):
     checkpoint_directory = self.get_temp_dir()
 
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.var = util.add_variable(
         root, name="var", initializer=0.)
     optimizer = adam.AdamOptimizer(0.1)
@@ -463,7 +463,7 @@ class CheckpointingTests(test.TestCase):
                                    14.))
     slots_path = util.CheckpointableSaver(root).save(
         os.path.join(checkpoint_directory, "with_slots"))
-    new_root = tracking.Checkpointable()
+    new_root = tracking.AutoCheckpointable()
     # Load the slot-containing checkpoint (deferred), then immediately overwrite
     # the non-slot variable (also deferred).
     slot_status = util.CheckpointableSaver(
@@ -508,7 +508,7 @@ class CheckpointingTests(test.TestCase):
       with graph.as_default(), self.session(graph):
         checkpoint_directory = self.get_temp_dir()
         checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
-        obj = tracking.Checkpointable()
+        obj = tracking.AutoCheckpointable()
         obj.var = variable_scope.get_variable(name="v", initializer=0.)
         obj.opt = adam.AdamOptimizer(0.1)
         obj.opt.minimize(obj.var.read_value())
@@ -526,7 +526,7 @@ class CheckpointingTests(test.TestCase):
       with graph.as_default(), self.session(graph):
         checkpoint_directory = self.get_temp_dir()
         checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
-        obj = tracking.Checkpointable()
+        obj = tracking.AutoCheckpointable()
         obj.var = variable_scope.get_variable(name="v", initializer=0.)
         obj.opt = adam.AdamOptimizer(0.1)
         obj.opt.minimize(obj.var.read_value())
diff --git a/tensorflow/python/data/ops/iterator_ops.py b/tensorflow/python/data/ops/iterator_ops.py
index bfa256f8d7..d6fb73813c 100644
--- a/tensorflow/python/data/ops/iterator_ops.py
+++ b/tensorflow/python/data/ops/iterator_ops.py
@@ -68,7 +68,7 @@ def _device_stack_is_empty():
 
 
 @tf_export(v1=["data.Iterator"])
-class Iterator(checkpointable.CheckpointableBase):
+class Iterator(checkpointable.Checkpointable):
   """Represents the state of iterating through a `Dataset`."""
 
   def __init__(self, iterator_resource, initializer, output_types,
@@ -491,7 +491,7 @@ def _generate_shared_name(prefix):
   return "{}{}".format(prefix, uid)
 
 
-class EagerIterator(checkpointable.CheckpointableBase):
+class EagerIterator(checkpointable.Checkpointable):
   """An iterator producing tf.Tensor objects from a tf.data.Dataset."""
 
   def __init__(self, dataset):
diff --git a/tensorflow/python/distribute/values.py b/tensorflow/python/distribute/values.py
index ec0ea4bc2d..e7db285be5 100644
--- a/tensorflow/python/distribute/values.py
+++ b/tensorflow/python/distribute/values.py
@@ -626,7 +626,7 @@ class _MirroredSaveable(saver.BaseSaverBuilder.ResourceVariableSaveable):
 
 
 class MirroredVariable(DistributedVariable, Mirrored,
-                       checkpointable.CheckpointableBase):
+                       checkpointable.Checkpointable):
   """Holds a map from device to variables whose values are kept in sync."""
 
   def __init__(
@@ -748,7 +748,7 @@ def _enclosing_tpu_context():
 # tpu.replicate() because it assumes that you're in a device context where you
 # can operate on a single version of the variable, but a tpu.replicate()
 # operates on all variables and is replicated during a rewrite pass.
-class TPUMirroredVariable(checkpointable.CheckpointableBase):
+class TPUMirroredVariable(checkpointable.Checkpointable):
   """Holds a map from device to TPU variables whose values are kept in sync."""
 
   def __init__(
@@ -1201,7 +1201,7 @@ def _assert_replica_context(strategy):
 
 
 class ReplicaLocalVariable(DistributedVariable, PerReplica,
-                           checkpointable.CheckpointableBase):
+                           checkpointable.Checkpointable):
   """Holds a map from device to variables whose values are reduced on save."""
 
   def __init__(
@@ -1432,7 +1432,7 @@ def value_container(val):
 
 
 # TODO(josh11b): Descend from Variable.
-class AggregatingVariable(checkpointable.CheckpointableBase):
+class AggregatingVariable(checkpointable.Checkpointable):
   """A wrapper around a variable that aggregates updates across replicas."""
 
   def __init__(self, strategy, v, aggregation):
diff --git a/tensorflow/python/feature_column/feature_column_v2.py b/tensorflow/python/feature_column/feature_column_v2.py
index 73e3001fc0..5976f647af 100644
--- a/tensorflow/python/feature_column/feature_column_v2.py
+++ b/tensorflow/python/feature_column/feature_column_v2.py
@@ -3175,7 +3175,7 @@ def _raise_shared_embedding_column_error():
                    '`DenseFeatures` or `LinearModel` instead.')
 
 
-class SharedEmbeddingColumnCreator(tracking.Checkpointable):
+class SharedEmbeddingColumnCreator(tracking.AutoCheckpointable):
 
   def __init__(self,
                dimension,
diff --git a/tensorflow/python/keras/engine/base_layer.py b/tensorflow/python/keras/engine/base_layer.py
index 47900e7533..5a3540d12d 100644
--- a/tensorflow/python/keras/engine/base_layer.py
+++ b/tensorflow/python/keras/engine/base_layer.py
@@ -57,7 +57,7 @@ from tensorflow.tools.docs import doc_controls
 
 
 @keras_export('keras.layers.Layer')
-class Layer(checkpointable.CheckpointableBase):
+class Layer(checkpointable.Checkpointable):
   """Base layer class.
 
   This is the class from which all layers inherit.
diff --git a/tensorflow/python/keras/engine/network.py b/tensorflow/python/keras/engine/network.py
index cca9874e8b..41f5f319bc 100644
--- a/tensorflow/python/keras/engine/network.py
+++ b/tensorflow/python/keras/engine/network.py
@@ -1434,7 +1434,7 @@ class Network(base_layer.Layer):
         session = backend.get_session()
       optimizer = getattr(self, 'optimizer', None)
       if (optimizer
-          and not isinstance(optimizer, checkpointable.CheckpointableBase)):
+          and not isinstance(optimizer, checkpointable.Checkpointable)):
         logging.warning(
             ('This model was compiled with a Keras optimizer (%s) but is being '
              'saved in TensorFlow format with `save_weights`. The model\'s '
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index d690df2986..f5a720b292 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -261,7 +261,7 @@ class Model(Network):
     self.optimizer = optimizer
     # We've disabled automatic dependency tracking for this method, but do want
     # to add a checkpoint dependency on the optimizer if it's checkpointable.
-    if isinstance(self.optimizer, checkpointable.CheckpointableBase):
+    if isinstance(self.optimizer, checkpointable.Checkpointable):
       self._track_checkpointable(
           self.optimizer, name='optimizer', overwrite=True)
     self.loss = loss
diff --git a/tensorflow/python/keras/layers/recurrent.py b/tensorflow/python/keras/layers/recurrent.py
index 797278a5e5..9404543ed9 100644
--- a/tensorflow/python/keras/layers/recurrent.py
+++ b/tensorflow/python/keras/layers/recurrent.py
@@ -468,7 +468,7 @@ class RNN(Layer):
     self.zero_output_for_mask = kwargs.pop('zero_output_for_mask', False)
     super(RNN, self).__init__(**kwargs)
     self.cell = cell
-    if isinstance(cell, checkpointable.CheckpointableBase):
+    if isinstance(cell, checkpointable.Checkpointable):
       self._track_checkpointable(self.cell, name='cell')
     self.return_sequences = return_sequences
     self.return_state = return_state
diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
index bc210132b5..6cd6cf0a8d 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py
@@ -70,7 +70,7 @@ def _deduplicate_indexed_slices(values, indices):
 
 @six.add_metaclass(abc.ABCMeta)
 @keras_export("keras.optimizers.Optimizer")
-class OptimizerV2(checkpointable.CheckpointableBase):
+class OptimizerV2(checkpointable.Checkpointable):
   """Updated base class for optimizers.
 
   This class defines the API to add Ops to train a model.  You never use this
diff --git a/tensorflow/python/keras/optimizers.py b/tensorflow/python/keras/optimizers.py
index 8c3bf110d4..b704b885cb 100644
--- a/tensorflow/python/keras/optimizers.py
+++ b/tensorflow/python/keras/optimizers.py
@@ -710,7 +710,7 @@ class Nadam(Optimizer):
     return dict(list(base_config.items()) + list(config.items()))
 
 
-class TFOptimizer(Optimizer, checkpointable.CheckpointableBase):
+class TFOptimizer(Optimizer, checkpointable.Checkpointable):
   """Wrapper class for native TensorFlow optimizers.
   """
 
diff --git a/tensorflow/python/layers/base.py b/tensorflow/python/layers/base.py
index 5354d437b4..1b84ec1f69 100644
--- a/tensorflow/python/layers/base.py
+++ b/tensorflow/python/layers/base.py
@@ -554,7 +554,7 @@ class Layer(base_layer.Layer):
 
   def __setattr__(self, value, name):
     # By-pass the automatic dependency tracking performed by the parent Layer.
-    super(checkpointable.CheckpointableBase, self).__setattr__(value, name)
+    super(checkpointable.Checkpointable, self).__setattr__(value, name)
 
 
 def _add_elements_to_collection(elements, collection_list):
diff --git a/tensorflow/python/ops/lookup_ops.py b/tensorflow/python/ops/lookup_ops.py
index 0cc3ea5ec7..6a7057fafd 100644
--- a/tensorflow/python/ops/lookup_ops.py
+++ b/tensorflow/python/ops/lookup_ops.py
@@ -164,7 +164,7 @@ class InitializableLookupTableBase(LookupInterface):
     self._default_value = ops.convert_to_tensor(
         default_value, dtype=self._value_dtype)
     self._default_value.get_shape().merge_with(tensor_shape.scalar())
-    if isinstance(initializer, checkpointable_base.CheckpointableBase):
+    if isinstance(initializer, checkpointable_base.Checkpointable):
       self._initializer = self._track_checkpointable(
           initializer, "_initializer")
     self._resource_handle = self.create_resource()
@@ -315,7 +315,7 @@ class HashTable(InitializableLookupTableBase):
     return exported_keys, exported_values
 
 
-class TableInitializerBase(checkpointable_base.CheckpointableBase):
+class TableInitializerBase(checkpointable_base.Checkpointable):
   """Base class for lookup table initializers."""
 
   def __init__(self, key_dtype, value_dtype):
diff --git a/tensorflow/python/ops/rnn_cell_impl.py b/tensorflow/python/ops/rnn_cell_impl.py
index 2da66cfcd5..40c3771f4e 100644
--- a/tensorflow/python/ops/rnn_cell_impl.py
+++ b/tensorflow/python/ops/rnn_cell_impl.py
@@ -1183,7 +1183,7 @@ class DropoutWrapper(RNNCell):
 
     # Set cell, variational_recurrent, seed before running the code below
     self._cell = cell
-    if isinstance(cell, checkpointable.CheckpointableBase):
+    if isinstance(cell, checkpointable.Checkpointable):
       self._track_checkpointable(self._cell, name="cell")
     self._variational_recurrent = variational_recurrent
     self._seed = seed
@@ -1424,7 +1424,7 @@ class ResidualWrapper(RNNCell):
     """
     super(ResidualWrapper, self).__init__()
     self._cell = cell
-    if isinstance(cell, checkpointable.CheckpointableBase):
+    if isinstance(cell, checkpointable.Checkpointable):
       self._track_checkpointable(self._cell, name="cell")
     self._residual_fn = residual_fn
 
@@ -1482,7 +1482,7 @@ class DeviceWrapper(RNNCell):
     """
     super(DeviceWrapper, self).__init__()
     self._cell = cell
-    if isinstance(cell, checkpointable.CheckpointableBase):
+    if isinstance(cell, checkpointable.Checkpointable):
       self._track_checkpointable(self._cell, name="cell")
     self._device = device
 
@@ -1551,7 +1551,7 @@ class MultiRNNCell(RNNCell):
     for cell_number, cell in enumerate(self._cells):
       # Add Checkpointable dependencies on these cells so their variables get
       # saved with this object when using object-based saving.
-      if isinstance(cell, checkpointable.CheckpointableBase):
+      if isinstance(cell, checkpointable.Checkpointable):
         # TODO(allenl): Track down non-Checkpointable callers.
         self._track_checkpointable(cell, name="cell-%d" % (cell_number,))
     self._state_is_tuple = state_is_tuple
diff --git a/tensorflow/python/ops/stateful_random_ops.py b/tensorflow/python/ops/stateful_random_ops.py
index c7a14f2015..155ad969f6 100644
--- a/tensorflow/python/ops/stateful_random_ops.py
+++ b/tensorflow/python/ops/stateful_random_ops.py
@@ -151,7 +151,7 @@ def _shape_tensor(shape):
 
 
 @tf_export("random.experimental.Generator")
-class Generator(tracking.Checkpointable):
+class Generator(tracking.AutoCheckpointable):
   """Random-number generator.
 
   It uses Variable to manage its internal state.
diff --git a/tensorflow/python/ops/template.py b/tensorflow/python/ops/template.py
index 4eaa16a22c..e02175d6fe 100644
--- a/tensorflow/python/ops/template.py
+++ b/tensorflow/python/ops/template.py
@@ -232,7 +232,7 @@ def _skip_common_stack_elements(stacktrace, base_case):
   return stacktrace[-1:]
 
 
-class Template(checkpointable.CheckpointableBase):
+class Template(checkpointable.Checkpointable):
   """Wrap a function to aid in variable sharing.
 
   Templates are functions that create variables the first time they are called
diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py
index d765936399..7eb2e14b42 100644
--- a/tensorflow/python/ops/variables.py
+++ b/tensorflow/python/ops/variables.py
@@ -204,7 +204,7 @@ class VariableMetaclass(type):
 
 @tf_export("Variable", v1=[])
 class Variable(six.with_metaclass(VariableMetaclass,
-                                  checkpointable.CheckpointableBase)):
+                                  checkpointable.Checkpointable)):
   """See the [Variables Guide](https://tensorflow.org/guide/variables).
 
   A variable maintains state in the graph across calls to `run()`. You add a
diff --git a/tensorflow/python/saved_model/load.py b/tensorflow/python/saved_model/load.py
index 93a081b62b..134373906d 100644
--- a/tensorflow/python/saved_model/load.py
+++ b/tensorflow/python/saved_model/load.py
@@ -150,7 +150,7 @@ class _Loader(object):
       # individually callable by adding a `__call__` method to the classes of
       # the objects instances that have a `__call__` property.
 
-      class _UserObject(tracking.Checkpointable):
+      class _UserObject(tracking.AutoCheckpointable):
         pass
 
       return _UserObject(), setattr
@@ -200,4 +200,3 @@ def load(export_dir):
         "Currently only SavedModels exported with `tf.saved_model.save` may be "
         "imported. Other SavedModels may eventually be supported via load().")
   return root
-
diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index ff6b72053b..9eea170bf0 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -43,17 +43,17 @@ class LoadTest(test.TestCase):
     return load.load(path)
 
   def test_structure_import(self):
-    root = tracking.Checkpointable()
-    root.dep_one = tracking.Checkpointable()
-    root.dep_two = tracking.Checkpointable()
-    root.dep_two.dep = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
+    root.dep_one = tracking.AutoCheckpointable()
+    root.dep_two = tracking.AutoCheckpointable()
+    root.dep_two.dep = tracking.AutoCheckpointable()
     root.dep_three = root.dep_two.dep
     imported = self.cycle(root)
     self.assertIs(imported.dep_three, imported.dep_two.dep)
     self.assertIsNot(imported.dep_one, imported.dep_two)
 
   def test_variables(self):
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.v1 = variables.Variable(1., trainable=True)
     root.v2 = variables.Variable(2., trainable=False)
     imported = self.cycle(root)
@@ -63,7 +63,7 @@ class LoadTest(test.TestCase):
     self.assertFalse(imported.v2.trainable)
 
   def test_capture_variables(self):
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.weights = variables.Variable(2.)
     root.f = def_function.function(
         lambda x: root.weights * x,
@@ -83,7 +83,7 @@ class LoadTest(test.TestCase):
     file1 = self._make_asset("contents 1")
     file2 = self._make_asset("contents 2")
 
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.asset1 = tracking.TrackableAsset(file1)
     root.asset2 = tracking.TrackableAsset(file2)
 
@@ -102,7 +102,7 @@ class LoadTest(test.TestCase):
       self.assertEquals("contents 2", f.read())
 
   def test_capture_assets(self):
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.vocab = tracking.TrackableAsset(self._make_asset("contents"))
     root.f = def_function.function(
         lambda: root.vocab.asset_path,
@@ -116,7 +116,7 @@ class LoadTest(test.TestCase):
 
   def test_dedup_assets(self):
     vocab = self._make_asset("contents")
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.asset1 = tracking.TrackableAsset(vocab)
     root.asset2 = tracking.TrackableAsset(vocab)
     imported = self.cycle(root)
@@ -128,7 +128,7 @@ class LoadTest(test.TestCase):
     def func(x):
       return 2 * x
 
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.f = func
 
     # Add two traces.
@@ -146,7 +146,7 @@ class LoadTest(test.TestCase):
     def func(x):
       return 2 * x
 
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.f = func
 
     imported = self.cycle(root)
@@ -157,7 +157,7 @@ class LoadTest(test.TestCase):
     def func(x):
       return 2 * x
 
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.f = func
 
     imported = self.cycle(
@@ -173,7 +173,7 @@ class LoadTest(test.TestCase):
         lambda x: f(x) + 1.0,
         input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)])
 
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.g = g
     imported = self.cycle(root)
     imported.g(constant_op.constant([1.0]))
@@ -186,7 +186,7 @@ class LoadTest(test.TestCase):
       else:
         return 7
 
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.f = def_function.function(func)
 
     self.assertEqual(20, root.f(constant_op.constant(10), True).numpy())
@@ -208,7 +208,7 @@ class LoadTest(test.TestCase):
       else:
         return 7
 
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.f = def_function.function(func)
 
     x = constant_op.constant(10)
@@ -240,7 +240,7 @@ class LoadTest(test.TestCase):
       named_tuple = named_tuple_type(a=input1 + input2, b=input1 * input2)
       return [named_tuple, input2, {"x": 0.5}]
 
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.f = def_function.function(func)
 
     result = root.f(constant_op.constant(2), constant_op.constant(3))
@@ -270,7 +270,7 @@ class LoadTest(test.TestCase):
       else:
         return 7
 
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.f = def_function.function(func)
 
     self.assertEqual(20, root.f(constant_op.constant(10), True).numpy())
@@ -285,7 +285,7 @@ class LoadTest(test.TestCase):
     self.assertEqual(6, imported.f(constant_op.constant(1), defg=7.0).numpy())
 
   def test_member_function(self):
-    class CheckpointableWithMember(tracking.Checkpointable):
+    class CheckpointableWithMember(tracking.AutoCheckpointable):
 
       def __init__(self):
         super(CheckpointableWithMember, self).__init__()
@@ -310,7 +310,7 @@ class LoadTest(test.TestCase):
     self.assertEqual(27, imported.f(constant_op.constant(2)).numpy())
 
   def test_side_effect_listing(self):
-    class M(tracking.Checkpointable):
+    class M(tracking.AutoCheckpointable):
 
       def __init__(self):
         super(M, self).__init__()
@@ -334,7 +334,7 @@ class LoadTest(test.TestCase):
         lambda x: x*weight + bias,
         input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)])
 
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.weight = weight
     root.bias = bias
     root.g = g
@@ -346,16 +346,16 @@ class LoadTest(test.TestCase):
       self.assertAllClose(grad, [3.5, 1.0])
 
   def test_callable(self):
-    class M1(tracking.Checkpointable):
+    class M1(tracking.AutoCheckpointable):
 
       @def_function.function(
           input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)])
       def __call__(self, x):
         return x
 
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.m1 = M1()
-    root.m2 = tracking.Checkpointable()
+    root.m2 = tracking.AutoCheckpointable()
     root.m2.__call__ = def_function.function(
         input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)])(
             lambda x: x*3.0)
@@ -378,9 +378,9 @@ class LoadTest(test.TestCase):
     func = def_function.function(
         input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)])(
             lambda x: x*3.0)
-    root = tracking.Checkpointable()
-    root.__call__ = tracking.Checkpointable()
-    root.__call__.__call__ = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
+    root.__call__ = tracking.AutoCheckpointable()
+    root.__call__.__call__ = tracking.AutoCheckpointable()
     root.__call__.__call__.__call__ = func
 
     imported = self.cycle(root)
@@ -395,7 +395,7 @@ class LoadTest(test.TestCase):
     def func(x):
       return 2 * x
 
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.f = func
 
     self.assertAllEqual([2], root.f(constant_op.constant([1])).numpy())
@@ -419,7 +419,7 @@ class LoadTest(test.TestCase):
                         imported.f(constant_op.constant([1, 2, 3])).numpy())
 
   def test_dict(self):
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.variables = dict(a=variables.Variable(1.))
     root.variables["b"] = variables.Variable(2.)
     root.variables["c"] = 1
@@ -429,7 +429,7 @@ class LoadTest(test.TestCase):
     self.assertEqual(set(["a", "b"]), set(imported.variables.keys()))
 
   def test_list(self):
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.variables = [variables.Variable(1.)]
     root.variables.append(1)
     root.variables.append(variables.Variable(3.))
diff --git a/tensorflow/python/saved_model/revived_types_test.py b/tensorflow/python/saved_model/revived_types_test.py
index 98e10cfec0..ede5922b80 100644
--- a/tensorflow/python/saved_model/revived_types_test.py
+++ b/tensorflow/python/saved_model/revived_types_test.py
@@ -25,7 +25,7 @@ from tensorflow.python.saved_model import saved_object_graph_pb2
 from tensorflow.python.training.checkpointable import tracking
 
 
-class CustomTestClass(tracking.Checkpointable):
+class CustomTestClass(tracking.AutoCheckpointable):
 
   def __init__(self, version):
     self.version = version
@@ -56,7 +56,7 @@ revived_types.register_revived_type(
 class RegistrationMatchingTest(test.TestCase):
 
   def test_save_typecheck(self):
-    self.assertIs(revived_types.serialize(tracking.Checkpointable()), None)
+    self.assertIs(revived_types.serialize(tracking.AutoCheckpointable()), None)
 
   def test_load_identifier_not_found(self):
     nothing_matches = revived_types.deserialize(
diff --git a/tensorflow/python/saved_model/save.py b/tensorflow/python/saved_model/save.py
index 207dc2cb74..d383bbecfc 100644
--- a/tensorflow/python/saved_model/save.py
+++ b/tensorflow/python/saved_model/save.py
@@ -796,7 +796,7 @@ def save(obj, export_dir, signatures=None):
             "tf.enable_eager_execution() must run first when calling it from "
             "TensorFlow 1.x.")
   # pylint: enable=line-too-long
-  if not isinstance(obj, base.CheckpointableBase):
+  if not isinstance(obj, base.Checkpointable):
     raise ValueError(
         "Expected a Checkpointable object for export, got {}.".format(obj))
 
diff --git a/tensorflow/python/saved_model/save_test.py b/tensorflow/python/saved_model/save_test.py
index 533b954190..5f9dbe2c25 100644
--- a/tensorflow/python/saved_model/save_test.py
+++ b/tensorflow/python/saved_model/save_test.py
@@ -87,7 +87,7 @@ def _import_and_infer(
 class SaveTest(test.TestCase):
 
   def test_method_save_signature(self):
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.f = def_function.function(
         lambda x: 2. * x,
         input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)])
@@ -99,7 +99,7 @@ class SaveTest(test.TestCase):
         _import_and_infer(save_dir, {"x": 1.}))
 
   def test_method_save_concrete(self):
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.f = def_function.function(
         lambda z: {"out": 2. * z})
     root.f(constant_op.constant(1.))
@@ -115,7 +115,7 @@ class SaveTest(test.TestCase):
             save_dir, {"z": 1.}, signature_key="non_default_key"))
 
   def test_non_concrete_error(self):
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.f = def_function.function(lambda x: 2. * x)
     root.f(constant_op.constant(1.))
     save_dir = os.path.join(self.get_temp_dir(), "saved_model")
@@ -124,7 +124,7 @@ class SaveTest(test.TestCase):
       save.save(root, save_dir, root.f)
 
   def test_nested_inputs(self):
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.f = def_function.function(
         lambda x: 2. * x[0],
         input_signature=([tensor_spec.TensorSpec(None, dtypes.float32),
@@ -137,7 +137,7 @@ class SaveTest(test.TestCase):
       root.f.get_concrete_function()
 
   def test_nested_outputs(self):
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.f = def_function.function(lambda x: (2. * x, (3. * x, 4. * x)))
     root.f(constant_op.constant(1.))
     to_save = root.f.get_concrete_function(constant_op.constant(1.))
@@ -158,7 +158,7 @@ class SaveTest(test.TestCase):
       save.save(root, save_dir, to_save)
 
   def test_variable(self):
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.v1 = variables.Variable(3.)
     root.v2 = variables.Variable(2.)
     root.f = def_function.function(
@@ -186,7 +186,7 @@ class SaveTest(test.TestCase):
   def test_trivial_save_exception(self):
     save_dir = os.path.join(self.get_temp_dir(), "saved_model")
     with self.assertRaisesRegexp(ValueError, "signature"):
-      save.save(tracking.Checkpointable(), save_dir)
+      save.save(tracking.AutoCheckpointable(), save_dir)
 
   def test_single_method_default_signature(self):
     model = _ModelWithOptimizer()
@@ -200,7 +200,7 @@ class SaveTest(test.TestCase):
                                     {"x": [[3., 4.]], "y": [2.]}))
 
   def test_single_function_default_signature(self):
-    model = tracking.Checkpointable()
+    model = tracking.AutoCheckpointable()
     model.f = def_function.function(lambda: 3., input_signature=())
     model.f()
     save_dir = os.path.join(self.get_temp_dir(), "saved_model")
@@ -369,7 +369,7 @@ class AssetTests(test.TestCase):
         _import_and_infer(second_dir, {"keys": ["gamma", "beta"]}))
 
   def test_unused_asset(self):
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.f = def_function.function(
         lambda x: 2. * x,
         input_signature=[tensor_spec.TensorSpec(None, dtypes.float32)])
diff --git a/tensorflow/python/training/checkpointable/base.py b/tensorflow/python/training/checkpointable/base.py
index f2797bc5ff..8257693055 100644
--- a/tensorflow/python/training/checkpointable/base.py
+++ b/tensorflow/python/training/checkpointable/base.py
@@ -460,16 +460,16 @@ def no_automatic_dependency_tracking(method):
       target=method, decorator_func=_method_wrapper)
 
 
-class CheckpointableBase(object):
+class Checkpointable(object):
   """Base class for `Checkpointable` objects without automatic dependencies.
 
   This class has no __setattr__ override for performance reasons. Dependencies
   must be added explicitly. Unless attribute assignment is performance-critical,
-  use `Checkpointable` instead. Use `CheckpointableBase` for `isinstance`
+  use `AutoCheckpointable` instead. Use `Checkpointable` for `isinstance`
   checks.
   """
 
-  # CheckpointableBase does not do automatic dependency tracking, but uses the
+  # Checkpointable does not do automatic dependency tracking, but uses the
   # no_automatic_dependency_tracking decorator so it can avoid adding
   # dependencies if a subclass is Checkpointable / inherits from Model (both of
   # which have __setattr__ overrides).
@@ -623,7 +623,7 @@ class CheckpointableBase(object):
     # assign again. It will add this variable to our dependencies, and if there
     # is a non-trivial restoration queued, it will handle that. This also
     # handles slot variables.
-    if not overwrite or isinstance(new_variable, CheckpointableBase):
+    if not overwrite or isinstance(new_variable, Checkpointable):
       return self._track_checkpointable(new_variable, name=name,
                                         overwrite=overwrite)
     else:
@@ -695,7 +695,7 @@ class CheckpointableBase(object):
       ValueError: If another object is already tracked by this name.
     """
     self._maybe_initialize_checkpointable()
-    if not isinstance(checkpointable, CheckpointableBase):
+    if not isinstance(checkpointable, Checkpointable):
       raise TypeError(
           ("Checkpointable._track_checkpointable() passed type %s, not a "
            "Checkpointable.") % (type(checkpointable),))
@@ -742,7 +742,7 @@ class CheckpointableBase(object):
       name: The name of the dependency within this object (`self`), used to
         match `checkpointable` with values saved in a checkpoint.
       checkpointable: The Checkpointable object to restore (inheriting from
-        `CheckpointableBase`).
+        `Checkpointable`).
     """
     self._maybe_initialize_checkpointable()
     checkpointable._maybe_initialize_checkpointable()  # pylint: disable=protected-access
diff --git a/tensorflow/python/training/checkpointable/base_test.py b/tensorflow/python/training/checkpointable/base_test.py
index 736cbee488..750799f030 100644
--- a/tensorflow/python/training/checkpointable/base_test.py
+++ b/tensorflow/python/training/checkpointable/base_test.py
@@ -29,13 +29,13 @@ from tensorflow.python.training.checkpointable import util
 class InterfaceTests(test.TestCase):
 
   def testOverwrite(self):
-    root = base.CheckpointableBase()
-    leaf = base.CheckpointableBase()
+    root = base.Checkpointable()
+    leaf = base.Checkpointable()
     root._track_checkpointable(leaf, name="leaf")
     (current_name, current_dependency), = root._checkpoint_dependencies
     self.assertIs(leaf, current_dependency)
     self.assertEqual("leaf", current_name)
-    duplicate_name_dep = base.CheckpointableBase()
+    duplicate_name_dep = base.Checkpointable()
     with self.assertRaises(ValueError):
       root._track_checkpointable(duplicate_name_dep, name="leaf")
     root._track_checkpointable(duplicate_name_dep, name="leaf", overwrite=True)
@@ -44,7 +44,7 @@ class InterfaceTests(test.TestCase):
     self.assertEqual("leaf", current_name)
 
   def testAddVariableOverwrite(self):
-    root = base.CheckpointableBase()
+    root = base.Checkpointable()
     a = root._add_variable_with_custom_getter(
         name="v", shape=[], getter=variable_scope.get_variable)
     self.assertEqual([root, a], util.list_objects(root))
@@ -61,15 +61,15 @@ class InterfaceTests(test.TestCase):
             getter=variable_scope.get_variable)
 
   def testAssertConsumedWithUnusedPythonState(self):
-    has_config = base.CheckpointableBase()
+    has_config = base.Checkpointable()
     has_config.get_config = lambda: {}
     saved = util.Checkpoint(obj=has_config)
     save_path = saved.save(os.path.join(self.get_temp_dir(), "ckpt"))
-    restored = util.Checkpoint(obj=base.CheckpointableBase())
+    restored = util.Checkpoint(obj=base.Checkpointable())
     restored.restore(save_path).assert_consumed()
 
   def testAssertConsumedFailsWithUsedPythonState(self):
-    has_config = base.CheckpointableBase()
+    has_config = base.Checkpointable()
     attributes = {
         "foo_attr": functools.partial(
             base.PythonStringStateSaveable,
@@ -78,7 +78,7 @@ class InterfaceTests(test.TestCase):
     has_config._gather_saveables_for_checkpoint = lambda: attributes
     saved = util.Checkpoint(obj=has_config)
     save_path = saved.save(os.path.join(self.get_temp_dir(), "ckpt"))
-    restored = util.Checkpoint(obj=base.CheckpointableBase())
+    restored = util.Checkpoint(obj=base.Checkpointable())
     status = restored.restore(save_path)
     with self.assertRaisesRegexp(AssertionError, "foo_attr"):
       status.assert_consumed()
diff --git a/tensorflow/python/training/checkpointable/data_structures.py b/tensorflow/python/training/checkpointable/data_structures.py
index d1e617ed46..5a5b444a6c 100644
--- a/tensorflow/python/training/checkpointable/data_structures.py
+++ b/tensorflow/python/training/checkpointable/data_structures.py
@@ -58,7 +58,7 @@ def _wrap_or_unwrap(value):
   """Wraps basic data structures, unwraps NoDependency objects."""
   if isinstance(value, NoDependency):
     return value.value
-  if isinstance(value, base.CheckpointableBase):
+  if isinstance(value, base.Checkpointable):
     return value  # Skip conversion for already checkpointable objects.
   elif isinstance(value, dict):
     return _DictWrapper(value)
@@ -99,7 +99,7 @@ def sticky_attribute_assignment(checkpointable, name, value):
   value = _wrap_or_unwrap(value)
   if not add_dependency:
     return value
-  if isinstance(value, base.CheckpointableBase):
+  if isinstance(value, base.Checkpointable):
     checkpointable._track_checkpointable(  # pylint: disable=protected-access
         value, name=name,
         # Allow the user to switch the Checkpointable which is tracked by this
@@ -109,7 +109,7 @@ def sticky_attribute_assignment(checkpointable, name, value):
   return value
 
 
-class CheckpointableDataStructure(base.CheckpointableBase):
+class CheckpointableDataStructure(base.Checkpointable):
   """Base class for data structures which contain checkpointable objects."""
 
   def __init__(self):
@@ -122,11 +122,11 @@ class CheckpointableDataStructure(base.CheckpointableBase):
         checkpointable=self, value=value, name=name)
     if isinstance(value, variables.Variable):
       self._extra_variables.append(value)
-    if not isinstance(value, base.CheckpointableBase):
+    if not isinstance(value, base.Checkpointable):
       raise ValueError(
           ("Only checkpointable objects (such as Layers or Optimizers) may be "
            "stored in a List object. Got %s, which does not inherit from "
-           "CheckpointableBase.") % (value,))
+           "Checkpointable.") % (value,))
     if hasattr(value, "_use_resource_variables"):
       # In subclassed models, legacy layers (tf.layers) must always use
       # resource variables.
@@ -410,7 +410,7 @@ class _ListWrapper(List, collections.MutableSequence,
 
   def __setitem__(self, key, value):
     self._check_external_modification()
-    if isinstance(self._storage[key], base.CheckpointableBase):
+    if isinstance(self._storage[key], base.Checkpointable):
       self._non_append_mutation = True
     self._storage[key] = self._track_value(value, self._name_element(key))
     self._update_snapshot()
@@ -693,14 +693,14 @@ class _DictWrapper(Mapping, collections.MutableMapping):
     else:
       value = _wrap_or_unwrap(value)
       existing_dependency = None
-      if not no_dep and isinstance(value, base.CheckpointableBase):
+      if not no_dep and isinstance(value, base.Checkpointable):
         # Non-string keys are OK as long as we have no reason to add a
         # dependency on the value (either because the value is not
         # checkpointable, or because it was wrapped in a NoDependency object).
         self._non_string_key = True
     current_value = self._storage.setdefault(key, value)
     if current_value is not value:
-      if ((not no_dep and isinstance(value, base.CheckpointableBase))
+      if ((not no_dep and isinstance(value, base.Checkpointable))
           # We don't want to just check that the existing object is
           # checkpointable, since it may have been wrapped in a NoDependency
           # object.
@@ -716,7 +716,7 @@ class _DictWrapper(Mapping, collections.MutableMapping):
   def __delitem__(self, key):
     self._check_external_modification()
     existing_value = self[key]
-    if isinstance(existing_value, base.CheckpointableBase):
+    if isinstance(existing_value, base.Checkpointable):
       # Deleting tracked checkpointable values means restoring is problematic,
       # so we'll throw an exception on save.
       self._non_append_mutation = True
diff --git a/tensorflow/python/training/checkpointable/data_structures_test.py b/tensorflow/python/training/checkpointable/data_structures_test.py
index 79191f3448..53cbd66482 100644
--- a/tensorflow/python/training/checkpointable/data_structures_test.py
+++ b/tensorflow/python/training/checkpointable/data_structures_test.py
@@ -210,8 +210,8 @@ class ListTests(test.TestCase):
   def testListWrapperBasic(self):
     # _ListWrapper, unlike List, compares like the built-in list type (since it
     # is used to automatically replace lists).
-    a = tracking.Checkpointable()
-    b = tracking.Checkpointable()
+    a = tracking.AutoCheckpointable()
+    b = tracking.AutoCheckpointable()
     self.assertEqual([a, a],
                      [a, a])
     self.assertEqual(data_structures._ListWrapper([a, a]),
@@ -343,7 +343,7 @@ class MappingTests(test.TestCase):
 
   def testLayerCollectionWithExternalMutation(self):
     d = {}
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.wrapper = d
     self.assertEqual([], root.wrapper.layers)
     self.assertEqual([], root.wrapper.trainable_weights)
@@ -361,7 +361,7 @@ class MappingTests(test.TestCase):
     self.assertEqual(2, len(has_mappings))
     self.assertNotIn(data_structures.Mapping(), has_mappings)
     # In contrast to Mapping, dict wrappers are not hashable
-    a = tracking.Checkpointable()
+    a = tracking.AutoCheckpointable()
     a.d = {}
     self.assertEqual({}, a.d)
     self.assertFalse({} != a.d)  # pylint: disable=g-explicit-bool-comparison
@@ -370,7 +370,7 @@ class MappingTests(test.TestCase):
       set([a.d])
 
   def testDictWrapperBadKeys(self):
-    a = tracking.Checkpointable()
+    a = tracking.AutoCheckpointable()
     a.d = {}
     a.d[1] = data_structures.List()
     model = training.Model()
@@ -380,7 +380,7 @@ class MappingTests(test.TestCase):
       model.save_weights(save_path)
 
   def testDictWrapperNoDependency(self):
-    a = tracking.Checkpointable()
+    a = tracking.AutoCheckpointable()
     a.d = data_structures.NoDependency({})
     a.d[1] = [3]
     self.assertEqual([a], util.list_objects(a))
@@ -391,7 +391,7 @@ class MappingTests(test.TestCase):
     model.load_weights(save_path)
 
   def testNonStringKeyNotCheckpointableValue(self):
-    a = tracking.Checkpointable()
+    a = tracking.AutoCheckpointable()
     a.d = {}
     a.d["a"] = [3]
     a.d[1] = data_structures.NoDependency([3])
@@ -405,15 +405,15 @@ class MappingTests(test.TestCase):
   def testNonAppendNotCheckpointable(self):
     # Non-append mutations (deleting or overwriting values) are OK when the
     # values aren't tracked.
-    a = tracking.Checkpointable()
+    a = tracking.AutoCheckpointable()
     a.d = {}
     a.d["a"] = [3]
     a.d[1] = 3
     a.d[1] = 2
     self.assertEqual(2, a.d[1])
     del a.d[1]
-    a.d[2] = data_structures.NoDependency(tracking.Checkpointable())
-    second = tracking.Checkpointable()
+    a.d[2] = data_structures.NoDependency(tracking.AutoCheckpointable())
+    second = tracking.AutoCheckpointable()
     a.d[2] = data_structures.NoDependency(second)
     self.assertIs(second, a.d[2])
     self.assertEqual([a, a.d, a.d["a"]], util.list_objects(a))
@@ -475,7 +475,7 @@ class MappingTests(test.TestCase):
     self.assertEqual({1: 3}, new_dict)
 
   def testListShallowCopy(self):
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     orig_list = [[1.]]
     root.a = orig_list
     copied = copy.copy(root.a)
@@ -492,7 +492,7 @@ class MappingTests(test.TestCase):
       util.list_objects(copy.copy(root.a))
 
   def testListDeepCopy(self):
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     orig_list = [[1.]]
     root.a = orig_list
     copied = copy.deepcopy(root.a)
@@ -509,7 +509,7 @@ class MappingTests(test.TestCase):
       util.list_objects(copy.deepcopy(root.a))
 
   def testDictShallowCopy(self):
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     orig_dict = {"a": [1.]}
     root.a = orig_dict
     copied = copy.copy(root.a)
@@ -526,7 +526,7 @@ class MappingTests(test.TestCase):
       util.list_objects(copy.copy(root.a))
 
   def testDictDeepCopy(self):
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     orig_dict = {"a": [1.]}
     root.a = orig_dict
     copied = copy.deepcopy(root.a)
@@ -543,8 +543,8 @@ class MappingTests(test.TestCase):
       util.list_objects(copy.deepcopy(root.a))
 
   def testShallowCopyCheckpointable(self):
-    original = tracking.Checkpointable()
-    original_sub = tracking.Checkpointable()
+    original = tracking.AutoCheckpointable()
+    original_sub = tracking.AutoCheckpointable()
     original.a = [[1.]]
     original.b = {"a": original_sub}
     shallow_copied = copy.copy(original)
@@ -557,15 +557,15 @@ class MappingTests(test.TestCase):
     self.assertIn(shallow_copied.b["a"], shallow_deps)
 
   def testDeepCopyCheckpointable(self):
-    original = tracking.Checkpointable()
-    original_sub = tracking.Checkpointable()
+    original = tracking.AutoCheckpointable()
+    original_sub = tracking.AutoCheckpointable()
     original.a = [[1.]]
     original.b = {"a": original_sub}
     deep_copied = copy.deepcopy(original)
     self.assertIsNot(original, deep_copied)
     self.assertIsNot(original_sub, deep_copied.b["a"])
     self.assertEqual([[1.]], deep_copied.a)
-    self.assertIsInstance(deep_copied.b["a"], tracking.Checkpointable)
+    self.assertIsInstance(deep_copied.b["a"], tracking.AutoCheckpointable)
     deps = util.list_objects(deep_copied)
     self.assertIn(deep_copied.a, deps)
     self.assertIn(deep_copied.b, deps)
diff --git a/tensorflow/python/training/checkpointable/tracking.py b/tensorflow/python/training/checkpointable/tracking.py
index 4e96aee0c5..04fd5547e1 100644
--- a/tensorflow/python/training/checkpointable/tracking.py
+++ b/tensorflow/python/training/checkpointable/tracking.py
@@ -41,7 +41,7 @@ class NotCheckpointable(object):
   pass
 
 
-class Checkpointable(base.CheckpointableBase):
+class AutoCheckpointable(base.Checkpointable):
   """Manages dependencies on other objects.
 
   `Checkpointable` objects may have dependencies: other `Checkpointable` objects
@@ -74,7 +74,7 @@ class Checkpointable(base.CheckpointableBase):
     if getattr(self, "_setattr_tracking", True):
       value = data_structures.sticky_attribute_assignment(
           checkpointable=self, value=value, name=name)
-    super(Checkpointable, self).__setattr__(name, value)
+    super(AutoCheckpointable, self).__setattr__(name, value)
 
   def _no_dependency(self, value):
     """Override to allow CheckpointableBase to disable dependency tracking."""
@@ -124,7 +124,7 @@ def resource_tracker_scope(resource_tracker):
     _RESOURCE_TRACKER_STACK = old
 
 
-class TrackableResource(base.CheckpointableBase):
+class TrackableResource(base.Checkpointable):
   """Base class for all resources that need to be tracked."""
 
   def __init__(self):
@@ -151,7 +151,7 @@ class TrackableResource(base.CheckpointableBase):
     return self._resource_handle
 
 
-class TrackableAsset(base.CheckpointableBase):
+class TrackableAsset(base.Checkpointable):
   """Base class for asset files which need to be tracked."""
 
   def __init__(self, path):
diff --git a/tensorflow/python/training/checkpointable/tracking_test.py b/tensorflow/python/training/checkpointable/tracking_test.py
index 17c5461bc2..eb70919b9c 100644
--- a/tensorflow/python/training/checkpointable/tracking_test.py
+++ b/tensorflow/python/training/checkpointable/tracking_test.py
@@ -35,10 +35,10 @@ from tensorflow.python.util import nest
 class InterfaceTests(test.TestCase):
 
   def testMultipleAssignment(self):
-    root = tracking.Checkpointable()
-    root.leaf = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
+    root.leaf = tracking.AutoCheckpointable()
     root.leaf = root.leaf
-    duplicate_name_dep = tracking.Checkpointable()
+    duplicate_name_dep = tracking.AutoCheckpointable()
     with self.assertRaisesRegexp(ValueError, "already declared"):
       root._track_checkpointable(duplicate_name_dep, name="leaf")
     # No error; we're overriding __setattr__, so we can't really stop people
@@ -50,10 +50,10 @@ class InterfaceTests(test.TestCase):
     self.assertIs(duplicate_name_dep, dep_object)
 
   def testNoDependency(self):
-    root = tracking.Checkpointable()
-    hasdep = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
+    hasdep = tracking.AutoCheckpointable()
     root.hasdep = hasdep
-    nodep = tracking.Checkpointable()
+    nodep = tracking.AutoCheckpointable()
     root.nodep = data_structures.NoDependency(nodep)
     self.assertEqual(1, len(root._checkpoint_dependencies))
     self.assertIs(root._checkpoint_dependencies[0].ref, root.hasdep)
@@ -66,16 +66,16 @@ class InterfaceTests(test.TestCase):
       def __init__(self):
         super(NoDependencyModel, self).__init__()
         self.a = []
-        self.b = tracking.Checkpointable()
+        self.b = tracking.AutoCheckpointable()
 
     nodeps = NoDependencyModel()
     self.assertEqual([nodeps], util.list_objects(nodeps))
 
   def testListBasic(self):
-    a = tracking.Checkpointable()
-    b = tracking.Checkpointable()
+    a = tracking.AutoCheckpointable()
+    b = tracking.AutoCheckpointable()
     a.l = [b]
-    c = tracking.Checkpointable()
+    c = tracking.AutoCheckpointable()
     a.l.append(c)
     a_deps = util.list_objects(a)
     self.assertIn(b, a_deps)
@@ -87,10 +87,10 @@ class InterfaceTests(test.TestCase):
 
   @test_util.run_in_graph_and_eager_modes
   def testMutationDirtiesList(self):
-    a = tracking.Checkpointable()
-    b = tracking.Checkpointable()
+    a = tracking.AutoCheckpointable()
+    b = tracking.AutoCheckpointable()
     a.l = [b]
-    c = tracking.Checkpointable()
+    c = tracking.AutoCheckpointable()
     a.l.insert(0, c)
     checkpoint = util.Checkpoint(a=a)
     with self.assertRaisesRegexp(ValueError, "A list element was replaced"):
@@ -98,11 +98,11 @@ class InterfaceTests(test.TestCase):
 
   @test_util.run_in_graph_and_eager_modes
   def testOutOfBandEditDirtiesList(self):
-    a = tracking.Checkpointable()
-    b = tracking.Checkpointable()
+    a = tracking.AutoCheckpointable()
+    b = tracking.AutoCheckpointable()
     held_reference = [b]
     a.l = held_reference
-    c = tracking.Checkpointable()
+    c = tracking.AutoCheckpointable()
     held_reference.append(c)
     checkpoint = util.Checkpoint(a=a)
     with self.assertRaisesRegexp(ValueError, "The wrapped list was modified"):
@@ -110,25 +110,25 @@ class InterfaceTests(test.TestCase):
 
   @test_util.run_in_graph_and_eager_modes
   def testNestedLists(self):
-    a = tracking.Checkpointable()
+    a = tracking.AutoCheckpointable()
     a.l = []
-    b = tracking.Checkpointable()
+    b = tracking.AutoCheckpointable()
     a.l.append([b])
-    c = tracking.Checkpointable()
+    c = tracking.AutoCheckpointable()
     a.l[0].append(c)
     a_deps = util.list_objects(a)
     self.assertIn(b, a_deps)
     self.assertIn(c, a_deps)
     a.l[0].append(1)
-    d = tracking.Checkpointable()
+    d = tracking.AutoCheckpointable()
     a.l[0].append(d)
     a_deps = util.list_objects(a)
     self.assertIn(d, a_deps)
     self.assertIn(b, a_deps)
     self.assertIn(c, a_deps)
     self.assertNotIn(1, a_deps)
-    e = tracking.Checkpointable()
-    f = tracking.Checkpointable()
+    e = tracking.AutoCheckpointable()
+    f = tracking.AutoCheckpointable()
     a.l1 = [[], [e]]
     a.l1[0].append(f)
     a_deps = util.list_objects(a)
@@ -183,7 +183,7 @@ class InterfaceTests(test.TestCase):
 
   @test_util.run_in_graph_and_eager_modes
   def testAssertions(self):
-    a = tracking.Checkpointable()
+    a = tracking.AutoCheckpointable()
     a.l = {"k": [numpy.zeros([2, 2])]}
     self.assertAllEqual(nest.flatten({"k": [numpy.zeros([2, 2])]}),
                         nest.flatten(a.l))
diff --git a/tensorflow/python/training/checkpointable/util.py b/tensorflow/python/training/checkpointable/util.py
index 1ff0cd9f25..5f96ce0db8 100644
--- a/tensorflow/python/training/checkpointable/util.py
+++ b/tensorflow/python/training/checkpointable/util.py
@@ -1655,7 +1655,7 @@ def frozen_saver(root_checkpointable):
 
 
 @tf_export("train.Checkpoint")
-class Checkpoint(tracking.Checkpointable):
+class Checkpoint(tracking.AutoCheckpointable):
   """Groups checkpointable objects, saving and restoring them.
 
   `Checkpoint`'s constructor accepts keyword arguments whose values are types
@@ -1757,7 +1757,7 @@ class Checkpoint(tracking.Checkpointable):
     """
     super(Checkpoint, self).__init__()
     for k, v in sorted(kwargs.items(), key=lambda item: item[0]):
-      if not isinstance(v, (base.CheckpointableBase,
+      if not isinstance(v, (base.Checkpointable,
                             def_function.PolymorphicFunction)):
         raise ValueError(
             ("`Checkpoint` was expecting a checkpointable object (an object "
diff --git a/tensorflow/python/training/checkpointable/util_test.py b/tensorflow/python/training/checkpointable/util_test.py
index 78a8282f25..cef1075e93 100644
--- a/tensorflow/python/training/checkpointable/util_test.py
+++ b/tensorflow/python/training/checkpointable/util_test.py
@@ -51,7 +51,7 @@ from tensorflow.python.training.checkpointable import tracking
 from tensorflow.python.training.checkpointable import util as checkpointable_utils
 
 
-class NonLayerCheckpointable(tracking.Checkpointable):
+class NonLayerCheckpointable(tracking.AutoCheckpointable):
 
   def __init__(self):
     super(NonLayerCheckpointable, self).__init__()
@@ -139,7 +139,7 @@ class InterfaceTests(test.TestCase):
 
   def testInitNotCalled(self):
 
-    class NoInit(tracking.Checkpointable):
+    class NoInit(tracking.AutoCheckpointable):
 
       def __init__(self):
         pass
@@ -148,7 +148,7 @@ class InterfaceTests(test.TestCase):
     checkpointable_utils.add_variable(NoInit(), "var", shape=[])
 
   def testShapeDtype(self):
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     v1 = checkpointable_utils.add_variable(
         root, name="v1", initializer=3., dtype=dtypes.float64)
     self.assertEqual(dtypes.float64, v1.dtype)
@@ -180,7 +180,7 @@ class InterfaceTests(test.TestCase):
   def testNotCheckpointable(self):
 
     class CallsFunctionalStuff(
-        tracking.NotCheckpointable, tracking.Checkpointable):
+        tracking.NotCheckpointable, tracking.AutoCheckpointable):
       pass
 
     test_dir = self.get_temp_dir()
@@ -190,7 +190,7 @@ class InterfaceTests(test.TestCase):
       checkpoint.save(prefix)
 
     class CallsFunctionalStuffOtherMRO(
-        tracking.Checkpointable, tracking.NotCheckpointable):
+        tracking.AutoCheckpointable, tracking.NotCheckpointable):
       pass
 
     checkpoint_reversed = checkpointable_utils.Checkpoint(
@@ -220,7 +220,7 @@ class _MirroringSaveable(saver_lib.BaseSaverBuilder.SaveableObject):
         self._mirrored_variable.assign(tensor))
 
 
-class _OwnsMirroredVariables(base.CheckpointableBase):
+class _OwnsMirroredVariables(base.Checkpointable):
   """A Checkpointable object which returns a more complex SaveableObject."""
 
   def __init__(self):
@@ -653,7 +653,7 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
   # pylint: enable=cell-var-from-loop
 
   def _get_checkpoint_name(self, name):
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     checkpointable_utils.add_variable(
         root, name=name, shape=[1, 2], dtype=dtypes.float64)
     (named_variable,), _, _ = checkpointable_utils._serialize_object_graph(
@@ -674,8 +674,8 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
 
   @test_util.run_in_graph_and_eager_modes(assert_no_eager_garbage=True)
   def testNumberedPath(self):
-    root = tracking.Checkpointable()
-    leaf = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
+    leaf = tracking.AutoCheckpointable()
     root.leaf = leaf
     checkpointable_utils.add_variable(leaf, name="v", shape=[])
     (named_variable,), _, _ = checkpointable_utils._serialize_object_graph(
@@ -684,8 +684,8 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
 
   @test_util.run_in_graph_and_eager_modes
   def testLocalNameValidation(self):
-    root = tracking.Checkpointable()
-    leaf = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
+    leaf = tracking.AutoCheckpointable()
     # Dots are escaped, which avoids conflicts with reserved names.
     root._track_checkpointable(leaf, name=".ATTRIBUTES")
     checkpointable_utils.add_variable(checkpointable=leaf, name="a", shape=[])
@@ -726,13 +726,13 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
   @test_util.run_in_graph_and_eager_modes
   def testLateDependencyTracking(self):
 
-    class Dependency(tracking.Checkpointable):
+    class Dependency(tracking.AutoCheckpointable):
 
       def build(self):
         self.var = checkpointable_utils.add_variable(
             self, "var", initializer=0.)
 
-    class LateDependencies(tracking.Checkpointable):
+    class LateDependencies(tracking.AutoCheckpointable):
 
       def add_dep(self):
         self.dep = Dependency()
@@ -759,13 +759,13 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
   @test_util.run_in_graph_and_eager_modes
   def testDepAfterVar(self):
 
-    class Dependency(tracking.Checkpointable):
+    class Dependency(tracking.AutoCheckpointable):
 
       def build(self):
         self.var = checkpointable_utils.add_variable(
             self, "var", initializer=0.)
 
-    class DepAfterVar(tracking.Checkpointable):
+    class DepAfterVar(tracking.AutoCheckpointable):
 
       def add_dep(self):
         dep = Dependency()
@@ -792,7 +792,7 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
   def testDeferredSlotRestoration(self):
     checkpoint_directory = self.get_temp_dir()
 
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.var = checkpointable_utils.add_variable(
         root, name="var", initializer=0.)
     optimizer = adam.Adam(0.1)
@@ -815,7 +815,7 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
         14.))
     slots_path = checkpointable_utils.CheckpointableSaver(root).save(
         os.path.join(checkpoint_directory, "with_slots"))
-    new_root = tracking.Checkpointable()
+    new_root = tracking.AutoCheckpointable()
     # Load the slot-containing checkpoint (deferred), then immediately overwrite
     # the non-slot variable (also deferred).
     slot_status = checkpointable_utils.CheckpointableSaver(
@@ -861,8 +861,8 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
   @test_util.run_in_graph_and_eager_modes
   def testOverlappingRestores(self):
     checkpoint_directory = self.get_temp_dir()
-    save_root = tracking.Checkpointable()
-    save_root.dep = tracking.Checkpointable()
+    save_root = tracking.AutoCheckpointable()
+    save_root.dep = tracking.AutoCheckpointable()
     save_root.dep.var = checkpointable_utils.add_variable(
         save_root.dep, name="var", initializer=0.)
     self.evaluate(state_ops.assign(save_root.dep.var, 12.))
@@ -871,13 +871,13 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
     self.evaluate(state_ops.assign(save_root.dep.var, 13.))
     second_path = saver.save(os.path.join(checkpoint_directory, "second"))
 
-    first_root = tracking.Checkpointable()
-    second_root = tracking.Checkpointable()
+    first_root = tracking.AutoCheckpointable()
+    second_root = tracking.AutoCheckpointable()
     first_status = checkpointable_utils.CheckpointableSaver(
         first_root).restore(first_path)
     second_status = checkpointable_utils.CheckpointableSaver(
         second_root).restore(second_path)
-    load_dep = tracking.Checkpointable()
+    load_dep = tracking.AutoCheckpointable()
     load_dep.var = checkpointable_utils.add_variable(
         load_dep, name="var", shape=[])
     first_root.dep = load_dep
@@ -891,13 +891,13 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
 
     # Try again with the order of the restore() reversed. The last restore
     # determines the final value.
-    first_root = tracking.Checkpointable()
-    second_root = tracking.Checkpointable()
+    first_root = tracking.AutoCheckpointable()
+    second_root = tracking.AutoCheckpointable()
     second_status = checkpointable_utils.CheckpointableSaver(
         second_root).restore(second_path)
     first_status = checkpointable_utils.CheckpointableSaver(
         first_root).restore(first_path)
-    load_dep = tracking.Checkpointable()
+    load_dep = tracking.AutoCheckpointable()
     load_dep.var = checkpointable_utils.add_variable(
         load_dep, name="var", shape=[])
     first_root.dep = load_dep
@@ -913,23 +913,23 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
   def testAmbiguousLoad(self):
     # Not OK to split one checkpoint object into two
     checkpoint_directory = self.get_temp_dir()
-    save_root = tracking.Checkpointable()
-    save_root.dep_one = tracking.Checkpointable()
-    save_root.dep_two = tracking.Checkpointable()
-    dep_three = tracking.Checkpointable()
+    save_root = tracking.AutoCheckpointable()
+    save_root.dep_one = tracking.AutoCheckpointable()
+    save_root.dep_two = tracking.AutoCheckpointable()
+    dep_three = tracking.AutoCheckpointable()
     save_root.dep_one.dep_three = dep_three
     save_root.dep_two.dep_three = dep_three
     checkpointable_utils.add_variable(dep_three, name="var", initializer=0.)
     self.evaluate(checkpointable_utils.gather_initializers(save_root))
     save_path = checkpointable_utils.CheckpointableSaver(save_root).save(
         os.path.join(checkpoint_directory, "ckpt"))
-    load_root = tracking.Checkpointable()
+    load_root = tracking.AutoCheckpointable()
     status = checkpointable_utils.CheckpointableSaver(load_root).restore(
         save_path)
-    load_root.dep_one = tracking.Checkpointable()
-    load_root.dep_two = tracking.Checkpointable()
-    load_root.dep_one.dep_three = tracking.Checkpointable()
-    load_root.dep_two.dep_three = tracking.Checkpointable()
+    load_root.dep_one = tracking.AutoCheckpointable()
+    load_root.dep_two = tracking.AutoCheckpointable()
+    load_root.dep_one.dep_three = tracking.AutoCheckpointable()
+    load_root.dep_two.dep_three = tracking.AutoCheckpointable()
     checkpointable_utils.add_variable(
         load_root.dep_one.dep_three, name="var", initializer=0.)
     with self.assertRaises(AssertionError):
@@ -941,9 +941,9 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
   def testObjectsCombined(self):
     # Currently fine to load two checkpoint objects into one Python object
     checkpoint_directory = self.get_temp_dir()
-    save_root = tracking.Checkpointable()
-    save_root.dep_one = tracking.Checkpointable()
-    save_root.dep_two = tracking.Checkpointable()
+    save_root = tracking.AutoCheckpointable()
+    save_root.dep_one = tracking.AutoCheckpointable()
+    save_root.dep_two = tracking.AutoCheckpointable()
     checkpointable_utils.add_variable(
         save_root.dep_one, name="var1", initializer=32., dtype=dtypes.float64)
     checkpointable_utils.add_variable(
@@ -951,8 +951,8 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
     self.evaluate(checkpointable_utils.gather_initializers(save_root))
     save_path = checkpointable_utils.CheckpointableSaver(save_root).save(
         os.path.join(checkpoint_directory, "ckpt"))
-    load_root = tracking.Checkpointable()
-    load_root.dep_one = tracking.Checkpointable()
+    load_root = tracking.AutoCheckpointable()
+    load_root.dep_one = tracking.AutoCheckpointable()
     load_root.dep_two = load_root.dep_one
     v1 = checkpointable_utils.add_variable(
         load_root.dep_one, name="var1", shape=[], dtype=dtypes.float64)
@@ -968,8 +968,8 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
   def testDependencyLoop(self):
     # Note: this test creates garbage during eager execution because it
     # purposefully creates a reference cycle.
-    first = tracking.Checkpointable()
-    second = tracking.Checkpointable()
+    first = tracking.AutoCheckpointable()
+    second = tracking.AutoCheckpointable()
     first.second = second
     second.first = first
     first.v = checkpointable_utils.add_variable(
@@ -982,10 +982,10 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
         os.path.join(checkpoint_directory, "ckpt"))
 
     # Test deferred loading
-    first_load = tracking.Checkpointable()
+    first_load = tracking.AutoCheckpointable()
     status = checkpointable_utils.CheckpointableSaver(
         first_load).restore(save_path)
-    second_load = tracking.Checkpointable()
+    second_load = tracking.AutoCheckpointable()
     first_load.second = second_load
     second_load.first = first_load
     with self.assertRaises(AssertionError):
@@ -1014,7 +1014,7 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
   def testRestoreOnAssign(self):
     checkpoint_directory = self.get_temp_dir()
     checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
-    first = tracking.Checkpointable()
+    first = tracking.AutoCheckpointable()
     first.var1 = variables_lib.Variable(0., name="outside_var")
     first.var2 = variables_lib.Variable(0., name="blah")
     self.evaluate(first.var1.assign(4.))
@@ -1022,7 +1022,7 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
     save_path = checkpointable_utils.CheckpointableSaver(first).save(
         checkpoint_prefix)
 
-    second = tracking.Checkpointable()
+    second = tracking.AutoCheckpointable()
     second.var2 = variables_lib.Variable(0., name="blah")
     status = checkpointable_utils.CheckpointableSaver(
         second).restore(save_path)
@@ -1042,7 +1042,7 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
       with graph.as_default(), self.session(graph):
         checkpoint_directory = self.get_temp_dir()
         checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
-        obj = tracking.Checkpointable()
+        obj = tracking.AutoCheckpointable()
         obj.var = variables_lib.Variable(0., name="v")
         obj.opt = adam.Adam(0.1)
         variables = [obj.var]
@@ -1059,7 +1059,7 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
     # No checkpoints are deleted by default
     checkpoint_directory = self.get_temp_dir()
     checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
-    obj = tracking.Checkpointable()
+    obj = tracking.AutoCheckpointable()
     obj.var = variable_scope.get_variable(name="v", initializer=0.)
     self.evaluate(checkpointable_utils.gather_initializers(obj))
     saver = checkpointable_utils.Checkpoint(obj=obj)
@@ -1079,7 +1079,7 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
   def testCheckpointStateChangingVarList(self):
     checkpoint_directory = self.get_temp_dir()
     checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
-    obj = tracking.Checkpointable()
+    obj = tracking.AutoCheckpointable()
     obj.var = variable_scope.get_variable(name="v", initializer=0.)
     self.evaluate(checkpointable_utils.gather_initializers(obj))
     checkpoint = checkpointable_utils.Checkpoint(obj=obj)
@@ -1132,7 +1132,7 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
       with graph.as_default(), self.session(graph):
         checkpoint_directory = self.get_temp_dir()
         checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
-        obj = tracking.Checkpointable()
+        obj = tracking.AutoCheckpointable()
         obj.var = variables_lib.Variable(0., name="v")
         obj.opt = adam.Adam(0.1)
         variables = [obj.var]
@@ -1286,7 +1286,7 @@ class CheckpointingTests(parameterized.TestCase, test.TestCase):
     load_status.assert_existing_objects_matched().run_restore_ops()
 
 
-class _ManualScope(tracking.Checkpointable):
+class _ManualScope(tracking.AutoCheckpointable):
 
   def __call__(self):
     with variable_scope.variable_scope("ManualScope") as vs:
diff --git a/tensorflow/python/training/checkpointable/util_with_v1_optimizers_test.py b/tensorflow/python/training/checkpointable/util_with_v1_optimizers_test.py
index bb6bb8bd1d..bd80fa60f0 100644
--- a/tensorflow/python/training/checkpointable/util_with_v1_optimizers_test.py
+++ b/tensorflow/python/training/checkpointable/util_with_v1_optimizers_test.py
@@ -47,7 +47,7 @@ from tensorflow.python.training.checkpointable import tracking
 from tensorflow.python.training.checkpointable import util as checkpointable_utils
 
 
-class NonLayerCheckpointable(tracking.Checkpointable):
+class NonLayerCheckpointable(tracking.AutoCheckpointable):
 
   def __init__(self):
     super(NonLayerCheckpointable, self).__init__()
@@ -461,7 +461,7 @@ class CheckpointingTests(test.TestCase):
   # pylint: enable=cell-var-from-loop
 
   def _get_checkpoint_name(self, name):
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     checkpointable_utils.add_variable(
         root, name=name, shape=[1, 2], dtype=dtypes.float64)
     (named_variable,), _, _ = checkpointable_utils._serialize_object_graph(
@@ -503,7 +503,7 @@ class CheckpointingTests(test.TestCase):
   def testDeferredSlotRestoration(self):
     checkpoint_directory = self.get_temp_dir()
 
-    root = tracking.Checkpointable()
+    root = tracking.AutoCheckpointable()
     root.var = checkpointable_utils.add_variable(
         root, name="var", initializer=0.)
     optimizer = adam.AdamOptimizer(0.1)
@@ -526,7 +526,7 @@ class CheckpointingTests(test.TestCase):
                                    14.))
     slots_path = checkpointable_utils.CheckpointableSaver(root).save(
         os.path.join(checkpoint_directory, "with_slots"))
-    new_root = tracking.Checkpointable()
+    new_root = tracking.AutoCheckpointable()
     # Load the slot-containing checkpoint (deferred), then immediately overwrite
     # the non-slot variable (also deferred).
     slot_status = checkpointable_utils.CheckpointableSaver(
@@ -572,7 +572,7 @@ class CheckpointingTests(test.TestCase):
       with graph.as_default(), self.session(graph):
         checkpoint_directory = self.get_temp_dir()
         checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
-        obj = tracking.Checkpointable()
+        obj = tracking.AutoCheckpointable()
         obj.var = variable_scope.get_variable(name="v", initializer=0.)
         obj.opt = adam.AdamOptimizer(0.1)
         obj.opt.minimize(obj.var.read_value())
@@ -590,7 +590,7 @@ class CheckpointingTests(test.TestCase):
       with graph.as_default(), self.session(graph):
         checkpoint_directory = self.get_temp_dir()
         checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
-        obj = tracking.Checkpointable()
+        obj = tracking.AutoCheckpointable()
         obj.var = variable_scope.get_variable(name="v", initializer=0.)
         obj.opt = adam.AdamOptimizer(0.1)
         obj.opt.minimize(obj.var.read_value())
@@ -739,7 +739,7 @@ class CheckpointingTests(test.TestCase):
       self.assertEqual(42., self.evaluate(optimizer.variables()[0]))
 
 
-class _ManualScope(tracking.Checkpointable):
+class _ManualScope(tracking.AutoCheckpointable):
 
   def __call__(self):
     with variable_scope.variable_scope("ManualScope") as vs:
diff --git a/tensorflow/python/training/checkpointable/util_xla_test.py b/tensorflow/python/training/checkpointable/util_xla_test.py
index d80ac003b4..4e96a7514a 100644
--- a/tensorflow/python/training/checkpointable/util_xla_test.py
+++ b/tensorflow/python/training/checkpointable/util_xla_test.py
@@ -29,7 +29,7 @@ from tensorflow.python.training.checkpointable import tracking
 from tensorflow.python.training.checkpointable import util as checkpointable_utils
 
 
-class NonLayerCheckpointable(tracking.Checkpointable):
+class NonLayerCheckpointable(tracking.AutoCheckpointable):
 
   def __init__(self):
     super(NonLayerCheckpointable, self).__init__()
diff --git a/tensorflow/python/training/optimizer.py b/tensorflow/python/training/optimizer.py
index a591dc9fec..3742ebb807 100644
--- a/tensorflow/python/training/optimizer.py
+++ b/tensorflow/python/training/optimizer.py
@@ -218,7 +218,7 @@ class Optimizer(
     # Optimizers inherit from CheckpointableBase rather than Checkpointable
     # since they do most of their dependency management themselves (slot
     # variables are special-cased, and non-slot variables are keyed to graphs).
-    checkpointable.CheckpointableBase):
+    checkpointable.Checkpointable):
   """Base class for optimizers.
 
   This class defines the API to add Ops to train a model.  You never use this
diff --git a/tensorflow/python/training/saver_test.py b/tensorflow/python/training/saver_test.py
index d1b51adaa4..dec23c50e8 100644
--- a/tensorflow/python/training/saver_test.py
+++ b/tensorflow/python/training/saver_test.py
@@ -2775,7 +2775,7 @@ class ScopedGraphTest(test.TestCase):
       self.assertEqual(2.0, self.evaluate(var_dict2["variable2:0"]))
 
 
-class _OwnsAVariableSimple(checkpointable_base.CheckpointableBase):
+class _OwnsAVariableSimple(checkpointable_base.Checkpointable):
   """A Checkpointable object which can be saved using a tf.train.Saver."""
 
   def __init__(self):
@@ -2808,7 +2808,7 @@ class _MirroringSaveable(
         self._mirrored_variable.assign(tensor))
 
 
-class _OwnsMirroredVariables(checkpointable_base.CheckpointableBase):
+class _OwnsMirroredVariables(checkpointable_base.Checkpointable):
   """A Checkpointable object which returns a more complex SaveableObject."""
 
   def __init__(self):
@@ -2831,7 +2831,7 @@ class _OwnsMirroredVariables(checkpointable_base.CheckpointableBase):
     return self.non_dep_variable.name
 
 
-class NonLayerCheckpointable(checkpointable_tracking.Checkpointable):
+class NonLayerCheckpointable(checkpointable_tracking.AutoCheckpointable):
 
   def __init__(self):
     super(NonLayerCheckpointable, self).__init__()
diff --git a/tensorflow/python/training/saving/saveable_object_util.py b/tensorflow/python/training/saving/saveable_object_util.py
index fa88d2c6eb..b8cc66249b 100644
--- a/tensorflow/python/training/saving/saveable_object_util.py
+++ b/tensorflow/python/training/saving/saveable_object_util.py
@@ -165,7 +165,7 @@ def saveable_objects_for_op(op, name):
         yield ResourceVariableSaveable(
             variable, variable._save_slice_info.spec, name)
     # pylint: enable=protected-access
-  elif isinstance(op, checkpointable.CheckpointableBase) and not isinstance(
+  elif isinstance(op, checkpointable.Checkpointable) and not isinstance(
       op, variables.Variable):
     # pylint: disable=protected-access
     for attr, factory in op._gather_saveables_for_checkpoint().items():
@@ -250,7 +250,7 @@ def op_list_to_dict(op_list, convert_variable_to_tensor=True):
         names_to_saveables[name].append(var)
       else:
         names_to_saveables[name] = [var]
-    elif (isinstance(var, checkpointable.CheckpointableBase)
+    elif (isinstance(var, checkpointable.Checkpointable)
           and not isinstance(var, variables.Variable)):
       checkpointable_saveables = [
           (factory() if callable(factory) else factory)
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.-variable.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.-variable.pbtxt
index 048f666305..f042097acc 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.-variable.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.-variable.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.Variable"
 tf_class {
   is_instance: "<class \'tensorflow.python.ops.variables.VariableV1\'>"
   is_instance: "<class \'tensorflow.python.ops.variables.Variable\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "SaveSliceInfo"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.-iterator.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.-iterator.pbtxt
index 4f0147a523..682a2b91b6 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.data.-iterator.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.-iterator.pbtxt
@@ -1,7 +1,7 @@
 path: "tensorflow.data.Iterator"
 tf_class {
   is_instance: "<class \'tensorflow.python.data.ops.iterator_ops.Iterator\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "initializer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.-model.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.-model.pbtxt
index eced2e1cb0..0a81c3a663 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.-model.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.-model.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.engine.training.Model\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.network.Network\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.-sequential.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.-sequential.pbtxt
index 2acb90173f..2a9ceec30e 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.-sequential.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.-sequential.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.engine.training.Model\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.network.Network\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.-peephole-l-s-t-m-cell.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.-peephole-l-s-t-m-cell.pbtxt
index b2ab5006dc..1bfd51cdcc 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.-peephole-l-s-t-m-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.experimental.-peephole-l-s-t-m-cell.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.PeepholeLSTMCell\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.LSTMCell\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-activation.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-activation.pbtxt
index da212382c1..8a0b8eb46f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-activation.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-activation.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Activation"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.Activation\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-activity-regularization.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-activity-regularization.pbtxt
index c910db027e..abb3c23694 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-activity-regularization.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-activity-regularization.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.ActivityRegularization"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.ActivityRegularization\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-add.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-add.pbtxt
index 8b7b33e98c..b27db4e7f2 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-add.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-add.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.merge.Add\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.merge._Merge\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-alpha-dropout.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-alpha-dropout.pbtxt
index 5e3e41ba20..50998ac9d6 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-alpha-dropout.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-alpha-dropout.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.AlphaDropout"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.noise.AlphaDropout\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling1-d.pbtxt
index e160b10153..be17aeafb5 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling1-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.AveragePooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling2-d.pbtxt
index b6b71358c8..7f21b444bc 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling2-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.AveragePooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling3-d.pbtxt
index 5c5ab1580e..2ac86f152f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average-pooling3-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.AveragePooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average.pbtxt
index 489de2e4d3..f6b1dd2f7e 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-average.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.merge.Average\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.merge._Merge\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool1-d.pbtxt
index 30fec249b8..3da1f43a92 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool1-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.AveragePooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool2-d.pbtxt
index 0e983c9234..a7be5ac818 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool2-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.AveragePooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool3-d.pbtxt
index ec50db7127..c5c29bead3 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-avg-pool3-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.AveragePooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-batch-normalization.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-batch-normalization.pbtxt
index cbbb000e25..3af3c2a501 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-batch-normalization.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-batch-normalization.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.normalization.BatchNormalizationV1\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.normalization.BatchNormalizationV2\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-bidirectional.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-bidirectional.pbtxt
index 23153d4284..880d18e1aa 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-bidirectional.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-bidirectional.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.wrappers.Bidirectional\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.wrappers.Wrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-concatenate.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-concatenate.pbtxt
index 766c3f267f..1eb0cf1a18 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-concatenate.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-concatenate.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.merge.Concatenate\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.merge._Merge\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt
index 8980982271..d9394e60f5 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional_recurrent.ConvRNN2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.RNN\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activation"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv1-d.pbtxt
index a74b8d2950..a0f6dc8097 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv1-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv1D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv2-d-transpose.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv2-d-transpose.pbtxt
index b093f8ead9..037b92f861 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv2-d-transpose.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv2-d-transpose.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv2-d.pbtxt
index 0ce9f6fd59..6a0d027d47 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv2-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv3-d-transpose.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv3-d-transpose.pbtxt
index c1f5bfae0d..66b5bd75fc 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv3-d-transpose.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv3-d-transpose.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv3-d.pbtxt
index 4aa872c4f0..e73133ff07 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-conv3-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution1-d.pbtxt
index 6e01f7c70c..7af6b2b3c3 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution1-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv1D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt
index c002042d77..baff492dfb 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution2-d.pbtxt
index f5e5446d2b..63d30a6185 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution2-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt
index d5f36f4bc3..7a29cbbec3 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution3-d.pbtxt
index 346fec6056..87c75c0224 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-convolution3-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping1-d.pbtxt
index 0f8fe9f05e..f69104ddfe 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping1-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Cropping1D"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Cropping1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping2-d.pbtxt
index 68fb7382a7..aa05471933 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping2-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Cropping2D"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Cropping2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping3-d.pbtxt
index deda82f9b3..d61f1ddc1d 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cropping3-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Cropping3D"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Cropping3D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cu-d-n-n-g-r-u.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cu-d-n-n-g-r-u.pbtxt
index 2eba3fb954..e2d05f8298 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cu-d-n-n-g-r-u.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cu-d-n-n-g-r-u.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.cudnn_recurrent._CuDNNRNN\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.RNN\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cu-d-n-n-l-s-t-m.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cu-d-n-n-l-s-t-m.pbtxt
index 6ed13d37f2..f650f48423 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cu-d-n-n-l-s-t-m.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-cu-d-n-n-l-s-t-m.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.cudnn_recurrent._CuDNNRNN\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.RNN\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dense.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dense.pbtxt
index 919aed5723..06e8b6b314 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dense.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dense.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Dense"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.Dense\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-depthwise-conv2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-depthwise-conv2-d.pbtxt
index f590ce1ef7..9fdf6f66d1 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-depthwise-conv2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-depthwise-conv2-d.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dot.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dot.pbtxt
index db4261fadc..cbe1020650 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dot.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dot.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.merge.Dot\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.merge._Merge\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dropout.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dropout.pbtxt
index 7369552b3b..0efba09b27 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dropout.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-dropout.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Dropout"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.Dropout\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-e-l-u.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-e-l-u.pbtxt
index f643ef9de2..b34c499eb2 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-e-l-u.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-e-l-u.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.ELU"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.advanced_activations.ELU\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-embedding.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-embedding.pbtxt
index ce053ae8c4..51dd853127 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-embedding.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-embedding.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Embedding"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.embeddings.Embedding\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-flatten.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-flatten.pbtxt
index db95043077..dcd18a9ced 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-flatten.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-flatten.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Flatten"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.Flatten\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-g-r-u-cell.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-g-r-u-cell.pbtxt
index a6edba6b7e..f029907ee8 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-g-r-u-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-g-r-u-cell.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.GRUCell"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.GRUCell\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-g-r-u.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-g-r-u.pbtxt
index f8c0dbb273..278ae06cba 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-g-r-u.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-g-r-u.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.GRU\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.RNN\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activation"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-gaussian-dropout.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-gaussian-dropout.pbtxt
index ac4bbe7d19..15cbcfe8ed 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-gaussian-dropout.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-gaussian-dropout.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.GaussianDropout"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.noise.GaussianDropout\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-gaussian-noise.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-gaussian-noise.pbtxt
index 947e3170ae..865b898c4c 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-gaussian-noise.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-gaussian-noise.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.GaussianNoise"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.noise.GaussianNoise\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt
index 17e202c581..3e17aca17c 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalAveragePooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt
index 9772c5df9b..b160687a2a 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalAveragePooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt
index cd65075591..70e8d51a5a 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalAveragePooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt
index 0423de7a24..809dc8554b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalAveragePooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt
index 4471cba245..3fbce8cb71 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalAveragePooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt
index c0e7fae456..70e4103ea1 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalAveragePooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool1-d.pbtxt
index 6975a6e88d..000bf54c45 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool1-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalMaxPooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool2-d.pbtxt
index 56bd70db7e..8ffbf07f9b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool2-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalMaxPooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool3-d.pbtxt
index 656319920e..3803d2b0a8 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pool3-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalMaxPooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt
index f815e66911..28668224e0 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalMaxPooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt
index f61f0e521b..b83ed67723 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalMaxPooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt
index c58c8ce63f..e689d69140 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalMaxPooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-input-layer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-input-layer.pbtxt
index 0efe9a4297..bb6eddae71 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-input-layer.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-input-layer.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.InputLayer"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.engine.input_layer.InputLayer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-l-s-t-m-cell.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-l-s-t-m-cell.pbtxt
index 5caa02e71a..5fb3f9dd3a 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-l-s-t-m-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-l-s-t-m-cell.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.LSTMCell"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.LSTMCell\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-l-s-t-m.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-l-s-t-m.pbtxt
index f21c7e5b21..8eb6dd9f4a 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-l-s-t-m.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-l-s-t-m.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.LSTM\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.RNN\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activation"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-lambda.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-lambda.pbtxt
index 381d5660b9..376bec0814 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-lambda.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-lambda.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Lambda"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.Lambda\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-layer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-layer.pbtxt
index 36b0a86628..c5f91a6338 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-layer.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-layer.pbtxt
@@ -1,7 +1,7 @@
 path: "tensorflow.keras.layers.Layer"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-leaky-re-l-u.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-leaky-re-l-u.pbtxt
index b41662e63a..bde8887359 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-leaky-re-l-u.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-leaky-re-l-u.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.LeakyReLU"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.advanced_activations.LeakyReLU\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-locally-connected1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-locally-connected1-d.pbtxt
index e4abfca913..16945f2c12 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-locally-connected1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-locally-connected1-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.LocallyConnected1D"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.local.LocallyConnected1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-locally-connected2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-locally-connected2-d.pbtxt
index cfcb92e293..f05741ffce 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-locally-connected2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-locally-connected2-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.LocallyConnected2D"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.local.LocallyConnected2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-masking.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-masking.pbtxt
index e0721353d1..7885db4ed2 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-masking.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-masking.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Masking"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.Masking\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool1-d.pbtxt
index 0618fbeead..9380d26cf4 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool1-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.MaxPooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool2-d.pbtxt
index 4af52ffec8..8eb8218df3 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool2-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.MaxPooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool3-d.pbtxt
index db9311ee58..0c96f86ed3 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pool3-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.MaxPooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling1-d.pbtxt
index bfb15cb447..0c6b230eb7 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling1-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.MaxPooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling2-d.pbtxt
index 1db962dbb8..eb7ca52fba 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling2-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.MaxPooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling3-d.pbtxt
index f80d5267e7..e724e9088f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-max-pooling3-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.MaxPooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-maximum.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-maximum.pbtxt
index cd772d4ac7..dafbd09ee2 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-maximum.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-maximum.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.merge.Maximum\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.merge._Merge\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-minimum.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-minimum.pbtxt
index 2bb6b3073a..3122fbec1c 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-minimum.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-minimum.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.merge.Minimum\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.merge._Merge\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-multiply.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-multiply.pbtxt
index e1a1f07355..0527cda1f0 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-multiply.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-multiply.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.merge.Multiply\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.merge._Merge\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-p-re-l-u.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-p-re-l-u.pbtxt
index 66c4446572..814e5a5d54 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-p-re-l-u.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-p-re-l-u.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.PReLU"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.advanced_activations.PReLU\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-permute.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-permute.pbtxt
index 0839554f43..aa1731afb8 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-permute.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-permute.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Permute"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.Permute\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-r-n-n.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-r-n-n.pbtxt
index b10695f6f7..9d7dd85fe0 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-r-n-n.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-r-n-n.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.RNN"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.RNN\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-re-l-u.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-re-l-u.pbtxt
index b96500f710..e9bba298bb 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-re-l-u.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-re-l-u.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.ReLU"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.advanced_activations.ReLU\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-repeat-vector.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-repeat-vector.pbtxt
index a27d93ec62..3c783eb512 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-repeat-vector.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-repeat-vector.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.RepeatVector"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.RepeatVector\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-reshape.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-reshape.pbtxt
index 6dda24d3d2..b8e0882541 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-reshape.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-reshape.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Reshape"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.Reshape\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-conv1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-conv1-d.pbtxt
index 8a4ae8aaa7..310f369ed6 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-conv1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-conv1-d.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.SeparableConv\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-conv2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-conv2-d.pbtxt
index a083c1da2e..df19d781c2 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-conv2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-conv2-d.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.SeparableConv\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-convolution1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-convolution1-d.pbtxt
index 5d5b361f82..bf909509bd 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-convolution1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-convolution1-d.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.SeparableConv\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-convolution2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-convolution2-d.pbtxt
index 392c338d73..5d66bc6fb6 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-convolution2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-separable-convolution2-d.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.SeparableConv\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-simple-r-n-n-cell.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-simple-r-n-n-cell.pbtxt
index 1143604903..88e9300de9 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-simple-r-n-n-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-simple-r-n-n-cell.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.SimpleRNNCell"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.SimpleRNNCell\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-simple-r-n-n.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-simple-r-n-n.pbtxt
index 5a15f1a55f..9d81c6d4bc 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-simple-r-n-n.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-simple-r-n-n.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.SimpleRNN\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.RNN\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activation"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-softmax.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-softmax.pbtxt
index c470d9c8e8..712eb0c6ec 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-softmax.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-softmax.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Softmax"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.advanced_activations.Softmax\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt
index d17d6495c0..dfc4ca2705 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.SpatialDropout1D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.core.Dropout\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt
index 2d538b4734..5e4f727f71 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.SpatialDropout2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.core.Dropout\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt
index b70923601a..9d893cb30a 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.SpatialDropout3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.core.Dropout\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-stacked-r-n-n-cells.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-stacked-r-n-n-cells.pbtxt
index f453ddd50e..a2ed954e4c 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-stacked-r-n-n-cells.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-stacked-r-n-n-cells.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.StackedRNNCells"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.StackedRNNCells\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-subtract.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-subtract.pbtxt
index 5759169e07..8a0818e78a 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-subtract.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-subtract.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.merge.Subtract\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.merge._Merge\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt
index bfde1c35f6..b5591b4826 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.ThresholdedReLU"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.advanced_activations.ThresholdedReLU\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-time-distributed.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-time-distributed.pbtxt
index e7f59a9cc5..210e4fd4e6 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-time-distributed.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-time-distributed.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.wrappers.TimeDistributed\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.wrappers.Wrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling1-d.pbtxt
index 0354149d4f..da2213a84f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling1-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.UpSampling1D"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.UpSampling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling2-d.pbtxt
index fff0e26bc1..e2c303d506 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling2-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.UpSampling2D"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.UpSampling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling3-d.pbtxt
index c49fa5663d..396e774c8a 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-up-sampling3-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.UpSampling3D"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.UpSampling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-wrapper.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-wrapper.pbtxt
index c961699053..8b6418d514 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-wrapper.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-wrapper.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Wrapper"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.wrappers.Wrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding1-d.pbtxt
index 1911e128eb..e8fda4c71a 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding1-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.ZeroPadding1D"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.ZeroPadding1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding2-d.pbtxt
index 88be914347..50c52d270b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding2-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.ZeroPadding2D"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.ZeroPadding2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding3-d.pbtxt
index 2bbb71ece2..84c6b78a2b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-zero-padding3-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.ZeroPadding3D"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.ZeroPadding3D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-accuracy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-accuracy.pbtxt
index e3e8dc00fe..6756beca1d 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-accuracy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-accuracy.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-binary-accuracy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-binary-accuracy.pbtxt
index 60b4f992c0..bec0b20aa5 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-binary-accuracy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-binary-accuracy.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-accuracy.pbtxt
index deab5a7a70..71dc294dc3 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-accuracy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-accuracy.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-hinge.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-hinge.pbtxt
index 10477e6662..43024e738a 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-hinge.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-categorical-hinge.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-cosine-proximity.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-cosine-proximity.pbtxt
index cdb3e1a79a..1e39385f81 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-cosine-proximity.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-cosine-proximity.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-negatives.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-negatives.pbtxt
index f4d067243d..5432f7f400 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-negatives.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-negatives.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics._ConfusionMatrixConditionCount\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-positives.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-positives.pbtxt
index f898e00eea..75541bf285 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-positives.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-false-positives.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics._ConfusionMatrixConditionCount\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-hinge.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-hinge.pbtxt
index 393ba964c4..f8d47f3771 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-hinge.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-hinge.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-absolute-error.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-absolute-error.pbtxt
index a86b26e618..bf7fc7cfc5 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-absolute-error.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-absolute-error.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-absolute-percentage-error.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-absolute-percentage-error.pbtxt
index 5e8f321a06..59bb767d35 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-absolute-percentage-error.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-absolute-percentage-error.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-squared-error.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-squared-error.pbtxt
index a2eb10c5f6..91f4712312 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-squared-error.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-squared-error.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-squared-logarithmic-error.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-squared-logarithmic-error.pbtxt
index 7df45a8caf..205e15d439 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-squared-logarithmic-error.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean-squared-logarithmic-error.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean.pbtxt
index 899502cfad..eec26ffce7 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-mean.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-precision.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-precision.pbtxt
index 3bdf3a4167..9aeaa5627a 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-precision.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-precision.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Precision\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-recall.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-recall.pbtxt
index 1ac2aacca0..748cec0866 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-recall.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-recall.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Recall\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sensitivity-at-specificity.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sensitivity-at-specificity.pbtxt
index 5c4f26929f..97aeb680be 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sensitivity-at-specificity.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sensitivity-at-specificity.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.SensitivitySpecificityBase\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt
index 48ac5c2fdc..571c2bf9d3 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-specificity-at-sensitivity.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-specificity-at-sensitivity.pbtxt
index 702f711722..85f80b062e 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-specificity-at-sensitivity.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-specificity-at-sensitivity.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.SensitivitySpecificityBase\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-squared-hinge.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-squared-hinge.pbtxt
index 6c3a9748c7..71c047e973 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-squared-hinge.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-squared-hinge.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-negatives.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-negatives.pbtxt
index 3f1a99b83b..4bc9383f6f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-negatives.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-negatives.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics._ConfusionMatrixConditionCount\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-positives.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-positives.pbtxt
index e5a4207a32..2eae4df0ae 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-positives.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.-true-positives.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics._ConfusionMatrixConditionCount\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-model.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-model.pbtxt
index 5885cd21c1..55bd285612 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-model.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-model.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.engine.training.Model\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.network.Network\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-sequential.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-sequential.pbtxt
index 935fa32f8c..a82424167e 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-sequential.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-sequential.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.engine.training.Model\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.network.Network\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adadelta.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adadelta.pbtxt
index 5426269793..0a56293e80 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adadelta.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adadelta.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.optimizers.Adadelta"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.adadelta.Adadelta\'>"
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "iterations"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adagrad.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adagrad.pbtxt
index c39fe6ba4f..14d0894e56 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adagrad.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adagrad.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.optimizers.Adagrad"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.adagrad.Adagrad\'>"
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "iterations"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adam.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adam.pbtxt
index 05d46d380b..fdb1ea838c 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adam.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adam.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.optimizers.Adam"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.adam.Adam\'>"
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "iterations"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adamax.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adamax.pbtxt
index 88fd96c71d..ece63ec168 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adamax.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adamax.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.optimizers.Adamax"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.adamax.Adamax\'>"
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "iterations"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-nadam.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-nadam.pbtxt
index 7b824e7f0b..f952f88b6d 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-nadam.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-nadam.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.optimizers.Nadam"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.nadam.Nadam\'>"
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "iterations"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-optimizer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-optimizer.pbtxt
index 58b7f27491..27bae902b0 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-optimizer.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-optimizer.pbtxt
@@ -1,7 +1,7 @@
 path: "tensorflow.keras.optimizers.Optimizer"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "iterations"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-r-m-sprop.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-r-m-sprop.pbtxt
index 8de796edde..e523443a00 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-r-m-sprop.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-r-m-sprop.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.optimizers.RMSprop"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.rmsprop.RMSprop\'>"
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "iterations"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-s-g-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-s-g-d.pbtxt
index 393eeb3d6c..d2721f8e92 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-s-g-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-s-g-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.optimizers.SGD"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.gradient_descent.SGD\'>"
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "iterations"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling1-d.pbtxt
index 85764cc8dc..6d826a8f8e 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling1-d.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling1D\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling2-d.pbtxt
index 259da2ad3e..9505c90aac 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling2-d.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling2D\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling3-d.pbtxt
index ffda9334cf..5b1b8f78dc 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-average-pooling3-d.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling3D\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-batch-normalization.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-batch-normalization.pbtxt
index 56a3fc3de7..ef4c57b694 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-batch-normalization.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-batch-normalization.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.normalization.BatchNormalizationV2\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv1-d.pbtxt
index d72f24b3d5..b5ee2e7302 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv1-d.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv2-d-transpose.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv2-d-transpose.pbtxt
index 72a7339368..57f6d7c7c0 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv2-d-transpose.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv2-d-transpose.pbtxt
@@ -6,7 +6,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv2-d.pbtxt
index 38a63df42d..88c616bd17 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv2-d.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv3-d-transpose.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv3-d-transpose.pbtxt
index 29620561f7..b70a907907 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv3-d-transpose.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv3-d-transpose.pbtxt
@@ -6,7 +6,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv3-d.pbtxt
index f1a2bcbb72..33e8765ce6 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-conv3-d.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-dense.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-dense.pbtxt
index d1e2d57570..1ac13b5791 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-dense.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-dense.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.Dense\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-dropout.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-dropout.pbtxt
index 92e40f6d96..77faa3c2b9 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-dropout.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-dropout.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.Dropout\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-flatten.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-flatten.pbtxt
index 087601a3c1..0b2631491b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-flatten.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-flatten.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.Flatten\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-layer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-layer.pbtxt
index b052c6bb0a..0a3414d20c 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-layer.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-layer.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.layers.Layer"
 tf_class {
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling1-d.pbtxt
index 9444a1bc76..ffc5cf1c8b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling1-d.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling1D\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling2-d.pbtxt
index 83dcb5e4e7..ff2cf2ba90 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling2-d.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling2D\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling3-d.pbtxt
index eb26e2220b..09c8a31a7b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-max-pooling3-d.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling3D\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-separable-conv1-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-separable-conv1-d.pbtxt
index 38d75e8bd5..549e13a7ac 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-separable-conv1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-separable-conv1-d.pbtxt
@@ -6,7 +6,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-separable-conv2-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-separable-conv2-d.pbtxt
index 90fc61cdfa..169ecdece5 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-separable-conv2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-separable-conv2-d.pbtxt
@@ -6,7 +6,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt
index adffc55227..4251206cda 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl.RNNCell\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-basic-r-n-n-cell.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-basic-r-n-n-cell.pbtxt
index 95746cc49c..20af24633a 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-basic-r-n-n-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-basic-r-n-n-cell.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl.RNNCell\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt
index 3547b66d19..3205c6a4dc 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl.RNNCell\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-dropout-wrapper.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-dropout-wrapper.pbtxt
index 7582fd52b6..14cf5ce456 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-dropout-wrapper.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-dropout-wrapper.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl.RNNCell\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-g-r-u-cell.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-g-r-u-cell.pbtxt
index 7ec61661fd..e43547b154 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-g-r-u-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-g-r-u-cell.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl.RNNCell\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-l-s-t-m-cell.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-l-s-t-m-cell.pbtxt
index 9617d07568..99381cd7e1 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-l-s-t-m-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-l-s-t-m-cell.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl.RNNCell\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt
index b31886f736..1fbde9df17 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl.RNNCell\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt
index c36ecaa4b2..8ba92fcc8d 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl.RNNCell\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt
index 42128ebd17..9de73076b1 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl.RNNCell\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.train.-adadelta-optimizer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.train.-adadelta-optimizer.pbtxt
index 1f1d8b6f9e..65a2b605d5 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.train.-adadelta-optimizer.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.train.-adadelta-optimizer.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.train.AdadeltaOptimizer"
 tf_class {
   is_instance: "<class \'tensorflow.python.training.adadelta.AdadeltaOptimizer\'>"
   is_instance: "<class \'tensorflow.python.training.optimizer.Optimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "GATE_GRAPH"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.train.-adagrad-d-a-optimizer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.train.-adagrad-d-a-optimizer.pbtxt
index a7c05d4849..179272d8a8 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.train.-adagrad-d-a-optimizer.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.train.-adagrad-d-a-optimizer.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.train.AdagradDAOptimizer"
 tf_class {
   is_instance: "<class \'tensorflow.python.training.adagrad_da.AdagradDAOptimizer\'>"
   is_instance: "<class \'tensorflow.python.training.optimizer.Optimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "GATE_GRAPH"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.train.-adagrad-optimizer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.train.-adagrad-optimizer.pbtxt
index bc8b92389c..15c2ef46c1 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.train.-adagrad-optimizer.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.train.-adagrad-optimizer.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.train.AdagradOptimizer"
 tf_class {
   is_instance: "<class \'tensorflow.python.training.adagrad.AdagradOptimizer\'>"
   is_instance: "<class \'tensorflow.python.training.optimizer.Optimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "GATE_GRAPH"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.train.-adam-optimizer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.train.-adam-optimizer.pbtxt
index 5d17be9378..9c902e582f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.train.-adam-optimizer.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.train.-adam-optimizer.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.train.AdamOptimizer"
 tf_class {
   is_instance: "<class \'tensorflow.python.training.adam.AdamOptimizer\'>"
   is_instance: "<class \'tensorflow.python.training.optimizer.Optimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "GATE_GRAPH"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.train.-checkpoint.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.train.-checkpoint.pbtxt
index 5be37200f3..42dcdac9e7 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.train.-checkpoint.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.train.-checkpoint.pbtxt
@@ -1,8 +1,8 @@
 path: "tensorflow.train.Checkpoint"
 tf_class {
   is_instance: "<class \'tensorflow.python.training.checkpointable.util.Checkpoint\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.tracking.Checkpointable\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.tracking.AutoCheckpointable\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "save_counter"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.train.-ftrl-optimizer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.train.-ftrl-optimizer.pbtxt
index d265fdeb01..f41d9f12d9 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.train.-ftrl-optimizer.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.train.-ftrl-optimizer.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.train.FtrlOptimizer"
 tf_class {
   is_instance: "<class \'tensorflow.python.training.ftrl.FtrlOptimizer\'>"
   is_instance: "<class \'tensorflow.python.training.optimizer.Optimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "GATE_GRAPH"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.train.-gradient-descent-optimizer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.train.-gradient-descent-optimizer.pbtxt
index c673e29cd4..7399750385 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.train.-gradient-descent-optimizer.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.train.-gradient-descent-optimizer.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.train.GradientDescentOptimizer"
 tf_class {
   is_instance: "<class \'tensorflow.python.training.gradient_descent.GradientDescentOptimizer\'>"
   is_instance: "<class \'tensorflow.python.training.optimizer.Optimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "GATE_GRAPH"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.train.-momentum-optimizer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.train.-momentum-optimizer.pbtxt
index 8199f63b9b..9bbaa14a6f 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.train.-momentum-optimizer.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.train.-momentum-optimizer.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.train.MomentumOptimizer"
 tf_class {
   is_instance: "<class \'tensorflow.python.training.momentum.MomentumOptimizer\'>"
   is_instance: "<class \'tensorflow.python.training.optimizer.Optimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "GATE_GRAPH"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.train.-optimizer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.train.-optimizer.pbtxt
index 876bb35e39..448e17a448 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.train.-optimizer.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.train.-optimizer.pbtxt
@@ -1,7 +1,7 @@
 path: "tensorflow.train.Optimizer"
 tf_class {
   is_instance: "<class \'tensorflow.python.training.optimizer.Optimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "GATE_GRAPH"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.train.-proximal-adagrad-optimizer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.train.-proximal-adagrad-optimizer.pbtxt
index 14349a74ef..eb1782e9ca 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.train.-proximal-adagrad-optimizer.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.train.-proximal-adagrad-optimizer.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.train.ProximalAdagradOptimizer"
 tf_class {
   is_instance: "<class \'tensorflow.python.training.proximal_adagrad.ProximalAdagradOptimizer\'>"
   is_instance: "<class \'tensorflow.python.training.optimizer.Optimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "GATE_GRAPH"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.train.-proximal-gradient-descent-optimizer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.train.-proximal-gradient-descent-optimizer.pbtxt
index 7d982dc51f..eb9a86183e 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.train.-proximal-gradient-descent-optimizer.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.train.-proximal-gradient-descent-optimizer.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.train.ProximalGradientDescentOptimizer"
 tf_class {
   is_instance: "<class \'tensorflow.python.training.proximal_gradient_descent.ProximalGradientDescentOptimizer\'>"
   is_instance: "<class \'tensorflow.python.training.optimizer.Optimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "GATE_GRAPH"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.train.-r-m-s-prop-optimizer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.train.-r-m-s-prop-optimizer.pbtxt
index 906384a287..2cf4c2e7ea 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.train.-r-m-s-prop-optimizer.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.train.-r-m-s-prop-optimizer.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.train.RMSPropOptimizer"
 tf_class {
   is_instance: "<class \'tensorflow.python.training.rmsprop.RMSPropOptimizer\'>"
   is_instance: "<class \'tensorflow.python.training.optimizer.Optimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "GATE_GRAPH"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.train.-sync-replicas-optimizer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.train.-sync-replicas-optimizer.pbtxt
index 2c0fda3c72..ecce08220d 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.train.-sync-replicas-optimizer.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.train.-sync-replicas-optimizer.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.train.SyncReplicasOptimizer"
 tf_class {
   is_instance: "<class \'tensorflow.python.training.sync_replicas_optimizer.SyncReplicasOptimizer\'>"
   is_instance: "<class \'tensorflow.python.training.optimizer.Optimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "GATE_GRAPH"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-variable.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-variable.pbtxt
index ee16ffa505..2206f0f8d8 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.-variable.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.-variable.pbtxt
@@ -1,7 +1,7 @@
 path: "tensorflow.Variable"
 tf_class {
   is_instance: "<class \'tensorflow.python.ops.variables.Variable\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "SaveSliceInfo"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.-model.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.-model.pbtxt
index eced2e1cb0..0a81c3a663 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.-model.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.-model.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.engine.training.Model\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.network.Network\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.-sequential.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.-sequential.pbtxt
index 2acb90173f..2a9ceec30e 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.-sequential.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.-sequential.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.engine.training.Model\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.network.Network\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.experimental.-peephole-l-s-t-m-cell.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.experimental.-peephole-l-s-t-m-cell.pbtxt
index b2ab5006dc..1bfd51cdcc 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.experimental.-peephole-l-s-t-m-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.experimental.-peephole-l-s-t-m-cell.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.PeepholeLSTMCell\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.LSTMCell\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-activation.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-activation.pbtxt
index da212382c1..8a0b8eb46f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-activation.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-activation.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Activation"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.Activation\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-activity-regularization.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-activity-regularization.pbtxt
index c910db027e..abb3c23694 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-activity-regularization.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-activity-regularization.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.ActivityRegularization"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.ActivityRegularization\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-add.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-add.pbtxt
index 8b7b33e98c..b27db4e7f2 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-add.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-add.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.merge.Add\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.merge._Merge\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-alpha-dropout.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-alpha-dropout.pbtxt
index 5e3e41ba20..50998ac9d6 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-alpha-dropout.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-alpha-dropout.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.AlphaDropout"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.noise.AlphaDropout\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling1-d.pbtxt
index e160b10153..be17aeafb5 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling1-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.AveragePooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling2-d.pbtxt
index b6b71358c8..7f21b444bc 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling2-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.AveragePooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling3-d.pbtxt
index 5c5ab1580e..2ac86f152f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average-pooling3-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.AveragePooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average.pbtxt
index 489de2e4d3..f6b1dd2f7e 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-average.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.merge.Average\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.merge._Merge\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool1-d.pbtxt
index 30fec249b8..3da1f43a92 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool1-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.AveragePooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool2-d.pbtxt
index 0e983c9234..a7be5ac818 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool2-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.AveragePooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool3-d.pbtxt
index ec50db7127..c5c29bead3 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-avg-pool3-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.AveragePooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-batch-normalization.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-batch-normalization.pbtxt
index 36ea9d5851..b13f963a6f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-batch-normalization.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-batch-normalization.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.BatchNormalization"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.normalization.BatchNormalizationV2\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-bidirectional.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-bidirectional.pbtxt
index 23153d4284..880d18e1aa 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-bidirectional.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-bidirectional.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.wrappers.Bidirectional\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.wrappers.Wrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-concatenate.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-concatenate.pbtxt
index 766c3f267f..1eb0cf1a18 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-concatenate.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-concatenate.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.merge.Concatenate\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.merge._Merge\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt
index 8980982271..d9394e60f5 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional_recurrent.ConvRNN2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.RNN\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activation"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv1-d.pbtxt
index a74b8d2950..a0f6dc8097 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv1-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv1D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv2-d-transpose.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv2-d-transpose.pbtxt
index b093f8ead9..037b92f861 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv2-d-transpose.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv2-d-transpose.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv2-d.pbtxt
index 0ce9f6fd59..6a0d027d47 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv2-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv3-d-transpose.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv3-d-transpose.pbtxt
index c1f5bfae0d..66b5bd75fc 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv3-d-transpose.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv3-d-transpose.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv3-d.pbtxt
index 4aa872c4f0..e73133ff07 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-conv3-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution1-d.pbtxt
index 6e01f7c70c..7af6b2b3c3 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution1-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv1D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt
index c002042d77..baff492dfb 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution2-d.pbtxt
index f5e5446d2b..63d30a6185 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution2-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt
index d5f36f4bc3..7a29cbbec3 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution3-d.pbtxt
index 346fec6056..87c75c0224 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-convolution3-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping1-d.pbtxt
index 0f8fe9f05e..f69104ddfe 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping1-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Cropping1D"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Cropping1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping2-d.pbtxt
index 68fb7382a7..aa05471933 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping2-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Cropping2D"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Cropping2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping3-d.pbtxt
index deda82f9b3..d61f1ddc1d 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-cropping3-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Cropping3D"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Cropping3D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense-features.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense-features.pbtxt
index ff00ca1bb2..28c1926bd7 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense-features.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense-features.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.DenseFeatures"
 tf_class {
   is_instance: "<class \'tensorflow.python.feature_column.feature_column_v2.DenseFeatures\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense.pbtxt
index 919aed5723..06e8b6b314 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Dense"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.Dense\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-depthwise-conv2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-depthwise-conv2-d.pbtxt
index f590ce1ef7..9fdf6f66d1 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-depthwise-conv2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-depthwise-conv2-d.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dot.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dot.pbtxt
index db4261fadc..cbe1020650 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dot.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dot.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.merge.Dot\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.merge._Merge\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dropout.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dropout.pbtxt
index 7369552b3b..0efba09b27 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dropout.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dropout.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Dropout"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.Dropout\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-e-l-u.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-e-l-u.pbtxt
index f643ef9de2..b34c499eb2 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-e-l-u.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-e-l-u.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.ELU"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.advanced_activations.ELU\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-embedding.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-embedding.pbtxt
index ce053ae8c4..51dd853127 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-embedding.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-embedding.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Embedding"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.embeddings.Embedding\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-flatten.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-flatten.pbtxt
index db95043077..dcd18a9ced 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-flatten.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-flatten.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Flatten"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.Flatten\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-g-r-u-cell.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-g-r-u-cell.pbtxt
index a6edba6b7e..f029907ee8 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-g-r-u-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-g-r-u-cell.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.GRUCell"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.GRUCell\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-g-r-u.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-g-r-u.pbtxt
index df2ea3fbe9..ac2d8c9aa3 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-g-r-u.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-g-r-u.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.GRU\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.RNN\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activation"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-gaussian-dropout.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-gaussian-dropout.pbtxt
index ac4bbe7d19..15cbcfe8ed 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-gaussian-dropout.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-gaussian-dropout.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.GaussianDropout"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.noise.GaussianDropout\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-gaussian-noise.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-gaussian-noise.pbtxt
index 947e3170ae..865b898c4c 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-gaussian-noise.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-gaussian-noise.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.GaussianNoise"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.noise.GaussianNoise\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt
index 17e202c581..3e17aca17c 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalAveragePooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt
index 9772c5df9b..b160687a2a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalAveragePooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt
index cd65075591..70e8d51a5a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalAveragePooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt
index 0423de7a24..809dc8554b 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalAveragePooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt
index 4471cba245..3fbce8cb71 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalAveragePooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt
index c0e7fae456..70e4103ea1 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalAveragePooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool1-d.pbtxt
index 6975a6e88d..000bf54c45 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool1-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalMaxPooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool2-d.pbtxt
index 56bd70db7e..8ffbf07f9b 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool2-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalMaxPooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool3-d.pbtxt
index 656319920e..3803d2b0a8 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pool3-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalMaxPooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt
index f815e66911..28668224e0 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalMaxPooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt
index f61f0e521b..b83ed67723 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalMaxPooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt
index c58c8ce63f..e689d69140 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalMaxPooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.GlobalPooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-input-layer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-input-layer.pbtxt
index 0efe9a4297..bb6eddae71 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-input-layer.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-input-layer.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.InputLayer"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.engine.input_layer.InputLayer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-l-s-t-m-cell.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-l-s-t-m-cell.pbtxt
index 5caa02e71a..5fb3f9dd3a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-l-s-t-m-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-l-s-t-m-cell.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.LSTMCell"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.LSTMCell\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-l-s-t-m.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-l-s-t-m.pbtxt
index 33082a6f06..89dfc2a256 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-l-s-t-m.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-l-s-t-m.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.LSTM\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.RNN\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activation"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-lambda.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-lambda.pbtxt
index 381d5660b9..376bec0814 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-lambda.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-lambda.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Lambda"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.Lambda\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-layer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-layer.pbtxt
index 36b0a86628..c5f91a6338 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-layer.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-layer.pbtxt
@@ -1,7 +1,7 @@
 path: "tensorflow.keras.layers.Layer"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-leaky-re-l-u.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-leaky-re-l-u.pbtxt
index b41662e63a..bde8887359 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-leaky-re-l-u.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-leaky-re-l-u.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.LeakyReLU"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.advanced_activations.LeakyReLU\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-linear-model.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-linear-model.pbtxt
index 5766528b31..d40a999bd8 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-linear-model.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-linear-model.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.engine.training.Model\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.network.Network\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-locally-connected1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-locally-connected1-d.pbtxt
index e4abfca913..16945f2c12 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-locally-connected1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-locally-connected1-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.LocallyConnected1D"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.local.LocallyConnected1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-locally-connected2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-locally-connected2-d.pbtxt
index cfcb92e293..f05741ffce 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-locally-connected2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-locally-connected2-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.LocallyConnected2D"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.local.LocallyConnected2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-masking.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-masking.pbtxt
index e0721353d1..7885db4ed2 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-masking.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-masking.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Masking"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.Masking\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool1-d.pbtxt
index 0618fbeead..9380d26cf4 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool1-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.MaxPooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool2-d.pbtxt
index 4af52ffec8..8eb8218df3 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool2-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.MaxPooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool3-d.pbtxt
index db9311ee58..0c96f86ed3 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pool3-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.MaxPooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling1-d.pbtxt
index bfb15cb447..0c6b230eb7 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling1-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.MaxPooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling2-d.pbtxt
index 1db962dbb8..eb7ca52fba 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling2-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.MaxPooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling3-d.pbtxt
index f80d5267e7..e724e9088f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-max-pooling3-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.MaxPooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.pooling.Pooling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-maximum.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-maximum.pbtxt
index cd772d4ac7..dafbd09ee2 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-maximum.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-maximum.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.merge.Maximum\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.merge._Merge\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-minimum.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-minimum.pbtxt
index 2bb6b3073a..3122fbec1c 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-minimum.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-minimum.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.merge.Minimum\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.merge._Merge\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-multiply.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-multiply.pbtxt
index e1a1f07355..0527cda1f0 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-multiply.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-multiply.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.merge.Multiply\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.merge._Merge\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-p-re-l-u.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-p-re-l-u.pbtxt
index 66c4446572..814e5a5d54 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-p-re-l-u.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-p-re-l-u.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.PReLU"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.advanced_activations.PReLU\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-permute.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-permute.pbtxt
index 0839554f43..aa1731afb8 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-permute.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-permute.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Permute"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.Permute\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-r-n-n.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-r-n-n.pbtxt
index b10695f6f7..9d7dd85fe0 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-r-n-n.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-r-n-n.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.RNN"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.RNN\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-re-l-u.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-re-l-u.pbtxt
index b96500f710..e9bba298bb 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-re-l-u.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-re-l-u.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.ReLU"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.advanced_activations.ReLU\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-repeat-vector.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-repeat-vector.pbtxt
index a27d93ec62..3c783eb512 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-repeat-vector.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-repeat-vector.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.RepeatVector"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.RepeatVector\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-reshape.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-reshape.pbtxt
index 6dda24d3d2..b8e0882541 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-reshape.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-reshape.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Reshape"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.Reshape\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-conv1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-conv1-d.pbtxt
index 8a4ae8aaa7..310f369ed6 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-conv1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-conv1-d.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.SeparableConv\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-conv2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-conv2-d.pbtxt
index a083c1da2e..df19d781c2 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-conv2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-conv2-d.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.SeparableConv\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-convolution1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-convolution1-d.pbtxt
index 5d5b361f82..bf909509bd 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-convolution1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-convolution1-d.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.SeparableConv\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-convolution2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-convolution2-d.pbtxt
index 392c338d73..5d66bc6fb6 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-convolution2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-separable-convolution2-d.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.SeparableConv\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.Conv\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-simple-r-n-n-cell.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-simple-r-n-n-cell.pbtxt
index 1143604903..88e9300de9 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-simple-r-n-n-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-simple-r-n-n-cell.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.SimpleRNNCell"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.SimpleRNNCell\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-simple-r-n-n.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-simple-r-n-n.pbtxt
index 5a15f1a55f..9d81c6d4bc 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-simple-r-n-n.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-simple-r-n-n.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.SimpleRNN\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.RNN\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activation"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-softmax.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-softmax.pbtxt
index c470d9c8e8..712eb0c6ec 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-softmax.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-softmax.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Softmax"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.advanced_activations.Softmax\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt
index d17d6495c0..dfc4ca2705 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.SpatialDropout1D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.core.Dropout\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt
index 2d538b4734..5e4f727f71 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.SpatialDropout2D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.core.Dropout\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt
index b70923601a..9d893cb30a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.core.SpatialDropout3D\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.core.Dropout\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-stacked-r-n-n-cells.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-stacked-r-n-n-cells.pbtxt
index f453ddd50e..a2ed954e4c 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-stacked-r-n-n-cells.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-stacked-r-n-n-cells.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.StackedRNNCells"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.recurrent.StackedRNNCells\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-subtract.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-subtract.pbtxt
index 5759169e07..8a0818e78a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-subtract.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-subtract.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.merge.Subtract\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.merge._Merge\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt
index bfde1c35f6..b5591b4826 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.ThresholdedReLU"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.advanced_activations.ThresholdedReLU\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-time-distributed.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-time-distributed.pbtxt
index e7f59a9cc5..210e4fd4e6 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-time-distributed.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-time-distributed.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.wrappers.TimeDistributed\'>"
   is_instance: "<class \'tensorflow.python.keras.layers.wrappers.Wrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling1-d.pbtxt
index 0354149d4f..da2213a84f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling1-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.UpSampling1D"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.UpSampling1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling2-d.pbtxt
index fff0e26bc1..e2c303d506 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling2-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.UpSampling2D"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.UpSampling2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling3-d.pbtxt
index c49fa5663d..396e774c8a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-up-sampling3-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.UpSampling3D"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.UpSampling3D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-wrapper.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-wrapper.pbtxt
index c961699053..8b6418d514 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-wrapper.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-wrapper.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.Wrapper"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.wrappers.Wrapper\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding1-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding1-d.pbtxt
index 1911e128eb..e8fda4c71a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding1-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding1-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.ZeroPadding1D"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.ZeroPadding1D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding2-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding2-d.pbtxt
index 88be914347..50c52d270b 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding2-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding2-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.ZeroPadding2D"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.ZeroPadding2D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding3-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding3-d.pbtxt
index 2bbb71ece2..84c6b78a2b 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding3-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-zero-padding3-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.layers.ZeroPadding3D"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.layers.convolutional.ZeroPadding3D\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-accuracy.pbtxt
index e3e8dc00fe..6756beca1d 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-accuracy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-accuracy.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-binary-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-binary-accuracy.pbtxt
index 60b4f992c0..bec0b20aa5 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-binary-accuracy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-binary-accuracy.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-accuracy.pbtxt
index deab5a7a70..71dc294dc3 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-accuracy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-accuracy.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-hinge.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-hinge.pbtxt
index 10477e6662..43024e738a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-hinge.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-categorical-hinge.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-cosine-proximity.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-cosine-proximity.pbtxt
index cdb3e1a79a..1e39385f81 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-cosine-proximity.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-cosine-proximity.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-negatives.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-negatives.pbtxt
index f4d067243d..5432f7f400 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-negatives.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-negatives.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics._ConfusionMatrixConditionCount\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-positives.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-positives.pbtxt
index f898e00eea..75541bf285 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-positives.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-false-positives.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics._ConfusionMatrixConditionCount\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-hinge.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-hinge.pbtxt
index 393ba964c4..f8d47f3771 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-hinge.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-hinge.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-absolute-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-absolute-error.pbtxt
index a86b26e618..bf7fc7cfc5 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-absolute-error.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-absolute-error.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-absolute-percentage-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-absolute-percentage-error.pbtxt
index 5e8f321a06..59bb767d35 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-absolute-percentage-error.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-absolute-percentage-error.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-squared-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-squared-error.pbtxt
index a2eb10c5f6..91f4712312 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-squared-error.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-squared-error.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-squared-logarithmic-error.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-squared-logarithmic-error.pbtxt
index 7df45a8caf..205e15d439 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-squared-logarithmic-error.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean-squared-logarithmic-error.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean.pbtxt
index 899502cfad..eec26ffce7 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-mean.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-precision.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-precision.pbtxt
index 3bdf3a4167..9aeaa5627a 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-precision.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-precision.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Precision\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-recall.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-recall.pbtxt
index 1ac2aacca0..748cec0866 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-recall.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-recall.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Recall\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sensitivity-at-specificity.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sensitivity-at-specificity.pbtxt
index 5c4f26929f..97aeb680be 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sensitivity-at-specificity.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sensitivity-at-specificity.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.SensitivitySpecificityBase\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt
index 48ac5c2fdc..571c2bf9d3 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-sparse-categorical-accuracy.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-specificity-at-sensitivity.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-specificity-at-sensitivity.pbtxt
index 702f711722..85f80b062e 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-specificity-at-sensitivity.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-specificity-at-sensitivity.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.SensitivitySpecificityBase\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-squared-hinge.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-squared-hinge.pbtxt
index 6c3a9748c7..71c047e973 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-squared-hinge.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-squared-hinge.pbtxt
@@ -5,7 +5,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics.Mean\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-negatives.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-negatives.pbtxt
index 3f1a99b83b..4bc9383f6f 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-negatives.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-negatives.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics._ConfusionMatrixConditionCount\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-positives.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-positives.pbtxt
index e5a4207a32..2eae4df0ae 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-positives.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.-true-positives.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.metrics._ConfusionMatrixConditionCount\'>"
   is_instance: "<class \'tensorflow.python.keras.metrics.Metric\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-model.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-model.pbtxt
index 5885cd21c1..55bd285612 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-model.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-model.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.engine.training.Model\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.network.Network\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-sequential.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-sequential.pbtxt
index 935fa32f8c..a82424167e 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-sequential.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-sequential.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.keras.engine.training.Model\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.network.Network\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adadelta.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adadelta.pbtxt
index 5426269793..0a56293e80 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adadelta.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adadelta.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.optimizers.Adadelta"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.adadelta.Adadelta\'>"
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "iterations"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adagrad.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adagrad.pbtxt
index c39fe6ba4f..14d0894e56 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adagrad.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adagrad.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.optimizers.Adagrad"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.adagrad.Adagrad\'>"
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "iterations"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adam.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adam.pbtxt
index 05d46d380b..fdb1ea838c 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adam.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adam.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.optimizers.Adam"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.adam.Adam\'>"
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "iterations"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adamax.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adamax.pbtxt
index 88fd96c71d..ece63ec168 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adamax.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adamax.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.optimizers.Adamax"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.adamax.Adamax\'>"
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "iterations"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-nadam.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-nadam.pbtxt
index 7b824e7f0b..f952f88b6d 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-nadam.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-nadam.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.optimizers.Nadam"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.nadam.Nadam\'>"
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "iterations"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-optimizer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-optimizer.pbtxt
index 58b7f27491..27bae902b0 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-optimizer.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-optimizer.pbtxt
@@ -1,7 +1,7 @@
 path: "tensorflow.keras.optimizers.Optimizer"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "iterations"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-r-m-sprop.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-r-m-sprop.pbtxt
index 8de796edde..e523443a00 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-r-m-sprop.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-r-m-sprop.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.optimizers.RMSprop"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.rmsprop.RMSprop\'>"
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "iterations"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-s-g-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-s-g-d.pbtxt
index 393eeb3d6c..d2721f8e92 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-s-g-d.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-s-g-d.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.keras.optimizers.SGD"
 tf_class {
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.gradient_descent.SGD\'>"
   is_instance: "<class \'tensorflow.python.keras.optimizer_v2.optimizer_v2.OptimizerV2\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "iterations"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt
index 3547b66d19..3205c6a4dc 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl.RNNCell\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt
index c36ecaa4b2..8ba92fcc8d 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt
@@ -3,7 +3,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl.RNNCell\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt
index 42128ebd17..9de73076b1 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl.RNNCell\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.rnn.-dropout-wrapper.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.rnn.-dropout-wrapper.pbtxt
index 7721eed65b..7781337c82 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.rnn.-dropout-wrapper.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.rnn.-dropout-wrapper.pbtxt
@@ -6,7 +6,7 @@ tf_class {
   is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl.RNNCell\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "activity_regularizer"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-checkpoint.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-checkpoint.pbtxt
index 5be37200f3..42dcdac9e7 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.-checkpoint.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.train.-checkpoint.pbtxt
@@ -1,8 +1,8 @@
 path: "tensorflow.train.Checkpoint"
 tf_class {
   is_instance: "<class \'tensorflow.python.training.checkpointable.util.Checkpoint\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.tracking.Checkpointable\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.tracking.AutoCheckpointable\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "save_counter"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-proximal-gradient-descent-optimizer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-proximal-gradient-descent-optimizer.pbtxt
index 7d982dc51f..eb9a86183e 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.-proximal-gradient-descent-optimizer.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.train.-proximal-gradient-descent-optimizer.pbtxt
@@ -2,7 +2,7 @@ path: "tensorflow.train.ProximalGradientDescentOptimizer"
 tf_class {
   is_instance: "<class \'tensorflow.python.training.proximal_gradient_descent.ProximalGradientDescentOptimizer\'>"
   is_instance: "<class \'tensorflow.python.training.optimizer.Optimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.CheckpointableBase\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "GATE_GRAPH"
-- 
GitLab


From 5b2fa842fafcffd7fb1251a1e550d8b6ff7a1467 Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Tue, 15 Jan 2019 16:28:44 -0800
Subject: [PATCH 0781/1095] [Grappler] Preserve function execution semantics
 wrt to Dataset ops.

PiperOrigin-RevId: 229463147
---
 tensorflow/core/grappler/grappler_item.cc     | 16 ++---
 tensorflow/core/grappler/grappler_item.h      | 24 +++----
 tensorflow/core/grappler/op_types.cc          |  7 +++
 tensorflow/core/grappler/op_types.h           |  3 +
 .../optimizers/arithmetic_optimizer.cc        |  2 +-
 .../grappler/optimizers/function_optimizer.cc | 26 ++++----
 .../grappler/optimizers/meta_optimizer.cc     | 12 ++--
 .../optimizers/meta_optimizer_test.cc         | 62 +++++++++----------
 tensorflow/core/grappler/utils/functions.cc   |  6 +-
 .../core/grappler/utils/functions_test.cc     |  2 +-
 10 files changed, 89 insertions(+), 71 deletions(-)

diff --git a/tensorflow/core/grappler/grappler_item.cc b/tensorflow/core/grappler/grappler_item.cc
index 2d71ac54cc..1323dd9a64 100644
--- a/tensorflow/core/grappler/grappler_item.cc
+++ b/tensorflow/core/grappler/grappler_item.cc
@@ -43,7 +43,7 @@ GrapplerItem GrapplerItem::WithGraph(GraphDef&& graph_def) const {
   item.save_restore_loc_tensor = save_restore_loc_tensor;
   item.queue_runners = queue_runners;
   item.devices_ = devices_;
-  item.allowed_optimizations_ = allowed_optimizations_;
+  item.optimization_options_ = optimization_options_;
   item.graph.Swap(&graph_def);
   return item;
 }
@@ -115,9 +115,11 @@ std::unordered_set<string> GrapplerItem::NodesToPreserve() const {
     }
   }
 
-  if (!allowed_optimizations_.prune_ops_with_side_effects) {
+  // Tensorflow functions do not prune side effects, or dataset-output ops from
+  // the function body (see PruneFunctionBody in common_runtime/function.cc).
+  if (optimization_options_.is_function_instantiation) {
     for (const NodeDef& node : graph.node()) {
-      if (!IsFreeOfSideEffect(node)) {
+      if (!IsFreeOfSideEffect(node) || IsDataset(node)) {
         result.insert(node.name());
       }
     }
@@ -175,13 +177,13 @@ Status GrapplerItem::InferDevicesFromGraph() {
 
 void GrapplerItem::ClearDevices() { devices_.clear(); }
 
-const GrapplerItem::AllowedOptimizations& GrapplerItem::allowed_optimizations()
+const GrapplerItem::OptimizationOptions& GrapplerItem::optimization_options()
     const {
-  return allowed_optimizations_;
+  return optimization_options_;
 }
 
-GrapplerItem::AllowedOptimizations& GrapplerItem::allowed_optimizations() {
-  return allowed_optimizations_;
+GrapplerItem::OptimizationOptions& GrapplerItem::optimization_options() {
+  return optimization_options_;
 }
 
 std::vector<const NodeDef*> ComputeTransitiveFanin(
diff --git a/tensorflow/core/grappler/grappler_item.h b/tensorflow/core/grappler/grappler_item.h
index 1ae551f5ac..75712e9f92 100644
--- a/tensorflow/core/grappler/grappler_item.h
+++ b/tensorflow/core/grappler/grappler_item.h
@@ -81,17 +81,17 @@ struct GrapplerItem {
   // fetch nodes, keep_ops, init_ops.
   std::unordered_set<string> NodesToPreserve() const;
 
-  // Restrict types of optimizations that are allowed for this GrapplerItem.
-  struct AllowedOptimizations {
+  struct OptimizationOptions {
     // Is it allowed to add nodes to the graph that do not have registered
     // gradient function.
-    bool non_differentiable_rewrites = true;
-
-    // By default we are allowed to prune ops with side-effects from the main
-    // graph if they are not in transitive fanin of the fetch nodes. If we are
-    // optimizing a graph that was instantiated by a function definition, we
-    // must keep all side effects intact.
-    bool prune_ops_with_side_effects = true;
+    bool allow_non_differentiable_rewrites = true;
+
+    // Tensorflow function execution semantics is slightly different from the
+    // main Tensorflow graph, and we need to make sure that we do not change it
+    // by running Grappler optimizer passes. One main difference is that
+    // functions do not prune ops with side-effects and dataset-output ops (see
+    // PruneFunctionBody in common_runtime/function.cc).
+    bool is_function_instantiation = false;
   };
 
   const std::unordered_set<string>& devices() const;
@@ -108,8 +108,8 @@ struct GrapplerItem {
   // Clears a set of available devices.
   void ClearDevices();
 
-  const AllowedOptimizations& allowed_optimizations() const;
-  AllowedOptimizations& allowed_optimizations();
+  const OptimizationOptions& optimization_options() const;
+  OptimizationOptions& optimization_options();
 
  private:
   // TODO(ezhulenev) Make GrapplerItem a class and hide all public data members.
@@ -120,7 +120,7 @@ struct GrapplerItem {
   // Example of a fully defined name: "/job:work/replica:1/task:1/device:CPU:0"
   std::unordered_set<string> devices_;
 
-  AllowedOptimizations allowed_optimizations_;
+  OptimizationOptions optimization_options_;
 };
 
 // Return the transitive fanin of a set of terminal nodes.
diff --git a/tensorflow/core/grappler/op_types.cc b/tensorflow/core/grappler/op_types.cc
index b201c3a717..e06580cebf 100644
--- a/tensorflow/core/grappler/op_types.cc
+++ b/tensorflow/core/grappler/op_types.cc
@@ -561,6 +561,13 @@ bool MaybeHasRefInput(const NodeDef& node) {
   return false;
 }
 
+bool IsDataset(const NodeDef& node) {
+  const string& op = node.op();
+  // See `GetNodeClassForOp` in core/graph/graph.cc.
+  return op == "IteratorGetNext" || op == "IteratorGetNextSync" ||
+         op == "DatasetToSingleElement" || op == "ReduceDataset";
+}
+
 bool IsFreeOfSideEffect(const NodeDef& node,
                         const OpRegistryInterface* op_registry) {
   // Placeholders must be preserved to keep the graph feedable.
diff --git a/tensorflow/core/grappler/op_types.h b/tensorflow/core/grappler/op_types.h
index cb7781ec6e..092748f4ff 100644
--- a/tensorflow/core/grappler/op_types.h
+++ b/tensorflow/core/grappler/op_types.h
@@ -183,6 +183,9 @@ bool IsCommutative(const NodeDef& node);
 // value.
 bool IsPersistent(const NodeDef& node);
 
+// Returns true if the node belongs to the NC_DATASET class (see graph/graph.h).
+bool IsDataset(const NodeDef& node);
+
 bool IsFreeOfSideEffect(const NodeDef& node,
                         const OpRegistryInterface* op_registry);
 bool IsFreeOfSideEffect(const NodeDef& node);  // use OpRegistry::Global()
diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
index e85e0473b3..2168dbd623 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
@@ -3576,7 +3576,7 @@ Status ArithmeticOptimizer::Optimize(Cluster* /*cluster*/,
 
   // Disable restricted graph rewrites.
   options_.unary_ops_composition &=
-      item.allowed_optimizations().non_differentiable_rewrites;
+      item.optimization_options().allow_non_differentiable_rewrites;
 
   if (options_.dedup_computations) {
     DedupComputations();
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.cc b/tensorflow/core/grappler/optimizers/function_optimizer.cc
index 1394aa5047..794e87dc38 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer.cc
@@ -260,7 +260,7 @@ class FunctionOptimizerContext {
       : grappler_item_id_(item.id),
         graph_version_(item.graph.versions().producer()),
         opt_level_(opt_level),
-        allowed_optimizations_(item.allowed_optimizations()),
+        optimization_options_(item.optimization_options()),
         function_library_(OpRegistry::Global(), item.graph.library()),
         available_device_names_(item.devices().begin(), item.devices().end()),
         graph_view_(&item.graph) {
@@ -272,8 +272,8 @@ class FunctionOptimizerContext {
 
   const RewriterConfig::Toggle opt_level() const { return opt_level_; }
 
-  const GrapplerItem::AllowedOptimizations& allowed_optimizations() const {
-    return allowed_optimizations_;
+  const GrapplerItem::OptimizationOptions& optimization_options() const {
+    return optimization_options_;
   }
 
   const FunctionLibraryDefinition& function_library() const {
@@ -409,7 +409,7 @@ class FunctionOptimizerContext {
   const string grappler_item_id_;
   const int graph_version_;
   const RewriterConfig::Toggle opt_level_;
-  const GrapplerItem::AllowedOptimizations allowed_optimizations_;
+  const GrapplerItem::OptimizationOptions optimization_options_;
   FunctionLibraryDefinition function_library_;
 
   // These fields initialized lazily only if needed.
@@ -1620,21 +1620,25 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
   // edges from inlined side-effectful ops.
   std::vector<string> side_effectful_nodes;
 
-  // We have to make sure that all side-effectful nodes inside a function body
-  // will be executed after function inlining.
+  // We have to make sure that all side-effectful and dataset-output nodes
+  // inside a function body will be executed after function inlining.
   for (NodeDef& func_body_node : *placed_graph_def.mutable_node()) {
-    if (!IsFreeOfSideEffect(func_body_node, &ctx->function_library())) {
+    const bool node_must_execute =
+        !IsFreeOfSideEffect(func_body_node, &ctx->function_library()) ||
+        IsDataset(func_body_node);
+
+    if (node_must_execute) {
       int num_fanouts = placed_graph_view.NumFanouts(
           func_body_node, /*include_controlled_nodes=*/true);
 
       // If the node doesn't have any outgoing edges and we do not have any
       // nodes in the `happens_after` set, we can't inline a function and
-      // guarantee that side-effects will be executed. The only exception if we
-      // do function library optimization, and the GrapplerItem was constructed
-      // for the function body, because functions have strict semantics.
+      // guarantee that it will be executed. The only exception if we do
+      // function library optimization, and the GrapplerItem was instantiated
+      // for the function body, because functions do not prune these ops.
 
       if (num_fanouts == 0 && happens_after.empty() &&
-          ctx->allowed_optimizations().prune_ops_with_side_effects) {
+          !ctx->optimization_options().is_function_instantiation) {
         return errors::Internal(
             "Can't inline a function with a side-effectful op with empty "
             "fanouts and empty output control edge set. Function body node: ",
diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer.cc b/tensorflow/core/grappler/optimizers/meta_optimizer.cc
index a84bb1d62f..c200b7db07 100644
--- a/tensorflow/core/grappler/optimizers/meta_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/meta_optimizer.cc
@@ -527,7 +527,8 @@ Status MetaOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
       // can't perform non-differentiable rewrites.
       if (differentiable_functions.find(func_name) !=
           differentiable_functions.end()) {
-        func_item.allowed_optimizations().non_differentiable_rewrites = false;
+        func_item.optimization_options().allow_non_differentiable_rewrites =
+            false;
       }
 
       // Function item is allowed to use all devices from the main graph.
@@ -536,10 +537,11 @@ Status MetaOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
         VLOG(3) << added_devices.error_message();
       }
 
-      // We are not allowed to prune side effects from the graph instantiated
-      // by the function definition, because we must guarantee function
-      // execution semantics wrt side effects (see function_optimizer.cc).
-      func_item.allowed_optimizations().prune_ops_with_side_effects = false;
+      // We are not allowed to prune certain types of ops from the graph
+      // instantiated by the function definition, because we must guarantee
+      // function execution semantics wrt side effects (see
+      // function_optimizer.cc).
+      func_item.optimization_options().is_function_instantiation = true;
 
       // Optimize function body graph.
       GraphDef optimized_func_graph;
diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc b/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc
index a061f6194a..b1b72075ca 100644
--- a/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc
@@ -87,12 +87,12 @@ REGISTER_GRAPH_OPTIMIZER(TestOptimizerWithParams);
 // Record various properties of the GrapplerItems passed for optimization.
 class GrapplerItemPropertiesAccumulator : public CustomGraphOptimizer {
  public:
-  static void SetAllowedOptimizations(
-      gtl::FlatMap<string, GrapplerItem::AllowedOptimizations>*
-          allowed_optimizations) {
-    allowed_optimizations_ = allowed_optimizations;
+  static void SetOptimizationOptions(
+      gtl::FlatMap<string, GrapplerItem::OptimizationOptions>*
+          optimization_options) {
+    optimization_options_ = optimization_options;
   }
-  static void ResetAllowedOptimizations() { allowed_optimizations_ = nullptr; }
+  static void ResetOptimizationOptions() { optimization_options_ = nullptr; }
 
   GrapplerItemPropertiesAccumulator() {}
   string name() const override {
@@ -107,8 +107,8 @@ class GrapplerItemPropertiesAccumulator : public CustomGraphOptimizer {
   Status Optimize(Cluster* cluster, const GrapplerItem& item,
                   GraphDef* optimized_graph) override {
     *optimized_graph = item.graph;
-    if (allowed_optimizations_) {
-      allowed_optimizations_->insert({item.id, item.allowed_optimizations()});
+    if (optimization_options_) {
+      optimization_options_->insert({item.id, item.optimization_options()});
     }
     return Status::OK();
   }
@@ -117,12 +117,12 @@ class GrapplerItemPropertiesAccumulator : public CustomGraphOptimizer {
                 const GraphDef& optimized_graph, double result) override {}
 
  private:
-  static gtl::FlatMap<string, GrapplerItem::AllowedOptimizations>*
-      allowed_optimizations_;
+  static gtl::FlatMap<string, GrapplerItem::OptimizationOptions>*
+      optimization_options_;
 };
 
-gtl::FlatMap<string, GrapplerItem::AllowedOptimizations>*
-    GrapplerItemPropertiesAccumulator::allowed_optimizations_;
+gtl::FlatMap<string, GrapplerItem::OptimizationOptions>*
+    GrapplerItemPropertiesAccumulator::optimization_options_;
 
 REGISTER_GRAPH_OPTIMIZER(GrapplerItemPropertiesAccumulator);
 
@@ -602,10 +602,9 @@ TEST_F(MetaOptimizerTest, OptimizeFunctionLibraryWithRestrictions) {
 
   // We will record what type of optimizations meta optimizer allows for each
   // GrapplerItem (main graph and graphs for each function).
-  gtl::FlatMap<string, GrapplerItem::AllowedOptimizations>
-      allowed_optimizations;
-  GrapplerItemPropertiesAccumulator::SetAllowedOptimizations(
-      &allowed_optimizations);
+  gtl::FlatMap<string, GrapplerItem::OptimizationOptions> optimization_options;
+  GrapplerItemPropertiesAccumulator::SetOptimizationOptions(
+      &optimization_options);
 
   // Just record properties of optimized Grappler items.
   ConfigProto config_proto;
@@ -664,22 +663,23 @@ TEST_F(MetaOptimizerTest, OptimizeFunctionLibraryWithRestrictions) {
 
   // Our custom optimizer must be called for the main graph and for the two
   // functions.
-  ASSERT_EQ(allowed_optimizations.size(), 3);
-
-  auto allowed_optimizations_main =
-      gtl::FindOrNull(allowed_optimizations, "main");
-  ASSERT_NE(allowed_optimizations_main, nullptr);
-  EXPECT_TRUE(allowed_optimizations_main->non_differentiable_rewrites);
-
-  auto allowed_optimizations_my_mul_1 =
-      gtl::FindOrNull(allowed_optimizations, "MyMul1");
-  ASSERT_NE(allowed_optimizations_my_mul_1, nullptr);
-  EXPECT_TRUE(allowed_optimizations_my_mul_1->non_differentiable_rewrites);
-
-  auto allowed_optimizations_my_mul_2 =
-      gtl::FindOrNull(allowed_optimizations, "MyMul2");
-  ASSERT_NE(allowed_optimizations_my_mul_2, nullptr);
-  EXPECT_FALSE(allowed_optimizations_my_mul_2->non_differentiable_rewrites);
+  ASSERT_EQ(optimization_options.size(), 3);
+
+  auto optimization_options_main =
+      gtl::FindOrNull(optimization_options, "main");
+  ASSERT_NE(optimization_options_main, nullptr);
+  EXPECT_TRUE(optimization_options_main->allow_non_differentiable_rewrites);
+
+  auto optimization_options_my_mul_1 =
+      gtl::FindOrNull(optimization_options, "MyMul1");
+  ASSERT_NE(optimization_options_my_mul_1, nullptr);
+  EXPECT_TRUE(optimization_options_my_mul_1->allow_non_differentiable_rewrites);
+
+  auto optimization_options_my_mul_2 =
+      gtl::FindOrNull(optimization_options, "MyMul2");
+  ASSERT_NE(optimization_options_my_mul_2, nullptr);
+  EXPECT_FALSE(
+      optimization_options_my_mul_2->allow_non_differentiable_rewrites);
 }
 
 class SleepingOptimizer : public CustomGraphOptimizer {
diff --git a/tensorflow/core/grappler/utils/functions.cc b/tensorflow/core/grappler/utils/functions.cc
index 150728d030..357a0b3b47 100644
--- a/tensorflow/core/grappler/utils/functions.cc
+++ b/tensorflow/core/grappler/utils/functions.cc
@@ -337,9 +337,9 @@ GrapplerFunctionItem::GrapplerFunctionItem(
     }
   }
 
-  // It's unsafe to prune side-effectful ops from the graph instantiated from a
-  // function definition (see inlining in function_optimizer.cc).
-  allowed_optimizations().prune_ops_with_side_effects = false;
+  // Tensorflow functions execution semantics is different from the main graph,
+  // and we need to preserve it when we do graph optimizations.
+  optimization_options().is_function_instantiation = true;
 }
 
 const string& GrapplerFunctionItem::description() const { return description_; }
diff --git a/tensorflow/core/grappler/utils/functions_test.cc b/tensorflow/core/grappler/utils/functions_test.cc
index c49920c79c..7720888828 100644
--- a/tensorflow/core/grappler/utils/functions_test.cc
+++ b/tensorflow/core/grappler/utils/functions_test.cc
@@ -641,7 +641,7 @@ TEST_F(FunctionsTest, FromFunctionDefWithSideEffectfulOps) {
   EXPECT_EQ(3, item.function_body().node_size());
   EXPECT_EQ(1, item.input_size());
   EXPECT_EQ(0, item.output_size());
-  EXPECT_EQ(false, item.allowed_optimizations().prune_ops_with_side_effects);
+  EXPECT_EQ(true, item.optimization_options().is_function_instantiation);
 }
 
 TEST_F(FunctionsTest, MakeFunctionDef) {
-- 
GitLab


From 7e090f6a5412fd5f22b5b75ca7ae7844c3d025e9 Mon Sep 17 00:00:00 2001
From: Nick Hamatake <hamatake@google.com>
Date: Tue, 15 Jan 2019 16:34:32 -0800
Subject: [PATCH 0782/1095] Force ICU to look for its data statically. This is
 required for Windows 10 builds. Patch suggested by github.com/cielavenir.

Fixes #23655

PiperOrigin-RevId: 229464102
---
 tensorflow/core/kernels/unicode_script_op.cc | 3 +++
 third_party/icu/BUILD.bazel                  | 1 +
 2 files changed, 4 insertions(+)

diff --git a/tensorflow/core/kernels/unicode_script_op.cc b/tensorflow/core/kernels/unicode_script_op.cc
index 085e397eba..f01bb52293 100644
--- a/tensorflow/core/kernels/unicode_script_op.cc
+++ b/tensorflow/core/kernels/unicode_script_op.cc
@@ -17,6 +17,9 @@ limitations under the License.
 #include "unicode/uscript.h"  // TF:icu
 #include "tensorflow/core/framework/op_kernel.h"
 
+// ICU codemap data is linked statically.
+#define U_STATIC_IMPLEMENTATION
+
 namespace tensorflow {
 
 class UnicodeScriptOp : public OpKernel {
diff --git a/third_party/icu/BUILD.bazel b/third_party/icu/BUILD.bazel
index 36d6b9006b..399962d249 100644
--- a/third_party/icu/BUILD.bazel
+++ b/third_party/icu/BUILD.bazel
@@ -44,6 +44,7 @@ cc_library(
     ]),
     copts = [
         "-DU_COMMON_IMPLEMENTATION",
+        "-DU_STATIC_IMPLEMENTATION",
         "-DU_HAVE_STD_ATOMICS",
     ] + select({
         ":android": [
-- 
GitLab


From bc246800912b450cf5ee7a485a95a0b77a3d9f53 Mon Sep 17 00:00:00 2001
From: Francois Chollet <fchollet@google.com>
Date: Tue, 15 Jan 2019 17:40:10 -0800
Subject: [PATCH 0783/1095] Reduce flakyness of integration test.

Tested by trying a range of random seeds.

PiperOrigin-RevId: 229473610
---
 tensorflow/python/keras/integration_test.py | 36 ++++++++++-----------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/tensorflow/python/keras/integration_test.py b/tensorflow/python/keras/integration_test.py
index febee44cd4..dc8d1deddb 100644
--- a/tensorflow/python/keras/integration_test.py
+++ b/tensorflow/python/keras/integration_test.py
@@ -35,7 +35,7 @@ class VectorClassificationIntegrationTest(keras_parameterized.TestCase):
   def test_vector_classification(self):
     np.random.seed(1337)
     (x_train, y_train), _ = testing_utils.get_test_data(
-        train_samples=150,
+        train_samples=100,
         test_samples=0,
         input_shape=(10,),
         num_classes=2)
@@ -47,10 +47,10 @@ class VectorClassificationIntegrationTest(keras_parameterized.TestCase):
          keras.layers.Dense(y_train.shape[-1], activation='softmax')],
         input_shape=x_train.shape[1:])
     model.compile(loss='categorical_crossentropy',
-                  optimizer=keras.optimizer_v2.adam.Adam(0.01),
+                  optimizer=keras.optimizer_v2.adam.Adam(0.005),
                   metrics=['accuracy'],
                   run_eagerly=testing_utils.should_run_eagerly())
-    history = model.fit(x_train, y_train, epochs=10, batch_size=16,
+    history = model.fit(x_train, y_train, epochs=10, batch_size=10,
                         validation_data=(x_train, y_train),
                         verbose=2)
     self.assertGreater(history.history['val_acc'][-1], 0.7)
@@ -64,7 +64,7 @@ class VectorClassificationIntegrationTest(keras_parameterized.TestCase):
     # and internal losses can be shared.
     np.random.seed(1337)
     (x_train, y_train), _ = testing_utils.get_test_data(
-        train_samples=150,
+        train_samples=100,
         test_samples=0,
         input_shape=(10,),
         num_classes=2)
@@ -82,13 +82,13 @@ class VectorClassificationIntegrationTest(keras_parameterized.TestCase):
     y = keras.layers.Dense(y_train.shape[-1], activation='softmax')(y)
     model = keras.models.Model(x, y)
     model.compile(loss='categorical_crossentropy',
-                  optimizer=keras.optimizer_v2.adam.Adam(0.01),
+                  optimizer=keras.optimizer_v2.adam.Adam(0.005),
                   metrics=['accuracy'],
                   run_eagerly=testing_utils.should_run_eagerly())
     if not testing_utils.should_run_eagerly():
       self.assertEqual(len(model.losses), 2)
       self.assertEqual(len(model.updates), 2)
-    history = model.fit(x_train, y_train, epochs=10, batch_size=16,
+    history = model.fit(x_train, y_train, epochs=10, batch_size=10,
                         validation_data=(x_train, y_train),
                         verbose=2)
     self.assertGreater(history.history['val_acc'][-1], 0.7)
@@ -106,7 +106,7 @@ class TimeseriesClassificationIntegrationTest(keras_parameterized.TestCase):
   def test_timeseries_classification(self):
     np.random.seed(1337)
     (x_train, y_train), _ = testing_utils.get_test_data(
-        train_samples=250,
+        train_samples=100,
         test_samples=0,
         input_shape=(4, 10),
         num_classes=2)
@@ -119,22 +119,22 @@ class TimeseriesClassificationIntegrationTest(keras_parameterized.TestCase):
     model = testing_utils.get_model_from_layers(
         layers, input_shape=x_train.shape[1:])
     model.compile(loss='categorical_crossentropy',
-                  optimizer=keras.optimizer_v2.adam.Adam(0.01),
+                  optimizer=keras.optimizer_v2.adam.Adam(0.005),
                   metrics=['accuracy'],
                   run_eagerly=testing_utils.should_run_eagerly())
-    history = model.fit(x_train, y_train, epochs=15, batch_size=16,
+    history = model.fit(x_train, y_train, epochs=15, batch_size=10,
                         validation_data=(x_train, y_train),
                         verbose=2)
     self.assertGreater(history.history['val_acc'][-1], 0.7)
     _, val_acc = model.evaluate(x_train, y_train)
     self.assertAlmostEqual(history.history['val_acc'][-1], val_acc)
     predictions = model.predict(x_train)
-    self.assertEqual(predictions.shape, (250, 2))
+    self.assertEqual(predictions.shape, (x_train.shape[0], 2))
 
   def test_timeseries_classification_sequential_tf_rnn(self):
     np.random.seed(1337)
     (x_train, y_train), _ = testing_utils.get_test_data(
-        train_samples=150,
+        train_samples=100,
         test_samples=0,
         input_shape=(4, 10),
         num_classes=2)
@@ -147,17 +147,17 @@ class TimeseriesClassificationIntegrationTest(keras_parameterized.TestCase):
                                                 activation='softmax',
                                                 dtype=dtypes.float32)))
     model.compile(loss='categorical_crossentropy',
-                  optimizer=keras.optimizer_v2.adam.Adam(0.01),
+                  optimizer=keras.optimizer_v2.adam.Adam(0.005),
                   metrics=['accuracy'],
                   run_eagerly=testing_utils.should_run_eagerly())
-    history = model.fit(x_train, y_train, epochs=15, batch_size=16,
+    history = model.fit(x_train, y_train, epochs=15, batch_size=10,
                         validation_data=(x_train, y_train),
                         verbose=2)
     self.assertGreater(history.history['val_acc'][-1], 0.7)
     _, val_acc = model.evaluate(x_train, y_train)
     self.assertAlmostEqual(history.history['val_acc'][-1], val_acc)
     predictions = model.predict(x_train)
-    self.assertEqual(predictions.shape, (150, 2))
+    self.assertEqual(predictions.shape, (x_train.shape[0], 2))
 
 
 @keras_parameterized.run_with_all_model_types
@@ -167,7 +167,7 @@ class ImageClassificationIntegrationTest(keras_parameterized.TestCase):
   def test_image_classification(self):
     np.random.seed(1337)
     (x_train, y_train), _ = testing_utils.get_test_data(
-        train_samples=250,
+        train_samples=100,
         test_samples=0,
         input_shape=(10, 10, 3),
         num_classes=2)
@@ -184,17 +184,17 @@ class ImageClassificationIntegrationTest(keras_parameterized.TestCase):
     model = testing_utils.get_model_from_layers(
         layers, input_shape=x_train.shape[1:])
     model.compile(loss='categorical_crossentropy',
-                  optimizer=keras.optimizer_v2.adam.Adam(0.01),
+                  optimizer=keras.optimizer_v2.adam.Adam(0.005),
                   metrics=['accuracy'],
                   run_eagerly=testing_utils.should_run_eagerly())
-    history = model.fit(x_train, y_train, epochs=10, batch_size=16,
+    history = model.fit(x_train, y_train, epochs=10, batch_size=10,
                         validation_data=(x_train, y_train),
                         verbose=2)
     self.assertGreater(history.history['val_acc'][-1], 0.7)
     _, val_acc = model.evaluate(x_train, y_train)
     self.assertAlmostEqual(history.history['val_acc'][-1], val_acc)
     predictions = model.predict(x_train)
-    self.assertEqual(predictions.shape, (250, 2))
+    self.assertEqual(predictions.shape, (x_train.shape[0], 2))
 
 
 if __name__ == '__main__':
-- 
GitLab


From 13ecb4c3bcdd93c85a4bf21080b240681c763eda Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Tue, 15 Jan 2019 17:47:32 -0800
Subject: [PATCH 0784/1095] Document the GPU accelerator best practices.

PiperOrigin-RevId: 229474455
---
 .../lite/g3doc/performance/best_practices.md  | 32 +++++++++++++++++--
 1 file changed, 29 insertions(+), 3 deletions(-)

diff --git a/tensorflow/lite/g3doc/performance/best_practices.md b/tensorflow/lite/g3doc/performance/best_practices.md
index b76414cebe..090f19e810 100644
--- a/tensorflow/lite/g3doc/performance/best_practices.md
+++ b/tensorflow/lite/g3doc/performance/best_practices.md
@@ -37,8 +37,34 @@ If your application is not careful, there can be redundant copies when feeding t
 Platform specific tools like [Android profiler](https://developer.android.com/studio/profile/android-profiler) and [Instruments](https://help.apple.com/instruments/mac/current/) provide a wealth of profiling information that can be used to debug your app. Sometimes the performance bug may be not in the model but in parts of application code that interact with the model. Make sure to familiarize yourself with platform specific profiling tools and best practices for your platform.
 
 ## Evaluate whether your model benefits from using hardware accelerators available on the device
-Tensorflow Lite is working on adding support for accelerators like GPU and provides acceleration through [Neural Networks API](https://developer.android.com/ndk/guides/neuralnetworks/) on Android.
-You can utilize these hardware accelerator backends to improve the speed and efficiency of your model. To enable Neural Networks API call [UseNNAPI](https://github.com/tensorflow/tensorflow/blob/6305a6d83552ba6a472cd72398b60d9241467f1f/tensorflow/lite/interpreter.h#L334) on the interpreter instance.
+
+TensorFlow Lite has added been new ways to accelerate models with faster
+hardware like GPUs, DSPs, and neural accelerators. Typically, these accelerators
+are exposed through *delegate* submodules that take over parts of the
+interpreter execution. TensorFlow Lite can use delegates by:
+
+*   Using Android's
+    [Neural Networks API](https://developer.android.com/ndk/guides/neuralnetworks/).
+    You can utilize these hardware accelerator backends to improve the speed and
+    efficiency of your model. To enable the Neural Networks API, call
+    [UseNNAPI](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/interpreter.h#L330)
+    on the interpreter instance.
+*   A binary-only GPU delegate has been released for Android and iOS—using
+    OpenGL and Metal, respectively. To try them out, see the
+    [GPU delegate tutorial](gpu.md) and [documentation](gpu_advanced.md).
+*   It is possible to create your own delegate if you have access to
+    non-standard hardware. View the NN API delegate in the source code as an
+    example.
+
+Be aware that some accelerators work better for different types of models. It is
+important to benchmark each delegate to see if it is a good choice for your
+application. For example, if you have a very small model, it may not be worth
+delegating the model to either the NN API or the GPU. Conversely, accelerators
+are a great choice for large models that have high arithmetic intensity.
 
 ## Need more help
-The Tensorflow team is happy to help diagnose and address specific performance issues you may be facing. Please file an issue on [GitHub](https://github.com/tensorflow/tensorflow/issues) with details of the issue.
+
+The Tensorflow team is happy to help diagnose and address specific performance
+issues you may be facing. Please file an issue on
+[GitHub](https://github.com/tensorflow/tensorflow/issues) with details of the
+issue.
-- 
GitLab


From a3fa57c6d2f4678597c0b76bb448c37b66c45376 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Tue, 15 Jan 2019 17:57:55 -0800
Subject: [PATCH 0785/1095] Add new distribution strategy test rule

PiperOrigin-RevId: 229475641
---
 tensorflow/contrib/distribute/python/BUILD    | 10 +++--
 .../core/platform/default/distribute.bzl      | 39 +++++++++++++++++++
 2 files changed, 45 insertions(+), 4 deletions(-)
 create mode 100644 tensorflow/core/platform/default/distribute.bzl

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index 1a54cd167e..384fe53923 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -1,6 +1,7 @@
 # Implementation of a prototype TF distributed computation library.
 
 load("//tensorflow/compiler/tests:build_defs.bzl", "tf_xla_py_test")
+load("//tensorflow/core:platform/default/distribute.bzl", "distribute_py_test")
 load("//tensorflow:tensorflow.bzl", "py_test")
 load("//tensorflow:tensorflow.bzl", "cuda_py_test")
 
@@ -621,12 +622,10 @@ py_library(
     ],
 )
 
-cuda_py_test(
+distribute_py_test(
     name = "keras_test",
     srcs = ["keras_test.py"],
-    additional_deps = [
-        ":keras_test_lib",
-    ],
+    main = "keras_test.py",
     shard_count = 16,
     tags = [
         "multi_and_single_gpu",
@@ -635,6 +634,9 @@ cuda_py_test(
         "no_windows_gpu",
         "notsan",
     ],
+    deps = [
+        ":keras_test_lib",
+    ],
 )
 
 # TODO(b/121200287): Remove this in 2.0
diff --git a/tensorflow/core/platform/default/distribute.bzl b/tensorflow/core/platform/default/distribute.bzl
new file mode 100644
index 0000000000..a3afc37f03
--- /dev/null
+++ b/tensorflow/core/platform/default/distribute.bzl
@@ -0,0 +1,39 @@
+"""Build rules for tf.distritbute testing."""
+
+load("//tensorflow:tensorflow.bzl", "cuda_py_test")
+
+def distribute_py_test(
+        name,
+        srcs = [],
+        deps = [],
+        tags = [],
+        data = [],
+        main = None,
+        args = [],
+        shard_count = 1,
+        **kwargs):
+    """Generates py_test targets for CPU and GPU.
+
+    Args:
+        name: test target name to generate suffixed with `test`.
+        srcs: source files for the tests.
+        deps: additional dependencies for the test targets.
+        tags: tags to be assigned to the different test targets.
+        data: data files that need to be associated with the target files.
+        main: optional main script.
+        args: arguments to the tests.
+        shard_count: number of shards to split the tests across.
+        **kwargs: extra keyword arguments to the test.
+    """
+
+    cuda_py_test(
+        name = name,
+        srcs = srcs,
+        data = data,
+        main = main,
+        additional_deps = deps,
+        shard_count = shard_count,
+        tags = tags,
+        args = args,
+        **kwargs
+    )
-- 
GitLab


From 6750ba5da025f71ea989ea7f9d924a5e9b22aedc Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 15 Jan 2019 18:06:59 -0800
Subject: [PATCH 0786/1095] Update ops-related pbtxt files.

PiperOrigin-RevId: 229476985
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 128 ++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 |  95 ++++++++++++-
 2 files changed, 217 insertions(+), 6 deletions(-)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 8705492c66..9990b8629c 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -1298,6 +1298,34 @@ op {
     type: DT_FLOAT
   }
 }
+op {
+  name: "AdjustContrastv2"
+  input_arg {
+    name: "images"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "contrast_factor"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    default_value {
+      type: DT_FLOAT
+    }
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+      }
+    }
+  }
+}
 op {
   name: "AdjustHue"
   input_arg {
@@ -1313,6 +1341,34 @@ op {
     type: DT_FLOAT
   }
 }
+op {
+  name: "AdjustHue"
+  input_arg {
+    name: "images"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "delta"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    default_value {
+      type: DT_FLOAT
+    }
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+      }
+    }
+  }
+}
 op {
   name: "AdjustSaturation"
   input_arg {
@@ -1328,6 +1384,34 @@ op {
     type: DT_FLOAT
   }
 }
+op {
+  name: "AdjustSaturation"
+  input_arg {
+    name: "images"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "scale"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    default_value {
+      type: DT_FLOAT
+    }
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+      }
+    }
+  }
+}
 op {
   name: "All"
   input_arg {
@@ -73373,6 +73457,50 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "StatefulStandardNormal"
+  input_arg {
+    name: "resource"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "shape"
+    type_attr: "shape_dtype"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "dtype"
+  }
+  attr {
+    name: "dtype"
+    type: "type"
+    default_value {
+      type: DT_FLOAT
+    }
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_BFLOAT16
+        type: DT_FLOAT
+        type: DT_DOUBLE
+      }
+    }
+  }
+  attr {
+    name: "shape_dtype"
+    type: "type"
+    default_value {
+      type: DT_INT64
+    }
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  is_stateful: true
+}
 op {
   name: "StatelessIf"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 012c03a834..8212282623 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -486,7 +486,7 @@ op {
   name: "AdjustContrastv2"
   input_arg {
     name: "images"
-    type: DT_FLOAT
+    type_attr: "T"
   }
   input_arg {
     name: "contrast_factor"
@@ -494,14 +494,27 @@ op {
   }
   output_arg {
     name: "output"
-    type: DT_FLOAT
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    default_value {
+      type: DT_FLOAT
+    }
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+      }
+    }
   }
 }
 op {
   name: "AdjustHue"
   input_arg {
     name: "images"
-    type: DT_FLOAT
+    type_attr: "T"
   }
   input_arg {
     name: "delta"
@@ -509,14 +522,27 @@ op {
   }
   output_arg {
     name: "output"
-    type: DT_FLOAT
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    default_value {
+      type: DT_FLOAT
+    }
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+      }
+    }
   }
 }
 op {
   name: "AdjustSaturation"
   input_arg {
     name: "images"
-    type: DT_FLOAT
+    type_attr: "T"
   }
   input_arg {
     name: "scale"
@@ -524,7 +550,20 @@ op {
   }
   output_arg {
     name: "output"
-    type: DT_FLOAT
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    default_value {
+      type: DT_FLOAT
+    }
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+      }
+    }
   }
 }
 op {
@@ -34365,6 +34404,50 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "StatefulStandardNormal"
+  input_arg {
+    name: "resource"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "shape"
+    type_attr: "shape_dtype"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "dtype"
+  }
+  attr {
+    name: "dtype"
+    type: "type"
+    default_value {
+      type: DT_FLOAT
+    }
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_BFLOAT16
+        type: DT_FLOAT
+        type: DT_DOUBLE
+      }
+    }
+  }
+  attr {
+    name: "shape_dtype"
+    type: "type"
+    default_value {
+      type: DT_INT64
+    }
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  is_stateful: true
+}
 op {
   name: "StatelessIf"
   input_arg {
-- 
GitLab


From d62f6223fc0a1e4d81dffd92c61ada3e9ec17fb7 Mon Sep 17 00:00:00 2001
From: Shashi Shekhar <shashishekhar@google.com>
Date: Tue, 15 Jan 2019 18:23:21 -0800
Subject: [PATCH 0787/1095] Move to new quantization parameters for subgraph
 quantizer.

PiperOrigin-RevId: 229478990
---
 .../lite/tools/optimize/subgraph_quantizer.cc | 37 ++++----
 .../tools/optimize/subgraph_quantizer_test.cc | 76 +++++++---------
 .../optimize/symmetric_per_channel_params.cc  | 86 -------------------
 .../optimize/symmetric_per_channel_params.h   | 58 -------------
 .../symmetric_per_channel_params_test.cc      | 51 -----------
 5 files changed, 47 insertions(+), 261 deletions(-)
 delete mode 100644 tensorflow/lite/tools/optimize/symmetric_per_channel_params.cc
 delete mode 100644 tensorflow/lite/tools/optimize/symmetric_per_channel_params.h
 delete mode 100644 tensorflow/lite/tools/optimize/symmetric_per_channel_params_test.cc

diff --git a/tensorflow/lite/tools/optimize/subgraph_quantizer.cc b/tensorflow/lite/tools/optimize/subgraph_quantizer.cc
index 56f2e6cf76..30320304ec 100644
--- a/tensorflow/lite/tools/optimize/subgraph_quantizer.cc
+++ b/tensorflow/lite/tools/optimize/subgraph_quantizer.cc
@@ -27,7 +27,6 @@ limitations under the License.
 #include "tensorflow/lite/model.h"
 #include "tensorflow/lite/schema/schema_generated.h"
 #include "tensorflow/lite/tools/optimize/quantization_utils.h"
-#include "tensorflow/lite/tools/optimize/symmetric_per_channel_params.h"
 
 namespace tflite {
 namespace optimize {
@@ -37,11 +36,14 @@ namespace {
 const int8_t kMinQuantizedValue = -127;
 const int8_t kMaxQuantizedValue = 127;
 
-TfLiteStatus AddQuantizationParams(const SymmetricPerChannelParams& params,
+TfLiteStatus AddQuantizationParams(const std::vector<float>& scales,
+                                   int quantized_dimension,
                                    const uint8_t* buffer_data,
                                    size_t buffer_size, TensorType output_type,
                                    ModelT* model, TensorT* tensor) {
-  params.AddToTensor(tensor);
+  tensor->quantization = absl::make_unique<QuantizationParametersT>();
+  tensor->quantization->scale.assign(scales.begin(), scales.end());
+  tensor->quantization->quantized_dimension = quantized_dimension;
   model->buffers[tensor->buffer]->data.assign(buffer_data,
                                               buffer_data + buffer_size);
   // Update the tensor type.
@@ -95,10 +97,6 @@ TfLiteStatus SymmetricPerChannelQuantizeTensor(ModelT* model, TensorT* tensor,
   TF_LITE_ENSURE_STATUS(utils::NumElements(*tensor, &num_elements));
   const uint64_t num_elements_per_channel = num_elements / channel_dim_size;
 
-  if (tensor->quantization == nullptr) {
-    tensor->quantization = absl::make_unique<QuantizationParametersT>();
-  }
-
   std::vector<float> min_vals(channel_dim_size);
   std::vector<float> max_vals(channel_dim_size);
   std::vector<bool> has_min_max_value(channel_dim_size, false);
@@ -170,9 +168,8 @@ TfLiteStatus SymmetricPerChannelQuantizeTensor(ModelT* model, TensorT* tensor,
   // Set the buffers and output type.
   uint8_t* uint8_buffer = reinterpret_cast<uint8_t*>(final_buffer.data());
   size_t buffer_size = num_elements * sizeof(int8_t);
-  SymmetricPerChannelParams symmetric_params(scales, channel_dim_index);
-  return AddQuantizationParams(symmetric_params, uint8_buffer, buffer_size,
-                               TensorType_INT8, model, tensor);
+  return AddQuantizationParams(scales, channel_dim_index, uint8_buffer,
+                               buffer_size, TensorType_INT8, model, tensor);
 }
 
 // Symmetrically quantizes the bias for ops like Conv and DepthwiseConv.
@@ -204,10 +201,8 @@ TfLiteStatus SymmetricPerChannelBiasQuantize(const TensorT* input_tensor,
     error_reporter->Report("Input tensor missing quantization information");
     return kTfLiteError;
   }
-  std::unique_ptr<SymmetricPerChannelParams> weight_params;
-  TF_LITE_ENSURE_STATUS(SymmetricPerChannelParams::ReadFromTensor(
-      *weight_tensor, &weight_params));
-  const auto& weight_scales = weight_params->scales();
+  TF_LITE_ENSURE(error_reporter, weight_tensor->quantization);
+  const std::vector<float>& weight_scales = weight_tensor->quantization->scale;
 
   if (weight_scales.size() != channel_dim_size) {
     error_reporter->Report("Mismatch weight scale dimension: %d",
@@ -226,9 +221,6 @@ TfLiteStatus SymmetricPerChannelBiasQuantize(const TensorT* input_tensor,
   uint64_t num_elements;
   TF_LITE_ENSURE_STATUS(utils::NumElements(*tensor, &num_elements));
 
-  if (tensor->quantization == nullptr) {
-    tensor->quantization = absl::make_unique<QuantizationParametersT>();
-  }
   std::vector<int32_t> final_buffer(num_elements);
   const int32_t kScale = std::numeric_limits<int32_t>::max();
 
@@ -244,9 +236,14 @@ TfLiteStatus SymmetricPerChannelBiasQuantize(const TensorT* input_tensor,
   // Set the buffers and output type.
   uint8_t* uint8_buffer = reinterpret_cast<uint8_t*>(final_buffer.data());
   size_t buffer_size = num_elements * sizeof(int32_t);
-  SymmetricPerChannelParams symmetric_params(scales, channel_dim_index);
-  return AddQuantizationParams(symmetric_params, uint8_buffer, buffer_size,
-                               TensorType_INT32, model, tensor);
+  // For Bias we only set the quantized values, the scale and quantized
+  // dimension is implicit.
+  tensor->quantization = nullptr;
+  model->buffers[tensor->buffer]->data.assign(uint8_buffer,
+                                              uint8_buffer + buffer_size);
+
+  tensor->type = TensorType_INT32;
+  return kTfLiteOk;
 }
 }  // namespace
 
diff --git a/tensorflow/lite/tools/optimize/subgraph_quantizer_test.cc b/tensorflow/lite/tools/optimize/subgraph_quantizer_test.cc
index 96ba20d4d9..085b66e90f 100644
--- a/tensorflow/lite/tools/optimize/subgraph_quantizer_test.cc
+++ b/tensorflow/lite/tools/optimize/subgraph_quantizer_test.cc
@@ -21,7 +21,6 @@ limitations under the License.
 #include "tensorflow/core/util/command_line_flags.h"
 #include "tensorflow/lite/schema/schema_generated.h"
 #include "tensorflow/lite/tools/optimize/subgraph_quantizer.h"
-#include "tensorflow/lite/tools/optimize/symmetric_per_channel_params.h"
 #include "tensorflow/lite/tools/optimize/test_util.h"
 
 namespace {
@@ -91,31 +90,22 @@ TEST(SubgraphQuantizerTest, VerifyConvQuantizationWithUnitScale) {
 
   ASSERT_TRUE(weights_tensor->quantization);
   const int out_channel_size = weights_tensor->shape[0];
-  std::unique_ptr<SymmetricPerChannelParams> bias_params;
-  std::unique_ptr<SymmetricPerChannelParams> weights_params;
-  ASSERT_EQ(kTfLiteOk, SymmetricPerChannelParams::ReadFromTensor(
-                           *weights_tensor, &weights_params));
-  ASSERT_EQ(kTfLiteOk, SymmetricPerChannelParams::ReadFromTensor(*bias_tensor,
-                                                                 &bias_params));
-
-  ASSERT_EQ(bias_params->scales().size(), out_channel_size);
-  ASSERT_EQ(weights_params->scales().size(), out_channel_size);
+
+  // Bias tensor doesn't contain quantization info.
+  ASSERT_FALSE(bias_tensor->quantization);
+
+  const std::vector<float>& weights_scales =
+      weights_tensor->quantization->scale;
+
+  ASSERT_EQ(weights_scales.size(), out_channel_size);
   ASSERT_EQ(input_tensor->quantization->scale.size(), 1);
   ASSERT_EQ(output_tensor->quantization->scale.size(), 1);
 
-  const float eps = 1e-7;
 
-  // Bias scale should be input * per_channel_weight_scale.
-  for (size_t i = 0; i < out_channel_size; i++) {
-    EXPECT_NEAR(
-        bias_params->scales()[i],
-        input_tensor->quantization->scale[0] * weights_params->scales()[i],
-        eps);
-  }
   for (size_t i = 0; i < out_channel_size; i++) {
-    EXPECT_EQ(weights_params->scales()[i], 1);
-    EXPECT_EQ(bias_params->scales()[i], 1);
+    EXPECT_EQ(weights_scales[i], 1);
   }
+
   EXPECT_EQ(input_tensor->quantization->scale[0], 1);
   EXPECT_EQ(output_tensor->quantization->scale[0], 1);
 
@@ -128,8 +118,11 @@ TEST(SubgraphQuantizerTest, VerifyConvQuantizationWithUnitScale) {
   const float* bias_float_buffer =
       reinterpret_cast<const float*>(original_bias_buffer->data()->data());
 
+  const float eps = 1e-7;
   for (size_t i = 0; i < bias_tensor->shape[0]; i++) {
-    auto dequantized_value = bias_values[i] * bias_params->scales()[i];
+    const float bias_scale =
+        input_tensor->quantization->scale[0] * weights_scales[i];
+    auto dequantized_value = bias_values[i] * bias_scale;
     EXPECT_NEAR(dequantized_value, bias_float_buffer[i], eps);
   }
 
@@ -147,7 +140,7 @@ TEST(SubgraphQuantizerTest, VerifyConvQuantizationWithUnitScale) {
     for (size_t j = 0; j < num_values_in_channel; j++) {
       size_t element_idx = channel_idx * out_channel_size + j;
       auto dequantized_value =
-          weight_values[element_idx] * weights_params->scales()[channel_idx];
+          weight_values[element_idx] * weights_scales[channel_idx];
       EXPECT_NEAR(dequantized_value, weights_float_buffer[element_idx], eps);
     }
   }
@@ -190,27 +183,17 @@ TEST(SubgraphQuantizerTest, VerifyConvQuantization) {
 
   ASSERT_TRUE(weights_tensor->quantization);
   const int out_channel_size = weights_tensor->shape[0];
-  std::unique_ptr<SymmetricPerChannelParams> bias_params;
-  std::unique_ptr<SymmetricPerChannelParams> weights_params;
-  ASSERT_EQ(kTfLiteOk, SymmetricPerChannelParams::ReadFromTensor(
-                           *weights_tensor, &weights_params));
-  ASSERT_EQ(kTfLiteOk, SymmetricPerChannelParams::ReadFromTensor(*bias_tensor,
-                                                                 &bias_params));
-
-  ASSERT_EQ(bias_params->scales().size(), out_channel_size);
-  ASSERT_EQ(weights_params->scales().size(), out_channel_size);
-  ASSERT_EQ(input_tensor->quantization->scale.size(), 1);
-  ASSERT_EQ(output_tensor->quantization->scale.size(), 1);
 
-  const float eps = 1e-7;
+  // Bias tensor doesn't contain quantization info.
+  ASSERT_FALSE(bias_tensor->quantization);
 
-  // Bias scale should be input * per_channel_weight_scale.
-  for (size_t i = 0; i < out_channel_size; i++) {
-    EXPECT_NEAR(
-        bias_params->scales()[i],
-        input_tensor->quantization->scale[0] * weights_params->scales()[i],
-        eps);
-  }
+  ASSERT_TRUE(weights_tensor->quantization);
+  const std::vector<float>& weights_scales =
+      weights_tensor->quantization->scale;
+
+  ASSERT_EQ(weights_scales.size(), out_channel_size);
+  ASSERT_EQ(input_tensor->quantization->scale.size(), 1);
+  ASSERT_EQ(output_tensor->quantization->scale.size(), 1);
 
   const auto bias_buffer = model.buffers[bias_tensor->buffer].get();
   ASSERT_EQ(bias_buffer->data.size(), sizeof(int32_t) * bias_tensor->shape[0]);
@@ -221,10 +204,11 @@ TEST(SubgraphQuantizerTest, VerifyConvQuantization) {
   const float* bias_float_buffer =
       reinterpret_cast<const float*>(original_bias_buffer->data()->data());
 
-  for (size_t i = 0; i < bias_tensor->shape[0]; i++) {
-    auto scale = bias_params->scales()[i];
-    auto dequantized_value = bias_values[i] * scale;
-    EXPECT_NEAR(dequantized_value, bias_float_buffer[i], scale / 2);
+  for (size_t i = 0; i < out_channel_size; i++) {
+    const float bias_scale =
+        input_tensor->quantization->scale[0] * weights_scales[i];
+    auto dequantized_value = bias_values[i] * bias_scale;
+    EXPECT_NEAR(dequantized_value, bias_float_buffer[i], bias_scale / 2);
   }
 
   const auto weights_buffer = model.buffers[weights_tensor->buffer].get();
@@ -240,7 +224,7 @@ TEST(SubgraphQuantizerTest, VerifyConvQuantization) {
   for (size_t channel_idx = 0; channel_idx < out_channel_size; channel_idx++) {
     for (size_t j = 0; j < num_values_in_channel; j++) {
       size_t element_idx = channel_idx * out_channel_size + j;
-      auto scale = weights_params->scales()[channel_idx];
+      auto scale = weights_scales[channel_idx];
       auto dequantized_value = weight_values[element_idx] * scale;
       EXPECT_NEAR(dequantized_value, weights_float_buffer[element_idx],
                   scale / 2);
diff --git a/tensorflow/lite/tools/optimize/symmetric_per_channel_params.cc b/tensorflow/lite/tools/optimize/symmetric_per_channel_params.cc
deleted file mode 100644
index ff7e5657f1..0000000000
--- a/tensorflow/lite/tools/optimize/symmetric_per_channel_params.cc
+++ /dev/null
@@ -1,86 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#include "tensorflow/lite/tools/optimize/symmetric_per_channel_params.h"
-
-#include "flatbuffers/flexbuffers.h"
-#include "absl/memory/memory.h"
-#include "tensorflow/lite/schema/schema_generated.h"
-
-namespace tflite {
-namespace optimize {
-namespace internal {
-
-namespace {
-const char* kCustomQuantizationScales = "scales";
-const char* kCustomQuantizationChannelDimIndex = "channel_dim_index";
-
-// Writes scales and dimensions to custom quantization.
-std::vector<uint8_t> SerializeToFlexBuffer(const std::vector<float>& scales,
-                                           int channel_dim_index) {
-  flexbuffers::Builder fbb;
-  fbb.Map([&]() {
-    fbb.Vector(kCustomQuantizationScales, [&]() {
-      for (float scale : scales) {
-        fbb.Float(scale);
-      }
-    });
-    fbb.Int(kCustomQuantizationChannelDimIndex, channel_dim_index);
-  });
-  fbb.Finish();
-  return fbb.GetBuffer();
-}
-
-}  // namespace
-
-/*static*/ TfLiteStatus SymmetricPerChannelParams::ReadFromTensor(
-    const TensorT& tensor, std::unique_ptr<SymmetricPerChannelParams>* params) {
-  const auto* q_params = tensor.quantization.get();
-  if (!q_params) return kTfLiteError;
-  auto custom_quantization = q_params->details.AsCustomQuantization();
-
-  auto buffer = flexbuffers::GetRoot(custom_quantization->custom.data(),
-                                     custom_quantization->custom.size());
-  if (!buffer.IsMap()) {
-    return kTfLiteError;
-  }
-
-  auto custom_quantization_map = buffer.AsMap();
-  auto quant_scales =
-      custom_quantization_map[kCustomQuantizationScales].AsVector();
-  auto channel_dim_index =
-      custom_quantization_map[kCustomQuantizationChannelDimIndex].AsInt32();
-  std::vector<float> scales;
-  scales.reserve(quant_scales.size());
-  for (int i = 0; i < quant_scales.size(); i++) {
-    scales.push_back(quant_scales[i].AsFloat());
-  }
-  *params =
-      absl::make_unique<SymmetricPerChannelParams>(scales, channel_dim_index);
-  return kTfLiteOk;
-}
-
-TfLiteStatus SymmetricPerChannelParams::AddToTensor(TensorT* tensor) const {
-  flatbuffers::FlatBufferBuilder fbb;
-  CustomQuantizationT* custom_quantization = new CustomQuantizationT;
-  custom_quantization->custom =
-      SerializeToFlexBuffer(scales_, channel_dim_index_);
-  tensor->quantization = absl::make_unique<QuantizationParametersT>();
-  tensor->quantization->details.type = QuantizationDetails_CustomQuantization;
-  tensor->quantization->details.value = custom_quantization;
-  return kTfLiteOk;
-}
-}  // namespace internal
-}  // namespace optimize
-}  // namespace tflite
diff --git a/tensorflow/lite/tools/optimize/symmetric_per_channel_params.h b/tensorflow/lite/tools/optimize/symmetric_per_channel_params.h
deleted file mode 100644
index 2ec3ea373a..0000000000
--- a/tensorflow/lite/tools/optimize/symmetric_per_channel_params.h
+++ /dev/null
@@ -1,58 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#ifndef TENSORFLOW_LITE_TOOLS_OPTIMIZE_SYMMETRIC_PER_CHANNEL_PARAMS_H_
-#define TENSORFLOW_LITE_TOOLS_OPTIMIZE_SYMMETRIC_PER_CHANNEL_PARAMS_H_
-
-#include <memory>
-#include <vector>
-
-#include "tensorflow/lite/context.h"
-#include "tensorflow/lite/model.h"
-#include "tensorflow/lite/schema/schema_generated.h"
-
-namespace tflite {
-namespace optimize {
-namespace internal {
-
-// Helper to read/write symmetric per channel parameters to tensors.
-//
-// TODO(shashishekhar): This maybe useful for kernels as well, refactor to
-// kernel utils.
-class SymmetricPerChannelParams {
- public:
-  SymmetricPerChannelParams(const std::vector<float> scales,
-                            int channel_dim_index)
-      : scales_(scales), channel_dim_index_(channel_dim_index) {}
-
-  // Creates a new instance by reading the information from existing tensor.
-  static TfLiteStatus ReadFromTensor(
-      const TensorT& tensor,
-      std::unique_ptr<SymmetricPerChannelParams>* params);
-
-  // Add the symmetric per channel info to the tensor.
-  TfLiteStatus AddToTensor(TensorT* tensor) const;
-
-  const std::vector<float>& scales() const { return scales_; }
-
-  int channel_dim_index() const { return channel_dim_index_; }
-
- private:
-  std::vector<float> scales_;
-  int channel_dim_index_;
-};
-}  // namespace internal
-}  // namespace optimize
-}  // namespace tflite
-#endif  // TENSORFLOW_LITE_TOOLS_OPTIMIZE_SYMMETRIC_PER_CHANNEL_PARAMS_H_
diff --git a/tensorflow/lite/tools/optimize/symmetric_per_channel_params_test.cc b/tensorflow/lite/tools/optimize/symmetric_per_channel_params_test.cc
deleted file mode 100644
index 4d3cbe6f49..0000000000
--- a/tensorflow/lite/tools/optimize/symmetric_per_channel_params_test.cc
+++ /dev/null
@@ -1,51 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#include "tensorflow/lite/tools/optimize/symmetric_per_channel_params.h"
-#include <gmock/gmock.h>
-#include <gtest/gtest.h>
-#include "flatbuffers/flexbuffers.h"  // TF:flatbuffers
-#include "tensorflow/lite/schema/schema_generated.h"
-
-namespace tflite {
-namespace optimize {
-namespace internal {
-namespace {
-
-TEST(SymmetricPerChannelParamsTest, TestReadWrite) {
-  TensorT tensor;
-  std::unique_ptr<SymmetricPerChannelParams> read_back_params;
-  const std::vector<float> scales = {3.0, 4.0, 5.0};
-  const int channel_dim_index = 42;
-
-  SymmetricPerChannelParams params(scales, channel_dim_index);
-  params.AddToTensor(&tensor);
-  auto status =
-      SymmetricPerChannelParams::ReadFromTensor(tensor, &read_back_params);
-  EXPECT_EQ(kTfLiteOk, status);
-  ASSERT_TRUE(read_back_params);
-
-  EXPECT_EQ(channel_dim_index, read_back_params->channel_dim_index());
-  ASSERT_EQ(read_back_params->scales(), scales);
-}
-
-}  // namespace
-}  // namespace internal
-}  // namespace optimize
-}  // namespace tflite
-
-int main(int argc, char** argv) {
-  ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}
-- 
GitLab


From dc91b313e86e6bcb8c88a4be2e098296ba6c3174 Mon Sep 17 00:00:00 2001
From: Billy Lamberta <blamb@google.com>
Date: Tue, 15 Jan 2019 18:25:10 -0800
Subject: [PATCH 0788/1095] * Cleaned up prose for clarity. * Refactored
 options example to highlight preferred usage.

PiperOrigin-RevId: 229479188
---
 tensorflow/lite/g3doc/_book.yaml              |   4 +
 .../lite/g3doc/performance/gpu_advanced.md    | 196 ++++++++++--------
 2 files changed, 114 insertions(+), 86 deletions(-)

diff --git a/tensorflow/lite/g3doc/_book.yaml b/tensorflow/lite/g3doc/_book.yaml
index 0c79e79fdd..9c48e1e54d 100644
--- a/tensorflow/lite/g3doc/_book.yaml
+++ b/tensorflow/lite/g3doc/_book.yaml
@@ -59,6 +59,10 @@ upper_tabs:
       - title: Post-training quantization example
         path: https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/tutorials/post_training_quant.ipynb
         status: external
+      - title: GPU delegate
+        path: /lite/performance/gpu
+      - title: Advanced GPU
+        path: /lite/performance/gpu_advanced
 
       - title: TF Mobile
         style: accordion
diff --git a/tensorflow/lite/g3doc/performance/gpu_advanced.md b/tensorflow/lite/g3doc/performance/gpu_advanced.md
index d0ae123613..6274948040 100644
--- a/tensorflow/lite/g3doc/performance/gpu_advanced.md
+++ b/tensorflow/lite/g3doc/performance/gpu_advanced.md
@@ -5,24 +5,32 @@ hardware accelerators.  This document describes how to use the GPU backend using
 the TensorFlow Lite delegate APIs on Android (requires OpenGL ES 3.1 or higher)
 and iOS (requires iOS 8 or later).
 
+## Benefits of GPU Acceleration
+
+### Speed
+
 GPUs are designed to have high throughput for massively parallelizable
-workloads.  Thus, they are well-suited for deep neural nets which consists of a
+workloads. Thus, they are well-suited for deep neural nets, which consist of a
 huge number of operators, each working on some input tensor(s) that can be
-easily divided into smaller workloads and carried out in parallel, typically
-resulting in lower latency.  In the best scenario, inference on the GPU may now
-run fast enough and now become suitable for real-time applications if it was not
-before.
-
-GPUs do their computation with 16-bit or 32-bit floating point numbers and do
-not require quantization for optimal performance unlike the CPUs.  If
-quantization of your neural network was not an option due to lower accuracy
-caused by lost precision, such concern can be discarded when running deep neural
-net models on the GPU.
-
-Another benefit that comes with GPU inference is its power efficiency.  GPUs
-carry out the computations in a very efficient and optimized way, so that they
-consume less power and generate less heat than when the same task is run on the
-CPUs.
+easily divided into smaller workloads and carried out in parallel. This
+parallelism typically results in lower latency. In the best scenario, inference
+on the GPU may run fast enough to become suitable for real-time applications
+that were not previously possible.
+
+### Accuracy
+
+GPUs do their computation with 16-bit or 32-bit floating point numbers and
+(unlike the CPUs) do not require quantization for optimal performance. If
+decreased accuracy made quantization untenable for your models, running your
+neural network on a GPU may eliminate this concern.
+
+### Energy Efficiency
+
+Another benefit that comes with GPU inference is its power efficiency. A GPU
+carries out computations in a very efficient and optimized way, consuming less
+power and generating less heat than the same task run on a CPU.
+
+## Supported Ops
 
 TensorFlow Lite on GPU supports the following ops in 16-bit and 32-bit float
 precision:
@@ -51,8 +59,8 @@ precision:
 
 ### Android
 
-Using TensorFlow Lite on GPU is achieved via `TfLiteDelegate`.  In Java, you can
-specify the GpuDelegate through `Interpreter.Options`.
+Run TensorFlow Lite on GPU with `TfLiteDelegate`. In Java, you can specify the
+GpuDelegate through `Interpreter.Options`.
 
 ```java
 // NEW: Prepare GPU delegate.
@@ -73,10 +81,9 @@ delegate.close();
 
 ### iOS
 
-Using TensorFlow Lite on GPU is as simple as getting the GPU delegate via
-`NewGpuDelegate()` and then passing it to
-`Interpreter::ModifyGraphWithDelegate()` instead of calling
-`Interpreter::AllocateTensors()`:
+To use TensorFlow Lite on GPU, get the GPU delegate via `NewGpuDelegate()` and
+then pass it to `Interpreter::ModifyGraphWithDelegate()` (instead of calling
+`Interpreter::AllocateTensors()`).
 
 ```c++
 // Set up interpreter.
@@ -87,7 +94,13 @@ std::unique_ptr<Interpreter> interpreter;
 InterpreterBuilder(*model, op_resolver)(&interpreter);
 
 // NEW: Prepare GPU delegate.
-auto* delegate = NewGpuDelegate(nullptr);  // default config
+
+const GpuDelegateOptions options = {
+  .allow_precision_loss = false,
+  .wait_type = kGpuDelegateOptions::WaitType::Passive,
+};
+
+auto* delegate = NewGpuDelegate(options);
 if (interpreter->ModifyGraphWithDelegate(delegate) != kTfLiteOk) return false;
 
 // Run inference.
@@ -99,17 +112,19 @@ ReadFromOutputTensor(interpreter->typed_output_tensor<float>(0));
 DeleteGpuDelegate(delegate);
 ```
 
-*IMPORTANT:* When calling `Interpreter::ModifyGraphWithDelegate()` or
-`Interpreter::Invoke()`, the caller must have a `EGLContext` in the current
-thread and `Interpreter::Invoke()` must be called from the same `EGLContext`.
-If such `EGLContext` does not exist, the delegate will internally create one,
-but then the developer must ensure that `Interpreter::Invoke()` is always called
-from the same thread `Interpreter::ModifyGraphWithDelegate()` was called.
+Note: When calling `Interpreter::ModifyGraphWithDelegate()` or
+`Interpreter::Invoke()`, the caller must have an `EGLContext` in the current
+thread and `Interpreter::Invoke()` must be called from the same `EGLContext`. If
+an `EGLContext` does not exist, the delegate will internally create one, but
+then the developer must ensure that `Interpreter::Invoke()` is always called
+from the same thread in which `Interpreter::ModifyGraphWithDelegate()` was
+called.
 
-## Advanced: Delegate Options for iOS
+## Advanced Usage
 
-There are a couple of GPU options that can be set and passed on to
-`NewGpuDelegate()`:
+### Delegate Options for iOS
+
+`NewGpuDelegate()` accepts a `struct` of options.
 
 ```c++
 struct GpuDelegateOptions {
@@ -130,45 +145,53 @@ struct GpuDelegateOptions {
 };
 ```
 
-When option is set to `nullptr` as shown in the Basic Usage, it translates to:
+Passing `nullptr` into `NewGpuDelegate()` sets the default options (which are
+explicated in the Basic Usage example above).
 
 ```c++
+
+// THIS:
 const GpuDelegateOptions options = {
   .allow_precision_loss = false,
   .wait_type = kGpuDelegateOptions::WaitType::Passive,
 };
+
+auto* delegate = NewGpuDelegate(options);
+
+// IS THE SAME AS THIS:
+auto* delegate = NewGpuDelegate(nullptr);
+
 ```
 
-While it is convenient to just supply `nullptr`, it is recommended to explicitly
-set the options to avoid any unexpected artifacts in case default values are
-changed.
+While it is convenient to use `nullptr`, we recommend that you explicitly set
+the options, to avoid any unexpected behavior if default values are changed in
+the future.
 
-## Advanced: Input/Output Buffers
+### Input/Output Buffers
 
-To do computation on the GPU, data must be made available to the GPU which often
-translates to performing a memory copy.  It is desirable not to cross the
-CPU/GPU memory boundary if possible, as this can take up a significant amount of
-time.  Usually, such crossing is inevitable, but in some special cases, one or
-the other can be omitted.
+To do computation on the GPU, data must be made available to the GPU. This often
+requires performing a memory copy. It is desirable not to cross the CPU/GPU
+memory boundary if possible, as this can take up a significant amount of time.
+Usually, such crossing is inevitable, but in some special cases, one or the
+other can be omitted.
 
-If the network's input is an image already loaded in the GPU memory, e.g. a GPU
-texture containing the camera feed, it can stay in the GPU memory without ever
-entering the CPU memory.  Similarly, if the network's output is in the form of a
-renderable image, e.g.
-[image style transfer](https://www.cv-foundation.org/openaccess/content_cvpr_2016/papers/Gatys_Image_Style_Transfer_CVPR_2016_paper.pdf),
+If the network's input is an image already loaded in the GPU memory (for
+example, a GPU texture containing the camera feed) it can stay in the GPU memory
+without ever entering the CPU memory. Similarly, if the network's output is in
+the form of a renderable image (for example,
+[image style transfer](https://www.cv-foundation.org/openaccess/content_cvpr_2016/papers/Gatys_Image_Style_Transfer_CVPR_2016_paper.pdf)_)
 it can be directly displayed on the screen.
 
-To let users achieve best performance, TensorFlow Lite makes it possible for
-them to directly read from/write to the TensorFlow hardware buffer and bypass
+To achieve best performance, TensorFlow Lite makes it possible for users to
+directly read from and write to the TensorFlow hardware buffer and bypass
 avoidable memory copies.
 
-### Android
+#### Android
 
-Assuming the image input is in the GPU memory, it must be first converted to a
-OpenGL Shader Storage Buffer Object (SSBO).  One can associate a TfLiteTensor
-with user-prepared SSBO with `Interpreter.bindGlBufferToTensor()`.
-
-*IMPORTANT:* `Interpreter.bindGlBufferToTensor()` must be called before
+Assuming the image input is in the GPU memory, it must first be converted to an
+OpenGL Shader Storage Buffer Object (SSBO). You can associate a TfLiteTensor to
+a user-prepared SSBO with `Interpreter.bindGlBufferToTensor()`. Note that
+`Interpreter.bindGlBufferToTensor()` must be called before
 `Interpreter.modifyGraphWithDelegate()`.
 
 ```java
@@ -176,7 +199,7 @@ with user-prepared SSBO with `Interpreter.bindGlBufferToTensor()`.
 EGLContext eglContext = eglGetCurrentContext();
 if (eglContext.equals(EGL_NO_CONTEXT)) return false;
 
-// Create a SSBO.
+// Create an SSBO.
 int[] id = new int[1];
 glGenBuffers(id.length, id, 0);
 glBindBuffer(GL_SHADER_STORAGE_BUFFER, id[0]);
@@ -198,9 +221,10 @@ float[] outputArray = new float[outputSize];
 interpreter.runInference(null, outputArray);
 ```
 
-Similar approach can be applied to the output tensor.  In that case,
+A similar approach can be applied to the output tensor. In that case,
 `Interpreter.Options.setAllowBufferHandleOutput(true)` should be passed on, to
-disable the default copying the network's output from GPU memory to CPU memory.
+disable the default copying of the network's output from GPU memory to CPU
+memory.
 
 ```java
 // Ensure a valid EGL rendering context.
@@ -230,21 +254,16 @@ interpreter.runInference(input, null);
 renderOutputSsbo(outputSsboId);
 ```
 
-### iOS
-
-Assuming the image input is in the GPU memory, it must be first converted to a
-`MTLBuffer` object for Metal.  One can associate a TfLiteTensor with a
-user-prepared `MTLBuffer` with `BindMetalBufferToTensor()`.
-
-*IMPORTANT:* `BindMetalBufferToTensor()` must be called before
-`Interpreter::ModifyGraphWithDelegate()`.
+#### iOS
 
-*IMPORTANT:* By default, the inference output is copied from GPU memory to CPU
-memory implicitly by the framework.  This behavior can be turned off by calling
-`Interpreter::SetAllowBufferHandleOutput(true)` during initialization.  To copy
-the inference output from GPU memory to CPU memory, explicit
-`Interpreter::EnsureTensorDataIsReadable()` calls are required for each output
-tensor.
+Assuming the image input is in GPU memory, it must first be converted to a
+`MTLBuffer` object for Metal. You can associate a TfLiteTensor to a
+user-prepared `MTLBuffer` with `BindMetalBufferToTensor()`. Note that
+`BindMetalBufferToTensor()` must be called before
+`Interpreter::ModifyGraphWithDelegate()`. Additionally, the inference output is,
+by default, copied from GPU memory to CPU memory. This behavior can be turned
+off by calling `Interpreter::SetAllowBufferHandleOutput(true)` during
+initialization.
 
 ```c++
 // Prepare GPU delegate.
@@ -258,22 +277,27 @@ if (interpreter->ModifyGraphWithDelegate(delegate) != kTfLiteOk) return false;
 if (interpreter->Invoke() != kTfLiteOk) return false;
 ```
 
+Note: Once the default behavior is turned off, copying the inference output from
+GPU memory to CPU memory requires an explicit call to
+`Interpreter::EnsureTensorDataIsReadable()` for each output tensor.
+
 ## Tips and Tricks
 
-* Some operations that are trivial on CPU side may be high cost in GPU land.
-  One class of such operation is various forms of reshape operations (including
-  `BATCH_TO_SPACE`, `SPACE_TO_BATCH`, `SPACE_TO_DEPTH`, etc.).  If those ops
-  are inserted into the network just for the network architect's logical
-  thinking, it is worth removing them for performance.
+*   Some operations that are trivial on the CPU may be high cost on a GPU. One
+    class of such operation includes various forms of reshape operations
+    (including `BATCH_TO_SPACE`, `SPACE_TO_BATCH`, `SPACE_TO_DEPTH`, and similar
+    operation). If these operations are not required (for example, they were
+    inserted to help the network architect reason about the system but do not
+    otherwise affect output), it is worth removing them for performance.
 
-* On GPU, tensor data is sliced into 4-channels.  Thus, a computation on a
-  tensor of shape `[B, H, W, 5]` will perform about the same on a tensor of
-  shape `[B, H, W, 8]`, but significantly worse than `[B, H, W, 4]`.
+*   On a GPU, tensor data is sliced into 4-channels. Thus, a computation on a
+    tensor of shape `[B, H, W, 5]` will perform about the same on a tensor of
+    shape `[B, H, W, 8]`, but significantly worse than `[B, H, W, 4]`.
 
-* In that sense, if the camera hardware supports image frames in RGBA, feeding
-  that 4-channel input is significantly faster as a memory copy (from 3-channel
-  RGB to 4-channel RGBX) can be avoided.
+    *   For example, if the camera hardware supports image frames in RGBA,
+        feeding that 4-channel input is significantly faster, because a memory
+        copy (from 3-channel RGB to 4-channel RGBX) can be avoided.
 
-* For best performance, do not hesitate to re-train your classifier with
-  mobile-optimized network architecture.  That is a significant part of
-  optimization for on-device inference.
+*   For best performance, do not hesitate to re-train your classifier with
+    mobile-optimized network architecture. That is a significant part of
+    optimization for on-device inference.
-- 
GitLab


From 8d89bfeaae9540d60e290254a8a82828656d3e23 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 15 Jan 2019 18:25:17 -0800
Subject: [PATCH 0789/1095] Go: Update generated wrapper functions for
 TensorFlow ops. PiperOrigin-RevId: 229479202

---
 tensorflow/go/op/wrappers.go | 511 +++++++++++++++++++----------------
 1 file changed, 276 insertions(+), 235 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index f8a5b5e7e1..fbabb37c0e 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -6606,6 +6606,27 @@ func DeleteSessionTensor(scope *Scope, handle tf.Output) (o *tf.Operation) {
 	return scope.AddOperation(opspec)
 }
 
+// Store the input tensor in the state of the current session.
+//
+// Arguments:
+//	value: The tensor to be stored.
+//
+// Returns The handle for the tensor stored in the session state, represented
+// as a string.
+func GetSessionHandle(scope *Scope, value tf.Output) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "GetSessionHandle",
+		Input: []tf.Input{
+			value,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Compute the regularized incomplete beta integral \\(I_x(a, b)\\).
 //
 // The regularized incomplete beta integral is defined as:
@@ -11629,220 +11650,6 @@ func OneHot(scope *Scope, indices tf.Output, depth tf.Output, on_value tf.Output
 	return op.Output(0)
 }
 
-// Computes exponential of x element-wise.  \\(y = e^x\\).
-func Exp(scope *Scope, x tf.Output) (y tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Exp",
-		Input: []tf.Input{
-			x,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// NthElementAttr is an optional argument to NthElement.
-type NthElementAttr func(optionalAttr)
-
-// NthElementReverse sets the optional reverse attribute to value.
-//
-// value: When set to True, find the nth-largest value in the vector and vice
-// versa.
-// If not specified, defaults to false
-func NthElementReverse(value bool) NthElementAttr {
-	return func(m optionalAttr) {
-		m["reverse"] = value
-	}
-}
-
-// Finds values of the `n`-th order statistic for the last dimension.
-//
-// If the input is a vector (rank-1), finds the entries which is the nth-smallest
-// value in the vector and outputs their values as scalar tensor.
-//
-// For matrices (resp. higher rank input), computes the entries which is the
-// nth-smallest value in each row (resp. vector along the last dimension). Thus,
-//
-//     values.shape = input.shape[:-1]
-//
-// Arguments:
-//	input: 1-D or higher with last dimension at least `n+1`.
-//	n: 0-D. Position of sorted vector to select along the last dimension (along
-// each row for matrices). Valid range of n is `[0, input.shape[:-1])`
-//
-// Returns The `n`-th order statistic along each last dimensional slice.
-func NthElement(scope *Scope, input tf.Output, n tf.Output, optional ...NthElementAttr) (values tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "NthElement",
-		Input: []tf.Input{
-			input, n,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Computes the maximum along segments of a tensor.
-//
-// Read
-// [the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
-// for an explanation of segments.
-//
-// This operator is similar to the unsorted segment sum operator found
-// [(here)](../../../api_docs/python/math_ops.md#UnsortedSegmentSum).
-// Instead of computing the sum over segments, it computes the maximum such that:
-//
-// \\(output_i = \max_{j...} data[j...]\\) where max is over tuples `j...` such
-// that `segment_ids[j...] == i`.
-//
-// If the maximum is empty for a given segment ID `i`, it outputs the smallest
-// possible value for the specific numeric type,
-// `output[i] = numeric_limits<T>::lowest()`.
-//
-// If the given segment ID `i` is negative, then the corresponding value is
-// dropped, and will not be included in the result.
-//
-// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
-// <img style="width:100%" src="https://www.tensorflow.org/images/UnsortedSegmentMax.png" alt>
-// </div>
-//
-// Arguments:
-//
-//	segment_ids: A tensor whose shape is a prefix of `data.shape`.
-//
-//
-// Returns Has same shape as data, except for the first `segment_ids.rank`
-// dimensions, which are replaced with a single dimension which has size
-// `num_segments`.
-func UnsortedSegmentMax(scope *Scope, data tf.Output, segment_ids tf.Output, num_segments tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "UnsortedSegmentMax",
-		Input: []tf.Input{
-			data, segment_ids, num_segments,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Transforms a vector of brain.Example protos (as strings) into typed tensors.
-//
-// Arguments:
-//	serialized: A vector containing a batch of binary serialized Example protos.
-//	names: A vector containing the names of the serialized protos.
-// May contain, for example, table key (descriptive) names for the
-// corresponding serialized protos.  These are purely useful for debugging
-// purposes, and the presence of values here has no effect on the output.
-// May also be an empty vector if no names are available.
-// If non-empty, this vector must be the same length as "serialized".
-//	sparse_keys: A list of Nsparse string Tensors (scalars).
-// The keys expected in the Examples' features associated with sparse values.
-//	dense_keys: A list of Ndense string Tensors (scalars).
-// The keys expected in the Examples' features associated with dense values.
-//	dense_defaults: A list of Ndense Tensors (some may be empty).
-// dense_defaults[j] provides default values
-// when the example's feature_map lacks dense_key[j].  If an empty Tensor is
-// provided for dense_defaults[j], then the Feature dense_keys[j] is required.
-// The input type is inferred from dense_defaults[j], even when it's empty.
-// If dense_defaults[j] is not empty, and dense_shapes[j] is fully defined,
-// then the shape of dense_defaults[j] must match that of dense_shapes[j].
-// If dense_shapes[j] has an undefined major dimension (variable strides dense
-// feature), dense_defaults[j] must contain a single element:
-// the padding element.
-//	sparse_types: A list of Nsparse types; the data types of data in each Feature
-// given in sparse_keys.
-// Currently the ParseExample supports DT_FLOAT (FloatList),
-// DT_INT64 (Int64List), and DT_STRING (BytesList).
-//	dense_shapes: A list of Ndense shapes; the shapes of data in each Feature
-// given in dense_keys.
-// The number of elements in the Feature corresponding to dense_key[j]
-// must always equal dense_shapes[j].NumEntries().
-// If dense_shapes[j] == (D0, D1, ..., DN) then the shape of output
-// Tensor dense_values[j] will be (|serialized|, D0, D1, ..., DN):
-// The dense outputs are just the inputs row-stacked by batch.
-// This works for dense_shapes[j] = (-1, D1, ..., DN).  In this case
-// the shape of the output Tensor dense_values[j] will be
-// (|serialized|, M, D1, .., DN), where M is the maximum number of blocks
-// of elements of length D1 * .... * DN, across all minibatch entries
-// in the input.  Any minibatch entry with less than M blocks of elements of
-// length D1 * ... * DN will be padded with the corresponding default_value
-// scalar element along the second dimension.
-func ParseExample(scope *Scope, serialized tf.Output, names tf.Output, sparse_keys []tf.Output, dense_keys []tf.Output, dense_defaults []tf.Output, sparse_types []tf.DataType, dense_shapes []tf.Shape) (sparse_indices []tf.Output, sparse_values []tf.Output, sparse_shapes []tf.Output, dense_values []tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"sparse_types": sparse_types, "dense_shapes": dense_shapes}
-	opspec := tf.OpSpec{
-		Type: "ParseExample",
-		Input: []tf.Input{
-			serialized, names, tf.OutputList(sparse_keys), tf.OutputList(dense_keys), tf.OutputList(dense_defaults),
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	if sparse_indices, idx, err = makeOutputList(op, idx, "sparse_indices"); err != nil {
-		scope.UpdateErr("ParseExample", err)
-		return
-	}
-	if sparse_values, idx, err = makeOutputList(op, idx, "sparse_values"); err != nil {
-		scope.UpdateErr("ParseExample", err)
-		return
-	}
-	if sparse_shapes, idx, err = makeOutputList(op, idx, "sparse_shapes"); err != nil {
-		scope.UpdateErr("ParseExample", err)
-		return
-	}
-	if dense_values, idx, err = makeOutputList(op, idx, "dense_values"); err != nil {
-		scope.UpdateErr("ParseExample", err)
-		return
-	}
-	return sparse_indices, sparse_values, sparse_shapes, dense_values
-}
-
-// Compute the pairwise cross product.
-//
-// `a` and `b` must be the same shape; they can either be simple 3-element vectors,
-// or any shape where the innermost dimension is 3. In the latter case, each pair
-// of corresponding 3-element vectors is cross-multiplied independently.
-//
-// Arguments:
-//	a: A tensor containing 3-element vectors.
-//	b: Another tensor, of same type and shape as `a`.
-//
-// Returns Pairwise cross product of the vectors in `a` and `b`.
-func Cross(scope *Scope, a tf.Output, b tf.Output) (product tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Cross",
-		Input: []tf.Input{
-			a, b,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // CudnnRNNAttr is an optional argument to CudnnRNN.
 type CudnnRNNAttr func(optionalAttr)
 
@@ -12696,6 +12503,261 @@ func ResourceApplyFtrl(scope *Scope, var_ tf.Output, accum tf.Output, linear tf.
 	return scope.AddOperation(opspec)
 }
 
+// Computes exponential of x element-wise.  \\(y = e^x\\).
+func Exp(scope *Scope, x tf.Output) (y tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Exp",
+		Input: []tf.Input{
+			x,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// NthElementAttr is an optional argument to NthElement.
+type NthElementAttr func(optionalAttr)
+
+// NthElementReverse sets the optional reverse attribute to value.
+//
+// value: When set to True, find the nth-largest value in the vector and vice
+// versa.
+// If not specified, defaults to false
+func NthElementReverse(value bool) NthElementAttr {
+	return func(m optionalAttr) {
+		m["reverse"] = value
+	}
+}
+
+// Finds values of the `n`-th order statistic for the last dimension.
+//
+// If the input is a vector (rank-1), finds the entries which is the nth-smallest
+// value in the vector and outputs their values as scalar tensor.
+//
+// For matrices (resp. higher rank input), computes the entries which is the
+// nth-smallest value in each row (resp. vector along the last dimension). Thus,
+//
+//     values.shape = input.shape[:-1]
+//
+// Arguments:
+//	input: 1-D or higher with last dimension at least `n+1`.
+//	n: 0-D. Position of sorted vector to select along the last dimension (along
+// each row for matrices). Valid range of n is `[0, input.shape[:-1])`
+//
+// Returns The `n`-th order statistic along each last dimensional slice.
+func NthElement(scope *Scope, input tf.Output, n tf.Output, optional ...NthElementAttr) (values tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "NthElement",
+		Input: []tf.Input{
+			input, n,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Computes the maximum along segments of a tensor.
+//
+// Read
+// [the section on segmentation](https://tensorflow.org/api_guides/python/math_ops#Segmentation)
+// for an explanation of segments.
+//
+// This operator is similar to the unsorted segment sum operator found
+// [(here)](../../../api_docs/python/math_ops.md#UnsortedSegmentSum).
+// Instead of computing the sum over segments, it computes the maximum such that:
+//
+// \\(output_i = \max_{j...} data[j...]\\) where max is over tuples `j...` such
+// that `segment_ids[j...] == i`.
+//
+// If the maximum is empty for a given segment ID `i`, it outputs the smallest
+// possible value for the specific numeric type,
+// `output[i] = numeric_limits<T>::lowest()`.
+//
+// If the given segment ID `i` is negative, then the corresponding value is
+// dropped, and will not be included in the result.
+//
+// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+// <img style="width:100%" src="https://www.tensorflow.org/images/UnsortedSegmentMax.png" alt>
+// </div>
+//
+// Arguments:
+//
+//	segment_ids: A tensor whose shape is a prefix of `data.shape`.
+//
+//
+// Returns Has same shape as data, except for the first `segment_ids.rank`
+// dimensions, which are replaced with a single dimension which has size
+// `num_segments`.
+func UnsortedSegmentMax(scope *Scope, data tf.Output, segment_ids tf.Output, num_segments tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "UnsortedSegmentMax",
+		Input: []tf.Input{
+			data, segment_ids, num_segments,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Transforms a vector of brain.Example protos (as strings) into typed tensors.
+//
+// Arguments:
+//	serialized: A vector containing a batch of binary serialized Example protos.
+//	names: A vector containing the names of the serialized protos.
+// May contain, for example, table key (descriptive) names for the
+// corresponding serialized protos.  These are purely useful for debugging
+// purposes, and the presence of values here has no effect on the output.
+// May also be an empty vector if no names are available.
+// If non-empty, this vector must be the same length as "serialized".
+//	sparse_keys: A list of Nsparse string Tensors (scalars).
+// The keys expected in the Examples' features associated with sparse values.
+//	dense_keys: A list of Ndense string Tensors (scalars).
+// The keys expected in the Examples' features associated with dense values.
+//	dense_defaults: A list of Ndense Tensors (some may be empty).
+// dense_defaults[j] provides default values
+// when the example's feature_map lacks dense_key[j].  If an empty Tensor is
+// provided for dense_defaults[j], then the Feature dense_keys[j] is required.
+// The input type is inferred from dense_defaults[j], even when it's empty.
+// If dense_defaults[j] is not empty, and dense_shapes[j] is fully defined,
+// then the shape of dense_defaults[j] must match that of dense_shapes[j].
+// If dense_shapes[j] has an undefined major dimension (variable strides dense
+// feature), dense_defaults[j] must contain a single element:
+// the padding element.
+//	sparse_types: A list of Nsparse types; the data types of data in each Feature
+// given in sparse_keys.
+// Currently the ParseExample supports DT_FLOAT (FloatList),
+// DT_INT64 (Int64List), and DT_STRING (BytesList).
+//	dense_shapes: A list of Ndense shapes; the shapes of data in each Feature
+// given in dense_keys.
+// The number of elements in the Feature corresponding to dense_key[j]
+// must always equal dense_shapes[j].NumEntries().
+// If dense_shapes[j] == (D0, D1, ..., DN) then the shape of output
+// Tensor dense_values[j] will be (|serialized|, D0, D1, ..., DN):
+// The dense outputs are just the inputs row-stacked by batch.
+// This works for dense_shapes[j] = (-1, D1, ..., DN).  In this case
+// the shape of the output Tensor dense_values[j] will be
+// (|serialized|, M, D1, .., DN), where M is the maximum number of blocks
+// of elements of length D1 * .... * DN, across all minibatch entries
+// in the input.  Any minibatch entry with less than M blocks of elements of
+// length D1 * ... * DN will be padded with the corresponding default_value
+// scalar element along the second dimension.
+func ParseExample(scope *Scope, serialized tf.Output, names tf.Output, sparse_keys []tf.Output, dense_keys []tf.Output, dense_defaults []tf.Output, sparse_types []tf.DataType, dense_shapes []tf.Shape) (sparse_indices []tf.Output, sparse_values []tf.Output, sparse_shapes []tf.Output, dense_values []tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"sparse_types": sparse_types, "dense_shapes": dense_shapes}
+	opspec := tf.OpSpec{
+		Type: "ParseExample",
+		Input: []tf.Input{
+			serialized, names, tf.OutputList(sparse_keys), tf.OutputList(dense_keys), tf.OutputList(dense_defaults),
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if sparse_indices, idx, err = makeOutputList(op, idx, "sparse_indices"); err != nil {
+		scope.UpdateErr("ParseExample", err)
+		return
+	}
+	if sparse_values, idx, err = makeOutputList(op, idx, "sparse_values"); err != nil {
+		scope.UpdateErr("ParseExample", err)
+		return
+	}
+	if sparse_shapes, idx, err = makeOutputList(op, idx, "sparse_shapes"); err != nil {
+		scope.UpdateErr("ParseExample", err)
+		return
+	}
+	if dense_values, idx, err = makeOutputList(op, idx, "dense_values"); err != nil {
+		scope.UpdateErr("ParseExample", err)
+		return
+	}
+	return sparse_indices, sparse_values, sparse_shapes, dense_values
+}
+
+// Compute the pairwise cross product.
+//
+// `a` and `b` must be the same shape; they can either be simple 3-element vectors,
+// or any shape where the innermost dimension is 3. In the latter case, each pair
+// of corresponding 3-element vectors is cross-multiplied independently.
+//
+// Arguments:
+//	a: A tensor containing 3-element vectors.
+//	b: Another tensor, of same type and shape as `a`.
+//
+// Returns Pairwise cross product of the vectors in `a` and `b`.
+func Cross(scope *Scope, a tf.Output, b tf.Output) (product tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Cross",
+		Input: []tf.Input{
+			a, b,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// StatefulStandardNormalAttr is an optional argument to StatefulStandardNormal.
+type StatefulStandardNormalAttr func(optionalAttr)
+
+// StatefulStandardNormalDtype sets the optional dtype attribute to value.
+//
+// value: The type of the output.
+// If not specified, defaults to DT_FLOAT
+func StatefulStandardNormalDtype(value tf.DataType) StatefulStandardNormalAttr {
+	return func(m optionalAttr) {
+		m["dtype"] = value
+	}
+}
+
+// Outputs random values from a normal distribution.
+//
+// The generated values will have mean 0 and standard deviation 1.
+//
+// Arguments:
+//	resource: The handle of the resource variable that stores the state of the RNG.
+//	shape: The shape of the output tensor.
+//
+// Returns A tensor of the specified shape filled with random normal values.
+func StatefulStandardNormal(scope *Scope, resource tf.Output, shape tf.Output, optional ...StatefulStandardNormalAttr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "StatefulStandardNormal",
+		Input: []tf.Input{
+			resource, shape,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Locks a mutex resource.  The output is the lock.  So long as the lock tensor
 //
 // is alive, any other request to use `MutexLock` with this mutex will wait.
@@ -17519,27 +17581,6 @@ func DecodeBase64(scope *Scope, input tf.Output) (output tf.Output) {
 	return op.Output(0)
 }
 
-// Store the input tensor in the state of the current session.
-//
-// Arguments:
-//	value: The tensor to be stored.
-//
-// Returns The handle for the tensor stored in the session state, represented
-// as a string.
-func GetSessionHandle(scope *Scope, value tf.Output) (handle tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "GetSessionHandle",
-		Input: []tf.Input{
-			value,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // ResourceSparseApplyProximalAdagradAttr is an optional argument to ResourceSparseApplyProximalAdagrad.
 type ResourceSparseApplyProximalAdagradAttr func(optionalAttr)
 
-- 
GitLab


From 492a0e17ae125c17fb5a0d0406e7c255fec835b6 Mon Sep 17 00:00:00 2001
From: tomguluson92 <314913739@qq.com>
Date: Wed, 16 Jan 2019 11:07:16 +0800
Subject: [PATCH 0790/1095] fix spell mistake

---
 tensorflow/python/util/nest.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/util/nest.py b/tensorflow/python/util/nest.py
index 70e5ebb3b6..dbae97a373 100644
--- a/tensorflow/python/util/nest.py
+++ b/tensorflow/python/util/nest.py
@@ -149,7 +149,7 @@ def assert_same_structure(nest1, nest2, check_types=True):
 
   Note that namedtuples with identical name and fields are always considered
   to have the same shallow structure (even with `check_types=True`).
-  For intance, this code will print `True`:
+  For instance, this code will print `True`:
 
   ```python
   def nt(a, b):
-- 
GitLab


From 1f2952ec47015a300184e340d6f53aa42eb6eadf Mon Sep 17 00:00:00 2001
From: Yu-Cheng Ling <ycling@google.com>
Date: Tue, 15 Jan 2019 19:23:23 -0800
Subject: [PATCH 0791/1095] Update the MobileNet models used in iOS examples.

PiperOrigin-RevId: 229484657
---
 .../lite/examples/ios/camera/data/labels.txt  | 1001 +++++++++++++++++
 .../lite/examples/ios/download_models.sh      |   43 +-
 .../lite/examples/ios/simple/data/labels.txt  | 1001 +++++++++++++++++
 3 files changed, 2018 insertions(+), 27 deletions(-)
 create mode 100644 tensorflow/lite/examples/ios/camera/data/labels.txt
 create mode 100644 tensorflow/lite/examples/ios/simple/data/labels.txt

diff --git a/tensorflow/lite/examples/ios/camera/data/labels.txt b/tensorflow/lite/examples/ios/camera/data/labels.txt
new file mode 100644
index 0000000000..572eccf900
--- /dev/null
+++ b/tensorflow/lite/examples/ios/camera/data/labels.txt
@@ -0,0 +1,1001 @@
+dummy
+tench
+goldfish
+great white shark
+tiger shark
+hammerhead
+electric ray
+stingray
+cock
+hen
+ostrich
+brambling
+goldfinch
+house finch
+junco
+indigo bunting
+robin
+bulbul
+jay
+magpie
+chickadee
+water ouzel
+kite
+bald eagle
+vulture
+great grey owl
+European fire salamander
+common newt
+eft
+spotted salamander
+axolotl
+bullfrog
+tree frog
+tailed frog
+loggerhead
+leatherback turtle
+mud turtle
+terrapin
+box turtle
+banded gecko
+common iguana
+American chameleon
+whiptail
+agama
+frilled lizard
+alligator lizard
+Gila monster
+green lizard
+African chameleon
+Komodo dragon
+African crocodile
+American alligator
+triceratops
+thunder snake
+ringneck snake
+hognose snake
+green snake
+king snake
+garter snake
+water snake
+vine snake
+night snake
+boa constrictor
+rock python
+Indian cobra
+green mamba
+sea snake
+horned viper
+diamondback
+sidewinder
+trilobite
+harvestman
+scorpion
+black and gold garden spider
+barn spider
+garden spider
+black widow
+tarantula
+wolf spider
+tick
+centipede
+black grouse
+ptarmigan
+ruffed grouse
+prairie chicken
+peacock
+quail
+partridge
+African grey
+macaw
+sulphur-crested cockatoo
+lorikeet
+coucal
+bee eater
+hornbill
+hummingbird
+jacamar
+toucan
+drake
+red-breasted merganser
+goose
+black swan
+tusker
+echidna
+platypus
+wallaby
+koala
+wombat
+jellyfish
+sea anemone
+brain coral
+flatworm
+nematode
+conch
+snail
+slug
+sea slug
+chiton
+chambered nautilus
+Dungeness crab
+rock crab
+fiddler crab
+king crab
+American lobster
+spiny lobster
+crayfish
+hermit crab
+isopod
+white stork
+black stork
+spoonbill
+flamingo
+little blue heron
+American egret
+bittern
+crane
+limpkin
+European gallinule
+American coot
+bustard
+ruddy turnstone
+red-backed sandpiper
+redshank
+dowitcher
+oystercatcher
+pelican
+king penguin
+albatross
+grey whale
+killer whale
+dugong
+sea lion
+Chihuahua
+Japanese spaniel
+Maltese dog
+Pekinese
+Shih-Tzu
+Blenheim spaniel
+papillon
+toy terrier
+Rhodesian ridgeback
+Afghan hound
+basset
+beagle
+bloodhound
+bluetick
+black-and-tan coonhound
+Walker hound
+English foxhound
+redbone
+borzoi
+Irish wolfhound
+Italian greyhound
+whippet
+Ibizan hound
+Norwegian elkhound
+otterhound
+Saluki
+Scottish deerhound
+Weimaraner
+Staffordshire bullterrier
+American Staffordshire terrier
+Bedlington terrier
+Border terrier
+Kerry blue terrier
+Irish terrier
+Norfolk terrier
+Norwich terrier
+Yorkshire terrier
+wire-haired fox terrier
+Lakeland terrier
+Sealyham terrier
+Airedale
+cairn
+Australian terrier
+Dandie Dinmont
+Boston bull
+miniature schnauzer
+giant schnauzer
+standard schnauzer
+Scotch terrier
+Tibetan terrier
+silky terrier
+soft-coated wheaten terrier
+West Highland white terrier
+Lhasa
+flat-coated retriever
+curly-coated retriever
+golden retriever
+Labrador retriever
+Chesapeake Bay retriever
+German short-haired pointer
+vizsla
+English setter
+Irish setter
+Gordon setter
+Brittany spaniel
+clumber
+English springer
+Welsh springer spaniel
+cocker spaniel
+Sussex spaniel
+Irish water spaniel
+kuvasz
+schipperke
+groenendael
+malinois
+briard
+kelpie
+komondor
+Old English sheepdog
+Shetland sheepdog
+collie
+Border collie
+Bouvier des Flandres
+Rottweiler
+German shepherd
+Doberman
+miniature pinscher
+Greater Swiss Mountain dog
+Bernese mountain dog
+Appenzeller
+EntleBucher
+boxer
+bull mastiff
+Tibetan mastiff
+French bulldog
+Great Dane
+Saint Bernard
+Eskimo dog
+malamute
+Siberian husky
+dalmatian
+affenpinscher
+basenji
+pug
+Leonberg
+Newfoundland
+Great Pyrenees
+Samoyed
+Pomeranian
+chow
+keeshond
+Brabancon griffon
+Pembroke
+Cardigan
+toy poodle
+miniature poodle
+standard poodle
+Mexican hairless
+timber wolf
+white wolf
+red wolf
+coyote
+dingo
+dhole
+African hunting dog
+hyena
+red fox
+kit fox
+Arctic fox
+grey fox
+tabby
+tiger cat
+Persian cat
+Siamese cat
+Egyptian cat
+cougar
+lynx
+leopard
+snow leopard
+jaguar
+lion
+tiger
+cheetah
+brown bear
+American black bear
+ice bear
+sloth bear
+mongoose
+meerkat
+tiger beetle
+ladybug
+ground beetle
+long-horned beetle
+leaf beetle
+dung beetle
+rhinoceros beetle
+weevil
+fly
+bee
+ant
+grasshopper
+cricket
+walking stick
+cockroach
+mantis
+cicada
+leafhopper
+lacewing
+dragonfly
+damselfly
+admiral
+ringlet
+monarch
+cabbage butterfly
+sulphur butterfly
+lycaenid
+starfish
+sea urchin
+sea cucumber
+wood rabbit
+hare
+Angora
+hamster
+porcupine
+fox squirrel
+marmot
+beaver
+guinea pig
+sorrel
+zebra
+hog
+wild boar
+warthog
+hippopotamus
+ox
+water buffalo
+bison
+ram
+bighorn
+ibex
+hartebeest
+impala
+gazelle
+Arabian camel
+llama
+weasel
+mink
+polecat
+black-footed ferret
+otter
+skunk
+badger
+armadillo
+three-toed sloth
+orangutan
+gorilla
+chimpanzee
+gibbon
+siamang
+guenon
+patas
+baboon
+macaque
+langur
+colobus
+proboscis monkey
+marmoset
+capuchin
+howler monkey
+titi
+spider monkey
+squirrel monkey
+Madagascar cat
+indri
+Indian elephant
+African elephant
+lesser panda
+giant panda
+barracouta
+eel
+coho
+rock beauty
+anemone fish
+sturgeon
+gar
+lionfish
+puffer
+abacus
+abaya
+academic gown
+accordion
+acoustic guitar
+aircraft carrier
+airliner
+airship
+altar
+ambulance
+amphibian
+analog clock
+apiary
+apron
+ashcan
+assault rifle
+backpack
+bakery
+balance beam
+balloon
+ballpoint
+Band Aid
+banjo
+bannister
+barbell
+barber chair
+barbershop
+barn
+barometer
+barrel
+barrow
+baseball
+basketball
+bassinet
+bassoon
+bathing cap
+bath towel
+bathtub
+beach wagon
+beacon
+beaker
+bearskin
+beer bottle
+beer glass
+bell cote
+bib
+bicycle-built-for-two
+bikini
+binder
+binoculars
+birdhouse
+boathouse
+bobsled
+bolo tie
+bonnet
+bookcase
+bookshop
+bottlecap
+bow
+bow tie
+brass
+brassiere
+breakwater
+breastplate
+broom
+bucket
+buckle
+bulletproof vest
+bullet train
+butcher shop
+cab
+caldron
+candle
+cannon
+canoe
+can opener
+cardigan
+car mirror
+carousel
+carpenter's kit
+carton
+car wheel
+cash machine
+cassette
+cassette player
+castle
+catamaran
+CD player
+cello
+cellular telephone
+chain
+chainlink fence
+chain mail
+chain saw
+chest
+chiffonier
+chime
+china cabinet
+Christmas stocking
+church
+cinema
+cleaver
+cliff dwelling
+cloak
+clog
+cocktail shaker
+coffee mug
+coffeepot
+coil
+combination lock
+computer keyboard
+confectionery
+container ship
+convertible
+corkscrew
+cornet
+cowboy boot
+cowboy hat
+cradle
+crane
+crash helmet
+crate
+crib
+Crock Pot
+croquet ball
+crutch
+cuirass
+dam
+desk
+desktop computer
+dial telephone
+diaper
+digital clock
+digital watch
+dining table
+dishrag
+dishwasher
+disk brake
+dock
+dogsled
+dome
+doormat
+drilling platform
+drum
+drumstick
+dumbbell
+Dutch oven
+electric fan
+electric guitar
+electric locomotive
+entertainment center
+envelope
+espresso maker
+face powder
+feather boa
+file
+fireboat
+fire engine
+fire screen
+flagpole
+flute
+folding chair
+football helmet
+forklift
+fountain
+fountain pen
+four-poster
+freight car
+French horn
+frying pan
+fur coat
+garbage truck
+gasmask
+gas pump
+goblet
+go-kart
+golf ball
+golfcart
+gondola
+gong
+gown
+grand piano
+greenhouse
+grille
+grocery store
+guillotine
+hair slide
+hair spray
+half track
+hammer
+hamper
+hand blower
+hand-held computer
+handkerchief
+hard disc
+harmonica
+harp
+harvester
+hatchet
+holster
+home theater
+honeycomb
+hook
+hoopskirt
+horizontal bar
+horse cart
+hourglass
+iPod
+iron
+jack-o'-lantern
+jean
+jeep
+jersey
+jigsaw puzzle
+jinrikisha
+joystick
+kimono
+knee pad
+knot
+lab coat
+ladle
+lampshade
+laptop
+lawn mower
+lens cap
+letter opener
+library
+lifeboat
+lighter
+limousine
+liner
+lipstick
+Loafer
+lotion
+loudspeaker
+loupe
+lumbermill
+magnetic compass
+mailbag
+mailbox
+maillot
+maillot
+manhole cover
+maraca
+marimba
+mask
+matchstick
+maypole
+maze
+measuring cup
+medicine chest
+megalith
+microphone
+microwave
+military uniform
+milk can
+minibus
+miniskirt
+minivan
+missile
+mitten
+mixing bowl
+mobile home
+Model T
+modem
+monastery
+monitor
+moped
+mortar
+mortarboard
+mosque
+mosquito net
+motor scooter
+mountain bike
+mountain tent
+mouse
+mousetrap
+moving van
+muzzle
+nail
+neck brace
+necklace
+nipple
+notebook
+obelisk
+oboe
+ocarina
+odometer
+oil filter
+organ
+oscilloscope
+overskirt
+oxcart
+oxygen mask
+packet
+paddle
+paddlewheel
+padlock
+paintbrush
+pajama
+palace
+panpipe
+paper towel
+parachute
+parallel bars
+park bench
+parking meter
+passenger car
+patio
+pay-phone
+pedestal
+pencil box
+pencil sharpener
+perfume
+Petri dish
+photocopier
+pick
+pickelhaube
+picket fence
+pickup
+pier
+piggy bank
+pill bottle
+pillow
+ping-pong ball
+pinwheel
+pirate
+pitcher
+plane
+planetarium
+plastic bag
+plate rack
+plow
+plunger
+Polaroid camera
+pole
+police van
+poncho
+pool table
+pop bottle
+pot
+potter's wheel
+power drill
+prayer rug
+printer
+prison
+projectile
+projector
+puck
+punching bag
+purse
+quill
+quilt
+racer
+racket
+radiator
+radio
+radio telescope
+rain barrel
+recreational vehicle
+reel
+reflex camera
+refrigerator
+remote control
+restaurant
+revolver
+rifle
+rocking chair
+rotisserie
+rubber eraser
+rugby ball
+rule
+running shoe
+safe
+safety pin
+saltshaker
+sandal
+sarong
+sax
+scabbard
+scale
+school bus
+schooner
+scoreboard
+screen
+screw
+screwdriver
+seat belt
+sewing machine
+shield
+shoe shop
+shoji
+shopping basket
+shopping cart
+shovel
+shower cap
+shower curtain
+ski
+ski mask
+sleeping bag
+slide rule
+sliding door
+slot
+snorkel
+snowmobile
+snowplow
+soap dispenser
+soccer ball
+sock
+solar dish
+sombrero
+soup bowl
+space bar
+space heater
+space shuttle
+spatula
+speedboat
+spider web
+spindle
+sports car
+spotlight
+stage
+steam locomotive
+steel arch bridge
+steel drum
+stethoscope
+stole
+stone wall
+stopwatch
+stove
+strainer
+streetcar
+stretcher
+studio couch
+stupa
+submarine
+suit
+sundial
+sunglass
+sunglasses
+sunscreen
+suspension bridge
+swab
+sweatshirt
+swimming trunks
+swing
+switch
+syringe
+table lamp
+tank
+tape player
+teapot
+teddy
+television
+tennis ball
+thatch
+theater curtain
+thimble
+thresher
+throne
+tile roof
+toaster
+tobacco shop
+toilet seat
+torch
+totem pole
+tow truck
+toyshop
+tractor
+trailer truck
+tray
+trench coat
+tricycle
+trimaran
+tripod
+triumphal arch
+trolleybus
+trombone
+tub
+turnstile
+typewriter keyboard
+umbrella
+unicycle
+upright
+vacuum
+vase
+vault
+velvet
+vending machine
+vestment
+viaduct
+violin
+volleyball
+waffle iron
+wall clock
+wallet
+wardrobe
+warplane
+washbasin
+washer
+water bottle
+water jug
+water tower
+whiskey jug
+whistle
+wig
+window screen
+window shade
+Windsor tie
+wine bottle
+wing
+wok
+wooden spoon
+wool
+worm fence
+wreck
+yawl
+yurt
+web site
+comic book
+crossword puzzle
+street sign
+traffic light
+book jacket
+menu
+plate
+guacamole
+consomme
+hot pot
+trifle
+ice cream
+ice lolly
+French loaf
+bagel
+pretzel
+cheeseburger
+hotdog
+mashed potato
+head cabbage
+broccoli
+cauliflower
+zucchini
+spaghetti squash
+acorn squash
+butternut squash
+cucumber
+artichoke
+bell pepper
+cardoon
+mushroom
+Granny Smith
+strawberry
+orange
+lemon
+fig
+pineapple
+banana
+jackfruit
+custard apple
+pomegranate
+hay
+carbonara
+chocolate sauce
+dough
+meat loaf
+pizza
+potpie
+burrito
+red wine
+espresso
+cup
+eggnog
+alp
+bubble
+cliff
+coral reef
+geyser
+lakeside
+promontory
+sandbar
+seashore
+valley
+volcano
+ballplayer
+groom
+scuba diver
+rapeseed
+daisy
+yellow lady's slipper
+corn
+acorn
+hip
+buckeye
+coral fungus
+agaric
+gyromitra
+stinkhorn
+earthstar
+hen-of-the-woods
+bolete
+ear
+toilet tissue
diff --git a/tensorflow/lite/examples/ios/download_models.sh b/tensorflow/lite/examples/ios/download_models.sh
index 4828617d95..a450aba042 100755
--- a/tensorflow/lite/examples/ios/download_models.sh
+++ b/tensorflow/lite/examples/ios/download_models.sh
@@ -17,42 +17,31 @@
 set -ex
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-MODELS_URL="https://storage.googleapis.com/download.tensorflow.org/models/tflite/mobilenet_v1_1.0_224_ios_lite_float_2017_11_08.zip"
-QUANTIZED_MODELS_URL="https://storage.googleapis.com/download.tensorflow.org/models/tflite/mobilenet_v1_224_android_quant_2017_11_08.zip"
+FLOAT_MODEL_URL="http://download.tensorflow.org/models/mobilenet_v1_2018_02_22/mobilenet_v1_1.0_224.tgz"
+QUANTIZED_MODEL_URL="http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_1.0_224_quant.tgz"
 DOWNLOADS_DIR=$(mktemp -d)
 
-cd $SCRIPT_DIR
+cd "$SCRIPT_DIR"
 
 download_and_extract() {
-  local usage="Usage: download_and_extract URL DIR"
-  local url="${1:?${usage}}"
-  local dir="${2:?${usage}}"
+  local url="$1"
+  local dir="$2"
   echo "downloading ${url}" >&2
   mkdir -p "${dir}"
   tempdir=$(mktemp -d)
-  tempdir2=$(mktemp -d)
 
-  curl -L ${url} > ${tempdir}/zipped.zip
-  unzip ${tempdir}/zipped.zip -d ${tempdir2}
-
-  # If the zip file contains nested directories, extract the files from the
-  # inner directory.
-  if ls ${tempdir2}/*/* 1> /dev/null 2>&1; then
-    # unzip has no strip components, so unzip to a temp dir, and move the
-    # files we want from the tempdir to destination.
-    cp -R ${tempdir2}/*/* ${dir}/
-  else
-    cp -R ${tempdir2}/* ${dir}/
-  fi
-  rm -rf ${tempdir2} ${tempdir}
+  curl -L ${url} > ${tempdir}/archive.tgz
+  cd ${dir}
+  tar zxvf ${tempdir}/archive.tgz
+  rm -rf ${tempdir}
 }
 
-download_and_extract "${MODELS_URL}" "${DOWNLOADS_DIR}/models"
-download_and_extract "${QUANTIZED_MODELS_URL}" "${DOWNLOADS_DIR}/quantized_models"
-
-file ${DOWNLOADS_DIR}/models
+download_and_extract "${FLOAT_MODEL_URL}" "${DOWNLOADS_DIR}/float_model"
+download_and_extract "${QUANTIZED_MODEL_URL}" "${DOWNLOADS_DIR}/quantized_model"
 
-cp ${DOWNLOADS_DIR}/models/models/* simple/data/
-cp ${DOWNLOADS_DIR}/models/models/* camera/data/
-cp "${DOWNLOADS_DIR}/quantized_models/mobilenet_quant_v1_224.tflite" \
+cd "$SCRIPT_DIR"
+cp "${DOWNLOADS_DIR}/float_model/mobilenet_v1_1.0_224.tflite" "simple/data/mobilenet_v1_1.0_224.tflite"
+cp "${DOWNLOADS_DIR}/float_model/mobilenet_v1_1.0_224.tflite" "camera/data/mobilenet_v1_1.0_224.tflite"
+cp "${DOWNLOADS_DIR}/quantized_model/mobilenet_v1_1.0_224_quant.tflite" \
    'camera/data/mobilenet_quant_v1_224.tflite'
+echo "Done"
diff --git a/tensorflow/lite/examples/ios/simple/data/labels.txt b/tensorflow/lite/examples/ios/simple/data/labels.txt
new file mode 100644
index 0000000000..572eccf900
--- /dev/null
+++ b/tensorflow/lite/examples/ios/simple/data/labels.txt
@@ -0,0 +1,1001 @@
+dummy
+tench
+goldfish
+great white shark
+tiger shark
+hammerhead
+electric ray
+stingray
+cock
+hen
+ostrich
+brambling
+goldfinch
+house finch
+junco
+indigo bunting
+robin
+bulbul
+jay
+magpie
+chickadee
+water ouzel
+kite
+bald eagle
+vulture
+great grey owl
+European fire salamander
+common newt
+eft
+spotted salamander
+axolotl
+bullfrog
+tree frog
+tailed frog
+loggerhead
+leatherback turtle
+mud turtle
+terrapin
+box turtle
+banded gecko
+common iguana
+American chameleon
+whiptail
+agama
+frilled lizard
+alligator lizard
+Gila monster
+green lizard
+African chameleon
+Komodo dragon
+African crocodile
+American alligator
+triceratops
+thunder snake
+ringneck snake
+hognose snake
+green snake
+king snake
+garter snake
+water snake
+vine snake
+night snake
+boa constrictor
+rock python
+Indian cobra
+green mamba
+sea snake
+horned viper
+diamondback
+sidewinder
+trilobite
+harvestman
+scorpion
+black and gold garden spider
+barn spider
+garden spider
+black widow
+tarantula
+wolf spider
+tick
+centipede
+black grouse
+ptarmigan
+ruffed grouse
+prairie chicken
+peacock
+quail
+partridge
+African grey
+macaw
+sulphur-crested cockatoo
+lorikeet
+coucal
+bee eater
+hornbill
+hummingbird
+jacamar
+toucan
+drake
+red-breasted merganser
+goose
+black swan
+tusker
+echidna
+platypus
+wallaby
+koala
+wombat
+jellyfish
+sea anemone
+brain coral
+flatworm
+nematode
+conch
+snail
+slug
+sea slug
+chiton
+chambered nautilus
+Dungeness crab
+rock crab
+fiddler crab
+king crab
+American lobster
+spiny lobster
+crayfish
+hermit crab
+isopod
+white stork
+black stork
+spoonbill
+flamingo
+little blue heron
+American egret
+bittern
+crane
+limpkin
+European gallinule
+American coot
+bustard
+ruddy turnstone
+red-backed sandpiper
+redshank
+dowitcher
+oystercatcher
+pelican
+king penguin
+albatross
+grey whale
+killer whale
+dugong
+sea lion
+Chihuahua
+Japanese spaniel
+Maltese dog
+Pekinese
+Shih-Tzu
+Blenheim spaniel
+papillon
+toy terrier
+Rhodesian ridgeback
+Afghan hound
+basset
+beagle
+bloodhound
+bluetick
+black-and-tan coonhound
+Walker hound
+English foxhound
+redbone
+borzoi
+Irish wolfhound
+Italian greyhound
+whippet
+Ibizan hound
+Norwegian elkhound
+otterhound
+Saluki
+Scottish deerhound
+Weimaraner
+Staffordshire bullterrier
+American Staffordshire terrier
+Bedlington terrier
+Border terrier
+Kerry blue terrier
+Irish terrier
+Norfolk terrier
+Norwich terrier
+Yorkshire terrier
+wire-haired fox terrier
+Lakeland terrier
+Sealyham terrier
+Airedale
+cairn
+Australian terrier
+Dandie Dinmont
+Boston bull
+miniature schnauzer
+giant schnauzer
+standard schnauzer
+Scotch terrier
+Tibetan terrier
+silky terrier
+soft-coated wheaten terrier
+West Highland white terrier
+Lhasa
+flat-coated retriever
+curly-coated retriever
+golden retriever
+Labrador retriever
+Chesapeake Bay retriever
+German short-haired pointer
+vizsla
+English setter
+Irish setter
+Gordon setter
+Brittany spaniel
+clumber
+English springer
+Welsh springer spaniel
+cocker spaniel
+Sussex spaniel
+Irish water spaniel
+kuvasz
+schipperke
+groenendael
+malinois
+briard
+kelpie
+komondor
+Old English sheepdog
+Shetland sheepdog
+collie
+Border collie
+Bouvier des Flandres
+Rottweiler
+German shepherd
+Doberman
+miniature pinscher
+Greater Swiss Mountain dog
+Bernese mountain dog
+Appenzeller
+EntleBucher
+boxer
+bull mastiff
+Tibetan mastiff
+French bulldog
+Great Dane
+Saint Bernard
+Eskimo dog
+malamute
+Siberian husky
+dalmatian
+affenpinscher
+basenji
+pug
+Leonberg
+Newfoundland
+Great Pyrenees
+Samoyed
+Pomeranian
+chow
+keeshond
+Brabancon griffon
+Pembroke
+Cardigan
+toy poodle
+miniature poodle
+standard poodle
+Mexican hairless
+timber wolf
+white wolf
+red wolf
+coyote
+dingo
+dhole
+African hunting dog
+hyena
+red fox
+kit fox
+Arctic fox
+grey fox
+tabby
+tiger cat
+Persian cat
+Siamese cat
+Egyptian cat
+cougar
+lynx
+leopard
+snow leopard
+jaguar
+lion
+tiger
+cheetah
+brown bear
+American black bear
+ice bear
+sloth bear
+mongoose
+meerkat
+tiger beetle
+ladybug
+ground beetle
+long-horned beetle
+leaf beetle
+dung beetle
+rhinoceros beetle
+weevil
+fly
+bee
+ant
+grasshopper
+cricket
+walking stick
+cockroach
+mantis
+cicada
+leafhopper
+lacewing
+dragonfly
+damselfly
+admiral
+ringlet
+monarch
+cabbage butterfly
+sulphur butterfly
+lycaenid
+starfish
+sea urchin
+sea cucumber
+wood rabbit
+hare
+Angora
+hamster
+porcupine
+fox squirrel
+marmot
+beaver
+guinea pig
+sorrel
+zebra
+hog
+wild boar
+warthog
+hippopotamus
+ox
+water buffalo
+bison
+ram
+bighorn
+ibex
+hartebeest
+impala
+gazelle
+Arabian camel
+llama
+weasel
+mink
+polecat
+black-footed ferret
+otter
+skunk
+badger
+armadillo
+three-toed sloth
+orangutan
+gorilla
+chimpanzee
+gibbon
+siamang
+guenon
+patas
+baboon
+macaque
+langur
+colobus
+proboscis monkey
+marmoset
+capuchin
+howler monkey
+titi
+spider monkey
+squirrel monkey
+Madagascar cat
+indri
+Indian elephant
+African elephant
+lesser panda
+giant panda
+barracouta
+eel
+coho
+rock beauty
+anemone fish
+sturgeon
+gar
+lionfish
+puffer
+abacus
+abaya
+academic gown
+accordion
+acoustic guitar
+aircraft carrier
+airliner
+airship
+altar
+ambulance
+amphibian
+analog clock
+apiary
+apron
+ashcan
+assault rifle
+backpack
+bakery
+balance beam
+balloon
+ballpoint
+Band Aid
+banjo
+bannister
+barbell
+barber chair
+barbershop
+barn
+barometer
+barrel
+barrow
+baseball
+basketball
+bassinet
+bassoon
+bathing cap
+bath towel
+bathtub
+beach wagon
+beacon
+beaker
+bearskin
+beer bottle
+beer glass
+bell cote
+bib
+bicycle-built-for-two
+bikini
+binder
+binoculars
+birdhouse
+boathouse
+bobsled
+bolo tie
+bonnet
+bookcase
+bookshop
+bottlecap
+bow
+bow tie
+brass
+brassiere
+breakwater
+breastplate
+broom
+bucket
+buckle
+bulletproof vest
+bullet train
+butcher shop
+cab
+caldron
+candle
+cannon
+canoe
+can opener
+cardigan
+car mirror
+carousel
+carpenter's kit
+carton
+car wheel
+cash machine
+cassette
+cassette player
+castle
+catamaran
+CD player
+cello
+cellular telephone
+chain
+chainlink fence
+chain mail
+chain saw
+chest
+chiffonier
+chime
+china cabinet
+Christmas stocking
+church
+cinema
+cleaver
+cliff dwelling
+cloak
+clog
+cocktail shaker
+coffee mug
+coffeepot
+coil
+combination lock
+computer keyboard
+confectionery
+container ship
+convertible
+corkscrew
+cornet
+cowboy boot
+cowboy hat
+cradle
+crane
+crash helmet
+crate
+crib
+Crock Pot
+croquet ball
+crutch
+cuirass
+dam
+desk
+desktop computer
+dial telephone
+diaper
+digital clock
+digital watch
+dining table
+dishrag
+dishwasher
+disk brake
+dock
+dogsled
+dome
+doormat
+drilling platform
+drum
+drumstick
+dumbbell
+Dutch oven
+electric fan
+electric guitar
+electric locomotive
+entertainment center
+envelope
+espresso maker
+face powder
+feather boa
+file
+fireboat
+fire engine
+fire screen
+flagpole
+flute
+folding chair
+football helmet
+forklift
+fountain
+fountain pen
+four-poster
+freight car
+French horn
+frying pan
+fur coat
+garbage truck
+gasmask
+gas pump
+goblet
+go-kart
+golf ball
+golfcart
+gondola
+gong
+gown
+grand piano
+greenhouse
+grille
+grocery store
+guillotine
+hair slide
+hair spray
+half track
+hammer
+hamper
+hand blower
+hand-held computer
+handkerchief
+hard disc
+harmonica
+harp
+harvester
+hatchet
+holster
+home theater
+honeycomb
+hook
+hoopskirt
+horizontal bar
+horse cart
+hourglass
+iPod
+iron
+jack-o'-lantern
+jean
+jeep
+jersey
+jigsaw puzzle
+jinrikisha
+joystick
+kimono
+knee pad
+knot
+lab coat
+ladle
+lampshade
+laptop
+lawn mower
+lens cap
+letter opener
+library
+lifeboat
+lighter
+limousine
+liner
+lipstick
+Loafer
+lotion
+loudspeaker
+loupe
+lumbermill
+magnetic compass
+mailbag
+mailbox
+maillot
+maillot
+manhole cover
+maraca
+marimba
+mask
+matchstick
+maypole
+maze
+measuring cup
+medicine chest
+megalith
+microphone
+microwave
+military uniform
+milk can
+minibus
+miniskirt
+minivan
+missile
+mitten
+mixing bowl
+mobile home
+Model T
+modem
+monastery
+monitor
+moped
+mortar
+mortarboard
+mosque
+mosquito net
+motor scooter
+mountain bike
+mountain tent
+mouse
+mousetrap
+moving van
+muzzle
+nail
+neck brace
+necklace
+nipple
+notebook
+obelisk
+oboe
+ocarina
+odometer
+oil filter
+organ
+oscilloscope
+overskirt
+oxcart
+oxygen mask
+packet
+paddle
+paddlewheel
+padlock
+paintbrush
+pajama
+palace
+panpipe
+paper towel
+parachute
+parallel bars
+park bench
+parking meter
+passenger car
+patio
+pay-phone
+pedestal
+pencil box
+pencil sharpener
+perfume
+Petri dish
+photocopier
+pick
+pickelhaube
+picket fence
+pickup
+pier
+piggy bank
+pill bottle
+pillow
+ping-pong ball
+pinwheel
+pirate
+pitcher
+plane
+planetarium
+plastic bag
+plate rack
+plow
+plunger
+Polaroid camera
+pole
+police van
+poncho
+pool table
+pop bottle
+pot
+potter's wheel
+power drill
+prayer rug
+printer
+prison
+projectile
+projector
+puck
+punching bag
+purse
+quill
+quilt
+racer
+racket
+radiator
+radio
+radio telescope
+rain barrel
+recreational vehicle
+reel
+reflex camera
+refrigerator
+remote control
+restaurant
+revolver
+rifle
+rocking chair
+rotisserie
+rubber eraser
+rugby ball
+rule
+running shoe
+safe
+safety pin
+saltshaker
+sandal
+sarong
+sax
+scabbard
+scale
+school bus
+schooner
+scoreboard
+screen
+screw
+screwdriver
+seat belt
+sewing machine
+shield
+shoe shop
+shoji
+shopping basket
+shopping cart
+shovel
+shower cap
+shower curtain
+ski
+ski mask
+sleeping bag
+slide rule
+sliding door
+slot
+snorkel
+snowmobile
+snowplow
+soap dispenser
+soccer ball
+sock
+solar dish
+sombrero
+soup bowl
+space bar
+space heater
+space shuttle
+spatula
+speedboat
+spider web
+spindle
+sports car
+spotlight
+stage
+steam locomotive
+steel arch bridge
+steel drum
+stethoscope
+stole
+stone wall
+stopwatch
+stove
+strainer
+streetcar
+stretcher
+studio couch
+stupa
+submarine
+suit
+sundial
+sunglass
+sunglasses
+sunscreen
+suspension bridge
+swab
+sweatshirt
+swimming trunks
+swing
+switch
+syringe
+table lamp
+tank
+tape player
+teapot
+teddy
+television
+tennis ball
+thatch
+theater curtain
+thimble
+thresher
+throne
+tile roof
+toaster
+tobacco shop
+toilet seat
+torch
+totem pole
+tow truck
+toyshop
+tractor
+trailer truck
+tray
+trench coat
+tricycle
+trimaran
+tripod
+triumphal arch
+trolleybus
+trombone
+tub
+turnstile
+typewriter keyboard
+umbrella
+unicycle
+upright
+vacuum
+vase
+vault
+velvet
+vending machine
+vestment
+viaduct
+violin
+volleyball
+waffle iron
+wall clock
+wallet
+wardrobe
+warplane
+washbasin
+washer
+water bottle
+water jug
+water tower
+whiskey jug
+whistle
+wig
+window screen
+window shade
+Windsor tie
+wine bottle
+wing
+wok
+wooden spoon
+wool
+worm fence
+wreck
+yawl
+yurt
+web site
+comic book
+crossword puzzle
+street sign
+traffic light
+book jacket
+menu
+plate
+guacamole
+consomme
+hot pot
+trifle
+ice cream
+ice lolly
+French loaf
+bagel
+pretzel
+cheeseburger
+hotdog
+mashed potato
+head cabbage
+broccoli
+cauliflower
+zucchini
+spaghetti squash
+acorn squash
+butternut squash
+cucumber
+artichoke
+bell pepper
+cardoon
+mushroom
+Granny Smith
+strawberry
+orange
+lemon
+fig
+pineapple
+banana
+jackfruit
+custard apple
+pomegranate
+hay
+carbonara
+chocolate sauce
+dough
+meat loaf
+pizza
+potpie
+burrito
+red wine
+espresso
+cup
+eggnog
+alp
+bubble
+cliff
+coral reef
+geyser
+lakeside
+promontory
+sandbar
+seashore
+valley
+volcano
+ballplayer
+groom
+scuba diver
+rapeseed
+daisy
+yellow lady's slipper
+corn
+acorn
+hip
+buckeye
+coral fungus
+agaric
+gyromitra
+stinkhorn
+earthstar
+hen-of-the-woods
+bolete
+ear
+toilet tissue
-- 
GitLab


From bc4a279ca963a901576a9a7a35b189dfae7f5c85 Mon Sep 17 00:00:00 2001
From: Tim Shen <timshen@google.com>
Date: Tue, 15 Jan 2019 19:58:24 -0800
Subject: [PATCH 0792/1095] Unbreak non-CUDA builds, since the testing bot
 tests all targets in the directory "//tensorflow/...".

PiperOrigin-RevId: 229487682
---
 tensorflow/stream_executor/cuda/BUILD | 134 +++++++++++++-------------
 1 file changed, 67 insertions(+), 67 deletions(-)

diff --git a/tensorflow/stream_executor/cuda/BUILD b/tensorflow/stream_executor/cuda/BUILD
index 87c8eae416..fcfda1337e 100644
--- a/tensorflow/stream_executor/cuda/BUILD
+++ b/tensorflow/stream_executor/cuda/BUILD
@@ -42,10 +42,10 @@ cc_library(
 
 cc_library(
     name = "cuda_platform",
-    srcs = ["cuda_platform.cc"],
-    hdrs = ["cuda_platform.h"],
+    srcs = if_cuda_is_configured(["cuda_platform.cc"]),
+    hdrs = if_cuda_is_configured(["cuda_platform.h"]),
     visibility = ["//visibility:public"],
-    deps = [
+    deps = if_cuda_is_configured([
         ":cuda_driver",
         ":cuda_gpu_executor",
         ":cuda_platform_id",
@@ -55,27 +55,27 @@ cc_library(
         "//tensorflow/stream_executor:stream_executor_pimpl_header",
         "//tensorflow/stream_executor/lib",
         "//tensorflow/stream_executor/platform",
-    ] + tf_additional_cuda_platform_deps(),
+    ] + tf_additional_cuda_platform_deps()),
     alwayslink = True,  # Registers itself with the MultiPlatformManager.
 )
 
 cc_library(
     name = "cuda_diagnostics",
-    srcs = ["cuda_diagnostics.cc"],
-    hdrs = ["cuda_diagnostics.h"],
-    deps = [
-        "//tensorflow/stream_executor/lib",
-        "//tensorflow/stream_executor/platform",
+    srcs = if_cuda_is_configured(["cuda_diagnostics.cc"]),
+    hdrs = if_cuda_is_configured(["cuda_diagnostics.h"]),
+    deps = if_cuda_is_configured([
         "@com_google_absl//absl/container:inlined_vector",
         "@com_google_absl//absl/strings",
-    ],
+        "//tensorflow/stream_executor/lib",
+        "//tensorflow/stream_executor/platform",
+    ]),
 )
 
 cc_library(
     name = "cuda_driver",
-    srcs = ["cuda_driver.cc"],
-    hdrs = ["cuda_driver.h"],
-    deps = [
+    srcs = if_cuda_is_configured(["cuda_driver.cc"]),
+    hdrs = if_cuda_is_configured(["cuda_driver.h"]),
+    deps = if_cuda_is_configured([
         ":cuda_diagnostics",
         "@com_google_absl//absl/base",
         "@com_google_absl//absl/container:inlined_vector",
@@ -84,49 +84,49 @@ cc_library(
         "//tensorflow/stream_executor:device_options",
         "//tensorflow/stream_executor/lib",
         "//tensorflow/stream_executor/platform",
-    ] + tf_additional_cuda_driver_deps(),
+    ] + tf_additional_cuda_driver_deps()),
 )
 
 # The activation library is tightly coupled to the executor library.
 # TODO(leary) split up cuda_gpu_executor.cc so that this can stand alone.
 cc_library(
     name = "cuda_activation_header",
-    hdrs = ["cuda_activation.h"],
+    hdrs = if_cuda_is_configured(["cuda_activation.h"]),
     visibility = ["//visibility:public"],
-    deps = ["//tensorflow/stream_executor/platform"],
+    deps = if_cuda_is_configured(["//tensorflow/stream_executor/platform"]),
 )
 
 cc_library(
     name = "cuda_activation",
-    srcs = ["cuda_activation.cc"],
-    hdrs = ["cuda_activation.h"],
-    deps = [
+    srcs = if_cuda_is_configured(["cuda_activation.cc"]),
+    hdrs = if_cuda_is_configured(["cuda_activation.h"]),
+    deps = if_cuda_is_configured([
         ":cuda_driver",
+        "@local_config_cuda//cuda:cuda_headers",
         "//tensorflow/stream_executor",
         "//tensorflow/stream_executor:stream_executor_internal",
         "//tensorflow/stream_executor/platform",
-        "@local_config_cuda//cuda:cuda_headers",
-    ],
+    ]),
 )
 
 cc_library(
     name = "cuda_gpu_executor_header",
-    textual_hdrs = ["cuda_gpu_executor.h"],
+    textual_hdrs = if_cuda_is_configured(["cuda_gpu_executor.h"]),
     visibility = ["//visibility:public"],
-    deps = [
+    deps = if_cuda_is_configured([
         ":cuda_kernel",
         "//tensorflow/stream_executor:event",
         "//tensorflow/stream_executor/lib",
         "//tensorflow/stream_executor/platform",
-    ],
+    ]),
 )
 
 cc_library(
     name = "cublas_plugin",
-    srcs = ["cuda_blas.cc"],
-    hdrs = ["cuda_blas.h"],
+    srcs = if_cuda_is_configured(["cuda_blas.cc"]),
+    hdrs = if_cuda_is_configured(["cuda_blas.h"]),
     visibility = ["//visibility:public"],
-    deps = [
+    deps = if_cuda_is_configured([
         ":cuda_activation",
         ":cuda_gpu_executor",
         ":cuda_helpers",
@@ -146,16 +146,16 @@ cc_library(
         "//tensorflow/stream_executor/lib",
         "//tensorflow/stream_executor/platform",
         "//tensorflow/stream_executor/platform:dso_loader",
-    ] + if_static(["@local_config_cuda//cuda:cublas"]),
+    ] + if_static(["@local_config_cuda//cuda:cublas"])),
     alwayslink = True,
 )
 
 cc_library(
     name = "cufft_plugin",
-    srcs = ["cuda_fft.cc"],
-    hdrs = ["cuda_fft.h"],
+    srcs = if_cuda_is_configured(["cuda_fft.cc"]),
+    hdrs = if_cuda_is_configured(["cuda_fft.h"]),
     visibility = ["//visibility:public"],
-    deps = [
+    deps = if_cuda_is_configured([
         ":cuda_activation_header",
         ":cuda_gpu_executor_header",
         ":cuda_helpers",
@@ -169,21 +169,21 @@ cc_library(
         "//tensorflow/stream_executor/lib",
         "//tensorflow/stream_executor/platform",
         "//tensorflow/stream_executor/platform:dso_loader",
-    ] + if_static(["@local_config_cuda//cuda:cufft"]),
+    ] + if_static(["@local_config_cuda//cuda:cufft"])),
     alwayslink = True,
 )
 
 cc_library(
     name = "cudnn_plugin",
-    srcs = ["cuda_dnn.cc"],
-    hdrs = ["cuda_dnn.h"],
+    srcs = if_cuda_is_configured(["cuda_dnn.cc"]),
+    hdrs = if_cuda_is_configured(["cuda_dnn.h"]),
     copts = [
         # STREAM_EXECUTOR_CUDNN_WRAP would fail on Clang with the default
         # setting of template depth 256
         "-ftemplate-depth-512",
     ],
     visibility = ["//visibility:public"],
-    deps = [
+    deps = if_cuda_is_configured([
         ":cuda_activation",
         ":cuda_diagnostics",
         ":cuda_driver",
@@ -208,15 +208,15 @@ cc_library(
         "//tensorflow/stream_executor/lib",
         "//tensorflow/stream_executor/platform",
         "//tensorflow/stream_executor/platform:dso_loader",
-    ] + tf_additional_cudnn_plugin_deps() + if_static(["@local_config_cuda//cuda:cudnn"]),
+    ] + tf_additional_cudnn_plugin_deps() + if_static(["@local_config_cuda//cuda:cudnn"])),
     alwayslink = True,
 )
 
 cc_library(
     name = "curand_plugin",
-    srcs = ["cuda_rng.cc"],
-    hdrs = ["cuda_rng.h"],
-    deps = [
+    srcs = if_cuda_is_configured(["cuda_rng.cc"]),
+    hdrs = if_cuda_is_configured(["cuda_rng.h"]),
+    deps = if_cuda_is_configured([
         ":cuda_activation",
         ":cuda_gpu_executor",
         ":cuda_helpers",
@@ -229,75 +229,75 @@ cc_library(
         "//tensorflow/stream_executor/lib",
         "//tensorflow/stream_executor/platform",
         "//tensorflow/stream_executor/platform:dso_loader",
-    ] + if_static(["@local_config_cuda//cuda:curand"]),
+    ] + if_static(["@local_config_cuda//cuda:curand"])),
     alwayslink = True,
 )
 
 cc_library(
     name = "cuda_kernel",
-    hdrs = ["cuda_kernel.h"],
-    deps = [
+    hdrs = if_cuda_is_configured(["cuda_kernel.h"]),
+    deps = if_cuda_is_configured([
         ":cuda_driver",
+        "@local_config_cuda//cuda:cuda_headers",
         "//tensorflow/stream_executor:event",
         "//tensorflow/stream_executor:stream_executor_pimpl_header",
         "//tensorflow/stream_executor/lib",
         "//tensorflow/stream_executor/platform",
-        "@local_config_cuda//cuda:cuda_headers",
-    ],
+    ]),
 )
 
 # TODO(leary) we likely need to canonicalize/eliminate this.
 cc_library(
     name = "cuda_helpers",
-    textual_hdrs = ["cuda_helpers.h"],
+    textual_hdrs = if_cuda_is_configured(["cuda_helpers.h"]),
 )
 
 cc_library(
     name = "cuda_event",
-    srcs = ["cuda_event.cc"],
-    hdrs = ["cuda_event.h"],
-    deps = [
+    srcs = if_cuda_is_configured(["cuda_event.cc"]),
+    hdrs = if_cuda_is_configured(["cuda_event.h"]),
+    deps = if_cuda_is_configured([
         ":cuda_driver",
         ":cuda_gpu_executor_header",
         ":cuda_stream",
         "//tensorflow/stream_executor:stream_executor_headers",
         "//tensorflow/stream_executor/lib",
-    ],
+    ]),
 )
 
 cc_library(
     name = "cuda_stream",
-    srcs = ["cuda_stream.cc"],
-    hdrs = ["cuda_stream.h"],
-    deps = [
+    srcs = if_cuda_is_configured(["cuda_stream.cc"]),
+    hdrs = if_cuda_is_configured(["cuda_stream.h"]),
+    deps = if_cuda_is_configured([
         ":cuda_driver",
         ":cuda_gpu_executor_header",
         "//tensorflow/stream_executor:stream_executor_headers",
         "//tensorflow/stream_executor:stream_header",
         "//tensorflow/stream_executor/lib",
         "//tensorflow/stream_executor/platform",
-    ],
+    ]),
 )
 
 cc_library(
     name = "cuda_timer",
-    srcs = ["cuda_timer.cc"],
-    hdrs = ["cuda_timer.h"],
-    deps = [
+    srcs = if_cuda_is_configured(["cuda_timer.cc"]),
+    hdrs = if_cuda_is_configured(["cuda_timer.h"]),
+    deps = if_cuda_is_configured([
         ":cuda_driver",
         ":cuda_gpu_executor_header",
         ":cuda_stream",
         "//tensorflow/stream_executor:stream_executor_headers",
         "//tensorflow/stream_executor/lib",
-    ],
+    ]),
 )
 
 # It implements :cuda_gpu_executor_header
 cc_library(
     name = "cuda_gpu_executor",
-    srcs = ["cuda_gpu_executor.cc"],
-    hdrs = ["cuda_gpu_executor.h"],
-    deps = [
+    srcs = if_cuda_is_configured(["cuda_gpu_executor.cc"]),
+    hdrs = if_cuda_is_configured(["cuda_gpu_executor.h"]),
+    deps = if_cuda_is_configured([
         ":cuda_activation",
         ":cuda_diagnostics",
         ":cuda_driver",
@@ -306,6 +306,7 @@ cc_library(
         ":cuda_platform_id",
         ":cuda_stream",
         ":cuda_timer",
+        "@com_google_absl//absl/strings",
         "//tensorflow/stream_executor:event",
         "//tensorflow/stream_executor:plugin_registry",
         "//tensorflow/stream_executor:stream_executor_internal",
@@ -313,8 +314,7 @@ cc_library(
         "//tensorflow/stream_executor:timer",
         "//tensorflow/stream_executor/lib",
         "//tensorflow/stream_executor/platform",
-        "@com_google_absl//absl/strings",
-    ],
+    ]),
     alwayslink = True,
 )
 
@@ -341,13 +341,13 @@ cc_library(
     name = "all_runtime",
     copts = tf_copts(),
     visibility = ["//visibility:public"],
-    deps = if_cuda_is_configured([
-        ":cudnn_plugin",
-        ":cufft_plugin",
+    deps = [
         ":cublas_plugin",
-        ":curand_plugin",
         ":cuda_driver",
         ":cuda_platform",
-    ]),
+        ":cudnn_plugin",
+        ":cufft_plugin",
+        ":curand_plugin",
+    ],
     alwayslink = 1,
 )
-- 
GitLab


From 742bd415306658a329be3f6581eab74b68c74601 Mon Sep 17 00:00:00 2001
From: Albin Joy <albin.joy@huawei.com>
Date: Tue, 15 Jan 2019 10:09:28 +0530
Subject: [PATCH 0793/1095] Typo error correction context.py

---
 tensorflow/python/eager/context.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/eager/context.py b/tensorflow/python/eager/context.py
index faaf742c26..6c5b191cf9 100644
--- a/tensorflow/python/eager/context.py
+++ b/tensorflow/python/eager/context.py
@@ -180,8 +180,8 @@ class _ContextSwitchStack(threading.local):
   def push(self, is_building_function, enter_context_fn):
     """Push metadata about a context switch onto the stack.
 
-    A context switch can take one of two forms: installing a graph as the
-    default graph, or entering the eager context. For each context switch,
+    A context switch can take any one of the two forms: installing a graph as
+    the default graph, or entering the eager context. For each context switch,
     we record whether or not the entered context is building a function.
 
     Args:
-- 
GitLab


From 0bc02af7db9e43c37a89278a2e7c99ecdfe4fb00 Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Tue, 15 Jan 2019 23:12:00 -0800
Subject: [PATCH 0794/1095] Fix demo app to handle some cases more properly

- Don't crash when threads are changes with a delegate. This is done by now recreating interpreter every time threads is changes.
- Don't call showToast() every updateFrame() when classifier is null. This prevents hanging and allows the proper error message to be displayed.

PiperOrigin-RevId: 229503428
---
 .../Camera2BasicFragment.java                 | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java b/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java
index a7b3440536..814d236872 100644
--- a/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java
+++ b/tensorflow/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java
@@ -190,6 +190,8 @@ public class Camera2BasicFragment extends Fragment
 
   int currentModel = -1;
 
+  int currentNumThreads = -1;
+
   /** An additional thread for running tasks that shouldn't block the UI. */
   private HandlerThread backgroundThread;
 
@@ -323,13 +325,16 @@ public class Camera2BasicFragment extends Fragment
     // Get UI information before delegating to background
     final int modelIndex = modelView.getCheckedItemPosition();
     final int deviceIndex = deviceView.getCheckedItemPosition();
+    final int numThreads = np.getValue();
 
     backgroundHandler.post(() -> {
-      if (modelIndex == currentModel && deviceIndex == currentDevice) {
+      if (modelIndex == currentModel && deviceIndex == currentDevice
+              && numThreads == currentNumThreads) {
         return;
       }
       currentModel = modelIndex;
       currentDevice = deviceIndex;
+      currentNumThreads = numThreads;
 
       // Disable classifier while updating
       if (classifier != null) {
@@ -357,7 +362,11 @@ public class Camera2BasicFragment extends Fragment
         classifier = null;
       }
 
-      // Customzie the interpreter to the type of device we want to use.
+      // Customize the interpreter to the type of device we want to use.
+      if (classifier == null) {
+        return;
+      }
+      classifier.setNumThreads(numThreads);
       if (device.equals(cpu)) {
       } else if (device.equals(gpu)) {
         if (!GpuDelegateHelper.isGpuDelegateAvailable()) {
@@ -437,7 +446,7 @@ public class Camera2BasicFragment extends Fragment
         new NumberPicker.OnValueChangeListener() {
           @Override
           public void onValueChange(NumberPicker picker, int oldVal, int newVal) {
-            backgroundHandler.post(() -> classifier.setNumThreads(newVal));
+            updateActiveModel();
           }
         });
 
@@ -807,7 +816,9 @@ public class Camera2BasicFragment extends Fragment
   /** Classifies a frame from the preview stream. */
   private void classifyFrame() {
     if (classifier == null || getActivity() == null || cameraDevice == null) {
-      showToast("Uninitialized Classifier or invalid context.");
+      // It's important to not call showToast every frame, or else the app will starve and
+      // hang. updateActiveModel() already puts a error message up with showToast.
+      // showToast("Uninitialized Classifier or invalid context.");
       return;
     }
     SpannableStringBuilder textToShow = new SpannableStringBuilder();
-- 
GitLab


From 9fd5c9930dc714c089ed137d57e8ba09e0e1a6af Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 16 Jan 2019 01:03:08 -0800
Subject: [PATCH 0795/1095] compat: Update forward compatibility horizon to
 2019-01-16

PiperOrigin-RevId: 229513368
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 83f31bd533..773b3a41f9 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 15)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 16)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From a331eaf593bb91eaafd8953f1bd7d502eaeee53a Mon Sep 17 00:00:00 2001
From: Tom Hennigan <tomhennigan@google.com>
Date: Wed, 16 Jan 2019 02:40:41 -0800
Subject: [PATCH 0796/1095] Move variable ownership test into
 extended.variable_created_in_scope.

We want to invert the relationship here, instead of tf.Variable being aware of
strategies, we make strategies keep track of variables created while their
scope was active. This change implements that (at least in terms of the public
API for variables and dist strat).

tf.Variable subclasses created by distribution strategy (e.g.
DistributedVariable) still export a public `distribution_strategy` property.

PiperOrigin-RevId: 229525171
---
 .../python/parameter_server_strategy_test.py  |  5 +++-
 .../python/distribute/distribute_lib.py       | 27 +++++++++++++++++++
 tensorflow/python/distribute/values.py        |  9 ++++---
 tensorflow/python/keras/engine/training.py    |  5 ++--
 .../resource_variable_ops_test.py             |  2 +-
 .../python/kernel_tests/variables_test.py     |  2 +-
 .../python/ops/resource_variable_ops.py       |  9 ++-----
 tensorflow/python/ops/variables.py            | 14 ++--------
 .../api/golden/v1/tensorflow.-variable.pbtxt  |  4 ---
 ...orflow.distribute.-strategy-extended.pbtxt |  4 +++
 .../api/golden/v2/tensorflow.-variable.pbtxt  |  4 ---
 ...orflow.distribute.-strategy-extended.pbtxt |  4 +++
 12 files changed, 53 insertions(+), 36 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
index c7760a8431..9e7e201519 100644
--- a/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
+++ b/tensorflow/contrib/distribute/python/parameter_server_strategy_test.py
@@ -873,7 +873,10 @@ class ParameterServerStrategyWithChiefTest(ParameterServerStrategyTestBase,
                              id(get_step), get_step.__class__.__name__)))
       self.assertIs(resource_variable_ops.ResourceVariable, type(created_step))
       self.assertIs(resource_variable_ops.ResourceVariable, type(get_step))
-      self.assertIs(strategy, created_step.distribute_strategy)
+      # All variables have an _distribute_strategy parameter. Only variable
+      # subclasses in distribution strategy expose it publicly.
+      self.assertFalse(hasattr(strategy, 'distribute_strategy'))
+      self.assertIs(strategy, created_step._distribute_strategy)
 
   @combinations.generate(
       combinations.combine(mode=['graph'], use_core_strategy=[True, False]))
diff --git a/tensorflow/python/distribute/distribute_lib.py b/tensorflow/python/distribute/distribute_lib.py
index f112787ba2..32789c821e 100644
--- a/tensorflow/python/distribute/distribute_lib.py
+++ b/tensorflow/python/distribute/distribute_lib.py
@@ -876,6 +876,30 @@ class DistributionStrategyExtended(object):
     # Note: should support "colocate_with" argument.
     raise NotImplementedError("must be implemented in descendants")
 
+  def variable_created_in_scope(self, v):
+    """Tests whether `v` was created while this strategy scope was active.
+
+    Variables created inside the strategy scope are "owned" by it:
+
+    >>> with strategy.scope():
+    ...   v = tf.Variable(1.)
+    >>> strategy.variable_created_in_scope(v)
+    True
+
+    Variables created outside the strategy are not owned by it:
+
+    >>> v = tf.Variable(1.)
+    >>> strategy.variable_created_in_scope(v)
+    False
+
+    Args:
+      v: A `tf.Variable` instance.
+
+    Returns:
+      True if `v` was created inside the scope, False if not.
+    """
+    return v._distribute_strategy == self._container_strategy_weakref()  # pylint: disable=protected-access
+
   def read_var(self, v):
     """Reads the value of a variable.
 
@@ -1519,6 +1543,9 @@ class _DefaultDistributionExtended(DistributionStrategyExtended):
     _require_strategy_scope_extended(self)
     return ops.colocate_with(colocate_with_variable)
 
+  def variable_created_in_scope(self, v):
+    return v._distribute_strategy is None  # pylint: disable=protected-access
+
   def _distribute_dataset(self, dataset_fn):
     return self._call_dataset_fn(dataset_fn)
 
diff --git a/tensorflow/python/distribute/values.py b/tensorflow/python/distribute/values.py
index e7db285be5..19967f2c5e 100644
--- a/tensorflow/python/distribute/values.py
+++ b/tensorflow/python/distribute/values.py
@@ -571,11 +571,12 @@ ops.register_dense_tensor_like_type(DistributedVariable)
 
 
 def _validate_colocate_extended(v, extended):
-  if v.distribute_strategy.extended is not extended:
+  variable_strategy = v._distribute_strategy  # pylint: disable=protected-access
+  if variable_strategy.extended is not extended:
     raise ValueError(
         "`colocate_vars_with` must only be passed a variable created in this "
         "tf.distribute.Strategy.scope(), not %s created in scope: %s" %
-        (v, v.distribute_strategy,))
+        (v, variable_strategy))
 
 
 def validate_colocate_distributed_variable(v, extended):
@@ -595,7 +596,7 @@ def validate_colocate_tpu_variable(v, extended):
 
 
 def validate_colocate(v, extended):
-  if not hasattr(v, "distribute_strategy"):
+  if not hasattr(v, "_distribute_strategy"):
     raise ValueError(
         "`colocate_vars_with` must only be passed a variable created in this "
         "tf.distribute.Strategy.scope(), not: %r" % (v,))
@@ -1174,7 +1175,7 @@ class _ReplicaLocalSaveable(saver.BaseSaverBuilder.SaveableObject):
     # We use a callable so that we don't have to evaluate this expression
     # in the case where we are trying to restore instead of save.
     def tensor():
-      strategy = replica_local_variable.distribute_strategy
+      strategy = replica_local_variable._distribute_strategy  # pylint: disable=protected-access
       return strategy.extended.read_var(replica_local_variable)
 
     spec = saver.BaseSaverBuilder.SaveSpec(
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index f5a720b292..6b0422e71c 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -547,7 +547,8 @@ class Model(Network):
       # Validate all variables were correctly created in distribution scope.
       if self._distribution_strategy and not self._compile_distribution:
         for v in self.variables:
-          if v.distribute_strategy is not self._distribution_strategy:
+          strategy = self._distribution_strategy
+          if not strategy.extended.variable_created_in_scope(v):
             raise ValueError(
                 'Variable (%s) was not created in the distribution strategy '
                 'scope of (%s). It is most likely due to not all layers or '
@@ -556,7 +557,7 @@ class Model(Network):
                 'to the following.\n'
                 'with strategy.scope():\n'
                 '  model=_create_model()\n'
-                '  model.compile(...)'% (v, self._distribution_strategy))
+                '  model.compile(...)'% (v, strategy))
 
   @property
   def metrics(self):
diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
index ec93038f1b..d31652eb89 100644
--- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py
+++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
@@ -1095,7 +1095,7 @@ class _MixedPrecisionVariableTest(test_util.TensorFlowTestCase):
   @test_util.run_in_graph_and_eager_modes()
   def testDistributeStrategy(self):
     v = resource_variable_ops.ResourceVariable(1, dtype=dtypes.int32)
-    self.assertIsNone(v.distribute_strategy)
+    self.assertIsNone(v._distribute_strategy)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/kernel_tests/variables_test.py b/tensorflow/python/kernel_tests/variables_test.py
index 4dc6d0a828..028ef11fc4 100644
--- a/tensorflow/python/kernel_tests/variables_test.py
+++ b/tensorflow/python/kernel_tests/variables_test.py
@@ -46,7 +46,7 @@ class VariablesTestCase(test.TestCase):
   @test_util.run_deprecated_v1
   def testDistributeStrategy(self):
     v = variables.VariableV1(0.0)
-    self.assertIsNone(v.distribute_strategy)
+    self.assertIsNone(v._distribute_strategy)
 
   @test_util.run_v1_only("b/120545219")
   def testInitialization(self):
diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index 2681deb78c..1a895fee87 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -593,7 +593,7 @@ class ResourceVariable(variables.VariableV1):
         constraint=self._constraint,
         dtype=self._dtype,
         name=self._shared_name + "_copy",
-        distribute_strategy=self.distribute_strategy)
+        distribute_strategy=self._distribute_strategy)
     memo[self._unique_id] = copied_variable
     return copied_variable
 
@@ -622,11 +622,6 @@ class ResourceVariable(variables.VariableV1):
     """The shape of this variable."""
     return self._shape
 
-  @property
-  def distribute_strategy(self):
-    """The `tf.distribute.Strategy` that this variable was created under."""
-    return self._distribute_strategy
-
   def _shape_as_list(self):
     if self.shape.ndims is None:
       return None
@@ -933,7 +928,7 @@ class ResourceVariable(variables.VariableV1):
         name=self._shared_name,
         dtype=self.dtype,
         constraint=self.constraint,
-        distribute_strategy=self.distribute_strategy), ()
+        distribute_strategy=self._distribute_strategy), ()
 
   def scatter_sub(self, sparse_delta, use_locking=False, name=None):
     """Subtracts `IndexedSlices` from this variable.
diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py
index 7eb2e14b42..3392b1aad9 100644
--- a/tensorflow/python/ops/variables.py
+++ b/tensorflow/python/ops/variables.py
@@ -1004,16 +1004,6 @@ class Variable(six.with_metaclass(VariableMetaclass,
     """The `Graph` of this variable."""
     raise NotImplementedError
 
-  @property
-  def distribute_strategy(self):
-    """The `tf.distribute.Strategy` that this variable was created under.
-
-    Returns:
-      A `tf.distribute.Strategy` or `None` if this variable was not created
-      inside the `scope()` of any strategy.
-    """
-    raise NotImplementedError
-
   @property
   def shape(self):
     """The `TensorShape` of this variable.
@@ -2206,7 +2196,7 @@ class RefVariable(VariableV1):
     return self._variable.graph
 
   @property
-  def distribute_strategy(self):
+  def _distribute_strategy(self):
     """The `tf.distribute.Strategy` that this variable was created under."""
     return None   # Ref variables are never created inside a strategy.
 
@@ -2603,7 +2593,7 @@ class PartitionedVariable(object):
     return self.get_shape()
 
   @property
-  def distribute_strategy(self):
+  def _distribute_strategy(self):
     """The `tf.distribute.Strategy` that this variable was created under."""
     # NOTE(yuefengz): Today, no partitioned variables in a distribute strategy.
     return None
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.-variable.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.-variable.pbtxt
index f042097acc..341ace0766 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.-variable.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.-variable.pbtxt
@@ -16,10 +16,6 @@ tf_class {
     name: "device"
     mtype: "<type \'property\'>"
   }
-  member {
-    name: "distribute_strategy"
-    mtype: "<type \'property\'>"
-  }
   member {
     name: "dtype"
     mtype: "<type \'property\'>"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy-extended.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy-extended.pbtxt
index 37b620891f..5c4f090753 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy-extended.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy-extended.pbtxt
@@ -82,4 +82,8 @@ tf_class {
     name: "value_container"
     argspec: "args=[\'self\', \'value\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "variable_created_in_scope"
+    argspec: "args=[\'self\', \'v\'], varargs=None, keywords=None, defaults=None"
+  }
 }
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-variable.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-variable.pbtxt
index 2206f0f8d8..a80726d3bb 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.-variable.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.-variable.pbtxt
@@ -15,10 +15,6 @@ tf_class {
     name: "device"
     mtype: "<type \'property\'>"
   }
-  member {
-    name: "distribute_strategy"
-    mtype: "<type \'property\'>"
-  }
   member {
     name: "dtype"
     mtype: "<type \'property\'>"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy-extended.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy-extended.pbtxt
index 37b620891f..5c4f090753 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy-extended.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy-extended.pbtxt
@@ -82,4 +82,8 @@ tf_class {
     name: "value_container"
     argspec: "args=[\'self\', \'value\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "variable_created_in_scope"
+    argspec: "args=[\'self\', \'v\'], varargs=None, keywords=None, defaults=None"
+  }
 }
-- 
GitLab


From 2225d4d16dcec29f5c4eeda4485618bc3ac4de99 Mon Sep 17 00:00:00 2001
From: Vojtech Bardiovsky <vbardiovsky@google.com>
Date: Wed, 16 Jan 2019 06:13:58 -0800
Subject: [PATCH 0797/1095] Return the same list of dataset optimizations every
 time to fix non-determinism.

PiperOrigin-RevId: 229545908
---
 tensorflow/python/data/experimental/ops/optimization_options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/data/experimental/ops/optimization_options.py b/tensorflow/python/data/experimental/ops/optimization_options.py
index 9fe1f9499e..02747d3e36 100644
--- a/tensorflow/python/data/experimental/ops/optimization_options.py
+++ b/tensorflow/python/data/experimental/ops/optimization_options.py
@@ -132,4 +132,4 @@ class OptimizationOptions(options.OptionsBase):
       for optimization in optimizations_to_disable:
         if getattr(self, optimization) is not False:
           result.add(optimization)
-    return list(result)
+    return sorted(list(result))
-- 
GitLab


From 2834c69d95a3034c587f3cee80282a47055fa750 Mon Sep 17 00:00:00 2001
From: Dheeraj Rajaram Reddy <dheeraj98reddy@gmail.com>
Date: Wed, 16 Jan 2019 20:53:35 +0530
Subject: [PATCH 0798/1095] Add take_while to v1 golden API

---
 .../tools/api/golden/v1/tensorflow.data.experimental.pbtxt    | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt
index a262c0f799..abc98a74b6 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.pbtxt
@@ -180,6 +180,10 @@ tf_module {
     name: "shuffle_and_repeat"
     argspec: "args=[\'buffer_size\', \'count\', \'seed\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
+  member_method {
+    name: "take_while"
+    argspec: "args=[\'predicate\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "unbatch"
     argspec: "args=[], varargs=None, keywords=None, defaults=None"
-- 
GitLab


From 625516717dc584267eaff196a3c9a62f0e18658f Mon Sep 17 00:00:00 2001
From: Dheeraj Rajaram Reddy <dheeraj98reddy@gmail.com>
Date: Wed, 16 Jan 2019 21:03:53 +0530
Subject: [PATCH 0799/1095] Add absl_py parameterized testing dependency to
 take_while_serialization test

---
 .../python/data/experimental/kernel_tests/serialization/BUILD    | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD b/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD
index 00b05e4d30..16c8aced6b 100644
--- a/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD
+++ b/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD
@@ -681,6 +681,7 @@ py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python/data/experimental/ops:take_while_ops",
         "//tensorflow/python/data/ops:dataset_ops",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
-- 
GitLab


From 1f0e45d698e07834c2e316843403f0742a206d8e Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Wed, 16 Jan 2019 07:42:12 -0800
Subject: [PATCH 0800/1095] [tf.data] Turning map paralellization off by
 default until an performance regression is resolved.

PiperOrigin-RevId: 229555646
---
 .../kernel_tests/optimization/optimize_dataset_test.py           | 1 -
 tensorflow/python/data/experimental/ops/optimization_options.py  | 1 -
 2 files changed, 2 deletions(-)

diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
index a08016ca89..e02f3b1572 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
@@ -268,7 +268,6 @@ class OptimizeDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     options = dataset_ops.Options()
     expected_optimizations = [
         "map_and_batch_fusion",
-        "map_parallelization",
         "noop_elimination",
         "shuffle_and_repeat_fusion",
     ]
diff --git a/tensorflow/python/data/experimental/ops/optimization_options.py b/tensorflow/python/data/experimental/ops/optimization_options.py
index 02747d3e36..be1fb4c7ca 100644
--- a/tensorflow/python/data/experimental/ops/optimization_options.py
+++ b/tensorflow/python/data/experimental/ops/optimization_options.py
@@ -125,7 +125,6 @@ class OptimizationOptions(options.OptionsBase):
       # user explicitly disables them.
       optimizations_to_disable = [
           "map_and_batch_fusion",
-          "map_parallelization",
           "noop_elimination",
           "shuffle_and_repeat_fusion",
       ]
-- 
GitLab


From 060b6e32adbcee5504377f5bd54a2e4044d78273 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 16 Jan 2019 07:51:23 -0800
Subject: [PATCH 0801/1095] Use Ophint to support unidirectional sequence rnn
 (static)

PiperOrigin-RevId: 229556771
---
 .../lite/experimental/examples/lstm/BUILD     |  35 ++++
 .../experimental/examples/lstm/tflite_rnn.py  | 150 ++++++++++++++
 .../lstm/unidirectional_sequence_rnn_test.py  | 195 ++++++++++++++++++
 .../propagate_array_data_types.cc             |   6 +
 .../propagate_fixed_sizes.cc                  |  44 ++++
 tensorflow/lite/toco/import_tensorflow.cc     |  22 ++
 tensorflow/lite/toco/model.h                  |  10 +-
 tensorflow/lite/toco/tflite/operator.cc       |  36 ++++
 tensorflow/lite/toco/tooling_util.cc          |   1 +
 9 files changed, 498 insertions(+), 1 deletion(-)
 create mode 100644 tensorflow/lite/experimental/examples/lstm/tflite_rnn.py
 create mode 100644 tensorflow/lite/experimental/examples/lstm/unidirectional_sequence_rnn_test.py

diff --git a/tensorflow/lite/experimental/examples/lstm/BUILD b/tensorflow/lite/experimental/examples/lstm/BUILD
index 0c351ee4ec..f39673c028 100644
--- a/tensorflow/lite/experimental/examples/lstm/BUILD
+++ b/tensorflow/lite/experimental/examples/lstm/BUILD
@@ -17,6 +17,19 @@ py_library(
     ],
 )
 
+py_library(
+    name = "tflite_rnn",
+    srcs = ["tflite_rnn.py"],
+    srcs_version = "PY2AND3",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//tensorflow:tensorflow_py",
+        "//tensorflow/lite/python:lite",
+        "//tensorflow/python:framework",
+        "@six_archive//:six",
+    ],
+)
+
 py_test(
     name = "unidirectional_sequence_lstm_test",
     size = "large",
@@ -38,3 +51,25 @@ py_test(
         "@six_archive//:six",
     ],
 )
+
+py_test(
+    name = "unidirectional_sequence_rnn_test",
+    size = "large",
+    srcs = ["unidirectional_sequence_rnn_test.py"],
+    srcs_version = "PY2AND3",
+    tags = [
+        "no_oss",
+        "no_pip",
+    ],
+    deps = [
+        ":tflite_rnn",
+        "//tensorflow:tensorflow_py",
+        "//tensorflow/examples/tutorials/mnist:input_data",
+        "//tensorflow/lite/python:lite",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:platform",
+        "//tensorflow/python/tools:optimize_for_inference",
+        "//third_party/py/numpy",
+        "@six_archive//:six",
+    ],
+)
diff --git a/tensorflow/lite/experimental/examples/lstm/tflite_rnn.py b/tensorflow/lite/experimental/examples/lstm/tflite_rnn.py
new file mode 100644
index 0000000000..e4aad18367
--- /dev/null
+++ b/tensorflow/lite/experimental/examples/lstm/tflite_rnn.py
@@ -0,0 +1,150 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""TfLite BasicRnnCell wrapper.
+
+TODO(renjieliu): Find a better home for this one.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+import itertools
+
+from tensorflow.lite.python import lite
+from tensorflow.python.keras import activations
+from tensorflow.python.layers import base as base_layer
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import init_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import nn_ops
+from tensorflow.python.ops import rnn_cell_impl
+
+
+class TfLiteRNNCell(rnn_cell_impl.LayerRNNCell):
+  """The most basic RNN cell.
+
+  This is used only for TfLite, it provides hints and it also makes the
+  variables in the desired for the tflite ops.
+  """
+
+  def __init__(self,
+               num_units,
+               activation=None,
+               reuse=None,
+               name=None,
+               dtype=None,
+               **kwargs):
+    """Initializes the parameters for an RNN cell.
+
+    Args:
+      num_units: int, The number of units in the RNN cell.
+      activation: Nonlinearity to use.  Default: `tanh`. It could also be string
+        that is within Keras activation function names.
+      reuse: (optional) Python boolean describing whether to reuse variables in
+        an existing scope. Raises an error if not `True` and the existing scope
+        already has the given variables.
+      name: String, the name of the layer. Layers with the same name will share
+        weights, but to avoid mistakes we require reuse=True in such cases.
+      dtype: Default dtype of the layer (default of `None` means use the type of
+        the first input). Required when `build` is called before `call`.
+      **kwargs: Dict, keyword named properties for common layer attributes, like
+        `trainable` etc when constructing the cell from configs of get_config().
+
+    Raises:
+      ValueError: If the existing scope already has the given variables.
+    """
+    super(TfLiteRNNCell, self).__init__(
+        _reuse=reuse, name=name, dtype=dtype, **kwargs)
+
+    # Inputs must be Rank-2.
+    self.input_spec = base_layer.InputSpec(ndim=2)
+
+    self._tflite_wrapper = lite.OpHint("UnidirectionalSequenceRnn")
+    self._num_units = num_units
+    if activation:
+      self._activation = activations.get(activation)
+    else:
+      self._activation = math_ops.tanh
+
+  @property
+  def state_size(self):
+    return self._num_units
+
+  @property
+  def output_size(self):
+    return self._num_units
+
+  def build(self, inputs_shape):
+    """Builds the RNN cell.
+
+    Args:
+      inputs_shape: Rnn input tensor shape.
+
+    Raises:
+      ValueError: If last dimension of the input shape is not known.
+    """
+    if inputs_shape[-1] is None:
+      raise ValueError("Expected inputs.shape[-1] to be known, saw shape: %s" %
+                       (inputs_shape,))
+
+    input_depth = inputs_shape[-1]
+
+    def add_variable_wrapped(name, shape, initializer, index):
+      var = self.add_variable(name, shape=shape, initializer=initializer)
+      return self._tflite_wrapper.add_input(
+          var, name=name, index_override=index)
+
+    self._input_weights = add_variable_wrapped(
+        "input_weights", [self._num_units, input_depth], None, 1)
+    self._recurrent_weights = add_variable_wrapped(
+        "recurrent_weights", [self._num_units, self._num_units], None, 2)
+    self._bias = add_variable_wrapped(
+        "bias",
+        shape=[self._num_units],
+        initializer=init_ops.zeros_initializer(dtype=self.dtype),
+        index=3)
+
+    self.built = True
+
+  def call(self, inputs, state):
+    """Most basic RNN: output = new_state = act(W * input + U * state + B)."""
+    inputs = self._tflite_wrapper.add_input(
+        inputs, tag="input", name="input", aggregate="stack", index_override=0)
+    state = self._tflite_wrapper.add_input(
+        state,
+        tag="hidden_state",
+        name="hidden_state",
+        aggregate="first",
+        index_override=4)
+    weights = array_ops.transpose(
+        array_ops.concat([self._input_weights, self._recurrent_weights], 1))
+    gate_inputs = math_ops.matmul(array_ops.concat([inputs, state], 1), weights)
+    gate_inputs = nn_ops.bias_add(gate_inputs, self._bias)
+    output = self._activation(gate_inputs)
+    output = self._tflite_wrapper.add_output(
+        output,
+        tag="output",
+        name="output",
+        index_override=1,
+        aggregate="stack")
+    return output, output
+
+  def get_config(self):
+    config = {
+        "num_units": self._num_units,
+        "activation": activations.serialize(self._activation),
+        "reuse": self._reuse,
+    }
+    base_config = super(TfLiteRNNCell, self).get_config()
+    return dict(itertools.chain(base_config.items(), config.items()))
diff --git a/tensorflow/lite/experimental/examples/lstm/unidirectional_sequence_rnn_test.py b/tensorflow/lite/experimental/examples/lstm/unidirectional_sequence_rnn_test.py
new file mode 100644
index 0000000000..6f9e2dd949
--- /dev/null
+++ b/tensorflow/lite/experimental/examples/lstm/unidirectional_sequence_rnn_test.py
@@ -0,0 +1,195 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+import tempfile
+import numpy as np
+import tensorflow as tf
+
+from tensorflow import flags
+
+from tensorflow.examples.tutorials.mnist import input_data
+from tensorflow.lite.experimental.examples.lstm.tflite_rnn import TfLiteRNNCell
+from tensorflow.lite.python.op_hint import convert_op_hints_to_stubs
+from tensorflow.python.framework import test_util
+from tensorflow.python.platform import test
+from tensorflow.python.tools import optimize_for_inference_lib
+
+FLAGS = flags.FLAGS
+
+# Number of steps to train model.
+TRAIN_STEPS = 1
+
+CONFIG = tf.ConfigProto(device_count={"GPU": 0})
+
+
+class UnidirectionalSequenceRnnTest(test_util.TensorFlowTestCase):
+
+  def __init__(self, *args, **kwargs):
+    super(UnidirectionalSequenceRnnTest, self).__init__(*args, **kwargs)
+    # Define constants
+    # Unrolled through 28 time steps
+    self.time_steps = 28
+    # Rows of 28 pixels
+    self.n_input = 28
+    # Learning rate for Adam optimizer
+    self.learning_rate = 0.001
+    # MNIST is meant to be classified in 10 classes(0-9).
+    self.n_classes = 10
+    # Batch size
+    self.batch_size = 16
+    # Rnn Units.
+    self.num_units = 16
+
+  def setUp(self):
+    super(UnidirectionalSequenceRnnTest, self).setUp()
+    # Import MNIST dataset
+    data_dir = tempfile.mkdtemp(dir=FLAGS.test_tmpdir)
+    self.mnist = input_data.read_data_sets(data_dir, one_hot=True)
+
+  def buildRnnLayer(self):
+    return tf.nn.rnn_cell.MultiRNNCell([
+        TfLiteRNNCell(self.num_units, name="rnn1"),
+        TfLiteRNNCell(self.num_units, name="rnn2")
+    ])
+
+  def buildModel(self, rnn_layer):
+    # Weights and biases for output softmax layer.
+    out_weights = tf.Variable(
+        tf.random_normal([self.num_units, self.n_classes]))
+    out_bias = tf.Variable(tf.random_normal([self.n_classes]))
+
+    # input image placeholder
+    x = tf.placeholder(
+        "float", [None, self.time_steps, self.n_input], name="INPUT_IMAGE")
+
+    # x is shaped [batch_size,time_steps,num_inputs]
+    rnn_input = tf.unstack(x, self.time_steps, 1)
+    outputs, _ = tf.nn.static_rnn(rnn_layer, rnn_input, dtype="float32")
+
+    # Compute logits by multiplying outputs[-1] of shape [batch_size,num_units]
+    # by the softmax layer's out_weight of shape [num_units,n_classes]
+    # plus out_bias
+    prediction = tf.matmul(outputs[-1], out_weights) + out_bias
+    output_class = tf.nn.softmax(prediction, name="OUTPUT_CLASS")
+
+    return x, prediction, output_class
+
+  def trainModel(self, x, prediction, output_class, sess):
+    # input label placeholder
+    y = tf.placeholder("float", [None, self.n_classes])
+    # Loss function
+    loss = tf.reduce_mean(
+        tf.nn.softmax_cross_entropy_with_logits(logits=prediction, labels=y))
+    # Optimization
+    opt = tf.train.AdamOptimizer(
+        learning_rate=self.learning_rate).minimize(loss)
+
+    # Initialize variables
+    sess.run(tf.global_variables_initializer())
+    for _ in range(TRAIN_STEPS):
+      batch_x, batch_y = self.mnist.train.next_batch(
+          batch_size=self.batch_size, shuffle=False)
+
+      batch_x = batch_x.reshape((self.batch_size, self.time_steps,
+                                 self.n_input))
+      sess.run(opt, feed_dict={x: batch_x, y: batch_y})
+
+  def saveAndRestoreModel(self, rnn_layer, sess, saver):
+    """Saves and restores the model to mimic the most common use case.
+
+    Args:
+      rnn_layer: The rnn layer either a single rnn cell or a multi rnn cell.
+      sess: Old session.
+      saver: saver created by tf.train.Saver()
+
+    Returns:
+      A tuple containing:
+
+      - Input tensor of the restored model.
+      - Prediction tensor of the restored model.
+      - Output tensor, which is the softwmax result of the prediction tensor.
+      - new session of the restored model.
+
+    """
+    model_dir = tempfile.mkdtemp(dir=FLAGS.test_tmpdir)
+    saver.save(sess, model_dir)
+
+    # Reset the graph.
+    tf.reset_default_graph()
+    x, prediction, output_class = self.buildModel(rnn_layer)
+
+    new_sess = tf.Session(config=CONFIG)
+    saver = tf.train.Saver()
+    saver.restore(new_sess, model_dir)
+    return x, prediction, output_class, new_sess
+
+  def getInferenceResult(self, x, output_class, sess):
+    b1, _ = self.mnist.train.next_batch(batch_size=1)
+    sample_input = np.reshape(b1, (1, self.time_steps, self.n_input))
+
+    expected_output = sess.run(output_class, feed_dict={x: sample_input})
+    frozen_graph = tf.graph_util.convert_variables_to_constants(
+        sess, sess.graph_def, [output_class.op.name])
+    return sample_input, expected_output, frozen_graph
+
+  def tfliteInvoke(self, graph, test_inputs, outputs):
+    tf.reset_default_graph()
+    # Turn the input into placeholder of shape 1
+    tflite_input = tf.placeholder(
+        "float", [1, self.time_steps, self.n_input], name="INPUT_IMAGE_LITE")
+    tf.import_graph_def(graph, name="", input_map={"INPUT_IMAGE": tflite_input})
+    with tf.Session() as sess:
+      curr = sess.graph_def
+      curr = convert_op_hints_to_stubs(graph_def=curr)
+
+    curr = optimize_for_inference_lib.optimize_for_inference(
+        curr, ["INPUT_IMAGE_LITE"], ["OUTPUT_CLASS"],
+        [tf.float32.as_datatype_enum])
+
+    tflite = tf.lite.toco_convert(
+        curr, [tflite_input], [outputs], allow_custom_ops=False)
+    interpreter = tf.lite.Interpreter(model_content=tflite)
+    interpreter.allocate_tensors()
+
+    input_index = interpreter.get_input_details()[0]["index"]
+    interpreter.set_tensor(input_index, test_inputs)
+    interpreter.invoke()
+    output_index = interpreter.get_output_details()[0]["index"]
+    result = interpreter.get_tensor(output_index)
+    # Reset all variables so it will not pollute other inferences.
+    interpreter.reset_all_variables()
+    return result
+
+  def testStaticRnnMultiRnnCell(self):
+    sess = tf.Session(config=CONFIG)
+
+    x, prediction, output_class = self.buildModel(self.buildRnnLayer())
+    self.trainModel(x, prediction, output_class, sess)
+
+    saver = tf.train.Saver()
+    x, prediction, output_class, new_sess = self.saveAndRestoreModel(
+        self.buildRnnLayer(), sess, saver)
+
+    test_inputs, expected_output, frozen_graph = self.getInferenceResult(
+        x, output_class, new_sess)
+
+    result = self.tfliteInvoke(frozen_graph, test_inputs, output_class)
+    self.assertTrue(np.allclose(expected_output, result, rtol=1e-6, atol=1e-2))
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/lite/toco/graph_transformations/propagate_array_data_types.cc b/tensorflow/lite/toco/graph_transformations/propagate_array_data_types.cc
index 6d9aad66b6..7aec6728da 100644
--- a/tensorflow/lite/toco/graph_transformations/propagate_array_data_types.cc
+++ b/tensorflow/lite/toco/graph_transformations/propagate_array_data_types.cc
@@ -252,6 +252,12 @@ void SetDataTypeForAllOutputs(Model* model, Operator* op,
       SetDataTypeForAllOutputs(model, op, data_type);
       break;
     }
+    case OperatorType::kUnidirectionalSequenceRnn: {
+      const ArrayDataType data_type = model->GetArray(op->inputs[0]).data_type;
+      if (data_type != ArrayDataType::kFloat) return ::tensorflow::Status::OK();
+      SetDataTypeForAllOutputs(model, op, data_type);
+      break;
+    }
     case OperatorType::kUnique: {
       CHECK_EQ(op->outputs.size(), 2);
       const UniqueOperator* unique_op = static_cast<UniqueOperator*>(op);
diff --git a/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc b/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
index 5185afd22e..1b1780a73b 100644
--- a/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
+++ b/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
@@ -1109,6 +1109,46 @@ void ProcessUnidirectionalSequenceLstmOperator(
   output_shape->ReplaceDims({timestamp, batch_size, output_size});
 }
 
+void ProcessUnidirectionalSequenceRnnOperator(
+    Model* model, UnidirectionalSequenceRnnOperator* op) {
+  auto& output_array = model->GetArray(op->outputs[0]);
+  if (output_array.has_shape()) {
+    // Shape already propagated.
+    return;
+  }
+
+  if (output_array.data_type == ArrayDataType::kNone) {
+    // Yield until the output type has been set by PropagateArrayDataTypes
+    return;
+  }
+
+  // TODO(renjieliu): check the inputs, as well as all kinds of weights.
+  const auto& input_array = model->GetArray(op->inputs[0]);
+  // Yield until input dims have been resolved.
+  if (!input_array.has_shape()) {
+    return;
+  }
+  const auto& input_shape = input_array.shape();
+  const int batch_size = input_shape.dims(1);
+  const int timestamp = input_shape.dims(0);
+
+  const auto& bias_array = model->GetArray(op->inputs[3]);
+  // Yield until input dims have been resolved.
+  if (!bias_array.has_shape()) {
+    return;
+  }
+
+  constexpr int kHiddenStateTensor = 4;
+  // b(115961645): This is a hack to work around.
+  model->GetArray(op->inputs[kHiddenStateTensor]).buffer.reset();
+
+  const auto& bias_shape = bias_array.shape();
+  const int output_size = bias_shape.dims(0);
+
+  Shape* output_shape = output_array.mutable_shape();
+  output_shape->ReplaceDims({timestamp, batch_size, output_size});
+}
+
 void ProcessSpaceToBatchNDOperator(Model* model, SpaceToBatchNDOperator* op) {
   const auto& input_array = model->GetArray(op->inputs[0]);
   // Yield until input dims have been resolved.
@@ -2037,6 +2077,10 @@ void ProcessUniqueOperator(Model* model, UniqueOperator* op) {
       ProcessUnidirectionalSequenceLstmOperator(
           model, static_cast<UnidirectionalSequenceLstmOperator*>(op));
       break;
+    case OperatorType::kUnidirectionalSequenceRnn:
+      ProcessUnidirectionalSequenceRnnOperator(
+          model, static_cast<UnidirectionalSequenceRnnOperator*>(op));
+      break;
     case OperatorType::kLstmCell:
       ProcessLstmCellOperator(model, static_cast<LstmCellOperator*>(op));
       break;
diff --git a/tensorflow/lite/toco/import_tensorflow.cc b/tensorflow/lite/toco/import_tensorflow.cc
index 86e04b2393..dac106b398 100644
--- a/tensorflow/lite/toco/import_tensorflow.cc
+++ b/tensorflow/lite/toco/import_tensorflow.cc
@@ -2317,6 +2317,27 @@ tensorflow::Status ConvertLeakyReluOperator(
   return tensorflow::Status::OK();
 }
 
+tensorflow::Status ConvertUnidirectionalSequenceRnn(
+    const NodeDef& node, const TensorFlowImportFlags& tf_import_flags,
+    Model* model) {
+  DCHECK_EQ(node.op(), "UnidirectionalSequenceRnn");
+
+  auto* op = new UnidirectionalSequenceRnnOperator();
+  const auto& indices = GetListAttr(node, "_tflite_input_indices");
+  if (indices.i_size() != node.input().size()) {
+    return tensorflow::errors::InvalidArgument("Input size does not match.");
+  }
+
+  for (const string& input : node.input()) {
+    op->inputs.push_back(input);
+  }
+  // Only use the last one as input.
+  op->outputs.push_back(node.name() + ":1");
+  model->operators.emplace_back(op);
+
+  return tensorflow::Status::OK();
+}
+
 }  // namespace
 
 namespace internal {
@@ -2454,6 +2475,7 @@ ConverterMapType GetTensorFlowNodeConverterMap() {
       {"Unpack", ConvertUnpackOperator},
       {"ZerosLike", ConvertSimpleOperator<TensorFlowZerosLikeOperator, 1, 1>},
       {"UnidirectionalSequenceLstm", ConvertUnidirectionalSequenceLstm},
+      {"UnidirectionalSequenceRnn", ConvertUnidirectionalSequenceRnn},
       {"MirrorPad", ConvertMirrorPadOperator},
       {"Unique", ConvertSimpleOperator<UniqueOperator, 1, 2>},
   });
diff --git a/tensorflow/lite/toco/model.h b/tensorflow/lite/toco/model.h
index bfa86c8059..296ed9fc74 100644
--- a/tensorflow/lite/toco/model.h
+++ b/tensorflow/lite/toco/model.h
@@ -158,7 +158,8 @@ enum class OperatorType : uint8 {
   kLeakyRelu,
   kAbs,
   kMirrorPad,
-  kUnique
+  kUnique,
+  kUnidirectionalSequenceRnn
 };
 
 // Helper to deal with TensorFlow arrays using a different ordering of
@@ -1965,6 +1966,13 @@ struct UniqueOperator : Operator {
   ArrayDataType idx_out_type = ArrayDataType::kInt32;
 };
 
+struct UnidirectionalSequenceRnnOperator : Operator {
+  UnidirectionalSequenceRnnOperator()
+      : Operator(OperatorType::kUnidirectionalSequenceRnn) {}
+  bool time_major;
+  FusedActivationFunctionType fused_activation_function;
+};
+
 // Alloc's are used for transient arrays only. An Alloc specifies which interval
 // of the "transient_data" workspace buffer passed to inference functions, is to
 // be used for the transient array at hand. The 'start' and 'end' values are
diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index 8eb4d321ef..d9c051ebf7 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -1509,6 +1509,39 @@ class Unique : public BuiltinOperator<UniqueOperator, ::tflite::UniqueOptions,
   }
 };
 
+class UnidirectionalSequenceRnn
+    : public BuiltinOperator<UnidirectionalSequenceRnnOperator,
+                             ::tflite::SequenceRNNOptions,
+                             ::tflite::BuiltinOptions_SequenceRNNOptions> {
+ public:
+  using BuiltinOperator::BuiltinOperator;
+  flatbuffers::Offset<TfLiteOptions> WriteOptions(
+      const TocoOperator& op,
+      flatbuffers::FlatBufferBuilder* builder) const override {
+    return ::tflite::CreateSequenceRNNOptions(
+        *builder, /*time_major=*/true,
+        /*fused_activation_function=*/
+        ::tflite::ActivationFunctionType_TANH);
+  }
+  void ReadOptions(const TfLiteOptions& options,
+                   TocoOperator* op) const override {
+    // Only support tanh actication, so check that tflite type is tanh.
+    DCHECK(options.fused_activation_function() ==
+           ::tflite::ActivationFunctionType_TANH);
+  }
+
+  int GetVersion(const OperatorSignature& op_signature) const override {
+    return 1;
+  }
+
+  std::vector<bool> GetMutatingInputVariables(
+      const Operator& op) const override {
+    std::vector<bool> mutating_input_variables(op.inputs.size(), false);
+    mutating_input_variables[4] = true;
+    return mutating_input_variables;
+  }
+};
+
 std::unique_ptr<flexbuffers::Builder> WriteFlexOpOptions(
     const string& tensorflow_node_def) {
   auto fbb = absl::make_unique<flexbuffers::Builder>();
@@ -1847,6 +1880,9 @@ std::vector<std::unique_ptr<BaseOperator>> BuildOperatorList(
                                       OperatorType::kMirrorPad));
   ops.push_back(MakeUnique<Unique>(::tflite::BuiltinOperator_UNIQUE,
                                    OperatorType::kUnique));
+  ops.push_back(MakeUnique<UnidirectionalSequenceRnn>(
+      ::tflite::BuiltinOperator_UNIDIRECTIONAL_SEQUENCE_RNN,
+      OperatorType::kUnidirectionalSequenceRnn));
 
   // Custom Operators.
   ops.push_back(
diff --git a/tensorflow/lite/toco/tooling_util.cc b/tensorflow/lite/toco/tooling_util.cc
index 2396de1a3d..9c0e8067d8 100644
--- a/tensorflow/lite/toco/tooling_util.cc
+++ b/tensorflow/lite/toco/tooling_util.cc
@@ -417,6 +417,7 @@ const char* OperatorTypeName(OperatorType type) {
     HANDLE_OPERATORTYPENAME_CASE(SquaredDifference)
     HANDLE_OPERATORTYPENAME_CASE(MirrorPad)
     HANDLE_OPERATORTYPENAME_CASE(Unique)
+    HANDLE_OPERATORTYPENAME_CASE(UnidirectionalSequenceRnn)
     default:
       LOG(FATAL) << "Unhandled op type";
 #undef HANDLE_OPERATORTYPENAME_CASE
-- 
GitLab


From d122a13913a1d058cb872b9c443ecbfef8c8bc8d Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Wed, 16 Jan 2019 08:23:59 -0800
Subject: [PATCH 0802/1095] Fix for ResourceVariables and variants when using
 while_v2 + GradientTape.

PiperOrigin-RevId: 229561249
---
 tensorflow/python/kernel_tests/BUILD          |  1 +
 .../python/kernel_tests/while_v2_test.py      | 18 ++++++
 tensorflow/python/ops/while_v2.py             | 59 ++++++++++++++-----
 3 files changed, 64 insertions(+), 14 deletions(-)

diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index 7a514875d6..40bf4aedd0 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -3538,6 +3538,7 @@ cuda_py_test(
         "@absl_py//absl/testing:parameterized",
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python:array_ops",
+        "//tensorflow/python/eager:def_function",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:constant_op",
         "//tensorflow/python:control_flow_ops",
diff --git a/tensorflow/python/kernel_tests/while_v2_test.py b/tensorflow/python/kernel_tests/while_v2_test.py
index 1f2c6f94c5..061d787760 100644
--- a/tensorflow/python/kernel_tests/while_v2_test.py
+++ b/tensorflow/python/kernel_tests/while_v2_test.py
@@ -23,6 +23,8 @@ from absl.testing import parameterized
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.core.protobuf import rewriter_config_pb2
 from tensorflow.python.eager import backprop
+from tensorflow.python.eager import context
+from tensorflow.python.eager import def_function
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import meta_graph
@@ -34,6 +36,7 @@ from tensorflow.python.ops import functional_ops
 from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import list_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import variables
 from tensorflow.python.ops import while_v2
 from tensorflow.python.ops.control_flow_ops import while_loop as while_loop_v1
 from tensorflow.python.ops.while_v2 import while_loop as while_loop_v2
@@ -65,6 +68,21 @@ class WhileV2Test(test.TestCase, parameterized.TestCase):
       self.assertEqual(16., eval_result[0])
       self.assertSequenceEqual(sess.run(grad), [32.])
 
+  def testGradientTapeResourceVariable(self):
+    with context.eager_mode():
+      v = variables.Variable(1.)
+
+      @def_function.function
+      def fnWithLoop():  # pylint: disable=invalid-name
+        with backprop.GradientTape() as tape:
+          _, x = while_loop_v2(
+              lambda i, _: i < 2,
+              lambda i, x: (i + 1, x * v),
+              [0, 2.])
+        return tape.gradient(x, v)
+
+      self.assertAllEqual(fnWithLoop(), 4.0)
+
   @test_util.run_deprecated_v1
   def testMultipleLoopVarsBasic(self):
     x = constant_op.constant(5.)
diff --git a/tensorflow/python/ops/while_v2.py b/tensorflow/python/ops/while_v2.py
index 50011e34b1..0e427d3c6a 100644
--- a/tensorflow/python/ops/while_v2.py
+++ b/tensorflow/python/ops/while_v2.py
@@ -38,6 +38,7 @@ from tensorflow.python.ops import control_flow_util
 from tensorflow.python.ops import control_flow_util_v2 as util
 from tensorflow.python.ops import custom_gradient
 from tensorflow.python.ops import gen_functional_ops
+from tensorflow.python.ops import gen_resource_variable_ops
 from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import list_ops
 from tensorflow.python.ops import math_ops
@@ -263,20 +264,9 @@ def _WhileGrad(op, *grads):  # pylint: disable=invalid-name
   assert not _is_in_xla_context() or maximum_iterations is not None
   maximum_iterations = _validate_and_convert_to_tensor(maximum_iterations)
 
-  # Set the incoming gradient of non-trainable inputs to None. It is possible
-  # that we receive non-None gradients for non-trainable types in nested while
-  # loops because we accumulate outputs of the inner while as variant tensors
-  # which are trainable and hence receive zeros_like tensors in the gradient
-  # pass. The non-trainable tensors then receive the popped zeros tensor from
-  # this zeros variant. The gradient for the loop vars corresponding to these
-  # tensors is None or zeros (this happens only if the loop var is accumulated
-  # as well) in _grad_fn so we reset these.
-  # TODO(b/118712257): Remove the IsTrainable filter once we can handle None
-  # output grads in _grad_fn.
-  grads = [
-      None if not _is_trainable(output) else grad
-      for grad, output in zip(grads, body_graph.outputs)
-  ]
+  grads = [_preprocess_grad(grad, body_out, while_out)
+           for grad, body_out, while_out
+           in zip(grads, body_graph.outputs, while_op.outputs)]
 
   # We compute the gradient for the sub-graph between trainable ys and xs
   # with non-None incoming gradients. We later pad the None's to the list of
@@ -352,6 +342,47 @@ def _WhileGrad(op, *grads):  # pylint: disable=invalid-name
   return none_padded_outputs
 
 
+def _preprocess_grad(grad, body_graph_output, while_op_output):
+  """Returns the initial gradient to be used for a given output tensor.
+
+  Args:
+    grad: the original gradient Tensor passed to the gradient function.
+    body_graph_output: the corresponding Tensor in the body graph.
+    while_op_output: the corresponding Tensor output of the While op.
+
+  Returns:
+    A Tensor or None.
+  """
+  # Set the incoming gradient of non-trainable inputs to None. It is possible
+  # that we receive non-None gradients for non-trainable types in nested while
+  # loops because we accumulate outputs of the inner while as variant tensors
+  # which are trainable and hence receive zeros_like tensors in the gradient
+  # pass. The non-trainable tensors then receive the popped zeros tensor from
+  # this zeros variant. The gradient for the loop vars corresponding to these
+  # tensors is None or zeros (this happens only if the loop var is accumulated
+  # as well) in _grad_fn so we reset these.
+  # TODO(b/118712257): Remove once we can handle None output grads in _grad_fn.
+  if not _is_trainable(body_graph_output):
+    return None
+
+  # GradientTape initializes resource and variant grads as None instead of
+  # zeros. Set to zeros so _GradientsHelper computes the gradients instead of
+  # returning None.
+  if (while_op_output.dtype in (dtypes.resource, dtypes.variant)
+      and grad is None):
+    return _zeros_like(while_op_output)
+
+  return grad
+
+
+def _zeros_like(op_output):
+  """Like array_ops.zeros_like() but also accepts resource var handles."""
+  if op_output.dtype == dtypes.resource:
+    return array_ops.zeros(
+        gen_resource_variable_ops.variable_shape(op_output))
+  return array_ops.zeros_like(op_output)
+
+
 def _is_trainable(tensor):
   """Returns whether the given tensor is trainable."""
   if not gradients_impl.IsTrainable(tensor):
-- 
GitLab


From e74388ec6d1e8aeadd7aaa787544f319175dd2a6 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Wed, 16 Jan 2019 17:15:17 +0000
Subject: [PATCH 0803/1095] Fix softmax_cross_entropy issue

This fix is from 24397/24736 and all credit goes to @felix-schneider

This fix fixes 24397. This fix fixes 24736.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/ops/losses/losses_impl.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/losses/losses_impl.py b/tensorflow/python/ops/losses/losses_impl.py
index 7f88ccd879..3393e75335 100644
--- a/tensorflow/python/ops/losses/losses_impl.py
+++ b/tensorflow/python/ops/losses/losses_impl.py
@@ -794,7 +794,7 @@ def softmax_cross_entropy(
 
     if label_smoothing > 0:
       num_classes = math_ops.cast(
-          array_ops.shape(onehot_labels)[1], logits.dtype)
+          array_ops.shape(onehot_labels)[-1], logits.dtype)
       smooth_positives = 1.0 - label_smoothing
       smooth_negatives = label_smoothing / num_classes
       onehot_labels = onehot_labels * smooth_positives + smooth_negatives
-- 
GitLab


From 59e9acf78787ee2e46106672533aaff8032f69b0 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Wed, 16 Jan 2019 09:20:44 -0800
Subject: [PATCH 0804/1095] Bump the googletest dependency version for TF

PiperOrigin-RevId: 229569726
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 7eb1d8a3c6..354148f034 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -408,11 +408,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
 
     tf_http_archive(
         name = "com_google_googletest",
-        sha256 = "61eee610f136c1edc693d979647a4bb2ca253d60e6964724b61af85d32a41251",
-        strip_prefix = "googletest-6729a1361150131bc5d394d5cd2b4cdf0953ee7b",
+        sha256 = "ff7a82736e158c077e76188232eac77913a15dac0b22508c390ab3f88e6d6d86",
+        strip_prefix = "googletest-b6cd405286ed8635ece71c72f118e659f4ade3fb",
         urls = [
-            "https://mirror.bazel.build/github.com/google/googletest/archive/6729a1361150131bc5d394d5cd2b4cdf0953ee7b.zip",
-            "https://github.com/google/googletest/archive/6729a1361150131bc5d394d5cd2b4cdf0953ee7b.zip",
+            "https://mirror.bazel.build/github.com/google/googletest/archive/b6cd405286ed8635ece71c72f118e659f4ade3fb.zip",
+            "https://github.com/google/googletest/archive/b6cd405286ed8635ece71c72f118e659f4ade3fb.zip",
         ],
     )
 
-- 
GitLab


From 899f788738014fe438105fe30100e768b5f7900c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 16 Jan 2019 09:45:13 -0800
Subject: [PATCH 0805/1095] Add a missing explicit dependency between
 :nn_ops_test (which references the AvgPool op) and :pooling_ops.

PiperOrigin-RevId: 229573584
---
 tensorflow/core/kernels/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 374aa88d2d..96536e6945 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -4011,6 +4011,7 @@ tf_cuda_cc_test(
         ":nn",
         ":ops_testutil",
         ":ops_util",
+        ":pooling_ops",
         "//tensorflow/cc:cc_ops",
         "//tensorflow/cc:cc_ops_internal",
         "//tensorflow/core:core_cpu",
-- 
GitLab


From 9728b12bf7a2e4cf4010a9baaee5626fd2bd8140 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 16 Jan 2019 09:53:50 -0800
Subject: [PATCH 0806/1095] Update Google Cloud Bigtable C++ Client to the
 v0.5.0 release.

PiperOrigin-RevId: 229574976
---
 .../kernels/test_kernels/bigtable_test_client.cc      | 11 +++++++++++
 .../kernels/test_kernels/bigtable_test_client.h       |  6 ++++++
 2 files changed, 17 insertions(+)

diff --git a/tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client.cc b/tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client.cc
index c04d5578ee..e6fda9e617 100644
--- a/tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client.cc
+++ b/tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client.cc
@@ -410,6 +410,17 @@ BigtableTestClient::AsyncCheckAndMutateRow(
   return nullptr;
 }
 
+std::unique_ptr<
+    grpc::ClientAsyncReaderInterface<google::bigtable::v2::ReadRowsResponse>>
+BigtableTestClient::AsyncReadRows(
+    grpc::ClientContext* context,
+    const google::bigtable::v2::ReadRowsRequest& request,
+    grpc::CompletionQueue* cq, void* tag) {
+  LOG(WARNING) << "Call to InMemoryDataClient::" << __func__
+               << "(); this will likely cause a crash!";
+  return nullptr;
+}
+
 std::shared_ptr<grpc::Channel> BigtableTestClient::Channel() {
   LOG(WARNING) << "Call to InMemoryDataClient::Channel(); this will likely "
                   "cause a crash!";
diff --git a/tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client.h b/tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client.h
index 8570590457..8e1326f2ce 100644
--- a/tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client.h
+++ b/tensorflow/contrib/bigtable/kernels/test_kernels/bigtable_test_client.h
@@ -87,6 +87,12 @@ class BigtableTestClient : public ::google::cloud::bigtable::DataClient {
       const google::bigtable::v2::CheckAndMutateRowRequest& request,
       grpc::CompletionQueue* cq) override;
 
+  std::unique_ptr<
+      grpc::ClientAsyncReaderInterface<google::bigtable::v2::ReadRowsResponse>>
+  AsyncReadRows(grpc::ClientContext* context,
+                const google::bigtable::v2::ReadRowsRequest& request,
+                grpc::CompletionQueue* cq, void* tag) override;
+
   std::shared_ptr<grpc::Channel> Channel() override;
 
  private:
-- 
GitLab


From 3a2741544dc6aaf0abfe47d8fdd35d710dc7ffb0 Mon Sep 17 00:00:00 2001
From: Billy Lamberta <blamb@google.com>
Date: Wed, 16 Jan 2019 10:00:19 -0800
Subject: [PATCH 0807/1095] Publish /lite

PiperOrigin-RevId: 229576111
---
 tensorflow/lite/g3doc/custom_operators.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/lite/g3doc/custom_operators.md b/tensorflow/lite/g3doc/custom_operators.md
index 18202cbfb9..2d80668f37 100644
--- a/tensorflow/lite/g3doc/custom_operators.md
+++ b/tensorflow/lite/g3doc/custom_operators.md
@@ -137,9 +137,9 @@ operations instead of a single operator.
 
 ## Special TF Graph Attributes
 
-When Toco converts a TF graph into TFLite format, it makes some assumption about
-custom operations that might be not correct. In this case, the generated graph
-can be not executable.
+When `tflite_convert` converts a TensorFlow graph into TFLite format, it makes
+some assumption about custom operations that might be not correct. In this case,
+the generated graph may not execute.
 
 It is possible to add aditional information about your custom op output to TF
 graph before it is converted. The following attributes are supported:
-- 
GitLab


From ef5d41d827b4130cc3cedf08454e54ce1340f6eb Mon Sep 17 00:00:00 2001
From: Vojtech Bardiovsky <vbardiovsky@google.com>
Date: Wed, 16 Jan 2019 10:11:41 -0800
Subject: [PATCH 0808/1095] Refactoring: Get signature directly from concrete
 function.

We can do this now, because signature is stored on the function, and it will clean the list_all_concrete_functions return value.

PiperOrigin-RevId: 229578645
---
 tensorflow/python/eager/def_function.py       | 51 ++++++++-----------
 tensorflow/python/eager/def_function_test.py  |  9 +++-
 .../saved_model/function_serialization.py     |  7 ++-
 tensorflow/python/saved_model/load_test.py    |  4 +-
 tensorflow/python/saved_model/save.py         |  2 +-
 5 files changed, 36 insertions(+), 37 deletions(-)

diff --git a/tensorflow/python/eager/def_function.py b/tensorflow/python/eager/def_function.py
index 977f548b61..aef8f9ccda 100644
--- a/tensorflow/python/eager/def_function.py
+++ b/tensorflow/python/eager/def_function.py
@@ -447,50 +447,41 @@ class PolymorphicFunction(object):
     return initialize_variables.get_concrete_function()
 
   def _list_all_concrete_functions_for_serialization(self):
-    """Returns all of the concrete functions.
+    """Returns all concrete functions for serialization.
 
     Returns:
-      A list of tuples in the form (signature, concrete_function), where
-      concrete function is an instance of `Function`.
+      A list of instances of `Function`.
     """
-    input_signature = self._input_signature
-    if input_signature is not None:
+    if self._input_signature is not None:
       self.get_concrete_function()
     concrete_functions = []
-    for signature in self._cached_input_signatures:
-      flattened = nest.flatten(signature)
-      if any(
-          isinstance(arg, func_graph_module.UnknownArgument)
-          for arg in flattened):
-        logging.info("Unsupported signature for serialization: %s.", signature)
-        continue
-      concrete_function = self.get_concrete_function(*signature)
-      concrete_functions.append((signature, concrete_function))
-    return concrete_functions
-
-  @property
-  def _cached_input_signatures(self):
-    """All input signatures used to call this PolymorphicFunction."""
-    seen = list()
-    # We are using a list so that:
-    #  - the returned collection is deterministic, and
-    #  - we can use a custom equality operator (is_same_structure).
-    # This is run only at serialization time on likely very small inputs so we
-    # are not concerned about O(n^2) runtime.
     # pylint: disable=protected-access
-    concrete_functions = []
     if self._stateful_fn:
       concrete_functions.extend(self._stateful_fn._function_cache.values())
     if self._stateless_fn:
       concrete_functions.extend(self._stateless_fn._function_cache.values())
+    # pylint: enable=protected-access
+    deduplicated_concrete_functions = list()
+    seen_signatures = list()
+    # We are using a list so that:
+    #  - the returned collection is deterministic, and
+    #  - we can use a custom equality operator (is_same_structure).
+    # This is run only at serialization time on likely very small inputs so we
+    # are not concerned about O(n^2) runtime.
     for concrete_function in concrete_functions:
       signature, _ = concrete_function.structured_input_signature
+      flattened = nest.flatten(signature)
+      if any(
+          isinstance(arg, func_graph_module.UnknownArgument)
+          for arg in flattened):
+        logging.info("Unsupported signature for serialization: %s.", signature)
+        continue
       equal_to_signature = functools.partial(
           function_lib.is_same_structure, signature, check_values=True)
-      if not any(equal_to_signature(s) for s in seen):
-        yield signature
-        seen.append(signature)
-    # pylint: enable=protected-access
+      if not any(equal_to_signature(s) for s in seen_signatures):
+        deduplicated_concrete_functions.append(concrete_function)
+        seen_signatures.append(signature)
+    return deduplicated_concrete_functions
 
   def get_concrete_function(self, *args, **kwargs):
     """Returns a `Function` object specialized to inputs and execution context.
diff --git a/tensorflow/python/eager/def_function_test.py b/tensorflow/python/eager/def_function_test.py
index 77cc8ee981..256ef511a3 100644
--- a/tensorflow/python/eager/def_function_test.py
+++ b/tensorflow/python/eager/def_function_test.py
@@ -247,7 +247,7 @@ class DefFunctionTest(test.TestCase):
     concrete = compute.get_concrete_function(
         tensor_spec.TensorSpec(None, dtypes.float32))
     self.assertAllClose(4., concrete(constant_op.constant(2.)))
-    input_signature, = compute._cached_input_signatures
+    input_signature, _ = concrete.structured_input_signature
     self.assertEqual(
         tuple(input_signature),
         (tensor_spec.TensorSpec(None, dtypes.float32),))
@@ -260,8 +260,13 @@ class DefFunctionTest(test.TestCase):
 
     f(constant_op.constant([[3., 4.]]), constant_op.constant([2.]))
     f(constant_op.constant([[3, 4, 5]]), constant_op.constant([2]))
+
+    concrete_functions = f._list_all_concrete_functions_for_serialization()
+    signatures_for_serialization = [
+        c.structured_input_signature[0] for c in concrete_functions
+    ]
     self.assertEqual(
-        set(f._cached_input_signatures),
+        set(signatures_for_serialization),
         set(((tensor_spec.TensorSpec([1, 2], dtypes.float32),
               tensor_spec.TensorSpec([1], dtypes.float32)),
              (tensor_spec.TensorSpec([1, 3], dtypes.int32),
diff --git a/tensorflow/python/saved_model/function_serialization.py b/tensorflow/python/saved_model/function_serialization.py
index 8e8e3bc856..6496b2b2a4 100644
--- a/tensorflow/python/saved_model/function_serialization.py
+++ b/tensorflow/python/saved_model/function_serialization.py
@@ -48,7 +48,7 @@ def serialize_polymorphic_function(polymorphic_function, node_ids):
   proto.function_spec.CopyFrom(function_spec_proto)
   all_concrete_functions = \
       polymorphic_function._list_all_concrete_functions_for_serialization()  # pylint: disable=protected-access
-  for signature, concrete_function in all_concrete_functions:
+  for concrete_function in all_concrete_functions:
     bound_inputs = []
     try:
       for capture in concrete_function.captured_inputs:
@@ -60,10 +60,13 @@ def serialize_polymorphic_function(polymorphic_function, node_ids):
           "captures tensor %s which is unsupported or not reachable from root.",
           concrete_function.name, capture)
       continue
+    signature_args, signature_kwargs = \
+        concrete_function.structured_input_signature
+    del signature_kwargs
     function_proto = proto.monomorphic_function.add()
     function_proto.concrete_function = concrete_function.name
     function_proto.canonicalized_input_signature.CopyFrom(
-        coder.encode_structure(signature))
+        coder.encode_structure(signature_args))
     structured_outputs = func_graph_module.convert_structure_to_signature(
         concrete_function.structured_outputs)
     function_proto.output_signature.CopyFrom(
diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index 9eea170bf0..c8c8f6e1b3 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -401,8 +401,8 @@ class LoadTest(test.TestCase):
     self.assertAllEqual([2], root.f(constant_op.constant([1])).numpy())
     self.assertAllEqual([2, 4], root.f(constant_op.constant([1, 2])).numpy())
 
-    concrete_functions = root.f._list_all_concrete_functions_for_serialization()
-    self.assertEqual(1, len(concrete_functions))  # pylint: disable=protected-access
+    concrete_functions = root.f._list_all_concrete_functions_for_serialization()  # pylint: disable=protected-access
+    self.assertEqual(1, len(concrete_functions))
 
     imported = self.cycle(root)
 
diff --git a/tensorflow/python/saved_model/save.py b/tensorflow/python/saved_model/save.py
index d383bbecfc..ef3f67bbce 100644
--- a/tensorflow/python/saved_model/save.py
+++ b/tensorflow/python/saved_model/save.py
@@ -560,7 +560,7 @@ def _fill_meta_graph_def(meta_graph_def, saveable_view, signature_functions,
 
   with exported_graph.as_default():
     signatures = _generate_signatures(signature_functions, resource_map)
-    for _, concrete_function in concrete_functions:
+    for concrete_function in concrete_functions:
       concrete_function.add_to_graph()
     saver_def = saver.to_proto()
     meta_graph_def.saver_def.CopyFrom(saver_def)
-- 
GitLab


From 7226c27f4d0161954ee36b0fcb32275a2942d5bc Mon Sep 17 00:00:00 2001
From: Tom Hennigan <tomhennigan@google.com>
Date: Wed, 16 Jan 2019 10:13:48 -0800
Subject: [PATCH 0809/1095] Use new _ property for distribute strategy on
 resource variable.

PiperOrigin-RevId: 229579047
---
 .../distribute/python/mirrored_strategy_multigpu_test.py        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
index 59d711ae01..6727c7d703 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
@@ -267,7 +267,7 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
     self.assertIs(strategy, var.distribute_strategy)
     for d in var.devices:
       self.assertEqual(d, var.get(d).device)
-      self.assertIs(strategy, var.get(d).distribute_strategy)
+      self.assertIs(strategy, var.get(d)._distribute_strategy)  # pylint: disable=protected-access
 
   def testVariableInFuncGraph(self, distribution):
     def model_fn():
-- 
GitLab


From 352525990469da6694abaa4e70edfdbb287dd984 Mon Sep 17 00:00:00 2001
From: Billy Lamberta <blamb@google.com>
Date: Wed, 16 Jan 2019 10:14:45 -0800
Subject: [PATCH 0810/1095] Add tutorial vid links.

PiperOrigin-RevId: 229579217
---
 tensorflow/lite/g3doc/performance/gpu.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tensorflow/lite/g3doc/performance/gpu.md b/tensorflow/lite/g3doc/performance/gpu.md
index 14ca54a158..c738922612 100644
--- a/tensorflow/lite/g3doc/performance/gpu.md
+++ b/tensorflow/lite/g3doc/performance/gpu.md
@@ -24,6 +24,9 @@ The easiest way to try out the experimental GPU delegate is to follow the below
 
 ### Android (with Android Studio)
 
+For a step-by-step tutorial, watch the
+[Experimental GPU Delegate for Android](https://youtu.be/Xkhgre8r5G0) video.
+
 Note: This requires OpenGL ES 3.1 or higher.
 
 #### Step 1. Clone the TensorFlow source code and open it in Android Studio
@@ -55,6 +58,9 @@ run on the GPU.
 
 ### iOS (with XCode)
 
+For a step-by-step tutorial, watch the
+[Experimental GPU Delegate for iOS](https://youtu.be/a5H4Zwjp49c) video.
+
 Note: This requires XCode v10.1 or later.
 
 #### Step 1. Get the demo source code and make sure it compiles.
-- 
GitLab


From ff12131917fb3f9c789063e3a78e1d8a41e49776 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 16 Jan 2019 10:17:33 -0800
Subject: [PATCH 0811/1095] Automated rollback of commit
 8154ec0b4a893657e48b838c06e1f988aeac4115

PiperOrigin-RevId: 229579722
---
 .../process_function_library_runtime.cc       |  10 +-
 tensorflow/core/grappler/optimizers/BUILD     |   1 -
 .../grappler/optimizers/function_optimizer.cc |  74 +++++------
 .../optimizers/function_optimizer_test.cc     | 122 ------------------
 4 files changed, 38 insertions(+), 169 deletions(-)

diff --git a/tensorflow/core/common_runtime/process_function_library_runtime.cc b/tensorflow/core/common_runtime/process_function_library_runtime.cc
index 76c75ad3d2..b236343a0f 100644
--- a/tensorflow/core/common_runtime/process_function_library_runtime.cc
+++ b/tensorflow/core/common_runtime/process_function_library_runtime.cc
@@ -568,12 +568,15 @@ Status ProcessFunctionLibraryRuntime::InstantiateMultiDevice(
   DumpGraph("Before running POST_PLACEMENT passes", graph.get());
   TF_RETURN_IF_ERROR(OptimizationPassRegistry::Global()->RunGrouping(
       OptimizationPassRegistry::POST_PLACEMENT, optimization_options));
+  DumpGraph("Before running POST_REWRITE_FOR_EXEC passes", graph.get());
+  TF_RETURN_IF_ERROR(OptimizationPassRegistry::Global()->RunGrouping(
+      OptimizationPassRegistry::POST_REWRITE_FOR_EXEC, optimization_options));
+  DumpGraph("After all optimization passes", graph.get());
 
   Device* cpu_device;
   TF_RETURN_IF_ERROR(device_mgr_->LookupDevice("CPU:0", &cpu_device));
 
   if (options.optimize_graph_fn) {
-    DumpGraph("Before running graph optimization fn", graph.get());
     Status status = options.optimize_graph_fn(std::move(ret_node_names),
                                               &data->overlay_lib_, device_set,
                                               cpu_device, &graph);
@@ -584,11 +587,6 @@ Status ProcessFunctionLibraryRuntime::InstantiateMultiDevice(
     DumpGraph("After optimization", graph.get());
   }
 
-  DumpGraph("Before running POST_REWRITE_FOR_EXEC passes", graph.get());
-  TF_RETURN_IF_ERROR(OptimizationPassRegistry::Global()->RunGrouping(
-      OptimizationPassRegistry::POST_REWRITE_FOR_EXEC, optimization_options));
-  DumpGraph("After all optimization passes", graph.get());
-
   std::unordered_map<string, std::unique_ptr<Graph>> subgraphs;
   TF_RETURN_IF_ERROR(
       PartitionFunctionGraph(device_set, std::move(graph), &subgraphs));
diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index 70e0f226fc..ee216f80f9 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -181,7 +181,6 @@ tf_cuda_cc_test(
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/utils:grappler_test",
-        "@com_google_absl//absl/algorithm:container",
     ],
 )
 
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.cc b/tensorflow/core/grappler/optimizers/function_optimizer.cc
index 794e87dc38..bd69405b1a 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer.cc
@@ -27,8 +27,6 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/device_mgr.h"
 #include "tensorflow/core/common_runtime/device_set.h"
 #include "tensorflow/core/common_runtime/function.h"
-#include "tensorflow/core/common_runtime/lower_if_while.h"
-#include "tensorflow/core/common_runtime/optimization_registry.h"
 #include "tensorflow/core/common_runtime/placer.h"
 #include "tensorflow/core/common_runtime/process_function_library_runtime.h"
 #include "tensorflow/core/framework/attr_value_util.h"
@@ -1367,6 +1365,20 @@ Status IsInlinableIndirectFunctionCall(const FunctionOptimizerContext& ctx,
         SummarizeNodeDef(func_node));
   }
 
+  // TODO(b/120991525, b/120986912): We need to lower `If` and `While` nodes to
+  // `Switch` nodes after function inlining (one more PRE_PLACEMENT pass?), but
+  // because of the reason described above we are not sure that it's safe, for
+  // now just disable inlining functions with functional control flow.
+  const auto is_functional_ctrl_flow_op = [](const NodeDef& node) {
+    return IsIf(node) || IsWhile(node);
+  };
+  if (absl::c_any_of(func.node_def(), is_functional_ctrl_flow_op)) {
+    return errors::FailedPrecondition(
+        "Can't inline function with `If` or `While` nodes in the function "
+        "body: ",
+        SummarizeNodeDef(func_node));
+  }
+
   return Status::OK();
 }
 
@@ -1476,46 +1488,22 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
 
   const string prefix = strings::StrCat(func_node.name(), "/");
 
-  // ------------------------------------------------------------------------ //
-  // Grappler receives the graph after PRE_PLACEMENT, Placer, and POST_PLACEMENT
-  // passes, so each node has a valid device assignment. Also V2 control
-  // flow ops (functional If and While) lowered to V1 control flow (Switch and
-  // Merge nodes). To keep graph valid for execution we must assign device to
-  // every inlined graph node, and also lower the control flow.
-
-  // Control flow lowering and Placer works with a Graph object.
-  std::unique_ptr<Graph> func_body_graph =
-      absl::make_unique<Graph>(ctx->function_library());
-
-  GraphConstructorOptions opts;
-  TF_RETURN_IF_ERROR(
-      ConvertGraphDefToGraph(opts, item.graph, func_body_graph.get()));
-
-  GraphOptimizationPassOptions opt_options;
-  opt_options.graph = &func_body_graph;
-  opt_options.flib_def = ctx->mutable_function_library();
-
-  // TODO(ezhulenev): Should we run full PRE_PLACEMENT pass here? And
-  // POST_PLACEMENT after placer?
-  LowerIfWhilePass pass;
-  TF_RETURN_IF_ERROR(pass.Run(opt_options));
-
   // ------------------------------------------------------------------------ //
   // Before placing the function body nodes we pin input placeholders to the
   // same device as their corresponding input nodes.
 
-  for (Node* func_body_node : func_body_graph->nodes()) {
+  for (NodeDef& func_body_node : *item.graph.mutable_node()) {
     const auto input_placeholder_idx =
-        input_placeholders_idx.find(func_body_node->name());
+        input_placeholders_idx.find(func_body_node.name());
 
     if (input_placeholder_idx != input_placeholders_idx.end()) {
       const int input_idx = input_placeholder_idx->second;
       const GraphView::OutputPort output_port =
           ctx->graph_view().GetRegularFanin({&func_node, input_idx});
 
-      VLOG(3) << "Pin inlined function input node '" << func_body_node->name()
+      VLOG(3) << "Pin inlined function input node '" << func_body_node.name()
               << "' to the '" << output_port.node->device() << "' device.";
-      func_body_node->set_requested_device(output_port.node->device());
+      func_body_node.set_device(output_port.node->device());
     }
   }
 
@@ -1523,6 +1511,8 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
   // After placing nodes corresponding to the function inputs, we need to assign
   // device placements to all other function body nodes.
 
+  GraphDef placed_graph_def;
+
   const DeviceSet* devices = ctx->devices();
 
   if (devices->devices().empty()) {
@@ -1532,8 +1522,9 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
     // of a batch job for graph analysis/optimization.
     VLOG(3) << "Assign function call node device to all function body nodes. "
             << "Device: " << func_node.device();
-    for (Node* func_body_node : func_body_graph->nodes()) {
-      func_body_node->set_requested_device(func_node.device());
+    placed_graph_def = item.mutable_function_body();
+    for (NodeDef& node : *placed_graph_def.mutable_node()) {
+      node.set_device(func_node.device());
     }
   } else {
     // If we are running in an active runtime session, Grappler will get the
@@ -1545,20 +1536,23 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
                    [](string* out, const Device* d) { out->append(d->name()); })
             << "]";
 
+    // Construct a Graph object from the instantiated function body.
+    GraphConstructorOptions opts;
+    Graph graph(ctx->function_library());
+    TF_RETURN_IF_ERROR(
+        ConvertGraphDefToGraph(opts, item.function_body(), &graph));
+
     // Use function caller node device as a default for placer.
     const Device* default_device =
         devices->FindDeviceByName(func_node.device());
 
-    Placer placer(func_body_graph.get(), devices,
-                  nullptr /* No session options */, default_device);
+    Placer placer(&graph, devices, nullptr, /* No session options */
+                  default_device);
     TF_RETURN_IF_ERROR(placer.Run());
-  }
-
-  // TODO(ezhulenev): Should we run POST_PLACEMENT runtime pass here?
 
-  // All following transformations will work with GraphDef.
-  GraphDef placed_graph_def;
-  func_body_graph->ToGraphDef(&placed_graph_def);
+    // Convert Graph back to the GraphDef.
+    graph.ToGraphDef(&placed_graph_def);
+  }
 
   // ------------------------------------------------------------------------ //
   // After all nodes placed we need to prepare them for inlining into the
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
index f4752bd584..827b0658c5 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
@@ -14,8 +14,6 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/core/grappler/optimizers/function_optimizer.h"
-
-#include "absl/algorithm/container.h"
 #include "tensorflow/cc/ops/functional_ops.h"
 #include "tensorflow/cc/ops/standard_ops.h"
 #include "tensorflow/core/framework/function_testlib.h"
@@ -1184,126 +1182,6 @@ TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithMergedDeadTensors) {
   test::ExpectTensorEqual<float>(tensors[0], tensors_expected[0]);
 }
 
-TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithFunctionalControlFlow) {
-  using test::function::NDef;
-  using FDH = FunctionDefHelper;
-
-  FunctionOptimizer optimizer(RewriterConfig::AGGRESSIVE);
-
-  FunctionDef add_func = FunctionDefHelper::Create(
-      "MyAdd", {"x:T", "y:T"}, {"z:T"}, {"T: {float, double}"},
-      {{{"add"}, "Add", {"x", "y"}, {{"T", "$T"}}}},
-      /* Mapping between function returns and function node outputs. */
-      {{"z", "add:z:0"}});
-
-  FunctionDef mul_func = FunctionDefHelper::Create(
-      "MyMul", {"x:T", "y:T"}, {"z:T"}, {"T: {float, double}"},
-      {{{"mul"}, "Mul", {"x", "y"}, {{"T", "$T"}}}},
-      /* Mapping between function returns and function node outputs. */
-      {{"z", "mul:z:0"}});
-
-  // Compute: return cond ? a + b : a * b
-  FunctionDef add_or_mul_func = FunctionDefHelper::Create(
-      "AddOrMul", {"cond:bool", "x:float", "y:float"}, {"z:float"}, {},
-      {
-          {{"if_node"},
-           "If",
-           {"cond", "x", "y"},
-           {
-               {"Tcond", DT_BOOL},
-               {"Tin", DataTypeSlice{DT_FLOAT, DT_FLOAT}},
-               {"Tout", DataTypeSlice{DT_FLOAT}},
-               {"then_branch", FDH::FunctionRef("MyAdd", {{"T", DT_FLOAT}})},
-               {"else_branch", FDH::FunctionRef("MyMul", {{"T", DT_FLOAT}})},
-               {"_lower_using_switch_merge", true},
-           }},
-      },
-      /* Mapping between function returns and function node outputs. */
-      {{"z", "if_node:output:0"}});
-
-  // Build a computation graph for:
-  //   is_add: bool
-  //   a: float
-  //   b: float
-  //   c = AddOrMul(is_add, a, b)  # is_add ? a + b : a * b
-  //   d = Identity(c)
-  //   return d
-
-  // c = MyMul(a, b)
-  GrapplerItem item;
-  item.fetch = {"d"};
-  item.graph = test::function::GDef(
-      {NDef("is_add", "Placeholder", {}, {{"dtype", DT_BOOL}}, kDevice),
-       NDef("a", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
-       NDef("b", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice),
-
-       NDef("c", "PartitionedCall", {"is_add", "a", "b"},
-            {{"Tin", DataTypeSlice{DT_BOOL, DT_FLOAT, DT_FLOAT}},
-             {"Tout", DataTypeSlice{DT_FLOAT}},
-             {"f", FDH::FunctionRef("AddOrMul")}},
-            kDevice),
-
-       NDef("d", "Identity", {"c"}, {{"T", DT_FLOAT}}, kDevice)},
-      // Function library.
-      {add_or_mul_func, add_func, mul_func});
-
-  GraphDef optimized_graph;
-  TF_EXPECT_OK(optimizer.Optimize(nullptr, item, &optimized_graph));
-
-  const auto count_nodes_with_op = [&](const string& op) {
-    return absl::c_count_if(optimized_graph.node(), [&](const NodeDef& node) {
-      return node.op() == op;
-    });
-  };
-
-  // All `PartitionedCall` nodes in the optimized graph must be inlined, and
-  // `If` node must be lowered to `Switch` and `Merge` nodes.
-  EXPECT_EQ(count_nodes_with_op("PartitionedCallOp"), 0);
-  EXPECT_EQ(count_nodes_with_op("If"), 0);
-  EXPECT_EQ(count_nodes_with_op("Switch"), 3);
-  EXPECT_EQ(count_nodes_with_op("Merge"), 1);
-
-  GrapplerItem optimized = item.WithGraph(std::move(optimized_graph));
-
-  Tensor one = test::AsScalar<float>(1.0);
-  Tensor two = test::AsScalar<float>(2.0);
-  Tensor three = test::AsScalar<float>(3.0);
-
-  const auto feed_args = [&](bool is_add) {
-    std::vector<std::pair<string, Tensor>> feed;
-    feed.emplace_back("a", one);
-    feed.emplace_back("b", two);
-    feed.emplace_back("is_add", test::AsScalar<bool>(is_add));
-    return feed;
-  };
-
-  {  // Check 'is_add == true': a + b
-    item.feed = feed_args(true);
-    optimized.feed = feed_args(true);
-
-    auto tensors_expected = EvaluateFetchNodes(item);
-    ASSERT_EQ(tensors_expected.size(), 1);
-    test::ExpectTensorEqual<float>(tensors_expected[0], three);
-
-    auto tensors = EvaluateFetchNodes(optimized);
-    ASSERT_EQ(tensors.size(), tensors_expected.size());
-    test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
-  }
-
-  {  // Check 'is_add == false': a * b
-    item.feed = feed_args(false);
-    optimized.feed = feed_args(false);
-
-    auto tensors_expected = EvaluateFetchNodes(item);
-    ASSERT_EQ(tensors_expected.size(), 1);
-    test::ExpectTensorEqual<float>(tensors_expected[0], two);
-
-    auto tensors = EvaluateFetchNodes(optimized);
-    ASSERT_EQ(tensors.size(), tensors_expected.size());
-    test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
-  }
-}
-
 TEST_F(FunctionOptimizerTest, SpecializeFunctionXTimesTwo) {
   using test::function::NDef;
 
-- 
GitLab


From 04181795a31d2b332aa0d0fc5daf9d254432c042 Mon Sep 17 00:00:00 2001
From: Igor Ganichev <iga@google.com>
Date: Wed, 16 Jan 2019 10:24:17 -0800
Subject: [PATCH 0812/1095] Add num_values argument to Tensor::DebugString()

Non-CPU Tensors cause a crash when DebugString() tries to access
their values. num_values can be used not to access the values or
print more than default of 3 during debugging.

PiperOrigin-RevId: 229581029
---
 tensorflow/compiler/jit/xla_launch_util.cc |  2 +-
 tensorflow/core/framework/tensor.cc        |  9 +++++++--
 tensorflow/core/framework/tensor.h         | 12 ++++++++++--
 3 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/tensorflow/compiler/jit/xla_launch_util.cc b/tensorflow/compiler/jit/xla_launch_util.cc
index 4e3cb0238f..c64981053f 100644
--- a/tensorflow/compiler/jit/xla_launch_util.cc
+++ b/tensorflow/compiler/jit/xla_launch_util.cc
@@ -377,7 +377,7 @@ Status XlaComputationLaunchContext::PopulateOutputs(
     }
 
     if (VLOG_IS_ON(3)) {
-      VLOG(3) << ctx->mutable_output(i)->DebugString();
+      VLOG(3) << ctx->mutable_output(i)->DeviceSafeDebugString();
     }
   }
 
diff --git a/tensorflow/core/framework/tensor.cc b/tensorflow/core/framework/tensor.cc
index 0c96ec8168..ab492a2189 100644
--- a/tensorflow/core/framework/tensor.cc
+++ b/tensorflow/core/framework/tensor.cc
@@ -1174,10 +1174,15 @@ bool Tensor::SharesBufferWith(const Tensor& b) const {
          buf_->root_buffer() == b.buf_->root_buffer();
 }
 
-string Tensor::DebugString() const {
+string Tensor::DebugString(int num_values) const {
   return strings::StrCat("Tensor<type: ", DataTypeString(dtype()),
                          " shape: ", shape().DebugString(),
-                         " values: ", SummarizeValue(3), ">");
+                         " values: ", SummarizeValue(num_values), ">");
+}
+
+string Tensor::DeviceSafeDebugString() const {
+  return strings::StrCat("Tensor<type: ", DataTypeString(dtype()),
+                         " shape: ", shape().DebugString(), ">");
 }
 
 void Tensor::FillDescription(TensorDescription* description) const {
diff --git a/tensorflow/core/framework/tensor.h b/tensorflow/core/framework/tensor.h
index 009dd0846d..44293c5c19 100644
--- a/tensorflow/core/framework/tensor.h
+++ b/tensorflow/core/framework/tensor.h
@@ -526,7 +526,15 @@ class Tensor {
   string SummarizeValue(int64 max_entries, bool print_v2 = false) const;
 
   /// A human-readable summary of the tensor suitable for debugging.
-  string DebugString() const;
+  // `num_values` is the number of actual data values in the tensor
+  // included in the message. If the tensor might be resident in
+  // GPU/TPU memory use DeviceSafeDebugString instead.
+  string DebugString(int num_values = 3) const;
+
+  // Variant of DebugString() that should be used for possibly non-CPU tensors.
+  // If the tensor is not resident on CPU, we can't read its values as
+  // DebugString() does.
+  string DeviceSafeDebugString() const;
 
   /// Fill in the `TensorDescription` proto with metadata about the
   /// tensor that is useful for monitoring and debugging.
@@ -594,7 +602,7 @@ class Tensor {
       OpKernelContext* ctx, Var* var);  // For access to RefCountIsOne().
   friend Status batch_util::CopyElementToSlice(
       Tensor element, Tensor* parent,
-      int64 index);                // For access to RefCountIsOne().
+      int64 index);  // For access to RefCountIsOne().
   friend Status batch_util::MaybeMoveSliceToElement(
       Tensor* parent, Tensor* element,
       int64 index);  // For access to RefCountIsOne().
-- 
GitLab


From 1b2b712b836f1c7e32526f62c797a9d9a46d9561 Mon Sep 17 00:00:00 2001
From: Shivani Agrawal <shivaniagrawal@google.com>
Date: Wed, 16 Jan 2019 10:41:46 -0800
Subject: [PATCH 0813/1095] Fix in evaluate for SparseTensors.

PiperOrigin-RevId: 229584367
---
 .../bucket_by_sequence_length_test.py         |  9 ++---
 .../experimental/kernel_tests/scan_test.py    |  4 +-
 .../experimental/kernel_tests/unbatch_test.py | 37 +++++--------------
 tensorflow/python/framework/test_util.py      |  5 ++-
 4 files changed, 17 insertions(+), 38 deletions(-)

diff --git a/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py b/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
index 71ae14b5e1..0bbf0e9a12 100644
--- a/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/bucket_by_sequence_length_test.py
@@ -77,12 +77,11 @@ def _get_record_shape(sparse):
 class BucketBySequenceLengthTest(test_base.DatasetTestBase,
                                  parameterized.TestCase):
 
-  # TODO(b/117581999): add eager coverage.
   @parameterized.named_parameters(
       ("WithoutPadding", True),
       ("WithPadding", False),
   )
-  def testSkipEagerBucketDropReminder(self, param_no_padding):
+  def testBucketDropReminder(self, param_no_padding):
 
     boundaries = [10, 20, 30]
     batch_sizes = [10, 8, 4, 2]
@@ -202,12 +201,11 @@ class BucketBySequenceLengthTest(test_base.DatasetTestBase,
 
     _test_bucket_by_padding(param_no_padding)
 
-  # TODO(b/117581999): add eager coverage.
   @parameterized.named_parameters(
       ("WithoutPadding", True),
       ("WithPadding", False),
   )
-  def testSkipEagerBucket(self, param_no_padding):
+  def testBucket(self, param_no_padding):
 
     boundaries = [10, 20, 30]
     batch_sizes = [10, 8, 4, 2]
@@ -383,12 +381,11 @@ class BucketBySequenceLengthTest(test_base.DatasetTestBase,
 
     _test_tuple_elements_by_padding(param_no_padding)
 
-  # TODO(b/117581999): add eager coverage
   @parameterized.named_parameters(
       ("DoDropRemainder", True),
       ("DoNotDropRemainder", False),
   )
-  def testSkipEagerBucketSparse(self, param_drop_remainder):
+  def testBucketSparse(self, param_drop_remainder):  # pylint: disable=g-doc-args
     """Tests bucketing of sparse tensors (case where `no_padding` == True).
 
     Test runs on following dataset:
diff --git a/tensorflow/python/data/experimental/kernel_tests/scan_test.py b/tensorflow/python/data/experimental/kernel_tests/scan_test.py
index f5ac0f5007..38e9b1e128 100644
--- a/tensorflow/python/data/experimental/kernel_tests/scan_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/scan_test.py
@@ -70,9 +70,7 @@ class ScanTest(test_base.DatasetTestBase):
     self.assertEqual(5, self.evaluate(next_element()))
     self.assertEqual(8, self.evaluate(next_element()))
 
-  # TODO(b/117581999): Add coverage for eager.
-  @test_util.run_deprecated_v1
-  def testSkipEagerSparseCount(self):
+  def testSparseCount(self):
 
     def _sparse(i):
       return sparse_tensor.SparseTensorValue(
diff --git a/tensorflow/python/data/experimental/kernel_tests/unbatch_test.py b/tensorflow/python/data/experimental/kernel_tests/unbatch_test.py
index e4034cc43a..613fe0da6b 100644
--- a/tensorflow/python/data/experimental/kernel_tests/unbatch_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/unbatch_test.py
@@ -68,9 +68,7 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
     self.assertDatasetProduces(
         data, [(i, compat.as_bytes(str(i)), i) for i in range(10)])
 
-  # TODO(b/117581999): Add eager coverage.
-  @test_util.run_deprecated_v1
-  def testSkipEagerUnbatchDatasetWithSparseTensor(self):
+  def testUnbatchDatasetWithSparseTensor(self):
     st = sparse_tensor.SparseTensorValue(
         indices=[[i, i] for i in range(10)],
         values=list(range(10)),
@@ -79,20 +77,12 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
     data = data.apply(batching.unbatch())
     data = data.batch(5)
     data = data.apply(batching.unbatch())
-    iterator = dataset_ops.make_one_shot_iterator(data)
-    next_element = iterator.get_next()
-
-    for i in range(10):
-      st_row = self.evaluate(next_element)
-      self.assertEqual([i], st_row.indices)
-      self.assertEqual([i], st_row.values)
-      self.assertEqual([10], st_row.dense_shape)
-    with self.assertRaises(errors.OutOfRangeError):
-      self.evaluate(next_element)
+    expected_output = [
+        sparse_tensor.SparseTensorValue([[i]], [i], [10]) for i in range(10)
+    ]
+    self.assertDatasetProduces(data, expected_output=expected_output)
 
-  # TODO(b/117581999): Add eager coverage.
-  @test_util.run_deprecated_v1
-  def testSkipEagerUnbatchDatasetWithDenseAndSparseTensor(self):
+  def testUnbatchDatasetWithDenseAndSparseTensor(self):
     st = sparse_tensor.SparseTensorValue(
         indices=[[i, i] for i in range(10)],
         values=list(range(10)),
@@ -101,16 +91,9 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
     data = data.apply(batching.unbatch())
     data = data.batch(5)
     data = data.apply(batching.unbatch())
-    next_element = self.getNext(data)
-
-    for i in range(10):
-      dense_elem, st_row = self.evaluate(next_element())
-      self.assertEqual(i, dense_elem)
-      self.assertEqual([i], st_row.indices)
-      self.assertEqual([i], st_row.values)
-      self.assertEqual([10], st_row.dense_shape)
-    with self.assertRaises(errors.OutOfRangeError):
-      self.evaluate(next_element())
+    expected_output = [(i, sparse_tensor.SparseTensorValue([[i]], [i], [10]))
+                       for i in range(10)]
+    self.assertDatasetProduces(data, expected_output=expected_output)
 
   def testUnbatchSingleElementTupleDataset(self):
     data = tuple([(math_ops.range(10),) for _ in range(3)])
@@ -150,7 +133,7 @@ class UnbatchTest(test_base.DatasetTestBase, parameterized.TestCase):
     with self.assertRaises(ValueError):
       data.apply(batching.unbatch())
 
-  # TODO(b/117581999): eager mode doesnt capture raised error, debug.
+  # Note: dynamic shape mismatch is graph specific test.
   @test_util.run_deprecated_v1
   def testSkipEagerUnbatchDynamicShapeMismatch(self):
     ph1 = array_ops.placeholder(dtypes.int32, shape=[None])
diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py
index e2b906e37c..732fd31389 100644
--- a/tensorflow/python/framework/test_util.py
+++ b/tensorflow/python/framework/test_util.py
@@ -1658,8 +1658,9 @@ class TensorFlowTestCase(googletest.TestCase):
     else:
       try:
         if sparse_tensor.is_sparse(tensor):
-          return sparse_tensor.SparseTensorValue(tensor.indices, tensor.values,
-                                                 tensor.dense_shape)
+          return sparse_tensor.SparseTensorValue(tensor.indices.numpy(),
+                                                 tensor.values.numpy(),
+                                                 tensor.dense_shape.numpy())
         return tensor.numpy()
       except AttributeError as e:
         six.raise_from(ValueError("Unsupported type %s." % type(tensor)), e)
-- 
GitLab


From b59aa4b89356333027bb38f7364f18c4df882835 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 16 Jan 2019 10:44:30 -0800
Subject: [PATCH 0814/1095] Update landscape layout with model, device, and
 number of thread selectors.

PiperOrigin-RevId: 229584823
---
 .../layout-land/fragment_camera2_basic.xml    | 128 +++++++++++-------
 1 file changed, 82 insertions(+), 46 deletions(-)

diff --git a/tensorflow/lite/java/demo/app/src/main/res/layout-land/fragment_camera2_basic.xml b/tensorflow/lite/java/demo/app/src/main/res/layout-land/fragment_camera2_basic.xml
index ee71ab808f..323b21dbce 100644
--- a/tensorflow/lite/java/demo/app/src/main/res/layout-land/fragment_camera2_basic.xml
+++ b/tensorflow/lite/java/demo/app/src/main/res/layout-land/fragment_camera2_basic.xml
@@ -16,67 +16,103 @@
 
 <LinearLayout
     xmlns:android="http://schemas.android.com/apk/res/android"
-    android:layout_width="match_parent"
+    xmlns:tools="http://schemas.android.com/tools" android:layout_width="match_parent"
     android:layout_height="match_parent"
     android:background="#bb7700"
     android:orientation="horizontal">
 
-  <com.example.android.tflitecamerademo.AutoFitTextureView
+    <com.example.android.tflitecamerademo.AutoFitTextureView
       android:id="@+id/texture"
       android:layout_width="0dp"
       android:layout_height="match_parent"
       android:layout_weight=".8"/>
 
-  <LinearLayout
+    <LinearLayout
       android:layout_width="0dp"
       android:layout_height="match_parent"
       android:layout_weight=".2"
       android:orientation="vertical">
 
-    <ImageView
-        android:id="@+id/logoview"
-        android:layout_width="wrap_content"
-        android:layout_height="wrap_content"
-        android:scaleType="centerInside"
-        android:src="@drawable/logo"/>
-
-    <RadioGroup
-        android:gravity="center"
-        android:layout_width="match_parent"
-        android:layout_height="match_parent"
-        android:orientation="horizontal">
-        <RadioButton
-            android:id="@+id/radio_cpu"
-            android:background="#0000000f"
-            android:layout_width="wrap_content"
-            android:layout_height="wrap_content"
-            android:text="@string/cpu"
-            android:textColor="@android:color/white" />
-        <RadioButton
-            android:id="@+id/radio_nnapi"
-            android:background="#0000000f"
+        <ImageView
+            android:id="@+id/logoview"
             android:layout_width="wrap_content"
+            android:layout_height="47dp"
+            android:scaleType="centerInside"
+            android:src="@drawable/logo"/>
+
+        <TextView
+            android:id="@+id/text"
+            android:layout_width="match_parent"
+            android:layout_height="160dp"
+            android:paddingTop="20dp"
+            android:textColor="#FFF"
+            android:textSize="20sp"
+            android:textStyle="bold"/>
+        <LinearLayout
+            android:id="@+id/modelLayout"
+            android:layout_width="match_parent"
+            android:layout_height="150dp"
+            android:orientation="vertical">
+
+            <TextView
+                android:id="@+id/textView"
+                android:layout_width="match_parent"
+                android:layout_height="20dp"
+                android:text="@string/modelLabel"
+                android:textAlignment="center"
+                android:textColor="@android:color/white"/>
+
+            <ListView
+                android:id="@+id/model"
+                android:layout_width="match_parent"
+                android:layout_height="wrap_content">
+
+            </ListView>
+        </LinearLayout>
+        <LinearLayout
+            android:id="@+id/deviceLayout"
+            android:layout_width="match_parent"
+            android:layout_height="150dp"
+            android:orientation="vertical">
+
+            <TextView
+                android:id="@+id/textView2"
+                android:layout_width="match_parent"
+                android:layout_height="20dp"
+                android:text="@string/deviceLabel"
+                android:textAlignment="center"
+                android:textColor="@android:color/white"/>
+
+            <ListView
+                android:id="@+id/device"
+                android:layout_width="match_parent"
+                android:layout_height="wrap_content"/>
+
+        </LinearLayout>
+
+        <LinearLayout
+            android:layout_width="match_parent"
             android:layout_height="wrap_content"
-            android:text="@string/nnapi"
-            android:textColor="@android:color/white" />
-        </RadioGroup>
-
-    <NumberPicker
-        android:id="@+id/np"
-        android:layout_width="wrap_content"
-        android:layout_height="47dp"
-        android:layout_gravity="center_horizontal"
-        android:visibility="visible"/>
-
-    <TextView
-        android:id="@+id/text"
-        android:textStyle="bold"
-        android:layout_width="match_parent"
-        android:layout_height="match_parent"
-        android:paddingTop="20dp"
-        android:textColor="#FFF"
-        android:textSize="20sp"/>
-
-  </LinearLayout>
+        >
+
+            <TextView
+                android:layout_width="wrap_content"
+                android:layout_height="wrap_content"
+                android:gravity="center"
+                android:text="Threads"
+                android:textAlignment="center"
+                android:textColor="@android:color/white"/>
+
+            <NumberPicker
+                android:id="@+id/np"
+                android:layout_width="match_parent"
+                android:layout_height="wrap_content"
+                android:layout_marginLeft="10dp"
+                android:theme="@style/AppTheme.Picker"
+                android:visibility="visible"/>
+
+        </LinearLayout>
+
+    </LinearLayout>
 </LinearLayout>
 
-- 
GitLab


From b4ae5ba323c20ad30e3f7559d547a6b76d5f33e6 Mon Sep 17 00:00:00 2001
From: Rachel Lim <rachelim@google.com>
Date: Wed, 16 Jan 2019 10:51:07 -0800
Subject: [PATCH 0815/1095] [tf.data] Add a ChooseFastestDataset, which will
 allow our optimizations to dynamically pick between two equivalent input
 datasets

- For the first N iterations, run two threads simultaneously that each pulls from one of the inputs. The threads also measure time taken. Wait for both threads to complete before returning.
- After N iterations, decide which dataset is faster and just use that.

There are more sophisticated implementations that could be better, e.g. instead of waiting for both threads to complete, return when the faster of them is finished. However, the code here is significantly simpler (I also implemented the other one elsewhere), at the cost of slower iterations for the first N iterations -- the iteration time for the first N iters is max(input_a, input_b). Open to ideas.

PiperOrigin-RevId: 229586112
---
 ...def_ExperimentalChooseFastestDataset.pbtxt |   4 +
 .../core/kernels/data/experimental/BUILD      |  13 +
 .../experimental/choose_fastest_dataset_op.cc | 346 ++++++++++++++++++
 .../core/ops/experimental_dataset_ops.cc      |   7 +
 .../python/data/experimental/benchmarks/BUILD |  14 +
 .../benchmarks/choose_fastest_benchmark.py    | 105 ++++++
 .../kernel_tests/optimization/BUILD           |  20 +
 .../choose_fastest_dataset_test.py            |  85 +++++
 .../kernel_tests/serialization/BUILD          |  18 +
 ...oose_fastest_dataset_serialization_test.py |  45 +++
 .../data/experimental/ops/optimization.py     |  45 +++
 11 files changed, 702 insertions(+)
 create mode 100644 tensorflow/core/api_def/base_api/api_def_ExperimentalChooseFastestDataset.pbtxt
 create mode 100644 tensorflow/core/kernels/data/experimental/choose_fastest_dataset_op.cc
 create mode 100644 tensorflow/python/data/experimental/benchmarks/choose_fastest_benchmark.py
 create mode 100644 tensorflow/python/data/experimental/kernel_tests/optimization/choose_fastest_dataset_test.py
 create mode 100644 tensorflow/python/data/experimental/kernel_tests/serialization/choose_fastest_dataset_serialization_test.py

diff --git a/tensorflow/core/api_def/base_api/api_def_ExperimentalChooseFastestDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_ExperimentalChooseFastestDataset.pbtxt
new file mode 100644
index 0000000000..7aa7a59bb6
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_ExperimentalChooseFastestDataset.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "ExperimentalChooseFastestDataset"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/kernels/data/experimental/BUILD b/tensorflow/core/kernels/data/experimental/BUILD
index 583fce6d89..29756f0cfd 100644
--- a/tensorflow/core/kernels/data/experimental/BUILD
+++ b/tensorflow/core/kernels/data/experimental/BUILD
@@ -143,6 +143,18 @@ tf_kernel_library(
     ],
 )
 
+tf_kernel_library(
+    name = "choose_fastest_dataset_op",
+    srcs = ["choose_fastest_dataset_op.cc"],
+    deps = [
+        "//tensorflow/core:experimental_dataset_ops_op_lib",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+        "//tensorflow/core/kernels/data:dataset",
+    ],
+)
+
 tf_kernel_library(
     name = "non_serializable_dataset_op",
     srcs = ["non_serializable_dataset_op.cc"],
@@ -343,6 +355,7 @@ tf_kernel_library(
     name = "dataset_kernels",
     deps = [
         ":assert_next_dataset_op",
+        ":choose_fastest_dataset_op",
         ":csv_dataset_op",
         ":dense_to_sparse_batch_dataset_op",
         ":directed_interleave_dataset_op",
diff --git a/tensorflow/core/kernels/data/experimental/choose_fastest_dataset_op.cc b/tensorflow/core/kernels/data/experimental/choose_fastest_dataset_op.cc
new file mode 100644
index 0000000000..f66d5d9955
--- /dev/null
+++ b/tensorflow/core/kernels/data/experimental/choose_fastest_dataset_op.cc
@@ -0,0 +1,346 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/framework/common_shape_fns.h"
+#include "tensorflow/core/framework/dataset.h"
+#include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/histogram/histogram.h"
+
+namespace tensorflow {
+namespace data {
+namespace {
+
+static const double kPercentile = 90.0;
+
+class ChooseFastestDatasetOp : public DatasetOpKernel {
+ public:
+  explicit ChooseFastestDatasetOp(OpKernelConstruction* ctx)
+      : DatasetOpKernel(ctx) {
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("num_experiments", &num_experiments_));
+  }
+
+  void MakeDataset(OpKernelContext* ctx, DatasetBase** output) override {
+    OpInputList input_list;
+    OP_REQUIRES_OK(ctx, ctx->input_list("input_datasets", &input_list));
+    OP_REQUIRES(
+        ctx, input_list.size() > 1,
+        errors::InvalidArgument(
+            "ChooseFastestDataset must have at least two input datasets."));
+
+    std::vector<DatasetBase*> inputs;
+    inputs.reserve(input_list.size());
+    for (const auto& tensor : input_list) {
+      DatasetBase* input;
+      OP_REQUIRES_OK(ctx, GetDatasetFromVariantTensor(tensor, &input));
+      inputs.push_back(input);
+    }
+
+    const DataTypeVector& output_types = inputs[0]->output_dtypes();
+    for (size_t i = 1, num_inputs = inputs.size(); i < num_inputs; ++i) {
+      OP_REQUIRES(ctx, inputs[i]->output_dtypes() == output_types,
+                  errors::InvalidArgument(
+                      "All inputs to ChooseFastestDataset "
+                      "must have the same output types. Input ",
+                      i, " has output types: ",
+                      DataTypeVectorString(inputs[i]->output_dtypes()),
+                      ", while all prior inputs have types: ",
+                      DataTypeVectorString(output_types), "."));
+    }
+
+    std::vector<PartialTensorShape> output_shapes = inputs[0]->output_shapes();
+    // Merge the output shapes of all the input datasets, returning an
+    // error if any of them are incompatible.
+    for (size_t i = 1, num_inputs = inputs.size(); i < num_inputs; ++i) {
+      OP_REQUIRES(
+          ctx, inputs[i]->output_shapes().size() == output_shapes.size(),
+          errors::InvalidArgument(
+              "All inputs to ChooseFastestDataset must have compatible outputs."
+              " Input ",
+              i, " has ", inputs[i]->output_shapes().size(),
+              " components, while all prior inputs have ", output_shapes.size(),
+              " components."));
+      for (size_t j = 0, num_components = output_shapes.size();
+           j < num_components; ++j) {
+        PartialTensorShape result;
+        OP_REQUIRES(
+            ctx,
+            output_shapes[j]
+                .MergeWith(inputs[i]->output_shapes().at(j), &result)
+                .ok(),
+            errors::InvalidArgument(
+                "All inputs to ChooseFastestDataset must have "
+                "compatible output shapes. Component ",
+                j, " of input ", i, " has shape: ",
+                inputs[i]->output_shapes().at(j), ", while components ", j,
+                " of all prior inputs have shape: ", output_shapes[j], "."));
+        output_shapes[j] = std::move(result);
+      }
+    }
+
+    int64 cardinality = inputs[0]->Cardinality();
+    for (size_t i = 1, num_inputs = inputs.size(); i < num_inputs; ++i) {
+      if (cardinality == kUnknownCardinality) {
+        cardinality = inputs[i]->Cardinality();
+      } else {
+        OP_REQUIRES(
+            ctx,
+            inputs[i]->Cardinality() == cardinality ||
+                inputs[i]->Cardinality() == kUnknownCardinality,
+            errors::InvalidArgument(
+                "All inputs to ChooseFastestDataset must have compatible "
+                "cardinalities. Input ",
+                i, " has cardinality: ", inputs[i]->Cardinality(),
+                ", while all prior inputs have cardinality: ", cardinality,
+                "."));
+      }
+    }
+    *output = new Dataset(ctx, std::move(inputs), std::move(output_shapes),
+                          cardinality, num_experiments_);
+  }
+
+ private:
+  class Dataset : public DatasetBase {
+   public:
+    Dataset(OpKernelContext* ctx, std::vector<DatasetBase*> inputs,
+            std::vector<PartialTensorShape> output_shapes, int64 cardinality,
+            int64 num_experiments)
+        : DatasetBase(DatasetContext(ctx)),
+          inputs_(std::move(inputs)),
+          output_shapes_(std::move(output_shapes)),
+          cardinality_(cardinality),
+          num_experiments_(num_experiments) {
+      for (auto input : inputs_) {
+        input->Ref();
+      }
+    }
+
+    ~Dataset() override {
+      for (auto input : inputs_) {
+        input->Unref();
+      }
+    }
+
+    std::unique_ptr<IteratorBase> MakeIteratorInternal(
+        const string& prefix) const override {
+      return absl::make_unique<ChooseFastestIterator>(
+          ChooseFastestIterator::Params{
+              this, strings::StrCat(prefix, "::ChooseFastest")});
+    }
+
+    const DataTypeVector& output_dtypes() const override {
+      return inputs_[0]->output_dtypes();
+    }
+
+    const std::vector<PartialTensorShape>& output_shapes() const override {
+      return output_shapes_;
+    }
+
+    string DebugString() const override {
+      return "ChooseFastestDatasetOp::Dataset";
+    }
+
+    int64 Cardinality() const override { return cardinality_; }
+
+   protected:
+    Status AsGraphDefInternal(SerializationContext* ctx,
+                              DatasetGraphDefBuilder* b,
+                              Node** output) const override {
+      std::vector<Node*> input_nodes;
+      input_nodes.reserve(inputs_.size());
+      for (const auto& input : inputs_) {
+        Node* input_node;
+        TF_RETURN_IF_ERROR(b->AddInputDataset(ctx, input, &input_node));
+        input_nodes.push_back(input_node);
+      }
+      AttrValue num_experiments_attr;
+      b->BuildAttrValue(num_experiments_, &num_experiments_attr);
+      return b->AddDataset(
+          this, {}, {std::make_pair(0, input_nodes)},
+          {std::make_pair("num_experiments", std::move(num_experiments_attr))},
+          output);
+    }
+
+   private:
+    class ChooseFastestIterator : public DatasetIterator<Dataset> {
+     public:
+      explicit ChooseFastestIterator(const Params& params)
+          : DatasetIterator<Dataset>(params),
+            histograms_(dataset()->inputs_.size()) {}
+
+      Status Initialize(IteratorContext* ctx) override {
+        mutex_lock l(mu_);
+        input_impls_.resize(dataset()->inputs_.size());
+        for (size_t i = 0, num_inputs = dataset()->inputs_.size();
+             i < num_inputs; ++i) {
+          TF_RETURN_IF_ERROR(dataset()->inputs_[i]->MakeIterator(
+              ctx, strings::StrCat(prefix(), "_", i), &input_impls_[i]));
+        }
+        return Status::OK();
+      }
+
+      Status GetNextInternal(IteratorContext* ctx,
+                             std::vector<Tensor>* out_tensors,
+                             bool* end_of_sequence) override {
+        mutex_lock l(mu_);
+
+        // The first num_experiments_ iterations, we fire up a thread for
+        // each input that calls its GetNext function and records the time
+        // taken. We only return when all the threads have completed.
+        if (experiment_counter_ < dataset()->num_experiments_) {
+          experiment_counter_++;
+          std::vector<ThreadInfo> threads = StartThreads(ctx);
+          for (const auto& thread : threads) {
+            thread.result->notification.WaitForNotification();
+          }
+
+          *out_tensors = std::move(threads[0].result->out_tensors);
+          *end_of_sequence = threads[0].result->end_of_sequence;
+
+          if (experiment_counter_ == dataset()->num_experiments_) {
+            SelectFastestInputIndex();
+          }
+          return threads[0].result->status;
+        }
+        return input_impls_[fastest_index_]->GetNext(ctx, out_tensors,
+                                                     end_of_sequence);
+      }
+
+     protected:
+      std::shared_ptr<model::Node> CreateNode(
+          IteratorContext* ctx, model::Node::Args args) const override {
+        return model::MakeKnownRatioNode(std::move(args), /*ratio=*/1);
+      }
+
+      // TODO(rachelim): Save and restore histogram state as well. Currently,
+      // if an iterator is saved and restored, the histograms start recording
+      // from scratch.
+      Status SaveInternal(IteratorStateWriter* writer) override {
+        mutex_lock l(mu_);
+        if (input_impls_.empty()) {
+          TF_RETURN_IF_ERROR(
+              writer->WriteScalar(full_name("input_impls_empty"), ""));
+        } else {
+          for (auto& input_impl : input_impls_) {
+            TF_RETURN_IF_ERROR(SaveInput(writer, input_impl));
+          }
+        }
+        TF_RETURN_IF_ERROR(writer->WriteScalar(full_name("experiment_counter"),
+                                               experiment_counter_));
+        TF_RETURN_IF_ERROR(
+            writer->WriteScalar(full_name("fastest_index"), fastest_index_));
+        return Status::OK();
+      }
+
+      Status RestoreInternal(IteratorContext* ctx,
+                             IteratorStateReader* reader) override {
+        mutex_lock l(mu_);
+        if (reader->Contains(full_name("input_impls_empty"))) {
+          input_impls_.clear();
+        } else {
+          DCHECK_EQ(input_impls_.size(), dataset()->inputs_.size());
+          for (auto& input_impl : input_impls_) {
+            TF_RETURN_IF_ERROR(RestoreInput(ctx, reader, input_impl));
+          }
+        }
+        TF_RETURN_IF_ERROR(reader->ReadScalar(full_name("experiment_counter"),
+                                              &experiment_counter_));
+        TF_RETURN_IF_ERROR(
+            reader->ReadScalar(full_name("fastest_index"), &fastest_index_));
+        return Status::OK();
+      }
+
+     private:
+      struct InvocationResult {
+        Notification notification;
+        Status status;
+        bool end_of_sequence;
+        std::vector<Tensor> out_tensors;
+      };
+
+      struct ThreadInfo {
+        std::unique_ptr<InvocationResult> result;
+        std::unique_ptr<Thread> thread;
+      };
+
+      std::vector<std::unique_ptr<IteratorBase>> input_impls_;
+      // For tracking the time taken for each input's iterations.
+      std::vector<histogram::Histogram> histograms_;
+
+      mutex mu_;
+      int64 experiment_counter_ GUARDED_BY(mu_) = 0;
+      int64 fastest_index_ = -1;
+
+      std::vector<ThreadInfo> StartThreads(IteratorContext* ctx)
+          EXCLUSIVE_LOCKS_REQUIRED(mu_) {
+        std::vector<ThreadInfo> threads(dataset()->inputs_.size());
+        for (size_t i = 0, num_inputs = dataset()->inputs_.size();
+             i < num_inputs; ++i) {
+          threads[i].result = absl::make_unique<InvocationResult>();
+          threads[i].thread.reset(ctx->env()->StartThread(
+              {}, strings::StrCat("tf_data_merge_", i),
+              std::bind(&ChooseFastestIterator::RunnerThread, this, ctx,
+                        threads[i].result.get(), i)));
+        }
+        return threads;
+      }
+
+      void RunnerThread(IteratorContext* ctx, InvocationResult* result, int i) {
+        int64 start = Env::Default()->NowNanos();
+        Status s = input_impls_[i]->GetNext(ctx, &result->out_tensors,
+                                            &result->end_of_sequence);
+        histograms_[i].Add(
+            static_cast<double>(Env::Default()->NowNanos() - start));
+
+        result->status = s;
+        result->notification.Notify();
+      }
+
+      // Select the fastest input to use based on the histograms of timings
+      // of the completed threads. The input with the best 90th percentile
+      // iteration time is selected.
+      void SelectFastestInputIndex() EXCLUSIVE_LOCKS_REQUIRED(mu_) {
+        fastest_index_ = 0;
+
+        double best_percentile = histograms_[0].Percentile(kPercentile);
+        for (size_t i = 1, num_inputs = histograms_.size(); i < num_inputs;
+             ++i) {
+          double percentile = histograms_[i].Percentile(kPercentile);
+          if (percentile <= best_percentile) {
+            best_percentile = percentile;
+            fastest_index_ = i;
+          }
+        }
+      }
+    };  // class Iterator
+
+    const std::vector<DatasetBase*> inputs_;
+    const std::vector<PartialTensorShape> output_shapes_;
+    const int64 cardinality_;
+    const int64 num_experiments_;
+  };  // class Dataset
+
+  int64 num_experiments_;
+};  // class ChooseFastestDatasetOp
+
+// Register the kernel implementation for ChooseFastestDataset.
+REGISTER_KERNEL_BUILDER(
+    Name("ExperimentalChooseFastestDataset").Device(DEVICE_CPU),
+    ChooseFastestDatasetOp);
+
+}  // namespace
+}  // namespace data
+}  // namespace tensorflow
diff --git a/tensorflow/core/ops/experimental_dataset_ops.cc b/tensorflow/core/ops/experimental_dataset_ops.cc
index f904e2536d..d70b1650ff 100644
--- a/tensorflow/core/ops/experimental_dataset_ops.cc
+++ b/tensorflow/core/ops/experimental_dataset_ops.cc
@@ -453,6 +453,13 @@ REGISTER_OP("ExperimentalLMDBDataset")
                       // stateful to inhibit constant folding.
     .SetShapeFn(shape_inference::ScalarShape);
 
+REGISTER_OP("ExperimentalChooseFastestDataset")
+    .Input("input_datasets: N * variant")
+    .Output("handle: variant")
+    .Attr("N: int >= 2")
+    .Attr("num_experiments: int")
+    .SetShapeFn(shape_inference::ScalarShape);
+
 REGISTER_OP("ExperimentalIdentityIndexedDataset")
     .Input("size: uint64")
     .Output("handle: variant")
diff --git a/tensorflow/python/data/experimental/benchmarks/BUILD b/tensorflow/python/data/experimental/benchmarks/BUILD
index 2443f7edad..4f2117ec9b 100644
--- a/tensorflow/python/data/experimental/benchmarks/BUILD
+++ b/tensorflow/python/data/experimental/benchmarks/BUILD
@@ -110,6 +110,20 @@ py_test(
     ],
 )
 
+py_test(
+    name = "choose_fastest_benchmark",
+    srcs = ["choose_fastest_benchmark.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:session",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//third_party/py/numpy",
+    ],
+)
+
 py_test(
     name = "optimize_benchmark",
     srcs = ["optimize_benchmark.py"],
diff --git a/tensorflow/python/data/experimental/benchmarks/choose_fastest_benchmark.py b/tensorflow/python/data/experimental/benchmarks/choose_fastest_benchmark.py
new file mode 100644
index 0000000000..4a5a264c6f
--- /dev/null
+++ b/tensorflow/python/data/experimental/benchmarks/choose_fastest_benchmark.py
@@ -0,0 +1,105 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Benchmarks for static optimizations."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import time
+
+import numpy as np
+
+from tensorflow.python.client import session
+from tensorflow.python.data.experimental.ops import optimization
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.platform import test
+
+
+# TODO(b/119837791): Add eager benchmarks too.
+class ChooseFastestBenchmark(test.Benchmark):
+  """Benchmarks for static optimizations."""
+
+  def benchmarkChooseFastest(self):
+
+    dataset = dataset_ops.Dataset.range(1000**2).repeat()
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    dataset = dataset.with_options(options)
+    map_batch_dataset = dataset.map(lambda x: x + 1).batch(100)
+    batch_map_dataset = dataset.batch(100).map(lambda x: x + 1)
+
+    merge_dataset = optimization._ChooseFastestDataset(  # pylint: disable=protected-access
+        [batch_map_dataset, map_batch_dataset])
+    self._benchmark(map_batch_dataset, "map_batch_dataset")
+    self._benchmark(batch_map_dataset, "batch_map_dataset")
+    self._benchmark(merge_dataset, "merge_dataset")
+
+  def benchmarkChooseFastestFirstNIterations(self):
+
+    dataset = dataset_ops.Dataset.range(1000**2).repeat()
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    dataset = dataset.with_options(options)
+    map_batch_dataset = dataset.map(lambda x: x + 1).batch(100)
+    batch_map_dataset = dataset.batch(100).map(lambda x: x + 1)
+
+    merge_dataset = optimization._ChooseFastestDataset(  # pylint: disable=protected-access
+        [batch_map_dataset, map_batch_dataset])
+
+    self._benchmarkFirstN(map_batch_dataset, "map_batch_dataset")
+    self._benchmarkFirstN(batch_map_dataset, "batch_map_dataset")
+    self._benchmarkFirstN(merge_dataset, "merge_dataset")
+
+  def _benchmarkFirstN(self, dataset, name):
+    n = 10  # The default num_experiments for ChooseFastestDataset
+    iterator = dataset_ops.make_one_shot_iterator(dataset)
+    next_element = iterator.get_next()
+
+    deltas = []
+    for _ in range(100):
+      with session.Session() as sess:
+        start = time.time()
+        for _ in range(n):
+          sess.run(next_element.op)
+        end = time.time()
+        deltas.append(end - start)
+    median_wall_time = np.median(deltas) / n
+    self.report_benchmark(
+        iters=n, wall_time=median_wall_time, name=name + "_first_%d" % n)
+
+  def _benchmark(self, dataset, name):
+    iterator = dataset_ops.make_one_shot_iterator(dataset)
+    next_element = iterator.get_next()
+
+    with session.Session() as sess:
+      # Run 10 steps to warm up the session caches before taking the first
+      # measurement. Additionally, 10 is the default num_experiments for
+      # ChooseFastestDataset.
+      for _ in range(10):
+        sess.run(next_element.op)
+      deltas = []
+      for _ in range(50):
+        start = time.time()
+        for _ in range(50):
+          sess.run(next_element.op)
+        end = time.time()
+        deltas.append(end - start)
+
+      median_wall_time = np.median(deltas) / 100
+      self.report_benchmark(iters=100, wall_time=median_wall_time, name=name)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD b/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD
index 93a9aca1f2..703d3350db 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD
@@ -242,6 +242,26 @@ py_test(
     ],
 )
 
+py_test(
+    name = "choose_fastest_dataset_test",
+    size = "small",
+    srcs = ["choose_fastest_dataset_test.py"],
+    srcs_version = "PY2AND3",
+    tags = [
+        "no_oss",
+        "no_pip",
+        "no_windows",
+    ],
+    deps = [
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:errors",
+        "//tensorflow/python/data/experimental/ops:optimization",
+        "//tensorflow/python/data/kernel_tests:test_base",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "@absl_py//absl/testing:parameterized",
+    ],
+)
+
 py_test(
     name = "model_dataset_test",
     size = "medium",
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/choose_fastest_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/choose_fastest_dataset_test.py
new file mode 100644
index 0000000000..ec7a85ae11
--- /dev/null
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/choose_fastest_dataset_test.py
@@ -0,0 +1,85 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for `tf.data.experimental._ChooseFastestDataset`."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from absl.testing import parameterized
+
+from tensorflow.python.data.experimental.ops import optimization
+from tensorflow.python.data.kernel_tests import test_base
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.eager import context
+from tensorflow.python.framework import errors
+from tensorflow.python.framework import test_util
+from tensorflow.python.platform import test
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class ChooseFastestDatasetTest(test_base.DatasetTestBase,
+                               parameterized.TestCase):
+
+  def testChooseFastestSimple(self):
+    dataset = dataset_ops.Dataset.from_tensor_slices([0, 1, 2, 3, 4])
+    merge = optimization._ChooseFastestDataset([dataset, dataset])
+    self.assertDatasetProduces(
+        merge,
+        expected_output=[0, 1, 2, 3, 4],
+        expected_shapes=dataset.output_shapes)
+
+  def testChooseFastestManyInputs(self):
+    dataset = dataset_ops.Dataset.from_tensor_slices([0, 1, 2, 3, 4])
+    merge = optimization._ChooseFastestDataset([dataset for _ in range(5)])
+    self.assertDatasetProduces(
+        merge,
+        expected_output=[0, 1, 2, 3, 4],
+        expected_shapes=dataset.output_shapes)
+
+  def testChooseFastest(self):
+    dataset = dataset_ops.Dataset.range(600)
+    f = lambda x: 2 * x
+    dataset_a = dataset.batch(50).map(f)
+    dataset_b = dataset.map(f).batch(50)
+    merge = optimization._ChooseFastestDataset([dataset_a, dataset_b])
+    self.assertDatasetProduces(
+        merge,
+        expected_output=[
+            [i * 2 for i in range(j * 50, (j + 1) * 50)] for j in range(12)
+        ],
+        expected_shapes=dataset_a.output_shapes)
+
+  @parameterized.named_parameters(
+      ("Shapes", [0], [[1, 2, 3]], "must have compatible output shapes."),
+      ("Types", [0], [0.0], "must have the same output types."),
+      ("NumComponents", [0], ([0], [1]), "must have the same output types."),
+      ("Cardinality", [1, 2, 3], [1], "must have compatible cardinalities."))
+  def testChooseFastestErrorWithIncompatibleInput(self, slices_a, slices_b,
+                                                  error_msg):
+    dataset_a = dataset_ops.Dataset.from_tensor_slices(slices_a)
+    dataset_b = dataset_ops.Dataset.from_tensor_slices(slices_b)
+
+    # The error is raised at dataset creation time.
+    if context.executing_eagerly():
+      with self.assertRaises(errors.InvalidArgumentError):
+        merge = optimization._ChooseFastestDataset([dataset_a, dataset_b])
+    else:
+      merge = optimization._ChooseFastestDataset([dataset_a, dataset_b])
+      self.assertDatasetProduces(
+          merge, expected_error=(errors.InvalidArgumentError, error_msg))
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD b/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD
index 4a2e28f496..3a2ce17228 100644
--- a/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD
+++ b/tensorflow/python/data/experimental/kernel_tests/serialization/BUILD
@@ -93,6 +93,24 @@ py_test(
     ],
 )
 
+py_test(
+    name = "choose_fastest_dataset_serialization_test",
+    size = "small",
+    srcs = ["choose_fastest_dataset_serialization_test.py"],
+    srcs_version = "PY2AND3",
+    tags = [
+        "no_oss",
+        "no_pip",
+        "no_windows",
+    ],
+    deps = [
+        ":dataset_serialization_test_base",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python/data/experimental/ops:optimization",
+        "//tensorflow/python/data/ops:dataset_ops",
+    ],
+)
+
 py_test(
     name = "concatenate_dataset_serialization_test",
     size = "small",
diff --git a/tensorflow/python/data/experimental/kernel_tests/serialization/choose_fastest_dataset_serialization_test.py b/tensorflow/python/data/experimental/kernel_tests/serialization/choose_fastest_dataset_serialization_test.py
new file mode 100644
index 0000000000..936dc22214
--- /dev/null
+++ b/tensorflow/python/data/experimental/kernel_tests/serialization/choose_fastest_dataset_serialization_test.py
@@ -0,0 +1,45 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for the ZipDataset serialization."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.data.experimental.kernel_tests.serialization import dataset_serialization_test_base
+from tensorflow.python.data.experimental.ops import optimization
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.platform import test
+
+
+class ChooseFastestDatasetSerializationTest(
+    dataset_serialization_test_base.DatasetSerializationTestBase):
+
+  def testCore(self):
+    num_outputs = 10
+    batch_size = 2
+
+    def build_ds():
+      dataset = dataset_ops.Dataset.range(num_outputs)
+      map_fn = lambda x: x * 2
+      return optimization._ChooseFastestDataset([  # pylint: disable=protected-access
+          dataset.map(map_fn).batch(batch_size),
+          dataset.batch(batch_size).map(map_fn)
+      ])
+
+    self.run_core_tests(build_ds, None, num_outputs // 2)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/data/experimental/ops/optimization.py b/tensorflow/python/data/experimental/ops/optimization.py
index 22a36646ea..75769b8998 100644
--- a/tensorflow/python/data/experimental/ops/optimization.py
+++ b/tensorflow/python/data/experimental/ops/optimization.py
@@ -129,3 +129,48 @@ class _NonSerializableDataset(dataset_ops.UnaryUnchangedStructureDataset):
             self._input_dataset._variant_tensor,  # pylint: disable=protected-access
             **dataset_ops.flat_structure(self)))
     super(_NonSerializableDataset, self).__init__(input_dataset, variant_tensor)
+
+
+class _ChooseFastestDataset(dataset_ops.DatasetV2):
+  """A `Dataset` that merges two input datasets."""
+
+  def __init__(self, datasets, num_experiments=10):
+    """Chooses the fastest of some input datasets.
+
+    Given input datasets, produces elements as quickly as the fastest of the
+    inputs. Note that this dataset assumes that input datasets have the same
+    elements in the same order, though this is not enforced besides checking
+    that the input datasets have compatible output types, output shapes, and
+    cardinality at runtime. The resulting dataset produces elements that are
+    identical to the input elements, and in the same order.
+
+    Note that the time to first iteration is longer when this dataset is used
+    due to the overhead of dynamically picking the faster dataset. Namely,
+    for the first num_experiments iterations, this dataset will pull from all
+    of its inputs simultaneously in order to determine which input is the
+    fastest. For all subsequent iterations, that input will be used.
+
+    Args:
+      datasets: A list of `Datasets` that all have the same elements in the same
+        order.
+      num_experiments: The number of experiments to run before deciding which
+        dataset is fastest. In each "experiment" iteration, the dataset will
+        call from all its inputs simultaneously, and update its knowledge of
+        which input is the fastest.
+
+    Returns:
+      A `Dataset` that has the same elements the inputs.
+    """
+    self._datasets = list(datasets)
+    variant_tensor = (
+        gen_experimental_dataset_ops.experimental_choose_fastest_dataset(
+            [dataset._variant_tensor for dataset in self._datasets],  # pylint: disable=protected-access
+            num_experiments=num_experiments))
+    super(_ChooseFastestDataset, self).__init__(variant_tensor)
+
+  def _inputs(self):
+    return self._datasets
+
+  @property
+  def _element_structure(self):
+    return self._datasets[0]._element_structure  # pylint: disable=protected-access
-- 
GitLab


From aba19e4143bd12070dcec103c55173cc3c0368b1 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Wed, 16 Jan 2019 10:53:44 -0800
Subject: [PATCH 0816/1095] Add a (failing) randomized test for FusedBatchNorm
 training

I had to add a --tf_xla_reference_device flag so that I could set the reference
device to GPU:0.  The CPU kernel for FusedBatchNorm does not support NCHW.

PiperOrigin-RevId: 229586592
---
 tensorflow/compiler/tests/randomized_tests.cc | 52 +++++++++++++++++--
 1 file changed, 47 insertions(+), 5 deletions(-)

diff --git a/tensorflow/compiler/tests/randomized_tests.cc b/tensorflow/compiler/tests/randomized_tests.cc
index d23fd12516..735014fc89 100644
--- a/tensorflow/compiler/tests/randomized_tests.cc
+++ b/tensorflow/compiler/tests/randomized_tests.cc
@@ -80,6 +80,7 @@ int64 tf_xla_random_seed = 0;
 int32 tf_xla_test_repetitions = 20;
 int64 tf_xla_max_tensor_size = 10000LL;
 string* tf_xla_test_device_ptr;  // initial value set in main()
+string* tf_xla_reference_device_ptr;  // initial value set in main()
 bool tf_xla_test_use_jit = true;
 
 string LocalDeviceToFullDeviceName(const string& device) {
@@ -321,6 +322,9 @@ class OpTest : public ::testing::Test {
   // for use as reduction indices.
   Tensor RandomReductionIndices(int rank);
 
+  // Returns a random bit.
+  bool RandomBool();
+
   struct WindowedSpatialDims {
     Padding padding;
     std::vector<int64> kernel_dims;
@@ -453,6 +457,11 @@ std::vector<int64> OpTest::RandomDims(int min_rank, int max_rank,
   return dims;
 }
 
+bool OpTest::RandomBool() {
+  std::bernoulli_distribution d(0.5);
+  return d(generator());
+}
+
 Tensor OpTest::RandomTensor(DataType dtype, bool needs_unique_values,
                             absl::Span<const int64> shape) {
   Tensor tensor(dtype, TensorShape(shape));
@@ -829,8 +838,8 @@ OpTest::TestResult OpTest::ExpectTfAndXlaOutputsAreClose(
     VLOG(1) << "Input: " << input_tensors.back().DebugString();
   }
 
-  string cpu_device =
-      LocalDeviceToFullDeviceName(absl::StrCat(DEVICE_CPU, ":0"));
+  string reference_device =
+      LocalDeviceToFullDeviceName(*tf_xla_reference_device_ptr);
   string test_device = LocalDeviceToFullDeviceName(*tf_xla_test_device_ptr);
 
   DeviceNameUtils::ParsedName parsed_name;
@@ -845,9 +854,9 @@ OpTest::TestResult OpTest::ExpectTfAndXlaOutputsAreClose(
   std::vector<string> expected_inputs, test_inputs;
   std::vector<string> expected_fetches, test_fetches;
   Status status = builder.BuildGraph(
-      absl::StrCat("test", num_tests_, "_expected"), cpu_device,
-      /* use_jit= */ false, &graph, /* test_node_def= */ nullptr,
-      &expected_inputs, &expected_fetches);
+      absl::StrCat("test", num_tests_, "_expected"), reference_device,
+      /*use_jit=*/false, &graph, /*test_node_def=*/nullptr, &expected_inputs,
+      &expected_fetches);
   if (!status.ok()) {
     LOG(ERROR) << "Expected graph construction failed: " << status;
     return kFatalError;
@@ -3346,11 +3355,41 @@ TEST_F(OpTest, ZerosLike) {
   });
 }
 
+// Example failing run:
+//   --tf_xla_reference_device=GPU:0
+//   --tf_xla_test_use_jit=true --tf_xla_test_device=GPU:0
+//   --tf_xla_test_repetitions=2
+//   --gunit_filter='OpTest.FusedBatchNormTraining'
+//   --tf_xla_random_seed=2838146746
+TEST_F(OpTest, FusedBatchNormTraining) {
+  bool is_nhwc = RandomBool();
+  std::vector<int64> x_dims = RandomDims(/*min_rank=*/4, /*max_rank=*/4,
+                                         /*min_size=*/5, /*max_size=*/20);
+  std::vector<int64> scale_dims = {x_dims[is_nhwc ? 3 : 1]};
+  std::vector<int64> offset_dims = {x_dims[is_nhwc ? 3 : 1]};
+  std::vector<int64> mean_dims = {0};
+  std::vector<int64> variance_dims = {0};
+  DataType type = DT_FLOAT;
+  Repeatedly([&] {
+    return ExpectTfAndXlaOutputsAreClose(
+        OpTestBuilder("FusedBatchNorm")
+            .RandomInput(type, x_dims)
+            .RandomInput(type, scale_dims)
+            .RandomInput(type, offset_dims)
+            .RandomInput(type, mean_dims)
+            .RandomInput(type, variance_dims)
+            .Attr("T", type)
+            .Attr("data_format", is_nhwc ? "NHWC" : "NCHW")
+            .Attr("epsilon", static_cast<float>(1.001e-05))
+            .Attr("is_training", true));
+  });
+}
 }  // anonymous namespace
 }  // namespace tensorflow
 
 int main(int argc, char** argv) {
   tensorflow::tf_xla_test_device_ptr = new tensorflow::string("GPU:0");
+  tensorflow::tf_xla_reference_device_ptr = new tensorflow::string("CPU:0");
   std::vector<tensorflow::Flag> flag_list = {
       tensorflow::Flag(
           "tf_xla_random_seed", &tensorflow::tf_xla_random_seed,
@@ -3366,6 +3405,9 @@ int main(int argc, char** argv) {
                        "Maximum number of elements for random input tensors."),
       tensorflow::Flag("tf_xla_test_device", tensorflow::tf_xla_test_device_ptr,
                        "Tensorflow device type to use for test"),
+      tensorflow::Flag("tf_xla_reference_device",
+                       tensorflow::tf_xla_reference_device_ptr,
+                       "Tensorflow device type to use for reference"),
       tensorflow::Flag("tf_xla_test_use_jit", &tensorflow::tf_xla_test_use_jit,
                        "Use JIT compilation for the operator under test"),
   };
-- 
GitLab


From 2eaf93a349fe2d4d4a2d89ea723302d2b34ec98e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 16 Jan 2019 10:54:45 -0800
Subject: [PATCH 0817/1095] Replace calls to deprecated googletest macros
 *TEST_CASE() with *TEST_SUITE()

PiperOrigin-RevId: 229586776
---
 .../compiler/tf2xla/literal_util_test.cc      |  2 +-
 .../partial_run_mgr_test.cc                   |  2 +-
 tensorflow/core/framework/bfloat16_test.cc    |  2 +-
 tensorflow/core/framework/model_test.cc       | 17 ++++----
 tensorflow/core/kernels/conv_ops_test.cc      | 42 +++++++++----------
 tensorflow/core/lib/gtl/int_type_test.cc      |  2 +-
 tensorflow/core/nccl/nccl_manager_test.cc     |  2 +-
 .../bidirectional_sequence_lstm_test.cc       |  2 +-
 tensorflow/lite/kernels/conv_test.cc          |  2 +-
 .../lite/kernels/depthwise_conv_test.cc       |  4 +-
 .../lite/kernels/fully_connected_test.cc      |  4 +-
 tensorflow/lite/kernels/reshape_test.cc       |  6 +--
 .../lite/kernels/transpose_conv_test.cc       |  2 +-
 tensorflow/lite/models/speech_test.cc         |  8 ++--
 .../lite/toco/import_tensorflow_test.cc       |  8 ++--
 tensorflow/lite/toco/tooling_util_test.cc     |  4 +-
 16 files changed, 55 insertions(+), 54 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/literal_util_test.cc b/tensorflow/compiler/tf2xla/literal_util_test.cc
index 15f4c38da2..44bccfe647 100644
--- a/tensorflow/compiler/tf2xla/literal_util_test.cc
+++ b/tensorflow/compiler/tf2xla/literal_util_test.cc
@@ -49,7 +49,7 @@ using Types =
                      std::pair<int16, qint16>, std::pair<uint16, quint16>,
                      std::pair<int32, qint32>>;
 
-TYPED_TEST_CASE(LiteralUtilTest, Types);
+TYPED_TEST_SUITE(LiteralUtilTest, Types);
 
 TYPED_TEST(LiteralUtilTest, LiteralToQuantizedHostTensor) {
   using int_type = typename TypeParam::first_type;
diff --git a/tensorflow/core/distributed_runtime/partial_run_mgr_test.cc b/tensorflow/core/distributed_runtime/partial_run_mgr_test.cc
index 5f7c0cb3ca..a2b799c3e4 100644
--- a/tensorflow/core/distributed_runtime/partial_run_mgr_test.cc
+++ b/tensorflow/core/distributed_runtime/partial_run_mgr_test.cc
@@ -139,7 +139,7 @@ TEST_P(StatusPropagationTest, PartialRunDoneFirst) {
 // ExecutorDone and PartialRunDone.
 Status ExecutorError() { return errors::Internal("executor error"); }
 Status PartialRunError() { return errors::Internal("partial run error"); }
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     PartialRunMgr, StatusPropagationTest,
     ::testing::Values(
         StatusTestParam{Status::OK(), Status::OK(), Status::OK()},
diff --git a/tensorflow/core/framework/bfloat16_test.cc b/tensorflow/core/framework/bfloat16_test.cc
index ce97085494..7da1727e47 100644
--- a/tensorflow/core/framework/bfloat16_test.cc
+++ b/tensorflow/core/framework/bfloat16_test.cc
@@ -75,7 +75,7 @@ TEST_P(Bfloat16Test, TruncateTest) {
   EXPECT_EQ(GetParam().expected_rounding, float(rounded));
 }
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     Bfloat16Test_Instantiation, Bfloat16Test,
     ::testing::Values(
         Bfloat16TestParam{
diff --git a/tensorflow/core/framework/model_test.cc b/tensorflow/core/framework/model_test.cc
index 013f1e61c8..1d7f407e18 100644
--- a/tensorflow/core/framework/model_test.cc
+++ b/tensorflow/core/framework/model_test.cc
@@ -83,9 +83,10 @@ TEST_P(AsyncInterleaveManyTest, Model) {
   EXPECT_GE(async_interleave_many->OutputTime(&input_times), 0);
 }
 
-INSTANTIATE_TEST_CASE_P(Test, AsyncInterleaveManyTest,
-                        ::testing::Combine(::testing::Values(1, 2),
-                                           ::testing::Values(0, 50, 100, 200)));
+INSTANTIATE_TEST_SUITE_P(Test, AsyncInterleaveManyTest,
+                         ::testing::Combine(::testing::Values(1, 2),
+                                            ::testing::Values(0, 50, 100,
+                                                              200)));
 
 class AsyncKnownRatioTest
     : public ::testing::TestWithParam<std::tuple<int64, int64, int64>> {};
@@ -156,10 +157,10 @@ TEST_P(AsyncKnownRatioTest, Model) {
   EXPECT_GE(async_known_many->OutputTime(&input_times), 0);
 }
 
-INSTANTIATE_TEST_CASE_P(Test, AsyncKnownRatioTest,
-                        ::testing::Combine(::testing::Values(1, 2, 4, 8),
-                                           ::testing::Values(0, 50, 100, 200),
-                                           ::testing::Values(0, 1, 2, 4)));
+INSTANTIATE_TEST_SUITE_P(Test, AsyncKnownRatioTest,
+                         ::testing::Combine(::testing::Values(1, 2, 4, 8),
+                                            ::testing::Values(0, 50, 100, 200),
+                                            ::testing::Values(0, 1, 2, 4)));
 
 TEST(InterleaveManyTest, Model) {
   std::shared_ptr<Node> interleave_many =
@@ -245,7 +246,7 @@ TEST_P(KnownRatioTest, Model) {
             num_inputs_per_output * (50 + 100) + 64);
 }
 
-INSTANTIATE_TEST_CASE_P(Test, KnownRatioTest, ::testing::Values(0, 1, 2, 4));
+INSTANTIATE_TEST_SUITE_P(Test, KnownRatioTest, ::testing::Values(0, 1, 2, 4));
 
 TEST(SourceTest, Model) {
   std::shared_ptr<Node> source = model::MakeSourceNode({0, "source", nullptr});
diff --git a/tensorflow/core/kernels/conv_ops_test.cc b/tensorflow/core/kernels/conv_ops_test.cc
index fc93915e16..a4cd67804e 100644
--- a/tensorflow/core/kernels/conv_ops_test.cc
+++ b/tensorflow/core/kernels/conv_ops_test.cc
@@ -952,8 +952,8 @@ class FusedConv2DWithBiasOpTest : public FusedConv2DOpTest<T> {};
 template <typename T>
 class FusedConv2DWithBatchNormOpTest : public FusedConv2DOpTest<T> {};
 
-TYPED_TEST_CASE_P(FusedConv2DWithBiasOpTest);
-TYPED_TEST_CASE_P(FusedConv2DWithBatchNormOpTest);
+TYPED_TEST_SUITE_P(FusedConv2DWithBiasOpTest);
+TYPED_TEST_SUITE_P(FusedConv2DWithBatchNormOpTest);
 
 // -------------------------------------------------------------------------- //
 // Conv2D + BiasAdd + {Relu}                                                  //
@@ -1035,29 +1035,29 @@ TYPED_TEST_P(FusedConv2DWithBatchNormOpTest, SpatialConvolutionAndRelu) {
   this->VerifyConv2DWithBatchNormAndRelu(filter_size, filter_count);
 }
 
-REGISTER_TYPED_TEST_CASE_P(FusedConv2DWithBiasOpTest,    //
-                           OneByOneConvolution,          //
-                           ImageSizeConvolution,         //
-                           SpatialConvolution,           //
-                           OneByOneConvolutionAndRelu,   //
-                           ImageSizeConvolutionAndRelu,  //
-                           SpatialConvolutionAndRelu);
-
-REGISTER_TYPED_TEST_CASE_P(FusedConv2DWithBatchNormOpTest,  //
-                           OneByOneConvolution,             //
-                           ImageSizeConvolution,            //
-                           SpatialConvolution,              //
-                           OneByOneConvolutionAndRelu,      //
-                           ImageSizeConvolutionAndRelu,     //
-                           SpatialConvolutionAndRelu);
+REGISTER_TYPED_TEST_SUITE_P(FusedConv2DWithBiasOpTest,    //
+                            OneByOneConvolution,          //
+                            ImageSizeConvolution,         //
+                            SpatialConvolution,           //
+                            OneByOneConvolutionAndRelu,   //
+                            ImageSizeConvolutionAndRelu,  //
+                            SpatialConvolutionAndRelu);
+
+REGISTER_TYPED_TEST_SUITE_P(FusedConv2DWithBatchNormOpTest,  //
+                            OneByOneConvolution,             //
+                            ImageSizeConvolution,            //
+                            SpatialConvolution,              //
+                            OneByOneConvolutionAndRelu,      //
+                            ImageSizeConvolutionAndRelu,     //
+                            SpatialConvolutionAndRelu);
 
 using FusedBiasAddDataTypes = ::testing::Types<float, double>;
-INSTANTIATE_TYPED_TEST_CASE_P(Test, FusedConv2DWithBiasOpTest,
-                              FusedBiasAddDataTypes);
+INSTANTIATE_TYPED_TEST_SUITE_P(Test, FusedConv2DWithBiasOpTest,
+                               FusedBiasAddDataTypes);
 
 using FusedBatchNormDataTypes = ::testing::Types<float>;
-INSTANTIATE_TYPED_TEST_CASE_P(Test, FusedConv2DWithBatchNormOpTest,
-                              FusedBatchNormDataTypes);
+INSTANTIATE_TYPED_TEST_SUITE_P(Test, FusedConv2DWithBatchNormOpTest,
+                               FusedBatchNormDataTypes);
 
 ////////////////////////////////////////////////////////////////////////////////
 // Performance benchmarks for the FusedConv2DWithBiasOp.                      //
diff --git a/tensorflow/core/lib/gtl/int_type_test.cc b/tensorflow/core/lib/gtl/int_type_test.cc
index 61d364017c..89d2d0e8fe 100644
--- a/tensorflow/core/lib/gtl/int_type_test.cc
+++ b/tensorflow/core/lib/gtl/int_type_test.cc
@@ -45,7 +45,7 @@ typedef ::testing::Types<Int8_IT, UInt8_IT, Int16_IT, UInt16_IT, Int32_IT,
                          Int64_IT, UInt64_IT, Long_IT>
     SupportedIntTypes;
 
-TYPED_TEST_CASE(IntTypeTest, SupportedIntTypes);
+TYPED_TEST_SUITE(IntTypeTest, SupportedIntTypes);
 
 TYPED_TEST(IntTypeTest, TestInitialization) {
   constexpr typename TestFixture::T a;
diff --git a/tensorflow/core/nccl/nccl_manager_test.cc b/tensorflow/core/nccl/nccl_manager_test.cc
index 58bb84ac57..e65af13389 100644
--- a/tensorflow/core/nccl/nccl_manager_test.cc
+++ b/tensorflow/core/nccl/nccl_manager_test.cc
@@ -209,7 +209,7 @@ const Scalar NcclManagerTest<Scalar>::max_ =
 
 // Instantiate tests for float and double.
 using TypeList = ::testing::Types<float, double>;
-TYPED_TEST_CASE(NcclManagerTest, TypeList);
+TYPED_TEST_SUITE(NcclManagerTest, TypeList);
 
 // Test basic sum reduction.
 TYPED_TEST(NcclManagerTest, BasicSumReduction) {
diff --git a/tensorflow/lite/kernels/bidirectional_sequence_lstm_test.cc b/tensorflow/lite/kernels/bidirectional_sequence_lstm_test.cc
index 59ea47a2a2..707f06af83 100644
--- a/tensorflow/lite/kernels/bidirectional_sequence_lstm_test.cc
+++ b/tensorflow/lite/kernels/bidirectional_sequence_lstm_test.cc
@@ -400,7 +400,7 @@ class BidirectionalLSTMOpModel : public SingleOpModel {
 // indicating whether to use quantization or not.
 class LSTMOpTest : public ::testing::TestWithParam<bool> {};
 
-INSTANTIATE_TEST_CASE_P(QuantizationOrNot, LSTMOpTest, ::testing::Bool());
+INSTANTIATE_TEST_SUITE_P(QuantizationOrNot, LSTMOpTest, ::testing::Bool());
 
 TEST_P(LSTMOpTest, BlackBoxTestNoCifgNoPeepholeNoProjectionNoClipping) {
   const int n_batch = 1;
diff --git a/tensorflow/lite/kernels/conv_test.cc b/tensorflow/lite/kernels/conv_test.cc
index 478df3354f..d0350b2fa7 100644
--- a/tensorflow/lite/kernels/conv_test.cc
+++ b/tensorflow/lite/kernels/conv_test.cc
@@ -1069,7 +1069,7 @@ TEST_P(ConvolutionOpTest, DISABLED_PointwiseMultifilterHybrid) {
                   0.0474)));
 }
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     ConvolutionOpTest, ConvolutionOpTest,
     ::testing::ValuesIn(SingleOpTest::GetKernelTags(*kKernelMap)));
 
diff --git a/tensorflow/lite/kernels/depthwise_conv_test.cc b/tensorflow/lite/kernels/depthwise_conv_test.cc
index d924e6f700..75aed4cc4a 100644
--- a/tensorflow/lite/kernels/depthwise_conv_test.cc
+++ b/tensorflow/lite/kernels/depthwise_conv_test.cc
@@ -437,11 +437,11 @@ TEST_P(QuantizedDepthwiseConvolutionOpTest, SimpleDilatedTestPaddingSame) {
               ElementsAreArray({4, 7, 3, 6, 10, 4, 2, 3, 1}));
 }
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     DepthwiseConvolutionOpTest, DepthwiseConvolutionOpTest,
     ::testing::ValuesIn(SingleOpTest::GetKernelTags(*kKernelMap)));
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     QuantizedDepthwiseConvolutionOpTest, QuantizedDepthwiseConvolutionOpTest,
     ::testing::ValuesIn(SingleOpTest::GetKernelTags(*kKernelMap)));
 
diff --git a/tensorflow/lite/kernels/fully_connected_test.cc b/tensorflow/lite/kernels/fully_connected_test.cc
index d1d29fc7e6..03f4ea7143 100644
--- a/tensorflow/lite/kernels/fully_connected_test.cc
+++ b/tensorflow/lite/kernels/fully_connected_test.cc
@@ -725,11 +725,11 @@ TEST_P(QuantizedFullyConnectedOpTest,
               ElementsAre(175, 177, 179, 243, 245, 247));
 }
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     FloatFullyConnectedOpTest, FloatFullyConnectedOpTest,
     ::testing::ValuesIn(SingleOpTest::GetKernelTags(*kKernelMap)));
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     QuantizedFullyConnectedOpTest, QuantizedFullyConnectedOpTest,
     ::testing::ValuesIn(SingleOpTest::GetKernelTags(*kKernelMapNoPie)));
 
diff --git a/tensorflow/lite/kernels/reshape_test.cc b/tensorflow/lite/kernels/reshape_test.cc
index f98f3eb9ae..e2210aeaf0 100644
--- a/tensorflow/lite/kernels/reshape_test.cc
+++ b/tensorflow/lite/kernels/reshape_test.cc
@@ -236,9 +236,9 @@ TEST_P(ReshapeOpTest, Strings) {
               ElementsAreArray({"1", "2", "3", "4", "5", "6", "7", "8"}));
 }
 
-INSTANTIATE_TEST_CASE_P(VariedShapeSpec, ReshapeOpTest,
-                        ::testing::Values(kAsReshapeOption, kAsConstantTensor,
-                                          kAsTensor));
+INSTANTIATE_TEST_SUITE_P(VariedShapeSpec, ReshapeOpTest,
+                         ::testing::Values(kAsReshapeOption, kAsConstantTensor,
+                                           kAsTensor));
 }  // namespace
 }  // namespace tflite
 
diff --git a/tensorflow/lite/kernels/transpose_conv_test.cc b/tensorflow/lite/kernels/transpose_conv_test.cc
index 0520d84a30..44d1336b99 100644
--- a/tensorflow/lite/kernels/transpose_conv_test.cc
+++ b/tensorflow/lite/kernels/transpose_conv_test.cc
@@ -252,7 +252,7 @@ TEST_P(TransposeConvOpTest, AccuracyTest) {
   EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 3, 4, 1}));
 }
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     TransposeConvOpTest, TransposeConvOpTest,
     ::testing::ValuesIn(SingleOpTest::GetKernelTags(*kKernelMap)));
 
diff --git a/tensorflow/lite/models/speech_test.cc b/tensorflow/lite/models/speech_test.cc
index 17b7e8f28e..f3509d1ece 100644
--- a/tensorflow/lite/models/speech_test.cc
+++ b/tensorflow/lite/models/speech_test.cc
@@ -196,10 +196,10 @@ TEST_P(SpeechTest, DISABLED_TtsTest) {
 // 200s just to bring up the Android emulator.)
 static const int kAllInvocations = -1;
 static const int kFirstFewInvocations = 10;
-INSTANTIATE_TEST_CASE_P(LongTests, SpeechTest,
-                        ::testing::Values(kAllInvocations));
-INSTANTIATE_TEST_CASE_P(ShortTests, SpeechTest,
-                        ::testing::Values(kFirstFewInvocations));
+INSTANTIATE_TEST_SUITE_P(LongTests, SpeechTest,
+                         ::testing::Values(kAllInvocations));
+INSTANTIATE_TEST_SUITE_P(ShortTests, SpeechTest,
+                         ::testing::Values(kFirstFewInvocations));
 
 }  // namespace
 }  // namespace tflite
diff --git a/tensorflow/lite/toco/import_tensorflow_test.cc b/tensorflow/lite/toco/import_tensorflow_test.cc
index de7f4cdb7e..8ff3f7733a 100644
--- a/tensorflow/lite/toco/import_tensorflow_test.cc
+++ b/tensorflow/lite/toco/import_tensorflow_test.cc
@@ -257,8 +257,8 @@ std::vector<tensorflow::DataType> TestTypes() {
   return {DT_FLOAT, DT_INT32, DT_INT64, DT_BOOL, DT_QUINT8, DT_COMPLEX64};
 }
 
-INSTANTIATE_TEST_CASE_P(ShapeImportTest, ShapeImportTest,
-                        ::testing::ValuesIn(TestTypes()));
+INSTANTIATE_TEST_SUITE_P(ShapeImportTest, ShapeImportTest,
+                         ::testing::ValuesIn(TestTypes()));
 
 class ContentImportTest : public ::testing::Test {
  public:
@@ -418,8 +418,8 @@ TEST_P(TypeImportTest, BasicTypeInference) {
           model.operators[0].get());
   ASSERT_THAT(op->output_data_types, ::testing::ElementsAre(GetParam().second));
 }
-INSTANTIATE_TEST_CASE_P(BasicTypeInference, TypeImportTest,
-                        ::testing::ValuesIn(UnaryTestTypes()));
+INSTANTIATE_TEST_SUITE_P(BasicTypeInference, TypeImportTest,
+                         ::testing::ValuesIn(UnaryTestTypes()));
 
 TEST(ImportTest, TypeInferenceWithFixedOutputType) {
   // Create an op that has a fixed output type (bool).
diff --git a/tensorflow/lite/toco/tooling_util_test.cc b/tensorflow/lite/toco/tooling_util_test.cc
index faa6fe412e..f063ce71e9 100644
--- a/tensorflow/lite/toco/tooling_util_test.cc
+++ b/tensorflow/lite/toco/tooling_util_test.cc
@@ -96,8 +96,8 @@ TEST_P(ShapeTest, Agrees) {
   }
 }
 
-INSTANTIATE_TEST_CASE_P(AgreeBroadcast, ShapeTest,
-                        ::testing::ValuesIn(CreateShapePairs()));
+INSTANTIATE_TEST_SUITE_P(AgreeBroadcast, ShapeTest,
+                         ::testing::ValuesIn(CreateShapePairs()));
 
 static const char kNegativeValuesMessage[] =
     "Tensor shape should not include negative values";
-- 
GitLab


From 168eca540fc3b2090b15535d6728ed88520fae49 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Wed, 16 Jan 2019 11:14:30 -0800
Subject: [PATCH 0818/1095] Update tests to use the new distribute_py_test
 build rule.

PiperOrigin-RevId: 229590924
---
 tensorflow/contrib/distribute/python/BUILD    | 50 +++++++++++--------
 .../core/platform/default/distribute.bzl      |  2 +
 2 files changed, 32 insertions(+), 20 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index 384fe53923..3a4f23e597 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -640,12 +640,11 @@ distribute_py_test(
 )
 
 # TODO(b/121200287): Remove this in 2.0
-cuda_py_test(
+distribute_py_test(
     name = "keras_backward_compat_test",
     srcs = ["keras_backward_compat_test.py"],
-    additional_deps = [
-        ":keras_test_lib",
-    ],
+    full_precision = True,
+    main = "keras_backward_compat_test.py",
     shard_count = 16,
     tags = [
         "multi_and_single_gpu",
@@ -654,6 +653,9 @@ cuda_py_test(
         "no_windows_gpu",
         "notsan",
     ],
+    deps = [
+        ":keras_test_lib",
+    ],
 )
 
 py_library(
@@ -680,13 +682,12 @@ py_library(
     ],
 )
 
-cuda_py_test(
+distribute_py_test(
     name = "keras_dnn_correctness_test",
     size = "medium",
     srcs = ["keras_dnn_correctness_test.py"],
-    additional_deps = [
-        ":keras_correctness_test_lib",
-    ],
+    full_precision = True,
+    main = "keras_dnn_correctness_test.py",
     # Shard count is set to an odd number to distribute tasks across
     # shards more evenly.
     shard_count = 19,
@@ -697,15 +698,17 @@ cuda_py_test(
         "no_windows_gpu",
         "notsan",
     ],
+    deps = [
+        ":keras_correctness_test_lib",
+    ],
 )
 
-cuda_py_test(
+distribute_py_test(
     name = "keras_image_model_correctness_test",
     size = "medium",
     srcs = ["keras_image_model_correctness_test.py"],
-    additional_deps = [
-        ":keras_correctness_test_lib",
-    ],
+    full_precision = True,
+    main = "keras_image_model_correctness_test.py",
     # Shard count is set to an odd number to distribute tasks across
     # shards more evenly.
     shard_count = 31,
@@ -716,15 +719,17 @@ cuda_py_test(
         "no_windows_gpu",
         "notsan",
     ],
+    deps = [
+        ":keras_correctness_test_lib",
+    ],
 )
 
-cuda_py_test(
+distribute_py_test(
     name = "keras_embedding_model_correctness_test",
     size = "medium",
     srcs = ["keras_embedding_model_correctness_test.py"],
-    additional_deps = [
-        ":keras_correctness_test_lib",
-    ],
+    full_precision = True,
+    main = "keras_embedding_model_correctness_test.py",
     # Shard count is set to an odd number to distribute tasks across
     # shards more evenly.
     shard_count = 31,
@@ -735,15 +740,17 @@ cuda_py_test(
         "no_windows_gpu",
         "notsan",
     ],
+    deps = [
+        ":keras_correctness_test_lib",
+    ],
 )
 
-cuda_py_test(
+distribute_py_test(
     name = "keras_lstm_model_correctness_test",
     size = "medium",
     srcs = ["keras_lstm_model_correctness_test.py"],
-    additional_deps = [
-        ":keras_correctness_test_lib",
-    ],
+    full_precision = True,
+    main = "keras_lstm_model_correctness_test.py",
     # Shard count is set to an odd number to distribute tasks across
     # shards more evenly.
     shard_count = 31,
@@ -754,6 +761,9 @@ cuda_py_test(
         "no_windows_gpu",
         "notsan",
     ],
+    deps = [
+        ":keras_correctness_test_lib",
+    ],
 )
 
 py_library(
diff --git a/tensorflow/core/platform/default/distribute.bzl b/tensorflow/core/platform/default/distribute.bzl
index a3afc37f03..ea8fa8708e 100644
--- a/tensorflow/core/platform/default/distribute.bzl
+++ b/tensorflow/core/platform/default/distribute.bzl
@@ -11,6 +11,7 @@ def distribute_py_test(
         main = None,
         args = [],
         shard_count = 1,
+        full_precision = False,
         **kwargs):
     """Generates py_test targets for CPU and GPU.
 
@@ -26,6 +27,7 @@ def distribute_py_test(
         **kwargs: extra keyword arguments to the test.
     """
 
+    _ignore = (full_precision)
     cuda_py_test(
         name = name,
         srcs = srcs,
-- 
GitLab


From 42f6aa2d5aeda0a586fd64c82e7cc71c98e8e047 Mon Sep 17 00:00:00 2001
From: Tong Shen <endlessroad@google.com>
Date: Wed, 16 Jan 2019 11:15:04 -0800
Subject: [PATCH 0819/1095] Add a DFSFrom() algorithm.

PiperOrigin-RevId: 229591031
---
 tensorflow/core/graph/algorithm.cc | 39 +++++++++++++++++++++++-------
 tensorflow/core/graph/algorithm.h  | 12 +++++++++
 2 files changed, 42 insertions(+), 9 deletions(-)

diff --git a/tensorflow/core/graph/algorithm.cc b/tensorflow/core/graph/algorithm.cc
index 25bd3516a1..5ad1c19dc1 100644
--- a/tensorflow/core/graph/algorithm.cc
+++ b/tensorflow/core/graph/algorithm.cc
@@ -22,25 +22,29 @@ limitations under the License.
 #include "tensorflow/core/platform/logging.h"
 
 namespace tensorflow {
-
-void DFS(const Graph& g, const std::function<void(Node*)>& enter,
-         const std::function<void(Node*)>& leave,
-         const NodeComparator& stable_comparator,
-         const EdgeFilter& edge_filter) {
+namespace {
+template <typename T>
+void DFSFromHelper(const Graph& g, gtl::ArraySlice<T> start,
+                   const std::function<void(T)>& enter,
+                   const std::function<void(T)>& leave,
+                   const NodeComparator& stable_comparator,
+                   const EdgeFilter& edge_filter) {
   // Stack of work to do.
   struct Work {
-    Node* node;
+    T node;
     bool leave;  // Are we entering or leaving n?
   };
-  std::vector<Work> stack;
-  stack.push_back(Work{g.source_node(), false});
+  std::vector<Work> stack(start.size());
+  for (int i = 0; i < start.size(); ++i) {
+    stack[i] = Work{start[i], false};
+  }
 
   std::vector<bool> visited(g.num_node_ids(), false);
   while (!stack.empty()) {
     Work w = stack.back();
     stack.pop_back();
 
-    Node* n = w.node;
+    T n = w.node;
     if (w.leave) {
       leave(n);
       continue;
@@ -80,6 +84,23 @@ void DFS(const Graph& g, const std::function<void(Node*)>& enter,
     }
   }
 }
+}  // namespace
+
+void DFS(const Graph& g, const std::function<void(Node*)>& enter,
+         const std::function<void(Node*)>& leave,
+         const NodeComparator& stable_comparator,
+         const EdgeFilter& edge_filter) {
+  DFSFromHelper(g, {g.source_node()}, enter, leave, stable_comparator,
+                edge_filter);
+}
+
+void DFSFrom(const Graph& g, gtl::ArraySlice<const Node*> start,
+             const std::function<void(const Node*)>& enter,
+             const std::function<void(const Node*)>& leave,
+             const NodeComparator& stable_comparator,
+             const EdgeFilter& edge_filter) {
+  DFSFromHelper(g, start, enter, leave, stable_comparator, edge_filter);
+}
 
 void ReverseDFS(const Graph& g, const std::function<void(Node*)>& enter,
                 const std::function<void(Node*)>& leave,
diff --git a/tensorflow/core/graph/algorithm.h b/tensorflow/core/graph/algorithm.h
index 45f8a29a92..3479605df8 100644
--- a/tensorflow/core/graph/algorithm.h
+++ b/tensorflow/core/graph/algorithm.h
@@ -55,6 +55,18 @@ extern void DFS(const Graph& g, const std::function<void(Node*)>& enter,
                 const NodeComparator& stable_comparator = {},
                 const EdgeFilter& edge_filter = {});
 
+// Perform a depth-first-search on g starting at the 'start' nodes.
+// If enter is not empty, calls enter(n) before visiting any children of n.
+// If leave is not empty, calls leave(n) after visiting all children of n.
+// If stable_comparator is set, a stable ordering of visit is achieved by
+// sorting a node's neighbors first before visiting them.
+// If edge_filter is set then ignores edges for which edge_filter returns false.
+extern void DFSFrom(const Graph& g, gtl::ArraySlice<const Node*> start,
+                    const std::function<void(const Node*)>& enter,
+                    const std::function<void(const Node*)>& leave,
+                    const NodeComparator& stable_comparator = {},
+                    const EdgeFilter& edge_filter = {});
+
 // Perform a reverse depth-first-search on g starting at the sink node.
 // If enter is not empty, calls enter(n) before visiting any parents of n.
 // If leave is not empty, calls leave(n) after visiting all parents of n.
-- 
GitLab


From 3e41d77991f3807eb796a62573f2451cedee0c63 Mon Sep 17 00:00:00 2001
From: Katherine Wu <kathywu@google.com>
Date: Wed, 16 Jan 2019 11:23:28 -0800
Subject: [PATCH 0820/1095] Add mean tensor to Keras metrics. (From
 tf.metrics.mean_tensor).

PiperOrigin-RevId: 229592753
---
 tensorflow/python/keras/metrics.py      | 111 ++++++++++++++++++
 tensorflow/python/keras/metrics_test.py | 150 ++++++++++++++++++++++++
 2 files changed, 261 insertions(+)

diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index 245dffe99e..d6010e3cb0 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -29,6 +29,7 @@ from tensorflow.python.eager import context
 from tensorflow.python.eager import function
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_shape
 from tensorflow.python.keras import backend as K
 from tensorflow.python.keras.engine.base_layer import Layer
 from tensorflow.python.keras.losses import binary_crossentropy
@@ -2300,6 +2301,116 @@ class MeanIoU(Metric):
     return dict(list(base_config.items()) + list(config.items()))
 
 
+class MeanTensor(Metric):
+  """Computes the element-wise (weighted) mean of the given tensors.
+
+  `MeanTensor` returns a tensor with the same shape of the input tensors. The
+  mean value is updated by keeping local variables `total` and `count`. The
+  `total` tracks the sum of the weighted values, and `count` stores the sum of
+  the weighted counts.
+
+  Usage:
+
+  ```python
+  m = tf.metrics.MeanTensor()
+  m.update_state([0, 1, 2, 3])
+  m.update_state([4, 5, 6, 7])
+  print('Result: ', m.result().numpy())  # Result: [2, 3, 4, 5]
+  m.update_state([12, 10, 8, 6], sample_weights= [0, 0.2, 0.5, 1])
+  print('Result: ', m.result().numpy())  # Result: [2, 3.636, 4.8, 5.333]
+  ```
+  """
+
+  def __init__(self, name='mean_tensor', dtype=None):
+    """Creates a `MeanTensor` instance.
+
+    Args:
+      name: (Optional) string name of the metric instance.
+      dtype: (Optional) data type of the metric result.
+    """
+    super(MeanTensor, self).__init__(name=name, dtype=dtype)
+    self._shape = None
+    self._total = None
+    self._count = None
+    self._built = False
+
+  def _build(self, shape):
+    self._shape = tensor_shape.TensorShape(shape)
+    # Create new state variables
+    self._total = self.add_weight(
+        'total', shape=shape, initializer=init_ops.zeros_initializer)
+    self._count = self.add_weight(
+        'count', shape=shape, initializer=init_ops.zeros_initializer)
+    with ops.init_scope():
+      if not context.executing_eagerly():
+        K._initialize_variables(K._get_session())  # pylint: disable=protected-access
+    self._built = True
+
+  @property
+  def total(self):
+    return self._total if self._built else None
+
+  @property
+  def count(self):
+    return self._count if self._built else None
+
+  def update_state(self, values, sample_weight=None):
+    """Accumulates statistics for computing the element-wise mean.
+
+    Args:
+      values: Per-example value.
+      sample_weight: Optional weighting of each example. Defaults to 1.
+
+    Returns:
+      Update op.
+    """
+    values = math_ops.cast(values, self._dtype)
+    if not self._built:
+      self._build(values.shape)
+    elif values.shape != self._shape:
+      raise ValueError('MeanTensor input values must always have the same '
+                       'shape. Expected shape (set during the first call): {}. '
+                       'Got: {}'.format(self._shape, values.get_shape()))
+
+    num_values = array_ops.ones_like(values)
+    if sample_weight is not None:
+      sample_weight = math_ops.cast(sample_weight, self._dtype)
+
+      # Update dimensions of weights to match with values if possible.
+      values, _, sample_weight = squeeze_or_expand_dimensions(
+          values, None, sample_weight)
+      try:
+        # Broadcast weights if possible.
+        sample_weight = weights_broadcast_ops.broadcast_weights(
+            sample_weight, values)
+      except ValueError:
+        # Reduce values to same ndim as weight array
+        ndim = K.ndim(values)
+        weight_ndim = K.ndim(sample_weight)
+        values = math_ops.reduce_mean(
+            values, axis=list(range(weight_ndim, ndim)))
+
+      num_values = math_ops.multiply(num_values, sample_weight)
+      values = math_ops.multiply(values, sample_weight)
+
+    update_total_op = state_ops.assign_add(self._total, values)
+    with ops.control_dependencies([update_total_op]):
+      return state_ops.assign_add(self._count, num_values)
+
+  def result(self):
+    if not self._built:
+      raise ValueError(
+          'MeanTensor does not have any result yet. Please call the MeanTensor '
+          'instance or use `.update_state(value)` before retrieving the result.'
+          )
+    return math_ops.div_no_nan(self.total, self.count)
+
+  def reset_states(self):
+    if self._built:
+      for v in self.variables:
+        K.set_value(v, np.zeros(self._shape.as_list()))
+
+
 def accuracy(y_true, y_pred):
   y_pred.get_shape().assert_is_compatible_with(y_true.get_shape())
   if y_true.dtype != y_pred.dtype:
diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py
index bf2f4d88fa..bd94132a9a 100644
--- a/tensorflow/python/keras/metrics_test.py
+++ b/tensorflow/python/keras/metrics_test.py
@@ -23,6 +23,7 @@ import os
 import numpy as np
 
 from tensorflow.python.eager import context
+from tensorflow.python.eager import function as eager_function
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -30,6 +31,7 @@ from tensorflow.python.framework import test_util
 from tensorflow.python.keras import keras_parameterized
 from tensorflow.python.keras import layers
 from tensorflow.python.keras import metrics
+from tensorflow.python.keras import Model
 from tensorflow.python.keras import testing_utils
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import variables
@@ -1050,6 +1052,154 @@ class MeanIoUTest(test.TestCase):
     self.assertAllClose(self.evaluate(result), expected_result, atol=1e-3)
 
 
+class MeanTensorTest(keras_parameterized.TestCase):
+
+  @test_util.run_in_graph_and_eager_modes
+  def test_config(self):
+    m = metrics.MeanTensor(name='mean_by_element')
+
+    # check config
+    self.assertEqual(m.name, 'mean_by_element')
+    self.assertTrue(m.stateful)
+    self.assertEqual(m.dtype, dtypes.float32)
+    self.assertEqual(len(m.variables), 0)
+
+    with self.assertRaisesRegexp(ValueError, 'does not have any result yet'):
+      m.result()
+
+    self.evaluate(m([[3], [5], [3]]))
+    self.assertAllEqual(m._shape, [3, 1])
+
+    m2 = metrics.MeanTensor.from_config(m.get_config())
+    self.assertEqual(m2.name, 'mean_by_element')
+    self.assertTrue(m2.stateful)
+    self.assertEqual(m2.dtype, dtypes.float32)
+    self.assertEqual(len(m2.variables), 0)
+
+  @test_util.run_in_graph_and_eager_modes
+  def test_unweighted(self):
+    m = metrics.MeanTensor(dtype=dtypes.float64)
+
+    # check __call__()
+    self.assertAllClose(self.evaluate(m([100, 40])), [100, 40])
+    self.assertAllClose(self.evaluate(m.total), [100, 40])
+    self.assertAllClose(self.evaluate(m.count), [1, 1])
+
+    # check update_state() and result() + state accumulation + tensor input
+    update_op = m.update_state(ops.convert_n_to_tensor([1, 5]))
+    self.evaluate(update_op)
+    self.assertAllClose(self.evaluate(m.result()), [50.5, 22.5])
+    self.assertAllClose(self.evaluate(m.total), [101, 45])
+    self.assertAllClose(self.evaluate(m.count), [2, 2])
+
+    # check reset_states()
+    m.reset_states()
+    self.assertAllClose(self.evaluate(m.total), [0, 0])
+    self.assertAllClose(self.evaluate(m.count), [0, 0])
+
+  @test_util.run_in_graph_and_eager_modes
+  def test_weighted(self):
+    m = metrics.MeanTensor(dtype=dtypes.float64)
+    self.assertEqual(m.dtype, dtypes.float64)
+
+    # check scalar weight
+    result_t = m([100, 30], sample_weight=0.5)
+    self.assertAllClose(self.evaluate(result_t), [100, 30])
+    self.assertAllClose(self.evaluate(m.total), [50, 15])
+    self.assertAllClose(self.evaluate(m.count), [0.5, 0.5])
+
+    # check weights not scalar and weights rank matches values rank
+    result_t = m([1, 5], sample_weight=[1, 0.2])
+    result = self.evaluate(result_t)
+    self.assertAllClose(result, [51 / 1.5, 16 / 0.7], 2)
+    self.assertAllClose(self.evaluate(m.total), [51, 16])
+    self.assertAllClose(self.evaluate(m.count), [1.5, 0.7])
+
+    # check weights broadcast
+    result_t = m([1, 2], sample_weight=0.5)
+    self.assertAllClose(self.evaluate(result_t), [51.5 / 2, 17 / 1.2])
+    self.assertAllClose(self.evaluate(m.total), [51.5, 17])
+    self.assertAllClose(self.evaluate(m.count), [2, 1.2])
+
+    # check weights squeeze
+    result_t = m([1, 5], sample_weight=[[1], [0.2]])
+    self.assertAllClose(self.evaluate(result_t), [52.5 / 3, 18 / 1.4])
+    self.assertAllClose(self.evaluate(m.total), [52.5, 18])
+    self.assertAllClose(self.evaluate(m.count), [3, 1.4])
+
+    # check weights expand
+    m = metrics.MeanTensor((2, 1), dtype=dtypes.float64)
+    self.evaluate(variables.variables_initializer(m.variables))
+    result_t = m([[1], [5]], sample_weight=[1, 0.2])
+    self.assertAllClose(self.evaluate(result_t), [[1], [5]])
+    self.assertAllClose(self.evaluate(m.total), [[1], [1]])
+    self.assertAllClose(self.evaluate(m.count), [[1], [0.2]])
+
+  @test_util.run_in_graph_and_eager_modes
+  def test_invalid_value_shape(self):
+    m = metrics.MeanTensor(dtype=dtypes.float64)
+    m([1])
+    with self.assertRaisesRegexp(
+        ValueError, 'MeanTensor input values must always have the same shape'):
+      m([1, 5])
+
+  @test_util.run_in_graph_and_eager_modes
+  def test_build_in_tf_function(self):
+    """Ensure that variables are created correctly in a tf function."""
+    m = metrics.MeanTensor(dtype=dtypes.float64)
+
+    @eager_function.defun
+    def call_metric(x):
+      return m(x)
+
+    self.assertAllClose(self.evaluate(call_metric([100, 40])), [100, 40])
+    self.assertAllClose(self.evaluate(m.total), [100, 40])
+    self.assertAllClose(self.evaluate(m.count), [1, 1])
+    self.assertAllClose(self.evaluate(call_metric([20, 2])), [60, 21])
+
+  def test_in_keras_model(self):
+    with context.eager_mode():
+      class ModelWithMetric(Model):
+
+        def __init__(self):
+          super(ModelWithMetric, self).__init__()
+          self.dense1 = layers.Dense(
+              3, activation='relu', kernel_initializer='ones')
+          self.dense2 = layers.Dense(
+              1, activation='sigmoid', kernel_initializer='ones')
+          self.mean_tensor = metrics.MeanTensor()
+
+        def call(self, x):
+          x = self.dense1(x)
+          x = self.dense2(x)
+          self.mean_tensor(self.dense1.kernel)
+          return x
+
+      model = ModelWithMetric()
+      model.compile(
+          loss='mae',
+          optimizer='rmsprop',
+          run_eagerly=True)
+
+      x = np.ones((100, 4))
+      y = np.zeros((100, 1))
+      model.evaluate(x, y, batch_size=50)
+      self.assertAllClose(self.evaluate(model.mean_tensor.result()),
+                          np.ones((4, 3)))
+      self.assertAllClose(self.evaluate(model.mean_tensor.total),
+                          np.full((4, 3), 2))
+      self.assertAllClose(self.evaluate(model.mean_tensor.count),
+                          np.full((4, 3), 2))
+
+      model.evaluate(x, y, batch_size=25)
+      self.assertAllClose(self.evaluate(model.mean_tensor.result()),
+                          np.ones((4, 3)))
+      self.assertAllClose(self.evaluate(model.mean_tensor.total),
+                          np.full((4, 3), 4))
+      self.assertAllClose(self.evaluate(model.mean_tensor.count),
+                          np.full((4, 3), 4))
+
+
 def _get_model(compile_metrics):
   model_layers = [
       layers.Dense(3, activation='relu', kernel_initializer='ones'),
-- 
GitLab


From 0f91a840d86210f298c8caf0e8640b4b9925a058 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 16 Jan 2019 11:25:00 -0800
Subject: [PATCH 0821/1095] Support lists of floating points in custom op
 attributes.

PiperOrigin-RevId: 229593054
---
 tensorflow/lite/toco/tflite/operator.cc      | 15 +++++++++++++++
 tensorflow/lite/toco/tflite/operator_test.cc | 14 +++++++++++++-
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index d9c051ebf7..088673fd95 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -1647,6 +1647,13 @@ class TensorFlowUnsupported : public BaseOperator {
             }
             fbb->EndVector(start, /*typed=*/true, /*fixed=*/false);
             has_valid_attr = true;
+          } else if (attr.list().f_size() > 0) {
+            auto start = fbb->StartVector(key);
+            for (const float v : attr.list().f()) {
+              fbb->Add(v);
+            }
+            fbb->EndVector(start, /*typed=*/true, /*fixed=*/false);
+            has_valid_attr = true;
           } else {
             LOG(WARNING)
                 << "Ignoring unsupported type in list attribute with key '"
@@ -1707,6 +1714,14 @@ class TensorFlowUnsupported : public BaseOperator {
           }
           break;
         }
+        case 13: {  // flexbuffers::FBT_VECTOR_FLOAT: {
+          auto* list = (*attr)[key].mutable_list();
+          const auto& vector = value.AsTypedVector();
+          for (size_t i = 0; i < vector.size(); i++) {
+            list->add_f(vector[i].AsFloat());
+          }
+          break;
+        }
         default:
           LOG(WARNING) << "Ignoring unsupported attribute type with key '"
                        << key << "'";
diff --git a/tensorflow/lite/toco/tflite/operator_test.cc b/tensorflow/lite/toco/tflite/operator_test.cc
index 215eda82f6..f77780488a 100644
--- a/tensorflow/lite/toco/tflite/operator_test.cc
+++ b/tensorflow/lite/toco/tflite/operator_test.cc
@@ -569,6 +569,12 @@ TEST_F(OperatorTest, TensorFlowUnsupported) {
   (*attr)["str_attr"].set_s("Hello World");
   (*attr)["int_attr"].set_i(17);
   (*attr)["bool_attr"].set_b(true);
+  {
+    auto* list = (*attr)["list_float_attr"].mutable_list();
+    list->add_f(std::numeric_limits<float>::min());
+    list->add_f(2.0);
+    list->add_f(-std::numeric_limits<float>::max());
+  }
   {
     auto* list = (*attr)["list_int_attr"].mutable_list();
     list->add_i(1);
@@ -588,7 +594,13 @@ TEST_F(OperatorTest, TensorFlowUnsupported) {
   EXPECT_EQ("Hello World", output_attr.at("str_attr").s());
   EXPECT_EQ(17, output_attr.at("int_attr").i());
   EXPECT_EQ(true, output_attr.at("bool_attr").b());
-
+  {
+    const auto& list = output_attr.at("list_float_attr").list();
+    ASSERT_EQ(3, list.f_size());
+    EXPECT_EQ(std::numeric_limits<float>::min(), list.f(0));
+    EXPECT_EQ(2.0, list.f(1));
+    EXPECT_EQ(-std::numeric_limits<float>::max(), list.f(2));
+  }
   {
     const auto& list = output_attr.at("list_int_attr").list();
     ASSERT_EQ(4, list.i_size());
-- 
GitLab


From 6d5147395469d35d695dbd9d2498ad36bcfcca46 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 16 Jan 2019 11:25:34 -0800
Subject: [PATCH 0822/1095] Support previously added 'truncate' attribute on
 floating datatype casts to apply in tf2xla as well.

PiperOrigin-RevId: 229593179
---
 tensorflow/compiler/tests/randomized_tests.cc | 34 +++++++++++++-
 tensorflow/compiler/tf2xla/kernels/cast_op.cc | 47 +++++++++++++++++++
 tensorflow/compiler/xla/primitive_util.cc     | 16 +++++++
 tensorflow/compiler/xla/primitive_util.h      |  4 ++
 4 files changed, 99 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/tests/randomized_tests.cc b/tensorflow/compiler/tests/randomized_tests.cc
index 735014fc89..1521cc760b 100644
--- a/tensorflow/compiler/tests/randomized_tests.cc
+++ b/tensorflow/compiler/tests/randomized_tests.cc
@@ -63,6 +63,7 @@ limitations under the License.
 #include "tensorflow/core/graph/graph.h"
 #include "tensorflow/core/graph/graph_constructor.h"
 #include "tensorflow/core/kernels/ops_util.h"
+#include "tensorflow/core/lib/bfloat16/bfloat16.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/platform/test.h"
@@ -769,8 +770,22 @@ Status TensorsAreEqualImpl(const Tensor& x, const Tensor& y) {
   for (int i = 0; i < Tx.size(); ++i) {
     if (Tx(i) != Ty(i)) {
       return errors::InvalidArgument(absl::StrCat(
-          i, "-th tensor element isn't equal: ", Tx(i), " vs. ", Ty(i),
-          ". x = ", x.DebugString(), "y = ", y.DebugString()));
+          i, "-th tensor element isn't equal: ", Str(Tx(i)), " vs. ",
+          Str(Ty(i)), ". x = ", x.DebugString(), "y = ", y.DebugString()));
+    }
+  }
+  return Status::OK();
+}
+
+Status TensorsAreEqualImplBfloat16(const Tensor& x, const Tensor& y) {
+  auto Tx = x.flat<bfloat16>();
+  auto Ty = y.flat<bfloat16>();
+  for (int i = 0; i < Tx.size(); ++i) {
+    if (Tx(i) != Ty(i)) {
+      return errors::InvalidArgument(absl::StrCat(
+          i, "-th tensor element isn't equal: ", static_cast<float>(Tx(i)),
+          " vs. ", static_cast<float>(Ty(i)), ". x = ", x.DebugString(),
+          "y = ", y.DebugString()));
     }
   }
   return Status::OK();
@@ -806,6 +821,8 @@ Status TensorsAreClose(const Tensor& a, const Tensor& b, double atol,
       return TensorsAreEqualImpl<int64>(a, b);
     case DT_BOOL:
       return TensorsAreEqualImpl<bool>(a, b);
+    case DT_BFLOAT16:
+      return TensorsAreEqualImplBfloat16(a, b);
     default:
       LOG(FATAL) << "Unexpected type : " << DataTypeString(a.dtype());
   }
@@ -1380,6 +1397,19 @@ TEST_F(OpTest, Cast) {
   });
 }
 
+TEST_F(OpTest, CastBF16) {
+  Repeatedly([this]() {
+    DataType src_type, dst_type;
+    src_type = Choose<DataType>({DT_FLOAT});
+    dst_type = Choose<DataType>({DT_BFLOAT16});
+    return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Cast")
+                                             .RandomInput(src_type)
+                                             .Attr("SrcT", src_type)
+                                             .Attr("DstT", dst_type)
+                                             .Attr("Truncate", true));
+  });
+}
+
 TEST_F(OpTest, Ceil) {
   Repeatedly([this]() {
     return ExpectTfAndXlaOutputsAreClose(
diff --git a/tensorflow/compiler/tf2xla/kernels/cast_op.cc b/tensorflow/compiler/tf2xla/kernels/cast_op.cc
index ce8131eeb4..db58c2e651 100644
--- a/tensorflow/compiler/tf2xla/kernels/cast_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/cast_op.cc
@@ -12,6 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
+#include "tensorflow/compiler/tf2xla/lib/util.h"
 
 #include "tensorflow/compiler/tf2xla/type_util.h"
 #include "tensorflow/compiler/tf2xla/xla_helpers.h"
@@ -19,6 +20,7 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/primitive_util.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/core/framework/kernel_def_builder.h"
 
 namespace tensorflow {
@@ -31,6 +33,20 @@ class CastOp : public XlaOpKernel {
     OP_REQUIRES_OK(ctx, ctx->GetAttr("DstT", &dst_dtype_));
     OP_REQUIRES_OK(ctx, DataTypeToPrimitiveType(src_dtype_, &src_type_));
     OP_REQUIRES_OK(ctx, DataTypeToPrimitiveType(dst_dtype_, &dst_type_));
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("Truncate", &use_truncation_));
+  }
+
+  xla::PrimitiveType GetUnsignedIntTypeOfSameWidth(int64 src_bitwidth) {
+    switch (src_bitwidth) {
+      case 16:
+        return xla::U16;
+      case 32:
+        return xla::U32;
+      case 64:
+        return xla::U64;
+      default:
+        return xla::PRIMITIVE_TYPE_INVALID;
+    }
   }
 
   void Compile(XlaOpKernelContext* ctx) override {
@@ -48,6 +64,36 @@ class CastOp : public XlaOpKernel {
       // imaginary part.
       output = xla::ConvertElementType(xla::Real(input), dst_type_);
     } else {
+      if (use_truncation_) {
+        OP_REQUIRES(
+            ctx,
+            xla::primitive_util::IsFloatingPointType(src_type_) &&
+                xla::primitive_util::IsFloatingPointType(dst_type_),
+            errors::Unimplemented("Truncate attribute is only "
+                                  "implemented for floating point datatypes."));
+        int mantissa_difference =
+            xla::primitive_util::SignificandWidth(src_type_) -
+            xla::primitive_util::SignificandWidth(dst_type_);
+        OP_REQUIRES(ctx, mantissa_difference > 0,
+                    errors::Unimplemented(
+                        "Truncate attribute is only implemented in cases where "
+                        "dst datatype "
+                        "has fewer mantissa bits than the src datatype"));
+        int src_bitwidth = xla::primitive_util::BitWidth(src_type_);
+
+        // Bitcast to same-width integer, mask off the LSBs, bitcast back to the
+        // source datatype.
+        int64 mask = ~((1L << mantissa_difference) - 1);
+        xla::PrimitiveType same_width_int =
+            GetUnsignedIntTypeOfSameWidth(src_bitwidth);
+        OP_REQUIRES(ctx, same_width_int != xla::PRIMITIVE_TYPE_INVALID,
+                    errors::Unimplemented("Unexpected type bitwidth"));
+        input = xla::BitcastConvertType(
+            xla::And(
+                xla::BitcastConvertType(input, same_width_int),
+                ::tensorflow::IntegerLiteral(builder, same_width_int, mask)),
+            src_type_);
+      }
       output = xla::ConvertElementType(input, dst_type_);
     }
 
@@ -57,6 +103,7 @@ class CastOp : public XlaOpKernel {
  protected:
   DataType src_dtype_, dst_dtype_;
   xla::PrimitiveType src_type_, dst_type_;
+  bool use_truncation_;
 
   TF_DISALLOW_COPY_AND_ASSIGN(CastOp);
 };
diff --git a/tensorflow/compiler/xla/primitive_util.cc b/tensorflow/compiler/xla/primitive_util.cc
index 04b40efbed..3386d2e097 100644
--- a/tensorflow/compiler/xla/primitive_util.cc
+++ b/tensorflow/compiler/xla/primitive_util.cc
@@ -18,11 +18,27 @@ limitations under the License.
 #include "absl/strings/ascii.h"
 #include "absl/strings/numbers.h"
 #include "tensorflow/compiler/xla/util.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/core/platform/logging.h"
 
 namespace xla {
 namespace primitive_util {
 
+int SignificandWidth(PrimitiveType type) {
+  switch (type) {
+    case F32:
+      return std::numeric_limits<float>::digits;
+    case F64:
+      return std::numeric_limits<double>::digits;
+    case BF16:
+      return kBFloat16MantissaBits + 1;
+    case F16:
+      return 11;
+    default:
+      LOG(FATAL) << "Not a floating data type " << type;
+  }
+}
+
 bool IsFloatingPointType(PrimitiveType type) {
   return type == F16 || type == F32 || type == F64 || type == BF16;
 }
diff --git a/tensorflow/compiler/xla/primitive_util.h b/tensorflow/compiler/xla/primitive_util.h
index 739a7c409c..d32505335d 100644
--- a/tensorflow/compiler/xla/primitive_util.h
+++ b/tensorflow/compiler/xla/primitive_util.h
@@ -29,6 +29,10 @@ limitations under the License.
 namespace xla {
 namespace primitive_util {
 
+// Returns the count of significand (mantissa) bits for float datatypes.
+// For non-float datatypes, results in a LOG(FATAL).
+int SignificandWidth(PrimitiveType type);
+
 // The number of exponent bits in a BF16 value.
 const int kBFloat16ExponentBits = 8;
 
-- 
GitLab


From d80c2aa144b092937ffd1f019cfe461ea668839f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 16 Jan 2019 11:29:39 -0800
Subject: [PATCH 0823/1095] - Fixed multiple broken links in the
 best_practices.md guide as a result of the recent move out of contrib

PiperOrigin-RevId: 229594101
---
 .../lite/g3doc/performance/best_practices.md  | 34 +++++++++++++++----
 1 file changed, 27 insertions(+), 7 deletions(-)

diff --git a/tensorflow/lite/g3doc/performance/best_practices.md b/tensorflow/lite/g3doc/performance/best_practices.md
index 090f19e810..5f41a70275 100644
--- a/tensorflow/lite/g3doc/performance/best_practices.md
+++ b/tensorflow/lite/g3doc/performance/best_practices.md
@@ -1,6 +1,9 @@
 # Performance best practices
 
-Mobile and embedded devices have limited computational resources and it is important to keep your application resource efficient. We have compiled a list of best practices and strategies you can use to optimize your model and application when using Tensorflow Lite.
+Mobile and embedded devices have limited computational resources and it is
+important to keep your application resource efficient. We have compiled a list
+of best practices and strategies that you can use to optimize your model and
+application when using TensorFlow Lite.
 
 ## Choose the best model for the task
 Depending on the task you will need to make a tradeoff between model complexity and size. If your task requires high accuracy then you may need a large and complex model. Some tasks may work with a less precise model, for these tasks it is better to use a smaller but less precise model. Smaller models not only use less disk space and memory but are generally faster and more energy efficient. For example, graphs below show accuracy and latency tradeoff for some common image classification models.
@@ -10,7 +13,7 @@ Depending on the task you will need to make a tradeoff between model complexity
 
 ![latency vs model size](../images/performance/model_size_vs_latency.png "Latency vs Model size")
 
-One example of models optimized for mobile devices are [MobileNets](https://arxiv.org/abs/1704.04861), which are optimized for mobile vision applications. Tensorflow Lite [models page](../models.md) lists several other models that have been optimized specifically for mobile and embedded devices.
+One example of models optimized for mobile devices are [MobileNets](https://arxiv.org/abs/1704.04861), which are optimized for mobile vision applications. TensorFlow Lite [models page](../models.md) lists several other models that have been optimized specifically for mobile and embedded devices.
 
 You can retrain the listed models on your own dataset by using transfer learning. Check out our transfer learning tutorial for
 [image classification](https://codelabs.developers.google.com/codelabs/tensorflow-for-poets/#0) and
@@ -18,20 +21,37 @@ You can retrain the listed models on your own dataset by using transfer learning
 
 
 ## Profile your model
-Once you have selected a candidate model that is right for your task, it is a good practice to profile and benchmark your model. Tensorflow Lite [benchmarking tool](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/benchmark) has a built-in profiler that shows per operator profiling statistics. This can help in understanding performance bottlenecks and which operators dominate the computation time.
+Once you have selected a candidate model that is right for your task, it is a good practice to profile and benchmark your model. TensorFlow Lite [benchmarking tool](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/benchmark) has a built-in profiler that shows per operator profiling statistics. This can help in understanding performance bottlenecks and which operators dominate the computation time.
 
 ## Profile and optimize operators in the graph
 If a particular operator appears frequently in the model and based on profiling you find the operator consuming the most amount of time, you can look into optimizing the operator.
- This scenario should be rare as Tensorflow Lite has optimized versions for most ops. However you may be able to write a faster version of a custom op, if you know the constraints in which the operator is executed. Check out our [custom operator documentation](../custom_operators.md).
+ This scenario should be rare as TensorFlow Lite has optimized versions for most ops. However you may be able to write a faster version of a custom op, if you know the constraints in which the operator is executed. Check out our [custom operator documentation](../custom_operators.md).
 
 ## Quantize your model
 If your model uses floating point weights or activations then it may be possible to reduce the size of model up to ~4x by using quantization and other model optimizations. Check out our [model optimization toolkit](model_optimization.md) for details about optimizing your model. 
 
 ## Tweak the number of threads
-Tensorflow Lite supports multi-threaded kernels for many operators. You can increase the number of threads and speed up execution of operators. Increasing the number of threads will however make your model use more resources and power. For some applications latency may be more important than energy efficiency. You can increase the number of threads by setting the number of [interpreter](https://github.com/tensorflow/tensorflow/blob/1084594657a5d139102ac794f84d1427a710e39a/tensorflow/lite/interpreter.h#L337) threads. Multi-threaded execution however comes at the cost of increased performance variability depending on what else is been executed concurrently. This is particularly the case for mobile apps. For example, isolated tests may show 2x speed up vs single-threaded but if another app is executing at the same time may result in worst performance than single-threaded.
+
+TensorFlow Lite supports multi-threaded kernels for many operators. You can
+increase the number of threads and speed up execution of operators. Increasing
+the number of threads will however make your model use more resources and power.
+For some applications latency may be more important than energy efficiency. You
+can increase the number of threads by setting the number of
+[interpreter](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/interpreter.h#L333)
+threads. Multi-threaded execution however comes at the cost of increased
+performance variability depending on what else is been executed concurrently.
+This is particularly the case for mobile apps. For example, isolated tests may
+show 2x speed up vs single-threaded but if another app is executing at the same
+time may result in worst performance than single-threaded.
 
 ## Eliminate redundant copies
-If your application is not careful, there can be redundant copies when feeding the input to the model and reading output from the model. Make sure to eliminate redundant copies. If you are using higher level APIs like Java API, make sure to carefully check the documentation for performance caveats. For example, the Java API is a lot faster if ByteBuffers are used as [inputs](https://github.com/tensorflow/tensorflow/blob/6305a6d83552ba6a472cd72398b60d9241467f1f/tensorflow/lite/java/src/main/java/org/tensorflow/lite/Interpreter.java#L151).
+
+If your application is not careful, there can be redundant copies when feeding
+the input to the model and reading output from the model. Make sure to eliminate
+redundant copies. If you are using higher level APIs like Java API, make sure to
+carefully check the documentation for performance caveats. For example, the Java
+API is a lot faster if ByteBuffers are used as
+[inputs](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/java/src/main/java/org/tensorflow/lite/Interpreter.java#L175).
 
 ## Profile your application with platform specific tools
 Platform specific tools like [Android profiler](https://developer.android.com/studio/profile/android-profiler) and [Instruments](https://help.apple.com/instruments/mac/current/) provide a wealth of profiling information that can be used to debug your app. Sometimes the performance bug may be not in the model but in parts of application code that interact with the model. Make sure to familiarize yourself with platform specific profiling tools and best practices for your platform.
@@ -64,7 +84,7 @@ are a great choice for large models that have high arithmetic intensity.
 
 ## Need more help
 
-The Tensorflow team is happy to help diagnose and address specific performance
+The TensorFlow team is happy to help diagnose and address specific performance
 issues you may be facing. Please file an issue on
 [GitHub](https://github.com/tensorflow/tensorflow/issues) with details of the
 issue.
-- 
GitLab


From 43d9f971759be4a0a683bfdbaa646025552e8f1e Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Wed, 16 Jan 2019 11:38:46 -0800
Subject: [PATCH 0824/1095] Fix more TFLite tests and binaries

Use the tf_cc_binary where appropriate. Also fix an issue
with schema generation and blacklist lstm zip tests temporarily.

PiperOrigin-RevId: 229596167
---
 tensorflow/lite/build_def.bzl         | 2 +-
 tensorflow/lite/schema/BUILD          | 1 -
 tensorflow/lite/testing/BUILD         | 5 +++--
 tensorflow/lite/tools/benchmark/BUILD | 3 ++-
 4 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/tensorflow/lite/build_def.bzl b/tensorflow/lite/build_def.bzl
index 03b217c120..3c48b655f7 100644
--- a/tensorflow/lite/build_def.bzl
+++ b/tensorflow/lite/build_def.bzl
@@ -263,7 +263,7 @@ def generated_test_models():
         "logical_and",
         "logical_or",
         "logical_xor",
-        "lstm",
+        #"lstm", TODO(b/122889684): Resolve toco structured line parsing in oss.
         "max_pool",
         "maximum",
         "mean",
diff --git a/tensorflow/lite/schema/BUILD b/tensorflow/lite/schema/BUILD
index 69d5458c6e..ea516764c9 100644
--- a/tensorflow/lite/schema/BUILD
+++ b/tensorflow/lite/schema/BUILD
@@ -70,7 +70,6 @@ flatbuffer_cc_library(
         "--no-union-value-namespacing",
         "--gen-object-api",
     ],
-    gen_reflections = True,
     out_prefix = "reflection/",
 )
 
diff --git a/tensorflow/lite/testing/BUILD b/tensorflow/lite/testing/BUILD
index 19c950b4f8..ce85a39385 100644
--- a/tensorflow/lite/testing/BUILD
+++ b/tensorflow/lite/testing/BUILD
@@ -12,6 +12,7 @@ load(
 load("//tensorflow/lite:special_rules.bzl", "tflite_portable_test_suite")
 load(
     "//tensorflow:tensorflow.bzl",
+    "tf_cc_binary",
     "tf_cc_test",
     "py_test",
 )
@@ -230,7 +231,7 @@ cc_test(
     ],
 )
 
-cc_binary(
+tf_cc_binary(
     name = "nnapi_example",
     srcs = ["nnapi_example.cc"],
     deps = [
@@ -380,7 +381,7 @@ tf_cc_test(
     ],
 )
 
-cc_binary(
+tf_cc_binary(
     name = "tflite_diff",
     srcs = ["tflite_diff_example_test.cc"],
     deps = [
diff --git a/tensorflow/lite/tools/benchmark/BUILD b/tensorflow/lite/tools/benchmark/BUILD
index bc47406cd9..e68997a00b 100644
--- a/tensorflow/lite/tools/benchmark/BUILD
+++ b/tensorflow/lite/tools/benchmark/BUILD
@@ -4,6 +4,7 @@ package(default_visibility = [
 
 licenses(["notice"])  # Apache 2.0
 
+load("//tensorflow:tensorflow.bzl", "tf_cc_binary")
 load("//tensorflow/lite:special_rules.bzl", "tflite_portable_test_suite")
 load("//tensorflow/lite:build_def.bzl", "tflite_copts")
 load("//tensorflow/lite:build_def.bzl", "tflite_linkopts")
@@ -35,7 +36,7 @@ cc_binary(
     ],
 )
 
-cc_binary(
+tf_cc_binary(
     name = "benchmark_model_plus_flex",
     srcs = [
         "benchmark_plus_flex_main.cc",
-- 
GitLab


From e388dfcde40d44d3c6bf5ee6835b823ede73bb7a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 16 Jan 2019 11:56:47 -0800
Subject: [PATCH 0825/1095] Export BUILD file so it is visible at
 //third_party/tensorflow/core:BUILD

PiperOrigin-RevId: 229599457
---
 tensorflow/core/BUILD | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 7ba9afb5a8..6c9afc51e9 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -70,6 +70,9 @@ package(default_visibility = [
 
 licenses(["notice"])  # Apache 2.0
 
+# Export the BUILD file so automated tooling can check licenses
+exports_files(["BUILD"])
+
 load(
     "//tensorflow:tensorflow.bzl",
     "cc_header_only_library",
-- 
GitLab


From 45a95587b973cd0f82ab240b038b94157d015fc3 Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Wed, 16 Jan 2019 11:57:21 -0800
Subject: [PATCH 0826/1095] Don't rematerialize copy nodes.

Copy insertion adds copies to prevent buffer clobbering. Rematerialize those copies could reintroduce the clobbering. This CL adds copies into the blacklist of rematerialization pass.

PiperOrigin-RevId: 229599559
---
 .../xla/service/hlo_rematerialization.cc      |  9 ++++
 .../xla/service/hlo_rematerialization_test.cc | 46 +++++++++++++++++++
 2 files changed, 55 insertions(+)

diff --git a/tensorflow/compiler/xla/service/hlo_rematerialization.cc b/tensorflow/compiler/xla/service/hlo_rematerialization.cc
index 9ca14ca18a..5c1793933c 100644
--- a/tensorflow/compiler/xla/service/hlo_rematerialization.cc
+++ b/tensorflow/compiler/xla/service/hlo_rematerialization.cc
@@ -57,6 +57,15 @@ using ::tensorflow::strings::HumanReadableNumBytes;
 
 // Returns true if the given instruction is rematerializable.
 bool IsRematerializable(const HloInstruction* instruction) {
+  if (instruction->opcode() == HloOpcode::kCopy) {
+    if (LayoutUtil::Equal(instruction->shape().layout(),
+                          instruction->operand(0)->shape().layout())) {
+      // Don't rematerialize copies added by copy insertion (layout doesn't
+      // change).
+      return false;
+    }
+  }
+
   // Don't rematerialize instructions with side effects or instructions which
   // cannot be cloned safely.
   switch (instruction->opcode()) {
diff --git a/tensorflow/compiler/xla/service/hlo_rematerialization_test.cc b/tensorflow/compiler/xla/service/hlo_rematerialization_test.cc
index c4bb475619..102a360ad8 100644
--- a/tensorflow/compiler/xla/service/hlo_rematerialization_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_rematerialization_test.cc
@@ -499,6 +499,52 @@ TEST_F(HloRematerializationTest, InstructionRematerializedMultipleTimes) {
   EXPECT_THAT(add_4->operand(0), op::Broadcast(param));
 }
 
+TEST_F(HloRematerializationTest, CopyNotRematerialized) {
+  // Test that copies are not rematerialized.
+  auto module = CreateNewVerifiedModule();
+
+  auto builder = HloComputation::Builder(TestName());
+  auto param = builder.AddInstruction(
+      HloInstruction::CreateParameter(0, vec1024_shape_, "param"));
+
+  auto copy = builder.AddInstruction(
+      HloInstruction::CreateUnary(vec1024_shape_, HloOpcode::kCopy, param));
+
+  auto negate_a_1 = builder.AddInstruction(
+      HloInstruction::CreateUnary(vec1024_shape_, HloOpcode::kNegate, copy));
+
+  auto negate_a_2 = builder.AddInstruction(HloInstruction::CreateUnary(
+      vec1024_shape_, HloOpcode::kNegate, negate_a_1));
+
+  auto negate_b_1 = builder.AddInstruction(
+      HloInstruction::CreateUnary(vec1024_shape_, HloOpcode::kNegate, copy));
+
+  auto negate_b_2 = builder.AddInstruction(HloInstruction::CreateUnary(
+      vec1024_shape_, HloOpcode::kNegate, negate_b_1));
+
+  builder.AddInstruction(HloInstruction::CreateTuple({negate_a_2, negate_b_2}));
+
+  HloComputation* entry_computation =
+      module->AddEntryComputation(builder.Build());
+
+  TF_ASSERT_OK_AND_ASSIGN(bool changed,
+                          RunHloRematerialization(
+                              /*memory_limit_bytes=*/1 * 1024, module.get()));
+
+  auto count_copies = [](const HloComputation* computation) {
+    int64 copy_count = 0;
+    for (auto* instruction : computation->instructions()) {
+      if (instruction->opcode() == HloOpcode::kCopy) {
+        copy_count++;
+      }
+    }
+    return copy_count;
+  };
+  EXPECT_TRUE(changed);
+
+  EXPECT_EQ(count_copies(entry_computation), 1);
+}
+
 class IndirectUseTest : public HloRematerializationTest,
                         public ::testing::WithParamInterface<bool> {};
 
-- 
GitLab


From ee5d6e6deaac2b129bd9ac0c5ac3a82bb4bde8b8 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Wed, 16 Jan 2019 11:58:08 -0800
Subject: [PATCH 0827/1095] Tag a bunch of tests oss_serial to address spurious
 nightly GPU failures

Still not entirely sure what changed to make these not work when run together, but the CUDA upgrade seems likely. They pass individually with test_output=streamed.

PiperOrigin-RevId: 229599721
---
 tensorflow/contrib/eager/python/examples/densenet/BUILD     | 2 ++
 tensorflow/contrib/eager/python/examples/l2hmc/BUILD        | 3 +++
 .../contrib/eager/python/examples/linear_regression/BUILD   | 5 ++++-
 tensorflow/contrib/eager/python/examples/resnet50/BUILD     | 2 ++
 tensorflow/contrib/eager/python/examples/revnet/BUILD       | 4 ++++
 tensorflow/contrib/eager/python/examples/rnn_colorbot/BUILD | 3 +++
 tensorflow/contrib/eager/python/examples/rnn_ptb/BUILD      | 3 +++
 tensorflow/contrib/eager/python/examples/spinn/BUILD        | 1 +
 tensorflow/contrib/sparsemax/BUILD                          | 6 ++++++
 9 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/eager/python/examples/densenet/BUILD b/tensorflow/contrib/eager/python/examples/densenet/BUILD
index e2154fcc5f..56a682ec55 100644
--- a/tensorflow/contrib/eager/python/examples/densenet/BUILD
+++ b/tensorflow/contrib/eager/python/examples/densenet/BUILD
@@ -27,6 +27,7 @@ cuda_py_test(
     tags = [
         "no_pip",
         "optonly",
+        "oss_serial",
     ],
 )
 
@@ -45,5 +46,6 @@ cuda_py_test(
         "nomsan",
         "notsan",
         "optonly",
+        "oss_serial",
     ],
 )
diff --git a/tensorflow/contrib/eager/python/examples/l2hmc/BUILD b/tensorflow/contrib/eager/python/examples/l2hmc/BUILD
index 7bdf9053de..78548c51c9 100644
--- a/tensorflow/contrib/eager/python/examples/l2hmc/BUILD
+++ b/tensorflow/contrib/eager/python/examples/l2hmc/BUILD
@@ -36,4 +36,7 @@ cuda_py_test(
         "//tensorflow/contrib/eager/python:tfe",
         "//third_party/py/numpy",
     ],
+    tags = [
+        "oss_serial",
+    ],
 )
diff --git a/tensorflow/contrib/eager/python/examples/linear_regression/BUILD b/tensorflow/contrib/eager/python/examples/linear_regression/BUILD
index 74ce9e84f0..6a178ddcec 100644
--- a/tensorflow/contrib/eager/python/examples/linear_regression/BUILD
+++ b/tensorflow/contrib/eager/python/examples/linear_regression/BUILD
@@ -23,7 +23,10 @@ cuda_py_test(
         ":linear_regression",
         "//tensorflow:tensorflow_py",
     ],
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
+    tags = [
+        "no_windows",  # TODO: needs investigation on Windows
+        "oss_serial",
+    ],
 )
 
 cuda_py_test(
diff --git a/tensorflow/contrib/eager/python/examples/resnet50/BUILD b/tensorflow/contrib/eager/python/examples/resnet50/BUILD
index f3135a9668..04ac78a2d3 100644
--- a/tensorflow/contrib/eager/python/examples/resnet50/BUILD
+++ b/tensorflow/contrib/eager/python/examples/resnet50/BUILD
@@ -40,6 +40,7 @@ cuda_py_test(
         "nomsan",  # Fix b/118130911
         "notsan",  # Fix b/118130911
         "optonly",
+        "oss_serial",
     ],
 )
 
@@ -58,5 +59,6 @@ cuda_py_test(
         "nomsan",
         "notsan",
         "optonly",
+        "oss_serial",
     ],
 )
diff --git a/tensorflow/contrib/eager/python/examples/revnet/BUILD b/tensorflow/contrib/eager/python/examples/revnet/BUILD
index 4f0d46b1ba..0d85bf63ad 100644
--- a/tensorflow/contrib/eager/python/examples/revnet/BUILD
+++ b/tensorflow/contrib/eager/python/examples/revnet/BUILD
@@ -73,6 +73,9 @@ cuda_py_test(
         ":ops",
         "//tensorflow:tensorflow_py",
     ],
+    tags = [
+        "oss_serial",
+    ],
 )
 
 cuda_py_test(
@@ -101,6 +104,7 @@ cuda_py_test(
     tags = [
         "no_pip",  # depends on blocks_test, which is not available in pip package
         "optonly",
+        "oss_serial",
     ],
 )
 
diff --git a/tensorflow/contrib/eager/python/examples/rnn_colorbot/BUILD b/tensorflow/contrib/eager/python/examples/rnn_colorbot/BUILD
index d500b632eb..576f60396e 100644
--- a/tensorflow/contrib/eager/python/examples/rnn_colorbot/BUILD
+++ b/tensorflow/contrib/eager/python/examples/rnn_colorbot/BUILD
@@ -25,4 +25,7 @@ cuda_py_test(
         "//tensorflow/contrib/eager/python:tfe",
         "//tensorflow:tensorflow_py",
     ],
+    tags = [
+        "oss_serial",
+    ],
 )
diff --git a/tensorflow/contrib/eager/python/examples/rnn_ptb/BUILD b/tensorflow/contrib/eager/python/examples/rnn_ptb/BUILD
index 2cc2fcbfeb..f9bf82a7d8 100644
--- a/tensorflow/contrib/eager/python/examples/rnn_ptb/BUILD
+++ b/tensorflow/contrib/eager/python/examples/rnn_ptb/BUILD
@@ -35,4 +35,7 @@ cuda_py_test(
         "//third_party/py/numpy",
         "//tensorflow:tensorflow_py",
     ],
+    tags = [
+        "oss_serial",
+    ],
 )
diff --git a/tensorflow/contrib/eager/python/examples/spinn/BUILD b/tensorflow/contrib/eager/python/examples/spinn/BUILD
index 5966f1d487..9b0fbaa679 100644
--- a/tensorflow/contrib/eager/python/examples/spinn/BUILD
+++ b/tensorflow/contrib/eager/python/examples/spinn/BUILD
@@ -42,5 +42,6 @@ cuda_py_test(
         "no-internal-py3",  # flaky
         "no_cuda_on_cpu_tap",
         "no_pip",  # because spinn.py is under third_party/.
+        "oss_serial",
     ],
 )
diff --git a/tensorflow/contrib/sparsemax/BUILD b/tensorflow/contrib/sparsemax/BUILD
index d7ba754f70..ed4eca1a60 100644
--- a/tensorflow/contrib/sparsemax/BUILD
+++ b/tensorflow/contrib/sparsemax/BUILD
@@ -49,6 +49,9 @@ cuda_py_tests(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:platform_test",
     ],
+    tags = [
+        "oss_serial",
+    ],
 )
 
 cuda_py_tests(
@@ -64,4 +67,7 @@ cuda_py_tests(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:platform_test",
     ],
+    tags = [
+        "oss_serial",
+    ],
 )
-- 
GitLab


From 8a5ddef315a5cd96e4e129bbd2b8ff1fa9f23ef6 Mon Sep 17 00:00:00 2001
From: Reed Wanderman-Milne <reedwm@google.com>
Date: Wed, 16 Jan 2019 12:00:48 -0800
Subject: [PATCH 0828/1095] Fix nn_fused_batchnorm_test.

Also set test size to medium, becuase it takes significantly less than five
minutes to run.

PiperOrigin-RevId: 229600169
---
 tensorflow/python/BUILD                          |  2 +-
 tensorflow/python/ops/nn_fused_batchnorm_test.py | 10 ++++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index adba0b49fa..e32be0039f 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -3516,7 +3516,7 @@ cuda_py_test(
 
 cuda_py_test(
     name = "nn_fused_batchnorm_test",
-    size = "large",
+    size = "medium",
     srcs = ["ops/nn_fused_batchnorm_test.py"],
     additional_deps = [
         ":array_ops",
diff --git a/tensorflow/python/ops/nn_fused_batchnorm_test.py b/tensorflow/python/ops/nn_fused_batchnorm_test.py
index 4e5cff5c16..69e753aa95 100644
--- a/tensorflow/python/ops/nn_fused_batchnorm_test.py
+++ b/tensorflow/python/ops/nn_fused_batchnorm_test.py
@@ -22,6 +22,7 @@ import numpy as np
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import gradients_impl
@@ -436,6 +437,7 @@ class BatchNormalizationTest(test.TestCase):
       self._test_training(
           x_shape, dtype, [6], np.float32, use_gpu=False, data_format='NHWC')
 
+  @test_util.run_deprecated_v1
   def testBatchNormGradShape1(self):
     for is_training in [True, False]:
       x_shape = [1, 1, 6, 1]
@@ -463,6 +465,7 @@ class BatchNormalizationTest(test.TestCase):
             data_format='NHWC',
             is_training=is_training)
 
+  @test_util.run_deprecated_v1
   def testBatchNormGradShape2(self):
     for is_training in [True, False]:
       x_shape = [1, 1, 6, 2]
@@ -483,6 +486,7 @@ class BatchNormalizationTest(test.TestCase):
             data_format='NHWC',
             is_training=is_training)
 
+  @test_util.run_deprecated_v1
   def testBatchNormGradShape3(self):
     for is_training in [True, False]:
       x_shape = [1, 2, 1, 6]
@@ -496,6 +500,7 @@ class BatchNormalizationTest(test.TestCase):
               data_format='NCHW',
               is_training=is_training)
 
+  @test_util.run_deprecated_v1
   def testBatchNormGradShape4(self):
     for is_training in [True, False]:
       x_shape = [5, 7, 11, 4]
@@ -523,6 +528,7 @@ class BatchNormalizationTest(test.TestCase):
             data_format='NHWC',
             is_training=is_training)
 
+  @test_util.run_deprecated_v1
   @test_util.disable_xla('This test never passed for XLA')
   def testBatchNormGradShape5(self):
     for is_training in [True, False]:
@@ -582,6 +588,7 @@ class BatchNormalizationTest(test.TestCase):
           is_training=is_training,
           err_tolerance=err_tolerance)
 
+  @test_util.run_deprecated_v1
   def testBatchNormGradGradConfig1(self):
     config = {
         'shape': [2, 3, 4, 5],
@@ -590,6 +597,7 @@ class BatchNormalizationTest(test.TestCase):
     }
     self._testBatchNormGradGrad(config)
 
+  @test_util.run_deprecated_v1
   def testBatchNormGradGradConfig2(self):
     config = {
         'shape': [2, 3, 2, 2],
@@ -598,6 +606,7 @@ class BatchNormalizationTest(test.TestCase):
     }
     self._testBatchNormGradGrad(config)
 
+  @test_util.run_deprecated_v1
   def testBatchNormGradGradConfig3(self):
     config = {
         'shape': [2, 3, 4, 5],
@@ -606,6 +615,7 @@ class BatchNormalizationTest(test.TestCase):
     }
     self._testBatchNormGradGrad(config)
 
+  @test_util.run_deprecated_v1
   def testBatchNormGradGradConfig4(self):
     config = {
         'shape': [2, 3, 2, 2],
-- 
GitLab


From d6f6485eaee272fa4bd72c9d4625c2520f699e7f Mon Sep 17 00:00:00 2001
From: Andy Ly <lyandy@google.com>
Date: Wed, 16 Jan 2019 12:08:53 -0800
Subject: [PATCH 0829/1095] [Grappler] Improve MutableGraphView tests by
 checking graph and node connectivity with GraphView, and checking node fanins
 and fanouts explicitly.

PiperOrigin-RevId: 229601739
---
 tensorflow/core/grappler/BUILD                |    1 +
 .../core/grappler/mutable_graph_view_test.cc  | 1215 ++++++++---------
 2 files changed, 569 insertions(+), 647 deletions(-)

diff --git a/tensorflow/core/grappler/BUILD b/tensorflow/core/grappler/BUILD
index 475fa64201..bd95e89878 100644
--- a/tensorflow/core/grappler/BUILD
+++ b/tensorflow/core/grappler/BUILD
@@ -233,5 +233,6 @@ tf_cc_test(
         "//tensorflow/core:testlib",
         "//tensorflow/core/grappler/inputs:trivial_test_graph_input_yielder",
         "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/types:span",
     ],
 )
diff --git a/tensorflow/core/grappler/mutable_graph_view_test.cc b/tensorflow/core/grappler/mutable_graph_view_test.cc
index 597bf8a7ee..b5bf33033c 100644
--- a/tensorflow/core/grappler/mutable_graph_view_test.cc
+++ b/tensorflow/core/grappler/mutable_graph_view_test.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/core/grappler/mutable_graph_view.h"
 #include "absl/strings/substitute.h"
+#include "absl/types/span.h"
 #include "tensorflow/cc/ops/standard_ops.h"
 #include "tensorflow/core/framework/function_testlib.h"
 #include "tensorflow/core/framework/types.pb.h"
@@ -29,6 +30,115 @@ namespace grappler {
 namespace {
 
 using ::tensorflow::test::function::NDef;
+using FDH = FunctionDefHelper;
+
+void CompareNodeFanins(const MutableGraphView& graph, NodeDef* node,
+                       absl::Span<const string> fanins) {
+  ASSERT_EQ(node->input_size(), fanins.size());
+  for (int i = 0; i < node->input_size(); ++i) {
+    TensorId tensor_id = ParseTensorName(fanins[i]);
+    EXPECT_EQ(ParseTensorName(node->input(i)), tensor_id);
+    int port;
+    if (tensor_id.index() == Graph::kControlSlot) {
+      port = Graph::kControlSlot;
+    } else {
+      port = i;
+    }
+    MutableGraphView::InputPort input_port(node, port);
+    MutableGraphView::OutputPort output_port =
+        graph.GetOutputPort(tensor_id.node(), tensor_id.index());
+    EXPECT_TRUE(graph.GetFanin(input_port).contains(output_port));
+    EXPECT_TRUE(graph.GetFanout(output_port).contains(input_port));
+  }
+}
+
+void CompareNodeFanouts(const MutableGraphView& graph, NodeDef* node,
+                        absl::Span<const string> fanouts) {
+  auto node_fanouts =
+      graph.GetFanouts(*node, /*include_controlled_nodes=*/true);
+  EXPECT_EQ(node_fanouts.size(), fanouts.size());
+  for (const string& fanout : fanouts) {
+    TensorId tensor_id = ParseTensorName(fanout);
+    MutableGraphView::InputPort input_port(graph.GetNode(tensor_id.node()),
+                                           tensor_id.index());
+    EXPECT_TRUE(node_fanouts.contains(input_port));
+  }
+}
+
+void CheckNode(const MutableGraphView& graph, absl::string_view node_name,
+               absl::string_view op, absl::string_view device,
+               absl::Span<const std::pair<string, FDH::AttrValueWrapper>> attrs,
+               absl::Span<const string> fanins,
+               absl::Span<const string> fanouts) {
+  NodeDef* node = graph.GetNode(node_name);
+  ASSERT_NE(node, nullptr);
+  EXPECT_EQ(node->op(), op);
+  EXPECT_EQ(node->device(), device);
+  EXPECT_EQ(node->attr_size(), attrs.size());
+  for (const auto& attr : attrs) {
+    auto it = node->attr().find(attr.first);
+    ASSERT_NE(it, node->attr().end());
+    EXPECT_TRUE(AreAttrValuesEqual(it->second, attr.second.proto));
+  }
+  CompareNodeFanins(graph, node, fanins);
+  CompareNodeFanouts(graph, node, fanouts);
+}
+
+void CheckGraph(const MutableGraphView& mutable_graph) {
+  GraphView immutable_graph(mutable_graph.graph());
+  EXPECT_EQ(mutable_graph.graph()->node_size(),
+            immutable_graph.graph()->node_size());
+  EXPECT_EQ(mutable_graph.graph(), immutable_graph.graph());
+
+  auto check_edges =
+      [](const absl::flat_hash_set<MutableGraphView::Edge>& mutable_edges,
+         const absl::flat_hash_set<GraphView::Edge>& immutable_edges) {
+        EXPECT_EQ(mutable_edges.size(), immutable_edges.size());
+        for (const auto& fanin_edge : mutable_edges) {
+          GraphView::Edge immutable_edge(
+              {fanin_edge.src.node, fanin_edge.src.port_id},
+              {fanin_edge.dst.node, fanin_edge.dst.port_id});
+          EXPECT_TRUE(immutable_edges.contains(immutable_edge));
+        }
+      };
+
+  // Check graph connectivity.
+  for (auto& node : *mutable_graph.graph()->mutable_node()) {
+    EXPECT_EQ(&node, immutable_graph.GetNode(node.name()));
+
+    auto mutable_fanins =
+        mutable_graph.GetFanins(node, /*include_controlling_nodes=*/true);
+    auto immutable_fanins =
+        immutable_graph.GetFanins(node, /*include_controlling_nodes=*/true);
+    EXPECT_EQ(mutable_fanins.size(), immutable_fanins.size());
+    for (const auto& fanin : mutable_fanins) {
+      GraphView::OutputPort immutable_fanin(fanin.node, fanin.port_id);
+      EXPECT_TRUE(immutable_fanins.contains(immutable_fanin));
+    }
+
+    auto mutable_fanouts =
+        mutable_graph.GetFanouts(node, /*include_controlled_nodes=*/true);
+    auto immutable_fanouts =
+        immutable_graph.GetFanouts(node, /*include_controlled_nodes=*/true);
+    EXPECT_EQ(mutable_fanouts.size(), immutable_fanouts.size());
+    for (const auto& fanout : mutable_fanouts) {
+      GraphView::InputPort immutable_fanout(fanout.node, fanout.port_id);
+      EXPECT_TRUE(immutable_fanouts.contains(immutable_fanout));
+    }
+
+    auto mutable_fanin_edges =
+        mutable_graph.GetFaninEdges(node, /*include_controlling_edges=*/true);
+    auto immutable_fanin_edges =
+        immutable_graph.GetFaninEdges(node, /*include_controlling_edges=*/true);
+    check_edges(mutable_fanin_edges, immutable_fanin_edges);
+
+    auto mutable_fanout_edges =
+        mutable_graph.GetFanoutEdges(node, /*include_controlled_edges=*/true);
+    auto immutable_fanout_edges =
+        immutable_graph.GetFanoutEdges(node, /*include_controlled_edges=*/true);
+    check_edges(mutable_fanout_edges, immutable_fanout_edges);
+  }
+}
 
 TEST(MutableGraphViewTest, AddAndUpdateFanouts) {
   // Actual node.op() is not important in this test.
@@ -43,42 +153,23 @@ TEST(MutableGraphViewTest, AddAndUpdateFanouts) {
   MutableGraphView graph(&graph_def);
 
   NodeDef* new_bar = graph.AddNode(NDef("new_bar", "NotImportant", {}, {}));
-  NodeDef* bar = graph.GetNode("bar");
-
-  EXPECT_TRUE(graph.UpdateFanouts(bar->name(), new_bar->name()).ok());
-
-  // Fanout nodes must have their inputs updated.
-  NodeDef* foo_1 = graph.GetNode("foo_1");
-  ASSERT_NE(foo_1, nullptr);
-  ASSERT_EQ(foo_1->input_size(), 3);
-  EXPECT_EQ(foo_1->input(0), "new_bar");
-  EXPECT_EQ(foo_1->input(1), "other");
-  EXPECT_EQ(foo_1->input(2), "new_bar:1");
-
-  NodeDef* foo_2 = graph.GetNode("foo_2");
-  ASSERT_NE(foo_2, nullptr);
-  ASSERT_EQ(foo_2->input_size(), 2);
-  EXPECT_EQ(foo_2->input(0), "other:1");
-  EXPECT_EQ(foo_2->input(1), "new_bar:2");
-
-  NodeDef* foo_3 = graph.GetNode("foo_3");
-  ASSERT_NE(foo_3, nullptr);
-  ASSERT_EQ(foo_3->input_size(), 2);
-  EXPECT_EQ(foo_3->input(0), "other:2");
-  EXPECT_EQ(foo_3->input(1), "^new_bar");
-
-  // And fanouts mapping must be also updated for both nodes.
-  bool include_control_fanouts = true;
-  auto old_node_fanouts = graph.GetFanouts(*bar, include_control_fanouts);
-  auto new_node_fanouts = graph.GetFanouts(*new_bar, include_control_fanouts);
-
-  EXPECT_TRUE(old_node_fanouts.empty());
-
-  EXPECT_EQ(new_node_fanouts.size(), 4);
-  EXPECT_EQ(new_node_fanouts.count(MutableGraphView::InputPort(foo_1, 0)), 1);
-  EXPECT_EQ(new_node_fanouts.count(MutableGraphView::InputPort(foo_1, 2)), 1);
-  EXPECT_EQ(new_node_fanouts.count(MutableGraphView::InputPort(foo_2, 1)), 1);
-  EXPECT_EQ(new_node_fanouts.count(MutableGraphView::InputPort(foo_3, -1)), 1);
+
+  EXPECT_TRUE(graph.UpdateFanouts("bar", new_bar->name()).ok());
+
+  // Fanins and fanouts must be updated.
+  CheckNode(graph, "bar", "NotImportant", "", {}, {}, {});
+  CheckNode(graph, "other", "NotImportant", "", {}, {},
+            {"foo_1:1", "foo_2", "foo_3"});
+  CheckNode(graph, "foo_1", "NotImportant", "", {},
+            {"new_bar", "other", "new_bar:1"}, {});
+  CheckNode(graph, "foo_2", "NotImportant", "", {}, {"other:1", "new_bar:2"},
+            {});
+  CheckNode(graph, "foo_3", "NotImportant", "", {}, {"other:2", "^new_bar"},
+            {});
+  CheckNode(graph, "new_bar", "NotImportant", "", {}, {},
+            {"foo_1:0", "foo_1:2", "foo_2:1", "^foo_3"});
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, AddAndUpdateFanoutsKeepControls) {
@@ -92,37 +183,21 @@ TEST(MutableGraphViewTest, AddAndUpdateFanoutsKeepControls) {
   MutableGraphView graph(&graph_def);
 
   NodeDef* new_bar = graph.AddNode(NDef("new_bar", "Identity", {"bar_1:2"}));
-  NodeDef* bar_2 = graph.GetNode("bar_2");
-
-  EXPECT_TRUE(graph.UpdateFanouts(bar_2->name(), new_bar->name()).ok());
-
-  // Fanout nodes must have their inputs updated.
-  NodeDef* foo_1 = graph.GetNode("foo_1");
-  ASSERT_NE(foo_1, nullptr);
-  ASSERT_EQ(foo_1->input_size(), 4);
-  EXPECT_EQ(foo_1->input(0), "new_bar");
-  EXPECT_EQ(foo_1->input(1), "other");
-  EXPECT_EQ(foo_1->input(2), "new_bar:1");
-  EXPECT_EQ(foo_1->input(3), "^new_bar");
-
-  NodeDef* foo_2 = graph.GetNode("foo_2");
-  ASSERT_NE(foo_2, nullptr);
-  ASSERT_EQ(foo_2->input_size(), 3);
-  EXPECT_EQ(foo_2->input(0), "other:1");
-  EXPECT_EQ(foo_2->input(1), "new_bar:2");
-  EXPECT_EQ(foo_2->input(2), "^new_bar");
-
-  // And fanouts mapping must be also updated for both nodes.
-  bool include_control_fanouts = true;
-  auto old_node_fanouts = graph.GetFanouts(*bar_2, include_control_fanouts);
-  auto new_node_fanouts = graph.GetFanouts(*new_bar, include_control_fanouts);
-
-  EXPECT_TRUE(old_node_fanouts.empty());
-  EXPECT_EQ(new_node_fanouts.count(MutableGraphView::InputPort(foo_1, 0)), 1);
-  EXPECT_EQ(new_node_fanouts.count(MutableGraphView::InputPort(foo_1, 2)), 1);
-  EXPECT_EQ(new_node_fanouts.count(MutableGraphView::InputPort(foo_1, -1)), 1);
-  EXPECT_EQ(new_node_fanouts.count(MutableGraphView::InputPort(foo_2, 1)), 1);
-  EXPECT_EQ(new_node_fanouts.count(MutableGraphView::InputPort(foo_2, -1)), 1);
+
+  EXPECT_TRUE(graph.UpdateFanouts("bar_2", new_bar->name()).ok());
+
+  // Fanins and fanouts must be updated.
+  CheckNode(graph, "bar_1", "Switch", "", {}, {}, {"bar_2", "new_bar"});
+  CheckNode(graph, "bar_2", "Identity", "", {}, {"bar_1:1"}, {});
+  CheckNode(graph, "other", "NotImportant", "", {}, {}, {"foo_1:1", "foo_2"});
+  CheckNode(graph, "foo_1", "NotImportant", "", {},
+            {"new_bar", "other", "new_bar:1", "^new_bar"}, {});
+  CheckNode(graph, "foo_2", "NotImportant", "", {},
+            {"other:1", "new_bar:2", "^new_bar"}, {});
+  CheckNode(graph, "new_bar", "Identity", "", {}, {"bar_1:2"},
+            {"foo_1", "foo_1:2", "^foo_1", "foo_2:1", "^foo_2"});
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, AddAndUpdateFanoutsWithoutSelfLoops) {
@@ -137,56 +212,16 @@ TEST(MutableGraphViewTest, AddAndUpdateFanoutsWithoutSelfLoops) {
 
   // `new_bar` reads the output of an original `bar` node.
   NodeDef* new_bar = graph.AddNode(NDef("new_bar", "NewBar", {"bar"}, {}));
-  NodeDef* bar = graph.GetNode("bar");
 
   EXPECT_TRUE(graph.UpdateFanouts("bar", new_bar->name()).ok());
 
-  // Foo node must read from `new_bar`.
-  NodeDef* foo_1 = graph.GetNode("foo_1");
-  ASSERT_NE(foo_1, nullptr);
-  ASSERT_EQ(foo_1->input_size(), 1);
-  EXPECT_EQ(foo_1->input(0), "new_bar");
-
-  NodeDef* foo_2 = graph.GetNode("foo_2");
-  ASSERT_NE(foo_2, nullptr);
-  ASSERT_EQ(foo_2->input_size(), 1);
-  EXPECT_EQ(foo_2->input(0), "^new_bar");
-
-  // And the `new_bar` should read from the original `bar`.
-  ASSERT_EQ(new_bar->input_size(), 1);
-  ASSERT_EQ(new_bar->input(0), "bar");
-
-  // And fanouts mapping must be also updated for both nodes.
-  bool include_control_fanouts = true;
-  auto bar_fanouts = graph.GetFanouts(*bar, include_control_fanouts);
-  auto new_bar_fanouts = graph.GetFanouts(*new_bar, include_control_fanouts);
+  // Fanins and fanouts must be updated.
+  CheckNode(graph, "bar", "NotImportant", "", {}, {}, {"new_bar"});
+  CheckNode(graph, "foo_1", "NotImportant", "", {}, {"new_bar"}, {});
+  CheckNode(graph, "foo_2", "NotImportant", "", {}, {"^new_bar"}, {});
+  CheckNode(graph, "new_bar", "NewBar", "", {}, {"bar"}, {"foo_1", "^foo_2"});
 
-  EXPECT_EQ(bar_fanouts.size(), 1);
-  EXPECT_EQ(bar_fanouts.count(MutableGraphView::InputPort(new_bar, 0)), 1);
-
-  EXPECT_EQ(new_bar_fanouts.size(), 2);
-  EXPECT_EQ(new_bar_fanouts.count(MutableGraphView::InputPort(foo_1, 0)), 1);
-  EXPECT_EQ(new_bar_fanouts.count(MutableGraphView::InputPort(foo_2, -1)), 1);
-}
-
-void CompareNodeInputs(const MutableGraphView& graph, const NodeDef* expected,
-                       NodeDef* actual) {
-  ASSERT_EQ(actual->input_size(), expected->input_size());
-  for (int i = 0; i < actual->input_size(); ++i) {
-    EXPECT_EQ(actual->input(i), expected->input(i));
-    TensorId tensor_id = ParseTensorName(expected->input(i));
-    int port;
-    if (tensor_id.index() == Graph::kControlSlot) {
-      port = Graph::kControlSlot;
-    } else {
-      port = i;
-    }
-    MutableGraphView::InputPort input_port(actual, port);
-    MutableGraphView::OutputPort output_port =
-        graph.GetOutputPort(tensor_id.node(), tensor_id.index());
-    EXPECT_EQ(graph.GetFanin(input_port).contains(output_port), true);
-    EXPECT_EQ(graph.GetFanout(output_port).contains(input_port), true);
-  }
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, UpdateFanoutsToSwitchWithControlFromSwitch) {
@@ -213,27 +248,13 @@ TEST(MutableGraphViewTest, UpdateFanoutsToSwitchWithControlFromSwitch) {
 
   EXPECT_EQ(graph.graph()->node_size(), 5);
 
-  NodeDef expected = NDef("", "", {}, {});
-  NodeDef* a = graph.GetNode("a");
-  ASSERT_NE(a, nullptr);
-  CompareNodeInputs(graph, &expected, a);
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {"^e"});
+  CheckNode(graph, "b", "Switch", "", {}, {}, {"e:1"});
+  CheckNode(graph, "c", "NotImportant", "", {}, {}, {"e:0"});
+  CheckNode(graph, "d", "NotImportant", "", {}, {}, {"^e"});
+  CheckNode(graph, "e", "NotImportant", "", {}, {"c", "b", "^a", "^d"}, {});
 
-  NodeDef* b = graph.GetNode("b");
-  ASSERT_NE(b, nullptr);
-  CompareNodeInputs(graph, &expected, b);
-
-  NodeDef* c = graph.GetNode("c");
-  ASSERT_NE(c, nullptr);
-  CompareNodeInputs(graph, &expected, c);
-
-  NodeDef* d = graph.GetNode("d");
-  ASSERT_NE(d, nullptr);
-  CompareNodeInputs(graph, &expected, d);
-
-  NodeDef* e = graph.GetNode("e");
-  ASSERT_NE(e, nullptr);
-  expected = NDef("", "", {"c", "b", "^a", "^d"});
-  CompareNodeInputs(graph, &expected, e);
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, UpdateFanoutsToSwitchWithNoControlFromSwitch) {
@@ -249,27 +270,13 @@ TEST(MutableGraphViewTest, UpdateFanoutsToSwitchWithNoControlFromSwitch) {
 
   EXPECT_EQ(graph.graph()->node_size(), 5);
 
-  NodeDef expected = NDef("", "", {}, {});
-  NodeDef* a = graph.GetNode("a");
-  ASSERT_NE(a, nullptr);
-  CompareNodeInputs(graph, &expected, a);
-
-  NodeDef* b = graph.GetNode("b");
-  ASSERT_NE(b, nullptr);
-  CompareNodeInputs(graph, &expected, b);
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {"^e"});
+  CheckNode(graph, "b", "Switch", "", {}, {}, {"e:0", "e:1"});
+  CheckNode(graph, "c", "NotImportant", "", {}, {}, {});
+  CheckNode(graph, "d", "NotImportant", "", {}, {}, {"^e"});
+  CheckNode(graph, "e", "NotImportant", "", {}, {"b", "b", "^a", "^d"}, {});
 
-  NodeDef* c = graph.GetNode("c");
-  ASSERT_NE(c, nullptr);
-  CompareNodeInputs(graph, &expected, c);
-
-  NodeDef* d = graph.GetNode("d");
-  ASSERT_NE(d, nullptr);
-  CompareNodeInputs(graph, &expected, d);
-
-  NodeDef* e = graph.GetNode("e");
-  ASSERT_NE(e, nullptr);
-  expected = NDef("", "", {"b", "b", "^a", "^d"});
-  CompareNodeInputs(graph, &expected, e);
+  CheckGraph(graph);
 }
 
 GraphDef SimpleMutateFaninGraph() {
@@ -287,129 +294,152 @@ GraphDef SimpleMutateFaninGraph() {
   return graph_def;
 }
 
-void TestAddRegularFanin(absl::string_view node_name,
+absl::flat_hash_map<string, std::vector<string>> GetNodeInputsFromGraph(
+    const GraphDef& graph, absl::string_view node_to_exclude) {
+  absl::flat_hash_map<string, std::vector<string>> node_inputs;
+  for (const auto& node : graph.node()) {
+    if (node.name() == node_to_exclude) {
+      continue;
+    }
+    node_inputs[node.name()] =
+        std::vector<string>(node.input().begin(), node.input().end());
+  }
+  return node_inputs;
+}
+
+void CheckUnmodifiedNodeFanins(
+    const GraphDef& graph, absl::string_view node_to_exclude,
+    const absl::flat_hash_map<string, std::vector<string>>&
+        unmodified_node_inputs) {
+  for (const auto& node : graph.node()) {
+    if (node.name() == node_to_exclude) {
+      continue;
+    }
+    auto it = unmodified_node_inputs.find(node.name());
+    ASSERT_NE(it, unmodified_node_inputs.end());
+    ASSERT_EQ(it->second.size(), node.input_size());
+    for (int i = 0; i < node.input_size(); ++i) {
+      EXPECT_EQ(node.input(i), it->second[i]);
+    }
+  }
+}
+
+void TestAddRegularFanin(absl::string_view node_name, bool node_exists,
                          const TensorId& fanin_to_add, bool success,
                          const string& error_msg,
-                         const NodeDef* expected_node) {
+                         absl::Span<const string> expected_fanins) {
   GraphDef graph_def = SimpleMutateFaninGraph();
 
   MutableGraphView graph(&graph_def);
 
-  auto node = graph.GetNode(node_name);
-  if (expected_node == nullptr) {
-    EXPECT_EQ(node, nullptr);
-  } else {
+  NodeDef* node = graph.GetNode(node_name);
+  if (node_exists) {
     EXPECT_NE(node, nullptr);
+  } else {
+    EXPECT_EQ(node, nullptr);
   }
 
+  absl::flat_hash_map<string, std::vector<string>> unmodified_node_inputs =
+      GetNodeInputsFromGraph(graph_def, node_name);
+
   Status s = graph.AddRegularFanin(node_name, fanin_to_add);
   EXPECT_EQ(s.ok(), success);
   if (!success) {
     EXPECT_EQ(s.error_message(), error_msg);
   }
-  if (expected_node != nullptr) {
-    CompareNodeInputs(graph, expected_node, node);
+  if (node_exists) {
+    CompareNodeFanins(graph, node, expected_fanins);
   }
+
+  CheckUnmodifiedNodeFanins(graph_def, node_name, unmodified_node_inputs);
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, AddRegularFanin) {
-  NodeDef expected_node;
   string error_msg;
   // Add input to node with 1 input 0 controls.
-  expected_node = NDef("", "", {"a", "b:1"});
-  TestAddRegularFanin("foo_1", {"b", 1}, /*success=*/true, error_msg,
-                      &expected_node);
+  TestAddRegularFanin("foo_1", /*node_exists=*/true, {"b", 1}, /*success=*/true,
+                      error_msg, {"a", "b:1"});
   // Add input to node with multiple inputs and 0 controls.
-  expected_node = NDef("", "", {"b", "a:1", "a:1", "b:2"});
-  TestAddRegularFanin("foo_3", {"b", 2}, /*success=*/true, error_msg,
-                      &expected_node);
+  TestAddRegularFanin("foo_3", /*node_exists=*/true, {"b", 2}, /*success=*/true,
+                      error_msg, {"b", "a:1", "a:1", "b:2"});
   // Add input to node with 1 input multiple controls.
-  expected_node = NDef("", "", {"b", "a", "^c"});
-  TestAddRegularFanin("foo_2", {"a", 0}, /*success=*/true, error_msg,
-                      &expected_node);
+  TestAddRegularFanin("foo_2", /*node_exists=*/true, {"a", 0}, /*success=*/true,
+                      error_msg, {"b", "a", "^c"});
   // Add input to node with multiple inputs and controls.
-  expected_node = NDef("", "", {"a", "b:2", "b:2", "a:1", "^d", "^c"});
-  TestAddRegularFanin("foo_4", {"a", 1}, /*success=*/true, error_msg,
-                      &expected_node);
+  TestAddRegularFanin("foo_4", /*node_exists=*/true, {"a", 1}, /*success=*/true,
+                      error_msg, {"a", "b:2", "b:2", "a:1", "^d", "^c"});
   // Add input to node with 0 inputs 0 controls.
-  expected_node = NDef("", "", {"a:1"});
-  TestAddRegularFanin("foo_5", {"a", 1}, /*success=*/true, error_msg,
-                      &expected_node);
+  TestAddRegularFanin("foo_5", /*node_exists=*/true, {"a", 1}, /*success=*/true,
+                      error_msg, {"a:1"});
   // Add input to node with 0 inputs multiple controls.
-  expected_node = NDef("", "", {"c:1", "^b", "^a"});
-  TestAddRegularFanin("foo_6", {"c", 1}, /*success=*/true, error_msg,
-                      &expected_node);
+  TestAddRegularFanin("foo_6", /*node_exists=*/true, {"c", 1}, /*success=*/true,
+                      error_msg, {"c:1", "^b", "^a"});
 
   // Add control to node with 1 input 0 controls.
-  expected_node = NDef("", "", {"a"});
   error_msg = "Can't add invalid fanin '^b' as regular fanin to node 'foo_1'.";
-  TestAddRegularFanin("foo_1", {"b", Graph::kControlSlot}, /*success=*/false,
-                      error_msg, &expected_node);
+  TestAddRegularFanin("foo_1", /*node_exists=*/true, {"b", Graph::kControlSlot},
+                      /*success=*/false, error_msg, {"a"});
   // Add control to node with multiple inputs and 0 controls.
-  expected_node = NDef("", "", {"b", "a:1", "a:1"});
   error_msg = "Can't add invalid fanin '^c' as regular fanin to node 'foo_3'.";
-  TestAddRegularFanin("foo_3", {"c", Graph::kControlSlot}, /*success=*/false,
-                      error_msg, &expected_node);
+  TestAddRegularFanin("foo_3", /*node_exists=*/true, {"c", Graph::kControlSlot},
+                      /*success=*/false, error_msg, {"b", "a:1", "a:1"});
   // Add control to node with 1 input multiple controls.
-  expected_node = NDef("", "", {"b", "^a", "^c"});
   error_msg = "Can't add invalid fanin '^d' as regular fanin to node 'foo_2'.";
-  TestAddRegularFanin("foo_2", {"d", Graph::kControlSlot}, /*success=*/false,
-                      error_msg, &expected_node);
+  TestAddRegularFanin("foo_2", /*node_exists=*/true, {"d", Graph::kControlSlot},
+                      /*success=*/false, error_msg, {"b", "^a", "^c"});
   // Add control to node with multiple input multiple controls.
-  expected_node = NDef("", "", {"a", "b:2", "b:2", "^c", "^d"});
   error_msg = "Can't add invalid fanin '^a' as regular fanin to node 'foo_4'.";
-  TestAddRegularFanin("foo_4", {"a", Graph::kControlSlot},
-                      /*success=*/false, error_msg, &expected_node);
+  TestAddRegularFanin("foo_4", /*node_exists=*/true, {"a", Graph::kControlSlot},
+                      /*success=*/false, error_msg,
+                      {"a", "b:2", "b:2", "^c", "^d"});
   // Add control to node with 0 inputs 0 controls.
-  expected_node = NDef("", "", {});
   error_msg = "Can't add invalid fanin '^a' as regular fanin to node 'foo_5'.";
-  TestAddRegularFanin("foo_5", {"a", Graph::kControlSlot}, /*success=*/false,
-                      error_msg, &expected_node);
+  TestAddRegularFanin("foo_5", /*node_exists=*/true, {"a", Graph::kControlSlot},
+                      /*success=*/false, error_msg, {});
   // Add control to node with 0 inputs multiple controls.
-  expected_node = NDef("", "", {"^a", "^b"});
   error_msg = "Can't add invalid fanin '^c' as regular fanin to node 'foo_6'.";
-  TestAddRegularFanin("foo_6", {"c", Graph::kControlSlot}, /*success=*/false,
-                      error_msg, &expected_node);
+  TestAddRegularFanin("foo_6", /*node_exists=*/true, {"c", Graph::kControlSlot},
+                      /*success=*/false, error_msg, {"^a", "^b"});
   // Add control to node with control that already exists.
-  expected_node = NDef("", "", {"b", "^a", "^c"});
   error_msg = "Can't add invalid fanin '^a' as regular fanin to node 'foo_2'.";
-  TestAddRegularFanin("foo_2", {"a", Graph::kControlSlot},
-                      /*success=*/false, error_msg, &expected_node);
+  TestAddRegularFanin("foo_2", /*node_exists=*/true, {"a", Graph::kControlSlot},
+                      /*success=*/false, error_msg, {"b", "^a", "^c"});
 
   // Add fanin to node where node is missing.
   error_msg =
       "Can't add fanin 'a:0' as regular fanin to missing node 'foo_missing'.";
-  TestAddRegularFanin("foo_missing", {"a", 0}, /*success=*/false, error_msg,
-                      nullptr);
+  TestAddRegularFanin("foo_missing", /*node_exists=*/false, {"a", 0},
+                      /*success=*/false, error_msg, {});
   // Add fanin to node where fanin is missing.
-  expected_node = NDef("", "", {"a"});
   error_msg =
       "Can't add missing fanin 'bar_missing:0' as regular fanin to node "
       "'foo_1'.";
-  TestAddRegularFanin("foo_1", {"bar_missing", 0}, /*success=*/false, error_msg,
-                      &expected_node);
+  TestAddRegularFanin("foo_1", /*node_exists=*/true, {"bar_missing", 0},
+                      /*success=*/false, error_msg, {"a"});
   // Add fanin to node where node and fanin are missing.
   error_msg =
       "Can't add missing fanin 'bar_missing:0' as regular fanin to missing "
       "node 'foo_missing'.";
-  TestAddRegularFanin("foo_missing", {"bar_missing", 0}, /*success=*/false,
-                      error_msg, /*expected_node=*/nullptr);
+  TestAddRegularFanin("foo_missing", /*node_exists=*/false, {"bar_missing", 0},
+                      /*success=*/false, error_msg, {});
   // Add control fanin to node where node and fanin are missing.
   error_msg =
       "Can't add invalid/missing fanin '^bar_missing' as regular fanin to "
       "missing node 'foo_missing'.";
-  TestAddRegularFanin("foo_missing", {"bar_missing", Graph::kControlSlot},
-                      /*success=*/false, error_msg, /*expected_node=*/nullptr);
+  TestAddRegularFanin("foo_missing", /*node_exists=*/false,
+                      {"bar_missing", Graph::kControlSlot},
+                      /*success=*/false, error_msg, {});
 
   // Add self to create cycle.
   error_msg = "Can't add fanin 'foo_6:2' as regular fanin to self.";
-  expected_node = NDef("", "", {"^a", "^b"});
-  TestAddRegularFanin("foo_6", {"foo_6", 2}, /*success=*/false, error_msg,
-                      &expected_node);
+  TestAddRegularFanin("foo_6", /*node_exists=*/true, {"foo_6", 2},
+                      /*success=*/false, error_msg, {"^a", "^b"});
 }
 
-void CheckFanout(const MutableGraphView& graph, const TensorId& fanin,
-                 absl::string_view node_name) {
+void CheckFanoutRemoved(const MutableGraphView& graph, const TensorId& fanin,
+                        absl::string_view node_name) {
   MutableGraphView::OutputPort output_port =
       graph.GetOutputPort(fanin.node(), fanin.index());
   auto fanouts = graph.GetFanout(output_port);
@@ -418,166 +448,166 @@ void CheckFanout(const MutableGraphView& graph, const TensorId& fanin,
   }
 }
 
-void TestRemoveRegularFanin(absl::string_view node_name,
+void TestRemoveRegularFanin(absl::string_view node_name, bool node_exists,
                             const TensorId& fanin_to_remove, bool success,
                             const string& error_msg,
-                            const NodeDef* expected_node) {
+                            absl::Span<const string> expected_fanins) {
   GraphDef graph_def = SimpleMutateFaninGraph();
 
   MutableGraphView graph(&graph_def);
 
-  auto node = graph.GetNode(node_name);
-  if (expected_node == nullptr) {
-    EXPECT_EQ(nullptr, node);
-  } else {
+  NodeDef* node = graph.GetNode(node_name);
+  if (node_exists) {
     EXPECT_NE(nullptr, node);
+  } else {
+    EXPECT_EQ(nullptr, node);
   }
 
+  absl::flat_hash_map<string, std::vector<string>> unmodified_node_inputs =
+      GetNodeInputsFromGraph(graph_def, node_name);
+
   Status s = graph.RemoveRegularFanin(node_name, fanin_to_remove);
   EXPECT_EQ(s.ok(), success);
   if (!success) {
     EXPECT_EQ(s.error_message(), error_msg);
   }
-  if (expected_node != nullptr) {
-    CompareNodeInputs(graph, expected_node, node);
+  if (node_exists) {
+    CompareNodeFanins(graph, node, expected_fanins);
     if (success) {
-      CheckFanout(graph, fanin_to_remove, node_name);
+      CheckFanoutRemoved(graph, fanin_to_remove, node_name);
     }
   }
+
+  CheckUnmodifiedNodeFanins(graph_def, node_name, unmodified_node_inputs);
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, RemoveRegularFanin) {
-  NodeDef expected_node;
   string error_msg;
   // Remove input from node with 1 input 0 controls.
-  expected_node = NDef("", "", {});
-  TestRemoveRegularFanin("foo_1", {"a", 0}, /*success=*/true, error_msg,
-                         &expected_node);
+  TestRemoveRegularFanin("foo_1", /*node_exists=*/true, {"a", 0},
+                         /*success=*/true, error_msg, {});
   // Remove input from node with multiple inputs and 0 controls.
-  expected_node = NDef("", "", {"b"});
-  TestRemoveRegularFanin("foo_3", {"a", 1}, /*success=*/true, error_msg,
-                         &expected_node);
+  TestRemoveRegularFanin("foo_3", /*node_exists=*/true, {"a", 1},
+                         /*success=*/true, error_msg, {"b"});
   // Remove input from node with 1 input multiple controls.
-  expected_node = NDef("", "", {"^a", "^c"});
-  TestRemoveRegularFanin("foo_2", {"b", 0}, /*success=*/true, error_msg,
-                         &expected_node);
+  TestRemoveRegularFanin("foo_2", /*node_exists=*/true, {"b", 0},
+                         /*success=*/true, error_msg, {"^a", "^c"});
   // Remove input from node with multiple inputs and controls.
-  expected_node = NDef("", "", {"a", "^c", "^d"});
-  TestRemoveRegularFanin("foo_4", {"b", 2}, /*success=*/true, error_msg,
-                         &expected_node);
+  TestRemoveRegularFanin("foo_4", /*node_exists=*/true, {"b", 2},
+                         /*success=*/true, error_msg, {"a", "^c", "^d"});
   // Remove input from node with multiple inputs and controls, and results in
   // shifting of ports.
-  expected_node = NDef("", "", {"b:2", "b:2", "^c", "^d"});
-  TestRemoveRegularFanin("foo_4", {"a", 0}, /*success=*/true, error_msg,
-                         &expected_node);
+  TestRemoveRegularFanin("foo_4", /*node_exists=*/true, {"a", 0},
+                         /*success=*/true, error_msg,
+                         {"b:2", "b:2", "^c", "^d"});
 
   // Remove control from node with 1 input multiple controls.
-  expected_node = NDef("", "", {"b", "^a", "^c"});
   error_msg =
       "Can't remove invalid fanin '^a' as regular fanin from node 'foo_2'.";
-  TestRemoveRegularFanin("foo_2", {"a", Graph::kControlSlot},
-                         /*success=*/false, error_msg, &expected_node);
+  TestRemoveRegularFanin("foo_2", /*node_exists=*/true,
+                         {"a", Graph::kControlSlot},
+                         /*success=*/false, error_msg, {"b", "^a", "^c"});
   // Remove control from node with multiple input multiple controls.
-  expected_node = NDef("", "", {"a", "b:2", "b:2", "^c", "^d"});
   error_msg =
       "Can't remove invalid fanin '^d' as regular fanin from node 'foo_4'.";
-  TestRemoveRegularFanin("foo_4", {"d", Graph::kControlSlot},
-                         /*success=*/false, error_msg, &expected_node);
+  TestRemoveRegularFanin(
+      "foo_4", /*node_exists=*/true, {"d", Graph::kControlSlot},
+      /*success=*/false, error_msg, {"a", "b:2", "b:2", "^c", "^d"});
   // Remove control from node with 0 inputs multiple controls.
-  expected_node = NDef("", "", {"^a", "^b"});
   error_msg =
       "Can't remove invalid fanin '^a' as regular fanin from node 'foo_6'.";
-  TestRemoveRegularFanin("foo_6", {"a", Graph::kControlSlot},
-                         /*success=*/false, error_msg, &expected_node);
+  TestRemoveRegularFanin("foo_6", /*node_exists=*/true,
+                         {"a", Graph::kControlSlot},
+                         /*success=*/false, error_msg, {"^a", "^b"});
 
   // Remove input from node with 0 inputs 0 controls.
-  expected_node = NDef("", "", {});
   error_msg = "";
-  TestRemoveRegularFanin("foo_5", {"a", 1}, /*success=*/true, error_msg,
-                         &expected_node);
+  TestRemoveRegularFanin("foo_5", /*node_exists=*/true, {"a", 1},
+                         /*success=*/true, error_msg, {});
   // Remove input from node with 0 inputs multiple controls.
-  expected_node = NDef("", "", {"^a", "^b"});
-  TestRemoveRegularFanin("foo_6", {"a", 1}, /*success=*/true, error_msg,
-                         &expected_node);
+  TestRemoveRegularFanin("foo_6", /*node_exists=*/true, {"a", 1},
+                         /*success=*/true, error_msg, {"^a", "^b"});
 
   // Remove control from node with 1 input 0 controls.
-  expected_node = NDef("", "", {"a"});
   error_msg =
       "Can't remove invalid fanin '^b' as regular fanin from node 'foo_1'.";
-  TestRemoveRegularFanin("foo_1", {"b", Graph::kControlSlot},
-                         /*success=*/false, error_msg, &expected_node);
+  TestRemoveRegularFanin("foo_1", /*node_exists=*/true,
+                         {"b", Graph::kControlSlot},
+                         /*success=*/false, error_msg, {"a"});
   // Remove control from node with multiple inputs and 0 controls.
-  expected_node = NDef("", "", {"b", "a:1", "a:1"});
   error_msg =
       "Can't remove invalid fanin '^c' as regular fanin from node 'foo_3'.";
-  TestRemoveRegularFanin("foo_3", {"c", Graph::kControlSlot},
-                         /*success=*/false, error_msg, &expected_node);
+  TestRemoveRegularFanin("foo_3", /*node_exists=*/true,
+                         {"c", Graph::kControlSlot},
+                         /*success=*/false, error_msg, {"b", "a:1", "a:1"});
   // Remove control from node with 0 inputs 0 controls.
-  expected_node = NDef("", "", {});
   error_msg =
       "Can't remove invalid fanin '^a' as regular fanin from node 'foo_5'.";
-  TestRemoveRegularFanin("foo_5", {"a", Graph::kControlSlot},
-                         /*success=*/false, error_msg, &expected_node);
+  TestRemoveRegularFanin("foo_5", /*node_exists=*/true,
+                         {"a", Graph::kControlSlot},
+                         /*success=*/false, error_msg, {});
 
   // Remove fanin from node where node is missing.
   error_msg =
       "Can't remove fanin 'a:0' as regular fanin from missing node "
       "'foo_missing'.";
-  TestRemoveRegularFanin("foo_missing", {"a", 0}, /*success=*/false, error_msg,
-                         /*expected_node=*/nullptr);
+  TestRemoveRegularFanin("foo_missing", /*node_exists=*/false, {"a", 0},
+                         /*success=*/false, error_msg, {});
   // Remove fanin from node where fanin is missing.
-  expected_node = NDef("", "", {"a"});
   error_msg =
       "Can't remove missing fanin 'bar_missing:0' as regular fanin from node "
       "'foo_1'.";
-  TestRemoveRegularFanin("foo_1", {"bar_missing", 0}, /*success=*/false,
-                         error_msg, &expected_node);
+  TestRemoveRegularFanin("foo_1", /*node_exists=*/true, {"bar_missing", 0},
+                         /*success=*/false, error_msg, {"a"});
   // Remove fanin from node where node and fanin are missing.
   error_msg =
       "Can't remove missing fanin 'bar_missing:0' as regular fanin from "
       "missing node 'foo_missing'.";
-  TestRemoveRegularFanin("foo_missing", {"bar_missing", 0}, /*success=*/false,
-                         error_msg,
-                         /*expected_node=*/nullptr);
+  TestRemoveRegularFanin("foo_missing", /*node_exists=*/false,
+                         {"bar_missing", 0}, /*success=*/false, error_msg, {});
   // Remove control from node where node and fanin are missing.
   error_msg =
       "Can't remove invalid/missing fanin '^bar_missing' as regular fanin from "
       "missing node 'foo_missing'.";
-  TestRemoveRegularFanin("foo_missing", {"bar_missing", Graph::kControlSlot},
-                         /*success=*/false, error_msg,
-                         /*expected_node=*/nullptr);
+  TestRemoveRegularFanin("foo_missing", /*node_exists=*/false,
+                         {"bar_missing", Graph::kControlSlot},
+                         /*success=*/false, error_msg, {});
 
   // Remove self.
   error_msg = "Can't remove fanin 'foo_6:2' as regular fanin from self.";
-  expected_node = NDef("", "", {"^a", "^b"});
-  TestRemoveRegularFanin("foo_6", {"foo_6", 2}, /*success=*/false, error_msg,
-                         &expected_node);
+  TestRemoveRegularFanin("foo_6", /*node_exists=*/true, {"foo_6", 2},
+                         /*success=*/false, error_msg, {"^a", "^b"});
 }
 
-void TestRemoveAllFanins(absl::string_view node_name,
+void TestRemoveAllFanins(absl::string_view node_name, bool node_exists,
                          bool keep_controlling_nodes, bool success,
                          const string& error_msg,
-                         const NodeDef* expected_node) {
+                         absl::Span<const string> expected_fanins) {
   GraphDef graph_def = SimpleMutateFaninGraph();
 
   MutableGraphView graph(&graph_def);
 
-  auto node = graph.GetNode(node_name);
+  NodeDef* node = graph.GetNode(node_name);
   absl::flat_hash_set<string> fanin_strings;
-  if (expected_node == nullptr) {
-    EXPECT_EQ(node, nullptr);
-  } else {
+  if (node_exists) {
     EXPECT_NE(node, nullptr);
     fanin_strings.insert(node->input().begin(), node->input().end());
+  } else {
+    EXPECT_EQ(node, nullptr);
   }
 
+  absl::flat_hash_map<string, std::vector<string>> unmodified_node_inputs =
+      GetNodeInputsFromGraph(graph_def, node_name);
+
   Status s = graph.RemoveAllFanins(node_name, keep_controlling_nodes);
   EXPECT_EQ(s.ok(), success);
   if (!success) {
     EXPECT_EQ(s.error_message(), error_msg);
   }
-  if (expected_node != nullptr) {
-    CompareNodeInputs(graph, expected_node, node);
+  if (node_exists) {
+    CompareNodeFanins(graph, node, expected_fanins);
     if (success) {
       TensorId tensor_id;
       auto retained_inputs = absl::flat_hash_set<string>(node->input().begin(),
@@ -585,177 +615,192 @@ void TestRemoveAllFanins(absl::string_view node_name,
       for (const string& fanin : fanin_strings) {
         if (!retained_inputs.contains(fanin)) {
           tensor_id = ParseTensorName(fanin);
-          CheckFanout(graph, tensor_id, node_name);
+          CheckFanoutRemoved(graph, tensor_id, node_name);
         }
       }
     }
   }
+
+  CheckUnmodifiedNodeFanins(graph_def, node_name, unmodified_node_inputs);
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, RemoveAllFanins) {
-  NodeDef expected_node;
   string error_msg;
   // Remove all fanins from node with no control dependencies.
-  expected_node = NDef("", "", {});
-  TestRemoveAllFanins("foo_3", /*keep_controlling_nodes=*/false,
-                      /*success=*/true, error_msg, &expected_node);
+  TestRemoveAllFanins("foo_3", /*node_exists=*/true,
+                      /*keep_controlling_nodes=*/false,
+                      /*success=*/true, error_msg, {});
   // Remove all fanins from node with control dependencies.
-  TestRemoveAllFanins("foo_4", /*keep_controlling_nodes=*/false,
-                      /*success=*/true, error_msg, &expected_node);
+  TestRemoveAllFanins("foo_4", /*node_exists=*/true,
+                      /*keep_controlling_nodes=*/false,
+                      /*success=*/true, error_msg, {});
 
   // Remove all fanins from node with no control dependencies and preserve
   // control dependencies.
-  TestRemoveAllFanins("foo_3", /*keep_controlling_nodes=*/true,
-                      /*success=*/true, error_msg, &expected_node);
+  TestRemoveAllFanins("foo_3", /*node_exists=*/true,
+                      /*keep_controlling_nodes=*/true,
+                      /*success=*/true, error_msg, {});
   // Remove all fanins from node with control dependencies and preserve control
   // dependencies.
-  expected_node = NDef("", "", {"^c", "^d"});
-  TestRemoveAllFanins("foo_4", /*keep_controlling_nodes=*/true,
-                      /*success=*/true, error_msg, &expected_node);
+  TestRemoveAllFanins("foo_4", /*node_exists=*/true,
+                      /*keep_controlling_nodes=*/true,
+                      /*success=*/true, error_msg, {"^c", "^d"});
 
   // Remove all fanins from node with no fanins.
-  expected_node = NDef("", "", {});
-  TestRemoveAllFanins("foo_5", /*keep_controlling_nodes=*/false,
-                      /*success=*/true, error_msg, &expected_node);
-  TestRemoveAllFanins("foo_5", /*keep_controlling_nodes=*/true,
-                      /*success=*/true, error_msg, &expected_node);
+  TestRemoveAllFanins("foo_5", /*node_exists=*/true,
+                      /*keep_controlling_nodes=*/false,
+                      /*success=*/true, error_msg, {});
+  TestRemoveAllFanins("foo_5", /*node_exists=*/true,
+                      /*keep_controlling_nodes=*/true,
+                      /*success=*/true, error_msg, {});
 
   // Remove all fanins from node with only control dependencies.
-  TestRemoveAllFanins("foo_6", /*keep_controlling_nodes=*/false,
-                      /*success=*/true, error_msg, &expected_node);
-  expected_node = NDef("", "", {"^a", "^b"});
-  TestRemoveAllFanins("foo_6", /*keep_controlling_nodes=*/true,
-                      /*success=*/true, error_msg, &expected_node);
+  TestRemoveAllFanins("foo_6", /*node_exists=*/true,
+                      /*keep_controlling_nodes=*/false,
+                      /*success=*/true, error_msg, {});
+  TestRemoveAllFanins("foo_6", /*node_exists=*/true,
+                      /*keep_controlling_nodes=*/true,
+                      /*success=*/true, error_msg, {"^a", "^b"});
 
   // Remove all fanins from node where node is missing.
   error_msg = "Can't remove all fanins from missing node 'foo_missing'.";
-  TestRemoveAllFanins("foo_missing", /*keep_controlling_nodes=*/false,
-                      /*success=*/false, error_msg, /*expected_node=*/nullptr);
-  TestRemoveAllFanins("foo_missing", /*keep_controlling_nodes=*/true,
-                      /*success=*/false, error_msg, /*expected_node=*/nullptr);
+  TestRemoveAllFanins("foo_missing", /*node_exists=*/false,
+                      /*keep_controlling_nodes=*/false,
+                      /*success=*/false, error_msg, {});
+  TestRemoveAllFanins("foo_missing", /*node_exists=*/false,
+                      /*keep_controlling_nodes=*/true,
+                      /*success=*/false, error_msg, {});
 }
 
-void TestUpdateFanin(absl::string_view node_name, const TensorId& from_fanin,
-                     const TensorId& to_fanin, bool success,
-                     const string& error_msg, const NodeDef* expected_node) {
+void TestUpdateFanin(absl::string_view node_name, bool node_exists,
+                     const TensorId& from_fanin, const TensorId& to_fanin,
+                     bool success, const string& error_msg,
+                     absl::Span<const string> expected_fanins) {
   GraphDef graph_def = SimpleMutateFaninGraph();
 
   MutableGraphView graph(&graph_def);
 
-  auto node = graph.GetNode(node_name);
-  if (expected_node == nullptr) {
-    EXPECT_EQ(node, nullptr);
-  } else {
+  NodeDef* node = graph.GetNode(node_name);
+  if (node_exists) {
     EXPECT_NE(node, nullptr);
+  } else {
+    EXPECT_EQ(node, nullptr);
   }
 
+  absl::flat_hash_map<string, std::vector<string>> unmodified_node_inputs =
+      GetNodeInputsFromGraph(graph_def, node_name);
+
   Status s = graph.UpdateFanin(node_name, from_fanin, to_fanin);
   EXPECT_EQ(s.ok(), success);
   if (!success) {
     EXPECT_EQ(s.error_message(), error_msg);
   }
-  if (expected_node != nullptr) {
-    CompareNodeInputs(graph, expected_node, node);
+  if (node_exists) {
+    CompareNodeFanins(graph, node, expected_fanins);
     if (success) {
-      CheckFanout(graph, from_fanin, node_name);
+      CheckFanoutRemoved(graph, from_fanin, node_name);
     }
   }
+
+  CheckUnmodifiedNodeFanins(graph_def, node_name, unmodified_node_inputs);
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, UpdateFanin) {
-  NodeDef expected_node;
   string error_msg;
   // Update fanin from non control to non control.
-  expected_node = NDef("", "", {"a", "b:3", "b:3", "^c", "^d"});
-  TestUpdateFanin("foo_4", {"b", 2}, {"b", 3}, /*success=*/true, error_msg,
-                  &expected_node);
+  TestUpdateFanin("foo_4", /*node_exists=*/true, {"b", 2}, {"b", 3},
+                  /*success=*/true, error_msg, {"a", "b:3", "b:3", "^c", "^d"});
   // Update fanin from non control to control.
-  expected_node = NDef("", "", {"a", "^c", "^d", "^b"});
-  TestUpdateFanin("foo_4", {"b", 2}, {"b", Graph::kControlSlot},
-                  /*success=*/true, error_msg, &expected_node);
+  TestUpdateFanin("foo_4", /*node_exists=*/true, {"b", 2},
+                  {"b", Graph::kControlSlot},
+                  /*success=*/true, error_msg, {"a", "^c", "^d", "^b"});
   // Update fanin from control to non control.
-  expected_node = NDef("", "", {"a", "b:2", "b:2", "d:1", "^c"});
-  TestUpdateFanin("foo_4", {"d", Graph::kControlSlot}, {"d", 1},
-                  /*success=*/true, error_msg, &expected_node);
+  TestUpdateFanin(
+      "foo_4", /*node_exists=*/true, {"d", Graph::kControlSlot}, {"d", 1},
+      /*success=*/true, error_msg, {"a", "b:2", "b:2", "d:1", "^c"});
   // Update fanin from control to control.
-  expected_node = NDef("", "", {"a", "b:2", "b:2", "^d"});
-  TestUpdateFanin("foo_4", {"c", Graph::kControlSlot},
+  TestUpdateFanin("foo_4", /*node_exists=*/true, {"c", Graph::kControlSlot},
                   {"b", Graph::kControlSlot}, /*success=*/true, error_msg,
-                  &expected_node);
+                  {"a", "b:2", "b:2", "^d"});
   // Update fanin from control to existing control.
-  expected_node = NDef("", "", {"a", "b:2", "b:2", "^d"});
-  TestUpdateFanin("foo_4", {"c", Graph::kControlSlot},
+  TestUpdateFanin("foo_4", /*node_exists=*/true, {"c", Graph::kControlSlot},
                   {"d", Graph::kControlSlot}, /*success=*/true, error_msg,
-                  &expected_node);
+                  {"a", "b:2", "b:2", "^d"});
 
   // Update fanin of node where from and to fanins are the same.
-  expected_node = NDef("", "", {"a"});
-  TestUpdateFanin("foo_1", {"a", -1}, {"a", -1}, /*success=*/true, error_msg,
-                  &expected_node);
-  TestUpdateFanin("foo_1", {"a", 0}, {"a", 0}, /*success=*/true, error_msg,
-                  &expected_node);
-  TestUpdateFanin("foo_1", {"a", 1}, {"a", 1}, /*success=*/true, error_msg,
-                  &expected_node);
+  TestUpdateFanin("foo_1", /*node_exists=*/true, {"a", -1}, {"a", -1},
+                  /*success=*/true, error_msg, {"a"});
+  TestUpdateFanin("foo_1", /*node_exists=*/true, {"a", 0}, {"a", 0},
+                  /*success=*/true, error_msg, {"a"});
+  TestUpdateFanin("foo_1", /*node_exists=*/true, {"a", 1}, {"a", 1},
+                  /*success=*/true, error_msg, {"a"});
 
   // Update fanin of node where node is missing.
   error_msg =
       "Can't update fanin 'a:0' to fanin 'a:1' in missing node 'foo_missing'.";
-  TestUpdateFanin("foo_missing", {"a", 0}, {"a", 1}, /*success=*/false,
-                  error_msg,
-                  /*expected_node=*/nullptr);
+  TestUpdateFanin("foo_missing", /*node_exists=*/false, {"a", 0}, {"a", 1},
+                  /*success=*/false, error_msg, {});
   // Update fanin of node where from fanin is missing.
   error_msg =
       "Can't update missing fanin 'from_bar_missing:0' to fanin 'a:1' in node "
       "'foo_1'.";
-  TestUpdateFanin("foo_1", {"from_bar_missing", 0}, {"a", 1},
-                  /*success=*/false, error_msg, &expected_node);
+  TestUpdateFanin("foo_1", /*node_exists=*/true, {"from_bar_missing", 0},
+                  {"a", 1},
+                  /*success=*/false, error_msg, {"a"});
   // Update fanin of node where to fanin is missing.
   error_msg =
       "Can't update fanin 'a:0' to missing fanin 'to_bar_missing:1' in node "
       "'foo_1'.";
-  TestUpdateFanin("foo_1", {"a", 0}, {"to_bar_missing", 1}, /*success=*/false,
-                  error_msg, &expected_node);
+  TestUpdateFanin("foo_1", /*node_exists=*/true, {"a", 0},
+                  {"to_bar_missing", 1}, /*success=*/false, error_msg, {"a"});
   // Update fanin of node where from/to fanins and node are missing.
   error_msg =
       "Can't update missing fanin 'from_bar_missing:0' to missing fanin "
       "'to_bar_missing:1' in missing node 'foo_missing'.";
-  TestUpdateFanin("foo_missing", {"from_bar_missing", 0}, {"to_bar_missing", 1},
-                  /*success=*/false, error_msg, /*expected_node=*/nullptr);
+  TestUpdateFanin("foo_missing", /*node_exists=*/false, {"from_bar_missing", 0},
+                  {"to_bar_missing", 1},
+                  /*success=*/false, error_msg, {});
   // Update fanin of node where from fanin is invalid.
   error_msg =
       "Can't update invalid fanin 'a:-2' to fanin 'a:0' in node 'foo_1'.";
-  TestUpdateFanin("foo_1", {"a", -2}, {"a", 0},
-                  /*success=*/false, error_msg, &expected_node);
+  TestUpdateFanin("foo_1", /*node_exists=*/true, {"a", -2}, {"a", 0},
+                  /*success=*/false, error_msg, {"a"});
   // Update fanin of node where to fanin is invalid.
   error_msg =
       "Can't update fanin 'a:0' to invalid fanin 'a:-2' in node 'foo_1'.";
-  TestUpdateFanin("foo_1", {"a", 0}, {"a", -2},
-                  /*success=*/false, error_msg, &expected_node);
+  TestUpdateFanin("foo_1", /*node_exists=*/true, {"a", 0}, {"a", -2},
+                  /*success=*/false, error_msg, {"a"});
   // Update fanin of node where from/to fanins are invalid and missing and node
   // is missing.
   error_msg =
       "Can't update invalid/missing fanin 'from_bar_missing:-2' to "
       "invalid/missing fanin 'to_bar_missing:-3' in missing node "
       "'foo_missing'.";
-  TestUpdateFanin("foo_missing", {"from_bar_missing", -2},
-                  {"to_bar_missing", -3},
-                  /*success=*/false, error_msg, /*expected_node=*/nullptr);
+  TestUpdateFanin("foo_missing", /*node_exists=*/false,
+                  {"from_bar_missing", -2}, {"to_bar_missing", -3},
+                  /*success=*/false, error_msg, {});
 
   // Update to self to create cycle.
-  expected_node = NDef("", "", {"a", "b:2", "b:2", "^c", "^d"});
   error_msg = "Can't update fanin 'b:2' to fanin 'foo_4:3' in self 'foo_4'.";
-  TestUpdateFanin("foo_4", {"b", 2}, {"foo_4", 3}, /*success=*/false, error_msg,
-                  &expected_node);
+  TestUpdateFanin("foo_4", /*node_exists=*/true, {"b", 2}, {"foo_4", 3},
+                  /*success=*/false, error_msg,
+                  {"a", "b:2", "b:2", "^c", "^d"});
   error_msg = "Can't update fanin 'b:2' to fanin '^foo_4' in self 'foo_4'.";
-  TestUpdateFanin("foo_4", {"b", 2}, {"foo_4", Graph::kControlSlot},
-                  /*success=*/false, error_msg, &expected_node);
+  TestUpdateFanin(
+      "foo_4", /*node_exists=*/true, {"b", 2}, {"foo_4", Graph::kControlSlot},
+      /*success=*/false, error_msg, {"a", "b:2", "b:2", "^c", "^d"});
   error_msg = "Can't update fanin '^c' to fanin 'foo_4:4' in self 'foo_4'.";
-  TestUpdateFanin("foo_4", {"c", Graph::kControlSlot}, {"foo_4", 4},
-                  /*success=*/false, error_msg, &expected_node);
+  TestUpdateFanin(
+      "foo_4", /*node_exists=*/true, {"c", Graph::kControlSlot}, {"foo_4", 4},
+      /*success=*/false, error_msg, {"a", "b:2", "b:2", "^c", "^d"});
   error_msg = "Can't update fanin '^c' to fanin '^foo_4' in self 'foo_4'.";
-  TestUpdateFanin("foo_4", {"c", Graph::kControlSlot},
+  TestUpdateFanin("foo_4", /*node_exists=*/true, {"c", Graph::kControlSlot},
                   {"foo_4", Graph::kControlSlot}, /*success=*/false, error_msg,
-                  &expected_node);
+                  {"a", "b:2", "b:2", "^c", "^d"});
 }
 
 void TestUpdateFaninFromFaninToNodeAsSwitchControl(const TensorId& fanin) {
@@ -777,20 +822,12 @@ void TestUpdateFaninFromFaninToNodeAsSwitchControl(const TensorId& fanin) {
 
   EXPECT_EQ(graph.graph()->node_size(), 3);
 
-  NodeDef expected;
-  NodeDef* a = graph.GetNode("a");
-  ASSERT_NE(a, nullptr);
-  expected = NDef("", "", {});
-  CompareNodeInputs(graph, &expected, a);
-
-  NodeDef* b = graph.GetNode("b");
-  ASSERT_NE(b, nullptr);
-  CompareNodeInputs(graph, &expected, b);
+  string fanout = IsControlInput(fanin) ? AsControlDependency("c") : "c";
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {fanout});
+  CheckNode(graph, "b", "Switch", "", {}, {}, {});
+  CheckNode(graph, "c", "NotImportant", "", {}, {tensor_id_str}, {});
 
-  NodeDef* c = graph.GetNode("c");
-  ASSERT_NE(c, nullptr);
-  expected = NDef("", "", {tensor_id_str});
-  CompareNodeInputs(graph, &expected, c);
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, UpdateFaninToNodeAsSwitchControl) {
@@ -816,59 +853,24 @@ TEST(MutableGraphViewTest, DedupControllingFaninsOnGraphInit) {
   MutableGraphView graph(&graph_def);
 
   EXPECT_EQ(graph.graph()->node_size(), 11);
-  NodeDef expected;
-  NodeDef* a = graph.GetNode("a");
-  ASSERT_NE(a, nullptr);
-  expected = NDef("", "", {});
-  CompareNodeInputs(graph, &expected, a);
-
-  NodeDef* b = graph.GetNode("b");
-  ASSERT_NE(b, nullptr);
-  CompareNodeInputs(graph, &expected, b);
-
-  NodeDef* c = graph.GetNode("c");
-  ASSERT_NE(c, nullptr);
-  CompareNodeInputs(graph, &expected, c);
-
-  NodeDef* d = graph.GetNode("d");
-  ASSERT_NE(d, nullptr);
-  expected = NDef("", "", {"c:1"});
-  CompareNodeInputs(graph, &expected, d);
-
-  NodeDef* foo_1 = graph.GetNode("foo_1");
-  ASSERT_NE(foo_1, nullptr);
-  expected = NDef("", "", {"a", "b:1"});
-  CompareNodeInputs(graph, &expected, foo_1);
-
-  NodeDef* foo_2 = graph.GetNode("foo_2");
-  ASSERT_NE(foo_2, nullptr);
-  expected = NDef("", "", {"a", "^b"});
-  CompareNodeInputs(graph, &expected, foo_2);
-
-  NodeDef* foo_3 = graph.GetNode("foo_3");
-  ASSERT_NE(foo_3, nullptr);
-  expected = NDef("", "", {"a", "b:1"});
-  CompareNodeInputs(graph, &expected, foo_3);
-
-  NodeDef* foo_4 = graph.GetNode("foo_4");
-  ASSERT_NE(foo_4, nullptr);
-  expected = NDef("", "", {"a:2", "b:1"});
-  CompareNodeInputs(graph, &expected, foo_4);
-
-  NodeDef* foo_5 = graph.GetNode("foo_5");
-  ASSERT_NE(foo_5, nullptr);
-  expected = NDef("", "", {"a:2", "b:1"});
-  CompareNodeInputs(graph, &expected, foo_5);
-
-  NodeDef* foo_6 = graph.GetNode("foo_6");
-  ASSERT_NE(foo_6, nullptr);
-  expected = NDef("", "", {"d", "^d"});
-  CompareNodeInputs(graph, &expected, foo_6);
-
-  NodeDef* foo_7 = graph.GetNode("foo_7");
-  ASSERT_NE(foo_7, nullptr);
-  expected = NDef("", "", {"a:3", "b:2", "d", "^d"});
-  CompareNodeInputs(graph, &expected, foo_7);
+
+  CheckNode(graph, "a", "NotImportant", "", {}, {},
+            {"foo_1", "foo_2", "foo_3", "foo_4", "foo_5", "foo_7"});
+  CheckNode(graph, "b", "NotImportant", "", {}, {},
+            {"foo_1:1", "^foo_2", "foo_3:1", "foo_4:1", "foo_5:1", "foo_7:1"});
+  CheckNode(graph, "c", "Switch", "", {}, {}, {"d"});
+  CheckNode(graph, "d", "Identity", "", {}, {"c:1"},
+            {"foo_6", "^foo_6", "foo_7:2", "^foo_7"});
+  CheckNode(graph, "foo_1", "IdentityN", "", {}, {"a", "b:1"}, {});
+  CheckNode(graph, "foo_2", "IdentityN", "", {}, {"a", "^b"}, {});
+  CheckNode(graph, "foo_3", "IdentityN", "", {}, {"a", "b:1"}, {});
+  CheckNode(graph, "foo_4", "IdentityN", "", {}, {"a:2", "b:1"}, {});
+  CheckNode(graph, "foo_5", "NotImportant", "", {}, {"a:2", "b:1"}, {});
+  CheckNode(graph, "foo_6", "Identity", "", {}, {"d", "^d"}, {});
+  CheckNode(graph, "foo_7", "NotImportant", "", {}, {"a:3", "b:2", "d", "^d"},
+            {});
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, DedupControllingFaninsOnAddFanin) {
@@ -881,17 +883,14 @@ TEST(MutableGraphViewTest, DedupControllingFaninsOnAddFanin) {
   MutableGraphView graph(&graph_def);
 
   EXPECT_TRUE(graph.AddRegularFanin("b", {"a", 2}).ok());
-  NodeDef expected;
-  NodeDef* b = graph.GetNode("b");
-  ASSERT_NE(b, nullptr);
-  expected = NDef("", "", {"a:2"});
-  CompareNodeInputs(graph, &expected, b);
+  CheckNode(graph, "b", "NotImportant", "", {}, {"a:2"}, {});
 
   EXPECT_TRUE(graph.AddControllingFanin("c", {"a", Graph::kControlSlot}).ok());
-  NodeDef* c = graph.GetNode("c");
-  ASSERT_NE(c, nullptr);
-  expected = NDef("", "", {"a:1"});
-  CompareNodeInputs(graph, &expected, c);
+  CheckNode(graph, "c", "NotImportant", "", {}, {"a:1"}, {});
+
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {"b:0", "c:0"});
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, NoDedupControlFlowControllingFaninsOnAddFanin) {
@@ -903,29 +902,18 @@ TEST(MutableGraphViewTest, NoDedupControlFlowControllingFaninsOnAddFanin) {
   MutableGraphView graph(&graph_def);
 
   EXPECT_TRUE(graph.AddRegularFanin("c", {"b", 2}).ok());
-  NodeDef expected;
-  NodeDef* c = graph.GetNode("c");
-  ASSERT_NE(c, nullptr);
-  expected = NDef("", "", {"b:2"});
-  CompareNodeInputs(graph, &expected, c);
+  CheckNode(graph, "c", "", "", {}, {"b:2"}, {});
   EXPECT_TRUE(graph.AddControllingFanin("c", {"b", Graph::kControlSlot}).ok());
-  expected = NDef("", "", {"b:2", "^b"});
-  CompareNodeInputs(graph, &expected, c);
+  CheckNode(graph, "c", "", "", {}, {"b:2", "^b"}, {});
   EXPECT_TRUE(graph.AddControllingFanin("c", {"b", Graph::kControlSlot}).ok());
-  expected = NDef("", "", {"b:2", "^b"});
-  CompareNodeInputs(graph, &expected, c);
+  CheckNode(graph, "c", "", "", {}, {"b:2", "^b"}, {});
 
   EXPECT_TRUE(graph.AddControllingFanin("d", {"b", Graph::kControlSlot}).ok());
-  NodeDef* d = graph.GetNode("d");
-  ASSERT_NE(d, nullptr);
-  expected = NDef("", "", {"^b"});
-  CompareNodeInputs(graph, &expected, d);
+  CheckNode(graph, "d", "", "", {}, {"^b"}, {});
   EXPECT_TRUE(graph.AddControllingFanin("d", {"b", Graph::kControlSlot}).ok());
-  expected = NDef("", "", {"^b"});
-  CompareNodeInputs(graph, &expected, d);
-  EXPECT_TRUE(graph.AddRegularFanin("d", {"b", 3}).ok());
-  expected = NDef("", "", {"b:3", "^b"});
-  CompareNodeInputs(graph, &expected, d);
+  CheckNode(graph, "d", "", "", {}, {"^b"}, {});
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, DedupControllingFaninsOnUpdateFanin) {
@@ -938,10 +926,12 @@ TEST(MutableGraphViewTest, DedupControllingFaninsOnUpdateFanin) {
   MutableGraphView graph(&graph_def);
 
   EXPECT_TRUE(graph.UpdateFanin("c", {"a", 1}, {"b", 2}).ok());
-  NodeDef* c = graph.GetNode("c");
-  ASSERT_NE(c, nullptr);
-  NodeDef expected = NDef("", "", {"b:2"});
-  CompareNodeInputs(graph, &expected, c);
+
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {});
+  CheckNode(graph, "b", "NotImportant", "", {}, {}, {"c"});
+  CheckNode(graph, "c", "NotImportant", "", {}, {"b:2"}, {});
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, NoDedupControlFlowControllingFaninsOnUpdateFanin) {
@@ -957,23 +947,16 @@ TEST(MutableGraphViewTest, NoDedupControlFlowControllingFaninsOnUpdateFanin) {
                   .UpdateFanin("d", {"b", Graph::kControlSlot},
                                {"c", Graph::kControlSlot})
                   .ok());
-  NodeDef expected;
-  NodeDef* d = graph.GetNode("d");
-  ASSERT_NE(d, nullptr);
-  expected = NDef("", "", {"c", "^c"});
-  CompareNodeInputs(graph, &expected, d);
+  CheckNode(graph, "d", "NotImportant", "", {}, {"c", "^c"}, {});
 
   EXPECT_TRUE(graph.UpdateFanin("e", {"b", 0}, {"c", 3}).ok());
-  NodeDef* e = graph.GetNode("e");
-  ASSERT_NE(e, nullptr);
-  expected = NDef("", "", {"c:3", "^c"});
-  CompareNodeInputs(graph, &expected, e);
+  CheckNode(graph, "e", "NotImportant", "", {}, {"c:3", "^c"}, {});
 
   EXPECT_TRUE(
       graph.UpdateFanin("e", {"c", 3}, {"c", Graph::kControlSlot}).ok());
-  ASSERT_NE(e, nullptr);
-  expected = NDef("", "", {"^c"});
-  CompareNodeInputs(graph, &expected, e);
+  CheckNode(graph, "e", "NotImportant", "", {}, {"^c"}, {});
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, UpdateMaxRegularOutputPortOnAddFanin) {
@@ -986,19 +969,12 @@ TEST(MutableGraphViewTest, UpdateMaxRegularOutputPortOnAddFanin) {
   MutableGraphView graph(&graph_def);
 
   EXPECT_TRUE(graph.AddRegularFanin("c", {"a", 3}).ok());
-  NodeDef* a = graph.GetNode("a");
-  ASSERT_NE(a, nullptr);
-
-  auto fanouts = graph.GetFanouts(*a, /*include_controlled_nodes=*/true);
-  EXPECT_EQ(fanouts.size(), 2);
 
-  NodeDef* b = graph.GetNode("b");
-  ASSERT_NE(b, nullptr);
-  EXPECT_EQ(fanouts.count(MutableGraphView::InputPort(b, 0)), 1);
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {"b", "c"});
+  CheckNode(graph, "b", "NotImportant", "", {}, {"a:1"}, {"^c"});
+  CheckNode(graph, "c", "NotImportant", "", {}, {"a:3", "^b"}, {});
 
-  NodeDef* c = graph.GetNode("c");
-  ASSERT_NE(c, nullptr);
-  EXPECT_EQ(fanouts.count(MutableGraphView::InputPort(c, 0)), 1);
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, UpdateMaxRegularOutputPortOnRemoveFanin) {
@@ -1011,15 +987,11 @@ TEST(MutableGraphViewTest, UpdateMaxRegularOutputPortOnRemoveFanin) {
   MutableGraphView graph(&graph_def);
 
   EXPECT_TRUE(graph.RemoveRegularFanin("c", {"a", 2}).ok());
-  NodeDef* a = graph.GetNode("a");
-  ASSERT_NE(a, nullptr);
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {"b"});
+  CheckNode(graph, "b", "NotImportant", "", {}, {"a:1"}, {});
+  CheckNode(graph, "c", "NotImportant", "", {}, {}, {});
 
-  auto fanouts = graph.GetFanouts(*a, /*include_controlled_nodes=*/true);
-  EXPECT_EQ(fanouts.size(), 1);
-
-  NodeDef* b = graph.GetNode("b");
-  ASSERT_NE(b, nullptr);
-  EXPECT_EQ(fanouts.count(MutableGraphView::InputPort(b, 0)), 1);
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, KeepMaxRegularOutputPortOnRemoveFanin) {
@@ -1032,15 +1004,12 @@ TEST(MutableGraphViewTest, KeepMaxRegularOutputPortOnRemoveFanin) {
   MutableGraphView graph(&graph_def);
 
   EXPECT_TRUE(graph.RemoveRegularFanin("b", {"a", 1}).ok());
-  NodeDef* a = graph.GetNode("a");
-  ASSERT_NE(a, nullptr);
 
-  auto fanouts = graph.GetFanouts(*a, /*include_controlled_nodes=*/true);
-  EXPECT_EQ(fanouts.size(), 1);
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {"c"});
+  CheckNode(graph, "b", "NotImportant", "", {}, {}, {});
+  CheckNode(graph, "c", "NotImportant", "", {}, {"a:2"}, {});
 
-  NodeDef* c = graph.GetNode("c");
-  ASSERT_NE(c, nullptr);
-  EXPECT_EQ(fanouts.count(MutableGraphView::InputPort(c, 0)), 1);
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, UpdateMaxRegularOutputPortOnUpdateFanin) {
@@ -1053,22 +1022,12 @@ TEST(MutableGraphViewTest, UpdateMaxRegularOutputPortOnUpdateFanin) {
   MutableGraphView graph(&graph_def);
 
   EXPECT_TRUE(graph.UpdateFanin("c", {"a", 2}, {"b", 3}).ok());
-  NodeDef* a = graph.GetNode("a");
-  ASSERT_NE(a, nullptr);
-
-  auto a_fanouts = graph.GetFanouts(*a, /*include_controlled_nodes=*/true);
-  EXPECT_EQ(a_fanouts.size(), 1);
 
-  NodeDef* b = graph.GetNode("b");
-  ASSERT_NE(b, nullptr);
-  EXPECT_EQ(a_fanouts.count(MutableGraphView::InputPort(b, 0)), 1);
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {"b"});
+  CheckNode(graph, "b", "NotImportant", "", {}, {"a:1"}, {"c"});
+  CheckNode(graph, "c", "NotImportant", "", {}, {"b:3"}, {});
 
-  auto b_fanouts = graph.GetFanouts(*b, /*include_controlled_nodes=*/true);
-  EXPECT_EQ(b_fanouts.size(), 1);
-
-  NodeDef* c = graph.GetNode("c");
-  ASSERT_NE(c, nullptr);
-  EXPECT_EQ(b_fanouts.count(MutableGraphView::InputPort(c, 0)), 1);
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, AddControllingFaninMissing) {
@@ -1096,15 +1055,11 @@ TEST(MutableGraphViewTest, AddControllingFaninMissing) {
   EXPECT_EQ(s.error_message(), expected_msg);
 
   ASSERT_EQ(graph.graph()->node_size(), 2);
-  NodeDef expected;
-  NodeDef* a = graph.GetNode("a");
-  ASSERT_NE(a, nullptr);
-  expected = NDef("", "", {});
-  CompareNodeInputs(graph, &expected, a);
-
-  NodeDef* b = graph.GetNode("b");
-  ASSERT_NE(b, nullptr);
-  CompareNodeInputs(graph, &expected, b);
+
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {});
+  CheckNode(graph, "b", "NotImportant", "", {}, {}, {});
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, AddControllingFaninExistingControl) {
@@ -1118,16 +1073,11 @@ TEST(MutableGraphViewTest, AddControllingFaninExistingControl) {
   EXPECT_TRUE(graph.AddControllingFanin("a", {"b", Graph::kControlSlot}).ok());
 
   ASSERT_EQ(graph.graph()->node_size(), 2);
-  NodeDef expected;
-  NodeDef* a = graph.GetNode("a");
-  ASSERT_NE(a, nullptr);
-  expected = NDef("", "", {"^b"});
-  CompareNodeInputs(graph, &expected, a);
-
-  NodeDef* b = graph.GetNode("b");
-  ASSERT_NE(b, nullptr);
-  expected = NDef("", "", {});
-  CompareNodeInputs(graph, &expected, b);
+
+  CheckNode(graph, "a", "NotImportant", "", {}, {"^b"}, {});
+  CheckNode(graph, "b", "NotImportant", "", {}, {}, {"^a"});
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, AddControllingFaninNotSwitch) {
@@ -1141,16 +1091,11 @@ TEST(MutableGraphViewTest, AddControllingFaninNotSwitch) {
   EXPECT_TRUE(graph.AddControllingFanin("a", {"b", 2}).ok());
 
   ASSERT_EQ(graph.graph()->node_size(), 2);
-  NodeDef expected;
-  NodeDef* a = graph.GetNode("a");
-  ASSERT_NE(a, nullptr);
-  expected = NDef("", "", {"^b"});
-  CompareNodeInputs(graph, &expected, a);
-
-  NodeDef* b = graph.GetNode("b");
-  ASSERT_NE(b, nullptr);
-  expected = NDef("", "", {});
-  CompareNodeInputs(graph, &expected, b);
+
+  CheckNode(graph, "a", "NotImportant", "", {}, {"^b"}, {});
+  CheckNode(graph, "b", "NotImportant", "", {}, {}, {"^a"});
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, AddControllingFaninSwitch) {
@@ -1167,14 +1112,11 @@ TEST(MutableGraphViewTest, AddControllingFaninSwitch) {
   EXPECT_EQ(s.error_message(), expected_msg);
 
   ASSERT_EQ(graph.graph()->node_size(), 2);
-  NodeDef* a = graph.GetNode("a");
-  ASSERT_NE(a, nullptr);
-  NodeDef expected = NDef("", "", {});
-  CompareNodeInputs(graph, &expected, a);
-
-  NodeDef* b = graph.GetNode("b");
-  ASSERT_NE(b, nullptr);
-  CompareNodeInputs(graph, &expected, b);
+
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {});
+  CheckNode(graph, "b", "Switch", "", {}, {}, {});
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, AddControllingFaninSwitchWithIdentity) {
@@ -1189,10 +1131,12 @@ TEST(MutableGraphViewTest, AddControllingFaninSwitchWithIdentity) {
   EXPECT_TRUE(graph.AddControllingFanin("a", {"switch", 0}).ok());
 
   ASSERT_EQ(graph.graph()->node_size(), 3);
-  NodeDef* a = graph.GetNode("a");
-  ASSERT_NE(a, nullptr);
-  NodeDef expected = NDef("", "", {"^identity"});
-  CompareNodeInputs(graph, &expected, a);
+
+  CheckNode(graph, "a", "NotImportant", "", {}, {"^identity"}, {});
+  CheckNode(graph, "switch", "Switch", "", {}, {}, {"identity"});
+  CheckNode(graph, "identity", "Identity", "", {}, {"switch"}, {"^a"});
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, AddControllingFaninSwitchWithNoExistingIdentity) {
@@ -1208,20 +1152,15 @@ TEST(MutableGraphViewTest, AddControllingFaninSwitchWithNoExistingIdentity) {
   EXPECT_TRUE(graph.AddControllingFanin("a", {"switch", 0}).ok());
 
   ASSERT_EQ(graph.graph()->node_size(), 3);
-  NodeDef expected;
-  NodeDef* a = graph.GetNode("a");
-  ASSERT_NE(a, nullptr);
-  expected = NDef("", "", {"^ConstantFoldingCtrl/switch_0"});
-  CompareNodeInputs(graph, &expected, a);
-
-  NodeDef* identity = graph.GetNode("ConstantFoldingCtrl/switch_0");
-  ASSERT_NE(identity, nullptr);
-  expected = NDef("", "", {"switch"});
-  CompareNodeInputs(graph, &expected, identity);
-  EXPECT_EQ(identity->op(), "Identity");
-  EXPECT_EQ(identity->device(), kDevice);
-  ASSERT_TRUE(identity->attr().count("T"));
-  EXPECT_EQ(identity->attr().at("T").type(), DT_FLOAT);
+
+  CheckNode(graph, "a", "NotImportant", "", {},
+            {"^ConstantFoldingCtrl/switch_0"}, {});
+  CheckNode(graph, "switch", "Switch", kDevice, {{"T", DT_FLOAT}}, {},
+            {"ConstantFoldingCtrl/switch_0"});
+  CheckNode(graph, "ConstantFoldingCtrl/switch_0", "Identity", kDevice,
+            {{"T", DT_FLOAT}}, {"switch"}, {"^a"});
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, AddControllingFaninSwitchWithExistingAddedIdentity) {
@@ -1236,10 +1175,15 @@ TEST(MutableGraphViewTest, AddControllingFaninSwitchWithExistingAddedIdentity) {
   EXPECT_TRUE(graph.AddControllingFanin("a", {"switch", 0}).ok());
 
   ASSERT_EQ(graph.graph()->node_size(), 3);
-  NodeDef* a = graph.GetNode("a");
-  ASSERT_NE(a, nullptr);
-  NodeDef expected = NDef("", "", {"^ConstantFoldingCtrl/switch_0"});
-  CompareNodeInputs(graph, &expected, a);
+
+  CheckNode(graph, "a", "NotImportant", "", {},
+            {"^ConstantFoldingCtrl/switch_0"}, {});
+  CheckNode(graph, "switch", "Switch", "", {}, {},
+            {"ConstantFoldingCtrl/switch_0"});
+  CheckNode(graph, "ConstantFoldingCtrl/switch_0", "Identity", "", {},
+            {"switch"}, {"^a"});
+
+  CheckGraph(graph);
 }
 
 void TestAddControllingFaninSelfLoops(absl::string_view node_name,
@@ -1259,30 +1203,14 @@ void TestAddControllingFaninSelfLoops(absl::string_view node_name,
   EXPECT_EQ(s.error_message(), error_msg);
 
   EXPECT_EQ(graph.graph()->node_size(), 5);
-  NodeDef expected;
-  NodeDef* a = graph.GetNode("a");
-  ASSERT_NE(a, nullptr);
-  expected = NDef("", "", {}, {});
-  CompareNodeInputs(graph, &expected, a);
-
-  NodeDef* b = graph.GetNode("b");
-  ASSERT_NE(b, nullptr);
-  CompareNodeInputs(graph, &expected, b);
-
-  NodeDef* c = graph.GetNode("c");
-  ASSERT_NE(c, nullptr);
-  expected = NDef("", "", {"b:0"});
-  CompareNodeInputs(graph, &expected, c);
-
-  NodeDef* d = graph.GetNode("d");
-  ASSERT_NE(d, nullptr);
-  expected = NDef("", "", {"b:1"});
-  CompareNodeInputs(graph, &expected, d);
-
-  NodeDef* e = graph.GetNode("e");
-  ASSERT_NE(e, nullptr);
-  expected = NDef("", "", {"^a"});
-  CompareNodeInputs(graph, &expected, e);
+
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {"^e"});
+  CheckNode(graph, "b", "Switch", "", {{"T", DT_FLOAT}}, {}, {"c", "d"});
+  CheckNode(graph, "c", "Identity", "", {}, {"b"}, {});
+  CheckNode(graph, "d", "Identity", "", {}, {"b:1"}, {});
+  CheckNode(graph, "e", "NotImportant", "", {}, {"^a"}, {});
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, AddControllingFaninSelfLoops) {
@@ -1332,23 +1260,13 @@ TEST(MutableGraphViewTest, AddControllingFaninSelfLoopsGeneratedIdentity) {
   EXPECT_EQ(s.error_message(), expected_msg);
 
   EXPECT_EQ(graph.graph()->node_size(), 4);
-  NodeDef expected;
-  NodeDef* a = graph.GetNode("a");
-  ASSERT_NE(a, nullptr);
-  expected = NDef("", "", {}, {});
-  CompareNodeInputs(graph, &expected, a);
-
-  NodeDef* b = graph.GetNode("b");
-  ASSERT_NE(b, nullptr);
-  CompareNodeInputs(graph, &expected, b);
-
-  NodeDef* c = graph.GetNode("c");
-  ASSERT_NE(c, nullptr);
-  CompareNodeInputs(graph, &expected, c);
-
-  NodeDef* d = graph.GetNode("ConstantFoldingCtrl/b_1");
-  ASSERT_NE(d, nullptr);
-  CompareNodeInputs(graph, &expected, d);
+
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {});
+  CheckNode(graph, "b", "Switch", "", {{"T", DT_FLOAT}}, {}, {});
+  CheckNode(graph, "c", "NotImportant", "", {}, {}, {});
+  CheckNode(graph, "ConstantFoldingCtrl/b_1", "Identity", "", {}, {}, {});
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, RemoveControllingFaninMissing) {
@@ -1364,10 +1282,13 @@ TEST(MutableGraphViewTest, RemoveControllingFaninMissing) {
   EXPECT_TRUE(graph.RemoveControllingFanin("d", "c").ok());
 
   ASSERT_EQ(graph.graph()->node_size(), 4);
-  NodeDef* d = graph.GetNode("d");
-  ASSERT_NE(d, nullptr);
-  NodeDef expected = NDef("", "", {"^a", "^b"});
-  CompareNodeInputs(graph, &expected, d);
+
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {"^d"});
+  CheckNode(graph, "b", "NotImportant", "", {}, {}, {"^d"});
+  CheckNode(graph, "c", "NotImportant", "", {}, {}, {});
+  CheckNode(graph, "d", "NotImportant", "", {}, {"^a", "^b"}, {});
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, RemoveControllingFaninExisting) {
@@ -1384,10 +1305,13 @@ TEST(MutableGraphViewTest, RemoveControllingFaninExisting) {
   EXPECT_TRUE(graph.RemoveControllingFanin("d", "a").ok());
 
   ASSERT_EQ(graph.graph()->node_size(), 4);
-  NodeDef* d = graph.GetNode("d");
-  ASSERT_NE(d, nullptr);
-  NodeDef expected = NDef("", "", {"^c", "^b"});
-  CompareNodeInputs(graph, &expected, d);
+
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {});
+  CheckNode(graph, "b", "NotImportant", "", {}, {}, {"^d"});
+  CheckNode(graph, "c", "NotImportant", "", {}, {}, {"^d"});
+  CheckNode(graph, "d", "NotImportant", "", {}, {"^c", "^b"}, {});
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, RemoveControllingFaninOnRegularFanin) {
@@ -1403,10 +1327,12 @@ TEST(MutableGraphViewTest, RemoveControllingFaninOnRegularFanin) {
   EXPECT_TRUE(graph.RemoveControllingFanin("c", "b").ok());
 
   ASSERT_EQ(graph.graph()->node_size(), 3);
-  NodeDef* c = graph.GetNode("c");
-  ASSERT_NE(c, nullptr);
-  NodeDef expected = NDef("", "", {"a", "b"});
-  CompareNodeInputs(graph, &expected, c);
+
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {"b", "c"});
+  CheckNode(graph, "b", "NotImportant", "", {}, {"a"}, {"c:1"});
+  CheckNode(graph, "c", "NotImportant", "", {}, {"a", "b"}, {});
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, RemoveControllingFaninSelfLoop) {
@@ -1424,10 +1350,12 @@ TEST(MutableGraphViewTest, RemoveControllingFaninSelfLoop) {
   EXPECT_EQ(s.error_message(), expected_msg);
 
   ASSERT_EQ(graph.graph()->node_size(), 3);
-  NodeDef* c = graph.GetNode("c");
-  ASSERT_NE(c, nullptr);
-  NodeDef expected = NDef("", "", {"a", "b"});
-  CompareNodeInputs(graph, &expected, c);
+
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {"b", "c"});
+  CheckNode(graph, "b", "NotImportant", "", {}, {"a"}, {"c:1"});
+  CheckNode(graph, "c", "NotImportant", "", {}, {"a", "b"}, {});
+
+  CheckGraph(graph);
 }
 
 TEST(MutableGraphViewTest, DeleteNodes) {
@@ -1444,21 +1372,14 @@ TEST(MutableGraphViewTest, DeleteNodes) {
   EXPECT_NE(graph.GetNode("foo_1"), nullptr);
   graph.DeleteNodes({"foo_1"});
 
+  EXPECT_EQ(graph.graph()->node_size(), 3);
   EXPECT_EQ(graph.GetNode("foo_1"), nullptr);
 
-  NodeDef* bar = graph.GetNode("bar");
-  NodeDef* other = graph.GetNode("other");
-  NodeDef* foo_2 = graph.GetNode("foo_2");
-
-  bool include_control_fanouts = true;
-  auto bar_fanouts = graph.GetFanouts(*bar, include_control_fanouts);
-  auto other_fanouts = graph.GetFanouts(*other, include_control_fanouts);
-
-  EXPECT_EQ(bar_fanouts.size(), 1);
-  EXPECT_EQ(bar_fanouts.count(MutableGraphView::InputPort(foo_2, 1)), 1);
+  CheckNode(graph, "bar", "NotImportant", "", {}, {}, {"foo_2:1"});
+  CheckNode(graph, "other", "NotImportant", "", {}, {}, {"foo_2"});
+  CheckNode(graph, "foo_2", "NotImportant", "", {}, {"other:1", "bar:2"}, {});
 
-  EXPECT_EQ(other_fanouts.size(), 1);
-  EXPECT_EQ(other_fanouts.count(MutableGraphView::InputPort(foo_2, 0)), 1);
+  CheckGraph(graph);
 }
 
 }  // namespace
-- 
GitLab


From b2723af89145e5238df417b9bfd47fef29d00354 Mon Sep 17 00:00:00 2001
From: Akshay Modi <nareshmodi@google.com>
Date: Wed, 16 Jan 2019 13:05:06 -0800
Subject: [PATCH 0830/1095] Don't run ops.device(None) in eager.

PiperOrigin-RevId: 229610975
---
 tensorflow/python/framework/ops.py             | 3 +++
 tensorflow/python/ops/resource_variable_ops.py | 4 +++-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index 079fad4210..6da6637b0b 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -102,6 +102,9 @@ class _UserDeviceSpec(object):
 
 class NullContextmanager(object):
 
+  def __init__(self, *args, **kwargs):
+    pass
+
   def __enter__(self):
     pass
 
diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index 1a895fee87..c8f20029ac 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -409,11 +409,13 @@ class ResourceVariable(variables.VariableV1):
         # Use attr_scope and device(None) to simulate the behavior of
         # colocate_with when the variable we want to colocate with doesn't
         # yet exist.
+        device_context_manager = (
+            ops.device if self._in_graph_mode else ops.NullContextmanager)
         attr = attr_value_pb2.AttrValue(
             list=attr_value_pb2.AttrValue.ListValue(
                 s=[compat.as_bytes("loc:@%s" % handle_name)]))
         with ops.get_default_graph()._attr_scope({"_class": attr}):
-          with ops.name_scope("Initializer"), ops.device(None):
+          with ops.name_scope("Initializer"), device_context_manager(None):
             initial_value = ops.convert_to_tensor(
                 initial_value() if init_from_fn else initial_value,
                 name="initial_value", dtype=dtype)
-- 
GitLab


From 71472ddbe17a51ba10003d271700bb0123ceb991 Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Wed, 16 Jan 2019 13:29:51 -0800
Subject: [PATCH 0831/1095] Remove dynamic dynamic dimensions in
 XLABuilder::Build

XLA backend cannot handle dynamic dimension yet, remove all dynamic
dimensions before building xla program until we have support in the
backend.

PiperOrigin-RevId: 229615685
---
 tensorflow/compiler/xla/client/xla_builder.cc | 19 +++++++++++++++++++
 .../compiler/xla/client/xla_builder_test.cc   |  4 +++-
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_builder.cc
index 4b360c6563..20298d175d 100644
--- a/tensorflow/compiler/xla/client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_builder.cc
@@ -322,6 +322,25 @@ StatusOr<XlaComputation> XlaBuilder::Build(int64 root_id) {
     return AppendStatus(first_error_, backtrace);
   }
 
+  // TODO(b/121223198): XLA backend cannot handle dynamic dimensions yet, remove
+  // all dynamic dimensions before building xla program until we have support in
+  // the backend.
+  std::function<void(ShapeProto*)> remove_dynamic_dimension =
+      [&](ShapeProto* shape) {
+        if (shape->tuple_shapes_size() != 0) {
+          for (int64 i = 0; i < shape->tuple_shapes_size(); ++i) {
+            remove_dynamic_dimension(shape->mutable_tuple_shapes(i));
+          }
+        }
+        for (int64 i = 0; i < shape->dimensions_size(); ++i) {
+          shape->set_is_dynamic_dimension(i, false);
+        }
+      };
+
+  for (auto& instruction : instructions_) {
+    remove_dynamic_dimension(instruction.mutable_shape());
+  }
+
   HloComputationProto entry;
   SetProtoIdAndName(&entry, name_, kNameSeparator, GetNextId());
   TF_ASSIGN_OR_RETURN(ProgramShape program_shape, GetProgramShape(root_id));
diff --git a/tensorflow/compiler/xla/client/xla_builder_test.cc b/tensorflow/compiler/xla/client/xla_builder_test.cc
index abc11b4732..feee8187c7 100644
--- a/tensorflow/compiler/xla/client/xla_builder_test.cc
+++ b/tensorflow/compiler/xla/client/xla_builder_test.cc
@@ -463,7 +463,9 @@ TEST_F(XlaBuilderTest, DynamicParameter) {
                                  ->parameter_instruction(0)
                                  ->shape()
                                  .tuple_shapes(1);
-  EXPECT_TRUE(param_shape.is_dynamic_dimension(0));
+  // TODO(b/121223198): The dynamic dimension should be set once we enable
+  // dynamic dimensions in xla builder.
+  EXPECT_FALSE(param_shape.is_dynamic_dimension(0));
 }
 
 TEST_F(XlaBuilderTest, AfterAllWithNonTokenOperands) {
-- 
GitLab


From c432f294845786ca4d26e4d8091cab4dfa1e796f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 16 Jan 2019 13:40:43 -0800
Subject: [PATCH 0832/1095] Restore 0-arg version of Tensor::DebugString.

PiperOrigin-RevId: 229617823
---
 tensorflow/core/framework/tensor.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/framework/tensor.h b/tensorflow/core/framework/tensor.h
index 44293c5c19..4cfad893d5 100644
--- a/tensorflow/core/framework/tensor.h
+++ b/tensorflow/core/framework/tensor.h
@@ -529,7 +529,8 @@ class Tensor {
   // `num_values` is the number of actual data values in the tensor
   // included in the message. If the tensor might be resident in
   // GPU/TPU memory use DeviceSafeDebugString instead.
-  string DebugString(int num_values = 3) const;
+  string DebugString(int num_values) const;
+  string DebugString() const { return DebugString(3); }
 
   // Variant of DebugString() that should be used for possibly non-CPU tensors.
   // If the tensor is not resident on CPU, we can't read its values as
-- 
GitLab


From 440ee60cda0912d2275a61b714a33564716bfd5a Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Wed, 16 Jan 2019 13:54:48 -0800
Subject: [PATCH 0833/1095] [TF:XLA] Bump open source llvm revision to r351319

PiperOrigin-RevId: 229620326
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 354148f034..b8ef43d27e 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -502,11 +502,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "llvm",
         build_file = clean_dep("//third_party/llvm:llvm.autogenerated.BUILD"),
-        sha256 = "c6e6de0b5cd524acf276188cb6aae427568070cd1b9cb9c0e3aab8044f3ac278",
-        strip_prefix = "llvm-a42cde684126a3797f3eda9c894c379e7a8c66c1",
+        sha256 = "2027c5043ea3458376fbf4263aaab3b257736b9e4a1e9e9984f15e245177979d",
+        strip_prefix = "llvm-82a2c96ee81a617918d9bcc0fbe9af860e437f5f",
         urls = [
-            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/a42cde684126a3797f3eda9c894c379e7a8c66c1.tar.gz",
-            "https://github.com/llvm-mirror/llvm/archive/a42cde684126a3797f3eda9c894c379e7a8c66c1.tar.gz",
+            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/82a2c96ee81a617918d9bcc0fbe9af860e437f5f.tar.gz",
+            "https://github.com/llvm-mirror/llvm/archive/82a2c96ee81a617918d9bcc0fbe9af860e437f5f.tar.gz",
         ],
     )
 
-- 
GitLab


From 955d8f1377c3f8e7a2ec4b548126d20d70e063f0 Mon Sep 17 00:00:00 2001
From: Alan Chiao <alanchiao@google.com>
Date: Wed, 16 Jan 2019 14:13:41 -0800
Subject: [PATCH 0834/1095] Clarify tensor_utils neon memory alignment
 assumptions.

PiperOrigin-RevId: 229624071
---
 .../internal/optimized/neon_tensor_utils.cc   | 34 ++++++++++++-------
 1 file changed, 22 insertions(+), 12 deletions(-)

diff --git a/tensorflow/lite/kernels/internal/optimized/neon_tensor_utils.cc b/tensorflow/lite/kernels/internal/optimized/neon_tensor_utils.cc
index cf1bda2661..3e7b6c22be 100644
--- a/tensorflow/lite/kernels/internal/optimized/neon_tensor_utils.cc
+++ b/tensorflow/lite/kernels/internal/optimized/neon_tensor_utils.cc
@@ -90,20 +90,28 @@ void NeonMatrixBatchVectorMultiplyAccumulate(
     int n_batch, float* __restrict__ result, int result_stride) {
   const int kWeightsPerUint32 = 4;
   const int kWeightsPerNeonLane = 16;
-  // If the number of rows is not divisible by kWeightsPerUint32, we set a
-  // flag and allocate an aligned memory block. The flag is used to use the
-  // aligned memory block later in the kernel loop.
+  // Assuming *matrix is kWeightsPerUint32-byte aligned,
+  // every row of the matrix is also
+  // kWeightsPerUint32-byte aligned as long as cols is
+  // a multiple of kWeightsPerUint32. The assumption
+  // is currently satisfied by TFLite's 16-byte memory
+  // alignment scheme.
+  //
+  // Otherwise, we allocate an aligned memory block and set
+  // a flag to later copy rows from matrix to the block
+  // for aligned multiplication.
   bool unaligned = false;
-  int8* aligned_row = nullptr;
+  int8_t* aligned_row = nullptr;
   void* aligned_row_free = nullptr;
   if ((m_cols & (kWeightsPerUint32 - 1)) != 0) {
     unaligned = true;
-    aligned_row = (int8*)aligned_alloc(kWeightsPerUint32, m_cols,  // NOLINT
-                                       &aligned_row_free);
+    aligned_row = (int8_t*)aligned_alloc(kWeightsPerUint32, m_cols,  // NOLINT
+                                         &aligned_row_free);
   }
   void* aligned_vec_free = nullptr;
-  int8* aligned_vec = (int8*)aligned_alloc(kWeightsPerUint32, m_cols,  // NOLINT
-                                           &aligned_vec_free);
+  int8_t* aligned_vec =
+      (int8_t*)aligned_alloc(kWeightsPerUint32, m_cols,  // NOLINT
+                             &aligned_vec_free);
 
   // If m_cols is not at least kWeightsPerNeonLane, we cannot use the main
   // vectorized loop, and we need to process sequentially. postamble_start shows
@@ -114,13 +122,13 @@ void NeonMatrixBatchVectorMultiplyAccumulate(
   for (batch = 0; batch < n_batch; ++batch) {
     const float batch_scaling_factor = scaling_factors[batch];
     // Copy the vector data to an aligned vector.
-    memcpy(aligned_vec, vectors + batch * m_cols, sizeof(int8) * m_cols);
+    memcpy(aligned_vec, vectors + batch * m_cols, sizeof(int8_t) * m_cols);
     // Compute dot-product for every column.
     for (row = 0; row < m_rows; ++row, result += result_stride) {
       // Get the address of the first element of the row.
-      int8* row_ptr = (int8*)matrix + row * m_cols;  // NOLINT
+      int8_t* row_ptr = (int8_t*)matrix + row * m_cols;  // NOLINT
       if (unaligned) {
-        memcpy(aligned_row, row_ptr, sizeof(int8) * m_cols);
+        memcpy(aligned_row, row_ptr, sizeof(int8_t) * m_cols);
         row_ptr = aligned_row;
       }
 
@@ -135,7 +143,8 @@ void NeonMatrixBatchVectorMultiplyAccumulate(
       col = 0;
       for (; col < postamble_start; col += kWeightsPerNeonLane) {
         // Load 16 8-bit values from the row and vector, each, to operate on.
-        // Here the assumption is that each buffer is 4-byte aligned.
+        // Here the assumption is that each buffer is 4-byte aligned. Otherwise,
+        // performance may suffer significantly.
         TFLITE_CHECK_EQ((uintptr_t)(&row_ptr[col]) & (kWeightsPerUint32 - 1),
                         0);
         const int8x16_t s1_8x16 = vld1q_s8((const int8_t*)(aligned_vec + col));
@@ -164,6 +173,7 @@ void NeonMatrixBatchVectorMultiplyAccumulate(
         if ((m_cols - postamble_start) >= (kWeightsPerNeonLane >> 1)) {
           // Load 8 8-bit values from the row and column each to operate on.
           // Here the assumption is that each buffer is 4-bytes aligned.
+          // Otherwise, performance may suffer significantly.
           TFLITE_CHECK_EQ((uintptr_t)(&row_ptr[col]) & (kWeightsPerUint32 - 1),
                           0);
           const int8x8_t s1_8x8 = vld1_s8((const int8_t*)(aligned_vec + col));
-- 
GitLab


From 8ae3ff414ca860f6d4fdc92b255b483c1d9c302f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 16 Jan 2019 14:20:57 -0800
Subject: [PATCH 0835/1095] Update tf.distribute.Strategy.broadcast() API to
 match accepted RFC:
 https://github.com/tensorflow/community/blob/master/rfcs/20181016-replicator.md

PiperOrigin-RevId: 229625402
---
 tensorflow/python/distribute/distribute_lib.py    | 15 +++++++++------
 tensorflow/python/distribute/values.py            |  4 ++--
 ...tensorflow.distribute.-mirrored-strategy.pbtxt |  2 +-
 .../v1/tensorflow.distribute.-strategy.pbtxt      |  2 +-
 ...tensorflow.distribute.-mirrored-strategy.pbtxt |  2 +-
 .../v2/tensorflow.distribute.-strategy.pbtxt      |  2 +-
 6 files changed, 15 insertions(+), 12 deletions(-)

diff --git a/tensorflow/python/distribute/distribute_lib.py b/tensorflow/python/distribute/distribute_lib.py
index 32789c821e..e7d52c6c98 100644
--- a/tensorflow/python/distribute/distribute_lib.py
+++ b/tensorflow/python/distribute/distribute_lib.py
@@ -500,10 +500,11 @@ class DistributionStrategy(object):
         inputs = input_iterator.get_next()
         return self._extended.call_for_each_replica(fn, args=(inputs,))
 
-  @doc_controls.do_not_generate_docs  # DEPRECATED, moving to `extended`
-  def broadcast(self, tensor, destinations=None):
-    """DEPRECATED: use extended.broadcast_to() instead."""
-    return self._extended.broadcast_to(tensor, destinations)
+  # TODO(b/121296772,b/121300973): Add logical_device argument (default of 0).
+  def broadcast(self, tensor):
+    """Broadcasts `tensor` to all replicas, returning a per-replica value."""
+    _require_cross_replica_context_extended(self._extended)
+    return self._extended._broadcast(tensor)  # pylint: disable=protected-access
 
   def reduce(self, reduce_op, value):
     """Reduce `value` across replicas.
@@ -758,8 +759,7 @@ class DistributionStrategyExtended(object):
   * `d.make_dataset_iterator(dataset)` (or the deprecated
     `d.distribute_dataset(dataset).make_one_shot_iterator()`): in cross-replica
     context, produces an iterator with locality T
-  * `d.extended.broadcast_to(t)`: in cross-replica context, produces a value
-    with locality M
+  * `d.broadcast(t)`: in cross-replica context, produces a value with locality M
   * `d.extended.broadcast_to(t, v)`: in cross-replica context, produces a value
     with locality V(`v`)
   * `d.extended.call_for_each_replica(fn, ...)`: in cross-replica context, runs
@@ -1025,6 +1025,9 @@ class DistributionStrategyExtended(object):
     assert not isinstance(destinations, (list, tuple))
     return self._broadcast_to(tensor, destinations)
 
+  def _broadcast(self, tensor):
+    return self._broadcast_to(tensor, None)  # Default implementation
+
   def _broadcast_to(self, tensor, destinations):
     raise NotImplementedError("must be implemented in descendants")
 
diff --git a/tensorflow/python/distribute/values.py b/tensorflow/python/distribute/values.py
index 19967f2c5e..feaeea80cb 100644
--- a/tensorflow/python/distribute/values.py
+++ b/tensorflow/python/distribute/values.py
@@ -605,8 +605,8 @@ def validate_colocate(v, extended):
 
 def _apply_aggregation(strategy, value, aggregation, destinations):
   if aggregation == vs.VariableAggregation.ONLY_FIRST_REPLICA:
-    return strategy.broadcast(strategy.unwrap(value)[0],
-                              destinations=destinations)
+    return strategy.extended.broadcast_to(strategy.unwrap(value)[0],
+                                          destinations=destinations)
   reduce_op = reduce_util.ReduceOp.from_variable_aggregation(aggregation)
   return strategy.extended.reduce_to(reduce_op, value, destinations)
 
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-mirrored-strategy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-mirrored-strategy.pbtxt
index c554df489a..4cb78b08f8 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-mirrored-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-mirrored-strategy.pbtxt
@@ -17,7 +17,7 @@ tf_class {
   }
   member_method {
     name: "broadcast"
-    argspec: "args=[\'self\', \'tensor\', \'destinations\'], varargs=None, keywords=None, defaults=[\'None\'], "
+    argspec: "args=[\'self\', \'tensor\'], varargs=None, keywords=None, defaults=None"
   }
   member_method {
     name: "colocate_vars_with"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt
index cce83be3d0..11c1479b5b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt
@@ -16,7 +16,7 @@ tf_class {
   }
   member_method {
     name: "broadcast"
-    argspec: "args=[\'self\', \'tensor\', \'destinations\'], varargs=None, keywords=None, defaults=[\'None\'], "
+    argspec: "args=[\'self\', \'tensor\'], varargs=None, keywords=None, defaults=None"
   }
   member_method {
     name: "colocate_vars_with"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-mirrored-strategy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-mirrored-strategy.pbtxt
index c554df489a..4cb78b08f8 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-mirrored-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-mirrored-strategy.pbtxt
@@ -17,7 +17,7 @@ tf_class {
   }
   member_method {
     name: "broadcast"
-    argspec: "args=[\'self\', \'tensor\', \'destinations\'], varargs=None, keywords=None, defaults=[\'None\'], "
+    argspec: "args=[\'self\', \'tensor\'], varargs=None, keywords=None, defaults=None"
   }
   member_method {
     name: "colocate_vars_with"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt
index cce83be3d0..11c1479b5b 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt
@@ -16,7 +16,7 @@ tf_class {
   }
   member_method {
     name: "broadcast"
-    argspec: "args=[\'self\', \'tensor\', \'destinations\'], varargs=None, keywords=None, defaults=[\'None\'], "
+    argspec: "args=[\'self\', \'tensor\'], varargs=None, keywords=None, defaults=None"
   }
   member_method {
     name: "colocate_vars_with"
-- 
GitLab


From b0b56368f0c9269e4b003e2bc07020043eef53c8 Mon Sep 17 00:00:00 2001
From: Thomas O'Malley <omalleyt@google.com>
Date: Wed, 16 Jan 2019 14:33:55 -0800
Subject: [PATCH 0836/1095] Add optional `validation_freq` argument to `fit`.
 This allows user to configure how many training epochs occur between each
 validation run, or by providing a list users can explicitly specify which
 epochs should be followed by a validation run.

PiperOrigin-RevId: 229627830
---
 tensorflow/python/keras/engine/training.py    | 21 +++++++++++++
 .../python/keras/engine/training_arrays.py    | 14 +++++++--
 .../keras/engine/training_distributed.py      | 23 +++++++++++---
 .../python/keras/engine/training_generator.py | 12 ++++++-
 .../python/keras/engine/training_test.py      | 29 +++++++++++++++++
 .../python/keras/engine/training_utils.py     | 31 +++++++++++++++++++
 .../golden/v1/tensorflow.keras.-model.pbtxt   |  4 +--
 .../v1/tensorflow.keras.-sequential.pbtxt     |  4 +--
 .../v1/tensorflow.keras.models.-model.pbtxt   |  4 +--
 .../tensorflow.keras.models.-sequential.pbtxt |  4 +--
 .../golden/v2/tensorflow.keras.-model.pbtxt   |  4 +--
 .../v2/tensorflow.keras.-sequential.pbtxt     |  4 +--
 ...ensorflow.keras.layers.-linear-model.pbtxt |  4 +--
 .../v2/tensorflow.keras.models.-model.pbtxt   |  4 +--
 .../tensorflow.keras.models.-sequential.pbtxt |  4 +--
 15 files changed, 140 insertions(+), 26 deletions(-)

diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 6b0422e71c..e6ede64573 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -636,6 +636,7 @@ class Model(Network):
           initial_epoch=0,
           steps_per_epoch=None,
           validation_steps=None,
+          validation_freq=1,
           max_queue_size=10,
           workers=1,
           use_multiprocessing=False,
@@ -740,6 +741,13 @@ class Model(Network):
             is a dataset or dataset iterator. Total number of steps (batches of
             samples) to draw before stopping when performing validation
             at the end of every epoch.
+        validation_freq: Only relevant if validation data is provided. Integer
+            or `collections.Container` instance (e.g. list, tuple, etc.). If an
+            integer, specifies how many training epochs to run before a new
+            validation run is performed, e.g. `validation_freq=2` runs
+            validation every 2 epochs. If a Container, specifies the epochs on
+            which to run validation, e.g. `validation_freq=[1, 2, 10]` runs
+            validation at the end of the 1st, 2nd, and 10th epochs.
         max_queue_size: Integer. Used for generator or `keras.utils.Sequence`
             input only. Maximum size for the generator queue.
             If unspecified, `max_queue_size` will default to 10.
@@ -846,6 +854,7 @@ class Model(Network):
           callbacks=callbacks,
           validation_data=validation_data,
           validation_steps=validation_steps,
+          validation_freq=validation_freq,
           class_weight=class_weight,
           max_queue_size=max_queue_size,
           workers=workers,
@@ -868,6 +877,7 @@ class Model(Network):
           callbacks=callbacks,
           validation_data=validation_data,
           validation_steps=validation_steps,
+          validation_freq=validation_freq,
           class_weight=class_weight,
           workers=0,
           shuffle=shuffle,
@@ -930,6 +940,7 @@ class Model(Network):
           callbacks=callbacks,
           validation_data=validation_data,
           validation_steps=validation_steps,
+          validation_freq=validation_freq,
           workers=0,
           shuffle=shuffle,
           initial_epoch=initial_epoch,
@@ -951,6 +962,7 @@ class Model(Network):
           initial_epoch=initial_epoch,
           steps_per_epoch=steps_per_epoch,
           validation_steps=validation_steps,
+          validation_freq=validation_freq,
           steps_name='steps_per_epoch')
 
   def evaluate(self,
@@ -1426,6 +1438,7 @@ class Model(Network):
                     callbacks=None,
                     validation_data=None,
                     validation_steps=None,
+                    validation_freq=1,
                     class_weight=None,
                     max_queue_size=10,
                     workers=1,
@@ -1480,6 +1493,13 @@ class Model(Network):
             to yield from `generator` before stopping.
             Optional for `Sequence`: if unspecified, will use
             the `len(validation_data)` as a number of steps.
+        validation_freq: Only relevant if validation data is provided. Integer
+            or `collections.Container` instance (e.g. list, tuple, etc.). If an
+            integer, specifies how many training epochs to run before a new
+            validation run is performed, e.g. `validation_freq=2` runs
+            validation every 2 epochs. If a Container, specifies the epochs on
+            which to run validation, e.g. `validation_freq=[1, 2, 10]` runs
+            validation at the end of the 1st, 2nd, and 10th epochs.
         class_weight: Dictionary mapping class indices to a weight
             for the class.
         max_queue_size: Integer. Maximum size for the generator queue.
@@ -1535,6 +1555,7 @@ class Model(Network):
         callbacks=callbacks,
         validation_data=validation_data,
         validation_steps=validation_steps,
+        validation_freq=validation_freq,
         class_weight=class_weight,
         max_queue_size=max_queue_size,
         workers=workers,
diff --git a/tensorflow/python/keras/engine/training_arrays.py b/tensorflow/python/keras/engine/training_arrays.py
index a356f8b328..3b6a13839a 100644
--- a/tensorflow/python/keras/engine/training_arrays.py
+++ b/tensorflow/python/keras/engine/training_arrays.py
@@ -56,6 +56,7 @@ def model_iteration(model,
                     initial_epoch=0,
                     steps_per_epoch=None,
                     validation_steps=None,
+                    validation_freq=1,
                     mode=ModeKeys.TRAIN,
                     validation_in_fit=False,
                     steps_name='steps',
@@ -84,6 +85,13 @@ def model_iteration(model,
         the default value of `None`.
       validation_steps: Number of steps to run validation for (only if doing
         validation from data tensors). Ignored with the default value of `None`.
+      validation_freq: Only relevant if validation data is provided. Integer or
+        `collections.Container` instance (e.g. list, tuple, etc.). If an
+        integer, specifies how many training epochs to run before a new
+        validation run is performed, e.g. `validation_freq=2` runs
+        validation every 2 epochs. If a Container, specifies the epochs on
+        which to run validation, e.g. `validation_freq=[1, 2, 10]` runs
+        validation at the end of the 1st, 2nd, and 10th epochs.
       mode: One of ModeKeys.TRAIN/ModeKeys.TEST/ModeKeys.PREDICT.
       validation_in_fit: DEPRECATED: if true, then this method is invoked from
         within training iteration (for validation). In this case, do not copy
@@ -320,8 +328,10 @@ def model_iteration(model,
     if len(results) == 1:
       results = results[0]
 
-    # Run the test loop every epoch during training.
-    if do_validation and not callbacks.model.stop_training:
+    # Run the test loop every `validation_freq` epochs during training.
+    if (do_validation and
+        training_utils.should_run_validation(validation_freq, epoch) and
+        not callbacks.model.stop_training):
       val_results = model_iteration(
           model,
           val_inputs,
diff --git a/tensorflow/python/keras/engine/training_distributed.py b/tensorflow/python/keras/engine/training_distributed.py
index 58a8637a84..8b7e5bb5cc 100644
--- a/tensorflow/python/keras/engine/training_distributed.py
+++ b/tensorflow/python/keras/engine/training_distributed.py
@@ -30,6 +30,7 @@ from tensorflow.python.keras import backend as K
 from tensorflow.python.keras import callbacks as cbks
 from tensorflow.python.keras.engine import distributed_training_utils
 from tensorflow.python.keras.engine import training_arrays
+from tensorflow.python.keras.engine import training_utils
 from tensorflow.python.keras.utils.generic_utils import Progbar
 from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import tf_logging as logging
@@ -51,7 +52,8 @@ def fit_distributed(model,
                     sample_weight=None,
                     initial_epoch=0,
                     steps_per_epoch=None,
-                    validation_steps=None):
+                    validation_steps=None,
+                    validation_freq=1):
   """Fit loop for Distribution Strategies."""
   distributed_training_utils.validate_callbacks(callbacks, model.optimizer)
   distributed_training_utils.validate_inputs(
@@ -111,7 +113,8 @@ def fit_distributed(model,
         val_dataset=val_dataset,
         initial_epoch=initial_epoch,
         steps_per_epoch=steps_per_epoch,
-        validation_steps=validation_steps)
+        validation_steps=validation_steps,
+        validation_freq=1)
   else:
     return training_arrays.fit_loop(
         model,
@@ -124,7 +127,8 @@ def fit_distributed(model,
         shuffle=shuffle,
         initial_epoch=initial_epoch,
         steps_per_epoch=steps_per_epoch,
-        validation_steps=validation_steps)
+        validation_steps=validation_steps,
+        validation_freq=validation_freq)
 
 
 def evaluate_distributed(model,
@@ -206,7 +210,8 @@ def experimental_tpu_fit_loop(model,
                               initial_epoch=0,
                               steps_per_epoch=None,
                               val_dataset=None,
-                              validation_steps=None):
+                              validation_steps=None,
+                              validation_freq=1):
   """Fit loop for training with TPU DistributionStrategy.
 
   Arguments:
@@ -224,6 +229,13 @@ def experimental_tpu_fit_loop(model,
       validation_steps: Number of steps to run validation for
           (only if doing validation from data tensors).
           Ignored with the default value of `None`.
+      validation_freq: Only relevant if validation data is provided. Integer or
+          `collections.Container` instance (e.g. list, tuple, etc.). If an
+          integer, specifies how many training epochs to run before a new
+          validation run is performed, e.g. `validation_freq=2` runs
+          validation every 2 epochs. If a Container, specifies the epochs on
+          which to run validation, e.g. `validation_freq=[1, 2, 10]` runs
+          validation at the end of the 1st, 2nd, and 10th epochs.
 
   Returns:
       Returns `None`.
@@ -363,7 +375,8 @@ def experimental_tpu_fit_loop(model,
       if callbacks.model.stop_training:
         break
 
-    if do_validation:
+    if (do_validation and
+        training_utils.should_run_validation(validation_freq, epoch)):
       logging.info('Running validation at fit epoch: %s', epoch)
 
       if model._compile_distribution:
diff --git a/tensorflow/python/keras/engine/training_generator.py b/tensorflow/python/keras/engine/training_generator.py
index 8a5e24856c..23b5d7e8c5 100644
--- a/tensorflow/python/keras/engine/training_generator.py
+++ b/tensorflow/python/keras/engine/training_generator.py
@@ -46,6 +46,7 @@ def model_iteration(model,
                     callbacks=None,
                     validation_data=None,
                     validation_steps=None,
+                    validation_freq=1,
                     class_weight=None,
                     max_queue_size=10,
                     workers=1,
@@ -74,6 +75,13 @@ def model_iteration(model,
         `keras.utils.data_utils.Sequence` object or Eager Iterator or Dataset.
       validation_steps: Total number of steps (batches of samples) before
         declaring validation finished.
+      validation_freq: Only relevant if validation data is provided. Integer or
+        `collections.Container` instance (e.g. list, tuple, etc.). If an
+        integer, specifies how many training epochs to run before a new
+        validation run is performed, e.g. `validation_freq=2` runs
+        validation every 2 epochs. If a Container, specifies the epochs on
+        which to run validation, e.g. `validation_freq=[1, 2, 10]` runs
+        validation at the end of the 1st, 2nd, and 10th epochs.
       class_weight: Dictionary mapping class indices to a weight for the class.
       max_queue_size: Integer. Maximum size for the generator queue. If
         unspecified, `max_queue_size` will default to 10.
@@ -257,7 +265,9 @@ def model_iteration(model,
       results = results[0]
 
     # Run the test loop every epoch during training.
-    if do_validation and not callbacks.model.stop_training:
+    if (do_validation and
+        training_utils.should_run_validation(validation_freq, epoch) and
+        not callbacks.model.stop_training):
       val_results = model_iteration(
           model,
           validation_data,
diff --git a/tensorflow/python/keras/engine/training_test.py b/tensorflow/python/keras/engine/training_test.py
index fa26385cc4..6be4da70f6 100644
--- a/tensorflow/python/keras/engine/training_test.py
+++ b/tensorflow/python/keras/engine/training_test.py
@@ -22,6 +22,7 @@ import io
 import logging
 import sys
 
+from absl.testing import parameterized
 import numpy as np
 import six
 
@@ -803,6 +804,34 @@ class TrainingTest(keras_parameterized.TestCase):
       test_model_loss = test_model.train_on_batch(train_x, train_y)
       self.assertAlmostEqual(test_model_loss, reference_model_loss, places=4)
 
+  @keras_parameterized.run_with_all_model_types
+  @keras_parameterized.run_all_keras_modes
+  @parameterized.named_parameters(
+      ('default', 1, 4), ('integer_two', 2, 2), ('integer_four', 4, 1),
+      ('simple_list', [1, 3, 4], 3), ('duplicated_list', [4, 2, 2], 2))
+  def test_validation_freq(self, validation_freq, expected_runs):
+    x, y = np.ones((10, 10)), np.ones((10, 1))
+    model = testing_utils.get_small_mlp(2, 1, 10)
+    model.compile('sgd', 'mse')
+
+    class ValCounter(keras.callbacks.Callback):
+
+      def __init__(self):
+        self.val_runs = 0
+
+      def on_test_begin(self, logs=None):
+        self.val_runs += 1
+
+    val_counter = ValCounter()
+    model.fit(
+        x,
+        y,
+        epochs=4,
+        validation_data=(x, y),
+        validation_freq=validation_freq,
+        callbacks=[val_counter])
+    self.assertEqual(val_counter.val_runs, expected_runs)
+
 
 class TestExceptionsAndWarnings(keras_parameterized.TestCase):
 
diff --git a/tensorflow/python/keras/engine/training_utils.py b/tensorflow/python/keras/engine/training_utils.py
index c36b637829..3affb84147 100644
--- a/tensorflow/python/keras/engine/training_utils.py
+++ b/tensorflow/python/keras/engine/training_utils.py
@@ -19,6 +19,7 @@ from __future__ import division
 from __future__ import print_function
 
 import abc
+import collections
 from collections import OrderedDict
 import copy
 import json
@@ -1502,3 +1503,33 @@ def should_run_multi_worker():
   tf_config = json.loads(os.environ.get('TF_CONFIG', '{}'))
   cluster_spec = server_lib.ClusterSpec(tf_config.get('cluster', {}))
   return tf_config and 'master' not in cluster_spec.jobs
+
+
+def should_run_validation(validation_freq, epoch):
+  """Checks if validation should be run this epoch.
+
+  Arguments:
+    validation_freq: Integer or list. If an integer, specifies how many training
+      epochs to run before a new validation run is performed. If a list,
+      specifies the epochs on which to run validation.
+    epoch: Integer, the number of the training epoch just completed.
+
+  Returns:
+    Bool, True if validation should be run.
+
+  Raises:
+    ValueError: if `validation_freq` is an Integer and less than 1, or if
+    it is neither an Integer nor a Sequence.
+  """
+  # `epoch` is 0-indexed internally but 1-indexed in the public API.
+  one_indexed_epoch = epoch + 1
+
+  if isinstance(validation_freq, int):
+    if validation_freq < 1:
+      raise ValueError('`validation_freq` can not be less than 1.')
+    return one_indexed_epoch % validation_freq == 0
+
+  if not isinstance(validation_freq, collections.Container):
+    raise ValueError('`validation_freq` must be an Integer or '
+                     '`collections.Container` (e.g. list, tuple, etc.)')
+  return one_indexed_epoch in validation_freq
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.-model.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.-model.pbtxt
index 0a81c3a663..283cc6a735 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.-model.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.-model.pbtxt
@@ -175,11 +175,11 @@ tf_class {
   }
   member_method {
     name: "fit"
-    argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'epochs\', \'verbose\', \'callbacks\', \'validation_split\', \'validation_data\', \'shuffle\', \'class_weight\', \'sample_weight\', \'initial_epoch\', \'steps_per_epoch\', \'validation_steps\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'1\', \'1\', \'None\', \'0.0\', \'None\', \'True\', \'None\', \'None\', \'0\', \'None\', \'None\', \'10\', \'1\', \'False\'], "
+    argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'epochs\', \'verbose\', \'callbacks\', \'validation_split\', \'validation_data\', \'shuffle\', \'class_weight\', \'sample_weight\', \'initial_epoch\', \'steps_per_epoch\', \'validation_steps\', \'validation_freq\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'1\', \'1\', \'None\', \'0.0\', \'None\', \'True\', \'None\', \'None\', \'0\', \'None\', \'None\', \'1\', \'10\', \'1\', \'False\'], "
   }
   member_method {
     name: "fit_generator"
-    argspec: "args=[\'self\', \'generator\', \'steps_per_epoch\', \'epochs\', \'verbose\', \'callbacks\', \'validation_data\', \'validation_steps\', \'class_weight\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'shuffle\', \'initial_epoch\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'1\', \'None\', \'None\', \'None\', \'None\', \'10\', \'1\', \'False\', \'True\', \'0\'], "
+    argspec: "args=[\'self\', \'generator\', \'steps_per_epoch\', \'epochs\', \'verbose\', \'callbacks\', \'validation_data\', \'validation_steps\', \'validation_freq\', \'class_weight\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'shuffle\', \'initial_epoch\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'1\', \'None\', \'None\', \'None\', \'1\', \'None\', \'10\', \'1\', \'False\', \'True\', \'0\'], "
   }
   member_method {
     name: "from_config"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.-sequential.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.-sequential.pbtxt
index 2a9ceec30e..95e405aeba 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.-sequential.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.-sequential.pbtxt
@@ -180,11 +180,11 @@ tf_class {
   }
   member_method {
     name: "fit"
-    argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'epochs\', \'verbose\', \'callbacks\', \'validation_split\', \'validation_data\', \'shuffle\', \'class_weight\', \'sample_weight\', \'initial_epoch\', \'steps_per_epoch\', \'validation_steps\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'1\', \'1\', \'None\', \'0.0\', \'None\', \'True\', \'None\', \'None\', \'0\', \'None\', \'None\', \'10\', \'1\', \'False\'], "
+    argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'epochs\', \'verbose\', \'callbacks\', \'validation_split\', \'validation_data\', \'shuffle\', \'class_weight\', \'sample_weight\', \'initial_epoch\', \'steps_per_epoch\', \'validation_steps\', \'validation_freq\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'1\', \'1\', \'None\', \'0.0\', \'None\', \'True\', \'None\', \'None\', \'0\', \'None\', \'None\', \'1\', \'10\', \'1\', \'False\'], "
   }
   member_method {
     name: "fit_generator"
-    argspec: "args=[\'self\', \'generator\', \'steps_per_epoch\', \'epochs\', \'verbose\', \'callbacks\', \'validation_data\', \'validation_steps\', \'class_weight\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'shuffle\', \'initial_epoch\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'1\', \'None\', \'None\', \'None\', \'None\', \'10\', \'1\', \'False\', \'True\', \'0\'], "
+    argspec: "args=[\'self\', \'generator\', \'steps_per_epoch\', \'epochs\', \'verbose\', \'callbacks\', \'validation_data\', \'validation_steps\', \'validation_freq\', \'class_weight\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'shuffle\', \'initial_epoch\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'1\', \'None\', \'None\', \'None\', \'1\', \'None\', \'10\', \'1\', \'False\', \'True\', \'0\'], "
   }
   member_method {
     name: "from_config"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-model.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-model.pbtxt
index 55bd285612..eb1ab1d9dd 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-model.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-model.pbtxt
@@ -175,11 +175,11 @@ tf_class {
   }
   member_method {
     name: "fit"
-    argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'epochs\', \'verbose\', \'callbacks\', \'validation_split\', \'validation_data\', \'shuffle\', \'class_weight\', \'sample_weight\', \'initial_epoch\', \'steps_per_epoch\', \'validation_steps\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'1\', \'1\', \'None\', \'0.0\', \'None\', \'True\', \'None\', \'None\', \'0\', \'None\', \'None\', \'10\', \'1\', \'False\'], "
+    argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'epochs\', \'verbose\', \'callbacks\', \'validation_split\', \'validation_data\', \'shuffle\', \'class_weight\', \'sample_weight\', \'initial_epoch\', \'steps_per_epoch\', \'validation_steps\', \'validation_freq\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'1\', \'1\', \'None\', \'0.0\', \'None\', \'True\', \'None\', \'None\', \'0\', \'None\', \'None\', \'1\', \'10\', \'1\', \'False\'], "
   }
   member_method {
     name: "fit_generator"
-    argspec: "args=[\'self\', \'generator\', \'steps_per_epoch\', \'epochs\', \'verbose\', \'callbacks\', \'validation_data\', \'validation_steps\', \'class_weight\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'shuffle\', \'initial_epoch\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'1\', \'None\', \'None\', \'None\', \'None\', \'10\', \'1\', \'False\', \'True\', \'0\'], "
+    argspec: "args=[\'self\', \'generator\', \'steps_per_epoch\', \'epochs\', \'verbose\', \'callbacks\', \'validation_data\', \'validation_steps\', \'validation_freq\', \'class_weight\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'shuffle\', \'initial_epoch\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'1\', \'None\', \'None\', \'None\', \'1\', \'None\', \'10\', \'1\', \'False\', \'True\', \'0\'], "
   }
   member_method {
     name: "from_config"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-sequential.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-sequential.pbtxt
index a82424167e..c69cf28174 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-sequential.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.models.-sequential.pbtxt
@@ -180,11 +180,11 @@ tf_class {
   }
   member_method {
     name: "fit"
-    argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'epochs\', \'verbose\', \'callbacks\', \'validation_split\', \'validation_data\', \'shuffle\', \'class_weight\', \'sample_weight\', \'initial_epoch\', \'steps_per_epoch\', \'validation_steps\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'1\', \'1\', \'None\', \'0.0\', \'None\', \'True\', \'None\', \'None\', \'0\', \'None\', \'None\', \'10\', \'1\', \'False\'], "
+    argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'epochs\', \'verbose\', \'callbacks\', \'validation_split\', \'validation_data\', \'shuffle\', \'class_weight\', \'sample_weight\', \'initial_epoch\', \'steps_per_epoch\', \'validation_steps\', \'validation_freq\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'1\', \'1\', \'None\', \'0.0\', \'None\', \'True\', \'None\', \'None\', \'0\', \'None\', \'None\', \'1\', \'10\', \'1\', \'False\'], "
   }
   member_method {
     name: "fit_generator"
-    argspec: "args=[\'self\', \'generator\', \'steps_per_epoch\', \'epochs\', \'verbose\', \'callbacks\', \'validation_data\', \'validation_steps\', \'class_weight\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'shuffle\', \'initial_epoch\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'1\', \'None\', \'None\', \'None\', \'None\', \'10\', \'1\', \'False\', \'True\', \'0\'], "
+    argspec: "args=[\'self\', \'generator\', \'steps_per_epoch\', \'epochs\', \'verbose\', \'callbacks\', \'validation_data\', \'validation_steps\', \'validation_freq\', \'class_weight\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'shuffle\', \'initial_epoch\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'1\', \'None\', \'None\', \'None\', \'1\', \'None\', \'10\', \'1\', \'False\', \'True\', \'0\'], "
   }
   member_method {
     name: "from_config"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.-model.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.-model.pbtxt
index 0a81c3a663..283cc6a735 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.-model.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.-model.pbtxt
@@ -175,11 +175,11 @@ tf_class {
   }
   member_method {
     name: "fit"
-    argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'epochs\', \'verbose\', \'callbacks\', \'validation_split\', \'validation_data\', \'shuffle\', \'class_weight\', \'sample_weight\', \'initial_epoch\', \'steps_per_epoch\', \'validation_steps\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'1\', \'1\', \'None\', \'0.0\', \'None\', \'True\', \'None\', \'None\', \'0\', \'None\', \'None\', \'10\', \'1\', \'False\'], "
+    argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'epochs\', \'verbose\', \'callbacks\', \'validation_split\', \'validation_data\', \'shuffle\', \'class_weight\', \'sample_weight\', \'initial_epoch\', \'steps_per_epoch\', \'validation_steps\', \'validation_freq\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'1\', \'1\', \'None\', \'0.0\', \'None\', \'True\', \'None\', \'None\', \'0\', \'None\', \'None\', \'1\', \'10\', \'1\', \'False\'], "
   }
   member_method {
     name: "fit_generator"
-    argspec: "args=[\'self\', \'generator\', \'steps_per_epoch\', \'epochs\', \'verbose\', \'callbacks\', \'validation_data\', \'validation_steps\', \'class_weight\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'shuffle\', \'initial_epoch\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'1\', \'None\', \'None\', \'None\', \'None\', \'10\', \'1\', \'False\', \'True\', \'0\'], "
+    argspec: "args=[\'self\', \'generator\', \'steps_per_epoch\', \'epochs\', \'verbose\', \'callbacks\', \'validation_data\', \'validation_steps\', \'validation_freq\', \'class_weight\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'shuffle\', \'initial_epoch\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'1\', \'None\', \'None\', \'None\', \'1\', \'None\', \'10\', \'1\', \'False\', \'True\', \'0\'], "
   }
   member_method {
     name: "from_config"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.-sequential.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.-sequential.pbtxt
index 2a9ceec30e..95e405aeba 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.-sequential.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.-sequential.pbtxt
@@ -180,11 +180,11 @@ tf_class {
   }
   member_method {
     name: "fit"
-    argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'epochs\', \'verbose\', \'callbacks\', \'validation_split\', \'validation_data\', \'shuffle\', \'class_weight\', \'sample_weight\', \'initial_epoch\', \'steps_per_epoch\', \'validation_steps\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'1\', \'1\', \'None\', \'0.0\', \'None\', \'True\', \'None\', \'None\', \'0\', \'None\', \'None\', \'10\', \'1\', \'False\'], "
+    argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'epochs\', \'verbose\', \'callbacks\', \'validation_split\', \'validation_data\', \'shuffle\', \'class_weight\', \'sample_weight\', \'initial_epoch\', \'steps_per_epoch\', \'validation_steps\', \'validation_freq\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'1\', \'1\', \'None\', \'0.0\', \'None\', \'True\', \'None\', \'None\', \'0\', \'None\', \'None\', \'1\', \'10\', \'1\', \'False\'], "
   }
   member_method {
     name: "fit_generator"
-    argspec: "args=[\'self\', \'generator\', \'steps_per_epoch\', \'epochs\', \'verbose\', \'callbacks\', \'validation_data\', \'validation_steps\', \'class_weight\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'shuffle\', \'initial_epoch\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'1\', \'None\', \'None\', \'None\', \'None\', \'10\', \'1\', \'False\', \'True\', \'0\'], "
+    argspec: "args=[\'self\', \'generator\', \'steps_per_epoch\', \'epochs\', \'verbose\', \'callbacks\', \'validation_data\', \'validation_steps\', \'validation_freq\', \'class_weight\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'shuffle\', \'initial_epoch\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'1\', \'None\', \'None\', \'None\', \'1\', \'None\', \'10\', \'1\', \'False\', \'True\', \'0\'], "
   }
   member_method {
     name: "from_config"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-linear-model.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-linear-model.pbtxt
index d40a999bd8..c4726cf824 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-linear-model.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-linear-model.pbtxt
@@ -180,11 +180,11 @@ tf_class {
   }
   member_method {
     name: "fit"
-    argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'epochs\', \'verbose\', \'callbacks\', \'validation_split\', \'validation_data\', \'shuffle\', \'class_weight\', \'sample_weight\', \'initial_epoch\', \'steps_per_epoch\', \'validation_steps\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'1\', \'1\', \'None\', \'0.0\', \'None\', \'True\', \'None\', \'None\', \'0\', \'None\', \'None\', \'10\', \'1\', \'False\'], "
+    argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'epochs\', \'verbose\', \'callbacks\', \'validation_split\', \'validation_data\', \'shuffle\', \'class_weight\', \'sample_weight\', \'initial_epoch\', \'steps_per_epoch\', \'validation_steps\', \'validation_freq\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'1\', \'1\', \'None\', \'0.0\', \'None\', \'True\', \'None\', \'None\', \'0\', \'None\', \'None\', \'1\', \'10\', \'1\', \'False\'], "
   }
   member_method {
     name: "fit_generator"
-    argspec: "args=[\'self\', \'generator\', \'steps_per_epoch\', \'epochs\', \'verbose\', \'callbacks\', \'validation_data\', \'validation_steps\', \'class_weight\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'shuffle\', \'initial_epoch\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'1\', \'None\', \'None\', \'None\', \'None\', \'10\', \'1\', \'False\', \'True\', \'0\'], "
+    argspec: "args=[\'self\', \'generator\', \'steps_per_epoch\', \'epochs\', \'verbose\', \'callbacks\', \'validation_data\', \'validation_steps\', \'validation_freq\', \'class_weight\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'shuffle\', \'initial_epoch\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'1\', \'None\', \'None\', \'None\', \'1\', \'None\', \'10\', \'1\', \'False\', \'True\', \'0\'], "
   }
   member_method {
     name: "from_config"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-model.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-model.pbtxt
index 55bd285612..eb1ab1d9dd 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-model.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-model.pbtxt
@@ -175,11 +175,11 @@ tf_class {
   }
   member_method {
     name: "fit"
-    argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'epochs\', \'verbose\', \'callbacks\', \'validation_split\', \'validation_data\', \'shuffle\', \'class_weight\', \'sample_weight\', \'initial_epoch\', \'steps_per_epoch\', \'validation_steps\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'1\', \'1\', \'None\', \'0.0\', \'None\', \'True\', \'None\', \'None\', \'0\', \'None\', \'None\', \'10\', \'1\', \'False\'], "
+    argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'epochs\', \'verbose\', \'callbacks\', \'validation_split\', \'validation_data\', \'shuffle\', \'class_weight\', \'sample_weight\', \'initial_epoch\', \'steps_per_epoch\', \'validation_steps\', \'validation_freq\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'1\', \'1\', \'None\', \'0.0\', \'None\', \'True\', \'None\', \'None\', \'0\', \'None\', \'None\', \'1\', \'10\', \'1\', \'False\'], "
   }
   member_method {
     name: "fit_generator"
-    argspec: "args=[\'self\', \'generator\', \'steps_per_epoch\', \'epochs\', \'verbose\', \'callbacks\', \'validation_data\', \'validation_steps\', \'class_weight\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'shuffle\', \'initial_epoch\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'1\', \'None\', \'None\', \'None\', \'None\', \'10\', \'1\', \'False\', \'True\', \'0\'], "
+    argspec: "args=[\'self\', \'generator\', \'steps_per_epoch\', \'epochs\', \'verbose\', \'callbacks\', \'validation_data\', \'validation_steps\', \'validation_freq\', \'class_weight\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'shuffle\', \'initial_epoch\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'1\', \'None\', \'None\', \'None\', \'1\', \'None\', \'10\', \'1\', \'False\', \'True\', \'0\'], "
   }
   member_method {
     name: "from_config"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-sequential.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-sequential.pbtxt
index a82424167e..c69cf28174 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-sequential.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.models.-sequential.pbtxt
@@ -180,11 +180,11 @@ tf_class {
   }
   member_method {
     name: "fit"
-    argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'epochs\', \'verbose\', \'callbacks\', \'validation_split\', \'validation_data\', \'shuffle\', \'class_weight\', \'sample_weight\', \'initial_epoch\', \'steps_per_epoch\', \'validation_steps\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'1\', \'1\', \'None\', \'0.0\', \'None\', \'True\', \'None\', \'None\', \'0\', \'None\', \'None\', \'10\', \'1\', \'False\'], "
+    argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'epochs\', \'verbose\', \'callbacks\', \'validation_split\', \'validation_data\', \'shuffle\', \'class_weight\', \'sample_weight\', \'initial_epoch\', \'steps_per_epoch\', \'validation_steps\', \'validation_freq\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'1\', \'1\', \'None\', \'0.0\', \'None\', \'True\', \'None\', \'None\', \'0\', \'None\', \'None\', \'1\', \'10\', \'1\', \'False\'], "
   }
   member_method {
     name: "fit_generator"
-    argspec: "args=[\'self\', \'generator\', \'steps_per_epoch\', \'epochs\', \'verbose\', \'callbacks\', \'validation_data\', \'validation_steps\', \'class_weight\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'shuffle\', \'initial_epoch\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'1\', \'None\', \'None\', \'None\', \'None\', \'10\', \'1\', \'False\', \'True\', \'0\'], "
+    argspec: "args=[\'self\', \'generator\', \'steps_per_epoch\', \'epochs\', \'verbose\', \'callbacks\', \'validation_data\', \'validation_steps\', \'validation_freq\', \'class_weight\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'shuffle\', \'initial_epoch\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'1\', \'None\', \'None\', \'None\', \'1\', \'None\', \'10\', \'1\', \'False\', \'True\', \'0\'], "
   }
   member_method {
     name: "from_config"
-- 
GitLab


From 661103ed87cdf0f5a792100b543006b71d62697a Mon Sep 17 00:00:00 2001
From: Revan Sopher <rsopher@google.com>
Date: Wed, 16 Jan 2019 14:52:57 -0800
Subject: [PATCH 0837/1095] Bump absl_py from 0.2.2 to 0.7.0, add absl_py's new
 dep enum34.

PiperOrigin-RevId: 229631203
---
 tensorflow/opensource_only.files               |  1 +
 tensorflow/workspace.bzl                       | 18 ++++++++++++++----
 .../systemlibs/absl_py.absl.testing.BUILD      |  6 +++++-
 third_party/systemlibs/enum34.BUILD            | 13 +++++++++++++
 4 files changed, 33 insertions(+), 5 deletions(-)
 create mode 100644 third_party/systemlibs/enum34.BUILD

diff --git a/tensorflow/opensource_only.files b/tensorflow/opensource_only.files
index 03b5e5a73c..30dfce3c83 100644
--- a/tensorflow/opensource_only.files
+++ b/tensorflow/opensource_only.files
@@ -150,6 +150,7 @@ tensorflow/third_party/systemlibs/cython.BUILD
 tensorflow/third_party/systemlibs/double_conversion.BUILD
 tensorflow/third_party/systemlibs/zlib.BUILD
 tensorflow/third_party/systemlibs/jsoncpp.BUILD
+tensorflow/third_party/systemlibs/enum34.BUILD
 tensorflow/third_party/systemlibs/re2.BUILD
 tensorflow/third_party/systemlibs/lmdb.BUILD
 tensorflow/third_party/systemlibs/googleapis.BUILD
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index b8ef43d27e..6fcccd4147 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -317,19 +317,29 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
 
     tf_http_archive(
         name = "absl_py",
-        sha256 = "95160f778a62c7a60ddeadc7bf2d83f85a23a27359814aca12cf949e896fa82c",
-        strip_prefix = "abseil-py-pypi-v0.2.2",
+        sha256 = "595726be4bf3f7e6d64a1a255fa03717b693c01b913768abd52649cbb7ddf2bd",
+        strip_prefix = "abseil-py-pypi-v0.7.0",
         system_build_file = clean_dep("//third_party/systemlibs:absl_py.BUILD"),
         system_link_files = {
             "//third_party/systemlibs:absl_py.absl.flags.BUILD": "absl/flags/BUILD",
             "//third_party/systemlibs:absl_py.absl.testing.BUILD": "absl/testing/BUILD",
         },
         urls = [
-            "https://mirror.bazel.build/github.com/abseil/abseil-py/archive/pypi-v0.2.2.tar.gz",
-            "https://github.com/abseil/abseil-py/archive/pypi-v0.2.2.tar.gz",
+            "https://mirror.bazel.build/github.com/abseil/abseil-py/archive/pypi-v0.7.0.tar.gz",
+            "https://github.com/abseil/abseil-py/archive/pypi-v0.7.0.tar.gz",
         ],
     )
 
+    tf_http_archive(
+        name = "enum34_archive",
+        urls = [
+            "https://mirror.bazel.build/pypi.python.org/packages/bf/3e/31d502c25302814a7c2f1d3959d2a3b3f78e509002ba91aea64993936876/enum34-1.1.6.tar.gz",
+            "https://pypi.python.org/packages/bf/3e/31d502c25302814a7c2f1d3959d2a3b3f78e509002ba91aea64993936876/enum34-1.1.6.tar.gz",
+        ],
+        sha256 = "8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1",
+        build_file = clean_dep("//third_party/systemlibs:enum34.BUILD"),
+    )
+
     tf_http_archive(
         name = "org_python_pypi_backports_weakref",
         build_file = clean_dep("//third_party:backports_weakref.BUILD"),
diff --git a/third_party/systemlibs/absl_py.absl.testing.BUILD b/third_party/systemlibs/absl_py.absl.testing.BUILD
index c1b794c1e9..7629509ebb 100644
--- a/third_party/systemlibs/absl_py.absl.testing.BUILD
+++ b/third_party/systemlibs/absl_py.absl.testing.BUILD
@@ -2,6 +2,10 @@ licenses(["notice"])  # Apache 2.0
 
 py_library(
     name = "parameterized",
-    testonly = 1,
     visibility = ["//visibility:public"],
 )
+
+py_library(
+    name = "absltest",
+    visibility = ["//visiblity:public"],
+)
diff --git a/third_party/systemlibs/enum34.BUILD b/third_party/systemlibs/enum34.BUILD
new file mode 100644
index 0000000000..a5ac25592b
--- /dev/null
+++ b/third_party/systemlibs/enum34.BUILD
@@ -0,0 +1,13 @@
+# Description:
+#   enum34 provides a backport of the enum module for Python 2.
+
+licenses(["notice"])  # MIT
+
+exports_files(["LICENSE"])
+
+py_library(
+    name = "enum",
+    srcs = ["enum34-1.1.6/enum/__init__.py"],
+    srcs_version = "PY2AND3",
+    visibility = ["//visibility:public"],
+)
-- 
GitLab


From 715f5c73cfaef2991d393024f019d7906ae27e8d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 16 Jan 2019 14:53:05 -0800
Subject: [PATCH 0838/1095] Update NASCell to LayerRNNCell.

PiperOrigin-RevId: 229631237
---
 .../rnn/python/kernel_tests/rnn_cell_test.py  | 13 +++
 tensorflow/contrib/rnn/python/ops/rnn_cell.py | 80 +++++++++++--------
 2 files changed, 59 insertions(+), 34 deletions(-)

diff --git a/tensorflow/contrib/rnn/python/kernel_tests/rnn_cell_test.py b/tensorflow/contrib/rnn/python/kernel_tests/rnn_cell_test.py
index aa1d7d2b01..d7ee7fb8fa 100644
--- a/tensorflow/contrib/rnn/python/kernel_tests/rnn_cell_test.py
+++ b/tensorflow/contrib/rnn/python/kernel_tests/rnn_cell_test.py
@@ -29,7 +29,9 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import random_seed
+from tensorflow.python.framework import test_util
 from tensorflow.python.keras import initializers
+from tensorflow.python.keras import layers as keras_layers
 from tensorflow.python.keras import testing_utils
 from tensorflow.python.keras import utils
 from tensorflow.python.ops import array_ops
@@ -763,6 +765,17 @@ class RNNCellTest(test.TestCase):
         self.assertEqual(new_h.shape[1], num_proj)
         self.assertAllClose(np.concatenate(res[1], axis=1), expected_state)
 
+  @test_util.run_in_graph_and_eager_modes
+  def testNASCellKerasRNN(self):
+    """Tests that NASCell works with keras RNN layer."""
+    cell = contrib_rnn_cell.NASCell(10)
+    seq_input = ops.convert_to_tensor(
+        np.random.rand(2, 3, 5), name="seq_input", dtype=dtypes.float32)
+    rnn_layer = keras_layers.RNN(cell=cell)
+    rnn_outputs = rnn_layer(seq_input)
+    self.evaluate([variables.global_variables_initializer()])
+    self.assertEqual(self.evaluate(rnn_outputs).shape, (2, 10))
+
   def testUGRNNCell(self):
     num_units = 2
     batch_size = 3
diff --git a/tensorflow/contrib/rnn/python/ops/rnn_cell.py b/tensorflow/contrib/rnn/python/ops/rnn_cell.py
index a2179cae0c..482e547a16 100644
--- a/tensorflow/contrib/rnn/python/ops/rnn_cell.py
+++ b/tensorflow/contrib/rnn/python/ops/rnn_cell.py
@@ -1462,7 +1462,7 @@ class LayerNormBasicLSTMCell(rnn_cell_impl.RNNCell):
     return new_h, new_state
 
 
-class NASCell(rnn_cell_impl.RNNCell):
+class NASCell(rnn_cell_impl.LayerRNNCell):
   """Neural Architecture Search (NAS) recurrent network cell.
 
   This implements the recurrent cell from the paper:
@@ -1475,23 +1475,28 @@ class NASCell(rnn_cell_impl.RNNCell):
   The class uses an optional projection layer.
   """
 
-  def __init__(self, num_units, num_proj=None, use_biases=False, reuse=None):
+  # NAS cell's architecture base.
+  _NAS_BASE = 8
+
+  def __init__(self, num_units, num_proj=None, use_bias=False, reuse=None,
+               **kwargs):
     """Initialize the parameters for a NAS cell.
 
     Args:
-      num_units: int, The number of units in the NAS cell
+      num_units: int, The number of units in the NAS cell.
       num_proj: (optional) int, The output dimensionality for the projection
         matrices.  If None, no projection is performed.
-      use_biases: (optional) bool, If True then use biases within the cell. This
+      use_bias: (optional) bool, If True then use biases within the cell. This
         is False by default.
       reuse: (optional) Python boolean describing whether to reuse variables
         in an existing scope.  If not `True`, and the existing scope already has
         the given variables, an error is raised.
+      **kwargs: Additional keyword arguments.
     """
-    super(NASCell, self).__init__(_reuse=reuse)
+    super(NASCell, self).__init__(_reuse=reuse, **kwargs)
     self._num_units = num_units
     self._num_proj = num_proj
-    self._use_biases = use_biases
+    self._use_bias = use_bias
     self._reuse = reuse
 
     if num_proj is not None:
@@ -1509,6 +1514,33 @@ class NASCell(rnn_cell_impl.RNNCell):
   def output_size(self):
     return self._output_size
 
+  def build(self, inputs_shape):
+    input_size = tensor_shape.dimension_value(
+        tensor_shape.TensorShape(inputs_shape).with_rank(2)[1])
+    if input_size is None:
+      raise ValueError("Could not infer input size from inputs.get_shape()[-1]")
+
+    num_proj = self._num_units if self._num_proj is None else self._num_proj
+
+    # Variables for the NAS cell. `recurrent_kernel` is all matrices multiplying
+    # the hiddenstate and `kernel` is all matrices multiplying the inputs.
+    self.recurrent_kernel = self.add_variable(
+        "recurrent_kernel", [num_proj, self._NAS_BASE * self._num_units])
+    self.kernel = self.add_variable(
+        "kernel", [input_size, self._NAS_BASE * self._num_units])
+
+    if self._use_bias:
+      self.bias = self.add_variable("bias",
+                                    shape=[self._NAS_BASE * self._num_units],
+                                    initializer=init_ops.zeros_initializer)
+
+    # Projection layer if specified
+    if self._num_proj is not None:
+      self.projection_weights = self.add_variable(
+          "projection_weights", [self._num_units, self._num_proj])
+
+    self.built = True
+
   def call(self, inputs, state):
     """Run one step of NAS Cell.
 
@@ -1535,38 +1567,20 @@ class NASCell(rnn_cell_impl.RNNCell):
     tanh = math_ops.tanh
     relu = nn_ops.relu
 
-    num_proj = self._num_units if self._num_proj is None else self._num_proj
-
     (c_prev, m_prev) = state
 
-    dtype = inputs.dtype
-    input_size = inputs.get_shape().with_rank(2).dims[1]
-    if input_size.value is None:
-      raise ValueError("Could not infer input size from inputs.get_shape()[-1]")
-    # Variables for the NAS cell. W_m is all matrices multiplying the
-    # hiddenstate and W_inputs is all matrices multiplying the inputs.
-    concat_w_m = vs.get_variable("recurrent_kernel",
-                                 [num_proj, 8 * self._num_units], dtype)
-    concat_w_inputs = vs.get_variable(
-        "kernel", [input_size.value, 8 * self._num_units], dtype)
-
-    m_matrix = math_ops.matmul(m_prev, concat_w_m)
-    inputs_matrix = math_ops.matmul(inputs, concat_w_inputs)
-
-    if self._use_biases:
-      b = vs.get_variable(
-          "bias",
-          shape=[8 * self._num_units],
-          initializer=init_ops.zeros_initializer(),
-          dtype=dtype)
-      m_matrix = nn_ops.bias_add(m_matrix, b)
+    m_matrix = math_ops.matmul(m_prev, self.recurrent_kernel)
+    inputs_matrix = math_ops.matmul(inputs, self.kernel)
+
+    if self._use_bias:
+      m_matrix = nn_ops.bias_add(m_matrix, self.bias)
 
     # The NAS cell branches into 8 different splits for both the hiddenstate
     # and the input
     m_matrix_splits = array_ops.split(
-        axis=1, num_or_size_splits=8, value=m_matrix)
+        axis=1, num_or_size_splits=self._NAS_BASE, value=m_matrix)
     inputs_matrix_splits = array_ops.split(
-        axis=1, num_or_size_splits=8, value=inputs_matrix)
+        axis=1, num_or_size_splits=self._NAS_BASE, value=inputs_matrix)
 
     # First layer
     layer1_0 = sigmoid(inputs_matrix_splits[0] + m_matrix_splits[0])
@@ -1598,9 +1612,7 @@ class NASCell(rnn_cell_impl.RNNCell):
 
     # Projection layer if specified
     if self._num_proj is not None:
-      concat_w_proj = vs.get_variable("projection_weights",
-                                      [self._num_units, self._num_proj], dtype)
-      new_m = math_ops.matmul(new_m, concat_w_proj)
+      new_m = math_ops.matmul(new_m, self.projection_weights)
 
     new_state = rnn_cell_impl.LSTMStateTuple(new_c, new_m)
     return new_m, new_state
-- 
GitLab


From f98110b559362b086409438b00f5b73c9d870b60 Mon Sep 17 00:00:00 2001
From: Blake Hechtman <blakehechtman@google.com>
Date: Wed, 16 Jan 2019 14:54:24 -0800
Subject: [PATCH 0839/1095] [TF] Use compare and cast instead of floor for
 dropout.

PiperOrigin-RevId: 229631446
---
 .../layers/python/layers/layers_test.py       |  2 +-
 .../python/util/parse_layer_parameters.py     | 23 ++++++++++---------
 .../python/keras/engine/topology_test.py      |  6 ++---
 .../python/keras/layers/wrappers_test.py      |  2 +-
 tensorflow/python/ops/nn_ops.py               | 14 +++++------
 5 files changed, 23 insertions(+), 24 deletions(-)

diff --git a/tensorflow/contrib/layers/python/layers/layers_test.py b/tensorflow/contrib/layers/python/layers/layers_test.py
index d791418c9d..1c0088186c 100644
--- a/tensorflow/contrib/layers/python/layers/layers_test.py
+++ b/tensorflow/contrib/layers/python/layers/layers_test.py
@@ -1356,7 +1356,7 @@ class DropoutTest(test.TestCase):
     with self.cached_session():
       images = np.random.uniform(size=(5, height, width, 3))
       output = _layers.dropout(images)
-      self.assertEqual(output.op.name, 'Dropout/dropout_1/mul')
+      self.assertEqual(output.op.name, 'Dropout/dropout_1/mul_1')
       output.get_shape().assert_is_compatible_with(
           ops.convert_to_tensor(images).get_shape())
 
diff --git a/tensorflow/contrib/receptive_field/python/util/parse_layer_parameters.py b/tensorflow/contrib/receptive_field/python/util/parse_layer_parameters.py
index 0e3c46f17d..92ae1021bc 100644
--- a/tensorflow/contrib/receptive_field/python/util/parse_layer_parameters.py
+++ b/tensorflow/contrib/receptive_field/python/util/parse_layer_parameters.py
@@ -27,7 +27,8 @@ from tensorflow.python.platform import tf_logging as logging
 _UNCHANGED_RF_LAYER_OPS = [
     "Add", "BiasAdd", "Cast", "Ceil", "ConcatV2", "Const", "Floor",
     "FusedBatchNorm", "Identity", "Log", "Mul", "Pow", "RealDiv", "Relu",
-    "Relu6", "Round", "Rsqrt", "Softplus", "Sub", "VariableV2", "LRN"
+    "Relu6", "Round", "Rsqrt", "Softplus", "Sub", "VariableV2", "LRN",
+    "GreaterEqual"
 ]
 
 # Different ways in which padding modes may be spelled.
@@ -276,11 +277,11 @@ def get_layer_params(node, name_to_node, input_resolution=None, force=False):
     kernel_size_x, kernel_size_y = _conv_kernel_size(node, name_to_node)
     # Compute the padding for this node separately for each direction.
     total_padding_x, padding_x = _padding_size_conv_pool(
-        node, kernel_size_x, stride_x, input_resolution[1]
-        if input_resolution is not None else None)
+        node, kernel_size_x, stride_x,
+        input_resolution[1] if input_resolution is not None else None)
     total_padding_y, padding_y = _padding_size_conv_pool(
-        node, kernel_size_y, stride_y, input_resolution[0]
-        if input_resolution is not None else None)
+        node, kernel_size_y, stride_y,
+        input_resolution[0] if input_resolution is not None else None)
   elif node.op == "Pad":
     # Kernel and stride are simply 1 in this case.
     kernel_size_x = 1
@@ -294,11 +295,11 @@ def get_layer_params(node, name_to_node, input_resolution=None, force=False):
     kernel_size_x, kernel_size_y = _pool_kernel_size(node, name_to_node)
     # Compute the padding for this node separately for each direction.
     total_padding_x, padding_x = _padding_size_conv_pool(
-        node, kernel_size_x, stride_x, input_resolution[1]
-        if input_resolution is not None else None)
+        node, kernel_size_x, stride_x,
+        input_resolution[1] if input_resolution is not None else None)
     total_padding_y, padding_y = _padding_size_conv_pool(
-        node, kernel_size_y, stride_y, input_resolution[0]
-        if input_resolution is not None else None)
+        node, kernel_size_y, stride_y,
+        input_resolution[0] if input_resolution is not None else None)
   elif node.op in _UNCHANGED_RF_LAYER_OPS:
     # These nodes do not modify the RF parameters.
     kernel_size_x = 1
@@ -320,7 +321,7 @@ def get_layer_params(node, name_to_node, input_resolution=None, force=False):
       total_padding_y = None
       padding_y = None
     else:
-      raise ValueError("Unknown layer for operation '%s': %s" % (node.name,
-                                                                 node.op))
+      raise ValueError(
+          "Unknown layer for operation '%s': %s" % (node.name, node.op))
   return (kernel_size_x, kernel_size_y, stride_x, stride_y, padding_x,
           padding_y, total_padding_x, total_padding_y)
diff --git a/tensorflow/python/keras/engine/topology_test.py b/tensorflow/python/keras/engine/topology_test.py
index 9022dc9481..951988d852 100644
--- a/tensorflow/python/keras/engine/topology_test.py
+++ b/tensorflow/python/keras/engine/topology_test.py
@@ -358,17 +358,17 @@ class TopologyConstructionTest(keras_parameterized.TestCase):
     x = keras.layers.Dropout(0.5)(x, training=True)
     model = keras.models.Model(inp, x)
     # Would be `dropout/cond/Merge` by default
-    self.assertTrue(model.output.op.name.endswith('dropout/mul'))
+    self.assertTrue(model.output.op.name.endswith('dropout/mul_1'))
 
     # Test that argument is kept when applying the model
     inp2 = keras.layers.Input(shape=(2,))
     out2 = model(inp2)
-    self.assertTrue(out2.op.name.endswith('dropout/mul'))
+    self.assertTrue(out2.op.name.endswith('dropout/mul_1'))
 
     # Test that argument is kept after loading a model
     config = model.get_config()
     model = keras.models.Model.from_config(config)
-    self.assertTrue(model.output.op.name.endswith('dropout/mul'))
+    self.assertTrue(model.output.op.name.endswith('dropout/mul_1'))
 
   def test_node_construction(self):
     # test basics
diff --git a/tensorflow/python/keras/layers/wrappers_test.py b/tensorflow/python/keras/layers/wrappers_test.py
index 00e5276d6a..313a2193eb 100644
--- a/tensorflow/python/keras/layers/wrappers_test.py
+++ b/tensorflow/python/keras/layers/wrappers_test.py
@@ -159,7 +159,7 @@ class TimeDistributedTest(test.TestCase):
       np.random.seed(1234)
       x = keras.layers.Input(shape=(3, 2))
       y = keras.layers.TimeDistributed(
-          keras.layers.Dropout(.999))(x, training=True)
+          keras.layers.Dropout(.9999))(x, training=True)
       model = keras.models.Model(x, y)
       y = model.predict(np.random.random((10, 3, 2)))
       self.assertAllClose(np.mean(y), 0., atol=1e-1, rtol=1e-1)
diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index f71fcef130..25609f9efa 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -3292,15 +3292,13 @@ def dropout_v2(x, rate, noise_shape=None, seed=None, name=None):  # pylint: disa
         return x
 
     noise_shape = _get_noise_shape(x, noise_shape)
-
-    keep_prob = 1 - rate
-    # uniform [keep_prob, 1.0 + keep_prob)
-    random_tensor = keep_prob
-    random_tensor += random_ops.random_uniform(
+    # Sample a uniform distribution on [0.0, 1.0) and select values larger than
+    # rate.
+    random_tensor = random_ops.random_uniform(
         noise_shape, seed=seed, dtype=x.dtype)
-    # 0. if [keep_prob, 1.0) and 1. if [1.0, 1.0 + keep_prob)
-    binary_tensor = math_ops.floor(random_tensor)
-    ret = math_ops.divide(x, keep_prob) * binary_tensor
+    keep_prob = 1 - rate
+    ret = (1 / keep_prob) * math_ops.cast(keep_prob >= random_tensor,
+                                          x.dtype) * x
     if not context.executing_eagerly():
       ret.set_shape(x.get_shape())
     return ret
-- 
GitLab


From 9850eea1c83422c90a84c5c520070577becec0a5 Mon Sep 17 00:00:00 2001
From: Mingming Liu <mingmingl@google.com>
Date: Wed, 16 Jan 2019 14:58:58 -0800
Subject: [PATCH 0840/1095] Add monitoring for file IO time and TF graph init
 time.

PiperOrigin-RevId: 229632265
---
 tensorflow/cc/saved_model/loader.cc | 43 +++++++++++++++++++++++------
 1 file changed, 35 insertions(+), 8 deletions(-)

diff --git a/tensorflow/cc/saved_model/loader.cc b/tensorflow/cc/saved_model/loader.cc
index 85d3dd01fa..aaf06a74c6 100644
--- a/tensorflow/cc/saved_model/loader.cc
+++ b/tensorflow/cc/saved_model/loader.cc
@@ -21,6 +21,7 @@ limitations under the License.
 #include "tensorflow/cc/saved_model/reader.h"
 #include "tensorflow/core/lib/io/path.h"
 #include "tensorflow/core/lib/monitoring/counter.h"
+#include "tensorflow/core/lib/monitoring/sampler.h"
 #include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/env.h"
@@ -42,9 +43,28 @@ auto* load_latency = monitoring::Counter<1>::New(
     "/tensorflow/cc/saved_model/load_latency",
     "Latency in microseconds for SavedModels that were successfully loaded.",
     "model_path");
+auto* load_latency_by_stage = monitoring::Sampler<2>::New(
+    {
+        "/tensorflow/cc/saved_model/load_latency_by_stage",  // metric name
+        "Distribution of wall time spent (in microseconds) in each stage "
+        "(restore graph from disk, run init graph op, etc) when loading the "
+        "model",
+        "model_path",
+        "stage",
+    },
+    // Scale of 10, power of 1.5 with bucket count 36 (~20 minutes).
+    monitoring::Buckets::Exponential(10, 1.5, 36));
+
 constexpr char kLoadAttemptFail[] = "fail";
 constexpr char kLoadAttemptSuccess[] = "success";
 
+uint64 GetLatencyMicroseconds(const uint64 start_microseconds) {
+  const uint64 end_microseconds = Env::Default()->NowMicros();
+  // Avoid clock skew.
+  if (end_microseconds < start_microseconds) return 0;
+  return end_microseconds - start_microseconds;
+}
+
 Status LoadMetaGraphIntoSession(const MetaGraphDef& meta_graph_def,
                                 const SessionOptions& session_options,
                                 std::unique_ptr<Session>* session) {
@@ -242,6 +262,7 @@ Status LoadSavedModelInternal(const SessionOptions& session_options,
                               const string& export_dir,
                               const std::unordered_set<string>& tags,
                               SavedModelBundle* const bundle) {
+  const uint64 read_start_microseconds = Env::Default()->NowMicros();
   TF_RETURN_IF_ERROR(ReadMetaGraphDefFromSavedModel(export_dir, tags,
                                                     &bundle->meta_graph_def));
 
@@ -256,12 +277,23 @@ Status LoadSavedModelInternal(const SessionOptions& session_options,
                  bundle->meta_graph_def.saver_def().restore_op_name(),
                  bundle->meta_graph_def.saver_def().filename_tensor_name(),
                  asset_file_defs, bundle->session.get()));
+  // Record walltime spent in restoring graph from disk, but postpone metric
+  // increments until graph init finishes.
+  const uint64 restore_graph_walltime =
+      GetLatencyMicroseconds(read_start_microseconds);
+
+  const uint64 graph_init_start_microseconds = Env::Default()->NowMicros();
   string init_op_name;
   TF_RETURN_IF_ERROR(
       GetInitOp(export_dir, bundle->meta_graph_def, &init_op_name));
   TF_RETURN_IF_ERROR(RunInitOp(run_options, export_dir, bundle->meta_graph_def,
                                asset_file_defs, bundle->session.get(),
                                init_op_name));
+  load_latency_by_stage->GetCell(export_dir, "restore_graph")
+      ->Add(restore_graph_walltime);
+  // Record wall time spent in init op.
+  load_latency_by_stage->GetCell(export_dir, "init_graph")
+      ->Add(GetLatencyMicroseconds(graph_init_start_microseconds));
   return Status::OK();
 }
 
@@ -275,16 +307,10 @@ Status LoadSavedModel(const SessionOptions& session_options,
   const uint64 start_microseconds = Env::Default()->NowMicros();
   const Status status = LoadSavedModelInternal(session_options, run_options,
                                                export_dir, tags, bundle);
-  const uint64 load_latency_microsecs = [&]() -> uint64 {
-    const uint64 end_microseconds = Env::Default()->NowMicros();
-    // Avoid clock skew.
-    if (end_microseconds < start_microseconds) return 0;
-    return end_microseconds - start_microseconds;
-  }();
   auto log_and_count = [&](const string& status_str) {
     LOG(INFO) << "SavedModel load for tags { " << str_util::Join(tags, " ")
               << " }; Status: " << status_str << ". Took "
-              << load_latency_microsecs << " microseconds.";
+              << GetLatencyMicroseconds(start_microseconds) << " microseconds.";
     load_attempt_count->GetCell(export_dir, status_str)->IncrementBy(1);
   };
   if (status.ok()) {
@@ -292,7 +318,8 @@ Status LoadSavedModel(const SessionOptions& session_options,
   } else {
     log_and_count(kLoadAttemptFail);
   }
-  load_latency->GetCell(export_dir)->IncrementBy(load_latency_microsecs);
+  load_latency->GetCell(export_dir)
+      ->IncrementBy(GetLatencyMicroseconds(start_microseconds));
   return status;
 }
 
-- 
GitLab


From 29b51d42ba4950658c566a0ca531b799e019ef7f Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Wed, 16 Jan 2019 15:10:08 -0800
Subject: [PATCH 0841/1095] Update tests to use the new distribute_py_test
 build rule.

PiperOrigin-RevId: 229634446
---
 tensorflow/contrib/distribute/python/BUILD | 66 +++++++---------------
 1 file changed, 21 insertions(+), 45 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index 3a4f23e597..6ae3ec7fb0 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -344,10 +344,14 @@ cuda_py_test(
     ],
 )
 
-py_library(
-    name = "minimize_loss_test_lib",
-    testonly = 1,
+distribute_py_test(
+    name = "minimize_loss_test",
     srcs = ["minimize_loss_test.py"],
+    main = "minimize_loss_test.py",
+    tags = [
+        "multi_and_single_gpu",
+        "no_pip",
+    ],
     deps = [
         ":combinations",
         ":mirrored_strategy",
@@ -367,18 +371,6 @@ py_library(
     ],
 )
 
-cuda_py_test(
-    name = "minimize_loss_test",
-    srcs = ["minimize_loss_test.py"],
-    additional_deps = [
-        ":minimize_loss_test_lib",
-    ],
-    tags = [
-        "multi_and_single_gpu",
-        "no_pip",
-    ],
-)
-
 cuda_py_test(
     name = "moving_averages_test",
     srcs = ["moving_averages_test.py"],
@@ -501,10 +493,14 @@ py_library(
     ],
 )
 
-py_library(
-    name = "step_fn_test_lib",
-    testonly = 1,
+distribute_py_test(
+    name = "step_fn_test",
     srcs = ["step_fn_test.py"],
+    main = "step_fn_test.py",
+    tags = [
+        "multi_and_single_gpu",
+        "no_pip",
+    ],
     deps = [
         ":combinations",
         ":single_loss_example",
@@ -517,18 +513,6 @@ py_library(
     ],
 )
 
-cuda_py_test(
-    name = "step_fn_test",
-    srcs = ["step_fn_test.py"],
-    additional_deps = [
-        ":step_fn_test_lib",
-    ],
-    tags = [
-        "multi_and_single_gpu",
-        "no_pip",
-    ],
-)
-
 py_library(
     name = "monitor",
     srcs = ["monitor.py"],
@@ -766,10 +750,14 @@ distribute_py_test(
     ],
 )
 
-py_library(
-    name = "metrics_v1_test_lib",
-    testonly = 1,
+distribute_py_test(
+    name = "metrics_v1_test",
     srcs = ["metrics_v1_test.py"],
+    main = "metrics_v1_test.py",
+    tags = [
+        "multi_and_single_gpu",
+        "no_pip",
+    ],
     deps = [
         ":combinations",
         "//tensorflow/python:math_ops",
@@ -781,18 +769,6 @@ py_library(
     ],
 )
 
-cuda_py_test(
-    name = "metrics_v1_test",
-    srcs = ["metrics_v1_test.py"],
-    additional_deps = [
-        ":metrics_v1_test_lib",
-    ],
-    tags = [
-        "multi_and_single_gpu",
-        "no_pip",
-    ],
-)
-
 cuda_py_test(
     name = "warm_starting_util_test",
     size = "medium",
-- 
GitLab


From e0d156b3bfef7388b31b35793a2add7b0a5fab6d Mon Sep 17 00:00:00 2001
From: Yifei Feng <yifeif@google.com>
Date: Wed, 16 Jan 2019 15:10:22 -0800
Subject: [PATCH 0842/1095] [WIP] Part II of #22562: Make TensorFlow core
 dynamically load cuda driver instead of static linking.

PiperOrigin-RevId: 229634483
---
 .../gpu/gpu_cudamalloc_allocator.cc           |   6 +-
 tensorflow/stream_executor/build_defs.bzl     |   3 +-
 tensorflow/stream_executor/cuda/BUILD         |   7 +-
 .../stream_executor/cuda/cuda_driver.cc       | 213 ++++++++++--------
 .../cuda/cuda_driver_wrapper.h                | 143 ++++++++++++
 .../stream_executor/cuda/cuda_gpu_executor.cc |   5 +-
 6 files changed, 277 insertions(+), 100 deletions(-)
 create mode 100644 tensorflow/stream_executor/cuda/cuda_driver_wrapper.h

diff --git a/tensorflow/core/common_runtime/gpu/gpu_cudamalloc_allocator.cc b/tensorflow/core/common_runtime/gpu/gpu_cudamalloc_allocator.cc
index d85ca8892f..4be1bbb7df 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_cudamalloc_allocator.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_cudamalloc_allocator.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #ifdef GOOGLE_CUDA
 #include "cuda/include/cuda.h"
 #include "tensorflow/stream_executor/cuda/cuda_activation.h"
+#include "tensorflow/stream_executor/cuda/cuda_driver_wrapper.h"
 #endif  // GOOGLE_CUDA
 
 #include "tensorflow/core/common_runtime/gpu/gpu_cudamalloc_allocator.h"
@@ -41,7 +42,7 @@ void* GPUcudaMallocAllocator::AllocateRaw(size_t alignment, size_t num_bytes) {
   // allocate with cudaMalloc
   se::cuda::ScopedActivateExecutorContext scoped_activation{stream_exec_};
   CUdeviceptr rv = 0;
-  CUresult res = cuMemAlloc(&rv, num_bytes);
+  CUresult res = tensorflow::wrap::cuMemAlloc(&rv, num_bytes);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "cuMemAlloc failed to allocate " << num_bytes;
     return nullptr;
@@ -54,7 +55,8 @@ void* GPUcudaMallocAllocator::AllocateRaw(size_t alignment, size_t num_bytes) {
 void GPUcudaMallocAllocator::DeallocateRaw(void* ptr) {
 #ifdef GOOGLE_CUDA
   // free with cudaFree
-  CUresult res = cuMemFree(reinterpret_cast<CUdeviceptr>(ptr));
+  CUresult res =
+      tensorflow::wrap::cuMemFree(reinterpret_cast<CUdeviceptr>(ptr));
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "cuMemFree failed to free " << ptr;
   }
diff --git a/tensorflow/stream_executor/build_defs.bzl b/tensorflow/stream_executor/build_defs.bzl
index a7ddf5a0d7..717c13d113 100644
--- a/tensorflow/stream_executor/build_defs.bzl
+++ b/tensorflow/stream_executor/build_defs.bzl
@@ -4,8 +4,9 @@ def stream_executor_friends():
 def tf_additional_cuda_platform_deps():
   return []
 
+# Use dynamic loading, therefore should be empty.
 def tf_additional_cuda_driver_deps():
-  return ["@local_config_cuda//cuda:cuda_driver"]
+  return []
 
 def tf_additional_cudnn_plugin_deps():
   return []
diff --git a/tensorflow/stream_executor/cuda/BUILD b/tensorflow/stream_executor/cuda/BUILD
index fcfda1337e..77a1851047 100644
--- a/tensorflow/stream_executor/cuda/BUILD
+++ b/tensorflow/stream_executor/cuda/BUILD
@@ -74,7 +74,10 @@ cc_library(
 cc_library(
     name = "cuda_driver",
     srcs = if_cuda_is_configured(["cuda_driver.cc"]),
-    hdrs = if_cuda_is_configured(["cuda_driver.h"]),
+    hdrs = if_cuda_is_configured([
+        "cuda_driver.h",
+        "cuda_driver_wrapper.h",
+    ]),
     deps = if_cuda_is_configured([
         ":cuda_diagnostics",
         "@com_google_absl//absl/base",
@@ -84,6 +87,7 @@ cc_library(
         "//tensorflow/stream_executor:device_options",
         "//tensorflow/stream_executor/lib",
         "//tensorflow/stream_executor/platform",
+        "//tensorflow/stream_executor/platform:dso_loader",
     ] + tf_additional_cuda_driver_deps()),
 )
 
@@ -314,6 +318,7 @@ cc_library(
         "//tensorflow/stream_executor:timer",
         "//tensorflow/stream_executor/lib",
         "//tensorflow/stream_executor/platform",
+        "//tensorflow/stream_executor/platform:dso_loader",
     ]),
     alwayslink = True,
 )
diff --git a/tensorflow/stream_executor/cuda/cuda_driver.cc b/tensorflow/stream_executor/cuda/cuda_driver.cc
index 7aea66e3af..c39e4f59eb 100644
--- a/tensorflow/stream_executor/cuda/cuda_driver.cc
+++ b/tensorflow/stream_executor/cuda/cuda_driver.cc
@@ -25,6 +25,7 @@ limitations under the License.
 #include "absl/container/inlined_vector.h"
 #include "absl/strings/str_cat.h"
 #include "tensorflow/stream_executor/cuda/cuda_diagnostics.h"
+#include "tensorflow/stream_executor/cuda/cuda_driver_wrapper.h"
 #include "tensorflow/stream_executor/lib/env.h"
 #include "tensorflow/stream_executor/lib/error.h"
 #include "tensorflow/stream_executor/lib/human_readable.h"
@@ -108,11 +109,11 @@ class CreatedContexts {
 // Formats CUresult to output prettified values into a log stream.
 string ToString(CUresult result) {
   const char *error_name;
-  if (cuGetErrorName(result, &error_name)) {
+  if (tensorflow::wrap::cuGetErrorName(result, &error_name)) {
     return absl::StrCat("UNKNOWN ERROR (", static_cast<int>(result), ")");
   }
   const char *error_string;
-  if (cuGetErrorString(result, &error_string)) {
+  if (tensorflow::wrap::cuGetErrorString(result, &error_string)) {
     return error_name;
   }
   return absl::StrCat(error_name, ": ", error_string);
@@ -167,7 +168,7 @@ namespace {
 
 // Call cuCtxtSynchronize and crash if it doesn't succeed.
 void SynchronizeOrDie() {
-  auto res = cuCtxSynchronize();
+  auto res = tensorflow::wrap::cuCtxSynchronize();
   if (res != CUDA_SUCCESS) {
     LOG(FATAL) << "Synchronize found "
                << ToString(res) << " :: " << port::CurrentStackTrace();
@@ -203,7 +204,8 @@ ScopedActivateContext::ScopedActivateContext(CudaContext* cuda_context) {
   to_restore_ = (tls->depth == 1 ? nullptr : tls->context);
 
   // Set the context and update thread local.
-  CHECK_EQ(CUDA_SUCCESS, cuCtxSetCurrent(cuda_context->context()));
+  CHECK_EQ(CUDA_SUCCESS,
+           tensorflow::wrap::cuCtxSetCurrent(cuda_context->context()));
   tls->id = cuda_context->id();
   tls->context = cuda_context;
 }
@@ -228,7 +230,8 @@ ScopedActivateContext::~ScopedActivateContext() {
   }
 
   // Set context and update thread local.
-  CHECK_EQ(CUDA_SUCCESS, cuCtxSetCurrent(to_restore_->context()));
+  CHECK_EQ(CUDA_SUCCESS,
+           tensorflow::wrap::cuCtxSetCurrent(to_restore_->context()));
   tls->id = to_restore_->id();
   tls->context = to_restore_;
 }
@@ -290,7 +293,7 @@ static port::Status InternalInit() {
   if (FLAGS_gpuexec_cuda_driver_inject_init_error) {
     LOG(ERROR) << "injecting CUDA init error; initialization will fail";
   } else {
-    res = cuInit(0 /* = flags */);
+    res = tensorflow::wrap::cuInit(0 /* = flags */);
   }
 
   if (res == CUDA_SUCCESS) {
@@ -323,7 +326,7 @@ static port::Status InternalInit() {
 
 /* static */ port::Status CUDADriver::GetDevice(int device_ordinal,
                                                 CUdevice *device) {
-  CUresult res = cuDeviceGet(device, device_ordinal);
+  CUresult res = tensorflow::wrap::cuDeviceGet(device, device_ordinal);
   if (res == CUDA_SUCCESS) {
     return port::Status::OK();
   }
@@ -337,7 +340,8 @@ static port::Status InternalInit() {
                                             string *device_name) {
   static const size_t kCharLimit = 64;
   absl::InlinedVector<char, 4> chars(kCharLimit);
-  CUresult res = cuDeviceGetName(chars.begin(), kCharLimit - 1, device);
+  CUresult res =
+      tensorflow::wrap::cuDeviceGetName(chars.begin(), kCharLimit - 1, device);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to get device name for " << device << ": "
                << ToString(res);
@@ -388,9 +392,9 @@ bool DeviceOptionsToContextFlags(const DeviceOptions &device_options,
 
   unsigned int former_primary_context_flags;
   int former_primary_context_is_active;
-  CHECK_EQ(CUDA_SUCCESS,
-           cuDevicePrimaryCtxGetState(device, &former_primary_context_flags,
-                                      &former_primary_context_is_active));
+  CHECK_EQ(CUDA_SUCCESS, tensorflow::wrap::cuDevicePrimaryCtxGetState(
+                             device, &former_primary_context_flags,
+                             &former_primary_context_is_active));
   if (former_primary_context_flags != flags) {
     if (former_primary_context_is_active) {
       LOG(ERROR)
@@ -398,15 +402,16 @@ bool DeviceOptionsToContextFlags(const DeviceOptions &device_options,
           << former_primary_context_flags << ") than the desired flag set ("
           << flags << ").";
     } else {
-      CHECK_EQ(CUDA_SUCCESS, cuDevicePrimaryCtxSetFlags(device, flags));
+      CHECK_EQ(CUDA_SUCCESS,
+               tensorflow::wrap::cuDevicePrimaryCtxSetFlags(device, flags));
     }
   }
 
   former_context = CUDADriver::CurrentContextOrDie();
-  res = cuDevicePrimaryCtxRetain(&new_context, device);
+  res = tensorflow::wrap::cuDevicePrimaryCtxRetain(&new_context, device);
   if (former_context != nullptr) {
     CUdevice former_device;
-    if (cuCtxGetDevice(&former_device) == CUDA_SUCCESS) {
+    if (tensorflow::wrap::cuCtxGetDevice(&former_device) == CUDA_SUCCESS) {
       if (former_device == device) {
         if (former_context == new_context) {
           VLOG(2) << "The primary context " << former_context << " for device "
@@ -425,7 +430,7 @@ bool DeviceOptionsToContextFlags(const DeviceOptions &device_options,
                  << former_context;
     }
   }
-  CHECK_EQ(CUDA_SUCCESS, cuCtxSetCurrent(former_context));
+  CHECK_EQ(CUDA_SUCCESS, tensorflow::wrap::cuCtxSetCurrent(former_context));
 
   if (res == CUDA_SUCCESS) {
     *context = CreatedContexts::Add(new_context);
@@ -454,12 +459,12 @@ bool DeviceOptionsToContextFlags(const DeviceOptions &device_options,
     return;
   }
   CUcontext former_context = CurrentContext();
-  CUresult res = cuCtxSetCurrent(context->context());
+  CUresult res = tensorflow::wrap::cuCtxSetCurrent(context->context());
   CUdevice device;
-  cuCtxGetDevice(&device);
-  cuCtxSetCurrent(former_context);
+  tensorflow::wrap::cuCtxGetDevice(&device);
+  tensorflow::wrap::cuCtxSetCurrent(former_context);
 
-  res = cuDevicePrimaryCtxRelease(device);
+  res = tensorflow::wrap::cuDevicePrimaryCtxRelease(device);
 
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to release CUDA context; leaking: " << ToString(res);
@@ -471,7 +476,8 @@ bool DeviceOptionsToContextFlags(const DeviceOptions &device_options,
 /* static */ bool CUDADriver::FuncGetAttribute(CUfunction_attribute attribute,
                                                CUfunction func,
                                                int *attribute_value) {
-  CUresult res = cuFuncGetAttribute(attribute_value, attribute, func);
+  CUresult res =
+      tensorflow::wrap::cuFuncGetAttribute(attribute_value, attribute, func);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to query kernel attribute. kernel: " << func
                << ", attribute: " << attribute;
@@ -482,7 +488,7 @@ bool DeviceOptionsToContextFlags(const DeviceOptions &device_options,
 
 /* static */ bool CUDADriver::FuncSetCacheConfig(CUfunction function,
                                                  CUfunc_cache cache_config) {
-  CUresult res = cuFuncSetCacheConfig(function, cache_config);
+  CUresult res = tensorflow::wrap::cuFuncSetCacheConfig(function, cache_config);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to set CUDA kernel cache config. kernel: " << function
                << ", config: " << cache_config << ", result: " << ToString(res);
@@ -496,10 +502,11 @@ bool DeviceOptionsToContextFlags(const DeviceOptions &device_options,
 CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   CUsharedconfig shared_mem_config;
   ScopedActivateContext activation(context);
-  CUresult result = cuCtxGetSharedMemConfig(&shared_mem_config);
+  CUresult result =
+      tensorflow::wrap::cuCtxGetSharedMemConfig(&shared_mem_config);
   if (result != CUDA_SUCCESS) {
     CUdevice device;
-    cuCtxGetDevice(&device);
+    tensorflow::wrap::cuCtxGetDevice(&device);
     LOG(ERROR) << "failed to get CUDA device shared memory config. "
                << "Context device ID: " << device
                << ", result: " << ToString(result);
@@ -513,10 +520,11 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
 /* static */ port::Status CUDADriver::ContextSetSharedMemConfig(
     CudaContext* context, CUsharedconfig shared_mem_config) {
   ScopedActivateContext activation(context);
-  CUresult result = cuCtxSetSharedMemConfig(shared_mem_config);
+  CUresult result =
+      tensorflow::wrap::cuCtxSetSharedMemConfig(shared_mem_config);
   if (result != CUDA_SUCCESS) {
     CUdevice device;
-    cuCtxGetDevice(&device);
+    tensorflow::wrap::cuCtxGetDevice(&device);
     LOG(ERROR) << "failed to set CUDA device shared memory config. "
                << "Context device ID: " << device
                << ", config: " << shared_mem_config
@@ -539,9 +547,9 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
           << " gdy: " << grid_dim_y << " gdz: " << grid_dim_z
           << " bdx: " << block_dim_x << " bdy: " << block_dim_y
           << " bdz: " << block_dim_z;
-  CUresult res = cuLaunchKernel(function, grid_dim_x, grid_dim_y, grid_dim_z,
-                                block_dim_x, block_dim_y, block_dim_z,
-                                shared_mem_bytes, stream, kernel_params, extra);
+  CUresult res = tensorflow::wrap::cuLaunchKernel(
+      function, grid_dim_x, grid_dim_y, grid_dim_z, block_dim_x, block_dim_y,
+      block_dim_z, shared_mem_bytes, stream, kernel_params, extra);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to launch CUDA kernel: " << function
                << "; result: " << ToString(res);
@@ -555,7 +563,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                                                 const char *cubin_bytes,
                                                 CUmodule *module) {
   ScopedActivateContext activation(context);
-  CUresult result = cuModuleLoadFatBinary(module, cubin_bytes);
+  CUresult result =
+      tensorflow::wrap::cuModuleLoadFatBinary(module, cubin_bytes);
   if (result != CUDA_SUCCESS) {
     return port::Status(port::error::INTERNAL,
                         "failed to load in-memory CUBIN: " + ToString(result));
@@ -598,8 +607,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
       // TODO(leary) Need to see if NVIDIA can expunge the leakiness in their
       // module loading: see http://b/13248943
 
-      res = cuModuleLoadDataEx(module, ptx_data, TF_ARRAYSIZE(options),
-                               options, option_values);
+      res = tensorflow::wrap::cuModuleLoadDataEx(
+          module, ptx_data, TF_ARRAYSIZE(options), options, option_values);
     }
 
     // The PTX JIT mutates the values in the option values array to reflect the
@@ -638,7 +647,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                                                      CUdeviceptr location,
                                                      uint8 value, size_t size) {
   ScopedActivateContext activation(context);
-  CUresult res = cuMemsetD8(location, value, size);
+  CUresult res = tensorflow::wrap::cuMemsetD8(location, value, size);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to memset memory: " << ToString(res);
     return false;
@@ -651,7 +660,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                                                       uint32 value,
                                                       size_t uint32_count) {
   ScopedActivateContext activation(context);
-  CUresult res = cuMemsetD32(location, value, uint32_count);
+  CUresult res = tensorflow::wrap::cuMemsetD32(location, value, uint32_count);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to memset memory: " << ToString(res);
     return false;
@@ -665,7 +674,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                                                       size_t uint32_count,
                                                       CUstream stream) {
   ScopedActivateContext activation(context);
-  CUresult res = cuMemsetD8Async(location, value, uint32_count, stream);
+  CUresult res =
+      tensorflow::wrap::cuMemsetD8Async(location, value, uint32_count, stream);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to enqueue async memset operation: " << ToString(res);
     return false;
@@ -680,7 +690,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                                                        size_t uint32_count,
                                                        CUstream stream) {
   ScopedActivateContext activation(context);
-  CUresult res = cuMemsetD32Async(location, value, uint32_count, stream);
+  CUresult res =
+      tensorflow::wrap::cuMemsetD32Async(location, value, uint32_count, stream);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to enqueue async memset operation: " << ToString(res);
     return false;
@@ -694,7 +705,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                                                 StreamCallback callback,
                                                 void *data) {
   // Note: flags param is required to be zero according to CUDA 6.0.
-  CUresult res = cuStreamAddCallback(stream, callback, data, 0 /* = flags */);
+  CUresult res = tensorflow::wrap::cuStreamAddCallback(stream, callback, data,
+                                                       0 /* = flags */);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "unable to add host callback: " << ToString(res);
     return false;
@@ -708,7 +720,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                                                 CUfunction *function) {
   ScopedActivateContext activated{context};
   CHECK(module != nullptr && kernel_name != nullptr);
-  CUresult res = cuModuleGetFunction(function, module, kernel_name);
+  CUresult res =
+      tensorflow::wrap::cuModuleGetFunction(function, module, kernel_name);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to get PTX kernel \"" << kernel_name
                << "\" from module: " << ToString(res);
@@ -726,7 +739,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   ScopedActivateContext activated{context};
   CHECK(module != nullptr && symbol_name != nullptr &&
         (dptr != nullptr || bytes != nullptr));
-  CUresult res = cuModuleGetGlobal(dptr, bytes, module, symbol_name);
+  CUresult res =
+      tensorflow::wrap::cuModuleGetGlobal(dptr, bytes, module, symbol_name);
   if (res != CUDA_SUCCESS) {
     // symbol may not be found in the current module, but it may reside in
     // another module.
@@ -741,7 +755,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
 /* static */ void CUDADriver::UnloadModule(CudaContext *context,
                                            CUmodule module) {
   ScopedActivateContext activated{context};
-  CUresult res = cuModuleUnload(module);
+  CUresult res = tensorflow::wrap::cuModuleUnload(module);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to unload module " << module
                << "; leaking: " << ToString(res);
@@ -752,7 +766,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
     CudaContext* context) {
   ScopedActivateContext activated{context};
   CUdevice device = -1;
-  CUresult result = cuCtxGetDevice(&device);
+  CUresult result = tensorflow::wrap::cuCtxGetDevice(&device);
   if (result == CUDA_SUCCESS) {
     return device;
   }
@@ -768,7 +782,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   // up synchronization with respect to memsets and any other things that have
   // to occur on the default stream?
   ScopedActivateContext activated{context};
-  CUresult res = cuStreamCreate(out, 0);
+  CUresult res = tensorflow::wrap::cuStreamCreate(out, 0);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "could not allocate CUDA stream for context "
                << context->context() << ": " << ToString(res);
@@ -787,7 +801,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   }
 
   ScopedActivateContext activated{context};
-  CUresult res = cuStreamDestroy(*stream);
+  CUresult res = tensorflow::wrap::cuStreamDestroy(*stream);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to destroy CUDA stream for context "
                << context->context() << ": " << ToString(res);
@@ -802,7 +816,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                                               uint64 bytes) {
   ScopedActivateContext activated{context};
   CUdeviceptr result = 0;
-  CUresult res = cuMemAlloc(&result, bytes);
+  CUresult res = tensorflow::wrap::cuMemAlloc(&result, bytes);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to allocate "
                << port::HumanReadableNumBytes::ToString(bytes) << " (" << bytes
@@ -819,7 +833,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                                                void *location) {
   ScopedActivateContext activation(context);
   CUdeviceptr pointer = absl::bit_cast<CUdeviceptr>(location);
-  CUresult res = cuMemFree(pointer);
+  CUresult res = tensorflow::wrap::cuMemFree(pointer);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to free device memory at " << location
                << "; result: " << ToString(res);
@@ -834,7 +848,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   ScopedActivateContext activation(context);
   CUdeviceptr result = 0;
   // "Portable" memory is visible to all CUDA contexts. Safe for our use model.
-  CUresult res = cuMemAllocManaged(&result, bytes, CU_MEM_ATTACH_GLOBAL);
+  CUresult res =
+      tensorflow::wrap::cuMemAllocManaged(&result, bytes, CU_MEM_ATTACH_GLOBAL);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to alloc " << bytes
                << " bytes unified memory; result: " << ToString(res);
@@ -850,7 +865,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                                                       void *location) {
   ScopedActivateContext activation(context);
   CUdeviceptr pointer = absl::bit_cast<CUdeviceptr>(location);
-  CUresult res = cuMemFree(pointer);
+  CUresult res = tensorflow::wrap::cuMemFree(pointer);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to free unified memory at " << location
                << "; result: " << ToString(res);
@@ -865,7 +880,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   ScopedActivateContext activation(context);
   void *host_mem = nullptr;
   // "Portable" memory is visible to all CUDA contexts. Safe for our use model.
-  CUresult res = cuMemHostAlloc(&host_mem, bytes, CU_MEMHOSTALLOC_PORTABLE);
+  CUresult res = tensorflow::wrap::cuMemHostAlloc(&host_mem, bytes,
+                                                  CU_MEMHOSTALLOC_PORTABLE);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to alloc " << bytes
                << " bytes on host: " << ToString(res);
@@ -876,7 +892,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
 /* static */ void CUDADriver::HostDeallocate(CudaContext* context,
                                              void *location) {
   ScopedActivateContext activation(context);
-  CUresult res = cuMemFreeHost(location);
+  CUresult res = tensorflow::wrap::cuMemFreeHost(location);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "error deallocating host memory at " << location << ": "
                << ToString(res);
@@ -887,8 +903,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                                            uint64 bytes) {
   ScopedActivateContext activation(context);
   // "Portable" memory is visible to all CUDA contexts. Safe for our use model.
-  CUresult res =
-      cuMemHostRegister(location, bytes, CU_MEMHOSTREGISTER_PORTABLE);
+  CUresult res = tensorflow::wrap::cuMemHostRegister(
+      location, bytes, CU_MEMHOSTREGISTER_PORTABLE);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "error registering host memory at " << location << ": "
                << ToString(res);
@@ -900,7 +916,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
 /* static */ bool CUDADriver::HostUnregister(CudaContext* context,
                                              void *location) {
   ScopedActivateContext activation(context);
-  CUresult res = cuMemHostUnregister(location);
+  CUresult res = tensorflow::wrap::cuMemHostUnregister(location);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "error unregistering host memory at " << location << ": "
                << ToString(res);
@@ -917,7 +933,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   }
 
   ScopedActivateContext activated{context};
-  CUresult res = cuEventDestroy(*event);
+  CUresult res = tensorflow::wrap::cuEventDestroy(*event);
   *event = nullptr;
 
   switch (res) {
@@ -941,7 +957,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                                                   CUevent event,
                                                   CUstream stream) {
   ScopedActivateContext activated{context};
-  CUresult res = cuEventRecord(event, stream);
+  CUresult res = tensorflow::wrap::cuEventRecord(event, stream);
   switch (res) {
     case CUDA_SUCCESS:
       return port::Status::OK();
@@ -962,7 +978,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
 /* static */ port::StatusOr<CUresult> CUDADriver::QueryEvent(
     CudaContext *context, CUevent event) {
   ScopedActivateContext activated{context};
-  CUresult res = cuEventQuery(event);
+  CUresult res = tensorflow::wrap::cuEventQuery(event);
   if (res != CUDA_SUCCESS && res != CUDA_ERROR_NOT_READY) {
     return port::Status(
         port::error::INTERNAL,
@@ -978,12 +994,12 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   ScopedActivateContext activated{context};
   // The stop event must have completed in order for cuEventElapsedTime to
   // work.
-  CUresult res = cuEventSynchronize(stop);
+  CUresult res = tensorflow::wrap::cuEventSynchronize(stop);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to synchronize the stop event: " << ToString(res);
     return false;
   }
-  res = cuEventElapsedTime(elapsed_milliseconds, start, stop);
+  res = tensorflow::wrap::cuEventElapsedTime(elapsed_milliseconds, start, stop);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to get elapsed time between events: "
                << ToString(res);
@@ -997,7 +1013,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                                                 CUstream stream,
                                                 CUevent event) {
   ScopedActivateContext activation(context);
-  CUresult res = cuStreamWaitEvent(stream, event, 0 /* = flags */);
+  CUresult res =
+      tensorflow::wrap::cuStreamWaitEvent(stream, event, 0 /* = flags */);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "could not wait stream on event: " << ToString(res);
     return false;
@@ -1008,7 +1025,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
 
 /* static */ bool CUDADriver::SynchronizeContext(CudaContext* context) {
   ScopedActivateContext activation(context);
-  CUresult res = cuCtxSynchronize();
+  CUresult res = tensorflow::wrap::cuCtxSynchronize();
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "could not synchronize on CUDA context: " << ToString(res)
                << " :: " << port::CurrentStackTrace();
@@ -1022,7 +1039,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                                                         CUstream stream) {
   ScopedActivateContext activated{context};
   CHECK(stream != nullptr);
-  CUresult res = cuStreamSynchronize(stream);
+  CUresult res = tensorflow::wrap::cuStreamSynchronize(stream);
   if (res != CUDA_SUCCESS) {
     port::Status status = port::InternalError(
         absl::StrCat("could not synchronize on CUDA stream: ", ToString(res)));
@@ -1038,7 +1055,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                                            CUstream stream) {
   ScopedActivateContext activated{context};
   CHECK(stream != nullptr);
-  CUresult res = cuStreamQuery(stream);
+  CUresult res = tensorflow::wrap::cuStreamQuery(stream);
   if (res == CUDA_SUCCESS) {
     return true;
   }
@@ -1054,7 +1071,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                                                            CUdeviceptr gpu_src,
                                                            uint64 size) {
   ScopedActivateContext activation(context);
-  CUresult res = cuMemcpyDtoH(host_dst, gpu_src, size);
+  CUresult res = tensorflow::wrap::cuMemcpyDtoH(host_dst, gpu_src, size);
   if (res != CUDA_SUCCESS) {
     return port::InternalError(
         port::Printf("failed to synchronous memcpy from device to host: %s; "
@@ -1072,7 +1089,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                                                            const void *host_src,
                                                            uint64 size) {
   ScopedActivateContext activation(context);
-  CUresult res = cuMemcpyHtoD(gpu_dst, host_src, size);
+  CUresult res = tensorflow::wrap::cuMemcpyHtoD(gpu_dst, host_src, size);
   if (res != CUDA_SUCCESS) {
     return port::InternalError(port::Printf(
         "failed to synchronous memcpy from host to device: %s; GPU dst: %p;"
@@ -1089,7 +1106,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                                                            CUdeviceptr gpu_src,
                                                            uint64 size) {
   ScopedActivateContext activation(context);
-  CUresult res = cuMemcpyDtoD(gpu_dst, gpu_src, size);
+  CUresult res = tensorflow::wrap::cuMemcpyDtoD(gpu_dst, gpu_src, size);
   if (res != CUDA_SUCCESS) {
     return port::InternalError(port::Printf(
         "failed to synchronous memcpy from host to device: %s; GPU dst: %p; "
@@ -1107,7 +1124,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                                                     uint64 size,
                                                     CUstream stream) {
   ScopedActivateContext activation(context);
-  CUresult res = cuMemcpyDtoHAsync(host_dst, gpu_src, size, stream);
+  CUresult res =
+      tensorflow::wrap::cuMemcpyDtoHAsync(host_dst, gpu_src, size, stream);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << port::Printf(
         "failed to enqueue async memcpy from device to host: %s; host dst: %p; "
@@ -1128,7 +1146,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                                                     uint64 size,
                                                     CUstream stream) {
   ScopedActivateContext activation(context);
-  CUresult res = cuMemcpyHtoDAsync(gpu_dst, host_src, size, stream);
+  CUresult res =
+      tensorflow::wrap::cuMemcpyHtoDAsync(gpu_dst, host_src, size, stream);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << port::Printf(
         "failed to enqueue async memcpy from host to device: %s; GPU dst: %p; "
@@ -1148,7 +1167,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
                                                     uint64 size,
                                                     CUstream stream) {
   ScopedActivateContext activation(context);
-  CUresult result = cuMemcpyDtoDAsync(gpu_dst, gpu_src, size, stream);
+  CUresult result =
+      tensorflow::wrap::cuMemcpyDtoDAsync(gpu_dst, gpu_src, size, stream);
   if (result != CUDA_SUCCESS) {
     LOG(ERROR) << port::Printf(
         "failed to enqueue async memcpy from device to device: %s"
@@ -1185,7 +1205,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   }
 
   ScopedActivateContext activated{context};
-  CUresult res = cuEventCreate(result, cuflags);
+  CUresult res = tensorflow::wrap::cuEventCreate(result, cuflags);
 
   if (res == CUDA_SUCCESS) {
     return port::Status::OK();
@@ -1201,7 +1221,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
 
 /* static */ int CUDADriver::GetDeviceCount() {
   int device_count = 0;
-  CUresult res = cuDeviceGetCount(&device_count);
+  CUresult res = tensorflow::wrap::cuDeviceGetCount(&device_count);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "could not retrieve CUDA device count: " << ToString(res);
     return 0;
@@ -1216,8 +1236,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
 /* static */ port::StatusOr<CudaContext*> CUDADriver::GetPointerContext(
     CUdeviceptr pointer) {
   CudaContext* context = nullptr;
-  CUresult result =
-      cuPointerGetAttribute(&context, CU_POINTER_ATTRIBUTE_CONTEXT, pointer);
+  CUresult result = tensorflow::wrap::cuPointerGetAttribute(
+      &context, CU_POINTER_ATTRIBUTE_CONTEXT, pointer);
   if (result == CUDA_SUCCESS) {
     CHECK(context != nullptr) << "success should entail non-null context";
     return context;
@@ -1232,8 +1252,8 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
 /* static */ port::StatusOr<MemorySpace> CUDADriver::GetPointerMemorySpace(
     CUdeviceptr pointer) {
   unsigned int value;
-  CUresult result =
-      cuPointerGetAttribute(&value, CU_POINTER_ATTRIBUTE_MEMORY_TYPE, pointer);
+  CUresult result = tensorflow::wrap::cuPointerGetAttribute(
+      &value, CU_POINTER_ATTRIBUTE_MEMORY_TYPE, pointer);
   if (result == CUDA_SUCCESS) {
     switch (value) {
       case CU_MEMORYTYPE_DEVICE:
@@ -1256,7 +1276,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
 /* static */ port::Status CUDADriver::GetPointerAddressRange(CUdeviceptr dptr,
                                                              CUdeviceptr *base,
                                                              size_t *size) {
-  CUresult result = cuMemGetAddressRange(base, size, dptr);
+  CUresult result = tensorflow::wrap::cuMemGetAddressRange(base, size, dptr);
   if (result == CUDA_SUCCESS) {
     return port::Status::OK();
   } else if (result == CUDA_ERROR_NOT_FOUND) {
@@ -1291,7 +1311,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
   *cc_major = 0;
   *cc_minor = 0;
 
-  CUresult res = cuDeviceGetAttribute(
+  CUresult res = tensorflow::wrap::cuDeviceGetAttribute(
       cc_major, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, device);
   if (res != CUDA_SUCCESS) {
     return port::Status(
@@ -1301,7 +1321,7 @@ CUDADriver::ContextGetSharedMemConfig(CudaContext* context) {
             ToString(res).c_str(), device));
   }
 
-  res = cuDeviceGetAttribute(
+  res = tensorflow::wrap::cuDeviceGetAttribute(
       cc_minor, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, device);
   if (res != CUDA_SUCCESS) {
     return port::Status(
@@ -1320,7 +1340,8 @@ template <typename T>
 static port::StatusOr<T> GetSimpleAttribute(CUdevice device,
                                             CUdevice_attribute attribute) {
   int value = -1;
-  CUresult result = cuDeviceGetAttribute(&value, attribute, device);
+  CUresult result =
+      tensorflow::wrap::cuDeviceGetAttribute(&value, attribute, device);
   if (result != CUDA_SUCCESS) {
     return port::Status(
         port::error::NOT_FOUND,
@@ -1375,24 +1396,24 @@ static port::StatusOr<T> GetSimpleAttribute(CUdevice device,
 /* static */ bool CUDADriver::GetGridLimits(int *x, int *y, int *z,
                                             CUdevice device) {
   int value;
-  CUresult res =
-      cuDeviceGetAttribute(&value, CU_DEVICE_ATTRIBUTE_MAX_GRID_DIM_X, device);
+  CUresult res = tensorflow::wrap::cuDeviceGetAttribute(
+      &value, CU_DEVICE_ATTRIBUTE_MAX_GRID_DIM_X, device);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to query max grid dim x: " << ToString(res);
     return false;
   }
   *x = value;
 
-  res =
-      cuDeviceGetAttribute(&value, CU_DEVICE_ATTRIBUTE_MAX_GRID_DIM_Y, device);
+  res = tensorflow::wrap::cuDeviceGetAttribute(
+      &value, CU_DEVICE_ATTRIBUTE_MAX_GRID_DIM_Y, device);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to query max grid dim y: " << ToString(res);
     return false;
   }
   *y = value;
 
-  res =
-      cuDeviceGetAttribute(&value, CU_DEVICE_ATTRIBUTE_MAX_GRID_DIM_Z, device);
+  res = tensorflow::wrap::cuDeviceGetAttribute(
+      &value, CU_DEVICE_ATTRIBUTE_MAX_GRID_DIM_Z, device);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to query max grid dim z: " << ToString(res);
     return false;
@@ -1402,7 +1423,7 @@ static port::StatusOr<T> GetSimpleAttribute(CUdevice device,
 }
 
 /* static */ bool CUDADriver::GetDriverVersion(int *driver_version) {
-  CUresult res = cuDriverGetVersion(driver_version);
+  CUresult res = tensorflow::wrap::cuDriverGetVersion(driver_version);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to query driver version: " << ToString(res);
     return false;
@@ -1414,7 +1435,8 @@ static port::StatusOr<T> GetSimpleAttribute(CUdevice device,
 /* static */ port::StatusOr<int> CUDADriver::GetDeviceAttribute(
     CUdevice_attribute attribute, CUdevice device) {
   int val;
-  CUresult res = cuDeviceGetAttribute(&val, attribute, device);
+  CUresult res =
+      tensorflow::wrap::cuDeviceGetAttribute(&val, attribute, device);
   if (res != CUDA_SUCCESS) {
     return port::Status(
         port::error::INTERNAL,
@@ -1426,8 +1448,8 @@ static port::StatusOr<T> GetSimpleAttribute(CUdevice device,
 
 /* static */ bool CUDADriver::IsEccEnabled(CUdevice device, bool *result) {
   int value = -1;
-  CUresult res =
-      cuDeviceGetAttribute(&value, CU_DEVICE_ATTRIBUTE_ECC_ENABLED, device);
+  CUresult res = tensorflow::wrap::cuDeviceGetAttribute(
+      &value, CU_DEVICE_ATTRIBUTE_ECC_ENABLED, device);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to query ECC status: " << ToString(res);
     return false;
@@ -1443,7 +1465,7 @@ static port::StatusOr<T> GetSimpleAttribute(CUdevice device,
   ScopedActivateContext activation(context);
   size_t free = 0;
   size_t total = 0;
-  CUresult res = cuMemGetInfo(&free, &total);
+  CUresult res = tensorflow::wrap::cuMemGetInfo(&free, &total);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to query device memory info: " << ToString(res);
     return false;
@@ -1457,7 +1479,7 @@ static port::StatusOr<T> GetSimpleAttribute(CUdevice device,
 /* static */ bool CUDADriver::GetDeviceTotalMemory(CUdevice device,
                                                    uint64 *result) {
   size_t value = -1;
-  CUresult res = cuDeviceTotalMem(&value, device);
+  CUresult res = tensorflow::wrap::cuDeviceTotalMem(&value, device);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to query total available memory: " << ToString(res);
     return false;
@@ -1472,7 +1494,8 @@ static port::StatusOr<T> GetSimpleAttribute(CUdevice device,
   static const int kBufferSize = 64;
   absl::InlinedVector<char, 4> chars(kBufferSize);
   chars[kBufferSize - 1] = '\0';
-  CUresult res = cuDeviceGetPCIBusId(chars.begin(), kBufferSize - 1, device);
+  CUresult res = tensorflow::wrap::cuDeviceGetPCIBusId(chars.begin(),
+                                                       kBufferSize - 1, device);
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to query PCI bus id for device: " << ToString(res);
     return pci_bus_id;
@@ -1500,7 +1523,7 @@ static port::StatusOr<T> GetSimpleAttribute(CUdevice device,
                << to_device.status();
     return false;
   }
-  CUresult res = cuDeviceCanAccessPeer(
+  CUresult res = tensorflow::wrap::cuDeviceCanAccessPeer(
       &can_access_peer, from_device.ValueOrDie(), to_device.ValueOrDie());
   if (res != CUDA_SUCCESS) {
     LOG(ERROR) << "failed to detect peer access capability: " << ToString(res);
@@ -1517,7 +1540,8 @@ static port::StatusOr<T> GetSimpleAttribute(CUdevice device,
   }
 
   ScopedActivateContext activated{from};
-  CUresult result = cuCtxEnablePeerAccess(to->context(), 0 /* = flags */);
+  CUresult result =
+      tensorflow::wrap::cuCtxEnablePeerAccess(to->context(), 0 /* = flags */);
   if (result != CUDA_SUCCESS &&
       result != CUDA_ERROR_PEER_ACCESS_ALREADY_ENABLED) {
     return port::Status(
@@ -1535,8 +1559,9 @@ static port::StatusOr<T> GetSimpleAttribute(CUdevice device,
   ScopedActivateContext activation(context);
 
   int max_blocks;
-  CUresult result = cuOccupancyMaxActiveBlocksPerMultiprocessor(
-      &max_blocks, kernel, threads_per_block, dynamic_shared_memory_bytes);
+  CUresult result =
+      tensorflow::wrap::cuOccupancyMaxActiveBlocksPerMultiprocessor(
+          &max_blocks, kernel, threads_per_block, dynamic_shared_memory_bytes);
   if (result != CUDA_SUCCESS) {
     return port::Status(
         port::error::INTERNAL,
@@ -1549,7 +1574,7 @@ static port::StatusOr<T> GetSimpleAttribute(CUdevice device,
 
 /* static */ CUcontext CUDADriver::CurrentContextOrDie() {
   CUcontext current = nullptr;
-  CUresult result = cuCtxGetCurrent(&current);
+  CUresult result = tensorflow::wrap::cuCtxGetCurrent(&current);
   if (result != CUDA_SUCCESS) {
     LOG(FATAL) << "failed to query current context: " << ToString(result);
   }
diff --git a/tensorflow/stream_executor/cuda/cuda_driver_wrapper.h b/tensorflow/stream_executor/cuda/cuda_driver_wrapper.h
new file mode 100644
index 0000000000..ee99908bd5
--- /dev/null
+++ b/tensorflow/stream_executor/cuda/cuda_driver_wrapper.h
@@ -0,0 +1,143 @@
+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// This file wraps cuda driver calls with dso loader so that we don't need to
+// have explicit linking to libcuda. All TF cuda driver usage should route
+// through this wrapper.
+
+#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DRIVER_WRAPPER_H_
+#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DRIVER_WRAPPER_H_
+
+#include "cuda/include/cuda.h"
+#include "tensorflow/stream_executor/lib/env.h"
+#include "tensorflow/stream_executor/platform/dso_loader.h"
+#include "tensorflow/stream_executor/platform/port.h"
+
+namespace tensorflow {
+namespace wrap {
+#ifdef PLATFORM_GOOGLE
+// Use static linked library
+#define STREAM_EXECUTOR_LIBCUDA_WRAP(cudaSymbolName)                       \
+  template <typename... Args>                                              \
+  auto cudaSymbolName(Args... args)->decltype(::cudaSymbolName(args...)) { \
+    return ::cudaSymbolName(args...);                                      \
+  }
+
+// This macro wraps a global identifier, given by cudaSymbolName, in a callable
+// structure that loads the DLL symbol out of the DSO handle in a thread-safe
+// manner on first use. This dynamic loading technique is used to avoid DSO
+// dependencies on vendor libraries which may or may not be available in the
+// deployed binary environment.
+#else
+#define TO_STR_(x) #x
+#define TO_STR(x) TO_STR_(x)
+
+#define STREAM_EXECUTOR_LIBCUDA_WRAP(cudaSymbolName)                        \
+  template <typename... Args>                                               \
+  auto cudaSymbolName(Args... args)->decltype(::cudaSymbolName(args...)) {  \
+    using FuncPtrT = std::add_pointer<decltype(::cudaSymbolName)>::type;    \
+    static FuncPtrT loaded = []() -> FuncPtrT {                             \
+      static const char *kName = TO_STR(cudaSymbolName);                    \
+      void *f;                                                              \
+      auto s = stream_executor::port::Env::Default()->GetSymbolFromLibrary( \
+          stream_executor::internal::CachedDsoLoader::GetLibcudaDsoHandle() \
+              .ValueOrDie(),                                                \
+          kName, &f);                                                       \
+      CHECK(s.ok()) << "could not find " << kName                           \
+                    << " in libcuda DSO; dlerror: " << s.error_message();   \
+      return reinterpret_cast<FuncPtrT>(f);                                 \
+    }();                                                                    \
+    return loaded(args...);                                                 \
+  }
+#endif
+
+// clang-format off
+#define LIBCUDA_ROUTINE_EACH(__macro)                   \
+  __macro(cuCtxEnablePeerAccess)                        \
+  __macro(cuCtxGetCurrent)                              \
+  __macro(cuCtxGetDevice)                               \
+  __macro(cuCtxGetSharedMemConfig)                      \
+  __macro(cuCtxSetCurrent)                              \
+  __macro(cuCtxSetSharedMemConfig)                      \
+  __macro(cuCtxSynchronize)                             \
+  __macro(cuDeviceCanAccessPeer)                        \
+  __macro(cuDeviceGet)                                  \
+  __macro(cuDeviceGetAttribute)                         \
+  __macro(cuDeviceGetCount)                             \
+  __macro(cuDeviceGetName)                              \
+  __macro(cuDeviceGetPCIBusId)                          \
+  __macro(cuDevicePrimaryCtxGetState)                   \
+  __macro(cuDevicePrimaryCtxRelease)                    \
+  __macro(cuDevicePrimaryCtxRetain)                     \
+  __macro(cuDevicePrimaryCtxSetFlags)                   \
+  __macro(cuDeviceTotalMem)                             \
+  __macro(cuDriverGetVersion)                           \
+  __macro(cuEventCreate)                                \
+  __macro(cuEventDestroy)                               \
+  __macro(cuEventElapsedTime)                           \
+  __macro(cuEventQuery)                                 \
+  __macro(cuEventRecord)                                \
+  __macro(cuEventSynchronize)                           \
+  __macro(cuFuncGetAttribute)                           \
+  __macro(cuFuncSetCacheConfig)                         \
+  __macro(cuGetErrorName)                               \
+  __macro(cuGetErrorString)                             \
+  __macro(cuInit)                                       \
+  __macro(cuLaunchKernel)                               \
+  __macro(cuMemAlloc)                                   \
+  __macro(cuMemAllocManaged)                            \
+  __macro(cuMemFree)                                    \
+  __macro(cuMemFreeHost)                                \
+  __macro(cuMemGetAddressRange)                         \
+  __macro(cuMemGetInfo)                                 \
+  __macro(cuMemHostAlloc)                               \
+  __macro(cuMemHostRegister)                            \
+  __macro(cuMemHostUnregister)                          \
+  __macro(cuMemcpyDtoD)                                 \
+  __macro(cuMemcpyDtoDAsync)                            \
+  __macro(cuMemcpyDtoH)                                 \
+  __macro(cuMemcpyDtoHAsync)                            \
+  __macro(cuMemcpyHtoD)                                 \
+  __macro(cuMemcpyHtoDAsync)                            \
+  __macro(cuMemsetD32)                                  \
+  __macro(cuMemsetD32Async)                             \
+  __macro(cuMemsetD8)                                   \
+  __macro(cuMemsetD8Async)                              \
+  __macro(cuModuleGetFunction)                          \
+  __macro(cuModuleGetGlobal)                            \
+  __macro(cuModuleLoadDataEx)                           \
+  __macro(cuModuleLoadFatBinary)                        \
+  __macro(cuModuleUnload)                               \
+  __macro(cuOccupancyMaxActiveBlocksPerMultiprocessor)  \
+  __macro(cuOccupancyMaxPotentialBlockSize)             \
+  __macro(cuPointerGetAttribute)                        \
+  __macro(cuStreamAddCallback)                          \
+  __macro(cuStreamCreate)                               \
+  __macro(cuStreamDestroy)                              \
+  __macro(cuStreamQuery)                                \
+  __macro(cuStreamSynchronize)                          \
+  __macro(cuStreamWaitEvent)
+
+// clang-format on
+
+LIBCUDA_ROUTINE_EACH(STREAM_EXECUTOR_LIBCUDA_WRAP)
+#undef LIBCUDA_ROUTINE_EACH
+#undef STREAM_EXECUTOR_LIBCUDA_WRAP
+#undef TO_STR
+#undef TO_STR_
+}  // namespace wrap
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDA_DRIVER_WRAPPER_H_
diff --git a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc
index 70004495cd..51dec6b246 100644
--- a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc
+++ b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc
@@ -28,6 +28,7 @@ limitations under the License.
 #include "absl/strings/string_view.h"
 #include "tensorflow/stream_executor/cuda/cuda_diagnostics.h"
 #include "tensorflow/stream_executor/cuda/cuda_driver.h"
+#include "tensorflow/stream_executor/cuda/cuda_driver_wrapper.h"
 #include "tensorflow/stream_executor/cuda/cuda_event.h"
 #include "tensorflow/stream_executor/cuda/cuda_platform_id.h"
 #include "tensorflow/stream_executor/cuda/cuda_stream.h"
@@ -501,7 +502,7 @@ int CUDAExecutor::CalculateOccupancy(
     CUfunction func) {
   int suggested_blocks = 0;
   int suggested_threads = 0;
-  CUresult err = cuOccupancyMaxPotentialBlockSize(
+  CUresult err = tensorflow::wrap::cuOccupancyMaxPotentialBlockSize(
       &suggested_blocks, &suggested_threads, func, nullptr,
       shared_memory_per_block, 0);
   CHECK_EQ(err, CUDA_SUCCESS);
@@ -518,7 +519,7 @@ int CUDAExecutor::CompareOccupancy(int *initial_blocks,
                                    CUfunction func) {
   int suggested_blocks = 0;
   int suggested_threads = 0;
-  CUresult err = cuOccupancyMaxPotentialBlockSize(
+  CUresult err = tensorflow::wrap::cuOccupancyMaxPotentialBlockSize(
       &suggested_blocks, &suggested_threads, func, nullptr,
       shared_memory_per_block, 0);
   CHECK_EQ(err, CUDA_SUCCESS);
-- 
GitLab


From be1e0458309715cb8d0eea6baf1863a90a4f9ab5 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 16 Jan 2019 15:10:59 -0800
Subject: [PATCH 0843/1095] Commented out a description of the delayed-build
 pattern. :)

PiperOrigin-RevId: 229634588
---
 tensorflow/python/keras/engine/sequential.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/keras/engine/sequential.py b/tensorflow/python/keras/engine/sequential.py
index 425c8c3ef6..37eb2840b3 100644
--- a/tensorflow/python/keras/engine/sequential.py
+++ b/tensorflow/python/keras/engine/sequential.py
@@ -87,8 +87,8 @@ class Sequential(Model):
   model.add(Dense(32))
   model.weights  # returns list of length 4
 
-  When using the delayed-build pattern (no input shape specified), you can
-  choose to manually build your model by calling `build(batch_input_shape)`:
+  # When using the delayed-build pattern (no input shape specified), you can
+  # choose to manually build your model by calling `build(batch_input_shape)`:
   model = Sequential()
   model.add(Dense(32))
   model.add(Dense(32))
-- 
GitLab


From a799b815e5b497fd85e36ec36e44df0f3cebf802 Mon Sep 17 00:00:00 2001
From: James Qin <jamesqin@google.com>
Date: Wed, 16 Jan 2019 15:23:15 -0800
Subject: [PATCH 0844/1095] Define .value() on DistributedVariable

PiperOrigin-RevId: 229636699
---
 tensorflow/python/distribute/values.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/python/distribute/values.py b/tensorflow/python/distribute/values.py
index feaeea80cb..585ae1bd6c 100644
--- a/tensorflow/python/distribute/values.py
+++ b/tensorflow/python/distribute/values.py
@@ -562,6 +562,9 @@ class DistributedVariable(DistributedDelegate):
   def read_value(self):
     return self._distribute_strategy.extended.read_var(self)
 
+  def value(self):
+    return self._get_closest().value()
+
   def _should_act_as_resource_variable(self):
     """Pass resource_variable_ops.is_resource_variable check."""
     pass
-- 
GitLab


From 7fa13a9954ddd0c0479f226a8cc71215b7dc260f Mon Sep 17 00:00:00 2001
From: Shivani Agrawal <shivaniagrawal@google.com>
Date: Wed, 16 Jan 2019 15:31:20 -0800
Subject: [PATCH 0845/1095] Adds either a Note or TODO for tests without Eager
 coverage. Adds coverage for some more SparseTensors related tests and
 optimization test.

PiperOrigin-RevId: 229638239
---
 .../optimization/map_vectorization_test.py    | 12 ++--
 .../optimization/model_dataset_test.py        | 14 ++--
 .../optimization/optimize_dataset_test.py     | 71 +++++++++++--------
 .../parse_example_dataset_test.py             | 45 ++++++++----
 .../kernel_tests/restructured_dataset_test.py |  2 +-
 .../python/data/kernel_tests/dataset_test.py  |  3 +
 .../from_sparse_tensor_slices_test.py         |  4 +-
 .../kernel_tests/from_tensor_slices_test.py   |  2 +-
 .../data/kernel_tests/from_tensors_test.py    |  3 +-
 .../python/data/kernel_tests/optional_test.py |  6 +-
 .../data/kernel_tests/padded_batch_test.py    |  2 +
 .../python/data/kernel_tests/reduce_test.py   |  1 +
 12 files changed, 97 insertions(+), 68 deletions(-)

diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
index 405b038eb2..ec543aeb76 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
@@ -370,19 +370,17 @@ class MapVectorizationTest(test_base.DatasetTestBase, parameterized.TestCase):
                                                      num_parallel_calls)
     self.assertDatasetsEqual(unoptimized, optimized)
 
-  # TODO(b/117581999): Add eager coverage for the following tests.
-  def testSkipEagerOptimizationBadMapFn(self):
+  def testOptimizationBadMapFn(self):
     # Test map functions that give an error
     def map_fn(x):
       # x has leading dimension 5, this will raise an error
       return array_ops.gather(x, 10)
-
-    base_dataset = dataset_ops.Dataset.range(5).repeat(5).batch(
-        5, drop_remainder=True)
-    _, optimized = self._get_test_datasets(base_dataset, map_fn)
-    nxt = dataset_ops.make_one_shot_iterator(optimized).get_next()
     with self.assertRaisesRegexp(errors.InvalidArgumentError,
                                  r"indices = 10 is not in \[0, 5\)"):
+      base_dataset = dataset_ops.Dataset.range(5).repeat(5).batch(
+          5, drop_remainder=True)
+      _, optimized = self._get_test_datasets(base_dataset, map_fn)
+      nxt = dataset_ops.make_one_shot_iterator(optimized).get_next()
       self.evaluate(nxt)
 
   def testOptimizationWithCapturedInputs(self):
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/model_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/model_dataset_test.py
index c6b9db0d52..5c1ae7a98a 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/model_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/model_dataset_test.py
@@ -23,10 +23,11 @@ from tensorflow.python.data.experimental.ops import optimization
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import errors
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 
 
-# TODO(b/117581999): Add eager coverage for the following tests.
+@test_util.run_all_in_graph_and_eager_modes
 class ModelDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
 
   def testAutotuneOption(self):
@@ -37,14 +38,11 @@ class ModelDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     options.experimental_autotune = True
     options.experimental_optimization.apply_default_optimizations = False
     dataset = dataset.with_options(options)
+    get_next = self.getNext(dataset)
 
-    iterator = dataset_ops.make_one_shot_iterator(dataset)
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      self.assertEqual(0, self.evaluate(get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        self.evaluate(get_next)
+    self.assertEqual(0, self.evaluate(get_next()))
+    with self.assertRaises(errors.OutOfRangeError):
+      self.evaluate(get_next())
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
index e02f3b1572..f44ade049b 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/optimize_dataset_test.py
@@ -29,6 +29,7 @@ from tensorflow.python.data.experimental.ops import scan_ops
 from tensorflow.python.data.experimental.ops import threadpool
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
@@ -112,35 +113,47 @@ class OptimizeDatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
     get_next = self.getNext(dataset)
     self.evaluate(get_next())
 
-  # TODO(b/117581999): Add eager coverage for the following tests.
-  def testSkipEagerOptimizationLargeInputFromTensor(self):
-    input_t = array_ops.placeholder(dtypes.int32, (None, None, None))
-    dataset = dataset_ops.Dataset.from_tensors(input_t)
-    options = dataset_ops.Options()
-    options.experimental_optimization.apply_default_optimizations = False
-    dataset = dataset.with_options(options)
-    iterator = dataset_ops.make_initializable_iterator(dataset)
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      sess.run(init_op, {input_t: np.ones([512, 1024, 1025], np.int32)})
-      self.evaluate(get_next)
-
-  # TODO(b/117581999): Add eager coverage for the following tests.
-  def testSkipEagerOptimizationLargeInputFromTensorSlices(self):
-    input_t = array_ops.placeholder(dtypes.int32, (None, None, None, None))
-    dataset = dataset_ops.Dataset.from_tensor_slices(input_t)
-    options = dataset_ops.Options()
-    options.experimental_optimization.apply_default_optimizations = False
-    dataset = dataset.with_options(options)
-    iterator = dataset_ops.make_initializable_iterator(dataset)
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
-    with self.cached_session() as sess:
-      sess.run(init_op, {input_t: np.ones([1, 512, 1024, 1025], np.int32)})
-      self.evaluate(get_next)
+  def testOptimizationLargeInputFromTensor(self):
+    def dataset_fn(input_t):
+      dataset = dataset_ops.Dataset.from_tensors(input_t)
+      options = dataset_ops.Options()
+      options.experimental_optimization.apply_default_optimizations = False
+      return dataset.with_options(options)
+
+    if context.executing_eagerly():
+      input_t = np.ones([512, 1024, 1025], np.int32)
+      get_next = self.getNext(dataset_fn(input_t))
+      self.evaluate(get_next())
+    else:
+      input_t = array_ops.placeholder(dtypes.int32, (None, None, None))
+      iterator = dataset_ops.make_initializable_iterator(dataset_fn(input_t))
+      init_op = iterator.initializer
+      get_next = iterator.get_next()
+
+      with self.cached_session() as sess:
+        sess.run(init_op, {input_t: np.ones([512, 1024, 1025], np.int32)})
+        self.evaluate(get_next)
+
+  def testOptimizationLargeInputFromTensorSlices(self):
+    def dataset_fn(input_t):
+      dataset = dataset_ops.Dataset.from_tensor_slices(input_t)
+      options = dataset_ops.Options()
+      options.experimental_optimization.apply_default_optimizations = False
+      return dataset.with_options(options)
+
+    if context.executing_eagerly():
+      input_t = np.ones([1, 512, 1024, 1025], np.int32)
+      get_next = self.getNext(dataset_fn(input_t))
+      self.evaluate(get_next())
+    else:
+      input_t = array_ops.placeholder(dtypes.int32, (None, None, None, None))
+      iterator = dataset_ops.make_initializable_iterator(dataset_fn(input_t))
+      init_op = iterator.initializer
+      get_next = iterator.get_next()
+
+      with self.cached_session() as sess:
+        sess.run(init_op, {input_t: np.ones([1, 512, 1024, 1025], np.int32)})
+        self.evaluate(get_next)
 
   def testOptimizationNestedDataset(self):
 
diff --git a/tensorflow/python/data/experimental/kernel_tests/parse_example_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/parse_example_dataset_test.py
index 76e0d4d72a..4dbb188f2c 100644
--- a/tensorflow/python/data/experimental/kernel_tests/parse_example_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/parse_example_dataset_test.py
@@ -27,6 +27,7 @@ from tensorflow.core.example import feature_pb2
 from tensorflow.python.data.experimental.ops import parsing_ops as contrib_parsing_ops
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import ops
@@ -671,8 +672,7 @@ class ParseExampleDatasetTest(test_base.DatasetTestBase):
     for batch_size in (1, 10, 20, 100, 256):
       self._testSerializedContainingVarLenDenseLargerBatch(batch_size)
 
-  @test_util.run_deprecated_v1
-  def testSkipEagerSerializedShapeMismatch(self):
+  def testSerializedShapeMismatch(self):
     aname = "a"
     bname = "b"
     cname = "c"
@@ -695,19 +695,34 @@ class ParseExampleDatasetTest(test_base.DatasetTestBase):
     ]
 
     serialized = [m.SerializeToString() for m in original]
-    self._test(
-        ops.convert_to_tensor(serialized), {
-            aname:
-                parsing_ops.FixedLenSequenceFeature((2, 1),
-                                                    dtype=dtypes.float32,
-                                                    allow_missing=True,
-                                                    default_value=[]),
-            bname:
-                parsing_ops.FixedLenSequenceFeature(
-                    (2, 1, 1), dtype=dtypes.string, allow_missing=True),
-        },
-        expected_err=(ValueError,
-                      "Cannot reshape a tensor with 0 elements to shape"))
+    if context.executing_eagerly():
+      self._test(
+          ops.convert_to_tensor(serialized), {
+              aname:
+                  parsing_ops.FixedLenSequenceFeature((2, 1),
+                                                      dtype=dtypes.float32,
+                                                      allow_missing=True,
+                                                      default_value=[]),
+              bname:
+                  parsing_ops.FixedLenSequenceFeature(
+                      (2, 1, 1), dtype=dtypes.string, allow_missing=True),
+          },
+          expected_err=(errors_impl.InvalidArgumentError,
+                        "Input to reshape is a tensor with 0 values"))
+    else:
+      self._test(
+          ops.convert_to_tensor(serialized), {
+              aname:
+                  parsing_ops.FixedLenSequenceFeature((2, 1),
+                                                      dtype=dtypes.float32,
+                                                      allow_missing=True,
+                                                      default_value=[]),
+              bname:
+                  parsing_ops.FixedLenSequenceFeature(
+                      (2, 1, 1), dtype=dtypes.string, allow_missing=True),
+          },
+          expected_err=(ValueError,
+                        "Cannot reshape a tensor with 0 elements to shape"))
 
   @test_util.run_deprecated_v1
   def testSerializedContainingVarLenDense(self):
diff --git a/tensorflow/python/data/experimental/kernel_tests/restructured_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/restructured_dataset_test.py
index 87a91415b0..ddac02b9e2 100644
--- a/tensorflow/python/data/experimental/kernel_tests/restructured_dataset_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/restructured_dataset_test.py
@@ -27,7 +27,7 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
 
 
-# TODO(b/117581999): Add eager coverage
+# TODO(b/117581999): Add eager specific test.
 class RestructuredDatasetTest(test_base.DatasetTestBase):
 
   @test_util.run_deprecated_v1
diff --git a/tensorflow/python/data/kernel_tests/dataset_test.py b/tensorflow/python/data/kernel_tests/dataset_test.py
index db8a999491..f319b24bee 100644
--- a/tensorflow/python/data/kernel_tests/dataset_test.py
+++ b/tensorflow/python/data/kernel_tests/dataset_test.py
@@ -266,6 +266,7 @@ class DatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
           round_trip_dataset, [self.evaluate(tf_value_fn())],
           requires_initialization=True)
 
+  # NOTE: This test is specific to graph mode and is skipped in eager mode.
   @test_util.run_deprecated_v1
   def testSkipEagerSameGraphErrorOneShot(self):
     dataset = dataset_ops.Dataset.range(10)
@@ -273,6 +274,7 @@ class DatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
       with self.assertRaisesRegexp(ValueError, "must be from the same graph"):
         dataset = dataset.batch(2)
 
+  # NOTE: This test is specific to graph mode and is skipped in eager mode.
   @test_util.run_deprecated_v1
   def testSkipEagerSameGraphErrorOneShotSimple(self):
     dataset = dataset_ops.Dataset.range(10)
@@ -283,6 +285,7 @@ class DatasetTest(test_base.DatasetTestBase, parameterized.TestCase):
             str(mock_log.call_args), "Please ensure that all datasets in the "
             "pipeline are created in the same graph as the iterator.")
 
+  # NOTE: This test is specific to graph mode and is skipped in eager mode.
   @test_util.run_deprecated_v1
   def testSkipEagerSameGraphErrorInitializable(self):
     dataset = dataset_ops.Dataset.range(10)
diff --git a/tensorflow/python/data/kernel_tests/from_sparse_tensor_slices_test.py b/tensorflow/python/data/kernel_tests/from_sparse_tensor_slices_test.py
index ef608ebb67..546c2fb2ed 100644
--- a/tensorflow/python/data/kernel_tests/from_sparse_tensor_slices_test.py
+++ b/tensorflow/python/data/kernel_tests/from_sparse_tensor_slices_test.py
@@ -29,11 +29,11 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
 
 
-@test_util.run_all_in_graph_and_eager_modes
+# NOTE: deprecated method in V2, no eager coverage added.
 class FromSparseTensorSlicesTest(test_base.DatasetTestBase):
 
   @test_util.run_deprecated_v1
-  def testSkipEagerFromSparseTensorSlices(self):
+  def testFromSparseTensorSlices(self):
     """Test a dataset based on slices of a `tf.SparseTensor`."""
     st = array_ops.sparse_placeholder(dtypes.float64)
     iterator = dataset_ops.make_initializable_iterator(
diff --git a/tensorflow/python/data/kernel_tests/from_tensor_slices_test.py b/tensorflow/python/data/kernel_tests/from_tensor_slices_test.py
index 9a480e5678..72db638771 100644
--- a/tensorflow/python/data/kernel_tests/from_tensor_slices_test.py
+++ b/tensorflow/python/data/kernel_tests/from_tensor_slices_test.py
@@ -53,7 +53,7 @@ class FromTensorSlicesTest(test_base.DatasetTestBase):
     with self.assertRaises(errors.OutOfRangeError):
       results = self.evaluate(get_next())
 
-  def testSkipEagerFromTensorSlicesSparse(self):
+  def testFromTensorSlicesSparse(self):
     """Test a dataset that represents the slices from a tuple of tensors."""
     components = (sparse_tensor.SparseTensorValue(
         indices=np.array([[0, 0], [1, 0], [2, 0]]),
diff --git a/tensorflow/python/data/kernel_tests/from_tensors_test.py b/tensorflow/python/data/kernel_tests/from_tensors_test.py
index ab3c15263f..82ccdebc7f 100644
--- a/tensorflow/python/data/kernel_tests/from_tensors_test.py
+++ b/tensorflow/python/data/kernel_tests/from_tensors_test.py
@@ -50,7 +50,7 @@ class FromTensorsTest(test_base.DatasetTestBase):
 
     self.assertDatasetProduces(dataset, expected_output=[components])
 
-  def testSkipEagerFromTensorsSparse(self):
+  def testFromTensorsSparse(self):
     """Test a dataset that represents a single tuple of tensors."""
     components = (sparse_tensor.SparseTensorValue(
         indices=np.array([[0]]),
@@ -224,6 +224,7 @@ class FromTensorsTest(test_base.DatasetTestBase):
     self.assertEquals(dtypes.int64, get_next().dtype)
     self.assertEquals([3], get_next().shape)
 
+  # TODO(b/121264236): needs mechanism for multiple device in eager mode.
   def testSkipEagerSplitPipelineFailsWithPlacementError(self):
     with session.Session(
         target="",
diff --git a/tensorflow/python/data/kernel_tests/optional_test.py b/tensorflow/python/data/kernel_tests/optional_test.py
index ba5ee9b661..2269bb8724 100644
--- a/tensorflow/python/data/kernel_tests/optional_test.py
+++ b/tensorflow/python/data/kernel_tests/optional_test.py
@@ -75,7 +75,6 @@ class OptionalTest(test_base.DatasetTestBase, parameterized.TestCase):
       self.assertAllEqual(expected.dense_shape,
                           self.evaluate(actual.dense_shape))
 
-  @test_util.run_deprecated_v1
   def testFromNone(self):
     value_structure = structure.TensorStructure(dtypes.float32, [])
     opt = optional_ops.Optional.none_from_structure(value_structure)
@@ -269,9 +268,7 @@ class OptionalTest(test_base.DatasetTestBase, parameterized.TestCase):
        optional_ops.OptionalStructure(
            structure.TensorStructure(dtypes.float32, []))),
   )
-  @test_util.run_deprecated_v1
-  def testSkipEagerOptionalStructure(self, tf_value_fn,
-                                     expected_value_structure):
+  def testOptionalStructure(self, tf_value_fn, expected_value_structure):
     tf_value = tf_value_fn()
     opt = optional_ops.Optional.from_value(tf_value)
 
@@ -306,6 +303,7 @@ class OptionalTest(test_base.DatasetTestBase, parameterized.TestCase):
       self.assertEqual(
           self.evaluate(tf_value), self.evaluate(round_trip_opt.get_value()))
 
+  # NOTE: This test is specific to graph mode and is skipped in eager mode.
   @parameterized.named_parameters(
       ("Tensor", np.array([1, 2, 3], dtype=np.int32),
        lambda: constant_op.constant([4, 5, 6], dtype=dtypes.int32), True),
diff --git a/tensorflow/python/data/kernel_tests/padded_batch_test.py b/tensorflow/python/data/kernel_tests/padded_batch_test.py
index dcfb2f507b..042af7a6f9 100644
--- a/tensorflow/python/data/kernel_tests/padded_batch_test.py
+++ b/tensorflow/python/data/kernel_tests/padded_batch_test.py
@@ -156,6 +156,7 @@ class PaddedBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
     next_element = self.getNext(padded_dataset)
     self.evaluate(next_element())
 
+  # NOTE: This test is specific to graph mode and is skipped in eager mode.
   @test_util.run_deprecated_v1
   def testSkipEagerPaddedBatchDatasetShapeSpecifications(self):
     int_placeholder = array_ops.placeholder(dtypes.int32)
@@ -228,6 +229,7 @@ class PaddedBatchTest(test_base.DatasetTestBase, parameterized.TestCase):
       _ = dataset_ops.Dataset.range(10).padded_batch(
           5, padded_shapes=shape_as_tensor)
 
+  # NOTE: This test is specific to graph mode and is skipped in eager mode.
   @test_util.run_deprecated_v1
   def testSkipEagerPaddedBatchShapeError(self):
     with self.assertRaisesRegexp(
diff --git a/tensorflow/python/data/kernel_tests/reduce_test.py b/tensorflow/python/data/kernel_tests/reduce_test.py
index 14bbc0bf72..93acc1565f 100644
--- a/tensorflow/python/data/kernel_tests/reduce_test.py
+++ b/tensorflow/python/data/kernel_tests/reduce_test.py
@@ -68,6 +68,7 @@ class ReduceTest(test_base.DatasetTestBase, parameterized.TestCase):
       self.assertEqual(((i + 1) * i) // 2, s)
       self.assertEqual(i, c)
 
+  # NOTE: This test is specific to graph mode and is skipped in eager mode.
   @test_util.run_deprecated_v1
   def testSkipEagerSquareUsingPlaceholder(self):
     delta = array_ops.placeholder(dtype=dtypes.int64)
-- 
GitLab


From a88c90db14f92509e0da39072dc9bfd772a1c1aa Mon Sep 17 00:00:00 2001
From: Ben Barsdell <bbarsdell@nvidia.com>
Date: Fri, 14 Dec 2018 10:14:54 -0800
Subject: [PATCH 0846/1095] [XLA:GPU] Make sm_35 the min supported LLVM proc

- XLA nvptx backend does not support sm_30, and TensorFlow also only
  supports sm_35 onward.
---
 .../xla/service/gpu/llvm_gpu_backend/nvptx_backend_lib.cc   | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/nvptx_backend_lib.cc b/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/nvptx_backend_lib.cc
index eddaa877f2..153aab97d9 100644
--- a/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/nvptx_backend_lib.cc
+++ b/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/nvptx_backend_lib.cc
@@ -110,11 +110,9 @@ static string GetLibdeviceFilename(const string& libdevice_dir_path,
 }
 
 // Gets the GPU name as it's known to LLVM for a given compute capability.  If
-// we see an unrecognized compute capability, we return "sm_30".
+// we see an unrecognized compute capability, we return "sm_35".
 static string GetSmName(std::pair<int, int> compute_capability) {
   static auto* m = new std::map<std::pair<int, int>, int>({
-      {{3, 0}, 30},
-      {{3, 2}, 32},
       {{3, 5}, 35},
       {{3, 7}, 37},
       {{5, 0}, 50},
@@ -127,7 +125,7 @@ static string GetSmName(std::pair<int, int> compute_capability) {
       {{7, 2}, 72},
       {{7, 5}, 75},
   });
-  int sm_version = 30;
+  int sm_version = 35;
   auto it = m->find(compute_capability);
   if (it != m->end()) {
     sm_version = it->second;
-- 
GitLab


From ea84f664dd7d4405240da8797bfd21b7bd860aa1 Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Wed, 16 Jan 2019 15:37:43 -0800
Subject: [PATCH 0847/1095] [tf.data] Fixing a bug in MultiDeviceIterator which
 would cause a deadlock if the iterator was re-initialized without trying to
 fetch elements from it.

PiperOrigin-RevId: 229639479
---
 .../core/kernels/data/multi_device_iterator_ops.cc    |  2 +-
 tensorflow/python/data/kernel_tests/BUILD             |  1 +
 .../data/kernel_tests/multi_device_iterator_test.py   | 11 ++++++++---
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/kernels/data/multi_device_iterator_ops.cc b/tensorflow/core/kernels/data/multi_device_iterator_ops.cc
index eb08d2b28a..167276032b 100644
--- a/tensorflow/core/kernels/data/multi_device_iterator_ops.cc
+++ b/tensorflow/core/kernels/data/multi_device_iterator_ops.cc
@@ -151,7 +151,7 @@ class MultiDeviceIterator : public ResourceBase {
     void Reset() LOCKS_EXCLUDED(mu_) {
       {
         mutex_lock l(mu_);
-        if (!background_thread_finished_) {
+        if (background_thread_ && !background_thread_finished_) {
           cancelled_ = true;
           // Wake up the background thread.
           for (int i = 0; i < size_; ++i) {
diff --git a/tensorflow/python/data/kernel_tests/BUILD b/tensorflow/python/data/kernel_tests/BUILD
index 737ba28ceb..8206ce382c 100644
--- a/tensorflow/python/data/kernel_tests/BUILD
+++ b/tensorflow/python/data/kernel_tests/BUILD
@@ -405,6 +405,7 @@ cuda_py_test(
     srcs = ["multi_device_iterator_test.py"],
     additional_deps = [
         ":test_base",
+        "@absl_py//absl/testing:parameterized",
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python/data/ops:dataset_ops",
         "//tensorflow/python/data/ops:multi_device_iterator_ops",
diff --git a/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py b/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py
index 2ae6a9893d..0c88d7533f 100644
--- a/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py
+++ b/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py
@@ -18,6 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from absl.testing import parameterized
+
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.data.experimental.ops import optimization
 from tensorflow.python.data.kernel_tests import test_base
@@ -36,17 +38,20 @@ from tensorflow.python.platform import test
 # / V2 mode, we should remove this annotation and the run_v1_only annotations
 # as well.
 @test_util.run_all_in_graph_and_eager_modes
-class MultiDeviceIteratorTest(test_base.DatasetTestBase):
+class MultiDeviceIteratorTest(test_base.DatasetTestBase,
+                              parameterized.TestCase):
 
   @test_util.run_v1_only
-  def testNoGetNext(self):
+  @parameterized.parameters(0, 1, 42,)
+  def testInitOnly(self, num_inits):
     dataset = dataset_ops.Dataset.range(10)
     multi_device_iterator = multi_device_iterator_ops.MultiDeviceIterator(
         dataset, ["/cpu:1", "/cpu:2"])
 
     config = config_pb2.ConfigProto(device_count={"CPU": 3})
     with self.test_session(config=config):
-      self.evaluate(multi_device_iterator.initializer)
+      for _ in range(num_inits):
+        self.evaluate(multi_device_iterator.initializer)
 
   @test_util.run_v1_only
   def testBasic(self):
-- 
GitLab


From b1075f10d9facfdca80d28b881a434419e34ae2d Mon Sep 17 00:00:00 2001
From: Shashi Shekhar <shashishekhar@google.com>
Date: Wed, 16 Jan 2019 15:59:42 -0800
Subject: [PATCH 0848/1095] Populate zero point for symmetric quantization as
 well.

PiperOrigin-RevId: 229643065
---
 tensorflow/lite/tools/optimize/subgraph_quantizer.cc  | 11 +++++++++--
 .../lite/tools/optimize/subgraph_quantizer_test.cc    | 10 ++++++++++
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/tools/optimize/subgraph_quantizer.cc b/tensorflow/lite/tools/optimize/subgraph_quantizer.cc
index 30320304ec..bd6e6e81b2 100644
--- a/tensorflow/lite/tools/optimize/subgraph_quantizer.cc
+++ b/tensorflow/lite/tools/optimize/subgraph_quantizer.cc
@@ -37,12 +37,17 @@ const int8_t kMinQuantizedValue = -127;
 const int8_t kMaxQuantizedValue = 127;
 
 TfLiteStatus AddQuantizationParams(const std::vector<float>& scales,
+                                   const std::vector<int64_t>& zero_point,
                                    int quantized_dimension,
                                    const uint8_t* buffer_data,
                                    size_t buffer_size, TensorType output_type,
                                    ModelT* model, TensorT* tensor) {
   tensor->quantization = absl::make_unique<QuantizationParametersT>();
   tensor->quantization->scale.assign(scales.begin(), scales.end());
+  if (zero_point.size() != scales.size()) {
+    return kTfLiteError;
+  }
+  tensor->quantization->zero_point.assign(zero_point.begin(), zero_point.end());
   tensor->quantization->quantized_dimension = quantized_dimension;
   model->buffers[tensor->buffer]->data.assign(buffer_data,
                                               buffer_data + buffer_size);
@@ -168,8 +173,10 @@ TfLiteStatus SymmetricPerChannelQuantizeTensor(ModelT* model, TensorT* tensor,
   // Set the buffers and output type.
   uint8_t* uint8_buffer = reinterpret_cast<uint8_t*>(final_buffer.data());
   size_t buffer_size = num_elements * sizeof(int8_t);
-  return AddQuantizationParams(scales, channel_dim_index, uint8_buffer,
-                               buffer_size, TensorType_INT8, model, tensor);
+  std::vector<int64_t> zero_point(scales.size(), 0);
+  return AddQuantizationParams(scales, zero_point, channel_dim_index,
+                               uint8_buffer, buffer_size, TensorType_INT8,
+                               model, tensor);
 }
 
 // Symmetrically quantizes the bias for ops like Conv and DepthwiseConv.
diff --git a/tensorflow/lite/tools/optimize/subgraph_quantizer_test.cc b/tensorflow/lite/tools/optimize/subgraph_quantizer_test.cc
index 085b66e90f..1ea49da899 100644
--- a/tensorflow/lite/tools/optimize/subgraph_quantizer_test.cc
+++ b/tensorflow/lite/tools/optimize/subgraph_quantizer_test.cc
@@ -97,13 +97,18 @@ TEST(SubgraphQuantizerTest, VerifyConvQuantizationWithUnitScale) {
   const std::vector<float>& weights_scales =
       weights_tensor->quantization->scale;
 
+  const std::vector<int64_t>& weights_zero_points =
+      weights_tensor->quantization->zero_point;
+
   ASSERT_EQ(weights_scales.size(), out_channel_size);
+  ASSERT_EQ(weights_zero_points.size(), out_channel_size);
   ASSERT_EQ(input_tensor->quantization->scale.size(), 1);
   ASSERT_EQ(output_tensor->quantization->scale.size(), 1);
 
 
   for (size_t i = 0; i < out_channel_size; i++) {
     EXPECT_EQ(weights_scales[i], 1);
+    EXPECT_EQ(weights_zero_points[i], 0);
   }
 
   EXPECT_EQ(input_tensor->quantization->scale[0], 1);
@@ -190,8 +195,11 @@ TEST(SubgraphQuantizerTest, VerifyConvQuantization) {
   ASSERT_TRUE(weights_tensor->quantization);
   const std::vector<float>& weights_scales =
       weights_tensor->quantization->scale;
+  const std::vector<int64_t>& weights_zero_points =
+      weights_tensor->quantization->zero_point;
 
   ASSERT_EQ(weights_scales.size(), out_channel_size);
+  ASSERT_EQ(weights_zero_points.size(), out_channel_size);
   ASSERT_EQ(input_tensor->quantization->scale.size(), 1);
   ASSERT_EQ(output_tensor->quantization->scale.size(), 1);
 
@@ -225,9 +233,11 @@ TEST(SubgraphQuantizerTest, VerifyConvQuantization) {
     for (size_t j = 0; j < num_values_in_channel; j++) {
       size_t element_idx = channel_idx * out_channel_size + j;
       auto scale = weights_scales[channel_idx];
+      auto zero_point = weights_zero_points[channel_idx];
       auto dequantized_value = weight_values[element_idx] * scale;
       EXPECT_NEAR(dequantized_value, weights_float_buffer[element_idx],
                   scale / 2);
+      EXPECT_EQ(zero_point, 0);
     }
   }
 }
-- 
GitLab


From 1961871d3fc8941b60212ea9f577001e7827d4a1 Mon Sep 17 00:00:00 2001
From: Shanqing Cai <cais@google.com>
Date: Wed, 16 Jan 2019 16:05:18 -0800
Subject: [PATCH 0849/1095] Disable testing of user-defined GPU kernel under
 eager mode

PiperOrigin-RevId: 229644310
---
 tensorflow/tools/ci_build/builds/test_user_ops.sh | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/tensorflow/tools/ci_build/builds/test_user_ops.sh b/tensorflow/tools/ci_build/builds/test_user_ops.sh
index 25ecee4725..9da9c3b881 100755
--- a/tensorflow/tools/ci_build/builds/test_user_ops.sh
+++ b/tensorflow/tools/ci_build/builds/test_user_ops.sh
@@ -239,8 +239,15 @@ function run_op() {
   fi
 }
 
+printf "\nTesting execution of user-defined op under graph mode:\n\n"
 run_op "$("${PYTHON_BIN_PATH}" -c "import tensorflow as tf; print(tf.Session('').run(tf.load_op_library('./${USER_OP_SO}').${USER_OP}(${OP_INPUT})))")"
-run_op "$("${PYTHON_BIN_PATH}" -c "import tensorflow as tf; tf.enable_eager_execution(); print(tf.load_op_library('./${USER_OP_SO}').${USER_OP}(${OP_INPUT}).numpy())")" " in eager mode"
+
+if [[ "${IS_GPU}" == "0" ]]; then
+  printf "\nTesting execution of user-defined op under eager mode:\n\n"
+  run_op "$("${PYTHON_BIN_PATH}" -c "import tensorflow as tf; tf.enable_eager_execution(); print(tf.load_op_library('./${USER_OP_SO}').${USER_OP}(${OP_INPUT}).numpy())")" " in eager mode"
+else
+  printf "\nSKIPPING the testing of execution of user-defined GPU kernel under eager mode. See b/122972785.\n\n"
+fi
 
 
 popd
-- 
GitLab


From 1d2e3ab3f291bd887a8dc292bb3fd31f9106204e Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Wed, 16 Jan 2019 16:12:51 -0800
Subject: [PATCH 0850/1095] Always call PyObject_ClearWeakRefs

This seems to work in Python 2 and be necessary for correctness in Python 3. weakreflist is not set and yet there are weakrefs which need to be cleared.

Oddly it's only the weak key dictionary that fails in Python 3 before removing the conditional; using weakref.ref on its own seems fine.

PiperOrigin-RevId: 229645639
---
 tensorflow/python/eager/ops_test.py      | 28 ++++++++++++++++++++++++
 tensorflow/python/eager/pywrap_tensor.cc |  4 +---
 2 files changed, 29 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/eager/ops_test.py b/tensorflow/python/eager/ops_test.py
index 17a090d526..ab4bdaa601 100644
--- a/tensorflow/python/eager/ops_test.py
+++ b/tensorflow/python/eager/ops_test.py
@@ -18,6 +18,8 @@ from __future__ import division
 from __future__ import print_function
 
 import threading
+import weakref
+
 import numpy as np
 
 from tensorflow.core.protobuf import config_pb2
@@ -397,6 +399,32 @@ class OpsTest(test_util.TensorFlowTestCase):
     t1.start()
     t1.join()
 
+  def testWeakrefEagerTensor(self):
+    x = constant_op.constant([[1.]])
+    x.at1 = constant_op.constant([[2.]])
+    x.at2 = 3.
+    weak_x = weakref.ref(x)
+    weak_xat1 = weakref.ref(x.at1)
+    del x
+    self.assertIs(weak_x(), None)
+    self.assertIs(weak_xat1(), None)
+
+  def testWeakKeyDictionaryTensor(self):
+    weak_key_dict = weakref.WeakKeyDictionary()
+    strong_x = constant_op.constant([[1.]])
+    strong_y = constant_op.constant([[2.]])
+    weak_key_dict[strong_x] = constant_op.constant([[3.]])
+    weak_key_dict[strong_y] = constant_op.constant([[4.]])
+    strong_y.a = constant_op.constant([[5.]])
+    weak_x = weakref.ref(strong_x)
+    del strong_x
+    self.assertIs(weak_x(), None)
+    self.assertEqual([strong_y], list(weak_key_dict))
+    self.assertEqual(1, len(list(weak_key_dict)))
+    self.assertEqual(1, len(weak_key_dict))
+    del strong_y
+    self.assertEqual([], list(weak_key_dict))
+
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/eager/pywrap_tensor.cc b/tensorflow/python/eager/pywrap_tensor.cc
index 30a93fb0e4..35040c5d56 100644
--- a/tensorflow/python/eager/pywrap_tensor.cc
+++ b/tensorflow/python/eager/pywrap_tensor.cc
@@ -501,9 +501,7 @@ int EagerTensor_init(EagerTensor* self, PyObject* args, PyObject* kwds) {
 void EagerTensor_dealloc(EagerTensor* self) {
   // Clear weak references to self.
   // Needs to happen before any actual destruction.
-  if (self->weakreflist != nullptr) {
-    PyObject_ClearWeakRefs((PyObject*)self);
-  }
+  PyObject_ClearWeakRefs((PyObject*)self);
 
   TF_DeleteStatus(self->status);
   Py_DECREF(self->handle_data);
-- 
GitLab


From 0eeb360d8f04fd93b799a07bedbea6e6e899f55e Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Wed, 16 Jan 2019 16:12:52 -0800
Subject: [PATCH 0851/1095] Remove dead code block from normalization layer

PiperOrigin-RevId: 229645644
---
 tensorflow/python/keras/layers/normalization.py | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/tensorflow/python/keras/layers/normalization.py b/tensorflow/python/keras/layers/normalization.py
index 9167a747cd..171e1e231e 100644
--- a/tensorflow/python/keras/layers/normalization.py
+++ b/tensorflow/python/keras/layers/normalization.py
@@ -413,15 +413,6 @@ class BatchNormalizationV2(Layer):
   def _assign_moving_average(self, variable, value, momentum):
     with ops.name_scope(None, 'AssignMovingAvg',
                         [variable, value, momentum]) as scope:
-      # TODO(b/120571621): We want to avoid colocating the variables here
-      # since TPUStrategy does not implement replica local variables.
-      # Remove this hack once we support TPULocalVariables.
-      is_tpu_strategy = False
-      if distribution_strategy_context.has_strategy():
-        distribute = distribution_strategy_context.get_strategy()
-        if distribute.__class__.__name__ == 'TPUStrategy':
-          is_tpu_strategy = True
-
       with ops.colocate_with(variable):
         decay = ops.convert_to_tensor(1.0 - momentum, name='decay')
         if decay.dtype != variable.dtype.base_dtype:
-- 
GitLab


From 889615c72ef16e1d3151d7ae52939258fdda796c Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Wed, 16 Jan 2019 16:45:06 -0800
Subject: [PATCH 0852/1095] Updates binary cross entropy logic in Keras when
 input is probabilities. Moves label smoothing logic into the cross entropy
 loss functions.

PiperOrigin-RevId: 229651194
---
 tensorflow/python/keras/backend.py            |  12 +-
 tensorflow/python/keras/losses.py             |  67 ++++++---
 tensorflow/python/keras/losses_test.py        | 127 +++++++++++++-----
 .../golden/v1/tensorflow.keras.losses.pbtxt   |   4 +-
 .../golden/v1/tensorflow.keras.metrics.pbtxt  |   4 +-
 .../golden/v2/tensorflow.keras.losses.pbtxt   |   4 +-
 .../golden/v2/tensorflow.keras.metrics.pbtxt  |   4 +-
 7 files changed, 156 insertions(+), 66 deletions(-)

diff --git a/tensorflow/python/keras/backend.py b/tensorflow/python/keras/backend.py
index 2e53ee4ec9..f357f0efc7 100644
--- a/tensorflow/python/keras/backend.py
+++ b/tensorflow/python/keras/backend.py
@@ -3866,7 +3866,8 @@ def categorical_crossentropy(target, output, from_logits=False, axis=-1):
       axis = axis % len(output.shape)
       # scale preds so that the class probas of each sample sum to 1
       output = output / math_ops.reduce_sum(output, axis, True)
-      # manual computation of crossentropy
+
+      # Compute cross entropy from probabilities.
       epsilon_ = _to_tensor(epsilon(), output.dtype.base_dtype)
       output = clip_ops.clip_by_value(output, epsilon_, 1. - epsilon_)
       return -math_ops.reduce_sum(target * math_ops.log(output), axis)
@@ -3948,10 +3949,13 @@ def binary_crossentropy(target, output, from_logits=False):
   """
   if not from_logits:
     if context.executing_eagerly() or output.op.type != 'Sigmoid':
-      # transform back to logits
       epsilon_ = _to_tensor(epsilon(), output.dtype.base_dtype)
-      output = clip_ops.clip_by_value(output, epsilon_, 1 - epsilon_)
-      output = math_ops.log(output / (1 - output))
+      output = clip_ops.clip_by_value(output, epsilon_, 1. - epsilon_)
+
+      # Compute cross entropy from probabilities.
+      bce = target * math_ops.log(output + epsilon())
+      bce += (1 - target) * math_ops.log(1 - output + epsilon())
+      return -bce
     else:
       # When sigmoid activation function is used for output operation, we
       # use logits from the sigmoid function directly to compute loss in order
diff --git a/tensorflow/python/keras/losses.py b/tensorflow/python/keras/losses.py
index 66780de0f0..9144eb0bf9 100644
--- a/tensorflow/python/keras/losses.py
+++ b/tensorflow/python/keras/losses.py
@@ -24,6 +24,7 @@ import abc
 import six
 
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import smart_cond
 from tensorflow.python.keras import backend as K
 from tensorflow.python.keras.utils.generic_utils import deserialize_keras_object
 from tensorflow.python.keras.utils.generic_utils import serialize_keras_object
@@ -291,7 +292,7 @@ class BinaryCrossentropy(Loss):
   Args:
     from_logits: Whether `output` is expected to be a logits tensor. By default,
       we consider that `output` encodes a probability distribution.
-    label_smoothing: If greater than `0` then smooth the labels.
+    label_smoothing: Float in [0, 1]. If > `0` then smooth the labels.
     reduction: Type of `tf.losses.Reduction` to apply to loss. Default value is
       `SUM_OVER_BATCH_SIZE`.
     name: Optional name for the op.
@@ -304,7 +305,8 @@ class BinaryCrossentropy(Loss):
                name=None):
     super(BinaryCrossentropy, self).__init__(reduction=reduction, name=name)
     self.from_logits = from_logits
-    self.label_smoothing = label_smoothing
+    self.label_smoothing = ops.convert_to_tensor(
+        label_smoothing, dtype=K.floatx())
 
   def call(self, y_true, y_pred):
     """Invokes the `BinaryCrossentropy` instance.
@@ -318,11 +320,11 @@ class BinaryCrossentropy(Loss):
     """
     y_pred = ops.convert_to_tensor(y_pred)
     y_true = math_ops.cast(y_true, y_pred.dtype)
-
-    if self.label_smoothing > 0:
-      y_true = y_true * (1 - self.label_smoothing) + 0.5 * self.label_smoothing
-
-    return binary_crossentropy(y_true, y_pred, from_logits=self.from_logits)
+    return binary_crossentropy(
+        y_true,
+        y_pred,
+        from_logits=self.from_logits,
+        label_smoothing=self.label_smoothing)
 
 
 @keras_export('keras.losses.CategoricalCrossentropy')
@@ -349,8 +351,9 @@ class CategoricalCrossentropy(Loss):
   Args:
     from_logits: Whether `output` is expected to be a logits tensor. By default,
       we consider that `output` encodes a probability distribution.
-    label_smoothing: If greater than `0` then smooth the labels. This option is
-      currently not supported when `y_pred` is a sparse input (not one-hot).
+    label_smoothing: Float in [0, 1]. If > `0` then smooth the labels. This
+      option is currently not supported when `y_pred` is a sparse input
+      (not one-hot).
     reduction: Type of `tf.losses.Reduction` to apply to loss. Default value is
       `SUM_OVER_BATCH_SIZE`.
     name: Optional name for the op.
@@ -364,7 +367,8 @@ class CategoricalCrossentropy(Loss):
     super(CategoricalCrossentropy, self).__init__(
         reduction=reduction, name=name)
     self.from_logits = from_logits
-    self.label_smoothing = label_smoothing
+    self.label_smoothing = ops.convert_to_tensor(
+        label_smoothing, dtype=K.floatx())
 
   def call(self, y_true, y_pred):
     """Invokes the `CategoricalCrossentropy` instance.
@@ -385,14 +389,11 @@ class CategoricalCrossentropy(Loss):
           y_true, y_pred, from_logits=self.from_logits)
     else:
       y_true = math_ops.cast(y_true, y_pred.dtype)
-      if self.label_smoothing > 0:
-        num_classes = math_ops.cast(array_ops.shape(y_true)[1], y_pred.dtype)
-        smooth_positives = 1.0 - self.label_smoothing
-        smooth_negatives = self.label_smoothing / num_classes
-        y_true = y_true * smooth_positives + smooth_negatives
-
       return categorical_crossentropy(
-          y_true, y_pred, from_logits=self.from_logits)
+          y_true,
+          y_pred,
+          from_logits=self.from_logits,
+          label_smoothing=self.label_smoothing)
 
 
 @keras_export('keras.losses.Hinge')
@@ -782,7 +783,29 @@ def logcosh(y_true, y_pred):
 
 @keras_export('keras.metrics.categorical_crossentropy',
               'keras.losses.categorical_crossentropy')
-def categorical_crossentropy(y_true, y_pred, from_logits=False):
+def categorical_crossentropy(y_true,
+                             y_pred,
+                             from_logits=False,
+                             label_smoothing=0):
+  """Computes the categorical crossentropy loss.
+
+  Args:
+    y_true: tensor of true targets.
+    y_pred: tensor of predicted targets.
+    from_logits: Whether `y_pred` is expected to be a logits tensor. By default,
+      we consider that `y_pred` encodes a probability distribution.
+    label_smoothing: Float in [0, 1]. If > `0` then smooth the labels.
+
+  Returns:
+    Categorical crossentropy loss value.
+  """
+
+  def _smooth_labels():
+    num_classes = math_ops.cast(array_ops.shape(y_true)[1], y_pred.dtype)
+    return y_true * (1.0 - label_smoothing) + (label_smoothing / num_classes)
+
+  y_true = smart_cond.smart_cond(label_smoothing,
+                                 _smooth_labels, lambda: y_true)
   return K.categorical_crossentropy(y_true, y_pred, from_logits=from_logits)
 
 
@@ -795,7 +818,13 @@ def sparse_categorical_crossentropy(y_true, y_pred, from_logits=False):
 
 @keras_export('keras.metrics.binary_crossentropy',
               'keras.losses.binary_crossentropy')
-def binary_crossentropy(y_true, y_pred, from_logits=False):
+def binary_crossentropy(y_true, y_pred, from_logits=False, label_smoothing=0):
+
+  def _smooth_labels():
+    return y_true * (1.0 - label_smoothing) + 0.5 * label_smoothing
+
+  y_true = smart_cond.smart_cond(label_smoothing,
+                                 _smooth_labels, lambda: y_true)
   return K.mean(
       K.binary_crossentropy(y_true, y_pred, from_logits=from_logits), axis=-1)
 
diff --git a/tensorflow/python/keras/losses_test.py b/tensorflow/python/keras/losses_test.py
index 004c30f84d..d276854e29 100644
--- a/tensorflow/python/keras/losses_test.py
+++ b/tensorflow/python/keras/losses_test.py
@@ -27,7 +27,6 @@ from tensorflow.python import keras
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import test_util
-from tensorflow.python.ops import array_ops
 from tensorflow.python.ops.losses import losses_impl
 from tensorflow.python.platform import test
 
@@ -593,74 +592,132 @@ class BinaryCrossentropyTest(test.TestCase):
     self.assertAlmostEqual(self.evaluate(loss), 0.0, 3)
 
   def test_unweighted(self):
+    y_true = np.asarray([1, 0, 1, 0]).reshape([2, 2])
+    y_pred = np.asarray([1, 1, 1, 0], dtype=np.float32).reshape([2, 2])
     bce_obj = keras.losses.BinaryCrossentropy()
-    y_true = constant_op.constant([1, 0, 1, 0, 0, 1], shape=(2, 3))
-    y_pred = constant_op.constant([1, 1, 1, 0, 1, 0],
-                                  shape=(2, 3),
-                                  dtype=dtypes.float32)
     loss = bce_obj(y_true, y_pred)
-    self.assertAlmostEqual(self.evaluate(loss), 8.0004, 3)
+
+    # EPSILON = 1e-7, y = y_true, y` = y_pred, Y_MAX = 0.9999999
+    # y` = clip_ops.clip_by_value(output, EPSILON, 1. - EPSILON)
+    # y` = [Y_MAX, Y_MAX, Y_MAX, EPSILON]
+
+    # Loss = -(y log(y` + EPSILON) + (1 - y) log(1 - y` + EPSILON))
+    #      = [-log(Y_MAX + EPSILON), -log(1 - Y_MAX + EPSILON),
+    #         -log(Y_MAX + EPSILON), -log(1)]
+    #      = [0, 15.33, 0, 0]
+    # Reduced loss = 15.33 / 4
+
+    self.assertAlmostEqual(self.evaluate(loss), 3.833, 3)
 
     # Test with logits.
-    logits = constant_op.constant([10., 10., 10., -10., 10, -10],
-                                  shape=(2, 3),
-                                  dtype=dtypes.float32)
+    y_true = constant_op.constant([[1, 0, 1], [0, 1, 1]])
+    logits = constant_op.constant([[100.0, -100.0, 100.0],
+                                   [100.0, 100.0, -100.0]])
     bce_obj = keras.losses.BinaryCrossentropy(from_logits=True)
     loss = bce_obj(y_true, logits)
-    self.assertAlmostEqual(self.evaluate(loss), 5., 3)
+
+    # Loss = max(x, 0) - x * z + log(1 + exp(-abs(x)))
+    #            (where x = logits and z = y_true)
+    #      = [((100 - 100 * 1 + log(1 + exp(-100))) +
+    #          (0 + 100 * 0 + log(1 + exp(-100))) +
+    #          (100 - 100 * 1 + log(1 + exp(-100))),
+    #         ((100 - 100 * 0 + log(1 + exp(-100))) +
+    #          (100 - 100 * 1 + log(1 + exp(-100))) +
+    #          (0 + 100 * 1 + log(1 + exp(-100))))]
+    #      = [(0 + 0 + 0) / 3, 200 / 3]
+    # Reduced loss = (0 + 66.666) / 2
+
+    self.assertAlmostEqual(self.evaluate(loss), 33.333, 3)
 
   def test_scalar_weighted(self):
     bce_obj = keras.losses.BinaryCrossentropy()
-    y_true = constant_op.constant([1, 0, 1, 0, 0, 1], shape=(2, 3))
-    y_pred = constant_op.constant([1, 1, 1, 0, 1, 0],
-                                  shape=(2, 3),
-                                  dtype=dtypes.float32)
+    y_true = np.asarray([1, 0, 1, 0]).reshape([2, 2])
+    y_pred = np.asarray([1, 1, 1, 0], dtype=np.float32).reshape([2, 2])
     loss = bce_obj(y_true, y_pred, sample_weight=2.3)
-    self.assertAlmostEqual(self.evaluate(loss), 18.4010, 3)
+
+    # EPSILON = 1e-7, y = y_true, y` = y_pred, Y_MAX = 0.9999999
+    # y` = clip_ops.clip_by_value(output, EPSILON, 1. - EPSILON)
+    # y` = [Y_MAX, Y_MAX, Y_MAX, EPSILON]
+
+    # Loss = -(y log(y` + EPSILON) + (1 - y) log(1 - y` + EPSILON))
+    #      = [-log(Y_MAX + EPSILON), -log(1 - Y_MAX + EPSILON),
+    #         -log(Y_MAX + EPSILON), -log(1)]
+    #      = [0, 15.33, 0, 0]
+    # Weighted loss = [0, 15.33 * 2.3, 0, 0]
+    # Reduced loss = 15.33 * 2.3 / 4
+
+    self.assertAlmostEqual(self.evaluate(loss), 8.817, 3)
 
     # Test with logits.
-    y_true = array_ops.ones((32, 1))
-    logits = array_ops.ones((32, 1), dtype=dtypes.float32)
+    y_true = constant_op.constant([[1, 0, 1], [0, 1, 1]])
+    logits = constant_op.constant([[100.0, -100.0, 100.0],
+                                   [100.0, 100.0, -100.0]])
     bce_obj = keras.losses.BinaryCrossentropy(from_logits=True)
     loss = bce_obj(y_true, logits, sample_weight=2.3)
-    self.assertAlmostEqual(self.evaluate(loss), 0.7205, 3)
+
+    # Loss = max(x, 0) - x * z + log(1 + exp(-abs(x)))
+    #            (where x = logits and z = y_true)
+    # Loss = [(0 + 0 + 0) / 3, 200 / 3]
+    # Weighted loss = [0 * 2.3, 66.666 * 2.3]
+    # Reduced loss = (0 + 66.666 * 2.3) / 2
+
+    self.assertAlmostEqual(self.evaluate(loss), 76.667, 3)
 
   def test_sample_weighted(self):
     bce_obj = keras.losses.BinaryCrossentropy()
-    y_true = constant_op.constant([1, 0, 1, 0, 0, 1], shape=(2, 3))
-    y_pred = constant_op.constant([1, 1, 1, 0, 1, 0],
-                                  shape=(2, 3),
-                                  dtype=dtypes.float64)
+    y_true = np.asarray([1, 0, 1, 0]).reshape([2, 2])
+    y_pred = np.asarray([1, 1, 1, 0], dtype=np.float32).reshape([2, 2])
     sample_weight = constant_op.constant([1.2, 3.4], shape=(2, 1))
     loss = bce_obj(y_true, y_pred, sample_weight=sample_weight)
-    self.assertAlmostEqual(self.evaluate(loss), 21.4907, 3)
+
+    # EPSILON = 1e-7, y = y_true, y` = y_pred, Y_MAX = 0.9999999
+    # y` = clip_ops.clip_by_value(output, EPSILON, 1. - EPSILON)
+    # y` = [Y_MAX, Y_MAX, Y_MAX, EPSILON]
+
+    # Loss = -(y log(y` + EPSILON) + (1 - y) log(1 - y` + EPSILON))
+    #      = [-log(Y_MAX + EPSILON), -log(1 - Y_MAX + EPSILON),
+    #         -log(Y_MAX + EPSILON), -log(1)]
+    #      = [0, 15.33, 0, 0]
+    # Reduced loss = 15.33 * 1.2 / 4
+
+    self.assertAlmostEqual(self.evaluate(loss), 4.6, 3)
 
     # Test with logits.
-    y_true = constant_op.constant([[0, 0, 1], [1, 0, 0], [0, 1, 0]])
-    logits = constant_op.constant(
-        [[100.0, -100.0, -100.0], [-100.0, 100.0, -100.0],
-         [-100.0, -100.0, 100.0]],
-        dtype=dtypes.float64)
-    weights = constant_op.constant([3, 2, 8])
+    y_true = constant_op.constant([[1, 0, 1], [0, 1, 1]])
+    logits = constant_op.constant([[100.0, -100.0, 100.0],
+                                   [100.0, 100.0, -100.0]])
+    weights = constant_op.constant([4, 3])
     bce_obj = keras.losses.BinaryCrossentropy(from_logits=True)
     loss = bce_obj(y_true, logits, sample_weight=weights)
-    self.assertAlmostEqual(self.evaluate(loss), 288.8888, 3)
+
+    # Loss = max(x, 0) - x * z + log(1 + exp(-abs(x)))
+    #            (where x = logits and z = y_true)
+    # Loss = [(0 + 0 + 0)/3, 200 / 3]
+    # Weighted loss = [0 * 4, 66.666 * 3]
+    # Reduced loss = (0 + 66.666 * 3) / 2
+
+    self.assertAlmostEqual(self.evaluate(loss), 100, 3)
 
   def test_no_reduction(self):
-    y_true = constant_op.constant(((1, 0, 1), (1, 1, 0), (0, 1, 1)))
-    logits = constant_op.constant(((100.0, -100.0, 100.0),
-                                   (100.0, -100.0, 100.0),
-                                   (100.0, 100.0, -100.0)))
+    y_true = constant_op.constant([[1, 0, 1], [0, 1, 1]])
+    logits = constant_op.constant([[100.0, -100.0, 100.0],
+                                   [100.0, 100.0, -100.0]])
     bce_obj = keras.losses.BinaryCrossentropy(
         from_logits=True, reduction=losses_impl.ReductionV2.NONE)
     loss = bce_obj(y_true, logits)
-    self.assertAllClose((0., 66.6666, 66.6666), self.evaluate(loss), 3)
+
+    # Loss = max(x, 0) - x * z + log(1 + exp(-abs(x)))
+    #            (where x = logits and z = y_true)
+    # Loss = [(0 + 0 + 0)/3, (200)/3]
+
+    self.assertAllClose((0., 66.6666), self.evaluate(loss), 3)
 
   def test_label_smoothing(self):
     logits = constant_op.constant([[100.0, -100.0, -100.0]])
     y_true = constant_op.constant([[1, 0, 1]])
     label_smoothing = 0.1
     # Loss: max(x, 0) - x * z + log(1 + exp(-abs(x)))
+    #            (where x = logits and z = y_true)
     # Label smoothing: z' = z * (1 - L) + 0.5L
     #                  1  = 1 - 0.5L
     #                  0  = 0.5L
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.pbtxt
index b71e89883e..ce942de6d2 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.losses.pbtxt
@@ -62,11 +62,11 @@ tf_module {
   }
   member_method {
     name: "binary_crossentropy"
-    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], "
+    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\', \'label_smoothing\'], varargs=None, keywords=None, defaults=[\'False\', \'0\'], "
   }
   member_method {
     name: "categorical_crossentropy"
-    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], "
+    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\', \'label_smoothing\'], varargs=None, keywords=None, defaults=[\'False\', \'0\'], "
   }
   member_method {
     name: "categorical_hinge"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt
index 54f5b21f55..cae251cdfe 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.metrics.pbtxt
@@ -110,7 +110,7 @@ tf_module {
   }
   member_method {
     name: "binary_crossentropy"
-    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], "
+    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\', \'label_smoothing\'], varargs=None, keywords=None, defaults=[\'False\', \'0\'], "
   }
   member_method {
     name: "categorical_accuracy"
@@ -118,7 +118,7 @@ tf_module {
   }
   member_method {
     name: "categorical_crossentropy"
-    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], "
+    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\', \'label_smoothing\'], varargs=None, keywords=None, defaults=[\'False\', \'0\'], "
   }
   member_method {
     name: "cosine"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt
index 673593ed6c..e8809450fa 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.losses.pbtxt
@@ -66,11 +66,11 @@ tf_module {
   }
   member_method {
     name: "binary_crossentropy"
-    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], "
+    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\', \'label_smoothing\'], varargs=None, keywords=None, defaults=[\'False\', \'0\'], "
   }
   member_method {
     name: "categorical_crossentropy"
-    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], "
+    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\', \'label_smoothing\'], varargs=None, keywords=None, defaults=[\'False\', \'0\'], "
   }
   member_method {
     name: "categorical_hinge"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt
index 54f5b21f55..cae251cdfe 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.metrics.pbtxt
@@ -110,7 +110,7 @@ tf_module {
   }
   member_method {
     name: "binary_crossentropy"
-    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], "
+    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\', \'label_smoothing\'], varargs=None, keywords=None, defaults=[\'False\', \'0\'], "
   }
   member_method {
     name: "categorical_accuracy"
@@ -118,7 +118,7 @@ tf_module {
   }
   member_method {
     name: "categorical_crossentropy"
-    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], "
+    argspec: "args=[\'y_true\', \'y_pred\', \'from_logits\', \'label_smoothing\'], varargs=None, keywords=None, defaults=[\'False\', \'0\'], "
   }
   member_method {
     name: "cosine"
-- 
GitLab


From af9f12801a72ef9e9cbfa3e0dbca3510388c7104 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 16 Jan 2019 17:03:57 -0800
Subject: [PATCH 0853/1095] Fork contrib's timeseries into the canned estimator
 folder and clean it up to only include LSTMAutoRegressor and its
 dependencies. After the fork, the following changes were made:

1. Imports are pointed to the new fork except for symbols that were removed during the next steps of the clean up (see below). tensorflow/.../estimator imports were changed to tensorflow_estimator/...

2. estimators.py had all its symbols removed except for LSTMAutoRegressor which is the one we want to keep.

3. ar_model.py had all its symbols removed except for LSTMPredictionModel and ARModel (which uses LSTMPredictionModel).

4. state_management.py had ChainingStateManager removed and as state_management_test.py only tests ChainingStateManager, that test file was removed.

5. math_utils.py had all its symbols removed except for InputStatisticsFromMiniBatch and replicate_state.

6. examples folder was removed. This will come back in some form (perhaps an updated docstring in estimators.py) before LSTMAutoRegressor is exported.

7. input_pipeline.py was removed. This is going to be replaced with a user supplied tf.data-backed input_fn. For now, tests point to contrib's input_pipeline.

8. Cleaned up the BUILD and __init__.py files. Fixed rules and removed stale tags.

PiperOrigin-RevId: 229654011
---
 tensorflow/contrib/timeseries/examples/BUILD          | 2 ++
 tensorflow/contrib/timeseries/python/timeseries/BUILD | 1 +
 2 files changed, 3 insertions(+)

diff --git a/tensorflow/contrib/timeseries/examples/BUILD b/tensorflow/contrib/timeseries/examples/BUILD
index 57797214d1..e10be88ece 100644
--- a/tensorflow/contrib/timeseries/examples/BUILD
+++ b/tensorflow/contrib/timeseries/examples/BUILD
@@ -105,6 +105,7 @@ py_binary(
     data = ["data/multivariate_periods.csv"],
     srcs_version = "PY2AND3",
     tags = ["no_pip"],
+    visibility = ["//visibility:public"],
     deps = select({
         ":empty_condition": [],
         "//conditions:default": [],
@@ -113,6 +114,7 @@ py_binary(
         "//tensorflow:tensorflow_py",
         "//tensorflow/contrib/timeseries/python/timeseries:estimators",
         "//tensorflow/contrib/timeseries/python/timeseries:model",
+        "//tensorflow/contrib/timeseries/python/timeseries:state_management",
     ],
 )
 
diff --git a/tensorflow/contrib/timeseries/python/timeseries/BUILD b/tensorflow/contrib/timeseries/python/timeseries/BUILD
index a0c3204d41..2a22295197 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/BUILD
+++ b/tensorflow/contrib/timeseries/python/timeseries/BUILD
@@ -281,6 +281,7 @@ py_library(
         "input_pipeline.py",
     ],
     srcs_version = "PY2AND3",
+    visibility = ["//visibility:public"],
     deps = [
         ":feature_keys",
         ":model_utils",
-- 
GitLab


From 574db93f256879e4182760b2661f2c398f48372e Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Wed, 16 Jan 2019 17:08:25 -0800
Subject: [PATCH 0854/1095] Add SerializableResource and
 GetSerializedResouceOp. This is needed by: 1. the new calibration design. The
 current int8 calibration workflow depends on    a global resouce manager
 singleton TRTResourceManager (in    resources/trt_resource_manager.h). This
 has been:    - violating the resource manager design: resource manager should
 be      per-device    - pollutes the BUILD dependencies, and makes the kernel
 implementation not      able to be used in other language bindings (Issue
 #23853) 2. the custom backend offline mode, where we'll do the conversion
 during    execution and provide an offline tool to get the serizlied engine

PiperOrigin-RevId: 229654702
---
 tensorflow/contrib/BUILD                      |  2 +-
 tensorflow/contrib/tensorrt/BUILD             | 49 +++++++++---
 .../kernels/get_serialized_resource_op.cc     | 73 +++++++++++++++++
 .../get_serialized_resource_op_test.cc        | 80 +++++++++++++++++++
 .../ops/get_serialized_resource_op.cc         | 40 ++++++++++
 .../contrib/tensorrt/python/__init__.py       |  2 +-
 .../ops/{trt_engine_op.py => trt_ops.py}      |  6 +-
 .../tensorrt/python/trt_convert_test.py       |  2 +-
 .../tensorrt/resources/trt_resources.cc       | 61 ++++++++++++++
 .../tensorrt/resources/trt_resources.h        | 40 +++-------
 .../test/dynamic_input_shapes_test.py         |  3 -
 .../tensorrt/test/quantization_mnist_test.py  |  2 +-
 .../test/tf_trt_integration_test_base.py      |  2 +-
 13 files changed, 314 insertions(+), 48 deletions(-)
 create mode 100644 tensorflow/contrib/tensorrt/kernels/get_serialized_resource_op.cc
 create mode 100644 tensorflow/contrib/tensorrt/kernels/get_serialized_resource_op_test.cc
 create mode 100644 tensorflow/contrib/tensorrt/ops/get_serialized_resource_op.cc
 rename tensorflow/contrib/tensorrt/python/ops/{trt_engine_op.py => trt_ops.py} (87%)
 create mode 100644 tensorflow/contrib/tensorrt/resources/trt_resources.cc

diff --git a/tensorflow/contrib/BUILD b/tensorflow/contrib/BUILD
index 307bb6eca3..a4c3d9623a 100644
--- a/tensorflow/contrib/BUILD
+++ b/tensorflow/contrib/BUILD
@@ -196,7 +196,7 @@ cc_library(
             "//tensorflow/contrib/kinesis:dataset_kernels",
         ],
     }) + if_not_windows([
-        "//tensorflow/contrib/tensorrt:trt_engine_op_kernel",
+        "//tensorflow/contrib/tensorrt:trt_op_kernels",
     ]),
 )
 
diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index 7ae3caaaa2..80d79164a2 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -54,8 +54,9 @@ tf_cuda_cc_test(
 )
 
 tf_custom_op_library(
-    name = "python/ops/_trt_engine_op.so",
+    name = "python/ops/_trt_ops.so",
     srcs = [
+        "ops/get_serialized_resource_op.cc",
         "ops/trt_engine_op.cc",
     ],
     deps = [
@@ -80,8 +81,9 @@ tf_cuda_library(
 )
 
 cc_library(
-    name = "trt_engine_op_kernel",
+    name = "trt_op_kernels",
     srcs = [
+        "kernels/get_serialized_resource_op.cc",
         "kernels/trt_engine_op.cc",
     ],
     hdrs = [
@@ -109,9 +111,33 @@ cc_library(
     alwayslink = 1,  # buildozer: disable=alwayslink-with-hdrs
 )
 
+tf_cuda_cc_test(
+    name = "get_serialized_resource_op_test",
+    size = "small",
+    srcs = ["kernels/get_serialized_resource_op_test.cc"],
+    tags = [
+        "no_cuda_on_cpu_tap",
+        "no_windows",
+        "nomac",
+    ],
+    deps = [
+        ":get_serialized_resource_op_op_lib",
+        ":trt_op_kernels",
+        ":trt_resources",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
+        "//tensorflow/core/kernels:ops_testutil",
+    ],
+)
+
 tf_gen_op_libs(
     op_lib_names = [
         "trt_engine_op",
+        "get_serialized_resource_op",
     ],
 )
 
@@ -128,8 +154,9 @@ tf_cuda_library(
 )
 
 tf_gen_op_wrapper_py(
-    name = "trt_engine_op",
+    name = "trt_ops",
     deps = [
+        ":get_serialized_resource_op_op_lib",
         ":trt_engine_op_op_lib",
         ":trt_logging",
         ":trt_shape_function",
@@ -137,16 +164,17 @@ tf_gen_op_wrapper_py(
 )
 
 tf_custom_op_py_library(
-    name = "trt_engine_op_loader",
-    srcs = ["python/ops/trt_engine_op.py"],
+    name = "trt_ops_loader",
+    srcs = ["python/ops/trt_ops.py"],
     dso = [
-        ":python/ops/_trt_engine_op.so",
+        ":python/ops/_trt_ops.so",
     ] + if_tensorrt([
         "@local_config_tensorrt//:nv_infer",
     ]),
     kernels = [
-        ":trt_engine_op_kernel",
+        ":trt_op_kernels",
         ":trt_engine_op_op_lib",
+        ":get_serialized_resource_op_op_lib",
         ":trt_shape_function",
     ],
     srcs_version = "PY2AND3",
@@ -176,8 +204,8 @@ py_library(
     name = "trt_ops_py",
     srcs_version = "PY2AND3",
     deps = [
-        ":trt_engine_op",
-        ":trt_engine_op_loader",
+        ":trt_ops",
+        ":trt_ops_loader",
     ],
 )
 
@@ -208,7 +236,7 @@ tf_py_wrap_cc(
     deps = [
         ":test_utils",
         ":trt_conversion",
-        ":trt_engine_op_kernel",
+        ":trt_op_kernels",
         "//third_party/python_runtime:headers",
     ],
 )
@@ -218,6 +246,7 @@ tf_cuda_library(
     srcs = [
         "resources/trt_int8_calibrator.cc",
         "resources/trt_resource_manager.cc",
+        "resources/trt_resources.cc",
     ],
     hdrs = [
         "resources/trt_int8_calibrator.h",
diff --git a/tensorflow/contrib/tensorrt/kernels/get_serialized_resource_op.cc b/tensorflow/contrib/tensorrt/kernels/get_serialized_resource_op.cc
new file mode 100644
index 0000000000..f68bc2b485
--- /dev/null
+++ b/tensorflow/contrib/tensorrt/kernels/get_serialized_resource_op.cc
@@ -0,0 +1,73 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CONTRIB_TENSORRT_KERNELS_GET_SERIALIZED_RESOURCE_OP_H_
+#define TENSORFLOW_CONTRIB_TENSORRT_KERNELS_GET_SERIALIZED_RESOURCE_OP_H_
+
+#include <memory>
+#include <vector>
+
+#include "tensorflow/contrib/tensorrt/resources/trt_resources.h"
+#include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/resource_mgr.h"
+#include "tensorflow/core/lib/core/refcount.h"
+
+#if GOOGLE_CUDA
+#if GOOGLE_TENSORRT
+
+namespace tensorflow {
+namespace tensorrt {
+
+class GetSerializedResourceOp : public OpKernel {
+ public:
+  explicit GetSerializedResourceOp(OpKernelConstruction* context)
+      : OpKernel(context) {}
+
+  ~GetSerializedResourceOp() override {}
+
+  void Compute(OpKernelContext* context) override {
+    // TODO(laigd): it will allocate the tensor on the device and copy the
+    // serialized string to that tensor, and later sess.run() will copy it back
+    // to host. We need to optimize this.
+    const string& container = context->input(0).scalar<string>()();
+    const string& resource_name = context->input(1).scalar<string>()();
+
+    // Get the resource.
+    SerializableResourceBase* resource = nullptr;
+    OP_REQUIRES_OK(context, context->resource_manager()->Lookup(
+                                container, resource_name, &resource));
+    ::tensorflow::core::ScopedUnref sc(resource);
+
+    // Serialize the resource as output.
+    string serialized_resource;
+    OP_REQUIRES_OK(context, resource->SerializeToString(&serialized_resource));
+
+    Tensor* output = nullptr;
+    OP_REQUIRES_OK(context,
+                   context->allocate_output(0, TensorShape({}), &output));
+    output->scalar<string>()() = serialized_resource;
+  }
+};
+
+REGISTER_KERNEL_BUILDER(Name("GetSerializedResourceOp").Device(DEVICE_GPU),
+                        GetSerializedResourceOp);
+
+}  // namespace tensorrt
+}  // namespace tensorflow
+
+#endif  // GOOGLE_TENSORRT
+#endif  // GOOGLE_CUDA
+#endif  // TENSORFLOW_CONTRIB_TENSORRT_KERNELS_GET_SERIALIZED_RESOURCE_OP_H_
diff --git a/tensorflow/contrib/tensorrt/kernels/get_serialized_resource_op_test.cc b/tensorflow/contrib/tensorrt/kernels/get_serialized_resource_op_test.cc
new file mode 100644
index 0000000000..a91228e4c4
--- /dev/null
+++ b/tensorflow/contrib/tensorrt/kernels/get_serialized_resource_op_test.cc
@@ -0,0 +1,80 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <dirent.h>
+#include <string.h>
+#include <fstream>
+#include <vector>
+
+#include "tensorflow/contrib/tensorrt/resources/trt_resources.h"
+#include "tensorflow/core/framework/fake_input.h"
+#include "tensorflow/core/framework/node_def_builder.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/kernels/ops_testutil.h"
+#include "tensorflow/core/platform/test.h"
+
+#if GOOGLE_CUDA
+#if GOOGLE_TENSORRT
+
+namespace tensorflow {
+namespace tensorrt {
+
+class GetSerializedResourceOpTest : public OpsTestBase {};
+
+TEST_F(GetSerializedResourceOpTest, Basic) {
+  // Create the GPU device.
+  std::unique_ptr<Device> device(
+      DeviceFactory::NewDevice("GPU", {}, "/job:worker/replica:0/task:0"));
+
+  // Create the resource.
+  class MySerializableResource : public SerializableResourceBase {
+   public:
+    string DebugString() const override { return ""; }
+    Status SerializeToString(string* serialized) override {
+      *serialized = "my_serialized_str";
+      return Status::OK();
+    }
+  };
+  const string container = "mycontainer";
+  const string resource_name = "myresource";
+  SerializableResourceBase* resource = new MySerializableResource();
+  ResourceMgr* rm = device->resource_manager();
+  EXPECT_TRUE(rm->Create(container, resource_name, resource).ok());
+
+  // Create the op.
+  SetDevice(DEVICE_GPU, std::move(device));
+  TF_ASSERT_OK(NodeDefBuilder("op", "GetSerializedResourceOp")
+                   .Input(FakeInput(DT_STRING))
+                   .Input(FakeInput(DT_STRING))
+                   .Finalize(node_def()));
+  TF_ASSERT_OK(InitOp());
+
+  // Execute the op.
+  AddInputFromArray<string>(TensorShape({}), {container});
+  AddInputFromArray<string>(TensorShape({}), {resource_name});
+  TF_ASSERT_OK(RunOpKernel());
+
+  // Verify the result.
+  // TODO(laigd): OpsTestBase::GetOutput() doesn't work.
+  Tensor* output = context_->mutable_output(0);
+  EXPECT_EQ("my_serialized_str", output->scalar<string>()());
+}
+
+}  // namespace tensorrt
+}  // namespace tensorflow
+
+#endif  // GOOGLE_TENSORRT
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/contrib/tensorrt/ops/get_serialized_resource_op.cc b/tensorflow/contrib/tensorrt/ops/get_serialized_resource_op.cc
new file mode 100644
index 0000000000..59da73f5ef
--- /dev/null
+++ b/tensorflow/contrib/tensorrt/ops/get_serialized_resource_op.cc
@@ -0,0 +1,40 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+#if GOOGLE_TENSORRT
+
+#include "tensorflow/core/framework/common_shape_fns.h"
+#include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/framework/shape_inference.h"
+#include "tensorflow/core/framework/tensor_shape.h"
+
+namespace tensorflow {
+
+REGISTER_OP("GetSerializedResourceOp")
+    .Input("container: string")
+    .Input("resource_name: string")
+    .Output("serialized_resource: string")
+    .SetShapeFn(shape_inference::ScalarShape)
+    .SetIsStateful()
+    .Doc(R"doc(
+Gets a resource from a container managed by the resource manager and returns
+its serialized representation.
+)doc");
+
+}  // namespace tensorflow
+
+#endif  // GOOGLE_TENSORRT
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/contrib/tensorrt/python/__init__.py b/tensorflow/contrib/tensorrt/python/__init__.py
index 7cdfe2b1a6..e2cf253ca0 100644
--- a/tensorflow/contrib/tensorrt/python/__init__.py
+++ b/tensorflow/contrib/tensorrt/python/__init__.py
@@ -19,7 +19,7 @@ from __future__ import division
 from __future__ import print_function
 
 # pylint: disable=unused-import,line-too-long
-from tensorflow.contrib.tensorrt.python.ops import trt_engine_op
+from tensorflow.contrib.tensorrt.python.ops import trt_ops
 from tensorflow.contrib.tensorrt.python.trt_convert import add_test_value
 from tensorflow.contrib.tensorrt.python.trt_convert import calib_graph_to_infer_graph
 from tensorflow.contrib.tensorrt.python.trt_convert import clear_test_values
diff --git a/tensorflow/contrib/tensorrt/python/ops/trt_engine_op.py b/tensorflow/contrib/tensorrt/python/ops/trt_ops.py
similarity index 87%
rename from tensorflow/contrib/tensorrt/python/ops/trt_engine_op.py
rename to tensorflow/contrib/tensorrt/python/ops/trt_ops.py
index 31a313182b..1fee06854f 100644
--- a/tensorflow/contrib/tensorrt/python/ops/trt_engine_op.py
+++ b/tensorflow/contrib/tensorrt/python/ops/trt_ops.py
@@ -22,13 +22,13 @@ import platform
 
 if platform.system() != "Windows":
   # pylint: disable=wildcard-import,unused-import,g-import-not-at-top
-  from tensorflow.contrib.tensorrt.ops.gen_trt_engine_op import *
+  from tensorflow.contrib.tensorrt.ops.gen_trt_ops import *
 
   from tensorflow.contrib.util import loader
   from tensorflow.python.platform import resource_loader
   # pylint: enable=wildcard-import,unused-import,g-import-not-at-top
 
-  _trt_engine_op = loader.load_op_library(
-      resource_loader.get_path_to_datafile("_trt_engine_op.so"))
+  _trt_ops = loader.load_op_library(
+      resource_loader.get_path_to_datafile("_trt_ops.so"))
 else:
   raise RuntimeError("Windows platforms are not supported")
diff --git a/tensorflow/contrib/tensorrt/python/trt_convert_test.py b/tensorflow/contrib/tensorrt/python/trt_convert_test.py
index 14ab4772c4..9562704aeb 100644
--- a/tensorflow/contrib/tensorrt/python/trt_convert_test.py
+++ b/tensorflow/contrib/tensorrt/python/trt_convert_test.py
@@ -22,7 +22,7 @@ import os
 
 from tensorflow.contrib.tensorrt.python import trt_convert
 # pylint: disable=unused-import
-from tensorflow.contrib.tensorrt.python.ops import trt_engine_op
+from tensorflow.contrib.tensorrt.python.ops import trt_ops
 # pylint: enable=unused-import
 from tensorflow.core.framework import graph_pb2
 from tensorflow.core.protobuf import config_pb2
diff --git a/tensorflow/contrib/tensorrt/resources/trt_resources.cc b/tensorflow/contrib/tensorrt/resources/trt_resources.cc
new file mode 100644
index 0000000000..c19eb34dab
--- /dev/null
+++ b/tensorflow/contrib/tensorrt/resources/trt_resources.cc
@@ -0,0 +1,61 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/contrib/tensorrt/resources/trt_resources.h"
+
+#if GOOGLE_CUDA
+#if GOOGLE_TENSORRT
+
+namespace tensorflow {
+namespace tensorrt {
+
+TRTCalibrationResource::~TRTCalibrationResource() {
+  VLOG(0) << "Destroying Calibration Resource " << std::endl << DebugString();
+  builder_.reset();
+  engine_.reset();
+  // We need to manually destroy the builder and engine before the allocator
+  // is destroyed.
+  allocator_.reset();
+}
+
+string TRTCalibrationResource::DebugString() const {
+  std::stringstream oss;
+  using std::dec;
+  using std::endl;
+  using std::hex;
+  oss << " Calibrator = " << hex << calibrator_.get() << dec << endl
+      << " Builder    = " << hex << builder_.get() << dec << endl
+      << " Engine     = " << hex << engine_.get() << dec << endl
+      << " Logger     = " << hex << &logger_ << dec << endl
+      << " Allocator  = " << hex << allocator_.get() << dec << endl
+      << " Thread     = " << hex << thr_.get() << dec << endl;
+  return oss.str();
+}
+
+Status TRTCalibrationResource::SerializeToString(string* serialized) {
+  calibrator_->waitAndSetDone();
+  thr_->join();
+  *serialized = calibrator_->getCalibrationTableAsString();
+  if (!serialized->size()) {
+    return tensorflow::errors::Unknown("Calibration table is empty.");
+  }
+  return Status::OK();
+}
+
+}  // namespace tensorrt
+}  // namespace tensorflow
+
+#endif  // GOOGLE_TENSORRT
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/contrib/tensorrt/resources/trt_resources.h b/tensorflow/contrib/tensorrt/resources/trt_resources.h
index 11930f3671..d0a87f2c31 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_resources.h
+++ b/tensorflow/contrib/tensorrt/resources/trt_resources.h
@@ -32,37 +32,23 @@ limitations under the License.
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
-
 #include "tensorrt/include/NvInfer.h"
 
 namespace tensorflow {
 namespace tensorrt {
 
-class TRTCalibrationResource : public tensorflow::ResourceBase {
+class SerializableResourceBase : public tensorflow::ResourceBase {
+ public:
+  virtual Status SerializeToString(string* serialized) = 0;
+};
+
+class TRTCalibrationResource : public SerializableResourceBase {
  public:
-  ~TRTCalibrationResource() {
-    LOG(INFO) << "Destroying Calibration Resource " << std::endl
-              << DebugString();
-    builder_.reset();
-    engine_.reset();
-    // We need to manually destroy the builder and engine before the allocator
-    // is destroyed.
-    allocator_.reset();
-  }
-
-  string DebugString() const override {
-    std::stringstream oss;
-    using std::dec;
-    using std::endl;
-    using std::hex;
-    oss << " Calibrator = " << hex << calibrator_.get() << dec << endl
-        << " Builder    = " << hex << builder_.get() << dec << endl
-        << " Engine     = " << hex << engine_.get() << dec << endl
-        << " Logger     = " << hex << &logger_ << dec << endl
-        << " Allocator  = " << hex << allocator_.get() << dec << endl
-        << " Thread     = " << hex << thr_.get() << dec << endl;
-    return oss.str();
-  }
+  ~TRTCalibrationResource() override;
+
+  string DebugString() const override;
+
+  Status SerializeToString(string* serialized) override;
 
   // Lookup table for temporary staging areas of input tensors for calibration.
   std::unordered_map<string, std::pair<void*, size_t>> device_buffers_;
@@ -82,6 +68,6 @@ class TRTCalibrationResource : public tensorflow::ResourceBase {
 }  // namespace tensorrt
 }  // namespace tensorflow
 
-#endif
-#endif
+#endif  // GOOGLE_TENSORRT
+#endif  // GOOGLE_CUDA
 #endif  // TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_RESOURCES_H_
diff --git a/tensorflow/contrib/tensorrt/test/dynamic_input_shapes_test.py b/tensorflow/contrib/tensorrt/test/dynamic_input_shapes_test.py
index 90addecb14..cc28cd6087 100644
--- a/tensorflow/contrib/tensorrt/test/dynamic_input_shapes_test.py
+++ b/tensorflow/contrib/tensorrt/test/dynamic_input_shapes_test.py
@@ -20,9 +20,6 @@ from __future__ import print_function
 
 import numpy as np
 
-# pylint: disable=unused-import
-from tensorflow.contrib.tensorrt.python.ops import trt_engine_op
-# pylint: enable=unused-import
 from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
diff --git a/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py b/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py
index 966476dc3e..753b47f4d9 100644
--- a/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py
+++ b/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py
@@ -20,7 +20,7 @@ from __future__ import print_function
 
 from tensorflow.contrib.tensorrt.python import trt_convert
 # pylint: disable=unused-import
-from tensorflow.contrib.tensorrt.python.ops import trt_engine_op
+from tensorflow.contrib.tensorrt.python.ops import trt_ops
 # pylint: enable=unused-import
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python import data
diff --git a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
index f13c615cb1..4f47f80e73 100644
--- a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
+++ b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
@@ -27,7 +27,7 @@ import six
 
 from tensorflow.contrib.tensorrt.python import trt_convert
 # pylint: disable=unused-import
-from tensorflow.contrib.tensorrt.python.ops import trt_engine_op
+from tensorflow.contrib.tensorrt.python.ops import trt_ops
 # pylint: enable=unused-import
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.core.protobuf import rewriter_config_pb2
-- 
GitLab


From 8b0e7dfcf0e5f4d0a15a1c32dd379292f2b344fb Mon Sep 17 00:00:00 2001
From: Katherine Wu <kathywu@google.com>
Date: Wed, 16 Jan 2019 17:16:40 -0800
Subject: [PATCH 0855/1095] Correct add_weight docstring by removing mention of
 outdated functionality.

PiperOrigin-RevId: 229655774
---
 tensorflow/python/keras/engine/base_layer.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/keras/engine/base_layer.py b/tensorflow/python/keras/engine/base_layer.py
index 5a3540d12d..7bfc301239 100644
--- a/tensorflow/python/keras/engine/base_layer.py
+++ b/tensorflow/python/keras/engine/base_layer.py
@@ -256,7 +256,7 @@ class Layer(checkpointable.Checkpointable):
                  synchronization=tf_variables.VariableSynchronization.AUTO,
                  aggregation=tf_variables.VariableAggregation.NONE,
                  **kwargs):
-    """Adds a new variable to the layer, or gets an existing one; returns it.
+    """Adds a new variable to the layer.
 
     Arguments:
       name: variable name.
-- 
GitLab


From 1d669ac6ccaa343add66d652f48b2ae943932a43 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Wed, 16 Jan 2019 17:28:46 -0800
Subject: [PATCH 0856/1095] [XLA] Delete dumped_computation_to_graphviz tool.

Superceded by the interactive_graphviz tool.

PiperOrigin-RevId: 229657392
---
 tensorflow/compiler/xla/tools/BUILD           | 27 ------
 .../tools/dumped_computation_to_graphviz.cc   | 84 -------------------
 2 files changed, 111 deletions(-)
 delete mode 100644 tensorflow/compiler/xla/tools/dumped_computation_to_graphviz.cc

diff --git a/tensorflow/compiler/xla/tools/BUILD b/tensorflow/compiler/xla/tools/BUILD
index a7ef32fd35..4412a6ec69 100644
--- a/tensorflow/compiler/xla/tools/BUILD
+++ b/tensorflow/compiler/xla/tools/BUILD
@@ -29,33 +29,6 @@ tf_cc_binary(
     ],
 )
 
-cc_library(
-    name = "dumped_computation_to_graphviz_library",
-    srcs = ["dumped_computation_to_graphviz.cc"],
-    deps = [
-        "//tensorflow/compiler/xla:debug_options_flags",
-        "//tensorflow/compiler/xla:statusor",
-        "//tensorflow/compiler/xla:types",
-        "//tensorflow/compiler/xla:xla_data_proto",
-        "//tensorflow/compiler/xla/client",
-        "//tensorflow/compiler/xla/client:client_library",
-        "//tensorflow/compiler/xla/client:local_client",
-        "//tensorflow/compiler/xla/client:xla_computation",
-        "//tensorflow/compiler/xla/service",
-        "//tensorflow/compiler/xla/service:hlo_proto",
-        "//tensorflow/core:lib",
-        "@com_google_absl//absl/types:span",
-    ],
-)
-
-tf_cc_binary(
-    name = "dumped_computation_to_graphviz",
-    deps = [
-        ":dumped_computation_to_graphviz_library",
-        "//tensorflow/compiler/xla/service:interpreter_plugin",
-    ],
-)
-
 tf_cc_binary(
     name = "show_signature",
     srcs = ["show_signature.cc"],
diff --git a/tensorflow/compiler/xla/tools/dumped_computation_to_graphviz.cc b/tensorflow/compiler/xla/tools/dumped_computation_to_graphviz.cc
deleted file mode 100644
index b623556468..0000000000
--- a/tensorflow/compiler/xla/tools/dumped_computation_to_graphviz.cc
+++ /dev/null
@@ -1,84 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-// Usage: dumped_computation_to_graphviz some_binary_snapshot_proto*
-//
-// Dumps a graphviz URL for a snapshot computation to the command line.
-//
-// some_binary_snapshot_proto is obtained by serializing the HloSnapshot from
-// ServiceInterface::SnapshotComputation to disk.
-//
-// The GraphViz URL is placed into the log stderr, whereas computation
-// statistics are printed on stdout (implementation note: getting computation
-// statistics is how we trigger compilation to split out a GraphViz URL).
-
-#include <stdio.h>
-#include <memory>
-#include <string>
-
-#include "absl/types/span.h"
-#include "tensorflow/compiler/xla/client/client.h"
-#include "tensorflow/compiler/xla/client/client_library.h"
-#include "tensorflow/compiler/xla/client/local_client.h"
-#include "tensorflow/compiler/xla/client/xla_computation.h"
-#include "tensorflow/compiler/xla/debug_options_flags.h"
-#include "tensorflow/compiler/xla/service/hlo.pb.h"
-#include "tensorflow/compiler/xla/service/service.h"
-#include "tensorflow/compiler/xla/statusor.h"
-#include "tensorflow/compiler/xla/types.h"
-#include "tensorflow/compiler/xla/xla_data.pb.h"
-#include "tensorflow/core/platform/env.h"
-#include "tensorflow/core/platform/init_main.h"
-#include "tensorflow/core/platform/logging.h"
-
-namespace xla {
-namespace tools {
-
-void RealMain(absl::Span<char* const> args) {
-  Client* client = ClientLibrary::LocalClientOrDie();
-  for (char* arg : args) {
-    HloSnapshot module;
-    TF_CHECK_OK(
-        tensorflow::ReadBinaryProto(tensorflow::Env::Default(), arg, &module));
-    XlaComputation computation =
-        client->LoadSnapshot(module).ConsumeValueOrDie();
-    DebugOptions debug_options = GetDebugOptionsFromFlags();
-    debug_options.set_xla_generate_hlo_graph(".*");
-    ComputationStats stats =
-        client->GetComputationStats(computation, debug_options)
-            .ConsumeValueOrDie();
-    fprintf(stdout, ">>> %s :: %s\n", arg, stats.DebugString().c_str());
-  }
-}
-
-}  // namespace tools
-}  // namespace xla
-
-int main(int argc, char** argv) {
-  std::vector<tensorflow::Flag> flag_list;
-  xla::AppendDebugOptionsFlags(&flag_list);
-  xla::string usage = tensorflow::Flags::Usage(argv[0], flag_list);
-  const bool parse_result = tensorflow::Flags::Parse(&argc, argv, flag_list);
-  if (!parse_result) {
-    LOG(ERROR) << "\n" << usage;
-    return 2;
-  }
-  tensorflow::port::InitMain(argv[0], &argc, &argv);
-
-  absl::Span<char* const> args(argv, argc);
-  args.remove_prefix(1);  // Pop off the binary name, argv[0]
-  xla::tools::RealMain(args);
-  return 0;
-}
-- 
GitLab


From 0f6bde31cf9593395529dce78cb4ec95eecbba3c Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Wed, 16 Jan 2019 18:04:59 -0800
Subject: [PATCH 0857/1095] tf.saved_model: Allow saving/loading concrete
 functions directly

We'll want this for representing signatures, and I don't see any reason to limit it to that.

Some fiddling so that the function gets registered with the eager context. Apparently that was only working when graph building.

PiperOrigin-RevId: 229661863
---
 tensorflow/python/eager/function.py           | 56 +++++------
 .../saved_model/function_deserialization.py   | 18 ++++
 .../saved_model/function_serialization.py     | 65 +++++++++----
 tensorflow/python/saved_model/load.py         | 96 +++++++++++--------
 tensorflow/python/saved_model/load_test.py    | 51 ++++++++++
 tensorflow/python/saved_model/save.py         | 35 ++++---
 .../saved_model/saved_object_graph.proto      |  1 +
 7 files changed, 224 insertions(+), 98 deletions(-)

diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index 9b63b6f31c..82dea51c29 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -256,13 +256,16 @@ class _EagerDefinedFunction(object):
     self._graph = graph
     self._stateful_ops = tuple(op for op in operations if op.op_def.is_stateful)
 
-  def add_to_graph(self, g):
+  def add_to_graph(self, g=None):
     # pylint: disable=protected-access
-    if self.name not in g._functions:
-      g._add_function(self)
-    for f in self._graph._functions.values():
-      if f.name not in g._functions:
-        g._add_function(f)
+    if not g and context.executing_eagerly():
+      context.context().add_function_def(self.definition)
+    else:
+      if self.name not in g._functions:
+        g._add_function(self)
+      for f in self._graph._functions.values():
+        if f.name not in g._functions:
+          g._add_function(f)
     # pylint: enable=protected-access
 
   @property
@@ -635,27 +638,26 @@ class ConcreteFunction(object):
     # method's functionality better. Remove register_gradient_functions argument
     # and figure out if these needs to be registered.
 
-    if not context.executing_eagerly() or g:
-      if not g:
-        g = ops.get_default_graph()
-      self._inference_function.add_to_graph(g)  # pylint: disable=protected-access
-
-      # pylint: disable=protected-access
-      if register_gradient_functions:
-        # There are two situations for the actual call of a defun:
-        # 1. If none of the input args are resource variables or watch by any
-        #   tape, and it will run the _inference_function of concrete_func for
-        #   forward pass, the gradient will be generated by standard mechanism.
-        # 2. Otherwise, defun will create two functions, one for forward pass,
-        #   and the backward pass will be created via tape.
-        #   When registering the function, we register both cases.
-        if self._backward_graph_function is None:
-          self._construct_backprop_function()
-        forward_function = self._forward_function
-        backward_function = self._backward_graph_function._inference_function
-        # pylint: enable=protected-access
-        forward_function.add_to_graph(g)
-        backward_function.add_to_graph(g)
+    if not context.executing_eagerly() and not g:
+      g = ops.get_default_graph()
+    self._inference_function.add_to_graph(g)  # pylint: disable=protected-access
+
+    # pylint: disable=protected-access
+    if register_gradient_functions:
+      # There are two situations for the actual call of a defun:
+      # 1. If none of the input args are resource variables or watch by any
+      #   tape, and it will run the _inference_function of concrete_func for
+      #   forward pass, the gradient will be generated by standard mechanism.
+      # 2. Otherwise, defun will create two functions, one for forward pass,
+      #   and the backward pass will be created via tape.
+      #   When registering the function, we register both cases.
+      if self._backward_graph_function is None:
+        self._construct_backprop_function()
+      forward_function = self._forward_function
+      backward_function = self._backward_graph_function._inference_function
+      # pylint: enable=protected-access
+      forward_function.add_to_graph(g)
+      backward_function.add_to_graph(g)
 
   def _construct_backprop_function(self):
     """Constructs the backprop function object for this function."""
diff --git a/tensorflow/python/saved_model/function_deserialization.py b/tensorflow/python/saved_model/function_deserialization.py
index 4ab85ec8a0..5479bbb355 100644
--- a/tensorflow/python/saved_model/function_deserialization.py
+++ b/tensorflow/python/saved_model/function_deserialization.py
@@ -71,6 +71,24 @@ def _deserialize_function_spec(function_spec_proto, coder):
                                    kwargs_to_include, input_signature)
 
 
+def recreate_concrete_function(saved_concrete_function, concrete_functions):
+  """Recreates a user-facing concrete function."""
+  concrete_function = concrete_functions[saved_concrete_function.name]
+  input_signature = (saved_concrete_function.canonicalized_input_signature
+                     .list_value.values)
+  # pylint: disable=protected-access
+  # Set metadata required for the concrete function to accept keyword and
+  # positional arguments in __call__. Normally this is set in
+  # get_concrete_function.
+  concrete_function._arg_keywords = [
+      spec.tensor_spec_value.name for spec in input_signature]
+  # TODO(allenl): Should we preserve the number of allowed positional arguments?
+  concrete_function._num_positional_args = len(input_signature)
+  # pylint: enable=protected-access
+  concrete_function.add_to_graph()
+  return concrete_function
+
+
 def recreate_function(saved_function, concrete_functions):
   """Creates a `Function` from a `SavedFunction`.
 
diff --git a/tensorflow/python/saved_model/function_serialization.py b/tensorflow/python/saved_model/function_serialization.py
index d973f97d42..c79c61b60c 100644
--- a/tensorflow/python/saved_model/function_serialization.py
+++ b/tensorflow/python/saved_model/function_serialization.py
@@ -19,6 +19,7 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.framework import func_graph as func_graph_module
+from tensorflow.python.framework import tensor_spec
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.saved_model import nested_structure_coder
 from tensorflow.python.saved_model import saved_object_graph_pb2
@@ -38,6 +39,46 @@ def _serialize_function_spec(function_spec, coder):
   return proto
 
 
+def _serialize_concrete_function_with_signature(
+    concrete_function, node_ids, signature_args, coder):
+  """Build a SavedConcreteFunction."""
+  bound_inputs = []
+  try:
+    for capture in concrete_function.captured_inputs:
+      bound_inputs.append(node_ids[capture])
+  except KeyError:
+    # TODO(andresp): Would it better to throw an exception?
+    logging.warning(
+        "Concrete function %s not added to object based saved model as it "
+        "captures tensor %s which is unsupported or not reachable from root.",
+        concrete_function.name, capture)
+    return None
+  concrete_function_proto = saved_object_graph_pb2.SavedConcreteFunction()
+  concrete_function_proto.name = concrete_function.name
+  concrete_function_proto.canonicalized_input_signature.CopyFrom(
+      coder.encode_structure(signature_args))
+  structured_outputs = func_graph_module.convert_structure_to_signature(
+      concrete_function.structured_outputs)
+  concrete_function_proto.output_signature.CopyFrom(
+      coder.encode_structure(structured_outputs))
+  concrete_function_proto.bound_inputs.extend(bound_inputs)
+  return concrete_function_proto
+
+
+def serialize_concrete_function(concrete_function, node_ids):
+  """Build a standalone SavedConcreteFunction."""
+  coder = nested_structure_coder.StructureCoder()
+  signature = []
+  # Construct a flat list of TensorSpecs for this concrete function. On load
+  # we'll parse them and recreate the call metadata.
+  for input_tensor, input_keyword in zip(
+      concrete_function.inputs, concrete_function._arg_keywords):  # pylint: disable=protected-access
+    signature.append(tensor_spec.TensorSpec(
+        input_tensor.shape, input_tensor.dtype, name=input_keyword))
+  return _serialize_concrete_function_with_signature(
+      concrete_function, node_ids, signature, coder)
+
+
 def serialize_function(function, node_ids):
   """Build a SavedFunction proto."""
   coder = nested_structure_coder.StructureCoder()
@@ -48,27 +89,11 @@ def serialize_function(function, node_ids):
   all_concrete_functions = \
       function._list_all_concrete_functions_for_serialization()  # pylint: disable=protected-access
   for concrete_function in all_concrete_functions:
-    bound_inputs = []
-    try:
-      for capture in concrete_function.captured_inputs:
-        bound_inputs.append(node_ids[capture])
-    except KeyError:
-      # TODO(andresp): Would it better to throw an exception?
-      logging.warning(
-          "Concrete function %s not added to object based saved model as it "
-          "captures tensor %s which is unsupported or not reachable from root.",
-          concrete_function.name, capture)
-      continue
     signature_args, signature_kwargs = \
         concrete_function.structured_input_signature
     del signature_kwargs
-    concrete_function_proto = proto.concrete_function.add()
-    concrete_function_proto.name = concrete_function.name
-    concrete_function_proto.canonicalized_input_signature.CopyFrom(
-        coder.encode_structure(signature_args))
-    structured_outputs = func_graph_module.convert_structure_to_signature(
-        concrete_function.structured_outputs)
-    concrete_function_proto.output_signature.CopyFrom(
-        coder.encode_structure(structured_outputs))
-    concrete_function_proto.bound_inputs.extend(bound_inputs)
+    concrete_function_proto = _serialize_concrete_function_with_signature(
+        concrete_function, node_ids, signature_args, coder)
+    if concrete_function_proto is not None:
+      proto.concrete_function.add().CopyFrom(concrete_function_proto)
   return proto
diff --git a/tensorflow/python/saved_model/load.py b/tensorflow/python/saved_model/load.py
index 98990905f0..2aa4924b29 100644
--- a/tensorflow/python/saved_model/load.py
+++ b/tensorflow/python/saved_model/load.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import functools
 import os
 
 from tensorflow.python.lib.io import file_io
@@ -48,53 +49,62 @@ class _Loader(object):
         function_deserialization.load_function_def_library(
             meta_graph.graph_def.library))
     self._load_all()
-    self._setup_concrete_functions()
+    self._setup_functions()
     self._restore_checkpoint()
 
-  def _setup_concrete_functions(self):
+  def _setup_concrete_function(
+      self, concrete_function, seen_functions, coder):
+    """Setup captured tensors and outputs for a single concrete function."""
+    name = concrete_function.name
+    bound_inputs = [
+        self._get_tensor_from_node(node_id)
+        for node_id in concrete_function.bound_inputs]
+    bound_variables = [
+        self._nodes[node_id]
+        for node_id in concrete_function.bound_inputs
+        if self._proto.nodes[node_id].WhichOneof("kind") == "variable"
+    ]
+    if name in seen_functions:
+      raise RuntimeError(
+          "Concrete function with a duplicate name: %s." % name)
+    else:
+      seen_functions.add(name)
+      # TODO(andresp): This is only injecting the captured inputs into the
+      # concrete function, note that we did not modify the FuncGraph
+      # itself.
+      function = self._concrete_functions[name]
+      function._captured_inputs = bound_inputs  # pylint: disable=protected-access
+      function._func_graph.variables = bound_variables  # pylint: disable=protected-access
+      # By setting the structured_outputs directly, we can rely on this
+      # function_lib.ConcreteFunction object to perform the output repacking
+      # logic. The only limitation of that logic is that it only works
+      # with output that is convertible to Tensors and the conversion
+      # always happens. For example tf.TensorShape([2, 3]) will be
+      # converted to Tensor representing [2, 3].
+      original_outputs = coder.decode_proto(
+          concrete_function.output_signature)
+      # The original_outputs here had Tensors converted to TensorSpecs, so
+      # the restored function's structured_outputs field will not be
+      # exactly the same. Fortunately the repacking logic cares only about
+      # the structure.
+      # TODO(vbardiovsky): Should we just replicate the structures, with
+      # Nones instead of real objects? Decide when we start solving
+      # idempotency.
+      function._func_graph.structured_outputs = original_outputs  # pylint: disable=protected-access
+
+  def _setup_functions(self):
+
     """Setup captures and output structure in restored concrete functions."""
     seen_concrete_functions = set()
     coder = nested_structure_coder.StructureCoder()
     for object_proto in self._proto.nodes:
-      if object_proto.WhichOneof("kind") == "function":
+      if object_proto.WhichOneof("kind") == "concrete_function":
+        self._setup_concrete_function(
+            object_proto.concrete_function, seen_concrete_functions, coder)
+      elif object_proto.WhichOneof("kind") == "function":
         for concrete_function in object_proto.function.concrete_function:
-          name = concrete_function.name
-          bound_inputs = [
-              self._get_tensor_from_node(node_id)
-              for node_id in concrete_function.bound_inputs
-          ]
-          bound_variables = [
-              self._nodes[node_id]
-              for node_id in concrete_function.bound_inputs
-              if self._proto.nodes[node_id].WhichOneof("kind") == "variable"
-          ]
-          if name in seen_concrete_functions:
-            raise RuntimeError(
-                "Concrete function with a duplicate name: %s." % name)
-          else:
-            seen_concrete_functions.add(name)
-            # TODO(andresp): This is only injecting the captured inputs into the
-            # concrete function, note that we did not modify the FuncGraph
-            # itself.
-            revived_function = self._concrete_functions[name]
-            revived_function._captured_inputs = bound_inputs  # pylint: disable=protected-access
-            revived_function._func_graph.variables = bound_variables  # pylint: disable=protected-access
-            # By setting the structured_outputs directly, we can rely on this
-            # function_lib.ConcreteFunction object to perform the output
-            # repacking logic. The only limitation of that logic is that it only
-            # works with output that is convertible to Tensors and the
-            # conversion always happens. For example tf.TensorShape([2, 3])
-            # will be converted to Tensor representing [2, 3].
-            original_outputs = coder.decode_proto(
-                concrete_function.output_signature)
-            # The original_outputs here had Tensors converted to TensorSpecs, so
-            # the restored concrete function's structured_outputs field will not
-            # be exactly the same. Fortunately the repacking logic cares only
-            # about the structure.
-            # TODO(vbardiovsky): Should we just replicate the structures, with
-            # Nones instead of real objects? Decide when we start solving
-            # idempotency.
-            revived_function._func_graph.structured_outputs = original_outputs  # pylint: disable=protected-access
+          self._setup_concrete_function(
+              concrete_function, seen_concrete_functions, coder)
 
   def _get_tensor_from_node(self, node_id):
     obj = self._nodes[node_id]
@@ -137,6 +147,8 @@ class _Loader(object):
         "user_object": lambda: self._recreate_user_object(proto.user_object),
         "asset": lambda: self._recreate_asset(proto.asset),
         "function": lambda: self._recreate_function(proto.function),
+        "concrete_function": functools.partial(
+            self._recreate_concrete_function, proto.concrete_function),
         "variable": lambda: self._recreate_variable(proto.variable),
     }
     kind = proto.WhichOneof("kind")
@@ -168,6 +180,10 @@ class _Loader(object):
     return function_deserialization.recreate_function(
         proto, self._concrete_functions), setattr
 
+  def _recreate_concrete_function(self, proto):
+    return function_deserialization.recreate_concrete_function(
+        proto, self._concrete_functions), setattr
+
   def _recreate_variable(self, proto):
     # TODO(andresp): Can we use the checkpointed value as initializer?
     dummy_value = init_ops.Zeros(dtype=proto.dtype)(shape=proto.shape)
diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index c8c8f6e1b3..e3dcbda50a 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -418,6 +418,57 @@ class LoadTest(test.TestCase):
     self.assertAllEqual([2, 4, 6],
                         imported.f(constant_op.constant([1, 2, 3])).numpy())
 
+  def test_concrete_function(self):
+
+    @def_function.function(
+        input_signature=[tensor_spec.TensorSpec([None], dtypes.int32)])
+    def func(x):
+      return 2 * x
+
+    root = tracking.AutoCheckpointable()
+    root.f = func.get_concrete_function()
+
+    self.assertAllEqual([2], root.f(constant_op.constant([1])).numpy())
+    self.assertAllEqual([2, 4], root.f(constant_op.constant([1, 2])).numpy())
+
+    imported = self.cycle(root)
+
+    self.assertAllEqual([2, 4, 6, 8],
+                        imported.f(constant_op.constant([1, 2, 3, 4])).numpy())
+    self.assertAllEqual([2, 4, 6],
+                        imported.f(constant_op.constant([1, 2, 3])).numpy())
+
+  def test_concrete_function_no_signature(self):
+    @def_function.function
+    def func(x):
+      return 2 * x
+
+    root = tracking.AutoCheckpointable()
+    root.f = func.get_concrete_function(constant_op.constant([1]))
+    self.assertAllEqual([4], root.f(constant_op.constant([2])).numpy())
+    imported = self.cycle(root)
+    self.assertAllEqual([6],
+                        imported.f(constant_op.constant([3])).numpy())
+
+  def test_concrete_function_backprop(self):
+    @def_function.function(
+        input_signature=[tensor_spec.TensorSpec([None], dtypes.float32)])
+    def func(x):
+      return x ** 2.
+    root = tracking.AutoCheckpointable()
+    root.f = func.get_concrete_function()
+
+    def _compute_gradient(function):
+      with backprop.GradientTape() as tape:
+        inp = constant_op.constant(1.)
+        tape.watch(inp)
+        output = function(inp)
+      return tape.gradient(output, inp)
+
+    self.assertEqual(2., _compute_gradient(root.f).numpy())
+    imported = self.cycle(root)
+    self.assertEqual(2., _compute_gradient(imported.f).numpy())
+
   def test_dict(self):
     root = tracking.AutoCheckpointable()
     root.variables = dict(a=variables.Variable(1.))
diff --git a/tensorflow/python/saved_model/save.py b/tensorflow/python/saved_model/save.py
index c35e09457f..9c3ace7325 100644
--- a/tensorflow/python/saved_model/save.py
+++ b/tensorflow/python/saved_model/save.py
@@ -73,18 +73,24 @@ class _SaveableView(object):
     self.functions = util.ObjectIdentityDictionary()
 
     # Also add `Function`s as nodes.
-    for obj in self.nodes:
+    nodes_without_functions = list(self.nodes)
+    for obj in nodes_without_functions:
       self.functions[obj] = self._list_functions(obj)
       for function in self.functions[obj].values():
         if function not in self.node_ids:
           self.node_ids[function] = len(self.nodes)
           self.nodes.append(function)
-        # Force listing the concrete functions for the side effects:
-        #  - populate the cache for `Function`s that have an
-        #  input_signature and have not been called.
-        #  - force side effects of creation of concrete functions, e.g. create
-        #  variables on first run.
-        function._list_all_concrete_functions_for_serialization()  # pylint: disable=protected-access
+          # Avoids recursing into functions to see if other functions are
+          # assigned to attributes. This is sometimes true for concrete
+          # functions but not helpful.
+          self.functions[function] = {}
+        if isinstance(function, def_function.Function):
+          # Force listing the concrete functions for the side effects:
+          #  - populate the cache for functions that have an input_signature
+          #  and have not been called.
+          #  - force side effects of creation of concrete functions, e.g. create
+          #  variables on first run.
+          function._list_all_concrete_functions_for_serialization()  # pylint: disable=protected-access
 
   @property
   def root(self):
@@ -96,7 +102,7 @@ class _SaveableView(object):
       assert self.node_ids[node] == node_id
       object_proto = proto.nodes.add()
       object_proto.slot_variables.extend(self.slot_variables.get(node, ()))
-      if isinstance(node, def_function.Function):
+      if isinstance(node, (def_function.Function, defun.ConcreteFunction)):
         continue
       for child in node._checkpoint_dependencies:  # pylint: disable=protected-access
         child_proto = object_proto.children.add()
@@ -117,7 +123,8 @@ class _SaveableView(object):
         # We really don't want to throw an exception just because some object's
         # attribute accessor is broken.
         attribute_value = None
-      if isinstance(attribute_value, def_function.Function):
+      if isinstance(attribute_value, (def_function.Function,
+                                      defun.ConcreteFunction)):
         functions[attribute_name] = attribute_value
     return functions
 
@@ -554,8 +561,11 @@ def _fill_meta_graph_def(meta_graph_def, saveable_view, signature_functions,
   concrete_functions = []
   for obj in accessible_objects:
     for function in saveable_view.functions[obj].values():
-      concrete_functions.extend(
-          function._list_all_concrete_functions_for_serialization())  # pylint: disable=protected-access
+      if isinstance(function, defun.ConcreteFunction):
+        concrete_functions.append(function)
+      else:
+        concrete_functions.extend(
+            function._list_all_concrete_functions_for_serialization())  # pylint: disable=protected-access
 
   with exported_graph.as_default():
     signatures = _generate_signatures(signature_functions, resource_map)
@@ -617,6 +627,9 @@ def _write_object_proto(obj, proto, asset_file_def_index, node_ids):
   elif isinstance(obj, def_function.Function):
     proto.function.CopyFrom(
         function_serialization.serialize_function(obj, node_ids))
+  elif isinstance(obj, defun.ConcreteFunction):
+    proto.concrete_function.CopyFrom(
+        function_serialization.serialize_concrete_function(obj, node_ids))
   else:
     registered_type_proto = revived_types.serialize(obj)
     if registered_type_proto is None:
diff --git a/tensorflow/python/saved_model/saved_object_graph.proto b/tensorflow/python/saved_model/saved_object_graph.proto
index 38940f4c1c..6f5a952083 100644
--- a/tensorflow/python/saved_model/saved_object_graph.proto
+++ b/tensorflow/python/saved_model/saved_object_graph.proto
@@ -54,6 +54,7 @@ message SavedObject {
     SavedAsset asset = 5;
     SavedFunction function = 6;
     SavedVariable variable = 7;
+    SavedConcreteFunction concrete_function = 8;
   }
 }
 
-- 
GitLab


From 6dae9a68f0e6e902aaf8811165b8f70d2cc3624a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 16 Jan 2019 18:23:04 -0800
Subject: [PATCH 0858/1095] Go: Update generated wrapper functions for
 TensorFlow ops. PiperOrigin-RevId: 229663723

---
 tensorflow/go/op/wrappers.go | 126 +++++++++++++++++------------------
 1 file changed, 63 insertions(+), 63 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index fbabb37c0e..134185a03b 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -18447,69 +18447,6 @@ func Timestamp(scope *Scope) (ts tf.Output) {
 	return op.Output(0)
 }
 
-// ResourceSparseApplyKerasMomentumAttr is an optional argument to ResourceSparseApplyKerasMomentum.
-type ResourceSparseApplyKerasMomentumAttr func(optionalAttr)
-
-// ResourceSparseApplyKerasMomentumUseLocking sets the optional use_locking attribute to value.
-//
-// value: If `True`, updating of the var and accum tensors will be protected
-// by a lock; otherwise the behavior is undefined, but may exhibit less
-// contention.
-// If not specified, defaults to false
-func ResourceSparseApplyKerasMomentumUseLocking(value bool) ResourceSparseApplyKerasMomentumAttr {
-	return func(m optionalAttr) {
-		m["use_locking"] = value
-	}
-}
-
-// ResourceSparseApplyKerasMomentumUseNesterov sets the optional use_nesterov attribute to value.
-//
-// value: If `True`, the tensor passed to compute grad will be
-// var + momentum * accum, so in the end, the var you get is actually
-// var + momentum * accum.
-// If not specified, defaults to false
-func ResourceSparseApplyKerasMomentumUseNesterov(value bool) ResourceSparseApplyKerasMomentumAttr {
-	return func(m optionalAttr) {
-		m["use_nesterov"] = value
-	}
-}
-
-// Update relevant entries in '*var' and '*accum' according to the momentum scheme.
-//
-// Set use_nesterov = True if you want to use Nesterov momentum.
-//
-// That is for rows we have grad for, we update var and accum as follows:
-//
-// accum = accum * momentum - lr * grad
-// var += accum
-//
-// Arguments:
-//	var_: Should be from a Variable().
-//	accum: Should be from a Variable().
-//	lr: Learning rate. Must be a scalar.
-//	grad: The gradient.
-//	indices: A vector of indices into the first dimension of var and accum.
-//	momentum: Momentum. Must be a scalar.
-//
-// Returns the created operation.
-func ResourceSparseApplyKerasMomentum(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.Output, grad tf.Output, indices tf.Output, momentum tf.Output, optional ...ResourceSparseApplyKerasMomentumAttr) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "ResourceSparseApplyKerasMomentum",
-		Input: []tf.Input{
-			var_, accum, lr, grad, indices, momentum,
-		},
-		Attrs: attrs,
-	}
-	return scope.AddOperation(opspec)
-}
-
 // VariableShapeAttr is an optional argument to VariableShape.
 type VariableShapeAttr func(optionalAttr)
 
@@ -23474,6 +23411,69 @@ func TakeManySparseFromTensorsMap(scope *Scope, sparse_handles tf.Output, dtype
 	return op.Output(0), op.Output(1), op.Output(2)
 }
 
+// ResourceSparseApplyKerasMomentumAttr is an optional argument to ResourceSparseApplyKerasMomentum.
+type ResourceSparseApplyKerasMomentumAttr func(optionalAttr)
+
+// ResourceSparseApplyKerasMomentumUseLocking sets the optional use_locking attribute to value.
+//
+// value: If `True`, updating of the var and accum tensors will be protected
+// by a lock; otherwise the behavior is undefined, but may exhibit less
+// contention.
+// If not specified, defaults to false
+func ResourceSparseApplyKerasMomentumUseLocking(value bool) ResourceSparseApplyKerasMomentumAttr {
+	return func(m optionalAttr) {
+		m["use_locking"] = value
+	}
+}
+
+// ResourceSparseApplyKerasMomentumUseNesterov sets the optional use_nesterov attribute to value.
+//
+// value: If `True`, the tensor passed to compute grad will be
+// var + momentum * accum, so in the end, the var you get is actually
+// var + momentum * accum.
+// If not specified, defaults to false
+func ResourceSparseApplyKerasMomentumUseNesterov(value bool) ResourceSparseApplyKerasMomentumAttr {
+	return func(m optionalAttr) {
+		m["use_nesterov"] = value
+	}
+}
+
+// Update relevant entries in '*var' and '*accum' according to the momentum scheme.
+//
+// Set use_nesterov = True if you want to use Nesterov momentum.
+//
+// That is for rows we have grad for, we update var and accum as follows:
+//
+// accum = accum * momentum - lr * grad
+// var += accum
+//
+// Arguments:
+//	var_: Should be from a Variable().
+//	accum: Should be from a Variable().
+//	lr: Learning rate. Must be a scalar.
+//	grad: The gradient.
+//	indices: A vector of indices into the first dimension of var and accum.
+//	momentum: Momentum. Must be a scalar.
+//
+// Returns the created operation.
+func ResourceSparseApplyKerasMomentum(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.Output, grad tf.Output, indices tf.Output, momentum tf.Output, optional ...ResourceSparseApplyKerasMomentumAttr) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ResourceSparseApplyKerasMomentum",
+		Input: []tf.Input{
+			var_, accum, lr, grad, indices, momentum,
+		},
+		Attrs: attrs,
+	}
+	return scope.AddOperation(opspec)
+}
+
 // Assigns a new value to a variable.
 //
 // Any ReadVariableOp with a control dependency on this op is guaranteed to return
-- 
GitLab


From f960af59b7d86ed5ece2c604f0400ef4f3afb4b2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 16 Jan 2019 18:24:53 -0800
Subject: [PATCH 0859/1095] Update ops-related pbtxt files.

PiperOrigin-RevId: 229663932
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 58 +++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 | 58 +++++++++++++++++++
 2 files changed, 116 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 9990b8629c..3b7a7b812a 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -22275,6 +22275,28 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "ExperimentalChooseFastestDataset"
+  input_arg {
+    name: "input_datasets"
+    type: DT_VARIANT
+    number_attr: "N"
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "N"
+    type: "int"
+    has_minimum: true
+    minimum: 2
+  }
+  attr {
+    name: "num_experiments"
+    type: "int"
+  }
+}
 op {
   name: "ExperimentalDatasetCardinality"
   input_arg {
@@ -23647,6 +23669,42 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "ExperimentalTakeWhileDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "other_arguments"
+    type_list_attr: "Targuments"
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "predicate"
+    type: "func"
+  }
+  attr {
+    name: "Targuments"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
 op {
   name: "ExperimentalThreadPoolDataset"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 8212282623..92a7bb9a34 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -10435,6 +10435,28 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "ExperimentalChooseFastestDataset"
+  input_arg {
+    name: "input_datasets"
+    type: DT_VARIANT
+    number_attr: "N"
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "N"
+    type: "int"
+    has_minimum: true
+    minimum: 2
+  }
+  attr {
+    name: "num_experiments"
+    type: "int"
+  }
+}
 op {
   name: "ExperimentalDatasetCardinality"
   input_arg {
@@ -11457,6 +11479,42 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "ExperimentalTakeWhileDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "other_arguments"
+    type_list_attr: "Targuments"
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "predicate"
+    type: "func"
+  }
+  attr {
+    name: "Targuments"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
 op {
   name: "ExperimentalThreadPoolDataset"
   input_arg {
-- 
GitLab


From 52af2770f94f0f660e4019d95821052c08b18a41 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 16 Jan 2019 20:07:31 -0800
Subject: [PATCH 0860/1095] Removing unnecessary parameters.

PiperOrigin-RevId: 229673560
---
 third_party/gpus/cuda_configure.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl
index f8f3e90708..1988c4ae96 100644
--- a/third_party/gpus/cuda_configure.bzl
+++ b/third_party/gpus/cuda_configure.bzl
@@ -1067,10 +1067,10 @@ def _create_dummy_repository(repository_ctx):
 
   # Create dummy files for the CUDA toolkit since they are still required by
   # tensorflow/core/platform/default/build_config:cuda.
-  repository_ctx.file("cuda/cuda/include/cuda.h", "")
-  repository_ctx.file("cuda/cuda/include/cublas.h", "")
-  repository_ctx.file("cuda/cuda/include/cudnn.h", "")
-  repository_ctx.file("cuda/cuda/extras/CUPTI/include/cupti.h", "")
+  repository_ctx.file("cuda/cuda/include/cuda.h")
+  repository_ctx.file("cuda/cuda/include/cublas.h")
+  repository_ctx.file("cuda/cuda/include/cudnn.h")
+  repository_ctx.file("cuda/cuda/extras/CUPTI/include/cupti.h")
   repository_ctx.file("cuda/cuda/lib/%s" % _lib_name("cuda", cpu_value))
   repository_ctx.file("cuda/cuda/lib/%s" % _lib_name("cudart", cpu_value))
   repository_ctx.file(
-- 
GitLab


From 77e2ba9188f4343214adc30de63a98d4999103f7 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 16 Jan 2019 20:45:42 -0800
Subject: [PATCH 0861/1095] Go: Update generated wrapper functions for
 TensorFlow ops. PiperOrigin-RevId: 229676560

---
 tensorflow/go/op/wrappers.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index 134185a03b..666ebe33df 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -13421,7 +13421,7 @@ func StaticRegexReplaceReplaceGlobal(value bool) StaticRegexReplaceAttr {
 // Arguments:
 //	input: The text to be processed.
 //	pattern: The regular expression to match the input.
-//	rewrite: The rewrite to be applied to the matched expresion.
+//	rewrite: The rewrite to be applied to the matched expression.
 //
 // Returns The text after applying pattern and rewrite.
 func StaticRegexReplace(scope *Scope, input tf.Output, pattern string, rewrite string, optional ...StaticRegexReplaceAttr) (output tf.Output) {
-- 
GitLab


From 69b9e5358b7e64efd52f742847eeaf5f893762ab Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 16 Jan 2019 22:02:05 -0800
Subject: [PATCH 0862/1095] Remove HostBuffers

HostBuffers add significant complexity and have no users.

PiperOrigin-RevId: 229682740
---
 tensorflow/stream_executor/BUILD         |  7 ----
 tensorflow/stream_executor/dnn.h         | 16 ---------
 tensorflow/stream_executor/host_buffer.h | 46 ------------------------
 tensorflow/stream_executor/stream.cc     | 33 -----------------
 4 files changed, 102 deletions(-)
 delete mode 100644 tensorflow/stream_executor/host_buffer.h

diff --git a/tensorflow/stream_executor/BUILD b/tensorflow/stream_executor/BUILD
index 7cf7e75a55..fca28e016a 100644
--- a/tensorflow/stream_executor/BUILD
+++ b/tensorflow/stream_executor/BUILD
@@ -206,7 +206,6 @@ cc_library(
 cc_library(
     name = "stream",
     srcs = [
-        "host_buffer.h",
         "stream.cc",
     ],
     hdrs = ["stream.h"],
@@ -453,12 +452,6 @@ cc_library(
     ],
 )
 
-cc_library(
-    name = "host_buffer",
-    hdrs = ["host_buffer.h"],
-    deps = [":dnn"],
-)
-
 tf_proto_library(
     name = "dnn_proto",
     srcs = ["dnn.proto"],
diff --git a/tensorflow/stream_executor/dnn.h b/tensorflow/stream_executor/dnn.h
index f5eef29109..f60d1ada24 100644
--- a/tensorflow/stream_executor/dnn.h
+++ b/tensorflow/stream_executor/dnn.h
@@ -2069,22 +2069,6 @@ class DnnSupport {
       QuantizedActivationMode mode,
       DeviceMemory<float>* gpu_unquantized_dst) = 0;
 
-  // Enqueues an asynchronous copy of the contents of buffer_src to
-  // gpu_unquantized_dst.
-  virtual bool DoCopyHostBuffer2Device(
-      Stream* stream, HostBuffer* buffer_src,
-      DeviceMemory<float>* gpu_unquantized_dst) {
-    return false;
-  }
-
-  // Enqueues an asynchronous copy of the contents of gpu_unquantized_src to
-  // buffer_dst.
-  virtual bool DoCopyDevice2HostBuffer(
-      Stream* stream, const DeviceMemory<float>& gpu_unquantized_src,
-      HostBuffer* buffer_dst) {
-    return false;
-  }
-
   // Create an RNN descriptor based on model shapes and configurations.
   // The caller retains the ownership of the descriptor.
   //
diff --git a/tensorflow/stream_executor/host_buffer.h b/tensorflow/stream_executor/host_buffer.h
deleted file mode 100644
index 20299da517..0000000000
--- a/tensorflow/stream_executor/host_buffer.h
+++ /dev/null
@@ -1,46 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#ifndef TENSORFLOW_STREAM_EXECUTOR_HOST_BUFFER_H_
-#define TENSORFLOW_STREAM_EXECUTOR_HOST_BUFFER_H_
-
-#include "tensorflow/stream_executor/dnn.h"
-
-namespace stream_executor {
-
-// A HostBuffer is a block of memory in host memory containing the data for a
-// dnn::BatchDescriptor using a device-dependent memory layout.
-// Derived classes provide methods to construct a HostBuffer for a specific
-// device, and to copy data in and out of the buffer.
-class HostBuffer {
- public:
-  const dnn::BatchDescriptor& descriptor() const { return descriptor_; }
-
-  // Returns a string describing the HostBuffer.
-  virtual string AsString() const = 0;
-
- protected:
-  // Construct a HostBuffer from the supplied dnn::BatchDescriptor.
-  explicit HostBuffer(const dnn::BatchDescriptor& descriptor)
-      : descriptor_(descriptor) {}
-  virtual ~HostBuffer() {}
-
- private:
-  const dnn::BatchDescriptor descriptor_;
-};
-
-}  // namespace stream_executor
-
-#endif  // TENSORFLOW_STREAM_EXECUTOR_HOST_BUFFER_H_
diff --git a/tensorflow/stream_executor/stream.cc b/tensorflow/stream_executor/stream.cc
index 0dceacb228..e7485ca426 100644
--- a/tensorflow/stream_executor/stream.cc
+++ b/tensorflow/stream_executor/stream.cc
@@ -20,7 +20,6 @@ limitations under the License.
 #include "absl/strings/str_cat.h"
 #include "third_party/eigen3/Eigen/Core"
 #include "tensorflow/stream_executor/blas.h"
-#include "tensorflow/stream_executor/host_buffer.h"
 #include "tensorflow/stream_executor/host_or_device_scalar.h"
 #include "tensorflow/stream_executor/lib/stacktrace.h"
 #include "tensorflow/stream_executor/platform.h"
@@ -95,8 +94,6 @@ string ToVlogString(const void *ptr) {
   return out.str();
 }
 
-string ToVlogString(const HostBuffer &buffer) { return buffer.AsString(); }
-
 template <class T>
 string ToVlogString(const std::complex<T> &c) {
   // StrCat does not convert std::complex to text.
@@ -2103,36 +2100,6 @@ Stream &Stream::ThenMemcpyH2DQuantized(
   return *this;
 }
 
-Stream &Stream::ThenCopyHostBuffer2Device(
-    HostBuffer *buffer_src, DeviceMemory<float> *gpu_unquantized_dst) {
-  VLOG_CALL(PARAM(*buffer_src), PARAM(gpu_unquantized_dst));
-
-  if (ok()) {
-    if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      CheckError(
-          dnn->DoCopyHostBuffer2Device(this, buffer_src, gpu_unquantized_dst));
-    } else {
-      SetErrorAndLogNoDnnSupport();
-    }
-  }
-  return *this;
-}
-
-Stream &Stream::ThenCopyDevice2HostBuffer(
-    const DeviceMemory<float> &gpu_unquantized_src, HostBuffer *buffer_dst) {
-  VLOG_CALL(PARAM(gpu_unquantized_src), PARAM(*buffer_dst));
-
-  if (ok()) {
-    if (dnn::DnnSupport *dnn = parent_->AsDnn()) {
-      CheckError(
-          dnn->DoCopyDevice2HostBuffer(this, gpu_unquantized_src, buffer_dst));
-    } else {
-      SetErrorAndLogNoDnnSupport();
-    }
-  }
-  return *this;
-}
-
 Stream *Stream::GetOrCreateSubStream() {
   mutex_lock lock(mu_);
 
-- 
GitLab


From 8eb3cbcb423c4d840e88a910c5db47404207b8a4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 16 Jan 2019 23:25:53 -0800
Subject: [PATCH 0863/1095] Fix typo.

PiperOrigin-RevId: 229689427
---
 tensorflow/compiler/xla/g3doc/broadcasting.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/xla/g3doc/broadcasting.md b/tensorflow/compiler/xla/g3doc/broadcasting.md
index 2870869a2c..5c0525c1e9 100644
--- a/tensorflow/compiler/xla/g3doc/broadcasting.md
+++ b/tensorflow/compiler/xla/g3doc/broadcasting.md
@@ -168,7 +168,7 @@ consult the
 
 Broadcasting of a lower-rank array to a higher-rank array **and** broadcasting
 using degenerate dimensions can both be performed in the same binary operation.
-For example, a vector of size 4 and an matrix of size 1x2 can be added together
+For example, a vector of size 4 and a matrix of size 1x2 can be added together
 using broadcast dimensions value of (0):
 
     |1 2 3 4| + [5 6]    // [5 6] is a 1x2 matrix, not a vector.
@@ -176,7 +176,7 @@ using broadcast dimensions value of (0):
 First the vector is broadcast up to rank 2 (matrix) using the broadcast
 dimensions. The single value (0) in the broadcast dimensions indicates that
 dimension zero of the vector matches to dimension zero of the matrix. This
-produces an matrix of size 4xM where the value M is chosen to match the
+produces a matrix of size 4xM where the value M is chosen to match the
 corresponding dimension size in the 1x2 array. Therefore, a 4x2 matrix is
 produced:
 
-- 
GitLab


From ca2d0b652e1bc8e6d637318a1fbcb56bd4e6f0d0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 17 Jan 2019 01:03:09 -0800
Subject: [PATCH 0864/1095] compat: Update forward compatibility horizon to
 2019-01-17

PiperOrigin-RevId: 229698933
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 773b3a41f9..c558722061 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 16)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 17)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 9331096d56002c7fcb8bae411684b8b78fc196c4 Mon Sep 17 00:00:00 2001
From: Tom Hennigan <tomhennigan@google.com>
Date: Thu, 17 Jan 2019 06:29:06 -0800
Subject: [PATCH 0865/1095] Experimental implementation of tf.Module.

PiperOrigin-RevId: 229735738
---
 tensorflow/python/BUILD                       |   1 +
 tensorflow/python/__init__.py                 |   1 +
 tensorflow/python/module/BUILD                |  30 ++
 tensorflow/python/module/module.py            | 402 ++++++++++++++++++
 tensorflow/python/module/module_test.py       | 355 ++++++++++++++++
 .../v1/tensorflow.experimental.-module.pbtxt  |  47 ++
 .../golden/v1/tensorflow.experimental.pbtxt   |   4 +
 .../v2/tensorflow.experimental.-module.pbtxt  |  47 ++
 .../golden/v2/tensorflow.experimental.pbtxt   |   4 +
 9 files changed, 891 insertions(+)
 create mode 100644 tensorflow/python/module/BUILD
 create mode 100644 tensorflow/python/module/module.py
 create mode 100644 tensorflow/python/module/module_test.py
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.experimental.-module.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.experimental.-module.pbtxt

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index e32be0039f..b61c33f319 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -178,6 +178,7 @@ py_library(
         "//tensorflow/python/eager:def_function",
         "//tensorflow/python/eager:profiler",
         "//tensorflow/python/eager:remote",
+        "//tensorflow/python/module",
         "//tensorflow/python/ops/distributions",
         "//tensorflow/python/ops/linalg",
         "//tensorflow/python/ops/losses",
diff --git a/tensorflow/python/__init__.py b/tensorflow/python/__init__.py
index 5172dbb836..398fb375e1 100644
--- a/tensorflow/python/__init__.py
+++ b/tensorflow/python/__init__.py
@@ -82,6 +82,7 @@ from tensorflow.python import distribute
 from tensorflow.python import keras
 from tensorflow.python.feature_column import feature_column_lib as feature_column
 from tensorflow.python.layers import layers
+from tensorflow.python.module import module
 from tensorflow.python.ops import bitwise_ops as bitwise
 from tensorflow.python.ops import image_ops as image
 from tensorflow.python.ops import manip_ops as manip
diff --git a/tensorflow/python/module/BUILD b/tensorflow/python/module/BUILD
new file mode 100644
index 0000000000..64c226e4c9
--- /dev/null
+++ b/tensorflow/python/module/BUILD
@@ -0,0 +1,30 @@
+package(default_visibility = ["//tensorflow:internal"])
+
+load("//tensorflow:tensorflow.bzl", "tf_py_test")
+
+licenses(["notice"])  # Apache 2.0
+
+exports_files(["LICENSE"])
+
+py_library(
+    name = "module",
+    srcs = ["module.py"],
+    deps = [
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:util",
+        "//tensorflow/python:variables",
+        "//tensorflow/python/training/checkpointable:tracking",
+        "@six_archive//:six",
+    ],
+)
+
+tf_py_test(
+    name = "module_test",
+    srcs = ["module_test.py"],
+    additional_deps = [
+        ":module",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python/compat:v2_compat",
+        "//tensorflow/python:variables",
+    ],
+)
diff --git a/tensorflow/python/module/module.py b/tensorflow/python/module/module.py
new file mode 100644
index 0000000000..16db0b0e01
--- /dev/null
+++ b/tensorflow/python/module/module.py
@@ -0,0 +1,402 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Modules encapsulate building stateful components."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import re
+import sys
+
+import six
+
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import variables
+from tensorflow.python.training.checkpointable import tracking
+from tensorflow.python.util import nest
+from tensorflow.python.util import tf_decorator
+from tensorflow.python.util import tf_inspect
+from tensorflow.python.util.tf_export import tf_export
+
+
+class ModuleMetaclass(type):
+  """Metaclass for `tf.Module`."""
+
+  def __new__(mcs, name, bases, clsdict):
+    for key, value in clsdict.items():
+      if key in ("__init__", "name_scope"):
+        continue
+
+      elif tf_inspect.isfunction(value):
+        if getattr(value, "_no_module_name_scope", False):
+          # The function has been annotated to say that no autoscoping should
+          # be applied, so do not patch it.
+          continue
+        clsdict[key] = with_name_scope(value)
+
+      elif isinstance(value, property):
+        clsdict[key] = property(
+            value.fget if not value.fget else with_name_scope(value.fget),
+            value.fset if not value.fset else with_name_scope(value.fset),
+            value.fdel if not value.fdel else with_name_scope(value.fdel),
+            doc=value.__doc__)
+
+    return type.__new__(mcs, name, bases, clsdict)
+
+  def __call__(cls, *args, **kwargs):
+    # Call new such that we have an un-initialized module instance that we can
+    # still reference even if there is an exception during __init__. This is
+    # needed such that we can make sure the name_scope constructed in __init__
+    # is closed even if there is an exception.
+    module = cls.__new__(cls, *args, **kwargs)
+
+    # Now attempt to initialize the object.
+    try:
+      module.__init__(*args, **kwargs)
+    except:
+      # We must explicitly catch so that in Python 2 sys.exc_info() is populated
+      # before entering the finally block.
+      raise
+
+    finally:
+      # The base Module constructor enters the modules name scope before
+      # returning such that other functionality in the ctor happens within the
+      # modules name scope.
+      scope = getattr(module, "_ctor_name_scope", None)
+      exc_info = sys.exc_info()
+      if scope is None:
+        if exc_info[0] is None:
+          raise ValueError(
+              "Constructing a tf.Module without calling the super constructor "
+              "is not supported. Add the following as the first line in your "
+              "__init__ method:\n\n"
+              "super(%s, self).__init__()" % cls.__name__)
+      else:
+        scope.__exit__(*exc_info)
+        del module._ctor_name_scope
+
+    return module
+
+
+def with_name_scope(unbound_method):
+  """Patches the given method so it enters the modules name scope."""
+  def enter_name_scope(self, *args, **kwargs):
+    """Decorator that calls the given function in the module name scope.
+
+    Args:
+      self: Module instance.
+      *args: Positional arguments to `unbound_method`.
+      **kwargs: Keyword arguments to `unbound_method`.
+
+    Returns:
+      `with self.name_scope: return unbound_method(self, *args, **kwargs)`
+    """
+    try:
+      module_name_scope = self.name_scope
+    except AttributeError as exc_value_from:
+      exc_value = AttributeError(
+          "The super constructor must be called before any other methods in "
+          "your constructor. If this is not possible then annotate all the "
+          "methods called with `@no_module_name_scope`.")
+      six.raise_from(exc_value, exc_value_from)
+
+    with module_name_scope:
+      # tf.Module enters the module name scope for all methods. To disable this
+      # for a particular method annotate it with `@no_module_name_scope`.
+      return unbound_method(self, *args, **kwargs)
+
+  return tf_decorator.make_decorator(unbound_method, enter_name_scope)
+
+
+@tf_export("experimental.Module")
+class Module(six.with_metaclass(ModuleMetaclass, tracking.AutoCheckpointable)):
+  """Base neural network module class.
+
+  A module is a named container for `tf.Variable`s, other `tf.Module`s and
+  functions which apply to user input. For example a dense layer in a neural
+  network might be implemented as a `tf.Module`:
+
+  >>> class Dense(tf.Module):
+  ...   def __init__(self, in_features, output_features):
+  ...     super(Linear, self).__init__()
+  ...     self.w = tf.Variable(
+  ...         tf.random_normal([input_features, output_features]), name='w')
+  ...     self.b = tf.Variable(tf.zeros([output_features]), name='b')
+  ...
+  ...   def __call__(self, x):
+  ...     x = tf.convert_to_tensor(x, name='x')
+  ...     y = tf.matmul(x, self.w) + self.b
+  ...     return tf.nn.relu(y)
+
+  You can use the dense layer as you would expect:
+
+  >>> d = Dense(input_features=64, output_features=10)
+  >>> d(tf.ones([100, 64]))
+  <tf.Tensor: ...>
+
+  By subclassing `tf.Module` instead of `object` any variables created inside
+  the module are automatically created within the modules name scope:
+
+  >> d.w.name
+  "dense/w:0"
+
+  In eager mode this is useful for debugging, and when used with `@tf.function`
+  the use of name scopes gives operations (e.g. matmul) useful names as well.
+
+  As well as automatic naming, the Dense module inherits methods for tracking
+  its variables:
+
+  >>> d.variables
+  (<tf.Variable 'dense/b:0' ...>, <tf.Variable 'dense/w:0' ...>)
+  """
+
+  def __init__(self, name=None):
+    if name is None:
+      name = camel_to_snake(type(self).__name__)
+    else:
+      if not valid_identifier(name):
+        raise ValueError(
+            "%r is not a valid module name. Module names must be valid Python "
+            "identifiers (e.g. a valid class name)." % name)
+
+    self._name = name
+    with ops.name_scope(name) as scope_name:
+      self._scope_name = scope_name
+
+    # Enter the name scope so subsequent code in the contructor (e.g. creating
+    # submodules) happens inside the modules name scope. This is exited when
+    # the subclass __init__ returns (this is implemented in ModuleMetaclass).
+    self._ctor_name_scope = self.name_scope
+    self._ctor_name_scope.__enter__()
+
+  @property
+  def name(self):
+    """Returns the name of this module as passed or determined in the ctor.
+
+    NOTE: This is not the same as the `self.name_scope.name` which includes
+    parent module names.
+    """
+    return self._name
+
+  @property
+  def name_scope(self):
+    """Returns a `tf.name_scope` instance for this class."""
+    # TODO(tomhennigan) Memoize once name scopes are re-entrant.
+    return ops.name_scope(self._scope_name)
+
+  @property
+  def variables(self):
+    """Returns a tuple of variables owned by this module and it's submodules.
+
+    Returns:
+      A tuple of variables for the current module (sorted by attribute name)
+      followed by variables from all submodules recursively (depth first).
+    """
+    return tuple(walk(self, recurse_if=_IS_MODULE, predicate=_IS_VARIABLE))
+
+  @property
+  def owned_variables(self):
+    """Returns a tuple of variables that are attributes of the current module.
+
+    See `variables` for a property which returns all variables from the current
+    module and all it's submodules recursively.
+
+    Returns:
+      A tuple of variables which are attributes of the current module. Will
+      yield variables inside nested structures (lists etc) but not in other
+      modules.
+    """
+    return tuple(walk(self, predicate=_IS_VARIABLE))
+
+  @property
+  def trainable_variables(self):
+    """Returns a tuple of variables owned by this module and it's submodules.
+
+    Returns:
+      A tuple of variables for the current module (sorted by attribute name)
+      followed by variables from all submodules recursively (depth first).
+    """
+    return tuple(
+        walk(self, recurse_if=_IS_MODULE, predicate=_IS_TRAINABLE_VARIABLE))
+
+  @property
+  def owned_trainable_variables(self):
+    """Returns a tuple of variables that are attributes of the current module.
+
+    See `variables` for a property which returns all variables from the current
+    module and all it's submodules recursively.
+
+    Returns:
+      A tuple of variables which are attributes of the current module. Will
+      yield variables inside nested structures (lists etc) but not in other
+      modules.
+    """
+    return tuple(walk(self, predicate=_IS_TRAINABLE_VARIABLE))
+
+  @property
+  def owned_submodules(self):
+    """Returns an iterator of immediate child modules.
+
+    Child modules are modules which are found as properties of the current
+    module.
+
+    >>> a = tf.experimental.Module()
+    >>> b = tf.experimental.Module()
+    >>> c = tf.experimental.Module()
+    >>> a.b = b
+    >>> b.c = c
+    >>> assert list(a.owned_submodules) == [b]
+    >>> assert list(b.owned_submodules) == [c]
+    >>> assert list(c.owned_submodules) == []
+
+    Returns:
+      A generator over all child modules.
+    """
+    return walk(self, predicate=_IS_MODULE)
+
+  @property
+  def submodules(self):
+    """Returns an iterator of all sub-modules recursively.
+
+    Submodules are modules which are properties of this module, or found as
+    properties of modules which are properties of this module (and so on).
+
+    >>> a = tf.experimental.Module()
+    >>> b = tf.experimental.Module()
+    >>> c = tf.experimental.Module()
+    >>> a.b = b
+    >>> b.c = c
+    >>> assert list(a.submodules) == [b, c]
+    >>> assert list(b.submodules) == [c]
+    >>> assert list(c.submodules) == []
+
+    Returns:
+      A generator over all submodules.
+    """
+    return walk(self, recurse_if=_IS_MODULE, predicate=_IS_MODULE)
+
+  @classmethod
+  def no_name_scope(cls, method):
+    """Decorator to wrap a method, preventing automatic name scope wrapping.
+
+    By default, any method on a module is considered as a forwards function, and
+    so any variables / modules created by the method will be scoped as belonging
+    to the module. In some cases this is undesirable, for example when
+    implementing .clone() / .transpose(), as in those cases we want the new
+    module to have the scope of wherever the .transpose() call is made. To
+    allow this, decorate any methods with `no_module_name_scope`.
+
+    This logic is tied to ModuleMetaclass.__new__, if anything is
+    changed here corresponding changes will be needed there.
+
+    Args:
+      method: the method to wrap.
+
+    Returns:
+      The method, with a flag indicating no name scope wrapping should occur.
+    """
+    setattr(method, "_no_module_name_scope", True)
+    return method
+
+_IS_VARIABLE = lambda o: isinstance(o, variables.Variable)
+_IS_TRAINABLE_VARIABLE = lambda o: (_IS_VARIABLE(o) and o.trainable)
+_IS_MODULE = lambda o: isinstance(o, Module)
+_CAMEL_TO_SNAKE_R = re.compile(r"((?<=[a-z0-9])[A-Z]|(?!^)[A-Z](?=[a-z]))")
+_VALID_IDENTIFIER = re.compile(r"^[a-zA-Z_]([a-zA-Z0-9_])*$")
+
+
+def valid_identifier(name):
+  return bool(_VALID_IDENTIFIER.match(name))
+
+
+def camel_to_snake(value):
+  return _CAMEL_TO_SNAKE_R.sub(r"_\1", value).lower()
+
+
+def walk(o, recurse_if=None, predicate=None):
+  """Flattened attributes of `o` in sorted order by attribute name.
+
+  >>> class Foo(object):
+  ...   def __init__(self, prefix=''):
+  ...     self.z = prefix + 'c'
+  ...     self.a = [prefix + 'a', prefix + 'b']
+
+  >>> tuple(walk(Foo()))
+  ('a', 'b', 'c')
+
+  If `predicate` is not None, then only values matching predicate are returned:
+
+  >>> tuple(walk(Foo(), predicate=lambda v: v != 'a'))
+  ('b', 'c')
+
+  If `recurse_if` is not None then it should be a callable which tests if the
+  given leaf should be expanded:
+
+  >>> is_string = lambda v: isinstance(v, str)
+  >>> is_foo = lambda l: isinstance(l, Foo)
+  >>> o = Foo(prefix='root_')
+  >>> o.b = Foo(prefix='child_')
+  >>> tuple(walk(o, predicate=is_string))
+  ('root_a', 'root_b', 'root_c')
+  >>> tuple(walk(o, recurse_if=is_foo, predicate=is_string))
+  ('root_a', 'root_b', 'root_c', 'child_a', 'child_b', 'child_c')
+
+  Args:
+    o: An object who's attributes are walked.
+    recurse_if: (Optional) Visited items of this type will be walked to extract
+      more leaves. If `None`, it will not recurse into leaves.
+    predicate: (Optional) If set then only values matching predicate are
+      yielded.
+
+  Returns:
+    Attributes of `o` in name order. If `recurse_if` is not `None` then
+    attributes for which `recurse_if(attribute) == True` will be walked
+    recursively. If `predicate` is not `None` then only attributes for which
+    `predicate(attribute) == True` will be yielded.
+  """
+  if predicate is None:
+    predicate = lambda _: True
+  return _walk_internal(
+      o, recurse_if=recurse_if, predicate=predicate, seen=set())
+
+
+def _walk_internal(o, recurse_if, predicate, seen):
+  """Implementation of `walk`."""
+  if seen is None:
+    seen = set([id(o)])
+
+  o_dict = vars(o)
+  to_walk = []
+
+  for key in sorted(o_dict):
+    values = nest.flatten(o_dict[key])
+    for value in values:
+      value_id = id(value)
+      if value_id in seen:
+        continue
+
+      seen.add(value_id)
+      if predicate(value):
+        yield value
+
+      if recurse_if is not None and recurse_if(value):
+        # Walk direct properties first then recurse.
+        to_walk.append(value)
+
+  for value in to_walk:
+    for subvalue in _walk_internal(value, recurse_if, predicate, seen):
+      # Predicate is already tested for these values.
+      yield subvalue
diff --git a/tensorflow/python/module/module_test.py b/tensorflow/python/module/module_test.py
new file mode 100644
index 0000000000..e7c736696e
--- /dev/null
+++ b/tensorflow/python/module/module_test.py
@@ -0,0 +1,355 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for `tf.Module`."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.compat import v2_compat
+from tensorflow.python.framework import ops
+from tensorflow.python.module import module
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import test
+
+
+class TestModuleNaming(test.TestCase):
+
+  def test_single_name(self):
+    mod = module.Module(name="simple")
+    self.assertEqual(mod.name, "simple")
+    self.assertEqual(mod.name_scope.name, "simple/")
+
+  def test_construct_in_scope(self):
+    with ops.name_scope("foo"):
+      mod = module.Module(name="bar")
+    self.assertEqual(mod.name, "bar")
+    self.assertEqual(mod.name_scope.name, "foo/bar/")
+
+  def test_enters_name_scope_in_call(self):
+    mod = ReturnsNameScopeModule()
+    for _ in range(3):
+      self.assertEqual(mod(), mod.name_scope.name)
+
+  def test_enters_name_scope_in_other_method(self):
+    mod = ReturnsNameScopeModule()
+    for _ in range(3):
+      self.assertEqual(mod.alternative_forward(), mod.name_scope.name)
+
+  def test_subclassed_module(self):
+    mod = SubclassedReturnsNameScopeModule()
+    for _ in range(3):
+      self.assertEqual(mod.alternative_forward(), mod.name_scope.name)
+      self.assertEqual(mod.alternative_alternative_forward(),
+                       mod.name_scope.name)
+
+  def test_submodule_created_late(self):
+    m = TreeModule()
+    self.assertEqual(m.name, "tree_module")
+    self.assertEqual(m.name_scope.name, "tree_module/")
+    leaf1 = m.new_leaf()
+    self.assertEqual(leaf1.name, "tree_module")
+    self.assertEqual(leaf1.name_scope.name, "tree_module/tree_module/")
+
+  def test_does_not_evaluate_property_methods(self):
+    mod = PropertyThrowsWhenCalledModule()
+    with self.assertRaises(AssertionError):
+      mod.raise_assertion_error  # pylint: disable=pointless-statement
+
+  def test_overridden_name_scope(self):
+    mod = ModuleOverridingNameScope()
+    self.assertEqual(mod(), mod.name_scope.name)
+    self.assertEqual(mod.alternative_forward(), mod.name_scope.name)
+
+  def test_patched_callable(self):
+    with ops.name_scope("foo"):
+      mod = module.Module(name="bar")
+    mod.foo = get_name_scope
+    # `foo` is not a method so we do not re-enter the name scope.
+    self.assertEqual(mod.foo(), "")
+
+  def test_invalid_name(self):
+    msg = ".* is not a valid module name"
+    with self.assertRaisesRegexp(ValueError, msg):
+      module.Module(name="$Foo")
+
+  def test_modules_not_numbered_in_eager(self):
+    mod = RecursiveModule(2)
+    self.assertEqual(mod.name_scope.name, "badger/")
+    self.assertEqual(mod.child.name_scope.name, "badger/badger/")
+
+    mod = RecursiveModule(2)
+    self.assertEqual(mod.name_scope.name, "badger/")
+    self.assertEqual(mod.child.name_scope.name, "badger/badger/")
+
+  def test_module_numbering_in_graph(self):
+    with ops.Graph().as_default():
+      mod = RecursiveModule(2)
+      self.assertEqual(mod.name_scope.name, "badger/")
+      self.assertEqual(mod.child.name_scope.name, "badger/badger/")
+
+      mod = RecursiveModule(2)
+      self.assertEqual(mod.name_scope.name, "badger_1/")
+      self.assertEqual(mod.child.name_scope.name, "badger_1/badger/")
+
+  def test_ctor_error_closes_name_scope(self):
+    with self.assertRaises(ErrorModuleError):
+      # If super constructor is called then a name scope is opened then an error
+      # is thrown. The metaclass should handle this and close the namescope
+      # before re-throwing the exception.
+      ErrorModule(call_super=True)
+
+    self.assertEqual("", get_name_scope())
+
+  def test_ctor_error_handles_ctor_not_opening_name_scope(self):
+    with self.assertRaises(ErrorModuleError):
+      # If super ctor is not called then the name scope isn't opened. We need to
+      # ensure that this doesn't trigger an exception (e.g. the metaclass trying
+      # to __exit__ a non-existant name scope).
+      ErrorModule(call_super=False)
+
+    self.assertEqual("", get_name_scope())
+
+  def test_forward_method_closes_name_scope(self):
+    mod = ErrorModule(call_super=True, raise_in_constructor=False)
+    with self.assertRaises(ErrorModuleError):
+      mod()
+
+    self.assertEqual("", get_name_scope())
+
+
+class VariableNamingTest(test.TestCase):
+
+  def test_variable_names(self):
+    mod = RecursiveModule(3)
+    self.assertEqual(mod.w.name, "badger/mushroom:0")
+    self.assertEqual(mod.child.w.name, "badger/badger/mushroom:0")
+    self.assertEqual(mod.child.child.w.name, "badger/badger/badger/mushroom:0")
+
+
+class VariableTrackingTest(test.TestCase):
+
+  def test_variables(self):
+    m = RecursiveModule(3)
+    self.assertEqual(m.variables, (m.w, m.child.w, m.child.child.w))
+    self.assertEqual(m.child.variables, (m.child.w, m.child.child.w))
+    self.assertEqual(m.child.child.variables, (m.child.child.w,))
+
+  def test_owned_variables(self):
+    m = RecursiveModule(3)
+    self.assertEqual(m.owned_variables, (m.w,))
+    self.assertEqual(m.child.owned_variables, (m.child.w,))
+    self.assertEqual(m.child.child.owned_variables, (m.child.child.w,))
+
+  def test_trainable_variables(self):
+    m = RecursiveModule(3)
+    self.assertEqual(m.trainable_variables,
+                     (m.w, m.child.w, m.child.child.w))
+    self.assertEqual(m.child.trainable_variables,
+                     (m.child.w, m.child.child.w))
+    self.assertEqual(m.child.child.trainable_variables, (m.child.child.w,))
+
+  def test_trainable_variables_ignores_non_trainable(self):
+    m = RecursiveModule(3, trainable=False)
+    self.assertEqual(len(m.trainable_variables), 0)
+    self.assertEqual(len(m.child.trainable_variables), 0)
+    self.assertEqual(len(m.child.child.trainable_variables), 0)
+
+  def test_owned_trainable_variables(self):
+    m = RecursiveModule(3)
+    self.assertEqual(m.owned_trainable_variables, (m.w,))
+    self.assertEqual(m.child.owned_trainable_variables, (m.child.w,))
+    self.assertEqual(m.child.child.owned_trainable_variables,
+                     (m.child.child.w,))
+
+  def test_owned_trainable_variables_ignores_non_trainable(self):
+    m = RecursiveModule(3, trainable=False)
+    self.assertEqual(len(m.owned_trainable_variables), 0)
+    self.assertEqual(len(m.child.owned_trainable_variables), 0)
+    self.assertEqual(len(m.child.child.owned_trainable_variables), 0)
+
+
+class ModuleTrackingTest(test.TestCase):
+
+  def test_owned_submodules(self):
+    m = RecursiveModule(3)
+    self.assertEqual(list(m.owned_submodules), [m.child])
+    self.assertEqual(list(m.child.owned_submodules), [m.child.child])
+    self.assertEqual(list(m.child.child.owned_submodules), [])
+
+  def test_submodules(self):
+    m = RecursiveModule(3)
+    self.assertEqual(list(m.submodules), [m.child, m.child.child])
+    self.assertEqual(list(m.child.submodules), [m.child.child])
+    self.assertEqual(list(m.child.child.submodules), [])
+
+  def test_non_ctor_submodule(self):
+    m = TreeModule()
+    leaf1 = m.new_leaf()
+    self.assertEqual(set(m.submodules), {leaf1})
+    leaf2 = m.new_leaf()
+    self.assertEqual(set(m.submodules), {leaf1, leaf2})
+
+
+class CommonErrorsTest(test.TestCase):
+
+  def test_not_calling_super_constructor(self):
+    msg = ("Constructing a tf.Module without calling the super constructor is "
+           "not supported")
+    with self.assertRaisesRegexp(ValueError, msg):
+      DoesNotCallSuperConstructorModule()
+
+  def test_calls_method_before_super(self):
+    msg = "super constructor must be called before any other methods"
+    with self.assertRaisesRegexp(AttributeError, msg):
+      CallsMethodBeforeSuperConstructorModule(allowed_method=False)
+
+  def test_annotated_method_is_allowed(self):
+    self.assertIsNotNone(
+        CallsMethodBeforeSuperConstructorModule(allowed_method=True))
+
+
+def get_name_scope():
+  with ops.name_scope("x") as ns:
+    return ns[:-2]
+
+
+class ErrorModuleError(Exception):
+  pass
+
+
+class ErrorModule(module.Module):
+
+  def __init__(self, call_super, raise_in_constructor=True):
+    if call_super:
+      super(ErrorModule, self).__init__()
+    if raise_in_constructor:
+      raise ErrorModuleError("Deliberate error!")
+
+  def __call__(self):
+    raise ErrorModuleError("Deliberate error!")
+
+
+class RecursiveModule(module.Module):
+
+  def __init__(self, depth, trainable=True):
+    super(RecursiveModule, self).__init__(name="badger")
+    self.child = None
+    if depth > 1:
+      self.child = RecursiveModule(depth - 1, trainable=trainable)
+    self.w = variables.Variable(1.0, trainable=trainable, name="mushroom")
+
+
+class TreeModule(module.Module):
+
+  def __init__(self, name=None):
+    super(TreeModule, self).__init__(name=name)
+    self._leaves = []
+
+  def new_leaf(self, name=None):
+    leaf = TreeModule(name=name)
+    self._leaves.append(leaf)
+    return leaf
+
+
+class ReturnsNameScopeModule(module.Module):
+
+  def alternative_forward(self):
+    return get_name_scope()
+
+  def __call__(self):
+    return get_name_scope()
+
+
+class SubclassedReturnsNameScopeModule(ReturnsNameScopeModule):
+
+  def alternative_alternative_forward(self):
+    return get_name_scope()
+
+
+class PropertyThrowsWhenCalledModule(module.Module):
+
+  @property
+  def raise_assertion_error(self):
+    raise AssertionError
+
+
+class ModuleOverridingNameScope(ReturnsNameScopeModule):
+
+  @property
+  def name_scope(self):
+    return ops.name_scope("yolo/")
+
+
+class DoesNotCallSuperConstructorModule(module.Module):
+
+  def __init__(self):
+    # NOTE: Intentionally does not call super constructor.
+    pass
+
+
+class CallsMethodBeforeSuperConstructorModule(module.Module):
+
+  def __init__(self, allowed_method):
+    if allowed_method:
+      self.no_name_scope()
+    else:
+      self.with_name_scope()
+    super(CallsMethodBeforeSuperConstructorModule, self).__init__()
+
+  @module.Module.no_name_scope
+  def no_name_scope(self):
+    pass
+
+  def with_name_scope(self):
+    pass
+
+
+class WalkTest(test.TestCase):
+
+  def test_walk(self):
+    parent = SimpleModule()
+    child = parent.c
+
+    self.assertEqual(
+        list(module.walk(parent, predicate=IS_MEMBER)),
+        [parent.a[0], parent.a[1], parent.z])
+
+    self.assertEqual(
+        list(module.walk(parent, recurse_if=IS_MODULE, predicate=IS_MEMBER)),
+        [parent.a[0], parent.a[1], parent.z, child.a[0], child.a[1], child.z])
+
+
+class MemberType(object):
+  """A simple type to search for."""
+  pass
+
+
+class SimpleModule(module.Module):
+
+  def __init__(self, create_child=True):
+    super(SimpleModule, self).__init__()
+    self.z = MemberType()
+    self.a = [MemberType(), MemberType()]
+    if create_child:
+      self.c = SimpleModule(create_child=False)
+
+
+IS_MEMBER = lambda v: isinstance(v, MemberType)
+IS_MODULE = lambda v: isinstance(v, module.Module)
+
+if __name__ == "__main__":
+  v2_compat.enable_v2_behavior()
+  test.main()
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.experimental.-module.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.experimental.-module.pbtxt
new file mode 100644
index 0000000000..c364b0217a
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.experimental.-module.pbtxt
@@ -0,0 +1,47 @@
+path: "tensorflow.experimental.Module"
+tf_class {
+  is_instance: "<class \'tensorflow.python.module.module.Module\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.tracking.AutoCheckpointable\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name_scope"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "owned_submodules"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "owned_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "owned_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "submodules"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "no_name_scope"
+    argspec: "args=[\'cls\', \'method\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.experimental.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.experimental.pbtxt
index 0c3f04e468..a7ee6d3e07 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.experimental.pbtxt
@@ -1,5 +1,9 @@
 path: "tensorflow.experimental"
 tf_module {
+  member {
+    name: "Module"
+    mtype: "<class \'tensorflow.python.module.module.ModuleMetaclass\'>"
+  }
   member_method {
     name: "function_executor_type"
     argspec: "args=[\'executor_type\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.experimental.-module.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.experimental.-module.pbtxt
new file mode 100644
index 0000000000..c364b0217a
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.experimental.-module.pbtxt
@@ -0,0 +1,47 @@
+path: "tensorflow.experimental.Module"
+tf_class {
+  is_instance: "<class \'tensorflow.python.module.module.Module\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.tracking.AutoCheckpointable\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name_scope"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "owned_submodules"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "owned_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "owned_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "submodules"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "no_name_scope"
+    argspec: "args=[\'cls\', \'method\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.experimental.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.experimental.pbtxt
index 0c3f04e468..a7ee6d3e07 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.experimental.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.experimental.pbtxt
@@ -1,5 +1,9 @@
 path: "tensorflow.experimental"
 tf_module {
+  member {
+    name: "Module"
+    mtype: "<class \'tensorflow.python.module.module.ModuleMetaclass\'>"
+  }
   member_method {
     name: "function_executor_type"
     argspec: "args=[\'executor_type\'], varargs=None, keywords=None, defaults=None"
-- 
GitLab


From 31c1c097d8ee260ab1411011f5276b326cfb2ace Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 17 Jan 2019 07:36:10 -0800
Subject: [PATCH 0866/1095] Remove 19 uneeded C++ BUILD Dependencies

PiperOrigin-RevId: 229744406
---
 tensorflow/compiler/xla/service/BUILD             | 10 ----------
 tensorflow/compiler/xla/service/cpu/BUILD         |  2 --
 tensorflow/compiler/xla/service/gpu/BUILD         |  4 ----
 tensorflow/compiler/xla/service/interpreter/BUILD |  2 --
 tensorflow/compiler/xla/service/llvm_ir/BUILD     |  1 -
 5 files changed, 19 deletions(-)

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index aef3c37159..e39e17c110 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -1062,7 +1062,6 @@ cc_library(
         "//tensorflow/compiler/xla:types",
         "//tensorflow/compiler/xla:util",
         "//tensorflow/core:lib",
-        "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/strings",
         "@com_google_absl//absl/strings:str_format",
@@ -1100,7 +1099,6 @@ cc_library(
         ":buffer_value_containers",
         ":heap_simulator",
         ":hlo",
-        ":hlo_memory_scheduler",
         ":hlo_proto",
         ":logical_buffer",
         ":tuple_points_to_analysis",
@@ -1240,7 +1238,6 @@ cc_library(
     deps = [
         ":hlo",
         ":hlo_proto",
-        "//tensorflow/compiler/xla:statusor",
         "@com_google_absl//absl/strings",
         "@com_google_absl//absl/types:span",
     ],
@@ -1508,7 +1505,6 @@ cc_library(
         "//tensorflow/compiler/xla:util",
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/core:lib",
-        "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/types:optional",
         "@com_google_absl//absl/types:span",
     ],
@@ -2069,7 +2065,6 @@ cc_library(
         "//tensorflow/compiler/xla:literal",
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:status_macros",
-        "//tensorflow/compiler/xla:statusor",
         "//tensorflow/compiler/xla:types",
         "//tensorflow/compiler/xla:util",
         "//tensorflow/compiler/xla:xla_data_proto",
@@ -3030,10 +3025,7 @@ cc_library(
         ":hlo",
         ":hlo_pass",
         ":shape_inference",
-        "//tensorflow/compiler/xla:literal",
         "//tensorflow/compiler/xla:literal_util",
-        "//tensorflow/compiler/xla:shape_util",
-        "//tensorflow/compiler/xla:types",
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/core:lib",
         "@com_google_absl//absl/algorithm:container",
@@ -3405,7 +3397,6 @@ cc_library(
         ":hlo_pass_pipeline",
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/core:lib",
-        "@com_google_absl//absl/container:flat_hash_map",
     ],
 )
 
@@ -3593,7 +3584,6 @@ cc_library(
         "//tensorflow/compiler/xla:util",
         "//tensorflow/core:lib",
         "@com_google_absl//absl/algorithm:container",
-        "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/container:inlined_vector",
     ],
 )
diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD
index a7feab5a0a..d4535b204d 100644
--- a/tensorflow/compiler/xla/service/cpu/BUILD
+++ b/tensorflow/compiler/xla/service/cpu/BUILD
@@ -790,8 +790,6 @@ cc_library(
         ":target_machine_features",
         "//tensorflow/compiler/xla:util",
         "//tensorflow/compiler/xla/service:computation_layout",
-        "//tensorflow/compiler/xla/service:hlo",
-        "//tensorflow/compiler/xla/service:hlo_casting_utils",
         "//tensorflow/compiler/xla/service:layout_assignment",
         "//tensorflow/core:lib",
         "@com_google_absl//absl/container:flat_hash_map",
diff --git a/tensorflow/compiler/xla/service/gpu/BUILD b/tensorflow/compiler/xla/service/gpu/BUILD
index 1caf753c45..dc17aa4426 100644
--- a/tensorflow/compiler/xla/service/gpu/BUILD
+++ b/tensorflow/compiler/xla/service/gpu/BUILD
@@ -1013,14 +1013,10 @@ cc_library(
     srcs = ["variadic_op_splitter.cc"],
     hdrs = ["variadic_op_splitter.h"],
     deps = [
-        ":ir_emission_utils",
-        "//tensorflow/compiler/xla:literal_util",
         "//tensorflow/compiler/xla:statusor",
         "//tensorflow/compiler/xla:util",
-        "//tensorflow/compiler/xla:window_util",
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/service:hlo",
-        "//tensorflow/compiler/xla/service:hlo_casting_utils",
         "//tensorflow/compiler/xla/service:hlo_pass",
         "//tensorflow/core:lib",
         "@com_google_absl//absl/strings",
diff --git a/tensorflow/compiler/xla/service/interpreter/BUILD b/tensorflow/compiler/xla/service/interpreter/BUILD
index 9b4cafa4b4..545662543c 100644
--- a/tensorflow/compiler/xla/service/interpreter/BUILD
+++ b/tensorflow/compiler/xla/service/interpreter/BUILD
@@ -32,7 +32,6 @@ cc_library(
         "//tensorflow/compiler/xla:status_macros",
         "//tensorflow/compiler/xla:statusor",
         "//tensorflow/compiler/xla/service:algebraic_simplifier",
-        "//tensorflow/compiler/xla/service:batchnorm_expander",
         "//tensorflow/compiler/xla/service:compiler",
         "//tensorflow/compiler/xla/service:computation_placer",
         "//tensorflow/compiler/xla/service:dynamic_index_splitter",
@@ -43,7 +42,6 @@ cc_library(
         "//tensorflow/compiler/xla/service:hlo_cost_analysis",
         "//tensorflow/compiler/xla/service:hlo_cse",
         "//tensorflow/compiler/xla/service:hlo_dce",
-        "//tensorflow/compiler/xla/service:hlo_get_dimension_size_rewriter",
         "//tensorflow/compiler/xla/service:hlo_module_config",
         "//tensorflow/compiler/xla/service:hlo_pass",
         "//tensorflow/compiler/xla/service:hlo_pass_pipeline",
diff --git a/tensorflow/compiler/xla/service/llvm_ir/BUILD b/tensorflow/compiler/xla/service/llvm_ir/BUILD
index 54e8fe1947..c5d59fb28e 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/BUILD
+++ b/tensorflow/compiler/xla/service/llvm_ir/BUILD
@@ -39,7 +39,6 @@ cc_library(
         "//tensorflow/compiler/xla/service:logical_buffer",
         "//tensorflow/core:lib",
         "@com_google_absl//absl/container:flat_hash_map",
-        "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/strings",
         "@llvm//:core",
     ],
-- 
GitLab


From cc6f6fb453f1cb864643452ff937f9f4606bda32 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Thu, 17 Jan 2019 07:40:48 -0800
Subject: [PATCH 0867/1095] Dump out an annotated TF graph from
 mark_for_compilation_pass for better visualization

PiperOrigin-RevId: 229745040
---
 .../compiler/jit/mark_for_compilation_pass.cc | 22 +++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass.cc b/tensorflow/compiler/jit/mark_for_compilation_pass.cc
index 50afec020d..cb01690845 100644
--- a/tensorflow/compiler/jit/mark_for_compilation_pass.cc
+++ b/tensorflow/compiler/jit/mark_for_compilation_pass.cc
@@ -41,6 +41,7 @@ limitations under the License.
 #include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/graph/algorithm.h"
 #include "tensorflow/core/graph/control_flow.h"
+#include "tensorflow/core/graph/graph_constructor.h"
 #include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/gtl/cleanup.h"
 #include "tensorflow/core/lib/strings/stringprintf.h"
@@ -1150,6 +1151,27 @@ Status MarkForCompilationPass::RunImpl(
   if (flags->tf_xla_clustering_debug) {
     dump_graph::DumpGraphToFile("mark_for_compilation", **options.graph,
                                 options.flib_def);
+
+    // We also dump out an annoated version of the TF graph where the nodes
+    // names are prefixed with the cluster names.  This can help visualizing the
+    // clustering decisions on TensorBoard.
+    Graph new_graph((*options.graph)->op_registry());
+    CopyGraph(**options.graph, &new_graph);
+
+    for (Node* n : new_graph.nodes()) {
+      if (absl::optional<absl::string_view> cluster_name =
+              GetXlaClusterForNode(*n)) {
+        n->set_name(absl::StrCat(*cluster_name, "/", n->name()));
+      } else {
+        // There is room for improvement here.  In particular, it may help to
+        // split these unclustered nodes into classes where every node in a
+        // specific class has edges to and from the same set of clusters.
+        n->set_name(absl::StrCat("unclustered/", n->name()));
+      }
+    }
+
+    dump_graph::DumpGraphToFile("mark_for_compilation_annotated", new_graph,
+                                options.flib_def);
   }
 
   VLogClusteringSummary(*graph);
-- 
GitLab


From 202525cd4bbab377e9ee67f1e4098e40b0925d9c Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Thu, 17 Jan 2019 07:43:30 -0800
Subject: [PATCH 0868/1095] Blacklist certain failing NNAPI zip tests

PiperOrigin-RevId: 229745363
---
 .../testing/generated_examples_zip_test.cc    | 29 ++++++++++++++++++-
 1 file changed, 28 insertions(+), 1 deletion(-)

diff --git a/tensorflow/lite/testing/generated_examples_zip_test.cc b/tensorflow/lite/testing/generated_examples_zip_test.cc
index a9a31ad088..45bd59a67d 100644
--- a/tensorflow/lite/testing/generated_examples_zip_test.cc
+++ b/tensorflow/lite/testing/generated_examples_zip_test.cc
@@ -107,6 +107,28 @@ std::map<string, string> kBrokenTests = {
     {R"(^\/strided_slice_buggy)", "119786029"},
 };
 
+// Additional list of tests that are expected to fail when
+//   --test_arg=--ignore_known_bugs=false
+// and
+//   --test_arg=--use_nnapi=true
+// Note that issues related to lack of NNAPI support for a particular op are
+// handled separately; this list is specifically for broken cases where
+// execution produces broken output.
+// Key is a substring of the test name and value is a bug number.
+std::map<string, string> kBrokenNnapiTests = {
+    // Certain NNAPI kernels silently fail with int32 types.
+    {R"(^\/add.*dtype=tf\.int32)", "122987564"},
+    {R"(^\/concat.*dtype=tf\.int32)", "122987564"},
+    {R"(^\/mul.*dtype=tf\.int32)", "122987564"},
+    {R"(^\/space_to_depth.*dtype=tf\.int32)", "122987564"},
+
+    // Certain NNAPI fully_connected shape permutations fail.
+    {R"(^\/fully_connected_constant_filter=True.*shape1=\[3,3\])", "122987564"},
+    {R"(^\/fully_connected_constant_filter=True.*shape1=\[4,4\])", "122987564"},
+    {R"(^\/fully_connected.*shape1=\[3,3\].*transpose_b=True)", "122987564"},
+    {R"(^\/fully_connected.*shape1=\[4,4\].*shape2=\[4,1\])", "122987564"},
+};
+
 // Allows test data to be unarchived into a temporary directory and makes
 // sure those temporary directories are removed later.
 class ArchiveEnvironment : public ::testing::Environment {
@@ -242,8 +264,13 @@ TEST_P(OpsTest, RunZipTests) {
   tflite::testing::TfLiteDriver test_driver(FLAGS_use_nnapi);
   test_driver.SetModelBaseDir(tflite_dir);
 
+  auto broken_tests = kBrokenTests;
+  if (FLAGS_use_nnapi) {
+    broken_tests.insert(kBrokenNnapiTests.begin(), kBrokenNnapiTests.end());
+  }
+
   string bug_number;
-  for (const auto& p : kBrokenTests) {
+  for (const auto& p : broken_tests) {
     if (RE2::PartialMatch(test_name, p.first)) {
       bug_number = p.second;
     }
-- 
GitLab


From f01d357ca3f08d7872dc1059e1a13f7fb4515ef2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 17 Jan 2019 08:34:02 -0800
Subject: [PATCH 0869/1095] Switch from deprecated
 strategy.distribute_dataset() to strategy.make_input_fn_iterator() in
 contrib/distribute tests.

PiperOrigin-RevId: 229752452
---
 .../distribute/python/metrics_v1_test.py      |  5 ++---
 .../distribute/python/minimize_loss_test.py   | 21 ++++++++-----------
 .../python/mirrored_strategy_multigpu_test.py | 11 +++-------
 .../contrib/distribute/python/monitor.py      |  2 +-
 .../distribute/python/optimizer_v2_test.py    |  9 ++------
 .../contrib/distribute/python/step_fn.py      | 14 ++++++-------
 .../contrib/distribute/python/step_fn_test.py |  3 ++-
 7 files changed, 26 insertions(+), 39 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/metrics_v1_test.py b/tensorflow/contrib/distribute/python/metrics_v1_test.py
index 96006276f6..a663e809dd 100644
--- a/tensorflow/contrib/distribute/python/metrics_v1_test.py
+++ b/tensorflow/contrib/distribute/python/metrics_v1_test.py
@@ -95,8 +95,7 @@ class MetricsV1Test(test.TestCase, parameterized.TestCase):
 
   def _test_metric(self, distribution, dataset_fn, metric_fn, expected_fn):
     with ops.Graph().as_default(), distribution.scope():
-      iterator = distribution.distribute_dataset(
-          dataset_fn).make_initializable_iterator()
+      iterator = distribution.make_input_fn_iterator(lambda _: dataset_fn())
       if isinstance(distribution, tpu_strategy.TPUStrategy):
         def step_fn(ctx, inputs):
           value, update = distribution.extended.call_for_each_replica(
@@ -121,7 +120,7 @@ class MetricsV1Test(test.TestCase, parameterized.TestCase):
         # replace "distribution.num_replicas_in_sync" with "1".
         batches_per_update = distribution.num_replicas_in_sync
 
-      self.evaluate(iterator.initializer)
+      self.evaluate(iterator.initialize())
       self.evaluate(variables.local_variables_initializer())
 
       batches_consumed = 0
diff --git a/tensorflow/contrib/distribute/python/minimize_loss_test.py b/tensorflow/contrib/distribute/python/minimize_loss_test.py
index 37e6b418a0..f06c9b7564 100644
--- a/tensorflow/contrib/distribute/python/minimize_loss_test.py
+++ b/tensorflow/contrib/distribute/python/minimize_loss_test.py
@@ -41,12 +41,9 @@ from tensorflow.python.ops.losses import losses_impl
 
 class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
 
-  def _get_iterator(self, ds):
-    if context.executing_eagerly():
-      iterator = ds.make_one_shot_iterator()
-    else:
-      iterator = ds.make_initializable_iterator()
-      self.evaluate(iterator.initializer)
+  def _get_iterator(self, strategy, input_fn):
+    iterator = strategy.make_input_fn_iterator(lambda _: input_fn())
+    self.evaluate(iterator.initialize())
     return iterator
 
   @combinations.generate(
@@ -70,7 +67,7 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
             distribution.extended.call_for_each_replica(
                 model_fn, args=(inputs,)))
 
-      iterator = self._get_iterator(distribution.distribute_dataset(dataset_fn))
+      iterator = self._get_iterator(distribution, dataset_fn)
 
       def run_step():
         return distribution.extended.experimental_run_steps_on_iterator(
@@ -102,7 +99,7 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
       model_fn, dataset_fn, layer = minimize_loss_example(
           optimizer_fn, use_bias=True, use_callable_loss=use_callable_loss)
 
-      iterator = self._get_iterator(distribution.distribute_dataset(dataset_fn))
+      iterator = self._get_iterator(distribution, dataset_fn)
 
       def run_step():
         return distribution.group(
@@ -161,7 +158,7 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
             distribution.extended.call_for_each_replica(
                 model_fn, args=(inputs,)))
 
-      iterator = self._get_iterator(distribution.distribute_dataset(dataset_fn))
+      iterator = self._get_iterator(distribution, dataset_fn)
 
       def run_step():
         return distribution.extended.experimental_run_steps_on_iterator(
@@ -230,7 +227,7 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
           fetches += tuple(ops.get_collection(ops.GraphKeys.UPDATE_OPS))
         return control_flow_ops.group(fetches)
 
-      iterator = self._get_iterator(distribution.distribute_dataset(dataset_fn))
+      iterator = self._get_iterator(distribution, dataset_fn)
 
       def run_step():
         return distribution.extended.experimental_run_steps_on_iterator(
@@ -322,7 +319,7 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
             distribution.extended.call_for_each_replica(
                 model_fn, args=(inputs,)))
 
-      iterator = self._get_iterator(distribution.distribute_dataset(dataset_fn))
+      iterator = self._get_iterator(distribution, dataset_fn)
 
       def run_step():
         return distribution.extended.experimental_run_steps_on_iterator(
@@ -413,7 +410,7 @@ class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
             output=loss)
         return distribution.group(train_op)
 
-      iterator = self._get_iterator(distribution.distribute_dataset(dataset_fn))
+      iterator = self._get_iterator(distribution, dataset_fn)
 
       def run_step():
         initial_loss = lambda: constant_op.constant(1e7)
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
index c22e4f8ee6..d6337d106f 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
@@ -366,14 +366,9 @@ class MirroredStrategyVariableCreationTest(test.TestCase):
                 (layer2.kernel, layer2.bias),
                 (layer3.kernel, layer3.bias)]
 
-    ds = distribution.distribute_dataset(
-        lambda: dataset_ops.Dataset.from_tensors([[1.]]).repeat(10))
-    if context.executing_eagerly():
-      iterator = ds.make_one_shot_iterator()
-    else:
-      iterator = ds.make_initializable_iterator()
-      self.evaluate([iterator.initializer])
-
+    iterator = distribution.make_input_fn_iterator(
+        lambda _: dataset_ops.Dataset.from_tensors([[1.]]).repeat(10))
+    self.evaluate(iterator.initialize())
     features = iterator.get_next()
 
     with distribution.scope():
diff --git a/tensorflow/contrib/distribute/python/monitor.py b/tensorflow/contrib/distribute/python/monitor.py
index 17b7ab74f6..53e35ea6b7 100644
--- a/tensorflow/contrib/distribute/python/monitor.py
+++ b/tensorflow/contrib/distribute/python/monitor.py
@@ -51,7 +51,7 @@ class Monitor(object):
     else:
       if session is None:
         raise ValueError("Should provide a `session` in Graph mode.")
-      session.run(step_callable._iterator.initializer)  # pylint: disable=protected-access
+      session.run(step_callable.initialize())
       self._run_step = session.make_callable(step_callable())
       session.run(variables.global_variables_initializer())
 
diff --git a/tensorflow/contrib/distribute/python/optimizer_v2_test.py b/tensorflow/contrib/distribute/python/optimizer_v2_test.py
index a832ef61f9..e388061b17 100644
--- a/tensorflow/contrib/distribute/python/optimizer_v2_test.py
+++ b/tensorflow/contrib/distribute/python/optimizer_v2_test.py
@@ -41,12 +41,7 @@ class MinimizeLossOptimizerV2Test(test.TestCase, parameterized.TestCase):
     with distribution.scope():
       model_fn, dataset_fn, layer = minimize_loss_example(
           optimizer_fn, use_bias=True, use_callable_loss=use_callable_loss)
-
-      ds = distribution.distribute_dataset(dataset_fn)
-      if context.executing_eagerly():
-        iterator = ds.make_one_shot_iterator()
-      else:
-        iterator = ds.make_initializable_iterator()
+      iterator = distribution.make_input_fn_iterator(lambda _: dataset_fn())
 
       def run_step():
         return control_flow_ops.group(
@@ -56,7 +51,7 @@ class MinimizeLossOptimizerV2Test(test.TestCase, parameterized.TestCase):
 
       if not context.executing_eagerly():
         with self.cached_session() as sess:
-          sess.run(iterator.initializer)
+          sess.run(iterator.initialize())
           run_step = sess.make_callable(run_step())
         self.evaluate(variables.global_variables_initializer())
 
diff --git a/tensorflow/contrib/distribute/python/step_fn.py b/tensorflow/contrib/distribute/python/step_fn.py
index 24b95f748b..27aad46b97 100644
--- a/tensorflow/contrib/distribute/python/step_fn.py
+++ b/tensorflow/contrib/distribute/python/step_fn.py
@@ -19,7 +19,6 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.eager import backprop
-from tensorflow.python.eager import context
 from tensorflow.python.training import optimizer as optimizer_lib
 
 
@@ -33,6 +32,9 @@ class Step(object):
   def distribution(self):
     return self._distribution
 
+  def initialize(self):
+    return []
+
   def __call__(self):
     """Perform one step of this training algorithm."""
     raise NotImplementedError("must be implemented in descendants")
@@ -50,12 +52,10 @@ class StandardInputStep(Step):
 
   def __init__(self, dataset_fn, distribution):
     super(StandardInputStep, self).__init__(distribution)
-    self._distributed_input = distribution.distribute_dataset(dataset_fn)
-    if context.executing_eagerly():
-      self._iterator = self._distributed_input.make_one_shot_iterator()
-    else:
-      # TODO(priyag): Expose initializer via some initializer property.
-      self._iterator = self._distributed_input.make_initializable_iterator()
+    self._iterator = distribution.make_input_fn_iterator(lambda _: dataset_fn())
+
+  def initialize(self):
+    return self._iterator.initialize()
 
 
 class StandardSingleLossStep(StandardInputStep):
diff --git a/tensorflow/contrib/distribute/python/step_fn_test.py b/tensorflow/contrib/distribute/python/step_fn_test.py
index a77d6d0bec..9f48560b26 100644
--- a/tensorflow/contrib/distribute/python/step_fn_test.py
+++ b/tensorflow/contrib/distribute/python/step_fn_test.py
@@ -46,10 +46,11 @@ class SingleLossStepTest(test.TestCase, parameterized.TestCase):
           optimizer_fn, distribution, use_bias=True, iterations_per_step=2)
 
       if context.executing_eagerly():
+        single_loss_step.initialize()
         run_step = single_loss_step
       else:
         with self.cached_session() as sess:
-          sess.run(single_loss_step._iterator.initializer)
+          sess.run(single_loss_step.initialize())
           run_step = sess.make_callable(single_loss_step())
       self.evaluate(variables.global_variables_initializer())
 
-- 
GitLab


From 890a58c43242becfc29f0bfe6c5100b5f7faea94 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 17 Jan 2019 08:46:54 -0800
Subject: [PATCH 0870/1095] s/AutographParseError/AutoGraphParseError/
 (properly camel-casing AutoGraph.)

PiperOrigin-RevId: 229754536
---
 tensorflow/python/autograph/__init__.py                     | 4 ++--
 tensorflow/python/autograph/converters/control_flow_test.py | 2 +-
 tensorflow/python/autograph/converters/decorators_test.py   | 2 +-
 tensorflow/python/autograph/converters/slices_test.py       | 2 +-
 tensorflow/python/autograph/pyct/transformer.py             | 6 +++---
 5 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/tensorflow/python/autograph/__init__.py b/tensorflow/python/autograph/__init__.py
index 6faeb01607..8fd6e72da6 100644
--- a/tensorflow/python/autograph/__init__.py
+++ b/tensorflow/python/autograph/__init__.py
@@ -50,7 +50,7 @@ from tensorflow.python.autograph.lang.directives import set_element_type
 from tensorflow.python.autograph.lang.directives import set_loop_options
 from tensorflow.python.autograph.lang.special_functions import stack
 from tensorflow.python.autograph.lang.special_functions import tensor_list
-from tensorflow.python.autograph.pyct.transformer import AutographParseError
+from tensorflow.python.autograph.pyct.transformer import AutoGraphParseError
 from tensorflow.python.util.all_util import remove_undocumented
 
 # TODO(mdan): Revisit this list once we finalize the generated code mechanism.
@@ -77,7 +77,7 @@ _allowed_symbols = [
     'stack',
     'tensor_list',
     # Exceptions
-    'AutographParseError',
+    'AutoGraphParseError',
     # Utilities: to be removed
     'utils',
 ]
diff --git a/tensorflow/python/autograph/converters/control_flow_test.py b/tensorflow/python/autograph/converters/control_flow_test.py
index 034fcbe386..1a38d0db4d 100644
--- a/tensorflow/python/autograph/converters/control_flow_test.py
+++ b/tensorflow/python/autograph/converters/control_flow_test.py
@@ -184,7 +184,7 @@ class ControlFlowTest(converter_testing.TestCase):
       return b
 
     node, ctx = self.prepare(test_fn, {})
-    with self.assertRaises(transformer.AutographParseError):
+    with self.assertRaises(transformer.AutoGraphParseError):
       control_flow.transform(node, ctx)
 
   @test_util.run_deprecated_v1
diff --git a/tensorflow/python/autograph/converters/decorators_test.py b/tensorflow/python/autograph/converters/decorators_test.py
index abd76849d6..bcf502c62b 100644
--- a/tensorflow/python/autograph/converters/decorators_test.py
+++ b/tensorflow/python/autograph/converters/decorators_test.py
@@ -121,7 +121,7 @@ class DecoratorsTest(converter_testing.TestCase):
       return inner_fn(a)
 
     # Expected to fail because simple_decorator could not be imported.
-    with self.assertRaises(transformer.AutographParseError):
+    with self.assertRaises(transformer.AutoGraphParseError):
       test_fn(1)
 
   def test_nested_decorators_imported(self):
diff --git a/tensorflow/python/autograph/converters/slices_test.py b/tensorflow/python/autograph/converters/slices_test.py
index bd049afdfc..d674f266c8 100644
--- a/tensorflow/python/autograph/converters/slices_test.py
+++ b/tensorflow/python/autograph/converters/slices_test.py
@@ -68,7 +68,7 @@ class SliceTest(converter_testing.TestCase):
     def_.directives[directives.set_element_type] = {
         'dtype': parser.parse_expression('tf.float32')
     }
-    with self.assertRaises(transformer.AutographParseError):
+    with self.assertRaises(transformer.AutoGraphParseError):
       slices.transform(node, ctx)
 
 
diff --git a/tensorflow/python/autograph/pyct/transformer.py b/tensorflow/python/autograph/pyct/transformer.py
index b6830534b3..9c776a737a 100644
--- a/tensorflow/python/autograph/pyct/transformer.py
+++ b/tensorflow/python/autograph/pyct/transformer.py
@@ -29,7 +29,7 @@ from tensorflow.python.autograph.pyct import pretty_printer
 from tensorflow.python.autograph.pyct import templates
 
 
-class AutographParseError(SyntaxError):
+class AutoGraphParseError(SyntaxError):
   pass
 
 
@@ -495,8 +495,8 @@ class Base(gast.NodeTransformer):
       # In other words, we need to find how to suppress the "During handling
       # of the above exception, another exception occurred" message.
       six.reraise(
-          AutographParseError,
-          AutographParseError(msg, (original_file_path, original_line_number,
+          AutoGraphParseError,
+          AutoGraphParseError(msg, (original_file_path, original_line_number,
                                     original_col_offset, original_source_line)),
           sys.exc_info()[2])
     finally:
-- 
GitLab


From a782c9df3e9c2cae1b49326f9c9b72eac1dfd794 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 17 Jan 2019 08:53:44 -0800
Subject: [PATCH 0871/1095] Replace --genrule_strategy w/ --strategy=Genrule

This is in preparation for --genrule_strategy becoming a no-op in a future Bazel release.

PiperOrigin-RevId: 229755652
---
 .bazelrc                                   | 2 +-
 tensorflow/tools/ci_build/install/.bazelrc | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.bazelrc b/.bazelrc
index d5432aeb17..c70c571361 100644
--- a/.bazelrc
+++ b/.bazelrc
@@ -80,7 +80,7 @@ build --define=use_fast_cpp_protos=true
 build --define=allow_oversize_protos=true
 
 build --spawn_strategy=standalone
-build --genrule_strategy=standalone
+build --strategy=Genrule=standalone
 build -c opt
 
 # Other build flags.
diff --git a/tensorflow/tools/ci_build/install/.bazelrc b/tensorflow/tools/ci_build/install/.bazelrc
index 2060babd4a..4662e2e60a 100644
--- a/tensorflow/tools/ci_build/install/.bazelrc
+++ b/tensorflow/tools/ci_build/install/.bazelrc
@@ -5,7 +5,7 @@ startup --batch
 
 # Similarly, we need to workaround sandboxing issues:
 #   https://github.com/bazelbuild/bazel/issues/418
-build  --verbose_failures --spawn_strategy=standalone --genrule_strategy=standalone
+build  --verbose_failures --spawn_strategy=standalone --strategy=Genrule=standalone
 test --spawn_strategy=standalone
 
 # Force bazel output to use colors (good for jenkins) and print useful errors.
-- 
GitLab


From c9bd0744ea3d527503e7cc2e44c7306f0d25e421 Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Thu, 17 Jan 2019 09:01:33 -0800
Subject: [PATCH 0872/1095] MOve backend config related functions to a
 dedicated file. Use backend epsilon if epsilon argument is None.

PiperOrigin-RevId: 229756902
---
 tensorflow/python/keras/BUILD                 |  19 +++
 tensorflow/python/keras/backend.py            | 143 ++----------------
 tensorflow/python/keras/backend_config.py     | 126 +++++++++++++++
 .../python/keras/backend_config_test.py       |  55 +++++++
 tensorflow/python/keras/backend_test.py       |  18 ---
 tensorflow/python/keras/optimizer_v2/BUILD    |   1 +
 .../python/keras/optimizer_v2/adadelta.py     |   3 +
 .../keras/optimizer_v2/adadelta_test.py       |   9 ++
 .../python/keras/optimizer_v2/adagrad.py      |   3 +
 .../python/keras/optimizer_v2/adagrad_test.py |  13 ++
 tensorflow/python/keras/optimizer_v2/adam.py  |   3 +
 .../python/keras/optimizer_v2/adam_test.py    |   9 ++
 .../python/keras/optimizer_v2/adamax.py       |   3 +
 .../python/keras/optimizer_v2/adamax_test.py  |   9 ++
 tensorflow/python/keras/optimizer_v2/nadam.py |   3 +
 .../python/keras/optimizer_v2/nadam_test.py   |   9 ++
 .../python/keras/optimizer_v2/rmsprop.py      |   3 +
 .../python/keras/optimizer_v2/rmsprop_test.py |   9 ++
 18 files changed, 287 insertions(+), 151 deletions(-)
 create mode 100644 tensorflow/python/keras/backend_config.py
 create mode 100644 tensorflow/python/keras/backend_config_test.py

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 9e08c766db..c4c403119c 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -82,6 +82,7 @@ py_library(
     srcs = ["backend.py"],
     srcs_version = "PY2AND3",
     deps = [
+        ":backend_config",
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:check_ops",
@@ -117,6 +118,12 @@ py_library(
     ],
 )
 
+py_library(
+    name = "backend_config",
+    srcs = ["backend_config.py"],
+    srcs_version = "PY2AND3",
+)
+
 py_library(
     name = "engine",
     srcs = [
@@ -1108,6 +1115,18 @@ tf_py_test(
     ],
 )
 
+tf_py_test(
+    name = "backend_config_test",
+    size = "medium",
+    srcs = ["backend_config_test.py"],
+    additional_deps = [
+        ":keras",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:util",
+    ],
+)
+
 tf_py_test(
     name = "keras_parameterized_test",
     size = "small",
diff --git a/tensorflow/python/keras/backend.py b/tensorflow/python/keras/backend.py
index f357f0efc7..9677f12d34 100644
--- a/tensorflow/python/keras/backend.py
+++ b/tensorflow/python/keras/backend.py
@@ -40,6 +40,7 @@ from tensorflow.python.framework import func_graph
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import tensor_util
+from tensorflow.python.keras import backend_config
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import clip_ops
 from tensorflow.python.ops import control_flow_ops
@@ -97,15 +98,6 @@ _DUMMY_EAGER_GRAPH = _DummyEagerGraph()
 # Change its value via `manual_variable_initialization(value)`.
 _MANUAL_VAR_INIT = False
 
-# The type of float to use throughout a session.
-_FLOATX = 'float32'
-
-# Epsilon fuzz factor used throughout the codebase.
-_EPSILON = 1e-7
-
-# Default image data format, one of "channels_last", "channels_first".
-_IMAGE_DATA_FORMAT = 'channels_last'
-
 # This list holds the available devices.
 # It is populated when `_get_available_gpus()` is called for the first time.
 # We assume our devices don't change henceforth.
@@ -119,6 +111,14 @@ _GRAPH_VARIABLES = weakref.WeakKeyDictionary()
 # the graph.
 _GRAPH_TF_OPTIMIZERS = weakref.WeakKeyDictionary()
 
+# The below functions are kept accessible from backend for compatibility.
+epsilon = backend_config.epsilon
+floatx = backend_config.floatx
+image_data_format = backend_config.image_data_format
+set_epsilon = backend_config.set_epsilon
+set_floatx = backend_config.set_floatx
+set_image_data_format = backend_config.set_image_data_format
+
 
 @keras_export('keras.backend.backend')
 def backend():
@@ -132,87 +132,6 @@ def backend():
   return 'tensorflow'
 
 
-@keras_export('keras.backend.epsilon')
-def epsilon():
-  """Returns the value of the fuzz factor used in numeric expressions.
-
-  Returns:
-      A float.
-
-  Example:
-  ```python
-      >>> keras.backend.epsilon()
-      1e-07
-  ```
-  """
-  return _EPSILON
-
-
-@keras_export('keras.backend.set_epsilon')
-def set_epsilon(value):
-  """Sets the value of the fuzz factor used in numeric expressions.
-
-  Arguments:
-      value: float. New value of epsilon.
-
-  Example:
-  ```python
-      >>> from keras import backend as K
-      >>> K.epsilon()
-      1e-07
-      >>> K.set_epsilon(1e-05)
-      >>> K.epsilon()
-      1e-05
-  ```
-  """
-  global _EPSILON
-  _EPSILON = value
-
-
-@keras_export('keras.backend.floatx')
-def floatx():
-  """Returns the default float type, as a string.
-
-  E.g. 'float16', 'float32', 'float64'.
-
-  Returns:
-      String, the current default float type.
-
-  Example:
-  ```python
-      >>> keras.backend.floatx()
-      'float32'
-  ```
-  """
-  return _FLOATX
-
-
-@keras_export('keras.backend.set_floatx')
-def set_floatx(value):
-  """Sets the default float type.
-
-  Arguments:
-      value: String; 'float16', 'float32', or 'float64'.
-
-  Example:
-  ```python
-      >>> from keras import backend as K
-      >>> K.floatx()
-      'float32'
-      >>> K.set_floatx('float16')
-      >>> K.floatx()
-      'float16'
-  ```
-
-  Raises:
-      ValueError: In case of invalid value.
-  """
-  global _FLOATX
-  if value not in {'float16', 'float32', 'float64'}:
-    raise ValueError('Unknown floatx type: ' + str(value))
-  _FLOATX = str(value)
-
-
 @keras_export('keras.backend.cast_to_floatx')
 def cast_to_floatx(x):
   """Cast a Numpy array to the default Keras float type.
@@ -238,49 +157,7 @@ def cast_to_floatx(x):
       dtype('float32')
   ```
   """
-  return np.asarray(x, dtype=_FLOATX)
-
-
-@keras_export('keras.backend.image_data_format')
-def image_data_format():
-  """Returns the default image data format convention.
-
-  Returns:
-      A string, either `'channels_first'` or `'channels_last'`
-
-  Example:
-  ```python
-      >>> keras.backend.image_data_format()
-      'channels_first'
-  ```
-  """
-  return _IMAGE_DATA_FORMAT
-
-
-@keras_export('keras.backend.set_image_data_format')
-def set_image_data_format(data_format):
-  """Sets the value of the image data format convention.
-
-  Arguments:
-      data_format: string. `'channels_first'` or `'channels_last'`.
-
-  Example:
-  ```python
-      >>> from keras import backend as K
-      >>> K.image_data_format()
-      'channels_first'
-      >>> K.set_image_data_format('channels_last')
-      >>> K.image_data_format()
-      'channels_last'
-  ```
-
-  Raises:
-      ValueError: In case of invalid `data_format` value.
-  """
-  global _IMAGE_DATA_FORMAT
-  if data_format not in {'channels_last', 'channels_first'}:
-    raise ValueError('Unknown data_format: ' + str(data_format))
-  _IMAGE_DATA_FORMAT = str(data_format)
+  return np.asarray(x, dtype=floatx())
 
 
 # A global dictionary mapping graph objects to an index of counters used
diff --git a/tensorflow/python/keras/backend_config.py b/tensorflow/python/keras/backend_config.py
new file mode 100644
index 0000000000..e7c63ac2c7
--- /dev/null
+++ b/tensorflow/python/keras/backend_config.py
@@ -0,0 +1,126 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Keras backend config API."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.util.tf_export import keras_export
+
+# The type of float to use throughout a session.
+_FLOATX = 'float32'
+
+# Epsilon fuzz factor used throughout the codebase.
+_EPSILON = 1e-7
+
+# Default image data format, one of "channels_last", "channels_first".
+_IMAGE_DATA_FORMAT = 'channels_last'
+
+
+@keras_export('keras.backend.epsilon')
+def epsilon():
+  """Returns the value of the fuzz factor used in numeric expressions.
+
+  Returns:
+      A float.
+
+  Example:
+  ```python
+  keras.backend.epsilon() >>>1e-07
+  ```
+  """
+  return _EPSILON
+
+
+@keras_export('keras.backend.set_epsilon')
+def set_epsilon(value):
+  """Sets the value of the fuzz factor used in numeric expressions.
+
+  Arguments:
+      value: float. New value of epsilon.
+  Example: ```python from keras import backend as K K.epsilon() >>> 1e-07
+    K.set_epsilon(1e-05) K.epsilon() >>> 1e-05 ```
+  """
+  global _EPSILON
+  _EPSILON = value
+
+
+@keras_export('keras.backend.floatx')
+def floatx():
+  """Returns the default float type, as a string.
+
+  E.g. 'float16', 'float32', 'float64'.
+
+  Returns:
+      String, the current default float type.
+
+  Example:
+  ```python
+  keras.backend.floatx() >>> 'float32'
+  ```
+  """
+  return _FLOATX
+
+
+@keras_export('keras.backend.set_floatx')
+def set_floatx(value):
+  """Sets the default float type.
+
+  Arguments:
+      value: String; 'float16', 'float32', or 'float64'.
+  Example: ```python from keras import backend as K K.floatx() >>> 'float32'
+    K.set_floatx('float16') K.floatx() >>> 'float16' ```
+
+  Raises:
+      ValueError: In case of invalid value.
+  """
+  global _FLOATX
+  if value not in {'float16', 'float32', 'float64'}:
+    raise ValueError('Unknown floatx type: ' + str(value))
+  _FLOATX = str(value)
+
+
+@keras_export('keras.backend.image_data_format')
+def image_data_format():
+  """Returns the default image data format convention.
+
+  Returns:
+      A string, either `'channels_first'` or `'channels_last'`
+
+  Example:
+  ```python
+  keras.backend.image_data_format() >>> 'channels_first'
+  ```
+  """
+  return _IMAGE_DATA_FORMAT
+
+
+@keras_export('keras.backend.set_image_data_format')
+def set_image_data_format(data_format):
+  """Sets the value of the image data format convention.
+
+  Arguments:
+      data_format: string. `'channels_first'` or `'channels_last'`.
+  Example: ```python from keras import backend as K K.image_data_format() >>>
+    'channels_first' K.set_image_data_format('channels_last')
+    K.image_data_format() >>> 'channels_last' ```
+
+  Raises:
+      ValueError: In case of invalid `data_format` value.
+  """
+  global _IMAGE_DATA_FORMAT
+  if data_format not in {'channels_last', 'channels_first'}:
+    raise ValueError('Unknown data_format: ' + str(data_format))
+  _IMAGE_DATA_FORMAT = str(data_format)
diff --git a/tensorflow/python/keras/backend_config_test.py b/tensorflow/python/keras/backend_config_test.py
new file mode 100644
index 0000000000..59e003d81e
--- /dev/null
+++ b/tensorflow/python/keras/backend_config_test.py
@@ -0,0 +1,55 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for backend_config."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python import keras
+from tensorflow.python.framework import test_util
+from tensorflow.python.platform import test
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class BackendConfigTest(test.TestCase):
+
+  def test_backend(self):
+    self.assertEqual(keras.backend.backend(), 'tensorflow')
+
+  def test_espilon(self):
+    epsilon = 1e-2
+    keras.backend_config.set_epsilon(epsilon)
+    self.assertEqual(keras.backend_config.epsilon(), epsilon)
+    keras.backend_config.set_epsilon(1e-7)
+    self.assertEqual(keras.backend_config.epsilon(), 1e-7)
+
+  def test_floatx(self):
+    floatx = 'float64'
+    keras.backend_config.set_floatx(floatx)
+    self.assertEqual(keras.backend_config.floatx(), floatx)
+    keras.backend_config.set_floatx('float32')
+    self.assertEqual(keras.backend_config.floatx(), 'float32')
+
+  def test_image_data_format(self):
+    image_data_format = 'channels_first'
+    keras.backend_config.set_image_data_format(image_data_format)
+    self.assertEqual(keras.backend_config.image_data_format(),
+                     image_data_format)
+    keras.backend_config.set_image_data_format('channels_last')
+    self.assertEqual(keras.backend_config.image_data_format(), 'channels_last')
+
+
+if __name__ == '__main__':
+  test.main()
diff --git a/tensorflow/python/keras/backend_test.py b/tensorflow/python/keras/backend_test.py
index 4b83f0bf66..e3f86ee2c0 100644
--- a/tensorflow/python/keras/backend_test.py
+++ b/tensorflow/python/keras/backend_test.py
@@ -99,24 +99,6 @@ class BackendUtilsTest(test.TestCase):
   def test_backend(self):
     self.assertEqual(keras.backend.backend(), 'tensorflow')
 
-  def test_espilon(self):
-    epsilon = 1e-2
-    keras.backend.set_epsilon(epsilon)
-    self.assertEqual(keras.backend.epsilon(), epsilon)
-    keras.backend.set_epsilon(1e-7)
-
-  def test_floatx(self):
-    floatx = 'float64'
-    keras.backend.set_floatx(floatx)
-    self.assertEqual(keras.backend.floatx(), floatx)
-    keras.backend.set_floatx('float32')
-
-  def test_image_data_format(self):
-    image_data_format = 'channels_first'
-    keras.backend.set_image_data_format(image_data_format)
-    self.assertEqual(keras.backend.image_data_format(), image_data_format)
-    keras.backend.set_image_data_format('channels_last')
-
   def test_get_reset_uids(self):
     self.assertEqual(keras.backend.get_uid('foo'), 1)
     self.assertEqual(keras.backend.get_uid('foo'), 2)
diff --git a/tensorflow/python/keras/optimizer_v2/BUILD b/tensorflow/python/keras/optimizer_v2/BUILD
index da757edc74..45afe2a134 100644
--- a/tensorflow/python/keras/optimizer_v2/BUILD
+++ b/tensorflow/python/keras/optimizer_v2/BUILD
@@ -35,6 +35,7 @@ py_library(
         "//tensorflow/python:variables",
         "//tensorflow/python/distribute:reduce_util",
         "//tensorflow/python/distribute:values",
+        "//tensorflow/python/keras:backend_config",
     ],
 )
 
diff --git a/tensorflow/python/keras/optimizer_v2/adadelta.py b/tensorflow/python/keras/optimizer_v2/adadelta.py
index b6cba263bc..a3d5538ea8 100644
--- a/tensorflow/python/keras/optimizer_v2/adadelta.py
+++ b/tensorflow/python/keras/optimizer_v2/adadelta.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 
 import numpy as np
 
+from tensorflow.python.keras import backend_config
 from tensorflow.python.keras.optimizer_v2 import optimizer_v2
 from tensorflow.python.training import training_ops
 from tensorflow.python.util.tf_export import keras_export
@@ -90,6 +91,8 @@ class Adadelta(optimizer_v2.OptimizerV2):
     invocations of optimizer functions.
     @end_compatibility
     """
+    if epsilon is None:
+      epsilon = backend_config.epsilon()
     super(Adadelta, self).__init__(name, **kwargs)
     self._set_hyper('learning_rate', kwargs.get('lr', learning_rate))
     self._set_hyper('decay', self._initial_decay)
diff --git a/tensorflow/python/keras/optimizer_v2/adadelta_test.py b/tensorflow/python/keras/optimizer_v2/adadelta_test.py
index 442a1d3189..06ff975212 100644
--- a/tensorflow/python/keras/optimizer_v2/adadelta_test.py
+++ b/tensorflow/python/keras/optimizer_v2/adadelta_test.py
@@ -181,6 +181,15 @@ class AdadeltaOptimizerTest(test.TestCase):
     self.assertAllClose(self.evaluate(opt_2.lr), (1.0))
     self.assertAllClose(self.evaluate(opt_3.lr), (0.1))
 
+  def testConstructAdadeltaWithEpsilonValues(self):
+    opt = adadelta.Adadelta(epsilon=None)
+    config = opt.get_config()
+    self.assertEqual(config["epsilon"], 1e-7)
+
+    opt = adadelta.Adadelta(epsilon=1e-8)
+    config = opt.get_config()
+    self.assertEqual(config["epsilon"], 1e-8)
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/keras/optimizer_v2/adagrad.py b/tensorflow/python/keras/optimizer_v2/adagrad.py
index 01c722521f..0840aa6fae 100644
--- a/tensorflow/python/keras/optimizer_v2/adagrad.py
+++ b/tensorflow/python/keras/optimizer_v2/adagrad.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python.framework import ops
+from tensorflow.python.keras import backend_config
 from tensorflow.python.keras.optimizer_v2 import optimizer_v2
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import init_ops
@@ -89,6 +90,8 @@ class Adagrad(optimizer_v2.OptimizerV2):
     if initial_accumulator_value < 0.0:
       raise ValueError('initial_accumulator_value must be non-negative: %s' %
                        initial_accumulator_value)
+    if epsilon is None:
+      epsilon = backend_config.epsilon()
     if epsilon < 1e-7:
       raise ValueError('epsilon must be larger than 1e-7: %s' % epsilon)
     super(Adagrad, self).__init__(name, **kwargs)
diff --git a/tensorflow/python/keras/optimizer_v2/adagrad_test.py b/tensorflow/python/keras/optimizer_v2/adagrad_test.py
index 1599a54745..864aefaf70 100644
--- a/tensorflow/python/keras/optimizer_v2/adagrad_test.py
+++ b/tensorflow/python/keras/optimizer_v2/adagrad_test.py
@@ -411,6 +411,19 @@ class AdagradOptimizerTest(test.TestCase):
     self.assertAllClose(self.evaluate(opt_2.lr), (1.0))
     self.assertAllClose(self.evaluate(opt_3.lr), (0.1))
 
+  def testConstructAdagradWithEpsilonValues(self):
+    opt = adagrad.Adagrad(epsilon=None)
+    config = opt.get_config()
+    self.assertEqual(config["epsilon"], 1e-7)
+
+    opt = adagrad.Adagrad(epsilon=1e-6)
+    config = opt.get_config()
+    self.assertEqual(config["epsilon"], 1e-6)
+
+    with self.assertRaisesRegexp(ValueError,
+                                 "epsilon must be larger than 1e-7"):
+      opt = adagrad.Adagrad(epsilon=1e-8)
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/keras/optimizer_v2/adam.py b/tensorflow/python/keras/optimizer_v2/adam.py
index 263004c9df..4fa7c73615 100644
--- a/tensorflow/python/keras/optimizer_v2/adam.py
+++ b/tensorflow/python/keras/optimizer_v2/adam.py
@@ -18,6 +18,7 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.framework import ops
+from tensorflow.python.keras import backend_config
 from tensorflow.python.keras.optimizer_v2 import optimizer_v2
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
@@ -131,6 +132,8 @@ class Adam(optimizer_v2.OptimizerV2):
         compatibility, recommended to use `learning_rate` instead.
     """
 
+    if epsilon is None:
+      epsilon = backend_config.epsilon()
     super(Adam, self).__init__(name, **kwargs)
     self._set_hyper('learning_rate', kwargs.get('lr', learning_rate))
     self._set_hyper('decay', self._initial_decay)
diff --git a/tensorflow/python/keras/optimizer_v2/adam_test.py b/tensorflow/python/keras/optimizer_v2/adam_test.py
index 837d0220e1..7918c09b7e 100644
--- a/tensorflow/python/keras/optimizer_v2/adam_test.py
+++ b/tensorflow/python/keras/optimizer_v2/adam_test.py
@@ -516,6 +516,15 @@ class AdamOptimizerTest(test.TestCase):
     self.assertAllClose(self.evaluate(opt_2.lr), (1.0))
     self.assertAllClose(self.evaluate(opt_3.lr), (0.1))
 
+  def testConstructAdamWithEpsilonValues(self):
+    opt = adam.Adam(epsilon=None)
+    config = opt.get_config()
+    self.assertEqual(config["epsilon"], 1e-7)
+
+    opt = adam.Adam(epsilon=1e-8)
+    config = opt.get_config()
+    self.assertEqual(config["epsilon"], 1e-8)
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/keras/optimizer_v2/adamax.py b/tensorflow/python/keras/optimizer_v2/adamax.py
index 9a1b63f6c6..3102e28cff 100644
--- a/tensorflow/python/keras/optimizer_v2/adamax.py
+++ b/tensorflow/python/keras/optimizer_v2/adamax.py
@@ -19,6 +19,7 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.framework import ops
+from tensorflow.python.keras import backend_config
 from tensorflow.python.keras.optimizer_v2 import optimizer_v2
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
@@ -95,6 +96,8 @@ class Adamax(optimizer_v2.OptimizerV2):
         allow time inverse decay of learning rate. `lr` is included for backward
         compatibility, recommended to use `learning_rate` instead.
     """
+    if epsilon is None:
+      epsilon = backend_config.epsilon()
     super(Adamax, self).__init__(name, **kwargs)
     self._set_hyper('learning_rate', kwargs.get('lr', learning_rate))
     self._set_hyper('decay', self._initial_decay)
diff --git a/tensorflow/python/keras/optimizer_v2/adamax_test.py b/tensorflow/python/keras/optimizer_v2/adamax_test.py
index fe658761c7..6934f1590e 100644
--- a/tensorflow/python/keras/optimizer_v2/adamax_test.py
+++ b/tensorflow/python/keras/optimizer_v2/adamax_test.py
@@ -375,6 +375,15 @@ class AdamaxOptimizerTest(test.TestCase):
     self.assertAllClose(self.evaluate(opt_2.lr), (1.0))
     self.assertAllClose(self.evaluate(opt_3.lr), (0.1))
 
+  def testConstructAdamaxWithEpsilonValues(self):
+    opt = adamax.Adamax(epsilon=None)
+    config = opt.get_config()
+    self.assertEqual(config["epsilon"], 1e-7)
+
+    opt = adamax.Adamax(epsilon=1e-8)
+    config = opt.get_config()
+    self.assertEqual(config["epsilon"], 1e-8)
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/keras/optimizer_v2/nadam.py b/tensorflow/python/keras/optimizer_v2/nadam.py
index 4c94b1358a..d515f98725 100644
--- a/tensorflow/python/keras/optimizer_v2/nadam.py
+++ b/tensorflow/python/keras/optimizer_v2/nadam.py
@@ -18,6 +18,7 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.framework import ops
+from tensorflow.python.keras import backend_config
 from tensorflow.python.keras.optimizer_v2 import optimizer_v2
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
@@ -85,6 +86,8 @@ class Nadam(optimizer_v2.OptimizerV2):
 
     # Backwards compatiblity with keras NAdam optimizer.
     kwargs['decay'] = kwargs.pop('schedule_decay', 0.004)
+    if epsilon is None:
+      epsilon = backend_config.epsilon()
     super(Nadam, self).__init__(name, **kwargs)
     self._set_hyper('learning_rate', kwargs.get('lr', learning_rate))
     self._set_hyper('decay', self._initial_decay)
diff --git a/tensorflow/python/keras/optimizer_v2/nadam_test.py b/tensorflow/python/keras/optimizer_v2/nadam_test.py
index b877a80cd8..8dd61956f6 100644
--- a/tensorflow/python/keras/optimizer_v2/nadam_test.py
+++ b/tensorflow/python/keras/optimizer_v2/nadam_test.py
@@ -178,6 +178,15 @@ class NadamOptimizerTest(test.TestCase):
     self.evaluate(variables.global_variables_initializer())
     self.assertAllClose(self.evaluate(opt.decay), (0.2))
 
+  def testConstructNAdamWithEpsilonValues(self):
+    opt = nadam.Nadam(epsilon=None)
+    config = opt.get_config()
+    self.assertEqual(config["epsilon"], 1e-7)
+
+    opt = nadam.Nadam(epsilon=1e-8)
+    config = opt.get_config()
+    self.assertEqual(config["epsilon"], 1e-8)
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/keras/optimizer_v2/rmsprop.py b/tensorflow/python/keras/optimizer_v2/rmsprop.py
index d459d9811e..e55e6375a3 100644
--- a/tensorflow/python/keras/optimizer_v2/rmsprop.py
+++ b/tensorflow/python/keras/optimizer_v2/rmsprop.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python.framework import ops
+from tensorflow.python.keras import backend_config
 from tensorflow.python.keras.optimizer_v2 import optimizer_v2
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
@@ -102,6 +103,8 @@ class RMSprop(optimizer_v2.OptimizerV2):
         allow time inverse decay of learning rate. `lr` is included for backward
         compatibility, recommended to use `learning_rate` instead.
     """
+    if epsilon is None:
+      epsilon = backend_config.epsilon()
     super(RMSprop, self).__init__(name, **kwargs)
     self._set_hyper("learning_rate", kwargs.get("lr", learning_rate))
     self._set_hyper("decay", self._initial_decay)
diff --git a/tensorflow/python/keras/optimizer_v2/rmsprop_test.py b/tensorflow/python/keras/optimizer_v2/rmsprop_test.py
index d4bc7ce50c..a9ddc2155a 100644
--- a/tensorflow/python/keras/optimizer_v2/rmsprop_test.py
+++ b/tensorflow/python/keras/optimizer_v2/rmsprop_test.py
@@ -471,6 +471,15 @@ class RMSpropOptimizerTest(test.TestCase):
       self.assertEqual(
           self.evaluate(opt.variables()[0]), self.evaluate(opt.iterations))
 
+  def testConstructRMSpropWithEpsilonValues(self):
+    opt = rmsprop.RMSprop(epsilon=None)
+    config = opt.get_config()
+    self.assertEqual(config["epsilon"], 1e-7)
+
+    opt = rmsprop.RMSprop(epsilon=1e-8)
+    config = opt.get_config()
+    self.assertEqual(config["epsilon"], 1e-8)
+
 
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From 53c44a9ac4e69c788ef5b707d21c0f364eedb9a8 Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Thu, 17 Jan 2019 09:06:26 -0800
Subject: [PATCH 0873/1095] Make a real repro for IConstantLayer: we need to
 call getType() before getDimensions(), then if CreateConstantLayer() doesn't
 set the dtype, the test will fail.

Before CreateConstantLayer() was added, graphs which trigger conversion of INT32
weights to IConstantLayer will crash the TRT conversion. This is a bug in TRT
5.0 and I has reported it to NVIDIA. CreateConstantLayer() fixed that by setting
the dtype of the output ITensor of the IConstantLayer, but the added test could
not reproduce the error when we remove the setting code. This change fix the
test, so when removing the dtype setting logic the test will fail.

Note that, after this fix, during conversion it'll report a lot of spammy logs
but won't crash. Once NVIDIA fixed the TRT bug we should remove the dtype
setting logic and the test should still pass.

PiperOrigin-RevId: 229758150
---
 tensorflow/contrib/tensorrt/convert/convert_nodes.cc      | 7 ++++---
 tensorflow/contrib/tensorrt/convert/convert_nodes.h       | 3 +++
 tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc | 4 +++-
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index a1c519d908..8b7279ad03 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -340,10 +340,11 @@ nvinfer1::ITensor* Converter::CreateConstantLayer(
   nvinfer1::IConstantLayer* layer = network()->addConstant(dims, trt_weights);
   if (!layer) return nullptr;
   const nvinfer1::DataType trt_dtype = trt_weights.type;
-  // NOTE(laigd): calling layer->setOutputType() does not work. This may be a
-  // bug and we should report to NVIDIA.
-  // layer->setOutputType(0, trt_dtype);
   nvinfer1::ITensor* trt_tensor = layer->getOutput(0);
+  // TODO(laigd): there is a bug in TensorRT 5.0 library that, if we don't set
+  // the data type below, it will always be kFLOAT regardless what the data type
+  // of the weights is. Once NVIDIA fixes this bug, we should remove the data
+  // type setting logic below and test should still pass.
   trt_tensor->setType(trt_dtype);
   return trt_tensor;
 }
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.h b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
index 25e1c42fdf..a02a52e430 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.h
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
@@ -158,7 +158,10 @@ class OutputEdgeValidator {
   bool operator()(const tensorflow::Edge* out_edge) const;
 };
 
+string DebugString(const nvinfer1::DimensionType type);
+string DebugString(const nvinfer1::DataType trt_dtype);
 string DebugString(const nvinfer1::Dims& dims);
+string DebugString(const nvinfer1::Permutation& permutation, int len);
 string DebugString(const nvinfer1::ITensor& tensor);
 int64_t TrtDimsNumElements(const nvinfer1::Dims& dims);
 
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
index 36dbbe3f6f..f90cdac359 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
@@ -922,8 +922,10 @@ TEST_F(ConverterTest, CreateConstantLayer) {
     nvinfer1::ITensor* tensor =
         converter_->CreateConstantLayer(weights, GetTestDims({3, 10}));
     ASSERT_NE(nullptr, tensor);
+    EXPECT_EQ(TfDataTypeToTrt(dtype), tensor->getType())
+        << "Expected " << DebugString(TfDataTypeToTrt(dtype)) << " vs. actual "
+        << DebugString(tensor->getType());
     ExpectTrtDimsEqualsArray({3, 10}, tensor->getDimensions());
-    EXPECT_EQ(TfDataTypeToTrt(dtype), tensor->getType());
   }
 }
 
-- 
GitLab


From ac7f91ee3cd48c01f9ff253440e81c36a729526a Mon Sep 17 00:00:00 2001
From: Vojtech Bardiovsky <vbardiovsky@google.com>
Date: Thu, 17 Jan 2019 09:14:03 -0800
Subject: [PATCH 0874/1095] Make serialization work with kwargs.

Additionally:
1) Unify the way how concrete functions and concrete functions from signatures are serialized. This can be achieved by using signature from the concrete_function.structured_input_signature field instead of constructing it manually.
2) Add a test for serializing a concrete function, then restoring it and calling with a positional argument.
PiperOrigin-RevId: 229759347
---
 tensorflow/python/eager/def_function_test.py  | 18 +++++----
 tensorflow/python/eager/function.py           | 13 +++++--
 .../eager/function_argument_naming_test.py    |  8 ++--
 tensorflow/python/framework/func_graph.py     | 39 +++++++++++++++----
 .../saved_model/function_deserialization.py   | 32 ++++++++-------
 .../saved_model/function_serialization.py     | 28 +++----------
 tensorflow/python/saved_model/load_test.py    | 39 +++++++++++++++++++
 7 files changed, 115 insertions(+), 62 deletions(-)

diff --git a/tensorflow/python/eager/def_function_test.py b/tensorflow/python/eager/def_function_test.py
index 256ef511a3..e0a82bb0be 100644
--- a/tensorflow/python/eager/def_function_test.py
+++ b/tensorflow/python/eager/def_function_test.py
@@ -247,10 +247,9 @@ class DefFunctionTest(test.TestCase):
     concrete = compute.get_concrete_function(
         tensor_spec.TensorSpec(None, dtypes.float32))
     self.assertAllClose(4., concrete(constant_op.constant(2.)))
-    input_signature, _ = concrete.structured_input_signature
-    self.assertEqual(
-        tuple(input_signature),
-        (tensor_spec.TensorSpec(None, dtypes.float32),))
+    signature_args, _ = concrete.structured_input_signature
+    self.assertEqual(signature_args,
+                     (tensor_spec.TensorSpec(None, dtypes.float32),))
 
   def test_serialization_signature_cache(self):
 
@@ -261,12 +260,15 @@ class DefFunctionTest(test.TestCase):
     f(constant_op.constant([[3., 4.]]), constant_op.constant([2.]))
     f(constant_op.constant([[3, 4, 5]]), constant_op.constant([2]))
 
+    signatures_args = set()
     concrete_functions = f._list_all_concrete_functions_for_serialization()
-    signatures_for_serialization = [
-        c.structured_input_signature[0] for c in concrete_functions
-    ]
+    for concrete_function in concrete_functions:
+      args, kwargs = concrete_function.structured_input_signature
+      signatures_args.add(args)
+      self.assertEqual(dict(), kwargs)
+
     self.assertEqual(
-        set(signatures_for_serialization),
+        signatures_args,
         set(((tensor_spec.TensorSpec([1, 2], dtypes.float32),
               tensor_spec.TensorSpec([1], dtypes.float32)),
              (tensor_spec.TensorSpec([1, 3], dtypes.int32),
diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index 82dea51c29..d17c7020f3 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -1306,10 +1306,15 @@ class Function(object):
           arglen = len(args)
         else:
           arglen = len(self._input_signature)
-        arg_names = (
-            self._function_spec.arg_names[:arglen]
-            + [self._function_spec.vararg_name] *
-            (arglen - len(self._function_spec.arg_names)))
+        base_arg_names = self._function_spec.arg_names[:arglen]
+        num_missing_args = arglen - len(self._function_spec.arg_names)
+        missing_arg_names = [self._function_spec.vararg_name] * num_missing_args
+        # Produce a list of missing args of the form ["arg_0", "arg_1", ...],
+        # where arg is based on the self._function_spec.vararg_name.
+        missing_arg_names = [
+            "%s_%d" % (arg, i) for i, arg in enumerate(missing_arg_names)
+        ]
+        arg_names = base_arg_names + missing_arg_names
         graph_function = ConcreteFunction(
             func_graph_module.func_graph_from_py_func(
                 self._name,
diff --git a/tensorflow/python/eager/function_argument_naming_test.py b/tensorflow/python/eager/function_argument_naming_test.py
index 9358c4fd07..08a50a8f51 100644
--- a/tensorflow/python/eager/function_argument_naming_test.py
+++ b/tensorflow/python/eager/function_argument_naming_test.py
@@ -220,10 +220,10 @@ class ArgumentNamingTests(test.TestCase, parameterized.TestCase):
         z=tensor_spec.TensorSpec(shape=(), dtype=dtypes.float32),
         zz=tensor_spec.TensorSpec(shape=(), dtype=dtypes.float32, name='cust'))
     self.assertEqual(
-        ['x', 'y', 'args', 'second_variadic', 'z', 'cust'],
+        ['x', 'y', 'args_1', 'second_variadic', 'z', 'cust'],
         [inp.op.name for inp in variadic_op.inputs])
     self.assertEqual(
-        [b'x', b'y', b'args', b'second_variadic', b'z', b'cust'],
+        [b'x', b'y', b'args_1', b'second_variadic', b'z', b'cust'],
         [inp.op.get_attr('_user_specified_name')
          for inp in variadic_op.inputs])
 
@@ -244,10 +244,10 @@ class ArgumentNamingTests(test.TestCase, parameterized.TestCase):
     variadic_op = variadic_fn.get_concrete_function()
     self.assertIn(b'variadic_fn', variadic_op.name)
     self.assertEqual(
-        ['x', 'y', 'args', 'z'],
+        ['x', 'y', 'args_1', 'z'],
         [inp.op.name for inp in variadic_op.inputs])
     self.assertEqual(
-        [b'x', b'y', b'args', b'z'],
+        [b'x', b'y', b'args_1', b'z'],
         [inp.op.get_attr('_user_specified_name')
          for inp in variadic_op.inputs])
 
diff --git a/tensorflow/python/framework/func_graph.py b/tensorflow/python/framework/func_graph.py
index 5202e9b8f9..36f7aa67c7 100644
--- a/tensorflow/python/framework/func_graph.py
+++ b/tensorflow/python/framework/func_graph.py
@@ -63,11 +63,20 @@ class UnknownArgument(object):
   pass
 
 
-def convert_structure_to_signature(structure):
+# TODO(vbardiovsky): Remove this when nest is updated with new
+# flatten_with_tuple_paths.
+def flatten_with_tuple_paths(structure):
+  return list(zip(nest.yield_flat_paths(structure), nest.flatten(structure)))
+
+
+def convert_structure_to_signature(structure, arg_names=None):
   """Convert a potentially nested structure to a signature.
 
   Args:
-    structure: Structure to convert.
+    structure: Structure to convert, where top level collection is a list or a
+      tuple.
+    arg_names: Optional list of arguments that has equal number of elements as
+      `structure` and is used for naming corresponding TensorSpecs.
 
   Returns:
     Identical structure that has TensorSpec objects instead of Tensors and
@@ -84,8 +93,19 @@ def convert_structure_to_signature(structure):
 
   # We are using the flattened paths to name the TensorSpecs. We need an
   # explicit name for them downstream.
-  flattened_with_paths = nest.flatten_with_joined_string_paths(structure)
-  mapped = [encode_arg(arg, path) for path, arg in flattened_with_paths]
+  flattened = flatten_with_tuple_paths(structure)
+  if arg_names:
+    if len(arg_names) != len(structure):
+      raise ValueError(
+          "Passed in arg_names don't match actual signature (%s)." % arg_names)
+    # Replace all top-level names with their actual arg_names. If a path before
+    # was "(2,'a',1)", it will become "(arg_names[2],'a',1)".
+    flattened = [((arg_names[0],) + path[1:], arg) for path, arg in flattened]
+
+  mapped = [
+      encode_arg(arg, "/".join([str(p) for p in path]))
+      for path, arg in flattened
+  ]
   return nest.pack_sequence_as(structure, mapped)
 
 
@@ -100,9 +120,9 @@ class FuncGraph(ops.Graph):
       inputs coming first.
     outputs: Tensors that will be returned by this function. The tensors are in
       this FuncGraph.
-    structured_input_signature: A possibly-nested python object that was
-      received by this function. Note that this structure might contain Python
-      `None`s.
+    structured_input_signature: A tuple of (args, kwargs), which are both
+      possibly-nested python objects that were received by this function. Note
+      that these structures might contain Python `None`s.
     structured_outputs: A possibly-nested python object which will be returned
       by this function. The Tensors in this structure are the same as those of
       self.outputs. Note that this structure might contain Python `None`s.
@@ -443,8 +463,11 @@ def func_graph_from_py_func(name,
     func_kwargs = _get_defun_inputs_from_kwargs(kwargs)
 
     # Convert all Tensors into TensorSpecs before saving the structured inputs.
+    # If storing pure concrete functions that are not called through polymorphic
+    # functions, we don't have access to FunctionSpec, so we need to call the
+    # TensorSpecs by their `arg_names` for later binding.
     func_graph.structured_input_signature = (
-        convert_structure_to_signature(func_args),
+        convert_structure_to_signature(func_args, arg_names),
         convert_structure_to_signature(func_kwargs))
 
     # Note: `nest.flatten` sorts by keys, as does `_deterministic_dict_values`.
diff --git a/tensorflow/python/saved_model/function_deserialization.py b/tensorflow/python/saved_model/function_deserialization.py
index 5479bbb355..ec5214824e 100644
--- a/tensorflow/python/saved_model/function_deserialization.py
+++ b/tensorflow/python/saved_model/function_deserialization.py
@@ -73,17 +73,23 @@ def _deserialize_function_spec(function_spec_proto, coder):
 
 def recreate_concrete_function(saved_concrete_function, concrete_functions):
   """Recreates a user-facing concrete function."""
+  coder = nested_structure_coder.StructureCoder()
+
   concrete_function = concrete_functions[saved_concrete_function.name]
-  input_signature = (saved_concrete_function.canonicalized_input_signature
-                     .list_value.values)
+  input_signature = coder.decode_proto(
+      saved_concrete_function.canonicalized_input_signature)
+  input_signature_args, input_signature_kwargs = input_signature
+  if input_signature_kwargs:
+    raise ValueError("Restoring concrete function with non-empty kwargs (%s)." %
+                     input_signature_kwargs)
+
   # pylint: disable=protected-access
   # Set metadata required for the concrete function to accept keyword and
   # positional arguments in __call__. Normally this is set in
   # get_concrete_function.
-  concrete_function._arg_keywords = [
-      spec.tensor_spec_value.name for spec in input_signature]
+  concrete_function._arg_keywords = [spec.name for spec in input_signature_args]
   # TODO(allenl): Should we preserve the number of allowed positional arguments?
-  concrete_function._num_positional_args = len(input_signature)
+  concrete_function._num_positional_args = len(input_signature_args)
   # pylint: enable=protected-access
   concrete_function.add_to_graph()
   return concrete_function
@@ -120,25 +126,21 @@ def recreate_function(saved_function, concrete_functions):
           concrete_function.canonicalized_input_signature)
 
       try:
-        can_args, can_kwargs = function_spec.canonicalize_function_inputs(
+        canonicalized_inputs = function_spec.canonicalize_function_inputs(
             *args, **kwargs)
-        if can_kwargs:
-          # TODO(vbardiovsky): Enable this along with the structured input and
-          # structured output.
-          raise ValueError(
-              "Received keywords arguments that could not be bound: %s" %
-              kwargs)
       except ValueError:
         continue
 
-      if _inputs_compatible(can_args,
+      if _inputs_compatible(canonicalized_inputs,
                             canonicalized_original_inputs):
-        flattened_inputs = nest.flatten(can_args)
+        flattened_inputs = nest.flatten(canonicalized_inputs)
         filtered_inputs = [t for t in flattened_inputs if _is_tensor(t)]
         return function_obj._call_flat(filtered_inputs)  # pylint: disable=protected-access
 
     raise AssertionError(
-        "Could not find matching function to call for arguments: %s" % (args,))
+        "Could not find matching function to call for args %r and kwargs %r" %
+        (args, kwargs))
+
   return restored_function
 
 
diff --git a/tensorflow/python/saved_model/function_serialization.py b/tensorflow/python/saved_model/function_serialization.py
index c79c61b60c..172ab41dc8 100644
--- a/tensorflow/python/saved_model/function_serialization.py
+++ b/tensorflow/python/saved_model/function_serialization.py
@@ -19,7 +19,6 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.framework import func_graph as func_graph_module
-from tensorflow.python.framework import tensor_spec
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.saved_model import nested_structure_coder
 from tensorflow.python.saved_model import saved_object_graph_pb2
@@ -39,8 +38,7 @@ def _serialize_function_spec(function_spec, coder):
   return proto
 
 
-def _serialize_concrete_function_with_signature(
-    concrete_function, node_ids, signature_args, coder):
+def serialize_concrete_function(concrete_function, node_ids):
   """Build a SavedConcreteFunction."""
   bound_inputs = []
   try:
@@ -53,10 +51,11 @@ def _serialize_concrete_function_with_signature(
         "captures tensor %s which is unsupported or not reachable from root.",
         concrete_function.name, capture)
     return None
+  coder = nested_structure_coder.StructureCoder()
   concrete_function_proto = saved_object_graph_pb2.SavedConcreteFunction()
   concrete_function_proto.name = concrete_function.name
   concrete_function_proto.canonicalized_input_signature.CopyFrom(
-      coder.encode_structure(signature_args))
+      coder.encode_structure(concrete_function.structured_input_signature))
   structured_outputs = func_graph_module.convert_structure_to_signature(
       concrete_function.structured_outputs)
   concrete_function_proto.output_signature.CopyFrom(
@@ -65,20 +64,6 @@ def _serialize_concrete_function_with_signature(
   return concrete_function_proto
 
 
-def serialize_concrete_function(concrete_function, node_ids):
-  """Build a standalone SavedConcreteFunction."""
-  coder = nested_structure_coder.StructureCoder()
-  signature = []
-  # Construct a flat list of TensorSpecs for this concrete function. On load
-  # we'll parse them and recreate the call metadata.
-  for input_tensor, input_keyword in zip(
-      concrete_function.inputs, concrete_function._arg_keywords):  # pylint: disable=protected-access
-    signature.append(tensor_spec.TensorSpec(
-        input_tensor.shape, input_tensor.dtype, name=input_keyword))
-  return _serialize_concrete_function_with_signature(
-      concrete_function, node_ids, signature, coder)
-
-
 def serialize_function(function, node_ids):
   """Build a SavedFunction proto."""
   coder = nested_structure_coder.StructureCoder()
@@ -89,11 +74,8 @@ def serialize_function(function, node_ids):
   all_concrete_functions = \
       function._list_all_concrete_functions_for_serialization()  # pylint: disable=protected-access
   for concrete_function in all_concrete_functions:
-    signature_args, signature_kwargs = \
-        concrete_function.structured_input_signature
-    del signature_kwargs
-    concrete_function_proto = _serialize_concrete_function_with_signature(
-        concrete_function, node_ids, signature_args, coder)
+    concrete_function_proto = serialize_concrete_function(
+        concrete_function, node_ids)
     if concrete_function_proto is not None:
       proto.concrete_function.add().CopyFrom(concrete_function_proto)
   return proto
diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index e3dcbda50a..280a26a0a5 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -284,6 +284,28 @@ class LoadTest(test.TestCase):
     self.assertEqual(7, imported.f(constant_op.constant(2)).numpy())
     self.assertEqual(6, imported.f(constant_op.constant(1), defg=7.0).numpy())
 
+  def test_additional_kwargs(self):
+    def func(x, training=False, **options):
+      del options
+      if training:
+        return 2 * x
+      else:
+        return 7
+
+    root = tracking.AutoCheckpointable()
+    root.f = def_function.function(func)
+
+    x = constant_op.constant(10)
+    self.assertEqual(7, root.f(x, learning_rate=0.5, epochs=3).numpy())
+
+    imported = self.cycle(root)
+
+    with self.assertRaisesRegexp(AssertionError,
+                                 "Could not find matching function to call.*"):
+      imported.f(x, learning_rate=0.5, epochs=4)
+
+    self.assertEqual(7, imported.f(x, learning_rate=0.5, epochs=3).numpy())
+
   def test_member_function(self):
     class CheckpointableWithMember(tracking.AutoCheckpointable):
 
@@ -438,6 +460,23 @@ class LoadTest(test.TestCase):
     self.assertAllEqual([2, 4, 6],
                         imported.f(constant_op.constant([1, 2, 3])).numpy())
 
+  def test_concrete_function_arg_names(self):
+
+    @def_function.function(
+        input_signature=[tensor_spec.TensorSpec([None], dtypes.int32)])
+    def func(x):
+      return 2 * x
+
+    root = tracking.AutoCheckpointable()
+    root.f = func.get_concrete_function()
+
+    self.assertAllEqual([2], root.f(constant_op.constant([1])).numpy())
+
+    imported = self.cycle(root)
+
+    self.assertAllEqual([2, 4, 6],
+                        imported.f(x=constant_op.constant([1, 2, 3])).numpy())
+
   def test_concrete_function_no_signature(self):
     @def_function.function
     def func(x):
-- 
GitLab


From 452907059d5844831b15d4202574a3b6be5d8b91 Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Thu, 17 Jan 2019 10:22:56 -0800
Subject: [PATCH 0875/1095] Cleanup: remove unused code.

PiperOrigin-RevId: 229772701
---
 tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
index f90cdac359..c8739b46c1 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
@@ -364,9 +364,6 @@ TEST(TRT_TensorOrWeights_Test, Basic) {
       EXPECT_EQ(false, ptr->is_tensor());
       EXPECT_EQ(true, ptr->is_weights());
       EXPECT_TRUE(TrtShapedWeightsEquals(weights, ptr->weights()));
-
-      nvinfer1::Dims dims;
-      dims.nbDims = 0;
       ExpectTrtDimsEqualsArray({}, ptr->GetTrtDims());
     }
   }
-- 
GitLab


From a8dbd3a680b06f617baef53dfad47a10e642f0ca Mon Sep 17 00:00:00 2001
From: Mark Daoust <markdaoust@google.com>
Date: Thu, 17 Jan 2019 10:31:28 -0800
Subject: [PATCH 0876/1095] Do not override `steps_per_epoch` if it is not
 None.

Currently if you pass a tf.keras.preprocessing.image.ImageDataGenerator to fit,
with a `steps_per_epoch`, your manual `steps_per_epoch` is overridden with the real value. Fix it to only override the value if the `steps_per_epoch` was not manually set.

PiperOrigin-RevId: 229774626
---
 tensorflow/python/keras/engine/training_generator.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/keras/engine/training_generator.py b/tensorflow/python/keras/engine/training_generator.py
index 23b5d7e8c5..b8defad8b2 100644
--- a/tensorflow/python/keras/engine/training_generator.py
+++ b/tensorflow/python/keras/engine/training_generator.py
@@ -410,7 +410,9 @@ def convert_to_generator_like(data,
       and may be `None` or `[None]`.
     batch_size: Used when creating a generator out of tuples of NumPy arrays or
       EagerTensors.
-    steps_per_epoch: Steps of the generator to run each epoch.
+    steps_per_epoch: Steps of the generator to run each epoch. If `None` the
+      number of steps will be read from the data (for
+      `keras.utils.data_utils.Sequence` types).
     epochs: Total number of epochs to run.
     shuffle: Whether the data should be shuffled.
 
@@ -431,7 +433,8 @@ def convert_to_generator_like(data,
   if data_utils.is_generator_or_sequence(data) or isinstance(
       data, iterator_ops.EagerIterator):
     if isinstance(data, data_utils.Sequence):
-      steps_per_epoch = len(data)
+      if steps_per_epoch is None:
+        steps_per_epoch = len(data)
     return data, steps_per_epoch
   if isinstance(data, dataset_ops.DatasetV2):
     return dataset_ops.make_one_shot_iterator(data), steps_per_epoch
-- 
GitLab


From 5f98ba7eba6538f6cb389fa7326a9549dcd1299a Mon Sep 17 00:00:00 2001
From: Andr? Susano Pinto <andresp@google.com>
Date: Thu, 17 Jan 2019 10:34:59 -0800
Subject: [PATCH 0877/1095] Refactor registering gradient of ConcreteFunction.

PiperOrigin-RevId: 229775288
---
 tensorflow/python/eager/function.py | 25 ++++++++++---------------
 1 file changed, 10 insertions(+), 15 deletions(-)

diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index d17c7020f3..9c3e20f936 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -506,26 +506,25 @@ class ConcreteFunction(object):
     if context.executing_eagerly() or not self.outputs:
       outputs = self._inference_function.call(ctx, args)
     else:
-      if not self._gradient_name:
-        self._gradient_name = "PartitionedCall-%s" % ops.uid()
-        self._register_gradient(self._gradient_name)
+      self._register_gradient()
       with ops.get_default_graph().gradient_override_map(
           {"PartitionedCall": self._gradient_name,
            "StatefulPartitionedCall": self._gradient_name}):
         outputs = self._inference_function.call(ctx, args)
     return self._build_call_outputs(outputs)
 
-  def _register_gradient(self, name):
-    """Registers the gradient for this `ConcreteFunction` under the given name.
+  def _register_gradient(self):
+    """Registers the gradient for this `ConcreteFunction`.
 
     The gradient rewrites an inference call op to a forward call op, but does
     not modify a pre-existing forward call op. It then computes the gradient
     from the output's gradients and the side outputs of the forward op.
-
-    Args:
-      name: The name to register the gradient as.
     """
-    @ops.RegisterGradient(name)
+    if self._gradient_name:
+      return
+    self._gradient_name = "PartitionedCall-%s" % ops.uid()
+
+    @ops.RegisterGradient(self._gradient_name)
     def _registered_grad_fn(op, *doutputs):  # pylint: disable=unused-variable
       return self._grad_fn(op, *doutputs)
 
@@ -724,9 +723,7 @@ class ConcreteFunction(object):
 
     ctx = context.context()
 
-    if not self._gradient_name:
-      self._gradient_name = "PartitionedCall-%s" % ops.uid()
-      self._register_gradient(self._gradient_name)
+    self._register_gradient()
     with ops.get_default_graph().gradient_override_map(
         {"PartitionedCall": self._gradient_name,
          "StatefulPartitionedCall": self._gradient_name}):
@@ -773,9 +770,7 @@ class ConcreteFunction(object):
     """
     ctx = context.context()
 
-    if not self._gradient_name:
-      self._gradient_name = "PartitionedCall-%s" % ops.uid()
-      self._register_gradient(self._gradient_name)
+    self._register_gradient()
     with ops.get_default_graph().gradient_override_map(
         {"PartitionedCall": self._gradient_name,
          "StatefulPartitionedCall": self._gradient_name}):
-- 
GitLab


From 0f39e8a552018eba3bd2916c0a72467617bebf88 Mon Sep 17 00:00:00 2001
From: Haoyu Zhang <haoyuzhang@google.com>
Date: Thu, 17 Jan 2019 10:41:18 -0800
Subject: [PATCH 0878/1095] Reuse existing dataset iterators in Keras for
 better memory efficiency.

* In Keras model iteration, create only one iterator for training dataset and
  one iterator for validation datasets.
* If training requires dataset to be reset at each epoch, reinitialize the
  training iterator;
* If we run multiple validation passes between training epochs, reinitialize the validation
  iterator.

Compared to recreating iterators, this approach reduces host memory usage and
prevents memory leak.

PiperOrigin-RevId: 229776545
---
 .../engine/distributed_training_utils.py      | 10 ++++-
 .../python/keras/engine/training_arrays.py    | 38 ++++++++++++++++---
 .../python/keras/engine/training_utils.py     | 18 +++++++--
 3 files changed, 54 insertions(+), 12 deletions(-)

diff --git a/tensorflow/python/keras/engine/distributed_training_utils.py b/tensorflow/python/keras/engine/distributed_training_utils.py
index 27ad6a5c0b..d13e0b6fa9 100644
--- a/tensorflow/python/keras/engine/distributed_training_utils.py
+++ b/tensorflow/python/keras/engine/distributed_training_utils.py
@@ -34,6 +34,7 @@ from tensorflow.python.keras import callbacks
 from tensorflow.python.keras import metrics as metrics_module
 from tensorflow.python.keras import optimizers
 from tensorflow.python.keras.optimizer_v2 import optimizer_v2
+from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import tf_logging as logging
@@ -528,10 +529,15 @@ def list_to_tuple(maybe_list):
 def get_iterator(dataset, distribution_strategy):
   with distribution_strategy.scope():
     iterator = distribution_strategy.make_dataset_iterator(dataset)
-    init_op = iterator.initialize()
+  initialize_iterator(iterator, distribution_strategy)
+  return iterator
+
+
+def initialize_iterator(iterator, distribution_strategy):
+  with distribution_strategy.scope():
+    init_op = control_flow_ops.group(iterator.initialize())
     if not context.executing_eagerly():
       K.get_session().run(init_op)
-  return iterator
 
 
 def _get_input_from_iterator(iterator, model):
diff --git a/tensorflow/python/keras/engine/training_arrays.py b/tensorflow/python/keras/engine/training_arrays.py
index 3b6a13839a..d0881c11a5 100644
--- a/tensorflow/python/keras/engine/training_arrays.py
+++ b/tensorflow/python/keras/engine/training_arrays.py
@@ -24,6 +24,7 @@ import functools
 import numpy as np
 
 from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.data.ops import iterator_ops
 from tensorflow.python.eager import context
 from tensorflow.python.framework import errors
 from tensorflow.python.keras import backend as K
@@ -119,18 +120,22 @@ def model_iteration(model,
 
   # In case we were passed a dataset, we extract symbolic tensors from it.
   reset_dataset_after_each_epoch = False
-  original_dataset = None
+  input_iterator = None
   is_dataset = isinstance(inputs,
                           (dataset_ops.DatasetV1, dataset_ops.DatasetV2))
   # TODO(fchollet): consider moving `steps_per_epoch` inference to
   # _standardize_user_data and set reset_dataset_after_each_epoch as an
   # attribute on the dataset instance.
   if is_dataset:
-    original_dataset = inputs
     if steps_per_epoch is None:
       reset_dataset_after_each_epoch = True
       steps_per_epoch = training_utils.infer_steps_for_dataset(
           inputs, steps_per_epoch, epochs=epochs, steps_name=steps_name)
+    input_iterator = _get_iterator(inputs, model._distribution_strategy)
+
+  val_iterator = None
+  if isinstance(val_inputs, (dataset_ops.DatasetV1, dataset_ops.DatasetV2)):
+    val_iterator = _get_iterator(val_inputs, model._distribution_strategy)
 
   if mode == ModeKeys.TRAIN:
     _print_train_info(inputs, val_inputs, steps_per_epoch, verbose)
@@ -150,6 +155,7 @@ def model_iteration(model,
       convert_eager_tensors_to_numpy((inputs, targets, sample_weights))
 
   # Prepare input data.
+  inputs = input_iterator or inputs
   ins = _prepare_feed_values(model, inputs, targets, sample_weights, mode)
   if not is_dataset:
     num_samples_or_steps = _get_num_samples_or_steps(ins, batch_size,
@@ -230,7 +236,7 @@ def model_iteration(model,
           actual_inputs = ins() if callable(ins) else ins
           batch_outs = f(actual_inputs)
         except errors.OutOfRangeError:
-          if original_dataset is None:
+          if not is_dataset:
             # We ran out of batches while the user passed an iterator (legacy).
             logging.warning(
                 'Your dataset iterator ran out of data; '
@@ -332,6 +338,7 @@ def model_iteration(model,
     if (do_validation and
         training_utils.should_run_validation(validation_freq, epoch) and
         not callbacks.model.stop_training):
+      val_inputs = val_iterator or val_inputs
       val_results = model_iteration(
           model,
           val_inputs,
@@ -348,15 +355,18 @@ def model_iteration(model,
         val_results = [val_results]
       epoch_logs = cbks.make_logs(
           model, epoch_logs, val_results, mode, prefix='val_')
+      if val_iterator and epoch < epochs - 1:
+        _reinitialize_iterator(val_iterator, model._distribution_strategy)
 
     if mode == ModeKeys.TRAIN:
       # Epochs only apply to `fit`.
       callbacks.on_epoch_end(epoch, epoch_logs)
     progbar.on_epoch_end(epoch, epoch_logs)
 
-    # Recreate dataset iterator for the next epoch.
+    # Reinitialize dataset iterator for the next epoch.
     if reset_dataset_after_each_epoch and epoch < epochs - 1:
-      ins = _prepare_feed_values(model, original_dataset, None, None, mode)
+      _reinitialize_iterator(input_iterator, model._distribution_strategy)
+      ins = _prepare_feed_values(model, input_iterator, None, None, mode)
 
   callbacks._call_end_hook(mode)
 
@@ -430,7 +440,8 @@ def _prepare_feed_values(model, inputs, targets, sample_weights, mode):
     else:
       return get_distributed_inputs()
 
-  if isinstance(inputs, (dataset_ops.DatasetV1, dataset_ops.DatasetV2)):
+  if isinstance(inputs, (dataset_ops.DatasetV1, dataset_ops.DatasetV2,
+                         iterator_ops.Iterator)):
     inputs, targets, sample_weights = model._standardize_user_data(
         inputs,
         extract_tensors_from_dataset=True)
@@ -445,6 +456,21 @@ def _prepare_feed_values(model, inputs, targets, sample_weights, mode):
   return ins
 
 
+def _get_iterator(inputs, distribution_strategy=None):
+  if distribution_strategy:
+    return distributed_training_utils.get_iterator(
+        inputs, distribution_strategy)
+  return training_utils.get_iterator(inputs)
+
+
+def _reinitialize_iterator(iterator, distribution_strategy=None):
+  if distribution_strategy:
+    distributed_training_utils.initialize_iterator(
+        iterator, distribution_strategy)
+  else:
+    training_utils.initialize_iterator(iterator)
+
+
 def _make_execution_function(model, mode):
   """Makes function to run one step of model execution."""
   if model._distribution_strategy:
diff --git a/tensorflow/python/keras/engine/training_utils.py b/tensorflow/python/keras/engine/training_utils.py
index 3affb84147..7e5bc08e5e 100644
--- a/tensorflow/python/keras/engine/training_utils.py
+++ b/tensorflow/python/keras/engine/training_utils.py
@@ -1232,6 +1232,19 @@ def is_dataset_or_iterator(data):
                            iterator_ops.Iterator))
 
 
+def get_iterator(dataset):
+  """Create and initialize an iterator from a dataset."""
+  iterator = dataset_ops.make_initializable_iterator(dataset)
+  initialize_iterator(iterator)
+  return iterator
+
+
+def initialize_iterator(iterator):
+  init_op = iterator.initializer
+  if not context.executing_eagerly():
+    K.get_session().run(init_op)
+
+
 def extract_tensors_from_dataset(dataset):
   """Extract a tuple of tensors `inputs, targets, sample_weight` from a dataset.
 
@@ -1241,10 +1254,7 @@ def extract_tensors_from_dataset(dataset):
   Returns:
     Tuple of tensors `x, y, weights`. `y` and `weights` entry may be None.
   """
-  iterator = dataset_ops.make_initializable_iterator(dataset)
-  init_op = iterator.initializer
-  if not context.executing_eagerly():
-    K.get_session().run(init_op)
+  iterator = get_iterator(dataset)
   inputs, targets, sample_weight = unpack_iterator_input(iterator)
   return inputs, targets, sample_weight
 
-- 
GitLab


From 887bcfa5116a11c1ee534b99cc1c118c29ff7e46 Mon Sep 17 00:00:00 2001
From: Cong Liu <congliu@google.com>
Date: Thu, 17 Jan 2019 10:51:03 -0800
Subject: [PATCH 0879/1095] [XLA] Fix a bug in dynamic parameter binding.

PiperOrigin-RevId: 229778432
---
 tensorflow/compiler/xla/service/dynamic_parameter_binding.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/dynamic_parameter_binding.cc b/tensorflow/compiler/xla/service/dynamic_parameter_binding.cc
index 3d0eddb0d8..5549cccfa8 100644
--- a/tensorflow/compiler/xla/service/dynamic_parameter_binding.cc
+++ b/tensorflow/compiler/xla/service/dynamic_parameter_binding.cc
@@ -70,7 +70,7 @@ StatusOr<DynamicParameterBinding> DynamicParameterBinding::CreateFromProto(
     int64 target_param_num = binding.target_param_num();
     ShapeIndex target_param_index(binding.target_param_index().begin(),
                                   binding.target_param_index().end());
-    int64 target_dim_num = binding.target_param_num();
+    int64 target_dim_num = binding.target_param_dim_num();
 
     TF_RETURN_IF_ERROR(
         result.Bind(DynamicParameter{dynamic_param_num, dynamic_param_index},
-- 
GitLab


From 8f275bdcede1de92ef2ec72324627ab5fc449a21 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 17 Jan 2019 10:54:21 -0800
Subject: [PATCH 0880/1095] Fixes bug preventing cond_v2 from working with
 gradient tapes inside functions.

PiperOrigin-RevId: 229779015
---
 .../python/kernel_tests/cond_v2_test.py       | 22 +++++++++++++++++++
 tensorflow/python/ops/cond_v2.py              | 21 ++++++++++--------
 2 files changed, 34 insertions(+), 9 deletions(-)

diff --git a/tensorflow/python/kernel_tests/cond_v2_test.py b/tensorflow/python/kernel_tests/cond_v2_test.py
index 050da5ff6c..356c6f0a16 100644
--- a/tensorflow/python/kernel_tests/cond_v2_test.py
+++ b/tensorflow/python/kernel_tests/cond_v2_test.py
@@ -20,7 +20,9 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.core.protobuf import config_pb2
+from tensorflow.python.eager import backprop
 from tensorflow.python.eager import context
+from tensorflow.python.eager import def_function
 from tensorflow.python.eager import function
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -628,6 +630,26 @@ class CondV2Test(test.TestCase):
         # d2[x]/dx2 = 0
         self.assertEqual(false_val, [0.0])
 
+  def testGradientTapeOfCondWithResourceVariableInFunction(self):
+    with context.eager_mode():
+      v = variables.Variable(2.)
+
+      @def_function.function
+      def fnWithCond():  # pylint: disable=invalid-name
+        with backprop.GradientTape() as tape:
+          pred = constant_op.constant(True, dtype=dtypes.bool)
+
+          def true_fn():
+            return math_ops.pow(v, 3)
+
+          def false_fn():
+            return v
+
+          cond = cond_v2.cond_v2(pred, true_fn, false_fn, name="cond")
+        return tape.gradient(cond, v)
+
+      self.assertAllEqual(fnWithCond(), 12.0)
+
   def testLowering(self):
     with ops.Graph().as_default() as g:
       with self.session(graph=g) as sess:
diff --git a/tensorflow/python/ops/cond_v2.py b/tensorflow/python/ops/cond_v2.py
index 5dca8e501b..6e8f2df39b 100644
--- a/tensorflow/python/ops/cond_v2.py
+++ b/tensorflow/python/ops/cond_v2.py
@@ -91,11 +91,13 @@ def cond_v2(pred, true_fn, false_fn, name="cond"):
 @ops.RegisterGradient("If")
 def _IfGrad(op, *grads):  # pylint: disable=invalid-name
   """The gradient of an If op produced by cond_v2."""
-  true_graph, false_graph = _get_func_graphs(op)
+  # Get the if operator (this logic handles the case where op is a MockOp)
+  if_op = op.outputs[0].op
+  true_graph, false_graph = _get_func_graphs(if_op)
   # Note: op.graph != ops.get_default_graph() when we are computing the gradient
   # of a nested cond.
-  assert true_graph.outer_graph == op.graph
-  assert false_graph.outer_graph == op.graph
+  assert true_graph.outer_graph == if_op.graph
+  assert false_graph.outer_graph == if_op.graph
 
   # Create grad functions that compute the gradient of the true/false forward
   # graphs. These functions will capture tensors from the forward pass
@@ -140,11 +142,12 @@ def _IfGrad(op, *grads):  # pylint: disable=invalid-name
     true_graph.name += "_rewritten"
     false_graph.name += "_rewritten"
 
-    op._set_func_attr("then_branch", util.create_new_tf_function(true_graph))
-    op._set_func_attr("else_branch", util.create_new_tf_function(false_graph))
-    op._set_type_list_attr("Tout", true_graph.output_types)
-    op._set_shape_list_attr("output_shapes", true_graph.output_shapes)
-    op._add_outputs(
+    if_op._set_func_attr("then_branch", util.create_new_tf_function(true_graph))
+    if_op._set_func_attr("else_branch",
+                         util.create_new_tf_function(false_graph))
+    if_op._set_type_list_attr("Tout", true_graph.output_types)
+    if_op._set_shape_list_attr("output_shapes", true_graph.output_shapes)
+    if_op._add_outputs(
         [t.dtype for t in extra_true_outputs],
         [t.shape for t in extra_true_outputs])
 
@@ -153,7 +156,7 @@ def _IfGrad(op, *grads):  # pylint: disable=invalid-name
   true_grad_inputs = _resolve_grad_inputs(true_graph, true_grad_graph)
   false_grad_inputs = _resolve_grad_inputs(false_graph, false_grad_graph)
 
-  outputs = _build_cond(op.inputs[0], true_grad_graph, false_grad_graph,
+  outputs = _build_cond(if_op.inputs[0], true_grad_graph, false_grad_graph,
                         true_grad_inputs, false_grad_inputs)
 
   # The predicate has no gradient.
-- 
GitLab


From d0cc3477ff5e3d9dd71abb2656a2a2b5fb39d04c Mon Sep 17 00:00:00 2001
From: Russell Power <power@google.com>
Date: Thu, 17 Jan 2019 11:03:31 -0800
Subject: [PATCH 0881/1095] TPUEstimator: Add export_to_cpu option to
 enable/disable CPU export.

Not all TPU ops are supported on the CPU (in particular some bfloat16
operations).  These models will fail to export on the CPU.  This CL adds a flag
to allow users to disable CPU export in this case.

PiperOrigin-RevId: 229780875
---
 .../contrib/tpu/python/tpu/tpu_estimator.py   | 39 ++++++++++++-------
 1 file changed, 26 insertions(+), 13 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 79340a9f21..022ac4152d 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -2143,6 +2143,7 @@ class TPUEstimator(estimator_lib.Estimator):
                batch_axis=None,
                eval_on_tpu=True,
                export_to_tpu=True,
+               export_to_cpu=True,
                warm_start_from=None,
                experimental_exported_model_uses_all_cores=False):
     """Constructs an `TPUEstimator` instance.
@@ -2187,9 +2188,11 @@ class TPUEstimator(estimator_lib.Estimator):
       eval_on_tpu: If False, evaluation runs on CPU or GPU. In this case, the
         model_fn must return `EstimatorSpec` when called with `mode` as `EVAL`.
       export_to_tpu: If True, `export_savedmodel()` exports a metagraph for
-        serving on TPU besides the one on CPU. Note that unsupported export
-        modes such as EVAL will be ignored. For those modes, only a CPU model
-        will be exported. Currently, export_to_tpu only supports PREDICT.
+        serving on TPU. Note that unsupported export modes such as EVAL will be
+        ignored. For those modes, only a CPU model will be exported.
+        Currently, export_to_tpu only supports PREDICT.
+      export_to_cpu: If True, `export_savedmodel()` exports a metagraph for
+        serving on CPU.
       warm_start_from: Optional string filepath to a checkpoint or SavedModel to
         warm-start from, or a `tf.estimator.WarmStartSettings` object to fully
         configure warm-starting.  If the string filepath is provided instead of
@@ -2264,6 +2267,7 @@ class TPUEstimator(estimator_lib.Estimator):
         self._config, train_batch_size, eval_batch_size, predict_batch_size,
         use_tpu, eval_on_tpu)
 
+    self._export_to_cpu = export_to_cpu
     self._export_to_tpu = export_to_tpu
     self._experimental_exported_model_uses_all_cores = (
         experimental_exported_model_uses_all_cores)
@@ -2284,14 +2288,18 @@ class TPUEstimator(estimator_lib.Estimator):
                       'when `export_to_tpu` is `True`; Mode {} will be ignored '
                       'for TPU.'.format(mode))
 
-    (super(TPUEstimator, self)._add_meta_graph_for_mode(
-        builder,
-        input_receiver_fn_map,
-        checkpoint_path,
-        save_variables,
-        mode=mode,
-        export_tags=export_tags,
-        check_variables=check_variables))
+    if not self._export_to_cpu and not self._export_to_tpu:
+      raise ValueError('One of export_to_cpu and export_to_tpu must be true.')
+
+    if self._export_to_cpu:
+      (super(TPUEstimator, self)._add_meta_graph_for_mode(
+          builder,
+          input_receiver_fn_map,
+          checkpoint_path,
+          save_variables,
+          mode=mode,
+          export_tags=export_tags,
+          check_variables=check_variables))
 
     if self._export_to_tpu and mode == model_fn_lib.ModeKeys.PREDICT:
       input_receiver_fn_map = {
@@ -2299,15 +2307,20 @@ class TPUEstimator(estimator_lib.Estimator):
       }
       export_tags = [tag_constants.SERVING, tag_constants.TPU]
       mode = _REWRITE_FOR_INFERENCE_MODE
+
       # See b/110052256 for why `check_variables` is `False`.
+      if not self._export_to_cpu:
+        check_variables = save_variables = True
+      else:
+        check_variables = save_variables = False
       (super(TPUEstimator, self)._add_meta_graph_for_mode(
           builder,
           input_receiver_fn_map,
           checkpoint_path,
-          save_variables=False,
+          save_variables=save_variables,
           mode=mode,
           export_tags=export_tags,
-          check_variables=False))
+          check_variables=check_variables))
 
   def _call_model_fn(self, features, labels, mode, config):
     if mode == _REWRITE_FOR_INFERENCE_MODE:
-- 
GitLab


From 568b9e56038d2d2fe8927f9e0b538bf5e49116f6 Mon Sep 17 00:00:00 2001
From: Yu-Cheng Ling <ycling@google.com>
Date: Thu, 17 Jan 2019 11:03:53 -0800
Subject: [PATCH 0882/1095] Fix Toco IdentifyL2Normalization bugs

PiperOrigin-RevId: 229780959
---
 tensorflow/lite/build_def.bzl                 |  1 +
 tensorflow/lite/testing/generate_examples.py  | 30 +++++++++++++++++++
 .../identify_l2_normalization.cc              | 16 +++-------
 tensorflow/lite/toco/tooling_util.cc          |  2 +-
 tensorflow/lite/toco/tooling_util.h           |  2 +-
 5 files changed, 37 insertions(+), 14 deletions(-)

diff --git a/tensorflow/lite/build_def.bzl b/tensorflow/lite/build_def.bzl
index 3c48b655f7..7e70503386 100644
--- a/tensorflow/lite/build_def.bzl
+++ b/tensorflow/lite/build_def.bzl
@@ -253,6 +253,7 @@ def generated_test_models():
         "greater_equal",
         "sum",
         "l2norm",
+        "l2norm_shared_epsilon",
         "l2_pool",
         "leaky_relu",
         "less",
diff --git a/tensorflow/lite/testing/generate_examples.py b/tensorflow/lite/testing/generate_examples.py
index d2c275f787..12b5a8b210 100644
--- a/tensorflow/lite/testing/generate_examples.py
+++ b/tensorflow/lite/testing/generate_examples.py
@@ -1424,6 +1424,36 @@ def make_conv_tests(zip_path):
   make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
 
 
+# Note: This is a regression test for a bug (b/122651451) that Toco incorrectly
+# erases the reduction indices array while it's shared with other ops.
+def make_l2norm_shared_epsilon_tests(zip_path):
+  """Regression test for a bug (b/122651451)."""
+
+  # Chose a set of parameters
+  test_parameters = [{
+      "input_shape": [[5, 7]],
+      "dim": [1],
+      "epsilon": [1e-8],
+  }]
+
+  def build_graph(parameters):
+    input_tensor = tf.placeholder(
+        dtype=tf.float32, name="input", shape=parameters["input_shape"])
+    epsilon = tf.constant(parameters["epsilon"])
+    out1 = tf.nn.l2_normalize(input_tensor, parameters["dim"], epsilon=epsilon)
+    out2 = tf.nn.l2_normalize(input_tensor, parameters["dim"], epsilon=epsilon)
+    out = out1 + out2
+    return [input_tensor], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    input_values = create_tensor_data(
+        np.float32, parameters["input_shape"], min_value=-4, max_value=10)
+    return [input_values], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_values])))
+
+  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+
+
 # Note: This is a regression test for a bug (b/112436267) that Toco incorrectly
 # fuses weights when multiple Conv2D/FULLY_CONNECTED ops share the same constant
 # weight tensor.
diff --git a/tensorflow/lite/toco/graph_transformations/identify_l2_normalization.cc b/tensorflow/lite/toco/graph_transformations/identify_l2_normalization.cc
index dabd4bd209..3b7c88ac62 100644
--- a/tensorflow/lite/toco/graph_transformations/identify_l2_normalization.cc
+++ b/tensorflow/lite/toco/graph_transformations/identify_l2_normalization.cc
@@ -151,20 +151,12 @@ std::vector<std::unique_ptr<Operator>>::iterator FindOperator(
 
   // Erase the subgraph that is now replaced by L2Normalization
   model->operators.erase(FindOperator(model, square_op));
-  model->EraseArray(sum_op->inputs[0]);
-  if (sum_op->inputs.size() > 1) {
-    model->EraseArray(sum_op->inputs[1]);
-  }
-  model->operators.erase(FindOperator(model, sum_op));
+  DeleteOpAndArraysIfUnused(model, sum_op);
   if (add_op) {
-    model->EraseArray(add_op->inputs[0]);
-    model->EraseArray(add_op->inputs[1]);
-    model->operators.erase(FindOperator(model, add_op));
+    DeleteOpAndArraysIfUnused(model, add_op);
   }
-  model->EraseArray(sqrt_or_rsqrt_op->inputs[0]);
-  model->operators.erase(FindOperator(model, sqrt_or_rsqrt_op));
-  model->EraseArray(div_or_mul_op->inputs[1]);
-  model->operators.erase(FindOperator(model, div_or_mul_op));
+  DeleteOpAndArraysIfUnused(model, sqrt_or_rsqrt_op);
+  DeleteOpAndArraysIfUnused(model, div_or_mul_op);
   *modified = true;
   return ::tensorflow::Status::OK();
 }
diff --git a/tensorflow/lite/toco/tooling_util.cc b/tensorflow/lite/toco/tooling_util.cc
index 9c0e8067d8..2c98ff1d92 100644
--- a/tensorflow/lite/toco/tooling_util.cc
+++ b/tensorflow/lite/toco/tooling_util.cc
@@ -173,7 +173,7 @@ bool DeleteArrayIfUsedOnce(const string& array_name, Model* model) {
   return false;
 }
 
-void DeleteOpAndArraysIfUnused(Model* model, Operator* op) {
+void DeleteOpAndArraysIfUnused(Model* model, const Operator* op) {
   for (const string& array_name : op->inputs) {
     DeleteArrayIfUsedOnce(array_name, model);
   }
diff --git a/tensorflow/lite/toco/tooling_util.h b/tensorflow/lite/toco/tooling_util.h
index 53131824b5..517da784d0 100644
--- a/tensorflow/lite/toco/tooling_util.h
+++ b/tensorflow/lite/toco/tooling_util.h
@@ -72,7 +72,7 @@ bool DeleteArrayIfUsedOnce(const string& array_name, Model* model);
 
 // Deletes the op and any of its input and output arrays if they are unused
 // after the op has been deleted.
-void DeleteOpAndArraysIfUnused(Model* model, Operator* op);
+void DeleteOpAndArraysIfUnused(Model* model, const Operator* op);
 
 std::vector<std::unique_ptr<Operator>>::const_iterator FindOpWithOutput(
     const Model& model, const string& array_name);
-- 
GitLab


From 2db14a0ffcb3cd46211a0da8baae2be2afe02b45 Mon Sep 17 00:00:00 2001
From: Andy Ly <lyandy@google.com>
Date: Thu, 17 Jan 2019 11:06:51 -0800
Subject: [PATCH 0883/1095] [Grappler] Add validation in
 MutableGraphView.DeleteNodes().

PiperOrigin-RevId: 229781551
---
 tensorflow/core/grappler/BUILD                |   1 +
 .../core/grappler/mutable_graph_view.cc       | 132 +++++++++--
 tensorflow/core/grappler/mutable_graph_view.h |  43 +++-
 .../core/grappler/mutable_graph_view_test.cc  | 218 ++++++++++++++----
 .../core/grappler/optimizers/data/BUILD       |  10 +
 .../grappler/optimizers/data/filter_fusion.cc |   5 +-
 .../optimizers/data/graph_utils_test.cc       |  14 +-
 .../optimizers/data/hoist_random_uniform.cc   |   5 +-
 .../optimizers/data/make_numa_aware.cc        |   5 +-
 .../optimizers/data/map_and_batch_fusion.cc   |   5 +-
 .../optimizers/data/map_and_filter_fusion.cc  |   5 +-
 .../grappler/optimizers/data/map_fusion.cc    |   5 +-
 .../optimizers/data/map_parallelization.cc    |   5 +-
 .../optimizers/data/map_vectorization.cc      |   5 +-
 .../optimizers/data/noop_elimination.cc       |   5 +-
 .../data/shuffle_and_repeat_fusion.cc         |   5 +-
 16 files changed, 379 insertions(+), 89 deletions(-)

diff --git a/tensorflow/core/grappler/BUILD b/tensorflow/core/grappler/BUILD
index bd95e89878..d5266247cf 100644
--- a/tensorflow/core/grappler/BUILD
+++ b/tensorflow/core/grappler/BUILD
@@ -211,6 +211,7 @@ cc_library(
         ":utils",
         "//tensorflow/core:graph",
         "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
         "//tensorflow/core:protos_all_cc",
         "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/strings",
diff --git a/tensorflow/core/grappler/mutable_graph_view.cc b/tensorflow/core/grappler/mutable_graph_view.cc
index 2ba53a685d..e31eedf166 100644
--- a/tensorflow/core/grappler/mutable_graph_view.cc
+++ b/tensorflow/core/grappler/mutable_graph_view.cc
@@ -19,6 +19,7 @@ limitations under the License.
 #include <utility>
 
 #include "absl/strings/str_cat.h"
+#include "absl/strings/str_join.h"
 #include "absl/strings/string_view.h"
 #include "absl/strings/substitute.h"
 #include "tensorflow/core/framework/graph.pb.h"
@@ -29,6 +30,7 @@ limitations under the License.
 #include "tensorflow/core/grappler/utils.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/stringpiece.h"
+#include "tensorflow/core/lib/gtl/map_util.h"
 #include "tensorflow/core/platform/types.h"
 
 namespace tensorflow {
@@ -134,7 +136,8 @@ void MutableGraphView::AddAndDedupFanouts(NodeDef* node) {
                               controlling_fanins.contains(input_node_name)));
     if (!gtl::InsertIfNotPresent(&fanins, input_node_name) &&
         can_dedup_control) {
-      node->mutable_input()->SwapElements(pos, last_pos--);
+      node->mutable_input()->SwapElements(pos, last_pos);
+      --last_pos;
     } else {
       OutputPort output(nodes()[input_node_name], tensor_id.index());
 
@@ -506,10 +509,11 @@ bool MutableGraphView::RemoveRegularFaninInternal(NodeDef* node,
       auto fanouts_set = remove_input(fanin_port, i, /*update_max_port=*/false);
       fanouts_set->insert({node, curr_pos});
       // Shift inputs to be retained.
-      mutable_inputs->SwapElements(i, curr_pos++);
+      mutable_inputs->SwapElements(i, curr_pos);
+      ++curr_pos;
     } else {
       // Skip inputs to be retained until first modification.
-      curr_pos++;
+      ++curr_pos;
     }
   }
 
@@ -717,15 +721,103 @@ Status MutableGraphView::UpdateFanin(absl::string_view node_name,
   return Status::OK();
 }
 
-void MutableGraphView::DeleteNodes(const std::set<string>& nodes_to_delete) {
-  // TODO(lyandy): Check if nodes have fanouts before deleting and return
-  // Status.
-  for (const string& node_name_to_delete : nodes_to_delete)
-    RemoveFaninsInternal(nodes().at(node_name_to_delete),
-                         /*keep_controlling_fanins=*/false);
-  for (const string& node_name_to_delete : nodes_to_delete)
+Status MutableGraphView::CheckNodesCanBeDeleted(
+    const absl::flat_hash_set<string>& nodes_to_delete) {
+  std::vector<string> missing_nodes;
+  std::vector<string> nodes_with_fanouts;
+  for (const string& node_name_to_delete : nodes_to_delete) {
+    NodeDef* node = GetNode(node_name_to_delete);
+    if (node == nullptr) {
+      // Can't delete missing node.
+      missing_nodes.push_back(node_name_to_delete);
+      continue;
+    }
+    const int max_port = gtl::FindWithDefault(max_regular_output_port(), node,
+                                              Graph::kControlSlot);
+    for (int i = Graph::kControlSlot; i <= max_port; ++i) {
+      auto it = fanouts().find({node, i});
+      bool has_retained_fanout = false;
+      if (it != fanouts().end()) {
+        for (const auto& fanout : it->second) {
+          // Check if fanouts are of nodes to be deleted, and if so, they can be
+          // ignored, as they will be removed also.
+          if (!nodes_to_delete.contains(fanout.node->name())) {
+            // Removing node will leave graph in an invalid state.
+            has_retained_fanout = true;
+            break;
+          }
+        }
+      }
+      if (has_retained_fanout) {
+        nodes_with_fanouts.push_back(node_name_to_delete);
+        break;
+      }
+    }
+  }
+
+  // Error message can get quite long, so we only show the first 5 node names.
+  auto sort_and_sample = [](std::vector<string>* s) {
+    constexpr int kMaxNodeNames = 5;
+    std::sort(s->begin(), s->end());
+    if (s->size() > kMaxNodeNames) {
+      return absl::StrCat(
+          absl::StrJoin(s->begin(), s->begin() + kMaxNodeNames, ", "), ", ...");
+    }
+    return absl::StrJoin(*s, ", ");
+  };
+
+  if (!missing_nodes.empty()) {
+    VLOG(1) << absl::Substitute("Attempting to delete missing node(s) [$0]",
+                                sort_and_sample(&missing_nodes));
+  }
+  if (!nodes_with_fanouts.empty()) {
+    std::vector<string> input_node_names(nodes_to_delete.begin(),
+                                         nodes_to_delete.end());
+    return errors::Internal(absl::Substitute(
+        "Can't delete node(s) with retained fanout(s) [$0] from node(s) [$1].",
+        sort_and_sample(&nodes_with_fanouts),
+        sort_and_sample(&input_node_names)));
+  }
+
+  return Status::OK();
+}
+
+Status MutableGraphView::DeleteNodes(
+    const absl::flat_hash_set<string>& nodes_to_delete) {
+  TF_RETURN_IF_ERROR(CheckNodesCanBeDeleted(nodes_to_delete));
+
+  // Find nodes in internal state and delete.
+  for (const string& node_name_to_delete : nodes_to_delete) {
+    NodeDef* node = GetNode(node_name_to_delete);
+    if (node != nullptr) {
+      RemoveFaninsInternal(node, /*keep_controlling_fanins=*/false);
+      RemoveFanoutsInternal(node);
+    }
+  }
+  for (const string& node_name_to_delete : nodes_to_delete) {
     nodes().erase(node_name_to_delete);
-  EraseNodesFromGraph(nodes_to_delete, graph());
+  }
+
+  // Find nodes in graph and delete by partitioning into nodes to retain and
+  // nodes to delete based on input set of nodes to delete by name.
+  // TODO(lyandy): Use a node name->idx hashmap if this is a performance
+  // bottleneck.
+  int pos = 0;
+  const int last_idx = graph()->node_size() - 1;
+  int last_pos = last_idx;
+  while (pos <= last_pos) {
+    if (nodes_to_delete.contains(graph()->node(pos).name())) {
+      graph()->mutable_node()->SwapElements(pos, last_pos);
+      --last_pos;
+    } else {
+      ++pos;
+    }
+  }
+  if (last_pos < last_idx) {
+    graph()->mutable_node()->DeleteSubrange(last_pos + 1, last_idx - last_pos);
+  }
+
+  return Status::OK();
 }
 
 void MutableGraphView::RemoveFaninsInternal(NodeDef* deleted_node,
@@ -741,10 +833,22 @@ void MutableGraphView::RemoveFaninsInternal(NodeDef* deleted_node,
     input.node = deleted_node;
     input.port_id = IsTensorIdControlling(tensor_id) ? Graph::kControlSlot : i;
 
-    absl::flat_hash_set<InputPort>* fanouts_set = &fanouts()[fanin];
-    fanouts_set->erase(input);
-    UpdateMaxRegularOutputPortForRemovedFanin(fanin, *fanouts_set);
+    auto it = fanouts().find(fanin);
+    if (it != fanouts().end()) {
+      absl::flat_hash_set<InputPort>* fanouts_set = &it->second;
+      fanouts_set->erase(input);
+      UpdateMaxRegularOutputPortForRemovedFanin(fanin, *fanouts_set);
+    }
+  }
+}
+
+void MutableGraphView::RemoveFanoutsInternal(NodeDef* deleted_node) {
+  const int max_port = gtl::FindWithDefault(max_regular_output_port(),
+                                            deleted_node, Graph::kControlSlot);
+  for (int i = Graph::kControlSlot; i <= max_port; ++i) {
+    fanouts().erase({deleted_node, i});
   }
+  max_regular_output_port().erase(deleted_node);
 }
 
 }  // end namespace grappler
diff --git a/tensorflow/core/grappler/mutable_graph_view.h b/tensorflow/core/grappler/mutable_graph_view.h
index 59740f2c71..c62129bcad 100644
--- a/tensorflow/core/grappler/mutable_graph_view.h
+++ b/tensorflow/core/grappler/mutable_graph_view.h
@@ -101,6 +101,10 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   // end up with:
   //
   //   fanin -> Identity{N} -^> target node.
+  //
+  // If the control dependency being added is redundant (control dependency
+  // already exists or control dependency can be deduped from regular fanins),
+  // this will not result in an error and the node will not be modified.
   Status AddControllingFanin(absl::string_view node_name,
                              const TensorId& fanin);
 
@@ -108,29 +112,41 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   // do not exist in the graph, nothing will be modified in the graph. If there
   // are multiple inputs that match the fanin, all of them will be removed. To
   // remove controlling fanins, use RemoveControllingFanin.
+  //
+  // If the fanin being removed doesn't exist in the node's inputs, this will
+  // not result in an error and the node will not be modified.
   Status RemoveRegularFanin(absl::string_view node_name, const TensorId& fanin);
 
   // Removes control dependency `fanin_node_name` from the target node named
   // `node_name`. If the node or fanin do not exist in the graph, nothing will
   // be modified in the graph. To remove regular fanins, use RemoveRegualrFanin.
+  //
+  // If the fanin being removed doesn't exist in the node's inputs, this will
+  // not result in an error and the node will not be modified.
   Status RemoveControllingFanin(absl::string_view node_name,
                                 absl::string_view fanin_node_name);
 
   // Removes all fanins from node `node_name`. Control dependencies will be
   // retained if keep_controlling_fanins is true.
+  //
+  // If no fanins are removed, this will not result in an error and the node
+  // will not be modified.
   Status RemoveAllFanins(absl::string_view node_name,
                          bool keep_controlling_fanins);
 
   // Replaces all fanins `from_fanin` with `to_fanin` in node `node_name`. If
   // the fanins or node do not exist, nothing will be modified in the graph.
   // Control dependencies will be deduped.
+  //
+  // If the fanin being updated doesn't exist in the node's inputs, this will
+  // not result in an error and the node will not be modified.
   Status UpdateFanin(absl::string_view node_name, const TensorId& from_fanin,
                      const TensorId& to_fanin);
 
-  // Deletes nodes from the graph.
-  // TODO(lyandy): Check if nodes have fanouts before deleting and return
-  // Status.
-  void DeleteNodes(const std::set<string>& nodes_to_delete);
+  // Deletes nodes from the graph. If a node can't be safely removed,
+  // specifically if a node still has fanouts, an error will be returned. Nodes
+  // that can't be found are ignored.
+  Status DeleteNodes(const absl::flat_hash_set<string>& nodes_to_delete);
 
  private:
   // Adds fanouts for fanins of node to graph, while deduping control
@@ -163,11 +179,6 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   // behavior is undefined.
   Status UpdateFanoutsInternal(NodeDef* from_node, NodeDef* to_node);
 
-  // Removes fanins of the deleted node from internal state. Control
-  // dependencies are retained iff keep_controlling_fanins is true.
-  void RemoveFaninsInternal(NodeDef* deleted_node,
-                            bool keep_controlling_fanins);
-
   // Adds fanin to node. If fanin is a control dependency, existing control
   // dependencies will be checked first before adding. Otherwise fanin will be
   // added after existing non control dependency inputs.
@@ -179,6 +190,20 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   // Removes controlling fanin `fanin_node` from node if such controlling fanin
   // exists.
   bool RemoveControllingFaninInternal(NodeDef* node, NodeDef* fanin_node);
+
+  // Checks if nodes to be deleted are missing or have any fanouts that will
+  // remain in the graph. If node is removed in either case, the graph will
+  // enter an invalid state.
+  Status CheckNodesCanBeDeleted(
+      const absl::flat_hash_set<string>& nodes_to_delete);
+
+  // Removes fanins of the deleted node from internal state. Control
+  // dependencies are retained iff keep_controlling_fanins is true.
+  void RemoveFaninsInternal(NodeDef* deleted_node,
+                            bool keep_controlling_fanins);
+
+  // Removes fanouts of the deleted node from internal state.
+  void RemoveFanoutsInternal(NodeDef* deleted_node);
 };
 
 }  // end namespace grappler
diff --git a/tensorflow/core/grappler/mutable_graph_view_test.cc b/tensorflow/core/grappler/mutable_graph_view_test.cc
index b5bf33033c..acfaba5ddd 100644
--- a/tensorflow/core/grappler/mutable_graph_view_test.cc
+++ b/tensorflow/core/grappler/mutable_graph_view_test.cc
@@ -23,6 +23,7 @@ limitations under the License.
 #include "tensorflow/core/grappler/grappler_item.h"
 #include "tensorflow/core/grappler/inputs/trivial_test_graph_input_yielder.h"
 #include "tensorflow/core/grappler/utils.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/platform/test.h"
 
 namespace tensorflow {
@@ -154,7 +155,7 @@ TEST(MutableGraphViewTest, AddAndUpdateFanouts) {
 
   NodeDef* new_bar = graph.AddNode(NDef("new_bar", "NotImportant", {}, {}));
 
-  EXPECT_TRUE(graph.UpdateFanouts("bar", new_bar->name()).ok());
+  TF_EXPECT_OK(graph.UpdateFanouts("bar", new_bar->name()));
 
   // Fanins and fanouts must be updated.
   CheckNode(graph, "bar", "NotImportant", "", {}, {}, {});
@@ -184,7 +185,7 @@ TEST(MutableGraphViewTest, AddAndUpdateFanoutsKeepControls) {
 
   NodeDef* new_bar = graph.AddNode(NDef("new_bar", "Identity", {"bar_1:2"}));
 
-  EXPECT_TRUE(graph.UpdateFanouts("bar_2", new_bar->name()).ok());
+  TF_EXPECT_OK(graph.UpdateFanouts("bar_2", new_bar->name()));
 
   // Fanins and fanouts must be updated.
   CheckNode(graph, "bar_1", "Switch", "", {}, {}, {"bar_2", "new_bar"});
@@ -213,7 +214,7 @@ TEST(MutableGraphViewTest, AddAndUpdateFanoutsWithoutSelfLoops) {
   // `new_bar` reads the output of an original `bar` node.
   NodeDef* new_bar = graph.AddNode(NDef("new_bar", "NewBar", {"bar"}, {}));
 
-  EXPECT_TRUE(graph.UpdateFanouts("bar", new_bar->name()).ok());
+  TF_EXPECT_OK(graph.UpdateFanouts("bar", new_bar->name()));
 
   // Fanins and fanouts must be updated.
   CheckNode(graph, "bar", "NotImportant", "", {}, {}, {"new_bar"});
@@ -266,7 +267,7 @@ TEST(MutableGraphViewTest, UpdateFanoutsToSwitchWithNoControlFromSwitch) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.UpdateFanouts("c", "b").ok());
+  TF_EXPECT_OK(graph.UpdateFanouts("c", "b"));
 
   EXPECT_EQ(graph.graph()->node_size(), 5);
 
@@ -882,10 +883,10 @@ TEST(MutableGraphViewTest, DedupControllingFaninsOnAddFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.AddRegularFanin("b", {"a", 2}).ok());
+  TF_EXPECT_OK(graph.AddRegularFanin("b", {"a", 2}));
   CheckNode(graph, "b", "NotImportant", "", {}, {"a:2"}, {});
 
-  EXPECT_TRUE(graph.AddControllingFanin("c", {"a", Graph::kControlSlot}).ok());
+  TF_EXPECT_OK(graph.AddControllingFanin("c", {"a", Graph::kControlSlot}));
   CheckNode(graph, "c", "NotImportant", "", {}, {"a:1"}, {});
 
   CheckNode(graph, "a", "NotImportant", "", {}, {}, {"b:0", "c:0"});
@@ -901,16 +902,16 @@ TEST(MutableGraphViewTest, NoDedupControlFlowControllingFaninsOnAddFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.AddRegularFanin("c", {"b", 2}).ok());
+  TF_EXPECT_OK(graph.AddRegularFanin("c", {"b", 2}));
   CheckNode(graph, "c", "", "", {}, {"b:2"}, {});
-  EXPECT_TRUE(graph.AddControllingFanin("c", {"b", Graph::kControlSlot}).ok());
+  TF_EXPECT_OK(graph.AddControllingFanin("c", {"b", Graph::kControlSlot}));
   CheckNode(graph, "c", "", "", {}, {"b:2", "^b"}, {});
-  EXPECT_TRUE(graph.AddControllingFanin("c", {"b", Graph::kControlSlot}).ok());
+  TF_EXPECT_OK(graph.AddControllingFanin("c", {"b", Graph::kControlSlot}));
   CheckNode(graph, "c", "", "", {}, {"b:2", "^b"}, {});
 
-  EXPECT_TRUE(graph.AddControllingFanin("d", {"b", Graph::kControlSlot}).ok());
+  TF_EXPECT_OK(graph.AddControllingFanin("d", {"b", Graph::kControlSlot}));
   CheckNode(graph, "d", "", "", {}, {"^b"}, {});
-  EXPECT_TRUE(graph.AddControllingFanin("d", {"b", Graph::kControlSlot}).ok());
+  TF_EXPECT_OK(graph.AddControllingFanin("d", {"b", Graph::kControlSlot}));
   CheckNode(graph, "d", "", "", {}, {"^b"}, {});
 
   CheckGraph(graph);
@@ -925,7 +926,7 @@ TEST(MutableGraphViewTest, DedupControllingFaninsOnUpdateFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.UpdateFanin("c", {"a", 1}, {"b", 2}).ok());
+  TF_EXPECT_OK(graph.UpdateFanin("c", {"a", 1}, {"b", 2}));
 
   CheckNode(graph, "a", "NotImportant", "", {}, {}, {});
   CheckNode(graph, "b", "NotImportant", "", {}, {}, {"c"});
@@ -943,17 +944,14 @@ TEST(MutableGraphViewTest, NoDedupControlFlowControllingFaninsOnUpdateFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph
-                  .UpdateFanin("d", {"b", Graph::kControlSlot},
-                               {"c", Graph::kControlSlot})
-                  .ok());
+  TF_EXPECT_OK(graph.UpdateFanin("d", {"b", Graph::kControlSlot},
+                                 {"c", Graph::kControlSlot}));
   CheckNode(graph, "d", "NotImportant", "", {}, {"c", "^c"}, {});
 
-  EXPECT_TRUE(graph.UpdateFanin("e", {"b", 0}, {"c", 3}).ok());
+  TF_EXPECT_OK(graph.UpdateFanin("e", {"b", 0}, {"c", 3}));
   CheckNode(graph, "e", "NotImportant", "", {}, {"c:3", "^c"}, {});
 
-  EXPECT_TRUE(
-      graph.UpdateFanin("e", {"c", 3}, {"c", Graph::kControlSlot}).ok());
+  TF_EXPECT_OK(graph.UpdateFanin("e", {"c", 3}, {"c", Graph::kControlSlot}));
   CheckNode(graph, "e", "NotImportant", "", {}, {"^c"}, {});
 
   CheckGraph(graph);
@@ -968,7 +966,7 @@ TEST(MutableGraphViewTest, UpdateMaxRegularOutputPortOnAddFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.AddRegularFanin("c", {"a", 3}).ok());
+  TF_EXPECT_OK(graph.AddRegularFanin("c", {"a", 3}));
 
   CheckNode(graph, "a", "NotImportant", "", {}, {}, {"b", "c"});
   CheckNode(graph, "b", "NotImportant", "", {}, {"a:1"}, {"^c"});
@@ -986,7 +984,7 @@ TEST(MutableGraphViewTest, UpdateMaxRegularOutputPortOnRemoveFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.RemoveRegularFanin("c", {"a", 2}).ok());
+  TF_EXPECT_OK(graph.RemoveRegularFanin("c", {"a", 2}));
   CheckNode(graph, "a", "NotImportant", "", {}, {}, {"b"});
   CheckNode(graph, "b", "NotImportant", "", {}, {"a:1"}, {});
   CheckNode(graph, "c", "NotImportant", "", {}, {}, {});
@@ -1003,7 +1001,7 @@ TEST(MutableGraphViewTest, KeepMaxRegularOutputPortOnRemoveFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.RemoveRegularFanin("b", {"a", 1}).ok());
+  TF_EXPECT_OK(graph.RemoveRegularFanin("b", {"a", 1}));
 
   CheckNode(graph, "a", "NotImportant", "", {}, {}, {"c"});
   CheckNode(graph, "b", "NotImportant", "", {}, {}, {});
@@ -1021,7 +1019,7 @@ TEST(MutableGraphViewTest, UpdateMaxRegularOutputPortOnUpdateFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.UpdateFanin("c", {"a", 2}, {"b", 3}).ok());
+  TF_EXPECT_OK(graph.UpdateFanin("c", {"a", 2}, {"b", 3}));
 
   CheckNode(graph, "a", "NotImportant", "", {}, {}, {"b"});
   CheckNode(graph, "b", "NotImportant", "", {}, {"a:1"}, {"c"});
@@ -1069,8 +1067,8 @@ TEST(MutableGraphViewTest, AddControllingFaninExistingControl) {
       /*funcs=*/{});
 
   MutableGraphView graph(&graph_def);
-  EXPECT_TRUE(graph.AddControllingFanin("a", {"b", Graph::kControlSlot}).ok());
-  EXPECT_TRUE(graph.AddControllingFanin("a", {"b", Graph::kControlSlot}).ok());
+  TF_EXPECT_OK(graph.AddControllingFanin("a", {"b", Graph::kControlSlot}));
+  TF_EXPECT_OK(graph.AddControllingFanin("a", {"b", Graph::kControlSlot}));
 
   ASSERT_EQ(graph.graph()->node_size(), 2);
 
@@ -1087,8 +1085,8 @@ TEST(MutableGraphViewTest, AddControllingFaninNotSwitch) {
       /*funcs=*/{});
 
   MutableGraphView graph(&graph_def);
-  EXPECT_TRUE(graph.AddControllingFanin("a", {"b", 2}).ok());
-  EXPECT_TRUE(graph.AddControllingFanin("a", {"b", 2}).ok());
+  TF_EXPECT_OK(graph.AddControllingFanin("a", {"b", 2}));
+  TF_EXPECT_OK(graph.AddControllingFanin("a", {"b", 2}));
 
   ASSERT_EQ(graph.graph()->node_size(), 2);
 
@@ -1127,8 +1125,8 @@ TEST(MutableGraphViewTest, AddControllingFaninSwitchWithIdentity) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.AddControllingFanin("a", {"switch", 0}).ok());
-  EXPECT_TRUE(graph.AddControllingFanin("a", {"switch", 0}).ok());
+  TF_EXPECT_OK(graph.AddControllingFanin("a", {"switch", 0}));
+  TF_EXPECT_OK(graph.AddControllingFanin("a", {"switch", 0}));
 
   ASSERT_EQ(graph.graph()->node_size(), 3);
 
@@ -1148,8 +1146,8 @@ TEST(MutableGraphViewTest, AddControllingFaninSwitchWithNoExistingIdentity) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.AddControllingFanin("a", {"switch", 0}).ok());
-  EXPECT_TRUE(graph.AddControllingFanin("a", {"switch", 0}).ok());
+  TF_EXPECT_OK(graph.AddControllingFanin("a", {"switch", 0}));
+  TF_EXPECT_OK(graph.AddControllingFanin("a", {"switch", 0}));
 
   ASSERT_EQ(graph.graph()->node_size(), 3);
 
@@ -1171,8 +1169,8 @@ TEST(MutableGraphViewTest, AddControllingFaninSwitchWithExistingAddedIdentity) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.AddControllingFanin("a", {"switch", 0}).ok());
-  EXPECT_TRUE(graph.AddControllingFanin("a", {"switch", 0}).ok());
+  TF_EXPECT_OK(graph.AddControllingFanin("a", {"switch", 0}));
+  TF_EXPECT_OK(graph.AddControllingFanin("a", {"switch", 0}));
 
   ASSERT_EQ(graph.graph()->node_size(), 3);
 
@@ -1279,7 +1277,7 @@ TEST(MutableGraphViewTest, RemoveControllingFaninMissing) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.RemoveControllingFanin("d", "c").ok());
+  TF_EXPECT_OK(graph.RemoveControllingFanin("d", "c"));
 
   ASSERT_EQ(graph.graph()->node_size(), 4);
 
@@ -1301,8 +1299,8 @@ TEST(MutableGraphViewTest, RemoveControllingFaninExisting) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.RemoveControllingFanin("d", "a").ok());
-  EXPECT_TRUE(graph.RemoveControllingFanin("d", "a").ok());
+  TF_EXPECT_OK(graph.RemoveControllingFanin("d", "a"));
+  TF_EXPECT_OK(graph.RemoveControllingFanin("d", "a"));
 
   ASSERT_EQ(graph.graph()->node_size(), 4);
 
@@ -1323,8 +1321,8 @@ TEST(MutableGraphViewTest, RemoveControllingFaninOnRegularFanin) {
 
   MutableGraphView graph(&graph_def);
 
-  EXPECT_TRUE(graph.RemoveControllingFanin("c", "a").ok());
-  EXPECT_TRUE(graph.RemoveControllingFanin("c", "b").ok());
+  TF_EXPECT_OK(graph.RemoveControllingFanin("c", "a"));
+  TF_EXPECT_OK(graph.RemoveControllingFanin("c", "b"));
 
   ASSERT_EQ(graph.graph()->node_size(), 3);
 
@@ -1370,7 +1368,7 @@ TEST(MutableGraphViewTest, DeleteNodes) {
   MutableGraphView graph(&graph_def);
 
   EXPECT_NE(graph.GetNode("foo_1"), nullptr);
-  graph.DeleteNodes({"foo_1"});
+  TF_EXPECT_OK(graph.DeleteNodes({"foo_1"}));
 
   EXPECT_EQ(graph.graph()->node_size(), 3);
   EXPECT_EQ(graph.GetNode("foo_1"), nullptr);
@@ -1382,6 +1380,148 @@ TEST(MutableGraphViewTest, DeleteNodes) {
   CheckGraph(graph);
 }
 
+GraphDef SimpleDeleteNodeGraph() {
+  // Actual node.op() is not important in this test.
+  GraphDef graph_def = test::function::GDef(
+      {NDef("a", "NotImportant", {}, {}), NDef("b", "NotImportant", {"a:2"}),
+       NDef("c", "NotImportant", {"a:5", "^b"}), NDef("d", "NotImportant", {}),
+       NDef("e", "NotImportant", {"d:2"}),
+       NDef("f", "NotImportant", {"d:3", "^e"})},
+      /*funcs=*/{});
+  return graph_def;
+}
+
+TEST(MutableGraphViewTest, DeleteNodesWithFanoutsBeingDeleted) {
+  GraphDef graph_def = SimpleDeleteNodeGraph();
+
+  MutableGraphView graph(&graph_def);
+  EXPECT_NE(graph.GetNode("a"), nullptr);
+  EXPECT_NE(graph.GetNode("b"), nullptr);
+  EXPECT_NE(graph.GetNode("c"), nullptr);
+  TF_EXPECT_OK(graph.DeleteNodes({"c", "a", "b"}));
+
+  EXPECT_EQ(graph.graph()->node_size(), 3);
+  EXPECT_EQ(graph.GetNode("a"), nullptr);
+  EXPECT_EQ(graph.GetNode("b"), nullptr);
+  EXPECT_EQ(graph.GetNode("c"), nullptr);
+
+  CheckNode(graph, "d", "NotImportant", "", {}, {}, {"e", "f"});
+  CheckNode(graph, "e", "NotImportant", "", {}, {"d:2"}, {"^f"});
+  CheckNode(graph, "f", "NotImportant", "", {}, {"d:3", "^e"}, {});
+
+  CheckGraph(graph);
+}
+
+TEST(MutableGraphViewTest, DeleteMissingNodes) {
+  GraphDef graph_def = SimpleDeleteNodeGraph();
+
+  MutableGraphView graph(&graph_def);
+
+  EXPECT_EQ(graph.GetNode("g"), nullptr);
+  EXPECT_EQ(graph.GetNode("h"), nullptr);
+  TF_EXPECT_OK(graph.DeleteNodes({"g", "h"}));
+
+  EXPECT_EQ(graph.graph()->node_size(), 6);
+  EXPECT_EQ(graph.GetNode("g"), nullptr);
+  EXPECT_EQ(graph.GetNode("h"), nullptr);
+
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {"b", "c"});
+  CheckNode(graph, "b", "NotImportant", "", {}, {"a:2"}, {"^c"});
+  CheckNode(graph, "c", "NotImportant", "", {}, {"a:5", "^b"}, {});
+  CheckNode(graph, "d", "NotImportant", "", {}, {}, {"e", "f"});
+  CheckNode(graph, "e", "NotImportant", "", {}, {"d:2"}, {"^f"});
+  CheckNode(graph, "f", "NotImportant", "", {}, {"d:3", "^e"}, {});
+
+  CheckGraph(graph);
+}
+
+TEST(MutableGraphViewTest, DeleteMissingNodesAndNodesWithFanoutsBeingDeleted) {
+  GraphDef graph_def = SimpleDeleteNodeGraph();
+
+  MutableGraphView graph(&graph_def);
+
+  EXPECT_NE(graph.GetNode("d"), nullptr);
+  EXPECT_NE(graph.GetNode("e"), nullptr);
+  EXPECT_NE(graph.GetNode("f"), nullptr);
+  TF_EXPECT_OK(graph.DeleteNodes({"d", "e", "f", "g", "h"}));
+
+  EXPECT_EQ(graph.graph()->node_size(), 3);
+  EXPECT_EQ(graph.GetNode("d"), nullptr);
+  EXPECT_EQ(graph.GetNode("e"), nullptr);
+  EXPECT_EQ(graph.GetNode("f"), nullptr);
+
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {"b", "c"});
+  CheckNode(graph, "b", "NotImportant", "", {}, {"a:2"}, {"^c"});
+  CheckNode(graph, "c", "NotImportant", "", {}, {"a:5", "^b"}, {});
+
+  CheckGraph(graph);
+}
+
+TEST(MutableGraphViewTest, DeleteNodesWithError) {
+  GraphDef graph_def = SimpleDeleteNodeGraph();
+
+  MutableGraphView graph(&graph_def);
+
+  Status s = graph.DeleteNodes({"b", "a"});
+  EXPECT_FALSE(s.ok());
+  string error_msg =
+      "Can't delete node(s) with retained fanout(s) [a, b] from node(s) [a, "
+      "b].";
+  EXPECT_EQ(s.error_message(), error_msg);
+
+  EXPECT_EQ(graph.graph()->node_size(), 6);
+
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {"b", "c"});
+  CheckNode(graph, "b", "NotImportant", "", {}, {"a:2"}, {"^c"});
+  CheckNode(graph, "c", "NotImportant", "", {}, {"a:5", "^b"}, {});
+  CheckNode(graph, "d", "NotImportant", "", {}, {}, {"e", "f"});
+  CheckNode(graph, "e", "NotImportant", "", {}, {"d:2"}, {"^f"});
+  CheckNode(graph, "f", "NotImportant", "", {}, {"d:3", "^e"}, {});
+
+  CheckGraph(graph);
+}
+
+TEST(MutableGraphViewTest, DeleteNodesWithLargeError) {
+  // Actual node.op() is not important in this test.
+  GraphDef graph_def = test::function::GDef(
+      {NDef("a", "NotImportant", {}, {}), NDef("b", "NotImportant", {"a:2"}),
+       NDef("c", "NotImportant", {"^b"}), NDef("d", "NotImportant", {"c:6"}),
+       NDef("e", "NotImportant", {"d:2"}),
+       NDef("f", "NotImportant", {"d:3", "^e"}),
+       NDef("g", "NotImportant", {"f"}), NDef("h", "NotImportant", {"a"}),
+       NDef("i", "NotImportant", {"b"}), NDef("j", "NotImportant", {"c"}),
+       NDef("k", "NotImportant", {"d"}), NDef("l", "NotImportant", {"e"}),
+       NDef("m", "NotImportant", {"f"})},
+      /*funcs=*/{});
+
+  MutableGraphView graph(&graph_def);
+
+  Status s = graph.DeleteNodes({"a", "b", "c", "d", "e", "f"});
+  EXPECT_FALSE(s.ok());
+  string error_msg =
+      "Can't delete node(s) with retained fanout(s) [a, b, c, d, e, ...] from "
+      "node(s) [a, b, c, d, e, ...].";
+  EXPECT_EQ(s.error_message(), error_msg);
+
+  EXPECT_EQ(graph.graph()->node_size(), 13);
+
+  CheckNode(graph, "a", "NotImportant", "", {}, {}, {"b", "h"});
+  CheckNode(graph, "b", "NotImportant", "", {}, {"a:2"}, {"^c", "i"});
+  CheckNode(graph, "c", "NotImportant", "", {}, {"^b"}, {"d", "j"});
+  CheckNode(graph, "d", "NotImportant", "", {}, {"c:6"}, {"e", "f", "k"});
+  CheckNode(graph, "e", "NotImportant", "", {}, {"d:2"}, {"^f", "l"});
+  CheckNode(graph, "f", "NotImportant", "", {}, {"d:3", "^e"}, {"g", "m"});
+  CheckNode(graph, "g", "NotImportant", "", {}, {"f"}, {});
+  CheckNode(graph, "h", "NotImportant", "", {}, {"a"}, {});
+  CheckNode(graph, "i", "NotImportant", "", {}, {"b"}, {});
+  CheckNode(graph, "j", "NotImportant", "", {}, {"c"}, {});
+  CheckNode(graph, "k", "NotImportant", "", {}, {"d"}, {});
+  CheckNode(graph, "l", "NotImportant", "", {}, {"e"}, {});
+  CheckNode(graph, "m", "NotImportant", "", {}, {"f"}, {});
+
+  CheckGraph(graph);
+}
+
 }  // namespace
 }  // namespace grappler
 }  // namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/data/BUILD b/tensorflow/core/grappler/optimizers/data/BUILD
index 413588b15c..fe59dfef15 100644
--- a/tensorflow/core/grappler/optimizers/data/BUILD
+++ b/tensorflow/core/grappler/optimizers/data/BUILD
@@ -38,6 +38,7 @@ cc_library(
         ":graph_utils",
         ":fusion_utils",
         ":optimizer_base",
+        "@com_google_absl//absl/container:flat_hash_set",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core:lib",
         "//tensorflow/core/grappler:grappler_item",
@@ -193,6 +194,7 @@ cc_library(
         ":function_utils",
         ":graph_utils",
         ":optimizer_base",
+        "@com_google_absl//absl/container:flat_hash_set",
         "//tensorflow/core:lib",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:grappler_item",
@@ -261,6 +263,7 @@ cc_library(
     deps = [
         ":graph_utils",
         ":optimizer_base",
+        "@com_google_absl//absl/container:flat_hash_set",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:op_types",
@@ -324,6 +327,7 @@ cc_library(
     deps = [
         ":graph_utils",
         ":optimizer_base",
+        "@com_google_absl//absl/container:flat_hash_set",
         "//tensorflow/core:lib",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:grappler_item",
@@ -358,6 +362,7 @@ cc_library(
         ":graph_utils",
         ":fusion_utils",
         ":optimizer_base",
+        "@com_google_absl//absl/container:flat_hash_set",
         "//tensorflow/core:lib",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:grappler_item",
@@ -396,6 +401,7 @@ cc_library(
         ":graph_utils",
         ":fusion_utils",
         ":optimizer_base",
+        "@com_google_absl//absl/container:flat_hash_set",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core:lib",
         "//tensorflow/core/grappler:grappler_item",
@@ -434,6 +440,7 @@ cc_library(
     deps = [
         ":graph_utils",
         ":optimizer_base",
+        "@com_google_absl//absl/container:flat_hash_set",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:op_types",
@@ -470,6 +477,7 @@ cc_library(
         ":graph_utils",
         ":optimizer_base",
         ":vectorization_utils",
+        "@com_google_absl//absl/container:flat_hash_set",
         "//tensorflow/core:lib",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:grappler_item",
@@ -526,6 +534,7 @@ cc_library(
     deps = [
         ":graph_utils",
         ":optimizer_base",
+        "@com_google_absl//absl/container:flat_hash_set",
         "//tensorflow/core:lib",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:grappler_item",
@@ -571,6 +580,7 @@ cc_library(
     deps = [
         ":graph_utils",
         ":optimizer_base",
+        "@com_google_absl//absl/container:flat_hash_set",
         "//tensorflow/core:lib",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:grappler_item",
diff --git a/tensorflow/core/grappler/optimizers/data/filter_fusion.cc b/tensorflow/core/grappler/optimizers/data/filter_fusion.cc
index 25dc10806b..7a20b8042b 100644
--- a/tensorflow/core/grappler/optimizers/data/filter_fusion.cc
+++ b/tensorflow/core/grappler/optimizers/data/filter_fusion.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/core/grappler/optimizers/data/filter_fusion.h"
 
+#include "absl/container/flat_hash_set.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/grappler/clusters/cluster.h"
@@ -66,7 +67,7 @@ Status FilterFusion::OptimizeAndCollectStats(Cluster* cluster,
   *output = sorted_old_graph;
 
   MutableGraphView graph(output);
-  std::set<string> nodes_to_delete;
+  absl::flat_hash_set<string> nodes_to_delete;
   FunctionLibraryDefinition function_library(OpRegistry::Global(),
                                              output->library());
 
@@ -125,7 +126,7 @@ Status FilterFusion::OptimizeAndCollectStats(Cluster* cluster,
     stats->num_changes++;
   }
 
-  graph.DeleteNodes(nodes_to_delete);
+  TF_RETURN_IF_ERROR(graph.DeleteNodes(nodes_to_delete));
   return Status::OK();
 }
 
diff --git a/tensorflow/core/grappler/optimizers/data/graph_utils_test.cc b/tensorflow/core/grappler/optimizers/data/graph_utils_test.cc
index 5c0f03dca8..3b6d223fd3 100644
--- a/tensorflow/core/grappler/optimizers/data/graph_utils_test.cc
+++ b/tensorflow/core/grappler/optimizers/data/graph_utils_test.cc
@@ -109,7 +109,7 @@ TEST(GraphUtilsTest, ContainsGraphNodeWithName) {
   AddNode("A", "OpA", {}, {}, &graph);
   EXPECT_TRUE(ContainsGraphNodeWithName("A", *graph.graph()));
 
-  graph.DeleteNodes({"A"});
+  EXPECT_TRUE(graph.DeleteNodes({"A"}).ok());
   EXPECT_TRUE(!ContainsGraphNodeWithName("A", *graph.graph()));
 }
 
@@ -131,7 +131,7 @@ TEST(GraphUtilsTest, ContainsNodeWithOp) {
   AddNode("A", "OpA", {}, {}, &graph);
   EXPECT_TRUE(ContainsNodeWithOp("OpA", *graph.graph()));
 
-  graph.DeleteNodes({"A"});
+  EXPECT_TRUE(graph.DeleteNodes({"A"}).ok());
   EXPECT_TRUE(!ContainsNodeWithOp("OpA", *graph.graph()));
 }
 
@@ -143,7 +143,7 @@ TEST(GraphUtilsTest, FindGraphNodeWithName) {
   AddNode("A", "OpA", {}, {}, &graph);
   EXPECT_NE(FindGraphNodeWithName("A", *graph.graph()), -1);
 
-  graph.DeleteNodes({"A"});
+  EXPECT_TRUE(graph.DeleteNodes({"A"}).ok());
   EXPECT_EQ(FindGraphNodeWithName("A", *graph.graph()), -1);
 }
 
@@ -164,10 +164,10 @@ TEST(GraphUtilsTest, FindGraphNodeWithOp) {
 
   AddNode("A", "OpA", {}, {}, &graph);
   AddNode("B", "OpB", {"A"}, {}, &graph);
-  AddNode("A2", "OpA", {"B"}, {}, &graph);
+  AddNode("A2", "OpA", {"A"}, {}, &graph);
   EXPECT_EQ(FindGraphNodeWithOp("OpA", *graph.graph()), 0);
 
-  graph.DeleteNodes({"B"});
+  EXPECT_TRUE(graph.DeleteNodes({"B"}).ok());
   EXPECT_EQ(FindGraphNodeWithOp("OpB", *graph.graph()), -1);
   EXPECT_EQ(FindGraphNodeWithName("A2", *graph.graph()), 1);
 }
@@ -186,7 +186,7 @@ TEST(GraphUtilsTest, FindAllGraphNodesWithOp) {
   EXPECT_EQ(result_indices.at(0), 0);
   EXPECT_EQ(result_indices.at(1), 2);
 
-  graph.DeleteNodes({"A2"});
+  EXPECT_TRUE(graph.DeleteNodes({"A2"}).ok());
   std::vector<int> result_indices_new =
       FindAllGraphNodesWithOp("OpA", *graph.graph());
   EXPECT_EQ(result_indices_new.size(), 1);
@@ -201,7 +201,7 @@ TEST(GraphUtilsTest, SetUniqueGraphNodeName) {
   NodeDef* node2 = AddNode("", "A", {}, {}, &graph);
   EXPECT_NE(node1->name(), node2->name());
 
-  graph.DeleteNodes({node1->name()});
+  EXPECT_TRUE(graph.DeleteNodes({node1->name()}).ok());
   NodeDef* node3 = AddNode("", "A", {}, {}, &graph);
   EXPECT_NE(node2->name(), node3->name());
 }
diff --git a/tensorflow/core/grappler/optimizers/data/hoist_random_uniform.cc b/tensorflow/core/grappler/optimizers/data/hoist_random_uniform.cc
index 237e4b63e3..496277780f 100644
--- a/tensorflow/core/grappler/optimizers/data/hoist_random_uniform.cc
+++ b/tensorflow/core/grappler/optimizers/data/hoist_random_uniform.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/core/grappler/optimizers/data/hoist_random_uniform.h"
 
+#include "absl/container/flat_hash_set.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/framework/op_def.pb.h"
@@ -227,7 +228,7 @@ Status HoistRandomUniform::OptimizeAndCollectStats(Cluster* cluster,
   *output = item.graph;
 
   MutableGraphView graph(output);
-  std::set<string> nodes_to_delete;
+  absl::flat_hash_set<string> nodes_to_delete;
   FunctionLibraryDefinition function_library(OpRegistry::Global(),
                                              item.graph.library());
 
@@ -277,7 +278,7 @@ Status HoistRandomUniform::OptimizeAndCollectStats(Cluster* cluster,
     stats->num_changes++;
   }
 
-  graph.DeleteNodes(nodes_to_delete);
+  TF_RETURN_IF_ERROR(graph.DeleteNodes(nodes_to_delete));
   return Status::OK();
 }
 
diff --git a/tensorflow/core/grappler/optimizers/data/make_numa_aware.cc b/tensorflow/core/grappler/optimizers/data/make_numa_aware.cc
index d8ea9adfb7..221f4c2525 100644
--- a/tensorflow/core/grappler/optimizers/data/make_numa_aware.cc
+++ b/tensorflow/core/grappler/optimizers/data/make_numa_aware.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/core/grappler/optimizers/data/make_numa_aware.h"
 
+#include "absl/container/flat_hash_set.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/grappler/clusters/cluster.h"
 #include "tensorflow/core/grappler/grappler_item.h"
@@ -43,7 +44,7 @@ Status MakeNumaAware::OptimizeAndCollectStats(Cluster* cluster,
                                               OptimizationStats* stats) {
   *output = item.graph;
   MutableGraphView graph(output);
-  std::set<string> nodes_to_delete;
+  absl::flat_hash_set<string> nodes_to_delete;
 
   for (const NodeDef& node : item.graph.node()) {
     if (node.op() != "ExperimentalMapAndBatchDataset") continue;
@@ -53,7 +54,7 @@ Status MakeNumaAware::OptimizeAndCollectStats(Cluster* cluster,
     nodes_to_delete.insert(node.name());
     stats->num_changes++;
   }
-  graph.DeleteNodes(nodes_to_delete);
+  TF_RETURN_IF_ERROR(graph.DeleteNodes(nodes_to_delete));
   return Status::OK();
 }
 
diff --git a/tensorflow/core/grappler/optimizers/data/map_and_batch_fusion.cc b/tensorflow/core/grappler/optimizers/data/map_and_batch_fusion.cc
index 3aeac021e6..5d26d1abe4 100644
--- a/tensorflow/core/grappler/optimizers/data/map_and_batch_fusion.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_and_batch_fusion.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/core/grappler/optimizers/data/map_and_batch_fusion.h"
 
+#include "absl/container/flat_hash_set.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/grappler/clusters/cluster.h"
@@ -104,7 +105,7 @@ Status MapAndBatchFusion::OptimizeAndCollectStats(Cluster* cluster,
                                                   OptimizationStats* stats) {
   *output = item.graph;
   MutableGraphView graph(output);
-  std::set<string> nodes_to_delete;
+  absl::flat_hash_set<string> nodes_to_delete;
   for (const NodeDef& node : item.graph.node()) {
     if (node.op() != "BatchDataset" && node.op() != "BatchDatasetV2") {
       continue;
@@ -131,7 +132,7 @@ Status MapAndBatchFusion::OptimizeAndCollectStats(Cluster* cluster,
     stats->num_changes++;
   }
 
-  graph.DeleteNodes(nodes_to_delete);
+  TF_RETURN_IF_ERROR(graph.DeleteNodes(nodes_to_delete));
   return Status::OK();
 }
 
diff --git a/tensorflow/core/grappler/optimizers/data/map_and_filter_fusion.cc b/tensorflow/core/grappler/optimizers/data/map_and_filter_fusion.cc
index 1ed4eae4ba..e257683b35 100644
--- a/tensorflow/core/grappler/optimizers/data/map_and_filter_fusion.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_and_filter_fusion.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/core/grappler/optimizers/data/map_and_filter_fusion.h"
 
+#include "absl/container/flat_hash_set.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/grappler/clusters/cluster.h"
@@ -103,7 +104,7 @@ Status MapAndFilterFusion::OptimizeAndCollectStats(Cluster* cluster,
   *output = sorted_old_graph;
 
   MutableGraphView graph(output);
-  std::set<string> nodes_to_delete;
+  absl::flat_hash_set<string> nodes_to_delete;
   FunctionLibraryDefinition function_library(OpRegistry::Global(),
                                              item.graph.library());
   auto get_map_node = [](const NodeDef& node) -> const NodeDef* {
@@ -168,7 +169,7 @@ Status MapAndFilterFusion::OptimizeAndCollectStats(Cluster* cluster,
     stats->num_changes++;
   }
 
-  graph.DeleteNodes(nodes_to_delete);
+  TF_RETURN_IF_ERROR(graph.DeleteNodes(nodes_to_delete));
   return Status::OK();
 }
 
diff --git a/tensorflow/core/grappler/optimizers/data/map_fusion.cc b/tensorflow/core/grappler/optimizers/data/map_fusion.cc
index 66f8404c05..ce41f7069c 100644
--- a/tensorflow/core/grappler/optimizers/data/map_fusion.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_fusion.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/core/grappler/optimizers/data/map_fusion.h"
 
+#include "absl/container/flat_hash_set.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/grappler/clusters/cluster.h"
@@ -86,7 +87,7 @@ Status MapFusion::OptimizeAndCollectStats(Cluster* cluster,
   *output = sorted_old_graph;
 
   MutableGraphView graph(output);
-  std::set<string> nodes_to_delete;
+  absl::flat_hash_set<string> nodes_to_delete;
   FunctionLibraryDefinition function_library(OpRegistry::Global(),
                                              item.graph.library());
 
@@ -147,7 +148,7 @@ Status MapFusion::OptimizeAndCollectStats(Cluster* cluster,
     stats->num_changes++;
   }
 
-  graph.DeleteNodes(nodes_to_delete);
+  TF_RETURN_IF_ERROR(graph.DeleteNodes(nodes_to_delete));
   return Status::OK();
 }
 
diff --git a/tensorflow/core/grappler/optimizers/data/map_parallelization.cc b/tensorflow/core/grappler/optimizers/data/map_parallelization.cc
index d8f01d720a..57e541cde6 100644
--- a/tensorflow/core/grappler/optimizers/data/map_parallelization.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_parallelization.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/core/grappler/optimizers/data/map_parallelization.h"
 
+#include "absl/container/flat_hash_set.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/grappler/clusters/cluster.h"
 #include "tensorflow/core/grappler/grappler_item.h"
@@ -73,7 +74,7 @@ Status MapParallelization::OptimizeAndCollectStats(Cluster* cluster,
                                                    OptimizationStats* stats) {
   *output = item.graph;
   MutableGraphView graph(output);
-  std::set<string> nodes_to_delete;
+  absl::flat_hash_set<string> nodes_to_delete;
   FunctionLibraryDefinition function_library(OpRegistry::Global(),
                                              item.graph.library());
   auto get_map_node = [](const NodeDef& node) -> const NodeDef* {
@@ -97,7 +98,7 @@ Status MapParallelization::OptimizeAndCollectStats(Cluster* cluster,
     stats->num_changes++;
   }
 
-  graph.DeleteNodes(nodes_to_delete);
+  TF_RETURN_IF_ERROR(graph.DeleteNodes(nodes_to_delete));
   return Status::OK();
 }
 
diff --git a/tensorflow/core/grappler/optimizers/data/map_vectorization.cc b/tensorflow/core/grappler/optimizers/data/map_vectorization.cc
index 848642041b..4c9e1f31e7 100644
--- a/tensorflow/core/grappler/optimizers/data/map_vectorization.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_vectorization.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #include "tensorflow/core/grappler/optimizers/data/map_vectorization.h"
 #include "tensorflow/core/grappler/optimizers/data/vectorization_utils.h"
 
+#include "absl/container/flat_hash_set.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/framework/tensor.pb.h"  // NOLINT
@@ -218,7 +219,7 @@ Status MapVectorization::OptimizeAndCollectStats(Cluster* cluster,
                                                  OptimizationStats* stats) {
   *output = item.graph;
   MutableGraphView graph(output);
-  std::set<string> nodes_to_delete;
+  absl::flat_hash_set<string> nodes_to_delete;
 
   for (const NodeDef& node : item.graph.node()) {
     // Find Map->Batch nodes.
@@ -274,7 +275,7 @@ Status MapVectorization::OptimizeAndCollectStats(Cluster* cluster,
     nodes_to_delete.insert(batch_node.name());
     stats->num_changes++;
   }
-  graph.DeleteNodes(nodes_to_delete);
+  TF_RETURN_IF_ERROR(graph.DeleteNodes(nodes_to_delete));
   return Status::OK();
 }
 
diff --git a/tensorflow/core/grappler/optimizers/data/noop_elimination.cc b/tensorflow/core/grappler/optimizers/data/noop_elimination.cc
index e474fddcc0..851bbbdc1a 100644
--- a/tensorflow/core/grappler/optimizers/data/noop_elimination.cc
+++ b/tensorflow/core/grappler/optimizers/data/noop_elimination.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/core/grappler/optimizers/data/noop_elimination.h"
 
+#include "absl/container/flat_hash_set.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/grappler/clusters/cluster.h"
@@ -76,7 +77,7 @@ Status NoOpElimination::OptimizeAndCollectStats(Cluster* cluster,
                                                 OptimizationStats* stats) {
   *output = item.graph;
   MutableGraphView graph(output);
-  std::set<string> nodes_to_delete;
+  absl::flat_hash_set<string> nodes_to_delete;
   for (const NodeDef& node : item.graph.node()) {
     if (!IsNoOp(node, graph)) continue;
 
@@ -87,7 +88,7 @@ Status NoOpElimination::OptimizeAndCollectStats(Cluster* cluster,
     stats->num_changes++;
   }
 
-  graph.DeleteNodes(nodes_to_delete);
+  TF_RETURN_IF_ERROR(graph.DeleteNodes(nodes_to_delete));
   return Status::OK();
 }
 
diff --git a/tensorflow/core/grappler/optimizers/data/shuffle_and_repeat_fusion.cc b/tensorflow/core/grappler/optimizers/data/shuffle_and_repeat_fusion.cc
index f30b2230ff..ff64ff1adb 100644
--- a/tensorflow/core/grappler/optimizers/data/shuffle_and_repeat_fusion.cc
+++ b/tensorflow/core/grappler/optimizers/data/shuffle_and_repeat_fusion.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/core/grappler/optimizers/data/shuffle_and_repeat_fusion.h"
 
+#include "absl/container/flat_hash_set.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/grappler/clusters/cluster.h"
@@ -39,7 +40,7 @@ Status ShuffleAndRepeatFusion::OptimizeAndCollectStats(
     OptimizationStats* stats) {
   *output = item.graph;
   MutableGraphView graph(output);
-  std::set<string> nodes_to_delete;
+  absl::flat_hash_set<string> nodes_to_delete;
 
   auto make_shuffle_and_repeat_node = [&output](const NodeDef& shuffle_node,
                                                 const NodeDef& repeat_node) {
@@ -95,7 +96,7 @@ Status ShuffleAndRepeatFusion::OptimizeAndCollectStats(
     stats->num_changes++;
   }
 
-  graph.DeleteNodes(nodes_to_delete);
+  TF_RETURN_IF_ERROR(graph.DeleteNodes(nodes_to_delete));
   return Status::OK();
 }
 
-- 
GitLab


From fa016bec45651ecdd789dbb197bca65ea3ab80d4 Mon Sep 17 00:00:00 2001
From: Cong Liu <congliu@google.com>
Date: Thu, 17 Jan 2019 11:09:33 -0800
Subject: [PATCH 0884/1095] [XLA] Actually test the dynamic dimension inference
 for select-and-scatter.

PiperOrigin-RevId: 229782019
---
 .../dynamic_dimension_inference_test.cc       | 25 +++++++++++++++----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/tensorflow/compiler/xla/service/dynamic_dimension_inference_test.cc b/tensorflow/compiler/xla/service/dynamic_dimension_inference_test.cc
index 1dd196821c..b42e67b4bb 100644
--- a/tensorflow/compiler/xla/service/dynamic_dimension_inference_test.cc
+++ b/tensorflow/compiler/xla/service/dynamic_dimension_inference_test.cc
@@ -62,6 +62,17 @@ class DynamicDimensionInferenceTest : public HloTestBase {
     return module_->AddEmbeddedComputation(embedded_builder.Build());
   }
 
+  HloComputation* GetGe() {
+    auto embedded_builder = HloComputation::Builder("ge");
+    auto lhs = embedded_builder.AddInstruction(HloInstruction::CreateParameter(
+        0, ShapeUtil::MakeShape(F32, {}), "lhs"));
+    auto rhs = embedded_builder.AddInstruction(HloInstruction::CreateParameter(
+        1, ShapeUtil::MakeShape(F32, {}), "rhs"));
+    embedded_builder.AddInstruction(HloInstruction::CreateBinary(
+        ShapeUtil::MakeShape(PRED, {}), HloOpcode::kGe, lhs, rhs));
+    return module_->AddEmbeddedComputation(embedded_builder.Build());
+  }
+
   std::unique_ptr<HloModule> module_;
   std::unique_ptr<DynamicDimensionInference> inference_;
   const Shape scalar_shape_ = ShapeUtil::MakeShape(S32, {});
@@ -487,7 +498,7 @@ TEST_F(DynamicDimensionInferenceTest, SelectAndScatterTest) {
   // Test the ability to trace select and scatter batch dimensions.
   auto builder = HloComputation::Builder(TestName());
   auto input_shape = ShapeUtil::MakeShape(F32, {2, 4, 4});
-  auto output_shape = ShapeUtil::MakeShape(F32, {2, 2, 2});
+  auto source_shape = ShapeUtil::MakeShape(F32, {2, 2, 2});
 
   Window window;
   // First dimension is unchanged.
@@ -514,22 +525,26 @@ TEST_F(DynamicDimensionInferenceTest, SelectAndScatterTest) {
       /*parameter_number=*/0, input_shape, "A"));
   auto* size_param = builder.AddInstruction(HloInstruction::CreateParameter(
       /*parameter_number=*/1, scalar_shape_, "size_param"));
+  auto* source = builder.AddInstruction(HloInstruction::CreateParameter(
+      /*parameter_number=*/2, source_shape, "B"));
 
   auto init = builder.AddInstruction(
       HloInstruction::CreateConstant(LiteralUtil::CreateR0<float>(0.0)));
 
-  auto* reduce_window =
-      builder.AddInstruction(HloInstruction::CreateReduceWindow(
-          output_shape, a_param, init, window, GetAdd()));
+  auto* sns = builder.AddInstruction(HloInstruction::CreateSelectAndScatter(
+      input_shape, a_param, GetGe(), window, source, init, GetAdd()));
 
   module_->AddEntryComputation(builder.Build());
 
   TF_CHECK_OK(module_->dynamic_parameter_binding().Bind(
       DynamicParameterBinding::DynamicParameter{1, {}},
       DynamicParameterBinding::DynamicDimension{0, {}, 0}));
+  TF_CHECK_OK(module_->dynamic_parameter_binding().Bind(
+      DynamicParameterBinding::DynamicParameter{1, {}},
+      DynamicParameterBinding::DynamicDimension{2, {}, 0}));
 
   TF_ASSERT_OK(RunInference());
-  EXPECT_EQ(inference_->GetDynamicSize(reduce_window, {}, 0), size_param);
+  EXPECT_EQ(inference_->GetDynamicSize(sns, {}, 0), size_param);
 }
 
 }  // namespace
-- 
GitLab


From 7462a8fcd7c526a00fa62036cde53f5d989c5a45 Mon Sep 17 00:00:00 2001
From: Michael Kuperstein <mkuper@google.com>
Date: Thu, 17 Jan 2019 11:49:03 -0800
Subject: [PATCH 0885/1095] [XLA] Remove the old HloInstruction
 CreateDS/CreateDUS interfaces.

Migrate all remaining use to thes new Span interfaces. Note that
this does not enforce using R0 indices - it's still possible to
pass a span with a single R1 element. That happens in practice
when going through the deprecated version of the XlaBuilder API.

PiperOrigin-RevId: 229790190
---
 .../xla/service/buffer_liveness_test.cc       |  8 +--
 .../cpu/cpu_instruction_fusion_test.cc        | 51 ++++++++-----------
 .../compiler/xla/service/hlo_instruction.cc   | 49 +++++-------------
 .../compiler/xla/service/hlo_instruction.h    | 10 ----
 .../service/tuple_points_to_analysis_test.cc  | 16 +++---
 tensorflow/compiler/xla/tests/fusion_test.cc  |  4 +-
 6 files changed, 47 insertions(+), 91 deletions(-)

diff --git a/tensorflow/compiler/xla/service/buffer_liveness_test.cc b/tensorflow/compiler/xla/service/buffer_liveness_test.cc
index 46218e6499..23b9af0281 100644
--- a/tensorflow/compiler/xla/service/buffer_liveness_test.cc
+++ b/tensorflow/compiler/xla/service/buffer_liveness_test.cc
@@ -638,10 +638,10 @@ class FusedDynamicUpdateSliceLivenessTest : public BufferLivenessTest {
     }
     // Create a DynamicUpdateSlice instruction of tuple element 1 with 'update'.
     auto starts = builder.AddInstruction(
-        HloInstruction::CreateConstant(LiteralUtil::CreateR1<int32>({2})));
+        HloInstruction::CreateConstant(LiteralUtil::CreateR0<int32>(2)));
     auto dynamic_update_slice =
         builder.AddInstruction(HloInstruction::CreateDynamicUpdateSlice(
-            data_shape, gte1, update, starts));
+            data_shape, gte1, update, {starts}));
     // Create output tuple.
     builder.AddInstruction(
         HloInstruction::CreateTuple({gte0, dynamic_update_slice}));
@@ -794,10 +794,10 @@ class DynamicUpdateSliceLivenessTest : public BufferLivenessTest {
     }
     // Create a DynamicUpdateSlice instruction of tuple element 1 with 'update'.
     auto starts = builder.AddInstruction(
-        HloInstruction::CreateConstant(LiteralUtil::CreateR1<int32>({2})));
+        HloInstruction::CreateConstant(LiteralUtil::CreateR0<int32>(2)));
     auto dynamic_update_slice =
         builder.AddInstruction(HloInstruction::CreateDynamicUpdateSlice(
-            data_shape, gte1, update, starts));
+            data_shape, gte1, update, {starts}));
     // Create output tuple.
     auto tuple_root = builder.AddInstruction(
         HloInstruction::CreateTuple({gte0, dynamic_update_slice}));
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion_test.cc b/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion_test.cc
index 86cc72dd7c..c4bde837e5 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion_test.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion_test.cc
@@ -390,10 +390,8 @@ TEST_F(OpcodeFusionTest, DynamicSlice_Negate) {
       HloInstruction::CreateParameter(0, param_shape, "param"));
   HloInstruction* param1 = builder.AddInstruction(
       HloInstruction::CreateParameter(1, slice_shape, "starts"));
-  HloInstruction* dynamic_slice2 =
-      builder.AddInstruction(HloInstruction::CreateDynamicSlice(
-          result_shape, param0,
-          std::initializer_list<HloInstruction*>({param1}), {2}));
+  HloInstruction* dynamic_slice2 = builder.AddInstruction(
+      HloInstruction::CreateDynamicSlice(result_shape, param0, {param1}, {2}));
   builder.AddInstruction(HloInstruction::CreateUnary(
       result_shape, HloOpcode::kNegate, dynamic_slice2));
 
@@ -591,49 +589,40 @@ TEST_F(OpcodeFusionTest, MessOfFusibleNodes) {
 
   Shape full_shape = ShapeUtil::MakeShape(F32, {4, 100, 10, 100, 50});
 
-  auto loop_idx = builder.AddInstruction(HloInstruction::CreateReshape(
-      ShapeUtil::MakeShape(S32, {1}),
-      builder.AddInstruction(HloInstruction::CreateParameter(
-          0, ShapeUtil::MakeShape(S32, {}), "param0"))));
-
+  auto loop_idx = builder.AddInstruction(HloInstruction::CreateParameter(
+      0, ShapeUtil::MakeShape(S32, {}), "param0"));
   auto param1 = builder.AddInstruction(HloInstruction::CreateParameter(
-      1, ShapeUtil::MakeShape(S32, {1}), "param1"));
-  auto concat = builder.AddInstruction(HloInstruction::CreateConcatenate(
-      ShapeUtil::MakeShape(S32, {5}),
-      {loop_idx, param1, param1, param1, param1}, /*dimension=*/0));
+      1, ShapeUtil::MakeShape(S32, {}), "param1"));
 
-  auto idx_choice = builder.AddInstruction(HloInstruction::CreateDynamicSlice(
-      ShapeUtil::MakeShape(S32, {1}),
-      builder.AddInstruction(HloInstruction::CreateParameter(
-          2, ShapeUtil::MakeShape(S32, {4}), "param2")),
-      loop_idx,
-      /*slice_sizes=*/{1}));
-
-  PaddingConfig padding_config;
-  padding_config.add_dimensions()->set_edge_padding_high(4);
-  auto pad = builder.AddInstruction(HloInstruction::CreatePad(
-      ShapeUtil::MakeShape(S32, {5}), idx_choice,
-      builder.AddInstruction(
-          HloInstruction::CreateConstant(LiteralUtil::CreateR0(0))),
-      padding_config));
+  auto idx_choice = builder.AddInstruction(HloInstruction::CreateReshape(
+      ShapeUtil::MakeShape(S32, {}),
+      builder.AddInstruction(HloInstruction::CreateDynamicSlice(
+          ShapeUtil::MakeShape(S32, {1}),
+          builder.AddInstruction(HloInstruction::CreateParameter(
+              2, ShapeUtil::MakeShape(S32, {4}), "param2")),
+          {loop_idx},
+          /*slice_sizes=*/{1}))));
+  auto zero = builder.AddInstruction(
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0(0)));
 
   auto slice = builder.AddInstruction(HloInstruction::CreateDynamicSlice(
       ShapeUtil::MakeShape(F32, {1, 100, 10, 100, 50}),
       builder.AddInstruction(HloInstruction::CreateParameter(
           3, ShapeUtil::MakeShape(F32, {100, 100, 10, 100, 50}), "param3")),
-      pad, /*slice_sizes=*/{1, 100, 10, 100, 50}));
+      {idx_choice, zero, zero, zero, zero},
+      /*slice_sizes=*/{1, 100, 10, 100, 50}));
 
   builder.AddInstruction(HloInstruction::CreateDynamicUpdateSlice(
       full_shape,
       builder.AddInstruction(
           HloInstruction::CreateParameter(4, full_shape, "param4")),
-      slice, concat));
+      slice, {loop_idx, param1, param1, param1, param1}));
 
   module->AddEntryComputation(builder.Build());
   RunFusionAndCheckOpcodesWereFused(
       module.get(),
-      {HloOpcode::kConcatenate, HloOpcode::kPad, HloOpcode::kDynamicSlice,
-       HloOpcode::kDynamicSlice, HloOpcode::kDynamicUpdateSlice,
+      {HloOpcode::kDynamicSlice, HloOpcode::kDynamicSlice,
+       HloOpcode::kDynamicUpdateSlice, HloOpcode::kReshape,
        HloOpcode::kParameter, HloOpcode::kParameter, HloOpcode::kParameter,
        HloOpcode::kParameter, HloOpcode::kParameter, HloOpcode::kParameter});
 }
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc
index 029d170317..3c92554ad4 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.cc
+++ b/tensorflow/compiler/xla/service/hlo_instruction.cc
@@ -458,21 +458,17 @@ StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
           << "DynamicSlice instruction should have at least 1 operands but "
              "sees "
           << proto.operand_ids_size();
-      if (proto.operand_ids_size() == 2 && operands(1)->shape().rank() == 1) {
-        // TODO(b/118437727): Old form, remove this path.
-        instruction =
-            CreateDynamicSlice(shape, operands(0), operands(1), slice_sizes);
-      } else {
-        // New form
+      // TODO(b/118437727): Old form, make the check unconditional.
+      if (proto.operand_ids_size() != 2 || operands(1)->shape().rank() != 1) {
         auto expected_operands = 1 + operands(0)->shape().rank();
         TF_RET_CHECK(proto.operand_ids_size() == expected_operands)
             << "DynamicSlice instruction should have " << expected_operands
             << " operands, but has " << proto.operand_ids_size();
-        const auto& operand_vector = all_operands();
-        instruction = CreateDynamicSlice(
-            shape, operands(0), absl::MakeSpan(operand_vector).subspan(1),
-            slice_sizes);
       }
+      const auto& operand_vector = all_operands();
+      instruction = CreateDynamicSlice(
+          shape, operands(0), absl::MakeSpan(operand_vector).subspan(1),
+          slice_sizes);
       break;
     }
     case HloOpcode::kDynamicUpdateSlice: {
@@ -480,22 +476,19 @@ StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
           << "DynamicUpdateSlice instruction should have at least 2 operands "
              "but sees "
           << proto.operand_ids_size();
-      if (proto.operand_ids_size() == 3 && operands(2)->shape().rank() == 1) {
-        // TODO(b/118437727): Old form, remove this path.
-        instruction = CreateDynamicUpdateSlice(shape, operands(0), operands(1),
-                                               operands(2));
-      } else {
-        // New form
+      // TODO(b/118437727): Old form, make the check unconditional.
+      if (proto.operand_ids_size() != 3 || operands(2)->shape().rank() != 1) {
         auto expected_operands = 2 + operands(0)->shape().rank();
         TF_RET_CHECK(proto.operand_ids_size() == expected_operands)
             << "DynamicUpdateSlice instruction should have "
             << expected_operands << " operands, but has "
             << proto.operand_ids_size();
-        const auto& operand_vector = all_operands();
-        instruction =
-            CreateDynamicUpdateSlice(shape, operands(0), operands(1),
-                                     absl::MakeSpan(operand_vector).subspan(2));
       }
+      const auto& operand_vector = all_operands();
+      instruction =
+          CreateDynamicUpdateSlice(shape, operands(0), operands(1),
+                                   absl::MakeSpan(operand_vector).subspan(2));
+
       break;
     }
     case HloOpcode::kGather: {
@@ -947,13 +940,6 @@ HloInstruction::CreateAddDependency(HloInstruction* data_operand,
                                                 limit_indices, strides);
 }
 
-/* static */ std::unique_ptr<HloInstruction> HloInstruction::CreateDynamicSlice(
-    const Shape& shape, HloInstruction* operand, HloInstruction* start_indices,
-    absl::Span<const int64> slice_sizes) {
-  return absl::make_unique<HloDynamicSliceInstruction>(
-      shape, operand, start_indices, slice_sizes);
-}
-
 /* static */ std::unique_ptr<HloInstruction> HloInstruction::CreateDynamicSlice(
     const Shape& shape, HloInstruction* operand,
     absl::Span<HloInstruction* const> start_indices,
@@ -962,15 +948,6 @@ HloInstruction::CreateAddDependency(HloInstruction* data_operand,
       shape, operand, start_indices, slice_sizes);
 }
 
-/* static */ std::unique_ptr<HloInstruction>
-HloInstruction::CreateDynamicUpdateSlice(const Shape& shape,
-                                         HloInstruction* operand,
-                                         HloInstruction* update,
-                                         HloInstruction* start_indices) {
-  return absl::make_unique<HloDynamicUpdateSliceInstruction>(
-      shape, operand, update, start_indices);
-}
-
 /* static */ std::unique_ptr<HloInstruction>
 HloInstruction::CreateDynamicUpdateSlice(
     const Shape& shape, HloInstruction* operand, HloInstruction* update,
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.h b/tensorflow/compiler/xla/service/hlo_instruction.h
index 789ace6a26..2c29b6c243 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.h
+++ b/tensorflow/compiler/xla/service/hlo_instruction.h
@@ -556,11 +556,6 @@ class HloInstruction {
   // Creates a slice instruction, where the first operand is sliced by
   // start indices specified in the second operand, and by size specified in
   // 'slice_sizes'.
-  ABSL_DEPRECATED("Use span-of-indices form instead")
-  static std::unique_ptr<HloInstruction> CreateDynamicSlice(
-      const Shape& shape, HloInstruction* operand,
-      HloInstruction* start_indices, absl::Span<const int64> slice_sizes);
-  // Same as above, but expects a span of scalar start indices.
   static std::unique_ptr<HloInstruction> CreateDynamicSlice(
       const Shape& shape, HloInstruction* operand,
       absl::Span<HloInstruction* const> start_indices,
@@ -568,11 +563,6 @@ class HloInstruction {
 
   // Creates a dynamic update slice instruction, which updates a slice
   // of 'operand' with 'update' and 'start_indices'.
-  ABSL_DEPRECATED("Use span-of-indices form instead")
-  static std::unique_ptr<HloInstruction> CreateDynamicUpdateSlice(
-      const Shape& shape, HloInstruction* operand, HloInstruction* update,
-      HloInstruction* start_indices);
-  // Same as above, but expects a span of scalar start indices.
   static std::unique_ptr<HloInstruction> CreateDynamicUpdateSlice(
       const Shape& shape, HloInstruction* operand, HloInstruction* update,
       absl::Span<HloInstruction* const> start_indices);
diff --git a/tensorflow/compiler/xla/service/tuple_points_to_analysis_test.cc b/tensorflow/compiler/xla/service/tuple_points_to_analysis_test.cc
index 7599e1e6ad..fd5759e442 100644
--- a/tensorflow/compiler/xla/service/tuple_points_to_analysis_test.cc
+++ b/tensorflow/compiler/xla/service/tuple_points_to_analysis_test.cc
@@ -623,7 +623,7 @@ class FusionPointsToAnalysisTest : public TuplePointsToAnalysisTest {
   void Run(const bool add_additional_gte0_user) {
     Shape input_shape = ShapeUtil::MakeShape(F32, {8});
     Shape update_shape = ShapeUtil::MakeShape(F32, {3});
-    Shape starts_shape = ShapeUtil::MakeShape(S32, {1});
+    Shape starts_shape = ShapeUtil::MakeShape(S32, {});
     Shape tuple_shape =
         ShapeUtil::MakeTupleShape({input_shape, update_shape, starts_shape});
 
@@ -657,7 +657,7 @@ class FusionPointsToAnalysisTest : public TuplePointsToAnalysisTest {
         HloInstruction::CreateGetTupleElement(starts_shape, tuple_param0, 2));
     // Update 'input' with 'update' at dynamic 'starts' indices.
     builder.AddInstruction(HloInstruction::CreateDynamicUpdateSlice(
-        input_shape, input, update, starts));
+        input_shape, input, update, {starts}));
 
     // Build computation and add it to module as entry computation.
     BuildModule(builder.Build());
@@ -882,12 +882,12 @@ TEST_F(DoesNotUseOperandBufferTest, FusedDynamicUpdateSlice) {
 
   // Create a DynamicUpdateSlice instruction of tuple element 1.
   auto starts = builder.AddInstruction(
-      HloInstruction::CreateConstant(LiteralUtil::CreateR1<int32>({2})));
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0<int32>(2)));
   auto update = builder.AddInstruction(HloInstruction::CreateConstant(
       LiteralUtil::CreateR1<float>({2.f, 2.f, 2.f})));
   auto dynamic_update_slice =
       builder.AddInstruction(HloInstruction::CreateDynamicUpdateSlice(
-          data_shape, gte1, update, starts));
+          data_shape, gte1, update, {starts}));
   builder.AddInstruction(
       HloInstruction::CreateTuple({gte0, dynamic_update_slice}));
 
@@ -976,12 +976,12 @@ TEST_F(CanShareOperandBufferWithUserTest, FusedDynamicUpdateSlice) {
 
   // Create a DynamicUpdateSlice instruction of tuple element 1.
   auto starts = builder.AddInstruction(
-      HloInstruction::CreateConstant(LiteralUtil::CreateR1<int32>({2})));
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0<int32>(2)));
   auto update = builder.AddInstruction(HloInstruction::CreateConstant(
       LiteralUtil::CreateR1<float>({2.f, 2.f, 2.f})));
   auto dynamic_update_slice =
       builder.AddInstruction(HloInstruction::CreateDynamicUpdateSlice(
-          data_shape, gte1, update, starts));
+          data_shape, gte1, update, {starts}));
   builder.AddInstruction(
       HloInstruction::CreateTuple({gte0, dynamic_update_slice}));
 
@@ -1003,7 +1003,7 @@ TEST_F(CanShareOperandBufferWithUserTest, DynamicUpdateSliceCanShare) {
 
   Shape data_shape = ShapeUtil::MakeShape(F32, {8});
   Shape update_shape = ShapeUtil::MakeShape(F32, {4});
-  Shape starts_shape = ShapeUtil::MakeShape(S32, {1});
+  Shape starts_shape = ShapeUtil::MakeShape(S32, {});
   auto data = builder.AddInstruction(
       HloInstruction::CreateParameter(0, data_shape, "data"));
   auto update = builder.AddInstruction(
@@ -1011,7 +1011,7 @@ TEST_F(CanShareOperandBufferWithUserTest, DynamicUpdateSliceCanShare) {
   auto starts = builder.AddInstruction(
       HloInstruction::CreateParameter(2, starts_shape, "starts"));
   auto dus = builder.AddInstruction(HloInstruction::CreateDynamicUpdateSlice(
-      data_shape, data, update, starts));
+      data_shape, data, update, {starts}));
 
   BuildModuleAndRunAnalysis(builder.Build());
 
diff --git a/tensorflow/compiler/xla/tests/fusion_test.cc b/tensorflow/compiler/xla/tests/fusion_test.cc
index d1fddf9d6b..2178c9b3f3 100644
--- a/tensorflow/compiler/xla/tests/fusion_test.cc
+++ b/tensorflow/compiler/xla/tests/fusion_test.cc
@@ -523,10 +523,10 @@ XLA_TEST_F(FusionTest, DynamicSliceNegate) {
   auto const0 = builder.AddInstruction(HloInstruction::CreateConstant(
       LiteralUtil::CreateR1<int32>({1, 2, 3, 4})));
   auto const1 = builder.AddInstruction(
-      HloInstruction::CreateConstant(LiteralUtil::CreateR1<int32>({1})));
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0<int32>(1)));
   auto dynamic_slice2 =
       builder.AddInstruction(HloInstruction::CreateDynamicSlice(
-          ShapeUtil::MakeShape(S32, {2}), const0, const1, {2}));
+          ShapeUtil::MakeShape(S32, {2}), const0, {const1}, {2}));
   auto negate3 = builder.AddInstruction(HloInstruction::CreateUnary(
       ShapeUtil::MakeShape(S32, {2}), HloOpcode::kNegate, dynamic_slice2));
   hlo_module->AddEntryComputation(builder.Build())
-- 
GitLab


From 5c08b1f021e0a40fece750b5ebc0cf85b3281265 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 17 Jan 2019 11:52:56 -0800
Subject: [PATCH 0886/1095] Remove alias redirection for RBE.

PiperOrigin-RevId: 229790895
---
 tensorflow/opensource_only.files            |   3 -
 third_party/gpus/crosstool/remote.BUILD.tpl |  10 --
 third_party/gpus/cuda/remote.BUILD.tpl      | 110 --------------------
 third_party/gpus/cuda_configure.bzl         |  17 ++-
 third_party/tensorrt/build_defs.bzl.tpl     |   4 +-
 third_party/tensorrt/remote.BUILD.tpl       |   7 --
 third_party/tensorrt/tensorrt_configure.bzl |  11 +-
 7 files changed, 14 insertions(+), 148 deletions(-)
 delete mode 100644 third_party/gpus/crosstool/remote.BUILD.tpl
 delete mode 100644 third_party/gpus/cuda/remote.BUILD.tpl
 delete mode 100644 third_party/tensorrt/remote.BUILD.tpl

diff --git a/tensorflow/opensource_only.files b/tensorflow/opensource_only.files
index 30dfce3c83..5f1ec1d673 100644
--- a/tensorflow/opensource_only.files
+++ b/tensorflow/opensource_only.files
@@ -82,7 +82,6 @@ tensorflow/third_party/gpus/crosstool/clang/bin/crosstool_wrapper_driver_is_not_
 tensorflow/third_party/gpus/crosstool/CROSSTOOL.tpl
 tensorflow/third_party/gpus/crosstool/CROSSTOOL_hipcc.tpl
 tensorflow/third_party/gpus/crosstool/LICENSE
-tensorflow/third_party/gpus/crosstool/remote.BUILD.tpl
 tensorflow/third_party/gpus/crosstool/windows/msvc_wrapper_for_nvcc.py.tpl
 tensorflow/third_party/gpus/crosstool/BUILD.tpl
 tensorflow/third_party/gpus/crosstool/BUILD
@@ -90,7 +89,6 @@ tensorflow/third_party/gpus/cuda/LICENSE
 tensorflow/third_party/gpus/cuda/BUILD.tpl
 tensorflow/third_party/gpus/cuda/BUILD.windows.tpl
 tensorflow/third_party/gpus/cuda/cuda_config.h.tpl
-tensorflow/third_party/gpus/cuda/remote.BUILD.tpl
 tensorflow/third_party/gpus/cuda/BUILD
 tensorflow/third_party/gpus/cuda/build_defs.bzl.tpl
 tensorflow/third_party/gpus/rocm/rocm_config.h.tpl
@@ -194,7 +192,6 @@ tensorflow/third_party/tensorrt/BUILD
 tensorflow/third_party/tensorrt/build_defs.bzl.tpl
 tensorflow/third_party/tensorrt/BUILD.tpl
 tensorflow/third_party/tensorrt/tensorrt_configure.bzl
-tensorflow/third_party/tensorrt/remote.BUILD.tpl
 tensorflow/third_party/kafka/config.patch
 tensorflow/third_party/kafka/BUILD
 tensorflow/third_party/android/BUILD
diff --git a/third_party/gpus/crosstool/remote.BUILD.tpl b/third_party/gpus/crosstool/remote.BUILD.tpl
deleted file mode 100644
index b2316331db..0000000000
--- a/third_party/gpus/crosstool/remote.BUILD.tpl
+++ /dev/null
@@ -1,10 +0,0 @@
-# Description:
-#   Template for crosstool Build file to use a pre-generated config.
-licenses(["restricted"])
-
-package(default_visibility = ["//visibility:public"])
-
-alias(
-    name = "toolchain",
-    actual = "%{remote_cuda_repo}:toolchain",
-)
diff --git a/third_party/gpus/cuda/remote.BUILD.tpl b/third_party/gpus/cuda/remote.BUILD.tpl
deleted file mode 100644
index 100c7bb7c4..0000000000
--- a/third_party/gpus/cuda/remote.BUILD.tpl
+++ /dev/null
@@ -1,110 +0,0 @@
-# Description:
-#   Template for cuda Build file to use a pre-generated config.
-licenses(["restricted"])  # MPL2, portions GPL v3, LGPL v3, BSD-like
-
-package(default_visibility = ["//visibility:public"])
-
-config_setting(
-    name = "using_nvcc",
-    values = {
-        "define": "using_cuda_nvcc=true",
-    },
-)
-
-config_setting(
-    name = "using_clang",
-    values = {
-        "define": "using_cuda_clang=true",
-    },
-)
-
-# Equivalent to using_clang && -c opt.
-config_setting(
-    name = "using_clang_opt",
-    values = {
-        "define": "using_cuda_clang=true",
-        "compilation_mode": "opt",
-    },
-)
-
-config_setting(
-    name = "darwin",
-    values = {"cpu": "darwin"},
-    visibility = ["//visibility:public"],
-)
-
-config_setting(
-    name = "freebsd",
-    values = {"cpu": "freebsd"},
-    visibility = ["//visibility:public"],
-)
-
-alias(
-    name = "cuda_headers",
-    actual = "%{remote_cuda_repo}/cuda:cuda_headers",
-)
-
-alias(
-    name = "cudart_static",
-    actual = "%{remote_cuda_repo}/cuda:cudart_static",
-)
-
-alias(
-    name = "cuda_driver",
-    actual = "%{remote_cuda_repo}/cuda:cuda_driver",
-)
-
-alias(
-    name = "cudart",
-    actual = "%{remote_cuda_repo}/cuda:cudart",
-)
-
-alias(
-    name = "cublas",
-    actual = "%{remote_cuda_repo}/cuda:cublas",
-)
-
-alias(
-    name = "cusolver",
-    actual = "%{remote_cuda_repo}/cuda:cusolver",
-)
-
-alias(
-    name = "cudnn",
-    actual = "%{remote_cuda_repo}/cuda:cudnn",
-)
-
-alias(
-    name = "cudnn_header",
-    actual = "%{remote_cuda_repo}/cuda:cudnn_header",
-)
-
-alias(
-    name = "cufft",
-    actual = "%{remote_cuda_repo}/cuda:cufft",
-)
-
-alias(
-    name = "curand",
-    actual = "%{remote_cuda_repo}/cuda:curand",
-)
-
-alias(
-    name = "cuda",
-    actual = "%{remote_cuda_repo}/cuda:cuda",
-)
-
-alias(
-    name = "cupti_headers",
-    actual = "%{remote_cuda_repo}/cuda:cupti_headers",
-)
-
-alias(
-    name = "cupti_dsos",
-    actual = "%{remote_cuda_repo}/cuda:cupti_dsos",
-)
-
-alias(
-    name = "libdevice_root",
-    actual = "%{remote_cuda_repo}/cuda:libdevice_root",
-)
diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl
index 1988c4ae96..beacf2f45a 100644
--- a/third_party/gpus/cuda_configure.bzl
+++ b/third_party/gpus/cuda_configure.bzl
@@ -1492,17 +1492,16 @@ def _create_remote_cuda_repository(repository_ctx, remote_config_repo):
               ),
       },
   )
-  _tpl(
-      repository_ctx,
-      "cuda:remote.BUILD",
-      {
-          "%{remote_cuda_repo}": remote_config_repo,
-      },
+  repository_ctx.template(
       "cuda/BUILD",
+      Label(remote_config_repo + "/cuda/BUILD"),
+      {},
+  )
+  repository_ctx.template(
+      "crosstool/BUILD",
+      Label(remote_config_repo + "/crosstool/BUILD"),
+      {},
   )
-  _tpl(repository_ctx, "crosstool:remote.BUILD", {
-      "%{remote_cuda_repo}": remote_config_repo,
-  }, "crosstool/BUILD")
 
 
 def _cuda_autoconf_impl(repository_ctx):
diff --git a/third_party/tensorrt/build_defs.bzl.tpl b/third_party/tensorrt/build_defs.bzl.tpl
index 0dc3a7ba2d..6d00513827 100644
--- a/third_party/tensorrt/build_defs.bzl.tpl
+++ b/third_party/tensorrt/build_defs.bzl.tpl
@@ -2,6 +2,4 @@
 
 def if_tensorrt(if_true, if_false=[]):
   """Tests whether TensorRT was enabled during the configure process."""
-  if %{tensorrt_is_configured}:
-    return if_true
-  return if_false
+  return %{if_tensorrt}
diff --git a/third_party/tensorrt/remote.BUILD.tpl b/third_party/tensorrt/remote.BUILD.tpl
deleted file mode 100644
index 7598e7aa4b..0000000000
--- a/third_party/tensorrt/remote.BUILD.tpl
+++ /dev/null
@@ -1,7 +0,0 @@
-licenses(["notice"])  # Apache 2.0
-
-package(default_visibility = ["//visibility:public"])
-
-alias(name="LICENSE", actual = "%{target}:LICENSE")
-alias(name = "tensorrt_headers", actual = "%{target}:tensorrt_headers")
-alias(name = "nv_infer", actual = "%{target}:nv_infer")
diff --git a/third_party/tensorrt/tensorrt_configure.bzl b/third_party/tensorrt/tensorrt_configure.bzl
index d1e116d9dd..7cf845d5cc 100644
--- a/third_party/tensorrt/tensorrt_configure.bzl
+++ b/third_party/tensorrt/tensorrt_configure.bzl
@@ -143,7 +143,7 @@ def _tpl(repository_ctx, tpl, substitutions):
 
 def _create_dummy_repository(repository_ctx):
   """Create a dummy TensorRT repository."""
-  _tpl(repository_ctx, "build_defs.bzl", {"%{tensorrt_is_configured}": "False"})
+  _tpl(repository_ctx, "build_defs.bzl", {"%{if_tensorrt}": "if_false"})
   substitutions = {
       "%{tensorrt_genrules}": "",
       "%{tensorrt_headers}": "",
@@ -158,11 +158,10 @@ def _tensorrt_configure_impl(repository_ctx):
   """Implementation of the tensorrt_configure repository rule."""
   if _TF_TENSORRT_CONFIG_REPO in repository_ctx.os.environ:
     # Forward to the pre-configured remote repository.
-    repository_ctx.template("BUILD", Label("//third_party/tensorrt:remote.BUILD.tpl"), {
-        "%{target}": repository_ctx.os.environ[_TF_TENSORRT_CONFIG_REPO],
-    })
+    remote_config_repo = repository_ctx.os.environ[_TF_TENSORRT_CONFIG_REPO]
+    repository_ctx.template("BUILD", Label(remote_config_repo + "/BUILD"), {})
     # Set up config file.
-    _tpl(repository_ctx, "build_defs.bzl", {"%{tensorrt_is_configured}": "True"})
+    _tpl(repository_ctx, "build_defs.bzl", {"%{if_tensorrt}": "if_true"})
     return
 
   if _TENSORRT_INSTALL_PATH not in repository_ctx.os.environ:
@@ -210,7 +209,7 @@ def _tensorrt_configure_impl(repository_ctx):
   ))
 
   # Set up config file.
-  _tpl(repository_ctx, "build_defs.bzl", {"%{tensorrt_is_configured}": "True"})
+  _tpl(repository_ctx, "build_defs.bzl", {"%{if_tensorrt}": "if_true"})
 
   # Set up BUILD file.
   substitutions = {
-- 
GitLab


From 2435a1875b574f5d299b1dee431ab5ceccd6132f Mon Sep 17 00:00:00 2001
From: Revan Sopher <rsopher@google.com>
Date: Thu, 17 Jan 2019 11:54:37 -0800
Subject: [PATCH 0887/1095] Switch out unittest for absltest, and point app.py
 to absl's app.py to resolve flag conflict.

PiperOrigin-RevId: 229791193
---
 tensorflow/python/BUILD                  |  6 +-
 tensorflow/python/platform/app.py        | 98 ++----------------------
 tensorflow/python/platform/googletest.py | 12 +--
 tensorflow/tools/pip_package/BUILD       |  5 ++
 4 files changed, 22 insertions(+), 99 deletions(-)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index b61c33f319..e0648085c3 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -213,6 +213,7 @@ py_library(
         ":pywrap_tensorflow",
         ":util",
         "//tensorflow/core:protos_all_py",
+        "@absl_py//absl:app",
         "@absl_py//absl/flags",
         "@six_archive//:six",
     ],
@@ -233,7 +234,10 @@ py_library(
     name = "platform_test",
     srcs = ["platform/googletest.py"],
     srcs_version = "PY2AND3",
-    deps = [":platform_benchmark"],
+    deps = [
+        ":platform_benchmark",
+        "@absl_py//absl/testing:absltest",
+    ],
 )
 
 tf_py_test(
diff --git a/tensorflow/python/platform/app.py b/tensorflow/python/platform/app.py
index 7b917235c0..303b70ff57 100644
--- a/tensorflow/python/platform/app.py
+++ b/tensorflow/python/platform/app.py
@@ -18,109 +18,23 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import errno as _errno
 import sys as _sys
 
+from absl.app import run as _run
+
 from tensorflow.python.platform import flags
 from tensorflow.python.util.tf_export import tf_export
 
 
-def _usage(shorthelp):
-  """Writes __main__'s docstring to stdout with some help text.
-
-  Args:
-    shorthelp: bool, if True, prints only flags from the main module,
-        rather than all flags.
-  """
-  doc = _sys.modules['__main__'].__doc__
-  if not doc:
-    doc = '\nUSAGE: %s [flags]\n' % _sys.argv[0]
-    doc = flags.text_wrap(doc, indent='       ', firstline_indent='')
-  else:
-    # Replace all '%s' with sys.argv[0], and all '%%' with '%'.
-    num_specifiers = doc.count('%') - 2 * doc.count('%%')
-    try:
-      doc %= (_sys.argv[0],) * num_specifiers
-    except (OverflowError, TypeError, ValueError):
-      # Just display the docstring as-is.
-      pass
-  if shorthelp:
-    flag_str = flags.FLAGS.main_module_help()
-  else:
-    flag_str = str(flags.FLAGS)
-  try:
-    _sys.stdout.write(doc)
-    if flag_str:
-      _sys.stdout.write('\nflags:\n')
-      _sys.stdout.write(flag_str)
-    _sys.stdout.write('\n')
-  except IOError as e:
-    # We avoid printing a huge backtrace if we get EPIPE, because
-    # "foo.par --help | less" is a frequent use case.
-    if e.errno != _errno.EPIPE:
-      raise
-
-
-class _HelpFlag(flags.BooleanFlag):
-  """Special boolean flag that displays usage and raises SystemExit."""
-  NAME = 'help'
-  SHORT_NAME = 'h'
-
-  def __init__(self):
-    super(_HelpFlag, self).__init__(
-        self.NAME, False, 'show this help', short_name=self.SHORT_NAME)
-
-  def parse(self, arg):
-    if arg:
-      _usage(shorthelp=True)
-      print()
-      print('Try --helpfull to get a list of all flags.')
-      _sys.exit(1)
-
-
-class _HelpshortFlag(_HelpFlag):
-  """--helpshort is an alias for --help."""
-  NAME = 'helpshort'
-  SHORT_NAME = None
-
-
-class _HelpfullFlag(flags.BooleanFlag):
-  """Display help for flags in main module and all dependent modules."""
-
-  def __init__(self):
-    super(_HelpfullFlag, self).__init__('helpfull', False, 'show full help')
-
-  def parse(self, arg):
-    if arg:
-      _usage(shorthelp=False)
-      _sys.exit(1)
-
-
-_define_help_flags_called = False
-
-
-def _define_help_flags():
-  global _define_help_flags_called
-  if not _define_help_flags_called:
-    flags.DEFINE_flag(_HelpFlag())
-    flags.DEFINE_flag(_HelpfullFlag())
-    flags.DEFINE_flag(_HelpshortFlag())
-    _define_help_flags_called = True
+def _parse_flags_tolerate_undef(argv):
+  """Parse args, returning any unknown flags (ABSL defaults to crashing)."""
+  return flags.FLAGS(_sys.argv if argv is None else argv, known_only=True)
 
 
 @tf_export(v1=['app.run'])
 def run(main=None, argv=None):
   """Runs the program with an optional 'main' function and 'argv' list."""
 
-  # Define help flags.
-  _define_help_flags()
-
-  # Parse known flags.
-  argv = flags.FLAGS(_sys.argv if argv is None else argv, known_only=True)
-
   main = main or _sys.modules['__main__'].main
 
-  # Call the main function, passing through any arguments
-  # to the final program.
-  _sys.exit(main(argv))
-
+  _run(main=main, argv=argv, flags_parser=_parse_flags_tolerate_undef)
diff --git a/tensorflow/python/platform/googletest.py b/tensorflow/python/platform/googletest.py
index fe4b0d0d37..802721e34b 100644
--- a/tensorflow/python/platform/googletest.py
+++ b/tensorflow/python/platform/googletest.py
@@ -13,7 +13,7 @@
 # limitations under the License.
 # ==============================================================================
 
-"""Imports unittest as a replacement for testing.pybase.googletest."""
+"""Imports absltest as a replacement for testing.pybase.googletest."""
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
@@ -26,7 +26,7 @@ import tempfile
 
 # go/tf-wildcard-import
 # pylint: disable=wildcard-import
-from unittest import *
+from absl.testing.absltest import *
 # pylint: enable=wildcard-import
 
 from tensorflow.python.framework import errors
@@ -41,7 +41,7 @@ from tensorflow.python.util.tf_export import tf_export
 
 Benchmark = benchmark.TensorFlowBenchmark  # pylint: disable=invalid-name
 
-unittest_main = main
+absltest_main = main
 
 # We keep a global variable in this module to make sure we create the temporary
 # directory only once per test binary invocation.
@@ -51,7 +51,7 @@ _googletest_temp_dir = ''
 # pylint: disable=invalid-name
 # pylint: disable=undefined-variable
 def g_main(argv):
-  """Delegate to unittest.main after redefining testLoader."""
+  """Delegate to absltest.main after redefining testLoader."""
   if 'TEST_SHARD_STATUS_FILE' in os.environ:
     try:
       f = None
@@ -67,7 +67,7 @@ def g_main(argv):
 
   if ('TEST_TOTAL_SHARDS' not in os.environ or
       'TEST_SHARD_INDEX' not in os.environ):
-    return unittest_main(argv=argv)
+    return absltest_main(argv=argv)
 
   total_shards = int(os.environ['TEST_TOTAL_SHARDS'])
   shard_index = int(os.environ['TEST_SHARD_INDEX'])
@@ -87,7 +87,7 @@ def g_main(argv):
   # Override getTestCaseNames
   base_loader.getTestCaseNames = getShardedTestCaseNames
 
-  unittest_main(argv=argv, testLoader=base_loader)
+  absltest_main(argv=argv, testLoader=base_loader)
 
 
 # Redefine main to allow running benchmarks
diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD
index c51b45a49c..3c1a9280a2 100644
--- a/tensorflow/tools/pip_package/BUILD
+++ b/tensorflow/tools/pip_package/BUILD
@@ -146,7 +146,11 @@ filegroup(
         "//third_party/eigen3:LICENSE",
         "//third_party/fft2d:LICENSE",
         "//third_party/hadoop:LICENSE.txt",
+        "@absl_py//absl:LICENSE",
+        "@absl_py//absl/logging:LICENSE",
         "@absl_py//absl/flags:LICENSE",
+        "@absl_py//absl/testing:LICENSE",
+        "@absl_py//absl/third_party/unittest3_backport:LICENSE",
         "@arm_neon_2_x86_sse//:LICENSE",
         "@astor_archive//:LICENSE",
         "@boringssl//:LICENSE",
@@ -155,6 +159,7 @@ filegroup(
         "@curl//:COPYING",
         "@double_conversion//:LICENSE",
         "@eigen_archive//:COPYING.MPL2",
+        "@enum34_archive//:LICENSE",
         "@farmhash_archive//:COPYING",
         "@fft2d//:fft/readme.txt",
         "@flatbuffers//:LICENSE.txt",
-- 
GitLab


From f5465080e9d3b2544c5b0fe364070d55b469e516 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Thu, 17 Jan 2019 11:56:22 -0800
Subject: [PATCH 0888/1095] Device Assignment support in TPUStrategy for
 StatefulRNN

PiperOrigin-RevId: 229791509
---
 .../contrib/distribute/python/tpu_strategy.py | 56 +++++++++++++++++--
 .../tpu/python/tpu/device_assignment.py       |  3 +
 2 files changed, 53 insertions(+), 6 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index b9d3924a24..f55e6f0b20 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -25,6 +25,8 @@ import copy
 import functools
 
 from tensorflow.contrib.tpu.python.ops import tpu_ops
+from tensorflow.contrib.tpu.python.tpu import device_assignment as device_assignment_lib
+from tensorflow.contrib.tpu.python.tpu import topology
 from tensorflow.contrib.tpu.python.tpu import tpu
 from tensorflow.contrib.tpu.python.tpu import tpu_system_metadata as tpu_system_metadata_lib
 from tensorflow.contrib.tpu.python.tpu import training_loop
@@ -58,6 +60,8 @@ def initialize_tpu_system(cluster_resolver=None):
   Args:
     cluster_resolver: A tf.contrib.cluster_resolver.TPUClusterResolver,
         which provides information about the TPU cluster.
+  Returns:
+    The tf.contrib.tpu.Topology object for the topology of the TPU cluster.
   """
   if cluster_resolver is None:
     cluster_resolver = resolver_lib.TPUClusterResolver("")
@@ -68,8 +72,9 @@ def initialize_tpu_system(cluster_resolver=None):
 
   with ops.Graph().as_default():
     with session_lib.Session(config=session_config, target=master) as sess:
-      sess.run([tpu.initialize_system()])
+      serialized_topology = sess.run(tpu.initialize_system())
   logging.info("Finished initializing TPU system.")
+  return topology.Topology(serialized=serialized_topology)
 
 
 def get_tpu_system_metadata(tpu_cluster_resolver):
@@ -149,6 +154,7 @@ class TPUStrategy(distribute_lib.DistributionStrategy):
   def __init__(self,
                tpu_cluster_resolver=None,
                steps_per_run=None,
+               device_assignment=None,
                **kwargs):
     """Initializes the TPUStrategy object.
 
@@ -160,10 +166,13 @@ class TPUStrategy(distribute_lib.DistributionStrategy):
           metrics, summaries etc.
           This parameter is only used when Distribution Strategy is used with
           estimator or keras.
+      device_assignment: Optional `tf.contrib.tpu.DeviceAssignment` to specify
+          the placement of replicas on the TPU cluster. Currently only supports
+          the usecase of using a single core within a TPU cluster.
       **kwargs: Additional experimental flags. Will be removed in future.
     """
     super(TPUStrategy, self).__init__(TPUExtended(
-        self, tpu_cluster_resolver, steps_per_run))
+        self, tpu_cluster_resolver, steps_per_run, device_assignment))
 
     self._disable_training_loop_on_host = False
     if len(kwargs) > 1:
@@ -188,7 +197,8 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
   def __init__(self,
                container_strategy,
                tpu_cluster_resolver=None,
-               steps_per_run=None):
+               steps_per_run=None,
+               device_assignment=None):
     super(TPUExtended, self).__init__(container_strategy)
 
     if tpu_cluster_resolver is None:
@@ -201,6 +211,21 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
 
     self._tpu_cluster_resolver = tpu_cluster_resolver
     self._tpu_metadata = get_tpu_system_metadata(self._tpu_cluster_resolver)
+    self._device_assignment = device_assignment
+
+    # Device assignment is currently only supported for 1 core case.
+    if self._device_assignment:
+      assert isinstance(self._device_assignment,
+                        device_assignment_lib.DeviceAssignment)
+      if self._device_assignment.num_replicas != 1:
+        raise ValueError("Device assignment is only supported for a single "
+                         "core single replica case currently.")
+      if self._device_assignment.num_cores_per_replica != 1:
+        raise ValueError("Device assignment is only supported for a single "
+                         "core single replica case currently.")
+      if not all(self._device_assignment.core_assignment[0][0] == [0, 0, 0]):
+        raise ValueError("Device assignment is only supported for a single "
+                         "core single replica case currently.")
 
     # TODO(jhseu): Switch to DeviceAssignment to support pods and model
     # parallelism.
@@ -604,15 +629,34 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
 
   @property
   def num_hosts(self):
-    return self._tpu_metadata.num_hosts
+    if self._device_assignment is None:
+      return self._tpu_metadata.num_hosts
+
+    return len(set([self._device_assignment.host_device(r)
+                    for r in range(self._device_assignment.num_replicas)]))
 
   @property
   def num_replicas_per_host(self):
-    return self._tpu_metadata.num_of_cores_per_host
+    if self._device_assignment is None:
+      return self._tpu_metadata.num_of_cores_per_host
+
+    # TODO(sourabhbajaj): Remove this method we use inputs and remove infeed
+    # as the computation of num_replicas_per_host is not a constant
+    # when using device_assignment. This is a temporary workaround to support
+    # StatefulRNN as everything is 1 in that case.
+    # This method needs to take host_id as input for correct computation.
+    max_models_per_host = (self._tpu_metadata.num_of_cores_per_host //
+                           self._device_assignment.num_cores_per_replica)
+    models_per_host = min(self._device_assignment.num_replicas,
+                          max_models_per_host)
+    return models_per_host * self._device_assignment.num_cores_per_replica
 
   @property
   def _num_replicas_in_sync(self):
-    return self._tpu_metadata.num_cores
+    if self._device_assignment is None:
+      return self._tpu_metadata.num_cores
+    return (self._device_assignment.num_replicas *
+            self._device_assignment.num_cores_per_replica)
 
   @property
   def experimental_between_graph(self):
diff --git a/tensorflow/contrib/tpu/python/tpu/device_assignment.py b/tensorflow/contrib/tpu/python/tpu/device_assignment.py
index 6906501ecf..3313dc749c 100644
--- a/tensorflow/contrib/tpu/python/tpu/device_assignment.py
+++ b/tensorflow/contrib/tpu/python/tpu/device_assignment.py
@@ -25,6 +25,9 @@ from six.moves import xrange  # pylint: disable=redefined-builtin
 from tensorflow.contrib.tpu.python.tpu.topology import Topology
 
 
+SINGLE_CORE_ASSIGNMENT = [[[0, 0, 0]]]
+
+
 def _compute_task_and_cores_to_replicas(core_assignment, topology):
   """Computes a nested dict which maps task and logical core to replicas."""
   task_and_cores_to_replicas = {}
-- 
GitLab


From 5f640305f37a58970f7070ecc074ada1ec14107b Mon Sep 17 00:00:00 2001
From: Jared Duke <jdduke@google.com>
Date: Thu, 17 Jan 2019 11:58:19 -0800
Subject: [PATCH 0889/1095] Disable nnapi_delegate_test portable tests

PiperOrigin-RevId: 229791853
---
 tensorflow/lite/delegates/nnapi/BUILD | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tensorflow/lite/delegates/nnapi/BUILD b/tensorflow/lite/delegates/nnapi/BUILD
index dda3c02567..63f6da1fa6 100644
--- a/tensorflow/lite/delegates/nnapi/BUILD
+++ b/tensorflow/lite/delegates/nnapi/BUILD
@@ -24,7 +24,11 @@ tf_cc_test(
     name = "nnapi_delegate_test",
     size = "small",
     srcs = ["nnapi_delegate_test.cc"],
-    tags = ["tflite_not_portable_ios"],
+    tags = [
+        # TODO(b/122987564): Enable on Android after resolving API 27 failures.
+        "tflite_not_portable_android",
+        "tflite_not_portable_ios",
+    ],
     deps = [
         ":nnapi_delegate",
         "//tensorflow/lite:framework",
-- 
GitLab


From 1e8a3d9e1fc542bca285682dd0fb9971ac3262d3 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Thu, 17 Jan 2019 12:06:17 -0800
Subject: [PATCH 0890/1095] Adds v2 binary crossentropy metric.

PiperOrigin-RevId: 229793647
---
 tensorflow/python/keras/metrics.py      |  66 +++++++++++++
 tensorflow/python/keras/metrics_test.py | 120 ++++++++++++++++++++++++
 2 files changed, 186 insertions(+)

diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index d6010e3cb0..d3bd523d29 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -2411,6 +2411,72 @@ class MeanTensor(Metric):
         K.set_value(v, np.zeros(self._shape.as_list()))
 
 
+class BinaryCrossentropy(MeanMetricWrapper):
+  """Computes the binary crossentropy between `y_true` and `y_pred`.
+
+  Usage:
+
+  ```python
+  m = tf.keras.metrics.BinaryCrossentropy()
+  m.update_state([1., 0., 1., 0.], [1., 1., 1., 0.])
+
+  # EPSILON = 1e-7, y = y_true, y` = y_pred, Y_MAX = 0.9999999
+  # y` = clip_ops.clip_by_value(output, EPSILON, 1. - EPSILON)
+  # y` = [Y_MAX, Y_MAX, Y_MAX, EPSILON]
+
+  # Metric = -(y log(y` + EPSILON) + (1 - y) log(1 - y` + EPSILON))
+  #        = [-log(Y_MAX + EPSILON), -log(1 - Y_MAX + EPSILON),
+  #           -log(Y_MAX + EPSILON), -log(1)]
+  #        = [(0 + 15.33) / 2, (0 + 0) / 2]
+  # Reduced metric = 7.665 / 2
+
+  print('Final result: ', m.result().numpy())  # Final result: 3.833
+  ```
+
+  Usage with tf.keras API:
+
+  ```python
+  model = keras.models.Model(inputs, outputs)
+  model.compile(
+      'sgd',
+      loss='mse',
+      metrics=[tf.keras.metrics.BinaryCrossentropy()])
+  ```
+  """
+
+  def __init__(self,
+               name='binary_crossentropy',
+               dtype=None,
+               from_logits=False,
+               label_smoothing=0):
+    """Creates a `BinaryCrossentropy` instance.
+
+    Args:
+      name: (Optional) string name of the metric instance.
+      dtype: (Optional) data type of the metric result.
+      from_logits: (Optional )Whether output is expected to be a logits tensor.
+        By default, we consider that output encodes a probability distribution.
+      label_smoothing: (Optional) Float in [0, 1]. When > 0, label values are
+        smoothed, meaning the confidence on label values are relaxed.
+        e.g. `label_smoothing=0.2` means that we will use a value of `0.1` for
+        label `0` and `0.9` for label `1`"
+    """
+    label_smoothing = ops.convert_to_tensor(label_smoothing, dtype=K.floatx())
+
+    super(BinaryCrossentropy, self).__init__(
+        binary_crossentropy,
+        name,
+        dtype=dtype,
+        from_logits=from_logits,
+        label_smoothing=label_smoothing)
+
+  @classmethod
+  def from_config(cls, config):
+    if 'fn' in config:
+      config.pop('fn')
+    return super(BinaryCrossentropy, cls).from_config(config)
+
+
 def accuracy(y_true, y_pred):
   y_pred.get_shape().assert_is_compatible_with(y_true.get_shape())
   if y_true.dtype != y_pred.dtype:
diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py
index bd94132a9a..4e755dd301 100644
--- a/tensorflow/python/keras/metrics_test.py
+++ b/tensorflow/python/keras/metrics_test.py
@@ -1200,6 +1200,126 @@ class MeanTensorTest(keras_parameterized.TestCase):
                           np.full((4, 3), 4))
 
 
+@test_util.run_all_in_graph_and_eager_modes
+class BinaryCrossentropyTest(test.TestCase):
+
+  def test_config(self):
+    bce_obj = metrics.BinaryCrossentropy(
+        name='bce', dtype=dtypes.int32, label_smoothing=0.2)
+    self.assertEqual(bce_obj.name, 'bce')
+    self.assertEqual(bce_obj._dtype, dtypes.int32)
+
+    old_config = bce_obj.get_config()
+    self.assertAllClose(self.evaluate(old_config['label_smoothing']), 0.2, 1e-3)
+
+    # Check save and restore config
+    bce_obj2 = metrics.BinaryCrossentropy.from_config(old_config)
+    self.assertEqual(bce_obj2.name, 'bce')
+    self.assertEqual(bce_obj2._dtype, dtypes.int32)
+    new_config = bce_obj2.get_config()
+    self.assertAllClose(self.evaluate(new_config['label_smoothing']), 0.2, 1e-3)
+
+  def test_unweighted(self):
+    bce_obj = metrics.BinaryCrossentropy()
+    self.evaluate(variables.variables_initializer(bce_obj.variables))
+    y_true = np.asarray([1, 0, 1, 0]).reshape([2, 2])
+    y_pred = np.asarray([1, 1, 1, 0], dtype=np.float32).reshape([2, 2])
+    result = bce_obj(y_true, y_pred)
+
+    # EPSILON = 1e-7, y = y_true, y` = y_pred, Y_MAX = 0.9999999
+    # y` = clip_ops.clip_by_value(output, EPSILON, 1. - EPSILON)
+    # y` = [Y_MAX, Y_MAX, Y_MAX, EPSILON]
+
+    # Metric = -(y log(y` + EPSILON) + (1 - y) log(1 - y` + EPSILON))
+    #        = [-log(Y_MAX + EPSILON), -log(1 - Y_MAX + EPSILON),
+    #           -log(Y_MAX + EPSILON), -log(1)]
+    #        = [(0 + 15.33) / 2, (0 + 0) / 2]
+    # Reduced metric = 7.665 / 2
+
+    self.assertAllClose(self.evaluate(result), 3.833, atol=1e-3)
+
+  def test_unweighted_with_logits(self):
+    bce_obj = metrics.BinaryCrossentropy(from_logits=True)
+    self.evaluate(variables.variables_initializer(bce_obj.variables))
+
+    y_true = constant_op.constant([[1, 0, 1], [0, 1, 1]])
+    y_pred = constant_op.constant([[100.0, -100.0, 100.0],
+                                   [100.0, 100.0, -100.0]])
+    result = bce_obj(y_true, y_pred)
+
+    # Metric = max(x, 0) - x * z + log(1 + exp(-abs(x)))
+    #              (where x = logits and z = y_true)
+    #        = [((100 - 100 * 1 + log(1 + exp(-100))) +
+    #            (0 + 100 * 0 + log(1 + exp(-100))) +
+    #            (100 - 100 * 1 + log(1 + exp(-100))),
+    #           ((100 - 100 * 0 + log(1 + exp(-100))) +
+    #            (100 - 100 * 1 + log(1 + exp(-100))) +
+    #            (0 + 100 * 1 + log(1 + exp(-100))))]
+    #        = [(0 + 0 + 0) / 3, 200 / 3]
+    # Reduced metric = (0 + 66.666) / 2
+
+    self.assertAllClose(self.evaluate(result), 33.333, atol=1e-3)
+
+  def test_weighted(self):
+    bce_obj = metrics.BinaryCrossentropy()
+    self.evaluate(variables.variables_initializer(bce_obj.variables))
+    y_true = np.asarray([1, 0, 1, 0]).reshape([2, 2])
+    y_pred = np.asarray([1, 1, 1, 0], dtype=np.float32).reshape([2, 2])
+    sample_weight = constant_op.constant([1.5, 2.])
+    result = bce_obj(y_true, y_pred, sample_weight=sample_weight)
+
+    # EPSILON = 1e-7, y = y_true, y` = y_pred, Y_MAX = 0.9999999
+    # y` = clip_ops.clip_by_value(output, EPSILON, 1. - EPSILON)
+    # y` = [Y_MAX, Y_MAX, Y_MAX, EPSILON]
+
+    # Metric = -(y log(y` + EPSILON) + (1 - y) log(1 - y` + EPSILON))
+    #        = [-log(Y_MAX + EPSILON), -log(1 - Y_MAX + EPSILON),
+    #           -log(Y_MAX + EPSILON), -log(1)]
+    #        = [(0 + 15.33) / 2, (0 + 0) / 2]
+    # Weighted metric = [7.665 * 1.5, 0]
+    # Reduced metric = 7.665 * 1.5 / (1.5 + 2)
+
+    self.assertAllClose(self.evaluate(result), 3.285, atol=1e-3)
+
+  def test_weighted_from_logits(self):
+    bce_obj = metrics.BinaryCrossentropy(from_logits=True)
+    self.evaluate(variables.variables_initializer(bce_obj.variables))
+    y_true = constant_op.constant([[1, 0, 1], [0, 1, 1]])
+    y_pred = constant_op.constant([[100.0, -100.0, 100.0],
+                                   [100.0, 100.0, -100.0]])
+    sample_weight = constant_op.constant([2., 2.5])
+    result = bce_obj(y_true, y_pred, sample_weight=sample_weight)
+
+    # Metric = max(x, 0) - x * z + log(1 + exp(-abs(x)))
+    #              (where x = logits and z = y_true)
+    #        = [(0 + 0 + 0) / 3, 200 / 3]
+    # Weighted metric = [0, 66.666 * 2.5]
+    # Reduced metric = 66.666 * 2.5 / (2 + 2.5)
+
+    self.assertAllClose(self.evaluate(result), 37.037, atol=1e-3)
+
+  def test_label_smoothing(self):
+    logits = constant_op.constant(((100., -100., -100.)))
+    y_true = constant_op.constant(((1, 0, 1)))
+    label_smoothing = 0.1
+    # Metric: max(x, 0) - x * z + log(1 + exp(-abs(x)))
+    #             (where x = logits and z = y_true)
+    # Label smoothing: z' = z * (1 - L) + 0.5L
+    # After label smoothing, label 1 becomes 1 - 0.5L
+    #                        label 0 becomes 0.5L
+    # Applying the above two fns to the given input:
+    # (100 - 100 * (1 - 0.5 L)  + 0 +
+    #  0   + 100 * (0.5 L)      + 0 +
+    #  0   + 100 * (1 - 0.5 L)  + 0) * (1/3)
+    #  = (100 + 50L) * 1/3
+    bce_obj = metrics.BinaryCrossentropy(
+        from_logits=True, label_smoothing=label_smoothing)
+    self.evaluate(variables.variables_initializer(bce_obj.variables))
+    result = bce_obj(y_true, logits)
+    expected_value = (100.0 + 50.0 * label_smoothing) / 3.0
+    self.assertAllClose(expected_value, self.evaluate(result), atol=1e-3)
+
+
 def _get_model(compile_metrics):
   model_layers = [
       layers.Dense(3, activation='relu', kernel_initializer='ones'),
-- 
GitLab


From f6b3d83c1112686658c0e6d34c04eef545240056 Mon Sep 17 00:00:00 2001
From: Alex Ilchenko <ilch@google.com>
Date: Thu, 17 Jan 2019 12:15:41 -0800
Subject: [PATCH 0891/1095] Add TPU support for resize using nearest neighbor

PiperOrigin-RevId: 229795283
---
 tensorflow/compiler/tests/image_ops_test.py   | 115 +++++-
 .../tf2xla/kernels/image_resize_ops.cc        | 331 ++++++++++--------
 2 files changed, 304 insertions(+), 142 deletions(-)

diff --git a/tensorflow/compiler/tests/image_ops_test.py b/tensorflow/compiler/tests/image_ops_test.py
index 0e2d840418..12741c4d4a 100644
--- a/tensorflow/compiler/tests/image_ops_test.py
+++ b/tensorflow/compiler/tests/image_ops_test.py
@@ -403,6 +403,117 @@ class AdjustSaturationTest(xla_test.XLATestCase):
           self.assertAllClose(y_fused, y_baseline, rtol=2e-5, atol=1e-5)
 
 
+class ResizeNearestNeighborTest(xla_test.XLATestCase):
+  # TODO(ilch): Wrap each test with `for dtype in self.float_types:`
+  # Some work to understand how that should be done was presented here:
+  # cl/227850213
+
+  def _assertForwardOpMatchesExpected(self,
+                                      image_np,
+                                      target_shape,
+                                      expected=None,
+                                      large_tolerance=False,
+                                      align_corners=True):
+    if expected is None:
+      self.fail("expected must be specified")
+    with self.cached_session() as sess, self.test_scope():
+      image = array_ops.placeholder(image_np.dtype)
+      resized = gen_image_ops.resize_nearest_neighbor(
+          image, target_shape, align_corners=align_corners)
+      out = sess.run(resized, {image: image_np[np.newaxis, :, :, np.newaxis]})
+      if large_tolerance:
+        self.assertAllClose(
+            expected[np.newaxis, :, :, np.newaxis], out, rtol=0.03, atol=0.1)
+      else:
+        self.assertAllClose(expected[np.newaxis, :, :, np.newaxis], out)
+
+  def testAlignCorners2x2To1x1(self):
+    self._assertForwardOpMatchesExpected(
+        np.array([[1, 2], [3, 4]], dtype=np.float32), [1, 1],
+        expected=np.array([[1]], dtype=np.float32))
+
+  def testAlignCorners1x1To2x2(self):
+    self._assertForwardOpMatchesExpected(
+        np.array([[1]], dtype=np.float32), [2, 2],
+        expected=np.array([[1, 1], [1, 1]], dtype=np.float32))
+
+  def testAlignCorners1x1To3x3(self):
+    self._assertForwardOpMatchesExpected(
+        np.array([[1]], dtype=np.float32), [3, 3],
+        expected=np.array([[1, 1, 1], [1, 1, 1], [1, 1, 1]], dtype=np.float32))
+
+  def testAlignCorners2x2To3x3(self):
+    self._assertForwardOpMatchesExpected(
+        np.array([[1, 2], [3, 4]], dtype=np.float32), [3, 3],
+        expected=np.array([[1, 2, 2], [3, 4, 4], [3, 4, 4]], dtype=np.float32))
+
+  def testAlignCorners2x2To4x4(self):
+    self._assertForwardOpMatchesExpected(
+        np.array([[1, 2], [3, 4]], dtype=np.float32), [4, 4],
+        expected=np.array(
+            [[1, 1, 2, 2], [1, 1, 2, 2], [3, 3, 4, 4], [3, 3, 4, 4]],
+            dtype=np.float32))
+
+  def testAlignCorners3x3To2x2(self):
+    self._assertForwardOpMatchesExpected(
+        np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32), [2, 2],
+        expected=np.array([[1, 3], [7, 9]], dtype=np.float32))
+
+  def testAlignCorners4x4To3x3(self):
+    self._assertForwardOpMatchesExpected(
+        np.array(
+            [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16]],
+            dtype=np.float32), [3, 3],
+        expected=np.array([[1, 3, 4], [9, 11, 12], [13, 15, 16]],
+                          dtype=np.float32))
+
+  def testAlignCorners3x3To4x4(self):
+    self._assertForwardOpMatchesExpected(
+        np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32), [4, 4],
+        expected=np.array(
+            [[1, 2, 2, 3], [4, 5, 5, 6], [4, 5, 5, 6], [7, 8, 8, 9]],
+            dtype=np.float32))
+
+  def testAlignCorners3x3To6x6(self):
+    self._assertForwardOpMatchesExpected(
+        np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32), [6, 6],
+        expected=np.array(
+            [[1, 1, 2, 2, 3, 3], [1, 1, 2, 2, 3, 3], [4, 4, 5, 5, 6, 6],
+             [4, 4, 5, 5, 6, 6], [7, 7, 8, 8, 9, 9], [7, 7, 8, 8, 9, 9]],
+            dtype=np.float32))
+
+  def testAlignCorners3x3To9x9(self):
+    # The expected matrix might look uneven in terms of how many of each number
+    # there is, but this is an artifact of doing the dilation and convolution
+    # iteratively. The behavior is less esoteric in the 3x3To12x12 case below.
+    self._assertForwardOpMatchesExpected(
+        np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32), [9, 9],
+        expected=np.array(
+            [[1, 2, 2, 2, 2, 3, 3, 3, 3], [4, 5, 5, 5, 5, 6, 6, 6, 6],
+             [4, 5, 5, 5, 5, 6, 6, 6, 6], [4, 5, 5, 5, 5, 6, 6, 6, 6],
+             [4, 5, 5, 5, 5, 6, 6, 6, 6], [7, 8, 8, 8, 8, 9, 9, 9, 9],
+             [7, 8, 8, 8, 8, 9, 9, 9, 9], [7, 8, 8, 8, 8, 9, 9, 9, 9],
+             [7, 8, 8, 8, 8, 9, 9, 9, 9]],
+            dtype=np.float32))
+
+  def testAlignCorners3x3To12x12(self):
+    self._assertForwardOpMatchesExpected(
+        np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32), [12, 12],
+        expected=np.array([[1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3],
+                           [1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3],
+                           [1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3],
+                           [4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6],
+                           [4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6],
+                           [4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6],
+                           [4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6],
+                           [4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6],
+                           [4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6],
+                           [7, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9],
+                           [7, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9],
+                           [7, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9]],
+                          dtype=np.float32))
+
+
 class ResizeBilinearTest(xla_test.XLATestCase):
 
   def _assertForwardOpMatchesExpected(self,
@@ -444,14 +555,14 @@ class ResizeBilinearTest(xla_test.XLATestCase):
       self.assertAllCloseAccordingToType(expected[np.newaxis, :, :, np.newaxis],
                                          out)
 
-  def testAlignCorners1x2To3x2(self):
+  def testAlignCorners1x2To3x3(self):
     for dtype in self.float_types:
       self._assertForwardOpMatchesExpected(
           np.array([[1, 2]], dtype=dtype), [3, 3],
           expected=np.array([[1, 1.5, 2], [1, 1.5, 2], [1, 1.5, 2]],
                             dtype=np.float32))
 
-  def testAlignCorners1x2To3x2Grad(self):
+  def testAlignCorners1x2To3x3Grad(self):
     for dtype in self.float_types:
       self._assertBackwardOpMatchesExpected(
           np.array([[1, 2], [3, 4], [5, 6]], dtype=np.float32),
diff --git a/tensorflow/compiler/tf2xla/kernels/image_resize_ops.cc b/tensorflow/compiler/tf2xla/kernels/image_resize_ops.cc
index 5a10c52ba8..b96d45316f 100644
--- a/tensorflow/compiler/tf2xla/kernels/image_resize_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/image_resize_ops.cc
@@ -72,10 +72,10 @@ namespace {
 // from in_size to out_size.
 struct ResizeConvolutionDims {
   // Size of the kernel to use.
-  std::vector<int64> kernel_size;
+  std::vector<int64> kernel_size;  // k
 
   // Stride of the convolution to use.
-  std::vector<int64> stride;
+  std::vector<int64> stride;  // S
 };
 ResizeConvolutionDims ComputeResizeConvolutionParameters(
     absl::Span<const int64> in_size, absl::Span<const int64> out_size,
@@ -117,8 +117,10 @@ ResizeConvolutionDims ComputeResizeConvolutionParameters(
 //                        + dims.stride * (out_size - 1)
 int64 CalculateUpperPadding(int64 in_size, int64 out_size, int64 kernel_size,
                             int64 stride) {
-  return (2 * kernel_size - 1) + (out_size - 1) * stride - (kernel_size - 1) -
-         1 - (kernel_size * (in_size - 1));
+  int64 padding = (2 * kernel_size - 1) + (out_size - 1) * stride -
+                  (kernel_size - 1) - 1 - (kernel_size * (in_size - 1));
+
+  return padding;
 }
 
 // Form a 2D convolution kernel like:
@@ -132,7 +134,7 @@ int64 CalculateUpperPadding(int64 in_size, int64 out_size, int64 kernel_size,
 // If the 2D kernel would be very large, the 1D kernel can be applied once in
 // each dimension due to the symmetry of the kernel along all axis to reduce the
 // computational intensity.
-xla::XlaOp Make1DKernel(xla::XlaBuilder* builder, int64 n) {
+xla::XlaOp MakeBilinear1DKernel(xla::XlaBuilder* builder, int64 n) {
   std::vector<float> kernel(n * 2 - 1);
   for (int64 i = 0; i < n; ++i) {
     float v = (i + 1.0f) / n;
@@ -142,43 +144,64 @@ xla::XlaOp Make1DKernel(xla::XlaBuilder* builder, int64 n) {
   return xla::ConstantR1<float>(builder, kernel);
 }
 
+// Unlike the bilinear kernel, which is triangular, the nearest neighbor
+// kernel is a square. For example, a 1D kernel with n=3 would look like
+// [0 1 1 1 0]
+// and n=4 would look like
+// [0 0 1 1 1 1 0].
+// Note that in the second case, the kernel is not symmetric and we default
+// to the right (because an existing non TPU kernel
+// for nearest neighbor resize already chose to default to the right,
+// so we want to be consistent).
+xla::XlaOp MakeNearestNeighbor1DKernel(xla::XlaBuilder* builder, int64 n) {
+  std::vector<float> kernel(n * 2 - 1, 0.0f);
+  std::fill(&kernel[n / 2], &kernel[(3 * n) / 2], 1.0f);
+
+  return xla::ConstantR1<float>(builder, kernel);
+}
+
 // Kernels with more than 16 spatial elements are considered intense and the
-// kernel should applied to each dimension independently.
+// kernel should be applied to each dimension independently.
 const int64 kMax2DKernelSize = 16;
 
-xla::XlaOp MakeBilinearResizeKernel(xla::XlaBuilder* builder,
-                                    absl::Span<const int64> kernel_size,
-                                    int64 channels) {
+xla::XlaOp MakeGeneralResizeKernel(xla::XlaBuilder* builder,
+                                   absl::Span<const int64> kernel_size,
+                                   int64 channels, bool is_kernel_bilinear) {
+  auto make_kernel_func =
+      is_kernel_bilinear ? MakeBilinear1DKernel : MakeNearestNeighbor1DKernel;
+
   auto depthwise_kernel = xla::Broadcast(
       xla::Zero(builder, xla::F32),
       {(2 * kernel_size[0] - 1), (2 * kernel_size[1] - 1), channels, 1});
 
   return xla::Mul(
-      xla::Add(depthwise_kernel, Make1DKernel(builder, kernel_size[1]),
+      xla::Add(depthwise_kernel, make_kernel_func(builder, kernel_size[1]),
                /*broadcast_dimensions=*/{1}),
-      Make1DKernel(builder, kernel_size[0]),
+      make_kernel_func(builder, kernel_size[0]),
       /*broadcast_dimensions=*/{0});
 }
 
-xla::XlaOp MakeBilinearResizeKernelInDim(xla::XlaBuilder* builder,
-                                         absl::Span<const int64> kernel_size,
-                                         int64 channels, int64 dim) {
+xla::XlaOp MakeGeneralResizeKernelInDim(xla::XlaBuilder* builder,
+                                        absl::Span<const int64> kernel_size,
+                                        int64 channels, int64 dim,
+                                        bool is_kernel_bilinear) {
+  auto make_kernel_func =
+      is_kernel_bilinear ? MakeBilinear1DKernel : MakeNearestNeighbor1DKernel;
+
   auto depthwise_kernel =
       xla::Broadcast(xla::Zero(builder, xla::F32),
                      {dim == 0 ? (2 * kernel_size[0] - 1) : 1,
                       dim == 1 ? (2 * kernel_size[1] - 1) : 1, channels, 1});
-  return xla::Add(depthwise_kernel, Make1DKernel(builder, kernel_size[dim]),
+  return xla::Add(depthwise_kernel, make_kernel_func(builder, kernel_size[dim]),
                   /*broadcast_dimensions=*/{dim});
 }
 
-xla::XlaOp ResizeUsingDilationAndConvolution(xla::XlaBuilder* builder,
-                                             const xla::XlaOp& input,
-                                             const int num_spatial_dims,
-                                             std::vector<int64> in_size,
-                                             std::vector<int64> out_size,
-                                             const int64 channels,
-                                             const bool align_corners) {
-  // Picture for a 1x3 to 1x4 resize:
+xla::XlaOp ResizeUsingDilationAndConvolution(
+    xla::XlaBuilder* builder, const xla::XlaOp& input,
+    const int num_spatial_dims, std::vector<int64> in_size,
+    std::vector<int64> out_size, const int64 channels, const bool align_corners,
+    bool is_kernel_bilinear) {
+  // Picture for a 1x3 to 1x4 bilinear resize:
   // stride = 2, kernel size = 3
   // Input:
   // 3 6 9
@@ -264,8 +287,8 @@ xla::XlaOp ResizeUsingDilationAndConvolution(xla::XlaBuilder* builder,
   // Split convolutions into independent dimensions if they would be a very
   // large kernel.
   if (dims.kernel_size[0] * dims.kernel_size[1] < kMax2DKernelSize) {
-    xla::XlaOp kernel =
-        MakeBilinearResizeKernel(builder, dims.kernel_size, channels);
+    xla::XlaOp kernel = MakeGeneralResizeKernel(builder, dims.kernel_size,
+                                                channels, is_kernel_bilinear);
     output =
         xla::ConvGeneralDilated(input_data, kernel, dims.stride,
                                 /*padding=*/
@@ -275,8 +298,8 @@ xla::XlaOp ResizeUsingDilationAndConvolution(xla::XlaBuilder* builder,
                                 /*rhs_dilation=*/{1, 1}, dimension_numbers,
                                 /*feature_group_count=*/channels);
   } else {
-    xla::XlaOp kernel0 =
-        MakeBilinearResizeKernelInDim(builder, dims.kernel_size, channels, 0);
+    xla::XlaOp kernel0 = MakeGeneralResizeKernelInDim(
+        builder, dims.kernel_size, channels, 0, is_kernel_bilinear);
     output = xla::ConvGeneralDilated(
         input_data, kernel0, {dims.stride[0], 1},
         /*padding=*/
@@ -284,8 +307,8 @@ xla::XlaOp ResizeUsingDilationAndConvolution(xla::XlaBuilder* builder,
         /*lhs_dilation=*/{dims.kernel_size[0], 1},
         /*rhs_dilation=*/{1, 1}, dimension_numbers,
         /*feature_group_count=*/channels);
-    xla::XlaOp kernel1 =
-        MakeBilinearResizeKernelInDim(builder, dims.kernel_size, channels, 1);
+    xla::XlaOp kernel1 = MakeGeneralResizeKernelInDim(
+        builder, dims.kernel_size, channels, 1, is_kernel_bilinear);
     output = xla::ConvGeneralDilated(
         output, kernel1, {1, dims.stride[1]},
         /*padding=*/
@@ -306,13 +329,11 @@ xla::XlaOp ResizeUsingDilationAndConvolution(xla::XlaBuilder* builder,
   return output;
 }
 
-xla::XlaOp ResizeUsingDilationAndConvolutionGradOp(xla::XlaBuilder* builder,
-                                                   const xla::XlaOp& grad,
-                                                   const int num_spatial_dims,
-                                                   std::vector<int64> in_size,
-                                                   std::vector<int64> grad_size,
-                                                   const int64 channels,
-                                                   const bool align_corners) {
+xla::XlaOp ResizeUsingDilationAndConvolutionGradOp(
+    xla::XlaBuilder* builder, const xla::XlaOp& grad,
+    const int num_spatial_dims, std::vector<int64> in_size,
+    std::vector<int64> grad_size, const int64 channels,
+    const bool align_corners, bool is_kernel_bilinear) {
   ResizeConvolutionDims dims =
       ComputeResizeConvolutionParameters(in_size, grad_size, align_corners);
 
@@ -332,8 +353,8 @@ xla::XlaOp ResizeUsingDilationAndConvolutionGradOp(xla::XlaBuilder* builder,
   dimension_numbers.set_kernel_output_feature_dimension(num_spatial_dims);
   xla::XlaOp output;
   if (dims.kernel_size[0] * dims.kernel_size[1] < kMax2DKernelSize) {
-    xla::XlaOp kernel =
-        MakeBilinearResizeKernel(builder, dims.kernel_size, channels);
+    xla::XlaOp kernel = MakeGeneralResizeKernel(builder, dims.kernel_size,
+                                                channels, is_kernel_bilinear);
 
     // Broadcast the input kernel where the forward op expanded from a size == 1
     // dimension to a size > 1 dimension. This has the effect of summing the
@@ -355,14 +376,14 @@ xla::XlaOp ResizeUsingDilationAndConvolutionGradOp(xla::XlaBuilder* builder,
         /*rhs_dilation=*/{1, 1}, dimension_numbers,
         /*feature_group_count=*/channels);
   } else {
-    xla::XlaOp kernel0 =
-        MakeBilinearResizeKernelInDim(builder, dims.kernel_size, channels, 0);
-    xla::XlaOp kernel1 =
-        MakeBilinearResizeKernelInDim(builder, dims.kernel_size, channels, 1);
-
-    // Broadcast the input kernel where the forward op expanded from a size == 1
-    // dimension to a size > 1 dimension. This has the effect of summing the
-    // gradient contributions in that dimension.
+    xla::XlaOp kernel0 = MakeGeneralResizeKernelInDim(
+        builder, dims.kernel_size, channels, 0, is_kernel_bilinear);
+    xla::XlaOp kernel1 = MakeGeneralResizeKernelInDim(
+        builder, dims.kernel_size, channels, 1, is_kernel_bilinear);
+
+    // Broadcast the input kernel where the forward op expanded from a
+    // size == 1 dimension to a size > 1 dimension. This has the effect of
+    // summing the gradient contributions in that dimension.
     if (in_size[0] == 1 && grad_size[0] > 1) {
       kernel0 =
           xla::Add(kernel0, xla::ConstantR1<float>(builder, grad_size[0], 0),
@@ -407,109 +428,139 @@ xla::XlaOp ResizeUsingDilationAndConvolutionGradOp(xla::XlaBuilder* builder,
   return output;
 }
 
-class ResizeBilinearOp : public XlaOpKernel {
- public:
-  explicit ResizeBilinearOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) {
-    OP_REQUIRES_OK(ctx, ctx->GetAttr("align_corners", &align_corners_));
+void GeneralCompile(XlaOpKernelContext* ctx, bool align_corners_,
+                    bool is_kernel_bilinear) {
+  xla::XlaBuilder* b = ctx->builder();
+
+  TensorShape input_shape = ctx->InputShape(0);
+  OP_REQUIRES(ctx, input_shape.dims() == 4,
+              errors::InvalidArgument("input must be 4-dimensional",
+                                      input_shape.DebugString()));
+  // First dimension always assumed to be batch
+  const int64 batch = input_shape.dim_size(0);
+  std::vector<int64> in_size = {input_shape.dim_size(1),
+                                input_shape.dim_size(2)};
+  // Last/4th dimension always assumed to be num channels
+  const int64 channels = input_shape.dim_size(3);
+  OP_REQUIRES(ctx, in_size[0] > 0 && in_size[1] > 0,
+              errors::InvalidArgument("input size must be positive, got [",
+                                      in_size[0], ",", in_size[1], "]"));
+
+  std::vector<int64> out_size;
+  OP_REQUIRES_OK(ctx, ctx->ConstantInputAsIntVector(1, &out_size));
+  OP_REQUIRES(ctx, out_size.size() == 2,
+              errors::InvalidArgument("output size must be length 2, got ",
+                                      out_size.size()));
+  OP_REQUIRES(ctx, out_size[0] > 0 && out_size[1] > 0,
+              errors::InvalidArgument("output size must be positive, got [",
+                                      out_size[0], ",", out_size[1], "]"));
+
+  const int num_spatial_dims = 2;
+
+  xla::XlaOp input = ctx->Input(0);
+
+  // If in_size[i] > 1 and out_size[i] == 1, slice out the first input in
+  // dimension i.
+  bool slice_input = false;
+  for (int i = 0; i < num_spatial_dims; ++i) {
+    if (in_size[i] > 1 && out_size[i] == 1) {
+      // If in_size[i] > 1 but out_size[i] == 1, then we slice out the first
+      // entry before resizing.
+      slice_input = true;
+      in_size[i] = 1;
+    }
+  }
+  if (slice_input) {
+    input = xla::Slice(input, {0, 0, 0, 0},
+                       {batch, in_size[0], in_size[1], channels}, {1, 1, 1, 1});
   }
 
-  void Compile(XlaOpKernelContext* ctx) override {
-    xla::XlaBuilder* b = ctx->builder();
+  // Output is always type float.
+  input = xla::ConvertElementType(input, xla::F32);
+
+  // Special Case:
+  // Instead of doing a ResizeUsingDilationAndConvolution directly,
+  // while (out_size[0]-1) = c * 2^x * (in_size[0]-1) for x>1 c>1, resize the
+  // image to 2*(in_size[0]-1)+1 x-times and then resize by scale c(int here).
+  // Instead of resizing directly we resize it iteratively.
+  //
+  // Since bilinear resize can be broken down as 2 sequential linear
+  // operations along different dimensions.
+  // Given sufficient numerical stability and a<e<c and b<f<d, bilinear resize
+  // from image of size axb -> cxd is same as resizing axb -> exf -> cxd.
+  // This does not work in the case of align_corners_=false because of special
+  // padding requirements that cause multiple resizes to be very different
+  // from a single resize.
+  //
+  // This makes the convolutions kernels smaller and the operation faster.
+  xla::XlaOp output = input;
+  while (in_size != out_size) {
+    if (in_size[0] != 1 && in_size[1] != 1) {
+      std::vector<float> k = {
+          (static_cast<float>(out_size[0]) - 1) / ((in_size[0] - 1) * 2),
+          (static_cast<float>(out_size[1]) - 1) / ((in_size[1] - 1) * 2)};
+      if ((k[0] == std::floor(k[0])) && (k[1] == std::floor(k[1])) &&
+          k[0] > 1 && k[1] > 1 && align_corners_) {
+        std::vector<int64> next_out_size = {(in_size[0] - 1) * 2 + 1,
+                                            (in_size[1] - 1) * 2 + 1};
+        output = ResizeUsingDilationAndConvolution(
+            b, input, num_spatial_dims, in_size, next_out_size, channels,
+            align_corners_, is_kernel_bilinear);
+        input = output;
+        in_size = next_out_size;
+      } else {
+        output = ResizeUsingDilationAndConvolution(
+            b, input, num_spatial_dims, in_size, out_size, channels,
+            align_corners_, is_kernel_bilinear);
+        in_size = out_size;
+      }
+    } else {
+      output = ResizeUsingDilationAndConvolution(
+          b, input, num_spatial_dims, in_size, out_size, channels,
+          align_corners_, is_kernel_bilinear);
+      in_size = out_size;
+    }
+  }
 
-    TensorShape input_shape = ctx->InputShape(0);
-    OP_REQUIRES(ctx, input_shape.dims() == 4,
-                errors::InvalidArgument("input must be 4-dimensional",
-                                        input_shape.DebugString()));
-    const int64 batch = input_shape.dim_size(0);
-    std::vector<int64> in_size = {input_shape.dim_size(1),
-                                  input_shape.dim_size(2)};
-    const int64 channels = input_shape.dim_size(3);
-    OP_REQUIRES(ctx, in_size[0] > 0 && in_size[1] > 0,
-                errors::InvalidArgument("input size must be positive, got [",
-                                        in_size[0], ",", in_size[1], "]"));
+  ctx->SetOutput(0, output);
+}
 
-    std::vector<int64> out_size;
-    OP_REQUIRES_OK(ctx, ctx->ConstantInputAsIntVector(1, &out_size));
-    OP_REQUIRES(ctx, out_size.size() == 2,
-                errors::InvalidArgument("output size must be length 2, got ",
-                                        out_size.size()));
-    OP_REQUIRES(ctx, out_size[0] > 0 && out_size[1] > 0,
-                errors::InvalidArgument("output size must be positive, got [",
-                                        out_size[0], ",", out_size[1], "]"));
+class ResizeNearestNeighborOp : public XlaOpKernel {
+ public:
+  explicit ResizeNearestNeighborOp(OpKernelConstruction* ctx)
+      : XlaOpKernel(ctx) {
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("align_corners", &align_corners_));
+    OP_REQUIRES(
+        ctx, align_corners_ == true,
+        errors::Unimplemented("ResizeNearestNeighbor with align_corners=False "
+                              "is not yet implemented"));
+  }
 
-    const int num_spatial_dims = 2;
+  void Compile(XlaOpKernelContext* ctx) override {
+    GeneralCompile(ctx, align_corners_, is_kernel_bilinear_);
+  }
 
-    xla::XlaOp input = ctx->Input(0);
+ private:
+  bool align_corners_ = true;
+  bool is_kernel_bilinear_ = false;
+};
 
-    // If in_size[i] > 1 and out_size[i] == 1, slice out the first input in
-    // dimension i.
-    bool slice_input = false;
-    for (int i = 0; i < num_spatial_dims; ++i) {
-      if (in_size[i] > 1 && out_size[i] == 1) {
-        // If in_size[i] > 1 but out_size[i] == 1, then we slice out the first
-        // entry before resizing.
-        slice_input = true;
-        in_size[i] = 1;
-      }
-    }
-    if (slice_input) {
-      input =
-          xla::Slice(input, {0, 0, 0, 0},
-                     {batch, in_size[0], in_size[1], channels}, {1, 1, 1, 1});
-    }
+REGISTER_XLA_OP(Name("ResizeNearestNeighbor").CompileTimeConstantInput("size"),
+                ResizeNearestNeighborOp);
 
-    // Output is always type float.
-    input = xla::ConvertElementType(input, xla::F32);
-
-    // Special Case:
-    // Instead of doing a ResizeUsingDilationAndConvolution directly,
-    // while (out_size[0]-1) = c * 2^x * (in_size[0]-1) for x>1 c>1, resize the
-    // image to 2*(in_size[0]-1)+1 x-times and then resize by scale c(int here).
-    // Instead of resizing directly we resize it iteratively.
-    //
-    // Since bilinear resize can be broken down as 2 sequential linear
-    // operations along different dimensions.
-    // Given sufficient numerical stability and a<e<c and b<f<d, bilinear resize
-    // from image of size axb -> cxd is same as resizing axb -> exf -> cxd.
-    // This does not work in the case of align_corners_=false because of special
-    // padding requirements that cause multiple resizes to be very different
-    // from a single resize.
-    //
-    // This makes the convolutions kernels smaller and the operation faster.
-    xla::XlaOp output = input;
-    while (in_size != out_size) {
-      if (in_size[0] != 1 && in_size[1] != 1) {
-        std::vector<float> k = {
-            (static_cast<float>(out_size[0]) - 1) / ((in_size[0] - 1) * 2),
-            (static_cast<float>(out_size[1]) - 1) / ((in_size[1] - 1) * 2)};
-        if ((k[0] == std::floor(k[0])) && (k[1] == std::floor(k[1])) &&
-            k[0] > 1 && k[1] > 1 && align_corners_) {
-          std::vector<int64> next_out_size = {(in_size[0] - 1) * 2 + 1,
-                                              (in_size[1] - 1) * 2 + 1};
-          output = ResizeUsingDilationAndConvolution(b, input, num_spatial_dims,
-                                                     in_size, next_out_size,
-                                                     channels, align_corners_);
-          input = output;
-          in_size = next_out_size;
-        } else {
-          output = ResizeUsingDilationAndConvolution(b, input, num_spatial_dims,
-                                                     in_size, out_size,
-                                                     channels, align_corners_);
-          in_size = out_size;
-        }
-      } else {
-        output = ResizeUsingDilationAndConvolution(b, input, num_spatial_dims,
-                                                   in_size, out_size, channels,
-                                                   align_corners_);
-        in_size = out_size;
-      }
-    }
+class ResizeBilinearOp : public XlaOpKernel {
+ public:
+  explicit ResizeBilinearOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) {
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("align_corners", &align_corners_));
+  }
 
-    ctx->SetOutput(0, output);
+  void Compile(XlaOpKernelContext* ctx) override {
+    GeneralCompile(ctx, align_corners_, is_kernel_bilinear_);
   }
 
  private:
-  bool align_corners_;
+  bool align_corners_ = true;
+  bool is_kernel_bilinear_ = true;
 };
 
 REGISTER_XLA_OP(Name("ResizeBilinear").CompileTimeConstantInput("size"),
@@ -581,19 +632,19 @@ class ResizeBilinearGradOp : public XlaOpKernel {
                                                (in_size[1] - 1) * 2 + 1};
           output = ResizeUsingDilationAndConvolutionGradOp(
               b, grad, num_spatial_dims, in_size, next_grad_size, channels,
-              align_corners_);
+              align_corners_, true);
           grad = output;
           in_size = next_grad_size;
         } else {
           output = ResizeUsingDilationAndConvolutionGradOp(
               b, grad, num_spatial_dims, in_size, grad_size, channels,
-              align_corners_);
+              align_corners_, true);
           in_size = grad_size;
         }
       } else {
         output = ResizeUsingDilationAndConvolutionGradOp(
             b, grad, num_spatial_dims, in_size, grad_size, channels,
-            align_corners_);
+            align_corners_, true);
         in_size = grad_size;
       }
     }
-- 
GitLab


From 66ae27d95e18f1abd728dfbc0ce4c876b8a4f219 Mon Sep 17 00:00:00 2001
From: Akshay Modi <nareshmodi@google.com>
Date: Thu, 17 Jan 2019 12:26:18 -0800
Subject: [PATCH 0892/1095] TensorScatter* ops should forward the correct
 input, and set output when forwarding.

PiperOrigin-RevId: 229796887
---
 tensorflow/core/kernels/scatter_nd_op.cc              |  4 +++-
 tensorflow/python/kernel_tests/scatter_nd_ops_test.py | 11 +++++++++++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/kernels/scatter_nd_op.cc b/tensorflow/core/kernels/scatter_nd_op.cc
index 1b1c59cf34..50e4c66b7e 100644
--- a/tensorflow/core/kernels/scatter_nd_op.cc
+++ b/tensorflow/core/kernels/scatter_nd_op.cc
@@ -197,7 +197,7 @@ class TensorScatterOp : public OpKernel {
     }
 
     std::unique_ptr<Tensor> forwarded_input = c->forward_input(
-        2, 0, input.dtype(), shape, DEVICE_MEMORY, AllocatorAttributes());
+        0, 0, input.dtype(), shape, DEVICE_MEMORY, AllocatorAttributes());
 
     if (forwarded_input == nullptr) {
       // We were not able to forward the input, so we deep copy the tensor and
@@ -215,6 +215,8 @@ class TensorScatterOp : public OpKernel {
       OP_REQUIRES_OK(c, functor::DoScatterNd<Device, T, Index, op>(
                             c, indices, updates, shape, forwarded_input.get(),
                             false /*allocate*/));
+
+      c->set_output(0, *forwarded_input);
     }
   }
 };
diff --git a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
index f30b5315aa..88f7b27b77 100644
--- a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
+++ b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
@@ -24,6 +24,7 @@ import numpy as np
 
 from tensorflow.python.client import session
 from tensorflow.python.eager import context
+from tensorflow.python.eager import def_function
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
@@ -750,6 +751,16 @@ class ScatterNdTensorTest(test.TestCase):
       self.assertLess(err_added_wrt_updates, 2e-4)
       self.assertLess(err_subbed_wrt_updates, 2e-4)
 
+  def testTensorScatterUpdateWithForwarding(self):
+    @def_function.function
+    def _TestFn():
+      indices = constant_op.constant([[4], [3], [1], [7]])
+      updates = constant_op.constant([9, 10, 11, 12], dtype=dtypes.float32)
+      t = array_ops.ones([8], dtype=dtypes.float32)
+
+      return array_ops.tensor_scatter_update(t, indices, updates)
+
+    self.assertAllEqual(_TestFn(), [1, 11, 1, 10, 9, 1, 1, 12])
 
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From bc7cf8062dc88b55130bc9b786707fca22c75206 Mon Sep 17 00:00:00 2001
From: Alan Chiao <alanchiao@google.com>
Date: Thu, 17 Jan 2019 12:32:26 -0800
Subject: [PATCH 0893/1095] Replace runtime memory alignment checks with debug
 time checks.

average 7.5% performance improvement across different models with hybrid inference.

PiperOrigin-RevId: 229797803
---
 .../lite/kernels/internal/optimized/neon_tensor_utils.cc  | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/lite/kernels/internal/optimized/neon_tensor_utils.cc b/tensorflow/lite/kernels/internal/optimized/neon_tensor_utils.cc
index 3e7b6c22be..730d9b662a 100644
--- a/tensorflow/lite/kernels/internal/optimized/neon_tensor_utils.cc
+++ b/tensorflow/lite/kernels/internal/optimized/neon_tensor_utils.cc
@@ -145,8 +145,8 @@ void NeonMatrixBatchVectorMultiplyAccumulate(
         // Load 16 8-bit values from the row and vector, each, to operate on.
         // Here the assumption is that each buffer is 4-byte aligned. Otherwise,
         // performance may suffer significantly.
-        TFLITE_CHECK_EQ((uintptr_t)(&row_ptr[col]) & (kWeightsPerUint32 - 1),
-                        0);
+        TFLITE_DCHECK_EQ(  // NOLINT
+            (uintptr_t)(&row_ptr[col]) & (kWeightsPerUint32 - 1), 0);
         const int8x16_t s1_8x16 = vld1q_s8((const int8_t*)(aligned_vec + col));
         const int8x16_t s2_8x16 = vld1q_s8((const int8_t*)(row_ptr + col));
         // Multiply the low bits (i.e. the lower 8 8bit numbers in the
@@ -174,8 +174,8 @@ void NeonMatrixBatchVectorMultiplyAccumulate(
           // Load 8 8-bit values from the row and column each to operate on.
           // Here the assumption is that each buffer is 4-bytes aligned.
           // Otherwise, performance may suffer significantly.
-          TFLITE_CHECK_EQ((uintptr_t)(&row_ptr[col]) & (kWeightsPerUint32 - 1),
-                          0);
+          TFLITE_DCHECK_EQ(  // NOLINT
+              (uintptr_t)(&row_ptr[col]) & (kWeightsPerUint32 - 1), 0);
           const int8x8_t s1_8x8 = vld1_s8((const int8_t*)(aligned_vec + col));
           const int8x8_t s2_8x8 = vld1_s8((const int8_t*)(row_ptr + col));
           const int16x8_t prod_16x8 = vmull_s8(s1_8x8, s2_8x8);
-- 
GitLab


From a851fa125b0ddaf91e64a279f568d8ff01f6f76f Mon Sep 17 00:00:00 2001
From: Yanan Cao <ycao@google.com>
Date: Thu, 17 Jan 2019 12:33:39 -0800
Subject: [PATCH 0894/1095] Add a warning for dump_graph when dirname is not
 specified

PiperOrigin-RevId: 229798002
---
 tensorflow/core/util/dump_graph.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tensorflow/core/util/dump_graph.cc b/tensorflow/core/util/dump_graph.cc
index 523d37ecc2..d275e076f8 100644
--- a/tensorflow/core/util/dump_graph.cc
+++ b/tensorflow/core/util/dump_graph.cc
@@ -84,6 +84,10 @@ string WriteTextProtoToUniqueFile(Env* env, const string& name,
     dir = getenv("TF_DUMP_GRAPH_PREFIX");
   }
   if (!dir) {
+    LOG(WARNING)
+        << "Failed to dump " << name << " because dump location is not "
+        << " specified through either TF_DUMP_GRAPH_PREFIX environment "
+        << "variable or function argument.";
     return "(TF_DUMP_GRAPH_PREFIX not specified)";
   }
   Status status = env->RecursivelyCreateDir(dir);
-- 
GitLab


From 155385b86579489f92c0f5643c8af97364fe5d24 Mon Sep 17 00:00:00 2001
From: Reed Wanderman-Milne <reedwm@google.com>
Date: Thu, 17 Jan 2019 12:38:07 -0800
Subject: [PATCH 0895/1095] Automated rollback of commit
 7e090f6a5412fd5f22b5b75ca7ae7844c3d025e9

PiperOrigin-RevId: 229798646
---
 tensorflow/core/kernels/unicode_script_op.cc | 3 ---
 third_party/icu/BUILD.bazel                  | 1 -
 2 files changed, 4 deletions(-)

diff --git a/tensorflow/core/kernels/unicode_script_op.cc b/tensorflow/core/kernels/unicode_script_op.cc
index f01bb52293..085e397eba 100644
--- a/tensorflow/core/kernels/unicode_script_op.cc
+++ b/tensorflow/core/kernels/unicode_script_op.cc
@@ -17,9 +17,6 @@ limitations under the License.
 #include "unicode/uscript.h"  // TF:icu
 #include "tensorflow/core/framework/op_kernel.h"
 
-// ICU codemap data is linked statically.
-#define U_STATIC_IMPLEMENTATION
-
 namespace tensorflow {
 
 class UnicodeScriptOp : public OpKernel {
diff --git a/third_party/icu/BUILD.bazel b/third_party/icu/BUILD.bazel
index 399962d249..36d6b9006b 100644
--- a/third_party/icu/BUILD.bazel
+++ b/third_party/icu/BUILD.bazel
@@ -44,7 +44,6 @@ cc_library(
     ]),
     copts = [
         "-DU_COMMON_IMPLEMENTATION",
-        "-DU_STATIC_IMPLEMENTATION",
         "-DU_HAVE_STD_ATOMICS",
     ] + select({
         ":android": [
-- 
GitLab


From 7b690b1687fb506ad9e7faabe5095bb266a0f68f Mon Sep 17 00:00:00 2001
From: Frank Chen <frankchn@google.com>
Date: Thu, 17 Jan 2019 12:42:39 -0800
Subject: [PATCH 0896/1095] Return the task_type if the Cluster Resolver is in
 "grpc" mode

PiperOrigin-RevId: 229799270
---
 .../python/distribute/cluster_resolver/tpu_cluster_resolver.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py b/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py
index 72b9990701..635948fd19 100644
--- a/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py
+++ b/tensorflow/python/distribute/cluster_resolver/tpu_cluster_resolver.py
@@ -378,7 +378,8 @@ class TPUClusterResolver(ClusterResolver):
     return self.master()
 
   def get_job_name(self):
-    if self._shouldResolve():
+    if (self._shouldResolve() or
+        self._tpu.startswith(compat.as_bytes('grpc://'))):
       return self.task_type
 
   def cluster_spec(self):
-- 
GitLab


From 094ae1c04d3c021e7e0048c91204a04bb08d3eb0 Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Thu, 17 Jan 2019 12:48:01 -0800
Subject: [PATCH 0897/1095] Remove useless comments.

PiperOrigin-RevId: 229800138
---
 tensorflow/core/lib/gif/gif_io.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/lib/gif/gif_io.cc b/tensorflow/core/lib/gif/gif_io.cc
index 3fad8c8b14..24aca854eb 100644
--- a/tensorflow/core/lib/gif/gif_io.cc
+++ b/tensorflow/core/lib/gif/gif_io.cc
@@ -94,8 +94,8 @@ uint8* Decode(const void* srcdata, int datasize,
   }
 
   const int num_frames = gif_file->ImageCount;
-  const int width = max_frame_width;    // gif_file->SWidth;
-  const int height = max_frame_height;  // gif_file->SHeight;
+  const int width = max_frame_width;
+  const int height = max_frame_height;
   const int channel = 3;
 
   uint8* const dstdata = allocate_output(num_frames, width, height, channel);
-- 
GitLab


From 76c2a2512c3fefe065415e338a98c1043acbb131 Mon Sep 17 00:00:00 2001
From: Tom Hennigan <tomhennigan@google.com>
Date: Thu, 17 Jan 2019 12:48:12 -0800
Subject: [PATCH 0898/1095] Use more specific assertions from absl base class.

PiperOrigin-RevId: 229800181
---
 tensorflow/python/module/module_test.py | 66 ++++++++++++-------------
 1 file changed, 33 insertions(+), 33 deletions(-)

diff --git a/tensorflow/python/module/module_test.py b/tensorflow/python/module/module_test.py
index e7c736696e..d5d36a7bbc 100644
--- a/tensorflow/python/module/module_test.py
+++ b/tensorflow/python/module/module_test.py
@@ -143,64 +143,64 @@ class VariableTrackingTest(test.TestCase):
 
   def test_variables(self):
     m = RecursiveModule(3)
-    self.assertEqual(m.variables, (m.w, m.child.w, m.child.child.w))
-    self.assertEqual(m.child.variables, (m.child.w, m.child.child.w))
-    self.assertEqual(m.child.child.variables, (m.child.child.w,))
+    self.assertCountEqual(m.variables, (m.w, m.child.w, m.child.child.w))
+    self.assertCountEqual(m.child.variables, (m.child.w, m.child.child.w))
+    self.assertCountEqual(m.child.child.variables, (m.child.child.w,))
 
   def test_owned_variables(self):
     m = RecursiveModule(3)
-    self.assertEqual(m.owned_variables, (m.w,))
-    self.assertEqual(m.child.owned_variables, (m.child.w,))
-    self.assertEqual(m.child.child.owned_variables, (m.child.child.w,))
+    self.assertCountEqual(m.owned_variables, (m.w,))
+    self.assertCountEqual(m.child.owned_variables, (m.child.w,))
+    self.assertCountEqual(m.child.child.owned_variables, (m.child.child.w,))
 
   def test_trainable_variables(self):
     m = RecursiveModule(3)
-    self.assertEqual(m.trainable_variables,
-                     (m.w, m.child.w, m.child.child.w))
-    self.assertEqual(m.child.trainable_variables,
-                     (m.child.w, m.child.child.w))
-    self.assertEqual(m.child.child.trainable_variables, (m.child.child.w,))
+    self.assertCountEqual(m.trainable_variables,
+                          (m.w, m.child.w, m.child.child.w))
+    self.assertCountEqual(m.child.trainable_variables,
+                          (m.child.w, m.child.child.w))
+    self.assertCountEqual(m.child.child.trainable_variables, (m.child.child.w,))
 
   def test_trainable_variables_ignores_non_trainable(self):
     m = RecursiveModule(3, trainable=False)
-    self.assertEqual(len(m.trainable_variables), 0)
-    self.assertEqual(len(m.child.trainable_variables), 0)
-    self.assertEqual(len(m.child.child.trainable_variables), 0)
+    self.assertEmpty(m.trainable_variables)
+    self.assertEmpty(m.child.trainable_variables)
+    self.assertEmpty(m.child.child.trainable_variables)
 
   def test_owned_trainable_variables(self):
     m = RecursiveModule(3)
-    self.assertEqual(m.owned_trainable_variables, (m.w,))
-    self.assertEqual(m.child.owned_trainable_variables, (m.child.w,))
-    self.assertEqual(m.child.child.owned_trainable_variables,
-                     (m.child.child.w,))
+    self.assertCountEqual(m.owned_trainable_variables, (m.w,))
+    self.assertCountEqual(m.child.owned_trainable_variables, (m.child.w,))
+    self.assertCountEqual(m.child.child.owned_trainable_variables,
+                          (m.child.child.w,))
 
   def test_owned_trainable_variables_ignores_non_trainable(self):
     m = RecursiveModule(3, trainable=False)
-    self.assertEqual(len(m.owned_trainable_variables), 0)
-    self.assertEqual(len(m.child.owned_trainable_variables), 0)
-    self.assertEqual(len(m.child.child.owned_trainable_variables), 0)
+    self.assertEmpty(m.owned_trainable_variables)
+    self.assertEmpty(m.child.owned_trainable_variables)
+    self.assertEmpty(m.child.child.owned_trainable_variables)
 
 
 class ModuleTrackingTest(test.TestCase):
 
   def test_owned_submodules(self):
     m = RecursiveModule(3)
-    self.assertEqual(list(m.owned_submodules), [m.child])
-    self.assertEqual(list(m.child.owned_submodules), [m.child.child])
-    self.assertEqual(list(m.child.child.owned_submodules), [])
+    self.assertCountEqual(m.owned_submodules, [m.child])
+    self.assertCountEqual(m.child.owned_submodules, [m.child.child])
+    self.assertEmpty(list(m.child.child.owned_submodules))
 
   def test_submodules(self):
     m = RecursiveModule(3)
-    self.assertEqual(list(m.submodules), [m.child, m.child.child])
-    self.assertEqual(list(m.child.submodules), [m.child.child])
-    self.assertEqual(list(m.child.child.submodules), [])
+    self.assertCountEqual(m.submodules, [m.child, m.child.child])
+    self.assertCountEqual(m.child.submodules, [m.child.child])
+    self.assertEmpty(list(m.child.child.submodules))
 
   def test_non_ctor_submodule(self):
     m = TreeModule()
     leaf1 = m.new_leaf()
-    self.assertEqual(set(m.submodules), {leaf1})
+    self.assertCountEqual(m.submodules, (leaf1,))
     leaf2 = m.new_leaf()
-    self.assertEqual(set(m.submodules), {leaf1, leaf2})
+    self.assertCountEqual(m.submodules, (leaf1, leaf2))
 
 
 class CommonErrorsTest(test.TestCase):
@@ -323,12 +323,12 @@ class WalkTest(test.TestCase):
     parent = SimpleModule()
     child = parent.c
 
-    self.assertEqual(
-        list(module.walk(parent, predicate=IS_MEMBER)),
+    self.assertCountEqual(
+        module.walk(parent, predicate=IS_MEMBER),
         [parent.a[0], parent.a[1], parent.z])
 
-    self.assertEqual(
-        list(module.walk(parent, recurse_if=IS_MODULE, predicate=IS_MEMBER)),
+    self.assertCountEqual(
+        module.walk(parent, recurse_if=IS_MODULE, predicate=IS_MEMBER),
         [parent.a[0], parent.a[1], parent.z, child.a[0], child.a[1], child.z])
 
 
-- 
GitLab


From aed554b573df4387e47c8f9e7d4ff52963c6e89b Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Thu, 17 Jan 2019 12:48:46 -0800
Subject: [PATCH 0899/1095] Prevent OOMs in too large JPG multi-channel images.

PiperOrigin-RevId: 229800275
---
 tensorflow/core/lib/jpeg/jpeg_mem.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/lib/jpeg/jpeg_mem.cc b/tensorflow/core/lib/jpeg/jpeg_mem.cc
index f7a359eb5b..9e7d1e6410 100644
--- a/tensorflow/core/lib/jpeg/jpeg_mem.cc
+++ b/tensorflow/core/lib/jpeg/jpeg_mem.cc
@@ -157,7 +157,8 @@ uint8* UncompressLow(const void* srcdata, FewerArgsForCompiler* argball) {
   jpeg_calc_output_dimensions(&cinfo);
 
   int64 total_size = static_cast<int64>(cinfo.output_height) *
-                     static_cast<int64>(cinfo.output_width);
+                     static_cast<int64>(cinfo.output_width) *
+                     static_cast<int64>(cinfo.num_components);
   // Some of the internal routines do not gracefully handle ridiculously
   // large images, so fail fast.
   if (cinfo.output_width <= 0 || cinfo.output_height <= 0) {
-- 
GitLab


From 05bef43400c83ca6d27617e27f1661e14f5f05e8 Mon Sep 17 00:00:00 2001
From: Saurabh Saxena <srbs@google.com>
Date: Thu, 17 Jan 2019 12:52:46 -0800
Subject: [PATCH 0900/1095] Do not constant fold nodes with DT_VARIANT type
 outputs in XLA. XLA does not support Const nodes of type Variant. It needs to
 see the source ops for the Variant operations to be able to build its own
 representation.

PiperOrigin-RevId: 229801068
---
 .../jit/encapsulate_subgraphs_pass.cc         | 28 +++++++++++-
 .../compiler/tests/tensor_list_ops_test.py    | 29 ++++++++++---
 tensorflow/compiler/tf2xla/xla_compiler.cc    | 28 +++++++++++-
 tensorflow/core/common_runtime/function.cc    | 10 ++++-
 tensorflow/core/common_runtime/function.h     |  3 ++
 .../core/common_runtime/graph_optimizer.cc    | 10 ++++-
 .../core/common_runtime/graph_optimizer.h     | 43 ++++++++++++-------
 7 files changed, 123 insertions(+), 28 deletions(-)

diff --git a/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc b/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc
index 03aba97bbe..a21e083131 100644
--- a/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc
+++ b/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc
@@ -2535,7 +2535,33 @@ Status EncapsulateSubgraphsPass::Run(
             std::vector<int>* input_permutation,
             std::vector<int>* output_permutation, NodeDef* node) {
         // Optimize the subgraph.
-        OptimizeGraph(flr, subgraph);
+        // Do not constant fold nodes that output DT_VARIANT type tensors.
+        // XLA does not support Const nodes of Variant type since it needs
+        // to know the original ops to be able to compile them to the relevant
+        // XLA form.
+        // TODO(srbs): This filter is a little conservative. E.g. a subgraph of
+        // the form:
+        //                          Const
+        //                            |
+        // EmptyTensorList -> TensorListPushBack -> TensorListPopBack -> Op
+        //                                                  |
+        //                                        (Discard popped list)
+        //
+        // Would have been reduced to "Const -> Op" without this filter.
+        // However since we are only allowed to specify the filter at the "Node"
+        // level there is no good way to allow the above behavior. So we
+        // disallow any sort of constant folding on Variant nodes for now.
+        auto cf_consider_fn = [](const Node* n) {
+          for (const auto& output_arg : n->op_def().output_arg()) {
+            if (output_arg.type() == DT_VARIANT) {
+              return false;
+            }
+          }
+          return true;
+        };
+        GraphOptimizer::Options graph_optimizer_options;
+        graph_optimizer_options.cf_consider_fn = cf_consider_fn;
+        OptimizeGraph(flr, subgraph, graph_optimizer_options);
 
         const int num_args = input_permutation->size();
         std::vector<bool> const_args(num_args);
diff --git a/tensorflow/compiler/tests/tensor_list_ops_test.py b/tensorflow/compiler/tests/tensor_list_ops_test.py
index 5c079d595c..3b48aed60c 100644
--- a/tensorflow/compiler/tests/tensor_list_ops_test.py
+++ b/tensorflow/compiler/tests/tensor_list_ops_test.py
@@ -48,24 +48,39 @@ class ListOpsTest(xla_test.XLATestCase):
 
   def testPushPop(self):
     with self.cached_session() as sess, self.test_scope():
-      num = array_ops.placeholder(dtypes.int32)
       l = list_ops.tensor_list_reserve(
-          element_shape=(7, 15), num_elements=num, element_dtype=dtypes.float32)
+          element_shape=(7, 15), num_elements=10, element_dtype=dtypes.float32)
       l = list_ops.tensor_list_push_back(
           l, constant_op.constant(1.0, shape=(7, 15)))
       l = list_ops.tensor_list_push_back(
           l, constant_op.constant(2.0, shape=(7, 15)))
       l, e2 = list_ops.tensor_list_pop_back(l, element_dtype=dtypes.float32)
       _, e1 = list_ops.tensor_list_pop_back(l, element_dtype=dtypes.float32)
-      self.assertAllEqual(sess.run(e2, {num: 10}), 2.0 * np.ones((7, 15)))
-      self.assertAllEqual(sess.run(e1, {num: 10}), 1.0 * np.ones((7, 15)))
+      self.assertAllEqual(sess.run(e2), 2.0 * np.ones((7, 15)))
+      self.assertAllEqual(sess.run(e1), 1.0 * np.ones((7, 15)))
+
+  def testDoNotConstantFoldVariants(self):
+    with self.cached_session() as sess, self.test_scope():
+      val = array_ops.placeholder(dtype=dtypes.float32)
+      l = list_ops.tensor_list_reserve(
+          element_shape=(7, 15), num_elements=10, element_dtype=dtypes.float32)
+      # Note: Pushing a Placeholder will force the constant folding code
+      # to build a Const node with a DT_VARIANT output. This tests that XLA
+      # passes a cf_consider_fn which prevent folding such nodes.
+      l = list_ops.tensor_list_push_back(
+          l, array_ops.fill(value=val, dims=(7, 15)))
+      l = list_ops.tensor_list_push_back(
+          l, constant_op.constant(2.0, shape=(7, 15)))
+      l, e2 = list_ops.tensor_list_pop_back(l, element_dtype=dtypes.float32)
+      _, e1 = list_ops.tensor_list_pop_back(l, element_dtype=dtypes.float32)
+      self.assertAllEqual(sess.run(e2, {val: 1.0}), 2.0 * np.ones((7, 15)))
+      self.assertAllEqual(sess.run(e1, {val: 1.0}), 1.0 * np.ones((7, 15)))
 
   def testPushPopSeparateLists(self):
     with self.cached_session() as sess, self.test_scope():
-      num = array_ops.placeholder(dtypes.int32)
       l = list_ops.tensor_list_reserve(
           element_shape=scalar_shape(),
-          num_elements=num,
+          num_elements=20,
           element_dtype=dtypes.float32)
       l = list_ops.tensor_list_push_back(l, constant_op.constant(1.0))
       l2 = list_ops.tensor_list_push_back(l, constant_op.constant(2.0))
@@ -75,7 +90,7 @@ class ListOpsTest(xla_test.XLATestCase):
       l2, e22 = list_ops.tensor_list_pop_back(l2, element_dtype=dtypes.float32)
       l3, e31 = list_ops.tensor_list_pop_back(l3, element_dtype=dtypes.float32)
       l3, e32 = list_ops.tensor_list_pop_back(l3, element_dtype=dtypes.float32)
-      result = sess.run([e11, [e21, e22], [e31, e32]], {num: 20})
+      result = sess.run([e11, [e21, e22], [e31, e32]])
       self.assertEqual(result, [1.0, [2.0, 1.0], [3.0, 1.0]])
 
   def testEmptyTensorList(self):
diff --git a/tensorflow/compiler/tf2xla/xla_compiler.cc b/tensorflow/compiler/tf2xla/xla_compiler.cc
index ee461a3c07..15fd265686 100644
--- a/tensorflow/compiler/tf2xla/xla_compiler.cc
+++ b/tensorflow/compiler/tf2xla/xla_compiler.cc
@@ -462,8 +462,34 @@ std::unique_ptr<Graph> XlaCompiler::GetGraph(const FunctionBody* fbody) {
   opts.set_do_function_inlining(true);
   opts.set_do_constant_folding(true);
   GraphOptimizer optimizer(opts);
+  // Do not constant fold nodes that output DT_VARIANT type tensors.
+  // XLA does not support Const nodes of Variant type since it needs
+  // to know the original ops to be able to compile them to the relevant
+  // XLA form.
+  // TODO(srbs): This filter is a little conservative. E.g. a subgraph of
+  // the form:
+  //                          Const
+  //                            |
+  // EmptyTensorList -> TensorListPushBack -> TensorListPopBack -> Op
+  //                                                  |
+  //                                        (Discard popped list)
+  //
+  // Would have been reduced to "Const -> Op" without this filter.
+  // However since we are only allowed to specify the filter at the "Node"
+  // level there is no good way to allow the above behavior. So we
+  // disallow any sort of constant folding on Variant nodes for now.
+  auto cf_consider_fn = [](const Node* n) {
+    for (const auto& output_arg : n->op_def().output_arg()) {
+      if (output_arg.type() == DT_VARIANT) {
+        return false;
+      }
+    }
+    return true;
+  };
+  GraphOptimizer::Options graph_optimizer_options;
+  graph_optimizer_options.cf_consider_fn = cf_consider_fn;
   optimizer.Optimize(flib_runtime_, flib_runtime_->env(),
-                     /*device=*/nullptr, &graph, /*shape_map=*/nullptr);
+                     /*device=*/nullptr, &graph, graph_optimizer_options);
 
   return graph;
 }
diff --git a/tensorflow/core/common_runtime/function.cc b/tensorflow/core/common_runtime/function.cc
index 3df09eac29..82bbb7e689 100644
--- a/tensorflow/core/common_runtime/function.cc
+++ b/tensorflow/core/common_runtime/function.cc
@@ -786,13 +786,19 @@ void DumpGraph(StringPiece label, const Graph* g) {
   }
 }
 
-void OptimizeGraph(FunctionLibraryRuntime* lib, std::unique_ptr<Graph>* g) {
+void OptimizeGraph(FunctionLibraryRuntime* lib, std::unique_ptr<Graph>* g,
+                   const GraphOptimizer::Options& graph_optimizer_options) {
   OptimizerOptions opts;
   opts.set_do_common_subexpression_elimination(true);
   opts.set_do_function_inlining(true);
   opts.set_do_constant_folding(true);
   GraphOptimizer optimizer(opts);
-  optimizer.Optimize(lib, lib->env(), lib->device(), g, /*shape_map=*/nullptr);
+  optimizer.Optimize(lib, lib->env(), lib->device(), g,
+                     graph_optimizer_options);
+}
+
+void OptimizeGraph(FunctionLibraryRuntime* lib, std::unique_ptr<Graph>* g) {
+  OptimizeGraph(lib, g, GraphOptimizer::Options());
 }
 
 namespace {
diff --git a/tensorflow/core/common_runtime/function.h b/tensorflow/core/common_runtime/function.h
index eeca66f5d0..df884f7577 100644
--- a/tensorflow/core/common_runtime/function.h
+++ b/tensorflow/core/common_runtime/function.h
@@ -21,6 +21,7 @@ limitations under the License.
 
 #include "tensorflow/core/common_runtime/device.h"
 #include "tensorflow/core/common_runtime/device_mgr.h"
+#include "tensorflow/core/common_runtime/graph_optimizer.h"
 #include "tensorflow/core/common_runtime/process_function_library_runtime.h"
 #include "tensorflow/core/framework/function.h"
 #include "tensorflow/core/graph/graph.h"
@@ -133,6 +134,8 @@ void DumpGraph(StringPiece label, const Graph* g);
 // OptimizeGraph mutates **g extensively and replaces '*g' with a
 // complete copy. Therefore, the caller should not keep any references
 // to nodes *g.
+void OptimizeGraph(FunctionLibraryRuntime* lib, std::unique_ptr<Graph>* g,
+                   const GraphOptimizer::Options& graph_optimizer_options);
 void OptimizeGraph(FunctionLibraryRuntime* lib, std::unique_ptr<Graph>* g);
 
 // Convert the Graph of a function to a GraphDef.
diff --git a/tensorflow/core/common_runtime/graph_optimizer.cc b/tensorflow/core/common_runtime/graph_optimizer.cc
index 37a979a8f1..7905944fb1 100644
--- a/tensorflow/core/common_runtime/graph_optimizer.cc
+++ b/tensorflow/core/common_runtime/graph_optimizer.cc
@@ -38,8 +38,7 @@ void GraphOptimizer::Optimize(
     std::unique_ptr<Graph>* graph,
     const std::unordered_map<string, std::vector<PartialTensorShape>>*
         shape_map,
-    const std::function<bool(const Node*)>& cse_consider_fn,
-    const std::function<bool(const Node*)>& cf_consider_fn) {
+    const NodePredicate& cse_consider_fn, const NodePredicate& cf_consider_fn) {
   Graph* g = graph->get();
   DumpGraph("Initial", g);
 
@@ -103,4 +102,11 @@ void GraphOptimizer::Optimize(
   DumpGraph("ReCopy", graph->get());
 }
 
+void GraphOptimizer::Optimize(FunctionLibraryRuntime* runtime, Env* env,
+                              Device* device, std::unique_ptr<Graph>* graph,
+                              const Options& options) {
+  Optimize(runtime, env, device, graph, options.shape_map,
+           options.cse_consider_fn, options.cf_consider_fn);
+}
+
 }  // end namespace tensorflow
diff --git a/tensorflow/core/common_runtime/graph_optimizer.h b/tensorflow/core/common_runtime/graph_optimizer.h
index 789cc56942..05150608f0 100644
--- a/tensorflow/core/common_runtime/graph_optimizer.h
+++ b/tensorflow/core/common_runtime/graph_optimizer.h
@@ -26,6 +26,28 @@ namespace tensorflow {
 
 class GraphOptimizer {
  public:
+  using NodePredicate = std::function<bool(const Node*)>;
+
+  struct Options {
+    // If not null it maps from nodes in graph to partially-known
+    // shapes of their outputs, and may be used, e.g., in the constant folding
+    // pass. The use of shape_map implies that the mapping from node name to the
+    // vector of partial shapes of its outputs is stable, i.e., no optimization
+    // pass may replace a node with a different node of the same name that has a
+    // different number of outputs, or outputs with different known shapes.
+    // TODO(b/65453533) introduce a unique way to name nodes in a graph.
+    std::unordered_map<string, std::vector<PartialTensorShape>>* shape_map =
+        nullptr;
+
+    // If not null then only nodes for which cse_consider_fn returns true will
+    // be considered for CSE.
+    NodePredicate cse_consider_fn = nullptr;
+
+    // If not null then only nodes for which cf_consider_fn returns true will be
+    // considered for CF.
+    NodePredicate cf_consider_fn = nullptr;
+  };
+
   GraphOptimizer(const OptimizerOptions& opts);
   ~GraphOptimizer();
 
@@ -34,26 +56,17 @@ class GraphOptimizer {
   // on which the 'graph' will execute. It's passed to the optimizers
   // so that they can respect constraints if any, that should be
   // respected.
-  //
-  // If shape_map is not null it maps from nodes in graph to partially-known
-  // shapes of their outputs, and may be used, e.g., in the constant folding
-  // pass. The use of shape_map implies that the mapping from node name to the
-  // vector of partial shapes of its outputs is stable, i.e., no optimization
-  // pass may replace a node with a different node of the same name that has a
-  // different number of outputs, or outputs with different known shapes.
-  // TODO(b/65453533) introduce a unique way to name nodes in a graph.
-  //
-  // If cse_consider_fn is not null then only nodes for which cse_consider_fn
-  // returns true will be considered for CSE.
-  // If cf_consider_fn is not null then only nodes for which cf_consider_fn
-  // returns true will be considered for CF.
+  void Optimize(FunctionLibraryRuntime* runtime, Env* env, Device* device,
+                std::unique_ptr<Graph>* graph,
+                const Options& graph_optimizer_options);
+  // DEPRECATED: Consider passing a GraphOptimizer::Options object instead.
   void Optimize(
       FunctionLibraryRuntime* runtime, Env* env, Device* device,
       std::unique_ptr<Graph>* graph,
       const std::unordered_map<string, std::vector<PartialTensorShape>>*
           shape_map,
-      const std::function<bool(const Node*)>& cse_consider_fn = nullptr,
-      const std::function<bool(const Node*)>& cf_consider_fn = nullptr);
+      const NodePredicate& cse_consider_fn = nullptr,
+      const NodePredicate& cf_consider_fn = nullptr);
 
   const OptimizerOptions& options() { return opts_; }
 
-- 
GitLab


From c1ccc03a08211dddc409759bfab22ed9af59078c Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Thu, 17 Jan 2019 12:53:51 -0800
Subject: [PATCH 0901/1095] [tf.data] Disabling default optimizations in
 benchmarks.

This CL:
- makes sure all benchmarks disable default optimizations
- removes redundant print statements
- various minor fixes

PiperOrigin-RevId: 229801289
---
 .../python/data/benchmarks/batch_benchmark.py | 13 +--
 .../data/benchmarks/filter_benchmark.py       | 10 +-
 .../from_tensor_slices_benchmark.py           | 26 +++---
 .../data/benchmarks/list_files_benchmark.py   |  8 +-
 .../python/data/benchmarks/map_benchmark.py   | 21 ++---
 .../python/data/benchmarks/range_benchmark.py | 17 ++--
 .../benchmarks/autotune_benchmark.py          | 16 +---
 .../benchmarks/csv_dataset_benchmark.py       |  5 +-
 .../benchmarks/map_and_batch_benchmark.py     | 91 +++++++------------
 .../benchmarks/map_vectorization_benchmark.py | 31 +++----
 .../benchmarks/matching_files_benchmark.py    | 10 +-
 .../benchmarks/optimize_benchmark.py          |  9 +-
 .../parallel_interleave_benchmark.py          |  6 +-
 .../rejection_resample_benchmark.py           |  3 +
 .../benchmarks/unbatch_benchmark.py           | 10 +-
 15 files changed, 117 insertions(+), 159 deletions(-)

diff --git a/tensorflow/python/data/benchmarks/batch_benchmark.py b/tensorflow/python/data/benchmarks/batch_benchmark.py
index e063849f70..0ccf5c57d1 100644
--- a/tensorflow/python/data/benchmarks/batch_benchmark.py
+++ b/tensorflow/python/data/benchmarks/batch_benchmark.py
@@ -42,6 +42,9 @@ class BatchBenchmark(test.Benchmark):
 
     dataset = dataset_ops.Dataset.from_tensors(sparse_placeholder).repeat(
         ).batch(batch_size_placeholder)
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    dataset = dataset.with_options(options)
     iterator = dataset_ops.make_initializable_iterator(dataset)
     next_element = iterator.get_next()
 
@@ -72,13 +75,11 @@ class BatchBenchmark(test.Benchmark):
 
         median_wall_time = np.median(deltas) / 100.0
 
-        print("Batch sparse dataset non-zeros per row: %d batch_size: %d "
-              "wall time: %f"
-              % (non_zeros_per_row, batch_size, median_wall_time))
         self.report_benchmark(
-            iters=10000, wall_time=median_wall_time,
-            name="batch_sparse_dataset_nnz_%d_batch_size_%d" % (
-                non_zeros_per_row, batch_size))
+            iters=10000,
+            wall_time=median_wall_time,
+            name="sparse_num_elements_%d_batch_size_%d" %
+            (non_zeros_per_row, batch_size))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/benchmarks/filter_benchmark.py b/tensorflow/python/data/benchmarks/filter_benchmark.py
index a6d86fe221..e0ecf19e11 100644
--- a/tensorflow/python/data/benchmarks/filter_benchmark.py
+++ b/tensorflow/python/data/benchmarks/filter_benchmark.py
@@ -36,6 +36,9 @@ class FilterBenchmark(test.Benchmark):
     with ops.Graph().as_default():
       dataset = (
           dataset_ops.Dataset.from_tensors(True).repeat(None).filter(predicate))
+      options = dataset_ops.Options()
+      options.experimental_optimization.apply_default_optimizations = False
+      dataset = dataset.with_options(options)
       iterator = dataset_ops.make_one_shot_iterator(dataset)
       next_element = iterator.get_next()
 
@@ -51,12 +54,7 @@ class FilterBenchmark(test.Benchmark):
           deltas.append(end - start)
 
         median_wall_time = np.median(deltas) / 100
-        print("Filter dataset using %s. Median wall time: %f" %
-              (name, median_wall_time))
-        self.report_benchmark(
-            iters=100,
-            wall_time=median_wall_time,
-            name=name)
+        self.report_benchmark(iters=100, wall_time=median_wall_time, name=name)
 
   def benchmarkSimpleFunction(self):
     self._benchmark(array_ops.identity, "simple_function")
diff --git a/tensorflow/python/data/benchmarks/from_tensor_slices_benchmark.py b/tensorflow/python/data/benchmarks/from_tensor_slices_benchmark.py
index d7f1a4e7af..4e5559ddba 100644
--- a/tensorflow/python/data/benchmarks/from_tensor_slices_benchmark.py
+++ b/tensorflow/python/data/benchmarks/from_tensor_slices_benchmark.py
@@ -41,6 +41,9 @@ class FromTensorSlicesBenchmark(test.Benchmark):
     dataset = (
         dataset_ops.Dataset.from_tensor_slices(input_data)
         .repeat(num_epochs + 1).batch(batch_size))
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    dataset = dataset.with_options(options)
     iterator = dataset_ops.make_initializable_iterator(dataset)
     next_element = iterator.get_next()
 
@@ -59,9 +62,6 @@ class FromTensorSlicesBenchmark(test.Benchmark):
         pass
 
     median_wall_time = np.median(deltas)
-    print("Slice/repeat/batch with sess.run() input size: %d batch size: %d "
-          "Median wall time per element: %f" % (input_size, batch_size,
-                                                median_wall_time))
     self.report_benchmark(
         iters=len(deltas),
         wall_time=median_wall_time,
@@ -77,6 +77,9 @@ class FromTensorSlicesBenchmark(test.Benchmark):
     dataset = (
         dataset_ops.Dataset.from_tensor_slices(input_data)
         .repeat(num_epochs + 1).batch(batch_size))
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    dataset = dataset.with_options(options)
     iterator = dataset_ops.make_initializable_iterator(dataset)
     next_element = iterator.get_next()
 
@@ -96,10 +99,6 @@ class FromTensorSlicesBenchmark(test.Benchmark):
         pass
 
     median_wall_time = np.median(deltas)
-    print(
-        "Slice/repeat/batch with callable input size: %d batch size: %d Median"
-        " wall time per element: %f" % (input_size, batch_size,
-                                        median_wall_time))
     self.report_benchmark(
         iters=len(deltas),
         wall_time=median_wall_time,
@@ -116,6 +115,9 @@ class FromTensorSlicesBenchmark(test.Benchmark):
     dataset = (
         dataset_ops.Dataset.from_tensor_slices(input_data.reshape(100, 100))
         .repeat(num_epochs + 1))
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    dataset = dataset.with_options(options)
     iterator = dataset_ops.make_initializable_iterator(dataset)
     next_element = iterator.get_next()
 
@@ -135,9 +137,6 @@ class FromTensorSlicesBenchmark(test.Benchmark):
         pass
 
     median_wall_time = np.median(deltas)
-    print("Reshape/slice/repeat with callable input size: %d batch size: %d "
-          "Median wall time per element: %f" % (input_size, batch_size,
-                                                median_wall_time))
     self.report_benchmark(
         iters=len(deltas),
         wall_time=median_wall_time,
@@ -154,6 +153,9 @@ class FromTensorSlicesBenchmark(test.Benchmark):
     dataset = (
         dataset_ops.Dataset.from_tensor_slices(input_data).batch(batch_size)
         .cache().repeat(num_epochs + 1))
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    dataset = dataset.with_options(options)
     iterator = dataset_ops.make_initializable_iterator(dataset)
     next_element = iterator.get_next()
 
@@ -173,10 +175,6 @@ class FromTensorSlicesBenchmark(test.Benchmark):
         pass
 
     median_wall_time = np.median(deltas)
-    print(
-        "Slice/batch/cache/repeat with callable input size: %d batch size: %d "
-        "Median wall time per element: %f"
-        % (input_size, batch_size, median_wall_time))
     self.report_benchmark(
         iters=len(deltas),
         wall_time=median_wall_time,
diff --git a/tensorflow/python/data/benchmarks/list_files_benchmark.py b/tensorflow/python/data/benchmarks/list_files_benchmark.py
index 0dc2147112..70f8eeec9e 100644
--- a/tensorflow/python/data/benchmarks/list_files_benchmark.py
+++ b/tensorflow/python/data/benchmarks/list_files_benchmark.py
@@ -58,6 +58,9 @@ class ListFilesBenchmark(test.Benchmark):
     for _ in range(iters):
       with ops.Graph().as_default():
         dataset = dataset_ops.Dataset.list_files(patterns)
+        options = dataset_ops.Options()
+        options.experimental_optimization.apply_default_optimizations = False
+        dataset = dataset.with_options(options)
         next_element = dataset.make_one_shot_iterator().get_next()
         with session.Session() as sess:
           sub_deltas = []
@@ -71,11 +74,6 @@ class ListFilesBenchmark(test.Benchmark):
               break
           deltas.append(sub_deltas)
     median_deltas = np.median(deltas, axis=0)
-    print('Nested directory size (width*depth): %d*%d Median wall time: '
-          '%fs (read first filename), %fs (read second filename), avg %fs'
-          ' (read %d more filenames)' %
-          (width, depth, median_deltas[0], median_deltas[1],
-           np.average(median_deltas[2:]), len(median_deltas) - 2))
     self.report_benchmark(
         iters=iters,
         wall_time=np.sum(median_deltas),
diff --git a/tensorflow/python/data/benchmarks/map_benchmark.py b/tensorflow/python/data/benchmarks/map_benchmark.py
index 65d945cdae..b620eaaed5 100644
--- a/tensorflow/python/data/benchmarks/map_benchmark.py
+++ b/tensorflow/python/data/benchmarks/map_benchmark.py
@@ -38,17 +38,14 @@ class MapBenchmark(test.Benchmark):
         if mode == "general":
           map_fn = lambda x: x + 1
           use_inter_op_parallelism = True
-          print_label = ""
           benchmark_label = ""
         if mode == "single-threaded":
           map_fn = lambda x: x + 1
           use_inter_op_parallelism = False
-          print_label = " (single threaded mode)"
           benchmark_label = "_single_threaded"
         if mode == "short-circuit":
           map_fn = lambda x: x
           use_inter_op_parallelism = True  # should not have any significance
-          print_label = " (short circuit mode)"
           benchmark_label = "_short_circuit"
 
         with ops.Graph().as_default():
@@ -58,6 +55,9 @@ class MapBenchmark(test.Benchmark):
                 dataset,
                 map_fn,
                 use_inter_op_parallelism=use_inter_op_parallelism)
+          options = dataset_ops.Options()
+          options.experimental_optimization.apply_default_optimizations = False
+          dataset = dataset.with_options(options)
           iterator = dataset_ops.make_one_shot_iterator(dataset)
           next_element = iterator.get_next()
 
@@ -73,13 +73,10 @@ class MapBenchmark(test.Benchmark):
               deltas.append(end - start)
 
             median_wall_time = np.median(deltas) / 100
-            print("Map dataset chain length%s: %d Median wall time: %f" %
-                  (print_label, chain_length, median_wall_time))
             self.report_benchmark(
                 iters=1000,
                 wall_time=median_wall_time,
-                name="map_dataset_chain_length_%d%s" % (chain_length,
-                                                        benchmark_label))
+                name="chain_length_%d%s" % (chain_length, benchmark_label))
 
   def benchmarkMapFanOut(self):
     fan_outs = [1, 2, 5, 10, 20, 50, 100]
@@ -88,17 +85,14 @@ class MapBenchmark(test.Benchmark):
         if mode == "general":
           map_fn = lambda *xs: [x + 1 for x in xs]
           use_inter_op_parallelism = True
-          print_label = ""
           benchmark_label = ""
         if mode == "single-threaded":
           map_fn = lambda *xs: [x + 1 for x in xs]
           use_inter_op_parallelism = False
-          print_label = " (single threaded mode)"
           benchmark_label = "_single_threaded"
         if mode == "short-circuit":
           map_fn = lambda *xs: xs
           use_inter_op_parallelism = True  # should not have any significance
-          print_label = " (short circuit mode)"
           benchmark_label = "_short_circuit"
 
         with ops.Graph().as_default():
@@ -108,6 +102,9 @@ class MapBenchmark(test.Benchmark):
               dataset,
               map_fn,
               use_inter_op_parallelism=use_inter_op_parallelism)
+          options = dataset_ops.Options()
+          options.experimental_optimization.apply_default_optimizations = False
+          dataset = dataset.with_options(options)
           iterator = dataset_ops.make_one_shot_iterator(dataset)
           next_element = iterator.get_next()
 
@@ -123,12 +120,10 @@ class MapBenchmark(test.Benchmark):
               deltas.append(end - start)
 
             median_wall_time = np.median(deltas) / 100
-            print("Map dataset fan out%s: %d Median wall time: %f" %
-                  (print_label, fan_out, median_wall_time))
             self.report_benchmark(
                 iters=1000,
                 wall_time=median_wall_time,
-                name="map_dataset_fan_out_%d%s" % (fan_out, benchmark_label))
+                name="fan_out_%d%s" % (fan_out, benchmark_label))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/data/benchmarks/range_benchmark.py b/tensorflow/python/data/benchmarks/range_benchmark.py
index a5020e2873..375ff339a8 100644
--- a/tensorflow/python/data/benchmarks/range_benchmark.py
+++ b/tensorflow/python/data/benchmarks/range_benchmark.py
@@ -31,14 +31,16 @@ class RangeBenchmark(test.Benchmark):
 
   def _benchmarkRangeHelper(self, modeling_enabled):
     num_elements = 10000000 if modeling_enabled else 50000000
-    options = dataset_ops.Options()
-    options.experimental_autotune = modeling_enabled
 
     # Use `Dataset.skip()` and `Dataset.take()` to perform the iteration in
     # C++, and focus on the minimal overheads (excluding Python invocation
     # costs).
     dataset = dataset_ops.Dataset.range(num_elements).skip(
-        num_elements - 1).take(1).with_options(options)
+        num_elements - 1).take(1)
+    options = dataset_ops.Options()
+    options.experimental_autotune = modeling_enabled
+    options.experimental_optimization.apply_default_optimizations = False
+    dataset = dataset.with_options(options)
     iterator = dataset_ops.make_initializable_iterator(dataset)
     next_element = iterator.get_next()
 
@@ -54,11 +56,10 @@ class RangeBenchmark(test.Benchmark):
       end = time.time()
 
       time_per_element = (end - start) / num_elements
-      print("Average time per element (%s modeling): %f nanoseconds" % (
-          "with" if modeling_enabled else "without", time_per_element * 1e9))
-      self.report_benchmark(iters=num_elements, wall_time=time_per_element,
-                            name="benchmark_tf_data_dataset_range%s"
-                            % ("_with_modeling" if modeling_enabled else ""))
+      self.report_benchmark(
+          iters=num_elements,
+          wall_time=time_per_element,
+          name="modeling_%s" % ("on" if modeling_enabled else "off"))
 
   def benchmarkRange(self):
     for modeling_enabled in [False, True]:
diff --git a/tensorflow/python/data/experimental/benchmarks/autotune_benchmark.py b/tensorflow/python/data/experimental/benchmarks/autotune_benchmark.py
index 391b6711e9..bda7d38792 100644
--- a/tensorflow/python/data/experimental/benchmarks/autotune_benchmark.py
+++ b/tensorflow/python/data/experimental/benchmarks/autotune_benchmark.py
@@ -60,9 +60,6 @@ class AutotuneBenchmark(test.Benchmark):
         end = time.time()
         deltas.append(end - start)
 
-    print("%f (median), %f (mean), %f (stddev), %f (min), %f (max)\n" %
-          (np.median(deltas), np.mean(deltas), np.std(deltas), np.min(deltas),
-           np.max(deltas)))
     self.report_benchmark(
         iters=10000,
         wall_time=np.median(deltas),
@@ -87,6 +84,7 @@ class AutotuneBenchmark(test.Benchmark):
             batch_size=batch_size))
     options = dataset_ops.Options()
     options.experimental_autotune = autotune
+    options.experimental_optimization.apply_default_optimizations = False
     dataset = dataset.with_options(options)
     iterator = dataset_ops.make_one_shot_iterator(dataset)
     get_next = iterator.get_next()
@@ -101,10 +99,6 @@ class AutotuneBenchmark(test.Benchmark):
         end = time.time()
         deltas.append(end - start)
 
-    print("%f (median), %f (mean), %f (stddev), %f (min), %f (max)\n" %
-          (np.median(deltas), np.mean(deltas), np.std(deltas), np.min(deltas),
-           np.max(deltas)))
-
     self.report_benchmark(
         iters=1000,
         wall_time=np.median(deltas),
@@ -128,6 +122,7 @@ class AutotuneBenchmark(test.Benchmark):
         num_parallel_calls=optimization.AUTOTUNE)
     options = dataset_ops.Options()
     options.experimental_autotune = autotune
+    options.experimental_optimization.apply_default_optimizations = False
     dataset = dataset.with_options(options)
     iterator = dataset_ops.make_one_shot_iterator(dataset)
     get_next = iterator.get_next()
@@ -142,9 +137,6 @@ class AutotuneBenchmark(test.Benchmark):
         end = time.time()
         deltas.append(end - start)
 
-    print("%f (median), %f (mean), %f (stddev), %f (min), %f (max)\n" %
-          (np.median(deltas), np.mean(deltas), np.std(deltas), np.min(deltas),
-           np.max(deltas)))
     self.report_benchmark(
         iters=10000,
         wall_time=np.median(deltas),
@@ -190,6 +182,7 @@ class AutotuneBenchmark(test.Benchmark):
     dataset = dataset.map(f2, num_parallel_calls=optimization.AUTOTUNE)
     options = dataset_ops.Options()
     options.experimental_autotune = autotune
+    options.experimental_optimization.apply_default_optimizations = False
     dataset = dataset.with_options(options)
     iterator = dataset_ops.make_one_shot_iterator(dataset)
     get_next = iterator.get_next()
@@ -204,9 +197,6 @@ class AutotuneBenchmark(test.Benchmark):
         end = time.time()
         deltas.append(end - start)
 
-    print("%f (median), %f (mean), %f (stddev), %f (min), %f (max)\n" %
-          (np.median(deltas), np.mean(deltas), np.std(deltas), np.min(deltas),
-           np.max(deltas)))
     self.report_benchmark(
         iters=1000,
         wall_time=np.median(deltas),
diff --git a/tensorflow/python/data/experimental/benchmarks/csv_dataset_benchmark.py b/tensorflow/python/data/experimental/benchmarks/csv_dataset_benchmark.py
index 03345ce4e6..2e91e08c79 100644
--- a/tensorflow/python/data/experimental/benchmarks/csv_dataset_benchmark.py
+++ b/tensorflow/python/data/experimental/benchmarks/csv_dataset_benchmark.py
@@ -63,6 +63,9 @@ class CsvDatasetBenchmark(test.Benchmark):
 
   def _runBenchmark(self, dataset, num_cols, prefix):
     dataset = dataset.skip(self._num_per_iter - 1)
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    dataset = dataset.with_options(options)
     deltas = []
     for _ in range(10):
       next_element = dataset_ops.make_one_shot_iterator(dataset).get_next()
@@ -79,8 +82,6 @@ class CsvDatasetBenchmark(test.Benchmark):
       deltas.append(end - start)
     # Median wall time per CSV record read and decoded
     median_wall_time = np.median(deltas) / self._num_per_iter
-    print('%s num_cols: %d Median wall time: %f' % (prefix, num_cols,
-                                                    median_wall_time))
     self.report_benchmark(
         iters=self._num_per_iter,
         wall_time=median_wall_time,
diff --git a/tensorflow/python/data/experimental/benchmarks/map_and_batch_benchmark.py b/tensorflow/python/data/experimental/benchmarks/map_and_batch_benchmark.py
index b17f2bcd12..4b7c173786 100644
--- a/tensorflow/python/data/experimental/benchmarks/map_and_batch_benchmark.py
+++ b/tensorflow/python/data/experimental/benchmarks/map_and_batch_benchmark.py
@@ -26,7 +26,6 @@ import numpy as np
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.client import session
 from tensorflow.python.data.experimental.ops import batching
-from tensorflow.python.data.experimental.ops.optimization_options import OptimizationOptions
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -41,7 +40,7 @@ _NUMPY_RANDOM_SEED = 42
 class MapAndBatchBenchmark(test.Benchmark):
   """Benchmarks for `tf.data.experimental.map_and_batch()`."""
 
-  def benchmarkMapAndBatchDense(self):
+  def benchmarkMapAndBatch(self):
     """Measures the performance of parallelized batching."""
     shapes = [(), (10,), (10, 10), (10, 10, 10), (224, 224, 3)]
     batch_size_values = [1, 32, 64, 128, 1024]
@@ -55,6 +54,9 @@ class MapAndBatchBenchmark(test.Benchmark):
 
     dataset = dataset.apply(batching.map_and_batch(
         lambda _: dense_value, batch_size_placeholder))
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    dataset = dataset.with_options(options)
     iterator = dataset_ops.make_initializable_iterator(dataset)
     next_element = iterator.get_next()
 
@@ -88,13 +90,9 @@ class MapAndBatchBenchmark(test.Benchmark):
         median_wall_time = np.median(deltas) / 100.0
         iters = len(deltas) * 100
 
-        print("Map and batch dense dataset shape: %r batch_size: %d "
-              "wall time: %f (%d iters)"
-              % (shape, batch_size, median_wall_time, iters))
         self.report_benchmark(
             iters=iters, wall_time=median_wall_time,
-            name="benchmark_batch_dense_dataset_nnz_%d_batch_size_%d" % (
-                np.prod(shape), batch_size))
+            name="num_elements_%d_batch_size_%d" % (np.prod(shape), batch_size))
 
   def benchmarkMapAndBatchChainingVersusFusing(self):
     """Compares the performance of chaining and fusing map and batch.
@@ -128,49 +126,25 @@ class MapAndBatchBenchmark(test.Benchmark):
     def benchmark(label, series):
       """Runs benchmark the given series."""
 
-      print("%s:" % label)
-
-      def make_base_dataset(element_size):
+      def make_dataset(element_size, num_calls, batch_size):  # pylint: disable=missing-docstring
         k = 1024 * 1024
         x = constant_op.constant(np.random.rand(element_size, 4 * k))
         y = constant_op.constant(np.random.rand(4 * k, 1))
-        return dataset_ops.Dataset.range(1000000000000).map(lambda _: (x, y))
+        dataset = dataset_ops.Dataset.range(1000000000000).map(lambda _: (x, y))
+        dataset = dataset.map(
+            math_ops.matmul,
+            num_parallel_calls=num_calls).batch(batch_size=batch_size)
+        options = dataset_ops.Options()
+        options.experimental_optimization.apply_default_optimizations = False
+        return dataset.with_options(options)
 
       for num_calls, inter_op, element_size, batch_size in series:
-
         num_iters = 1024 // (
             (element_size * batch_size) // min(num_calls, inter_op))
-        fused_dataset = make_base_dataset(element_size)
-        fused_dataset = fused_dataset.map(
-            math_ops.matmul,
-            num_parallel_calls=num_calls).batch(batch_size=batch_size)
-
-        fused_iterator = dataset_ops.make_one_shot_iterator(fused_dataset)
-        fused_get_next = fused_iterator.get_next()
-
-        fused_deltas = []
-        with session.Session(
-            config=config_pb2.ConfigProto(
-                inter_op_parallelism_threads=inter_op,
-                use_per_session_threads=True)) as sess:
-
-          for _ in range(5):
-            sess.run(fused_get_next.op)
-          for _ in range(num_iters):
-            start = time.time()
-            sess.run(fused_get_next.op)
-            end = time.time()
-            fused_deltas.append(end - start)
-
-        # `map_and_batch_fusion` is optimized by default. To get the chained
-        # dataset, with have to disable it.
-        options = dataset_ops.Options()
-        options.experimental_optimization = OptimizationOptions()
-        options.experimental_optimization.map_and_batch_fusion = False
-        chained_dataset = fused_dataset.with_options(options)
+        # By default the chained map().batch() calls will not be fused.
+        chained_dataset = make_dataset(element_size, num_calls, batch_size)
         chained_iterator = dataset_ops.make_one_shot_iterator(chained_dataset)
         chained_get_next = chained_iterator.get_next()
-
         chained_deltas = []
         with session.Session(
             config=config_pb2.ConfigProto(
@@ -184,27 +158,32 @@ class MapAndBatchBenchmark(test.Benchmark):
             end = time.time()
             chained_deltas.append(end - start)
 
-        print(
-            "batch size: %d, num parallel calls: %d, inter-op parallelism: %d, "
-            "element size: %d, num iters: %d\nchained wall time: %f (median), "
-            "%f (mean), %f (stddev), %f (min), %f (max)\n  fused wall time: "
-            "%f (median), %f (mean), %f (stddev), %f (min), %f (max)\n    "
-            "chained/fused:    %.2fx (median),    %.2fx (mean)" %
-            (batch_size, num_calls, inter_op, element_size, num_iters,
-             np.median(chained_deltas), np.mean(chained_deltas),
-             np.std(chained_deltas), np.min(chained_deltas),
-             np.max(chained_deltas), np.median(fused_deltas),
-             np.mean(fused_deltas), np.std(fused_deltas), np.min(fused_deltas),
-             np.max(fused_deltas),
-             np.median(chained_deltas) / np.median(fused_deltas),
-             np.mean(chained_deltas) / np.mean(fused_deltas)))
-
         self.report_benchmark(
             iters=num_iters,
             wall_time=np.median(chained_deltas),
             name=name("chained", label, num_calls, inter_op, element_size,
                       batch_size))
 
+        # Apply an option to the default dataset that will fuse map().batch().
+        options = dataset_ops.Options()
+        options.experimental_optimization.map_and_batch_fusion = True
+        fused_dataset = chained_dataset.with_options(options)
+        fused_iterator = dataset_ops.make_one_shot_iterator(fused_dataset)
+        fused_get_next = fused_iterator.get_next()
+        fused_deltas = []
+        with session.Session(
+            config=config_pb2.ConfigProto(
+                inter_op_parallelism_threads=inter_op,
+                use_per_session_threads=True)) as sess:
+
+          for _ in range(5):
+            sess.run(fused_get_next.op)
+          for _ in range(num_iters):
+            start = time.time()
+            sess.run(fused_get_next.op)
+            end = time.time()
+            fused_deltas.append(end - start)
+
         self.report_benchmark(
             iters=num_iters,
             wall_time=np.median(fused_deltas),
diff --git a/tensorflow/python/data/experimental/benchmarks/map_vectorization_benchmark.py b/tensorflow/python/data/experimental/benchmarks/map_vectorization_benchmark.py
index a60ba0a857..50e3a5c469 100644
--- a/tensorflow/python/data/experimental/benchmarks/map_vectorization_benchmark.py
+++ b/tensorflow/python/data/experimental/benchmarks/map_vectorization_benchmark.py
@@ -24,7 +24,6 @@ import numpy as np
 from tensorflow.core.example import example_pb2
 from tensorflow.core.example import feature_pb2
 from tensorflow.python.client import session
-from tensorflow.python.data.experimental.ops import optimization_options
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.util import nest
 from tensorflow.python.framework import constant_op
@@ -115,31 +114,27 @@ class MapVectorizationBenchmark(test.Benchmark):
 
   def _compare(self, input_dataset, map_fn, batch_size, input_size, str_id):
     num_elems = int(np.sum([np.prod(x) for x in input_size]))
-    name_template = "{}__batch_size_{}_input_element_size_{}_{}"
+    name_template = "{}_batch_size_{}_input_element_size_{}_{}"
 
-    base_dataset = input_dataset.map(map_fn).batch(batch_size)
+    unoptimized_dataset = input_dataset.map(map_fn).batch(batch_size)
 
     options = dataset_ops.Options()
-    opt_options = optimization_options.OptimizationOptions()
-    # Disable default map_and_batch_fusion optimization
-    opt_options.map_and_batch_fusion = False
-    options.experimental_optimization = opt_options
-    base_dataset = base_dataset.with_options(options)
+    options.experimental_optimization.apply_default_optimizations = False
+    unoptimized_dataset = unoptimized_dataset.with_options(options)
+    unoptimized_next = dataset_ops.make_one_shot_iterator(
+        unoptimized_dataset).get_next()
 
-    unoptimized_op = dataset_ops.make_one_shot_iterator(base_dataset).get_next()
-
-    optimized_options = dataset_ops.Options()
-    opt_options = optimization_options.OptimizationOptions()
-    opt_options.map_vectorization = True
-    optimized_options.experimental_optimization = opt_options
-    optimized = base_dataset.with_options(optimized_options)
-    optimized_op = dataset_ops.make_one_shot_iterator(optimized).get_next()
+    options = dataset_ops.Options()
+    options.experimental_optimization.map_vectorization = True
+    optimized_dataset = unoptimized_dataset.with_options(options)
+    optimized_next = dataset_ops.make_one_shot_iterator(
+        optimized_dataset).get_next()
 
     unoptimized_time = self._run(
-        unoptimized_op,
+        unoptimized_next,
         name=name_template.format(str_id, batch_size, num_elems, "unoptimized"))
     optimized_time = self._run(
-        optimized_op,
+        optimized_next,
         name=name_template.format(str_id, batch_size, num_elems, "optimized"))
 
     print("Batch size: {}\n"
diff --git a/tensorflow/python/data/experimental/benchmarks/matching_files_benchmark.py b/tensorflow/python/data/experimental/benchmarks/matching_files_benchmark.py
index c53f8dd7c5..cb5bf2946d 100644
--- a/tensorflow/python/data/experimental/benchmarks/matching_files_benchmark.py
+++ b/tensorflow/python/data/experimental/benchmarks/matching_files_benchmark.py
@@ -60,6 +60,9 @@ class MatchingFilesBenchmark(test.Benchmark):
     for _ in range(iters):
       with ops.Graph().as_default():
         dataset = matching_files.MatchingFilesDataset(patterns)
+        options = dataset_ops.Options()
+        options.experimental_optimization.apply_default_optimizations = False
+        dataset = dataset.with_options(options)
         next_element = dataset_ops.make_one_shot_iterator(dataset).get_next()
 
         with session.Session() as sess:
@@ -75,11 +78,6 @@ class MatchingFilesBenchmark(test.Benchmark):
           deltas.append(sub_deltas)
 
     median_deltas = np.median(deltas, axis=0)
-    print('Nested directory size (width*depth): %d*%d Median wall time: '
-          '%fs (read first filename), %fs (read second filename), avg %fs'
-          ' (read %d more filenames)' %
-          (width, depth, median_deltas[0], median_deltas[1],
-           np.average(median_deltas[2:]), len(median_deltas) - 2))
     self.report_benchmark(
         iters=iters,
         wall_time=np.sum(median_deltas),
@@ -92,7 +90,7 @@ class MatchingFilesBenchmark(test.Benchmark):
             (len(median_deltas) - 2):
                 np.average(median_deltas[2:])
         },
-        name='dataset_nested_directory(%d*%d)' %
+        name='nested_directory(%d*%d)' %
         (width, depth))
 
     shutil.rmtree(tmp_dir, ignore_errors=True)
diff --git a/tensorflow/python/data/experimental/benchmarks/optimize_benchmark.py b/tensorflow/python/data/experimental/benchmarks/optimize_benchmark.py
index 1bbee5e7a3..395a529f85 100644
--- a/tensorflow/python/data/experimental/benchmarks/optimize_benchmark.py
+++ b/tensorflow/python/data/experimental/benchmarks/optimize_benchmark.py
@@ -47,6 +47,7 @@ class OptimizationBenchmark(test.Benchmark):
         dataset = dataset.map(lambda x: x)
       if optimize_dataset:
         options = dataset_ops.Options()
+        options.experimental_optimization.apply_default_optimizations = False
         options.experimental_optimization.map_fusion = True
         dataset = dataset.with_options(options)
 
@@ -66,8 +67,6 @@ class OptimizationBenchmark(test.Benchmark):
 
         median_wall_time = np.median(deltas) / 100
         opt_mark = "opt" if optimize_dataset else "noopt"
-        print("Map dataset {} chain length: {} Median wall time: {}".format(
-            opt_mark, chain_length, median_wall_time))
         self.report_benchmark(
             iters=100,
             wall_time=median_wall_time,
@@ -90,6 +89,7 @@ class OptimizationBenchmark(test.Benchmark):
             lambda x: math_ops.greater_equal(x - 5, 0))
       if optimize_dataset:
         options = dataset_ops.Options()
+        options.experimental_optimization.apply_default_optimizations = False
         options.experimental_optimization.map_and_filter_fusion = True
         dataset = dataset.with_options(options)
       iterator = dataset_ops.make_one_shot_iterator(dataset)
@@ -108,8 +108,6 @@ class OptimizationBenchmark(test.Benchmark):
 
         median_wall_time = np.median(deltas) / 100
         opt_mark = "opt" if optimize_dataset else "noopt"
-        print("Map and filter dataset {} chain length: {} Median wall time: {}"
-              .format(opt_mark, chain_length, median_wall_time))
         self.report_benchmark(
             iters=100,
             wall_time=median_wall_time,
@@ -131,6 +129,7 @@ class OptimizationBenchmark(test.Benchmark):
         dataset = dataset.filter(lambda x: math_ops.greater_equal(x - 5, 0))
       if optimize_dataset:
         options = dataset_ops.Options()
+        options.experimental_optimization.apply_default_optimizations = False
         options.experimental_optimization.filter_fusion = True
         dataset = dataset.with_options(options)
 
@@ -150,8 +149,6 @@ class OptimizationBenchmark(test.Benchmark):
 
         median_wall_time = np.median(deltas) / 100
         opt_mark = "opt" if optimize_dataset else "no-opt"
-        print("Filter dataset {} chain length: {} Median wall time: {}".format(
-            opt_mark, chain_length, median_wall_time))
         self.report_benchmark(
             iters=1000,
             wall_time=median_wall_time,
diff --git a/tensorflow/python/data/experimental/benchmarks/parallel_interleave_benchmark.py b/tensorflow/python/data/experimental/benchmarks/parallel_interleave_benchmark.py
index f5108e5ff4..37375af27f 100644
--- a/tensorflow/python/data/experimental/benchmarks/parallel_interleave_benchmark.py
+++ b/tensorflow/python/data/experimental/benchmarks/parallel_interleave_benchmark.py
@@ -56,8 +56,10 @@ class ParallelInterleaveBenchmark(test.Benchmark):
 
   def _benchmark(self, dataset_fn, iters, num_elements):
     with ops.Graph().as_default():
-      iterator = dataset_ops.make_one_shot_iterator(dataset_fn())
-      next_element = iterator.get_next()
+      options = dataset_ops.Options()
+      options.experimental_optimization.apply_default_optimizations = False
+      dataset = dataset_fn().with_options(options)
+      next_element = dataset_ops.make_one_shot_iterator(dataset).get_next()
       with session.Session() as sess:
         deltas = []
         for _ in range(iters):
diff --git a/tensorflow/python/data/experimental/benchmarks/rejection_resample_benchmark.py b/tensorflow/python/data/experimental/benchmarks/rejection_resample_benchmark.py
index a64f7ecb00..9a8ac7ef65 100644
--- a/tensorflow/python/data/experimental/benchmarks/rejection_resample_benchmark.py
+++ b/tensorflow/python/data/experimental/benchmarks/rejection_resample_benchmark.py
@@ -39,6 +39,9 @@ def _time_resampling(data_np, target_dist, init_dist, num_to_sample):  # pylint:
           initial_dist=init_dist,
           seed=142))
 
+  options = dataset_ops.Options()
+  options.experimental_optimization.apply_default_optimizations = False
+  dataset = dataset.with_options(options)
   get_next = dataset_ops.make_one_shot_iterator(dataset).get_next()
 
   with session.Session() as sess:
diff --git a/tensorflow/python/data/experimental/benchmarks/unbatch_benchmark.py b/tensorflow/python/data/experimental/benchmarks/unbatch_benchmark.py
index 6f80df50b8..3f5b9b9130 100644
--- a/tensorflow/python/data/experimental/benchmarks/unbatch_benchmark.py
+++ b/tensorflow/python/data/experimental/benchmarks/unbatch_benchmark.py
@@ -42,6 +42,9 @@ class UnbatchBenchmark(test.Benchmark):
       dataset = dataset.batch(batch_size_placeholder)
       dataset = dataset.apply(batching.unbatch())
       dataset = dataset.skip(elems_per_trial)
+      options = dataset_ops.Options()
+      options.experimental_optimization.apply_default_optimizations = False
+      dataset = dataset.with_options(options)
       iterator = dataset_ops.make_initializable_iterator(dataset)
       next_element = iterator.get_next()
 
@@ -58,8 +61,6 @@ class UnbatchBenchmark(test.Benchmark):
             deltas.append((end - start) / elems_per_trial)
 
           median_wall_time = np.median(deltas)
-          print("Unbatch (native) batch size: %d Median wall time per element:"
-                " %f microseconds" % (batch_size, median_wall_time * 1e6))
           self.report_benchmark(
               iters=10000,
               wall_time=median_wall_time,
@@ -78,6 +79,9 @@ class UnbatchBenchmark(test.Benchmark):
       dataset = dataset.batch(batch_size_placeholder)
       dataset = dataset.flat_map(dataset_ops.Dataset.from_tensor_slices)
       dataset = dataset.skip(elems_per_trial)
+      options = dataset_ops.Options()
+      options.experimental_optimization.apply_default_optimizations = False
+      dataset = dataset.with_options(options)
       iterator = dataset_ops.make_initializable_iterator(dataset)
       next_element = iterator.get_next()
 
@@ -94,8 +98,6 @@ class UnbatchBenchmark(test.Benchmark):
             deltas.append((end - start) / elems_per_trial)
 
           median_wall_time = np.median(deltas)
-          print("Unbatch (unfused) batch size: %d Median wall time per element:"
-                " %f microseconds" % (batch_size, median_wall_time * 1e6))
           self.report_benchmark(
               iters=10000,
               wall_time=median_wall_time,
-- 
GitLab


From f602445c23feb67f2e35256ffdf24e090195fab6 Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Thu, 17 Jan 2019 12:59:15 -0800
Subject: [PATCH 0902/1095] [tf.data] Replacing IteratorStateMetadata with a
 simpler alternative.

Currently, serialization of iterator state uses the `IteratorStateMetadata` proto, which AFAICT is only used to convert a map to and from string. This CL replaces the proto-based conversion with a simpler mechanism. This change is motivated by making it possible to move the VariantTensorDataReader and VariantTensorDataWriter into a header file and depending on protos from TensorFlow header files is discouraged (c.f. b/62899350).

PiperOrigin-RevId: 229802166
---
 .../contrib/cmake/tf_core_framework.cmake     |  1 -
 tensorflow/core/BUILD                         |  1 -
 tensorflow/core/framework/iterator.proto      | 18 --------
 tensorflow/core/kernels/data/iterator_ops.cc  | 41 ++++++-------------
 4 files changed, 13 insertions(+), 48 deletions(-)
 delete mode 100644 tensorflow/core/framework/iterator.proto

diff --git a/tensorflow/contrib/cmake/tf_core_framework.cmake b/tensorflow/contrib/cmake/tf_core_framework.cmake
index 9ae5831f47..d8d1cc3aa2 100644
--- a/tensorflow/contrib/cmake/tf_core_framework.cmake
+++ b/tensorflow/contrib/cmake/tf_core_framework.cmake
@@ -147,7 +147,6 @@ set(tf_proto_text_srcs
     "tensorflow/core/framework/function.proto"
     "tensorflow/core/framework/graph.proto"
     "tensorflow/core/framework/graph_transfer_info.proto"
-    "tensorflow/core/framework/iterator.proto"
     "tensorflow/core/framework/kernel_def.proto"
     "tensorflow/core/framework/log_memory.proto"
     "tensorflow/core/framework/node_def.proto"
diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 6c9afc51e9..084db7a0fd 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -181,7 +181,6 @@ COMMON_PROTO_SRCS = [
     "framework/function.proto",
     "framework/graph.proto",
     "framework/graph_transfer_info.proto",
-    "framework/iterator.proto",
     "framework/kernel_def.proto",
     "framework/log_memory.proto",
     "framework/node_def.proto",
diff --git a/tensorflow/core/framework/iterator.proto b/tensorflow/core/framework/iterator.proto
deleted file mode 100644
index f015342e13..0000000000
--- a/tensorflow/core/framework/iterator.proto
+++ /dev/null
@@ -1,18 +0,0 @@
-syntax = "proto3";
-
-package tensorflow;
-option cc_enable_arenas = true;
-option java_outer_classname = "IteratorProtos";
-option java_multiple_files = true;
-option java_package = "org.tensorflow.util";
-option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework";
-
-// Protocol buffer representing the metadata for an iterator's state stored
-// as a Variant tensor.
-message IteratorStateMetadata {
-  // A user-specified version string.
-  string version = 1;
-
-  // Keys for tensors in the VariantTensorDataProto.
-  repeated string keys = 2;
-}
diff --git a/tensorflow/core/kernels/data/iterator_ops.cc b/tensorflow/core/kernels/data/iterator_ops.cc
index a8c3f7f77f..da497a5f72 100644
--- a/tensorflow/core/kernels/data/iterator_ops.cc
+++ b/tensorflow/core/kernels/data/iterator_ops.cc
@@ -20,7 +20,6 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/threadpool_device.h"
 #include "tensorflow/core/framework/function.h"
 #include "tensorflow/core/framework/function_handle_cache.h"
-#include "tensorflow/core/framework/iterator.pb.h"
 #include "tensorflow/core/framework/partial_tensor_shape.h"
 #include "tensorflow/core/framework/resource_op_kernel.h"
 #include "tensorflow/core/framework/stats_aggregator.h"
@@ -277,12 +276,19 @@ class IteratorResource : public ResourceBase {
 
 namespace {
 
+constexpr char kDelimiter[] = "@@";
+
 // Helper class for reading data from a VariantTensorData object.
 class VariantTensorDataReader : public IteratorStateReader {
  public:
   explicit VariantTensorDataReader(const VariantTensorData* data)
       : data_(data) {
-    PreProcess();
+    string metadata;
+    data_->get_metadata(&metadata);
+    auto keys = str_util::Split(metadata, kDelimiter, str_util::SkipEmpty());
+    for (size_t i = 0; i < keys.size(); ++i) {
+      map_[keys[i]] = i;
+    }
   }
 
   // Returns OK iff the initialization was successful, i.e.,
@@ -306,21 +312,6 @@ class VariantTensorDataReader : public IteratorStateReader {
   }
 
  private:
-  void PreProcess() {
-    string metadata;
-    data_->get_metadata(&metadata);
-    IteratorStateMetadata proto;
-    if (!proto.ParseFromString(metadata)) {
-      status_ = errors::Internal("Error parsing IteratorStateMetadata.");
-      return;
-    }
-    size_t num_entries = proto.keys_size();
-    CHECK_EQ(num_entries, data_->tensors_size());
-    for (size_t i = 0; i < num_entries; i++) {
-      map_[proto.keys(i)] = i;
-    }
-  }
-
   template <typename T>
   Status ReadScalarInternal(StringPiece key, T* val) {
     if (map_.find(string(key)) == map_.end()) {
@@ -361,11 +352,10 @@ class VariantTensorDataWriter : public IteratorStateWriter {
     return WriteTensorInternal(key, val);
   }
 
-  // Writes the metadata to `data_`.
   Status Flush() {
     string metadata;
-    if (!metadata_proto_.SerializeToString(&metadata)) {
-      return errors::Internal("Unable to serialize IteratorStateMetadata.");
+    for (size_t i = 0; i < keys_.size(); ++i) {
+      strings::StrAppend(&metadata, kDelimiter, keys_[i]);
     }
     data_->set_metadata(metadata);
     return Status::OK();
@@ -380,19 +370,14 @@ class VariantTensorDataWriter : public IteratorStateWriter {
   }
 
   Status WriteTensorInternal(StringPiece key, const Tensor& val) {
-    // Write key to the metadata proto. This gets written to `data_`
-    // when `Flush()` is called. We do this lazily to avoid multiple
-    // serialization calls.
-    metadata_proto_.add_keys(string(key));
-
-    // Update tensors.
+    DCHECK_EQ(key.find(kDelimiter), string::npos);
+    keys_.push_back(string(key));
     *(data_->add_tensors()) = val;
     return Status::OK();
   }
 
   VariantTensorData* data_;
-  // TODO(srbs): Set the version string.
-  IteratorStateMetadata metadata_proto_;
+  std::vector<string> keys_;
 };
 
 // Wrapper for encoding/decoding the iterator state stored in a Variant tensor.
-- 
GitLab


From 1757ebeaa21b356b954fa8f54e1d0796a8e60dc6 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Thu, 17 Jan 2019 13:01:06 -0800
Subject: [PATCH 0903/1095] Add oss-serial tag to elastic_optimizer_test

This requires allocating many ports, and attaching to them later.
Even with portserver, this is a flaky action when multiple tests execute on the same environment.

PiperOrigin-RevId: 229802480
---
 tensorflow/contrib/opt/BUILD | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/contrib/opt/BUILD b/tensorflow/contrib/opt/BUILD
index 0446e823d9..12320d9e45 100644
--- a/tensorflow/contrib/opt/BUILD
+++ b/tensorflow/contrib/opt/BUILD
@@ -319,6 +319,9 @@ tf_py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
         "//third_party/py/numpy",
     ],
+    tags = [
+        "oss_serial",
+    ],
 )
 
 tf_py_test(
-- 
GitLab


From 26cb1cd0e402b179650bba742caa4796f12da963 Mon Sep 17 00:00:00 2001
From: Suharsh Sivakumar <suharshs@google.com>
Date: Thu, 17 Jan 2019 13:10:15 -0800
Subject: [PATCH 0904/1095] Add support for per channel quantization to C API.

Also add future proof quantization params structure.

PiperOrigin-RevId: 229804132
---
 tensorflow/lite/c/c_api_internal.c       | 37 ++++++++++++
 tensorflow/lite/c/c_api_internal.h       | 71 ++++++++++++++++++++++--
 tensorflow/lite/c/c_api_internal_test.cc | 21 +++++++
 3 files changed, 124 insertions(+), 5 deletions(-)

diff --git a/tensorflow/lite/c/c_api_internal.c b/tensorflow/lite/c/c_api_internal.c
index 2923dbad4e..29dba15c63 100644
--- a/tensorflow/lite/c/c_api_internal.c
+++ b/tensorflow/lite/c/c_api_internal.c
@@ -70,6 +70,20 @@ TfLiteIntArray* TfLiteIntArrayCopy(const TfLiteIntArray* src) {
 
 void TfLiteIntArrayFree(TfLiteIntArray* a) { free(a); }
 
+int TfLiteFloatArrayGetSizeInBytes(int size) {
+  static TfLiteFloatArray dummy;
+  return sizeof(dummy) + sizeof(dummy.data[0]) * size;
+}
+
+TfLiteFloatArray* TfLiteFloatArrayCreate(int size) {
+  TfLiteFloatArray* ret =
+      (TfLiteFloatArray*)malloc(TfLiteFloatArrayGetSizeInBytes(size));
+  ret->size = size;
+  return ret;
+}
+
+void TfLiteFloatArrayFree(TfLiteFloatArray* a) { free(a); }
+
 void TfLiteTensorDataFree(TfLiteTensor* t) {
   if (t->allocation_type == kTfLiteDynamic && t->data.raw) {
     free(t->data.raw);
@@ -77,10 +91,30 @@ void TfLiteTensorDataFree(TfLiteTensor* t) {
   t->data.raw = NULL;
 }
 
+void TfLiteQuantizationFree(TfLiteTensor* t) {
+  if (t->quantization.type == kTfLiteAffineQuantization) {
+    TfLiteAffineQuantization* q_params =
+        (TfLiteAffineQuantization*)(t->quantization.params);
+    if (q_params->scale) {
+      TfLiteFloatArrayFree(q_params->scale);
+      q_params->scale = NULL;
+    }
+    if (q_params->zero_point) {
+      TfLiteIntArrayFree(q_params->zero_point);
+      q_params->zero_point = NULL;
+    }
+    free(q_params);
+  }
+  t->quantization.params = NULL;
+  t->quantization.type = kTfLiteNoQuantization;
+}
+
 void TfLiteTensorFree(TfLiteTensor* t) {
   TfLiteTensorDataFree(t);
   if (t->dims) TfLiteIntArrayFree(t->dims);
   t->dims = NULL;
+
+  TfLiteQuantizationFree(t);
 }
 
 void TfLiteTensorReset(TfLiteType type, const char* name, TfLiteIntArray* dims,
@@ -98,6 +132,9 @@ void TfLiteTensorReset(TfLiteType type, const char* name, TfLiteIntArray* dims,
   tensor->allocation_type = allocation_type;
   tensor->allocation = allocation;
   tensor->is_variable = is_variable;
+
+  tensor->quantization.type = kTfLiteNoQuantization;
+  tensor->quantization.params = NULL;
 }
 
 void TfLiteTensorRealloc(size_t num_bytes, TfLiteTensor* tensor) {
diff --git a/tensorflow/lite/c/c_api_internal.h b/tensorflow/lite/c/c_api_internal.h
index 1b1bc6db8f..31f483370c 100644
--- a/tensorflow/lite/c/c_api_internal.h
+++ b/tensorflow/lite/c/c_api_internal.h
@@ -98,8 +98,32 @@ int TfLiteIntArrayEqualsArray(TfLiteIntArray* a, int b_size, int b_data[]);
 // You are expected to free memory with TfLiteIntArrayFree
 TfLiteIntArray* TfLiteIntArrayCopy(const TfLiteIntArray* src);
 
-// Free memory of array `v`.
-void TfLiteIntArrayFree(TfLiteIntArray* v);
+// Free memory of array `a`.
+void TfLiteIntArrayFree(TfLiteIntArray* a);
+
+// Fixed size list of floats. Used for per-channel quantization.
+typedef struct {
+  int size;
+// gcc 6.1+ have a bug where flexible members aren't properly handled
+// https://github.com/google/re2/commit/b94b7cd42e9f02673cd748c1ac1d16db4052514c
+#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ == 6 && \
+    __GNUC_MINOR__ >= 1
+  float data[0];
+#else
+  float data[];
+#endif
+} TfLiteFloatArray;
+
+// Given the size (number of elements) in a TfLiteFloatArray, calculate its size
+// in bytes.
+int TfLiteFloatArrayGetSizeInBytes(int size);
+
+// Create a array of a given `size` (uninitialized entries).
+// This returns a pointer, that you must free using TfLiteFloatArrayFree().
+TfLiteFloatArray* TfLiteFloatArrayCreate(int size);
+
+// Free memory of array `a`.
+void TfLiteFloatArrayFree(TfLiteFloatArray* a);
 
 // Since we must not depend on any libraries, define a minimal subset of
 // error macros while avoiding names that have pre-conceived meanings like
@@ -185,14 +209,48 @@ typedef enum {
 // Return the name of a given type, for error reporting purposes.
 const char* TfLiteTypeGetName(TfLiteType type);
 
+// SupportedQuantizationTypes.
+typedef enum {
+  // No quantization.
+  kTfLiteNoQuantization = 0,
+  // Affine quantization (with support for per-channel quantization).
+  // Corresponds to TfLiteAffineQuantization.
+  kTfLiteAffineQuantization = 1,
+} TfLiteQuantizationType;
+
+// Structure specifying the quantization used by the tensor, if-any.
+typedef struct {
+  // The type of quantization held by params.
+  TfLiteQuantizationType type;
+  // Holds a reference to one of the quantization param structures specified
+  // below.
+  void* params;
+} TfLiteQuantization;
+
+// Legacy. Will be deprecated in favor of TfLiteAffineQuantization.
+// If per-layer quantization is specified this field will still be populated in
+// addition to TfLiteAffineQuantization.
 // Parameters for asymmetric quantization. Quantized values can be converted
 // back to float using:
-//    real_value = scale * (quantized_value - zero_point);
+//     real_value = scale * (quantized_value - zero_point)
 typedef struct {
   float scale;
   int32_t zero_point;
 } TfLiteQuantizationParams;
 
+// Parameters for asymmetric quantization across a dimension (i.e per output
+// channel quantization).
+// quantized_dimension specifies which dimension the scales and zero_points
+// correspond to.
+// For a particular value in quantized_dimension, quantized values can be
+// converted back to float using:
+//     real_value = scale * (quantized_value - zero_point)
+typedef struct {
+  TfLiteFloatArray* scale;
+  TfLiteIntArray* zero_point;
+  int32_t quantized_dimension;
+} TfLiteAffineQuantization;
+
 // A union of pointers that points to memory for a given tensor.
 typedef union {
   int* i32;
@@ -274,12 +332,15 @@ typedef struct {
 
   // True if the tensor is a variable.
   bool is_variable;
+
+  // Quantization information. Replaces params field above.
+  TfLiteQuantization quantization;
 } TfLiteTensor;
 
-// Free data memory of tensor `t`;
+// Free data memory of tensor `t`.
 void TfLiteTensorDataFree(TfLiteTensor* t);
 
-// Free memory of tensor `t`;
+// Free memory of tensor `t`.
 void TfLiteTensorFree(TfLiteTensor* t);
 
 // Set all of a tensor's fields (and free any previously allocated data).
diff --git a/tensorflow/lite/c/c_api_internal_test.cc b/tensorflow/lite/c/c_api_internal_test.cc
index acf0dfc5be..d01cf63a3e 100644
--- a/tensorflow/lite/c/c_api_internal_test.cc
+++ b/tensorflow/lite/c/c_api_internal_test.cc
@@ -65,6 +65,13 @@ TEST(IntArray, TestIntArrayEqual) {
   TfLiteIntArrayFree(d);
 }
 
+TEST(FloatArray, TestFloatArrayCreate) {
+  TfLiteFloatArray* a = TfLiteFloatArrayCreate(0);
+  TfLiteFloatArray* b = TfLiteFloatArrayCreate(3);
+  TfLiteFloatArrayFree(a);
+  TfLiteFloatArrayFree(b);
+}
+
 TEST(Types, TestTypeNames) {
   auto type_name = [](TfLiteType t) {
     return std::string(TfLiteTypeGetName(t));
@@ -81,6 +88,20 @@ TEST(Types, TestTypeNames) {
   EXPECT_EQ(type_name(kTfLiteString), "STRING");
 }
 
+TEST(Quantization, TestQuantizationFree) {
+  TfLiteTensor t;
+  // Set these values, otherwise TfLiteTensorFree has uninitialized values.
+  t.allocation_type = kTfLiteArenaRw;
+  t.dims = nullptr;
+  t.quantization.type = kTfLiteAffineQuantization;
+  auto* params = reinterpret_cast<TfLiteAffineQuantization*>(
+      malloc(sizeof(TfLiteAffineQuantization)));
+  params->scale = TfLiteFloatArrayCreate(3);
+  params->zero_point = TfLiteIntArrayCreate(3);
+  t.quantization.params = reinterpret_cast<void*>(params);
+  TfLiteTensorFree(&t);
+}
+
 }  // namespace tflite
 
 int main(int argc, char** argv) {
-- 
GitLab


From f132194cf910343d4e1c3aedfad45f7b76fc5f61 Mon Sep 17 00:00:00 2001
From: Russell Power <power@google.com>
Date: Thu, 17 Jan 2019 13:24:19 -0800
Subject: [PATCH 0905/1095] SessionRef: add a timeout to avoid blocking
 Session::Close() forever.

Some operations/sub-systems do not respect TF cancellation operations.  This
can result in Session:Close() hanging when waiting for these operations to
finish.

This CL adds a default timeout of 60 seconds to ensure we always return control
back to the client eventually.

PiperOrigin-RevId: 229806493
---
 tensorflow/python/BUILD                 |  1 +
 tensorflow/python/client/session_ref.cc | 31 +++++++++++++++++++++----
 2 files changed, 28 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index e0648085c3..d17c57b05f 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -4067,6 +4067,7 @@ cc_library(
         "//tensorflow/core:master_proto_cc",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:replay_log_proto_cc",
+        "@com_google_absl//absl/time",
     ],
 )
 
diff --git a/tensorflow/python/client/session_ref.cc b/tensorflow/python/client/session_ref.cc
index 6639cf506e..895624f592 100644
--- a/tensorflow/python/client/session_ref.cc
+++ b/tensorflow/python/client/session_ref.cc
@@ -18,6 +18,9 @@ limitations under the License.
 #include <memory>
 #include <utility>
 
+#include "absl/time/clock.h"
+#include "absl/time/time.h"
+#include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/io/path.h"
 #include "tensorflow/core/lib/io/record_writer.h"
 #include "tensorflow/core/lib/strings/stringprintf.h"
@@ -477,6 +480,8 @@ Status SessionRef::ReleaseCallable(CallableHandle handle) {
   LOG_AND_RUN_OPERATION(ReleaseCallable, handle);
 }
 
+static const absl::Duration kMaxCloseWaitTime = absl::Seconds(60);
+
 Status SessionRef::Close(const RunOptions& run_options) {
   TF_RETURN_IF_ERROR(CheckNotClosed());
   mutex_lock l(run_lock_);
@@ -487,8 +492,17 @@ Status SessionRef::Close(const RunOptions& run_options) {
     status = session_->Close(run_options);
   }
   session_.reset();
-  while (run_count_ > 0) {
-    run_finished_.wait(l);
+
+  // Wait no more than kMaxCloseWaitTime for all pending operations to finish.
+  absl::Time start_time = absl::Now();
+  absl::Duration wait_time = start_time + kMaxCloseWaitTime - absl::Now();
+  while (run_count_ > 0 && wait_time > absl::ZeroDuration()) {
+    if (run_finished_.wait_for(l, absl::ToChronoMilliseconds(wait_time)) ==
+        std::cv_status::timeout) {
+      status.Update(errors::DeadlineExceeded("Timeout closing session."));
+      return status;
+    }
+    wait_time = start_time + kMaxCloseWaitTime - absl::Now();
   }
   return status;
 }
@@ -503,8 +517,17 @@ Status SessionRef::Close() {
     status = session_->Close();
   }
   session_.reset();
-  while (run_count_ > 0) {
-    run_finished_.wait(l);
+
+  // Wait no more than kMaxCloseWaitTime for all pending operations to finish.
+  absl::Time start_time = absl::Now();
+  absl::Duration wait_time = start_time + kMaxCloseWaitTime - absl::Now();
+  while (run_count_ > 0 && wait_time > absl::ZeroDuration()) {
+    if (run_finished_.wait_for(l, absl::ToChronoMilliseconds(wait_time)) ==
+        std::cv_status::timeout) {
+      status.Update(errors::DeadlineExceeded("Timeout closing session."));
+      return status;
+    }
+    wait_time = start_time + kMaxCloseWaitTime - absl::Now();
   }
   return status;
 }
-- 
GitLab


From 8879131afb5e0470279a838f4b47bb969a361cf2 Mon Sep 17 00:00:00 2001
From: Mingming Liu <mingmingl@google.com>
Date: Thu, 17 Jan 2019 13:30:56 -0800
Subject: [PATCH 0906/1095] Use Sampler to record warm up latency distribution
 (by status).

PiperOrigin-RevId: 229807679
---
 tensorflow/cc/saved_model/loader.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/cc/saved_model/loader.cc b/tensorflow/cc/saved_model/loader.cc
index aaf06a74c6..10f7abf09e 100644
--- a/tensorflow/cc/saved_model/loader.cc
+++ b/tensorflow/cc/saved_model/loader.cc
@@ -52,8 +52,8 @@ auto* load_latency_by_stage = monitoring::Sampler<2>::New(
         "model_path",
         "stage",
     },
-    // Scale of 10, power of 1.5 with bucket count 36 (~20 minutes).
-    monitoring::Buckets::Exponential(10, 1.5, 36));
+    // Scale of 10, power of 1.8 with bucket count 33 (~20 minutes).
+    monitoring::Buckets::Exponential(10, 1.8, 33));
 
 constexpr char kLoadAttemptFail[] = "fail";
 constexpr char kLoadAttemptSuccess[] = "success";
-- 
GitLab


From 3fa896d2aa9f67f88c825c3a05eff2cf01b5546a Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Thu, 17 Jan 2019 13:33:33 -0800
Subject: [PATCH 0907/1095] Revert the changes to TRTEngineOp's OpDef, so
 existing models won't get broken.

PiperOrigin-RevId: 229808206
---
 .../contrib/tensorrt/convert/convert_graph.cc | 12 +++++-----
 .../contrib/tensorrt/convert/convert_graph.h  |  5 ++--
 .../contrib/tensorrt/convert/convert_nodes.h  |  2 +-
 .../tensorrt/convert/trt_optimization_pass.cc |  6 ++---
 .../contrib/tensorrt/kernels/trt_engine_op.cc | 17 +++++++------
 .../contrib/tensorrt/kernels/trt_engine_op.h  |  2 +-
 .../contrib/tensorrt/ops/trt_engine_op.cc     |  5 +++-
 .../contrib/tensorrt/python/trt_convert.py    | 24 +++++++++----------
 .../tensorrt/python/trt_convert_test.py       |  7 +++---
 .../contrib/tensorrt/test/test_tftrt.py       |  6 ++---
 .../test/tf_trt_integration_test_base.py      |  8 +++----
 11 files changed, 47 insertions(+), 47 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index 87c2d367f8..e2350f69a6 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -239,7 +239,7 @@ tensorflow::Status ConvertGraphDefToTensorRT(
     const std::vector<string>& output_names, size_t max_batch_size,
     size_t max_workspace_size_bytes, tensorflow::GraphDef* new_graph_def,
     int precision_mode, int minimum_segment_size, bool is_dyn_op,
-    int max_cached_engines, std::vector<int> cached_engine_batch_sizes,
+    int max_cached_engines, std::vector<int> cached_engine_batches,
     bool use_calibration) {
   // Create GrapplerItem.
   tensorflow::grappler::GrapplerItem item;
@@ -301,9 +301,9 @@ tensorflow::Status ConvertGraphDefToTensorRT(
   TF_RETURN_IF_ERROR(GetPrecisionModeName(
       precision_mode, parameters["precision_mode"].mutable_s()));
   parameters["maximum_cached_engines"].set_i(max_cached_engines);
-  if (!cached_engine_batch_sizes.empty()) {
-    auto list = parameters["cached_engine_batch_sizes"].mutable_list();
-    for (const int batch : cached_engine_batch_sizes) {
+  if (!cached_engine_batches.empty()) {
+    auto list = parameters["cached_engine_batches"].mutable_list();
+    for (const int batch : cached_engine_batches) {
       list->add_i(batch);
     }
   }
@@ -689,7 +689,7 @@ tensorflow::Status CreateTRTNode(const std::vector<EngineInfo>& infos, int pos,
   }
 
   if (info.engine_type == EngineInfo::EngineType::TRTStatic &&
-      !info.cached_engine_batch_sizes.empty()) {
+      !info.cached_engine_batches.empty()) {
     LOG(WARNING) << "Cached engine batches are ignored for static engines";
   }
   tensorflow::NodeDef trt_node;
@@ -983,7 +983,7 @@ tensorflow::Status ConvertAfterShapes(ConversionParams& params) {
                                    ? EngineInfo::EngineType::TRTDynamic
                                    : EngineInfo::EngineType::TRTStatic);
     curr_engine.use_calibration = params.use_calibration;
-    curr_engine.cached_engine_batch_sizes = params.cached_engine_batch_sizes;
+    curr_engine.cached_engine_batches = params.cached_engine_batches;
     curr_engine.maximum_cached_engines = params.max_cached_engines;
     StrAppend(&curr_engine.engine_name, "TRTEngineOp_", t);
     status = RegisterSegmentFunctionToFunctionLibrary(
diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.h b/tensorflow/contrib/tensorrt/convert/convert_graph.h
index 5616bbefd9..1f39f56f63 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.h
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.h
@@ -82,7 +82,7 @@ struct ConversionParams {
   bool fixed_input_size;   // Assume non-batch ranks of input tensors are fixed
   int max_cached_engines;  // maximum number of cached engines
   bool use_calibration;
-  std::vector<int> cached_engine_batch_sizes;  // list of cached engines
+  std::vector<int> cached_engine_batches;  // list of cached engines
 };
 
 // This method extracts calibration information from the resource managers
@@ -101,8 +101,7 @@ tensorflow::Status ConvertGraphDefToTensorRT(
     size_t max_workspace_size_bytes, tensorflow::GraphDef* new_graph_def,
     int precision_mode = 1, int minimum_segment_size = 3,
     bool is_dyn_op = false, int max_cached_engines = 1,
-    std::vector<int> cached_engine_batch_sizes = {},
-    bool use_calibration = true);
+    std::vector<int> cached_engine_batches = {}, bool use_calibration = true);
 
 // Method to call from optimization pass
 tensorflow::Status ConvertAfterShapes(ConversionParams& params);
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.h b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
index a02a52e430..4ea5775f04 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.h
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
@@ -108,7 +108,7 @@ struct EngineInfo {
   EngineType engine_type;
   int64 max_workspace_size_bytes;
   int maximum_cached_engines;
-  std::vector<int> cached_engine_batch_sizes;
+  std::vector<int> cached_engine_batches;
   int precision_mode;
   bool use_calibration;
 };
diff --git a/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc b/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc
index e7ca126f55..d57f2300f8 100644
--- a/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc
+++ b/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc
@@ -50,8 +50,8 @@ tensorflow::Status TRTOptimizationPass::Init(
   if (params.count("is_dynamic_op")) {
     is_dynamic_op_ = params.at("is_dynamic_op").b();
   }
-  if (params.count("cached_engine_batch_sizes")) {
-    auto batch_vec = params.at("cached_engine_batch_sizes").list();
+  if (params.count("cached_engine_batches")) {
+    auto batch_vec = params.at("cached_engine_batches").list();
     batches_.reserve(batch_vec.i_size());
     for (const auto i : batch_vec.i()) {
       batches_.push_back(i);
@@ -258,7 +258,7 @@ tensorflow::Status TRTOptimizationPass::Optimize(
   cp.graph_properties = &static_graph_properties;
   cp.cluster = cluster;
   cp.is_dyn_op = is_dynamic_op_;
-  cp.cached_engine_batch_sizes = batches_;
+  cp.cached_engine_batches = batches_;
   cp.max_cached_engines = max_cached_batches_;
   cp.use_calibration = use_calibration_;
   auto status = tensorflow::tensorrt::convert::ConvertAfterShapes(cp);
diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
index f089f2c1cc..7548c8ccda 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
+++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
@@ -136,13 +136,12 @@ TRTEngineOp::TRTEngineOp(OpKernelConstruction* context)
   native_func_ = tensorflow::kInvalidHandle;
   OP_REQUIRES_OK(context, context->GetAttr("max_cached_engines_count",
                                            &max_cached_engines_));
-  OP_REQUIRES_OK(context, context->GetAttr("cached_engine_batch_sizes",
-                                           &cached_engine_batch_sizes_));
-  std::sort(cached_engine_batch_sizes_.begin(),
-            cached_engine_batch_sizes_.end());
+  OP_REQUIRES_OK(context, context->GetAttr("cached_engine_batches",
+                                           &cached_engine_batches_));
+  std::sort(cached_engine_batches_.begin(), cached_engine_batches_.end());
   if (VLOG_IS_ON(1)) {
     string s("Engine Batches= ");
-    for (auto i : cached_engine_batch_sizes_) {
+    for (auto i : cached_engine_batches_) {
       StrAppend(&s, i, " ");
     }
     VLOG(1) << s;
@@ -247,7 +246,7 @@ bool TRTEngineOp::GetCompatibleCachedEngine(
   // Output shape will always be the same as the input but we will overwrite the
   // batch size.
   *engine_input_shapes = actual_input_shapes;
-  for (const int cached_batch_size : cached_engine_batch_sizes_) {
+  for (const int cached_batch_size : cached_engine_batches_) {
     // Check if compatible: batch <= cached batch.
     //
     // TODO(laigd): here it only compare the first dim a.k.a the batch size,
@@ -495,12 +494,12 @@ EngineContext* TRTEngineOp::GetEngine(
   // exact shape and possibly create a new engine if it is not in cache.
   if (!matched_successfully) {
     engine_input_shapes = input_shapes;
-    if (!cached_engine_batch_sizes_.empty()) {
-      // If user has explicitly defined cached_engine_batch_sizes, we should
+    if (!cached_engine_batches_.empty()) {
+      // If user has explicitly defined cached_engine_batches, we should
       // warn them that their input was non-compatible (batch size too high)
       LOG(WARNING) << "No compatible cached engine was found for batch size: "
                    << batch_size << ". A new engine will be created.";
-      cached_engine_batch_sizes_.push_back(batch_size);
+      cached_engine_batches_.push_back(batch_size);
     }
   }
 
diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
index c9b325cfd2..7f0f05aa0a 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
+++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
@@ -104,7 +104,7 @@ class TRTEngineOp : public AsyncOpKernel {
   bool calibration_mode_;
 
   // Batches of the cached engines
-  std::vector<int> cached_engine_batch_sizes_;
+  std::vector<int> cached_engine_batches_;
 
   // Maximum number of cached engines
   int max_cached_engines_;
diff --git a/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc b/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc
index 73cc046bc3..b84d2fe0b8 100644
--- a/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc
+++ b/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc
@@ -28,6 +28,9 @@ namespace shape_inference {
 extern Status TRTEngineOpShapeInference(InferenceContext* c);
 }
 
+// NOTE: please try NOT to add/modify/remove attributes or inputs/outputs to the
+// list below, this will break backward compatibility!
+//
 // TODO(laigd): consider making this op stateful. The only problem is it uses TF
 // function which has to be stateless, but we can use function library as the
 // key to cache the instantiated functions for different executor subgraphs.
@@ -40,7 +43,7 @@ REGISTER_OP("TRTEngineOp")
     .Attr("OutT: list({int8,float16,float32,int32})")
     .Attr("static_engine: bool = true")
     .Attr("fixed_input_size: bool = true")
-    .Attr("cached_engine_batch_sizes: list(int) >= 0 = []")
+    .Attr("cached_engine_batches: list(int) >= 0 = []")
     .Attr("max_cached_engines_count: int = 1")
     .Attr("workspace_size_bytes: int")
     .Attr("precision_mode: {'FP32', 'FP16', 'INT8'}")
diff --git a/tensorflow/contrib/tensorrt/python/trt_convert.py b/tensorflow/contrib/tensorrt/python/trt_convert.py
index 8de17347f8..49d72232aa 100644
--- a/tensorflow/contrib/tensorrt/python/trt_convert.py
+++ b/tensorflow/contrib/tensorrt/python/trt_convert.py
@@ -77,7 +77,7 @@ def get_tensorrt_rewriter_config(rewriter_config=None,
                                  minimum_segment_size=3,
                                  is_dynamic_op=False,
                                  maximum_cached_engines=1,
-                                 cached_engine_batch_sizes=None,
+                                 cached_engine_batches=None,
                                  use_calibration=True):
   """Returns a RewriterConfig proto for TRT transformation.
 
@@ -97,7 +97,7 @@ def get_tensorrt_rewriter_config(rewriter_config=None,
       If the number of cached engines is already at max but none of them can
       serve the input, the TRTEngineOp will fall back to run the TF function
       based on which the TRTEngineOp is created.
-    cached_engine_batch_sizes: a list of batch sizes used to create cached
+    cached_engine_batches: a list of batch sizes used to create cached
       engines, only used when is_dynamic_op is True. The length of the list
       should be <= maximum_cached_engines, and the dynamic TRT op will
       use this list to determine the batch sizes of the cached engines, instead
@@ -150,14 +150,14 @@ def get_tensorrt_rewriter_config(rewriter_config=None,
       "max_workspace_size_bytes"].i = max_workspace_size_bytes
   optimizer.parameter_map["precision_mode"].s = _to_bytes(precision_mode)
   optimizer.parameter_map["maximum_cached_engines"].i = maximum_cached_engines
-  if cached_engine_batch_sizes:
-    if not isinstance(cached_engine_batch_sizes, list):
-      raise TypeError("cached_engine_batch_sizes should be a list.")
-    if len(cached_engine_batch_sizes) > maximum_cached_engines:
-      raise ValueError("cached_engine_batch_sizes should not contain more than "
+  if cached_engine_batches:
+    if not isinstance(cached_engine_batches, list):
+      raise TypeError("cached_engine_batches should be a list.")
+    if len(cached_engine_batches) > maximum_cached_engines:
+      raise ValueError("cached_engine_batches should not contain more than "
                        "maximum_cached_engines items.")
-    optimizer.parameter_map["cached_engine_batch_sizes"].list.i.extend(
-        cached_engine_batch_sizes)
+    optimizer.parameter_map["cached_engine_batches"].list.i.extend(
+        cached_engine_batches)
   optimizer.parameter_map["use_calibration"].b = use_calibration
   return rewriter_config_with_trt
 
@@ -170,7 +170,7 @@ def create_inference_graph(input_graph_def,
                            minimum_segment_size=3,
                            is_dynamic_op=False,
                            maximum_cached_engines=1,
-                           cached_engine_batch_sizes=None,
+                           cached_engine_batches=None,
                            use_calibration=True,
                            input_saved_model_dir=None,
                            input_saved_model_tags=None,
@@ -197,7 +197,7 @@ def create_inference_graph(input_graph_def,
       If the number of cached engines is already at max but none of them can
       serve the input, the TRTEngineOp will fall back to run the TF function
       based on which the TRTEngineOp is created.
-    cached_engine_batch_sizes: a list of batch sizes used to create cached
+    cached_engine_batches: a list of batch sizes used to create cached
       engines, only used when is_dynamic_op is True. The length of the list
       should be <= maximum_cached_engines, and the dynamic TRT op will
       use this list to determine the batch sizes of the cached engines, instead
@@ -361,7 +361,7 @@ def create_inference_graph(input_graph_def,
   rewriter_config_with_trt = get_tensorrt_rewriter_config(
       rewriter_config, max_batch_size, max_workspace_size_bytes, precision_mode,
       minimum_segment_size, is_dynamic_op, maximum_cached_engines,
-      cached_engine_batch_sizes, use_calibration)
+      cached_engine_batches, use_calibration)
   session_config_with_trt.graph_options.rewrite_options.CopyFrom(
       rewriter_config_with_trt)
 
diff --git a/tensorflow/contrib/tensorrt/python/trt_convert_test.py b/tensorflow/contrib/tensorrt/python/trt_convert_test.py
index 9562704aeb..3ef18e3e15 100644
--- a/tensorflow/contrib/tensorrt/python/trt_convert_test.py
+++ b/tensorflow/contrib/tensorrt/python/trt_convert_test.py
@@ -57,7 +57,7 @@ class TrtConvertTest(test_util.TensorFlowTestCase):
         minimum_segment_size=10,
         is_dynamic_op=True,
         maximum_cached_engines=2,
-        cached_engine_batch_sizes=[1, 128])
+        cached_engine_batches=[1, 128])
     self.assertEqual(["constfold", "layout", "constfold"],
                      rewriter_cfg.optimizers)
     self.assertEqual(rewriter_config_pb2.RewriterConfig.ONE,
@@ -71,7 +71,7 @@ class TrtConvertTest(test_util.TensorFlowTestCase):
     for key in [
         "minimum_segment_size", "max_batch_size", "is_dynamic_op",
         "max_workspace_size_bytes", "precision_mode", "maximum_cached_engines",
-        "cached_engine_batch_sizes"
+        "cached_engine_batches"
     ]:
       self.assertTrue(key in trt_optimizer.parameter_map)
     self.assertEqual(10, trt_optimizer.parameter_map["minimum_segment_size"].i)
@@ -84,8 +84,7 @@ class TrtConvertTest(test_util.TensorFlowTestCase):
         trt_optimizer.parameter_map["precision_mode"].s)
     self.assertEqual(2, trt_optimizer.parameter_map["maximum_cached_engines"].i)
     self.assertEqual(
-        [1, 128],
-        trt_optimizer.parameter_map["cached_engine_batch_sizes"].list.i)
+        [1, 128], trt_optimizer.parameter_map["cached_engine_batches"].list.i)
 
   def _GetConfigProto(self):
     """Get ConfigProto for session creation."""
diff --git a/tensorflow/contrib/tensorrt/test/test_tftrt.py b/tensorflow/contrib/tensorrt/test/test_tftrt.py
index d26f260086..090aa8bdb0 100644
--- a/tensorflow/contrib/tensorrt/test/test_tftrt.py
+++ b/tensorflow/contrib/tensorrt/test/test_tftrt.py
@@ -191,7 +191,7 @@ def user(multi_engine,
       minimum_segment_size=2,  # minimum number of nodes in an engine
       is_dynamic_op=False,
       maximum_cached_engines=1,
-      cached_engine_batch_sizes=[])
+      cached_engine_batches=[])
   o1 = run_graph(orig_graph, dummy_input)
   o2 = run_graph(trt_graph, dummy_input)
   o3 = run_graph(trt_graph, dummy_input)
@@ -206,7 +206,7 @@ def user(multi_engine,
       minimum_segment_size=2,  # minimum number of nodes in an engine
       is_dynamic_op=False,
       maximum_cached_engines=1,
-      cached_engine_batch_sizes=[])
+      cached_engine_batches=[])
   int8_calib_gdef = trt.create_inference_graph(
       input_graph_def=orig_graph,
       outputs=["output"],
@@ -216,7 +216,7 @@ def user(multi_engine,
       minimum_segment_size=2,  # minimum number of nodes in an engine
       is_dynamic_op=False,
       maximum_cached_engines=1,
-      cached_engine_batch_sizes=[])
+      cached_engine_batches=[])
   o4 = run_graph(fp16_graph, dummy_input)
   _ = run_calibration(int8_calib_gdef, dummy_input)
   int8_graph = trt.calib_graph_to_infer_graph(int8_calib_gdef)
diff --git a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
index 4f47f80e73..db43581404 100644
--- a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
+++ b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
@@ -61,7 +61,7 @@ RunParams = namedtuple("RunParams", [
 ConversionParams = namedtuple("ConversionParams", [
     "max_batch_size", "max_workspace_size_bytes", "precision_mode",
     "minimum_segment_size", "is_dynamic_op", "maximum_cached_engines",
-    "cached_engine_batch_sizes", "rewriter_config", "use_calibration"
+    "cached_engine_batches", "rewriter_config", "use_calibration"
 ])
 
 PRECISION_MODES = ["FP32", "FP16", "INT8"]
@@ -186,7 +186,7 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
         minimum_segment_size=2,
         is_dynamic_op=run_params.dynamic_engine,
         maximum_cached_engines=1,
-        cached_engine_batch_sizes=None,
+        cached_engine_batches=None,
         rewriter_config=None,
         use_calibration=run_params.use_calibration)
 
@@ -266,7 +266,7 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
           conversion_params.minimum_segment_size,
           conversion_params.is_dynamic_op,
           conversion_params.maximum_cached_engines,
-          conversion_params.cached_engine_batch_sizes,
+          conversion_params.cached_engine_batches,
           conversion_params.use_calibration)
 
       graph_options = config_pb2.GraphOptions(rewrite_options=rewriter_cfg)
@@ -369,7 +369,7 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
         minimum_segment_size=conversion_params.minimum_segment_size,
         is_dynamic_op=conversion_params.is_dynamic_op,
         maximum_cached_engines=conversion_params.maximum_cached_engines,
-        cached_engine_batch_sizes=conversion_params.cached_engine_batch_sizes,
+        cached_engine_batches=conversion_params.cached_engine_batches,
         use_calibration=conversion_params.use_calibration,
         session_config=config_for_trt)
 
-- 
GitLab


From 741fb41f3cafc5750b3d1f6ab3ba366b8cf93b34 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Thu, 17 Jan 2019 13:49:30 -0800
Subject: [PATCH 0908/1095] [TF:XLA] Bump open source llvm revision to r351428

PiperOrigin-RevId: 229811121
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 6fcccd4147..ad73b19ff1 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -512,11 +512,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "llvm",
         build_file = clean_dep("//third_party/llvm:llvm.autogenerated.BUILD"),
-        sha256 = "2027c5043ea3458376fbf4263aaab3b257736b9e4a1e9e9984f15e245177979d",
-        strip_prefix = "llvm-82a2c96ee81a617918d9bcc0fbe9af860e437f5f",
+        sha256 = "986122dee6053db98d4cf8a60efedd6d94e20c557cc97299a0c7d33b9efae12d",
+        strip_prefix = "llvm-329f768b5fc380a4bfa327396f108a8d8f33e77b",
         urls = [
-            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/82a2c96ee81a617918d9bcc0fbe9af860e437f5f.tar.gz",
-            "https://github.com/llvm-mirror/llvm/archive/82a2c96ee81a617918d9bcc0fbe9af860e437f5f.tar.gz",
+            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/329f768b5fc380a4bfa327396f108a8d8f33e77b.tar.gz",
+            "https://github.com/llvm-mirror/llvm/archive/329f768b5fc380a4bfa327396f108a8d8f33e77b.tar.gz",
         ],
     )
 
-- 
GitLab


From 793d66993c0210fd22eeda7c7048c379521ebd29 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 17 Jan 2019 13:52:51 -0800
Subject: [PATCH 0909/1095] internal changes

PiperOrigin-RevId: 229811646
---
 tensorflow/python/autograph/impl/api.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/autograph/impl/api.py b/tensorflow/python/autograph/impl/api.py
index 6633441e9f..122ea1726b 100644
--- a/tensorflow/python/autograph/impl/api.py
+++ b/tensorflow/python/autograph/impl/api.py
@@ -35,9 +35,9 @@ from tensorflow.python.autograph.operators import py_builtins
 from tensorflow.python.autograph.pyct import compiler
 from tensorflow.python.autograph.pyct import inspect_utils
 from tensorflow.python.autograph.utils import py_func
-from tensorflow.python.util import nest
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.util import nest
 from tensorflow.python.util import tf_decorator
 from tensorflow.python.util import tf_inspect
 from tensorflow.python.util.tf_export import tf_export
@@ -263,7 +263,7 @@ def converted_call(f, owner, options, *args, **kwargs):
     partial_types = (f.__class__,)
 
   else:
-    NotImplementedError('unknown callable type "%s"' % type(f))
+    raise NotImplementedError('unknown callable type "%s"' % type(f))
 
   arg_values = tf_inspect.getcallargs(arg_map_target, *args, **kwargs)
   arg_types = {}
-- 
GitLab


From 8dacf726e788afadf7b8a7ab366f66c10fb2c0c4 Mon Sep 17 00:00:00 2001
From: Russell Power <power@google.com>
Date: Thu, 17 Jan 2019 13:58:28 -0800
Subject: [PATCH 0910/1095] TPUEstimator: always force dequeue operations to
 execute.

If a hostcall does not use the result of an outfeed_dequeue operation, the
resulting values remains on the outfeed.  This eventually stalls the training
process.  This CL executes all outfeed_dequeue operations explicitly when
running hostcalls to ensure the outfeed is drained.

PiperOrigin-RevId: 229812586
---
 .../contrib/tpu/python/tpu/tpu_estimator.py   | 28 +++++++++++++++++--
 1 file changed, 26 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 022ac4152d..27c8ccf6bd 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -124,6 +124,16 @@ def _is_iterable(obj):
     return False
 
 
+class CatchInvalidHostcallFunctions(control_flow_ops.XLAControlFlowContext):
+
+  def AddOp(self, op):
+    if op.type in [
+        'AudioSummary', 'AudioSummaryV2', 'HistogramSummary', 'ImageSummary',
+        'MergeSummary', 'ScalarSummary', 'TensorSummary', 'TensorSummaryV2'
+    ]:
+      raise ValueError('Use tf.contrib.summary inside of host_calls.')
+
+
 def _create_global_step(graph):
   graph = graph or ops.get_default_graph()
   if training.get_global_step(graph) is not None:
@@ -1802,6 +1812,10 @@ class _OutfeedHostCall(object):
           dequeue_ops[j].append(item)
 
     # Deconstruct dequeue ops.
+    flat_dequeue_ops = []
+    for l in dequeue_ops:
+      flat_dequeue_ops.extend(l)
+
     dequeue_ops_by_name = {}
     pos = 0
     for name in self._names:
@@ -1809,6 +1823,14 @@ class _OutfeedHostCall(object):
                                               len(self._tensors[name])]
       pos += len(self._tensors[name])
 
+    def _call_host_fn(fn, *args, **kw):
+      context = CatchInvalidHostcallFunctions()
+      context.Enter()
+      result = fn(*args, **kw)
+      context.Exit()
+      context.ExitResult(result)
+      return result
+
     # It is assumed evaluation always happens on single host TPU system. So,
     # place all ops on tpu host if possible.
     #
@@ -1842,7 +1864,7 @@ class _OutfeedHostCall(object):
           # The user-provided eval_metrics[1] is a dict.
           dequeue_ops = dict(zip(self._tensor_keys[name], dequeue_ops))
           try:
-            ret[name] = self._host_fns[name](**dequeue_ops)
+            ret[name] = _call_host_fn(self._host_fns[name], **dequeue_ops)
           except TypeError as e:
             logging.warning(
                 'Exception while calling %s: %s. It is likely the tensors '
@@ -1850,8 +1872,10 @@ class _OutfeedHostCall(object):
                 'function\'s arguments', name, e, name)
             raise
         else:
-          ret[name] = self._host_fns[name](*dequeue_ops)
+          ret[name] = _call_host_fn(self._host_fns[name], *dequeue_ops)
 
+    # force all dequeue operations to be run if not consumed by the host calls
+    ret['__force_dequeue'] = control_flow_ops.group(*flat_dequeue_ops)
     return ret
 
 
-- 
GitLab


From 1cb48000f4c8e22bb444920a999942f8c50d17a9 Mon Sep 17 00:00:00 2001
From: Fei Hu <hufei68@gmail.com>
Date: Thu, 3 Jan 2019 15:39:40 -0800
Subject: [PATCH 0911/1095] Move VariantTensorDataWriter and
 VariantTensorDataReader to dataset_utils

---
 tensorflow/core/kernels/data/dataset_utils.cc | 94 +++++++++++++++++++
 tensorflow/core/kernels/data/dataset_utils.h  | 49 ++++++++++
 2 files changed, 143 insertions(+)

diff --git a/tensorflow/core/kernels/data/dataset_utils.cc b/tensorflow/core/kernels/data/dataset_utils.cc
index 4d92d314d3..aba991c080 100644
--- a/tensorflow/core/kernels/data/dataset_utils.cc
+++ b/tensorflow/core/kernels/data/dataset_utils.cc
@@ -141,5 +141,99 @@ Status VerifyShapesCompatible(const std::vector<PartialTensorShape>& expected,
   return Status::OK();
 }
 
+Status VariantTensorDataReader::status() const { return status_; }
+
+Status VariantTensorDataReader::ReadScalar(StringPiece key, int64* val) {
+  return ReadScalarInternal(key, val);
+}
+
+Status VariantTensorDataReader::ReadScalar(StringPiece key, string* val) {
+  return ReadScalarInternal(key, val);
+}
+
+Status VariantTensorDataReader::ReadTensor(StringPiece key, Tensor* val) {
+  return ReadTensorInternal(key, val);
+}
+
+bool VariantTensorDataReader::Contains(StringPiece key) {
+  return map_.find(string(key)) != map_.end();
+}
+
+void VariantTensorDataReader::PreProcess() {
+  string metadata;
+  data_->get_metadata(&metadata);
+  IteratorStateMetadata proto;
+  if (!proto.ParseFromString(metadata)) {
+    status_ = errors::Internal("Error parsing IteratorStateMetadata.");
+    return;
+  }
+  size_t num_entries = proto.keys_size();
+  CHECK_EQ(num_entries, data_->tensors_size());
+  for (size_t i = 0; i < num_entries; i++) {
+    map_[proto.keys(i)] = i;
+  }
+}
+
+template <typename T>
+Status VariantTensorDataReader::ReadScalarInternal(StringPiece key, T* val) {
+  if (map_.find(string(key)) == map_.end()) {
+    return errors::NotFound(key);
+  }
+  *val = data_->tensors(map_[string(key)]).scalar<T>()();
+  return Status::OK();
+}
+
+Status VariantTensorDataReader::ReadTensorInternal(StringPiece key,
+                                                   Tensor* val) {
+  if (map_.find(string(key)) == map_.end()) {
+    return errors::NotFound(key);
+  }
+  *val = data_->tensors(map_[string(key)]);
+  return Status::OK();
+}
+
+Status VariantTensorDataWriter::WriteScalar(StringPiece key, const int64 val) {
+  return WriteScalarInternal(key, val);
+}
+
+Status VariantTensorDataWriter::WriteScalar(StringPiece key,
+                                            const string& val) {
+  return WriteScalarInternal(key, val);
+}
+
+Status VariantTensorDataWriter::WriteTensor(StringPiece key,
+                                            const Tensor& val) {
+  return WriteTensorInternal(key, val);
+}
+
+Status VariantTensorDataWriter::Flush() {
+  string metadata;
+  if (!metadata_proto_.SerializeToString(&metadata)) {
+    return errors::Internal("Unable to serialize IteratorStateMetadata.");
+  }
+  data_->set_metadata(metadata);
+  return Status::OK();
+}
+
+template <typename T>
+Status VariantTensorDataWriter::WriteScalarInternal(StringPiece key,
+                                                    const T& val) {
+  Tensor val_t = Tensor(DataTypeToEnum<T>::v(), TensorShape({}));
+  val_t.scalar<T>()() = val;
+  return WriteTensorInternal(key, val_t);
+}
+
+Status VariantTensorDataWriter::WriteTensorInternal(StringPiece key,
+                                                    const Tensor& val) {
+  // Write key to the metadata proto. This gets written to `data_`
+  // when `Flush()` is called. We do this lazily to avoid multiple
+  // serialization calls.
+  metadata_proto_.add_keys(string(key));
+
+  // Update tensors.
+  *(data_->add_tensors()) = val;
+  return Status::OK();
+}
+
 }  // namespace data
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/data/dataset_utils.h b/tensorflow/core/kernels/data/dataset_utils.h
index 23a3d93ed1..b2a19ac869 100644
--- a/tensorflow/core/kernels/data/dataset_utils.h
+++ b/tensorflow/core/kernels/data/dataset_utils.h
@@ -16,6 +16,7 @@ limitations under the License.
 #define TENSORFLOW_CORE_KERNELS_DATA_DATASET_UTILS_H_
 
 #include "tensorflow/core/framework/dataset.h"
+#include "tensorflow/core/framework/iterator.pb.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/kernels/data/captured_function.h"
 
@@ -57,6 +58,54 @@ Status VerifyTypesMatch(const DataTypeVector& expected,
 Status VerifyShapesCompatible(const std::vector<PartialTensorShape>& expected,
                               const std::vector<PartialTensorShape>& received);
 
+// Helper class for reading data from a VariantTensorData object.
+class VariantTensorDataReader : public IteratorStateReader {
+ public:
+  explicit VariantTensorDataReader(const VariantTensorData* data)
+      : data_(data) {
+    PreProcess();
+  }
+
+  // Returns OK iff the initialization was successful, i.e.,
+  // pre-processing did not have errors.
+  Status status() const;
+  Status ReadScalar(StringPiece key, int64* val) override;
+  Status ReadScalar(StringPiece key, string* val) override;
+  Status ReadTensor(StringPiece key, Tensor* val) override;
+  bool Contains(StringPiece key) override;
+
+ private:
+  void PreProcess();
+  template <typename T>
+  Status ReadScalarInternal(StringPiece key, T* val);
+  Status ReadTensorInternal(StringPiece key, Tensor* val);
+  std::map<string, size_t> map_;
+  const VariantTensorData* data_;  // Not owned.
+  Status status_;
+};
+
+// Helper class for writing data to a VariantTensorData object.
+class VariantTensorDataWriter : public IteratorStateWriter {
+ public:
+  // Does not take ownership of data.
+  explicit VariantTensorDataWriter(VariantTensorData* data) : data_(data) {}
+  Status WriteScalar(StringPiece key, const int64 val) override;
+  Status WriteScalar(StringPiece key, const string& val) override;
+  Status WriteTensor(StringPiece key, const Tensor& val) override;
+
+  // Writes the metadata to `data_`.
+  Status Flush();
+
+ private:
+  template <typename T>
+  Status WriteScalarInternal(StringPiece key, const T& val);
+  Status WriteTensorInternal(StringPiece key, const Tensor& val);
+  VariantTensorData* data_;
+
+  // TODO(srbs): Set the version string.
+  IteratorStateMetadata metadata_proto_;
+};
+
 }  // namespace data
 }  // namespace tensorflow
 
-- 
GitLab


From 0762324b237eca697b376e3e5d067d761717230e Mon Sep 17 00:00:00 2001
From: Fei Hu <hufei68@gmail.com>
Date: Thu, 3 Jan 2019 22:37:28 -0800
Subject: [PATCH 0912/1095] Add the test cases for VariantTensorDataWriter and
 VariantTensorDataReader

---
 tensorflow/core/kernels/data/BUILD            |  1 +
 .../core/kernels/data/dataset_utils_test.cc   | 44 ++++++++++++++++++-
 2 files changed, 43 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/kernels/data/BUILD b/tensorflow/core/kernels/data/BUILD
index fab1ec8b6e..dcbdb74691 100644
--- a/tensorflow/core/kernels/data/BUILD
+++ b/tensorflow/core/kernels/data/BUILD
@@ -39,6 +39,7 @@ tf_cc_test(
     srcs = ["dataset_utils_test.cc"],
     deps = [
         ":dataset_utils",
+        "//tensorflow/core:framework",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
     ],
diff --git a/tensorflow/core/kernels/data/dataset_utils_test.cc b/tensorflow/core/kernels/data/dataset_utils_test.cc
index 43295b8ebb..de5dc59bb1 100644
--- a/tensorflow/core/kernels/data/dataset_utils_test.cc
+++ b/tensorflow/core/kernels/data/dataset_utils_test.cc
@@ -13,15 +13,16 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#include "tensorflow/core/framework/variant.h"
 #include "tensorflow/core/kernels/data/dataset_utils.h"
-
+#include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/platform/test.h"
 
 namespace tensorflow {
 namespace data {
 namespace {
 
-TEST(DatasetUtils, ComputeMoveVector) {
+TEST(DatasetUtilsTest, ComputeMoveVector) {
   struct TestCase {
     std::vector<int> indices;
     std::vector<bool> expected;
@@ -41,6 +42,45 @@ TEST(DatasetUtils, ComputeMoveVector) {
   }
 }
 
+TEST(DatasetUtilsTest, VariantTensorData_Writer_Reader) {
+  VariantTensorData data;
+
+  // Basic test cases.
+  VariantTensorDataWriter writer(&data);
+  TF_ASSERT_OK(writer.WriteScalar("Int64", 24));
+  TF_ASSERT_OK(writer.WriteScalar("", "Empty_Key"));
+  Tensor input_tensor(DT_FLOAT, {1});
+  input_tensor.flat<float>()(0) = 2.0f;
+  TF_ASSERT_OK(writer.WriteTensor("Tensor", input_tensor));
+  TF_ASSERT_OK(writer.Flush());
+
+  VariantTensorDataReader reader(&data);
+  EXPECT_OK(reader.status());
+  int64 val_int64;
+  TF_ASSERT_OK(reader.ReadScalar("Int64", &val_int64));
+  EXPECT_EQ(val_int64, 24);
+  string val_string;
+  TF_ASSERT_OK(reader.ReadScalar("", &val_string));
+  EXPECT_EQ(val_string, "Empty_Key");
+  Tensor val_tensor;
+  TF_ASSERT_OK(reader.ReadTensor("Tensor", &val_tensor));
+  EXPECT_EQ(input_tensor.NumElements(), val_tensor.NumElements());
+  EXPECT_EQ(input_tensor.flat<float>()(0), val_tensor.flat<float>()(0));
+
+  // Test the non-existing key for VariantTensorDataReader.
+  EXPECT_EQ(error::NOT_FOUND,
+            reader.ReadScalar("Non_Existing_Key", &val_int64).code());
+  EXPECT_EQ(error::NOT_FOUND,
+            reader.ReadScalar("Non_Existing_Key", &val_string).code());
+  EXPECT_EQ(error::NOT_FOUND,
+            reader.ReadTensor("Non_Existing_Key", &val_tensor).code());
+
+  // Test the invalid parameter for the constructor of VariantTensorDataReader.
+  data.metadata_ = "Invalid Metadata";
+  VariantTensorDataReader reader2(&data);
+  EXPECT_EQ(error::INTERNAL, reader2.status().code());
+}
+
 }  // namespace
 }  // namespace data
 }  // namespace tensorflow
-- 
GitLab


From 55c6645ec7c65ebc6ce9e036052a915b66c2fc46 Mon Sep 17 00:00:00 2001
From: Fei Hu <hufei68@gmail.com>
Date: Fri, 4 Jan 2019 10:00:48 -0800
Subject: [PATCH 0913/1095] Use error::InvalidArgument to replace CHECK_EQ in
 VariantTensorDataReader::PreProcess

---
 tensorflow/core/kernels/data/dataset_utils.cc      |  7 ++++++-
 tensorflow/core/kernels/data/dataset_utils_test.cc | 13 ++++++++++---
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/kernels/data/dataset_utils.cc b/tensorflow/core/kernels/data/dataset_utils.cc
index aba991c080..cb0a183c3b 100644
--- a/tensorflow/core/kernels/data/dataset_utils.cc
+++ b/tensorflow/core/kernels/data/dataset_utils.cc
@@ -168,7 +168,12 @@ void VariantTensorDataReader::PreProcess() {
     return;
   }
   size_t num_entries = proto.keys_size();
-  CHECK_EQ(num_entries, data_->tensors_size());
+  if (num_entries != data_->tensors_size()) {
+    status_ = errors::InvalidArgument("Unmatched number of keys and tensors: ",
+                                      num_entries, " vs. ",
+                                      data_->tensors_size());
+    return;
+  }
   for (size_t i = 0; i < num_entries; i++) {
     map_[proto.keys(i)] = i;
   }
diff --git a/tensorflow/core/kernels/data/dataset_utils_test.cc b/tensorflow/core/kernels/data/dataset_utils_test.cc
index de5dc59bb1..2cc5d357b0 100644
--- a/tensorflow/core/kernels/data/dataset_utils_test.cc
+++ b/tensorflow/core/kernels/data/dataset_utils_test.cc
@@ -75,10 +75,17 @@ TEST(DatasetUtilsTest, VariantTensorData_Writer_Reader) {
   EXPECT_EQ(error::NOT_FOUND,
             reader.ReadTensor("Non_Existing_Key", &val_tensor).code());
 
-  // Test the invalid parameter for the constructor of VariantTensorDataReader.
-  data.metadata_ = "Invalid Metadata";
-  VariantTensorDataReader reader2(&data);
+  // Test the invalid metadata for VariantTensorDataReader.
+  VariantTensorData data_invalid_meta(data);
+  data_invalid_meta.metadata_ = "Invalid Metadata";
+  VariantTensorDataReader reader2(&data_invalid_meta);
   EXPECT_EQ(error::INTERNAL, reader2.status().code());
+
+  // Test the unmatched number of keys and tensors for VariantTensorDataReader.
+  VariantTensorData data_unmatched_entries(data);
+  data_unmatched_entries.tensors_.push_back(Tensor(DT_INT64, {1}));
+  VariantTensorDataReader reader3(&data_unmatched_entries);
+  EXPECT_EQ(error::INVALID_ARGUMENT, reader3.status().code());
 }
 
 }  // namespace
-- 
GitLab


From 7bcbe79420f52f9fbee7d43327fb2edab088bb78 Mon Sep 17 00:00:00 2001
From: Fei Hu <hufei68@gmail.com>
Date: Fri, 4 Jan 2019 11:08:56 -0800
Subject: [PATCH 0914/1095] Separate the big test to small tests and address
 the clang-format issues

---
 tensorflow/core/kernels/data/dataset_utils.cc |  6 +--
 .../core/kernels/data/dataset_utils_test.cc   | 43 +++++++++++--------
 2 files changed, 28 insertions(+), 21 deletions(-)

diff --git a/tensorflow/core/kernels/data/dataset_utils.cc b/tensorflow/core/kernels/data/dataset_utils.cc
index cb0a183c3b..8ac74af6ba 100644
--- a/tensorflow/core/kernels/data/dataset_utils.cc
+++ b/tensorflow/core/kernels/data/dataset_utils.cc
@@ -169,9 +169,9 @@ void VariantTensorDataReader::PreProcess() {
   }
   size_t num_entries = proto.keys_size();
   if (num_entries != data_->tensors_size()) {
-    status_ = errors::InvalidArgument("Unmatched number of keys and tensors: ",
-                                      num_entries, " vs. ",
-                                      data_->tensors_size());
+    status_ = errors::InvalidArgument(
+        "Unmatched number of keys and tensors: ", num_entries, " vs. ",
+        data_->tensors_size());
     return;
   }
   for (size_t i = 0; i < num_entries; i++) {
diff --git a/tensorflow/core/kernels/data/dataset_utils_test.cc b/tensorflow/core/kernels/data/dataset_utils_test.cc
index 2cc5d357b0..b503e58c1b 100644
--- a/tensorflow/core/kernels/data/dataset_utils_test.cc
+++ b/tensorflow/core/kernels/data/dataset_utils_test.cc
@@ -13,8 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/core/framework/variant.h"
 #include "tensorflow/core/kernels/data/dataset_utils.h"
+#include "tensorflow/core/framework/variant.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/platform/test.h"
 
@@ -42,10 +42,8 @@ TEST(DatasetUtilsTest, ComputeMoveVector) {
   }
 }
 
-TEST(DatasetUtilsTest, VariantTensorData_Writer_Reader) {
+TEST(DatasetUtilsTest, VariantTensorDataRoundtrip) {
   VariantTensorData data;
-
-  // Basic test cases.
   VariantTensorDataWriter writer(&data);
   TF_ASSERT_OK(writer.WriteScalar("Int64", 24));
   TF_ASSERT_OK(writer.WriteScalar("", "Empty_Key"));
@@ -66,26 +64,35 @@ TEST(DatasetUtilsTest, VariantTensorData_Writer_Reader) {
   TF_ASSERT_OK(reader.ReadTensor("Tensor", &val_tensor));
   EXPECT_EQ(input_tensor.NumElements(), val_tensor.NumElements());
   EXPECT_EQ(input_tensor.flat<float>()(0), val_tensor.flat<float>()(0));
+}
 
-  // Test the non-existing key for VariantTensorDataReader.
+TEST(DatasetUtilsTest, VariantTensorDataNonExistentKey) {
+  VariantTensorData data;
+  VariantTensorDataReader reader(&data);
+  TF_ASSERT_OK(reader.status());
+  int64 val_int64;
+  string val_string;
+  Tensor val_tensor;
   EXPECT_EQ(error::NOT_FOUND,
-            reader.ReadScalar("Non_Existing_Key", &val_int64).code());
+            reader.ReadScalar("NonExistentKey", &val_int64).code());
   EXPECT_EQ(error::NOT_FOUND,
-            reader.ReadScalar("Non_Existing_Key", &val_string).code());
+            reader.ReadScalar("NonExistentKey", &val_string).code());
   EXPECT_EQ(error::NOT_FOUND,
-            reader.ReadTensor("Non_Existing_Key", &val_tensor).code());
+            reader.ReadTensor("NonExistentKey", &val_tensor).code());
+}
 
-  // Test the invalid metadata for VariantTensorDataReader.
-  VariantTensorData data_invalid_meta(data);
-  data_invalid_meta.metadata_ = "Invalid Metadata";
-  VariantTensorDataReader reader2(&data_invalid_meta);
-  EXPECT_EQ(error::INTERNAL, reader2.status().code());
+TEST(DatasetUtilsTest, VariantTensorDataInvalidMetadata) {
+  VariantTensorData data;
+  data.metadata_ = "Invalid Metadata";
+  VariantTensorDataReader reader(&data);
+  EXPECT_EQ(error::INTERNAL, reader.status().code());
+}
 
-  // Test the unmatched number of keys and tensors for VariantTensorDataReader.
-  VariantTensorData data_unmatched_entries(data);
-  data_unmatched_entries.tensors_.push_back(Tensor(DT_INT64, {1}));
-  VariantTensorDataReader reader3(&data_unmatched_entries);
-  EXPECT_EQ(error::INVALID_ARGUMENT, reader3.status().code());
+TEST(DatasetUtilsTest, VariantTensorDataUnmatchedKeys) {
+  VariantTensorData data;
+  data.tensors_.push_back(Tensor(DT_INT64, {1}));
+  VariantTensorDataReader reader(&data);
+  EXPECT_EQ(error::INVALID_ARGUMENT, reader.status().code());
 }
 
 }  // namespace
-- 
GitLab


From cc1713c0c0cf20972c34c447fe4afc28a7f6ce34 Mon Sep 17 00:00:00 2001
From: Fei Hu <hufei68@gmail.com>
Date: Fri, 4 Jan 2019 13:01:38 -0800
Subject: [PATCH 0915/1095] Change the indentation

---
 tensorflow/core/kernels/data/dataset_utils.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/core/kernels/data/dataset_utils.cc b/tensorflow/core/kernels/data/dataset_utils.cc
index 8ac74af6ba..67a0b1165a 100644
--- a/tensorflow/core/kernels/data/dataset_utils.cc
+++ b/tensorflow/core/kernels/data/dataset_utils.cc
@@ -169,9 +169,9 @@ void VariantTensorDataReader::PreProcess() {
   }
   size_t num_entries = proto.keys_size();
   if (num_entries != data_->tensors_size()) {
-    status_ = errors::InvalidArgument(
-        "Unmatched number of keys and tensors: ", num_entries, " vs. ",
-        data_->tensors_size());
+    status_ =
+        errors::InvalidArgument("Unmatched number of keys and tensors: ",
+                                num_entries, " vs. ", data_->tensors_size());
     return;
   }
   for (size_t i = 0; i < num_entries; i++) {
-- 
GitLab


From abf9518088797e505ead96aedebd4d71b7e03b4d Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Thu, 17 Jan 2019 14:02:43 -0800
Subject: [PATCH 0916/1095] Disable InterpreterTest.java for now and fix
 missing include.

PiperOrigin-RevId: 229813499
---
 tensorflow/lite/java/BUILD                       | 3 +++
 tensorflow/lite/models/smartreply/ops/predict.cc | 1 +
 2 files changed, 4 insertions(+)

diff --git a/tensorflow/lite/java/BUILD b/tensorflow/lite/java/BUILD
index bd9d8f4806..a539a0cf77 100644
--- a/tensorflow/lite/java/BUILD
+++ b/tensorflow/lite/java/BUILD
@@ -145,6 +145,9 @@ java_test(
         "//tensorflow/lite:testdata/multi_add_flex.bin",
     ],
     javacopts = JAVACOPTS,
+    tags = [
+        "no_mac",  # TODO(b/122888913): libtensorflowlite_test_jni broke on mac.
+    ],
     test_class = "org.tensorflow.lite.InterpreterTest",
     visibility = ["//visibility:private"],
     deps = [
diff --git a/tensorflow/lite/models/smartreply/ops/predict.cc b/tensorflow/lite/models/smartreply/ops/predict.cc
index bb2ed4a315..24b7d54897 100644
--- a/tensorflow/lite/models/smartreply/ops/predict.cc
+++ b/tensorflow/lite/models/smartreply/ops/predict.cc
@@ -28,6 +28,7 @@ limitations under the License.
 //
 
 #include <algorithm>
+#include <cstdlib>
 #include <unordered_map>
 #include <vector>
 
-- 
GitLab


From f986acda003966de4efed0e2fe4854a2b6dc776f Mon Sep 17 00:00:00 2001
From: Andy Ly <lyandy@google.com>
Date: Thu, 17 Jan 2019 14:05:09 -0800
Subject: [PATCH 0917/1095] [Grappler] Exclude Atan2 from IsUnaryElementWise.

PiperOrigin-RevId: 229814062
---
 tensorflow/core/grappler/op_types.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tensorflow/core/grappler/op_types.cc b/tensorflow/core/grappler/op_types.cc
index e06580cebf..f7931c615c 100644
--- a/tensorflow/core/grappler/op_types.cc
+++ b/tensorflow/core/grappler/op_types.cc
@@ -713,7 +713,6 @@ bool IsUnaryElementWise(const NodeDef& node) {
           "Asin",
           "Asinh",
           "Atan",
-          "Atan2",
           "Atanh",
           "Ceil",
           "ComplexAbs",
-- 
GitLab


From 703ab9db70f61ab26d7379614149a07ef3e8db37 Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Thu, 17 Jan 2019 14:22:45 -0800
Subject: [PATCH 0918/1095] Make tolerance more reasonable on
 _testOptimizersCompat...

PiperOrigin-RevId: 229817236
---
 tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
index 374ba60460..8069703b7a 100644
--- a/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
+++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2_test.py
@@ -533,8 +533,10 @@ class OptimizersCompatibilityTest(keras_parameterized.TestCase):
 
       hist_1 = model_v1.fit(x, y, batch_size=5, epochs=1, shuffle=False)
       hist_2 = model_v2.fit(x, y, batch_size=5, epochs=1, shuffle=False)
-      self.assertAllClose(model_v1.get_weights(), model_v2.get_weights())
-      self.assertAllClose(hist_1.history['loss'], hist_2.history['loss'])
+      self.assertAllClose(model_v1.get_weights(), model_v2.get_weights(),
+                          rtol=1e-5, atol=1e-5)
+      self.assertAllClose(hist_1.history['loss'], hist_2.history['loss'],
+                          rtol=1e-5, atol=1e-5)
 
   def testAdadeltaCompatibility(self):
     opt_v1 = optimizers.Adadelta(lr=0.01)
-- 
GitLab


From 779f2f83fa07866ca34b6aff1a45ff5b09bfcfbd Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Thu, 17 Jan 2019 14:22:48 -0800
Subject: [PATCH 0919/1095] Minor fix to error message; NFC

PiperOrigin-RevId: 229817246
---
 tensorflow/core/graph/control_flow.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/graph/control_flow.cc b/tensorflow/core/graph/control_flow.cc
index 8e1e56d29b..66237a3497 100644
--- a/tensorflow/core/graph/control_flow.cc
+++ b/tensorflow/core/graph/control_flow.cc
@@ -59,7 +59,7 @@ Status ValidateControlFlowInfo(const Graph* graph,
           "Invalid loop structure: Mismatched parent frames for \"",
           cf.frame_name, "\": \"", parent->name, "\" vs \"", frame.parent->name,
           "\". The node giving this error: ", FormatNodeForError(*node),
-          "This is an internal bug, please file a bug report with "
+          ". This is an internal bug, please file a bug report with "
           "instructions on how to reproduce the error.");
     }
     if (IsLoopCond(node)) {
-- 
GitLab


From 22bbbb2280f1ce9a04ab67cdb1f63d85da6f33b3 Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Thu, 17 Jan 2019 14:28:59 -0800
Subject: [PATCH 0920/1095] Make tolerance larger for cudnn comparison

PiperOrigin-RevId: 229818348
---
 .../contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py b/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py
index 7e1b4062ce..ca92c31236 100644
--- a/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py
+++ b/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py
@@ -1023,7 +1023,7 @@ class CudnnRNNTestCompatibleRNNCells(test_util.TensorFlowTestCase):
           outputs_v, output_state_v = sess.run(
               [outputs, output_state],
               feed_dict={cell_inputs: inference_input})
-          self.assertAllClose(cudnn_outputs_v, outputs_v, atol=2e-5, rtol=2e-5)
+          self.assertAllClose(cudnn_outputs_v, outputs_v, atol=1e-4, rtol=2e-4)
           (cudnn_output_h_v,) = cudnn_output_states_v
           self.assertAllClose(cudnn_output_h_v, output_state_v, atol=2e-5,
                               rtol=2e-5)
-- 
GitLab


From 98984347b66d0642d042841e47e571cdd2191104 Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Thu, 17 Jan 2019 14:39:06 -0800
Subject: [PATCH 0921/1095] Disable broken tests for 1.13

PiperOrigin-RevId: 229820397
---
 tensorflow/lite/build_def.bzl | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/lite/build_def.bzl b/tensorflow/lite/build_def.bzl
index 7e70503386..5e6b13a46a 100644
--- a/tensorflow/lite/build_def.bzl
+++ b/tensorflow/lite/build_def.bzl
@@ -327,6 +327,7 @@ def generated_test_models_failing(conversion_mode):
     if conversion_mode == "toco-flex":
         return [
             "lstm",  # TODO(b/117510976): Restore when lstm flex conversion works.
+            "unroll_batch_matmul",  # TODO(b/123030774): Fails in 1.13 tests.
         ]
 
     return []
-- 
GitLab


From ee0cc9c80dfbc52ce63dea5e9bd9d789bdbb4893 Mon Sep 17 00:00:00 2001
From: Wenhao Jia <jiawenhao@google.com>
Date: Thu, 17 Jan 2019 14:42:11 -0800
Subject: [PATCH 0922/1095] Internal change

PiperOrigin-RevId: 229820977
---
 tensorflow/compiler/jit/xla_device.cc      | 78 +++++++---------------
 tensorflow/compiler/jit/xla_device.h       | 38 ++---------
 tensorflow/core/common_runtime/device.h    | 19 +++++-
 tensorflow/core/common_runtime/executor.cc | 54 ++++++++++++++-
 tensorflow/core/framework/op_kernel.h      | 22 ++++++
 5 files changed, 120 insertions(+), 91 deletions(-)

diff --git a/tensorflow/compiler/jit/xla_device.cc b/tensorflow/compiler/jit/xla_device.cc
index 77cd2f4462..e2397f6fcb 100644
--- a/tensorflow/compiler/jit/xla_device.cc
+++ b/tensorflow/compiler/jit/xla_device.cc
@@ -219,9 +219,6 @@ XlaDevice::XlaDevice(const SessionOptions& session_options,
 XlaDevice::~XlaDevice() {
   VLOG(1) << "Destroying XLA device " << jit_device_name_ << " " << this;
   mutex_lock lock(mu_);
-  while (outstanding_asynchronous_operations_ > 0) {
-    outstanding_asynchronous_operations_cv_.wait(lock);
-  }
   if (device_context_) {
     device_context_->Unref();
   }
@@ -398,12 +395,6 @@ Status XlaDevice::Sync() {
   if (!stream) return Status::OK();
 
   Status status = stream->BlockHostUntilDone();
-  {
-    mutex_lock lock(mu_);
-    while (outstanding_asynchronous_operations_ > 0) {
-      outstanding_asynchronous_operations_cv_.wait(lock);
-    }
-  }
   TF_RETURN_IF_ERROR(status);
   if (!stream->ok()) {
     return errors::Internal("XlaDevice::Sync() failed.");
@@ -412,6 +403,8 @@ Status XlaDevice::Sync() {
   return Status::OK();
 }
 
+// TODO(b/112409994): This is no longer necessary. Consolidate it with the
+// synchronous version.
 void XlaDevice::Sync(const DoneCallback& done) {
   VLOG(1) << "XlaDevice::Sync (asynchronous)";
   std::shared_ptr<se::Stream> stream;
@@ -424,14 +417,20 @@ void XlaDevice::Sync(const DoneCallback& done) {
     return;
   }
 
+  // The call to ThenEnqueueOnBackgroundThread below enqueues a host callback at
+  // the end of the stream, after everything that has already been enqueued
+  // there at this moment. When the host callback is called, everything before
+  // it must have already finished, and the host callback will then place the
+  // task below onto a background thread. (See the implementation of
+  // ThenEnqueueOnBackgroundThread for details.) Therefore, when the done
+  // callback is finally called from that background thread, we know for sure
+  // that everything enqueued onto the stream (i.e., the device) at this very
+  // moment--when ThenEnqueueOnBackgroundThread is called--will have finished.
+  // This achieves a device-wide sync.
   stream->ThenEnqueueOnBackgroundThread(
       [this, stream, done](se::StreamExecutor*) {
         tracing::ScopedActivity activity("XlaDevice::Sync::Callback",
                                          /*is_expensive=*/true);
-        mutex_lock lock(mu_);
-        while (outstanding_asynchronous_operations_ > 0) {
-          outstanding_asynchronous_operations_cv_.wait(lock);
-        }
         done(stream->ok() ? Status::OK()
                           : errors::Internal("XlaDevice::Sync() failed."));
       });
@@ -470,57 +469,26 @@ Status XlaDevice::MakeTensorFromProto(const TensorProto& tensor_proto,
   return status;
 }
 
-void XlaDevice::SetRequiresSyncOnCompletion(bool sync_on_completion) {
+void XlaDevice::SetAllowsSyncOnCompletion(bool sync_on_completion) {
   mutex_lock lock(mu_);
   sync_on_completion_ = sync_on_completion;
 }
 
-bool XlaDevice::RequiresSyncOnCompletion() const {
+bool XlaDevice::AllowsSyncOnCompletion() const {
   mutex_lock lock(mu_);
   return sync_on_completion_;
 }
 
-XlaDevice::AsynchronousOperationHandle::AsynchronousOperationHandle(
-    XlaDevice* device)
-    : device_(device) {
-  mutex_lock lock(device_->mu_);
-  ++device_->outstanding_asynchronous_operations_;
-}
-
-XlaDevice::AsynchronousOperationHandle::~AsynchronousOperationHandle() {
-  if (device_) {
-    mutex_lock lock(device_->mu_);
-    --device_->outstanding_asynchronous_operations_;
-    device_->outstanding_asynchronous_operations_cv_.notify_all();
+Status XlaDevice::CurrentStatus() {
+  std::shared_ptr<se::Stream> stream;
+  {
+    mutex_lock lock(mu_);
+    stream = stream_;
   }
-}
-
-XlaDevice::AsynchronousOperationHandle::AsynchronousOperationHandle(
-    const XlaDevice::AsynchronousOperationHandle& other)
-    : device_(other.device_) {
-  mutex_lock lock(device_->mu_);
-  ++device_->outstanding_asynchronous_operations_;
-}
-
-XlaDevice::AsynchronousOperationHandle::AsynchronousOperationHandle(
-    XlaDevice::AsynchronousOperationHandle&& other)
-    : device_(other.device_) {
-  other.device_ = nullptr;
-}
-
-XlaDevice::AsynchronousOperationHandle& XlaDevice::AsynchronousOperationHandle::
-operator=(const XlaDevice::AsynchronousOperationHandle& other) {
-  device_ = other.device_;
-  mutex_lock lock(device_->mu_);
-  ++device_->outstanding_asynchronous_operations_;
-  return *this;
-}
-
-XlaDevice::AsynchronousOperationHandle& XlaDevice::AsynchronousOperationHandle::
-operator=(XlaDevice::AsynchronousOperationHandle&& other) {
-  device_ = other.device_;
-  other.device_ = nullptr;
-  return *this;
+  if (!stream) {
+    return Status::OK();
+  }
+  return stream->ok() ? Status::OK() : errors::Internal("XlaDevice is not OK.");
 }
 
 XlaDeviceOpRegistrations* RegisterXlaDeviceKernels(const char* device,
diff --git a/tensorflow/compiler/jit/xla_device.h b/tensorflow/compiler/jit/xla_device.h
index 45f18ac9ee..e35a1c7d29 100644
--- a/tensorflow/compiler/jit/xla_device.h
+++ b/tensorflow/compiler/jit/xla_device.h
@@ -167,35 +167,14 @@ class XlaDevice : public LocalDevice {
   Status UseGpuDeviceInfo() LOCKS_EXCLUDED(mu_);
 
   // Instructs this XlaDevice to return 'sync_on_completion' for
-  // RequiresSyncOnCompletion().
-  void SetRequiresSyncOnCompletion(bool sync_on_completion) LOCKS_EXCLUDED(mu_);
+  // AllowsSyncOnCompletion().
+  void SetAllowsSyncOnCompletion(bool sync_on_completion) LOCKS_EXCLUDED(mu_);
 
-  bool RequiresSyncOnCompletion() const override LOCKS_EXCLUDED(mu_);
+  bool AllowsSyncOnCompletion() const override LOCKS_EXCLUDED(mu_);
 
-  // A simple RAII handle. On construction the device's
-  // outstanding_asynchronous_operations_ field is incremented; on destruction
-  // it is decremented.
-  class AsynchronousOperationHandle {
-   public:
-    AsynchronousOperationHandle(XlaDevice* device);
-    ~AsynchronousOperationHandle();
-    AsynchronousOperationHandle(const AsynchronousOperationHandle& other);
-    AsynchronousOperationHandle(AsynchronousOperationHandle&& other);
-    AsynchronousOperationHandle& operator=(
-        const AsynchronousOperationHandle& other);
-    AsynchronousOperationHandle& operator=(AsynchronousOperationHandle&& other);
-
-   private:
-    XlaDevice* device_ = nullptr;
-  };
-
-  AsynchronousOperationHandle CreateAsynchronousOperationHandle() {
-    return AsynchronousOperationHandle(this);
-  }
+  Status CurrentStatus() override LOCKS_EXCLUDED(mu_);
 
  private:
-  friend class AsynchronousOperationHandle;
-
   xla::LocalClient* client() const;
   Allocator* GetAllocatorLocked(AllocatorAttributes attr)
       EXCLUSIVE_LOCKS_REQUIRED(mu_);
@@ -255,14 +234,9 @@ class XlaDevice : public LocalDevice {
   // Thread pool used for running closures
   std::unique_ptr<thread::ThreadPool> thread_pool_;
 
-  // True if the device requires XlaDevice::Sync to be called on completion
+  // True if the device allows XlaDevice::Sync to be called on completion
   // regardless of status.
-  bool sync_on_completion_ GUARDED_BY(mu_) = false;
-
-  // Count of outstanding asynchronous operations which must be zero on Sync()
-  // completion.
-  int64 outstanding_asynchronous_operations_ GUARDED_BY(mu_) = 0;
-  condition_variable outstanding_asynchronous_operations_cv_;
+  bool sync_on_completion_ GUARDED_BY(mu_) = true;
 
   // Set of devices to use. This controls which of the devices on the given
   // platform will have resources allocated. For GPUs this will be
diff --git a/tensorflow/core/common_runtime/device.h b/tensorflow/core/common_runtime/device.h
index 8dfbb21eda..5a0ef28ff2 100644
--- a/tensorflow/core/common_runtime/device.h
+++ b/tensorflow/core/common_runtime/device.h
@@ -44,6 +44,7 @@ limitations under the License.
 #include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/graph/graph.h"
 #include "tensorflow/core/graph/types.h"
+#include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/macros.h"
 #include "tensorflow/core/platform/types.h"
@@ -122,9 +123,21 @@ class Device : public DeviceBase {
   // version.
   virtual void Sync(const DoneCallback& done);
 
-  // Override this to return true for devices that require a Sync() call before
-  // session completion.
-  virtual bool RequiresSyncOnCompletion() const { return false; }
+  // On session completion, the executor may call Device::Sync() depending on
+  // flag settings. Override this to return false for devices that don't allow
+  // such calls. Instead, these devices must use other mechanisms (such as
+  // num_deferred_ops) to ensure the device has finished processing necessary
+  // work at session completion.
+  //
+  // Devices that override this function must also implement CurrentStatus.
+  virtual bool AllowsSyncOnCompletion() const { return true; }
+
+  // This is used in conjunction with AllowsSyncOnCompletion to allow the
+  // executor to get execution result status at session completion.
+  virtual Status CurrentStatus() {
+    return errors::Unimplemented(
+        "CurrentStatus is not supported on this device.");
+  }
 
   // Optionally modify the device's GraphDef before execution.
   //
diff --git a/tensorflow/core/common_runtime/executor.cc b/tensorflow/core/common_runtime/executor.cc
index 07c8c4a5d4..d068bbf1e4 100644
--- a/tensorflow/core/common_runtime/executor.cc
+++ b/tensorflow/core/common_runtime/executor.cc
@@ -1276,6 +1276,11 @@ class ExecutorState {
 
   std::atomic_int_fast32_t num_outstanding_ops_;
 
+  // Available via OpKernelContext to every OpKernel invocation.
+  mutex num_deferred_ops_mu_;
+  condition_variable num_deferred_ops_cv_;
+  int64 num_deferred_ops_ GUARDED_BY(num_deferred_ops_mu_) = 0;
+
   mutex mu_;
   Status status_ GUARDED_BY(mu_);
 
@@ -1631,6 +1636,15 @@ void ExecutorState::Process(TaggedNode tagged_node, int64 scheduled_nsec) {
   params.input_alloc_attrs = &input_alloc_attrs;
   params.runner = &runner_;
   params.stats_collector = stats_collector_;
+  params.inc_num_deferred_ops_function = [this]() {
+    mutex_lock lock(num_deferred_ops_mu_);
+    num_deferred_ops_++;
+  };
+  params.dec_num_deferred_ops_function = [this]() {
+    mutex_lock lock(num_deferred_ops_mu_);
+    num_deferred_ops_--;
+    num_deferred_ops_cv_.notify_all();
+  };
 
   Status s;
   NodeExecStatsInterface* stats = nullptr;
@@ -2416,7 +2430,45 @@ void ExecutorState::Finish() {
   CHECK(done_cb != nullptr);
   Device* device = impl_->params_.device;
 
-  if ((sync_on_finish_ && status.ok()) || device->RequiresSyncOnCompletion()) {
+  // There are several potential race conditions below. To name a few:
+  // 1. Even if the device's status is OK at the precise moment when
+  // num_deferred_ops_ reaches 0, it could go bad before device->CurrentStatus()
+  // is called below, caused by work enqueued onto the same device by other
+  // concurrent ExecutorState objects.
+  // 2. Some implementations of Device::CurrentStatus, such as
+  // XlaDevice::CurrentStatus, may be inherently racy because it releases the
+  // device mutex after a stream pointer is acquired and before the stream is
+  // queried for status.
+  // 3. It's the same for some implementations of Device::Sync, such as
+  // XlaDevice::Sync.
+  //
+  // However, these race conditions are acceptable because a stream (and
+  // therefore an XlaDevice) can only go from OK to not-OK, never the opposite,
+  // which means we will at worst report errors when there isn't any, never the
+  // opposite.
+
+  // If inc_num_deferred_ops_function has ever been called, ExecutorState must
+  // wait for all corresponding dec_num_deferred_ops_function calls to happen
+  // regardless of status. This ensures that dec_num_deferred_ops_function can
+  // safely use ExecutorState's resources.
+  {
+    mutex_lock lock(num_deferred_ops_mu_);
+    while (num_deferred_ops_ > 0) {
+      num_deferred_ops_cv_.wait(lock);
+    }
+  }
+
+  // An early exit for devices don't allow sync on completion. Ops that run on
+  // these devices should have used num_deferred_ops correctly to ensure the
+  // device has finished all relevant work at this point.
+  if (!device->AllowsSyncOnCompletion()) {
+    status.Update(device->CurrentStatus());
+    delete this;
+    runner([=]() { done_cb(status); });
+    return;
+  }
+
+  if (sync_on_finish_ && status.ok()) {
     // Block until the device has finished all queued operations. For
     // devices like GPUs that continue to execute Ops after their Compute
     // methods have completed, this ensures that control is not returned to
diff --git a/tensorflow/core/framework/op_kernel.h b/tensorflow/core/framework/op_kernel.h
index aa07cbd380..06b90964ad 100644
--- a/tensorflow/core/framework/op_kernel.h
+++ b/tensorflow/core/framework/op_kernel.h
@@ -646,6 +646,10 @@ class OpKernelContext {
     static const int kNoReservation = -1;
     // Values in [0,...) represent reservations for the indexed output.
     const int* forward_from_array = nullptr;
+
+    // For tracking actively running deferred ops.
+    std::function<void()> inc_num_deferred_ops_function = []() {};
+    std::function<void()> dec_num_deferred_ops_function = []() {};
   };
 
   // params must outlive the OpKernelContext.
@@ -1172,6 +1176,24 @@ class OpKernelContext {
 
   bool input_is_ref(int index) const;
 
+  // Used by OpKernel implementations to track actively running deferred ops.
+  //
+  // A deferred op is one whose Compute method returns (or whose ComputeAsync
+  // method invokes the callback) when work is scheduled onto a device. At that
+  // point, we don't know when the work will actually complete (or if it has
+  // already completed) on the device. These functions allow the executor to
+  // track the status of deferred ops and act accordingly.
+  //
+  // Deferred OpKernel implementations must use these methods to get two
+  // functions. It then must call these two functions in pairs, before and after
+  // device execution, respectively.
+  TF_MUST_USE_RESULT std::function<void()> inc_num_deferred_ops_function() {
+    return params_->inc_num_deferred_ops_function;
+  }
+  TF_MUST_USE_RESULT std::function<void()> dec_num_deferred_ops_function() {
+    return params_->dec_num_deferred_ops_function;
+  }
+
  private:
   Allocator* get_allocator(AllocatorAttributes attr);
 
-- 
GitLab


From b797012b39f89c45f13336b36c6b73e9ad93d815 Mon Sep 17 00:00:00 2001
From: Scott Zhu <scottzhu@google.com>
Date: Thu, 17 Jan 2019 14:46:46 -0800
Subject: [PATCH 0923/1095] Add AttentionMechanismV2 that follows Keras API
 style.

1. Adding new base class for AttentionMechanism that use keras.layer as parent.
2. All the new AttentionMechanismV2 will accept tensors from call() instead of
   as __init__ parameters.
3. memory_length has been update to memory_mask in v2 which is more align with
the keras API.
4. _luong_score() and _bahdanau_score() has been updated to not create variable
by itself. All the variable are now passed in as parameters.
5. All the existing unit tests are still passing. I haven't add new unit test
for v2 class yet, will add them once I update the AttentionWrapper to v2.

PiperOrigin-RevId: 229821776
---
 .../seq2seq/python/ops/attention_wrapper.py   | 962 ++++++++++++++++--
 1 file changed, 851 insertions(+), 111 deletions(-)

diff --git a/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py b/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
index 60ec3efffe..31c62d5849 100644
--- a/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
+++ b/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
@@ -28,6 +28,7 @@ from tensorflow.contrib.framework.python.framework import tensor_util
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.keras.engine.base_layer import Layer
 from tensorflow.python.layers import base as layers_base
 from tensorflow.python.layers import core as layers_core
 from tensorflow.python.ops import array_ops
@@ -72,77 +73,6 @@ class AttentionMechanism(object):
     raise NotImplementedError
 
 
-def _prepare_memory(memory, memory_sequence_length, check_inner_dims_defined):
-  """Convert to tensor and possibly mask `memory`.
-
-  Args:
-    memory: `Tensor`, shaped `[batch_size, max_time, ...]`.
-    memory_sequence_length: `int32` `Tensor`, shaped `[batch_size]`.
-    check_inner_dims_defined: Python boolean.  If `True`, the `memory`
-      argument's shape is checked to ensure all but the two outermost
-      dimensions are fully defined.
-
-  Returns:
-    A (possibly masked), checked, new `memory`.
-
-  Raises:
-    ValueError: If `check_inner_dims_defined` is `True` and not
-      `memory.shape[2:].is_fully_defined()`.
-  """
-  memory = nest.map_structure(
-      lambda m: ops.convert_to_tensor(m, name="memory"), memory)
-  if memory_sequence_length is not None:
-    memory_sequence_length = ops.convert_to_tensor(
-        memory_sequence_length, name="memory_sequence_length")
-  if check_inner_dims_defined:
-    def _check_dims(m):
-      if not m.get_shape()[2:].is_fully_defined():
-        raise ValueError("Expected memory %s to have fully defined inner dims, "
-                         "but saw shape: %s" % (m.name, m.get_shape()))
-    nest.map_structure(_check_dims, memory)
-  if memory_sequence_length is None:
-    seq_len_mask = None
-  else:
-    seq_len_mask = array_ops.sequence_mask(
-        memory_sequence_length,
-        maxlen=array_ops.shape(nest.flatten(memory)[0])[1],
-        dtype=nest.flatten(memory)[0].dtype)
-    seq_len_batch_size = (
-        tensor_shape.dimension_value(memory_sequence_length.shape[0])
-        or array_ops.shape(memory_sequence_length)[0])
-  def _maybe_mask(m, seq_len_mask):
-    rank = m.get_shape().ndims
-    rank = rank if rank is not None else array_ops.rank(m)
-    extra_ones = array_ops.ones(rank - 2, dtype=dtypes.int32)
-    m_batch_size = tensor_shape.dimension_value(
-        m.shape[0]) or array_ops.shape(m)[0]
-    if memory_sequence_length is not None:
-      message = ("memory_sequence_length and memory tensor batch sizes do not "
-                 "match.")
-      with ops.control_dependencies([
-          check_ops.assert_equal(
-              seq_len_batch_size, m_batch_size, message=message)]):
-        seq_len_mask = array_ops.reshape(
-            seq_len_mask,
-            array_ops.concat((array_ops.shape(seq_len_mask), extra_ones), 0))
-        return m * seq_len_mask
-    else:
-      return m
-  return nest.map_structure(lambda m: _maybe_mask(m, seq_len_mask), memory)
-
-
-def _maybe_mask_score(score, memory_sequence_length, score_mask_value):
-  if memory_sequence_length is None:
-    return score
-  message = ("All values in memory_sequence_length must greater than zero.")
-  with ops.control_dependencies(
-      [check_ops.assert_positive(memory_sequence_length, message=message)]):
-    score_mask = array_ops.sequence_mask(
-        memory_sequence_length, maxlen=array_ops.shape(score)[1])
-    score_mask_values = score_mask_value * array_ops.ones_like(score)
-    return array_ops.where(score_mask, score, score_mask_values)
-
-
 class _BaseAttentionMechanism(AttentionMechanism):
   """A base AttentionMechanism class providing common functionality.
 
@@ -205,12 +135,14 @@ class _BaseAttentionMechanism(AttentionMechanism):
           self._memory_layer.dtype).as_numpy_dtype(-np.inf)
     self._probability_fn = lambda score, prev: (  # pylint:disable=g-long-lambda
         probability_fn(
-            _maybe_mask_score(score, memory_sequence_length, score_mask_value),
+            _maybe_mask_score(score,
+                              memory_sequence_length=memory_sequence_length,
+                              score_mask_value=score_mask_value),
             prev))
     with ops.name_scope(
         name, "BaseAttentionMechanismInit", nest.flatten(memory)):
       self._values = _prepare_memory(
-          memory, memory_sequence_length,
+          memory, memory_sequence_length=memory_sequence_length,
           check_inner_dims_defined=check_inner_dims_defined)
       self._keys = (
           self.memory_layer(self._values) if self.memory_layer  # pylint: disable=not-callable
@@ -286,6 +218,199 @@ class _BaseAttentionMechanism(AttentionMechanism):
     return self.initial_alignments(batch_size, dtype)
 
 
+class _BaseAttentionMechanismV2(AttentionMechanism, Layer):
+  """A base AttentionMechanism class providing common functionality.
+
+  Common functionality includes:
+    1. Storing the query and memory layers.
+    2. Preprocessing and storing the memory.
+
+  Note that this layer only support Keras functional API since it takes multiple
+  input tensors, which is not available in sequential model.
+  """
+
+  def __init__(self,
+               probability_fn,
+               query_layer=None,
+               memory_layer=None,
+               **kwargs):
+    """Construct base AttentionMechanism class.
+
+    Args:
+      probability_fn: A `callable`. Converts the score and previous alignments
+        to probabilities. Its signature should be:
+        `probabilities = probability_fn(score, state)`.
+      query_layer:  (optional): Instance of `tf.keras.Layer`.  The layer's depth
+        must match the depth of `memory_layer`.  If `query_layer` is not
+        provided, the shape of `query` must match that of `memory_layer`.
+      memory_layer: (optional): Instance of `tf.keras.Layer`. The layer's
+        depth must match the depth of `query_layer`.
+        If `memory_layer` is not provided, the shape of `memory` must match
+        that of `query_layer`.
+      **kwargs: Dictionary that contains other common arguments for layer
+        creation.
+    """
+    if (query_layer is not None
+        and not isinstance(query_layer, layers_base.Layer)):
+      raise TypeError(
+          "query_layer is not a Layer: %s" % type(query_layer).__name__)
+    if (memory_layer is not None
+        and not isinstance(memory_layer, layers_base.Layer)):
+      raise TypeError(
+          "memory_layer is not a Layer: %s" % type(memory_layer).__name__)
+    self.query_layer = query_layer
+    self.memory_layer = memory_layer
+    if self.memory_layer is not None and "dtype" not in kwargs:
+      kwargs["dtype"] = self.memory_layer.dtype
+    super(_BaseAttentionMechanismV2, self).__init__(**kwargs)
+    if not callable(probability_fn):
+      raise TypeError("probability_fn must be callable, saw type: %s" %
+                      type(probability_fn).__name__)
+    self.probability_fn = probability_fn
+
+    self.keys = None
+    self.values = None
+    self.batch_size = None
+    self._memory_initialized = False
+    self._check_inner_dims_defined = True
+
+  def build(self, input_shape):
+    if self.query_layer is not None:
+      self.query_layer.build(input_shape)
+    if self.memory_layer is not None:
+      self.memory_layer.build(input_shape)
+    # dtype of the layer is known at this moment, create the score_mask_value if
+    # needed.
+    self.score_mask_value = dtypes.as_dtype(self.dtype).as_numpy_dtype(-np.inf)
+    self.built = True
+
+  def _setup_memory(self, memory, memory_mask=None):
+    """Pre-process the memory before actually query the memory.
+
+    This should only be called once at the first invocation of call().
+
+    Args:
+      memory: The memory to query; usually the output of an RNN encoder. This
+        tensor should be shaped `[batch_size, max_time, ...]`.
+      memory_mask: The boolean tensor with shape `[batch_size, max_time]`. For
+        any value equal to False, the corresponding value in memory should be
+        ignored.
+    """
+    if self._memory_initialized:
+      raise ValueError("The memory for the attention has already been setup.")
+    with ops.name_scope(
+        self.name, "BaseAttentionMechanismInit", nest.flatten(memory)):
+      self.values = _prepare_memory(
+          memory, memory_mask=memory_mask,
+          check_inner_dims_defined=self._check_inner_dims_defined)
+      if self.memory_layer is not None:
+        self.keys = self.memory_layer(self.values)
+      else:
+        self.keys = self.values
+      self.batch_size = (
+          tensor_shape.dimension_value(self.keys.shape[0]) or
+          array_ops.shape(self.keys)[0])
+      self.alignments_size = (tensor_shape.dimension_value(self.keys.shape[1])
+                              or array_ops.shape(self.keys)[1])
+      if memory_mask is not None:
+        self.probability_fn = lambda score, prev: (  # pylint:disable=g-long-lambda
+            self.probability_fn(_maybe_mask_score(
+                score, self.score_mask_value, memory_mask=memory_mask), prev))
+    self._memory_initialized = True
+
+  def call(self, inputs, mask=None, **kwargs):
+    """Base method to calculate the attention score.
+
+    Args:
+      inputs: a list of tensor that contains `query`, `state`, and `memory`.
+        `query` is the tensor of dtype matching `memory` and shape
+        `[batch_size, query_depth]`.
+        `state` is the tensor of dtype matching `memory` and shape
+        `[batch_size, alignments_size]`. (`alignments_size` is memory's
+        `max_time`).
+        `memory` is the memory to query; usually the output of an RNN encoder.
+        This tensor should be shaped `[batch_size, max_time, feature]`.
+      mask: optional bool tensor with shape `[batch, max_time]` for the mask of
+        memory. If it is not None, the corresponding item of the memory should
+        be filtered out during calculation.
+      **kwargs: Dict, other keyword arguments for the call method.
+    """
+    query, state, memory, memory_mask = self._process_inputs(inputs, mask)
+    if not self._memory_initialized:
+      self._setup_memory(memory, memory_mask=memory_mask)
+    return self.calculate_attention(query, state)
+
+  def calculate_attention(self, query, state):
+    raise NotImplementedError(
+        "calculate_attention need to be implemented by subclasses.")
+
+  def get_config(self):
+    config = {}
+    # Since the probability_fn is likely to be a wrapped function, the child
+    # class should preserve the original function and how its wrapped.
+
+    if self.query_layer is not None:
+      config["query_layer"] = {
+          "class_name": self.query_layer.__class__.__name__,
+          "config": self.query_layer.get_config(),
+      }
+    if self.memory_layer is not None:
+      config["memory_layer"] = {
+          "class_name": self.memory_layer.__class__.__name__,
+          "config": self.memory_layer.get_config(),
+      }
+    base_config = super(_BaseAttentionMechanismV2, self).get_config()
+    return dict(list(base_config.items()) + list(config.items()))
+
+  def _process_inputs(self, inputs, mask):
+    if len(inputs) != 3:
+      raise ValueError(
+          "Expect to have 3 inputs for attention, got %d" % len(inputs))
+    query, state, memory = inputs
+    return query, state, memory, mask
+
+  def _process_probability_fn(self, func_name):
+    """Helper method to retrieve the probably function by string input."""
+    valid_probability_fns = {
+        "softmax": nn_ops.softmax,
+        "hardmax": hardmax,
+    }
+    if func_name not in valid_probability_fns.keys():
+      raise ValueError("Invalid probability function: %s, options are %s" %
+                       (func_name, valid_probability_fns.keys()))
+    return valid_probability_fns[func_name]
+
+  @classmethod
+  def deserialize_inner_layer_from_config(cls, config, custom_objects):
+    """Helper method that reconstruct the query and memory from the config.
+
+    In the get_config() method, the query and memory layer configs are
+    serialized into dict for persistence, this method perform the reverse action
+    to reconstruct the layer from the config.
+
+    Args:
+      config: dict, the configs that will be used to reconstruct the object.
+      custom_objects: dict mapping class names (or function names) of custom
+        (non-Keras) objects to class/functions.
+    Returns:
+      config: dict, the config with layer instance created, which is ready to be
+        used as init parameters.
+    """
+    # Reconstruct the query and memory layer for parent class.
+    from tensorflow.python.keras.layers import deserialize as deserialize_layer  # pylint: disable=g-import-not-at-top
+    query_layer_config = config.pop("query_layer", None)
+    if query_layer_config:
+      query_layer = deserialize_layer(query_layer_config,
+                                      custom_objects=custom_objects)
+      config["query_layer"] = query_layer
+    memory_layer_config = config.pop("memory_layer", None)
+    if memory_layer_config:
+      memory_layer = deserialize_layer(memory_layer_config,
+                                       custom_objects=custom_objects)
+      config["memory_layer"] = memory_layer
+    return config
+
+
 def _luong_score(query, keys, scale):
   """Implements Luong-style (multiplicative) scoring function.
 
@@ -304,7 +429,7 @@ def _luong_score(query, keys, scale):
   Args:
     query: Tensor, shape `[batch_size, num_units]` to compare to keys.
     keys: Processed memory, shape `[batch_size, max_time, num_units]`.
-    scale: Whether to apply a scale to the score function.
+    scale: the optional tensor to scale the attention score.
 
   Returns:
     A `[batch_size, max_time]` tensor of unnormalized score values.
@@ -320,7 +445,6 @@ def _luong_score(query, keys, scale):
         "Query (%s) has units: %s.  Keys (%s) have units: %s.  "
         "Perhaps you need to set num_units to the keys' dimension (%s)?"
         % (query, depth, keys, key_units, key_units))
-  dtype = query.dtype
 
   # Reshape from [batch_size, depth] to [batch_size, 1, depth]
   # for matmul.
@@ -338,12 +462,8 @@ def _luong_score(query, keys, scale):
   score = math_ops.matmul(query, keys, transpose_b=True)
   score = array_ops.squeeze(score, [1])
 
-  if scale:
-    # Scalar used in weight scaling
-    g = variable_scope.get_variable(
-        "attention_g", dtype=dtype,
-        initializer=init_ops.ones_initializer, shape=())
-    score = g * score
+  if scale is not None:
+    score = scale * score
   return score
 
 
@@ -354,8 +474,8 @@ class LuongAttention(_BaseAttentionMechanism):
   as described in:
 
   Minh-Thang Luong, Hieu Pham, Christopher D. Manning.
-  "Effective Approaches to Attention-based Neural Machine Translation."
-  EMNLP 2015.  https://arxiv.org/abs/1508.04025
+  [Effective Approaches to Attention-based Neural Machine Translation.
+  EMNLP 2015.](https://arxiv.org/abs/1508.04025)
 
   The second is the scaled form inspired partly by the normalized form of
   Bahdanau attention.
@@ -429,13 +549,121 @@ class LuongAttention(_BaseAttentionMechanism):
         `max_time`).
     """
     with variable_scope.variable_scope(None, "luong_attention", [query]):
-      score = _luong_score(query, self._keys, self._scale)
+      attention_g = None
+      if self._scale:
+        attention_g = variable_scope.get_variable(
+            "attention_g", dtype=query.dtype,
+            initializer=init_ops.ones_initializer, shape=())
+      score = _luong_score(query, self._keys, attention_g)
     alignments = self._probability_fn(score, state)
     next_state = alignments
     return alignments, next_state
 
 
-def _bahdanau_score(processed_query, keys, normalize):
+class LuongAttentionV2(_BaseAttentionMechanismV2):
+  """Implements Luong-style (multiplicative) attention scoring.
+
+  This attention has two forms.  The first is standard Luong attention,
+  as described in:
+
+  Minh-Thang Luong, Hieu Pham, Christopher D. Manning.
+  [Effective Approaches to Attention-based Neural Machine Translation.
+  EMNLP 2015.](https://arxiv.org/abs/1508.04025)
+
+  The second is the scaled form inspired partly by the normalized form of
+  Bahdanau attention.
+
+  To enable the second form, construct the object with parameter
+  `scale=True`.
+  """
+
+  def __init__(self,
+               units,
+               scale=False,
+               probability_fn="softmax",
+               dtype=None,
+               name="LuongAttention",
+               **kwargs):
+    """Construct the AttentionMechanism mechanism.
+
+    Args:
+      units: The depth of the attention mechanism.
+      scale: Python boolean. Whether to scale the energy term.
+      probability_fn: (optional) string, the name of function to convert the
+        attention score to probabilities. The default is `softmax` which is
+        `tf.nn.softmax`. Other options is `hardmax`, which is hardmax() within
+        this module. Any other value will result intovalidation error. Default
+        to use `softmax`.
+      dtype: The data type for the memory layer of the attention mechanism.
+      name: Name to use when creating ops.
+      **kwargs: Dictionary that contains other common arguments for layer
+        creation.
+    """
+    # For LuongAttention, we only transform the memory layer; thus
+    # num_units **must** match expected the query depth.
+    self.probability_fn_name = probability_fn
+    probability_fn = self._process_probability_fn(self.probability_fn_name)
+    wrapped_probability_fn = lambda score, _: probability_fn(score)
+    if dtype is None:
+      dtype = dtypes.float32
+    super(LuongAttentionV2, self).__init__(
+        query_layer=None,
+        memory_layer=layers_core.Dense(
+            units, name="memory_layer", use_bias=False, dtype=dtype),
+        probability_fn=wrapped_probability_fn,
+        name=name,
+        dtype=dtype,
+        **kwargs)
+    self.units = units
+    self.scale = scale
+
+  def build(self, input_shape):
+    super(LuongAttentionV2, self).build(input_shape)
+    if self.scale:
+      self.scale_weight = self.add_weight(
+          "attention_g", initializer=init_ops.ones_initializer, shape=())
+    else:
+      self.scale_weight = None
+    self.built = True
+
+  def calculate_attention(self, query, state):
+    """Score the query based on the keys and values.
+
+    Args:
+      query: Tensor of dtype matching `self.values` and shape
+        `[batch_size, query_depth]`.
+      state: Tensor of dtype matching `self.values` and shape
+        `[batch_size, alignments_size]`
+        (`alignments_size` is memory's `max_time`).
+
+    Returns:
+      alignments: Tensor of dtype matching `self.values` and shape
+        `[batch_size, alignments_size]` (`alignments_size` is memory's
+        `max_time`).
+    """
+    score = _luong_score(query, self.keys, self.scale_weight)
+    alignments = self.probability_fn(score, state)
+    next_state = alignments
+    return alignments, next_state
+
+  def get_config(self):
+    config = {
+        "units": self.units,
+        "scale": self.scale,
+        "probability_fn": self.probability_fn_name,
+    }
+    base_config = super(LuongAttentionV2, self).get_config()
+    return dict(list(base_config.items()) + list(config.items()))
+
+  @classmethod
+  def from_config(cls, config, custom_objects=None):
+    config = _BaseAttentionMechanismV2.deserialize_inner_layer_from_config(
+        config, custom_objects=custom_objects)
+    return cls(**config)
+
+
+def _bahdanau_score(processed_query, keys, attention_v,
+                    attention_g=None, attention_b=None):
   """Implements Bahdanau-style (additive) scoring function.
 
   This attention has two forms.  The first is Bhandanau attention,
@@ -453,41 +681,28 @@ def _bahdanau_score(processed_query, keys, normalize):
    Training of Deep Neural Networks."
   https://arxiv.org/abs/1602.07868
 
-  To enable the second form, set `normalize=True`.
+  To enable the second form, set please pass in attention_g and attention_b.
 
   Args:
     processed_query: Tensor, shape `[batch_size, num_units]` to compare to keys.
     keys: Processed memory, shape `[batch_size, max_time, num_units]`.
-    normalize: Whether to normalize the score function.
+    attention_v: Tensor, shape `[num_units]`.
+    attention_g: Optional scalar tensor for normalization.
+    attention_b: Optional tensor with shape `[num_units]` for normalization.
 
   Returns:
     A `[batch_size, max_time]` tensor of unnormalized score values.
   """
-  dtype = processed_query.dtype
-  # Get the number of hidden units from the trailing dimension of keys
-  num_units = tensor_shape.dimension_value(
-      keys.shape[2]) or array_ops.shape(keys)[2]
   # Reshape from [batch_size, ...] to [batch_size, 1, ...] for broadcasting.
   processed_query = array_ops.expand_dims(processed_query, 1)
-  v = variable_scope.get_variable(
-      "attention_v", [num_units], dtype=dtype)
-  if normalize:
-    # Scalar used in weight normalization
-    g = variable_scope.get_variable(
-        "attention_g", dtype=dtype,
-        initializer=init_ops.constant_initializer(math.sqrt((1. / num_units))),
-        shape=())
-    # Bias added prior to the nonlinearity
-    b = variable_scope.get_variable(
-        "attention_b", [num_units], dtype=dtype,
-        initializer=init_ops.zeros_initializer())
-    # normed_v = g * v / ||v||
-    normed_v = g * v * math_ops.rsqrt(
-        math_ops.reduce_sum(math_ops.square(v)))
+  if attention_g is not None and attention_b is not None:
+    normed_v = attention_g * attention_v * math_ops.rsqrt(
+        math_ops.reduce_sum(math_ops.square(attention_v)))
     return math_ops.reduce_sum(
-        normed_v * math_ops.tanh(keys + processed_query + b), [2])
+        normed_v * math_ops.tanh(keys + processed_query + attention_b), [2])
   else:
-    return math_ops.reduce_sum(v * math_ops.tanh(keys + processed_query), [2])
+    return math_ops.reduce_sum(
+        attention_v * math_ops.tanh(keys + processed_query), [2])
 
 
 class BahdanauAttention(_BaseAttentionMechanism):
@@ -578,12 +793,145 @@ class BahdanauAttention(_BaseAttentionMechanism):
     """
     with variable_scope.variable_scope(None, "bahdanau_attention", [query]):
       processed_query = self.query_layer(query) if self.query_layer else query
-      score = _bahdanau_score(processed_query, self._keys, self._normalize)
+      attention_v = variable_scope.get_variable(
+          "attention_v", [self._num_units], dtype=query.dtype)
+      if not self._normalize:
+        attention_g = None
+        attention_b = None
+      else:
+        attention_g = variable_scope.get_variable(
+            "attention_g", dtype=query.dtype,
+            initializer=init_ops.constant_initializer(
+                math.sqrt((1. / self._num_units))),
+            shape=())
+        attention_b = variable_scope.get_variable(
+            "attention_b", [self._num_units], dtype=query.dtype,
+            initializer=init_ops.zeros_initializer())
+
+      score = _bahdanau_score(processed_query, self._keys, attention_v,
+                              attention_g=attention_g, attention_b=attention_b)
     alignments = self._probability_fn(score, state)
     next_state = alignments
     return alignments, next_state
 
 
+class BahdanauAttentionV2(_BaseAttentionMechanismV2):
+  """Implements Bahdanau-style (additive) attention.
+
+  This attention has two forms.  The first is Bahdanau attention,
+  as described in:
+
+  Dzmitry Bahdanau, Kyunghyun Cho, Yoshua Bengio.
+  "Neural Machine Translation by Jointly Learning to Align and Translate."
+  ICLR 2015. https://arxiv.org/abs/1409.0473
+
+  The second is the normalized form.  This form is inspired by the
+  weight normalization article:
+
+  Tim Salimans, Diederik P. Kingma.
+  "Weight Normalization: A Simple Reparameterization to Accelerate
+   Training of Deep Neural Networks."
+  https://arxiv.org/abs/1602.07868
+
+  To enable the second form, construct the object with parameter
+  `normalize=True`.
+  """
+
+  def __init__(self,
+               units,
+               normalize=False,
+               probability_fn="softmax",
+               dtype=None,
+               name="BahdanauAttention",
+               **kwargs):
+    """Construct the Attention mechanism.
+
+    Args:
+      units: The depth of the query mechanism.
+      normalize: Python boolean.  Whether to normalize the energy term.
+      probability_fn: (optional) string, the name of function to convert the
+        attention score to probabilities. The default is `softmax` which is
+        `tf.nn.softmax`. Other options is `hardmax`, which is hardmax() within
+        this module. Any other value will result into validation error. Default
+        to use `softmax`.
+      dtype: The data type for the query and memory layers of the attention
+        mechanism.
+      name: Name to use when creating ops.
+      **kwargs: Dictionary that contains other common arguments for layer
+        creation.
+    """
+    self.probability_fn_name = probability_fn
+    probability_fn = self._process_probability_fn(self.probability_fn_name)
+    wrapped_probability_fn = lambda score, _: probability_fn(score)
+    if dtype is None:
+      dtype = dtypes.float32
+    super(BahdanauAttentionV2, self).__init__(
+        query_layer=layers_core.Dense(
+            units, name="query_layer", use_bias=False, dtype=dtype),
+        memory_layer=layers_core.Dense(
+            units, name="memory_layer", use_bias=False, dtype=dtype),
+        probability_fn=wrapped_probability_fn,
+        name=name,
+        dtype=dtype,
+        **kwargs)
+    self.units = units
+    self.normalize = normalize
+
+  def build(self, input_shape):
+    super(BahdanauAttentionV2, self).build(input_shape)
+    self.attention_v = self.add_weight(
+        "attention_v", [self.units], dtype=self.dtype)
+    if self.normalize:
+      self.attention_g = self.add_weight(
+          "attention_g", initializer=init_ops.constant_initializer(
+              math.sqrt((1. / self.units))), shape=())
+      self.attention_b = self.add_weight(
+          "attention_b", shape=[self.units],
+          initializer=init_ops.zeros_initializer())
+    else:
+      self.attention_g = None
+      self.attention_b = None
+    self.built = True
+
+  def calculate_attention(self, query, state):
+    """Score the query based on the keys and values.
+
+    Args:
+      query: Tensor of dtype matching `self.values` and shape
+        `[batch_size, query_depth]`.
+      state: Tensor of dtype matching `self.values` and shape
+        `[batch_size, alignments_size]`
+        (`alignments_size` is memory's `max_time`).
+
+    Returns:
+      alignments: Tensor of dtype matching `self.values` and shape
+        `[batch_size, alignments_size]` (`alignments_size` is memory's
+        `max_time`).
+    """
+    processed_query = self.query_layer(query) if self.query_layer else query
+    score = _bahdanau_score(processed_query, self.keys, self.attention_v,
+                            attention_g=self.attention_g,
+                            attention_b=self.attention_b)
+    alignments = self.probability_fn(score, state)
+    next_state = alignments
+    return alignments, next_state
+
+  def get_config(self):
+    config = {
+        "units": self.units,
+        "normalize": self.normalize,
+        "probability_fn": self.probability_fn_name,
+    }
+    base_config = super(BahdanauAttentionV2, self).get_config()
+    return dict(list(base_config.items()) + list(config.items()))
+
+  @classmethod
+  def from_config(cls, config, custom_objects=None):
+    config = _BaseAttentionMechanismV2.deserialize_inner_layer_from_config(
+        config, custom_objects=custom_objects)
+    return cls(**config)
+
+
 def safe_cumprod(x, *args, **kwargs):
   """Computes cumprod of x in logspace using cumsum to avoid underflow.
 
@@ -766,6 +1114,34 @@ class _BaseMonotonicAttentionMechanism(_BaseAttentionMechanism):
         dtype=dtype)
 
 
+class _BaseMonotonicAttentionMechanismV2(_BaseAttentionMechanismV2):
+  """Base attention mechanism for monotonic attention.
+
+  Simply overrides the initial_alignments function to provide a dirac
+  distribution, which is needed in order for the monotonic attention
+  distributions to have the correct behavior.
+  """
+
+  def initial_alignments(self, batch_size, dtype):
+    """Creates the initial alignment values for the monotonic attentions.
+
+    Initializes to dirac distributions, i.e. [1, 0, 0, ...memory length..., 0]
+    for all entries in the batch.
+
+    Args:
+      batch_size: `int32` scalar, the batch_size.
+      dtype: The `dtype`.
+
+    Returns:
+      A `dtype` tensor shaped `[batch_size, alignments_size]`
+      (`alignments_size` is the values' `max_time`).
+    """
+    max_time = self._alignments_size
+    return array_ops.one_hot(
+        array_ops.zeros((batch_size,), dtype=dtypes.int32), max_time,
+        dtype=dtype)
+
+
 class BahdanauMonotonicAttention(_BaseMonotonicAttentionMechanism):
   """Monotonic attention mechanism with Bahadanau-style energy function.
 
@@ -860,7 +1236,22 @@ class BahdanauMonotonicAttention(_BaseMonotonicAttentionMechanism):
     with variable_scope.variable_scope(
         None, "bahdanau_monotonic_attention", [query]):
       processed_query = self.query_layer(query) if self.query_layer else query
-      score = _bahdanau_score(processed_query, self._keys, self._normalize)
+      attention_v = variable_scope.get_variable(
+          "attention_v", [self._num_units], dtype=query.dtype)
+      if not self._normalize:
+        attention_g = None
+        attention_b = None
+      else:
+        attention_g = variable_scope.get_variable(
+            "attention_g", dtype=query.dtype,
+            initializer=init_ops.constant_initializer(
+                math.sqrt((1. / self._num_units))),
+            shape=())
+        attention_b = variable_scope.get_variable(
+            "attention_b", [self._num_units], dtype=query.dtype,
+            initializer=init_ops.zeros_initializer())
+      score = _bahdanau_score(processed_query, self._keys, attention_v,
+                              attention_g=attention_g, attention_b=attention_b)
       score_bias = variable_scope.get_variable(
           "attention_score_bias", dtype=processed_query.dtype,
           initializer=self._score_bias_init)
@@ -870,6 +1261,140 @@ class BahdanauMonotonicAttention(_BaseMonotonicAttentionMechanism):
     return alignments, next_state
 
 
+class BahdanauMonotonicAttentionV2(_BaseMonotonicAttentionMechanismV2):
+  """Monotonic attention mechanism with Bahadanau-style energy function.
+
+  This type of attention enforces a monotonic constraint on the attention
+  distributions; that is once the model attends to a given point in the memory
+  it can't attend to any prior points at subsequence output timesteps.  It
+  achieves this by using the _monotonic_probability_fn instead of softmax to
+  construct its attention distributions.  Since the attention scores are passed
+  through a sigmoid, a learnable scalar bias parameter is applied after the
+  score function and before the sigmoid.  Otherwise, it is equivalent to
+  BahdanauAttention.  This approach is proposed in
+
+  Colin Raffel, Minh-Thang Luong, Peter J. Liu, Ron J. Weiss, Douglas Eck,
+  "Online and Linear-Time Attention by Enforcing Monotonic Alignments."
+  ICML 2017.  https://arxiv.org/abs/1704.00784
+  """
+
+  def __init__(self,
+               units,
+               normalize=False,
+               sigmoid_noise=0.,
+               sigmoid_noise_seed=None,
+               score_bias_init=0.,
+               mode="parallel",
+               dtype=None,
+               name="BahdanauMonotonicAttention",
+               **kwargs):
+    """Construct the Attention mechanism.
+
+    Args:
+      units: The depth of the query mechanism.
+      normalize: Python boolean. Whether to normalize the energy term.
+      sigmoid_noise: Standard deviation of pre-sigmoid noise. See the docstring
+        for `_monotonic_probability_fn` for more information.
+      sigmoid_noise_seed: (optional) Random seed for pre-sigmoid noise.
+      score_bias_init: Initial value for score bias scalar. It's recommended to
+        initialize this to a negative value when the length of the memory is
+        large.
+      mode: How to compute the attention distribution. Must be one of
+        'recursive', 'parallel', or 'hard'. See the docstring for
+        `tf.contrib.seq2seq.monotonic_attention` for more information.
+      dtype: The data type for the query and memory layers of the attention
+        mechanism.
+      name: Name to use when creating ops.
+      **kwargs: Dictionary that contains other common arguments for layer
+        creation.
+    """
+    # Set up the monotonic probability fn with supplied parameters
+    if dtype is None:
+      dtype = dtypes.float32
+    wrapped_probability_fn = functools.partial(
+        _monotonic_probability_fn, sigmoid_noise=sigmoid_noise, mode=mode,
+        seed=sigmoid_noise_seed)
+    super(BahdanauMonotonicAttentionV2, self).__init__(
+        query_layer=layers_core.Dense(
+            units, name="query_layer", use_bias=False, dtype=dtype),
+        memory_layer=layers_core.Dense(
+            units, name="memory_layer", use_bias=False, dtype=dtype),
+        probability_fn=wrapped_probability_fn,
+        name=name,
+        dtype=dtype,
+        **kwargs)
+    self.units = units
+    self.normalize = normalize
+    self.sigmoid_noise = sigmoid_noise
+    self.sigmoid_noise_seed = sigmoid_noise_seed
+    self.score_bias_init = score_bias_init
+    self.mode = mode
+
+  def build(self, input_shape):
+    super(BahdanauMonotonicAttentionV2, self).build(input_shape)
+    self.attention_v = self.add_weight(
+        "attention_v", [self.units], dtype=self.dtype)
+    self.attention_score_bias = self.add_weight(
+        "attention_score_bias", shape=(), dtype=self.dtype,
+        initializer=init_ops.constant_initializer(
+            self.score_bias_init, dtype=self.dtype))
+    if not self.normalize:
+      self.attention_g = None
+      self.attention_b = None
+    else:
+      self.attention_g = self.add_weight(
+          "attention_g", dtype=self.dtype,
+          initializer=init_ops.constant_initializer(
+              math.sqrt((1. / self.units))),
+          shape=())
+      self.attention_b = self.add_weight(
+          "attention_b", [self.units], dtype=self.dtype,
+          initializer=init_ops.zeros_initializer())
+    self.built = True
+
+  def calculate_attention(self, query, state):
+    """Score the query based on the keys and values.
+
+    Args:
+      query: Tensor of dtype matching `self.values` and shape
+        `[batch_size, query_depth]`.
+      state: Tensor of dtype matching `self.values` and shape
+        `[batch_size, alignments_size]`
+        (`alignments_size` is memory's `max_time`).
+
+    Returns:
+      alignments: Tensor of dtype matching `self.values` and shape
+        `[batch_size, alignments_size]` (`alignments_size` is memory's
+        `max_time`).
+    """
+    processed_query = self.query_layer(query) if self.query_layer else query
+    score = _bahdanau_score(processed_query, self.keys, self.attention_v,
+                            attention_g=self.attention_g,
+                            attention_b=self.attention_b)
+    score += self.attention_score_bias
+    alignments = self.probability_fn(score, state)
+    next_state = alignments
+    return alignments, next_state
+
+  def get_config(self):
+    config = {
+        "units": self.units,
+        "normalize": self.normalize,
+        "sigmoid_noise": self.sigmoid_noise,
+        "sigmoid_noise_seed": self.sigmoid_noise_seed,
+        "score_bias_init": self.score_bias_init,
+        "mode": self.mode,
+    }
+    base_config = super(BahdanauMonotonicAttentionV2, self).get_config()
+    return dict(list(base_config.items()) + list(config.items()))
+
+  @classmethod
+  def from_config(cls, config, custom_objects=None):
+    config = _BaseAttentionMechanismV2.deserialize_inner_layer_from_config(
+        config, custom_objects=custom_objects)
+    return cls(**config)
+
+
 class LuongMonotonicAttention(_BaseMonotonicAttentionMechanism):
   """Monotonic attention mechanism with Luong-style energy function.
 
@@ -960,7 +1485,12 @@ class LuongMonotonicAttention(_BaseMonotonicAttentionMechanism):
     """
     with variable_scope.variable_scope(None, "luong_monotonic_attention",
                                        [query]):
-      score = _luong_score(query, self._keys, self._scale)
+      attention_g = None
+      if self._scale:
+        attention_g = variable_scope.get_variable(
+            "attention_g", dtype=query.dtype,
+            initializer=init_ops.ones_initializer, shape=())
+      score = _luong_score(query, self._keys, attention_g)
       score_bias = variable_scope.get_variable(
           "attention_score_bias", dtype=query.dtype,
           initializer=self._score_bias_init)
@@ -970,6 +1500,125 @@ class LuongMonotonicAttention(_BaseMonotonicAttentionMechanism):
     return alignments, next_state
 
 
+class LuongMonotonicAttentionV2(_BaseMonotonicAttentionMechanismV2):
+  """Monotonic attention mechanism with Luong-style energy function.
+
+  This type of attention enforces a monotonic constraint on the attention
+  distributions; that is once the model attends to a given point in the memory
+  it can't attend to any prior points at subsequence output timesteps.  It
+  achieves this by using the _monotonic_probability_fn instead of softmax to
+  construct its attention distributions.  Otherwise, it is equivalent to
+  LuongAttention.  This approach is proposed in
+
+  [Colin Raffel, Minh-Thang Luong, Peter J. Liu, Ron J. Weiss, Douglas Eck,
+  "Online and Linear-Time Attention by Enforcing Monotonic Alignments."
+  ICML 2017.](https://arxiv.org/abs/1704.00784)
+  """
+
+  def __init__(self,
+               units,
+               scale=False,
+               sigmoid_noise=0.,
+               sigmoid_noise_seed=None,
+               score_bias_init=0.,
+               mode="parallel",
+               dtype=None,
+               name="LuongMonotonicAttention",
+               **kwargs):
+    """Construct the Attention mechanism.
+
+    Args:
+      units: The depth of the query mechanism.
+      scale: Python boolean.  Whether to scale the energy term.
+      sigmoid_noise: Standard deviation of pre-sigmoid noise.  See the docstring
+        for `_monotonic_probability_fn` for more information.
+      sigmoid_noise_seed: (optional) Random seed for pre-sigmoid noise.
+      score_bias_init: Initial value for score bias scalar.  It's recommended to
+        initialize this to a negative value when the length of the memory is
+        large.
+      mode: How to compute the attention distribution.  Must be one of
+        'recursive', 'parallel', or 'hard'.  See the docstring for
+        `tf.contrib.seq2seq.monotonic_attention` for more information.
+      dtype: The data type for the query and memory layers of the attention
+        mechanism.
+      name: Name to use when creating ops.
+      **kwargs: Dictionary that contains other common arguments for layer
+        creation.
+    """
+    # Set up the monotonic probability fn with supplied parameters
+    if dtype is None:
+      dtype = dtypes.float32
+    wrapped_probability_fn = functools.partial(
+        _monotonic_probability_fn, sigmoid_noise=sigmoid_noise, mode=mode,
+        seed=sigmoid_noise_seed)
+    super(LuongMonotonicAttentionV2, self).__init__(
+        query_layer=None,
+        memory_layer=layers_core.Dense(
+            units, name="memory_layer", use_bias=False, dtype=dtype),
+        probability_fn=wrapped_probability_fn,
+        name=name,
+        dtype=dtype,
+        **kwargs)
+    self.units = units
+    self.scale = scale
+    self.sigmoid_noise = sigmoid_noise
+    self.sigmoid_noise_seed = sigmoid_noise_seed
+    self.score_bias_init = score_bias_init
+    self.mode = mode
+
+  def build(self, input_shape):
+    super(LuongMonotonicAttentionV2, self).build(input_shape)
+    if self.scale:
+      self.attention_g = self.add_weight(
+          "attention_g", initializer=init_ops.ones_initializer, shape=())
+    else:
+      self.attention_g = None
+    self.attention_score_bias = self.add_weight(
+        "attention_score_bias", shape=(),
+        initializer=init_ops.constant_initializer(
+            self.score_bias_init, dtype=self.dtype))
+    self.built = True
+
+  def calculate_attention(self, query, state):
+    """Score the query based on the keys and values.
+
+    Args:
+      query: Tensor of dtype matching `self.values` and shape
+        `[batch_size, query_depth]`.
+      state: Tensor of dtype matching `self.values` and shape
+        `[batch_size, alignments_size]`
+        (`alignments_size` is memory's `max_time`).
+
+    Returns:
+      alignments: Tensor of dtype matching `self.values` and shape
+        `[batch_size, alignments_size]` (`alignments_size` is memory's
+        `max_time`).
+    """
+    score = _luong_score(query, self.keys, self.attention_g)
+    score += self.attention_score_bias
+    alignments = self.probability_fn(score, state)
+    next_state = alignments
+    return alignments, next_state
+
+  def get_config(self):
+    config = {
+        "units": self.units,
+        "normalize": self.normalize,
+        "sigmoid_noise": self.sigmoid_noise,
+        "sigmoid_noise_seed": self.sigmoid_noise_seed,
+        "score_bias_init": self.score_bias_init,
+        "mode": self.mode,
+    }
+    base_config = super(LuongMonotonicAttentionV2, self).get_config()
+    return dict(list(base_config.items()) + list(config.items()))
+
+  @classmethod
+  def from_config(cls, config, custom_objects=None):
+    config = _BaseAttentionMechanismV2.deserialize_inner_layer_from_config(
+        config, custom_objects=custom_objects)
+    return cls(**config)
+
+
 class AttentionWrapperState(
     collections.namedtuple("AttentionWrapperState",
                            ("cell_state", "attention", "time", "alignments",
@@ -1026,6 +1675,97 @@ class AttentionWrapperState(
         super(AttentionWrapperState, self)._replace(**kwargs))
 
 
+def _prepare_memory(memory, memory_sequence_length=None, memory_mask=None,
+                    check_inner_dims_defined=True):
+  """Convert to tensor and possibly mask `memory`.
+
+  Args:
+    memory: `Tensor`, shaped `[batch_size, max_time, ...]`.
+    memory_sequence_length: `int32` `Tensor`, shaped `[batch_size]`.
+    memory_mask: `boolean` tensor with shape [batch_size, max_time]. The memory
+      should be skipped when the corresponding mask is False.
+    check_inner_dims_defined: Python boolean.  If `True`, the `memory`
+      argument's shape is checked to ensure all but the two outermost
+      dimensions are fully defined.
+
+  Returns:
+    A (possibly masked), checked, new `memory`.
+
+  Raises:
+    ValueError: If `check_inner_dims_defined` is `True` and not
+      `memory.shape[2:].is_fully_defined()`.
+  """
+  memory = nest.map_structure(
+      lambda m: ops.convert_to_tensor(m, name="memory"), memory)
+  if memory_sequence_length is not None and memory_mask is not None:
+    raise ValueError("memory_sequence_length and memory_mask can't be provided "
+                     "at same time.")
+  if memory_sequence_length is not None:
+    memory_sequence_length = ops.convert_to_tensor(
+        memory_sequence_length, name="memory_sequence_length")
+  if check_inner_dims_defined:
+    def _check_dims(m):
+      if not m.get_shape()[2:].is_fully_defined():
+        raise ValueError("Expected memory %s to have fully defined inner dims, "
+                         "but saw shape: %s" % (m.name, m.get_shape()))
+    nest.map_structure(_check_dims, memory)
+  if memory_sequence_length is None and memory_mask is None:
+    seq_len_mask = None
+    seq_len_batch_size = None
+  elif memory_sequence_length is not None:
+    seq_len_mask = array_ops.sequence_mask(
+        memory_sequence_length,
+        maxlen=array_ops.shape(nest.flatten(memory)[0])[1],
+        dtype=nest.flatten(memory)[0].dtype)
+    seq_len_batch_size = (
+        tensor_shape.dimension_value(memory_sequence_length.shape[0])
+        or array_ops.shape(memory_sequence_length)[0])
+  else:
+    # For memory_mask is not None
+    seq_len_mask = memory_mask
+    seq_len_batch_size = (
+        tensor_shape.dimension_value(memory_mask.shape[0])
+        or array_ops.shape(memory_mask)[0])
+  def _maybe_mask(m, seq_len_mask):
+    """Mask the memory based on the memory mask."""
+    rank = m.get_shape().ndims
+    rank = rank if rank is not None else array_ops.rank(m)
+    extra_ones = array_ops.ones(rank - 2, dtype=dtypes.int32)
+    m_batch_size = tensor_shape.dimension_value(
+        m.shape[0]) or array_ops.shape(m)[0]
+    if seq_len_batch_size is not None:
+      message = ("memory_sequence_length and memory tensor batch sizes do not "
+                 "match.")
+      with ops.control_dependencies([
+          check_ops.assert_equal(
+              seq_len_batch_size, m_batch_size, message=message)]):
+        seq_len_mask = array_ops.reshape(
+            seq_len_mask,
+            array_ops.concat((array_ops.shape(seq_len_mask), extra_ones), 0))
+        return m * seq_len_mask
+    else:
+      return m
+  return nest.map_structure(lambda m: _maybe_mask(m, seq_len_mask), memory)
+
+
+def _maybe_mask_score(score, memory_sequence_length=None, memory_mask=None,
+                      score_mask_value=None):
+  """Mask the attention score based on the masks."""
+  if memory_sequence_length is None and memory_mask is None:
+    return score
+  if memory_sequence_length is not None and memory_mask is not None:
+    raise ValueError("memory_sequence_length and memory_mask can't be provided "
+                     "at same time.")
+  if memory_sequence_length is not None:
+    message = "All values in memory_sequence_length must greater than zero."
+    with ops.control_dependencies(
+        [check_ops.assert_positive(memory_sequence_length, message=message)]):
+      memory_mask = array_ops.sequence_mask(
+          memory_sequence_length, maxlen=array_ops.shape(score)[1])
+  score_mask_values = score_mask_value * array_ops.ones_like(score)
+  return array_ops.where(memory_mask, score, score_mask_values)
+
+
 def hardmax(logits, name=None):
   """Returns batched one-hot vectors.
 
-- 
GitLab


From 92122d1818bb8dbbab371506f81b2f7344bf8595 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 17 Jan 2019 14:54:37 -0800
Subject: [PATCH 0924/1095] - Add post_training_quantization to index under how
 it works - Add post_training_quantization to devguide as an optional step -
 Add GPU dev preview to index as an update - Remove testimonials

PiperOrigin-RevId: 229823165
---
 tensorflow/lite/g3doc/_index.yaml             | 122 +++++++-----------
 tensorflow/lite/g3doc/devguide.md             |  45 ++++++-
 .../landing-page/facial_contour_detection.png | Bin 0 -> 294705 bytes
 3 files changed, 94 insertions(+), 73 deletions(-)
 create mode 100644 tensorflow/lite/g3doc/images/landing-page/facial_contour_detection.png

diff --git a/tensorflow/lite/g3doc/_index.yaml b/tensorflow/lite/g3doc/_index.yaml
index 1b3f1d616a..7153b7c6f6 100644
--- a/tensorflow/lite/g3doc/_index.yaml
+++ b/tensorflow/lite/g3doc/_index.yaml
@@ -4,7 +4,7 @@ description: <!--no description-->
 landing_page:
   custom_css_path: /site-assets/css/style.css
   rows:
-  - heading: TensorFlow Lite is for mobile and embedded devices.
+  - heading: TensorFlow Lite is for mobile and embedded devices
     description: >
       <p style="max-width: 75%;">
         TensorFlow Lite is the official solution for running machine learning
@@ -13,9 +13,6 @@ landing_page:
         iOS, and other operating systems.
       </p>
       <style>
-      .tfo-landing-row-heading {
-        padding-top: 0 !important;
-      }
       .tfo-landing-row-heading h2 {
         margin-top: 0 !important;
       }
@@ -71,58 +68,16 @@ landing_page:
           icon_name: lens
           foreground: theme
 
-  - classname: devsite-landing-row-logos tfo-landing-row-heading
-    heading: Companies using TensorFlow Lite
-    items:
-    - custom_image:
-        path: ./images/landing-page/photos_logo.png
-      path: https://www.photos.google.com
-    - custom_image:
-        path: ./images/landing-page/gboard_logo.png
-      path: https://play.google.com/store/apps/details?id=com.google.android.inputmethod.latin&hl=en_US
-    - custom_image:
-        path: ./images/landing-page/gmail_logo.png
-      path: https://www.google.com/gmail/
-    - custom_image:
-        path: ./images/landing-page/assistant_logo.png
-      path: https://assistant.google.com/
-
-  - classname: devsite-landing-row-logos
-    items:
-    - custom_image:
-        path: ./images/landing-page/vsco_logo.png
-      path: https://vsco.co
-    - custom_image:
-        path: ./images/landing-page/shazam_logo.png
-      path: https://www.shazam.com/
-    - custom_image:
-        path: ./images/landing-page/nest_logo.png
-      path: https://nest.com/
-    - custom_image:
-        path: ./images/landing-page/loseit_logo.png
-      path: https://www.loseit.com/
-
-  - classname: devsite-landing-row-no-image-background devsite-landing-row-67
-    background: grey
-    items:
-    - description: >
-        <em>“TensorFlow Lite helped us introduce machine learning and AI into our
-        app in an easy and streamlined way. We could reduce the size of our
-        models while keeping the accuracy high. This helped us create an amazing
-        fishing experience for our users by allowing them to identify any fish
-        species with just a photo.”</em>
-      image_path: ./images/landing-page/fishbrain_logo_big.png
-
   - heading: How it works
     items:
-    - heading: Build
+    - heading: Pick a model
       icon:
         icon_name: build
       description: >
-        Build a new model or retrain an existing one, such as using transfer learning.
+        Pick a new model or retrain an existing one.
       buttons:
-      - label: Read the developer guide
-        path: /lite/devguide
+      - label: Pick
+        path: /lite/devguide#1_choose_a_model
         classname: button button-primary tfo-button-primary
     - heading: Convert
       icon:
@@ -131,18 +86,29 @@ landing_page:
         Convert a TensorFlow model into a compressed flat buffer with the
         TensorFlow Lite Converter.
       buttons:
-      - label: Read the converter guide
-        path: /lite/convert/
+      - label: Convert
+        path: /lite/devguide#2_convert_the_model_format
         classname: button button-primary tfo-button-primary
     - heading: Deploy
+      icon:
+        icon_name: settings_cell
+      description: >
+        Take the compressed <code>.tflite</code> file and load it into a mobile or embedded device.
+      buttons:
+      - label: Deploy
+        path: /lite/devguide#3_use_the_tensorflow_lite_model_for_inference_in_a_mobile_app
+        classname: button button-primary tfo-button-primary
+    - heading: Optimize
       icon:
         icon_name: bolt
       description: >
-        Take the compressed <code>.tflite</code> file and load it into a mobile
-        or embedded device.<br/>
-        See the <a href="#build-your-first-tensorflow-lite-app">tutorials below</a> to build an app.
+        [optional] Quantize by converting 32-bit floats to more efficient 8-bit integers or run on GPU.
+      buttons:
+      - label: Optimize
+        path: /lite/devguide#4_optimize_your_model_optional
+        classname: button button-primary tfo-button-primary
 
-  - heading: Build your first TensorFlow Lite app
+  - heading: Build your first TensorFlow Lite app with Codelabs
     background: grey
     items:
     - classname: tfo-landing-row-item-inset-white
@@ -160,28 +126,40 @@ landing_page:
         We love to hear what you're working on—it may even get highlighted on
         our social media! <a href="https://groups.google.com/a/tensorflow.org/forum/#!forum/discuss" class="external">Tell us</a>.
 
-  - classname: devsite-landing-row-no-image-background devsite-landing-row-67
+  - classname: devsite-landing-row-logos tfo-landing-row-heading
+    heading: TensorFlow Lite users
     items:
-    - description: >
-        <p>
-          <em>“The release of TensorFlow Lite has allowed us to deploy an engaging
-          real-time experience to our users that eliminates the requirement
-          for a data connection. TensorFlow Lite’s ability to compress and
-          optimize the TensorFlow graph for mobile deployment has been
-          transformative in expanding the capabilities of Snap It.</em>
-        </p>
-        <p>
-          <em>Through TensorFlow Lite, our users can now enjoy a state of the
-          art, computer-vision-based food logging experience without worrying
-          about signal strength. We look forward to future collaborations
-          with the TensorFlow Lite team.”</em>
-        </p>
-      image_path: ./images/landing-page/loseit_logo_big.png
+    - custom_image:
+        path: ./images/landing-page/photos_logo.png
+    - custom_image:
+        path: ./images/landing-page/gboard_logo.png
+    - custom_image:
+        path: ./images/landing-page/gmail_logo.png
+    - custom_image:
+        path: ./images/landing-page/assistant_logo.png
+
+  - classname: devsite-landing-row-logos
+    items:
+    - custom_image:
+        path: ./images/landing-page/vsco_logo.png
+    - custom_image:
+        path: ./images/landing-page/shazam_logo.png
+    - custom_image:
+        path: ./images/landing-page/nest_logo.png
+    - custom_image:
+        path: ./images/landing-page/loseit_logo.png
+
 
   - classname: devsite-landing-row-cards
     background: grey
     heading: Updates
     items:
+    - heading: "TensorFlow Lite Now Faster with Mobile GPUs (Developer Preview)"
+      image_path: ./images/landing-page/facial_contour_detection.png
+      path: https://medium.com/tensorflow/tensorflow-lite-now-faster-with-mobile-gpus-developer-preview-e15797e6dee7
+      buttons:
+      - label: Read more
+        path: https://medium.com/tensorflow/tensorflow-lite-now-faster-with-mobile-gpus-developer-preview-e15797e6dee7
     - heading: "AI in motion: react in the real world"
       image_path: ./images/landing-page/ai_in_motion.png
       path: https://cloud.google.com/blog/products/ai-machine-learning/ai-motion-designing-simple-system-see-understand-and-react-real-world-part-ii
diff --git a/tensorflow/lite/g3doc/devguide.md b/tensorflow/lite/g3doc/devguide.md
index fdd02638f9..cbad036407 100644
--- a/tensorflow/lite/g3doc/devguide.md
+++ b/tensorflow/lite/g3doc/devguide.md
@@ -180,7 +180,6 @@ bazel run tensorflow/lite/tools:visualize -- model.tflite model_viz.html
 This generates an interactive HTML page listing subgraphs, operations, and a
 graph visualization.
 
-
 ## 3. Use the TensorFlow Lite model for inference in a mobile app
 
 After completing the prior steps, you should now have a `.tflite` model file.
@@ -221,3 +220,47 @@ devices. To use the converter, refer to the
 Compile Tensorflow Lite for a Raspberry Pi by following the
 [RPi build instructions](rpi.md) This compiles a static library file (`.a`) used
 to build your app. There are plans for Python bindings and a demo app.
+
+## 4. Optimize your model (optional)
+
+There are two options. If you plan to run on CPU, we recommend that you quantize
+your weights and activation tensors. If the hardware is available, another
+option is to run on GPU for massively parallelizable workloads.
+
+### Quantization
+Compress your model size by lowering the precision of the parameters (i.e.
+neural network weights) from their training-time 32-bit floating-point
+representations into much smaller and efficient 8-bit integer ones.
+
+This will execute the heaviest computations fast in lower precision, but the
+most sensitive ones with higher precision, thus typically resulting in little to
+no final accuracy losses for the task, yet a significant speed-up over pure
+floating-point execution.
+
+The post-training quantization technique is integrated into the TensorFlow Lite
+conversion tool. Getting started is easy: after building your TensorFlow model,
+simply enable the ‘post_training_quantize’ flag in the TensorFlow Lite
+conversion tool. Assuming that the saved model is stored in saved_model_dir, the
+quantized tflite flatbuffer can be generated in command line:
+
+```
+converter=tf.contrib.lite.TocoConverter.from_saved_model(saved_model_dir)
+converter.post_training_quantize=True
+tflite_quantized_model=converter.convert()
+open(“quantized_model.tflite”, “wb”).write(tflite_quantized_model)
+```
+
+Read the full documentation [here](performance/post_training_quantization) and see a tutorial [here](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/tutorials/post_training_quant.ipynb).
+
+### GPU
+Run on GPU GPUs are designed to have high throughput for massively
+parallelizable workloads. Thus, they are well-suited for deep neural nets, which
+consist of a huge number of operators, each working on some input tensor(s) that
+can be easily divided into smaller workloads and carried out in parallel,
+typically resulting in lower latency.
+
+Another benefit with GPU inference is its power efficiency. GPUs carry out the
+computations in a very efficient and optimized manner, so that they consume less
+power and generate less heat than when the same task is run on CPUs.
+
+Read the tutorial [here](performance/gpu) and full documentation [here](performance/gpu_advanced).
diff --git a/tensorflow/lite/g3doc/images/landing-page/facial_contour_detection.png b/tensorflow/lite/g3doc/images/landing-page/facial_contour_detection.png
new file mode 100644
index 0000000000000000000000000000000000000000..27bb49826ff3246bd5a971e7ecd0926121dbf749
GIT binary patch
literal 294705
zcmeAS@N?(olHy`uVBq!ia0y~yU{+vYU<&16Vqjoc*~Rvgfq`pFrn7T^OJ;FFPGThk
zgGcAoY2_iom14)=SL^S+AS5tBs7iPV2a5p5Tn4U72i-iFSe$w^J8p;u9^AM<$yfHG
zqaf>XrV|kYfjP}djK3Rrv`$$>2#8Mo+pK!-&AXen)#odp*Pi=+_x|^LzpMYpFx*m}
z7oz26%CJe5E3%}+UfWP}rt2GKmMaZBRSl0nJ`LKxVg(07L+AJVkDbL87#Oy{tqfZE
z<Np5t*Dr{0Jgs3+;bI74EvhqPC}LvBeCnOb#h@e3a7IjX2^Yf}A%+>7jx0%Gc%#8!
zaQXO7UWOSfAJ_0R%-D3}1p`BlK!eNUk0~!1uJkh;IQ4M3HbcfPh6fsg9V!f40vJ9#
zO#i^bu;nX*fvR<NBSS+O6T_O||BojBGd~|PRlR@ux<?-)85kNwG$wf;-E;K+^U^=>
zPl#!&XU$+_&~+A*;$_HUIMKO=^`X|ZCFgqPBur#TXfmF%uu%BSbaoa04GrQF>#Vfr
zNg5u^T$Hfxl>5Qnxd}I$GN$eikP|UJXmj&>)TuQV_Mx$g;ZJ8y7o2Ty&~nj_nZi2f
zyEy;P6JBsB^;Aaa`OHlTS9@-L+J8p%(W+0Pk{cJUlB~R6EqqEi)Rv>A(NQZ_X!)s$
zT7N?S1r<)wir3%^eEDgP#MQ`Fop06ql)OsqXYSdsg0n54gGo%KncJd?!$I*(C<6nF
z&IG+_otKP7bnI7#=%f^?*0gKtq`Ry0Hyl}#a_vWcT{*u1gF}hO{jEmn4GXv!Tt37m
z+p}5dFkX1|Yu(K4%op4l&h2}wooe}5ijRSTUnGEC?M0*Dj7HNN&Bi+##S0WT4k-vF
zDRMqzls8h4PT9aQ?U0s6gVGHq>jg634%k_6xId6EIh1L^KmCE!otE7n1gaLu#~l=?
zX$WL-?BI}1RH*7OOlrKLc#cE*qhr$q4iza+r=STXE=?K}m!6PX!lCJQYT{lG-cQCw
zT$cSYj#d+Yd*~ZA>~XbQEaBxmbA@77r=RQ174lMj#}*Y|EGbc()&6Yp><d9#xNLjO
z7CgV0{UUY?du=OUazmh@>=~BL$8SV1h;^zaXU^b>?h{Q?jSvlMH%)$fBX<qE^bxy<
z92RVSt(%WmK6EN9w6M!l%(Ki>UMEv7aJ?b>@ZDoyAGH<S|4_1r@gCp%&gYMWKfe6o
z`h%&4bDvbbP`zNi`2GI>4hjM#9APZwP8y1x6FeiFZg|{qU7@~3z>2fBB~fYO3imC_
zRl>@hHjA1f9CVcB)aEJQQ~D-sKWT+Z@+4!;8B=^FO$eLp<<ahu?z!FLyyxN+-c77G
zC(C$>O%R`KKKc9P*(c3U{CyJGrFOLIk#FbHl%TLcFU`^^-BaABl!s_to$|_YuB&KZ
zc#!u>_m$IE?!Dq~qB~1hx52D^*^=BvvJ19ecz5~U#g`@CR_*hA!Y`O#%zh#M!vBT*
zRf#OVtQgbATU=YYwz6%F&RT!<<SVgPb+0~O`FzFuO8@Hn0rFQAvLmusGCr70%MQty
zbLqu3iEAktR+(;DY1w&KY?f5Ier^>DJRPL%;k{t$`mO%2o5JLolV=z*Nz@#ZI#zeg
z)y;nLfh|czGiRPrdM230UhEO=T|KL8PuaS%)^Cz;<z#)12eq$OzIJm>vCnev=bu#$
zd)7^EnQZ2>YIaHXrH*$gv3=_Twg+sF`@Zg7y6v>M331czh3$3Qd;FJ0ZS>#BU)^8I
zU&mjOpL1r;8`GKgI}1K-Jju@!z#QG2(X7h$n`fOgkF=Czn}iqdX&!wEKB<=yg*`^S
z+ob23N*Sa{{FdnM*)BcbxbEbow4WP~tO*K}&OUMEfXWq@%uUyq98S4%<&?>}lC7D0
zUnU!ON#52uo3lG|_00)8H~!@Ly!6@DXZ$)R#B#cMy61|$6;qxTHjQ)IYaMCb_=uNL
zg=;^p$yzHHd3kN^TJfmx$om@=Z*0ijlv8%|&KB3)*j(cr?VS7@3(NA#ly=9K%?<t#
zY_rT`S%~k;ZOd=|zWMyd^;_SqURbzTtSQSVsj$6TllAG!+Th#E>xxTCMXcYI*SuKr
zROH!~TPttAT)TO#@%^2Le)e(SRlSkg>OJ}N?98&uEtkvucRxQ>F!$lEb6c-|T|ImE
z>@#c6<sP_u$M)#nOM7RZO>y67zDd9Of$bmLu!=d~emr^kY~=@=XFr!u-y1&dy5f5A
z%WIc$FMmJ(-t*j7XFuP2efRO<Po5utUR-|s`jP3+qo4ksl6*|CV3p<hzd3)5{__3a
z`>p)T^q=#;$a8++i4eOW)WO=uBF0w6lF53Mbt=0prx~vw|2kedu0q~Op-Vzt{Ji{k
zrEZ-yee!9i>79iJ-#4Tt=04o=VaA6~AC?Pl6WrACO!7s~2FcrLHnTDfgN-&D3MuWJ
zcw0%{J<?UT=U1v&V%U{6U6JP(&0RFzZToS9q-V!vx_%10mfvmpEpbli9HVunakh1S
zHNJm5oD!T0o%|nNIg;9WebSf7E}p(8k39MD<kgdhHyt;ZpLsX`vDNYfzw0rpckV1+
zf9CRh^?BuK`+n8j`}0`y$CQ>S&YJNR6ZdG{E6_6EYE;@Gf8hSfdesLU+ql_U?j1^c
z_}S$BmPcE}U*3Cp`x3iRjj@*zn=9j@qKmW7TsUJpQ#o>CB>SdgM>rq-$_Xm-`li`A
zO~w1twgVZC@0$Ep&fVg0Yx9NThkdQin+hVg%1k^z)m(E1*UCq`E=BDOyE*&u<lu>)
z3j_Zwdc1Si&PjJ(cjt=dNBptTw|Ku?b%*L7)wSNf)7C}Z+I=Sf=DnvUK3#dbR{KoY
zll3p+B{zL~pml21n$Z30%3@;o{Q5hq#q{jYwx8Oce_v}`vsO1idxB`m`p;3@({1zi
zX4D>Mo6egmy-$>jYrUAI*i@0)n<ws9d}UuQ?^Q2xd&;(z$(wIKyBp0BtrER!+pj#^
zh&7RO!tZR)J+`gw?V7i(x9`87vH!8x$t{y?tu?KuT3351d(A#|=Jd8HwVLNO_D4UA
z4~)+Y*txJMWqQK(wRh*-Ev@GHF7o#0pT#eZS64fKGYg3foBBri&FP!pbL#*6`!@fE
z{ne1y;o_n7whERPiVc1xME6$vo>gpio}GO$D)sGwhnDU6a-24oLek^6Z;xBP?}dg;
zs9f#O;#22ik00vaDwi(5-a4^x(I=J1DT&ONHta}TVYqnXa?O=1KZo2{c`a@Iv&s7{
z{uDjfa`I)D@vU^T`Ag@VwYM(Qdw-{(drRtvnqNQnUHiF~Utc9|$7QqCYab{5IC^?|
zlYVT*_iX#}o4-H3)Sega7n>TNyXRTi+K0Pdf1SL4*T1Swhu1EC*!H`3*=#xU+1AU;
z=k0x3{rb<_1GlHYD|jEX+xl}}dU(wBz3(jV+J0yG9(PCle*I^K=LcULp4$JRpG!X5
zMz1pI!;)9#v+uvEzEgGX3){E2da2J}66N>5zxw{u{rz8_+b_1)uFU(h=UMCL?#s*v
zna$(R)!qE>^((7n!vp@a_FD1L`|UoeKR^H4xzqV?eoFqadwDg>zrQ>1JM(>X`{K6w
zH7EWVSOxtEeE;~M{~Q0e^M&o~<zv6J{#g2D_1Wvs@*m!p>rL(N^{h0KV_-;NXjrzC
zagQy7gFjQk9hL=W<nB*qWMGiI|CaGs`Go&x9`80`VA#QUh0#HU{lHe%2NvuH7O^(m
zV-h&Z*zuI%)uTmK|0F+|h(9mX&r)XCmy+`)mWkma+k@=yH*RD+zH#Gv7Q-7xhtusJ
z|H?2NDB(D;k%!^Ix30(9591%n8!~ZmuszO|VPIh3EbxddW?<kJ24O~qS#u;97+3={
zLn2Bde0{8v^K<nQQ}UBi^(t};7(l?rzM>#8IXksPAt^OIGtXA({qFrr3YjUkO5vuy
z2EGN(sTr9bRYj@6RemAKRoTgwDN6Qs3N{s16}bhusU?XD6}dTi#a0!zN?>!X@`|lM
z!um=IU?nBlwn~m53l$=K0~DO|i&7O#^~`jW4NXiH%q{f{&5TVfEp-%(3=A#w4J`Bx
zjdcyptV|57jEoeZK*>(QrYI%ND#*nRYFAN8nypesNlAf~zJ7Umxn8-kUVc%!zM-Y1
zrM`iYzLAk`QA(O_ab;dfVufyAu`<jAm(=3qqRfJl%=|nBkco*&`K2YcN=i^;Q!6qN
z=H?Yc!znpGS05y!Z>VPg@)X3kxdm_+ps1`!EI?71k_=Z@P+F7&c4tbmerir?Zfagh
zvA&_6A>7b{jQo=P;*5M8M#H61EU3sW@b!h;fz>U!xo|6rOI*uJ@T)EksVqp<4@xc0
zFD*(=buCNHD^XTb0EK{6KxRd1PJ~NlZfag}W`3TnlBtOqvQ#9pl%+W+Tnnt6i&7Iy
zQe6^DQf-xt3=A!G4GeV+%|Z-~tc=a93{7<nEUXL+AWD7nQ!>*ku`0E&GC)<DT9TNO
zSdvIkaZ0jPT7FS(Vu`I%W^Q77s(wLUx{?AsEUldLbMlLP^HWl7m5h<aJ)NBcit^Jk
zb5d=UpuIt`BYl0XJoAc667!N%JzZ>-Dsl_-GBZ=G3=@+RQ<BULb&bs|QglsB4N`Oy
zjg2gIQ<4o043bkV&CHUFVftP2lS@Id57VDQqJICP%=FB>#2m8C0cBrMva#~WEH23}
zs)S~+;M77WD<@L{oKmb3lR=rqR;f6#EHy<5Dws*O{~?B!XQbv~qy^*%hA1n}Pb(=;
zEJ}4uPt7ZV6(Rw!Jfh(23@S7%6*PPklQZ)`8AlW5z~qd?y!6x*TP1xZeLDpkeNbM9
zcupUjU6A|@mP0BRV9JAB+(1ll5e6#G>=Y2?WNKcDtx}Paz1^n$#&HZRdB;3m978JR
zy!jhjtbB5+YL{L)i^MyIg9Yp}W~QV)W#6o_)x>yG-><U%cb3=tZW_)!xn};CqWAo>
z=9lEg9ba>Pj-2(i@~FF^YgfISwrZ2s%&BL5CV2|HbDY5Ook5^ZzU+Wx%Csx-OV?|~
zPZMKfV`6k?@rsD5e!Df2gOL*k;$?rluZDBu%~hJFPJn<9qRN76TA*OPL}!Ak3Ix;~
z^<dFfgn)<IZb}hwvkZj}dU-&=kFJRvVNMWmD7?pIgAf$#OCrIFhuUq1&#pDMw6x?)
zMQ}X)zPAnJ9{Vtf<Sp0szYkSWQTcOF@prAhf}-NX+1*Qdrynf)o@0D9{LOWlJjSo7
z{q^5Bct>A2we^~J_AjkMkZ+_txWb&A4!xK1mj2Zxs2>@{yyA4icF7lv3{^7yjBhTn
zS?yRK@%>}|lE`n{T@GdZ{XdUcNJwbkH?Wr<mUk^Roz8fhr}pTyZ7+l-F6C`lny&9$
z{h`hK<&RJMF5Tu~&|}`B)|Yz!VNh8ncl?I8@rR55O<iicdy8xQ)GbTPd%F9(_q_SD
zUbD6(#MZ~-$1x>v5Z{-}d14#;Pmpu+wKd)`)68PnN~Y+}Xk|RGRBwjv6s_FqHofCc
zt1GttW~`PBiDR>RxPN`c^>y}o^D=^KUcZydI5;;orvB2|)glSjRxwYqI5|1vox!2}
zAv7siHd|FbndxHp3Hy#3o&>QAu}ih)DQ;SAv|wdk$-|e8X55R}c>*RTKa4u%G<}_<
zaNXR7FSETXw%deyXD1maOu3xV`jOFU$*a4;M};pn<!{p7{V>98;@Xy$^}^uT{!#u=
zwtGA4l}l`kEDrEyUDdsj8_^wm)8KwqxlrY$sCYJ>1dmOtcy~wCd$E6E+x_vxKC2f2
z(;Bj;B{P;vPM98_WZ!@4^7MmwEBLQ!-+x*^EknRFyYS%0#Yd-_Je=j2XKZq?VA<o%
zg3{76ecxy)KFn?dC#8Q+H#8U>%Q)M^x$(gDk1A8WqZ!1cvz)v{bEjWpxTeF@@HOd{
z^qY0!!B+V-Hn#KM`?+m-elZ~3OE|dl_p?fyisj$u>gIM#EHPabUAq1Lg*)G0i>!IU
zuY7WCfW}(W?gi{?cvdC6eZ<(4b!NN4s)?Gs3j!Xz`Bt6#=ZCq6sNnH;2hA;~`TA%p
zK78E>cFw)l+U0qvo_$?6OnyzBXj`Hi*2-kK%4y*i18(7;!QWMOPY#@(=ISzQ>8996
zhQ1Q<Pd6`=x3B+QzI*b;cki}8KKLar(19uBjqRWCK(VB7>5ZqDv`&S*FgSEC!F<!|
zpVxAid2nBA)~_s&TCTyn(T!ny#<gRH-}GY(oSY8H8-b%?2V*jir$z13&!61VV^~Dr
z|MJp*t0;OxN#C&hgWmM0xVfeA_vG&}I*7ayE33M{^HlngSigS0>EY=m1`YXltv9UJ
zoI7oX<kaUE1Sc%&5Pkdh{i}#8t8awwTAduI6#hM!LE`FL?-#Yo&o{50=6Ce`jy+b!
z#)aP>YUovz8+Dz3=jWz%`>Y7FK~k37I&q(+hreyw(jE9^UT2!B%AY_;F{1b85?lO>
zeYN`z@WvIbJ|k`_og|miYh`Q2^5I*?zcfo$fjC?K-A%r+H$)$A-Qv8~DEQ%n=*d%O
zcIuw#JABS=_X@*h!MTac<m}i!tlfQUUHIgp*LzlUE)#HlJxRA}Las;WwOm8N>wD{R
zCtb5H<ec_uM*OQ3^{ZVanzjj-U*6Qc+ACIU8hv5Yc7xT1%g!sEc4qMMWw;{odgE*U
z_jjTyQ!-~JwFb?(8@4&9Inn9Rb!KpJ7qd9q@3=3!XkL*HLu$b3&*?|(@5!BIoYOw_
zh~lYh2kxbYACstJU|<!QeWRdH=!^Q^B}*MyMP~CHXJ+{OaPIf2tJ+nfYd^f=Iq+}s
z@*{yK6y)~?1pZLGUKyVm<!P?B+Iy$R5u;;wl==dfb$#iRNbD_KU-PZ(dBctn@h$(N
z53^_XUz1{W6ORzT-={JuMR%>KzJp1(Mc9`I>i?^q4#}HBBHrzId&=%-icXog`}TiL
zlB&FV$d1|Ib!g73N0m3K{jW~#oYOf^b>G*#l0zMd`<RcaU7L4#-VdjxYAURy{q=wU
zc&t3R{onNeOE$%_a4+lNnfU+Z$y(oa>vw;jpS70p_qPJEpJHaF7iNg~&)?1yG0*<6
z+l1Nk&cD4VuHt*~+uiKc!#{avRS0WmKa2lnzs_Zw$ZUoHe);3C9(Jrev1Qk-hiy`7
zDk^^_DuRo-+jom+-xUjZB4+gLL->3f_7C4O)*XMBwe7{Eg%$I%d$XFOyZl<h47S}B
za^v}2kSx7YLiDc3<*xj{4_-Ro*;2eR$2{2T)~l}$*=v7$?YFJT6MOcudAUkPpm6+|
zxccAEeqQL;WJ{{DQx^_R34hzl@myo!l@trPn6qLZgV&t?{FBd3d`5d%J99%qSZc_X
zTecsQ@9p4->X#_vll#_QzVElXKljpx?EEmD@@0<goSgMNpfdmAY##52FLy|O_58@f
za7Jmu<v&^5I^-7X?c1VfHg{{inbz*p`BpcZ886?su<~8=R@1#q49nPdOC{{Odw5Hj
zL?Q2jaG!OVVHV*zpXSfIQ~FBJ>TlbT9F6`dO`GD*ocZ+U=Jx%6=WkT3ak<;!8`Q2G
zsBJ8~#b;62_7@@IuRqP1ws(`7_69A+k`)UNt#S!t@VU$we@ZQx(eTomYJUDN4VkqK
zvpu6ck9&FiSnLd{hwuADBs<BtUoKsfe3^0k9;@oVe^%I5tvI~;c~rL6x4qu0E<6m?
z&Nf=QJNm3+2<Jh8(5^YnD?Gy(b}ZB6NQ*sWVt(%b+vT%PNM?q+)V4il{JG`GyZVlc
zoQtK(T=l+Qn~^G(_*sHM<HVM#k9)GWR8^b2pL+g;nrMH_^yhn%ocbltg{SVesrs<)
z)wR1!<{YVidcG{Et$8byJ^lNGms=OOZsc3np(x^c3glczuyZwTy!aG%>+p=9lJnm^
z<Gav&gI)aOwKd@rGi7CiSRFo36Pox>#Pia+W4m0|AK0~JSzrmn+suuMaVo5_+)llb
z(;3!eURjqnRoAx8D<#z+FLN62bAC1c--oU5?^=COq-pDGQHH5+{pP)x!CF7{JVQz5
zwe@yC8$z^g%a+9I9x^U+kB!%_?$b-%qv>_gODVX=y761^8D_T)p9Kz-oZdJiE|OUx
zc-D*Sr)E`N{$#LfBj1(nUvIp%uaa7<u}krX^k?fy|7{f?@=Jn>ikjkt`!ko_`F8dw
zhp$rfp?$SmryW_nqPnl**w!r9Lp}!AZ8X!;@=T<2UW9AMD_&YtCM`HKOk}&#S*2%=
zTMvEPJLB4_TKA&k{dx{c!8+nek3++6>J_Zb%ssS1r1nGj{$I1tpWDoNx+B5$q1Lqj
zyY6XfeQ?cuwTpwpC9->+=l9JC!RBx3)_p5tb#A|yo>Qu`b>}2jIUl)WX`T!h`LE07
z&6qIZ1A8~9s@nH&LC}M<-yFCXnEgunyYf_-y|wKU6P8=I7VEFGeY5%b_LG;}oHEzu
zSFD;P{clwnKSPZ0vT{=`2IXVV0yx~V_udj%uP19LHu22FKG)lV*Pra@s_ihmb5O2!
zp`2ZwsV+kXrx*iQ`HweWCPXQ)zuKiHVQ-owtsuCp_g{*R+lJy5@18I17O+i;mGkU)
zBr~_*$PJs86Kap=e?E3Wed%R6nVM`1If3J|@-JR+4tOMft7xIGB_Ahe{RVJW%;xUW
zS$;f3m_<PSz=Rde<&REimNg!kowq*z&bBvRjM5*Mt$w#f>U>*QY<8fzt!L~yv9oWE
z#4fTZY3|D1!8cWQrHtp+6WLeOFXwmkmcNSrf4ZK@Vb810mk+PV{QK$WeTA;#DccHG
zYZ~eK>{Y&!>otGY_Uo3nPCs(*mg(kueJVf1P=#@Q_?9O{EVDIPV?Li-dGKAtYxSs&
zoJ%bO-e_?~MaeI*xzy5PUjS-)efZiIduxN{+g!B+Kdvn`v7Nu)c2<7kHofzVXP6hA
zT~S-Q#L=r?&)`0zm&@W`rA`OWzKZywc$PUY<a<G^RCo2hJxM1_F0PSKHEm)IzCJ0a
zJ@e}o-agZ$7ox)1=U1BM^USvY6+HFrzfV8!zp=0Kn*ZyMyxoFTkJdiX-4!tLL+MJP
zhI_|MMXx)|*T{<EkS@$tesGaH*Esf;9K)yIm(zRpnIw7aedN~q<yW(DidV3cOMQhp
zTaeGZ7cu?9pwgt-6_h>W%}dhWN-`XqQJ${5Jt?%}n$Z%yeN&w8=u9}8zMCnuN3nzb
zzR(H}fvzdsJMOCAXnnSJ``<R%y5<PIBgca8mpv?%NqE7MA^RprJ}xHda&<)V9j;Oa
zt9zTWvse{o<XIZ+TYqWSZC#E#bp@wOTvG$0KJ1*Sm#ZHfRqTCrZ`{JX$jcp>sjNMQ
z<{nF~dHG&BeEj`at>y26UQEAc#kx;j<xe8G@w{I3%_X)iR}T6uPd~0d|M>HyjwxH8
zU;j9xW!>pjhqvr9wZ8Kt^zN(#75^i1I#dEJ`{j&%n9Gub<Kovcmzi#7$l1K;c>3no
z{{xjys$XMS{D(OqYn^9=N@&#U+e&xZ((VSlH2W#IJ?HmEO(UV*D|g0n?FnVxyEXUf
zn%-rluZtGu_AWVnDX+TqknVFa!>H1`2|KnOGqITQ!slh)p;zVqpRByb-tcO5QD(Qr
zZYxE_hu{0ac~EV^#wzdZUtNKb7j$?e*DCGq+Bnl}PVmWoku_m8uL4@KS+gg5HN+oY
ztu(jmNNe2n@YlUj%$FNph=iVc6wI@Bk@I@hU#`}VpDPKq^(`~qJM+NRE}!JPzMoV|
zHa}Qv`~T9Lbu;uX?o-*dyFfR!^mTvysdafQ7xq8)c)jb?t~8@bb31C8R$VvTlg{cE
zBP^S<_T%ij!xk<bIv;lLF4yp%`mXwwo{z^5W^fT5uO8uWF=fW(#m+fijV5>47%V48
z#cjPgscEa>jOoY3%u^&DMep#teq{RRN!c4;?OInJe0^tV%^%a1yw{z=r)LVDUG#VE
ztB0i<#0u9`OnZ6eO<$?VmR;KSWw(1tuM7^9bM~3iaqQ<=_dMQ2-`jx_(cHV$)=H}x
zFI%bkY1ay_JEwkSNxpq%C9rjoagW3r!@yHY+2N-*n*DEH`*rJ-t=5un-_^}N)p}%e
z@J<7G;L2_Aj%K(f;rw&viz!T*Pr6O##~#-Z*qvqe=K1yi?;T$Mz8A49Z|bXEs=8cP
zRkVNQ&ye^$Tj;!XBo9Lpui>j`@7u3e#V^TmebZN(JxA*GYYx4o7d@XWN<E&ui$!~y
z)cU_)4_@%v7xMH~T+8grRp#5h+`Yrh_tf3nvX^P1{YBTXU$+`WJ#XtqOPh5@ig<D*
z{J(hl?%n+00I|Fl3s8TxZURRbsPU%rb)s&r!LtG@+r@(BH$s@NKdqd8E#~@Wsbbxk
zcU-1qop@4W#y!PU^s`g+om2l=9?eWPs@_t~Y@XeH_N~zEwHg<<+P;z6mUmTb{}-nV
zO1oKI7NuUFpZ)o{Zjh+$o4d!3RXA+#`1yLr-BJPPGbKB&=AS58Bb{U{{^C|-ypyKe
zP2P>u4Wb@j<V;#RXT!A2+;0pDvs{C&{l5RpET4OpmZ^lA${#sU5B`w6{4}!(D`zIH
z>xh!wU*mVx!79zrn#r|9%)BS;dezmlVUCZYwY@cc&Hr4F6{uD?Z@N&l$jr*uT1IWx
z4lBmM?ORtrYs>nY_wLHchq<Rk9(renm0hmh^6JdGcUFgYwQrTlzEd}6oloiM$okc{
zCnm1?k$2tNRp)Kkw<Ejnttrc$+t>J#TX^#(f#$pBe7af<$I2a#hl-s~)>(0If>YSZ
zsi7KICvFlF5~|||*AV@jE^8lY?AqSG`Q?|J_J99MGepn0=>O(wPU2h3i?i~!a_`9O
z+V%7Pzf~+wr`PS963%!%t?X81`k52lU*C1iUiWt8YQNH>ihB+xFXG#>jcx8L>4xZY
z<xSh?KYvwgZYlDdy-@w)%FBC?zY4FPx%zwk-KlR*70umtJX=pRb=8k+v!}=GKJw^h
z-pZtjnXZ!SXV0k8_MEz^(PjC*zaf*Ge&qSxDBZAXpP}eR*}IFk22GZFtdV+g&1=6m
z)u8bR5l}1Q&&7k64y0|04h+%=)Mi+3|9^$;s$|{xn`Rn&qQ!FBFR@s+znZZx|Htv<
z&Iul(p6*9hn{2l&E%2U_<=MPz+cf6umeGdn+&dgM@7l@vJ}-abyrpkl-9#g=oVWjX
zbbkH+_U(F;x6iLHDe?TBFI#YS-IY=k=kPCG=`Vt|L?1o4?y=l<V@)fru;jcc-^`9)
zmT2XVxvaBFbYJ!9$7?fFk48JLG4GAkFi5z+?{~o#P~N+@@W%HkQ>RY&U=1DwsyDfz
z(y{5qjzxc1yIiu^IgZWvdW>P|M%It>ZtQPM=;bwJo~$AMx9)ZttHZ}0ky$S^-ml*J
zXm{*l(S_a%7sdX)wOy{z_|N2TC9$9H3PnE&+U_LJkm6;ol^r+dsIANFJ%8`ieOLeP
zw{OSM_PT!`_2vG4N&oJ*Z^2;@r|v_+Y`%^*-Kyj7b|3kbT=V_K`v0ZhRoOgUT&}sk
z)7$l<<g3Q5RoUB>bh}JkANg!Qn>)Kp(Y4jkN4?6tZKB)qKM!?W=Ks6Q9BcC@m|@DF
zeJ>4t&V6rbxi1UqV*OA)a5nSo48QrV$3yEY(kn}^8Ly8&XVu*0eMWlmasSK{vOiZw
zyIn2(8ROo*`C5vsu%oBu%W1pTHM^OYx;>JQza@0Tv?`u`NB*DC?OwftvBFi?W9wcU
z2(9Y~xhR)!&z`XV_vGpS-|jD)x=JI-b<@?_#jj6qRSeqyP;Ad`o;xMYM=yjdT7D#W
z{o7!la@lv6#jk7fM9%Rz!SLW|rF7J{cNboiR@#=PzjwI2c28hm&N|*)6YXEs{{M}A
z=k!?efs<9<h2Q%BpYH#&@A`xZ6Fy{b+_FqP<#CMgnM1p#n$EBLcv^h^Jnx;8idYls
ztZc3AG-DZL8bUes4zMnnqWtV&NMz{^(d5qoy*uQzb{<v9ef?mS(6U)!i4J?R6b`mV
zOuuHLv_6n$-OZ~V8?8bN%NCYg-?M5{+w;8GT@P*-w3NQwG2!6On}>EbxIWeM^O<{k
zs%dZWN8_sBm1@TrR!y#dGNJoap<?*8dd=ORzI{$Qdi|V;wp{r<`=1kM$cbt1n;k#b
z{^v#g|G(DD-MOuty1M4y$>!?n-9M+!kV{LNqj|QK#UXn^eaWO_mQe?$+c8Ygyek|Y
z<FjMJ7k-Hg&GU9B<*wvPKAS6gOpoL2MQs-G8B!rvwSx<zcQ$$JZ8)03YuPl{`HSs2
zF*mL!K0<$AT{mhM4|@LLTqE}y^X;bVUt3+D+&Dk9>F4AJtP?jp*eD`Y@TIK#)YW%q
zFMU1M|87-Ixh7lq)78%ot#3KE=QSrk#{s5C+rRAf_3!`<G9Ho_OH#0&nWo45*2LuF
zGHag?x6be}TxVHfD;WIwSb*;Yxy6^8#pYhw=unn^=FypHn<KO9UWs0aia+1}(@0U>
z>Slhb{2ivf6INB+)or-E_xN3=`)j@A7rx%z<uqls%%=0RUmWZbb!Pas+dgpruhZ*l
zO1-~SiMyQGb?wLfJt~LRzG>^t&oMsC?r_}E^|(w=<T8=cl4G3L4EM^;?qR$0#zsN4
zC#!X@(^~KB*`ibPR_ZtQ1x6j8(tCHKoyCu7RX=`2f9Eqgy6mzs!|vVp_f>x|xymIh
zRCgQdv8pEB+z+j1_Uz@kk+x&gGP5%=vVyy#;^vy3KKlBza(EcWu?UYsTUCSf962Z6
zD+eW$Up;7xGMBTgIam5PFznf@D|2<?V(0v@nm+3``}g^Inv>QTtlqG8gT36Ho7b6D
ze{X%l)DRtUym-5oY(u33FVEJzIk$aI7sg)xb?EMeb`SO_>&cxT+t2rHlK=PPGQa)3
zv+LiK*KewgoTT*ibASEB|3CD9t+)H};pWA~$M66Cd3=9GLH_&a^8b^}_b(D(*Ps5}
zzwS!?*Txxg>*i*j|7NcLGPGZ=o}Xc_%Kwk?HNPGn@1MTD_^aL9p9c@WO*(39cyFK1
z{k;#CSNH#3Z+GIz3_b<jUWfM{pCZcb8G;T~J$e;pbS>pU?T$5vB8pl3A5AWG^mrB{
zFpFPh^KtHj4-cg9nYT-9%zm4dcvq`jd0nyE>jyg+#23YTE8Z|VzH3Xv3sEMA4~JS?
zxx4qsF*I?ryw54-Jm1~8PLci4TJP|%{6BXpV`g>#`ge}`x{g?P<cgmGr&Z!rRsMuQ
zYG<(?L+7`Vk%GH-@nqcboE{r{RxJIv`Uit!7VUz};u^9`G#CS!Gq_t*_lH$iO(@=b
zN>+T)nlm5YTs_3Avbm|Z{M!?UMG|k$Gc&l~n_KN{8#0M|qt?X=hKxD7b5}IKyk2x~
zhE&m!0OoT$t!9+IOcRY^Xz(uH{X(qp-xK$vUFlx-HGfjlpUT_SZCd({f0_E1E8kLo
z{_x*tK4V@=(w_(S?!CPC`1tAT@^;ot4mmFm?zLUTxo+;y7cY$7-8s9yOH}=mTk1jc
zeE<E%BK+^(rY8C5^v;O+f0aGy{ofrMF7G`)G3RT1twWL2#g%#!mG-$tS9Q*oNt?>H
z;pdV)>#rSXNOXL1(!}d?Blm<ev)p)83>AMAaPp{Z-s_|*^XV2_`_c<h3)lF_-nskm
zuhouYiwn!9Zf~(@U*C94Ie5|gy~Rpt<_rRB#Cm4Cd8F+Wi0RlP$51vixKJ#H!N>af
zw5yu0;$GhC<KjKgY#x3i=>_LZu2xRY{i@);9Xspn%F@7{x^22_$$aP2pRJk`WLLd8
zW=~yRNzETOuI*Z}ycw;R7QW9p$0DATn){elTtj8`L~&-`9hED34A1Q><vf^m^2oc%
ze|6QgU+-{8^>1Pp*I=t!k@EQ9(b6&oO^u)h@!kxkDorb!qo?axn6Hisa&=ppu~cco
zvBRx7bCk`T+e6j=@7R)Wp~4xmBu)3N^drx|;qnu1P5Ju)lzNYT`S4{?-REypw)M-^
zzdyKl@7vR#&fX3VI@IA*`|IKHjV>O`ZR<ZR{`6D*CV#!py)`~Pov{~{cDrP~<@vbZ
zV~NMsMW*Rqm$^fa?VJ#uXfL%PqyDjl;aaQddK?UsBE?&{or<kjAKB&<=e*$-tK!TD
zOw65;sxc1l{A(CqhOWLdk>_Sb;I3(DR}XX;Zb~$b%Xxot-*;1iJ#R&oDlTff#6@3T
zamx2|)dg*VJqyxLZez7Dcq!wvE#j;gclFILOW&*)@0NW2RZQ4+qS7WIp*n33xiu{<
z_8b#lzA}nfd%JS4h>{g}K>zY*%cTYq$r=-EmuaNBa*8nIwZ?8e`Nlk0+dU@TH8ZC$
z`OF737lmt*(MM!X`@WX{dZe_NcadeM{m#R0yULEWux@f@{PE(2&;fqAy!w)opD&i%
zSO2{KPrE@+wX44D`|;K}?`)TSJ5a$e_vhlp-d4|dv#p7K+j1kt%EdnHOGv42?0&}g
z#}1ZH*4gk<>5ZHH5yq>A-xa^5X{}wm^IXC8sQw*wmH%pvNX$6JwPwv$vm?t-_=)FV
zez<dCb)&D0_R}fHJ})*-XwOS6FH1D<+_HYlwUt{96d$g3SBl`|++QA%+;mN9gSWKC
znvNTwL9AO>zii(Bzy0~=@SIl+cV_(N?%8o|RcZ)B-jfbC>6u|Z=?ow8emKe4$kZQd
zoHp}l{5y$0(d&1YdG?wMok%k&y3Qf*ylO*dIKzdn+b$>GUB0WYv?yTV;h7!DiH8q&
z=$4*+qyPVQz3Mg5;PsEBON%!zdj8j&_l?VyeOGSWv;7zLC8KA5*PB;&U%UP0SKDPh
zaRcwxO|vB~`Ob2i&gg#3=Zp5T>4DjYI;OU7USu8e-0W;pl(xoS-LD@ETayJf_AzR{
zwN-q$8r(+Q@2|7=rM7H3chG-@){bv%yI&tIsj{p8_=)-RqNO@2F<X<HowXwwJKkia
z>b_MBy82JT`R)vUjysEjuS<qEY&3kasaK7CUGqwY6tC@n3w@F=+jq=vd~^4_M@_f=
z&dYl&PEV=REz5fv7+9n6v||#F+vdC%29KQMPp!MKFmhseDeH#3E7eb;e{A~ZP<6rg
z#v;}`m$v6^+$dov)vEm{`tF49<*D{n3m@(Fitm?rb!nx?hNMsbIt+JR|Cq+O@t}a0
zp@HasmVGJ?tC!zuynH3RqC5Gnt%SOuw&KIp-~p5U<`a7NGJ6ZYO%wdmu(s#to3%e5
z%>S2fR_fBT<ywf0$*~Eq1XeeGsY><<*|6o=>P-q)SDoE>#O*>;*G`5PJQDA?!soth
zyv}v)>y86Vugi@ta88Vk)sALhnC0Po{6F{VqWyY{Wan1NGfdi15d60*g<CpIq`Yiq
z*!SL~y9UcVj$FRftiLqI@XbH5#fxNExK_HSH$7bI@?$Ua+T;H2Jo@M7ueY}C3ry-M
zPdO6tFGBHL#k-Up$6wyPD>(W3)JNwY<Z6lZNvQpQ>A%19eZs!h%Zm@+J-I^o>%0jQ
zK3r}1|G6F1do9~=_SQ>thH2Myau+lSHcaK6zt>bhuI|Dn?PZ;@iV{iT%IhBMuh@7{
z;&YMN=54Z$#z{OH0y<`IxDK9iX1M%Wz>O#Bb;kCr;(Kf7wwpC-Zd$RDaY31BmFwws
z9KprSv8i6i>t`j0=idl5$j)A7Ed2bh;0h^@1HOEnDtmXUPIJy%t9P_>Py2`G_y0e1
zcc1=kw|%FXzR(s1n_n-_FA80~d-6H4``Vkdtsb7Y|Hry<W6WOv^uE`Majv)bMP2?l
z(O~VGg})SH4_FvP7aRB_@2Or^62JC$mM{M`)v#w1RJ50tuFqJkx9()vhN$eNUz5}&
zHglToUg@~bdyV(*ZSB|E&faCe;MG%TAjq`$&Kc%67sc~_`M*9oMW*iZOXFmoT2Wyk
zp?yM2^bWPO+*hw^Vob{P6+R=H^4KrTETpdL$0PZ^y08C@j~`DE@|l!n`lx8dk|{zC
zvNEbhI&0>(zv^{-x=hcc>FVUae~)z^yIIK*m96J;U-`aE`|}v)T;~6)N_vm)F+6vA
zPG{BQn&l>IN;`9}%6SSt{4#|*Va73m&%1vct>3uztHX}QBWn&Fdidza4OShQm;>vM
z9ZD=;!+yx}#Ic<-^p~EKa1FKn_453$^>#bLa{|BLZ?3Ms67r+srxlyt5uWd#)$M<s
z3z+lHR&Rd&r)O_98;+m8K4sq2|3CEQ^yBM)Jv{!sOV`kL%l*G6K~3_1e}3C9@jG07
zR4g^mcE@({`{6$P_J2P-zT7?QT*2vop=*B3-oAXxJc(1R3)GohcO8#&%Go6SDn?&M
zjBnD>A72k{bBg2lQ8sB$C>1>x;&vx&h1Z^WJAF<qY?`tup>C#f#ThMw_tPwYZ9T*F
zB|DNmoPBz+@ABL^Pn}HSbc&C+a>Oy1RXmj8@_cf;d8N!fOX)oim`V%Id27xo-poDU
zyw+0UNwcG!?R@6x3*~Y)Nec-H{oCuI3W^;4H<#MZPH^N-NIi4^-`)H2a&_12<a@5&
zl2~lIJkQZZxo_(=w%b088uJ(z%nH%@87Nsed1t`7Q%bACu7}O@ZCrU<{YI%<{A>9a
z$M(IBOV?0b`#G$tdd{Oidb@l(Hky6UtB>FQUO(*VqPKGep04<DE06t=?je!y<>?~#
zit_GqwJu+twDD6|^<&+DJAZmr%6GY#uTL@Aa#zy$vc@i{>%v<#&G%oPxIlz?g67su
z)AoeT47zLWQGZT)vz~5&?!1Y`-7+6%Z~4A0b7QVc>aUaE|9|n{bW(lw=k4yF-~Z3I
zulV&Kb5{6_2_L2^Xo6}Id7a!BoZGfk3inN)p!{&1->REWzkKLEZ&z7;LQH$@0~v<r
zd#*&S^K#@2?cLm<y?T|_+FU(Gjd@?WOmE8PZ1sG1wZrg|rP)QVj6E~j%|6fl(jVGc
z=^(3>vy3C#?uBTRx9y`%Z7eZoR`cxBjrC<KO7|~Xc~GUHjJx4v;}qtHpQLVgEt*$Q
zSfqS@Z(-jvhDjfacYTlW;XnGtVAG=be_!M8YHzcvDcka^-mdbS{XadSxzYzi?Ee_H
zcF&8u^0WVc(<fI=neHb`*}s2daGA+<VD|RqyMKn-%xKZObW$U_J7}w%e*F5m3o|OL
zeq4O~KCtEG;^X%ho7SXjY&fa;_1kBSjIUQ`b#t0CIXu1-U?hIzkmC2XpTzHcovLD*
zo$g<x$R;#>+tmZFVxF0%aUb9Fm`Qov>jzEk#Sza6&OMC@4hVakv&`kp*{;j#wHd-c
zo3|hAPVL*5@2aDy!kSU2IHy=yZzscyE&2U-6DsrT-=Dkp;?0wilL0xIyCT7JT;CNy
zv!3_)rP8IJGl)N^TDrU7+hu*e(=T6NKi~iB*(SC*&0!OQycii`rmPN1agDSJZGZLT
zf{vCC+unsz;z7@1%;U~DPc8NhU2;QWhoEIk@r(Sr8Q$$O+xV_tQCP35(3a&8xwzIp
zszce#LDl}`hP4IxE4F`6**`60u8Tp!rn`$oH^v+|+PQPfI_cSR&9?uapa1{d!sq|H
z$NutnZol6Dt0C#;$}ab?<(Ipq4qR1V&-mt9_;PX6sn7Qri%b#xb$b6l?OXnnL<NnM
zf<i(=B=hHVn{=-TEf*>I`DgNb9#+wR8QzN*WEz4S4py$B&Zk=0FUacO4YD=t+tUBY
zO*49HVP4?TqJ=UZEd3I59r!b5@y>{uo;Nf8DA!}7{fiEKe{eMD#KmPX9!E6NCvls|
zB`o?V(x}FDQlsA_z3160i*hE0f|!muhO?ObtW?FDrYW{OcUt3h-J(47DtGnI+@Ern
zBO~-I<UKThcj@L%wFq-Mv|n<i%Zv#hejh9fR?BX(tWVT<)3wN@aa)aWu;>L%?x3B=
z%~rp=k+9b5m1K|SREAq8*p@w8=zIRv7tXlvyN@nenit_<vNQW$*^kScgD<-Y3G7nl
z*uCIM>pPL>(V2@{BzUFxf0Vv>W^`d*eE8bMX2<sblmGvhoqv0NZ%top%MG7qH}ypA
zb)WB^xSS`%lJ_k6=z@#)Kd7@tm+0r{JU&*?Ue0`M#W5-7jVl^fe_^cUO`0k&`^@Io
zzw2W4LQEv3WS>{B;n3c@^Ce?c_EI&G`womtW*4jp0gc3)D}l3ew$9pxi53h!hT-bZ
zHeWt$KHtvT=Fb_+cHbMKocs)KJy#jT4UUDV`k0n346G@S-_AFCPUo?ml`Gt0nDv6+
zo{=$_sClu1C-i28W8F8a9V;2n%1t}-^~%4FpO!7FeNz2z3F?{pG&dgE%Xs_V<)UE!
zIa13jqo2PLoe|1#%dv&Q<oLEMbzAn`KYkwHuOR&ErR&_kAC9ZA3Z7P+$;DuM``Y8W
zUz?AgzP?=iD`=!K|K7Z^?frGXjDvQ*;MwxlNO#r8iGFUOC;l!1b;}<=eZ6k~tHbH;
zTU3|6ire`0;ziZHnKF*P#+4_;|3BQkSl;H&GF{)t3$IV=ayup2>0h{r@9)j<Y^h+6
zEh`J(U5Z;B8S|>jdG1b!xW!z|QnJ4K?>n{IS`$h?B+c9{c>Q*@(G-)@KM%3<Y*h=o
zYH_^v0n-=O7uPpU$eF5quHEcvYTWgh4yhH5b2dtunB3AWj+whF_1%G#`@4gUC-1rb
zW$8YveHk{-w@%qrc=GeS2_N!Zf;I>V)qUTP&z~OkT5ZPvpZ4|<adY?9)wLw_urgfg
zN@~>a^L%;A&r3d_%6jL#lLz<|j&*H|Z5OHETIjRiIv{m%U~GEK!?bhOp7AES&%Jfu
z#{TS5a$6twM^{x%po7g-*y^Nf>}9LGU)Pe4RhdUlnY4OeNa&*G-Hab^^lbfk$w%kV
z?nP3v&9hG~eQ(#-ewS(PoGI!?Ys-(nykKZJb<5r>KEVsL8}sxIPx+^E*E_$dq-_3{
zm1&0VCS84R-Sq^F({>9jyZfY}q;yShM=Imh;OpC`maKVL;jp<R2%Kx!HQBWlAC_~S
z+THc$lw56LVTB8$t8$y*?tdq*bTQ5_PfLCKaOc8J4-U=<oY=8!8N&?rH904^6mEoa
za*3#xv0mG>dga3V`6WpPj0v$y-*yMCj*|YlcV@ji_kwTRx0=2d63G%hY<xZ~E%kQ&
zy}Ezb|3BXEKI>C+`)wDmj_Co5rvADb|L^DbdXKji44|Q0zwdI)9Seg?BBsZJ8Y)W5
zE5BTxU%LCxySX(VFJ4qN<*zF#k=(o^@96xp<Sj4eOzHWW*J`*)BkI`8dynO9?x=ke
zi}zUHwO%#r#oOoi>guv19v+;bx$pMQueQ4aaxaBSPxsGlUbCF9%#^8Y`nKg2Wy^ar
zw@&FXt-Z)lVIr6Bu_4!AO6P7BLxNUt*NMxgZiROGth}opr70UPRQ%Ar=PTD$?G4-7
z#F!6A7}tKhc{OL_9xG<mYDwlBPpu0#adLA0*9q!0P<&X<d0Ti*;SQdPpEr-^+tsGM
zt>j(fHC;4g&fIpbcJ=B{i(DQDhAq=t%Rjy8qpj`9DUX;!7?<#Ek`ePUl?&lr)4b3q
z;M~_ehEr`h@9k8m*%1?YUpTy}xAC=+yy>R3-ZJ_I6K1^0=W6QhR5Q4=WyQpR;@3?P
zk<MXxP21-ON6Ry4txHY%^KkQG^L+pM4+XzLbz^zl#z)Wn>qLJ4W3bt={rl<t|5k&7
z>%C>@>lJJJ&%E5u^Rj>XdQI)u*3N>zSyqO8e{e7EO^{94_eI`&S65v8`SL+V#d4u1
z%jN5QUN2mwTD$J}>FYbI=5uv#jPpDh{n6j*$CpLRZauQ_F*cmXvxmE>@(Y(m%=LeJ
z{?;g5^FFRReQu`NI<^(76BHgQ==+*Zsf+H6RMj%Sd$sw&#xt>xj|QC7{qeZ<M9b@H
ze?Og9T4!hKs62L>GIeF+oRi0H9_#k$)ZLJ(cWo)}J>B9!X6LiNx1M@9{pVMs15W}T
zek!?k-hJP`H~pV<>jN#-m%iiV@3{Z6UYnD%{t0+isGNECme9uzZ(jY_ykF7Vnt|o;
z(zu(#+05c6b}n<XXR+oN<>arfQhH+dHa{jiJucF^{lvp(0?wgQ!B0O}CMRUAc=G1R
zgy_{=$!Fi**)U^^NR9V~TP%$EzRG2$9y_8U#BW~R#WLN>!hE%YWXyr3E0d}fwtoHl
z=W+am!mbYPRQFW-KZQU4O#c71{=dD&lb81%Cmq$iz98Zw`}c1v)=xNb4OCuUk@jGp
zJhkiBzaw8h#BH3~S*YUNFITT$CSq6h$2Q;IzW&o<P*XfciZ`P9rDsZP;L2BOMgh#p
zTWSPPDi@13{D|yOPI6xU!ne~@Xzrrs-F=2t@voHb+?<?X^*wI|o6BYiwQQc>S+T8-
zPnS8HE)WRU`g8H^jj#zZ-?$hR)+^pnsJN|tWY-oED=U7{!ldVG_)k1EUixdR%LZ-5
zFx4Wp*AI#+7!sPgCvk5rOyy)yK9;7QSnXf*@VDG)8HQ`BTo1j~UnN{i-8%L9;i_$G
z<xS3csy2zA?3p^@!+NJr392f8^fpL7mzkAd_y0{~N_fh&U8b+MU)J5R(K&b7qm-_-
zr<yJ~KDJ3<NYHBQUgs52ykjNXA+yy>-`<tXUV6iZW!1Ytwr98Gf@(L+&0;S}7e6K?
zw)WwZ3k!^w_ix*na!q!%%=aT=hu&#CjrLnP<DsYZV)@sL7keAo*Jd_p%-w(culV`S
zUT!V9|L>UZ%xS;zR{qbW+v3(TD~&&1d-76mk>_i6{_Xdxf2CiaJA-xEvnr7ilQfXU
zw|`eya9^L>ywLa&+jX^_g2~s~{1ygWT=M+K+?Q?LO=_=LxE1P(*YJMvU7%!iAxz@s
zkAtkyZ&-dQY5rYn|GVCXQDnv}n~OXBm=_8Q{SyTboaf7WXER*4VEFN7bANxloqhBi
z4a=9mSCvHEoLsqrfu-NcXwu4O3m3^>ciJ?^@Yl!FlG!!VPwuW)E!>paHz%9Ln=|6v
zPIKeuhnGJ4dQ<1h&TD(_-Mk#Lz)gF?9qA2oHkxntGTOfO`_{b6Q=`g`ewAtRu{_9}
zq3rCr{?w5fO`>)+WnW%={J!E6dxMGME3qg3?^t#(XwaCI^Z8}Fe7gOgk9#NU8yPN}
z;+N!PV&?w$mHtZup)E;UCvr8-ulrG`^KD{^tLw41r#~IsyVoa^k29%4lHFm++SwLf
z#T6?qxvySYq{tZ(xUx&7=aB14DT&pS`|Eyf{`6CQxwz?CP*s29L^wlIcFLQknlC@9
zyH8nqESNiU^7Vp5MxF-85JvH-yE!C0p2VadI;(X@+Q4Aa!~`i1F=kVl)ON8+p{hF^
z3~$6gTX(A^$~5lDhY)=>nb5A+4_^GfygoE+N^kh-J6EMo#oe5o+PCmpuXyQ#i0IR;
z5!1`Kb<(ePK771q=ba?md$*IcM7e~8{yl}pgi3s+arMVzdwyNLA9GfWLE@b6fmcT#
z@-FSl4qCp#XvLJibt#utEZW!F`@~_v{nMqLn`iDcPD|g+@Zj8x!0ihz-I7X{lJPOQ
zaO<AwdlkjpTRe+XT=&1bvBEJ{@s;;fxk-yR`5nJ`wR?Z>rH~r4TL*SC2|e_#{{CX+
z3hBMK`6X0W|G2jL{)-cJ<#OkC9}NHaAWvG^D!Gy&K>3NT;Og7S9&`Ikjv6g3KYrb|
z`SqoFXPH_pavrWZCYvRpqN!acsbqF{vH3ebhNTh9rWm#zdzs>uIC(l-%z4MISz*-;
zTbDHUnjP62>Xys!;aq_1y04S}|9Db;`uG2bCu`42wk9v#cc`VMUe5ENm&cFX!)E8!
zY8C%GWK;jmJbW|j;hkHju0Hc>$=6uXNl#;*`zS?Z`+i^&*y}Wb!_j_1@9}_>$0m#0
zJy|o=P(yWfUi~(0hHkEzm&(7Z`ue^UU9pCF!`cA$@JRQ|QC$an3j(FKJMF*oBt3n`
zx;5_0o7}c7UvB^B{-X(xv)}Z->-{cv_3Js^(s~2ig$18atV@a$iQJULQ7tpod1tVR
zT)T+TtlrJFk?)>`f7@-pP|#c@^=o|X0maN`Tb@nenZB&oW5?D#KjI3t`1X2FPo9{_
zEgK#BXR?c_&=!sL-ah<`f6l$PC|YJucut__6u#T~TqiFCR~<<`_r$2Z`O2Pk#k_(V
ztaUTvzp}^|?mZ(LdEf2Kb)Wtj@z0djn+txJbk#@vki7l@n{6{>v|cu^ZmiRN%<@7o
zd9}<E@4!RWyf<!E+^gNEX|?^=wB>3p_t#GqSYsTb&Je->b&1~euO3<kYLok>PWYhj
z2p-F|jd0M(Wj%0fvj2XIoN3*uTeoam^ls0lRUL|{UV+>XH_i-8)S1c3B7Q=$dtUh@
zRjG7FfyG?+#DXRji0)Cnx@!Hk^6%>3+jjA-TkTQ$_s#89rm<6}E<7Y4&T#qlrQEEb
zE9y6n?Ql6<vFyvVUHdn+afB^1`M4@~t@h>}7IuBLH7r$oHa#nO|1qKJ)93#Bga4nf
zKXLp2^Z5Q9;W<ZtHr(W|zccO65sN<VB{u%2{@c`lQZ}?*vc<nj_p3@?{IoC2_~+Yw
z&EOPUtDAT7+1HAnv;6#o`I8r9Jc-!9&VPgVLO=a1Es;sL=k5|rE}X!(xX(#y|K!!*
z%RjH)YHa5rUXUJdCRbX(eDhoWv(nh-yVTc-++P}YkoS0z(*~(Ud{<3h3$17roSvX0
ztXJ_m=Jdv2ro5Fb;z^-NQ<IO(m@?}{#r@xArPpq5T2-`;F??#ovlZ)|@5Mfzpn1_E
zyg5zk)`M35yvW!!?NN8C4_v!vnX@sa!r$Y^bMU-aK7ZehsfO*BU*7EYKcC+19KP*M
z*P>M#F}n-T-AY)s@{y#f(HE|ZvQ^H9LLc9;Vc8VKnqS?z%8MmGx7gFS2^8XN&PDfJ
zbi0gn8O3d?S|xbCba+45r6Xa}<X)h@Oi-j-bahUwaqyah6Jxe)uK$1V@9*8si{9Uz
zJyWst)w{pHnIEzVakDc_dB(8d|EuP$#wL&7FJ9~&H0hK4|D$^+>v!0!jQ^{;ah*rT
zsps<(4oAt^{*&{K>HEl=R5hd1@a7(e9xpEjy#^NDDD~P2Ta8V!4BSoz&z*Qq>^CFh
z5%#<GyJeehTv&5s(Tx8;^jAn5?(5=P;5zAxtK6o)i%zg)d0g5Um8Jci{m@-$p3G+t
z)-ZiuCy`O))#!P^RmkUi?aXG0sl{t;UN)besbV`RtueTyns2|*qW6DoS{ANqoMsc)
z{17xVd3Qqju1Pmo_OZXOYK@rwVZ!Z2_g+8rx-+qMzgW~Ml_XV_KgQtJ)pnNMTOJxc
zI3@S*)#T?-zmx{Yx|D6*G_CGx)cV5A+~S>E_AX3mDqL#v_rcMH=8=w@1EMx^6^NYq
z`|ZY>RhyUIJ~!oB&4H!soVV&FuF4c;yUi67#MQtpXS<Zq{E^VXbBY?W3SMDT^0uD2
z9wK&RnGuJ=w4WVOZ~MRg`};e_gT=VEux3vI!@GB<4?1MCK3AQwBr0*u<%=;-{;lR;
zam+ogV$vqNnlhJJ0uv=pmG3CkT+BSTX!)t<^IcwQ%ROc)davQl)bOleVXVWK$HyBj
zZ@9z>W>%M5oL^gMx24yv{MTf@cl&=Sguf5Ixluc{HeI75Sg3dLp3cwW|6gcrTlzI%
zVf^ViCuO#!&E9=2A=P*8X`dZ}*GmgZ`gbt=J;|Cj^+dM7;;3iWk960(i)l!!`@T1S
zPsCG(TTY4h7q3}W6D+)h{dCL)bFF8|7dJbf{VnI@G3%m<X`Vvi!_UpTSAAqU5b(Z!
z>;J7PDm8z>i^lSqrwK`leR~o<-zGk8ZtPhxvpI()lP`WhF{NBDXU@xOX4*$LFHB*m
zYCW}6PU~icqv-r&&E1<yy1XiGdx->HJ8*KLV)vK1hPT6-xYu~QyE>d@|2^wzO!zgH
zd`3slNg<MR+OsC_P03VR$UptgzNq9I!V3#)xYM(Fp2i(Kx<q|lL*uiubpQL(uio08
zTpAY<tk<{RZ|<dE=l=c{?yb|!cHsJA`zj$rNinv2e*GWrRbS@DUAZsGwQ8<wc<}dC
zdn>D>?EWwD&f6e3VZ~<sB893uwYuV$SUuWu*A~BT$Zm3-w|3pjE;Y_e9nLu?#8yUb
z-{iJ?!PZhs$v_i^+tXE_mLA;_%COOLqMF1d!z~8qo&+o`IbqG)JnQzt6upi;+s<t@
z(|LY?_0@wFZPKTYn?C9IIp?+1q~~kdSARUgH6dl{;mPm0e0V}*!*Z3gcrs?RXC1j=
z#4g^%)f7@FXLzk*>dI}p8?`pBeK2S9wo~Qb-|sVLi{YJEIh_kM>B|UOL{sy0Lqpb_
z&EHb2Y_*rKU!TUE9bML%(B{Ro=>6WU$3mtz6&^hCc1cyMMB!Y~r>jJ^EM;gtw$mzP
z*{rNM@#b1Ch1ie>UOsHXTubkq2=p_$s2wmh;#qFw1a5}65zbnEzGqAS@6Wt)s%qK2
z$4*8%QpYuUj1$6(Im~WctJ<`D>6D`m*AGg*ewE-BlmGty{5PJO5BB8Dk=Byvma?$4
zQA=~aqV@k{{Dkr@mgO@JeR%DVby=7D%HyZ6{~WIu{Cu<iL&5A5T&;Et3hL8R9ys}Z
zd#}HylcoF3Pebc@|CYp5c!e%lb~NL|p|d|O=skRGwS7aTu}YDfq}WAA`N=O%E)QL^
z?)fCsbFos3*{#wU9HK(v4x8xSRF#h_h~PR~?4EkZ>BrIwHu<XB^LEdAX*EOebBJl&
zvp$c|Wh_r82C-#6&UqYqE`?A2v)us}QLfjoGao&=)qm9Gan3Rp@g(kjT7ote@7DG8
z9q*UpWjp-q2+LG(m~&=|B|04n=ia>~F_ZV_y1sb3{j+q}{Ny<(kt|@gb3wz_W#Jk}
zjT!bjT}-f8TN&2Cq-Y*}Mmjk+VtU!4@~L(UCa}KxJpaXAi-nR5zJ+eLZ|pzGvGn5^
zALR&x+}_CPCQBkS^Kv&c6{Mdr*{+?PaqPofL+%?_?`W+ppZ~%=**1yCM>?6oM(Xu)
z|M~CieqK{P%dDn+%$+B)@g=wM+?k(m8vkmzSNHGv{r`#U?k)e>Y{LG*`>^H4i8V=1
znhl_C^*v`}UVE!uOHMYwz54r||DL~lW`A`MGrXK|#DPDl%sKqEQjwdLnB~R{PZ7>S
z?M5ZJZ|^)WNLuqY?9wq_D!-#UX0N}Q{;vlNZT|ms;wJxi@ZnAh>n5hp7w&29()==Y
z!EYzl)L#~#1>82geQ=0L`Ix|&j_hpayMIeFpJ(`1M{N|mKDY3}g;l+xs&`B7&8Xx^
z@XGEzTjEruuar15X|>RoPjMH^yk+Li-v8JB|NhG_B^#F7vWWOj^7HVh5C<pr&<!60
zQ^LD%PTcz3rSL`ggiRgG^!ai(?~=Q@`)8|k?xd~8SO47#JXN;d!CyAx>{`RumyCSg
z=df{>AKI5Fwbe6lSD){q`H3vX{E4}}nGXA>FNwW)-cq$hc<ZgJ3|H12dpEgxjqf!5
z%(gFc)uUeK+geKS9^)^cv6JK1!yWrLW~l`UFYlCmDlZ!=G=IbWzno?~%ec9w|N8v?
zzriN$N7|qv=cW7Xt7Il`-xV!&phnx%yLiFex>{>4?Y_!mX$i&V|FqVzNPqgWRP)0Q
zPZs6qLuooDXN#8g`<!|6T>gJko5S&$*Gm3;I5~gqn#yl?)_5sw`5AF!BS%;IvP^Z8
z<A1NQ?dngs6_gX2TsA*rX~g=5%qzF#{Ng3J9;x?<hFkDEYA@T$7X13sw)oizw`+8k
z87<XN+jlo!EHbcZ<r)q5J*lO$4oiIA{{Qd)O{+D{CZC=-VZw)0a9z2&Y0>@JY7(0{
zE`R=cHOJA^k!5pMvh>$gErxAA;VLnjOEq5xX8R@=Gpxy3Fy(~BhZGj;+KV^Rb|t!M
z9Sij1JsEi8?w@%=-AuREnqIeD#GbrO&yl(KMf&U&5$glkg?*W3T>5pf!sgMe68+?E
z!JK7RIzB$E2$-dP%Ij|8v!f61%sRhJ|I9-j1E1Z~AD`d<|FFl}yrt!!!M7hTmfJHs
z+y<4~N)8jhPl&r3Iy-67@5S@~^|*bxaPp#})w0HCGtTkU#xZq9sHk1^%$vDAsV>yU
ze|q3UwXLt}ChTBfh%N0cSre^ZKRy1Q<mRcS`se2N?od;6G0&KM`(9W+V~<4euDx5=
zMu<wLX!~oe{d#Z)<0%t`wNqMF`)Iz>d(-sfQfcN|-p>MIs$beJXdk_{bnCS<($4Wt
zK|!}uSA?DT^XL8l^)J8Vr1HJ=0`2ydRpAeFIuzbE?}La&)t6@lm8E6dEE<g&dX5G>
ze3uZwoSJ%VsqCsk#nf{*0zMy849+_8#OB5(uD3n+e3zNEx}|xY@zXl6>q17{Y&Xxe
z^r<JZvuAs^wLWA1b<(8wM0Sq$0^x%WkKayn>f14=y{pNsS$r>349BcrHc31_0UZt3
ztb&`kzs#NMoZ0HcXLIak#iG-fTW?K&_twT_d0JW4ovSAoX0kA3*`Jy6u2cKeg+(S8
zGZGdxo=Tay=mx9Tu0_WjN`8G+l3XNtzQpI-VfkPYG09a);)*jjsxj|eTUy|^X<Fvh
zojrS7veQ>A@6Snm?qBC3yjaucA!r<L&P(0;&@G7%(^8n%w(G>~{&`h5IzpOloB6K!
z{YHlG+8k|mY`-3!?IpJK$LghTze;Qa%?OE<ZC!h!wC};)ZHpw&E48nm`E2K6X|64w
zRL;ll&tsN)CeZrf%;Sp20IORkS;JJfXo)Y)IJ}WX`TH*;@q@P?v<lRz{#hlGvg;G$
z!QBt$X4t;r^I_BrUiDF_{p>?O-KO~7t&1!#^m?~_Z)v&hvA9S`XrEBTs;Lsi9=Xlk
zHv{K9>N(5vtTH)+kMn1&(NYha8SJ|^H7@(Ww4iFvRyR%FWoCzqFKFNDJk5}CfvY#-
zB_FfegmZ$&-=AT2<GK4er^|8LlnvW<Z!qjX_SmAZHEyaX#|(xFi*@~J(Ob4Y+x+8V
zM%%|r-!eS!c3+yy)_b;MPQ=viN&YT6dtGk_n{U&a*Slm{M4s{mi&=uqysS!v>y~e^
z-m?7JbA~tN^+N8xmnC;?Jydb-$MfA&kG!+W=e<$+oqOZ=FE2~3-COf+$HI#Dd;Zq^
zcyhkX^xWAmC!gFdpZ8+3c$?nK=}lbq$`M+E#|~PTykF$wnr3m}8`sjUagnP_vyUxO
zQuxYwNzRDrn5XrZttQ7Vd{;TwvTq)btj>%@j#sMwd^vcz{r>;U^5R~F%P#+!)6x<R
z9+L=Xy1wIDRj6P5emnajRfcdY20owfbC-6Wi{aQJ8J?3GyR4?SCRMf}^<JCpvx3h8
zT$(4R`q(z_b@JggDnBxH22WMf$JBk-_L%if@X67hAmF;uGjQvvnJfY9!I9fVs*Wt3
zu!(P7Ry1GhrK%MwXHV|5-ovlPby7U_1eb^7DY5f2PtS}#vHNt^&rfF`K7HAv^gz?;
zB&*}CP@lT~1?zT6u<xpkbX!)mdg6pC9{=xh|39t&4{CGNq-!MQT)idfwRcx|&cra#
z2-BmUxg94R#Ddbig8wVLj1WJ5TwB%R$*q}5M?DjnK0MsKxMJJ3?t<cT7XK_iEnipW
zD%<*{p>JVnP}h;QG3%N;&$UEbgsA`bux<My605QC`@d6C>^oPpPc6+o<WxJ?(QEp4
zi8s$bSY9vg5PTz(btLtiz~;N!uUm57E4&PK@_KHtI>3N`S<F?x6Ooax_h_Ve&zK`V
z#guDDs@|`A-dx#VlU6(ttC=2q)_K87>xR=aPdxN)&kgLJYi(;c(dg`XPmdkF;4L~r
zJ3c3GHagxOUsGlp9k4({a>JHuK?~MsEl9OrB5V@J*vGw9i_wC?<Uq@&TaQbgTkR;#
zWHeoU;O+9>U+0#&ylBgku?f_c)~?;m#eB1G-^s-18OhTa3KWZP{8YRADd9{@!b~CM
zYkp2=xC$CuO=k)zAG1AXeI}k|Zm;^Cv+GwC{d&Y3xO(NmLkC%Z6ztrzZ~e6x9}TCu
z%U<j_`LMnI^C!?sN}*#@%63J#1k3&Xl73&3i%WLtOf!@232BdAx+eesp`XEIA}FF{
zDAL_<_QpY<m$_@(xmI7&ZH)=ZeU$N7M%DDfLHqwd#P9A1<O+ze7Jj^b*THwOajx}c
zq1$J?cbNFj<oWX7tr>H6bL=p`vaQJbcUzP6iXhMKg-%(I?<_5K)||d`a`W%j59!_4
z4BQW;sL8mV`(~@ru!=ADn%jp5L7(pg&Zs_fFl&xzlOg}}oYS{vYN$*O+I4E3%jqp!
zl43eqw>EIy<5HX}zFMX=g*%(^ibH4<hh)P+*4?L^&)u*IvAk|-$;}=uY&nr@<`hta
zq-3JcJ5J93emPyu+@EhoC#M~1E3R_V&N#dBdhKO{jO$C6&R<=bd_W{%QnN*Hu4X|&
z;az)=yK#G+7^fvh_3U>xD?6K+o_$17+l1j%`~vpu>=iG2vYPfTy6gPyTXW~Kdxi{a
zrK^IIFYOYIDw`4cn%(*g!;JQ4KR-YJ+`F*h*3+al&Ij}y4llgTHQmC#<06m9$_8J}
zx9t4eSDx&vV#xdrn$)_?y|Uw)?*F43m|vZEE$5~n{ri@b!CXz-q$>)XXKybzTACck
z-0B|i^kd2UkM8c%cUJAsUibbR=c8qZ_s+iV@UGfQL2u3N@EdiOJvXmP*!?uNvi!UF
zFjLw4o=BhR5z`GeKTqiLs=HFQVRf|juP?19xf)gqE}ql6&1lEAwB!lfuRnENrV-7U
zv0HMJQk(vYZMg;!la@`n6r!2e`*v<zc+uAFUmX)R3SUdDmiu*ysp9WX_xW?>rd``-
zJ4g3^<~<*eA8O!j`t|A}343R*6%Vfd;@X$4vHo%Q3g&D%r}Y6xHcq+pq*Kyxn&{J2
zIt&vO+j{i7=WbI<UTfIPr1RG;&q?ylGS)X%3miZDKDqjAxm50^=04txB~gsu63$Gy
z7WG>2`jwOZCq1>ZH!~?8>&<oRS^u@!iT@>UT>Za`pP!#UXmWAu3ufjE;n{5~pNTQt
z+^|8>WagzKy&74KyZ(bx_+j;WvB#nsdlV+kd-v_4fBiMzrHiibJH47Aqwj^i^5q-L
z1mFMKkXPVrqsy7>(w}#8RR{CYA8qT{yX7u>DM>58yZTl;=e@7-Tqzl!FZUY%eSZIc
zf}ez?*jBE4lG$f<?_RD<-qbs>FN61P-ff@d`aDg&A)M0vclQ{2y`E>a;P{aV9XY)-
zuHA{s?szIK?bb6<R$%Q}y#~#s4{vwpb%-+<xZXc`u1cqt^YN#iR@+~XK5ecy;Vz!Z
zvzL=oc8O4;)1m#`-r0K0dT}!Qt*lGMmu?kpHM;P}B7E+J^AFW7cs%VfjpGTuTXD5Q
zu>SlS-h;Qxju{6`WtfxlE=Nr2dE2C~T!yO^BzuoACNK)xFu40aEw#>L6)1f3KxWaV
zwqNJAiSD=~uNBBs61L#u4Cf~s-a3db_?R@c-RaZTx(mkZ_eSrl`uynVZ1Zy$Gi-{B
zGSZsk)26aHY+fp!w_(#gnfh~zjCbzNFF9@#?AjxA<0RA8+Q=YR{_4dE;vo)O?*H}t
zf0aFDOP7nf-LhYI+t2sO>BqZE3p40Fxm!E)Izxig6Vq)HZq=76CB2fR6eU%fQ?9W(
z=10x_lf3R;f8x84&C1_HuZM}cKMi<b9m;=Sc2cvi<V3D+6TXgLEIViTuzk8Dp{K?Z
zTw!<f`pkx6qiIsd6JIs2Dc-yG7H2a<O~Lo~_v5E391iY@JpZ~ecbTo>f@(qK2dy9S
zd5j*q#iT8bf3{+c@S)(d-xsZ_YUPNtc;1qbkTg~Kc@)=vp?!sY`x;HA7*vSwiFo#+
z`u!Yb@9J|iiX)!op568St&w<%M5tUl*SYqu4t(r8AAA)PUOCr5v2lXPnwFM&>lJF7
zwzSWmE2p>q+!|Z;*P+`cioMvdB6gz5()_lLOXn&WPO>dyWVpOScg|hO&2bBtu?0Tw
zkPI)8uyuXY#lqj!v;EkSl)$TAVIneIuNY+xPP)wPuxv`$pS>SnD7~0hY0H21^v#`y
zX-BqR`}N_@92u#6`}=YKGnak2*T{Fn);!nOApHGhzpm7%$j5J=+yDO|zO%|c{QZ6F
zpGx8HCr7Y8D0%<!)U0PmCI5N`zMoQEr~UO+*t%b9w>dJr7V>&M?-IN4U2OyYh^Z2W
zU#9fU68jqdXjzBr)<ZFS<!wU)TX#*Zy1G{GUe)&%`4@vjvRE9}PFZ$vuHj)dqmtCO
z9k2H?|K0!Z{eSoGD%#o0*zWRO*eve$I_%k%S;0~5K{<=c{rPxT#(ZX2^{OdQQt;Z0
zqka;FzRypZ-51DWV_CCRI#=gLC5Osfr7fC^m5)uaTd<_xtX4N-mBR749Az69GiY4x
ziYhtC7;{$axc>b0IXiefHb!uAwk!e765hTj{BO~r+u`R=y#%e1>M;y**yeY?DE00x
zlO?=gxI&pNlbzR!Uo+chdho(IskKvQ$Q*fnX_rFQ=c1QCD*nW+7kGUsZ*`fHzCzRX
zWr3Bss=^JEeHfgKE=W8OynaPJWA@8CKX(5%D*JS~`PlZYOG5cx1)QG3P*A?7*5r0l
z#XgPCpM`_N+_GcCo*mWimwx^$>u&gcrdt-5TX*yA+qGMtotbyX<in3|7=HiZzwf$*
zcslDPv*0QNTgC@(pF_5kILi1|R0X;}uMnA>_fhxLw#nK1tbQIZEwrB;lpX7Fv*Jq1
zlXQ(GbJ%@)PwU*i$v*wpB?*R^5|^ty4jwvYBtQ4PO2EXUc4j<_76%?Z<}7qIV{S#d
z#=|s&MT;YEO<%T)!-@Y%fG6McId{&kXPEGLuJg5>XI@X^*DXHIn$dLj)UJis>=v+X
zxpwCE=1A3@&n<s#eZq9q!F->rm8bDS_n7m}SHH{o@Jx+~*nTF^qjHbHLYtk-k1v_A
zuIR`$r?pEzH!X9qR&UB*ZWpW)eNN(S$+^Ia+cr;@?EG}+sn4f7mhx2#?RV_ft=M}s
z<duexhl5fCC+GjL8%xhhKR+dYo<IIi^UThrvqNvcm@VoX78hyGpE$F-G;`{uE4{4C
zR!B_ObWH5GOisz!e~Ou5)~eguM~-$b>^$vk8ZlkrnThERldI?E@x*zgX>N-)a(_HW
zbMC^wQ&Z1ah9rLY_t){<)@xB-o)dXF%H&o}-8xf9`5s$W(0PSBM;FDIoRf@Bb-lU%
z>ht_FM>`oEHqS3NW!rRC|87{pgvpKf!<c;h)89^<DUt3~tjJiP$bMP)*W_K?($728
zYfDNZ?f$n&h+dSEn(T9O(cN1zTVFC(6y*0y1Z{jY^ODS@Woqe%3RiM#N<Zh)T>0y!
zg}JR=oaV>7-{+m{>ldH8I8|WpVh;|jfS}q9-fuEOCtB{UtoC}{`KBw*^>_)VWT<p!
zq}e%LEt7M)pTDMDD0Vcc-*mL(+!I%Z_9fG6N=kk{xEJ?0X71@m7sbfXL+tKr=dRwv
z(lTXJ{OL_bM-%oRkILJ7=HmLL#eUkyp6=sUTV^K0Y|9^dF6FV~QzwIooZ@>nU*nk)
zH{}6SZ}wy9tOXw*EXvsv^-Qng`5X-uW&?i3=T$SUXL;>A<9g0iaD8fDDW_u1W7kN_
zV@+G556_i*-n?*GpVRD&utVnH<17BG-neC<dd=fIpRb?qzb7L%w`P-0`mq@wuUQ_A
zS-XaHg0b7XX{)w)o%Au(JHfLg?AkeJ*Ed<QNed!El`6P9;u>#0T%OJIo$t%ugCB1A
ztTwn27BT%=`{|Pl#P=|>OkA~RTZ2-Zsp$kBm-nlL7`~aFp7Myva`KVYuO5i0O7X9K
z8t~fblT_l7A2ue|XU;ocu8@d6D_vYF<tMg&NA1>Q2EkFE8J8Yz&YWjmHJ$zYR(_Qk
zxi_x1MY&#>pK@|hK=a){r|Q0|Pbs-7IsvpuxunGN;|rIyo8<rPsQJffD_tzPP5-oY
zvh2&)xvyKDHFz7=IGj@v7S>}conhGD&Tf@nbWh^s(|-}^L2-G@UI^?yb@lh|pI7%L
zX<wR~v2mAZ<yFzEC-zV6EZkB&ae?dnKIf|6l|9K1l9XmnSaUY!#%1P%N2gso8aA)f
z@aF-KFOy>QTJ@H!Y1Ur7O5(-BpubViJ_K#pw5rW3uW{<tkPU{0FRgYQ<DR#BcQYev
z!56M4K9^+gA6d`L;JSUk?YbGq?>g>1<Er>n>-ec#$9DQuW}Ue4W8<w5Goz@dA3m5i
zY(IW=n$tec<4V<GeDUq4EK_n0R$PAFKKG!)WT}GA1l5|!6Kz1bGV6<I^sR)Rh35|5
zOik_K+|8sBc8%%cLZLOLOA;6(rhd_3h^Ur*D6#whluJ{t#y#fS`ZzM1smyfy>q}*B
z0$K~+D&@$AAK0~}a98mgr6o&#2ws=I&HG2Z?4HEh8)}EPD3-AvxRI8+)=*9RR4M0e
zpXuA&wQpXX+f$K!%d#y>nw#O;mg-+wdpD__GgmqC^EiXX4#tLGSL4f!`F7b>UCj8}
z+<yDXsYT*tyb;SBUq=VAvkTW6T-@z{tM5>pVWq{o_QwZT<>aP+KGw72o^8ddt#SJ&
zD}VUhU+?J2yW{1@D)Wqg)0lOnwk`5rGy9FhGvh4Llo>(KnC>2*zIkTYn~M?eWnWZ8
zSxq`q9A&s^)7E<ypF%WUU7go{b$K6@|MEdouY~i4V@YyNa`vGn4`v#P%$U_~P@CBB
z`rZNS<h0o@Uza-TwyZ7+?$+UEFiD!TZ?Xu#Bln3ZLBAUEKAv2-adr-e6sN;~^~56$
z332xJR&_uB{Au-^$jiwI>Z1ISPGIy|e*WLf_xB^J&n@FS_-2lE)wvmLvpt&_LZpu_
z5m>X8&1dbJyD3~7e7P51ymQW3LAf%1t?2n{OShi&D!!3cYL*@QVOb|{CqvGYpk;wt
zRnHlJEnM1mq%yhZuE{s;uNgMW9%wThSr?}2)|1ZK!~2;bdDiBcj0NdyzVm&V{884x
zw|V|;p0$R|mqiwPz2xT!_BwoOUryYqm-ik^2)jB@d|O^G^!Lq;p89uuR~a4lJYT%n
zo9pr`H=ng@6odcYk~(Fop#HDpb!lat$!)2_S!<1#F6ORIJ$w4|^YqQH<7+3af3f!d
z+0GYkaYtubZh2Pz-2XCXat-s_p5N2AJ&)V6{g_Es?4zeSX8ApG_PsLN;qQgZXG+P)
zozr-8O5(Cn(Xt*x<zp|nKWBUj(L8HVyOS?LD>&;&LQmvO=SOE}2F;ylU3PDV&q~4T
z*V^J1zF6Iw&^hO*zIw(Hi{K1pi7C0uuC-0ws(fq{lUL?ZW2?;Njm(Aog6~$}e6;zp
zbMl!@bB|7$@PQY+2j{<)&f3UZAO3v({CN6#@rN@F->x{g#vy$7EtPJj*)yA`v~5i^
zyPncf_l=>nC}UfJS<R>PuJkFgtA6Ndde&~WlQ7OcypGM6zvq*vXBhkAuqi=N@9i#1
zH)<sa7q~j}zqh$~MOVUL;R$VqNXBI8UkcgP7N#ergw>RP@2oSa^qM)DL+qf8&z4J8
z!CVI?NL|rhV5wFWUi{`&_Cqe&w_1J-TkikmoG6qfy`*l}jrH#JK}=!OKL*UWrpvf;
z_FAv~uS*hp4PSq%VH3UXFz5Zv=jWHd_s}}>=h2J9(H%>;q+3^Nmt5r9{pM9zb%MyT
zZ3Shaa{^`luVd@<*Uon5=~wfA-*a0d<dtp8j9Y10y))Yl9=+u`IOCP1@G7IF^LpNA
z<gLwQUG=TuSVZ!Q@5RDytFA3oowhQn(QEt0MT!EY%ool{x~wie>-<7Az=rva4a=-m
zGc}h^xo3T0TV!DT!;Xzd`D(A~HVCdUJa((N`peN%K|0f_z8_Lksd1hFE_kd=&Pf*k
zIQseU;>rF2PF&kFrH`x<nfa+UfAyuCX{r*-BQCP0Dj7^OcAFf1K*aaxLeGMOyEjL&
zCiFf`d0=iiv+Ksyxf|tHIr0YF5M3Ex_(>t5I?F!2DJwCuahv;@o5wDnig`Teno)4p
zkqYZ~6LwZ+yqIz9u!OO-;c;&F-u#f<H`hO{DGFCtvv@AE<3Pa4OAlW?DG1{{d0;7<
zjPBy6HC~_ZTojr6Z8B5R``W5I^<NJzK7L>MWyhSmg=J^?<?=4u%#^peqZc;k*6T-4
z4pu~zB<0@v!mvP~PPOFz;&30fEF0^}%jew>hI3~#e7IvOXPmra!sN!7khiNctRA+X
z?>pi6^zHL6ZuS*B=5`v+d|m!|b<E!X&)&Y=oAmk`hf~*&P51wD@_ySHo)ah}(fB6v
z%G(oDBW~SWbjWDG)Cot$rPFvCrp!tczR4A?|8JVz^}hSEd|8(}_OSU%^qgJZTfBue
zhF7HdZ)GrRzix%WRAWp2s~y?10ySFBKRnPPaL@ALoz4lf?u3L_K43c7-D-RD-s9fz
z-pw<GdW)|dpJhBP*5H+G$+^9@>$Xif@$jYEvz~+3w?#fJ)or;`u?@8HWVQ3v?<)jn
zEtJXMEYdyogRdL7c~=xM%hF=5+}@vGW@>AnN@ZAJy5Gid=BHyOp)VOYjxlm5hC7*@
z6Iz?uEU}>I2Jh~my-sJe1eaSJ7b&{jG4*Ttu{GfwSpwZ}wA>O48TG!qZz%2%oW3MP
zP%>-JiLOnBd_JjOPhu|nD>QU35dY>I8ESd`bHnAEpUsWSGkqS%%zgdhY0){>=L}!C
z3RZ5Nwpu7Z?b)S}$&K$tlVZ2MWH_i?@nuR}wDIZ<o7A@Prg^#ZhgsWhFcvd7C-dgp
zyQ3O@Rt<9=xI{ks<EWR?6LNND#+=DPMTR#L*?cr!&gEWxr*LKX($ic9_xtMmw)9H~
z3EC}+JhCw(Cn;q4x%q!yeEeiu`B2{WpWNP-jQh7F4Q-c9FA)y7`@~?i$w!Zpv;=>)
zy$_Clo2SS2d$kPrYYk;*1D#LHtSrwQxv?<nS@v1>hS+Blg4TI?9F)>GdK$izL1xCO
zhnXEuY(h*OidBA_@_s8z30&VOBiAFveAKAivSwMx{oix4mE!DLWa{Lc_D`EBRQq1j
zs%4VvbHxHhmG1|4r|K0QGg>P3yk)|Z%JOrZimQdZHvXy7DN$t&`2KB+)T_TDHz$1P
zW#yi|lA9B>2zgKCzn|&n?SB8Rudu5>y39|z;iX_s^eM(QrrRPJ18jn2or9w9+!RSt
z+P&yjS8iC>`$?N;^z|!+`M<RAFW7RjB;rlWk9}+VXB&RHc}Cj#@UvSBi&$?hkI2l~
z*eOxFQHy)+*FA=p#oJ1Z7-sHD3l~YOvTbV1X8D~|=GGN;#yWT9y|gYr6U)8U3!cTg
z@Q2k1czS+sh%Q_#_%>^*oMDQW{ew4I>3a^JalKHOony?e&^^htJtejJ`;+C>{VV$_
zvl;KI-nClhedwOA!`{|?QQG^wzwZjySo^hO<Mhpv4epN(LyPKv9xP^vwpd-r&M=j4
z<%;Q=uI7cIO4_HU@_u~nvVHg8xQi(#6U^5YyCqM5RF{(`c<EzF#FlqkTi+bJAg&X(
z@$E6MtJ+sGKK~asyZ3Ed<KZah^V^bZW?YN1ulV)hcm1#Awfw)jmPREWG8X_ZloLOs
zmz(<b<n#T%+BZKIHs5`Pi($*U)2rsHL`(&(yD-+`Q}POvEi-Lj<-FB!6}R4z+mBbN
zHzkyQU=mm+ILY#4OXiVnhj!UKbP6!+3)+;WP{HzCqay>fT_K@~TW8H<rq(IlLL6b)
zGmY7nah+t;WQ~saT=ebO&JD}iKL<!Ar-@34^*)|?i9Olv`JD~gx3yVj$Q4@VpOJ2!
z`n8(9YSa3;({0L?*G+fo+p(s3rqq_id0xeen$sWN?Uv}SHC)Vi;IaMBIPLP%5`~f=
z2gZP%x^}K_yfmj7H*QPPXj;D{!LoQ_EaQ#KcXk$RvsmZ<#EUaZv@^6&ds#b!$MHo@
zPi{$e#dhR_W&l|ACT*@hDyEtHzV4FG%aaG*zsWqgHzE1Jtu#~fGmlqkRxWedKP~qJ
zQ|E$7XAi5k<j8#g_PMk$-e~ukqXF3r+}Z6>=I5q`N@U$#|I{XA%HnN8i$0#1l5gt0
zzNq54Mww_Rr=rc%b!VNgu3viXk|l#)dXDi&w?NsME^AiXxG1>_?l@<;iqq=(v`Xu<
zE7_XLx3nu2v`#4f{c>(`#ICBV!oQubUAO9t;JojkGP&C6(0+le)U)gDD*yfZb3(%T
zP1X_9uRKgIcHPqSR9s}LHd%zln|b!j|Ch^EwHX?&-no=~cBN+kqk_AoZ?+k4j<M?3
zj%yC;@wvZt_V9i_7W4Z~^W64lN-w5w{(SiG^VdQbZEZ~>B84|pr>`kLZWY`WqH*@^
zIyZmWo*i<AXQYesmN9(e`_hpu<?j6UuQFTqnUf0@t$)tls5Za)+Bd&fH^OhoItq)f
ze8(7ntI?ywD9B{0u@9q{5<}Bm^Jh=bDBS2?u()Zd=zG!0-3xjR6Lzg%&#au(#$@JO
zv$JacpEq07%lIyxdVFxpgLlgnWKa9@<*%LjE;;AtlIp7TGajdmj$K&t_M7yft#a}<
zGi#zh=Dc@!{_5{k(`}J!y^mz@3#qyH<Q^+BE?Rkcdv+?bSthrGj%qZ+6;6-b=Fi6@
zk~5C+J({u1cMWsEB~H(dE)(v6uJk4UkLJJW*!aV?zerp&oxyE`cJ`Z#;`-u=m;Ey8
z@Ba1m_z?x(ru1J)XYJ2FFOoOKc|K$au;tSTU%KJjj4z!h7l><XI=l|Z6J^q8NY7>P
z$lD02{dX9rH5HzedEm3y>~rXv%q!cnnw^`EugR+4Y`VRvx7mF?LvQY~<cxw-hvrPM
zn5cPCYMN;D)TFXM{+lN!J7})m%ye1&Sp{R;%ajX~eZEY&#&uczSx%bK((lKAJFEVd
zi)`Fvq~5-)aE{Z%mv47uESH>A`TAVM`!^@|Y+EL%myqh~A7k1Vyufm4-lYsFr!!^h
z3s+nHbm#LlF56XCc4@++2<e<Z(F{_@#izbD6bX9uvCi;9>xW})b6VXGZ{_c8KclxQ
zRqy(@;_#cN!Wvc-TT1?Ly*jV!oZBsllYxd_CT-p8=34Gu;rU|GPn+mh|F$0X>FNA)
zks*g6+TyX7Nndc!?|@Bb1C-6pb7rn*m7cItFL=>}oT<s7Nn6x+X6Ob_Ew!!=DzZ1v
znHm4}NCQuhX+gO`Oy{08T1MWDF<}R^)bk^M_lpE8&M0Quu-7R>b(&?-I_=kUHb?tp
zADtuqIY4l(R939VHjU3~93IQm=C737dnV9h+nLrU`~K!9K0f~HdJsq0)G!XsmO1SQ
zSc}503Cv#TAG+>KGrLvAMdAI^9Mn~6bis=$?faLSo{zV$|NPT_j_iB}2ALV>5{r*z
zJxNV%D%Ufa?8xt6cw@P|<s);G%euj<3cnSIfeztV>&@^kN9<RYZ`lvNtwob8m4XiL
ze&G6nLC5y#;iU<W;~1sF-CdS{)L^q}ueYprw(nIbz3@nS{j=Vgb7j7Bh4Pp*Y%)6N
zY!uyUxN)`Mo9FqfypzB2eQEx^>to9N>zBok=lAnA>}9sS^7db7=cR?4+BbV;^v$)6
z<SCK8bNlhC!aV1->#i1V{Qhs+uKQ~xggz>{xw5F{J+_#&)%bgtZq|$Lv(4K#thnRa
zkPz`y?|Ic0b?N@ox~1iJns;l&a_nOgKIiD6*^++GO*(AZ3gz{xx%}_mUbH^uQGP4v
zzrxvX3wL}goKbXQ-i}8}hg#p5@qfEo^>Ukf^Swof<{Cza+_1PNyP)mTdEV#{zoMJk
z7rxi29_cU)zd2{)G48eOGe4z<PTaooxXe_Msc#A&y8Fxv?cUwI>tk0(AnzQhV2?=V
zUhxE`g5ni|pTB%n?i00{k`$Ub-Lh1rBlVq>{+F~Z45r(IR&Fdb3tkhyJzwy2Ylr|#
z!tBsE{+5K>TjRwS6<+`MecyM}fT>Mv*+It1{nk0XUn(YePTO=>cwZoR8H#ygrtjl9
zhR?d6@3*n~$s)&aqBMlLTL`pf!ge-mQn=SkpP!q!w!b-0`+lwIq@BFGH*JeF?VJ3_
zbal(KZ#iOBkGu@zWZ$J6o)uc)n30#end`>g>ucEDVu~Bjo{-&n>h4NI6V(6@S$Fqz
zuetK_(?UZ%W)|D#pZWMEH17H`!4kiv-`lxTLv>};m`}3)D!twN<D~t+%ZrN2kG=W$
zeV&&lV?gk|?$7Ds$5ZFNZZ;GSx_)r)%$*im?w=j!o<8YtZReid`ghyvCp=+pU|D-(
zhx?JGw-Y?3sRhmG$T_;8UGH5_>0Ix#Z!bz8&XD4n%+R!6B23(oJznvKPiO1W44#KF
zVpD^2_JmEp{q(}ZuJpx93ZZPu$C(^hx{n10S1UL0&zF6<;lYIR?@zz|__wX;$-Z-6
zPyN~#b)I*(aoV!L$&KPY%uNxl>jfsAPOVdoINN$c?}kl?32#B-+TsgU6O4Ci9KI-R
ze6grHe}3ibvm!a0Sl6^fL~?h`U4H-Cn>TB?Rts7Fi46T}yK@c04yzE$<WSD%3M|Fz
zLN_otHuJ4~%wM4Wkny5LdM;?fg5QE6RkTRWQ@T6oTE|4WXE7NptEcZ&5PCo7wB+V%
zKQ8oLD_m!q9BBM_l}rAH&=+@gmsK^XHckC-7P9c1(^gUXdEDM#JLc@0H$P?pgUzc8
zQI%5Z{b#Step#_6clOnJ^HRLluIUz)eAOs#=r;9WoZ~W?^x0Q0eS6m<`Bt$sYv%gt
z+rpnm72nnl`4DC#xJUJT^?HFy+Z)$=c6~W`<joPC)@xB_^D5-3Sh-Fbsfwv`o!la)
zzvk!D*;?z@PWa97?dify8F{a=Of9`9+wPs0^lC4Cxh-^dcBpgQ+!>RPu0Hm0FSG5=
z|MCpxkF3%^m3;g2^z-9mAFY$WeMs0Z@n(K#UhKRb3Tn(dH(&45KDXrA&ePc}{3jnU
zt&97k>-*yLk&3rG46A+x{$PsrnKWh52A2ityQ^QQ%)B%yG2U45yL$58GqLBEh{*Z8
zF({kaUp}Kyuc7l;vzpR;8R-LjOW%1~o3A;&Zo|XkFJ-%bimv@|_V(q=-Lu++E`9s5
zXkU0vph@%g`PS;Hn~x^kxAaJBpTub<=f%0hJD6AXwn-P)r!$&Ir^s+AuD-O6d!ydN
zJ1501H_UF&O6<Lsyp~;9*Z$vf^QcX4_a1y{7#)(gNOftD<@0SvY^49I-`F0sVX8p*
z@^hBES=rQ*t}sn4EIu|zI0m%;ipBEU#+Q3<s3fe^{jn?f+_{I37o54FyW<S&SN{M9
z=H6@5CIxxSDemhIJN}8$)F3HmCbNU=olUzUQhyy}p6fZ&Wlc*yUyzTp)1mdvv9}l=
z{Cxd<ece~{_4DN~#u?SF_foiVI8q_e^JCAg_bcZLT(sM>F)O#{ZLWz%<a!2%Qzi>0
z3p`ykcl)Px1((nGHG3t!w%dA#x#REusO+^{4by%dVt5<v&nvO4>-{XYD^2dDG7EUE
z8FsFpZ5H?N_IvwW<p*2mnLoQ(eE2PB0mSzCSy4Qe%O2mlwK^@z-Y&Cf%jVY_>;=2N
z|G0Nj{NmMrvoFi+sxbfi@V5W)C)eYmZwGfNeRU5h-P#-WN9o1O?(;7gc6oUp{&h>I
zcUxjv`k7tbJ<q=9eR3-IIzRp0`K!NYUSb#K-IB>syu#5lKb>)@$Y!%6x|Oq^9qf@1
zv(7pqaaU(oGs6}+dwZMh&n8p_#-BN-@l`pvXW2~~+ZhZ7X>121J|~r#UED2V?0ug-
zDxzI6a9LZZSMV119_~sLIce!f8xH-*@#%H9&UtX_b6wuDFRl6e8?PNZ(PqY$Bz#9;
z|ABrr|Nrs--~adT%JKN&rl<)zXn{|uZQFVO<>x{>-kgd$5Sg9zdDAKRjJWSyOHIv}
zt@1kE$}qt=Y`e+oR~xjsIc~>%bCFo}agETV$dfV+T0NSFYooW;{x;0$T6AgpPHC;3
zT8X_6nMBTBs5rx1v}4Zeg_D%mtrV@_v_4!_>{-gpW^ThrR@$}k5AS_woM0sU?qu`w
z)iQshxHMa*IJYJ(2;cenVC<Y3dzt^r@B1nrzB792j?XUBr~Z8NbMw@%<$U*kziT#}
zxv?YlnGfIBzssx`g+Ct)xi*vSNpXCMh|;FSiMD$R^L}qSa;)>~(oDYXPLfJ$%=$AW
zyey{_Y1Gcmex|tON*C{!eIoN_an9im+K_5@ZszXG(Um!#5B+^z6MtzZPIJ>gr2hEh
zw@WWHQUkwath={drt;DXkDFcpH)s}Kny7MOqTy>T`TIAgFG=GS)o_jFcqubybAv#W
zZgGmZ563aqU-xv&t~_WeOq|O+<I*nO<zAXw<w8QgzWJH6Xz8TGlaH#_>^=I;NL4ER
z<&3E(b_!TtKmFk3hbtA1cQke<gjX(juQ+q#s!Oxn4D-7W*5yc_c{o=*CFlIjt%Y53
z6S%x*avzNT6!o;{Ab-Z6h8t6Ai;uNTxHe1Iz`&G0RI#>q+vH~Mz%VwQ>WU8vppJwL
z_`t8%3FrIIuDAQS{OMK6%G>T6W`3H!JW*~{vhiGLx5>F3v1?B);_Kr!yj5}grFvGZ
zlV+sxUOoSFX^AuAS>I&EvFq2&nDn@7(?v#A$)@cGyC1C0bcu_8W${{6qVTRocCPCr
z2R{GOr8kT{Qur7)uCDyN=%ZHpCcbrixf$0!9x|*C;`w%7?0ER{yqUXC?Mi+3Yir55
zvwmmR9Qw`wZhv{fd~S}P_vWQ7PfP1L`TAh<--Ffu^UdO>OtqXV>#f<{IPH*rjwyd=
zs_fdzRNm58fdW?oUM-v=B3$3?_x0+o!acit{cAOMGi_M0Va1(*Q*n(+8`dus4^obJ
zepIWxyy|hO;<oH(jj48Wac8b9_@8Afb@knmkaIg5lVVowOV>_pk<sgYmu)=dO;~*5
zmzK42cW#|D<DuX2;~VX>)2DswHOx`iBlkX7XzH~~mg2jAW^CZOAHaR@U*_|LCnRq_
zoOfx-4uNx_=Rz0%tqq?S5VdR4;l<`xU5vFvgmz0A&WlZ0P;t)T>p>@@=dR~e&d06^
zxb6^o$ZYzo>)Rp^#&alUJ}7@IGvmRcoIQ&cY>jJNwd9R<Kyvc3Dg1pELRSS^Kd5(m
zAKX#gG-=C=SPe06j$i+tzZTl-bb*^uC-!|-=iW{(yR+v#f4G59>CkWenpRet<aggL
z`b=L;s$0xC=ch3lbJluUoA*e4VtqXM@87AVOG38VJ7pxxbxL17u*+uS*({?MJ1<;#
zA0x{4CBc~A{j64iRWtu(tsc!Oi3)-?JFb)(JPA-s>{%rLC|N3sJ>f{H;?IvC*Q{A(
zwd0i4q}^JtbBwRnTnPSc>6|Lt@PM&v@s@WL4^=kb_IlI%e)`mx*NuL>n*RSwdAyDJ
z?I^pQ_WQoe|KD6NeYMR5r?PT`61_9dS1qEaGTh?eG?KHuA#cEcnd2nGjcEr<G_IeU
zcjooM0I!3u7G7z(+P!LN%$0ig=%dFLZ(MA@{Z566nDfi?X;Yu<doL-@AROhb`SsA8
zDSCGvHm{!haq&Z`2$ufOx?2zXylh&1E3t;5so3;L$Iht&%FovI$o0l9iHt3|DY`P!
zByqx))$bT5&N=I2DiFu_ZRwPT$w5~7EmNGfY}c6=ko`=C;n$%E`Iaf4w8};1axIv^
ztk|-(ZOLM};&VHFHacY`JM-x;-E->jp;<mFW$F@hrLCeGy2D;9%I_!+GpIA;wiSHV
zmaAzPV&t~VJdTk;X6LdOyF)zO&g3jexnKP}<DQ>1&qd+p2eJwtHy8a6-gxHd8lf}W
zLxMUswJq?A(0ib@x%A~bnVzLfep>2zMsmjIgR*e}w^)I<^cvfrU%C(fd|473=B#@9
z)S5L<51I41=W9u2UJ_39_unCKB}#sI>xK1EPa+Q9{&9QTOtT1YwH(ksrhQS|W_<g$
z?%J&&A;!af@R(kM%NoJyQ*9OJf9iPcQz>=!w%xZT?jDKT(*AR})4LDf^c3B*bmE!G
zwXtE(-fo=9qvrK`PU5{8Neia$RMs}RxnY}V*TH`QC+B$n+VlU*&*R01zwv!p{88dS
zNtsvU%ho5RhZgaeEP8&X^MQ<lit<sX30#-73-S-NT=8D`-E?-(_pM)Q^{=~H?=EOA
zQ!1UCefG7k`Ml@Tt?Ra{-woI_uOxl*f*U*3WBs`pl+}zi861^A7pcu{p7QD6KNg=K
z3C+eaPZ)M2Old1k;yF6~{f0}<AMSj$eJ#4_{Y{%4k)dl>ZLT!Eap<O$^R0cqw|30!
ziM>|pU9tGO&~d4^nO%N6_S>s0Q)N|d+Y-CE@lwNtOUFBJFf93UlXYFuc8y}q8(vRh
zejHnUYF@yFD<>Znwe!9RVVJX8Bc}J5*RE4$=gvC23iD)UCCbK>9lX}cagSGZwlteV
zk^fd3hJ=vNH7~rQr!IIeG(&1tRpQ|=>k9%0Z#$hbt8dA^<?OaPl~1eh`PQRq8mpC)
zBK25iJlk1pcl+XDU-yN-y#96jdi*%-0vfxHjc{Om_vc4}m2LIsm&VfPvfKA<E`2R|
zsQYGQY0-@C8k>7^t3C#No?053$>=4)WX4ziDB@uB<rT+v&f}OHd5&vC@!>~Pc3C|B
zc_}2u$7DU@^QIeNOg`b?E|_SozpZv~YOeY}ratbgJHjlZJNK+Pv-$beZB8G~v2I*{
z^Uac3p)+`vT$+^=6tjS@&q#K~Q67QGwP%;g8S<;0I^`2Daqt!U@5c+Tg)m==5-$o+
z)>)&u`c+9;O=(q9u13rKdykoyg7%&6JetC%zvQ0u{pms0&x_cywV%%^w#Y3gOcI+~
z{ajw^xtOTTs*?-c_hf0@y!7JYjju}<%Ri1|RGcv(io<`Y#v^u<->$D4r(F73b6WY$
z{^hS;DLq+NrOe749OR{UWt#ohfEbPgu{#Wsj3)b?csNa(|JYtv)}-(y;S=w6$sO6V
zdS6tr;mgj|^Ei(5uRLNY`K3i>5|5ej)P#Tk{v8QD_A}-+W95$}Q!IsJERwY}Z5Noy
zoMt*?$s-bdV?n9(f(sR;_4PZ>+<l{>Q~XBn_39XtIDvyq4#ov9W^b7E+azp{V)RsT
z4zu*NlG+uFTb%A1#UJL+_+vbSLD=+3$JWUUqrZ#Vh-Y!%yQR<~pS5>GLEhnw4_2y4
z>{+a}EhuGI5ra=DXS{e2m*aoVfT<V0W><z@T|KYjgF|}Je7mn{vnPBgb(Wesc|rmA
zx3>>-tZc38e#$hYIT(BvVDMUR<bC_})!&Z27r)gUeW=o9wN`1j(`{ppEyh7cNr8Kv
zX0lz`v#g@+Tt@osHAYKUT4c-?4b)@VT&dVlrgCnN;mjZvr-082GZt+-7|YxB%u2#I
zo4I%2OU<mboZip5pA#;wi^;T2^3lHV=fO+v%i0@$KY4KOAY*A)=Gv8}{Ruv|#cyV}
zuM=4<6lJ*f>s<9;=MFNO%1I?Z>)m+l!i*`KXYRaeu%p%Z)zRCBBx9HA&e82%rsw_1
zgHg*%wz>OkhWN6i&k8dxbxBHJOJ(@*B%x%h@w=JV#m~OldvA}eQAF~YTl;o!9OcvA
znq(rpEIi@JmyV0lvFo+UxXYXaGS6T7X0p92z3crw$;}P1w>TLB?$=o@3k%3T%PwEz
z7N*V+)xWjx()3He+UhoM&OB_gEH`sW+wnCTy}52a7d@WEExf*T>$57I^Q&6IWO5Hb
zc-A^w&xxUM8lR${Qa=0j3pHnsU3l=JLZsF<Y_|SMo0d?{r86?8Y-MZMnrN)mZr3`u
zU5W2}pv_|Qt_6(NU#3RH&7FEpZ|%)JVTy_mUw44kez`<UTCaCje}4U^n^$uV7Vy1V
z$P{*M?~J)SFQmMiqs^G$@6V;_-@DT53zuV6=)TO7<+Hp$Tr<-?``RyHH^;qS1)iU$
zsunL3bXnq~9Cpp0ZC+306i2Rv*re#jH=m1F$V_{(y4P9K)ZogWKTEW<omK5`$M~^L
zPUSn5c>m_**OxAq$UchrGUJ-h+rQ3h-FvLX{nC%OrZ}(FEL8$cH=er@W&a?rE-TIS
z)V}r!zpJYPJSyw!54GO7e0b`KaMe?%zU(#Mw#d?;oSoVB9{Z7h{}K!?Ox`#z!|dXw
z<9E)@yZ30aY;C>9-U<9|mzZDd{Qm1!j^!%ONd>9yYvwA&9A?vRNqDl1N!X@5q+z$Y
zM10GJGsmsJF1nzz{%~Y41GlCGYm()2HZQM(Ueoh`8l~51ip=0RceYtGJwJJBL8AYn
zM%mf36moAfYw2k5EDiC>I$3pwSJirU>%uSRIG3jMcZSX8)JyM4ieg;j61Qbi=#`w7
zDU-O>RbP8WPB98Sv2itz;pSs=BW6#|m0_4WnN7)Z{$w##*2Rkyee?46%rI9o6VfZ5
zc->UDG|;g~bBW5&E~&tk&$L@?;u<PB9%_p3W7B%wk}%WZL*DLbuOC^x>0*+4<$L^*
zk@!LDgB5SAO4gmd{*<X9de=R-BS|Mar+zR6&n>cWCuFYv{P=MFj}PMO&-b%P8-%DO
zl)gH3Dr9Nt!pK{48&|78-goWyu^Fqj99fqU!ZG92%q4TQCQYnT;xL=W!|=5AP{?$V
zN>TSm0U6eZON?gBa5K<RYnEG(cJ)~E(TH<jWs+F98*WJZvMpII_57Szn~lk`#%IlO
zjgy@xYySQB*YRvgf!?OoJ4A!{r>a<)CjEK(=1)nP&qg293*X~j7n;@e@2*_f{(ST1
z;9`%RPg<uewmsXb@#MgbWrg3?M8(Xr3ZGk`D5m<(=3=Xi!AA$rZD)>G&sytJ*UrB~
zAm~+SpYF2aAE|z6Yp%tFHhYLlsaIdp?S1FTy<$dXpmWcSW4G7M5IIxnc;t|?!i_*)
z&1$zdd`DJI$X*n=_*SI@*Z#w%r8ia{{AbiIa>kOSye9Qa!kybik5$jPzj^58krTy{
ztik*@`O)48+b@paW(Yf3scS5qyQ67B)y&-w7EWP#ut!0P#p_ecluMqL(=NHXxo$35
z^5tA?Ft4Vns93mC{+@!++Y7dzVLf|nVv}eQg9F#I8oxvLw8}-!?PKLvd-dRv$r;|?
zg59TbAFes2ZoT#S+O_w#vKRAOJY$?teRuLi?FprGU#IeA8N^IqJ9BOo=gaeFC0t(4
z;l6h<XyqgiWeY|N_Va7bxM%%Xpk9&`&KV!RL@3dT$&t|jeC3z9F=N9M?TE>uPj_wE
z7HMexdUdG|+u7R{{!2~SH)u`NG|lH}ik-e-dWx6!zSSbHMHl=CST*&6hG<pt?KM8m
zf(s-Q!mqKs`NnlstZ)ItnS}y5YZ!YaqzmVYrs{|X`7GUf&@7lsaIO@C#f%Jt)`Thh
zY_qn9FNpT8va~io*0W<@vB9$KH!6Sqb>ct1fx)YoHQ~sVYrig5IBg31z3S#d!?e`3
zVLtu*ZasG^FI^M9{;fE9_SV05X53tL<mj?0(e5yj-ColZKPQSZXx+#?RVKy|>%8>q
zJ7Hf_N2|k&4Hs9<536Z^y?U-8L+reB3%cfh*PC^f&p6!r@$tuN=T+?w-w=0k*6#8h
zanr>PGF92+-Crkh|LC%~n`?zGeGA)p;r99SX%BCHmP~N((Of(A(q5)(GX#^p3=L9q
z9X6~MYJDg5{AJt=zl9s_W}P*g&h+{0T}i`rqSqfcpPipmaB$NimV`4oa#9RYg<5<;
zFSL)&TqImETXXBofXv@nv5(K4nl=5k!}?uzI}AVjoy)z$zao!U&|!VRHMUa<ORF|+
zO}u+-^&;UvZRP)CY7_h7?aZGEY?xio$r*31!m6#v*bsZ`)uKZ`&)j&NzWM0FB;mGG
z5e&;Oi}P;%J+aJG@=F_+rL+lOXy(*Oz08rhVb`uXoGmF(-xxaK({9bC#4yfJ8_J(s
zg(vRW)b{7SU5p>w-dxpdOHF%kUCrHY!^!!v-5}q1H|xiq;N(@8*z|LZLv4bq{uhL;
zU31g=f@z8@&)*l<4hH^?%U@g_SQWNR#Bj;NYm;mDX>~7LC4aYGBhzT^&ROlt+$0S5
zmDiWMKf9-!edD^mSgD1qq2|&(_5U50g($MGwlO(3HMF!YNV2#lEVOs?xt*I=W!c1(
zIyHuN3q@_55wEc1UP}I^IUB1Mzc`Z1`%vcByh6n#3FYiQ26K&P8e8%wwFfT{dE9jO
z<=VKl?yYIn*1Mk`UA;NWI$bYaFTJMyx}C{uBOVccIq_c2)mgy?Q!OWlyM0@-drchQ
zjrk@OJCA+1rjW%YzSREl<E_SaFJ`V(GdbFE(LtJHC1cL}lgGF<|D-;!Om3B{+OG2@
z;DoN?_C2pXHRoEhxh#u#?C{l*VP=x@vL3@AFU_YmCNWbi??*qKF;|9vN{-AN@gM^&
zwR6tKk$g1;CK{beHY?UmFYZlP$-HGD(`tF0&#RApJb8fqeaoHmGgTOM(kH1o7Ame_
zZ!$4B6&U?2@5l`!@z+AFDOWsBv@UpJrWG$1#ALFxeUU@V`T*syOV5?ISM%;H`MOH|
zo0`_--z&XleAo(ZTr(V5Wfga0^6H|h$XS)8ijk>00z40DIi^OZyqmL=%Zck4w<Vha
ze<Qc!waGr53`>QTpS=!XeXk;H()h0T;8MdF52et_jr~&{jnXE{cRynevNf!@@vJDE
z-@(!?Cf%!mQHiBBC1vlT#Ny8a>-_u~QfIGxw0o~p$TiQlxwe*%cJZ<KJm=w<_UygT
z?o(41)vWR1eAN2n<0B{0NzZ-sp2R9eTt3w=vETZhSWz~c>cc(JdO4=IE|;?>Yd*dA
zH}?HB%jC}&F3gzX%E>ZY^XmH4WiEy{^i?J|d1}thcGj8O9F{Zr7^eZ_j<vR|YVY{6
z)V_68f3;ox(9P%C24>akOFti+I(7Fc<_AS;>|MfJqoy}<ZwvTcZ}j2#jfekoH-5e+
z%D-yexmk~to@B4xIlK5E!xTM+bCnC1EMMj{&-&omsTaThI9kYIt8+~vXsSWp;xbpq
z#anuvuAd2s=CYjZ<n>(U>FMV&-VYf~&gmZg>$I-*&aQvAPU<Rdm)N#+N~O1lLr+%X
z>Zt;)Q;yB(<@MVpwp!ru7Ma#QFHP~(2=R<L%68B9oOn2Q^SREye;MamNo{WV@WIsb
z<hDCIG*`_zY3af@TeGnARa5>?;jOcR@^e0PtPDSGQ?=p9t}R}kRSwxo-$kE#aUA2;
ze6FaUQq5DS^L&Hp_P=|Zlf0KHUf_)P1{dYit}z&N7ySA3^J)&GLPAqQm}*bvix~@!
zh1f4@%r>dvo!!__!m~ALt@nkNgK>>351yK}FY0oFU7IGuWsR(5mt~d(J~_UqYvH5a
z3^~SgU&Ow8qQq|1@Ur`GY39_d`+c`W8GYinl$xw^J;uLvW@E?eXGXWqM=ZP2k-duj
zqJpKB^xp~MRUT6hUW(Z}>+Ee?>-UTc<}%E)sr_-UBIL!xD0_v@(>JDpHuse|McwN#
z)QT6{#B_}9ZlpQ`i+AbOm)(cAZWZ^9h`zmzi^-(Fe_Nr^;{S)FlK0Ix+cQ)0_Jt5T
z2HVfKZfP?t{L(Hyo&C+EZ~L#$?oTYQC=)bkkUe(b%3{OUE7hYm%7!&*I(*+cQ+sdg
zX2#!>^YjycFZ=enzfX_xg4w$s-Zu62W3vvgVNgwy*u8RPVBn;gmtromvIM-@6TV^1
zjmmyKiv{y-YCX=j%}lBazSMj9FT)w<=7g6k{SNYdzL0U$eT|Cso@<ZSEMOK%=;e*d
zwwjl9DuFBD%3F!s>MPr?NC@i}ReN$XNW^T2FjqOk?r=NmhPKF5CiB%?{MF?QduGfn
zwU*>mzEPPR_xDx)uK0e?RlSp%rcOwZX3zHDS5a3{ck9esZB=!i=jV2AGGg%K^Yv$W
zWm9joZ}C>UyDcIC7EX$h=WZ-wY27*7%_A*VpfY#mlk1x&cgOg!)#XLZ%|4quH~S<@
zg1)co-j=r$TW5q`*we>4YvCk@W%_4suDNydvh&*TYfYV@eNq*lA6+nMQrTF0HbY>2
zLUY_jwb-v+Wh?y7e(9(@CA?fv<Fi26wRJ7>hBGyfw$|s@r0@JK|M_VS+tml3k8ujE
zW2mTi6n}kh?{Z^3m6*(Rd!nbQFI~;^H2PVOr1TC3*Digp6xO0+8e2}@cpxLi-xadp
zr<wH{omjujH*!h!^>55ivoTmm={Vn(J!f-;@4H;r^CgdVti3fov|#GfAkTw(P4^yd
z6~28b-g4vFjRr~O{&9vAWzTMFI6C*<sdlxBCp~AS*)5Wt&+XyYtB8?Y$<}u=@$r+r
zp6^##b<Zmo+W1a1n&F*jw7hKWG2Mo(#>^X+u{l3mJHaYlsG&H|_2qM)ou}5g%rky=
z_tI|F9iKB^91^pB-)Nm<vi3n$kk>241tpiK7FbTcGT}wc(_3$}86r3@e!us#W@`7G
zjiGts+jd=+4xhU8+`&`xGS==Cc$?d_+_Li=r<cOyJKQUGIw(dyc0HH!MqcG@R%}x2
zg<gTpTv0(6Rw!47a-6Cx50PKOq;zx#_rW>EdL|#X3KXsQDi%?ec=FG?Ju$m}o!lpO
z{pLH*<6b|0DG07<$+ys1pr`fy+ttnH^X+~o{qt%2_vGKZ+mE@E?{@rLwWXWOR*>bf
zv*|fW>(yGXyydgbw$6C-`Pk~p6{k*3DD*1Mo-JxH;f&zoRjex%ZdkYr&o$k?*J+RK
zI)<EYSN1y1m64LlyKzoFWzEN=vZ4yJ)%*UI%qhNWar@U^uQPr;4>H2OY!kA)o}cFT
zSxUfg{#@=SG11YYhtj<QSyUzGhpQSrb?@18{?EG#aff|#7&AWn6yV==R7Z<%+Ot(U
z=TA?*G;8~Db#AG)8L?9e9VSev^5;KzWu9D?t;Q1vubrXYrHVxruWsG`v@}09>>T?A
z=I1h|%vSnp)sw22*Z)7*s3|WqW5Ehm&8J~+Z9~reaq!esol!7V@@m90$J1?{YnpB(
zYwtDw_AiO8bp5=a7oPO|zEE(m!<PBRr>CwO*OoGG*n9og#w8{UWo+-~>f60cbo`%H
zE^F_<r|X%}B-Pbr^SU~_xr~$5W;JqVpAp-0=S;ZTevt$Q%_m`*>+(K^H%*AnuH2?w
zzI>|wlLXZ}t8|iftSG*!rN2G<8S?=V@u!EvHZI=acCTWi-l;k7cb*D*p78i+O1%D>
zrCnw-X0&Tn=zL!N>d6ImTVprxZ9=&(cYbWT<nZbNn|RJ$NvnP9C0+ZkG5EHfn-Udq
z<5`e_Udx)sx3}V+X2mL{Ip0V#UDa-)`9iB-=hRo*X-2m%9$skY@kR@Bv-ZA=5w}(q
zzPnWOqrCp_tEUf-KdxCK`fAntRp#rAOcPd}J~Man*RBkfn^W6nE$-zBNKMtK_?#oQ
zW5V=hSu*cCS8MgCGy6Bs=?u};;!#~}q7&?!`EFjO$l)o=p6Yzwnvm=j{@jUY>%6%#
zOgC;`HobLncdz^7#)STR5<!PRC+Fn$eoWN5AtAk9bmy99y_zZ7F@ev1FLF`z)=W9)
zvP>jEXt&aoXIU!h+x0|#ZalZ@ZELNw{p2sz^No+SD!f|u)qyMR3{Qc{x*I?5gg#3t
z+oSPN>-8MwJ1aSz=0wf@ZmT?%k70(AY4@u3AOoY#(?!p259BCoH|F2?^z@~t=M>wb
zJP(%b_;!%V!ccU^jP^jq>q32JA{f5ic(qx(aQSwlqqd>p+`GR$NlEH@zF-Rbyxpae
z?krvh&Njc^`TgC;Zqa>vk5~%7zpMNC>ASrit30kL8H*{hSxuRGCrE>>N^aHDm9gTw
z?6I8FjeC0|uI@dtbZ69w-CHfc=k1*G{`#{T3q5J|BDL_Hs-8RzQ#KVgayuFqi%nZ3
z!?JPv9K-OL#~2H?7^mmRtrAQQ{JzNk<lA@6OCEG~D{wuN)|X1}Y<&8`^s<wMSBY}y
z)SkDeV<wcD@-KY9SAb_9n*-n4)iG<lZ)>}R-g+Cgj(g(uueYBCom8w%f25;Xw0rgL
z@+$?CT-LO#mlQha_2bwfz1*weZ~ok<`SkMh=1=mflQk4>q{Pm<elT{fRj}{M%7R7Z
zx96M}*zzn$D=C~o>bcmpeTE9_s~*ev`Yy~oo4$F1=G((hPyKRbty=T5;n@?06~%iq
z6-De~wB~)~5`5jidGe}o4ln+w={N4nPp+3KXy5EmC85A|a?M%>>yYG46TWlx>{zE*
zeC^uuO-z@?+nr80d1>CTdGe(?{)FT7SVjfo>fCcLpU>a2I^b7$_dz~mlWw=y2ZQd^
z*VNBe?6!Y(E_DB&r`g&4Yk3>2EmsP>IeBtp@u8i<&*u~`dVkZwaKqlee-Ao%3eE4(
z2xwPeNIpDG|MV&?ZFUR(t%tqj9Zb4i*2}&-y=sw?vUt;tOULd$O;g`|=lim=-{#-B
ztaSh5zef_*-(THy-F#_NWBmMl&A;C^otiDHS6g5m!(RSqVN>z!3_a!-b1XIQu0C4R
zS<G-$v+wid#t7q&_r%V5Y+BU)?c&9!FJ15IN~I;${Cgz7{mBpa{~x1MQ`6LsCwO_C
zZB<|O<nE^G)|aQuf7Ga`b61x0yuN$;^W-M3Ze=r#F9o{irQCzL=YIBjeK2xsh8bI#
zrs`CMa3%fKT~2o4i{>sZ4hwZwmA=jwIm`K}k41UV^I6W9bQZ`7JDu|BJhqr`U0M~3
z_g1IpEE~%ht{Cq)6TQkMUgbH<gQjwoq}*jMR;2n}UbcR*`^J-xW2bAhJP*=;v-H>h
z&ENBVCuQH9sqihA;T_)&YnklB%1X+gUGAN4J-W&&Cv0=Y{CAn1*K*fhywP)Z#@S|{
zYnPf+mZjUi3h%j>qBc1$#Y}hanHwIqYZuP)HC=u(X>zE|)zZYdR?~Y<c&EhN-DoZ&
z_pYFvA)252Mq|gBq~J8$HIrTAr?s^lE@Zd)@G$!RuP}SA_s62czvuo@Ix3cs`#3-M
z$GZJdE1&2D-v9kEQ2i={h6M8rUH>kT^zP(o^Z7O0*7AxwI=H^Fu8Uf-l`D%^CH=I+
zm0s4>cM=b?HhL~QlBaWH`odGi)^Z#%YTo@SpQFQ?UHjN4JU+ap<~V!w45k<7k1T6n
zq5pYl+XvfElQ&=98MVn{rnST7*mRMOGZM>Mt0H($l-Ia8PS7tc+NQ!4@n(<5&gBa`
znk*(YADz3ZxYW5>Mq&FCuZB5Sy=^CI#~N-IIm!6x=A)XqdRJfGbXs}Y=KN=y=Zoz9
zezqzV)ZE!yo&RLbqrbZajx1O;Elzj(-LGA~S1<25{^#1&pD&#bzkK@qR7qIXwBK{5
zmQ^LOHhqvxW0bH=N?XMK_`pj}xA{TG?rwemrHDhy@?gdO`Onm1p8u-(R26Cag1h%!
z+DxVo&3a*~8v<VR$e4+6F#KS=q^c32Yr^1hlb`9}0!Nl+>(_}{>#iO4J^wzn>e<i2
zlD?hc-1pwQ+_et;UbD?^{#yHe_D?j=6s^3xf6uAE@&BHFpI+&|T6xAE^%}8|${_uz
zui0bW!Wo1DTJlz{+jVfYp4;j>x>d(MhPR&Mwm07?&6aHcV%;ZR0fR+%OCPVEm)JA)
zd8L1cO?=eTYbPeoj11e!wt>y+wZo|eFBV4q%wD|1|69ye(TVfq`J1jTYggcqk)L~Y
zmHMAEI?fwU^=zy?C_6uYn@!{|qaAD!v*ji&mG7A2sq^~cfeGDj@7}DslBA>OdWXB^
z@p10!Z~9MiAK7O8(5x;oiF?DgkK2vExaqH!iF&TjXTR6W{wP<+gYJD{@mfo#S)ISK
zs{8c8f7K3egWmoK$(TDcHJ$Zas>T+->E7?!4?F!R?YCT2x{T#>=p4N@Md<=3O7$o7
zg-wlr`g4c;i!8f$yL+<_-?Ex$v@U5z=(1gF<~|C)+SRFXG`5<NLFDVxp!tkDrGK9O
za;u}UxM`t)m-JfyP9{lvwok3sg4pB)Co5fivB1lyI&R&EUEPBHn|o@f=2tCpWB6U(
zp7VIhs@2m^mVEm2<@kBOXRkhcKi#2p{&P+59FMv$_paZs{v-D_wS0ct>x3Qh<=6jJ
z?7tuT+g7mWd${%+<J6-w_u07X#(r02U2S)A;%2FL_oo`Y;M~Y$+u<tG;d}nRG)tiU
zjW2%=i0_-=W4G34-t*ITUz-(f?6>AwaXWhU>b%OLRhN!to`3ms5m#8a9qZn$mT_Nh
z3TS_5jXc=f@|b(GQ`pnHp<D{*6;pZ5C6m9p*u73H-zrv=UcU8Q)%UPpl8d!XlGe!Y
zn^j*^e?n#2(oJPm;S6DFGhROZyZ-L#Pchj~1xv$Dd`{&zGM;*}+iV(Z!sq6tHTe=U
z?^)QR&aRTF+&5=-j!2Ki_7zuo93<CTX+K#VwJhw*0Uj~tHAbI4+iYJWs?O`I*d@2_
zT=D(*+$xSaSCcx8!teioboE{H+<Uw3AJuwwU$-hjE_>#+w47~>4R3wtJ-17rzjIIX
z(}tc!JLdB<m@8$!STODLy{fmrOV<_8JoHiU>-%$`RIg><ywCss*d470_kZ)(N#Fi4
zn?EsX>%QAJj@@5&K|1%}m-~NCKi9W+<nnxR{Cj-Or^9Q4f=zE`Y<E_*oTqC&{gURE
zhf(F4a{~g>c)ZuI*)Ug9pP_7yJ449|lN%HH8m#g)WZz{rA3Iii+-xIbR`#2+wMyK#
z6TgVJb27v^e?G9fdHUAR+Dn}jx0tykeJ_m3SUhD?keB3r6S4iWD|emh;c|T|B5?BH
z>Lu1fUMp3M(?0FVySHZbqkX$yyO}O6IbGA}&3duQz&rB1n3ceUJe5;dc3k`#^i8?{
z#=|v1Z?E>bSgk&DLNL$ej@r!WQkM7DXu0*3$F#BMnXTqx@Vf5y+*tYj@jI6U)M7Qa
zX8O%~pm&iWPUU3q^4UFRAJ-IT>M=$zcbrLB{Nzy0HF0^}GwCz;DSERPly9xSWu5Jx
zc;)D0cK>_ze|~fZd1Zh7@p1CAWkL-pGiIObTYT|g=7&x7?_|_(I?60Eyuf}yE_n{~
z*6HtmK3x4QI{B^Hk|}b56Pu*DKS?Hw$831%%vH8VH&}pkt?2Kz4UJm&SM(nb3X0`S
ztZupO&3WM7Hx>EMq0iQue^b5g%WZKZ)j50V>Li`=sIY&lzJD-m;bA>?|HrlbKgYIL
zm3rqKGTZjC@A$NXw~pOEcR@OLij`IE&&q9c5?lAM>(8GnQ~T<N-R-U*`zK=CQdb=>
z|LqleI%KnuHkXS1rD=DZ*t>l?3-8OaEPAQ6<AF=!!f*zw5WY)y9pjnnoa5K-IBv{e
z`*6p^{Kk&R%+QNQ^LaG*WM5wF;#Eof^7+Z6r5_tlFy6nCnmfza_u`I8VcJW)Lzmt7
zBEah86}eW!I6d(H#woQ0%iiP`wq6%MnbT_{a!IPs-u{$N-K^)Qr+XHa)|JiFVQ*BJ
zB*HVN!R=gDn$o#nv2&ku&z-;evXSfCB?=v@nU4Q%jC{H#h`(t!Pv?zqn|*k{T5AV~
zYNv+PZ4Gjmr#GFuwrrW!<oP#i9>vAGG;%cU4wRR!k3Y5VF?VJ8jcDn00#De^+fH2X
z!<z7<!tcCE)t{QF`BmA@)%|*3mhQ2RncW+yc-GgS!(1q>^<$m<mDg*gXKqj8Yunty
zP{q_b|NXja!NrUTbG+0xM!e-aP!wgh<Jg4b;cn}~Z+&lmSXaMr`khrfKe+m{tgaEh
z*mqxg>E`5fSNode8dy}CyGp}t6%SRf&gwifL+d=Njs1&BJu{9ctqo1z>h$>5fw}C)
z$yI$iRh0{GhUl#S6mx39hrj2nPJA}D-?97U!tKYBW^Ojy@BCgos+9BUsq&f-?Jq6|
z*xt?QeCD!H`sd!QY#EnN?^Jb2-<s5ObSY1uz>}Tpwdy_UL|A4TzIi)OQ_w>De!QV?
z>DABHt3#*0J~Z7Y>1X4eUwwIx#g(@)WJ}juE6>;cI9a~t)6bi`7WXSKNjhk5HPQ%P
zb7kYg3WtwUCqrW2o-Ukv_*Gf^?{fB_bLY<)C7;nc@9WyP{pD0|VRohT16xc!#^|oL
zdH-nBkp-vb8l1b^aiu8eg=je6CYF0&*VhOZ`%MsT%6qc<X^hot#-j@b<bJlEGfuvy
z_B|~rE$hWGvnZX2Kv7nw*;DQJY2Tdkx@h4n(WD*v)lM3FW$wQ%R{ei|(fXj$(5L^T
z_BK~cpWGZjXU<%iy%yHCm!dM)N)#8A8J(}aFP^hCV40<~+S#H|w|@n)9+A~yUyo>=
ztyZ2mEA*U2PMXr>`EysU@;M)NtmNA1HK$fz4c!&<jb)S8d(IS5Cw9jIbFNRm=l2`$
znErg1f1~~q@vkfIb2c?zHr`tO`)Ac<V}`J0n{N8_oz_Zvtj(Z&JbGqUs+L7%es0VA
zpN{4W@}}mc9b?&7P-9rMXI_EX-njI=UzqLBe^y9%tto$^nrWu|n{Vu!pI)u;m?!^!
z>E5Rsvb(u1dTgrr@mD{tkV%lC{7Rwgz2$#3IE)roA6j3;;y7<++(Vf#ZI{K&`^|%v
zScfMSE|-d38!`3huGL!Kn-x5!r`tY!{xfK`WVl3u(+2e;H*daN_Pji7^@95Ai+9yc
zm?-vJzd$&n|Bq6u_(JWJu$31kR?f?>3TJw36mIuSmCcgrCjTOb8!n2!S8Uu~?Y#9@
z+S18O^_xZ0Rjwb{G;dyh)xO7zw_i(}`h4g69}&!f0y~uU`}Am^_}=QiXmiuGw9C%v
zt)^$VJ|ElUm~VK^?B3TKRy*?_=SOESNayPL@tKEDcdTM*$l!U<?ab;BF3j+xg5g{r
zOMk(m!@V8GvUB=ouIgOJ#W1~-Q9(9$`$4z2u`#z?CaqfGv+?4(@b!)#H$AB^J9}L0
zL+Yx?w{>%0XU05^pE~`IsC>t&4*quYw`OeNISO)I2hY8~(;YR5BY$@0eLnvzhJ^*|
zKK%Od;ii1u`};og<ZBCOEID~JHL0SVQE8IPrn|PYHn%Xy9Vygknc}o^$rX1mRWHf4
zeN(*BLgyN`tlWDmed<h`aQ)@0d`y+K7?j<sCQ4Xd>P|8jiWBjc+&!`4)04v!m3gl`
zyl4_A?v;D$%8rS8mr~Mr3g$Y?Sl;`(BS7iZ*`6nR5>?oh+hyG+HYh!uS@P7)X;Sj$
zmOX5YoDC}Tvph8KcrZ*l|L|bN`_9F?cg(pTSyEV8zBPaPU-#<w#c3D#-iF_isrvNC
zLsPV?>g-I%iHibQ4#=-({BmB_y2eI|X~&`0LHnM2%ARBt*t*M6+fTO2MZiVHl0RFx
zsg&d0M;^_k(vF&@LYnu5Qn@5@*WRxFVshCy&t^jCs&_(XC)q8&@Wz<^{+3IP*R-mg
zvx1gg*WLT`+>@VoJ7k~y^iyVV$(;FWO_0GWtw<lW>C$VrX&%b`!*m2(m2CUKxYsUt
zzI@%US8t5hFbMX%e;YGrkHO2Ckx#Wv6MwDp`MNB8(@u@c;ei4bE<vYWzI<tP^-OHx
zM+Hkcb1@#5b*mShl4Ve3n<vlwB0zKQ)^*Q9-xUNOIx1vyUSUGaeAWZoPNznf`MJmG
ziF8Mu@iO*gi!tviQ-1Wb{fWJV{mC<%F3j@eW|%#1zx?sg!!KDJ-+0D8{Wtrk<F(D<
zJKuZS+P?nx<>u?uRka7JHQ0W?zuWu!Vbz8<{ui5%%B(J(_xkhX>C$^Q3u>MJe6n-D
zpcYqC>#=3*$N7KAe`VcWy_x?vm)+}|uhkm%&iNW$T{|)K1t-JK$K6?5*MxcdiGMm_
zG@pxwsqymZ=~4MrFM{izwD3-ST5RpK`OvcTQ&-MZe_R(G6BZ~kB`|>Ne)R?M<r1u`
zS5Mq57I@zOa_IR)xzqf+_8FYX|JuTwaM`#|K4kS$`5&|6^1@zpDsZ?6tod|@;l!VW
zt$VBrtb~pA+A~Y~&YqJ!-+wxBSN(I&>V3w$9=&o3+ZQdz?eOrot>P*fxwRI75l^Jk
zHk94@y<A(};T+qN*d8q#ZIQK~d_}(KPY+$cjpJZ}tk0^==eXwlJbT8-GB{AcL-P60
zNy5c3avKBE=h&u5Mm^@8T^x6QTm82`J5J3l_5XHlm-dp!i#-@0e&3MveU0z>XOU;S
zPHGjb+O;wHLf?C9yWbW+0tM#i{YkZ3n6Tkx<JV7j77MK2>VD|s%2S2gW_-W)ck0R3
z54#^$SWIBn+%_-bcE$B%a2G_&<U#(IPd~qe%&(g`_iVzHDUI{b&!0cXV&D2{#S`_9
zm%nGeuv~woeBb+TmzOe5(&(^V6wpz!)90Mu&Uf#P413lw6c{S(-p>5uQPINlHul>-
zoW6Kml`Y3A>gi7}d+u+_@n2;3n<W`&+ebD{ocx}>?|r)dmzzn4xO6wK^1W@jmt%si
znT(L!5yJ`V8TBL|vp=7#{#c{K-{1ex?!OuW4gpW?t}_@N+jZ#m(xUL6ZF`R@TUuE9
zOtpIIw|8!6fZM*vr>{3BEV7%eGdJmHR#DZiYgNl8&Gc%ipYTdvI@rh0U$*`od$U$N
z)6PiNE8@0_zf{VvJ=A*jj<?D8{QcM5W?>vP8#nU_hE_K>d~RU4#>U)r`RS(3Yz&N<
z3-)N3Y2WX4*&4;g!gfmBh$F?HrDti?p^us$T3>ReUpCfxRaCR_wV2W?S4T<C_ack`
zUgWECF8s*V(Dm+3_o4f>2gD*Leh6F9^8V|!wu@6cs&rl%X`kcywRV%m{MlE(u(<s_
z_))2NBf|!{yc=(29W>tcU&`8CoW8ug)NXa^*E4Bz&tJ3NtDW4{dykW;tdu>RL#!@e
z<jbFXE4(&c%{nI9dd)F;<()GjZ>(Sa(B_`?d&{b#`L}<1N`7bAp{sD8x##eur7sgh
zf7lhr&b{iDeR+Ap#HQyTJ9a4*)TXzt)++x|+we8Cavs0Nim*Furxt14Ut?m<lW~u0
zOXSZQe)D|p9MdL;jChNf{1ewAXBzJPD&-(vC$i>~OvKv8J0JfZV86D`{*&9^ed2fi
z%D+B;z--$`bD`*lt-bq0A2L@}RegE%A*1eJ<UD4PH<ss4r&N{JJ^N9Yzc=pNz1^4n
z{jbW@ZO}jd^3s%*Wi^%=yTtXkocmmp`fOjpjE<=9Dal(G=+<t%kbXz2;!b&Ks7O_4
zQiAUKY3c5^RaWo%JC8^uWO5&9eRrfS^tsp5%Ck|X{^d*+mp`6Uz8T4$J$bh1qJ275
z5gD;-Yr>8g>u!|4AN<OIBW3FKFQF1W!KI=n8E+fvOug-}T2?G}?!0*n8o^)Rq*QLL
znVIE2*KYSI{~IBfzjY;^47{&jUB}<RW4F(^th&_AgGuZBUj>1*srnlVYWAydVEgK_
z&#WVTZtD7|Z>uIM?_Kf!N7K%A=c*#tPTM!Z;hk&Y)}S~M=7h`;0iGG5o9=P>>Bg3)
zou2B?7`D{^pw|5jeO-Z9G~J3sPRE(gmHoYUqWn?qjW^`qF3~=_=u<{X!Ine+GSaus
zx_EE#)Ci9I%nptDVN0KGtYWHr@VjyU%b$T0xC2*ja{GI5g}uh7X5M9oKmV)Raq`QV
zkI#FyA6t2+{!x8$-PfnbB9@feMepBr;#l=W?V_7ro5L#Y%>B+sGIqGJuQ{^mUrOBt
z)5?Q-i)|-9x3m!o?K>nMD8RfyB75fdEp}5yDqKS6Y3D9-&{(=i;l$@XQ9tE=#;%#O
zOS><A?Vl`GeW!HU9s7hWR{M1Nu8)~jm(P3LJzcg!uu0*8*+l7htG#n9waoSzY0ozl
zTR(a4rW-dK%@f0Ru6riJGi_zs^fkWug@s*uKl}Xq_D9W}7x|P=IA@Xi9IZ*#u@86m
ziq%Qqt`JXNeel*X{kohjnQc{zeHUK&Jbk_fFGEhnk|{}rq7AY$ZT!ug)RT8IUs?KW
zp+JtwEXl=NFNPl6JAe0%l!ILQ)iY=GmhIlLW%j#(m0Fwpcx}|{zj*8te7!ng+178|
z3Q7+JHJ2{SDw;VX?J0{xC_};qXO*s7mt<X-91f=Ox{2S)SnH%X&!%!!$fTJbUW%za
z2@T&RVs93H(`KKrN_*)ff4iUauD$9${G=c&?AOite_o_4onJU_^;*?O9GO?d<rys`
z>P3$0G)@X!`*r8X_Ey!fJ=>MtXSFY!^6~J;xZ4XBdI;-x?e*C3j*Efcz0bVJM)2ze
zq41AKCO-bS@k5DOs)Xgdu6G?PzE_yo7#%*oSaFkafq80!;MBh-6fb`+DLeCc;)g{S
z9Zjq}d=(eT<+pxcVwcr^Qqe<mkz2Azw==_MYi@=$$4`CHxokYosPfm@{`dU<3;+L`
z^XKut&v)hjKKSwB^m+ThpkqPD!tSNiN=7ta?epL>T5J>?zsX5j({&*)LqChoF(pTp
zsX+@f);b2H^|-Svc2=}eV)z>wwd8}+zS6Ai{GnH?gI?U~QC=NVxK{9rfmTUM_PnUi
z>sB9nweHlGZ#@~)4!P})ivNGg$?#sU>&z?#-7TE03Bf`P2Cu$;x%%hy{xxT|uFC^Q
zG^h&dmX{T>Ny%>W@3(utoKt(oP4$z<LPDL?+a9rRoGkg+?TOnF{w1fMPYvR_@T8(=
zDbLX-Zx}9w9`snzecH5)al=vp%XtjS>Jv6EUfm@6(Y9)r@<p2!Th^(~aZA}*IMFHl
zXk1L#CB6npqn8=iM2sy?_cUHKh-v4q>v?O}H`R+lmh0dYp$6?eX)Z;r*Uvuu6#go;
zsG_8#qUOibj}ND7|9!LD{@1~^#TkF+FtnINy-d%3bL^2p*LR7fu1*YErfdOTu0hMZ
zBpHl_898KI_#(FFH3~+}SjlL3{akdJ#-e$i;aa;6f8ITDMkuSZ-oI(bZ~AsE6n(57
z)}g#7Y{BNcA`Eh$+;{!|B9-6z+f+d~PGw!AY_om$mJWskijtfYB^^#_+_NsQVt7_A
zHgn$AM6uuAd*#0Cp7~X;uT@Yk`}MBB-A}m&_x{5Zl^acrMWY-(aIFt6O<jN8vF3nQ
zwI=I=sYY_0O1@5tKgCk7>!}@^;<cnl^7<y%nOl!O5^sx4-Q*>jd1S|(?to<vZclx0
z8+y@KU;Ch|gn?CwibD+l>CY1$zs#B>>3k%EbC2u3rPd*fzMb;=`7X8HlC4Ac=Dt8L
z$JL8(+FYF#-?gDPchBsNtP!_=9PMvA8059`<dYy(9ic$4mD}VWhxlH4(j+c<TJ{+G
zn>#0rQ>JPKuU*~r`sJ=2CN~~t?dvU*v@Bfn#kg)!#IG!Kv0bx1STy`SzA5tISDQ<X
zcQ2$Sl_jrBypmDTx~z}i!Ixv<M$xC5QPN636Lf9{izt|>eqYk@B%$H^kH%PytQXo#
zr&!wfsGXnu_^^Iu>suv@J1t=hRbNUf?(t{3D4p|YW9r*>!OKWqc`3t-kHV5&j~sQ%
z<ZM4KvY1w#vBx2})HGYxd-lXf^=h#nCxGg0Lz869d5jEq>$}>zS7k0|n_dtR*Y!BV
zEdTs@V-<xh48GPu!81x&8_s&WpL=)WqfOdforfte4hS+ht?gUQ8(dW${yW#K`+d8d
zEpO%#18xT8wetIB{r_zLYkT~khx;q0mfZDdacVnc>DTvGZ|#RthMI>AI`2XzZu#xW
zz?Hjc)lS|8Q-kELK3naz@=VAb8P29Ikw0sXn>IJuEZSr!`_#)>DRBeyg6`A3Wk2UG
z^vM=Ftz5^bu+}%YI%0S4huEwQm*-y<TgD*aVfg3K)uNqj3~m4NTxQO^#pQCW?~iar
z?T;*@op&Zh+^$#-ZVYzI&pnnHqkHA$>BDmS8Uh7Ok39Tx{PUBde}60+ddn79tpB)V
z-DLfYd14GaWi5UT5?wQcHpz&7w)$g|dPc=yW5Q{+-Fu7E1TM6O*)Z>aot=9s#JpP|
z>Wc(JQQ0eZk?A`=KNjAzph($6bGq`uzPDx1XSXPDx$;&oF%s5X8mD%$`|x?4O?z6`
z<=lyV+itRJ_pSGjjCLAZ6j&9m;gexJ5x_Jd$ZFeRp4%1-FKqt=OwtGz<~nMylu`RS
z=ROZU2}{FiObLZkyWT~-<1*hEVq#@6>+Ir-jKY39I4AX;W-$NUkioGwdd2IH85Pp8
zhL1(x8OtB*JQP`Kxtc3kFS~E8<Mtq~_RDcWay{S0o^57ce)z>IcmK5;f8V|O&&&RC
zXS{s<hmU_BzAltMaQu6_{C)X<_H`c~oqqqn`^<`Crglj^tO{+<7O{LQ%y`QC^HG*V
zwaOH)B_cs~Yyla2B_uUdXEyzHbyiH+qLTPmYD&<bxPzXr0yy-WE*(>@lIiA~dgjn3
z^Y&E%W<OtCm{>FYQ;^L&(@uManL8hTTemfF*{eqFA72~9uiP>?rWL&0r!;L*z>;pk
zt35gli~Zu|@;uk%?zs%kebIu_YoD<_O)8qIGli|i<#W*)UB_qV=WkZq>%26yOQ17A
zpqSz9I*UI)Bu~r6in3Yf-kti^{18Xx;@CLd?`IOvhD~<3n8y;`9<luhBg0uKeYtSa
zLbp#whBgyDTKsYaPxhZ&lcdRG+mWo99QNEp(@8P+(%H#Yq2V@9dLBN#cya&V?)m$E
z{QGV(`KZ?F?VG|Bd!Lny)qWJbd)MXTz2EDYHJA1TJNFpO*rddCacg;q&i7xNU9yXm
z=SpT?oZ1>Zr@6bO-R;kjB_en0XEn^&8gy~)@uWI#Eklngoo5ccEd|1UvO7-CFIB&B
z^WLiCtC}67%h(zW<>wzeQ#7UNO(N5T(sFT|g7+yZzaK4TUVizgs&f#-{`!xq41VHu
zkzwUewmVDRoc!nG--jO`PPhMgSjG6)hYyh&bDU;BR`E_hUM}~3;Yan|b}W`Bdbn5_
zPVsQ%+6c|jTeE(80mFu;qMt*(r2AX0SNy#l8d<HS{jynM-`O|1y0W|(XKSZ<L`<D}
zZOta7mm5@XZxxlfoVoVitSK{py-RgE-Miv#iD;LJvCo>qAZykSRli;>pD)XFqbe!)
zv23sIfm_G!PupR2X}LcCYkuBcT&*X1?5u+ndw1<H5!t+LZ{1q;N7F1?ul-37m=UAa
z=DE`7s`T4U=gNEU*3?|NH^+DD6)&B+28IXvCF_pPU)5P!nQ+c0yZP9&Df6}GG`)N@
z>8qa0qTE*!R+qjtR0|a6`7z#7o+-(oHHn!mlz9*PhJ{OmPSt!mxmmvc(_Q<Y7f<Sb
z{k67VOi9K<w!cpyRYSm5Ea=!mjWW5;01?x5i(W6?<FVnz@!l3`PQP^v6<1%-m=hFy
z<KfmHD<}FJ`hM+wF+C{wLE4oG8h4IM_gsm$uTcHB<NvhxI-ROEv!}~g#?^1*UJ>zb
zqeOS-t^g4)NzKO|9bG;e0>`@&uSxAbIoGj&p3$^N#k((^KX<iUa&fT%i+%Y0&{a$m
zW_q34altAy{O1<#gof`nD=ocGI7@b)Ij}oZPWp^Q=2ahw)tj=82QKb5;c5-=irgs|
z9CqbV#?6Pa3jIqKN*r+8%`<Pt@kg$2#3BoiN^m#${@+-!F>&4t-+-g#ImUv!rnN53
z3rRhu5vs|Xv32LJkXIR}6JT+pUz5QwHE{E%&#O}wo}IE>O!J59&(nvyr`Dxun2B)D
zE!e8S&nOb5z0@fAhv8wJD<3W@r%k=EqI~xbk@KtM_s#nLV&!)Yb_Ln9NtFz{w0<<+
zI;v_P7;(Dw*2%dSW*jxHy?U=WbfW0F;=*}L*E6JCH4RFUYh9Ym;Kb<PrM+icf4cvx
zGa5^IUf;a9WW_(OcU%m2?=mz@QEpc{xbK4t%k(lQ#_44W98ETpzV7$vnG@U2*R!+a
z#er)KQIS^1O?OGjD%|l;KW;i9E2vH;{F&Rob#nP<Ka2f)nftK+{lnY>t@(QYUa>B|
z$XJ}iAhvi>BFp-%jx#@6n4j9|q^FxQ<7nP{pVY4}UfSQPme=KwzI0+;%klV{|J5sg
zvxL<Aj-0hN;E0#b>f?{rR;VzZ4f^=hV}rTXbm1uF(z4@QqqG^~rievdKfCj_W`foh
z4@Z@IIo=Em7jBkb&voKfHx1sh$|Kx$rjeklS?rB~V5zk^)zX%!$5Paovh)Iie3NEc
zNmtuGH{14c-4W2ty12y_*~+>_YL|H!f|jihlF(gRGgHM${ZEDR@$N^q3=S#b2e0v@
zu8PcKP3XP5@lIkG=K;6fKYw&YKQgeIxS4@>Mb?fLRu6@aR-2#NBy&D&d9m8ksWNMt
zjVJBi$dK_okatx?=wan;++9JQX}8>C%@a1hT7CMkmu6{6$qY5|+-0xA+{OE>Ki!V+
zJ|4K|_$wubAghnco-Qh#spdS(RXFv#oEs-@wk|4Du1q+y)H9q<cEQ2WX*;s=7afb(
zoSc##w)9*A_m18z_mn38dhP00&G6|ZXG6`dgr1{^Wh-s($=$mV5FC<Gsl%Y0Za%I0
zj$iv!f0;REuCG5ItmvQmeogG=9I?}18NR$&Da-!BW+zL)5{*kI!bF%7RK*yMMPzvB
z9E*_S+?F#T$mCwstnd>F-I?mEw?49-6RlM#sNkV_luK7X$*_8^#*^>tv7$WhEa!Wu
z+zUzC*xpfMVm%`x_YTK2xyzvwB@dtczH-(Awsb*;c@kYFBAOFglvod>$OYAWtK76|
zY38r>wP&|)>|O7Y!+6cC&d_Op?wl+kkC{b=p4b1o&CZOqS<yQ+G_}f1`uxw858Tek
zERhow5RK9f-FTc|3e>3k)3Ra7l!~zMyZrLDuL3gDpMC5tYxS#0GppT@`Zl9LwEk*|
z;+}P4cQi7)g?~n-3O8L#=V)-hVv}?;cT?WX=bmTzlNr2JXI`A{-hRA2KCP;ZJ0$W^
zSNGk0V*S=Zj#dj9Hircrb1@B8^<*%bR^79r<%X=~Jb~R}t{*MRe?87oh<fp=>WJ3@
ziQ_5h<}QIfi}DVw*<Tm2;>qi%{ri=FGZ#+q7ca92<YvedTD)*m#R*%+^@|S{tHlcP
zUZ}|Pyt#4bq1ZrW70shJB%@zOrwJU_>iWK@qTg|w>bi(^*X8&#P3r&I7`OeL9bj?3
z-^fm}+j?qfl5F|1vg<~R8AYNZYiEStSO2qiW$_)~$oPp)lQe=da%Zro2C!av+G11u
z!Xq@W-D};|4OiE%%4+%Cx96s_@L8TKXVyy_S~Dme&yLgWknztxy=Pg2TG9qrhdDDh
z**~@nme{&;*Bm+7+%1{Er{*L}zjw)*duRK{Zoz-jkEKp9rlifC|L?=q!dK7s=KK`0
zv|1gS)-%O#;aTz3^Lu=Dh|3t1n(VV?3&?o<Lp$N9Z;_1si?!EtF3KHuW|%3Y{rAl0
z$2B!7)81Fso!R@G<=DFX6CaC$D}PS;m6)e>-CQZ&>PhtTpjgp5U1zn;t730eu2|g^
zE%Rxj%IQ?`>3c3d{C8LB=&dKV?%y^<F*Cp1AXui+bWrN0ki)x)G0xwQ#QLw}a!oCW
zeERQKP2oHlUw{92d0&P<AFlrKyRug}wp@iLQ>D3Iw*KFRn>~+Y)33f~>u|X+z3+qS
z#P{qh550wK85PTR&)_l?2;^TK<YLS9C-i)0%k$rNuZW2JsA>3fQM^xI#=5D%KKA7<
zh6xKp^CSK<zv{PPdG#><zn#SQ?x$=F{I>&ntxg}kBLAIz?^C7wJ03Uv@iTq#!98w5
z`_WCugulu$z4Y4N&$q8Vlyial9EF0<d!m1ZKh%mzU_9`HgXv=QQ|*Fs*DowPww}BC
zsi-%0Un0W>>3tvgPsKzouJN}x{P%d^gOp{f!e-1qJ+JxP)P(1?>L<!qSxcudG&wvu
zE%RZu&H7Jw7B5gw@n>NW`?cjm=KB49)0dt4TwOl*^is}NjjDv^uj@FfB1HGS;jGX7
zq`)C_|NWZgGdk5uACzX8oWF8v-riDu4wre-@oNn3@UIfBig;@IeQrzS&&Gtw+e{lA
z6&RlE^jDdt+2JzpIYW*8hrh2s&hs!9{_4W;!*(K5rT?016ZY+oI()uZn8h*e+En#+
zX2S`L6Y^Q#->zs+0@Z!u`X#?jx37MF`DI#h>CP86-p_2W#TH5_8N1o6<~h1ctHtlm
z3Z_@Hbaq=zpU8Y8WBOZBhLXaQcMfM*v3$5I|M!4RY1J1Kd3nh<A8u7_^@*44(v82m
zA=qW@qcXLP7CO6<mj5s<D-C4`$yhrzDOl-S7C$Sa=IOjm$8=3aI9Mx#HY+o)zE%I@
z&4;58fBM{g(f$2iN#nF0Td{WjJJx+2%8Ds6_8sngCu2V{dQJ*DR^&fPASwKmJA;f>
zf?j+3)&N-s;gDXBJJWt#Reoq!U)sEQ*VOmBEkB<9b2r|$=5^3^p?cx137%rc5q~Cb
zZa<Sy_-ezw-Zy8s7QRU|TiVJHkg;Q;1ZPgh;iat$HL{W;Du468Iatx}a_8Hx=TANz
zocw*}&7|}-rx(9>FB0Hlche3nEHaRiHRwDlrSnY9*|9Z@fiWawvZNmeSN{!*$DH@~
zdb#WF4LP-D>a|rDx>7@@@$$-^%zK~l$7R;xkc^OqIb~-YTn_Ep{&?04C5CG<t2vnh
zmgT;k^>FSM;|djHVMfor>PoBv=N8284dA}4v~k<U-g4z_JlD+by}WQIZo|>2pFS~#
zYeF_1`FT-%^3R{?uPVy)SADWsUsNF?a%;{iPkzzQJ|CK`|H>X059P}%3eVW=_UOl@
zHAS(ag%g$@``lVQQ$oC3sCjDLwyON#d1~d0-u2hb*jt`{Kl)T*{n2b^<?}+PzsKC?
z(U_;fq7*+X!YbSR)AhBJO1-mx9@_XPEN_*s-JM=m^Xj<8d26DV&GM*n$zIZG=C~nC
z)^11rhl0J)L7PI1m-^Lznmqr^6&{9aLxaLn^;c<s`5NZkkLEXLXu0!}O@dX)bc=TP
znF^JapR|vKGAy}s`m^u6{0=tlkbf`c?T=Hq^Zhz^eF^_-%R{wa+9J+pTfJJVGiC0N
z&4)JyWOS@Z+WGPL>zb&o%S|q4hdlXeeYEKC<;pv+%lFi=AK2hzdP6p>GJC~tw!aHD
zi@DU@G_ACMac0udzBl@xeC=Q8d}?EO&HMLjQ-AkM^EJ=+-T(Uc-Kysap;mUgX3V-8
zC#kziw8nn^(?02+#*e=r(c*9s=<wygQvT@<14mQM75N<%uYMFSTXQN#GI-6#+uyx!
z8b9B>HR6oyE1MnV0n3aPc$t12)QGILI#7Kkmhr(vE%%4(FQwiSwww@N8TaGC?FAQ>
zeW<?EWN=>a-RW~#SA~-%e(REB&@uV+XN`8s)<aL73eO+Zy3Ww_FIJ*<*N4?M>=%xv
zb{^cN&Kzid?B<tG0c#X8E}x#Rd50tIDSy+{RT=B9C<rt=Fjda`oyU@|c8Viy>Uzh0
z`a60TM?TfQe7e|pLf_-=lFUzcT9jg)^)BD}_;((MO#N0fH4dM~2YbIS5%0*&K9sx1
zS<z?plWOVP72!(8;_2P~bw6J5Gu)|vzxe&UKQFhx|9frr;qdkL%m#aZ#mHXwoWIE8
z`yOl8LcRx2H*5dez`Y>;jQG8Rkg{8|XOzDCcHKU*!kOXM=daDnn`iF6IDdV+{F>+S
zC6ylXWnZ7au1SCB|LZXSwRt;uWxqVnzQ@uKcys1XX}Oi=7i;G1l>WVT?xEASd_Ny~
z=e<T=L5$tC(5~3rCdqvJ?~Lc4ex`iC$k%ZCxv<@(>F3wo+C3%h<T{RPha2qQ{k+><
zv-f%V-|u-cb`0lN%i2A4%$<@IC-L#atg90Ic5~ut@5xmf=hc>GPc*;&;eG0pwayG>
zPtNWz{U2W&@OJaT?bB>D&T=n~U0-YYuj@p7zTPL(S)chI+?%{r{mGfTdwz5qGswSr
z$1g8;qWRLgc-h@Gif=fMKmF`keRg+$S%u)ftCcKkS7$C?@RRe#?u=_Yt#8fw<dLJb
zukwf568_z9qeW__m%ZyVTmE+U^!Mk#>Hgi^R+IdB<qggE50yAvxdPIbEi7P6QQMR&
z?78p%?ET;QzscD%Zg{#8))bEA`CDoAGu^%Z9{0b5t<~$xD?dFGaQJIjH*fA+i8-Gp
zr5Z9ns9Nm0u4*3lgQwrc%lF-UTzFl2Yx?5|(bV(BZv_tg><{Cw+nRs;ea&Vwg^crM
z*}G!>tGDi%7d<<pKD)eU`IhU)s-wk><=ozXy>wl>(jZP&d@pl;X^hT<PZvL~oA*D1
z^})gQ-+orSU6$`EfAN&@UMGeJU++jUy-@8B`&cY@i}mgCXyyNP?f=j5{uMRe;(o8^
zTX_Dt`qww!y34IRQ}g*&{(KF)cJ3v$E;Wi`=db^Dzx(aPTI+?ifpK;92gSSf-6PNZ
z>Zs^us3?BRZDPH;kpDja;=g@!FU{k7a4r1X^AFE{ti69|ZS<FGd$ZK<*{wEzpS9<s
zHp2sXzEAJV%8uMs>zp}d<>5``Z836!9{=7wUcdKy_WsM0KZ3duv7Mmq_HOaw=F3Ox
z{~rzqjpy7qKKi@dU72C|X3733zh-*-`=@W-`|8uC+5f)WeDvpob?VofAHCTR^c{3b
zIR3!<>q>K1!*$nx?)x+A|AtfX7B+Kss8>x2`eQn`=9F<oFoQ+!^zz9K?F@Ba)~8wj
zk8D5AD3ZM8I%mVR@{Rr$mv>crz242R!Jpy5oEM&44Ug8J-Tz~YMDw-sO1ZnXPtSg=
zdLzYfX7lBT?D4k^XY5e7ero$ak+tEw<N-N12AlVJfpg*#*d8qZ|H=4X=skai$^WYk
zTh}D3mn+TrveDm~tt@n2xp`mowEK&%F)=j!+4x$1D#N3PaYp+$$bUZmZtrQshRJtV
z)`xsLKKbs;^HN^V|9!go`N2zNhE-LnbAN9)+fwswPPm*+?UCYpQlGZJeY?-{k^Iz;
z1?T%d%$od@_5GgM=z@J2@~3A{JiF{$&IgO*N00jdIP+P+!K{&e?puqW-09^3F>#fu
zda-Y&?O<7O&br3&{+%Nq7H9m6tdi1K|Nr&b_J7;=e_YFcc)I>G;qOl#B>b1I-hc8o
zQ$gcR^KW&ZbKKwhKLMH96~A`|pIl$_$&8)GkENAo^jx_*=lxH1{&w*htdlQRz0ZrR
zowmO~jmhG2&C5zTK5IKch6R)T_iCPVkKb+lyF}t%t=fx~7r*A7{BHc}X2WZ1$pgI;
ze`J5<Unm*L&#ZO~luF(SOpfSTU2$TsDQGVH;^h3g&(Zr8pFZa6)IE0Vn6)5iDzjYw
z8JqfM{rZw$^}n9o{eSoFMURC18#~@^$=-RM-~Q*D&o29`7M>OPwQK!m_bpWm`ImdP
zdzl5?=skLF@o^oA^F4Z7PWc>R<66_VDfQLch?nWx(&PK*RjIJ5ZGU#BUPqm)HB9cL
zY-R4LfMvgo40mr#$v>8;^=#rqzW4U!=UF><D8HV+d{sckR_W7o-6xg)%9!<dlcZ3?
zUG<Hn3(w3Ccd!2_K3^@EL9zGW6L<ChJKpb+`+ob`H07JJmMw1GXL)NpgrfOPbh!(k
zf0X;($^7B?>zZ3zH>pqSDt>ZV_}>hF|38KD3lf(;=i@hAbHI6lZBYGfiL477F4-^#
z=rc{Q^^sm{8m#fkc*5O3-q+?O9j}~do+p}hdgI%5=c3bQ|J-u=c%J{EuhTXeZ}WI6
z*0pliqkF9kzl@EI@3cs}T)6()dFf7Dv%ck9yIEG>W&HBuWuDHZCy)H=N<LZN-}C)T
z^nSsvl20~;${!!zuRV3^oAw^PDQp5K*Kany_W7_p;}prWx+QNGrY_s2c=eV|oy_L7
zD^%QHa^2|;&SsFC7_`b~<7bbyzBRK&t&CQO)T_l!Yjb_~re00<LoIiL`n08CtC~t4
zOtP`I`QqWlZo_*0U&M7@u3XRKN9H8WZ|k4?bH1AOBK9M^TZGKdhV3xj`e*C^5AW@3
z|2OM)ZtX2s-4@z%yW)G2#+DEJ-mI&Su83!ni0f`zVY_I$AVZ^OgV`>vpgmgpyLYG@
zHx}7Gi{FibD>rYUwn<sH%$=>`zVRt{T*BX4R?exmeDATKkdtYGnz*c1c7K3-`I1d9
zx}LubxH++FtyHJW-bwD0*6!rWl`tu=6`$axDH$tzO5~J`tenxVUr{#azVuuaZFpEI
zf1vTQrNs9PwZ*so&%N^^@P)wQ)q$s9nn=l-tXOp4>hP2P-~WEhn{CnXOmVTcdEuXr
zwhecVB+fA@=UBkzn)oh2BjcSwvhu%a*Dcq4xl?<@m_a2bsbuO*DJNcSH-3S2u~$v^
zu^jl((=PL8s^)9WJr8^XT+be_GM{z#*?NunWk0pKSABZUdF!HxP1NeD+rqbcvsQm8
zd+fy5{>p6PwazJb?_Io;(DtbRJ-@N-?@u4N8S+%NnoVzazjjQvqv6D_zD;*CG!|^x
zdU9!+8DH#uUAOdQlb#f8O?W$X>LQ<ot2v#u;~LzJlGkjkUO2U6%c@swPcAv0)IIjo
zGii$ZrVCrWHOg;K>6_M<mRY0G^0hJB@5o8rtFzuFM=WbOmU6m#-EI5Nvo`kT);tBz
zTD&`YN1#D})t~h<(w9yuGQ1fR%cS|`tMTFsR;%Y3o==&5wZg?ADr2tX*?n*GTpSLy
zac8_bzw>s&Z_SKerJtNfj`hT}++X-9aL?&!ORiNqTJ7>N^_Mw4cTHyJiG%BZFbT)?
zuj>ln&{W-B-T#ztGH;O8n`^OG^!gHBw`lL-+rns2S@-4TX8rmfS#{-G&l|k{y5oEt
z&u8my=H<+qjVqK6-;KS~;lv&%V!TTz@V%qV!)2AbN)108|65bV6kuno`S_R&(-Sqm
z1|vU-{HcXU`F|uXZJ(Ma95v17)EQm_lX8tU(JL8tJX`7%*v~Qj-kgk^nr+z&l_e`~
zu2{Z(LYRhzh-vpR>jS^<FhyU#_^B$Q+1BImUunN3InMd9T2B;=jYPO6%yiWGzDMkg
zlIOOm^5!=-`YoybC}@6_ae|uo_t^X&F9aKYetzyXlReo}MeY1)CrJ}IUIwmIv2ra2
z_K%JWebW>)Q@;nw#l1e_nRe_0|H2DVFS(j_w5nFu?+`rkQEgS8;8|a$QzE=kt9A><
zsJU6)N_cK$xNBC7+E?40nXCAwUb@I6{M#zg{YLqf-WB${_c>WFI%ikRoXMc<&JcBK
zi_ghklUaRT>-ppVe%X9@?Z)3`_om)pT9dn{+E7I5z<ax`M+<-TMt=FS)2g;|hew>~
zts8S@T`%!^7jQC`iSdn%kWI6D$QAwuE{B;`If+MWXYJmRa(#78&5;+UE``f#g}T0d
zqEH-O7ClEeG3Ke8==y27br-7sY`OntX2R`S8F^V*uDNfH9kOC;Z8}sH(l7H<p^Tv-
z=*wl<cD^Y=_kZ4dzNdQmUTL!zioJ3>$~P@boi}BpwBSjpSjG*t%pcwQ?cKuflpT}5
zzj=yigV8!Em!*uGE_AKuo*cTv(|TdrwhJn6KBUPB==PLy7v0RyKg$!d{o^^FS7p6x
zQ{O%_YBp;PJI1+qHCyRI-px{zzvpFpX}jLYS{__|?KI==+n*gpS5CYZxi)jlofew{
zmWF`Utv`7VG+usr@v_f(zU9yMy*9jeDJpd;+lr%+Ouh1<|L=6Ri1+d>nX^$-Dt~*~
z=jFR*PI6ZgS-G{O{UUGkQpOggt#8l#-kkls{E9yR_1|}nE?w8<wEAwu21}Xm7er=E
zSy`t1VK?LYRX(QMo1XPq<%P^n({;Np{{F|0p4OOMS~^C0!K=1BoY&u59ky!9rS)-3
z-iEw3+xAhkZ!KH4^!uQk8&Vfv*q@&F_-A?D%lco{i$CYhbi4LSKtcbKYnT4~zyH4c
z)Ly$KNKMGfRH{Qs@zSwu!Btl4%-H?h?JBtzTsoz=FobnU-?D&5p(|I-uk!gR6F%GL
z{JVAE{HxP#w){Le!7Xr)cvfjlZrk&}Kj*SGw5c9ietzGdTbI8pE5=RDnlekH?|)H|
zoz45>-w$>1{7BSP?N3p0UdwPpwm9fx^J#znIesG73k)VHHfZ0~^f@G7dx!Ul%gWv<
zvJCbWWfs3bi?2Wa_xmz#Ii?kD{*V4uiXLT<5o{Nl*TKk>ak!gV-sREhpBh5Gr^OdF
zUJG|&F1!BZ<L&Fs8;_os;B(twX8QZySCJU$#s5u{75LVj`S@0I^Lt*m{bkv|W_Rzg
z{PZp3{iM79_s3~`jXCB1?My~3Yk~>?rGux9Oc@sCG+cZcX~g#=Wo22>oOf}S)0?>t
zyqo&@snyaewV_unpPo{w;XeBJ^M0MPcdy#rz3brhck<87_ixPa|5uolxN4Kn!-wln
zsr%QJe){9Fl9NHxx_85Q4bz9!hlM{jn9cm?6v*Ha%BEndCe(EO@fBHy?lToHC$2i-
z;*`QQ)l)_=+$H(=%&^NI9g8)OJ`!lKdwb@+&JVTc@~yH~N_k=?ePZ>Ne;;Lkv@rA+
zd$##kv{9d!XsmsnjBcIJ`8Ii1hG+Aa?(Z>VSaIs!tC>z3uLbYzy>j84^3RL)Kjznb
zIJmw%>x@_Z+3u~qxqr?c2erTFcJ_wd|G)pA{B$qhuPOh|c4RL9=e=3G?E7B5hRsQ~
zay1`*Ew9%+6)xQK<X5+R{Fi_G;{TnPd1v+$b!$8RuSc)yTfg&PHR&6h&C}Ue&-Kr_
z<#Xa3^MmQK+imR`Zb{!*U*Tn1^Qre}xBPc`tIy8=zD<|?e*ABH;r~DPRGwE>Jo|R*
zjar4^?kGO%kF&3y+dpqlLd6m5%=XJqB>KM{H_o@xdedZYxrJ%&>)vhc-xAheH`Ze4
zNuPI5l6lAbj^{sbD%bz7tb11W{=<~;sXO0u9Ip-#)qg9{AX~Yy?&F5vACCTNEqveo
zNtD0x)Zur|*6Q_Jv?Kf2r+41zy{aH<udgk)vujydhWPe_!9Q-BzZZymzV_&C?JcwG
zH2LIaJf8mO!Nk{hZml-&f6-`J7b<Bdx<_HYfZV4}t?SQgF5Pmzr@hau?1tUX?eES+
ze^<ZHStI^;)8j`M|DBAvUd(q=?(5p>2WNe6+}>Tgef#^LWjAhrx;Me^nN{SSjpk*h
zHIL5l-;a4>f1~n;`%dZa=PTuY7w_1ocUb-0p9|-Xm7BihWw81GY_IjXFCQlVdA;tz
z+RwT7?Tr5f|4rNa*?p(<eeV4~%j+M{{(sY+Q|>6kS>NcUoxQm=Tfy7cuJWIKzw-3+
zSM&eP|Ifc)u5R!DhoAKK7Bdv;yw{UAa1E@j68iUPef-SFN8^{w-~aEjquDO5bmzJI
zR(ZY9U#(%Mv3l}OwgdirabN%bP-a-a>eIf*(_R-dtoq#;Q-833+K&2pI`0@B2+G}3
zXMZfm@J;*oN9pR-^6_6@ygTM8eBt*=&u_hZuWo#OW?zNruLt`ktCh*g9s65+Py2Vs
z@2Skc4i@g6eE&xD+zRXOvgPUJ7CYxKYg~VRGq3$pMf&$!w@QjE>!;SF$xq0T3oko<
z%RiOPpP_=c`NhA`Us(cMb%flOhp+FF4*nPW@YLKO-<X}Buh0CMes-olTiyRQ<E$W^
zWc`X_zj^;Q@B90|{PNOSOCBnmT369AyZx}T=f<8*+Vduf=qJ5TtZp;elXQw}^X@(8
zR(}!g_tAT9z5DQU>F<HdpY*@=-S1_WeSfl<#QHy{824)Er^oMEwD;iOAIkf@@4l1W
zS1|eX&c0LMcZZ#<OMY5&_>3l(o}GQ!-1c{R_g`)AudRJo_WsX<i2tV;r@k((X8*L^
z-s0NZ-}BqQ?b{k$_p~8Sbv`4<qkR3z?;kyTzo;uo-S^#b`Mq9t%zNc6ie}b)PnB@n
z|B%_^zmEL8@0G8QuPY7+Q@?j^Z_=UdwL9&e3#R?PSO4kZ_J5!MhyEy^w<pwidv9({
zDLB^89r>v4e*WL=|DUG+`&X|u&#q?QXZ`rS|K2?Qy?^`t|6$?hWdp6vM59(ddGXo*
z-+txe!CA>?0zbtpUFPR+@#|M^dC<ka#fy($5A*kuWSDs2_*6~%xy76KLCxJYE4oiJ
zFF!o15H#IZxMq^eu}2d&SI<`qp2y+gkjl1jLc^@kswV+y0$E2_I`2Lime)Fo<@u}K
z<xjT1uTJ{2!ouzhL(8OKl`lU!Sqe%jG?p5#K67HC%Y?OCYd@Xb>_5Nu<YUENi>byo
z?=Ro&RZe0s>f@el5_y!jVXk}s{knDcr=8d=`%5irqw;TWS@tPDSDF~#^}YFAVtsi3
z=O(V{5vBJeyPez_d{$N_1-~m#K6QRpp8dX3?d{Jt9^QWL40~y|lDyyV1v@$w=L=ic
zDd#<tw5;Y>a6w$|t>`Z%2K687O4uFNo!zucEd1*B7SA7t7(M)qcI|#xvMs1{%DKtv
z`}58$otkp=W4qF;&|eXmk6jme^onyjh-DtK6P&}lW($*mechU=Y#X%8mRAM2x@j7h
z$;f#M9od-F88<U)DbE5K-MDGb4}aeGdu!sXm@h}&zdiFmGdV)W>6FSuhDQ;pZDw9K
z*7vREJ<so<={kAq@}s}IJFA%z{zX;=9CZ?ys1y9uE$N9z*`^6b+tLLWEVi$dywCmr
z;QwFsh5v8=|GZZ5n%TZb8{fL_=q*?K6eIiJ{(tF@vl$<rgbO`a-<)%c*LwHe_*H+J
z6(T|{wHBxC{CC>FzM^K<y6TWGV$;2gmtB$K__p*hyXLgx+>gE1)a-DSdvn_EWXaAq
zlRc&%`}j3<{q$#*vtrh?&(^qidfH#1X6?ySC8kJdJf6PuQCXdBRn&y{9X64AQ$LD(
zYdo&5o0EL=_Vmhi=e4<9_PNysiWbXOG<r<#?v$)<d@*^((;X|g)Fonc^wiScc&(YZ
z+tDt#bI<kf?|!TgIknr$IO6Av;0Herrnk4Jx+tq_EKV?Qa4+Khuz$Xs+`lL4*I#fk
zluN|$n{PXr(7M!llm6pV>)DFVe`LQCec`+GNi%cdoGR8|k4ra}?6jS4|L*)GSJM|I
zysI43brYIY`}wO*XKgtB_D{vZx(hR{uN42;`JvLHBYgAKUQOHM><5%X-!VtHEZY4|
zd}74fKa*V2OKq2OG@aw}mV9FQV|7&wn@x1JptZlPUbD?{hpwAd``c^u{+JeI`#(Dr
z&%OWokIIB=O7Yx}60bigW`9!tra<?AFV9c;#p|*PLUm&8V!zIOa(TzPlPd3GXFQia
zE-z7>8CGpv{Pf4)n?<%fT~Ccl|LhT0_}+WMRcdD3y#1@vlpih&tlK!F^<%>3%eFR^
zwmeO{%bU+U;m)n8H{9|2AlvMcncrnT{SAKjp0j?7p=3iz=Q*jXSub4|Mg5#TZ~x1$
zXSM8KepwYdL2|!;W8~BHWtYExp4mAkZr=AZTu;6f-4E3Fj(YBP>Hz1raJT&tKV0S3
zWYj#Jde>%O)8yortWEj<COx~HdY^lVbiy^~tUnGJ|Fn)9b-&9iQa>J~`;c|thm%is
zN_{o6jXBf&V0zlU6QybwQ;H=Wf7KOSjb5W@E41@tdZk{~&r?6N7gsyAExO^eZ_oYv
zrH>N!)Gt?7oxuEao8v>XeX};MRl0L*x5&T0?;K>lEM(YoV!!I#-};;M#6`<IbQIJl
zpI)8Pc<i5-=Fw$y8McJ&(kgo_J*oGMME?p+lhvPfvfoH`U4FuJpwH*}4b}@sRjs@0
ze$U>2Xliptk6gu_UB)d=E4s>5)+~IKr&Hm_;NtX}qqn?SH*JrP&#ag)FETY2Y~AI<
zvwniJ*G$Q|YT`jwo4r~WZL4__;I-FcPoi4MgOZ99Of?KE4f#CHQlw0uH)J@NIP8f3
zzgsd!`?!SCe7zkx1uGvvyM6S|!o<|SPMIr{qm6I0<b5-6ioLL0d}2nm_Kg?I)tEo-
z>vgFOR7hFrYj5g3>F<>tD!+`QTozqv>t<o(xHw~CQ^njxk#nA}yt~k6uW{~oo!!i9
zf4OX0x4fo7c<TmdyL&sEJEP9bwx9oV%Wmc0=3d;dr(RobV%mBwT<KNk3iTuF+!Nlj
zO?vWpo%1EdlTXD(CGRcOxaWOFcwfg>UDn>d<>LC%(Jg0|thAD|+`TX%d$mUB@u~Z!
zeNLZxPD>+o)3Nl#v_%0OM^7BcSbFx1i^276^P^@5WixcYFudJ&WgB19lIGPfCY_Gh
zx>C|ERIpQ`x3hC=X2!WhS*Gh=`sPeMaVF6vAGjR;Wr_b5nZ{od$`EyGZom>>+xK6q
z(x#s5DbM#<zba_u*^5X1S*?41^O)JbAkaFviu9zlM?(D9o!O|oq~x9b7kkDJ1~KP(
zKg!?iwVUN*yx?2d9vdsI;M`ath6#@&7d&<g5nq=bn{in5<r|$-CyoW&Ir`Ma_0Pn_
zYsbI0U&{!wnkBFMX6jMHY1TPsZhVW`;lp!&r!@CkWu=3=k33k<yWg(j(HA*8whKp7
zMc#k3ak->ySUJz-V_bch^Xjsl|1Jh5)gG|yTfOUkjK%>^Pfc03@Hmm`E$!0`pG6;D
zc9`u^gJIa|ms)(RZ%_Q&a7iqm@kRgYZR<rF0w#Q2ll3pq-ZZy7fZ+kp`U#396NMPI
zalSau;Z_;sbVA_klr=$rB{z18ByV!i=D0b3;*u@)i_a){2zFH!vZX8(2<qyoec1YB
z;oUE~t1oRR40_<4ma<d%caQAdBa&esj{i+$^_#j)xOIE+;)iyJ{$`Xom?X%vFKX<l
zQce$dV7d`dyL3s9vf?Y9ZBsIf1iKH<j7nFz{6xw4!EeWG<DwSnPNQC~wJujad!(_s
zW*wCia=2l!d)wh|g{3tD$K@uf9DF9|B`q;m(m7;yUt8~nl^)sQtV##X3vRn_zpzSq
z{oA`YU&&t3T^v}Ua{JxxVAGR5oUVJL-UR>4D*tg~BkxKhVN<^^x0EkhB!B#up)RUk
zT)TcoMOEFeO}QOQL1RB(A&rq1Wg%NBUYED$uf+a0YiK)o$!q4xCq>1joDN(Xy=Nrc
z3>JDw+P_?taIeGG#@X>q)W)3}(!r5ijub~poqVK~trzv4P4D`RrwdKf^P_bpJ$`$v
zDZM~B>3z<(<?jMK`^slZES#*T$9t4PG%ETU?}0p>B~w<5a~*hme53S2duNTTuTz8E
zG>@i8vp;8_xqDO9dXFnvqS;sU7R9~ot(?3gsU|_yow-mT;8Fi?Z`tXEOb2`)8TlDC
zFYRmfPL1JRojp-_wm0YZUHg4+KUkIP**b0FRB^@wMp;%HJ$6cDYAzK|^fdB#=o*#D
ztT;We_JL2BuBVrf+{vRJpIuM)Y~-#IG-B3ws&|WHS-Mz2%IVs=&29&0O*OwVy|kVE
zfcx?46HjD{qf+M>Y~Fh1;~htblXnjGz4`p|5u5e)#B09wTy~%HRCM}}UY4DdlQKcl
zrr^DSm*3=W=H<mRCHC3>jOv%MTpf}Y@bdAk;;om?YCe5@iNVI}c6H#MmkO#J2Tlmq
zL<@BqvG%X%>NLD3pZA)jC{Kj_L+!Z>T{{!!DapQ_81pjSCw}YnS*tw5G~;qh#eRL)
zf9@`NG@e<YvvTIEGY6CxZq?kfP|TNmLD#x^@pV<UwpLko7vDa<b<AuZ6F3u3f3){H
z!v)2at9IGFPTcX#e!aMQ`{c}HKOL6_u2{TsmdS^Xh9^db*8=CtoL64BCGm}D*Ogw6
z?8U2_W@;@ImV4eWV_#$H&b;$hiUfmOc>kkV4y|Ba*|n>xT*6t`cg=Tr<8zjmy;56N
z%h)zL`TM@tl`c`*ZpmNQO=9_A<eAfNY3`KvYI&5#m+$jq?<`tB<JE>WJyQ(T+3)wQ
z%R9C4_N2Evo3)E=$|v}X=Nz5sFP<8+Q94~RTt{>}+v{sCU&JOnnzO+D+B>0c4~7J<
zlgbOHeDYw^WYAF-(R6F-5|I>G!n5CF(g&rM3trOtP8=126@v5XB^eu34yLyi`<O}@
zvt5|v!uCn2rLSdzB!|<d3f?_DPVV=Tu1ka)_T}(hP+I#d<U{Gq6|R!%bM*G;eoimB
z>UFehg?Z7QiJMQ0trtt)xXfB)#fc4jt>e$?>GdyP-TYCmVY=s2Hwnv04?CRM<t$UP
zEL9jdWSL3=x1YUO_Uq!sg+Z^RG9`6A&-k!7{*}D_A<OOmrHk&Jnn4B^k52Fo7v6Hu
z>efs9SkYChqNP6gFLw8zJ&jYCLHlpc;=8?Nr`f)~?L74P-H-43h09KU3^I1%y{^Q)
z_Rfk^K1bRgEMYouGUmAj>w?osuV3x!U0(!RMs{!N1%|l&><rRtmlPkibI`nOa;(wp
zvXgA;v79a08g(M&M#-nX99!$B;_PS;qsaQMitT`rXZD*<7XInKUfsU`NA&ZR#21HU
z7f9b)X%^SMU`-Ii%T*83`HRzXD&D$@bcL9&eRRWjo{ayC7T4H374Ef{pCt9(Re$_T
z@qfXemd%aU{jXADPV`qgwua3-_~&PFb&OAZD^uj>nn&#4&bx%~IG9ivEpd!}nkGwz
z0^bgEKK0WU!Qax>KaG~1t~=q;9EI=xYy5YZ)m&9s+G-Z$m3Q9Ac$Q%3wFm0UmPzD@
zY8p*YVa!<SdpUR0F@Hxchx7*bpiE^=*5~^fJT=Q-a4%$>8+xj{gyXI2*9x{(x(}q4
ze@ydR{p|Uz&@P*E=?DD2AOCKVnX#JhW0=<JjkW9stGgyX=Je1PT3grpb!&3<_kh)h
zyZDY(>l~9hnKvuu2FsS9RhN!CgdBdS7I=au?8Sn+6@gxvji*nm1m1F(5aczf;bQMu
zD@LD~z=cy)H{N3VB6eZ><$5;jOBbhiKHRb(=kEiFw3}{^d$yknNzUiImSFl;m?1??
zYfGBYY30CWeYQ8Wg6^*=d&k@9t-sUsp_ghjTefukS@1Zmwj)D^8S@19*|Xy_!{TFA
zxf?aNoa(W(4w}=ztRbQ)>}s=pYRX6Xj{!@%CnkOkoxPqZp}2nA{{{B7doG=yWqSBr
z{KfV>mpcdF8U;$f{lt^IXtfWcz{CeJ{LK1o!KQZ1`z(AI-W+S1c<S+S|Fo%>qFS3Q
zIsMoeSF}$Kdz|>2p*d*bgDneK944uBmT{*qI2yZELMPAEB9OcOPWt!HvNz8!SDz#E
z<@onocc)nLS8&JPb&C3XC1Mkou9kL*N`LWhafYB%W`akrOc761S>YSmy49vTQS4*R
zv{^w`Nh_zR9PIE4nq_Lv&(NPTG3dn!rKPFaHZAAQ@Sg9!w(-Jwt7B!CogZG?A~s*c
zPij$~RPE<2+dbBJ@G*GYOzb!>;eG1Yp;VQdT6?>_HA-W0ofu~sE{-gGe#&TXq+Wu_
z4om*v^3dAbclaE)a-G`tPVh=wKq}XjT1DR14--t1)%sJcz4HQ=|KIU*fv(cUt1X*b
zzMV<0I(FA4H+W_DX90#e!VC^J)A(k!1-6>y9XZV9%jmWAZ2z2-Cu90E-B%t7U9Z0N
zO6>lp65BWS&Myb&x1|!7`55fq-kD`LGbV1&sVy_Ln3rC;IWg(PBeyA%i?<3|C9*8m
z;&Q#kF=JIgMzl?b+V@0hv)dWl%idiKZkBEHE!!sB8PK7`s{Z3biszbF-5vp#IVWyE
zQQFkAZc)Yg-V=JK9aWs!-Q77EG#{^FT5vJTWrn1TfA-g@Q{%oKb=TSJ)jM(1Ggl*J
z#*P)ogI-rL>$0tWnR4g+^6o~96U*0R^*{IXJyWnno25SNzQgq3#vrB$mqqr<3_Rs6
z3L74rToGs+C;rQ@%ZYjIw6mrs8?trH7_R&gt4}?yyDmwnTezu4Z05~Pd!~!OJR_lh
zsWFT}OqqA_8Xs%p2|;(RJ@`L0F@2ko?&>e!kAMH+V`N&Pt@Lna$<fDoOIX)Elzn_L
zpn;o{Q6_x0#=YjC%oM(3)jmmQwU}Sb+M4wsaG~h$1V)azvrYw_N^#T9T_Pf&;+b(y
z-Kc*x@8pP^TXh$)WtfYlcB!rO%4>0q+8_8qf01+QLec%Z44Ri}ZJO!zGKO1h=7q1z
z`zu!(U0tQbzW$khU*O%tc8s1gt>*YYznj2uqu~74?jw6W@BMh1wRH96>FramN!Nak
zyK~HJpA@7jY+B9n-2DHXI2l>LWixw<H5u0KD4MWS^;y@9SxJw)tfU?ZGAs?5`RZs#
z&gImF!f~hc_!7+KZNDhWtROav!9XPbdxhAPP_J;;W$8Tcb;`F&bF+p1IrHNDt0-4L
z{ZALTf-5fPxdeHwHCf=yXr2_~8o4x5GIrSvoBgpiGbZ*VvpSrLI5p$q)Y_u^4PoX>
z!*dJ|@PtXKGZ!9U(qV9j)w&UK<Ep`HvAN7wv{wJ?u@Pzwdb8mGr-f%`T9!n}d#wjo
znaUcr2J>dKD&z=+I!E4Zh}<LBmXx^JtWRmF07J*;RmO|=a!9T?TV1Gh=GA@ii5fvI
zOx}}nUL4Sq$klG&?zHaPmoo<zt=?A6I$e^fLi<e8V>Slm^y_;yE1oYp)E9ECP}x^6
z&Gc>i#5Y=@y)u@oeO9V7>^b(QNw_u6gj@EJ*hPjVB7Y?lvTCOWb$Jx+Pk(S$><UA)
zRs%!qk`GE5OBJ4kmT<;Iw|qF8zv1%H)P+;mc0FC6qpQeJR95w6O1;+|-YBaZJDpd&
z+sMY<AT7AR*P!#MAfv*hU0SEEJdD3^vX5WkxTeFN&%B?mZP!0<w(aBRKJZc#A*lm1
z%Ge$??CxK?`~E}yYtMc@d-X{xQe)|lji)XNC{3KX!e{FR&iy9Mjoh494$X}!IrP(U
zi^R%o$@iOAx%;0DZlBLPb6)9+6vnW2uk3vc3~v4*YdcoGaIU)%G2>`x)Sh$UsT(im
ziM-F<vRgU0$VQ3t$}8=-+s!u^KNziisU5RAE9>RdAkW$RR-I;hy~4Pefn~{*xf35>
zUB+|m{VU$<tDk-J?_DDMZ^wHz!v>>u4Oah4xwB%Db=<X5o!s@@wq9)#zgv^T%~Zn1
zwQzdKZ9N0lt8HOsQe3OjHg8H>bTsH~s&2{VE8>%<oSEFZ$N1~v2xf+o+{IT5GP^xk
zG_yanu2pQ9Fz3-8RsG$iZ)RRJOTO`J;|;^rJ;K{Ao?5PZ{(Hgv-5K%z0$r@{&4Q9|
z740%Db6@%Ju7st@YLkYE%0`d3+DIMP7O^#yQ-0-zRe}F@3P#!+FbFJ>Tp&?9*PNxW
z({1mbAci{}3mGh}nS>iT7cb?T>XUTST)6RMYSN6$LNh*`y>nDGokw@!){A-z)Sp-V
zxRJ;kwz*nV_<Y8?oHQQGdD5YOXR;OKPh#G2X>LFU!@8=q-JYpe!rHd>u1^4$ms1z4
z>0UXHVMUKq^5qQ{zdk;EKGmzHQL|-BK<Cx))Tqnh-`1DqE?y8a^VPI@+so27sJ2{-
znIf<9?~7somlrEV*M80`U6B%`V%#?Wlx>xkZ0^J9y0JRJ_on47SFp@JrncgGyVoR3
ziFGIQzsWRwQ##n%b^LO-&-r)jvgVXT1g;QumiIGY?072oS?T)4jy1OrUg&zcx~A%o
zymZFYf{QW6HQkfu+8Z@}RGN}`<XLE|S>6;e&5r?-G=e_OaL&rL%~>$j)q8uB=NbX?
zO^wq|MY(NHugf->q_eQ(NNVSv1@2q-)m=$Ac&0$k*Gn@&MEi^Fq?#pj9JZuOd@X&L
zYU-bP`eAO_Rr&k74<@9hUwqMEIL%an^?|FQ=c=36mMt>q`rfztY@SZvYTp|w57M%1
zSBp4o_A=}|TCh!R<CY|a&}9t@H-DIFEL}21`S~sF7US!47U>47tvhgXU)4me$<Mpq
z>i&p6eJk$r_O(im*0qac?I!q~H@l^qmv(?LWQToyA&*13|MFE_u3T6CRvIN&{9jtq
zkuuY&X0~Co>4M(1&2xM<zbxLG85)q6EFC`)(hga8*!0+z!)}WiuHAkAX5C`mFV9{T
zCGHDg)3OW^<(zBqE763xLHoz4km7s2p^*%CIHII~FFRMHVIsCT`^`4tf@>>cL$55H
z<SQTX_MO{@j#FF2mQ|?Ce6)#gk;|mLtqd>R^owTBU|TBK&|5ZhpVFi=`!+n7J2hg>
zj#V#Ld+qmG<#XOi>g;=+8T)SavMyms*_|VB;Ka(gx{EI!oZa_UH>33YJI=`wzTS3D
z-mxuiRjGxd#o>C*>YChZZG;3aN$@J3Hu<C!yG$^gW7GMjr3*G)o1l0_SLk%z?wn6;
z8G(^D&8{J@mI*u;i+TK$d+n=<Tc*}bxpYlMOw-e=d&(v+QO%9rEaEv#1)dvzJ1^{Q
zjnv)a`c^h!#!>a$cL8>`JC7zXcC=i7w$ZKcxL#QAnRimjH(eekHdniCieAQc;Kao4
ziO0{fuU)9!aOT6DTWVa49-33kHlJ#7O7T6JQm>X^GJ8+ElR)RL7J;bH5cMOyTEPx?
z4ES7|ms*7wSzrCfb-;zO!I$sWcQJ<PorNDx)vsr6xK@7gSLT+*9{G2K@0|YQRep!V
zOM2;~vsY7{lM-iZ+&LDmwT&TL`n}f`o@)*=A#O7t)iO-KYceyQLBX&sjA6;*>o-y(
z4xi}UyUgjJ9;cHg>ykh1Gt?IeE{P5lu{RF;C7I6l^W?;rl9$7C9-QN!wCmrUiC!yD
zFIl!e;{0xD?sGz))MglPw{Cc~x@@y-M78duKYPNxHvZVFnXA0u+~b_M6(&1>MA@(z
z*t}o-^mg2Q&3EmW4brdoe=y&nowhbg*rXwG>EFWntu8;mv~+4m8(uFHSi3ZQV%oJ;
z7mg`MYJ_IbcC!+3dg&1qwM3w)>&vbS{JdtHb{^W^HdlSO<x<()iDK(c>fT)w9KGb}
z#1u}O=KODWvZgI{&=$*KoDfvCHCa+$BO-{6p{deB<W_;MO11x<&{d{u4)%UK{$1ke
zJ&T7A>t-LH73w<klJED6Rq{rYj%9e<JevBpSt}zfbE%MlZ)(Xzss45Um$W+>?>=^C
z`C6+fQ{tBA^`?EbYMy;zVkGm?A8gw~#3n~8ZV^o|%-IzxapT#R3mLcH_IE6fT|ImM
zt=v@{7ga6?n=5^@Ui)33VDYMRJKyvAJ=<l#DR0!UI_2T(?vTZXOKu!9+ouI?e55Zr
zo51<rn_*SeQvb4jbE<CnOg-k(_*LFaJN<L>pN&4EQruq8Cg0-Cz45#2rM5vxRoKnx
z`3!eT+vl6_Imb8E<x+P+*%F=F(#O%qA1_G>KEGndN~Pn;{N+V94%_E?>@4LJue9xW
zoOa$c;CVw(vXQOzI<uMc&c4^#QDor78g<f9J1716>MJ4#qD#VK3{EWPYIwT()5|7J
z?fRPYYJ2W|N!)%(n?a6KY?ZOYlO@q%nx0OYlO=^Sp1R)3P7q*TX7)Bk@k+FPn|-oX
zROBJCs7v1>PxJAWMLRAoOSi1~t=$&Svsr@MOQ_Xsp@1<ngB~ZxOSjVHHvKM<w`X#h
zsEL@>XRVcq;tV@mIxGLID3gTcJZa9?%}?D}0`C5DOL!tM#i_Z_xzTOX($}jmPl~&F
zw?H|A*Ka!WjL^3J>?bB37cw^VuP~dGCAe2Qwp@7vpN`$-TW^jOEsouHzWztnxz@(F
z8&tx?r`s(5E4cONTC3O>b9m(C7*5=o?xdc&%2>G7$z)j!?-{2%M^m$U_U+hZuHt`W
z^QjxvFEh--xv$RX0~LUf1vADU)tXD|Ov>8lvsyTDNvIkmKApVPBVqG}Y1&&@4%~2k
z-7(+CM!G>y&wUg30X6ou@4AjgJb5!^W!lSi(ic;cc*+!3T=<$@+doe%DpHu8Z@pvD
zp93!*O==9umP^X4nJZ%wm3_3STu<)ItWEX$v!+%Zy7i5D`QeiXA7=?4Kd)_+o4Z@O
zGjj94D<T(9>}rc+pB?-2vE5;v`kH>$h{Il+Ckr$9`7Zhz(0(+csdhup=|cqp6O#-t
zz4;rv-Ag27$%iE=iPZ~zPvmj<@+x#+>lP6ZF$-Ii5ma$~`aaWtYL_;xbYA^5%UH6k
zZrP4KFSaeaH*4zkOs`3<T>6)Cxb~}vomMtpHeX62H0$yWt1AZHnvW-Luob&4u|>D;
z`QBZv8zrJ#7c;AEoW(86%ptquqG`rC9-~R@kM}-)>6)gHI%gu!)>)xyBCTtkSRYKh
ztahp6f<!MDgYv{C?{6Nj+9oq!_$I>J`ryohRUN{usa1C^r5XNT+xPcu{Kob289V0~
zOjsq_tGoUCwqF-MMSNcDRqlWIQ{!A+<6~m2AJ)5hrK{}j6FZX-I#J59DY|3Y%PXPl
zPajpedP8gDwvSW0Ann9oV$DnIxEMSaF=#A}czf`ade+RWqHF!Tc^7HwF*{5tdbmB;
zE<?lbYyjx6k&SCb(m5|igul+Zv`WRj{S^DUwu^GH)&EOYY*FKjk&DP!yfta|>eENo
zY&`vwZ+EQOws!$HI~or}-|1ytu$jw&OF=~A(#6+Cr@y?({`h0ZEuQngk6(S3u;=(5
z*V+4|Eak%aw!OZR$66fzv*e(`X{oLLR~EVMx^nAVneo9{%MDFsk3LYokd?XnpXSkH
z3*|UxTk!-LL_8EalzRHmsqP7qy|=u!$T=oYVcjE}bN7^JTYis8ah6$=`(uv#X=}S~
zsXIk0uT%3d-Oan$C_v+g(yF8Hy)?hAeEw&_(vKG&ysYwHH&@3Zdih0-dpgIG-poGN
z7aTmJ<Y?zzCEqoY><pT+anUP6(+VDxIIW%Hwd2?ZAK$b++fViDJb279Uv=Y~Gcn!i
zuen}tPTX<SsIXOm;blX#@GikkUK_)AZ<Sy@(0M4@Zto7yq>m!j{^HX=znPLN717M<
z5YGDFPcr%8tb<JgQ_S{q%ndR7t-s0ZY|k8@%}%TRUO&lCmbU*6X=6At%nN<pXz{>)
z@u%q<-qpF!EuH*xlJy*kHyhfURNJrJs1<o{v@5A~chD)TozpYl_AoT;zP6!hiM#*J
z(_NceY?u;s%5Nx43^ir5&QmcCe!X<ZUEag%zinQgdN1SUYKh**%niL|)nC5s+uglO
zB`#h~ZHeG@A*QI6r+TyePl&fa&u|H!BmZQ}McrSoer?%1Q7p%OG1sni$u)_g(i3-g
zUcR<1e!aiR+KJL3flF8xOLiBe&%Q8m+x1@eOF`~$s}6ivmhqP@DvLvO+19+Hx{tL@
z<kw!%%D%;KetOc|s1v_0$aI`oSE(~;ZF=$BqgQPnWS2-RoIfj-<=xxKl3&t8wD0sL
zRbLIbBHMUxL+$c21+I!W%Wl2>+waDH@kPUFPWQg&u7=gS&A(^WavDuqc%X3WOw;K*
zj)q*Ey>`y}{mKirZDVRMVx8TW*0v=~r{dDB0~u+;oh%D}rtS0LF_%o)wc6)vA4}8~
zk?A)-?NHtLBU!pAa$WcHl`jq_R*0YT{8z<%;6mH|)>{&%<5(G7l)PBG@33!~8OqiW
zq$T2%z~vx5d8*eom1}1EtiUza^hYZ%vmQ8h_Dlk!z~nhEwf}XW|D+~&v)8qIchRdo
zqC(qFJUW{DIZbZ+g;QyqJ9lkxPMy29V2kuaBSY)2(I%%tw!h+CIkP>YT(oP;+>J%b
zNe9KepOuDwXg=P%WAWw-x2}{e3u@y#C8F%!!W5@c@|k0+X{w~>x7_JLADpLL4ga&E
zujRPV{OIKupDnjgKXFHtU$<5?;LyGBw@(xnuilWk=+K;*Uw5C<I_f)XI`6ckC70Gc
z+ibU}LO9Blcizl^jmmE~@WfB{Z~2??bNPm+EuC!Dcazp`lI|_f^Emh>YcGe6_ao(`
z>1&*%IaW?dI<B9#VU0w$@8)~AqLw$Vx^pw2nEUN(=F{gc-876!+i*u@PpIDuhK`_v
zJYfsvjn|(vT>VOFmy>pX%0lh1-Jh5J-rThExKZnrq$eGbN7Xp4aIi!^3D^F=C+f4`
zszuS~b}`&C|GakA!Krm-(&s<AZQ0yXp<H@GYK`p0$NT1F7=-bzWm$Oc*fk!p?@B4W
zGq0DpKd@V}@>=xV->`^CZ!(Pte-^QF@qD}J6Y)*IG$;MMWOj1vp1|E^6E04jnwWa5
z<<ME5BW16?Hi<BA$hju9JFM#H|7$m7Ps_e*IQ{xgZ|l^~g4)UMlXRumSXqc<&QvlM
z3_SIE``m>WZRX6__v?Y@nRj7&S5M3Je82HED@@Z|sf;V2J^0_Z9k(|8dbRAiI%C1&
zU2hXqY|1vX9N^EF{*$p?j+eP`g1`Tcw+Y_0ta=W+dBqnPPne?A5_W3C<|#`;3eP_~
zc9h}3rln<%CQ7RWXUp);*>%uh>%OhZVLGMUiVIoi-AcSGKWp82^BpFq6Yefre<+PV
zQYkFBQ1JA<%c(4G;cuAE`^25iT)OOT&hxds+_yu*qC1Zot+^uNlf<}5PkqgMj;@v!
zE4In?9!)&$C6bzK619wVg3&Q1hBFL0ZgI-7>gPnGT6ard6_(!hCF(O@#=4Vh{qtQj
zK6>unxS`}dfBf0-Uy}Q$Z83cxmY3@KagL|UWGj}2S8~ybi~);PXHEH=nsn=mK12VJ
zMrDQqQAOj3zmi;QyG8ol51%yA&Qgx-&CPkNEdbu*pqj;d%|S-QYp#6V@w$V1M5KMT
zUk%UQYMu37>GJ%-nL)pn8JZi1uGyDVx%+6ePyENjo!75z2yS}2KH}cF*NMNhM4Voq
z=-m;^?|U_+_wL4$bA2M#b|21MpH*JU!r-?2q^rpLofC8RbObE>uxRGBS=#Jtx6Q58
ziP)~fzVXiPt5;@9PTbygM|wtSziEWGZ1J%RzDciL>wfE)p4{y;TlM5E9@&h`Y;i38
zEv{zMcn|sdmqq^ymp0^BwQ=GCp6g6&%<gT{$i4Z%;M+B))u{{bDn)H7-!efmN<k+_
zMkdcUe|8b8Cf8aC9WSq`eLK!<xY8zk*<e*zY9V`e*_QflF^|ukT&{N+cfEZZr@hf9
zQCg%==6%@YgO!(0TVGja_AKCS$qY6PW1WYa>Ufp6_?$i!vRSD6qUX$46P14%KXfa7
z+HPCZ*E?z3Mz!lgRW4VuI(ZY8&abVh&ybAgi#ye1TvqYC#o9kIdHTU?M}JuwN8I3k
zqVvF9AkKNN#+r5IXB0iFo4@^x={fLO=Ik?u62n0Ab0<NgMm><$z>)uZ>wdcDJu;{*
zE&b9e(4G+_G40@{vf!L1(`nbgTbj#1`LiIjt6){%Os%%auzPvwCwelCG{34bI0$%O
zxo8lSqk4bOi)=k>_5<IRr@p#zy{gFO*_n<dwH>#f-#H=JaE_-}=AoNL$<irFB}a{2
zKFTfWX_;We=bBi3<Lb$SY=KwVc=L}q$e8emJ!cnBowK8p`Gt|6#muYkqHBEPXYLA6
zxUj13$KHu2w51MKT#Ja<>UjQ^js59=HM#Soi*v2G7EfPeZe*2nt#WsHVoC4JWzT+!
z-<+SnVEHr4bq~Y$imW%wU|eRlD!cRM&n21J&g-S#@$P7AlRLSg?$TtHjVh5Ds!LlY
zy!pSb<cQar>?^IiHYY~69libNZQG)EeigYlFQ^pXlo0dPRg@Cy4vl0mnzUwb_V4+c
zp&w7p+`TE%>B<(d9g9<EA1qQn<MYIAmZZ)i5ru6=4_|ns-YvhK$os_XmaNS$)3gou
zUX_%3PfC5hanoAc{ij1)c0}w7Y;E&<X1e7x=S;Cw-f6u~(r%yERq-Ymon?6Ucg=R2
zYi4!tz;z3I(!CjfeZHoc@GrZ#>$fGt4}R?!mf%olnYPH7_1iD5TWHbs{lv$JkiKQ#
zjc4t7xb}Na9qaky?q>|PPL|CJnYwb@i>|phPlu#Wt-74F@$90Vf~`tdeK&8bGSArZ
z|Hgu+vhkP08NMImWWB?&(D^`jWN7!Y{^i=cC(HDSX)b5YWasrf{D*y$arFQH*Mh53
z`*Ig;2u#{oU7g!>^lP)(Mpo?uLR(w@%sME<a%Ig_ccttvk8>B7vDqGbvH8vU|2co>
z{$KZJ{lEN|^*@uoNA4=WILm$e_YJ)#*L<wvIMZWy`rn`ZcmHe#tGRvF{`cmu<zZX0
zR_JV!;O3o>#2B{QzU20@L`lX4iw^45@EsPbOxrv&Mx4{^l+w3fm%KDz_Z+=&c^OBu
z`IT$BTX&!HIh!1-y>V6%i=H}*L6OZlp-o@>PEFCTnwyoA5SDh~QgY^kw&guroEndw
z$gW>KC9x;;+CPz&U1zRcStnyI_*rWA9jk!QkR=}$g=8lew=zxcf6xE1t!=N}lV@e7
zH*WT_)~8wr*d||iIp>Dfp{lu753{BkUEQPG&oVvTs5x}Wbsee2Kc}zaTECCMU}lWj
zT29W}H;$b*1rIwo@XSvCGovhE|MDcQd9wRUHf--XB6B@=>iWBv3WF^sNgkGK+oQF5
z$EGL^we}rv6{0?@c23ne7Iyt;l1|#HWm!pAqwY-Eu=lo0(|mypDW~NpU6-Y|Ju)io
zxhSg@Ej#OV$D>`131KgddS~Qpu&TP}xlWB+b&1GA4Y#!|Q?{m6%-&X;+s+(Ve|w8U
z%!Wg<zL|;ES^aDR+#75r2~4}P@}bCW69bKnFZb)MePznPkXC+ef4xzK;XjpPpZA&n
zj{md|y}DX-?SZ8o8)iB>b$4(@CuF33`T6H_{oUW6Hy@4vGVhu9&&T4A{q4Tj-I?qd
z7MyyeCua4U9JUMHCXd%D9X<16I`?L?^XYD%XMNn3){(fmP3*zryO(QrTE?C@%f-ER
zO`&tzj?yo=)!&Z^b02=itN!;5&%?Ug0^OTi!rE9DC^7G2v<N)$X3eUA$q`@I<j$<u
zU|6#D?uDL>7BiIIo;OKf8r|;4AR$^>c+$yPB6f*j^uuZu$&iga+~y0f>h4iBNHo7&
zQ<D;J_U??Gtewq{6W2e#DM<=rn2}>9vR&q(WU?TG`da=2-BUz&GBX58Ol{!1^E74F
z#?`;wR)aQa&Cdsq%r33?`^QLg8hiKR&s-s5)@+lDd)-o9``lHGCcHJ(mi1NHc|}7+
zbGldIxpQVRInUYGa$L7#+qFZ+`}nKen=|Jc%G}G~Ike8HwE6ZdAq6E*6+PkLa;AlX
z!7;{LpUI}xuL+2`9Iotc)|;_KdZK$@(A2wKQ{xP4pFVzf#$nQfyn|=g?KwJEaemob
zi`VN44u22&{kVFYVc*omSp9sHmM5#M^quCaIj`+V>R-*HvBl7Bb?e2CFZ|OUmd{=H
z_xtgejX%Dzz01G<^F_IW+l#aNt3NHDeph6h$i{O#*QEC@E#_xbe9`@G`sSn4FIL-b
z|DBlrCGqJW{r?{IpZ7<`<$LJz{7*j5VYVZyAoT64cdox`{{8H~TmP$m=db<$KmM9;
zfBz@<y%yEIQM0e*EV%VVd*?3S^jsaK6&Jds#fw-O)VaE+<jw3$dv;4W``r~b$?bZR
zuZ3@$w&9$RnPkdMmEF#3Z<L6vdh{`}cIm6&aFb2n&nOsY&UD*TYVI_1mQiRbm$}kX
zk=Gk<G=Eo9y4bUE&HHsFuZ*1Q@+bDayBPeB)1Wi-`PVON<{EA(=#n{~#`gK6;CTc4
zRo`~?GjQBxOj4UY^WdXDb`x`?q5~6!`~<~TK8bg^xQH{|eSP2QgmjsrX&Wv#%>d<7
zRd7Cy(3$GxYq-vD>+7jISSHL$e0$nK=zvOK&$Jn{PW2>ybxTvyo0{}nTQBU956kxT
zFCSf+Z}&_*um0Bz@$_vc^X~SZP+quo)3Qh(O~Zib9uD5bYx7?o?dWK6zkZJUx}@-8
z>8YMm)WmmhOqiCTvO7;@M(hcmJGV{+ZLGG=&lc<KUA0z{At!^m;bQ37nwH<&PqAGT
z2xlm%ERfps=kRKI8QnUo{l%&CSN<$H=wEAIc;Snd*uBSR7@EW;7I@AnIdi(z^5qQs
zl3mOF<^C<a{MUK;@4IO~)}QNoZ*|xH+-Z9&`F{yv|8CD$30LQ42v_g!mJ&XsQ+M;N
z_yI$Pj*I&`-^z(eZIB5%ojRrYYT&UK{qO(B{oY~Ke7^jSVna7yMe%<7-M{wKd~A;V
z>HggL-u(a9BR|@o(BrS1|Np|x-_2QH9-loOTKY?Twf*mRC+DAgw<dhcu9FFw?^YjP
zD;j8R92o0!D&oRK$yt20ZktsjVi^@Cgs!@4!|tGAu;xbS)C)IKw(Ps-Ugl=<Aj)Zr
zl$V*5b=Qohsgr!&8B<gihxBB*xTZ!qT;b)d>|WS$N8{?QwyK^>QB6*@cg>cIirp+Q
z=v=DRbnz0S^r6U8r>+TovPfr;O#67Gw=B0b_qI9rfup}H8_v{5E|c;o2yR=tRcVs`
zWR?c?BtOBbz+7*}3D-*=ta~kgx6E$JJ)3p8Igj&o>Xx1UY_{!VC|B+4&sTPOKTSXW
zTy~-L#2L5uFNu<F<Xo)1E7CuFnUu=yt@m^EW~qt$y!jyi<z~e@|MJ^x_E)03yp~^Z
z3so_Asbna!{;S^Z^vo)kmCI)73eRK=;+@oLHof!a#MlJ8d^tJet@nCa19Njz<qtbu
zC`g-qaiNd0z?4-IGdG^7oifX3Ra+55L&Up<Q)Q0a{?zOfFFU<)_pf{LZ1rWILSn_3
z;{QFX|ME}x&dn#QZ7&!+Z|=LM@!ELZ>C-1oZcB>W+f8R^I@rB<_1xv(A9eHTtt%>F
zNC-Mte&^wmIqz*`;utG#uzz^_@E~{Tp8z?A7nf4DOZgXlWemOb^K$*aqw}X6_1U-o
zwsmE}|GoWX*Dp$)(apS3W`4}bO=F9r@<G9-rq9pi+4lc)stH<WzrKiN)$gy>^<G!)
z85G`q2{(GyyD#9lxa^I;4+XU+%$rjD{h<Gf|12-2&EA|k@BfFZ@-;R8zb|^>?69~j
zYk>jNg6=8nYB%UGWv|(<lEQeVhmUoShclzYLXlgyOp9wf9||y-7oL(}xTC_BHz6X%
zd&SqO9aU<vM~n0W#lJ<jrz{E0@=2O^<dJ2A$xe>r)eUn(tu7^f3~1x|I_m__sVf3k
zr@dp?DQFzga49t_ZQ{za8?(*}x~gmq3%lI*?7E^X@5dXxS6n8Rd{HgiS#0^eYEz;2
z=XVoDX5?&n)M*p*GF`%2{PpwdyY9s`ul(h+wT=~>_}FvNc<Z|Wosy=Ce&bJG9f8eK
z-62U+Zrnbs9GEdzuy>u_&*Q<8tTs23*5v-FNd}!c>&L=)D=R;^v}UU8d3m40d}i6D
zX&k`?3+JzT@Pg%r>hWMvPxsqeOOA1AUku%PFi(8?9GN>?V%9S(sN61Q^)lehYTnid
zW#$VPOjn!OcTM!#jaIkzQ(N!7y{KoYvtwmy+@Fp~LHm66y-AuhE8>O9lE5Qsp&#c-
z3o2ZCvRUKaX`V|@l2mptY`E-|yCaax<@(ZY6RDGVYA+55W^#F$DXf2P{*vp7?97Lz
zmJAFZj%T}t|EqYb7xypqL;CtT;-4<+HN1N8gFo)S?9T6o4R2h9?0P<??P=dE-hI$k
zEy}J&it)fbHvXrkExSZxjT>jyIN15}|NN9&P@-Tk`<3laUYlR)>*MOn<mFE`|K9yN
zV{Nr!{6q5|<3rQ_JLwknGkkpU`1jx6SKUEs{_pJ<cyRcUbha%shg0$@#wBfa(fimR
zZ2c{k_pH^*E&a!*2%dNQ-n8`H{%N|s`S2h6xl5hDh*i&i*Pbfxw0K(XMDeSK%Dxrz
z2OnG)E2+YAw&xx9&H0MPT343kEi_h^V0F}y&c84<&x<|aSi$5Q@{3)XGJntbuB<Ru
zR+x9oJ|=_Jvo-~LMt!^}>E3M-*0*fR8OG%&r$~9ulu{11o4Iz%Y6r22*A!-@UhG(#
zojz4^Dbs>88ccUO++Ol(SC}s{y6W=DXyrnm_EV>ilq!pI8D_2vTb9oJtKdV%^W>WT
z<q?lg9a`z6ec-9x!nZFi``$eJ^f9p9s^!mx-bpK^xKyOyzB^XEOG|^5VeMHBvF~jq
zM%V2+S93*4EAa+syzo$G>3x3u*TKoM3Rk1JuCDa7sQN5-<Cxh$Ht@34tNgqPi+Pu?
zv-s0Kwg2PClRCxm_ssn(l4fQ#E;^?Z!umV#WBA{h=iEfPw)o8TIFlCf+2%=wrk;cU
z=Z}KUtG}K8?zTIo;e>#%m%4GOuHs>RxyYB1{quGOZgq%RCZ!T_<J^RIZ#+sE7Yfhb
z`sD4m^r+d{t*1Q7S{2H+IGwf1P`b2ho&1wO4>U|Cc<|ml*KGFU-{q3BKNTh{6VCPN
zcBbys{<LnNas7slI)(ec`|qABX2^TKpyq?{6Z!YK4ecNAv6)S0PgrlrSn=a4|L!!U
zWqp$2Nv~X-m|I>xlnbz_7UO#8dY$imhUN0bddDx@Y~(-0!|(2IzvpwWxB6bU@D-n}
z-^|H7E34Ca^zvHAKaAhpmTAAge?QipAuLPNMc-rVT*aP}k51Ks^FEyu5RGC^Q17t)
za{0yQ|Nh^qIOo)VdvwnIM$BJ-i}NQw{{76^aB#0(^7Ze}wbiR+9L0}S2+TB;b9v#B
zv9-gc-p$~Yvf!&-AB!y(yZu|A|E*xjnJa7CKfUa<KQ^05XJ5g!!YfNwPu#qEgW&^F
zv6&y2^oR;Eh|UaH*|S9UoRZ>02b)t`cO~8xPb-+_y6mIc)y=1-teoZA#H+j|Oxaz>
zEpUa9b})bIg@$X(oK-)RES+EZCHnyHW^<LUkHYtF*w>i;H;diwvA=bT#VtQx3G10v
zTgtzuNZiw%_q)wYf%9m^trR)S+Wr~uwR2B5GP*eFxJ}fZU!t6HY>%Gs9-BS)Z^pX$
zBz5ZUx^?Wl;Wnu?xiw##rx;BQ41ZR^zP0V;kEO+bE?vBPc|*;GB4O1#I;ooiuRop1
zalqv&e|^}+sIQ{ku1-=PSDQUntm|ig;1;q(&^!A!*LTx0XQ{Wx7WpJ^Yuw;l!Jtuk
zq{-l(p<cxkMf;N6(uD#kzZo|lntO3!(DnSE44cIEZ`Cei^tgFY`l6rUvn}N@Yz$vc
z_U(TE?PY!ZO!Whd4Zhztl~-;u{`u$Q(bxAY{{H&=^Obqr{(tkIivPcu^|$|A`1|wW
z`)94GGmShN|IhW~&&`c?Qg+pb7u;LTTCJD_mP`n-f5^10HK)hn;)-QXSJ<zvIBh6&
z<?`O<?!EKgZLE5>>;0$m8P`r<&b2?tBf;6F_wCxQ?bmF@JZ`o%pXS)X`IT#}^!0!?
zS*u+)ez&v#Kl=FP-XHgB-@AT)UwZm}Z(n`-ga3K=>NWW5e!iBEvwag^a^U|y<?8vn
zyX#BC{<Qvf()GF48Z{%eMfIdnuh`<K5A$Z;THUE!^5oO9b5_O~v(mF~vwmt5z2<v7
z(E4Ir(znM92fWs(WKQMEGVMO1Vd@oXuxm@&s;x)rj%2<LU|29w`L0LA|8fQ|)rA5<
z$98YbuoC8I=m?5FdC*KFb(OI|be7?|SC1|wEO+;}-)FzYNN4x6>^upE1M}Y7#M-QO
z`YChO_O9*O<L*xCYszK3m%rVgdNlgAxW&xZ)y`5E((Ih-m(AY#Dxkw?ZEo^rua&Ki
zmxHsVZ~v$S-z~XXzDi?4R;|fwe}h@CIH&Xd{H}ldo8_Hl$C#A04`i*eKB;zV&Htu_
z5igGXljYCca_RgUzPa+!3q2&0Tb&MM<Voq4OnNE#{TSnfmoudWCtkR<L-1?*EPJL4
zw{Ex}mD=qjz>t{hx|*rM@D?9qlI^CocC$l|{p#Mf=5}`F?z8+tygJ!O3tsKu`EkFW
z`{&-jUpM#Hzr1;T|Ch$cyDuMJJ^BB>GyiHPhdMs0|9vmtHkS9qTxO2&Suy$^Aqz#$
ze6G&s4NgycUp+6^U4lz-DM!!&jce+nQIYpQr-lTFO$u_*&SgjvX5rprbmv9v;ltVg
zR=UO;re!HEzx^#LW9mO$WozTD`3l`{Z!A+}>AAT0^Sk5U>)+eI{6D$A;zN8*L``zZ
zy}k3#C@uWHtbMQNCu`>U|KH|*c>9pC;o#nXDbEkTKOy)3ao5o`(eb|TKAw5T5xG=B
zV|B>Fklm9Puf34+sB1OXr<ae+`lrriOxVxHuzA_S$$PJvJ&JI=_I5twnJd3`+&ZJR
zc5B`o*@iaWvu9RpU9nAVb(!4KD@h^?8UMOU339Gg{&!>Xa-KaQt_G`o4*os&^RHvV
zrc3AReq`Ni3%hQ8!6^Unvw1dg)jzG&x9ofEFTZ}{YS8-hxVLV*t&MW66fa%X*lDi9
z5WdT}Xvx>98Pf%0RJmit9(pa0Dz9UUyMN5=UmCbZnaWqkK396p)G04_#?&)5v_Aa#
zy7%3@MX3+^!k(waiuz7A4P0hr+*T+Qt+>FoWMj28!#DA_7lZW-Ohmld|A^1&W@p^+
zUG)6PYj)d>6f)+e?@5>ybA$KptDf$*Q?pi@HcWKRQ(?F}pZ86&f3|J#>-FYNiy~Dn
zY!Kg_6z*VTTprWL@m+X=uAGft^Ot$WVzwUdEz?ifPm+J{V|RaM&A)}eKYX8a)7*+*
z&Ts$MimLawzt^3%*E%_u*~D-6VXx<xCm);JSRuaVw1ccuT4?WTg(pT%3$9K|Nw!+*
z?C_G~f|omk;nAag&wD1TbZ$^wDxq$8BWtn7mY6kbs&|*BrMDbewpek&7S3o}mT0%P
z8E=FPjCXlVY~16K?ojWh-1t}j8=u}TcJb=$yV9Mz3<DQjZM;z4pH|+VHsws6Y2;V6
zlkKVh7ux*z_8}p^c&_DMtH>J5H-A@ee)PNWeSOFKIVCTj9@?Q|qiHK7kka~g*_Dhl
z`ZuiPFZ|oVFJK(8J&SYkMaKA3%k&kKb{>y(TBa*IsYco&N~QD3w);K%vxQdg6lH8-
za`Dr+P_QvGdT$@we@2eh1n-Gw3d}PicI&*(zMA(($Kc^2-n^ARKmL7KUjK#p_gY=W
zC-NuUcFSs2o?3gq>D|1t6;G~l=ZcFm2#cCN*;2OU$H|A;49u(6Gv0UoD`}&oG&6LS
z&+>iy7#4`7EuFMh^qN^+A80J{hnIVM(6I<UzL)=kul`L~5xi8(LY*O__WFfm|BgPd
ztKYpbVWMz(T=pHwSIfDRHGhXN9Oy0l@xIQ^F6w3aAKl=jnTDyS&T$J)PRxC;-IkPL
zdQdmm_iVNC`Fv@qNskMSC%it&8hN07mo1;JTX@F2${70}Mm&r%7P5ZIMgCT0=@ORn
zI;xs?)W@*RJ(y?qqhaCI{k`ARtU-mb^~Lsn_Q}V^*Ds#@>Sfji%bzFxF8;h4zW-Bn
z_x)dMk6!27*U|o*IhOD9OHax0H)oFUMQmMZe!^}^_VQ0&g{vPsXzO2U4PtOgFudd#
zHM`)Ga!k-$@vEPHu5rI`DO_~x{0iNY&0nU9UtJSdP+WNKV=v3u9yv}X6Z_QL8fSNP
zM(sa4aY37+@|AlFkC?raG?&RqOPgNCu&t<I!YapAkJ8USN<Y6ZcCEe6@BOpCA5_vk
zpeDRuxf;}^p11pZ`px2VD)UNCsxn5T2ilf*Toh%9<!gMtr~2);+YWclmfwAr5f-~?
z?we%?#ojc9P3(O9{P@+jTKAovsl~q++HcO5U^F&gWTboQ%8fkB)T}!mTO<`?x4*9I
zyLH-^`9(nH#YyY9Zciy*9ILkR&7+)-qjg2!?jCM3Hd*nytudzbe0Ivewj{G`h2zPu
zwl^k8Ii$Xte|ze?hS}^Hk0(a-WQCcQq=c=Mx8I*2l>L$I`S#x2KfcF6hbw5yO3O0d
zc%jT-Usw0>*N>-jl+1-JUjBY>dT5H<$#d&ZWEf1C>YusR;&QuKVQR12^&3w^Si7BW
zm55Y#?Gm(E706^67hQg6C;z#HH!j=?a4x^8TUPkeO@vGQ`ppBc6MkzmEYygQJbd0}
z+kUBe-)H}2Y?!Fb$l)>VnToGfPvKQijqoBXUH;!?`F|zppDKU6^q;?uZQs6%djAiz
zYwg}{+PUJZtZMnaFRCfpI$1#;TUD2(CULEe5sF;;%lMRotJfWwHcLhQOFB-k&Yw_Y
zm#k)tFzb7DP2z4{N>%66w{s3Xp0@X|<jt~kKB=Ob3_(^*20}cmP6#ENdEfXDC*}L<
zg4ykjogQ+D-?WXlN@m{hIJn1yUDGJJNn}!sd(%pb<Sz=L6EECfcWuA%DT$nq`|nHs
z@PB`lAxG`w>BRTd@9up+UcN~#)i#OEd-~CfkCxZp`hVH~aQ(;SfBn_(sWPZ-%-edl
zvn73MgmQYh;jX~PCo_}WyU(~xJGiR2b?P-Kmx!J{Vii)YOg#5zCYCB6^jdT5s@P0J
zmrdQLJG-`oE&LIwRLGmK`0m|^4cc9Qo9=G7@$A&UDS3f=-u745O|RWx5g~R{Z1eVo
zGn8MwWWIGq>+lOLkvqp@Hz!LnG-@)$ib>aAIkDl(?Y<=G+dt&ZK?7s)cW-1|UN}YA
zJU(v4L7mRu&s#lDT}hb#>ALir%Y82juS;b=%M<5sRhYEUqcbIK-C4eCH+qlEdH%#q
zp=b3Q?sDcU9KO3F#S&hw3J^8S4!zhu&9eBVq13*vj^7W2qW*}z-s?2c$acc4T?uEE
z)=qRjEXk0P#&_l4qNvxP)v4c}my7*jdD{MTjS-Jn|LG6s-M`eB%)9q{clyQV#jEEY
zZ$5eM$>-zEC(UhGAH2*ec%XOE_Ly(qlB=x+h1p({ZIX`Kek@+?oAfo9*)eWO;ui7D
zrM#~<?(xu;H55OZ%b4n{)61f{^pKcGQo-SG?7SAt#-28Z<UV#!Ok=cWHu^VBo9m$N
zG*5<GPUhV6E@*7jd4H5U>cr8XE=xt$FXTOEl<btTba5(+XJ#5tz7=z;6Gx!9H=Cx|
z`P~yQ{5F>5jMbfL>1VI=TYkd%%p-ew*PMU#{`|UqGT;7Hi2i$hw}0N(b^qSJ+bo|S
z_wVSd^LFw7ev04K*A_Y;om<SnaIH{;VS!uH`R<n7H@tc(A`X|28g+f=n{&=*t@hd*
zU$yoy{!8!OFn7I?&aCr1Tf+WEc7;{(F`Pg4Ui91bgYg?={!P1m=lJ*b+&@R|i1PoO
zd!+7(_|)#2yL0s`x1Nz<)(C#<7TSI)dhN2Ez86l$7&o^r^SKl>BPKOyNvXN+v|agg
zrM7+i3|=o3Zf>Pwb}NN#PWvp$$vW?rcutZ$U}<S_qrKS3P}`5=)9)?oxXj$1n?}6O
z{dfAb*4<yMb6KB>-2R>){qBfVW!U^FnhF}mZ5t;pwvSJq&KePO=gQ*c=JOL@oM&0U
zn0q~@I?r&Im)?nwPJ!=xc-vVn_TFC{6zZ|~=Umo+)idsP-0`UTR48_F`oqPi-M`=e
z{o&x%g8w|04ab{3OM+_u{(1HKdHBA476sF%ZQAs@MmTNjvKQNKS*_Nc!f}0S#@7iS
z-S7UEjCz@2c*)gk5{t3_v;|51Y}_-}L}+D1{nnK()-+-Isq{3;=IjX`hRG3kUJEoV
z3CTRX;)GFzZe_LYJ*ig$8e1N1vT(~ZO;<}j#+y59?<~o<TouoVfUE1A4BVVO_HBJt
z*~1ysk{vLiXyMcQ!LRo(zXs~27!}zo)EBHje{IbipFfLEh422(_WRzs$#L)hzMJXc
zSGIhYOwsK*bE5WY9<JS%lh7l|rQ=q5YW>9x*EKgBv9XkRf73;=;mQo7wdvZInM!}I
zy}C1-A;!C78e8{XH{G;HOLYGS{m4D|?9C~WZB^gb*1ap^nziMD)b2|c=d^E4$X@?D
z!<L7k;N`m<o9C>z-wK=EdMe+&IRDQjBVohjQ{^kgm(J|mk@V$2ER%-#<edj~E&qcq
zRTBfv(y!0U*)mJI;pV^9rzZ*&{=ZXK{pS*+LP`7ofA5YpZRFnW|4e0W>e^FRZm2U(
zmvl?ocWm7S&)wR&tC=R=SUAPQ<ZhE`RPWUt7Zn-$mxy0|(Hkjmeu?4cH;e2yZXe~|
z^Izt1NitluYsZ?WZ#S_y<WB#2=7GA!oZjbG^LK;L;t5})KR4&=%Dta!|KrxH+48Zx
z-}TRR@1Gp^=kBNY_sjMF=PmrPGim9&ZMJ;<H`>3O7P?mZrgiPOzuEL~4u?(e?9iy5
z)ps;plNa$Os8l@PI^|N?yUqXRT<dOpsbtdh#p(JMG3FO*f)o$znRsd9k}2V)LQ8hc
zF5<n_VJHv~y?XoRx{QQrOE*c_8)|87c{I!YqQmwD-wF*(p9vf}c`A}SWNWs4tHQlR
zwY@hOxf`UH{*}DC(`@l3ufPdy+Ood!`|RWYMpb^io%Qv0)>h-6HpO<V7dYj2%#AO$
z-oNeEUiJkqWM6J#IPi3iOpWYXA@7dZZ0DnPpMtoi?K=LkSmUUPdIR%v<80H3C5z40
zX20FXabem)BRA>Ar?<S*Kl$DAe~m=G;mjjbYZ>mWn0GEoGw!jMeKPMfUNMVBj#0a`
zSk(5I$!+|3@Jub|f%$fMn_8~FtYEM<ay^_-SfS!HF)i)Zofg4{fF*LAORrp9W7jKP
zd!?jSEhkyJelw(H$Ns-?4QGAn6rFb#nMpmzJn!$X@0?usY~P!Jll2`x`D+jJ1RAFp
ze}8?2O<bGPaZa<tYu?@6-oO8>a1PL5`R#V=Tk9XW^U}8_YFzTj-YVUFwBuCBU9F3|
zo}LyxTFtOE{=)2Ej#IAlRxLiW;-UM+d;in!|7G9zonLN!eRcZBxBHF%U&~)N?~Oli
z()R1Me?7l{Rqs!`eD><oyHA^$%PY57tTw+mBcZlZ@al!Gxp%&@MtzxT|L*s%MOR<$
zobT^n@jSBaXoVo};li`Yl6U7{R1z|sdemQkG2<f>8Rtbxtl{Fj0;85pt5mxBK(~KM
z>ZC4_c~f|H&frPjYqS1TnzZh3)!ge+ix06`vPLZwtWG@}zBSDDQmffkX=~4ChktFp
zmE0sa+gCa2Q;*|SUAEQtLXQ<YMZMQPc6{bRC8325_NENKR&s{#x}p}7$#p4S_EGeY
zt8czOe?DK9>G;z_%pZ=QXIsDMKI??badxphM;N7ZQ=0iNXl$J2vM}YE*6LU9T69X=
zR~b!`JZZ8sK~*fBv-0?<FHt(@rF0l-y3KgPv{*PDbFMk4ScPmmt^Z?NzWN)T=G6VH
z1~p$^w6A}bF*DZi?&;4T`cGf`leZ<hXXEwSDQTRu-p${0#cg+3#7XXDX)9JAy4X}L
zxJpHH%SO*XPSTpP3a5A)R$Qo>xQEGM-Ey&IiW^O~ZS38D6SU;8#?;yT%hWW**41&h
z-@Eahw3a@<+xWa6W6qZB$x^w|ENx-m&R+d@bbHu)pZLhHp>}pzr%M^9^uB1fzm>mJ
zMWD1U;7V+o=dCCwlN8Y%j}<u^bZS{&nCwhwv;j?U?wFXNqsY?Wss8pqyW{%<(zdTI
zvfr)#|DpNo@4c1BW798xDBAq{&sF`GC)b}Z;gNMd|2{_1diU;{hBrNeqWt`KmgY`M
zZqH>D%Ui;o>Aua>LFd4s4ZBj^`4}RX{#tqRMyunAH&QK}deSLhgE;0&8VgO4Jp6K-
zlyv6NC&KluY;1-K(LE;F+a#`U^SKvn;xhf|Ns0EtRTq^rmqsZbus#;C%44VYhE393
zmv()+xX7XU!Jhm-x~~uDNIg!xu%?|gu6v^1WsR1!jL?7oZ~We_c6{R(FOe8&2G0hS
z4&F<1>p08%KYZ8|k`kKc89FsoYo-3HJlWTmkEc{!$drDaoNRW@|Fcp3qWWWJPDh+S
zeMZu{_?OT0t#iY+UzrqDdu8daw9vFvw=|ckik98&_dSyG8TK@&G}!XXZ{YmSk$L8r
zmhtJFACXeRj_mmg4mxY{YqLckKAQT+@z&fWMoY@&&F`;}&3VJOR=kuU|M>r=MO}T;
zljE1x9r`_g>0;aBvU!#z+zKHpxt^;Y6Pk9tls6#9@4NxS1nFtfN~>7SzV}3}5J|o7
zV;#ztoth;0bXCd*8wM{?H8t7A1{IfMb1p>%K2u}3eA=|x^=OKN?Y{5#|1PWJPdT+f
zGU-q{kF(L7%{$JETbp-3pLg3;^US4o^`&3F{Ndd-S+Ba|;3V@F!PGTVPM^`4l4?Es
zqMyecrP!&dmlHjc3fE-E#^<eNytb>(xcl|e9IF^Vwuzs5CrCa{*Es*>vH0`rbKUdG
z0v^viVOh%H>F39<BG)nP@B_EoE3X?aFiR5kn>e#tw)w1{uw}XA&Ni{8j9qc_PbX(7
zT&vhNY2Jpb(XO2RDMy`VNr#DPO*)``@9ZI`bFTLm_JqFF=KPR$#cQcRPVl_kXE)5U
z?=yaTayk9-^Z63>pBMj~EdSo$Z`MUaFK(~?$hWV<J|?|0Q(R@e<JGsh+9IKik-MKX
zOzqKQ$oQ=-BF4PBK8yW|Q;(|16ou5FB}>>szwmM`j5(&H8hh+TowyNi_@&ibCN1!3
zuD&9?M8tlV>aG*ddt6>$IlRa>Tw$kZm(p7sUIVd!o|dGe(yR5fUFZ7tGI@U1YuVAf
zy2I>pg?DknG<Tz)2Hdvu+c#=05Y1b>y6Cd!ulsJhe(wGC{@=l;|KI(+`~LsIpZCl+
ztN;J<%8Gyf93L~a@3ZTtS%u#_D61;Qa4wp`s*UwWL&As7gQxZu`3pC(aVqy;HLQ^7
zIp=QflC+a&<Etq~Yp(X)?CZK4s`=8$;Mm?gdu~?EjR9uQ0$$zJRw{9}K6=_lwaB}_
zM4;2v^^53Evlo2IvgMha?5`i+Ox=EAr?A>$<?dq=o`wEzre8~Vsx@Jf(pB5br`yjn
zXNg1|b?uwp7U9Qc<h^sN;xaA$I(C(7Ti5AtyvC3nyMO0ao%$yAgHsYsP8DYfR46hx
z)NBv0mYKbrOPc5Ks`lM4KkA6wn&joRkbUtgjy0=}T@@1MGK|^F>1r}#Np?n_2ru8H
zc{%dG0yeFh%e2Cq!#7|j&t*Ty__wpCy)^XAyBL3RqQ~@?ksD7HsLtlUXLzQaVbfFp
z=RZ^bM*b=@*I##W@2MmEcJJM_H+DU9wC~#7>%lEg&N&=7{mwM@*w;S^Gw&VFPiCDl
zXJ?!J*RI=XN4h)hRaeGF@V#(M7E!eAHZGd4R8n;NkX&Yj9p{p57EBdU+ujB^m~$v@
zc{8_W?SYzCY|DZKLd7&+O4%juo^elTFXKv`b1ODWPS==FF!lX!C$0P6-y4Kx-t^zH
z>U(iK$3vqF!ApZ03-70IXUIskSTI-I@7-IOwh-x;&#v3Owu{J-x@64!IOE>0<#Vqc
z`*6}MXL;$r^p(Qzq*!Z3=LVZ3KJETGRV;a`Xz_fJcM~JtNSaPB-%`J8N<rij(RVDX
z7(Cmu<dS8ZC-Z3Y<gJR|!Z+1v>1EsL$9GwJTvjQ`(72~su_QHOm*)At;wkMBM{jW*
zarpdh@x%R*V(Z>DoVMo95ql!}*tR~o@oa>f={&(^S9_+Tr+QKsCRW*m8BNW)y6)^z
zl^Nfvw(j6#2>y{RlpPyi2hQ2d-g-+<O<fVX=*}ep+ru2oqJQ7rUc9blCrd)_g|c!E
z{+Fj?7$P;6%l2_i-ZDjPZvGcn%e5=t3YQAnDgQc^%G1NG_(jt7nKJ{IG|QxzV>S1e
z|B?Cid#=BelHA&@=C=F2@)$2<Zq@iBuArJ1vj6)3uup6LcmIDHVppqXC!eC;xt?u_
zb>Zp7(`@z$#w+f-bf_#=wAI>_ku%5Fbi%469pe|<iUn##!*h0N8F1@qUvW*j7Er=j
zJh4@7$_b}_AAyz-J?32V>mPy@(^M}z2Xt-;SpRB{4u7PFlg9a0m1!DU40#`)2JD^?
zad_qNJ59@!9&8PYj9g#xMdyCG;?zlo6<RIloudr)PAHJ@s@(m~Z`Xk<$ye)=s@e~J
zsufwcN<$>m>EgGQeA8aF%y|`e?VO?6j{u*qcEP)UJju9M6!pCA`?Tu))kZft%6kj0
zcSPsUGw1)RY`r~KkYNqeA#daIDJ#9?`PXZjxu5b|<D|54mLQ|ZJ;R9O2`rmUnpqXN
zk8U}<H*bGm<Gs(FHJAK&m$4txcCmix+|S1Ov1{jwxNCp+?X=0;ZT+Nq>&N4!Oo>|#
zz1sM6`}<FPlO5M=u=erj=nVRjR<+Y^-wMkYMr`L8dfdGCpZueyt$!6M1J6`l{^4Z2
zNJEIShUKZxS4AGW3LkU5#XQ+`lXpj|3y+wV*y24c8MF2+I%r$VzjFPHS2G1>oVonS
ze6C@4F+)(-Qj?eN^Uv|_Q+&C#=DvC|Q<X=lns@!A%yw<nWXZt)uB+Fdm^pXC%$J6D
zr(3?ydt3k9`Z~Y-z8@FoFP-0h`2S1sAN8N^%Kxl+Fmb!e#qcK&;|iqTm6_h1>9yN9
zWNX;=J35REy4%<lUbCe<_Dero)D>`Vrs|q)2Tmki`CfZT&T*;^AOBX<i(8aAlYj2h
zU*euJDNpKA*!Dj0Lu;Jo9evq)qA~Wp=$3<-Qij}nFJ(2pPS?xX*ik9ew5X_rr&z42
zWkrjQn8w<jl7-#I8iLI0Hcpx%n57cq@j!oa*D?-M$!V=}4L(|T#jc!njM7`&-nD3&
z_T5{1V`pwvUp42~6m$2_*A}bG%hdjOnV6kXkw4eROyj)<1OI`o>S66)1CKomn8@Zm
zd*}S;-qXv!^~@5eT;nji+ha}8b3I16r(U0}s+RYPp18)6I!oBAxvlF&>>`oO%mXgm
zvkqRmtbQf>dVZ+B{Pee9=N|a^A!p7PzZL)e?iNqES9GN*CgJkbM@fg@nXNx~UxMXS
zYT_CZ!-&j8hxJKIla}7;Fz)ea3q6y2XXm=e>$~>-h+<w-`j<`Y{8WV!QyD%z|M`wx
z5t{uc6LytOOJ_*=EA&&3;pp@$@>lf2tePg&MI<kMVdAj=R#*A-OIIf8ByF%-Y_`^+
zEwt5|dFOU*&7yVsD^E5|VTs`1HQQ;`?Z1u}LW@+yz7&Ztxc}dyT-j%{tdcMHzeF|v
zo95s9_srb;<Klc-&gZv3{VMqJ;X=ke<F$PccOUE8w@Ay%eSPwVlF+Sh%XQo)Uo?5m
z_T(bN1O{)WnwULajXdvmB^tgn65%ycWJp=~+~~}X_$?fg<x3bIpA9%`vt!MK6~c3(
zdf!j{VDltZPqBQX)W^$$?OrEMo-7le^pHu=FY`pLoY1sY+oY0<c0FQUwYYxPoSi+s
zw$F^Fd76Gc<#?j6g!{=xqa9iXH?!XLh;LV|4CAvAOD|cWw7UIQ^R6$?7PZJ28*pUK
zbo{D4B{aUR<$LC(&%JiPe|=TqpD(km&F}S}luaK$zPmoZyozzhhiz7Erf-+<)LmE@
zBIBtt?OMXZlnv)TFZDb2#767>jOA+=dOej-YV}(YvQRAE@c6H$7Csr>XcJpL58j<y
z4ov*#r7~@vWX^}hOHEd?#+T;a=#o>uue$i+_iwkqy$E==QqFi!vPk&SBSoS)Ws)a1
zbR6uOdg;qlx!<vVYz#JilX9Y)*Ob<-+{za5`j@Zv&M8V$ZnaA4&!5Zav6N@(E3@_G
zmlFkkSnc5{Vt?AN-?=(QY?j=L<Ln=FrrbKx_-1{{HD22QS;?q|(5ZWO_e9Dq35n{x
z{lJ||x*^3#<6212c6El#s@2BA4C`;tFZ*=r)7mEoL*$qecZbMD-|08ow*2nD(+8Do
ze$W1`-kE>GLnr20y5e<{=BI{JzKAv`c|9%V33Dw8)qk(KXT8?hgRu-&brpy6Ux_Q#
z1aPXZ@z};0WU%h=?*%G}QrACl>#(l7U9hF;*-Vdysa;ofZ)yuPDaN?6MKLUJX)p9u
z2wmw|wMz2)?1h_b1ZHU4KArk(*Q1yciREEwQv(-QbBMfpr?U5Uh}PSqs!J|i@Uma1
z?D~;WG{!Bej$>iNZWS@Zh=!61h6Igek>=_z=Jza{<-6r6@6NOZhm0nruPL7>zpaho
zSM;^de(x)nm(SZd@sn@=PPS8FbC+E2vJvZaJ-+_6%OQguCzmoQq?)o%Hd8jYU8dSF
z@py!z=3cjTF2OD35;}L<!hU^FwEw=-CQafJ)5d_Kg;AdA?S+nRi=`%KvZ`?ISrTI6
z{o>QP%6&@&55>LjO_C4jlJSobTXytA+Q;3BDryHqG(8I=8Lp{Z+qLfrIMXf@U4L=v
z3#OF<)0dxraW6uXL(pft-m#)pcHY%TZzX?zzQw`*BbW5io;X*Q#l9=1Us@8(aWd`Y
zdYAZxIZXl8hGASi%Y)YOY-C{A7kFarHhymF{J&8_T8mg#DI7>uYt&jR(wXWq{k@p|
zM+x(;lnlOeZ<~KBw?4kS|K7g`kFD6eAHM(ktKq|qSDc$CFhAI{^pNFK#_G9yuU`s}
zDhb`0-ZgcqDm#OR@rz0P)6~|ju=l7{-E?TxuLQv@O3&1$GF`d#xptT8<Fi58XBI9A
z>$@+Q@Y60@(yP#(_u+3HHRh9(92UvN>?tzQ%~;O&_=xe;S}S|KWl|?^E=-x7H~qZi
zLXCiVTOQ7id(m<sAZKZ@di0u2!7qQWG3eAtm*zCO6u(t(?W)}CA!5014U^frdki|n
zw^Z+zt`mKBuS@sR*7S4l9zWZDX!gyN{7d)dZQ0Jw|Bcb*RR8BkuEi^Dj%A$i`+e=5
zr0K1b&P(6DIi}%rTeU%B|L^S)3zH4b{k8u7T6=;O+m-3=m3N;0(cL@iTe141t_CC7
z<}J@(YfpRh^hopRcW?Io-NrY0+3(JSMz$u(D|KJrGd%MuxpTGJ#U*Ch8hN%JroY~J
zI`D*jp1QH2bnWZzb*2ZRuIERcTpD8H?so5*p0ie}m{)C}-02fi3u`kN3N|wb)w%37
zjQ#iNP~>fP2j<!@m)D3!U0KVpps$C`{*p&`SoB{9ZAtA=#T5(=&oaflj_zLi!s~x&
zYD&P@boDLEE2bySTlwOub?d8{v*qT?*~}=rFQuts|2zKOe3cTpS7oeom!I^RbZ+9Z
z<2L(Q7o1AS{PpUR<<<30#u?={ceJH+s-D}tW@9+BAX;l(istcYdXKNp3qN@6#HA0N
z%Qz%gYaRTmdvYNcKX;S$727p?wumKM)ZO}7srAy-DI6;2bq`J!=(_Uy$(ePh+!jta
z)vEF~gt1L)yIT6KqgS*7r%Vf-_->WPs}`rAL;g`|;$qW;FMYOLE#aSVYvQf1UitDO
zOj`vadU-D`H)C#AcyohcYy5Gx%m*rQg=(wAzJ<J*x%uAF^L5&LEOU69bN>C2eqX=s
z{`Px$aZl#GJow<u1D)rql0Wnv?3~hN<5v6X^F?2=Q(f)WKK<7;OWr+=Q4BcrLPMNk
zPhPydtj2Q>sXrdSH@v=hH>4|EcrK?w=IgSMTQwydG39cM1%IC}IjE$ma`Es~A2~^F
zzdw<`cU5jo48Om88fVeDmJ{9^7cf1Z(8T6Zq{-;h9iiA+y7u+yRp3|+(_Pwbv~;SK
z8iPm9zVs_SA2%oLVpVvXSTl(^pvH6mttUyW3Ssk3OU!(6K}S(ka>wPbhI7qs6#D5p
z?d|*Kw<Nsg>bhCdVl(xkYD)cfH9K)c86K<L$(Q<m!k)cW_V({S9IGw6_UmotcKe`=
zxqA~fx!E~XyZO%F;CcI<)SXYeWDhGP?moW8z)HQG?QFu^GkjmHD;LW@+Sa~XuWRq>
zPpT}NO^#`;m0a>KK)abupJjnM`^r{#<8qDI<6&VDUQ7`aBU)dJF4vJ->vMf#earl1
zvYkf!?K$gW*w*fIa_#d~Zf`l<A$R$H%Ec)yDXMo}*S!|q=M_0;;ybT$vy<`jA`F9S
zC-T}(S+f6^oi@*w1G5+Iy3Bip+o?iBoqg*G+t<%bR(Vdy*7!2*x$4xfQf%C+4ey+G
zO<sO>`StkdKbuVVWmSB)Ut3@N>%-gEhu{AzTKw(wzXxC5UVM3b@n)UY6MOUa3$Od;
zt@tV8uB<(a3h%lyp8KgCN1QZ0C9NY@ta_*T<!vzcJU;Wh2qS;<-D{2+zin75-QnW?
za-uWuwij|<z78u7UgE!)$M?%=#gmgNd#&zkMztt$lvvN}D-mK|5_HrnsCa&srR0%@
z<Kgr4Uq@Y^9+us(b>00fTF+{OR-Q9X7xH-=v*d{f?~}Lfcb|PdS94oR^cAmKHUo!>
z<~`ky6HILNPR|d?6<S*{U9a_Px9N|GwpwWqCg1A}GZ9u(*qv2-bL|Gl^Zvo!L0p#~
z>oGsq;hy?(<C-EbXUT?$8m&z=Ph<~8RJ-wh&^_0>EB^g;H?H50IHhA=6^nXKeZS;(
zkm!q5mt<-}8(L3^ESx9L;=p?_sATEY@D*y@jC*pp*Uf5fQStb&<f;3TO-xI&e_SYg
zHBV)edpU!q<4m>RUT;D(=FNF@?odQy(cF`tg%|i7I8+fE-8`*I)AP0P)D`K!PDLmF
z+$S7XBlcyr@EoD1TuW<LoU2g#G1V)uhj-Fnmd9H<zpP$h!rF0=lQH=2H}R!^r+GN~
zahK&!4~bk-p)9;<;S8ZywO^}swDw&x$j-m{`g~oD#Sh2pIXdrZvwykW|8KwN^<Vz{
zS^rPu{i}Wc>*0eh2l$$$XKAdlY1J_*me9}Cx@!1GC+1@)gVjPQmabz_v0*)J7u1#B
zn}14N-clOkq4V~$-_o~kCuO|v-+$m9y6$z_$36MiPiAa+HPcCD+O_1s__*R5yVx2R
zot_af(XmZypMq4=7lt)g<}h3fi+KIZD4Tb~wXd^7P2*-<JHl$?klB3X<@`@+MK^bN
zRWJCde5xb$xaGlR^Y%|{o#df&E>g_Pwmam8^YfKqju&$eR-ep>w>7c7z>%os`9o**
zVxL<D-9-z&{+(jdbZJ3jO57ZA#*UJET45UJ9iIQZ^u?g|$G0uF?XB(a+W*`red<}a
za8o>YPMOnFqYb~crz<a7f2ud=MpstPS1b87i!O)E+h*CYMmoebd}`hl6Xq(#MbUYj
zw@v*gi|tSsxuiXbYpP1~1(O)T@F|yl8rV|jDP$IQ>~j)a)f_6aMPU1asqsfdCLOw~
zX*ADS(KgrF>T{T<^$|-^q0-vw)1eC=EbTfb|5)Y5C9#!D&E%zYg0wf>DDe_2v^sNr
zQ*G7r*13PP(lr)@uK9B83B#?OOY|4b^*&W2Bc1X5(z~~><dpIjKl~)QzAN8m-=Y{N
zhVN~;-;ces|9|w!r~5~bJ&v(mvy=Vp+rP)8OaFz*?ha;nay{TpsmO9)m9U=+t=P10
zdNCYbCtX~Bn1AI~-?fKdrLdiC+OcQx<`R#tzS}!Fc<Yv!hVI~9RZ_us_wsuC9iCz-
zCSKpOy>+J;>AX5!eAvoh`xoyilMiXHx4xWwWt0EXO?xhUugwxnyR6tC_5Jqxol&6f
z%mHxAi0x_+SM9o@=SAn|81{GsJ~WbC(SG+?>AgCR2K};x7zTxZD-ZUssCS$<KV^cm
z`Zg=JBhrBl3X>UQn=bQPnJQ=5WPebex!j6DB}{9o(1Tg;7Ku)~bY*i_iS^qLX)4R-
z&UvAItU}^W#a+9<?_RfyGk@6QJjLPrhqZrpe`lC8cmB8a`Y~)(nbXT}T|dif-(#71
z!oqe=Q6PiJz3_`Wuiso8tS=~C65SfPnse)-k0pU0Q!gyi&Jgv!#@m&s-F#ngTXxv7
z?~y#~_Hv#G4tXpgBH@v_yR6}jgvJXtk$z^6uH3TjmV;BHyhZMDc`Q|6*C|R{p586O
zB^vVb*@mK^in*))=G~pz#MqPF*Htw=&v3_vPy4j2o9z}F7R~L*zrAkXxm|O^j~kp^
zdGOuaF!2}brvxl%w_2Si<@!=@!*1VoGA$de+MXIczV~5i?Wffb{`fb1o)vA<@zj9D
zVYk4}UGtxND}Ksqv3%~JSo-|e(k`7t0lZ98Tbs&WY874T<cNCW*XvYw``?rWm)g^l
zxir=!FE}Wy)qCjmxy7@Oey;m_aQFWE7uh#GJKS+ZX;qKJn#*r?DHVxt{B+9G`QfGs
zb{$ic!i;!r8RmRfJ1BBMq)s?#&9&Y`u>Q;0Rc6ZfrB3Lm&NS0ZVAy|p9_wY@E@c(@
zHI1_Z&&<@jeA;t@<DThff~6!>`nCU_XR}W_UfC>fD5-I8=`k5|oy*!KufG&b6rU5k
z^}geCA<gc|p`9Cf)`~0>p7iqR)gNY4zKA;Rz4rZE%ips?v8`t{&MvKx>0#*EzU)>_
z$GMKslF+{!N_MR^ir5t%vD85;vw3xJ#npeSwj6x&R%X_v3(pq)nl^`{>)C?$!H0Z4
zsvlOr;c`n;!%?E+5F=-D;P3TM7D*gs6|>{G)pF>2@Xjh7hVI{fmHx($928yEr98X0
zY*`g+-8$|2kE=W~h2G7fJEzWLtKafJqghY)u+sf9-~Ps*GeZ*gMQXi$*TVm5#hap*
z0Hz|f-Kt$ndHpzDkLsjb#(Jn+Tl6b7Tf#Tq>Mf`2;YH7u%#PQ6EW<EuKl9>-$SHqg
za#yD+=Jm>ogz8+r(i$|$V@-r8PuSL_>(*{CnZ0|;vNOSVxz=wA*0A+ht@-jQ|5j03
zHTCcP|BMzfN8bK9X*u)S_#Zb6-+cWw|B2+OE6=?gnAF8)7DTL?YuXUD-BRZ0Yr#XM
zwW^Sg=E|ia$*of9Up9O0%2CYxAjgpOPI#Bo$%I^^_LhyF?^86-&fCve64;fzL?o)v
zRrr|?n+cmMSIjQ2Wh<9Towv|;%CEVvFYSNSJN)rgulHXc^&UB;%;vpa>xJm%3l}na
zM2<u?T>ba{|IM#O{&l|`<7Id2yphle3tJJg^6a5`fr}?^5tDlT?}6;ssq3tHuF3xD
z+BD~@Zr;t^rx(r$d&XXQbK@@Wq_aET8!b^>zH`r`y2JWsE<5{Gu(qCb614D8u5%G#
zRCwgoZt}WMG4uJ!>t4bhdRzH+%1=KpHt|Ny)8!7yZJS)8VrEatI{jmJ$EvO^S{2<#
zA`Exboe!N}xp$3V^c?Q#isxL*ZtvT~wAmtz)6$jQ?DaDSSD}k+`evoqcqd&fiTP=J
zO=#Dm6@e1*7x@+(s{C2>XYTB@e>$>%zFzMB|HH&C{Ve<Y60!TCd#bM$2KsHDwl`C=
zm8Ha~(lEcm-TH;sLPa~%9i`I?4)d?B=w=Ww*SUM8dh4p!;mo|8Eq7JA>;4`*CV$<e
zd1<)zr3<M6k7x4jfAn|So|zNOHlA7TWIRcAam<U2FY4L^4I>zwBL5r$&C*1H$8wvy
zT!iG>cOR&HrlZcR5xDK`k)R(8D_lKHqwJJcvsRaQoVA<YYQo^*Iz{wiIK!fQ5vv!x
z_!UvE^d%u-t9Qlow^K7Z86`RuXGSi5!8>z~@zNz{f^~OYdZ888su}qEVDmluAMy-7
z@$>fW-BHBe-FGnN8rzefFVv<5D$aViZ=TPUbU`KEEv37UuhBYF75cWWTE+L$T<N;W
zOzf-cRi_8O4UfE}TV*Z6J%wLb+x6Hy)}2$i8z%e7R8_=z3UI5kvYJg_>mX2Q`kCQV
zu72#wj=4%RZNiFHN6!)CvYZ}n5ITWX(k&+HrM1_sGI5O_iBn2PZ51u6+Gox3ayB@&
zmT9Kw`54}BAGJfmT6K4;=IfhJUt<)ZaaQUJhvHeDuzgY?`vn(CWVS|ot+_q%-kj|W
zE|pwA^d7%wZNI<grM-{%*YnS>rSE;VegEW-xeT)V*DbvC^F`WK)exbEZ)M;AUhPy6
z>YnkN<@7s-^#(g6x%wSM1H(#r%y#!Q@djkAjk7lJ_<tphce|0pdKIsCU2Yks7ne@=
z?Kf9`xi(1Ck?(8DHCNlnDdAy}8efm4OX#XM2b}t35LM23{p-`KQVDBcn=gIwXwoB{
zEr<3deSI1Iho!-=?%5~4vgIerw=X)LvNfPmYTl_=JM1!cbXsVA61l1tv8Aa`-)HHh
z%2j(sHx;O=F06jBWD&2_tch3Fi0T>~+bY~rcj8NL+t0Me!toaSzgq9E|Ipq)QD~P}
z>cYaAWok}O801Q;HJtsl+*WSdHI?gi8N<6W(b&Cu-36(umb9~HM)~rupQ_yQ>D!-K
zE`p8~-px;?>N0<DKB^1wI&@Okme=sB#8YOm<d`%1tzn8`U8|KOCrEWL++435G^tI%
zbaUvPV510yiDKzTq9$@XOD#&ca;mzcSZH<a0t0K+qqdhL%@w_}xKch&2(su8xq3!M
zD*wv-z{7d3oL`naKlSWUDBJ6+f=(iT9apYAa<*4=xku2>g){DX_4ce^P<?RKjomY@
zzmNz!zTr}U<-~CNufLAxeO6z7dHQmO4R_D~*ya{){W*8nmJGMmbB)7f_XM%5RobZW
z_)Mc$cK+tmFFsscwAyjODnTojc`NRR^Qta!UOCHy<H4@aE9~Ya&YsLajV&=sW~F}n
zrT{<d$`v9vO+vio^GtpP$nB5QT6&tNRj%P-!31y5G+xs>apCpZvGD=mjHjHn=0e(n
z`}+b;fhS4z`|Wi)Se6EKx-xfdcz*g_kA%+M9%hCU5t^G^FU;jGdaY>I?Z-AZaQ2&p
zo^7tzBEP6?4ASFvovGvWoO9CayfR7FQ$>fBTD$s=s)~9XUtfIKXs=Aenc$1|doA_|
z)C*YrKd$?2zkNlRHTU^{%j#<{_M{w7I3i=RSd6#wV$9xi8ejgdd%5<_`WK;Gcc*$C
zelf>GXRqf!G5hZ;?`Hd5vD6KiKH=QEmW>{5p<E3X3$<r8?-riNA+nBLEd7b{thNm{
zo*i7NClnM_CM5<|Pge-s>2OKA?a)y#;V9R2#ZpB^M}!t}oK(zl;Spn8e5q}UD;L8=
z|5kGrE|!=XIiiKOiXD%bxV$--%3Q)0nh0K1v`l&Jdb7iXZ>8bP_^%lWG2VAqpJd!F
zxhC||W=RQ_>9;KpYH}WU_m;0sylT;AqmT&Aa_Oix*E37KOs`x!cs{bDdfCP5PoA&e
zu>QZ5pS*j0#q@9AK3)@Xm~wjh%+9P=fh+5bZQX3NcP0vQORqk(;rrEX3-Y9131s*t
zoqf7-o1Npz%ZvK<9BND3#*pLMFP`hp*1z7m|K8uf=ltz{ss26rRF?PFmNTm*CFPR$
z+-G9=xwEdW_QCp3%nc8VC#7|TZNI5HYiCUTT#p^=^w)xBMB{69mu9rIw1>pa3iWHs
z)Jj!sO;q%nnRTyjNBWhHgSR-jc*PiWbJR~eGF;iIq4L(~0&_q}`J>F_t)JEuNeW*w
zej@y9`Mz!Ik`t;{ubVLC)EA450V$k{pQi*Qh)H>Q9-V$Rzw+zv?sLt@uHD{$eEa-(
zUiQ7WlB?zit}9v@bFJ^{51V)~@z?tTpB=ntttqx;&s?^z+%ch(BCa|fvJU;ozW$a@
zVUpLEN?S?a)8`cOpSK=8Ap76v)Nkc{|5=hOVT)9!C^ol#x7wOhW6vwtT+rF!ouTT*
z=It-ONMXy8Z&~UW!VamuYhf@sdX!V*B11w{lbMv|l_0H4X@ZM{oL;a_StnrSKWokA
zEi1ga_q<r+v)<KjRf<wPpYf>(QNMQ|v(Md%>F2uCw)E%OqMQQp2PUUIy>_ISEUmw{
zbp5Gat`SDwr#Rn=Zj3Ve!4NY|O7XUF#D^z~Z>igxOWnSC{a*B2?>&pxy2pl{*svl|
z%Vl2csnnwn7A{*Kx7Kli*&l)JrLX0r7niKt<*Uv-m05RMpy{>AVo4tx@++b*KJNc#
zRQ3OE{=LeMx~RN}Ev=mgm9{5NI5+9HM(wjz-}XCoL^PU29KLvJUA^vM*hs+oeb45_
zKkljf@kwB1$l*Ypry6X_H_VEtT=ra;#qikqq<6y2rFSbsJ|<58#p$_yiSo?VN#9nO
zU*js@dnjmDq(<jErENOrjxx=@B6FqB&ZT7H59WrKPgU}li<e5;n7#W^`qo(Q_4Ggc
z9>4#3=9XE*1EthA?KOv;8TqChj9hbU=M<GXc7}=Bzql0S8Lt`Y=w|7qef9jr^WgoK
zLrzNrBW{@ptlM=s;p8Pr?ueJ0-tXdfFqWUd(6LV`CqyDX^gvhWwznsEwRO)pEtGW$
z6bffJQ9k>{YKaR`8eWGKud_|cy=cX7;E%+IN{$I8U6WUfF|6&=fAgT#!7?ghQtnZE
z-FLz{MW>J7<NU06LP9)1=-MgI(4BAcw@T<bf9%rrys|*<s!BiGBf+hyZr1I0vnO5I
zyCr9bwReO>@s8U|ikI7XG3<*E7VzH3GNGF5z`o3<^X=cA)tmjDZ{7Cym;K~UYdov7
zp0sh%->g)wsH_c6(+|G>W?i>8>7H6?Lf<>NMN1a!$=ly|FhC?eB{a|VntBP#OBKgh
zsdm}7F?UqB8Q#C${?(-*v@>*KN+APlbLr{1-mh9iv>b}g`RGp%S`zfOyqkB1*VOi^
z)V~u-*S_Xn0~)9ckMY)=7NoN@%lL`c5ih5cPg5E`x%k<yIp5rrwW{lmXLG6hjfKoC
zo@X4(FIkoHyt%0OGr@JQ;i=UrZ`)k;_`mQJSFUkYDLJ<w%|&bK%Z~;-PC6yuI$3%m
zVU0j1Yw{e86#-irPi*$R{8w)S`yA%!()*sKx9;3_`~}bP^P6N2ILa_LykBhdaOyiA
zJFSz=tX)bQi(kI`8^z}Bz2qzB?`@ZZ4E;Q|n>=<qyRuC0y~!Sq#EjbV&SZvy)69yB
zMk&jmNhPjSiVNU+G?^>f@36X&V$h|l@sTT!Us^qN*#`x;t_cDw?{Z%CwZ7!acw&~|
zb{mb*8FRBrZM>3o8I1X#3JaO&WEcmgxlaDKcaLgw+tkY~b;$uoYaU%mPBHY_u`27D
zgY7kI1MkCOxiweM1@?)tB{wfHSbI3|TiN?_Lc4Bvtlu)#{<Bo*b3SQnJ#XH3v*n5<
z+02_?p1!>Je)aRJ-*ybs%Qqc={5gg(@#wOJ2i#Kb@;PW$t`_&N4ZWw>S}n^gs}}oK
z=+U<pg;Vcr4laAuqqtC`cW*}5gnuqMCW6OGDi*QiHaiGvDY31-z2}<02KV{sR~Nls
zT}fZas1WwaMq{UTsK)*4O5aOsGr=8@uGPnCLn^8bLyf%E51QzzzdboS>(8H^bsft(
zIu1s8O+NJRh2LHIHT$~$bzWEYi&As?X_;7Xn3wxSkX^~whw}4!Up`sL{!8P%WP@9H
zWW<W>(4&bfHvd=`#Q$WnR6*U}o88mxcUS%VY$#u0B5nV!LNA+J@WbVar>&Q-i{><Y
zIQ6?;*bA?vSCn)@-lleccpnfymD{Q&Ak3;+=qZQv&7@vlzlH@;8`&AWCK)njo?rQT
z`ZR{t59|-6os9i%ij{;nH!di&>CNdn^Gqm8aQ4!tAN(967e1M=;33~ZQ=xgnXO!fB
z@;Sd06J%QYvgRen`)M|jFUwn3F=s{nIx*Q{-UAWC#3+rh5}uj<J3jPriaiy2)g{__
zhuis&*Q?EU?%dFwd#Cbhug~v<Oz-b0CVL;Zv94j`?9OlV&^gB-(8;<W`3HkfWok_o
zW5RRC-#NdnxpU&o)=injvMOli<?TMzTjeBDda}jLc=Apd%ndl#CSUev*R$1e(N`s8
zWQDxM4a%QH{(9AF65-;S&0VoqYUS<|0VZOs&7wZrlO!kFPkY%YBQt9vONH^Ijh|NJ
z&+vL`q|8vR?tj|+X!f1>(;KdReR>7doYh{pR3w+}ZdRc<!-Q64+3Q}n)b`eGue@t$
zA=#Vf+W)dW;@Qf|-|72jS+01T?Ah<Sif>nYRQs~&;XMy(8LWgq9Nzq~`gg9?K@;Uj
z(aH;@o2RUru*G|6#q<~2pK6Ym-H(1F|NqI+^Z(!5SN+_)`lsioe;?%L{t4ij+|M{?
z*Q`^F4E=_84{flyX4rMF>zVEs%`krceTQO2(^GplmYny#9ImmeoptIWTPZC)=A-FL
zemWXOtMdvssx)av1-tH0(F=We<*~LsD?`90-zd8P>7<K`IZ7vPT%LDqVTnwrs?luG
zQ(CUU?JS#LEcM<wRVU=_K7XS>pVqg&^4`DBZ}~OhiJ|(O`%9)@m_M=R-dC{#$uzlt
zeS4lqcuJ|LeE(K7M_^-A<?G23dBw@otG1Tw+>Jk+Fg3oabee#}nm@0sc8V~3^X*Qw
z_<PfT_IJKAIgt!CA?~;5U;bwMx_fcVOu5$*b2iNj_{J#ue%3q|haYJgdpvY9cD~4%
z&U0DT^|9>owxuSkn_F%zez@y12SetD8-@QK-rVux==9q9xd#i<l|v-g=uA25!L;P~
z^o@&lPJU#}C0L^P_}XkE@4Kb1KJVIh2Q>axd(`xMjqi<?pkMp!WsYuSl2m%NvSNDq
zmRKhNk>!(5zted+_rjb&<0P?ks|Pa;W=?CkF!#&pb*hnp`7y^GKI)l@Xo<eMUs5~w
zEN}L$hqo9GB_|t+Kb@1BnE2Q)D}K?Y9`@P4tJ%-bxA(W_-)*1F|8Mc`-;>>zZ<!(F
zRH2e4t~4$1VC+vjhGyf2@ZZfQ`pYEOJQDeBmCdGQr5*l4`>xWS9c$lbz6!8;+x*n1
z?)Y3rhQjbG;YGar6&5XVIGOzNqr;;`cY^M@FgI-BTWhba{$thW=66|+=e)YQCDgJ^
z#pDE1qbDr6DSPZ>jkgD@N7o(28Lo4}TpRk8YHl%WF{ifN<2tr_hKY+7tEEdu9k*(h
z%gRSeTHczTZG0&Qqk7LSjBWcDskL!c{Dmz(ic`wgORm3Eozl5Z?;oFPn4!ZeUTX&1
zl&nv@3~b*|8fD$z!ZXt^exBKTS&99hziRDeQJ89J7~!&VmRxb<35otAO}`p^*n+f_
zwp-QB{ndO#dLqc8iq%$jL2F-WUovWK56Z3oV8~#}(|NEef4}cq=TC7frIPL{F7bS_
zE`Ew$*Ot8;cZ2R!uL;pmJCbnp>H9MgiQx840BELT{ick%4ab~)Zyop2Q)gCW*~mR5
znn_`rVB*|_T~||E=l}M5aAUEv(h8sV)BcK1X0&b5+qyGRba~5EjeRE!uO#1daY;0N
z&l{pJ>rC+WN5v;3&O0?1yPb7>+sx|VHTC1$XWRE12c3xXdp}=AMX#{3{YdD}kcRXE
zwU!E*o)@xVznWbq?wMQ3>l+>r{o(x_$(SkJXS?>)R=Kye-u8Pp^?;0=_LSfj)|dsI
z##|n18!bP&sl+;UhMRnUd~fgOgl!qM+mED#S=z5)jX0{2^fEH4j5AfSeU(6htebz3
zL4-=_^>>klCo=^l7gx^;yV<&C8@K1PsjtL&x!r>=m0qmw)~=C0sx0(-YOCDarF|kB
zcQvnT5OtcWdS(*K<~MP{%LNuSZ9kRo&UC>`=T2d1|N4%DtE^04yp7-Yj;l{*hR*-l
z{hY}YCA>KpMYz6aMNgigd-w6RC~byj2iK&H2hvo;ST<)Ym*wiecHJiI>KaB-!`*s!
ze`kMK-2J`#B5%@$j9RNRrk7)CbIx(g>+IXCQq+AxZtK_06PPCzh@KS*&e|>D$Z*d0
zbZqmg(%PNiDo<<mv72|aRRmKmC@k|@9=*Wl-rmnEKG$USHcUF`#5ie3+(SPO1#_L5
z%a1;JP}tY?thUkjP^foi+QNf7^9$v)mwM{&T9sy`Amg?{fhX@v<%*KSQG0DKKR%mZ
z&wk+Z^Yh;)rE6HrR_|qd8S4~U6*N6A`uR_xeT|FkBUnD}TGLVH<{8h(vH0_<cdz<>
zX-Qt?Ob%s@xGL&(jZ1V!=!=aV$1nAGZw=&7<-WHfL?JZwWnQOKfaTfUqKi8It<2Y5
zzlmw7Se3@y)eCZ^PU$I@pV}v9JMG<^vL$|#`VMk_$w*;q2#IVte2VADRqHE%X8sd7
z+CA;;StA4Y%W132vkT)rT+<SQ`**gjpSa?fw$|j;=g)E7h?>-@U6RcPYJJ|8?b6Kq
z`R4t!&-2#12=BUGwWG0&=arDrHckhwla@)TnL?*06m5=9SY!U5(f;;7zcn@uQZ*Cr
z<#!(GD!OVY@pFIhbOW9|pZ<xa%o9UdHopjDY1orDKkULB;mXK`LbcAZPIpaIShoG1
z%%*u~wUo}TC5kD_=J%RR`O;ajUsXT*s=2}2g6WC30)w>PdO!c|<0Yyss(rO}N~Bl3
zRVn9nqcx?qoRBE==5XbUxD{tPn{TNMFEit_*V^a8C!f0{$B<)mTuxT#*d~kEok~1;
zCz=|s#Uunj)AZc5%5Aa6*2pXJO9Y*E-B0z|uk`nh{)UysDKA4;<%LGB@JQo!crxd0
zz4U>f3Dfq@S#;^d2Z`%;x7gA(nk&Bd+?p4t{BmPp9{cI!l2dCgg<lc-W39#>_>w<t
z?j7y=xnV30<q|qC)=djbe=X>AI(nkj0Wafq6TL*!GQ~@E+pjOG*z4DrvqS9YdNtnn
zC7jx|2OjxNk*fT3*2%N~<sR80TU8de4_lZnKJqgEFs&(W_3j;I^V1J_`T8I3xzfkN
z@WDd-Y{0^jyvs>COe?EqUYI&hiDkj!qD^`0V()$}QQVn$(K}@SwX5MT^Io%Uyl0y+
z|DoOb)}!xi-!V9x6J4nB_;UAU?f);n-;><?^_;7pU4$${B**2)f$K8(`-@z4oz5nt
zE?o8Q?fO|K1$$dg-N|c~&lD~TahXtDx#nVMv;0cs$7vsHE|zU!<4z6UC*`%`cTb|G
z!~<*Zvol){`if<}oT|rJP_*28DbIo2%H+an(^s##yW!ebRdDNM^%^r}Ua@IGl4sZp
zDu4NKY<$Svy4XtCa=n?<uCv*{-tLRE+3x01rT*=Oz=L1Ur`}n;x-iW|rQ_7j#s}hm
zJB!^o)<{Zj3|cwmNW#VQzp8)d1|7-H+@N<Us+8yc7m2^2PxgGj=C?QATK>$g<?_O{
zuJ51Rd)RjH<`(0H{}!CSb?e_Nn|Qa@meotvOSm765m_-O`0;A*<pCS-EL`95WQVL#
z#q|XXw`4x}!nrE4#bsT_^ov(NYrQmbSyHQaDJQI9<(J)BzcS9fwdS^+ww)=(_`O(v
z>nWqo`6qm*xP;`r<l~>hwDM}UXKwl`*Vji{1dqB-^YFX*V0Y^)ud-8G>1SWuENL&C
z>Z;4yxQFkt!t?TDEB;RLdidLJMt<S{KK}^O%+Qb1#C8d;xb-1(La}JH%Zl<v=K>Ws
z#(BGi&2A0}oOkK=4r6bQ0~6jP_Dg;``RAmkQOpy5zSQ5_AIq>DUG>-H{&#Ctfh#6g
zyQXQ&-Yd=hk*FdjU3Wr4y(F|(`AYlRWeeUJM0~matAee;B;sm&K;Ob^zL(f{eqLF6
zO|fAWgLk?R!w%Usn$DWpck=JoE-PL8`g%OLCfkv*LOon5Jjp_4&97#!9X*~aw=A}5
z*z<S7m$lb8{{Ct{{jR5GgGIWN27{5mt=YW~hhK3hYNnXJT9xdu@_Z}f;!_;?g50iE
ze7+46#g_bP%YU7_b6%W?<@udY_x=7Ge-9K>m*@JhncVTyHeVdJV#@JF5xw?uY8o%K
ztyYVE5nY@!=W6#otJ%A5Tbr|PeNg&s)u;4F>+-U1ERT43XY0`yYf~9czBrURCnZ8v
zWQVSLQeC@fQt<+bU%MYyEwEId@@LvS-ZhgV)(gd!L_ci*k#ls~N3D6q+HYN@J{B`N
z2(>6B>lQsxI$Qnn`M#>yxw^5LB3^T!EdCgIwVqpp_fDwdUZJ3iYm^io$1GXOz^(Dd
znRCfKPAMhTYU}AwYE#+nZfrayqPX5_^4(*{cdwr`(aJDWbVhQ7)2emxtPYd@O)qwh
z^!Xib@B9CAJyXE@>u&jG@o)I`t#Y>5A3T*V%3xKuS6J(8!pjoBV-~S1Ui`D>+${0f
z>!cOi6|pNS*JOSF2F<FUEhe{=Q|y%9!aALWhP5J-eVY&N`t@{$Pk*}N&RyPf55<X@
zA3go~jN)^?2K}WMIt5=xT^A1rN8FleL05M8v+=ThcGhfP9I<NR`O_SUOH=Hw^u1!}
z`Di@r!oD}>7~Zh-Ss7_*wmfJP?PQJo`oeu;^6ozG=!DIyjxcp?ZawYv$~0wROu}TV
z{wdb0jkyAo^e>-2dC4HGwLRe6+O^I3aVNV=ig*`@^JcEtAaUBL>!vva+j9oKM>Afm
zj((**KRfq#miN>zS9PZcGKl{Cy!y-4``46Qo{FsdFR2~*i(?O;j0j(Zl@;5x?T0*9
zB#1A+B)q1`)%g81M$w58EkzkA{o8bXmBZvTyk}0ivh97u&4ZV}$gA7N7(Y_ke(8+Z
z($^VmtGTv)$xzh1v-MHj>s@zNId#_r9}MK-F=`OEw3*5~wP%^`w(vH-sqwKpnPw(T
zy%&CB*G{!{8IQwnZ4C)>FS-Bi7Q>f>Oi3?~YVYgs-G4G@cSrd>F5T#K=C4|{w1X`3
z<NP|4pboj3*H3<*escTc&9J{$W-%Pnx>tHaV)--E#5JpqOaI*S`ZT|kt_+JS|3|(~
z)9J@O-!HlKW_6m=WvBb=`lt2l?cV&YnZ4wI+Fs#Bn@mnG6|RVpzq%p))sJvT>9?+W
z+{sT<6?M*8qz83HblMg#GtcS$&Gz|vbIa>ri@-}QM8(*j$ZpmAnHcC%Al4)ub~P~8
zIKsNoIr03r7Z<+l*tT4XW!2;B**2Gh+e3bRx1Ba|o2|}yVU`&d2U6!<s$bD#9XM5Q
z@uoeCz28}{4B6Op;%8dD@x3*+_5b$ROMP1;;-DfHw|)JTH$O`I4X0Nw>}V8d5cqj~
zrjwzK$1Cmm=elK+gRQ&6?l;_xt~j%)x;x9S?_Y4~g*94zb*X2U^+{fHY_V#LxSw^w
zA)!0Q%`<<UtYhip^;?!l-Hn`@BB~Y?6uI=Q<JAMQtx|Ktb%nP0cxm4`t#e37LUdcG
zTtlY2(E4Rh!!$J3Iki1q^DgqNP*}R$+EcUj3l|>Iwlem*yYPE3$DxpgA>MA{N9=vC
zSZ)m1F1Sig_QBM73Yr@$UbdMojn2A$>3WH}<?@j0x(!UlljEI3+m?1eQ}Nj4$Mi?9
z+HLZxpPwBg?7Mr9ug%rA$~Z5_cbvUBQP)S+O2^37uX@e5><iNcu9&R(wMV#l>1EFN
zuLn1|w}<Ym+R9O3D;x7QtG}En{m&mu*$uyTT#o3KYV~1T7-Xf@zCqS@?U~DIOHw0b
z9z<$%MxArwOZZ@8-+AoU(&<3~lP;~kog4f5m+3W7mwdHW<l{9~_8bY8k{qEO)fsbF
z%@r+W>G^2PaQQc1tF7?)DPFGP>s8ntU(SB|sqdWQV&R57?K4jcGxf4`HoV+q5a#92
zeaW?W>+y-L1(6j(6$~Qp(-vMiRCf9C-tA>-zrQo4=r3>ciB)^)@3PL|81Fl8P3NUs
z{)#r(E_QphTB-8Gw+*&)Ew*m6t&}@(*U#<iM``WkOGRlLEWX}*$GK0<_($25xobW|
z)Z}DbR*QTqbc}1INco%f8ISy<C0^`Ez59N(kz3)*sSEzD%UiuN?7vDki*L-rP5ao~
zKIXnyCi}2+{iWBzlN7X=))dWM^u_qjq`l%^f2N+cIc3g&;Ct=e`Yk)&OSS$w_3rJY
zlvPK+WxT$7uI|~?`l6zg3pESN4)5|!`uOSh-MxROuC+Lwbm!n*H3h!~G3C>i>cpM@
z&g*Pc#eBqDb+7QTdAqAuJ(DuM{vt<QxbjFs=GukxCWor_)#L_>EsTrgzVVf-D)O-Y
zuIK9Ve}xXro_+gB!osA5PTCn#K9^(LQf3`gnl{U5i6+;rFFSPH1eUxk^fH`P#--uY
z+_Ex6F46ylY4g@~`n$k$434@>l~h^&8uL%o4xF>(=3LYKv;0E4oHTp(zBza2O~boF
z$;p*o_PcMftd7`u#!LCGs9^Z7^7v<(g;7=v%WoAg(U4=9vpJ%2pW@59%coz4h-kAu
z+x@<-KE}BJ|F6li)vu&GM5i6PZ=A(?OPl4KE9b=d&%JrSc0DYffB8wm)maaxEzGzn
znG?VLOYq$!(*uROy*<80X|M5O4*VQ(A&=>5?mMG1JNQ+(q_y6O$Gu&>A^lgzB(?V*
z67Rwuh=(R#iTtR{%H`j($}lwMqtE(v@u7v8Mm$s7lqJnYI6Sqc>{_&=X?4l`K;g_?
z&5;@3^38=!Empt!b$J5!%2R({E}F}G;N9D_yL*f}Bj56S)mrPFN!!V`!`ZNQ>XAkI
zA+PUz^xFICvZKW5PhEF88%_r)7TJ32SoKHbX8qEmKe7&Q+UApErp)$?(RGS2!>8<o
zp51Sz>54V5ntin`<&k)t@+e^H_Ur_gfR}#t6Zp>0PI>t2eEa{jm3=(69xLSic&5D*
zotJif)<LCPJgI(*UVcndu}o)B-`u8jb++NTzY#01DlD8L$9~|tA1}OI6e$efsuiug
z)TjNZeb<zjWlx`66;mwnX)oL;Y_V$2%t=8L_QXERyxP&o5)qicMpXJ%O#8KuH!3uJ
zbwBYo&wgbbFlpAy!lV;lqE9(3-ZA-SZ~op|RsA;q{_;1O`dWWHmcQD3A~~_ZSy%M(
z<DDT39#$%T5vV@?K<xE2-nr2$Ey|aMZ(l7EE6Q_C*3&vpxoH1X4-SX(>n~<SseE~r
zJ<)1L%RP;=i!|o8<<*`m;Yi?l$Fx#Jd;9lE*PJ{Z9XH3c-khrPYPD6qM&9P6R@uX^
zS>A7PdU)}{-73)#k?#j|Z-kdt#MfR5;_E;4Ik~jIq|rj$F5UQ@bx_04LOqkIp>3S^
zk~ZW$xAm^Ax88sCjIQSzrIi8>SJy0jv^%imMc9MgDjQd<dMvnc)1FJR*AukUEq<mH
z{&lXGe17vwWA%GohJQtj3|cG>E6?u|_23D+x{~ogn|zov=hR8f4uSgqpZzr!&wVX1
zfm=2%{OiY@$NkxFES4)c)&8HGc?7g0c2?R34WC$X!&6U8Pn}Ml$rhiqG)By?dGq9q
zIb4Yac1NQcwyx7(49-uXo_FQ1>|;2}q4-#1`Og>IL;Xbsn?+A|l*=zHIebZURhf#|
zRS_qKuJYFsC2qM@3&OfGN^4ZaOy>!5TzD1xmuGhDF<xzM)te3iG0I=9I=A1rcX-F$
z+52qX_P4e9Fs^*gCHZ(s)yLY!l1raFy4>gR<?6gZ<;v=Jng)KK<6YtfcR5uG-L;D8
zZ3{idX8F(W`s(YQcb>jAP5pYs&dh<&C$%q5@qm)PZ};X9twYYUjvMWrAj6xnq0c2y
zZ%J9j_EXd3y{380-YKpV{;>JL={@@&HLfV}UR||P>->`5tr@Ma43xDFyg2t|QRu`Q
z*~0qjd$bR`{}fzhBU$TUAgB4#LvXRd4)=}x$vJik2ZN^?Ug}Hw70sN$);3Q`B_rA>
zLj1|3(2aMePtE@yW|f%ODmpj(IH%`xzJ}j-QdgGAoZY!(??l<>XT>+Szf}6Ye)Eqx
z55LTvXT|hKBRpZ=OT(lMCXS+=thI}@I$huK{F=A$gBfV0uC9mvy#m(&*Y<g7eP4OF
z-rL>cl$nyzlNTR-<&lnD=%q7JZBI8%;Il0%x}LqO=jbV6C!eO3XD-E@Qey!x?|hUE
zTHbjzWbK|)CNtg}EeYarFI;__<wS*M&(&4-5qsC2z0DtcZ{oq^C%YNUZup6L+1_7X
zF<tGXXG5rI_KM^MeLQCqUQWpnvA?!mY3a)Fh~o#oSi}k(KXuy4^U|%Nwd-!($=q9M
zZ?)5Ar=8i;P%YN=PZYIO&wo~(+p<jdbh2PWz{zF{Hir8QHvNW+)=!Zth}&YnCDWQ=
z{pRxB{2U>>>dL3Bj8om1_y6TJj)m`h7*Fi7<Vv)@zPaUWfUmXD)Kjf0$xNr6wOAti
z&)F;qnt0i<af?N<laKAQmZLl5&)#2}>a}HC7`wsaI#;uG>1-3H<-L)5B%m2Omt_xg
z+bTQ3a-}?z-%AyvOpngI$8hq)>J(|MH{z;VLJLgxZjikAEzwCseA9-<N!mB7ud;M|
z=?kBBoh{XH_qJ+=_px_YuFuP?9_qTrmijk{B~QI-XeRmRlPkB#BSSam_0?`?pUyqE
zBKCLYudr)-zUQzh_<r{g^}i#!){{r<?cTt+2@#>%I>&aXZ7ez5=b<xYx?9V%RXv;P
z=9dI~fBWlWsiBBZyGmfBR9)z6zd2JL-HmW7u?fmiVu&>Pm-19hFL~t@9_2MmY#xgg
z;%ldZI&Y!iKESef911B`F@9{56TIx^e-U24(8tu_gWug1p~WeUzIhQQS|0ZIuPBA=
z@z&D*#d<yb#uNRnBBu#AG(#%Z+!kf9FlVa0y3VZpP4n;h{_EBs6L&xKux;zZ_8=|W
zcG1pOJJl3Lr>h?&oGtA+CRX=I-eK3i5<aW_g5KM*d`AQC^PFM38|$O*3dZjJR36Ez
zs?yh@c593IW7*Yb(pDV!v-0?*3A@|)*A#KeYNp-sUuv*-!lRN)6a1|=NfypJ%G@<i
zU({Fm)kIq+^NEw5)Gml!f1oDU*7ZfC$5M9A($9Y0E4^H&TrqAIDQxayt@*m~>H!&9
zjdkAsUW+O|C7BvAXSZcNyErZINr}J6T`oVv`#LYTztT~xOnq5+Na#Tv!>!$l3#wOb
zZdreIr|_{$(>7~3TAtjXu@6+SO<&5EzAz~6gi`<9=cm7wepcwSX4~wx;=Ygd`FgqS
z)!lWI`K+|&e)r*aNfOl;x$}bcQtVM9r6sOUHp(fwR33TB92vb-K>hE<csKKtk20P&
znM4#<uIb}>`@GIw$A0?pBMEP}S>AoTaif^gv<tq|Ru?mfOnv;jhpG6Iz~R6>vjjA&
zJ3yVd5J&{BV$!J<N|HG0mAX)cQS5I<U4@yN`Nda<e<m%j&0sKiDjKjP<%;2sw@gv%
zaw1&XGJkO~?2K+b=xgaz!F}oFCFjs7D=tfJJ>RPklKU|%c~gD;5`&tmJMz5de-wD~
z+73OQW2)?V=|Nw-{rTy#&AZp|dKhJB{A%9IR&)PBPjN{^;@N<b%Dme#f%O6EOaJ87
zT)l5*8KGDFE4}+|s9aH_k#YN;quxELt6y~OY6(~*`O#?Wv0_Ef7MBl7Z0`+@>|Am0
zSd_<+ipuCO^)LTz$<Pt|x0JP6l~qzTz&(hAF?54_MCD3n?#&%4i|?hzuz4>wijAn)
z6y7Fjs_nLU0?W1&FPq;L9#YbBH`n$|S8MrX*&RBO(^tCDZeLp%!vdGECgWc3JCBT~
z-DZhdtij8DqB$gbS9r~axbBU!tT!|Nb*-Mech$>`<ue$zb{zR6aeC()iJJ2(dbclI
zzHYYn)0R`|52rB-bY7kR^1IIhuBGSZo{?(U?WH3ncO|4SbWenC?g#I*{Y$paI{mo3
zet(0zbbZY47y66Y>=%XYTc;J4{XuzG%uWgK>-zV2Vtp6d6fF%|s=KOZD$9h{AP(2x
zK6ARCrYI~?GA?&m_xH~AudU#&fcA=~S{Ha%iX>^}R5CkwEO;{c1mkP)l-t{H&Fl=T
zUZy?%A;b{>=EARypGr3S$1Az-HPq#AQra`~OGNoSA6e$^-z-bE9JV%J<CV3LWx*7a
zy*96UZ|7I9t$(H`7j<s--wk!s_RiUN_bg{!?hi%P=P%UbIU|dn8j5A<E$^s&mQZsm
zu_E>d=hd|WyRKhd{X*-Z_SvvQzpqaIqJ7Qla@H&3jhb)&^qpql&=b&SRn_=vDav%l
zd~W~;t7`B{o%_pgRw#-{lrJ%`^mgV`Rc4*O(c`32e2}0<qV_L!@hSO>|K92TDtVw{
zPIpgKg;NHXSG4+)#EDOWO>@;9t+*^YEW~%UM|?d}q@|>*I5jNkqJ;`W)u%^iCG7SF
ztPyGScISE}Dpp}4UjF>+sjlwzanqusg*vaETe$bGX|rpKQtGo8?t9;P$A9HI{6gm4
zHm>Iki)@z8o$xtJ$j>q0t>o5b3%;pLkNuu>u3l*^$NBhCn{243j@H=((e*)>!dHCx
z{iQK-s(qGbsBClA;tzcu+y{I&-Qb+_{o8HN_bofpAFbQ+X;K|~Uan6d$K5QO)xUOL
z-pwGO-CuD$>p;dv&C7}nOKlEDwQXIezXRN?alC4Ac}~~EgZaOn&tw*H&YCho`SX>h
zvai1AE?xOG;>nYF-wf<drv9CD?Yd*j)q~E}B_8`<n#Q#})LAO|^%#THpAKoUwQDNO
zr+Pih)KRW3i3rUT)aedoJMd`k^NXK*>-Qx7Q@;HtHvRQ`Tdp(lhTs0o`#V$M=%2>{
z6-$yX?pP%(WHNJ$a?2{_iN}p@8E1W{?KiagnVEhuU}<Y+N~UU+d+z7*5K*(6pO)?V
z^UC8^?(C)AmsUDX%PRk8WxU*}hxM88JtNER;y@PGUv+CHa5>K7)6?cy?xQuS(cIbf
z;t9iwB`OEkhqfEGy#K9g9lAi0LH1a3frh8@_o%CKnlq<Wwmv!MnrnSmW6g{{FCz}u
zmvOJB7**z8Tk4u%mwsiUPL2Cc|Cp%uk6HB>zSmCw^UnL7*16WSXOC{HiHk~#T^Ey>
zE$ynL!nwht-}T1yechLoy!#dIRGm9DJ#IapsM5YidyejLJHA0fQ)YX{=0)c|KVA9y
z)yqT7&8E$>_x+yxt<jaY{MWrq`D<O;O<}V8*A*9iQBZW0S*l_f(bA=%)41hr)GI#~
zv9%5}R_0$%{@7p=v1H$+_rdPMS;t<VYg;QO`%=m;v@<=6@$y>DI6iknhW_G)^^$8!
zYZrp&60)*1SxY%r&lKJFo;h*Nl$VK`XCvHxoeJH~(fl&(_b;)<$%Sf5qr5ghzU~?9
z+JC8uf7j%acUGT#xHnY&V~gw&+R~Nk68<D^yT7)8HLJqggB2NDw?8wzZZP}&ZvMJC
zY4h)tEUi;l@nqz(EI6Jp@p#Ss6HfdMFOOa|)Mz%od|4``n<e6kz!v#{)fs`k;(wx9
zLS1j}U3TVH{b~7^vY*1=Uo_6X;;Ev=$}lm)Md(GGV&_JVWj@wMlTVrcu@K)EuOQJB
z>GDconv&V+)+K>QRvwQzW_0SnL%W6AvTtLy)+y}s;s`WhX=TuM`=+yApdv7CVIC_J
z!{@t~+`P}LXGxv;UVAAa^X1fe=}WfAG-aE_dhm$NEaFtGk$ZJ)g#go;?KScH*JS?)
znDndpopz*i)xs?YCT!GNlD~9D?5k%AjT^f5TAy;6d;Lp&=86dhVule5y*iIvO05z*
zxxwZb&q<~Jw~|{U+!j7^aJ=$UC#Frja$EP?eMf6ApVpL_T$Y^0o#i<#Ov5c{VR;3E
z^TSZdHLD_-w`}EVxc)QY?Y|`-AKsji60ybalC>h6_j$p^My@;ZXKYFQs*@<oeEIb3
ziHr@!ZiOvJTGJP@sN9_Dy6VTZPXeaz)(dmO3t!EZ%Q-Y3S6WIg$PlcWb;kXtcFgew
zrxJR%FS{bcvfxKh$DC`5*>hGM)yTB#Sh-92K-Gn{g4wQuEmvfEQkR@(DKTA>ADhNB
zBl=W<6~m^dq7BcQts>`TJ<|#6Dsr`ATWu1f(x7uDv+jdUe9$KaiSL%$O@&rV`c~cF
zxl}AJGG_HWPv20Dq6&EibIzWk5QE4=zopFfZadZ%7Amfky=DPJT(@XUm<cau&cvV_
zsjV_z>tBA}pfy?jm-@NZWxKgvv-^v^(t8%nKXF!JNzk0KCD}jrosSVon&lz-_lSkK
z$<-H5O+URn=brL0s@$c{y*EPC(Y!@n$4PX~(ucAAY)VhKmUjjPr2e$?^%ANn;g$Ca
zXrHnn`Qyx~2XZ@uTpL37&Yo`eD5idAq`5+jt8&Xyo9ykk4qyH7YwEHBS=HvH#higz
zr?Vcao!sEGv}j#NP-!K*K!8!09fP{aw9<RY8TYJz=oowv_Z8>Vm>Q8-u$XggR;Fl3
zDC0`qH7o2R-o9V#Q?l&ouZpsd|Nh*)|9->c7+J|L>eVGH`jfhUzkj>k-bPw}Ms&&f
z$ayb$S657*5)qm4LUwD)hNhzYFFN`x2MiVkI2~rri@&=zJ9hs!@KOoYqtk;fC@7}d
zT#uR^>rpXj8OO`-9}jOfj6E8be6Hzbn2vJ&>K)VWCRndlH_baaqvSyG*MFy`n#6lu
zek$AK7!@mhdd0#1lzF?>uq$x$TIC+mdfYwtbiI7yPwNA63?F|!c;?S7J7eFP^G+*G
zBfL&}=)_!O6VP(~+}z3>Hd8G9gY5;5h?>x;+ndiV-s8`x)3PFE+O`|>BqemWxyvq{
zKllHK^yl^>pXc+fbzJ#==L(_Ny&LA0GCL)Ixv}bC&{g);lP)N|v^OeSUo}yD_KRs7
zFKY!Z-xW|09%`)nXrlz^XqAQ2K5>e6ZZckZpyp6?=83O=oitwAoU;j-?8o6)>ulhc
zu-f^W&=)p^g)R$EgiYG9`s(e);!Dl>J1?1VYaEmeII(TXhKhw|!cG^?DC~?AtQOav
zpy%5!9HyAbI^$H)VX@zFI~``v$}TqIy0mcZ+KJb4``fPbn7n9k3OQ24GH;rkti_%e
zoWJU(y7?YvXgJ!t)=}2s?%TxO^^1Sin6mEdI>;!%X1>d{V^hAz@hS6eEVa*k*Y&Hp
zd`i&OhKsJZZ*s~kns&1=?(gNfr><u0nSO8TOGDq;yi9jZpG?`kp!iOs!qnC)djpgi
zp0sWCQ;C7kSgdgM_#%2=mhHg`pDUO9pRK;do&4ub;CA(O*#>$)lGy)(b`$PXDXM!e
zDe_(@*tPr;llhv6-WPJyTg#WtfA(od*GU^Qwp~XOzB1Q+jZgdJ`SzFop4s-UwKmC{
zmQ{Cz97&kCtYZE34$;_^?QRkbH;l95w!i+VcQO2^cZU38uL;FQvu=G-d-%80YI$_@
zy$K&G7oD4Y@yq#L|1YoqR{!VmU;DqSuhmyX{jbT;<usUjIX!sy^(&wM8+mWeGYEXn
zkiXIPZi`WKp3$s(^R%xtw-@Dn@H%ihc1Eab+Ny2-8@1X#<rzmD)t%|O^yn9P@1;$X
zQr<aCnL00`M3K|xrMOc2+AR~#D+jSUF%{hY)sfvR60k+e^Gztn)z14&EgDr5TP3=G
zn<&mWH7QfuB>L&<DH+LM*M(T>N0xDZWm~y#{;9S5+_Q={nut7IvTxSObz;XtdGE*1
z*6H+eP!yG1RmyYv)2?z4(+qRXTmR&kC!CV0xX4q-w4m}m!_s#fk`0tLa;y@nsJeQj
z<;H#M$I7=-7QVXZz%5l>`(=7yXtV40Y=1G|!lgkIcwQG8TsrTcuNgMa%|p6wg=qvs
zdE^Ttx3`CMOmyN}40lXkzU7!moz}*6rL~3N>hzQs$5K;)Pf3fTXK+qk<974q>Q`}r
z%buS2rTQuMjbGv83A4KD_LYVye2imR-m!Akr39ttdd4d=J<he1C&X;MDcH$snB{Y=
z=*{d`2cM)!)oi=Z|Ks5YMROgisd^h$7BW0vlcI0KATlZVM*PL)!D0-@%((YCUJl>x
zsl#x`hTS1}noEe!m7RC9|NVS2{q#4t-2C(P$(ujA{q17)e?9;InIz8l{d-tnZcMp$
zt<HPfp3LVA6MFPIE-N*)%v(F_rj!0^c?+`!rl^zF2CGk94$1F!JXAE-x`!`Ub#;!2
zkhIBL58amKFPw$ebp~1ZU(!20`A8Atg3>iXhv&8Jk%(ehanj>)?~!SZhxDdi*gPR>
z)%wLnk76d2%~NCH<P_ZEmHKt}?(8e;Cdz2dnpedBU{>Unta7GJD{l0L*%*EKJ4xWf
zjR&)0YOX$fv(D)JT{eMLA+xWQMNIV0_#paY^*-nB-y6B+1~7h@953~Zb=&VFe_eJa
z^<2Kg#-P@0-PsayB=DuTh}EerHCodGd(%zkcpqP2cMY`m-gncBt8&@$I#YtmA8nj;
z@1*SWa;rZpr%Z~}u;hQa)cNW;C*}#NJHDTKE6?Duv_w)^h|!|)x0&-JNA*SP;AL;>
z)s`94rd_TnuMiX2fA@$-=G3--miK1nGrurffBv&-{F?(i4!8=ZrccmYpE~cAd!QbB
z;(AuI$@e(_3hdrfoN#LLfyMLdE<g5RD^HNIRe!F-lV??T_sge<$}WBL-?Qib@v%9%
z&0RicLz`CHH%6|>>wL^E-n{aqtMcUMi*w?vp4l<HTmNG9BCR{2TRY}R<ohzz)IILq
zuJ(MU^5(KH5282s?hh~KKXa@6%&qb?>3aQFe@-v2EV{J6{LYL&T3_w&W%0UiJo8QH
z-tx;I0(`deUpui_;4t^@>wo^Z&GoVW8}DCVVf42ogSX`Ph9_G*q+Rl#IrU41NiODk
z5Roeova5Z&-3q3FBO+qj7a9_GO3nSB!8dWslgyRxyqw?tKKeyztE@8X^$))1IE*Li
z^svcsMrBGo|7^f^ee;B0iY;9yrZ_TNYwX-KXT_e+;r4-!cQ0N`sC&Bp=(mLoO1rm9
zuTENg_uAF9T}H*L`qoR=8Q#rgWAn-rS~@?n`|Vz@?2B7m4Ev(YxK@j<XlF=zm#)ZQ
zqWsi|-{EtJuiw01Kl47{JJ`F4>zdceq@=}<tgD`Lmd>#H``PcB`YCOhkdg_%Uc9^g
z)%CWbyw2I!m&adC6bZU~U)s0#<K~22Q;R#fI+vck!|1WHtKmb@rW>it&IIo&`fRCT
z!0xcFMQ$r-k8lUL;x#o5n{h}#RbYa+v}~BpY`J)^$$E*>e>&>Qk|le$cL|@I7kK#H
zmV^zzj2W68UY=w9829SLg43y4{{v+f`t{B&D{5PK+S;~WnS1G@yVg8nTcwhpYToNT
zqQuRsaAKqV&HeUvzP}egWV?ORsrj6(OS$fblc)F1$cs){$ds}|q%f1c@Xe+R8o5Vi
zUf&U*uU7GMV`=Tp{jX0H%(I++?q%h(IqB!Pzu4c)YTbP--tXSdPkePH@8uYpZKIDZ
zKVVsqw;`S3&%f%*A1P)xv)`JfPu~0E;`M(zrFBz(_22*X_W$*o&mW`r|9!mgN9OO`
zLvhb%PX6%kxw+OIQ@gUdFP~m~@!xCzQ@>Z9f9=5wtF~3YQ;}FC^n{5a&EZi}CzFzc
zv0>rF+p%kYW;xA&w&k4;!-*xA<{k;vRBVYdjN`iElhfA4ChB|6gU>a&r%&W5hvH&G
zhpH309^sCzR}NlzzwOt$RToyRGTZHX@#(L-UcHk!PS0u#l>9tpd!xwRAo;M}mL6-C
z@p7&H-f5Np{6bwCpL#m`p+wivi=oFih^&&nz@G4Nvh89qHgi4iz_^DEg_oPHGv>{m
z|4pLj_xqDJy+;-$7fcdbWaLrtn)_jFh)S4u<~73=TI*h1YVA9y)xBmp2jc|(X|`4>
zXH|dhe4RaSr;V6peg6Kx@0VL@ObB_Rk(oBVb(YQQbF+?|(tfy)U193^)y@(-r?7p_
zzO!FgbK|<wzf$0_<Ef#-O=-1RYke6uo(i;36b*QJc=N{?x$KELJB?W`>oOEcypw&h
zBjeX;u?AlICGu6aIV;xJmVOVA{;=TDo(D$DN~J@M4<F9sP~cWI*Qt6YcvfrI-r1>s
z^-90G4E=88Z9bh+_>w26!}wV3{S!;Oihh<(X0#}cO;{>r$$yUBr{>dxFQ4vB?w>1P
zyWzigQ}FYdpFf7ZUOBVgs`~TFng0*Hz8|w(eRA*QlWy5HrSEF?8P+`F)tR&JG0V1^
zT^r)AyuYt9_eFd@&-^oL)wdtMe3Hbaz&}6daNb+9e*LD!ci(+<HMmpJz2Un4o_|xz
zKdin|pY_cDb4tw*x%$oT_E*+l^Zn=j_&9^f`_RA)U+)=PSI^XK2)6I8m^JCnqtK-j
z+v3Zwe5`1F^y`0DGQ*k~%U>DoES~x&*zf>{<zqYD3DGPw;%_uc{d=|5%XI3#wa++T
zUHY#R9#OmKw#eKo+)p+f@a<L*G(Wy&Dg$4^qrfu<R+#Xro|>0)_4C{*5e)Y~%ua6J
z^5H_Zka^_IWw%4a%TK=!GFkYtu-}tmrHzjL#U05J6Qgq9_6dGlukXe+_wd}FeEkiT
zcDr7-oO=DO{ZO@Tt*f<tDFfq`3b(ZmQ%pn)PtMxCNKvTPHu!3S>R#bXF`FL$`CGpK
z=j8r6&(|hD1V46{A9?R$>>YW7<G4|h(S*vZ!fQQWwlhp`y|bb1Q1{eDE>6++B<4)E
zOMGj(Yn}eOxO}<2>q=`cUQJuFY3djM<WJKb+I==`y1*d9khCG7O#WE(<#Q5upOyCM
zH9To;%+<TXav-20)LSlOO0<!GWF;eS0^gO2v?^(vDTxnPB^X#9c{~4oM5Uhgo=qnv
zZFw$j9{qR1R>m2hPMPW&$Z2b+uD<?sfpOuAm^O{+4(1`+jA4fMFNzta9e2^2{QIM8
zY>Ax7rzQJ&m@=O+sJ(qV`*dKr_Stnt#kbF_i}P6~|MOv<afMKiv~*@s&P@ihTjmYV
zlWyOSTk%?!p+C*vzR+)4U_h`vyTSK+CIYv_PhM|~+VL}*!EVpaPkanA+uGD8_g=C5
z{(k>-^E#fd7hJdw7_80w%pZS7|H6!lnVTnWJ$Rd863>=~i@kRr>u#H_xFC9#=bRPA
z(ov#EGH(k{VhBjxvuaDgpHna2Ufuu8Zuim`TvxkHCTZz~DOl@0{k>42z?qMu+xU~+
z+^RFZ7E@MeU1n_fd6eOScFdL4`R}()@(Ggrmffjgdt&B?eZ7T7yF4oHx3Qj>xcz}f
zk3>#o^F6(Tm&I<Mp72^d^SSnlEH<}mt`iyV-d=i~@95FC`!f3;^|&V4omziosltx)
zODC{>S8UR&E}N(Sy=RudyddV&$?J3~c58&|akuOLU-WNlec#)Cs=?<I3=KH0q)l&S
zEPa}nXKVR>QoGD4yV$*#RBQg<&7X8#LrZV*EjyEgoIev}|KFcxz5o4-Zl%(cGyUDi
zFMZx3Z^O{{?!uW}RsYx~GdAww6<bzReCuJPM&P#pKK=PCo;T>XY_O>l>MdWg<<DWW
z$TgulXRmEt7oWTqG-{)&dYtPPSGJ`j%fW>K8kcn~yzD&xFVWlJ`JzQahvC~_8HT(x
z2D3i}0va=ym-3VpH`ZNv$+f#G@hq>mKVO(L|7zD&WqA?HrVF3h6<EvQ_eH7N^6U?<
zu5%LgrgiKLh7nUHeEyWuv8gwyYM+SFIYI3+W{Xbmx?6d{Ap5rI_k?X9zaKhla^l&?
zpXXnnZ@*Swt7-rDz>m-Kuh{*p{Ji|*m-GMAesrh*d_H~Ul(`Q>(<@B>xT?)t{iRjn
zPsXIy#X2&3Biz<%R0y&ho;>;CNzX5TjtBoVv9j#;JLk6lS7qJ5?x!gicK^6pR`DwK
z$$rs$dopiGA6B0{UH!y=o#=li(f>pmZvHAi+`HXRP5Suz+PCL}p1Xg4_gCxdOM8(s
z*IY|yOyto0G|7N@u}nwv=^dx@c;&)67g=9e^-EnV>)S$+8T+}KPw+UcpWwB0ucAl8
z(psf0_E|sYeoR_;Wy$@@sH*BO=VF)niglUBvds!QvP$)o>9I+Xv-ez|d~wR`<Xd(o
zizoCfyR(Hu@uZF}?>ZB1j@+;7teYFo?5cF_Z{?le)PLe_t!h#I)s>FYNuv1|pSJ{i
zHj7rSQP^@wwBoIVkJr}1w>usOe!h3md$;pFc79$@aiOa#<|IW=TU%+n!n-kqw{FSC
zkgp6EFSoh`e0^)x&}+Xg)OUl)rAteUO669|6koMr;xK7T?uo7ued};(wa%2DLaP;b
zIT(@_9E)7%@>cZi|6SL<zSjC2|95r$*U<g#rGNh?-gGO9bS@G-bEU}2e{JO#ory|w
z&c+-|Tj-O{ynmscjkNro`jE#rjWVwrc7)s$NS?1h<?Wf<b<g9Jf9d2Hd7q6`S(LXu
z*Erd3_7C2v_qb<o4iWa|&WSpC;>eXFB@@s5W#^YZBN=n<wZx1)=HAP~jG|ON^PfK!
z9XB_w<mQPTmv_zTljgph)^;X1b^G`I-(GzD{QUOm4d*Z2vROF!qrwhH)j$>nzthWD
zj~AwON_8<DP?~AJE&3~~%<;g_M;@J<X`N}46~TAYpmai8xct0pX<Pq3H-Gg?#k%&I
z%+c=0o@It|UEMErW}drH^vp9!opTZkL!O=<vtrT7I3L#<CHBn^qB?HX$g#{gTOb&7
zBJ$F+hy#%thP!4-o?Ik&>|;$}|NQocdrYHs9`pAqq`plJ6RzBq+Mjm+`TYQK<LaAV
zqfG_Re~UgBDU<)U?A)V@S-Z`576;xJsQZ+&P5;l5;LqpV6ZhT@J^8flK2N%GUi!=z
zTIpqpbLaNXKQ}9D<CmPn3>z9)a(0zUiZR?}Ix41I%Fie}Rpft6$c@EphnZIY7T><{
zBGZRjv2z#Z-H+OT@InH^1%p2ks(Rk<5={5rT2wQcL8>kG-{k2Ar}g6JUYp2$YGWI(
ziqYnr<V5N26qBv9@~pboK6|#~;SD#FydT>HO(*FF#&3&0#WSlnY(@{$>Y1jKbf0#l
zh(!37AIw=ZZ;wULi~6*LGqZUaxTi&&^{vXWiq89SR$~T}N4jhA9FH|WZI~J=&B80~
z<}*%kvf}febaPYT)}01yn^X7SOyk>bGfkZ{clGSd!`q&7`{?T*y7eyadqQw*ote1v
zvwRn&Pk*;OobmZd*qf^3>o42#?mkd_dP#25tc9l2kH0#R;G{cgXKFcb-t*3jTQ;xU
zll5V7?=hBGwcBhB_u`++&+EN^qrSMT^wR%nxvSeFl{UHLRF<B8m)5!AO<vnIx8zGR
zbs0^n?wyv*w`7$w&#6~DFLrWIqWL7Z#fN7#%vCwL>0qoFQ_al{S8WV!?|tfB8B&w}
zx_7RvLEQJ*JKTEDYfIO}zW%lK>a2rN*TVz6HJ5$heU^S%(Y2YARiQ+(d#aJ`z5{AU
zZ61r>pX<~dx}T*Xm(StZ%Ij~1n_a~>m9WMfUy#+c!kuGtz%j3j@tXsTB4hqtI=y%M
zeYO0c%Ido6&HF8bUR>u1j!S*0Yk$)EsD%2wX2W@{TpX+GnFO9@-D(VY)w{i4ZMyqu
zi}TaMPro?7c>4dH?dfkn@0XZob9#vc3)6&Te^1ezvf5XTd`zYsE&uu@m{vBO{5;=F
z^upZpZj5(16F&V}-e3Rk&C8FQm!Ef8Qy|qL?l_@US@AQU@Wqpn#g$u97ku1NUH@h6
z;fr~X3(J3=p8f3d>-RnO#q#yP+8>L*d+g62?fm`Ya`)YLwN)nnKC7*AxOe-y33GhR
z>Zbk*zjv@OxoDo{^Hp;{{M(t;zvt-rbM-&WPyheQe(?PKYxebT{;fQ7DNBCsrM$@u
zTkfq$=~sz!FAtSe6WX<KQPZWIYpZ8{>1z0E%6ssVPw26(A4{5ccTK%g_4(BUrcldk
z%ky{Y=BHfKWB=Qg;J#~ep8BgfUm}$A7s~$JoR`3MH|W~3?QFUt+n*%5ovu;~^l!Vj
z^+v(=)xs-VrNfqPVe4+wHECvDkUZhb1%uR&Jk#CppOD{S<ez!#aj2x&QS}h-<xvZ5
zgrxth=wp;UvHqMymU_8Rz@+OLS6|J2<@cmGROrE`0Qr}9HY6>L`&hyoaN$zioNiBp
zs~^2O+n0XXu&%V0ackO!YhOh}=BbpjDIO`2@=5Q0n$ocPa<*YC!;yr}(~ou6^EIrC
zVVp5z+xpq7-j&b3eyZ$QzNykp#tU<noV<FXEK0Zi<{5$Mt$UQ6Pe1q;G^@lgx8Syy
z8$*)hPkuS387BpEj;Fa@$~d#mk3pcBb-}*9$;QkM%!0|^6axOvz4s*hKEnjtzs1vL
zd<lOP_-r#L!|lva`vq#@%jW+1r4*;q>V7zFvnWg5>Gp>*e&zgf?Ehl;mOWp5ve>%n
z<E39qH~vw~<`89DzAf1=ea1P5H%u49Ly9YXu78hIVz|JxHTK!38=H1B2#I80woeb2
zmo5-$eeuaCC_-!EdBMYR8iJ=?jf~8TF58&i{n)IWs#4Ldum9}xij6$aKCgIh@_TaX
zlC1An_9!2Tei>u=aJSOTXYbNpuiCh?R_yrAck^!C`Vq3^{HI$c<;m$^itS{g6TfeN
z8u`H~@@(yvWyiM66h1vkCxQ7=s+nN;K6$lH-#-E;XLnc?&Gg+f@$?0arBA=yV0~k4
zG9zuXmX7$|BX{r3Z!2f5JZ8apY3JOhYkiF~+BOF&f1F}`+9llb5YMhtfhS)a%G#Bd
zAYr_0&degIlNtuqZ&O5$+-cjfse1ReL=E>7YjUNzcgdR9Fua}S6=J(JS|FVB{p9)_
z?<m9Vicf?t?s&K*+AA%$LwohhQ)f2yW@PZLJMM4)bsnGed70GPvf6XDNUs%eFyeZ1
zFVBdJVaevD`<8M&7h-4?a;v=6(`1=oT{cOjw=&6M^XJSx=~sl_{Mh%nc$x}l?sgT=
zXIxt=Gd84!tl$35wxq6l&K9}!{4?q5=3kGQd8dU6t$utxciVz`y$zOA!+Ccvl+?Lp
zzcTQ|o4q`1rOtQg=6zVKBlu#I!`{rw$<zNQZ<%}bQnjKmYrgID<Luw%U)IiMxH&ES
zI?uHPsn^YUHy`vKRT5$}jOaajXj|l@1I5oAm~QZH^AI{=^VVQ<;*Z(<-?x>0lQPpe
z?kTk84|t8|l4}gvvGI<so0puMpb~!hWr<bvk?V&u#8f<;c&5!#_gZRSYqf~Yu{XUk
zX~vAcOB2QK-B|l8@@`OC_4~P1BJpO&{qANkb2QJcxVEItX<Em!xd+QQLT1j;oBVA_
z`Q;f`XYSp8IaVj({F98CTdZ$u?J|1d<M4vZVgGK$&sWmiR;<}6QMUVS;^NrbcX?l@
zU!T9bJ#WP`AI+09dW8~l&7!Y9EZv@Cd9%!1M42gqclSiK>brW!s&BtKxObvcow0da
zvUiWq#Orp(HzOyXOt71+vti~rwicb$1zz*>?JBC=?)u!lJm>V|i_Q1*D)iUP{6D$B
zt}_4iv_#E=I)}0<C-tNXX|g!6C>csjQt>>T!pQ!pWKB=+Nsk-SWfJrJXL~F<u%Y*-
zZ?fc;#f?!sw@=N9>GIwB#_a5$v^xK@Q-wD*KHi=cd;R6EbDqB}0y2~9FWmVSQg`vg
zveUPG_Z3t}Zl9SgF^eJQ`>jbT3;U!KUdts}`yX555LkGzRCj~;LAARPIX@F8-6&tN
zk~KhSo9L4)gDb_BX=T>auk>d>_Y`3X+ni>a&E1wFGGp(RtZO|pjc509*w5T6vO&62
z{gwCBm`iiRE^nPBnXr7b#n$M1cg)j#u1pswIw1G|!>5<r|I`;>(@DB5`}*n8(9OF!
z)~o&O+qrg<_n`=HmoOvGiu+z_VMd<mr^~!;<)hEKsN6eyG_-i-xjKPAHJ56tI&*GF
zoJq?sT{Hb%!TlMQY3qG=ZCA}p{S$bzw{P!^bJA?jBAX>2uHjP&@?a=3>lQV8-u=0z
z@_*dfvWiIz9vP2yq(7?!KFmq`_-oDQ$Zz+0cVGW37rAm)%&Lu*qNZtIv$_lBrSKl_
znXLEq{z~tRg4wFZyc;t5*Cso(C$>#XE${5RRdwM;-G(_!W>#(~(AQC$Vr67<>C*0B
zd)MYh-K+aKDen5lO^(U_+c~b<2wHA8>`T4kkY~MCrSauQw%OBprp;C9Jj-*tOegQR
z&ey+hm%WefOFo-;D_io~u6+}twsTzn`gD4ZagVp>?|GKV8bSso;e9!GI;SeuW{0->
z*+pF1GT}YbgvwR!KUXdK@hZ1BeQw}vCflT^6YrG%4oF?NP;|$Yh6M^vc@dyZ{-<Am
zOP%*^dlJJV&%!Ll#@vTSTFN1Z3Ri0-Tu#1fr+m}r%)WnXPyYKmz4_>ur!SAaJa28k
z>&yHzz56Y*_usgBHb836rM??mWWT)ixBK<<_xrlw|DQqQ36C!1{Vo6Z?0vn#-uD;Q
zSZj8CU0|*IVx|tmN)eM^c06AE{+cZz&66j`%f!2@r#tCH2P&lGbxqfr!~S5ulw`W2
zZO^&g6B0}EPS3V2Hj_CTzkgoMVWIwcH4p6mmDvBef6wGi+voI;r`1pY5MhWop0ItF
z$5Q>BYs`38N`}8WWyvu2(qbde*rP?(-nPfxt9u1ji}`nMRQcPPmsy*|YBYJ?9s}Q{
zVz*Cz3I3J8$8=)k>|2#(mYdx3xP?po3U8&XSJ-SKDLQRxh;qK6%+cxRUdG>T(eL+L
z-283*wB^ft>%UKaowp-9faPBL%NI7qZHs>_f4lu;f3iiF{k^P-hR2_qa~(Jyz4rev
zm7gg!|GLjUUc3MMif!AU+i|4beX`}_`zJ?_xy<#yV!72Q^U8v#FItHu>(*{qWMgXE
zSyU$aVef=M8AVo}h`1@OnPFcx?k!_E_T%%5pN|;U=AO=3#vo?7?OyH@TbI2bnPy+r
zdl*n5UDt6*fSW;8=-$jf3k3H|o&Vza>)-DCE-#z6{+{f#-!&{E<=MLO(pM7ao_;*N
ziC4;J`#YQceFYk9krvvY=dD!fJvZT*ZJ1^5fu$j)=dPu_d;O~^>ukc>*WP-ZA8b6;
zKB>$<>GE{tU7>LONh+R`T+Rw#TBO$LtRupiY`o=_#22Uex<{tGy`3lbZsoq)Jx<r=
zEZOXGGiT$LY@L~g)&1*c&&};Wso;G2$dw0XoNH&5u^sqWoi*=Fy72M*RhHf9XLSVi
z-p{#v<jBmgv(z4{ymD00%D?;j_(h%F`+j}c_jQxtRmsiDf~GQ)4Wn0vtUoy;*IIY!
zlNi;Wn@gPK<Kix<hv(Ov{rztH<FmIu#@7q=H(guqf85B^VD>qKso`z5eW_bKo}Iau
zeJip}%rGLv#j*Comz>3hmOK7)nDl<#`+r;6zt2zh7Cip@^jQ6Kv+vXQ-rp40lfKFF
z{*B*<kK6yzHWJ>oW9PEg=HLBwpDyZe-%~Y>qe_d}Z=TAu6!V`wXN7MU9(ZFR{?O-+
z&(R-VeoJ|TuO{%G$hcV$6=-ng_S~Z(H>CCTQx6N&E|+wBo}0?_LL##^D1Y12Lvx;n
zO}jPUD0`lIarB<AFYe_>21zofnP$Jt$u!*z>h_o}+Eg+>@1Vu0XAhV*C4OLi()6)R
zuk-AsjGTiNE8X;8X#CZ9B{wO?)6%I}!X@%5_q$aSy85@P_50h`T>JZB`r7!TvSnF&
z@3^n-?!0<ud2xYNxA4982_@C@EbqEj%(J|FQz>Zc%bdXXW$lZ9-27&<e`fpcW85#h
zn*+1e*X(S&YyA9K(Zs!izT4No37c~)eePQA-KDA9k9lN1Je7Ac_46I^izUDBi0^lK
zu5`ks+&U-U<??!K>8(-Lrf2Tj)Y{AMz542k*u>EHW$V>;O4)Ch-Yzudr8a}Xe0RCc
zlM0UQaZ6tKY}Lf`>86phlV+(n-F$II{K2$zi@x{IPv%_exN5U8x0<ox>}MBVg)_>#
zOD`Bp8#cHt{`_YD95<z%<?nviNv$&TPGi`Uw?DJp`QgvHT@zXzx_YF=Zm3vA%1Yi$
z4LGr3(#(DTo0q3Y^Gp3Jd!4mG{_XtjbAP$ZAKP1=b~}>Sp-O*C+CR7CMV6Vbw1T=X
zJn!CcaY1w6iL}e7oo_CZ=e?a7*nMGJ<jvokrFCM=D#T8yeb4?}x$Eb(d-weA=1;n~
zBGY<*+x=~&YhRbGtp#m@6~FFn<h?UfkbS}D2PyOP{EwtfpX;r&fwNQUe8;VwIWc#Y
z<#qmZNX(rtWuUi3)O77)-Lf4A*;F}0VuDUP>MZ`&%&x~hpKTT2jx#m$^q+sS)N+1y
z)o5>x%BP<E<FmK^dAic}cG5`}HVw;`oI9&uXaDMI<MP#)U$xW9$nMwa760AuTU-6O
zmOQo1{{Q`)oZQ&G7mi&^*())9`pd`L{ls$H_lurT@BUSBt*(e;LA5RCwy2Ve^?Pgo
zfAOy`_;CM^V8iPB|8{-9XMgh-yE#MFS&;+(eji)0S@z!>`>+3BRA1k`w)ySv=y3fG
z&IQp7<zh>?6bu<$`sPHl>CIfDyYysa`J0vBj4O?V+HK_>C;YTKbdmeOnPswSPK}!%
zr0wUfEIYHO?YPd4+i%Z5e!Fi!?|*TIhTN-n_kZ73J8A3s=<SQYnAnHhnbe)889V(J
z*TK||(>5+e?OL%sT$@(;c<k96W%_dcj?HsV&wR5;dWNI<YqqlWm6CJo7WJi<UfgLv
zfAi%ZmEX^QIyZY-ZLw|9?fiJfn{BmI)iUn8b8UH_fA}Bkq{VmNJxTK7Ti4$Eu=Be8
zkq;|pKDOdF51$jZZ4cA0aQ(Vx>HA;ozQ1#l_{rzz-}YKHe0EfvbUWc$Kq^B~@}+R6
zRpKw7NHsjHtNmW?|I~Hc&u5k!)b;JwOFp~&XX4YhTc3p1Uq5towxfx~n=Sr}3fFFZ
z={Jw<L3@r*pyih*Du=xeGaQ(tulT@FOsjOB{`W&sFIKOc#IWgt^1Ss=Ha<vWfAoC$
z>EF+9++MwtUH;#LCc`%@eqSRlTmMO$-s*T!u|IXo^|$R)WK@+IzP%{uQ<}2j?*m4a
zR}yC$zkj>UWwGy$?X8_Z9{x|=x%+-a{{Fif;oIEXXYKg)A;)L;>UI6bPp7?$Nim<*
zdMr&`(Q0Fl$opAY&adz2o(t046p-pQEs}ecvE-AWL+_t?nf!@%W_TTSef?^1L%Pm$
zuZa1M+K-$Jr=7pWoU&0}e8_P_QEa@I@QpRe8;tL>&0}m3I8lDP-R*5;?uut^J<j)1
zTgCla4wWyDe=ParRMM2XcRLf0YOVe#BgByYX|r{H-?GU`k~{ZY?&%TyboSq~OK)xC
z?Y>W|KiS*PAn`b{;Bjep#x@U~EqS{hUw6Jbt#Z|mO@FOp=U9BbfA>ZCrQi2|H?LlP
zeDZ@Q&)Ds&^7r%BUD=lxXV!PHzrmy2Z2EoEZToj_tU3KA=lZsPorQnrJFU*%U$ZaO
zW=5R)S=s8WJN55(&;H8Zf6sjP<jaBC-wTwpZ|=-uHu%WGaKD!M$DZf^ycxbeKRli5
z+~V*1|MUMp$@`gq|KAtCo{Ahe`0a1@gD?K|7yd7P|Mv;I{i5mb{$Kna^WX6QE$I`>
z_3O&ECR-J;cz7@f`7eF2=68tg?Ki*fe6LkyeQamWk>+~pKKJ7b&EoL~@7|lK*mLgx
z%{TI~R?G92w@=PdUh8@9_=*k9UKR3-xldW?q`xaW&Ja0m^2%j*mG}EfeK~w%N2>69
zBROq9i=KB^*j}XOBt0w#4eIkAV>@#|qP$V4RCV#nyU{VZ9eYf(^3NW$O3S^y_dCOn
zlo@}xlbMANxK7S{sdzZY#&EXYEOUl;=IRG7=6(Oc#9+5~2E&V4y>GN`Thv(Y@rt#U
zu$#U6XXV?J+b5@=&#SwTo~bl}LE3H~8_(%uBGV5&s<TcBV&#u0|E$O^Fmry-zdtuW
z9c*pj-IKA@XU;zRgXdpud+Z{9;9W0gNmYj%!;_oboxkQ)>N1{-4)U|e|M*wAvW#cJ
z>+j5k!IMoH7Cvg561Tlx?#>&=phIhVE)@1%c3}K)V}@;}{r}4o>T@^$T)?*AxAC(y
zcJ3K-0}icsPjWoY(O|IS<E~n!XubD5Orcf1ZYy|rcJ0|5vGw<2{`1G)%a!((>E%s2
zdQAR$NWzB4HZSLHxO(0>an@J4zD&peOmEI?KR)A9qTfdg{#{$2ADm<oDJPx2@TleF
zphCUb6XhFc&vSV=J#3EykA#xJ4e<7+Rp1=p__Bhlyz#uTJIBKMuK_KYYbKo*d4K7T
znPCLOgWm>vZ_;-=7js?_d0%vxt8u4IS!H^o@2**KUcuh3FJ|3-Dm$^*eu@v@-n{uN
zk|DQ5ot|m#o%1O6kPmaS;Wf5|e!i*t)@=PnH$T3<`k5=>LX`8vs};+B6n(V5eDm;~
z|95A-tlR!`LD`?l^JD(J{_l}JX-7+(UvM77q<beber9}Mm~f*_cE;AXYwJH)e{|j^
z`Ztd+htbmD&()u21sJB)|K)7x_Md2RUEJSp=l8d7d)s*#<p19OU-|Qk?1SX$ZLABv
zZT%jv)$sG!u`@R+c6|J6&d9RgioM{=*6*KxJZ}D<c6!hCIV|eG?{7ET|G$(u{4n=!
z!5h<O@c#caJ@i>ajXYax+Ue~!iheI{f2h@4Br{il^_bRPz1i2@s;@Ei7CmHhaEw1G
zeKvL>$Fygn|9w2a3*T;05^l>D%gV{fSn7M8vqkj%mbuJ(V;PQwOs#uS#VA~w_kEVp
zYv%s1>}RHhS5A}G|6g#pqR1qFVk1Yx(-Z}!+waxb*WaEgt*l-7@AIv;w_EPdToS(Z
z$)i`7FFsl*TKiAvz=O~Ax9a}AU+3P=u%-ME$4%MPL@%L|1<$lKj|3T1%0%4RzV6<+
zf6sUvCcibgc_p=@s@~M##^;AhcMHwrztqj!D*iDezV{Btet|>ZYZa$0JKmDzKkxpQ
zUD1;hZY7>)P}#wvbDy7eiZ)AO=p1H;H;UX7TZ5X)dBiF^=IJt+#GiTf)#T*wUWIM$
z?cNE8`OhES)NG~ov$;g%Me@Q`eM($ib8a5&|Khk-QfIE~i9XP@!TZnoLd~V@nSp`R
zm^JzevS*)(G~4Z(!1tGfq2tJ**G}nzixf6yOl00*F`q4c<)6IA8?40{H-JX)TCYSk
zZ(XOqf6@;x2G81yCVM};`eGt>{b5epjk2QBeFxMyHAUV}`laR1!w_7_y1+R4*4kY`
zI(LGPE!-}UTQ;vUM@B1q=9bMaCU4k!JGRSAJIBW0<0Jb1QoQ)dn(r6Fip}Qx<}6RW
z5qv3^;evCe``+G@IU78l&E9u(hVAEPYrg9qm)T$Qjd6PB<(c{k%$&(J#rq_DlGop!
ze_m~FS~lbCGZO93LjPVUxnCT`FTHFtvrFLx$v<EB=kEXc@5$!r@851e={RwlYqR{s
zeXIH$_ilV^zhd>`_dBa?FZ(e{nEw1Y-=%xG_{V-buS1s)w%2dtm@g5YXt#%9L+RH=
zwtUX_KRs4_^W>BLnfZT@R;pc|JD>G@U6GL8?=#j_zkmLJ{QphWyOK|g)n;>f>}J(h
zh|Su6K;3+e;^oQWZp*jruHOH3Yrbsl`?t@`<*waYd2z?}dlox>Oi2Iq^J3WVX8HM(
z_}bbeJ!Q=1s;W89wG0sQn8cD$Hc4aZ6XTr;I-B2Ctq9tEapw8OKW#Qi&;5Kxu`<9a
z`qG^ktM0dP#*(bPDW~5oeQR?v+H}(k6>(pNQ!@oyx|&vAEfLpY%-I#|xjw?M&VxJc
zLWp1T!5m|QDaunWXFNaIbaUzLrhs#2cy=9CNiZsuoqqFNWr^wZ$irrD5+48lzI~1V
zo0mBo)6ZGCtvYu8t$~T5(WDBS&G+to>iv1v;KX(@$0F7GpPTp3u-mt(=J&q0dh7h2
zJ=}cpbm6a`KT2|X;=di!uiiaBCenER8{;nyFPyw?Nxz)`Mg11<vdt-0-cCQYPrp02
z&dzMIo8H=(DDKK@#<#RQUnS-AJ$mv==<sRr>4$R77+n0c@7Qms^4+-NeoVa1o|F&g
z&Y$1(`R2jNUz7|I)Qzt^yUfa?T#}Y0cKh738ph*^XEx@Zabn<<ogKK&O2;c9N3W-1
zm+@TNxAlMLw@sFLyg%a{*W$oky{RtA&7W_&EM6Vq_Vul-t*Zz7^tEzg)qQ)iZb`~M
zX0ZEZ@b6ane#4Vbmj5?<QGfc{=jJa0x>uF<vz>l-&wScv1*Qqx<@IizuDEvRjnSoq
zix*Ai`@Yb<>36#L*^}d^rM5laprZKtVrH}HXO+yK|NbqmHZ|dS_WUF`=Rn2@>r}aK
zEOyZGxuSM5pf9N~cJ_JwqZ4Fqd%l>f?k3ATY5VJm@0k+bzS(ym)c)p-S>lf$xILZi
z!}jFTy0>LQQztLFc4UgG@vrP=`Lh!zKivIx3s1D}lL<B=Ce2r?J{U|%xs!0yFZ<Sw
zo7KhjUp`%tU+(2M``jEuX%DN1Z*1br{#2<fvRy28yYDnZ^-a#up!dnea#}a<G4?Dy
zb$s%JnLk|k{@q{xMacT|+o#8-rIx>(v+HK6Zu?EY>`&j!quBqh$u_kz$olkav$(fl
z*}NFrgqyQhZm%}Ansc4+x#_&v-_DQM9@~2V&8j0e)06EIUoAhDm-PND-@CKFil5%E
z|9w%pE~9t-yWHIE#~&U(?<5?oclo&epVjMoZ}->#2%meleR=(_=5x;u_y0d$zc+&?
z|Jn0izo_l^&rG-7Y<J7%*S8N{M}@bSe|W1dUsF<8?5l2>_WokHhMqlNx=xVU?8Doh
z_nmD>SvSctXYs;>XoeF$cV53~y;yla^VRzMUTPOY!q%^xBiW!aBkHX0>Y0q4SJyDw
zNV0|%8%xGMS?jy{<vp>r-U^zrSA=)n=b3eCre->u%3GV6=F(dX%yvjhJc(hwv9$7P
z>)q^So1ZS)yN3H}QFZh)x%qFmJk06+bL*;i)!|QXe$?DzSvK*L`B{_Ce}BAs@+wO%
zCN`$?WOsMBqe1cRZC;anPF-|05>fJ>wA1FDs_gTh>3f$jHyq>UN^TN59^TKM8`gJe
z>f$Q(#PuupbWK%GGoEU3(5K|t*_kq?`t7eiZ9f0xbnDi&b87BbomgA3adY7_-?jNV
zJ$uC$r&!Ko{krn<40X>WhJ<3X;LU7@f112AiSB!R!)zMwib|oUs}4pq8!Emoo_Z#C
znp)xBjkc8^5)PZ`$8X#7@u<zojFy6VDYtg*eSE!f$$_}C<}dF<<#iZ>D_!Szlr21T
z@F9Ol+S%V_cgq<cEardw``z^~$FINAPO8-1Z5>$X+`|~N<8#~H7iG7b9lp+5c5l+h
zHJ;)jX2uyuSQ>KA%O3X>y78?>%;<Jm($XFOkNK7Fe)+p}f9AR^A3!bs&EWP<neNjZ
zMhWGMMUnGV^ykg(UC*p=u*UIo3B!vkPRcE8+tb6<_1=GRioaUB+^lntp3d!x^*bx(
z=!rdU`^4d$+F7v8R$Eec^0uixD<mByyoHnZ&Yd<RRcGebZSPYb*4WLS@Nk;hl%k$W
zD`}lcmG$NE&sJ7_Feu%dDwAZh@}zj#W_AV-u|E6xzFe1G^9<9>CD<HU@^qg>yp{IO
ze<)`%!<7A*|NqMdlV5%*-1qcq?7fY5+Rci~sym<l&`m$LGvC_!rS$!Edw-_I*n71d
zpS=0}<v)vNZv6GR_xJVa+C8QD6O-+3TiUQJd2TW7{LW+xzSm_lpIc;|vp(Cq@1V#Z
zHgBEw-lJ#Us(oFOZLEGU=i0i<m9zeA<S4DJ{XLhz?uY!n`tP&-|DQfDWxf2dXa4iO
z|4*IiXPd9S`Nij)PuAbwIWMlf|25|Qi?^WhA}RYN#`0V1a(X7W9ooaG;yKBp@3B;8
zqV=}TOad&MJC3h<$M!PkW>WR$ml|_**FH<L>OOt(kP`c`tZNLj_bi)qvGwX2!G%+M
z^wvCEHcx{^=&8wzZFAMRkDXm$nSYMqHqWfjY0MM0G)bC7^G@7urdccL#I(Zy)$Do3
zb&m>}->&@p?8V!gPk9-t4Q*#8TXL_>SoSQ!Y>v(x7o|OWcqXY-R{hZtNV2*tx1n>!
zS)nt}KFyF2x7}QspZ)vMv#Q00ifcX`yll*LbIaz7r>y4jtFF6rZC2IHyv;5?lel-O
z`f2$0_@0Trt}f9weg3|VPn}<1RL{O&mi6{|+T!%9c~e5y>qf|#q;+0T_?#fdprF0x
zMw!j=jk!7urv6erj2%TwpJi<L8Iw3KWrJ))Z=3x|yZ@7qt?~2y@YYpmWzEq9)tPtS
zxF(sZEvmTg<z`=N^{W2#IX&jwmeuafl2!6Hw$;gZzQ}Lw*Drs3E%JHzvU?&XkGu-4
zs_HhJN+?Y`y>O!T)Uw^$>;jh`b62u8s2bnSEIh}NeACQZ=#YPB?&b*7m}60O$^Yjn
z-+q{t2i_It2wuiM$@>|nPRH!bM^_{^KNnY*k7u9GePBZ_m&oF;I=iz~m5Y+q?aOD*
zn|Lwf?)!ZO=lQnn3Vf#1q{z5t<(?q-HpN|6l&)-VS=V*p=G(c)4)w4a8?KdBYk2rn
z^&;nygo{1qlG;z=*c(2d{m!_D$L0B_na@*tZi4ol<SaLx?`tORw&c^<u=NK8o)#4+
znmp?WxwpBhS=#VN%h|&>m;W!GaI)k_{<Qc1>e@M<TE8#dy~uRF_U-(lG;8a}d&{%#
z|66NVF7{EQ+VaM(PV1y6>Amls*Omv}jxCyPxifaVCHn)5eQysG&v^Xi2A@p&@wwNl
z{Is_(zjp1M%(lee+s^KIXfk8+twSY~raj&+v;6UUAq9ubAM+BHeZ2Pk|IbAmi>>y|
zNSQwQ^8YV7e>T|fx0x9~&*Vq+vtxVD=6{Lm-)EXwx2f>QkM{bn(cc<<&NJvt`ss9P
zXOU0ZZL`(-&rY|^RWUxl!SMWyTt<!q+as;q^USRhOX6B1)AV&Oe`&H_+&XLes%O7K
z-`aAf-Q9UMdNxB8uVWeiDM6OaQ<QI6K0AA?Nj2?)Nwp@|N#S$G%V%mN*9Fhs(3{z7
zbAh=)tm~=H87EeSpi{9b6N8*MG>_aox-{=wsC0eHG`+laoFR8_-`cl*t8eivc^lis
zn&-ZMyDdMjcl~15(iM~Um;_!5|CIfI#)F)zX`AKEj)gtE9G1Yh<&{Lq?zd+RWX{g=
z^80!0Yw4sCv$}`(w)f0p`F6%FS10wyLX#PHXQTvoMyPTt@Z>2?z3q2;uKLRDlUq4W
z*`<5VidJ0SG4q^~W87N4+Y+<ie%FiNHs|A}t0k9iHvE*n(sNm&n~$AgW6zAe?ri(n
zqu)I;TKIj=7r!+#{dvS#n}6{x?c=#=_U&)xV;kl4z)vL$&aZpgIqTpqA^nrf6!Yz4
z@^U5$F)R&ko~6!PZMjEi%8`J?4H5~z?YxDo!+Le5ynQQsyf7z!|LwO||315wQ3Aa|
zVc(NYizAJuhHvFOBr`eV&zsBQeZ9A8>J%BGb(buW<1&m`V(zY)uwM7KU+yB2#Vcea
zZN432XPD->J2O3}%zt&pYR~hY?^{}AOV35ia9gr$*n8Pe$VJJaa~)_oRnE?o?xlS_
zzm&J^-@LQp@LrBve@~wOyk+~IU7O}!JM~8E<bk!7lMcq2&Dh&;*JIL*-@Mf;pYciW
z){?6IQO~=)|MN5D;$If`&t-k*Ge7fv(VE+lZ)TN+Mce&2e)DkOO{2q6wSxWMj-G8U
z+|=(GU%Knk=ee<M?R{r&bw`~x&o18lboS{sqnzU#Wbds}^}PE%yLdg@y$pkU34Zyz
zU;aMZ_i(n{tbgYmug`4$KAStO_;z8HopGemT+ww=S9fjSdbdWb_MiIWp7g`@o9EPg
zoAgiE{vY@C;>k}XzA+a+zIX81-naWIH*Ng=q`7}ZrpI%8_U}2&r`cY0?@YX%6&G(7
zlDyF1{H!u7_r6J|Rxzz9*|w$ntcF62t3<a@g5H{E&vJfU%DT4ZnVQ1dXKD>4V%e=?
zidvTry-xMotSGWnId8@49SLPmGtDnYDki9>&ba$R#k^T1(&%>3!x~HO<wn89E?#G+
zvTR=8;*l2aKR>qPbpOlX;P&Y?M>&iyrd?rp9lN8x^iu@$O07#ue%`?`Er<2;)=A(0
zoj<?k!^MLKm;UCjJaAuVwtU^)s_740w=U1QdH1Z|>~jw`{Y@|nzLEPoJX6ScXF;m-
z6~E1ydEb3E3#MOYFWezfSGL(Q%jNv+K4q~3K|j2E7|x_mepa^oe#&XKD|hlXR8Cpl
zcd+GzbPnI`IqH($X*0KOug(Ad^yLCuzGyz>mx{Tzr^4M>6HMQ*S?&xj;wuq&_(twj
zg!-$;^&h_ESg+RQe?8e`b?eKo&E>bknbkt39^AF~;H!4tZ2}B8oR-8YJ7-_~AUiEU
zX7^d1V{bfImdv}dThf%h^GMc)3)>>?p6k9V*)CtQ+tzfuZ?){y&Fl1+Tnlb_{cDo<
zHc6I>81b7W9_5^h3_sdrSkz8kc)Fy#>7pX*aU)5A^{GAc7e?O5e<%B7$E}5_hk~vi
zJiJTiLZ~cr58v|zdOp)yTjk8c=AG#_cX;t?@84~Y?*9Co%y9X)-`|6cKmWRMAJ2cc
z^7a3-v)Jsv{bXcF+ThXY#NAV<5R#&=;(DfbI%Auu-MN=D%j+*$)O?qH?Wk0~{G09J
z3XLZ>pUrOnu5|m|EV+hro;Q-?=BCQ(ZMtB-SBW|0d=F#yp2IqP4|9J$H2o~a5MuG(
zE@O4UJwqLa4}woynJ2$Al)tMJ((wPogw(|A>`F_!8tvm#gN}Jk{oHUNE@uC6_K*IH
z?*3(C$jbctWO4kwe=qqNCcM42{M^>x5BReW*#FvFKF_{>-rhau?f-sev!A5(qA7Ku
z|GE?|Cl9`Pm#Tvw##BVN&)vQ90K=q<TaJD+V$rA(4qM#HutG87-c6lc*%$L;?3?fI
z{ynFxe|vGyZcEMWryNq(GEctx^>D`1_H~CUr!0?Lu_%UPrIk+4l<SG-m!2ryvh7*O
zmOTCgzu(t=esb17ZppWc+qbPzcqw1^-u}-6_wox<&93R2>9cFCON$QIU)T6gzIOe;
z+SPm?{)E`QjLW}&e-ZD&uo*j^buay3>3mk4!yt6NBg4}lxu1{bSp9pkc;T+^Z*vna
zCggv=`<`e1Uz<=zaUGt#2B+SA-#_p&Ed6_RZ7x%T#G1Z%eopnhr89K?7a3UbI9@t@
zG<)si4_;QkW8@aUDSBo7T4K-k`}->D-z^N4KhKa;|6$|3ZSUW1zdEr|a0^?v-0J5Y
z|NolQ`@8Y(dnrBX&W351T*T*WE>;N1jZ<^nKkI~?cE8Qr+)Cy>k&k_>YxjJ6ye2#L
zzx{R4TD$2no+i%oVy>!L$j7uFo?iGt$y10S$?Tl4vNwxi#I~x4$2Ws!EnAzx_TlU`
zN26WyHvLV^UiYonqwE6bv4g(yX>&@Jf6A)M&pcbxWpnxenF~s6+TJd6i-UIPZ`gS`
zwq^QZSEJ?AwUg#q#(S*n>rq|*^ZoL*viEZg7OP$Uqhxb7)a26xcco9e{(88`?zi*b
zR5{7?O=|N;)|{@2&2z(=69w)}Si8q>?Yq6U-EvR<yx-n)Zr$$(YZpeYK5Bb>!=C4p
z)r{q4>02F3noxb)IxTr&&bypk+c}#x-`np<`8xOCwp^W>Z8OYy7~<0N&n=VN{EW9b
zEaTQoo5(QpEs4g5)2a?`oSD4&*CDf<>fN{JGPm0<J}aANapq;|pJyd(!og|p1%sV@
zSFg|CfBAK_bZ@7vP35P9?t7c_pC9h7sW2?Pb(j4%_wTxI*RCJh`DofhNzMr>own_{
zCNr{Cvhu9^D(!ORO0tDY_XqC3thB_a+S&L<tdf@G8nu(RJJ&1ahAH1%RPZ$Xt*7_5
z?CCw%*DgE#Hs|1*Nylepwu+TXT#j5*Jn`k<8{IE^JJ@%!`#je=?<A@@wKcqWY7BSi
z?DLad_VA?LI^UTR#IU8~)uU&>9zFY&vHI>`_S0|kw|>4K6j*fWM&SJa&$$_{neP|3
z``z>BOXk_{_x~<?C%5gmqUR%>cl<Fs|JR<j-1G8p*{p`n0FyJ)k3VirEqDH@eONAE
za^mHqE<a4;SvKD+X_Go{68q#+U+2q5Mus_R&pUs(w8?*S-@oggZPnvD9<!$(KACf2
zUtZrOC7IxrUlu%<+Ln8x^~By}1BSlClT==w>GgQ3ayUw6Uuwbcrv}fA_U7xbPuY=U
zzU5r3al|gQz0%RPOL(tt+1z&b!lrUtj~8(frw{Mi>A}9$eeJal&Z+mf?SC2E`}E33
z<Mg{MUNO_HTPOdXZQ6EsUZ*Cn5L1Sw&oj;Ps?9fNs2a~oTDYauG=017-I<B!bDw8C
z%zLxT+Vq{y)114iG1tWdv)Qs^<I4g+P4{V+w*CI&&2uM?T1MaJ8?>5T`4@WMxcp|<
z-MSmOas3Z}MXoKix3_PfEjuB%WNx|GVz1%}@`)i)pL+P7TZFl@g&*aMe>~Id;H5>6
z?^l;!H(pct)9&`|-&-c=t{1ap*m76aMS)#Mq|CZ@Y0A`tpLW%5-&1vM`4cw#RUdQ8
z7@nUvclAa=-kDhs-`Vbn|Nr#K`d99@(`B}|ZvV<_#wR-C`Uk6O^@~3yL?)KhozST)
z)7_)X(4NCnZryY4q?v8UF}Vw^EE9fy-!;KS{O3c{yKa@Y4A*=<bJzH>hQIpUJasOH
zAN$@Lr9Ert=T+e5Jr^owbfAAK!v>LmA+>6vGf&8$efshD`?}(PFXszSFW`-_Ke%`M
zeUI|neLw9Qo^L+BW7kXRb*oh#9Jgd(d{}NHxGMD7`PEZ<UX|$|;JEcfwxLgOOSSK!
z1zLZ?rXJ6_cQBfvZPBTOi5E5U>|5d%%_vy%G)yGml1gECKv&u6h(v)EuflgOT3_I^
zvESub%<}$wc3D@ptzDMAW&3ygKaF3X{eKx~wewt@AOHWS`wi}_=YQPaU-y6h_o+J)
z>*VTdzWn}wCn#yfCfQ=<hMRNu{QY*~ZgARV#tmQ1cJ2P%EPw92OV;^q+ZtF4=JC&J
zjQ{K^apz3xl?U#(+RFL1te^dO?aI^E9}7z?s?zoP8~dgIyh#3Y^y0zyPx;$E>i0c9
zrh7~C)LuW1%iP-EHG<{^#w8wKKVN$J&C|i3FWz4s`|Hg?X0KOk?9R{iXl6CDR@Tw_
z&%(w0o&Wr?_n(!c!ra*!TyNj3`Zz=OIm3-YzvGL)71ZbNuWkQ(sOw<VUCqx|l9LxK
z<5f<XcW8r!v8qX{s`so@$r5j8)*fbVnDuNoXqQMfc$dh^ty4CiDf{Gd?qUwlJo`#J
zpR_|86w{I!c=lTJgw2z1$-c;Q@zJZ+yQ|uJkG<K~mBOxb>{3eSO7+8<l?5}mJpVL9
zZ;I#Jd8&uP_&tTJ=lfP%RyEgIyYl|0SG>Co=ljmsJKgN*r0buKWWU=pQ}y5rlNt8X
zk4hf)>@qWMQ1RQPwv0jQn~_j=+uoRl@}5c)WwX<4f2L1YXDEAiJLj;0Ve8un_tUe|
zul>@DaDJMs<Nq^fx$XIwM5FbAQ?)O4*6-e2XvBW^-IwiuT_0>d+L<N&S(>3Nefl-d
z#l`D4o6mP&qALE)O?=YLYdqVG=llHZGoD{zXFcCcT4&>;nJ*0wr)4@{eSS`6Teb=}
z>)~y($1{ueAAC0NaYf%@39GZb`)B%`)tzmb`~2Rz!pvLuPZVY)RR#FJJ{|b$W^uBP
z-dWGT3zFaLIn8(9^Wj1><Egs?HfM62_vE=fQ{DS%NzYH-TD{#D6}wsXhQD|UYU1v8
zF%lH=O84G8ooUVrok_d0SER7TZk`_YCAzhXdjdCG(uTClf|)ztEzn-|(}wFPi-UzQ
z?=zR33=__zh;>D%GMt-VyY4MphP{g4(|gyxvHPFjw&&NuzwY0@1x~6@{{8;_mwg*^
zx5h4*y!?HB?b}9)10QN%FX+!H|6Gu4mMbm2IrDVT<yn>ILoeTIvCNt5dihpLvYPwZ
zlWWesDqQ`XNw#WI$|wHH>G}6>d+cS~yy4;ROS|WszhRUaUADci@nCLpHRG1obJ!mj
zFPbuA?{dM!X&v9LCLFMN|KegprN7+|kJ&B;)$5<1m(Iz%5p#C(&O<Q^lP9QXzcg8X
z@ysmso6GLjJ-oa7`u!bMA3gP7mdxE@W#BROhMa73NJ&Y?sVvpKUMy!9iOpOXyY0!=
zUlq$Fb*g@}r$4@DZ5(a8<fV5|3eyFtHRq>ionCGJw#Q@5w8I_^u7|!YxtMUWM|Ja(
ziv{9t&pxuv4t}-tXym*D8{XD!7Tx_j?{?<Te?hCZ@?^)xyFk);=;NH9{}bQepC#jd
z*~Ib9PlgE=6Bz6o{H*TIty@s-;x=!y*6vN4c2!sJN-L|Xs@@VdDN5UT?Ma_q6JN5w
z>7TG}%lBQ9x24zSh3(DOJ=>S#cKfpN<;!<NBiq-VS#zqZ=gWtVf?pH;9xxwZW)S*O
z@^KPJn|tZ`g`11>f6S7WVR4+}!N5Q7d*%G^Ihq$GL{49Nv0H#=2KUr=T<>e|AI#)b
z2#l9_5kBu+b5Y%e%#PcfZ&aJV{)~5>SK#?saQm@y9G16A4c9O~P(L8R@N%Qkg1&{X
z_8D@$kE@Sad&FejeMy_YU7Qg!IBQa6_u8hri`)N9`S;|lu;z|M6B$<bC0^~G;FcqH
zbSaC2r<lT)os&B%_SpKm?0H-JWOd>vCS#pbwi;_#F4SCH6TiSb-!sMNlt-ki<CasA
zHt#kUif}h5@3fnIcHiYMj~ORM>j^aMJ}~KZQC0JyH@@c;SQ{AjC~eHR5xm*);nzJW
z0XO)#*Ung}xA>W2f=rOe#DG{o=>@y)G&3x4(fzUXw2O-BgP$C=B@E|N52Z;(S%_bc
z-KgPSE@U6Vci;TCb#YAv^O<?e-T&xxPMpFhaQ4Re(5Dxp(=v~8HB@V)+jg|Bda*)u
z+Nzq(*PcHWw)E)kYt%gOvB=}vhp>m?Nech0b}ef5x$|&S)CL|#k%<fwJoq}QZF=`D
zQ{DAaPOgOiXS95*2g8N!X}$jpc4*w<XXI%*7|d|df7*w;Gjikqz4|xros8-FdA-w>
z>&@<2X5E`N_4BWvzB_f^Xw0*p`00f9RiWM2?m7NFo6jG4xo5uitJIH43m=pe{@r%T
z_fvB3d;a(Hc-8%-&MvnTX8h><Iz{^2e~BLl&jlL)J$QWnzmNPImGZhh{P=CPcWuu!
zY&ht_ejz)xS<uR~^_x=e-<Nk+#BXI^dnomB#nops+rINjWv<$A>06>}d|=M;c%8X-
z!naQQwZdO||2^*omlFOqE{)MKO;?|y#A3kn<Zv>p!xs|<g`Mv=6)IV#2P(zRHe}#d
zG(9KgbmU%h`Blk=ofD)buc`g(OFEp8`$)RC>)@(hR|C(ZZ$ovP)0TWJown^9PvxJ@
z>d)K1%?_En?yL<%$lYAY)}wMypZixVSKPhrg^tYf%4%C9^}I_GuQprFemhZ<`NO9F
z3Re{;ZRuDe$+Jww`1H$D1z|nw^D|bzn7MEBZhM|3sUl}yWE!tu`2BgWJGY_Mx%2$%
zvf|d==CS0tJ0twkYKt?scTRdKY5VJNfHCJ>Nd~=j>vG<i<m+r(8nJH{bM!es%XMef
z^D6UpY}!`*v8^`y#ZHx9U9YasSh&;gy;iW#lGtqxch7Fx@j6FQp8LwwH)agsn{Kb$
z-1_8J@t@5Vduli>CLcWTs@&)FiRJSir{~;C?bg1l-ZNF|>^jTN^qJ=_MN}=D$Yj7P
zwshComV<kC@-am3v^^S<mN|Rt@)=&t%TCTnoVCpII&WTN@!bbv!p4U`>s&UuU75Eb
z<m{_$FZ+&G&Qj}&)BiN{Vxd}|^@}ZwXL;QFxin+>%wo%HZu9nur^Z^=ecL6`-E?B@
zs#E!Q7bnK%g$9Oku=X-vSd(NpJ3Fkm&81=O#OTYV&w5r#J&t5knAOT4bM$nIRLuDl
zv1cC77n(P2GWqr0E~<bz_IKa@iFUV2@BKJ%^V3}2|59oVMn7FYeObQveEy`!yqSkz
ztYcU=?_c?R@29U`TzQ;yQO5nt_4UWqmu`xG_B-`lbD?R`lZby$`pzG_X|uiSwTV<^
z;N=#ZohQY%FPmtdHdFcPiook3cUQfAzv}7Qou9u?p1bj5V*h^8>sBi_-`2UbSEVXR
z<gQzpfv5iUtx3_po}8XvyKLL9cJ=$K_SS0rdsY1NZ@%=a$uo0hB7+QhW-guCV|%_Q
zP3S`Ev>;JM9&K-Kv%c+Bzpo!XVi|j-;-Ki;d5^rzL+^$dUx~CTKHH=bHz`q2MRy(3
zt$C+>eP>r54Rw%OBVD}Bw06r<f3fPo?JE*f8Qel!LiHQxuJPu%=6YHw?X*!>$aUUj
z9-2~OJ3EaOr}K#IytY&`#yEOgHq$8)sq|$lt=->8gdH}qy_x_{_Il7{zlu9?(dVB}
zyiZKHWpllvF>pRtN5+h8=XwI?m%LMuY_IY8{>0+U-a`VfK3#v^Gwby-o-2=M9($b>
z@z{=K#g>55)31%UJ#}?GvU5>L(dw>)t5yjL3(n49=Gd=OHgDVNx9V)%43>I%m4d3?
zGOhe>+Zn2Kl@?8#bD*M{t05$kTakZHTiI!bzUb>|U5hi$T-E;7)%NJmmh;l-lXjO)
zPKikWc;on@_a=?YX2;C;?5mhyB6sWK<b9W!XQhAK@!sTo%I5Xw7Kt9cT3*HQr{cl8
z`5P)87`^o=@;~WRzFYk65!IxZmO3R8!Ledonc5y+JWm5kcgkOH`;_Xw^J!sNgz1+(
zs<{ga&NE--b2BS8oRj#&B2Th#@xouipA+3}SFFC)Xt&=js*mkamA1{k%12gv_rDVl
zn&`vQd95Yv`;zd>Y_EH}?i?tweD`d<+2pwF+8t&~TEoAD>ey=Au|Ck4=OV$yslc&k
zmiqB^R~}^ku@-Op{NioD5hsKCzm#V(_vYO_oAp2KW2g(S{Ll6OQo83eRPfb&e6zfL
z|3%UGXZbwxri;uZvf@0p=FBm6|HyDbkZ+D9Lo{Qv_;kgH`;#9f@aev2z4SGL<$%|Y
z=N{T(g_Ug*ozjeX^P<dM^*vl4-^hCNHM`ex!I{gKUg_8T?E5&qYGKRmA01WhVK&!(
z|32&g|ET=(XUE_D`)>DV|DX5k-?x38)-8T{bJD$B1>U8HJ5E-IE)jj!7rvkS>gR*q
ze!e?j$>qh{PoK9UZAFyb@x*skk0o|+$6EEz37nh~5xTF)_LamkZJ}&+gG{+c8}I$z
z=R8S=YY|UrU6HC^=fPVicg=sKJ$Js!qb<+N84QlC{<%_f;gi$5yszpus(xNs$d(|u
z^Fu<%?bdCfuP^KWe0tn`W9NEpGoEW|`%Vg|OxnOJQhvQWX-bET$8x5XbIz?h&k)k<
zvZ5#BR)rtWwR3sevwgo5%<+5ntMh!6j@Yz|Gy0pOYs1vd11CjizRb9rA<|~KZ{J1U
zxh3J<Hw{c<jLw}5mtA%;VXoG~6V?+YZI4O!zMb2>+`hPX+Vt4T;njT2X{Rq8`YcvB
zLqB@+$t`y;Jc?v=zB-dJrTk%t`Il$CQ_Y|EU1d0+dwOey?QKJz&1<h2MZWS$Yzw;>
zzHsB#Cvzg!CCs|V=Q;Ii>9%!7M^86z=9%|+@(j<u`GONxN$B{mXWns2#DD!|3kK)%
zbB_wI-)y?~<xADEWokib8Eui}XPwSJjhXJ65)m_T=0fAk1`CZ@HanDBZj5wE{d@NQ
z>Q_ZGms*{Eaek(+6vye4>ET{#TaCA#Gr2tN)Dn)auw4Iep6Z8T^Vi-I2~!qvvC~Z}
zel$mGd-<$MpZa{(Zz*y59CB~rl@}9seN0&TPRTDj_UqKtLI%6}u9q`r9WD?NW@y&D
z{Mw}T&8KVU@`}DJ+*N$%&%N7U+uyfT6^M8L`t0AOA9v#Q@rz#z{vR!AyA~%UKW*mY
zw`M+@vm^O?ZYh7?@NMe%ww-UQd+)}s-<n)CFTL>m=ijoYP0q?*{_;vEblc9%Szqs5
zi19X@kvU6^*EcwQ+36Y2><upMOfr~jn0Ix@)c~Hmsq;?z<X(EWYkH~V*`tqVmW37j
ziGKABEzeG?E<9$vYgca7ol`z(t8RE3uJ-ZTe)rd9|8sr+uJcRPzW$KB{pRku+eKwo
zyqui5Iq%FRpY*nKJwC6DE|;41q)RpE=!)d!{G6+q{OZH}*}Iap#6Ip|@c24)$-d{h
z@0`MqUimB*`K#-Xp!S52iCI_vw6_-BTQB}q>Z_89@EYA(r=&Hd$5sdJ4-{2A?Y#Q7
z!nE#J?43uF&dhinleRL^Y|{rn^JhJhM#;r@FGyat*qn4GE$DGu<}Bf(Yi?TQzgfw4
zJ7a^{$8y0_`>Ut<<~+*gzL&B=fJxve6GMsw*9yH2yIAyf3&Qk!1Lq%WZ&bN2doPoL
z<7(yR^$Z)jnWIdu@=waYwo6e=xbd9-s^d4k6gKGz|NK(JlOW;2uw?G4k4$&@7#3+L
z-P|G>GTl@2ov6=5#c7K_9MmY@V<l*_<hT-VnEy&uA-!aVtVL>qObZ^_F}!}IE$`IP
z$uD@4wc*0uvYBEH$~(U)o=e$${#g3Q83mzR8IQ1q2A{~wx_i5G*_@-vD%aOt=05OJ
zIVAr@74P}gj0&H1+4>JYzBei9q1^n>mJMF>zGgR<c;{YYJ7CLqAZ?;~{v45bzw=y2
zcj!L5{#0k(*Q&eL=XV88h`B80_Hn`TbJyIjHFKyh-r7CY%$xVml=2NGhEA?ty6t<M
zFK5nid*`{nC+uJM&GYp~{{1<A%f9~2|C%of*Y9M!I??&q-ZH13?_b@=@Al{Z-^-sX
zUm8-s=+|TgQS<r#w4dIb`Tvpl+KP+$mG`fms=p)iJ^G$~UZ4GqAikQV_rL$Ht$%&O
zI-}C;G5fxJZTH<;><&|=F0PR4m$12-x=P#RI(vZ2CL<eWfz#nZA#o*-t9XBL-_3pY
z>gwrQ`}b#~_Xlpdm!@uAIm_nr^ON04t5lhn$=7GU+P|<+&inf38z(ud7iiU_`=wuA
z{`~P;`!$B&)@&?Z6?HyA<6NSd>lKR}MvX;9xqky=Zkar*C{=5E`1Zmr>m7ynCfyKv
z%EU0Y-{j6GC59!wDqZD~2e=cSntBPpjA2&r7qPnh`ti-``|;1ZR~8AZ4xZ#zQWx~G
zKxB&2lArC3B{wCsg;sa8oK-l*T6o4}|Jipd&nPj}F29*|^~Kki9XF1s?R$5FVU2Ii
zV#8@OlHR%<vRRXNJ|Z^M{`X~xl*=2Wu2lp|>UQ1cWLUrAW0K-KC5FQ69@D#`S6kK}
z|Fv~r*VmgTlBO6nUv+VP&*X7d+EpjUtj{~p<x=2}DkCOsdpCyF(cP<eo#|9**#0Lq
z=}=mS3cEu>s_M$PBTt)lR*Cp=q+OigFtulsowUB=)SO*D%mOD{`4!Lk|MoLx=zqj0
zY2xzn!Mxt*+6p>5*&Hlg*4W5073`~bdR+3kOyun6tW~E&rW;li+ie%U;=;@Dy+V-F
z$*)2*=jbKXn5DaFY*%M!-$^rgFB3oQW_YD#_t_7z&yQLtln3vQ&V9aj;`PNryO}lq
zG$@EXRQLK5s;^zL)o!0trlstP`8@k}-1`0L)xs5j!%Tk8_52oA@X&V82dl%2|1P=N
z#oyCdE3xs}j%CZre|~x7`rUp1AK!_xwmW}q3zPfNwRu;l{Ewb*v-tYfHWaPd=5&9D
zhZjS~=?h^eejN;r73|vdi*e0jZ5`j#hZ<3=t6pTU+Gk|t^7?T38GgTce-G~7+Wapg
zTt04J^!%O9YoEP~`nRv_#Y(;K*d4X!zy7sqv$C(O-@j9M#@D~=MPkH1XGg7PEjQS)
ztXA{rrU2*Csi8ZMosR!;#xm!}U+%*XyVx3xnnUY0Sn01a?)LLM<Q7`rdTQ3xNe!}#
z?Pl#SeR_VCyXLHgM%fBy88~;AZF+h>P5t#H9<%bSJv+E#tKM`rMF;8$ajG?0oY}s|
z!)xKL7!J>4^1<v3VoppFB58`#CbdfKU;p|35w(BcHZn%c{^7PbX~T|ftPDn=awhm?
zaB1$<pPaGvy`hGJoL&qc{0;|3oVe9|(DZ)M#$^Yij{AI>)T!V;`&B0c53^^eUj=We
z;6m4OZU*MZH*T!rZ`(XM{O2l{d)xmM?%B1tV9gaR6T`AdQ=c8vJ$CpoESUZ7*W6II
znXgSZ?3&7?AjExOW=_>bUS8#$9~RAKxRV<7BG3Kfg7e3YxP~s$Uelp1wQtF$H}8tL
zq?1eYzlEQ>dn7Z|W!B=1zC|nbl&7@{C^EeJxFp6&`;dT1-2>x8p)Hd_r*{XeNqb!O
z<lUW!_W3uy&Ch-RPwDo~ipA#n7sD=2Tc2ML8u}}{IpLaj>owayTiSNLW7Yk(I$Hcd
z$?iQdHgcMWUS3PkSoQbr?i~G}KdLXY&s}ih%}0;=s=vpU&)@R@s(t;Blls4#H&1?;
z<KHPG_O;)3%?ZnQg|~d0!mb`Xc#D6rQ&fVD=5(Gg!NPv+hI0O>K!b4oJ+_-1xDWi!
zd1GZOf2g$TOZeHy+LsJjz2CN<x6iQueQ$f4`rGvT)e#zHpI?aYet+io{i446#Z&CQ
z?>H2>{_Bdj#=k1<e=dIT{QvvUMPkm!9%hEw#oCzMi`a9gY}27ljN5<NG6xjI^#&I{
z{dL*nshum6j@8kuJqw;sduk|Xcx{EI($&1ndqRIFeHZ>a`Fx$8|CFwSi&pG;udQBQ
zRk<|&a?X}B+s;coKC=4a%zbwz8uX`Z?r3>6jUk?6XN2M^SC%9nZ%L+_eY5s;`quP9
z(~Ho~?xV5{2Kt(Zm7MNK?LTM3=n<X%WLZS)vZHTR^p>i(pFJFRG$GW|ab1SVypD>S
z#_Y~&O}j6MK218^t>LP1>gcamIqxpGZxDL>?b34Hwxds;O(|^K{p#hNG{NMWzsG`E
z4I@^a?&`SgY+rjyY~fjhGK;VaDT1s=5(6giPu_oRb;{iBD>#-veb>#YsLN1mwvOS{
zujAMM8pq%3nVfyf#_eN5zzpy8?91IxE0(yf)L@vh?@G-4e-bvgDxxEsI1h=5&QTKL
z&Q3`Q;`((W@6kIg_x>hk{TH{lxAhn$J=Tct6use8bT>Ht^~|%?7ZYsPD2Vi4_LfqY
z%bt7i5LeX2+LGFbGu09=EX_~&{>0?Z^3Ceozg48LZvTC`AYAbMmK&Wx_8)FGp5JkN
z=k}Ih&z(N$;u9ALzWexep3y{)eSS&0?n^${*y}kT2##$GZ|>Xt+El~dW5LWHCp`3C
zaIdnlesO!r+*KdKww7COZk}iLdgbP8PyK)OS~HxVBi2`C-0shB^LIkwIff0Fi@pon
zT6@p`V}A4K|0xgdU-|A{Ydxv&ktLV3^_EbdQb_^JQ>&gU+<N?ERffdg``fd3+pC<L
zG3ljAv#!(i_fIl{*G&lE3pc4cbkOU0oUO~nmob_*_3cmDRlRRM|IcKax>xF-+1t)t
zvz59Q+;SvA@YtfuGsCoY`Y(?wEfc;O=E7wdA(h_Xv7|+$`OuM1dzQU-%}KbXw(lOK
zQo5KDFoV4zn0Eos$@$BYR$Xi6;(YJ>WG<Vb+vDfW)jZRl{4<!qvWmm_Y^&Bz|Le8u
zGI|Qtn6AGU;9EUy=Rd(&Mh1QELJV>(`ll}nckY-Hs;KyIx#+j9M+Wb=eo>lKBENNK
zvhkNGQHS17uiqSQ$&<99!&&Kx3orZm<|@-AnkFgh?xiOys!fbJp|RQ|G|*<j?c!FI
zbB{J!*e~r&-}KVTwy^G%m2LH>mUoX#4$sUH>U5oLs1f!^+O=3DSCDnpX_M)O6|+un
zIF@j8^=tO4a+h|z(yN@iB<9eXqg=jvx78;77WsM6davqKRq-Fv!mYxV2e_OrTI`*3
z;^6CV3q_{x$_|(jXfq>k`nBFot8Fid_xzY$`6u<;pZNE-bN8<2zb8_@yR+cNr|Oj}
ztZz+Jb!Sb~_TG@HJ?Ea5#?B*ga=cvUlRKY;<h`t%^5EO5^|OmMEotOF@cpk0L*(4h
z88a+SsB)Ce`m%N3x{zwsNw4jsqy6*b#JC#L{azn>9h37m)^=8B*^~M`Ier^{ci#Ik
z+4H|}>FZOQl|6q4t$W~l)wn$+VsF^AZRck3-#40LpZ{jgwz~flKk;8pf7_5C#(Hz#
zb;BjWkM&Aq*ZVCxRx13b>qykR4chAENgGyFOh~x=`?v2dwHJR<=3ZW;pw+!VLhvfX
zBLj7drMps7`&@5y7puiJssAf-3}TqUz5SRZ@Bd}VyE}dVupeUskHkrRXK-O!6yekK
zQprhasm6Pc)q!WrF54dQdibyE$kKp|X5X@QY!Uj}`ou2t_RdAQl^0`Hl*q16jyQfH
z^|6%xi(UGsT3?sD>=byKWP9+S=-OA$7!};B-)Ce_bX5~%I1x5w<8&#VQ=*p)^p-xp
z$|$bg>Z|m~CF$l5$+)E%k**O7d6`0&98@~5HOp_&oO!cCxYk~{tLCD5q~#c6#l>C=
z`>BC$^2;R|YW}_17A5twuj_MgdQR8p;OUu0r!v|c6RR)s&f}fO{!?W}m~W|!>*9A8
z_gya(IU9Ium3CK0@GAlB8AomTI!Y&sEnKbSB-p7Gbv)wqy-Q9$3<^7^1(x+^uI}?w
zRro#obDPN2Lw~RD^KQ6QT6L|L>A-i}|7YgLOPrYXnR{{Qr(D)=YZH%3HLHh9dpc^~
zUAo4L@$PI+udsZ<{j8eT(%Bd)%!DI?WrS87;z~9QUw+L^^!E$r3yur&3+C=VB=zgh
z?7q&stQX2>eLnd?L(X^B-Pz`DzrMfTT0f!X{``)=`qKArt=<xRc=^}8PmB8)j@(e>
z=X<A-^KzEAZLwR`=ZV>W^G@8KaXlyXwZ)B}xz_o!BD`0ApAsc<=UIkz++4YErJFJ5
zZ>_K1B4n^gcti4)*N-I)Bi?Muk&_MG{pn=7{c``cpCi_@&)r;-<$iGO>+)4??;;8_
zJ5_jhZ@7N$(9aYOhs3yEckRU&eRCeY?%_Rf>xj7w(*f^^TSOur&i<+%`){@V!a|{)
z>5oce`7OEj2kUQ-@V>Hs);GUUevMPC$$OYQm+b30T%#9qe06t`qwSSzs=KyW1?qXU
zdk4Rgm}bFfz<F-f)OA&SdcJEjsy!M+r!#5m8UzMUN#7#*O!01f{LWnc%N^PRk6q0p
znj>9;Lu-UGA1*Esn$B&yXGL3u8OPzDpB3umo|pvf?KyvL{#;d8lV3Gcj;BBW^4;dd
z|5sn0#oxE7mU~kFRsa91$_e5ZU#$1b@5=aMbE3Yks_NU9Ou=5M!Vfklx;>6PO%h}+
z+}UyZq0=1@+w*E_w+6$3upNq;j(^x01i99<c!df)WoG!mvGHP_ThT_H`9ABeKaUrm
z@G$hyv-QmZBHuqXS8ScKNlIPl^?TpC{U7tC-afy7cW1@neRD-8&apaN{emmv)iq9=
zz*G8~uBL06s%z#RycF|kHy3A*(7II)SNDd7RP56F5VMRUTx<39(@y`{gt}GAJ=X1g
zvFuvL>VNgd>8)A13>EyhzD_nPTmJU(o?jo=vzCj5tvm8yi|a?J^Ru$=R(@(a)~vnv
zr@`@Ckq)sIx9z$Q&*^O7;%5{vjL}_EG;RHl$G1(`3%<J-TU~zcv^L~}=(?)8x9>^&
zYDaZnVZD;7=stT%Ld{WE*GW9DTc;$kZcZ?ETkGMvQ6tr3=D}OpKA+XhK5p;nz4$_H
z-@^wE5$kwX9I!}{VBPK@-Ehv|?&q85@*k(a>j=#aRP)PymoSsFYg^Y{Ys)7;&spr;
z^z+inM(tS|;>BAUcYMlV*znY~e4%$F^Q8OhmMD5(a5D*2^E>FD(<hU=D?BKFLR8^`
zU8#Z}HFntA+*?<~+jod<anH}w6XUhdtg8I}{$|A0=fb;`yplO*GqiB8X%juVb&b`O
zONlyqhh4n1mz}d|x@R?+LE)6Ea#nxr&7O!*VFrcVxsl(>Md#?xlVq6obw+vmRhyC(
zrFBW&|6h7nzWKd_IpmzISYgA#Zb2okn)Sjh+xJ~pe6w`tnvNZ7TNotvy!y7KDOf+g
zsNcEEm4#vBYvzJ5)998h8`P@TSIqia*i|jC$|&{6R?S0>iQlf7#J%n<{C1%yKV4pP
zzu<Id_srunR_aBsS@}p~l8~A3wnLMPZ2um=-nO1aFoEZu5|`=p&^uRm3Adl#;-R-9
z@Ou$^_-@Ig&Hru*F0k{s>Mg~`^*P^9Fi331`}qcz9iR5E>3#pFIsIr%VD??5|3zIq
zE5wVge|)q`ZE?r6YT@ZZeAgy~KhfwpCr~K<R&SkX#({TtgXMQf*{_VAAy`qQB*}b4
zF7fF>?=PaJClen%&F{%+xqg1%p3HsUYiqtO|5$y$eADIN%0(ddUb*w%4X(MKuByLk
za#G^!YY8{TnEjidc2<-gy7=<_Cd0RJDzn{t7%x~Tul%ht;}JuIY4yDBg6558AHPCt
zkDVNh8gm}!oiOlH>GGJM(0prKsNB1fT;J&QsX}IgA78zCe{bb<+iQRK{EJ}rV4A;p
z?~@e;vnMXK=9*<a>Fl=iOGGB-d=kxCc`ah*(aTr&pOa<ix+i=6%9TAg_q}TtcU|J4
zsq(#bf?4#fEytIZ{5^MCuVvzr=qx41hBxbT1d~!r*R5Mq7BW3@qnNd~)$&KBtJcUk
zu5NWudd{HYbZL@3zY~K~ONsH~_R{_PMNj3-iMeTE(3hgiWdAI;fMHhm&E|u_M|0)&
z37k*=m;dYW=Az@TZl8{hw&!scY*YNtcUV^V@bc8%eaFq08-|re=h`n2QP1p@*dw~@
zQ@85ER;|s`rmR@?>RO^$qK=f_q6URqw&K6JV)yoK`L}CFxJQNlo0Gz83a(o1I&`W<
zM^C)(65pZdvtK)>tzI-KT2(b(ZR!#a74`&U#Y4Y!WOW<++FltQh_C-%dQEE*_nuvw
z873WbKH^`umZ5vjua8pYp%2$Comvl?3Vo4x`B>y<;iob)KP`5d82wlOw1`;D-iU}o
zrdCTY-ibZ7-xIcOJS*`yyhGPqk9BHZhH}im@7KTme-`~OkMqIH`tLdF6*HH~OesFT
z+Ik|7)A^4_HXU~CuxcxP_hfNd+w>P*a*OI_^B?pNn!~uFKYhWThnbuW7exR5xt4hI
z*R;1Ov05rM_4^~^x5}{YJgI-an=gDzV~47Jk)lTp=YF;?;z^tbWyJ3FPO6jsu&S?R
z_KhRgC8gEB`_?R8D3vyInbVS|PuI6S;`3PU<dd#&JiVyW?1^E6&zzvcMrS{qJaY3{
zpIl$BTdd`p%xB-e{x~I2H?M!~O{b!GZ}&T!TzAe`?)xhC#HE=}{%z8pc{AaR!tUoD
zSt;-RW`{hV<MZUCMAXrchi)%T4`ue=J!ATJ{kBN&($hv~iwxhrGkGj>GiTEBnU5p;
zCNZu%x2YnmAzRFRw&9{6m84lOO%LZhX<KPr5hvHS>)eb)DF%_lbC#Ch*<>8)=ekhh
zw05=HbT!Z7SU=Z^i7ZQ$_V`^k;dwfJxqbhkz>ljV7jZG%%z5-{?#*pMbyNKKH(j&;
z@P#jY^WSN=3w<w}`0ma=#-6pp<y^k^*}z%syRP2I)U9Ga7P$HQgWJ2``)%K`H}$-n
z*td_NKie`*HywT(dpu|B8e7roi9hyu9Jb(jcSvFKV;(QHhfx}vCEq$3^FQCEY3y9u
zdd5cKswB^41E0roovaGJ!RHv$7-lRL&bwK-^FlFaL(CgvGuDI`e$$S&y$e`=NJH*U
z=Iz(t-lbgeJG}p$x!BQWo$i%Bmt43WP2k~;6r9dISM$lj^~S1t;;CknXD-|sxzi<n
z%0J_x%ohTdJaZMNUzj>w`HL#kg_!+k(q_G8PBeah=4+7QyVZ+jT)Wd}&ht*oU@)4Z
zciCu4gsJlq-?N)`Bu7Px-Os%J%SPdUt^a!Wj|VQu`mew2eQWot2PHn;ZYNH~pUmS;
z4?mfgc~b3C&}`kN!jqeou3XyBwbuXZ+MUrZRtY@II6rUO_4yc&0nd_^0>&qGRrQvh
zi%?VC>>*xcG21#$wlS;k)!~;HB<p{C$-Grrl`PJ_t0Zz4OHE|dkxg4_u6Cvg&y)!3
zo+YZLwCvfuk}~0(TbI-;XNhKV%&)xbwnpR0le`m;m3aC2?yvgAf2lM6t13%?<xcgm
z+}msGtL)z8$*-QVEA8NniDz_9cdTGlY_Q-dnexSGwr|M3=d+e&re$&+Ju##3j`h9o
zQfsBmLzQoroW6PFx|}pyw($J|j$h^)KAqQ;xHi7}yWK8$s;q)p_q=#*w$&YP7TKD7
z%er139#=TQS#{GZyXK?ZUaXT?<|*+Nzso-Im*K+C&K7o?!&1$v8@I;ZcdRQ-yO$IH
zy?p}zPAv)cJnib%shf@*J+=LAV1(ewji(<iw`_=8&CHNEqkw;5wCQ{10GCNC7Mwag
z+pWt(NOVhwf|KC%O&y0^l(_O<{&<seH(__-ovc;s*c~3{{L2XYz4nBO4nt?yYS}uI
zzpu*Py<_6&%9-&u^n3X|&F<x<(KFSbp7DR*cXmg~-Nc_Rd<&WUw5@-r%yVPd#GG3;
zZASU;=r>O+_~x<i5qR`cLxC|y=~0KZQ%TOVT-nNhpH-eEOq{~e{bH%v9;J<SXT|kD
zWU;9o@%j>0Xd_zaI;H3JmzXmf+^*^rH?2Rpc9ndn|J+c%=zm(O?cp!Aqe=ztF%?J%
zacWQdc>3+}uFjgw-44#tjH+&-;SAnWr;4%%AD?f<tq@bq?eIb)cg>FzU8@(rjqER#
z>;CfN`=!<|LMOk@%=s2CH~(Ihq*~a!P5-n4ITHAN&)dq(Yu#+kc%yq~^ha~srEG$F
zHP7_-i@vLDdc}Tt!xtOX7U|uG9Pf6j?5UD@cH95<y^rg@%`|ZPYZ+krXi34IUAl*s
ztKR>#NMlFoo?l0A%Fb2WKds}^PtPvnO9_F06WTy^>Zj%Q>TKD6gG(G&*u|OkH!dyd
zmsqp=LhkPypOlVOOkJ~NYj=@jQ2A_EseD})b|<T1Mvc?||6KpK$ZTWhe%~Ak@X)dN
zW~p>9li)+k<J<JT3TsVGpRIK2UoP_Hz^rZOw#;09)@Bydf}{;9-?Q)Mu|4-yUe)_t
z{94Y#S+0|QMOk^3?oHZ|Fhgf<ptV!8uB*w)b;~y`sbt7#TQ4<PzTRxXv~=$qDJLG?
z-LBUZz+2+aU1PlRxf@@~X3n0S&u2b~xtg~{h;y#u<jiSe23OYQr%t*ZSYtjrwdmj(
zx4ao<%oCQaOq}J-)_pJ}Ht}fLLpIkG4YurOTTOH|L}x6!zWwrtb#r3o&yA0XJ(@US
z`OLF(zxBmdU$|p`&+As|y&2|Kzpg%9^K)6a$^Ee2b&KAuI(Gb0Y|K)3M_>M|Uu*I-
zuWj5Jy4qB{L3q!HH?r<?{+=^2UmSDo?K#(3nsZ~O%-k4$WzNcH8eMyjr=6>rRK4tx
z#<EjCt9iF?D-JRL>obd0>fzN)wd0$0<d>^DZ+ny-^r%Q;kCiv?<r%NrvL!W5=f!lJ
z_`c*gbBS}i)e-eTZMRvC8=5Bi+a5n2Rqo^-xn^@%y==4U@$;+hb$i!wJQcc`!kWrv
zY^kMWcb$3B8Y2b|W6c9Rf?P)pVooc}@K9c|?~<g+^pj_TdqNC-6j*(Jy-R!PV|d0@
z<iwfa*o_@2CbO?@TK)5^#pH)Nx@j*{rum6o|9RCU<-(?tycu6L=3L;&wanEGj1=80
z+Ms>&ZRoO+n~!vwn=-H8jIy^o@=fr}qKSIZF$V)TMW<yf+?bj+`R1B^Gn~>d%v^ZK
zDLrzQ7sHW+LVwTTW(ggJU`5}_FC{1anso3K&ooJ%hP+jhufBbhUViwzvj5|x#F;bm
zR$ZAW;4n?>LDcp{zUCQ1-Dh6-9yKlfwx(mnD%r%N3BF>kSGy;)PKufswq|PD5BusO
zzva*GX^Gcl>OcJCJEwZr72dm_#d;)rBpEi{bTNN6@k&QR^2;+DqE@ae4!$bMzQJHq
ztH<trH($v8s`EBJyK_eFEy27;+5NoDTRZbjvgVv^^;zOkdAx6b1>-x-^zTwP*OV(R
z`SGRZ&&6u_gwV<*yL2k;jQ9#<?^GUT6gZL?_n38KvdX%s>j|OnEL@jN;1}B%!_e_`
z)vqFzM<Mexru{W*@blGNKfn30Udi5MwZj)azUui{+8;36_3Gjc!Y6M`ZjhK#Jlpz6
z0&63?J&Oa+HSgy9iq1K!KQQd<Xkw9DxY~f{QBcgByC;QPJ1;%!d%Zu=*sN5=YV((f
zpRA{U)M=c%efj9Tx*uzIzv0e1+sN3E@S)^GGIN964+gb^Grt|!ThVi`Jm>+}0snvU
zwGaO9{Gr8EV0ru6{TKa{j-7lz?^+J)550!nt7p}Ix#s+I`Ny+-G8Tm!9{BouwY$#w
z?J6?w^WvBl&CBC&-tkOP=TVKE)1H{19kKq9^2HmkbFKw_=4*L4=f>_GF)|yDY416=
zY9impup{1k@3Acx+8O>QEHy~}=;^O)59IPzNKC%w+9udN^;BF-Q!xjl(QJ>fw?C%I
zii(D86Ny;$Vvo`!p`hZ-ddcSP9ZgGIB<w_&D~7o}e|>S!#$&gG8N{le{7$jra!*V=
zA$zp@^o2m1tEquMBpP0uE-H9b)w{H_S-a)L1GCs=ou4mXxLfvCP4i~F<%Z|WLku0}
zGd)P0$+A7kPgRiN(WRxcuFVVbZU{ON%B!L8>T&<XjfK0o#N$NzH^q6oN7(6F8Xmkd
z*I2loDS!2=Ma&1jT5scEf2zB1b!p;LEAg%Gjx~jfDjG&GJn3`_i;&;@-SKGD-tV>U
z@?VV$t1fQhNDh_KiSp?C7TdAy|E4R`4bO!~c}_|%bQN~0l4IB?Ak-s%Pp%<|qvE)!
zo|(!uwSC`i7(~qe(Wa~$ykzAJzvbr-JvS|^wUwQ|-Y{a;#yzVA`?oIj*IAr#FGB7-
z<A!zf%{CR@@k=l7HaWFs&e?BUN>~!+nT7hyytq0!g40FnuI<dFM#&68vHnhGqSqNp
z>J~{A%`<x`7-?K{TH{7e|K6MHE#J8csBL<v<8<F=)0qO1WQikfH%~0tv77I<^|o_8
zE>c$AMmuY2%zZaYCR@x^Q9AeJ&_=0cGnwW+kKCfVF4dqy`<Ithm)Z5ynP;;Wailp)
zur*(Nd$;t*%bz=<ujIV@@z;DxbL6|X{cjdpTr9B3**i5`cJ0mI-+V3~O4o0Cn5S58
zC&{~Lb$WQc>52KZrJw!hz4<KO|G#~{k6io<YyG@`Cnx*F`K7<SJO94uqnl^#D~&2^
z3@d92E%r-9R+g16lh@jHF2q*S?xEI{A3mJL(J#L}5?DMpD$1`Y_K3Rk-n=7c7u$-@
zo4N6eia>gy#Fj|4AY;wTRr6*(<C|c#M<n^;hxN}lUNV|}w)Ae`7L&R9s$P+M-hFfV
z61qG#{>(GY_dGYn*DQT{c4u;eVer-xqkHqNsF%)KE2MtWspChWL28@Wc_W6tMX8%q
zcqNYp)#?Ym`?xdjiO_3H$L7jevqdKcrYU-E6>%217*s44GyiO9a*=Z0@`s;SuU7YO
zUcLGJV*`a_AMf3dkTzbtJZab3Bh8si%91fBS0|Y5`BhZ0bYA?wnu?|Ox)N$+-8L6g
z6<5nnyQ!n6TpIG+&yM$QuKuo_muih?&$y}5>-_B1%eNCGXHE1so)Rf$>|8uo&Uej~
zG#2gCI#(pqlvoW!8G;y^GcTPh%0A9Cjc1zA#Mdia4SdY5C(nH8z~+2b#i%(lEzB_D
z)}otnJ)5=^?D^%*mTks(ooBUZ`^Lvt#WR^4%)huCIpQj)`L!$NZhCRt>i%@|XKOOA
z@mBuXEWg*nq9)sW{+jcDZZ1DL&2C}q1UBz$dpDZhD|>BTxbN*X-Ng~<8%mP>lvl0`
z7xH14V|@7wgR<{smvi^-Je{@s@~yj<eZ21HzgX>Cb9Q6m2D6X$(Kig^JWh({c7$s^
z(X{vA5z{ngc#;wSa`~yZQ>p~#t^6FfUWD<@qlQie_L%*~JSv<<D^^Unp?UIe=<Ge8
ze)Z&5K9;DgU3;4SYVYemEq1fFozvOFykN(SU7zjecH4cnWLUK#D$la~g=3@_<FVwR
z&l+nwQnt)WTzN1`TkeTm`UeZ~&X}x{LlcAxSsN~h=3f2%+jqA0(}}g)ZBO^>{Y+eu
zRkS<0dEx4<dKa{x&DgB;XVVOp|G_^#&%gBP{lAP%JHNY|R%@2mo3O|IniO19Wq0f+
zKjRg-TMpkJsb4?9zx2=9qv7*!wg3NlW@YKU>XOpW#gD6BO`CV`>9Y{;+Mhh;(~ke@
zQ~L8k*|kIbD6d=p+6NPhmFBS@Y&;)l{;Fo~@hJU+i?jAE-D!DQ>m66NXz`+qwrA^`
zR?m-rTPrzlCBNKUm5u57+h>{gKJ#5X>wfLkDGUFv`gJPML@4&|?MbWH{ZfxOthwr?
z`;2jt#~N1=9VLc`ca0fa5;Z12c=_OE`1ih7@5|+Swl9CO^4d4s*@k~t<Qq?LuhB`p
z&ML5X<DVn>I!Y~y`Yj$RLYmLheJU6Qc;;7`zF&EEkM%iQ{p*W7-#km;IdJ4-bs&=h
z!}LdOQZ4R9j=NT8)n~GXtvOn8Q~hG<qND`o${quo58@4M1+CpmI->rZhcA7;Bj9az
zzEedqNl^IChu>$@|0gS*y7#fUbY0(%yI+5WU4MV!<Id#^4$YF&=Ir+?)7$@J|MA_w
ztJ*I2-}|kQJGb`R%^gMu7X=-xmJdJX|Ml(nYs)9FbuaqF@~msyy~B}@<rEm+Rq`~k
zKd{c;&zh4Y9ABQpa1As*IQ8|iXH3d1s{YaTrUl@k=8#x5KVF^B99LVn_C4HqUno+1
z2j}5S*;m<`vl%*n9dax_-R*L7O>$(8|K7qA$rCym>e}-@N6+g>nVgu}boHC$g1<k1
z&hRNW`qZTL$4e-`=T~%Gim}-AqSDV1`%*VAm1PzP^kea6zxXty>Ux^@MU#bGS5ut|
z&R;$hS|{KjaZ2HU?~hMmy8cr>O`Ij~^K;>ss<O^?soD&I*3C)(9`FAA=8^jK$T|1^
zeyn}+MP5q$|Mn+LC$Gt0d-pHn-cQGp-^cCu9OZw$|9kUA>Dqm_SvFX_DR}bjV(F(d
z`wxjYy{np2y`s}k^25OwA4BZw1WR(%Sdy>UFE46Qnyyn(6Z>c1J9n>RQ7V&Ho#Fnl
z>fvQ4d*`<E-|MGjZu&X<X0ceW+S&IH<)zokJos)SrFHbXt!`zE-ikZxKkZ(mlGeI0
zL+N98=toD<ZjFQwhd%H)NSr$OqEPA4+%?@(nVZk~-?Fp*Q^xs2eIC=fHzz`Wdh-20
zd`<T20n6^=QE$I2V@b%9|36K>KCnmr?B9p%@_Ss%-ey$@di%G;w<dD5tU5i}L&-+k
zPSjL?PW{uLb|L5dk61kMW!ScZ(OtxENzs2VKgk<PlMM41Lyi?+y=}YHOhbNt#a)-I
zIeqRCRr8n*e4b*U)2N}hRQRN$WqH8yRhb7>EM{Be+dg@|BxU`kmckj#+t=M+c=N<9
z@%7tu)=UU1-8BDAOxNy=*ZkQ(SU$O&^z)y|Ft@)X<o<EZFwbLi*Kpe$4SMp=D9l#C
zGE6;9=2@Tmid>CYb_SgmAB)YB41AKS9<uUfSk{Er9NyUJTT=_3-fnXaNfuhW`-A4;
zxyD&)kG2HGUf69K*E8Stuvf!W=6IIn?<05`PT$}zz4yBGXzY`oS%uki`hVrlzj#@z
z(_ZQ2<<HrD+g?q(GIK`<!_EU<%T-UgUgd0;61-+O<-^*&%~{e6QHn3hnx-aKixz6G
z)^t^Q9HgRDcRF8nb@tw~P6yV<EMQ<!+@K}O<2>`kgbSi~tDoHuUBiEOrOf-k-_G5A
zIxY2jD8mcJ=Gn(BSFY5nE1ElZsc7og?fg+!-pc=<5V!A#c>Nti`*|<k9Xx52xcu5K
zhkX%uSZ*CJn`qgk&L27deDta4^Id@=cMh>TRX98<O24^g!o(|@nh%d}P3S$filO(}
zFMY{1rjJ+O`_gk`&6M&Zsh?JV`#&S|lI11Fzwa`qX)3*4U7r5=+1Xn5HLK>m+q&=K
zLtE{@TWqh4JEmXYQklT?Ktt-_6jzas00t#1?)_S)x-52HTEBg7f7ZMw^Oo;qNZ-7+
zcwt)RdJguidpB>&_4QqSvV8GjtEThKFLQstO8fWx)W&Jqca8pkdiJh2_o;sBhTyH-
z8hiG#G1z?ABiP{D62R%I(ZciUph5ebzBY-48KnXTKL2@s$#%-VSDQ}Rt=;q^s%ov^
zE-u&ETic5`6egW+Hqdx(yHu=gY52s_c|ly3Wfyv01|BS(qEYs6fnV6JkXhk|5#QfD
z&}Pg2+0AuVS6_<R@|KzVCMKURdJ}_AYaLp)^RtYDr_q|~ugymiE}qsd+-1(jkow2u
z^4hGcr9AHsnHff`J}&37>7=C6O_lY5S9KY6TGmvFPh{ghz!sbP=)H!S)EeKK!-*^0
z%mg2nSn;mkf8tWXtSEyNJ(tjE7pXNyp(i-nStMsY+@Gj<@n>x9_UswYa<iql)#u(l
zJ5l(&h-dCRv*nL&8Fk6Lowk&Zd1cb7q^R&&b2dj#@zON95^2>Po$P+=^Ol*N!K<Q9
zT|Q&P#bDs`_6M))UY>c&-`!oAx2iRoKb`felBcFe$)gC~xrq`--nKo9n3e3?$j1<S
z=3UxmpY>bUvPt%;wM8!5Y%<qy!_K@jEejrAy{hAT_}~lIw=;9L+z;w0Y&@2vU&I&l
z_3xg~(qdU{cMBWk9xyb|K7L-m?o()y>Av5ech9ajnq3$8&$s>OePzD7_w2uR*L?c%
z==wfyp>^l13-9g>y`t7o^5wzN{@b?~XJ0odO*8v!(7wDl^Vq4gU)IfiYT$d>WN}FO
z*(nFLHwP5g&Xg>j=JxecfA-Aia31G0&DKPz6d6~uzz_#_E3Fsa*BrgoCm*{!$vp2g
z_v*XBSC+k8*1IMC*yXn`^G&3S468ehuD^b{s`Jc+FEMFFTE_1~_iL|O%j_W9<TmrH
zgwM>zf6sPJkvyfdrt_A_WpT%Xs-K~u7jE6WYyV>Yn=Lc$fAwiLwNYi=R}%DmyUV`4
zGd4U?o3;Au_RTN99lhD_lv}madUwcty}q5Br@xBMymullX32H`_s??b>#8TW7`oJm
zAM=!4zQZeVNy9uv-^+7l?{C`EJS}(Al$pILX|eHh(~1hE@7z3Pqjx#L!(PwHQgd_X
znVy|;A`=yjQzUf#EiZGVF*XFXZ28`tc_}!nxoF@0jH+1So?ubM$VAmrlVqJcE@^$|
zHu)$`zThkBG5O!_#qlw2zMH37X)XJ{isR&LU-M^?Q5E+d>6Pd%p3#-PN=fBFNT}ey
zHz$97n;4h5;f;50?5eMu)t^`2zI#fndlg6dCcV?n45^=I<ZA9(IRC!d?Yvm?lKc5x
zMUF=gEs6=bq}{ApC?TEZoOZ^*=zxK*u){3L<JP;Ly$cY1*mdOAi4A5SOCfVKe1_ta
zT;BPwpXWbsernMfmkG-ySF~wNVK6zo{^KH_ss47qEg2Qgeo!~Mx$flNS$11|jz%w8
zGx6NJDUxe$DtU9~-R!<;;BS2QLa5K?n=TL8Dto1TX1<O|)2!@G+srxpt!T!T3P(0x
zhLXCV1q+k3djEd8m{=I=@78F&&xiNAQKYSRNpR{JhD)Lcq8QkEcAkrf@tP~yuyEHK
zn~qZF_nM|NmYtg*u}s%#hFZhUyepq|^qxLZ^SvzKA<Sv|wvzp6PFmjfkh3X<+^)fe
za+X&52TELX#Xjq&R%&*8pVrQ+o4L;~ruNG{_UYTr{|nxC`SRuCNt0cjpQ|tP*S%-Y
zH{p5P+`nB-{QSIoE7u<NNtrxxs_o`8(<aS3FBZ9XTCDNm1OwCSt7cx>I!kkcN8GXp
zX=yV~Zx;)6IQ89U*{sNCLN2PObaFSBh3IGh?1?sGNXvL~U?)q2_QdK_)?3}gRya<H
z*Xre;5*3`rzW?O?yK<FwSuZoPwyfbw4&|HkdiQ26MW56(4Yw#pow<z%FW>P=o0l|$
zg<Z7yB+shT9LYyhD}+yNu-VI*uA+YHo870fa_PHa+3D_vJ!$33XU=ZTZZYz$Ghe%(
zG0o>{)b*Ug?B@@k?yTK?b*9{0oB1YF{;@}uKYCjmo-O72>+gqCLfhXTQ)Vg6uKGK5
z*3{+V;pMlV?fI@!_xoq1rMA@J&HCx4%ce`(obOKilyUw1foQqx$ZJY(BllVH<gurx
z>v~I`^_rWy-8gpH_0lv3P0gFG8XTvYHr!P5F1}T*e!BDEDW1ch&nQW~UQ-vI=67z!
z!IEQ>s$yKs8g}?RHpn;Mk~l#+>~xMCUvCKSms<~(?^?K?dCk0_<a^VmG;!S3(GBt0
zSXH8!9yr@NB%bNPgAZ$4RiEuy_Ke$n#l`@k4@xp_uIeq@x81dhl{|K9hmq9Y;8Qkd
zJ)WHFxtC<tlK84)b+HUX=I!9v>w?*P))_xlJur*2_WQ+s_qW;j8~Zl!-2$y-fvsI=
z%SySpIl*9$;Q9JW!34!kp0B6120d3l8rl#l=V<!m&9608bys=*2h4PTUj8{sbLafu
zl6=cv>Mv*b5-ICowD|<zJcDUZ{>7}&T2}I3?UBorZczUx>DBMEI(t5s{CxRy2G_)#
zZT81K%06%U{_W1r6}CU0&zxGmU|Ocw$uzxVyA0WX{$(%-nJYQd=cKkrI*-AeIlRk-
z8P42R%5q6pC}FKI;`Y5fhpR$v_UpDYK67WD)e||KQ_&HeEOBPb%<{W63tx9XTX!yN
zm(KACA1|%tY|vVKJo0F4Y2Am9x4*8P{qCVO?_+zu<l55~70zAlrr$Ok+c!&pvUa{n
z&%2L>+m|hw-f_3~<CDlA-h%&%ChWPecQ3#6ZR_~7$DOs-Rx^F>YS*rs-L~FX&U^8+
z)K%PbH{PDP?nS2Ve8Y-a*(Z6#wx7yLGtl=v8kXm8c=nFf%Tv#Htut<wjA|>g^}dnm
zzcS>4VS-m(!Kbz2=f81X{<``8`xnf<9;<V7(u&WnT01!+;<MX&o;${odW&X0OLINK
z8JL;3OrmaPYuCfmJY9++hxL_KN~}5KG*eROVRwSrPCMz?O}jr-n$0dL`|f=`v!=lO
z|A~k%T_3OA&^sk!b5ZbzY{~uIW@&8SLz=h5?|pcAy`EqG;veCQ=l?V4-#6!k`bF0@
z*4=KeW7eKu&bYh!*Xl!aWFM`St}`hqPm?|VH+I_x=hqhba=hwZeHYJKY|Hl1zkKEM
zp4kOuFMfBO)XiVM_3(ioF(#tPHHD>9PZrMAG)i4|^355(G{@G}t4=KJxXHtCXu^f7
z3+7md9scZdG~kiRHA!J(!|S|rH?CZ<I#0<sHeHe@R9q-#H+N~x^N9k7IRCXO_J{gh
z=jKd~OxxU<xRqz~#ItIB;r_hSf2qD`^UI3}E{wHx5o~^1D*e(#nXUWYDhZucGD|Mc
zw+k|iU=Ui()^O&|@7r2ob>XX`PxS~MSXAUBYnU99cE-Slfn|eOTj7(}-09Uv6PC^v
zc?cTo0&Q~eJs#uHYkWt6XS&%tiLb_U?q!>siDzAvDy#jKH(A(mV%!;VU6cK9!v95_
zzL%b%{%Fg|zuV3)O2~Ylx7mXsG&Hc!tkYGi)xB?9N&BSRXYYTx{O|1(p74KfE>1Sn
zG>b9o3%}~w+422XG(!W2!=4c5-FnGW4;-!Ax_0LCpM{#TN7*-&9_iv>*VX?R;G#4w
z@8c?_N_7MMI;*hiRbkh~D>)YYKEcHBtmJ;lmS4B4{--m%TKvCN>3`}2#_8?f`R?wo
z*Lb`<U)x*s)@|Rf#jihj?Prtp{?N|8?>g_H*Xo*{i4F2nn{z+9?f?F}EP1(L%)Ij=
z3<mFyG~TvvkuMf&OVo*NcinbK_@wQ*4a@%apPo={qw!p8hx~P>^nKS{B=xGF7ES4U
zrLbAMXX~2GIdd!LexLZzzL5RtW`60j&jr?9Y?Aunmc3Rsu_%^v!;ICFRdqIO**uAZ
zA-?ovQLW|1PjS;ZR`x#ltWXdU!eJ?KSMAa2-1m!Ti5I<F`SxY1y;1-BIeEG}@BNaw
zYo2mW>b`8}>Bvaw+MC;|w|`!;=H{uhkC%(v|GfFMe(sw07vI%??Z5G8eq2ph#Ia)D
z6Z36@#pM5S{g-FB#k~Dt-SUmwW3$gRRvETm)~&y{XWnKP&GV_xopV3x<mDcB<v6yQ
zbz6U>qMWVkgpSMlE;+^QFIpy~Y}&f-RZz?KE8)#Wd_m%R&67nkzaGr6RMmN0<mdM;
zcYY+#`3C9dYu~-pUy=RXR$C`&L(!JT4s9vP(7@~IbGI`}EVjGKx_|%uDNmZezBY*n
z-Fy1i%lf79?|b8aGOnM$-!!hH=%6z1YQOU;hr15fgw9sl<5kGU@*scZRsX)S`N_hw
zfBXXPX`Ib>DIs?6k5AvUw)L0@d8zrmd>Hy%eOjPM#uY=I3kr7v41Kt5zHHsC;;$N>
zA2s)0-Q#anAx=-iSLVH7l}eHJwFx_ZUrCeAFyf1W|6H!+k8Zu$r88GmX_*K!!`Tm0
z^XC1(CT4x@O`LY<IhkU?=bW>*@J_oNyCm1tXr@}ys|qoOxgQe+UN3v)BD8F0!OSxP
z=hYZwR&U&Qy7+F!u_j41hNBO<5=|E0db;L9WSMZ>%QJJVg3?ZuZuYr6|L=>|uhXi2
z)yt+Eb;_l`Q$Cz^Z3de`?AqFNv2Azumh;swURY9>YJ1@|`{f6#e^o3G_FM^SKfgP<
z_{Gb|&G&yhe);k-{=4p;{rCS&EBtWo(NsRy^2iAhTj$=LoO^$@q>tg6d0lJo${s7{
z|2jA9_}@qR|F(X#Ubp|a^zLmh@A<tybfbICof&?w=4H-Y_EKZsqUBvT&!%!M`Fdx8
zUw7BRl(ZLTe3m3?uRJGk)!Qk&D=~~`+C+ctucA-?&En-$m|D8@!?R_^ffM`Y)_z^7
zbSF)-sO@IS7QZCdnG4$5z8HBwPpMDcY|!rcS6=q$yw@#ehm?#~u{EWf_@^|1uez>k
zRlINW${p`y?K9rau(saaou~Ir{LPZ9Z>na!l~{XtlS$jF_jZ3DbthB^{{PZ_^26rS
z_WQ5$=kNRTy#M{bKaq-JuVWtX&fjCY*JSRBWpjVW|6OYN^J1WG{R-V#w-#0YJpWJS
zXLs<?|K8sfR$DIbx)pN8++F|M_Oko(wRM;OtX=l)VEFS=Tk*v^CutltnQoLmFKNcQ
ziKzxV4u!n&y>!~<?21B_8BxoOv*ZNY9A}-ICXyj>tm@i1QKxW*lFbKaL|X1#p1CTW
z&G<&l`R)w19R|e`tAsrZbon0tS<COXe@Dy}wX?4m=DlEu+3#E;*wA$_>P%Z?)RJB6
z7B3N5nD-*M@b51EY1eaO+MKJyy0`t`SNMICs)k^AD7RBr*TG#Gx}xunBvkHNa&K|T
zCgue?il!P%L==<T5-YfypL&$0WfZ2oli9X++SAHeJB`H`X6?CjB2#$wkK4!F4uTd=
zCh6VzdfhZGh3U(OS=-KSy1c3K$@4|M;j^bd&G`N&{n2X9_a-sNY%OJYrkQiDN?XNq
z=e0!Be3yf|KR)f;x+!UqUu)?*2OVD7RnH%9DK-0gc5!Xw>u(|18~?7TV^;oe<nN|`
zXHyhk_xxvDtlS?m$gF>-;rZqkpOX8>dD9b}T%05heP&pt<RaKA*bu;^$ou}m%5@!u
zB@djRx+h8;YL05V&h-Ahx&7b8hH<~5Tz|>0dN*&~j2)GKL>kV?U47I1)}5UpPtN}C
z|Fe(&=!-E}+5d>BexNtu+4+B}4R2pR*KcUP|NCHf{m0|=cc*_my=jGn4nw|n*38n-
zcPsN&FIQt=>wACXVB`CP-_OlEb^p)Y_s`#Q-bh}s?%KY`wm%AgTrDuzaZhUs^X+mG
z-7;~xKT4Al;#^}ayUa9lj`K3!2=q|nTc3Z%R$iw;T32A@fzXDJmrp-Pxo-cZ>Qp9M
z$Hqg9iCfApUwnGHb&aazNx?3yHxk*&sdqM9W4#m{|4-tu(j!kzE=A`l!9QngcK_G8
zzx2sXu1P^HlRZD54O_c+brkoebLXEt?mm8A{Q1R)kx{lhQ=)psw(CvZ{r9naP~qL~
z{m;5z{CV*Bw*9Y%m;asrc52=JXtwG8?Dl3-@5<%=PJe$sGVt}QWo7H$Z;t=hy+Zgy
z`@c;~6=Yqee%_|t(4Tk6pCRC}$877pVsR|qMpLt1FFAdpjM2e=WvyV(g-f~5R6afJ
zO?Y#Fv1`SVNmVQjj+X8m*WX9*G@Ji=`IC2T#`^uK-5Wm5T4dzB^V6M2U1kxl4Xfm1
zFHZFQY-Y^(A^tC?@6Rtaf9&FnA|kE|ra#(}wBegu@S%t!2{-R_Mkl9VQQ0-C%8Wr_
zvsIQ02ZKOE`q2jNsq$-cTUUJK-F558b$Q6T6p2-WzAZMtkET1ZI7r2(ckg|%*<Uy)
zYQpZSnOYICah-{qdJb)jx_Pws;)>;WPXE<$o$m7PNW#+${b&Iutz{t|GtR2D-JEbU
zsK<$CTGz@ac3YKIs(2!1c}4g4z4CRvD$MZmL(M0VxSQwJr+0Q(@BY1&=VQ*KjvF6!
zx@Q|5Ht2Xf^IBfUs?(*%o_saAoZZ>+HP1PnP2j0PiJ)5BOlB)@Va~Z1k1grCSso$c
zU0zfxdw$(H{lgn0)E<NOJ{<1fS@h#JJA3%~e>Zhp?WFYb>LgXeub0>VTXS(mwDoPT
zwJZx}%inIF6=Y@o|K;y@b$=en-^(hiwc2fb`~JC}z2;KK#Opt<2lWZtKXd<m{Qu2^
z2VC##PuD#Ema_Y#Q>M7@wDz~5(~reZI~xD<S%mcW`&OB6zq$phPODPpQ|ntVwOOOJ
z^oWSM#s1n`H5TiwS1x*|l{J&)o>P~Or%CAA%KYlImCrA|iej5QQ>Sy^sg#>x;xly^
zHQGFL!sadwIllc|LTG4<+i5TN#<Z<h9WtYg*bnrqNoMZ2RFNKbL+7ew(UOLRk*#T4
z#W%BP+m|P8n6XeYE%tn>n)h7InL3%$CmjB7jbA2t_K=cS`(^`np3Q;v@n+w@-Z`<w
z**<E;!FPIZ?t4wLUL49)_~XfDar=K-_1~^uT>1TEy1l(c&FL5O-pI>sUB;gI^W9bZ
zze%+h-@i9sIpYDp&eBV}*0KvuI2r$BrjOh9?I*8(eOmvw_0j*--*fXVtJSwYJCvj8
zsPN|To0HS0&(K*Epb(rDsdl*}X#QEHg|k+63ae}V|MRNq_S)x}(@wgY&9;b?do7vV
zrhDkl7rkZ278>=ZN}e|I-@I*q^zA$mK@XAn%T}HVx@_`%m*VCrXYLqxmaPln2{YI;
z<Gjn~s*GZFZ|-Ngdk-bd<UQ8o#WXc}xrg(kU0RG`s)rX%_B$&+b+WbRt@nlzAx<8v
z6((9Q)Shj<WZ{<*&sheqtX|FXn$>hma1ujz&t2Zb!8~TOJujOymK?u4`=7OMez>Vb
zLe44<^N(APgZhcmy*oKVn|+U5eeiN~!0b&gD|9YB(pnKz>MC{3QoSLRae@E&<9FXC
zpHE`h@UHVtq{hsP-qGnL6VG0pw<T}u%yh}XNx%HUgC@%`tkl_BqO$Y6sP3MqMdndQ
z&tAM;R>->g?v3jqwf~=ey!&AJ^WCY*HI>WHzp9*dGv?w<!SiY%Hdp7(mQ<2hmA7iw
zgge_Ac$Q8)tEL_ne13w&vU5DKEu7zT*qS2EnxFr)-fjJMv9P=0<&Zlq;gc_foZXUG
z#e4b8lFA=1)}DK}?&vLx`mh*Q_V)alId5v~&V*dFNWFaPchRRU`)Xs$^HM7<tLg%K
z+&^E)t^cuqt9(n==8U|R(eAqsFPy5r_{8OV>+(&HUpzl=&g=I@ORaDI`xdsUKjrSH
zJLggoegD2y*`AmEM~|EBey-pC+_!({-+p6jF1fUK<=u|v2(OuOuD{Oe_vJmyx840V
z*6ne~?4@UA9V9igHt#IWkYy|G?@ecvR=pk8Uf#E&`1H4LA9re4@3S(sp0-h8fitJ|
z%*LWt29d*OQg-G>zE=8vG$(IM8KXz=D;~?o6CQ<V99qBk?YA=qXHsV|H%#o)`&v?M
z7MtdA-Bv1P%F)d|4pR@F)tS3HPTBjp$vcfG(@7_b1X;}&o=LH~&U<u*7W=Uu2^Mwb
zbLwr*rYe2=zwue={69ZG&z`;B=kw(UpKd-#4=kJZu5M9w+y3o$ukQH%;mMB`Pfcyl
z>l$re7ktz4IlH{STzt;%ul(ZO|Cg@YI`dO>>D26q_Q=;K@4OUG{Ul!bN!*g*wXW&B
zD(m<EcP^UduzqLh1eRCVW?#-Ut`m!&`ZCtY=+I4_!zxpk9u&FiBOF?>>yEVV|F;^S
zOJ**QTz37+o5rM=*OEnt-=r-{+VC!I^S1Z-np<T~aV5NTNSpPnWF@yrLyp9gK6`1l
zGap|i|L)zAmiqC*OZ9a2+hz@A(~f^yn(T2qD<vo;bK>H=8_ulSFnwM<hm-C_(Ksd@
z^A#H}Z*mH>uP*wxtJ<2CQ77HZe@)c9v@3ld4n7pKH8m8JNK4+EGQ-f>Ya>%c<V>k~
z#~GM=rr#@@Z7y~199Kf_g{x}&o<Vkr1upZJUvuzs`lCmN4I4egs{^-3sD*VLymTqB
zu`TV#^zG)i^X8gF_)VJ~#n`}b%_7hL=3FCYZq=^>OXAFf3-`XCd*-W>cyg<B)w-f{
z>y~pjYqABGB<F3gnG~hR;;7A3*6-6PRd2Q;dxGo5+JBEbExDw&m#ME!&zuNapUbj=
z;l;95MZsyAZu2hI<*t+7${{4kQ))5WI6r%3vAOTugP&_Ie~!I;=jPqavU1VGDnhE0
zF9k;IelNSN@B1&^!%wQMulZ#kH$D6FYpKQTiHtMMJ~Lc#kGmeAQ){U2v^|aO{Iab<
zGi{~q{{D|D`V^e}zw>HoqeR21_xbi8vhK-m`*zQC&*IyQxBu<`@A!P}rJwJj*KR*{
zyZ>8{?0=oRvwXL!`L0X)_2%H=hX<w4)l2=lUu)nxY5J~vx@^gso5Es@#od3de`CFR
zebn)tS8dkaygh6C+c&i<R_tytp1)_I<l$YD`<^X+5t*Z3CA-yNPOQ=*t38QNf?0zZ
z&RkfQmTuS0puuL$t7hDMM|el*>5`+VQlAVnH-zy_<8??skvi)|%^f3Y)g3Q&)E<U;
zGH}mI^$yOO=F20-khCFXO6xnb^XY8nC(~cwGbq{Kt1;`7?CFS<mw#?vy^=WbsO@zV
z<!UoKiC6nKugHylnpFL6f92no-Jkv0KBqgse{_D6yuV&tkF|gH+k?_>-(3vPxWC`?
z`QNnlyL79|S5LUT{NVb!FZutP5`-laYj#AeS)SA$w2oQl`xC*F&)&_|xvcM1`@Hp5
zlu3<77N^D8wVhY*`JLYR-E9BSAAUQ9)+XA{e&#iWy~a@bSw?o7yw>AaCl&oSw<eZ-
zE(tityLsVq%|buEV9tb}cim<+eYWv^8q=Su=6iY0>z!wooIb;&$rCDG+P-q;!4Kyq
z9kiP1)-vasq|WvCDM1FuHs5qPI^`6n_H&W6FDE=_G+bH!Jb9H;kLXe{-3P|ba>9kK
z(*AZ4uh~=fc73?tw~4(ydY)eXqdS+E=Dm1wa*^^hNrsRes}4o99b{1UT{*!Z)}#9W
zTiMHZ0(tYD<8vRqS2APB7QUac!p+4lqIvmwC*MskEe@yDiPcrvy{pOC5HRWJiudM+
zS6^1*)lTb{+}r8%L?I@9!!h&V><jvrHr%_b8SEN+??m-wNez3B!*8lq30#U$JtSeQ
zv~uF-uu@^B3CYWsURe`)&Em)2KO2|*{nNj9(^;3ZAv(J}zeMfsJaYHy)VvpO+m7iP
z`+oIV_V$>_g`KHJ&AjQmgwM*<XLGI!J7}9X%lhm(f1|9tn<_j;zMEff-+5Ky>dtdP
zmlL?1^PlNVPSOgRIm@>=|6uRpwx^NLn$s4ulz(oGeVu=PkKe|pF}wchM@ajcMK9Sl
z^Qrx%ogr7!48L}ttiPwTE@x|YPX5ooSAIV@d+eO`xqsrT-mR(%GWM4;Gr#kATV=%S
z{dcR5TxHLE{6ly5|0D0d^h~o_|NgwmS>M^0-=5pEUEMbAd(^6p0bl<<o3?zm{OYfQ
z4L_gsMm_dcnD+F?m-xOseV1aB`Yq)_oL3ng^6zEdmc49L(tdZI@SVt|+%wPcTuq&+
z!@D`pKy?zgQ<`WqD^o<zbFHRbWzmP@?mjF})YIl?n%Ku=Tl&00^ZU+IBBvuH0$Ca)
zrAj9)Q{1FvvP~yBQ7N@8O(`~1YT3HfDSG{1Lf-gqVqbYNZ^hPitKMsG{dAH)|A+Yh
zOEsUSv=)7@s{YQpy`A^&JFDIIW>@bP%fDJ4Rl3=Xb6MZ{SCZVff9wA{Dlh-%9{;{S
zH`C?!es2`L7q&8jA$Qpy*MF0~cFwc=_xhCJviK>*sdeR+nuXUtB)irMcg&8yQ*!y8
z^_^SO4vKHyxU=Hh`LyU?D*nc2KfIQhv@!kbXT3`Xm$MnJY)Ri#VByPfWYZPiyBno?
zX3lZuko8!e<Yw<J6}hABOi<DWw&1jgwHs?4c)~tcaX6TN={o4;qVqKAZcn*vLZITp
zv$Crk8JGU6Rhz}t5Vgg??eD3t)9<al5L!AZ%r1gqO<wbv&o6`~zCHUja_-X2R|;OH
z^e?&g^@@q=lB0}gz6Pa>y<Kq3^6;60@6l^*ubwdb+C2NmZjPt-ZBMJ2eKb89<I!;4
zBAP?<NeA!xRom~~&!2Q|Z(m}s_+f`h(${|f_C3Xv^5~1h-Dh*%c53bPJAeBYN9Jpu
z12HSETx)ZAw}pj4Pg?y_ZLaZ^>foGrTWZ-75@fbZtUEqE@a=B_>zIot&K`W$_iEFr
z8pVluF<uH?hi&w753mMsX)aaB-ThqGd{XtCDjV+>SCw3`Nu~|2O=Vmciy1P=h%sp>
zWg4sM9S-og>bkqXO#bh*?WIq0Wh>83++F(QWqa-UV4L%wCF}!!&HMkh`PxV4|8M86
z=lk@T{iJdIyeYpv6n#7?Ut1tmtG(-QSjp|#winocn*3RQSe<L-=Fn`$1+`Zmy|;h#
z-hNZHd>MZ|b3@c`zFhxEo2{p}FXOlBewlv#e7#dJgUDz3`!@f!Yy7XjA{4mt|MsJQ
zj=TPQdiOHV0l$Q}`5SkqhrXVk`sIq`*ISQ&t1~#2@6K}d(f$7-RYg3s>%+W~xmBX`
z<Jjg}zWSuMa=mn7!Q#~3J9`dRF=V*bnz0>-nY**fbpGir40Cqw+J9|5r-RP)cuDIj
z{@XvVehYoOUFo?;yln5mvybaf--r*<-Ln46>rK1k%f8<}{^!7VzQ0xt`!0W3G+U-l
zLc-r~eqOx&{}+4tIg)EX-8}#CeHW+~zc*2|yx!#exl6OpZ~74#_N0E3&7!UAqFz_=
z-#Y&Ps9?id`!%&ycK^QA%;~)BawIhE?RRVb7he>g?ml*Y-n2c-uP}U>5~cKLhsdgK
zrKlC}PH#I=xHIMY3b%#JuLzhidT92n{gt$-BxZ3|zoX@Hv7j%ToQxS(h%2t0>&B6`
z_I>Q}2&Gk9N|${Wz7^HwfBtxh;0h`G@WP9mu03Zt;OCvS!B?rAK~I`napR<2>suL)
zB<<z?aykyWm#ATDXTDz6oQacnN8C|ln$a$LXQkeB%UdxQ8Lk*E>Ad>%>*_@!+b72b
zPGX$8(LQF&w5RQ+aUt^;y?!%IoUtS2a@zVBFV~$b*4J#l=3Bxs;jP5wi7thPcfFI_
zc7Np;o4DrIoSQmDB5XTsmp;=tllmquKY?M6b7|$*pyJx|t5q+p-oW~tm*LFS$5(4T
zN+UH+8Sp3`O?-W0@q;sgHs_OPy{?{D|6se#dyhvq(&QNWwdKNhe1CA`ovqIA>gDWz
zFRuI7`v1F?9z)^3Ki_Nu?f$#|eYE`jwRF>!=O!1;oAT)Nt=(T|T3;!%w^tQNt1aEV
zH2%zF@uPpDY)+m%U|~|1Ra^SI;LEj-HTNxk%QzVBXG@r#^F#dRmu~T!;g)~z{^`^?
z7xY|``NFqIoyL@x*L-zCPFdG3?-c#@QPHXVcGvsq2D|;%|4OjhWfHYpKTzZ`gI`%K
zgXQf_pQrEL!nmUG{MC;xd<+-5-j!A@YYjf0D#d?*^NenL+~N=4mTkSfdar)^@p1!^
zt=Ciysy_etdDMR$<D5V8+wcAl_MNBEqEaK@zVE%kw=EVm+e7Caef;s=?s@Z>UTohp
z``ttJ`G4maTyuY4TX1nxU1XT;k&SkKj#KBocJ4aIZ};xcgXGkY4-al%`>c#DZ0rBm
zfuG;+V%ih6F=DQlmHp03>tBlhd-=0M{dL@&yH^cWc*S_<dWZ>zgljMQ`h0?fLyXr|
z-;8rNQdv%4+FkSdVW;8ke_v{RHmqEJ<z4CXHBEk=v#rg_UrF4$v$K8Ox<5B_XI(#T
zJi-3j>UH6vr=MRJe047)>TUSdl@rdr*4VLB=+LU;#t(J7l(>Xnwk^C8!|-6$wp|)`
z>v?X2&t&lw%zGrwx7g>h!j2t0^Oq~JUaFQ_GxzQ^4e``fK1WS&?#O@X-FjSlT2hwl
zq~(*|PhfjpQ+3bewa%1W|HW}FS866ie~Vpl`9#d~)GAreULp3CI+0e)4z*?)`x7Kv
z-<!8LrtDcL!7m)^V*GT04@1gkPo+oS`qmmNPi~syE3?}4y5eSyYZE;y_C1~T$7;^Y
zB(}>eHkVT}xvod8vNADe_R&f)-f_}b*?DGS<=y38+gECcG%ww3;2V7YvrgXC;>G7K
zv3*}0nK8@T>r9}@_02JxXUvkFcYaw-`}56*K5zScV8#3DOXh8L$E&XJ9NxVk=SbsB
z*3TtZL?(YMopf%#nN(TsGTGIp|GFoi6pxpRoRGeA)!ANy^UIE=6udrXdicpN@qg#e
zo^&?*c6Q^@Susar;x^{Ed*5+>>sbG!^pUN7z^=#U)*tUNT5rm?%0HFN{@46l&DpPu
zBcFZJW$2Df+M?sU@z}1FODE4OtXVm8uIrga2Dk2oU20)@c3SI@j@g{ls0$g9XN_cy
zo2Am2GuKHv#xifPjZ87RJL#F%bH2>BtOu*tK1-SwAi8j(|M~|rJYtu{9(-vP9cc5y
zDmwk+k9!skeY5VE`u+Ix>Eprk?p+yQjI)AR7R-o`o>%tybjIn+QS#L{Ha)Ys8l&uQ
zbN~CcO-e6I_^;kxujY1lO>f?*{$2cW91B0&KK-m8B`bd5>D#mS%D$)jpIe)kyzI-n
zO%Ik`Os}kXH225x-_y67UlIKD_Sdv;2ajL8eEj`;bAG=2M*kU(|KFW|_U`M4|357H
zz4z$1?c4v9mt~z3%N2ca$ENPqomo8lSEb+lxohWt-#OtDOT9L}O}X1PKe%*Z-ix!+
z+WF@^pUiu?)%|$1Q$umcv-e#OPw8B34Y8OW$>rR8cur+>p~UIM<;^OyedPDY-jABb
zUX;5xk*{=$SLxg+Mm6K^J1>O<jTdjf(S2<-$6N2S)vG<F*0ic-B!ynzG*54ho_wfi
zqUwS*mljOvP<bof;3ewd+G=8Sapzg30~aKNd6LV&-!Lc-X<q5JO;hC#Xp5Wt8s9(X
zMFd~mi+t=_wrh{=i$5t6C$_EURb&)kpRnzm-S5jk7Js*^h)R0>?a$lCds8RQD7u-Z
zW*AYvklU!0y=aT;&K1fG3a426U1Xl!iT`)y<mA0wU)7&kwQ+9=tq-aJmBktoY?Jif
zPDxGZQGc|0yWg$V{yY3c4h1uCFrE)~3fQE{WbxgCfz7`(@@Q13r6a@ChZZL{Uj4-6
zAjo<Cv(5Q<mxE^>nk7egTdb?U)6Q|{t(|o8f0dr&H6Y^su6(~I3^NYCWR+0n)APzS
zUXj<m_-MYr#O_Jv*1Y#;u>{Y4=qhk4F~Viu;Tz0pU-s6U?6uhXOnu9r&kOE-(3NNU
zJb7}u`mr(rmc;yfQMv!w8P=T<zt(#xd+UWmf(na!c2;c&+OlL%vV*t#^gUZ=GYMP1
zSlMuw_vwxH30-2lZpW24ckU_W)^6w*HSwOPaVKrYE*@haz2&{J+P6Rby=*Y?Ay+ly
zgI4(vcToow@!I8*N14j5_p>`p?kl_hlF>toKlY4P(AxJU)iZkKtJeOKUVO8k!Qt3Z
z8HOuS;xB4H?Ef_{%XYT<(xf|9cjwOD|KZhBpV{*E><Vs&j$ZvXpD|;r|G#QM+b>_=
zsvmWo|2WgZ{=w{b4;{Z>Fj*GA<l47VW#Q%47JS>J&F>!eyPxs@8OPJq2Tw0Qna;3o
z|6jfa^M3~z^Y;Iq&Hw&iw=P5U@BDx;@dJ=q&idT_%m07bf0kj*_hozBUnQj4isqOr
zOna}U<o88_iTNGF3r$m}@JCb2m<$<yB{Fv$-n0D5fkhk|o3rb#oZoln+>!YSGk+Ir
zxbBeiVPRMzQu?Iyor;oLo2IpHY2Bo`dX;NV_ZyWTO?Vi}|2-@I1Y>vEtIMB1&RfcG
zL)v%at>angn*Sr7^X*K$=%VSW)UUy;(eh-D%&Uckd=4VruTqxD3z==~T(57YdC0fs
zu;R7&8$E&EGF3A>FK;`q_3{|#bWXe9I}Kk2Z+Z|RxaaZ?nf22G-eoDD5N16vP5f$w
z<H>_oZ7~;Bz6%Q$yDqko)eWA)-IO#{qvexkNqa=yoxH!d-hQ)D<;u%_H{sW+4i$m5
zzn&aO>o!n%${xMn$GgR0$Cq6trJMHkJUNsa^;GRZ%-!y3yTqo-SITKCt=zd!)Hv%Q
z(}Ay2qPtcVMb|E`RGS^~^e9KHX+;0OUw+2ZE~Qy~dHr6l^q%c2h7Hfuf7kgbsk9r;
zJ{a9~EUR_<_4MTIx<}4m=B6?H%Uiuf^rwJbOX=c0$#ox!q$g!FSZ`*&{xY<_=6{P>
zPnz)&-LOt42F>>uICSM3#P7De`}Qeet*V%3!_g2vg^3z1EEjHUWq48E_{yy!G-7h!
zvPBh3mkPOx@N3<9X0_Kbk?F-sW2uF@o|VfFRz-JzJN(CS^^=F1q6?2sC^X!!_sl+Q
zE$fBVH@l*K_hr?Wf0wt7k@<UA|Nj3MGcViQznW9)@7tffwSK~p{eLe%+Ur06#GCN7
z$Kw9)p7Znld-JXDCEwN?Wc_cx_-K1t@bCF|OZYX5*$>S9|MK7Q>0$A?|Gyo+YhMv!
z|Lgbe|5xoV$?fR>cl!I>z4yP${o9sRA08-|V{a7~Ih(KRXMg7A_&<eH=H7k($Jc0$
z*EWuZRo`Aoy!w-(9&KL8I&)&!w0Cp%+sx~G#MI$^vq#ygmhGtx=v*4nz0xt#D}Q$J
ztd4qcTD78dk6NA5jIHxj@BDRnFY$FxCX<2N-z}wKxs|H?3;rl9Tw%Xqn)#Y)sWo29
zB4a%n3eIInuZdWb%4x~aySY_BNdNp7$Z6NG{YcHs5AS9~2F~~BoXTk(mOJmc)86L~
ze3YAuUfl81Vmp09zKdb2{8R>|n{&9gCT&>sU|Z08k%<bAwRK*u4*#N5@~|)Ea%;x#
znk#RgoaAsgX)4*N%U`%-Pq#qbp-qvugEk7b@4S2|_u3pQVHO4f&z<i$*Y51Obx*f^
zo;ia{k)x90^vv{4H`5=VXcbn}6PF7Q`}Szzt9>rvffkAlu0B4jQd%0u8fV<47<%r=
zG$`h=x31i^wk`kTRU6*>Ro82U54iBfl<8iPdT@Kqv=fyt^1jbiyYNQ+I`8x9`R@Cx
zuGfAx`gMBW<(T-p3k<6L-s{NiKCpV-Yle@ZkG37GQfbfHaPP#~a7l5-1)^8gSI_+I
z_xW?etq*JMq}^RyZpKDUm(p-M%@oJA_MKtGBbz&|D^v=fJnx&TvG3N2f2XAW>BwDf
ze6`Z}*135*OryfAW-Z%u_-f(CB^(Rq>`xRa-?&Vy`^MeNjTSpDl|Ic3J9PhQ);ES1
zo%-2t7l#!zKReF5pz+?^ELrB#D-%}kcHJCWT>Aaykw0_!(<|=(ojZSLtgm6-=`GAP
z;eTGWmsi*C53W{UvHk7d^0oV}zL<9{r}g&n`?oGnV%+_26EBZ<<(fy{x0XB<UaGVI
zl4#1)sZqynZsz{4enw)i@ADJy_W7-NI`747zolGf3!e3FZ@>L!@yZ$=1;LNozF!Qn
zQ%e<<@b_}uZz^Q5YTYB3r#A|_J%pmV4&V9s>gKy%{j*Pg7U`xu^OMr4zB=o|*Xg`m
zOGQqf<63RE@Z@T#HKlCY!pqtG?%j_+_aVDlFmB75y%$B+8%0!Xm1lPQyG!EG`w53+
z4_tZZsuK3?UDD4#hb<0sIA}T_xNosDA2fmmTIhUv<BZtYn3&Gn+OGbWP0k**oGHMx
zq34i+^kqqt!|Iz4o)TZ~aaefuOxBNk!out#mK-cva#nY?PlHFZ>dy0*X6C(Ea_qz*
zYpKq|7J0p&SEshtFFp9`M|6*5U_tb^zO26L((gZ&m)(my={k$S!{trIzH`%R*%DVg
zxg<Az(_C$Xok?eAR~pYsWr#Vp;`xF^DTY%#`=0NaBDr=+_!8bv5$h7Xf@dsDeOdF?
zLRVAF{o<p@u#4f5V#3ExZJJSYesRSGqkCqN1z~4oQx`53)}6~xFy-jl86{^s7g{$T
zDw)!H;#1o^y|+AHmOcEHZhUXkqQJTN&l?~9@>q91W%ES)&oci{|CprW^<?&k%jw<h
z_Zlt#seW0u{5s>Dqsjj3FHg!`u~c~RR&8I;uGN;0(hjpOVCypZx-sw67p<qV=AWN+
zEUtJktF`mXQZe81jM(BMvqG79W-GE6IT%LlGB~CyeBfKk!Pk+xzQxC8KP^3XLMf=H
zbp5Tuph(x5kW+`*F1s;hm_3R3F2Lm!p7p>x*x%i;Hpu<fmu%n9;j8xkSkULwKl|Tj
z{y7(7cl2Lgx%8UfWbsQDyX%b4-SD~Abn;HV%H#U%Ej9;c`tMVIxH7d?A+}`u#>=1k
zrbfNzcoSh<5?bE-E??^Q?X<_{rx&~bR{!Nwecbk1)>+mE3>u%<Kf7cE27gxj`pkcU
z&BV(GYig#=t&{xs!t7_3iS=2j>0f7A+>(<m(e*wWcll1Vgc^g9_1#PAOFS0|w;$=A
z^>Fo!Yd5aREVgtLs}4zy?lQa=mh>$5Y1_+L%MzDZInP?QOmO+iH91l{QqPrM`JAZs
zHS)Sw|L1S)^%1|$FE$bFUH{_B=}Sp_WfwlaDrsDjJa1`h-l}esg_~!exij<PWuE;)
z?WSjUUC&)|xo6Iu?$r+SG_G7>WQg1zqCfp~eP}alL(ct|E@n>(Wt~sf3MH(}(=@(z
z=JKJZu2Txn3B7#Pqsf+Cz$Q`m{lxd^S()dK`j{Fi+yyO%Gd-r}x})>|O(l^9e(9py
z<DdSxHb3TRrxWY#Dd#j|@2B!GOgTOGK+5t|rKD2~VLW1Cr7y4UTD3*lS~oNH%aQ;8
z>VoHKgg8Zbdj7htUat5`V%_!m$CoZ(4H9=s_&H<UuFkWwI%7VuPpnxg*!ApJ;m(Sc
z2a}{aS|@SKKZ@HZ{gg5DS(e1~?$Vc{45xI1y!tfecgnbYd@ymFPi4Tr*E(|Ve_K91
zT6KAI*U^O*-N!xF?Kj=cd-v_USpoCj`Iy}=y&tno_U4Rd3oq^Gk1iLsw3vK0i|eVR
z=3Tcl0t{YnB_B=ul(#5?VbKR~hk#uZR^M{r%iz}Zju8D4nh07Nv1L-k&77GzR(p9E
zPK50gFq*hw`dg07y}?ocs#qQzIdJiG*FGWF$*Yr{!WrauR!)_BWqtPEp0D%2{NKEN
z+aB}w|Lx^z^545B-`9wGKd=7#-CK_KKdR;D$^KDa`<~&)jI;OhzVY?W&$~MN^Apyc
z;XZojx?VpzsFT^Ve)9{<hLcAncSbQVx7uz!z5UmzKL>RR+ix{TCf)lx^T(#L&p+;;
zn=Jn9?)<NL_pA?o-n-n`*kIH8PdOV5=I6bg_tviJ+qK&dqx0KuKYhBmS$*c&8K-u>
zKm7gh<7ww_e)4~=Ze_o4)$RN5ZC3O(AC>ccntc8JlMMf-$x7>l-nkxJdv?P#X%~%Y
zXD5Yj-(`BFc_QC|+w-2YA2_K0cHWy>b=HEX=Y<4?6MbSi7>^|CEnV@7D|Wy0x9~4U
z!ppyNapqsP$)8oC^!iWB+-a|0t$y`%?}}9%tLHral{1x}d#bRS(1D;u?u*<Qx=*lf
zSahkfH#n{SZA4CzaQ$oWKI3K1X&&k2)6dUao+;(WenohLhx2UfefL==oYAQ|vo!tV
zc{|^G_w&PermYm!{GN9!dCKnd8&fu{;%nBKyK=_4%ga(ljwEyz9lRvDJxA&yZ>`ok
z#(?}Cx!d1)L_cghIK`r;^WZFX!RVTkqH^ut9oM!=9agx;d-vljtJ4h34|!rP>$;}(
z-OpW;Icph{!8^kUBV&=~mp+QJ<{M3qIE6YhoSyD;Iv|MMHl_8T@1u2#3Y`8u-ePEC
z{CB5c{|pu9@@E~LOBB6cUI^*fKk;#T+h*-eb2yfNdh}6r_4k_+W#4C>`;}9(^mklI
zvc^t_+PdG}vcl<-Hq)y6>ZcuS_sDObxo*k!!*l)2PqBzT5-eFA_FSWTO8J%&r@mKF
zR)NvoZ5ij5DWBbR^?=uoz`d_ExSh*w_SQ|+tTpR1+Ss>o#W#ca<PE3JM9kqiZvO3X
z^FA{#-^XrgF~>J`ZHVmJbW!bA&oqZ4aa-p`TnwLbB*#pJ!FglHIs*|^Sys2~azpQ+
zdzoHWGa}P+EKePZ+fkBZz4hHW)|_(p&?o2KEC_xrwM+T<`E5Vq)}7l_@-9O+TVq!G
z^*>tox>{79wa+R!wtr6keYe0_Y_3uwYcIBbRC>SM?f<u%dfi`rqTaQy{WUXa^O}V#
z=6>7O-F@!Sk;(tAy}f5rao8`XE_=r2=G^|NGZz~F_war0Hhp#^Tl4IjcV3+Oa?0d#
znt65wTe#0z6RBmHN%}b=vmR!*&F>68D)BtydT{;rsd|^@JzYKd=DUh-hd#ctGPbE_
z_h@C95a#!8=9QJ23(xi3PI&X?+&h`2{}v{eF1#}9>{`#-*{%CUzAbg*kqiurFpAid
zv_ZmnNsNk8+^v1mdzE-sYaIfmuRrtZCHdY)^4dkGZ@AX$wtB%Ig@#R)KCTm8rC$AM
z*|E$2{PDlJq6~A-t&b~~U^tx6d8PE2Nb~D6Av|CD?tl0@Eludvt%C5`KV~H#PRM;E
z&DZ$qU|^e2^i^g(TdvDKQ{Sx;dK<=bEpIaOhb}3@h=}0pL2rZl%6i4W<gDGL>Kk%*
zQcA?@En%YH*SN2r*E@5Cr)I{#S1)C}ubon@td@Dk;e9^1tD{3hT$`=ii-R$M^Wi41
zJlFWO(sqfyVXlwL*JOQMR1g#xzoliVTV3zLL#IL}ADU;gaN<0x)ib1(x;hFrsr&!<
zck(pTuB+*m`4gnpOq#=ODENGNt=Tdq_vx8ut}d=nSgg~<zU9o<tBI~LiK=P-?!iJW
zffH<2SAEx!>;8Os-{m!bTr-<{udTbdsl(etC5P?oXOnZ!_NOq+YiGWx)qP^?b(4tX
z*5!*&hlQv=y=G<Ww(;D?wPOE1#jZ*D>Xtoyb@Jk^tJ&-f3$`4%dho4TzLfR$)g_@t
zZ>{z|^L_k7eC_>Te;#gcQ@6IiTkkISZtK0xGaqfZ(0;)8?}Lr`Hgof@&c3U@d2YA4
z)nVC9jNAv*4~G2vu;yES*}Xr17SEl(@3!Q-6SL>8y7N%+^6#UZ>FRT~AK3A{vVGtB
zPs;tqSD(tI-+9b#7hHM&bItkJ&4pK4pPxxP^Znwt$lK|2ewvl*d|T0%c$T~Q!0O6{
zXCM4dTz_@tjtGVnlPDL}xKp>Mm$M&Vot`M{u(6~2rJ%1$zCq<8f!vsvH?Oz8y;@P)
zBz*fwSm@#C`UvI+x9q~M6n(w6{CV=K_=B?^zc~8pb-*$I%q5rRUu<o<@p@;#%O|nr
z2bd;!Opo~}<}`snOqGrGN4C}XuOEGL9@(pbXGse_v~n<BnDFr2GrvBwNWrjxWs_bA
z?OLSeB=lrmxM18*<Fcq*cUI=yVZ36PqO^cDsL!Eje}~8^p>@|KLoa^2X)xi-$3ow`
zhRcMaLuWPXT@&dGXIDB<>hb6G+WC%x(?VMq4txkQ`@N}RgOgC%b}hD;MXN=R9a?qo
z!dI!noeKn`oSu5eMcQ0gq@}a<9CP4T`G$R$ZMu)Y-Zx9+{jU6V+*fNk6xF8e__9d4
zDAs<m?>XtX66Z>1r#gl`FWQ7!BN(Nce5WkWcKxdAAh?~uW#a-#k3V@Gw>9Lx%uRcd
zcRB9zqKw;@E-sb&@$$u_cLkkCoIEsJvvL_LPVkyV$rTq#=1ya3xOi*s2}i40Nv+>9
z?q|-lE#0=Sa&<)A+Sk)QzRb|-V$FT-!guAs@x2>%i(0JfV`!Lrd)JYTqPZ&{y6)LD
zdF#wev$x;dS9|(jc<kqQdq3>rdtkGoy>a8;B@1@>F-%U-x~R1%P_UP+_3yghYNAC+
zsUO$x(!H9yQZM-KwC6v+usan0WuCuMZ@Xy497`@{2T>uj&pB<g#J@%LbNDfS_<JEu
z?m^`|e^rrr;j(AK=a`9}^OR)U`Q_!^ee<54f6ef0vU#4IzCUmBmHp)vFH#gUTs2Q<
zt(Z~xm*vi1m;JuQuYbNdHUH(GmR&{xd-mU7v*xq*@r$6PvXN7{;-0IAhD}Uco$Mj)
zu-x8N<c`=x0VlPhDIT0ILTkCon3JZeu1Tt`5uW{{zY~14s8q3_&_@U7MSZU%gdVzl
z4v|_@o%+1FD9)^(F~slPkp$0o!oHygbv|Vr@TzjXBYL*jpQ*_-BJ6SN*76H;#J8_q
zTPpqiYkQIV!`D1w2UofXo&R!5*GuV2%>2A9AvPXIu2u!c9d&)U^^-%=2Cc85PrvQ!
zc_OvGO5o3}ioG{JJU#yQ-O=v))+bRNp7VRpr%axqC7EiXbk6^zz0xFZsg-FF>h0$q
z75dK&jp!=4dg3f^q4$^2Q`T!&_yu<yloCl}6$t+pvOXv7A^Q>Ii*IipJN-`E^0rpG
z;zix;&=&9K->>&Ov^K6f8(lqlnr6?nby6Cd?=LuYv@ib2nR%CgmCuq-=PLvnf=yrS
zUOQ=Lm7r&PYYN*ljafZLo!1%{6<oWO{notZ_1(qiHvY2t5hm_%c>4=}Z4uYUo{tqJ
zb7f@gl)8J%`>oj9`@j92(^9qW`iH%I(`1?cOwXA2QR&dCe^)k3hAydUIBxrB*No?Z
z)k@DQ?G7j&wLQ<xl;x_zaOBd$>B|G->KjuIsEMaH%S%snW(nLQ`dy~Il66(g{>dIY
z>e_V_KdfR~{b<Fq_GfqH&tASJc`^Lz&wsmqtxDQZwn^`7uwlge_vYG*^@|Se{vLTU
zJ}GtfMgtw`eW5!`_pmKLvi|2i^NpSV4?q_YJ(g2=Hu>SNBV7lX6i&IeZh5gN^6LEW
zQ*LRQK49`Ve>?o2&pZtumXfbaCR()>FTBLM_mlLRu#l&+8d^Vd`^*;ZJy;MH<HLAl
zt>M`O<0DHM9#nkY5tFuTjt5i666OPG+zcG=z1=istQIcc>K7Aur`cWf(xUx`ZZiu^
z=YHiT(>hsW&It``{#Q;qN8JK#R&W@s5s>CR|HyFf?^9o%%(@^tDf<*#+q-Q7!l!&H
zFZ#ylJt}fk-ohoB8fd|JxKiTW)ennq-P(6~jm~<vJiiNFCT|!xiUfmZo?r|&U8AG2
z!Zgu!L#VXB<2z21eMcTW=)Ssf?P=?yU%E~#eWS>5Y3KT)8hzuc(|=w{l`QU!)Qt(b
z+_~w|DmRJMPT^PYPIrsTODXd8&a=3`aLPNQH+o@xNetIMukBl8TC@7SNjYDi^qTGE
zA9j~;eVLnfKXc#feLoBTA1?2=vNc_}_=`vOMa8457i{>I%aE`x?qPoCg|G#??j%lp
zzEO!ufJu?TJu%vS-gCot*Hx!aUeDUwDJ*fkYjN!5ZHFIw#dtY|GgK`-U-vTZ|4h^U
zzXT5WB}{7VI_IAwxU24XQvU{~P!Z8rOcOdR@_Mz|yyt1?-|p{AkFJkcy~_We?c=y$
z*(!;{Ttz!`dbWxz)DYK`UjMDCbIXh4GMq*S53OF1_4Y?jlJNhH;LZAO$$Cx<lM|wR
z6_?NHI=D(fFxF7JWNTiR2T$13Cvl9MR`n+?J-A9PWwnv>uD<oN*+SQHn!a0FUb#N^
zon$|w#bULK39oC}WLu+h69UdvuRW96<Hg~vxbp1hsx{*L|EzDSF}QpPapZXR;6vb3
zBPnlvp67hrhYR8!uGN^+@i};s8{>+UptzF9M!m=4`JOf%31wNOmEr2RU{!}oFU$F3
z&4>67S88}&O`SS#&G}>1?E9m0H(b1x>stDXO{2?|b#upVrU{?=q%<N<Ul81)q_)N@
zUT^=(MUj~sa@b}&mMWk9%rK*uv7t!pQgn&(a*;%<T{#=#TF+=`dAM+NYHXQO9yd#o
zk-_}PTcP=#(T7@J*DVROS>SSM(su84E2jt@$-jAb-{qLJsL9!zqGG-7gw5Jk9-Z2{
zg(J`J$%6MLtJlw7#<1XK$A@g!(pwv<uTBiKvFJYj<?Jbq#rGqElp-ffuhg3uaauk_
zkh3YwRprkPH%(Wa$(`$61aIos^Ju^0()QzaSf4Al`QW3~FE-ovYKvcXw4C2AXPL8h
zIqOP}a4*Mx!z;N3D{orqUJng(aSiU;P_*d(*53J=TD)PYGr|oc7;@ePC?8tt<iNEq
zqi_Z@<Iesdl}#)O`jeZJm{*y>7twA!a`$dPhhT~Y*ZZtJ3MH;Q(=@~v#e|eze_wl_
zF>c$B6=FiuPp3awIOnAB(qr9SyANq4dvL3Ge}C2NGWBk;{u1^4uDxY{Z*EQ(e_6zJ
zSE9>flHBP)iMv-G2Ys6o6?1F-EF%{NlWj4wskt5-zf3w<W!h0_7`1!Ky+v0f_L@Y6
zl``Ke-S6)4`fk`uc~iy}dfG~y3OvtURSp;YvpKQ2K*&b=w1)E2uo%6L7fOG_OnXkd
zN}S8|@LrG@qu1pjHgO7vb6{Jd!1RTOovx<3*WUgr#opfk<KM}3zin^a^;+t3A?3-l
ztOx7AEfP6;DC2EWsIt^DFYz$Pgw`X8-#<0C?FePwzere!Q+pD(V)x6tIhU9h%vjHG
zbLNMa9%BEryVBMOwoG+A?83)W$o=8!CsyC(PZCa9Z<)()71z4aL&@#7<~e`ut5Sk3
z5et2qB2TZk*cn$;T{ivF`dYKN@U?*x-qyJWm`3zp32%+x=d)p-*J(kY_nQr$=59Q6
zWE)fU+o*RMX96Rp@`n5j6y$7*T%MYz?cr6>KiTtCRaZdi<W<F$f}LIsts6>2o^AS=
zutP+5qNK~z_n&VDHwE{Fs~27TYF%>8;MmJ2s=Q)9cZQ!SJidbe>6=sY6PFgR_sCvV
zxbUWyq}b7;RUZ$o<`Fx4Z2HPdwg-A8OGUgDmrXG1-cmZFwd>%mBk<*-7gHDnRv0q~
zMNgT!dfm2|i>sRU`JX>N|8HhYZ0>A#ckk;jwNBk#%0KNF6N4~g=98HmCs?`{i0V#g
zQ@M2L)x#hBI<0dWr=5O$%Y*5_ypI-v@fLBmf*T)Sy)LOU+qF=0t_R<#ox+~)MJMj&
zUQjaqq|W)(j0H2F8NDrStQ6(F#>{f}!PNuo6;``)raAG}9E#?>wp!|KVB16v-6*?;
zh*LXwa@9D5Y~}B6dVccr#kWdalQw_&`NE7N|8RkyGgAtKR!y7gE-ua}#tSj`6W?sO
zYH4*(J?5#^-q`QeTXtzNcZix!{lph;dMI@53O){{bqo@Zrm?(t;R|JpooYMv|Ee3R
zPv>%J>9R)fB+NS|*?H%thi1Csk`+3exa=8@c!~cvS{U}!b9HOD(VCKyor|<9tad$F
z<z}$)R!*RfT&8YQOoZyEbB_!?FUQs0_ginJa3pi5&uWJoLOE+S)_r`pd&~7BT^F~l
zWsqlU^|YDx`iqwS#;1mni=Srnisr48I9|%uEtHsht#2uJL-ihk^BIfeuiKnca24bf
z<l5tvRJcxLqHWo<8?p>DxKBw}dBu1g`ymz_y1CsnW9s6!6<=GbS(a<@imkkx`^~GD
zgQ4n>r;Cr#x7#N9#+$Q#Z0=k1&X!~C@v7VNO9k3OMNb<{SQRZcG30L1>28U~J}j%E
zCtZK>DAeWatUa~Dvwu`2F>f&Y=-VdY&G3QyvtG+`jw}Ta{vDxi{XrE|8M@ca_F)O_
zx)3;N!q$Z`lh{s6>Ym>G^07<s%Repil5X2EJN!HM{f)%Gn_Jc-d(2<ELA*ie_qi34
z?K{uP9;=XO+tS)uQ)&H5b87`}h2T`zUJmVkwvgDuYd(@19n(DAFCXH+a^U_a9it~X
z(+*|4?Gt?!cZM<U$J4!M#2Kb%F5K;FaQZRV$vLvAo^N@c3N&yW4dCnc*u#+`F|RYA
z<)P%?qp2RwznC9r%zI^Y_N(psvIo2We!s7>X%$oJX{H0B>&=RaPc0Mtv6x38Fl@_N
z7e2|gcMtJ<^q5X$-c-6`N%;3k&2FbxnFN_V);E1)#IyXg_@+>;W$ec^t7mwu=?ilB
z6WQt=cRfM(x>8=|Ij#vS8<o0Pa}_l=YIttv+qt1B^2*#_K4yDgxXqbA>86Vazs5(d
z-KW3%d%Sj&4iJpp`%*F&bTTSS+|;XUPu+aCF=qXdRht$*TBl*ZV%M2K8>NjJf0ys7
zIrlsyAwt;n>eDHbGN%@<b>Jw-zgc~uE57TXu%@kl<}o9^#nbPIR+_ggU92;^MQrnQ
z7oCqs3TL<<%zb*<yUBHxw}*<v<Ez3Vg=a!H^=o9U`6IQeC-G}U>H7cvqF2sL6f5O#
z5p-Go@le;)m4~}MI#V_$ZHP;9XVA|2d_8jmXlhjv+yP7}NZN47;K(e!u6wILHfi75
zE9A6>MQrK?E(fLWww5doJYp+%sC@Ul8THj}byy(lQsvdtOET41w)O1|DU4MWyPmr;
zSFF)WsLJb&k@}>i6FOy>@1*9e<YC_M?BWIy$Nl%V<(n+EQ7{ZmZk?>mu+@B<<Td$~
zz1QAV>76XCI?^q0Mrq=-`YT`d%zFN_eQn?Kud+u!@3H$U`=iG#q;AWs7nd&t?l~NG
z+&=5u{rb-b$~8G7a^Jo-EqR#{6{#EWtKV$)3vC9KwB_%m9ISS!9B|p8o@sN0E9#*s
z>#tWc-k5m@X5L^_c`)y*(CJBR+7tCnBUs*FyDO&gE_1^w(Upyxa^iQ~5?q({<gH(D
zhtuLBk2PM(t5pl`DF{AXu)*`7TSMm$-9<Y*mb<#edkbWiYkvrN`=jdRi@nKC`th2<
z5uYDV2o&3Y>5I^;*Txmwno3vXYR~v=xk09=zdJzZ_I&OKsf;=dH_qs`%2=NXykj(J
z)#@oG!ZAr3?wB4o6>`0J)8))ZquzH{SG%=TOg*^D&sHdZZS(To_Onh4FEu*2YLQU#
z-POK|Y^y(|M)1#_|0Kg$a6;qmvkx{I8b$<ltqcxZ_fAOnSK^dO5$S!P%kp}`{WM8$
zHt#&i=zH3#b)6kAKlz?mmbBrTY~Csn#gk`FvUi(AMC>#WShPy-=@zb875)1bDU=5X
z>SgW8D4!<bm!usSzbDUAYI;#-&DN$!28-tn9ZOj5cs020+?M)R;qp7f2<>05CP}O*
zF=2Ii$ey)eO?AW?_lMJ`oLu!!YR&b6ea_#LSLr<dt-kL1gWu}^0x#QVefvEpApCX4
z{m)tR!mYo&{xa>g>96<uJ)QT}2Prb_QQLC-l;D&ZGfsWVQq8%Zad<;h*rEu5J1Zr0
zoOGuy>6~`Ce3r+$>xKOr-|VaD+{$Kg`mw^%s#z<UqFs^%Z!c&2w=1jn(LA=*QG&9?
zw?Zy;1zwsEpunh~^!OU5?k|g(?Qegdo}Os2K0%E4<jJNtx(pKx*%=ovU7P(idBGZ`
z&d-xxt8Q6TKKb>kvt~Q1L!NC}vXuEk@Z?pOCU3fN#4*T{>7ni~<^W;S=td6K?-ozl
zuUkLK@YXX6v&cRsbl^tf*&CA9TW+u<diGqItE1ui<KFkVr<87p>Ay~2scsU+6Bg5T
z{jD406~iSWiN{wwKl}82LhP}9d}~)#9j^YE&CRJTYulg{w9N0Mg!t1#A74$(iP{(x
zDrlv1C+jA+qUbY5@E+~#en_0|Iay%t^~beJTuH~0Q9)>L&_n)%3rg9#<6ixF(w(s7
z)4?L=Qa`(h*WveeUs%dIsdDA3X|H+29&WhqeK2;v^Mg0ytt=N-YBJ0Ty!?#st`<k;
z*Ql9`H>@hS8z|JW^pNDu9g`KqR1YZiD@ms6T}~)}|5YYtl9l&;HNFFBF}yWif=Qv0
z-><XojsBs!a>>TFE)UJEd$e!gRee=dRr9~)-;1?b+pdfMv&z0&J}X3W`|G;?DV{O6
z7xC^eUoNG=vU7!6cC2@Z<D9OGmXptxmX|7hUQzd9>fQ8tX9HaH7&aRwD7Ri}{a^p>
z-`brMb<YYNxmL#RDbTVzpIu+@;NcG8o)b1Rr*7YslIrXGb?K(;95=J73(l`I<-VvG
z&-C=)^po|o`V#fbd&f+SZuXqKl@?_iY<u}?Z0zOOsi9NnPCvEAM*XpY9uHfy%7*Q`
zn;)<?<lC-!D{sbdY~`yp9m>X%cHGVujt3qV2)sLeefz$xH{@d+7I91OzmO+vQ+`+8
zbl1j%rT<$d_j7!SKGN}qf6^nN*CBrHMbqux+<o-%pkvHli97ow?i4)|I^W+}ov|+Q
zt*YY0+OIullj@~!q<=WPsG045NX6@y=U;6pn7qhF?~cow6Hf6aoolDm|BFr0I;SHg
z*R8a6mD`DgjcTpl%D(9r6IW$rU)Io`<D2<3YS{_pq&u>rn_W-5pPN7TwT$Bw^=!wg
z)(4x~JzgJQ-**3BT~%4Fwi&OYtZG<9%|iM8QeN`~8|n;>#U5p5W!lU5Fl1TV(w(7Y
z241f-6E%yTizqgju03{bt;Cwpz2Bo4*M!#2O#OL(!PRrhyC+xYd9u~B@PB-u?f?7Z
z>W5b6gy){BD6@;@uH8@(c_vLh)~&ocE6U|a^ulv<x3z3a>(dLG=bG{L+0PBvir+=9
z*_5Yt!#6hV)bAMcPz(2-t*gr}o++z*{MD`Ru+XBVnVC0rO9L$R4=7b0c6(SPSjlnl
z^1XZWzh&)A_0IN|t*FbEtKRryF~?O-%_oLO1Ws>{nei=)e^tf1Ipxcb$UM2E(w?Dx
zbhnfFO4j0wum5?R*prvr{xI?6%u@@a*cto;Cmoz5x9Gsx*Z)@7+Q%H5E)b(y_m1I2
z)XmItZ`l_W9!*}qX4fe1Q9bIq$gfRl=kafw@2mc*shRP_YSHVemvao8#hau3$}ffc
zxLI^=UlH_u`}%o-N1r~eI;bZUqsr)&vFDUP#d#yXm{jRM%fC)r(zg6!-W<(I5uQg*
zI(Dm_>QuS5WyThx&}-ACb7f9(oyj!eb;r%~fyTwhy%mfiGS+op_z~-=9xEo+#lP&;
zM783g(j#II=bjf5%GTTc_w1}cWe-0lJv;Z_?*96qSyz5+>3HPRuJnCcSn}&@O*;>T
zu6<Q|l`CQGtJAADp4BM#_<g-_nLba+l|^c?J{vsVZ>|;K%PBSpcwd?BD%?NQb;Z1^
zNf#Vy-1^xV3YS&BP%l4v{6wk2Z*ITuC08y5dM8;u>bKhX)mLVz$$7C(KhOBHesas^
zoj;kF$#v$WQp&2(9i9^Fn%s`LXl&TM*`HTz{k+aOySBzEq)l=QGVYOAmRbLp?f0}f
z`O0Pw5-yz2>)gw*rF@^FLZ|(<?cZOndndl_yR(Yn+pGN1n^W(zt1OyyyZpeKv}88!
z%LPwXyR@}dW{R|H+`cV4aq-U}%Z5FR?dI;Aaa+BYe_EHRJIDQ}&08<XKWWl9*71tz
z<qdJs-HSFIZ!uWCZJ$-}p_dxHMHZ(SnUdKqAKEK%r>G@3OtdP!!13S7{!a{tjFL<r
zX(g@|__Oh3-_uRkqP1>&EEcnFt63P?ZE#&><D)gMY})7k_^q%^%zACpFSWILnwVbj
zVQ%xeakAUjzuUQ7u$}R#Y+&cqXRi~x1$>zepUls2|30_<?&SABxBq|pemP6@X@l3*
z_y2u;`fq_j#Oc1sj9n9+GqWq4GkNPaRn|o2wnEK^BfD<RRO>DXU0pSQD^tYkU!Ssk
z#g(5tKK;4qHv4t)X5*cYT#_H_-DQ+iTypF2!Lzm1Wt<Dv<l4nDZ+P(JT=q2wK6{R(
zAODCxKP)ISC3D|Z&${%#{M*#xUw_~C?bMsSPu2^CvTP32)0JswzwDvdzei}bW@n{X
z>+S;n)6FeM1fGkl8>x6$I0Xu_xVl<4tTK!7m{noNVzqWdsQL=+z$gdj>T3COe$`vw
zSLVJKmKM}r@Um5R&61=bUyfB7Geqv4Iuzs?_lc>w)WAw;ev`njnD71?OJ7|pTlez?
z&-MSFe++xw#Lc!A6}JROB({}sJTq7>@!Is-;?G;ZEPGR;q<?UE{A=5N?)wBcYQMI9
zSGQ;HaY-hIV={vITi<**`rw=E|2MApqtiajx0qb$Xw}fPF~h+t&MQq&uJ(`V+P#)D
zOIjEecD_F@GU+!%$3_93xEC?OZ}swZLyv0A4YUYcmi4+wc4}#$xMqV*|0y5Qb>6ev
zUZ36mWer#2<PQ-GPwX|>o}-|YA^2$)SHlkD3+&vC%a5+#UoG|bfx7*q=|9rmn16cS
zyzh=ApU@}ov(23PtCzL<hVUGC8CP6-h1p<ZS(0wUw6H4)YdzqFVx{S}b<SVEAN{(;
zK`zqrN}GoVL%@v5!8bphJ@&HaV9;&`C-v_p@y|`Q`PUaNTkd)!)r{RTr^Dgi!+CC2
zS(dhC_che^mrs@1@#H`$-{afw)S@M(o=!2->J6Ex=_b&oRKC*ZB7fDgg6~W^eVu3d
zg(e+Y#3}aM*SeCOH%BSYV^Pck(K!eA{J879PI3`b!zx3E_-Q8}>v0&B@4pn2yICmI
zQX%A$25<UgjqlqJygB~h&JjslE1S;qm$%+i=CXVz(^b9PruEyZ>whn93Eyn5(cE%}
z?fIMjQ#1S7W{D+!Hq&KN>)$HSpqp^+`S~yY3>OOHj<0<hke+l?@(dTlYLUOMTmLUk
zK03{6NBIwH7KirxKYty+UF|=w#%Y7;#aM>DMQ0<zuRS}O;4tm==f(RB-H-9;&8zoQ
zH%`u;b^Oz`g?h{x;djpbRb)NQ{WkEWm-mc*iRFv}B~w>)Zq<mLc4l#RV$?LgyGE(?
z6O#AKI4@-D?lg&5#mc5Fl6KNOwyn~YO~J#vE52SRHp3($#53t0yZ4zpf-{6pe{{K2
zv|=-h18<o3>Qj~s!pT|bVPCGUvU6F>aBbDTJyDk{6VvV6gzNX*;fZzYT<4@J#$XhY
z`A}f)f_wR2AI@{Lu3Y$LfywsWwxa2B`Kn$E59aQ+Rehng=b7D#>Q2*(pSc{K8}wOE
z-toESb)9$pCcXuC<bD+%bqI43I=%dn2~U`<wN>{2A61W*n)NbxpZo5+$46c6%Z=nm
ze-?aUSA4P1tn-k@R?jJ$gVpvrF@)w`P|)Hq;?`KVe`m{Fr7b~RUWX+QEep)bG-q8Z
zd?sS`DziPlH5ax{4p%uc^IMjK_m@o-@*muz5_PjTNG-^8-p!@ICFoXM>i)|7b}60x
z^LDO~IeS-v>)bt2AC>4s3tfNpom&$l&asD8!!q9FZ`P9-;cXVn_eM=xe*5;^-}^uE
z+f{w}%@PfYxyOf()Xw>H>~eMGz3QSJzW-PCSMNRA;gEj%))KGSWqbRpUuRxf^ncp5
zX!oZ9H!i4gF4#P|+@)!ozzGSyh?ZU_t_@n(FG&?#JEPo_H?!%Lb~wWcsRf$BEi?XW
zd48O&#I-Y^U$ySto;ANdu2%fH>h`;5+xO4cGhuhuUX#i@EyRLdd9oqThfI@*x~++e
z4j-{f&SDV%yK3t5iKoKBQTKQ46!z6oSAT1qK7B;3<3eBVnbqf`->kb`yx2bSviRZW
z=LNs8*Z*-Zb`Ld*DEVPi5%RzL-_?KoW_l&Zw#YDCR*E{Vv%7cG_oRYt_gr4Iw7>kd
zKKkj0&-Z&DrTcHnD&n$XG_a^NEj(<#clG<&nvg>uKW=tQT*k0O!-+TSp-ceBt}N4;
zwTjJKAKZA^d+TBC28&8@{;I?CzNuHVyDVJ#xz99UpSQ<}ryqrXs@zx=*>-8_i$yP-
zM4lc=EWT=U!(QAYVf`l=)<<g`W^OmpN}go2e$L7%ZI2JX$>D9<V&GnQa`%D_oyO&l
zLe6FEZ~l@{Cha3=7G>hUywyi8UC6%IeRlNm1AWC}C(`&kw(#$IIiI1sZ_VjIo1l*i
z*n3Lu>|#5_Y1FuCYP0(sFLkfppl+x3_u<FaRtS9E9KCn%{k#dOHXSOAoOZ9%=F6M(
z-r8L9t;)4>&eC}OQ;+3#Ww=!TD`OM*zU@k5+41O`*O%}872WJ1)V0fW!(RD5)|SAV
z4d%50H{EVkY+1^+lzEosAs62@xi4d$N3|wCZQWcs<*?{d@8H%$H%)?$|DD}@_iSRa
z$*Gpr1=E-P+!bUsM}Ep5nOomJ`CW{hSMy%&@U>qbxDNRLTai*_BCq_Z;l0h9MK7h*
ztX%6^9-KXvxhOdE`|02x73==jJoxc7{o{`Zb8<7yY>OElpDupj`Icw@Z%_(YE84tu
zmHyfi;Z?gLws4+R?mp(`D6xih_A|+5RvG#E?^gMqRN|T4^YCicq74Gdy03fM=GQBC
zG1mN>e*Nq7$uC#1{@8r0`QWop22=9I#rjrStg(<z^<s=P+08L8bHj;*R?QZh-)7&!
z?;Y;^ogU0@aNb@rdBLYuW|i{}OMZLr6xKP{rT$v}lYmZ^pSp{+|Al2*+`g1!v`Tw!
zepImKb<M9DTW7N}#H89~+%z^<I(*1l^xa2=MZMOoz9l9P)^{xBd!${cJ6}AZ(*1Wp
zUz@5{AJ4TdCzfbR+8oe3<6ZT7C0nen_QsA9=7O|zf4;JurxRW;T6K2IiK1AJhsTAa
z4fYGhEnm7cUeia8T~xJv()RCs_3hh#-+Z5E^K0$eFWbLwzW-ErR%5W4e0ljd>)WM^
zF5FyKUj9wEb?zSHo|5KuU;N$)Z+cnz(7fZO*Q%q9*G#f{FUi>-kKMZ3RO`Cel_iNA
zW@YnFeQK5TC|zphCOxC2r*B!_-M#NaJeT1?uI_2?v~Jih-C24rZU)b1DILA{{}Y<O
z?|B)kyM$M7_4UpbJN`U)W;gvX`-$&SD~u{uHq18VO22Kkne~}_JE&=T9n>`S*8Z)x
z@UU*#akFh-T`Y80vi7*juX$YMGtrviq*BarOa8@EbxXH2r}eBTnqV`hV6OaEcb9O6
zf62wgzyI&D`}=tH_5X)|e2t%T?SIXzZ@W3Hn<svHKD#MyG3&}TE~U#|+j+ul|GztO
zICjmhNZx}BZVD<N6Z+p8z;*f3&Mh43*Fr9=J)`EewM6cdhD+e3lTH`RrwHw=5_%e9
zs>=0-Ypan@&&may%2rF)WnDh?**9D3#mdstK2wjIXZ4*s*mJN;qW`dt>_4v{4(IDd
zjn+?Mm+iHBQa7zz>HD!G9`CkY$=98cpj(nAJ|*Jp>vyNwI69A>Tks}ABezHShS9Nu
zk^Gwt>=_vnPQFmo&{bxwiDfL?lX~Fu7V`%`)RQ$@ufD!kJehaT)A-W{mpU(fP+oB+
zMMl>zcvokScGYy9rN^ssbcIYMm+ETVJ>^v>w{oqn?5@;<I)_3Q7W5^a7tiBr2ySVb
z%<_H7Ppi8P!mWW;hfS<28y76~I4#Pcet);-s>XXKR~#(zUwmO%-oxGUMd$aPi`L`d
zn_Q!NPyhb+%*4k&o{I%GFI%|ZElWo_P5#lUu&?{FST}@yT{@LbVPVli))!XiRvyeU
zWp-c4pj{%fl>eIJmb6EPuWA|IWlr&t?>yPNR=v~jG0){g9?c48uU{{W<2mOv<7-u%
z$Q(QCyY*quFI)WkXn$yu#dPJCjHnia(#*WZHAO{?wvV?|@O|Dpud+n-Xz{9swc58M
z4*2WN`ls-6wckUn=TT9rUcOwhmgXDVEHtlk9bEEoPKsB^hd*mSFEa6OGps+x%*wsA
zI96h|Dp$B9Lynk4YM1}R$Koy*x_^C-{Q1GfD56MzV($mf?C9MynysZAln-nwF8a#0
zFJ@lPqBoOowj1fMUhY#@nVs_d{>+c%ckLGnb+6aX&uw#;cIP>CCv2WN*Riz1u!7}h
zc|(#WUi#f6B-h=R*wX3q``X6|+?V6-u5D!VK5Ovw&*~5PZBZsxi*(;`C2nA_D4)0L
z7?025qP2dOjhiPciLz;*y0XP%(HZUy3pV*o{i0!c<1WhqO{YgoHUvh@y8XE7j9B9G
zIkg>eGfKsqr>fdASpTo+owP>!_Ziz>r=!2-no8Vdc2E?T*F1UQp~A!aA&Q-;|Hb;_
zude#{7aB)T>n|_jPhXO_ec}9Rw-0aZExi)7!@=QqFtc`}Ww&nUi<W-L+cTKDEiO+v
z>CY{lZtPi<nU}wD_SbI}UIKsL_*={WU-NBq=KYRGEB4w~?Jh3(YdLuyYjWteCmf2i
z-l=?gp%8UCWC6diT83a%ci{RZnK`?cKKXO}q+^g>OoYltqirj$9NBehXINVC+66`P
z-uHeyw8rMVaI1^vqD7|^&ld{?ez43o&N$j>V3D13>^4i?%)L(k4NdQe%GaA~Z{ygb
z<5GI1ApEr3;zj$SHux#byq@A}e)YhPwSUh&J;`z4NDK2+u?v^pMBi|F_wDJO!$18U
zlRFE9eneR6ADbl>89KYuaH{hh(YZm#BNKC`WwS0VUYqt#^vCT3I@;TN_r<lJ-emjf
zyMemb<dXeur<@WN9gE#4Q_14+S~vTv#$1LOx4H|ggR{69FEDzCmp)uzv||ycMo8wq
zb7G0VU(I6?)C-xlezT&5*WXjFlSGXoQdZr$ykZ4g`t|MnY_pk`A3Axk{H>i)#Azv=
zNfD1$r>6%x)g{lW&RKM2$q7C7#a7jZPWxsl#awM)A$lO{>h!3~7hRt+Oub!oX!RNC
z-)|OHZ(6oIN3d<3*_sIllPaZFCT=)U$LJCA(493jFEf8dLz>YXp0J9r3;f5b@?WPc
z|GT-W>d)=_Ys#)OY<PI$QM2&eOBsR<DT32xYQ@Z2u}*!X&DqDg+m32nvM$u06ftdk
z_%`+n-#okjPnHl+IrUv%jdj7N813ZEou+GfM7&R~xqoDq_T2DurCjU^=cjJe>-^Md
z7;fdi$mUvCgHME>Ns#U#qesl!SPPd|WLy4hdAn+7iuB$eIty~ot8{#**fn!0dq|Sv
z7QY#s3%5L2B3TkFb>QjAjO_i)3#Oi5u(^fLP^0SSRYrp={N5^IH<l;|MAzJXmiw#F
zsWn1d!**-llv{o)qoX8~i`M?(h%+eqVV<bdwdKntzuk``nbxL6-q_@#yQ?%{s_V^b
zJjteFk=Np${k*kg!Ikx_3W36UM{d5!Jsue%{?Z_HYiU5_l&N=(QoHZG-*tBTLN@I`
zep8>7sq1S!w|$}&-StNEN&m~&tLI&fy!3j{4k3A+m8r)s%I}w|yRyaZ=z7Px)`STg
zHmUYA+jU*B>SOkPkm>z=YVq1N3<5WEa?WOT6@<z!3S-C)jgQLwXTFi`Tu<&9=67v%
zUK^+MMP*m*5=cE9)es~1+C_b>-IdKzGI?6hf0r!2(yG2>L%cwisonao`jgeaUHta?
z?YY%2Sr}B+%427VAO9+`z|^@crvLJx)Kx0GPEASL-S}MV{p%UI3p!b25;k<Ot*{bu
zH1Y49QC_rs$=<1z@;Y1_)L1K;*3a>tYQE-{&V<*Cn(|T&jy;W<xaroGXIjj^dp3s2
z`EiObDhZvvvQ+8CyqBx@ZAjBN;%T<|X)g2BhPHi56Bg~6yE{ohp)-5$^?UC+e4VSS
z?awHQxQbj1txZ)sxFy<R<@N&!f@co;zmiSpJi>MHs#ABtx+!8BmRtLzA9L!y=7^4t
zQsr7}GymnoS(@j{-yS$1acJLb!-Xd$?R7F4xwne(O)J{J@ve=!*J{7F+WNUVuQ{Vz
ztgaVKtKZll{zP>Z_kw6EHbJ*Y&Txm+IZ7-6_vbKW#NSok#kJ01+P#E_9BW$+3ROPU
zEn8W*gf08a$7jt4pB>sBSD~}=z0Di1=>G@L{{7(kGE<#rTD8<guI`$t*RrB4^3*Ta
zsa;d`+PF6UwNhl`>d@L5;E_s6SGIj+&lbg><m}<yT{<UN$CZtncMjKbj>->j_r0xH
z9C*p&tKHl?s<(P}CU3v-Rc)rD;F-0L=S1-eX1#fG^z~xy`IDyQ`0L#l%f7$=wQP!9
z=~Kag3+HVYO%({R;7)1yE~fRm^ORW{&$T@~-c1~-6~0>@CJHQ6-<ZHC@w#Jm)Xcde
z3e8uuT;it|SI!GR=gTFVesqb$yywTP?rkgC7x}>JIj2sd<MJmnMc3L)pLVThlhV8&
zvn^({-qoJa;o-`2$9bZ4In&}IzRT}?9_F3*a5?F))T2-6>EfbAXXpGqm|?v9L8rRw
z{9~K%GOqce6xsAj?8>_t-=$oQ5C71cxw$7r@5G<aF$W{hD@YYD{oGl~;=p3d(OT$z
zB;rF+`JTLttBnVybVpg`JH4LZQn~0){$!2k_xW?La@g8-PQ8<!!`EPvHFe!d?oA$7
zR01bw8ZMltf6k3zk<ZT$pOluKn;WsjWc$Tqf8SM9?fp6JVkpyQF%KPv!ew(dFYMjW
z!+7cfr)fmWh97HV1a{uv$@AdsG0k|}DZ+pD=D$B}#IN&Y^_tScWg3^KbQ?14(dfz%
zOO4pd5&l&XT#KcO@JcB?)qkPilRni&%aK>CUYo7CsfA;6+jqAunIF17sday-`mOP{
zd?}-%tTRjc)(=lvwj0_uKW}*aZSujbQ`Ott_}MQ;MMmp=xOy_-<8~RjC!CxOPV?M!
z!|&|rKDemTv3mDq3ALB6|9LLj^F~e1#Yod_!o^FKOy1YBf?kBNCC6$rm#T=CEE1Vs
zdtZzBci3bm-L*fCoK0GmHT|K*skM_d`Zp&n%;(tSefZv+tCe>QdQ28DdM7=*B>TGJ
zU5~~h?M!aHh}F!y7Jj*~R@c#))%Vr9eMW~8D!=Qqo9KjWce8oHIb*U=i2CJj9lK3?
z_sUxoXR#O<#>AE7rY^o9!M;?|VDj^<B-!m<9!I^eNp0i&Ja?b|t&In}xlC2HIrm(U
zS(_0Z_4B@2((bfx7cRB;l(#jvoH?`c*e3gkondpf_84X^(LB70JKW;F3=2beir2+u
zj0XyI-6ni_v%RMCh!9&h-vMs^$45nfH!SWq|6af0VE)rh!G}K?HT3^loGQP@{^v=~
zdXZX*s|lS>PnTI`adPGSY)fcUn_6@gQGbO+O)D$1xVt*1GI{>pB=cbBtPMYIzpwp$
z{pm#RqU&l}$NT>W>^fN1{Draf$}F+XdP2YZL*jMMh;c4q{;<II)mGDnoIma}_FvL9
z7VQy8OWAN@k?N|`98U3G$7jy1zI^rNCj~|S57Xbxc(H2l-gry@6#@%d{`w~Sct1~>
z7bU57?Sn{O&b3nq_!Jta*E<|exLDu#J=}DuQ?G_~ctKdn>)_W~KW;iZvQ?B%d=%B;
zl9S>ibzti~W$~>NE;kxh?VOQbE!K0<RiXX5MRnnZl8FbI7Sy<J-MhCw(e3iRgvNth
z@$Y2g7MqyP;M_kePJKhlOu@3(H#DBuJ#kTFXVoZw{G{>y7OQ>{!~Zkcs&n%Ge!AT%
zzgIJSve(O%zEhcJh*Y^{Xm%`FzwjwTg12nSs-^N(MIC{D4#`cdKHZ+)&m9gr9y+ws
z$5gq6A%smN>9Rn+T+Ua%_1pWMu4%>CbzBq5elkf}hjSz6)d{Z;ub%w=E=P!vP<G>1
zF4Kth+pQmJ=e=0I=q}IvquFauEev*ZTArA8mZ3Fr*~6|QQCF+ii-KAgA<5Z2hL2Bw
zj(L12Atzbx{k1P!3SyaNmCpJ9<LU&XHP-5#&m?pn983^)*l8=Grt|u3;=eOTRzH{i
zo#SlJo4QT<*UGi(3uT#C-?cwt(>zs7PeM`0VM(fnhrHfgkD?BS?2D7Eb>7CW6XZAj
z;MC3L{b^q1IkjC`FYlaA-P2w@kD;J#-kT!tcGe`pZvwrg1!X}dpPL;Pg{+y{Z=Len
z^N90%0XE&@aNbWJjMN^9Mp-2u6D>Wo(ea~{Yr}zczZbjCv^QCCBe<&Zpwb~}w<+r@
zyDv^SD8#URL0$H@ozp*E5SqKHVBwl~PuA_2^L*deJEw|`B6jL6YjqD7POdT8I{R!a
z*Uyshm#trw8^XA)Bd^3xxpwkf#iLU*3+8eMw@iO0ak6$|qUWp48h03dxWv=Dc&7cI
z92R+5{)j<n>f0!*+D}gwl)vY4Fs~HY_L*h(lJkD2&&SEySYLFVV<Norp~AwLCoM~^
zAFa0*D`!*OxX+>4m96{q>GO-4<PN3=%~^Tj-<GAJrv4ZD)<#c^SY7oVJgPs*t1tdo
zk~+J_v`Z`dH(ao7mo56YM<Vvyp_!Q*0-jtpy_m4{Z=Qw8lIQ$WlIpkieV%YIDJnAK
zxWSHX)0-EIx98@GE>-FDoA^BQq0rne&M>LaVoS?q-{g4D8L1rS-f$-+@*!V>w`_~l
z8nx#0vrRerjl5NQiY%19*2IX{-P{%SEHP}yza^a<K2ugTuw099f46F?BhQ={esXSJ
z=f!4S$T-uyDsY*YK(ouufMYI?Iaj*9ebUMPNbA&w2wtsCZ{OBdJ~sY4^U_*t|7zY^
z*Yj%+1oAxOW;}7JS=G4X<^;xxV#R4GClVTWPx57Q2$-tj)1%nzdN=y%gde=8AFuhK
zvp~Z3&QU+zHIm#Hvc!yRgCiYZhCOR{OJ4h+XZiYB>>XF;2DkWVe|o*r?|sS!29|%C
zb;7oq+B{<NQ*v*--zW99zRTU-`bWHKx~=r2h^e~rp`pLte31_H`G4=_a{I2XgNs*L
zbzD2L;d|_o>*ws_qQka!hcI}5t=g?D?tLjj<iPS-8FOzQi&k%EU3>M&t|$3#Betyy
z{d)=$FIhRU1sdBK^4@GdHO=pP$rY*F5eK`^-anea`~IY-iSfrBe?QETT2oZ5v6h{o
ztn(E=YsBfJi~rA?RdW3D+ItU=yI<%2cg4s_qxaC`e{BNcGqqTA95bs-IGS5`2er?f
zb!m#&6otCk#`g|YUUHfek;C!KxVfRHpiBJ+YX)mVlI-pr;p`0U2^y(A<;4q%yr<Y2
zH*VW}NYU}^mJf3e^%`xx^EPXZ<J;Dm%vKA9yc^ENc=X*<=6E2LmYsL5{>?L&**|Ye
z>G``|Oyg5%l#kt0TvxDA?9Oe?hR4FYO_#WOgqPT*N%EZBtu{xfx_surCD9SupLt#W
zyPQ`zw>GQs0@uwjld#`)ej7R`iL@I(vRttzIBUh8sXzNQ9rl(<ynOwS^WxQ<u#d7l
z*K(}?EMQ|u@d|7<-M;Ym;**z4|DLpX=aY9wU+&NHmoFZlO*L-anI03vY_aZmZ-wBZ
zN^#?IvAK7CrcN(Cw~1r=tHu8^7T(oo=Q^ZwuV{in;J-!PTefUnw>0$WGrKm?+O4bj
z1GO8ruF_ww^MAKYPZ4jlzefLr<~OtEUE1N;vAT7l62sl?bC<bBJh{B}b@=W@2f3p8
zi`U<Ju}f%f(0kSWyN>ETjZE2aZdTmMuiq{<C6!-eIisxLz4%qaZ?|)9t;G&u(bvz2
zH1S+x*i|rjcXSAEuJWC!8zSEbojW;uj$-D)6?1&om$_@7@0_WTc3fyn((Lt1MdjJ1
z`mdR9ex{i9fJC^Z`BuKT9dp}vbS<w8%E;+`x@(1u)zY|(1k*2~EDByv3OsalXLL`x
zbLr!RD3f)G+tSXcJ!7~Mo|fOU_OD-UszAkb^DyqYsf^c*3RSJ=-?>`)KSd%lQ?SjC
z{ea9{m%F0Vty>qD#O_*M__%}dz>Ksy0jq1P3Z-U8hwk|IlvPRVyvJ3w&Z1jcOT5Cj
zW=<)awr=yhgIW7xOp2e+53b%_{Ic)xU;XA5Zyqsw(<xh7Pu;SbsnnvKapyo+^vfMp
zoDc3@zs4vMYB7<$x=izC@S3;t_HS7+tEH`=t03*oy}N(DeGmR_qBM=`=W^Cxul#t$
z_(cvS2l5<P%ozFV)X#3ci!pxhCPl9`ffr;(S~e9QqvrSVEPS}BLZc(}s*~Qs*}a#S
zxz3nb(eolDJeMg>|9t4q%1qYhTPjVSCx-9Xcz50+8*d36yQO#SV+y_}9)A+(dM9W~
zDbr=$Ij+V|9153CMLBG`*C40k&LLjb`d$BnT|{%mh1coZ-%K<WyC=5r$Fhm@C$DEZ
zzxAA|4KH)Jq#5^=bH5Dcw4Kc|aaggyoI56UU-*l=x2G=B6?4CoF~wEEHD;<tkI_-F
zsT(8bt@d1ab7t4<)3u4d5;1o_cq)JVAM?*!z^+qc@<J<y3nr@0{7n187Kupe1U$KC
z5y_)0KdJbxyl%gMo^W`^mGC<r8!yVUxgSh@{^de*m%$X3u;o`6P9!+~`N*8%>htWd
z{Ol+h{wMiU#bREso_M5n^1|Nkm5uVUR||YUOl&qNm|b<DMrlTA&gx)El{x7fEW9mR
zE@duhFuAl!FJ_6U?1mR#rfEG`68!fQ(}9vJQsG}O^81xvUDmMBWqDP;cadPk-;$to
zTLz_LT+2CHS4PRl{e9B!^>q#3LlM<TZj-VV2OlxXZ%!y_ck_K;($CJa_Ui5&3%y@|
z;$Dk(Ze8^~4xFhLUn_~TR(5Pjmfq}MkUan8ilf@z3#W5zcKM?9^}(~c6A26N*_VgE
z%UtrtK|;q=e{F(?U&SBMQt?@g&eIGck^=W8nEx{RXe6C=@ACnF8I46s*QM6_Om#e#
z6ezdYD)xpb?}iUXPrS?5Jib<1@3MEBM9ukgr_MP&oqBkQ1Jmnjw+*4rVS+ZLUbROB
zo!0*0SXla{WWmdfN1xe6r}#~o6qByRb}4-Fq4%~<tNk7x;W!|1{nQ1`81Lto*}{r)
zcrWxyTGh|_S^p`KdvnXfxo_W|E8FPz&F|jU`BPW(hS=$D3d}m0(5Upi?V!`*gH_Yp
zci(@=63RQz`EtY_1LXsGuNdE}l$$o&ea;Oye^2z>cb2K&cbqC#Wis9W;LVm3X}$ZG
zeV5|8m3By{@JyCT`lm~GnCAus*FBfq#Ol5!?UCY|#3hd%*YCZ1Gl+e2UvANS-q)Mv
zweW<6Jeex3w(9m4j=%S7a&vDe>Bzow{d-&9PyfEn+P<s(|2?z++%%8+E19q==cke9
ztk?1k2Oj^vsNVi&?etz>n?nA4OW#Acejd+0lpv_sI!Px|Am-?Wa+CPyA68c-&+j}q
zZ&UMwh6M4lkVR7p<vRMMId-fH-FqC|leJ7$o~pi`q2Qv-sS{eF&93hn=lu8omvPbW
zuOFMY?dr-CWxK@LJeW1u(*3@7tTX!({3&tIlc$a`4b#u?#PRh#;hOPXCg=VMuAAnX
z##7xwb9Ea$isu-tHfBzUdZBY)>&(&_38Hy<lb77O{X3_3qxX#ysU-;??oPQ>u*~!5
z$>(pJjX6wJ`RDfvFbT}9o+6wX*nHJz(Zo$fa}UL)dProhDOmd1@8JWUR`)l~Pj@PL
z9@e|D$Gv&!3wFk)nbn(l_ln#yF)4plSLUt0N}%b?u}5>BS{fy6IAVS8@fH1R38p1c
z>U(l+?i8k0$1FH8_rme?%;#KRe1c~AF4dl^&L#86TjbFigJYf9+P7zIK9?{#;7M<^
zq3|@$c-8%L1E-$4t#`p=+tkVkpPzGUHW?+&Jn$)b(s?EZ(Rb6|MLbhInZW3=IP65i
z$M@Tc)@B-NX5T&{m=?I~hxj=!(LA5<`*s<BlYamDy!WJjclFcrm+Qj!skubyPk!>_
z<-e2R`&y5GUwwD(p^!KGyHCvB(Z2m*>UmJ(k4^i@T5-LNEeDfcSZCk7uNab|Cm%QK
z^r}Tgdf_{+RVzQy{*?tAFmmf%yL#@%$=v@m!p^<65HAqsi<*2SD#CM7aN-6Dmm9AR
z+xD)Q6;vkhD~j<%RrM*pA5Kq}=50Lf9k(y++U?y(^Jc9-ZDGz^C-BASloeyZ%})u{
z&CJaV3A?rwu228d$aZ&i_iUZ}Lek&MD@=audN-eit6cxfGhI&6i`IuFFR2SOvpo;J
zD*Rflb>At4tn#ickJbp7>oAy9J`dJ-<uzs2g!@hRwk%uvIdh_-M{t^W&x-8GZI{;S
zzFaME@XWnilQgvU{MW9wl<MY^<<FdDS{(6}NoR}E>9>jL0`m?|`r_=n^rBI7>GLNK
zF1__k7E3++)A8L~oAc_|Dl?wygy!8bNRFMinrl|o8k_K*+9%&Ghw%7*FEF!Zc)NGg
ztZ4b8YfI)mcM1HW>AUF9s*lFpnkUaPeX)FOB(<n;Uci@=-YFZLc&Gi*)H8@E>0muz
zAj8nQxpbo2B)=D3mzHKODM=2!wQraE=baM{D&4SJcihbMxb^MU;x}u<w~0)w_AqU2
zIx)9?PuuP<Zywrq`c0SAVNvkmQz$Y$`dZJVSwHIejM%W$gj4Dl8Y4M3gu#X$uFMK+
z-TU}o^cyW}gROkIcPdq@o>zQZvuN@+_w32JE7DCvgBKQyGDQ1p>=$6uw%Ji%vV(bn
z#GJR^ro_ZC?~su()N+@b<F;zeU7v6B+%rR1*9V&@x9r>R+`8K!;<U7$rLOP1p8NVn
z9v&M^jrT5LZ9b)0Gf$*NA+F(Qq*G8N&(YZ;GsGgb4{te9k*5~v`gEHA#fx00zCX~o
zmz9~i#yjD8j^Lb+wHq5`8FQZMu(PfIz1X0Djd7Er<579%#N8zx>%Yc1E1#{{z3lol
zb*=zQ^{+{5Z%9s=^yToeg2agjcLg`Jx~hgP@6?szRrZ~s%JyrS<5GsCwKcD{m8Psp
znZ<uj>T#p~?usu$v$yh1&CB%T|8c^@>(P_LXVVwYZR_ybFy;91*7El0N_*0dhi&2z
zX05W``10nDo#v(2MHCyXj2qUj?cOZ*c#iPZvs1b@ZOV(0Ua%=Ik|Dq=Uwg|v9X8|7
zw+%NRn)UbKk7;{5?i_wHhn0=nTUAL{j!P?=tNCkM^0mtNw$v*-KR>N_yJYJoGv+sk
zG&Vl*U7NY}5WEQz)P3#bmxAw=iF<8ZyFWKRHvDMKzuW%D$^IkLEp>MCu^rHK%Gz+F
zB%n-S8^_DSrzQRpI^RtmmA);PE8|?1w=tUM+!6Nd9b((pzq@#H?$j4s*34b<h=JkM
z<9}?QI_`u%++(QRzV=cjS8inU)}?$$swY<(O*w7yu+S_Z$yC*E(mAK6C7~K~U6T)4
zYVJw4p0e!a3Lm~n$7hKYw?%XY31=TT>DYI)@fyD?>)rz)AD`&nTRwI3l@0AzJgR@2
zZAsv`smwIjHL+SaSYl7A?RQUW<5MDYP8O>wc3w%~oO-m=k|D{YDqYt_>74dPw>g`I
zJKXNLZE)IKzsjL&#ZpoBUuN95Laf@JZ21%u)p$%|<Ell1Zr<7!-nG9%+3LBgr!s0i
zUw*7H;K1k5C+pTdD-30=U;aO!fvYNLL6aE&_frd_o8K?mdv{`jyXM;M0vQ{m`q^Tm
z#WPKH)lxRB$THvd&FcGtM{D9vfAeqeE(mktj0}nS$lvzw<<SQ{d<SYiY|7;_jVSUx
zni_aPU!F_u&YkH`S)18R4D!wupHA)E#GtYzec`cdYhOM(7r!<ubbk?eiA~R=tBJ-N
zm>HyhY0BQnwN<R$dO>N5q|!Y#tECBL`&TmtG>iU9xajY_j6?Ilr9f{99l7<D7W<B_
zE1veD&#8jJXX9JHdhI)(OoD1+#jM*LJku}w{A~WAF}FEvjgGBDk^P=_#})>uf(1vH
zG%V=)BARb#AL^beH1*}mPk+@cP8<K~iEHv>n4~(#@w4*ZYmQHK*FHEi_1r}VZN2SA
z**)IV9{GBN@hVTy6N+*D&6%|F^}nhU61q1GAISD`@~)Z5>z9<ad1mqLo<h4>%PuCI
z%u|%e$}bmNF^ko4?}3|7d1u%j6;7}-Sf02cW>G@VW0l8CKJ1n~XY#Xu&eJUl{)f}%
zf7@uOvTM)n>Urm-Y8vbGI&Bu+(z$RdC;5W>txdn%7R;C<`TB>)A`?Daj^?AahwN1j
z9e>a~hw*R7+k-b*r(ZmERzl~#Ov%wX=k}I4<j2f;v({BU_WTC#8>&y5f^M1mC3wAT
zO%yD9C72fdHswkJ=Y%LT+1M4<rV&L96LT{EWxV!W)MfFxxrIB#Jll8m#<SDjgXRhK
zz&+nfr<e?`{VzzKzx8)}(S)fjX*=bn7TPQ6TKdfA(PhZmpmw{!e{tFW2|Uw^w-s!D
zT(qohVQ;mB|L(rxrCSUlUR(69wP)siy#G(r_X%!S9k)zd$>WeKE2-1aq{!Ov*D?Ig
zotF>$e!5J5`cJ{I-5|nPdyN~z5l-u5p4Q-iS#MW)E<74h6LjJ0?7dFh2|*c7+6)1f
z(IMeh?Z&gOg>0QE67?x!$(<`qyP}gBqYSMrefE5OuXG`{)gW6@VtMX)mDjr}mrE@a
zUEaU=+un;7854BBaCNRb<aj96(e~Nf9fxMk&iZ1|%^SeK`9yLlm%HLD!)IR4XJ0!}
zp(a&q^#0kYziU2q8j6<8du})H#O^6KT{JiK&*oKrnjXARc&SH>U}NVUF2-8d^?}<8
zYv*qiJhyx0QdeC?YpMP=-O`3vA}9YI@(tNG(QWOexu%c1gc&A=Z7uEOlF*sj`CWSD
z%|km)II==MB%Qi&GIn1~r>WP=ndgrN@qJ-xPSx+4TfgU<xd#)&ncoR_e==C@VOa1|
z*1Z3E{?yaUS}#WR=E|f^-4w=i4YoY3N6N})jdXmKg{W1>rw{JYIxN>0D>i?99Ddkk
z`BdH1E`##A&{vUr`-+Vs#MKsm6kTnyJu<)Fz1U1u@+h;~XN%pFt5Z1}R_^0ZSY57R
z7m?^$zG;oH;+92g&V*=MT3&Wq?5P-X^Vn{Ycf4F1Yra2pnXS{pBPO8oaK+63OZc72
zzOB0WXv?QLFU{WwNhJ$KBpW+v2MQjXxyo$q8NZ%|j4ZOtk61JG@-wynaV{>~r<tbS
z{(jxwyP5gr9dn+2yKOW3>bCUWiN99NzuWd@>WlddDuI4H2iMqoz5i=>|I0T28m^tI
z)v7YSUE-c|v*}UY!QeW%v`P)n8Atu(8&;OKRL$<SWcpn`nW;hSi}E{(wKn3BQzZ^e
z`m$(!XhOKzYL?<1T?Tx;S0x)xS*q29XYAOLrqJ+?XX<9_%ICZ4H%%_rJCorpYtSL!
zuq<lb4$aq|D-If|C3kNqa!%avfUB^mf3skN&&o~P^1d@N%$av``_%hhXV|$nxz5ly
zGE>M<Y-K8A=)%gbdndouZ}@n0il}!%=nbWmz}+VwN^ER+xPFDqiz(AouKC4XI6Q5C
z?(NMJ?<-cR9*(;Ddo6hIH9*W^E1z+O@{T{TAL5>0T^`e1_xtxtre@R62aS#$NuCt(
zF4N_leemJAdm=s@6bhZ@V6`sTU3aO;)QEFum#n{hUD>niYibsYn_cF^v}3$l+H;)e
zU9WRH$y9Lk%(aKyk~(?Id)aL_7K%%`h1?c$e=%R@^1pKnE++iEth*tyqMyM(@8I$~
zIZ=l<n%3JrYq8f|KdT_=;>Qm?8V73@*6q>gXTMMr?EaMJL|1-w_3q^5u{vL0b|lZ}
zx-7Bs{r0z!%xnK>D}6XPtD<)MI+-YM9<jn32AAd}&iTf6@4=SJyBGMGy4O6s>$Gu$
z)OAkjUBxFA)E_IPE6?Pe=`v$#cD~=WQ)$T!e{`ou^GdCLaJ!yqdCJ^S;|FfdZ7Mg+
zs(w|+TwS33>WN3wrHV&~dRBb??7!NNal%2a<s6^)Fa$(jx7aq_nC-M^l>6mH%n4?J
z919-K&fKfrwEV-SB@%N&`7R&gO!4P-Up2%2)S<ZQyKy4w_jhwju5nq;t;x5hf7*-_
z6F!LL?v~&d{-2r^u~X-q&6+9BN0u}c?zmQ+T+Nb@bN1Al#L)NK!IQX7FZs@tlo%Gx
zKPcHQ>#4Mujr+i?d7oYdpSd%=c0=2Dw|NIo70)RVn7ic{gTvyBhe|^qTz(O6k@azh
zJlCEb59WwgR&#tk^55WM=B)L?Q9nfv-7@S5kej2)`{SI`(;^cdu?GDuh11z2tz+tD
z^l_E5u-#dtH-GcIA~}`N3nq!5_*Y18I(~EKkMyaN*L?c-K;`@$n+)!Zg|Gj4+H{C@
zXg|^_3U_}xfqTYe^Ovvxc^uabie%_pK0C2^L+*JE$Hm*`=1cN!;d(4?*=MfFw7X@Q
z6C1a0lwY#a{mhon2ZG9r8ZSpZST{|)VS}O7ZP$_&vqYk1m2)JBH3nX4P%&9?I%4`6
zW%XNG1v5=VJ6E@wdK4QhIGCpTg6qK4w~1+g3Y(T5d(kwxdXZY^tPi&uc*Nwz(jG@d
zAHV6b<XvF({;O7X#c%CDPWKdgw5p)X;Pu;`YOim-zfra3-KlW9-TRiZX{T&BVSc)w
zY1_2r9PdqvC$2Y^p1ySF+bP%dB0IMksXXPW;g_9u)+2oDs{P)OHbPg3U+eCT0Xq_Q
z@HWidbK{)&e~y>IU+1>pZC$)PMKEpEky_pFlMf2D{?=H#bHdkeT?Zw3r7FI}U#M(h
z<6ZOm?Dp%o-RJPn&lY(m*kBU0M?im0_A-vYKRLbBPbnm9uyEQ@5I0@ZEQtGaa8B5r
z4J{5)9rE)kn}S{5=sXTqk&aj}^}5~lvn)voQgTV_FX&JC@b1*N6E!+L&r)ADUI@3n
zvFMB1w6a@I6;zZt7`=tK84Ft-a{L}?$$d#lDSD{vA)<CI=H;HLks9;2cP!=d&}ocZ
z@YVUpt02YftuoeUSp%o)W^6u@#yQ(!BilKNWT)N`4|mTqW>Q7ozdBFaNM)yPzQ0Y%
zb6N$v%2_MVTHiI9DbfFe#ol@StGFTm<lc<z>uTSPuX1hX(7wJZru**OhJ&AiTU;l%
z=H#A@OY~lzpMT#+jh+9N-1g9If3_|c{+Kc0{pmv6vU6HXAFC<O-umF@$>Wm`F5(aS
z@l_;1by-+qYoI0fexd1smD4WoJ@s=>`o~|N_}B6Nx64l7(4`!6_4hh(2^Q!qw}XqJ
z;KswbPrn6d+Umahp`5a~;Nc5NtDTRpFwMxh|1oN=(W9-KTx1qm&T8*iYqFjFwlu?w
zY|*>x7u}dY+jHK>GlC5pcpPS~ymTkct2MZVCEDxlqQnh0<x}s(=h-jxoHWr!_+qkY
za4GYT1Cu#Ex6E0-zN1UZ{B)$v<_4{+5@~FT<(s!yKUyof;!x@T7GAYNJ%)mpHrYqd
zDXh9Ir_;b_a?&#If1^}T+C7WDR~Nb&KVReNm5OEG_Vw&GPRYeZ46|9)u5}tRzJC0N
z;k%vdf#;rPo8y1}@zz)QtaD&x>v0)>^He>>oqu>f=LQI-c`8qx&%IFeL1KW$illP=
zl(jZRK~lU$O&J@ew0+5ac>PTDu0+Y?6stxiol2GHrHXdn7;hX(Fja23lCbpDdwHXX
zdwf^U)?P1qyMJnwJA>EsZ@zbbz7S+c+#obp?tAs=rlThD&$p*V?TDy)%^R(A^u4&`
z8un)2i!q<VTinYRX|se}o69rJru=7T>f0@0h08APcpLv^HA_dxy@RV<cdYsz0~*NV
zk4w(p#l^t(QFP8p<<}b~znGGCp>Wwx137`p>gAJnXKdhlzQNvU?|HF@PDf{M`nqJ(
z(izkBI5sSqUU+9eW56}fzttblJ`BH<A7}H2KRUlE{QkdxJ{zTH<cmHKNnCs|%g--z
ziRheI55=16A`7QHnc2a8<z1h7%A=E$<{a_)byw!k><jVQ2g{sNFZP5~=mfrUxb&{!
z)Y$^jw~DuJ{|?`#{;?xbcB*su9hp@RC%#TwW!iAAd;1^J?SBsSXk1Od-p{uv<I9^F
z=^NI4jZ6M=G_X=vX}RRFRleTO_uSpWrZt`8wD>cLe0$FCoRT^lb3U9o8{6D+I`hJv
z4XjtDE@d}TuAEV3Y4lS5zI`XVZKos4a@o0=4*VAmtZ~VFxW(z(DgQsFPF8%?YWFOz
zgnc;ob<JwG8LElyYkOW;^6%c&GDGPA*S|Sm-mEo?Q{1fS^#6<GidE@lA7U9wA3F83
zF{I4;%_qOLUhyQm(&4Cv$UoPP%$z0mF5!2ce|=@wUJV2OsanyKT-8E^jt8zxooL;C
zxOvkVeLME}9D!XERxB!tTqN>6h4sog7p-{1h}Bj5!Hw$&TbE5WY*TI#XDeLx`9v41
z#smIqHZ%S`P`7{7fBMP)5A*gb^{+X#Jexu7w#TymDY6Ax9}cDLvs}^E>F4@zpUwd-
zwsafzB|m?CzU%m0KIh%vd$Ug-ul{)aqx=4=wU>806*F0^JO1!!d8E3*-9)LlmtqVH
zN~C^@@dTIN56ww7Jl3-7{V}b}^Wt01Jrm+EzJB7f=7~eUJ~+jg$~stnZ95aT;E~dn
zq~@SY%hvx^zqIK~{z}2vr$?iw=8EK9YPK!9>umbx#l`oZcOLw+I9FoLWVgL`#<|nw
zpFVYZU8Z%1HFj%>-}Wl`MxMK;CX~N>Aj*<lnjNJV$(E+?{zd1;j^ds|(B8Qd<%<0(
zvt1YU$vk0*x{}7QY%k-|%#R9OU;WQqYP%44&MVvB!{S#4$GS(!4|El`+!1OC+<bL`
ziMRLWPE#Z2D)l3K-pv<S++CVA!?j~sy{?o_(*~cFha#(bnKX~HE%phlO3jr`*Xq5}
z`te)f`t|$O!h-5eOI^Y{{SG@i`Q7s4E=o4`)7GinbMNGaFSXCFinqRcWTAES^`6)r
z?s|(XmN@yiecNTS{oGB9&)O%8S>LC$X<b(2LW)5lug~_Sg?lf*sNeQkc9)7zx5tDD
z)l!wxt!avT*1g^LHN873`tvqV^X_=22dbvIv$Usw%=o9rwtDftwr4U0GQ3v%r{>CR
zS@cb^;Lh#*qwI#}3KugM-1zYP<d%XH?RSl~d^_o~|52B8_#GV<8~xK8)_s)|{<7k?
z*rSCXTR*Yzr=I54Qk|)h!hWH<<LaC*(%v7q53G=9+&2I2q)kdIxMY_s>H0JyC3IPo
z+V112;dj0yOuVDcp}gBI=7i%3x9xe(r%qW<G~+5yn$>V{mb4s8jdIV%YiB+zWccI$
zROG<%pN_SuNw&-1NV?tO+_iVF{aaqK8OAIzQn4zxzOA|W@J0Iex-R2{&4(_sOp0JK
zzghD7+P6!x?W@+f9E?zplQ_0Z>jOvbuVyiaj@1ess~V5iFc^F;@MlP_{U9jS7Iel{
znj`PAql@CL74oJ=YxtbzRwq=<exA)BwK!1ZT$=Z%;*%#Q?tdG#y)E0Yq51XsqdcuI
zUd`Ufb@iQ6TL1HO{=+J4EK+OEpYvyI*|IU@gh%Vw@XM~NJVBARXx`ONYs+_q@5oz}
zp_`bw_u<`hT+(&6;c<)M`OY-&S@|75>+{7cY|eezvZ2MXFzodT*A|t@&95S5t4u!l
zT}*HdORjG@;k_(x<LP9LrCTqneVwrAR`eS^g9zpC+??+i672Vy9^ZF;$26Bsj0~r4
zE&W*~wQ|$8jAa~+CZC?~XL48;^0aO3vzSGjdRf`BBimFfYchDwGDm-QOtyLz-SVWu
zY2(BYkNAZX544wrS9o_$?TeoNZpTUHd(+Q;s9xhJ<i2jUc>dbGDsv}`=lhoRpK!j_
zamVjw)XDjrtpPRfFU=GFYdv47Zq=Ql{etI~oVvX%{M^$h-!;F#ny`I1B;!7NbI#+N
zbpaXoI@mVEu1Vgf?vdeoEuhTez54WP+tS~ky1j6heX#CJQ8Se=|1_IM$#Rd<DHg3t
zAL3`2Tw73g@a8eOX&-NkHBRte5<X@7?``=BO<ihpt>nHe+gl~Ucwniu+`Y<x{2ZxM
zRYn`{WKLGoRo9-g_WAd$DVoVQ7s{P_k+gp8di}iq+x{(mO1qT$ReMBUCO&_&rC{OC
zs2Lp%4$WV$-u=z7@AoBvC(KJOi(fN;ynE#`-oGJ}q@6y7Eo*b^7Feqmv+0z{*RnOB
zjT_&iAr)KEjaR<r3qP}L%hzk!s?>b+^_2<hXZ_SPQA{<K*Lh|3s<!qw!-22gCT?Q>
z6uNERnL9?M{GY{mF10CqJ5^Q9UYcK9VOmueZTsHWz5o8dN8g^>e{_C%Nm3{K_=)Fg
z&u-4=JaSAn#Lm=}?REZ?M9){B@64SS^r^Tkqhig4(+*lk{4-QOEKKO$e#CHL@vklB
z3oM@>Qk%HO>9Qn?L4#XS{Oh#=RjxfwdR_|>H4e;bwKBIC+V`TuV_l#B%be>1J09pf
z2>#hqT4xocnH+nyB%}49(}hr%<Zi8y82j0;-8%Q}n}4@W!%O7*k!Y8w)3R<8ju%aK
zHsxVX=SV$aaf<z-MDWT}XZ@Z}J}|8|=*)r}OFnD~)BfDTHC^+S(95ZY%a#@!+B;vC
z`x+afp0U+kXf@+RWtKYmlnpV3lik)%dnO$1l$7e_{`tlty`Wcj9{XJ6>Mnh_jQ#q2
zW&zblr;qF|Is572=hA~gcRZLjvCjRy+a%(Wz?37iQ|JCpD9L{)=a|f`+>-Gkm%(Xi
zx!0C0Y!jwDkpD9`@XP9`tM@m8I!W?-j$LEC#&hdL1Jil-=gHEi1N(WzrdwK`aMnF|
zp<J%)X@j(n;IB=u^BI=>)iaIodg;T<TJS;jkQnpzGu4~!$r;;L%`eNZw~&?fU7cL|
z=lR==Ptm{aZ*asfbqs&CWB)`q(FxbC9Tat#s2OUpeCu|jNt4Wir^)H;X8xe_u5;Sk
zzq+Lf%jNRgLc~lTJi6PkR4pN5^BhU`A8iZ7V*^j#JD<05<@1~6dMnm_bWtrldQQQ(
z{?ntiwO`+?;8fsZ<a()X;v_Gaz`4t6US1bZH($iZ1=lSa-zs0(=58sP9J2Lt)jlSz
z=S8tkWRHq1tdD)o&|@SjWO-I^)23wcV-h=aFW+<EiJhI6s&dz)^HJQ2MK)ze&rW<T
zmM5f?cC+`>rNk{4mpE{2j(Y#XuF%hbvu?_s=f2sqUSvG3O*&k7I`U>-Z|3@a28;7@
zQ}wp)y*qQ}Ox6n<)~K&+I#+r)fl(vt^@@E=mt0C0uTg2fs=0mEZ^i}X)$hJYH0(Io
z$?o-{nRS_f*^bE<vK-sC$}AM_x_Ojq^&PpSOqGJjX4f6t7S4~$mv8;p;rIPg&EA5$
zE}{*rCElwom|s5EVr}R&^b~&D8M^k>U+@NS=jo|_=T6kJWuJNQd1LSOLcb}(>02{4
ztk^w?;}GkN)`o+twyKt^PF<Ne!%0I_t;i#ARqKNEjo;G0{(3(7_4)5+Qn?rYT(tLA
zW(}5i$+lVV5_Q#;Ye7NgvMW&=XJ@O5E#|PfbNGN=<b%jLF5K=vUrf-fdw2JiS&_Vn
zSR}6;ZxWZYeEZD6YdOn!Ky84xK1W-EuWbl$t*lwmrnWo%t*WF>_?;PV<YtK}N{QAm
zd*v95zUVdxH1S`^FZTM9?CTw>Gfz1rpSrqa-<FbdJTtkfHao1Fy7A_bUHA8FZ(im)
zLnG$po?r{6K9y59?WZzYy^Cn{QVf>hukzLY+#(PrvGrS7dgCu4hduNEt~+Js^4722
zxKd*6la*StON-{V8gs5;ShgXy;;KN@8|F!Cn4TqA?M;-F)qUKqv}d;YstbobibIkl
zUV4|G+4gD9#f$|9y%vYPG}tPhvcYI6*L1h4V<8vM%;gDt+U5{-E$@9?=!$nYcdYwp
zaP~U?w)_<<C*{pOvEjwp*G!3#nR(N5rCgd>8<JJIlgjGYSFVgw*f>p6XQrLs@7N@x
zXW5$88fSIS-nsZA!IwKAbmoMeLAyR)zYuYC)xCI7581BdC120nm<0_RcNi$_in<)U
zy!x_|QuEfn;#cdYN$Nx{tNmOT&MXm@FEIDTT%DPAMZ3ahMPEwbHt2rgWz}r+Y#x6`
z*W1wTK37-xPmYUb{_*35RgANgX55=;Gp*JgXTNP8!XjQ56UZo${(S1`qe&*gYbP8$
zB%bz+#dfan%Lvd!(a{@|^LP!dgr2mxW%_OEa%exD^r}~ZFS4qsyeNF#4oQdLlBJt?
zr{?m^ntb-iql#HFs}^1eZ&3>=5&T(L<8R=5Lw?HBw@GtjF3Qv%eds>*cqHT0@ZJqP
zzC!WdtEYw8*Hkn9N?=^<lN+V{*TTY?&HGlO)wh-vEuT+x`&Bbb*6n%MKWpAC&+2M>
zxi8a%OLr6m{kg4UyJ7vjz&HF1C;zf)b@%;Qbo2eS-rM_%dvvo*4Z3|D_U(4jWLkfg
zDKxjpv$sU;{O46>n;VoFFZ7CwH{|6??>DKwE1#P+pUdG<*13!=7ZXm(ZoKDt#pOy1
zx0Fs{KD&bL$9sFV83f~|KKr&~|H0S7nSx38to%&Va$KU`&r-74y^r-n!iLVJN=%Q#
zD;nQ%N$NC){rM=idyYfOzObFPPqxaef8?fVrRuiw`;qj;1$ieDLRVMSZw2k~zmUop
z^F{mIL(P~8p^LJ-vP+k<X}3gj-))?{P&fO!uN=bx&*fe5mzFi|=RG`cY06fcXJ&`n
zR+PM(Gc~98_NxB0Gty^|=0#e^y}5K)cA2zR6Zi93KflfVn0&+I(IIgYF7Y2r4*YnK
zA|ZIHB6O<>?^Dg*1*_urJzbG-S^mqjHxCaixPR!?l+$n6HW_yu=(p3q7nV0itN6*T
zrZY}iPj=aFy8Ld7`Gt=K2L&8Ia0T*JPkA`|tF3;9XURRCf~B=}^2}2;-NYtZH-x<8
zvN|6sakzb1pofg+1uqZHjmIX4ocLDu{M5H2%7@RF>YWYw^w_yHYg4t#w>aKwZ&$m9
z3P`Sb{f{&7jlm@$pAgHCbCSs!DMzn8T;j0kipExkRkyYn{RnhfwSEr&ES9w8{|)ps
zzhq?CJ^HJCdv|rq1}oR1{RNX-KX#wmdT4)2i`#Q8Vfi&1F7Dhe#%R%TZN)(@*$s!D
zn#8BeC^9T9nYCWtCA&Op!?#<nx5xhPjrg`ZDq{17RF~34ySe0b-t4$Psjgi1dF$&>
zOgTxha}twe?@w7ST71%MLwVkTRbgLOgNx3?*KX{Zwdk%;L0h}!LFa3`ELIzv^05D{
z`;&EINqQNNINyTbv2(@Ft=sRoH~;#5flSVIQ3)Hw()OH8IN5(L@|A1eM(NwZehW9N
zDylb6IH+}2_tQfk!5o{JBCll?j@vjpwC)aw<nDEPSS@xdXw3y%M}=@<{mb09>WwSI
z3bH#_b4cA_-Tkir#2uZSdlO$@iaI1}e*Ch_mbXpY8Df_0HOca9&N;}qdhgwf9}T_~
zxHGTac7<_bnEmdhELtu4ou4M^@6=oHDW-ahVq4IWh(y10;ib~w?Nm(+)5KKPB<cR{
znozPkV){o1lLr5)O-t6d++IB6jPmBn?N=8UX}sFvY|3KLVAr92{zK^UrX{SkLXR$(
zCw@yyyx3BFSaOzW$kvxrX6*|*Cc#j!ZRykvReuB<RgX7aTF3e1S+vDM-{~56(s{y)
z<gaEfi@Nqpa^jWq4n7wgwD}HC+v@!IUR*Rc!@3z_dgqlH!spMO+nddB{OgRppDx6g
z+`Kf`tLAx5W?uhmL(es>?KiudVx4rhatNw;es(WRuLuck+9h^kZBg}!6~Ujb`F)@B
zE;UO-*ZY=<$|?utp4k$UHDawdX9=!*_4gWRgX!UGv%;=N)(OOiY6>m&S?enDbWX3a
zt?r^rck*5=H`4tvxBc$?KY^M_I!bHh_e)*c_cJBq;tXLg&83+t8Xnqbug`tgs(Z*~
z`MnhF7aSYTZtVN~_}k=n6<rMnh2CrLcd}xhnYU!>{&(jq<sN;!HRY>IhrCH}3q#-X
z*+o)fzq~eZh~6x-;*@n?|8?Gu@}RdWSqt5Ip04eec&EMg@CEUE);6ZoSh_Fne_FUK
z$|dCZWtIbn@99cuuXik1X4#=V`7ZawmM<orw{K*Iw_H{1EPdJc(`<WpDPzi}TU=FB
z0?)jQ&yGqFdeW}ez|XM2e(t^-i{swCRbR_fpw?N`zn6(${pI#kbu(TInN@aNnZWtc
z`INiFK7mhi%VO0GE}fQRNX=TeV_~oB=d`p*It!(5u6Ww=*eSsLYzX6lnFTfiUjAzq
z_h&4ga$Ue+SIX=9)W1&>n4g&>KH4pzvvxbj0!be|DIJE^%}JH1xx7`qOaU{mO*AxZ
z_<F=Wn6tT+;eLxl`y%bc4GbxJZR57+JXm=9o!ah}l6i+GU9qk4dT~x+f0RQ`=6>l*
z4^QzKnd+Qf8#K$0xANWHPYfO21ydA*epYA-RE4howfhPr5^wC<Xlpeu?~s(n`-grK
zAv^MxZ^>Bl;#H*Xg07?TPtC3*-0W5=f6g!^R(pH<l+f1D9qSr&<+8M+i=Lj6PCuWS
zzxks)@AXI5uG!{?tk`1z!>lx>pMR0*6X)bt6ApGs^(pqI{hc0ARVom#`#FL|Df(5z
z@?s;sEkX;nb<PQXpkK<~@48Z~roM~0@OMUnnC`8OPgor9sRrvRv4!7}IU?F{C}!H%
zC2Q2xc9*YcQ`s1@?clYDjQyGmT5hjN?o>W9*Jw>$+3gLQRof)bm3hVPv^>>y&?wx|
zFP>?}#GfkvBo3^-WoiAh{?Ez!-}S!_|NpfA&(HsV{EyW(J8a$>aPRf$=$+?Ym?*BB
z)jIitymk#YL*J=~3#E2Fdba#;$8VOCwN_yh?kWF?+Az)4AmPT2qgC5H?|j>vxaP?{
zmb(W6vu;|>JzC0s!K+SXm7H7L7Wa&&XVsn^ELkD(SvTQ=MXu0GnGf|RC(Tjz==R^R
zEidy)bS%Tf-{Pm%O`WOKytOBPiC6J&R?qW6uMcY*MZAhU8#?j0(u$;@-8L3q#6^p;
zC8xC(>u$fH8xyWk-1sA9gPJSbkGpZZL&YAhx>pQtw&Z-AVY<&|dhy8{$673srTZ^M
z1~)LCUo?G^{L^XQxtmk3=rJrmp1FVbWO1I}X<~^M;X4i_Y_zaFr@J!e<~#YkeYyU6
z_xD-k<iA*+7x+C&&L#W!SBYh_p9kj3`oyMbW!p#yv!)4tVsQ)hdfyy6m#cH@frN>V
z6>lCct}*cAI3U;i@7LL9r@OCN?k-xmUHx}}uX6y8{ZiI4p&5dA0+W=yKA+q)Pe5T=
zptk-!qsj?p@22cMpS`Pgy6%>kLqQt6D;F=5^m@PgW}Ir(iO)4jyruUK#paqh*2OIF
zJ$(G<!Hb{&|4jdXz5dhU|G)n4{h9uM)&Kv0_V54s{{P#bM>(e*VE2w$-|@D^Z?Z}D
zA)P}mGZ``-ZD;HYT^5_I?_2g>VciYCrE$CtIdQ^OpM>Kjbf0*sycKLr{~N<s{`llO
z-Ytvv6up?BU0Nm?qAO{_;2T_YYjydjK9lLsMcy$mY~xtkeOB|(tyzmEMsTkC<l&uK
zz#1Z!{_5)GMZf;`{wS|oxc{H%=I@^@q-1h_Z@k9p)>U!hw-Lv(n)174=5JjOYC1Kk
zR84E+Dqoq(<}JoDEp*$SqwD^Z$v&Sc#ir}T+rVU(uq=P|+TJM|8^gA)`W_7Kr$=x8
z6Oyi>BRak1?mQ)}sLR1DNuf$w&usO7?|fhWwb1t1$vZz67sSdPZ{4~`U2M8DoA<iZ
zpA~Ttm(P6I>YV*L?)R&Ey|rrh!fOg5)$d(al)rY(HA=DnlE=xDC!^-Nt=%;51B=^U
zzTGXy=Q=BHbPE4|VMXU3FXMA!Ten;1o5laTJl+0(c=Z3n`)~b{w~PM&^x}*4l8d%C
z%zyBA#UJ6NCp+49aHX@=zUrRMuz;JnA-uWNNt?-KA8XQjxtxb;UksN&(eSZjyO87m
z-0zpy8=+gbWv%CK*>0_5;La@GQ>r`HdsWXAmY*Nb#rr=Ne^&9y{$J(Ko2&C(&e$A$
z{{PE<wPl<M?Nj|$7lbR$RespKP%%B3|Jm)VO~)nvoJn1~ox{Y^BV$vw%OZvpqXVBK
zuC9Dj_HHKY?JEA+Qh!A5{fb+7>U~Sk=W9H1Ve<To5}(-a)jQ)6)l#eyR(i_b{^x~@
zK7!jq^n<x7`GUolW!=kRX?U?}P50tyo%6P=tGZtJyz;rhyFX#qeC7V|bZ_HMtW3Z7
zQ)Y?v>J@BN>W@VpZer_ZnW~{VUAXAaJkFl=7Mq0**~Mji3|h5Rbb<EsovW{PZHl_u
zz6La})4tYpt=h2|J~sxTr8%}URr$@P_n+d++o*l~NNUX11qZD<uD$s+yV-s7;f;M!
zi?`^prB7-;Hr4trcQnua${S06ui$Xu+xGoQ)%T0vo<8~X_Up6TF3YE$Zc=bEk$acm
z$5$Gj`qpSEht!sD?_JMr-MsC=iSAuu&)5%#iSRDEP$0M8rg*vavmY<#_XXtG?5*4Q
zcl!Ul?>AE(dn`(O5}D-Ex4`&D$D%aV!##fjWo|Rd91&eO)ga=ZX5OJG5?#HY9_dc{
zz4vF`oFk7pB6ud>P21RDXYcNzV;kMNx#8>PQU-(FxrMj54V3Gne5VSPl%{v8iq`t3
zs-I}O5Uw~~P1EXhbxzpHzl<f*<TeDJ_?xh)I@PxG?CWw{?_0Zb+#ktKbZ^?|^ty{R
z`r?J1brLU+y*gv|WU2Y1tcWv5#S~K`W`3%C!nM<MThUspWbRE17EBP0+I~Cs`?rFc
z%SC~|c^p1fZ20#vw=SQ@!9yoZ!gR-hgu9t0r+&S9S8KI1nNxjX;*yr*Q648E>ula#
zy49ZY?TEJ8#Me%BECQLmH?J;;y>~8j?W=I`#&C9B>uu-!)USPG3_165#nSZUAJll<
zzh5iVb3X>!s{H=;{(ToDkL;PedtYrqP0gxuqtxpjUcTKLuiu_^n0-mHvow9`vu#Ba
zUf+HEdG+tRf4^PbdurO0jDi)b*4!<ue;oL<xadT})UDp!;TFdaZ;a$R@p$Jpjeb|g
z%?Tz_Rt1yxeLDR&@Xp_(%fsuh?Jb{df0v=5_o455%Z8K30#~%ozkJmEOqj<#kByO<
zn&;#U7>{3GlJh}tcS{cUC(UnHH}9!fEpKOf)3%KBzUV*YghIs%!;;@Bo0S9|G8$M*
z?=Ywx+hXL?^YGIWhdDok-!H!`&QSN9|KEd@<+|GSf4<&lV3->0p_irA)Kj3Y^M;pY
z<JlSMYa`sBPI&*t;B8g+#aC0d=gjk3aDk_PmiNXJQ7=S;9Qn_iz1+#-%;5Ou{G7n*
z&(mef*pBk9-nf4f%Mv-y9TRTN2`$x%R+ZARaxgXHmGp_VFxORM$hhmLeRpmlo5J4x
zV(ZVY@#;Ur+VJA1)c(yaVTo537(~Q7#H(8UJt;pm{rYyJ4NlDgcU0d#GB0U7xGVVI
z(dVaD?J%^;GIeLWm*o(im*-}BA>n0t-j`KjU*AIt%BU*e)!qA+*Gm|f|BhY270<Ed
z=RyBEl~>v(A=hHW9{!p=_vOC58moS!E^Cufbh*soa5y1$#?c0$IG@ug!t%$$?+1Q-
z`N2g($HT+#A>X!qDQDO72}>tB$)$P)?o>)V-xPk~2}}FqmWWA-tvS==zhACqxO6T&
z#kMrLc9-!@(OFp!x4q<jtN$Z5ao#?yJ1rMZy<T@ECr7I{O6&H69WtAHx85_lY5V)t
z&ExC6w;j7-R2_cc(V@TGIboOerFC9xTy?8p1NV%ZbD0~AgI}v%dn#L6TFLOhOa9;D
z|9}5K{CK@Sd3X7TC(-|BZ+-vo@7gch?;a6udHeS7h3${J&HHy}Oj9xbzv1u}?H@PH
zxn8!+yR$r&^>*u`*vb5JmDV2eWl82t-<II^>VEZuY}4coQX-;eT!$k(Z=LLN-=E<1
z@lljW0;}1(lnp^!({+VRHRrG~Bu{X65KPzA;9c_IOPe;slFM6Pm84r$*~re1tEn@t
zs;i#yo?(T2|KF*c&+VtJTBEx^UQv?g;r4Cq_y5&#IhaJeiuA2eIpN92{eIG=-`}^h
zoUm9WG;Nme?5fY*E|t?wHDaT!o$TUv`9xh^^$)TzZbNA@+vC&42cp%p*IwA6<-LBw
zL(3Sxt#3C8{nH3;IsP@`zFkIr40FXCJH43cQF<-8?n@b4`Yd#u^q%Yd|Jl3cJi{D5
ze>wZQJNvkTEsPjWEPnjYE%kKsTSp5^js*qlikvO1HDe<E8JxWiTg!$lzZ5+AjLbc~
z-=+!@-@Pxr{k_h-?ylOFu8{BVxBoXbsjOUh!OP2N#^J9&zF)Wyz9r$okK0=c_B>^p
z6|5iJ%WeMhpjX<t^JdYX+)m`{{+Ob)VX1{ydr0<1bFB?(AxF%=ePLK}<kX%&CnOn6
zT9wXqY(CU;=h?N7$A31yz54d}{;R+5|9x)#|L6ag2mgQMfA_cfep-ylOPRurQOgz|
zIvaiCYGUOieOrZQ)!lh|D(nmN&Wf5ZJ?&i5DiC-7X!6C6p-tS3Iuj}n$n2UVfA>n?
zufRnDGYS>tzE~7%an+V7PQN{Og<s&@ASE}p)g|dsi;LF2)nIbq6>G>ejND-Qe#)zd
zn{)ndKF*+L_x^58-Nm1mZNF#!<y|BDeeax)3Q@oJT-zL;y}>UtPg*GZomKng-61h+
z`PaKNyUr2$Wq0RBdZ*v%x31eCwnk+Lc9k03YdoWNjj=3k!K<7lUr#Mg41K>hj3GNz
ze#@HvmHBh(GF&IGOuTk`@}B3WwxX}4vTh5uJmHuMs`2V18Fv(zGfb1zS$S&O!w(Pq
z;!Yf%a_LR#iHkpfJ^EJlc`e7o)3+uavAQ9rVrBd8JJb8sYYe5XURyMErQc1aqFsrc
zqDQ`pJT2V%H8ejmlJDXH9ldrR{)TO~ulL>kwe^+#zpy{`b~1ks_xt~Qct8Jt`rE~s
z-``I2m{pM`y0v<8*XAFkd>dUQbRw=zwg_gZnC+&US$c6_W2`}y$%W$&uO`er&wF3Q
z_Vv-Asreh&y|+%AA<iOk+TXoxrP1aKvmFF<&-Cs+#SjxKcj4GwKlQn9tIE^kkItIw
zxbI`;dcKy7f3BQgZ`|T8KT%Z3y>ip1b>UA1eAfrasjcW<d$@gXv{3}h${4k-ScT6g
zl3RO{m8-93SWR7cJamT6$+ibu!mE#1cKqn{uQsi86r3X%-kBhu!Lw`9Z;=GW4~Js4
z%Y;o=8V6fEm(&UF4rc!0zRrkYZV;oyp9i}yi(kGi@AuyIuSt+WY~tF!*B>XJxVJO;
z=<Aecccg#pT%BH;@z^hC3qz+?^rc=|Z>cq{?Ybv6Ogp@UG2(TG30KBVv(BjJ{_m?m
zL)iDhYqjJz9J|K&t;itDs5W;_Cd1O5TfernM@9RyWO}wPYza!)u%t|e;mwL(AKuo3
zPR>c3PofewX#6QUr?qtY9*GP0BJ<<#O5fg*nVWG==u^_(iHW8>)2_2duQk-Xqw$qT
zER$!FLru*zIhE=2CrT;G9{OxPUA)9QqvqazYs2`xe@}(y@89$}|MuPO(Hv`xrdO^{
zeRgd9ek<05c-zIx85nMLaW#vkd$q}L{<5H8^5HF~dza5uOT6hA^Rd%5LjO|Mk|@Pp
zT=_rEj&9qc(XVO7-zjrYY{vZO;!8|rV;OGTv0{7B*S%x&<7Lw>ygK#mhrzi(E=||p
zrKdh~GIDaYcJJDJG;+#u+X%ZryWJbYcl;4Cu-NIiTIb!;XSrwoAN?gW`>(>|6N?H9
z7MN+j;n7QBY-O-Ia7~%PO77GhC8vqiMe@C^E)UgOXIDP#`#K}g(#SO;P0L56XH#3p
zs#x2*_jTgu{6ByD=gq~vN{kIQ{I1~-Kh>oE(fT+^?d<h*&sM8a|HX?w-LL(|emGso
zxHawF>EIT{yQ({0`DGlv6lLnX#w>7GQJzUf>#=QhzN@D)WQV?oOdD;uw&jy@!nu%h
zq313}n!J8nxaGK|wAMzI$^yoyyOH8t40?BbAMPp)Yd?E%&35$(-V!?NW)@v%EjjA^
z`EUC2vuXa)YCYF?{PX#2<oU?w`n8#AD#xc^Vo*EEqt-U<9Y=@oylqc?{_8m%J-zwY
z>$}^d<)SK!7uOak@=i4kHql?KePH$MldhHijK+RbXNu{a)43Sx5nsuCsYt^r?~qZ}
z!}sA=KbjOC?LDrYH)*Aqb^j!Z-ecVJxUXnEJsREiy-@k5n1-BC4&#{#EKCLqA73je
z`RMuQ<I|HI4n@iB6Xo1)`ReK}3wiTgqx0Sgd%;-71pW5`lcqS$x$?#Q@Vkc$v;3cL
zO5tm>nRaLLQg-9$m2F2BWv_m-bi)xhMS(?Ux4b&68r;k(DQ4Z^GC9K_>_qHCz5R72
zJ1P}B3bOPWT%tnXzv@}JbKkwHO|^S}n$3{XtY{3mIVt4YTq7NpeftDHRy!oUU3GLu
z`b>GAuMvBKC3GC47D`WZ*tq-Xd#(>WyVw{~8bm{1{1JJ$D(q_=_$&>^?3}wLC5F1c
z7=)HCYkkjRIL`;N*wp;n&9=H<d-MPQ>a7(M@{13!KF9uS!ojEs+x5j;!^{HCoCvPS
zsJlC_Ta#zrtVIjg2Zf5At9)?vfKIkfL)fxqt!eW+p2s#uDm_XJjgEiX6cEE5>vZkT
zj6{WXi{v$DB|NsiuTyw<;_F3Le}9yo{P*C9(u1fpW_G9eg`9VVx!Sk07l!Wqm0L7h
zEBQ>tvvn`k7`EJ-WUjrTXT8Q3(aO03_gieHojGF4eXrT!-rfFFnw~mk#p`~nYhBCP
z9(wM5yLY+Izs^gBLiu<8EuQ&zO?&_5e`yMu!k1RFdoNkKskil@=fNXI>JKyLXa;Vc
zc*rKidHz!24-r#N6;7GdVc_%9t-*A5^zSUA%6)Zy_m29hpE>5py#Dq#pC#cB<r!A$
zc#50|(O$BC$s#5Hq90p)^A}%OApD#?GurBY*p9Wo+VvC}&#;tuT^F7^X`j`)<K;o_
zom!`EP2(+5>zulx$w)`4x5)PCH-nbo({t?(hMwMgsN68ZzRpx-p0NL8oA3tTkiea5
zuJ){C<GHpf?CU*nEG52P6n}Bit>ZVDIC!O)WQvpL3#Qc?)|yIG3jS(k;y)WRTVXk?
z<g_gul}|#;9DZr3Y^=!mx8iU2#n;n6y34l9zW?y<?glRg{mCk4ulM?XvSBh>pkXW;
zsr8*PA;@B_Zo;XmLknvcnzu-PnqBf!@>%NjOR6z5_B?KqSo~XF`bMF8w&`Tym13>0
z^zFPDpB%j%Xti+F+>D0pEJ_S-R;PcdRC>vJAmuyTh4tla_800uUcUeD`u|_|^y@DE
zzWDmd&+db|OVifvs5Fh({_T>ed8(~pTF>+2E(v?)MqE(q(pncVN90q)-CJgT&$n$X
z*~G)8^5Iul_P68Cf|q_C?d<q__^(3EY=elX{HI5wgCb=X@jShE#34p3qRlOLn}A*?
zgZAlk_4!8Mm%q64Q|rr14*Lbi?#(?H5%%WC;WEXBC~@(%=l{JtUH^Ig;{T`Xzy9I>
z@B93I%$9SP-+KGmox5$Ap|iA&d+Jl^9eGM>5jt#wes>lW`00sqr8izl*0I{joOZVV
zom<7$H`Skxey+U}dO&B|s>l;w#joeNiR}Dbn7Q9O`eN#gqq~H|b@$(E`YPM_cuU8R
zTJvL*BRY2p``z6C^6i?e(E8ip9hy5fIk$g&y7S3;&(^xvGwc;6rE_h4%dP#k^j=Y8
zWXqL=*4JCUrY!t>d}<4$luJ1Gxpq@IH|g)+uPL>1AKH41SNmbrr57f_UK5g}Rz_Jf
zY^yZQHkCTN<CGcK<MTcX*SOnEdv`@w_l(3QmB*}~w_Ka+s&ahUjnxOO9v(e%XojKY
zG>hO%XU}{#y%abz>t(xt>z7vtg&caW&HH=Cw)%7E?r#^rzI(r~>T~GtXZKck#n;qb
zslRAnebN4O+`d2M=kNX7ot|%R|7yPkPmiV6+-k!ex^s=wVs?JqQgHloO47l5lC|0?
z*`3U76ZgIAH#24C-C82VYV)_m=<)4}`sB)|#sb}+ZtSZ%COdD{Od&s|nVs7sE*+U7
zpINAP(LCn-La9k}xSw;SPjv4o@!`KPA>vR%VdLthb8lQ)zC?O*gq2RmC)tLl`wxD2
zbKdTM>d){0Hq}&Az5l&FF6aN#^w;}kc+SrVc=@~e-mh#!r@&MBinAB;Hiuq&&ama1
z8E;&(Y52-(f^+XM&Udn?`dj3l%d+mp(L*^8tXEA~roKP#lINMK>V+q4dA4&VUY^S{
z?TXQ9)xC#UyW3qHHSd0j)~($jG?h8y_U2RfJEyWCs<30<IL>U1m3mjaOWloA`q#@t
z65ong5+k-`CWtATMsQqtJyS|(d+S{X2AQ93&a*X_UaqmVaX9{M%ihg)5%*iazj*X>
zvCi4Ii&uG-L|yx?nSXu?)27G!>^)yUJ8S4X{hJ3z%thy|)%h=lBG27#vT?fkdv3z(
zbCZ@EJMtfJjQP7oo^i#U8D|~YjdkZZ`F#~-DrYl_n8T=Y^-6PX^y0cJ@9*vIG`qa@
z%j9=|+wZP_-uy65yLscgrwJc&e!aYMYo+b1TZ>n2J+`BJvGVsvt%lEzCVWUT(8%5#
z%BN-i%x<Sl&z{IDb8_w^x!sFkcocd6-oDyxg(v?hyme!ZYQ3fW>db{L&wC`><y)>e
z_50N6oLSlueLA==W_y9Kq$|(cBYclmPnfR6*KI6n#<enh^ZI!|RgNzR)Hv`W>ixvs
zla?q~C472&o!?&9|G41V(AyW6hu4?YzUY1Pw#4Z*yHNQ1rn^aXVq6<8Wr&{*bzG*i
zoa3|)tJ11Vvv=-hI*_n~Jz@K|&-*{zU%;cmw=G}HVu|a*U8fJ-Y&D73H*P(q)Xg*P
zzs@|jhpk~5J>fTv&R&1-64j%>XR@pC@-~;{=XCp4GMNRgDk@#1^s@_mYyo6JW`pU*
zcSQzG=bYKPuLvei*b(X}pg1e%nZCv?6NU@cGndWVC3sqVo%Y`SKD^W2_sxCx(RHU}
zqI6Qu<JaBM41WE&wv1hy${c1rd#yit$);4PP?HReIjmjH8NG?;4<&5fB)8Q!^U5R3
z>W4l193nfNuQqvTv&f$|ZaZyy*=YK4_jTT<S`IC4-m&P5)|Cg{Lf$i^Uu`|=;bdR)
zs%v)E>($?%^$TQsmK1)zcw5~+PPQ)h$3@}ozYpA6Q^a?o(I%_RW&gV6*OslC5tY5h
z_!$So?R1}KiHH5R#VDnwL>_l$5MTRw@x_<7&;Or3UHbps_($#AU;jUy@_5C|)|VG1
zHlCT><0kaduj}9v37#{O|0Rvpo<x<c(~3Q(_k8uiUt0>6o{sm=_x`%R|N5`RThl$>
zwb$`3{Ml-e%-JcLE_T*{k-_TmW>4{Vo>kfbj-_|yZ@zT7VR7Ko+y33wpMLyWm9r;Z
zm!WczyiP6qn>e{To!6k<7`|H9C+Qs8QgHn1j-4wq7WABAh_Q`hUU234+)MvWmrPV^
zR(-OL=Z2ouhSiITij`Y<G^W_?Gl<y2rn|kpC9K6=sUluscKzDzZ_Nud_Z2k${q=Em
zVyd^C@R2ijC9YqxY;bX%x_a{I^J>N!sqgPhULAFHeE@iTI;1Se*lljc*1JNJ*cm2#
z{pM9HeMsR~H`4;P*XP6@x_=f9+G)LAy!EVE$JHB>Jnby{KF1cXRWQriaN&kYOZM#x
z{_D5*znUG)U@G@&g)8H=!vAW$E!7^%YajU>ypsC%R8QjhsCzQ!Urb3{_|Vnmh3$5B
zm)bdUCn~LC4p>il&u#krQk-1+D%Wl`6JNvRV^iXEZ31}$Z#r*TKhJYTaoir=Q^`|v
z1GKlrzJ7SS_t&a9eg1Ekw=<N<=dV}4ZSMar-tNziuWR4C#a9}fDtQ_+F}3+XL35jP
zaQ5k{wv9cK%LBV_PT}4Y8_^;adei={;rjXe_T0RCeZI`!FJ+%Me~o|j!@@`Gk;LKy
z%OrPgn!jJhB72SLH11<EzFh~8em>v1t4DJ33#Nkot8O^GopS!snW*n?(yQ(Nzdjsp
zU;86H-|p9!v(HXnFqoDz?Mr2){rAc_D>D~%Zd}$n`^GZ&$hn)$cl}9wnSO0sy3?)L
z@9t%{-j;rUxBK2Pm#f!TtQTDQuJ*e0{|*PQ#fm=<oj9uh%ewI1Jl)=?xkcg*Ysw8S
zC(JbHHaNfd6mRFcD693$e>2RfZc@9wVCT1n7k6gQVNM9Np6VF&meIp&%KyUkd+W+-
zxgS*h=4hx2<Nn;TY^^^VcO#p2);ck<8kuR{d<si<PF-IkwX$>W(<eL&KK7lQ5xVx(
z`=F2YALIXs|KESFE%fjI)UA)pK7N|6&%QjJL1dlVx1V2488oiyv8DTcH(SitaB0f+
z>*klUi}j<~4fdKT?)vp{^~1e6f8)0OdmMiF+~2idvhLiADt~)+-|lyRz9s&AR<rs3
z>u0&;i}TL7?%sS>bes8huIsOI!^D@XUCi9~aL2sfZ)^?Uo{1H#4f*}-RQ8e`jHh<=
z=Krh9-)>u1T=etVr@G~9Vz%9$Df9ihcJ#knmpRW|OS>l-+3j0ux8v^K-VHl<?yM}X
zy_=W6jd6=n=zWE(@0V7TK7Iu{k0CWWKkrdh;`=*yrep|QJ?kr}$DncQ@i(_wZu34{
z%*(sCvnGV^{c`IXoA8R*nAquT=E0NiMpsu`#?0R#eZ`!uj(HuUMvhx<>El<tqAPPA
z@7fV`v+Z7vyTsPeC0vW;?P`mK?}kr~=ZoSuH@lPSp4;nYf7(j=`|J39H9xv@zgBVo
z?LK$wtZ0Bu#inh$YwPrPzI~IoR6}pI$Ia-N{1w>;{ui5Hf84um_r}M(cNWOz?zmgQ
zl<@i4C%0a<3qRj3*s<|#@6LCZK9(3qz5l=TSNPn&**Sa4z7`~Ze){$6u@irP)!qMO
z7$<1@>dmpq6X#mZSha88rf<J~{rV-PT)lT^q{L>y+&O(Y)q4`zmrLX)6jbG&HBDH*
z{%vn@^X2QSY+2WdnyvoztA_ddeOoi@=Cy}suP}ZY-=80!|9bDYqYCHDuid}n^7GiQ
zv%WX)d4QO2AF!D+gx|?o@#5Ikz-Zs@ZOxgfb6>xHS#dab#f!6U?*n*mYh^9pEVB68
zSJfMLjh8d*ioW?|)vX=pwC?8J-n=y<_wrVG+rPap)>IeI{nP%le%4a!=<v6<KOTR(
zVopr&&o9@$K9%Uc(NwR(eRJZfyGlZPckN?H==Htw<k_!nYp?k3<}D80_3PoalHk{`
zv$vOB-c|ZM;O$4Z-@h)Fo8_%v_LToR8}p1`%hsAr*;RNW!2Hjm_&Kwd%P-uM^>KAc
zb?xSh_a2n<zAcp6divL|XH~akB#vb+(Yk&wqu;wUb@#<zAGCHk?XJI(>Hl~0tS-Y{
zd-r*koLVj)e|JfEGQ;cN-}Tq6d-9Y)#p>xHx9Z1YyA9nwU1gi_;>E7HxeG#8-OXDq
zef{2@JuBDWo;`c^_6^_7uJ7KP@0PoD=gi}|tXnp&3f<cuWf-ygSAeTybLro;ap^$~
z)9>CpU>tw`$_2~p*Pa^N4p>X9N!|K8Mf&h!sfNpsGX*bho3cn-jpx-0*HAJ2-2I<7
zNPpv-a%s;hwt@||*2}J&FF(9h;=rtVOp%MaHkFCY`tv7Jd(SERj%%lUQoSnKPDe6a
zcG|w@$dfs-PAu;NPTO2#n*T8QJ(C-k<1xPsM*aiUQ<zuyowP|^bu=W6q4MLlqVj6{
zpf*(=vFZrB`U|C@+Ya(A-a4JXdi%N^mJ<2)6@O2fU%BkQ|3&TNH`l*u|K9TJ;o*rd
zW#;DYt$JAZ@A$X%^*5@DuilH?U-P5zxBai!&u{Ht)PL%o9vo|VcZTDW%ZH|Hoh??D
zHD5Y<vehE5&g5FB4>JDdKaWj5eV&b(d4X8<$6m#TxSz!rXMEPZV6$jVuW4CSgo)-|
zhMtv-#=0@Z-I<9-sf-ejdkV$R^EOOgxjHLm+1~B#^Y`3)xi9kf*FUT4_2d4l-16D|
zJuj~=m*IiI+b(~dM((d!|MazPa~$Us{p4vV;3<|aw07^5?;Q$lMkS`ZOO9K~<*j6P
z*q+CC+x_zTdv#Cc@BH!mol*56^2#pDIiF|LwtwlmviQfj;D)3751VY5yE>-ZE@F|7
zR!l=7v;3NU{koS<D9c}vT&cExvsX#go%zgen@cr#MJ_3^1kAd<scgy|<{R1TH+y@q
zaes~xnO=E%x$(QZ`#u&KW^FKd-z-{`ut`$sEX#$AyE+UC(V*1f0AAP@5Ncg~as8HC
zeP>^<syp{Ye^&RTb<YF!L*f}1@Lv;Otlf1-a4y4*K<UD;^sI_RBdvm5+mD_WjgEgd
zt@A#2kR!(V{ISw<o7&>DOC~NVij1;g6ZEQX^4Bq*Dz>hGZ^N~!^?_!~XSMrYS~2x$
z^r9aN&3B#XkLXEXc;HU=##NUE(mHv6ooU@;<*K}K)B1TA7tUVi{V3q5_*@a@yh*0p
zR&7kX&JghMTzLAD{;kvhnY|S%>NHm0yT<DAa{1Vj$(z5qf2`7e&FC@ZZhy|L{r}E(
zUtXS`|4Q6mZf{*>O`T~>w>rPs*X#AgZ_WQNzP5dz^{Fo!)-{%;h12}Gr)<^ER#!JZ
zC%VXR!6T!CjS*2bn|@m71RR>W<K?|_Tf1ci!oTcf{3mBVJax;5JG)0aJH2PN*ow7{
z|1CZ}yQVMVcU<xj&*Ce&e||kY`7gh3Ud`LL57pHxV@kK1&-=sH@bRhood%De`)wC#
zt9{U^n9S8A_~TK6NKv6Ys{^a8eb}OuMlYe?z8|e;bza<NdwXV0Y5Jzq_1FKFW^LY+
z;`@HDchy#g3$Fh{uL(bwdbxcT!-}=5w@&WvT~nynVf^2XZEd7U`gTXVea0&+JbqlL
zpLM%iEqbbMruox3Uzyg_{8fk&7F3aUn_GSB`|nD{k8bZY&%QNJKDwj)?}2z5o$L(^
z9#=vmyj~_1T+?`G%5Z7}!-mAWQ{w90!B^2SPnA_<UvNP`*3XQi&g#58<JPd&*Lr98
zyZEQ|2ymU5bUfh6KI6OW1rPHwZu$K#otUVb(`#Y6;A4rO^kKzYDyI@_cZk)0`LXtL
z^0JyKGp%mOF)TQ>WLutN{_EA1M?D`tQO)RJU0^WXMN1}Bc(ZE3felusH}}r`XS-AW
z*Ph_N%0)kPY!$Xxg*b;S|Mf6f$t+txb!)1va4hG#Sz={t{$^&jF?Y>YzJ9f;`;O@L
zBi-9KRnN>46f2PSdj9qI+BM}<U!8i{U%l7ja=@=b)4Tt^q{sh#xj*OcX8X>6Z|C3I
z|1a^*?wZZ_?f(D$pYmgN{O!Nz%ICket(bR2bj#Lr5#N&x1WsB??UcCBktjSdDg95$
z-~Q|0^FM!l{q5N7{L=Wc_wi5WZWr7>=SR+JnJHThuT9;0y4Y0rg-WN#a|gNfRTm#H
zAJDkG*eBn{PN)3c{eRc$H+;N%zxMZw>uUSIUwolmef;F{(DL2l`6<uWevQ8uGw;vl
z!k?NkI}Iyp?tfi&czx%WR|ap}n&<n(KAF-mA%e?tYQz)nmz%YA)d?o+-kx_+uE)4-
zPcQr4*Z*GZvd_vdcUt_d{HKvZf}3;KmFNenj_o<H)#IT{SnG9(vj_4y7p#e0U(KBG
z<yWfRJ*_)$X0?0?jg0oIX5jGDkvumy_E^EvXWw?nwLi4XD0%nD=ohQ%;pT;G*$-U}
z>r;<@Dms5J&cfAV<GP}@<4dE&d(Yji`)CFlIr$A9Ihi08eZ5gKejanc3;#7w1tae)
zuQ+iTbR3A1ZebCBuG9{ht~=4SJo1(DYb;i$&OI7r$0=2C>>eZkrltcaCVMw6t(JN^
z|H$tC?fLV!oU1Z^pUEaDm&0$gRCz7mHuDN6t98e3P6|)d4sKW}r8DViG{dHvZVy)j
zv02L$rHfN~S}G=7bn0JmDse@V`WB@#LZ`JO*Dla53cq}a?~)<I+}EqxoX#B=;LX^3
zbh-Sp*qcI+bc>=Enm%=mxyUEyusmdC5FeY@@>LyM{X*txJehb`Pyf;We;;4y|NC=v
z{=eeypFw3v<lA*G*Z*Jk%l-fDhxv79YkqG(_128R>)cMmEo{AKUq8-NTG)R4vWosa
zHLa=-zb5DJ{WbUZy6<OK|39#|mU)4}be8lSQ>oMU<WJp7Jtg(nC^b8qVM*XT4bxL!
zo?iYPe*e$fhxv7Hzsdi*xBULkdq1T2hn8;t&9^;t|L4=2&*`7%TU==L=Dl70*6)Ar
z*RBq&&wd;D>s8(N&A-3LzW?Xim!{<C#^L;$*CfErW6h1*zs#p)o;q*)LHKzJv(&D<
zofeuWqouaSzuqelTid9Axamum{t;g7*C!8DL`+-Z8S;c7>+Q3e4S&L&8@m14{@Q-$
zak#hT#A%<6)qD?j)ECF}d-^`qdw#yW^@+#CgI>$}?;VXjX=`MoarF1nz#V_IuQ8>I
zg@o<&Ss4FqJHwT8Ye1I_9?$xFmOt|9s=BF{T5XvgY*_oD@~MmggVNHHYtb@~ZeQ@v
ze0)ycnN2%~Z{NLl9LHwmJpOf6ub}W@UfC5khAbVUi01*bZtJ`dVu@^dUX!`hgekp}
z;a%pI&+OKd5A*TWZTo$Z?N#aRd8_89@;^3LHQOh-divkEe_Kr2JhumCWj%S_T5Ks|
zFkdqKg~eh;v4kSQ<bysM)#pFyDy@}Uv(<cW&ld5EL9b7gzWiJn!pCpBC`-{&pu5as
z_EPq}pCmG#9*sWzRzHuyXN|qFP;98__T$H9ZaelWhIs+An0dpM(}9*};%y&)6Iqve
z<E8F5F3xRzMsvmUKR?nn38=pQX_edhJu)d)x;jOvKJzq!Ep2pG2l2|gzxurR^|W*I
zZRPW7iZ~qJmshv*pAxV9eeo&l1^JxZYwl6GmoF;xMy>FTY!2gNFHAjSasGP2HNVAT
zdJpb&2JPNz&GyODP$9zPnMUZ@BT0=+R_6>t`Z*sj>$eMQUfX+5RZ8*VgIi4BK2Ltw
z*Hu3E(sZ>zkEy4MqpwXi$zyz-HRr+l4=n!^w|>3)@$+Z)td1wIIINTlJKix^wd}Y3
zwR=~d{^pYI4&$c@mo&-^_|l?&Y}2vVf7-`>;nA~QF$;e(6^3TKzP~L!-XZ>OX0ms|
zd^g=E)y&0dy{u6ucyyFjXXU162QF0by#MZQT>U?$3Zsqny(j*6E}y8EecCONQ^qCx
z__MN^wX1SwKC_<c9A)w3_0pZHXSB}dN+~S8{Vr|w`>Cx@zfGunw~0;fgPh_zF&X#w
zIU-&c*}Z)dpR9elbJd+QdZy>PJZv+cM6QmyS`Areqjh>ldh5R<!aRbrx;>Z{WIpUV
zD0xwI{^!Txhu_!!zFz-fvV3hL+v$0+%Od%=WHNNDoAYjJ+JvtGJKp(pUbSh>-m&rg
zwbENI=f$R;TYbL1@{Y_Y_1in@ie}nYh?UD7?J%3a?S4(vyO)KLDhJ-i``%rjdtYL=
zc>2{X3=NN~Y6R{wbiGwyv^s2=T5EQy7{i3g#u~3B*J+(mmF(LzFXcmvSn~UfuW=SK
z%VI;EAFW!nh2h2Fk`+<zPYv|%rB!v`V-YjIbn&wq>$!yuFV^)GR<84!zO?n~&bWa1
zpTz~GRVy0W{XN?%t=>Mo>?UBg`?A^1^K&<udc9mS%irX}2b0R?jT4@qVm$Wdn@EEE
zT-m*Q^hEXU^p~&6E8X||_VJU)88*G!Ao~B9XW^!$c1pEdL-`*0RK{^7GaT!A>|<)R
zttRU@AIs|n?^c!EymHHec}iZY#Fkru)%CUcxp(gBg_Q5QdaRbGt|}(He(QFzw{v4l
z=Xy>#)w(&*>bl@8J}D`kLw}n;ZT8tZ;g0^P<fglfUlzW<{cHb8XHy*+i9?SYHq8^L
z)q0Y$VG_pz-Uzda&)zJQQu;ajt?0DZ+&b&UmV|%O-X}ZpDdPpH%Sk~J;TG(?Go&h8
z?BsKwc0N7#_uG>Pf8Lzmf9<uoUYukAgR7LLg7WF;w63h}%^AJC<t(d;?G$p>EdN>c
zG=S&a23`}MY5zm_OS^HkX)``vbxx15#yITsx_9vb;%c{JucSsMm(2UF74FX!^JVqi
zx$0Yu4@}<wH!FPWD*bh#-Jt6rbQ)g1XnJ)~%Xv}AoU7c|CtkfIILSaGWy2vi#h2Q9
zw?CbG=dob}clqQM5)QN0x8LeBFv)dPp05#5vHQ-s2yZ2Z{Q5#}hx$!+Ywj^D=>1(-
zuY7)S=EJ{re2ca|{&&nN>co<QBN2Zl_VqDztnS!Y;V-o{WyJ*Lv<aSvm;F^2&Qki4
zXx%Sax$vfFw^#t*vQ@SJO6Ei+$E@nIxExwMQ9ts^zbSL>q`xW5C_ClyWN}f%4$eux
zkJQ)f+%#{C$)y#pkBb;QTWsvVyjN9!bVj(fvD)d5Wr4rkRCoTzVRAQ5_^tcDy#DpS
zqxFyL{vCgMarX83;-Tfg-TgPq%hbI6^YL)JTEF-7>E{a1E-hYsW$&8Tr>;)@Y|z(x
zZdKbxy=(pL^0t+q=i0ycfB65`=l?(0|NL_O|K#rce}}Wm-+ukM_U~)?UGx7Je!2fQ
z{Nnuo56^wB|EXL2`O6#MMIQoXmQI`{wk>>}s$fy<hQ(5S*E9DXUFjy(!WFFQ_UGSm
zwfIS|46`0Aaph3xE%KUmYL?Bk$!-gML>6UoJr39Pk1<_&m3z$$7RJKEMQ_&@S~|_p
ze7d;Er&eICYm{SVUh$mFb&9*<C5?S;Yh5k(h$k$aXw`9z#b)YejgOJPHWi!c#@vw6
zIh&jD-j8=$O``bT2f3OyVGm6IzYjlM+`7ajaxD-0$E#k=-~(3KvP0z;nI!1_5@l$A
z8zcJsz?Ke`RLhzdheQ;oHF2amMzyeCyw!jGentvI*%h~^%VtVV+I{NE1Ri_-UpA*s
zOXY2xt#SXHHgm=F7hB%&$Nwu3*&bK&{`&WQ-<~|FpR;a%&Cc-e3>S|5y*@AR%C(c6
z%g(%U+{h-Pv|w6R>RwO74ATqUHq%a-C8!-rZgg%vz_4q2B%j14Rd=n`og!a$X*`nF
zy6Gx3Px-+*sb*g%?RoZ}Y^JaER=O`E5*hj;yDEfl^Q$i>N^P@6@5oI(?9AHvN|;Tr
zH)_fBWv=fMR(zZ#HchHH`O{rzYuyQ=r(P+GtdKdTR=>o~eV+UJkl0v;e7BX>yKjZR
zl7EwWyZI5T+}7wqGrdRK8BBaigpSUC%6@si?D5<0R$5B;*!^R<QOo`1?z?S=q7!6q
zyxGQmb#9#O?atXUwfnY<J^b<}(`2p~8^fw@pOuMLzRIE3#Jo0!TCF%<xvatFT-u9M
z@8>+J)88QUG>$7$dsD5^-0RJ*iyM<SdS{$b=v25cZSNtY*&Q7H?|wQ>I{BRGLB+0}
z6K06~llT;BeDtdI4sA&t-zf{@dX-z|9c0Qdsk*cO<eJa*244Pa-~KDhxpn(%Ceu~{
z^#F!QDbduor`k3pEe?=Vd@?y`*O@gL8>CFami(;ZoD+M0(m}15@Wh7QI?nHXN~XoP
z=DO6~{D1G#W9JHU-IzIwy&}DCmd`6}?*54i>-q2`Z#!u1O86?!CTIS@vKI^v3-@1r
zePVCZx=r`y<>gEfo*Q&euej-b(uO5_e4XFgN}o*7wB`_fClewftaESYWO0}9sm@ZN
zf8HMawqVoJs<7|>oMLTv@3WbE!~Z(NgC)#A+EuM|U#nayx*Kc4E$7ynb7zS}z$?Bd
zjjMkfzFa-wxYOgi4aW={>q3P1e6Mv)5oJ=fe)niy2=lU@!oswhGAsh=R-G44Rk?(3
zX4rSY{hz_xNh=?%_c-%mn)HINVf<mQ*QU;E+vd-3NkhGial-Sgs0@a!WtmH~=W=Hn
zN<85?xo*x(u^ZLZpPt13?dSjhWd2h9_0=DbrGEd{{X5>;&#vyb=l^fo(jVXF^4Uus
z-M_u^b9VEq@9S5;`&Rhr;|uNT!|Y&H7rSgDzv};+{k#5``u?x_|6abgso(wg<;lmd
z4n;A)x1Qi`rOtRbT4|oiDb?#7OlxP^I@e7!2%j)b=eNIBL6!j9>xx+u?#&kC^D1tc
ztRHoCU-fBDFOxY-C!`4IZ+TUs_$~Uv)uxyB&jdqfSZd^ZO)~5)UCGp7ksZ43>YMpW
zL3`T{o?<)@X`#*BaPcpf_t9qwQksS7AA-JDZf0XMGx$E)Ud|;ubKkj%;tbVqzowmT
z|8uTxqw((h4;NR@a1W38wP`V9CHLx2Pcq)BU1Kn~#&6-jR{FMO?6Xy2U&|p&tu^j)
z$}CiQuKi?P&V<!oLa(j)cPm~}+7h*3(>}qY6Lb!pXkWTzi+|NMUjB*CUdvpamL0#&
z_zuId6|y<^3p_qtuezPNzxsFk(I@r0cK)kb|MlnbQ|p#aob>Rs375Zd<m9{0CWhVk
z#2|1Xs`B%f*v7C!$v;yby^(Y5-#VQoG*My8pB)JcuI}mXEC@d)6PY}lXYw71?<<y<
ztoyv@I+w}+d%d1pk0g5AC}@0hDw;1g>yE@sjg;i>1AWbPnOD3Pi`|rJEH7MEel_pq
zCY$ied)%g3d*?aMTmO{pSiuPe?JmEQYhFCBKFYs({vYW-AN<Y#zp^NHW;odx?JajS
zd;i{PHoI>9{C(X1|Jj3EL+96j`)vPb^NaKUFZ}(}zqtPQa_jYd^XBcXjEOB-5#D@p
zSI1_XK+6gI!M<+3+kBtTTC`$WxB2qjPLF-AHK%S!mgrGFT_<k(^K8q!JH|=xMANkb
z>sls%TfVnw?W&Ce(JI$cAFk$DSZeU;+w8<TRk4oM3tB>itD`gdHYLVd9f-`HxWe<{
ziG<Xx(k*Jc7#5Uxul^kM?#)rh-~DcjV;+3(h-@+Go!lO}ZAtg~Gko$n`5SsA_`X!m
zW-DI(roqBz-?sOU)_rOIxXI~tALFrShNrJ-Rj%{cvVGH{qMN&qS%$+7u(I<I6?=TT
zxcx2fpU|f=Q+aJxRW-LsO0#S{#C5y%G|#nixw85&e@C|LbhSF8v{R?!6Hj;*OB+RS
ztjkW<bt>*YwnzQq%k%7in={@ntZoXvE$eSBZ#(PN?=6MTg)5aNeh*K3<>$(A_Ky0|
zhA&g^*q_pymKxC#X&d1fmEqUBMj)g!ICZn_WP#X;Qw@)qMoB)|x#`%YcauCa-Z5v1
z#C~!T_}=)cgu(ZHFxQ8dlS)^#m2`#QJmwg4^4OD~QkK#TT$NH!lWo<uIWIUh>AKgN
zl_hmC{leLLS+#fUn+^5Zo4cRSzrO#^&nL_4C))m>f9?O%di$z>n_qoh&d~7q^Z7*w
zKibb;^?dE~|25B_#@A&R{Q9xkyx+e1^Nv|{B70|u8SCkuD>6#ovT)Mg#<r8Ivswy*
zyLPSi*~)h{VA0&efsQu=zep`rytz<HX{WzfS9RM3VTRIo0dnUhU!S{c*^svGg6<JN
zW`X@TCF`;#gsoUpCB922K-HcLH2a^qWp?&dt@kCNI~R&4FkVSe>{kihoMq$1c0EQy
zKs;#1)N{u;MfGi72$je5tBN;jeSQ4TS1L1cL*H-n%!dZ`D}zJo%r3Z3d38^>Z2tmX
z4h1c?(<0aJXDoJ|Vy`Rnd0uCg^-{Ob-}!HBEPY?~3$(l&d|t)(_~h)46(YaV48#MP
ze<o{ZM>cisme5(*oRRyZo2lV|&eAPAxG$N_nsxIjcg(j>Hy&^-Ke%6V_2a5JE5c8$
zlbs)bGOP0d-!}HMo~`R=EN$gh-Mc1i*`qUlT-Q%k<p^@EnJKZ?z;DGS-i<jkp2<5t
znknscTGZSrHSp+B9ZQoVp(}Pjy4mD<PVHL%(dmv1cVf6bOYE<{TqxCc>?6aM!^^*2
zH7*KgFuE$_bIeE1vVQZW6;q#QGj0B%f39TCOPlaw1MTySHeK&!uRosmYmUD@TYB%u
z8E^MkNXgV%{rWt=XP)fL^Z!y}>Q<@!|9kei{mmcd_3Qrq+7x`tFOyMftF-W}6eoXf
z{<RkE@w<A~h<2_HyXLLr!+m|$^7R)CT1CDV&u_Wt)>!BkzU1ALpDT_lDQkRN-ee)1
z+FASgSyz*irQzS{af;hhZe}?ugj>i}sP7Vz^SiYE<fKozes5-OT(?Gdj_;M0preO&
z`m9uYV>w%Fa@ewEM_CxQoO=CtOI_&vb9%Z#TRD;rard+S^K_ckdXOpLr47$2rVXpj
zekQoZYFa;FRD2_$7c%$jyC<u~-bF6hp~Zf7Ls<qxAzQL>osVsV^<0LC%iZy>K$|VV
z2c5BNZex|*rhd&!-uD+%=d5f&y(5Bc1zD9{0tX6h->LBjyuaO@w9A~OYSS}|)l)C^
zJUSS=Gp*BUQJUGiy6Tcs-Uah?tF}L9D5yWbB>a!=?GA=n--8nNb=+2Koo%9?FXqf?
zXusrHm-Hho$0*wfJ%NywIghU<%;o6Wp0v#M<eqr{0~1<<0~Mkdzd7XgEunK-#GGd3
z9*d&b_1p5~Jj2UXzK3oroKd!RuS)vkE@`t9TkbI}*u*Q8uk=OGi1W+O9)Z`#8JuQJ
z)exEadc}I#>&f?+Jh;Po&UGD>tbbE`OZIQ-$7lb8_x+sw>J5M4@53wg^5l+9-}G5|
z^(^!3<*j|2ZF)oIX|Fh*x#Yy<CsC?`TFaw-g*Q*#czbGlRlSMr<qe+db0aHxICWVV
zJoe}^bhy`_@D4S-q|tBuR#ug(yDe{yNVBKmguaCvrmou&c_2vX{2`mEs<Zd@tosqE
za*Cngoy?;P4_|8MKc1%<Y9YQ-?tA*2twOD*wk~6>&^4^F?EKJW<es3nAfPLdJwwgg
z>FUnw+t}mHpXV&Pk@BT{p7np-(^m}6UQf*luxblB{&j{@<Nnl62JdUhYbVR*^>|NN
zJI6D`q#=#x;*DKb6&|j-7rvI^+A2Gbn<tMxO;~av|L3}9x5KeVBXzg0VP(s<G!IzD
z@iTeF!LIYWI<=-;^83zped+CMbK5R5oMXQdcK@Nk!PU9`!E@*3YM01wof6wTeNxLq
z>Bwa@uh!mCm0H6VJJUod>x^d%XYJ9?RVNY@1NLn`7$PCR#YcwCAyxX(DpTDU38Nbp
zMh)x28N8-%)D@MA-{_IFQCy5`?TLdYjwl7X@=sG(qtWkM#(y=Ff7+R82ly7=^f<I|
z;iervxnZkBR<5zQyzzha3E|eQcT)D+PKwwP@M_;%h6lP0_m<y1*S-4Je1B{Gb#n8+
zGaR^LVEQ#W<P`JIEf*@+rPXSO9+_EpCAhL~hMQ$WjQfoHqD{f9$)b5qe^fSJc8b*1
znD3BtN^X&q*OrMX7HtYM;udd8{8u|Q+bBxo&mx|7ri9|K5AVba!oS=Xicx*hRqr>^
z&SP=d(+ir>bL3k#UryR|AmM7>gQ*eoVz(8x1)g7S_-+NS+VhsH9uGE~1Qq=(+)x{Q
zE$^V3$+?io=$~F@6TGKxeXGXL@MhK7`F{c<^Y|+>RnEue{j^=RYVusOdx44fK0mr<
zwmMfrXWH5<l~d;HofrQJUHfV;c&qQ;>GFyxzFZ6|V&r+7uWBw1j1uPD_%3C`5=)I)
z<wg;|V{2pjqibbk?yxuPJ$uS`?&ec`H^gef6ZjfR=NzBuC31+bRiyvWOewbt)&=w4
zS-(x@VrJuZxX82pI=5Yf$w9Bhu?owV%{Vf(yomk6i<L{mI~+dqJxF{n=J>kHf~Rpq
zTGS4U%~=<v4;eMDUlTj$<?@TW&a654)#|!&aa#VQvI{#V9Ij6D+rxLVl(mIXVddeK
zR)0UNT)@bv(GuzOkmHQddYx4h<-Ue2wmN@OEv)KWT-*0Fb<0C*@@Lq*E#IG5^pSD;
zarbN99glbz{+v*eUhEZ|A)2lg64M&BCG+3JH=*`BBjq;#b+2K5f4i?hs-a(8%X`l$
zrx{1Z7k!g=2wfb$E^x2QqQVJ%OP6{JWh+j*xUWj=nQ_`n#{9$mF-?bmmN&88Nc}1H
z)9+-d-{QN0ReH9k(poNDVqaarS1A`M%JwXTPt7WGYoD^sY#qOcnP*ZW{#Cn7tX@&z
zetGk3F+CT*54^v2>Yv-bfBv$+d4?9JtEy|JvDmcpuih+oc;4Z!?W$@^E8Ii%W{XK}
zXz*RU=jxlua&JDC6esc6bhZ{GhOYi~6}*o1o;UN37va%+`wo|BYd5S3cAH@s@o&Mi
z*t^@?7jBDvAo9!*)EqjQFqh}8;})}<$DVfYyLYmnZ0d>`JYv@uOgVRDg7));)3Uuw
zW_|s(gK_HVY5K*k=l`v^m9w>v@k)h87~k2&Mn4{wDfBli+NPK1C-#Pa+Si>15jGDQ
zcLdkAHte1xuefT#GGmQ9qBYS5{JQH;-IAPKd1X<FP~+rm&l2`#RWId7icB1v6f)8-
zaBO=fV|tlcYq#g;8{y}_9A3_nbwnd^%GOh6PG;N=^VV<Obnp1y($+miR_FQ4<(|*~
z^35W7hK5Yedq#_nYkZ5gzPsI5XTY{VzQ;4W^vd-=>>_JJt(LpVIjoI+n06<0-t)@3
z8Cq9QRhgtOe=jZ-b2N4GUFFUbeQ_CGjS2-9oHe}GT+&;0*`PzfceUS<_j@K6FW~rd
zM{<U<i|VzxF5!QEh<p-YZ-319CuYk%E3Nij8{fMdOKbFHd=fuYe#(BK&i$aQ?FQ3R
zYvZmxpL}R0n_fkyXh8IH(+G`2Tjp#k3y)uEHt)}|v+H>oa-JNX_~l6KMcwXAWd@~l
zG7oyb+0ZV>o0z{&Vh(4sXkXif6VroRYWC#Lu>V=rcK`AUi5Gik9zA*glHux2s=OZ<
zO1!j%_HHT@ejT-w6?7=X`lPj4=`L+sFD0*i^*#c$_Ue1wG&$bI2Uo9+YriQaE&pA}
zG@IM6+oNKcK@|^UZR^qrGMew7tSfu#Drgk(C-=p3xv0kll~EQB-D%oqucx2ww$jX=
zF1D^nqR#Gam@)f;$93y1r;B_RzaP2e>K4o2CT8y}CD%;Y7gZ1;aLTKV>#%(K^AiV-
zdt_Zw*)(ypr?P8{c)CZZ<97dT9EXj!=QwC+33-Y!&x<W|{?%n8)*5+9**e5o>g?<{
z^J<?O8ZMlAV(N+Wol)|<#~%IeGX7eaFY7SxJKN1`>u$~My#HDK#AlDMU(9zge}AG7
zCA~wPLGI{xw|C-umS6gL<H^)>n-3KyRIckOoU(Dbm#!IOEMLN``XhHw2zsnKzGLM=
zS+{9&A<^-<e`4E>FSi`Cl|LAuctmB*#{JLd{(2Z(S!yC1<d*b0=5p)R8H|-{Vz*}Z
zPGw8VxG%7cJF{2u+ux)euHk=oe!p$PzBGJ>o32yDhDzJ{Wj^udR;C};@0z;u)WaA5
zjy~A1V$uBjZ{FRvv#YGG51tpat7JwR!vy!shZtQWtq=O7Y?vnfh5felAy?07d<@$-
zuD6HV|6Dw&@oXXkbGD*Z`u|5(f37a-`Mb{XkjwJd7SG>&(6~B>PqC9VA<3kH_s`Xj
zCB;!G(v|tHF>7Ddf{#J@E52KP?!Ch|_ON|eG2MSnxAD{!7E|IpL$`@6Z%bv<j$znv
z|7!oLQ|tDhy!+Fr`Dihx?&H+IhZ1g938iywQ2aB=xn3$XbIB<mPv(usg5|#RZ_7V&
z*fw}+z93V|h9|8q59etpztT4O;MDS#qeDAsWq9*ffy|TZmu!E*ez1#?;T!wfjaR;O
zm>%B3aP*4R^%VD6Yd<yg$f`!o{`BOmic9z%j>w=LoPpCNPIZ4de#<*aKZ#H4zAJ-^
zldDK_#*2*s9(HH#FIBz%U>#q7<+NSbHT7MypSO$b7tU0*R9ao8B)wA3cQb=b&il76
z)54av-!==ZTw6Nt`=^+<tM01%&s!3x7ZW>g{Zu{w#nVo{yXdW?e=lv7>+DXSt$X%5
z-8vhx`qnS4`yAihzCT(1=8Gp=*|MnPbDcw$FWh6>62|>XgO5w6oHKRWhLV}f?Hh~B
z<<jqUPBG|avc4Is=*O!V{=!gO_3={1Ddy+qGP|i6RIJyy^K-fO?Z@*kZ{FU&p{!tm
zz2xz^&u9DJUi(+EA@J1$t_3{ooW@HH`)6J{<k{g=@Y1L{xcrs!{=a`scyjo5w<Lyc
zi@A`X`=g_7<4)(Nd(0Hh-Qc;}ur}uIskglQEce;;Z%)d&8+BoG+*Rmq$Ex$D+w>Y)
z4W#z0ikZ7&MUlYf<C%iK+{fcOE6kM(4(z%-z43cMbE(l5{mCwyI8G)M=6Yq$^Uks5
zJgd-h@KRv6x`AA}<x`c-PmllQH8KAG>d)D)r^PR}h2NM`s&piDiD`h#wAbgEs>>5M
z{L$3yjcT<_yY6<bKEkNjU3qWP$@NYJ$JB%TROYlBu3IC!H2mzltqgH|GECMCQHfK(
z>MUe+h})yOh3owm(58Z44}*guGc@!p)4P}d`Dn6n{k%?N^>7Wb0Ol7ZJAPiSTk<t5
z<D7{z+sVIkk4IPToAx|=|HkjHE9+Ji|Mkq*zf@T^<9gKdy>h-cvez0f%~~se{PvfW
z^Tl61^Yg>sKKuF5rTlZv+V4ANKhKWb_w~i&^(%88NB4g};kS|bM5IUfot&lnYoF|L
z@xIg5{KM-=`ju&&$EThVQky>i;`t=DwO7tOi&OjkgF%w-!7t~_Tt!_gwc1vmo1)rp
zs{3ZA^>qKP0E<7t2W{qXHSaxI@A!_B*VWQ8VyDar_15fEMy^becZaX|3%1pF{e9v1
z<8f}U(J}p~?VVi<j1*c=UFxVWI%>dLyLQjTjD2xwU11SZbuG_Gr)%YIh!u#5&{*1j
zb@n&+d4H>>FVKqlk+#N(H_T#PNBQ+PiCou&K1Kc4Zg&wi|Dk4YR6X_Tx#%-%D%OPl
zy_?OtA?&N%=}$LQrakZM(h4{CdpGME&*Ck9A%U}QA1Za;UwYA-b5|6HD$jC`zYBjZ
zY2K{OqG+?G)vb9${JF?o56;8}xr_c;@xV7B*YriH`EBdhM{j-4-xxjXw$D_CPd2O8
zGH+7RXDFD%yFq4umc*Jpl8P)7s-J&xxT?4HK(g+XjBROYQM}?ugNx>A$GV2>xPEQQ
z*2kIFU#GvUI=e3L)w#HZhRJPP3K-A#i=TVC=GSxej00lN-fF+EHDfjSRr0wmlu=^;
zhYguYf!8Mg4n5lX<ml~9)ujUAUpKO^Y1->HKW~wh>|fW!x2aZlSOUt{f2vtK{r$uX
zw?cQ%|MKn9@yI1BF9*e}F#hhDuYaDub2Z0WK7P~Rweq&V{$4oV-NwzZ?b3ZUxl8=>
z<90oEHofCjf190g=eucE6Zjr%-?YA%+qF>ioRHV<0<|+i&g;FE)q-{ZGnh}*n-`L?
zVxrrs0;z@{7rgcfRPOM*IqAG=%!GOlgD{4*5~o}~&R;p_$DLVW%hX(rZ$3}s3EO^z
zX&2Lm*!_-QUVJKHO0cqv?Kk+7^EZ2PDMNv))XGhI4_x0xZeJoKSZ`X&;8pDX#&voi
zL#NiV#^gEQA9si>F4}dn`StaAf1?&me!nH-`Raf+)!LI!4d#5PtJJ$18>JnWw9)&-
zo$ojHJez;Eq~st!i#Fr6RrlOM-K2ZTnGBQF&Yw=)UG()x)GxK-gyWXGA1n#T<FBj{
z*&8C^r6J+HjN{^ugxLLZ$@Np78#c{Lm~+*}ZFl8P&TfUAs2Mjs>i5;%xxc{T`;PsJ
z_iQ@566Z7TxL32g@3Tp~ehk-v8C5fs&M!W+)WG!9o^5;=Lbh`>yUwki;Uv+=rN_NN
ztRcK2eMNj8Yo`fAPhsh^MM~@e7SA{LCUR^(Vs*!SdDi7k&J3?uop<;~N++}F|4N^n
z<Z!}t<BW(YcUO0^GW^q+v1X0>;;-GEM>GB|Dtg?>n%^rKo@b*Q^YqwM>GD%z@8?JG
zsD;Puhz~v1!`O38UvFMKlR=u!{-^t0)>_W!eAoBYBDA`)&f;3KW5^!gD^mZT{*?MC
z+2CxtN9TXd7jagpDo>wR`ls#HvssJ$PS)~ozH)G(RNSuQ#0`3PKl;Q!wK>M*pmqAU
zk;?b<nU;Bzma?lH{-G?jQmk0|Y4ReW)vFx$2QyT@doJHA$*|#=*W!Yx9f4K%G@}_R
zv_F_TupQWw*6sH8%Qo2=#tQtKS)B5>9rJf#o)z43N$JTM$5KYyO0Mswzi&Rj`uX0~
z&9A>cyxRMcNh6j^q1Wigx_HGOIcaAfY%&RIJb6fVUu5drIh#`^wjOj!@Yxlh*v#6X
za;?yoPq=hn0|V&FmZ{At3Y)w$dXit)GuN{^9N#l(<LTQQR)zlM0xj38>Q6m>ZEFc%
zLQUm7t^O^0SdQ#+anzJe-@3JLa~;E_I&sISb4JZaQ)ep8$=t9ko1tgssnf?Jbsu+~
zuKW1m!?~l8aXWj&cjO4NJ&MuaGE1ya>|y)&d6N!4k}_Hv6ngC%BkOXG!(X|ZuTF?i
z>y3)6Q+V;pk5TO2*#yNG|E7F(dB*<KCbws`@nlWuNY-Sw<~#QfZP}Fc#ba;I-gTQE
zrDXM%o_`gc+;{HoPD6Kn_H{+uPIXORdtKer@keQ@&(mX9C+;*lHsj+|!}ZJ4*H3<~
z|62Lpw0CPt(!-A>n=(w5ZomB|VV%wDa<%<gnG7CRqzogU9?QRe?@ZjeUdDo%7RsU3
zORc@H%yRNu?Q<}6lNm$m_pC23pG|!IY1`+MR|73&YImJ%NSWlruy1N@?U$Eo43D&a
zzmlBvdwu-B(=-3F7+hz`sp?<}jjU3T(otO9a(U4&uQbk;-zT|gb#HF*c`2A1<D69a
zxNYOSx5qpmy1iTy%;)y9#Xr8|8k^7ZoI<H}!IQ3h*}-M^`L;ly@1Z-+>8qc)x;9@u
zBFJ>C!28?koF7G(EPtl_-F*DJ{q9@#xBmOp_v++Nwzv2_n}0di4OO2toNG56I?nNY
ziknuiO?S=Tsx{@C=Kl%gJYd`FB(Jmc>8(pOb2oRLZf+^xWvufjVpjCW^p!Dc8eM;W
z@$Tb1;HR$qgkzaQ7;O0ZUh&$@qsd`UU-`XH^j^KBP+#TJqAPnP<}CC)?C`*J*O>&}
zHUCpG-Y$L{CLiA4Fq!Y0P4C0f*L<1#&P~-97gcQVT{H1pV@474<+)1dL<}VM)^RTQ
zT4A;$#Nznt3Wde|Vx|#~V!YT^D{d@0Hr0dc;ras!7q4|R%{IvjY&-h7BxHq6__37M
z;;qs@p3Jm~KYdnXZM<#mzhBks_4fV$^YwdN?X`bvZduNZ%I<kwx#gDSLuR-8=Ji{@
zzdS$h&u;OPKX+Q|&$G8LfByX4p49f|@AeeA=fCT<`+d(pzW)8y*YEdMmHk+?D!$<J
zx*pFfvlMh0bge^9n13!j&A0wc)M-9#^Sapk)4rxm@#E-u*DHTV+MZ+nq4vVcMZ5nL
zo?SQf9nXQOk;)mu8g6PmORm4Y=lbc9E`yD|T<xw(ONqPhVoUt%tbUak>)*R}{A>KT
z^0@tfraxXE_Vv7fU(EJjH|H0!1liYE{px1As`<aNNozv-!k40ptfsQ0%1?4GzxJu(
z+$Pb4bc2$t3852Rp8R2azfddihO%MLT-&2h8H7@gbeeRqoN#}c5|weF=aQWC@yg48
zy3#M4`mp|}@M<shQ)cxmq&6H5N%yLB{$q3gGt+_4Rs8bq3}uU}K2JZn_~eo`!U|JO
zCm+murX?d)&URn*Oi0Fo=$(?6)uMiEDw{gHyYtS^<JlSOwjS~-(Q4-~k;uEr)h+Jr
z)Npb7!@dnp?5|9nrT0wJ(+V$^VRzUak+~u4>%CQ9&Fxb{SO5CHQrfs<^RbOmWh@f&
z?G`dVp2PYnQ*i0esuc&dwzvC~MP7*ASaH7o*2nA2zf%AIEWi59JLY}=r`P9P1ve^M
z=x(|8>ge<Bmohv`eyQ7Aa)z7g#?1R3m%`S&`Rc)MXAA2e2Znz9CE>6vL}X<a*PMBv
zRVm5|5r4}M$koqwJGEn1;)Z}|@yrt{HOZF~RC<;g9M`fwQPBSVs%l*RL*+ZF)zd%i
zH!4ojJjZ$M<{Oz0TE27S_TQ0TIp25HkJRsuV)d8&<ky#9+sFIB%&w-^<W|Og<_)i}
zct6eic0bSd=7soJ_6=nL@%t)&Mcw6z_mlf~Z1L^QAAjAub@FCK?fVa(w}wZz&z<YP
zjiG1tuPvsM4+`Vnm$Rj7G`!TkwD!o(5KV?;+qlh#wbwA-ICFgI;gerpywbk9ypa1q
z^Y*#5(-UPGgw`mYx8833#!M}1?WNcz-J~_0@0Mk~JvF2A-8$CtI7#NaPyaF`6y*PT
z@b=bhm$i4_#oAcj-g$TRwew~7>P-Lr`1-87|I+=6TDF{jIk&2eZk_1f-pu*wk(O+p
z+|<K0?;ow*UH*P^=JVY*_wLvD5ZhySx90cNhYOi}HB9^%Pq;i>SYD8k7i)fa1AF$`
z`3s)pN`y7LnN4@uaO~L4J0<D!zV{SX#$7npbEDgtb8^y!Q&pWt*IrpTFY<Z%Dd2Y8
zwdaM?mS22#V2<IBO+}S+Bu&oeo}DJzd9~9tz^Wu`hFF8sTG^%Vj-^=lf3&(Eef!q_
zt?IASrfBerWQH(i-k8|4IV0*y>f1{zG_u2IS=I+CF#i$tTg0Uh{ee4Gc^*&L@5Ng$
z%g5SC<S>10dfp@+s9UT2<=(Y_TkP^4%(U@dH&JNrvdWLQlrFy1EpwLs<tJ&{GfQp4
zBX@(d`=4Ha_C{@9!qS6LSAT<-cU9}u&Yiopd(Zv<JPymhf2jD7l@zP3trqtCNBUB2
zbJfyWD-#!1pW@SsUgnYi^<8i6H`d^m^|LJwS-oaVd;K`~bNbVxdzH@Zy6m(#aD&xd
z<~<iPQvdIN^EYzMlZ;ua9dqZK7*9%b`?P%K-J|bK_s{-a%KtsW$jC_PdemmE-J7y6
zbk3CX_22%?^Mbc`|E0|hbrTdE9GurzNtv>2S$6TR25-?S)4Ox;hDK)V2~M0TI8$(*
z+`c<U-@Z#bHh+ok63LVC{PHJHSFd*u4RhgQV)^}G!RJdED!s2C{ybN1_x~1C=<J^>
zvnxMo#P66t>wu1Pcplg6%1fJCO>|2i%ww{czkK^7-mO#b76d!;tL&a4FVgtr*;C$|
zrjksY_In#%Gx&Axcy-fnJIiJxXTMoSEctgY)KAIg)a7?*<C%U#$a9TMyR7QUJx|Y?
zCEU5VTsltu<&Ue;+OPR9-So3!Za8M;^grwlTZ8bgU(fP4{yJX#h)Klcn&RB{mz$~?
z8oboQGq<hZ^y~QNk7j4Xp7Vd?<Kv6DdM9Svt%>Cqg0?CaJd-Vt+1;k7!;tagu*0Jp
zCH`T{F3Tq7X1=^)Z*^&T^}kQX5naUwGWP<rf_LP;@jLGSz=Czlt|uESKHqw~l4pO1
zWq0N0HD2}{8y$LQtc|*Fb3h>A?t|pR8)xi(6|T0mp=|BzbIEL<6Qr26pP$UxX<vPD
z{X*vn*`H7R=-l`xOuBm0NyA%{&x##+Qk(z#UP#ABvvz@G_T&$3kLSL9+Of(Z{6wyO
z=42s%Hcbu79Y?0=?aB5%+xocR`c~QF%s(Cm<ZY4g>Z(xOk;Q!=F^ALWaOkv6F3#8I
z{x0)*q4jY`R&bjXU%I8QE00*`9o4yu)SMHPKQa^?7hE@cp|-0*_3h1b-L6K>u0Gx?
z&YspZ=i?~{{^wl+nOZ73+yc`My%c(}ZL$0LO;c99Key)FQ~fW&nNu&RJdt_5yvZ<6
z^ZM-6Nt+LsnmoJ0AoKPs<7xA=>%SdNUa(wi+FG%Ku4jAN&TC}tSn>4Fw7XGZxoRyY
zkrxGRxUbr3-6?$Dw8za-Y-7iHaqfn#9r5We;!LOJ?T)pzJd*OGN$r?Z6qoXQ1_rfY
zR+Z#C((`VgJ8?dLqG^oF{B;Ml+8^;Uipgn2%wC~B^Tq5P8y(W>cv~m`7M(pO=Hct-
zJYm~qzp~w|-G969k4d(>#imVHjy-u%ke!`v^n&~1_NX&yIVlTg>R(Fh32Iq(DDYAO
z>m}9%P46S-0;?Ghl<hj<xpMZDtG*?neTy`?Z1wd&xoOmC{nmE-Ywlca8^Uuo>i^pX
z$5-hv2IT(XJp1|I{t$*E{`$ts?&WWL^~<N11utMPx&MFq@dMwtCQj=MJQMzWe@*!A
z?Q?m4ZI$IcaOvH{qmy|byu7)8zE1emr?=K}e@pQ+a>|Hm<rg?TEo$p^hL{K6cI=dr
zw*6XmgXPLA^_OqrN|}?F&0qTb;Ju$|tFO+xUlnB8_B}=6m(tw3fmtnc7-p=oVr7u%
z6Fy<d>-o^G^lp+;ZdUCz#t(aB-|}qn>|J%hVB@31CGS%*nHtnT&bqVW_Z*cNlO-0b
z=gnBerLgX+EdR!a^UL%5&b7{1wX&SU;?xZp$%A<gCZb0o^$p{*mpk>uR;PYG@qM;L
zleI;*qPJ$C-E==W`?E{JJJs9F^tr#ql)tdhwV!Gx+|axxSaG?(cA56KHr@1_{&R0g
z|0?4y_&bFsCs}HhnR1%;M~#nfx6Mhl-@D+K<DtaX6S57De{A$%?z|%&{$xFuki#j~
z4S$m&npv}w9r9h&l520Ql4TH3Vl~)Q<Gu0jna2(19E49WgoUq6^I{P4)0Am=dZq8%
zFSYG8vk&TSiJ307=Hdb72ix6uOjLE5_V@keL-u~cd?k~&EZ_2Xj=FM2=)Oy}<==ul
z!}`{732Fc4WZmU-EcWB<*&91t|HLLUZV>yJ&Exo=m#vuL>$TiN5u)cp*mx9|ZkW<@
za8gi8q!Z7{j45k2oN5U+tkZR^$e1<tphvy2D}SwI>5Q;#nF=LGCD_lM_IszcovD-e
z=)z07T=u5B-!3p`VA$bnKlk9uNG_e$of8)St$jSzJj3s*-`^>!K~`N%e9;Fce^|1G
zRpedDOz~sM{!wd;HU>PBPLiHAKkmj0u?w&APjKGjnH_2)DeSd*O4sY6s)AK!`9H5N
zUi$Z)TUF3R7MUx*{>{mi`M2YH(_e-Ir(R1j99xv3|E9h??!+yt{~Lal$Y0(*tJa*Y
z&g3@#?|PH^$7P?xws0xL+OD=*SN&thrkZ#5uC=SL)wjI-_xr2(+3Rh7cf)U;nmI8|
zjHhwRB<tsy8}{V?tN8Ts*WcX?=Vo4y;tl=Uvi|N_zrM%Ib@zyw+?LIIpKp5S@5b$4
zpZ%QIew(4-!P}WUdB<`X8(ONTbSO;s$lv#ZWlyFRUqjD*p~W-In_qdl_Jn1hi!9~n
z?2_PJTREXhU3>CNzx_Lf=FZDyY|q~C@H}TMYf)XRy0G%$!X{huXTCcuHtF^KRBYHM
zQxZ|Fd($9x`MYqRLNVsm9KRUV!q)6^?>8#E=iys=>fyA|02dd*T~2v{)8?11d%We(
zm8~M|cNXg1o$~D|-})OmC#&?1w)S=TsXRDU>1w5$v~xo0gv}y~o(wOysJX~hhbbf)
zP0VKxw_sqLK4Ib8<wcoKS$6E*K7ZQmW~;tOyEJ8HtdTqs_-JXtf?IdKup~rxv73J_
zuDR^}JeKVxC@C`3EKk$i(_nqX^5;sHmZclGF1x+{e3}23=q{W2KOaok<9e#`(2j#e
zM@+Yf=#(1l+0&J8WqM=n(|eznKQ(er+F+6HG~vqp$$5$s3O<$3{-^MwWZE;QFmM?u
z|H}S%`u{iouixI#@qd4(0k5K<U1iaM=f|5Ly6U(nxZZ6ypIV%&q@2ulqfMyoLu~x|
z?aJHv9WI-S3j1D9OAJ|{-FyG%%gN#_9AR&6b;WYO3eC2<E^S(EX1(w5v~cTx`j`1G
z`z&*tsJGOBN13NP%(BWvR{V6##_Y`oQTbb+MXh+@C(-5LV-g!ZOLN~c*ThSqtM;9p
zmgcp@xc6n-!9xKqo0?87*z_$y?{QVF=j<<)wwGfogP10s<?;7A+))zi-uFSMn#m!#
z*TYX~Wu({9`EJ`TYc^}$-4WN97kyXbQ)WeOQj18#S*=8i@0TMkx=JbXI7_ygE*CqV
zd1uzyw<~tZG-n!F&Z~A$um8Gv^Zz=jKM7{x&mQ*V?%(|CPI%hK?6a4<jqBMGK55Lg
z@jCxcg{RJdyUt*?jThsbpPSPkCoVQkUcL1t^Y(x3Q&)(dcYU;Pwb{ML+xwz#1t!<7
zd(pFWhS}7rxf`vLIRko~>?ZG8n14m&b-{f*``Xfd`iqat6{kL3|L0$~+w+P4F9((y
z7{{FuUn3U%rXkw2xbo9dQ;Bmf^4ZyU6S6GMy}hx{SKxeD-qBeWvHLCRzwX)p(l_sC
z^X80sI+;3i6-A>$lXb3UYk!Vb_X;rGVw`=vqb+&0=*?}5w$Ce_aw^{^+c(<q(W<;v
z%WOZDm#jRU@g{1Im~Zpbt4T9nZI!$fvYv59#HSs9Q+B*Py1sJj@0T@m=kh*(nK370
zS<j_?K1?%Z)NB2%t#jq;7O?z#tG>LsIrHxPp5?yte+QZE*{FSd`MZ#ddk*xaC0blx
z@LD2!M)nN9yB=)}YDueJ^c|C6IQ)UZmdANg;!;7aGTU#<_T=>)jkw4=d1s@_+{x0O
zyaq}$7e;%pykIGIk~1N~D|KC_T5)=T&$Dv^$L9GchvZCiyP&|(qUa-b{=xDjt!2|s
ztqLr+j{E=Wj`g?N_uu~bB+onc#Uaax>#%{zm&m!{hXS%D%3ZneMNl#1?M`!F<+-kj
zi(c*&@}B?uozG=c-|JthqLUl!-p}84l*haG6pziMg?DE3yA|(Ucb@Iy8x<9sBZo8_
zA|+<tnbj(&t1_#PF~VoB@-fG~n-=Np-LNchMyP=Ji>`IwBPBc~<MaIe3SYV{mP%xM
zVa4pUXk||G(uL;&W=fpa-X8Y)Yg?klDmlh*oyX!EJI)Jmvt<kJKlWf{%{$JEnX_x(
zB>fO(*f>G>oL0hDW>pQzzma~vOFrITqrCUpo1HWKc*Gdi_!cZ;HJW%yovnL^wx;XC
zgNJz8xCQd&8NFCqyd!ea1*f$IE;@5OX09-clX>t>El!5zl3&b2i>^bF=Mp%N7y4Wi
zVVaX(wz!p{`;y6-cZYg|FDmbE>r$4A_;13t`o-~}cLyg~GM$)lbJo*?{;~-xZp}G!
z(6>!<f?faLu4^$LkEcf;`~35L-NgU1?W?wb=3hNKJ}UN?RWJ{4V)biz{_h`OS=+FD
zcz52u;`vwoJ-<Km=e;X_e0IIrJ~O`Zs9Ra_m!~&yC#cm`9Xe;8x0>r^DFegd5I)gs
zK@680)py>xZ!Q}$MU0_h$6M36KR5i>nVFaHt7KEz4TIl~t>G3ie`KN#GtTp~e-fNf
zd%W+v4a2>#<U4zqAKa1O5|+p6wYu<i<lT^Gj#_o^&%OC^Px3;LDTDs3)eLq_4LwOV
z$L1E-Z^+r2yx2UrW!bwGUnK&rX#G-uE!z3uTT$G~4Qp=Bo>;!%=Ip2U>)!E4X<uIQ
zcW&0-$61es8CEF%?XCVR{!2Z*O?PwE$w-3_S^5&(uWzr~7W!GcNTT{Hv#K1Q?74YM
zh1PLQQ|sT}-)p@~%(&1X_U7ucXVDuAIxSAhu8p~0rt{QD(Y0x<(w`|RVH&A_=iJ%Q
z{IAFRRQa#X1_kBJFC)B;Zr3lky54cCV1;PX1{sd3=nugQnNDj@<!7B1B*{HJ%VJYk
zl=wP>m90JMt&1)8sXkQv=yPdj@KTSBU3t|%FK%9LE*kjr{iJD3@lLk+uk!m(WQW`f
zY)b!UdXCfl%*H<JT^91QZ+EKdM}NDpJ#UujosiU(X3Z0xTy^4^_WIP)+_1uyUblLI
z4_?c6?e!4UdvskG+)$1_(`wzw-0@Gr;JAO#G8wPPhi6heY<7fk`7;`vbk8x<y8J8T
zeei|wA0@ok+ywr-iEC>KDVuX*>mA3CMOWV3EMiGuJ+%BO%c1w%rvIHHSLD3wr%<7*
zkxs>><=f<TPES{;xhbc5O1kucQ{w4{$+o3kVmm!rd46;h&zqWIDCcoxo{D47Qk}}Y
z^Fh;Q+&$}fb!moew0H7$>zmp!dfeqp7cqK&Q(rBzcc!DJdONd$=sTXNPp6d{a(`eF
zI~4t9-tF~&o8P<GA5ebWH}7R~$<pHA+qc#qI#xaT!HUn(`+x7BvHacjy5HYdbrq{@
zoacPE^3|)kQ9Eo6uBXllSrxkUb3$lH+TP{ay63k1ShL^WWak^+bGpmfHq86<rY}}F
zZ}M-ConPAYLhsM!+p(NUH#Vt+kzv{;hR@Ufyh;2Sw#iJwp0Ua>|Cjs;hor3h<JI#f
z%-guK*_8QdU~=v5TQ`^kOhtpIdR%>CBB?g<_^fT^4>m5-b(hO#GWgTHa%<%6_v~w#
z86MwaS>gVvGJCC&P2c0coxR__D8wXMOlB}Rnk*&v^gwdv?&6Kt*{wpK-aqTQY0GS1
zdHJ_z&);KmSj`+@e0%lk(>DyR{Ic8d>Wx#)rnm2W%JS|nTV}@0x&B+y&)2=HR;qly
zJT>nr597CQmS)2I3^QjONC{uV)xZ-wl_kP(<r<x(kCyud+Gl8b9|*dr)jp+iON$fJ
z`{jJ1p$vT|W43RZcBJ`a(x>9`?t1|}ucc>KKR)O6b-nwJI|h>t6@#vZ>9}&Ls5<O?
zy{qr>4hGX|Hiqqz9gGcM-}-%fy~8zpPL{EnMdyl7y=PK>)JiIr7j0)d%MgC4<|bPK
zcf;K%wfCZhcF#`pzFT=O^h23%+tz(|cdAWef53a*GJfIN#{$zH1x)`Eabkw0b9?^k
z%UA4XbkEfi=kD3sab6yrYSXVgeCNN!vQEgPtofn8&oL&?)(dA!LK!(1ywlz%QhWWK
z#k@1CIT&Ziw4SgPI&V_<TzdWbi6;-4&M3{8Rg>1o&lWK8gy}4gS6ou+thFB>{SM*{
zJD5<qcUAn4?@NUGk~g2>xbtM%)N5_apKlG{XK_$*)AaXe4xX6I)S&*BH!Y1POipz6
zRPKEq4X*OBkJsgd9sJ?Beq(mu{5>M!qK}UxFg8dVTezq{;kz4sPQW`zWEub0Z@XS5
zZ?ToKdmBAljbZDqs&9WL->a_rbH)1qyZvAGeV+RH{lAWFiI1c2|NQ*Ew*04c{Ke(k
z3k$DF<XW4Sp4+nYF7K>`8_V7w-2UZfrjaDC)t@_etM6^9h@bc3{h91lf0<Uus#QEO
zI_ax0)tTX*0K0nVycmXnJ7Nu23bogmI@oVr`e<Y0i|v){4z4{>4%`j@yVGC3**^8w
zh577eeupL-R-Y-)KKaLlclz<=^RLW1ua-Xd?!*OVCfW`<cXszVpZ`5=yZC1{w-p@|
z_Y2Ca&*#~`w~{|v^ZI!|-~3q>Nu9b|pB@#8UbHp-MECtPZw9?5M=xEx)Vk+|Zx~zu
z|4kofAG^aZD*M+~*D_+A?SBJXQ^qSIpKW+I?5g?qdCPD1w8vplrlJhR(^eF%PA}8{
z_G|6+T@2IYBMz)Spq{mL_5)iD>Hg>b@_N1-SmqR8N@f+xs<WE9SRg@gomR|+39Hw5
z^tz`DiuwmAvfkM-<KU+i^9nsrbvsyz-d5Gt@i=JNvVy^*Ri!pm!SSu*cIm&1ciir(
z=Q)sK*1+{#!1+?_%pT4YlU_ISR!lqeH{^#|^mc3a>c5M2s=F#ARt0$l=EVdXL};Hn
z5WT(Q;HTTl+&67XqBV9#m%doLa{9Ubt5(R*??3xiamC&AxYCF?J4uemsfsN+FHKz)
zDs^IxKNNLWiTM5}uJ+gWpqEV6`hR~ZiG3{ZVm%ObWWBLXW6C9dsok-quNXzoJ+d`A
zcuP!4c&VQbpQ2UY^WDa6-tud1KIlsJRhj2H|I+VDWsbTq=~DCeyY~jqO1gK#G|I2H
z<lRw`pq;-~uD*4qzOM5B*<Xt*FKrK;A`s>E%=!IpW7lJw69VGzH`GkCnOd-^@&V(9
z++$3ZkLECIq(qd)`>hc(y~7$%C}ArYf2l9v7{ji~wo?kOdbh24baIA=j`zwRWisho
z6Q<ZKayrUos<v_NF@YO7SDOyb&iH@EJ|I1lMMh?Cy_9{e$BtDF;S7BHzu)~dr8j@=
zmiMtd->3XM)%~zj?0n4k-~0C$FArO}U(V#V<L2I8cDd}Z{J%$DcrE_r-11VAsWIiZ
ziN_2k^I8*`2@&($eym-1C~Dr63Cs)y7sGCCl$p2FnL)JdrOL@hMutYU0`C_6N1N+x
z%HCxhe!SwNiR9ks-HX{upZ&bFZ}Aldhm{AVw@;DUW7zO6qfF|2n6|0Rp$})Crl%zs
zb3}e$`o)Gdr@@Ywk;kRJQ=ivG-r?iLw(@hc{Qm3TxyhDfxhmf^@X~rsU)Iv&`==hr
zW&Y4Vwfgeo`u+F+F|XcQR$cMu(Tl(GXXo2oS+31+uI^`E#vEDOZTr96U(0CAkoEQL
zt8JI=g|sqtPJX3+@+JG<wcTRLcY^e^**PBtSUbH+oteBQ$D=G`+vNpSK~uQA86)Jn
z`a5<gv7HW@rl9|7-xtQDh*s^6gFzPB5^FwB-6~tn>u`LlI>Q0xmx9Nb9u%-viYg`q
zT6Sl#ce2}yGU&8^WeZ%G@_yT1ZCySF2DR-cE*vbfD)~5PX|9a#y0gyr!dX*U4@~x*
z-OKdQK4YuBpYXmr58_U5PC6}^`uw}mv0a(p16<;g-Z0GJn!~Iqb3M~iDdSDnWIM0B
zD?r0%!rT*oE7ip+hRp3fT4E%*E$!Ys`&sj3w5(=#XbMQNaX;B%nVhV<_V+~|tK_N`
znJO8)E%~0oM^nVyw99yR9-4dX{2$3)28D^g7iOL_eEp^*#9XQ`Wp3!92`{Z?`@Ott
z!s4{6*DW!q<Bo5h?p5iocP4*7Bo)ofzRkhpfBAs+v_pSQuFTL=mSD4<?-Q=qvOM!b
zYEJH2lPfoZdrBM*Jz3Sxp}4@kA#p=iNMH#o$H6bpc!axW>hLT{HZq?u`B~0M|Hm3O
z>MN@k7Vt~$zE#7a;N->i-KczNc9_V|n-`+GmEzOAmS}be^=cOT27eLYV02IBTGZ0P
z8~$E`gSTbfy^XK5a~8CG-je3)|8KFq9k<@U<A=@j%srp~y98?S-iZ}@bTq@jZeH*5
zPU}LJdfT<RQNEvdOs?AO@X^LHc$HXv!J2EyuCc);E7xzCbZ)wHt*=&gVeUzjDGs8m
ziy}C~qrYwz-`e%$^O1FPuN+#n?p4+5G!Db3R@{tBW@zbbew_c|r%gz*rH`R3_sI}#
z-{%jF{mZvLdwA*9mP5^w-81A|FBceHJGRh#ZrkKX_m8S9SXdNz@Z?dQOH(&5e0M#v
zg!91mRh=GF+2*^?y1$*xxY}xBaBUhx23P(f--2a7+8dKJzlp!uvM6)yu1&EQzf^oV
z6Bl;mhUujVJNN&H4E2nA^OgPi^S}Gw&HwxT^2f)@?|(iHkK3CYK97Huyn0f6-}j(1
zfA^Wo9eJC#Ja7BfZ<7;dEU8)YUHajjh<TyUrLGqlmrJ(pS{VN3M39etuj#T3pTuoq
zt;KVtZ~6SXd?HB8XT5^gjJS_?HlE{U2(;Kfx6QtHQGwja#=MAIK9_eY?)7-CTVIu>
z>Am=tN-x7QXRg@xo=cJ<4(2@?JgG^aD_)-7c=S?dkvAvP_X4Y_ZGxFT{?7V5sjpgn
zu7}P($sv`TC?VA*=;`a0sKV%wd}e-}+1jRcmzJg%xhDVOsWOpeKh&S%eA)DDz{}{e
zyWa1vKUSEl@Mv$ELd~RO%olpyOk~d=;%I0KS-*X0PwmXQi?i9jcEyA$WlZGZp1XTl
zcKGHN(bFqMd6+t%GG_d`6ZFNSEpUmkZ@J7f->oJr3yP1+oe#cT7PYqQXaXqz*wyPu
zta1IbtLeP!wN#}iryu`qvJzs537_R78dS85Yo_MX87JgEr}k<c-EIHBA!7CHFH<e#
z%imsmSX*6VP;j-fePZz>_78{tt$K5%!NygB!#+bdUt?Z=ESJ-+w=V?L7`!<drd6xo
z+A_hp_*&?y#-PAu>Ic*s9Mr_L7(~vU;NSG3j46fZ+L1-iEL+yCsQY)QY}b|Uoo5ne
zK2~SodKvmNeczN*Obq+om#}h{Cg<NM(yQGzW9?Q2qvwUIcRugmX14F;nYtHs=V#ch
z*L{3Y{C<KH-+u+E|Lvcb>)UosthiQw|DDB)ke@&M&!mc3_Z^<Q`|i)28(h}!CO%xY
z`RPrCKa(G<kXjnCb3b#*!aJ+2`LE7>{<-Q>(AJNW=bl~sys<^=n~h0ynN?LaOHWBc
z%TD8nAlrR!KPaTG+o1dGptDZ9=(SIG9u)B#q}lKOFfVFVuKGNdMJ$3-yv4nbCNm^i
zKABfFAx2}3>!NCgCq-q^6}1+IOU^~=7&Co+$<xT4_$$fN&zwQQW>$Gh(D$j0PaSqO
z_F6};pOfQv*z0Q4Tj%gAuU1XCUGeduMXiCCA4A3`zmC2C<j*#qmTOx7zb~}@`Ofv{
z%r{nj`1kUnzuS+KEB?EG{`&uKrmtn@x_j%cg<D%Me;0hlEWo-gi+$Q6hM27rD<1Hr
z7c{&n(Xv?8^kd6HMSsKlZ}v>teVxhSArI4aL&-m1l!SLa=LxHOE<Qy*WLn<JwT27c
ztDUkjU*~PZId3V4=<iL2d{0D+x^`%<u3cHMea@_>p7y_|unV<n#{P`1)8b}ZZKU9Q
z@r_-9?<+s6^3y9n9%lTRAlU5sS;TBPYjf#l&vT+rck6iQuvs(|$gn%)OI(Sbv-)~<
zv2K#!d`mHgY{CEWoo=h^R1&#nraQ1~`n2mv3fJAor+Dle4smf?svNp1qIh%1=cjI5
z9s91=ezxOdd(v#Wag&+(Q=vn%WjTG>Ufl@x(e#@C=Za3WAG1O0YsS4LewmXRTQ-HR
zH9ow5MkJSa>z-#pzD(1(%3magZRCudxY~N=&fN#5>Lgw?uweGDtkK%V<);~*p(%RO
zGy3wP<(i2)=cKDQ?L4;YfIeeRwD0-)q(!=OqvQ8{_K~adpHOs-=~(@Sn(cdzG396e
z>05jtCGOLd`x}LqZ+%>Fv@Gq3k$uUv`1Xpzz{5iG-n^B({rvL+rUMf5+=K7#Tj8*_
zXZGqJk6tCDUrj!na*KJwhQG(oaTjNI7<~1A{!pi&@7>ntZ>MeFvvqOhJBx2^ho)|n
zVAJzqZ{)tLI3s(tnUMb~Gv*zSH!Mo*KKaO@FG!J*!#ePeSZVT`ZTu@g24-!KTcUfw
zRCdSVoQ)!y;?LugK38WIt1=19&y|0#I`#SNu5GKzvrISt)e!mhtui?`I=6-?!ERrA
z+5GrDfBohEF1r5fTc3s8QZM!c?^gfa_^snjwDj7yxBiMh_AO0KpW*lS^1`|==Z!a1
zd^z8I?*GQ5+{}|Q6YLtQY>dKJ?br}Hd-BR%`l|Y0UG*l_vTXKfa1G{4|F5kc#CYh`
zC%fg^do_X!Z;Ge#hCP~iV(#i~tG#Y#N+l;$2_#*1xaT6XBZqfxRnWtL+-Vg;XTDut
z*nHY#*V8M1LZ)pl+FRn+(0O6%(gKU7J#L$C&XG#4aeXN&wPv2tmI<+6r<~R2*~n0_
z>Eu){)d?><Zl74iQO$hziPXalMiaEm+&*2)sNer%h0ZmW1Wi8$#tz<NLU-z_^tAg5
zW^CSYDC)@besFtN{n;a-tuG$c7}X>`YrEL8k!j|PZBgGvv=k-<#Rz+fyjZm%O+mhq
zDMxne>(|Yl7V^<$-RJYBO0L<po+XS)L+$gQKfGHf6i&(9&>8w;;iB7p56r4x)+lqd
zTvogIPBo`Ugy%x&s(m6ow&rE`Ij1zP2nj0)oO!#{;Plgm=bpD2gWtXhI(7Dr@n$xs
z@Q^>QX1q_`ADD(#>l^O+Ss@u7tE4M=TSr^wbL?^rCFLte^Y4HBb8p*KnbUC>x6GJk
zaO4Efw0^UndVFk|R!4FfPQC2@U-0h>|FrkhZmzj@Q$p8>on0dQ?Y6gPnvUg(Zn&B@
zK|-S2ZFSxD^@}&#R?Lgq9JsrX|G=fpY2Ry%ta`uiIUp%5!SEqVKdk;T!@HGNC9cj(
ze)MGW+l$AVUw+ZMmS9r#t?<;G?pdauKN=#dKeWC)FnQXx&bq6!^A-d}?ko*sf1q(&
zrQg$L<*h}J-g+HByX*H(JsDNgvxiMX<JZ5FU%B{ro_XrDL!$SRrtw&=&vP&-zx1JE
zS7`71tZTbMUjKHO9^J6!TK0E_Gfj>A%P+~xTKlm6{vOyh_4n4q_dAchlmD~(zjVU`
z1_r6MZ$B3BH`v8?@~$tszV3##_Wp|@uYaGE>FRr19=5n%;@f|X$J>A9_53d>c`d!H
zT;ytEw*Di(<l8-}7sDGAE}48>@;HQhjnA(=PK8saT099fd2*7kWaZh^#_fN;97_2R
z9AgyGAfO?!W?6sw*;6MXm+gD@tb+4Eqqc7AN;A)iIp>Zhp3;fgmaF#fWROI{WVY2c
zy3<Po?|$2)FX%C4cCg~=pf7xvxAmRYa*|TyIwd{tw%^)>MKdH;-Ja`wdCR%u*?$dg
zEHk<{J!5TNl<|!mmo*|Q-?vVfV|=|;;8Nj&TOF}4Ep#T7$vVWf?dfo>`4_>k#`RCt
zqEnGi`0jST;cxP*maJe=n84cHx@-M`V@JF!gIgzDmsnHef9=q*xlzJmt9KkqxazlS
zb}F0p+TUIbESJ<6T-h!>Wo6<}k&!>~dRrCq*&>M;nOSM(^D{KpC<rlpxm1}o``0no
zH8w`OjV5K4?7sAW-}Q|-J6zYbsK>2Ivp1=p!CI*<e1LV@3gw5sbE_piW<FNea9r)(
z+&1UF7WX|yk(rJ&R-c$@aQ^+;oAG~Jzn{MTCS2}#^E#`ys>doS?96;m%m11^`)zZx
zo9n)g)Lm7-=T1E)@Z;L6_w#nXpLF<lXW6fYyJ{s*pS;c36gyFZfg|T^<L<wUQ-4bL
z?ug6PIeT0@gTWzuZdv2rvYPV$hy1^5-LZ@|visScnf#pf{$0jhpFi$BW;N$Uuc3ep
zkKwxm$G*)fDik|!yERXdrEPKM>dLxJMx5n7#ard)gf)koybWA^%ka=<v4*VjOuKyz
z%x!sRXK{Sjygg;|(x_thneUby%QCfldu`jKh-J5KHrHiKHf*`h;NrRDX1wI@X05tO
zr({J98@koKL_5^&^XKTT?YSGV<LvWCCIA0E_FcTw`t@>C;cx%G%P~hkzyCKdPwty?
zOz+nJvTg3M`(|lbG59!bxA<;XV|eS&$7MAyRP7_KKi>THV6I8Rw4RkKD$Ngc3-{lY
zYJ7S6(#4=XCBIBFrkbt~nP#+}J<Ivrj)K!XVm6x7r%ivBR$agE|19Z=tM@(Luw)7w
z&ypotP7JHgPCCRT$6&)0ol@>J-FPWS^}C)|Wtug!XTCHwE-#sJZNFC0{!iU0lSOrJ
zHtv$YGIbTdOnHg0Tte#VyzIS>A!ov-?Mh#*_w$7CF|7$nj~eGlF>$iTaW7kIcJxXf
z<I&j<mBc>IOjaz4KeAr^S<<N)7wxNq)@Sds_<y8!mFtXy-p-6K99F#0`M6}A_7y&z
zU8fQz9-4d1b8?*A#UFE3O1Ra_#e5H&?|9|5&gy;7EH&xy8rumEY&e|Hme=~u&u+`A
zyfi&?W5lMSZM`S_Qy)rixiaI&WiE!*af~GjpMoQ2Pn^8_ng4NyCx@4OTefVy`qX9V
zH^jRP?c`Ov?60>eJaX(lRI+rTq6)9r&NS_+Y_?{bt;&`g=K6`ud)-!g_VC^d+r%f8
zF_@?v2=Q51S#tK15~G88R^+*k&^;bgp3ga4zU7w8KK<4Di@x(e-&>fxWzG7Y%Uzb-
z%8FW6&3aEyDb&0;c7KLny!5vj<>evvpVRYmY7ZMMkdSFel<11{=LueGy6e@I>7fj<
zrCRR4gI_GpWIga#^mx#t!lV0kpLz9g%XEopLe^2f5>>KOJD02e7599uRCZ<6SD$6a
zi|W2b-&$rf&usNuePh<#&8bq87Ye>#7n=9}$E{x(-tt%8|KfLgyLG)uSg`Bls_41b
z^W5L`E}ms<@Gk0me5FbFtktK~e@!d>Kh<{e)57dsJF64d&D<Zletnr}Zfw5wuUFS2
zp6}<1zWPi%e)h6GuUnqy-0X8M`{=3t_r;m&rc=f1|8Gp3e>`c``?>34Gq#81-&~=7
z?<L2(oD*wT7OpN@z3=?$-^ccTdH90eK4xRK=-$$rx-h#Y>HEL>zW>>{@k(d)d%ar2
zR;eB57d_@bH1++;kf@KpnytONf=cHsjZD@q;b{v?$o84<c{jvhGQ)(5g^729O5App
zKGAq}D>QAhSBr_0m-5s8uFqX=K^K#EJ@xnZ6`N!z&ET<6I;Nw5t+h&UmstMVD=|yg
z+5f(5r6tYVF8%A{zm@a74kz#)XAJ1Id70^WcKI)UgTFOKYtJejHaKL!^2VcW;<>KV
zl`8{Z`ld|_?mpyVF17Eq{#EHz$6Zga=(L_a&>pp7f0Knrdb7r=^II~%342V=ljhWX
zn(39C{q>A&<~t3U^KbR$bKhO<%jhCBYav?_Q^S-utxNVs#EPiRY`Pa9dHMAm*NJ&X
zCq$i-J4(}xG<FF}OF!<a^-gEIu6;~bCrNO=KB#fKf1bzT0u@6kImU{@x-d@tO9CNI
z94|V}WwSF)lah6F&qXL~4l_G?{qHxA>DHbvm~PytaZqhJ%HgZd*1d|O*>l6Uqqjp`
zBN`XJ_L5NIG1Y8MTftCMer!(Pvhy#`aXVbzy0q%BflROK^TRQxR$brxd*8ZiKV#g2
zl6|)F_HHx~`DgNSTYKE{hn|V~2Cl}UTW{VsUn=0cFw6PC+(QOR3l@j`4Y<fV_t(DA
zoL5H$&uyCYK45m{#4C#m=1GW~#&{-9dCl(M<7~XmcK`9~U%Xy1D8#fH9h>F*;^C<;
zTnu-kE}L>6)ZC(dm0|CZfZ6@~&1+3=Rpg&zKUj6*aEh4z?6xu+8NW3LU%Z`Iv-_Ei
z$D&i)D$Om#zQ6aeag~q!+PCO61J7ZD>sAa`q7I*rl07v2YUJ9v=hRIXM;Whr+?A_Y
zxc2I+uZyErhpx}wc(3E@u}O3IMQw%Znm;dHb~kQY<p!y~toZUe;RCBbM%4x}UI<*f
zZOyiAyFT8?Uvhyd^XOUwn~Tz?7k`vrz188W1cT-CzHimDw@#gw$&fLV?eguTJM6EQ
z-Tv76*=O^D5PnAa4}zyZ{+xXM#@k;h7vf%>%J`6$yZ@p251UI`d4JZ}TUA`Vv*O(S
z?yxhr{5I@frh4@P;{roFL3Pn)UCmtE*sPGVsX0C_FMMzM&)@SR<FRr5H}=!z^LyJF
z{w4g(oBi^=&5d<(_WtRUkIT=K{rgqprR3Mo5f^7>=70K?d#?A}+O2vz#|=!DW&2E@
z7`##Wa7N|j%+u@rO1&=zJQqyJFh0rq_i{kUHItLCgqG-)hwc*d+0Kw*tUEzUU|FEq
zREaePEDO3{mR!3#ZFM=Dd;6O9saDasrAoeq{^ylDpIUhz4!GEB^2P7&6lvoPYc}|~
zEzHsu4rEqRxYV*|#<|^ZJMM+fxY)NWdDj|~TDexvFFTjUOp^=}UAWZcl$({*S;jML
zf#$gv!&UOnw#;aq`1A^QLu%3LwQ6BYT%70Yn>(%acE27`VmRmGSH6%H6E>`H-g77O
z=CllxbuWt#8<fobshGvVdw2$i<m)w3FQ;A3Klgm&)=e5JF-O*`PT5~O%~j`7c(+sA
z)usa0og&ZbUN2oDebjWJ^TGoTD_bT6wLE#n_h3aVd#Iq+)!XIPJK4Nf|85kxlp=RB
zv$ouACqsTkoZ3mIhnLetatkJi@+%4KTgtL-_J!r1nzfguw-m&z-=Nk|IpN&#`&UF9
zZU$^*xpBqsNSe+B(V|02C8E_r1v;snEMKQ>GTHc>A!oVZ?md4FcYnE`qRLv3y~|Pf
zu;bbj{PQH*H7xrM#q9lKwVJ`9AZi|m!jsdVKOS{I)<13EuR}W*XO<sY`MLY>@5A4k
z3v#x;+d6Gyj)wKrXI@Jr!qU@s?D^XE^37@MQ#{@bJKl3&_kDBb+@95bA2lzAE?10a
zyO25i>C3%GZfu*i>Mq-bx!d;#{r>gt&Aa=r|1I9yy{-1gv>CVj=317j&)vnaN`GE$
zp+s$A!MCWRx$^t}pL)}L?DPM`P17}g{j~9T?#T7P{Eb}0?!_<Sf|l2t)Rl|;ZM@15
zP%XvkKL1R!$;|oZnnQM5#MRr~J@RvP`s3~MWA1cHP0iT7+Hd>XwDNmpQm=O>rvJTr
zJD+E}{JZ>;=WjXI-||x~yvE2fi8<!!orgN#5{xEtIHoJB2K_($^YZ@q@Ghsuw)3;n
zOTX#FR)_!o^x$gyyHnE5bK7s+%zB^q{>_^=cM9i(9nE9ptJGOp88~mLg>yEarVnf9
z@jx#=rSBfqFK<qg{j%Tin8MY`(Jz>arw9IuSz&hB+>Bj_LE_i6LtR@IFRgG|a7cDC
z<AxAxhRuux>wc_knXMMr_$TLn+O**2+bj;(=RUo%%S*IjX+cV+<L(;<-ibYh2g=T`
z`c)<BP!lQ5wMoKs=hKsE4<n^G{<~gcI51uMQOzCSpb3dDLwT<)H+_Ea>NM-t6LV|c
z%e24qJ0v@8|GEFm=I-ur{lm49A)@=oSC^^AS(BI_9NqA&AjG1(m8GK9STW^pxY5!n
zE+>5?Cv9J+yQ-omP~=fL*S<S1mKJ~4Q4R?yv)SmQ>8g<Xr#G>vJ8#XC#kFr27pa>T
zIjZ<ZY?m#a5%xRr@0;~aA2V(2ZmqVx8C5MdTU>whq+Lx?v9E;L5BNX6@pM{HW#`2V
zzS()c<{W{7bC?87%D#T<u{(6N{q@rwJF9q$CI)$lKmT>AU;zh1)?2w5OC_fB_s=k&
z>lQ4NvcYrhI;~gdw)|~P`y`gW_Mt$tiK*o0@AWoveqE`HS{AM9Rh4S1i4?!S+O*qJ
zdhIS3>x?&H2dzKm?b;ub9Q};PZ|>O`b^*V+H+?Kh!yfYVeD-Q*xR8B1{;u!#t&e$K
z?;5uK$`3JT+%tC$^M;uo%h#Q4NwP2#W;i4H&ZYN$)VAyMb7TG}bj>^0vv8kQbp4Kx
z3o5TE{5gKrC*P|+PeXTdu654l+f}`9-bV|BXj*QIx>INs-S^m{aSC6$W7k80?TfDR
z$M^h8_^Ic(ddp7cguS~9cOPO{vU>adxL==|t>4c$v(q-JXwJ2B=cbo!yFGjDthXjQ
zOM|pNDhFK-%x&Cb>o>t^YwKGRu8hzt<}sF!ELUt=t9RxqPuzZ;CE^#~Gfhzb=dB#W
zz_5Q$ir@L2K@0iVFLnP3`L*(_Ws?7-j#EiDvpWS=E;Y!yx$;)z%q2?=?$5pGa(`~=
zRIR^}+e*%_*JSt*thqEGciSSKLyLD#4$)@*G9^*ck8#7z6O(-p`KWE5A#rQ1*w#}o
zS5Eg|=l@wmcm3U>EVb<~_O6~Wvp8%|cA)2}s{-@4<%@6Zh&O~Z4Q6n2et5L;R_@G=
z6BxAST&-<*vaynL;nELH2N!w0k(_qQVU@u_rC+NQHcl{(ICpB##+zl6UPazAySby_
z0Mnk;+rMfj8eQ6wc7Ez2=Ujz9dDc!Vh5P1v*_$#R$W@znDe=~=dr^PpFTQnoMNs|U
zId?VV3PcaOi8&mN{K>o~Y4Xv&=(1M5tT)`O60Og_HyzV!3a;u9n!}XPH@~G>%6E@n
z65|f73GQ)|J#vH{l~&i4mn>8L9K3Z`*UxZ9mBqh%!(z8eZn&9r_vgeugVOmjEVkdC
zP2YF7{xP?*b@e%(_8k>^>)tvrU0`_jm|H6(r`ohh$Gmx&dDPe4r~Mw?Y&<sCs=HMs
zZnN9oH(oZ5t=6LFzI=<iaAWO!&G`8L*Ye%s|Mk9lW}O{hJmvPwe?lMQ7|uvCnAl7|
zrakAQ*Yj&7$*u>(lmBiNkN<b}#p(FjHM}>XUah&iE@V;|SHo)2TPn9-{kpWWK=0x*
z``p#iCS}{#J`<2}ING+-g2CtL!ep%{A6%?i3l`0l+xpsM-@UVaS4>LpU#Z9y&UFp6
z%>R7O?e)CJA-T&B=HHiWx-YVd*~4btEhUdeo;1sO)@r|2)lBI=;SjpA{gQ82chip5
zAzw1~x-L(D$hBy_kJ#^(Q@pn``1J9K3hvg_|CO`oik`vii4&rC34eFH?(t}8!`fRt
zGnOoR&hMywf9^)@ig#SishVd>?{40*c71qksoeHN-dU_)6c<`0&*8blovrQfm9$~P
z@AHxK{+N`6`kbF_#h^O3T5w&o$fFw@1xlDI!dw1yM;uA-X$7t0n&)x&LrR6xb72P4
z{CtjiC%QXlEL-T2<NS459mlyyQzw<9JXQsoRjztP`_=OGuY{-aipA&cSrNKy_9T~g
z+R64)-ab1szma+2u^UGV*c{exH?6&Azt^PyLP@VAuXpF6fJs$zRg~|3GnjN|ewc~#
z(p@UOhLV|5-G4F<&FMS2;Pt657V~T^H_rN;@^az5oud8C88^&rl%HOjWw^|}Vcl!R
z8S7r?d`pq?6`jk-_8_WD@}vpZ^CiXohklx_+-)ZR>Y5bi1_lwSHA!5&rDreQ-n>y_
zveG`C3GRBm%QsB9(erlV)6-seW#?S)F)#e_VcpO8tiSuBvLmt#xX-@Ih@0#B&(Yvp
z_)UL5w;fj_?j_C=dvs|2vct#x-HvgpeVS^zFXsw3Z&=x$qR?v!bFaO=evvs`d)J~D
zS*LyW6dn!xl682dwCQ4paE2sB(YrfO3Ab#Vknk&LiI!B--oUG6uQ#_#M<wgzTYo&W
z;!^p}MeaWZy3R2yQFJpE+v#)onAa+u+tLi?l1v{YOAKT=4(>RYw(C}xy14P?VD58T
zYs>EJ$ki;=H{_Vnl_BQufA~a3friy34zc9r@4lqGw>A1Lu)b#V-?F`B<&|ufbFXC!
zgr?=><|kZ^Tc;oGciUh~$a+(msh`{O)>hW#Uwt(p!mCQKa+AB!Qr-T&FMZNfrY(|g
z*{{4rWwSPe$;tOC*baTJ*1mnZd9(1Pgu5Y0p8Bg^@?F;c7qZs%xv|38>^CL0CX1JH
zsu^ocWT}37weiTGlasIIWNes_xhK!0G;hv4n`<)`zPKPTY0}>sBh^1kUii3K%=a-p
z=GmaC?3-Mr;@n%fYmJHX*+QR)jN5Cge7?P7STLEbJM`-={o@Re!~?&JsjZ9p(-kS<
zDe0aVbZu?)#U9t-y@8W=3BLTrQ?<l3aE6G{x24g`(pCh{xG+IXwAEAbyUK&5(!E(M
zuH`yMJ5*L`EZnFuVc}t|*ROMr7G(9T-qq-;^Qibp)4`}C>vcO~84Osy&;M^DcBOT~
zaVN1wlRKwgy8UG9cfU#hVv6&pm#>>MKb3dd<Pb)I=FsoW4!10uQ!h)ensVsVy2<}i
zOrkgrb!>gbyGds0QWn-`*Y{6P*gV?2EARBmhj(VMGBb#CF;6JstVlT-%;Fi>al~jQ
zx5X6gEpy#wUeNj+;W}-@4`Z1t0ZE&?6#Q0CmtZ|#cy`H_j<}>ZHCO6BKRUO4PyYLF
z_m~RejtZMDW&RRn{q&9B*T3%$KhNLyIXU-tQtnIpf1=g@T;9Fe_Wz^&xqk)c{=b}k
zSpTGa$iG+j_x-*&Z+>b{x5&eq8b+hk9g63JUdv8A(s-;{%4y%7%-MS`dG7J{jhkq3
zZvVmV`o70YHkocncQKu5e8cBV%dvLzt9O+C>vb*Ns<3VCvfR>x+kB2P7I#f;D!lrQ
z;X>HccIIb4I5*brJ)&W>kk9>%;<pF8-W@Ec_!tv#Zr3mWxo7`xE%>+F;qTsszka(f
zU;F)S?(=uz?->muO(%Rd{`}}f$yVL-Tf;T-qh)j+ZCp2TI)e|ZV^`N*r+YhZrcKT6
zvU*}7?e(1PsP7`}_pTfY*(wuSMH2ov#a)Q6OE=Bf*16AUCbPoFE;Hsk{&$ZsOYo;0
zJH1Rbs6A#)#_kMlu4COx!c9_oCbM-v+f#P(_sTb?J>t)=dzyYvbf-##<g_=}yiWus
zZD5#j=JVai?Q2E7mkI9P8l!gp1na+BPA6ifObeft#BNkur8hBKE^l3}VE2!!$>7Zm
zoQ9LWJYdl*V$x*RTz7NFWVY^Nx61a}ue7UrM7M>$xj9v6Rnx(sJquT#lb)R$K6kxF
zb7$(O)62KIn@1k2I1qnn%T@ooM<!ctxx`R%`|uxS;hijYA6u3hY+9jLq`KwF?RPsZ
zrS81T<5|OZcM4;`<rqaH$*?ykZ%qCvd~WMpwg=PX*O?!Um@fa*mSOLa3%743FAhi*
zy%@Uu3lE>v?uwhi)1GZy`)=(^H=|wo^2>L3-Lsf6<Luht>czJrUR~WDpS^tc+G$HI
zymMam1>Fw1dUvA1pJs>3EuW<MrB+Y<s=t=0BJA(x#p&`w(-w&&hR@yQJXdC$)SBNx
zykh1y3|V`b9k%XWz?O2MOQ2ulL-|ijaSx3gjs<V8RxJH|>F?zF>Cf+<nsz{Y?t+cd
zpHlaJ7Pu*?;mHxvu+!mjnu<riDa(4ZX&hSxe3mSHy=&tMv$jWPYQ+z{-k&U4{^eMf
ztLgPMy1w_`m+ZS4nET_?oq{=?qV{+HeR%xz@#6BjkK)$yUE%XZ-Fx}MxS1Rttuc<)
zwsGPNpUIV!-!Uou%~tt8mr4{Z7ws)!zv*Y5{W&IR2X~{w(pcW>vfE~*RvlwCa$j*K
za;=Ey?d>9Fo7QE>{0&)U#%^OT`ebLm(fOjRy(Q-))cx1^7r#04W|PRZ#Hb3B<<mK`
zFYIA!w(1MldAO9ffZ@j4r}q?IZ8LqsRUWneq@HPnd2z?lq?bwMvEmIUEgmq=v#b-^
z{qD~vndl=*cF8&n*OcmV6(89(OweEbPVm$6%!k*P-&*rdd)3*N33XaJ%fr4LoUkYK
zFss*vS4$mv#2TD0mrQ!~BO$;bqNIDtr6o-VcU9K&tLPjJ%nkFnwfpS~^9<HQ@3*z@
zvsNzc$ogEgbm7U43qsR^qr?AJ)blf3jq8j#R=3$_(}cvxQ<I+eF)nY2+;(}z6SWl#
zZ4Xilr{<QrYd>CRrMf6-i;<e#>I2cP6?Odv2faOY7R}Z8c{OzTw?eK5B8kbmw|jYC
z=U(5n+i1V(7ULD-+}l4sVrYoEzkb%KQ-9C0|9_FqQ1?y!{_mszDw|7pzxq<Qly^bf
z!zZ=<6>FOAFY-{U-Fi<jSh_)ylOZW<a@MLNQ40^vZL05T<t`9DbjGr|(bhYayMa?e
z?ul{!m&^Zk=2y0b*M#4x`FA(=rd~p1>t>@}Tv<2MI#k3w?}RSDdQ2$s$T}-sfio93
z^Z)+6PJ7AgT^kF86}jTi8oy>dymnRPNzE`V?S^pC(D-@s%c>9OuK!i)zdQQ!yK8nc
z|NYqVSZs<!?}Uzh1!?R?F`*lyMR&Wd=zj9{dX&o4{`JqlulT}vz;_XM!}D)8<vG@|
z_In)`7EEul-m`)Ks#)~pYn%3$Hw7~tkNbDmFgN;<=U1VCn<q-c&6lO{Oq<{3kR}uA
zrlWavwUmy6_`J)nc@xx6D>GctI-cKuN=j$KWv@(5we3|e=eWExIqJVuGv@TlgTAL9
zyji+RCq^{QQ&Y(Q+1KWuS`snlvQ6w#@dgpyKmI_wh7T_;=>B{pQlC3bQ^VflvHANw
zU#g-+__O}-^-e#{*?ExZLg=b@ky|(3WOLvVOI6|s6#af7N%yFePg(0hrVG*;cfVbD
z5|)2KX&;wmt?Rc7J%tCnZDvmOS}G*)^<m}NoaEBTPn8oST~B3iPfpsPliE42IK;`r
z;B@ivji<Stx_9&*V&``F{ASXfta+v!QKA*Ae`KUq|6F=U)b`x=wQ3979~tg{C7Wmd
z>wI7P@^5Sn8{hnW_U+o|vJ^wH&uO#E-)Wd0n|FPZy?0YG!^xEFzTM~6{$E@1bawy0
zCwuF^FaQ6mnP2YjZu`I2@BjZ7Z~trZ_kT~%|5LAje>CD-+~)Yq>a9-y`4nb;={(4_
z<&6M~zNU@9(G7=W85k=V_zncUXAHQp;n3V0hZo!bd?)|^{(t@d>;HfL|MmaJ`d{{Q
z{{H;V!~aaR@aSm|?<2LRn8kLU;7Whz!eo$ckj{~%z*6QrL*Arp_p6fg^0t-l;&@^o
zzEgE_;5s17ck%Ad-&5;9{9eEB>+T8PpC5d9^Ko;@t8Bx^o;*3n_E>KHvE?!If^!!b
zw&bxk+^X?;G5Lmz)rHi9)`k(RwXKKB7mKhg=L&GuJT%>m<9xM!x$m@#OI!EM-6b$f
zCoI=TN9OL^u(w-HtU^B=3Uv8rrFhC)d-hJ|@Hk5@=JH?7La(RzY@Z_~s{ZYgWo$o7
z?VpevYg13%y8OC#KG)YhkxMvsXTFwdE}gt-(V66pysX}Ly`MAeTH1EGPAl(Tip1li
zS`v>}`L=z%nZx=k-*Lvo8Qh(&I*&ddWlGR}bX|0&yYq?Z(UtjIxzb(P(vv>!`v2*3
zZ>Qo_PBEUaiz?6OUf<uar)$H71jd4}XM69hTzyV>rx^Eqtz(=#qTd6I>#di%PJMs<
zx`d9@_9+VWzT3IGWhYmLXoc?C8qFo$w`_{44cB{S4wb7WA3yc4x}o!N$NV^kNc}@S
zi%!gIy8gNO%ZWAfW8W=e@)q2pG$Zs_T&vvcTZ_12Ln0NX7RDZbsrBVp)ZZ)jUZ@^>
zTyY`eYWTD5F|YG>Z<PML>~cgn-|awszoy6ky8YA6FYXO3e_9?_|LfYR_`b=@|GzuW
zUjO&=!TA4ws^$N^+-?8=!Q}7XIUH>I*ynDGJQmS>P^NNcyy2&pvG-UMIu5QXc&0vI
zfX7Hut@i!-f6M=G`SwMeU-awNlLtT4|C86(v+LSBU8m`QuwnX%i=jFxfx9HGB)B;p
zsr3>R`x0s>!>|7M@ArFuf3oj=?r)#7fB$wH%R5)y;%D-_v)%Le+ieD|{@b7Tzqs{#
z?i2RcuixK`t4o{owEd(==uZ39YnDAUnowG@Okqa1bi#@|o*NddU-y*#<rItC*84eE
z#c{34UGDg5wS!P>m1n$Tm&2noYh=QXx!-?v_FdNAsq>FpvmbR-bZehBxnjEH;Wr;P
zN<=Dd;BC;jcVx0lZRET!!Je_%U96W<BbG$!w$A3M^ZODtZ%x5!{mtL!EUV0%9XHKq
zCQmDmwm_Z!T>B6%#XY)94Q8fS8qL1HcaesH_Y;xMck92}``Wx*p2w)&>Befn!tSCK
zi!`^;;jLfeToANzhR)%LPIcdBE6u!ZA}@KzvMuE(ygHF_Cui8rg9|+bE_AOy)u(;O
z%SO0iiqqE0mvXNb`otG%Pl#N$ct`Vhwy1+?KUYSIOcsu3U2q}c;w6vl)UW=B4>zyc
zTb8}rGt$XFjVDa;FjIojF89k5mb`oCy)=A^>YitZ0t&)en?qALPrp30NuSxZxs>~1
zStXm4q14{=<IW66lJ}<VH>xTAI`yDL!^u_IjCpgfPCfkX%X#PO`g_MdAA1?({e1h{
z@a=MY9{*qb{>S&-SCoGB-ac#m{qFnv_vil|{`~&pq|<SW_gq@|M<HC>MV3+Hg1_7%
zlXJ`p4E?Q7ruqe(mS}zOSbo2mrAC?*Pxa*Qd%Nad)fQ2iW<Alhm$i1|37()G!kOnL
zB(XPmJW9HLG4JMQ_h&y&UY?)EE7^a)^Hu)MSMPbX_b+$7U4D7(+s}@Re_lSmz3$tO
zsp*TW=ik}&{p8xM2l9S;_ibK#>r9!HwspMu<o`uAMd@-&4j!Clb|>ns^~o1?8^wZF
z#(O_go2Mew{V1w#f?C{fu8Y&It9j|4%60WDe-Sq^=xL1K{$2HZYS)x%|G78k{_N}9
zuYR5;aK&@ik?$+EeXc2J-9Iy8>YuyLE6luY5;Yw^Uc2c1+{O3j$}M~Dyf`v<A<If;
z&Z}wD4!y8FzCH3Tzm*U3gH^?W)|%HsQ`oL=w|+9y{jz5MkHV-aa@ErgF=zfZOYXT}
za_>{|!N|&ul5@*k<&PC|vb3JubMk|@Qsy#!=8xSGNA7Eiot-EC@u%~dR-Jh%F|w`#
z?0L_MQj8ultvbsza{}AyKfWiUTZOungwA(q9ZGt#^x}=2-Fq*)tYchI(|FQGQSa<r
z=1gYs>2v3`-p|$ly)q>Cnf-jl?+3MR=y-EB+&R5`ih!bvgom`n`R6H9Eov0|L~}zw
zeQ#}885VQ>+$?2dK80^f_(LAd&?%PLI)!WV?s>PhuL}hQe0q@<s!$ki*&NEx&ahy6
zI_vUKkttJ8uG;vf^y=x(y+?M$XEHPYx?<<I{ra!2+WZ<D#t#;9CjIq=+jc*yNEbaI
zbedy>*bT=j?uL~Ojz9JnoL`&E@bBB&m7l7mcd33kxkJ?FcD%#}`HsKydEEaWmAyP6
z!#t|=)A6cVET2ERZryk#bM_vlyoz^6%+eiXRj*pAzw2`IImWE1u`>Ofz<Cv!8T|Y2
z<dw1?)^72A^KoVGrbBmE8kZ{tIv)(b=p=J4NwGP~ZK0s~GQK~@KQf%S@O+;At!=a9
zYd$Rc{%3EF_{_Dx=Dt0awKOYl#_Oq<_utrl>}A{Gs5;T5TSCvwE{T}W8{U6>?p?R~
zu-Qo)+O)#-=cJU(zY@Fc>Qd&wCqHcVAJ5^<TXU=D%%!|%?CW2y>neVn^IpbWLMJVo
zcTM%X8!7K3m<pEcVbF^fxRi65_k*vq>GcM=1`XTHcju1y`&&c>aT(03m~`%J-{YNm
zr_b<S^PQHMSml07BhG)7Su^+Pl|LDh?}+3bko91Br1?<s^kl~JhT_SUJGW-W9h>2`
z%|>Z)VByt;T^G(@jN8Baux{(`pQYOcSss0Izg?%L)7n^g$^Fu+uE&QZ^!09hO3H63
zkWv50q5a8$Ye#0iKu$T!g{UnxaYvqaw(?(7va5fQDe>+6JfCHoO*bn}I`!{fLYt>l
zTU#PSmBc(Q#aw5_(~FKwHg2fd5MjY;aQM=W+PE)DD%v_ynsYV=ymHmZSKT}<?s4gT
z`K#`2CqrDXFAsL)J;x=lGr6+1yy;Hp=_k`d^8?OjuD1KKeCnFBn}egXeER&<q{>Z>
zg;uY7QI#te_}b)bK(=x0#YhRK%g?1QKbJBQ^IdM?ADq0^baL;~CyS&E6-;;zr);#J
z`?9T1$I-ZV>$H=1Bzrvg76ojb%l9+kVgW~<!V>ukA<flc$3ItpY_qbp)UR3l<(tmD
zRUb>%ZGZi-M9zP+&dGl(GBvMjpIdRa$gs{k*7Qb!{>;n9_kxn*!@fG~`n>Dni&_C6
zzwoC9Rs|{AS7vKBm&`h5d_{t%SCUshvcLCaN|wv#zWQ|=Hp*JA5sl!F@7#8O&tLvC
zPpg}!{(hZv_wR@0@7?_uPj<K8S7Gw+$8vwWe=jHBWAC^7_hNad>z;d6&eu%KZ8lw=
zc}iI!<V#0t>|3XX;LC0dn``Vtj&o#X%<%bo=)#3ZH&)J*_gP-I`|8`8-}Re+zFi;w
zCLsA2kK$KB_6nV4QB1myQcV{`&21!K#&ONkdsNq3s%-fpcFxVY?e>2k?*D!8|DWT(
ze^=kDjL(|=ZS#ZlM;rFpSDMM?)x5Ue{(rW<>9zYS&5{>PZ#nFlpRltz_hQ`ID{h%d
z7Jf2qZI3g2N_FR1oO^Ok{K<;t8+xZ2U+y}7?C0#FYl@GuZmzg2sA(N_@%$OyjXY0e
zXPsp1zO%XO+T<Ot>cjHfBKgk0z3JEf^4gV9-)C%>3?n7ds!do7z8-lH_M$iJ)%LQf
zt2~}9zj&bZe(2T6GwN&%y)W+=zPe*)YiVt7`P^@L?P|H|po=rjc~eDBacn#IJjz7S
zDcNW1^Um<g5h9)u^Ln;uT#BqS(O#g|uxC>2OSj#|$#Ggth8IhE*R0xN;_>c!vh>5_
zlULXsHn864t;;c0Qg8OOS#HKszurq^r?-7EHCq3OK}qu0J0IgK9_cCdqOB)`Y`-m!
z>`ym3b&2O{klxu{X%|Z8GOjqgsB>vHkHZQ>lR3MhGCxJ%m3qADoR}=9_{W*a2NQH3
zeQw_&<DImwPv=?J9zD$oOw+44{508K?uuk<+&)9%T0x++NwW>V#}QH9(>&J#ch7ry
z=WflT!n>I(*Ui52*Xi%%k3MRjyRJXoA0OV=;P7K#aPfl&mli0#;x)NHH!^p|9VIsH
zkLy<WpLo#DykPnpGgV1Tg{kp!v$sYs+;*$H-}>B!ciMGYff?CLI&9-MMxN|D;h1+J
zfG1kjXf^A@pNF>XvS420<`^*bht>j@n7gsxPEGR=VOg@G**AP{U+Q`fJ~8FkQm*$O
zY?I9*e=jL~eRHCe{mt9Id2{M>?z~!M;A%Rdcyj1Ni_ekmOAa@GoP4(;x8TR-&I}pT
z^sD(SS+-#d88+-*pSMqV=lj2VcFi|iEWSnZ@KiUId2|14`KtU}egEyc%WL!)Ia*d1
zsVy^8IN7+@qSB{<@1g3$kmFa{UK)PB9dFOD;!EMnx8aO()iZ7_viSaJ&w)o5Kg;h=
z+s~9Y+gN4uo}P&x7qIp9b=YTCn`Zy?wy*jB+y3|Y|FbqVetaGO;QY~r%AxOUejPfz
z{_g+2dn+Yh8r?YVpg(hdpPzE}Jd563o<h)0o~^;xO=XI1I2~U1K03GO8{?X)_Rk+(
z!|#4L=HF32t6x9C^z_tuth;u-Yzw#%pQ@Ag?#=xA)w4@4?p(M}$#2RUNuM?)?}&GQ
zl~_V%oAa+e^-Pu_y+7Sl)#|_8w6C|iJWNzy@S3dd_3|?~zk#LWMzPk+=D7~XoU1$4
zL!&+Sn0ySn)cN}zkLt%AQ<Mb${VHoe*6C>^X;b-fmtV`}$XnM8&G{~x<QM9HD*TzL
z@Zn+i#FM(KL-JQwI5XIX|0>J&EPOkKdzBYkfv-ZorRgqF5!TMwrBViQ=cL7ybgw?E
z%$v;~G{b38>y6UYx2?H<WeLt#7n54!TJ!J0%97vB4PD<3A5@u;`Q|1AljaJ4KgGfY
z|2>ie%d#}s8&@fLR}{B7X?1Vyj9xp_M|Vr`QjsOPOAU5@eb_Cy^6}~31l{Ff*Y^Bg
zDam1fUNr6^C+j6MNe}0+KXDEbUC%WhrO&syXub8z)UEq&&x)9H<zVEKPA7(q=3AzC
z=^azrw6O3j*XyX`(&yicK3KWr;Hr4(gQwp}PuM*%aEj^s)&<eIc75~ypLug?)y~8`
z-s$_6&0SMpRBSmpYQK~$Lxy=7`(yRC$7{56!v8Lq_UERJ!u;iWQ|_<Nx9ti4AKtrm
zd%JdT$=7cgsh6yzv;UlPv8(%{{{HXm`rrBU|2)|(E-ufgB)Cp8^(_0#m1#zC3$(d}
z@{c+la|+7|$;qkv{d)HOf4`n>|9@fE?#PELKD-HDFsI;1g~iYC`cLcK{q2n;nH`)N
zp7FdokYBy^_{RtB@9Y0JfB)ZH@pN|noxFXYe~DXU_vWlDI)1w*`}INVqIB^CFAx4s
zum5?Xw&=lo_oy>@{M*dqOL&e)uDxQjIdFmY^|+O0)6P7bcJt0H-zh(TB+c&Kbj-cs
z`FD-zs2dRiMT?lv=B&LEX=$8sqNrnvhR*gc%ZeYTF5|h>owGdKx;NxY&T?sw2(MPX
z?YrtDolaN^@%ts;^}N35{rO4FPoC6&o8CUhK1g}{%jId?*Rl##_Ls?ZraPTr%AfdN
zk5j-hoMG1#ue<p+p5msaG4AV%7ajTgG)FaG>(fD}bqBS+EBz5Y<hN5{q3y#ZK87#P
zc_`11%$=(DInK&_X860AkC)F}XFp-O>SQ%%h1OScPn~Q(YUf{45-YS7;{i1kI9|<*
z$<-+=6`52h|1+0sr4`RRQMT0uuK!JT&+<>+(6zxvlA+eJVA3k%`8$m|FTXLH6|-x5
zb<al!o4u<JMtK!I<S+`gu9RbQpK06gCUq~z)$eS|y(31(Wm^CD)@kV;Wo-NQ=lAd5
zU)Qc(d-7%Fqpw%*+zQ#zWIg9tvVYhcv6-d{+Fy4aN=}{qTT_<pqY&F_xu9)ZY`NJ@
ze{GpAQ9eD=Vn)p4)p>lGGy8wN%lW=xR!E3OR;ej(@b=~Vekr`MH3}D;v4(4b-D{t$
z+VeRKe}Apn9JTUnx~#R;jEI`Dg4N+aCoawY`<z{Bb=&#r<z*ISyZ+tll%KoIUS?j#
zd7p^brN3-s{pQ)$eagvP`PKRAGwHR{wtoKgZGq&cr#mhcT6DM0xTVm%TH&mq*2z=}
z{vwW4ri7WYmP-n{f;gofo~eKK_51vJcJ=>i+s<CoKOVOFAwT1fph><_3KKcX6G9)^
z&(TjT_{FxGS56~z>ArmjcjZ4kd|aEy;)V3RpW&jlNq5Wr!_V}!r*B(oae$*C-7I5%
zh{Zhh*{69wr+@oYwm7`L$c%m4%?y9D?Q56yYBns`=&j7MxoZKl?%osr$=)jsqN{f{
zRu@F6Z8>!$bmxr={`xs7hwM%-uABAK{U_(_jyeVHeOuDkRy%IC{rPnMHNT^^|EHa_
z(ct;;_sSbyrQNJ2{wmg8PSKxGxyN#&`I;L%8+2WNtejp^mT)*|2KS1x1kG;?w={l^
zvr^x3v}l@!K;8D*>;A3`aT@~mS)bk#>DOx#Ve+>wrn>Fs{a5i{1C_2fZmpBZN%wZ{
zyzj*Kz@u~OPv+<AL!Zb8p4gD#S0%stx6@Wj1_u@)wa1M<?^iB9A)9rs)c6T#aT0iY
zz;hm>9>Fhv#5{wKvp6U$JvrlXfZ18y?AtGog-=`X>+M9l8*c05^(5B#i|({m7A?2d
z`P=$?$K@@)%hb5`hna-zn;A6Y&Zpo^&A3N5rwRSIv*?JU{@mXyGmF-mJ8b7=Sap^&
z`RBWyt<!w}Z@RVTi*rlX=lS*5<NuwL+qP}PE*l1)_5IWDPE5{^*cg2zp)&7$*{*rk
z`#Lvjbh*Fuo#A>xOU9?D=P097)Yo?jP9Ih`1kByWY4~>4<q7)bM=iaY@)-E7{6eJT
z=Gbj4@dd38>D%<vpyrurl*5FJo|B{Ie`t~Iu@zon`pUQ>Ejc#V!c+eIzDeioG8e7e
zGvnGhf3>wwYA)Q^wcGT2*7N<p9cJ29ZM2{C@bf9{s#Plve<(4T6~-p<&h+Cy{`<+_
zSNA>XyL@vO_kz`IbyB}CB}z=um5^DdGA}3BG<Wsy(??#c3f{YS8J{gf^31oYzg*|t
zzkQ!uNYNrY)<)cao^Afz-gfQYBo2kxy|%Y(HtxH|wS9HX_H8127OyzlZ6ANoMo``^
zE-_<s*JJxw&ueN~`>So{W%H{qwhQrzy<Q-5OLU9>v$6+~y<2u}<hpTOQ_B1EN7ge}
zr`>0swZ(Ia{F3H~wrwJ3uIcc%tTWQ;z7X2CnrlPp(QFfouPJX%TG&@^Dn6MKU{Pse
zp|*RH49kLBqW8CI&+glD{m=5>FIVmqb<yxk*`8_1C?nkP+UvM2*D2W@n}zRv4$jmJ
zKXLbnac%UhGm}#?lw$6u9Z8T~*ng+zRLTd{KOqrZ@3)z5n*RCj%)3!x=Yw;47;M&^
z*>-8#Hsf6t^Q27*xb@u{Kj=Qcbm4#KRL%!G7B4@nShU>s^|7t*Z=Z2`yKKo^3DeU%
zO|lnU*R`KIO<hgQT+%S2X!%on+v#s}XL;)gwRZo=0S`&fIml_A(#MeVWu3i|WOs4N
z(_PgJOqthf7pCo!+|?bccSwtKL)Q<JlC9UC*oJ!+FmCzzW?J;xnKMMaxmEQ3u5Pi}
zwM*mEys+G7yVML`Z(DqifuW{cjN|`7S>6b?Kg;qj2JgGw_a~&`;~_`&k18_LDuV77
z8N6YR`1{g*@%le?uF<}Qd=_=DqPm!$#Bevf(R@0s=dM0Cr*<Dh+@sFW4`SN4PtQM=
zkm|gz=b-njt0Iw3v2(<Eu6;|LC9--`>#aPIcj}*YW$ais<Zqq+E#5+A?~yx_8Iz;t
zKbw<1+u=UnvEsGN2c#r7zSwiL@nQ6r=LYk_v=%)JtJ`TWcY>|{aov-bU%wt)@vCZ8
z&HjCRYq`}gr<;B}c}7Y>psu0ZwDNh<i#QwcwPNSIX3KhrA2@1Yy{J6&>Ca+2nYtTU
z4>g~63TQp=6gaHN5F(pt^s({C(SWc@rJpg9EDU8+UMc4myw3URVtuthJ$A3{rqokc
zpH1H{l$ZT%7t5nnoheN%Z@1rkJ2heb(_Y@=E6*D{FKgO9`^vAj$3C&;n<Sa@6y8>O
z&wX>x{CMn>U8}`fa!bGe5j>EaFuS)wjq&E`Fq?y!H`mPP;#?}e^P3m9k~g>5i&Y}#
zk~$3AE>DPKv6v$HB}V++jTDZQwsk9<`DP|?Hg@%TW=q~HtoraU?85$STv<Y`YxW03
zl}$Kh{Z^n^mD|4jY8XSB=9$*FyPq>K#obs`edXq{^7OlAC+@4fcBoqF*d(rFyh^EL
zPTw96<?Wt5MLQQwSP-#OE?`c7eEwZ~2cIK1OiNr}e`<SvREX)JaKG!%Q};e_`9IsW
z?y&{KQlnf3wd8jfRy|e{W%A2zTnE|<u<&5ik>@;&3T3ejv87f)72Hi#2Q1I|83x}K
zcio?U)sI_lMS#bYx9ja{3@4R~ByBh)Kk?At3Y7;>4z1bRX*{3t!JLVons1dbC^XCL
z+9>AGYa=D(ALZ3>=a}sD*!jl3XEfXu_cppIeZN%v^wYz~7P7Z)ZGQhh=KYm(x94`M
z3iroLB|Y3^n)q>T(Bt$8ORd+L)g22}EM9m-%X7WjPWv!fhVbegS-yUcSt>L4T-YXl
zE@JyK>D9H*&vskSZr*No{&1}8dY|2opK0yi`_ty;n^gyMcJe0#z4nitK4DtAnMXhC
z1TL<P4sZ7`F9<cvcrx|ikM6~bx5nGtt`poY8L~L~epSZ0`^9DZH-728yEVW5`IogW
zdp`Lxtlj)EHk9$ghnoc*4+VJVw*>cPEuXq}!oSLGAuT&%6B)iZ_qQEZwCW1*i&yEd
zDu~r|efs+Fi`#L3_fLIjANKd#i^@~Gt;?^n-+g!P_Nnmr-n!?rKkfgOfAjsn>;FEz
zX9{?2c!9Yp^V1#AE5ANWE?s&p@69EB)xNH}G*hb?yYw1nnDbwLs2*SQ@n87=NBZyo
zzMO1g<0{ef?fK*G+|`yW>esSn_cFXW`C6ZC-u>pCQ2~>VeoWow7Jrf@LO(b%mFF67
zdwE*+w#yt#CU2KrvO{Oqg3W#$J~|E7j+_53D&KJ?$tmHdb?4jLXFjF;`^Rv-lz+;~
zWAD$jN}ZS@p)+}*L->@|{9j@ZrZWngE<e@ny@^AGDKmOr%}Up%F4fw-uT~X*U*lC4
zlNu8mvn}-idxO4ti>*$Tt4NdI{kfvgqRma;@BBVBR!5$p(Y{DVmH+CUABUL*7(Sn_
ziw6x{T7p}K=>^wo|9!Dh-uUiWxzh2%oaGnq*Vmo>*Y(SmJxpn7{GP~j8$u4d28Mn;
z_Ev4)(i4q=D)Tb`_^Hf`U9F_T8>Yi>pfq;Zd9l1uwa-i+9Bg#+S1;XgWb!K66rO28
zs?y)@IWRBEvu3hsh~~2PC|($$rTg^LS?iB1M%%UZtvonb+m1!v*uOg_lr2xb?~#Ik
zmWQB6-P?~2yO%3$<jCB6<;LDWHs>mJ7_V?S>vZu{@Sd9}&i|^?Ng({{yB%3pRoS~P
zdiUu2EuVLvTk7SvsCuh+|IWnat$6wQ?^)Te4OyI?mu*Y$mL*>|4xC>aAbFf=ZI@xs
z@~Xc=QfZGTpVMysxa1nUS?p7G<BY6Yd55ca3_^E)d;7lS{B-T_?|)T#{aE~><Yx`n
zgT*E{xF*#GuibAuYi04{KD%16UB6@g6xBq_GM&3IN$YuXhu8)AqBWm7YhrHiQ4d@8
z^?u3RJl0)7>(^W>e-|#defa^cc;<lbeK!RxUYx!r!FcDh$A-VhvQkTa9jdrj$?Be#
z(3}|bdizH2(wydx!i%5z#BRTLqj$DCgW5EUNV}QWVzRx1S6d%we|50_*4KX<PP<jx
zZM(_wOzWe6>Z}bOCxrHIt(|&sQKg6P*GqYmwm4+Hs#(jt<%^R2-VJm09t8bWc)%j`
z^xZjzrjQk83HiQJr#B~T(3#K3Hg|%WvqEn0jykL5%!P(iymc88#MMviske=aYPFU~
z`($x7ZQblcpBPUE&FJ2f*k}EE+jM5m$xl^}R2r_YUmUFN+QgmZ@?y=IZ1K7lp1@e)
zj7evwzCXw3d)aFF=d*5c8zb8CrUftVoThI4AmZ1HXNUfV1hrjkTzpF|A$*58w;Z?S
z!R{M-!|&dUT*3hwqj}i8wd1^Gl8)am=5o7z5j(f8(%vkt)M&AtcbW9HgCCQ7HIsF(
z&Uj?emniXhbHLWfqv_(#^<r+b)-eVc&P?p|*>LJuxQX6-jguz@!fQ52Z5CTyRdDrZ
z(9(Hr9EUd?%07G3CwXT5R8h@YhTSQxJf#d7yV9;|H|Qzr&h8C2aqc~|<hGpKq~Djj
zLf4=0Fy>KPd}02iLO$*#d76TiuXMIv<7v(m3^}Ks^h;e{j!DAaF6hnDsp0R>nbh9u
z-g?5chvULwfo+0pMptIcbMsida*^@mR$~c=49zqZ_FajU?`GsoI3?(6XWMr2SI3_9
zTBXMG&Uv>*TzA(>ycgT9FaB8MN$1w-XLMEddN%2(s;yV^?mVup7ZzN5?^0HcD|huF
zgGCQ{Uh=Fwc4bxKnX8?$J?>v`1gz3|GWFG+fK?2KJQoMe4PW&+NoC&kRi1Z^{0~jE
z-uYFpMEF#sQK$RywIA=fFh2M8PJBJ>tm#?9ujkgLeE!Daon1bC(J`*w>(7;!|8%~#
z<LL|SwFx@<m+t<&RCF~{E&F@gG-3Vwvm%QsOHb~3s<Hch&%3-?@xM-m<X*X+r}=8u
zy|PtpL2Gj~i)*zezdE=>_}ca6=lsWx9SeBb)3#c(d$Aj@dRmf&|E$%D(<eSY6aMVg
zl37>Ox7L}iT&($eGvBPDb$%KAUv}l}`FSN`cdVft!`@1TRhLgi%~__rGpqEQlEvo4
z_4{<)f1lpSzt)re+{LWwe<v<x&AYGMZoKKxqQ}uPQnN$%c*JbBWoB5*>2O^q<E+M~
z+%U!8+UMWC+_UtoNxf8L*GqqY&nnYq=9aE=T*pe(zB;r`-dfGvdGcc7V;hs1?te2y
z4l2pcN<5NS&XcO-r(Kn8#H-!L+@dU~mT@P`lJ_2y(*f^27yOqVo6)ntQ0(N1pc4D3
zd)BoSxxEX%@O;{$)n=1tuqPRDE8mSYj(irJdR1Ed@ag!oJ!Qu)sn7mw;c+<lZo}j@
zL58ydoD3JIu6{WyE`RdOu<(^`&h|3r1J_IX+482UI5UXmElqTZzw*U0+2)cGLrQn(
zjXcH~j>}yyb}?l5tS@GEknlQea7n;{<3*R;)$p>lReO^pPRLvr+t{%`8g%kR%{+l(
z#g#MjxSOK(tbC`>q^Whuao)F8zqtfIuhZ^4wJYoa^MZs8r*>&eU1z^7IB(&T=2YWp
zdE1V&Gpn!Mbu~&!iJ{a!AopDw-=`0`$Gq;nej1SbY?qM3#fv*L+&K;uie#;6JHv8%
zS^3|ZK)DARb5!RA<c2*l<lq1OuJ!$j3OoPsmD(O!x~wPuv~N-k-*o@olvxhe=C1EV
zo=yokm=IXBA?1F6KIc9C<qxz9U28-ulS;2@3p37mtbXFh)U@{&TMK^}S6c=I=a^>w
zUVBx6!6D?@mC}b6!Pj}E=Vp}%BpF9%*2?&;`@^zmQyN2hc^doIgAX@<^jjT#dGg-)
zKW=}%%e>bSYvyv-H)K3>f$7298z1GQc9-qh5*N$gnEOCj<NLIIjlZtl`}0Ki67S8T
zCmWw_`a0G4{eq=gk9MZds9IoSw7W=7Cd@ZgZppEL>)#IVuHE}CoZ(8k?A|=p#mo$A
ztClbFxi4k*_n-8XSF1K>)!uWAGxBt1X#P0atW14toommDjkU}EZCmD*U_W!dmdEPF
z&kAmE6{-uz&Ro50{>^y7$ksC}A6l7)3QV4EGi_(q<DdrD<KHjsof~Q+Io(soJMz;V
z!<E05?9t%79orPPWFMEuoFm&`M4wJSuJ*ah==KzEj{~LZpKOHe8p`Yc9o&BJM=V2U
zJ6Cy8xuMjp0~3N;cFwepF)x)lS{gjbVAdTaG0WwXe>3>BEh==fTw%98KJ`cfW5CVA
zL%a0Ps_ZLC($YKKnEX&!u=Mhag%4JHTzBoN7Wvx7rM-l^Z!v?);YW=kd~?p6znjTe
zqjyKEd_#xppV%aDi<Gl(PrrWUe+G^vvejiecO25r26z8i@Yuk+u;m1cWUBhAPd`lG
zuYS??Pj<s`_U4ytF{SU$q|aez2${`YsClS%<(uTBnGMq`E(J*}{m)QxJ1~5u*yjn?
zR9-Ki=5^_@c%0o5{g1(%AC5eJ+G_uMb^rEl+b)G)i^-n#ePZj$M@N`TZ|mA@3|O_)
ze5SW(>-E%#o!UaS8QpFzuDTD;DmD}tC|(S$jELx8q;gfO>(+|{TNPHw-><3Jc=rXf
z!R+YceN`EGLc6%0?74e`HNoOs!k;B;j&rH=?zK}3SXv^=@a(WYbHx20Z?E5aSyuF1
zarftI_l_NVeC_tf`?qaAt(_P+zjV#Sj|EcpbB^Wvou3=_W9s+wx7P^0ue_}O{PE-Z
z{x9~7woCqJ)@%v4-&g*7!Tz;p?%RmyhE{(Lm~ia$J$d^Ot!s%pK6ys(wvd~D?Bt)2
zgDbzS`06E8rk+-NEi-hJSZ{G|aH&!2K2syBzgMUJ)SVpqHi2is4_%S3k;e;UoT7>@
z9|@aoy3a>`uI;?9^WAS1tiHP=eExmK$E)LNXY*a2XRq~UeLz#d(W{L?p%12NU6lUx
z?err@qil_(9Dhx=&1G9UL-d~N{jwOYOA}2i*!XkoGIfP(TPJmTDllH3yW3BH?)eQZ
ze-;TBaUAeHcwv3ft6$}^p(^1lt>Hd)OGPT5E|_jvvrI{Imf`e6<>_@=S3fV?7sslQ
zrQR8%ZJp+MZ;|SR2s!gZldtbmU3Ow>sM=?p*7URN-O^3L+TWvfta{bvX=HIS&c3yL
z7SF~B&whC79Tws)+1hbl0CcY3dFfSeAA0HA@CWT!*VFo1q||k1N23C_>&$~%{o*Ru
zOeb0&jhf7SFYEBj4cjJ%#F}%a94=4qRJY{+W3qep@3L!acjmMNe`j8$^d;DH(v3B;
ziB{|va`_o=2Ss!qeAKdz>+ZTKN@o-2WX0cnbNGGj_ks%<c_H%*|0#7cXEG~IP(G)4
zWa;~ib^V2z_AHk+PU(7U!gaIy=kZDUs<unKx3=gzhR^&rS6{e1R4yoH_9gEgw$pcy
zH7?k=JJ)pcr0wec<-)2GZ_i)5AGI%c*R{KyqO;dnt^CStseZA%_L^mHdTNiWf%e;_
zE$cFFY`9l<w(jZGiq?O*azDMoHPblMd8H;@_<i=}PwT0R?}yF1H>>q?T>iUyqxWCq
zR{QQZ`*#r}Ubp4@voCYn^0)q8&vVRAyQaA6-`c*4*8v&*QFEh$t?%9VdoWs?>s4I)
zwsy1iJijw~kLJzHo_+Pj1Cw)d4_Q`qeL5L@`ORtT-(k0N@@Bqde*35*Hh<mg-K#f+
z{${xyR_t$<yE?mX{)I%XZ;D0rfljfmqKaN_o5D-NRZE`!wB2j#T;TP1dX{dv&fXOU
z84NMrD=YHW$4u~6u=8L%#;los`fH<$QI5`h|JVC=#7H~q&-H(j{N%2^zPRGuh#kcX
zPx*Mgc$77jE!J*l@9sTe8=PLO+7b6c>deI6@0nqXYRXQhB->AAW!rR<<EN}b^6Yi>
z*O?ub>N2dD?rhiZn{&-H^tb8Oux&Rl>D4(sO3X4=HWzr!+kB&RwSKnW{tnkaTHs+C
z8$~yv&h+wIY5Inr_Nh)kcZ*>K%dU!kmX`Tx)7a-9OArn0xbyP-qOO3Akrx7wu``Fy
zweHK>wDIW`CeEO{lXg#ie{NUq+C3@v=U$vUJ?>!lDMu;CX&PtlEi7j94yfBulO1NU
z_f6OA*|IW!Hm<+Jrtou*zJH(4Y5%1H29Z80Vht9?Mw1##e4a6_n;%z^cfPn<|Lc)O
ze-gf3&=nQkXHrrar?56D$uKTN+$>_cQ5(aywQ7nC=ih%7N}k=@bnIPC&Boapy%%2!
zn||FU)w#JYBaQRm4D-$-Oa)Ts*&eH3ebFGmd2mhlcRMb-e}$E+rLNuna=zxr)3&c>
zoBGTxZii;x*l=&%{n=Xl*Vatjc+9N6{>S}2-yWU*UjHL{{oWr&|6lkqzn){xpb^{p
z;pP+J#dZ0+KB+AHcrNU*x99KI`)|~Ke&knMm%jIY$=j*F_G&U(?6q6=Q~%!Y6E#1c
zc7L~9v^ChNz@V;3>ii3F(WA{GqJ7<KZ{G_$oRi18`0sZA?Dg;eKAbIHes9l?&7T8z
z+wFJm$T}}C*rm0=z)}0rMlPq-;pbzfu6xe@nK`ycIMH)e<?UBuCzEbx3Y1R`_pq3f
z$`)Y4x#EV7N9#%j55qMt(m(4lWdAX2XzD&ybMIy?gRxGma?kIbS646QcyeJ?(&jmj
z83hkK5e;9$-8~^l<hu9AUlR?lUDCVD&agm#F<&;b1grWJKL31K*SP+746c3r4Dopi
zuV)tTeaxggXO17YdF39RQ&#1gt*>`|(GHO=d>1e8@p4NI!$j}9YDc`qw|2Px;RUCt
zH-_S6vg<m2|8*=VpUG$=<HMwx6u9qs(ecDpy-i0xeNrmh_227X(fTK|eT!B+=4A+u
zl$Ym`nw_h@lBwV>%ZW`DM-skX+WGX#gE+GzXWxH~E9`#HYf{y%dq;BDTgAPV>hqR7
z6_IN@+FknbtIpmX#ouqay|RA({r^enllGA_YfCRpR~O}TVA&KjAwuLy&r>c=i)ou&
z_CEQP{yg=<VGEfno7&%{Cd(A&?wzg@=GpApu}VUJ6*p(p#8l_RE0QK<XFm$hSe;_)
zAaOb;kMT#w+pFqt-R0l)+}r*8<BcrVBEO}rCST6*hzOmz^g8+T!|Cf2*Hoo%zPIu9
z+_P2b!VZ_Vy^{TXy8Iu*f$VSEU(cl-uBy&_DgErQzW&=k_by#p9slZI?XU0Wo_(72
z_bSsI(I0CP_xf6N9=rZ#{kog+d!_6Cn*N^pwfBDA*}BwEog2S=Tg|fS%dZ*xEWX`m
zF8FxQonhHJZ-b2;n_qD<G(|37_Ve;>enx>^w?45y`{5Zrw=I2b^m9|CKQE$nIi3`B
z1vRp6(OOnf_kPyJ7fX+5m0O=aH~-t31v^%>Z1CW#dUqp8D_LgZi&X_-i~S$_h^zXx
zvTHKDkd{<S`#a%}G4n}g&fp#A**7g%|6x<d5o7bGjeE{m3a!&h^RQWYYtNa4nRh<j
zjdavp_~n=5^z)|d><rqc7U;7x%zCz#GcJNle|B0d_pT_{FjtS0?u9$cp84tsN3NF>
zo6;Iqd3esJZ<pPVb``0HeY7;6`J8X<o<u&e)|T!cJdpGg_;jAS!>zK3Dg2_7KXRo{
zJM?My)CK=FVrNuLI`Lnr%U)A7;=!ufdpY7mKc}?Fn3~-_lCbk?)5VRk*LB~$?<w^9
zGxf0l(yZcLox-PE=bU1?Aic%s+0T_*BEM<LoY2`EU=YE|wAg-qq(R8LTWR5+s*4$p
z$<}_m+8Moe>fWXa`t2S%O_5hz)kTjal%9?}^e8ggKxQY~>s>;te{l7tAC}g1x-%g$
zH~Q=$nQRWe1-BxU?xou8i}=dNc*M>B_uN@Oj(sdy{y*cVe!q|Tgkpuw7Rx6lxw<5Y
zhJ=0i*nj+A_2hpCm%R7=#Xk8TBSWb}f80c8y}<W3eop9<Hy3r6)mrn(m4Velxn>&w
z+e_|cKb4=T%#4#4KBjcQ{O1vd0^V>@`xYq{->kpEm-pP*Wt+E7XxFb;mlKxiUfi=D
ztGB-BS|*&G+Bbi}6d9d@tKM^*ynh~KvQ=ike3R$k%cnCR^mQIwRV|ycR4DvHXkLUW
z!wfE_*~}TnEq`ZDmh4WM&vqv=Yop{erib@ydOa(j-ko1R;r#rm%^^9{%qO2*bb|lV
zR)=$oPI%rg5(#F?yc`|CxJPJbm_bBm%;LF?O&@o8@!wyv_*AEd{nXO<lh-wT*_1X1
z9tt(FPrg&0e#PeF(odDwRr%%}Pug$Kap}y(cdEsDNrLsopdAl()f4_qj{CG@mTd^r
z%+iOtmF|oTcUPXSn&|MO&U*L$*>@IAe|f!L-CO&@+Wj#hKLV8|W^Qoc5o0it+A30Z
z_QKCwv33Ico?bb@o}Dyv+M$}+k$!7ehj2^3R`Rg-t~F-(5<A<_uQaITz>C`Jg%Jw+
z+c$jXUwwD`_e)PKFJ!L`O?#=!Ab<Y5XAi5xjz?0RqUme1<~l8(({*q8#|RN~(}<()
zCVn2@FFlcEa41@x!}&-?^%oa|XUDfc7tN%<3N5z2dAfG?EQV+I3K!Tg1(=Fvdvlks
zdN{pG*gI&QfTfk@^as9Kf1SR``AJRo@7efY?xWigMaCIl_A(VceH^Et+Pi|e{?c~_
zpI_`>`{sUTXOe6Y`r0@5yaV^g6HN0MWEvhkmKQ#!EZ1EU{zSQJwZ!VXH+m~4cdT|e
z$A8w>DSY*X+sEJM|7>0qZOnKf*GwrlW@C5TgWHwiGEaEK64xh%i?eUMGw1ixZ{gEd
z{95&0bgpP!72}SE>`mSQ(Hm>8Po4Qi>u8~p`r6bYv0Q)uq=#xwA_9r;tnPh~(O2#~
zA7*~}Z+Cgzf2WUvOMgFJU3uWrQr$p-ke$(1r+V@|qvA9c8U5XD?LEzX@4c0iL-Oyx
zJ#r}g?dfUvrv+;p^(-mgK4;dW*q6NeOEuz3yQ&<XoGsRSI%DM$SHV5X(w7>nS$8G%
zTrPT1KJ#br#zV5je~#2kazr0ds+$TrT&m`qgr~r?SF3_2nlNsB-k-!T>SyT~xaiJ9
z3!RTF=buKFFmGk^JtZNQzuNx)zo~sKM|<x*xz-#e{p1NBgGP?jgz5hnHJ*qU#O^L#
z>AdJqYnC>{t2&3zzu&F7a41+Yai=6lk;RQdn<wvITu%$gjmt^=-~a65=YKE0urj#b
zeR*eJxs}LiXTBIa(H8B3uXVB>D)Y`1E|>N=UVQwz*Y;CqgYvRA=y`LjS7QlGTFdrr
z&4Gxgmz)-@=g{7wQ1$igyOex?vzPu1&wh9=y)^mF@61m!4N<kXWA>Zpm-Wqg`lvGP
z2@B`R8624t<+SIR7`gxO=Q$M3dSh3``^Ae7+cO({yT5qK(}-_HY1|s6(sRDpI{5rz
zZ=90q5O$9<`M+)HgUgFL)+lALK6tdZKSMg>^9jSL`&=)qVrX5y>}6E><?C^*7bNBi
zRc}A*rrVKgrev}8d4I<L3+^UPzqQ^^o2;lZYi8M3@3lYebvt%zCF{R*P0`3QGPgIM
zv-(!q=|Ty?5QYP?4FB$(4ANTb$e}Rd-JP?1lP4Uu+8uu(<mrR`e<pL4Yu#ZyaDiQ*
z+N`L3B2&0%+uobco}PQz_+hEYQ{LU4)}o@jrv)$8aw;lXU!eUl=*h0qO?3+_xgDn6
zTHD@LmD||&IKZ{w(aATvS8fpTYi)V@UC(~1u=di*75Wdio?Yy1?cun!g`w)Of}qg!
zY41Q|?24ex>GN%06*NmRd=XMMKWNn?qV_m7BlOPnzgPHDnkJpNQk3Px6L{(HG;fY>
zxh9r8ji=kMJyFYMn(=<kg*knT#j-V@h%=s#yWG8DPSm5NNAK>hIQVwkVuv3E=NKa9
z?U{6undxD=+{Z)aJ5x1n_D?mwK2<F_weRf<^~HP3{m;wEe~GUV`Vx`*A-?05=OfL$
zb!?CS1a-QuX*j9ioy)b-<c(Tq>0-sX8g^PiEiOvaCO5M-FYW%!w0;UV=fw1_D`v};
z>aaOnEvWvdk@3dvg+<gEkvVFgb&^GW>-T4^|2wJN<EP(Crzd=TCu%ov><HkNZL)Al
z3Y_;YCw%SPa=m-MS0wJWt-R0daJcEkB702^Kfym*A0!JyzT|)E{3u@id$NBI+v~Q;
zZjsrn3StYUHEpRh$uTcGze?SG+3(A1zu({e`(wMc@}w>EmG1aXx$BZC_y5qxb2gIv
z2fwFGX1Hu7of~~NceQ4m&Fib%x35)eT)1h&-F+qpv&4?CizzEG?owi#awYuc{C79}
zlQ(7SCyFmxuXk56CS0giRAF;k#H60sISySe9XnrdiON&htk$3YZnyqjo>xgOd=bS9
z*@CR@IG3%f3Xj)~cr^2bwUxL>>n`hf?yG4>Rpp=5vG21EzG=vyzS2tjbHtAmahaPQ
z$E9tUuzH!DpYWxflkdI$R8*GU;GV{NLRv&DE!^Z{P{c$xmp<;(orjtZD%t;vI8pW5
zOhKu9#|qV$Bk3KWa}(41Ic195{weG*37w?b;Lx=x>2Q$5N(U{Q-3LX_#&l1d9omwA
zCr`|59!s8CQ^UKpjWUcKkBu45)jZq9dDr=*#g&w}xP`nirVS37r85j8-u0ZCej@wI
z{y<UIhK8tzubbBd-CY(Ra&GgN{~2{(&Nn~*U-tj_|I_yMU-JLCrhh+o<G-x=(}rLZ
z4IdZz6GtLMXLBCZHamTGqu!@W1{2s{E`R%T`X+^^d>713BYuczuW|Bl;pwgT*fYW9
z=K0K*4^H1IIDNl=(u3ny#H#MZ%w4<xX|5^XS>H)}FV8uW#`{>4;nRXeYhvE+G45H;
zveUzR-zODC8MoBh>upOs_j4}k>YsM6&UB8+tMAJgGJedy9sLE=DG^NDk`?vR^xdff
z8K-b{ZT025-M-Fl{2#q*y9oc}7ryUn?Doy**PpTW<L&Y#;<?dh#at8*C><5uU#lJ*
zb5hMK(~%*hKk^BCri=J<))uL?F_xREs>7xw#s<gV^kXjY<~?+JRhjw6ifP$ln<i&%
z+ACvnd-CGLCN^f0qDOXy9@iE2HlN%qqP?7*q2plJ5##HsFWv|2Pd_BT)LcM+!UmCl
znb)O0y|((Ol00YKm!!1QD^-imS{x~1eitd)ko8%q_lMBw3o`0cOW#eOzB%}FaHi4K
zpdY42v(09_NPklGZOzXu*2-Gk4i{ZbJdBg}9lDwFgZtFZ%d_HlT^8He;ol8T5IvP?
zUwZs*>+GM){cu*#Y%ZDOio(TGyVkAw{pgdZar%aJJ3~bSuO$dJmu93TX(ny(ko%ot
z;W+P>a?mON)Qfw>EG<q?&zgR(Fm>I*Q*simxAs=Fi(Q$qb8C+fOU^Hb1Wh@H>%R9Z
z|8ig7w&%ZlvDCNuHkaS!F5hFcQDtI;T6gHZMv+$CX=mn7|Dv+^lEEtR&2x{L9{g=8
z>)*I^UBjv(-9)`Q{dAu7GxsR_rv)B5z12AGLYnBN{1x*x<L~YK{p{%6>#O5`o_c)$
z_o<Bc+~@Zkd$GmPO4dBQKYn-d#?sqc-_4s9z9eUIl>HnNjo65c%?v!p{I!&-cZt`9
zOu3?b=;fom{T?&>Vphbd{AQSNn`zyoM3W_&E$b#U+b{%QSG8t~vUWW1YmM>7CpkOi
z-i7?k^If}d?X|}am0LD{Jy=uYXnC+{x)Oud-i{m{k3!Wr8|~8@0v4^wS?&GcYt+Yt
zK=xZUyOWLuI;@`kYV{=MwZUm?CuIK!i9Rxq=hB*ws+qUb4<=|%ee+1=W6Z?o+n%i}
zHk@J_v&Y2Jqj1|UN$%+qI=%N9Pb<i7TzfRzr|#s9DFREfd-a6#E9^G@IIAjNQlxnD
z)7{8;x68Z|?MGvdh*j%*-m^5Fw8B0lr0O(Ng65xP{w#VYbNc4IP;zvtoZ0gE&9zNW
zQ*Q-a6+6MQxXDB30Nb0%hoA2}H&@@_n$kbNji57hdK~BM?D*{v<}u^<wqgcR>(xp=
zdwlM_vv}gaV!@XxFP>@l<pgUq_eV@OjbLqNby&MrpgEN9cp!JAXhWQpKKF($rCfW_
zC#GG8UY<&uHp$boidlF1skEz}Q}!)c<0Yuf{dCq_f9(?X4d3<uh1%Fz=a<{fh-&97
zkCi;pRp45-TQ5}PnMxSrwix@~e4_|g%fx>x&TR<U*|mXZEm!jQ?2s_QFF(%DkxH49
z*!n=kVWP<9wRxY{1lkAh`|*hTtFfx&-TP^ab?Y}g<>#95G}lybTORvt?bI2e8M8Uk
zF1j2p<aiXb`O6Z8%o;~u!5dor0u{Tqb!i9N8D>1XQu<Opm5uj!rg>TBRm1#$o9<c^
zP3te?DgG;Wfwjtk|INnVr*ka#{xX|bxo&TegTK&<sS`C;uI~~%ceFy^FfXMvu*^vN
zhK`h`dE>sIgcm03ZW_iE|2rdemW$~m^UTcK>hA4t9<|JI^__On=E>;~L5FuLf$lii
zeD!S8K_kVE$?IawvY%`0JR;bkStFn!<i8+g|0CY>p|fwUeR?PDZ&9i2vR3!vRbh-1
zDwa(-<gYsM!qJ?TUXMlp3TKI2=B~XO8qK~wAW1QKo6E;Y#uFLIOGNga<;a=ZD0Om!
z&$ElWr|Gkbe$hE2CH4_=TSR)#&JEl`+%p%h=xFt@d1#?~y|i=3r{L{hZcTq_+!gMb
zv|)|oF~%uhed?K;Lt9p#3!lPRq^)y6@7?KpANEYR{OqP}<i5{hvmNas8b5R%^s<RO
z;}GkTwO~$c+}R}t-XH&5VRN~ge!b_}`lI*u{|N8;`Rdo--A9r|FE@v96v`$u`t6MA
z_E_`i{OMP<3#Lni_<6|R_2G*vQ{SU?Ov|lfx6?`X*PAA6*!Q*~-F4Rlqokv^j3*?T
zGUohjzUxwX*;?K_{P@|c{nz4j6RsWUQV+A)sHYYd^H3vs&8FqdpTgQ&y8b*f(f<DX
zh=$!RGyb9rVe6t@_|7qG(PGnF$7PUi!FSfvwP{`H_1jrK@APKBYz}$Hus5Y{7R&Lu
zQJ)=;Ukq<!*!Akl)^LG2R`(<uwGJ&{NLaT`;Z$lftBRITQ*3Eg^xa42{}^25iV6Le
zdf@7ogvi>d>1^v~#6In?*?z?5=*E|<+st=g=a}_2(%`_$%CrszK@XnxKFvFl)7H)2
zcE6)Hzc&5p7lrpu9qB8xj94o<H)L(zBx`d}O7p1qUz?ppH+_Pqh&*cCQ?ozlrs398
zlC0{pjF}QNdkpz*v{o=!YN)Vv>zix1MySZEhedt4;b|l3@q5<EYRgl{V)adx&!$B9
zTv*5a=3{}8#F^liaq-ub#6E5ZZ(KEYn_A8AtC>OJ;WhzJ;hy4mAu<nlZF;ay=P2Wx
zh#jKmnzT~{PHn%KT{!um7K89}u1{KrHk(xc-73OXk@_O+M}jC%SepF|ZR4WD-w&~S
zpB6vLxWY`i>{Om^JWuMxbe$_-82Fb&|GYhG3M+g4n)0lO*mt`vWR_i7JmJM&hfhYw
zPR(DOw{H>mDFIJ?r^x4WbIPu2+Rot2y!fX+yJ&*jsnC`apPO6Qwyj+z@R%o>VclLQ
z`P8_`gD>aVD{+c6UCv#-^X#T?t@%~Y^Q)e(zQ8DO=9gpkw~P65{~vhU|G#+2-F@!G
zm6?87f4R=z`g3-g@t)9!3%m~hT4Nmj?)o`vm3DW&RWHsf-4B0uc-_|YWs8n_uJt#L
z-<<ls+>ih4*}vxd3jW-h`}q1hh7I~!-&|AwrSG3(V|As^d{+C}(3@Q6qrMuaa?Cp@
zTRioq)Z1xHsncXPWISx`cMhK`RNiW1U(tI|%k<T})WZ*^xN+>AI=d!XwWn$IjxGa%
zuQEJ2Wt)sFtDYadIQQ_$Tk4Y+A5Ik$UZ%NY^>fYt(HBJ<4!^Gz-^s@9aQppkCDsQP
zlCvCFZe4NGh*wPYLDl*B)5KLj<;`ZWNWVEdRLiGKli$Mow(I8SCvt@<eOiy*-4^yC
zNNqb;=u!*$&rjX5wtw1{5cXa4>a#0l7yMWH-CS16YUG@ws=da0nU=Ph&dyzz+ZMlY
zJ1=t}S%Tq%(XC3s?jP1iKzo{v*{<rA-%@k%;kka)a><GUvE4G3<x?wywkBHlIw*;~
zJIl5B)7`ay7<RmD4t05Sl~H77tJ3js5w^JKoW9;Sds5j~%q`ZRTYfw0<Vo98)}md9
zq}t6EKdxw<wt9+}07KHleI?4>7XRP%+x<E@|Jzsgm)hBRT#6}r+gu;T79Lxap1!iZ
z<l3?gFL>tKuJx{c&9$U8zW3lR%hm$Xg^ZW-#H-finpK^xbKO(n_AJ66eEZsG5#P4m
zT#>j}H_oPcr&p88r0VVc_t`ga-Hq-_FW;Jxx%SrERq^J0@A>;>w;p1AkQ1&eWpIAl
z?`tO%I6`@{pU>SKyVW6l>*u~W`R!V67b65Jr%N}b9<}(pIxkDIGVHbgnjcq8n$J$1
zpYtJoc78=PFT<2;EE}Hk-}^LwZ<uF&_8<1uo4&sKqWx#F^^r4zk*<fnDekEFcxG!j
zmlwa~+i;nQpDcEEh~5c}Gz<6RyHs&ah~470wfdKutt*X74;6Ap-<lKs>BSb#pu(Ju
z;&1o9^xJ>y?Weo43?I)q#?KR3zRTQZ{pR2Mxppya__+2xgT*CX2C4mzm(5wlp|5M}
zZMw}>`->9O!}A-K+|p!N7xJQNW@2ZL2baIjtHh_fjTg;lG&vi+<-}=M&uFi}FV$&1
zlOwt$bc~K{niF-<BQS5{nW#vwYoZhF7Imt94w)_5_1mH8*voxScCA~wfh)c5@sUHU
zF^96$+t2^M7Q4N}^-d3@i8#?^;_qNfZie&kpXx2nyVzHdv{db|g!{A862G$p-L%Z-
z-qD<Nb7ofjuI#?1$%iyeCb!)wso8gLlXCh(ueT*Wd0*M29!!mDvN)RdC0088i+0S`
zi4~KcMApAv{QA#sd#~nxlV$@kaV6!A7WJLR)6Te<9p2;n@W-rMvIk#!dP?{d&SW`L
z8ve91mG_#dU-yQ<wBWcMwi7RTm>u8r{FZiXsn^z`nJ=Qvj&CxLs_w1Oc~q2Pxx3TJ
zfV&~8fAurBt09Y)Ze4Km-^;h{=O>@dN`9GsY*Y67Id503+gr3DZ|AozQ=^#brDgXD
z--<d#vA+(xxKg0u{+wTDH3bty`?!wh?3Cv|Gwt`<F2~32C*Q>W`&Yl_&yP%Xo@s|S
z?_RubPw~dm>ib#euiamy@+@!Bjo**;_x-3iEkB`Vm3990^CJH)&iyshse{YXJ$3Ub
zftuO9G1W_18y{TR8va7LY01$#?R`uQ0zor(FKP~b!oGfDgoydtd&Sl!;yYQi#PY(V
z6)!2cPFQp(vVPC<Hz8}TW?c<oFv+uhyj|brElb0EF3qDn-cEe=q9v_!4las1<Y9CB
z?!rUd5^={?iG>JouI(xkc(UHmG-kp%p>X3jbBw(os+8(KI2U<qnaIM;PBzoN23@oQ
z9m)B_TA;c0%mLfaQhU=BCz|ivbn?>N8Co%{9SloEShu<zIPL$E?^);$VUHO;g-aP8
z_&qtql`U9*9Xt$mOo1`1UZ7xW(D#%#?|2U~o-dS9TAg7wd1a%^o3!uCl=gqv?)Gz{
z!g~#dt|{{cUT&SKtGdAGQ&E?}ZHBzqT&wKWs@GlY@6S{E&#}O~!)IEkXl%97({9mD
z_N!$t9)2`8ju5H-|0w=%(f|I-Yu{eBUY&O;^R1J^)=CA(&;*Un>Dp!8uefxVI=Qw^
zKW!AA?`z)myl`5MR_P&4b(U`Zy&^fHF5b_sC>Yx}iLr35J<{!S$lu|W>2J{|Exf9d
zD>v}%oW0YsE+|*WbaFlGH;bA3*}pI>DPmv$PHVkg&Hsb1Hk-#;*1bLFp1q#$+P*FM
zTUYn0GO)iED-wDw{_$t^)c-%kzyF(WfA6P!-6g4&?<U@#^G)col;lyHn|hBnxV7D%
z!(_4B`d7uzizn9w>@QQ<SNP|N`4@HReA)eF>33p4!%+LG{#;yrT)tDB-B0_;_V+oP
zFV6aVRneky(_~%=-EAkBg-+=hmA8K4QMCvSTR!!_PWIJ&U-M1Q0ZAK9y>wa?tFd}(
zk+{kB3|_yf%N^J8nTXEt*>N;AHE90KdmnzZTFm%#bM2uvh6DEl%qny{4q8Rub-A}o
z>;Bc9m7!5FCv4<*))@vB1$YG>W{s^1+2tK`eXBLYUd<ipq28Pf`%gH<WhW_Db|08&
zAurA&=Ee4|mDTy(Nxd$&MRi);LGG6V()J&n8z`&9;QesjYcEY1_d*%=4WDBczBaIv
z^PA*pKlR<Bt^&dO<=}Mktw8<e-D!Dqbvcr*Jf6Py)y2(<)pyTaJinZCLxJel4e6R*
z5*;~G3_D)^>5J9gvqq;?^z)Rt3%VCn=RJ!3Wf8%(_~r`xFRGha8RpFK({D2pI^X4|
zu<C3P+v&N9D-X?$pC|{aUygjt$j;WTP1T)XF5{~jP#`*w<xbO)MLLO&NwL*(_O|CQ
znO~nE=Bk)hzw*}HOHB)w%(#Dh+brjTCfUAnH_5|Ey|=?Y-}tRqJ9nRJ)7njfBDq)B
zF0S65rg-Sfn~w)=JbM4duWEkY+k8@YtDp=Iqlw${gBkJ%giU4F)V^H2HT?V9Z}E0N
zp1EhQe^<M8ZQb1&OSf;ybFF#&<L<k^^Y7JFY%KkKUM}bVmv4t3-M`FU|Ksv){{Q#i
z*SuZ6UH;$G=6t)G720>|&ffg%cJ)`<)sQPybGDRcxbE)!bW~iPxnR#`x7GI!{cqn}
zpI!9CV>8Ra#c2$8WciFkng56h%9R@_=c{PXJh^J!%yrY=O<Of{ZREsBGh-zu*4}CV
ztN3?I{%-x>%-=G@Q^K>ByBCG;T4w$F+STCDnMcp0O!P6T?EbLhfbAc)6{{p=-tCz2
z(ynG1SL;D#BZ)I!n<o8GVM#PFpCQRpW?8lG^IZRl(!c*b{{JWT-&wAF7XM^FnR)tM
zPGuf<i|!o!t~lMgCn8~q&Wc&0)xR9)&b#}`tv|;){ffovLbk={Dw!59a+o3~FIL}^
z_wV$MpQlu``|RZ8YyV{YPj7!U*)aI)<b9q`D|*g~yN7>}SaW^-_Iq{zW4!|ecU}5?
zI_SCMWuMJn`Dv^6`aYepefx@O2G<?e`5rm7WZwP_Jn9US-g-v4?Cf(o=b`p=Qf*0L
zq1l7CYaPBHy=HaOWXCn`X$OTiNPUZ$t-|Ta>7ZqJ%OtHV@|*RIuy^Z}K7C1?cQL^I
zV$z+H8>VgSIWO(qu(fA8H>-mDylGN8Y7Ac#*KAy<B_5P=K_H}hX5_q=mP+4UuROZ3
zY^LvF$ItubMK|AnaP#}MO`J@je3g?MqJ*cpI+uA)PvNmV{Z;0~;u>}@d7Y#U5zOkr
zs^V)m%oAPg|BdbIhSJ4lIaxDbE^TFe7S(;BaZ^;9R*An(vuXVCDY8lTGEbJvuieoZ
z_CkC9(@(NSp5?A8VmE~aX1h)}y*$0+f);CZA@@y1Max_B?|MGBe!jna^Y>?4uZHJ;
zUw7}vj=#rh|GqjH|L5Sr`~ObI|9e`_XSd*!Geh9(Yl?adU-nlrAIR9gvF>e|?XHcr
z@BJiVc@J4X^jl%Re9D93|8~*ce1(qk3$O1fPXBxB?a7<wYU%eG%KYtke|-G^^V>=O
z{H1%Z9n^m@n``5{8LMs=ddKXqPTR#Y!{h3`_61uKJl${FtkhOf5tUL@+^^lIU!l$@
z_Ut#$t@&;T!{cw>-<TxreV5I6@|hRfYS%99Jjio($H`we(*jEBf9~D9y!?Cpw{tHJ
zzI>Hgyjp+8EAs-U)aL2)JI;4V*MCvZ-~a#J#dE$=Q65L%x=meNmwQAdMdn!X*1al+
zm4v*%_jBgle7{u7+lp7r^~7e2gmwP}9p;$4kh*i<+SKi9`|S@sZu-${+e&BlOE~Fy
zlsvuga8aGY^<y(6?}`?g>AsJeqI88}*4x*sUq$Hew$p!H?DpgRzq|W?Z~C4n`TtMS
zqI3FsdTodQW$I^dU)K^d|NZyPhxfjG<+|Y%Tl@0^tPP)ww{F|U_NJ$o>4HSRnzlvV
ztYe<ZC4s+g_UIM4o9_Ldvf=FM_OpRj_n!6U-diwJd$soL#|^#b79V@{>e#oFQMUxn
z9#`kjpR<*5jRD)5Wk-)btJ1ZdZ}~JuD>gEgal<CDbsMf4hK1i)CCL~eJF{E#SlATF
zwO2AXUUZc+w`REXd9y|YyWYP~dyl)ntVn*!<4|lG9~m1N8_i#_>5yCI#!0U~U;aF4
zvW10R)x^ik<ZEKMC!afeHM7n@V$1C_Cdby;HA%+K5DW{~KM=+!qRhY%roZeK%WClh
zw@!R%NGU1Yvnwp=pkUWUjna<v($sS+x0V&MEttLj=)o&{_X<Rt1n#z2{IYbLwIi28
zWagT6M^1Vr=eHEEd3Qwm;ENxckLFf9eyNe+-0Sv|Z}z+132RK|^);m|ow&y2MRWB_
zh0fcB26w&Ltv|Y%$KQ_1es$3=x&G;?o9~ZY-t6|z`e@3IRi<v1kF}pium<?FC6(Me
zGcQhigRxcipPKy{eVh00-;wcWi`;$lqmBaGK6U?l^`-FjOY?ojc5lAh{d=}x{pEMr
zYBi-cHznuh+V6?4pQ1W{?!6gyvZ<3d|Nbv8`sZ%dbM>D$<BYB1{vIvgzN|cd^T7;N
zf#~g%3hU2rDSO15((ZYOZNqfFNrmwXK@MKuX(zRoC40vX3%kVCTV>6b&g@yg=HVmF
zM+qf=N<`iqk^XX&@j{}1m{iG%m+cFZ-yP3Nx3T5VKATZxxA5i7KXa6=WfuBuk+hYX
zDf%F3)%ol>qRLk+wDm-8Xx-52k#b(+(A#+Cfz6I}mARi*<bSh!)c@-1->=s$Ye=ro
z={vT3+2^K1-%kDfc`ZAC3tLt4(>Ht7o2$Q<mE9_-efa2+(sgm==3^3y>EhyU^XK>T
zL_OXry6h`gy7+VZzgxbaI@>GYe6eTK=Vt-3)57n+QhDBcts*x?Z&uP;!?>fbSXRig
zJ@4(kYLVyr#j+|b*SCk~Oa|)(?VQrqwS8w?m$kjOmE2QQoSdrr<m;cWpT8b=-n;AG
zh1+K@y6)Q~#+1<$&N$~w@S3*Xk69Z|HH9uaADX;p*37HFI1aqxlD;_CQAOAI(21Be
zYg$*zB$pnV5s|n{bJDSj6gO?(jb^d3t-oq=b9?tyC%;`Hp}6nUsj8c0#b1kMucp@I
zCZ9~ueVp1P^6tsS2^?;J15QpZSmh&QaDR`2XOPhgZ|=oYK1vH-V(RYY$h;*gG>MIS
z+sY03CSKnx&slGoR6e^>J7Ti4^*vo)*`RYWdzAOHJ-_UnoUz@z?_d|R@4R!0^J|Zb
zt8bM1|DwIQ-a?~5+&X%u;mU`3^J|%d&cD$3xJK>yujf-r)8lqsP-Qut|Kw)J;m;Wg
zY}s?)e*1B!;ojF<#rv3#Ocq@K*u`V&HiNz^AGb#Ce)#k4j;+%3;w|?J&aJJo&g0x@
zx8qmrq~60(CEFIR&_6r(!^O2*zOUc*!)@_YJDIt55>Ia)jH)mF`tkp>uv@d|%-wS;
zjNy-*_53+=SE%=2ne4vs&i>nWdxY(>ocGSW_up)Hb@r?+c6TEu-`{()`g622C&Q{@
zmIpZ@bED=<X#VQ+n!9zOM7*Mq_orOB_hD&4^=|tUE=(-@@Qd%D_qUxhl*Ru>9cr)L
zUv%Mx;KlHN>756e3gm)6DumAKs>`kZdhG1k1qU8}Ja*AC_^V2+u%qwy{@>5ec`wk8
zVB>bU$u>om!6=*Y>GoNQzqCAs9QYr1Y<650v@4?H;4Kf6o7X2E)zG}JIrD{a;_B<K
z()+6zCaIj+x=CzViM^)&?d$8e-*NpF?<e26_UmfhiKh%xIZytXdrS6`(Y3<3mN|mW
zqMvTgRlI)cK4X-{6Q%2FPYxRB6|KFt<z&zEXIifEha^lV{at$fg;AW%G=^+ReMw_R
zF^1m(6E`uO=-zpT|C)<Z>O_^NzB6+;=U?r(U@IiEWzn?rZ|V--j-Fw@f+gT}q6#O&
zSxy(Pbf>AmjC4+kMC7{9{N2I)Oftr`p)ca7L{;AL2QO>pMr~7@bbe`xrrbQ6S__V_
zvcB!>m(2||FgzR)G|_U}<|%8^*c59G!&1-MEn0MZe{;;bih{XsgX#?OzrS4j`^4KF
ztr_Y1vH@Cad0R8uFQv3}o!~m%9?YU?bo+6B+nHzO$#dVnn|W<-6gz|JEruQ2j2TW;
zzJ9dgueEyl`o788tJkmll7IKdj#}-vj2>SE_kL5n74Z7)v-14hrzcH+DZD*ge(m0m
zb3fnLKL7K*?)j5<=Q1bpzuUSur|*9M&7Uh*segZYn{(s6c7^?hheAV)JuY;8yZJxs
zoUcfo)`e%D;w%xV+{&NM-S9Z!zSwc~(>2dh;*QPv@pra&?(VDley3YvZ(jWQPFt~I
zZ#!#4413C(mA{WC=g6Jh9?ulZSn+4E{JyWZq}{E}?^nI;{XUN;AtjBge4~n|L(}q@
z*;@=0`HR*%=@rQus3{5V|DG~o<(5Wa!7D31Y_o}pi*EJ0IKQ~)e5vD2x9++v!hQ$i
z8W^*g1CFQPHMqh1>TXgH-{hQp8LxMf4YZ%k{3@#&C|=y!;>YebNhf)VXuhi;H)CKQ
zf4Wok&9+xxJ?b8$=_O0<SAwL1pC3w!$~Il>yk^31RiC{q$}9M@)p_&asS(dKb$jCP
zJW&&q)Omi<L3h3*Ys1MN!$mX7WST?Q%~!j_RVx{`uD3Si#g>3~dEN?VZuJ&z;)_~S
zR#Nw_@PJdFg~z2sIonSE`YE2iAz}5du8WDEZ}m=YNR?*4`OH9<r}F;!GaGxf=5K0v
z?pU>Redf^}3umvlRO@_ZaZz+**z26>yKX#tyHm^HQb=K#@*0P#g6G_>m+L0mRh33B
zG;-d~|2k*7&Drxl*DEtF#{U+061nSK*~x;}Wp{STZvS*}ul<|<{riiHzR%uYvHGsx
zt$<vOliP)BUD84}+~^Bf(EoXs`@yN7rN5j%TT*ko{HHmmf>2+M?4LuXaXP!6Iu=Y3
z^_~9U((Me@8ZGXNeb#*Y7*2bwKh@{9{L-wR30>zFuQy`8{$o9lTEI3>2N4Brh81UK
zJbb(MyW4Kt=CzBP{i?VWf+qPm$Mx+lD%?{1`D1^b$MOsj6$YiSERDG>Hcxc;Bpj83
z)-!*Q)L|&eJn-6ShNQ&f8;gbN-oAS$&@B2%A?fJ}MZSnMi-~7jZpZDv=k;`lK`lf5
zhQiDDA9iHUmnbgnP?=e-_Ws<d?vNSNf9<*5<l=et!an!d?elx({?}+8jym?dOZ7n1
zG4b9r5lXQuR~nffoTZu7B)ND?W}xNv)#~d*;(zmmJzeQg9%??>@L*)+=`z1}4aplc
zTs7vr)DUxd_e8;qJNUX&R(!`yPO*$TXKqJNS7UnP5ECJ+z`8(f()rT~;Ph-)Ya#K@
z{@tI~@-lbU%HCOP`}g1C<6Ac;@-)m_e4=N?mN}ahx4sFp5{<K*C+ua-&sz0*IiF@=
zMcS^_uPU})=J|av>-nnR*)!$0L^Qs+<$1*SrtNX&pX*<*X^BZ+mXgO2Rco_$T72p3
z?RVQU9-S!>d^>06y3<=<Zn<s4m3MsYv%m>!_j?@?s?@f)Tv<_6bju{tujI4oEYD@H
z-t89N<Gkf_>_1<nhbtT={aE!T%CSWJV?@hp28+<$$xF1_uPon`rkl6w%%X<W4bQkZ
z7t6M9%qcP`<Sn?va&hK1j&R1Lisy2sGBW?(EMG3f&=Xi&yu5L;&(e67SjHQ6HD7+$
ze^~F>xzovS>ZzVJQ>Sj;9Oxc3V?uY@tck47C%B!QKI|&qwZ>}AW07R1v(qKYceh_N
zv^e*5jbOE=&2~+v|K5A$=dkVaf2w20%9^l!vs0^1-orBmPA!6$ukm&tu1ZZ*%=Uae
zanp>f;KwU>7n<I_o|*KrZEKHsKPXa9v&>AG*co^t{pHSC3BjpthRaSFs96+mcXj>$
z^QO7_F(s9dOc#&xtCRNavMygViT~QKvwafr8+pPawWo+p^yZ!>Ta=?775#f{qV4^^
z(uNHY(nU*++@40w`{H}_&6U2RQ;nrqC$PNwY!o5#Xz~6z_bR8b?!WWr@6@;Ti`DOQ
zu$%j(%uSels%z558Kv$Qk7awCA5VU{<@N2|y9-*pTzgzM-rAWpTS6|xK+JcFPw1A*
zZA#8MOV2eJ1#;*sdaz90U(^&PIw$<5@UJUY+J%!&8~<|cp1fJO@UPz2S1---g`J;;
zd0V<K|EqNGoa1V)c6Vd`Fa4d17P}}O)KTB76n*j2>&>%1#}xF0Je^=QJ<w)x-GnH&
zU4Ir7-&lV5#f#NjX6%wLZ`(LUXJJ`mhobfsgJ+WBk*g=avpxFoW0-(yqt(mV!6#Qv
z|19nj&H(Ba8*PjhEcaJE;BsWrniA1x=PYgcX84MqTj#qgF6>C`Dy5Po`9Dj2IXBB3
z%W@4rkugQ-N#m5_npby@zb#$ccYfXW_l*yx$XLs;H5`~ch4o^{PBW(+LQZ+P8%6fW
zEHyNCcxJ~c<hUy6HiN>H-ztY!b?=;g%xK!jIT1{2Qfn@D*q#wF3>3~;m9lQCf0(6o
zdGeI|XP-q%>Ff&LJ#qDGwzo-t)1JQx^f#T?m!i!larYOaoc;}m<wb7-BWIP!9Z4vy
zUVL-Aht%C9r=NV;Wv18H9e=B$s#M&qm8uxmba2sB`FZ=c*FCMX<vR4sZ%?rQVNS1J
zj-s`joGgt!yEm?QvA0Gj+S=<tROmya#A*Idewe*Ww2ydea^9zWYw^~mjR^-QEBDI$
zD2OzkwEeNsT~UT6yDdAbxB9u(^7EJeb*g`&ta!b(?NH~b_gAfUKm0lK<u3kTJdv$i
zKPAq&uOYB)?WWBM0sikMzZ8Gyap_RDv99k1F`jGM4|hHK5matpQ@L9-xsvVOnzl<n
zxTeUId<hL>@Nr>ul(}6Uq2g}7Z|XZ+UAOk=oT`n>8SZdSNqFnCwEp+MgT-ZEKf8Il
z++1zJ#lo9Br$O^cWR%1!4~dNhJ-)l*Jer@b_xj#1DO{2|alegDb?rubMXg(FFJ~{^
z@L=+EmHT{78@K+`u)oRs%GfmJQh^$Gp@Na~+`HcVLGDRAw<u(uxD!<yDt79N62lc2
z1@F7*LEa(KTH!@1FQr1}e=g8_{8|D$qQSv>`&nd5iF)Y5JLPP(8#G$pt@;q-Ci%I2
zcCnPs@$iF6(&p;HEAu6ET^`(@a`{k1oL6ag=j(X4`cfV-h6eM?3>S_h{CxLT>}}@#
zsjChpeEc>|_Wi0Y8ttDv18!a0&(g{I;7x(mNv;HQ>pKDSLwu@S(pY>eU0W4Y!;U=b
z;+h+9OFw1tvBP#XA9?R<&fR{lOusg2iiY~Fg{fYQ(=Mx|wyW-+Dtl>?UCXOy*;|Tk
z-8p`FUjFf?$qLW;`1Pl|?|$1Q$$4<clg0c=GE+kpr)iyZ++H`gZ0@QxHb<_%4_fV*
z((cyp^{nUW9HII+-%`(zCGVy)ZK@Pn&3J}$!-}E{+Sv}h%L5xe&q!@O#pM*fC;oQb
zq@3Q(afVhrc2+00u2P?r#n%4pvHrQ(odtgHf>k{k3__<Yp7ZkB<wH-bny#8Xy=7AH
z$RuJ3+u{#yYF%ntt}14(GnfoHmuJ+Sm}wtZ_5LJ7kX+6hrX7-blMdPXNxf?-Rp`_U
z5>kJ1D?l-Eo$t|()hb)o%-`)CyEE*{#M)JN5BT^kz3}PHma?SOh~oG}$<sSbK&_hj
z)@r9NC|52PZf#=pdb-;tlD)z=<juTT{fNd^K8d^vuk8<9``L~sxGLo>yIMRosAYpt
zLcq(4OA8)3UROMxzF}2)#PgU{m+qd^n3{ZDKuUmjPKaF8F^yMYVnq)fI(1IDDd{^V
z^?uWn+w=F_&4(eiLG3%mxBTO`h%3HyD`s`f>P<_xoE5S%oW{?RD1X)JZP0S3<-ChH
z<^Ng;y>`lp{nD1ctoovKoL5rdgvM%4g+;PKE2j&oo;me;h4@1Lc{Qs#)j0AkGg#Go
zKZz-yv{}l{`N-$oDWA#F3z_3)Y>!%-v+<Dn#fusXr>buB=yClziDh%fub`-Nt+yxe
zFBkjV{B--g{QbP%Ntwo1Klz+CzJ6L#LsD5YnIT1QR<=i`!i}G6s|4hDzRTy!&R1`o
zbwVXbsKMy((PgahDlz(pW_c(bGg6-1!gy%gTkZ2U%geeX+6o_vg!9g^Xx+`dtIQ)(
zbYswGl`ki5nxv%3depiHEaiz^=;yRx-nEmdUjpld)OjyS1Z;OS&nebrEsnL?d3%wQ
z7<*IM4fAgGZ`#+5_s1vgVTgUKlf+UvIcibR<R|lYO^W-PZS&ytkKA5R_4b^X_55PR
zv<a6wCYW5WTD7D8b?1!Dh2fQ=RcEs}Wt*S=d$L2jeV5ZYrh^PEy}6eX0=qUye_0tO
z)9~8WgUx29m#llLcY9i&h22CqrShZ=62eXf4m($Ah?{&W``LYIxm|$%*@U_O{3rkY
z@BQuc|L^zj*Z;nF^Ikwqu55mMXwb5wI<1b!b+w=Q+?!j@Y#LH{@NsPJvhZ{H7W2z1
z1X=_d?yoG^dO0S2w|LU&McS_QMd#l0nM_FPREgB<em={2viS7#Di4i}PR?KPA#Uo=
z$(x1M=alD6j@~I8`*vyb>Ob*1N;^;e+ua#?@A9OGbBFH+7S_E@OwFCTzO8cKhF4Z;
z!Hq0y|D1W?y8hvn@OR5~roQn>VO3hjkm7o?Ylqs6<v-Ud?YFA1Ir^}0-4hqJuP)!t
z%GNyGbkKY8mQ02$Cl{TGm(DnIy?ZTw+3BfOL6`RPC-1uN{5{>fz-N+5*ZKoS5rWE}
zL*K=DPV<vrxba#-HCs^f?UK`si?3XI681Un*HMd9(b(TTQ@_3Y>@1?^GV`y>ANQ;c
zX9B)!Sx!|-o0JkU|96Jo<JVH4V(`55A_F7&d6%SjeU4gqk|#FM%-~7Rs>f%x#jIWP
zBHSoqhI?$Q5yPjC69o?bI@^~r`};E9?K_Xk_PV^WdLE(d7`XA_$MS}KPDdCz_SzkH
zvM4$o&z!m8LlBSH(U<>CEPw3(xBT+{Q&E%mRK9i7Uzd7JL-nrU=B9Gi1{X!Uepeli
zy>mr31$IBGoV?2Q&%#gFX1=^-qpw%x9KVEhedI$>dwz;ViEBdlvdv$^#b++*y7cGg
zlQ7n~Q|9r_WBC`fz^z-iPQU*D>CT68d!#&@+Ki8e-dUt<mi%mwf=buv3D-79Xji;i
zeJS|L`}WDQNosqU7$mf^1tJ$NVwe#+DM^C)(Y38-*$TtwZgENbsn32S_V2BWS1+DB
zyZwIs_ph(lu3V&bNQo;zn6vbIH*=WUO=dO+4mW20T^(UZ?y~u*X|+7R5>V&5IKz`Y
z)$GQpqyLUXtMFB}&5UibJvmpjAfP>W>f}$8wl3m(chAc!;#o?BUo)!$pI6J1JF2IA
zXUzO!_vDPPdrqI0n)#ZgJlT@^;9BXt=&MURzwKPN)TAW8=%$0V^r|E&9fOFvkkuTn
zyFY#0?7sKNoSGQv_hN5b^)eTTBrXd&rF_6BV$0-|AMe5i&#ivI?QoYv|L3wP{R-Px
zZP9rDl4tpm1jZd7-Rp~9eypsUaQ*sY`M(7}HqU3usQb@;e3@9$#BB8y;gJg&s(y8?
z(Gd0wz8k;g=%QQZ@pFAoTE|@bF>?X)#qck$*0{^gP~dPqTK;ZrRnh%}I$p1SK8vVp
zJJ<R8@AmGb0;Nl0{C$W0zeGh%wOg%sDZ3<g>Q#5^0<K?gHr(FhYqorYL;EuAtrJx5
zZ27d(OF4JD^m)NU>&uQ!4QtMOxMjwqFI`{W{5$H(|20P0Smx2S4ST;X`F*1(oG0Ma
zm*2Nv{};I<+VJ+Ud*8vTJMZ0fPH?&Y^zvpXzTO(3nS3V3wOjBlo1aIUYxp$()WE7f
zy<@?_%Q&2KuCdfU-Nbfus>FM?uaRB<FRNd9c~h79wx!6@SzWC$7G+%>Z#x<bX5ZF3
z&|diP(nJ+e(WGOUt~_6^pE*$4$a10G;%%z8<Tax>15h=jFTurdH~0ig*A&Br2_}<&
zJz?<bxb&{yZFOI^;@uVUqHL$@Lw>yAYT(E!S>NI-7!<YenzQ+f;CJV1uXWy07CbNG
z%N;)J$lWiG1a7W+aP)CZOqEDXtKfp3z*RHWz4pB>{^DbQUB%1g^W@)k&;S3p-~PwX
zc+*9EiEGYy+zQM3QPULK$ocw$GNUB>R>{uPWp=wh9h|m!qK3{cskjc^RT@jei<H(j
zbausC?vxgYEA&{MbmC?5O0K0Ya!dA|w)t5U+o4;(Z~u0a`<&bEu!}FxOuBeQZeEeS
z)$c1WPo9|mFi1X7kT0RQM)c3~_qR7&J$YdCsB5#-%Jm$vbC37DIH@6E(3N1)(DVB5
zxtUpaD<|c=o!?VfA+ti$admw6!WZ2w#amW&s%-w{nzVtz<Z_OLdx-Q3-OuJh-(3Tg
zdqj&2k3H+!@%mcwv)c!PH$Oi)_pOY2Fzb}-(?6~YQ{cSwbm#p{CV}YJjqi2~J+R1p
zvUzsT)EX|erDsehsd#X^?l>#oW_p?D!I4tQ#chR&mb!`J3r$p4oM2!O6>>|TFSU3L
zU+CU18)J`MSA;~<A-@$jb)ua(UK??7%s=R5^Zu*FjK@)TIIbRD#D8t+?}ZiiNozN|
z+e-RHegBbnaorR4+JpMJt$7E(J<G1$@XYXL-tzdmI}8Dy2ES+3J&9^DlP%P;OmD7I
zl5PH4)Y|evV)xb0Z_EFEo`31g1a6@hM||4n-0^Ll?{O+8F*WMUodXYV8d+veab^&G
z)PClUtA)aYN!x#m?_J2j-siunW9A(8mG{fo;tUt)+wT{7w8P#q>(thFj2l+<W!k-M
zh_Al!IQhTs!?k^R7i6{zEqbbTdu>g2P~ODKh@+xRa?_vH9P;=b?>NV<yfkys1@YLa
zf^P$!J31ZSDaLSZi%JZuf-zG-e|Eb3y~-W`uL*x`*Zvg0%&{V%htK-u6rJ_2l-NIf
z`~Cjk<%O$3rCY}HWxE-SgDZ6WR!&@WCHlxYkDo_wCb-^Ns_OTv%|PtuN3()Qr+jn%
zF5s)2m|YrD!f-`irq#Ghd+z-SZSDr~uX!0)Jl@>2eeRx1re<eDUhR@(>R@)f7h9-s
z$!VvWy~ac_^{DP^7krpcSFg12a5}eSij8Ldg_TAPG7qh5JM3K6n`@r>*YhZ~|D(}9
zv5hPdrz<)&ZriM`@yea|TkcQMDYme(6{#kAKHM{-W3_CA*F100R{U~R$Z7lOm5*LM
zNbddFsHw&F?pvO8QQF(<>Azx*u>`0I@7`;s>sIZp(pa6ggX5>q+WQAQwT?I3*je2E
zY|k8?)9<-mWNsAv->AA#A$5PqQ|U=NR<3DyY?%|gkY&k5<@PNP3d5iD^7;2XeZXgE
z@?}myrS_Zc;nu#|XQX~yI4-%{I^*eVg&j!_>h|BGWiH9M>*z{Zg`7}#&$#H7Azvfl
z#{H)4d0)=rsXwguoJiie<1AbHvfUbK%NL0CPO)72^#Y6D<#WIN-pyFPRcof>f|Lz8
z9?qZ2)K0wQVpw3*+--gEiS)x$+PyEr>JJr#?v{UG$D8bN@5`mE{o*~_<EoQ?e-MnV
z-N|<KGGo(&zTJ0}mNm}EVwMzEDXM>evf+o;q{OSP@3yn3vS}OJBq<&+-*)uqzbAj#
zKFw5p>C@(__cm|6mh!qh!@@Ft6+Y&vJ8$&gSk`hXbo&9RR8LLkb5pb~Pu##%nRLH)
z-Q7u-xejpIaXj6ZUGTDN<@Aj`=|_bUbRQSD&eC}2#WB}n7w^=Wn^+$jh<yloo*i{*
z$DCuWAKd1MvYkGf#HqOTae-mac?JcG8D#;&g)$q`B^_?-Z8#IWBfFg2%#CsSf{mZ*
zwBGH~=`sHC@k|kqS7E2s+tcYEbM=Ml(qCU+zrF0<&j&9LJAQw%=jUJfFOPVws)B^h
zH^wM)F=%|sQ(axo;1Kk1Bg2H~sr_3rw#`;x>vn1}oqv$&K|uHZ?cJueBBFh#4le8Z
zu;-BX#AU0T>?|j^*e&V{ib~&Y^0rxbdqZ>NjP#>=ou^~ozsz2@`EK{U3#M5rfwOMk
zKI*dkcg4~P8z$+gNZ+{R<vg!3=(*d(Iewn3YEwdX9?SfjSh)E5ExFdOb2sd}yzeVN
zL*H@!P{XT#ANwr*f8_<2T=Um^lO0bOUUgB^Oq%*a>C{%qV-u%*y7cTtU#9Z$i=u_o
zQzEV~S|xq?_~?tr+PKxfV@)16TvYWu$Gv3bqKP+@lHF_BUanUZSK<7i^Q?1?Qa0Q6
z=cyL;I@dLB#W$$cDBnMmpyis@Wydm?&D;BX>Yr2KQHON`4U2@FUQJ&7NpoqS(lXAc
z7h?5wLr!#S2y8paCe!Ttu}1p)_jx>bqLpt6zSh}gqf)sy?XFwwY^O8JIbsExUB3sL
z-{@s_)8zE7D!jpH#e2N-V3BGs?;MN0)5D`xKD0(p38~o+nhRR&|4k}z`492^*SoW;
z|8BAg_7X0#VCz10cFoo5`4u|4=Ovc!>$;G(>&KGgdv4u5dAoDD;+%=wxs8qsCFTlh
z#Y%~OdG_U#iv?rxzi2*&pAzew>~@A)R$iO=^4pIahS7dK@e6lnoLl;7&8g^i+3Ppo
z&3tL~S8`&+yy~N>pUTc0Uek3kLpAn9Wbd4WwIP05D~*)XdiOT^ZsSxqvaD*8<duKY
z3E~GHy0>5cbNJW;8*cgkRcF`DlfT{G|KiE{t*fS5e%!>m{Oo$eP0NZ~rmatrU-{eT
zYpX|yVXw=b1Dvq|UuKE5ow;v*<IJ+gq>a}k%APnUOO)2QN$K2p_Hm_*u;I4bk&ZW2
zChpvKC8YHA<U69f#j*l2TtA*UtvHSCkjbQtD!bwivfr&a`|8g0jXlqySw-rG^aJf^
zPmLKI*{kOk$}fF(YpS`invz&jShUKchMz7|g57ua#I>@_s$cEIm%H1`y@zqa!<1W&
zlUf+q;v|z)U0>YVy(q{1Mcx`6V;fziU%|`L)Y`pf8k$L46h4aCek-QCHNByz`gf6(
z&i?6Z7r)!aFTeNm{Qq0_YBkMXFTTjF?xsetS=O@SNn8w>OBn+KAIALMFK)bj)kC?n
zI|ZWk&ThPw$)uG3{b8y47jN!bJ@(Ig%WF<JHF!R~T)OS{<K*v-m%J}V%Wnytp+0}X
zE`jLP()X9;S{8MOaW}-x<vA^R`L}o2lgOv$TlY+FF1NLdduwvIZuSjpbzkNMpERE-
z-dZ&8O#N%_dv{lx1^eH)xBOK9iQT`oR{EXcWJrFObN|PEW{Y1x9PR$G9C&b<t>Nmu
z-1;jY{bfxjUN|XVBxramuxR$>K6xb(r~KYPmU%By)C#6a2EUwXEdO$)WbOt&$=)E*
zg~=12uQ&1wzdxV%Ew9TSrs%8E>b%D~<~(CqvZOI)=Gia0cjpF9J0Ep%(z;laTFqVk
zQ*V{ci&{IC#b9buQJIk$zxpX1(}>d*bvlgKjN*)(4i<&ZN@y*TpSF9EhOJairR>aq
zrkBt7a|<1dO4;xrvCrd((u%1}m70ti_tX~$GCX<mE~@k5slRHL`!>uyl)mAQY@bU`
z;S7cg2RfxE3U8mXvZUwi1nvI{IooEQPuLst>)9naM~1R{KOO#lt?$W?KX~ri$|bY+
zd2u`2h|Sy+%w(v&VVaep*ri7<r;X*)1h+I@-sx_AuH=eUkM?@$jRngVub#U)Bx=TX
zZhPM{#chAy%$r@AwoCS|LjKx4v1?W(e?KTE|NG7)=3E!$(vri^7iE06+9mic=tta#
zxvT9a6iM)j&9OZ5L_u8agmIO(uSEFO!h7$8y_E0i<$w2{S)ae-R%Y0yX%%kA4}V(w
zwS4Jf-&+3D^ADu5HLhvUo_c~yRmk!3-EZHX+|HM?`*qX)-|yoGF8|$sqVJx(Dcke4
z{&~M6{+kN~t8%KyzB{?{U3yzISG(|;McS*?lAKg}IOiW+bh^EbanIhT|4)hvJ+r#S
zSG9bZ`OeL}39Gz%Sf}Jg@184Ky>;64m-6PP*~1clsT8T7ae1xsaaw8ahq)WmR)3u%
z^<b8R@TE=HB-o7iacPJq?>k?8v90QZXQ0GR9hI14pOZN^O#Ap~A%kIHHG`Rz;K|$E
zjz5heg>{VgT$&{p7yM+?uPVnI`%WZ;M(M0h@^bsI=PsK<Np#1jJ-*x2gPxW7-io(e
zd+nRiW4+ym$s4vPKL38Vk6FpexN=>(&~>gvg$U`-j!{4KUo^L}-q%;JuCKZnBER?Y
zp0dRPrrO5A4N5&vH<>YPa;j8&&)KkLhShtvGt<R>O?%y97IJUX+K$=pUSB$_oc}#@
z)~C)RIf1VVwiaLA=yEOeb=JLizu$=GC~aCSo!7K&_I7RtTRXKIPb}2K6K*_CW}kH8
z=EFth1&Z4^4*wMBx}nD8wk2O_P2``D3uT;>11C+~=W&BS%sTvVopt{A!*#doY6~w)
z3)zZ&yS;AFh5vhN)`ptLJC{W)OR_D!-t^APuhd>!<M8StTWzM$pjH`0@$~EYeVga~
zQQ1H9<%jjRj?RB>{{LaxU-_>3OS2B9w@T~IIj)m*YunK)eMgucNJYH-_tE2Szss9%
z#Y^6;PV(ARc>jqk!^=m;%{8Cj9sTF>BzU*kGvV-aGnbb52pHZpiOJ<;IOlw$HAwL1
zy~=HibaXAgo@tzJ+r#|ud-bwb&X9?xU+lTQVqNDE$8{a7wH>{FAv$+AO#8Sx2^^7%
zYF?)nE);Sy5agQn&FlXDm6HT7Z{v>8)Ms_ze|csK^O2c`Q|Iou!onTLxsW64aK*G4
zr;oe$9b^*NXdHd(toU5-VE^ieWjz7mSyw~u#;uGv#j`vs_m=kcJ=4n6gbHmZhD5y&
zP}Z#6zkok%=~5Hlo-2Q^uit*ey*~2KkLK>ng`a=hYq*^KKK;CDOuqt`=Gn?O24S5)
z^?b#Ju0JY2R~Rx)c?-jh_VyhwWChlWeq(l+m@mdK)t~d!j#bW4lm6}6St!cTC^0=_
z_X}A8v#8MHiS`OxT!fbjWZ7+fzRcW8kl~Pj<?aQX-^B#(Wp}n`T;9ODmh)t3$>BP|
zLwy_M-?aa2*!L#uxmN95FV{rXj+IJo1<!1!$%b`k|Kf~^nouEoJR$I$`Qt~2qwg~q
z+^OGl)%vc5mA{?j(Y<0<AN_sy`0e-mOKZM+++F_dt!L1!6B8M_*U8Fv1nel9AX-?y
z$Z2+j&O%W|U8Oln%L;4WRS8~mIjJZ7h0`*7Y0HG3-6|)SJxo}@l)vJ!$~Ti;#paAr
z{*#t$nXk9-<+9u<;>A%Kzw}tRY=u@kwVgD(lBs_>?%58#$FD`f14y!8Rw&nA{TjvF
zFr`)cWaYe<GqgPWo(HSeUXyj0W0K_mInMCls+@^ZT@%@~b8-%6FnGqf-&8hZo0_H0
z+myYuy*i*T>F+tY4J)+Ef_P3>cW%;I@g+kd`Lofj29wPT)?QG4a+L43ar3rM_w)8L
zJvfzdd~Z<CmA}9H|BEqnZv6IZ&XkmI?^C1~zWA{|bEc?DwSUg8f>-(3&#a~1{*YWZ
z({AOGtd{8g=E0WHEqh|ul+BX8t5AK3d!wt3fUc*;@-sWPa5G%b-nu7t%9=~ZwsSuH
z@?*zhX<NHxDzd7-wK_d+ah|gI_QN7yTufRtL3Pe57TpJCyxxZbFKVA%5}vd+qw-e0
zL*(B57RGlsc~<W4t(2U$n0fclS;A3MJtIIH9}ilrSbFqi$alsR)=f<e?WcGiCb!5O
z{_$+`6Y1tEi#zo)H6<53{flK^{4=am&RaD7Sg-vrVJWjc-%}<?K9*vekllP(#!IsL
z>Y3PW%BAx+Olgjv^2lUSucXSOpVt!J7HSr$FAj{b)4Z-G@MLpBQLNGZ-^Nv2b7nvG
z=8LJ^G<PB6lUeODJD)K;atP6h%`ktl-$KAC<y-cw#Fz;Va&oS+h8~%}1a?2%IW=)*
ztEgVGWGbXi%<>d6&REB&a`KH!(NqftBjra43;*yl9GLOJ^PhL_9a%wFS^kwF2J-Ta
zg$zbtidlAXJ=i1Cu)Dp@rr0jxNP?ik;@L(^*%GQ({N$O|ov6d0dR3LHe>&f-Q}s6g
zcn(~idieVK?HRW^Tg&hLe9%1qzi8)1uE5hh3dK79{jHajPIFB1yDYrd#w2c$K!e}z
zD^?qP@)>9B+$GO2BR!ji;k(Go7j2;#jZeJJnVfc-_Q|I}_{(mA8(kjW=_0y5PbPC@
zC+_zZewwZR?PAa6D9fcH3<iRZ6FWn8rdjXp*u|l!+s3{v<euK4y4{mLD*lyJRhrAQ
zI_b`-$X>qW&Rr2_L-V9;y}E_>&z-n7(BX(zklEwy^X}|pSE$&rz4Qxr(GJ^}(wq8!
z1zivo+i~N}jy*d+E$v#jTH&a~?02)48VNG&49nZckrndg_zVwjQMv8s@);Vkr#heB
z{dIZK8*YvDU(?RcOMGt`))1=3$QaRTy;c6|PPY@8eqo19^94M<9^GR9q^I=~zy6xe
zC2uy^U7oZ^Ekfga>8;Wye~qSnTmx?T_esn+#UrTkSWbQBNoBU{k_JfyoJ{FWPo2K^
z*PgtR#JR9~OUoVxfucgqW>F(2<{Am{;=_^N!KrP#|26OIyYqLJz>Lo``X;*_NprK%
ze)Mi>=*P1mj7oyj7ayC>9bfhI_4&B{_mr5DUGMDOS^n`{{lDOkXWz?~n}rlw^|+>0
z&3t+AmdOD%z0+>mN7r}?w$9h?IcN9uW8Tbrn;zP1)H)W<utu(1Lb>#!b->RlSG+ZT
z>I!~ip1R%U(uz4Jd^%SzQaH&Kxl71EaO2JFdwNRgU(U1iD9v!4(0wJ!OteEN&~TdL
z6&B?gQzeBB*%sgV^>^=no}$o(1r28<e$ACU@ALgo{R-v}Hg{xGU5`)x_iBgsr`ow$
z$uG5CHDiCxTw;`<^mzL`m1u^x-~ELZH9L+o%=yCm(KcSJJane8q1eVx*FWz1J+)r-
zzP0kLl3ihTtGhIF(h7QflSM@TPE9(y)->yz&)TA#TNAyQL{CjTzvi-rVZ^H+YYb}5
zYn1GIBJ`@+A8fdJAm%bhi*>v1{nbZf-<M1-4AuW~J#AHtcIfFeRYB&RiF3bgT6w3s
zQ+ZF%dVg?kvJq*hsi?Yj+G=hU%Mv%nQ#W@!w~{!<dh^(krOaWH)6JB=e4OxKXJtjT
zo6Z`}Y1uz`!a97DHWUS~>Yx2`j)v0I#})#2lahKq{o<c?v)3ZQgy~#})NQUKXU_a7
za(nf!YlnBW(^l`WJbnhdW>xNsm%1z4&x=oWU-|#r+Uxfec78l(EnoBL-}$_|yGlPE
z&yV@jeenT%bFlF#v-YR&7bV_)Ic1{gIT`MKmSV1&2fyWPoTD!GkVpT!qwpc$3HH;3
z6Jo#cbuezaeDwsEq+!Z2mo;Z}4zD=F;Fc;7xbl**>XRld#+i2+18)~pbl7pLWyLT?
zT(%JBY;f6dX2yg|p*~YQPP0dIPspB@#S);jc4w-F)~B*bTZB7Z**^%yvsEs8D*3f&
znS9f%hf{J7@v5I*eQ>Mw;cq;#rI&&$Uc5TZHGj8~WLVg(#VYGo9NND{d6_cXvR_Y>
z9&)j*oKUdHNnKdScf*?8&?t%J2bI`PtFcer%_AvnqSW==o%>0%tFp<X!-p)lE^RuV
z8L0J5GEiw%+N`4oee9z?rO$hG@BJ@PL*Kf0pdsFQ@}PQ5od4mnE0*j6#tT2)HCw^G
zkV#<UhGUYYiw^X>b)Td%wXgAf=G4cR!<Q(j&&+)gsb0O;RAY<&ceV|;BR#oOr$+P@
zXh~0)dN%!u#I)B+)1Aa8cHAv~cfO5v;oj1{o{vk<-{<yRz_#bqo7#0BUaszHwOw=T
z@1o7@57z8oF71CU-(3IQmF4$u$o{+cF|%UAOdH*lC<%w9ZMT;%Se0{Bm5WnASZBdA
z%Xu4rPl;LJ(UM~CC49=a<*&ojU_rhUXAerc9}J#;Ir<d8n`6)+P2=RX4MLn3&qzKo
zI>E)M(7fRMG)>p<f@&|;3nm|qSevQR<e4gTu;{Gvv8<-4+`?xzO;}gt@Mlii4SuaJ
zQ$^o|3oToqckxQ?i%W9n&v{Mxvu+d96SY~{uBDs|@5CP;+-m)L{XLs^Yx%#+m)lvs
zSP~q`k-qnt&iZ{G?G5@8zgKVeRxoB&P+q1LF4+(>`%J_3?HS>*Npqisl}>yfb<w0D
z_+(~^>lWdk34yEUq(yy8UeP105p+&TaJNX}!P`BZUCn#GZf!Cyj^}AVoN**oQAGaO
zMH`*j(gJ>4HyY{8oGF=}Bj&d%Rp&5x1<mu0PSNTvQ(2CQj7o2^D$|bdU{F}PrB+>_
zNHEfTf6l+YaKnRDf2&U4;Z2a&^=nXVl2Vw|(kj!s>(8>xx%^XS3wQ3=(|D>dN@D&0
z1?RSAI~*&wUL3hMF>y|p_LE1K{v_B=+%omn`mNr_mVTM~pR3`nK10c?{XThnWZz1C
z%yQmyt0V4a#NIG_)6{qVT5`s^Q#maQbGn3D{rX+c*GbjvzNTkqr*PhX(xT{ug>P1G
z@b27mS3L3UBJ+@+Q?@)*S|j*jk?Nm63IR;ZjC3Y6Jzk;n(1d+YfS&PbEvqdlAxb8R
zdB<4hR(B>y3K*81+qg`yNY;=|?Zo75PVs>iJYt!LSWdB>Pc6Doyk<r#bLcwui;s+U
znz5{%67sZAgX6c7>7HYz3s<+#x0RXuf79oaj0}+ptCqYM4_&GIx3|?>Fi+ktYX8^f
z>+82KnIg#Duw=`;?GLw`D0td6_$*bCnX_U;qp*=w>DCF-oy#VQI!=DO;QCv+ybXRG
zXA|G7dpAWR{md&rFSgU2cRCLmxqbX5#yk1huQ^fAPF`(#BK2>=`9u2ZXH(9aDEF;u
zI<xE+gJ-X?`r#;VU*8SJZ{D{(-+M1*WpVtmVo0lhVOsEWnQx3UXEI$lmvv|AJqE4|
zDc_V+V+0rK8%4Az#_hjX#21m7-mu9o=1EoGL8b{$N-IP9BVJ554NTEm`f%lHRhES5
znm5->*qoQ&I!(tuRqtSy-k)RV&u+1L&V6iY+Peq^1HUsnt2TB1>VKJ={ruPC-{}l5
z9vr_itM_Hh^vey#(X$+7gpvjOw@&BdQ2Md$jn;fC?$zlx{&1|4S@Ny*-0_7<rL#`I
zjul(_Y3@1g;3~@?sf&%eNgGaWcabnq6!m9Oz9V*1&of)fG;Y$|ozEf_Pp95t-zyvZ
z^GfoDmbn4*7*<D4JoekG?96kPyNyeevlULh;cD=HaLh;eSksQHH&)sId6W3@#olx3
z@@_r8Dzmk29^ic1^mm!({+|V!hqm9ZFRw3smL2LXV#B6!y(O6?_WB-MZLfko(*<)+
ztrF(%Wt~3lca@LkAsKatr1eK;xJ=%nGWYhwwPrRGtLu08&UBnqG--8M+MciCTynmm
zML+HtuKQ@*5Tnh$>QTea!URrv?kQ~V%5o~@Hm@`a)?D$rGV#eJu58Kk;AMx;IdThb
zoi?)yyp+Na)O^0ra=}uaWv?DS`n2<z=OK@`@4}fA<C;SS6LxMdTrc`*6Q}<P!Hq(>
z0#j8Fdv3CIth{_>U&0~7h#A5`8M_S_?s;vE7Gg+lYFWn+VyMb^ZL^Tcmx&X1y`Lwe
zFKZY0P;T}4svU7J*ZY3v_m}Tci)2*bct5B7UO?~iDZ)#3Io;a3#lTE)Y0bgi!pU0~
zR_wXm_G!1sd7F(Z`hDLs1h@nTPE}ecz%XsAR_$hv-jy}4jPiE)MAdm31TN#yowjH@
zU(Q4`?dAfeZ!QKBrdJt+v!$NP1kGIO*4r%9I@7Q-A)#s7$BUwg?$Ns}ZhOx1{AAR<
zvF+T^DNTG6Kkrsb@Uoa%5E|ukIms(;QMcort52TIb~oR*wcbciZm;+X&FgFKw;TPP
zxa<Gy9Md0jKDfTiH*scIF2t~G5?6@qYv=#xzXZ9k{a*ggrS(X{MeUEpB@e9~$~M{;
z?&YwR6)pd}W<@9O*F8l%VY@4*oZ9$K{&JPab=TbDjK+|@)9?6NJDXchaoZ$TzcLP8
zXX|%@)pQY?!#|_EkeQN<8A3amPxLHV_dF``<@0^e!Ft!^MLvabe2-VmvJ6+V`aNZe
zaERm+Z^ws!7HRmWXzFprKdj3XciI*2?i1a9I5Ok?#F;Z~u6Mor{ZF|oiuaJW@Q!II
z3{LZSlip2ZN$4rkTdcHV;-{BCb(*#YiJXso7T%`6wUGJCbB+9O^>!5>mh<<^@B9Aw
z|8sx7|EGVRJNC@=%l(s18mT9m>lV&C@HWo!{#4;h0&|Y<HhFlcj5#Dk&f{!K$h}Q-
zxmu&w-rpp!^lj7TcQqR~Fs<Bl>_T7|%Qgii4X5zDeN%*dQd)Z?PIK*C#yI7LR#Fc~
zNYz=cmy1_--(yU*=j8bHsqBBWPTTod%Mw@iwYl@xY!mjoFST`b%wemH8v@(?CT}iz
zpJ`~tbH#-7N~U7lvt;+5HJx4B;{EksoZC0{b-w{k?@yYk$uQ^mmsFEAyY%=bojQIu
zG+4Yc_Q|RldR=!qi$A?*_^|8fZ^yTDwC&0mejUzb*t7TPHD^2Syt##!G_HPRTpc5*
zn7&g_%k^w#VoSo#u#gg-hCiuy&QDj#S(V(+`HW$ib~Vddwf$lDr=|Y6b_6t!;<`A`
zxMZ%Mt-vE0_iy@E*S$o8FU_&?*O6+jk~}QVRo=WU!?WURcCn)wuihhtZWo{b2G_GD
zw7lJ%XUR|$dm;Mz`U!@HJ>0AK)fZp%y72g5Z*!<&?t>#)AMEd!$MF0*mDli3C*k-z
z@$ma`J1@mo|8cgxwfn`tAI3jD_g&r<Rr5JZ|K16nBHy^rT+@zg&HZ94)csNPP=rg;
z%Pro?h4+QI54h&PVB}Lg)aaf%HR4F<4a+N!a_;E!FK>-n@xC}p`%=hM*VZSW4&2DG
zYcyC96*SB4f{dnWPtp8YQ?;x1%)0GaR8#QTv+e2oG}g?YccxxW*t+mk%L2KHvsXKZ
zGu&)fGM2cP@O$p&zvm>c^6-6^7w^BvZC|C&dBsJ|`SV6$-NeQde}X<-i(bF?;ii_x
zS^K`H7zBOST_NO<c`MQ^En)><*`D_<t$ha<eGpCCkg9s_q^<Fsw)D@TnkjP~q`cm_
zDZG6qnDbX*V#G0xNp;JL-pn?&@SC0z;Sz6o=K0n-@gqfn7Y$Eo+j??)8tE{Yo@-$7
z$ZBHTHGBQmzH5v=Q&w{CeX?=dM(`>O@!n3ODUxr#s<j4bpPG8NkU?pM<YI=2*?vqN
z776R_uzmg;Y?!{oM*JO{Z08Lzc}?}N3<U-OCW?nUdDI2D-`bi~g-ueu9x_)XV#ZPN
z?uANiX;M6onFLyv)&x$OU8Z_pX+^4*=?#V{@11`LIhqNaU!}C>NY@T;gV(QaYp;K2
zn`!%p|LBVim2cU~wxuW<8Gml+VU}e-Q#C<JTOn0G&T`>J<>Rq$momqHcz282%6HPz
zO<YsA*PLpfl*e!)`%32QXNMkYX)l_!`c1LQ<(iA}D_hk$j;3iIikg%n5UlCFK2ZDc
ztW=4$bGi?;2lpj&|86vO57k>+ndvQaJ@-PS@tJ*M@sTd!JKfePE}in#IMTj)m#xi?
zXz6=<%y(`6yyUk;`|*WVB96TmMAt5Vbozl`v%YKFQ*)i=6AyRo&_2r(GBGp#NPuwX
zdxM_V&(qrE*DVZwnq5|{d}b*}>blb==DPWj+I~D?lUP2kzW#Fak%WnC-OqGFM2tS2
zI^J_Q>{^gE+t0MhDG?`(jY<lPbZ$vb+2Qr6WtF7IH}(ei^!a6<O_M_c;|h~Bk43qu
zg690^D?howz%g6;gU^O#sh5Ly9bdU+f{D=AJAXtNls?o>JhF7Q!W_X)i{O>^f9Cd!
z9C)?0YSMLMXDyAnr#5PHZeh}wI;1sQMkm0m=4{OBaQ#M)@+S-dCRe^m3+}7id-Qhn
zksQISkfSC-hKDmx92HO5aDr{orS1#97FEq^FEm~6n*?j_ju)~2m$2jij>@-eA0rqv
zVrPii=-!+gxllM%WY<5-b&Dpbt@zHJW4W;SSkZdE=@V?ac{X-PDjABI#;$8S94f{Z
z`zYXPC|{h>@x4ZdQ;XORDI{dB&6($`=<+mLcSm0HhQI7Dbu-t-B+J-yguHEge#=K&
zbUMSFn^P~x2D)kL`uM7dmp47!KTW*+)$INIH+@<!z5MZiON&2|EBLN|Rcl!Iv!chh
z-s6G!pKrW3_8qafwKKA>?Aa+huMJygT-hxm81yRlTjJ_{U6b9kcQGxvt^3<dsH-VF
zRY~>fr;4RJ{9|S^6iDfuvfQ;s{^BK<+C<0JOjo8eik1vJ`zEO@v@Fw6_n7mb@B^pU
z#(JmKi=3Q(hi^^UFfFhXv;wgE*Z~Fkc}yK?7d%>u<}d3Bd?fjdRZ-ka@>q-Tws^02
z%ZYDZUyq*YB^$+5@N#bU+l~7>cN*VR+8NX`F`M~CqD)|E&^qN^;=Av5Om<tx`!XeQ
zbK_JK-rlLL37(TQR9yq4lY?2dT24HBnyof_{*fZ#ygToO8CV!!3iMw2`*8XH{lDvf
zgSM(X`(E!L!OdV;pP86-D6sR?uKsDAYwcK`?wqqCqkPBn_Y#i6ulr5U-mYDpmEZPv
zn}7UGmYCVUtilfm1&f|;)0(}NW0h9m%$N77SlC24gWfx_^SVYbyY%<lT60d_{<|!n
zRqt|w;EGwM1tqhuDl0L~RdzmF{pRwq$onPXW!aCey4iC1&97m4U?)6>x9wT-_4Et6
z`#aXf?Z5YH^Fwa$oBuKoJ>`;b)!v}8CD*+pM*8^OpsfqeC*1tS?-=B9UAO&H6+@7v
zT#7&An|JRRW@ue;N$OP9Tl%HAwcD1(c&gB@B9&=Dsi&Q$E2(JR<$iZflKsr<&?vJr
zA1!sI=0r@C+<WRwf)=aph4vfX?92E1Ojq{#xSIck(Y`eBvWtrf!fSf{T=W8bZ6CWj
z#TlqgQc>LU@S+pLgltt7Wy@E8WDC}PIw5~nxOwZ06Q38qNjk>I;4$-k*{^@`wrr;-
zu&th%#!wz`X4j7i0&yvAK2sUE<Fztp8Jd~j_L;iPeDj?nn|&9>o7`#BI)D9C(vgI<
z9lKMSyxKEd?<`%SIRDSrzx)3d1n@`iuMR!-yubJ%+uvYUiR^XrY;>0u26%rFY_Mzb
z+r9m!!}7bf#><a0ZMe15%j$b*>6%Ysin>`Y9YUF`PkglwIrjIj+*Kg<`AUd~-h#d(
zp9)1Ea4*-h4a=)5W&G_JzgO=N<4w;4VJ<>vibAfr=4%{ctn3Ptx&2b%$0F@>S?e+z
zmoh9ZO<`!@x{`NqudLgC@5|EVzb^kd?Duzi$+O`3j`fN6bEb8yelD`{-?2k0@7})5
z9r4cWiPQtZ!v<n!LmCYxzuKMea(3>T?^Br+SXOu6(RGl}_LC~FWInL+CF2TCpQ$R!
z@#{AxaCW{(dG>Z@;jyU~`C^;*>^(Ze<Eh1}Bm-IY&8rXWnqic6^zrmNQKic^Y<qvE
z?nsMF1H+R07bdH}4o(gG;8<Hb`Ag+BUhh4BHpU&xPX>)E*}2d8X2g&!HS@Zjva#Cr
zMgbo$&%^t;47|AC<uGW^blmm)S8DTy7g?@%dS`5l>hdm2Yc5T(SQutmcwpCF%j=8o
zp5sV8EaRrj@v_c~)n(_h>g^x$Po10oGl;D_&5cK8m%6~SsMEC*SA0?0^JU{MrFTLP
zFCO5$SpCK8-u;OkH~%L6{%~>e{6CdaF~`sT&sEIw|FqIOdV=+owQFaqa$Qb3^@Ho>
z*6%*uJ<<+!PZ<~{CB}BF?Y?)xJ!+kx;;AQ6ktKYqUvPb%Gdb|=79CEWo!|c6xvSJN
zC&Sj{?f#%{xq7B)n-+D-$umcU9=Ej%tL*+J7I)aiVWQ=#zd5%ncl@sQ?Va;bI)0|6
z><ZU+r(TxcN#1?nuEziOpA&f<>@26upF8JN=Gt=3Ek(~8tc7EqH|U?{I(-zh@X_^-
z?X8`j>3v_XKYb!Cv+)vFx5P@1Bkv{^ep7W)*%{U#;CS}y-z8ni4DPP%##&aL%+Gc)
zGA)>PhGp}E)L%!gRW11KAI^7o(v!B(jYi9!YWvRBPOB)>IMHZw{Z!?_+xw?=e@vOK
z{(2gl9{*9Rb#L3ASHE=N>(yll(yz&w{&8yus8D>~SKP<J!MgheuW_)8Py3z%W!KkN
zBlm4pi_jO@cc^yrS&hS{p(cGT(~7xf1Yh?4WMvq!k5kMw%B5kOYtyasd|NK<>|qwI
z*!^nrnJ0BSC*M}Rl+bF^sH?~#VEFB*q|TZ2j)I$$r&cNN+OF67O0DBsRq6Yai|7CO
z`O0na&%g3dlAb5lJ58DG+!z$7JMU<Onqk$Ic7++P`xd+0wlEZ1V?52cW6?`F`Sq(F
zZ@9m08{ZCr=>4~1rhExH_T<!=r5v_FT?~^i&l8@u$T#5K`TyD{UvW;be1Cy^<JF72
z3@zMmKRvm@H|f>n*%|%);<qh#KlqqeKW|#Wq_Rirlzx|bC*N*JJ;qaZU*!IudnXUy
zyZHat>E{0>`~E-wecb*~#`~tb`gi(6Z(dls;+J>&28Nz1f8QQ<_Y_K5wqY^1#f(WR
zoC`t>OPA=sH4d&l6fx6KQT*v@FGh)rWpDHwu4b1?`aKDhop3nf$*lG{Kl#d7Hp{38
zMZbRaV~zctZ~Y6iy<R+<61#!x)9D$PHy>x*{CCOe?Piltv==feTt8Ns!PpV8^Oj0t
zaB9x33FW_UUfz0q`{QdDC-0r8vPUoOpM(D6?S`PXhQ3_azix(=KI>E(R=TBCtYfZU
z!jVxWZJ6G~dSmy!&9B&FE1YXg^P57GX7)BP-bj$W;rO9AEZcpl)U+t+BMb*5@9s!v
zy{B>ZPIKG!7Z;wsDfwTqIrd#o_TPz;pN~rF#Fvz2)?GMxTei?uIFE5wpNq}0XQ1KR
z51;o-T{-;hdws&boqCTy=WI6Ysh;_gu^}*MriMe9!gaMYBi&`^&UDtea|XR>d#)Gg
zA<)(J=8nn7h<(!PQzakmlzUe^@68PFw?CiFcX908P?olfWtB(L`)iZ8a|cBV-fLow
z-opQGs!4>>8o|v^(~TKd>~Y<(R;W%t{r<~SPdhg?@hvDiTX2FS!CrsT?Q*yBvR~p_
z6+5o~*nRof{kdZG^UUSv#@ALK{hK8!bb9&veKFGCj|SZTvwg!)(=#4AzEev|J{6l4
zEYY94x@Egdz~VR2M@rU-Iw~#uRd{K^5;>jPsX6}3xCAdNw(mY8d!|TLQ}4*s&c{ri
z)ebtd;=Lu<b@*IPv@Sg;vs{&-!q(GLXX9NT<EVS)YtHkW+raPO6)fCysnyeZu2yN`
z6}6_#8mSSly5&El{xJhD$8k;8d47gNuj=sLk_YPLd(Op&+MVRwAhx+h#4hQ>v1Lr#
zLirfXyKHrp7}EQzjOOI9+s=xNTzfeo@P?DqQOU)ExeCv$4qehPOA58-Jg&9#?&%&T
zgJWvF7x!B<Z=E5w#wMOK>S76N)XyG|WtZ6e!j9aXTW+`G$Ho6^tBdURer;cT|9|73
z!~YJxKY3Z(=(f&EsV3HAyVSBO-E!kNq6E(@l99BvpJP1D{_PJf!E4uU>M>NQii(7O
z*cl#iCAG*^^HAh-ckbC7>T6deZ)#NF%9{JOtmN=D-DxYGZ?zj=e*41h-n+|_uDDM8
z#yIVGQPsD;ya{<{7Oi_|bvxE2eChhX1^F8j4EVAbmhIB_-sO~-_RY=A+hXs>XBT_(
zmsa~X%l}{U`DDB2F10Tj@3uaQJI#K4YVWTdr}oylCQSSBLcdmg(#lCY+ulFVyBZjs
zoU^ZPuWRR<T_;=vtY6Na(4DqwiqYnEP1ACf<aL6APA#tTJ)pR4!e$GBBOH@7PAy-+
zz_QD~J@9+K=Q-D1dh?IG`sU`h=jkCy^8o*#kf)#T=sL^G96MpU>{3Q$nDCJV#seJp
zPxcf&u+~nEI2~~;22{btnVbvzHF>k}r;LU`@ubzZ_DP?YgfltF82rEHbK#WwMls77
zUPl-mwG$(&jwhrtoXb}@`MU35kz-bQxU0v*k~PgUs)eGiZ}!=_G2iv%$Kz)YRG;Hy
z2tF%&^lP@J;t{8=3lDYGbJPQ+dT-rbEM@m*bFq{~r^}vOyVuVDW0B5(J>NXq`ug_&
zuQmRduU`JpahqjJprUw?*Y$K>hR52OhN@?+pIM9Q{n~Tc#ohdz&-8m$e(VKvuC?Ac
zb-PuxfBMUvjtUGs{-tSYvZWl81t%~~C{OZooV2XVV@>I{*Uu~>tNQyv>!vh=?$4j8
zcQGf$*s(sSvb)Um&GxxBLzfEk&5WHosm!?bI$OJ%hNZ`(Z)Y?#dtJl-PFOv4cFvz?
z?~iU@Z&Pt$shP!=sQ9aX_IDnw+}E-0Pg`Jo-kOfxtt*~J`A%h;q#-a>(DlU4l}|RP
z%Z0{XnEy)Y{8pvARcRKYe=S#a_T4-BPvzw69aeL+7>*>I4G{dPro(qb&VAzlZ6b}a
zVMlU$Tc4M3&+y#jaJ1QO?Nk0U%e>lT))fjbXjn3BMZi0C&qYsf7VN&kd*0vQ|JD9$
zY#&blcnhxKxQ{;i;nTvb?qB$br<yzX!-p`<j14?udd3HpTGyNx<($|OGevD?+pFJO
ztn8R49G5ZM`9MM^XluIG*Q@X1mYTC)@Y-6M>#)u@YRbuj76N{;(i@A|SIo%1x5Vox
zV{_mUQQwdkR=oR5bWYv=tFHZ#@t?MhPV3K84E0-i=j>5BZ&kstK#n!Bq%<a>hI#oz
z$F!ccjp4z$9zCun^^4A#RJmN3xnj-j9E0$XjL7*xx7YP?t`~^f^+HzQmtu`GPuRV8
zla=h=u&;GG*tL`+sxtrgk=WL~QzUefmcJ=^xcq;|%0-z;8$3F<-rDIkA#d*6b!F><
zCN8K}o!uGwdWL=Xn*wti_btt9ruY?yWlxv6YBJ?_N||cv{96x9f-O>`(_D6aJo|9&
z{j*VX+4*<9Juc6ZRizxVt6rv(EmQQ>N6Y68$s6{S$m?*ii!olY=w@=%Tf8lrFZ+Q?
zl0x(YHa{r~MRx0DzcwUG@7h;U-OUo=Vw%go_iA=2gGI(Q7Nuj1CYMWZv<a*g*|4<U
zuCrA6v=+PZ)W>HQEDBn$BIbN9Mt7n4?Dm+54O@SNGelgTr2d+LVWQ;AIZ_py?%8m4
zyxC&3CRHa{?7UG?%rT>9i}XtVz09nwHI`!j+`l|#b-)Id#B&q--(`DF@>#oFUZp0Z
zG}Cfn^_GeVM@A*C38GIA`{b-;o=_c})50*XrcCO4l=&v1<hutX6rHCrI<z%yXTGan
zA?>i>z4&&iUH6^#vpaaP?9yjgpx$u&;t#iJoC_X2mHio&|0Dfy)4YP;2jtD0Ya8r$
ztvA*7`F|>7`CQrB3CkTNOj7sPhOmD)e)hp<De=FB#y^a;PQT~9|JiI;zW9!7(n0@a
z%s<YZP&;AvkAI3QYmZ+0#lCarHR-OkU;h>CzH9S}e|z)6=D&W%OuyXLZ@gTyN$6Vr
zhQhzsE-sE=b@_YqUjJE+u}qBF{z9A!j+GotT`+m#j5#Z-4xUxJZDXiDJ4e{T-{;k;
z-wr+u1}bcmrdq|jteH3K@r>N1&u07+*}Rth*?Y-J9aoKw8SNGwH8SpukUj9|$eqjt
zwY<dlp=oLxFMq$Qy;gF^{vVnQ?9XO=yZm;;?c2qg*C(CaxhDQWW?I&}(_#1K&6wQb
zdp<<=z?CWe{&(NAuZf+K|J!m>z#X}m{gsz$|9pAaex1L5m-Ln0M^#R2l6I)yao6eZ
z%To`hUi2&Yf3SCLX=%;2`<a*4d-ld>EV8+|Zu`2*^g~<IJ`_E_y58`Xd%FMCztMBU
z)%(A>y<9o%$j6N1wP`}T9c?Qr?4|Oy&%VjNlhJ`)G5BR(K)C<wx6cCC-4YadQD=9j
zuldrP&Dzp`Z@qk*l03my*4FRah39WS6xu(38}PmOmd}RpxI;oW&UT-<$08%^A<O9<
zbSI`L|K!$>29wzi#S}X%YuzciVG6Uy<wHlV72dn~%(WoM`^u9@-NX%RHM2UM<nPuU
zTfd)E;dN?x{Lke-8K-?bZ#l#8^qG%Fb)_Df_y3=aXZZ1KueG4U(mgL^Uu^k(?)<%A
zFRp`=o|LVVw|n*V<S|>%NzvtTmp5rIwW+<KZ(LaP=KjBVrVH2wI)6NyTOG1ErCRXS
zN#&Vur|#VO**uKF=Eu*Y<%d39^L!$fGdJ+*RIvq|+xJxO{-HjrK;rb;%ED-tj6YBB
zYM=e`e8+Wh>sR_xuCM!Uv;Xfduiw7)YyGX0Oq&bSm+rV>^>4SfcK!d$4-(6o4OWPU
zM>8F8%zgaoi%Cw-^T*2Hzb)8SXv;BQ<mXp~5F7DbHG50^*_yUrG5)-a`C$Il&+U^Z
zOk3Fc_qu+OKnCxY{N)|y6MkK0(AxcW|Jmd2UraM^)s{2dHMnt>yTW!>%$N4tz4aSU
zudDz6;^Xe4hbK?m^?&Y!oz48eqiZJWKRw=kK&Sjstz*Qq7q{*6*j!)po&EFfTE<6(
z195g!*T?*Pm&foReyaSFXXj?Gp0R&Iytdgrem}+-+r1VyBKDSi*KqYPpOC%Xd_}t7
z9Fs>2Zhf-!oD?&ALVDqWznhN4NSAM3lxDZBGt6yo$sPgELz7e(C!O&qn%W&PNpj|v
zn^x->Ph5-t`)sdu^!Aqyo#(5!O#gUW%9|})a=!kuQw|0UtLHw`moqbZd|Ha-wYW)K
zNmYl}=I5s#e2T0TzU}WkkwM$|gsJ=y*TT~b{*7Nhrtv&DH(TS2LPbP-d)>NiEC*Tt
ziyrv?uJ(V%f4hda+2#KJY~j&M@4ZM^u&?&{tlM7>I_yk$KA8}?iSg0Y_qm@hPiFjJ
zXsftx+mWl+UWo4y=8=o(TK8b#x>H}e9rWZC?9BTv%+Yl|-M#j!Ucra={L=Tizso;Z
zQeRrT<>3DR_FJ0zxz}C`-qu)>y!mT^(7jd9Z~iJPDJywi#rv{c(4P0OJwt)}ytb3a
zRx>nMbI;}}YhDz+Bb;ISzOVL9#c}n=YU+QUN}s>^oOk%AuJ7NbroV}P`gncS<-ac%
zTi>*5IQ;TQ{8g#+Uz`meQW8F@UJTz;_jp>?T}vr@-YX_)Z+rb5X7Jp*xr5_lt;Hwp
z<c}p)VHc9lJBI7`?UmV8RJ?Du>8*Wb<#XfrGhev4z`|CNVaF}2S8WHkM>jpVzWsfS
z^@?1FTD5&2*{-DjeYkCh(N4L%xLjFTo)5Ro@7G-V|KoP~y3)fBtaq(9PiDBk%Ua{f
zx3dMs$(vap{PJXXxM#I1Z=;>-ww;U)-+ybUz1z3YxBH3zIgJ@j7P^ZTpZL&vZ<!3I
zOO!_tw~FV76$UORgO(qvJjOk3w)pmUYtsMP-q!SBs?lOrk;oI5I3Qsfxa{B2Ta)x2
zZ`YZ|bImAD<;usn|9|KIKmP02v*_>dQVxW8J$R^Yf4}<szx179=C^VucB~2Dm$O;m
z_|SY`{dcY1KNIrb7v}qGZq5i~J|A=7dxM(osqOFf34K!Wj5snul5xeE6Dt>OOcOA+
zxo2Sg`sDp``@N-?`kHw*UDZ|F?I`PDIgw=!C&Mj`a}f$I3qurK7WybiG<>(WzVkMR
zkwZk==YFkLz@z4S%Of(r{rUCv+V*$*CaJvLVz76Az0nOhrW<eXFkg88*!}(AtKRZ4
z{~OC!e!V#L;<?LHoxS+nl=5Uvk529Uz3)|KY|P9R%fBA{Ho?f~%k`U#@2A8snYzEz
z{9aFfykz>`uj>6bPG0WF%wkh=$y|QVZ`&Pq&q=Hf9hEKaQL5ix9kse`Ykc>V_TQ89
z>pEAg`4GNt?WTs`6+i8)_k0QVy6Jc08H+>OgVI9>>z==vlKAzP=h|(tPu7e5y!!Cp
zn)+gs`;#_B`@EPQ<p26fm*@V+o6Bx+Zm5gZxw2mS-;1yH!TwL)-Y(DEBDwPWnr)hA
zOO1`0e=cZODX``0^qIBl_S<c5zV)rNd+gUP$n5f3#q(PGi>>cIUd{g6z5TUQ|GPUc
zC)fpbIILu1=+P_T$;joo8^M)2?}^dW<nqfq%Bq?D9!YYzPh(zSc_iF&YxRjp85PCr
z{BmC&Ijp&pve=#Nf}*&?E{FJqYLiwPO*^tkaPbPxU8Wy9Sk3e9y?&N0J@>^s?e*(a
z-l{fl<q-!J6Z%3-6$-ys-`^+5xL{uGOu?C&Ii>T~eN?HB(3539X|*oX_-vOugHqd-
z&j&v(K9pcNJ<ut<;e=PpvgKS0$~GLBaJ=E)mcI|a-E6beyrksSZP<`!*g4s)$BpaA
zgIhBuq+gIYV#BtjQ!!QPl)B>_6N}_#iA@t2isuJybji=Fk4bTBX+62MW>?XUrro@A
z1R9t3)Vy3&^<7}ENYq4`+=j%KABGM~gPY$zT)<Ep&v0WN4_88U(7A=%<BseV%g^^s
z3G!c3^Zj{N$=1rZ$DZ8IU)6u_prFGu2epZp?qqD<GQ+QQ|1rr44t|dp9Q)B_cG=EF
zdes)ixo_{w>^hYCw54w2y*Zg-%U|A-(u=B%v)a7rkeB3xq!6(g&fypRmTp&gtQ|G)
zV@mAlb*7VjJg)k&c`X0h?!H|;yFC1QS(sp*`s+=b_o{zA{Mf<Q^YYxHGObEx_2itO
zxi1r{e{7lWcDu0UQ+ds_V4;Ppv-c{l+y3v2U%luBQHIoCUzN`HoH{dimcuHQn>*6g
z&h>h6PwsU2-d8Ct*lgJ468P1ntonFz4pSzx`pla`OaZC$=bR}>77>k%ed9kVSHf1<
zTX5H;ptcwVKW}k8O=rENgGrh}Xa1OO$?RM(ZRPoj-BTPx|9o4bbNl@9rH{B=dXGEG
zeacytbY|AfV}%D!y}k6ip_NBS%h+>?Z}0S}7lTu?%9}jc7`|OP_*O#v=~c~tnRP9x
ze>V55cbR(1Xx}fzcD2m-3ngbJNXC3=6VyI0ulac8q)f$?aaK>H8m8Ti3GU|(tux4E
zIBzUfDxvBbv5rwg`>@vgZtar_+n&^ZQMoyJIm4p`r?_@B>g>FGVS|OBB*V%y)_ctr
zVLM}|U7l^Z-sz0_8NsHjE;HLCR{l=T3DTc6%Oh^rihEyO52mucTeoU{T(tY<UrdYE
zYZboqVc1n~V<IoTnCYwMkBimc|9?ANcr80_cj2B|)4MlL_MF_h>tfoI5A#G%Y+dl`
zP@dexgK3NUe+j%gS(_EJBR%Q_-^qiG6T0Fve<(ft5X|Z!`1XrT`B#0REvs+&$nH|=
zI_E30id!Z4?)<Fezw2CNGBo|K=a*f3w{Bb6y7r4#H(1Kf?3b*Tc;G3d!6eqx$>#TH
z$;AVmzZGMS9@{+YNx%im2hMNnxz^qj=Z^Ch@%Wgl%NbKBAA2R`X2B)%5>v^G%Pxg~
zk`47b6s2|H;i9+}h9`k{Pc8IWw^M*IXZnE{cTcAv%k+ImAIL^5`<1fkX-3ig{C<^4
z6TUCXNl&f_EL@{^CQ~xqHp}@2pXE~L;~|oh5`CAR@Coz|k?!)VTYKeFc8O$N%~HJw
zsWqw2j1&G|^;^!TVA>%OX8rig6y-M-7iK!H>owHk6VlqQ#w|JFIq%&!&s^U{T-dpL
znF&Li;Ub3!&gpCuvP~zg4d7rfnltDAfww8l2lU_DPwrS+Z8){YJCxzhEfemFe}e0{
zO&Rw0w|8%n-D-IG(k96(>=TwP_xcyLYnIL}lYs2>=cfansphIpnO1ndNbByZGYdJ>
z9bMv!zdt(oA>Cf~oqgK<U(KJRzdnAu`ZsHN=LQW8?f1)e?Voq9>%4HSi)^XIYSmvH
z7x{i#2&{FT``b?cOjJ+$#~_x@8jteKm|e_IW=Abrz2$bhf;pG&c@18#TuH_hHAN<U
zZ3Yd^rBfyuC26fOj9KY!o?+elVCAnD#tf@wIGqaVEB#+#zT)!Bx63oM`F8o8-7B%~
ztnuEzT?P?)b8HXJGE14wq@e3_<kZK`ptQ|r&4g@^J@dTZv0$6hyM5o6e5n)<oA0t$
zxF_&?X{ENuHuae&FE>4zefi80296@ZUkA@D;hK?Y!sxV4t>0zNslpSI$x7dDJ=ozN
zvd=qhf=c6AX)W{lA|Ew+A4HxA6xQLJv~p5|Qo+lu(=WW<ajhNHg|EJO!)8nD{OKEc
z5;!+Z`<SHo>0#!jeF4v9jM-K{yx<m9@Thc>iYNDE$&8z4eO4Cq&rpkb8uvH!a&Y?@
z&X*UJT$nmK9^Tm$+S1}?!#qKoRpI38?Ea1F{^64v=co6aR!cf{l`HWaUwgwW#m?3D
zANSaKaPsNwSUr8G_p<D!xnE9iy*O!;z>lQbN^QP~yxmvgE=Y##dV2Tp>^>D$x4%F4
z$*hf;I+cBC(V3eKn;lp1H6NZoGydOnlO1~de(rs|b}NsPsg3r7JG@zYaus5;x8=!A
ze67Yg_1h)J^sPI?YM-d3CJG!~6ZvU#;KlBi;+gCYWecXeEp;}2%c*<*aqOa%C;X>=
z6s_P1letoqYoxrHX*s`YD%<I)o1&Y||L^{F_P^k>dl!%Me>(Z_;_?1X=l!Ici<iBc
z!z(9}<oDlP{#@?-Ixi!m9VT<Wboq6z=V&$!`p#?i$L!1$rII)KCua6nY{@h&aH?G@
zWl{OJ>U8>sgsU$K4@6xQou%CWN`v>>PXoJRRikSkRKM)K&M2|?Pus$?TWkVMj(M+j
zu{FN)?RMZ6v;C(Nm%1ySDmql;oSAs3sb;4LugFpp!NVobI5ZgR+G?fJ_G~(O>0VW$
zYuf1uHt;yLkgzJ_1kNxvkIW37>1UU`&aw*rG0k{!Wb^)@&s$C;9PE4l_gnpzPqRF(
zmF)DJ<g(_4hNn={1_puV(oYK>a?HDXuH)U>jb>L4@mwegFU>u1s>f|YO51PSb3x}~
z+>OtCXP@(S)x+RaGa=sTUp1CV$FIqG(Lej(<|u(*b3<$7FWy}crxv}n(RRa^m~#K}
z`KNC2zT)Q+RPXihymzl_Yobq#|K^v6uRd7ZeA)kf|K@o=o_)VkHK{CY>6G6;?^T{T
zm+WR~^lHUqG4&&9uKj%s9{c9LSR5oYPgbb&X^Zdn$6`9CH2<dr_W3*RwRmzV^LyDH
zrENL-vOD^%Rl*-`zq9Q{aP`XrFD}MeO{keIAlm5qOQ$eA$d5<G*6#8lfnBE**mmdl
z?Jws)pkM!m|8Tqfv+C-XhcCW3`0~eG8_RtMZz@cEY~2{uvh&~j7j<W!r*J-0Wq&#4
zN0&jwmA(t2F9NMso;hR589b@4K`^-XRS#c6&)M(uc;>jTJN?u(=Nzla&3OyH<hmc(
zaa$~Q?U97Qh}Eo)41IbsS(oOlypq1*(Go$!ShF03&}n=5FU&po+Q#vZ7=Pu1l7zwo
zj~X;~P5Qfv({1~{GXiJ4l$Lc~J$)_ScI{k-2oG1i$Jw%=MauEA%*~hM*cKFP&-9w9
z;^XDqZxVVYW}a*(YlHgkt*p*Zj(!l;%JcJ8X_%$;_}%7tHcDrG3_~>;k~UmAcVMlG
z?kU4{-zPlYx6@d0)r9T|*-{g&x0GxB%nMMLWWRap@ar#s?oD=7NlM!ALitKZ0L#|o
z&#hXnf7)LxmD15s3EF+MWr5a5)s}THw3mNAen+;Q%}+x3%Qr_wB~{~zIol$IR^-2}
zHTiSX|9E76N&53YVSoQ!o4Wk_Iex2teye`tbt_kIY@E{7KF3#-Z!yO-mPs|g&7>#D
zoeqrDlXt(+_3JHzLPCjk-cHNZcq#sz($8CuhAfDd6Eoh)DpR*8H`GC5@6x@hE;C*%
z&XZeuw8UKJvu~!Ms@gG!ph;7zmZuzBB5kdme^2`E&B_q&_TT;cZ`<emY5#R~zRJ)4
z${GJsUSIQ^y|Ql2**p9fBN+wMdFLNrWe}m~T9F!}E4ZK_balKSn|IBZ=?ostrdOZM
zQ8yJlknQW-;lkl2o-@b9@7FQIh^bp$dr~5HsU=-+dw4Q^xyqJPTa~JKiaG6s7G~G0
z#jaxUES9ZU#`84O|1Qs6)^~rBYVvNc3t;A`oc8*J=_^lesU>HBo%NrpQX*=2h2?b5
z8K0Hj-RJCbWdoP4t=ieVr|15uA6_vBPXFj)H;j;;yW#=|E3=D+#DmRG1G-Ms{O{2U
zUA@n}xtpuxL-f;#4Ab9JiaA&wb+DRd9WIZ(H{0w{TzA-}+b^fTnPO(fy>jk8yYhLT
z7PM=e=lOm&&rE1;ky~GDiZ*}$!A*B26~xcH)zA`Ca_RE<N{jiDqHL>`4m71Msi^Lj
zd%kC{>k-bqRvTCw=6yL*TH4K-pYQ4=)VcL$+Ds3POb>hCj~|{q(yv=%SGD|F&DxLo
z*AM^Cdi!$qs`}{Dn;9ogiENy4Q8<+;Vbz7R%1t`e%UsuQzuC~CB=`QCVvMk}Q;5~u
zAi+iFwFG}DraWtj5naJDTkOx#!^t<^*@j&5DEe*3l6z?H(p<&`e_uLZSXcI$!{JPd
z%#}GWxEdN&T<u+>W+a}u`Srd3zQ1Yb9csV$eRw=SM^^SplxpLwRVJR-^HP@P^>_F)
z?bx&T>f7EI;VV8woV#fnyz=$p?HM;8T(C*y+#kdfcH~A#*XJ2)mxg`uEjXnlx76pz
zg07W|8~uMBQ~l)kt4%6P@HUIW^~TEyzSf}C7EWSSa<<13tBr3zTgbLe;wW!R$^RRS
zn(XJ&OlG`Qd>GX7L8`JfCW5*A)#S%5kzXnw_AwiN6lC5wJ9}jz&$WWx45tEL7)^T!
zUh2F^F-w}!eH~B3GP7Ea`>7!uVul`)%uOC^R#t7;Ar#;I^!vr)wM7-x+yN!ir(9xK
z=R1S5l3@wxCd7Fv^4E`@d$RI&_VU(FW<&q@i5If2eZFc|7Ml0R_s5zw)t^ivK1k@?
zn4_>@#dE2>&!SE`UD?;L`@!a20!$eXj<|<!a4t?32~=b0Ps**R)}H#<N2Nezn$DeK
zRX?d+^?w!!Y~fk?WA{;&pwz4CfsqFvp7(S6FY$I(Rf%%t*R<O5?+@~Lx5eq`J_-4}
zWy0&sdI|sQm+j=r7+w1NWG?MG%Qh*0wfgtl0bjTh^kpx!G~2bT&RE9m-1o6aB!u(g
zro%aZL+6&Q-+p{UMvs?N{$@tSwedDK`_7(nc0bTv5GuWDwtJ@?*VDQvpJcVmNwa3D
ze9Z`FUn}?BcU|cAy>s*4`0V?hzv=9+X7<=kJ6|nXes+tF=#t0wr**0gwu|g6X4AKf
zUT4$AAW-{tj*enr*!uYVpIev4C*Aq6*q-sj4|n?%i?cl^eH>4DXma=O6JlU+<Pl3}
z4)rm=-LoV2(ME<Vu6~P`b+j_H6^a<%?OGS0e<+H7=E>z=&#I=$vpg^rnp-O9TDkq=
zhhr)wH+M0HMooEaHQ(#Z)`jOT7apid+%23Y*y$?0N%vv;tW_PEnyL<bMi%{%X(tLr
z&Mr>aQ2XuFv}w-9lm4pqZ{_I%P5-awQEeA;xaP1hMAwrc#ck~aX7J*|j(tU{W;Qof
zCtClBzPD%3-IsxPSRV4s&JdWXXc+dh&?vHxnXf@YN764(UAl0+XlKRt%kC>T2K>{#
zs<rXKT9%z*`~Ecx_{=ehVYJk{oNzOxUEy8VntqLQa{s<8<zKgX`VpJvtxE6oR+W`C
zTP14VyQj5t^0}~6Gn1D;TH;u-I@%{Gw<AokD^>a8m8|JEbfTEoD7cAUPN_b0=Dx!J
z-9^C<=RRAT|2{vaT4dc6k6RiHD-DB08O}Z0Hf3_6&C-&@Mg4Ubxiuu+4OZ}9Y*#RF
zX;MFA`{mWWqOi}xO{%AldR@vszh%*8|GVsr1!2k_?s_q9{bh%HI!w0nv|e2NwM^I{
zYf{YYD=G|0&*Wwr8Gll^6uFSEDD-hv#p7$QdS_40-(R_cukCmL{-00oo>OO^`(=v7
zR3q1=z4!em7l_K1xn~?abcd_|>5J9ci|2Y3=XL${yX^ig-s)J@pT+XKUe@KBCe4bF
zshFfA-SuzEsgEwLeey-7JB3Asnx|$@>RNup57e#Qb8px2Vp#?b%OYQ{s$Ef;#ZnA`
z=UCc`a*Yn({i_nJ8M`gS<Hh?h%~MMbx-Fcwl8NP@&Zpf<OJ<xZl-Up?u9zNj#^ZZ$
z7gwCu<5>oNQxz?5Nq%Fz;<nx1`q|G3KXN|q?E7p~7Y*L8uAQ{V$Apbx<1$0z*@Bzo
z-FtTSIHY~*NL;n#veku_$qu&17aXj=q2g&2niglOpO6|+@_o*#khaMx>+OHOF?q{l
zG_|Zcmw`cjY2X^q8|!46O+W9}@;TAkzw6Gt3FiI#C)+iry6i66zSQtnh^JO#&CZHz
zi?6(F(&iCMT<meuLSWBcR!vFanI1{GMiZA5z4;(Eb(`QKrJK_XPl@iHnq{oIa!L2*
z#$P*bR`A?>`*iaBXOri*oo#2x%K!V`Z~y1@^X}}M?wIm!V`4}aPv*Vi+$D2LKA*5U
z=(pH?=@re3?&*IOZ8$~F>ReywzAtEAgJ5Lt)`e57ic_XW7fsY+KJbHU?X)=6`(;N&
z8v@>WTXxRmv5fZrouZ=@su=olhrazCs~h#NPrBFNsI7YcXZrmge-}&NuL|L|ul%2W
zc>dolpHH?kv|JGM4w(0C4xemd*!RUhSZ8lE+dWs*KK%9Z?YCr^&Njb&xVb_{{$~Be
zOIPCaeL`;J^`6)v!w@<1^8Ai8Hij>p;id(XvR#!Y7@FO$ddnL%Nn*_e<Jqb@r-iQP
zm$AubTPYT=b@J0ZbgA6ry4n+_2X&IkCuaT^(=cvhh+27c-?bLjzQvQ0HM=LgPFtlB
z>oG~6Yju5ih}rAcTmDQ4ikcen^JZzx>1*+^p~u(f7&16dD4kz5lX-8?{dJ(Nn}1%N
zkYbpSJ@4Sp0<B+@m5Y?bekqlA#Yh|d>So@Sae?8$9bPRD?dwxrWxUVZ)(ERt#<oqq
zX3LkY<DzZuHr2(+u)MhB@hyvloy)8(PH+8KTp-H)f@`%()2wK9#Z{eLj8|M99k_7%
z=+ge6Rq|i>`QApT2_=W9P1+K$B$VAj_}GfO=cYMF&CYepXfU{z7hpft=4p?FxYjl;
z`*SAAkE$XLI|~0_TKGl0>i_xg{QoZhym{}#!T;aa?~r`cvD$ro?dy|k*Dk(&zd~e=
zcdF8|tgH!A&i}7nwmZ3*LB>t+>Em6ME|Hc^)+)Nozr<*KXS{G`i-zR}^}lS9xd+2E
zrEHZRPH9dx@vRYU*!*vTgwDzBFHJAHN6mZNCfRv2c=8YbQyr{9-x5Q_?5nEY|M_$7
z<%+adyLXFk-!HxH?Yy3UA78A$|3B~lgoxE06Yt$*s?}r+NW12FD>*kUWRcABIgVm}
z674@luIm4`{pCBqb=@4c*V?V?Kr4v;C8#Djrg821wtUr*P#5F$HuK93ZjMuY_;)H?
zn!c`&&Ce-r!4@H2jbQr|^O#<g@vq;%X`yeohHBD=8rIr%p;0SNiTYMt7Zup?bgN-W
z|DyXxi^7d#)7I^rTx`j9n)knx9>d!bR<Y8Zy;_>7M+~mEtx567l$u++>z3r|J*)|f
zzA@-{u1N)Vshc)l?96I%Z7|f@;Ko_9dg;RDT37Ec{b=Z4#@I3^R6?g*d9k(f?syA{
zkH5Z#%`tI`TlzHD!j!G8f+?ZAJpIGQHqjkRisncjIoQ**$Vh2=@P|1I=1NbS_0ut0
z_*`(LUf4y~i@g@tje;{-18W_Hb#xMR?CW2p1*HXmPSREVtRhq;Qu$1z%V4tb&fjaU
zrp><Sm6R=>($h6%$rHb{2}`n1y3ak<KimA@!(XTBFI=zt^KZ8NpXbN!|GC@DU&r&n
zX5Y-MRu{AP+wc7OO#jBI#TBw{u0B&&9(H=RYkIb@v8PMm85PxM*DmjyGWm+imsdH0
zE1LALKMSnadD`+|`m}VnewlAY0=x~vtA2NtvR{<xVRpFDCpP2NzSnNkvzgV5`u_H2
zAMK6HeqQzI)!C|&<iEGx9=}^3|Nr-5^J{WUOO1D|RnzX|TErQv`YbQNpF8-{Qcvbh
zXU?Z7-F&-7pgFWLl8?7WJ!Seb4()5%@l{;{VQru@nX0QMs$W+U^AYtI3KQ1(G*3?O
z^5Go{98a~_kKd44`CI#u9<N7{EQ8XMGZIrJ=Vfn=-s0J;+W%?}?*qe#I<8AQS0DJi
ziD}FIi5n(KE@8I&T)h2z%N7OxJxA_|O_}`axM%YzHb%ZBd{Zv#{WMqAOzqvgsXb^>
z#<TR}HKDST&($u{Texr9N7Eiqt^b@uGu3W)d4|-|iAr8mG?|}@g)LngV0qlUxc&M|
z$<GE#*EuZrZ0A~fXzr|SS`0gPay@txn14%Cd~ti^kyB?a%MV&41c|AvD$wL?xV?9;
z%86%OpLwoz?U+4tldO^G_nSwLif=3|Fa1$BZNioCe@QBPnI3#<J`(QgHOZ{QP|z?%
zN%2i;Pm>NqNjK+HpSf1E6~xb~JFgK+Gvwt~%1_A_=eNFoIO^s7j3;*=FYn#|b9XQQ
zKefBnx6Zr2?7jNv{=TmU`~If<{q}fYXY-T5{rqZ5%i1>1d{uXZDSVsf%bizd%<yF1
zIqzF8mz!^?z3!66mO!>of%R`5uidFEkjAn(qv@lldmE!dq3FgillDu~gG6_lJujRo
zIBk_jujQ_-UsZk{J2m(I%KJa;m3}xgIBdWFcmI9i`K%Q}8#l~kd-u@G|Gvk`kUnMA
zPQjk7h3cFuZ*?ADWLfedww2MQz4pfCf6aGn+K&{S__x|2$x?Ef?_ci#{+!fJPOimr
z8ji-iD}FeOUhXh7=ljT};-x9f?$n`U9P@E%_urH6@-o96TkAH2MQ_`2o<UPm?@alg
znT8QtBIcOf2`rVo-DRXW^>NJX36pK(PHqk6GMf6>;sk@-C!UVlBVmELyF7%<raGP8
zob&f;__kBQNm{9~8&bOpBthl;o(i9-fj><|)&#s-WIpHCGJ%lG?ggvVCTSe__V&`X
zorhe1`peqi$u;!M`~5=8lPO!k>roh=!gaNz^y*I~f793*cE=y_Wac#04|bWcwAEKw
zq-~D!nn}zvl>ek#yD_cl2$j@f(0=sJ<8FV1Q;2WtzF$hc9N(?ap4yi2SF~zzw9lL=
zUf=sqrYQ1xv2{PuoauOEOLOG9X=}gKr6ifJ5WcY}^LWC|-h$*QxsSTbqHnbwpQ1Tg
zQ0~}r`COZSm46S<@85jy=knv%-+lNRp8xINgZ;mBc1<k{+w7Fay3A{$^U7+wsY!{)
zigG0;2?a6+O+7xd=kaDH11;T@EFNDA_DTU|(JLM6nLaGJzo7GqN?}0pg)R%@{}*_Z
znICL^ac#M|`S1Kc=i1G;Gngzock=MPkN<x;?*F~tXOq%=nKsv|jrKQhh>2!A?W^Zp
zlJo0^tY?+CL^t1V`^}G*^PlH^9mK|czF@~+p~IRhsy>N0=q743?qHZO_3_Ls(X|{|
zGYws0ObZ^lba_cC-Z0QBl6Cd5me&#AX*T5`yKstU>gwoIPWBN(r`a7!m=z11qE>WF
zb!}BWvuV5RqSDJwo|4BIy}2ce+$2v<NH4tP(XF8?^3&A9AiX_(^X{2{Co`BRRBn=+
zTl=cMd*U1Bq|S}gKuhR<h#DGn9OIZR&7z;g=J0aM?3qC=j{YKwGYnVFTy*TV=cJ5E
zHJ6nM3h!jOFRc31)9ErtrRUw<pi?TGOV<2in5gA+>QC9G!fz|s58UB>q@di)cKz{8
z!@wvBmL@LFxdQ4>4kcRbUvKQ0WE^Z1;m7OzBFptoX_QP=R6u3OiTwe}hrF+vZK?bJ
zz~UFzfxPtWWe1ZOO41k#oTB*B4AqpDWg7lB+<zkBqITy~Wq$tjwikx0ZaA%DZRpte
z<I7_0_kWgrJ$L*!|CfbU7uJ|oy}fty>+Dsf&rkk(+v6;CwR-E%!rym~yV@IH+I~{v
zm>Qo@TZ+W&j+JLRG-iYzI_@aM+7LLUT4{r;8JFSYBbkDlPpwpTi75K+kpHEVXRp`w
z_{X=pO<U*ZEHw|?9QJ(k))zMWcGeq($?x6#dGh|)o4<AL-@bg8|J(l>`Tt8Bo74Wi
z>^R5BG%IuAsm&UhORg#1VGL#unmbE3$%@bANXf~}i7MF#H$8MsiHM(D@%wLiX40ZV
ziT_Vpt-D$Gyz80!!8c8(c&t1bPA6PEV`vt8DS*RGN{8XFg_V{S!>+rBQ>qvFo-XUT
zwsYB}(05<Co)lhEU}jA}@h;nHrFfRgBMX6<jfPz(ivCNz|K`@R@y)sE6;{R`lV*RJ
zJkdhudRH8y0Jq)k49kU+TLSauuHs<aIGwNT@Mout-JLg=P5<~6T*QBNW(>G|$ve2k
zbor)Px2}EkYd$(d;@E?eRjH!uHpxz{essuK%~PoS{WPPpZA(3cI203JPMDg);LtWX
zihl#!yUc|e>JfPw2`goOx3=&lTe5XK2&oEcykuK_u;^&5sGX*<#LD02><q8RcN{#F
z8pHZke$M58+MLk^^CWL8F3;I<x;8|_Fs04eE6dd>wxaXnt+zRXzg;VZf4yT7R8n1A
zJ#CBjOkpW^9>wLav^@QHaCF)K`uS_JyR`hDvlspUJ$JhzdG}`JuV3Zg|GB2W|E_*_
zx4q@>=FNY%*L{3=?cn1@I%YhU*~@w~mZqIzIMS2qda{1Y<*F!;Hc4?IRs{jGtAAsp
znLU4%D9ITGrz|_+v-0~QorjTk{Ct%jhx1MP@Z#9^OVX?FzDP6Q#hEbY>$*MPyL<Wn
zr9D1*@9yK};YPK;D{8)~bN-8ebn7q+`&pIP%`OTZXP7lVzqzOuBc1K6lc_ezV#BQk
zDj(`t<=D??ZoI=8zcDAmr!Tn1wPUsS;+y-{J&ioF<-Nh2NL~ilBZmyl)V03&emZvJ
zOEi1i9*)XqQXKEzy|dsAOPi3LCpi0C&uf#0y!@6)$Av`8V!rpUR9z%n!N8ze_V8|5
z%<4H?3_P6zPU_8P@oRf(5cZVA!Y}qz(kkC0CKGPR-FRB)QT;k9@#fb`a7q7Xm4I;i
z^as7HTWwDF6fU#aV4$7!pyw6W+>=Z(d8yltXG!W+M+!Z6VO6-k^p>*cjSx$=)msiP
z%(GO!d28}=&p377FqtWAr+PTG#FZ)>o#WD5McH*cnU)`!H8uV8MsLAGmlFa*bR=p_
zoYlJGTMjBIepYIBWj}5x@AvEQZPnV;9;e8)k!Q<;rd`-uD*y6ZU%7R(ii(nJePo&_
z$9|EC1zAV7JUwP&lOt6mR^+yJlk$qY{{m~OD)MXmDi42qAmUK+R{G>>^TRK+LZA0;
zpVd<LV(Uz!Nhwz=q#a~b*o1t(JmwS(4{TWDV7)f|%VX)Oq8{9l`%;xWBhtE>&;9nl
z|L3TF-G_gd&Feo-KFiNH>4Wmd!lKu|POg1jeLXvJweh(shHIDHqrcm)uK)4*nzj7)
zUH0kf{<Ew21DYeAM8%vublcJRm(IS4Yj<w?zhyznMzw&aL5(KHPkW?9g=;h3GQaqv
zbN<qq4<0AC80>bu`|?rjofAgWHDY9vYpzP@h=XoGjI59F7MU=?{qP#eZMp6q(<)}?
zZo8@XbW+LBE3SUiML9W@#I!^&Zb<(cHFre@S8~XSSiU~t<lNThCEXVo9yPdxEGlAn
z`RR<0=d^S!UvZ;OhMddgqU@e0B)_hSZwVIJ_v@6u!*#FOdM``fZGRkQ9xwcmA)xdi
zXjKbnnq|L{NW+qoS;mPwPWeoH9VGd7@(0gJYwl%vXZp!|S^V`4*ri`qt)1uj|J(}|
zk>K2#*D3qO*_Sd*Qdw|r{r1VcNgK+#%c5Mi)UDd^x!l(H&Q`h2=BI>2cCoLTeDD{~
zH16<}xM`Cl8E)ikHZ)6B{BYDHVtU;Ko;!(_{Lh8rq+(c=Hcgm*si`}A!(AtrG_U2_
zMaLFNIY&ts8k<a>R=BidO8D1>&DHC+1O=VEaDE9#rtj2SZ{=evpR4!A|9g4!-i8zM
z47(5H+3sRg*!%UFqfnjL-JoUPBTKUFjb>_Q?U^#CeU71uVKT$H<ZbdN*EonIXhpSb
zNn4ub)6gm9p7l;9x9|7%uZL&FR4z5#VYuS|!Sc5CIjL*U#RcBie)V`w`M(!RRx@+<
ze=ooO{olf?e}Crvf6TFJ>Z*=RZn4l4Q_YU4b~5<Qk>|*knpY&oZoF6OtV%|z^tOUS
z5n|7#wXPFnp47m%GB`<h*RJrKUz#$dzphS9yqLG=>n&*>+obFl8dq70Qw(H7%fC)^
zmHGNQ=Yb20>8>>=+-B@?W6aqAYTf-h%eRj2?z5ua3EcaY7bb7;V12)M&d*;nHdSXH
zu{o7B&w6{lWy1vSNxxcU1Q)+McP1}M&($Nl;LB9;BMFiW;(L!>-g^Av&E;D8zEdNO
zB2FIwZQuTB7#PIIdsWVVDyQzTSz4l>nvN~@)6jKcQd#RH^R{-9SdixRyhTwBk1tG}
zZ|E~gCHlqQXF@gGD*Yy^JoGYBSbXuC?Y>Do9ZgC*9vJ(ql-Zr0*!og&&(nO{smWC~
z7JECY_n6+a-EREPYmx)w2cAIt@(aaZ_TQ~KdbW`5-Ir;rlsk88@vh@)RrzQg@xRN%
z>%NY<q89&@#hbP$1`0ksG=b&Fg(a;_6;|^ZE*(2_{V?~T2vx56Yd*ic&(Al1Lg-7C
zYMuvA64y)LmDyYN{!96O>#Dc+F81cXRgY$WR(8UgFYT@SryuVo^Ka)nXFl&sT<*N%
zN>6em?)y&qR;lAuxLwWVgtg*(nHjf_&-T9@oxkqRpYYq?|GnPuui4<&-9vXyHpTaC
zzIW~K`F$Vn-M%g_cmI<8{<>}Ep;=MWHhqdY?Wpreq0mY<{=8q8_NiT$3YRfY5<A6t
zBUf(1WND5CwhhsV8yIHaDCL=UKG`<D;pujX<KYUKKkiLF*lYi5^2A;GycL`LEF;g%
z|GCrb@Qwl#v&bWoIRSQaXFWgAb?36vMgM{<x7F8Q-F>IXH_KG8TFUor_#%t`sa~E7
zxOc_P`I2=owQl2vHkor<CO^NuP57z$-{@(x=S^>UmH9H?$eQD<@^2f{RR%>T6Lc4U
zTP+@W%t)h$Nn(Lsl{3RmKBu@RRtm0`$x5CQ{@2(Wo6fSHk<ek7qO>JvGI!ODnB98?
zSr;vO?x5NCY}<eBW|N(IZm!xlc~UF_msd%bt;kaK>gk+dTz-G^qL8W|oD2*rx4LW(
z|Mqc#w0Y>O-i@BJ{GCn_g5^_RUN*b4_L{`jn_}v<n~SAqY~}oS$8q+Let)}S{)=5t
zR!94kmF|u9dGzVx(vyC7uX(@XYBo*(@gy*E%Jx$(OpEVi=S+DgFa7%7lp@jTNjg&<
zk2I<+3B8%VI(c!=!c$v4ck1Mdi$wLCvs+BOJb8oI{P(@Lo{2wh-_HAOQ?~i1e^odB
zRVu$qS<4sYv2oX-`%*968GeapeSN3du-xKq&X@YY9F?P?4D#FGZ9P1-&MvF{`uQkp
zhHLEYfB*CfoY>`4v)INXot?jp|MRVXXJ0=qW(ek$n96?U-5#&*Pa2*#+ucuCu9SSZ
zYKn1$J-3zG#z`JRDpG6{ScUA>BMz-(71|hb^y2a0b4eyzorlEr75{Y2Fzx97SLAx9
zb&?gwi8FiEggjn8d8?Q%c*woTrcJQC>8#M|)hl;&TUsZjsP!|iO7e^oJ-dACRTcY5
z;q4W>9`ROKEHF}j@kqD(hso=2Zbx>uxeBqr*=Shs;?*X@+1BSwE<a02R_V1^aB<eT
zttYk$Ii<`wI;rx6d8ta+iB|rd9^d*{CQcLJWYFDx`0b*ZY_%Qvt<3={QHiU4WV_-|
z`)}x}@7tKNVcJLSLuJ>SPAxYSX2{`GT#>&zSbt~5j|ZGymaNLLB4^)!3#%!4dY?;D
zc<UxHtA*j&>kkC>>2IypTc^TY8Nyy*eN*D3ScAoh6{mS^^^3JvufM@>b!ndA!(!ue
zhi#Ll8$_5KFHF{cq?i0?N&YLP@})ugf_@F5nT{`(X|8l((!AZjB1a-ga*fumtfj_#
zr^wHI=(ol13|Hr!m3D&D+Lq)Vy=BpBqxaG(JdZnUm*4ueA`FWr7ry?O_UZcTpYdC^
zzqjA}>Eo@tkC)%Kw%zloU;h3j`}q1#8BY%;?)qLb=c`G!>~h<)Q4jrg{9NC6WOezG
z)xA%y&R%?W_2cj`ji=iW<$wDexI=i}*0N>UcPG|eIq&{>jd|6F{=@C^ew&U@{-XbM
zc|SwUUi-U7-)FC!r^UKhXH{6xx1~BozN|lsV-_o~OPFnKX?MPKsn1Ef-YRMJlSxj~
zTC6`WpIO#^WvcM67q8r=G42Ryerg=?<-Pav^$ei7oQ0P+#YeLSc!#9@2=UyX&U}#L
z-o2vf=RNn^<V#ruUfZZ+{M5JEPWyH5%P$-Ze6tj-7o=Wty5-iAy2I@Jk!OecX0^<a
zOy6<KN9w(<P;B10y!Kt8lX$w9w{Cf*?I&C~VQG<18t(>s!^YC*VvgHa7F(}qUb_BP
z!1Aq=Yo^JI=vPdb{xMn-v|jOi_pXH0x(lxjZnQJp<KG$JCAe!7PuQWyce~|`7Bd*A
zO);8xS?IyUKNCA7h1ZF{=`D-)n*4fGhW(QnuOtE#)z2|_eVmYe;?v8eQvUhRrK>-E
z5xd23NvVr@t#0e&xMMAm@6)2=(hHqsIz*1=c^*&Dbz7%e7{051&qNbH)$@~1UpU@<
zbSdY8iM#AKDd$&dxqHo8^6-r7!6Tbrtd5c9RdI}76#w$Tiwla$O2QqI6ErTRr7Bel
z`LBQ7)5RNn&S;{GnCg<&PUb0xa?Q4{J$3!!v6h6LW`f5U?hCvQHl61*tz%~9k_@Nz
znkzxe%sdk8*t~x|{P{DoR%VUM?yVbkT1Wp+l;f?E_!9Q(bw!WqAN{7%H_NIcZ8GwE
zuHF|t+I#i0`s|NOX0N$_Y|R$mrL78!bIWD^$Sz~f=xH)4oRq>eagv8nvT&1T_)L!^
zH&;;`;f}N@?RWBJ^Upu-$xObJv{8@$ncl~UX-8INY^doe)od=kx%0Dh@7u%fCKu<$
zF}x1e<jC8#Zn55*r#cLtmW~w)^CXfO6w|$CO21WLHj7v;EvI(-MF09_pRQP$3Y4+y
zOyjaVZOR_9ea4Xtfg=+)?SJtnX?xL|ptnm~7PKT4Zg016jFM<-Pjz?FQ=0aSv3s)H
zc5~_9t$W`5pZsZ(-{R|ACTvV=aA-2sHJ;}%@2ZgLq4GTWwc3Vve|(kwpZe!>uWIww
zp65rM&a<;BO!*z0TC`U0tFygFz0uZNI$3icE>c=p@Ip)I<1yVLuCF5RS{EHlWO&M6
zx~_N1PlG<UO|ERbNzJ>{)`c9(oy$3K<&;$%r+wLZ;ucOgf6BtdEw}%!z?`JbXI}_f
z^thf7X6N!(E$^Q&z0b>B?h)s<$=hG(O78gGHq9j_)+?T|V4WDloEsU1i<jQ(kcqTp
zK5+BERtuFKK3RFwmA1TC6}7RWa>-M-pF7RYhHMQw;^?%0<>rnvTblxX{}!wY3AWGi
z@1NCgCujG^;P!u&`G3B=c+7tDpL%-r;U|yoK5OpZ{@w28rT2Hv-s{}*s9*m1E4%tX
z5^k<hxp(`+<VDWz_gi;we%$_g?o&RYntRz<ZhGout-a**^Zuc&B|etQykZPSYnK`w
z*l}h{*ejNYUD}q~rcHA;Ha?GWJ8s6;@=jivQ%fc6r?buCt_?+DH7^Ud6boPJ+px{6
zIP3Gt^)bqPCOXrC!e*Cd#>TDVHEokWwt4@l#lJ$9ZtJ`8LyWU7U*=3!?@FJ>kKe8(
z?c{lOvrr(}mjCFZ22agQ)uoxMH1FR2d;XbK)7z`g%_g_x)4M@yJ0{;>Kga80+pA~U
z_sXuMx|c}CPX`U!+30fx-_s7Yiw!-#l96+RSG^AR5)Q@*r;QoB6pYk77Cyc9ET>UR
zO61<XS?%0U-IwUB$bX<?JDq3NOFIUm43DDQg=-9E+`aFsb^OQ##xMM$oUA$0y!GZ<
zDbjD3pDr;8c0C<nJb{a^y0akaSr<cghTXF5Pj2s%zH(Z~>Dh({st)X_tPTpP_MW#g
z6)lq+6s|m-5Vq2M5tH+>+TuVfoprAaba@nYgNi3VyL-E8CqIkPsW_8O9G07R2l6C&
z+*CiE<8ZT`cW+Zn$1KfBCx4tPvDvfV{8x^QX!}<GUH5IRa%zekUS5w`y`~{#nY@3|
z+80w}9<2X*ZT(lZi=LZju3Bj{g>i|ye(oZt%@@Bf%{21R*7g)W<;DGI=A4P4G6pK0
z$3?QrRVJ$}oU_%et!Q!cj$M{oA_YgAr_B6RZU1<FGe79m9!0xf-VEB0^tLLqIBad{
z+<UJx>yvM^hSIS=jvNc;PuSeis+b<EI{D;}D_<BUTwD=5dvjXDjIFjuYj&4kD3#rK
znl-nFv%c;o^S!Ti(~@sRYjXcTe)emQ@VfSv{mdJBW@*n|S-5SQlxAw0#lv$aZ+3F*
z4o+P&|F_Y;eV}o;3cnQ%s*^VhGtBjqby%n3)&E|mL16PS1*Z+AF_$yHKXH6G#ZysV
zCabdV=!&;%S0~6%;C?nU^=`cIb!WX@j`Jnf*haoOd}_<$(l`xiStEu8M&{WUcpRo`
zI>pU$HwfGHGGxse!+a;%Pb>jFoJB4&$&*bsOg(3*FE{<x4r_b2MeEblA{sXw&Hivu
z<DvR@t<;GsxeiaGSeh6$uK1|6e){;VPwV^5HJ>hYw&*;N6n^6E*RQm@XWRLUpEkNE
zEKYhlORR14&E~F{2c9BR#nkvpqE5tUJH?)za<$&-?>Y7LX8)}+|CJU!O*ilCJGFJw
z@|Lcd=eBzB>T5rrxhzC~<#p}8|J^UWZ-w5kSd^lW;<%MB#K5CNOIIZ2;W^dkhXjfq
zE>Jn^KS|+9Phatwpx@4C7&J6@?b>@WFF?J2((~&=VO{L|xm2!QJm<SzdY!DIsL(IF
zh=v09Gn(tJ)HJTWZ@wWWaj)ZEj;DdOt7|)MX1XoRb<FPEz9(_gvUz9oE?Q)UTzMy=
za%R<~aL>cPYfDcs7G%BRDfo1?yIJRYO7NSi;H>TZ>so8l7#3$Wx%vpd%W+;=bA6Wa
zYVW?6M*Gabkr!t%)Ah`Sv)6MzAKY}4we@Jkm#*V~?YuTWYZ6<$apn6g-fH<d`AW~`
z$hxGZr8P4!`2MN+S@U*l%<Shy-&ZtldHCVa&BWrqYx{P}WLfW*VK~Bnac$)He&c3~
zOq<6Cmo)_Cx*g~ilGOh$5)*Pepzgv!(^nUKVm~lAc)g#<aATusA~%ysW@5)~CYMPW
z4o4d0jxae)+EVLiwf5k8!;*^A60WtJve{1+jP`Dom{OL;w)ugr1!pPa27}Xu-l_rT
zwiZ_V8P8ssRcE&A+Uz5Za<)c=3p`z1-%Ovv7NGmrKsP1IL+BvGs;OG@C91>1rX2eB
z;%TDFDXt5<=ZZdBy@J(ChC`+CYSh#VTn_7|nzcEI9=zrx8NtDQjPb3Zl7hPKt3NYt
z-3dM_yjrNlsHXImAn5LCl>=6wb1c#{zBjh$yr0|>d+I{K*@P3V8r7-ijxdNQZa?ti
z$DACgNpC&#)~R_;5<8Vt^6t&12byR9E^GO+IJp10_gm?j-xcC6se#M--SSd66<Zek
z=1Jk+X3sKr>9i1Se}(DtEC!)%QSpb<{C*3pekIWmmgShVv1ffaXlwR7(XL-pPyKu4
zrtZaCcKGw&hYS67FBR$*Pr2`7eo!UR=HAs$J$AESH=JsfdFtG8{G#i+P!3<4iVNjk
z6_<(w1Zr6ouDf|jetIPteNXS{j-)2n4d;Kl98fc2Xqof!U*XHII?vb+G6Y#JV~<vO
zR5^8O!`q@82RK!^ivLcsl&z`>tFk$~TImk&tCcdJzlEKW+`1!j$`<QKTG!PCboe3`
zW<>ZOW6{#Ju${bQNh|C8iQRGUjulK`Z~3UfqA4lduq}79>_yY&u+K+XHuKDyJZ<vi
z=Shjmp3`HJE*9lCPH|o6GbuxpThlUq)znMQe#=~Unmw|}Y!WKockz$$3+;qs8B)I|
zguTnOji3FpUYV~U%h>W}J;Pd&BT*L(<&R`cUBdai=Vxe3VZb)UMSh2u9O;_u7P;?3
zj{5iY9oJvne!stZOY!Hguh+gk?Ec<XvO!fN(S<4DRcf;Ag9mSQn}W}Vq#kRrS$OUA
zI@8NZ;wfHV7TYX6{o2qoXxAg1U8Yw~DC(Z-Yustpz+qXuf0neAV5{3=1^4WoB1@W+
zR42zy$*THP+|SImI#uNW?-9e*|9s-k>^mL8rF;HXK=a$p-r^C*uIqvZuFmrmN&4Or
zSCTPME4j6|sLuJcvC)^=tFF|~Qad^K>bV?&m0c@?j`<7z%6qA|Y38O-AxjZ<&Xw`b
z=O(GF>^xvA&dBglz$$p<jR$#KMKz~ZXILs%WW}XvZrrJtrgCmU|B-Ju-*{-MA9(au
zcezrM&xdbf!L<yVDk=>(&F61w4bxNhJkifHv&=fWrS4C`f+ZWa%%9?_AmreA^07np
zvUnTMEmLFy8WVPk*(o*pUi_8I``F<|n9b)!KPpwP)fg(rFWFRM&Tjs2iMPqy#j7V6
z=Xy1IPkP}P6E%C+ojaSuJZ4vi6<v_ps(tC>new+!Kc0E|f6{_jC4=i;6UCOD;bha%
z^;yW1sPoH}%`h#rg|WEoL*b<cosNT_c1}yr+)!ixJ^k&Ce1(pKRqszGEP1#8>n&*m
zp#vQE6}>i@y<fu6(BVJHY0?S(>z{&V^NUNbQFfkgHn(?8QLaaQX81a1y_nDL>3vU|
z^b)PA&XsY?b-Et5Vc2%xIKgq-)#}DspIc%k1Ut*5gt*R_btCsmq`ZFH>zRxB%v@gl
zFL+hio*=n@??#r0(>rX`PF-Q$tQL`^mAdWUtIJ!DZ{Rz(d7H$k3&Ot^g)_}P^JEUM
zQ=G}=16SXNSl!<3G0ElBr3l5O1=~a!igJ0Sd2K$asXkt<vi@?d<=Y7L@~nf|@1L37
zek#G8Y8oMWaJPc1O@oh$$+@yEjmaDK{3@1M{37>+tI!JlOC1-MYjhm^G+FA4!8w;n
zGiPdxik;cZX4tWN)f4u>qQ)jMZy~MPNAK3heSf`b-u10A9}hp?zU6!Ok5lD*3QuO<
zKF_uBiGRjzo*SYPN%yqx=PT{x*jy*N;?rWCMinPLW1F&b4;YzECModExc)Kio1@a7
zj{I(Bw<%M98=q|~p0~#(tluKxT>9&fqEn*N_|iY??tQrM?RutNOa)ttKfnEcpLK)M
zyC7Y~2GN=ah7qY3I*-qI@vZM@kEqxb-V)7276M1?rq$n%dR@jZ+ss;^sd|^oNw4T^
z?BO{RxZRTkGksmuOojK9RWr+(mR>SfO;o$pcf{-IjgD=Tx9@IUs?(uznx}|&wS4c}
z1M3$EIc>^2xpiZ;Q+4X*w~Y)2TNjyR%}{P%W>B=~j8f+Ip7oxfQc$mdSAuY;*-1`T
zX1RB5&(AJCu(eI#&~|Qdo5=XLJxkxX`!MGo3VO%3<%ZN5Rj$b@j3TA0I4w)f7pDkk
zDa~BPbfIC}pI{mG_GftthfltH=a+j;RgB@1)6}z8E;gG>^yTjz>WT<m*ZV(V^^Y%S
zI1HwqGmbc97-3?3{ne8ExJQ~lbSC<$JYw^E@zQ&0^Pat;nwe+j@XgJT_Ks;(dD0xg
zaLVrA>DZ+%i82Nb7A($xrWLC$t)C%oP<f%Np`+of;<?4zx$%`N<2O!yyU%cq{lkJu
z`Q2WTenGo_#7{0-;J5tgl`o<)pLyI4zMUQ1|MFAmI<o?|{kjX>GLoN7ZkYb1c2U9A
zS#u2YbFGZ;d{x~eJjH&3w_uorPPyHWq&4?zKt1-xwemHux0+|Db#4rr?6Z_1<w$|t
z=8Uo^u6D&Klgctu=IqR#V{+_<%7b?w3XLS+zk7GVLo+FTUZm{73>O|#?&~_+kE9)K
zXjMF}Q~&(#R}WhO&8hweZ#LZ8^?SSWr0Qq8WbUS_ZOc=ZysCOc$Cq1}-9~?=-=n<e
z=bUdhDSb?hNILtl^s8lkW9px2;KMgB1?;QYdVMy7)ACrk_t%7;-Fk3iV_Idj3Y+)J
zO;6fQBJ@1=x?G;>-u-FGf=tDwJ!{|Y*rpZ2{9yvS!>`l7O&L7n-k0&eRK7M#@&m&h
zW4-fRH%jTS+pTh3a7xovBkaiC-XqIRrqpXM*K*o(CHO*l`hv_;{6e)~7YI!8E^=ES
z`mBFJr|P<%%bTtJxxRnC=%aOivw`;{dnXx{W{C!c7Y+ue_09I=9N+nzqt&&?$bQrF
zds}AD2$i)`v`Jf`T58eep}eS-qdjpR!{_ThciQf%R$aAI-g@K1ac<lGb*KC%&X}7p
zIeFEdP|o+F8?rKf#qRm~q2uKigLQMwLPd7@+b-99IQ@po^N;;?9y?y`RcCgO5xkIC
z!LVT3hBFDerq4ghv=!{Akr3Yyy4NHx(Pzh(?bc^g_C7R>D0xt!b^PB;^`$B?sion2
zwp^Ypl=}3PPn&D`)LxhVjh@UZ(<Q&8Mu@On=?P@H!jb8^MnrvyYseuBVbEN9czUwU
zY_<zGllJeJ<8`q;!#TENZCw7MsjImEw*=1reLUir{qqQ>HK{RDhxUtiY+y^+5&L3`
z_0&|S0txX#g*VLpA1cC^-_5!9W9G!E&PTJS9I_41yLWThmb-iUye@i7f1&N2lcBK9
zMvOtFvi6qao$SE9{Bt6kJN|CH*{}Ucr&ayzxe%`^qh0#ai(}%`8F=RiY$}YnaQD0M
z)F4sCxN8ENlA2t9I#w}VEZcqb$j<H6d=aWAK74WDtlj-{{uy8HDKi`{$k(ixn5|{3
z6YJ?2^C&`9{zwJCP5ZhH2a?b71-k0Ho(`x={g!CfTv9eOLOS$doF7wbr5aDD+-fJ6
zP>GdZKaR}a|9{pGb)lAvg2#CNbG{D^<+E@!{1v<AarRFBD;iSQs&v1eTlCySb<Gur
zKgS-ey?zgLQ|qU1zu)is`)_yt-fwNp(ZQ<zsto5QUpn*GCN2Br8;;%2-Z9L|{F$;+
zq13z~aFxurH}BrfxzA9#Z&AnOw!gJgbqfOHyp?=UFS$4C<*R~n(PJ#C-=!@qr_S+M
z<8j32>*+3D?WhHro6J}RiY7|R8oskhyL^aY<rE`6i)X?&uh}1(6!9%3<{&$RTYo)Y
z!m2YL?=_#?RDABbDWu)-+{nxE>^7O^t3J%%ulP?gW6(XnOzen8YRAD-r4FxFyGr{{
zu{ro=ZS@~@PbcMPHS11#&Jqh$d9-^EV}lWQ$@CuPiE-EDj;~e{SvPP0o7c-Pc^p}n
zdlR%4@y^Veq=m<<ue+@b;t=ipaf$1Lc>mEq3s%d^Bu{4ho0|K74zn0T!Lu(8or{+-
z{Meq)ojPSdr`Ob7M!QrlW*$75W9xZKZY}F_+oJV5SaL$QEloUoM&pfh_q69T#Ftl}
zo3r$P%DSoNtAGB;oX7TAy{1d>^NddW-&g-$p2In7*?P;4L|%QS<vTw8otD$_(|s9J
zfJXTI7dZzkPCe_tBhjkj`SixspqMCEj`hDDdM9q^E6cO{cacAS|Gh;n3$49#SNR1m
zHCVZL)6a{SlM0y?Dyp;p-I}lZZt}?;w{7>_n6GlX*z7yYk~!9GhoAB-p5A(xu{g3?
zew9$qV$0dNYE8G4Cd`$;IQ8b+G%MBxSJq53;4_+(cvCju-R(-T&jnG2?nz%2{i-F;
zI5RY#-hArxkGK7x6=j>9GFXcwSs7N?u`ncUkPucp|KjHS?8B2wqgEU<wOQ_XgUj@;
z{kkVRji!G37AA6MN%iEbIh+iI2bSFms4MtaC^wn?fycHK3(rqgYuocK9RDM5ZRJbZ
zU7sxE4($H_tAG2H&0eo(cZFA<kX%{*^6~84GEK|xN`;xrEuS|${WNK#Uaj_>&nH#c
zPbOThKDPRl+M1WsXB=NIRb1&T$bQRKea|+h&g)-U8#PjcT=I5KQCd|qm4Tta|B2ol
zq0|1)TWyMsbZ#v;pC7pIe0$Q2=j$^5o>M!U*T*Ln-w+dVA?n~W0mgz{kxK21<Vh-u
zhq;f(>bBX=dv$3E|5b@eTJ_g<p0GNa&UQ|i`+V94Ew45ABLo(KR(dm4yg%8XcE$X^
z?%%16uVPQVmpAm;s%ZTEj$G0DgaD3%M$G?zOyD`~uXd^`xxAdc!DdyunmXTAt{bPX
zM#ZY-xH}26_gq?YF2O8m`VWcVX#$_OD_<`=wDm?8XjpL3q*;kEr%m}QPtLNI6l>UW
zAtkJ*^3|1<n_{jAU2V&9wp{t!<!z+1PHSq-T+m*RLeB`lXvP)m^O;^{yx+lF+E%-(
zU21WW>{3_F$OZnsSDB6OJkCkzIpt%w`I)y=i;&E+P8S`P)rHTb#CM%*QxrEnx9pVu
z=|%oa@80veCCL!5T<XA<=DUkxtvG&MeSgjNw3VsK3|lGIJz=}w1$WEbR%(8Fea`$Z
z-rKKEzm~A`yioa8rFWOaPX5yh`sAp%!gkT(^)Ej5@BDP&ruJ4*PRl?4l)h=Q@(XiJ
zDOhE4s;J`h&gA2a-OVWi-4-4>|GXzWn!UjJXf0Rqn~*2proFnSZLoUkF_s5&f^PWB
zPMUVJdUm2)=7m+Kz7>>d|BH@o3_hY_*<5O8Z1Si<O_skj<6Ex2$NgurdaNBCzH4u`
zM<i@u$nIuc5U})n%ir#07nv-NFDu$3eMQ{bcJ~jVrQr;7R?Ssg;qT|G!X_Q8y4>sF
zFF}jhZCf<{u71H4bW|i@U2)$AUOmq?*Yu{{eJeuGm0f6Fs{43};3u8yHcuF~%$>o@
z;G}oL?@Xv`OU<>QrgfU3KA8<Am&~862zs|_x=hRUd+NGNbRPt0w9Z!2U3B+cN6nMR
z4ZHo$e-XSsy}(e|H~C`D->aA7#CqixscgJr<octu8<f9P9aUS3G8o>pJ#RR5E&l4K
z9NFv3PNk^%?{Tc0q%!9!gGQ*?@eRG%hb9=m;^DZRUs`uUZ`roYY^~!{D(Cng&xm$(
zOIUs5ZO@}P?uO3?FD-kj&Tc&QQryM8aU9QjSl(JxZ`&Y0angnbUGoY=*|d%4F>e*u
zk9B=v_nIX_xpcluf_m-bqO#?OJsO_Qb5q=JlDw3mMaE0<$%(Hq+S%`SPtTclJd?A3
z>!o|ML>W0=|A@LZP0iC<H_f3XL)W}eb7!k$-KDZwk>4-&^sKK@ofh4B+I!oj8(N23
znv=>5ZTE{l`yiuZBz;7ox%3=MgX5Xlsqx#*f2Hnszv*BvnN>JxMaC_o;Lj6Queu@}
zGi_<=JPB@wq}&rNE2Dkxw0@XV*0R5H$=7Sy`4jY+bNbV#@0_P)8T5VS*%fb&e~5az
zV{=CUU+~|U>G#&It82|;+Os!w<I!6QvJCF7>Pd=!YgWp5&%E7yey;R1`6sttaWb4b
z5#nX7(<Bvpns-i!o7cXd95W9tnqqP$Mxuqo>0Aw)w^!zA&h!mNd^ZgAc)zXrW70S^
z(KUTO(*~7kF|$l-ctKk#3Oy%qhMBN6>@6v{nC+52|5M6ep5F{gV$b5X&+K5l$hYb6
z76ZlgpB&!hwjN}fprWHKqVPgbhvl?s+tbI(*2r&RZD6#G+`V_IMdlin=7Yzh+Jp`F
zJqqV!>~N7OdiL|(1`Cgl(hrhrjxI>DWeQy%pRc=WjS}1ce=i<fKMgvZbDFl|62J3%
z_PU<*TCZ(vFj<wez2c5@Y=@Eh4WH#9cl<m~pJhK~ski6(v`>89OfmlgZ?ML6CUNd`
znm56-<vz=V8>aDU{Mm620wd3f&b_(HFyxGs$86WEj}wfmUSFE>FKMci>++suxmzzu
z%(JxUT>a+Ut1SjIz4l*ycjT5pYN&?r8LoXTwtHM+G-m~HHE`_w#((Zla4;hWi#K<&
z?F-F{BVUq4A1?ivY8YX+OQ}1s>7VIDpQ$y!|CY<%=#tnfupnd~hvgCzy8_8$t*=?$
z=RaDKFB9ndaF*@vpn&v)3=I-GQJ0i*p8b3EYWnHDLWT{yPSl?9+Rl1^&BCS+8KHug
zXI?zlXJK48i6z9=c~i>-?v+aan~x*}PStq8vqhz$H0HH)%Io#}HigEe#w5%;ILlh+
z^pD(LNDku>`>Guax<Iw}ng3FS?M;a>0-J>luN;}%yJY2AkpREPF@Z6<Q9YA>Pnj~w
z=1ykX3BypMq!4=x^#iB2D)l=usPjv$t+qM&dez5}kD~UL6`#*-lQwCe^K(mYXwixE
z+U(>h%%LIb5s?bl4>_9OyBBoy3wK%7vX`g4PQ`VZO@1^lx<yxv$5Qk`aE$6y3B!m@
z>3Jvojw?PXm?^$pf}wlSjM&W|=Ckls+`Q$Sc5;<azThea50^P&g(2$_ChR#U+f@|M
zxnYWb&(736YXu}D*9jNvgjcRo@pRCg`F47$-F>%5s~meAIz#&QUwx;*Imue~jHKa|
z_{fNDTE?cc<c^tscRnL8|L||sAvq;6j%Paz(+ghxda|QnNq4jUyCXjyKg<8}@6Wg2
z?<etaAMm)V?&5l@%*xgM)xK;$uMpo}<8W&y`-5k#Z|~Xbdb2or%KM&#o?AKb_fC7r
zU2QL%m_1MBfBm<AHhz;{Tk*5*m!3H_I@ImXiKt!Dv3D7EIW{S9I<#;E-Psr<;(1`d
z`tQn|{`t@MemryH^=8}i-zqL1+Ncw`Dd+XcNvS6%>7I<zoj9eV^~j}&oedEkQyDI1
zl!TX;ehrGgCoLEnx${_U{hS@@`EF^ZiQY?G7M)vPe=~3*7?|(-J@<Xh@zbTEC9W;o
zgp_vwn;P^u$nw#;n7dbFB))#RV5Ih1LU39Pf66L$9hSp}%k^5`cOJdTcW{m45^l#w
z%Q!j@Zj!Lt#4Wiy=H~Snoi}^-i7~7wzj*4}>$qzS(S7q<_WpR+ZQHO_e0ta0wAIb!
z0@r=Nh&DJWtl*j55Gdf6zanf&7(=A^jR3|uGqoL8e-v`Eo;bBr^3-y+z_2w_Bs>qv
zrZez9@X+vRR(;MS$?Ln})g|?Oj$4<$A951lRhsucNuakPZ>OVE^|#t*yJhWXONcP}
z?wC*@(R*vB?*7b}Th2utA5u8f=Q>vXx8O}nn!9`b?jxdKgC`4hPApw`a?Y&Wo5fqi
z6#c#n9(XrX|LrE$uoI$MQ#V~_t}OZ$$Ce#oeZHV8&*_fS#0C6Dq9Qw@`PM)GdqFvs
zpHp|DR1e=#Q>N4-$Ltb&j(IfNb@*s~eUxLFVe(?prxc5M&mJ3}Jl)JudU5$BMmzh`
ziM9PpE))f3uHyab&*G5xsZgPYJ#GTeYQ4WQho1d9sK3RC>CNrtE$ggJoQ-rAh4bI9
zQ~xXRJIZk7Hp9d!rMbs-dM>sq^oE94xjJ}Dvh}t`lwO}58NGZL!`~21SBZD8uevT2
za9BBoHOMb?<!n_RgOlIqcW(T;Y-WDPJiRF!{Pv&a6)pJ7wY^m>G9qmy>ycHb<Nkz&
zy=OjnD($9G#65*oldsGzd-^SAQ`EYS+8HaKRpo8tQ_x(<#=2m6v*yXjAA5hi>j%Z+
z<i+dV|K&LOJyuu{VI#LZibJw3?t#A2&J|M>|7&SAE#AYUDP6ReDe>eVM;5sZA=&ms
ziRG)rgpXY^xg?nWxcZ>SFNe~|=Bc5AQKi=vKJ{D-StPOCZRM0-nK#O>y)(R!cJsd1
z!NStz)p9pg4;Xpc{FGDQe$QjaxkZ0Y{OFk)@ud9JEA2z<-|lWzch2%Yv~sVb%T<4d
zkD-xaf$DD8SW`3@PXq|9RaF%-J`xdhNHlHdCmo*49k&*BJ={FUM5VoNX5yppWw$1T
zbU*R+D}LHDOKH*RTy^m_xx4EkCZD^#<M+q5RDq`KQ#0NrRkt?g)}A%3<ykx7W1OQz
z+dPrP8xqr3Uw4&Xd|{P9$^|XM=#Z27!R;Pr!eYO0H~RQYyjE}_+PK+OTUfHFYr=1V
z>r3OOxl5+?I)BjLaw$L4RZ%c}r<-lcUA~4-ZP#UIX`WuZa5*E(Lz4v^YghX3YCp2d
zy2Iwz+a1AD+C|qlxE-0Iv_#8EAZz1g-D&2GjIsAzXTEJ;_22F9$u|2qt}x}2FP_sc
zc$N5kwpCOyeq9%#v^Y>xQ_!<{#&(O_ijvn9l<t{id%e)O(|-1laPo|UOb3qCM+kqh
zT(;yyY)<vZyRy{`+4^cO{?hsZ(z8#NEDj5J&M@Vp(bmdk*0#6VZ)_2)@bh;qD68wA
z!!%dSEmcg}G14i*yx@^mQS`CR(ZBWHt#I<scj?Y-E}vn$dRNFI4~?@`CIU+)ygS&F
zVXBjSg#G4Gn?%`t%FkTh$<|K#^<dw@+4EoPoa!%$);+y=&5@+%+2U6X#XL<#e`Pwc
zmArR97G;^Qrm{dWWzwr%>WL+a3@kUZ7@T(Xh_?q9%~&ePDA2}iu<VVT*THRn)=1BN
z@9KBht}~G{ovnn!TX64@D<Y1G?`F-(Vm@H%U!I@YvNyvvoxgH^$ieo7z8618Wb8c}
zx?y_8{0l;hHEwh<G6WrC$SS<=6mj!Ycf-1FyHuW~Q#v*Ilyp^vHXl9EUA8VH%ih(2
zJ?8o;U%^kVFD6CgDLXw|w{Pho9i?>|>#R+RA8)<B+4}Iem+3!N_SvlrnmJ=)OoXAp
zwkW4vzqICln|o)=Gu6DO=dw?n+^e_E@&2s8y4RE5MkF#i#HQq*Vr~lR@{a$hkhxH&
z|C7s#<=lFIHicR!pV?&?wbI0}WGy>`>E`n>-Q11l9Xv1ZwEvb=z4XdSsQGG(%D#B9
zFf%jZt45713ZAdt{pfplw7l{S<C?uQ1cHNHqL^>wZDV`r=pD~{_1cD=_V<OS-qhO^
zwo;FCN#a(=$4olA-Li9cR4Xv7^-T(xvQYbYnX<QF-nljJ)=z5YGmMaUZMA5tWSGIy
zV`@jY7|v?H<#OCsl<C7^A<g+h#y&qUzPNMLCUL!!!VBgivHr=O`5&H{pGzn#`Qo{)
zuw6HENrA5b+eFFK4Q*dU8Jl*gPYy}XTfua5)5Xq>{*{{q7~G=!ygnV1xVrm8_<?u)
zVfT6G)faD-eailF!i~~1QhA#D7%yj^yf~%w)E4<?yCk%pu6h*Q&XH(!N>%8^#9#jt
zDsEpsmpN@+!`ul150^Lvl$@E<RbsZJ<yxJhLsHiq`87w+gj@ce8uZ#{o2x<hz3D#k
z5<V(w9n25A?!<Py&)V?g(*m7`+iwY+zH5J0@>{Hp^Q~`pTD|;uC*DeXZOxFVv4T}t
zd<}!ZE>GU#hBfjwe+#lUvK^RyVa|-ZmOKodGv!=34o|3i+<ZbngqicqRHtiZ?fbG{
z%3b{tlDkjw&M!x`jc(6E*!HPwdS|Q`D{2*J_E1;5ed+ro3)5x|T^7%rnx42)3D@eI
zjH`<;Tim!Em!CgvT@;6I^U~!h908x_OniL5`~9^W%n{XwikoI$`(|Yr!TVzBO5cbB
zN4`4NrEoFcn!WVIm)}(p;WKVN%y2%W7BrP_H%r==NlOyNua!)hxA#nQ>0-`TN^Nh$
z9M7lidz;94;M+{!xS5-`GYXhqUtzgX)a!B<=kj>=hLr({Cw4KO<C$?{$<MZqS=o<&
z9-PO;dFgMVgpP$;L}%Y->j^uowq-g-WQ1}YnmI$m#Qd?PNP_1}1uezNA<idqZZ>$V
zQT(upOZSwD(eJ5~4<9b9UvOo{)MLe$0xm~xopJtYuSv50>C%I)PL?NJ6?(*foIa$n
zF!k=GrHjAwzda^2VcI?EcV%C^FYI|fEq+b6UFW42#?P9HS5EOz6>uxKRw`AzeY^Gb
zKNn+aBVt`PzUu3~ll>?wyz^<Cy2R1w`&}pA{y&zz{`}IlmT@+_L#HTOn!04o{_5s6
z(=+Y6!b<BL`-*kFrOAhL!lpa=y<@rWxi*wTi2G=p;k{s^u-K&{w+{){MdeG){kD5X
zx@6e5qV$>*k_#*Zdhdl?bM;x7QkKO~6W?>@(H#FQ29c!<7sA=Dajn)B=L%v_csbQH
z`$o69H#n=wteV<8ula*Q!_j*>3_+0-CtEkTwB?C4%sIRACr_jLr^QM=N2gw@i+Hqq
z!L_zqF;`fp1bvxQ_vEsT>Bi5lMw7oNPhM@Hl_sFQc)iDur!EKVRy1ok^Y64xxgB{^
zSm)77@A*#ymYhHFlzn^inF|N62G(9K$WL3x8RkBJNjb{`-^=A^ByN5>@X3ntKC{P)
zd!M>XR-eewVm=ncGke8S&!i=(n;90on`odNoW1k*b2&zpWhY*GtV?^|qnNJiE*r2l
zI8?lM-I+_>+iv@3bm}=JAFJ&QzCYbk;ezA_lNn_T<m9Iw@koC5aALr<^Q$g&onR@e
z{WbZ3u%ouuLZNAkq8Ot*8_vv|_Qqc`u={sV(w_<&^+GWdu^9)Gk|$2HJoiU?`t&~i
z8tq5Nve(P22Zw);zc(-T)Wz`B$}hW|jcePZOTW7H9jvwMxu_feYN9yP1)ZY#`{o>*
zcIIl+mgiB|@0u)n-+Je(B!j?C6|q?FIq$U-W2=t-Y<rP3&v4NN-&Ok?l}p>1ZzWw_
zs&}qbk;`#O>8V}*mv%B6TADuD5gJkBHu=mEp0jr*Zv0y0#<0z*THpT1vrbTZ<>!mc
zsZypHf$2OkFVE*ZjuAHESibTP`?STYy4Squ_u^Q&Wx|aGP7@>_ZgAf9dU*`Tf!e<>
zZYGv)QBN$N_u5gg|E1ZAo5sr(*e52IX<aNl?aw)P!*vZ!o3d<Wjg~dtJqwiVI*)E}
zn8vX5MN(Xq>!1F6e{QgxUVDD|;zgO;qn$D)H^12^ck_*0jFzhNgpz*2B6Wc|+A1zj
zZG_H9acySs73t&Ne8*ksq}$amB|Le@0-jxazAe$bGHHKNPRmv9VmrZU^_R+obI&sb
zhbh-eM0U=9o#wnPbjzeob?f5J{t6T6T%Byeno_T(7Ok-7X#F#Xh@=TKlq6Gol};~=
zj+irR#m-|qVX?m~4_eD`yt7*R)MBmu#1n6OmxQ<H?=ODVrTs0|=JTb+A1~|gop8I2
zuURN@j*?935|!`$k2tTkaj^wnTdtU)v%#;st($lA&P_gPnR2!7kCb^^nv^VHieT__
z-5?_?v*ms3o!4`BJpZ~Zk*#}ebL`X9Ia!*r>EAZE2urUG{&n%{qnAz!HU3dTr}<Nv
zme)-vbIzWkR4*AS9^gB_W<$zC2@gq+C2n80Fg}T^``{}IYMB%*WZb~e?X%&Rm)V=z
zzPGbmC$u`SJWLW<&!8y%kio$1@(WL+pz!9<>2(rqCT`Csu`XzO9`bo@ZG!Zzesc-7
z1zdX{srBR(<U3AJS>?T7EL}D><AEPfjlPi6xz<3wWI^AjazY<HS;R8`w>o^-T%7pi
zn(E$<I*P3KEoSa(I%~6u?X<*Oqlnf9i+KxkMW1ie$f$ibJGI1=?~p;ugrg2uY$JJ+
z9K@P;ygPZ3Z7)l@r=5yr@G;iHC9#k9NZg)wE1j9=VF=4|SA{mMbve>PowsY<HXMs;
zO_*`Ic)`>%qn-zUlHUo2z2wiZGmBc$E%`g$;N^+Un~(R1=w7g#vha!f^i7L4yA*U8
ztv&D|^4baOg~j#8ix#!U>?*Jkec`m){-%qfB7d2f?&-z;`@bC8*PYH*WIvsE*ZUI{
z-4#h$KY3c#F|L2_y7k88O#2L-3oTx!)*OAVU=-9A)Fn83*H&4E?)H;k^p-9&DHZ&g
zm?<Esw_u^XlSEsq!l}na+WZV{x4eSleNQs#USC>&cj3%AZpPWcC;sj{x#r+D;SKM1
zvqpP+92b+GzxLmoi7BP61<zDU1*d-el$5w!!F81>gZ|{_ug$+beRH(j6I5PGbj~wd
z8OX4rWy8%qlYBOOi`^;kquXocBAG=R4z~)!7!!_KH8?5sT)d*(GpTLMJ$`SSMrBEs
zNWr$m6NjGfVi4e2q?lAJQz+2J%8+H6-!5w+){#==eo4k?vEppm+m>6yg~PcGO$47>
z>MdR!!=ZSRMO%GoRrA9|8)KB6B6!6vW^UjSTe&~+B6Iqtjxr@*(UvvULQb1%0}TcK
zo;dKL?#XrAFUs;E;rc4YQGO|>w8Ar8-XA?9mObf(fyBWCjq{V<q*`^EwFUIVA6nZ~
z>mPAqs^|6Mn{K<^d}_o_JP`5z`kJY8q1OI~6L;qbua#F?9-vel&&(ik)<>)J21icn
z)|uxdljYf$Ukcd$%4Bz{$nsnLJj$!4Zh3xf>qS@Vb-RCsIjsJ;<K2Zbm-Y7+Jl=X;
zJaxzCKjE3F?|gqwkXauXabL`{eoM{F8H(Y1@5y965K_*~b^PYFmRD>Q!?yD|tFCGm
z<!Z1P89J`n)w-~C<>xf5Qol^TGhRKb)-Y{!{X0+H=vwJZ_q4V)sk*7}RqvQzHadQ#
z@j!p5`4qVZCx@EZ%Um9AxV!51#^Vc`%UwQloz>0Ux9fQ9Ue09xgtxACQ5-87KbLJ*
z-5|O0tz$*O%2LCw@K;Umf^3$#T=^!tEuh2q58K3r$8)@`TUrx6-f_;6ykk{+i{E>T
z+_g)Sw*<#%9+PCW*O@4nt-aI1`bn7OlKZ!G|F#F-)_oW3_InOjXe~?UQ(0AwqT2<3
z%J2Sv;dWHNP+IWPtZ9~VtJeiP$nhx&FF#dygdsxTbzS&94%r+1K86AUS6&%<KMu-{
ztGe@%b+1^$!QE@Biv$+xy{Ju9R1gwuW$2uE=jnuN!2z!X_=<|=JaYfH<uoI+e)((*
zQ<j}F>|6#ul3!munmQ@mNGXNmg~R!Ui$!%WUtPNO>3oi;rBV`JmpzV8^SpfL5ks`|
zf&RR0f_$Hkw|h;vcv;A=Z}tMw>s>+2UKTZ1cQCy6SFt~_yJ+{L3%Bo9y#0Dl^z-fI
z^Ji8VP7djgixWR&W!fOAvn%8k(;OwAlALOBFNPqu1{s%A8@<@CmEC$?b$R;vw%=7H
zQzr>bHj|jXg?Ym?ZRfOG|7KRQUY@nn;8uX+x?H}$uU;;n`(j30pT*TDt5_3KH@{h;
zkhwiDmSy3WKO2^mcvS^lKQp)d-09l2ziv(WSR`EZ`s?)RX4fW~R=erzyWH-6{|VHj
zoY*fl<z4=lZ6}@7ynJpo*lj45a0qOhqr@VxeA#**g)T42>3`an_Pw|#yjt(H@{7o#
z{N-h9OJDw;7vgm^^Feizx<F~;krdk#9%Wn&>T9-~|Cgh7xQl1<e<88;DpqQ#R-c+U
z84?zX^6n_f>6H8wdPU6OcTb;&ouTvVz->1c&c6IVLU&=blgDY(!maYtnvON)@3s|V
z;+^1cmHH@$tLH)BD$BF%DN;I0>m&ld8qcu4vL+;%nLRW`LOV6gN9c@{i{!rrEOTO~
zt@*1otz)9Fwd{n9WqNKx#d;qeEijFFb~5*0jHtL`a#e|^QN#)FHzEcuAtu5eJt_06
zPQ5$$NL|v4Th&o}^CfSEVm{^$kux3I$D+468Q;Fa#O+j}yMIQqX2zrmUc9&eWfX4n
zxuA8@XYa;U^Om)KY;uhItnQR_aqFvhT^B<G+vdI7Ke60X;dhj^8LyzxackB|L9MDc
z<bNMkRZUe%T~TV#F{Lzafjf`bHZ68ztyxb#tE6q$JoG00^mQKZcW>Tkp9rd6sdlJ(
z@65d|cW!%XoMgXzQ9W|f)SY|M><u|z{S}y&v15Ua!ugL1o4PIhcumhFCL4eI`Pug1
zamh9(Q(woqzkj{l%6)pdnc21Ks!pjrb(b&i7QgSik$uCrL%g=z)DG;6xj8p2Qq;6#
zfnLi_F_Sy{B^Rqqh<@?BeQDrz$5@UOQ==vO7DeZ*+hEV*>}4TlzVJwmy|(?KM-x|<
z&3vC!vumb==ZqY^o)hfdi`Iqm+3Ym?(yU=TRpG1X+eWX|ckFgd=Uc@1W1Xk+zG$Z#
zM#ul2a5-zb@As|d@ON{o1ybX}N)1Ie9M0h2Je{z!pP_-l?ecujt|WyQyKj74ta$8H
zQOeBxFvSxN5o$I{4r*Jx^Ab7YHyIy0Z&b~~Uevo=@rg_9oU#&4z3{_)JojgLY98|W
z;&#UGYU{G<<)OiuYd3$H-|FtXw|<(~6C>`^ttnfUo|v@ZrqsQNo;fV%R`(rRvqn(y
z=8p9_WkQmA=lE=MVyC4l`TQ4Nvc6+S(yy)ySN|!y|BABxc+_40?=qhKLdiF{x<8%M
z>lI(Ew^QYYs8GX{MFJhSHe3{DSaEgbx1DmGcQ#*N*7fay+V`Ja3$sMGtz%tu^rq1!
zrUM7Qvur=2nP{7}?qXuemy;D{+Fx_e>s@qT?Bt}?aqgFNo2r&dNlFA`!x5L9$)|b3
z6g?77bIPvFD4S!#?b{=CBggdozE6LChWF18@>;8F>Mp#d`eUpdsI7BnLV?ITrr8W)
z)tejlS1jDcvtYfbq;`OGc$R?RN>M3Oqw4aT52T#Kr?Ms#mN8D4BYz}7_-x$6lKg9}
z4xcu-7O^WZF&)bN_$I7g?8CX!?;4kE=*eNqjS;+BA-V0n>z7ZvF5FqNCbf^h*4^EZ
zSLgDxc2%#)tlXw!O#a=8oW@2e^~w_2dp^wjc7QLUL?V*wWY&g57n0{I<`qtsNXc8b
z(lkk6Qc=BSiD}Ii`3VM*_D0tGRvdGjv`9la>}1OPyBi;<pK$fR*mOE@<D|}oqW0Om
z3MzX~)r$E|n{e1y^V6(5RZa#sqf=@d8B*$2U0V`od@RYjy6DfTe<Cvlc*CYlYxJJD
zP}^$$;nb!WxAb&{$6Gd?x32m#MYZI7TtZJDbKrF?z83juucB1hy6;_C8NxIpa_z@u
z!j?<y)oM#0Z@q53Z2!kwuPbeza&;Ts$<S(xd7OWW<!xzuT9;8vV5wrl+6^nx7CqBA
z(VOG3KH|`??)2>y>lSU@bZ_CTYXTiBW#%YNw06y#^<-Xjl2zjgr6^{W!%`0VSwCk^
zwGn@EDUo4CPMqiE8K<62edoS7>e-qcr_#4MU8xbXQX-m9OmJqgocl{j&ZRVLx3&9;
zY!ksEFU_SJC++#soBH*~#^Z6e|K7A-pKn$8DkOh;--_ywv0~upzOXWIRqC&&OJBP6
z?pR$Lt{0NFduynE(Cu|jO-KDtpPgtaxz^lma{uONCl9-sk!ySVZXRTGP;`*kwJvJr
zlbI|LIz{S%dn{%$IaJNmi~V4p{_4_lBe(6Rd*!EbIi5OH$*^Iq4VR<GhPG$Bc}^d*
z4!Z8T@1P{-=guRKSmwXZ`09H&x8+dvg6rGP6k5&GX-(meSve)`o5Z#?+rG=)7FzyJ
z(bM$lzh4)Zw&gGNF%-CJQLVzZT52XwE=TUv6GD;vM|5OgTB+?YV^^tYx}2%LI=7^+
z_e!j!cACS4U3zUNG?(WW3yQv-c4C8%0;}gDLpMJ$1JCy8OPv!JpNpGyG4xohU>DQd
z%8VjL1)gNK&_oI28&a?Cw=Qu>dB?NmkJ8S)%(r%(cb_5?<5^vo(lSBim{*F&_Wx^Z
zbs5;Le*b*Dndh~?SM$`%e*&jGy|{?yxm04`?jru?c)#iw>EUYe35v01J)KKMoXi(L
zixU6rA~KPA+Tny$kt+&DzxSN|wo|U5Y3c48ig!<$o6dRN^+NUc@)KS5IrrB_@i_cS
z?Bp&gy`{dL<5*=($O-ipuEG<cr`2S5wKo=pSjo<~l&R<uu_b_GQsd|D<i4&cYYe<{
zw%lGm-*)?#E1#E7>$Cm)=<sp5sz<BmTNb{$A~*YMtUM^P+Z;cy+MxF3=gzlj+^ba?
zRxY*mxOr;H6j#5*_}MbSuKp8)nWqc<O4Q_O4bsY5SD2qBEp=oy|3wLJnG7>Y0d;><
zuV*X5@0^iXQ&Gn4aQePVS=UCtl(b(~X&+5<zBM+<&hO3HA<$(iv*B-i@e1v*yVLYn
z`gOeveS1^2Xv?CU)8@7(-^ET2ZpmpCH4$q%=A`7Dy!EQY%GN7oy=Ny1#&{(5O1MmV
zm88(sxAI7ze#aW72}?K*M`#A~BrQGDCm;Ga;dsB0v508opDBM|s&IWinE%kskmdUv
zzs)Hzld>eeev1j2Rhlaqo8Md=^HJ>OyOs0%yH~t&y59QCtb0SOxaIxn7sKzdr0X{&
zNq!QR;tLO-qB!@YifjCGYe$a7<xLype|oO!Hrk@J_RSgL=BM*`w>Jn*xDsXP$2+z4
zS7PVB$nq@4*?Ic{_H_jIT|WBgf3k|$MZYtCJDJwJ<~zIQ-F}$^Gt=eoynU*-XYqBh
zR1U#IeQZYqHota1S3f^Sh+&O6FXN^oEV}lpg`t6My7sn`rP}Pv!@~6VHk|976MHm8
zQ9y6;%U7E0g2lJ@7(O`jV%LoKFP#+H><;YgI^=RZu=s42^qj^djom#73*!_8uKBlw
zU0W;BJ&m<{d(qr|Q_hK9yTsBEY5Dbuw!i)FQ`7es+)tPL^+n#c?*4NBIaXV5|GeM*
z{)<M<tKj*+F59*3{jpi7)oXcW;qvUb(AmoZ-kc2y-}5Sz`)cNbAfMH*f2DY@j<YnK
zt93zaQP9c`gEeZ;oV=YoxGucg!an!gf|!?_il>=m7b_bJhp^B6mQYu<O>MQf{=1M>
zm!EpAzbqHB>hi`e#uEh_uZWsz?pnrspyXEEQsK8VHVMcEo6Y5TF{3Y@V^Z1fJ2!8>
zi_7?aUrB2#*Q>qn($cf;zd5MPes1fmyD#7VZPqNl;_N)7>6TCFt<@bF=?1b@u_1fC
zOjj+xx8Qp0fp1e@`sIgz3k~$NwoX!iaVCT-D?RIB!;NXI32HOr{A(?PH7llW3fKI8
zD$VN4xuw=_)3Seet$zDCgX5RD@7*uAiuKR#t4+Tz`n1mX{q9A3%62`Tcl9;jS)Drp
zH_Kfs?~A6IxNLd<>~!^Wy?w7`jm58@4P3uN_#405)lK=|#HEw0jgD^fT&{YlEd8o^
z@KmSuIlDM^XNaaQ59izMu)as{0_(1zlG_?yf>W(09xU0qIqTNt(2ZHQ<ZhkXGP&re
z^Aexk5id0^f4i_uclXpc8+lcp`bobCv&vM>3Saf-#`=<5Z@skI>-nzt*p)GE*3a6m
zohqohEJ%#OAwKo`7RJc$TAr(3$1Tmi_2ot@OG)gO-3R*eszoEk-|D4Yf8p2?C+X;0
z=DR!lmZNBF$gT|Lx0_a!`0mcuO1<dUv!=T&Nh|fDUi1UjEvMsJ<%BQmb-&rzWA-LF
z=~nFJb=)e;F1gO`GvHlRxAU^AwN|;!fz>6~PA}5Bbu)RTb9(76t!t+jMZZ)@y|}jH
z<*FNc{d3ww+%Ebp4G)VAsZJGr%Tty5IP}z6Bdxc&vQyrQx@zA(z@v3_rS`Jjmv7g&
za;A#qtyrM=qG_Ra`u~KihizQ;e0j!e^OqFcTU`C+#?>Gu_O9mrw~xhpYMwZ3sycex
z-*&Id>-Bq2+He1V;M>Py29Wfhe~Z5+y|+o56~52Qy72L>*?&$&@0+i8<<-l_;`hJu
zt^7SbzLvKkYHQTm1yOIy0`uNa*?TH0MC@*tf#dT-2?l!u#JI0s{q{4dpfvxNjntw|
z>!XDCzH?I5*}CfL%@n)2db!MRx5eKSke*sQH|nv6WA5}>qE)YhwhO=K`EzSaxZu)t
zIxK<{Y*Ve?_Br~N-Of$6(qOq7v^_{nHiT=gpDNGlrFkN6XQV6-b6DMQjp6d&vLJ`m
z-9M9-8fL7HN@ZAZ``wyjcWWJ2#%P|uzKq?`>gvj4*0&i>U3J<SrBSMF{ClsX2*X<o
z(^N$t_Eg11bLVanXgy#&ciYPuaXXW&au+09J&m)>-ePdMH>z%_&V(684d42jR&*Uz
zwoX5nTY4>>FLrajdR5`#;=;|9Nwuk4_usOaz2(~UTXW|g3fUU9wl2Hv+uk3SITL$+
zK8t>Pbhgr#pk>Rao0(lRH=DfT_tmKP8V%X&Vi+#m%h_A=YxQeyUkBmMx-C<JCS+?E
zMogS3xa(!Rs=2O*@z;n_efv%P6TEaUR(<-QQuq3ka`<G`NxWK$OI~K$bOfb6ejGAC
z`WC~5>VEa-P45z4T;KI+t!ll&RlDoa+3CUOTfJ6R+zk8whv&M6X!q}5e?z05>GI3Z
z&71%FqZz|A|2%m+tGzWvv*(qDU*gtgU;bj^wuKY&bzfd@TgvcuXIScDxBf40ZmR2h
zcb~8NEB)%>%g*I>3xlGv%YRmA<R5x}wEeA|R_!87!`0lbaTA*V<@HOaZH~C(&b_lY
zlh^*!JpH|A^w03U?+M>&vo<f~NQ#Dr>!DkfXa7tNi)uS#ocdZfc&Y4R6{CIz<Iqk?
z#ba$*C677Q9=n##+8}4G|6;Dfo7W2$N7sv;WMg=9=Bf4Bw`E4f{!jfD>~8<IQ!c*i
zol$Ja+hW^YZR&BNB7v89Uh&3tOj2WrI($9){gK<3A8mej>f?m`-LKAE_P=P>-tV{l
z{JHg|{+II@r>tL7GVRuv7w^;q!!NkbUZMR$w=br)W&T`KclMWscAS4EUcVK)Yl*m5
z_?l9G{SdF6-822Jd^(x?^V+3}CH231!sFjFeXuP|+bp^I-J_hH0w)Z<?0CmGbMo#@
zF%uQrTaU1q^u$zO{pVD0^wK5Gb*#_MBwSN)Jf2!v;>EYkZ@ya5I=xzD>2|K8P6|_k
z&TI{~^;Hz`TIv$KwI;ILR9u27A#~c)>$O%J40zZDqwHQeX)P5>?)ZE=W7ST#A5Pf?
zwQD~(WfyFaoxl3W7mf$j_kK+~z3kb#{a=-f6FWF#_kVk2ZS+lI&E6A>mX~k)8Wg{B
zn<uxl{`bqL8B7*5Yp*wYTeflA-qMY0r(CLPKcThY?#qw{t_x~Wt9y1_mYcGgRZ)xK
z6;rH3hu4}%3pN%!a9Diz+S{^d4$du^n<I|aMXU-t928z5${^ulFt_aV{h6;i`VJ~+
zG0Og4zv*7j?zeml>y@Hjbo~AM)p+8boMn%fK4+_{Hho!AZgqEVTDJDe%DFd$KQGGR
zVz^Wy?eIHhz34L8`s}|xEzN1aWz15QvhIGp9&D(8?!_{jjBD}!w~N;%WoK;_m|MTq
zvNSSkmh6{3kG6GGJC^_bTJp-{VNYJ(dNqrTuC*(_#bkdL^?l#noK|fr`J!(5Zs-2F
z@BB&<PgjUDNW5oS`>=kCE#IsaSC*B(ITE*|>d`6Lt!*z0em}BWcKGeo;Fl#kVm~c^
z?9^|^SHI|`_Ok7>UhgreJz8S@ZB@JPRdXk<x;qzUe-1x$kxlpRqu#rwx_7e@+rR(u
zou0Wc`pfo9UuUflU79)nV?ku~s!+BIY^P*JQkUO#I~Fz5j8S2|(r&kM#n(>8Kd)$S
ze;uc4_;y#gW;EBSp!heFF8KKzt$ov!vC2zw*Beg;mte<`7pkcl9Fc8)K1<((zBvDl
z@6d)@uF?%gFSZ5DcTOq2wJTtaoLSZC2-}<&DZ8F`$(et7_U7TMwtG)LKd>t@v0ZvC
zfS-N2Z}xAQBU7zst9w5>HTUcF%QLr^T#xrRPu=vh>)GiBd3Qab1F;KicT4k>%-7p+
z=jzno;&biYZ-<<^+I{c#b$y$QFW=4iy>~~MFXI7+?>{frPW`BNLGfaA$f;0cd+V$F
zS8)GJ?f(1DF>uA(9MkK&UFYlZGpsep&F9<po0;MH*Ogv7ciokDS!G^b>E`maH|sD%
zM@YV7*qKk~a`Lruz3;z1vNCh~?YZ9$tUp^N8u)6FK**1pD$y;s-|~6Rw$kYKJH7sV
ziR<w#)^8m{<<^8*H)pi(SiuzKX0}alUDqk4%H6`vs@Y#7Ogj(0+Gfq&yK_@RxZsgF
zj0?)&PYryvNQcERB6#k$DREnpeU`^^EP1)bcllYKY2ln~6BUKDZ{3t+xf<2;w&>>S
zkTAV1*;ln1gqMZ*ELswA&;IG!Z$3df?{`-`Yr0duL00{!kzka1d_}@jC+)ktE_~IJ
z+u)v2yZpFZ^Hy<&3zxG_Cp`b6dH4Q}8$KskRx{4^+5hy_b~Cmp#)?l5*X!obTXodR
zZ|m2di7Na`vdm|dI}aMU{hpV?J6YgU!Qzc~a-LZ|zYsH7LTA>ut4U{hSqv{#sV-Xi
zYUfhk;w#J#^3P3Rc-^&n-OK&ks`4_Mzo-`5Ta+%C$?Nle*Pj(Z%?A%%i+RswI`uAt
z-~NBRb`8HCY?x@vP`s$d%I@s%lH_G|34T+|eb^6#6(ugKOVHk_;J2(q;!(<}_q#vd
zl)L9}VcM69x4w37TfT+e+_TAcw~M>#mnjo%t6p0Mwv}#NuHnV+5WhU}hDXmj?v*?;
zashF>^x5-$*8BFnb8b_Lo?p%KiTg`Le@UR1me>036&k7aOKiN199f_2SYhKhb;*X@
z<?XBuhtC?-{(9k+-hL*5@kZ|{EjA}j;kHHVMJ;!=CeD!v7FViFKD?kh>c^gsGyQws
zy<VWFq~OsL6(7XVb!qwQQ~ciAam9wYYbwtEdQcD)vCO^R*tjUU^vYKApZd15o?rTB
zv9aKJ_4RjElcw?CuY2C;mHqtZ`hA6g;_)}G-+OZX)%(aB(^Gcus$p~3)c8wg60gX@
z-xIQ`vy($N^BXYy+I!{-|CzUw8CKZxiLLs4`EU1aueE-64Xl$pl_ZyU%}`0&kgzvB
z*4;!-GUq4vqbmaLel5Rd2CsIBl)QPvs#dDO@>QtW`X^#mPUidnT>5#&JoMKU#lY8V
zcxFoID6QDBe)D@yx#?@X#bkrF`5bp<&(XGQ+IP*{*Z4XIV~kB{R_XtzK9O&CSG1WF
z?0K`!G9Xjd?$g9<zIOI|Rfik3`_H`jnO>WI44lX&Y`g8cVry9S-$$>K64Q4}EZKXF
zPis?oUHsbzw-!Wv+_m~__*ARwdEA$xrmk+))0)~P;c_g^!ie`;P42F4FW;^*X@}X9
zI!z;Oi(l~wh97$oadhgdlJxZ3C)(Edg|9xe+U4jDrY(EBzG%%264%X*s>@t;x%YBe
z<cYqE+Ht`e%2Pi2$~sgoH;JFgdFj%v)*Zzgw$~eMl`3C<Yt_$Xe*RNkOZNo`+<tmA
zYW=Kh=^>}@doC`%`^m5K-^r-jh1O3?m!<7mlyvCaxo<V8$IL@bxVNz157D0M7_j!|
z$8FbVe(b%m%Od#pOZ(!a;{OR=8q5jt0a3lX*XA!VkE{M0Q(0|#&-8Ke-G!1?PE~ir
zr@VMP<>|#+e*5<XYc1Fs5W9vWf1$N#McjqCvtDsj&3VP)aHiz&O26enoO`#PTC;-T
z>Px<5yMHgg^|mTJO#7-+%(}2usg0XfOgYL}-Z*d3@s~SnuCDYc)w&$(+5gqAdX@IY
zJpb~BZFAmjdAaf4)aI*Yu?y}UOb}&ww(CaR(LE00Q}1a#Gd78SxmIb#qU@92O6xo<
zj>d#cO1pDaW7j30c-3V=;=8)NBDd^b;nRM%Z1)}4r9BV5s@_P&et*)Zziijxx%{Hx
ziw>!Ar(SGZu|#$X>m|OfO}F~bY!Ugp!shDAC38Q8?6$o+HFz%9T=`#{qh5+B8DE_m
z{I+bXl-Q!V8$d)A`_)y~um4_V{r08b(!0lE#NV1p-Svu1P+v8JZ<&?s{^-URQ^K>0
zts8WHO`N$tE=+80m!*Nc>*uSdg|-IX(=2@Xdf$SQt51s)Z|?b$RCITt<gOJjntne1
zvt_-tN%5<fI{NjGw_cAoeXqLQe7<$zx8D6VFHf57-4<kLv}1>5@T*qUoA-Lswb|2G
zhi<ujk8c%|ZtS}933++#Nm_RL-PVeKy-PRl5I$EG?D%SWbZ}~*c6Z)RQ-+k4nG3ep
zymyEUd0etwXI1LO&a$!&XCv7P(ZI=}>w_d+Zmka4<E3_($w6CKc#7+C?wdDnn!RX!
zIQ6ZW($<?9XQpWH)s0nsIW=HeM5f4vte4%~y*tmH`*w3{N_+rQz`7}?r^c;}e#!KC
zYuz5>7hBlt*1ibUN>%JQ$P^%Us3b-4#gttNk4}}`+7uR>{dP*L!CW(e3Q@)d$+zB?
zsjleSv|`)s+`Y}${Hj;qG2dmnSu}RS?{#s$X6dKyS3LUo;KkM7dh>41n|Rw;s9|^7
zYWvc>zkfcjpC8j{eX&?SZ_WHK>HJ%6zqR#yZLR%n6~}iaZMhspnJs1vYj{?DeJeb*
z)|cVrmuMm8$OT-v#vPx;d1imwd9~?_dgYmb3692YvC<ij9dd5|T6|#UDydf=W4brL
zG}_y}-;%lEgdCURCb7~dcO5nd?aDm5>U64m)MPV{V=EVKI=JxfR7Lf?lh!5u6TEh?
zPgD#KReVz8tYWq7l;bb==t<IH=hUpX8^o3RS1;IUI%l1xM0Q(l?b5B6@Aqau+El<J
zbto!_;c7pRz@?_wovgRb*h9q9wcj@+PLy4+@{7OMgr|+sAK728d$}t*vOBPI&S{(K
zv(e(~0^f0YzMA1^u~um6r`K|a49+pkbd=z_l-Zg%W626Pu4^5QDWQ>P6HaXr>)2Pc
z_HB3L2^sdFw`PKiPRz1&tGkx!(e_tEyU@2s=hO!IpE=AAO|GA=lD+0?e^t$hjU#Af
z_@%ftyEtsM9J*&^bL^dZmeW@6ZeL4%&1|;om&6YIwkw%je)P}LrJU2k9~VB}dOg@v
zC-UVa=CJtHeRexn1}(kBds*`J_KqVf7V$7z-WOOW?8W&yWPbMTb?h7T*k;;0Wu4fq
z=-2y~(e2A*c^8SbVlw+;6(^JmZvA6b%ewIu)2|OXY0B=p?k^2qsIx85tNj#Fc%Emk
za_Q{0<Q|_o-agam=S&@*qzemASZY00rfngE5u;da+Prtn3bS7Q*!OE+cRJtwy62yo
z%LP-n{&^6~z<0k+eGU7YPtD~QPaXgNx4-7!Q}w#P``G`T&HwY`{r`i@SH3DZfB&Ji
z>NkdKaS2{-3|S}YwtRgneDX{5t<}A3Uy>5t`F}Con!WajgeL3KKNUPmzkMytYMngX
z*tQ8D-C!%J`{CHeS6qtf-@bjaWjyjk^+k1)u3KWFn7rE@se8t}vz{D!(5`w<k&&q=
z>j78gKaoyG%P$8u9$mCS+P$=KSI3=KzxwQ6?&8|%)|b~G_ub4=PIb*@k?Z{4w@hxb
zbW80o5SX_0=fsJ|b1HL~3~oJAzxt{5{aJD5!0y=+Z(S{z)R-okc9lgPzp!q?%_PU{
z-3v0Fc03hz)Q;M)EbP{{4mr=yyc<Q7wX$uG_HnZ8Tyx**kbz&&A*-eUsk<9rHC<R@
znjv818Q9D7fa{@Pn_TU*x|FsTmowtdeN%p-_wD=P1z!_0Z1r1I()B;duB{F_b*9+V
zFnZo)k=8f27%#l7EG$#>>)l*b^h9rl`HYLo1}@xhF0eiLD_`|IrsK<(U*@jiIY0c~
z&fn?HxNx=j`Mk4j#SX=OQyNdd?N<4F{d&XOhqf2Xua_0~zX@U7w$k`-^Lx%s`f8Ha
zM(#o{s}HF?c01z{UAI@P`pHA41cB6W@z#mFRy^B2ms%hEENSMZ&bIZ&V%y`ZyzgI8
zxU?p8gR<wdPS?*4--UO|2Wv9FSR^t#i}`?$iiWO7M^{keY5sez65L`@J5RUg@1Hzh
zb;H*u>;Igayx!*T>+iN7*8ltKztbc8bV@JZ_ebXwKr1CqNUZT_z7l@&OEj19@%3NQ
zXHAIyy-mA{*Fj`G!$dh_g|x4trn45LzWR0MaE$J^M^`d(W8Q0D=hEOwW@z@feQRdN
zgd+;8F7Gk$^Nm#qi(Ry`E!gL3>X&D2);)c#GUAOB19w;FI3_pVJNi9kPS&i2m(JYg
z{l?L1tX?K=@JeUa?(86OrzpKd+pt(osgoBfaw=9UL?4|gv2MqyQiJrp6S?^wmB>1r
z7H7!%Xrpt~lu2#NDGv32dA)9ItJ;2a&U<?7q)*JoUhB7w8hUF3Ia10(rtuV4Y95Pr
z&Y1c!{mGM=l8x7FzbM=Du=c#mKK%RSj8!6~jtPdhXPmuu*x<=@^~`1Ai!U#lqU5BV
z{O`@t4Xgjwz6)HPq_Aq@so#B4b0w@|YZ<&Qrb;qKcvg4co1$vZ^hq|>Q(yIZ^83|y
z*UjlXSbK!?sqnO2?=GDAZC7&H;az?6ovGOm{3K5APLlI0pUca-VB^Z5)GsD{N%2>I
z?MPE_)>KaASkO{B_0-R{4Xm|auJY7pNG|YX7D|{nbw%<6Po8OVE?eUI<a2fXxEdu~
zR?f}VRLJ(q=n>mBQ(NUpYER-dCxsX6@7h(3innWMrO5}TPfDJBuv*E__w@S9o4>g)
znQBy<%k#|Y*ORaWzHP}2?fLsJi$?8~=$;-Ma5SM??&nSWA3NoLe*G=?<LmCahu{A{
zxP1QJ|Cij}l9x^$ofrRSeib-Ls5!Xs`1qdxzU_MU>U*aaa3%b0S$XE#t=Vfk^-Jz<
zHxD_$Bi8G_q(Ho><-;ro2^pqNhDgN;Wgb(@(hn+}3RyabafxSE>@Vfqf<r;EB7WxW
z-?hHmJz>&3{>y*2{KD=^r7pI`@16%1_s080OyFMLd$IK8=ND%po#GDHO!bc0IY+Gh
zc0&7|=h4r8R9(KJ>-pHCQi@@s;kO^AI&7zQ=ye-&a(z)gb-s<ECST6sbo}uQ`HbpW
zDvUAZyb~0cI32j4o~_?;X75}bhtHKhYfbKI>MFBZ>~u<6YmsZ-5?10FVb*P~aNECV
zfhwD}_r1)cl4tgXMP9m)p)_q$NN~Ku>6V4^KmQ!m6r8CVnUkOM=F$Pf?;g9Cmg{@v
z-`Eu3vDQ&#TB+pH4d0Z`ar*llxI3Y2Hs_NES1N>GbRL|m_b?{&a`+zWFZTq>r*`+*
zd2avq<;$+w?BX|0Y`ah;{rpgzG=sw3FOf++se7IMnH74LmOA+E-kB<VQ0-Mw^6yIi
z!$uL8Zkc2}50z>B%yIkCu`{iLf9r}>!Xo8FcX_I<sF{>`+A-yo(xT3!B&Or@@2anO
zzU8T4LoVBVsr#p=@c1lY-1H(~M(x@n9x<6dt?wM%+w}gvu~`1~&ENTU|2|#6^CmI9
zr}b`ke(@W|HRjb{w#*U?@Ot@=jbW9%j{Lrdt5`2K<T<=P=p(Y;MN9jjazXNJTmFle
zVk8XTd7hb&k|dz!;wF1BZApuR*~h68zZ%?|Ii?*nDiyrAW7!o3oprPBdX?ndcPhFp
zXYqY%McL`2^^a}aR-Vu1eBIeNMTuv#SXasO;?3<Qrt0h0&vEK@%bkAY)r-k*roX)>
ze!P;S{7(7#w>8cJol}!HFt9Y8-K4s}(r~(I^)`hDF(GD$tFFhA1#cXZn8(oZ?4n<q
z?_o9e%!8&`%Tu#{&e^T3lwG*E^OutT>RsyX%fgzz@$i@iNpRiEJ)N~&!r--P49}yi
z93kb^6Q#bzop}-yc}c)2t!fRAlcZN|P;vG-&RhQO@#~%hKb$9H=epCqFG=B-mF&0M
ziuP5DI}a8KIBeN;voTa^W7I{7xu;!%4f+(7w9=o=>y{}i7Y+B{|Anb?{r>mP=8O^Z
zuVfWGT6f3vSMK&VyDOG0_7B=Mp*+vPeJ{uFTVJNW(SEsa!~CtkMcm9hs$6X@#7JCa
zH9WXVZ1Vzved3Ila#K&l7~NbvnT^|FPTKy&?bH1f1)4o3D~d5F?a54Xds|heyfMZs
zcKP334yXMa8|s=%1ur%nHaPv#*urRvKyzyAyi#kqEC!teTBi!`A2kNK%P?Zirdw}c
zCv3`+N&dKb#*xgwU)KNGG~0H`tq*&D1owc}sCx7k<;w?O{=6xGMf1+``<Ei?wsFps
zxO!ddxRc?7jx#)x?Y|N=rz`2OEWG{Tro`OW?~Wgtx;L7=RpHS^<H>WY4!w+TJh~-t
z`oA`=Lk2otva8qKW@r)nb9UyFNzL4hYQHSjY+3MNk0OI6D`);vV^z~lj^CyI1dVbZ
zXl+z8ig>)2TQf26&@TCDyJoD<K6-bPZnDaZgP&|2D+RKio9FyVFG*U`&CE8DWx|XL
z>jLz`66UoXaJhHjvRNtH1c^lgFS1Wf<Fxmj`TlH9rmm)Hg!20|w)P#Tq|=yl<qy1h
zJLC1OkE}g>&I_mTFYXl8@H+J_)m^%+X=3CuXMa--QB@U3?PV{1O*|=}@;N1Js~O*_
zeVU!FH`l#c+uL?RV$ICJsIDNt4KogW-*m3LAn$X}w#>%P6`c?5o*vTA%dLF8^}2uj
z$2_I3ZT|Mfoq98cJ40`sO!_Ty>_x=AP0P7E4I8))Wft7GRbLXW;?L5grgF9<!K;kz
zk;#I(u#nV8)k?=zxBtmWzTfTkJL~g%!-A7(OQ*bVH903Rr|9<M3-hnM>QcH`sIX0_
z+3|^JQ)0R^L*c5Gyysphx_)EWb7$c-*OY#jZ(DC}hzW|<7Tdi3-TpZpD?Uo7uh!eh
z+fY_&`hC??_V-GD-J9C?|BK%Lhui+AzSZ1sM`GU|-LCkJ;o58)lLn8)+bffjjTs9b
ze3-ae?2rKux6zvX`7840GhE23o4YK@>BI@1%C9HRq^#+`!L;DAUFS9Si3{acCrY#}
zR0wSmm=vMiq;GOe^5zHIK=CNe$6t756#q>0=SndYciSgEY27v5%a3`F%{FJ2xVDLt
zLoxQww5K9Hd{W#c_6vSF_H;$Go~RNOihufy{pG$H!uvlr7XCjd(|>exmf{mZ|M_0J
zSEr}F<+xjD<HvlTJ2pgX^U=iJNlwhI4%b<K1qyS>9yw5<&0VMUdSXuQ>J_%ewoH=x
zu9Y&2)-$X%Rr?gt>=9kF%su4op=rEP@ud^)i@tubW=dB{_|+Q@9*-84HBI<;=7-km
zgF&Z``ZzPKnO|EtONv`fb5h7<wy@2rn$IobB6mtRo9ft~iP<V*ee|+oCa**O&RjD_
z`|M?wEH6Gr<h$>i@Zyx>l6Pf_GV4`K=BHj1)fGwIQTuD=+so%K?A+y2Rro0Q{r?Nr
z-9~R-G8O8UB%C>Yu2sX!@x;k<iB92>Pt%Ssd9LQ;SQcEEb?5EL)qBO#L#t~Ru}$o}
zyg)4bw(FGQ)}k*J%ClI1CZ6<rbwFTRQ$vpX{5MJa7W-d%VN$bK_V^aFD@I1z9ow?1
ze(j5B%yRv%>DIo?!c?zWv})I4|5w}ZHU;j>x_{I-#`T3<n|JU>sq)%)6Zf9YnB8i9
z|Mcl8wl(jTPM`Pd)ZhPq=KpP$|GoVG@A`jJ=l{R-m_JuFdH>bDKf;d59jFR+_n)nq
zZo;C#9evb*#o&V0%sTHS+rnpENws8X<lpEj(LJs6*20e(0?|hjRM|{4m&ATL$2on6
zu-8k`wWr+qm5j?)pZiz#x{cw;s@X3r!tPpxTHdvt&3-7bInk$C)I=<%ZnZ#%<B|$c
zL!e6c_{wu9^)>D=-ntsN^ZU_Gqgmd|ITk;QQaZfTK#5n(a;uel23O;Zlkd!>%+ima
z<M3Q%`TXknl!8P<fzpK>Q4<T9b)O$`+mj%vlXada>r#xk^TjGfhL$CtH4m?mG!_is
z>AvV@+p04DZFfHf+?%IckzgpCsBkIvqETSj)f)nvMcd|mExE8qn604C`N)LJix%iz
zoa482d)m48vyCGDo)Briv1+%;$~$v>mR_B(vms(>U4+w%o!Xo`jQgXf|NAkOf&1Fe
z`LC}Zh|-R8-dZRB|Ag-m@Y1hJ`{`o&y`NLe<Em@@#kX;t-{dy$Q}Y*&2ahH#(!5Z0
zc}LfL1C@1ASKnIBpTr!-6mVgEsetAr#tT*vkBq+?mn&#i)@dz$(RwjsS^%HY<kz~v
z94&^oPPi>f4r=~bx!~YI+18f4OH<tX_}U7y9)$#d^qrGd#F}Rm%i;EJ-pt6J2~pCk
zD<|mByE|#U-I-No5klRoISe_&cX~QZKeekxh5Nv*SpR36+OyBw3C_3MS^4hF%B9!k
ztN%Q#|6Bj(g8jcv`JeIj9~=LFnZ5tlhrRQ2-}jWr-ssME1vMV>zigQ`Y0+}t1)ckt
z7*05xx{<-K_wCms8Vawnokjcte{HA{HICpZR5EsU(w=p!aJf^##)UZy1(nt1f3{{_
z<2ZNYRaN9hr5R#vZeht@iBZWuZ2c7v)Yi*Je=+$q$w@$Jab>s(GvDC}eRqy#wJPu)
z5o=!cv8ZI#gyS>Vw9Ac?uj<b2U9eHWM9d>*{)q{jTM8~rec!1&KjgoVgp;JEaZ$(A
zh@%ca?$y|5F$LUWNVLz|RwEW`T5QJR^YBT}%1I%`k_uY|StcyeV0mD#9(7EdsVFxg
z@VWq7YR9RR($=F2X{J`j5z~4vuUT{JqjJ>xDetYkHk|*xA?9KwSAorzAff7$NeV)f
zRexA`?vieHO=Qe)5aSQoJ3}I(yK>sxoe?%&%#L5C+N#fF<Mx`UFVf(0yys%9ZYAH=
z!;SA_bWez1?Y$lsKjmRnaPIuNwhEo9pXQ?5p6A@Y_kUserPXm&CS{lJgv|7i-t1EA
z#rNx$OO|xNP0dM*1UgihX9ZOqm1#a|Z>6-9ZKC4+;uOuyw|PEce1%d!;#_S1P3Zi?
z^g2rX=H68OC0U|NMHFiNFYOLqHpz~)QTL^&=ks$5FYhy~5|BI=vGG-qk-1-QP}eNE
zs0>%0u)C>BKdYw)U0}Z%BK_s<gjKTJ<R)C(u|P|G-J{;Siw?<LI+bg-ui(cwYxcS4
z{_vW=+fsflzBaS^ZB51B-P7OwUVJF@dd;)+x;N!=RS)!QUdYyeUtjac{NL^Ty007A
zfA2NF(v+nCuR57OVeiW3S#xr7)-b=wU4Q=UwZj2RqWHJ`I(8{)wP)FR$G}grwG{^g
zL`~M54PBJ{Qq)rWcod7AN#<`K+4ae<_G;}_%0C=iI4AJNgz2GqS>1uXuYLquvT6JA
z?sc7`rI7vp@RNYp3x!HycXxg9w3A<c>ymDyXrkhYj$MvVE;eZ|J@qv2hHb~YpbZJ#
ztPwT8Rdz%ks=5->JuBNWxzQwI%Hk!-3MHl;JwcjxbK2z1gtPglMr_~-o3l>%j22r{
zhxT?0z2&!-X|N@&=oY$V_+O=6F_uH+pqfM0(xqpXb4Y#`Tp7l(X2qSa*#?q@uKp9J
zuGrc&K|yD#qWYU3ORl+_@@X8Oelf)Pf6#%<e&*<191Z)voO!{>ad?B?mp=VpQCH_q
z)KA}SRb=*t&3T@GRmq{vR=(Zq*aGKYTIS3+%`5q8>1+0vSt<;^*X_5;AB)(^qWj*>
zZs*Szn{2FVkM3stvbE~hQt=p>dd}}#KPrU&zV`C+nUJSS<#YR$mKzwV{4rk>q0ch&
zT&M10kKl|o>9gOrwmRJSBA;d0vGUPeF(yaTHkJJFQ$OY@bg$gi!SV6Jv@0byzb{$L
zbwSs@w13XxmalUv8d4${qUv5PdmLTI<zg1^|8hS=<n(!et{!4yPTRoqjA2hjS@xaj
z2cK?{u-v(1%J1hETcnonJ2o}?H2=M=|M$vA)ql&JqV@0McGc(Sohp*DvYt-VUvg|o
z=B>9Sx7hjqpY#8>+5hj}{r|q#J(2(S#Qxvx?|=T~&$Hb>iO+4p^SD2SN&E?WGo6_^
zoEKgZ`fF=ytIr+1k#RwMyz$y+*A3Tja^?DIm8M1<@eq8K<dfWUFQD<H$!RM?#{S4N
zmKS}O>iylREw#b)vyDLThL`i@CAE$&n*QX+DxV4O9_Z$&y)cTHa%|DQwB`(#jd_ke
zTvFVfeJ9zRI8NSZ{iStCQ}@f^AkNn%n+qe^;~p%(b6fOzWJYb@uSG(Pr-Xt!9b#Q9
z*;Wg1EM>aS{c@WN*Y}<yT(X7}MDv8t6n%9z3+za`Ao0EIy?Yz)inddQ9y55N7_AJS
zELfYkdGfjwbAww#{5TQ>cuQj?d8AzPw`~bvKA|v?Z&Rw`@2FIL^IL1(R1P&;sGN#b
zo0;Of=~A}^f0m<1|AO_QCuHMyoIKcBq}?+yg5j4{YjgHXUtwk8V3h@uGksjQO!_94
ze%jJdz%}i#VdkU<TB{AFJ$QUZp)h0}tH#7oo)>p_?e}JR&wJogbNNoMvSX_6T2GzU
znreG?uAyk>!>*u$^$rT&i;K70X<c8cm?R*%tF2al$>(dI9BV}1@aGCh78NY;a<<;O
zK&*w4UtI6fB9Ai+6G{zp?r4^5*8BUK@2rZ|yQDadlMEk^ZjLT}d3lDLRe6(FUBs%R
zoyw~hNxX6?mgv^5>p1vG=+NQq_ch~dXPECc$)5UIO4j_;A&qHL$5Y<<>DhdIu`t^1
zV$D}qhN(Mi#pl=Txt}M$=jUUCw4Voe^WFW`Y0Q6b>ayn3dB6Sz^VhuFGvin9{}0CY
z-{;poeE;XF^}oaZ|36IM|LbXSy5HNaTK|$E?HYH}-=I^jHf{)>@X$eeX~?cmHkPVl
zOL?RV1^k$e44Zdp>@d<<IGbmig_5zen#mOTHQgP{CT)BrwOCXA#N5PJPLqCh#h?A%
z8T{Bit#Dh<h0llAq)Ia?FgPYJ)N9)mEMB_ty!Gwd7drgS+L})=blsn5C@k>F<CMdM
zU4L&dX*kO_e)D<o>FDO@IWLP}uf4PC*qo4=aZXVc-ygh=ns~4_cJ+dd0?QA7Q+V_4
zarGjB42R%#lUFKhu06iXckk09p?qCIPpn@RMe%JGW!~_Pv%&YkN9CWJC-0i+#j$h3
z1im)@lUI|U*2v$9kz4w<tY+ej#7sqo=?lDqp0_p}4J}X-UTgi*X~I*h>?J#<O^CRi
zm2-?!tLv`Ld(Le|uQMcK#TC@Q@466@H6cs5ZQkmGMf#b2!7G=%pSvm`mc>}_Zn6B#
ze@mRW{v8h&f7?6t)8qWNY>Q16?Ms;YX6MZ2?yFY6R3f9#wrV8aH9N_-vUTF$Ad`tc
z8@_Cta_C#yIVr8^=a!PoyJqZ<@?n(`{Cgoma?QT5meo1@%!{WiYMODFm;YMpjBk={
zij|xCZ{PjMsd#S2QO=fK9S2u&7wk3ux%2r4Ltcd^ybmYkZjnsX2^XE!e7fL;hPcGj
zd8Zbth9&tPiMI54_RhWH-l^p8=J(3hR=s`NTD~`&LuTLKzNzi+ZEJqdb?SK9{?4-G
zwL0&HPZyrEHy>QdZh!sl_j5KS{+o9euDben>-@hT<Nuuf|LL*)ul~CC_WzHI$JLY-
z*EX#_5m$5h26IIIxeK<r+jx}Lp8V$(x9#FIZsxUL7VO{kB`PV%BT!sU!1u92!;u$`
zPcm~O+NMe-y-e%NPk(CNHtX6OqwX_43fnS-Jomm^n|S8K)oZgABwW<a6;(2-zE6<U
ziSpx7yS3wR-|~`#H_Mn0d|%|GVDXhL+m>~KKu<+Xpi(oRo7cgx_j~q!>AJFv*Y8`3
z`Q_Yum(JeW@T*Ji<zX(x#||!MO}Ee3KOvNrYiW>{;*%xn%Cmh=Ov)-Vp5L+2;MA!L
zRf~$t{&e^(kKiyhS-GkIT5SlQG@n&)j8$9Ar3n$L%N-YLe{hOj@NS{h%3bW?pYxV}
z+@M=Nfv1}N;=N7fCvW|@u~@Y}@OYSsb&twbub&%onUBTZ+UWi9L4olrj|>xa^P5Uj
zcL@ajUGVy0--X3Xg05{m&i4JNcH&;W+O}JBT*_oO6iJ*unzHVpb$s~r6S}(!=Ux_S
z5ARbpezZO2it8)GZwdjcPV$H`WM5KyyFGqI_ZpXFVy~yo59-XjR(QJm#qBMkO;&!*
zP6|!OnxgMzJ8ug8r63g)mM9_g*2TcjD{*O{^P*X4CGYryt6ZhrUL3yUVrZDa#_gc!
zQS^1*K6Ng~MIo~m-_A4+*Yp>EUfP}((eSjbyG!F3Ys9->g{QKTX3xE|*z>aU_L@ES
zZ;2hUu730GXtaIhw}sU$&)3>~`gwT!ea_@*{dbv@`R@H;J~rcY-mk{G{qiy2-{q|r
zkF{BTe!lJhSKZ=zb87$0Twn8S-T8la^#49t|L50wx!;TL+ZMk1y0+rboJ4&!P!Tlw
z*tC<g_=RdsH(Tx9ICG16wD>gtz?%uSObxp@wioIJ-7aO!a=CGRBe#oDw9d_jxoa;4
zh_;3IWzTQT%Ui+ljm?=!XKBcln8z;{sH!W?`?cAXY4K{8<3U`Stj^nGwzab8)y!dW
zXA^v6vSAyyGgpgvM7fOV^*NR%I~JN6MhoASFUpIU!rRgswq;JS(4y%Vbf^7#@*^>G
zZ|0v90?n*0mb}wU*)C7{vZni=KuX0VA9u~b8LN5UxVIHGNi+){br6{AtfaoJzi2m0
zpG2e8OPgp{3&%Fs#N1t5E-#qyX@-?&={5bEyMj`my}9PuxG8_#Zq0?-Mx6_m=Sa`j
zS?ej$;doWRM$(r_=fJLA(d*gGi>%7OaW2k3sJctz{E2-l-%K$SPQTu>JTtUy-4w6L
zpVPn2x?;HXSn=tb9Zgc}B~v(U7HjK$e0=>`j;x+6&y{2IADNx!*>>mbi`v&t962S1
z_8lkY=p=9GGTKsPK1=9?V@gz=mwn==SxvsL)!SyPzP2!J=Gdp+aWJWQmxb!4nSRd#
zex?LJnZ%)cUzjC3qxbK_Z1d6+^AGWbP4Z{WX*6b)T`ND0EzmZ8|D0KlF0q9riYJyi
zEYWC?l@t=r$n|J?#<fACNb}Igua;@2V<pyWJh4A6xdv3BwQT?NqBhljiu*mI;tyTL
z{P(MWzAdjhWL<KZe@^ui?nBx#`@RZ0bxd{Nuh0@<S5}_2;oG5LwvK7jPZj_EefQn1
z&U};p{Uv|8`StdH`t@;r?azDa^Z%UR|2cdAr}+P;WA}gT_P77Ob?IaM{O#K|itj!y
zcc99+E!S>^=e@Z5b{maWeA{+?amCWBe|80VX&y~8t4*Dfb?8!{>&xe_-X~q<JrZ^y
zLc(R9VN{i?)SD?rCie~seM))fvwqWeQQ!J)HJ(wtWp$f4H#Hci&S78UEq`LgkB>r!
zbbe~oO1;0+CT$XP=|>#Lp`7iJKPS!MUVMk|=KCqr4R1BJAJYACXom7(7a=CUep$Xx
z2a8fJI=&0JEZb3{)y&bUDaLfgb-zG^%5#B92LjR*Z7)fvH;X);l~?pwMO){QqLISU
zspacJMQ^x^L}iM58&03*Eplt|)SQ!y4YD#E-ia?ZuG3v}=7r&<nSpla7TJ3Ko%(WO
zhjxmD#xV^EuON|B4<?-=J=K*r^Su5Go?e}JS#bli*yWazifxA+m~VV?RQev0CZH&A
zXTEi{{yV1`i`Q%1x#iF$y?*aJ;rf<*i*)meSyKO9@;1C=Ud~!P@$r_5`FR)Jk~Zm8
zTfN;awzw=Zm~DgJqG|SvS|6oxZi_p@&~aIThc)#>=fOu$>^L9JowH-(mFWp*)=pUz
zw!Gk(6GM&un(hv-xI6Nup2yiHp4*vdc{h+brPfclDWhb|&E!=prwKSnR6Ut(8p0<L
z+pQ`2>f+5Z)(8o2!Dok_ZY+v^m-f`)qHXsziOFX7>S}8qrRJ}SS#y<v<A8R>YJZ!e
zAJh7)ojA@LXI8zK=D+0cy@k>CH}Bu%TUWLFV^?t9g6aO9NgHi~*T;SB%UmBbC)(oU
zoAB%Ya^}y^@A-2lT;8(qQ!#(tGx7g-{r`S@|MwPi{g2)2W&R&)%lh_Pd74ahqiw_2
zXFXSVmcPGpcP5W({4F1igM~W`ci)KMIrLItPM8;?R7b}GDHpkq4@>?h@QT@bD=*f4
za;<b?p+U;Tr+&iT5z~VV{8k;Ewfx5yA)|~Fx30IVKFQRI6_?B_j#fK-Dfvu7;pvF#
zIZO_x{oS806S{C@-NkI<lh*@YE1!%!ZriK6=J~BTzF+o~rAAC)cr<Ba;quH~ztdFO
zC9loDJlA>UloG8HHU=K0Eq|0A&sb>vQ=s5UqRY9d3*=`Sh^%2*&plT%X(G2qb>HoU
zO2#`D<#=c$p7oQ>t=^UpKVxIewx@l|Oj)u{b|2h!nX&RnG=J>{uS?g{`QEz)sV0PV
zWt0Z;E#fkm#i(K^w$(^wb&z?fMVDiHwnna3ZW+Ul4QF2dvH0`jT5gO>n}c)MuTvAa
z`D$6ckADh3|8(h29wRkT38SF>d0VERGdE3D*s>*BY4-gqR}&__WZt}?M8a$3PJs;x
zs)m}W`G*58RKCbgRyXkV6QAUw>Fkln8M&J6sMy<m;+H%h-|CxYKPBtnw*w1ripM4d
zaowLBHG5YF+r-Yfb{>UUHTo(dPtT=n`m^Tmq1Y*w$Ikv<cX)%a$>EC|WVa_=49)wt
z^nRH)-_du+5_-Kim1%3$Jf8KvS^i(fo-a`g1ANN2T{@UBVcAMU1B(x}=CL)8CLX$S
zWv9EHzrOwy?Z4N`EGMjv|LwKZ(ELTolhFL^wQ)MSaW^f0@)kewc>nbIv-`g%_CAic
z-@o_q$z$2yYrp+@?d^HD{^3^P{a<hIFMI!Y{{JuQ|Li{h_ucw^KR$UspEiA4c~o=B
zri#aB#i|)z`%3I;-LQOmi1y^|-?v>4pXIr_QY`B0R=&d@K0FP-yli3=<ANzcr<Pkv
z@+;-&b)6AV+&yiH$X1EYw>3^|#v5M=i9ELMn#g;r;m49``k6PR`1%q%cWPJZy9K(L
z%{lh<#f!=7<}@u9*rDO2q#Vw1zH;%sg|j4ex^{iBOy+3F@u>XLGb_84v8Z*Ksf)pr
zUtJ|%Jk5BUCNn%Ty`#c@V3GOOdnR=epOhz`m~bpRyzQvl;%hgYZY7`oz&q!*+cGh!
z;DY2S2a`k=g|NAtu#}y{zD2b4rsa|%r|gb|{@fKuPI2Dzd^m4r+RKI2V!8qPa~F#0
zHk)Nzow|8)5l=hc`xL9J=`Sb0b8{~YQ9jBuZT6?)nOfxwrKA;3cYRadEZh=#KV$8j
zWnC&)UK=;3vPOKcWt_VI_?4|mzq<6&)JpDhH*a}fV%eL$HfA$_-5!?mr@N-s1+zSR
zb@Qh9y}cV=R+x#rij4NrF`r#3sH(QWIBjPB+}aew)7Q1WvbXH@TKw#mf{|WS)7ift
zpN4d2+Rxg1KZ4h?gTZB=__390%&pVr2fb#!eaG$i%3X^LpUz8{(kQv5Zk!TfGV{fZ
zDwj)XF~>NpfBcARTF7GD^frX|xx4$tPr--cTqU2q(0+3+;c3TenR~NVr0?J2tl3c9
zZ-1e8ajkjH{co*Pw_G>Ota^6CcCX3a4{xT6-}`&;%8b)?e>QIBzrVZUGC#wP4@Z=L
z=l_`?@FMqU^z!)pmuY_=Ed1<m6IAzm_lH~8_m<ba|Fm)U^Lt<KJ?jj&^0fKB_x<;@
zrlsufZ{3Pm8~bxw{+<t4v@d>}?l<Spo&5i|HlF``Z}Ip0pRXS8v;F;K`ZWExfQfa+
z#?jRbug{7_X<okUee8u}_Tue(_ub8j+^_A>wP;Fruhn1Cm7+!g9M1hOG{UnSf}P*p
z$uSksd>Lw4=&(lg`cuPn<-JO!7Z-lb`8QL7;iAH&$sy|+Oz$jGoKbd8v*FH?8iC#g
z8uP?{CuXJ=^IlkUTj4ib+J+2~b2AK>x1C$nz3_PEx{Hdcg?FAP-2b4L=y~e#KKW^W
zOFwliZ=Il>d0ygd2)pm8of8VQSErpXGTdOLuDQ_EB9paY#jKCQ78k=Wr<S~u+nM1N
zA-i34N%5`;KEB4cS&fnwOgC<JJ$-Yt(&r0I6PC=X&^O9E@kVaN(gd!mC&>@$>c4c(
z2;C$z`>%!5BoFz?9=nfpoh<v)F+p;=z_Ta+YuGA$O@p5By6NBf$U5VU@RbRItqV>_
zG#z=<$rvIP-7WBFi%|0s$J>z$+rA!LRAVT-Nav;W^{CkACL-xt*)hIP)0W<4WB&8T
zLZMjGaNWHVOF0rf*DRXS!o)6k^7i64PdEaO8Zwlt8JKJjzI()><IL%KUaXco-&7|q
zJNRkFN}+-^ZxUi-ABFHPQt%U<S$JgSx}%;+M*7-iTNWultd#v*w={fBcTer}&krW}
z3wj&eTvwaAndd;+#glAY)59OnRQEOQj9G4HnYV-I+L3w9>jGqV8$?Sy-uOdv`YEj`
zr;gq#sd;7U5b{K(C1TG{cAx#`#XnYYpSRoi{<mGxw~ObikFMNuzxlcBzOR!{AAh&*
zt$mD+=;7^qFTY4z7%;=KqOw-~CSO{sz{`XUA3qfrU%y}R@uhy}qboPv?du;fZ(p<b
z$C;JC|F2MxUSI!kv%1~?ncXiROk8jG@yN;N@^v2?ljHwh(*J*DdEeiM@9%v-!94%x
z`u%^te2%x?U-s@S+cwYyoL*Jb%4s+6oJqTMxp$V)wTafR%NsS~A~V+&$n(saF5V=_
zH)l2{(}cLd^uEYe1Ced>IHIJLr6pVQJ~uCUxh!>(g+&~v4C9XAzZ#dM%K0Dda+1<^
z*O6}H5&I~1>AO={g~I0Q6KkxWOE;AnEjh&#sh#;tr&H2vcSg0i>&~qS1$kY1-Vf!<
zBIVvD&1<VuOyPW;@?hT!--+T&-=DO&nzzv6Zq5|0H_sRGJEaRbs1=<*^)9HEvsX}=
z=bl=6$I*SSwq2ZeyX@GvWHFB*kD0bik@~W3u`R13=I|N(R+VM*p8Ty_!=x%@zSe7_
ztc1OyQ;eSMKTyx)5ft*6L99j4YjYouW81V%ylpnA>$(I^1T`v{voD@`e(`5d1MO_j
z6cfSrE8k}Y<j?xcCBWXa&thqio?=PxnVO45JKDH~Kc9Y6$l9KDYGGy1Qa=4e^I6gU
z(#28}d2c6_CTxj}lzS@FD1JUqv`LLu<f?`8)b5}wNxx3qe6HKKZNb7|LDTDZ@+=wD
zr?<|&(>~d%WwPoP{V5DB6P~oGE?sdnPVvO4cp>2{N&PXef8>-K`kNjK@i?L07`2j5
zDVSr4{IneNdAq7_97*1}cK6PVV=HZa5_=<#dvnCHy%y@ye&dw%cH2Q0<L`gp?_PQT
z*1H^6Zu5QhkC(2mY<+Yyxm-HA%<P)gM62&Vm_NU-iof-~`sgC{4Sz1!w&(9&ymgn*
ze5=4{iwpC$j<4T4XHM*{1y>%GY@O`_IxFdawOz@-)BN$@7T!K@`|m)i{GS)6lE2GW
z{MS|U^|$}9^Y8XMV*5U~eLi2i)m*~nyY>F+efKNhO=Q>Gw*CFzJMuNp#Q%N0es0ds
zt@D39od1u#{_*m<f9d}&=l^?E%WuD7PtCTiqO;$B^LZq9pz3qWvsTA!<^#vlHr>t1
zHIKfaHf7?$91WF*sVjZ8q_-H#_#QGaadS^=^Sb9?#=9v{-1#_*_uGsr_d}smq#WWV
z>uK(sB&^4y#O$)=e<6Q4msrBigUL^S-0kL`_#?l)>C~+A7aTA7aeR-+QExo6rg2H^
zu?#PTPk-iVKQEr<x<%V>{f-od><wajHNO?8XM~C;+|)5xbYxbiVB3=7>+jTy62&h2
z3H#r*Xl4y+igq#D(jX)h_R5bhbY0`2CGOd(v$7<;)|v=UK72N3y>r<LrcK|@d+|>7
zzj{MJ*(oIB+1q(enxWq}Ja%vSZ&WDSc17IL(COzA_2kNZ+?zki@wy#;(R;ZoawAXV
zu0Bz@gI?8tzWsVA<fOKGf@#IM2{LE>UomZbrB@ZaWs_cQc=hoQs^JnRvh((Qs|ug?
zE-p=P>eIN2ypyi)qza7tcPd=t+NQ|3ePUbaj?FyXWp#y`zh{-zI7)PHs<`uy@j`7b
zSL2Dl32%>2KT^5-=<+ppRdTIO6BaCwc;2_>V<k(&EZfQ5QxC_cp7MJq=eS-gaqiT`
zAKy&4p*(kmul}J~&s7=3(l?)ccq(n?67IZ2{RSrm9mjdslJfUvmfU)uJyl!r$UB~E
zM>ZY28@FRNb6DR6tM51d{=WZvqT~MkA1`foQ+WHGukOd<r89qh+iCp!{^x~GI~T>*
zOCFo^|I^`@3b8vL?tOjUR<og5e%ISa;qtTfe3usg=D+*vq?@)~-J>1mp;2`o^zv7w
zu+>gF#(kbS;ZleQ*Yw&K-!^}*t^54r`f`4msOq0PndN`}3;zB$aiQFt>PLoK>oykr
zIKlnB?$;IbxO?}?Kb-#m)&Ab!&-S(7|9=1TcmJP@^`F+)y?Outh5fHH>Gr=*75nFy
zm|M>+TYuYk|89lrpvmagdzN=YbeSI<`n2-iw_OL89Z*}sz!ZOBjme=b&VY61q76<8
zJGA99#Wzhmzo~FVz!RpNy2XwuYMX-RoZH;sbNJ=PR~pl1w9MC8llfCKGHd7WCmSbS
zx6r>HbL&}xJLiG#E~nXZ<R+IrdbM%W^=P*4TddPI#ECn<?0Nm}#l1_vq67`uR&Orc
zsrQNb@fktGh|;w}%=U|_Ls_zuITi?X=+2i|bK+jn3&YjVlnpJ+PaK}{P`K=3f?&Q&
zr)5%u5z|StU7nW&Rj&nDJmq*;?3lZ-^Vg;Wr+00y-t=yx=(J-3e<hANU(@5Zwu|(*
zFiq&%E8EErZ|6^rwh#_dd3r4%I<48b=MQV<(yiCLz0X>N9?m}Oq<uT_mY8kCU&(mC
zKi+}P7ehtZE2|3`y6;U{eNbw>Qpwh^cicNw4_G`enS1%u&AZIG6SOm)PQ9UeL#F%x
z6PZ-G-p&Fqz7<KzhO;+FoHm;`sZX$tWx@=`;u|T=htBA|6qAVk8FxaC>&>@wvZWe1
z?QI>}Can{$ylOaM>>i+c++OYWj_%2t^LewM*{yMwaY+-n`gg;YO_o9VFRGRNOqVOL
z^If&-*mESoi*HG|&Zz<!=0G1Eo|5^ebmN<@%{q~A@$tfFySk4jk6${N@UME``6Vy^
zz5jjURPy)m@3CRG_g8#esV|m&`SNf6si})UetR!ja_eID`YZhMf&xwljF;71HU0P!
zRB`(~)63rTxVr1=tsj{-HNWSk%kTUCz*pY-`;EWtdAk=+cZpkRV;Y>l`s^g0K#hv4
zuilitxBUE~mizs`1%er(=Xo!c{Hrye`{PjhlV16nukY^He|>j<|9fx#KX=~$xpw~l
zH~;_7^8a0{|FN|G^XC8G;^$kICcQnhGjF-^zrOp08`(GPd+q#rQjnUTzyI8N*V1`~
z+c{cIX84NTDt*Wp;H9dT%;zD<%CJF<Q>p6=&vX{Hr0!WKKNMWMu4c$MQ#SqDv9f1g
zS1XU`_A+gHQ+D<GDm@O-i?i9LO_t~MV(FQEVs|-TmAKl0wyMh2uO<Yg+^sND&p2zn
zFwfGGnPcDANztNnWz?45Id^uCeAOxek9$0eWj$WVIe!Vg5>qU>#-(kE>@kgI(+(dA
zH!s#D4_RNFP&JO&T5&jaL(wwk8?Mc+sx2G7b1Jnkv?QLGJ44s;X@o<ikn{4=nYL-O
zbo2OLFc~eHV#1)bqRU7)N>D{fa$3-_rE??BZx?Q9*<Rrt<zF!2hW<I@wEMI6o|++D
z{PoS3DM7x6+(IlT+N!2pYmGXw>w?$M!{=3tO1!rnSb5+G%RbZc2QyY2Eb`RpD?A=`
zefq&%Zr_Tzj#n*|pB}3He%<gQ<LO#X{oT6rD^uO)ujy9wk#v#cyqhyE!1~Knl^;jb
zcysz3=HFd)eQQu#R76qbf?Bb{RX3I$^c9TSFBHt-66<2ZoO0ej>%arPhR4s3Ki!#m
zuj7oI)313NEKOe{@&Y<Fc1{X%6FR3Oc<tf(^;eJR$hqBI*_rj-dq?8V^}Duk)ji$F
z&fw57Ro{*w`P%$?E{Bp&KfbkYzWiHn^;_!;7tg*6+w$>H@YQ2?&OCMBHZ7i=?fJaA
zL!XZv>pp+wTRhtWBkTPa_UzbD_3c9Ld>QqteD}+a-%S3_@lbGT>A%zbyu0pp`_~%Q
z+}<i3Utjiam-PACrw@<Lu7B`GdA@zc<>S}h)mpv%&bTCV*PjP>i|1E-UvS_1@|P>M
z^*^Tm{}LTv`Em96y+1C6%k6x9|L>XoKVH^<?5}?~{r`FQcl%dgpS$MBU%yV<hOe<g
z(UUvnHto685Eq>lSMhh{*|Yb8_FRq2o$~Qhz->MgDetdaPZbq^zOvF+;Z^q32<>8T
zwN&N6B&KbJ-wz9gi$v{M%y=c~Y>9Y|)WrMR7Y(;^hj^)~DcsA*D4Y7%-)OOhpSy{`
zhA9zAZhIItkFf{b)8)NzLZ*aAzCSoqqf*_c=jP*!H3?rP#@3x)yShA9DSk_><&Ddh
z>P7EgOfKnP*}0lC`6BO(Ps%0FUNb#5T4TX`$Uw<fGS$DL&VL=RkJLpkwVarciJPTU
zp9(L%k+9Iy-l!};;X>M4$BCL7Wt2|+>S!vwdSSWA*)3tE{#NgnNrt38amlxBiD5gk
zS~T_Bq<wNxTZFeLSxjX&d0D_*r~KV!LRIZf&#p{fm&6xWZI_>(;Kz6%gQ@Ih=E*aE
zjc4nv(mgR%;F{}`ZxzcF7;J59WRrPb>`+~?rOmkE%LyZ$pzNnH4tM8%DDqqWG-=b_
zxEDeRJLS$#G0{1_*mD(E<C4Fxx5SGJwIsFwooCA65d2tb`uWA3VrTVaCL8cZt4#du
zxohFp+cE~x9%mRhmWbq-J)6d2^ri4=*w;VbnadA}EuZCCB2^%kKD)C|Jl&ch>FUx*
z$<-l0YddFL&z!%m_2aDrcTX&j+2i=~>eAS|XBWKpo%(J6|1gfi2hH0L&G=cqcl)Io
zb9cYHd+$of=c(f5*YwZDlvNgdS=(B)_2((>%vpZ&`@G)WkGH8vRh@G?$}%?djIMpf
z^<(=Eul_uJZtaVEANy=Y+Vg%tx!L~C^Y>rp?KO`M{LW(vSabGEd6fA4UymC3@9p|@
zLL8Ki4K`Q4d3Shwoe*fPklen%=ZufPYdiSptbBaki+Nk0&+nN&J?y+*?v~y0KM%?O
zon8O!`@d`dzdfH{`EBLL&!6WCRWrQat&*nZJ=LV4FH5fe=aXZ{I(yHy?9?_5WjxUK
za@9-it07*zN^3*B7B94JVpw3Q^n2fis<|9mv;GUN(_(2-jhtzJO+(_7dHO4+&YN3O
zZ)`Z4bZJ|g@0xSA$s1<)pDO)blsx@TJBx$v`L@r=>ocax%kA{o%_F8O@T+rH_D&-W
zFTqJ6*J`E1+m4oRbvfRkcWT1BR=vLrdEIHVR1R<Yd9cA$qu^7sNb?CECDG+i{;kUV
z<{vuYN{-6nO**AdyzU;~{q%`Y$Ieeyzpu8udG#)ZM|bMeg%U?1&3dcEtFE!mJG4!<
zq;SS!nRf1NIhW-g9#U)alH8RPzdB)uuhGk*4I$Siw4JHFQI{gbe4qR9h0nH<ZQ-Xr
zaFx|;5Hj%%ZFk;X$+J*Y-$J$J%pSiVjzx6>eQc*&?>-I*)?@DQyqi;0|E#FVk}X?F
z?e^Qdv7+MolO^UGFS#x|>*Y%(!-zAc+iDDKvVIEn*;>81qMD(j(&i^p`QW5QXmiGs
zttqV!s+*Px6lQT-xa}{@x-@62)dI0Nr>NBBM>K*xQcqk_xH|WTprZKlT@yq#Tnf@M
zv}gD3`m%G{gl%Wo9Ok~<xJ1O|isYH6QLH`5E4|bC3q6Ihldd1?{OwbHcXNM@+<~9^
za<8|So5k0yyL;zO#mAXju4}h%{`BJZnFZo=YjS@-I<+v`?&ig_*FL>|d)Bl#(b96a
zij99<sqe-h-6^Mjhn<((_jU5){3(kcowKWXzw`Jz_JVH@tNHJK>fCbuE0^o!WvBV?
zKF-@#^XA>d)4hr{WgnF9-+y27ZSV8HZ|7cXU0=I&>8yJ`f6sWI|99g45AFZo?El>U
ze0upuS&3?f*VZN`4fDcmbEgFDohZNgUFOzpC0V&IZ$8$aZ&Uba<>dAABPzeH<UYOp
z#Y?u5)s<#%w!ZYwo4ahvtq^Iex6EhvUAty!kkaG2qT6MeW3r+^w8@QBqcY>iUU%Dl
zb8@m81cM^wOgA}hT03q2uC=>gIb9Ag?Fh=^c2<^Z*!3l|WB!emZB1qiqNh*fU4CCu
zHpt9=TjYicfs1dZE$A#MpZa3QV*g2^PZXrGaxCKw#N9T&I%nSU`p|^Qi5K@~?ah34
zXyP-KIsPf~Yh2nKlN)6=F!WUK*zNeXW$nbp&ys#PE{W7X<LSSs`$g+f2L{Dsn_AYb
zPvdJ5v`fBmghxzYgxS&dUz^<<%^N4Wd~fGoO}wT(p+Ru_*22>8FE8XyZkWH|WM9pZ
zu(})5tA2DOhFy_Ab>sM=cb=Mo98adY_iYH98)^G%#+1u4Wede7<vcwp)hs5%)}7I>
z0lH9SOY*NS(}*Klg?8ue-k2I{Sm^3+C~->n$&<J5yta!Nia%!UeD`s}DY<L<;Y%Ah
ze%_ol*-!t}iNp;mkqu2!d8^ia>H6~V`s`z3AA(N#|Bj2`2<UeFw`iaE#WRgM&Wr)E
zx;f#`-lRxLES?(Vr!4!{AmY~69m>hySL}3So^s)AjA_qVd$%RbpKTAa+De2@`4#uf
zMkn>qqpI?E-+g0`t4)g7oyvT-^{$kS{~3>EYNjhKe>{AzAC{fD-TSuB*&niZ?^x}(
zdD_m9wtnx!H;2Dp<CkA68_^T`r2f7ygU_tIJ%8@(leXXS=Rs^X`?`iwyJG(2JHM|i
zdHL}6JA-CM8%9MVozmN~5=-~B{aaDC{kh!k`ggme_3VD8J{CRwa83Qu(9G?-|Nk}r
zzHR%y58d{k|NlG~UH94k-|hDn)7xzuzRs1HRWLP3%hcO;t>0W_i?3TB>({@Xx*VJu
z!~5-jzxo*5vRWeQD`>k-oUqXEU8`O)Pc*WTVd*qvnaVbCx`eZl&eEQ*Oa?(}1<!I-
z3MD>$h%+>loFip1D{1?}UdvaEpmFc2<~g%EBeYHhtQ6_!U{;W7;94&vdztsZ_a%n6
z)_S~3n&;-J*8A*`$BSjU2f1_V3@%@~^fy9teyT~Hwyn90JEz}_G)EP&aQ9uh6~&QG
zxAyOLT-P;0DQn$Lk%u2*)ryR-+{hD9+&b4YCvc(6eGAXc+bqS*O$@`FUhK<U_pHw(
zf@ijeh3gzKRrcRzJDpiVpYdhI8$H>zX|tQ=p*IsU@^8+{XxUd}w<a+lRQ%?|%Y4Nj
zx)0twvcRl$u}YZFYhxZQ6>+~y!HE(=`_3FmX_3k;PuFzcH`l_nQD8Bz$>yg=9F8eA
zxaOZaYj@e_KrY*XpBCob(~~zGQ++F=w@7L~N8-=rcRw}je0XFUvi-!-=5;^CPO6Ic
z$GG~>m0og&@q1~2#366q1y8-#@-CG5_I~Lj{^_r}gI)Irtoqj-e7Nn$9G3H5OmmIr
z`+jb_$II}bI*21vpG~)=q*3H%*E1=z;2V4&C;0}5r(3V%>7FiIyJWt^m8AY(3z!9N
z9MC&kaOcF<ntQpwJ6S~C8n5io6?2wn<X5x(ew3YW&wsI~;QfCx_FrZb*!OnVv(x8g
z_q_aV9``4zv}~b4=&Pd>_q>|*<Me#?fK}{&->xl=uMu!MFr#zL)s**_lgpVcs^7>R
z3Z4F8mO5MaD`EZL4{~3hx3~R$PC82b+_JxoR{y^G$;|!#$nc>kgHFwdZTEk)&j0zk
zyza+Mb-DdtBL5nTf=WZZ693s(1hSX<&kwKNx}~b@<<j)|F_m9eK8~;aGSgk&`rDW8
z<?&TzZ*vyS5@mR9c0SKMnnCOLCL7s{;a*Ekeygo1EA%WiynmUMiTAJ1o~e~LM3@$q
z&gEg~EIwuEsrSgz`o(27pNznc$(o=7aIfrXBZbiNj7qsU!&wmv-)734$&Q)f^!t7Q
zqbY~9iy9B>)}E!)(wjM2K?_z^H&45<H*BJW+NQ!uK|2obGPL@>Pe!jf)!1lOp1LL5
z=~~f;9$Jbrb<+%Y9OxB1Y@ig(pr`djM3J+3Q`?bm>hFr2cDg9GKAz1`H|_Mt3|q~a
zE7r>wMzt)`Iom87e<65g-pNUE=jJ!)iZ5LwuX!!<%(;BWOVy{oTx42b@^{xU+dHw-
zvTbx<*!&W%-M}EAe*OBYcd5_bPdhki!?mBCj=JwQyX31Ko5ruO;Y;WNrFpv=zKJ$W
z-E#cjBA32t8{Tc2_eb!C{K35*9Hmc7UO(n^G+)flUiwjP-%8zS+N{ctURmAVcR@AD
zv&~iiW{7l|`7J$<I0=_oa~9pnTVj$FVcxRF<L394Am>9f=6uMQY5nj*!NW2R)#p>z
zXon{=R$2us<qI_Pie5=N%XUI0XOg>V=LWIG-J4}!8_l{H<rnT!HJf8wJ)ddTo|0CD
zGJht&mKfoet3R@evsE5lq#?$C?~h@ihRmM}Z`0*>Ex$B_{XyluYd4qfEU$b2%=<jU
zjt@)Qzu*7SIQQ(<Ur*h)EncyAW7ao$-H_$GJ)ej!=ezrfQ^KLc`^4wtj`wy%WpCPj
zB)6R7ymig*xtZ&;FEt7*T0DE|&DOqqHa{l5XYUEu(YyDD`Om82bicoc>VL1T|H)tf
z;?d594L{Boy<uGQe!+#kfo}ujS6<!heLte=?HBL!^Q=q1-C4OgDxYb>Z7#+`Zo6C5
z8Uh?<pEZihTW=QiJBXoHb?QZjB_XS(G>Ob%&pEHBY9;3q!@eXkd}6Tu4&LGiR~|6T
z`u+Ss;en>$&CY-Jyf&Izt2FmeU!+OM#8p>vKl?BjUWqCGD!emm&Cj+4V&7X8wy0ff
zDsDaDme{$5y^qJ)FwkUj@ahedt!sCEyeOHxp>uPK)>KRH@_Uw@cRKe;iMu~f5|CUK
zrp)f`6ve<1m74eRi|oX-6De~$i|rN_n7m(;UbL}0bwkpnPGPo1U$52Ga_s4N=J;&$
z0>`zPYm~)jHZf_OnlLXlPAj|NMWe4$gU{P@?JU|Wnid}~pBPs7WywyR>+fD@{htxp
zV>OrS<%7*dw@PMwTyo7J#6O_>cZp)9b8Bh9^;4I^1#4yPo^O_6>t4`yy6Vu$cfH#c
z&U}qLRT%QP`as6(7q4@cFOymB%AIGp=JgWSfd0P<8NAgyKFd|@S}?`a?T_`A-3~e1
zt@S6xUCi3Bz*IaoYy0jg(aaqa>&_HNZ#*n?+b3z`WEoY-#L3OEXMz=rIbU~5&J;`I
z)#~x%;c}d07*ST!efigsdqO7CPnzPp_Juu`dp9E`*L2;%Ln<d)V_#^$VT#GHeEVJ8
z{?EnD{P(#P&)NPvaHZ#ce4$9g3@zpx6)$IQxqtirz5XSC?>#$xo?AIg?x$1bsoV8u
zxBhyv`tP+c%Q>cMtUL+lqOOX_pWnf7I@aDV(l_jp{O;RJUVdsW7d<tVJHGOWgt2L`
z|87~uBOoEKrCaXXzkad(MQeZE!{Yj{i_QP7pa0`P{-1yOwts%w=d4@bH~*Er@<#Rz
z|JJliwFPE%%K7iBOG<ux#C%$u_1-^kuJ8Z0Xmc+U!(xHN-8Y3m=i>%#f0p+76kAE|
zRqe-5k1S!B;$<#kCbHmc+o@yMPCc8-5TeDXa*Sb;#F`^KJCs>^+tw5<sAkLDz>tx9
z!=gl>a!u5NS2nx0_83La66SJ!DX4p1ier{t-|p>)8%uW1YmjJ+zg2nk!qnNSUOnRK
z&$uOoHu7Zu@lcyqxU<|&Y2&&rc@aJg8`xHBOh1t-(jZVY{okg;3q%wfEO?m|A6?N9
zHP{&>{%xM)p3?z*riyC25_@mBxY|9GGw(jGq4NBzgZ{!ahHG<*U*x8}UD>w2RJ?-u
z+Mdgb8%@J6emkGI?WC_r-{<0;ruSu%PYUq8xbu<qW@m5260Qv1BH;~6-`^;0aq)Ih
zi*Rt5no+pwu2>7Z)$fNTCl1ayn&Fr1H%B=C&7!3;;#0jjo)xu97>l1(l{~(hJ2ryt
zxY~5*r7Nlx*Q#&GP*0X>IGu2@Ue9LH{XpGi%lxCf4o=^4HQzJk=ii4{J+|q0a?a^%
zX<HDT)W)TI<Nj8|l+;5zohE#j@SmK)v#>ewPl&jL%hxkm-EaD0UryaT$CT@`<aNLK
zeaogij^(&3aoltIy`JPnzq0li6~4Ig;Y6*$vkPbC*)LT5{u-^F&hX;;zPcaVtoN+E
z6mmDd_L=v+kj#J8_xTg~>VACNrx922)b;1I>`lUU5_4*Qd~Pr>bDd!=^yb`Q-S?M{
z9j_I&=d*hI_x;}u0#5TQTt&K$dX;`$*3Y{uR?+5-)AhA;?0(Mv|G)mn<o_@7{~evZ
z?*H@uua5uwyZ?`R|K0lSc}A0er7L}7xK{4=<;$g8KZB*E)%EuLyR!PYe{6)^*HfRL
zpPy^h&9U&zHcnNi&Rq@?ZKaHjzHKpIIhbbOwz%H%=7B7OgBjNig`Eys-FcOQYgr80
zx3TwC+vwVhI$in^G<~1^v>o;eZv-Z9EiqNDWbbl(VtFR)*~1VXmoN!_5vF_r$?Yu@
z*uF2=c;G?L_n1RgD|aVEGG2OcJC@g0>d*uuzDd>#B4@K%1|D`%Z{eSMEXMr0fQi_P
zeW8bTy;gX2@BfRyY}f4t`(ih;X@|uoZCGJ^eutwf`)#-LM;IPd2d1c0?%rHlm((>!
z^^AnhuGN+**DR*CdTOvN=HM}_f4=Ty>gzkTM(sx?mBmS{4_#7V7;G<fXpPNnmM8b5
z7FVACX!=gt_sL$jLoc#&x{OjXZ5FORvOpl^+S`(?;!Gzsoi6b)HOkMlxLOysRKsuS
zB;GsQzinlhux+o}I=Sx$i$bfeEO@rdb1j?UtP`iNh8FC|5!-(H4Clj2@3a#|)ooXk
zpK3||jym{a5zl&SlgTeEP6}N+((1iUsX6zT=iC23o^QTqlJdhZM$r52;}B7%%)+T<
zhGLE;zm#G&b-h@fxLu>Wbj~-^n{pioPwh)tT*+fu@pMl2oFdcCf(O?t46;<t+s}GG
z_31_Lj15J1uI3qJZ_i24`)amNzV6inU-?#nbh#fg4KKdtZ@H$&;=X=f?eXtA^1G}5
zRo}aO@e~gm`@Q<ht@^UUt&eu<%V{t9%bZ{D{^i%|xWZM7&03$!?t9C3fMd4OnWGQy
z?*7{o9Fw<h!n9sb#x*uo_fzF}#qRwz$#9v~_Vv=*%ya%<GGqV^)BJdTNMcX*r|oq=
z*WLg7RsR3~fBXOU%m07=f4%+piT^*WulwPB+;0EYy?bx}{J-s}+<{+w9rMBhpXs*r
zo@F_3<x9<t40)TP7Y`;nhsTt^w%-4JS9#t0QuhA1^0yZcy9RZ$-}JbCJxX&bOGM5U
z>k4<rx!Nm=8T{r1HG6z7S-70rOVj^*{+@4DO`7`i?n<5bTbujJNhK}g`HY`GR<1Lg
z^+Al^bDR0Ajz?W=rs|~}6ZTy3d@H$l=0T<A;4a4yeU{GbCfDZOhPSp^o<6n8*!kkw
z#K|(3=6$}A6{&smQ{<U-Zv}%I%fE@0PTc=^+S9;Wihp1FExT)Jm~%C8-GmEwuU1`d
znYV25(z81P>@#$9&gn6Hu$?*i((|M(Pm90(w2@qP>qzDShKUzM19b!@1+}<X?_}eZ
zXjRDTxhlk$*tI&hXnRUw)wM^*91R}d&}N_aqqbX8SJANT?H%5t-B0b+>(+hTe(r4|
zQ-km(367;9#}0EcOuE$Ml5{huGqJkigoK{r61SE@g;!B)bISTEtd=XQXq9V)JT|g3
zH@3LAS8Vr|9g_Uw-Wz#mXYXhTT-5#l4CfxTeX_BKI}MA?79KZRvvHE3=l`vLuJ{W2
zon&m-XI8k%UDolb!X;h#ZAAu~v&+4+5)1>9Cd^p#>DZ^}n`~R!WVdN7JpZ&M<kE{<
z$1f;|t=20){ZJzC(du>DH_tHYzZ6~B^I-Ngfxg4aU3Vk2eGUZ)d+L1uc<1qPhM0dB
z?b#RHy)(nI#I@wp4h@mhjyryRnXdQyz*>fL^*<Atm;AkVkpEuWsV_6t_b2tNl#LLX
z7p@WRv9Zrk#Db4kT}-s^|I4g<=jC>{vqU`K|06*?=wbBzix+1z9pd?QOoHLhyZj6M
zZ_Y5!xBmO<<NCGhYG1woYyW@w`|8vDe;=&<|8aT!mri&28(wZA75BH@V2-F?ws^U(
zL|d>C56ht_3v=`R?=MUZf4jTr{zC2bmY?Qj=U2Qp4d3_m*y{d0|J%OD6+b;>8lE;u
zfhkmSi_tPY7mX)o=NLVFX3YrERNb&JXl5^y(vnr{rpxtO|9bM<{>wSmi_ib9QTf>R
z`1<i1t<USP^li}NddwGUx&P~~FLxcoI9`_N$ho9ADRVhG<%aF#mdKwkF?&kRy58G~
z#dqa<kLnbCx1CtL?Bq82zBxvQTGJ;kw7wG~q008e&h@Z~|8&h$pEUlS78Tl+A9Bb_
zp}5DD^R;7g;@->z(*hG~dH#7#)=K66D;0Qbm(H3eVmx#447L>|iiaaoTA!ZUkYWGW
z$3O4j(-jf2Pd5cfv|7JeXqva|=#A_covF??ENi(KYsF68G%Z`a_|3gTNeVwU%1i#X
zkc{e87gl$(Hwtm}&~$RR&dSwjyINs}YUX1L(`(0UKW$-QSlTjyWAB2E8zy~gI>t0P
zVva`ttn3(94t9^X%dQ?)z0JMrbu6Qp+sR}fr%d7R(;gqYBjl8}+gwD!$;yvO+f_5>
z&%q~!u{|dF3dULvd`hCL*X3?-UikRRhWE@jw;s-1A(Ni=OQp^!)@4eBy4no$;*Zl_
z3m;_Kv0&qae{z+(JgU_uo#FSsQgh+1j&{zLZ@euVJ&YD!w!7FeBjoaPH@_`MUY84-
zB?>LM(R<{PyYY+3^E%CLSXEc0eeCKLpa0V3)h7NsPC6GKweOcdCbR45<LCRo2;7SP
zY+fVZaP4R6<Yj;7UVC)bUZI6ukz;AE{BswrJads5%eUt3K74oC^;=RUR!<$4?Y;NN
z-Hwgn*q(_ZDd*lciSgb2b#n6Y__|+*j>`YMUH_KdzToWrzo+H@`~UxTe16TJh1b4G
z&zk?I^N8GmU&gY*K@3%H9!-(KpH8)&?mH7F7*`l+QSB<>VOjlj>O9jAk+D^8b2eYO
zUbXY)9gDpz(U*dzY|`7qckg0W-P~E)+P(AiIC3sHY?!9Dq1F4@igW|<w#4|^i)6|)
z87ItWG@NxzgemBV-_Jx%f#@kE&8}uaT;;PL`jz+`3VO7$<kogm$1i)9<bAtyq%L*y
zMD72Q{e@BoPPD0BZ3#N|-RYH?#Bwd3%&J*S4SsB3)J>gn`ox8~%I2P>sS(FA*0C-=
ztk7e6E}_)We{0Jf#r9sG=h8_x`U=F^x|d&B^EOnA_prxG6CP==rSoJeXHL16az||S
z<bzf5t3Q@VJI|3}kz{vxywcgn>*v!Qs%<gChmTt6ZF%MHtH2U+@m6`H$kc}hyMr#K
zZHeJ&sg=2Df9!#>>~j|(?ak6xyKd`T+uL(+(J#kOA*+whymR(D%k3S@jjUsjWvnpC
zS~sy<XL7_E##NoGpC9VbH*<UbPRinz=Dy<H;nQbJmWU@*YR)tEDd{a&o_^(As$A?}
z>;2yG+B0;%-#xnL+Nr6D8yz){@wmU~_fS6l`*XkG6P+lI6NMrT%M_a=bre-@Hb${&
zH(Wh%@x>FN*mvtYUuetuFM1{=92K)ES=yCH&r#)b^-g8x=-JaY_@$p@6@Kh_pzwNg
zp`7;ivw1vWdF?yhlg}t-JF_m(k^XX}b-kS)7b}Bye;wn2naSTJCeQlx<=6Fj0!}{i
zdt{t^t}Z)oTQ%QsrRQa1{u;gIi;M1;n|q2KGq``kP=5FC3%c_)rkn!pUo1HMJ4*a~
z<<XV*zKJzVJ^t?W++Ce3FLlaS%4F-^`_%bZfBy&e{r|l8|JeQitNy*J!{D^|#ddz}
zv(x=0w{mN=llc?=-sHDbJvxQwa6n6<MB5?ZM9Z8CrQNE_qqMhv`86vtQ!T1CH}j=P
z@mH(2*IO-$wg|LbW-aYIvyS0_27_(vg2se{t2f)M*i`v6hGVJ7LTz`CM#m=~uS^IR
znzHfTE2+bdUYFQ}C3TWb&X&tMuX!_J?Jvc`_(ysL5BX1O_T=o?#l_`R+ILx@qRYpO
zH)?fV$`6ItauJ8$oqHz9x$#YN+uge?l4oq6cs+f2=i1xwW2HW^XVTyCd^36f>@<UW
z11tCDpj@s-kF8oPN>`6%cC6l7CZUt%t6{>xv2*qXE(gPmgrG+~VY^&H`dD_gtl^q0
zF#C6uo41eM)z&1PP;rGnPaf>K@W`MkY{A>?jf_saZQFPA^faFMn6ByS5%n*zc|p!C
zhPKGV>s%&WW8rpixFed%DJ-d16<ny0^(gCm><h2FnUAHlW=9xoIaT?$c*l<VJDa(Z
zzir!dX|8e4<V2a4dv`<<^PWqH=3M+1;_Wf%tkAWYmx8Sr4rJO{{P})R`?*BYueglF
z$t+t$lV4i!vP~@W(3#YjRxf1A+907*cj{aDbshfHWiz)u+3f4sBKH1r?#j4s+f7yH
z`3{t^Ew=xA!eCdCyPQyVnW6o;gx=$MB_c;HY>flO?fVRhGT0n4kCmHeubsy}IXdp)
zg|+hyGOK=kvu=H^d+$@>`s=sLAG1HKj831<a^Zfa?!7;S9F9_JyfxMxvp>4jJYmwy
zH;2FT?V8K{ez&FF=Uw$r)@JTE|8DzUGej%R;KbaGo#0e9$2z9$&w*$6KR*nQuX;EA
zUw!>+^}0vx|KHa?TwMQu{okGTKi<5z{eD%yth)YFE&q3hYvo=U4x4<P-A$)P-HM5_
z`uVFfeE#15(%#t@Z@OQbKF!R}{Q7LSDpyTsBR*X(%id;Rk5w`)98-g&7w_D()#PP%
z>D`5r+qdjk?<>)^?e%6hg{2D?`f_;I#<omwGS9qIDBb9q#&#l$huKg^j>RFC<Ge0!
zw(S~0zMR?VLE%d*Kfmrgm{Y%QNB^;XyrMix!BZ43Caa|Bvn*({x8BF$>}bv~iT9$*
z%=b#`E?ccV8IpQ!ZsWNFdvy6rx+OJ>%$H2dU&7<b=l}ZTVcFy(p^5@iijpij^e-se
zzV<ZiP?2P*F#2LUF)e;tOK4K#+O<a9XE{9`oXj?uEfGHGesSC9(<|4jsXch1y6%2e
zUU)&qW~T+R%e+LJ+g67?dzy4XLBX|upZnUTZU!M?rd1l^45~-=q;BTexbw8~d%MO(
zn`?_sL^Z|kZj*QMP}}%wk$p!(v}k9CZS~;|dP+e%f-k>|D3yK`@=j&Kp}G3HO<{9V
z)nxm<GoGGezWelU%UivLM@<`qFL&oSoVs(><>8*)Zz@h@M_oSpN8yrWx$2@L3&bw#
zM;(sa-qZW)()0VfPyEqWY5UABy1eb^#4pOJd_8q{TaLX@5ni!K?ybt+Me`QsWVs4i
zZEw}cnKnUSlJ>nt&TG4!=Ddn1jh$=l+g_yGpRjzQn^{ZVq-{M(bJ9;8JrXkO_pLkY
z?t~`_rJNMBWWQHlT&^wVWU_AOJhK(?buH!w&Hrw;ziSmZ&3{id0W=_EWxDm(nd#~4
zci#KVb?Ec5EMx8dI!A^OR|~%1YI;5E@4ro+SNlRPX#>L+^Xp%;*I)T28&myEJ99br
zg#Wuw%sp_o;CbGyHBUY35}p?DH9W7Zdbzdx?!o$>C+olU|NHg-PqTc*Z~s46`u}~?
z|NrLW_5S~!rZev`N8A@&$awQ|*69~pmj<i**?e8v{=WX@P4)RdZ+%{G|NrZBKh^Yo
z-+rtNp1pkb>of0K7AnZC)^ib_)bf@gVa?QLVeUBD1KsJ{pRDE=Pu;Q5pra(v>y^G(
zThkBS@OW2Yi&rxrn>|^SzIk2TD(_Nfy<Z>A(@SnWH@%ZR)9Qvr(xr(zmr2{sy=r{v
z?8(rQo$c)IuHM?2Yr;=mNSXD3iT6>A%=M!-_v<qL-}zKIC;HRkHScvE8eCditaohh
z<eNM9eX~8d{pv?9Ba29(Q-3$Up8EQi^H0Z=u!t$DU#4U!O^#~T;%d)(yhfs!i}Sd|
znjVu)*;1BIf4gk7ycl)E^?8WAbgY|eV94tS%YG(Iu2ozhBy6<E%jZS)(IXA=k4x@;
zfA)CG`|8gPpY~SnPMf<yJkj*`xx-I8GcV-qoiN3zbwZ*<x?9o{U;hlvt31)d!BdQq
z^<{X)Y(*y=vNc?h?P|2<&Ps(=>v-mMmF)5MJ{gHuSWVKeO}PE}h{+=EzsDyx@2)I-
zddE#o?r_y*i8D{rEmpqDe*9yn_oE3huclv_))yU`W`64N8HH2VgiTNPz5Vi9O*ufR
zH-?$x*W@n|Gy8iOC%=B@7-Y&LYbmNOq?6no$aQ_C;<kIdJ7YQTF4;JJ&#av-AB0?+
zgoD$B&F|@!c6d%ub$^nx=gZbs>3in22^)X*ic971=4+^Xn|klT1>N}!YtEf?IB@&>
z-uL<yt#v}2ihBG1o^z~r=Qyah*pl^XUc33e2bZFhd9LM3HgEm-$e01t@VoZPk~w%q
z2!oD&34cP-^AvNte5Ib~y5zcq=D70z)%KU}-TU<^z5e<3x{vAq|Hbe4t6%%%?f-Z8
z|DL}8Z|VHnpF5ZT+WV*fd+%|%1HU%Q-6{BRq;vVa+8^KL?7tmij^FeCb$s>5J1aLc
z&5qi8@4A*@T5vAMU6Ea$E_;KPE$4PKJG9|q!JY+Ej<ubVnp$wJ!6ahCW1VV~FI!h0
z4bVE>9XLDPNqyt1T}K;CrtRd=c|9T7h^_njWtIN_CEY&~Zt8dmzMIyzI8)>P%pITE
z=A<Y+WopTbst@G!4%paetzGSLaP5}PVB`A&CASrRb=?-+bX2@y*M_`{Mk!ZWb51JW
z7by;|w+gA^wyU`1AN%A&^o#0=#z(v@I-M-y7AL+C<XbEvdiPoAH1>%MY=LY$Hmgj3
z+-4e=8!pauSSPV~&cRbx?<Y>V)|4#&YG2F!6A$;kI2GvkPG2}sQ2A(o%Tk-292$Zt
zv6{^VhJM#yUUZDQsVs2pr4<8HT#4P=+fRdM-dr4Z)7@Z})vE&;OKZIN)H2)FafB;s
z+o?8PNbhI6mcSSo>#|Q>NYd}h#|zu<2Pa;LD0tdD#Z*pQ#Ash(aZ}2d3BmDCdqfRR
z#TF(XlzD&jOQ>J-)aJa$_jvWA@-OWRvyoC!Tfn*S{Y1|-Wwrqpr^QUf);yM-;HdHT
zM(-!DgeeD;&Wq&zalg;UtRJvXUnh~LBJs?NsW)Fll)8r|1!W&DZEaZ*BhPM8_UFsZ
z%177d#lCtm`M2J@gYW)v3yZ$n9cUzcx#My5j1vi8w>{dLKL5tP4UggrLqA^?=f7W<
zZr;~cy;*MFY5seqFSc3RG=5E=-{7D+M^}XVx!lh^AC4@XZO=DZviI4&UqbEX`wTw*
zIB!>?-*B|QX3h4wZ>qnuU-`J6SwP_LTrq}-unO~s;qMOj9k!@@)%&|qatXVA&F!$=
zzdxkcJ@0?F|M&a<`t|ShYacQH|I+^N)ce1$&euO~UC+z*zxJ#E_xxX1mjBxww?X{=
zx(nIf&$n-_`q#Q%N4EY|_xU})ZiUPL{I&D?^=W=SvsW7MB!=w18|L-)MfbAHep56S
zTs>N|*zj?vmiQI7je9MtA6)ozG249h>sSc}uaztLzW7<^ZfCS9f27lJczS%WgJtga
z=cjHT^*X#d<oB+}@A<{Ew$?4(ym|4V4O|QEuFKfVyx?w|;jYVaELR=Z?3((jMA4(X
zWc6jaH}(dX^A^XvY)#yH*DXE0x~e`j-s99%qyFqoVejAYy!Bl6U4G6>uJufpFD?sy
zIm3)$y%Jj>=Tx}{ziQCMC>Lhj(3`%yYITgJ_=QEbA{V%%xKH}!UI=^YwX^#tQ&i%O
z_gmQOB3^}i?YsU#E7zfFYT3%@-ZGvGY+8vCthe5l?atP^x{6`L-l~P%B|a7ztD{l_
z9K1usG9yElX9xL|25_{zT%dEoFsj2Vb^-T=-Tb#!>GWUIYM!<)Vw(H&Q@8c!-Mu9~
zB}39-t<i*scS4ru-s+Vz<IX58+opZ>rQQYBQ%tQ2GA`Dw3oc&@x@J`vHEpY&Z5hv8
zF=zG*+J{R5-Yt;tSh?b^?&2U(-P~27fosbcE~KyV%vX0>-(z$u#3N!#D3ev3>lfw8
zWe-AI!UY-1lwOA3`kQflwaFf@-SU#_l`@*QG;e86GT3y0p>_MrX_IYjUhd_%ST)t`
z0^6awEssNQy^~sS*D!U#T|-@Q<Egu!_GwPH%D!4)cyH&7-^Lu`zQ2=>&hEcl7rR9}
zZg29b(u;mR-QPfW0y&)i;`#G&I=klT%;@EFx#rGEQn*yM`%S3OWxZ}M$+w#vl2QV4
zr`xV9wO*05>hgWP>SfFen6^w7N;#Xo{bgN%r_a(g<)^M@&3Tn)%3^T;<uSFjovW*F
z?78#0vmh@!eP_^Izp|{OUdb22j(RoM?PvD6JZqDUm-y9(r`K1$J96_i-&vkpk`Hc(
zPucaBPxpf9Hhm4oh~C|+3%~DjHs0d!8g#n%vF!EBV=tO*5sp&WQu5<d_OnyBx0qLD
z#Rn{k6<qX4_LK747Bh<-`TsATe#W&3bemMLY{k@93Idt4&-E|kzEpC-mDORX74HJI
zqYAeyuj|O$Jm6Ile*3~;&9Ph$%|~WhqKpgFb}wDGMC-wp++8cCXxmDaz4$J1J3in_
z%<3y?*4DS5diirmuC~a~aXGuyuR36*pMPn<WT)9h;n@s}HhD02Sc{##zU<`(v8>)r
z9VOluA22V7(VendvaTz1dyH<$EoP}HTp3?)88iEofAMULWN7C0{kttXZQG(pU*Gro
z$=cWao_lfS<3D;|)YnViPuRXZqw)HO&3hVutJ{Z36*qt5Yj}B@@BNziC11PE>;JrQ
z{$6oZsl}q^wff@V^|Jfkf=;Orb(&GK;s0IvjD3IZyt%zjn&%g{`JVT8#N|ygxfFLi
zI&XXH<Kbt@XLNU*=4s%&`|aeDFDHL|=}aoMinP1gyY&Ot)StC`*B+UzHM7Lv<L2jn
zP1PZfmrs`y&;R%D$r;VRzg9D_S+AYjyY<0~tH#r1cl~(snEU?keJ`$Ldt5$kSMvI&
zzFou&t)t81^=BMxe4QUr`_eK{V_R}<-P1GAdj9OK|Ge`2ziaE)ExQnCa>Gw*^XEgq
z)kLn!HGf?$b7$Mkq}-PxUvujI>`Siy@xXci=XU%0|BLItylC!UYxDbpTX5wr`|BJh
z-9;SYUpp?jK3gqo-A>{AzFRkz{@OCJcb5QbDPPBp!YS#QS9V>sog{ufZ{y1{iK<S+
zT5HE86M_tCS=;lMzk0o3ql2)Qc164Vv>5g8oq^&rlXvHapLts3yp=^bA@&S=lJ;x<
z70FTaCworaaE@i4z$UTkCpuo!T-R`H+<8L4LBec;rAtde%i9@GF8K+t_l2{{2+o-O
zr7`)#iR1q1X<8wU$qUui@kwz%*6pg*=Q*t5$9(^>yK%>gEna7iJaAtnv&A<`<zm!I
zGo5?#F14DQm$e*8;Prm8MtExN?Eo*{*?Mbwul{|zjO)0b)VI#6vP%`c*VwdH`;_lC
zO~1a7<CH_jnJNK4b+(DOzU}=Pw!v{r*pjfCe7mdKohL1{6gj3%e4aZkaY<_0h6gA1
zYy7&#EOuhkiY+I;geEO{%eO@H^{g364$FQ@RkCrH3a0K<S;V&0YN{85Zn2Dm(&2er
zKTj>}Zr7jmvn?TOrfI}ut<^H#uhbS7N3AZLA2fa0p?5sfewRItbp5`v&#rRUPQ6qP
zhf^B_TCOCW{?4Mkg>7Qy&q>xd0xo(g3ViNeBVkaiD|R?&l4HsWO_c@<UasWpa>Y?g
zLLZbY+}$fb?Zm~9*K20(y?jvCXV1^X&KVnaIVg5ao4s;3`wrWKQrsI~?P?Y}R&)AW
ztj%WWPE)_&-tN_9lO+>{Vy8ULFIwW4`~TpBuGMK`>ofL!s?p>Ct!ud#lKFJvdG3cx
z|NeXAE3dqG<<sPH=?4$a${QbBWB+}Lz@o}OOV{1o^I>Yc(Uiv~)#qd{<=s4B{%Co@
zw;Q%<vO4#^Z2Wib+rIBLE4IBin#@;rO`pl)!@k)DQo<|ub*d-+vH8RQt8r=O+XG51
z8dJ2><@TJGxO~g&-Oru#fBfBT_jmjMeEHw=?F!EJ|C{^&ICH;Tv|TCFZqU%-<(~h4
z9=xxA$6xmsbXxM4_vZEA!|UI3&)@lfWBdF1HxHl7)o1*?l018TtcU;l`Ky00q}{K2
zpzGc*v(t!W!jk3Ye)@AQu3pKUpZ_ZM%;Z-qmUEZBls+T6R3W$e?&II>OaW6K?huuG
zxOQLI=iUnw9Zq&;^4aTLyz#{7!=*y51k=^CUl!NiSlhU~WD>^#193ASnW9JQTfRn=
z7Av%^z5P=+>FQH^?#2gB9<$h^x3aGdZdg*iC%g2p)eOO}GsN2x7YnqkYu&Q?Y!Uat
z9U<1=MHWcfiaZr+`(diXlB9OdA#Tp4vz}2sqCtlpH%Tnrcc>)zyRBnhM9jHozf4t@
zc|2{&dB&%3x`|POQN}Ct)Ojxk-S``i>=Hzr5_QU$;|h$LuUSu-+<fBX+K3X@(=2O)
z_C9i2owS1S%haCBb*sW(_^7Gz9Lh|-)?6@g-xQ0tn;qsEOshR!6E6STHN~z#@6sc=
z^QDZJ5(*b>(7SPFuSZ1LX?KN~hkg&Ll?)wTFnXj^9J&0bEU-9ND<t;J^{qvw5ezF5
zny$Pn$h-0Qrq#8~k5WBf&lieDz4JSExn>=kHv7dQ)6U1!qV`1J=9+A`(q-Dm>5=PG
zo1@kwGERu*iRN(f+kJUj&JsVpBZupn&V7GY)p$waaCL%l`Mlc9+#QE5hU)ZaTW~i9
zoH~E<Xw2!=JPKa(RTxd46v@B)_C~1T+nb<Bx!Ak)mSurE;-e($IDO1E-YckS|8t`^
zq~zDvCvSurc9!pZ{bzdL(!K9M3kTIe2M$eHxVSd1?zQ@K`Cm2+$(J7JzGn!Sv{}vc
zgU#b5ua5N>Y|l|BbL>*?-+ouoCjaTh-wZ3l<FnSzo1R~3-NSg~sJi^~?I+$An6JH>
zmpyI2X<fo?&p%V$-~GQ{|9kzvuh#Z|w*PzNF8}Y6^5fpBZw%Mwr`7ztm%RT={lClV
z@s+<$hR?J6`(gR~`nN|~kN3w`{JHUY{oEZDudcL~@j3*yUBAwiZ&~%kqjsa-f+??7
zuZh~wQW}1q#o<Lxt@vjq>Dt^WL4P$of><3iT}`i+SU=oYv`+9+ZR@q&i!=_DUYWvh
zZFXemrY~Jv1Wd2T6t8*X@y6y->V}eA?_W!l^~|2cEaWjUUc8WDTJ)>~36eJ#7+!fh
z>Gsh*e!l~Yd;h6;hFytyeDT`Xsd+zD-YiI79KDjw>GYb0vJ)!Pti>iPwdBny@iTh3
z`R@hgMHRBUw=1<fO{l4Aop7aogPub`{^{xG{&wEl5_n$VtZdz#CevNst%)oitK`pY
z{kyW;Zfn`eFV5HUL?%mUMg?<h;>~F6l`fk7asK12Ey-8HnVp+gFPt$~C0^JyoMkNo
zhoQeosDR(`msRne3`YZYA6m03U}Drl-Iv}Qd8U2ZTNxhl_)<9EL9y?-<*q_IFL$nT
z>+%avQYn`1oOmQ5vp4FFK1;&hz!aZ#@fEQQTC5MgDaCwQ){*^?DQI5>2k(XV*Gmsb
zrEjn(TF13pRALubRnMg8lqs&c1;;WseB9f+KE1!<lSx|m&WrUEihdkcoW*tcW^0=D
zg(E3nPx#%;`F44+y=V4Ofv$G(?JNJBdM4$&Wyc}$0}7|ip7C+LTfIW*xKqkm)icUm
z<z*)?8RnGhOp5!MGPP=FnA!R_FD9Q(-Z$$8tM-#5nFR^ryI1D_df_{j@lEx2_GM|W
zp6~y&AzbH>*ZOJldtP5yJHMe-_B8)pWv-*l_sffCEjt???{WFyxo5Ksk8Sn7;j22W
z`#+z3YjU~%y+4HvtzM<yWbZRO%&%Blc=l_9WXY|#!dDv}KQIl<Ez6vAzV^?xpN*2q
zan(PM&i}J;|DWl3b=Uj<zW;xFdSC6lZh`L%*XBFr<gKZFdsDss?eqGow>Q_H+xNNc
zDf@e=l3S~P>)$W`c;?l=)0qo34mZD*V0g-sI3;Yo`pQ??YqusnI+eO=r_GHMZk(@e
zV?B6w25K{SF&|hK|K=srK}{*`ZqQZC-#;8OSi{ZelJYFBd!j^3B2R+A%$){ZXM8S1
z7`S~_=|A#*j`I_{3ld)s3hmNtd6KNA{d|!~N~7^CrV|Mq{he3dO*pgur6!Y!>YB?Y
zQ`my#g5ENwG_)mz>B^mJi_KQuweu72rL|l~1Ri=uSaWW7&^-R3P<_+J^|B1t<_7P*
zXYx*0?10Z2S=)Ujc{eWIVr+@o;I@xPC9^;Hspmn1Zpnj3axStzFFNGYY5J_$v~R+j
zrc;T_3a#1V=HFia^uP}re%p0A*{w5T($88gW%69)w?_1HP~olqD_#O=9jnC7ew^iz
z%WW7hF5oz`i(Qtj`^r1XNy*8&;U)1rF7}_g9eJ`3aw)z%zv#<}xeMQ=Uf^mty?DzX
zrQ0S99M0zp4>Jic2!5Sjvj1uDN|i71!r{ATFnp_;H>p|gS*JqbeA)erZj@hHy5?A>
z`t_tK8*WwnWM#L`d8;<_ye#v3_o|~Y$)(nJ7kyTnS)i1!m6SEHROpPKRAmyclJogn
z-_Cbs^KiDOzLoI_(>*bZu{`RiV~yGEQvz>~XgW>*xy(dKtuXn?zXxLc@1E89c)Wf&
zJ?=vO?xTOj)WY^APT_jAgGIe1R$|s&x$R-oK>JQ!8OOiNRb}9-dy;(O4gUngh-+GR
zXOvucKkxS`XEoD<(f23vU795S?*9SZb!RfRt2}Tyu{C;;@pjN20>(G>@7wilK4z5G
z&Hu3HUYSzMj9K$MfA4*o{QbhkzW3?dFCDsj?)>(H{A=#Nul>1D|Ie+x^M8A9-}hR)
z{+0Wz*Q=`;Uf&kU7C&cO_W$nxx5f1z%Ku&B|MSBC&xiAWPuJI-um9bfw`Tcwd7ZH3
z=JA#P-W>kEujJvQSH|&bGhEjvm9Q>Y|MggA>8_po?wvWpQlh-vy2^e2o0m+zrqRp&
zh3|14-f;d;Z=k@#rWc2eLWLL4xM?J*@FIKii~~$l7#tJ@oIHxS&hGqnbArTM@As2F
zxS8$H`*n14^syHfrnR|`N-Pq$3Lcymefhpm=f{})x(^$3ZJbW$$W6Q}Xq#1(bVII6
z;NZQ&)Ha5mXF7`g`?nn4WD_MW;Zkx*C3Blm<=KdI=~>sGWgWaDWUyO)N2W5@A)jyS
z-$X9*-1JVytkgo7`GVKdDGXc7`er4FC9c`^g1P8Lo9cmm%#Zn4g?rZ+oQmz8&eC^F
z@5$ztYY)1Nn>E;)j2gbEcduL1!L=`ZO}CDs#7a?rJ-4jgdLaUOZdt4x%3GZ{cp5sb
zrkJSMiZxiomVfSX;q|+c(3@t%%;5TR=hx=HC)RhLS!`;UaGKL<W{5|IX@n3%B8R%Y
zG^1mz_A||&-s<HHmy&lj*;g%bO4wCzeB}5HnG0{LPHGoht-2Ge=aKZ`$loQ!%ianH
z_dfpV_(mv--%|ZliEz}TO7Xv3OGH`~mZVwO1Qu>?|57ylppt0#HIIEepS7$0Tl>_;
zo-=%-S@$vR>&E#vl1+3BS5D(L%KLCw=xvK(SnMyGPdlQzrPuHM_i}oiMa}QIpRTr_
z{*=*Ib(1%x=eSGaWtVf$E~$jwdcXU&@T{3LLSy!P<&!9W@OS$XwYwERe_mN)o%Q&S
zxV+Er!@G7qy(o9}-LE@`eiXiYwY=cAEQ4OnQ$Ysn*Q<NC$3|;RRXFm-I$ywv!(Ydm
zwP)w{85Psij3Vy)*>3<%NQRxY|26mjr`P*GtpCn%ZM%WqIvx8zKkfhgtpDj>|Kj}p
zZ;$Q22it$w|MxNf{}KDxxT@N(PoJBgpQ9shArkkuZ!`aW>)+?fx0+v1^m+XDx}SUm
zOTh-&<(n+7b2xJ@4Y6P3c7pAK$v>fT*E{>HKkYib`Jqfh5<|y?7pr@hMH#y?tp2T^
z-dY{MdZ*irQ-$U$IUMdvvEAAx+Hf`eY0U4EWoxg!I~7wOcBs{aukOLE(=8KRHDAdU
z?G*jqtG4cWN(|@gJZ-t}6W&zlCiW^V74b^7J;%lP;#%}~hL)L=mrP)Yk!Dpp=^^eU
zFT!9ceKBUv6J?!U?aYj^!pgt0GQvfCi(C3P3$&!H%h=qQaOjLz^n;?PXuCD5=K4)6
z+wykSzK$azuNSTVdSTZ|Q&oBMBe#5$wn~31vh}W9x;vVoXX%@w7prb*o{%!(SS~Dl
zEc5>4?$U)F6P0+UsT&uD3i$oLXVa*gv~5|TfWwk)okv!2pW=V^<n4|Yw(eZ1UshZB
zA`U&$DmpOfNdxPGcc~3iJ$1g%Dqp)HhV6BTR(mf;sz<5Bleh2uIPV<!V!?63Fk;T9
z(@Vc)B+Br=U3RU=+Ir`dsk`TQrU?{$iJoRQ?T=H^rnQVy(>Pq8&G$3vi(8br=+GhG
zaH;pzH~V)9aA;0Gvfzb=vS#k{Wq&-Xj_&A6zVm-kOX;=Dz~dRy-tA=4xyAofrqcFj
zY@!ZF&fDD;&s=ZovQIbBzoFZ>W42RG&Cg3J(=<eurM>#|{9&|xXW)l}+}iwib9*em
zA7xMUc|7a2@2s#xkqlEp=bVu4J#G;<hod~^SNk;m-v@M$X662`w!8A@Soh0>fZO#S
z9!>7AUTS1*`lQV6<B_<ggTKsvH_GmM%Qt!1@_74w>>K{IJihbkQuF-Zf9L<bV*l&m
z{9haQe-zLA|0TKq&VTp+zA`(1U2&fOeoOhc`X7tu&Yin<EBDmD({ttOBP&{iY!_&;
zO<pFx?^A~DEtOwc2UoWXc`fFfn<8wn_nwK#zg@n@UvB3qHnmh_N*RWKkGJb!5Y%Fp
zV4b{kd-X2<g!@O0ZC)?hzWA%F<hfRc0P#b9ubs_!zSuhE?%z>p`?fKyXidTLj~ZdI
zrzAe5txK5WCZAlQ>t-l$=e(%HoLP=j8hN%CvG*k$O5JcGY~HoYDp$*dm1~wUMt)x)
z@;6}XY{NCj+AcdV)J)`ky3W%2`_oic4vqDK(>%62=0B;k3M=dtS5&i#%TIF+3SPKq
zx;T>~Z_Sz}E$;4ho6Mh<^-j@VGFxDGTxHOXUu$>GV4N^Rx2%-ih~?z8chd!y2S4f1
z>nb!cTX*5nuOp5oI+GNVwf40O3Mst^=4tJ;V%fYy%fRn?=RqY~vGq^Bui_S6nY+qR
zh4)&9=)z5p%p&IhvZ<X~%jMn>dL~NXlJ$!>fhTmlGfI+-d}rV8o^Wnwv<HWRpKq+T
zy^-pg&1~KEvbRhUH(H2iO<?GDamp)QIQv;~^~)VW-wOk_&K9iE^R9R)*Sd4=ZvFJH
zQ~f%%W#d$Ij?29N_im0M!w1`iTRnLeJ(J?J+G@5>eABtP=T`mwdX@1?(s}`od9|F1
zv2j_>tg35-%Ux3*9@(xga7_5<M8>o4X1>&p*v8|xKF_LjXOQBq(?XqFcm4lV*m_~<
zboEQdkH1{~@#`*!uCc_K`t#N8`pah>^yIm=Fxu{==yJW^4|0#qI6Hm5@v%2K>!g!A
z4xTCu-+ge+-43hpKC_RH=ls9q{CDoPqs#YMRKMl@c&YR4=k(|Izt(2%Pw$yILut;8
zg}YgrzrOv>Y<x^-z71&A_m0FK%iSfP_MQLtZvW4T^&hU=eXFg1eE$Bonfms1Z{FEg
zJp2C7yLx%)t9|xwN_VTj|9v=axvd0`pZ$lOSC+8n|Et)uCuH}V3l*Zy3!`@LlR9te
z#Nc(2_Xf|RGkpg|%Y~;^eS9NUyZ*|m$bG*I*2d?|*9y40Wk#r!OUy+d+wj{?TaTPc
zXie-`lRD2k!1Gmhpt#^~O@8(rfeUR}qlF4~u-Nt9aysF}u(WM*NSB4_KK00;S;n2m
zG@4o6Jsgu&!+&uo8tbXECW|{ov4oY?1d3a6Eayx+y7J+}8SgZvtXZgheks$+V#&_L
zzk!dA|DAPgsd3M}9UO%MGo2P#iXJ-pwr3sJL4h<&#YE9}TlR{lb~-tzrF93jlwLf~
z#c3GiZ4tO)-)yt&IrpW?6Hj=ac)3v^L+`0_y!JH9<C**QjM%?LnYf=1in{Q<>quar
z(1pp}b9lED^a|ZnsM8M8K2ou>Z1Fd3X`=|s{HvU&jJ@XZT@Bw>>*W@j7SENq#<69Z
zeM#UVA(nrKFFSF}JYg^~@j`?GTen`wqjfRK6YqRKb-h^p^J81CVh?R`y)#K=T-)3v
zl9x6Kzh*yPK7;$-B)+sYV*Z!wzZgH!Tl$CFi)Sf^>6+~ZbFyaaa+tFUbdcw%tn%gS
z1SF2;+6Zh6<(ljK>Z#4fc&3J(21?D++Y1%<dY^mrLx@Rd`*w-e?x<<y`MotSpWc~u
zcSCsJs+lM5R2*G-b@{%I*aY`ucNp69_kTXKad+$Ui*qx+$-ZX_m^rsSbwgX!4{xVq
zA16<+KF6tYEw`M(;>W|=+)F<-e!sux=hx?qKf?E={@uBL)<FjmuG7cg-Sa(c_wV3w
ze{S0_sp2m=8_f>C7SI3p!^rf~$2~sN%Wli2%kMfaF}ck8_x-<&wSQi3&fmRH=3MNz
z#@k_XyXT)f{Lpjhl%0o_S6=%TwugPgKleitn`>Sjc^v=ulK<by`#<lmd9%0vvHqQ3
z?fx|{A6DD%F8dg>{j%lH*7M)0_8yJ4GxT6*&;MC5i;wSqS-a=ug9#fdPOGorVLbLG
z?C6At(u-d?sjQT8iCSK()tCIPt3b5B7Ub()&c>^M>(_sLb8TOD`igC-i&qLby!MrF
zWf5%%5#!(XczJwPqds5ShGkD0UT8}kRg0+2?K*cvgyE~h*{{`VnObui^`|==QJLc+
z9DhooC!O=|Cuj4;CSQ8a^~f=t&fKWeb4lQfTEiOdw<0yX{A<?pw%j>pdE?d51sfgY
zTp$0Mw2L8FD)GjN1yj{m+@JHCai@h%_+D3T<!hhK>^V-%w79&1`N1EBd^dyBZBP22
z=`CqYvaE?cGa)=!F{AOfP?WjOjrIBFKSk6y*A{W-U6^*oEd9;R3)k4v4o2Ge^V#0i
zeCjUI?R)!iNv#yS@0Cne>8%-jhi7kjwAnqiP3_f(bqTRAD^Ew+F#YaGIK5E&xZ;`h
z+`d(&dU9t(a@d||r%A5K{9T&Cuupx)5wE2yyH7~OUeDT_8FoqT?kDlRnvd2!@K`Sq
zDEK2TUn)u=SJn2YE!WTUcfuzK?|H5zQ2AS?`RHR$z3=9F)8xN59N;)1{f+ZM8~Y#Q
zl%FEDlfInVS#>bPyz?yM%}>Eet1|MhbY=@kYO<EzUHDd5cC&8x>}Bn~uLSQEUEf?V
zd7e^M*KV<A&jOFM6gu*A8lSoNW?yBejbQIdliIVk)trpiT$Em2`nkT-MciI2dn>d2
z4py<_PZEybY=8e$mP?WCF5lmsKR*?J(#_A$U&oxld(9%;q)uqx%a8sF8~g6^COp0P
zo9*to*7Y_?ml#_F4mGDfXnEedcvn8x-8;-X{w%Ivx9$IeJ)d;%o7o#o%92d}#&AuV
zck;63{(GvvzyJG7|Nr&)zuMdHeK7z3{PzE=_y1^vw&F;O>wl^KyY%<}71v+O?y2Nh
z?sxycd&|-u!!HrDK6WP9?5P$#{BQMrKgqqje(b-sd)|GGQwD$PjJ>_1pM}2^^QeEw
z$uDq(rJ22_bz7iLfzh!a>={XWW>zSDOmOnq^Xg`Er+Bgwm#@PbPD42#pB^tI6#)&=
zoW$?5=LY#$ThDJ#-&cNr?o3_rJs)Mwc5l94V{QEV>>1<pzxSM9eIWLw&fP0|dRt@o
zw)yxm6*(rIl&YK%z`>W6*n2a_(z5LCZdSqCovKRz&R&=G-|2FsHKbD`AY0>sDr>~X
zn-ix`KN<9Lv%>M?&H2`96`6~~6xSVX&{x=ath`UDsbTAqFO7*YwGpg`7Ykk8c;axe
zhT(&gN(&24Me7E5{A;hP*_xR3(6_U3SM<Hf9>4u&ybw!T)_P1z=jU`m!6Wa#CLYz}
zH=FJithf5riHmD`X5CRpxpq8BC)eEV`2{cY);X%GT*ASdO>QW7tT7PbnRYZHXldNd
zRiC(4@1D4~dD<p!=@jScwLQsMzS|Nuy!zU{Nse!tbKcUXe_JG1Etqf2uGr%5b0lfa
z*9Uec(ffkcyVhjMC9Ti68!r3X(DA4!Z=m7QR9{8WO+wn?8w8)S8|%B@D3#&Li%{68
zkmDYBNIEj&!Ma}=rJE&>_KKddVqp)LX8T_BMdw3A%E~oNovN|x9rCA7_uk}{#d6xR
zd2bNMQgfE~-OH8S-gb!YYAkBz*~Z=Z@`_gJ=PyCI+pSa<mE|OeWleu^a2hk)>EMsM
z=2vRDsZ2cmc!RuXX|>6NE5>fq&nG3UoT%YuwfRq%b4!tiy_rKe&$AEu2U-^v&lL5Y
zWd1?wi0*zKz7;JKPQ`~hP2RL=cSD-WQK7fi1?h#2IU;Kq_OD>Ky1DazW$oWr4z=cj
zmbbS(WGFmeRI$z`YW>^ytdkP18I+ZX?QfModsAVFA(!~-Lo<G6+^_d`@t(dv*y~}h
zO@?$~@cljK>JrN9cg1{TZa#Q*@>k|+t_|C=B6rt5zIC-YzV6?}-}(Cu|K71)f4;8j
z+nMY0YkoZVck0=*v{}oNkNa)8ojI*^g=SmukL?v^Zz>C`J|{Z4oJcY__GU&+n{|QM
z;aitN4I)@eTHiXJn18q_WAf(ZKf)~<pH9DD-)yUCJ|W`6wMnc0Z?5LzICFtxbxH8r
zhgp*)l4Bni3$$_1kxmI>4zw$2j5>Td{l%3VZx(3HSP>l=H;Ly`xOu?ZXKn9S9NO?;
zHP_SITr+!E^X><WxN<#|yC1l1OQeU{h2{nR>0i%WX&1Hsdg?*P(GDpK9+y_}U-M3}
z%CzpC<ap*#uR-3LiY^<`z<J)S=5tGz@<%4Ekj-w3oOb+`m66MnmWiy<iAmQq%q974
zr>^^LYAzrsBU)CWb6YO^Xk1y=i=`dMGo~`SZws5gj3+|Q#o1+Uh+{z5zTp1Z{A-ug
zc6=0@o#6PnPu1b%f+HQXrX9>OGTVCZ<JFHB&aKXSwsYDx0mU0SD>fNSk2ogYwq&Q1
zqM+c~pL0`8j?_Os#?|t6fh9NRs*Kmq)+bbKv0L-wot~?!iptwbJ09^py*0hHUq*h)
z%f>ksiq9(*FHi7V7O8SUB520OB)iPqyXF3YNlj&MD!PO&FSAj&cdRJ8t8H21fdy6x
z^DGQaZWIYQcX9^Z_3}AzvU%#uIu)mu329|Z_mqTd-d$jIF7g?l<F<!8`P6sIe^qpM
zTxH|g@^AaJYT=7d-mE;#&Uf$LCEo6;6P+pNPd&Jvva%yF(bDR5c0C^tpXEC1tImS&
zcbQrgG?lE(l~-8AUjN6-r6A;dz^fatc5fGG2`$Qenis-4>D`}##*S(0W0V%%dZ5rH
z^vv8n`PlOK)UDNwH+{~{Q(CH)_Uz=0lQS!g(jNWTQ*?K4@$#vrtaE2rYW_X&?%yBf
z-;Ig)AF`Ldeq{dhkot4>Zz2hzvrle%T3sLi<5l-&`8~g{-Ff)^p3TRb$J5uv*vOkp
zCttRlV-#stSTj#>>dHmDpX<F-46pFZR_2Hu?)$j+!DKm;_v_7g7Hel{o>HtAI#^eC
z^}L*Q&4ULw_Uqb;EOg=%WXx~eG)=h4^UcJLHNQjZ61{pphMZC1<<O1TxY(wTvt97S
zVdK>zZV#N=&TTrNb+2OK{1?+W{<KbM?mSRFMgHqnqr+BO<|p!Plmpjn4%t=z_2Q}?
zIk7KdKhJKh?#s?zW_7{n+MSLLhj~w%6XrNgF)%Ff>%ZCAwWf(BLcmITkJ9GO<{7MK
ztyOO9Y2CM&VRlsH=BYFD)`q@d6HBvHTw<Tc@aM|b=xG-&9SCiRyuy38$h-Vt!dIaU
zm#3{Y+7~vn>xOTG>4wJvshs=U)Pf@&P2R0c)An|ZbUc<97;^Vh=<1xC49B=_Z{E9G
z{;IRH;ogEZ>4ssBCSe<&YOP__J<n<B!f}D2nmb}wwI7F&wxI|Ym-gmYIW<Sl$lRal
z&NlG@8>3jH%mHIb6|n@dQumqKyc<(2WvA^rJ@3-$y_Zs_m_`VhiAw0aa*TPXePLs)
z@!^b3^A^30INmGzKxOXI&V^-fHq76hyX%49nZCvI7M0q67OB(<(>kIp^!LK@aKj(G
ztNL9Em-h2iTYb5x^*K}^{9)E%F&+;6XHuJVR3F}GiaIy-OX$M8`%3ukF7aC-KG#{5
zYl--Rl~ZI+KHcQJBY%I<oRxB|CCB1s1<#Q*FP_v<wzpu4wZhW7Z}01wZwkLMX;)*&
z!{kb3?q(mE`reLX@pVh<UU>X^F)=8spv0e#@%OLyI(iz%pZ4nTrJk94ly&KqOHV)N
z#sB~GsA=__r$@Q-)^1BtobrX)I8yEJfo0G9{@2$f{AXPH@Y)X3#Fb_Y3=9mOu6{1-
HoD!M<W{h*h

literal 0
HcmV?d00001

-- 
GitLab


From 390959cbb9ab81014ca6f68456ce9058114a7d78 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 17 Jan 2019 14:56:45 -0800
Subject: [PATCH 0925/1095] Internal change.

PiperOrigin-RevId: 229823501
---
 tensorflow/compiler/xla/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/compiler/xla/BUILD b/tensorflow/compiler/xla/BUILD
index aa43fc8dab..636e5ef721 100644
--- a/tensorflow/compiler/xla/BUILD
+++ b/tensorflow/compiler/xla/BUILD
@@ -826,6 +826,7 @@ cc_library(
         "debug_options_parsers.h",
     ],
     hdrs = ["debug_options_flags.h"],
+    visibility = [":friends"],
     deps =
         [
             ":parse_flags_from_env",
-- 
GitLab


From 099b9479b548c2bb1b4460c2fe69ee3c319ed56b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 17 Jan 2019 15:08:23 -0800
Subject: [PATCH 0926/1095] Redirect python.training.distribute and
 python.training.distribution_strategy_context (which are currently just
 forwarding stubs) to python.distribute.*.

PiperOrigin-RevId: 229825818
---
 tensorflow/contrib/distribute/__init__.py     | 4 ++--
 tensorflow/python/eager/tape.py               | 4 ++--
 tensorflow/python/keras/engine/training.py    | 2 +-
 tensorflow/python/training/session_manager.py | 2 +-
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/tensorflow/contrib/distribute/__init__.py b/tensorflow/contrib/distribute/__init__.py
index 1bcc453a7e..59d76f5d1c 100644
--- a/tensorflow/contrib/distribute/__init__.py
+++ b/tensorflow/contrib/distribute/__init__.py
@@ -35,8 +35,8 @@ from tensorflow.contrib.distribute.python.tpu_strategy import TPUStrategy
 from tensorflow.python.distribute.cross_device_ops import *
 from tensorflow.python.distribute.distribute_config import DistributeConfig
 from tensorflow.python.distribute.distribute_coordinator import run_standard_tensorflow_server
-from tensorflow.python.training.distribute import *
-from tensorflow.python.training.distribution_strategy_context import *
+from tensorflow.python.distribute.distribute_lib import *
+from tensorflow.python.distribute.distribution_strategy_context import *
 
 from tensorflow.python.util.all_util import remove_undocumented
 
diff --git a/tensorflow/python/eager/tape.py b/tensorflow/python/eager/tape.py
index 56d696bf92..e5d6007b48 100644
--- a/tensorflow/python/eager/tape.py
+++ b/tensorflow/python/eager/tape.py
@@ -27,8 +27,8 @@ from tensorflow.python.util.lazy_loader import LazyLoader
 # distribution_strategy_context.
 # TODO(b/117329403): Remove this circular dependency.
 distribution_strategy_context = LazyLoader(
-    "distribute_lib", globals(),
-    "tensorflow.python.training."
+    "distribution_strategy_context", globals(),
+    "tensorflow.python.distribute."
     "distribution_strategy_context")
 
 
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index e6ede64573..b80b906f67 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -26,6 +26,7 @@ from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import iterator_ops
 from tensorflow.python.distribute import distribute_coordinator as dc
 from tensorflow.python.distribute import distribute_coordinator_context as dc_context
+from tensorflow.python.distribute import distribution_strategy_context
 from tensorflow.python.eager import context
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
@@ -48,7 +49,6 @@ from tensorflow.python.keras.utils.generic_utils import slice_arrays
 from tensorflow.python.keras.utils.losses_utils import squeeze_or_expand_dimensions
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import tf_logging as logging
-from tensorflow.python.training import distribution_strategy_context
 from tensorflow.python.training import optimizer as tf_optimizer_module
 from tensorflow.python.training.checkpointable import base as checkpointable
 from tensorflow.python.training.mode_keys import ModeKeys
diff --git a/tensorflow/python/training/session_manager.py b/tensorflow/python/training/session_manager.py
index 7bd0891e35..104247e60e 100644
--- a/tensorflow/python/training/session_manager.py
+++ b/tensorflow/python/training/session_manager.py
@@ -21,11 +21,11 @@ import time
 import numpy as np
 
 from tensorflow.python.client import session
+from tensorflow.python.distribute import distribution_strategy_context
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.training import checkpoint_management
-from tensorflow.python.training import distribution_strategy_context
 from tensorflow.python.util.tf_export import tf_export
 
 
-- 
GitLab


From 1c73610b59ee174a2d1d54a3624158bb199a6439 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Thu, 17 Jan 2019 15:08:38 -0800
Subject: [PATCH 0927/1095] Internal change.

PiperOrigin-RevId: 229825850
---
 .../tools/ci_build/windows/cpu/pip/build_tf_windows.sh      | 6 +++++-
 .../tools/ci_build/windows/gpu/pip/build_tf_windows.sh      | 6 +++++-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
index 7ec386a60e..2eeb8dfa1a 100644
--- a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
+++ b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
@@ -104,7 +104,11 @@ else
 fi
 
 if [[ "$TF_NIGHTLY" == 1 ]]; then
-  python tensorflow/tools/ci_build/update_version.py --nightly
+  if [[ ${PROJECT_NAME} == *"2.0_preview"* ]]; then
+    python tensorflow/tools/ci_build/update_version.py --version=2.0.0-preview
+  else
+    python tensorflow/tools/ci_build/update_version.py --nightly
+  fi
   if [ -z ${PROJECT_NAME} ]; then
     EXTRA_PIP_FLAGS="--nightly_flag"
   else
diff --git a/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh
index c076157b08..4cca4f9acf 100644
--- a/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh
+++ b/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh
@@ -104,7 +104,11 @@ else
 fi
 
 if [[ "$TF_NIGHTLY" == 1 ]]; then
-  python tensorflow/tools/ci_build/update_version.py --nightly
+  if [[ ${PROJECT_NAME} == *"2.0_preview"* ]]; then
+    python tensorflow/tools/ci_build/update_version.py --version=2.0.0-preview
+  else
+    python tensorflow/tools/ci_build/update_version.py --nightly
+  fi
   if [ -z ${PROJECT_NAME} ]; then
     EXTRA_PIP_FLAGS="--nightly_flag"
   else
-- 
GitLab


From 28401a50f1839f44c76f7b202c8766ecb61fc569 Mon Sep 17 00:00:00 2001
From: Reed Wanderman-Milne <reedwm@google.com>
Date: Thu, 17 Jan 2019 15:13:50 -0800
Subject: [PATCH 0928/1095] Automated rollback of commit
 2435a1875b574f5d299b1dee431ab5ceccd6132f

PiperOrigin-RevId: 229826749
---
 tensorflow/python/BUILD                  |  6 +-
 tensorflow/python/platform/app.py        | 98 ++++++++++++++++++++++--
 tensorflow/python/platform/googletest.py | 12 +--
 tensorflow/tools/pip_package/BUILD       |  5 --
 4 files changed, 99 insertions(+), 22 deletions(-)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index d17c57b05f..d4e6cbf804 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -213,7 +213,6 @@ py_library(
         ":pywrap_tensorflow",
         ":util",
         "//tensorflow/core:protos_all_py",
-        "@absl_py//absl:app",
         "@absl_py//absl/flags",
         "@six_archive//:six",
     ],
@@ -234,10 +233,7 @@ py_library(
     name = "platform_test",
     srcs = ["platform/googletest.py"],
     srcs_version = "PY2AND3",
-    deps = [
-        ":platform_benchmark",
-        "@absl_py//absl/testing:absltest",
-    ],
+    deps = [":platform_benchmark"],
 )
 
 tf_py_test(
diff --git a/tensorflow/python/platform/app.py b/tensorflow/python/platform/app.py
index 303b70ff57..7b917235c0 100644
--- a/tensorflow/python/platform/app.py
+++ b/tensorflow/python/platform/app.py
@@ -18,23 +18,109 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import errno as _errno
 import sys as _sys
 
-from absl.app import run as _run
-
 from tensorflow.python.platform import flags
 from tensorflow.python.util.tf_export import tf_export
 
 
-def _parse_flags_tolerate_undef(argv):
-  """Parse args, returning any unknown flags (ABSL defaults to crashing)."""
-  return flags.FLAGS(_sys.argv if argv is None else argv, known_only=True)
+def _usage(shorthelp):
+  """Writes __main__'s docstring to stdout with some help text.
+
+  Args:
+    shorthelp: bool, if True, prints only flags from the main module,
+        rather than all flags.
+  """
+  doc = _sys.modules['__main__'].__doc__
+  if not doc:
+    doc = '\nUSAGE: %s [flags]\n' % _sys.argv[0]
+    doc = flags.text_wrap(doc, indent='       ', firstline_indent='')
+  else:
+    # Replace all '%s' with sys.argv[0], and all '%%' with '%'.
+    num_specifiers = doc.count('%') - 2 * doc.count('%%')
+    try:
+      doc %= (_sys.argv[0],) * num_specifiers
+    except (OverflowError, TypeError, ValueError):
+      # Just display the docstring as-is.
+      pass
+  if shorthelp:
+    flag_str = flags.FLAGS.main_module_help()
+  else:
+    flag_str = str(flags.FLAGS)
+  try:
+    _sys.stdout.write(doc)
+    if flag_str:
+      _sys.stdout.write('\nflags:\n')
+      _sys.stdout.write(flag_str)
+    _sys.stdout.write('\n')
+  except IOError as e:
+    # We avoid printing a huge backtrace if we get EPIPE, because
+    # "foo.par --help | less" is a frequent use case.
+    if e.errno != _errno.EPIPE:
+      raise
+
+
+class _HelpFlag(flags.BooleanFlag):
+  """Special boolean flag that displays usage and raises SystemExit."""
+  NAME = 'help'
+  SHORT_NAME = 'h'
+
+  def __init__(self):
+    super(_HelpFlag, self).__init__(
+        self.NAME, False, 'show this help', short_name=self.SHORT_NAME)
+
+  def parse(self, arg):
+    if arg:
+      _usage(shorthelp=True)
+      print()
+      print('Try --helpfull to get a list of all flags.')
+      _sys.exit(1)
+
+
+class _HelpshortFlag(_HelpFlag):
+  """--helpshort is an alias for --help."""
+  NAME = 'helpshort'
+  SHORT_NAME = None
+
+
+class _HelpfullFlag(flags.BooleanFlag):
+  """Display help for flags in main module and all dependent modules."""
+
+  def __init__(self):
+    super(_HelpfullFlag, self).__init__('helpfull', False, 'show full help')
+
+  def parse(self, arg):
+    if arg:
+      _usage(shorthelp=False)
+      _sys.exit(1)
+
+
+_define_help_flags_called = False
+
+
+def _define_help_flags():
+  global _define_help_flags_called
+  if not _define_help_flags_called:
+    flags.DEFINE_flag(_HelpFlag())
+    flags.DEFINE_flag(_HelpfullFlag())
+    flags.DEFINE_flag(_HelpshortFlag())
+    _define_help_flags_called = True
 
 
 @tf_export(v1=['app.run'])
 def run(main=None, argv=None):
   """Runs the program with an optional 'main' function and 'argv' list."""
 
+  # Define help flags.
+  _define_help_flags()
+
+  # Parse known flags.
+  argv = flags.FLAGS(_sys.argv if argv is None else argv, known_only=True)
+
   main = main or _sys.modules['__main__'].main
 
-  _run(main=main, argv=argv, flags_parser=_parse_flags_tolerate_undef)
+  # Call the main function, passing through any arguments
+  # to the final program.
+  _sys.exit(main(argv))
+
diff --git a/tensorflow/python/platform/googletest.py b/tensorflow/python/platform/googletest.py
index 802721e34b..fe4b0d0d37 100644
--- a/tensorflow/python/platform/googletest.py
+++ b/tensorflow/python/platform/googletest.py
@@ -13,7 +13,7 @@
 # limitations under the License.
 # ==============================================================================
 
-"""Imports absltest as a replacement for testing.pybase.googletest."""
+"""Imports unittest as a replacement for testing.pybase.googletest."""
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
@@ -26,7 +26,7 @@ import tempfile
 
 # go/tf-wildcard-import
 # pylint: disable=wildcard-import
-from absl.testing.absltest import *
+from unittest import *
 # pylint: enable=wildcard-import
 
 from tensorflow.python.framework import errors
@@ -41,7 +41,7 @@ from tensorflow.python.util.tf_export import tf_export
 
 Benchmark = benchmark.TensorFlowBenchmark  # pylint: disable=invalid-name
 
-absltest_main = main
+unittest_main = main
 
 # We keep a global variable in this module to make sure we create the temporary
 # directory only once per test binary invocation.
@@ -51,7 +51,7 @@ _googletest_temp_dir = ''
 # pylint: disable=invalid-name
 # pylint: disable=undefined-variable
 def g_main(argv):
-  """Delegate to absltest.main after redefining testLoader."""
+  """Delegate to unittest.main after redefining testLoader."""
   if 'TEST_SHARD_STATUS_FILE' in os.environ:
     try:
       f = None
@@ -67,7 +67,7 @@ def g_main(argv):
 
   if ('TEST_TOTAL_SHARDS' not in os.environ or
       'TEST_SHARD_INDEX' not in os.environ):
-    return absltest_main(argv=argv)
+    return unittest_main(argv=argv)
 
   total_shards = int(os.environ['TEST_TOTAL_SHARDS'])
   shard_index = int(os.environ['TEST_SHARD_INDEX'])
@@ -87,7 +87,7 @@ def g_main(argv):
   # Override getTestCaseNames
   base_loader.getTestCaseNames = getShardedTestCaseNames
 
-  absltest_main(argv=argv, testLoader=base_loader)
+  unittest_main(argv=argv, testLoader=base_loader)
 
 
 # Redefine main to allow running benchmarks
diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD
index 3c1a9280a2..c51b45a49c 100644
--- a/tensorflow/tools/pip_package/BUILD
+++ b/tensorflow/tools/pip_package/BUILD
@@ -146,11 +146,7 @@ filegroup(
         "//third_party/eigen3:LICENSE",
         "//third_party/fft2d:LICENSE",
         "//third_party/hadoop:LICENSE.txt",
-        "@absl_py//absl:LICENSE",
-        "@absl_py//absl/logging:LICENSE",
         "@absl_py//absl/flags:LICENSE",
-        "@absl_py//absl/testing:LICENSE",
-        "@absl_py//absl/third_party/unittest3_backport:LICENSE",
         "@arm_neon_2_x86_sse//:LICENSE",
         "@astor_archive//:LICENSE",
         "@boringssl//:LICENSE",
@@ -159,7 +155,6 @@ filegroup(
         "@curl//:COPYING",
         "@double_conversion//:LICENSE",
         "@eigen_archive//:COPYING.MPL2",
-        "@enum34_archive//:LICENSE",
         "@farmhash_archive//:COPYING",
         "@fft2d//:fft/readme.txt",
         "@flatbuffers//:LICENSE.txt",
-- 
GitLab


From 097a123794a5ee83b2212991245c9ac0ff8f6c8d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 17 Jan 2019 15:23:45 -0800
Subject: [PATCH 0929/1095] Wrap word2vec_basic inside a main method

PiperOrigin-RevId: 229828434
---
 .../tutorials/word2vec/word2vec_basic.py      | 634 +++++++++---------
 1 file changed, 319 insertions(+), 315 deletions(-)

diff --git a/tensorflow/examples/tutorials/word2vec/word2vec_basic.py b/tensorflow/examples/tutorials/word2vec/word2vec_basic.py
index b09ee99768..805ec203b4 100644
--- a/tensorflow/examples/tutorials/word2vec/word2vec_basic.py
+++ b/tensorflow/examples/tutorials/word2vec/word2vec_basic.py
@@ -18,12 +18,12 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import argparse
 import collections
 import math
 import os
-import sys
-import argparse
 import random
+import sys
 from tempfile import gettempdir
 import zipfile
 
@@ -34,320 +34,324 @@ import tensorflow as tf
 
 from tensorflow.contrib.tensorboard.plugins import projector
 
-# Give a folder path as an argument with '--log_dir' to save
-# TensorBoard summaries. Default is a log folder in current directory.
-current_path = os.path.dirname(os.path.realpath(sys.argv[0]))
-
-parser = argparse.ArgumentParser()
-parser.add_argument(
-    '--log_dir',
-    type=str,
-    default=os.path.join(current_path, 'log'),
-    help='The log directory for TensorBoard summaries.')
-FLAGS, unparsed = parser.parse_known_args()
-
-# Create the directory for TensorBoard variables if there is not.
-if not os.path.exists(FLAGS.log_dir):
-  os.makedirs(FLAGS.log_dir)
-
-# Step 1: Download the data.
-url = 'http://mattmahoney.net/dc/'
-
-
-# pylint: disable=redefined-outer-name
-def maybe_download(filename, expected_bytes):
-  """Download a file if not present, and make sure it's the right size."""
-  local_filename = os.path.join(gettempdir(), filename)
-  if not os.path.exists(local_filename):
-    local_filename, _ = urllib.request.urlretrieve(url + filename,
-                                                   local_filename)
-  statinfo = os.stat(local_filename)
-  if statinfo.st_size == expected_bytes:
-    print('Found and verified', filename)
-  else:
-    print(statinfo.st_size)
-    raise Exception('Failed to verify ' + local_filename +
-                    '. Can you get to it with a browser?')
-  return local_filename
-
-
-filename = maybe_download('text8.zip', 31344016)
-
-
-# Read the data into a list of strings.
-def read_data(filename):
-  """Extract the first file enclosed in a zip file as a list of words."""
-  with zipfile.ZipFile(filename) as f:
-    data = tf.compat.as_str(f.read(f.namelist()[0])).split()
-  return data
-
-
-vocabulary = read_data(filename)
-print('Data size', len(vocabulary))
-
-# Step 2: Build the dictionary and replace rare words with UNK token.
-vocabulary_size = 50000
-
-
-def build_dataset(words, n_words):
-  """Process raw inputs into a dataset."""
-  count = [['UNK', -1]]
-  count.extend(collections.Counter(words).most_common(n_words - 1))
-  dictionary = dict()
-  for word, _ in count:
-    dictionary[word] = len(dictionary)
-  data = list()
-  unk_count = 0
-  for word in words:
-    index = dictionary.get(word, 0)
-    if index == 0:  # dictionary['UNK']
-      unk_count += 1
-    data.append(index)
-  count[0][1] = unk_count
-  reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys()))
-  return data, count, dictionary, reversed_dictionary
-
-
-# Filling 4 global variables:
-# data - list of codes (integers from 0 to vocabulary_size-1).
-#   This is the original text but words are replaced by their codes
-# count - map of words(strings) to count of occurrences
-# dictionary - map of words(strings) to their codes(integers)
-# reverse_dictionary - maps codes(integers) to words(strings)
-data, count, dictionary, reverse_dictionary = build_dataset(
-    vocabulary, vocabulary_size)
-del vocabulary  # Hint to reduce memory.
-print('Most common words (+UNK)', count[:5])
-print('Sample data', data[:10], [reverse_dictionary[i] for i in data[:10]])
-
 data_index = 0
 
 
-# Step 3: Function to generate a training batch for the skip-gram model.
-def generate_batch(batch_size, num_skips, skip_window):
-  global data_index
-  assert batch_size % num_skips == 0
-  assert num_skips <= 2 * skip_window
-  batch = np.ndarray(shape=(batch_size), dtype=np.int32)
-  labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32)
-  span = 2 * skip_window + 1  # [ skip_window target skip_window ]
-  buffer = collections.deque(maxlen=span)  # pylint: disable=redefined-builtin
-  if data_index + span > len(data):
-    data_index = 0
-  buffer.extend(data[data_index:data_index + span])
-  data_index += span
-  for i in range(batch_size // num_skips):
-    context_words = [w for w in range(span) if w != skip_window]
-    words_to_use = random.sample(context_words, num_skips)
-    for j, context_word in enumerate(words_to_use):
-      batch[i * num_skips + j] = buffer[skip_window]
-      labels[i * num_skips + j, 0] = buffer[context_word]
-    if data_index == len(data):
-      buffer.extend(data[0:span])
-      data_index = span
+def word2vec_basic(log_dir):
+  """Example of building, training and visualizing a word2vec model."""
+  # Create the directory for TensorBoard variables if there is not.
+  if not os.path.exists(log_dir):
+    os.makedirs(log_dir)
+
+  # Step 1: Download the data.
+  url = 'http://mattmahoney.net/dc/'
+
+  # pylint: disable=redefined-outer-name
+  def maybe_download(filename, expected_bytes):
+    """Download a file if not present, and make sure it's the right size."""
+    local_filename = os.path.join(gettempdir(), filename)
+    if not os.path.exists(local_filename):
+      local_filename, _ = urllib.request.urlretrieve(url + filename,
+                                                     local_filename)
+    statinfo = os.stat(local_filename)
+    if statinfo.st_size == expected_bytes:
+      print('Found and verified', filename)
     else:
-      buffer.append(data[data_index])
-      data_index += 1
-  # Backtrack a little bit to avoid skipping words in the end of a batch
-  data_index = (data_index + len(data) - span) % len(data)
-  return batch, labels
-
-
-batch, labels = generate_batch(batch_size=8, num_skips=2, skip_window=1)
-for i in range(8):
-  print(batch[i], reverse_dictionary[batch[i]], '->', labels[i, 0],
-        reverse_dictionary[labels[i, 0]])
-
-# Step 4: Build and train a skip-gram model.
-
-batch_size = 128
-embedding_size = 128  # Dimension of the embedding vector.
-skip_window = 1  # How many words to consider left and right.
-num_skips = 2  # How many times to reuse an input to generate a label.
-num_sampled = 64  # Number of negative examples to sample.
-
-# We pick a random validation set to sample nearest neighbors. Here we limit the
-# validation samples to the words that have a low numeric ID, which by
-# construction are also the most frequent. These 3 variables are used only for
-# displaying model accuracy, they don't affect calculation.
-valid_size = 16  # Random set of words to evaluate similarity on.
-valid_window = 100  # Only pick dev samples in the head of the distribution.
-valid_examples = np.random.choice(valid_window, valid_size, replace=False)
-
-graph = tf.Graph()
-
-with graph.as_default():
-
-  # Input data.
-  with tf.name_scope('inputs'):
-    train_inputs = tf.placeholder(tf.int32, shape=[batch_size])
-    train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])
-    valid_dataset = tf.constant(valid_examples, dtype=tf.int32)
-
-  # Ops and variables pinned to the CPU because of missing GPU implementation
-  with tf.device('/cpu:0'):
-    # Look up embeddings for inputs.
-    with tf.name_scope('embeddings'):
-      embeddings = tf.Variable(
-          tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0))
-      embed = tf.nn.embedding_lookup(embeddings, train_inputs)
-
-    # Construct the variables for the NCE loss
-    with tf.name_scope('weights'):
-      nce_weights = tf.Variable(
-          tf.truncated_normal(
-              [vocabulary_size, embedding_size],
-              stddev=1.0 / math.sqrt(embedding_size)))
-    with tf.name_scope('biases'):
-      nce_biases = tf.Variable(tf.zeros([vocabulary_size]))
-
-  # Compute the average NCE loss for the batch.
-  # tf.nce_loss automatically draws a new sample of the negative labels each
-  # time we evaluate the loss.
-  # Explanation of the meaning of NCE loss:
-  #   http://mccormickml.com/2016/04/19/word2vec-tutorial-the-skip-gram-model/
-  with tf.name_scope('loss'):
-    loss = tf.reduce_mean(
-        tf.nn.nce_loss(
-            weights=nce_weights,
-            biases=nce_biases,
-            labels=train_labels,
-            inputs=embed,
-            num_sampled=num_sampled,
-            num_classes=vocabulary_size))
-
-  # Add the loss value as a scalar to summary.
-  tf.summary.scalar('loss', loss)
-
-  # Construct the SGD optimizer using a learning rate of 1.0.
-  with tf.name_scope('optimizer'):
-    optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(loss)
-
-  # Compute the cosine similarity between minibatch examples and all embeddings.
-  norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keepdims=True))
-  normalized_embeddings = embeddings / norm
-  valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings,
-                                            valid_dataset)
-  similarity = tf.matmul(
-      valid_embeddings, normalized_embeddings, transpose_b=True)
-
-  # Merge all summaries.
-  merged = tf.summary.merge_all()
-
-  # Add variable initializer.
-  init = tf.global_variables_initializer()
-
-  # Create a saver.
-  saver = tf.train.Saver()
-
-# Step 5: Begin training.
-num_steps = 100001
-
-with tf.Session(graph=graph) as session:
-  # Open a writer to write summaries.
-  writer = tf.summary.FileWriter(FLAGS.log_dir, session.graph)
-
-  # We must initialize all variables before we use them.
-  init.run()
-  print('Initialized')
-
-  average_loss = 0
-  for step in xrange(num_steps):
-    batch_inputs, batch_labels = generate_batch(batch_size, num_skips,
-                                                skip_window)
-    feed_dict = {train_inputs: batch_inputs, train_labels: batch_labels}
-
-    # Define metadata variable.
-    run_metadata = tf.RunMetadata()
-
-    # We perform one update step by evaluating the optimizer op (including it
-    # in the list of returned values for session.run()
-    # Also, evaluate the merged op to get all summaries from the returned "summary" variable.
-    # Feed metadata variable to session for visualizing the graph in TensorBoard.
-    _, summary, loss_val = session.run(
-        [optimizer, merged, loss],
-        feed_dict=feed_dict,
-        run_metadata=run_metadata)
-    average_loss += loss_val
-
-    # Add returned summaries to writer in each step.
-    writer.add_summary(summary, step)
-    # Add metadata to visualize the graph for the last run.
-    if step == (num_steps - 1):
-      writer.add_run_metadata(run_metadata, 'step%d' % step)
-
-    if step % 2000 == 0:
-      if step > 0:
-        average_loss /= 2000
-      # The average loss is an estimate of the loss over the last 2000 batches.
-      print('Average loss at step ', step, ': ', average_loss)
-      average_loss = 0
-
-    # Note that this is expensive (~20% slowdown if computed every 500 steps)
-    if step % 10000 == 0:
-      sim = similarity.eval()
-      for i in xrange(valid_size):
-        valid_word = reverse_dictionary[valid_examples[i]]
-        top_k = 8  # number of nearest neighbors
-        nearest = (-sim[i, :]).argsort()[1:top_k + 1]
-        log_str = 'Nearest to %s:' % valid_word
-        for k in xrange(top_k):
-          close_word = reverse_dictionary[nearest[k]]
-          log_str = '%s %s,' % (log_str, close_word)
-        print(log_str)
-  final_embeddings = normalized_embeddings.eval()
-
-  # Write corresponding labels for the embeddings.
-  with open(FLAGS.log_dir + '/metadata.tsv', 'w') as f:
-    for i in xrange(vocabulary_size):
-      f.write(reverse_dictionary[i] + '\n')
-
-  # Save the model for checkpoints.
-  saver.save(session, os.path.join(FLAGS.log_dir, 'model.ckpt'))
-
-  # Create a configuration for visualizing embeddings with the labels in TensorBoard.
-  config = projector.ProjectorConfig()
-  embedding_conf = config.embeddings.add()
-  embedding_conf.tensor_name = embeddings.name
-  embedding_conf.metadata_path = os.path.join(FLAGS.log_dir, 'metadata.tsv')
-  projector.visualize_embeddings(writer, config)
-
-writer.close()
-
-# Step 6: Visualize the embeddings.
-
-
-# pylint: disable=missing-docstring
-# Function to draw visualization of distance between embeddings.
-def plot_with_labels(low_dim_embs, labels, filename):
-  assert low_dim_embs.shape[0] >= len(labels), 'More labels than embeddings'
-  plt.figure(figsize=(18, 18))  # in inches
-  for i, label in enumerate(labels):
-    x, y = low_dim_embs[i, :]
-    plt.scatter(x, y)
-    plt.annotate(
-        label,
-        xy=(x, y),
-        xytext=(5, 2),
-        textcoords='offset points',
-        ha='right',
-        va='bottom')
-
-  plt.savefig(filename)
-
-
-try:
-  # pylint: disable=g-import-not-at-top
-  from sklearn.manifold import TSNE
-  import matplotlib.pyplot as plt
-
-  tsne = TSNE(
-      perplexity=30, n_components=2, init='pca', n_iter=5000, method='exact')
-  plot_only = 500
-  low_dim_embs = tsne.fit_transform(final_embeddings[:plot_only, :])
-  labels = [reverse_dictionary[i] for i in xrange(plot_only)]
-  plot_with_labels(low_dim_embs, labels, os.path.join(gettempdir(), 'tsne.png'))
-
-except ImportError as ex:
-  print('Please install sklearn, matplotlib, and scipy to show embeddings.')
-  print(ex)
+      print(statinfo.st_size)
+      raise Exception('Failed to verify ' + local_filename +
+                      '. Can you get to it with a browser?')
+    return local_filename
+
+  filename = maybe_download('text8.zip', 31344016)
+
+  # Read the data into a list of strings.
+  def read_data(filename):
+    """Extract the first file enclosed in a zip file as a list of words."""
+    with zipfile.ZipFile(filename) as f:
+      data = tf.compat.as_str(f.read(f.namelist()[0])).split()
+    return data
+
+  vocabulary = read_data(filename)
+  print('Data size', len(vocabulary))
+
+  # Step 2: Build the dictionary and replace rare words with UNK token.
+  vocabulary_size = 50000
+
+  def build_dataset(words, n_words):
+    """Process raw inputs into a dataset."""
+    count = [['UNK', -1]]
+    count.extend(collections.Counter(words).most_common(n_words - 1))
+    dictionary = dict()
+    for word, _ in count:
+      dictionary[word] = len(dictionary)
+    data = list()
+    unk_count = 0
+    for word in words:
+      index = dictionary.get(word, 0)
+      if index == 0:  # dictionary['UNK']
+        unk_count += 1
+      data.append(index)
+    count[0][1] = unk_count
+    reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys()))
+    return data, count, dictionary, reversed_dictionary
+
+  # Filling 4 global variables:
+  # data - list of codes (integers from 0 to vocabulary_size-1).
+  #   This is the original text but words are replaced by their codes
+  # count - map of words(strings) to count of occurrences
+  # dictionary - map of words(strings) to their codes(integers)
+  # reverse_dictionary - maps codes(integers) to words(strings)
+  data, count, unused_dictionary, reverse_dictionary = build_dataset(
+      vocabulary, vocabulary_size)
+  del vocabulary  # Hint to reduce memory.
+  print('Most common words (+UNK)', count[:5])
+  print('Sample data', data[:10], [reverse_dictionary[i] for i in data[:10]])
+
+  # Step 3: Function to generate a training batch for the skip-gram model.
+  def generate_batch(batch_size, num_skips, skip_window):
+    global data_index
+    assert batch_size % num_skips == 0
+    assert num_skips <= 2 * skip_window
+    batch = np.ndarray(shape=(batch_size), dtype=np.int32)
+    labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32)
+    span = 2 * skip_window + 1  # [ skip_window target skip_window ]
+    buffer = collections.deque(maxlen=span)  # pylint: disable=redefined-builtin
+    if data_index + span > len(data):
+      data_index = 0
+    buffer.extend(data[data_index:data_index + span])
+    data_index += span
+    for i in range(batch_size // num_skips):
+      context_words = [w for w in range(span) if w != skip_window]
+      words_to_use = random.sample(context_words, num_skips)
+      for j, context_word in enumerate(words_to_use):
+        batch[i * num_skips + j] = buffer[skip_window]
+        labels[i * num_skips + j, 0] = buffer[context_word]
+      if data_index == len(data):
+        buffer.extend(data[0:span])
+        data_index = span
+      else:
+        buffer.append(data[data_index])
+        data_index += 1
+    # Backtrack a little bit to avoid skipping words in the end of a batch
+    data_index = (data_index + len(data) - span) % len(data)
+    return batch, labels
+
+  batch, labels = generate_batch(batch_size=8, num_skips=2, skip_window=1)
+  for i in range(8):
+    print(batch[i], reverse_dictionary[batch[i]], '->', labels[i, 0],
+          reverse_dictionary[labels[i, 0]])
+
+  # Step 4: Build and train a skip-gram model.
+
+  batch_size = 128
+  embedding_size = 128  # Dimension of the embedding vector.
+  skip_window = 1  # How many words to consider left and right.
+  num_skips = 2  # How many times to reuse an input to generate a label.
+  num_sampled = 64  # Number of negative examples to sample.
+
+  # We pick a random validation set to sample nearest neighbors. Here we limit
+  # the validation samples to the words that have a low numeric ID, which by
+  # construction are also the most frequent. These 3 variables are used only for
+  # displaying model accuracy, they don't affect calculation.
+  valid_size = 16  # Random set of words to evaluate similarity on.
+  valid_window = 100  # Only pick dev samples in the head of the distribution.
+  valid_examples = np.random.choice(valid_window, valid_size, replace=False)
+
+  graph = tf.Graph()
+
+  with graph.as_default():
+
+    # Input data.
+    with tf.name_scope('inputs'):
+      train_inputs = tf.placeholder(tf.int32, shape=[batch_size])
+      train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])
+      valid_dataset = tf.constant(valid_examples, dtype=tf.int32)
+
+    # Ops and variables pinned to the CPU because of missing GPU implementation
+    with tf.device('/cpu:0'):
+      # Look up embeddings for inputs.
+      with tf.name_scope('embeddings'):
+        embeddings = tf.Variable(
+            tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0))
+        embed = tf.nn.embedding_lookup(embeddings, train_inputs)
+
+      # Construct the variables for the NCE loss
+      with tf.name_scope('weights'):
+        nce_weights = tf.Variable(
+            tf.truncated_normal([vocabulary_size, embedding_size],
+                                stddev=1.0 / math.sqrt(embedding_size)))
+      with tf.name_scope('biases'):
+        nce_biases = tf.Variable(tf.zeros([vocabulary_size]))
+
+    # Compute the average NCE loss for the batch.
+    # tf.nce_loss automatically draws a new sample of the negative labels each
+    # time we evaluate the loss.
+    # Explanation of the meaning of NCE loss:
+    #   http://mccormickml.com/2016/04/19/word2vec-tutorial-the-skip-gram-model/
+    with tf.name_scope('loss'):
+      loss = tf.reduce_mean(
+          tf.nn.nce_loss(
+              weights=nce_weights,
+              biases=nce_biases,
+              labels=train_labels,
+              inputs=embed,
+              num_sampled=num_sampled,
+              num_classes=vocabulary_size))
+
+    # Add the loss value as a scalar to summary.
+    tf.summary.scalar('loss', loss)
+
+    # Construct the SGD optimizer using a learning rate of 1.0.
+    with tf.name_scope('optimizer'):
+      optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(loss)
+
+    # Compute the cosine similarity between minibatch examples and all
+    # embeddings.
+    norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keepdims=True))
+    normalized_embeddings = embeddings / norm
+    valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings,
+                                              valid_dataset)
+    similarity = tf.matmul(
+        valid_embeddings, normalized_embeddings, transpose_b=True)
+
+    # Merge all summaries.
+    merged = tf.summary.merge_all()
+
+    # Add variable initializer.
+    init = tf.global_variables_initializer()
+
+    # Create a saver.
+    saver = tf.train.Saver()
+
+  # Step 5: Begin training.
+  num_steps = 100001
+
+  with tf.Session(graph=graph) as session:
+    # Open a writer to write summaries.
+    writer = tf.summary.FileWriter(log_dir, session.graph)
+
+    # We must initialize all variables before we use them.
+    init.run()
+    print('Initialized')
+
+    average_loss = 0
+    for step in xrange(num_steps):
+      batch_inputs, batch_labels = generate_batch(batch_size, num_skips,
+                                                  skip_window)
+      feed_dict = {train_inputs: batch_inputs, train_labels: batch_labels}
+
+      # Define metadata variable.
+      run_metadata = tf.RunMetadata()
+
+      # We perform one update step by evaluating the optimizer op (including it
+      # in the list of returned values for session.run()
+      # Also, evaluate the merged op to get all summaries from the returned
+      # "summary" variable. Feed metadata variable to session for visualizing
+      # the graph in TensorBoard.
+      _, summary, loss_val = session.run([optimizer, merged, loss],
+                                         feed_dict=feed_dict,
+                                         run_metadata=run_metadata)
+      average_loss += loss_val
+
+      # Add returned summaries to writer in each step.
+      writer.add_summary(summary, step)
+      # Add metadata to visualize the graph for the last run.
+      if step == (num_steps - 1):
+        writer.add_run_metadata(run_metadata, 'step%d' % step)
+
+      if step % 2000 == 0:
+        if step > 0:
+          average_loss /= 2000
+        # The average loss is an estimate of the loss over the last 2000
+        # batches.
+        print('Average loss at step ', step, ': ', average_loss)
+        average_loss = 0
+
+      # Note that this is expensive (~20% slowdown if computed every 500 steps)
+      if step % 10000 == 0:
+        sim = similarity.eval()
+        for i in xrange(valid_size):
+          valid_word = reverse_dictionary[valid_examples[i]]
+          top_k = 8  # number of nearest neighbors
+          nearest = (-sim[i, :]).argsort()[1:top_k + 1]
+          log_str = 'Nearest to %s:' % valid_word
+          for k in xrange(top_k):
+            close_word = reverse_dictionary[nearest[k]]
+            log_str = '%s %s,' % (log_str, close_word)
+          print(log_str)
+    final_embeddings = normalized_embeddings.eval()
+
+    # Write corresponding labels for the embeddings.
+    with open(log_dir + '/metadata.tsv', 'w') as f:
+      for i in xrange(vocabulary_size):
+        f.write(reverse_dictionary[i] + '\n')
+
+    # Save the model for checkpoints.
+    saver.save(session, os.path.join(log_dir, 'model.ckpt'))
+
+    # Create a configuration for visualizing embeddings with the labels in
+    # TensorBoard.
+    config = projector.ProjectorConfig()
+    embedding_conf = config.embeddings.add()
+    embedding_conf.tensor_name = embeddings.name
+    embedding_conf.metadata_path = os.path.join(log_dir, 'metadata.tsv')
+    projector.visualize_embeddings(writer, config)
+
+  writer.close()
+
+  # Step 6: Visualize the embeddings.
+
+  # pylint: disable=missing-docstring
+  # Function to draw visualization of distance between embeddings.
+  def plot_with_labels(low_dim_embs, labels, filename):
+    assert low_dim_embs.shape[0] >= len(labels), 'More labels than embeddings'
+    plt.figure(figsize=(18, 18))  # in inches
+    for i, label in enumerate(labels):
+      x, y = low_dim_embs[i, :]
+      plt.scatter(x, y)
+      plt.annotate(
+          label,
+          xy=(x, y),
+          xytext=(5, 2),
+          textcoords='offset points',
+          ha='right',
+          va='bottom')
+
+    plt.savefig(filename)
+
+  try:
+    # pylint: disable=g-import-not-at-top
+    from sklearn.manifold import TSNE
+    import matplotlib.pyplot as plt
+
+    tsne = TSNE(
+        perplexity=30, n_components=2, init='pca', n_iter=5000, method='exact')
+    plot_only = 500
+    low_dim_embs = tsne.fit_transform(final_embeddings[:plot_only, :])
+    labels = [reverse_dictionary[i] for i in xrange(plot_only)]
+    plot_with_labels(low_dim_embs, labels, os.path.join(gettempdir(),
+                                                        'tsne.png'))
+
+  except ImportError as ex:
+    print('Please install sklearn, matplotlib, and scipy to show embeddings.')
+    print(ex)
+
+
+# All functionality is run after tf.app.run() (b/122547914). This could be split
+# up but the methods are laid sequentially with their usage for clarity.
+def main(unused_argv):
+  # Give a folder path as an argument with '--log_dir' to save
+  # TensorBoard summaries. Default is a log folder in current directory.
+  current_path = os.path.dirname(os.path.realpath(sys.argv[0]))
+
+  parser = argparse.ArgumentParser()
+  parser.add_argument(
+      '--log_dir',
+      type=str,
+      default=os.path.join(current_path, 'log'),
+      help='The log directory for TensorBoard summaries.')
+  flags, unused_flags = parser.parse_known_args()
+  word2vec_basic(flags.log_dir)
+
+if __name__ == '__main__':
+  tf.app.run()
-- 
GitLab


From 47cf89738d3dd09dd4c0ff24df3c5c433c544d0d Mon Sep 17 00:00:00 2001
From: Jacques Pienaar <jpienaar@google.com>
Date: Thu, 17 Jan 2019 15:30:05 -0800
Subject: [PATCH 0930/1095] [TF:XLA] Extract max_num_elements from
 empty_tensor_list.

PiperOrigin-RevId: 229829562
---
 .../compiler/tests/tensor_list_ops_test.py    | 22 +++++++++-----
 .../tf2xla/kernels/tensor_list_ops.cc         | 29 ++++++++++++++++---
 2 files changed, 40 insertions(+), 11 deletions(-)

diff --git a/tensorflow/compiler/tests/tensor_list_ops_test.py b/tensorflow/compiler/tests/tensor_list_ops_test.py
index 3b48aed60c..1ecdb22cd0 100644
--- a/tensorflow/compiler/tests/tensor_list_ops_test.py
+++ b/tensorflow/compiler/tests/tensor_list_ops_test.py
@@ -93,18 +93,26 @@ class ListOpsTest(xla_test.XLATestCase):
       result = sess.run([e11, [e21, e22], [e31, e32]])
       self.assertEqual(result, [1.0, [2.0, 1.0], [3.0, 1.0]])
 
-  def testEmptyTensorList(self):
-    dim = 7
+  def testEmptyTensorListNoMax(self):
     with self.cached_session() as sess, self.test_scope():
-      p = array_ops.placeholder(dtypes.int32)
       l = list_ops.empty_tensor_list(
-          element_shape=(p, 15), element_dtype=dtypes.float32)
+          element_shape=(7, 15), element_dtype=dtypes.float32)
       l = list_ops.tensor_list_push_back(
-          l, constant_op.constant(1.0, shape=(dim, 15)))
+          l, constant_op.constant(1.0, shape=(7, 15)))
       _, e = list_ops.tensor_list_pop_back(l, element_dtype=dtypes.float32)
       with self.assertRaisesRegexp(errors.InvalidArgumentError,
-                                   "Use TensorListReserve instead"):
-        self.assertEqual(sess.run(e, {p: dim}), 1.0 * np.ones((dim, 15)))
+                                   "Set the max number of elements"):
+        self.assertEqual(sess.run(e), 1.0 * np.ones((7, 15)))
+
+  def testEmptyTensorListMax(self):
+    with self.cached_session() as sess, self.test_scope():
+      l = list_ops.empty_tensor_list(
+          element_shape=(10, 15), element_dtype=dtypes.float32,
+          max_num_elements=2)
+      l = list_ops.tensor_list_push_back(
+          l, array_ops.fill(value=3.0, dims=(10, 15)))
+      _, e = list_ops.tensor_list_pop_back(l, element_dtype=dtypes.float32)
+      self.assertAllEqual(sess.run(e), 3.0 * np.ones((10, 15)))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/compiler/tf2xla/kernels/tensor_list_ops.cc b/tensorflow/compiler/tf2xla/kernels/tensor_list_ops.cc
index ba3d7eb00d..5dda62fd49 100644
--- a/tensorflow/compiler/tf2xla/kernels/tensor_list_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/tensor_list_ops.cc
@@ -85,19 +85,40 @@ REGISTER_XLA_OP(Name("TensorListReserve")
 
 class EmptyTensorListOp : public XlaOpKernel {
  public:
-  explicit EmptyTensorListOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) {}
+  explicit EmptyTensorListOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) {
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("element_dtype", &dtype_));
+  }
 
   void Compile(XlaOpKernelContext* ctx) override {
-    ctx->CtxFailure(
+    TensorShape element_shape;
+    OP_REQUIRES_OK(ctx, ctx->ConstantInputAsShape(0, &element_shape));
+    int64 max_num_elements;
+    OP_REQUIRES_OK(ctx, ctx->ConstantInputAsIntScalar(1, &max_num_elements));
+    OP_REQUIRES(
+        ctx, max_num_elements >= 0,
         errors::InvalidArgument("XLA compilation requires a fixed tensor list "
-                                "size. Use TensorListReserve instead."));
+                                "size. Set the max number of elements."));
+
+    TensorShape tensor_shape;
+    tensor_shape.AddDim(max_num_elements);
+    tensor_shape.AppendShape(element_shape);
+
+    xla::XlaBuilder* b = ctx->builder();
+    ctx->SetOutput(0, xla::Tuple(b, {xla::Broadcast(XlaHelpers::Zero(b, dtype_),
+                                                    tensor_shape.dim_sizes()),
+                                     xla::ConstantR0<int32>(b, 0)}));
   }
 
  private:
+  DataType dtype_;
+
   TF_DISALLOW_COPY_AND_ASSIGN(EmptyTensorListOp);
 };
 
-REGISTER_XLA_OP(Name("EmptyTensorList"), EmptyTensorListOp);
+REGISTER_XLA_OP(Name("EmptyTensorList")
+                    .CompileTimeConstantInput("element_shape")
+                    .CompileTimeConstantInput("max_num_elements"),
+                EmptyTensorListOp);
 
 class TensorListElementShapeOp : public XlaOpKernel {
  public:
-- 
GitLab


From 307b1f368aa36173a799575e93737f150a4db9cd Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 17 Jan 2019 15:46:15 -0800
Subject: [PATCH 0931/1095] Introduce a Graph Verifier interface in Grappler.

This can be used to implement specific graph verifiers that can then be plugged into the overall grappler workflow or run standalone.
Some examples of specific verifiers are:
 1. A general structural verifier that verifies that the specified graph has
   a valid structure that meets the specification of what it means to be
      a valid TF graph.
 2. A backend specific verifier that verifies that the specified graph,
     generated after a grappler transformation to convert the input TF graph
     to a corresponding backend graph, is a valid graph in the specification
     of the backend.

PiperOrigin-RevId: 229832249
---
 tensorflow/core/grappler/verifiers/BUILD      | 13 +++++
 .../core/grappler/verifiers/graph_verifier.h  | 53 +++++++++++++++++++
 2 files changed, 66 insertions(+)
 create mode 100644 tensorflow/core/grappler/verifiers/BUILD
 create mode 100644 tensorflow/core/grappler/verifiers/graph_verifier.h

diff --git a/tensorflow/core/grappler/verifiers/BUILD b/tensorflow/core/grappler/verifiers/BUILD
new file mode 100644
index 0000000000..00b1325176
--- /dev/null
+++ b/tensorflow/core/grappler/verifiers/BUILD
@@ -0,0 +1,13 @@
+licenses(["notice"])  # Apache 2.0
+
+cc_library(
+    name = "graph_verifier",
+    hdrs = [
+        "graph_verifier.h",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+    ],
+)
diff --git a/tensorflow/core/grappler/verifiers/graph_verifier.h b/tensorflow/core/grappler/verifiers/graph_verifier.h
new file mode 100644
index 0000000000..f5acd59266
--- /dev/null
+++ b/tensorflow/core/grappler/verifiers/graph_verifier.h
@@ -0,0 +1,53 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CORE_GRAPPLER_VERIFIERS_GRAPH_VERIFIER_H_
+#define TENSORFLOW_CORE_GRAPPLER_VERIFIERS_GRAPH_VERIFIER_H_
+
+#include <string>
+#include <vector>
+#include "tensorflow/core/framework/graph.pb.h"
+#include "tensorflow/core/lib/core/status.h"
+
+namespace tensorflow {
+namespace grappler {
+
+// An abstract interface for verifying a graph.
+// This will be used to implement specific verifiers to verify that a grappler
+// transformed graph is valid.
+// Some examples of specific verifiers are:
+// 1. A general structural verifier that verifies that the specified graph has
+//    a valid structure that meets the specification of what it means to be
+//      a valid TensorFlow graph.
+// 2. A backend specific verifier that verifies that the specified graph,
+//     generated after a grappler transformation to convert the input TensorFlow
+//     graph to a corresponding backend graph, is a valid graph in the
+//     specification of the backend.
+class GraphVerifier {
+ public:
+  GraphVerifier() {}
+  virtual ~GraphVerifier() {}
+
+  // A name for the verifier.
+  virtual string name() const = 0;
+
+  // Implement an algorithm to verify the specified graph.
+  virtual Status Verify(const GraphDef& graph, std::vector<string>* errors) = 0;
+};
+
+}  // end namespace grappler
+}  // end namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_GRAPPLER_VERIFIERS_GRAPH_VERIFIER_H_
-- 
GitLab


From a108131f4ef24cf6e7eecfd22f98f20fd1b6e390 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Thu, 17 Jan 2019 15:47:02 -0800
Subject: [PATCH 0932/1095] Cleanup training_distributed unnecessary scopes
 since we're already in the strategy scope in the entire function

PiperOrigin-RevId: 229832356
---
 .../keras/engine/training_distributed.py      | 65 ++++++++-----------
 1 file changed, 28 insertions(+), 37 deletions(-)

diff --git a/tensorflow/python/keras/engine/training_distributed.py b/tensorflow/python/keras/engine/training_distributed.py
index 8b7e5bb5cc..9b11c721d4 100644
--- a/tensorflow/python/keras/engine/training_distributed.py
+++ b/tensorflow/python/keras/engine/training_distributed.py
@@ -314,10 +314,9 @@ def experimental_tpu_fit_loop(model,
       dtype='int32',
       name='steps_per_run')
 
-  with current_strategy.scope():
-    ctx = current_strategy.extended.experimental_run_steps_on_iterator(
-        step_fn, iterator, iterations=steps_per_run,
-        initial_loop_values=initial_loop_values)
+  ctx = current_strategy.extended.experimental_run_steps_on_iterator(
+      step_fn, iterator, iterations=steps_per_run,
+      initial_loop_values=initial_loop_values)
 
   train_op = ctx.run_op
   output_tensors = ctx.last_step_outputs
@@ -325,9 +324,8 @@ def experimental_tpu_fit_loop(model,
   do_validation = bool(validation_steps)
 
   if model._compile_distribution:
-    with current_strategy.scope():
-      distributed_training_utils._copy_weights_to_distributed_model(
-          model, model._distributed_model_train)
+    distributed_training_utils._copy_weights_to_distributed_model(
+        model, model._distributed_model_train)
 
   callbacks = cbks.configure_callbacks(
       callbacks,
@@ -346,9 +344,8 @@ def experimental_tpu_fit_loop(model,
 
   callbacks.on_train_begin()
   for epoch in range(initial_epoch, epochs):
-    with current_strategy.scope():
-      distributed_training_utils._reset_metrics(
-          model, model._distributed_model_train)
+    distributed_training_utils._reset_metrics(
+        model, model._distributed_model_train)
     callbacks.on_epoch_begin(epoch)
     epoch_logs = {}
     step_index = 0
@@ -382,9 +379,8 @@ def experimental_tpu_fit_loop(model,
       if model._compile_distribution:
         # Since we create a new clone from the original model we need to copy
         # the weights back to the original model before we can run validation.
-        with current_strategy.scope():
-          distributed_training_utils._copy_weights_to_original_model(
-              model, model._distributed_model_train, ModeKeys.TRAIN)
+        distributed_training_utils._copy_weights_to_original_model(
+            model, model._distributed_model_train, ModeKeys.TRAIN)
 
       val_outs = experimental_tpu_test_loop(  # pylint: disable=undefined-variable
           model,
@@ -404,9 +400,8 @@ def experimental_tpu_fit_loop(model,
 
   if model._compile_distribution:
     # Copy the weights back from the replicated model to the original model.
-    with current_strategy.scope():
-      distributed_training_utils._copy_weights_to_original_model(
-          model, model._distributed_model_train, ModeKeys.TRAIN)
+    distributed_training_utils._copy_weights_to_original_model(
+        model, model._distributed_model_train, ModeKeys.TRAIN)
   scope.__exit__(None, None, None)
   return model.history
 
@@ -491,12 +486,11 @@ def experimental_tpu_test_loop(model,
     tensor = model._all_stateful_metrics_tensors[name]
     initial_loop_values[name] = array_ops.zeros(tensor.shape, tensor.dtype)
 
-  with current_strategy.scope():
-    # TODO(priyag): Use steps_per_run when we use new metrics as they will
-    # allow handling metric computation at each step using variables.
-    ctx = current_strategy.extended.experimental_run_steps_on_iterator(
-        step_fn, iterator, iterations=1,
-        initial_loop_values=initial_loop_values)
+  # TODO(priyag): Use steps_per_run when we use new metrics as they will
+  # allow handling metric computation at each step using variables.
+  ctx = current_strategy.extended.experimental_run_steps_on_iterator(
+      step_fn, iterator, iterations=1,
+      initial_loop_values=initial_loop_values)
 
   test_op = ctx.run_op
   output_tensors = ctx.last_step_outputs
@@ -505,13 +499,12 @@ def experimental_tpu_test_loop(model,
     progbar = Progbar(target=steps)
 
   if model._compile_distribution:
-    with current_strategy.scope():
-      distributed_training_utils._copy_weights_to_distributed_model(
-          model, model._distributed_model_test)
-  with current_strategy.scope():
-    distributed_training_utils._reset_metrics(
+    distributed_training_utils._copy_weights_to_distributed_model(
         model, model._distributed_model_test)
 
+  distributed_training_utils._reset_metrics(
+      model, model._distributed_model_test)
+
   assert steps is not None
   outs = [0.] * len(model.metrics_names)
   for step in range(steps):
@@ -609,11 +602,10 @@ def experimental_tpu_predict_loop(model, dataset, verbose=0, steps=None):
     shape.dims = [batch_dimension] + shape.dims[1:]
     initial_loop_values[name] = array_ops.zeros(shape, tensor.dtype)
 
-  with current_strategy.scope():
-    # TODO(priyag, sourabhbajaj): Support steps_per_run if/when we add outfeed.
-    ctx = current_strategy.extended.experimental_run_steps_on_iterator(
-        step_fn, iterator, iterations=1,
-        initial_loop_values=initial_loop_values)
+  # TODO(priyag, sourabhbajaj): Support steps_per_run if/when we add outfeed.
+  ctx = current_strategy.extended.experimental_run_steps_on_iterator(
+      step_fn, iterator, iterations=1,
+      initial_loop_values=initial_loop_values)
 
   predict_op = ctx.run_op
   output_tensors = ctx.last_step_outputs
@@ -622,13 +614,12 @@ def experimental_tpu_predict_loop(model, dataset, verbose=0, steps=None):
     progbar = Progbar(target=steps)
 
   if model._compile_distribution:
-    with current_strategy.scope():
-      distributed_training_utils._copy_weights_to_distributed_model(
-          model, model._distributed_model_predict)
-  with current_strategy.scope():
-    distributed_training_utils._reset_metrics(
+    distributed_training_utils._copy_weights_to_distributed_model(
         model, model._distributed_model_predict)
 
+  distributed_training_utils._reset_metrics(
+      model, model._distributed_model_predict)
+
   assert steps is not None
   # Since we do not know how many samples we will see, we cannot pre-allocate
   # the returned Numpy arrays. Instead, we store one array per batch seen
-- 
GitLab


From 04e8de0b1753ba7c229105d10113ac05fe5aa9d1 Mon Sep 17 00:00:00 2001
From: Blake Hechtman <blakehechtman@google.com>
Date: Thu, 17 Jan 2019 15:49:07 -0800
Subject: [PATCH 0933/1095] [TF] Make behavior of dropout backward
 compatability.

PiperOrigin-RevId: 229832672
---
 tensorflow/python/keras/layers/wrappers_test.py |  4 ++--
 tensorflow/python/ops/nn_ops.py                 | 10 ++++++++--
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/keras/layers/wrappers_test.py b/tensorflow/python/keras/layers/wrappers_test.py
index 313a2193eb..f3aa5c4d68 100644
--- a/tensorflow/python/keras/layers/wrappers_test.py
+++ b/tensorflow/python/keras/layers/wrappers_test.py
@@ -158,8 +158,8 @@ class TimeDistributedTest(test.TestCase):
       # test layers that need learning_phase to be set
       np.random.seed(1234)
       x = keras.layers.Input(shape=(3, 2))
-      y = keras.layers.TimeDistributed(
-          keras.layers.Dropout(.9999))(x, training=True)
+      y = keras.layers.TimeDistributed(keras.layers.Dropout(.999))(
+          x, training=True)
       model = keras.models.Model(x, y)
       y = model.predict(np.random.random((10, 3, 2)))
       self.assertAllClose(np.mean(y), 0., atol=1e-1, rtol=1e-1)
diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index 25609f9efa..e0ef9e5e34 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -3294,11 +3294,17 @@ def dropout_v2(x, rate, noise_shape=None, seed=None, name=None):  # pylint: disa
     noise_shape = _get_noise_shape(x, noise_shape)
     # Sample a uniform distribution on [0.0, 1.0) and select values larger than
     # rate.
+    #
+    # NOTE: Random uniform actually can only generate 2^23 floats on [1.0, 2.0)
+    # and subtract 1.0.
     random_tensor = random_ops.random_uniform(
         noise_shape, seed=seed, dtype=x.dtype)
     keep_prob = 1 - rate
-    ret = (1 / keep_prob) * math_ops.cast(keep_prob >= random_tensor,
-                                          x.dtype) * x
+    scale = 1 / keep_prob
+    # NOTE: if (1.0 + rate) - 1 is equal to rate, then we want to consider that
+    # float to be selected, hence we use a >= comparison.
+    keep_mask = random_tensor >= rate
+    ret = x * scale * math_ops.cast(keep_mask, x.dtype)
     if not context.executing_eagerly():
       ret.set_shape(x.get_shape())
     return ret
-- 
GitLab


From f8c19045a1ed9e54ee02e8690b5c1a185023c9db Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 17 Jan 2019 15:54:13 -0800
Subject: [PATCH 0934/1095] Improve how the library to link is determined.

PiperOrigin-RevId: 229833513
---
 tensorflow/contrib/tensorrt/BUILD             |  26 +--
 .../tensorrt/custom_plugin_examples/BUILD     |   4 +-
 third_party/gpus/cuda_configure.bzl           | 154 ++++++++++--------
 third_party/nccl/nccl_configure.bzl           |   1 +
 third_party/tensorrt/BUILD.tpl                |   7 +-
 third_party/tensorrt/tensorrt_configure.bzl   |  66 +++-----
 .../preconfig/ubuntu14.04/tensorrt5/BUILD     |   2 +-
 7 files changed, 128 insertions(+), 132 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index 80d79164a2..3f10906768 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -49,7 +49,7 @@ tf_cuda_cc_test(
         "//tensorflow/core:test_main",
     ] + if_tensorrt([
         "@local_config_cuda//cuda:cuda_headers",
-        "@local_config_tensorrt//:nv_infer",
+        "@local_config_tensorrt//:tensorrt",
     ]),
 )
 
@@ -63,7 +63,7 @@ tf_custom_op_library(
         ":trt_shape_function",
         "//tensorflow/core:lib_proto_parsing",
     ] + if_tensorrt([
-        "@local_config_tensorrt//:nv_infer",
+        "@local_config_tensorrt//:tensorrt",
     ]),
 )
 
@@ -76,7 +76,7 @@ tf_cuda_library(
         ":trt_logging",
         ":trt_plugins",
     ] + if_tensorrt([
-        "@local_config_tensorrt//:nv_infer",
+        "@local_config_tensorrt//:tensorrt",
     ]) + tf_custom_op_library_additional_deps(),
 )
 
@@ -105,7 +105,7 @@ cc_library(
         "//tensorflow/core:stream_executor_headers_lib",
         "//tensorflow/core/grappler/costs:graph_properties",
     ] + if_tensorrt([
-        "@local_config_tensorrt//:nv_infer",
+        "@local_config_tensorrt//:tensorrt",
     ]) + tf_custom_op_library_additional_deps(),
     # TODO(laigd): fix this by merging header file in cc file.
     alwayslink = 1,  # buildozer: disable=alwayslink-with-hdrs
@@ -149,7 +149,7 @@ tf_cuda_library(
     deps = [
         "//tensorflow/core:lib_proto_parsing",
     ] + if_tensorrt([
-        "@local_config_tensorrt//:nv_infer",
+        "@local_config_tensorrt//:tensorrt",
     ]),
 )
 
@@ -169,7 +169,7 @@ tf_custom_op_py_library(
     dso = [
         ":python/ops/_trt_ops.so",
     ] + if_tensorrt([
-        "@local_config_tensorrt//:nv_infer",
+        "@local_config_tensorrt//:tensorrt",
     ]),
     kernels = [
         ":trt_op_kernels",
@@ -262,7 +262,7 @@ tf_cuda_library(
         "//tensorflow/core:framework_lite",
         "//tensorflow/core:lib_proto_parsing",
     ] + if_tensorrt([
-        "@local_config_tensorrt//:nv_infer",
+        "@local_config_tensorrt//:tensorrt",
     ]),
 )
 
@@ -275,7 +275,7 @@ tf_cuda_library(
         "//tensorflow/core:framework_lite",
         "//tensorflow/core:lib_proto_parsing",
     ] + if_tensorrt([
-        "@local_config_tensorrt//:nv_infer",
+        "@local_config_tensorrt//:tensorrt",
     ]),
 )
 
@@ -347,7 +347,7 @@ tf_cuda_library(
         "//tensorflow/core/grappler/costs:graph_properties",
         "//tensorflow/core/grappler/optimizers:meta_optimizer",
     ] + if_tensorrt([
-        "@local_config_tensorrt//:nv_infer",
+        "@local_config_tensorrt//:tensorrt",
     ]) + tf_custom_op_library_additional_deps(),
 )
 
@@ -377,7 +377,7 @@ tf_cuda_cc_test(
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
     ] + if_tensorrt([
-        "@local_config_tensorrt//:nv_infer",
+        "@local_config_tensorrt//:tensorrt",
     ]),
 )
 
@@ -409,7 +409,7 @@ tf_cuda_cc_test(
         "//tensorflow/core:testlib",
     ] + if_tensorrt([
         "@local_config_cuda//cuda:cuda_headers",
-        "@local_config_tensorrt//:nv_infer",
+        "@local_config_tensorrt//:tensorrt",
     ]),
 )
 
@@ -468,7 +468,7 @@ tf_cuda_library(
         "//tensorflow/core:framework_lite",
         "//tensorflow/core:lib_proto_parsing",
     ] + if_tensorrt([
-        "@local_config_tensorrt//:nv_infer",
+        "@local_config_tensorrt//:tensorrt",
     ]),
 )
 
@@ -488,7 +488,7 @@ tf_cuda_cc_test(
         "//tensorflow/core:test_main",
     ] + if_tensorrt([
         "@local_config_cuda//cuda:cuda_headers",
-        "@local_config_tensorrt//:nv_infer",
+        "@local_config_tensorrt//:tensorrt",
     ]),
 )
 
diff --git a/tensorflow/contrib/tensorrt/custom_plugin_examples/BUILD b/tensorflow/contrib/tensorrt/custom_plugin_examples/BUILD
index 69058c5826..09990fb5e3 100644
--- a/tensorflow/contrib/tensorrt/custom_plugin_examples/BUILD
+++ b/tensorflow/contrib/tensorrt/custom_plugin_examples/BUILD
@@ -48,7 +48,7 @@ tf_custom_op_library(
         "//tensorflow/contrib/tensorrt:trt_plugins",
         "//tensorflow/core:framework_lite",
     ] + if_tensorrt([
-        "@local_config_tensorrt//:nv_infer",
+        "@local_config_tensorrt//:tensorrt",
     ]),
 )
 
@@ -67,7 +67,7 @@ tf_kernel_library(
         "//tensorflow/contrib/tensorrt:trt_plugins",
         "//tensorflow/core:stream_executor_headers_lib",
     ] + if_tensorrt([
-        "@local_config_tensorrt//:nv_infer",
+        "@local_config_tensorrt//:tensorrt",
     ]) + tf_custom_op_library_additional_deps(),
 )
 
diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl
index beacf2f45a..7a6c7f4fe5 100644
--- a/third_party/gpus/cuda_configure.bzl
+++ b/third_party/gpus/cuda_configure.bzl
@@ -643,9 +643,7 @@ def _cudnn_version(repository_ctx, cudnn_install_basedir, cpu_value):
     auto_configure_fail(("cuDNN version detected from %s (%s) does not match " +
                          "TF_CUDNN_VERSION (%s)") %
                         (str(cudnn_h_path), full_version, environ_version),)
-
-  # We only use the major version since we use the libcudnn libraries that are
-  # only versioned with the major version (e.g. libcudnn.so.5).
+  # Only use the major version to match the SONAME of the library.
   version = major_version
   if cpu_value == "Windows":
     version = "64_" + version
@@ -691,11 +689,11 @@ def _is_windows(repository_ctx):
   return get_cpu_value(repository_ctx) == "Windows"
 
 
-def _lib_name(lib, cpu_value, version = "", static = False):
+def lib_name(base_name, cpu_value, version = None, static = False):
   """Constructs the platform-specific name of a library.
 
     Args:
-      lib: The name of the library, such as "cudart"
+      base_name: The name of the library, such as "cudart"
       cpu_value: The name of the host operating system.
       version: The version of the library.
       static: True the library is static or False if it is a shared object.
@@ -703,31 +701,55 @@ def _lib_name(lib, cpu_value, version = "", static = False):
     Returns:
       The platform-specific name of the library.
     """
+  version = "" if not version else "." + version
   if cpu_value in ("Linux", "FreeBSD"):
     if static:
-      return "lib%s.a" % lib
-    else:
-      if version:
-        version = ".%s" % version
-      return "lib%s.so%s" % (lib, version)
+      return "lib%s.a" % base_name
+    return "lib%s.so%s" % (base_name, version)
   elif cpu_value == "Windows":
-    return "%s.lib" % lib
+    return "%s.lib" % base_name
   elif cpu_value == "Darwin":
     if static:
-      return "lib%s.a" % lib
-    elif version:
-      version = ".%s" % version
-    return "lib%s%s.dylib" % (lib, version)
+      return "lib%s.a" % base_name
+    return "lib%s%s.dylib" % (base_name, version)
   else:
     auto_configure_fail("Invalid cpu_value: %s" % cpu_value)
 
+def find_lib(repository_ctx, paths, check_soname = True):
+  """
+    Finds a library among a list of potential paths.
+
+    Args:
+      paths: List of paths to inspect.
+
+    Returns:
+      Returns the first path in paths that exist.
+  """
+  objdump = repository_ctx.which("objdump")
+  mismatches = []
+  for path in [repository_ctx.path(path) for path in paths]:
+    if not path.exists:
+      continue
+    if check_soname and objdump != None:
+      output = repository_ctx.execute([objdump, "-p", str(path)]).stdout
+      output = [line for line in output.splitlines() if "SONAME" in line]
+      sonames = [line.strip().split(" ")[-1] for line in output]
+      if not any([soname == path.basename for soname in sonames]):
+        mismatches.append(str(path))
+        continue
+    return path
+  if mismatches:
+    auto_configure_fail(
+        "None of the libraries match their SONAME: " + ", ".join(mismatches))
+  auto_configure_fail("No library found under: " + ", ".join(paths))
+
 
 def _find_cuda_lib(
         lib,
         repository_ctx,
         cpu_value,
         basedir,
-        version = "",
+        version,
         static = False):
   """Finds the given CUDA or cuDNN library on the system.
 
@@ -740,16 +762,12 @@ def _find_cuda_lib(
       static: True if static library, False if shared object.
 
     Returns:
-      Returns a struct with the following fields:
-        file_name: The basename of the library found on the system.
-        path: The full path to the library.
+      Returns the path to the library.
     """
-  file_name = _lib_name(lib, cpu_value, version, static)
-  for relative_path in CUDA_LIB_PATHS:
-    path = repository_ctx.path("%s/%s%s" % (basedir, relative_path, file_name))
-    if path.exists:
-      return struct(file_name=file_name, path=str(path.realpath))
-  auto_configure_fail("Cannot find cuda library %s" % file_name)
+  file_name = lib_name(lib, cpu_value, version, static)
+  return find_lib(repository_ctx, [
+      "%s/%s%s" % (basedir, path, file_name) for path in CUDA_LIB_PATHS
+  ], check_soname = version and not static)
 
 
 def _find_cupti_header_dir(repository_ctx, cuda_config):
@@ -785,23 +803,17 @@ def _find_cupti_lib(repository_ctx, cuda_config):
       cuda_config: The cuda configuration as returned by _get_cuda_config.
 
     Returns:
-      Returns a struct with the following fields:
-        file_name: The basename of the library found on the system.
-        path: The full path to the library.
+      Returns the path to the library.
     """
-  file_name = _lib_name(
+  file_name = lib_name(
       "cupti",
       cuda_config.cpu_value,
       cuda_config.cuda_version,
   )
-  cuda_toolkit_path = cuda_config.cuda_toolkit_path
-  for relative_path in CUPTI_LIB_PATHS:
-    path = repository_ctx.path(
-        "%s/%s%s" % (cuda_toolkit_path, relative_path, file_name),)
-    if path.exists:
-      return struct(file_name=file_name, path=str(path.realpath))
-
-  auto_configure_fail("Cannot find cupti library %s" % file_name)
+  basedir = cuda_config.cuda_toolkit_path
+  return find_lib(repository_ctx, [
+      "%s/%s%s" % (basedir, path, file_name) for path in CUPTI_LIB_PATHS
+  ])
 
 
 def _find_libs(repository_ctx, cuda_config):
@@ -817,8 +829,12 @@ def _find_libs(repository_ctx, cuda_config):
   cpu_value = cuda_config.cpu_value
   return {
       "cuda":
-          _find_cuda_lib("cuda", repository_ctx, cpu_value,
-                         cuda_config.cuda_toolkit_path),
+          _find_cuda_lib(
+              "cuda",
+              repository_ctx,
+              cpu_value,
+              cuda_config.cuda_toolkit_path,
+              None),
       "cudart":
           _find_cuda_lib(
               "cudart",
@@ -1035,9 +1051,9 @@ def _create_dummy_repository(repository_ctx):
       "cuda:BUILD",
       {
           "%{cuda_driver_lib}":
-              _lib_name("cuda", cpu_value),
+              lib_name("cuda", cpu_value),
           "%{cudart_static_lib}":
-              _lib_name(
+              lib_name(
                   "cudart_static",
                   cpu_value,
                   static=True,
@@ -1045,19 +1061,19 @@ def _create_dummy_repository(repository_ctx):
           "%{cudart_static_linkopt}":
               _cudart_static_linkopt(cpu_value),
           "%{cudart_lib}":
-              _lib_name("cudart", cpu_value),
+              lib_name("cudart", cpu_value),
           "%{cublas_lib}":
-              _lib_name("cublas", cpu_value),
+              lib_name("cublas", cpu_value),
           "%{cusolver_lib}":
-              _lib_name("cusolver", cpu_value),
+              lib_name("cusolver", cpu_value),
           "%{cudnn_lib}":
-              _lib_name("cudnn", cpu_value),
+              lib_name("cudnn", cpu_value),
           "%{cufft_lib}":
-              _lib_name("cufft", cpu_value),
+              lib_name("cufft", cpu_value),
           "%{curand_lib}":
-              _lib_name("curand", cpu_value),
+              lib_name("curand", cpu_value),
           "%{cupti_lib}":
-              _lib_name("cupti", cpu_value),
+              lib_name("cupti", cpu_value),
           "%{copy_rules}":
               "",
           "%{cuda_headers}":
@@ -1071,16 +1087,16 @@ def _create_dummy_repository(repository_ctx):
   repository_ctx.file("cuda/cuda/include/cublas.h")
   repository_ctx.file("cuda/cuda/include/cudnn.h")
   repository_ctx.file("cuda/cuda/extras/CUPTI/include/cupti.h")
-  repository_ctx.file("cuda/cuda/lib/%s" % _lib_name("cuda", cpu_value))
-  repository_ctx.file("cuda/cuda/lib/%s" % _lib_name("cudart", cpu_value))
+  repository_ctx.file("cuda/cuda/lib/%s" % lib_name("cuda", cpu_value))
+  repository_ctx.file("cuda/cuda/lib/%s" % lib_name("cudart", cpu_value))
   repository_ctx.file(
-      "cuda/cuda/lib/%s" % _lib_name("cudart_static", cpu_value))
-  repository_ctx.file("cuda/cuda/lib/%s" % _lib_name("cublas", cpu_value))
-  repository_ctx.file("cuda/cuda/lib/%s" % _lib_name("cusolver", cpu_value))
-  repository_ctx.file("cuda/cuda/lib/%s" % _lib_name("cudnn", cpu_value))
-  repository_ctx.file("cuda/cuda/lib/%s" % _lib_name("curand", cpu_value))
-  repository_ctx.file("cuda/cuda/lib/%s" % _lib_name("cufft", cpu_value))
-  repository_ctx.file("cuda/cuda/lib/%s" % _lib_name("cupti", cpu_value))
+      "cuda/cuda/lib/%s" % lib_name("cudart_static", cpu_value))
+  repository_ctx.file("cuda/cuda/lib/%s" % lib_name("cublas", cpu_value))
+  repository_ctx.file("cuda/cuda/lib/%s" % lib_name("cusolver", cpu_value))
+  repository_ctx.file("cuda/cuda/lib/%s" % lib_name("cudnn", cpu_value))
+  repository_ctx.file("cuda/cuda/lib/%s" % lib_name("curand", cpu_value))
+  repository_ctx.file("cuda/cuda/lib/%s" % lib_name("cufft", cpu_value))
+  repository_ctx.file("cuda/cuda/lib/%s" % lib_name("cupti", cpu_value))
 
   # Set up cuda_config.h, which is used by
   # tensorflow/stream_executor/dso_loader.cc.
@@ -1269,9 +1285,9 @@ def _create_local_cuda_repository(repository_ctx):
   cuda_libs = _find_libs(repository_ctx, cuda_config)
   cuda_lib_srcs = []
   cuda_lib_outs = []
-  for lib in cuda_libs.values():
-    cuda_lib_srcs.append(lib.path)
-    cuda_lib_outs.append("cuda/lib/" + lib.file_name)
+  for path in cuda_libs.values():
+    cuda_lib_srcs.append(str(path))
+    cuda_lib_outs.append("cuda/lib/" + path.basename)
   copy_rules.append(make_copy_files_rule(
       repository_ctx,
       name = "cuda-lib",
@@ -1317,25 +1333,25 @@ def _create_local_cuda_repository(repository_ctx):
       "cuda:BUILD.windows" if _is_windows(repository_ctx) else "cuda:BUILD",
       {
           "%{cuda_driver_lib}":
-              cuda_libs["cuda"].file_name,
+              cuda_libs["cuda"].basename,
           "%{cudart_static_lib}":
-              cuda_libs["cudart_static"].file_name,
+              cuda_libs["cudart_static"].basename,
           "%{cudart_static_linkopt}":
               _cudart_static_linkopt(cuda_config.cpu_value,),
           "%{cudart_lib}":
-              cuda_libs["cudart"].file_name,
+              cuda_libs["cudart"].basename,
           "%{cublas_lib}":
-              cuda_libs["cublas"].file_name,
+              cuda_libs["cublas"].basename,
           "%{cusolver_lib}":
-              cuda_libs["cusolver"].file_name,
+              cuda_libs["cusolver"].basename,
           "%{cudnn_lib}":
-              cuda_libs["cudnn"].file_name,
+              cuda_libs["cudnn"].basename,
           "%{cufft_lib}":
-              cuda_libs["cufft"].file_name,
+              cuda_libs["cufft"].basename,
           "%{curand_lib}":
-              cuda_libs["curand"].file_name,
+              cuda_libs["curand"].basename,
           "%{cupti_lib}":
-              cuda_libs["cupti"].file_name,
+              cuda_libs["cupti"].basename,
           "%{copy_rules}":
               "\n".join(copy_rules),
           "%{cuda_headers}": ('":cuda-include",\n' + '        ":cudnn-include",'
diff --git a/third_party/nccl/nccl_configure.bzl b/third_party/nccl/nccl_configure.bzl
index f7d7d55383..59b5bd3fea 100644
--- a/third_party/nccl/nccl_configure.bzl
+++ b/third_party/nccl/nccl_configure.bzl
@@ -134,6 +134,7 @@ def _nccl_configure_impl(repository_ctx):
     nccl_version = ""
     if _TF_NCCL_VERSION in repository_ctx.os.environ:
         nccl_version = repository_ctx.os.environ[_TF_NCCL_VERSION].strip()
+        nccl_version = nccl_version.split(".")[0]
 
     if nccl_version == "":
         # Alias to open source build from @nccl_archive.
diff --git a/third_party/tensorrt/BUILD.tpl b/third_party/tensorrt/BUILD.tpl
index 17ab2fa119..a41ab808c7 100644
--- a/third_party/tensorrt/BUILD.tpl
+++ b/third_party/tensorrt/BUILD.tpl
@@ -16,9 +16,9 @@ cc_library(
 )
 
 cc_library(
-    name = "nv_infer",
-    srcs = [%{nv_infer}],
-    data = [%{nv_infer}],
+    name = "tensorrt",
+    srcs = %{tensorrt_libs},
+    data = %{tensorrt_libs},
     copts= cuda_default_copts(),
     deps = [
         "@local_config_cuda//cuda:cuda",
@@ -28,6 +28,5 @@ cc_library(
     visibility = ["//visibility:public"],
 )
 
-
 %{copy_rules}
 
diff --git a/third_party/tensorrt/tensorrt_configure.bzl b/third_party/tensorrt/tensorrt_configure.bzl
index 7cf845d5cc..10e243fa84 100644
--- a/third_party/tensorrt/tensorrt_configure.bzl
+++ b/third_party/tensorrt/tensorrt_configure.bzl
@@ -12,6 +12,8 @@ load(
     "auto_configure_fail",
     "get_cpu_value",
     "find_cuda_define",
+    "find_lib",
+    "lib_name",
     "matches_version",
     "make_copy_dir_rule",
     "make_copy_files_rule",
@@ -94,10 +96,11 @@ def _trt_lib_version(repository_ctx, trt_install_path):
         ("TensorRT library version detected from %s/%s (%s) does not match " +
          "TF_TENSORRT_VERSION (%s). To fix this rerun configure again.") %
         (trt_header_dir, "NvInfer.h", full_version, environ_version))
-  return environ_version
+  # Only use the major version to match the SONAME of the library.
+  return major_version
 
 
-def _find_trt_libs(repository_ctx, trt_install_path, trt_lib_version):
+def _find_trt_libs(repository_ctx, cpu_value, trt_install_path, trt_lib_version):
   """Finds the given TensorRT library on the system.
 
   Adapted from code contributed by Sami Kama (https://github.com/samikama).
@@ -109,30 +112,13 @@ def _find_trt_libs(repository_ctx, trt_install_path, trt_lib_version):
       by _trt_lib_version.
 
   Returns:
-    Map of library names to structs with the following fields:
-      src_file_path: The full path to the library found on the system.
-      dst_file_name: The basename of the target library.
+    The path to the library.
   """
-  objdump = repository_ctx.which("objdump")
   result = {}
   for lib in _TF_TENSORRT_LIBS:
-    dst_file_name = "lib%s.so.%s" % (lib, trt_lib_version)
-    src_file_path = repository_ctx.path("%s/%s" % (trt_install_path,
-                                                   dst_file_name))
-    if not src_file_path.exists:
-      auto_configure_fail(
-          "Cannot find TensorRT library %s" % str(src_file_path))
-    if objdump != None:
-      objdump_out = repository_ctx.execute([objdump, "-p", str(src_file_path)])
-      for line in objdump_out.stdout.splitlines():
-        if "SONAME" in line:
-          dst_file_name = line.strip().split(" ")[-1]
-    result.update({
-        lib:
-            struct(
-                dst_file_name=dst_file_name,
-                src_file_path=str(src_file_path.realpath))
-    })
+    file_name = lib_name("nvinfer", cpu_value, trt_lib_version)
+    path = find_lib(repository_ctx, ["%s/%s" % (trt_install_path, file_name)])
+    result[file_name] = path
   return result
 
 
@@ -144,15 +130,12 @@ def _tpl(repository_ctx, tpl, substitutions):
 def _create_dummy_repository(repository_ctx):
   """Create a dummy TensorRT repository."""
   _tpl(repository_ctx, "build_defs.bzl", {"%{if_tensorrt}": "if_false"})
-  substitutions = {
-      "%{tensorrt_genrules}": "",
-      "%{tensorrt_headers}": "",
-  }
-  for lib in _TF_TENSORRT_LIBS:
-    k = "%%{%s}" % lib.replace("nv", "nv_")
-    substitutions.update({k: ""})
-  _tpl(repository_ctx, "BUILD", substitutions)
 
+  _tpl(repository_ctx, "BUILD", {
+      "%{tensorrt_genrules}": "",
+      "%{tensorrt_headers}": "[]",
+      "%{tensorrt_libs}": "[]"
+  })
 
 def _tensorrt_configure_impl(repository_ctx):
   """Implementation of the tensorrt_configure repository rule."""
@@ -168,7 +151,8 @@ def _tensorrt_configure_impl(repository_ctx):
     _create_dummy_repository(repository_ctx)
     return
 
-  if (get_cpu_value(repository_ctx) != "Linux"):
+  cpu_value = get_cpu_value(repository_ctx)
+  if (cpu_value != "Linux"):
     auto_configure_fail("TensorRT is supported only on Linux.")
   if _TF_TENSORRT_VERSION not in repository_ctx.os.environ:
     auto_configure_fail("TensorRT library (libnvinfer) version is not set.")
@@ -179,12 +163,12 @@ def _tensorrt_configure_impl(repository_ctx):
 
   # Copy the library files.
   trt_lib_version = _trt_lib_version(repository_ctx, trt_install_path)
-  trt_libs = _find_trt_libs(repository_ctx, trt_install_path, trt_lib_version)
+  trt_libs = _find_trt_libs(repository_ctx, cpu_value, trt_install_path, trt_lib_version)
   trt_lib_srcs = []
   trt_lib_outs = []
-  for lib in trt_libs.values():
-    trt_lib_srcs.append(lib.src_file_path)
-    trt_lib_outs.append("tensorrt/lib/" + lib.dst_file_name)
+  for path in trt_libs.values():
+    trt_lib_srcs.append(str(path))
+    trt_lib_outs.append("tensorrt/lib/" + path.basename)
   copy_rules = [make_copy_files_rule(
       repository_ctx,
       name = "tensorrt_lib",
@@ -212,15 +196,11 @@ def _tensorrt_configure_impl(repository_ctx):
   _tpl(repository_ctx, "build_defs.bzl", {"%{if_tensorrt}": "if_true"})
 
   # Set up BUILD file.
-  substitutions = {
+  _tpl(repository_ctx, "BUILD", {
       "%{copy_rules}": "\n".join(copy_rules),
       "%{tensorrt_headers}": '":tensorrt_include"',
-  }
-  for lib in _TF_TENSORRT_LIBS:
-    k = "%%{%s}" % lib.replace("nv", "nv_")
-    v = '"tensorrt/lib/%s"' % trt_libs[lib].dst_file_name
-    substitutions.update({k: v})
-  _tpl(repository_ctx, "BUILD", substitutions)
+      "%{tensorrt_libs}": str(trt_lib_outs),
+  })
 
 
 tensorrt_configure = repository_rule(
diff --git a/third_party/toolchains/preconfig/ubuntu14.04/tensorrt5/BUILD b/third_party/toolchains/preconfig/ubuntu14.04/tensorrt5/BUILD
index 941d99a0db..04442d9c51 100755
--- a/third_party/toolchains/preconfig/ubuntu14.04/tensorrt5/BUILD
+++ b/third_party/toolchains/preconfig/ubuntu14.04/tensorrt5/BUILD
@@ -17,7 +17,7 @@ cc_library(
 )
 
 cc_library(
-    name = "nv_infer",
+    name = "tensorrt",
     srcs = ["tensorrt/lib/libnvinfer.so.5"],
     copts = cuda_default_copts(),
     data = ["tensorrt/lib/libnvinfer.so.5"],
-- 
GitLab


From 3dedb64c95cb1fed7ce6f521e86c775a098b0525 Mon Sep 17 00:00:00 2001
From: Pete Warden <petewarden@google.com>
Date: Thu, 17 Jan 2019 16:00:25 -0800
Subject: [PATCH 0935/1095] Update Micro documentation

PiperOrigin-RevId: 229834532
---
 tensorflow/lite/experimental/micro/README.md | 131 +++++++++++++++++++
 1 file changed, 131 insertions(+)

diff --git a/tensorflow/lite/experimental/micro/README.md b/tensorflow/lite/experimental/micro/README.md
index 4c3eab38fa..b47e0feb79 100644
--- a/tensorflow/lite/experimental/micro/README.md
+++ b/tensorflow/lite/experimental/micro/README.md
@@ -509,3 +509,134 @@ Another  badly-formed  format string
 
 If not, you'll need to debug what went wrong, but hopefully with this small
 starting project it should be manageable.
+
+### Troubleshooting
+
+When we've been porting to new platforms, it's often been hard to figure out
+some of the fundamentals like linker settings and other toolchain setup flags.
+If you are having trouble, see if you can find a simple example program for your
+platform, like one that just blinks an LED. If you're able to build and run that
+successfully, then start to swap in parts of the TF Lite Micro codebase to that
+working project, taking it a step at a time and ensuring it's still working
+after every change. For example, a first step might be to paste in your
+`DebugLog()` implementation and call `DebugLog("Hello World!")` from the main
+function.
+
+Another common problem on embedded platforms is the stack size being too small.
+Mbed defaults to 4KB for the main thread's stack, which is too small for most
+models since TensorFlow Lite allocates buffers and other data structures that
+require more memory. The exact size will depend on which model you're running,
+but try increasing it if you are running into strange corruption issues that
+might be related to stack overwriting.
+
+### Optimizing for your Platform
+
+The default reference implementations in TensorFlow Lite Micro are written to be
+portable and easy to understand, not fast, so you'll want to replace performance
+critical parts of the code with versions specifically tailored to your
+architecture. The framework has been designed with this in mind, and we hope the
+combination of small modules and many tests makes it as straightforward as
+possible to swap in your own code a piece at a time, ensuring you have a working
+version at every step.
+
+### Code Module Organization
+
+We have adopted a system of small modules with platform-specific implementations
+to help with portability. Every module is just a standard `.h` header file
+containing the interface (either functions or a class), with an accompanying
+reference implementation in a `.cc` with the same name. The source file
+implements all of the code that's declared in the header. If you have a
+specialized implementation, you can create a folder in the same directory as the
+header and reference source, name it after your platform, and put your
+implementation in a `.cc` file inside that folder. We've already seen one
+example of this, where the Mbed and Bluepill versions of `DebugLog()` are inside
+[mbed](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/experimental/micro/mbed)
+and
+[bluepill](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/experimental/micro/bluepill)
+folders, children of the
+[same directory](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/experimental/micro)
+where the stdio-based
+[`debug_log.cc`](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/experimental/micro/debug_log.cc)
+reference implementation is found.
+
+The advantage of this approach is that we can automatically pick specialized
+implementations based on the current build target, without having to manually
+edit build files for every new platform. It allows incremental optimizations
+from a always-working foundation, without cluttering the reference
+implementations with a lot of variants.
+
+To see why we're doing this, it's worth looking at the alternatives. TensorFlow
+Lite has traditionally used preprocessor macros to separate out some
+platform-specific code within particular files, for example:
+
+```
+#ifndef USE_NEON
+#if defined(__ARM_NEON__) || defined(__ARM_NEON)
+#define USE_NEON
+#include <arm_neon.h>
+#endif
+```
+
+There’s also a tradition in gemmlowp of using file suffixes to indicate
+platform-specific versions of particular headers, with kernel_neon.h being
+included by kernel.h if `USE_NEON` is defined. As a third variation, kernels are
+separated out using a directory structure, with
+tensorflow/lite/kernels/internal/reference containing portable implementations,
+and tensorflow/lite/kernels/internal/optimized holding versions optimized for
+NEON on Arm platforms.
+
+These approaches are hard to extend to multiple platforms. Using macros means
+that platform-specific code is scattered throughout files in a hard-to-find way,
+and can make following the control flow difficult since you need to understand
+the macro state to trace it. For example, I temporarily introduced a bug that
+disabled NEON optimizations for some kernels when I removed
+tensorflow/lite/kernels/internal/common.h from their includes, without realizing
+it was where USE_NEON was defined!
+
+It’s also tough to port to different build systems, since figuring out the right
+combination of macros to use can be hard, especially since some of them are
+automatically defined by the compiler, and others are only set by build scripts,
+often across multiple rules.
+
+The approach we are using extends the file system approach that we use for
+kernel implementations, but with some specific conventions:
+
+-   For each module in TensorFlow Lite, there will be a parent directory that
+    contains tests, interface headers used by other modules, and portable
+    implementations of each part.
+-   Portable means that the code doesn’t include code from any libraries except
+    flatbuffers, or other TF Lite modules. You can include a limited subset of
+    standard C or C++ headers, but you can’t use any functions that require
+    linking against those libraries, including fprintf, etc. You can link
+    against functions in the standard math library, in <math.h>.
+-   Specialized implementations are held inside subfolders of the parent
+    directory, named after the platform or library that they depend on. So, for
+    example if you had my_module/foo.cc, a version that used RISC-V extensions
+    would live in my_module/riscv/foo.cc. If you had a version that used the
+    CMSIS library, it should be in my_module/cmsis/foo.cc.
+-   These specialized implementations should completely replace the top-level
+    implementations. If this involves too much code duplication, the top-level
+    implementation should be split into smaller files, so only the
+    platform-specific code needs to be replaced.
+-   There is a convention about how build systems pick the right implementation
+    file. There will be an ordered list of 'tags' defining the preferred
+    implementations, and to generate the right list of source files, each module
+    will be examined in turn. If a subfolder with a tag’s name contains a .cc
+    file with the same base name as one in the parent folder, then it will
+    replace the parent folder’s version in the list of build files. If there are
+    multiple subfolders with matching tags and file names, then the tag that’s
+    latest in the ordered list will be chosen. This allows us to express “I’d
+    like generically-optimized fixed point if it’s available, but I’d prefer
+    something using the CMSIS library” using the list 'fixed_point cmsis'. These
+    tags are passed in as `TAGS="<foo"` on the command line when you use the
+    main Makefile to build.
+-   There is an implicit “reference” tag at the start of every list, so that
+    it’s possible to support directory structures like the current
+    tensorflow/kernels/internal where portable implementations are held in a
+    “reference” folder that’s a sibling to the NEON-optimized folder.
+-   The headers for each unit in a module should remain platform-agnostic, and
+    be the same for all implementations. Private headers inside a sub-folder can
+    be used as needed, but shouldn’t be referred to by any portable code at the
+    top level.
+-   Tests should be at the parent level, with no platform-specific code.
+-   No platform-specific macros or #ifdef’s should be used in any portable code.
-- 
GitLab


From 07706e401e64577720476e341744ead9eb2bf3e0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 17 Jan 2019 16:00:57 -0800
Subject: [PATCH 0936/1095] Remove alias redirection for RBE python.

PiperOrigin-RevId: 229834629
---
 tensorflow/opensource_only.files    |  1 -
 third_party/py/python_configure.bzl |  4 +---
 third_party/py/remote.BUILD.tpl     | 13 -------------
 3 files changed, 1 insertion(+), 17 deletions(-)
 delete mode 100644 third_party/py/remote.BUILD.tpl

diff --git a/tensorflow/opensource_only.files b/tensorflow/opensource_only.files
index 5f1ec1d673..9c4dd2f4b8 100644
--- a/tensorflow/opensource_only.files
+++ b/tensorflow/opensource_only.files
@@ -217,7 +217,6 @@ tensorflow/third_party/protobuf/BUILD
 tensorflow/third_party/tflite_mobilenet.BUILD
 tensorflow/third_party/py/BUILD
 tensorflow/third_party/py/BUILD.tpl
-tensorflow/third_party/py/remote.BUILD.tpl
 tensorflow/third_party/py/numpy/BUILD
 tensorflow/third_party/py/python_configure.bzl
 tensorflow/third_party/termcolor.BUILD
diff --git a/third_party/py/python_configure.bzl b/third_party/py/python_configure.bzl
index 53264630a1..9a7581c246 100644
--- a/third_party/py/python_configure.bzl
+++ b/third_party/py/python_configure.bzl
@@ -294,9 +294,7 @@ def _create_local_python_repository(repository_ctx):
 def _create_remote_python_repository(repository_ctx, remote_config_repo):
   """Creates pointers to a remotely configured repo set up to build with Python.
   """
-  _tpl(repository_ctx, "remote.BUILD", {
-      "%{REMOTE_PYTHON_REPO}": remote_config_repo,
-  }, "BUILD")
+  repository_ctx.template("BUILD", Label(remote_config_repo + ":BUILD"), {})
 
 
 def _python_autoconf_impl(repository_ctx):
diff --git a/third_party/py/remote.BUILD.tpl b/third_party/py/remote.BUILD.tpl
deleted file mode 100644
index edcac41ec6..0000000000
--- a/third_party/py/remote.BUILD.tpl
+++ /dev/null
@@ -1,13 +0,0 @@
-licenses(["restricted"])
-
-package(default_visibility = ["//visibility:public"])
-
-alias(
-    name = "python_headers",
-    actual = "%{REMOTE_PYTHON_REPO}:python_headers",
-)
-
-alias(
-    name = "numpy_headers",
-    actual = "%{REMOTE_PYTHON_REPO}:numpy_headers",
-)
-- 
GitLab


From fe342218ad5d083b825f9347f5c35d929a8d745b Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Thu, 17 Jan 2019 16:12:06 -0800
Subject: [PATCH 0937/1095] [tf.data] Fix usage of `test_util.run_v1_only` in
 MultiDeviceIterator and temporarily disable a failing test.

PiperOrigin-RevId: 229836608
---
 .../multi_device_iterator_test.py             | 30 +++++++++----------
 1 file changed, 14 insertions(+), 16 deletions(-)

diff --git a/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py b/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py
index 0c88d7533f..2bc0d8f90d 100644
--- a/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py
+++ b/tensorflow/python/data/kernel_tests/multi_device_iterator_test.py
@@ -34,15 +34,12 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
 
 
-# TODO(b/121264236): Once we have a mechanism to have multiple devices in eager
-# / V2 mode, we should remove this annotation and the run_v1_only annotations
-# as well.
 @test_util.run_all_in_graph_and_eager_modes
 class MultiDeviceIteratorTest(test_base.DatasetTestBase,
                               parameterized.TestCase):
 
-  @test_util.run_v1_only
   @parameterized.parameters(0, 1, 42,)
+  @test_util.run_v1_only("b/121264236")
   def testInitOnly(self, num_inits):
     dataset = dataset_ops.Dataset.range(10)
     multi_device_iterator = multi_device_iterator_ops.MultiDeviceIterator(
@@ -53,7 +50,7 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase,
       for _ in range(num_inits):
         self.evaluate(multi_device_iterator.initializer)
 
-  @test_util.run_v1_only
+  @test_util.run_v1_only("b/121264236")
   def testBasic(self):
     dataset = dataset_ops.Dataset.range(10)
     multi_device_iterator = multi_device_iterator_ops.MultiDeviceIterator(
@@ -71,7 +68,7 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase,
         self.evaluate(elem_on_1)
         self.evaluate(elem_on_2)
 
-  @test_util.run_v1_only
+  @test_util.run_v1_only("b/121264236")
   def testOneOnSameDevice(self):
     with ops.device("/cpu:0"):
       dataset = dataset_ops.Dataset.range(10)
@@ -90,7 +87,7 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase,
         self.evaluate(elem_on_1)
         self.evaluate(elem_on_2)
 
-  @test_util.run_v1_only
+  @test_util.run_v1_only("b/121264236")
   def testRepeatDevices(self):
     with ops.device("/cpu:0"):
       dataset = dataset_ops.Dataset.range(20)
@@ -115,7 +112,7 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase,
         self.evaluate(elem_on_3)
         self.evaluate(elem_on_4)
 
-  @test_util.run_v1_only
+  @test_util.run_v1_only("b/121264236")
   def testNotFullyDivisible(self):
     dataset = dataset_ops.Dataset.range(9)
     multi_device_iterator = multi_device_iterator_ops.MultiDeviceIterator(
@@ -135,7 +132,7 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase,
         self.evaluate(elem_on_1)
         self.evaluate(elem_on_2)
 
-  @test_util.run_v1_only
+  @test_util.run_v1_only("b/121264236")
   def testGetNextAsOptional(self):
     if context.executing_eagerly():
       return
@@ -172,7 +169,7 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase,
       with self.assertRaises(errors.InvalidArgumentError):
         self.evaluate(elem_on_2_t)
 
-  @test_util.run_v1_only
+  @test_util.run_v1_only("b/121264236")
   def testUneven(self):
     dataset = dataset_ops.Dataset.range(10)
     multi_device_iterator = multi_device_iterator_ops.MultiDeviceIterator(
@@ -192,7 +189,7 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase,
         self.evaluate(elem_on_1)
         self.evaluate(elem_on_2)
 
-  @test_util.run_v1_only
+  @test_util.run_v1_only("b/121264236")
   def testMultipleInitializationsGraph(self):
     if context.executing_eagerly():
       return
@@ -214,8 +211,9 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase,
         self.assertEqual([(i, 0), (i, 1)], self.evaluate([elem_on_1,
                                                           elem_on_2]))
 
-  @test_util.run_v1_only
+  @test_util.run_v1_only("b/121264236")
   def testMultipleInitializationsEager(self):
+    self.skipTest("b/123023614")
     if not context.executing_eagerly():
       return
 
@@ -230,7 +228,7 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase,
       elem_on_1, elem_on_2 = multi_device_iterator.get_next()
       self.assertEqual([(0, 0), (1, 1)], self.evaluate([elem_on_1, elem_on_2]))
 
-  @test_util.run_v1_only
+  @test_util.run_v1_only("b/121264236")
   def testBasicGpu(self):
     if not test_util.is_gpu_available():
       self.skipTest("No GPU available")
@@ -251,7 +249,7 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase,
         self.evaluate(elem_on_1)
         self.evaluate(elem_on_2)
 
-  @test_util.run_v1_only
+  @test_util.run_v1_only("b/121264236")
   def testUnevenGpu(self):
     if not test_util.is_gpu_available():
       self.skipTest("No GPU available")
@@ -274,7 +272,7 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase,
         self.evaluate(elem_on_1)
         self.evaluate(elem_on_2)
 
-  @test_util.run_v1_only
+  @test_util.run_v1_only("b/121264236")
   def testGetNextAsOptionalGpu(self):
     if not test_util.is_gpu_available() or context.executing_eagerly():
       self.skipTest("No GPU available")
@@ -311,7 +309,7 @@ class MultiDeviceIteratorTest(test_base.DatasetTestBase,
       with self.assertRaises(errors.InvalidArgumentError):
         self.evaluate(elem_on_2_t)
 
-  @test_util.run_v1_only
+  @test_util.run_v1_only("b/121264236")
   def testOptimization(self):
     dataset = dataset_ops.Dataset.range(10)
     dataset = dataset.apply(optimization.assert_next(["MemoryCacheImpl"]))
-- 
GitLab


From ef10203524cd332afb96ce0fcd3a911654d40284 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Thu, 17 Jan 2019 16:25:57 -0800
Subject: [PATCH 0938/1095] Remove tf.train.ProximalGradientDescentOptimizer
 from the v2 API

I believe this was an oversight during API QA efforts

PiperOrigin-RevId: 229838796
---
 .../training/proximal_gradient_descent.py     |  2 +-
 ...-proximal-gradient-descent-optimizer.pbtxt | 51 -------------------
 .../api/golden/v2/tensorflow.train.pbtxt      |  4 --
 tensorflow/tools/compatibility/renames_v2.py  |  3 +-
 4 files changed, 3 insertions(+), 57 deletions(-)
 delete mode 100644 tensorflow/tools/api/golden/v2/tensorflow.train.-proximal-gradient-descent-optimizer.pbtxt

diff --git a/tensorflow/python/training/proximal_gradient_descent.py b/tensorflow/python/training/proximal_gradient_descent.py
index 369b6cbb50..6eca0e6cb5 100644
--- a/tensorflow/python/training/proximal_gradient_descent.py
+++ b/tensorflow/python/training/proximal_gradient_descent.py
@@ -27,7 +27,7 @@ from tensorflow.python.training import training_ops
 from tensorflow.python.util.tf_export import tf_export
 
 
-@tf_export("train.ProximalGradientDescentOptimizer")
+@tf_export(v1=["train.ProximalGradientDescentOptimizer"])
 class ProximalGradientDescentOptimizer(optimizer.Optimizer):
   # pylint: disable=line-too-long
   """Optimizer that implements the proximal gradient descent algorithm.
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.-proximal-gradient-descent-optimizer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.-proximal-gradient-descent-optimizer.pbtxt
deleted file mode 100644
index eb9a86183e..0000000000
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.-proximal-gradient-descent-optimizer.pbtxt
+++ /dev/null
@@ -1,51 +0,0 @@
-path: "tensorflow.train.ProximalGradientDescentOptimizer"
-tf_class {
-  is_instance: "<class \'tensorflow.python.training.proximal_gradient_descent.ProximalGradientDescentOptimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.optimizer.Optimizer\'>"
-  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
-  is_instance: "<type \'object\'>"
-  member {
-    name: "GATE_GRAPH"
-    mtype: "<type \'int\'>"
-  }
-  member {
-    name: "GATE_NONE"
-    mtype: "<type \'int\'>"
-  }
-  member {
-    name: "GATE_OP"
-    mtype: "<type \'int\'>"
-  }
-  member_method {
-    name: "__init__"
-    argspec: "args=[\'self\', \'learning_rate\', \'l1_regularization_strength\', \'l2_regularization_strength\', \'use_locking\', \'name\'], varargs=None, keywords=None, defaults=[\'0.0\', \'0.0\', \'False\', \'ProximalGradientDescent\'], "
-  }
-  member_method {
-    name: "apply_gradients"
-    argspec: "args=[\'self\', \'grads_and_vars\', \'global_step\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
-  member_method {
-    name: "compute_gradients"
-    argspec: "args=[\'self\', \'loss\', \'var_list\', \'gate_gradients\', \'aggregation_method\', \'colocate_gradients_with_ops\', \'grad_loss\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'None\', \'False\', \'None\'], "
-  }
-  member_method {
-    name: "get_name"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_slot"
-    argspec: "args=[\'self\', \'var\', \'name\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "get_slot_names"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-  member_method {
-    name: "minimize"
-    argspec: "args=[\'self\', \'loss\', \'global_step\', \'var_list\', \'gate_gradients\', \'aggregation_method\', \'colocate_gradients_with_ops\', \'name\', \'grad_loss\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'1\', \'None\', \'False\', \'None\', \'None\'], "
-  }
-  member_method {
-    name: "variables"
-    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-  }
-}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt
index a3ace15ca2..8b39086ed1 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.train.pbtxt
@@ -60,10 +60,6 @@ tf_module {
     name: "JobDef"
     mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
   }
-  member {
-    name: "ProximalGradientDescentOptimizer"
-    mtype: "<type \'type\'>"
-  }
   member {
     name: "SequenceExample"
     mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index fb2303e9dd..6695eb4883 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -613,10 +613,10 @@ renames = {
     'tf.summary.FileWriter': 'tf.compat.v1.summary.FileWriter',
     'tf.summary.FileWriterCache': 'tf.compat.v1.summary.FileWriterCache',
     'tf.summary.SessionLog': 'tf.compat.v1.summary.SessionLog',
-    'tf.summary.audio': 'tf.compat.v1.summary.audio',
     'tf.summary.Summary': 'tf.compat.v1.summary.Summary',
     'tf.summary.SummaryDescription': 'tf.compat.v1.summary.SummaryDescription',
     'tf.summary.TaggedRunMetadata': 'tf.compat.v1.summary.TaggedRunMetadata',
+    'tf.summary.audio': 'tf.compat.v1.summary.audio',
     'tf.summary.get_summary_description': 'tf.compat.v1.summary.get_summary_description',
     'tf.summary.histogram': 'tf.compat.v1.summary.histogram',
     'tf.summary.image': 'tf.compat.v1.summary.image',
@@ -664,6 +664,7 @@ renames = {
     'tf.train.Optimizer': 'tf.compat.v1.train.Optimizer',
     'tf.train.ProfilerHook': 'tf.estimator.ProfilerHook',
     'tf.train.ProximalAdagradOptimizer': 'tf.compat.v1.train.ProximalAdagradOptimizer',
+    'tf.train.ProximalGradientDescentOptimizer': 'tf.compat.v1.train.ProximalGradientDescentOptimizer',
     'tf.train.QueueRunner': 'tf.compat.v1.train.QueueRunner',
     'tf.train.RMSPropOptimizer': 'tf.compat.v1.train.RMSPropOptimizer',
     'tf.train.Saver': 'tf.compat.v1.train.Saver',
-- 
GitLab


From c0ea4d61fee3ce6fd563de359bc46dc3260be6c4 Mon Sep 17 00:00:00 2001
From: Edward Loper <edloper@google.com>
Date: Thu, 17 Jan 2019 16:31:45 -0800
Subject: [PATCH 0939/1095] Add CompositeTensor base class.

CompositeTensor is a new base class for tensor-like objects that are composed from tf.Tensors.

This CL:

1. Adds the CompositeTensor base class, which defines two special methods for composing & decomposing CompositeTensors.

2. Adds an "expand_composites" argument to several of the functions in tensorflow nest, which controls whether composite tensors are expanded or treated as atomic values.

3. Updates control_flow_ops.py ops to use composite tensors, rather than special-casing behavior based on type checks.

PiperOrigin-RevId: 229839748
---
 tensorflow/contrib/framework/__init__.py      |   1 +
 tensorflow/python/BUILD                       |  28 ++
 tensorflow/python/eager/function.py           |   3 +-
 .../python/framework/composite_tensor.py      | 100 ++++
 .../python/framework/composite_tensor_test.py | 101 ++++
 tensorflow/python/framework/func_graph.py     |  65 +--
 tensorflow/python/framework/ops.py            | 170 ++++++-
 tensorflow/python/framework/sparse_tensor.py  |  29 +-
 .../kernel_tests/control_flow_ops_py_test.py  | 196 ++++++--
 tensorflow/python/ops/control_flow_ops.py     | 450 +++++++-----------
 tensorflow/python/ops/ragged/ragged_tensor.py |  50 +-
 tensorflow/python/util/nest.py                |  79 ++-
 tensorflow/python/util/util.cc                |  87 +++-
 tensorflow/python/util/util.h                 |  32 +-
 tensorflow/python/util/util.i                 |  30 +-
 .../v1/tensorflow.-indexed-slices.pbtxt       |   1 +
 .../golden/v1/tensorflow.-ragged-tensor.pbtxt |   1 +
 .../golden/v1/tensorflow.-sparse-tensor.pbtxt |   1 +
 .../v1/tensorflow.sparse.-sparse-tensor.pbtxt |   1 +
 .../v2/tensorflow.-indexed-slices.pbtxt       |   1 +
 .../golden/v2/tensorflow.-ragged-tensor.pbtxt |   1 +
 .../golden/v2/tensorflow.-sparse-tensor.pbtxt |   1 +
 .../v2/tensorflow.sparse.-sparse-tensor.pbtxt |   1 +
 23 files changed, 1008 insertions(+), 421 deletions(-)
 create mode 100644 tensorflow/python/framework/composite_tensor.py
 create mode 100644 tensorflow/python/framework/composite_tensor_test.py

diff --git a/tensorflow/contrib/framework/__init__.py b/tensorflow/contrib/framework/__init__.py
index e72e50585a..ff65c1586c 100644
--- a/tensorflow/contrib/framework/__init__.py
+++ b/tensorflow/contrib/framework/__init__.py
@@ -130,6 +130,7 @@ _allowed_symbols = ['nest']
 _nest_allowed_symbols = [
     'assert_same_structure',
     'is_sequence',
+    'is_sequence_or_composite',
     'flatten',
     'flatten_dict_items',
     'pack_sequence_as',
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index d4e6cbf804..e6f8201fc7 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -671,6 +671,7 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":common_shapes",
+        ":composite_tensor",
         ":cpp_shape_inference_proto_py",
         ":errors",
         ":framework_fast_tensor_util",
@@ -996,12 +997,39 @@ py_library(
     srcs = ["framework/sparse_tensor.py"],
     srcs_version = "PY2AND3",
     deps = [
+        ":composite_tensor",
         ":dtypes",
         ":framework_ops",
         ":tensor_util",
     ],
 )
 
+py_library(
+    name = "composite_tensor",
+    srcs = ["framework/composite_tensor.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":dtypes",
+        ":framework_ops",
+        ":tensor_util",
+    ],
+)
+
+py_test(
+    name = "framework_composite_tensor_test",
+    srcs = ["framework/composite_tensor_test.py"],
+    main = "framework/composite_tensor_test.py",
+    srcs_version = "PY2AND3",
+    deps = [
+        ":composite_tensor",
+        ":framework",
+        ":framework_for_generated_wrappers",
+        ":framework_test_lib",
+        ":platform_test",
+        "//tensorflow/core:protos_all_py",
+    ],
+)
+
 # This target is maintained separately from :util to provide separate visibility
 # for legacy users who were granted visibility when the functions were private
 # members of ops.Graph.
diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index 9c3e20f936..9642b2ba7c 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -689,7 +689,8 @@ class ConcreteFunction(object):
     # Clear captures, since we pass them in as inputs.
     backwards_graph.captures = {}
     backwards_graph.outputs.extend(
-        grad for grad in func_graph_module.flatten(gradients_wrt_inputs)
+        grad
+        for grad in nest.flatten(gradients_wrt_inputs, expand_composites=True)
         if grad is not None)
     backwards_graph.structured_outputs = gradients_wrt_inputs
     self._backward_graph_function = ConcreteFunction(
diff --git a/tensorflow/python/framework/composite_tensor.py b/tensorflow/python/framework/composite_tensor.py
new file mode 100644
index 0000000000..5c9a292e36
--- /dev/null
+++ b/tensorflow/python/framework/composite_tensor.py
@@ -0,0 +1,100 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tensor-like objects that are composed from tf.Tensors."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import abc
+
+import six
+
+from tensorflow.python import pywrap_tensorflow
+
+
+@six.add_metaclass(abc.ABCMeta)
+class CompositeTensor(object):
+  """Abstract base class for Tensor-like objects that are composed from Tensors.
+
+  Each `CompositeTensor` can be decomposed into a structured collection of
+  component `tf.Tensor`s, and reconstructed from those components.
+
+  The `tensorflow.python.util.nest` module has support for treating composite
+  tensors as structure, which makes it easy to flatten and reconstruct
+  composite tensors (or larger structures that contain composite tensors).
+  E.g.:
+
+  ```python
+  ct = ...  # Create a composite tensor.
+  flat_list_of_tensors = nest.flatten(ct, expand_composites=True)
+  transformed_list_of_tensors = ...  # do something with the flat tensors.
+  result = nest.pack_sequence_as(ct, transformed_list_of_tensors)
+  ```
+  """
+
+  @abc.abstractmethod
+  def _to_components(self):
+    """Decomposes this composite tensor into its components.
+
+    Returns:
+      The components that comprise this composite tensor: a nested structure
+      (as defined by `tf.python.util.nest`) whose values are `tf.Tensor`s or
+      `CompositeTensor`s.
+    """
+    raise NotImplementedError("CompositeTensor._to_components")
+
+  @abc.abstractmethod
+  def _from_components(cls, components):  # pylint: disable=no-self-argument
+    """Creates a composite tensor of type `cls` from components.
+
+    Args:
+      components: The components that should be used to form the
+        composite tensor: a nested structure (as defined by
+        `tf.python.util.nest`) whose values are tf.Tensors or composite
+        tensors.
+
+    Returns:
+      A `CompositeTensor` of type `cls`.
+    """
+    raise NotImplementedError("CompositeTensor._from_components")
+
+  @abc.abstractmethod
+  def _shape_invariant_to_components(self, shape=None):
+    """Converts a shape invariant into invariants for individual components.
+
+    Args:
+      shape: A `tf.TensorShape` object.  The shape invariant for this
+        `CompositeTensor`, or `None` if a default shape invariant should be
+        used (based on the value of this `CompositeTensor`).
+
+    Returns:
+      A nested structure whose values are `tf.TensorShape` objects, specifying
+      the shape invariants for the tensors that comprise this `CompositeTensor`.
+    """
+    raise NotImplementedError("CompositeTensor._shape_invariant_to_components")
+
+  @abc.abstractproperty
+  def _is_graph_tensor(self):
+    """Returns True if this tensor's components belong to a TF graph."""
+    raise NotImplementedError("CompositeTensor._is_symbolic_tensor")
+
+
+pywrap_tensorflow.RegisterType("CompositeTensor", CompositeTensor)
+
+
+# @TODO(edloper): Can we replace convert_to_tensor_or_xyz with just
+# convert_to_tensor_or_composite?  Alternatively, should composite tensors
+# register a dispatch override for tf.convert_to_tensor?
diff --git a/tensorflow/python/framework/composite_tensor_test.py b/tensorflow/python/framework/composite_tensor_test.py
new file mode 100644
index 0000000000..f249faa5d6
--- /dev/null
+++ b/tensorflow/python/framework/composite_tensor_test.py
@@ -0,0 +1,101 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for tensorflow.python.framework.composite_tensor."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import composite_tensor
+from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.framework import test_util
+from tensorflow.python.platform import googletest
+from tensorflow.python.util import nest
+
+
+class TestCompositeTensor(composite_tensor.CompositeTensor):
+
+  def __init__(self, *components):
+    self._components = components
+
+  def _to_components(self):
+    return self._components
+
+  @classmethod
+  def _from_components(cls, components):
+    return cls(*components)
+
+  def _shape_invariant_to_components(self, shape=None):
+    raise NotImplementedError('CompositeTensor._shape_invariant_to_components')
+
+  def _is_graph_tensor(self):
+    return True
+
+
+class CompositeTensorTest(test_util.TensorFlowTestCase):
+
+  def assertNestEqual(self, a, b, expand_composites=False):
+    if isinstance(a, dict):
+      self.assertIsInstance(b, dict)
+      self.assertEqual(set(a), set(b))
+      for key in a:
+        self.assertNestEqual(a[key], b[key])
+    elif isinstance(a, (list, tuple)):
+      self.assertIsInstance(b, (list, tuple))
+      self.assertEqual(len(a), len(b))
+      for a_val, b_val in zip(a, b):
+        self.assertNestEqual(a_val, b_val)
+    elif expand_composites and isinstance(a, composite_tensor.CompositeTensor):
+      self.assertIsInstance(b, composite_tensor.CompositeTensor)
+      self.assertNestEqual(a._to_components(),
+                           b._to_components())
+
+  def testNestFlatten(self):
+    st1 = sparse_tensor.SparseTensor([[0, 3], [7, 2]], [1, 2], [10, 10])
+    st2 = sparse_tensor.SparseTensor([[1, 2, 3]], ['a'], [10, 10, 10])
+    structure = [[st1], 'foo', {'y': [st2]}]
+    x = nest.flatten(structure, expand_composites=True)
+    self.assertEqual(len(x), 7)
+    self.assertIs(x[0], st1.indices)
+    self.assertIs(x[1], st1.values)
+    self.assertIs(x[2], st1.dense_shape)
+    self.assertEqual(x[3], 'foo')
+    self.assertIs(x[4], st2.indices)
+    self.assertIs(x[5], st2.values)
+    self.assertIs(x[6], st2.dense_shape)
+
+  def testNestPackSequenceAs(self):
+    st1 = sparse_tensor.SparseTensor([[0, 3], [7, 2]], [1, 2], [10, 10])
+    st2 = sparse_tensor.SparseTensor([[1, 2, 3]], ['a'], [10, 10, 10])
+    structure1 = [[st1], 'foo', {'y': [st2]}]
+    flat = [st2.indices, st2.values, st2.dense_shape, 'bar',
+            st1.indices, st1.values, st1.dense_shape]
+    result = nest.pack_sequence_as(structure1, flat, expand_composites=True)
+    expected = [[st2], 'bar', {'y': [st1]}]
+    self.assertNestEqual(expected, result)
+
+  def testAssertSameStructure(self):
+    st1 = sparse_tensor.SparseTensor([[0]], [0], [100])
+    st2 = sparse_tensor.SparseTensor([[0, 3]], ['x'], [100, 100])
+    test = TestCompositeTensor(st1.indices, st1.values, st1.dense_shape)
+    nest.assert_same_structure(st1, st2, expand_composites=False)
+    nest.assert_same_structure(st1, st2, expand_composites=True)
+    nest.assert_same_structure(st1, test, expand_composites=False)
+    with self.assertRaises(TypeError):
+      nest.assert_same_structure(st1, test, expand_composites=True)
+
+
+if __name__ == '__main__':
+  googletest.main()
diff --git a/tensorflow/python/framework/func_graph.py b/tensorflow/python/framework/func_graph.py
index 36f7aa67c7..28d829e89a 100644
--- a/tensorflow/python/framework/func_graph.py
+++ b/tensorflow/python/framework/func_graph.py
@@ -26,7 +26,6 @@ from tensorflow.python.eager import context
 from tensorflow.python.eager import tape
 from tensorflow.python.eager.graph_only_ops import graph_placeholder
 from tensorflow.python.framework import ops
-from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import tensor_spec
 from tensorflow.python.framework.auto_control_deps import AutomaticControlDependencies
 from tensorflow.python.ops import array_ops
@@ -489,7 +488,7 @@ def func_graph_from_py_func(name,
           x = array_ops.identity(op_return_value)
       elif not isinstance(x, tensor_array_ops.TensorArray):
         try:
-          x = ops.convert_to_tensor_or_indexed_slices(x)
+          x = ops.convert_to_tensor_or_composite(x)
         except (ValueError, TypeError):
           raise TypeError(
               "To be compatible with tf.contrib.eager.defun, Python functions "
@@ -623,36 +622,25 @@ def flatten(sequence):
   Flattens non-tensor objects into their constituent tensors.
 
   Args:
-    sequence: A nested structure of Tensors, IndexedSlices, SparseTensors and
+    sequence: A nested structure of Tensors, CompositeTensors, and
       TensorArrays.
 
   Returns:
     A list of tensors.
   """
   # TODO(akshayka): Support `SparseTensor` in a similar fashion.
-  flat_sequence = nest.flatten(sequence)
-  outputs = []
-  for item in flat_sequence:
-    if isinstance(item, ops.IndexedSlices):
-      if item.dense_shape is not None:
-        outputs.extend([item.values, item.indices, item.dense_shape])
-      else:
-        outputs.extend([item.values, item.indices])
-    elif isinstance(item, sparse_tensor.SparseTensor):
-      outputs.extend([item.indices, item.values, item.dense_shape])
-    elif isinstance(item, tensor_array_ops.TensorArray):
-      outputs.append(item.flow)
-    else:
-      outputs.append(item)
-  return outputs
+  flat_sequence = nest.flatten(sequence, expand_composites=True)
+  return [
+      item.flow if isinstance(item, tensor_array_ops.TensorArray) else item
+      for item in flat_sequence]
 
 
 def pack_sequence_as(structure, flat_sequence):
   """Like `nest.pack_sequence_as` but also packs other Tensor-like objects.
 
   Args:
-    structure: The structure to pack into. May contain Tensors, IndexedSlices,
-      TensorArrays or SparseTensors.
+    structure: The structure to pack into. May contain Tensors,
+      CompositeTensors, or TensorArrays.
     flat_sequence: An iterable containing tensors.
 
   Returns:
@@ -661,33 +649,16 @@ def pack_sequence_as(structure, flat_sequence):
   Raises:
     AssertionError if `structure` and `flat_sequence` are not compatible.
   """
-  flattened_structure = nest.flatten(structure)
-  flat_sequence_with_slices_and_tas = []
-  index = 0
-  for t in flattened_structure:
-    if isinstance(t, ops.IndexedSlices):
-      if t.dense_shape is not None:
-        flat_sequence_with_slices_and_tas.append(
-            ops.IndexedSlices(*flat_sequence[index:index + 3]))
-        index += 3
-      else:
-        flat_sequence_with_slices_and_tas.append(
-            ops.IndexedSlices(*flat_sequence[index:index + 2]))
-        index += 2
-    elif isinstance(t, sparse_tensor.SparseTensor):
-      flat_sequence_with_slices_and_tas.append(
-          sparse_tensor.SparseTensor(*flat_sequence[index:index + 3]))
-      index += 3
-    elif isinstance(t, tensor_array_ops.TensorArray):
-      flow = flat_sequence[index]
-      ta = tensor_array_ops.build_ta_with_new_flow(t, flow)
-      flat_sequence_with_slices_and_tas.append(ta)
-      index += 1
-    else:
-      flat_sequence_with_slices_and_tas.append(flat_sequence[index])
-      index += 1
-  assert len(flattened_structure) == len(flat_sequence_with_slices_and_tas)
-  return nest.pack_sequence_as(structure, flat_sequence_with_slices_and_tas)
+  flat_sequence = list(flat_sequence)
+  flattened_structure = nest.flatten(structure, expand_composites=True)
+  if len(flattened_structure) != len(flat_sequence):
+    raise ValueError("Mismatch in element count")
+  for i in range(len(flat_sequence)):
+    if isinstance(flattened_structure[i], tensor_array_ops.TensorArray):
+      flat_sequence[i] = tensor_array_ops.build_ta_with_new_flow(
+          old_ta=flattened_structure[i], flow=flat_sequence[i])
+  return nest.pack_sequence_as(structure, flat_sequence, expand_composites=True)
+
 
 
 def _create_substitute_placeholder(value, name=None, dtype=None):
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index 6da6637b0b..e0f61622e4 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -41,6 +41,7 @@ from tensorflow.python.eager import context
 from tensorflow.python.eager import core
 from tensorflow.python.eager import tape
 from tensorflow.python.framework import c_api_util
+from tensorflow.python.framework import composite_tensor
 from tensorflow.python.framework import device as pydev
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
@@ -1032,7 +1033,7 @@ def convert_to_tensor(value, dtype=None, name=None, preferred_dtype=None):
       `preferred_dtype` is not possible, this argument has no effect.
 
   Returns:
-    An `Tensor` based on `value`.
+    A `Tensor` based on `value`.
 
   Raises:
     TypeError: If no conversion function is registered for `value` to `dtype`.
@@ -1085,7 +1086,7 @@ def convert_to_tensor_v2(value, dtype=None, dtype_hint=None, name=None):
     name: Optional name to use if a new `Tensor` is created.
 
   Returns:
-    An `Tensor` based on `value`.
+    A `Tensor` based on `value`.
 
   Raises:
     TypeError: If no conversion function is registered for `value` to `dtype`.
@@ -1228,7 +1229,7 @@ def internal_convert_n_to_tensor(values,
       value.
   """
   if not isinstance(values, collections.Sequence):
-    raise TypeError("values must be a list.")
+    raise TypeError("values must be a sequence.")
   ret = []
   if ctx is None: ctx = context.context()
   for i, value in enumerate(values):
@@ -1292,7 +1293,7 @@ def convert_to_tensor_or_indexed_slices(value, dtype=None, name=None):
     name: (Optional.) A name to use if a new `Tensor` is created.
 
   Returns:
-    An `Tensor`, `IndexedSlices`, or `SparseTensor` based on `value`.
+    A `Tensor`, `IndexedSlices`, or `SparseTensor` based on `value`.
 
   Raises:
     ValueError: If `dtype` does not match the element type of `value`.
@@ -1305,7 +1306,7 @@ def internal_convert_to_tensor_or_indexed_slices(value,
                                                  dtype=None,
                                                  name=None,
                                                  as_ref=False):
-  """Converts the given object to an `Tensor` or an `IndexedSlices`.
+  """Converts the given object to a `Tensor` or an `IndexedSlices`.
 
   If `value` is an `IndexedSlices` or `SparseTensor` it is returned
   unmodified. Otherwise, it is converted to a `Tensor` using
@@ -1320,7 +1321,7 @@ def internal_convert_to_tensor_or_indexed_slices(value,
     as_ref: True if the caller wants the results as ref tensors.
 
   Returns:
-    An `Tensor`, `IndexedSlices`, or `SparseTensor` based on `value`.
+    A `Tensor`, `IndexedSlices`, or `SparseTensor` based on `value`.
 
   Raises:
     ValueError: If `dtype` does not match the element type of `value`.
@@ -1351,7 +1352,7 @@ def internal_convert_n_to_tensor_or_indexed_slices(values,
   Args:
     values: A list of `None`, `IndexedSlices`, `SparseTensor`, or objects that
       can be consumed by `convert_to_tensor()`.
-    dtype: (Optional.) The required `DType` of the returned `Tensor`
+    dtype: (Optional.) The required `DType` of the returned `Tensor` or
       `IndexedSlices`.
     name: (Optional.) A name prefix to used when a new `Tensor` is
       created, in which case element `i` will be given the name `name
@@ -1359,7 +1360,7 @@ def internal_convert_n_to_tensor_or_indexed_slices(values,
     as_ref: True if the caller wants the results as ref tensors.
 
   Returns:
-    A list of `Tensor`, `IndexedSlices`, and/or `SparseTensor` objects.
+    A list of `Tensor`, `IndexedSlices`, `SparseTensor` and/or `None` objects.
 
   Raises:
     TypeError: If no conversion function is registered for an element in
@@ -1368,7 +1369,7 @@ def internal_convert_n_to_tensor_or_indexed_slices(values,
       value.
   """
   if not isinstance(values, collections.Sequence):
-    raise TypeError("values must be a list.")
+    raise TypeError("values must be a sequence.")
   ret = []
   for i, value in enumerate(values):
     if value is None:
@@ -1409,6 +1410,132 @@ def convert_n_to_tensor_or_indexed_slices(values, dtype=None, name=None):
       values=values, dtype=dtype, name=name, as_ref=False)
 
 
+def convert_to_tensor_or_composite(value, dtype=None, name=None):
+  """Converts the given object to a `Tensor` or `CompositeTensor`.
+
+  If `value` is a `CompositeTensor` it is returned unmodified. Otherwise, it
+  is converted to a `Tensor` using `convert_to_tensor()`.
+
+  Args:
+    value: A `CompositeTensor` or an object that can be consumed
+      by `convert_to_tensor()`.
+    dtype: (Optional.) The required `DType` of the returned `Tensor` or
+      `CompositeTensor`.
+    name: (Optional.) A name to use if a new `Tensor` is created.
+
+  Returns:
+    A `Tensor` or `CompositeTensor`, based on `value`.
+
+  Raises:
+    ValueError: If `dtype` does not match the element type of `value`.
+  """
+  return internal_convert_to_tensor_or_composite(
+      value=value, dtype=dtype, name=name, as_ref=False)
+
+
+def internal_convert_to_tensor_or_composite(value,
+                                            dtype=None,
+                                            name=None,
+                                            as_ref=False):
+  """Converts the given object to a `Tensor` or `CompositeTensor`.
+
+  If `value` is a `CompositeTensor` it is returned unmodified.  Otherwise, it
+  is converted to a `Tensor` using `convert_to_tensor()`.
+
+  Args:
+    value: A `CompositeTensor`, or an object that can be consumed
+      by `convert_to_tensor()`.
+    dtype: (Optional.) The required `DType` of the returned `Tensor` or
+      `CompositeTensor`.
+    name: (Optional.) A name to use if a new `Tensor` is created.
+    as_ref: True if the caller wants the results as ref tensors.
+
+  Returns:
+    A `Tensor` or `CompositeTensor`, based on `value`.
+
+  Raises:
+    ValueError: If `dtype` does not match the element type of `value`.
+  """
+  if isinstance(value, composite_tensor.CompositeTensor):
+    value_dtype = getattr(value, "dtype", None)
+    if dtype and not dtypes.as_dtype(dtype).is_compatible_with(value_dtype):
+      raise ValueError(
+          "Tensor conversion requested dtype %s for Tensor with dtype %s: %r" %
+          (dtypes.as_dtype(dtype).name, value.dtype.name, str(value)))
+    return value
+  else:
+    return internal_convert_to_tensor(
+        value, dtype=dtype, name=name, as_ref=as_ref)
+
+
+def internal_convert_n_to_tensor_or_composite(values,
+                                              dtype=None,
+                                              name=None,
+                                              as_ref=False):
+  """Converts `values` to a list of `Tensor` or `CompositeTensor` objects.
+
+  Any `CompositeTensor` objects in `values` are returned unmodified.
+
+  Args:
+    values: A list of `None`, `CompositeTensor`, or objects that
+      can be consumed by `convert_to_tensor()`.
+    dtype: (Optional.) The required `DType` of the returned `Tensor`s or
+      `CompositeTensor`s.
+    name: (Optional.) A name prefix to used when a new `Tensor` is
+      created, in which case element `i` will be given the name `name
+      + '_' + i`.
+    as_ref: True if the caller wants the results as ref tensors.
+
+  Returns:
+    A list of `Tensor`, `CompositeTensor`, and/or `None` objects.
+
+  Raises:
+    TypeError: If no conversion function is registered for an element in
+      `values`.
+    RuntimeError: If a registered conversion function returns an invalid
+      value.
+  """
+  if not isinstance(values, collections.Sequence):
+    raise TypeError("values must be a sequence.")
+  ret = []
+  for i, value in enumerate(values):
+    if value is None:
+      ret.append(value)
+    else:
+      n = None if name is None else "%s_%d" % (name, i)
+      ret.append(
+          internal_convert_to_tensor_or_composite(
+              value, dtype=dtype, name=n, as_ref=as_ref))
+  return ret
+
+
+def convert_n_to_tensor_or_composite(values, dtype=None, name=None):
+  """Converts `values` to a list of `Output` or `CompositeTensor` objects.
+
+  Any `CompositeTensor` objects in `values` are returned unmodified.
+
+  Args:
+    values: A list of `None`, `CompositeTensor``, or objects that
+      can be consumed by `convert_to_tensor()`.
+    dtype: (Optional.) The required `DType` of the returned `Tensor`s or
+      `CompositeTensor`s.
+    name: (Optional.) A name prefix to used when a new `Tensor` is
+      created, in which case element `i` will be given the name `name
+      + '_' + i`.
+
+  Returns:
+    A list of `Tensor` and/or `CompositeTensor` objects.
+
+  Raises:
+    TypeError: If no conversion function is registered for an element in
+      `values`.
+    RuntimeError: If a registered conversion function returns an invalid
+      value.
+  """
+  return internal_convert_n_to_tensor_or_composite(
+      values=values, dtype=dtype, name=name, as_ref=False)
+
+
 # TODO(josh11b): Add ctx argument to conversion_func() signature.
 @tf_export("register_tensor_conversion_function")
 def register_tensor_conversion_function(base_type,
@@ -1488,7 +1615,7 @@ def register_tensor_conversion_function(base_type,
 
 
 @tf_export("IndexedSlices")
-class IndexedSlices(_TensorLike):
+class IndexedSlices(_TensorLike, composite_tensor.CompositeTensor):
   """A sparse representation of a set of tensor slices at given indices.
 
   This class is a simple wrapper for a pair of `Tensor` objects:
@@ -1571,6 +1698,29 @@ class IndexedSlices(_TensorLike):
   def __neg__(self):
     return IndexedSlices(-self.values, self.indices, self.dense_shape)
 
+  def _to_components(self):
+    if self._dense_shape is None:
+      return (self._values, self._indices)
+    else:
+      return (self._values, self._indices, self._dense_shape)
+
+  @classmethod
+  def _from_components(cls, components):
+    return cls(*components)
+
+  def _shape_invariant_to_components(self, shape=None):
+    if shape is None:
+      shape = self._values.shape
+    if self._dense_shape is None:
+      return [shape, shape[:1]]  # values, indices
+    else:
+      # values, indices, dense_shape
+      return [shape, shape[:1], tensor_shape.TensorShape([shape.ndims])]
+
+  @property
+  def _is_graph_tensor(self):
+    return hasattr(self._values, 'graph')
+
 
 IndexedSlicesValue = collections.namedtuple(
     "IndexedSlicesValue", ["values", "indices", "dense_shape"])
diff --git a/tensorflow/python/framework/sparse_tensor.py b/tensorflow/python/framework/sparse_tensor.py
index 8546c2299a..c7f505fff3 100644
--- a/tensorflow/python/framework/sparse_tensor.py
+++ b/tensorflow/python/framework/sparse_tensor.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Classes and functions used to construct graphs."""
+"""Sparse tensors."""
 # pylint: disable=g-bad-name
 from __future__ import absolute_import
 from __future__ import division
@@ -21,8 +21,10 @@ from __future__ import print_function
 import collections
 
 from tensorflow.python import pywrap_tensorflow
+from tensorflow.python.framework import composite_tensor
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.util.tf_export import tf_export
 
@@ -34,7 +36,7 @@ _override_helper = ops._override_helper
 
 
 @tf_export("sparse.SparseTensor", "SparseTensor")
-class SparseTensor(_TensorLike):
+class SparseTensor(_TensorLike, composite_tensor.CompositeTensor):
   """Represents a sparse tensor.
 
   TensorFlow represents a sparse tensor as three separate dense tensors:
@@ -237,6 +239,29 @@ class SparseTensor(_TensorLike):
   def _override_operator(operator, func):
     _override_helper(SparseTensor, operator, func)
 
+  def _to_components(self):
+    return (self._indices, self._values, self._dense_shape)
+
+  @classmethod
+  def _from_components(cls, components):
+    return cls(*components)
+
+  def _shape_invariant_to_components(self, shape=None):
+    if shape is None:
+      shape = self.dense_shape.shape
+      # TODO(edloper): What if shape.dense_shape.shape.ndims is None?
+    if shape.ndims != 1:
+      raise ValueError("Shape invariant for SparseTensor must have the form "
+                       "TensorShape([r]), got %r" % shape)
+    rank = tensor_shape.dimension_value(shape[0])
+    return [tensor_shape.TensorShape([None, rank]),  # indices
+            tensor_shape.TensorShape([None]),  # values
+            tensor_shape.TensorShape([rank])]  # dense_shape
+
+  @property
+  def _is_graph_tensor(self):
+    return hasattr(self._values, 'graph')
+
 
 SparseTensorValue = collections.namedtuple(
     "SparseTensorValue", ["indices", "values", "dense_shape"])
diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
index 104e9e82ee..27b3e26c77 100644
--- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
@@ -63,6 +63,7 @@ from tensorflow.python.ops import nn_ops
 from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import script_ops
+from tensorflow.python.ops import sparse_ops
 from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import tensor_array_grad  # pylint: disable=unused-import
 from tensorflow.python.ops import tensor_array_ops
@@ -70,6 +71,7 @@ from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
 from tensorflow.python.ops import while_v2  # pylint: disable=unused-import
 # pylint: disable=unused-import
+from tensorflow.python.ops.ragged import ragged_factory_ops
 import tensorflow.python.ops.tensor_array_grad
 # pylint: enable=unused-import
 from tensorflow.python.platform import test
@@ -427,23 +429,31 @@ class ControlFlowTest(test.TestCase):
     self.assertAllEqual(11, val)
     self.assertAllEqual(0, ind)
 
-  @test_util.run_v1_only("b/120545219")
   def testCondSparseTensor(self):
-    with self.cached_session():
-      values = constant_op.constant([2.0, 4.0], name="values")
-      indices = constant_op.constant(
-          [[0], [3]], dtype=dtypes.int64, name="indices")
-      shape = constant_op.constant([10], dtype=dtypes.int64, name="dense_shape")
-      x = sparse_tensor.SparseTensor(indices, values, dense_shape=shape)
-      pred = math_ops.less(1, 2)
-      fn1 = lambda: sparse_tensor.SparseTensor(
-          indices + 1, x.values + 1, dense_shape=shape)
-      fn2 = lambda: sparse_tensor.SparseTensor(
-          indices, x.values - 1, dense_shape=shape)
-      r = control_flow_ops.cond(pred, fn1, fn2)
-      self.assertAllEqual([3.0, 5.0], r.values)
-      self.assertAllEqual([[1], [4]], r.indices)
-      self.assertAllEqual(r.values.get_shape(), (2,))
+    values = constant_op.constant([2.0, 4.0], name="values")
+    indices = constant_op.constant([[0], [3]],
+                                   dtype=dtypes.int64,
+                                   name="indices")
+    shape = constant_op.constant([10], dtype=dtypes.int64, name="dense_shape")
+    x = sparse_tensor.SparseTensor(indices, values, dense_shape=shape)
+    pred = math_ops.less(1, 2)
+    fn1 = lambda: sparse_tensor.SparseTensor(
+        indices + 1, x.values + 1, dense_shape=shape)
+    fn2 = lambda: sparse_tensor.SparseTensor(
+        indices, x.values - 1, dense_shape=shape)
+    r = control_flow_ops.cond(pred, fn1, fn2)
+    self.assertAllEqual([3.0, 5.0], r.values)
+    self.assertAllEqual([[1], [4]], r.indices)
+    self.assertAllEqual(r.values.get_shape(), (2,))
+
+  def testCondRaggedTensor(self):
+    rt = ragged_factory_ops.constant([[1, 2], [3], [4, 5, 6]])
+    pred = math_ops.less(1, 2)
+    fn1 = lambda: array_ops.concat([rt + 2, [[100]]], axis=0)
+    fn2 = lambda: rt[:2] - 2
+    result = control_flow_ops.cond(pred, fn1, fn2)
+    self.assertAllEqual([3, 4, 5, 6, 7, 8, 100], result.values)
+    self.assertAllEqual([0, 2, 3, 6, 7], result.row_splits)
 
   @test_util.run_v1_only("b/120545219")
   def testCondResource(self):
@@ -1519,35 +1529,80 @@ class ControlFlowTest(test.TestCase):
   @test_util.disable_control_flow_v2("b/116328420 (SparseTensor)")
   @test_util.run_v1_only("b/120545219")
   def testWhileShapeInferenceSparseTensor(self):
-    with self.cached_session():
-      values = constant_op.constant([2.0, 4.0], name="values")
-      indices = constant_op.constant(
-          [[0], [3]], dtype=dtypes.int64, name="indices")
-      shape = constant_op.constant([10], dtype=dtypes.int64, name="dense_shape")
-      i = constant_op.constant(0)
-      x = sparse_tensor.SparseTensor(indices, values, dense_shape=shape)
-
-      def c(i, _):
-        return i < 10
+    values = constant_op.constant([2.0, 4.0], name="values")
+    indices = constant_op.constant([[0], [3]],
+                                   dtype=dtypes.int64,
+                                   name="indices")
+    shape = constant_op.constant([10], dtype=dtypes.int64, name="dense_shape")
+    i = constant_op.constant(0)
+    x = sparse_tensor.SparseTensor(indices, values, dense_shape=shape)
+
+    def c(i, _):
+      return i < 10
+
+    def b1(i, x):  # modifies values.  (shape of components is not changed.)
+      return [
+          i + 1,
+          sparse_tensor.SparseTensor(x.indices, x.values * 2.0, x.dense_shape)
+      ]
 
-      def b(i, x):
-        return [
-            i + 1,
-            sparse_tensor.SparseTensor(x.indices, x.values * 2.0, x.dense_shape)
-        ]
+    def b2(i, x):  # adds new values.  (shape of components is changed.)
+      return [
+          i + 1,
+          sparse_ops.sparse_add(
+              x,
+              sparse_tensor.SparseTensor(
+                  indices=math_ops.cast(
+                      array_ops.fill([1, 1], i), dtypes.int64),
+                  values=array_ops.fill([1], 1.0),
+                  dense_shape=x.dense_shape))
+      ]
 
-      _, r = control_flow_ops.while_loop(c, b, [i, x])
-      self.assertEqual(r.dense_shape.get_shape()[0], 1)
+    def b3(i, x):  # modifies rank.  (shape of all components is changed.)
+      return [
+          i + 1,
+          sparse_tensor.SparseTensor(
+              array_ops.concat([x.indices, [[i], [i]]], axis=1), x.values * 2.0,
+              array_ops.concat([x.dense_shape, [10]], axis=0))
+      ]
 
+    # Default shape invariant; b1 only modifies values.
+    _, r = control_flow_ops.while_loop(c, b1, [i, x])
+    self.assertEqual(r.indices.get_shape().as_list(), [None, 1])
+    self.assertEqual(r.values.get_shape().as_list(), [None])
+    self.assertEqual(r.dense_shape.get_shape().as_list(), [1])
+
+    # Default shape invariant; b2 adds new values
+    _, r = control_flow_ops.while_loop(c, b2, [i, x])
+    self.assertEqual(r.indices.get_shape().as_list(), [None, 1])
+    self.assertEqual(r.values.get_shape().as_list(), [None])
+    self.assertEqual(r.dense_shape.get_shape().as_list(), [1])
+
+    # Default shape invariant; b3 modifies rank (which is not allowed).
+    with self.assertRaises(ValueError):
+      _, r = control_flow_ops.while_loop(c, b3, [i, x])
+
+    # Explicit shape invariant, allowing any rank; b1 only modifies values.
+    _, r = control_flow_ops.while_loop(
+        c, b1, [i, x],
+        [i.get_shape(), tensor_shape.TensorShape([None])])
+    self.assertEqual(r.indices.get_shape().as_list(), [None, None])
+    self.assertEqual(r.values.get_shape().as_list(), [None])
+    self.assertEqual(r.dense_shape.get_shape().as_list(), [None])
+
+    # Explicit shape invariant, allowing any rank; b3 modifies rank.
+    _, r = control_flow_ops.while_loop(
+        c, b3, [i, x],
+        [i.get_shape(), tensor_shape.TensorShape([None])])
+    self.assertEqual(r.indices.get_shape().as_list(), [None, None])
+    self.assertEqual(r.values.get_shape().as_list(), [None])
+    self.assertEqual(r.dense_shape.get_shape().as_list(), [None])
+
+    # Explicit shape invariant, with a specific (incompatible) rank.
+    with self.assertRaisesRegexp(ValueError, "is not compatible with"):
       _, r = control_flow_ops.while_loop(
-          c, b, [i, x],
-          [i.get_shape(), tensor_shape.TensorShape([None])])
-      self.assertEqual(r.dense_shape.get_shape().as_list(), [None])
-
-      with self.assertRaisesRegexp(ValueError, "is not compatible with"):
-        _, r = control_flow_ops.while_loop(
-            c, b, [i, x],
-            [i.get_shape(), tensor_shape.TensorShape([5])])
+          c, b1, [i, x],
+          [i.get_shape(), tensor_shape.TensorShape([5])])
 
   @test_util.disable_control_flow_v2("b/116282023 (IndexedSlices)")
   @test_util.run_v1_only("b/120545219")
@@ -1583,6 +1638,65 @@ class ControlFlowTest(test.TestCase):
             c, b, [i, x],
             [i.get_shape(), tensor_shape.TensorShape([None, 5])])
 
+  @test_util.disable_control_flow_v2("b/116328420 (RaggedTensor)")
+  def testWhileShapeInferenceRaggedTensor(self):
+    i = constant_op.constant(0)
+    x = ragged_factory_ops.constant([[1, 2], [3], [4, 5, 6]])
+    c = lambda i, _: i < 10
+
+    def b1(i, x):  # Adds new values to rows (but doesn't create new rows)
+      return [
+          i + 1,
+          array_ops.concat([x, x], axis=1)
+      ]
+
+    def b2(i, x):  # Adds new rows.
+      return [
+          i + 1,
+          array_ops.concat([x, x], axis=0)
+      ]
+
+    # Default shape invariant; b1 adds new values to rows.
+    _, r = control_flow_ops.while_loop(c, b1, [i, x])
+    self.assertEqual(r.row_splits.shape.as_list(), [4])
+
+    self.assertTrue(r.values.shape.as_list() in ([6 * 2**10], [None]))
+
+    # Default shape invariant; b2 adds new rows (not allowed).
+    if not context.executing_eagerly():
+      with self.assertRaises(ValueError):
+        _, r = control_flow_ops.while_loop(c, b2, [i, x])
+
+    # Explicit shape invariant; b1 adds new values to rows.
+    _, r = control_flow_ops.while_loop(
+        c, b1, [i, x],
+        [i.get_shape(), tensor_shape.TensorShape([None, None])])
+    self.assertTrue(r.row_splits.shape.as_list() in ([4], [None]))
+    self.assertTrue(r.values.shape.as_list() in ([6 * 2**10], [None]))
+
+    # Explicit shape invariant; b2 adds new rows.
+    _, r = control_flow_ops.while_loop(
+        c, b2, [i, x],
+        [i.get_shape(), tensor_shape.TensorShape([None, None])])
+    self.assertTrue(r.row_splits.shape.as_list() in ([3 * 2**10 + 1], [None]))
+    self.assertTrue(r.values.shape.as_list() in ([6 * 2**10], [None]))
+
+  @test_util.disable_control_flow_v2("b/116328420 (RaggedTensor)")
+  def testWhileShapeInferenceRaggedTensorRaggedRank2(self):
+    i = constant_op.constant(0)
+    x = ragged_factory_ops.constant([[[1, 2], [3], [4, 5, 6]],
+                                     [[], [8, 9, 10]]])
+    c = lambda i, _: i < 10
+    def b(i, x):
+      return [
+          i + 1,
+          array_ops.concat([x, x[..., i:i+1]], axis=-1)
+      ]
+    _, r = control_flow_ops.while_loop(c, b, [i, x])
+    self.assertEqual(r.row_splits.shape.as_list(), [3])
+    self.assertTrue(r.values.row_splits.shape.as_list() in ([6], [None]))
+    self.assertTrue(r.values.values.shape.as_list() in ([49], [None]))
+
   def _testNestedWhile_1(self, use_gpu):
     with self.cached_session(use_gpu=use_gpu):
       n = constant_op.constant(0)
diff --git a/tensorflow/python/ops/control_flow_ops.py b/tensorflow/python/ops/control_flow_ops.py
index 3b3bfbbfc2..2bc3a9e9b6 100644
--- a/tensorflow/python/ops/control_flow_ops.py
+++ b/tensorflow/python/ops/control_flow_ops.py
@@ -30,11 +30,11 @@ import six
 from tensorflow.core.framework import attr_value_pb2
 from tensorflow.core.protobuf import control_flow_pb2
 from tensorflow.python.eager import context
+from tensorflow.python.framework import composite_tensor
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
-from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import array_ops
@@ -181,47 +181,25 @@ def _Identity(data, name=None):
   Returns:
     A Tensor with the same type and value as the input Tensor.
   """
-  data = ops.internal_convert_to_tensor_or_indexed_slices(data, as_ref=True)
+  data = ops.internal_convert_to_tensor_or_composite(data, as_ref=True)
   if isinstance(data, ops.Tensor):
     if data.dtype._is_ref_dtype:  # pylint: disable=protected-access
       return gen_array_ops.ref_identity(data, name=name)
     else:
       return array_ops.identity(data, name=name)
   else:
-    if not isinstance(data, (ops.IndexedSlices, sparse_tensor.SparseTensor)):
-      raise TypeError("Type %s not supported" % type(data))
-    values = _Identity(data.values, name=name)
-    indices = array_ops.identity(data.indices, name="indices")
-    if isinstance(data, ops.IndexedSlices):
-      dense_shape = data.dense_shape
-      if dense_shape is not None:
-        dense_shape = array_ops.identity(dense_shape, name="dense_shape")
-      return ops.IndexedSlices(values, indices, dense_shape)
-    else:
-      dense_shape = array_ops.identity(data.dense_shape, name="dense_shape")
-      return sparse_tensor.SparseTensor(indices, values, dense_shape)
+    raise TypeError("Type %s not supported" % type(data))
 
 
 def _NextIteration(data, name=None):
-  data = ops.internal_convert_to_tensor_or_indexed_slices(data, as_ref=True)
+  data = ops.internal_convert_to_tensor_or_composite(data, as_ref=True)
   if isinstance(data, ops.Tensor):
     if data.dtype._is_ref_dtype:  # pylint: disable=protected-access
       return ref_next_iteration(data, name=name)
     else:
       return next_iteration(data, name=name)
   else:
-    if not isinstance(data, (ops.IndexedSlices, sparse_tensor.SparseTensor)):
-      raise TypeError("Type %s not supported" % type(data))
-    values = _NextIteration(data.values, name=name)
-    indices = next_iteration(data.indices, name="indices")
-    if isinstance(data, ops.IndexedSlices):
-      dense_shape = data.dense_shape
-      if dense_shape is not None:
-        dense_shape = next_iteration(dense_shape, name="dense_shape")
-      return ops.IndexedSlices(values, indices, dense_shape)
-    else:
-      dense_shape = next_iteration(data.dense_shape, name="dense_shape")
-      return sparse_tensor.SparseTensor(indices, values, dense_shape)
+    raise TypeError("Type %s not supported" % type(data))
 
 
 def _Enter(data,
@@ -244,12 +222,13 @@ def _Enter(data,
     is_constant: If true, the output is constant within the child frame.
     parallel_iterations: The number of iterations allowed to run in parallel.
     use_ref: If true, use ref_enter if data is of ref type.
+    use_input_shape: If true, set the result's shape based on data's shape.
     name: A name for this operation (optional).
 
   Returns:
     The same tensor as `data`.
   """
-  data = ops.internal_convert_to_tensor_or_indexed_slices(data, as_ref=True)
+  data = ops.internal_convert_to_tensor_or_composite(data, as_ref=True)
   if isinstance(data, ops.Tensor):
     if data.dtype._is_ref_dtype and use_ref:  # pylint: disable=protected-access
       result = gen_control_flow_ops.ref_enter(
@@ -261,45 +240,7 @@ def _Enter(data,
       result.set_shape(data.get_shape())
     return result
   else:
-    if not isinstance(data, (ops.IndexedSlices, sparse_tensor.SparseTensor)):
-      raise TypeError("Type %s not supported" % type(data))
-    values = _Enter(
-        data.values,
-        frame_name,
-        is_constant,
-        parallel_iterations=parallel_iterations,
-        use_input_shape=use_input_shape,
-        name=name)
-    indices = gen_control_flow_ops.enter(
-        data.indices,
-        frame_name,
-        is_constant,
-        parallel_iterations,
-        name="indices")
-    if use_input_shape:
-      indices.set_shape(data.indices.get_shape())
-    if isinstance(data, ops.IndexedSlices):
-      dense_shape = data.dense_shape
-      if dense_shape is not None:
-        dense_shape = gen_control_flow_ops.enter(
-            dense_shape,
-            frame_name,
-            is_constant,
-            parallel_iterations,
-            name="dense_shape")
-        if use_input_shape:
-          dense_shape.set_shape(data.dense_shape.get_shape())
-      return ops.IndexedSlices(values, indices, dense_shape)
-    else:
-      dense_shape = gen_control_flow_ops.enter(
-          data.dense_shape,
-          frame_name,
-          is_constant,
-          parallel_iterations,
-          name="dense_shape")
-      if use_input_shape:
-        dense_shape.set_shape(data.dense_shape.get_shape())
-      return sparse_tensor.SparseTensor(indices, values, dense_shape)
+    raise TypeError("Type %s not supported" % type(data))
 
 
 def exit(data, name=None):  # pylint: disable=redefined-builtin
@@ -314,25 +255,14 @@ def exit(data, name=None):  # pylint: disable=redefined-builtin
   Returns:
     The same tensor as `data`.
   """
-  data = ops.internal_convert_to_tensor_or_indexed_slices(data, as_ref=True)
+  data = ops.internal_convert_to_tensor_or_composite(data, as_ref=True)
   if isinstance(data, ops.Tensor):
     if data.dtype._is_ref_dtype:  # pylint: disable=protected-access
       return gen_control_flow_ops.ref_exit(data, name)
     else:
       return gen_control_flow_ops._exit(data, name)
   else:
-    if not isinstance(data, (ops.IndexedSlices, sparse_tensor.SparseTensor)):
-      raise TypeError("Type %s not supported" % type(data))
-    values = exit(data.values, name=name)
-    indices = gen_control_flow_ops._exit(data.indices, name="indices")
-    if isinstance(data, ops.IndexedSlices):
-      dense_shape = data.dense_shape
-      if dense_shape is not None:
-        dense_shape = gen_control_flow_ops._exit(dense_shape, name)
-      return ops.IndexedSlices(values, indices, dense_shape)
-    else:
-      dense_shape = gen_control_flow_ops._exit(data.dense_shape, name)
-      return sparse_tensor.SparseTensor(indices, values, dense_shape)
+    raise TypeError("Type %s not supported" % type(data))
 
 
 def switch(data, pred, dtype=None, name=None):
@@ -355,32 +285,19 @@ def switch(data, pred, dtype=None, name=None):
     to `output_true`, otherwise it goes to `output_false`.
   """
   with ops.name_scope(name, "Switch", [data, pred]) as name:
-    data = ops.internal_convert_to_tensor_or_indexed_slices(
+    data = ops.internal_convert_to_tensor_or_composite(
         data, dtype=dtype, name="data", as_ref=True)
     pred = ops.convert_to_tensor(pred, name="pred")
     if isinstance(data, ops.Tensor):
       return gen_control_flow_ops.switch(data, pred, name=name)
     else:
-      if not isinstance(data, (ops.IndexedSlices, sparse_tensor.SparseTensor)):
+      if not isinstance(data, composite_tensor.CompositeTensor):
         raise TypeError("Type %s not supported" % type(data))
-      val, ind = data.values, data.indices
-      val_f, val_t = gen_control_flow_ops.switch(val, pred, name=name)
-      ind_f, ind_t = gen_control_flow_ops.switch(ind, pred, name="indices")
-      if isinstance(data, ops.IndexedSlices):
-        dense_shape = data.dense_shape
-        if dense_shape is not None:
-          dense_shape_f, dense_shape_t = gen_control_flow_ops.switch(
-              dense_shape, pred, name="dense_shape")
-        else:
-          dense_shape_f, dense_shape_t = None, None
-        return (ops.IndexedSlices(val_f, ind_f, dense_shape_f),
-                ops.IndexedSlices(val_t, ind_t, dense_shape_t))
-      else:
-        dense_shape = data.dense_shape
-        dense_shape_f, dense_shape_t = gen_control_flow_ops.switch(
-            data.dense_shape, pred, name="dense_shape")
-        return (sparse_tensor.SparseTensor(ind_f, val_f, dense_shape_f),
-                sparse_tensor.SparseTensor(ind_t, val_t, dense_shape_t))
+      tensors = nest.flatten(data, expand_composites=True)
+      mapped = [gen_control_flow_ops.switch(tensor, pred) for tensor in tensors]
+      mapped_f, mapped_t = zip(*mapped)
+      return (nest.pack_sequence_as(data, mapped_f, expand_composites=True),
+              nest.pack_sequence_as(data, mapped_t, expand_composites=True))
 
 
 def _SwitchRefOrTensor(data, pred, name="Switch"):
@@ -403,7 +320,7 @@ def _SwitchRefOrTensor(data, pred, name="Switch"):
   Raises:
     TypeError: if data is not a Tensor or IndexedSlices
   """
-  data = ops.convert_to_tensor_or_indexed_slices(data, name="data")
+  data = ops.convert_to_tensor_or_composite(data, name="data")
   # NOTE(vrv): ops.colocate_with(data, ignore_existing=True) below
   # addresses the following scenario.
   #
@@ -456,7 +373,7 @@ def merge(inputs, name=None):
     raise ValueError("At least one of the merge inputs is None: %s" % inputs)
   with ops.name_scope(name, "Merge", inputs) as name:
     inputs = [
-        ops.internal_convert_to_tensor_or_indexed_slices(inp, as_ref=True)
+        ops.internal_convert_to_tensor_or_composite(inp, as_ref=True)
         for inp in inputs
     ]
     if all(isinstance(v, ops.Tensor) for v in inputs):
@@ -464,30 +381,27 @@ def merge(inputs, name=None):
         return gen_control_flow_ops.ref_merge(inputs, name)
       else:
         return gen_control_flow_ops.merge(inputs, name)
-    elif all(isinstance(v, sparse_tensor.SparseTensor) for v in inputs):
-      # Only handle the case when all inputs are SparseTensor.
-      values, _ = merge([inp.values for inp in inputs], name=name)
-      indices, chosen_index = gen_control_flow_ops.merge(
-          [inp.indices for inp in inputs], name="indices")
-      dense_shape, _ = gen_control_flow_ops.merge(
-          [inp.dense_shape for inp in inputs], name="dense_shape")
-      return (sparse_tensor.SparseTensor(indices, values, dense_shape),
-              chosen_index)
     else:
-      # For now convert all the inputs as IndexedSlices.
-      inputs = math_ops._as_indexed_slices_list(inputs, optimize=False)
-      values, _ = merge([inp.values for inp in inputs], name=name)
-      indices, chosen_index = gen_control_flow_ops.merge(
-          [inp.indices for inp in inputs], name="indices")
-      if any(inp.dense_shape is not None for inp in inputs):
-        if any(inp.dense_shape is None for inp in inputs):
-          raise ValueError("Either all merged IndexedSlices must have a "
-                           "dense_shape, or none must have a dense_shape.")
-        dense_shape, _ = gen_control_flow_ops.merge(
-            [inp.dense_shape for inp in inputs], name="dense_shape")
-      else:
-        dense_shape = None
-      return ops.IndexedSlices(values, indices, dense_shape), chosen_index
+      # If there is a mix of tensors and indexed slices, then convert the
+      # tensors to indexed slices.
+      if all(isinstance(v, (ops.IndexedSlices, ops.Tensor)) for v in inputs):
+        inputs = math_ops._as_indexed_slices_list(inputs, optimize=False)
+
+      for v in inputs:
+        if not isinstance(v, composite_tensor.CompositeTensor):
+          raise TypeError("Type %s not supported" % type(v))
+
+      for v in inputs[1:]:
+        nest.assert_same_structure(inputs[0], v, expand_composites=True)
+
+      flat_inputs = [nest.flatten(v, expand_composites=True) for v in inputs]
+      merged_results = [gen_control_flow_ops.merge(component)
+                        for component in zip(*flat_inputs)]
+      flat_merged = [tensor for (tensor, _) in merged_results]
+      chosen_index = merged_results[0][1]
+      merged_inputs = nest.pack_sequence_as(inputs[0], flat_merged,
+                                            expand_composites=True)
+      return (merged_inputs, chosen_index)
 
 
 # pylint: enable=protected-access
@@ -537,6 +451,30 @@ def _ShapeLessThanOrEqual(shape1, shape2):
   return True
 
 
+def _get_shape_invariant(var, shape=None):
+  """Returns a shape invariant for the given variable.
+
+  If `var` is a `CompositeTensor`, then this uses
+  `_shape_invariant_to_components()` to get shape invariants for the
+  component tensors.
+
+  Args:
+    var: The tensor whose shape is described.
+    shape: The shape invariant for the tensor.  If not specified, then a default
+      shape invariant for `var` is returned.
+
+  Returns:
+    The shape invariant for `var` (if it is a `Tensor`), or the shape invariants
+    for the components that comprise `var` (if it is a `CompositeTensor`).
+  """
+  if isinstance(var, composite_tensor.CompositeTensor):
+    return var._shape_invariant_to_components(shape)  # pylint: disable=protected-access
+  elif shape is None:
+    return var.shape
+  else:
+    return shape
+
+
 def _SetShapeInvariants(input_vars, enter_vars, shapes):
   """Set the shapes of the tensors in `enter_vars` to `shapes`.
 
@@ -566,31 +504,7 @@ def _SetShapeInvariants(input_vars, enter_vars, shapes):
             (inp.name, inp.get_shape(), shape))
       var.set_shape(shape)
     else:
-      if not isinstance(var, (ops.IndexedSlices, sparse_tensor.SparseTensor)):
-        raise TypeError("Type %s not supported" % type(var))
-      if isinstance(var, ops.IndexedSlices):
-        if not _ShapeLessThanOrEqual(inp.values.get_shape(), shape):
-          raise ValueError(
-              "The shape invariant specified for %s is not compatible with "
-              "the initial shape of the values tensor of this IndexedSlices. "
-              "It enters the loop with shape %s, but the specified shape "
-              "invariant is %s." % (inp.values.name, inp.values.get_shape(),
-                                    shape))
-        var.values.set_shape(shape)
-        var.indices.set_shape(tensor_shape.TensorShape([shape[0]]))
-        if var.dense_shape is not None:
-          var.dense_shape.set_shape(tensor_shape.TensorShape([shape.ndims]))
-      else:
-        if not _ShapeLessThanOrEqual(inp.dense_shape.get_shape(), shape):
-          raise ValueError(
-              "The shape invariant specified for %s is not compatible with "
-              "the initial shape of the shape tensor of this SparseTensor. "
-              "It enters the loop with shape %s, but the specified shape "
-              "invariant is %s." % (inp.dense_shape.name,
-                                    inp.dense_shape.get_shape(), shape))
-        var.values.set_shape(tensor_shape.TensorShape([None]))
-        var.indices.set_shape(tensor_shape.TensorShape([None, shape.ndims]))
-        var.dense_shape.set_shape(shape)
+      raise TypeError("Type %s not supported" % type(var))
 
 
 def _EnforceShapeInvariant(merge_var, next_var):
@@ -619,49 +533,7 @@ def _EnforceShapeInvariant(merge_var, next_var):
           "use the `shape_invariants` argument of tf.while_loop to specify a "
           "less-specific shape." % (input_t.name, input_t.shape, n_shape))
   else:
-    if not isinstance(merge_var,
-                      (ops.IndexedSlices, sparse_tensor.SparseTensor)):
-      raise TypeError("Type %s not supported" % type(merge_var))
-    if isinstance(merge_var, ops.IndexedSlices):
-      m_values_shape = merge_var.values.get_shape()
-      m_indices_shape = merge_var.indices.get_shape()
-      m_shape_shape = tensor_shape.TensorShape(None)
-      if merge_var.dense_shape is not None:
-        m_shape_shape = merge_var.dense_shape.get_shape()
-      n_values_shape = next_var.values.get_shape()
-      n_indices_shape = next_var.indices.get_shape()
-      n_shape_shape = tensor_shape.TensorShape(None)
-      if next_var.dense_shape is not None:
-        n_shape_shape = next_var.dense_shape.get_shape()
-      if (not _ShapeLessThanOrEqual(n_values_shape, m_values_shape) or
-          not _ShapeLessThanOrEqual(n_indices_shape, m_indices_shape)):
-        if not _ShapeLessThanOrEqual(n_values_shape, m_values_shape):
-          raise ValueError(
-              "The shape for %s is not an invariant for the loop. It enters "
-              "the loop with shape (%s, %s, %s), but has shape (%s, %s, %s) "
-              "after one iteration. Provide shape invariants using either the "
-              "`shape_invariants` argument of tf.while_loop or set_shape() "
-              "on the loop variables." %
-              (merge_var.name, m_values_shape, m_indices_shape, m_shape_shape,
-               n_values_shape, n_indices_shape, n_shape_shape))
-    else:
-      m_values_shape = merge_var.values.get_shape()
-      m_indices_shape = merge_var.indices.get_shape()
-      m_shape_shape = merge_var.dense_shape.get_shape()
-      n_values_shape = next_var.values.get_shape()
-      n_indices_shape = next_var.indices.get_shape()
-      n_shape_shape = next_var.dense_shape.get_shape()
-      if (not _ShapeLessThanOrEqual(n_values_shape, m_values_shape) or
-          not _ShapeLessThanOrEqual(n_indices_shape, m_indices_shape) or
-          not _ShapeLessThanOrEqual(n_shape_shape, m_shape_shape)):
-        raise ValueError(
-            "The shape for %s is not an invariant for the loop. It enters "
-            "the loop with shape (%s, %s, %s), but has shape (%s, %s, %s) "
-            "after one iteration. Provide shape invariants using either "
-            "the `shape_invariants` argument of tf.while_loop or set_shape() "
-            "on the loop variables." %
-            (merge_var.name, m_values_shape, m_indices_shape, m_shape_shape,
-             n_values_shape, n_indices_shape, n_shape_shape))
+    raise TypeError("Type %s not supported" % type(merge_var))
 
 
 def _AddNextAndBackEdge(m, v, enforce_shape_invariant=True):
@@ -676,26 +548,6 @@ def _AddNextAndBackEdge(m, v, enforce_shape_invariant=True):
       # TODO(skyewm): call this for other cases below (needs testing)
       _EnforceShapeInvariant(m, v)
     m.op._update_input(1, v)  # pylint: disable=protected-access
-  elif isinstance(m, ops.IndexedSlices):
-    # pylint: disable=protected-access
-    v = math_ops._as_indexed_slices(v, optimize=False)
-    v = _NextIteration(v)
-    m.values.op._update_input(1, v.values)
-    m.indices.op._update_input(1, v.indices)
-    # pylint: enable=protected-access
-    if m.dense_shape is not None:
-      if v.dense_shape is None:
-        raise ValueError("Must have dense shape: %s" % v.name)
-      m.dense_shape.op._update_input(1, v.dense_shape)
-  elif isinstance(m, sparse_tensor.SparseTensor):
-    if not isinstance(v, sparse_tensor.SparseTensor):
-      raise ValueError("Must be a sparse tensor: %s" % v.name)
-    v = _NextIteration(v)
-    # pylint: disable=protected-access
-    m.values.op._update_input(1, v.values)
-    m.indices.op._update_input(1, v.indices)
-    m.dense_shape.op._update_input(1, v.dense_shape)
-    # pylint: enable=protected-access
   else:
     raise TypeError("Type %s not supported" % type(m))
   return v
@@ -1613,7 +1465,8 @@ class ControlFlowContext(object):
   def ExitResult(self, result):
     """Make a list of tensors available in the outer context."""
     if self._outer_context:
-      nest.map_structure(lambda x: self._outer_context.AddName(x.name), result)
+      nest.map_structure(lambda x: self._outer_context.AddName(x.name), result,
+                         expand_composites=True)
 
   def GetWhileContext(self):
     """Return the while context containing this context."""
@@ -1920,19 +1773,9 @@ class CondContext(ControlFlowContext):
     if isinstance(v, ops.Operation):
       # Use pivot as the proxy for this op.
       return with_dependencies([v], self._pivot)
-    elif isinstance(v, (ops.IndexedSlices, sparse_tensor.SparseTensor)):
-      values = self._ProcessOutputTensor(v.values)
-      indices = self._ProcessOutputTensor(v.indices)
-      if isinstance(v, ops.IndexedSlices):
-        dense_shape = v.dense_shape
-        if dense_shape is not None:
-          dense_shape = self._ProcessOutputTensor(dense_shape)
-        return ops.IndexedSlices(values, indices, dense_shape)
-      else:
-        dense_shape = self._ProcessOutputTensor(v.dense_shape)
-        return sparse_tensor.SparseTensor(indices, values, dense_shape)
     else:
-      v = nest.map_structure(_convert_tensorarray_to_flow, v)
+      v = nest.map_structure(_convert_tensorarray_to_flow, v,
+                             expand_composites=True)
       return self._ProcessOutputTensor(ops.convert_to_tensor(v))
 
   def BuildCondBranch(self, fn):
@@ -1949,11 +1792,13 @@ class CondContext(ControlFlowContext):
           return no_op(), None
         else:
           original_result = nest.map_structure(array_ops.identity,
-                                               original_result)
+                                               original_result,
+                                               expand_composites=True)
     if original_result is None:
       return None, None
 
-    result = nest.map_structure(self._BuildCondTensor, original_result)
+    result = nest.map_structure(self._BuildCondTensor, original_result,
+                                expand_composites=True)
     if not isinstance(result, (list, _basetuple)):
       result = [result]
     return original_result, result
@@ -2120,7 +1965,8 @@ def cond(pred,
 
     # Check that the return values of the two branches have the same structure.
     try:
-      nest.assert_same_structure(orig_res_t, orig_res_f)
+      nest.assert_same_structure(orig_res_t, orig_res_f,
+                                 expand_composites=True)
     except TypeError as e:
       raise TypeError(
           "Incompatible return types of true_fn and false_fn: {}".format(e))
@@ -2132,24 +1978,21 @@ def cond(pred,
     if not res_t:
       raise ValueError("true_fn and false_fn must return at least one result.")
 
-    res_t_flat = nest.flatten(res_t)
-    res_f_flat = nest.flatten(res_f)
-
-    for x, y in zip(res_t_flat, res_f_flat):
-      assert ((isinstance(x, ops.IndexedSlices) and
-               isinstance(y, ops.IndexedSlices)) or
-              (isinstance(x, sparse_tensor.SparseTensor) and
-               isinstance(y, sparse_tensor.SparseTensor)) or
-              (isinstance(x, ops.Tensor) and isinstance(y, ops.Tensor)))
-      val_x = x if isinstance(x, ops.Tensor) else x.values
-      val_y = y if isinstance(y, ops.Tensor) else y.values
-      if val_x.dtype.base_dtype != val_y.dtype.base_dtype:
-        raise ValueError(
-            "Outputs of true_fn and false_fn must have the same type: %s, %s" %
-            (val_x.dtype.name, val_y.dtype.name))
+    res_t_flat = nest.flatten(res_t, expand_composites=True)
+    res_f_flat = nest.flatten(res_f, expand_composites=True)
+
+    for i, (x, y) in enumerate(zip(res_t_flat, res_f_flat)):
+      assert isinstance(x, ops.Tensor) and isinstance(y, ops.Tensor)
+      if x.dtype.base_dtype != y.dtype.base_dtype:
+        _cast_indexed_slice_indices(res_t, res_t_flat, res_f_flat)
+        if res_t_flat[i].dtype.base_dtype != res_f_flat[i].dtype.base_dtype:
+          raise ValueError(
+              "Outputs of true_fn and false_fn must have the same type: "
+              "%s, %s" % (x.dtype.name, y.dtype.name))
 
     merges = [merge(pair)[0] for pair in zip(res_f_flat, res_t_flat)]
-    merges = _convert_flows_to_tensorarrays(nest.flatten(orig_res_t), merges)
+    merges = _convert_flows_to_tensorarrays(
+        nest.flatten(orig_res_t, expand_composites=True), merges)
 
     # Only add non-nested conds to the collection. Any nested control flow will
     # be encapsulated in the root context.
@@ -2158,7 +2001,8 @@ def cond(pred,
       ops.add_to_collection(ops.GraphKeys.COND_CONTEXT, context_t)
       ops.add_to_collection(ops.GraphKeys.COND_CONTEXT, context_f)
 
-    merges = nest.pack_sequence_as(structure=orig_res_t, flat_sequence=merges)
+    merges = nest.pack_sequence_as(structure=orig_res_t, flat_sequence=merges,
+                                   expand_composites=True)
 
     # Singleton lists and tuples are automatically unpacked if strict == False.
     if not strict:
@@ -2166,6 +2010,48 @@ def cond(pred,
     return merges
 
 
+def _cast_indexed_slice_indices(structure, flat_a, flat_b):
+  """Cast IndexedSlice.indices from int32 to int64 where necessary.
+
+  For each `IndexedSlices` in the nested structure `structure`, find its
+  indices `Tensor` in the corresponding flattened lists `flat_a` and `flat_b`
+  (where composites have been expanded); and if those indices tensors have
+  different dtypes (i.e., if one is int64 but the other is int32), then cast
+  them to both be int64.
+
+  Args:
+    structure: The nested structure that was flattened.
+    flat_a: A flattened list of `Tensors` whose structure matches
+        `structure`.  Will be modified in place to cast `IndexedSlices`
+        indices tensors to int64, where necessary.
+    flat_a: A flattened list of `Tensors` whose structure matches
+        `structure`.  Will be modified in place to cast `IndexedSlices`
+        indices tensors to int64, where necessary.
+  """
+  # Find the locations (in flat_a and flat_b) of the IndexedSlices'
+  # indices tensors.
+  indexed_slice_indices = []
+  current_index = 0
+  for item in nest.flatten(structure, expand_composites=False):
+    if isinstance(item, ops.IndexedSlices):
+      # indices is the second component of the composite tensor.
+      indexed_slice_indices.append(current_index + 1)
+    if nest.is_sequence_or_composite(item):
+      current_index += len(nest.flatten(item, expand_composites=True))
+    else:
+      current_index += 1
+  assert current_index == len(flat_a)
+
+  for index in indexed_slice_indices:
+    assert flat_a[index].dtype in (dtypes.int32, dtypes.int64)
+    assert flat_b[index].dtype in (dtypes.int32, dtypes.int64)
+    if flat_a[index].dtype != flat_b[index].dtype:
+      if flat_b[index].dtype == dtypes.int32:
+        flat_b[index] = math_ops.cast(flat_b[index], dtypes.int64)
+      else:
+        flat_a[index] = math_ops.cast(flat_a[index], dtypes.int64)
+
+
 # pylint: enable=g-doc-args
 # pylint: enable=redefined-outer-name
 
@@ -2939,21 +2825,12 @@ class WhileContext(ControlFlowContext):
       if isinstance(x, ops.Tensor):
         self._values.add(x.name)
       else:
-        self._values.add(x.values.name)
-        self._values.add(x.indices.name)
-        if isinstance(x, ops.IndexedSlices):
-          dense_shape = x.dense_shape
-        elif isinstance(x, sparse_tensor.SparseTensor):
-          dense_shape = x.dense_shape
-        else:
-          raise TypeError("Type %s not supported" % type(x))
-        if dense_shape is not None:
-          self._values.add(dense_shape.name)
+        raise TypeError("Type %s not supported" % type(x))
 
   def _BuildLoop(self, pred, body, original_loop_vars, loop_vars,
                  shape_invariants):
     """Core: Add the loop termination condition and body to the graph."""
-    flat_loop_vars = nest.flatten(original_loop_vars)
+    flat_loop_vars = nest.flatten(original_loop_vars, expand_composites=True)
 
     # Let the context know the loop variables so the loop variables
     # would be added in the outer contexts properly.
@@ -3005,7 +2882,8 @@ class WhileContext(ControlFlowContext):
         _convert_flows_to_tensorarrays(flat_loop_vars, merge_vars))
     packed_vars = nest.pack_sequence_as(
         structure=original_loop_vars,
-        flat_sequence=merge_vars_with_tensor_arrays)
+        flat_sequence=merge_vars_with_tensor_arrays,
+        expand_composites=True)
     c = ops.convert_to_tensor(pred(*packed_vars))
     self._pivot = loop_cond(c, name="LoopCond")
     switch_vars = [_SwitchRefOrTensor(x, self._pivot) for x in merge_vars]
@@ -3019,11 +2897,12 @@ class WhileContext(ControlFlowContext):
         _convert_flows_to_tensorarrays(flat_loop_vars, vars_for_body))
     packed_vars_for_body = nest.pack_sequence_as(
         structure=original_loop_vars,
-        flat_sequence=vars_for_body_with_tensor_arrays)
+        flat_sequence=vars_for_body_with_tensor_arrays,
+        expand_composites=True)
     pre_summaries = ops.get_collection(ops.GraphKeys._SUMMARY_COLLECTION)  # pylint: disable=protected-access
     body_result = body(*packed_vars_for_body)
     post_summaries = ops.get_collection(ops.GraphKeys._SUMMARY_COLLECTION)  # pylint: disable=protected-access
-    if not nest.is_sequence(body_result):
+    if not nest.is_sequence_or_composite(body_result):
       body_result = [body_result]
     if len(post_summaries) > len(pre_summaries):
       new_summaries = post_summaries[len(pre_summaries):]
@@ -3037,20 +2916,24 @@ class WhileContext(ControlFlowContext):
             return x
           return array_ops.identity(x)
 
-        body_result = nest.map_structure(map_fn, body_result)
+        body_result = nest.map_structure(map_fn, body_result,
+                                         expand_composites=True)
 
     # Compare the structure types of input and output of body.
     # For backwards compatibility, the first layer is forced to a list
     # during this comparison, because inputs are typically lists and
     # outputs of the body are typically tuples.
-    nest.assert_same_structure(list(packed_vars_for_body), list(body_result))
+    nest.assert_same_structure(list(packed_vars_for_body), list(body_result),
+                               expand_composites=True)
 
     # Store body_result to keep track of TensorArrays returned by body
     original_body_result = body_result
     # Convert TensorArrays returned by body into their flow variables
-    result = nest.map_structure(_convert_tensorarray_to_flow,
-                                nest.flatten(body_result))
-    result = ops.convert_n_to_tensor_or_indexed_slices(result)
+    result = nest.map_structure(
+        _convert_tensorarray_to_flow,
+        nest.flatten(body_result, expand_composites=True),
+        expand_composites=True)
+    result = ops.convert_n_to_tensor_or_composite(result)
 
     # Add NextIteration and the back edges to complete the loop.
     if len(merge_vars) != len(result):
@@ -3076,9 +2959,15 @@ class WhileContext(ControlFlowContext):
     # Keep original_loop_vars to identify which are TensorArrays
     original_loop_vars = loop_vars
     # Convert TensorArrays to their flow variables
-    loop_vars = nest.map_structure(_convert_tensorarray_to_flow,
-                                   nest.flatten(loop_vars))
-    loop_vars = ops.convert_n_to_tensor_or_indexed_slices(loop_vars)
+    loop_vars = nest.map_structure(
+        _convert_tensorarray_to_flow,
+        nest.flatten(loop_vars, expand_composites=False),
+        expand_composites=True)
+    loop_vars = ops.convert_n_to_tensor_or_composite(loop_vars)
+    if shape_invariants is None:
+      shape_invariants = nest.map_structure(
+          _get_shape_invariant, loop_vars, expand_composites=False)
+    loop_vars = nest.flatten(loop_vars, expand_composites=True)
     try:
       self.Enter()
       # _BuildLoop calls _update_input in several places. _mutation_lock()
@@ -3090,14 +2979,15 @@ class WhileContext(ControlFlowContext):
     finally:
       self.Exit()
 
-    flat_result = nest.flatten(original_body_result)
+    flat_result = nest.flatten(original_body_result, expand_composites=True)
     # Convert TensorArray flow variables outside the context back into
     # their associated TensorArrays for returning to caller.
     exit_vars_with_tensor_arrays = (
         _convert_flows_to_tensorarrays(flat_result, exit_vars))
     packed_exit_vars = nest.pack_sequence_as(
         structure=original_body_result,
-        flat_sequence=exit_vars_with_tensor_arrays)
+        flat_sequence=exit_vars_with_tensor_arrays,
+        expand_composites=True)
 
     if return_same_structure:
       return packed_exit_vars
@@ -3111,12 +3001,7 @@ class WhileContext(ControlFlowContext):
       if isinstance(e, ops.Tensor):
         xs = [e]
       else:
-        if not isinstance(e, (ops.IndexedSlices, sparse_tensor.SparseTensor)):
-          raise TypeError("Type %s not supported" % type(e))
-        xs = [e.values, e.indices]
-        shape = e.dense_shape
-        if shape is not None:
-          xs.append(shape)
+        raise TypeError("Type %s not supported" % type(e))
       for x in xs:
         inp_op = x.op.inputs[0].op
         control_inputs = graph._control_dependencies_for_inputs([inp_op])
@@ -3546,7 +3431,12 @@ def while_loop(cond,
     if shape_invariants is not None:
       if maximum_iterations is not None:
         shape_invariants = (tensor_shape.TensorShape([]), shape_invariants)
-      nest.assert_same_structure(loop_vars, shape_invariants)
+
+      nest.assert_same_structure(loop_vars, shape_invariants,
+                                 expand_composites=False)
+      shape_invariants = nest.map_structure(
+          _get_shape_invariant, loop_vars, shape_invariants,
+          expand_composites=False)
 
     loop_context = WhileContext(
         maximum_iterations=maximum_iterations,
@@ -3588,7 +3478,7 @@ def _AsTensorList(x, p):
   for v in x:
     if isinstance(v, ops.Operation):
       v = with_dependencies([v], p)
-    v = ops.convert_to_tensor_or_indexed_slices(v)
+    v = ops.convert_to_tensor_or_composite(v)
     if isinstance(v, ops.Tensor):
       l.append(array_ops.identity(v))
     else:
@@ -3636,7 +3526,7 @@ def with_dependencies(dependencies, output_tensor, name=None):
                       list(dependencies) + [output_tensor]) as name:
     with ops.colocate_with(output_tensor):
       with ops.control_dependencies(dependencies):
-        output_tensor = ops.convert_to_tensor_or_indexed_slices(output_tensor)
+        output_tensor = ops.convert_to_tensor_or_composite(output_tensor)
         if isinstance(output_tensor, ops.Tensor):
           return _Identity(output_tensor, name=name)
         else:
@@ -3687,7 +3577,7 @@ def group(*inputs, **kwargs):
 
     # Sorts *inputs according to their devices.
     ops_on_device = {}  # device -> operations specified on the device.
-    for inp in nest.flatten(inputs):
+    for inp in nest.flatten(inputs, expand_composites=True):
       if not hasattr(inp, "device"):
         raise TypeError("Expected tf.group() expected Tensor arguments not "
                         "'%s' with type '%s'" % (inp, type(inp)))
diff --git a/tensorflow/python/ops/ragged/ragged_tensor.py b/tensorflow/python/ops/ragged/ragged_tensor.py
index fd334e6cc7..8fb3c1f44c 100644
--- a/tensorflow/python/ops/ragged/ragged_tensor.py
+++ b/tensorflow/python/ops/ragged/ragged_tensor.py
@@ -19,6 +19,7 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.client import session
+from tensorflow.python.framework import composite_tensor
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -45,7 +46,7 @@ _eval_using_default_session = ops._eval_using_default_session
 
 
 @tf_export("RaggedTensor")
-class RaggedTensor(object):
+class RaggedTensor(composite_tensor.CompositeTensor):
   """Represents a ragged tensor.
 
   A `RaggedTensor` is a tensor with one or more *ragged dimensions*, which are
@@ -1437,6 +1438,53 @@ class RaggedTensor(object):
       values = values.values
     return values
 
+  #=============================================================================
+  # Composite Tensor
+  #=============================================================================
+
+  def _to_components(self):
+    return (self.flat_values,) + self.nested_row_splits
+
+  @classmethod
+  def _from_components(cls, components):
+    return cls.from_nested_row_splits(components[0], components[1:])
+
+  def _shape_invariant_to_components(self, shape=None):
+    ragged_rank = self.ragged_rank
+    flat_values = self.flat_values
+
+    if shape is None:
+      # Default shape invariant
+      value_shape = flat_values.shape[1:]
+      values_shape = tensor_shape.TensorShape([None]).concatenate(value_shape)
+      return ((values_shape, self._row_splits.shape) +
+              tuple(tensor_shape.TensorShape([None])
+                    for i in range(1, ragged_rank)))
+    else:
+      # Explicitly specified shape invariant
+      if shape.ndims is not None and shape.ndims <= ragged_rank:
+        raise ValueError("Shape invariant %s does not have sufficient rank "
+                         "for a RaggedTensor with %d ragged dimensions." %
+                         (shape, self.ragged_rank))
+      if any(tensor_shape.dimension_value(shape[dim]) is not None
+             for dim in range(1, self.ragged_rank + 1)):
+        raise ValueError("Shape invariant dimension size must be None for "
+                         "ragged dimenions.")
+      nrows = tensor_shape.dimension_value(shape[0])
+      value_shape = shape[self.ragged_rank + 1:]
+      values_shape = tensor_shape.TensorShape([None]).concatenate(value_shape)
+      if nrows is None:
+        outer_splits_shape = tensor_shape.TensorShape([None])
+      else:
+        outer_splits_shape = tensor_shape.TensorShape([nrows + 1])
+      return ((values_shape, outer_splits_shape) +
+              tuple(tensor_shape.TensorShape([None])
+                    for i in range(1, ragged_rank)))
+
+  @property
+  def _is_graph_tensor(self):
+    return hasattr(self._values, 'graph')
+
 
 def is_ragged(value):
   """Returns true if `value` is a ragged tensor or ragged tensor value."""
diff --git a/tensorflow/python/util/nest.py b/tensorflow/python/util/nest.py
index dbae97a373..83fd8bdf60 100644
--- a/tensorflow/python/util/nest.py
+++ b/tensorflow/python/util/nest.py
@@ -74,6 +74,7 @@ def _is_namedtuple(instance, strict=False):
 # See the swig file (util.i) for documentation.
 _is_mapping = _pywrap_tensorflow.IsMapping
 _is_attrs = _pywrap_tensorflow.IsAttrs
+_is_composite_tensor = _pywrap_tensorflow.IsCompositeTensor
 
 
 def _sequence_like(instance, args):
@@ -97,6 +98,8 @@ def _sequence_like(instance, args):
     return type(instance)((key, result[key]) for key in _six.iterkeys(instance))
   elif _is_namedtuple(instance) or _is_attrs(instance):
     return type(instance)(*args)
+  elif _is_composite_tensor(instance):
+    return instance._from_components(args)  # pylint: disable=protected-access
   else:
     # Not a namedtuple
     return type(instance)(args)
@@ -115,6 +118,9 @@ def _yield_value(iterable):
   elif _is_attrs(iterable):
     for value in _get_attrs_values(iterable):
       yield value
+  elif _is_composite_tensor(iterable):
+    for value in _yield_value(iterable._to_components()):  # pylint: disable=protected-access
+      yield value
   else:
     for value in iterable:
       yield value
@@ -124,6 +130,10 @@ def _yield_value(iterable):
 is_sequence = _pywrap_tensorflow.IsSequence
 
 
+# See the swig file (util.i) for documentation.
+is_sequence_or_composite = _pywrap_tensorflow.IsSequenceOrComposite
+
+
 # See the swig file (util.i) for documentation.
 flatten = _pywrap_tensorflow.Flatten
 
@@ -144,7 +154,8 @@ class _DotString(object):
 _DOT = _DotString()
 
 
-def assert_same_structure(nest1, nest2, check_types=True):
+def assert_same_structure(nest1, nest2, check_types=True,
+                          expand_composites=False):
   """Asserts that two structures are nested in the same way.
 
   Note that namedtuples with identical name and fields are always considered
@@ -168,6 +179,8 @@ def assert_same_structure(nest1, nest2, check_types=True):
         considered the same if they are both list subtypes (which allows "list"
         and "_ListWrapper" from checkpointable dependency tracking to compare
         equal).
+    expand_composites: If true, then composite tensors such as `tf.SparseTensor`
+        and `tf.RaggedTensor` are expanded into their component tensors.
 
   Raises:
     ValueError: If the two structures do not have the same number of elements or
@@ -176,7 +189,8 @@ def assert_same_structure(nest1, nest2, check_types=True):
       their substructures. Only possible if `check_types` is `True`.
   """
   try:
-    _pywrap_tensorflow.AssertSameStructure(nest1, nest2, check_types)
+    _pywrap_tensorflow.AssertSameStructure(nest1, nest2, check_types,
+                                           expand_composites)
   except (ValueError, TypeError) as e:
     str1 = str(map_structure(lambda _: _DOT, nest1))
     str2 = str(map_structure(lambda _: _DOT, nest2))
@@ -242,13 +256,14 @@ def flatten_dict_items(dictionary):
   return flat_dictionary
 
 
-def _packed_nest_with_indices(structure, flat, index):
+def _packed_nest_with_indices(structure, flat, index, is_seq):
   """Helper function for pack_sequence_as.
 
   Args:
     structure: Substructure (list / tuple / dict) to mimic.
     flat: Flattened values to output substructure for.
     index: Index at which to start reading from flat.
+    is_seq: Function used to test if a value should be treated as a sequence.
 
   Returns:
     The tuple (new_index, child), where:
@@ -263,8 +278,8 @@ def _packed_nest_with_indices(structure, flat, index):
   """
   packed = []
   for s in _yield_value(structure):
-    if is_sequence(s):
-      new_index, child = _packed_nest_with_indices(s, flat, index)
+    if is_seq(s):
+      new_index, child = _packed_nest_with_indices(s, flat, index, is_seq)
       packed.append(_sequence_like(s, child))
       index = new_index
     else:
@@ -273,7 +288,7 @@ def _packed_nest_with_indices(structure, flat, index):
   return index, packed
 
 
-def pack_sequence_as(structure, flat_sequence):
+def pack_sequence_as(structure, flat_sequence, expand_composites=False):
   """Returns a given flattened sequence packed into a given structure.
 
   If `structure` is a scalar, `flat_sequence` must be a single-element list;
@@ -293,6 +308,8 @@ def pack_sequence_as(structure, flat_sequence):
         tuples, and dicts. Note: numpy arrays and strings are considered
         scalars.
     flat_sequence: flat sequence to pack.
+    expand_composites: If true, then composite tensors such as `tf.SparseTensor`
+        and `tf.RaggedTensor` are expanded into their component tensors.
 
   Returns:
     packed: `flat_sequence` converted to have the same recursive structure as
@@ -303,17 +320,19 @@ def pack_sequence_as(structure, flat_sequence):
       element counts.
     TypeError: `structure` is or contains a dict with non-sortable keys.
   """
-  if not is_sequence(flat_sequence):
+  is_seq = is_sequence_or_composite if expand_composites else is_sequence
+  if not is_seq(flat_sequence):
     raise TypeError("flat_sequence must be a sequence")
 
-  if not is_sequence(structure):
+  if not is_seq(structure):
     if len(flat_sequence) != 1:
       raise ValueError("Structure is a scalar but len(flat_sequence) == %d > 1"
                        % len(flat_sequence))
     return flat_sequence[0]
 
   try:
-    final_index, packed = _packed_nest_with_indices(structure, flat_sequence, 0)
+    final_index, packed = _packed_nest_with_indices(structure, flat_sequence,
+                                                    0, is_seq)
     if final_index < len(flat_sequence):
       raise IndexError
   except IndexError:
@@ -326,7 +345,7 @@ def pack_sequence_as(structure, flat_sequence):
   return _sequence_like(structure, packed)
 
 
-def map_structure(func, *structure, **check_types_dict):
+def map_structure(func, *structure, **kwargs):
   """Applies `func` to each entry in `structure` and returns a new structure.
 
   Applies `func(x[0], x[1], ...)` where x[i] is an entry in
@@ -337,12 +356,18 @@ def map_structure(func, *structure, **check_types_dict):
     func: A callable that accepts as many arguments as there are structures.
     *structure: scalar, or tuple or list of constructed scalars and/or other
       tuples/lists, or scalars.  Note: numpy arrays are considered as scalars.
-    **check_types_dict: only valid keyword argument is `check_types`. If set to
-      `True` (default) the types of iterables within the structures have to be
-      same (e.g. `map_structure(func, [1], (1,))` raises a `TypeError`
-      exception). To allow this set this argument to `False`.
-      Note that namedtuples with identical name and fields are always
-      considered to have the same shallow structure.
+    **kwargs: Valid keyword args are:
+
+      * `check_types`: If set to `True` (default) the types of
+        iterables within the structures have to be same (e.g.
+        `map_structure(func, [1], (1,))` raises a `TypeError`
+        exception). To allow this set this argument to `False`.
+        Note that namedtuples with identical name and fields are always
+        considered to have the same shallow structure.
+      * `expand_composites`: If set to `True`, then composite tensors such
+        as `tf.SparseTensor` and `tf.RaggedTensor` are expanded into their
+        component tensors.  If `False` (the default), then composite tensors
+        are not expanded.
 
   Returns:
     A new structure with the same arity as `structure`, whose values correspond
@@ -364,21 +389,25 @@ def map_structure(func, *structure, **check_types_dict):
   if not structure:
     raise ValueError("Must provide at least one structure")
 
-  if check_types_dict:
-    if "check_types" not in check_types_dict or len(check_types_dict) > 1:
-      raise ValueError("Only valid keyword argument is check_types")
-    check_types = check_types_dict["check_types"]
-  else:
-    check_types = True
+  check_types = True
+  expand_composites = False
+  if kwargs:
+    check_types = kwargs.pop("check_types", check_types)
+    expand_composites = kwargs.pop("expand_composites", expand_composites)
+    if kwargs:
+      raise ValueError("Only valid keyword arguments are check_types "
+                       "and expand_composites")
 
   for other in structure[1:]:
-    assert_same_structure(structure[0], other, check_types=check_types)
+    assert_same_structure(structure[0], other, check_types=check_types,
+                          expand_composites=expand_composites)
 
-  flat_structure = [flatten(s) for s in structure]
+  flat_structure = [flatten(s, expand_composites) for s in structure]
   entries = zip(*flat_structure)
 
   return pack_sequence_as(
-      structure[0], [func(*x) for x in entries])
+      structure[0], [func(*x) for x in entries],
+      expand_composites=expand_composites)
 
 
 def map_structure_with_paths(func, *structure, **kwargs):
diff --git a/tensorflow/python/util/util.cc b/tensorflow/python/util/util.cc
index e69eec73a0..bda0cba82f 100644
--- a/tensorflow/python/util/util.cc
+++ b/tensorflow/python/util/util.cc
@@ -455,6 +455,14 @@ class SparseTensorValueIterator : public ValueIterator {
   Safe_PyObjectPtr tensor_;
 };
 
+// Returns nullptr (to raise an exception) when next() is called.  Caller
+// should have already called PyErr_SetString.
+class ErrorValueIterator : public ValueIterator {
+ public:
+  ErrorValueIterator() {}
+  Safe_PyObjectPtr next() override { return nullptr; }
+};
+
 class AttrsValueIterator : public ValueIterator {
  public:
   explicit AttrsValueIterator(PyObject* nested) : nested_(nested) {
@@ -497,6 +505,35 @@ bool IsSparseTensorValueType(PyObject* o) {
              o, reinterpret_cast<PyTypeObject*>(sparse_tensor_value_type)) == 1;
 }
 
+// Returns 1 if `o` is an instance of CompositeTensor.
+// Returns 0 otherwise.
+// Returns -1 if an error occurred.
+bool IsCompositeTensorHelper(PyObject* o) {
+  static auto* const check_cache = new CachedTypeCheck([](PyObject* to_check) {
+    PyObject* composite_tensor_type = GetRegisteredType("CompositeTensor");
+    if (TF_PREDICT_FALSE(composite_tensor_type == nullptr)) {
+      PyErr_SetString(PyExc_RuntimeError,
+                      tensorflow::strings::StrCat(
+                          "CompositeTensor type has not been set. "
+                          "Please register the type with the identifier "
+                          "\"CompositeTensor\" using RegisterType.")
+                          .c_str());
+      return -1;
+    }
+    int is_instance = PyObject_IsInstance(to_check, composite_tensor_type);
+
+    // Don't cache a failed is_instance check.
+    if (is_instance == -1) return -1;
+
+    return static_cast<int>(is_instance != 0);
+  });
+  return check_cache->CachedLookup(o);
+}
+
+int IsSequenceOrCompositeHelper(PyObject* o) {
+  return IsSequence(o) || IsCompositeTensor(o);
+}
+
 int IsSequenceForDataHelper(PyObject* o) {
   return IsSequenceHelper(o) == 1 && !PyList_Check(o) &&
          !IsSparseTensorValueType(o);
@@ -529,6 +566,18 @@ ValueIteratorPtr GetValueIteratorForData(PyObject* nested) {
   }
 }
 
+// Similar to GetValueIterator above, but expands CompositeTensors.
+ValueIteratorPtr GetValueIteratorForComposite(PyObject* nested) {
+  if (IsCompositeTensor(nested)) {
+    static char expand_method_name[] = "_to_components";
+    nested = PyObject_CallMethod(nested, expand_method_name, nullptr);
+    if (PyErr_Occurred() || nested == nullptr) {
+      return absl::make_unique<ErrorValueIterator>();
+    }
+  }
+  return GetValueIterator(nested);
+}
+
 bool FlattenHelper(
     PyObject* nested, PyObject* list,
     const std::function<int(PyObject*)>& is_sequence_helper,
@@ -596,7 +645,8 @@ void SetDifferentKeysError(PyObject* dict1, PyObject* dict2, string* error_msg,
 bool AssertSameStructureHelper(
     PyObject* o1, PyObject* o2, bool check_types, string* error_msg,
     bool* is_type_error,
-    const std::function<int(PyObject*)>& is_sequence_helper) {
+    const std::function<int(PyObject*)>& is_sequence_helper,
+    const std::function<ValueIteratorPtr(PyObject*)>& value_iterator_getter) {
   DCHECK(error_msg);
   DCHECK(is_type_error);
   const bool is_seq1 = is_sequence_helper(o1);
@@ -702,8 +752,8 @@ bool AssertSameStructureHelper(
     }
   }
 
-  ValueIteratorPtr iter1 = GetValueIterator(o1);
-  ValueIteratorPtr iter2 = GetValueIterator(o2);
+  ValueIteratorPtr iter1 = value_iterator_getter(o1);
+  ValueIteratorPtr iter2 = value_iterator_getter(o2);
 
   if (!iter1->valid() || !iter2->valid()) return false;
 
@@ -714,9 +764,9 @@ bool AssertSameStructureHelper(
       if (Py_EnterRecursiveCall(" in assert_same_structure")) {
         return false;
       }
-      bool no_internal_errors =
-          AssertSameStructureHelper(v1.get(), v2.get(), check_types, error_msg,
-                                    is_type_error, is_sequence_helper);
+      bool no_internal_errors = AssertSameStructureHelper(
+          v1.get(), v2.get(), check_types, error_msg, is_type_error,
+          is_sequence_helper, value_iterator_getter);
       Py_LeaveRecursiveCall();
       if (!no_internal_errors) return false;
       if (!error_msg->empty()) return true;
@@ -742,9 +792,13 @@ bool IsAttrs(PyObject* o) { return IsAttrsHelper(o) == 1; }
 bool IsTensor(PyObject* o) { return IsTensorHelper(o) == 1; }
 bool IsIndexedSlices(PyObject* o) { return IsIndexedSlicesHelper(o) == 1; }
 
-PyObject* Flatten(PyObject* nested) {
+PyObject* Flatten(PyObject* nested, bool expand_composites) {
   PyObject* list = PyList_New(0);
-  if (FlattenHelper(nested, list, IsSequenceHelper, GetValueIterator)) {
+  const std::function<int(PyObject*)>& is_sequence_helper =
+      expand_composites ? IsSequenceOrCompositeHelper : IsSequenceHelper;
+  const std::function<ValueIteratorPtr(PyObject*)>& get_value_iterator =
+      expand_composites ? GetValueIteratorForComposite : GetValueIterator;
+  if (FlattenHelper(nested, list, is_sequence_helper, get_value_iterator)) {
     return list;
   } else {
     Py_DECREF(list);
@@ -752,6 +806,12 @@ PyObject* Flatten(PyObject* nested) {
   }
 }
 
+bool IsSequenceOrComposite(PyObject* o) {
+  return IsSequenceOrCompositeHelper(o) == 1;
+}
+
+bool IsCompositeTensor(PyObject* o) { return IsCompositeTensorHelper(o) == 1; }
+
 bool IsSequenceForData(PyObject* o) { return IsSequenceForDataHelper(o) == 1; }
 
 PyObject* FlattenForData(PyObject* nested) {
@@ -850,11 +910,16 @@ PyObject* SameNamedtuples(PyObject* o1, PyObject* o2) {
   }
 }
 
-PyObject* AssertSameStructure(PyObject* o1, PyObject* o2, bool check_types) {
+PyObject* AssertSameStructure(PyObject* o1, PyObject* o2, bool check_types,
+                              bool expand_composites) {
+  const std::function<int(PyObject*)>& is_sequence_helper =
+      expand_composites ? IsSequenceOrCompositeHelper : IsSequenceHelper;
+  const std::function<ValueIteratorPtr(PyObject*)>& get_value_iterator =
+      expand_composites ? GetValueIteratorForComposite : GetValueIterator;
   string error_msg;
   bool is_type_error = false;
   AssertSameStructureHelper(o1, o2, check_types, &error_msg, &is_type_error,
-                            IsSequenceHelper);
+                            is_sequence_helper, get_value_iterator);
   if (PyErr_Occurred()) {
     // Don't hide Python exceptions while checking (e.g. errors fetching keys
     // from custom mappings).
@@ -878,7 +943,7 @@ PyObject* AssertSameStructureForData(PyObject* o1, PyObject* o2,
   string error_msg;
   bool is_type_error = false;
   AssertSameStructureHelper(o1, o2, check_types, &error_msg, &is_type_error,
-                            IsSequenceForDataHelper);
+                            IsSequenceForDataHelper, GetValueIterator);
   if (PyErr_Occurred()) {
     // Don't hide Python exceptions while checking (e.g. errors fetching keys
     // from custom mappings).
diff --git a/tensorflow/python/util/util.h b/tensorflow/python/util/util.h
index f37cd527d8..4a5db93401 100644
--- a/tensorflow/python/util/util.h
+++ b/tensorflow/python/util/util.h
@@ -33,6 +33,30 @@ namespace swig {
 //   dict.
 bool IsSequence(PyObject* o);
 
+// Implements the same interface as nest.is_sequence_or_composite
+// Returns a true if its input is a collections.Sequence (except strings)
+// or a CompositeTensor.
+//
+// Args:
+//   seq: an input sequence.
+//
+// Returns:
+//   True if the sequence is a not a string and is a collections.Sequence or a
+//   dict or a CompositeTensor.
+bool IsSequenceOrComposite(PyObject* o);
+
+// Implements the same interface as nest.is_sequence_or_composite
+// Returns a true if its input is a collections.Sequence (except strings)
+// or a CompositeTensor.
+//
+// Args:
+//   seq: an input sequence.
+//
+// Returns:
+//   True if the sequence is a not a string and is a collections.Sequence or a
+//   dict or a CompositeTensor.
+bool IsCompositeTensor(PyObject* o);
+
 // Implements the same interface as tensorflow.util.nest._is_namedtuple
 // Returns Py_True iff `instance` should be considered a `namedtuple`.
 //
@@ -118,7 +142,8 @@ PyObject* SameNamedtuples(PyObject* o1, PyObject* o2);
 //
 // Returns:
 //  Py_None on success, nullptr on error.
-PyObject* AssertSameStructure(PyObject* o1, PyObject* o2, bool check_types);
+PyObject* AssertSameStructure(PyObject* o1, PyObject* o2, bool check_types,
+                              bool expand_composites);
 
 // Implements the same interface as tensorflow.util.nest.flatten
 //
@@ -139,6 +164,9 @@ PyObject* AssertSameStructure(PyObject* o1, PyObject* o2, bool check_types);
 // Args:
 //   nest: an arbitrarily nested structure or a scalar object. Note, numpy
 //       arrays are considered scalars.
+//   expand_composites: If true, then composite tensors (such as
+//       `tf.SparseTensor` and `tf.RaggedTensor` are flattened into their
+//       component tensors.
 //
 // Returns:
 //   A Python list, the flattened version of the input.
@@ -146,7 +174,7 @@ PyObject* AssertSameStructure(PyObject* o1, PyObject* o2, bool check_types);
 //
 // Raises:
 //   TypeError: The nest is or contains a dict with non-sortable keys.
-PyObject* Flatten(PyObject* nested);
+PyObject* Flatten(PyObject* nested, bool expand_composites = false);
 
 // The tensorflow.python.data package has its own nest utility that follows very
 // slightly different semantics for its functions than the tensorflow.python
diff --git a/tensorflow/python/util/util.i b/tensorflow/python/util/util.i
index 4d34d61eee..6e2a3d8ccf 100644
--- a/tensorflow/python/util/util.i
+++ b/tensorflow/python/util/util.i
@@ -35,7 +35,7 @@ limitations under the License.
 %noexception tensorflow::swig::IsTensor;
 
 %feature("docstring") tensorflow::swig::IsSequence
-"""Returns a true if its input is a collections.Sequence (except strings).
+"""Returns true if its input is a collections.Sequence (except strings).
 
 Args:
   seq: an input sequence.
@@ -47,6 +47,31 @@ Returns:
 %unignore tensorflow::swig::IsSequence;
 %noexception tensorflow::swig::IsSequence;
 
+%feature("docstring") tensorflow::swig::IsSequenceOrComposite
+"""Returns true if its input is a sequence or a `CompositeTensor`.
+
+Args:
+  seq: an input sequence.
+
+Returns:
+  True if the sequence is a not a string and is a collections.Sequence or a
+  dict or a CompositeTensor.
+"""
+%unignore tensorflow::swig::IsSequenceOrComposite;
+%noexception tensorflow::swig::IsSequenceOrComposite;
+
+%feature("docstring") tensorflow::swig::IsCompositeTensor
+"""Returns true if its input is a `CompositeTensor`.
+
+Args:
+  seq: an input sequence.
+
+Returns:
+  True if the sequence is a CompositeTensor.
+"""
+%unignore tensorflow::swig::IsCompositeTensor;
+%noexception tensorflow::swig::IsCompositeTensor;
+
 %unignore tensorflow::swig::IsNamedtuple;
 %noexception tensorflow::swig::IsNamedtuple;
 
@@ -103,6 +128,8 @@ running.
 Args:
   nest: an arbitrarily nested structure or a scalar object. Note, numpy
       arrays are considered scalars.
+  expand_composites: If true, then composite tensors such as `tf.SparseTensor`
+      and `tf.RaggedTensor` are expanded into their component tensors.
 
 Returns:
   A Python list, the flattened version of the input.
@@ -112,6 +139,7 @@ Raises:
 """
 %unignore tensorflow::swig::Flatten;
 %noexception tensorflow::swig::Flatten;
+%feature("kwargs") tensorflow::swig::Flatten;
 
 %feature("docstring") tensorflow::swig::IsSequenceForData
 """Returns a true if `seq` is a Sequence or dict (except strings/lists).
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.-indexed-slices.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.-indexed-slices.pbtxt
index fee84d8530..5b47c718a5 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.-indexed-slices.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.-indexed-slices.pbtxt
@@ -2,6 +2,7 @@ path: "tensorflow.IndexedSlices"
 tf_class {
   is_instance: "<class \'tensorflow.python.framework.ops.IndexedSlices\'>"
   is_instance: "<class \'tensorflow.python.framework.ops._TensorLike\'>"
+  is_instance: "<class \'tensorflow.python.framework.composite_tensor.CompositeTensor\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "dense_shape"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.-ragged-tensor.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.-ragged-tensor.pbtxt
index c0ed956535..481a8c73ac 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.-ragged-tensor.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.-ragged-tensor.pbtxt
@@ -1,6 +1,7 @@
 path: "tensorflow.RaggedTensor"
 tf_class {
   is_instance: "<class \'tensorflow.python.ops.ragged.ragged_tensor.RaggedTensor\'>"
+  is_instance: "<class \'tensorflow.python.framework.composite_tensor.CompositeTensor\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "dtype"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.-sparse-tensor.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.-sparse-tensor.pbtxt
index 3add49e90d..64f7260369 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.-sparse-tensor.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.-sparse-tensor.pbtxt
@@ -2,6 +2,7 @@ path: "tensorflow.SparseTensor"
 tf_class {
   is_instance: "<class \'tensorflow.python.framework.sparse_tensor.SparseTensor\'>"
   is_instance: "<class \'tensorflow.python.framework.ops._TensorLike\'>"
+  is_instance: "<class \'tensorflow.python.framework.composite_tensor.CompositeTensor\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "dense_shape"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.sparse.-sparse-tensor.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.sparse.-sparse-tensor.pbtxt
index 02e59a63e1..a31689a58b 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.sparse.-sparse-tensor.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.sparse.-sparse-tensor.pbtxt
@@ -2,6 +2,7 @@ path: "tensorflow.sparse.SparseTensor"
 tf_class {
   is_instance: "<class \'tensorflow.python.framework.sparse_tensor.SparseTensor\'>"
   is_instance: "<class \'tensorflow.python.framework.ops._TensorLike\'>"
+  is_instance: "<class \'tensorflow.python.framework.composite_tensor.CompositeTensor\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "dense_shape"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-indexed-slices.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-indexed-slices.pbtxt
index fee84d8530..5b47c718a5 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.-indexed-slices.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.-indexed-slices.pbtxt
@@ -2,6 +2,7 @@ path: "tensorflow.IndexedSlices"
 tf_class {
   is_instance: "<class \'tensorflow.python.framework.ops.IndexedSlices\'>"
   is_instance: "<class \'tensorflow.python.framework.ops._TensorLike\'>"
+  is_instance: "<class \'tensorflow.python.framework.composite_tensor.CompositeTensor\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "dense_shape"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-ragged-tensor.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-ragged-tensor.pbtxt
index c0ed956535..481a8c73ac 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.-ragged-tensor.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.-ragged-tensor.pbtxt
@@ -1,6 +1,7 @@
 path: "tensorflow.RaggedTensor"
 tf_class {
   is_instance: "<class \'tensorflow.python.ops.ragged.ragged_tensor.RaggedTensor\'>"
+  is_instance: "<class \'tensorflow.python.framework.composite_tensor.CompositeTensor\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "dtype"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-sparse-tensor.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-sparse-tensor.pbtxt
index 3add49e90d..64f7260369 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.-sparse-tensor.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.-sparse-tensor.pbtxt
@@ -2,6 +2,7 @@ path: "tensorflow.SparseTensor"
 tf_class {
   is_instance: "<class \'tensorflow.python.framework.sparse_tensor.SparseTensor\'>"
   is_instance: "<class \'tensorflow.python.framework.ops._TensorLike\'>"
+  is_instance: "<class \'tensorflow.python.framework.composite_tensor.CompositeTensor\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "dense_shape"
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.sparse.-sparse-tensor.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.sparse.-sparse-tensor.pbtxt
index 02e59a63e1..a31689a58b 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.sparse.-sparse-tensor.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.sparse.-sparse-tensor.pbtxt
@@ -2,6 +2,7 @@ path: "tensorflow.sparse.SparseTensor"
 tf_class {
   is_instance: "<class \'tensorflow.python.framework.sparse_tensor.SparseTensor\'>"
   is_instance: "<class \'tensorflow.python.framework.ops._TensorLike\'>"
+  is_instance: "<class \'tensorflow.python.framework.composite_tensor.CompositeTensor\'>"
   is_instance: "<type \'object\'>"
   member {
     name: "dense_shape"
-- 
GitLab


From 54e40b2b13ff9cd1067050a505b091a496490f8f Mon Sep 17 00:00:00 2001
From: Rachel Lim <rachelim@google.com>
Date: Thu, 17 Jan 2019 17:01:11 -0800
Subject: [PATCH 0940/1095] [tf.data] C++ test cleanup for map_vectorization

PiperOrigin-RevId: 229844456
---
 .../core/grappler/optimizers/data/BUILD       |   2 +
 .../optimizers/data/map_vectorization_test.cc | 332 +++++++++++-------
 2 files changed, 198 insertions(+), 136 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/data/BUILD b/tensorflow/core/grappler/optimizers/data/BUILD
index fe59dfef15..4ca353d7ba 100644
--- a/tensorflow/core/grappler/optimizers/data/BUILD
+++ b/tensorflow/core/grappler/optimizers/data/BUILD
@@ -497,10 +497,12 @@ tf_cc_test(
         ":graph_utils",
         ":map_vectorization",
         "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
         "//tensorflow/core/grappler:grappler_item",
+        "//tensorflow/core/kernels/data",
     ],
 )
 
diff --git a/tensorflow/core/grappler/optimizers/data/map_vectorization_test.cc b/tensorflow/core/grappler/optimizers/data/map_vectorization_test.cc
index f4faf41549..9d3d3af9f2 100644
--- a/tensorflow/core/grappler/optimizers/data/map_vectorization_test.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_vectorization_test.cc
@@ -21,191 +21,251 @@ limitations under the License.
 #include "tensorflow/core/grappler/grappler_item.h"
 #include "tensorflow/core/grappler/optimizers/data/graph_utils.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/lib/gtl/array_slice.h"
 #include "tensorflow/core/platform/test.h"
 
 namespace tensorflow {
 namespace grappler {
 namespace {
 
-using test::function::GDef;
 using test::function::NDef;
 
-NodeDef MakeMapNodeHelper(StringPiece name, StringPiece input_node_name,
-                          StringPiece function_name, StringPiece map_op_name,
-                          gtl::ArraySlice<PartialTensorShape> output_shapes,
-                          gtl::ArraySlice<DataType> output_types) {
-  return test::function::NDef(
-      name, map_op_name, {string(input_node_name)},
-      {{"f", FunctionDefHelper::FunctionRef(string(function_name))},
-       {"Targuments", {}},
-       {"output_shapes", output_shapes},
-       {"output_types", output_types}});
+NodeDef* AddConstBoolNode(GraphDef* graph, StringPiece name, bool value) {
+  NodeDef* node = graph->add_node();
+  *node = NDef(/*name=*/name, /*op=*/"Const", /*inputs=*/{},
+               /*attrs=*/{{"value", value}, {"dtype", DT_BOOL}});
+
+  return node;
 }
 
-NodeDef MakeMapNode(StringPiece name, StringPiece input_node_name,
-                    StringPiece function_name,
-                    gtl::ArraySlice<PartialTensorShape> output_shapes,
-                    gtl::ArraySlice<DataType> output_types) {
-  return MakeMapNodeHelper(name, input_node_name, function_name, "MapDataset",
-                           output_shapes, output_types);
+NodeDef* AddConstInt32Node(GraphDef* graph, StringPiece name, int value) {
+  NodeDef* node = graph->add_node();
+  *node = NDef(/*name=*/name, /*op=*/"Const", /*inputs=*/{},
+               /*attrs=*/{{"value", value}, {"dtype", DT_INT32}});
+
+  return node;
 }
 
-NodeDef MakeBatchNode(StringPiece name, StringPiece input_node_name,
-                      StringPiece input_batch_size_name,
-                      gtl::ArraySlice<PartialTensorShape> output_shapes,
-                      gtl::ArraySlice<DataType> output_types) {
-  return NDef(
-      name, "BatchDataset",
-      {string(input_node_name), string(input_batch_size_name)},
-      {{"output_types", output_types}, {"output_shapes", output_shapes}});
+NodeDef* AddConstInt64Node(GraphDef* graph, StringPiece name, int64 value) {
+  NodeDef* node = graph->add_node();
+  *node = NDef(/*name=*/name, /*op=*/"Const", /*inputs=*/{},
+               /*attrs=*/{{"value", value}, {"dtype", DT_INT64}});
+
+  return node;
 }
 
-NodeDef MakeBatchV2Node(StringPiece name, StringPiece input_node_name,
-                        StringPiece input_batch_size_name,
-                        StringPiece input_drop_remainder_name,
-                        gtl::ArraySlice<PartialTensorShape> output_shapes,
-                        gtl::ArraySlice<DataType> output_types) {
-  return NDef(
-      name, "BatchDatasetV2",
-      {string(input_node_name), string(input_batch_size_name),
-       string(input_drop_remainder_name)},
-      {{"output_types", output_types}, {"output_shapes", output_shapes}});
+// Adds a simple vectorizable map function that is akin to
+// dataset.map(lambda x: tf.identity(x))
+FunctionDef* AddMapFn(GraphDef* graph) {
+  FunctionDef* map_fn = graph->mutable_library()->add_function();
+  *map_fn = FunctionDefHelper::Create(
+      /*function_name=*/"map_fn",
+      /*in_def=*/{"x: int64"},
+      /*out_def=*/{"res: int64"},
+      /*attr_def=*/{},
+      /*node_def=*/{{{"node"}, "Identity", {"x"}, {{"T", DT_INT64}}}},
+      /*ret_def=*/{{"res", "node:output"}});
+
+  return map_fn;
 }
 
-NodeDef MakeRangeNode(StringPiece name, gtl::ArraySlice<string> inputs) {
-  return NDef(name, "RangeDataset", inputs,
-              {{"output_shapes", gtl::ArraySlice<TensorShape>({{}})},
-               {"output_types", gtl::ArraySlice<DataType>({DT_INT64})}});
+NodeDef* AddMapNode(GraphDef* graph, const string& input_dataset,
+                    bool parallel = false) {
+  NodeDef* map_node = graph->add_node();
+  auto map_fn = AddMapFn(graph);
+
+  if (parallel) {
+    auto num_parallel_calls = AddConstInt32Node(graph, "num_parallel_calls", 8);
+    *map_node =
+        NDef(/*name=*/"map", /*op=*/"ParallelMapDataset",
+             /*inputs=*/{input_dataset, num_parallel_calls->name()},
+             /*attrs=*/
+             {{"f", FunctionDefHelper::FunctionRef(map_fn->signature().name())},
+              {"Targuments", gtl::ArraySlice<DataType>({})},
+              {"output_types", gtl::ArraySlice<DataType>({DT_INT64})},
+              {"output_shapes", gtl::ArraySlice<TensorShape>({{}})},
+              {"use_inter_op_parallelism", false},
+              {"sloppy", true},
+              {"preserve_cardinality", true}});
+  } else {
+    *map_node =
+        NDef(/*name=*/"map", /*op=*/"MapDataset",
+             /*inputs=*/{input_dataset},
+             /*attrs=*/
+             {{"f", FunctionDefHelper::FunctionRef(map_fn->signature().name())},
+              {"Targuments", gtl::ArraySlice<DataType>({})},
+              {"output_types", gtl::ArraySlice<DataType>({DT_INT64})},
+              {"output_shapes", gtl::ArraySlice<TensorShape>({{}})},
+              {"use_inter_op_parallelism", false},
+              {"preserve_cardinality", true}});
+  }
+
+  return map_node;
 }
 
-TEST(MapVectorizationTest, VectorizeMapWithBatch) {
-  GrapplerItem item;
-  item.graph = GDef(
-      {NDef("start", "Const", {}, {{"value", 0}, {"dtype", DT_INT32}}),
-       NDef("stop", "Const", {}, {{"value", 10}, {"dtype", DT_INT32}}),
-       NDef("step", "Const", {}, {{"value", 1}, {"dtype", DT_INT32}}),
-       NDef("batch_size", "Const", {}, {{"value", 1}, {"dtype", DT_INT32}}),
-       MakeRangeNode("range", {"start", "stop", "step"}),
-       MakeMapNode("map", "range", "XTimesTwo", {{}}, {DT_INT32}),
-       MakeBatchNode("batch", "map", "batch_size", {{-1}}, {DT_INT32})},
-      // FunctionLib
-      {
-          test::function::XTimesTwo(),
-      });
-  MapVectorization optimizer;
-  GraphDef output;
-  TF_ASSERT_OK(optimizer.Optimize(nullptr, item, &output));
+NodeDef* AddBatchNode(GraphDef* graph, const string& input_dataset,
+                      bool v2 = false) {
+  NodeDef* batch_node = graph->add_node();
+
+  auto batch_size = AddConstInt64Node(graph, "batch_size", 10);
+
+  if (v2) {
+    auto drop_remainder = AddConstBoolNode(graph, "drop_remainder", true);
+    *batch_node = NDef(
+        /*name=*/"batch", /*op=*/"BatchDatasetV2",
+        /*inputs=*/{input_dataset, batch_size->name(), drop_remainder->name()},
+        /*attrs=*/
+        {{"output_types", gtl::ArraySlice<DataType>({DT_INT64})},
+         {"output_shapes", gtl::ArraySlice<TensorShape>({{10, 1}})}});
+  } else {
+    *batch_node = NDef(
+        /*name=*/"batch", /*op=*/"BatchDataset",
+        /*inputs=*/{input_dataset, batch_size->name()},
+        /*attrs=*/
+        {{"output_types", gtl::ArraySlice<DataType>({DT_INT64})},
+         {"output_shapes", gtl::ArraySlice<PartialTensorShape>({{-1, 1}})}});
+  }
+  return batch_node;
+}
+
+NodeDef* AddRangeNode(GraphDef* graph) {
+  auto start = AddConstInt64Node(graph, "start", 0);
+  auto stop = AddConstInt64Node(graph, "stop", 10);
+  auto step = AddConstInt64Node(graph, "step", 1);
+  NodeDef* range_node = graph->add_node();
+  *range_node = NDef(/*name=*/"range", /*op=*/"RangeDataset",
+                     /*inputs=*/{start->name(), stop->name(), step->name()},
+                     /*attrs=*/
+                     {{"output_shapes", gtl::ArraySlice<TensorShape>({{}})},
+                      {"output_types", gtl::ArraySlice<DataType>({DT_INT64})}});
+
+  return range_node;
+}
+
+void CheckNotVectorized(const GraphDef& output, const string& map_op,
+                        const string& batch_op, const string& map_input_name) {
+  ASSERT_EQ(graph_utils::FindAllGraphNodesWithOp(map_op, output).size(), 1);
+  ASSERT_EQ(graph_utils::FindAllGraphNodesWithOp(batch_op, output).size(), 1);
+  const NodeDef& map_node =
+      output.node(graph_utils::FindGraphNodeWithOp(map_op, output));
+  const NodeDef& batch_node =
+      output.node(graph_utils::FindGraphNodeWithOp(batch_op, output));
+  EXPECT_EQ(map_node.input(0), map_input_name);
+  EXPECT_EQ(batch_node.input(0), map_node.name());
+}
 
-  EXPECT_EQ(graph_utils::FindAllGraphNodesWithOp("MapDataset", output).size(),
-            1);
-  EXPECT_EQ(graph_utils::FindAllGraphNodesWithOp("BatchDataset", output).size(),
-            1);
+void CheckVectorized(const GraphDef& output, const string& map_op,
+                     const string& batch_op, const string& map_input_name) {
+  ASSERT_EQ(graph_utils::FindAllGraphNodesWithOp(map_op, output).size(), 1);
+  ASSERT_EQ(graph_utils::FindAllGraphNodesWithOp(batch_op, output).size(), 1);
   const NodeDef& map_node =
-      output.node(graph_utils::FindGraphNodeWithOp("MapDataset", output));
+      output.node(graph_utils::FindGraphNodeWithOp(map_op, output));
   const NodeDef& batch_node =
-      output.node(graph_utils::FindGraphNodeWithOp("BatchDataset", output));
+      output.node(graph_utils::FindGraphNodeWithOp(batch_op, output));
   EXPECT_EQ(map_node.input(0), batch_node.name());
-  EXPECT_EQ(batch_node.input(0), "range");
+  EXPECT_EQ(batch_node.input(0), map_input_name);
+
+  // Check that the function is actually vectorized.
+  // The vectorization of the identity function is itself.
+  string function_name = map_node.attr().at("f").func().name();
+  int found =
+      graph_utils::FindGraphFunctionWithName(function_name, output.library());
+  ASSERT_NE(found, -1);
+  const auto& function = output.library().function(found);
+  EXPECT_EQ(function.node_def(0).op(), "Identity");
 }
 
-TEST(MapVectorizationTest, VectorizeMapWithBatchV2) {
+class MapThenBatchTest
+    : public ::testing::TestWithParam<std::tuple<bool, bool>> {};
+
+TEST_P(MapThenBatchTest, IsVectorized) {
+  bool use_parallel_map = std::get<0>(GetParam());
+  bool use_batch_v2 = std::get<1>(GetParam());
   GrapplerItem item;
-  item.graph = GDef(
-      {NDef("start", "Const", {}, {{"value", 0}, {"dtype", DT_INT32}}),
-       NDef("stop", "Const", {}, {{"value", 10}, {"dtype", DT_INT32}}),
-       NDef("step", "Const", {}, {{"value", 1}, {"dtype", DT_INT32}}),
-       NDef("batch_size", "Const", {}, {{"value", 1}, {"dtype", DT_INT32}}),
-       NDef("drop_remainder", "Const", {},
-            {{"value", false}, {"dtype", DT_BOOL}}),
-       MakeRangeNode("range", {"start", "stop", "step"}),
-       MakeMapNode("map", "range", "XTimesTwo", {{}}, {DT_INT32}),
-       MakeBatchV2Node("batch", "map", "batch_size", "drop_remainder", {{-1}},
-                       {DT_INT32})},
-      // FunctionLib
-      {
-          test::function::XTimesTwo(),
-      });
+  auto range_node = AddRangeNode(&item.graph);
+  auto map_node = AddMapNode(&item.graph, range_node->name(), use_parallel_map);
+  auto batch_node = AddBatchNode(&item.graph, map_node->name(), use_batch_v2);
   MapVectorization optimizer;
   GraphDef output;
   TF_ASSERT_OK(optimizer.Optimize(nullptr, item, &output));
+  CheckVectorized(output, map_node->op(), batch_node->op(), range_node->name());
+}
 
-  EXPECT_EQ(graph_utils::FindAllGraphNodesWithOp("MapDataset", output).size(),
-            1);
-  EXPECT_EQ(
-      graph_utils::FindAllGraphNodesWithOp("BatchDatasetV2", output).size(), 1);
-  const NodeDef& map_node =
-      output.node(graph_utils::FindGraphNodeWithOp("MapDataset", output));
-  const NodeDef& batch_node =
-      output.node(graph_utils::FindGraphNodeWithOp("BatchDatasetV2", output));
-  EXPECT_EQ(map_node.input(0), batch_node.name());
-  EXPECT_EQ(batch_node.input(0), "range");
+INSTANTIATE_TEST_SUITE_P(MapThenBatchTest, MapThenBatchTest,
+                         ::testing::Combine(::testing::Bool(),
+                                            ::testing::Bool()));
+
+// Not all dataset types have "output_shapes" and "output_types" attrs defined.
+// Add a generic input node which may not have these attrs defined.
+NodeDef* AddArbitraryInputNode(GraphDef* graph,
+                               std::vector<PartialTensorShape>* output_shapes,
+                               std::vector<DataType>* output_types) {
+  auto input_node = graph->add_node();
+  std::vector<std::pair<string, FunctionDefHelper::AttrValueWrapper>> attrs;
+  if (output_shapes) {
+    attrs.push_back({"output_shapes", *output_shapes});
+  }
+  if (output_types) {
+    attrs.push_back({"output_types", *output_types});
+  }
+
+  *input_node = NDef(/*name=*/"input", /*op=*/"InputDataset", /*inputs=*/{},
+                     /*attrs=*/attrs);
+
+  return input_node;
 }
 
-TEST(MapVectorizationTest, VectorizeWithUndefinedOutputShape) {
+TEST(MapVectorizationTest, VectorizeWithUndefinedOutputShapes) {
+  // Tests that the optimization doesn't break when the input to MapDataset
+  // doesn't have an output_shapes attr defined. In this case, the map and
+  // batch swap does not occur.
   GrapplerItem item;
-  item.graph = GDef(
-      {NDef("batch_size", "Const", {}, {{"value", 1}, {"dtype", DT_INT32}}),
-       NDef("input", "InputDataset", {},
-            {{"output_types", gtl::ArraySlice<DataType>({DT_INT32})}}),
-       MakeMapNode("map", "input", "XTimesTwo", {{}}, {DT_INT32}),
-       MakeBatchNode("batch", "map", "batch_size", {{-1}}, {DT_INT32})},
-      // FunctionLib
-      {
-          test::function::XTimesTwo(),
-      });
+  std::vector<DataType> input_types({DT_INT64});
+  auto input_node = AddArbitraryInputNode(&item.graph, nullptr, &input_types);
+  auto map_node = AddMapNode(&item.graph, input_node->name());
+  auto batch_node = AddBatchNode(&item.graph, map_node->name());
   MapVectorization optimizer;
   GraphDef output;
   TF_ASSERT_OK(optimizer.Optimize(nullptr, item, &output));
+  CheckNotVectorized(output, map_node->op(), batch_node->op(),
+                     input_node->name());
 }
 
-TEST(MapVectorizationTest, VectorizeWithUndefinedOutputTypes) {
+TEST(MapVectorizationTest, VectorizeWithUnknownDim) {
+  // Tests that the optimization doesn't break when the input to MapDataset
+  // doesn't has some components with unknown dimensions. In this case, the
+  // optimization does not occur.
   GrapplerItem item;
-  item.graph = GDef(
-      {NDef("batch_size", "Const", {}, {{"value", 1}, {"dtype", DT_INT32}}),
-       NDef("input", "InputDataset", {},
-            {{"output_shapes", gtl::ArraySlice<TensorShape>({{}})}}),
-       MakeMapNode("map", "input", "XTimesTwo", {{}}, {DT_INT32}),
-       MakeBatchNode("batch", "map", "batch_size", {{-1}}, {DT_INT32})},
-      // FunctionLib
-      {
-          test::function::XTimesTwo(),
-      });
+  std::vector<PartialTensorShape> input_shapes({{-1, 2}});
+  std::vector<DataType> input_types({DT_INT64});
+  auto input_node =
+      AddArbitraryInputNode(&item.graph, &input_shapes, &input_types);
+  auto map_node = AddMapNode(&item.graph, input_node->name());
+  auto batch_node = AddBatchNode(&item.graph, map_node->name());
   MapVectorization optimizer;
   GraphDef output;
   TF_ASSERT_OK(optimizer.Optimize(nullptr, item, &output));
+  CheckNotVectorized(output, map_node->op(), batch_node->op(),
+                     input_node->name());
 }
 
-TEST(MapVectorizationTest, VectorizeWithFullyDefinedFunction) {
+TEST(MapVectorizationTest, VectorizeWithUndefinedOutputTypes) {
+  // Tests that the optimization doesn't break when the input doesn't have
+  // an output_types attr defined. The output_types of the input node, even
+  // if not present, can be inferred from the map function input signature.
   GrapplerItem item;
-  item.graph = GDef(
-      {NDef("start", "Const", {}, {{"value", 0}, {"dtype", DT_INT32}}),
-       NDef("stop", "Const", {}, {{"value", 10}, {"dtype", DT_INT32}}),
-       NDef("step", "Const", {}, {{"value", 1}, {"dtype", DT_INT32}}),
-       NDef("batch_size", "Const", {}, {{"value", 1}, {"dtype", DT_INT32}}),
-       MakeRangeNode("range", {"start", "stop", "step"}),
-       MakeMapNode("map", "range", "Func", {{}}, {DT_INT32}),
-       MakeBatchNode("batch", "map", "batch_size", {{-1}}, {DT_INT32})},
-      // FunctionLib
-      {FunctionDefHelper::Create(
-          "Func", {"x: int64", "y: int64"}, {"res: int64", "res2: int64"}, {},
-          {{{"o"}, "Mul", {"x", "x"}, {{"T", DT_INT64}}}},
-          {{"res", "o:z"}, {"res2", "o:z"}})});
+  std::vector<PartialTensorShape> input_shapes({{1}});
+  auto input_node = AddArbitraryInputNode(&item.graph, &input_shapes, nullptr);
+  auto map_node = AddMapNode(&item.graph, input_node->name());
+  auto batch_node = AddBatchNode(&item.graph, map_node->name());
   MapVectorization optimizer;
   GraphDef output;
   TF_ASSERT_OK(optimizer.Optimize(nullptr, item, &output));
-
-  EXPECT_EQ(graph_utils::FindAllGraphNodesWithOp("MapDataset", output).size(),
-            1);
-  EXPECT_EQ(graph_utils::FindAllGraphNodesWithOp("BatchDataset", output).size(),
-            1);
-  const NodeDef& map_node =
-      output.node(graph_utils::FindGraphNodeWithOp("MapDataset", output));
-  const NodeDef& batch_node =
-      output.node(graph_utils::FindGraphNodeWithOp("BatchDataset", output));
-  EXPECT_EQ(map_node.input(0), batch_node.name());
-  EXPECT_EQ(batch_node.input(0), "range");
+  CheckVectorized(output, map_node->op(), batch_node->op(), input_node->name());
 }
 
+// TODO(rachelim): Add test that has a polymorphic function.
+
 }  // namespace
 }  // namespace grappler
 }  // namespace tensorflow
-- 
GitLab


From 15dca2b2fd1fc8ecab3f605fc810cd5d76fa3851 Mon Sep 17 00:00:00 2001
From: Reed Wanderman-Milne <reedwm@google.com>
Date: Thu, 17 Jan 2019 17:26:35 -0800
Subject: [PATCH 0941/1095] Automated rollback of commit
 76c2a2512c3fefe065415e338a98c1043acbb131

PiperOrigin-RevId: 229848074
---
 tensorflow/python/module/module_test.py | 66 ++++++++++++-------------
 1 file changed, 33 insertions(+), 33 deletions(-)

diff --git a/tensorflow/python/module/module_test.py b/tensorflow/python/module/module_test.py
index d5d36a7bbc..e7c736696e 100644
--- a/tensorflow/python/module/module_test.py
+++ b/tensorflow/python/module/module_test.py
@@ -143,64 +143,64 @@ class VariableTrackingTest(test.TestCase):
 
   def test_variables(self):
     m = RecursiveModule(3)
-    self.assertCountEqual(m.variables, (m.w, m.child.w, m.child.child.w))
-    self.assertCountEqual(m.child.variables, (m.child.w, m.child.child.w))
-    self.assertCountEqual(m.child.child.variables, (m.child.child.w,))
+    self.assertEqual(m.variables, (m.w, m.child.w, m.child.child.w))
+    self.assertEqual(m.child.variables, (m.child.w, m.child.child.w))
+    self.assertEqual(m.child.child.variables, (m.child.child.w,))
 
   def test_owned_variables(self):
     m = RecursiveModule(3)
-    self.assertCountEqual(m.owned_variables, (m.w,))
-    self.assertCountEqual(m.child.owned_variables, (m.child.w,))
-    self.assertCountEqual(m.child.child.owned_variables, (m.child.child.w,))
+    self.assertEqual(m.owned_variables, (m.w,))
+    self.assertEqual(m.child.owned_variables, (m.child.w,))
+    self.assertEqual(m.child.child.owned_variables, (m.child.child.w,))
 
   def test_trainable_variables(self):
     m = RecursiveModule(3)
-    self.assertCountEqual(m.trainable_variables,
-                          (m.w, m.child.w, m.child.child.w))
-    self.assertCountEqual(m.child.trainable_variables,
-                          (m.child.w, m.child.child.w))
-    self.assertCountEqual(m.child.child.trainable_variables, (m.child.child.w,))
+    self.assertEqual(m.trainable_variables,
+                     (m.w, m.child.w, m.child.child.w))
+    self.assertEqual(m.child.trainable_variables,
+                     (m.child.w, m.child.child.w))
+    self.assertEqual(m.child.child.trainable_variables, (m.child.child.w,))
 
   def test_trainable_variables_ignores_non_trainable(self):
     m = RecursiveModule(3, trainable=False)
-    self.assertEmpty(m.trainable_variables)
-    self.assertEmpty(m.child.trainable_variables)
-    self.assertEmpty(m.child.child.trainable_variables)
+    self.assertEqual(len(m.trainable_variables), 0)
+    self.assertEqual(len(m.child.trainable_variables), 0)
+    self.assertEqual(len(m.child.child.trainable_variables), 0)
 
   def test_owned_trainable_variables(self):
     m = RecursiveModule(3)
-    self.assertCountEqual(m.owned_trainable_variables, (m.w,))
-    self.assertCountEqual(m.child.owned_trainable_variables, (m.child.w,))
-    self.assertCountEqual(m.child.child.owned_trainable_variables,
-                          (m.child.child.w,))
+    self.assertEqual(m.owned_trainable_variables, (m.w,))
+    self.assertEqual(m.child.owned_trainable_variables, (m.child.w,))
+    self.assertEqual(m.child.child.owned_trainable_variables,
+                     (m.child.child.w,))
 
   def test_owned_trainable_variables_ignores_non_trainable(self):
     m = RecursiveModule(3, trainable=False)
-    self.assertEmpty(m.owned_trainable_variables)
-    self.assertEmpty(m.child.owned_trainable_variables)
-    self.assertEmpty(m.child.child.owned_trainable_variables)
+    self.assertEqual(len(m.owned_trainable_variables), 0)
+    self.assertEqual(len(m.child.owned_trainable_variables), 0)
+    self.assertEqual(len(m.child.child.owned_trainable_variables), 0)
 
 
 class ModuleTrackingTest(test.TestCase):
 
   def test_owned_submodules(self):
     m = RecursiveModule(3)
-    self.assertCountEqual(m.owned_submodules, [m.child])
-    self.assertCountEqual(m.child.owned_submodules, [m.child.child])
-    self.assertEmpty(list(m.child.child.owned_submodules))
+    self.assertEqual(list(m.owned_submodules), [m.child])
+    self.assertEqual(list(m.child.owned_submodules), [m.child.child])
+    self.assertEqual(list(m.child.child.owned_submodules), [])
 
   def test_submodules(self):
     m = RecursiveModule(3)
-    self.assertCountEqual(m.submodules, [m.child, m.child.child])
-    self.assertCountEqual(m.child.submodules, [m.child.child])
-    self.assertEmpty(list(m.child.child.submodules))
+    self.assertEqual(list(m.submodules), [m.child, m.child.child])
+    self.assertEqual(list(m.child.submodules), [m.child.child])
+    self.assertEqual(list(m.child.child.submodules), [])
 
   def test_non_ctor_submodule(self):
     m = TreeModule()
     leaf1 = m.new_leaf()
-    self.assertCountEqual(m.submodules, (leaf1,))
+    self.assertEqual(set(m.submodules), {leaf1})
     leaf2 = m.new_leaf()
-    self.assertCountEqual(m.submodules, (leaf1, leaf2))
+    self.assertEqual(set(m.submodules), {leaf1, leaf2})
 
 
 class CommonErrorsTest(test.TestCase):
@@ -323,12 +323,12 @@ class WalkTest(test.TestCase):
     parent = SimpleModule()
     child = parent.c
 
-    self.assertCountEqual(
-        module.walk(parent, predicate=IS_MEMBER),
+    self.assertEqual(
+        list(module.walk(parent, predicate=IS_MEMBER)),
         [parent.a[0], parent.a[1], parent.z])
 
-    self.assertCountEqual(
-        module.walk(parent, recurse_if=IS_MODULE, predicate=IS_MEMBER),
+    self.assertEqual(
+        list(module.walk(parent, recurse_if=IS_MODULE, predicate=IS_MEMBER)),
         [parent.a[0], parent.a[1], parent.z, child.a[0], child.a[1], child.z])
 
 
-- 
GitLab


From 21734ca1e7f61ab8d62119b5a99be2926da58a7c Mon Sep 17 00:00:00 2001
From: Rachel Lim <rachelim@google.com>
Date: Thu, 17 Jan 2019 17:27:48 -0800
Subject: [PATCH 0942/1095] [tf.data] Fixes a bug with how map_vectorization
 checks that the output shapes of the dataset nodes are fully defined.

PiperOrigin-RevId: 229848249
---
 .../optimizers/data/map_vectorization.cc      |  1 +
 .../optimizers/data/map_vectorization_test.cc | 22 +++++++++++++++++--
 .../optimization/map_vectorization_test.py    |  8 +++----
 3 files changed, 25 insertions(+), 6 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/data/map_vectorization.cc b/tensorflow/core/grappler/optimizers/data/map_vectorization.cc
index 4c9e1f31e7..2dbd6e4698 100644
--- a/tensorflow/core/grappler/optimizers/data/map_vectorization.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_vectorization.cc
@@ -121,6 +121,7 @@ bool IsOutputShapesFullyDefined(const NodeDef& node) {
   const auto& shapes = shapes_attr->list().shape();
 
   for (const TensorShapeProto& shape : shapes) {
+    if (shape.unknown_rank()) return false;
     for (const auto& dim : shape.dim()) {
       if (dim.size() == -1) {
         return false;
diff --git a/tensorflow/core/grappler/optimizers/data/map_vectorization_test.cc b/tensorflow/core/grappler/optimizers/data/map_vectorization_test.cc
index 9d3d3af9f2..890041f57c 100644
--- a/tensorflow/core/grappler/optimizers/data/map_vectorization_test.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_vectorization_test.cc
@@ -231,10 +231,28 @@ TEST(MapVectorizationTest, VectorizeWithUndefinedOutputShapes) {
                      input_node->name());
 }
 
+TEST(MapVectorizationTest, VectorizeWithUnknownRank) {
+  // Tests that the optimization doesn't break when the input to MapDataset
+  // has components with unknown rank. In this case, the optimization does not
+  // occur.
+  GrapplerItem item;
+  std::vector<PartialTensorShape> input_shapes({{}});
+  std::vector<DataType> input_types({DT_INT64});
+  auto input_node =
+      AddArbitraryInputNode(&item.graph, &input_shapes, &input_types);
+  auto map_node = AddMapNode(&item.graph, input_node->name());
+  auto batch_node = AddBatchNode(&item.graph, map_node->name());
+  MapVectorization optimizer;
+  GraphDef output;
+  TF_ASSERT_OK(optimizer.Optimize(nullptr, item, &output));
+  CheckNotVectorized(output, map_node->op(), batch_node->op(),
+                     input_node->name());
+}
+
 TEST(MapVectorizationTest, VectorizeWithUnknownDim) {
   // Tests that the optimization doesn't break when the input to MapDataset
-  // doesn't has some components with unknown dimensions. In this case, the
-  // optimization does not occur.
+  // has components with unknown dimensions. In this case, the optimization does
+  // not occur.
   GrapplerItem item;
   std::vector<PartialTensorShape> input_shapes({{-1, 2}});
   std::vector<DataType> input_types({DT_INT64});
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
index ec543aeb76..9ffb8a1825 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
@@ -225,6 +225,10 @@ def _generate_csv_test_case():
 
 
 def _generate_parse_single_example_test_case():
+  # When sparse tensors are used, map_vectorization is not
+  # attempted because the output_shapes of the map dataset are not defined.
+  # TODO(rachelim): Consider being more lax with checking the output_shapes of
+  # the map node.
 
   def parse_example_factory():
 
@@ -243,8 +247,6 @@ def _generate_parse_single_example_test_case():
                     feature={
                         "dense_int": _int64_feature(i),
                         "dense_str": _bytes_feature(str(i)),
-                        "sparse_int": _int64_feature(i, i * 2, i * 4, i * 8),
-                        "sparse_str": _bytes_feature(*["abc"] * i)
                     })).SerializeToString() for i in range(10)
         ]))
 
@@ -252,8 +254,6 @@ def _generate_parse_single_example_test_case():
     features = {
         "dense_int": parsing_ops.FixedLenFeature((), dtypes.int64, 0),
         "dense_str": parsing_ops.FixedLenFeature((), dtypes.string, ""),
-        "sparse_int": parsing_ops.VarLenFeature(dtypes.int64),
-        "sparse_str": parsing_ops.VarLenFeature(dtypes.string),
     }
     return parsing_ops.parse_single_example(x, features)
 
-- 
GitLab


From fd1ab41166633dc5176c792fbd4c843ec2595857 Mon Sep 17 00:00:00 2001
From: Smit Hinsu <hinsu@google.com>
Date: Thu, 17 Jan 2019 17:31:43 -0800
Subject: [PATCH 0943/1095] Fix handling of truncated tensors in TensorRT Const
 op conversion

Currently TensorRT converter manually decodes tensor from protocol buffer and
it does not handle the case where tensor is encoded after truncating trailing
elements that have the same values. Fixed this by simplifying the conversion to
let TensorFlow decode tensor from the protocol buffer and then copying the
values to weights, possibly after casting to an appropriate DataType.

PiperOrigin-RevId: 229848825
---
 .../contrib/tensorrt/convert/convert_nodes.cc | 161 +++++++-----------
 .../contrib/tensorrt/convert/convert_nodes.h  |   4 +
 .../tensorrt/convert/convert_nodes_test.cc    |  61 ++++++-
 3 files changed, 119 insertions(+), 107 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 8b7279ad03..59d295f2e1 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -2712,43 +2712,69 @@ tensorflow::Status ConvertBiasAdd(OpConverterParams* params) {
   return Status::OK();
 }
 
-Status GetTensorDimsWithProtoShape(const Tensor& tensor,
-                                   int tensor_proto_array_len,
-                                   nvinfer1::Dims* dims) {
+void GetTensorDimsWithProtoShape(const Tensor& tensor, nvinfer1::Dims* dims) {
   if (tensor.dims() > 0) {
     *dims = GetTrtDimsForTensor(tensor);
-    if (TrtDimsNumElements(*dims) != tensor_proto_array_len &&
-        tensor_proto_array_len != 1) {
-      return errors::InvalidArgument(
-          "Broadcast on weights only supports kCHANNEL and kUNIFORM");
-    }
   } else {
     dims->nbDims = 1;
     // No dimension provided. Flatten it.
-    dims->d[0] = tensor_proto_array_len;
+    dims->d[0] = tensor.NumElements();
     dims->type[0] = nvinfer1::DimensionType::kSPATIAL;
     for (int i = 1; i < nvinfer1::Dims::MAX_DIMS; ++i) {
       dims->d[i] = 0;
     }
   }
-  return Status::OK();
 }
 
-template <typename CType>
-Status TfTensorToTrtWeights(const DataType dtype, const Tensor& tensor,
-                            const CType* tensor_proto_array,
-                            int tensor_proto_array_len, TrtWeightStore* store,
+Status TfTensorToTrtWeights(const Tensor& tensor, TrtWeightStore* weight_store,
                             TRT_ShapedWeights* weights) {
+  const DataType dtype = tensor.dtype();
+
+  // We always convert the integer constants to INT32, since TRT INT8 is for
+  // quantized inference.
+  //
+  // TODO(aaroey): FP16 will remain in half format and is not converted to
+  // FP32, but the converter currently uses all float weights as FP32. Fix
+  // this.
+  const DataType converted_dtype =
+      (dtype == DT_INT16 || dtype == DT_INT8 || dtype == DT_UINT8 ? DT_INT32
+                                                                  : dtype);
+
+  // Verify that the dtype is supported by TensorRT. Otherwise, return an error.
+  nvinfer1::DataType trt_dtype;
+  TF_RETURN_IF_ERROR(ConvertDType(converted_dtype, &trt_dtype));
+
+  if (tensor.NumElements() == 0) {
+    // Return empty weights having converted dtype.
+    *weights = TRT_ShapedWeights(converted_dtype);
+    return Status::OK();
+  }
+
   nvinfer1::Dims weight_dims;
-  TF_RETURN_IF_ERROR(GetTensorDimsWithProtoShape(tensor, tensor_proto_array_len,
-                                                 &weight_dims));
-  *weights = store->GetTempWeights(dtype, weight_dims);
-  void* dst = const_cast<void*>(weights->GetValues());
-  if (tensor_proto_array_len == 1) {
-    std::fill_n((CType*)dst, TrtDimsNumElements(weight_dims),
-                *tensor_proto_array);
+  GetTensorDimsWithProtoShape(tensor, &weight_dims);
+  *weights = weight_store->GetTempWeights(converted_dtype, weight_dims);
+
+  // Copy the tensor directly if the tensor does not require cast to the
+  // supported type.
+  if (converted_dtype == dtype) {
+    char* dst = static_cast<char*>(const_cast<void*>(weights->GetValues()));
+    memcpy(dst, tensor.tensor_data().data(), tensor.TotalBytes());
+    return Status::OK();
+  }
+
+  // Copy tensor elements after casting them to the converted DataType.
+  int32* dst = static_cast<int32*>(const_cast<void*>(weights->GetValues()));
+  if (dtype == DT_INT16) {
+    const int16* src = tensor.flat<int16>().data();
+    std::copy(src, src + tensor.NumElements(), dst);
+  } else if (dtype == DT_INT8) {
+    const int8* src = tensor.flat<int8>().data();
+    std::copy(src, src + tensor.NumElements(), dst);
   } else {
-    memcpy(dst, tensor_proto_array, weights->size_bytes());
+    // dtype can only be DT_UINT8 at this point.
+    TFTRT_CHECK_EQ_TYPE(dtype, DT_UINT8);
+    const uint8* src = tensor.flat<uint8>().data();
+    std::copy(src, src + tensor.NumElements(), dst);
   }
   return Status::OK();
 }
@@ -2766,15 +2792,6 @@ tensorflow::Status ConvertConst(OpConverterParams* params) {
         "Constant node is expected to have empty input list: ",
         node_def.name());
   }
-  TFAttrs attrs(node_def);
-  const DataType dtype = attrs.get<tensorflow::DataType>("dtype");
-  // We always convert the integer constants to kINT32, since TRT kINT8 is for
-  // quantized inference.
-  const DataType converted_dtype =
-      (dtype == DT_INT16 || dtype == DT_INT8 || dtype == DT_UINT8 ? DT_INT32
-                                                                  : dtype);
-  nvinfer1::DataType trt_dtype;
-  TF_RETURN_IF_ERROR(ConvertDType(converted_dtype, &trt_dtype));
 
   // Create shaped weights as output
   const auto& tensor_proto = node_def.attr().at("value").tensor();
@@ -2784,78 +2801,18 @@ tensorflow::Status ConvertConst(OpConverterParams* params) {
                                         node_def.name());
   }
 
-  TRT_ShapedWeights weights(converted_dtype);
-  if (tensor.NumElements() == 0) {
-    // Do nothing.
-  } else if (!tensor_proto.float_val().empty()) {
-    TF_RETURN_IF_ERROR(TfTensorToTrtWeights(
-        converted_dtype, tensor, tensor_proto.float_val().begin(),
-        tensor_proto.float_val_size(), params->weight_store, &weights));
-  } else if (!tensor_proto.int_val().empty()) {
-    TF_RETURN_IF_ERROR(TfTensorToTrtWeights(
-        converted_dtype, tensor, tensor_proto.int_val().begin(),
-        tensor_proto.int_val_size(), params->weight_store, &weights));
-  } else if (!tensor_proto.half_val().empty()) {
-    // TODO(aaroey): implement fp16 conversion.
-    return errors::Unimplemented("fp16 constant is not supported yet.");
-  } else if (!tensor_proto.tensor_content().empty()) {
-    // TODO(aaroey): fp16 will remain in half format and is not converted to
-    // fp32, but the converter currently uses all float weights as fp32. Fix
-    // this.
-    const auto& content = tensor_proto.tensor_content();
-    if (content.size() > 0) {
-      const int dtype_size = tensorflow::DataTypeSize(dtype);
-      if (content.size() % dtype_size != 0) {
-        return errors::FailedPrecondition("Tensor content size ",
-                                          content.size(),
-                                          " is not a multiple of ", dtype_size);
-      }
-      nvinfer1::Dims weights_dim;
-      TF_RETURN_IF_ERROR(GetTensorDimsWithProtoShape(
-          tensor, content.size() / dtype_size, &weights_dim));
-      const int64_t size_bytes = TrtDimsNumElements(weights_dim) * dtype_size;
-      if (content.size() != size_bytes) {
-        return errors::FailedPrecondition(
-            "Tensor size and TensorProto content size mismatch: ", size_bytes,
-            " vs ", content.size());
-      } else if (tensor.NumElements() != content.size() / dtype_size) {
-        return errors::FailedPrecondition(
-            "Tensor elements count and TensorProto content size mismatch: ",
-            tensor.NumElements(), " vs ", content.size() / dtype_size);
-      }
-      weights =
-          params->weight_store->GetTempWeights(converted_dtype, weights_dim);
-      if (dtype_size == tensorflow::DataTypeSize(converted_dtype)) {
-        port::CopyToArray(content, static_cast<char*>(
-                                       const_cast<void*>(weights.GetValues())));
-      } else {
-        // Copy out the weights as original data type.
-        std::vector<uint8_t> temp_weights(content.size());
-        port::CopyToArray(content,
-                          reinterpret_cast<char*>(temp_weights.data()));
-        int32* dst =
-            static_cast<int32*>(const_cast<void*>(weights.GetValues()));
-        // Copy to the weight store as converted data type.
-        if (dtype == DT_INT16) {
-          int16* data = reinterpret_cast<int16*>(temp_weights.data());
-          std::copy(data, data + tensor.NumElements(), dst);
-        } else if (dtype == DT_INT8) {
-          int8* data = reinterpret_cast<int8*>(temp_weights.data());
-          std::copy(data, data + tensor.NumElements(), dst);
-        } else if (dtype == DT_UINT8) {
-          uint8* data = reinterpret_cast<uint8*>(temp_weights.data());
-          std::copy(data, data + tensor.NumElements(), dst);
-        } else {
-          return errors::FailedPrecondition(
-              "Unexpected data type: ", DataTypeString(dtype),
-              " at: ", node_def.name());
-        }
-      }
-    }
-  } else {
-    return errors::Unimplemented("Not supported constant type, at ",
-                                 node_def.name());
+  TFAttrs attrs(node_def);
+  const DataType dtype = attrs.get<tensorflow::DataType>("dtype");
+  if (dtype != tensor.dtype()) {
+    return errors::InvalidArgument("DataType mismatch between attr (",
+                                   DataTypeString(dtype), ") and tensor (",
+                                   DataTypeString(tensor.dtype()), ")");
   }
+
+  TRT_ShapedWeights weights;
+  TF_RETURN_IF_ERROR(
+      TfTensorToTrtWeights(tensor, params->weight_store, &weights));
+
   if (params->outputs != nullptr) {
     params->outputs->push_back(TRT_TensorOrWeights(weights));
   }
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.h b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
index 4ea5775f04..8495a1a19a 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.h
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
@@ -197,6 +197,10 @@ class TRT_ShapedWeights {
   // underlying buffer.
   TRT_ShapedWeights(DataType type, nvinfer1::Dims dims, Tensor tensor);
 
+  // All weights should be stored inside TrtWeightStore to make sure lifetime of
+  // all the underlying tensors are available until the engine is built. For
+  // this reason, tensor_ should never be reassigned to a different value that
+  // is not already present in the TrtWeightStore.
   Tensor tensor_;
 
   friend class TrtWeightStore;
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
index c8739b46c1..fa6acdb782 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
@@ -1122,6 +1122,30 @@ class OpConverterTest : public ::testing::Test {
   std::unordered_map<string, NodeDef> validator_inputs_;
 };
 
+template <typename T>
+void CopyTensorElements(const Tensor& tensor, proto2::RepeatedField<T>* out) {
+  out->Clear();
+  if (tensor.NumElements() == 0) return;
+
+  // TensorProto does not need to have all the elements present and can truncate
+  // trailing elements with the same value for compressed representation. Such
+  // elements are derived based on the tensor shape.
+  const auto flat = tensor.flat<T>();
+  int64 last_index = 0;
+  for (int64 i = 0; i < tensor.NumElements(); ++i) {
+    if (flat(i) != flat(last_index)) {
+      last_index = i;
+    }
+  }
+
+  int num_out_elements = last_index + 1;
+  out->Reserve(num_out_elements);
+  out->AddNAlreadyReserved(num_out_elements);
+  const T* src = flat.data();
+  T* dst = out->mutable_data();
+  std::copy(src, src + num_out_elements, dst);
+}
+
 template <DataType dtype, typename InputCType, typename OutputCType>
 void TestConvertConst(OpConverterTest* test) {
   NodeDef node_def;
@@ -1134,11 +1158,23 @@ void TestConvertConst(OpConverterTest* test) {
                             const std::vector<OutputCType>& expected_value) {
     test->Reset();
 
-    auto& attr = *node_def.mutable_attr();
+    TensorProto* tensor_attr =
+        (*node_def.mutable_attr())["value"].mutable_tensor();
+    tensor_attr->Clear();
+
     if (as_tensor_content) {
-      tensor.AsProtoTensorContent(attr["value"].mutable_tensor());
+      tensor.AsProtoTensorContent(tensor_attr);
     } else {
-      tensor.AsProtoField(attr["value"].mutable_tensor());
+      tensor.shape().AsProto(tensor_attr->mutable_tensor_shape());
+      tensor_attr->set_dtype(tensor.dtype());
+
+      if (tensor.dtype() == DT_FLOAT) {
+        CopyTensorElements<float>(tensor, tensor_attr->mutable_float_val());
+      } else if (tensor.dtype() == DT_INT32) {
+        CopyTensorElements<int32>(tensor, tensor_attr->mutable_int_val());
+      } else {
+        tensor.AsProtoField(tensor_attr);
+      }
     }
     test->RunValidationAndConversion(node_def);
     TRT_TensorOrWeights output;
@@ -1151,8 +1187,7 @@ void TestConvertConst(OpConverterTest* test) {
   {
     // By default empty tensor will pick DT_FLOAT as data type and we fix it
     // here.
-    attr["value"].mutable_tensor()->set_dtype(dtype);
-    Tensor t;  // Empty tensor.
+    Tensor t(dtype);  // Empty tensor.
     reset_and_test(t, false, {}, {});
   }
   {
@@ -1171,6 +1206,22 @@ void TestConvertConst(OpConverterTest* test) {
     reset_and_test(t, false, {2, 3}, {1, 2, 3, 4, 5, 6});
     reset_and_test(t, true, {2, 3}, {1, 2, 3, 4, 5, 6});
   }
+  {
+    // Set all tensor elements to the same value. Such tensors are encoded
+    // using a single element list in tensor proto.
+    Tensor t = ::tensorflow::test::AsTensor<InputCType>({1, 1, 1, 1, 1, 1},
+                                                        TensorShape({2, 3}));
+    reset_and_test(t, false, {2, 3}, {1, 1, 1, 1, 1, 1});
+    reset_and_test(t, true, {2, 3}, {1, 1, 1, 1, 1, 1});
+  }
+  {
+    // Set trailing tensor elements to the same value. Such tensors are
+    // encoded by truncating all equal elements except the first one.
+    Tensor t = ::tensorflow::test::AsTensor<InputCType>({2, 2, 1, 1, 1, 1},
+                                                        TensorShape({2, 3}));
+    reset_and_test(t, false, {2, 3}, {2, 2, 1, 1, 1, 1});
+    reset_and_test(t, true, {2, 3}, {2, 2, 1, 1, 1, 1});
+  }
 }
 
 TEST_F(OpConverterTest, ConvertConst) {
-- 
GitLab


From 799ec6e59608762d98009db256844ed589b4d6dd Mon Sep 17 00:00:00 2001
From: Reed Wanderman-Milne <reedwm@google.com>
Date: Thu, 17 Jan 2019 17:38:34 -0800
Subject: [PATCH 0944/1095] Fix flaky conv_ops_test

PiperOrigin-RevId: 229849889
---
 tensorflow/python/kernel_tests/BUILD            | 2 --
 tensorflow/python/kernel_tests/conv_ops_test.py | 9 +++++----
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index 40bf4aedd0..aef4354ee0 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -2754,8 +2754,6 @@ cuda_py_test(
     ],
     shard_count = 4,
     tags = [
-        # TODO(b/118887316): Re-enable this test in Kokoro.
-        "no_oss",
         "optonly",  # times out
     ],
     xla_enable_strict_auto_jit = True,
diff --git a/tensorflow/python/kernel_tests/conv_ops_test.py b/tensorflow/python/kernel_tests/conv_ops_test.py
index 7ff1a61e47..390a547fb6 100644
--- a/tensorflow/python/kernel_tests/conv_ops_test.py
+++ b/tensorflow/python/kernel_tests/conv_ops_test.py
@@ -256,7 +256,7 @@ class Conv2DTest(test.TestCase):
       tensors.append(_SetupVal(data_format, use_gpu))
     values = self.evaluate(tensors)
     for i in range(1, len(values)):
-      self.assertAllClose(values[0], values[i], rtol=1e-5, atol=1e-5)
+      self.assertAllClose(values[0], values[i], rtol=1e-3, atol=1e-3)
 
   def _ComputeReferenceDilatedConv(self, tensor_in_sizes, filter_in_sizes,
                                    stride, dilation, padding, data_format,
@@ -297,7 +297,7 @@ class Conv2DTest(test.TestCase):
     return expected, computed
 
   def _VerifyDilatedConvValues(self, tensor_in_sizes, filter_in_sizes, strides,
-                               padding, dilations):
+                               padding, dilations, rtol=1e-4):
     expected_results = []
     computed_results = []
     for data_format, use_gpu in GetTestConfigs():
@@ -313,7 +313,7 @@ class Conv2DTest(test.TestCase):
         tf_logging.debug("expected = %s", e_value)
         tf_logging.debug("actual = %s", c_value)
         self.assertAllClose(
-            e_value.flatten(), c_value.flatten(), atol=tolerance, rtol=1e-4)
+            e_value.flatten(), c_value.flatten(), atol=tolerance, rtol=rtol)
 
   def _VerifyValues(self,
                     tensor_in_sizes,
@@ -2891,7 +2891,8 @@ def GetInceptionFwdDilatedConvTest(input_size, filter_size, stride, padding):
           filter_in_sizes=filter_size,
           strides=[stride, stride],
           dilations=[2, 2],
-          padding=padding)
+          padding=padding,
+          rtol=5e-4)
 
   return Test
 
-- 
GitLab


From 4e8bf7f9ca79e085c37c5d2d3a32c7f409faf999 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 17 Jan 2019 17:40:34 -0800
Subject: [PATCH 0945/1095] Fix for CL 229790895.

PiperOrigin-RevId: 229850165
---
 third_party/gpus/cuda_configure.bzl         | 2 +-
 third_party/tensorrt/tensorrt_configure.bzl | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl
index 7a6c7f4fe5..d1326333d3 100644
--- a/third_party/gpus/cuda_configure.bzl
+++ b/third_party/gpus/cuda_configure.bzl
@@ -1515,7 +1515,7 @@ def _create_remote_cuda_repository(repository_ctx, remote_config_repo):
   )
   repository_ctx.template(
       "crosstool/BUILD",
-      Label(remote_config_repo + "/crosstool/BUILD"),
+      Label(remote_config_repo + "/crosstool:BUILD"),
       {},
   )
 
diff --git a/third_party/tensorrt/tensorrt_configure.bzl b/third_party/tensorrt/tensorrt_configure.bzl
index 10e243fa84..22fd80e91a 100644
--- a/third_party/tensorrt/tensorrt_configure.bzl
+++ b/third_party/tensorrt/tensorrt_configure.bzl
@@ -142,7 +142,7 @@ def _tensorrt_configure_impl(repository_ctx):
   if _TF_TENSORRT_CONFIG_REPO in repository_ctx.os.environ:
     # Forward to the pre-configured remote repository.
     remote_config_repo = repository_ctx.os.environ[_TF_TENSORRT_CONFIG_REPO]
-    repository_ctx.template("BUILD", Label(remote_config_repo + "/BUILD"), {})
+    repository_ctx.template("BUILD", Label(remote_config_repo + ":BUILD"), {})
     # Set up config file.
     _tpl(repository_ctx, "build_defs.bzl", {"%{if_tensorrt}": "if_true"})
     return
-- 
GitLab


From f028498d0d1c2f799a55edf7d47bece78ebdff4d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 17 Jan 2019 18:41:14 -0800
Subject: [PATCH 0946/1095] Fix for CL 229790895.

PiperOrigin-RevId: 229856980
---
 third_party/gpus/cuda_configure.bzl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl
index d1326333d3..5c9873f334 100644
--- a/third_party/gpus/cuda_configure.bzl
+++ b/third_party/gpus/cuda_configure.bzl
@@ -1510,7 +1510,7 @@ def _create_remote_cuda_repository(repository_ctx, remote_config_repo):
   )
   repository_ctx.template(
       "cuda/BUILD",
-      Label(remote_config_repo + "/cuda/BUILD"),
+      Label(remote_config_repo + "/cuda:BUILD"),
       {},
   )
   repository_ctx.template(
-- 
GitLab


From a905c626ca971c4258770b37082b919ba2d9715b Mon Sep 17 00:00:00 2001
From: Reed Wanderman-Milne <reedwm@google.com>
Date: Thu, 17 Jan 2019 18:46:12 -0800
Subject: [PATCH 0947/1095] Disable broken tests

PiperOrigin-RevId: 229857529
---
 tensorflow/contrib/eager/python/examples/revnet/BUILD  | 1 +
 tensorflow/contrib/eager/python/examples/rnn_ptb/BUILD | 1 +
 2 files changed, 2 insertions(+)

diff --git a/tensorflow/contrib/eager/python/examples/revnet/BUILD b/tensorflow/contrib/eager/python/examples/revnet/BUILD
index 0d85bf63ad..d05270089f 100644
--- a/tensorflow/contrib/eager/python/examples/revnet/BUILD
+++ b/tensorflow/contrib/eager/python/examples/revnet/BUILD
@@ -87,6 +87,7 @@ cuda_py_test(
         "//tensorflow:tensorflow_py",
     ],
     tags = [
+        "no_oss",  # b/123045964
         "optonly",
     ],
 )
diff --git a/tensorflow/contrib/eager/python/examples/rnn_ptb/BUILD b/tensorflow/contrib/eager/python/examples/rnn_ptb/BUILD
index f9bf82a7d8..5d48ee4a5f 100644
--- a/tensorflow/contrib/eager/python/examples/rnn_ptb/BUILD
+++ b/tensorflow/contrib/eager/python/examples/rnn_ptb/BUILD
@@ -25,6 +25,7 @@ cuda_py_test(
         "//tensorflow/contrib/eager/python:tfe",
         "//tensorflow:tensorflow_py",
     ],
+    tags = ["no_oss"],  # b/123045964
 )
 
 cuda_py_test(
-- 
GitLab


From c10447d959d07a51a321b622e7a8da6b61768651 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 17 Jan 2019 19:01:05 -0800
Subject: [PATCH 0948/1095] Allow easier re-running of toco on already
 quantized, mixed-bit-depth tflite models, by:   - fixing a bug whereby
 PropagateArrayDataTypes was brutally propagating     the data type of its
 first input onto all outputs;   - not letting QUANTIZED_UINT8 override
 existing non-8bit quantized data     types.

PiperOrigin-RevId: 229858747
---
 .../graph_transformations/identify_lstm.cc    | 22 +++++++++++++++++--
 .../propagate_array_data_types.cc             |  8 +++++++
 tensorflow/lite/toco/toco_tooling.cc          | 17 ++++++++++++++
 3 files changed, 45 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/toco/graph_transformations/identify_lstm.cc b/tensorflow/lite/toco/graph_transformations/identify_lstm.cc
index 089ecee959..65dbb8a176 100644
--- a/tensorflow/lite/toco/graph_transformations/identify_lstm.cc
+++ b/tensorflow/lite/toco/graph_transformations/identify_lstm.cc
@@ -147,12 +147,26 @@ bool MatchOperatorInputs(const Operator& op, const Model& model,
   if (final_output_mul->type != OperatorType::kMul) {
     return ::tensorflow::Status::OK();
   }
+  // final_output_mul->outputs[0] would be one of the two outputs of our
+  // LstmCell. Exit if it does not already have a data type.
+  // We won't be able to propagate data types through a fused LstmCell.
+  if (model->GetArray(final_output_mul->outputs[0]).data_type ==
+      ArrayDataType::kNone) {
+    return ::tensorflow::Status::OK();
+  }
   Operator *state_output_tanh, *fc_output_sig;
   if (!MatchOperatorInputs(*final_output_mul, *model, OperatorType::kTanh,
                            &state_output_tanh, OperatorType::kLogistic,
                            &fc_output_sig)) {
     return ::tensorflow::Status::OK();
   }
+  // state_output_tanh->inputs[0] would be one of the two outputs of our
+  // LstmCell. Exit if it does not already have a data type.
+  // We won't be able to propagate data types through a fused LstmCell.
+  if (model->GetArray(state_output_tanh->inputs[0]).data_type ==
+      ArrayDataType::kNone) {
+    return ::tensorflow::Status::OK();
+  }
 
   // State output TanH
   // (We don't count an operator as ID'd until we verify it has the correct
@@ -262,11 +276,15 @@ bool MatchOperatorInputs(const Operator& op, const Model& model,
       lstm_cell_op->outputs[LstmCellOperator::ACTIV_OUTPUT]));
   const string& concat_temp_array_name =
       AvailableArrayName(*model, base_name + "concat_temp");
-  model->GetOrCreateArray(concat_temp_array_name);
+  auto& concat_temp_array = model->GetOrCreateArray(concat_temp_array_name);
+  concat_temp_array.data_type =
+      model->GetArray(concat_inputs->outputs[0]).data_type;
   lstm_cell_op->outputs[LstmCellOperator::CONCAT_TEMP] = concat_temp_array_name;
   const string& activ_temp_array_name =
       AvailableArrayName(*model, base_name + "activ_temp");
-  model->GetOrCreateArray(activ_temp_array_name);
+  auto& activ_temp_array = model->GetOrCreateArray(activ_temp_array_name);
+  activ_temp_array.data_type =
+      model->GetArray(fully_connected->outputs[0]).data_type;
   lstm_cell_op->outputs[LstmCellOperator::ACTIV_TEMP] = activ_temp_array_name;
   AddMessageF("Created temp outputs %s and %s on operator %s",
               concat_temp_array_name, activ_temp_array_name,
diff --git a/tensorflow/lite/toco/graph_transformations/propagate_array_data_types.cc b/tensorflow/lite/toco/graph_transformations/propagate_array_data_types.cc
index 7aec6728da..d546ff4ec0 100644
--- a/tensorflow/lite/toco/graph_transformations/propagate_array_data_types.cc
+++ b/tensorflow/lite/toco/graph_transformations/propagate_array_data_types.cc
@@ -266,6 +266,14 @@ void SetDataTypeForAllOutputs(Model* model, Operator* op,
       model->GetArray(op->outputs[1]).data_type = unique_op->idx_out_type;
       break;
     }
+    case OperatorType::kLstmCell: {
+      // It's tricky to propagate data types through a LstmCell, as that has
+      // multiple inputs and outputs, and there are quantized cases with
+      // mixed (8bit vs 16bit) cases. Fortunately, that should never be needed,
+      // as the data formats, such as TFLITE, that have LstmCell nodes, also
+      // have data type fields for all their arrays.
+      break;
+    }
     default: {
       // These operators produce outputs with the same type as their 1st input
       CHECK_GT(op->inputs.size(), 0);
diff --git a/tensorflow/lite/toco/toco_tooling.cc b/tensorflow/lite/toco/toco_tooling.cc
index 55a454e66d..61dca56d23 100644
--- a/tensorflow/lite/toco/toco_tooling.cc
+++ b/tensorflow/lite/toco/toco_tooling.cc
@@ -178,6 +178,23 @@ void SetFinalDataTypeOnInputs(const TocoFlags& toco_flags, Model* model) {
       // Ignore non-real data types.
       continue;
     }
+    // The enum value QUANTIZED_UINT8 for --inference_type and
+    // --inference_input_type has long meant just 'QUANTIZED', being used as
+    // well in mixed 8-bit / 16-bit quantized models. However,
+    // ConvertIODataTypeToArrayDataType still interpretes it as meaning 8bit,
+    // and people have run into issues in the situation where they have an
+    // already mixed 8-bit / 16-bit quantized model in TFLITE format and
+    // want to run it again through toco, without having to re-specify all the
+    // extra array info that was used in the (complicated) process of initially
+    // quantizing that model. In order to have --inference_type=QUANTIZED_UINT8
+    // just work in that case, we implement the logic that when an array is
+    // already quantized, if  --inference_type is quantized (so we're not
+    // asking to dequantize here), no change of quantized data type is to be
+    // recorded.
+    if (array->data_type != toco::ArrayDataType::kFloat &&
+        type != toco::ArrayDataType::kFloat) {
+      continue;
+    }
 
     array->final_data_type = type;
   }
-- 
GitLab


From 2a5abdb77806e06ce7e2820ede6233ca79cf5625 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 17 Jan 2019 19:02:03 -0800
Subject: [PATCH 0949/1095] Nit: Add a missing "\" in lite benchmark
 documentation

PiperOrigin-RevId: 229858857
---
 tensorflow/lite/tools/benchmark/android/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/lite/tools/benchmark/android/README.md b/tensorflow/lite/tools/benchmark/android/README.md
index f5b67e3f79..db82c59acd 100644
--- a/tensorflow/lite/tools/benchmark/android/README.md
+++ b/tensorflow/lite/tools/benchmark/android/README.md
@@ -51,7 +51,7 @@ and can be appended to the `args` string alongside the required `--graph` flag
 args key).
 
 ```
-adb shell am start -S -n
+adb shell am start -S -n \
   org.tensorflow.lite.benchmark/org.tensorflow.lite.benchmark.BenchmarkModelActivity \
   --es args '"--graph=/data/local/tmp/mobilenet_quant_v1_224.tflite --num_threads=4"'
 ```
-- 
GitLab


From 79aeb13673a9107e76622fbc3181a92b398bee28 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 17 Jan 2019 19:16:38 -0800
Subject: [PATCH 0950/1095] Automated rollback of commit
 f132194cf910343d4e1c3aedfad45f7b76fc5f61

PiperOrigin-RevId: 229860333
---
 tensorflow/python/BUILD                 |  1 -
 tensorflow/python/client/session_ref.cc | 31 ++++---------------------
 2 files changed, 4 insertions(+), 28 deletions(-)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index e6f8201fc7..30de1fa440 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -4091,7 +4091,6 @@ cc_library(
         "//tensorflow/core:master_proto_cc",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:replay_log_proto_cc",
-        "@com_google_absl//absl/time",
     ],
 )
 
diff --git a/tensorflow/python/client/session_ref.cc b/tensorflow/python/client/session_ref.cc
index 895624f592..6639cf506e 100644
--- a/tensorflow/python/client/session_ref.cc
+++ b/tensorflow/python/client/session_ref.cc
@@ -18,9 +18,6 @@ limitations under the License.
 #include <memory>
 #include <utility>
 
-#include "absl/time/clock.h"
-#include "absl/time/time.h"
-#include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/io/path.h"
 #include "tensorflow/core/lib/io/record_writer.h"
 #include "tensorflow/core/lib/strings/stringprintf.h"
@@ -480,8 +477,6 @@ Status SessionRef::ReleaseCallable(CallableHandle handle) {
   LOG_AND_RUN_OPERATION(ReleaseCallable, handle);
 }
 
-static const absl::Duration kMaxCloseWaitTime = absl::Seconds(60);
-
 Status SessionRef::Close(const RunOptions& run_options) {
   TF_RETURN_IF_ERROR(CheckNotClosed());
   mutex_lock l(run_lock_);
@@ -492,17 +487,8 @@ Status SessionRef::Close(const RunOptions& run_options) {
     status = session_->Close(run_options);
   }
   session_.reset();
-
-  // Wait no more than kMaxCloseWaitTime for all pending operations to finish.
-  absl::Time start_time = absl::Now();
-  absl::Duration wait_time = start_time + kMaxCloseWaitTime - absl::Now();
-  while (run_count_ > 0 && wait_time > absl::ZeroDuration()) {
-    if (run_finished_.wait_for(l, absl::ToChronoMilliseconds(wait_time)) ==
-        std::cv_status::timeout) {
-      status.Update(errors::DeadlineExceeded("Timeout closing session."));
-      return status;
-    }
-    wait_time = start_time + kMaxCloseWaitTime - absl::Now();
+  while (run_count_ > 0) {
+    run_finished_.wait(l);
   }
   return status;
 }
@@ -517,17 +503,8 @@ Status SessionRef::Close() {
     status = session_->Close();
   }
   session_.reset();
-
-  // Wait no more than kMaxCloseWaitTime for all pending operations to finish.
-  absl::Time start_time = absl::Now();
-  absl::Duration wait_time = start_time + kMaxCloseWaitTime - absl::Now();
-  while (run_count_ > 0 && wait_time > absl::ZeroDuration()) {
-    if (run_finished_.wait_for(l, absl::ToChronoMilliseconds(wait_time)) ==
-        std::cv_status::timeout) {
-      status.Update(errors::DeadlineExceeded("Timeout closing session."));
-      return status;
-    }
-    wait_time = start_time + kMaxCloseWaitTime - absl::Now();
+  while (run_count_ > 0) {
+    run_finished_.wait(l);
   }
   return status;
 }
-- 
GitLab


From a33ceb20cfa9af339e79ce814d3eb849b68e8398 Mon Sep 17 00:00:00 2001
From: Eugene Brevdo <ebrevdo@google.com>
Date: Thu, 17 Jan 2019 19:38:57 -0800
Subject: [PATCH 0951/1095] Keras Dense layer dtype changes.

* Dense layer now explicitly requires floating point inputs (better error messages than before).
* Dense layer call() now casts inputs to self.dtype.

PiperOrigin-RevId: 229862412
---
 tensorflow/python/keras/engine/base_layer.py | 2 ++
 tensorflow/python/keras/layers/core.py       | 6 ++++++
 tensorflow/python/keras/layers/core_test.py  | 8 ++++++++
 3 files changed, 16 insertions(+)

diff --git a/tensorflow/python/keras/engine/base_layer.py b/tensorflow/python/keras/engine/base_layer.py
index 7bfc301239..f63fb4e694 100644
--- a/tensorflow/python/keras/engine/base_layer.py
+++ b/tensorflow/python/keras/engine/base_layer.py
@@ -307,6 +307,8 @@ class Layer(checkpointable.Checkpointable):
     if dtype is None:
       dtype = self.dtype or backend.floatx()
     dtype = dtypes.as_dtype(dtype)
+    if self._dtype is None:
+      self._dtype = dtype.base_dtype.name
     initializer = initializers.get(initializer)
     regularizer = regularizers.get(regularizer)
     constraint = constraints.get(constraint)
diff --git a/tensorflow/python/keras/layers/core.py b/tensorflow/python/keras/layers/core.py
index dfbab80be3..fdc0a76877 100644
--- a/tensorflow/python/keras/layers/core.py
+++ b/tensorflow/python/keras/layers/core.py
@@ -27,6 +27,7 @@ import numpy as np
 
 from tensorflow.python.eager import context
 from tensorflow.python.framework import common_shapes
+from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.keras import activations
@@ -932,6 +933,10 @@ class Dense(Layer):
     self.input_spec = InputSpec(min_ndim=2)
 
   def build(self, input_shape):
+    dtype = dtypes.as_dtype(self.dtype or K.floatx())
+    if not (dtype.is_floating or dtype.is_complex):
+      raise TypeError('Unable to build `Dense` layer with non-floating point '
+                      'dtype %s' % (dtype,))
     input_shape = tensor_shape.TensorShape(input_shape)
     if tensor_shape.dimension_value(input_shape[-1]) is None:
       raise ValueError('The last dimension of the inputs to `Dense` '
@@ -972,6 +977,7 @@ class Dense(Layer):
         output_shape = shape[:-1] + [self.units]
         outputs.set_shape(output_shape)
     else:
+      inputs = math_ops.cast(inputs, self.dtype)
       outputs = gen_math_ops.mat_mul(inputs, self.kernel)
     if self.use_bias:
       outputs = nn.bias_add(outputs, self.bias)
diff --git a/tensorflow/python/keras/layers/core_test.py b/tensorflow/python/keras/layers/core_test.py
index 9df40f806f..d61c8e3d08 100644
--- a/tensorflow/python/keras/layers/core_test.py
+++ b/tensorflow/python/keras/layers/core_test.py
@@ -22,6 +22,7 @@ import numpy as np
 
 from tensorflow.python import keras
 from tensorflow.python.eager import context
+from tensorflow.python.framework import ops
 from tensorflow.python.keras import keras_parameterized
 from tensorflow.python.keras import testing_utils
 from tensorflow.python.ops import math_ops
@@ -283,6 +284,13 @@ class CoreLayersTest(keras_parameterized.TestCase):
     testing_utils.layer_test(
         keras.layers.Dense, kwargs={'units': 3}, input_shape=(3, 4, 5, 2))
 
+  def test_dense_dtype(self):
+    inputs = ops.convert_to_tensor(
+        np.random.randint(low=0, high=7, size=(2, 2)))
+    layer = keras.layers.Dense(5, dtype='float32')
+    outputs = layer(inputs)
+    self.assertEqual(outputs.dtype, 'float32')
+
   def test_dense_regularization(self):
     layer = keras.layers.Dense(
         3,
-- 
GitLab


From 0be3b45806074bc386d3e72113dc48f2a332abe6 Mon Sep 17 00:00:00 2001
From: Fei Hu <hufei68@gmail.com>
Date: Thu, 17 Jan 2019 20:14:42 -0800
Subject: [PATCH 0952/1095] Update the code and remove the proto header

---
 tensorflow/core/kernels/data/dataset_utils.cc |  32 +-----
 tensorflow/core/kernels/data/dataset_utils.h  |  17 +--
 .../core/kernels/data/dataset_utils_test.cc   |  20 +---
 tensorflow/core/kernels/data/iterator_ops.cc  | 104 ------------------
 4 files changed, 16 insertions(+), 157 deletions(-)

diff --git a/tensorflow/core/kernels/data/dataset_utils.cc b/tensorflow/core/kernels/data/dataset_utils.cc
index 67a0b1165a..8f3b1f7066 100644
--- a/tensorflow/core/kernels/data/dataset_utils.cc
+++ b/tensorflow/core/kernels/data/dataset_utils.cc
@@ -159,26 +159,6 @@ bool VariantTensorDataReader::Contains(StringPiece key) {
   return map_.find(string(key)) != map_.end();
 }
 
-void VariantTensorDataReader::PreProcess() {
-  string metadata;
-  data_->get_metadata(&metadata);
-  IteratorStateMetadata proto;
-  if (!proto.ParseFromString(metadata)) {
-    status_ = errors::Internal("Error parsing IteratorStateMetadata.");
-    return;
-  }
-  size_t num_entries = proto.keys_size();
-  if (num_entries != data_->tensors_size()) {
-    status_ =
-        errors::InvalidArgument("Unmatched number of keys and tensors: ",
-                                num_entries, " vs. ", data_->tensors_size());
-    return;
-  }
-  for (size_t i = 0; i < num_entries; i++) {
-    map_[proto.keys(i)] = i;
-  }
-}
-
 template <typename T>
 Status VariantTensorDataReader::ReadScalarInternal(StringPiece key, T* val) {
   if (map_.find(string(key)) == map_.end()) {
@@ -213,8 +193,8 @@ Status VariantTensorDataWriter::WriteTensor(StringPiece key,
 
 Status VariantTensorDataWriter::Flush() {
   string metadata;
-  if (!metadata_proto_.SerializeToString(&metadata)) {
-    return errors::Internal("Unable to serialize IteratorStateMetadata.");
+  for (size_t i = 0; i < keys_.size(); ++i) {
+    strings::StrAppend(&metadata, kDelimiter, keys_[i]);
   }
   data_->set_metadata(metadata);
   return Status::OK();
@@ -230,12 +210,8 @@ Status VariantTensorDataWriter::WriteScalarInternal(StringPiece key,
 
 Status VariantTensorDataWriter::WriteTensorInternal(StringPiece key,
                                                     const Tensor& val) {
-  // Write key to the metadata proto. This gets written to `data_`
-  // when `Flush()` is called. We do this lazily to avoid multiple
-  // serialization calls.
-  metadata_proto_.add_keys(string(key));
-
-  // Update tensors.
+  DCHECK_EQ(key.find(kDelimiter), string::npos);
+  keys_.push_back(string(key));
   *(data_->add_tensors()) = val;
   return Status::OK();
 }
diff --git a/tensorflow/core/kernels/data/dataset_utils.h b/tensorflow/core/kernels/data/dataset_utils.h
index b2a19ac869..b960671594 100644
--- a/tensorflow/core/kernels/data/dataset_utils.h
+++ b/tensorflow/core/kernels/data/dataset_utils.h
@@ -16,7 +16,6 @@ limitations under the License.
 #define TENSORFLOW_CORE_KERNELS_DATA_DATASET_UTILS_H_
 
 #include "tensorflow/core/framework/dataset.h"
-#include "tensorflow/core/framework/iterator.pb.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/kernels/data/captured_function.h"
 
@@ -58,16 +57,22 @@ Status VerifyTypesMatch(const DataTypeVector& expected,
 Status VerifyShapesCompatible(const std::vector<PartialTensorShape>& expected,
                               const std::vector<PartialTensorShape>& received);
 
+constexpr char kDelimiter[] = "@@";
+
 // Helper class for reading data from a VariantTensorData object.
 class VariantTensorDataReader : public IteratorStateReader {
  public:
   explicit VariantTensorDataReader(const VariantTensorData* data)
       : data_(data) {
-    PreProcess();
+    string metadata;
+    data_->get_metadata(&metadata);
+    auto keys = str_util::Split(metadata, kDelimiter, str_util::SkipEmpty());
+    for (size_t i = 0; i < keys.size(); ++i) {
+      map_[keys[i]] = i;
+    }
   }
 
-  // Returns OK iff the initialization was successful, i.e.,
-  // pre-processing did not have errors.
+  // Returns OK iff the initialization was successful.
   Status status() const;
   Status ReadScalar(StringPiece key, int64* val) override;
   Status ReadScalar(StringPiece key, string* val) override;
@@ -75,7 +80,6 @@ class VariantTensorDataReader : public IteratorStateReader {
   bool Contains(StringPiece key) override;
 
  private:
-  void PreProcess();
   template <typename T>
   Status ReadScalarInternal(StringPiece key, T* val);
   Status ReadTensorInternal(StringPiece key, Tensor* val);
@@ -102,8 +106,7 @@ class VariantTensorDataWriter : public IteratorStateWriter {
   Status WriteTensorInternal(StringPiece key, const Tensor& val);
   VariantTensorData* data_;
 
-  // TODO(srbs): Set the version string.
-  IteratorStateMetadata metadata_proto_;
+  std::vector<string> keys_;
 };
 
 }  // namespace data
diff --git a/tensorflow/core/kernels/data/dataset_utils_test.cc b/tensorflow/core/kernels/data/dataset_utils_test.cc
index b503e58c1b..ba81760fc0 100644
--- a/tensorflow/core/kernels/data/dataset_utils_test.cc
+++ b/tensorflow/core/kernels/data/dataset_utils_test.cc
@@ -46,7 +46,6 @@ TEST(DatasetUtilsTest, VariantTensorDataRoundtrip) {
   VariantTensorData data;
   VariantTensorDataWriter writer(&data);
   TF_ASSERT_OK(writer.WriteScalar("Int64", 24));
-  TF_ASSERT_OK(writer.WriteScalar("", "Empty_Key"));
   Tensor input_tensor(DT_FLOAT, {1});
   input_tensor.flat<float>()(0) = 2.0f;
   TF_ASSERT_OK(writer.WriteTensor("Tensor", input_tensor));
@@ -57,9 +56,6 @@ TEST(DatasetUtilsTest, VariantTensorDataRoundtrip) {
   int64 val_int64;
   TF_ASSERT_OK(reader.ReadScalar("Int64", &val_int64));
   EXPECT_EQ(val_int64, 24);
-  string val_string;
-  TF_ASSERT_OK(reader.ReadScalar("", &val_string));
-  EXPECT_EQ(val_string, "Empty_Key");
   Tensor val_tensor;
   TF_ASSERT_OK(reader.ReadTensor("Tensor", &val_tensor));
   EXPECT_EQ(input_tensor.NumElements(), val_tensor.NumElements());
@@ -68,6 +64,8 @@ TEST(DatasetUtilsTest, VariantTensorDataRoundtrip) {
 
 TEST(DatasetUtilsTest, VariantTensorDataNonExistentKey) {
   VariantTensorData data;
+  strings::StrAppend(&data.metadata_, "key1", "@@");
+  data.tensors_.push_back(Tensor(DT_INT64, {1}));
   VariantTensorDataReader reader(&data);
   TF_ASSERT_OK(reader.status());
   int64 val_int64;
@@ -81,20 +79,6 @@ TEST(DatasetUtilsTest, VariantTensorDataNonExistentKey) {
             reader.ReadTensor("NonExistentKey", &val_tensor).code());
 }
 
-TEST(DatasetUtilsTest, VariantTensorDataInvalidMetadata) {
-  VariantTensorData data;
-  data.metadata_ = "Invalid Metadata";
-  VariantTensorDataReader reader(&data);
-  EXPECT_EQ(error::INTERNAL, reader.status().code());
-}
-
-TEST(DatasetUtilsTest, VariantTensorDataUnmatchedKeys) {
-  VariantTensorData data;
-  data.tensors_.push_back(Tensor(DT_INT64, {1}));
-  VariantTensorDataReader reader(&data);
-  EXPECT_EQ(error::INVALID_ARGUMENT, reader.status().code());
-}
-
 }  // namespace
 }  // namespace data
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/data/iterator_ops.cc b/tensorflow/core/kernels/data/iterator_ops.cc
index da497a5f72..8f64639a8b 100644
--- a/tensorflow/core/kernels/data/iterator_ops.cc
+++ b/tensorflow/core/kernels/data/iterator_ops.cc
@@ -276,110 +276,6 @@ class IteratorResource : public ResourceBase {
 
 namespace {
 
-constexpr char kDelimiter[] = "@@";
-
-// Helper class for reading data from a VariantTensorData object.
-class VariantTensorDataReader : public IteratorStateReader {
- public:
-  explicit VariantTensorDataReader(const VariantTensorData* data)
-      : data_(data) {
-    string metadata;
-    data_->get_metadata(&metadata);
-    auto keys = str_util::Split(metadata, kDelimiter, str_util::SkipEmpty());
-    for (size_t i = 0; i < keys.size(); ++i) {
-      map_[keys[i]] = i;
-    }
-  }
-
-  // Returns OK iff the initialization was successful, i.e.,
-  // pre-processing did not have errors.
-  Status status() const { return status_; }
-
-  Status ReadScalar(StringPiece key, int64* val) override {
-    return ReadScalarInternal(key, val);
-  }
-
-  Status ReadScalar(StringPiece key, string* val) override {
-    return ReadScalarInternal(key, val);
-  }
-
-  Status ReadTensor(StringPiece key, Tensor* val) override {
-    return ReadTensorInternal(key, val);
-  }
-
-  bool Contains(StringPiece key) override {
-    return map_.find(string(key)) != map_.end();
-  }
-
- private:
-  template <typename T>
-  Status ReadScalarInternal(StringPiece key, T* val) {
-    if (map_.find(string(key)) == map_.end()) {
-      return errors::NotFound(key);
-    }
-    *val = data_->tensors(map_[string(key)]).scalar<T>()();
-    return Status::OK();
-  }
-
-  Status ReadTensorInternal(StringPiece key, Tensor* val) {
-    if (map_.find(string(key)) == map_.end()) {
-      return errors::NotFound(key);
-    }
-    *val = data_->tensors(map_[string(key)]);
-    return Status::OK();
-  }
-
-  std::map<string, size_t> map_;
-  const VariantTensorData* data_;  // Not owned.
-  Status status_;
-};
-
-// Helper class for writing data to a VariantTensorData object.
-class VariantTensorDataWriter : public IteratorStateWriter {
- public:
-  // Does not take ownership of data.
-  explicit VariantTensorDataWriter(VariantTensorData* data) : data_(data) {}
-
-  Status WriteScalar(StringPiece key, const int64 val) override {
-    return WriteScalarInternal(key, val);
-  }
-
-  Status WriteScalar(StringPiece key, const string& val) override {
-    return WriteScalarInternal(key, val);
-  }
-
-  Status WriteTensor(StringPiece key, const Tensor& val) override {
-    return WriteTensorInternal(key, val);
-  }
-
-  Status Flush() {
-    string metadata;
-    for (size_t i = 0; i < keys_.size(); ++i) {
-      strings::StrAppend(&metadata, kDelimiter, keys_[i]);
-    }
-    data_->set_metadata(metadata);
-    return Status::OK();
-  }
-
- private:
-  template <typename T>
-  Status WriteScalarInternal(StringPiece key, const T& val) {
-    Tensor val_t = Tensor(DataTypeToEnum<T>::v(), TensorShape({}));
-    val_t.scalar<T>()() = val;
-    return WriteTensorInternal(key, val_t);
-  }
-
-  Status WriteTensorInternal(StringPiece key, const Tensor& val) {
-    DCHECK_EQ(key.find(kDelimiter), string::npos);
-    keys_.push_back(string(key));
-    *(data_->add_tensors()) = val;
-    return Status::OK();
-  }
-
-  VariantTensorData* data_;
-  std::vector<string> keys_;
-};
-
 // Wrapper for encoding/decoding the iterator state stored in a Variant tensor.
 // The get() method returns an IteratorStateReader which can be used
 // to restore iterator state.
-- 
GitLab


From a45e7253c19ec04cbbc116ceda4052a018d9cc09 Mon Sep 17 00:00:00 2001
From: Blake Hechtman <blakehechtman@google.com>
Date: Thu, 17 Jan 2019 21:16:13 -0800
Subject: [PATCH 0953/1095] [XLA] Support no dimension numbers set in Dot.

PiperOrigin-RevId: 229870615
---
 tensorflow/compiler/xla/service/BUILD         |  1 +
 .../compiler/xla/service/dot_decomposer.cc    |  5 ++
 .../compiler/xla/service/hlo_cost_analysis.cc | 21 ++---
 .../xla/service/hlo_cost_analysis_test.cc     | 82 +++++++++++++++++++
 .../xla/service/hlo_evaluator_typed_visitor.h | 18 ++--
 .../compiler/xla/tests/dot_operation_test.cc  | 30 +++++++
 6 files changed, 137 insertions(+), 20 deletions(-)

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index e39e17c110..2b2c32a681 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -2105,6 +2105,7 @@ tf_cc_test(
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:statusor",
         "//tensorflow/compiler/xla:test_helpers",
+        "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/client",
         "//tensorflow/compiler/xla/client:client_library",
         "//tensorflow/compiler/xla/client:local_client",
diff --git a/tensorflow/compiler/xla/service/dot_decomposer.cc b/tensorflow/compiler/xla/service/dot_decomposer.cc
index 855424067d..e8bc6d0571 100644
--- a/tensorflow/compiler/xla/service/dot_decomposer.cc
+++ b/tensorflow/compiler/xla/service/dot_decomposer.cc
@@ -301,6 +301,11 @@ StatusOr<bool> DotDecomposer::Run(HloModule* module) {
         non_canonical_dots.push_back(instruction);
         continue;
       }
+      if (dnums.lhs_batch_dimensions().empty() &&
+          dnums.lhs_contracting_dimensions().empty()) {
+        non_canonical_dots.push_back(instruction);
+        continue;
+      }
       if (dnums.lhs_batch_dimensions().empty()) {
         continue;
       }
diff --git a/tensorflow/compiler/xla/service/hlo_cost_analysis.cc b/tensorflow/compiler/xla/service/hlo_cost_analysis.cc
index c4b4fa62dd..76fd402b2c 100644
--- a/tensorflow/compiler/xla/service/hlo_cost_analysis.cc
+++ b/tensorflow/compiler/xla/service/hlo_cost_analysis.cc
@@ -237,24 +237,17 @@ Status HloCostAnalysis::HandleDomain(const HloInstruction* domain) {
 
 Status HloCostAnalysis::HandleDot(const HloInstruction* dot) {
   const Shape& lhs_shape = dot->operand(0)->shape();
-  const Shape& rhs_shape = dot->operand(1)->shape();
+  const Shape& dot_shape = dot->shape();
   const DotDimensionNumbers& dnums = dot->dot_dimension_numbers();
   // Count of elements along the reduction dimension (last dimension for the
   // rhs).
-  int64 reduction_width =
-      lhs_shape.dimensions(dnums.lhs_contracting_dimensions(0));
-  // First divide by reduction width before multiplying by rhs elements to avoid
-  // overflow.
-  int64 fma_count;
-  if (reduction_width == 0) {
-    fma_count = 0;
-  } else {
-    fma_count = (ShapeUtil::ElementsIn(lhs_shape) / reduction_width) *
-                ShapeUtil::ElementsIn(rhs_shape);
+  int64 reduction_width = 1;
+  for (auto dim : dnums.lhs_contracting_dimensions()) {
+    reduction_width *= lhs_shape.dimensions(dim);
   }
-
-  // We count an FMA operation as 2 floating point operations.
-  current_properties_[kFlopsKey] = kFmaFlops * fma_count;
+  // Each output elment requires reduction_widht FMA operations.
+  current_properties_[kFlopsKey] =
+      kFmaFlops * ShapeUtil::ElementsIn(dot_shape) * reduction_width;
   return Status::OK();
 }
 
diff --git a/tensorflow/compiler/xla/service/hlo_cost_analysis_test.cc b/tensorflow/compiler/xla/service/hlo_cost_analysis_test.cc
index 82dd57d3c6..4d42770ba7 100644
--- a/tensorflow/compiler/xla/service/hlo_cost_analysis_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_cost_analysis_test.cc
@@ -29,6 +29,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/service.h"
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/tests/hlo_test_base.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/core/platform/logging.h"
 
 #include "tensorflow/compiler/xla/statusor.h"
@@ -157,6 +158,87 @@ TEST_F(HloCostAnalysisTest, MatrixMultiply) {
             sizeof(float) * (10 * 5 + 5 * 30 + 10 * 30));
 }
 
+TEST_F(HloCostAnalysisTest, DotGeneral) {
+  XlaBuilder builder("matrix_multiply");
+  auto lhs =
+      Parameter(&builder, 0, ShapeUtil::MakeShape(F32, {10, 5, 5}), "lhs");
+  auto rhs =
+      Parameter(&builder, 1, ShapeUtil::MakeShape(F32, {5, 5, 30}), "rhs");
+  DotDimensionNumbers dnums;
+  dnums.add_lhs_contracting_dimensions(1);
+  dnums.add_lhs_contracting_dimensions(2);
+  dnums.add_rhs_contracting_dimensions(0);
+  dnums.add_rhs_contracting_dimensions(1);
+  DotGeneral(lhs, rhs, dnums);
+
+  // Run HLO cost analysis.
+  auto hlo_module = BuildHloGraph(&builder);
+  HloCostAnalysis analysis(ShapeSize);
+  ASSERT_IS_OK(
+      hlo_module->entry_computation()->root_instruction()->Accept(&analysis));
+
+  // Check the number of computations returned from the analysis (1500 FMAs).
+  EXPECT_EQ(analysis.flop_count(), 2 * 10 * 30 * 5 * 5);
+
+  EXPECT_EQ(analysis.transcendental_count(), 0);
+
+  // Bytes accessed is sum of inputs and output.
+  EXPECT_EQ(analysis.bytes_accessed(),
+            sizeof(float) * (10 * 5 * 5 + 5 * 5 * 30 + 10 * 30));
+}
+
+TEST_F(HloCostAnalysisTest, DotGeneral2) {
+  XlaBuilder builder("matrix_multiply");
+  auto lhs =
+      Parameter(&builder, 0, ShapeUtil::MakeShape(F32, {10, 5, 5}), "lhs");
+  auto rhs =
+      Parameter(&builder, 1, ShapeUtil::MakeShape(F32, {5, 5, 30}), "rhs");
+  DotDimensionNumbers dnums;
+  dnums.add_lhs_contracting_dimensions(1);
+  dnums.add_lhs_batch_dimensions(2);
+  dnums.add_rhs_contracting_dimensions(0);
+  dnums.add_rhs_batch_dimensions(1);
+  DotGeneral(lhs, rhs, dnums);
+
+  // Run HLO cost analysis.
+  auto hlo_module = BuildHloGraph(&builder);
+  HloCostAnalysis analysis(ShapeSize);
+  ASSERT_IS_OK(
+      hlo_module->entry_computation()->root_instruction()->Accept(&analysis));
+
+  // Check the number of computations returned from the analysis (1500 FMAs).
+  EXPECT_EQ(analysis.flop_count(), 2 * 10 * 30 * 5 * 5);
+
+  EXPECT_EQ(analysis.transcendental_count(), 0);
+
+  // Bytes accessed is sum of inputs and output.
+  EXPECT_EQ(analysis.bytes_accessed(),
+            sizeof(float) * (10 * 5 * 5 + 5 * 5 * 30 + 5 * 10 * 30));
+}
+
+TEST_F(HloCostAnalysisTest, DotGeneral3) {
+  XlaBuilder builder("matrix_multiply");
+  auto lhs = Parameter(&builder, 0, ShapeUtil::MakeShape(F32, {10, 5}), "lhs");
+  auto rhs = Parameter(&builder, 1, ShapeUtil::MakeShape(F32, {5, 30}), "rhs");
+  DotDimensionNumbers dnums;
+  DotGeneral(lhs, rhs, dnums);
+
+  // Run HLO cost analysis.
+  auto hlo_module = BuildHloGraph(&builder);
+  HloCostAnalysis analysis(ShapeSize);
+  ASSERT_IS_OK(
+      hlo_module->entry_computation()->root_instruction()->Accept(&analysis));
+
+  // Check the number of computations returned from the analysis (1500 FMAs).
+  EXPECT_EQ(analysis.flop_count(), 2 * 10 * 30 * 5 * 5);
+
+  EXPECT_EQ(analysis.transcendental_count(), 0);
+
+  // Bytes accessed is sum of inputs and output.
+  EXPECT_EQ(analysis.bytes_accessed(),
+            sizeof(float) * (10 * 5 + 5 * 30 + 5 * 5 * 10 * 30));
+}
+
 TEST_F(HloCostAnalysisTest, Map) {
   XlaBuilder builder("map");
   auto input = Parameter(&builder, 0, ShapeUtil::MakeShape(F32, {10}), "in");
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
index 95a78408b0..648c7d0e67 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
@@ -1187,7 +1187,8 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
   }
 
   Status HandleDot(HloInstruction* dot) override {
-    if (parent_->use_fast_path_) {
+    if (dot->dot_dimension_numbers().rhs_contracting_dimensions_size() == 1 &&
+        parent_->use_fast_path_) {
       return HandleDot<ReturnT>(dot);
     }
     return HandleDotSlowPath(dot);
@@ -1349,12 +1350,17 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
                 static_cast<ElementwiseT>(lhs_literal.Get<ReturnT>(lhs_index)) *
                 static_cast<ElementwiseT>(rhs_literal.Get<ReturnT>(rhs_index));
 
-            for (int64 i = accumulate_index_sizes.size() - 1; i >= 0; --i) {
-              int64 value = ++accumulate_index[i];
-              if (value != accumulate_index_sizes[i]) {
-                break;
+            // If there are no contracting dimension accumulate_index_sizes is
+            // empty, do not try to count down from -1 to 0 since it is and
+            // infinite loop.
+            if (!accumulate_index_sizes.empty()) {
+              for (int64 i = accumulate_index_sizes.size() - 1; i >= 0; --i) {
+                int64 value = ++accumulate_index[i];
+                if (value != accumulate_index_sizes[i]) {
+                  break;
+                }
+                accumulate_index[i] = 0;
               }
-              accumulate_index[i] = 0;
             }
           }
 
diff --git a/tensorflow/compiler/xla/tests/dot_operation_test.cc b/tensorflow/compiler/xla/tests/dot_operation_test.cc
index 33d816292c..4576a04ec3 100644
--- a/tensorflow/compiler/xla/tests/dot_operation_test.cc
+++ b/tensorflow/compiler/xla/tests/dot_operation_test.cc
@@ -1189,5 +1189,35 @@ ENTRY %test {
   EXPECT_TRUE(RunAndCompare(hlo_string, ErrorSpec{1e-3, 1e-3}));
 }
 
+XLA_TEST_F(DotOperationTextTest, DotWithNoDnums) {
+  absl::string_view hlo_string =
+      R"(
+HloModule DotWithNoDnums
+
+ENTRY %test {
+  %lhs = f32[2,3]{1,0} parameter(0)
+  %rhs = f32[4,5]{1,0} parameter(1)
+  ROOT %dot = f32[2,3,4,5]{3,2,1,0} dot(%lhs, %rhs)
+}
+)";
+
+  EXPECT_TRUE(RunAndCompare(hlo_string, ErrorSpec{1e-3, 1e-3}));
+}
+
+XLA_TEST_F(DotOperationTextTest, Einsum) {
+  absl::string_view hlo_string =
+      R"(
+HloModule Einsum
+
+ENTRY %test {
+  %lhs = f32[8,64,96]{2,1,0} parameter(0)
+  %rhs = f32[96,32,4]{2,1,0} parameter(1)
+  ROOT %dot = f32[8,64,32,4]{3,2,1,0}  dot(%lhs, %rhs), lhs_contracting_dims={2}, rhs_contracting_dims={0}
+}
+)";
+
+  EXPECT_TRUE(RunAndCompare(hlo_string, ErrorSpec{4e-3, 4e-3}));
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From a3335403457d1005f5bd556f740a2dd4324edf60 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 17 Jan 2019 23:00:29 -0800
Subject: [PATCH 0954/1095] Split ragged_array_ops.py into multiple targets to
 avoid dependency cycles. Specifically, refactor out ragged gather,
 batch_gather, where, concat and stack ops into their own file to avoid
 dependency cycles.

PiperOrigin-RevId: 229878011
---
 tensorflow/python/ops/ragged/BUILD            |  89 ++-
 tensorflow/python/ops/ragged/__init__.py      |   4 +
 .../python/ops/ragged/ragged_array_ops.py     | 710 ------------------
 .../ops/ragged/ragged_batch_gather_op_test.py |  12 +-
 .../ops/ragged/ragged_batch_gather_ops.py     | 120 +++
 .../ops/ragged/ragged_concat_op_test.py       |  12 +-
 .../python/ops/ragged/ragged_concat_ops.py    | 302 ++++++++
 .../python/ops/ragged/ragged_dispatch.py      |  18 +-
 .../ops/ragged/ragged_gather_nd_op_test.py    |  12 +-
 .../ops/ragged/ragged_gather_op_test.py       |  33 +-
 .../python/ops/ragged/ragged_gather_ops.py    | 258 +++++++
 .../python/ops/ragged/ragged_getitem.py       |   4 +-
 .../python/ops/ragged/ragged_stack_op_test.py |  10 +-
 .../python/ops/ragged/ragged_where_op.py      | 166 ++++
 .../python/ops/ragged/ragged_where_op_test.py |   8 +-
 15 files changed, 989 insertions(+), 769 deletions(-)
 create mode 100644 tensorflow/python/ops/ragged/ragged_batch_gather_ops.py
 create mode 100644 tensorflow/python/ops/ragged/ragged_concat_ops.py
 create mode 100644 tensorflow/python/ops/ragged/ragged_gather_ops.py
 create mode 100644 tensorflow/python/ops/ragged/ragged_where_op.py

diff --git a/tensorflow/python/ops/ragged/BUILD b/tensorflow/python/ops/ragged/BUILD
index 972b14955f..e3bdb74cfe 100644
--- a/tensorflow/python/ops/ragged/BUILD
+++ b/tensorflow/python/ops/ragged/BUILD
@@ -24,10 +24,13 @@ py_library(
     tags = ["nofixdeps"],
     deps = [
         ":ragged_array_ops",
+        ":ragged_batch_gather_ops",
+        ":ragged_concat_ops",
         ":ragged_conversion_ops",
         ":ragged_dispatch",
         ":ragged_factory_ops",
         ":ragged_functional_ops",
+        ":ragged_gather_ops",
         ":ragged_getitem",
         ":ragged_map_ops",
         ":ragged_math_ops",
@@ -37,6 +40,7 @@ py_library(
         ":ragged_tensor_shape",
         ":ragged_tensor_value",
         ":ragged_util",
+        ":ragged_where_op",
         ":segment_id_ops",
         "//tensorflow/python:util",
     ],
@@ -55,6 +59,7 @@ py_library(
         ":segment_id_ops",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:check_ops",
+        "//tensorflow/python:constant_op",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:math_ops",
@@ -64,6 +69,45 @@ py_library(
     ],
 )
 
+py_library(
+    name = "ragged_batch_gather_ops",
+    srcs = ["ragged_batch_gather_ops.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":ragged_array_ops",
+        ":ragged_concat_ops",
+        ":ragged_conversion_ops",
+        ":ragged_gather_ops",
+        ":ragged_tensor",
+        ":ragged_tensor_shape",
+        ":ragged_util",
+        ":ragged_where_op",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:check_ops",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:math_ops",
+    ],
+)
+
+py_library(
+    name = "ragged_concat_ops",
+    srcs = ["ragged_concat_ops.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":ragged_array_ops",
+        ":ragged_conversion_ops",
+        ":ragged_gather_ops",
+        ":ragged_tensor",
+        ":ragged_util",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:check_ops",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:tensor_shape",
+    ],
+)
+
 py_library(
     name = "ragged_conversion_ops",
     srcs = ["ragged_conversion_ops.py"],
@@ -115,13 +159,30 @@ py_library(
     ],
 )
 
+py_library(
+    name = "ragged_gather_ops",
+    srcs = ["ragged_gather_ops.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":ragged_array_ops",
+        ":ragged_conversion_ops",
+        ":ragged_tensor",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:ragged_array_ops_gen",
+        "//tensorflow/python:tensor_shape",
+    ],
+)
+
 py_library(
     name = "ragged_getitem",
     srcs = ["ragged_getitem.py"],
     srcs_version = "PY2AND3",
     deps = [
-        ":ragged_array_ops",
         ":ragged_factory_ops",
+        ":ragged_gather_ops",
         ":ragged_math_ops",
         ":ragged_tensor",
         "//tensorflow/python:array_ops",
@@ -244,6 +305,22 @@ py_library(
     ],
 )
 
+py_library(
+    name = "ragged_where_op",
+    srcs = ["ragged_where_op.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":ragged_concat_ops",
+        ":ragged_functional_ops",
+        ":ragged_gather_ops",
+        ":ragged_tensor",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:math_ops",
+    ],
+)
+
 py_library(
     name = "segment_id_ops",
     srcs = ["segment_id_ops.py"],
@@ -412,6 +489,7 @@ py_test(
     deps = [
         ":ragged_array_ops",
         ":ragged_factory_ops",
+        ":ragged_gather_ops",
         ":ragged_test_util",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:constant_op",
@@ -430,6 +508,7 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":ragged_array_ops",
+        ":ragged_batch_gather_ops",
         ":ragged_factory_ops",
         ":ragged_tensor",
         ":ragged_test_util",
@@ -448,8 +527,8 @@ py_test(
     srcs = ["ragged_gather_nd_op_test.py"],
     srcs_version = "PY2AND3",
     deps = [
-        ":ragged_array_ops",
         ":ragged_factory_ops",
+        ":ragged_gather_ops",
         ":ragged_test_util",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:dtypes",
@@ -691,7 +770,7 @@ py_test(
     srcs = ["ragged_concat_op_test.py"],
     srcs_version = "PY2AND3",
     deps = [
-        ":ragged_array_ops",
+        ":ragged_concat_ops",
         ":ragged_factory_ops",
         ":ragged_test_util",
         "//tensorflow/python:array_ops",
@@ -710,7 +789,7 @@ py_test(
     srcs = ["ragged_stack_op_test.py"],
     srcs_version = "PY2AND3",
     deps = [
-        ":ragged_array_ops",
+        ":ragged_concat_ops",
         ":ragged_factory_ops",
         ":ragged_test_util",
         "//tensorflow/python:constant_op",
@@ -785,9 +864,9 @@ py_test(
     srcs = ["ragged_where_op_test.py"],
     srcs_version = "PY2AND3",
     deps = [
-        ":ragged_array_ops",
         ":ragged_factory_ops",
         ":ragged_test_util",
+        ":ragged_where_op",
         "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:platform_test",
         "@absl_py//absl/testing:parameterized",
diff --git a/tensorflow/python/ops/ragged/__init__.py b/tensorflow/python/ops/ragged/__init__.py
index 30ed9c53ab..a5ffd8a950 100644
--- a/tensorflow/python/ops/ragged/__init__.py
+++ b/tensorflow/python/ops/ragged/__init__.py
@@ -29,10 +29,13 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.ops.ragged import ragged_array_ops
+from tensorflow.python.ops.ragged import ragged_batch_gather_ops
+from tensorflow.python.ops.ragged import ragged_concat_ops
 from tensorflow.python.ops.ragged import ragged_conversion_ops
 from tensorflow.python.ops.ragged import ragged_dispatch
 from tensorflow.python.ops.ragged import ragged_factory_ops
 from tensorflow.python.ops.ragged import ragged_functional_ops
+from tensorflow.python.ops.ragged import ragged_gather_ops
 from tensorflow.python.ops.ragged import ragged_getitem
 from tensorflow.python.ops.ragged import ragged_map_ops
 from tensorflow.python.ops.ragged import ragged_math_ops
@@ -41,6 +44,7 @@ from tensorflow.python.ops.ragged import ragged_string_ops
 from tensorflow.python.ops.ragged import ragged_tensor
 from tensorflow.python.ops.ragged import ragged_tensor_shape
 from tensorflow.python.ops.ragged import ragged_tensor_value
+from tensorflow.python.ops.ragged import ragged_where_op
 from tensorflow.python.ops.ragged import segment_id_ops
 
 # Add a list of the ops that support Ragged Tensors.
diff --git a/tensorflow/python/ops/ragged/ragged_array_ops.py b/tensorflow/python/ops/ragged/ragged_array_ops.py
index f1e61fd7f6..8c62cc4a72 100644
--- a/tensorflow/python/ops/ragged/ragged_array_ops.py
+++ b/tensorflow/python/ops/ragged/ragged_array_ops.py
@@ -20,11 +20,8 @@ from __future__ import print_function
 
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
-from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import check_ops
-from tensorflow.python.ops import gen_ragged_array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops.ragged import ragged_conversion_ops
 from tensorflow.python.ops.ragged import ragged_functional_ops
@@ -34,326 +31,6 @@ from tensorflow.python.ops.ragged import ragged_util
 from tensorflow.python.ops.ragged import segment_id_ops
 
 
-#===============================================================================
-# ragged_gather
-#===============================================================================
-# TODO(edloper): Add an `axis` argument
-def gather(params, indices, validate_indices=None, axis=0, batch_dims=0,
-           name=None):
-  """Gathers ragged slices from `params` axis `0` according to `indices`.
-
-  Returns `RaggedTensor` output, such that:
-
-  ```python
-  output.shape = indices.shape + params.shape[1:]
-  output.ragged_rank = indices.shape.ndims + params.ragged_rank
-  output[i...j, d0...dn] = params[indices[i...j], d0...dn]
-  ```
-
-  `params` may be ragged.  `indices` may be ragged.
-  `indices` must have dtype `int32` or `int64`. If any index is out of bounds,
-  then an error is returned.
-
-  Examples:
-
-  ```python
-  >>> params = tf.constant(['a', 'b', 'c', 'd', 'e'])
-  >>> indices = tf.constant([3, 1, 2, 1, 0])
-  >>> ragged_params = tf.ragged.constant([['a', 'b', 'c'], ['d'], [], ['e']])
-  >>> ragged_indices = tf.ragged.constant([[3, 1, 2], [1], [], [0]])
-
-  >>> print ragged.gather(params, ragged_indices)
-  [['d', 'b', 'c'], ['b'], [], ['a']]
-
-  >>> print ragged.gather(ragged_params, indices)
-  [['e'], ['d'], [], ['d'], ['a', 'b', 'c']]
-
-  >>> print ragged.gather(ragged_params, ragged_indices)
-  [[['e'], ['d'], []], [['d']], [], [['a', 'b', 'c']]]
-  ```
-
-  Args:
-    params: The potentially ragged tensor from which to gather values. Must be
-      at least rank 1.
-    indices: The potentially ragged tensor indicating which values to gather.
-      Must have dtype `int32` or `int64`.  Values must be in the range `[0,
-      params.shape[0]]`.
-    validate_indices: Ignored.
-    axis: Must be zero.
-    batch_dims: Must be zero.
-    name: A name for the operation (optional).
-
-  Returns:
-    A `RaggedTensor`, where `output.dtype=params.dtype` and
-    `output.shape=indices.shape + params.shape[1:]` and
-    `output.ragged_rank=indices.shape.ndims + params.ragged_rank`.
-
-  Raises:
-    ValueError: If indices.shape.ndims is not known statically.
-  """
-  del validate_indices
-  if not isinstance(axis, int) or axis != 0:
-    raise ValueError('axis != 0 is not supported for ragged gather yet.')
-  if not isinstance(batch_dims, int) or batch_dims != 0:
-    raise ValueError('batch_dims != 0 is not supported for ragged gather yet.')
-  with ops.name_scope(name, 'RaggedGather', [params, indices]):
-    params = ragged_tensor.convert_to_tensor_or_ragged_tensor(
-        params, name='params')
-    indices = ragged_tensor.convert_to_tensor_or_ragged_tensor(
-        indices, name='indices')
-
-    if ragged_tensor.is_ragged(indices):
-      return indices.with_values(gather(params, indices.values))
-
-    if not ragged_tensor.is_ragged(params):
-      return array_ops.gather(params, indices)
-
-    indices = ops.convert_to_tensor(indices)
-    if indices.shape.ndims is None:
-      raise ValueError('indices.shape.ndims must be known statically')
-
-    result = gen_ragged_array_ops.ragged_gather(
-        indices=indices,
-        params_dense_values=params.flat_values,
-        params_nested_splits=params.nested_row_splits,
-        OUTPUT_RAGGED_RANK=indices.shape.ndims + len(params.nested_row_splits) -
-        1)
-
-    # Compose the RaggedTensor from splits & values.
-    return ragged_tensor.RaggedTensor.from_nested_row_splits(
-        result.output_dense_values, result.output_nested_splits)
-
-
-#===============================================================================
-# ragged.batch_gather
-#===============================================================================
-def batch_gather(params, indices, name=None):
-  """Gathers slices from `params` according to `indices` with batch dims.
-
-  This operation is similar to `gather`, but it assumes that the leading `N`
-  dimensions of `indices` and `params` are batch dimensions, and performs a
-  gather within each batch.  In particular, when using this operation with `N`
-  batch dimensions `B1...BN`:
-
-  * `indices` has shape `[B1...BN, I]`
-  * `params` has shape `[B1...BN, P1...PM]`.
-  * `result` has shape `[B1...BN, I, P2...PM]`.
-  * `result[b1...bN, i, p2...pM] =
-    params[b1...bN, indices[b1...bN, i], p2...pM]`
-
-  Args:
-    params: A potentially ragged tensor with shape `[B1...BN, P1...PM]` (`N>=0`,
-      `M>0`).
-    indices: A potentially ragged tensor with shape `[B1...BN, I]` (`N>=0`).
-    name: A name for the operation (optional).
-
-  Returns:
-    A potentially ragged tensor with shape `[B1...BN, I, P2...PM]`.
-    `result.ragged_rank = max(indices.ragged_rank, params.ragged_rank)`.
-
-  #### Example:
-    ```python
-    >>> params = tf.ragged.constant([['a', 'b', 'c'], ['d'], [], ['e']])
-    >>> indices = tf.ragged.constant([[1, 2, 0], [], [], [0, 0]])
-    >>> ragged.batch_gather(params, indices)
-    [['b', 'c', 'a'], [], [], ['e', 'e']]
-    ```
-  """
-  if not (ragged_tensor.is_ragged(params) or ragged_tensor.is_ragged(indices)):
-    return array_ops.batch_gather(params, indices, name)
-
-  with ops.name_scope(name, 'RaggedBatchGather', [params, indices]):
-    params = ragged_tensor.convert_to_tensor_or_ragged_tensor(
-        params, name='params')
-    indices = ragged_tensor.convert_to_tensor_or_ragged_tensor(
-        indices, name='indices')
-    indices_ndims = indices.shape.ndims
-    if indices_ndims is None:
-      raise ValueError(
-          'batch_gather does not allow indices with unknown shape.')
-    if indices_ndims == 0:
-      raise ValueError('indices.rank must be at least 1.')
-
-    if ragged_tensor.is_ragged(indices):
-      # If the outermost ragged dimension is a batch dimension, recurse.
-      if indices_ndims > 2:
-        if not ragged_tensor.is_ragged(params):
-          raise ValueError('batch shape from indices does '
-                           'not match params shape')
-        checks = [check_ops.assert_equal(params.row_splits, indices.row_splits)]
-        with ops.control_dependencies(checks):
-          return ragged_tensor.RaggedTensor.from_row_splits(
-              batch_gather(params.values, indices.values), indices.row_splits)
-
-      # Otherwise, indices is a 2D ragged tensor with 1 ragged dimension.
-      else:
-        # Ensure that `params` is ragged and has at least 2 dimensions.
-        if not ragged_tensor.is_ragged(params):
-          if params.shape.ndims is not None and params.shape.ndims < 2:
-            raise ValueError('batch shape from indices does '
-                             'not match params shape')
-          params = ragged_conversion_ops.from_tensor(params, ragged_rank=1)
-
-        # Adjust indices from within-batch to global (in params.values), and
-        # then use ragged.gather to gather them.
-        num_indices = indices.row_lengths()
-        params_starts = params.row_starts()
-        adjustments = ragged_util.repeat(params_starts, num_indices, axis=0)
-        adjusted_index_values = math_ops.to_int64(indices.values) + adjustments
-        return ragged_tensor.RaggedTensor.from_row_splits(
-            gather(params.values, adjusted_index_values), indices.row_splits)
-
-    else:  # params is a RaggedTensor and indices is a Tensor.
-      if indices_ndims == 1:
-        return gather(params, indices)
-      elif indices_ndims == 2:
-        # Adjust indices from batch-local to global (in params.values)
-        adjustments = array_ops.expand_dims(params.row_starts(), 1)
-        adjusted_indices = math_ops.to_int64(indices) + adjustments
-        return gather(params.values, adjusted_indices)
-      else:
-        raise ValueError('batch shape from indices does not match params shape')
-
-
-#===============================================================================
-# ragged.gather_nd
-#===============================================================================
-def gather_nd(params, indices, name=None):
-  """Gather slices from `params` using `n`-dimensional indices.
-
-  This operation is similar to `gather`, but it uses the innermost dimension
-  of `indices` to define a slice into `params`.  In particular, if:
-
-  * `indices` has shape `[A1...AN, I]`
-  * `params` has shape `[B1...BM]`
-
-  Then:
-
-  * `result` has shape `[A1...AN, B_{I+1}...BM]`.
-  * `result[a1...aN] = params[indices[a1...aN, :]]`
-
-  Args:
-    params: A potentially ragged tensor with shape `[A1...AN, I]`.
-    indices: A potentially ragged tensor with shape `[B1...BM]`.
-    name: A name for the operation (optional).
-
-  Returns:
-    A potentially ragged tensor with shape `[A1...AN, B_{I+1}...BM]`.
-
-  #### Examples:
-    ```python
-    >>> params = tf.ragged.constant_value(
-    ...     [ [ ['000', '001'], ['010'              ]          ],
-    ...       [ ['100'       ], ['110', '111', '112'], ['120'] ],
-    ...       [ [            ], ['210'              ]          ] ])
-
-    >>> # Gather 2D slices from a 3D tensor
-    >>> ragged.gather_nd(params, [[2], [0]])
-    [ [ [            ], ['210'] ]
-      [ ['000', '001'], ['010'] ] ]
-
-    >>> # Gather 1D slices from a 3D tensor
-    >>> ragged.gather_nd(params, [[2, 1], [0, 0]])
-    [['210'], ['000', '001']]
-
-    >>> # Gather scalars from a 3D tensor
-    >>> ragged.gather_nd(params, [[0, 0, 1], [1, 1, 2]])
-    ['001', '112']
-    ```
-  """
-  if not (ragged_tensor.is_ragged(params) or ragged_tensor.is_ragged(indices)):
-    return array_ops.gather_nd(params, indices, name)
-
-  with ops.name_scope(name, 'RaggedGatherNd', [params, indices]):
-
-    params = ragged_tensor.convert_to_tensor_or_ragged_tensor(
-        params, name='params')
-    indices = ragged_tensor.convert_to_tensor_or_ragged_tensor(
-        indices, name='indices')
-    indices_shape = indices.shape
-    indices_ndims = indices_shape.ndims
-    if indices_ndims is None:
-      raise ValueError('indices.rank be statically known.')
-    if indices_ndims == 0:
-      raise ValueError('indices.rank must be at least 1.')
-    if (ragged_tensor.is_ragged(indices) and
-        indices_ndims == indices.ragged_rank + 1):
-      raise ValueError('The innermost dimension of indices may not be ragged')
-
-    # `index_size` is the "n" in "gather_nd" -- i.e., the number of dimensions
-    # that each index slices into.
-    index_size = tensor_shape.dimension_value(indices_shape[-1])
-    if index_size is None:
-      raise ValueError('indices.shape[-1] must be statically known.')
-
-    # If `indices` has more than 2 dimensions, then recurse.  If `indices` is
-    # dense, then we convert it to ragged before recursing, and then convert
-    # the result back to `dense` if appropriate.
-    if indices_ndims > 2:
-      indices_is_dense = not ragged_tensor.is_ragged(indices)
-      if indices_is_dense:
-        indices = ragged_conversion_ops.from_tensor(
-            indices, ragged_rank=indices_ndims - 2)
-      result = indices.with_flat_values(gather_nd(params, indices.flat_values))
-      if (indices_is_dense and ragged_tensor.is_ragged(result) and
-          result.ragged_rank == indices_ndims - 2):
-        result = ragged_conversion_ops.to_tensor(result)
-      return result
-
-    # indices_ndims <= 2, and the innermost dimension of indices may not be
-    # ragged, so `indices` must not be ragged.
-    assert not ragged_tensor.is_ragged(indices)
-    assert ragged_tensor.is_ragged(params)
-
-    # Handle corner case: An empty index tuple selects the entire `params`
-    # value.  So if `index_size` is zero, then tile `params`.
-    if index_size == 0:
-      params_ndims = params.ragged_rank + array_ops.rank(params.flat_values)
-      for dim in range(indices_ndims - 1):
-        params = expand_dims(params, axis=0)
-      multiples = array_ops.concat([
-          array_ops.shape(indices)[:-1],
-          array_ops.ones([params_ndims], dtypes.int32)
-      ],
-                                   axis=0)
-      return tile(params, multiples)
-
-    # When index_size=1, we can just flatten the index tuples and use gather.
-    elif index_size == 1:
-      flattened_index_tuples = array_ops.reshape(indices, [-1])
-      return gather(params, flattened_index_tuples)
-
-    # Otherwise, params is a RaggedTensor, and indices is a 1D or 2D Tensor.
-    # Flatten both the index tuples and the params, such that the flattened
-    # index tuples point to the correct values in the flattened params; and
-    # then use ragged.gather on the flattened index tuples & params.
-    else:
-      indices = math_ops.to_int64(indices)
-
-      # Flatten the outermost 2 dimensions of the index tuples & params.
-      flattened_index_tuples = array_ops.gather(params.row_splits,
-                                                indices[..., 0])
-      flattened_index_tuples += indices[..., 1]
-      flattened_params = params.values
-
-      # Flatten any remaining dimensions.
-      for dim in range(2, index_size):
-        if not ragged_tensor.is_ragged(flattened_params):
-          flattened_index_tuples = array_ops.expand_dims(
-              flattened_index_tuples, axis=1)
-          flattened_index_tuples = array_ops.concat(
-              [flattened_index_tuples, indices[..., dim:]], axis=1)
-          return array_ops.gather_nd(flattened_params, flattened_index_tuples)
-
-        flattened_index_tuples = array_ops.gather(
-            flattened_params.row_starts(), flattened_index_tuples)
-        flattened_index_tuples += indices[..., dim]
-        flattened_params = flattened_params.values
-
-      # Gather using the flattened index tuples and params.
-      return gather(flattened_params, flattened_index_tuples)
-
-
 #===============================================================================
 # Masking
 #===============================================================================
@@ -547,260 +224,6 @@ def boolean_mask(data, mask, keepdims=False, name=None):
       return masked_values
 
 
-#===============================================================================
-# Concatenation and Stacking
-#===============================================================================
-def concat(values, axis, name=None):
-  """Concatenates potentially ragged tensors along one dimension.
-
-  Given a list of tensors with the same rank `K` (`K >= axis`), returns a
-  rank-`K` `RaggedTensor` `result` such that `result[i0...iaxis]` is the
-  concatenation of `[rt[i0...iaxis] for rt in values]`.
-
-  Args:
-    values: A list of potentially ragged tensors.  May not be empty. All
-      `values` must have the same rank and the same dtype; but unlike
-      `tf.concat`, they can have arbitrary shapes.
-    axis: A python integer, indicating the dimension along which to concatenate.
-      (Note: Unlike `tf.concat`, the `axis` parameter must be statically known.)
-        Negative values are supported only if the rank of at least one
-        `values` value is statically known.
-    name: A name prefix for the returned tensor (optional).
-
-  Returns:
-    A `RaggedTensor` with rank `K`.
-    `result.ragged_rank=max(axis, max(rt.ragged_rank for rt in values]))`.
-
-  Raises:
-    ValueError: If `values` is empty, if `axis` is out of bounds or if
-      the input tensors have different ranks.
-
-  #### Example:
-    ```python
-    >>> t1 = tf.ragged.constant([[1, 2], [3, 4, 5]])
-    >>> t2 = tf.ragged.constant([[6], [7, 8, 9]])
-    >>> ragged.concat([t1, t2], axis=0)
-    [[1, 2], [3, 4, 5], [6], [7, 8, 9]]
-    >>> ragged.concat([t1, t2], axis=1)
-    [[1, 2, 6], [3, 4, 5, 7, 8, 9]]
-    ```
-  """
-  if not isinstance(values, (list, tuple)):
-    values = [values]
-  with ops.name_scope(name, 'RaggedConcat', values):
-    return _ragged_stack_concat_helper(values, axis, stack_values=False)
-
-
-def stack(values, axis=0, name=None):
-  """Stacks potentially ragged tensors along one dimension.
-
-  Given a list of tensors with the same rank `K` (`K >= axis`), returns a
-  rank-`K+1` `RaggedTensor` `result` such that `result[i0...iaxis]` is the
-  list `[rt[i0...iaxis] for rt in values]`.
-
-  Args:
-    values: A list of potentially ragged tensors.  May not be empty. All
-      `values` must have the same rank and the same dtype; but unlike
-      `tf.concat`, they can have arbitrary shapes.
-    axis: A python integer, indicating the dimension along which to stack.
-      (Note: Unlike `tf.stack`, the `axis` parameter must be statically known.)
-        Negative values are supported only if the rank of at least one
-        `values` value is statically known.
-    name: A name prefix for the returned tensor (optional).
-
-  Returns:
-    A `RaggedTensor` with rank `K+1`.
-    `result.ragged_rank=max(axis, max(rt.ragged_rank for rt in values]))`.
-
-  Raises:
-    ValueError: If `values` is empty, if `axis` is out of bounds or if
-      the input tensors have different ranks.
-
-  #### Example:
-    ```python
-    >>> t1 = tf.ragged.constant([[1, 2], [3, 4, 5]])
-    >>> t2 = tf.ragged.constant([[6], [7, 8, 9]])
-    >>> ragged.stack([t1, t2], axis=0)
-    [[[1, 2], [3, 4, 5]], [[6], [7, 9, 0]]]
-    >>> ragged.stack([t1, t2], axis=1)
-    [[[1, 2], [6]], [[3, 4, 5], [7, 8, 9]]]
-    ```
-  """
-  if not isinstance(values, (list, tuple)):
-    values = [values]
-  with ops.name_scope(name, 'RaggedConcat', values):
-    return _ragged_stack_concat_helper(values, axis, stack_values=True)
-
-
-def _ragged_stack_concat_helper(rt_inputs, axis, stack_values):
-  """Helper function to concatenate or stack ragged tensors.
-
-  Args:
-    rt_inputs: A list of RaggedTensors or Tensors to combine.
-    axis: The axis along which to concatenate or stack.
-    stack_values: A boolean -- if true, then stack values; otherwise,
-      concatenate them.
-
-  Returns:
-    A RaggedTensor.
-  Raises:
-    ValueError: If rt_inputs is empty, or if axis is out of range.
-  """
-  # Validate parameters.
-  if not rt_inputs:
-    raise ValueError('rt_inputs may not be empty.')
-
-  # Convert input tensors.
-  rt_inputs = [
-      ragged_tensor.convert_to_tensor_or_ragged_tensor(
-          rt_input, name='rt_input') for rt_input in rt_inputs
-  ]
-
-  # Special case: if there's only one input, then return it as-is.
-  if len(rt_inputs) == 1:
-    if stack_values:
-      return expand_dims(rt_inputs[0], axis=axis)
-    else:
-      return rt_inputs[0]
-
-  # Check the rank (number of dimensions) of the input tensors.
-  ndims = None
-  for rt in rt_inputs:
-    if ndims is None:
-      ndims = rt.shape.ndims
-    else:
-      rt.shape.assert_has_rank(ndims)
-
-  out_ndims = ndims if (ndims is None or not stack_values) else ndims + 1
-  axis = ragged_util.get_positive_axis(axis, out_ndims)
-
-  # If all the inputs are Tensors, and we're combining the final dimension,
-  # then we can delegate to the tf.stack/tf.concat operation, and return a
-  # Tensor.
-  if all(not ragged_tensor.is_ragged(rt) for rt in rt_inputs):
-    if ndims is not None and (axis == out_ndims - 1 or axis == ndims - 1):
-      if stack_values:
-        return array_ops.stack(rt_inputs, axis)
-      else:
-        return array_ops.concat(rt_inputs, axis)
-
-  # Convert any Tensor inputs to RaggedTensors.  This makes it
-  # possible to concatenate Tensors and RaggedTensors together.
-  for i in range(len(rt_inputs)):
-    if not ragged_tensor.is_ragged(rt_inputs[i]):
-      rt_inputs[i] = ragged_conversion_ops.from_tensor(
-          rt_inputs[i], ragged_rank=1)
-
-  # Convert the input tensors to all have the same ragged_rank.
-  ragged_rank = max(max(rt.ragged_rank for rt in rt_inputs), 1)
-  rt_inputs = [_increase_ragged_rank_to(rt, ragged_rank) for rt in rt_inputs]
-
-  if axis == 0:
-    return _ragged_stack_concat_axis_0(rt_inputs, stack_values)
-  elif axis == 1:
-    return _ragged_stack_concat_axis_1(rt_inputs, stack_values)
-  else:  # axis > 1: recurse.
-    values = [rt.values for rt in rt_inputs]
-    splits = [[rt_input.row_splits] for rt_input in rt_inputs]
-    with ops.control_dependencies(ragged_util.assert_splits_match(splits)):
-      return ragged_tensor.RaggedTensor.from_row_splits(
-          _ragged_stack_concat_helper(values, axis - 1, stack_values),
-          splits[0][0])
-
-
-def _ragged_stack_concat_axis_0(rt_inputs, stack_values):
-  """Helper function to concatenate or stack ragged tensors along axis 0.
-
-  Args:
-    rt_inputs: A list of RaggedTensors, all with the same rank and ragged_rank.
-    stack_values: Boolean.  If true, then stack values; otherwise, concatenate
-      them.
-
-  Returns:
-    A RaggedTensor.
-  """
-  # Concatenate the inner values together.
-  flat_values = [rt.flat_values for rt in rt_inputs]
-  concatenated_flat_values = array_ops.concat(flat_values, axis=0)
-
-  # Concatenate the splits together for each ragged dimension (adjusting
-  # split offsets as necessary).
-  nested_splits = [rt.nested_row_splits for rt in rt_inputs]
-  ragged_rank = rt_inputs[0].ragged_rank
-  concatenated_nested_splits = [
-      _concat_ragged_splits([ns[dim]
-                             for ns in nested_splits])
-      for dim in range(ragged_rank)
-  ]
-
-  # If we are performing a stack operation, then add another splits.
-  if stack_values:
-    stack_lengths = array_ops.stack([_nrows(rt) for rt in rt_inputs])
-    stack_splits = ragged_util.lengths_to_splits(stack_lengths)
-    concatenated_nested_splits.insert(0, stack_splits)
-
-  return ragged_tensor.RaggedTensor.from_nested_row_splits(
-      concatenated_flat_values, concatenated_nested_splits)
-
-
-def _ragged_stack_concat_axis_1(rt_inputs, stack_values):
-  """Helper function to concatenate or stack ragged tensors along axis 1.
-
-  Args:
-    rt_inputs: A list of RaggedTensors, all with the same rank and ragged_rank.
-    stack_values: Boolean.  If true, then stack values; otherwise, concatenate
-      them.
-
-  Returns:
-    A RaggedTensor.
-  """
-  num_inputs = len(rt_inputs)
-
-  rt_nrows = _nrows(rt_inputs[0])
-  nrows_msg = 'Input tensors have incompatible shapes.'
-  nrows_checks = [
-      check_ops.assert_equal(_nrows(rt), rt_nrows, message=nrows_msg)
-      for rt in rt_inputs[1:]
-  ]
-
-  with ops.control_dependencies(nrows_checks):
-    # Concatentate the inputs together to put them in a single ragged tensor.
-    concatenated_rt = _ragged_stack_concat_axis_0(rt_inputs, stack_values=False)
-
-    # Use ragged.gather to permute the rows of concatenated_rt.  In particular,
-    #   permuted_rt = [rt_inputs[0][0], ..., rt_inputs[N][0],
-    #                  rt_inputs[0][1], ..., rt_inputs[N][1],
-    #                      ...,
-    #                  rt_inputs[0][M], ..., rt_input[N][M]]
-    # where `N=num_inputs-1` and `M=rt_nrows-1`.
-    row_indices = math_ops.range(rt_nrows * num_inputs)
-    row_index_matrix = array_ops.reshape(row_indices, [num_inputs, -1])
-    transposed_row_index_matrix = array_ops.transpose(row_index_matrix)
-    row_permutation = array_ops.reshape(transposed_row_index_matrix, [-1])
-    permuted_rt = gather(concatenated_rt, row_permutation)
-
-    if stack_values:
-      # Add a new splits tensor to group together the values.
-      stack_splits = math_ops.range(0, rt_nrows * num_inputs + 1, num_inputs)
-      _copy_row_shape(rt_inputs, stack_splits)
-      return ragged_tensor.RaggedTensor.from_row_splits(permuted_rt,
-                                                        stack_splits)
-    else:
-      # Merge together adjacent rows by dropping the row-split indices that
-      # separate them.
-      concat_splits = permuted_rt.row_splits[::num_inputs]
-      _copy_row_shape(rt_inputs, concat_splits)
-      return ragged_tensor.RaggedTensor.from_row_splits(permuted_rt.values,
-                                                        concat_splits)
-
-
-def _copy_row_shape(rt_inputs, splits):
-  """Sets splits.shape to [rt[shape[0]+1] for each rt in rt_inputs."""
-  for rt in rt_inputs:
-    if rt.shape[0] is not None:
-      splits.set_shape(tensor_shape.TensorShape(rt.shape[0] + 1))
-
-
 #===============================================================================
 # Tiling
 #===============================================================================
@@ -1061,139 +484,6 @@ def expand_dims(input, axis, name=None):  # pylint: disable=redefined-builtin
     return ragged_tensor.RaggedTensor.from_row_splits(values, splits)
 
 
-#===============================================================================
-# ragged.where
-#===============================================================================
-def where(condition, x=None, y=None, name=None):
-  """Return the elements, either from `x` or `y`, depending on the `condition`.
-
-  : If both `x` and `y` are `None`:
-    Returns the coordinates of true elements of `condition`. The coordinates
-    are returned in a 2-D tensor with shape
-    `[num_true_values, dim_size(condition)]`, where `result[i]` is the
-    coordinates of the `i`th true value (in row-major order).
-
-  : If both `x` and `y` are non-`None`:
-    Returns a tensor formed by selecting values from `x` where condition is
-    true, and from `y` when condition is false.  In particular:
-
-    : If `condition`, `x`, and `y` all have the same shape:
-
-      * `result[i1...iN] = x[i1...iN]` if `condition[i1...iN]` is true.
-      * `result[i1...iN] = y[i1...iN]` if `condition[i1...iN]` is false.
-
-    : Otherwise:
-
-      * `condition` must be a vector.
-      * `x` and `y` must have the same number of dimensions.
-      * The outermost dimensions of `condition`, `x`, and `y` must all have the
-        same size.
-      * `result[i] = x[i]` if `condition[i]` is true.
-      * `result[i] = y[i]` if `condition[i]` is false.
-
-  Args:
-    condition: A potentially ragged tensor of type `bool`
-    x: A potentially ragged tensor (optional).
-    y: A potentially ragged tensor (optional).  Must be specified if `x` is
-      specified.  Must have the same rank and type as `x`.
-    name: A name of the operation (optional)
-
-  Returns:
-    : If both `x` and `y` are `None`:
-      A `Tensor` with shape `(num_true, dim_size(condition))`.
-    : Otherwise:
-      A potentially ragged tensor with the same type, rank, and outermost
-      dimension size as `x` and `y`.
-      `result.ragged_rank = max(x.ragged_rank, y.ragged_rank)`.
-
-  Raises:
-    ValueError: When exactly one of `x` or `y` is non-`None`; or when
-      `condition`, `x`, and `y` have incompatible shapes.
-
-  #### Examples:
-    ```python
-    >>> # Coordinates where condition is true.
-    >>> condition = tf.ragged.constant_value(
-    ...     [[True, False, True], [False, True]])
-    >>> ragged.where(condition)
-    [[0, 0], [0, 2], [1, 1]]
-
-    >>> # Elementwise selection between x and y, based on condition.
-    >>> condition = tf.ragged.constant_value(
-    ...     [[True, False, True], [False, True]])
-    >>> x = tf.ragged.constant_value([['A', 'B', 'C'], ['D', 'E']])
-    >>> y = tf.ragged.constant_value([['a', 'b', 'c'], ['d', 'e']])
-    >>> ragged.where(condition, x, y)
-    [['A', 'b', 'C'], ['d', 'E']]
-
-    >>> # Row selection between x and y, based on condition.
-    >>> condition = [True, False]
-    >>> x = tf.ragged.constant_value([['A', 'B', 'C'], ['D', 'E']])
-    >>> y = tf.ragged.constant_value([['a', 'b', 'c'], ['d', 'e']])
-    >>> ragged.where(condition, x, y)
-    [['A', 'B', 'C'], ['d', 'e']]
-    ```
-  """
-  if (x is None) != (y is None):
-    raise ValueError('x and y must be either both None or both non-None')
-  with ops.name_scope('RaggedWhere', name, [condition, x, y]):
-    condition = ragged_tensor.convert_to_tensor_or_ragged_tensor(
-        condition, name='condition')
-    if x is None:
-      return _coordinate_where(condition)
-    else:
-      x = ragged_tensor.convert_to_tensor_or_ragged_tensor(x, name='x')
-      y = ragged_tensor.convert_to_tensor_or_ragged_tensor(y, name='y')
-      return _elementwise_where(condition, x, y)
-
-
-def _elementwise_where(condition, x, y):
-  """Ragged version of tf.where(condition, x, y)."""
-  condition_is_ragged = isinstance(condition, ragged_tensor.RaggedTensor)
-  x_is_ragged = isinstance(x, ragged_tensor.RaggedTensor)
-  y_is_ragged = isinstance(y, ragged_tensor.RaggedTensor)
-
-  if not (condition_is_ragged or x_is_ragged or y_is_ragged):
-    return array_ops.where(condition, x, y)
-
-  elif condition_is_ragged and x_is_ragged and y_is_ragged:
-    return ragged_functional_ops.map_flat_values(array_ops.where, condition, x,
-                                                 y)
-  elif not condition_is_ragged:
-    # Concatenate x and y, and then use `gather` to assemble the selected rows.
-    condition.shape.assert_has_rank(1)
-    x_nrows = _nrows(x)
-    x_and_y = concat([x, y], axis=0)
-    indices = array_ops.where(condition, math_ops.range(x_nrows),
-                              x_nrows + math_ops.range(_nrows(y)))
-    return gather(x_and_y, indices)
-
-  else:
-    raise ValueError('Input shapes do not match.')
-
-
-def _coordinate_where(condition):
-  """Ragged version of tf.where(condition)."""
-  if not isinstance(condition, ragged_tensor.RaggedTensor):
-    return array_ops.where(condition)
-
-  # The coordinate for each `true` value in condition.values.
-  selected_coords = _coordinate_where(condition.values)
-
-  # Convert the first index in each coordinate to a row index and column index.
-  first_index = selected_coords[:, 0]
-  selected_rows = array_ops.gather(condition.value_rowids(), first_index)
-  selected_row_starts = array_ops.gather(condition.row_splits, selected_rows)
-  selected_cols = first_index - selected_row_starts
-
-  # Assemble the row & column index with the indices for inner dimensions.
-  return array_ops.concat([
-      array_ops.expand_dims(selected_rows, 1),
-      array_ops.expand_dims(selected_cols, 1), selected_coords[:, 1:]
-  ],
-                          axis=1)
-
-
 #===============================================================================
 # RaggedTensor Size
 #===============================================================================
diff --git a/tensorflow/python/ops/ragged/ragged_batch_gather_op_test.py b/tensorflow/python/ops/ragged/ragged_batch_gather_op_test.py
index 431d350db8..72692fcef0 100644
--- a/tensorflow/python/ops/ragged/ragged_batch_gather_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_batch_gather_op_test.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for ragged_array_ops.batch_gather."""
+"""Tests for ragged_batch_gather_ops.batch_gather."""
 
 from __future__ import absolute_import
 from __future__ import division
@@ -25,7 +25,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops.ragged import ragged_array_ops
+from tensorflow.python.ops.ragged import ragged_batch_gather_ops
 from tensorflow.python.ops.ragged import ragged_factory_ops
 from tensorflow.python.ops.ragged import ragged_tensor
 from tensorflow.python.ops.ragged import ragged_test_util
@@ -146,7 +146,7 @@ class RaggedBatchGatherOpTest(ragged_test_util.RaggedTensorTestCase,
               [[[[b'c', b'a'], [b'd', b'd']], [[b'f', b'e']]]], ragged_rank=2)),
   ])
   def testRaggedBatchGather(self, descr, params, indices, expected):
-    result = ragged_array_ops.batch_gather(params, indices)
+    result = ragged_batch_gather_ops.batch_gather(params, indices)
     self.assertRaggedEqual(result, expected)
 
   def testRaggedBatchGatherUnknownRankError(self):
@@ -159,11 +159,11 @@ class RaggedBatchGatherOpTest(ragged_test_util.RaggedTensorTestCase,
 
     with self.assertRaisesRegexp(
         ValueError, 'batch_gather does not allow indices with unknown shape.'):
-      ragged_array_ops.batch_gather(params, indices)
+      ragged_batch_gather_ops.batch_gather(params, indices)
 
     with self.assertRaisesRegexp(
         ValueError, 'batch_gather does not allow indices with unknown shape.'):
-      ragged_array_ops.batch_gather(params, ragged_indices)
+      ragged_batch_gather_ops.batch_gather(params, ragged_indices)
 
   @parameterized.parameters(
       [
@@ -208,7 +208,7 @@ class RaggedBatchGatherOpTest(ragged_test_util.RaggedTensorTestCase,
                                        message=None,
                                        error=ValueError):
     with self.assertRaisesRegexp(error, message):
-      ragged_array_ops.batch_gather(params, indices)
+      ragged_batch_gather_ops.batch_gather(params, indices)
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/ops/ragged/ragged_batch_gather_ops.py b/tensorflow/python/ops/ragged/ragged_batch_gather_ops.py
new file mode 100644
index 0000000000..9c57aead91
--- /dev/null
+++ b/tensorflow/python/ops/ragged/ragged_batch_gather_ops.py
@@ -0,0 +1,120 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Batch gather operations for RaggedTensors."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import check_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops.ragged import ragged_conversion_ops
+from tensorflow.python.ops.ragged import ragged_gather_ops
+from tensorflow.python.ops.ragged import ragged_tensor
+from tensorflow.python.ops.ragged import ragged_util
+
+
+#===============================================================================
+# ragged.batch_gather
+#===============================================================================
+def batch_gather(params, indices, name=None):
+  """Gathers slices from `params` according to `indices` with batch dims.
+
+  This operation is similar to `gather`, but it assumes that the leading `N`
+  dimensions of `indices` and `params` are batch dimensions, and performs a
+  gather within each batch.  In particular, when using this operation with `N`
+  batch dimensions `B1...BN`:
+
+  * `indices` has shape `[B1...BN, I]`
+  * `params` has shape `[B1...BN, P1...PM]`.
+  * `result` has shape `[B1...BN, I, P2...PM]`.
+  * `result[b1...bN, i, p2...pM] =
+    params[b1...bN, indices[b1...bN, i], p2...pM]`
+
+  Args:
+    params: A potentially ragged tensor with shape `[B1...BN, P1...PM]` (`N>=0`,
+      `M>0`).
+    indices: A potentially ragged tensor with shape `[B1...BN, I]` (`N>=0`).
+    name: A name for the operation (optional).
+
+  Returns:
+    A potentially ragged tensor with shape `[B1...BN, I, P2...PM]`.
+    `result.ragged_rank = max(indices.ragged_rank, params.ragged_rank)`.
+
+  #### Example:
+    ```python
+    >>> params = tf.ragged.constant([['a', 'b', 'c'], ['d'], [], ['e']])
+    >>> indices = tf.ragged.constant([[1, 2, 0], [], [], [0, 0]])
+    >>> tf.batch_gather(params, indices)
+    [['b', 'c', 'a'], [], [], ['e', 'e']]
+    ```
+  """
+  if not (ragged_tensor.is_ragged(params) or ragged_tensor.is_ragged(indices)):
+    return array_ops.batch_gather(params, indices, name)
+
+  with ops.name_scope(name, 'RaggedBatchGather', [params, indices]):
+    params = ragged_tensor.convert_to_tensor_or_ragged_tensor(
+        params, name='params')
+    indices = ragged_tensor.convert_to_tensor_or_ragged_tensor(
+        indices, name='indices')
+    indices_ndims = indices.shape.ndims
+    if indices_ndims is None:
+      raise ValueError(
+          'batch_gather does not allow indices with unknown shape.')
+    if indices_ndims == 0:
+      raise ValueError('indices.rank must be at least 1.')
+
+    if ragged_tensor.is_ragged(indices):
+      # If the outermost ragged dimension is a batch dimension, recurse.
+      if indices_ndims > 2:
+        if not ragged_tensor.is_ragged(params):
+          raise ValueError('batch shape from indices does '
+                           'not match params shape')
+        checks = [check_ops.assert_equal(params.row_splits, indices.row_splits)]
+        with ops.control_dependencies(checks):
+          return ragged_tensor.RaggedTensor.from_row_splits(
+              batch_gather(params.values, indices.values), indices.row_splits)
+
+      # Otherwise, indices is a 2D ragged tensor with 1 ragged dimension.
+      else:
+        # Ensure that `params` is ragged and has at least 2 dimensions.
+        if not ragged_tensor.is_ragged(params):
+          if params.shape.ndims is not None and params.shape.ndims < 2:
+            raise ValueError('batch shape from indices does '
+                             'not match params shape')
+          params = ragged_conversion_ops.from_tensor(params, ragged_rank=1)
+
+        # Adjust indices from within-batch to global (in params.values), and
+        # then use ragged.gather to gather them.
+        num_indices = indices.row_lengths()
+        params_starts = params.row_starts()
+        adjustments = ragged_util.repeat(params_starts, num_indices, axis=0)
+        adjusted_index_values = math_ops.to_int64(indices.values) + adjustments
+        return ragged_tensor.RaggedTensor.from_row_splits(
+            ragged_gather_ops.gather(params.values, adjusted_index_values),
+            indices.row_splits)
+
+    else:  # params is a RaggedTensor and indices is a Tensor.
+      if indices_ndims == 1:
+        return ragged_gather_ops.gather(params, indices)
+      elif indices_ndims == 2:
+        # Adjust indices from batch-local to global (in params.values)
+        adjustments = array_ops.expand_dims(params.row_starts(), 1)
+        adjusted_indices = math_ops.to_int64(indices) + adjustments
+        return ragged_gather_ops.gather(params.values, adjusted_indices)
+      else:
+        raise ValueError('batch shape from indices does not match params shape')
diff --git a/tensorflow/python/ops/ragged/ragged_concat_op_test.py b/tensorflow/python/ops/ragged/ragged_concat_op_test.py
index 254afdaa21..62989d3025 100644
--- a/tensorflow/python/ops/ragged/ragged_concat_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_concat_op_test.py
@@ -26,7 +26,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops.ragged import ragged_array_ops
+from tensorflow.python.ops.ragged import ragged_concat_ops
 from tensorflow.python.ops.ragged import ragged_factory_ops
 from tensorflow.python.ops.ragged import ragged_test_util
 from tensorflow.python.platform import googletest
@@ -235,7 +235,7 @@ class RaggedConcatOpTest(ragged_test_util.RaggedTensorTestCase,
                        expected_ragged_rank=None,
                        expected_shape=None):
     rt_inputs = self._rt_inputs_to_tensors(rt_inputs, ragged_ranks)
-    concatenated = ragged_array_ops.concat(rt_inputs, axis)
+    concatenated = ragged_concat_ops.concat(rt_inputs, axis)
     if expected_ragged_rank is not None:
       self.assertEqual(concatenated.ragged_rank, expected_ragged_rank)
     if expected_shape is not None:
@@ -276,7 +276,7 @@ class RaggedConcatOpTest(ragged_test_util.RaggedTensorTestCase,
                       message=None,
                       ragged_ranks=None):
     rt_inputs = self._rt_inputs_to_tensors(rt_inputs, ragged_ranks)
-    self.assertRaisesRegexp(error, message, ragged_array_ops.concat, rt_inputs,
+    self.assertRaisesRegexp(error, message, ragged_concat_ops.concat, rt_inputs,
                             axis)
 
   @parameterized.parameters([
@@ -294,7 +294,7 @@ class RaggedConcatOpTest(ragged_test_util.RaggedTensorTestCase,
     rt_inputs = [
         array_ops.placeholder_with_default(rt, shape=None) for rt in rt_inputs
     ]
-    concatenated = ragged_array_ops.concat(rt_inputs, axis)
+    concatenated = ragged_concat_ops.concat(rt_inputs, axis)
     with self.assertRaisesRegexp(error, message):
       self.evaluate(concatenated)
 
@@ -307,7 +307,7 @@ class RaggedConcatOpTest(ragged_test_util.RaggedTensorTestCase,
     ]
     self.assertRaisesRegexp(
         ValueError, r'axis may only be negative if ndims is statically known.',
-        ragged_array_ops.concat, rt_inputs, -1)
+        ragged_concat_ops.concat, rt_inputs, -1)
 
   def testSingleTensorInput(self):
     """Tests ragged_concat with a single tensor input.
@@ -317,7 +317,7 @@ class RaggedConcatOpTest(ragged_test_util.RaggedTensorTestCase,
     returns that tensor.  This test exercises that path.
     """
     rt_inputs = ragged_factory_ops.constant([[1, 2], [3, 4]])
-    concatenated = ragged_array_ops.concat(rt_inputs, 0)
+    concatenated = ragged_concat_ops.concat(rt_inputs, 0)
     self.assertRaggedEqual(concatenated, [[1, 2], [3, 4]])
 
 
diff --git a/tensorflow/python/ops/ragged/ragged_concat_ops.py b/tensorflow/python/ops/ragged/ragged_concat_ops.py
new file mode 100644
index 0000000000..0f86b05e17
--- /dev/null
+++ b/tensorflow/python/ops/ragged/ragged_concat_ops.py
@@ -0,0 +1,302 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Concat and stack operations for RaggedTensors."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_shape
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import check_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops.ragged import ragged_array_ops
+from tensorflow.python.ops.ragged import ragged_conversion_ops
+from tensorflow.python.ops.ragged import ragged_gather_ops
+from tensorflow.python.ops.ragged import ragged_tensor
+from tensorflow.python.ops.ragged import ragged_util
+
+
+def concat(values, axis, name=None):
+  """Concatenates potentially ragged tensors along one dimension.
+
+  Given a list of tensors with the same rank `K` (`K >= axis`), returns a
+  rank-`K` `RaggedTensor` `result` such that `result[i0...iaxis]` is the
+  concatenation of `[rt[i0...iaxis] for rt in values]`.
+
+  Args:
+    values: A list of potentially ragged tensors.  May not be empty. All
+      `values` must have the same rank and the same dtype; but unlike
+      `tf.concat`, they can have arbitrary shapes.
+    axis: A python integer, indicating the dimension along which to concatenate.
+      (Note: Unlike `tf.concat`, the `axis` parameter must be statically known.)
+        Negative values are supported only if the rank of at least one
+        `values` value is statically known.
+    name: A name prefix for the returned tensor (optional).
+
+  Returns:
+    A `RaggedTensor` with rank `K`.
+    `result.ragged_rank=max(axis, max(rt.ragged_rank for rt in values]))`.
+
+  Raises:
+    ValueError: If `values` is empty, if `axis` is out of bounds or if
+      the input tensors have different ranks.
+
+  #### Example:
+    ```python
+    >>> t1 = tf.ragged.constant([[1, 2], [3, 4, 5]])
+    >>> t2 = tf.ragged.constant([[6], [7, 8, 9]])
+    >>> ragged.concat([t1, t2], axis=0)
+    [[1, 2], [3, 4, 5], [6], [7, 8, 9]]
+    >>> ragged.concat([t1, t2], axis=1)
+    [[1, 2, 6], [3, 4, 5, 7, 8, 9]]
+    ```
+  """
+  if not isinstance(values, (list, tuple)):
+    values = [values]
+  with ops.name_scope(name, 'RaggedConcat', values):
+    return _ragged_stack_concat_helper(values, axis, stack_values=False)
+
+
+def stack(values, axis=0, name=None):
+  """Stacks potentially ragged tensors along one dimension.
+
+  Given a list of tensors with the same rank `K` (`K >= axis`), returns a
+  rank-`K+1` `RaggedTensor` `result` such that `result[i0...iaxis]` is the
+  list `[rt[i0...iaxis] for rt in values]`.
+
+  Args:
+    values: A list of potentially ragged tensors.  May not be empty. All
+      `values` must have the same rank and the same dtype; but unlike
+      `tf.concat`, they can have arbitrary shapes.
+    axis: A python integer, indicating the dimension along which to stack.
+      (Note: Unlike `tf.stack`, the `axis` parameter must be statically known.)
+        Negative values are supported only if the rank of at least one
+        `values` value is statically known.
+    name: A name prefix for the returned tensor (optional).
+
+  Returns:
+    A `RaggedTensor` with rank `K+1`.
+    `result.ragged_rank=max(axis, max(rt.ragged_rank for rt in values]))`.
+
+  Raises:
+    ValueError: If `values` is empty, if `axis` is out of bounds or if
+      the input tensors have different ranks.
+
+  #### Example:
+    ```python
+    >>> t1 = tf.ragged.constant([[1, 2], [3, 4, 5]])
+    >>> t2 = tf.ragged.constant([[6], [7, 8, 9]])
+    >>> ragged.stack([t1, t2], axis=0)
+    [[[1, 2], [3, 4, 5]], [[6], [7, 9, 0]]]
+    >>> ragged.stack([t1, t2], axis=1)
+    [[[1, 2], [6]], [[3, 4, 5], [7, 8, 9]]]
+    ```
+  """
+  if not isinstance(values, (list, tuple)):
+    values = [values]
+  with ops.name_scope(name, 'RaggedConcat', values):
+    return _ragged_stack_concat_helper(values, axis, stack_values=True)
+
+
+def _ragged_stack_concat_helper(rt_inputs, axis, stack_values):
+  """Helper function to concatenate or stack ragged tensors.
+
+  Args:
+    rt_inputs: A list of RaggedTensors or Tensors to combine.
+    axis: The axis along which to concatenate or stack.
+    stack_values: A boolean -- if true, then stack values; otherwise,
+      concatenate them.
+
+  Returns:
+    A RaggedTensor.
+  Raises:
+    ValueError: If rt_inputs is empty, or if axis is out of range.
+  """
+  # Validate parameters.
+  if not rt_inputs:
+    raise ValueError('rt_inputs may not be empty.')
+
+  # Convert input tensors.
+  rt_inputs = [
+      ragged_tensor.convert_to_tensor_or_ragged_tensor(
+          rt_input, name='rt_input') for rt_input in rt_inputs
+  ]
+
+  # Special case: if there's only one input, then return it as-is.
+  if len(rt_inputs) == 1:
+    if stack_values:
+      return ragged_array_ops.expand_dims(rt_inputs[0], axis=axis)
+    else:
+      return rt_inputs[0]
+
+  # Check the rank (number of dimensions) of the input tensors.
+  ndims = None
+  for rt in rt_inputs:
+    if ndims is None:
+      ndims = rt.shape.ndims
+    else:
+      rt.shape.assert_has_rank(ndims)
+
+  out_ndims = ndims if (ndims is None or not stack_values) else ndims + 1
+  axis = ragged_util.get_positive_axis(axis, out_ndims)
+
+  # If all the inputs are Tensors, and we're combining the final dimension,
+  # then we can delegate to the tf.stack/tf.concat operation, and return a
+  # Tensor.
+  if all(not ragged_tensor.is_ragged(rt) for rt in rt_inputs):
+    if ndims is not None and (axis == out_ndims - 1 or axis == ndims - 1):
+      if stack_values:
+        return array_ops.stack(rt_inputs, axis)
+      else:
+        return array_ops.concat(rt_inputs, axis)
+
+  # Convert any Tensor inputs to RaggedTensors.  This makes it
+  # possible to concatenate Tensors and RaggedTensors together.
+  for i in range(len(rt_inputs)):
+    if not ragged_tensor.is_ragged(rt_inputs[i]):
+      rt_inputs[i] = ragged_conversion_ops.from_tensor(
+          rt_inputs[i], ragged_rank=1)
+
+  # Convert the input tensors to all have the same ragged_rank.
+  ragged_rank = max(max(rt.ragged_rank for rt in rt_inputs), 1)
+  rt_inputs = [_increase_ragged_rank_to(rt, ragged_rank) for rt in rt_inputs]
+
+  if axis == 0:
+    return _ragged_stack_concat_axis_0(rt_inputs, stack_values)
+  elif axis == 1:
+    return _ragged_stack_concat_axis_1(rt_inputs, stack_values)
+  else:  # axis > 1: recurse.
+    values = [rt.values for rt in rt_inputs]
+    splits = [[rt_input.row_splits] for rt_input in rt_inputs]
+    with ops.control_dependencies(ragged_util.assert_splits_match(splits)):
+      return ragged_tensor.RaggedTensor.from_row_splits(
+          _ragged_stack_concat_helper(values, axis - 1, stack_values),
+          splits[0][0])
+
+
+def _ragged_stack_concat_axis_0(rt_inputs, stack_values):
+  """Helper function to concatenate or stack ragged tensors along axis 0.
+
+  Args:
+    rt_inputs: A list of RaggedTensors, all with the same rank and ragged_rank.
+    stack_values: Boolean.  If true, then stack values; otherwise, concatenate
+      them.
+
+  Returns:
+    A RaggedTensor.
+  """
+  # Concatenate the inner values together.
+  flat_values = [rt.flat_values for rt in rt_inputs]
+  concatenated_flat_values = array_ops.concat(flat_values, axis=0)
+
+  # Concatenate the splits together for each ragged dimension (adjusting
+  # split offsets as necessary).
+  nested_splits = [rt.nested_row_splits for rt in rt_inputs]
+  ragged_rank = rt_inputs[0].ragged_rank
+  concatenated_nested_splits = [
+      _concat_ragged_splits([ns[dim]
+                             for ns in nested_splits])
+      for dim in range(ragged_rank)
+  ]
+
+  # If we are performing a stack operation, then add another splits.
+  if stack_values:
+    stack_lengths = array_ops.stack([rt.nrows() for rt in rt_inputs])
+    stack_splits = ragged_util.lengths_to_splits(stack_lengths)
+    concatenated_nested_splits.insert(0, stack_splits)
+
+  return ragged_tensor.RaggedTensor.from_nested_row_splits(
+      concatenated_flat_values, concatenated_nested_splits)
+
+
+def _ragged_stack_concat_axis_1(rt_inputs, stack_values):
+  """Helper function to concatenate or stack ragged tensors along axis 1.
+
+  Args:
+    rt_inputs: A list of RaggedTensors, all with the same rank and ragged_rank.
+    stack_values: Boolean.  If true, then stack values; otherwise, concatenate
+      them.
+
+  Returns:
+    A RaggedTensor.
+  """
+  num_inputs = len(rt_inputs)
+
+  rt_nrows = rt_inputs[0].nrows()
+  nrows_msg = 'Input tensors have incompatible shapes.'
+  nrows_checks = [
+      check_ops.assert_equal(rt.nrows(), rt_nrows, message=nrows_msg)
+      for rt in rt_inputs[1:]
+  ]
+
+  with ops.control_dependencies(nrows_checks):
+    # Concatentate the inputs together to put them in a single ragged tensor.
+    concatenated_rt = _ragged_stack_concat_axis_0(rt_inputs, stack_values=False)
+
+    # Use ragged.gather to permute the rows of concatenated_rt.  In particular,
+    #   permuted_rt = [rt_inputs[0][0], ..., rt_inputs[N][0],
+    #                  rt_inputs[0][1], ..., rt_inputs[N][1],
+    #                      ...,
+    #                  rt_inputs[0][M], ..., rt_input[N][M]]
+    # where `N=num_inputs-1` and `M=rt_nrows-1`.
+    row_indices = math_ops.range(rt_nrows * num_inputs)
+    row_index_matrix = array_ops.reshape(row_indices, [num_inputs, -1])
+    transposed_row_index_matrix = array_ops.transpose(row_index_matrix)
+    row_permutation = array_ops.reshape(transposed_row_index_matrix, [-1])
+    permuted_rt = ragged_gather_ops.gather(concatenated_rt, row_permutation)
+
+    if stack_values:
+      # Add a new splits tensor to group together the values.
+      stack_splits = math_ops.range(0, rt_nrows * num_inputs + 1, num_inputs)
+      _copy_row_shape(rt_inputs, stack_splits)
+      return ragged_tensor.RaggedTensor.from_row_splits(permuted_rt,
+                                                        stack_splits)
+    else:
+      # Merge together adjacent rows by dropping the row-split indices that
+      # separate them.
+      concat_splits = permuted_rt.row_splits[::num_inputs]
+      _copy_row_shape(rt_inputs, concat_splits)
+      return ragged_tensor.RaggedTensor.from_row_splits(permuted_rt.values,
+                                                        concat_splits)
+
+
+def _copy_row_shape(rt_inputs, splits):
+  """Sets splits.shape to [rt[shape[0]+1] for each rt in rt_inputs."""
+  for rt in rt_inputs:
+    if rt.shape[0] is not None:
+      splits.set_shape(tensor_shape.TensorShape(rt.shape[0] + 1))
+
+
+def _increase_ragged_rank_to(rt_input, ragged_rank):
+  """Adds ragged dimensions to `rt_input` so it has the desired ragged rank."""
+  if ragged_rank > 0:
+    if not ragged_tensor.is_ragged(rt_input):
+      rt_input = ragged_conversion_ops.from_tensor(rt_input)
+    if rt_input.ragged_rank < ragged_rank:
+      rt_input = rt_input.with_values(
+          _increase_ragged_rank_to(rt_input.values, ragged_rank - 1))
+  return rt_input
+
+
+def _concat_ragged_splits(splits_list):
+  """Concatenates a list of RaggedTensor splits to form a single splits."""
+  pieces = [splits_list[0]]
+  splits_offset = splits_list[0][-1]
+  for splits in splits_list[1:]:
+    pieces.append(splits[1:] + splits_offset)
+    splits_offset += splits[-1]
+  return array_ops.concat(pieces, axis=0)
diff --git a/tensorflow/python/ops/ragged/ragged_dispatch.py b/tensorflow/python/ops/ragged/ragged_dispatch.py
index 5284750198..0c9e6efe8b 100644
--- a/tensorflow/python/ops/ragged/ragged_dispatch.py
+++ b/tensorflow/python/ops/ragged/ragged_dispatch.py
@@ -32,10 +32,14 @@ from tensorflow.python.ops import parsing_ops
 from tensorflow.python.ops import string_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.ops.ragged import ragged_array_ops
+from tensorflow.python.ops.ragged import ragged_batch_gather_ops
+from tensorflow.python.ops.ragged import ragged_concat_ops
+from tensorflow.python.ops.ragged import ragged_gather_ops
 from tensorflow.python.ops.ragged import ragged_math_ops
 from tensorflow.python.ops.ragged import ragged_tensor
 from tensorflow.python.ops.ragged import ragged_tensor_shape
 from tensorflow.python.ops.ragged import ragged_util
+from tensorflow.python.ops.ragged import ragged_where_op
 from tensorflow.python.util import dispatch
 from tensorflow.python.util import tf_decorator
 from tensorflow.python.util import tf_export
@@ -401,7 +405,7 @@ _V1_OPS_THAT_DELEGATE_TO_V2_OPS = [
 
 def _ragged_gather_v1(params, indices, validate_indices=None, name=None,
                       axis=0, batch_dims=0):
-  return ragged_array_ops.gather(
+  return ragged_gather_ops.gather(
       params=params,
       indices=indices,
       validate_indices=validate_indices,
@@ -422,20 +426,20 @@ def _ragged_size_v1(input, name=None, out_type=dtypes.int32):  # pylint: disable
 
 # (original_op, ragged_op, ragged_args)
 _RAGGED_DISPATCH_OPS = [
-    (array_ops.batch_gather, ragged_array_ops.batch_gather,
+    (array_ops.batch_gather, ragged_batch_gather_ops.batch_gather,
      ['params', 'indices']),
-    (array_ops.concat, ragged_array_ops.concat, ['[values]']),
+    (array_ops.concat, ragged_concat_ops.concat, ['[values]']),
     (array_ops.expand_dims, _ragged_expand_dims_v1, ['input']),
     (array_ops.expand_dims_v2, ragged_array_ops.expand_dims, ['input']),
     (array_ops.gather, _ragged_gather_v1, ['params', 'indices']),
-    (array_ops.gather_v2, ragged_array_ops.gather, ['params', 'indices']),
-    (array_ops.gather_nd, ragged_array_ops.gather_nd, ['params', 'indices']),
+    (array_ops.gather_v2, ragged_gather_ops.gather, ['params', 'indices']),
+    (array_ops.gather_nd, ragged_gather_ops.gather_nd, ['params', 'indices']),
     (array_ops.rank, ragged_array_ops.rank, ['input']),
     (array_ops.size, _ragged_size_v1, ['input']),
     (array_ops.size_v2, ragged_array_ops.size, ['input']),
-    (array_ops.stack, ragged_array_ops.stack, ['[values]']),
+    (array_ops.stack, ragged_concat_ops.stack, ['[values]']),
     (array_ops.tile, ragged_array_ops.tile, ['input']),
-    (array_ops.where, ragged_array_ops.where, ['condition', 'x', 'y']),
+    (array_ops.where, ragged_where_op.where, ['condition', 'x', 'y']),
     (math_ops.unsorted_segment_sum, ragged_math_ops.segment_sum,
      ['data', 'segment_ids']),
     (math_ops.unsorted_segment_prod, ragged_math_ops.segment_prod,
diff --git a/tensorflow/python/ops/ragged/ragged_gather_nd_op_test.py b/tensorflow/python/ops/ragged/ragged_gather_nd_op_test.py
index d4bffeb401..8e44368d47 100644
--- a/tensorflow/python/ops/ragged/ragged_gather_nd_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_gather_nd_op_test.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for ragged_array_ops.gather_nd."""
+"""Tests for ragged_gather_ops.gather_nd."""
 
 from __future__ import absolute_import
 from __future__ import division
@@ -26,8 +26,8 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops.ragged import ragged_array_ops
 from tensorflow.python.ops.ragged import ragged_factory_ops
+from tensorflow.python.ops.ragged import ragged_gather_ops
 from tensorflow.python.ops.ragged import ragged_test_util
 from tensorflow.python.platform import googletest
 
@@ -201,7 +201,7 @@ class RaggedGatherNdOpTest(ragged_test_util.RaggedTensorTestCase,
           expected=[[b'c', b'd'], [b'a', b'b'], [b'e', b'f']]),
   ])  # pyformat: disable
   def testRaggedGatherNd(self, descr, params, indices, expected):
-    result = ragged_array_ops.gather_nd(params, indices)
+    result = ragged_gather_ops.gather_nd(params, indices)
     self.assertRaggedEqual(result, expected)
 
   def testRaggedGatherNdUnknownRankError(self):
@@ -213,10 +213,10 @@ class RaggedGatherNdOpTest(ragged_test_util.RaggedTensorTestCase,
 
     with self.assertRaisesRegexp(ValueError,
                                  'indices.rank be statically known.'):
-      ragged_array_ops.gather_nd(params, indices1)
+      ragged_gather_ops.gather_nd(params, indices1)
     with self.assertRaisesRegexp(
         ValueError, r'indices.shape\[-1\] must be statically known.'):
-      ragged_array_ops.gather_nd(params, indices2)
+      ragged_gather_ops.gather_nd(params, indices2)
 
   @parameterized.parameters([
       dict(
@@ -238,7 +238,7 @@ class RaggedGatherNdOpTest(ragged_test_util.RaggedTensorTestCase,
                                     message=None,
                                     error=ValueError):
     with self.assertRaisesRegexp(error, message):
-      ragged_array_ops.gather_nd(params, indices)
+      ragged_gather_ops.gather_nd(params, indices)
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/ops/ragged/ragged_gather_op_test.py b/tensorflow/python/ops/ragged/ragged_gather_op_test.py
index 9914b56448..eb64bb4ad1 100644
--- a/tensorflow/python/ops/ragged/ragged_gather_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_gather_op_test.py
@@ -17,7 +17,6 @@
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
-
 from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -25,8 +24,8 @@ from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops.ragged import ragged_array_ops
 from tensorflow.python.ops.ragged import ragged_factory_ops
+from tensorflow.python.ops.ragged import ragged_gather_ops
 from tensorflow.python.ops.ragged import ragged_test_util
 from tensorflow.python.platform import googletest
 
@@ -41,35 +40,35 @@ class RaggedGatherOpTest(ragged_test_util.RaggedTensorTestCase):
                                                  ['e']])
     ragged_indices = ragged_factory_ops.constant([[3, 1, 2], [1], [], [0]])
     self.assertRaggedEqual(
-        ragged_array_ops.gather(params, ragged_indices),
+        ragged_gather_ops.gather(params, ragged_indices),
         [[b'd', b'b', b'c'], [b'b'], [], [b'a']])
     self.assertRaggedEqual(
-        ragged_array_ops.gather(ragged_params, indices),
+        ragged_gather_ops.gather(ragged_params, indices),
         [[b'e'], [b'd'], [], [b'd'], [b'a', b'b', b'c']])
     self.assertRaggedEqual(
-        ragged_array_ops.gather(ragged_params, ragged_indices),
+        ragged_gather_ops.gather(ragged_params, ragged_indices),
         [[[b'e'], [b'd'], []], [[b'd']], [], [[b'a', b'b', b'c']]])
 
   def testTensorParamsAndTensorIndices(self):
     params = ['a', 'b', 'c', 'd', 'e']
     indices = [2, 0, 2, 1]
     self.assertRaggedEqual(
-        ragged_array_ops.gather(params, indices), [b'c', b'a', b'c', b'b'])
-    self.assertIsInstance(ragged_array_ops.gather(params, indices), ops.Tensor)
+        ragged_gather_ops.gather(params, indices), [b'c', b'a', b'c', b'b'])
+    self.assertIsInstance(ragged_gather_ops.gather(params, indices), ops.Tensor)
 
   def testRaggedParamsAndTensorIndices(self):
     params = ragged_factory_ops.constant([['a', 'b'], ['c', 'd', 'e'], ['f'],
                                           [], ['g']])
     indices = [2, 0, 2, 1]
     self.assertRaggedEqual(
-        ragged_array_ops.gather(params, indices),
+        ragged_gather_ops.gather(params, indices),
         [[b'f'], [b'a', b'b'], [b'f'], [b'c', b'd', b'e']])
 
   def testTensorParamsAndRaggedIndices(self):
     params = ['a', 'b', 'c', 'd', 'e']
     indices = ragged_factory_ops.constant([[2, 1], [1, 2, 0], [3]])
     self.assertRaggedEqual(
-        ragged_array_ops.gather(params, indices),
+        ragged_gather_ops.gather(params, indices),
         [[b'c', b'b'], [b'b', b'c', b'a'], [b'd']])
 
   def testRaggedParamsAndRaggedIndices(self):
@@ -77,7 +76,7 @@ class RaggedGatherOpTest(ragged_test_util.RaggedTensorTestCase):
                                           [], ['g']])
     indices = ragged_factory_ops.constant([[2, 1], [1, 2, 0], [3]])
     self.assertRaggedEqual(
-        ragged_array_ops.gather(params, indices),
+        ragged_gather_ops.gather(params, indices),
         [[[b'f'], [b'c', b'd', b'e']],                # [[p[2], p[1]      ],
          [[b'c', b'd', b'e'], [b'f'], [b'a', b'b']],  #  [p[1], p[2], p[0]],
          [[]]]                                        #  [p[3]            ]]
@@ -88,14 +87,14 @@ class RaggedGatherOpTest(ragged_test_util.RaggedTensorTestCase):
                                           [], ['g']])
     indices = 1
     self.assertRaggedEqual(
-        ragged_array_ops.gather(params, indices), [b'c', b'd', b'e'])
+        ragged_gather_ops.gather(params, indices), [b'c', b'd', b'e'])
 
   def test3DRaggedParamsAnd2DTensorIndices(self):
     params = ragged_factory_ops.constant([[['a', 'b'], []],
                                           [['c', 'd'], ['e'], ['f']], [['g']]])
     indices = [[1, 2], [0, 1], [2, 2]]
     self.assertRaggedEqual(
-        ragged_array_ops.gather(params, indices),
+        ragged_gather_ops.gather(params, indices),
         [[[[b'c', b'd'], [b'e'], [b'f']], [[b'g']]],            # [[p1, p2],
          [[[b'a', b'b'], []], [[b'c', b'd'], [b'e'], [b'f']]],  #  [p0, p1],
          [[[b'g']], [[b'g']]]]                                  #  [p2, p2]]
@@ -109,7 +108,7 @@ class RaggedGatherOpTest(ragged_test_util.RaggedTensorTestCase):
         inner_shape=(2,))
     params = ['a', 'b', 'c', 'd', 'e', 'f', 'g']
     self.assertRaggedEqual(
-        ragged_array_ops.gather(params, indices),
+        ragged_gather_ops.gather(params, indices),
         [[[[b'd', b'e'], [b'a', b'g']], []],
          [[[b'c', b'b'], [b'b', b'a']], [[b'c', b'f']], [[b'c', b'd']]],
          [[[b'b', b'a']]]])  # pyformat: disable
@@ -121,13 +120,13 @@ class RaggedGatherOpTest(ragged_test_util.RaggedTensorTestCase):
     ragged_indices = ragged_factory_ops.constant([[0, 3]])
     with self.assertRaisesRegexp(errors.InvalidArgumentError,
                                  r'indices\[1\] = 3 is not in \[0, 3\)'):
-      self.evaluate(ragged_array_ops.gather(tensor_params, ragged_indices))
+      self.evaluate(ragged_gather_ops.gather(tensor_params, ragged_indices))
     with self.assertRaisesRegexp(errors.InvalidArgumentError,
                                  r'indices\[2\] = 2 is not in \[0, 2\)'):
-      self.evaluate(ragged_array_ops.gather(ragged_params, tensor_indices))
+      self.evaluate(ragged_gather_ops.gather(ragged_params, tensor_indices))
     with self.assertRaisesRegexp(errors.InvalidArgumentError,
                                  r'indices\[1\] = 3 is not in \[0, 2\)'):
-      self.evaluate(ragged_array_ops.gather(ragged_params, ragged_indices))
+      self.evaluate(ragged_gather_ops.gather(ragged_params, ragged_indices))
 
   def testUnknownIndicesRankError(self):
     if context.executing_eagerly():
@@ -137,7 +136,7 @@ class RaggedGatherOpTest(ragged_test_util.RaggedTensorTestCase):
     indices = array_ops.placeholder_with_default(indices, None)
     self.assertRaisesRegexp(ValueError,
                             r'indices\.shape\.ndims must be known statically',
-                            ragged_array_ops.gather, params, indices)
+                            ragged_gather_ops.gather, params, indices)
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/ops/ragged/ragged_gather_ops.py b/tensorflow/python/ops/ragged/ragged_gather_ops.py
new file mode 100644
index 0000000000..b49e0e549f
--- /dev/null
+++ b/tensorflow/python/ops/ragged/ragged_gather_ops.py
@@ -0,0 +1,258 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Gather operations for RaggedTensors."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_shape
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import gen_ragged_array_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops.ragged import ragged_array_ops
+from tensorflow.python.ops.ragged import ragged_conversion_ops
+from tensorflow.python.ops.ragged import ragged_tensor
+
+
+#===============================================================================
+# ragged_gather
+#===============================================================================
+# TODO(edloper): Add an `axis` argument
+def gather(params, indices, validate_indices=None, axis=0, batch_dims=0,
+           name=None):
+  """Gathers ragged slices from `params` axis `0` according to `indices`.
+
+  Returns `RaggedTensor` output, such that:
+
+  ```python
+  output.shape = indices.shape + params.shape[1:]
+  output.ragged_rank = indices.shape.ndims + params.ragged_rank
+  output[i...j, d0...dn] = params[indices[i...j], d0...dn]
+  ```
+
+  `params` may be ragged.  `indices` may be ragged.
+  `indices` must have dtype `int32` or `int64`. If any index is out of bounds,
+  then an error is returned.
+
+  Examples:
+
+  ```python
+  >>> params = tf.constant(['a', 'b', 'c', 'd', 'e'])
+  >>> indices = tf.constant([3, 1, 2, 1, 0])
+  >>> ragged_params = tf.ragged.constant([['a', 'b', 'c'], ['d'], [], ['e']])
+  >>> ragged_indices = tf.ragged.constant([[3, 1, 2], [1], [], [0]])
+
+  >>> print ragged.gather(params, ragged_indices)
+  [['d', 'b', 'c'], ['b'], [], ['a']]
+
+  >>> print ragged.gather(ragged_params, indices)
+  [['e'], ['d'], [], ['d'], ['a', 'b', 'c']]
+
+  >>> print ragged.gather(ragged_params, ragged_indices)
+  [[['e'], ['d'], []], [['d']], [], [['a', 'b', 'c']]]
+  ```
+
+  Args:
+    params: The potentially ragged tensor from which to gather values. Must be
+      at least rank 1.
+    indices: The potentially ragged tensor indicating which values to gather.
+      Must have dtype `int32` or `int64`.  Values must be in the range `[0,
+      params.shape[0]]`.
+    validate_indices: Ignored.
+    axis: Must be zero.
+    batch_dims: Must be zero.
+    name: A name for the operation (optional).
+
+  Returns:
+    A `RaggedTensor`, where `output.dtype=params.dtype` and
+    `output.shape=indices.shape + params.shape[1:]` and
+    `output.ragged_rank=indices.shape.ndims + params.ragged_rank`.
+
+  Raises:
+    ValueError: If indices.shape.ndims is not known statically.
+  """
+  del validate_indices
+  if not isinstance(axis, int) or axis != 0:
+    raise ValueError('axis != 0 is not supported for ragged gather yet.')
+  if not isinstance(batch_dims, int) or batch_dims != 0:
+    raise ValueError('batch_dims != 0 is not supported for ragged gather yet.')
+  with ops.name_scope(name, 'RaggedGather', [params, indices]):
+    params = ragged_tensor.convert_to_tensor_or_ragged_tensor(
+        params, name='params')
+    indices = ragged_tensor.convert_to_tensor_or_ragged_tensor(
+        indices, name='indices')
+
+    if ragged_tensor.is_ragged(indices):
+      return indices.with_values(gather(params, indices.values))
+
+    if not ragged_tensor.is_ragged(params):
+      return array_ops.gather(params, indices)
+
+    indices = ops.convert_to_tensor(indices)
+    if indices.shape.ndims is None:
+      raise ValueError('indices.shape.ndims must be known statically')
+
+    result = gen_ragged_array_ops.ragged_gather(
+        indices=indices,
+        params_dense_values=params.flat_values,
+        params_nested_splits=params.nested_row_splits,
+        OUTPUT_RAGGED_RANK=indices.shape.ndims + len(params.nested_row_splits) -
+        1)
+
+    # Compose the RaggedTensor from splits & values.
+    return ragged_tensor.RaggedTensor.from_nested_row_splits(
+        result.output_dense_values, result.output_nested_splits)
+
+
+#===============================================================================
+# ragged.gather_nd
+#===============================================================================
+def gather_nd(params, indices, name=None):
+  """Gather slices from `params` using `n`-dimensional indices.
+
+  This operation is similar to `gather`, but it uses the innermost dimension
+  of `indices` to define a slice into `params`.  In particular, if:
+
+  * `indices` has shape `[A1...AN, I]`
+  * `params` has shape `[B1...BM]`
+
+  Then:
+
+  * `result` has shape `[A1...AN, B_{I+1}...BM]`.
+  * `result[a1...aN] = params[indices[a1...aN, :]]`
+
+  Args:
+    params: A potentially ragged tensor with shape `[A1...AN, I]`.
+    indices: A potentially ragged tensor with shape `[B1...BM]`.
+    name: A name for the operation (optional).
+
+  Returns:
+    A potentially ragged tensor with shape `[A1...AN, B_{I+1}...BM]`.
+
+  #### Examples:
+    ```python
+    >>> params = tf.ragged.constant_value(
+    ...     [ [ ['000', '001'], ['010'              ]          ],
+    ...       [ ['100'       ], ['110', '111', '112'], ['120'] ],
+    ...       [ [            ], ['210'              ]          ] ])
+
+    >>> # Gather 2D slices from a 3D tensor
+    >>> ragged.gather_nd(params, [[2], [0]])
+    [ [ [            ], ['210'] ]
+      [ ['000', '001'], ['010'] ] ]
+
+    >>> # Gather 1D slices from a 3D tensor
+    >>> ragged.gather_nd(params, [[2, 1], [0, 0]])
+    [['210'], ['000', '001']]
+
+    >>> # Gather scalars from a 3D tensor
+    >>> ragged.gather_nd(params, [[0, 0, 1], [1, 1, 2]])
+    ['001', '112']
+    ```
+  """
+  if not (ragged_tensor.is_ragged(params) or ragged_tensor.is_ragged(indices)):
+    return array_ops.gather_nd(params, indices, name)
+
+  with ops.name_scope(name, 'RaggedGatherNd', [params, indices]):
+
+    params = ragged_tensor.convert_to_tensor_or_ragged_tensor(
+        params, name='params')
+    indices = ragged_tensor.convert_to_tensor_or_ragged_tensor(
+        indices, name='indices')
+    indices_shape = indices.shape
+    indices_ndims = indices_shape.ndims
+    if indices_ndims is None:
+      raise ValueError('indices.rank be statically known.')
+    if indices_ndims == 0:
+      raise ValueError('indices.rank must be at least 1.')
+    if (ragged_tensor.is_ragged(indices) and
+        indices_ndims == indices.ragged_rank + 1):
+      raise ValueError('The innermost dimension of indices may not be ragged')
+
+    # `index_size` is the "n" in "gather_nd" -- i.e., the number of dimensions
+    # that each index slices into.
+    index_size = tensor_shape.dimension_value(indices_shape[-1])
+    if index_size is None:
+      raise ValueError('indices.shape[-1] must be statically known.')
+
+    # If `indices` has more than 2 dimensions, then recurse.  If `indices` is
+    # dense, then we convert it to ragged before recursing, and then convert
+    # the result back to `dense` if appropriate.
+    if indices_ndims > 2:
+      indices_is_dense = not ragged_tensor.is_ragged(indices)
+      if indices_is_dense:
+        indices = ragged_conversion_ops.from_tensor(
+            indices, ragged_rank=indices_ndims - 2)
+      result = indices.with_flat_values(gather_nd(params, indices.flat_values))
+      if (indices_is_dense and ragged_tensor.is_ragged(result) and
+          result.ragged_rank == indices_ndims - 2):
+        result = ragged_conversion_ops.to_tensor(result)
+      return result
+
+    # indices_ndims <= 2, and the innermost dimension of indices may not be
+    # ragged, so `indices` must not be ragged.
+    assert not ragged_tensor.is_ragged(indices)
+    assert ragged_tensor.is_ragged(params)
+
+    # Handle corner case: An empty index tuple selects the entire `params`
+    # value.  So if `index_size` is zero, then tile `params`.
+    if index_size == 0:
+      params_ndims = params.ragged_rank + array_ops.rank(params.flat_values)
+      for dim in range(indices_ndims - 1):
+        params = ragged_array_ops.expand_dims(params, axis=0)
+      multiples = array_ops.concat([
+          array_ops.shape(indices)[:-1],
+          array_ops.ones([params_ndims], dtypes.int32)
+      ],
+                                   axis=0)
+      return ragged_array_ops.tile(params, multiples)
+
+    # When index_size=1, we can just flatten the index tuples and use gather.
+    elif index_size == 1:
+      flattened_index_tuples = array_ops.reshape(indices, [-1])
+      return gather(params, flattened_index_tuples)
+
+    # Otherwise, params is a RaggedTensor, and indices is a 1D or 2D Tensor.
+    # Flatten both the index tuples and the params, such that the flattened
+    # index tuples point to the correct values in the flattened params; and
+    # then use ragged.gather on the flattened index tuples & params.
+    else:
+      indices = math_ops.to_int64(indices)
+
+      # Flatten the outermost 2 dimensions of the index tuples & params.
+      flattened_index_tuples = array_ops.gather(params.row_splits,
+                                                indices[..., 0])
+      flattened_index_tuples += indices[..., 1]
+      flattened_params = params.values
+
+      # Flatten any remaining dimensions.
+      for dim in range(2, index_size):
+        if not ragged_tensor.is_ragged(flattened_params):
+          flattened_index_tuples = array_ops.expand_dims(
+              flattened_index_tuples, axis=1)
+          flattened_index_tuples = array_ops.concat(
+              [flattened_index_tuples, indices[..., dim:]], axis=1)
+          return array_ops.gather_nd(flattened_params, flattened_index_tuples)
+
+        flattened_index_tuples = array_ops.gather(
+            flattened_params.row_starts(), flattened_index_tuples)
+        flattened_index_tuples += indices[..., dim]
+        flattened_params = flattened_params.values
+
+      # Gather using the flattened index tuples and params.
+      return gather(flattened_params, flattened_index_tuples)
diff --git a/tensorflow/python/ops/ragged/ragged_getitem.py b/tensorflow/python/ops/ragged/ragged_getitem.py
index 001a400596..1f1bb51253 100644
--- a/tensorflow/python/ops/ragged/ragged_getitem.py
+++ b/tensorflow/python/ops/ragged/ragged_getitem.py
@@ -23,7 +23,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
-from tensorflow.python.ops.ragged import ragged_array_ops
+from tensorflow.python.ops.ragged import ragged_gather_ops
 from tensorflow.python.ops.ragged import ragged_math_ops
 from tensorflow.python.ops.ragged import ragged_tensor
 
@@ -344,7 +344,7 @@ def _build_ragged_tensor_from_value_ranges(starts, limits, step, values):
 
   # Use `ragged_gather` or `array_ops.gather` to collect the values.
   if isinstance(values, ragged_tensor.RaggedTensor):
-    gathered_values = ragged_array_ops.gather(
+    gathered_values = ragged_gather_ops.gather(
         params=values, indices=value_indices.values)
   else:
     gathered_values = array_ops.gather(
diff --git a/tensorflow/python/ops/ragged/ragged_stack_op_test.py b/tensorflow/python/ops/ragged/ragged_stack_op_test.py
index b2b96a9465..e52ad4de20 100644
--- a/tensorflow/python/ops/ragged/ragged_stack_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_stack_op_test.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for ragged_array_ops.stack."""
+"""Tests for ragged_concat_ops.stack."""
 
 from __future__ import absolute_import
 from __future__ import division
@@ -22,7 +22,7 @@ from absl.testing import parameterized
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import test_util
-from tensorflow.python.ops.ragged import ragged_array_ops
+from tensorflow.python.ops.ragged import ragged_concat_ops
 from tensorflow.python.ops.ragged import ragged_factory_ops
 from tensorflow.python.ops.ragged import ragged_test_util
 from tensorflow.python.platform import googletest
@@ -330,7 +330,7 @@ class RaggedStackOpTest(ragged_test_util.RaggedTensorTestCase,
         if rrank != 0 else constant_op.constant(rt_input)
         for (rt_input, rrank) in zip(rt_inputs, ragged_ranks)
     ]
-    stacked = ragged_array_ops.stack(rt_inputs, axis)
+    stacked = ragged_concat_ops.stack(rt_inputs, axis)
     if expected_ragged_rank is not None:
       self.assertEqual(stacked.ragged_rank, expected_ragged_rank)
     if expected_shape is not None:
@@ -360,7 +360,7 @@ class RaggedStackOpTest(ragged_test_util.RaggedTensorTestCase,
           message='axis=3 out of bounds: expected -3<=axis<3'),
   )
   def testError(self, rt_inputs, axis, error, message):
-    self.assertRaisesRegexp(error, message, ragged_array_ops.stack, rt_inputs,
+    self.assertRaisesRegexp(error, message, ragged_concat_ops.stack, rt_inputs,
                             axis)
 
   def testSingleTensorInput(self):
@@ -371,7 +371,7 @@ class RaggedStackOpTest(ragged_test_util.RaggedTensorTestCase,
     equivalent to expand_dims(axis=0).  This test exercises that path.
     """
     rt_inputs = ragged_factory_ops.constant([[1, 2], [3, 4]])
-    stacked = ragged_array_ops.stack(rt_inputs, 0)
+    stacked = ragged_concat_ops.stack(rt_inputs, 0)
     self.assertRaggedEqual(stacked, [[[1, 2], [3, 4]]])
 
 
diff --git a/tensorflow/python/ops/ragged/ragged_where_op.py b/tensorflow/python/ops/ragged/ragged_where_op.py
new file mode 100644
index 0000000000..d60ee49ee8
--- /dev/null
+++ b/tensorflow/python/ops/ragged/ragged_where_op.py
@@ -0,0 +1,166 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""where operation for RaggedTensors."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops.ragged import ragged_concat_ops
+from tensorflow.python.ops.ragged import ragged_functional_ops
+from tensorflow.python.ops.ragged import ragged_gather_ops
+from tensorflow.python.ops.ragged import ragged_tensor
+
+
+def where(condition, x=None, y=None, name=None):
+  """Return the elements, either from `x` or `y`, depending on the `condition`.
+
+  : If both `x` and `y` are `None`:
+    Returns the coordinates of true elements of `condition`. The coordinates
+    are returned in a 2-D tensor with shape
+    `[num_true_values, dim_size(condition)]`, where `result[i]` is the
+    coordinates of the `i`th true value (in row-major order).
+
+  : If both `x` and `y` are non-`None`:
+    Returns a tensor formed by selecting values from `x` where condition is
+    true, and from `y` when condition is false.  In particular:
+
+    : If `condition`, `x`, and `y` all have the same shape:
+
+      * `result[i1...iN] = x[i1...iN]` if `condition[i1...iN]` is true.
+      * `result[i1...iN] = y[i1...iN]` if `condition[i1...iN]` is false.
+
+    : Otherwise:
+
+      * `condition` must be a vector.
+      * `x` and `y` must have the same number of dimensions.
+      * The outermost dimensions of `condition`, `x`, and `y` must all have the
+        same size.
+      * `result[i] = x[i]` if `condition[i]` is true.
+      * `result[i] = y[i]` if `condition[i]` is false.
+
+  Args:
+    condition: A potentially ragged tensor of type `bool`
+    x: A potentially ragged tensor (optional).
+    y: A potentially ragged tensor (optional).  Must be specified if `x` is
+      specified.  Must have the same rank and type as `x`.
+    name: A name of the operation (optional)
+
+  Returns:
+    : If both `x` and `y` are `None`:
+      A `Tensor` with shape `(num_true, dim_size(condition))`.
+    : Otherwise:
+      A potentially ragged tensor with the same type, rank, and outermost
+      dimension size as `x` and `y`.
+      `result.ragged_rank = max(x.ragged_rank, y.ragged_rank)`.
+
+  Raises:
+    ValueError: When exactly one of `x` or `y` is non-`None`; or when
+      `condition`, `x`, and `y` have incompatible shapes.
+
+  #### Examples:
+    ```python
+    >>> # Coordinates where condition is true.
+    >>> condition = tf.ragged.constant_value(
+    ...     [[True, False, True], [False, True]])
+    >>> ragged.where(condition)
+    [[0, 0], [0, 2], [1, 1]]
+
+    >>> # Elementwise selection between x and y, based on condition.
+    >>> condition = tf.ragged.constant_value(
+    ...     [[True, False, True], [False, True]])
+    >>> x = tf.ragged.constant_value([['A', 'B', 'C'], ['D', 'E']])
+    >>> y = tf.ragged.constant_value([['a', 'b', 'c'], ['d', 'e']])
+    >>> ragged.where(condition, x, y)
+    [['A', 'b', 'C'], ['d', 'E']]
+
+    >>> # Row selection between x and y, based on condition.
+    >>> condition = [True, False]
+    >>> x = tf.ragged.constant_value([['A', 'B', 'C'], ['D', 'E']])
+    >>> y = tf.ragged.constant_value([['a', 'b', 'c'], ['d', 'e']])
+    >>> ragged.where(condition, x, y)
+    [['A', 'B', 'C'], ['d', 'e']]
+    ```
+  """
+  if (x is None) != (y is None):
+    raise ValueError('x and y must be either both None or both non-None')
+  with ops.name_scope('RaggedWhere', name, [condition, x, y]):
+    condition = ragged_tensor.convert_to_tensor_or_ragged_tensor(
+        condition, name='condition')
+    if x is None:
+      return _coordinate_where(condition)
+    else:
+      x = ragged_tensor.convert_to_tensor_or_ragged_tensor(x, name='x')
+      y = ragged_tensor.convert_to_tensor_or_ragged_tensor(y, name='y')
+      return _elementwise_where(condition, x, y)
+
+
+def _elementwise_where(condition, x, y):
+  """Ragged version of tf.where(condition, x, y)."""
+  condition_is_ragged = isinstance(condition, ragged_tensor.RaggedTensor)
+  x_is_ragged = isinstance(x, ragged_tensor.RaggedTensor)
+  y_is_ragged = isinstance(y, ragged_tensor.RaggedTensor)
+
+  if not (condition_is_ragged or x_is_ragged or y_is_ragged):
+    return array_ops.where(condition, x, y)
+
+  elif condition_is_ragged and x_is_ragged and y_is_ragged:
+    return ragged_functional_ops.map_flat_values(array_ops.where, condition, x,
+                                                 y)
+  elif not condition_is_ragged:
+    # Concatenate x and y, and then use `gather` to assemble the selected rows.
+    condition.shape.assert_has_rank(1)
+    x_nrows = _nrows(x)
+    x_and_y = ragged_concat_ops.concat([x, y], axis=0)
+    indices = array_ops.where(condition, math_ops.range(x_nrows),
+                              x_nrows + math_ops.range(_nrows(y)))
+    return ragged_gather_ops.gather(x_and_y, indices)
+
+  else:
+    raise ValueError('Input shapes do not match.')
+
+
+def _coordinate_where(condition):
+  """Ragged version of tf.where(condition)."""
+  if not isinstance(condition, ragged_tensor.RaggedTensor):
+    return array_ops.where(condition)
+
+  # The coordinate for each `true` value in condition.values.
+  selected_coords = _coordinate_where(condition.values)
+
+  # Convert the first index in each coordinate to a row index and column index.
+  first_index = selected_coords[:, 0]
+  selected_rows = array_ops.gather(condition.value_rowids(), first_index)
+  selected_row_starts = array_ops.gather(condition.row_splits, selected_rows)
+  selected_cols = first_index - selected_row_starts
+
+  # Assemble the row & column index with the indices for inner dimensions.
+  return array_ops.concat([
+      array_ops.expand_dims(selected_rows, 1),
+      array_ops.expand_dims(selected_cols, 1), selected_coords[:, 1:]
+  ],
+                          axis=1)
+
+
+def _nrows(rt_input, out_type=dtypes.int64, name=None):
+  if isinstance(rt_input, ragged_tensor.RaggedTensor):
+    return rt_input.nrows(out_type=out_type, name=name)
+  else:
+    with ops.name_scope(name, 'RaggedNRows', [rt_input]):
+      return array_ops.shape(rt_input, out_type=out_type)[0]
diff --git a/tensorflow/python/ops/ragged/ragged_where_op_test.py b/tensorflow/python/ops/ragged/ragged_where_op_test.py
index 3dd9565826..e76a04072a 100644
--- a/tensorflow/python/ops/ragged/ragged_where_op_test.py
+++ b/tensorflow/python/ops/ragged/ragged_where_op_test.py
@@ -17,13 +17,11 @@
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
-
 from absl.testing import parameterized
-
 from tensorflow.python.framework import test_util
-from tensorflow.python.ops.ragged import ragged_array_ops
 from tensorflow.python.ops.ragged import ragged_factory_ops
 from tensorflow.python.ops.ragged import ragged_test_util
+from tensorflow.python.ops.ragged import ragged_where_op
 from tensorflow.python.platform import googletest
 
 
@@ -182,7 +180,7 @@ class RaggedWhereOpTest(ragged_test_util.RaggedTensorTestCase,
               [[[[], [b'A']]], [[[b'b']]]])),
   ])   # pyformat: disable
   def testRaggedWhere(self, condition, expected, x=None, y=None):
-    result = ragged_array_ops.where(condition, x, y)
+    result = ragged_where_op.where(condition, x, y)
     self.assertRaggedEqual(result, expected)
 
   @parameterized.parameters([
@@ -201,7 +199,7 @@ class RaggedWhereOpTest(ragged_test_util.RaggedTensorTestCase,
   ])
   def testRaggedWhereErrors(self, condition, error, message, x=None, y=None):
     with self.assertRaisesRegexp(error, message):
-      ragged_array_ops.where(condition, x, y)
+      ragged_where_op.where(condition, x, y)
 
 
 if __name__ == '__main__':
-- 
GitLab


From ff088a9e52e9085025c09e50a48fa9be7606f6da Mon Sep 17 00:00:00 2001
From: Dimitris Vardoulakis <dimvar@google.com>
Date: Thu, 17 Jan 2019 23:15:50 -0800
Subject: [PATCH 0955/1095] Fix some Clang-Tidy findings.

PiperOrigin-RevId: 229879297
---
 tensorflow/compiler/xla/service/heap_simulator.h              | 2 +-
 tensorflow/compiler/xla/service/pattern_matcher_gmock_test.cc | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/tensorflow/compiler/xla/service/heap_simulator.h b/tensorflow/compiler/xla/service/heap_simulator.h
index dbbf43082f..3e0631aeb4 100644
--- a/tensorflow/compiler/xla/service/heap_simulator.h
+++ b/tensorflow/compiler/xla/service/heap_simulator.h
@@ -158,7 +158,7 @@ class HeapSimulator {
   void FillDebugTrace(HeapSimulatorTrace::Event::Kind kind,
                       const BufferValue* buffer,
                       const HloInstruction* instruction,
-                      const BufferValue* shared_with_canonical);
+                      const BufferValue* share_with_canonical);
 
   // Counterintuitive: the algorithm_ itself can be a NoFragmentationStatsHeap,
   // in which case we are calculating the same allocs/frees twice in the
diff --git a/tensorflow/compiler/xla/service/pattern_matcher_gmock_test.cc b/tensorflow/compiler/xla/service/pattern_matcher_gmock_test.cc
index 9ca2fb05c1..f51a18b138 100644
--- a/tensorflow/compiler/xla/service/pattern_matcher_gmock_test.cc
+++ b/tensorflow/compiler/xla/service/pattern_matcher_gmock_test.cc
@@ -23,7 +23,6 @@ namespace xla {
 namespace {
 
 namespace m = ::xla::match;
-using ::testing::Eq;
 using ::testing::Not;
 
 template <typename MatchedTy>
-- 
GitLab


From dda435ef8413ac47cfd8992bb5eba912ab0cd281 Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Fri, 18 Jan 2019 00:49:06 -0800
Subject: [PATCH 0956/1095] Improve key-value sort in the HloEvaluator.

Instead of doing n sorts for n sort operands, do only one (indirect) sort of
indices, and then use them to extract the values from the literals in the
sorted order. We avoid having to use n template arguments by using the
functionality of Literal to abstract away the native type.

PiperOrigin-RevId: 229886785
---
 .../compiler/xla/service/hlo_evaluator.cc     | 296 +++++++++---------
 1 file changed, 148 insertions(+), 148 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc
index 7589c99202..1d9af45745 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc
@@ -18,9 +18,9 @@ limitations under the License.
 #include <cmath>
 #include <cstdlib>
 #include <functional>
+#include <iterator>
 #include <string>
 #include <type_traits>
-#include <utility>
 #include <vector>
 
 #include "absl/algorithm/container.h"
@@ -1327,141 +1327,51 @@ Status HloEvaluator::HandleWhile(HloInstruction* while_hlo) {
   return Status::OK();
 }
 
-// Key-value sort is a special snowflake: it's templated on two different
-// element types, one for the keys, and one for the values. Jump through some
-// hoops to make this work.
 namespace {
-template <typename KeyType, typename ValueType>
-StatusOr<Literal> EvaluateSortInternal(HloInstruction* sort,
-                                       const Literal& keys_literal,
-                                       const Literal& values_literal) {
-  auto rank = keys_literal.shape().rank();
-  TF_RET_CHECK(
-      ShapeUtil::SameDimensions(keys_literal.shape(), values_literal.shape()))
-      << "Sort keys and values must have the same dimensions";
-  TF_RET_CHECK(sort->operand_count() >= 2) << "Expected key-value sort";
-  // We need to sort an array of keys and an array of values, where the
-  // sorted order of the values is determined by the keys. The simplest(?)
-  // way to do this is to go to an array-of-pairs representation, sort the
-  // array using the keys, and then go back to pair-of-arrays.
-  VLOG(3) << "HandleSort keys_literal: " << keys_literal.ToString();
-  VLOG(3) << "HandleSort values_literal: " << values_literal.ToString();
-
-  if (rank == 0) {
-    // Nothing to sort.
-    return LiteralUtil::MakeTuple({&keys_literal, &values_literal});
-  }
-
-  Literal keys_result_literal(keys_literal.shape());
-  Literal values_result_literal(values_literal.shape());
-  std::vector<int64> zero_base(rank, 0);
-  std::vector<int64> increment(rank, 1);
-  int64 sort_dim = sort->dimensions(0);
-  int64 sort_dim_elements = keys_literal.shape().dimensions(sort_dim);
-  increment[sort_dim] = sort_dim_elements;
-  // Iterate through each dimension except 'sort_dim'.
-  TF_RETURN_IF_ERROR(ShapeUtil::ForEachIndexWithStatus(
-      keys_literal.shape(), zero_base,
-      AsInt64Slice(keys_literal.shape().dimensions()), increment,
-      [&](absl::Span<const int64> indices) -> StatusOr<bool> {
-        // Extract a slice from the keys and values literals that correspond to
-        // exactly the row in dimension 'sort_dim'.
-        std::vector<int64> limit_indices(indices.begin(), indices.end());
-        absl::c_for_each(limit_indices, [](int64& index) { ++index; });
-        limit_indices[sort_dim] = sort_dim_elements;
-        TF_ASSIGN_OR_RETURN(auto keys_to_sort,
-                            keys_literal.Slice(indices, limit_indices)
-                                .Reshape({sort_dim_elements}));
-        const auto& keys_data = keys_to_sort.data<KeyType>();
-        TF_ASSIGN_OR_RETURN(auto values_to_sort,
-                            values_literal.Slice(indices, limit_indices)
-                                .Reshape({sort_dim_elements}));
-        const auto& values_data = values_to_sort.data<ValueType>();
-        using kv_pair = std::pair<KeyType, ValueType>;
-        std::vector<kv_pair> key_value_vector;
-        key_value_vector.reserve(keys_data.size());
-        for (int i = 0; i < keys_data.size(); ++i) {
-          key_value_vector.push_back(
-              std::make_pair(keys_data[i], values_data[i]));
-        }
-        std::stable_sort(key_value_vector.begin(), key_value_vector.end(),
-                         [](const kv_pair& a, const kv_pair& b) {
-                           return SafeLess<KeyType>(a.first, b.first);
-                         });
-        std::vector<KeyType> result_keys;
-        // We use a InlinedVector here because we need to convert it to an
-        // absl::Span later, and this would not work with std::vector<bool>.
-        absl::InlinedVector<ValueType, 10> result_values;
-        for (const auto& key_value : key_value_vector) {
-          result_keys.push_back(key_value.first);
-          result_values.push_back(key_value.second);
-        }
-        Literal sorted_keys(ShapeUtil::MakeShape(
-            keys_literal.shape().element_type(), {sort_dim_elements}));
-        sorted_keys.PopulateR1(absl::Span<const KeyType>(result_keys));
-        Literal sorted_values(ShapeUtil::MakeShape(
-            values_literal.shape().element_type(), {sort_dim_elements}));
-        sorted_values.PopulateR1(absl::Span<const ValueType>(result_values));
-        std::vector<int64> slice_dimensions(rank, 1);
-        slice_dimensions[sort_dim] = sort_dim_elements;
-        std::vector<int64> start_indices(rank, 0);
-        TF_ASSIGN_OR_RETURN(auto sorted_keys_reshaped,
-                            sorted_keys.Reshape(slice_dimensions));
-        TF_RETURN_IF_ERROR(keys_result_literal.CopySliceFrom(
-            sorted_keys_reshaped, start_indices, indices, slice_dimensions));
-        TF_ASSIGN_OR_RETURN(auto sorted_values_reshaped,
-                            sorted_values.Reshape(slice_dimensions));
-        TF_RETURN_IF_ERROR(values_result_literal.CopySliceFrom(
-            sorted_values_reshaped, start_indices, indices, slice_dimensions));
-        return true;
-      }));
-
-  Literal result_tuple;
-  result_tuple =
-      LiteralUtil::MakeTuple({&keys_result_literal, &values_result_literal});
-  VLOG(3) << "HandleSort result_tuple: " << result_tuple.ToString();
-  return std::move(result_tuple);
-}
-
-template <typename KeyType>
-StatusOr<Literal> EvaluateSortCurried(HloInstruction* sort,
-                                      const Literal& keys_literal,
-                                      const Literal& values_literal) {
-  switch (values_literal.shape().element_type()) {
-    case PRED:
-      return EvaluateSortInternal<KeyType, bool>(sort, keys_literal,
-                                                 values_literal);
-    case F32:
-      return EvaluateSortInternal<KeyType, float>(sort, keys_literal,
-                                                  values_literal);
-    case U32:
-      return EvaluateSortInternal<KeyType, uint32>(sort, keys_literal,
-                                                   values_literal);
-    case S32:
-      return EvaluateSortInternal<KeyType, int32>(sort, keys_literal,
-                                                  values_literal);
-    case BF16:
-      return EvaluateSortInternal<KeyType, bfloat16>(sort, keys_literal,
-                                                     values_literal);
-    default:
-      return InvalidArgument("Unsupported type for Sort");
-  }
-}
-
-StatusOr<Literal> EvaluateSort(HloInstruction* sort,
-                               const Literal& keys_literal,
-                               const Literal& values_literal) {
-  switch (sort->operand(0)->shape().element_type()) {
-    case F32:
-      return EvaluateSortCurried<float>(sort, keys_literal, values_literal);
-    case U32:
-      return EvaluateSortCurried<uint32>(sort, keys_literal, values_literal);
-    case S32:
-      return EvaluateSortCurried<int32>(sort, keys_literal, values_literal);
-    case BF16:
-      return EvaluateSortCurried<bfloat16>(sort, keys_literal, values_literal);
+StatusOr<Literal> ExtractFromIndexPositions(const Literal& from,
+                                            absl::Span<int64 const> indices) {
+  PrimitiveType type = from.shape().element_type();
+  switch (type) {
+    case PRED: {
+      // We use a InlinedVector here because we need to convert it to an
+      // absl::Span later, and this would not work with std::vector<bool>.
+      absl::InlinedVector<bool, 10> values;
+      for (int64 index : indices) {
+        values.push_back(from.Get<bool>({index}));
+      }
+      return LiteralUtil::CreateR1<bool>(values);
+    }
+    case F32: {
+      std::vector<float> values;
+      for (int64 index : indices) {
+        values.push_back(from.Get<float>({index}));
+      }
+      return LiteralUtil::CreateR1<float>(values);
+    }
+    case U32: {
+      std::vector<uint32> values;
+      for (int64 index : indices) {
+        values.push_back(from.Get<uint32>({index}));
+      }
+      return LiteralUtil::CreateR1<uint32>(values);
+    }
+    case S32: {
+      std::vector<int32> values;
+      for (int64 index : indices) {
+        values.push_back(from.Get<int32>({index}));
+      }
+      return LiteralUtil::CreateR1<int32>(values);
+    }
+    case BF16: {
+      std::vector<bfloat16> values;
+      for (int64 index : indices) {
+        values.push_back(from.Get<bfloat16>({index}));
+      }
+      return LiteralUtil::CreateR1<bfloat16>(values);
+    }
     default:
-      return InvalidArgument("Unsupported type for Sort");
+      return InvalidArgument("Unsupported type for Sort: %s",
+                             PrimitiveType_Name(type));
   }
 }
 }  // namespace
@@ -1470,23 +1380,113 @@ Status HloEvaluator::HandleSort(HloInstruction* sort) {
   if (!sort->shape().IsTuple()) {
     return DefaultAction(sort);
   } else {
-    // This is a really stupid work-around for the fact it's hard to support a
-    // multi-value sort directly, due to the fact we need to template the
-    // evaluation function on all of the value types.
-    std::vector<Literal> sort_results_backing;
-    for (int64 i = 0; i < sort->operand_count(); ++i) {
-      auto result = EvaluateSort(sort, GetEvaluatedLiteralFor(sort->operand(0)),
-                                 GetEvaluatedLiteralFor(sort->operand(i)));
-      if (!result.ok()) {
-        return result.status();
+    TF_RET_CHECK(sort->operand_count() >= 2) << "Expected key-value sort";
+    for (int64 i = 1; i < sort->operand_count(); ++i) {
+      TF_RET_CHECK(ShapeUtil::SameDimensions(sort->operand(0)->shape(),
+                                             sort->operand(i)->shape()))
+          << "All Sort operands must have the same dimensions";
+    }
+
+    if (VLOG_IS_ON(3)) {
+      for (int64 i = 0; i < sort->operand_count(); ++i) {
+        VLOG(3) << "HandleSort operand " << i << " literal: "
+                << GetEvaluatedLiteralFor(sort->operand(i)).ToString();
       }
-      sort_results_backing.push_back(
-          std::move(result.ValueOrDie().DecomposeTuple()[1]));
     }
-    std::vector<const Literal*> sort_results;
-    absl::c_transform(sort_results_backing, std::back_inserter(sort_results),
+    Shape key_shape = sort->operand(0)->shape();
+    auto rank = key_shape.rank();
+    PrimitiveType keys_type = key_shape.element_type();
+    if (keys_type != F32 && keys_type != U32 && keys_type != S32 &&
+        keys_type != BF16) {
+      return InvalidArgument("Unsupported type for Sort: %s",
+                             PrimitiveType_Name(keys_type));
+    }
+    std::vector<Literal> result_literals;
+    result_literals.reserve(sort->operand_count());
+    for (int64 i = 0; i < sort->operand_count(); ++i) {
+      result_literals.emplace_back(sort->operand(i)->shape());
+    }
+    std::vector<int64> zero_base(rank, 0);
+    std::vector<int64> increment(rank, 1);
+    int64 sort_dim = sort->dimensions(0);
+    int64 sort_dim_elements = key_shape.dimensions(sort_dim);
+    increment[sort_dim] = sort_dim_elements;
+    // Iterate through each dimension except 'sort_dim'.
+    TF_RETURN_IF_ERROR(ShapeUtil::ForEachIndexWithStatus(
+        key_shape, zero_base, AsInt64Slice(key_shape.dimensions()), increment,
+        [&](absl::Span<const int64> indices) -> StatusOr<bool> {
+          // Extract a slice from each operand literal that corresponds to
+          // exactly the row in dimension 'sort_dim'.
+          std::vector<int64> limit_indices(indices.begin(), indices.end());
+          absl::c_for_each(limit_indices, [](int64& index) { ++index; });
+          limit_indices[sort_dim] = sort_dim_elements;
+          std::vector<Literal> literals_to_sort;
+          literals_to_sort.reserve(sort->operand_count());
+          for (int64 i = 0; i < sort->operand_count(); ++i) {
+            TF_ASSIGN_OR_RETURN(auto literal_to_sort,
+                                GetEvaluatedLiteralFor(sort->operand(i))
+                                    .Slice(indices, limit_indices)
+                                    .Reshape({sort_dim_elements}));
+            literals_to_sort.push_back(std::move(literal_to_sort));
+          }
+          std::vector<int64> indices_to_sort(sort_dim_elements);
+          std::iota(indices_to_sort.begin(), indices_to_sort.end(), 0);
+          std::stable_sort(
+              indices_to_sort.begin(), indices_to_sort.end(),
+              [keys_type, &literals_to_sort](int64 a, int64 b) {
+                switch (keys_type) {
+                  case F32: {
+                    auto key_lhs = literals_to_sort[0].Get<float>({a});
+                    auto key_rhs = literals_to_sort[0].Get<float>({b});
+                    return SafeLess(key_lhs, key_rhs);
+                  }
+                  case U32: {
+                    auto key_lhs = literals_to_sort[0].Get<uint32>({a});
+                    auto key_rhs = literals_to_sort[0].Get<uint32>({b});
+                    return SafeLess(key_lhs, key_rhs);
+                  }
+                  case S32: {
+                    auto key_lhs = literals_to_sort[0].Get<int32>({a});
+                    auto key_rhs = literals_to_sort[0].Get<int32>({b});
+                    return SafeLess(key_lhs, key_rhs);
+                  }
+                  case BF16: {
+                    auto key_lhs = literals_to_sort[0].Get<bfloat16>({a});
+                    auto key_rhs = literals_to_sort[0].Get<bfloat16>({b});
+                    return SafeLess(key_lhs, key_rhs);
+                  }
+                  default:
+                    // We should never reach here, because we checked earlier
+                    // that 'key_type' is one of the cases above.
+                    LOG(FATAL) << "Invalid key type in Sort: %s",
+                        PrimitiveType_Name(keys_type);
+                    return false;
+                }
+              });
+          std::vector<int64> slice_dimensions(rank, 1);
+          slice_dimensions[sort_dim] = sort_dim_elements;
+          std::vector<int64> start_indices(rank, 0);
+          for (int64 i = 0; i < sort->operand_count(); ++i) {
+            TF_ASSIGN_OR_RETURN(Literal sorted_literal,
+                                ExtractFromIndexPositions(literals_to_sort[i],
+                                                          indices_to_sort));
+            TF_ASSIGN_OR_RETURN(auto sorted_literal_reshaped,
+                                sorted_literal.Reshape(slice_dimensions));
+            TF_RETURN_IF_ERROR(result_literals[i].CopySliceFrom(
+                sorted_literal_reshaped, start_indices, indices,
+                slice_dimensions));
+          }
+          return true;
+        }));
+
+    std::vector<const Literal*> literal_ptrs;
+    absl::c_transform(result_literals, std::back_inserter(literal_ptrs),
                       [](const Literal& literal) { return &literal; });
-    evaluated_[sort] = LiteralUtil::MakeTuple(sort_results);
+
+    Literal result_tuple = LiteralUtil::MakeTuple(literal_ptrs);
+    VLOG(3) << "HandleSort result_tuple: " << result_tuple.ToString();
+
+    evaluated_[sort] = std::move(result_tuple);
     return Status::OK();
   }
 }
-- 
GitLab


From 43fb03cc5fb1d61c30e783fa5c771c626d29dd82 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 18 Jan 2019 01:03:34 -0800
Subject: [PATCH 0957/1095] compat: Update forward compatibility horizon to
 2019-01-18

PiperOrigin-RevId: 229888500
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index c558722061..ccc0735b6e 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 17)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 18)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From d29b2faa5fe9c79679171bcafe930b819f74dca5 Mon Sep 17 00:00:00 2001
From: Tamara Norman <tamaranorman@google.com>
Date: Fri, 18 Jan 2019 01:03:42 -0800
Subject: [PATCH 0958/1095] Test that arg positions are correct where used in
 the conversion script

PiperOrigin-RevId: 229888522
---
 tensorflow/tools/compatibility/tf_upgrade_v2_test.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index 23446e30b1..b160d4de08 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -293,6 +293,18 @@ class TestUpgrade(test_util.TensorFlowTestCase):
     visitor.private_map["tf.compat"] = ["v1", "v2"]
     traverse.traverse(tf.compat.v1, visitor)
 
+  def testPositionsMatchArgGiven(self):
+    full_dict = tf_upgrade_v2.TFAPIChangeSpec().function_arg_warnings
+    method_names = full_dict.keys()
+    for method in method_names:
+      # doesn't test methods on objects
+      if not method.startswith("*."):
+        args = full_dict[method].keys()
+        method = get_symbol_for_name(tf, method)
+        arg_spec = tf_inspect.getfullargspec(method)
+        for (arg, pos) in args:
+          self.assertEqual(arg_spec[0][pos], arg)
+
   def testReorderFileNeedsUpdate(self):
     reordered_function_names = (
         tf_upgrade_v2.TFAPIChangeSpec().reordered_function_names)
-- 
GitLab


From 9132586162e08666368eea1eed2f90d8cbb140af Mon Sep 17 00:00:00 2001
From: Andr? Susano Pinto <andresp@google.com>
Date: Fri, 18 Jan 2019 01:53:36 -0800
Subject: [PATCH 0959/1095] Register the gradients of imported functions and
 fix the FunctionDef graphs to point to them.

Note:
 This CL enables computing gradients through a nested imported object based saved
 model function. It assumes that "StatefulPartitionedCall" and "PartionedCall"
 ops always use the default gradient that can be computed by the loader when
 needed.
PiperOrigin-RevId: 229894692
---
 .../saved_model/function_deserialization.py   | 50 +++++++++++++++----
 tensorflow/python/saved_model/load_test.py    | 43 +++++++++++++++-
 2 files changed, 83 insertions(+), 10 deletions(-)

diff --git a/tensorflow/python/saved_model/function_deserialization.py b/tensorflow/python/saved_model/function_deserialization.py
index ec5214824e..f6427864bb 100644
--- a/tensorflow/python/saved_model/function_deserialization.py
+++ b/tensorflow/python/saved_model/function_deserialization.py
@@ -28,7 +28,9 @@ from tensorflow.python.framework import function_def_to_graph as function_def_li
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_spec
 from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.saved_model import nested_structure_coder
+from tensorflow.python.util import compat
 from tensorflow.python.util import nest
 
 
@@ -160,24 +162,29 @@ def load_function_def_library(library):
   Raises:
     ValueError: if functions dependencies have a cycle.
   """
-  # TODO(andresp): Look into restoring gradient function information.
   functions = {}
-  name_mapping = {}
+
   # Note: Use a new graph to allow function_def_to_graph to help validating
   # that the functions are loaded correctly. This is not possible to do
   # just in eager mode as there is no python API to find if a function has
   # been registered in eager. Note also that despite this the created
   # func_graphs can still be used in eager or in other graphs.
-  with ops.Graph().as_default() as import_graph:
-    for fdef in _sort_function_defs(library):
-      copy = _fix_fdef(fdef, name_mapping)
+  import_graph = ops.Graph()
+
+  for fdef in _sort_function_defs(library):
+    with import_graph.as_default():
+      copy = _fix_fdef(fdef, functions)
 
       func_graph = function_def_lib.function_def_to_graph(copy)
       func = function_lib.ConcreteFunction(func_graph)
       func.add_to_graph(import_graph)
 
-      name_mapping[fdef.signature.name] = func.name
       functions[fdef.signature.name] = func
+
+    # Also register the gradients in the current root context.
+    with ops.init_scope():
+      func._register_gradient()  # pylint: disable=protected-access
+
   return functions
 
 
@@ -215,14 +222,39 @@ def _sort_function_defs(library):
   return [reverse[x] for x in output]
 
 
-def _fix_fdef(orig_fdef, name_map):
+def _fix_fdef(orig_fdef, functions):
+  """Fixes a FunctionDef proto to be loaded in current context.
+
+  In particular, when loading a function library into an eager context, one
+  must rename the functions to avoid conflicts with existent functions.
+
+  Args:
+    orig_fdef: FunctionDef proto to fix. It is not modified.
+    functions: map from function name to a ConcreteFunction instance.
+
+  Returns:
+    A fixed copy of the original FunctionDef.
+  """
   fdef = function_pb2.FunctionDef()
   fdef.CopyFrom(orig_fdef)
-  fdef.signature.name = _clean_function_name(fdef.signature.name)
   for node_def in fdef.node_def:
+    if "_gradient_op_type" in node_def.attr:
+      if node_def.op in ["StatefulPartitionedCall", "PartitionedCall"]:
+        # TODO(andresp): This code assumes that the gradient registered for this
+        # function call is the default gradient for the function and not a
+        # custom one.
+        fname = node_def.attr["f"].func.name
+        node_def.attr["_gradient_op_type"].s = compat.as_bytes(
+            functions[fname]._gradient_name)  # pylint: disable=protected-access
+      else:
+        logging.warning("Importing a function (%s) with ops with custom "
+                        "gradients. Will likely fail if a gradient is "
+                        "requested.", fdef.signature.name)
     for _, attr_value in node_def.attr.items():
       if attr_value.func.name:
-        attr_value.func.name = name_map[attr_value.func.name]
+        attr_value.func.name = functions[attr_value.func.name].name
+
+  fdef.signature.name = _clean_function_name(fdef.signature.name)
   return fdef
 
 
diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index 280a26a0a5..1c999a625d 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -361,12 +361,53 @@ class LoadTest(test.TestCase):
     root.bias = bias
     root.g = g
     imported = self.cycle(root)
-    with backprop.GradientTape(watch_accessed_variables=True) as t:
+    with backprop.GradientTape() as t:
       x = constant_op.constant([3.5])
       loss = imported.g(x)
       grad = t.gradient(loss, [imported.weight, imported.bias])
       self.assertAllClose(grad, [3.5, 1.0])
 
+  def test_nested_backprop(self):
+    weight = variables.Variable(1., trainable=True)
+    bias = variables.Variable(0., trainable=True)
+
+    # Note: this function gets called from other function defs via a
+    # "PartitionedCall" op node.
+    @def_function.function(input_signature=[
+        tensor_spec.TensorSpec(None, dtypes.float32),
+        tensor_spec.TensorSpec(None, dtypes.float32)])
+    def mul(x, y):
+      return x * y
+
+    # Note: this function gets called from other function defs via a
+    # "StatefulPartitionedCall" op node.
+    @def_function.function(input_signature=[
+        tensor_spec.TensorSpec(None, dtypes.float32)])
+    def f(x):
+      return mul(weight.read_value(), x)
+
+    @def_function.function(input_signature=[
+        tensor_spec.TensorSpec(None, dtypes.float32)])
+    def g(x):
+      return f(x) + bias,
+
+    @def_function.function(input_signature=[
+        tensor_spec.TensorSpec(None, dtypes.float32)])
+    def h(x):
+      return g(x) + bias,
+
+    root = tracking.AutoCheckpointable()
+    root.weight = weight
+    root.bias = bias
+    root.g = h
+
+    imported = self.cycle(root)
+    with backprop.GradientTape() as t:
+      x = constant_op.constant([3.5])
+      loss = imported.g(x)
+    grad = t.gradient(loss, [imported.weight, imported.bias])
+    self.assertAllClose(grad, [3.5, 2.0])
+
   def test_callable(self):
     class M1(tracking.AutoCheckpointable):
 
-- 
GitLab


From 815d49bb5855f121707b2a5b03656a18e22062f6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 18 Jan 2019 02:02:47 -0800
Subject: [PATCH 0960/1095] Automated rollback of commit
 5c08b1f021e0a40fece750b5ebc0cf85b3281265

PiperOrigin-RevId: 229895719
---
 tensorflow/opensource_only.files            |   3 +
 third_party/gpus/crosstool/remote.BUILD.tpl |  10 ++
 third_party/gpus/cuda/remote.BUILD.tpl      | 110 ++++++++++++++++++++
 third_party/gpus/cuda_configure.bzl         |  17 +--
 third_party/tensorrt/build_defs.bzl.tpl     |   4 +-
 third_party/tensorrt/remote.BUILD.tpl       |   7 ++
 third_party/tensorrt/tensorrt_configure.bzl |  11 +-
 7 files changed, 148 insertions(+), 14 deletions(-)
 create mode 100644 third_party/gpus/crosstool/remote.BUILD.tpl
 create mode 100644 third_party/gpus/cuda/remote.BUILD.tpl
 create mode 100644 third_party/tensorrt/remote.BUILD.tpl

diff --git a/tensorflow/opensource_only.files b/tensorflow/opensource_only.files
index 9c4dd2f4b8..850b5f5ca2 100644
--- a/tensorflow/opensource_only.files
+++ b/tensorflow/opensource_only.files
@@ -82,6 +82,7 @@ tensorflow/third_party/gpus/crosstool/clang/bin/crosstool_wrapper_driver_is_not_
 tensorflow/third_party/gpus/crosstool/CROSSTOOL.tpl
 tensorflow/third_party/gpus/crosstool/CROSSTOOL_hipcc.tpl
 tensorflow/third_party/gpus/crosstool/LICENSE
+tensorflow/third_party/gpus/crosstool/remote.BUILD.tpl
 tensorflow/third_party/gpus/crosstool/windows/msvc_wrapper_for_nvcc.py.tpl
 tensorflow/third_party/gpus/crosstool/BUILD.tpl
 tensorflow/third_party/gpus/crosstool/BUILD
@@ -89,6 +90,7 @@ tensorflow/third_party/gpus/cuda/LICENSE
 tensorflow/third_party/gpus/cuda/BUILD.tpl
 tensorflow/third_party/gpus/cuda/BUILD.windows.tpl
 tensorflow/third_party/gpus/cuda/cuda_config.h.tpl
+tensorflow/third_party/gpus/cuda/remote.BUILD.tpl
 tensorflow/third_party/gpus/cuda/BUILD
 tensorflow/third_party/gpus/cuda/build_defs.bzl.tpl
 tensorflow/third_party/gpus/rocm/rocm_config.h.tpl
@@ -192,6 +194,7 @@ tensorflow/third_party/tensorrt/BUILD
 tensorflow/third_party/tensorrt/build_defs.bzl.tpl
 tensorflow/third_party/tensorrt/BUILD.tpl
 tensorflow/third_party/tensorrt/tensorrt_configure.bzl
+tensorflow/third_party/tensorrt/remote.BUILD.tpl
 tensorflow/third_party/kafka/config.patch
 tensorflow/third_party/kafka/BUILD
 tensorflow/third_party/android/BUILD
diff --git a/third_party/gpus/crosstool/remote.BUILD.tpl b/third_party/gpus/crosstool/remote.BUILD.tpl
new file mode 100644
index 0000000000..b2316331db
--- /dev/null
+++ b/third_party/gpus/crosstool/remote.BUILD.tpl
@@ -0,0 +1,10 @@
+# Description:
+#   Template for crosstool Build file to use a pre-generated config.
+licenses(["restricted"])
+
+package(default_visibility = ["//visibility:public"])
+
+alias(
+    name = "toolchain",
+    actual = "%{remote_cuda_repo}:toolchain",
+)
diff --git a/third_party/gpus/cuda/remote.BUILD.tpl b/third_party/gpus/cuda/remote.BUILD.tpl
new file mode 100644
index 0000000000..100c7bb7c4
--- /dev/null
+++ b/third_party/gpus/cuda/remote.BUILD.tpl
@@ -0,0 +1,110 @@
+# Description:
+#   Template for cuda Build file to use a pre-generated config.
+licenses(["restricted"])  # MPL2, portions GPL v3, LGPL v3, BSD-like
+
+package(default_visibility = ["//visibility:public"])
+
+config_setting(
+    name = "using_nvcc",
+    values = {
+        "define": "using_cuda_nvcc=true",
+    },
+)
+
+config_setting(
+    name = "using_clang",
+    values = {
+        "define": "using_cuda_clang=true",
+    },
+)
+
+# Equivalent to using_clang && -c opt.
+config_setting(
+    name = "using_clang_opt",
+    values = {
+        "define": "using_cuda_clang=true",
+        "compilation_mode": "opt",
+    },
+)
+
+config_setting(
+    name = "darwin",
+    values = {"cpu": "darwin"},
+    visibility = ["//visibility:public"],
+)
+
+config_setting(
+    name = "freebsd",
+    values = {"cpu": "freebsd"},
+    visibility = ["//visibility:public"],
+)
+
+alias(
+    name = "cuda_headers",
+    actual = "%{remote_cuda_repo}/cuda:cuda_headers",
+)
+
+alias(
+    name = "cudart_static",
+    actual = "%{remote_cuda_repo}/cuda:cudart_static",
+)
+
+alias(
+    name = "cuda_driver",
+    actual = "%{remote_cuda_repo}/cuda:cuda_driver",
+)
+
+alias(
+    name = "cudart",
+    actual = "%{remote_cuda_repo}/cuda:cudart",
+)
+
+alias(
+    name = "cublas",
+    actual = "%{remote_cuda_repo}/cuda:cublas",
+)
+
+alias(
+    name = "cusolver",
+    actual = "%{remote_cuda_repo}/cuda:cusolver",
+)
+
+alias(
+    name = "cudnn",
+    actual = "%{remote_cuda_repo}/cuda:cudnn",
+)
+
+alias(
+    name = "cudnn_header",
+    actual = "%{remote_cuda_repo}/cuda:cudnn_header",
+)
+
+alias(
+    name = "cufft",
+    actual = "%{remote_cuda_repo}/cuda:cufft",
+)
+
+alias(
+    name = "curand",
+    actual = "%{remote_cuda_repo}/cuda:curand",
+)
+
+alias(
+    name = "cuda",
+    actual = "%{remote_cuda_repo}/cuda:cuda",
+)
+
+alias(
+    name = "cupti_headers",
+    actual = "%{remote_cuda_repo}/cuda:cupti_headers",
+)
+
+alias(
+    name = "cupti_dsos",
+    actual = "%{remote_cuda_repo}/cuda:cupti_dsos",
+)
+
+alias(
+    name = "libdevice_root",
+    actual = "%{remote_cuda_repo}/cuda:libdevice_root",
+)
diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl
index 5c9873f334..90020b6608 100644
--- a/third_party/gpus/cuda_configure.bzl
+++ b/third_party/gpus/cuda_configure.bzl
@@ -1508,16 +1508,17 @@ def _create_remote_cuda_repository(repository_ctx, remote_config_repo):
               ),
       },
   )
-  repository_ctx.template(
+  _tpl(
+      repository_ctx,
+      "cuda:remote.BUILD",
+      {
+          "%{remote_cuda_repo}": remote_config_repo,
+      },
       "cuda/BUILD",
-      Label(remote_config_repo + "/cuda:BUILD"),
-      {},
-  )
-  repository_ctx.template(
-      "crosstool/BUILD",
-      Label(remote_config_repo + "/crosstool:BUILD"),
-      {},
   )
+  _tpl(repository_ctx, "crosstool:remote.BUILD", {
+      "%{remote_cuda_repo}": remote_config_repo,
+  }, "crosstool/BUILD")
 
 
 def _cuda_autoconf_impl(repository_ctx):
diff --git a/third_party/tensorrt/build_defs.bzl.tpl b/third_party/tensorrt/build_defs.bzl.tpl
index 6d00513827..0dc3a7ba2d 100644
--- a/third_party/tensorrt/build_defs.bzl.tpl
+++ b/third_party/tensorrt/build_defs.bzl.tpl
@@ -2,4 +2,6 @@
 
 def if_tensorrt(if_true, if_false=[]):
   """Tests whether TensorRT was enabled during the configure process."""
-  return %{if_tensorrt}
+  if %{tensorrt_is_configured}:
+    return if_true
+  return if_false
diff --git a/third_party/tensorrt/remote.BUILD.tpl b/third_party/tensorrt/remote.BUILD.tpl
new file mode 100644
index 0000000000..d5afc4a5c1
--- /dev/null
+++ b/third_party/tensorrt/remote.BUILD.tpl
@@ -0,0 +1,7 @@
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//visibility:public"])
+
+alias(name="LICENSE", actual = "%{target}:LICENSE")
+alias(name = "tensorrt_headers", actual = "%{target}:tensorrt_headers")
+alias(name = "tensorrt", actual = "%{target}:tensorrt")
diff --git a/third_party/tensorrt/tensorrt_configure.bzl b/third_party/tensorrt/tensorrt_configure.bzl
index 22fd80e91a..d3bc388346 100644
--- a/third_party/tensorrt/tensorrt_configure.bzl
+++ b/third_party/tensorrt/tensorrt_configure.bzl
@@ -129,7 +129,7 @@ def _tpl(repository_ctx, tpl, substitutions):
 
 def _create_dummy_repository(repository_ctx):
   """Create a dummy TensorRT repository."""
-  _tpl(repository_ctx, "build_defs.bzl", {"%{if_tensorrt}": "if_false"})
+  _tpl(repository_ctx, "build_defs.bzl", {"%{tensorrt_is_configured}": "False"})
 
   _tpl(repository_ctx, "BUILD", {
       "%{tensorrt_genrules}": "",
@@ -141,10 +141,11 @@ def _tensorrt_configure_impl(repository_ctx):
   """Implementation of the tensorrt_configure repository rule."""
   if _TF_TENSORRT_CONFIG_REPO in repository_ctx.os.environ:
     # Forward to the pre-configured remote repository.
-    remote_config_repo = repository_ctx.os.environ[_TF_TENSORRT_CONFIG_REPO]
-    repository_ctx.template("BUILD", Label(remote_config_repo + ":BUILD"), {})
+    repository_ctx.template("BUILD", Label("//third_party/tensorrt:remote.BUILD.tpl"), {
+        "%{target}": repository_ctx.os.environ[_TF_TENSORRT_CONFIG_REPO],
+    })
     # Set up config file.
-    _tpl(repository_ctx, "build_defs.bzl", {"%{if_tensorrt}": "if_true"})
+    _tpl(repository_ctx, "build_defs.bzl", {"%{tensorrt_is_configured}": "True"})
     return
 
   if _TENSORRT_INSTALL_PATH not in repository_ctx.os.environ:
@@ -193,7 +194,7 @@ def _tensorrt_configure_impl(repository_ctx):
   ))
 
   # Set up config file.
-  _tpl(repository_ctx, "build_defs.bzl", {"%{if_tensorrt}": "if_true"})
+  _tpl(repository_ctx, "build_defs.bzl", {"%{tensorrt_is_configured}": "True"})
 
   # Set up BUILD file.
   _tpl(repository_ctx, "BUILD", {
-- 
GitLab


From c188ec5ca9481fe35ce8420fa12d60a62b30c82c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 18 Jan 2019 03:02:37 -0800
Subject: [PATCH 0961/1095] Include device assignment in models exported by
 TPUEstimator

For model parallel usecases the num_cores_per_replica and the
device_assignment attributes are part of the model so we should export
it to get a consistent picture.

PiperOrigin-RevId: 229901092
---
 .../contrib/tpu/python/tpu/tpu_context.py     |  2 +-
 .../contrib/tpu/python/tpu/tpu_estimator.py   | 25 +++++++++++++++++--
 2 files changed, 24 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_context.py b/tensorflow/contrib/tpu/python/tpu/tpu_context.py
index 6724624479..9ff0c55187 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_context.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_context.py
@@ -208,7 +208,7 @@ class _InternalTPUContext(object):
         use_tpu and config.tpu_config.num_cores_per_replica)
     self._mode = None
     num_cores_per_replica = config.tpu_config.num_cores_per_replica
-    if num_cores_per_replica:
+    if self._model_parallelism_enabled:
       self._computation_shape = _NUM_CORES_TO_COMPUTATION_SHAPE[
           num_cores_per_replica]
     else:
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 27c8ccf6bd..41d0562513 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -2169,7 +2169,8 @@ class TPUEstimator(estimator_lib.Estimator):
                export_to_tpu=True,
                export_to_cpu=True,
                warm_start_from=None,
-               experimental_exported_model_uses_all_cores=False):
+               experimental_exported_model_uses_all_cores=False,
+               experimental_export_device_assignment=False):
     """Constructs an `TPUEstimator` instance.
 
     Args:
@@ -2228,6 +2229,10 @@ class TPUEstimator(estimator_lib.Estimator):
         cores for inference with TPUPartitionedCall(). Once outside compilation
         is supported in TPUPartitionedCall(), this flag will be enabled by
         default.
+      experimental_export_device_assignment: Whether to include the device
+        assignment in the exported model. Doing so is useful in case of model
+        parallel inference but will tie the exported model to the TPU topology
+        used to export the model.
 
     Raises:
       ValueError: `params` has reserved keys already.
@@ -2295,6 +2300,13 @@ class TPUEstimator(estimator_lib.Estimator):
     self._export_to_tpu = export_to_tpu
     self._experimental_exported_model_uses_all_cores = (
         experimental_exported_model_uses_all_cores)
+    self._experimental_export_device_assignment = (
+        experimental_export_device_assignment)
+    if (experimental_exported_model_uses_all_cores and
+        experimental_export_device_assignment):
+      raise ValueError('experimental_exported_model_uses_all_cores and '
+                       'experimental_export_device_assignment is not supported '
+                       'at the same time.')
 
     self._is_input_fn_invoked = None
     self._rendezvous = {}
@@ -2404,7 +2416,16 @@ class TPUEstimator(estimator_lib.Estimator):
       tpu_computation, tpu_capture = self._build_tpu_computation_for_inference(
           features, labels, mode, config)
 
-      tensors_on_cpu = tpu.rewrite_for_inference(tpu_computation)
+      if self._experimental_export_device_assignment:
+        # Export the device assignment as part of the model. This is useful for
+        # model parallel usecases where the model relies on the mapping between
+        # logical and physical devices.
+        with self._ctx.with_mode(mode) as ctx:
+          device_assignment = ctx.device_assignment
+      else:
+        device_assignment = None
+      tensors_on_cpu = tpu.rewrite_for_inference(
+          tpu_computation, device_assignment=device_assignment)
       (estimator_spec, export_outputs_dict, export_outputs_list,
        predictions_dict) = (
            tpu_capture.get())
-- 
GitLab


From c861dc1dcfb1c9f6182335b3fbd611a1e225e8cd Mon Sep 17 00:00:00 2001
From: Andr? Susano Pinto <andresp@google.com>
Date: Fri, 18 Jan 2019 05:29:33 -0800
Subject: [PATCH 0962/1095] Fix another mismatch between TensorSpec versions
 when toggling tensorflow version.

PiperOrigin-RevId: 229913286
---
 tensorflow/python/saved_model/nested_structure_coder.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/saved_model/nested_structure_coder.py b/tensorflow/python/saved_model/nested_structure_coder.py
index 31ee239f13..e080a6b3de 100644
--- a/tensorflow/python/saved_model/nested_structure_coder.py
+++ b/tensorflow/python/saved_model/nested_structure_coder.py
@@ -83,7 +83,6 @@ class StructureCoder(object):
     """
     return self._map_structure(nested_structure, self._get_encoders())
 
-
   def can_encode(self, nested_structure):
     """Determines whether a nested structure can be encoded into a proto.
 
@@ -361,7 +360,10 @@ class _TensorShapeCodec(object):
   """Codec for `TensorShape`."""
 
   def can_encode(self, pyobj):
-    return isinstance(pyobj, tensor_shape.TensorShape)
+    return isinstance(pyobj, (tensor_shape.TensorShape,
+                              # TODO(b/121255889): Should not need these.
+                              tensor_shape.TensorShapeV1,
+                              tensor_shape.TensorShapeV2))
 
   def do_encode(self, tensor_shape_value, encode_fn):
     del encode_fn
-- 
GitLab


From faad607b60e6bed6e780a2ac93afcdfc96f01429 Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Fri, 18 Jan 2019 07:13:43 -0800
Subject: [PATCH 0963/1095] Pass a compare function to the KeyValueSort runtime
 function.

This is in preparation of supporting calling a HloComputation for comparisons.

PiperOrigin-RevId: 229924424
---
 .../compiler/xla/service/cpu/cpu_runtime.cc   |  27 +--
 .../compiler/xla/service/cpu/cpu_runtime.h    |  13 +-
 .../compiler/xla/service/cpu/ir_emitter.cc    | 190 +++++++++--------
 .../xla/service/cpu/runtime_key_value_sort.cc | 197 ++----------------
 .../xla/service/cpu/runtime_key_value_sort.h  |  81 ++-----
 .../xla/service/cpu/simple_orc_jit.cc         |  13 +-
 6 files changed, 140 insertions(+), 381 deletions(-)

diff --git a/tensorflow/compiler/xla/service/cpu/cpu_runtime.cc b/tensorflow/compiler/xla/service/cpu/cpu_runtime.cc
index a9febe891b..d8878e622c 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_runtime.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_runtime.cc
@@ -84,31 +84,8 @@ extern const char* const kReleaseOutfeedBufferAfterPopulationSymbolName =
     "__xla_cpu_runtime_ReleaseOutfeedBufferAfterPopulation";
 extern const char* const kParallelForkJoinSymbolName =
     "__xla_cpu_runtime_ParallelForkJoin";
-extern const char* const kKeyValueSortPREDSymbolName =
-    "__xla_cpu_runtime_KeyValueSortPRED";
-extern const char* const kKeyValueSortS8SymbolName =
-    "__xla_cpu_runtime_KeyValueSortS8";
-extern const char* const kKeyValueSortU8SymbolName =
-    "__xla_cpu_runtime_KeyValueSortU8";
-extern const char* const kKeyValueSortS16SymbolName =
-    "__xla_cpu_runtime_KeyValueSortS16";
-extern const char* const kKeyValueSortU16SymbolName =
-    "__xla_cpu_runtime_KeyValueSortU16";
-extern const char* const kKeyValueSortF16SymbolName =
-    "__xla_cpu_runtime_KeyValueSortF16";
-extern const char* const kKeyValueSortS32SymbolName =
-    "__xla_cpu_runtime_KeyValueSortS32";
-extern const char* const kKeyValueSortU32SymbolName =
-    "__xla_cpu_runtime_KeyValueSortU32";
-extern const char* const kKeyValueSortF32SymbolName =
-    "__xla_cpu_runtime_KeyValueSortF32";
-extern const char* const kKeyValueSortS64SymbolName =
-    "__xla_cpu_runtime_KeyValueSortS64";
-extern const char* const kKeyValueSortU64SymbolName =
-    "__xla_cpu_runtime_KeyValueSortU64";
-extern const char* const kKeyValueSortF64SymbolName =
-    "__xla_cpu_runtime_KeyValueSortF64";
-
+extern const char* const kKeyValueSortSymbolName =
+    "__xla_cpu_runtime_KeyValueSort";
 extern const char* const kXlaCpuRuntimeSymbolNamePrefix = "__xla_cpu_runtime_";
 }  // namespace runtime
 }  // namespace cpu
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_runtime.h b/tensorflow/compiler/xla/service/cpu/cpu_runtime.h
index b2e760a224..3a2b44d8c1 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_runtime.h
+++ b/tensorflow/compiler/xla/service/cpu/cpu_runtime.h
@@ -64,18 +64,7 @@ extern const char* const kReleaseInfeedBufferAfterDequeueSymbolName;
 extern const char* const kAcquireOutfeedBufferForPopulationSymbolName;
 extern const char* const kReleaseOutfeedBufferAfterPopulationSymbolName;
 extern const char* const kParallelForkJoinSymbolName;
-extern const char* const kKeyValueSortPREDSymbolName;
-extern const char* const kKeyValueSortS8SymbolName;
-extern const char* const kKeyValueSortU8SymbolName;
-extern const char* const kKeyValueSortS16SymbolName;
-extern const char* const kKeyValueSortU16SymbolName;
-extern const char* const kKeyValueSortF16SymbolName;
-extern const char* const kKeyValueSortS32SymbolName;
-extern const char* const kKeyValueSortU32SymbolName;
-extern const char* const kKeyValueSortF32SymbolName;
-extern const char* const kKeyValueSortS64SymbolName;
-extern const char* const kKeyValueSortU64SymbolName;
-extern const char* const kKeyValueSortF64SymbolName;
+extern const char* const kKeyValueSortSymbolName;
 
 // All symbol names for XLA CPU runtime functions need to start with this
 // prefix.
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
index f8a997045a..0226e8275c 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
@@ -495,6 +495,26 @@ Status IrEmitter::HandleSort(HloInstruction* hlo) {
   const HloSortInstruction* sort = Cast<HloSortInstruction>(hlo);
   TF_RETURN_IF_ERROR(EmitTargetAddressForOp(sort));
   Shape keys_shape = sort->keys()->shape();
+  PrimitiveType keys_type = keys_shape.element_type();
+  switch (keys_type) {
+    case PRED:
+    case S8:
+    case U8:
+    case S16:
+    case U16:
+    case F16:
+    case S32:
+    case U32:
+    case F32:
+    case S64:
+    case U64:
+    case F64:
+      break;
+    default:
+      return Unimplemented(
+          "Element type %s not supported in the Sort op on CPU.",
+          PrimitiveType_Name(keys_type));
+  }
   std::vector<llvm::Value*> destination_addresses(sort->operand_count());
   for (int64 i = 0; i < sort->operand_count(); ++i) {
     ShapeIndex shape_index =
@@ -542,105 +562,101 @@ Status IrEmitter::HandleSort(HloInstruction* hlo) {
     lower_dimensions *= normalized_keys_shape.dimensions(i);
   }
 
-  PrimitiveType keys_type = keys_shape.element_type();
-  const char* fn_name = nullptr;
-  llvm::Type* keys_native_type = nullptr;
-  switch (keys_type) {
-    case PRED:
-      fn_name = runtime::kKeyValueSortPREDSymbolName;
-      keys_native_type = b_.getInt8PtrTy();
-      break;
-    case S8:
-      fn_name = runtime::kKeyValueSortS8SymbolName;
-      keys_native_type = b_.getInt8PtrTy();
-      break;
-    case U8:
-      fn_name = runtime::kKeyValueSortU8SymbolName;
-      keys_native_type = b_.getInt8PtrTy();
-      break;
-    case S16:
-      fn_name = runtime::kKeyValueSortS16SymbolName;
-      keys_native_type = b_.getInt16Ty()->getPointerTo();
-      break;
-    case U16:
-      fn_name = runtime::kKeyValueSortU16SymbolName;
-      keys_native_type = b_.getInt16Ty()->getPointerTo();
-      break;
-    case F16:
-      fn_name = runtime::kKeyValueSortF16SymbolName;
-      keys_native_type = b_.getHalfTy()->getPointerTo();
-      break;
-    case S32:
-      fn_name = runtime::kKeyValueSortS32SymbolName;
-      keys_native_type = b_.getInt32Ty()->getPointerTo();
-      break;
-    case U32:
-      fn_name = runtime::kKeyValueSortU32SymbolName;
-      keys_native_type = b_.getInt32Ty()->getPointerTo();
-      break;
-    case F32:
-      fn_name = runtime::kKeyValueSortF32SymbolName;
-      keys_native_type = b_.getFloatTy()->getPointerTo();
-      break;
-    case S64:
-      fn_name = runtime::kKeyValueSortS64SymbolName;
-      keys_native_type = b_.getInt64Ty()->getPointerTo();
-      break;
-    case U64:
-      fn_name = runtime::kKeyValueSortU64SymbolName;
-      keys_native_type = b_.getInt64Ty()->getPointerTo();
-      break;
-    case F64:
-      fn_name = runtime::kKeyValueSortF64SymbolName;
-      keys_native_type = b_.getDoubleTy()->getPointerTo();
-      break;
-    default:
-      return Unimplemented(
-          "Element type %s not supported in the Sort op on CPU.",
-          PrimitiveType_Name(keys_type));
+  llvm::FunctionType* less_than_type = llvm::FunctionType::get(
+      b_.getInt1Ty(), {b_.getInt8PtrTy(), b_.getInt8PtrTy()},
+      /*isVarArg=*/false);
+  auto less_than_function = llvm_ir::CreateFunction(
+      less_than_type, llvm::GlobalValue::InternalLinkage,
+      /*enable_fast_math=*/false,
+      /*optimize_for_size=*/true, absl::StrCat(IrName(sort), "_comparator"),
+      module_);
+  // Emit the code for the less_than function.
+  {
+    llvm::IRBuilder<>::InsertPointGuard guard(b_);
+
+    auto* entry_bb =
+        llvm::BasicBlock::Create(b_.getContext(), "entry", less_than_function);
+
+    b_.SetInsertPoint(entry_bb);
+    auto keys_ir_type = llvm_ir::PrimitiveTypeToIrType(keys_type, module_);
+    CHECK_EQ(less_than_function->arg_size(), 2);
+    llvm::Value* keys_lhs_ptr = less_than_function->arg_begin();
+    keys_lhs_ptr = PointerCast(keys_lhs_ptr, keys_ir_type->getPointerTo());
+    llvm::Value* keys_rhs_ptr = less_than_function->arg_begin() + 1;
+    keys_rhs_ptr = PointerCast(keys_rhs_ptr, keys_ir_type->getPointerTo());
+
+    // TODO(b/122298745): Replace the custom compare logic with a call to the
+    // computation specified for the Sort op.
+    llvm::Value* keys_lhs = Load(keys_ir_type, keys_lhs_ptr);
+    llvm::Value* keys_rhs = Load(keys_ir_type, keys_rhs_ptr);
+    bool is_signed_comparison = true;
+    if (primitive_util::IsFloatingPointType(keys_type)) {
+      // We would like a total order of floating point numbers so that the
+      // sort has a predictable behavior in the presence of NaNs. Rather
+      // than using floating point comparison, we use the following trick:
+      // If f is a float, and
+      // x = bit_cast<int32>(f);
+      // y = x < 0 ? 0x7FFFFFFF - x : x;
+      // then y is ordered as an int32 such that finite values have the
+      // obvious order, -0 is ordered before 0, and -NaN and NaN appear at
+      // the beginning and end of the ordering.
+      auto k = b_.getInt(llvm::APInt::getSignedMaxValue(
+          keys_lhs->getType()->getPrimitiveSizeInBits()));
+      auto comparison_type = k->getType();
+      auto zero = llvm::ConstantInt::get(comparison_type, 0);
+      auto maybe_flip = [&](llvm::Value* v) {
+        return b_.CreateSelect(b_.CreateICmp(llvm::ICmpInst::ICMP_SLT, v, zero),
+                               b_.CreateSub(k, v), v);
+      };
+      keys_lhs = b_.CreateBitCast(keys_lhs, comparison_type);
+      keys_rhs = b_.CreateBitCast(keys_rhs, comparison_type);
+      keys_lhs = maybe_flip(keys_lhs);
+      keys_rhs = maybe_flip(keys_rhs);
+    } else if (!primitive_util::IsSignedIntegralType(keys_type)) {
+      is_signed_comparison = false;
+    }
+    llvm::Value* result =
+        b_.CreateICmp(is_signed_comparison ? llvm::ICmpInst::ICMP_SLT
+                                           : llvm::ICmpInst::ICMP_ULT,
+                      keys_lhs, keys_rhs);
+    llvm::ReturnInst::Create(b_.getContext(),
+                             /*retVal=*/result, entry_bb);
   }
 
   llvm::FunctionType* key_value_sort_type = llvm::FunctionType::get(
       b_.getVoidTy(),
-      {keys_native_type, b_.getInt64Ty(), b_.getInt64Ty(), b_.getInt64Ty(),
+      {b_.getInt64Ty(), b_.getInt64Ty(), b_.getInt64Ty(),
        b_.getInt8PtrTy()->getPointerTo(), b_.getInt32Ty(),
-       b_.getInt32Ty()->getPointerTo()},
+       b_.getInt32Ty()->getPointerTo(), less_than_function->getType()},
       /*isVarArg=*/false);
-  auto* key_value_sort_func = llvm::cast<llvm::Function>(
-      module_->getOrInsertFunction(fn_name, key_value_sort_type));
+  auto* key_value_sort_func =
+      llvm::cast<llvm::Function>(module_->getOrInsertFunction(
+          runtime::kKeyValueSortSymbolName, key_value_sort_type));
   key_value_sort_func->setCallingConv(llvm::CallingConv::C);
   key_value_sort_func->setDoesNotThrow();
-  llvm::Value* values;
-  llvm::Value* sizes;
-  if (sort->values_count() == 0) {
-    values = llvm::Constant::getNullValue(b_.getInt8PtrTy()->getPointerTo());
-    sizes = llvm::Constant::getNullValue(b_.getInt32Ty()->getPointerTo());
-  } else {
-    values = llvm_ir::EmitAllocaAtFunctionEntryWithCount(
-        b_.getInt8PtrTy(), b_.getInt32(sort->values_count()),
-        "cc_values_alloca", &b_);
-    sizes = llvm_ir::EmitAllocaAtFunctionEntryWithCount(
-        b_.getInt32Ty(), b_.getInt32(sort->values_count()), "cc_sizes_alloca",
-        &b_);
-    for (int64 i = 0; i < sort->values_count(); ++i) {
-      llvm::Value* value_as_i8ptr =
-          PointerCast(destination_addresses[i + 1], b_.getInt8PtrTy());
-      llvm::Value* slot_in_values_alloca =
-          ConstInBoundsGEP1_32(b_.getInt8PtrTy(), values, i);
-      Store(value_as_i8ptr, slot_in_values_alloca);
-      llvm::Value* slot_in_sizes_alloca =
-          ConstInBoundsGEP1_32(b_.getInt32Ty(), sizes, i);
-      llvm::Value* size = b_.getInt32(ShapeUtil::ByteSizeOfPrimitiveType(
-          sort->operand(i + 1)->shape().element_type()));
-      Store(size, slot_in_sizes_alloca);
-    }
+  llvm::Value* values = llvm_ir::EmitAllocaAtFunctionEntryWithCount(
+      b_.getInt8PtrTy(), b_.getInt32(sort->operand_count()), "cc_values_alloca",
+      &b_);
+  llvm::Value* sizes = llvm_ir::EmitAllocaAtFunctionEntryWithCount(
+      b_.getInt32Ty(), b_.getInt32(sort->operand_count()), "cc_sizes_alloca",
+      &b_);
+  for (int64 i = 0; i < sort->operand_count(); ++i) {
+    llvm::Value* value_as_i8ptr =
+        PointerCast(destination_addresses[i], b_.getInt8PtrTy());
+    llvm::Value* slot_in_values_alloca =
+        ConstInBoundsGEP1_32(b_.getInt8PtrTy(), values, i);
+    Store(value_as_i8ptr, slot_in_values_alloca);
+    llvm::Value* slot_in_sizes_alloca =
+        ConstInBoundsGEP1_32(b_.getInt32Ty(), sizes, i);
+    llvm::Value* size = b_.getInt32(ShapeUtil::ByteSizeOfPrimitiveType(
+        sort->operand(i)->shape().element_type()));
+    Store(size, slot_in_sizes_alloca);
   }
 
   Call(key_value_sort_func,
-       {PointerCast(destination_addresses[0], keys_native_type),
-        b_.getInt64(higher_dimensions), b_.getInt64(sort_dimension_elements),
+       {b_.getInt64(higher_dimensions), b_.getInt64(sort_dimension_elements),
         b_.getInt64(lower_dimensions), values,
-        b_.getInt32(sort->values_count()), sizes});
+        b_.getInt32(sort->operand_count()), sizes, less_than_function});
 
   if (sort->values_count() > 0) {
     llvm_ir::EmitTuple(GetIrArrayFor(sort), destination_addresses, &b_,
diff --git a/tensorflow/compiler/xla/service/cpu/runtime_key_value_sort.cc b/tensorflow/compiler/xla/service/cpu/runtime_key_value_sort.cc
index 722aa3120e..a0667d0d9d 100644
--- a/tensorflow/compiler/xla/service/cpu/runtime_key_value_sort.cc
+++ b/tensorflow/compiler/xla/service/cpu/runtime_key_value_sort.cc
@@ -15,12 +15,10 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/cpu/runtime_key_value_sort.h"
 
 #include <algorithm>
-#include <cmath>
 #include <cstring>
-#include <limits>
 #include <memory>
+#include <numeric>
 #include <string>
-#include <utility>
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/platform/dynamic_annotations.h"
@@ -28,80 +26,14 @@ limitations under the License.
 #include "tensorflow/core/platform/types.h"
 
 namespace {
-using tensorflow::int16;
 using tensorflow::int32;
 using tensorflow::int64;
-using tensorflow::int8;
-using tensorflow::uint16;
-using tensorflow::uint32;
-using tensorflow::uint64;
-using tensorflow::uint8;
-
-template <typename KeyType>
-void KeyValueSort(std::pair<KeyType, int64>* row_to_sort, int64 num_elements) {
-  std::sort(row_to_sort, row_to_sort + num_elements);
-}
-
-// We would like a total order of floating point numbers so that the
-// sort has a predictable behavior in the presence of NaNs. Rather
-// than using floating point comparison, we use the following trick:
-// If f is a float, and
-// x = bit_cast<int32>(f);
-// y = x < 0 ? 0x7FFFFFFF - x : x;
-// then y is ordered as an int32 such that finite values have the
-// obvious order, -0 is ordered before 0, and -NaN and NaN appear at
-// the beginning and end of the ordering.
-template <typename CastType, typename UnsignedCastType, typename KeyType>
-CastType Convert(KeyType value) {
-  CastType casted_value;
-  memcpy(&casted_value, &value, sizeof(CastType));
-  if (casted_value < 0) {
-    return static_cast<UnsignedCastType>(std::numeric_limits<CastType>::max()) -
-           casted_value;
-  }
-  return casted_value;
-}
-
-template <typename CastType, typename UnsignedCastType, typename KeyType>
-bool LessThan(KeyType lhs, KeyType rhs) {
-  return Convert<CastType, UnsignedCastType>(lhs) <
-         Convert<CastType, UnsignedCastType>(rhs);
-}
-
-template <>
-void KeyValueSort(std::pair<double, int64>* row_to_sort, int64 num_elements) {
-  std::stable_sort(row_to_sort, row_to_sort + num_elements,
-                   [](const std::pair<double, int64>& lhs,
-                      const std::pair<double, int64>& rhs) -> bool {
-                     return LessThan<int64, uint64>(lhs.first, rhs.first);
-                   });
-}
-
-template <>
-void KeyValueSort(std::pair<float, int64>* row_to_sort, int64 num_elements) {
-  std::stable_sort(row_to_sort, row_to_sort + num_elements,
-                   [](const std::pair<float, int64>& lhs,
-                      const std::pair<float, int64>& rhs) -> bool {
-                     return LessThan<int32, uint32>(lhs.first, rhs.first);
-                   });
-}
-
-template <>
-void KeyValueSort(std::pair<Eigen::half, int64>* row_to_sort,
-                  int64 num_elements) {
-  std::stable_sort(row_to_sort, row_to_sort + num_elements,
-                   [](const std::pair<Eigen::half, int64>& lhs,
-                      const std::pair<Eigen::half, int64>& rhs) -> bool {
-                     return LessThan<int32, uint32>(
-                         Eigen::half_impl::half_to_float(lhs.first),
-                         Eigen::half_impl::half_to_float(rhs.first));
-                   });
-}
+}  // namespace
 
-template <typename KeyType>
-void KeyValueSortImpl(KeyType* keys, int64 a, int64 b, int64 c, char** values,
-                      int32 values_count,
-                      int32* values_primitive_type_size_in_bytes) {
+TF_ATTRIBUTE_NO_SANITIZE_MEMORY void __xla_cpu_runtime_KeyValueSort(
+    int64 a, int64 b, int64 c, char** values, int32 values_count,
+    int32* values_primitive_type_size_in_bytes,
+    bool (*less_than)(char*, char*)) {
   // 'values' and 'values_primitive_type_size_in_bytes' are managed by the JIT
   // code, so msan can't tell they are initialized.
   TF_ANNOTATE_MEMORY_IS_INITIALIZED(values, values_count * sizeof(char*));
@@ -121,8 +53,8 @@ void KeyValueSortImpl(KeyType* keys, int64 a, int64 b, int64 c, char** values,
   int64 num_iteration_elements = a * c;
   int64 sort_dimension_offset = c;
 
-  std::unique_ptr<std::pair<KeyType, int64>[]> row_to_sort(
-      new std::pair<KeyType, int64>[sort_dimension_elements]);
+  std::unique_ptr<int64[]> indices(new int64[sort_dimension_elements]);
+  std::iota(indices.get(), indices.get() + sort_dimension_elements, 0);
   std::unique_ptr<std::string[]> reordered_values(
       new std::string[sort_dimension_elements]);
   for (int64 index = 0; index < num_iteration_elements; ++index) {
@@ -135,24 +67,22 @@ void KeyValueSortImpl(KeyType* keys, int64 a, int64 b, int64 c, char** values,
     int64 base_offset =
         index % sort_dimension_offset +
         (index - index % sort_dimension_offset) * sort_dimension_elements;
-    // TODO(b/26783907): We could define a custom iterator class that references
-    // all arrays. Then we could avoid the intermediate copy. However this
-    // would become more complicated, and it is not clear if the benefit is high
-    // enough.
-    for (int64 i = 0; i < sort_dimension_elements; ++i) {
-      row_to_sort[i] =
-          std::make_pair(keys[base_offset + i * sort_dimension_offset], i);
-    }
-    KeyValueSort(row_to_sort.get(), sort_dimension_elements);
-    for (int64 i = 0; i < sort_dimension_elements; ++i) {
-      keys[base_offset + i * sort_dimension_offset] = row_to_sort[i].first;
-    }
-
-    // Reorder the values according to the order defined by the keys.
+    std::stable_sort(
+        indices.get(), indices.get() + sort_dimension_elements,
+        [&](int64 a, int64 b) {
+          int64 memory_index_lhs = (base_offset + a * sort_dimension_offset) *
+                                   values_primitive_type_size_in_bytes[0];
+          int64 memory_index_rhs = (base_offset + b * sort_dimension_offset) *
+                                   values_primitive_type_size_in_bytes[0];
+          return less_than(values[0] + memory_index_lhs,
+                           values[0] + memory_index_rhs);
+        });
+
+    // Reorder the values according to the order defined by 'indices'.
     for (int32 idx = 0; idx < values_count; ++idx) {
       for (int64 i = 0; i < sort_dimension_elements; ++i) {
         int64 memory_index =
-            (base_offset + row_to_sort[i].second * sort_dimension_offset) *
+            (base_offset + indices[i] * sort_dimension_offset) *
             values_primitive_type_size_in_bytes[idx];
 
         reordered_values[i] =
@@ -168,88 +98,3 @@ void KeyValueSortImpl(KeyType* keys, int64 a, int64 b, int64 c, char** values,
     }
   }
 }
-}  // namespace
-
-TF_ATTRIBUTE_NO_SANITIZE_MEMORY void __xla_cpu_runtime_KeyValueSortPRED(
-    bool* keys, int64 a, int64 b, int64 c, char** values, int32 values_count,
-    int32* values_primitive_type_size_in_bytes) {
-  KeyValueSortImpl(keys, a, b, c, values, values_count,
-                   values_primitive_type_size_in_bytes);
-}
-
-TF_ATTRIBUTE_NO_SANITIZE_MEMORY void __xla_cpu_runtime_KeyValueSortS8(
-    int8* keys, int64 a, int64 b, int64 c, char** values, int32 values_count,
-    int32* values_primitive_type_size_in_bytes) {
-  KeyValueSortImpl(keys, a, b, c, values, values_count,
-                   values_primitive_type_size_in_bytes);
-}
-
-TF_ATTRIBUTE_NO_SANITIZE_MEMORY void __xla_cpu_runtime_KeyValueSortU8(
-    uint8* keys, int64 a, int64 b, int64 c, char** values, int32 values_count,
-    int32* values_primitive_type_size_in_bytes) {
-  KeyValueSortImpl(keys, a, b, c, values, values_count,
-                   values_primitive_type_size_in_bytes);
-}
-
-TF_ATTRIBUTE_NO_SANITIZE_MEMORY void __xla_cpu_runtime_KeyValueSortS16(
-    int16* keys, int64 a, int64 b, int64 c, char** values, int32 values_count,
-    int32* values_primitive_type_size_in_bytes) {
-  KeyValueSortImpl(keys, a, b, c, values, values_count,
-                   values_primitive_type_size_in_bytes);
-}
-
-TF_ATTRIBUTE_NO_SANITIZE_MEMORY void __xla_cpu_runtime_KeyValueSortU16(
-    uint16* keys, int64 a, int64 b, int64 c, char** values, int32 values_count,
-    int32* values_primitive_type_size_in_bytes) {
-  KeyValueSortImpl(keys, a, b, c, values, values_count,
-                   values_primitive_type_size_in_bytes);
-}
-
-TF_ATTRIBUTE_NO_SANITIZE_MEMORY void __xla_cpu_runtime_KeyValueSortF16(
-    Eigen::half* keys, int64 a, int64 b, int64 c, char** values,
-    int32 values_count, int32* values_primitive_type_size_in_bytes) {
-  KeyValueSortImpl(keys, a, b, c, values, values_count,
-                   values_primitive_type_size_in_bytes);
-}
-
-TF_ATTRIBUTE_NO_SANITIZE_MEMORY void __xla_cpu_runtime_KeyValueSortS32(
-    int32* keys, int64 a, int64 b, int64 c, char** values, int32 values_count,
-    int32* values_primitive_type_size_in_bytes) {
-  KeyValueSortImpl(keys, a, b, c, values, values_count,
-                   values_primitive_type_size_in_bytes);
-}
-
-TF_ATTRIBUTE_NO_SANITIZE_MEMORY void __xla_cpu_runtime_KeyValueSortU32(
-    uint32* keys, int64 a, int64 b, int64 c, char** values, int32 values_count,
-    int32* values_primitive_type_size_in_bytes) {
-  KeyValueSortImpl(keys, a, b, c, values, values_count,
-                   values_primitive_type_size_in_bytes);
-}
-
-TF_ATTRIBUTE_NO_SANITIZE_MEMORY void __xla_cpu_runtime_KeyValueSortF32(
-    float* keys, int64 a, int64 b, int64 c, char** values, int32 values_count,
-    int32* values_primitive_type_size_in_bytes) {
-  KeyValueSortImpl(keys, a, b, c, values, values_count,
-                   values_primitive_type_size_in_bytes);
-}
-
-TF_ATTRIBUTE_NO_SANITIZE_MEMORY void __xla_cpu_runtime_KeyValueSortS64(
-    int64* keys, int64 a, int64 b, int64 c, char** values, int32 values_count,
-    int32* values_primitive_type_size_in_bytes) {
-  KeyValueSortImpl(keys, a, b, c, values, values_count,
-                   values_primitive_type_size_in_bytes);
-}
-
-TF_ATTRIBUTE_NO_SANITIZE_MEMORY void __xla_cpu_runtime_KeyValueSortU64(
-    uint64* keys, int64 a, int64 b, int64 c, char** values, int32 values_count,
-    int32* values_primitive_type_size_in_bytes) {
-  KeyValueSortImpl(keys, a, b, c, values, values_count,
-                   values_primitive_type_size_in_bytes);
-}
-
-TF_ATTRIBUTE_NO_SANITIZE_MEMORY void __xla_cpu_runtime_KeyValueSortF64(
-    double* keys, int64 a, int64 b, int64 c, char** values, int32 values_count,
-    int32* values_primitive_type_size_in_bytes) {
-  KeyValueSortImpl(keys, a, b, c, values, values_count,
-                   values_primitive_type_size_in_bytes);
-}
diff --git a/tensorflow/compiler/xla/service/cpu/runtime_key_value_sort.h b/tensorflow/compiler/xla/service/cpu/runtime_key_value_sort.h
index 7821099386..5460af3485 100644
--- a/tensorflow/compiler/xla/service/cpu/runtime_key_value_sort.h
+++ b/tensorflow/compiler/xla/service/cpu/runtime_key_value_sort.h
@@ -21,76 +21,19 @@ limitations under the License.
 
 extern "C" {
 
-// 'keys' represents a 3-dimensional shape with dimensions [a, b, c]. The 'b'
-// dimension of 'keys' is sorted into ascending order. If 'values_count' is <=
-// 0, 'values' and 'values_primitive_type_size_in_bytes' can be nullptr.
-// If 'values_count' > 0, they contain exactly 'values_count' many elements.
-// Each element of 'values' also represents a 3-dimensional shape with
-// dimensions [a, b, c], and the size of the primitive type of the i-th shape
-// has exactly 'values_primitive_type_size_in_bytes[i]' bytes. The elements in
-// each 'values' shape are reordered in such a way that if the element at index
-// 'i' in 'keys' was moved to index 'j', the element at index 'i' in a 'values'
-// shape is also moved to index 'j' (which means that the same elements
-// correspond to each other as before).
-extern void __xla_cpu_runtime_KeyValueSortPRED(
-    bool* keys, tensorflow::int64 a, tensorflow::int64 b, tensorflow::int64 c,
+// Each entry in 'values' represents a 3-dimensional shape with dimensions
+// [a, b, c]. The 'b' dimension of the first shape is sorted into ascending
+// order according to the results of comparisons using the provided 'less_than'
+// function. 'values_count' must be > 0 and specifies the number of entries in
+// 'values' and 'values_primitive_type_size_in_bytes'. The size of the primitive
+// type of the i-th shape has exactly 'values_primitive_type_size_in_bytes[i]'
+// bytes. The elements in each 'values' shape are reordered in the same way
+// according to the comparisons using the first shape.
+extern void __xla_cpu_runtime_KeyValueSort(
+    tensorflow::int64 a, tensorflow::int64 b, tensorflow::int64 c,
     char** values, tensorflow::int32 values_count,
-    tensorflow::int32* values_primitive_type_size_in_bytes);
-
-extern void __xla_cpu_runtime_KeyValueSortS8(
-    tensorflow::int8* keys, tensorflow::int64 a, tensorflow::int64 b,
-    tensorflow::int64 c, char** values, tensorflow::int32 values_count,
-    tensorflow::int32* values_primitive_type_size_in_bytes);
-
-extern void __xla_cpu_runtime_KeyValueSortU8(
-    tensorflow::uint8* keys, tensorflow::int64 a, tensorflow::int64 b,
-    tensorflow::int64 c, char** values, tensorflow::int32 values_count,
-    tensorflow::int32* values_primitive_type_size_in_bytes);
-
-extern void __xla_cpu_runtime_KeyValueSortS16(
-    tensorflow::int16* keys, tensorflow::int64 a, tensorflow::int64 b,
-    tensorflow::int64 c, char** values, tensorflow::int32 values_count,
-    tensorflow::int32* values_primitive_type_size_in_bytes);
-
-extern void __xla_cpu_runtime_KeyValueSortU16(
-    tensorflow::uint16* keys, tensorflow::int64 a, tensorflow::int64 b,
-    tensorflow::int64 c, char** values, tensorflow::int32 values_count,
-    tensorflow::int32* values_primitive_type_size_in_bytes);
-
-extern void __xla_cpu_runtime_KeyValueSortF16(
-    Eigen::half* keys, tensorflow::int64 a, tensorflow::int64 b,
-    tensorflow::int64 c, char** values, tensorflow::int32 values_count,
-    tensorflow::int32* values_primitive_type_size_in_bytes);
-
-extern void __xla_cpu_runtime_KeyValueSortS32(
-    tensorflow::int32* keys, tensorflow::int64 a, tensorflow::int64 b,
-    tensorflow::int64 c, char** values, tensorflow::int32 values_count,
-    tensorflow::int32* values_primitive_type_size_in_bytes);
-
-extern void __xla_cpu_runtime_KeyValueSortU32(
-    tensorflow::uint32* keys, tensorflow::int64 a, tensorflow::int64 b,
-    tensorflow::int64 c, char** values, tensorflow::int32 values_count,
-    tensorflow::int32* values_primitive_type_size_in_bytes);
-
-extern void __xla_cpu_runtime_KeyValueSortF32(
-    float* keys, tensorflow::int64 a, tensorflow::int64 b, tensorflow::int64 c,
-    char** values, tensorflow::int32 values_count,
-    tensorflow::int32* values_primitive_type_size_in_bytes);
-
-extern void __xla_cpu_runtime_KeyValueSortS64(
-    tensorflow::int64* keys, tensorflow::int64 a, tensorflow::int64 b,
-    tensorflow::int64 c, char** values, tensorflow::int32 values_count,
-    tensorflow::int32* values_primitive_type_size_in_bytes);
-
-extern void __xla_cpu_runtime_KeyValueSortU64(
-    tensorflow::uint64* keys, tensorflow::int64 a, tensorflow::int64 b,
-    tensorflow::int64 c, char** values, tensorflow::int32 values_count,
-    tensorflow::int32* values_primitive_type_size_in_bytes);
-
-extern void __xla_cpu_runtime_KeyValueSortF64(
-    double* keys, tensorflow::int64 a, tensorflow::int64 b, tensorflow::int64 c,
-    char** values, tensorflow::int32 values_count,
-    tensorflow::int32* values_primitive_type_size_in_bytes);
+    tensorflow::int32* values_primitive_type_size_in_bytes,
+    bool (*less_than)(char*, char*));
 }
 
 #endif  // TENSORFLOW_COMPILER_XLA_SERVICE_CPU_RUNTIME_KEY_VALUE_SORT_H_
diff --git a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
index 12ab6360c5..9c2685674f 100644
--- a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
+++ b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
@@ -240,18 +240,7 @@ bool RegisterKnownJITSymbols() {
   REGISTER_CPU_RUNTIME_SYMBOL(ParallelForkJoin);
   REGISTER_CPU_RUNTIME_SYMBOL(ReleaseInfeedBufferAfterDequeue);
   REGISTER_CPU_RUNTIME_SYMBOL(ReleaseOutfeedBufferAfterPopulation);
-  REGISTER_CPU_RUNTIME_SYMBOL(KeyValueSortPRED);
-  REGISTER_CPU_RUNTIME_SYMBOL(KeyValueSortS8);
-  REGISTER_CPU_RUNTIME_SYMBOL(KeyValueSortU8);
-  REGISTER_CPU_RUNTIME_SYMBOL(KeyValueSortS16);
-  REGISTER_CPU_RUNTIME_SYMBOL(KeyValueSortU16);
-  REGISTER_CPU_RUNTIME_SYMBOL(KeyValueSortF16);
-  REGISTER_CPU_RUNTIME_SYMBOL(KeyValueSortS32);
-  REGISTER_CPU_RUNTIME_SYMBOL(KeyValueSortU32);
-  REGISTER_CPU_RUNTIME_SYMBOL(KeyValueSortF32);
-  REGISTER_CPU_RUNTIME_SYMBOL(KeyValueSortS64);
-  REGISTER_CPU_RUNTIME_SYMBOL(KeyValueSortU64);
-  REGISTER_CPU_RUNTIME_SYMBOL(KeyValueSortF64);
+  REGISTER_CPU_RUNTIME_SYMBOL(KeyValueSort);
 
   registry->Register("__gnu_f2h_ieee", reinterpret_cast<void*>(__gnu_f2h_ieee));
   registry->Register("__gnu_h2f_ieee", reinterpret_cast<void*>(__gnu_h2f_ieee));
-- 
GitLab


From f9f39e43c8ced8377cbb5e4775815445e825c655 Mon Sep 17 00:00:00 2001
From: Tom Hennigan <tomhennigan@google.com>
Date: Fri, 18 Jan 2019 07:15:12 -0800
Subject: [PATCH 0964/1095] Loosen return type for variables.

PiperOrigin-RevId: 229924537
---
 tensorflow/python/module/module.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/tensorflow/python/module/module.py b/tensorflow/python/module/module.py
index 16db0b0e01..2ba6b39bc0 100644
--- a/tensorflow/python/module/module.py
+++ b/tensorflow/python/module/module.py
@@ -199,23 +199,23 @@ class Module(six.with_metaclass(ModuleMetaclass, tracking.AutoCheckpointable)):
 
   @property
   def variables(self):
-    """Returns a tuple of variables owned by this module and it's submodules.
+    """Collection of variables owned by this module and it's submodules.
 
     Returns:
-      A tuple of variables for the current module (sorted by attribute name)
-      followed by variables from all submodules recursively (depth first).
+      A collection of variables for the current module (sorted by attribute
+      name) followed by variables from all submodules recursively (depth first).
     """
     return tuple(walk(self, recurse_if=_IS_MODULE, predicate=_IS_VARIABLE))
 
   @property
   def owned_variables(self):
-    """Returns a tuple of variables that are attributes of the current module.
+    """Collection of variables that are attributes of the current module.
 
     See `variables` for a property which returns all variables from the current
     module and all it's submodules recursively.
 
     Returns:
-      A tuple of variables which are attributes of the current module. Will
+      A collection of variables which are attributes of the current module. Will
       yield variables inside nested structures (lists etc) but not in other
       modules.
     """
@@ -223,24 +223,24 @@ class Module(six.with_metaclass(ModuleMetaclass, tracking.AutoCheckpointable)):
 
   @property
   def trainable_variables(self):
-    """Returns a tuple of variables owned by this module and it's submodules.
+    """Collection of variables owned by this module and it's submodules.
 
     Returns:
-      A tuple of variables for the current module (sorted by attribute name)
-      followed by variables from all submodules recursively (depth first).
+      A collection of variables for the current module (sorted by attribute
+      name) followed by variables from all submodules recursively (depth first).
     """
     return tuple(
         walk(self, recurse_if=_IS_MODULE, predicate=_IS_TRAINABLE_VARIABLE))
 
   @property
   def owned_trainable_variables(self):
-    """Returns a tuple of variables that are attributes of the current module.
+    """Collection of variables that are attributes of the current module.
 
     See `variables` for a property which returns all variables from the current
     module and all it's submodules recursively.
 
     Returns:
-      A tuple of variables which are attributes of the current module. Will
+      A collection of variables which are attributes of the current module. Will
       yield variables inside nested structures (lists etc) but not in other
       modules.
     """
-- 
GitLab


From c5d6b96282d6c5ef5a730e9bba7f6133f9e9d671 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 18 Jan 2019 07:32:53 -0800
Subject: [PATCH 0965/1095] Change UNDEFINED to UNSPECIFIED in code handling
 unspecified built-in arguments.

PiperOrigin-RevId: 229926077
---
 .../python/autograph/operators/py_builtins.py | 20 +++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/tensorflow/python/autograph/operators/py_builtins.py b/tensorflow/python/autograph/operators/py_builtins.py
index ddf05f73f3..9f54cf4fe5 100644
--- a/tensorflow/python/autograph/operators/py_builtins.py
+++ b/tensorflow/python/autograph/operators/py_builtins.py
@@ -37,7 +37,7 @@ from tensorflow.python.ops import list_ops
 from tensorflow.python.ops import math_ops
 
 
-UNDEFINED = object()
+UNSPECIFIED = object()
 
 
 def overload_of(f):
@@ -77,14 +77,14 @@ def _py_float(x):
   return float(x)
 
 
-def int_(x=0, base=UNDEFINED):
+def int_(x=0, base=UNSPECIFIED):
   if tensor_util.is_tensor(x):
     return _tf_int(x, base)
   return _py_int(x, base)
 
 
 def _tf_int(x, base):
-  if base not in (10, UNDEFINED):
+  if base not in (10, UNSPECIFIED):
     raise NotImplementedError('base {} not supported for int'.format(base))
 
   # TODO(mdan): We shouldn't assume int32.
@@ -94,7 +94,7 @@ def _tf_int(x, base):
 
 
 def _py_int(x, base):
-  if base is UNDEFINED:
+  if base is UNSPECIFIED:
     return int(x)
   return int(x, base)
 
@@ -167,7 +167,7 @@ def print_(*objects, **kwargs):
 
 def _tf_py_func_print(objects, kwargs):
   """Overload of print_ as a py_func implementation."""
-  override_kwargs = {k: v for k, v in kwargs.items() if v is not UNDEFINED}
+  override_kwargs = {k: v for k, v in kwargs.items() if v is not UNSPECIFIED}
   if 'flush' not in override_kwargs:
     # Defaulting to flushing the console in graph mode, which helps reduce
     # garbled output in IPython.
@@ -187,7 +187,7 @@ def _tf_py_func_print(objects, kwargs):
       print_wrapper, None, objects, use_dummy_return=True)
 
 
-def range_(start_or_stop, stop=UNDEFINED, step=UNDEFINED):
+def range_(start_or_stop, stop=UNSPECIFIED, step=UNSPECIFIED):
   if any(tensor_util.is_tensor(s) for s in (start_or_stop, stop, step)):
     return _tf_range(start_or_stop, stop, step)
   return _py_range(start_or_stop, stop, step)
@@ -200,10 +200,10 @@ def _tf_range(start_or_stop, stop, step):
   # graph construction error aligns the semantics with Python.
 
   # TODO(mdan): We should optimize this when a full tensor is not required.
-  if step is not UNDEFINED:
+  if step is not UNSPECIFIED:
     # TODO(mdan): Add argument coercion similar to other cases.
     return math_ops.range(start_or_stop, stop, step)
-  if stop is not UNDEFINED:
+  if stop is not UNSPECIFIED:
     stop = math_ops.maximum(start_or_stop, stop)
     return math_ops.range(start_or_stop, stop)
   start_or_stop = math_ops.maximum(start_or_stop, 0)
@@ -211,9 +211,9 @@ def _tf_range(start_or_stop, stop, step):
 
 
 def _py_range(start_or_stop, stop, step):
-  if step is not UNDEFINED:
+  if step is not UNSPECIFIED:
     return range(start_or_stop, stop, step)
-  if stop is not UNDEFINED:
+  if stop is not UNSPECIFIED:
     return range(start_or_stop, stop)
   return range(start_or_stop)
 
-- 
GitLab


From 8690cf7bf33f1e2d96ac1f2dea765d8aaca18f4d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 18 Jan 2019 08:03:30 -0800
Subject: [PATCH 0966/1095] Auto-linting on
 third_party/tensorflow/python/autograph/pyct/transformer.py

PiperOrigin-RevId: 229929152
---
 .../python/autograph/pyct/transformer.py      | 40 +++++++++++--------
 1 file changed, 24 insertions(+), 16 deletions(-)

diff --git a/tensorflow/python/autograph/pyct/transformer.py b/tensorflow/python/autograph/pyct/transformer.py
index 9c776a737a..3578f38c65 100644
--- a/tensorflow/python/autograph/pyct/transformer.py
+++ b/tensorflow/python/autograph/pyct/transformer.py
@@ -35,15 +35,17 @@ class AutoGraphParseError(SyntaxError):
 
 # TODO(mdan): Use namedtuple.
 class EntityInfo(object):
-  """Contains information about a Python entity. Immutable.
+  """Contains information about a Python entity.
+
+  Immutable.
 
   Examples of entities include functions and classes.
 
   Attributes:
     source_code: The entity's source code.
     source_file: The entity's source file.
-    namespace: Dict[str, ], containing symbols visible to the entity
-        (excluding parameters).
+    namespace: Dict[str, ], containing symbols visible to the entity (excluding
+      parameters).
     arg_values: dict[str->*], containing parameter values, if known.
     arg_types: dict[str->*], containing parameter types, if known.
     owner_type: The surrounding class type of the function, if present.
@@ -199,7 +201,9 @@ class Base(gast.NodeTransformer):
   # TODO(mdan): Document all extra features.
 
   def __init__(self, entity_info):
-    """Initialize the transformer. Subclasses should call this.
+    """Initialize the transformer.
+
+    Subclasses should call this.
 
     Args:
       entity_info: An EntityInfo object.
@@ -232,13 +236,15 @@ class Base(gast.NodeTransformer):
     return len(self._local_scope_state)
 
   def enter_local_scope(self, inherit=None):
-    """Deprecated. Use self.state instead.
+    """Deprecated.
+
+    Use self.state instead.
 
     Marks entry into a new local scope.
 
     Args:
-      inherit: Optional enumerable of variable names to copy from the
-          parent scope.
+      inherit: Optional enumerable of variable names to copy from the parent
+        scope.
     """
     scope_entered = {}
     if inherit:
@@ -249,13 +255,15 @@ class Base(gast.NodeTransformer):
     self._local_scope_state.append(scope_entered)
 
   def exit_local_scope(self, keep=None):
-    """Deprecated. Use self.state instead.
+    """Deprecated.
+
+    Use self.state instead.
 
     Marks exit from the current local scope.
 
     Args:
-      keep: Optional enumerable of variable names to copy into the
-          parent scope.
+      keep: Optional enumerable of variable names to copy into the parent scope.
+
     Returns:
       A dict containing the scope that has just been exited.
     """
@@ -390,11 +398,11 @@ class Base(gast.NodeTransformer):
 
     Args:
       targets: list, tuple of or individual AST node. Should be used with the
-          targets field of an ast.Assign node.
+        targets field of an ast.Assign node.
       values: an AST node.
       apply_fn: a function of a single argument, which will be called with the
-          respective nodes of each single assignment. The signature is
-          apply_fn(target, value), no return value.
+        respective nodes of each single assignment. The signature is
+        apply_fn(target, value), no return value.
     """
     if not isinstance(targets, (list, tuple)):
       targets = (targets,)
@@ -429,9 +437,9 @@ class Base(gast.NodeTransformer):
       # call `visit`.  The error needs to be raised before the exception handler
       # below is installed, because said handler will mess up if `node` is not,
       # in fact, a node.
-      msg = (
-          'invalid value for "node": expected "ast.AST", got "{}"; to'
-          ' visit lists of nodes, use "visit_block" instead').format(type(node))
+      msg = ('invalid value for "node": expected "ast.AST", got "{}"; to'
+             ' visit lists of nodes, use "visit_block" instead').format(
+                 type(node))
       raise ValueError(msg)
 
     did_enter_function = False
-- 
GitLab


From 7c65c591fb52b3a343af88835d560196d8290e8c Mon Sep 17 00:00:00 2001
From: Ayush Dubey <ayushd@google.com>
Date: Fri, 18 Jan 2019 08:08:57 -0800
Subject: [PATCH 0967/1095] Invert static ordering of collective ops when
 enabled.

PiperOrigin-RevId: 229929957
---
 tensorflow/core/graph/collective_order.cc      |  4 ++--
 tensorflow/core/graph/collective_order_test.cc | 16 ++++++++--------
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/tensorflow/core/graph/collective_order.cc b/tensorflow/core/graph/collective_order.cc
index bbba9264ce..c24166cb8e 100644
--- a/tensorflow/core/graph/collective_order.cc
+++ b/tensorflow/core/graph/collective_order.cc
@@ -92,8 +92,8 @@ Status CreateControlDependencies(
       const auto& deps_j = (*data_dependencies)[collective_nodes[j]];
       if (deps_i.find(instance_keys[j]) == deps_i.end() &&
           deps_j.find(instance_keys[i]) == deps_j.end()) {
-        int src_idx = instance_keys[i] < instance_keys[j] ? i : j;
-        int dst_idx = instance_keys[i] < instance_keys[j] ? j : i;
+        int src_idx = instance_keys[i] > instance_keys[j] ? i : j;
+        int dst_idx = instance_keys[i] > instance_keys[j] ? j : i;
         Node* src_node = collective_nodes[src_idx];
         Node* dst_node = collective_nodes[dst_idx];
         VLOG(1) << "Adding control dependency from node " << src_node->name()
diff --git a/tensorflow/core/graph/collective_order_test.cc b/tensorflow/core/graph/collective_order_test.cc
index 241c98b549..9a158e5c3f 100644
--- a/tensorflow/core/graph/collective_order_test.cc
+++ b/tensorflow/core/graph/collective_order_test.cc
@@ -137,18 +137,18 @@ std::unique_ptr<Graph> InitGraph() {
 }
 
 // Tests that in the graph created by `InitGraph`, exactly 2 control edges are
-// added after calling `OrderCollectives`: c2_0 -> c3_0 and c2_1 -> c3_1.
+// added after calling `OrderCollectives`: c3_0 -> c2_0 and c3_1 -> c2_1.
 TEST(CollectiveOrderTest, SimpleOrder) {
   std::unique_ptr<Graph> graph = InitGraph();
   TF_EXPECT_OK(OrderCollectives(graph.get(), GraphCollectiveOrder::kEdges));
   VerifyGraph(*graph, {"c1_0", "c1_1", "c2_0", "c2_1", "c3_0", "c3_1"},
-              {{"c2_0", "c3_0"}, {"c2_1", "c3_1"}});
+              {{"c3_0", "c2_0"}, {"c3_1", "c2_1"}});
 }
 
 TEST(CollectiveOrderTest, SimpleOrderAttr) {
   std::unique_ptr<Graph> graph = InitGraph();
   TF_EXPECT_OK(OrderCollectives(graph.get(), GraphCollectiveOrder::kAttrs));
-  VerifyAttrs(*graph, {{"c3_0", {2}}, {"c3_1", {2}}});
+  VerifyAttrs(*graph, {{"c2_0", {3}}, {"c2_1", {3}}});
 }
 
 // Initialize the following graph:
@@ -185,12 +185,12 @@ std::unique_ptr<Graph> InitGraph2() {
 }
 
 // Tests that in the graph created by `InitGraph2`, we add the following control
-// edges after calling `OrderCollectives`: c2 -> c3, c3 -> c4.  c2->c4 is
+// edges after calling `OrderCollectives`: c4 -> c3, c3 -> c2.  c4->c2 is
 // pruned because it follows from the other two edges.
 TEST(CollectiveOrderTest, SimpleOrder2) {
   std::unique_ptr<Graph> graph = InitGraph2();
   TF_EXPECT_OK(OrderCollectives(graph.get(), GraphCollectiveOrder::kEdges));
-  VerifyGraph(*graph, {"c1", "c2", "c3", "c4"}, {{"c2", "c3"}, {"c3", "c4"}});
+  VerifyGraph(*graph, {"c1", "c2", "c3", "c4"}, {{"c4", "c3"}, {"c3", "c2"}});
 }
 
 // Initialize the following graph:
@@ -223,12 +223,12 @@ std::unique_ptr<Graph> InitGraphForPruning() {
   return graph;
 }
 
-// Tests that in the graph created by `InitGraphForPruning`, we only add c1 ->
-// c2, c2 -> c3, c3 -> c4, and other edges are pruned away.
+// Tests that in the graph created by `InitGraphForPruning`, we only add c4 ->
+// c3, c3 -> c2, c2 -> c1, and other edges are pruned away.
 TEST(CollectiveOrderTest, Pruning) {
   std::unique_ptr<Graph> graph = InitGraphForPruning();
   TF_EXPECT_OK(OrderCollectives(graph.get(), GraphCollectiveOrder::kAttrs));
-  VerifyAttrs(*graph, {{"c4", {3}}, {"c3", {2}}, {"c2", {1}}});
+  VerifyAttrs(*graph, {{"c3", {4}}, {"c2", {3}}, {"c1", {2}}});
 }
 
 }  // namespace
-- 
GitLab


From 72571e08245505a8ca0a1550ee7f97212b1df113 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 18 Jan 2019 08:09:15 -0800
Subject: [PATCH 0968/1095] Roll CL 229790895 forward with fix.

PiperOrigin-RevId: 229930009
---
 tensorflow/opensource_only.files            |   3 -
 third_party/gpus/crosstool/remote.BUILD.tpl |  10 --
 third_party/gpus/cuda/remote.BUILD.tpl      | 110 --------------------
 third_party/gpus/cuda_configure.bzl         |  22 ++--
 third_party/tensorrt/build_defs.bzl.tpl     |   4 +-
 third_party/tensorrt/remote.BUILD.tpl       |   7 --
 third_party/tensorrt/tensorrt_configure.bzl |  16 +--
 7 files changed, 23 insertions(+), 149 deletions(-)
 delete mode 100644 third_party/gpus/crosstool/remote.BUILD.tpl
 delete mode 100644 third_party/gpus/cuda/remote.BUILD.tpl
 delete mode 100644 third_party/tensorrt/remote.BUILD.tpl

diff --git a/tensorflow/opensource_only.files b/tensorflow/opensource_only.files
index 850b5f5ca2..9c4dd2f4b8 100644
--- a/tensorflow/opensource_only.files
+++ b/tensorflow/opensource_only.files
@@ -82,7 +82,6 @@ tensorflow/third_party/gpus/crosstool/clang/bin/crosstool_wrapper_driver_is_not_
 tensorflow/third_party/gpus/crosstool/CROSSTOOL.tpl
 tensorflow/third_party/gpus/crosstool/CROSSTOOL_hipcc.tpl
 tensorflow/third_party/gpus/crosstool/LICENSE
-tensorflow/third_party/gpus/crosstool/remote.BUILD.tpl
 tensorflow/third_party/gpus/crosstool/windows/msvc_wrapper_for_nvcc.py.tpl
 tensorflow/third_party/gpus/crosstool/BUILD.tpl
 tensorflow/third_party/gpus/crosstool/BUILD
@@ -90,7 +89,6 @@ tensorflow/third_party/gpus/cuda/LICENSE
 tensorflow/third_party/gpus/cuda/BUILD.tpl
 tensorflow/third_party/gpus/cuda/BUILD.windows.tpl
 tensorflow/third_party/gpus/cuda/cuda_config.h.tpl
-tensorflow/third_party/gpus/cuda/remote.BUILD.tpl
 tensorflow/third_party/gpus/cuda/BUILD
 tensorflow/third_party/gpus/cuda/build_defs.bzl.tpl
 tensorflow/third_party/gpus/rocm/rocm_config.h.tpl
@@ -194,7 +192,6 @@ tensorflow/third_party/tensorrt/BUILD
 tensorflow/third_party/tensorrt/build_defs.bzl.tpl
 tensorflow/third_party/tensorrt/BUILD.tpl
 tensorflow/third_party/tensorrt/tensorrt_configure.bzl
-tensorflow/third_party/tensorrt/remote.BUILD.tpl
 tensorflow/third_party/kafka/config.patch
 tensorflow/third_party/kafka/BUILD
 tensorflow/third_party/android/BUILD
diff --git a/third_party/gpus/crosstool/remote.BUILD.tpl b/third_party/gpus/crosstool/remote.BUILD.tpl
deleted file mode 100644
index b2316331db..0000000000
--- a/third_party/gpus/crosstool/remote.BUILD.tpl
+++ /dev/null
@@ -1,10 +0,0 @@
-# Description:
-#   Template for crosstool Build file to use a pre-generated config.
-licenses(["restricted"])
-
-package(default_visibility = ["//visibility:public"])
-
-alias(
-    name = "toolchain",
-    actual = "%{remote_cuda_repo}:toolchain",
-)
diff --git a/third_party/gpus/cuda/remote.BUILD.tpl b/third_party/gpus/cuda/remote.BUILD.tpl
deleted file mode 100644
index 100c7bb7c4..0000000000
--- a/third_party/gpus/cuda/remote.BUILD.tpl
+++ /dev/null
@@ -1,110 +0,0 @@
-# Description:
-#   Template for cuda Build file to use a pre-generated config.
-licenses(["restricted"])  # MPL2, portions GPL v3, LGPL v3, BSD-like
-
-package(default_visibility = ["//visibility:public"])
-
-config_setting(
-    name = "using_nvcc",
-    values = {
-        "define": "using_cuda_nvcc=true",
-    },
-)
-
-config_setting(
-    name = "using_clang",
-    values = {
-        "define": "using_cuda_clang=true",
-    },
-)
-
-# Equivalent to using_clang && -c opt.
-config_setting(
-    name = "using_clang_opt",
-    values = {
-        "define": "using_cuda_clang=true",
-        "compilation_mode": "opt",
-    },
-)
-
-config_setting(
-    name = "darwin",
-    values = {"cpu": "darwin"},
-    visibility = ["//visibility:public"],
-)
-
-config_setting(
-    name = "freebsd",
-    values = {"cpu": "freebsd"},
-    visibility = ["//visibility:public"],
-)
-
-alias(
-    name = "cuda_headers",
-    actual = "%{remote_cuda_repo}/cuda:cuda_headers",
-)
-
-alias(
-    name = "cudart_static",
-    actual = "%{remote_cuda_repo}/cuda:cudart_static",
-)
-
-alias(
-    name = "cuda_driver",
-    actual = "%{remote_cuda_repo}/cuda:cuda_driver",
-)
-
-alias(
-    name = "cudart",
-    actual = "%{remote_cuda_repo}/cuda:cudart",
-)
-
-alias(
-    name = "cublas",
-    actual = "%{remote_cuda_repo}/cuda:cublas",
-)
-
-alias(
-    name = "cusolver",
-    actual = "%{remote_cuda_repo}/cuda:cusolver",
-)
-
-alias(
-    name = "cudnn",
-    actual = "%{remote_cuda_repo}/cuda:cudnn",
-)
-
-alias(
-    name = "cudnn_header",
-    actual = "%{remote_cuda_repo}/cuda:cudnn_header",
-)
-
-alias(
-    name = "cufft",
-    actual = "%{remote_cuda_repo}/cuda:cufft",
-)
-
-alias(
-    name = "curand",
-    actual = "%{remote_cuda_repo}/cuda:curand",
-)
-
-alias(
-    name = "cuda",
-    actual = "%{remote_cuda_repo}/cuda:cuda",
-)
-
-alias(
-    name = "cupti_headers",
-    actual = "%{remote_cuda_repo}/cuda:cupti_headers",
-)
-
-alias(
-    name = "cupti_dsos",
-    actual = "%{remote_cuda_repo}/cuda:cupti_dsos",
-)
-
-alias(
-    name = "libdevice_root",
-    actual = "%{remote_cuda_repo}/cuda:libdevice_root",
-)
diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl
index 90020b6608..76f2f3a07b 100644
--- a/third_party/gpus/cuda_configure.bzl
+++ b/third_party/gpus/cuda_configure.bzl
@@ -1508,17 +1508,21 @@ def _create_remote_cuda_repository(repository_ctx, remote_config_repo):
               ),
       },
   )
-  _tpl(
-      repository_ctx,
-      "cuda:remote.BUILD",
-      {
-          "%{remote_cuda_repo}": remote_config_repo,
-      },
+  repository_ctx.template(
       "cuda/BUILD",
+      Label(remote_config_repo + "/cuda:BUILD"),
+      {},
+  )
+  repository_ctx.template(
+      "cuda/build_defs.bzl",
+      Label(remote_config_repo + "/cuda:build_defs.bzl"),
+      {},
+  )
+  repository_ctx.template(
+      "cuda/cuda/cuda_config.h",
+      Label(remote_config_repo + "/cuda:cuda/cuda_config.h"),
+      {},
   )
-  _tpl(repository_ctx, "crosstool:remote.BUILD", {
-      "%{remote_cuda_repo}": remote_config_repo,
-  }, "crosstool/BUILD")
 
 
 def _cuda_autoconf_impl(repository_ctx):
diff --git a/third_party/tensorrt/build_defs.bzl.tpl b/third_party/tensorrt/build_defs.bzl.tpl
index 0dc3a7ba2d..6d00513827 100644
--- a/third_party/tensorrt/build_defs.bzl.tpl
+++ b/third_party/tensorrt/build_defs.bzl.tpl
@@ -2,6 +2,4 @@
 
 def if_tensorrt(if_true, if_false=[]):
   """Tests whether TensorRT was enabled during the configure process."""
-  if %{tensorrt_is_configured}:
-    return if_true
-  return if_false
+  return %{if_tensorrt}
diff --git a/third_party/tensorrt/remote.BUILD.tpl b/third_party/tensorrt/remote.BUILD.tpl
deleted file mode 100644
index d5afc4a5c1..0000000000
--- a/third_party/tensorrt/remote.BUILD.tpl
+++ /dev/null
@@ -1,7 +0,0 @@
-licenses(["notice"])  # Apache 2.0
-
-package(default_visibility = ["//visibility:public"])
-
-alias(name="LICENSE", actual = "%{target}:LICENSE")
-alias(name = "tensorrt_headers", actual = "%{target}:tensorrt_headers")
-alias(name = "tensorrt", actual = "%{target}:tensorrt")
diff --git a/third_party/tensorrt/tensorrt_configure.bzl b/third_party/tensorrt/tensorrt_configure.bzl
index d3bc388346..c6de25b33e 100644
--- a/third_party/tensorrt/tensorrt_configure.bzl
+++ b/third_party/tensorrt/tensorrt_configure.bzl
@@ -129,7 +129,7 @@ def _tpl(repository_ctx, tpl, substitutions):
 
 def _create_dummy_repository(repository_ctx):
   """Create a dummy TensorRT repository."""
-  _tpl(repository_ctx, "build_defs.bzl", {"%{tensorrt_is_configured}": "False"})
+  _tpl(repository_ctx, "build_defs.bzl", {"%{if_tensorrt}": "if_false"})
 
   _tpl(repository_ctx, "BUILD", {
       "%{tensorrt_genrules}": "",
@@ -141,11 +141,13 @@ def _tensorrt_configure_impl(repository_ctx):
   """Implementation of the tensorrt_configure repository rule."""
   if _TF_TENSORRT_CONFIG_REPO in repository_ctx.os.environ:
     # Forward to the pre-configured remote repository.
-    repository_ctx.template("BUILD", Label("//third_party/tensorrt:remote.BUILD.tpl"), {
-        "%{target}": repository_ctx.os.environ[_TF_TENSORRT_CONFIG_REPO],
-    })
-    # Set up config file.
-    _tpl(repository_ctx, "build_defs.bzl", {"%{tensorrt_is_configured}": "True"})
+    remote_config_repo = repository_ctx.os.environ[_TF_TENSORRT_CONFIG_REPO]
+    repository_ctx.template("BUILD", Label(remote_config_repo + ":BUILD"), {})
+    repository_ctx.template(
+        "build_defs.bzl",
+        Label(remote_config_repo + ":build_defs.bzl"),
+        {},
+    )
     return
 
   if _TENSORRT_INSTALL_PATH not in repository_ctx.os.environ:
@@ -194,7 +196,7 @@ def _tensorrt_configure_impl(repository_ctx):
   ))
 
   # Set up config file.
-  _tpl(repository_ctx, "build_defs.bzl", {"%{tensorrt_is_configured}": "True"})
+  _tpl(repository_ctx, "build_defs.bzl", {"%{if_tensorrt}": "if_true"})
 
   # Set up BUILD file.
   _tpl(repository_ctx, "BUILD", {
-- 
GitLab


From f6b81f458da1f7b68e06c6a4501c5c380d33d590 Mon Sep 17 00:00:00 2001
From: Ayush Dubey <ayushd@google.com>
Date: Fri, 18 Jan 2019 08:15:00 -0800
Subject: [PATCH 0969/1095] Automated rollback of commit
 681f6a6ac9344425549c8a38fcbfae854e7deddf

PiperOrigin-RevId: 229930698
---
 tensorflow/core/BUILD                         |   2 +-
 .../base_collective_executor.cc               |  33 ++
 .../common_runtime/base_collective_executor.h |  18 +
 .../collective_param_resolver_local.cc        | 159 ++++++---
 .../collective_param_resolver_local.h         |  15 +-
 .../core/common_runtime/collective_util.cc    |  31 ++
 .../core/common_runtime/collective_util.h     |  21 ++
 .../hierarchical_tree_broadcaster.h           |   5 +
 .../core/common_runtime/ring_reducer.cc       |  43 +--
 tensorflow/core/common_runtime/ring_reducer.h |  24 +-
 tensorflow/core/distributed_runtime/BUILD     |   1 +
 .../collective_param_resolver_distributed.cc  |  21 +-
 ...lective_param_resolver_distributed_test.cc |   2 +
 tensorflow/core/framework/collective.cc       |  11 +
 tensorflow/core/framework/collective.h        |  32 +-
 tensorflow/core/kernels/BUILD                 |  25 ++
 .../core/kernels/collective_nccl_reducer.cc   | 204 +++++++++++
 .../core/kernels/collective_nccl_reducer.h    |  49 +++
 .../kernels/collective_nccl_reducer_test.cc   | 331 ++++++++++++++++++
 tensorflow/core/kernels/collective_ops.cc     |  23 +-
 tensorflow/core/nccl/BUILD                    |   4 +-
 tensorflow/core/protobuf/worker.proto         |   1 +
 22 files changed, 927 insertions(+), 128 deletions(-)
 create mode 100644 tensorflow/core/kernels/collective_nccl_reducer.cc
 create mode 100644 tensorflow/core/kernels/collective_nccl_reducer.h
 create mode 100644 tensorflow/core/kernels/collective_nccl_reducer_test.cc

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 084db7a0fd..ceec270f99 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -3719,7 +3719,6 @@ tf_cc_tests(
     srcs = [
         "common_runtime/buf_rendezvous_test.cc",
         "common_runtime/collective_executor_mgr_test.cc",
-        "common_runtime/collective_param_resolver_local_test.cc",
         "common_runtime/collective_rma_local_test.cc",
         "common_runtime/device_resolver_local_test.cc",
         "common_runtime/device_set_test.cc",
@@ -3835,6 +3834,7 @@ tf_cc_tests(
     name = "higher_level_tests_needing_kernels",
     size = "small",
     srcs = [
+        "common_runtime/collective_param_resolver_local_test.cc",
         "graph/graph_constructor_test.cc",
     ],
     linkopts = select({
diff --git a/tensorflow/core/common_runtime/base_collective_executor.cc b/tensorflow/core/common_runtime/base_collective_executor.cc
index 92e56df181..03c0e9ce2b 100644
--- a/tensorflow/core/common_runtime/base_collective_executor.cc
+++ b/tensorflow/core/common_runtime/base_collective_executor.cc
@@ -296,4 +296,37 @@ Status BaseCollectiveExecutor::CreateCollective(
   return status;
 }
 
+bool BaseCollectiveExecutor::CheckDependencies(
+    const CollectiveParams& col_params) {
+  for (int32 instance : col_params.instance.impl_details.dependencies) {
+    auto find_iter = launched_.find(instance);
+    if (find_iter == launched_.end() || find_iter->second != 0) {
+      return false;
+    }
+  }
+  return true;
+}
+
+void BaseCollectiveExecutor::WaitForDependencies(
+    const CollectiveParams& col_params) {
+  mutex_lock l(launch_mu_);
+  while (!CheckDependencies(col_params)) {
+    launch_cv_.wait(l);
+  }
+}
+
+void BaseCollectiveExecutor::Launched(const CollectiveParams& col_params) {
+  mutex_lock l(launch_mu_);
+  if (launched_.find(col_params.instance.instance_key) == launched_.end()) {
+    const string& task_name =
+        col_params.instance.task_names[col_params.default_rank];
+    const int32 num_devices =
+        col_params.instance.num_devices_per_task.at(task_name);
+    launched_[col_params.instance.instance_key] = num_devices;
+  }
+  if (--launched_[col_params.instance.instance_key] == 0) {
+    launch_cv_.notify_all();
+  }
+}
+
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/base_collective_executor.h b/tensorflow/core/common_runtime/base_collective_executor.h
index 09826a8814..b711aa6d50 100644
--- a/tensorflow/core/common_runtime/base_collective_executor.h
+++ b/tensorflow/core/common_runtime/base_collective_executor.h
@@ -135,15 +135,33 @@ class BaseCollectiveExecutor : public CollectiveExecutor {
                                client_locality, done);
   }
 
+  // If we need to enforce an ordering on any portion of collective
+  // implementation, and the ordering is encoded via attribute on the collective
+  // op, this function will block until all dependencies for this collective
+  // have completed.
+  void WaitForDependencies(const CollectiveParams& col_params) override;
+  // Record that this collective has completed the portion of the implementation
+  // that needs to be ordered wrt other collectives, to unblock any of its
+  // dependent ops.
+  void Launched(const CollectiveParams& col_params) override;
+
  protected:
   const int64 step_id_;
   const DeviceMgr* dev_mgr_;  // Not owned.
   std::unique_ptr<PerStepCollectiveRemoteAccess> remote_access_;
   const string* gpu_ring_order_;  // Not owned.
+  mutex launch_mu_;
+  condition_variable launch_cv_;
+  // collective instance key -> number of local devices for which NCCL ops have
+  // been launched.
+  std::unordered_map<int32, int32> launched_ GUARDED_BY(launch_mu_);
 
  private:
   Status CreateCollective(const CollectiveParams& col_params,
                           CollectiveImplementationInterface** col_impl);
+  // Check if all ops on which this collective depends on have launched.
+  bool CheckDependencies(const CollectiveParams& col_params)
+      EXCLUSIVE_LOCKS_REQUIRED(launch_mu_);
 };
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/collective_param_resolver_local.cc b/tensorflow/core/common_runtime/collective_param_resolver_local.cc
index a8e3f4c881..8907f6d56a 100644
--- a/tensorflow/core/common_runtime/collective_param_resolver_local.cc
+++ b/tensorflow/core/common_runtime/collective_param_resolver_local.cc
@@ -26,6 +26,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/gtl/flatmap.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/types.h"
 #include "tensorflow/core/util/device_name_utils.h"
@@ -39,7 +40,10 @@ void CollectiveParamResolverLocal::InstanceRec::WaitForOutMu(mutex_lock& lock) {
 CollectiveParamResolverLocal::CollectiveParamResolverLocal(
     const DeviceMgr* dev_mgr, DeviceResolverInterface* dev_resolver,
     const string& task_name)
-    : dev_mgr_(dev_mgr), dev_resolver_(dev_resolver), task_name_(task_name) {}
+    : nccl_(false),  // (b/111897089): turn on NCCL collectives.
+      dev_mgr_(dev_mgr),
+      dev_resolver_(dev_resolver),
+      task_name_(task_name) {}
 
 void CollectiveParamResolverLocal::CompleteGroupAsync(
     const CompleteGroupRequest* request, CompleteGroupResponse* response,
@@ -316,29 +320,28 @@ GlobalDeviceMap EstablishGlobalRank(
 // cp->same_num_devices_per_task.  Requires cp->instance.task_names
 // be sorted.
 void SetDevPerTask(CollectiveParams* cp) {
-  cp->instance.same_num_devices_per_task = false;
-  if (cp->instance.task_names.empty()) return;
-  int dev_per_task = -1;
-  int count = 0;
+  cp->instance.num_devices_per_task.clear();
   const string* last_task_name = &cp->instance.task_names[0];
+  int count = 0;
   for (const string& task_name : cp->instance.task_names) {
-    if (task_name != *last_task_name) {
-      CHECK_GT(count, 0);
-      if (dev_per_task < 0) {
-        dev_per_task = count;
-      } else {
-        CHECK_GT(dev_per_task, 0);
-        if (count != dev_per_task) return;
-      }
+    if (task_name == *last_task_name) {
+      ++count;
+    } else {
+      cp->instance.num_devices_per_task[*last_task_name] = count;
       count = 1;
       last_task_name = &task_name;
-    } else {
-      ++count;
     }
   }
-  CHECK_GT(count, 0);
-  if ((dev_per_task > 0) && (count != dev_per_task)) {
-    return;
+  cp->instance.num_devices_per_task[*last_task_name] = count;
+
+  cp->instance.same_num_devices_per_task = false;
+  int dev_per_task = -1;
+  for (const auto& task_dev : cp->instance.num_devices_per_task) {
+    if (dev_per_task == -1) {
+      dev_per_task = task_dev.second;
+    } else if (dev_per_task != task_dev.second) {
+      return;
+    }
   }
   cp->instance.same_num_devices_per_task = true;
   CHECK_EQ((cp->group.group_size % cp->group.num_tasks), 0);
@@ -398,7 +401,6 @@ void CollectiveParamResolverLocal::SetDefaultRank(const string& device,
 void CollectiveParamResolverLocal::InitInstanceSharedParams(
     const GroupRec* gr, const CollectiveParams* cp, InstanceRec* ir,
     const StatusCallback& done) {
-  VLOG(1) << "InitInstanceSharedParams " << ir;
   ir->shared.instance = cp->instance;
   {
     mutex_lock gl(gr->mu);
@@ -412,8 +414,8 @@ void CollectiveParamResolverLocal::InitInstanceSharedParams(
   }
   ir->shared.default_rank = -1;
 
-  // Sort devce_names lexicographcally, keeping task_names in
-  // corresponding order.
+  // Sort device_names lexicographically, keeping task_names in corresponding
+  // order.  Also set number of devices per task.
   SortDevicesAndTasks(&ir->shared);
 
   // Get Locality data for all devices.
@@ -605,6 +607,25 @@ void CollectiveParamResolverLocal::CompleteInstanceAsync(
                        "intended only for non-distributed deployment."));
 }
 
+// TODO(b/111897089): we need a better way to pick the collective
+// implementation.  The ideal way would depend upon the topology and link
+// strength before picking a particular implementation.
+void CollectiveParamResolverLocal::AssignCollectiveType(CollectiveParams* cp) {
+  if (cp->instance.type == BROADCAST_COLLECTIVE) {
+    cp->instance.impl_details.collective_name = "HierarchicalTreeBroadcast";
+  } else if (cp->instance.type == REDUCTION_COLLECTIVE) {
+    if (nccl_) {
+      cp->instance.impl_details.collective_name = "NcclReduce";
+    } else {
+      cp->instance.impl_details.collective_name = "RingReduce";
+    }
+  } else {
+    cp->instance.impl_details.collective_name = "undef";
+  }
+  VLOG(1) << "AssignCollectiveType "
+          << cp->instance.impl_details.collective_name;
+}
+
 void CollectiveParamResolverLocal::CompleteInstanceLocal(
     const string& device, const GroupRec* gr, CollectiveParams* cp,
     bool is_source, const StatusCallback& done) {
@@ -641,48 +662,57 @@ void CollectiveParamResolverLocal::CompleteInstanceFromInitializedIRec(
     // custom operator= does a deep copy.
     cp->instance = ir->shared.instance;
   }
-  // Populate the fields common across task, also default_rank.
+  // Populate the fields common across task.
+  AssignCollectiveType(cp);
   SetDefaultRank(device, cp);
   CompleteTaskIsLocal(task_name_, cp);
-  // TODO(b/113171733): we need a better way to pick the collective
-  // implementation.  The ideal way would depend upon the topology and link
-  // strength before picking a particular implementation.
-  cp->instance.impl_details.collective_name =
-      (cp->instance.type == BROADCAST_COLLECTIVE) ? "HierarchicalTreeBroadcast"
-                                                  : "RingReduce";
+
   CollectiveImplementationInterface* col_impl;
-  Status lookup_status = CollectiveRegistry::LookupParamResolverInstance(
+  Status status = CollectiveRegistry::LookupParamResolverInstance(
       cp->instance.impl_details.collective_name, &col_impl);
-  if (!lookup_status.ok()) {
-    done(lookup_status);
+  if (status.ok()) {
+    status = col_impl->InitializeInstanceBeforeGroupDiscovery(cp);
+  }
+  if (!status.ok()) {
+    done(status);
     return;
   }
-  // If broadcast, may need to wait for source discovery.
-  if (cp->instance.type == BROADCAST_COLLECTIVE) {
-    CompleteInstanceSource(ir, cp, is_source,
-                           [col_impl, ir, device, cp, done](InstanceRec* irec) {
-                             CHECK_EQ(ir, irec);
-                             Status s;
-                             {
-                               mutex_lock l(irec->out_mu);
-                               irec->WaitForOutMu(l);
-                               s = irec->status;
-                               cp->source_rank = irec->source_rank;
-                             }
-                             if (s.ok()) {
-                               s = col_impl->InitializeCollectiveParams(cp);
-                             }
-                             done(s);
-                           });
+
+  //  We may need to wait for the group if:
+  //  * this is a broadcast, for source discovery;
+  //  * we are using NCCL with more than 1 worker, for the communicator key from
+  //    rank 0.
+  bool broadcast = cp->instance.type == BROADCAST_COLLECTIVE;
+  bool nccl = cp->instance.type == REDUCTION_COLLECTIVE &&
+              cp->instance.impl_details.collective_name == "NcclReduce" &&
+              cp->group.num_tasks > 1;
+  if (broadcast || nccl) {
+    WaitForGroup(ir, cp, is_source, broadcast, nccl,
+                 [col_impl, ir, device, cp, done](InstanceRec* irec) {
+                   Status s;
+                   if (ir != irec) {
+                     s = errors::Internal("Expected ir ", ir, " and irec ",
+                                          irec, " to be equal");
+                   } else {
+                     mutex_lock l(irec->out_mu);
+                     irec->WaitForOutMu(l);
+                     s = irec->status;
+                     cp->source_rank = irec->source_rank;
+                     cp->instance.communicator_key = irec->communicator_key;
+                   }
+                   if (s.ok()) {
+                     s = col_impl->InitializeCollectiveParams(cp);
+                   }
+                   done(s);
+                 });
   } else {
     done(col_impl->InitializeCollectiveParams(cp));
   }
 }
 
-void CollectiveParamResolverLocal::CompleteInstanceSource(InstanceRec* ir,
-                                                          CollectiveParams* cp,
-                                                          bool is_source,
-                                                          const IRConsumer& f) {
+void CollectiveParamResolverLocal::WaitForGroup(
+    InstanceRec* ir, CollectiveParams* cp, bool is_source, bool init_source,
+    bool init_nccl, const IRConsumer& f) {
   std::vector<IRConsumer> ready_waiters;
   {
     mutex_lock l(ir->out_mu);
@@ -692,7 +722,8 @@ void CollectiveParamResolverLocal::CompleteInstanceSource(InstanceRec* ir,
     if (!ir->known[cp->default_rank]) {
       ir->known[cp->default_rank] = true;
       ++ir->known_count;
-      if (is_source) {
+      if (init_source && is_source) {
+        // Initialize source rank.
         if (ir->source_rank >= 0) {
           ir->status = errors::Internal("Instance ", cp->instance.instance_key,
                                         " already has source ", ir->source_rank,
@@ -702,13 +733,26 @@ void CollectiveParamResolverLocal::CompleteInstanceSource(InstanceRec* ir,
           ir->source_rank = cp->default_rank;
         }
       }
+      if (init_nccl && cp->default_rank == 0) {
+        // Initialize communicator key.
+        if (!ir->communicator_key.empty()) {
+          ir->status =
+              errors::Internal("Instance ", cp->instance.instance_key,
+                               " already has communicator_key ",
+                               str_util::CEscape(ir->communicator_key),
+                               ", received second claim from device ",
+                               cp->instance.device_names[cp->default_rank]);
+        } else {
+          ir->communicator_key = cp->instance.communicator_key;
+        }
+      }
     }
     if (ir->known_count < ir->shared.group.group_size) {
       ir->known_waiters.push_back(f);
       return;
     }
     CHECK_EQ(ir->known_count, ir->shared.group.group_size);
-    if (ir->source_rank < 0) {
+    if (init_source && ir->source_rank < 0) {
       // NOTE(ayushd): changing the error message below would also require
       // updating CompleteParamsBroadcastForgotSend test in
       // CollectiveParamResolverLocalTest.
@@ -718,6 +762,13 @@ void CollectiveParamResolverLocal::CompleteInstanceSource(InstanceRec* ir,
                            "could mean that there were group_size=",
                            ir->known_count, " BcastRecvs but no BcastSend.");
     }
+    if (init_nccl && ir->communicator_key.empty()) {
+      ir->status = errors::Internal(
+          "Instance ", cp->instance.instance_key, " device ",
+          cp->instance.device_names[cp->default_rank],
+          " did not find rank 0 for setting communicator key.  This is an "
+          "internal error in collective param resolution");
+    }
     if (!ir->known_waiters.empty()) {
       ready_waiters = std::move(ir->known_waiters);
     }
diff --git a/tensorflow/core/common_runtime/collective_param_resolver_local.h b/tensorflow/core/common_runtime/collective_param_resolver_local.h
index 365bddc787..fd408e4ef3 100644
--- a/tensorflow/core/common_runtime/collective_param_resolver_local.h
+++ b/tensorflow/core/common_runtime/collective_param_resolver_local.h
@@ -130,8 +130,10 @@ class CollectiveParamResolverLocal : public ParamResolverInterface {
     Status status GUARDED_BY(out_mu);
 
     // These fields are used to count the instances that have called
-    // in and become known while resolving broadcast source identity.
+    // in and become known while resolving broadcast source identity and
+    // communicator key.
     int source_rank GUARDED_BY(out_mu);
+    string communicator_key GUARDED_BY(out_mu);
     int known_count GUARDED_BY(out_mu);
     std::vector<bool> known GUARDED_BY(out_mu);
     std::vector<IRConsumer> known_waiters GUARDED_BY(out_mu);
@@ -197,10 +199,10 @@ class CollectiveParamResolverLocal : public ParamResolverInterface {
                                            const StatusCallback& done)
       LOCKS_EXCLUDED(ir->out_mu);
 
-  // Complete source data for a broadcast instance.
+  // Complete source data and/or nccl communicator key.
   // Precondition: *cp has complete group data and default_rank.
-  void CompleteInstanceSource(InstanceRec* ir, CollectiveParams* cp,
-                              bool is_source, const IRConsumer& f)
+  void WaitForGroup(InstanceRec* ir, CollectiveParams* cp, bool is_source,
+                    bool init_source, bool init_nccl, const IRConsumer& f)
       LOCKS_EXCLUDED(ir->out_mu);
 
   // If cp.device_names contains only devices local to this process
@@ -216,10 +218,15 @@ class CollectiveParamResolverLocal : public ParamResolverInterface {
   // current ordering of cp->instance.device_names.
   void SetDefaultRank(const string& device, CollectiveParams* cp);
 
+  // Sets cp->instance.type based on collective op type, and attempts to assign
+  // best implementation.
+  void AssignCollectiveType(CollectiveParams* cp);
+
   // Helper to grab status under lock, invoke callback out of lock.
   void CallbackWithStatus(const InstanceRecCallback& done, InstanceRec* irec)
       LOCKS_EXCLUDED(irec->out_mu);
 
+  const bool nccl_;
   const DeviceMgr* dev_mgr_;
   DeviceResolverInterface* dev_resolver_;  // Not owned.
   string task_name_;
diff --git a/tensorflow/core/common_runtime/collective_util.cc b/tensorflow/core/common_runtime/collective_util.cc
index 195521a078..bee4a13d18 100644
--- a/tensorflow/core/common_runtime/collective_util.cc
+++ b/tensorflow/core/common_runtime/collective_util.cc
@@ -79,5 +79,36 @@ string SubdivPermDebugString(const CollectiveParams& col_params) {
   return buf;
 }
 
+SubContext::SubContext(OpKernelContext* ctx, OpKernelContext::Params* params,
+                       OpKernel* op, Tensor* output, Tensor* input)
+    : sub_params_(*params),
+      sub_inputs_({output, input}),
+      sub_input_attr_({ctx->input_alloc_attr(0), ctx->input_alloc_attr(0)}),
+      sub_input_dc_(
+          {ctx->input_device_context(0), ctx->input_device_context(0)}) {
+  sub_params_.op_kernel = op;
+  sub_params_.inputs = &sub_inputs_;
+  sub_params_.input_alloc_attrs = &sub_input_attr_;
+  sub_params_.input_device_contexts = &sub_input_dc_;
+  sub_params_.eigen_gpu_device = nullptr;
+  sub_params_.ensure_eigen_gpu_device();
+  sub_params_.forward_from_array = &forward_from_;
+  sub_ctx_.reset(new OpKernelContext(&sub_params_, 1));
+}
+
+Status ComputeBinOp(OpKernelContext* op_ctx, OpKernelContext::Params* params,
+                    Device* device, OpKernel* op, Tensor* output,
+                    Tensor* input) {
+  // Prepare an OpKernelContext that is identical to that of the original Op
+  // (i.e. the collective), except for the input output sizes and identities and
+  // the Op itself.
+  // TODO(ayushd, tucker): Is it possible to cache and reuse these objects?
+  // They're mostly identical inside one device execution.
+  std::unique_ptr<SubContext> sub_ctx(
+      new SubContext(op_ctx, params, op, output, input));
+  device->Compute(op, sub_ctx->sub_ctx_.get());
+  return sub_ctx->sub_ctx_->status();
+}
+
 }  // namespace collective_util
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/collective_util.h b/tensorflow/core/common_runtime/collective_util.h
index ebb5731bec..01fb8b8c81 100644
--- a/tensorflow/core/common_runtime/collective_util.h
+++ b/tensorflow/core/common_runtime/collective_util.h
@@ -32,6 +32,27 @@ Status InitializeDeviceAndLocality(const DeviceMgr* dev_mgr,
                                    DeviceLocality* device_locality);
 string SubdivPermDebugString(const CollectiveParams& col_params);
 
+// Used for executing a sub-operation, e.g. a merge_op instance, with
+// an OpKernelContext based on the one passed into this Op.
+class SubContext {
+ public:
+  OpKernelContext::Params sub_params_;
+  gtl::InlinedVector<TensorValue, 4> sub_inputs_;
+  gtl::InlinedVector<AllocatorAttributes, 4> sub_input_attr_;
+  gtl::InlinedVector<DeviceContext*, 4> sub_input_dc_;
+  // Used only for Binary and Unary Ops for which we require
+  // the calculation to be in-place on the first input.
+  int forward_from_ = 0;
+  std::unique_ptr<OpKernelContext> sub_ctx_;
+  SubContext(OpKernelContext* ctx, OpKernelContext::Params* params,
+             OpKernel* op, Tensor* output, Tensor* input);
+  ~SubContext() = default;
+};
+
+Status ComputeBinOp(OpKernelContext* op_ctx, OpKernelContext::Params* params,
+                    Device* device, OpKernel* op, Tensor* output,
+                    Tensor* input);
+
 }  // namespace collective_util
 }  // namespace tensorflow
 
diff --git a/tensorflow/core/common_runtime/hierarchical_tree_broadcaster.h b/tensorflow/core/common_runtime/hierarchical_tree_broadcaster.h
index ceb9baad30..76392b8e59 100644
--- a/tensorflow/core/common_runtime/hierarchical_tree_broadcaster.h
+++ b/tensorflow/core/common_runtime/hierarchical_tree_broadcaster.h
@@ -41,6 +41,11 @@ class HierarchicalTreeBroadcaster : public CollectiveImplementationInterface {
   // and device_locality.  Also saves the CollectiveContext in this object.
   Status InitializeCollectiveContext(CollectiveContext* col_ctx) override;
 
+  // No-op for hierarchical tree broadcaster.
+  Status InitializeInstanceBeforeGroupDiscovery(CollectiveParams*) override {
+    return Status::OK();
+  }
+
   // Begins async execution of the hierarchical tree broadcast.
   // Must be called in a blockable thread.
   // TODO(b/80529858): remove the previous warning when we have a dedicated
diff --git a/tensorflow/core/common_runtime/ring_reducer.cc b/tensorflow/core/common_runtime/ring_reducer.cc
index 092f15e49e..8ed2fc2f1c 100644
--- a/tensorflow/core/common_runtime/ring_reducer.cc
+++ b/tensorflow/core/common_runtime/ring_reducer.cc
@@ -394,37 +394,6 @@ void RingReducer::Finish(bool ok) {
   done_(s);
 }
 
-RingReducer::SubContext::SubContext(OpKernelContext* ctx,
-                                    OpKernelContext::Params* params,
-                                    OpKernel* op, Tensor* output, Tensor* input)
-    : sub_params_(*params),
-      sub_inputs_({output, input}),
-      sub_input_attr_({ctx->input_alloc_attr(0), ctx->input_alloc_attr(0)}),
-      sub_input_dc_(
-          {ctx->input_device_context(0), ctx->input_device_context(0)}) {
-  sub_params_.op_kernel = op;
-  sub_params_.inputs = &sub_inputs_;
-  sub_params_.input_alloc_attrs = &sub_input_attr_;
-  sub_params_.input_device_contexts = &sub_input_dc_;
-  sub_params_.eigen_gpu_device = nullptr;
-  sub_params_.ensure_eigen_gpu_device();
-  sub_params_.forward_from_array = &forward_from_;
-  sub_ctx_ = new OpKernelContext(&sub_params_, 1);
-}
-
-Status RingReducer::ComputeBinOp(Device* device, OpKernel* op, Tensor* output,
-                                 Tensor* input) {
-  // Prepare an OpKernelContext that is identical to that of the original Op
-  // (i.e. the collective), except for the input output sizes and identities and
-  // the Op itself.
-  // TODO(tucker): Is it possible to cache and reuse these objects?  They're
-  // mostly identical inside one device execution.
-  std::unique_ptr<SubContext> sub_ctx(
-      new SubContext(col_ctx_->op_ctx, col_ctx_->op_params, op, output, input));
-  device->Compute(op, sub_ctx->sub_ctx_);
-  return sub_ctx->sub_ctx_->status();
-}
-
 // At the beginning of the algorithm initialize a RingField struct for
 // every independent field of the tensor.
 void RingReducer::InitRingField(RingField* rf, int chunk_idx, int subdiv_idx,
@@ -632,9 +601,9 @@ bool RingReducer::RunAsyncParts() {
           --recv_pending_count;
           if (!rf->second_pass) {
             rf->action = RF_REDUCE;
-            Status s =
-                ComputeBinOp(col_ctx_->device, col_params_->merge_op.get(),
-                             &rf->chunk, &rf->tmp_chunk);
+            Status s = collective_util::ComputeBinOp(
+                col_ctx_->op_ctx, col_ctx_->op_params, col_ctx_->device,
+                col_params_->merge_op.get(), &rf->chunk, &rf->tmp_chunk);
             if (!s.ok()) {
               aborted = true;
               StartAbort(s);
@@ -647,9 +616,9 @@ bool RingReducer::RunAsyncParts() {
           if (!rf->second_pass && col_params_->final_op.get() && rf->is_final) {
             rf->action = RF_FINALIZE;
             group_size_tensor_ready_.WaitForNotification();
-            Status s =
-                ComputeBinOp(col_ctx_->device, col_params_->final_op.get(),
-                             &rf->chunk, &group_size_tensor_);
+            Status s = collective_util::ComputeBinOp(
+                col_ctx_->op_ctx, col_ctx_->op_params, col_ctx_->device,
+                col_params_->final_op.get(), &rf->chunk, &group_size_tensor_);
             if (!s.ok()) {
               aborted = true;
               StartAbort(s);
diff --git a/tensorflow/core/common_runtime/ring_reducer.h b/tensorflow/core/common_runtime/ring_reducer.h
index 0848e37b52..a5aa8fad70 100644
--- a/tensorflow/core/common_runtime/ring_reducer.h
+++ b/tensorflow/core/common_runtime/ring_reducer.h
@@ -40,6 +40,11 @@ class RingReducer : public CollectiveImplementationInterface {
   // and device_locality.  Also saves the CollectiveContext in this object.
   Status InitializeCollectiveContext(CollectiveContext* col_ctx) override;
 
+  // No-op for ring reducer.
+  Status InitializeInstanceBeforeGroupDiscovery(CollectiveParams*) override {
+    return Status::OK();
+  }
+
   // Begins async execution of the ring reduce algorithm.
   // Must be called in a blockable thread.
   // TODO(b/80529858): remove the previous warning when we have a dedicated
@@ -52,27 +57,8 @@ class RingReducer : public CollectiveImplementationInterface {
   void StartAbort(const Status& s);
   void ContinueAfterInputCopy();
   void Finish(bool ok);
-  Status ComputeBinOp(Device* device, OpKernel* op, Tensor* output,
-                      Tensor* input);
   bool RunAsyncParts();
 
-  // Used for executing a sub-operation, e.g. a merge_op instance, with
-  // an OpKernelContext based on the one passed into this Op.
-  class SubContext {
-   public:
-    OpKernelContext::Params sub_params_;
-    gtl::InlinedVector<TensorValue, 4> sub_inputs_;
-    gtl::InlinedVector<AllocatorAttributes, 4> sub_input_attr_;
-    gtl::InlinedVector<DeviceContext*, 4> sub_input_dc_;
-    // Used only for Binary and Unary Ops for which we require
-    // the calculation to be in-place on the first input.
-    int forward_from_ = 0;
-    OpKernelContext* sub_ctx_;
-    SubContext(OpKernelContext* ctx, OpKernelContext::Params* params,
-               OpKernel* op, Tensor* output, Tensor* input);
-    ~SubContext() { delete sub_ctx_; }
-  };
-
   // Current status of a RingField
   enum RingFieldAction {
     RF_INIT = 0,    // Just initialized for a pass
diff --git a/tensorflow/core/distributed_runtime/BUILD b/tensorflow/core/distributed_runtime/BUILD
index e388d3e6f0..197a3f24d4 100644
--- a/tensorflow/core/distributed_runtime/BUILD
+++ b/tensorflow/core/distributed_runtime/BUILD
@@ -594,6 +594,7 @@ tf_cc_test(
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
+        "//tensorflow/core/kernels:collective_ops",
     ],
 )
 
diff --git a/tensorflow/core/distributed_runtime/collective_param_resolver_distributed.cc b/tensorflow/core/distributed_runtime/collective_param_resolver_distributed.cc
index 1dd10d309b..9f94a24fcd 100644
--- a/tensorflow/core/distributed_runtime/collective_param_resolver_distributed.cc
+++ b/tensorflow/core/distributed_runtime/collective_param_resolver_distributed.cc
@@ -181,6 +181,10 @@ void CollectiveParamResolverDistributed::CompleteInstanceAsync(
                           ir->WaitForOutMu(l);
                           response->set_instance_key(cp->instance.instance_key);
                           response->set_source_rank(ir->source_rank);
+                          if (!cp->instance.communicator_key.empty()) {
+                            response->set_communicator_key(
+                                cp->instance.communicator_key);
+                          }
                           done_and_cleanup(fi_status);
                         } else {
                           done_and_cleanup(fi_status);
@@ -283,8 +287,10 @@ void CollectiveParamResolverDistributed::UpdateInstanceCache(
   using InstanceRecPointer = InstanceRec*;
   InstanceRecPointer* irp = new InstanceRecPointer(nullptr);
   int32 source_rank = resp.source_rank();
+  string communicator_key = resp.communicator_key();
 
-  auto continue_with_ir = [this, cp, irp, source_rank, done](const Status& s) {
+  auto continue_with_ir = [cp, irp, source_rank, communicator_key,
+                           done](const Status& s) {
     if (!s.ok()) {
       done(s);
       delete irp;
@@ -306,6 +312,19 @@ void CollectiveParamResolverDistributed::UpdateInstanceCache(
         }
         ir->source_rank = source_rank;
       }
+      if (ir->communicator_key != communicator_key) {
+        if (!ir->communicator_key.empty()) {
+          ir->status = errors::Internal(
+              "UpdateInstanceCache: CompleteInstanceResponse for instance ",
+              cp->instance.instance_key,
+              " gives communicator_key with size =", communicator_key.size(),
+              " but cache already holds communicator_key with size=",
+              ir->communicator_key.size());
+          status = ir->status;
+          break;
+        }
+        ir->communicator_key = communicator_key;
+      }
       if (ir->known_count < cp->group.group_size) {
         ir->known_count = cp->group.group_size;
         if (ir->known.size() != cp->group.group_size) {
diff --git a/tensorflow/core/distributed_runtime/collective_param_resolver_distributed_test.cc b/tensorflow/core/distributed_runtime/collective_param_resolver_distributed_test.cc
index 40b18d321a..823d7d5eb9 100644
--- a/tensorflow/core/distributed_runtime/collective_param_resolver_distributed_test.cc
+++ b/tensorflow/core/distributed_runtime/collective_param_resolver_distributed_test.cc
@@ -268,6 +268,8 @@ class DeviceResDistTest : public ::testing::Test {
         EXPECT_EQ(cp_[idx].instance.device_names[idx], device_name);
         EXPECT_EQ(cp_[idx].instance.task_names[idx], task_name);
         if (idx > 0) {
+          EXPECT_EQ(cp_[0].instance.communicator_key,
+                    cp_[idx].instance.communicator_key);
           for (int i = 0; i < dev_count; ++i) {
             EXPECT_EQ(cp_[0].instance.device_names[i],
                       cp_[idx].instance.device_names[i]);
diff --git a/tensorflow/core/framework/collective.cc b/tensorflow/core/framework/collective.cc
index 7fa58347f2..b83d183f14 100644
--- a/tensorflow/core/framework/collective.cc
+++ b/tensorflow/core/framework/collective.cc
@@ -17,6 +17,7 @@ limitations under the License.
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/hash/hash.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 
 namespace tensorflow {
@@ -64,7 +65,9 @@ CollInstanceParams& CollInstanceParams::operator=(
     device_names.assign(other.device_names.begin(), other.device_names.end());
     task_names.assign(other.task_names.begin(), other.task_names.end());
     same_num_devices_per_task = other.same_num_devices_per_task;
+    num_devices_per_task = other.num_devices_per_task;
     gpu_ring_order = other.gpu_ring_order;
+    communicator_key = other.communicator_key;
     impl_details.subdiv_offsets.assign(
         other.impl_details.subdiv_offsets.begin(),
         other.impl_details.subdiv_offsets.end());
@@ -76,6 +79,7 @@ CollInstanceParams& CollInstanceParams::operator=(
     impl_details.subdiv_source_rank.assign(
         other.impl_details.subdiv_source_rank.begin(),
         other.impl_details.subdiv_source_rank.end());
+    impl_details.dependencies = other.impl_details.dependencies;
   }
   return *this;
 }
@@ -91,6 +95,13 @@ string CollInstanceParams::ToString() const {
   for (const auto& n : task_names) {
     strings::StrAppend(&v, n, ", ");
   }
+  strings::StrAppend(&v, "} num_devices_per_task={");
+  for (const auto dpt : num_devices_per_task) {
+    strings::StrAppend(&v, dpt.first, ": ", dpt.second, ", ");
+  }
+  strings::StrAppend(&v, "}, collective_name=", impl_details.collective_name,
+                     ", communicator_key=", str_util::CEscape(communicator_key),
+                     ", subdiv_offsets={");
   strings::StrAppend(&v, "}, subdiv_offsets={");
   for (const auto& d : impl_details.subdiv_offsets) {
     strings::StrAppend(&v, d, ",");
diff --git a/tensorflow/core/framework/collective.h b/tensorflow/core/framework/collective.h
index 0321429702..546e3938a8 100644
--- a/tensorflow/core/framework/collective.h
+++ b/tensorflow/core/framework/collective.h
@@ -70,6 +70,8 @@ struct CollImplDetails {
   std::vector<std::vector<int>> subdiv_permutations;
   std::vector<int> subdiv_offsets;
   std::vector<int> subdiv_source_rank;  // rank of source in each subdiv
+  std::vector<int32>
+      dependencies;  // collective instances on which this node depends
 };
 
 // Data common to all members of a collective instance.
@@ -85,9 +87,13 @@ struct CollInstanceParams {
   std::vector<string> task_names;
   // True if every task has the same number of devices.
   bool same_num_devices_per_task = false;
+  // Task -> number of devices on that task.
+  std::unordered_map<string, int32> num_devices_per_task;
   // If passed in to GPUOptions in ConfigProto, defines a good ring order for
   // GPUs.  Assumes same GPU configuration at each worker.
   string gpu_ring_order = "";
+  // Valid when using a communicator-based collective mechanism, e.g. NCCL.
+  string communicator_key;
   CollImplDetails impl_details;
   string ToString() const;
   CollInstanceParams& operator=(const struct CollInstanceParams& other);
@@ -269,6 +275,21 @@ class CollectiveExecutor : public PeerAccessInterface, public core::RefCounted {
 
   virtual PerStepCollectiveRemoteAccess* remote_access() { return nullptr; }
 
+  // `WaitForDependencies` and `Launched` are used for fine-grained control of
+  // execution order between collective instances.  These functions are intended
+  // to be called in `Run` function of collective implementations, and may be
+  // used to make part, or whole, of the collective execution ordered with
+  // respect to other collective instances.
+  //
+  // `WaitForDependencies` will block until it is safe to continue the callee's
+  // execution, where safety is defined as: ordered with respect to the
+  // collective instances defined in the callee's `wait_for` attribute.
+  virtual void WaitForDependencies(const CollectiveParams& col_params) {}
+  // `Launched` unblocks the dependent collective instances by recording that
+  // this callee device has completed the critical portion of the collective
+  // execution.
+  virtual void Launched(const CollectiveParams& col_params) {}
+
   // Used to designate an invalid group or instance key.
   static int64 kInvalidId;
 
@@ -347,7 +368,8 @@ class CollectiveImplementationInterface {
 
   // Initializes the portions of `col_params` specific to this
   // implementation.  Called exactly once for every Collective instance during
-  // the CollectiveParams resolution process when the graph is first executed.
+  // the CollectiveParams resolution process when the graph is first executed,
+  // at the end of `CompleteInstanceLocal()`.
   // NOTE(ayushd): This is effectively a static function because it modifies the
   // `col_params` passed in and should not manipulate any data members.  However
   // because it is virtual and needs to be implemented by every derived class we
@@ -360,6 +382,14 @@ class CollectiveImplementationInterface {
   // object.
   virtual Status InitializeCollectiveContext(CollectiveContext* col_ctx) = 0;
 
+  // Initializes instance params at the beginning of `CompleteInstanceLocal()`,
+  // unlike `InitializeCollectiveParams` which is called at the end.  This
+  // function is called before all devices in the instance are discovered, and
+  // may be used to broadcast data via the shared `InstanceRec` object in
+  // collective param resolution to all devices.
+  virtual Status InitializeInstanceBeforeGroupDiscovery(
+      CollectiveParams* col_params) = 0;
+
   // Processes and moves data according to the logic of this Collective
   // implementation.  Relies on appropriate initialization of op-specific
   // CollectiveParams in InitializeCollectiveParams(), as well as appropriate
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 96536e6945..c26c9af53f 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -60,6 +60,7 @@ load(
     "mkl_deps",
 )
 load("@local_config_cuda//cuda:build_defs.bzl", "if_cuda")
+load("//tensorflow:tensorflow.bzl", "if_nccl")
 
 config_setting(
     # Add "--define tensorflow_xsmm=1" to your build command to use libxsmm for
@@ -180,12 +181,36 @@ tf_cc_test(
 
 tf_kernel_library(
     name = "collective_ops",
+    srcs = if_nccl([
+        "collective_nccl_reducer.h",
+        "collective_nccl_reducer.cc",
+    ]),
     prefix = "collective_ops",
     deps = [
         "//tensorflow/core:collective_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:protos_all_cc",
+    ] + if_nccl([
+        "@local_config_nccl//:nccl",
+        "//tensorflow/core/nccl:nccl_lib",
+    ]),
+)
+
+tf_cuda_cc_test(
+    name = "collective_nccl_reducer_test",
+    size = "small",
+    srcs = ["collective_nccl_reducer_test.cc"],
+    tags = tf_cuda_tests_tags() + ["no_cuda_on_cpu_tap"],
+    deps = [
+        "//tensorflow/core:all_kernels",
+        "//tensorflow/core:core_cpu",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
     ],
 )
 
diff --git a/tensorflow/core/kernels/collective_nccl_reducer.cc b/tensorflow/core/kernels/collective_nccl_reducer.cc
new file mode 100644
index 0000000000..113f1487f0
--- /dev/null
+++ b/tensorflow/core/kernels/collective_nccl_reducer.cc
@@ -0,0 +1,204 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/core/kernels/collective_nccl_reducer.h"
+
+#ifdef GOOGLE_CUDA
+
+#include "tensorflow/core/common_runtime/collective_util.h"
+#include "tensorflow/core/nccl/nccl_manager.h"
+
+namespace tensorflow {
+namespace {
+string NcclCollectiveKey(const string& exec_key, int step_id) {
+  return strings::StrCat(exec_key, ":", step_id);
+}
+}  // namespace
+
+NcclReducer::NcclReducer() : col_ctx_(nullptr), col_params_(nullptr) {}
+
+Status NcclReducer::InitializeCollectiveParams(CollectiveParams* col_params) {
+  if (col_params->instance.type != REDUCTION_COLLECTIVE ||
+      col_params->instance.impl_details.collective_name != "NcclReduce") {
+    return errors::Internal("Unexpected collective type ",
+                            col_params->instance.type, " expected ",
+                            REDUCTION_COLLECTIVE, "; or collective name ",
+                            col_params->instance.impl_details.collective_name,
+                            " expected NcclReduce");
+  } else {
+    return Status::OK();
+  }
+}
+
+Status NcclReducer::InitializeCollectiveContext(CollectiveContext* col_ctx) {
+  col_ctx_ = col_ctx;
+  col_params_ = &col_ctx->col_params;
+  return collective_util::InitializeDeviceAndLocality(
+      col_ctx->dev_mgr, col_ctx->device_name, &col_ctx->device,
+      &col_ctx->device_locality);
+}
+
+Status NcclReducer::InitializeInstanceBeforeGroupDiscovery(
+    CollectiveParams* col_params) {
+  if (col_params->default_rank == 0 && col_params->group.num_tasks > 1) {
+    col_params->instance.communicator_key =
+        NcclManager::instance()->GenerateCommunicatorKey();
+  }
+  return Status::OK();
+}
+
+Status ReductionOp(const string& merge_op, ncclRedOp_t* reduction_op) {
+  if (merge_op == "Add") {
+    *reduction_op = ncclSum;
+    return Status::OK();
+  } else if (merge_op == "Mul") {
+    *reduction_op = ncclProd;
+    return Status::OK();
+  } else {
+    return errors::Internal("Expected merge_op to be either Add or Mul, found ",
+                            merge_op);
+  }
+}
+
+void NcclReducer::Run(StatusCallback done) {
+  ncclRedOp_t reduction_op;
+  Status s = ReductionOp(col_params_->merge_op->type_string(), &reduction_op);
+  if (!s.ok()) {
+    done(s);
+    return;
+  }
+
+  Tensor group_size;
+  Notification group_size_ready;
+  Status group_size_status;
+  if (col_params_->final_op) {
+    // Create an on-device scalar value from group_size_.
+    // TODO(ayushd, tucker): avoid this copy by either reusing across
+    // invocations or providing the scalar to the kernel in host memory.
+    Tensor group_size_val(col_ctx_->output->dtype(), TensorShape({}));
+    switch (col_ctx_->output->dtype()) {
+      case DT_FLOAT:
+        group_size_val.scalar<float>()() = col_params_->group.group_size;
+        break;
+      case DT_DOUBLE:
+        group_size_val.scalar<double>()() = col_params_->group.group_size;
+        break;
+      case DT_INT32:
+        group_size_val.scalar<int32>()() = col_params_->group.group_size;
+        break;
+      case DT_INT64:
+        group_size_val.scalar<int64>()() = col_params_->group.group_size;
+        break;
+      default:
+        done(errors::Internal("Unsupported type ", col_ctx_->output->dtype()));
+        return;
+    }
+    group_size = Tensor(
+        col_ctx_->device->GetAllocator(col_ctx_->op_ctx->input_alloc_attr(0)),
+        col_ctx_->output->dtype(), TensorShape({}));
+    DeviceContext* op_dev_ctx = col_ctx_->op_ctx->op_device_context();
+    // Enqueue copy on gpu stream.
+    op_dev_ctx->CopyCPUTensorToDevice(
+        &group_size_val, col_ctx_->device, &group_size,
+        [&group_size_ready, &group_size_status](const Status& s) {
+          group_size_status = s;
+          group_size_ready.Notify();
+        });
+  } else {
+    group_size_ready.Notify();
+  }
+
+  Notification nccl_done;
+  Status nccl_status;
+  auto* compute_stream = col_ctx_->op_ctx->op_device_context()->stream();
+  auto* gpu_info = col_ctx_->op_ctx->device()->tensorflow_gpu_device_info();
+  // `AddToAllReduce` performs consistency checks for the NCCL call and enqueues
+  // the `Participant` struct locally.  When all local participants with this
+  // `nccl_collective_key` have called `AddToAllReduce` and
+  // `SignalMultiNodeReady`, all devices at this worker are ready to process
+  // this NCCL op.
+  //
+  // The `NcclManager` uses a dedicated CUDA stream for NCCL kernels.  At this
+  // point, it synchronizes the NCCL stream with the compute stream, and then
+  // enqueues the NCCL kernel on the NCCL stream.
+  const int num_global_devices = col_params_->group.group_size;
+  const int num_local_devices = col_params_->instance.num_devices_per_task.at(
+      col_params_->instance.task_names[col_params_->default_rank]);
+  const string nccl_collective_key =
+      NcclCollectiveKey(col_ctx_->exec_key, col_ctx_->step_id);
+  auto done_callback = [&nccl_done, &nccl_status](const Status& s) {
+    nccl_status = s;
+    nccl_done.Notify();
+  };
+  auto participant = absl::make_unique<NcclManager::Participant>(
+      compute_stream->parent(), compute_stream, gpu_info->event_mgr,
+      gpu_info->gpu_id, col_ctx_->input, col_ctx_->output,
+      col_params_->default_rank, std::move(done_callback));
+  VLOG(1) << "NcclReducer calling NcclManager::AddToAllReduce num_tasks "
+          << col_params_->group.num_tasks << " current task "
+          << col_params_->instance.task_names[col_params_->default_rank]
+          << " num local devices " << num_local_devices
+          << " num global devices " << num_global_devices;
+  NcclManager::instance()->AddToAllReduce(
+      std::move(participant),
+      {nccl_collective_key, num_local_devices, num_global_devices,
+       col_params_->instance.communicator_key},
+      reduction_op);
+
+  // NOTE(ayushd): We need to synchronize NCCL launches across nodes to prevent
+  // deadlocks.  In the current implementation, we define a deterministic
+  // sequential launch order between potentially concurrent collective instances
+  // by introducing control information during static graph analysis in
+  // graph/collective_order.cc.  This can be either in the form of explicit
+  // control edges or via `wait_for` attribute on the collective op.
+  //
+  // The other end of the design spectrum would have a distinguished node
+  // dynamically signal the next collective to launch to all other participants.
+  // This has higher degree of runtime coordination, but it may be able to
+  // achieve better performance if the (arbitrary) static execution order
+  // assigned in the first approach turns out to not be good from a scheduling
+  // perspective.  e.g. consider a graph in which c1, c2, and c3 are three
+  // concurrent collective instances, and the static ordering assigns c1 -> c2
+  // -> c3.  In practice, it could turn out that c3 is always ready to execute
+  // before c1 or c2.
+  //
+  // `WaitForDependencies` may block if the collective instances on which this
+  // op depends have not yet launched.  When this function returns, this op is
+  // ready to go.
+  col_ctx_->col_exec->WaitForDependencies(*col_params_);
+  NcclManager::instance()->SignalMultiNodeReady(nccl_collective_key);
+  // When all devices at this worker have called `SignalMultiNodeReady`, the
+  // `NcclManager` will enqueue the NCCL kernel on the NCCL stream.  Thus the
+  // implementation of `Launched` keeps track of the number of devices that have
+  // launched.
+  col_ctx_->col_exec->Launched(*col_params_);
+
+  // Wait for nccl op and group_size copy to succeed, then do final_op.
+  group_size_ready.WaitForNotification();
+  nccl_done.WaitForNotification();
+  Status final_status =
+      group_size_status.ok() ? nccl_status : group_size_status;
+  if (final_status.ok() && col_params_->final_op) {
+    final_status = collective_util::ComputeBinOp(
+        col_ctx_->op_ctx, col_ctx_->op_params, col_ctx_->device,
+        col_params_->final_op.get(), col_ctx_->output, &group_size);
+  }
+  done(final_status);
+}
+
+REGISTER_COLLECTIVE(NcclReduce, NcclReducer);
+
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/collective_nccl_reducer.h b/tensorflow/core/kernels/collective_nccl_reducer.h
new file mode 100644
index 0000000000..dc70b280c5
--- /dev/null
+++ b/tensorflow/core/kernels/collective_nccl_reducer.h
@@ -0,0 +1,49 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_CORE_KERNELS_COLLECTIVE_NCCL_REDUCER_H_
+#define TENSORFLOW_CORE_KERNELS_COLLECTIVE_NCCL_REDUCER_H_
+
+#include "tensorflow/core/framework/collective.h"
+
+namespace tensorflow {
+#ifdef GOOGLE_CUDA
+
+class NcclReducer : public CollectiveImplementationInterface {
+ public:
+  NcclReducer();
+  ~NcclReducer() override = default;
+
+  // No-op for this collective implementation.
+  Status InitializeCollectiveParams(CollectiveParams* col_params) override;
+
+  // Initializes the device objects and device localities.
+  Status InitializeCollectiveContext(CollectiveContext* col_ctx) override;
+
+  // Initialize nccl communicator key.
+  Status InitializeInstanceBeforeGroupDiscovery(
+      CollectiveParams* col_params) override;
+
+  // Hands off all reduce to NcclManager.
+  void Run(StatusCallback done) override;
+
+ private:
+  CollectiveContext* col_ctx_;          // Not owned
+  const CollectiveParams* col_params_;  // Not owned
+};
+
+#endif  // GOOGLE_CUDA
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_KERNELS_COLLECTIVE_NCCL_REDUCER_H_
diff --git a/tensorflow/core/kernels/collective_nccl_reducer_test.cc b/tensorflow/core/kernels/collective_nccl_reducer_test.cc
new file mode 100644
index 0000000000..ff33c1322c
--- /dev/null
+++ b/tensorflow/core/kernels/collective_nccl_reducer_test.cc
@@ -0,0 +1,331 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/core/kernels/collective_nccl_reducer.h"
+
+#include <algorithm>
+#include "absl/memory/memory.h"
+#include "tensorflow/core/common_runtime/base_collective_executor.h"
+#include "tensorflow/core/common_runtime/device.h"
+#include "tensorflow/core/common_runtime/device_factory.h"
+#include "tensorflow/core/common_runtime/device_mgr.h"
+#include "tensorflow/core/common_runtime/device_resolver_local.h"
+#include "tensorflow/core/common_runtime/process_util.h"
+#include "tensorflow/core/common_runtime/test_collective_executor_mgr.h"
+#include "tensorflow/core/framework/collective.h"
+#include "tensorflow/core/framework/fake_input.h"
+#include "tensorflow/core/framework/node_def_builder.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/lib/core/notification.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/platform/test.h"
+#include "tensorflow/core/public/session_options.h"
+#include "tensorflow/core/public/version.h"
+
+#ifdef GOOGLE_CUDA
+
+namespace tensorflow {
+static constexpr int kStepId = 10;
+
+std::unique_ptr<OpKernel> GetKernel(const NodeDef& node, DeviceBase* device) {
+  Status status;
+  std::unique_ptr<OpKernel> k = CreateOpKernel(
+      DEVICE_GPU, device, device->GetAllocator(AllocatorAttributes()), node,
+      TF_GRAPH_DEF_VERSION, &status);
+  if (!status.ok()) LOG(FATAL) << status;
+  return k;
+}
+
+std::unique_ptr<OpKernel> GetAdd(DeviceBase* device) {
+  NodeDef node_def;
+  NodeDefBuilder builder("add_node", "Add");
+  TF_CHECK_OK(builder.Attr("T", DT_FLOAT)
+                  .Input(FakeInput(DT_FLOAT))
+                  .Input(FakeInput(DT_FLOAT))
+                  .Finalize(&node_def));
+  return GetKernel(node_def, device);
+}
+
+std::unique_ptr<OpKernel> GetDiv(DeviceBase* device) {
+  NodeDef node_def;
+  NodeDefBuilder builder("add_node", "Div");
+  TF_CHECK_OK(builder.Attr("T", DT_FLOAT)
+                  .Input(FakeInput(DT_FLOAT))
+                  .Input(FakeInput(DT_FLOAT))
+                  .Finalize(&node_def));
+  return GetKernel(node_def, device);
+}
+
+class NcclReducerTest : public ::testing::Test {
+ protected:
+  ~NcclReducerTest() override {
+    if (col_exec_) col_exec_->Unref();
+  }
+
+  void InitGPUDevices() {
+    std::vector<std::unique_ptr<Device>> all_devices;
+    SessionOptions session_options;
+    session_options.config.mutable_gpu_options()
+        ->set_per_process_gpu_memory_fraction(0.1);
+    session_options.env = Env::Default();
+    Status s = DeviceFactory::GetFactory(DEVICE_GPU)
+                   ->AddDevices(session_options, "", &all_devices);
+    TF_CHECK_OK(s);
+    for (std::unique_ptr<Device>& d : all_devices) {
+      if (d->device_type() == "GPU") {
+        gpus_.emplace_back(std::move(d));
+      }
+    }
+  }
+
+  void Init(int num_ranks) {
+    setenv("NCCL_DEBUG", "INFO", 1 /* replace */);
+    setenv("NCCL_LAUNCH_MODE", "PARALLEL", 1 /* replace */);
+    InitGPUDevices();
+    std::vector<std::unique_ptr<Device>> local_devices;
+    std::vector<string> device_names;
+    for (int rank = 0; rank < num_ranks; ++rank) {
+      if (rank < gpus_.size()) {
+        local_devices.emplace_back(std::move(gpus_[rank]));
+      }
+    }
+    int num_gpus = local_devices.size();
+    for (const auto& device : local_devices) {
+      device_names.push_back(device->name());
+      VLOG(2) << device->name();
+    }
+    if (!dev_mgr_) dev_mgr_.reset(new DeviceMgr(std::move(local_devices)));
+    col_exec_ = new BaseCollectiveExecutor(
+        &col_exec_mgr_, /*remote_access=*/nullptr, kStepId, dev_mgr_.get(),
+        /*gpu_ring_order=*/nullptr);
+
+    // Initialize collective params.
+    col_params_.name = "test_nccl_collective_op";
+    const int group_key = 5;
+    col_params_.group.group_key = group_key;
+    col_params_.group.device_type = DEVICE_GPU;
+    col_params_.group.group_size = num_ranks;
+    const int instance_key = 23;
+    col_params_.instance.instance_key = instance_key;
+    col_params_.instance.type = REDUCTION_COLLECTIVE;
+    col_params_.instance.data_type = DT_FLOAT;
+    col_params_.instance.impl_details.collective_name = "NcclReduce";
+    const string task_name = "/job:worker/replica:0/task:0";
+    col_params_.instance.num_devices_per_task[task_name] = num_ranks;
+    for (int rank = 0; rank < num_ranks; ++rank) {
+      col_params_.instance.device_names.push_back(
+          device_names[rank % num_gpus]);
+      col_params_.instance.task_names.push_back(task_name);
+    }
+    for (int rank = 0; rank < num_ranks; ++rank) {
+      instances_.push_back(absl::make_unique<DeviceInstance>(
+          rank, col_params_.instance.device_names[rank], this));
+    }
+  }
+
+  void Reduce() {
+    int done = 0;
+    mutex done_mu;
+    condition_variable done_cv;
+    for (const auto& instance : instances_) {
+      DeviceInstance* di = instance.get();
+      SchedClosure([di, &done, &done_mu, &done_cv] {
+        di->DoReduce();
+        mutex_lock l(done_mu);
+        ++done;
+        done_cv.notify_all();
+      });
+    }
+
+    mutex_lock l(done_mu);
+    while (done < instances_.size()) done_cv.wait(l);
+  }
+
+  void RunTest(int num_ranks, int tensor_length) {
+    Init(num_ranks);
+    std::vector<float> expected(tensor_length, 0.0);
+    for (int rank = 0; rank < num_ranks; ++rank) {
+      DeviceInstance* instance = instances_[rank].get();
+      instance->InitTensor(DT_FLOAT, TensorShape({tensor_length}),
+                           [&expected, rank](Tensor* t) {
+                             for (size_t i = 0; i < t->NumElements(); ++i) {
+                               float value = pow(10, rank) * i;
+                               t->flat<float>()(i) = value;
+                               expected[i] += value;
+                             }
+                           });
+    }
+    Reduce();
+    // Confirm that every rank computed the same correct value.
+    for (int i = 0; i < tensor_length; ++i) {
+      expected[i] /= num_ranks;
+    }
+    for (int rank = 0; rank < instances_.size(); ++rank) {
+      TF_ASSERT_OK(instances_[rank]->status_);
+      Tensor* dev_tensor = &instances_[rank]->tensor_;
+      Tensor actual(DT_FLOAT, TensorShape({tensor_length}));
+      Notification note;
+      Device* dev = instances_[rank]->device_;
+      auto* dev_info = dev->tensorflow_gpu_device_info();
+      dev_info->default_context->CopyDeviceTensorToCPU(
+          dev_tensor, /*tensor_name=*/"", dev, &actual,
+          [&note](const Status&) { note.Notify(); });
+      note.WaitForNotification();
+      for (int i = 0; i < tensor_length; ++i) {
+        EXPECT_FLOAT_EQ(expected[i], actual.template flat<float>()(i))
+            << "Mismatch at rank " << rank << " index " << i;
+      }
+    }
+  }
+
+  std::unique_ptr<OpKernel> GetCollectiveReduce(const CollectiveParams& params,
+                                                Tensor* input,
+                                                DeviceBase* device) {
+    mutex_lock l(mu_);
+    NodeDef node_def;
+    NodeDefBuilder builder(
+        strings::StrCat("collective_reduce_", reduce_counter_++),
+        "CollectiveReduce");
+    TF_CHECK_OK(
+        builder.Attr("T", params.instance.data_type)
+            .Attr("merge_op", "Add")
+            .Attr("final_op", "Div")
+            .Attr("group_size", params.group.group_size)
+            .Attr("group_key", params.group.group_key)
+            .Attr("instance_key", params.instance.instance_key)
+            .Attr("subdiv_offsets", params.instance.impl_details.subdiv_offsets)
+            .Input(FakeInput(params.instance.data_type))
+            .Finalize(&node_def));
+    return GetKernel(node_def, device);
+  }
+
+  class DeviceInstance {
+   public:
+    DeviceInstance(int rank, const string& device_name, NcclReducerTest* parent)
+        : parent_(parent), device_name_(device_name), rank_(rank) {
+      TF_CHECK_OK(parent_->dev_mgr_->LookupDevice(device_name_, &device_))
+          << "Could not find device " << device_name_ << " existing devices "
+          << parent_->dev_mgr_->DebugString();
+      col_params_.name = parent_->col_params_.name;
+      col_params_.default_rank = rank;
+      col_params_.group.group_key = parent_->col_params_.group.group_key;
+      col_params_.group.device_type = parent_->col_params_.group.device_type;
+      col_params_.group.group_size = parent_->col_params_.group.group_size;
+      col_params_.instance = parent->col_params_.instance;
+    }
+
+    void InitTensor(DataType dtype, const TensorShape& shape,
+                    const std::function<void(Tensor*)>& init_f) {
+      tensor_ =
+          Tensor(device_->GetAllocator(AllocatorAttributes()), dtype, shape);
+      Tensor cpu_tensor(dtype, shape);
+      init_f(&cpu_tensor);
+      VLOG(2) << "cpu_tensor " << cpu_tensor.DebugString();
+      auto* dev_info = device_->tensorflow_gpu_device_info();
+      Notification note;
+      dev_info->default_context->CopyCPUTensorToDevice(
+          &cpu_tensor, device_, &tensor_,
+          [&note](const Status&) { note.Notify(); });
+      note.WaitForNotification();
+    }
+
+    void DoReduce() {
+      col_params_.merge_op = GetAdd(device_);
+      col_params_.final_op = GetDiv(device_);
+
+      // Prepare an OpKernelContext.
+      OpKernelContext::Params op_params;
+      op_params.step_id = kStepId;
+      op_params.device = device_;
+      gtl::InlinedVector<TensorValue, 4> inputs;
+      inputs.push_back(TensorValue(&tensor_));
+      op_params.inputs = &inputs;
+      gtl::InlinedVector<AllocatorAttributes, 4> input_aa(
+          {AllocatorAttributes()});
+      op_params.input_alloc_attrs = &input_aa;
+      gtl::InlinedVector<DeviceContext*, 4> input_dc;
+      DeviceContext* dev_ctx = nullptr;
+      auto* dev_info = device_->tensorflow_gpu_device_info();
+      if (dev_info) {
+        dev_ctx = dev_info->default_context;
+        dev_ctx->Ref();
+      } else {
+        dev_ctx = new DeviceContext;
+      }
+      input_dc.push_back(dev_ctx);
+      op_params.input_device_contexts = &input_dc;
+      op_params.op_device_context = dev_ctx;
+      int forward_from = 0;
+      op_params.forward_from_array = &forward_from;
+      AllocatorAttributes generic_alloc_attr;
+      op_params.output_attr_array = &generic_alloc_attr;
+      std::unique_ptr<OpKernel> op =
+          parent_->GetCollectiveReduce(col_params_, &tensor_, device_);
+      op_params.op_kernel = op.get();
+      OpKernelContext ctx(&op_params, 1);
+
+      // We never actually execute the kernel, so we need to do the output
+      // allocation it would do, ourselves.
+      Tensor* output_tensor_ptr = nullptr;
+      TF_CHECK_OK(ctx.forward_input_or_allocate_output({0}, 0, tensor_.shape(),
+                                                       &output_tensor_ptr));
+      CHECK_EQ(output_tensor_ptr, ctx.mutable_output(0));
+
+      // Prepare a NcclReducer instance.
+      string exec_key =
+          strings::StrCat(col_params_.instance.instance_key, ":0:0");
+      NcclReducer reducer;
+      CollectiveContext col_ctx(parent_->col_exec_, parent_->dev_mgr_.get(),
+                                &ctx, &op_params, col_params_, exec_key,
+                                kStepId, &tensor_, &tensor_);
+      TF_CHECK_OK(reducer.InitializeCollectiveContext(&col_ctx));
+
+      // Run the all-reduce.
+      reducer.Run([this](Status s) { status_ = s; });
+      if (status_.ok()) {
+        CHECK(tensor_.CopyFrom(*ctx.mutable_output(0), tensor_.shape()));
+      }
+
+      dev_ctx->Unref();
+    }
+
+    NcclReducerTest* parent_;
+    string device_name_;
+    int rank_;
+    Tensor tensor_;
+    Device* device_;
+    CollectiveParams col_params_;
+    Status status_;
+  };
+
+  std::vector<std::unique_ptr<tensorflow::Device>> gpus_;
+  TestCollectiveExecutorMgr col_exec_mgr_;
+  CollectiveExecutor* col_exec_;
+  std::unique_ptr<DeviceMgr> dev_mgr_;
+  std::vector<std::unique_ptr<DeviceInstance>> instances_;
+  CollectiveParams col_params_;
+  mutex mu_;
+  int32 reduce_counter_ GUARDED_BY(mu_) = 0;
+};
+
+TEST_F(NcclReducerTest, Test2Dev16Len) { RunTest(2, 16); }
+TEST_F(NcclReducerTest, Test4Dev16Len) { RunTest(4, 16); }
+TEST_F(NcclReducerTest, Test8Dev16Len) { RunTest(8, 16); }
+TEST_F(NcclReducerTest, Test8Dev128Len) { RunTest(8, 128); }
+TEST_F(NcclReducerTest, Test8Dev1045991Len) { RunTest(8, 1048576); }
+
+}  // namespace tensorflow
+
+#endif
diff --git a/tensorflow/core/kernels/collective_ops.cc b/tensorflow/core/kernels/collective_ops.cc
index 82e2913b64..04e37e8edc 100644
--- a/tensorflow/core/kernels/collective_ops.cc
+++ b/tensorflow/core/kernels/collective_ops.cc
@@ -43,16 +43,17 @@ class CollectiveOpKernel : public AsyncOpKernel {
       // Call in a blockable thread because it's not guaranteed that
       // this call cannot block.
       c->env()->SchedClosure([this, c, done, col_exec]() {
-        col_exec->CompleteParamsAsync(c->device()->name(), &col_params_,
-                                      c->cancellation_manager(),
-                                      [this, c, done](const Status& s) {
-                                        if (s.ok()) {
-                                          ComputeAsync(c, done);
-                                        } else {
-                                          c->SetStatus(s);
-                                          done();
-                                        }
-                                      });
+        col_exec->CompleteParamsAsync(
+            c->device()->name(), &col_params_, c->cancellation_manager(),
+            [this, c, done](const Status& s) {
+              if (s.ok()) {
+                col_params_.instance.impl_details.dependencies = dependencies_;
+                ComputeAsync(c, done);
+              } else {
+                c->SetStatus(s);
+                done();
+              }
+            });
       });
       return false;
     }
@@ -60,6 +61,7 @@ class CollectiveOpKernel : public AsyncOpKernel {
   }
 
   CollectiveParams col_params_;
+  std::vector<int32> dependencies_;
 };
 
 class CollectiveReduceOpKernel : public CollectiveOpKernel {
@@ -87,6 +89,7 @@ class CollectiveReduceOpKernel : public CollectiveOpKernel {
                     "final_op must be one of {\"Id\", \"Div\"} but got ",
                     final_op_name));
     OP_REQUIRES_OK(c, c->GetAttr("T", &col_params_.instance.data_type));
+    OP_REQUIRES_OK(c, c->GetAttr("wait_for", &dependencies_));
 
     const NodeDef& real_node = c->def();
     col_params_.name = strings::StrCat(real_node.name(), ": Reduce(",
diff --git a/tensorflow/core/nccl/BUILD b/tensorflow/core/nccl/BUILD
index 4be33b2a0c..a19e1af888 100644
--- a/tensorflow/core/nccl/BUILD
+++ b/tensorflow/core/nccl/BUILD
@@ -20,9 +20,11 @@ cc_library(
     name = "nccl_lib",
     srcs = if_cuda([
         "nccl_manager.cc",
-        "nccl_manager.h",
         "nccl_rewrite.cc",
     ]),
+    hdrs = if_cuda([
+        "nccl_manager.h",
+    ]),
     copts = tf_copts(),
     deps = if_cuda([
         "@local_config_nccl//:nccl",
diff --git a/tensorflow/core/protobuf/worker.proto b/tensorflow/core/protobuf/worker.proto
index 74058c8465..4284dd119e 100644
--- a/tensorflow/core/protobuf/worker.proto
+++ b/tensorflow/core/protobuf/worker.proto
@@ -535,6 +535,7 @@ message CompleteInstanceRequest {
 message CompleteInstanceResponse {
   int32 instance_key = 1;
   int32 source_rank = 2;
+  bytes communicator_key = 3;
 }
 
 // Request for next agreed-upon step_id for the specified graph_keys.
-- 
GitLab


From 8eafc6c7941e8488db6452af68851a6f531528d9 Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Fri, 18 Jan 2019 08:43:06 -0800
Subject: [PATCH 0970/1095] Prevent out-of-bounds memory access in decoding gif
 images.

PiperOrigin-RevId: 229934641
---
 tensorflow/core/lib/gif/gif_io.cc | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/tensorflow/core/lib/gif/gif_io.cc b/tensorflow/core/lib/gif/gif_io.cc
index 24aca854eb..ce842e9723 100644
--- a/tensorflow/core/lib/gif/gif_io.cc
+++ b/tensorflow/core/lib/gif/gif_io.cc
@@ -147,6 +147,14 @@ uint8* Decode(const void* srcdata, int datasize,
         GifByteType color_index =
             this_image->RasterBits[(i - img_desc->Top) * (img_desc->Width) +
                                    (j - img_desc->Left)];
+
+        if (color_index >= color_map->ColorCount) {
+          *error_string = strings::StrCat("found color index ", color_index,
+                                          " outside of color map range ",
+                                          color_map->ColorCount);
+          return nullptr;
+        }
+
         const GifColorType& gif_color = color_map->Colors[color_index];
         p_dst[j * channel + 0] = gif_color.Red;
         p_dst[j * channel + 1] = gif_color.Green;
-- 
GitLab


From 06444e2c956f1aa6265bf467ebb8b3072a280b7f Mon Sep 17 00:00:00 2001
From: Fei Hu <hufei68@gmail.com>
Date: Fri, 18 Jan 2019 09:36:09 -0800
Subject: [PATCH 0971/1095] move kDelimiter to an anonymous namespace

---
 tensorflow/core/kernels/data/dataset_utils.cc | 17 +++++++++++++++++
 tensorflow/core/kernels/data/dataset_utils.h  | 12 +-----------
 2 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/tensorflow/core/kernels/data/dataset_utils.cc b/tensorflow/core/kernels/data/dataset_utils.cc
index 8f3b1f7066..dd47ed8e8d 100644
--- a/tensorflow/core/kernels/data/dataset_utils.cc
+++ b/tensorflow/core/kernels/data/dataset_utils.cc
@@ -141,6 +141,23 @@ Status VerifyShapesCompatible(const std::vector<PartialTensorShape>& expected,
   return Status::OK();
 }
 
+namespace {
+
+constexpr char kDelimiter[] = "@@";
+
+}  // namespace
+
+VariantTensorDataReader::VariantTensorDataReader(
+    const tensorflow::VariantTensorData* data)
+    : data_(data) {
+  string metadata;
+  data_->get_metadata(&metadata);
+  auto keys = str_util::Split(metadata, kDelimiter, str_util::SkipEmpty());
+  for (size_t i = 0; i < keys.size(); ++i) {
+    map_[keys[i]] = i;
+  }
+}
+
 Status VariantTensorDataReader::status() const { return status_; }
 
 Status VariantTensorDataReader::ReadScalar(StringPiece key, int64* val) {
diff --git a/tensorflow/core/kernels/data/dataset_utils.h b/tensorflow/core/kernels/data/dataset_utils.h
index b960671594..eae2bc4242 100644
--- a/tensorflow/core/kernels/data/dataset_utils.h
+++ b/tensorflow/core/kernels/data/dataset_utils.h
@@ -57,20 +57,10 @@ Status VerifyTypesMatch(const DataTypeVector& expected,
 Status VerifyShapesCompatible(const std::vector<PartialTensorShape>& expected,
                               const std::vector<PartialTensorShape>& received);
 
-constexpr char kDelimiter[] = "@@";
-
 // Helper class for reading data from a VariantTensorData object.
 class VariantTensorDataReader : public IteratorStateReader {
  public:
-  explicit VariantTensorDataReader(const VariantTensorData* data)
-      : data_(data) {
-    string metadata;
-    data_->get_metadata(&metadata);
-    auto keys = str_util::Split(metadata, kDelimiter, str_util::SkipEmpty());
-    for (size_t i = 0; i < keys.size(); ++i) {
-      map_[keys[i]] = i;
-    }
-  }
+  explicit VariantTensorDataReader(const VariantTensorData* data);
 
   // Returns OK iff the initialization was successful.
   Status status() const;
-- 
GitLab


From 2e2a3ca8e827138a02da1cf129ab28057b9f9ddd Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 18 Jan 2019 09:32:48 -0800
Subject: [PATCH 0972/1095] Update Eigen to
 https://bitbucket.org/eigen/eigen/commits/bcc817c0ba985fc27b0c8b9998351cf0f789687b

PiperOrigin-RevId: 229941993
---
 tensorflow/core/kernels/cwise_ops.h | 34 +++--------------------------
 tensorflow/workspace.bzl            |  8 +++----
 2 files changed, 7 insertions(+), 35 deletions(-)

diff --git a/tensorflow/core/kernels/cwise_ops.h b/tensorflow/core/kernels/cwise_ops.h
index a22d76717a..15bd533be0 100644
--- a/tensorflow/core/kernels/cwise_ops.h
+++ b/tensorflow/core/kernels/cwise_ops.h
@@ -51,15 +51,12 @@ struct scalar_arg_op<std::complex<double>> {
 };
 #endif
 
+#if EIGEN_HAS_CXX11_MATH == 0
 template <typename T>
 struct scalar_asinh_op {
   EIGEN_EMPTY_STRUCT_CTOR(scalar_asinh_op)
   EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T operator()(const T& a) const {
-#if EIGEN_HAS_CXX11_MATH
-    return numext::asinh(a);
-#else
     return std::asinh(a);
-#endif  // EIGEN_HAS_CXX11_MATH
   }
 };
 template <typename T>
@@ -71,11 +68,7 @@ template <typename T>
 struct scalar_acosh_op {
   EIGEN_EMPTY_STRUCT_CTOR(scalar_acosh_op)
   EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T operator()(const T& a) const {
-#if EIGEN_HAS_CXX11_MATH
-    return numext::acosh(a);
-#else
     return std::acosh(a);
-#endif  // EIGEN_HAS_CXX11_MATH
   }
 };
 template <typename T>
@@ -87,35 +80,14 @@ template <typename T>
 struct scalar_atanh_op {
   EIGEN_EMPTY_STRUCT_CTOR(scalar_atanh_op)
   EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T operator()(const T& a) const {
-#if EIGEN_HAS_CXX11_MATH
-    return numext::atanh(a);
-#else
     return std::atanh(a);
-#endif  // EIGEN_HAS_CXX11_MATH
   }
 };
 template <typename T>
 struct functor_traits<scalar_atanh_op<T>> {
   enum { Cost = 5 * NumTraits<T>::MulCost, PacketAccess = false };
 };
-
-// TODO(rmlarsen): This is a workaround for upstream change
-// https://bitbucket.org/eigen/eigen/commits/f339468d04d0f87caeb6cab9aef568627e9f6ea9
-// that renamed scalar_binary_pow_op to scalar_pow_op and deleted the unary
-// version of the latter. Remove once we upgrade to Eigen 3.3.
-template <typename Scalar, typename Exponent>
-struct scalar_binary_pow_op_google {
-  EIGEN_EMPTY_STRUCT_CTOR(scalar_binary_pow_op_google)
-  EIGEN_DEVICE_FUNC inline Scalar operator()(const Scalar& a,
-                                             const Exponent& b) const {
-    return numext::pow(a, b);
-  }
-};
-
-template <typename Scalar, typename Exponent>
-struct functor_traits<scalar_binary_pow_op_google<Scalar, Exponent>> {
-  enum { Cost = 5 * NumTraits<Scalar>::MulCost, PacketAccess = false };
-};
+#endif
 
 template <typename Scalar, typename Exponent>
 struct safe_scalar_binary_pow_op {
@@ -865,7 +837,7 @@ template <typename T>
 struct floor_div_real : base<T, Eigen::internal::google_floor_div_real<T>> {};
 
 template <typename T>
-struct pow : base<T, Eigen::internal::scalar_binary_pow_op_google<T, T>> {};
+struct pow : base<T, Eigen::internal::scalar_pow_op<T, T>> {};
 
 template <typename T>
 struct safe_pow : base<T, Eigen::internal::safe_scalar_binary_pow_op<T, T>> {
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index ad73b19ff1..71ecd4d902 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -140,11 +140,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "eigen_archive",
         build_file = clean_dep("//third_party:eigen.BUILD"),
-        sha256 = "df54d805573871d03aaef6d27d45439c18ff6e8db215a3b922b2daefd6914147",
-        strip_prefix = "eigen-eigen-ad3bcd81cc49",
+        sha256 = "48678550a32665331d729be87076e576f2502fff325f5b6c2c78ebf7b1b22c7b",
+        strip_prefix = "eigen-eigen-bcc817c0ba98",
         urls = [
-            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/ad3bcd81cc49.tar.gz",
-            "https://bitbucket.org/eigen/eigen/get/ad3bcd81cc49.tar.gz",
+            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/bcc817c0ba98.tar.gz",
+            "https://bitbucket.org/eigen/eigen/get/bcc817c0ba98.tar.gz",
         ],
     )
 
-- 
GitLab


From cf677e677b11cb503832be9a9cc2e7f6a5076d76 Mon Sep 17 00:00:00 2001
From: Francois Chollet <fchollet@google.com>
Date: Fri, 18 Jan 2019 09:47:51 -0800
Subject: [PATCH 0973/1095] Fix an issue with Sequential manual building when
 in deferred mode.

PiperOrigin-RevId: 229944372
---
 tensorflow/python/keras/engine/sequential_test.py | 14 ++++++++++++++
 tensorflow/python/keras/engine/training.py        |  2 +-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/keras/engine/sequential_test.py b/tensorflow/python/keras/engine/sequential_test.py
index 8e5d6fe93a..a65c200ce6 100644
--- a/tensorflow/python/keras/engine/sequential_test.py
+++ b/tensorflow/python/keras/engine/sequential_test.py
@@ -307,6 +307,20 @@ class TestSequential(keras_parameterized.TestCase):
     self.assertTrue(model.built)
     self.assertEqual(len(model.weights), 8)
 
+  @keras_parameterized.run_all_keras_modes(always_skip_v1=True)
+  def test_sequential_deferred_manual_build(self):
+    model = testing_utils.get_small_sequential_mlp(4, 5)
+    self.assertFalse(model.built)
+    model(array_ops.zeros([1, 2]))
+    self.assertTrue(model.built)
+    self.assertEqual(len(model.outputs), 0)
+    model.compile('rmsprop',
+                  loss='mse',
+                  run_eagerly=testing_utils.should_run_eagerly())
+    self.assertEqual(len(model.outputs), 0)
+    model.train_on_batch(np.zeros((1, 2)), np.zeros((1, 5)))
+    self.assertEqual(len(model.outputs), 1)
+
   @keras_parameterized.run_all_keras_modes
   def test_sequential_nesting(self):
     model = testing_utils.get_small_sequential_mlp(4, 3)
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index b80b906f67..53caac25c9 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -279,7 +279,7 @@ class Model(Network):
     self._distributed_model = None
     # Initialize model metric attributes.
     self._init_metric_attributes()
-    if not self.built:
+    if not self.built or not self.inputs or not self.outputs:
       # Model is not compilable because it does not know its number of inputs
       # and outputs, nor their shapes and names. We will compile after the first
       # time the model gets called on training data.
-- 
GitLab


From 26f4194fe4c26ea723e7828091801846fad44487 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 18 Jan 2019 09:51:54 -0800
Subject: [PATCH 0974/1095] Add some utils for parsing Tensors as simple
 functions at runtime.

Also add some utils for calculating row endpoints of sparse matrices in coordinate list form.

PiperOrigin-RevId: 229945000
---
 tensorflow/core/kernels/BUILD                 |  53 +++
 tensorflow/core/kernels/sparse_utils.cc       | 161 +++++++++
 tensorflow/core/kernels/sparse_utils.h        |  71 ++++
 tensorflow/core/kernels/sparse_utils_test.cc  | 263 ++++++++++++++
 tensorflow/core/kernels/tensor_flag_utils.cc  | 187 ++++++++++
 tensorflow/core/kernels/tensor_flag_utils.h   |  78 +++++
 .../core/kernels/tensor_flag_utils_test.cc    | 322 ++++++++++++++++++
 7 files changed, 1135 insertions(+)
 create mode 100644 tensorflow/core/kernels/sparse_utils.cc
 create mode 100644 tensorflow/core/kernels/sparse_utils.h
 create mode 100644 tensorflow/core/kernels/sparse_utils_test.cc
 create mode 100644 tensorflow/core/kernels/tensor_flag_utils.cc
 create mode 100644 tensorflow/core/kernels/tensor_flag_utils.h
 create mode 100644 tensorflow/core/kernels/tensor_flag_utils_test.cc

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index c26c9af53f..9fcd4106fa 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -367,6 +367,59 @@ tf_kernel_library(
     ]),
 )
 
+cc_library(
+    name = "sparse_utils",
+    srcs = [
+        "sparse_utils.cc",
+    ],
+    hdrs = ["sparse_utils.h"],
+    deps = [
+        "//tensorflow/core:framework",
+        "//tensorflow/core:framework_lite",
+        "//tensorflow/core:lib_internal",
+    ],
+)
+
+tf_cc_test(
+    name = "sparse_utils_test",
+    srcs = ["sparse_utils_test.cc"],
+    deps = [
+        ":sparse_utils",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:framework_lite",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "@com_google_absl//absl/base:core_headers",
+    ],
+)
+
+cc_library(
+    name = "tensor_flag_utils",
+    srcs = [
+        "tensor_flag_utils.cc",
+    ],
+    hdrs = ["tensor_flag_utils.h"],
+    deps = [
+        "//tensorflow/core:framework",
+        "//tensorflow/core:framework_lite",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+tf_cc_test(
+    name = "tensor_flag_utils_test",
+    srcs = ["tensor_flag_utils_test.cc"],
+    deps = [
+        ":tensor_flag_utils",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:framework_lite",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "@com_google_absl//absl/base:core_headers",
+    ],
+)
+
 tf_cuda_library(
     name = "ops_testutil",
     testonly = 1,
diff --git a/tensorflow/core/kernels/sparse_utils.cc b/tensorflow/core/kernels/sparse_utils.cc
new file mode 100644
index 0000000000..198862940d
--- /dev/null
+++ b/tensorflow/core/kernels/sparse_utils.cc
@@ -0,0 +1,161 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/kernels/sparse_utils.h"
+
+#include <cstddef>
+
+#include "tensorflow/core/framework/tensor_shape.h"
+
+namespace tensorflow {
+namespace sparse_utils {
+
+template <typename Tindices>
+Tindices FindNextDenseRowStartIndex(
+    const Tindices sparse_index_begin,
+    const typename TTypes<Tindices>::ConstMatrix& indices_mat) {
+  // Search in the index range [begin, end) of indices_mat.
+  Tindices begin = sparse_index_begin;
+  Tindices end = indices_mat.dimension(0);
+  const Tindices orig_sparse_index_end = end;
+
+  // The first dense row we search.
+  const Tindices orig_dense_index_begin = indices_mat(begin, 0);
+  // Early exit if no next dense row index.
+  if (orig_dense_index_begin == static_cast<int64>(indices_mat(end - 1, 0))) {
+    return orig_sparse_index_end;
+  }
+
+  Tindices increment = 1;
+  while (begin + increment < end &&
+         indices_mat(begin + increment, 0) == orig_dense_index_begin) {
+    increment *= 2;
+  }
+  // Narrow the search space as an optimization.
+  if (begin + increment < end) {
+    end = begin + increment;
+  }
+  begin += increment / 2;
+
+  // Perform a binary search on the interval [begin, end) for
+  // dense_row_index_to_find.
+  const Tindices dense_row_index_to_find = orig_dense_index_begin;
+  while (begin < end) {
+    const Tindices m = begin + (end - begin) / 2;
+    const Tindices m_dense_row_index = static_cast<Tindices>(indices_mat(m, 0));
+    if (m_dense_row_index == dense_row_index_to_find &&
+        (m + 1 == orig_sparse_index_end ||
+         static_cast<Tindices>(indices_mat(m + 1, 0)) !=
+             dense_row_index_to_find)) {
+      return m + 1;
+    } else if (m_dense_row_index <= dense_row_index_to_find) {
+      begin = m + 1;
+    } else {
+      end = m;
+    }
+  }
+
+  // No next dense row index.
+  return orig_sparse_index_end;
+}
+
+template <typename Tindices>
+std::vector<Tindices> GetStartIndicesOfEachDenseRow(
+    const typename TTypes<Tindices>::ConstMatrix& indices_mat,
+    bool* contains_empty_rows) {
+  int64 start_sparse_index_of_cur_dense_row = 0;
+  std::vector<Tindices> segment_indices;
+  const Tindices num_entries_in_sparse_tensor = indices_mat.dimension(0);
+  const Tindices num_dense_rows_in_sparse_tensor =
+      1 + indices_mat(num_entries_in_sparse_tensor - 1, 0) - indices_mat(0, 0);
+  // Reserve an extra slot for the 0 we store in the first entry by convention.
+  segment_indices.reserve(1 + num_dense_rows_in_sparse_tensor);
+  segment_indices.push_back(0);
+  *contains_empty_rows = false;
+  while (true) {
+    const Tindices start_sparse_index_of_next_dense_row =
+        FindNextDenseRowStartIndex<Tindices>(
+            start_sparse_index_of_cur_dense_row, indices_mat);
+    if (start_sparse_index_of_next_dense_row == num_entries_in_sparse_tensor) {
+      segment_indices.push_back(start_sparse_index_of_next_dense_row);
+      break;
+    }
+    // Encode the length of the current dense row as well as the lengths of all
+    // the empty rows until the next dense row,
+    for (Tindices i = 0;
+         i < indices_mat(start_sparse_index_of_next_dense_row, 0) -
+                 indices_mat(start_sparse_index_of_cur_dense_row, 0);
+         ++i) {
+      segment_indices.push_back(start_sparse_index_of_next_dense_row);
+    }
+    // If there is more than one row between the current and next non-empty
+    // rows then those rows are empty.
+    *contains_empty_rows |=
+        indices_mat(start_sparse_index_of_next_dense_row, 0) -
+            indices_mat(start_sparse_index_of_cur_dense_row, 0) >
+        1;
+    start_sparse_index_of_cur_dense_row = start_sparse_index_of_next_dense_row;
+  }
+  return segment_indices;
+}
+
+template <typename Tindices>
+std::vector<Tindices> ParseRowStartIndices(
+    const tensorflow::Tensor& tensor,
+    const Tindices num_nonzero_entries_in_sparse_mat) {
+  std::vector<Tindices> out;
+  auto vec = tensor.vec<Tindices>();
+  out.reserve(vec.size() + 1);
+  for (size_t i = 0; i < vec.dimension(0); ++i) {
+    out.push_back(vec(i));
+  }
+  out.push_back(num_nonzero_entries_in_sparse_mat);
+  return out;
+}
+
+template <typename Tindices>
+bool ContainsEmptyRows(const std::vector<Tindices>& row_start_indices) {
+  // Skip checking the lengths of the first and last dense rows since those are
+  // always non-empty.
+  for (size_t i = 2; i < row_start_indices.size() - 1; ++i) {
+    if (row_start_indices.at(i) - row_start_indices.at(i - 1) == 0) {
+      return true;
+    }
+  }
+  return false;
+}
+
+#define REGISTER_SPARSE_UTIL_FUNCTIONS(TypeIndex)                           \
+  template TypeIndex FindNextDenseRowStartIndex<TypeIndex>(                 \
+      const TypeIndex sparse_index_begin,                                   \
+      const TTypes<TypeIndex>::ConstMatrix& indices_mat);                   \
+  template std::vector<TypeIndex> GetStartIndicesOfEachDenseRow<TypeIndex>( \
+      const TTypes<TypeIndex>::ConstMatrix& indices_mat,                    \
+      bool* contains_empty_rows);                                           \
+  template bool ContainsEmptyRows<TypeIndex>(                               \
+      const std::vector<TypeIndex>& row_start_indices);                     \
+  template std::vector<TypeIndex> ParseRowStartIndices<TypeIndex>(          \
+      const tensorflow::Tensor& tensor,                                     \
+      const TypeIndex num_nonzero_entries_in_sparse_mat);
+
+REGISTER_SPARSE_UTIL_FUNCTIONS(int32);
+REGISTER_SPARSE_UTIL_FUNCTIONS(int64);
+REGISTER_SPARSE_UTIL_FUNCTIONS(uint8);
+REGISTER_SPARSE_UTIL_FUNCTIONS(uint16);
+REGISTER_SPARSE_UTIL_FUNCTIONS(uint32);
+REGISTER_SPARSE_UTIL_FUNCTIONS(uint64);
+
+}  // namespace sparse_utils
+}  // namespace tensorflow
diff --git a/tensorflow/core/kernels/sparse_utils.h b/tensorflow/core/kernels/sparse_utils.h
new file mode 100644
index 0000000000..9e3c41a496
--- /dev/null
+++ b/tensorflow/core/kernels/sparse_utils.h
@@ -0,0 +1,71 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// Helpers for writing OpKernels for sparse tensors.
+#ifndef TENSORFLOW_CORE_KERNELS_SPARSE_UTILS_H_
+#define TENSORFLOW_CORE_KERNELS_SPARSE_UTILS_H_
+
+#include <vector>
+
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/tensor_types.h"
+#include "tensorflow/core/platform/types.h"
+
+namespace tensorflow {
+namespace sparse_utils {
+
+// Find the index i of the first element for which
+// indices_mat(sparse_index_begin, 0) < indices_mat(i, 0).
+// The search is conducted in the open interval
+// [sparse_index_begin, indices_mat.dimension(0)) and when no such i is found,
+// indices_mat.dimension(0) is returned.
+// indices_mat(k, 0) should be non-decreasing over the interval
+// [begin, indices_mat.dimension(0)).
+// Requires 0 <= sparse_index_begin < indices_mat.dimension(0).
+template <typename Tindices>
+Tindices FindNextDenseRowStartIndex(
+    const Tindices sparse_index_begin,
+    const typename TTypes<Tindices>::ConstMatrix& indices_mat);
+
+// Returns the vector v of indices in indices_mat at which new dense matrix
+// rows begin.
+// v.front() = 0, v.back() = indices_mat.dimension(0), and for i > 0,
+// v[i] - v[i-1] is the length of the ith dense row in indices_mat.
+// *contains_empty_rows = true if and only if indices_mat contains empty rows
+// (rows without values) between its first and last row.
+template <typename Tindices>
+std::vector<Tindices> GetStartIndicesOfEachDenseRow(
+    const typename TTypes<Tindices>::ConstMatrix& indices_mat,
+    bool* contains_empty_rows);
+
+// Converts tensor.vec<Tindices> to an std::vector<Tindices> object, appends
+// the value num_nonzero_entries_in_sparse_mat, and returns the result.
+template <typename Tindices>
+std::vector<Tindices> ParseRowStartIndices(
+    const tensorflow::Tensor& tensor,
+    const Tindices num_nonzero_entries_in_sparse_mat);
+
+// Returns true if and only if the sparse matrix indices_mat whose row start
+// indices are represented by row_start_indices has empty dense rows
+// (between its first and last dense rows).
+// This function satisfies the identity row_start_indices ==
+// GetStartIndicesOfEachDenseRow(indices_mat, &return_value).
+template <typename Tindices>
+bool ContainsEmptyRows(const std::vector<Tindices>& row_start_indices);
+
+}  // namespace sparse_utils
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_KERNELS_SPARSE_UTILS_H_
diff --git a/tensorflow/core/kernels/sparse_utils_test.cc b/tensorflow/core/kernels/sparse_utils_test.cc
new file mode 100644
index 0000000000..5d0adff886
--- /dev/null
+++ b/tensorflow/core/kernels/sparse_utils_test.cc
@@ -0,0 +1,263 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/kernels/sparse_utils.h"
+
+#include <vector>
+
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/tensor_types.h"
+#include "tensorflow/core/framework/types.pb.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace {
+
+using tensorflow::DataType;
+using tensorflow::int32;
+using tensorflow::int64;
+using tensorflow::Tensor;
+using tensorflow::TTypes;
+using tensorflow::uint16;
+using tensorflow::uint32;
+using tensorflow::uint64;
+using tensorflow::sparse_utils::ContainsEmptyRows;
+using tensorflow::sparse_utils::FindNextDenseRowStartIndex;
+using tensorflow::sparse_utils::GetStartIndicesOfEachDenseRow;
+using tensorflow::sparse_utils::ParseRowStartIndices;
+
+TEST(SparseUtilsTest, GetStartIndicesOfEachDenseRow) {
+  {
+    int32 data[] = {0, 0, 1, 0, 4, 0, 6, 0, 7, 0, 8, 0, 10, 0, 12, 0};
+    TTypes<int32>::ConstMatrix indices_mat(data, 8, 2);
+    // indices_list = {0, 1, 4, 6, 7, 8, 10, 12};
+    bool contains_empty_rows;
+    EXPECT_TRUE(GetStartIndicesOfEachDenseRow<int32>(indices_mat,
+                                                     &contains_empty_rows) ==
+                std::vector<int32>({0, 1, 2, 2, 2, 3, 3, 4, 5, 6, 6, 7, 7, 8}));
+    EXPECT_TRUE(contains_empty_rows);
+  }
+  {
+    int32 data[] = {0, 0, 1, 0, 1, 0, 4, 0, 4, 0, 4, 0,  6, 0,  7,
+                    0, 7, 0, 7, 0, 7, 0, 8, 0, 8, 0, 10, 0, 12, 0};
+    TTypes<int32>::ConstMatrix indices_mat(data, 15, 2);
+    // indices_list = {0, 1, 1, 4, 4, 4,  6, 7, 7, 7, 7, 8, 8, 10, 12};
+    bool contains_empty_rows;
+    EXPECT_TRUE(
+        GetStartIndicesOfEachDenseRow<int32>(indices_mat,
+                                             &contains_empty_rows) ==
+        std::vector<int32>({0, 1, 3, 3, 3, 6, 6, 7, 11, 13, 13, 14, 14, 15}));
+    EXPECT_TRUE(contains_empty_rows);
+  }
+  {
+    int64 data[] = {3, 0};
+    TTypes<int64>::ConstMatrix indices_mat(data, 1, 2);
+    bool contains_empty_rows;
+    EXPECT_TRUE(GetStartIndicesOfEachDenseRow<int64>(indices_mat,
+                                                     &contains_empty_rows) ==
+                std::vector<int64>({0, 1}));
+    EXPECT_FALSE(contains_empty_rows);
+  }
+  {
+    uint32 data[] = {3, 0, 3, 0};
+    TTypes<uint32>::ConstMatrix indices_mat(data, 2, 2);
+    bool contains_empty_rows;
+    EXPECT_TRUE(GetStartIndicesOfEachDenseRow<uint32>(indices_mat,
+                                                      &contains_empty_rows) ==
+                std::vector<uint32>({0, 2}));
+    EXPECT_FALSE(contains_empty_rows);
+  }
+  {
+    uint16 data[] = {0, 0, 0, 0, 0, 0, 1, 0};
+    TTypes<uint16>::ConstMatrix indices_mat(data, 4, 2);
+    // indices_list = {0, 0, 0, 1};
+    bool contains_empty_rows;
+    EXPECT_TRUE(GetStartIndicesOfEachDenseRow<uint16>(indices_mat,
+                                                      &contains_empty_rows) ==
+                std::vector<uint16>({0, 3, 4}));
+    EXPECT_FALSE(contains_empty_rows);
+  }
+  {
+    uint64 data[] = {0, 0, 0, 0, 0, 0, 3, 0};
+    TTypes<uint64>::ConstMatrix indices_mat(data, 4, 2);
+    bool contains_empty_rows;
+    // indices_list = {0, 0, 0, 3};
+    EXPECT_TRUE(GetStartIndicesOfEachDenseRow<uint64>(indices_mat,
+                                                      &contains_empty_rows) ==
+                std::vector<uint64>({0, 3, 3, 3, 4}));
+    EXPECT_TRUE(contains_empty_rows);
+  }
+}
+
+TEST(SparseUtilsTest, ParseRowStartIndices) {
+  {
+    Tensor t(DataType::DT_INT32, {1});
+    int indx = 0;
+    for (const int32 v : {0}) {
+      t.flat<int32>()(indx++) = v;
+    }
+    EXPECT_TRUE(ParseRowStartIndices<int32>(t, 1) ==
+                std::vector<int32>({0, 1}));
+  }
+  {
+    Tensor t(DataType::DT_INT64, {1});
+    int indx = 0;
+    for (const int64 v : {0}) {
+      t.flat<int64>()(indx++) = v;
+    }
+    EXPECT_TRUE(ParseRowStartIndices<int64>(t, 2) ==
+                std::vector<int64>({0, 2}));
+  }
+  {
+    Tensor t(DataType::DT_UINT64, {2});
+    int indx = 0;
+    for (const uint64 v : {0, 3}) {
+      t.flat<uint64>()(indx++) = v;
+    }
+    EXPECT_TRUE(ParseRowStartIndices<uint64>(t, 4) ==
+                std::vector<uint64>({0, 3, 4}));
+  }
+  {
+    Tensor t(DataType::DT_UINT16, {2});
+    int indx = 0;
+    for (const uint16 v : {0, 3}) {
+      t.flat<uint16>()(indx++) = v;
+    }
+    EXPECT_TRUE(ParseRowStartIndices<uint16>(t, 4) ==
+                std::vector<uint16>({0, 3, 4}));
+  }
+}
+
+TEST(SparseUtilsTest, ContainsEmptyRows) {
+  {
+    int32 data[] = {0, 0, 1, 0, 4, 0, 6, 0, 7, 0, 8, 0, 10, 0, 12, 0};
+    TTypes<int32>::ConstMatrix indices_mat(data, 8, 2);
+    bool contains_empty_rows;
+    const auto segment_indices =
+        GetStartIndicesOfEachDenseRow<int32>(indices_mat, &contains_empty_rows);
+    // indices_list = {0, 1, 4, 6, 7, 8, 10, 12};
+    EXPECT_TRUE(ContainsEmptyRows(segment_indices));
+  }
+  {
+    int64 data[] = {0, 0, 1, 0, 4, 0, 6, 0, 7, 0, 8, 0, 10, 0, 12, 0};
+    TTypes<int64>::ConstMatrix indices_mat(data, 8, 2);
+    bool contains_empty_rows;
+    const auto segment_indices =
+        GetStartIndicesOfEachDenseRow<int64>(indices_mat, &contains_empty_rows);
+    // indices_list = {0, 1, 4, 6, 7, 8, 10, 12};
+    EXPECT_TRUE(ContainsEmptyRows(segment_indices));
+  }
+  {
+    int32 data[] = {1, 0, 1, 1, 2, 0, 2, 1, 2, 2, 3, 4};
+    TTypes<int32>::ConstMatrix indices_mat(data, 6, 2);
+    bool contains_empty_rows;
+    const auto segment_indices =
+        GetStartIndicesOfEachDenseRow<int32>(indices_mat, &contains_empty_rows);
+    // indices_list = {1, 1, 2, 2, 2, 3};
+    EXPECT_FALSE(ContainsEmptyRows(segment_indices));
+  }
+  {
+    uint16 data[] = {1, 0, 1, 1, 2, 0, 2, 1, 2, 2, 3, 4};
+    TTypes<uint16>::ConstMatrix indices_mat(data, 6, 2);
+    bool contains_empty_rows;
+    const auto segment_indices = GetStartIndicesOfEachDenseRow<uint16>(
+        indices_mat, &contains_empty_rows);
+    // indices_list = {1, 1, 2, 2, 2, 3};
+    EXPECT_FALSE(ContainsEmptyRows(segment_indices));
+  }
+  {
+    int32 data[] = {0, 0, 1, 0, 1, 1, 2, 0, 2, 1, 2, 2, 3, 4};
+    TTypes<int32>::ConstMatrix indices_mat(data, 7, 2);
+    bool contains_empty_rows;
+    const auto segment_indices =
+        GetStartIndicesOfEachDenseRow<int32>(indices_mat, &contains_empty_rows);
+    // indices_list = {0, 1, 1, 2, 2, 2, 3};
+    EXPECT_FALSE(ContainsEmptyRows(segment_indices));
+  }
+  {
+    int64 data[] = {0, 0, 1, 0, 1, 1, 2, 0, 2, 1, 2, 2, 3, 4};
+    TTypes<int64>::ConstMatrix indices_mat(data, 7, 2);
+    bool contains_empty_rows;
+    const auto segment_indices =
+        GetStartIndicesOfEachDenseRow<int64>(indices_mat, &contains_empty_rows);
+    // indices_list = {0, 1, 1, 2, 2, 2, 3};
+    EXPECT_FALSE(ContainsEmptyRows(segment_indices));
+  }
+  {
+    uint32 data[] = {0, 0, 0, 1, 0, 2, 2, 0, 2, 1, 2, 2, 3, 4};
+    TTypes<uint32>::ConstMatrix indices_mat(data, 7, 2);
+    bool contains_empty_rows;
+    const auto segment_indices = GetStartIndicesOfEachDenseRow<uint32>(
+        indices_mat, &contains_empty_rows);
+    // indices_list = {0, 0, 0, 2, 2, 2, 3};
+    EXPECT_TRUE(ContainsEmptyRows(segment_indices));
+  }
+  {
+    int64 data[] = {0, 0, 0, 1, 0, 2, 2, 0, 2, 1, 2, 2, 3, 4};
+    TTypes<int64>::ConstMatrix indices_mat(data, 7, 2);
+    bool contains_empty_rows;
+    const auto segment_indices =
+        GetStartIndicesOfEachDenseRow<int64>(indices_mat, &contains_empty_rows);
+    // indices_list = {0, 0, 0, 2, 2, 2, 3};
+    EXPECT_TRUE(ContainsEmptyRows(segment_indices));
+  }
+  {
+    uint64 data[] = {0, 0, 0, 1, 0, 2, 1, 0, 2, 1, 2, 2, 3, 4};
+    TTypes<uint64>::ConstMatrix indices_mat(data, 7, 2);
+    bool contains_empty_rows;
+    const auto segment_indices = GetStartIndicesOfEachDenseRow<uint64>(
+        indices_mat, &contains_empty_rows);
+    // indices_list = {0, 0, 0, 1, 2, 2, 3};
+    EXPECT_FALSE(ContainsEmptyRows(segment_indices));
+  }
+}
+
+TEST(SparseUtilsTest, FindNextDenseRowStartIndex) {
+  {
+    int32 data[] = {0, 0, 1, 0, 4, 0, 6, 0, 7, 0, 8, 0, 10, 0, 12, 0};
+    TTypes<int32>::ConstMatrix indices_mat(data, 8, 2);
+    // indices_list = {0, 1, 4, 6, 7, 8, 10, 12};
+    for (int32 i = 0; i < 8; ++i) {
+      EXPECT_EQ(i + 1, FindNextDenseRowStartIndex<int32>(i, indices_mat));
+    }
+  }
+  {
+    uint16 data[] = {0, 0, 1, 0, 4, 0, 6, 0, 7, 0, 8, 0, 10, 0, 12, 0};
+    TTypes<uint16>::ConstMatrix indices_mat(data, 8, 2);
+    // indices_list = {0, 1, 4, 6, 7, 8, 10, 12};
+    for (uint16 i = 0; i < 8; ++i) {
+      EXPECT_EQ(i + 1, FindNextDenseRowStartIndex<uint16>(i, indices_mat));
+    }
+  }
+  {
+    int64 data[] = {0, 0, 1, 0, 1, 0, 4, 0, 4, 0, 4, 0,  6, 0,  7,
+                    0, 7, 0, 7, 0, 7, 0, 8, 0, 8, 0, 10, 0, 12, 0};
+    TTypes<int64>::ConstMatrix indices_mat(data, 15, 2);
+    // indices_list = {0, 1, 1, 4, 4, 4,  6, 7, 7, 7, 7, 8, 8, 10, 12};
+    EXPECT_EQ(3, FindNextDenseRowStartIndex<int64>(static_cast<int64>(1),
+                                                   indices_mat));
+    EXPECT_EQ(3, FindNextDenseRowStartIndex<int64>(static_cast<int64>(2),
+                                                   indices_mat));
+    EXPECT_EQ(6, FindNextDenseRowStartIndex<int64>(static_cast<int64>(3),
+                                                   indices_mat));
+    EXPECT_EQ(6, FindNextDenseRowStartIndex<int64>(static_cast<int64>(4),
+                                                   indices_mat));
+    EXPECT_EQ(14, FindNextDenseRowStartIndex<int64>(static_cast<int64>(13),
+                                                    indices_mat));
+    EXPECT_EQ(15, FindNextDenseRowStartIndex<int64>(static_cast<int64>(14),
+                                                    indices_mat));
+  }
+}
+
+}  // namespace
diff --git a/tensorflow/core/kernels/tensor_flag_utils.cc b/tensorflow/core/kernels/tensor_flag_utils.cc
new file mode 100644
index 0000000000..b6f9192729
--- /dev/null
+++ b/tensorflow/core/kernels/tensor_flag_utils.cc
@@ -0,0 +1,187 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/kernels/tensor_flag_utils.h"
+
+#include "absl/strings/str_cat.h"
+#include "tensorflow/core/framework/tensor_shape.h"
+
+namespace tensorflow {
+namespace tensor_flag_utils {
+
+Status ValidateSparseMatrixShardingConfig(const Tensor& config) {
+  if (TensorShapeUtils::IsScalar(config.shape())) {
+    const float scalar_config = config.template scalar<float>()();
+    if (0 < scalar_config && scalar_config <= 1.0) {
+      return Status::OK();
+    }
+    return Status(
+        error::INVALID_ARGUMENT,
+        absl::StrCat("Expected config to be in range (0, 1] but instead found ",
+                     scalar_config));
+  }
+  if (!TensorShapeUtils::IsMatrix(config.shape())) {
+    return Status(error::INVALID_ARGUMENT,
+                  absl::StrCat("Expected config to be either scalar or matrix "
+                               "but instead found tensor of rank ",
+                               config.dims()));
+  }
+  if (config.dim_size(1) != 3) {
+    return Status(
+        error::INVALID_ARGUMENT,
+        absl::StrCat(
+            "Expected config matrix to have dim(1) = 3 but instead found ",
+            config.dim_size(1)));
+  }
+
+  auto config_matrix = config.matrix<float>();
+  for (int i = 0; i < config.dim_size(0); ++i) {
+    if (0 > config_matrix(i, 0)) {
+      return errors::InvalidArgument(
+          "First column of fraction_rows_per_thread_config "
+          "should "
+          "have non-negative values but found ",
+          config_matrix(i, 0), " in row ", i);
+    }
+    if (0 > config_matrix(i, 1)) {
+      return errors::InvalidArgument(
+          "Second column of fraction_rows_per_thread_config "
+          "should "
+          "have non-negative values but found ",
+          config_matrix(i, 1), " in row ", i);
+    }
+    if (!(0 < config_matrix(i, 2) && config_matrix(i, 2) <= 1)) {
+      return errors::InvalidArgument(
+          "Last column of fraction_rows_per_thread_config should "
+          "have values in the range (0, 1] but found ",
+          config_matrix(i, 2), " in row ", i);
+    }
+  }
+  return Status::OK();
+}
+
+template <typename MatrixType, typename K>
+MatrixType FindConfigValueForKey(
+    const typename TTypes<MatrixType>::ConstMatrix& config_mat,
+    const std::pair<K, K>& key) {
+  const int last_row_index = config_mat.dimension(0) - 1;
+  for (int i = 0; i < last_row_index; ++i) {
+    if (key.first >= config_mat(i, 0) && key.second >= config_mat(i, 1)) {
+      return config_mat(i, 2);
+    }
+  }
+  return config_mat(last_row_index, 2);
+}
+
+Status ValidateScalarQuantityShardingConfig(const Tensor& config) {
+  if (TensorShapeUtils::IsScalar(config.shape())) {
+    const float scalar_config = config.template scalar<float>()();
+    if (0 < scalar_config && scalar_config <= 1.0) {
+      return Status::OK();
+    }
+    return Status(
+        error::INVALID_ARGUMENT,
+        absl::StrCat("Expected config to be in range (0, 1] but instead found ",
+                     scalar_config));
+  }
+  if (!TensorShapeUtils::IsMatrix(config.shape())) {
+    return Status(error::INVALID_ARGUMENT,
+                  absl::StrCat("Expected config to be either scalar or matrix "
+                               "but instead found tensor of rank ",
+                               config.dims()));
+  }
+  if (config.dim_size(1) != 2) {
+    return Status(
+        error::INVALID_ARGUMENT,
+        absl::StrCat(
+            "Expected config matrix to have dim(1) = 2 but instead found ",
+            config.dim_size(1)));
+  }
+
+  auto config_matrix = config.matrix<float>();
+  for (int i = 0; i < config.dim_size(0); ++i) {
+    if (0 > config_matrix(i, 0)) {
+      return errors::InvalidArgument(
+          "First column of fraction_rows_per_thread_config "
+          "should "
+          "have non-negative values but found ",
+          config_matrix(i, 0), " in row ", i);
+    }
+    if (!(0 < config_matrix(i, 1) && config_matrix(i, 1) <= 1)) {
+      return errors::InvalidArgument(
+          "Last column of fraction_rows_per_thread_config should "
+          "have values in the range (0, 1] but found ",
+          config_matrix(i, 1), " in row ", i);
+    }
+  }
+  return Status::OK();
+}
+
+template <typename MatrixType, typename K>
+MatrixType FindConfigValueForKey(
+    const typename TTypes<MatrixType>::ConstMatrix& config_mat, const K key) {
+  const int last_row_index = config_mat.dimension(0) - 1;
+  for (int i = 0; i < last_row_index; ++i) {
+    if (key >= config_mat(i, 0)) {
+      return config_mat(i, 1);
+    }
+  }
+  return config_mat(last_row_index, 1);
+}
+
+template <typename Tindices>
+Tindices GetLinearBucket(const Tindices value, const Tindices bucket_size) {
+  const Tindices next_multiple_of_bucket_size =
+      (value + bucket_size - 1) / bucket_size * bucket_size;
+  return next_multiple_of_bucket_size - (bucket_size - 1);
+}
+
+template <typename Tindices>
+Tindices GetPowerBucket(const Tindices value, const Tindices bucket_size) {
+  if (bucket_size == 1) {
+    return 1;
+  }
+  return std::pow(bucket_size, std::floor(std::log(bucket_size * (value - 1)) /
+                                          std::log(bucket_size)) -
+                                   1) +
+         1;
+}
+
+#define REGISTER_SPARSE_UTIL_FUNCTIONS(TypeIndex)                           \
+  template float FindConfigValueForKey<float, TypeIndex>(                   \
+      const TTypes<float>::ConstMatrix& config_mat,                         \
+      const std::pair<TypeIndex, TypeIndex>& key);                          \
+  template float FindConfigValueForKey<float, TypeIndex>(                   \
+      const TTypes<float>::ConstMatrix& config_mat, const TypeIndex key);   \
+  template int64 FindConfigValueForKey<int64, TypeIndex>(                   \
+      const TTypes<int64>::ConstMatrix& config_mat, const TypeIndex key);
+
+REGISTER_SPARSE_UTIL_FUNCTIONS(int32);
+REGISTER_SPARSE_UTIL_FUNCTIONS(int64);
+REGISTER_SPARSE_UTIL_FUNCTIONS(uint8);
+REGISTER_SPARSE_UTIL_FUNCTIONS(uint16);
+REGISTER_SPARSE_UTIL_FUNCTIONS(uint32);
+REGISTER_SPARSE_UTIL_FUNCTIONS(uint64);
+
+template int32 GetLinearBucket(const int32 value, const int32 bucket_size);
+
+template int64 GetLinearBucket(const int64 value, const int64 bucket_size);
+
+template int32 GetPowerBucket(const int32 value, const int32 bucket_size);
+
+template int64 GetPowerBucket(const int64 value, const int64 bucket_size);
+
+}  // namespace tensor_flag_utils
+}  // namespace tensorflow
diff --git a/tensorflow/core/kernels/tensor_flag_utils.h b/tensorflow/core/kernels/tensor_flag_utils.h
new file mode 100644
index 0000000000..f406c73a29
--- /dev/null
+++ b/tensorflow/core/kernels/tensor_flag_utils.h
@@ -0,0 +1,78 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// Helpers for parsing tensors as runtime flags.
+#ifndef TENSORFLOW_CORE_KERNELS_TENSOR_FLAG_UTILS_H_
+#define TENSORFLOW_CORE_KERNELS_TENSOR_FLAG_UTILS_H_
+
+#include <utility>
+#include <vector>
+
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/tensor_types.h"
+#include "tensorflow/core/platform/types.h"
+
+namespace tensorflow {
+namespace tensor_flag_utils {
+
+// Converts tensor.vec<Tindices> to an std::vector<Tindices> object, appends
+// the value num_nonzero_entries_in_sparse_mat, and returns the result.
+template <typename Tindices>
+std::vector<Tindices> ParseRowStartIndices(
+    const tensorflow::Tensor& tensor,
+    const Tindices num_nonzero_entries_in_sparse_mat);
+
+// Returns Status::OK() if and only if config is a float scalar or a matrix with
+// dimensions M x 3. If config is a scalar then config must be in the range
+// [0, 1.0). If confix is a matrix then config must have shape M x 3, all of
+// its entries must be positive, and entries in the last column may not
+// exceed 1.0. If config is a matrix then it may not be empty.
+Status ValidateSparseMatrixShardingConfig(const Tensor& config);
+
+// Returns Status::OK() if and only if config is a float scalar or a non-empty
+// matrix with dimensions M x 2.
+Status ValidateScalarQuantityShardingConfig(const Tensor& config);
+
+// Returns the last entry of the first row in config_mat for which the first
+// two entries are no smaller than the respective entries in key. If no such
+// row exists then returns the last entry in the last row in config_mat.
+// config_mat may not be empty.
+template <typename MatrixType, typename K>
+MatrixType FindConfigValueForKey(
+    const typename TTypes<MatrixType>::ConstMatrix& config_mat,
+    const std::pair<K, K>& key);
+
+// Returns the last entry of the first row in config_mat for which the first
+// two entries are no smaller than the respective entries in key. If no such
+// row exists then returns the last entry in the last row in config_mat.
+// config_mat may not be empty.
+template <typename MatrixType, typename K>
+MatrixType FindConfigValueForKey(
+    const typename TTypes<MatrixType>::ConstMatrix& config_mat, const K key);
+
+// Returns largest multiple of bucket_size less than value.
+// Expects 1 <= bucket_size <= value.
+template <typename Tindices>
+Tindices GetLinearBucket(const Tindices value, const Tindices bucket_size);
+
+// Returns the largest power of bucket_size less than value.
+// Expects 1 <= bucket_size <= value. If bucket_size = 1, returns 1.
+template <typename Tindices>
+Tindices GetPowerBucket(const Tindices value, const Tindices bucket_size);
+
+}  // namespace tensor_flag_utils
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_KERNELS_TENSOR_FLAG_UTILS_H_
diff --git a/tensorflow/core/kernels/tensor_flag_utils_test.cc b/tensorflow/core/kernels/tensor_flag_utils_test.cc
new file mode 100644
index 0000000000..23ccc7ad7a
--- /dev/null
+++ b/tensorflow/core/kernels/tensor_flag_utils_test.cc
@@ -0,0 +1,322 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/kernels/tensor_flag_utils.h"
+
+#include <vector>
+
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/tensor_types.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace {
+
+using tensorflow::DataType;
+using tensorflow::int32;
+using tensorflow::int64;
+using tensorflow::Tensor;
+using tensorflow::TTypes;
+using tensorflow::error::INVALID_ARGUMENT;
+using tensorflow::tensor_flag_utils::FindConfigValueForKey;
+using tensorflow::tensor_flag_utils::GetLinearBucket;
+using tensorflow::tensor_flag_utils::GetPowerBucket;
+using tensorflow::tensor_flag_utils::ValidateScalarQuantityShardingConfig;
+using tensorflow::tensor_flag_utils::ValidateSparseMatrixShardingConfig;
+
+TEST(SparseUtilsTest, ValidateSparseMatrixShardingConfig) {
+  // Only a default is specified.
+  {
+    Tensor t(DataType::DT_FLOAT, {});
+    t.scalar<float>()() = 0.7;
+    EXPECT_TRUE(ValidateSparseMatrixShardingConfig(t).ok());
+  }
+  {
+    Tensor t(DataType::DT_FLOAT, {});
+    t.scalar<float>()() = 1.0;
+    EXPECT_TRUE(ValidateSparseMatrixShardingConfig(t).ok());
+  }
+
+  // Misshapen.
+  {
+    Tensor t(DataType::DT_FLOAT, {1, 1});
+    int indx = 0;
+    for (const float v : {60.0}) {
+      t.flat<float>()(indx++) = v;
+    }
+    EXPECT_EQ(INVALID_ARGUMENT, ValidateSparseMatrixShardingConfig(t).code());
+  }
+  {
+    Tensor t(DataType::DT_FLOAT, {1, 2});
+    int indx = 0;
+    for (const float v : {
+             60.0,
+             50.0,
+         }) {
+      t.flat<float>()(indx++) = v;
+    }
+    EXPECT_EQ(INVALID_ARGUMENT, ValidateSparseMatrixShardingConfig(t).code());
+  }
+
+  // Only one key is specified.
+  {
+    Tensor t(DataType::DT_FLOAT, {1, 3});
+    int indx = 0;
+    for (const float v : {30.0, 20.0, 1.0}) {
+      t.flat<float>()(indx++) = v;
+    }
+    EXPECT_TRUE(ValidateSparseMatrixShardingConfig(t).ok());
+  }
+
+  // Two keys are specified.
+  {
+    Tensor t(DataType::DT_FLOAT, {2, 3});
+    int indx = 0;
+    for (const float v : {60.0, 50.0, 0.41, 30.0, 20.0, 0.7}) {
+      t.flat<float>()(indx++) = v;
+    }
+    EXPECT_TRUE(ValidateSparseMatrixShardingConfig(t).ok());
+  }
+
+  // Out of range.
+  {
+    Tensor t(DataType::DT_FLOAT, {2, 3});
+    int indx = 0;
+    for (const float v : {60.0, 40.0, 0.41, 30.0, 20.0, 10.7}) {
+      t.flat<float>()(indx++) = v;
+    }
+    EXPECT_EQ(INVALID_ARGUMENT, ValidateSparseMatrixShardingConfig(t).code());
+  }
+  {
+    Tensor t(DataType::DT_FLOAT, {2, 3});
+    int indx = 0;
+    for (const float v : {60.0, 40.0, 0.41, 30.0, 20.0, -0.7}) {
+      t.flat<float>()(indx++) = v;
+    }
+    EXPECT_EQ(INVALID_ARGUMENT, ValidateSparseMatrixShardingConfig(t).code());
+  }
+  {
+    Tensor t(DataType::DT_FLOAT, {2, 3});
+    int indx = 0;
+    for (const float v : {60.0, -40.0, 0.41, 30.0, 20.0, 0.7}) {
+      t.flat<float>()(indx++) = v;
+    }
+    EXPECT_EQ(INVALID_ARGUMENT, ValidateSparseMatrixShardingConfig(t).code());
+  }
+  {
+    Tensor t(DataType::DT_FLOAT, {});
+    t.scalar<float>()() = -0.5;
+    EXPECT_EQ(INVALID_ARGUMENT, ValidateSparseMatrixShardingConfig(t).code());
+  }
+  {
+    Tensor t(DataType::DT_FLOAT, {});
+    t.scalar<float>()() = 0;
+    EXPECT_EQ(INVALID_ARGUMENT, ValidateSparseMatrixShardingConfig(t).code());
+  }
+  {
+    Tensor t(DataType::DT_FLOAT, {});
+    t.scalar<float>()() = 1.2;
+    EXPECT_EQ(INVALID_ARGUMENT, ValidateSparseMatrixShardingConfig(t).code());
+  }
+}
+
+TEST(SparseUtilsTest, ValidateScalarQuantityShardingConfig) {
+  // Only a default is specified.
+  {
+    Tensor t(DataType::DT_FLOAT, {});
+    t.scalar<float>()() = 0.7;
+    EXPECT_TRUE(ValidateScalarQuantityShardingConfig(t).ok());
+  }
+  {
+    Tensor t(DataType::DT_FLOAT, {});
+    t.scalar<float>()() = 1.0;
+    EXPECT_TRUE(ValidateScalarQuantityShardingConfig(t).ok());
+  }
+  {
+    Tensor t(DataType::DT_FLOAT, {});
+    t.scalar<float>()() = 1.2;
+    EXPECT_EQ(INVALID_ARGUMENT, ValidateScalarQuantityShardingConfig(t).code());
+  }
+
+  // Misshapen.
+  {
+    Tensor t(DataType::DT_FLOAT, {1, 1});
+    int indx = 0;
+    for (const float v : {60.0}) {
+      t.flat<float>()(indx++) = v;
+    }
+    EXPECT_EQ(INVALID_ARGUMENT, ValidateScalarQuantityShardingConfig(t).code());
+  }
+  {
+    Tensor t(DataType::DT_FLOAT, {1, 2});
+    int indx = 0;
+    for (const float v : {
+             60.0,
+             50.0,
+         }) {
+      t.flat<float>()(indx++) = v;
+    }
+    EXPECT_EQ(INVALID_ARGUMENT, ValidateScalarQuantityShardingConfig(t).code());
+  }
+
+  // Two keys are specified.
+  {
+    Tensor t(DataType::DT_FLOAT, {1, 3});
+    int indx = 0;
+    for (const float v : {30.0, 20.0, 1.0}) {
+      t.flat<float>()(indx++) = v;
+    }
+    EXPECT_EQ(INVALID_ARGUMENT, ValidateScalarQuantityShardingConfig(t).code());
+  }
+
+  // Only one key is specified.
+  {
+    Tensor t(DataType::DT_FLOAT, {2, 2});
+    int indx = 0;
+    for (const float v : {60.0, 0.41, 30.0, 0.7}) {
+      t.flat<float>()(indx++) = v;
+    }
+    EXPECT_TRUE(ValidateScalarQuantityShardingConfig(t).ok());
+  }
+
+  // Out of range.
+  {
+    Tensor t(DataType::DT_FLOAT, {2, 2});
+    int indx = 0;
+    for (const float v : {60.0, 0.41, 30.0, 10.7}) {
+      t.flat<float>()(indx++) = v;
+    }
+    EXPECT_EQ(INVALID_ARGUMENT, ValidateScalarQuantityShardingConfig(t).code());
+  }
+  {
+    Tensor t(DataType::DT_FLOAT, {2, 2});
+    int indx = 0;
+    for (const float v : {60.0, 0.41, 30.0, -0.7}) {
+      t.flat<float>()(indx++) = v;
+    }
+    EXPECT_EQ(INVALID_ARGUMENT, ValidateScalarQuantityShardingConfig(t).code());
+  }
+  {
+    Tensor t(DataType::DT_FLOAT, {2, 2});
+    int indx = 0;
+    for (const float v : {-40.0, 0.41, 20.0, 0.7}) {
+      t.flat<float>()(indx++) = v;
+    }
+    EXPECT_EQ(INVALID_ARGUMENT, ValidateScalarQuantityShardingConfig(t).code());
+  }
+  {
+    Tensor t(DataType::DT_FLOAT, {});
+    t.scalar<float>()() = -0.5;
+    EXPECT_EQ(INVALID_ARGUMENT, ValidateScalarQuantityShardingConfig(t).code());
+  }
+  {
+    Tensor t(DataType::DT_FLOAT, {});
+    t.scalar<float>()() = 0;
+    EXPECT_EQ(INVALID_ARGUMENT, ValidateScalarQuantityShardingConfig(t).code());
+  }
+  {
+    Tensor t(DataType::DT_FLOAT, {});
+    t.scalar<float>()() = 1.2;
+    EXPECT_EQ(INVALID_ARGUMENT, ValidateScalarQuantityShardingConfig(t).code());
+  }
+}
+
+TEST(SparseUtils, FindConfigValueForKey) {
+  {
+    float data[] = {60.0, 50.0, 0.41, 30.0, 20.0, 0.1, 0, 0, 0.7};
+    TTypes<float>::ConstMatrix config_mat(data, 3, 3);
+    auto val = FindConfigValueForKey<float, int32>(config_mat, {70, 40});
+    EXPECT_FLOAT_EQ(0.1, val);
+    val = FindConfigValueForKey<float, int32>(config_mat, {60, 50});
+    EXPECT_FLOAT_EQ(0.41, val);
+    val = FindConfigValueForKey<float, int32>(config_mat, {60, 60});
+    EXPECT_FLOAT_EQ(0.41, val);
+    val = FindConfigValueForKey<float, int32>(config_mat, {60, 40});
+    EXPECT_FLOAT_EQ(0.1, val);
+    val = FindConfigValueForKey<float, int32>(config_mat, {50, 60});
+    EXPECT_FLOAT_EQ(0.1, val);
+    val = FindConfigValueForKey<float, int32>(config_mat, {20, 30});
+    EXPECT_FLOAT_EQ(0.7, val);
+    val = FindConfigValueForKey<float, int32>(config_mat, {30, 10});
+    EXPECT_FLOAT_EQ(0.7, val);
+  }
+  {
+    float data[] = {0, 0, 0.7};
+    TTypes<float>::ConstMatrix config_mat(data, 1, 3);
+    auto val = FindConfigValueForKey<float, int64>(config_mat, {70, 40});
+    EXPECT_FLOAT_EQ(0.7, val);
+    val = FindConfigValueForKey<float, int64>(config_mat, {60, 50});
+    EXPECT_FLOAT_EQ(0.7, val);
+    val = FindConfigValueForKey<float, int64>(config_mat, {60, 60});
+    EXPECT_FLOAT_EQ(0.7, val);
+    val = FindConfigValueForKey<float, int64>(config_mat, {60, 40});
+    EXPECT_FLOAT_EQ(0.7, val);
+    val = FindConfigValueForKey<float, int64>(config_mat, {50, 60});
+    EXPECT_FLOAT_EQ(0.7, val);
+    val = FindConfigValueForKey<float, int64>(config_mat, {20, 30});
+    EXPECT_FLOAT_EQ(0.7, val);
+    val = FindConfigValueForKey<float, int64>(config_mat, {30, 10});
+    EXPECT_FLOAT_EQ(0.7, val);
+  }
+  {
+    float data[] = {60.0, 50.0, 0.41, 0, 0, 0.7};
+    TTypes<float>::ConstMatrix config_mat(data, 2, 3);
+    auto val = FindConfigValueForKey<float, int32>(config_mat, {70, 40});
+    EXPECT_FLOAT_EQ(0.7, val);
+    val = FindConfigValueForKey<float, int32>(config_mat, {60, 50});
+    EXPECT_FLOAT_EQ(0.41, val);
+    val = FindConfigValueForKey<float, int32>(config_mat, {60, 60});
+    EXPECT_FLOAT_EQ(0.41, val);
+    val = FindConfigValueForKey<float, int32>(config_mat, {60, 40});
+    EXPECT_FLOAT_EQ(0.7, val);
+    val = FindConfigValueForKey<float, int32>(config_mat, {50, 60});
+    EXPECT_FLOAT_EQ(0.7, val);
+    val = FindConfigValueForKey<float, int32>(config_mat, {20, 30});
+    EXPECT_FLOAT_EQ(0.7, val);
+    val = FindConfigValueForKey<float, int32>(config_mat, {30, 10});
+    EXPECT_FLOAT_EQ(0.7, val);
+  }
+  {
+    float data[] = {60.0, 0.41, 50.0, 0.14, 0, 0.7};
+    TTypes<float>::ConstMatrix config_mat(data, 3, 2);
+    auto val = FindConfigValueForKey<float, int32>(config_mat, 70);
+    EXPECT_FLOAT_EQ(0.41, val);
+    val = FindConfigValueForKey<float, int32>(config_mat, 60);
+    EXPECT_FLOAT_EQ(0.41, val);
+    val = FindConfigValueForKey<float, int32>(config_mat, 55);
+    EXPECT_FLOAT_EQ(0.14, val);
+    val = FindConfigValueForKey<float, int32>(config_mat, 50);
+    EXPECT_FLOAT_EQ(0.14, val);
+    val = FindConfigValueForKey<float, int32>(config_mat, 20);
+    EXPECT_FLOAT_EQ(0.7, val);
+    val = FindConfigValueForKey<float, int32>(config_mat, 30);
+    EXPECT_FLOAT_EQ(0.7, val);
+  }
+}
+
+TEST(SparseUtils, GetLinearBucket) {
+  EXPECT_EQ(11, GetLinearBucket(11, 5));
+  EXPECT_EQ(11, GetLinearBucket(12, 5));
+  EXPECT_EQ(1, GetLinearBucket(4ll, 5ll));
+}
+
+TEST(SparseUtils, GetPowerBucket) {
+  EXPECT_EQ(6, GetPowerBucket(11, 5));
+  EXPECT_EQ(6, GetPowerBucket(12, 5));
+  EXPECT_EQ(1332, GetPowerBucket(1335, 11));
+  EXPECT_EQ(5, GetPowerBucket(5ll, 4ll));
+  EXPECT_EQ(1, GetPowerBucket(4ll, 1ll));
+}
+
+}  // namespace
-- 
GitLab


From 0ed026100272867a270e1766c9ca0ae9804ff802 Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Fri, 18 Jan 2019 10:00:49 -0800
Subject: [PATCH 0975/1095] Disable jni on mac since mac doesn't use gnu linker

This does not preclude building Android Java JNI since that uses a different toolchain that does support version-script

PiperOrigin-RevId: 229946385
---
 tensorflow/lite/java/BUILD      | 15 +++++++++++++++
 tensorflow/lite/java/ovic/BUILD | 10 ++++++++--
 2 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/java/BUILD b/tensorflow/lite/java/BUILD
index a539a0cf77..8983079a31 100644
--- a/tensorflow/lite/java/BUILD
+++ b/tensorflow/lite/java/BUILD
@@ -90,6 +90,9 @@ java_test(
     size = "small",
     srcs = ["src/test/java/org/tensorflow/lite/TensorFlowLiteTest.java"],
     javacopts = JAVACOPTS,
+    tags = [
+        "no_mac",  # TODO(b/122888913): libtensorflowlite_test_jni broke on mac.
+    ],
     test_class = "org.tensorflow.lite.TensorFlowLiteTest",
     deps = [
         ":tensorflowlitelib",
@@ -103,6 +106,9 @@ java_test(
     size = "small",
     srcs = ["src/test/java/org/tensorflow/lite/DataTypeTest.java"],
     javacopts = JAVACOPTS,
+    tags = [
+        "no_mac",  # TODO(b/122888913): libtensorflowlite_test_jni broke on mac.
+    ],
     test_class = "org.tensorflow.lite.DataTypeTest",
     deps = [
         ":tensorflowlitelib",
@@ -126,6 +132,9 @@ java_test(
         "src/testdata/with_custom_op.lite",
     ],
     javacopts = JAVACOPTS,
+    tags = [
+        "no_mac",  # TODO(b/122888913): libtensorflowlite_test_jni broke on mac.
+    ],
     test_class = "org.tensorflow.lite.NativeInterpreterWrapperTest",
     deps = [
         ":tensorflowlitelib",
@@ -186,6 +195,9 @@ java_test(
         "src/testdata/add.bin",
     ],
     javacopts = JAVACOPTS,
+    tags = [
+        "no_mac",  # TODO(b/122888913): libtensorflowlite_test_jni broke on mac.
+    ],
     test_class = "org.tensorflow.lite.TensorTest",
     deps = [
         ":tensorflowlitelib",
@@ -199,6 +211,9 @@ filegroup(
     srcs = select({
         "//conditions:default": [":libtensorflowlite_jni.so"],
     }),
+    tags = [
+        "no_mac",  # TODO(b/122888913): libtensorflowlite_test_jni broke on mac.
+    ],
     visibility = ["//visibility:public"],
 )
 
diff --git a/tensorflow/lite/java/ovic/BUILD b/tensorflow/lite/java/ovic/BUILD
index 774320871e..b00c9cd058 100644
--- a/tensorflow/lite/java/ovic/BUILD
+++ b/tensorflow/lite/java/ovic/BUILD
@@ -19,7 +19,10 @@ java_test(
         "//tensorflow/lite/java/ovic/src/testdata:ovic_testdata",
     ],
     javacopts = JAVACOPTS,
-    tags = ["no_oss"],
+    tags = [
+        "no_mac",
+        "no_oss",  # TODO(b/122888913): libtensorflowlite_test_jni broke on mac.
+    ],
     test_class = "org.tensorflow.ovic.OvicClassifierTest",
     visibility = ["//visibility:public"],
     deps = [
@@ -87,7 +90,10 @@ java_test(
         "//tensorflow/lite/java/ovic/src/testdata:ovic_testdata",
     ],
     javacopts = JAVACOPTS,
-    tags = ["no_oss"],
+    tags = [
+        "no_mac",
+        "no_oss",  # TODO(b/122888913): libtensorflowlite_test_jni broke on mac.
+    ],
     test_class = "org.tensorflow.ovic.OvicDetectorTest",
     visibility = ["//visibility:public"],
     deps = [
-- 
GitLab


From e3f2b2067355da6197e8c9500fb74e40357a0f80 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 18 Jan 2019 10:19:29 -0800
Subject: [PATCH 0976/1095] Use google-cloud-cpp v0.5.0 in tensorflow's
 WORKSPACE.

PiperOrigin-RevId: 229949984
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 71ecd4d902..3d8ea6d9e1 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -185,15 +185,15 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
 
     tf_http_archive(
         name = "com_github_googlecloudplatform_google_cloud_cpp",
-        sha256 = "44eee8bd47cbd5ff192e895b45f9f913e2e117f10fdb9af0fd3b1a87a7b53bc3",
-        strip_prefix = "google-cloud-cpp-0.4.0",
+        sha256 = "886bcba3616d5f362838a2d86ae0198dd3670a84a84c82291cda6c30e14779fc",
+        strip_prefix = "google-cloud-cpp-0.5.0",
         system_build_file = clean_dep("//third_party/systemlibs:google_cloud_cpp.BUILD"),
         system_link_files = {
             "//third_party/systemlibs:google_cloud_cpp.google.cloud.bigtable.BUILD": "google/cloud/bigtable/BUILD",
         },
         urls = [
-            "https://mirror.bazel.build/github.com/GoogleCloudPlatform/google-cloud-cpp/archive/v0.4.0.tar.gz",
-            "https://github.com/GoogleCloudPlatform/google-cloud-cpp/archive/v0.4.0.tar.gz",
+            "https://mirror.bazel.build/github.com/GoogleCloudPlatform/google-cloud-cpp/archive/v0.5.0.tar.gz",
+            "https://github.com/GoogleCloudPlatform/google-cloud-cpp/archive/v0.5.0.tar.gz",
         ],
     )
 
-- 
GitLab


From e8d74b48d05e0673d93633ac38e1ab88d71f4dff Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 18 Jan 2019 10:20:52 -0800
Subject: [PATCH 0977/1095] Simplify BufferMap by moving forwardability out of
 it.

PiperOrigin-RevId: 229950204
---
 tensorflow/lite/delegates/flex/buffer_map.h   | 18 -------------
 .../lite/delegates/flex/buffer_map_test.cc    |  9 -------
 tensorflow/lite/delegates/flex/kernel.cc      | 27 +++++++++++++------
 3 files changed, 19 insertions(+), 35 deletions(-)

diff --git a/tensorflow/lite/delegates/flex/buffer_map.h b/tensorflow/lite/delegates/flex/buffer_map.h
index 45bc571f92..b73ed88d37 100644
--- a/tensorflow/lite/delegates/flex/buffer_map.h
+++ b/tensorflow/lite/delegates/flex/buffer_map.h
@@ -56,25 +56,7 @@ class BufferMap {
   // given TfLiteTensor's data.
   void SetFromTfLite(int tensor_index, const TfLiteTensor* tensor);
 
-  // Sets a bit indicating that the tensor associated with 'tensor_index' can
-  // be use by TF's forwarding optimizations.
-  void SetForwardable(int tensor_index) { forwardable_.insert(tensor_index); }
-
-  // Removes all information about which tensors are forwardable.
-  void ClearForwardable() { forwardable_.clear(); }
-
-  // Returns true if this tensor has been explicitly marks as forwardable by
-  // a call to SetForwardable().
-  bool IsForwardable(int tensor_index) const {
-    return forwardable_.count(tensor_index) > 0;
-  }
-
  private:
-  // List of tensors that can be used by TF in its forwarding optimization.
-  // Doing so allows an input tensor to be modified and used as the output
-  // tensor. The delegate takes care of not holding any references to tensors
-  // in this list while Eager is executing the corresponding op.
-  std::set<int> forwardable_;
   // Mapping from TL Lite tensor ID to TensorFlow's Tensor. All tensors that
   // are inputs or outputs of a subgraph will be added here, irrespective of
   // whether their data are managed by TF Lite or TensorFlow.
diff --git a/tensorflow/lite/delegates/flex/buffer_map_test.cc b/tensorflow/lite/delegates/flex/buffer_map_test.cc
index 8edeb280eb..9e8472f1e7 100644
--- a/tensorflow/lite/delegates/flex/buffer_map_test.cc
+++ b/tensorflow/lite/delegates/flex/buffer_map_test.cc
@@ -222,15 +222,6 @@ TEST(BufferMapTest, TensorFlowOverwritesTfLite) {
               ElementsAre(0, 0, 0, 0.123f, 0, 0));
 }
 
-TEST(BufferMapTest, Forwardable) {
-  BufferMap buffer_map;
-  EXPECT_FALSE(buffer_map.IsForwardable(0));
-  buffer_map.SetForwardable(0);
-  EXPECT_TRUE(buffer_map.IsForwardable(0));
-  buffer_map.ClearForwardable();
-  EXPECT_FALSE(buffer_map.IsForwardable(0));
-}
-
 }  // namespace
 }  // namespace flex
 }  // namespace tflite
diff --git a/tensorflow/lite/delegates/flex/kernel.cc b/tensorflow/lite/delegates/flex/kernel.cc
index 2e0fc22ad6..ceb9918f6f 100644
--- a/tensorflow/lite/delegates/flex/kernel.cc
+++ b/tensorflow/lite/delegates/flex/kernel.cc
@@ -67,6 +67,7 @@ class OpInputs {
     for (int index : TfLiteIntArrayView(indexes)) {
       inputs_.push_back(index);
     }
+    forwardable_.resize(inputs_.size());
   }
   ~OpInputs() {}
 
@@ -89,11 +90,21 @@ class OpInputs {
     }
   }
 
+  void SetForwardable(int i, bool v) { forwardable_[i] = v; }
+
+  bool IsForwardable(int i) const { return forwardable_[i]; }
+
   TensorSource GetTensorSource(int i) const { return sources_[i]; }
 
  private:
   std::vector<int> inputs_;
   std::vector<TensorSource> sources_;
+
+  // List of tensors that can be used by TF in its forwarding optimization.
+  // Doing so allows an input tensor to be modified and used as the output
+  // tensor. The delegate takes care of not holding any references to tensors
+  // in this list while Eager is executing the corresponding op.
+  std::vector<int> forwardable_;
 };
 
 // A list of outputs of a given node of the TensorFlow/Eager graph, along with
@@ -279,7 +290,7 @@ class OpNode {
       } else {
         // If this is a forwardable tensor, we will remove it from the previous
         // op's list, giving TF the opportunity to reuse its buffer.
-        bool unref_handle = buffer_map->IsForwardable(input_index);
+        bool unref_handle = inputs_.IsForwardable(i);
         auto* handle =
             s.node->outputs_.GetHandle(s.node_output_index, unref_handle);
         op_->MutableInputs()->push_back(handle);
@@ -485,13 +496,13 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
     }
   }
 
-  buffer_map->ClearForwardable();
-  for (const auto& x : tensor_ref_count) {
-    if (x.second == 1) {
-      // This tensor is referenced once by a single op. We can allow the TF
-      // kernel to "forward" it to the output, meaning its buffer will be
-      // reused and overwritten.
-      buffer_map->SetForwardable(x.first);
+  // All tensors that are referenced exactly once are marked as "forwardable",
+  // meaning that we will allow TensorFlow to reuse its buffer as the output of
+  // an op.
+  for (auto& node_data : op_data->nodes) {
+    for (int i = 0; i < node_data->inputs().Size(); ++i) {
+      bool f = (tensor_ref_count[node_data->inputs().TfLiteIndex(i)] == 1);
+      node_data->mutable_inputs()->SetForwardable(i, f);
     }
   }
 
-- 
GitLab


From d70cb7c2a548a664fbe85d273443412af6a24d80 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Fri, 18 Jan 2019 10:26:13 -0800
Subject: [PATCH 0978/1095] In tf2 tf.dtypes.as_string -> tf.strings.as_string

PiperOrigin-RevId: 229951239
---
 tensorflow/core/api_def/python_api/api_def_AsString.pbtxt | 4 ++++
 tensorflow/tools/api/golden/v1/tensorflow.strings.pbtxt   | 4 ++++
 tensorflow/tools/api/golden/v2/tensorflow.dtypes.pbtxt    | 4 ----
 tensorflow/tools/api/golden/v2/tensorflow.strings.pbtxt   | 4 ++++
 4 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/api_def/python_api/api_def_AsString.pbtxt b/tensorflow/core/api_def/python_api/api_def_AsString.pbtxt
index d51defc376..bc8cc309f5 100644
--- a/tensorflow/core/api_def/python_api/api_def_AsString.pbtxt
+++ b/tensorflow/core/api_def/python_api/api_def_AsString.pbtxt
@@ -2,6 +2,10 @@ op {
   graph_op_name: "AsString"
   endpoint {
     name: "dtypes.as_string"
+    deprecation_version: 2
+  }
+  endpoint {
+    name: "strings.as_string"
   }
   endpoint {
     name: "as_string"
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.strings.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.strings.pbtxt
index ad26ded10b..6f4f4c2f31 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.strings.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.strings.pbtxt
@@ -1,5 +1,9 @@
 path: "tensorflow.strings"
 tf_module {
+  member_method {
+    name: "as_string"
+    argspec: "args=[\'input\', \'precision\', \'scientific\', \'shortest\', \'width\', \'fill\', \'name\'], varargs=None, keywords=None, defaults=[\'-1\', \'False\', \'False\', \'-1\', \'\', \'None\'], "
+  }
   member_method {
     name: "format"
     argspec: "args=[\'template\', \'inputs\', \'placeholder\', \'summarize\', \'name\'], varargs=None, keywords=None, defaults=[\'{}\', \'3\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.dtypes.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.dtypes.pbtxt
index 01b870a816..956e4d93e5 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.dtypes.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.dtypes.pbtxt
@@ -112,10 +112,6 @@ tf_module {
     name: "as_dtype"
     argspec: "args=[\'type_value\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "as_string"
-    argspec: "args=[\'input\', \'precision\', \'scientific\', \'shortest\', \'width\', \'fill\', \'name\'], varargs=None, keywords=None, defaults=[\'-1\', \'False\', \'False\', \'-1\', \'\', \'None\'], "
-  }
   member_method {
     name: "cast"
     argspec: "args=[\'x\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.strings.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.strings.pbtxt
index 962cf9a723..e2da65eee4 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.strings.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.strings.pbtxt
@@ -1,5 +1,9 @@
 path: "tensorflow.strings"
 tf_module {
+  member_method {
+    name: "as_string"
+    argspec: "args=[\'input\', \'precision\', \'scientific\', \'shortest\', \'width\', \'fill\', \'name\'], varargs=None, keywords=None, defaults=[\'-1\', \'False\', \'False\', \'-1\', \'\', \'None\'], "
+  }
   member_method {
     name: "format"
     argspec: "args=[\'template\', \'inputs\', \'placeholder\', \'summarize\', \'name\'], varargs=None, keywords=None, defaults=[\'{}\', \'3\', \'None\'], "
-- 
GitLab


From 9f3a726641379b3c11c60693e3fc44c40e71d4fd Mon Sep 17 00:00:00 2001
From: Edward Loper <edloper@google.com>
Date: Fri, 18 Jan 2019 10:30:08 -0800
Subject: [PATCH 0979/1095] For SparseTensor shape invariant: accept
 TensorShape(None), and treat is as TensorShape([None]).

PiperOrigin-RevId: 229951972
---
 tensorflow/python/framework/sparse_tensor.py      |  3 ++-
 .../kernel_tests/control_flow_ops_py_test.py      | 15 +++++++++++++++
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/framework/sparse_tensor.py b/tensorflow/python/framework/sparse_tensor.py
index c7f505fff3..c69fa41677 100644
--- a/tensorflow/python/framework/sparse_tensor.py
+++ b/tensorflow/python/framework/sparse_tensor.py
@@ -249,7 +249,8 @@ class SparseTensor(_TensorLike, composite_tensor.CompositeTensor):
   def _shape_invariant_to_components(self, shape=None):
     if shape is None:
       shape = self.dense_shape.shape
-      # TODO(edloper): What if shape.dense_shape.shape.ndims is None?
+    if shape.ndims is None:
+      shape = tensor_shape.TensorShape([None])
     if shape.ndims != 1:
       raise ValueError("Shape invariant for SparseTensor must have the form "
                        "TensorShape([r]), got %r" % shape)
diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
index 27b3e26c77..d9d7af1f0d 100644
--- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
@@ -1598,6 +1598,21 @@ class ControlFlowTest(test.TestCase):
     self.assertEqual(r.values.get_shape().as_list(), [None])
     self.assertEqual(r.dense_shape.get_shape().as_list(), [None])
 
+    # Shape invariant with ndims=None.  Technically, this isn't supported
+    # according to the docs, but we support it for backwards compatibility.
+    _, r = control_flow_ops.while_loop(
+        c, b1, [i, x],
+        [i.get_shape(), tensor_shape.TensorShape(None)])
+    self.assertEqual(r.indices.get_shape().as_list(), [None, None])
+    self.assertEqual(r.values.get_shape().as_list(), [None])
+    self.assertEqual(r.dense_shape.get_shape().as_list(), [None])
+    _, r = control_flow_ops.while_loop(
+        c, b3, [i, x],
+        [i.get_shape(), tensor_shape.TensorShape(None)])
+    self.assertEqual(r.indices.get_shape().as_list(), [None, None])
+    self.assertEqual(r.values.get_shape().as_list(), [None])
+    self.assertEqual(r.dense_shape.get_shape().as_list(), [None])
+
     # Explicit shape invariant, with a specific (incompatible) rank.
     with self.assertRaisesRegexp(ValueError, "is not compatible with"):
       _, r = control_flow_ops.while_loop(
-- 
GitLab


From 91e843486d1f230554dd0662273d88eb6fa7e295 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 18 Jan 2019 10:48:05 -0800
Subject: [PATCH 0980/1095] Add TFLite NNAPI interface which uses a struct
 instead of using directly function calls. Add an empty implementation for
 NNAPI outside Android.

PiperOrigin-RevId: 229955475
---
 tensorflow/lite/BUILD                         |   2 +-
 tensorflow/lite/delegates/nnapi/BUILD         |   2 +-
 .../lite/delegates/nnapi/nnapi_delegate.cc    | 163 ++---
 tensorflow/lite/nnapi/BUILD                   |  23 +
 tensorflow/lite/nnapi/NeuralNetworksShim.h    |   5 +
 tensorflow/lite/nnapi/nnapi_implementation.cc | 137 +++++
 tensorflow/lite/nnapi/nnapi_implementation.h  | 582 ++++++++++++++++++
 .../lite/nnapi/nnapi_implementation_test.cc   | 120 ++++
 tensorflow/lite/nnapi_delegate.cc             | 137 ++---
 tensorflow/lite/testing/BUILD                 |   2 +-
 tensorflow/lite/testing/nnapi_example.cc      |   2 +-
 11 files changed, 1013 insertions(+), 162 deletions(-)
 create mode 100644 tensorflow/lite/nnapi/nnapi_implementation.cc
 create mode 100644 tensorflow/lite/nnapi/nnapi_implementation.h
 create mode 100644 tensorflow/lite/nnapi/nnapi_implementation_test.cc

diff --git a/tensorflow/lite/BUILD b/tensorflow/lite/BUILD
index 18fd1c1012..2cc661f2ba 100644
--- a/tensorflow/lite/BUILD
+++ b/tensorflow/lite/BUILD
@@ -193,7 +193,7 @@ cc_library(
         "//tensorflow/lite/core/api:api",
         "//tensorflow/lite/kernels:eigen_support",
         "//tensorflow/lite/kernels:gemm_support",
-        "//tensorflow/lite/nnapi:nnapi_lib",
+        "//tensorflow/lite/nnapi:nnapi_implementation",
         "//tensorflow/lite/profiling:profiler",
         "//tensorflow/lite/schema:schema_fbs",
     ] + select({
diff --git a/tensorflow/lite/delegates/nnapi/BUILD b/tensorflow/lite/delegates/nnapi/BUILD
index 63f6da1fa6..99cd6d3f85 100644
--- a/tensorflow/lite/delegates/nnapi/BUILD
+++ b/tensorflow/lite/delegates/nnapi/BUILD
@@ -16,7 +16,7 @@ cc_library(
         "//tensorflow/lite:kernel_api",
         "//tensorflow/lite/c:c_api_internal",
         "//tensorflow/lite/kernels:kernel_util",
-        "//tensorflow/lite/nnapi:nnapi_lib",
+        "//tensorflow/lite/nnapi:nnapi_implementation",
     ],
 )
 
diff --git a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
index 94fcc8740b..d14e449b3e 100644
--- a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
+++ b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc
@@ -25,7 +25,7 @@ limitations under the License.
 #include "tensorflow/lite/context_util.h"
 #include "tensorflow/lite/delegates/nnapi/nnapi_delegate.h"
 #include "tensorflow/lite/kernels/kernel_util.h"
-#include "tensorflow/lite/nnapi/NeuralNetworksShim.h"
+#include "tensorflow/lite/nnapi/nnapi_implementation.h"
 
 #ifdef __ANDROID__
 #include <sys/mman.h>
@@ -49,57 +49,39 @@ namespace {
   } while (0)
 
 namespace {
-int32_t GetAndroidSdkVersion() {
-#ifdef __ANDROID__
-  const char* sdkProp = "ro.build.version.sdk";
-  char sdkVersion[PROP_VALUE_MAX];
-  int length = __system_property_get(sdkProp, sdkVersion);
-  if (length != 0) {
-    for (int i = 0; i < length; ++i) {
-      int digit = sdkVersion[i] - '0';
-      if (digit < 0 || digit > 9) {
-        // Non-numeric SDK version, assume it's higher then expected;
-        return std::numeric_limits<int32_t>::max();
-      }
-    }
-    return atoi(sdkVersion);
-  }
-#endif  // __ANDROID__
-  return 0;
-}
-
 constexpr int32_t kMinSdkVersionForNNAPI = 27;
 constexpr int32_t kMinSdkVersionForNNAPI11 = 28;
-static const int32_t kAndroidSdkVersion = GetAndroidSdkVersion();
-
 }  // namespace
 
 // RAII NN API Model Destructor for use with std::unique_ptr
 struct NNFreeModel {
   void operator()(ANeuralNetworksModel* model) {
-    ANeuralNetworksModel_free(model);
+    NnApiImplementation()->ANeuralNetworksModel_free(model);
   }
 };
 // RAII NN API Compilation Destructor for use with std::unique_ptr
 struct NNFreeCompilation {
   void operator()(ANeuralNetworksCompilation* model) {
-    ANeuralNetworksCompilation_free(model);
+    NnApiImplementation()->ANeuralNetworksCompilation_free(model);
   }
 };
 
 // Manage NNAPI shared memory handle
 class NNMemory {
  public:
-  NNMemory(const char* name, size_t size) {
 #ifdef __ANDROID__
+  NNMemory(const NnApi* nnapi, const char* name, size_t size) {
+    nnapi_ = nnapi;
     byte_size_ = size;
-    fd_ = ASharedMemory_create(name, size);
+    fd_ = nnapi_->ASharedMemory_create(name, size);
     data_ptr_ = reinterpret_cast<uint8_t*>(
         mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd_, 0));
-    ANeuralNetworksMemory_createFromFd(size, PROT_READ | PROT_WRITE, fd_, 0,
-                                       &nn_memory_handle_);
-#endif
+    nnapi_->ANeuralNetworksMemory_createFromFd(size, PROT_READ | PROT_WRITE,
+                                               fd_, 0, &nn_memory_handle_);
   }
+#else
+  NNMemory(const NnApi* /*nnapi*/, const char* /*name*/, size_t /*size*/) {}
+#endif
 
   ~NNMemory() {
 #ifdef __ANDROID__
@@ -107,7 +89,7 @@ class NNMemory {
       munmap(data_ptr_, byte_size_);
     }
     if (nn_memory_handle_) {
-      ANeuralNetworksMemory_free(nn_memory_handle_);
+      nnapi_->ANeuralNetworksMemory_free(nn_memory_handle_);
     }
     if (fd_ > 0) close(fd_);
 #endif
@@ -118,6 +100,7 @@ class NNMemory {
 
  private:
 #ifdef __ANDROID__
+  const NnApi* nnapi_;
   int fd_ = 0;
   size_t byte_size_ = 0;
 #endif
@@ -167,9 +150,10 @@ class OperandMapping {
 // operands for both tensors and parameters, and TFLite separates the two.
 class NNAPIOpBuilder {
  public:
-  NNAPIOpBuilder(TfLiteContext* context, OperandMapping* tensor_mapping,
-                 ANeuralNetworksModel* nn_model)
-      : context_(context),
+  NNAPIOpBuilder(const NnApi* nnapi, TfLiteContext* context,
+                 OperandMapping* tensor_mapping, ANeuralNetworksModel* nn_model)
+      : nnapi_(nnapi),
+        context_(context),
         operand_mapping_(tensor_mapping),
         nn_model_(nn_model) {}
 
@@ -225,7 +209,8 @@ class NNAPIOpBuilder {
         .dimensionCount = dimension_count,
         .dimensions = dims.data()};
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context_, ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
+        context_,
+        nnapi_->ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
     int ann_operand = operand_mapping_->add_new_non_tensor_operand();
     augmented_outputs_.push_back(ann_operand);
     return kTfLiteOk;
@@ -242,7 +227,8 @@ class NNAPIOpBuilder {
         reinterpret_cast<uint32_t*>(tensor->dims->data), tensor->params.scale,
         tensor->params.zero_point};
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context_, ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
+        context_,
+        nnapi_->ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
     augmented_outputs_.push_back(ann_index);
 
     *ann_tensor_index_out = ann_index;
@@ -299,13 +285,14 @@ class NNAPIOpBuilder {
         nn_type, static_cast<uint32_t>(tensor->dims->size),
         reinterpret_cast<uint32_t*>(tensor->dims->data), scale, zeroPoint};
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context_, ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
+        context_,
+        nnapi_->ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
 
     if (tensor->allocation_type == kTfLiteMmapRo) {
       // TODO(b/80630405): Use NNAPIAllocation.
       RETURN_TFLITE_ERROR_IF_NN_ERROR(
           context_,
-          ANeuralNetworksModel_setOperandValue(
+          nnapi_->ANeuralNetworksModel_setOperandValue(
               nn_model_, ann_tensor_index, tensor->data.raw, tensor->bytes));
     }
 
@@ -318,7 +305,7 @@ class NNAPIOpBuilder {
     // Actually add a NN API operation
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
         context_,
-        ANeuralNetworksModel_addOperation(
+        nnapi_->ANeuralNetworksModel_addOperation(
             nn_model_, type, static_cast<uint32_t>(augmented_inputs_.size()),
             augmented_inputs_.data(),
             static_cast<uint32_t>(augmented_outputs_.size()),
@@ -333,11 +320,12 @@ class NNAPIOpBuilder {
   TfLiteStatus AddScalarOperand(T value, int32_t nn_type) {
     ANeuralNetworksOperandType operand_type{.type = nn_type};
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context_, ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
+        context_,
+        nnapi_->ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
     int ann_operand = operand_mapping_->add_new_non_tensor_operand();
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context_, ANeuralNetworksModel_setOperandValue(nn_model_, ann_operand,
-                                                       &value, sizeof(T)));
+        context_, nnapi_->ANeuralNetworksModel_setOperandValue(
+                      nn_model_, ann_operand, &value, sizeof(T)));
     augmented_inputs_.push_back(ann_operand);
     return kTfLiteOk;
   }
@@ -348,15 +336,19 @@ class NNAPIOpBuilder {
     ANeuralNetworksOperandType operand_type{
         .type = nn_type, .dimensionCount = 1, .dimensions = &num_values};
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context_, ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
+        context_,
+        nnapi_->ANeuralNetworksModel_addOperand(nn_model_, &operand_type));
     int ann_operand = operand_mapping_->add_new_non_tensor_operand();
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context_, ANeuralNetworksModel_setOperandValue(
+        context_, nnapi_->ANeuralNetworksModel_setOperandValue(
                       nn_model_, ann_operand, values, sizeof(T) * num_values));
     augmented_inputs_.push_back(ann_operand);
     return kTfLiteOk;
   }
 
+  // Access to NNAPI.
+  const NnApi* const nnapi_;
+
   // TfLiteContext for error handling.
   TfLiteContext* const context_;
 
@@ -392,7 +384,7 @@ ANeuralNetworksOperationType BasicMappingFn(
 // The kernel that represents the node sub set of TF Lite being run on NN API.
 class NNAPIDelegateKernel {
  public:
-  NNAPIDelegateKernel() = default;
+  NNAPIDelegateKernel() { nnapi_ = NnApiImplementation(); }
 
   typedef ANeuralNetworksOperationType (*MappingFn)(
       const NNAPIOpMappingArgs& mapping_args);
@@ -401,7 +393,7 @@ class NNAPIDelegateKernel {
   // when called. You can use this function to see if a node is supported
   // (i.e. that MappingFn is not nullptr).
   static MappingFn Map(TfLiteContext* context, int builtin_code, int version,
-                       TfLiteNode* node) {
+                       int android_sdk_version, TfLiteNode* node) {
     switch (builtin_code) {
       case kTfLiteBuiltinAdd:
         if (version == 1) {
@@ -520,7 +512,7 @@ class NNAPIDelegateKernel {
         }
         break;
       case kTfLiteBuiltinSqueeze:
-        if (version == 1 && kAndroidSdkVersion >= kMinSdkVersionForNNAPI11) {
+        if (version == 1 && android_sdk_version >= kMinSdkVersionForNNAPI11) {
           return [](const NNAPIOpMappingArgs& mapping_args)
                      -> ANeuralNetworksOperationType {
             auto builtin = reinterpret_cast<TfLiteSqueezeParams*>(
@@ -636,7 +628,7 @@ class NNAPIDelegateKernel {
         }
         break;
       case kTfLiteBuiltinSub:
-        if (version == 1 && kAndroidSdkVersion >= kMinSdkVersionForNNAPI11 &&
+        if (version == 1 && android_sdk_version >= kMinSdkVersionForNNAPI11 &&
             context->tensors[node->inputs->data[0]].type == kTfLiteFloat32) {
           // NNAPI only support float sub.
           return [](const NNAPIOpMappingArgs& mapping_args)
@@ -649,7 +641,7 @@ class NNAPIDelegateKernel {
         }
         break;
       case kTfLiteBuiltinDiv:
-        if (version == 1 && kAndroidSdkVersion >= kMinSdkVersionForNNAPI11 &&
+        if (version == 1 && android_sdk_version >= kMinSdkVersionForNNAPI11 &&
             context->tensors[node->inputs->data[0]].type == kTfLiteFloat32) {
           // NNAPI only support float div.
           return [](const NNAPIOpMappingArgs& mapping_args)
@@ -662,7 +654,7 @@ class NNAPIDelegateKernel {
         }
         break;
       case kTfLiteBuiltinPad:
-        if (version == 1 && kAndroidSdkVersion >= kMinSdkVersionForNNAPI11 &&
+        if (version == 1 && android_sdk_version >= kMinSdkVersionForNNAPI11 &&
             node->inputs->size == 2 &&
             context->tensors[node->inputs->data[0]].type == kTfLiteFloat32) {
           // NNAPI does not support specifying the padding value.
@@ -672,12 +664,12 @@ class NNAPIDelegateKernel {
         }
         break;
       case kTfLiteBuiltinSpaceToBatchNd:
-        if (version == 1 && kAndroidSdkVersion >= kMinSdkVersionForNNAPI11) {
+        if (version == 1 && android_sdk_version >= kMinSdkVersionForNNAPI11) {
           return BasicMappingFn<ANEURALNETWORKS_SPACE_TO_BATCH_ND>;
         }
         break;
       case kTfLiteBuiltinStridedSlice:
-        if (version == 1 && kAndroidSdkVersion >= kMinSdkVersionForNNAPI11) {
+        if (version == 1 && android_sdk_version >= kMinSdkVersionForNNAPI11) {
           return [](const NNAPIOpMappingArgs& mapping_args)
                      -> ANeuralNetworksOperationType {
             auto builtin = reinterpret_cast<TfLiteStridedSliceParams*>(
@@ -695,7 +687,7 @@ class NNAPIDelegateKernel {
         // dimensions.
         // TODO(b/110888333): Support dynamically-sized tensors in delegates.
         if ((version == 1) &&
-            (kAndroidSdkVersion >= kMinSdkVersionForNNAPI11) &&
+            (android_sdk_version >= kMinSdkVersionForNNAPI11) &&
             (node->inputs->size > 1) &&
             (context->tensors[node->inputs->data[1]].allocation_type ==
              kTfLiteMmapRo)) {
@@ -793,7 +785,7 @@ class NNAPIDelegateKernel {
         break;
       case kTfLiteBuiltinMean:
         // NNAPI does not support generating a scalar as output for MEAN.
-        if (version == 1 && kAndroidSdkVersion >= kMinSdkVersionForNNAPI11 &&
+        if (version == 1 && android_sdk_version >= kMinSdkVersionForNNAPI11 &&
             context->tensors[node->inputs->data[0]].type == kTfLiteFloat32 &&
             context->tensors[node->outputs->data[0]].dims->size > 0) {
           return [](const NNAPIOpMappingArgs& mapping_args)
@@ -837,8 +829,8 @@ class NNAPIDelegateKernel {
 
     if (!nn_model_) {
       ANeuralNetworksModel* model;
-      RETURN_TFLITE_ERROR_IF_NN_ERROR(context,
-                                      ANeuralNetworksModel_create(&model));
+      RETURN_TFLITE_ERROR_IF_NN_ERROR(
+          context, nnapi_->ANeuralNetworksModel_create(&model));
       nn_model_.reset(model);
 
       TF_LITE_ENSURE_STATUS(
@@ -848,10 +840,10 @@ class NNAPIDelegateKernel {
     if (!nn_compilation_) {
       ANeuralNetworksCompilation* compilation;
       RETURN_TFLITE_ERROR_IF_NN_ERROR(
-          context,
-          ANeuralNetworksCompilation_create(nn_model_.get(), &compilation));
+          context, nnapi_->ANeuralNetworksCompilation_create(nn_model_.get(),
+                                                             &compilation));
       RETURN_TFLITE_ERROR_IF_NN_ERROR(
-          context, ANeuralNetworksCompilation_finish(compilation));
+          context, nnapi_->ANeuralNetworksCompilation_finish(compilation));
       nn_compilation_.reset(compilation);
     }
     return kTfLiteOk;
@@ -860,8 +852,8 @@ class NNAPIDelegateKernel {
   TfLiteStatus Invoke(TfLiteContext* context, TfLiteNode* node) {
     ANeuralNetworksExecution* execution = nullptr;
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context,
-        ANeuralNetworksExecution_create(nn_compilation_.get(), &execution));
+        context, nnapi_->ANeuralNetworksExecution_create(nn_compilation_.get(),
+                                                         &execution));
 
     // Set the input tensor buffers. Note: we access tflite tensors using
     // absolute indices but NN api indices inputs by relative indices.
@@ -881,7 +873,7 @@ class NNAPIDelegateKernel {
                tensor->data.raw, tensor->bytes);
         RETURN_TFLITE_ERROR_IF_NN_ERROR(
             context,
-            ANeuralNetworksExecution_setInputFromMemory(
+            nnapi_->ANeuralNetworksExecution_setInputFromMemory(
                 execution, relative_input_index, nullptr,
                 nn_input_memory_->get_handle(), input_offset, tensor->bytes));
         input_offset += tensor->bytes;
@@ -896,7 +888,7 @@ class NNAPIDelegateKernel {
       TfLiteTensor* tensor = &context->tensors[output_index];
       RETURN_TFLITE_ERROR_IF_NN_ERROR(
           context,
-          ANeuralNetworksExecution_setOutputFromMemory(
+          nnapi_->ANeuralNetworksExecution_setOutputFromMemory(
               execution, relative_output_index, nullptr,
               nn_output_memory_->get_handle(), output_offset, tensor->bytes));
       output_offset += tensor->bytes;
@@ -912,7 +904,7 @@ class NNAPIDelegateKernel {
       // reading and writing into the same buffer during a invocation.
       // TODO(110369471): using double shared buffer to minimize the copies.
       RETURN_TFLITE_ERROR_IF_NN_ERROR(
-          context, ANeuralNetworksExecution_setOutput(
+          context, nnapi_->ANeuralNetworksExecution_setOutput(
                        execution, relative_output_index, nullptr,
                        tensor->data.raw, tensor->bytes));
       relative_output_index++;
@@ -920,10 +912,12 @@ class NNAPIDelegateKernel {
     // Invoke ANN in blocking fashion.
     ANeuralNetworksEvent* event = nullptr;
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context, ANeuralNetworksExecution_startCompute(execution, &event));
-    RETURN_TFLITE_ERROR_IF_NN_ERROR(context, ANeuralNetworksEvent_wait(event));
-    ANeuralNetworksEvent_free(event);
-    ANeuralNetworksExecution_free(execution);
+        context,
+        nnapi_->ANeuralNetworksExecution_startCompute(execution, &event));
+    RETURN_TFLITE_ERROR_IF_NN_ERROR(context,
+                                    nnapi_->ANeuralNetworksEvent_wait(event));
+    nnapi_->ANeuralNetworksEvent_free(event);
+    nnapi_->ANeuralNetworksExecution_free(execution);
 
     // copy results from shared memory to the destination.
     output_offset = 0;
@@ -938,6 +932,8 @@ class NNAPIDelegateKernel {
   }
 
  private:
+  // Access to NNApi.
+  const NnApi* nnapi_;
   // ANN API state.
   std::unique_ptr<ANeuralNetworksModel, NNFreeModel> nn_model_;
   std::unique_ptr<ANeuralNetworksCompilation, NNFreeCompilation>
@@ -958,7 +954,7 @@ class NNAPIDelegateKernel {
     // The operand builder allows creating a single op. We create it at this
     // reduced power position rather than in the for loop to avoid reallocating
     // the vectors.
-    NNAPIOpBuilder builder(context, &operand_mapping_, nn_model_.get());
+    NNAPIOpBuilder builder(nnapi_, context, &operand_mapping_, nn_model_.get());
     // Add Tensors
     // allocate outside to avoid realloc
     for (auto node_index : nodes_) {
@@ -981,9 +977,10 @@ class NNAPIDelegateKernel {
         }
       }
       // Get op type and operands
-      int nn_op_type = Map(context, reg->builtin_code, reg->version, node)(
-          {context, &builder, node, &model_state_outputs_,
-           &model_state_tfl_inputs_});
+      int nn_op_type = Map(
+          context, reg->builtin_code, reg->version, nnapi_->android_sdk_version,
+          node)({context, &builder, node, &model_state_outputs_,
+                 &model_state_tfl_inputs_});
       // Map outputs to NN API tensor indices.
       for (auto output_index : TfLiteIntArrayView(node->outputs)) {
         TF_LITE_ENSURE_STATUS(builder.AddTensorOutput(output_index));
@@ -1029,25 +1026,27 @@ class NNAPIDelegateKernel {
 
     // Tell ANN to declare inputs/outputs
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context, ANeuralNetworksModel_identifyInputsAndOutputs(
+        context, nnapi_->ANeuralNetworksModel_identifyInputsAndOutputs(
                      nn_model_.get(), inputs.size(), inputs.data(),
                      outputs.size(), outputs.data()));
 
     // Set relaxed computation mode for fp32 if possible.
-    if (kAndroidSdkVersion >= kMinSdkVersionForNNAPI11) {
+    if (nnapi_->android_sdk_version >= kMinSdkVersionForNNAPI11) {
       RETURN_TFLITE_ERROR_IF_NN_ERROR(
-          context, ANeuralNetworksModel_relaxComputationFloat32toFloat16(
-                       nn_model_.get(), context->allow_fp32_relax_to_fp16));
+          context,
+          nnapi_->ANeuralNetworksModel_relaxComputationFloat32toFloat16(
+              nn_model_.get(), context->allow_fp32_relax_to_fp16));
     }
 
     // Finalize the model
     RETURN_TFLITE_ERROR_IF_NN_ERROR(
-        context, ANeuralNetworksModel_finish(nn_model_.get()));
+        context, nnapi_->ANeuralNetworksModel_finish(nn_model_.get()));
 
     // Create shared memory pool for inputs and outputs.
-    nn_input_memory_.reset(new NNMemory("input_pool", total_input_byte_size));
+    nn_input_memory_.reset(
+        new NNMemory(nnapi_, "input_pool", total_input_byte_size));
     nn_output_memory_.reset(
-        new NNMemory("output_pool", total_output_byte_size));
+        new NNMemory(nnapi_, "output_pool", total_output_byte_size));
 
     return kTfLiteOk;
   }
@@ -1063,7 +1062,9 @@ TfLiteDelegate* NnApiDelegate() {
       .Prepare = [](TfLiteContext* context,
                     TfLiteDelegate* delegate) -> TfLiteStatus {
         // Do not check nodes_ if NN API is unavailable.
-        if (kAndroidSdkVersion < kMinSdkVersionForNNAPI || !NNAPIExists()) {
+        const NnApi* nnapi = NnApiImplementation();
+        if (nnapi->android_sdk_version < kMinSdkVersionForNNAPI ||
+            !nnapi->nnapi_exists) {
           return kTfLiteOk;
         }
 
@@ -1076,6 +1077,7 @@ TfLiteDelegate* NnApiDelegate() {
         TfLiteIntArray* plan;
         TF_LITE_ENSURE_STATUS(context->GetExecutionPlan(context, &plan));
 
+        int android_sdk_version = NnApiImplementation()->android_sdk_version;
         // Check for every node if it is supported
         // TODO(b/80625235): Fix this to do more careful checking of versioning.
         for (int node_index : TfLiteIntArrayView(plan)) {
@@ -1084,7 +1086,8 @@ TfLiteDelegate* NnApiDelegate() {
           TF_LITE_ENSURE_STATUS(context->GetNodeAndRegistration(
               context, node_index, &node, &registration));
           if (NNAPIDelegateKernel::Map(context, registration->builtin_code,
-                                       registration->version, node)) {
+                                       registration->version,
+                                       android_sdk_version, node)) {
             supported_nodes.push_back(node_index);
           }
         }
diff --git a/tensorflow/lite/nnapi/BUILD b/tensorflow/lite/nnapi/BUILD
index bd3a8a69af..172d0c8758 100644
--- a/tensorflow/lite/nnapi/BUILD
+++ b/tensorflow/lite/nnapi/BUILD
@@ -12,3 +12,26 @@ cc_library(
     ],
     linkopts = ["-ldl"],
 )
+
+cc_library(
+    name = "nnapi_implementation",
+    srcs = [
+        "nnapi_implementation.cc",
+    ],
+    hdrs = [
+        "nnapi_implementation.h",
+    ],
+    linkopts = ["-ldl"],
+    deps = [
+        "//tensorflow/lite/nnapi:nnapi_lib",
+    ],
+)
+
+cc_test(
+    name = "nnapi_implementation_test",
+    srcs = ["nnapi_implementation_test.cc"],
+    deps = [
+        "//tensorflow/lite/nnapi:nnapi_implementation",
+        "@com_google_googletest//:gtest_main",
+    ],
+)
diff --git a/tensorflow/lite/nnapi/NeuralNetworksShim.h b/tensorflow/lite/nnapi/NeuralNetworksShim.h
index 2ce6e50de6..3a4e15006e 100644
--- a/tensorflow/lite/nnapi/NeuralNetworksShim.h
+++ b/tensorflow/lite/nnapi/NeuralNetworksShim.h
@@ -22,6 +22,11 @@ limitations under the License.
 
 #include "tensorflow/lite/nnapi/NeuralNetworksTypes.h"
 
+// This interface is now deprecated. You should use instead
+// nnapi_implementation.
+
+// TODO(b/123017568): Update all current usages of this file.
+
 // helpers
 
 #define NNAPI_LOG(format, ...) fprintf(stderr, format "\n", __VA_ARGS__);
diff --git a/tensorflow/lite/nnapi/nnapi_implementation.cc b/tensorflow/lite/nnapi/nnapi_implementation.cc
new file mode 100644
index 0000000000..5c96957f4a
--- /dev/null
+++ b/tensorflow/lite/nnapi/nnapi_implementation.cc
@@ -0,0 +1,137 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/nnapi/nnapi_implementation.h"
+
+#include <cstdlib>
+
+#ifdef __ANDROID__
+#include <dlfcn.h>
+#include <sys/mman.h>
+#include <sys/system_properties.h>
+#include <unistd.h>
+#endif
+
+#define NNAPI_LOG(format, ...) fprintf(stderr, format "\n", __VA_ARGS__);
+
+namespace {
+
+#ifdef __ANDROID__
+int32_t GetAndroidSdkVersion() {
+  const char* sdkProp = "ro.build.version.sdk";
+  char sdkVersion[PROP_VALUE_MAX];
+  int length = __system_property_get(sdkProp, sdkVersion);
+  if (length != 0) {
+    int32_t result = 0;
+    for (int i = 0; i < length; ++i) {
+      int digit = sdkVersion[i] - '0';
+      if (digit < 0 || digit > 9) {
+        // Non-numeric SDK version, assume it's higher than expected;
+        return 0xffff;
+      }
+      result = result * 10 + digit;
+    }
+    return result;
+  }
+  return 0;
+}
+
+void* LoadFunction(void* handle, const char* name) {
+  if (handle == nullptr) {
+    return nullptr;
+  }
+  void* fn = dlsym(handle, name);
+  if (fn == nullptr) {
+    NNAPI_LOG("nnapi error: unable to open function %s", name);
+  }
+  return fn;
+}
+
+#define LOAD_FUNCTION(handle, name) \
+  nnapi.name = reinterpret_cast<name##_fn>(LoadFunction(handle, #name));
+
+#else
+
+#define LOAD_FUNCTION(handle, name) nnapi.name = nullptr;
+
+#endif
+
+const NnApi LoadNnApi() {
+  NnApi nnapi = {};
+
+#ifdef __ANDROID__
+  void* libneuralnetworks = nullptr;
+  void* libandroid = nullptr;
+  nnapi.android_sdk_version = GetAndroidSdkVersion();
+  if (nnapi.android_sdk_version < 27) {
+    NNAPI_LOG("nnapi error: requires android sdk version to be at least %d",
+              27);
+  } else {
+    // TODO: change RTLD_LOCAL? Assumes there can be multiple instances of nn
+    // api RT
+    libneuralnetworks = dlopen("libneuralnetworks.so", RTLD_LAZY | RTLD_LOCAL);
+    if (libneuralnetworks == nullptr) {
+      NNAPI_LOG("nnapi error: unable to open library %s",
+                "libneuralnetworks.so");
+    }
+    libandroid = dlopen("libandroid.so", RTLD_LAZY | RTLD_LOCAL);
+    if (libandroid == nullptr) {
+      NNAPI_LOG("nnapi error: unable to open library %s", "libandroid.so");
+    }
+  }
+  nnapi.nnapi_exists = libneuralnetworks != nullptr;
+#else
+  nnapi.nnapi_exists = false;
+  nnapi.android_sdk_version = 0;
+#endif
+
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksMemory_createFromFd);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksMemory_free);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksModel_create);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksModel_free);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksModel_finish);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksModel_addOperand);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksModel_setOperandValue);
+  LOAD_FUNCTION(libneuralnetworks,
+                ANeuralNetworksModel_setOperandValueFromMemory);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksModel_addOperation);
+  LOAD_FUNCTION(libneuralnetworks,
+                ANeuralNetworksModel_identifyInputsAndOutputs);
+  LOAD_FUNCTION(libneuralnetworks,
+                ANeuralNetworksModel_relaxComputationFloat32toFloat16);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksCompilation_create);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksCompilation_free);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksCompilation_setPreference);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksCompilation_finish);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksExecution_create);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksExecution_free);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksExecution_setInput);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksExecution_setInputFromMemory);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksExecution_setOutput);
+  LOAD_FUNCTION(libneuralnetworks,
+                ANeuralNetworksExecution_setOutputFromMemory);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksExecution_startCompute);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksEvent_wait);
+  LOAD_FUNCTION(libneuralnetworks, ANeuralNetworksEvent_free);
+  LOAD_FUNCTION(libandroid, ASharedMemory_create);
+
+  return nnapi;
+}
+
+}  // namespace
+
+const NnApi* NnApiImplementation() {
+  static const NnApi nnapi = LoadNnApi();
+  return &nnapi;
+}
diff --git a/tensorflow/lite/nnapi/nnapi_implementation.h b/tensorflow/lite/nnapi/nnapi_implementation.h
new file mode 100644
index 0000000000..82d7cc75c1
--- /dev/null
+++ b/tensorflow/lite/nnapi/nnapi_implementation.h
@@ -0,0 +1,582 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_LITE_NNAPI_NNAPI_IMPLEMENTATION_H_
+#define TENSORFLOW_LITE_NNAPI_NNAPI_IMPLEMENTATION_H_
+
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "tensorflow/lite/nnapi/NeuralNetworksTypes.h"
+
+struct NnApi {
+  bool nnapi_exists;
+  int32_t android_sdk_version;
+
+  /**
+   * Creates a shared memory object from a file descriptor.
+   *
+   * The shared memory is backed by a file descriptor via mmap.
+   * See {@link ANeuralNetworksMemory} for a description on how to use
+   * this shared memory.
+   *
+   * @param size The requested size in bytes.
+   *             Must not be larger than the file size.
+   * @param prot The desired memory protection for the mapping.
+   *             It is either PROT_NONE or the bitwise OR of one or
+   *             more of the following flags: PROT_READ, PROT_WRITE.
+   * @param fd The requested file descriptor.
+   *           The file descriptor has to be mmap-able. The file
+   *           descriptor will be duplicated.
+   * @param offset The offset to the beginning of the file of the area to map.
+   *               The offset has to be aligned to a page size.
+   * @param memory The memory object to be created.
+   *               Set to NULL if unsuccessful.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if the request completed normally.
+   */
+  int (*ANeuralNetworksMemory_createFromFd)(size_t size, int protect, int fd,
+                                            size_t offset,
+                                            ANeuralNetworksMemory** memory);
+
+  /**
+   * Delete a memory object.
+   *
+   * Destroys the object used by the run time to keep track of the memory.
+   * This will free the underlying actual memory if no other code has open
+   * handles to this memory.
+   *
+   * @param memory The memory object to be freed.
+   */
+  void (*ANeuralNetworksMemory_free)(ANeuralNetworksMemory* memory);
+
+  /**
+   * Create an empty {@link ANeuralNetworksModel}.
+   *
+   * <p>This only creates the object. Computation is performed once
+   * {@link ANeuralNetworksExecution_startCompute} is invoked.
+   *
+   * The model should be constructed with calls to
+   * {@link ANeuralNetworksModel_addOperation} and
+   * {@link ANeuralNetworksModel_addOperand}
+   *
+   * <p>{@link ANeuralNetworksModel_finish} should be called once the model
+   * has been fully constructed.</p>
+   *
+   * <p>{@link ANeuralNetworksModel_free} should be called once the model
+   * is no longer needed.</p>
+   *
+   * @param model The {@link ANeuralNetworksModel} to be created.
+   *              Set to NULL if unsuccessful.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful.
+   */
+  int (*ANeuralNetworksModel_create)(ANeuralNetworksModel** model);
+
+  /**
+   * Destroy a model.
+   *
+   * The model need not have been finished by a call to
+   * {@link ANeuralNetworksModel_finish}.
+   *
+   * See {@link ANeuralNetworksModel} for information on multithreaded usage.
+   *
+   * @param model The model to be destroyed. Passing NULL is acceptable and
+   *              results in no operation.
+   */
+  void (*ANeuralNetworksModel_free)(ANeuralNetworksModel* model);
+
+  /**
+   * Indicate that we have finished modifying a model. Required before
+   * calling {@link ANeuralNetworksCompilation_compile}.
+   *
+   * An application is responsible to make sure that no other thread uses
+   * the model at the same time.
+   *
+   * See {@link ANeuralNetworksModel} for information on multithreaded usage.
+   *
+   * @param model The model to be finished.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful.
+   */
+  int (*ANeuralNetworksModel_finish)(ANeuralNetworksModel* model);
+
+  /**
+   * Add an operand to a model.
+   *
+   * The order in which the operands are added is important. The first one added
+   * to a model will have the index value 0, the second 1, etc. These indexes
+   * are used as operand identifiers in
+   * {@link ANeuralNetworksModel_addOperation},
+   * {@link ANeuralNetworksExecution_setInput},
+   * {@link ANeuralNetworksExecution_setInputFromMemory},
+   * {@link ANeuralNetworksExecution_setOutput},
+   * {@link ANeuralNetworksExecution_setOutputFromMemory} and
+   * {@link ANeuralNetworksExecution_setOperandValue}.
+   *
+   * To build a model that can accommodate inputs of various sizes, as you may
+   * want to do for a CNN, set the size of the dimensions that will vary at run
+   * time to 0. If you do so, provide the full dimensions when calling
+   * {@link ANeuralNetworksExecution_setInput} or {@link
+   * ANeuralNetworksExecution_setInputFromMemory}.
+   *
+   * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
+   * been called will return an error.
+   *
+   * See {@link ANeuralNetworksModel} for information on multithreaded usage.
+   *
+   * @param model The model to be modified.
+   * @param type The {@link ANeuralNetworksOperandType} that describes the shape
+   * of the operand.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful.
+   */
+  int (*ANeuralNetworksModel_addOperand)(
+      ANeuralNetworksModel* model, const ANeuralNetworksOperandType* type);
+
+  /**
+   * Sets an operand to a constant value.
+   *
+   * For scalar values, the content of buffer is copied into the model.
+   *
+   * For tensor values, a pointer to the buffer is stored within the model.
+   * The application is responsible for not changing the content of this region
+   * until all executions using this model have completed. As the data may
+   * be copied during processing, modifying the data after this call yields
+   * undefined results.
+   *
+   * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
+   * been called will return an error.
+   *
+   * See {@link ANeuralNetworksModel} for information on multithreaded usage.
+   *
+   * @param model The model to be modified.
+   * @param index The index of the model operand we're setting.
+   * @param buffer A pointer to the data to use.
+   * @param length The size in bytes of the data value.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful.
+   */
+  int (*ANeuralNetworksModel_setOperandValue)(ANeuralNetworksModel* model,
+                                              int32_t index, const void* buffer,
+                                              size_t length);
+
+  /**
+   * Sets an operand to a value stored in a memory object.
+   *
+   * The content of the memory is not copied. A reference to that memory is
+   * stored inside the model. The application is responsible for not changing
+   * the content of the memory region until all executions using this model have
+   * completed.
+   * As the data may be copied during processing, modifying the data after this
+   * call yields undefined results.
+   *
+   * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
+   * been called will return an error.
+   *
+   * See {@link ANeuralNetworksModel} for information on multithreaded usage.
+   *
+   * @param model The model to be modified.
+   * @param index The index of the model operand we're setting.
+   * @param buffer A pointer to the data to use.
+   * @param memory The memory containing the data.
+   * @param offset This specifies the location of the data within the memory.
+   *               The offset is in bytes from the start of memory.
+   * @param length The size in bytes of the data value.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful.
+   */
+  int (*ANeuralNetworksModel_setOperandValueFromMemory)(
+      ANeuralNetworksModel* model, int32_t index,
+      const ANeuralNetworksMemory* memory, size_t offset, size_t length);
+
+  /**
+   * Add an operation to a model.
+   *
+   * @param model The model to be modified.
+   * @param type The type of the operation.
+   * @param inputCount The number of entries in the inputs array.
+   * @param inputs An array of indexes identifying each operand.
+   * @param outputCount The number of entries in the outputs array.
+   * @param outputs An array of indexes identifying each operand.
+   *
+   * The operands specified by inputs and outputs must have been
+   * previously added by calls to {@link ANeuralNetworksModel_addOperand}.
+   *
+   * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
+   * been called will return an error.
+   *
+   * See {@link ANeuralNetworksModel} for information on multithreaded usage.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful.
+   */
+  int (*ANeuralNetworksModel_addOperation)(ANeuralNetworksModel* model,
+                                           ANeuralNetworksOperationType type,
+                                           uint32_t inputCount,
+                                           const uint32_t* inputs,
+                                           uint32_t outputCount,
+                                           const uint32_t* outputs);
+
+  /**
+   * Specifies which operands will be the model's inputs and outputs.
+   *
+   * An operand cannot be used for both input and output. Doing so will
+   * return an error.
+   *
+   * @param model The model to be modified.
+   * @param inputCount The number of entries in the inputs array.
+   * @param inputs An array of indexes identifying the input operands.
+   * @param outputCount The number of entries in the outputs array.
+   * @param outputs An array of indexes identifying the output operands.
+   *
+   * The operands specified by inputs and outputs must have been
+   * previously added by calls to {@link ANeuralNetworksModel_addOperand}.
+   *
+   * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
+   * been called will return an error.
+   *
+   * See {@link ANeuralNetworksModel} for information on multithreaded usage.
+   *
+   */
+  int (*ANeuralNetworksModel_identifyInputsAndOutputs)(
+      ANeuralNetworksModel* model, uint32_t inputCount, const uint32_t* inputs,
+      uint32_t outputCount, const uint32_t* outputs);
+
+  /**
+   * Specifies whether {@link ANEURALNETWORKS_TENSOR_FLOAT32} is allowed to be
+   * calculated with range and/or precision as low as that of the
+   * IEEE 754 16-bit floating-point format. By default,
+   * {@link ANEURALNETWORKS_TENSOR_FLOAT32} must be calculated using at least
+   * the range and precision of the IEEE 754 32-bit floating-point format.
+   *
+   * @param model The model to be modified.
+   * @param allow 'true' indicates {@link ANEURALNETWORKS_TENSOR_FLOAT32} may be
+   *              calculated with range and/or precision as low as that of the
+   *              IEEE 754 16-bit floating point format. 'false' indicates
+   *              {@link ANEURALNETWORKS_TENSOR_FLOAT32} must be calculated
+   *              using at least the range and precision of the IEEE 754 32-bit
+   *              floating point format.
+   *
+   * Attempting to modify a model once {@link ANeuralNetworksModel_finish} has
+   * been called will return an error.
+   *
+   * Available since API level 28.
+   *
+   * See {@link ANeuralNetworksModel} for information on multithreaded usage.
+   */
+  int (*ANeuralNetworksModel_relaxComputationFloat32toFloat16)(
+      ANeuralNetworksModel* model, bool allow);
+
+  /**
+   * Create a {@link ANeuralNetworksCompilation} to compile the given model.
+   * This only creates the object. Compilation is only performed once
+   * {@link ANeuralNetworksCompilation_start} is invoked.
+   *
+   * <p>The provided model must outlive the compilation.</p>
+   *
+   * The model must already have been finished by a call to
+   * {@link ANeuralNetworksModel_finish}.
+   *
+   * See {@link ANeuralNetworksCompilation} for information on multithreaded
+   * usage.
+   *
+   * @param model The {@link ANeuralNetworksModel} to be compiled.
+   * @param compilation The newly created object or NULL if unsuccessful.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA
+   *         if the model is invalid.
+   */
+  int (*ANeuralNetworksCompilation_create)(
+      ANeuralNetworksModel* model, ANeuralNetworksCompilation** compilation);
+
+  /**
+   * Destroy a compilation.
+   *
+   * <p>If called on a compilation for which
+   * {@link ANeuralNetworksCompilation_start} has been called, the
+   * function will return immediately but will mark the compilation to be
+   * deleted once the compilation completes. The
+   * {@link ANeuralNetworksCompilation_wait} will return ERROR_DELETED.
+   *
+   * See {@link ANeuralNetworksCompilation} for information on multithreaded
+   * usage.
+   *
+   * @param compilation The compilation to be destroyed. Passing NULL is
+   * acceptable and results in no operation.
+   */
+  void (*ANeuralNetworksCompilation_free)(
+      ANeuralNetworksCompilation* compilation);
+
+  /**
+   * Sets the execution preference.
+   *
+   * <p>Provides guidance to the runtime when trade-offs are possible.</p>
+   *
+   * See {@link ANeuralNetworksCompilation} for information on multithreaded
+   * usage.
+   *
+   * @param compilation The compilation to be modified.
+   * @param preference Either {@link PREFER_LOW_POWER},
+   *                  {@link PREFER_SINGLE_FAST_ANSWER}, or
+   *                  {@link PREFER_SUSTAINED_SPEED}.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful.
+   */
+  int (*ANeuralNetworksCompilation_setPreference)(
+      ANeuralNetworksCompilation* compilation, int32_t preference);
+
+  /**
+   * Waits until the compilation completes.
+   *
+   * More than one thread can wait on a compilation. When the compilation
+   * completes, all threads will be released.
+   *
+   * See {@link ANeuralNetworksCompilation} for information on multithreaded
+   * usage.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if the compilation completed normally.
+   */
+  int (*ANeuralNetworksCompilation_finish)(
+      ANeuralNetworksCompilation* compilation);
+
+  /**
+   * Create a {@link ANeuralNetworksExecution} to apply the given compilation.
+   * This only creates the object. Computation is only performed once
+   * {@link ANeuralNetworksExecution_startCompute} is invoked.
+   *
+   * <p>The provided compilation must outlive the execution.</p>
+   *
+   * See {@link ANeuralNetworksExecution} for information on multithreaded
+   * usage.
+   *
+   * @param compilation The {@link ANeuralNetworksCompilation} to be evaluated.
+   * @param execution The newly created object or NULL if unsuccessful.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA
+   *         if the compilation is invalid.
+   */
+  int (*ANeuralNetworksExecution_create)(
+      ANeuralNetworksCompilation* compilation,
+      ANeuralNetworksExecution** execution);
+
+  /**
+   * Destroy an execution.
+   *
+   * <p>If called on an execution for which
+   * {@link ANeuralNetworksExecution_startCompute} has been called, the
+   * function will return immediately but will mark the execution to be deleted
+   * once the computation completes.   The {link ANeuralNetworksExecution_wait}
+   * will return ANEURALNETWORKS_ERROR_DELETED.
+   *
+   * See {@link ANeuralNetworksExecution} for information on multithreaded
+   * usage.
+   *
+   * @param execution The execution to be destroyed. Passing NULL is acceptable
+   * and results in no operation.
+   */
+  void (*ANeuralNetworksExecution_free)(ANeuralNetworksExecution* execution);
+
+  /**
+   * Associate a user buffer with an input of the model of the
+   * {@link ANeuralNetworksExecution}.
+   *
+   * <p>The provided buffer must outlive the execution.</p>
+   *
+   * See {@link ANeuralNetworksExecution} for information on multithreaded
+   * usage.
+   *
+   * @param execution The execution to be modified.
+   * @param index The index of the input argument we are setting. It is
+   *              an index into the lists passed to
+   *              {@link ANeuralNetworksModel_identifyInputsAndOutputs}. It is
+   *              not the index associated with {@link
+   * ANeuralNetworksModel_addOperand}.
+   * @param type The type of the operand. This should be used to specify the
+   *             dimensions that were set to 0 when the operand was added to the
+   *             model. All other properties of the type must be the same as
+   *             specified in the model. If the type is the same as specified
+   *             when the model was built, NULL can be passed.
+   * @param buffer The buffer containing the data.
+   * @param length The length in bytes of the buffer.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA if
+   * the name is not recognized or the buffer is too small for the input.
+   */
+  int (*ANeuralNetworksExecution_setInput)(
+      ANeuralNetworksExecution* execution, int32_t index,
+      const ANeuralNetworksOperandType* type, const void* buffer,
+      size_t length);
+
+  /**
+   * Associate part of a memory object with an input of the model of the
+   * {@link ANeuralNetworksExecution}.
+   *
+   * <p>The provided memory must outlive the execution.</p>
+   *
+   * See {@link ANeuralNetworksExecution} for information on multithreaded
+   * usage.
+   *
+   * @param execution The execution to be modified.
+   * @param index The index of the input argument we are setting. It is
+   *              an index into the lists passed to
+   *              {@link ANeuralNetworksModel_identifyInputsAndOutputs}. It is
+   *              not the index associated with {@link
+   * ANeuralNetworksModel_addOperand}.
+   * @param type The type of the operand. This can be used to specify the
+   *             dimensions that were set to 0 when the operand was added to the
+   *             model. All other values must be the same as specified in the
+   *             model. If the type is the same as specified when the model
+   *             was built, NULL can be passed.
+   * @param memory The memory containing the data.
+   * @param offset This specifies the location of the data within the memory.
+   *               The offset is in bytes from the start of memory.
+   * @param length The size in bytes of the data value.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA if
+   * the name is not recognized or the buffer is too small for the input.
+   */
+  int (*ANeuralNetworksExecution_setInputFromMemory)(
+      ANeuralNetworksExecution* execution, int32_t index,
+      const ANeuralNetworksOperandType* type,
+      const ANeuralNetworksMemory* memory, size_t offset, size_t length);
+
+  /**
+   * Associate a user buffer with an output of the model of the
+   * {@link ANeuralNetworksExecution}.
+   *
+   * <p>The provided buffer must outlive the execution.</p>
+   *
+   * See {@link ANeuralNetworksExecution} for information on multithreaded
+   * usage.
+   *
+   * @param execution The execution to be modified.
+   * @param index The index of the output argument we are setting. It is
+   *              an index into the lists passed to
+   *              {@link ANeuralNetworksModel_identifyInputsAndOutputs}. It is
+   *              not the index associated with {@link
+   * ANeuralNetworksModel_addOperand}.
+   * @param type The type of the operand. This can be used to specify the
+   *             dimensions that were set to 0 when the operand was added to the
+   *             model. All other values must be the same as specified in the
+   *             model. If the type is the same as specified when the model
+   *             was built, NULL can be passed.
+   * @param buffer The buffer where the data is to be written.
+   * @param length The length in bytes of the buffer.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA if
+   * the name is not recognized or the buffer is too small for the output.
+   */
+  int (*ANeuralNetworksExecution_setOutput)(
+      ANeuralNetworksExecution* execution, int32_t index,
+      const ANeuralNetworksOperandType* type, void* buffer, size_t length);
+
+  /**
+   * Associate part of a memory object with an output of the model of the
+   * {@link ANeuralNetworksExecution}.
+   *
+   * <p>The provided memory must outlive the execution.</p>
+   *
+   * See {@link ANeuralNetworksExecution} for information on multithreaded
+   * usage.
+   *
+   * @param execution The execution to be modified.
+   * @param index The index of the output argument we are setting. It is
+   *              an index into the lists passed to
+   *              {@link ANeuralNetworksModel_identifyInputsAndOutputs}. It is
+   *              not the index associated with {@link
+   * ANeuralNetworksModel_addOperand}.
+   * @param type The type of the operand. This can be used to specify the
+   *             dimensions that were set to 0 when the operand was added to the
+   *             model. All other values must be the same as specified in the
+   *             model. If the type is the same as specified when the model
+   *             was built, NULL can be passed.
+   * @param memory The memory where the data is to be stored.
+   * @param offset This specifies the location of the data within the memory.
+   *               The offset is in bytes from the start of memory.
+   * @param length The length in bytes of the data value.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful, ANEURALNETWORKS_BAD_DATA if
+   * the name is not recognized or the buffer is too small for the output.
+   */
+  int (*ANeuralNetworksExecution_setOutputFromMemory)(
+      ANeuralNetworksExecution* execution, int32_t index,
+      const ANeuralNetworksOperandType* type,
+      const ANeuralNetworksMemory* memory, size_t offset, size_t length);
+
+  /**
+   * Schedule evaluation of the execution.
+   *
+   * <p>Schedules evaluation of the execution. Once the model has been
+   * applied and the outputs are ready to be consumed, the execution will be
+   * signaled. Use {@link ANeuralNetworksExecution_wait} to wait for that
+   * signal.
+   * </p>
+   *
+   * Multiple executions can be scheduled and evaluated concurrently, and
+   * compilations can be performed concurrently with executions. The runtime
+   * makes no guarantee on the ordering of the completion of compilations and
+   * executions. If it's important to the application, the application should
+   * enforce the ordering by using {@link ANeuralNetworksCompilation_wait} and
+   * {@link ANeuralNetworksExecution_wait}.
+   *
+   * ANeuralNetworksExecution_wait must be called to recuperate the resources
+   * used by the execution.
+   *
+   * See {@link ANeuralNetworksExecution} for information on multithreaded
+   * usage.
+   *
+   * @param execution The execution to be scheduled and executed.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if successful.
+   */
+  int (*ANeuralNetworksExecution_startCompute)(
+      ANeuralNetworksExecution* execution, ANeuralNetworksEvent** event);
+
+  /**
+   * Waits until the execution completes.
+   *
+   * More than one thread can wait on an event. When the execution completes,
+   * all threads will be released.
+   *
+   * See {@link ANeuralNetworksExecution} for information on multithreaded
+   * usage.
+   *
+   * @return ANEURALNETWORKS_NO_ERROR if the execution completed normally.
+   */
+  int (*ANeuralNetworksEvent_wait)(ANeuralNetworksEvent* event);
+
+  /**
+   * Destroys the event.
+   *
+   * See {@link ANeuralNetworksExecution} for information on multithreaded
+   * usage.
+   */
+  void (*ANeuralNetworksEvent_free)(ANeuralNetworksEvent* event);
+
+  // ASharedMemory_create was added in Android 8.0, so safe to use with NNAPI
+  // which was added in 8.1.
+  int (*ASharedMemory_create)(const char* name, size_t size);
+
+  /**/
+};
+
+/**
+ * Load the NNAPI implementation from the shared libraries.
+ * The NnApi structure is filled with all the pointers. If one function doesn't
+ * exist, a null pointer is stored.
+ */
+const NnApi* NnApiImplementation();
+
+#endif  // TENSORFLOW_LITE_NNAPI_NNAPI_IMPLEMENTATION_H_
diff --git a/tensorflow/lite/nnapi/nnapi_implementation_test.cc b/tensorflow/lite/nnapi/nnapi_implementation_test.cc
new file mode 100644
index 0000000000..35de769bc8
--- /dev/null
+++ b/tensorflow/lite/nnapi/nnapi_implementation_test.cc
@@ -0,0 +1,120 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/nnapi/nnapi_implementation.h"
+#include <gtest/gtest.h>
+
+namespace {
+
+TEST(NnapiLibTest, NnApiImplementation) {
+  const NnApi* nnapi = NnApiImplementation();
+  EXPECT_NE(nnapi, nullptr);
+#ifdef __ANDROID__
+  EXPECT_GT(nnapi->android_sdk_version, 0);
+  if (nnapi.android_sdk_version < 27) {
+    EXPECT_FALSE(nnapi->nnapi_exists);
+    EXPECT_EQ(nnapi->ANeuralNetworksMemory_createFromFd, nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksMemory_free, nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksModel_create, nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksModel_free, nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksModel_finish, nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksModel_addOperand, nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksModel_setOperandValue, nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksModel_setOperandValueFromMemory, nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksModel_addOperation, nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksModel_identifyInputsAndOutputs, nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksModel_relaxComputationFloat32toFloat16,
+              nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksCompilation_create, nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksCompilation_free, nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksCompilation_setPreference, nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksCompilation_finish, nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksExecution_create, nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksExecution_free, nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksExecution_setInput, nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksExecution_setInputFromMemory, nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksExecution_setOutput, nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksExecution_setOutputFromMemory, nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksExecution_startCompute, nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksEvent_wait, nullptr);
+    EXPECT_EQ(nnapi->ANeuralNetworksEvent_free, nullptr);
+    EXPECT_EQ(nnapi->ASharedMemory_create, nullptr);
+  } else {
+    EXPECT_TRUE(nnapi->nnapi_exists);
+    EXPECT_NE(nnapi->ANeuralNetworksMemory_createFromFd, nullptr);
+    EXPECT_NE(nnapi->ANeuralNetworksMemory_free, nullptr);
+    EXPECT_NE(nnapi->ANeuralNetworksModel_create, nullptr);
+    EXPECT_NE(nnapi->ANeuralNetworksModel_free, nullptr);
+    EXPECT_NE(nnapi->ANeuralNetworksModel_finish, nullptr);
+    EXPECT_NE(nnapi->ANeuralNetworksModel_addOperand, nullptr);
+    EXPECT_NE(nnapi->ANeuralNetworksModel_setOperandValue, nullptr);
+    EXPECT_NE(nnapi->ANeuralNetworksModel_setOperandValueFromMemory, nullptr);
+    EXPECT_NE(nnapi->ANeuralNetworksModel_addOperation, nullptr);
+    EXPECT_NE(nnapi->ANeuralNetworksModel_identifyInputsAndOutputs, nullptr);
+    if (nnapi->android_sdk_version >= 28) {
+      // relaxComputationFloat32toFloat16 only available with Android 9.0 (P).
+      EXPECT_NE(nnapi->ANeuralNetworksModel_relaxComputationFloat32toFloat16,
+                nullptr);
+    } else {
+      EXPECT_EQ(nnapi->ANeuralNetworksModel_relaxComputationFloat32toFloat16,
+                nullptr);
+    }
+    EXPECT_NE(nnapi->ANeuralNetworksCompilation_create, nullptr);
+    EXPECT_NE(nnapi->ANeuralNetworksCompilation_free, nullptr);
+    EXPECT_NE(nnapi->ANeuralNetworksCompilation_setPreference, nullptr);
+    EXPECT_NE(nnapi->ANeuralNetworksCompilation_finish, nullptr);
+    EXPECT_NE(nnapi->ANeuralNetworksExecution_create, nullptr);
+    EXPECT_NE(nnapi->ANeuralNetworksExecution_free, nullptr);
+    EXPECT_NE(nnapi->ANeuralNetworksExecution_setInput, nullptr);
+    EXPECT_NE(nnapi->ANeuralNetworksExecution_setInputFromMemory, nullptr);
+    EXPECT_NE(nnapi->ANeuralNetworksExecution_setOutput, nullptr);
+    EXPECT_NE(nnapi->ANeuralNetworksExecution_setOutputFromMemory, nullptr);
+    EXPECT_NE(nnapi->ANeuralNetworksExecution_startCompute, nullptr);
+    EXPECT_NE(nnapi->ANeuralNetworksEvent_wait, nullptr);
+    EXPECT_NE(nnapi->ANeuralNetworksEvent_free, nullptr);
+    EXPECT_NE(nnapi->ASharedMemory_create, nullptr);
+  }
+#else
+  EXPECT_FALSE(nnapi->nnapi_exists);
+  EXPECT_EQ(nnapi->android_sdk_version, 0);
+  EXPECT_EQ(nnapi->ANeuralNetworksMemory_createFromFd, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksMemory_free, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksModel_create, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksModel_free, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksModel_finish, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksModel_addOperand, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksModel_setOperandValue, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksModel_setOperandValueFromMemory, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksModel_addOperation, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksModel_identifyInputsAndOutputs, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksModel_relaxComputationFloat32toFloat16,
+            nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksCompilation_create, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksCompilation_free, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksCompilation_setPreference, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksCompilation_finish, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksExecution_create, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksExecution_free, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksExecution_setInput, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksExecution_setInputFromMemory, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksExecution_setOutput, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksExecution_setOutputFromMemory, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksExecution_startCompute, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksEvent_wait, nullptr);
+  EXPECT_EQ(nnapi->ANeuralNetworksEvent_free, nullptr);
+  EXPECT_EQ(nnapi->ASharedMemory_create, nullptr);
+#endif
+}
+
+}  // namespace
diff --git a/tensorflow/lite/nnapi_delegate.cc b/tensorflow/lite/nnapi_delegate.cc
index be035fe60c..5a3a21258a 100644
--- a/tensorflow/lite/nnapi_delegate.cc
+++ b/tensorflow/lite/nnapi_delegate.cc
@@ -21,7 +21,7 @@ limitations under the License.
 #include "tensorflow/lite/c/builtin_op_data.h"
 #include "tensorflow/lite/core/api/error_reporter.h"
 #include "tensorflow/lite/model.h"
-#include "tensorflow/lite/nnapi/NeuralNetworksShim.h"
+#include "tensorflow/lite/nnapi/nnapi_implementation.h"
 
 #ifdef __ANDROID__
 #include <android/log.h>
@@ -84,56 +84,27 @@ void logError(const char* format, ...) {
 static const int64_t kOperandIdNotSet = -1;
 static const int64_t kOperandNotNeeded = -2;
 
-namespace {
-
-int32_t GetAndroidSdkVersion() {
-#ifdef __ANDROID__
-  const char* sdkProp = "ro.build.version.sdk";
-  char sdkVersion[PROP_VALUE_MAX];
-  int length = __system_property_get(sdkProp, sdkVersion);
-  if (length != 0) {
-    for (int i = 0; i < length; ++i) {
-      int digit = sdkVersion[i] - '0';
-      if (digit < 0 || digit > 9) {
-        // Non-numeric SDK version, assume it's higher then expected;
-        return 0xFFFF;
-      }
-    }
-    return atoi(sdkVersion);
-  }
-  FATAL("No %s prop", sdkProp);
-#endif  // __ANDROID__
-  return 0;
-}
-
-int32_t GetAndroidSdkVersionCached() {
-  static int32_t androidSdkVersion = GetAndroidSdkVersion();
-  return androidSdkVersion;
-}
-
-}  // namespace
-
 NNAPIAllocation::NNAPIAllocation(const char* filename,
                                  ErrorReporter* error_reporter)
     : MMAPAllocation(filename, error_reporter) {
   if (mmapped_buffer_ != MAP_FAILED)
-    CHECK_NN(ANeuralNetworksMemory_createFromFd(buffer_size_bytes_, PROT_READ,
-                                                mmap_fd_, 0, &handle_));
+    CHECK_NN(NnApiImplementation()->ANeuralNetworksMemory_createFromFd(
+        buffer_size_bytes_, PROT_READ, mmap_fd_, 0, &handle_));
 }
 
 NNAPIAllocation::~NNAPIAllocation() {
   if (handle_) {
-    ANeuralNetworksMemory_free(handle_);
+    NnApiImplementation()->ANeuralNetworksMemory_free(handle_);
   }
 }
 
 NNAPIDelegate::~NNAPIDelegate() {
   if (nn_compiled_model_) {
-    ANeuralNetworksCompilation_free(nn_compiled_model_);
+    NnApiImplementation()->ANeuralNetworksCompilation_free(nn_compiled_model_);
     nn_compiled_model_ = nullptr;
   }
   if (nn_model_) {
-    ANeuralNetworksModel_free(nn_model_);
+    NnApiImplementation()->ANeuralNetworksModel_free(nn_model_);
     nn_model_ = nullptr;
     // TODO(aselle): Is this thread-safe and callable multiple times?
   }
@@ -145,6 +116,7 @@ TfLiteStatus addTensorOperands(tflite::Subgraph* subgraph,
                                ANeuralNetworksModel* nn_model,
                                uint32_t* no_of_operands_added,
                                std::vector<int64_t>* nnapi_ids) {
+  const NnApi* nnapi = NnApiImplementation();
   uint32_t next_id = 0;
   for (size_t i = 0; i < subgraph->tensors_size(); i++) {
     // Skip temporaries and RNN back-edges.
@@ -198,24 +170,24 @@ TfLiteStatus addTensorOperands(tflite::Subgraph* subgraph,
         nn_type, static_cast<uint32_t>(tensor->dims->size),
         reinterpret_cast<uint32_t*>(tensor->dims->data), scale, zeroPoint};
     RETURN_ERROR_IF_NN_FAILED(
-        ANeuralNetworksModel_addOperand(nn_model, &operand_type));
+        nnapi->ANeuralNetworksModel_addOperand(nn_model, &operand_type));
     // TODO(aselle): Based on Michael's suggestion, limiting this to read
     // only memory
     if (tensor->allocation_type == kTfLiteMmapRo) {
       if (const NNAPIAllocation* alloc = dynamic_cast<const NNAPIAllocation*>(
               static_cast<const Allocation*>(tensor->allocation))) {
         RETURN_ERROR_IF_NN_FAILED(
-            ANeuralNetworksModel_setOperandValueFromMemory(
+            nnapi->ANeuralNetworksModel_setOperandValueFromMemory(
                 nn_model, next_id, alloc->memory(),
                 alloc->offset(tensor->data.raw), tensor->bytes));
       } else {
-        RETURN_ERROR_IF_NN_FAILED(ANeuralNetworksModel_setOperandValue(
+        RETURN_ERROR_IF_NN_FAILED(nnapi->ANeuralNetworksModel_setOperandValue(
             nn_model, next_id, tensor->data.raw, tensor->bytes));
       }
     } else if (tensor->bytes == 0) {
       // These size 0 tensors are optional tensors reserved.
-      RETURN_ERROR_IF_NN_FAILED(
-          ANeuralNetworksModel_setOperandValue(nn_model, next_id, nullptr, 0));
+      RETURN_ERROR_IF_NN_FAILED(nnapi->ANeuralNetworksModel_setOperandValue(
+          nn_model, next_id, nullptr, 0));
     }
 
     ++next_id;
@@ -244,6 +216,7 @@ TfLiteStatus AddOpsAndParams(
     uint32_t next_id, std::vector<int>* model_state_inputs,
     std::vector<int>* model_state_outputs,
     const std::vector<int64_t>& tensor_id_to_nnapi_id) {
+  const NnApi* nnapi = NnApiImplementation();
   for (size_t i = 0; i < subgraph->nodes_size(); i++) {
     const auto* node_and_registration = subgraph->node_and_registration(i);
     const TfLiteNode& node = node_and_registration->first;
@@ -258,21 +231,21 @@ TfLiteStatus AddOpsAndParams(
     MapAndAddTensorIds(node.outputs->data, node.outputs->size,
                        &augmented_outputs, tensor_id_to_nnapi_id);
 
-    auto add_scalar_int32 = [&nn_model, &augmented_inputs,
+    auto add_scalar_int32 = [nnapi, &nn_model, &augmented_inputs,
                              &next_id](int value) {
       ANeuralNetworksOperandType operand_type{.type = ANEURALNETWORKS_INT32};
-      CHECK_NN(ANeuralNetworksModel_addOperand(nn_model, &operand_type))
-      CHECK_NN(ANeuralNetworksModel_setOperandValue(nn_model, next_id, &value,
-                                                    sizeof(int32_t)))
+      CHECK_NN(nnapi->ANeuralNetworksModel_addOperand(nn_model, &operand_type))
+      CHECK_NN(nnapi->ANeuralNetworksModel_setOperandValue(
+          nn_model, next_id, &value, sizeof(int32_t)))
       augmented_inputs.push_back(next_id++);
     };
 
-    auto add_scalar_float32 = [&nn_model, &augmented_inputs,
+    auto add_scalar_float32 = [nnapi, &nn_model, &augmented_inputs,
                                &next_id](float value) {
       ANeuralNetworksOperandType operand_type{.type = ANEURALNETWORKS_FLOAT32};
-      CHECK_NN(ANeuralNetworksModel_addOperand(nn_model, &operand_type))
-      CHECK_NN(ANeuralNetworksModel_setOperandValue(nn_model, next_id, &value,
-                                                    sizeof(float)))
+      CHECK_NN(nnapi->ANeuralNetworksModel_addOperand(nn_model, &operand_type))
+      CHECK_NN(nnapi->ANeuralNetworksModel_setOperandValue(
+          nn_model, next_id, &value, sizeof(float)))
       augmented_inputs.push_back(next_id++);
     };
 
@@ -281,8 +254,8 @@ TfLiteStatus AddOpsAndParams(
           .type = ANEURALNETWORKS_TENSOR_INT32,
           .dimensionCount = 1,
           .dimensions = &num_values};
-      CHECK_NN(ANeuralNetworksModel_addOperand(nn_model, &operand_type))
-      CHECK_NN(ANeuralNetworksModel_setOperandValue(
+      CHECK_NN(nnapi->ANeuralNetworksModel_addOperand(nn_model, &operand_type))
+      CHECK_NN(nnapi->ANeuralNetworksModel_setOperandValue(
           nn_model, next_id, values, sizeof(int32_t) * num_values));
       augmented_inputs.push_back(next_id++);
     };
@@ -291,15 +264,16 @@ TfLiteStatus AddOpsAndParams(
     // For each state_out tensor, a corresponding state_in operand needs to be
     // created for NNAPI.
     auto duplicate_state_tensor_float32 =
-        [subgraph, &nn_model, &next_id, &augmented_inputs, &model_state_inputs,
-         &model_state_outputs](int tensor_id) {
+        [nnapi, subgraph, &nn_model, &next_id, &augmented_inputs,
+         &model_state_inputs, &model_state_outputs](int tensor_id) {
           const TfLiteTensor* tensor = subgraph->tensor(tensor_id);
           ANeuralNetworksOperandType operand_type{
               ANEURALNETWORKS_TENSOR_FLOAT32,
               static_cast<uint32_t>(tensor->dims->size),
               reinterpret_cast<uint32_t*>(tensor->dims->data),
               tensor->params.scale, tensor->params.zero_point};
-          CHECK_NN(ANeuralNetworksModel_addOperand(nn_model, &operand_type));
+          CHECK_NN(
+              nnapi->ANeuralNetworksModel_addOperand(nn_model, &operand_type));
           augmented_inputs.push_back(next_id);
           model_state_inputs->push_back(next_id);
           model_state_outputs->push_back(tensor_id);
@@ -388,7 +362,7 @@ TfLiteStatus AddOpsAndParams(
     };
 
     // LSTM in NNAPI requires scratch tensor as an output operand.
-    auto add_lstm_scratch_tensor_float32 = [subgraph, &node, &nn_model,
+    auto add_lstm_scratch_tensor_float32 = [nnapi, subgraph, &node, &nn_model,
                                             &next_id, &augmented_outputs]() {
       if (node.temporaries->size == 0) return;
       int scratch_buffer_index = node.temporaries->data[0];
@@ -398,7 +372,7 @@ TfLiteStatus AddOpsAndParams(
           static_cast<uint32_t>(tensor->dims->size),
           reinterpret_cast<uint32_t*>(tensor->dims->data), tensor->params.scale,
           tensor->params.zero_point};
-      CHECK_NN(ANeuralNetworksModel_addOperand(nn_model, &operand_type));
+      CHECK_NN(nnapi->ANeuralNetworksModel_addOperand(nn_model, &operand_type));
       augmented_outputs.insert(augmented_outputs.begin(), next_id++);
     };
 
@@ -427,15 +401,16 @@ TfLiteStatus AddOpsAndParams(
     };
 
     // Handle optional input tensors.
-    auto add_optional_tensors = [&nn_model, &augmented_inputs,
+    auto add_optional_tensors = [nnapi, &nn_model, &augmented_inputs,
                                  &next_id](int nn_type) {
       for (size_t idx = 0; idx < augmented_inputs.size(); idx++) {
         if (augmented_inputs[idx] == kOptionalTensor) {
           const std::vector<uint32_t> dim = {0, 0};
           ANeuralNetworksOperandType operand_type{nn_type, 2, dim.data(), 0, 0};
-          CHECK_NN(ANeuralNetworksModel_addOperand(nn_model, &operand_type))
-          CHECK_NN(ANeuralNetworksModel_setOperandValue(nn_model, next_id,
-                                                        nullptr, 0))
+          CHECK_NN(
+              nnapi->ANeuralNetworksModel_addOperand(nn_model, &operand_type))
+          CHECK_NN(nnapi->ANeuralNetworksModel_setOperandValue(
+              nn_model, next_id, nullptr, 0))
           augmented_inputs[idx] = next_id++;
         }
       }
@@ -697,13 +672,13 @@ TfLiteStatus AddOpsAndParams(
         break;
     }
 
-    if (nnapi_version == 11 && GetAndroidSdkVersionCached() < 28) {
+    if (nnapi_version == 11 && nnapi->android_sdk_version < 28) {
       logError("Op %d needs NNAPI1.1", builtin);
       return kTfLiteError;
     }
 
     // Add the operation.
-    RETURN_ERROR_IF_NN_FAILED(ANeuralNetworksModel_addOperation(
+    RETURN_ERROR_IF_NN_FAILED(nnapi->ANeuralNetworksModel_addOperation(
         nn_model, nn_op_type, static_cast<uint32_t>(augmented_inputs.size()),
         augmented_inputs.data(),
         static_cast<uint32_t>(augmented_outputs.size()),
@@ -715,9 +690,10 @@ TfLiteStatus AddOpsAndParams(
 TfLiteStatus NNAPIDelegate::BuildGraph(Subgraph* subgraph) {
   if (nn_model_ && nn_compiled_model_) return model_status_;
 
+  const NnApi* nnapi = NnApiImplementation();
   // TODO(aselle): This is not correct. need to handle resize invalidation.
   if (!nn_model_) {
-    CHECK_NN(ANeuralNetworksModel_create(&nn_model_));
+    CHECK_NN(nnapi->ANeuralNetworksModel_create(&nn_model_));
 
     // Find which tensors should be added to NNAPI. TFLite has temporaries
     // and RNN back-edges which are are not valid for NNAPI. We look through all
@@ -764,21 +740,22 @@ TfLiteStatus NNAPIDelegate::BuildGraph(Subgraph* subgraph) {
                        model_states_outputs_.size(), &augmented_outputs,
                        tensor_id_to_nnapi_id);
 
-    CHECK_NN(ANeuralNetworksModel_identifyInputsAndOutputs(
+    CHECK_NN(nnapi->ANeuralNetworksModel_identifyInputsAndOutputs(
         nn_model_, static_cast<uint32_t>(augmented_inputs.size()),
         reinterpret_cast<const uint32_t*>(augmented_inputs.data()),
         static_cast<uint32_t>(augmented_outputs.size()),
         reinterpret_cast<const uint32_t*>(augmented_outputs.data())));
 
-    if (GetAndroidSdkVersionCached() >= 28) {
-      CHECK_NN(ANeuralNetworksModel_relaxComputationFloat32toFloat16(
+    if (nnapi->android_sdk_version >= 28) {
+      CHECK_NN(nnapi->ANeuralNetworksModel_relaxComputationFloat32toFloat16(
           nn_model_, subgraph->GetAllowFp16PrecisionForFp32()));
     }
-    CHECK_NN(ANeuralNetworksModel_finish(nn_model_));
+    CHECK_NN(nnapi->ANeuralNetworksModel_finish(nn_model_));
   }
   if (!nn_compiled_model_) {
-    CHECK_NN(ANeuralNetworksCompilation_create(nn_model_, &nn_compiled_model_));
-    CHECK_NN(ANeuralNetworksCompilation_finish(nn_compiled_model_));
+    CHECK_NN(nnapi->ANeuralNetworksCompilation_create(nn_model_,
+                                                      &nn_compiled_model_));
+    CHECK_NN(nnapi->ANeuralNetworksCompilation_finish(nn_compiled_model_));
   }
   return kTfLiteOk;
 }
@@ -794,8 +771,10 @@ TfLiteStatus NNAPIDelegate::Invoke(Subgraph* subgraph) {
     return model_status_;
   }
 
+  const NnApi* nnapi = NnApiImplementation();
   ANeuralNetworksExecution* execution = nullptr;
-  CHECK_NN(ANeuralNetworksExecution_create(nn_compiled_model_, &execution));
+  CHECK_NN(
+      nnapi->ANeuralNetworksExecution_create(nn_compiled_model_, &execution));
 
   // Currently perform deep copy of input buffer
   for (size_t i = 0; i < subgraph->inputs().size(); i++) {
@@ -803,7 +782,7 @@ TfLiteStatus NNAPIDelegate::Invoke(Subgraph* subgraph) {
     // TODO(aselle): Is this what we want or do we want input instead?
     // TODO(aselle): This should be called setInputValue maybe to be cons.
     TfLiteTensor* tensor = subgraph->tensor(input);
-    CHECK_NN(ANeuralNetworksExecution_setInput(
+    CHECK_NN(nnapi->ANeuralNetworksExecution_setInput(
         execution, i, nullptr, tensor->data.raw, tensor->bytes));
   }
 
@@ -811,7 +790,7 @@ TfLiteStatus NNAPIDelegate::Invoke(Subgraph* subgraph) {
   for (size_t i = 0; i < subgraph->outputs().size(); i++) {
     int output = subgraph->outputs()[i];
     TfLiteTensor* tensor = subgraph->tensor(output);
-    CHECK_NN(ANeuralNetworksExecution_setOutput(
+    CHECK_NN(nnapi->ANeuralNetworksExecution_setOutput(
         execution, i, nullptr, tensor->data.raw, tensor->bytes));
   }
 
@@ -823,21 +802,21 @@ TfLiteStatus NNAPIDelegate::Invoke(Subgraph* subgraph) {
     // Here we are using a deep copy for state_in tensors so that we are not
     // reading and writing into the same buffer during a invocation.
     // TODO(miaowang): using double shared buffer to minimize the copies.
-    CHECK_NN(ANeuralNetworksExecution_setInput(
+    CHECK_NN(nnapi->ANeuralNetworksExecution_setInput(
         execution, i + subgraph->inputs().size(), nullptr, tensor->data.raw,
         tensor->bytes));
     // Tell NNAPI where to output the state_out.
-    CHECK_NN(ANeuralNetworksExecution_setOutput(
+    CHECK_NN(nnapi->ANeuralNetworksExecution_setOutput(
         execution, i + subgraph->outputs().size(), nullptr, tensor->data.raw,
         tensor->bytes));
   }
 
   // Currently use blocking compute.
   ANeuralNetworksEvent* event = nullptr;
-  CHECK_NN(ANeuralNetworksExecution_startCompute(execution, &event));
-  CHECK_NN(ANeuralNetworksEvent_wait(event));
-  ANeuralNetworksEvent_free(event);
-  ANeuralNetworksExecution_free(execution);
+  CHECK_NN(nnapi->ANeuralNetworksExecution_startCompute(execution, &event));
+  CHECK_NN(nnapi->ANeuralNetworksEvent_wait(event));
+  nnapi->ANeuralNetworksEvent_free(event);
+  nnapi->ANeuralNetworksExecution_free(execution);
 
 #if 0
   printf("From the NN API:\n");
@@ -855,6 +834,8 @@ TfLiteStatus NNAPIDelegate::Invoke(Subgraph* subgraph) {
   return kTfLiteOk;
 }
 
-bool NNAPIDelegate::IsSupported() { return NNAPIExists(); }
+bool NNAPIDelegate::IsSupported() {
+  return NnApiImplementation()->nnapi_exists;
+}
 
 }  // namespace tflite
diff --git a/tensorflow/lite/testing/BUILD b/tensorflow/lite/testing/BUILD
index ce85a39385..3840b52a0a 100644
--- a/tensorflow/lite/testing/BUILD
+++ b/tensorflow/lite/testing/BUILD
@@ -237,7 +237,7 @@ tf_cc_binary(
     deps = [
         ":parse_testdata_lib",
         ":tflite_driver",
-        "//tensorflow/lite/nnapi:nnapi_lib",
+        "//tensorflow/lite/nnapi:nnapi_implementation",
     ],
 )
 
diff --git a/tensorflow/lite/testing/nnapi_example.cc b/tensorflow/lite/testing/nnapi_example.cc
index 0ff1fbcd41..309cb19628 100644
--- a/tensorflow/lite/testing/nnapi_example.cc
+++ b/tensorflow/lite/testing/nnapi_example.cc
@@ -26,7 +26,7 @@ limitations under the License.
 #include <iostream>
 #include <sstream>
 #include <string>
-#include "tensorflow/lite/nnapi/NeuralNetworksShim.h"
+#include "tensorflow/lite/nnapi/nnapi_implementation.h"
 #include "tensorflow/lite/testing/parse_testdata.h"
 #include "tensorflow/lite/testing/tflite_driver.h"
 
-- 
GitLab


From 4665e7096e19f18eae8b1830a0d4c1c0ffc8d46e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 18 Jan 2019 10:57:00 -0800
Subject: [PATCH 0981/1095] Add some kernel-specific DepthwiseConv profiling
 labels.

PiperOrigin-RevId: 229957162
---
 .../lite/kernels/internal/optimized/depthwiseconv_uint8.h      | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/lite/kernels/internal/optimized/depthwiseconv_uint8.h b/tensorflow/lite/kernels/internal/optimized/depthwiseconv_uint8.h
index d3dca799a7..0f4226dd15 100644
--- a/tensorflow/lite/kernels/internal/optimized/depthwiseconv_uint8.h
+++ b/tensorflow/lite/kernels/internal/optimized/depthwiseconv_uint8.h
@@ -1983,6 +1983,7 @@ inline void DepthwiseConv(
           input_shape, filter_shape, stride_width, stride_height,
           dilation_width_factor, dilation_height_factor, pad_width, pad_height,
           depth_multiplier, output_shape, output_shift)) {
+    gemmlowp::ScopedProfilingLabel specialized_label("DepthwiseConv/8bit/3x3");
     DepthwiseConv3x3Filter(params, input_shape, input_data, filter_shape,
                            filter_data, bias_shape, bias_data, output_shape,
                            output_data);
@@ -1990,6 +1991,8 @@ inline void DepthwiseConv(
   }
 #endif
 
+  gemmlowp::ScopedProfilingLabel specialized_label(
+      "DepthwiseConv/8bit/General");
   DepthwiseConvGeneral(params, input_shape, input_data, filter_shape,
                        filter_data, bias_shape, bias_data, output_shape,
                        output_data);
-- 
GitLab


From cfe98499e525da4f17e8cac494d5d128ecde4da2 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Fri, 18 Jan 2019 11:02:54 -0800
Subject: [PATCH 0982/1095] Fix build

PiperOrigin-RevId: 229958434
---
 tensorflow/tools/compatibility/renames_v2.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index 6695eb4883..9d8d25016a 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -169,6 +169,7 @@ renames = {
     'tf.distributions.Uniform': 'tf.compat.v1.distributions.Uniform',
     'tf.distributions.kl_divergence': 'tf.compat.v1.distributions.kl_divergence',
     'tf.div': 'tf.compat.v1.div',
+    'tf.dtypes.as_string': 'tf.strings.as_string',
     'tf.enable_eager_execution': 'tf.compat.v1.enable_eager_execution',
     'tf.enable_resource_variables': 'tf.compat.v1.enable_resource_variables',
     'tf.enable_v2_batch_normalization': 'tf.compat.v1.enable_v2_batch_normalization',
-- 
GitLab


From 691f6c23580c62960ffd59ca27afcf36d075946d Mon Sep 17 00:00:00 2001
From: Karim Nosir <karimnosseir@google.com>
Date: Fri, 18 Jan 2019 11:09:54 -0800
Subject: [PATCH 0983/1095] Disable smartreply tests by default.

PiperOrigin-RevId: 229959758
---
 tensorflow/lite/models/smartreply/BUILD                       | 4 ++++
 tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh | 4 ++++
 2 files changed, 8 insertions(+)

diff --git a/tensorflow/lite/models/smartreply/BUILD b/tensorflow/lite/models/smartreply/BUILD
index 9b2f0da64c..5be2aaff1f 100644
--- a/tensorflow/lite/models/smartreply/BUILD
+++ b/tensorflow/lite/models/smartreply/BUILD
@@ -57,6 +57,7 @@ tf_cc_test(
         "//tensorflow/lite/models:testdata/smartreply_samples.tsv",
         "@tflite_smartreply//:smartreply.tflite",
     ],
+    tags = ["no_oss"],
     deps = [
         ":predictor_lib",
         "//tensorflow/core:test",
@@ -71,6 +72,7 @@ tf_cc_test(
     name = "extract_feature_op_test",
     size = "small",
     srcs = ["ops/extract_feature_test.cc"],
+    tags = ["no_oss"],
     deps = [
         ":custom_ops",
         "//tensorflow/lite:framework",
@@ -85,6 +87,7 @@ tf_cc_test(
     name = "normalize_op_test",
     size = "small",
     srcs = ["ops/normalize_test.cc"],
+    tags = ["no_oss"],
     deps = [
         ":custom_ops",
         "//tensorflow/lite:framework",
@@ -99,6 +102,7 @@ tf_cc_test(
     name = "predict_op_test",
     size = "small",
     srcs = ["ops/predict_test.cc"],
+    tags = ["no_oss"],
     deps = [
         ":custom_ops",
         "//tensorflow/lite:framework",
diff --git a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
index 2eeb8dfa1a..d33b732cc5 100644
--- a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
+++ b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
@@ -125,6 +125,10 @@ fi
 
 run_configure_for_cpu_build
 
+bazel build --announce_rc --config=opt ${EXTRA_BUILD_FLAGS}  \
+  --build_tag_filters=-no_pip,-no_windows,-no_oss,-gpu \
+  tensorflow/lite:framework tensorflow/lite/examples/minimal:minimal || exit $?
+
 bazel build --announce_rc --config=opt ${EXTRA_BUILD_FLAGS} \
   tensorflow/tools/pip_package:build_pip_package \
   --incompatible_remove_native_http_archive=false || exit $?
-- 
GitLab


From 7fce32e9be8c43a96f0c55063be6ab215d12b0ae Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Fri, 18 Jan 2019 11:24:02 -0800
Subject: [PATCH 0984/1095] [TF] Add version_script option to tf_py_wrap_cc
 Bazel rule. [XLA] Hide all symbols except PyInit_* from the pywrap_xla
 module.

The XLA Python extension includes parts of TensorFlow core. If the TensorFlow symbols are exported by the XLA extension as well as TensorFlow itself, a segfault occurs if both tensorflow and jax are imported into the same Python interpreter.

JAX issue https://github.com/google/jax/issues/120 will be fixed when this change is integrated into the JAX repository.

PiperOrigin-RevId: 229962366
---
 tensorflow/compiler/xla/python/BUILD                 |  5 +++++
 .../xla/python/pywrap_xla_exported_symbols.lds       |  1 +
 .../xla/python/pywrap_xla_version_script.lds         |  6 ++++++
 tensorflow/tensorflow.bzl                            | 12 ++++++++----
 tensorflow/tf_exported_symbols.lds                   |  1 -
 tensorflow/tf_version_script.lds                     |  1 -
 6 files changed, 20 insertions(+), 6 deletions(-)
 create mode 100644 tensorflow/compiler/xla/python/pywrap_xla_exported_symbols.lds
 create mode 100644 tensorflow/compiler/xla/python/pywrap_xla_version_script.lds

diff --git a/tensorflow/compiler/xla/python/BUILD b/tensorflow/compiler/xla/python/BUILD
index ddffafa901..8276ee5a32 100644
--- a/tensorflow/compiler/xla/python/BUILD
+++ b/tensorflow/compiler/xla/python/BUILD
@@ -98,6 +98,11 @@ tf_py_wrap_cc(
         "local_computation_builder.i",
         "//tensorflow/python:platform/base.i",
     ],
+    version_script = select({
+        "//tensorflow:darwin": "pywrap_xla_exported_symbols.lds",
+        "//tensorflow:windows": None,
+        "//conditions:default": "pywrap_xla_version_script.lds",
+    }),
     deps = [
         ":local_computation_builder",
         ":numpy_bridge",
diff --git a/tensorflow/compiler/xla/python/pywrap_xla_exported_symbols.lds b/tensorflow/compiler/xla/python/pywrap_xla_exported_symbols.lds
new file mode 100644
index 0000000000..bce6c1acf8
--- /dev/null
+++ b/tensorflow/compiler/xla/python/pywrap_xla_exported_symbols.lds
@@ -0,0 +1 @@
+_PyInit__pywrap_xla
diff --git a/tensorflow/compiler/xla/python/pywrap_xla_version_script.lds b/tensorflow/compiler/xla/python/pywrap_xla_version_script.lds
new file mode 100644
index 0000000000..d31cfce7be
--- /dev/null
+++ b/tensorflow/compiler/xla/python/pywrap_xla_version_script.lds
@@ -0,0 +1,6 @@
+xla {
+  global:
+    PyInit_*;
+  local:
+    *;
+};
diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl
index 37d52cdb39..874d9e872c 100644
--- a/tensorflow/tensorflow.bzl
+++ b/tensorflow/tensorflow.bzl
@@ -1649,7 +1649,9 @@ def tf_py_wrap_cc(
         swig_includes = [],
         deps = [],
         copts = [],
+        version_script = None,
         **kwargs):
+    """Builds a Python extension module."""
     module_name = name.split("/")[-1]
 
     # Convert a rule name such as foo/bar/baz to foo/bar/_baz.so
@@ -1668,6 +1670,11 @@ def tf_py_wrap_cc(
         toolchain_deps = ["@bazel_tools//tools/cpp:current_cc_toolchain"],
         deps = deps + extra_deps,
     )
+    if not version_script:
+        version_script = select({
+            "@local_config_cuda//cuda:darwin": clean_dep("//tensorflow:tf_exported_symbols.lds"),
+            "//conditions:default": clean_dep("//tensorflow:tf_version_script.lds"),
+        })
     vscriptname = name + "_versionscript"
     _append_init_to_versionscript(
         name = vscriptname,
@@ -1676,10 +1683,7 @@ def tf_py_wrap_cc(
             "//conditions:default": True,
         }),
         module_name = module_name,
-        template_file = select({
-            "@local_config_cuda//cuda:darwin": clean_dep("//tensorflow:tf_exported_symbols.lds"),
-            "//conditions:default": clean_dep("//tensorflow:tf_version_script.lds"),
-        }),
+        template_file = version_script,
     )
     extra_linkopts = select({
         "@local_config_cuda//cuda:darwin": [
diff --git a/tensorflow/tf_exported_symbols.lds b/tensorflow/tf_exported_symbols.lds
index 9f6114f503..04632330c5 100644
--- a/tensorflow/tf_exported_symbols.lds
+++ b/tensorflow/tf_exported_symbols.lds
@@ -4,5 +4,4 @@
 *TF_*
 *TFE_*
 *nsync_*
-*pywrap_xla*
 *stream_executor*
diff --git a/tensorflow/tf_version_script.lds b/tensorflow/tf_version_script.lds
index 39d258c3b7..563d178de7 100644
--- a/tensorflow/tf_version_script.lds
+++ b/tensorflow/tf_version_script.lds
@@ -5,7 +5,6 @@ tensorflow {
     *TF_*;
     *TFE_*;
     *nsync_*;
-    *pywrap_xla*;
     *stream_executor*;
   local:
     *;
-- 
GitLab


From b58fa42c78e1441e1230b7c03ffc98a1f389493d Mon Sep 17 00:00:00 2001
From: Edward Loper <edloper@google.com>
Date: Fri, 18 Jan 2019 11:40:42 -0800
Subject: [PATCH 0985/1095] Fix CompositeDelegation handling for
 control_flow_ops such as _Enter.

PiperOrigin-RevId: 229965591
---
 .../kernel_tests/control_flow_ops_py_test.py  | 15 ++++++++++++++
 tensorflow/python/ops/control_flow_ops.py     | 20 +++++++++++++++++++
 2 files changed, 35 insertions(+)

diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
index d9d7af1f0d..90d77050e9 100644
--- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
@@ -3760,6 +3760,21 @@ class ControlFlowTest(test.TestCase):
       result = func(qint)
       self.evaluate(result)
 
+  def testSparseIdentity(self):
+    st1 = sparse_tensor.SparseTensor([[0, 5]], ['x'], [10, 10])
+    st2 = control_flow_ops._Identity(st1)
+    self.assertAllEqual(st1.indices, st2.indices)
+    self.assertAllEqual(st1.values, st2.values)
+    self.assertAllEqual(st1.dense_shape, st2.dense_shape)
+
+  def testSparseEnterExit(self):
+    st1 = sparse_tensor.SparseTensor([[0, 5]], ['x'], [10, 10])
+    st2 = control_flow_ops._Enter(st1, "foo_1")
+    st3 = control_flow_ops.exit(st2)
+    self.assertAllEqual(st1.indices, st3.indices)
+    self.assertAllEqual(st1.values, st3.values)
+    self.assertAllEqual(st1.dense_shape, st3.dense_shape)
+
 
 class ControlFlowContextCheckTest(test.TestCase):
 
diff --git a/tensorflow/python/ops/control_flow_ops.py b/tensorflow/python/ops/control_flow_ops.py
index 2bc3a9e9b6..cfdbe63c2f 100644
--- a/tensorflow/python/ops/control_flow_ops.py
+++ b/tensorflow/python/ops/control_flow_ops.py
@@ -187,6 +187,8 @@ def _Identity(data, name=None):
       return gen_array_ops.ref_identity(data, name=name)
     else:
       return array_ops.identity(data, name=name)
+  elif isinstance(data, composite_tensor.CompositeTensor):
+    return nest.map_structure(_Identity, data, expand_composites=True)
   else:
     raise TypeError("Type %s not supported" % type(data))
 
@@ -198,6 +200,8 @@ def _NextIteration(data, name=None):
       return ref_next_iteration(data, name=name)
     else:
       return next_iteration(data, name=name)
+  elif isinstance(data, composite_tensor.CompositeTensor):
+    return nest.map_structure(_NextIteration, data, expand_composites=True)
   else:
     raise TypeError("Type %s not supported" % type(data))
 
@@ -239,6 +243,11 @@ def _Enter(data,
     if use_input_shape:
       result.set_shape(data.get_shape())
     return result
+  elif isinstance(data, composite_tensor.CompositeTensor):
+    def enter_component(t):
+      return _Enter(t, frame_name, is_constant, parallel_iterations,
+                    use_ref, use_input_shape)
+    return nest.map_structure(enter_component, data, expand_composites=True)
   else:
     raise TypeError("Type %s not supported" % type(data))
 
@@ -261,6 +270,8 @@ def exit(data, name=None):  # pylint: disable=redefined-builtin
       return gen_control_flow_ops.ref_exit(data, name)
     else:
       return gen_control_flow_ops._exit(data, name)
+  elif isinstance(data, composite_tensor.CompositeTensor):
+    return nest.map_structure(exit, data, expand_composites=True)
   else:
     raise TypeError("Type %s not supported" % type(data))
 
@@ -548,6 +559,15 @@ def _AddNextAndBackEdge(m, v, enforce_shape_invariant=True):
       # TODO(skyewm): call this for other cases below (needs testing)
       _EnforceShapeInvariant(m, v)
     m.op._update_input(1, v)  # pylint: disable=protected-access
+  elif isinstance(m, composite_tensor.CompositeTensor):
+    # pylint: disable=protected-access
+    def update_component(m_component, v_component):
+      m_component.op._update_input(1, v_component)
+    if isinstance(m, ops.IndexedSlices):
+      v = math_ops._as_indexed_slices(v, optimize=False)
+    # pylint: enable=protected-access
+    v = _NextIteration(v)
+    return nest.map_structure(update_component, m, v, expand_composites=True)
   else:
     raise TypeError("Type %s not supported" % type(m))
   return v
-- 
GitLab


From 216851bb33c348bd8b33973b450d9b4be06815d7 Mon Sep 17 00:00:00 2001
From: Yanan Cao <ycao@google.com>
Date: Fri, 18 Jan 2019 11:44:34 -0800
Subject: [PATCH 0986/1095] Add back a missing arg documentation for
 validate_indices in tf.gather

PiperOrigin-RevId: 229966279
---
 tensorflow/python/ops/array_ops.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py
index 014fdd25cc..b9ff2f9717 100644
--- a/tensorflow/python/ops/array_ops.py
+++ b/tensorflow/python/ops/array_ops.py
@@ -3266,7 +3266,7 @@ def gather(params,
            validate_indices=None,
            name=None,
            axis=None,
-           batch_dims=0):  # pylint: disable=g-doc-args
+           batch_dims=0):
   r"""Gather slices from params axis axis according to indices.
 
   Gather slices from params axis axis according to indices.  `indices` must be
@@ -3320,13 +3320,14 @@ def gather(params,
       `axis + 1`.
     indices: The index `Tensor`.  Must be one of the following types: `int32`,
       `int64`. Must be in range `[0, params.shape[axis])`.
+    validate_indices: Deprecated, does nothing.
+    name: A name for the operation (optional).
     axis: A `Tensor`. Must be one of the following types: `int32`, `int64`. The
       `axis` in `params` to gather `indices` from. Must be greater than or equal
       to `batch_dims`.  Defaults to the first non-batch dimension. Supports
       negative indexes.
     batch_dims: An `integer`.  The number of batch dimensions.  Must be less
       than `ndims(inices)`.
-    name: A name for the operation (optional).
 
   Returns:
     A `Tensor`. Has the same type as `params`.
-- 
GitLab


From 45679c956cda0d8ffa28834fb917559057fdde86 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 18 Jan 2019 11:48:41 -0800
Subject: [PATCH 0987/1095] Add keras correctness test for stateful LSTM models
 with DeviceAssignment.

PiperOrigin-RevId: 229966996
---
 tensorflow/contrib/distribute/python/BUILD    | 22 +++++
 .../contrib/distribute/python/combinations.py | 22 ++++-
 .../python/keras_correctness_test_base.py     | 23 +++--
 ...as_stateful_lstm_model_correctness_test.py | 99 +++++++++++++++++++
 tensorflow/python/keras/engine/training.py    | 14 +++
 5 files changed, 171 insertions(+), 9 deletions(-)
 create mode 100644 tensorflow/contrib/distribute/python/keras_stateful_lstm_model_correctness_test.py

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index 6ae3ec7fb0..7a77dbe29c 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -651,6 +651,7 @@ py_library(
         "keras_embedding_model_correctness_test.py",
         "keras_image_model_correctness_test.py",
         "keras_lstm_model_correctness_test.py",
+        "keras_stateful_lstm_model_correctness_test.py",
     ],
     deps = [
         ":combinations",
@@ -750,6 +751,27 @@ distribute_py_test(
     ],
 )
 
+distribute_py_test(
+    name = "keras_stateful_lstm_model_correctness_test",
+    size = "medium",
+    srcs = ["keras_stateful_lstm_model_correctness_test.py"],
+    full_precision = True,
+    main = "keras_stateful_lstm_model_correctness_test.py",
+    # Shard count is set to an odd number to distribute tasks across
+    # shards more evenly.
+    shard_count = 31,
+    tags = [
+        "multi_and_single_gpu",
+        "no_oss",  # TODO(b/117919883): Fix python error.
+        "no_pip",
+        "no_windows_gpu",
+        "notsan",
+    ],
+    deps = [
+        ":keras_correctness_test_lib",
+    ],
+)
+
 distribute_py_test(
     name = "metrics_v1_test",
     srcs = ["metrics_v1_test.py"],
diff --git a/tensorflow/contrib/distribute/python/combinations.py b/tensorflow/contrib/distribute/python/combinations.py
index 304762e69e..88170d4085 100644
--- a/tensorflow/contrib/distribute/python/combinations.py
+++ b/tensorflow/contrib/distribute/python/combinations.py
@@ -53,6 +53,7 @@ from tensorflow.contrib.distribute.python import tpu_strategy as tpu_lib
 from tensorflow.contrib.optimizer_v2 import adagrad as adagrad_v2
 from tensorflow.contrib.optimizer_v2 import adam as adam_v2
 from tensorflow.contrib.optimizer_v2 import gradient_descent as gradient_descent_v2
+from tensorflow.contrib.tpu.python.tpu import device_assignment as device_assignment_lib
 from tensorflow.python.distribute import distribution_strategy_context
 from tensorflow.python.eager import context
 from tensorflow.python.framework import ops
@@ -321,11 +322,18 @@ class NamedDistribution(object):
     return self._required_tpu
 
 
-def _get_tpu_strategy_creator(steps_per_run, **kwargs):
+def _get_tpu_strategy_creator(steps_per_run, use_single_core=False, **kwargs):
   def _create_tpu_strategy():
     resolver = cluster_resolver.TPUClusterResolver("")
-    tpu_lib.initialize_tpu_system(resolver)
+    topology = tpu_lib.initialize_tpu_system(resolver)
+    device_assignment = None
+    if use_single_core:
+      device_assignment = device_assignment_lib.DeviceAssignment(
+          topology, core_assignment=device_assignment_lib.
+          SINGLE_CORE_ASSIGNMENT)
+
     strategy = tpu_lib.TPUStrategy(resolver, steps_per_run=steps_per_run,
+                                   device_assignment=device_assignment,
                                    **kwargs)
     return strategy
   return _create_tpu_strategy
@@ -345,6 +353,16 @@ tpu_strategy = NamedDistribution(
 tpu_strategy_one_step = NamedDistribution(
     "TPUOneStep", _get_tpu_strategy_creator(steps_per_run=1),
     required_tpu=True)
+tpu_strategy_loop_on_device_one_core = NamedDistribution(
+    "TPULoopOnDeviceOneCore", _get_tpu_strategy_creator(
+        steps_per_run=2, use_single_core=True,
+        _disable_training_loop_on_host=True),
+    required_tpu=True)
+tpu_strategy_one_step_loop_on_device_one_core = NamedDistribution(
+    "TPUOneStepLoopOnDeviceOneCore", _get_tpu_strategy_creator(
+        steps_per_run=1, use_single_core=True,
+        _disable_training_loop_on_host=True),
+    required_tpu=True)
 # TODO(b/122327153): Remove below two NamedDistributions.
 tpu_strategy_loop_on_device = NamedDistribution(
     "TPULoopOnDevice", _get_tpu_strategy_creator(
diff --git a/tensorflow/contrib/distribute/python/keras_correctness_test_base.py b/tensorflow/contrib/distribute/python/keras_correctness_test_base.py
index 08ed933f29..1ceb95cdd6 100644
--- a/tensorflow/contrib/distribute/python/keras_correctness_test_base.py
+++ b/tensorflow/contrib/distribute/python/keras_correctness_test_base.py
@@ -207,8 +207,8 @@ def get_correctness_test_inputs(use_numpy, use_validation_data,
   return training_inputs, eval_inputs, predict_inputs
 
 
-def fit_eval_and_predict(
-    initial_weights, input_fn, model_fn, distribution=None):
+def fit_eval_and_predict(initial_weights, input_fn, model_fn,
+                         distribution=None, is_stateful_model=False):
   """Generates results for fit/predict/evaluate for given model."""
   model = model_fn(initial_weights=initial_weights, distribution=distribution)
   training_inputs, eval_inputs, predict_inputs = input_fn(distribution)
@@ -222,7 +222,15 @@ def fit_eval_and_predict(
   result['weights_1'] = model.get_weights()
 
   if predict_inputs is not None:
-    result['predict_result_1'] = model.predict(**predict_inputs)
+    # Check correctness of the result of predict() invoked
+    # multiple times -- as for stateful models, result of
+    # predict may differ for each batch.
+    predict_length = 1
+    if is_stateful_model:
+      predict_length = 3
+    for i in range(predict_length):
+      result_key = 'predict_result_{}'.format(i)
+      result[result_key] = model.predict(**predict_inputs)
 
   # Train and eval again to mimic user's flow.
 
@@ -334,7 +342,8 @@ class TestDistributionStrategyCorrectnessBase(test.TestCase,
                            distribution,
                            use_numpy,
                            use_validation_data,
-                           with_batch_norm=False):
+                           with_batch_norm=False,
+                           is_stateful_model=False):
     with self.cached_session():
       self.set_up_test_config(use_numpy, use_validation_data, with_batch_norm)
       self.skip_unsupported_test_configuration(distribution)
@@ -355,10 +364,10 @@ class TestDistributionStrategyCorrectnessBase(test.TestCase,
 
       results_with_ds = fit_eval_and_predict(
           initial_weights, input_fn=input_fn, model_fn=self.get_model,
-          distribution=distribution)
+          distribution=distribution, is_stateful_model=is_stateful_model)
       results_without_ds = fit_eval_and_predict(
           initial_weights, input_fn=input_fn, model_fn=self.get_model,
-          distribution=None)
+          distribution=None, is_stateful_model=is_stateful_model)
 
       # First, special case, for multi-replica distributed training, batch norm
       # is not aggregated globally. So it is expected to have different weights.
@@ -448,7 +457,7 @@ class TestDistributionStrategyEmbeddingModelCorrectnessBase(
         features, maxlen=max_words)
     x_train = np.asarray(features, dtype=np.float32)
     y_train = np.asarray(labels, dtype=np.int32).reshape((count, 1))
-    x_predict = x_train
+    x_predict = x_train[:_GLOBAL_BATCH_SIZE]
     return x_train, y_train, x_predict
 
 
diff --git a/tensorflow/contrib/distribute/python/keras_stateful_lstm_model_correctness_test.py b/tensorflow/contrib/distribute/python/keras_stateful_lstm_model_correctness_test.py
new file mode 100644
index 0000000000..54eb07e100
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/keras_stateful_lstm_model_correctness_test.py
@@ -0,0 +1,99 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for stateful tf.keras LSTM models using DistributionStrategy."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.distribute.python import combinations
+from tensorflow.contrib.distribute.python import keras_correctness_test_base
+from tensorflow.python import keras
+from tensorflow.python.eager import test
+from tensorflow.python.training import gradient_descent
+
+
+def strategies_for_stateful_embedding_model():
+  """Returns TPUStrategy with single core device assignment."""
+
+  return [combinations.tpu_strategy_loop_on_device_one_core,
+          combinations.tpu_strategy_one_step_loop_on_device_one_core]
+
+
+def test_combinations_for_stateful_embedding_model():
+  return (
+      combinations.combine(
+          distribution=strategies_for_stateful_embedding_model(),
+          mode='graph',
+          use_numpy=False,
+          use_validation_data=False
+      ))
+
+
+class DistributionStrategyStatefulLstmModelCorrectnessTest(
+    keras_correctness_test_base.
+    TestDistributionStrategyEmbeddingModelCorrectnessBase):
+
+  def get_model(self, max_words=10, initial_weights=None, distribution=None):
+    batch_size = keras_correctness_test_base._GLOBAL_BATCH_SIZE
+
+    with keras_correctness_test_base.MaybeDistributionScope(distribution):
+      word_ids = keras.layers.Input(
+          shape=(max_words,),
+          batch_size=batch_size,
+          dtype=np.int32, name='words')
+      word_embed = keras.layers.Embedding(input_dim=20,
+                                          output_dim=10)(word_ids)
+      lstm_embed = keras.layers.LSTM(units=4,
+                                     return_sequences=False,
+                                     stateful=True)(word_embed)
+
+      preds = keras.layers.Dense(2, activation='softmax')(lstm_embed)
+      model = keras.Model(inputs=[word_ids], outputs=[preds])
+
+      if initial_weights:
+        model.set_weights(initial_weights)
+
+      model.compile(
+          optimizer=gradient_descent.GradientDescentOptimizer(
+              learning_rate=0.1),
+          loss='sparse_categorical_crossentropy',
+          metrics=['sparse_categorical_accuracy'])
+    return model
+
+  @combinations.generate(test_combinations_for_stateful_embedding_model())
+  def test_stateful_lstm_model_correctness(self,
+                                           distribution,
+                                           use_numpy,
+                                           use_validation_data):
+    self.run_correctness_test(distribution, use_numpy, use_validation_data,
+                              is_stateful_model=True)
+
+  @combinations.generate(keras_correctness_test_base.
+                         test_combinations_for_embedding_model())
+  def test_incorrectly_use_multiple_cores_for_stateful_lstm_model(
+      self, distribution, use_numpy, use_validation_data):
+    with self.assertRaisesRegexp(ValueError,
+                                 'Single core must be used for computation '
+                                 'on stateful models. Consider adding '
+                                 '`device_assignment` parameter to '
+                                 'TPUStrategy'):
+      self.run_correctness_test(distribution, use_numpy, use_validation_data,
+                                is_stateful_model=True)
+
+
+if __name__ == '__main__':
+  test.main()
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 53caac25c9..366a96e7a6 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -2214,6 +2214,20 @@ class Model(Network):
       raise NotImplementedError('`sample_weight` is currently not supported '
                                 'when using TPUStrategy.')
 
+    if (self.stateful and distributed_training_utils.is_tpu_strategy(
+        self._distribution_strategy) and self._distribution_strategy.
+        num_replicas_in_sync != 1):
+      raise ValueError('Single core must be used for computation on '
+                       'stateful models. Consider adding `device_assignment` '
+                       'parameter to TPUStrategy using\n'
+                       'topology = tf.contrib.distribute.'
+                       'initialize_tpu_system()\n'
+                       'device_assignment = tf.contrib.tpu.DeviceAssignment('
+                       'topology, core_assignment=tf.contrib.tpu.'
+                       'SINGLE_CORE_ASSIGNMENT)\n'
+                       'tpu_strategy = tf.contrib.distribute.TPUStrategy('
+                       'device_assignment=device_assignment)')
+
     # Validates `steps` and `shuffle` arguments right at the beginning
     # since we use it to construct the dataset object.
     # TODO(anjalisridhar): Remove this check once we refactor the
-- 
GitLab


From b3c326376263f8e1af0d44f29ac1dc6106f699de Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Fri, 18 Jan 2019 11:49:00 -0800
Subject: [PATCH 0988/1095] Fix kokoro test failure by adding necessary
 protobuf dependencies.

PiperOrigin-RevId: 229967055
---
 tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
index fa6acdb782..3bd4ff94d1 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
@@ -36,6 +36,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
+#include "tensorflow/core/platform/protobuf.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/core/protobuf/config.pb.h"  // NOLINT
 #include "tensorflow/core/public/session.h"
@@ -1123,7 +1124,7 @@ class OpConverterTest : public ::testing::Test {
 };
 
 template <typename T>
-void CopyTensorElements(const Tensor& tensor, proto2::RepeatedField<T>* out) {
+void CopyTensorElements(const Tensor& tensor, protobuf::RepeatedField<T>* out) {
   out->Clear();
   if (tensor.NumElements() == 0) return;
 
-- 
GitLab


From a206216c854e23d77bd727f039fd194ad831185f Mon Sep 17 00:00:00 2001
From: Martin Wicke <wicke@google.com>
Date: Fri, 18 Jan 2019 12:06:21 -0800
Subject: [PATCH 0989/1095] Make converter output filenames in logs.

PiperOrigin-RevId: 229970291
---
 tensorflow/tools/compatibility/ast_edits.py   | 11 +++---
 .../tools/compatibility/tf_upgrade_v2_main.py | 34 ++++++++++++++-----
 2 files changed, 32 insertions(+), 13 deletions(-)

diff --git a/tensorflow/tools/compatibility/ast_edits.py b/tensorflow/tools/compatibility/ast_edits.py
index 940be4c23c..17573e4f43 100644
--- a/tensorflow/tools/compatibility/ast_edits.py
+++ b/tensorflow/tools/compatibility/ast_edits.py
@@ -509,7 +509,8 @@ class ASTCodeUpgrader(object):
       in_place: Allow the conversion of an entire directory in place.
 
     Returns:
-      A tuple of files processed, the report string ofr all files, and errors
+      A tuple of files processed, the report string ofr all files, and a dict
+        mapping filenames to errors encountered in that file.
     """
 
     if output_root_directory == root_directory:
@@ -556,7 +557,7 @@ class ASTCodeUpgrader(object):
           files_to_copy.append((fullpath, fullpath_output))
 
     file_count = 0
-    tree_errors = []
+    tree_errors = {}
     report = ""
     report += ("=" * 80) + "\n"
     report += "Input tree: %r\n" % root_directory
@@ -568,7 +569,7 @@ class ASTCodeUpgrader(object):
         os.makedirs(output_directory)
       file_count += 1
       _, l_report, l_errors = self.process_file(input_path, output_path)
-      tree_errors += l_errors
+      tree_errors[input_path] = l_errors
       report += l_report
     for input_path, output_path in files_to_copy:
       output_directory = os.path.dirname(output_path)
@@ -586,7 +587,7 @@ class ASTCodeUpgrader(object):
       files_to_process += py_files
 
     file_count = 0
-    tree_errors = []
+    tree_errors = {}
     report = ""
     report += ("=" * 80) + "\n"
     report += "Input tree: %r\n" % root_directory
@@ -595,7 +596,7 @@ class ASTCodeUpgrader(object):
     for path in files_to_process:
       file_count += 1
       _, l_report, l_errors = self.process_file(path, path)
-      tree_errors += l_errors
+      tree_errors[path] = l_errors
       report += l_report
 
     return file_count, report, tree_errors
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_main.py b/tensorflow/tools/compatibility/tf_upgrade_v2_main.py
index 870bc6f216..871da9ad6b 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_main.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_main.py
@@ -98,15 +98,33 @@ Simple usage:
   else:
     parser.print_help()
   if report_text:
-    open(report_filename, "w").write(report_text)
-    print("TensorFlow 2.0 Upgrade Script")
-    print("-----------------------------")
-    print("Converted %d files\n" % files_processed)
-    print("Detected %d errors that require attention" % len(errors))
-    print("-" * 80)
-    print("\n".join(errors))
-    print("\nMake sure to read the detailed log %r\n" % report_filename)
+    num_errors = 0
+    report = ""
+    for f in errors:
+      if errors[f]:
+        num_errors += len(errors[f])
+        report += "-" * 80 + "\n"
+        report += "File: %s\n" % f
+        report += "-" * 80 + "\n"
+        report += "\n".join(errors[f]) + "\n"
+
+    report = (
+        "TensorFlow 2.0 Upgrade Script\n"
+        "-----------------------------\n"
+        "Converted %d files\n" % files_processed +
+        "Detected %d errors that require attention" % num_errors + "\n" +
+        "-" * 80 + "\n"
+    ) + report
 
+    with open(report_filename, "w") as report_file:
+      report_file.write(report)
+      report_file.write("=" * 80 + "\n")
+      report_file.write("Detailed log follows:\n\n")
+      report_file.write("=" * 80 + "\n")
+      report_file.write(report_text)
+
+    print(report)
+    print("\nMake sure to read the detailed log %r\n" % report_filename)
 
 if __name__ == "__main__":
   main()
-- 
GitLab


From f8b8e49bac488b16b1fe94cc07b8fb96dfb244bb Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Fri, 18 Jan 2019 12:09:53 -0800
Subject: [PATCH 0990/1095] Fix estimator import in tensorflow.compat.v1
 module.

PiperOrigin-RevId: 229970907
---
 tensorflow/compat_template_v1.__init__.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compat_template_v1.__init__.py b/tensorflow/compat_template_v1.__init__.py
index b966c22b23..9549a71c41 100644
--- a/tensorflow/compat_template_v1.__init__.py
+++ b/tensorflow/compat_template_v1.__init__.py
@@ -28,7 +28,8 @@ from tensorflow.python import pywrap_tensorflow  # pylint: disable=unused-import
 from tensorflow.python.tools import component_api_helper as _component_api_helper
 _component_api_helper.package_hook(
     parent_package_str=__name__,
-    child_package_str=('tensorflow_estimator.python.estimator.api.estimator'))
+    child_package_str=(
+        'tensorflow_estimator.python.estimator.api._v1.estimator'))
 _component_api_helper.package_hook(
     parent_package_str=__name__,
     child_package_str=('tensorflow.python.keras.api._v1.keras'))
-- 
GitLab


From c4894a845f6c52740d32f566c862b9b6a99ffccc Mon Sep 17 00:00:00 2001
From: Artem Belevich <tra@google.com>
Date: Fri, 18 Jan 2019 12:14:27 -0800
Subject: [PATCH 0991/1095] Make interactive graphviz buildable in opensource
 TF and add a build test for it.

PiperOrigin-RevId: 229971593
---
 tensorflow/compiler/xla/tools/BUILD           |  6 ++++++
 .../xla/tools/interactive_graphviz.cc         | 16 ++++++++--------
 .../xla/tools/interactive_graphviz_test.sh    | 19 +++++++++++++++++++
 3 files changed, 33 insertions(+), 8 deletions(-)
 create mode 100755 tensorflow/compiler/xla/tools/interactive_graphviz_test.sh

diff --git a/tensorflow/compiler/xla/tools/BUILD b/tensorflow/compiler/xla/tools/BUILD
index 4412a6ec69..52fee4770a 100644
--- a/tensorflow/compiler/xla/tools/BUILD
+++ b/tensorflow/compiler/xla/tools/BUILD
@@ -255,3 +255,9 @@ tf_cc_binary(
         "@com_google_absl//absl/strings",
     ],
 )
+
+sh_test(
+    name = "interactive_graphviz_build_only_test",
+    srcs = ["interactive_graphviz_test.sh"],
+    data = [":interactive_graphviz"],
+)
diff --git a/tensorflow/compiler/xla/tools/interactive_graphviz.cc b/tensorflow/compiler/xla/tools/interactive_graphviz.cc
index 6c90cde5a7..ac865707f8 100644
--- a/tensorflow/compiler/xla/tools/interactive_graphviz.cc
+++ b/tensorflow/compiler/xla/tools/interactive_graphviz.cc
@@ -1,4 +1,4 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -29,8 +29,7 @@ limitations under the License.
 #include "absl/strings/numbers.h"
 #include "absl/strings/str_cat.h"
 #include "absl/strings/str_join.h"
-#include "absl/strings/string_view_utils.h"
-#include "absl/strings/util.h"
+#include "absl/strings/str_split.h"
 #include "tensorflow/compiler/xla/client/client_library.h"
 #include "tensorflow/compiler/xla/client/local_client.h"
 #include "tensorflow/compiler/xla/service/compiler.h"
@@ -56,7 +55,8 @@ bool ReadLine(const char *prompt, string *line) {
   return util::ReadLine(prompt, line);
 #else
   std::cout << prompt;
-  return std::getline(std::cin, *line);
+  std::getline(std::cin, *line);
+  return std::cin.good();
 #endif
 }
 
@@ -391,9 +391,9 @@ void DisplayGraphHandle(const Options &opts, const string& handle) {
   std::cout << handle << std::endl;
 
   // If it is a url, try to open it up in the user's browser too.
-  if (strings::StartsWithIgnoreCase(handle, "http://") ||
-      strings::StartsWithIgnoreCase(handle, "https://") ||
-      strings::StartsWithIgnoreCase(handle, "file://")) {
+  if (absl::StartsWithIgnoreCase(handle, "http://") ||
+      absl::StartsWithIgnoreCase(handle, "https://") ||
+      absl::StartsWithIgnoreCase(handle, "file://")) {
     const char* browser_bin = opts.browser.empty() ? "/usr/bin/sensible-browser"
                                                    : opts.browser.c_str();
     tensorflow::SubProcess p;
@@ -515,7 +515,7 @@ void InteractiveDumpGraphs(const Options& opts, const HloModule& module) {
                 << std::endl;
       continue;
     }
-    std::vector<string> tokens = strings::Split(line, ' ');
+    std::vector<string> tokens = absl::StrSplit(line, ' ');
     if (tokens[0] == "quit" || tokens[0] == "exit") {
       break;
     } else if (tokens[0] == "help") {
diff --git a/tensorflow/compiler/xla/tools/interactive_graphviz_test.sh b/tensorflow/compiler/xla/tools/interactive_graphviz_test.sh
new file mode 100755
index 0000000000..b3e43aa7da
--- /dev/null
+++ b/tensorflow/compiler/xla/tools/interactive_graphviz_test.sh
@@ -0,0 +1,19 @@
+#! /bin/bash
+# /* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================*/
+
+# This is a placeholder for a compile-only test for intractive_graphviz tool.
+
+exit 0
-- 
GitLab


From 9cd051ee4b41d7bccaf8ad977644177d55a56baa Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 18 Jan 2019 12:24:53 -0800
Subject: [PATCH 0992/1095] Change the output type of eager profiler result
 from tfprof.ProfileProto to tpu.Trace. Then we can use Tensorboard profile
 plugin to visualize the profiling result.

PiperOrigin-RevId: 229973284
---
 tensorflow/c/eager/BUILD                      |  1 +
 tensorflow/c/eager/c_api_experimental.h       |  5 +-
 tensorflow/c/eager/c_api_experimental_test.cc |  8 +--
 .../contrib/tpu/profiler/trace_events.proto   |  3 +
 tensorflow/core/profiler/lib/BUILD            |  2 +-
 .../core/profiler/lib/eager_profiler.cc       | 56 ++++++++++++++++---
 tensorflow/python/eager/BUILD                 |  2 +-
 tensorflow/python/eager/profiler.py           | 23 +++++---
 tensorflow/python/eager/profiler_test.py      |  4 +-
 9 files changed, 78 insertions(+), 26 deletions(-)

diff --git a/tensorflow/c/eager/BUILD b/tensorflow/c/eager/BUILD
index 51665fb9db..777e2e626a 100644
--- a/tensorflow/c/eager/BUILD
+++ b/tensorflow/c/eager/BUILD
@@ -229,6 +229,7 @@ tf_cuda_cc_test(
         ":c_api_test_util",
         "//tensorflow/c:c_test_util",
         "//tensorflow/cc/profiler",
+        "//tensorflow/contrib/tpu/profiler:trace_events_proto_cc",
         "//tensorflow/core:lib",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:test",
diff --git a/tensorflow/c/eager/c_api_experimental.h b/tensorflow/c/eager/c_api_experimental.h
index 9eb80f5216..5cd5569468 100644
--- a/tensorflow/c/eager/c_api_experimental.h
+++ b/tensorflow/c/eager/c_api_experimental.h
@@ -35,9 +35,8 @@ typedef struct TFE_Profiler TFE_Profiler;
 TF_CAPI_EXPORT extern TFE_Profiler* TFE_NewProfiler(TFE_Context* ctx);
 TF_CAPI_EXPORT extern void TFE_DeleteProfiler(TFE_Profiler* profiler);
 
-// The output string is a binary string of tensorflow.tfprof.ProfileProto.
-// User can write the string to file for offline analysis by tfprof command-line
-// tools or graphical user interface.
+// The output string is a binary string of tensorflow.tpu.Trace. User can write
+// the string to file for offline analysis by tensorboard.
 TF_CAPI_EXPORT extern void TFE_ProfilerSerializeToString(TFE_Context* ctx,
                                                          TFE_Profiler* profiler,
                                                          TF_Buffer* buf,
diff --git a/tensorflow/c/eager/c_api_experimental_test.cc b/tensorflow/c/eager/c_api_experimental_test.cc
index c3d29bd2e0..af55fee66e 100644
--- a/tensorflow/c/eager/c_api_experimental_test.cc
+++ b/tensorflow/c/eager/c_api_experimental_test.cc
@@ -18,12 +18,12 @@ limitations under the License.
 #include <string.h>
 #include "tensorflow/c/eager/c_api_test_util.h"
 #include "tensorflow/cc/profiler/profiler.h"
+#include "tensorflow/contrib/tpu/profiler/trace_events.pb.h"
 #include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/protobuf.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/core/platform/test_benchmark.h"
-#include "tensorflow/core/profiler/tfprof_log.pb.h"
 
 using tensorflow::string;
 
@@ -70,17 +70,17 @@ void ExecuteWithProfiling(bool async) {
   TFE_ProfilerSerializeToString(ctx, profiler, profiler_result, status);
   TFE_DeleteProfiler(profiler);
   ASSERT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
-  tensorflow::tfprof::ProfileProto profile_proto;
+  tensorflow::tpu::Trace profile_proto;
   EXPECT_TRUE(profile_proto.ParseFromString(
       {reinterpret_cast<const char*>(profiler_result->data),
        profiler_result->length}));
   string profile_proto_str = profile_proto.DebugString();
   if (!gpu_device_name.empty()) {
-    EXPECT_TRUE(HasSubstr(profile_proto_str, "gpu:0"));
+    EXPECT_TRUE(HasSubstr(profile_proto_str, "GPU:0"));
     // device name with "stream:all" is collected by Device Tracer.
     EXPECT_TRUE(HasSubstr(profile_proto_str, "stream:all"));
   }
-  EXPECT_TRUE(HasSubstr(profile_proto_str, "cpu:0"));
+  EXPECT_TRUE(HasSubstr(profile_proto_str, "CPU:0"));
   TF_DeleteBuffer(profiler_result);
 
   TF_Tensor* t = TFE_TensorHandleResolve(retvals[0], status);
diff --git a/tensorflow/contrib/tpu/profiler/trace_events.proto b/tensorflow/contrib/tpu/profiler/trace_events.proto
index cb2b916267..96c4784c69 100644
--- a/tensorflow/contrib/tpu/profiler/trace_events.proto
+++ b/tensorflow/contrib/tpu/profiler/trace_events.proto
@@ -56,4 +56,7 @@ message TraceEvent {
   // The duration of the event in picoseconds if applicable.
   // Events without duration are called instant events.
   uint64 duration_ps = 10;
+
+  // Extra arguments that will be displayed in trace view.
+  map<string, string> args = 11;
 }
diff --git a/tensorflow/core/profiler/lib/BUILD b/tensorflow/core/profiler/lib/BUILD
index 3d46606cbd..0320ae1a68 100644
--- a/tensorflow/core/profiler/lib/BUILD
+++ b/tensorflow/core/profiler/lib/BUILD
@@ -23,7 +23,7 @@ tf_cuda_library(
     visibility = ["//tensorflow:internal"],
     deps = [
         "//tensorflow/core/common_runtime/eager:context",
-        "//tensorflow/cc/profiler",
+        "//tensorflow/contrib/tpu/profiler:trace_events_proto_cc",
     ] + select({
         "//tensorflow:android": [
             "//tensorflow/core:android_tensorflow_lib_lite",
diff --git a/tensorflow/core/profiler/lib/eager_profiler.cc b/tensorflow/core/profiler/lib/eager_profiler.cc
index 7702febbc3..6b2e8b9a19 100644
--- a/tensorflow/core/profiler/lib/eager_profiler.cc
+++ b/tensorflow/core/profiler/lib/eager_profiler.cc
@@ -14,7 +14,8 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/core/profiler/lib/eager_profiler.h"
-#include "tensorflow/cc/profiler/profiler.h"
+#include <string>
+#include "tensorflow/contrib/tpu/profiler/trace_events.pb.h"
 #include "tensorflow/core/common_runtime/eager/context.h"
 #include "tensorflow/core/common_runtime/step_stats_collector.h"
 #include "tensorflow/core/framework/graph.pb.h"
@@ -25,6 +26,47 @@ limitations under the License.
 
 namespace tensorflow {
 
+namespace {
+
+void ConvertRunMetadataToTraceEvent(RunMetadata* run_metadata,
+                                    tpu::Trace* trace) {
+  auto trace_devices = trace->mutable_devices();
+  // TODO(fishx): use a lighter representation instead of GraphDef to insert
+  // python information into trace event.
+
+  for (size_t device_id = 0;
+       device_id < run_metadata->step_stats().dev_stats_size(); ++device_id) {
+    // Create device
+    auto* device_stats =
+        run_metadata->mutable_step_stats()->mutable_dev_stats(device_id);
+    tensorflow::tpu::Device device;
+    device.set_name(device_stats->device());
+    device.set_device_id(device_id);
+    tensorflow::tpu::Resource resource;
+    resource.set_name("0");
+    resource.set_resource_id(0);
+    (*device.mutable_resources())[0] = resource;
+    (*trace_devices)[device_id] = device;
+
+    // Emit events.
+    for (auto node :
+         run_metadata->step_stats().dev_stats(device_id).node_stats()) {
+      auto* event = trace->add_trace_events();
+      auto* args = event->mutable_args();
+      event->set_device_id(device_id);
+      event->set_resource_id(0);
+      event->set_name(node.node_name());
+      event->set_timestamp_ps(node.all_start_micros() * 1000000000);
+      event->set_duration_ps(node.all_end_rel_micros() * 1000000000);
+      (*args)["label"] = node.timeline_label();
+    }
+  }
+
+  // TODO(fishx): Convert allocation data as well.
+}
+
+}  // namespace
+
 /*static*/ std::unique_ptr<EagerProfiler> EagerProfiler::Create(
     EagerContext* const context) {
   return absl::WrapUnique(new EagerProfiler(context));
@@ -58,12 +100,12 @@ Status EagerProfiler::SerializeToString(string* content) {
     step_stats_collector->Finalize();
   }
 
-  // TODO(fishx): update tfprof to use a lighter representation instead of
-  // GraphDef.
-  GraphDef graph;
-  std::unique_ptr<tfprof::Profiler> tfprof(new tfprof::Profiler(graph));
-  tfprof->AddStep(0, run_metadata_);
-  return tfprof->SerializeToString(content);
+  tpu::Trace trace;
+
+  ConvertRunMetadataToTraceEvent(&run_metadata_, &trace);
+
+  trace.SerializeToString(content);
+  return Status::OK();
 }
 
 EagerProfiler::EagerProfiler(EagerContext* const context) : context_(context) {
diff --git a/tensorflow/python/eager/BUILD b/tensorflow/python/eager/BUILD
index 3a35734051..6f798fcdb0 100644
--- a/tensorflow/python/eager/BUILD
+++ b/tensorflow/python/eager/BUILD
@@ -108,8 +108,8 @@ cuda_py_test(
     additional_deps = [
         ":profiler",
         ":test",
-        "//tensorflow/core/profiler:protos_all_py",
         "//tensorflow/python:constant_op",
+        "//tensorflow/contrib/tpu/profiler:trace_events_proto_py",
     ],
 )
 
diff --git a/tensorflow/python/eager/profiler.py b/tensorflow/python/eager/profiler.py
index f1d4373085..de96ae6ce9 100644
--- a/tensorflow/python/eager/profiler.py
+++ b/tensorflow/python/eager/profiler.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import os
 import threading
 
 from tensorflow.python import pywrap_tensorflow
@@ -25,8 +26,11 @@ from tensorflow.python.eager import context
 from tensorflow.python.framework import c_api_util
 from tensorflow.python.platform import gfile
 
+LOGDIR_PLUGIN = 'plugins/profile'
+
 _profiler = None
 _profiler_lock = threading.Lock()
+_run_num = 0
 
 
 def start():
@@ -48,14 +52,14 @@ def stop():
   """Stop current profiling session and return its result.
 
   Returns:
-    A binary string of tensorflow.tfprof.ProfileProto. User can write the string
-    to file for offline analysis by tfprof command-line tools or graphical user
-    interface.
+    A binary string of tensorflow.tpu.Trace. User can write the string
+    to file for offline analysis by tensorboard.
 
   Raises:
     AssertionError: If there is no active profiling session.
   """
   global _profiler
+  global _run_num
   if _profiler is None:
     raise AssertionError('Cannot stop profiling. No profiler is running.')
   with c_api_util.tf_buffer() as buffer_:
@@ -67,6 +71,7 @@ def stop():
   with _profiler_lock:
     pywrap_tensorflow.TFE_DeleteProfiler(_profiler)
     _profiler = None
+    _run_num += 1
   return result
 
 
@@ -75,19 +80,21 @@ class Profiler(object):
 
   Example usage:
   ```python
-  with Profiler("/path/to/save/result"):
+  with Profiler("/path/to/logdir"):
     # do some work
   ```
   """
 
-  def __init__(self, filename):
-    self._filename = filename
+  def __init__(self, logdir):
+    self._logdir = logdir
 
   def __enter__(self):
     start()
 
   def __exit__(self, typ, value, tb):
     result = stop()
-    with gfile.Open(self._filename, 'wb') as f:
+    plugin_dir = os.path.join(self._logdir, LOGDIR_PLUGIN,
+                              'run{}'.format(_run_num))
+    gfile.MakeDirs(plugin_dir)
+    with gfile.Open(os.path.join(plugin_dir, 'local.trace'), 'wb') as f:
       f.write(result)
-
diff --git a/tensorflow/python/eager/profiler_test.py b/tensorflow/python/eager/profiler_test.py
index b3fe2efabe..319d063990 100644
--- a/tensorflow/python/eager/profiler_test.py
+++ b/tensorflow/python/eager/profiler_test.py
@@ -18,7 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.core.profiler import tfprof_log_pb2
+from tensorflow.contrib.tpu.profiler import trace_events_pb2
 from tensorflow.python.eager import profiler
 from tensorflow.python.eager import test
 from tensorflow.python.framework import constant_op
@@ -34,7 +34,7 @@ class ProfilerTest(test_util.TensorFlowTestCase):
     product = three * five
     self.assertAllEqual(15, product)
     profile_result = profiler.stop()
-    profile_pb = tfprof_log_pb2.ProfileProto()
+    profile_pb = trace_events_pb2.Trace()
     profile_pb.ParseFromString(profile_result)
     profile_pb_str = '%s' % profile_pb
     self.assertTrue('Mul' in profile_pb_str)
-- 
GitLab


From 6fb73564d4d831fdb3699225235931c132cc3eb5 Mon Sep 17 00:00:00 2001
From: Karim Nosir <karimnosseir@google.com>
Date: Fri, 18 Jan 2019 12:27:23 -0800
Subject: [PATCH 0993/1095] Enable more tests.

PiperOrigin-RevId: 229973629
---
 tensorflow/lite/kernels/internal/BUILD | 22 ++++------------------
 1 file changed, 4 insertions(+), 18 deletions(-)

diff --git a/tensorflow/lite/kernels/internal/BUILD b/tensorflow/lite/kernels/internal/BUILD
index b734b2d6cc..1ff5164d2c 100644
--- a/tensorflow/lite/kernels/internal/BUILD
+++ b/tensorflow/lite/kernels/internal/BUILD
@@ -254,9 +254,6 @@ cc_library(
 cc_test(
     name = "tensor_test",
     srcs = ["tensor_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",  # TODO(b/117786830)
-    ],
     deps = [
         ":tensor",
         "@com_google_googletest//:gtest",
@@ -286,9 +283,6 @@ cc_library(
 cc_test(
     name = "quantization_util_test",
     srcs = ["quantization_util_test.cc"],
-    tags = [
-        "tflite_not_portable_ios",  # TODO(b/117786830)
-    ],
     deps = [
         ":quantization_util",
         "@com_google_googletest//:gtest",
@@ -568,9 +562,6 @@ tf_cc_test(
         "//conditions:default": [],
     }),
     linkstatic = 1,
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":tensor_utils",
         "//tensorflow/lite/c:c_api_internal",
@@ -595,9 +586,6 @@ cc_test(
     name = "depthwiseconv_quantized_test",
     srcs = ["depthwiseconv_quantized_test.cc"],
     shard_count = 2,
-    tags = [
-        "tflite_not_portable_ios",
-    ],
     deps = [
         ":optimized_base",
         ":reference_base",
@@ -610,9 +598,6 @@ cc_test(
 cc_test(
     name = "resize_bilinear_test",
     srcs = ["resize_bilinear_test.cc"],
-    tags = [
-        "tflite_not_portable",
-    ],
     deps = [
         ":optimized_base",
         ":reference_base",
@@ -625,9 +610,6 @@ cc_test(
 cc_test(
     name = "resize_nearest_neighbor_test",
     srcs = ["resize_nearest_neighbor_test.cc"],
-    tags = [
-        "tflite_not_portable",
-    ],
     deps = [
         ":optimized_base",
         ":reference_base",
@@ -679,6 +661,10 @@ cc_test(
 cc_test(
     name = "log_quantized_test",
     srcs = ["log_quantized_test.cc"],
+    linkopts = select({
+        "//tensorflow:windows": [],
+        "//conditions:default": ["-lm"],
+    }),
     deps = [
         ":optimized_base",
         ":reference_base",
-- 
GitLab


From 2bd14a830f8143cee455d5abe8540e8923b25f63 Mon Sep 17 00:00:00 2001
From: Thomas O'Malley <omalleyt@google.com>
Date: Fri, 18 Jan 2019 12:27:52 -0800
Subject: [PATCH 0994/1095] Fix to make sure correctness tests run in all modes

PiperOrigin-RevId: 229973693
---
 tensorflow/python/keras/engine/correctness_test.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/keras/engine/correctness_test.py b/tensorflow/python/keras/engine/correctness_test.py
index c2f3b040de..68634235d1 100644
--- a/tensorflow/python/keras/engine/correctness_test.py
+++ b/tensorflow/python/keras/engine/correctness_test.py
@@ -66,7 +66,10 @@ class SimpleBiasTest(keras_parameterized.TestCase):
 
   def _get_simple_bias_model(self):
     model = testing_utils.get_model_from_layers([Bias()], input_shape=(1,))
-    model.compile(keras.optimizer_v2.gradient_descent.SGD(0.1), 'mae')
+    model.compile(
+        keras.optimizer_v2.gradient_descent.SGD(0.1),
+        'mae',
+        run_eagerly=testing_utils.should_run_eagerly())
     return model
 
   def test_simple_bias_fit(self):
@@ -101,7 +104,10 @@ class MultipleInputTest(keras_parameterized.TestCase):
       model = MultiInputSubclassed()
     else:
       model = multi_input_functional()
-    model.compile(keras.optimizer_v2.gradient_descent.SGD(0.1), 'mae')
+    model.compile(
+        keras.optimizer_v2.gradient_descent.SGD(0.1),
+        'mae',
+        run_eagerly=testing_utils.should_run_eagerly())
     return model
 
   @parameterized.named_parameters(('subclassed', True), ('functional', False))
-- 
GitLab


From 49cd875b9ae1bc9da96871c321010c25e7ae3b32 Mon Sep 17 00:00:00 2001
From: Ayush Dubey <ayushd@google.com>
Date: Fri, 18 Jan 2019 12:28:08 -0800
Subject: [PATCH 0995/1095] Build fix for collective_nccl_reducer_test_gpu for
 non-CUDA builds.

PiperOrigin-RevId: 229973734
---
 tensorflow/core/kernels/collective_nccl_reducer_test.cc | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/kernels/collective_nccl_reducer_test.cc b/tensorflow/core/kernels/collective_nccl_reducer_test.cc
index ff33c1322c..26c92f1f74 100644
--- a/tensorflow/core/kernels/collective_nccl_reducer_test.cc
+++ b/tensorflow/core/kernels/collective_nccl_reducer_test.cc
@@ -12,6 +12,9 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
+
+#ifdef GOOGLE_CUDA
+
 #include "tensorflow/core/kernels/collective_nccl_reducer.h"
 
 #include <algorithm>
@@ -34,8 +37,6 @@ limitations under the License.
 #include "tensorflow/core/public/session_options.h"
 #include "tensorflow/core/public/version.h"
 
-#ifdef GOOGLE_CUDA
-
 namespace tensorflow {
 static constexpr int kStepId = 10;
 
-- 
GitLab


From 603afda1e43adef70bbd6764cc7416417e7baba7 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 18 Jan 2019 12:38:36 -0800
Subject: [PATCH 0996/1095] [TF:XLA] Make HloComputation equality comparison
 iterative.

When comparing HLOs with large subcomputations (like fusion), the previous recursive implementation can cause a stack overflow.

PiperOrigin-RevId: 229975468
---
 .../compiler/xla/service/hlo_computation.cc   | 43 ++++++++++++-------
 .../xla/service/hlo_computation_test.cc       | 23 ++++++++++
 2 files changed, 51 insertions(+), 15 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_computation.cc b/tensorflow/compiler/xla/service/hlo_computation.cc
index f9b64d12ae..98164752b9 100644
--- a/tensorflow/compiler/xla/service/hlo_computation.cc
+++ b/tensorflow/compiler/xla/service/hlo_computation.cc
@@ -695,21 +695,34 @@ bool HloComputation::operator==(const HloComputation& other) const {
   }
   absl::flat_hash_set<std::pair<const HloInstruction*, const HloInstruction*>>
       visited;
-  std::function<bool(const HloInstruction*, const HloInstruction*)> eq =
-      [&visited, &eq](const HloInstruction* a, const HloInstruction* b) {
-        // If <a,b> are visited but not identical, the recursion should have
-        // been aborted. So, if <a,b> are visited at this point, they must be
-        // identical.
-        if (visited.contains(std::make_pair(a, b))) {
-          return true;
-        }
-        visited.emplace(a, b);
-        return a->Identical(
-            *b, eq, [](const HloComputation* a, const HloComputation* b) {
-              return *a == *b;
-            });
-      };
-  return eq(root_instruction(), other.root_instruction());
+  std::vector<std::pair<const HloInstruction*, const HloInstruction*>> worklist;
+
+  worklist.push_back({root_instruction(), other.root_instruction()});
+
+  while (!worklist.empty()) {
+    auto pair = worklist.back();
+    worklist.pop_back();
+
+    if (visited.contains(pair)) {
+      continue;
+    }
+    visited.emplace(pair);
+    // TODO(b/123082518): Avoid recursively invoking == becasue it may
+    // cause a stack overflow with deeply nested subcomputations.
+    bool identical_ignoring_operands = pair.first->Identical(
+        *pair.second,
+        [](const HloInstruction*, const HloInstruction*) { return true; },
+        [](const HloComputation* a, const HloComputation* b) {
+          return *a == *b;
+        });
+    if (!identical_ignoring_operands) {
+      return false;
+    }
+    for (size_t i = 0; i < pair.first->operands().size(); ++i) {
+      worklist.push_back({pair.first->operand(i), pair.second->operand(i)});
+    }
+  }
+  return true;
 }
 
 Status HloComputation::ReplaceWithNewInstruction(
diff --git a/tensorflow/compiler/xla/service/hlo_computation_test.cc b/tensorflow/compiler/xla/service/hlo_computation_test.cc
index 251c7bbec4..35ee532704 100644
--- a/tensorflow/compiler/xla/service/hlo_computation_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_computation_test.cc
@@ -645,5 +645,28 @@ TEST_F(HloComputationTest, StringificationCanonical) {
   EXPECT_EQ(computation->ToString(options), expected_computation2);
 }
 
+std::unique_ptr<HloComputation> MakeAddNComputation(int n) {
+  auto builder = HloComputation::Builder("add_n");
+  auto result = builder.AddInstruction(HloInstruction::CreateParameter(
+      0, ShapeUtil::MakeShape(F32, {}), "x_value"));
+  auto one = builder.AddInstruction(
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0<float>(1.0)));
+  for (int i = 0; i < n; ++i) {
+    result = builder.AddInstruction(HloInstruction::CreateBinary(
+        one->shape(), HloOpcode::kAdd, result, one));
+  }
+  return builder.Build();
+}
+
+TEST_F(HloComputationTest, DeepEquality) {
+  auto computation_a = MakeAddNComputation(200000);
+  auto computation_b = MakeAddNComputation(200000);
+  EXPECT_TRUE(*computation_a == *computation_b);
+
+  auto computation_c = MakeAddNComputation(199999);
+  EXPECT_FALSE(*computation_a == *computation_c);
+  EXPECT_FALSE(*computation_c == *computation_b);
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From fec674bdcdc5fbd3ee277031e100ba3c438d4bcc Mon Sep 17 00:00:00 2001
From: Russell Power <power@google.com>
Date: Fri, 18 Jan 2019 12:50:23 -0800
Subject: [PATCH 0997/1095] EncapsulateSubgraphs: skip duplicate control edge
 checking.

This check should be unnecessary (as we are reconstructing the edges only as
needed).  This speeds up startup of large mesh-tf models by over a minute.

PiperOrigin-RevId: 229977388
---
 .../jit/encapsulate_subgraphs_pass.cc         | 32 ++++++++++++-------
 1 file changed, 21 insertions(+), 11 deletions(-)

diff --git a/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc b/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc
index a21e083131..d0d7a3f378 100644
--- a/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc
+++ b/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc
@@ -1008,13 +1008,15 @@ Status Encapsulator::Subgraph::AddHostComputes(
       // subgraph.
       for (const auto& src_node : oc_subgraph.control_inputs) {
         Node* src_image = node_images.at(src_node);
-        graph_->AddControlEdge(src_image, host_compute);
+        graph_->AddControlEdge(src_image, host_compute,
+                               /* allow_duplicates= */ true);
       }
 
       // Connect the _HostCompute node to its ancestor host compute nodes.
       for (const auto& ancestor_name : host_compute_ancestors) {
         Node* ancestor = host_compute_node[ancestor_name];
-        graph_->AddControlEdge(ancestor, host_compute);
+        graph_->AddControlEdge(ancestor, host_compute,
+                               /* allow_duplicates= */ true);
       }
 
       // Connect the consumers in the subgraph to the _HostCompute node.
@@ -1031,7 +1033,8 @@ Status Encapsulator::Subgraph::AddHostComputes(
       // node.
       for (const auto& dst_node : oc_subgraph.control_outputs) {
         Node* dst_image = node_images.at(dst_node);
-        graph_->AddControlEdge(host_compute, dst_image);
+        graph_->AddControlEdge(host_compute, dst_image,
+                               /* allow_duplicates= */ true);
       }
     }
   }
@@ -1059,7 +1062,8 @@ Status Encapsulator::Subgraph::MakeSequencingNode(const string& subgraph_name,
 void Encapsulator::Subgraph::ConnectSequencerToCallNode(Graph* graph_out) {
   if (sequencer_ != nullptr) {
     VLOG(2) << "ConnectSequencerToCallNode";
-    graph_out->AddControlEdge(sequencer_, call_node_);
+    graph_out->AddControlEdge(sequencer_, call_node_,
+                              /* allow_duplicates= */ true);
   }
 }
 
@@ -1279,7 +1283,8 @@ Status Encapsulator::Subgraph::AddRecvAtHostNode(
   // completes. This has no effect on execution order but prevents the
   // RecvAtHost being pruned.
   TF_RETURN_IF_ERROR(MakeSequencingNode(subgraph_name, graph_out));
-  graph_out->AddControlEdge(oc_subgraph->recv_at_host, sequencer_);
+  graph_out->AddControlEdge(oc_subgraph->recv_at_host, sequencer_,
+                            true /* skip duplicates check */);
 
   return Status::OK();
 }
@@ -1336,7 +1341,8 @@ Status Encapsulator::Subgraph::AddSendFromHostNode(
   // subgraph completes. This has no effect on execution order but prevents the
   // RecvAtHost being pruned.
   TF_RETURN_IF_ERROR(MakeSequencingNode(subgraph_name, graph_out));
-  graph_out->AddControlEdge(oc_subgraph->send_from_host, sequencer_);
+  graph_out->AddControlEdge(oc_subgraph->send_from_host, sequencer_,
+                            /* allow_duplicates= */ true);
 
   return Status::OK();
 }
@@ -1446,7 +1452,8 @@ Status Encapsulator::CopySubgraphEdges(
         src_func_id == dst_func_id) {
       Graph* g = subgraphs_[src_func_id].GetGraph();
       if (edge->IsControlEdge()) {
-        g->AddControlEdge(src_image, dst_image);
+        g->AddControlEdge(src_image, dst_image,
+                          /* allow_duplicates= */ true);
       } else {
         g->AddEdge(src_image, edge->src_output(), dst_image, edge->dst_input());
       }
@@ -1732,7 +1739,8 @@ Status Encapsulator::CopyEdgeToOutputGraph(
     if (edges_added
             ->emplace(OutputTensor(src_image, -1), InputTensor(dst_image, -1))
             .second) {
-      graph_out->AddControlEdge(src_image, dst_image);
+      graph_out->AddControlEdge(src_image, dst_image,
+                                /* allow_duplicates= */ true);
     }
 
     return Status::OK();
@@ -1761,7 +1769,8 @@ Status Encapsulator::AddCallNodeDependencies(Graph* graph_out) {
     const string& subgraph = ancestors.first;
     for (const string& ancestor : ancestors.second) {
       graph_out->AddControlEdge(subgraphs_[ancestor].GetCallNode(),
-                                subgraphs_[subgraph].GetCallNode());
+                                subgraphs_[subgraph].GetCallNode(),
+                                /* allow_duplicates= */ true);
     }
   }
   return Status::OK();
@@ -2129,7 +2138,8 @@ Status CheckClusterDependencyForCycles(
     const string& ancestor, const string& successor,
     const std::unordered_map<string, std::unordered_set<string>>& ancestors,
     const std::unordered_map<Node*, PathDetails>& node_ancestors_map,
-    GraphCycles* cycle_detector, std::map<string, int>* cycle_detector_map) {
+    GraphCycles* cycle_detector,
+    std::unordered_map<string, int>* cycle_detector_map) {
   if (cycle_detector_map->find(ancestor) == cycle_detector_map->end()) {
     (*cycle_detector_map)[ancestor] = cycle_detector->NewNode();
   }
@@ -2173,7 +2183,7 @@ Status Encapsulator::FindClusterDependencies() {
   // We check that clusters are acyclic using this cycle detector.
   GraphCycles cycle_detector;
   // Map from cluster name to cycle detector node id.
-  std::map<string, int> cycle_detector_map;
+  std::unordered_map<string, int> cycle_detector_map;
   // Process the nodes in topologically-sorted order.
   std::vector<Node*> nodes;
   GetReversePostOrder(*graph_in_, &nodes);
-- 
GitLab


From 2515feb397b22822e012a916798ccb5c3e1cd997 Mon Sep 17 00:00:00 2001
From: Andy Ly <lyandy@google.com>
Date: Fri, 18 Jan 2019 12:54:05 -0800
Subject: [PATCH 0998/1095] [Grappler] Check regular input port upper bound
 before returning fanin node. Have GraphView also store max regular input port
 per node.

PiperOrigin-RevId: 229977962
---
 tensorflow/core/grappler/graph_view.h         | 102 ++++++---
 tensorflow/core/grappler/graph_view_test.cc   | 193 ++++++++++--------
 .../core/grappler/mutable_graph_view.cc       |  99 ++++-----
 3 files changed, 231 insertions(+), 163 deletions(-)

diff --git a/tensorflow/core/grappler/graph_view.h b/tensorflow/core/grappler/graph_view.h
index dc4ab93894..88b1e975c1 100644
--- a/tensorflow/core/grappler/graph_view.h
+++ b/tensorflow/core/grappler/graph_view.h
@@ -143,13 +143,20 @@ class GraphViewInternal {
 
   // Gets the output port(s) in the immediate fanin of an input port.
   absl::flat_hash_set<OutputPort> GetFanin(const InputPort& port) const {
-    if (port.port_id >= 0) return {GetRegularFanin(port)};
+    if (port.port_id >= 0) {
+      OutputPort regular_fanin = GetRegularFanin(port);
+      if (regular_fanin.node == nullptr) {
+        return {};
+      }
+      return {regular_fanin};
+    }
 
     // Collect fanin for the control input.
     absl::flat_hash_set<OutputPort> result;
-    for (int i = port.node->input_size() - 1; i >= 0; --i) {
+    const int first_control_port =
+        gtl::FindWithDefault(max_regular_input_port_, port.node, -1) + 1;
+    for (int i = first_control_port; i < port.node->input_size(); ++i) {
       TensorId tensor_id = ParseTensorName(port.node->input(i));
-      if (tensor_id.index() >= 0) break;  // we reached regular inputs
 
       auto it = nodes_.find(tensor_id.node());
       if (it != nodes_.end()) result.emplace(it->second, tensor_id.index());
@@ -158,22 +165,36 @@ class GraphViewInternal {
   }
 
   // Special case: regular (i.e. non-control) input ports can only have one
-  // fanin.
+  // fanin. If port.port_id is out of range or is a control dependency, then an
+  // empty OutputPort is returned.
   const OutputPort GetRegularFanin(const InputPort& port) const {
-    DCHECK_GE(port.port_id, 0);
-    if (port.port_id < 0) return OutputPort();
+    if (port.port_id < 0 ||
+        port.port_id >
+            gtl::FindWithDefault(max_regular_input_port_, port.node, -1)) {
+      return OutputPort();
+    }
 
     TensorId tensor_id = ParseTensorName(port.node->input(port.port_id));
     return GetOutputPort(tensor_id.node(), tensor_id.index());
   }
 
   // Checks if a tensor id is a fanin of the node.
-  bool HasFanin(const NodeDef& node, const TensorId& fanin) const {
-    if (fanin.index() < -1) {
+  bool HasFanin(const NodeDefT& node, const TensorId& fanin) const {
+    int end = node.input_size();
+    if (end == 0 || fanin.index() < -1) {
       return false;
     }
-    for (const string& input : node.input()) {
-      if (ParseTensorName(input) == fanin) {
+
+    const int num_regular_fanins =
+        gtl::FindWithDefault(max_regular_input_port_, &node, -1) + 1;
+    int start = 0;
+    if (fanin.index() > -1) {
+      end = num_regular_fanins;
+    } else {
+      start = num_regular_fanins;
+    }
+    for (int i = start; i < end; ++i) {
+      if (ParseTensorName(node.input(i)) == fanin) {
         return true;
       }
     }
@@ -183,14 +204,14 @@ class GraphViewInternal {
   // Gets all the input ports in the immediate fanout of a node. Include the
   // controlled nodes iff include_controlled_nodes is true.
   absl::flat_hash_set<InputPort> GetFanouts(
-      const NodeDef& node, bool include_controlled_nodes) const {
+      const NodeDefT& node, bool include_controlled_nodes) const {
     absl::flat_hash_set<InputPort> result;
 
     OutputPort port;
     port.node = const_cast<NodeDefT*>(&node);
     const int first_port_id = include_controlled_nodes ? -1 : 0;
     const int last_port_id =
-        gtl::FindWithDefault(max_regular_output_port_, port.node, -1);
+        gtl::FindWithDefault(max_regular_output_port_, &node, -1);
 
     for (int i = first_port_id; i <= last_port_id; ++i) {
       port.port_id = i;
@@ -205,11 +226,14 @@ class GraphViewInternal {
   // Gets all the output ports in the immediate fanin of a node. Include the
   // controlling nodes iff include_controlling_nodes is true.
   absl::flat_hash_set<OutputPort> GetFanins(
-      const NodeDef& node, bool include_controlling_nodes) const {
+      const NodeDefT& node, bool include_controlling_nodes) const {
     absl::flat_hash_set<OutputPort> result;
-    for (int i = 0; i < node.input_size(); ++i) {
+    const int max_input_port =
+        include_controlling_nodes
+            ? node.input_size() - 1
+            : gtl::FindWithDefault(max_regular_input_port_, &node, -1);
+    for (int i = 0; i <= max_input_port; ++i) {
       TensorId tensor_id = ParseTensorName(node.input(i));
-      if (tensor_id.index() < 0 && !include_controlling_nodes) break;
 
       auto it = nodes_.find(tensor_id.node());
       if (it != nodes_.end()) result.emplace(it->second, tensor_id.index());
@@ -219,27 +243,23 @@ class GraphViewInternal {
 
   // Gets the number of ports in the immediate fanin of a node. Count the
   // controlling nodes iff include_controlling_nodes is true.
-  int NumFanins(const NodeDef& node, bool include_controlling_nodes) const {
-    int count = 0;
-    for (const string& input : node.input()) {
-      if (!include_controlling_nodes && IsControlInput(input)) {
-        break;
-      }
-      count += 1;
+  int NumFanins(const NodeDefT& node, bool include_controlling_nodes) const {
+    if (include_controlling_nodes) {
+      return node.input_size();
     }
-    return count;
+    return gtl::FindWithDefault(max_regular_input_port_, &node, -1) + 1;
   }
 
   // Gets the number of ports in the immediate fanout of a node. Count the
   // controlled nodes iff include_controlled_nodes is true.
-  int NumFanouts(const NodeDef& node, bool include_controlled_nodes) const {
+  int NumFanouts(const NodeDefT& node, bool include_controlled_nodes) const {
     int count = 0;
 
     OutputPort port;
     port.node = const_cast<NodeDefT*>(&node);
     const int first_port_id = include_controlled_nodes ? -1 : 0;
     const int last_port_id =
-        gtl::FindWithDefault(max_regular_output_port_, port.node, -1);
+        gtl::FindWithDefault(max_regular_output_port_, &node, -1);
 
     for (int i = first_port_id; i <= last_port_id; ++i) {
       port.port_id = i;
@@ -253,7 +273,7 @@ class GraphViewInternal {
   // Gets all the edges in the immediate fanout of a node. Include the
   // controlled edges iff include_controlled_edges is true.
   absl::flat_hash_set<Edge> GetFanoutEdges(
-      const NodeDef& node, bool include_controlled_edges) const {
+      const NodeDefT& node, bool include_controlled_edges) const {
     absl::flat_hash_set<Edge> result;
 
     OutputPort port;
@@ -267,8 +287,7 @@ class GraphViewInternal {
       auto it = fanouts_.find(port);
       if (it != fanouts_.end()) {
         for (auto itr = it->second.begin(); itr != it->second.end(); ++itr) {
-          result.emplace(/*src=*/OutputPort(const_cast<NodeDefT*>(&node), i),
-                         /*dst=*/*itr);
+          result.emplace(/*src=*/port, /*dst=*/*itr);
         }
       }
     }
@@ -278,11 +297,14 @@ class GraphViewInternal {
   // Gets all the edges in the immediate fanin of a node. Include the
   // controlling edges iff include_controlling_edges is true.
   absl::flat_hash_set<Edge> GetFaninEdges(
-      const NodeDef& node, bool include_controlling_edges) const {
+      const NodeDefT& node, bool include_controlling_edges) const {
     absl::flat_hash_set<Edge> result;
-    for (int i = 0; i < node.input_size(); ++i) {
+    const int max_input_port =
+        include_controlling_edges
+            ? node.input_size() - 1
+            : gtl::FindWithDefault(max_regular_input_port_, &node, -1);
+    for (int i = 0; i <= max_input_port; ++i) {
       TensorId tensor_id = ParseTensorName(node.input(i));
-      if (tensor_id.index() < 0 && !include_controlling_edges) break;
 
       auto it = nodes_.find(tensor_id.node());
       if (it != nodes_.end()) {
@@ -303,7 +325,10 @@ class GraphViewInternal {
     CHECK(result.second) << "Non unique node name detected: " << node->name();
   }
 
+  // TODO(lyandy): Checks for self loops, Switch control dependencies, fanins
+  // exist, and all regular fanins come before controlling fanins.
   void AddFanouts(NodeDefT* node) {
+    int max_input_port = -1;
     for (int i = 0; i < node->input_size(); ++i) {
       TensorId tensor_id = ParseTensorName(node->input(i));
       OutputPort output(nodes_[tensor_id.node()], tensor_id.index());
@@ -311,11 +336,15 @@ class GraphViewInternal {
       if (output.port_id < 0) {
         fanouts_[output].emplace(node, -1);
       } else {
+        max_input_port = i;
         max_regular_output_port_[output.node] =
             std::max(max_regular_output_port_[output.node], output.port_id);
         fanouts_[output].emplace(node, i);
       }
     }
+    if (max_input_port > -1) {
+      max_regular_input_port_[node] = max_input_port;
+    }
   }
 
   // Access to the mutable internal state for MutableGraphView.
@@ -325,7 +354,11 @@ class GraphViewInternal {
     return fanouts_;
   }
 
-  absl::flat_hash_map<const NodeDef*, int>& max_regular_output_port() {
+  absl::flat_hash_map<const NodeDefT*, int>& max_regular_input_port() {
+    return max_regular_input_port_;
+  }
+
+  absl::flat_hash_map<const NodeDefT*, int>& max_regular_output_port() {
     return max_regular_output_port_;
   }
 
@@ -338,10 +371,13 @@ class GraphViewInternal {
   // A mapping from the output port to all inputs that read from it.
   absl::flat_hash_map<OutputPort, absl::flat_hash_set<InputPort>> fanouts_;
 
+  // Keep a maximum index of input tensors of the node.
+  absl::flat_hash_map<const NodeDefT*, int> max_regular_input_port_;
+
   // Keep a maximum index of tensor fetched from the node. It doesn't guarantee
   // that all tensors in the [0, max_regular_output_port] range are actually
   // fetched by other nodes.
-  absl::flat_hash_map<const NodeDef*, int> max_regular_output_port_;
+  absl::flat_hash_map<const NodeDefT*, int> max_regular_output_port_;
 
   // If the node has no fanouts at given output port (output tensor consumers)
   // we return a reference to this set from `GetFanout` (we can't construct new
diff --git a/tensorflow/core/grappler/graph_view_test.cc b/tensorflow/core/grappler/graph_view_test.cc
index 404dcd30c1..839057065b 100644
--- a/tensorflow/core/grappler/graph_view_test.cc
+++ b/tensorflow/core/grappler/graph_view_test.cc
@@ -20,6 +20,7 @@ limitations under the License.
 #include "tensorflow/cc/ops/standard_ops.h"
 #include "tensorflow/core/grappler/grappler_item.h"
 #include "tensorflow/core/grappler/inputs/trivial_test_graph_input_yielder.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/platform/test.h"
 
 namespace tensorflow {
@@ -42,26 +43,24 @@ TEST_F(GraphViewTest, OpPortIdToArgIdShapeN) {
 
   const OpDef* a_op_def = nullptr;
   const OpDef* b_op_def = nullptr;
-  EXPECT_TRUE(
-      OpRegistry::Global()->LookUpOpDef(a_node_def.op(), &a_op_def).ok());
-  EXPECT_TRUE(
-      OpRegistry::Global()->LookUpOpDef(b_node_def.op(), &b_op_def).ok());
+  TF_EXPECT_OK(OpRegistry::Global()->LookUpOpDef(a_node_def.op(), &a_op_def));
+  TF_EXPECT_OK(OpRegistry::Global()->LookUpOpDef(b_node_def.op(), &b_op_def));
 
   // Const has 0 inputs, 1 output.
-  EXPECT_EQ(-1, OpInputPortIdToArgId(a_node_def, *a_op_def, 0));
-  EXPECT_EQ(0, OpOutputPortIdToArgId(a_node_def, *a_op_def, 0));
-  EXPECT_EQ(-1, OpOutputPortIdToArgId(a_node_def, *a_op_def, 1));
+  EXPECT_EQ(OpInputPortIdToArgId(a_node_def, *a_op_def, 0), -1);
+  EXPECT_EQ(OpOutputPortIdToArgId(a_node_def, *a_op_def, 0), 0);
+  EXPECT_EQ(OpOutputPortIdToArgId(a_node_def, *a_op_def, 1), -1);
 
   // ShapeN has N=3 inputs and outputs.
-  EXPECT_EQ(0, OpInputPortIdToArgId(b_node_def, *b_op_def, 0));
-  EXPECT_EQ(0, OpInputPortIdToArgId(b_node_def, *b_op_def, 1));
-  EXPECT_EQ(0, OpInputPortIdToArgId(b_node_def, *b_op_def, 2));
-  EXPECT_EQ(-1, OpInputPortIdToArgId(b_node_def, *b_op_def, 3));
-  EXPECT_EQ(0, OpOutputPortIdToArgId(b_node_def, *b_op_def, 0));
-  EXPECT_EQ(0, OpOutputPortIdToArgId(b_node_def, *b_op_def, 1));
-  EXPECT_EQ(0, OpOutputPortIdToArgId(b_node_def, *b_op_def, 2));
-  EXPECT_EQ(-1, OpOutputPortIdToArgId(b_node_def, *b_op_def, 3));
-  EXPECT_EQ(-1, OpOutputPortIdToArgId(b_node_def, *b_op_def, 4));
+  EXPECT_EQ(OpInputPortIdToArgId(b_node_def, *b_op_def, 0), 0);
+  EXPECT_EQ(OpInputPortIdToArgId(b_node_def, *b_op_def, 1), 0);
+  EXPECT_EQ(OpInputPortIdToArgId(b_node_def, *b_op_def, 2), 0);
+  EXPECT_EQ(OpInputPortIdToArgId(b_node_def, *b_op_def, 3), -1);
+  EXPECT_EQ(OpOutputPortIdToArgId(b_node_def, *b_op_def, 0), 0);
+  EXPECT_EQ(OpOutputPortIdToArgId(b_node_def, *b_op_def, 1), 0);
+  EXPECT_EQ(OpOutputPortIdToArgId(b_node_def, *b_op_def, 2), 0);
+  EXPECT_EQ(OpOutputPortIdToArgId(b_node_def, *b_op_def, 3), -1);
+  EXPECT_EQ(OpOutputPortIdToArgId(b_node_def, *b_op_def, 4), -1);
 }
 
 TEST_F(GraphViewTest, OpPortIdToArgIdSparseSplit) {
@@ -76,22 +75,21 @@ TEST_F(GraphViewTest, OpPortIdToArgIdSparseSplit) {
 
     const NodeDef& b_node_def = *graph_view.GetNode("b");
     const OpDef* b_op_def = nullptr;
-    EXPECT_TRUE(
-        OpRegistry::Global()->LookUpOpDef(b_node_def.op(), &b_op_def).ok());
+    TF_EXPECT_OK(OpRegistry::Global()->LookUpOpDef(b_node_def.op(), &b_op_def));
 
     // We have 4 inputs.
-    EXPECT_EQ(0, OpInputPortIdToArgId(b_node_def, *b_op_def, 0));
-    EXPECT_EQ(1, OpInputPortIdToArgId(b_node_def, *b_op_def, 1));
-    EXPECT_EQ(2, OpInputPortIdToArgId(b_node_def, *b_op_def, 2));
-    EXPECT_EQ(3, OpInputPortIdToArgId(b_node_def, *b_op_def, 3));
-    EXPECT_EQ(-1, OpInputPortIdToArgId(b_node_def, *b_op_def, 4));
+    EXPECT_EQ(OpInputPortIdToArgId(b_node_def, *b_op_def, 0), 0);
+    EXPECT_EQ(OpInputPortIdToArgId(b_node_def, *b_op_def, 1), 1);
+    EXPECT_EQ(OpInputPortIdToArgId(b_node_def, *b_op_def, 2), 2);
+    EXPECT_EQ(OpInputPortIdToArgId(b_node_def, *b_op_def, 3), 3);
+    EXPECT_EQ(OpInputPortIdToArgId(b_node_def, *b_op_def, 4), -1);
 
     for (int port_id = 0; port_id <= num_splits * 3; ++port_id) {
       int arg_id = -1;
       if (port_id < num_splits * 3) {
         arg_id = port_id / num_splits;
       }
-      EXPECT_EQ(arg_id, OpOutputPortIdToArgId(b_node_def, *b_op_def, port_id));
+      EXPECT_EQ(OpOutputPortIdToArgId(b_node_def, *b_op_def, port_id), arg_id);
     }
   }
 }
@@ -110,18 +108,17 @@ TEST_F(GraphViewTest, ParseSingleExample) {
   const NodeDef& c_node_def = *graph_view.GetNode("c");
 
   const OpDef* c_op_def = nullptr;
-  EXPECT_TRUE(
-      OpRegistry::Global()->LookUpOpDef(c_node_def.op(), &c_op_def).ok());
-
-  EXPECT_EQ(0, OpOutputPortIdToArgId(c_node_def, *c_op_def, 0));
-  EXPECT_EQ(0, OpOutputPortIdToArgId(c_node_def, *c_op_def, 1));
-  EXPECT_EQ(1, OpOutputPortIdToArgId(c_node_def, *c_op_def, 2));
-  EXPECT_EQ(1, OpOutputPortIdToArgId(c_node_def, *c_op_def, 3));
-  EXPECT_EQ(2, OpOutputPortIdToArgId(c_node_def, *c_op_def, 4));
-  EXPECT_EQ(2, OpOutputPortIdToArgId(c_node_def, *c_op_def, 5));
-  EXPECT_EQ(3, OpOutputPortIdToArgId(c_node_def, *c_op_def, 6));
-  EXPECT_EQ(3, OpOutputPortIdToArgId(c_node_def, *c_op_def, 7));
-  EXPECT_EQ(-1, OpOutputPortIdToArgId(c_node_def, *c_op_def, 8));
+  TF_EXPECT_OK(OpRegistry::Global()->LookUpOpDef(c_node_def.op(), &c_op_def));
+
+  EXPECT_EQ(OpOutputPortIdToArgId(c_node_def, *c_op_def, 0), 0);
+  EXPECT_EQ(OpOutputPortIdToArgId(c_node_def, *c_op_def, 1), 0);
+  EXPECT_EQ(OpOutputPortIdToArgId(c_node_def, *c_op_def, 2), 1);
+  EXPECT_EQ(OpOutputPortIdToArgId(c_node_def, *c_op_def, 3), 1);
+  EXPECT_EQ(OpOutputPortIdToArgId(c_node_def, *c_op_def, 4), 2);
+  EXPECT_EQ(OpOutputPortIdToArgId(c_node_def, *c_op_def, 5), 2);
+  EXPECT_EQ(OpOutputPortIdToArgId(c_node_def, *c_op_def, 6), 3);
+  EXPECT_EQ(OpOutputPortIdToArgId(c_node_def, *c_op_def, 7), 3);
+  EXPECT_EQ(OpOutputPortIdToArgId(c_node_def, *c_op_def, 8), -1);
 }
 
 TEST_F(GraphViewTest, BasicGraph) {
@@ -132,26 +129,26 @@ TEST_F(GraphViewTest, BasicGraph) {
   GraphView graph(&item.graph);
 
   GraphView::InputPort input = graph.GetInputPort("AddN", 0);
-  EXPECT_EQ("AddN", input.node->name());
-  EXPECT_EQ(0, input.port_id);
+  EXPECT_EQ(input.node->name(), "AddN");
+  EXPECT_EQ(input.port_id, 0);
   GraphView::OutputPort fanin = graph.GetRegularFanin(input);
-  EXPECT_EQ("Square", fanin.node->name());
-  EXPECT_EQ(0, fanin.port_id);
+  EXPECT_EQ(fanin.node->name(), "Square");
+  EXPECT_EQ(fanin.port_id, 0);
 
   input = graph.GetInputPort("AddN", 1);
-  EXPECT_EQ("AddN", input.node->name());
-  EXPECT_EQ(1, input.port_id);
+  EXPECT_EQ(input.node->name(), "AddN");
+  EXPECT_EQ(input.port_id, 1);
   fanin = graph.GetRegularFanin(input);
-  EXPECT_EQ("Square_1", fanin.node->name());
-  EXPECT_EQ(0, fanin.port_id);
+  EXPECT_EQ(fanin.node->name(), "Square_1");
+  EXPECT_EQ(fanin.port_id, 0);
 
   GraphView::OutputPort output = graph.GetOutputPort("AddN", 0);
-  EXPECT_EQ("AddN", output.node->name());
-  EXPECT_EQ(0, output.port_id);
-  EXPECT_EQ(2, graph.GetFanout(output).size());
+  EXPECT_EQ(output.node->name(), "AddN");
+  EXPECT_EQ(output.port_id, 0);
+  EXPECT_EQ(graph.GetFanout(output).size(), 2);
   for (auto fanout : graph.GetFanout(output)) {
     if (fanout.node->name() == "AddN_2" || fanout.node->name() == "AddN_3") {
-      EXPECT_EQ(0, fanout.port_id);
+      EXPECT_EQ(fanout.port_id, 0);
     } else {
       // Invalid fanout
       EXPECT_FALSE(true);
@@ -159,7 +156,7 @@ TEST_F(GraphViewTest, BasicGraph) {
   }
 
   const NodeDef* add_node = graph.GetNode("AddN");
-  EXPECT_NE(nullptr, add_node);
+  EXPECT_NE(add_node, nullptr);
 
   absl::flat_hash_set<string> fanouts;
   absl::flat_hash_set<string> expected_fanouts = {"AddN_2:0", "AddN_3:0"};
@@ -190,44 +187,44 @@ TEST_F(GraphViewTest, ControlDependencies) {
   GraphView graph(&item.graph);
 
   GraphView::OutputPort output = graph.GetOutputPort("a", -1);
-  EXPECT_EQ("a", output.node->name());
-  EXPECT_EQ(-1, output.port_id);
+  EXPECT_EQ(output.node->name(), "a");
+  EXPECT_EQ(output.port_id, -1);
   auto fanout = graph.GetFanout(output);
-  EXPECT_EQ(1, fanout.size());
-  EXPECT_EQ("d", (*fanout.begin()).node->name());
-  EXPECT_EQ(-1, (*fanout.begin()).port_id);
+  EXPECT_EQ(fanout.size(), 1);
+  EXPECT_EQ((*fanout.begin()).node->name(), "d");
+  EXPECT_EQ((*fanout.begin()).port_id, -1);
 
   output = graph.GetOutputPort("a", 0);
-  EXPECT_EQ("a", output.node->name());
-  EXPECT_EQ(0, output.port_id);
+  EXPECT_EQ(output.node->name(), "a");
+  EXPECT_EQ(output.port_id, 0);
   fanout = graph.GetFanout(output);
-  EXPECT_EQ(1, fanout.size());
-  EXPECT_EQ("b", (*fanout.begin()).node->name());
-  EXPECT_EQ(0, (*fanout.begin()).port_id);
+  EXPECT_EQ(fanout.size(), 1);
+  EXPECT_EQ((*fanout.begin()).node->name(), "b");
+  EXPECT_EQ((*fanout.begin()).port_id, 0);
 
   GraphView::InputPort input = graph.GetInputPort("d", -1);
-  EXPECT_EQ("d", input.node->name());
-  EXPECT_EQ(-1, input.port_id);
+  EXPECT_EQ(input.node->name(), "d");
+  EXPECT_EQ(input.port_id, -1);
   auto fanin = graph.GetFanin(input);
-  EXPECT_EQ(1, fanin.size());
-  EXPECT_EQ("a", (*fanin.begin()).node->name());
-  EXPECT_EQ(-1, (*fanin.begin()).port_id);
+  EXPECT_EQ(fanin.size(), 1);
+  EXPECT_EQ((*fanin.begin()).node->name(), "a");
+  EXPECT_EQ((*fanin.begin()).port_id, -1);
 
   input = graph.GetInputPort("d", 0);
-  EXPECT_EQ("d", input.node->name());
-  EXPECT_EQ(0, input.port_id);
+  EXPECT_EQ(input.node->name(), "d");
+  EXPECT_EQ(input.port_id, 0);
   fanin = graph.GetFanin(input);
-  EXPECT_EQ(1, fanin.size());
-  EXPECT_EQ("b", (*fanin.begin()).node->name());
-  EXPECT_EQ(0, (*fanin.begin()).port_id);
+  EXPECT_EQ(fanin.size(), 1);
+  EXPECT_EQ((*fanin.begin()).node->name(), "b");
+  EXPECT_EQ((*fanin.begin()).port_id, 0);
 
   input = graph.GetInputPort("d", 1);
-  EXPECT_EQ("d", input.node->name());
-  EXPECT_EQ(1, input.port_id);
+  EXPECT_EQ(input.node->name(), "d");
+  EXPECT_EQ(input.port_id, 1);
   fanin = graph.GetFanin(input);
-  EXPECT_EQ(1, fanin.size());
-  EXPECT_EQ("c", (*fanin.begin()).node->name());
-  EXPECT_EQ(0, (*fanin.begin()).port_id);
+  EXPECT_EQ(fanin.size(), 1);
+  EXPECT_EQ((*fanin.begin()).node->name(), "c");
+  EXPECT_EQ((*fanin.begin()).port_id, 0);
 }
 
 TEST_F(GraphViewTest, HasNode) {
@@ -238,8 +235,8 @@ TEST_F(GraphViewTest, HasNode) {
   TF_CHECK_OK(s.ToGraphDef(&item.graph));
   GraphView graph(&item.graph);
 
-  EXPECT_EQ(true, graph.HasNode("a"));
-  EXPECT_EQ(false, graph.HasNode("b"));
+  EXPECT_EQ(graph.HasNode("a"), true);
+  EXPECT_EQ(graph.HasNode("b"), false);
 }
 
 TEST_F(GraphViewTest, HasFanin) {
@@ -254,14 +251,42 @@ TEST_F(GraphViewTest, HasFanin) {
   GraphView graph(&item.graph);
 
   const NodeDef* d_node = graph.GetNode("d");
-  EXPECT_NE(nullptr, d_node);
-
-  EXPECT_EQ(true, graph.HasFanin(*d_node, {"a", Graph::kControlSlot}));
-  EXPECT_EQ(false, graph.HasFanin(*d_node, {"a", 0}));
-  EXPECT_EQ(true, graph.HasFanin(*d_node, {"b", 0}));
-  EXPECT_EQ(false, graph.HasFanin(*d_node, {"b", Graph::kControlSlot}));
-  EXPECT_EQ(true, graph.HasFanin(*d_node, {"c", 0}));
-  EXPECT_EQ(false, graph.HasFanin(*d_node, {"c", Graph::kControlSlot}));
+  EXPECT_NE(d_node, nullptr);
+
+  EXPECT_EQ(graph.HasFanin(*d_node, {"a", Graph::kControlSlot}), true);
+  EXPECT_EQ(graph.HasFanin(*d_node, {"a", 0}), false);
+  EXPECT_EQ(graph.HasFanin(*d_node, {"b", 0}), true);
+  EXPECT_EQ(graph.HasFanin(*d_node, {"b", Graph::kControlSlot}), false);
+  EXPECT_EQ(graph.HasFanin(*d_node, {"c", 0}), true);
+  EXPECT_EQ(graph.HasFanin(*d_node, {"c", Graph::kControlSlot}), false);
+}
+
+TEST_F(GraphViewTest, GetRegularFaninPortOutOfBounds) {
+  tensorflow::Scope s = tensorflow::Scope::NewRootScope();
+  Output a = ops::Const(s.WithOpName("a"), 0.0f, {10, 10});
+  Output b = ops::Square(s.WithOpName("b"), {});
+  Output c = ops::Sqrt(s.WithOpName("c"), {b});
+  Output d = ops::AddN(s.WithOpName("d").WithControlDependencies(a), {b, c});
+
+  GrapplerItem item;
+  TF_CHECK_OK(s.ToGraphDef(&item.graph));
+  GraphView graph(&item.graph);
+
+  const NodeDef* b_node = graph.GetNode("b");
+  EXPECT_NE(b_node, nullptr);
+  const NodeDef* c_node = graph.GetNode("c");
+  EXPECT_NE(c_node, nullptr);
+  const NodeDef* d_node = graph.GetNode("d");
+  EXPECT_NE(d_node, nullptr);
+
+  auto d_output_0 = graph.GetRegularFanin({d_node, 0});
+  EXPECT_EQ(d_output_0, GraphView::OutputPort(b_node, 0));
+  auto d_output_1 = graph.GetRegularFanin({d_node, 1});
+  EXPECT_EQ(d_output_1, GraphView::OutputPort(c_node, 0));
+  auto d_output_2 = graph.GetRegularFanin({d_node, 2});
+  EXPECT_EQ(d_output_2, GraphView::OutputPort());
+  auto d_output_control = graph.GetRegularFanin({d_node, Graph::kControlSlot});
+  EXPECT_EQ(d_output_control, GraphView::OutputPort());
 }
 
 }  // namespace
diff --git a/tensorflow/core/grappler/mutable_graph_view.cc b/tensorflow/core/grappler/mutable_graph_view.cc
index e31eedf166..43dec07c1d 100644
--- a/tensorflow/core/grappler/mutable_graph_view.cc
+++ b/tensorflow/core/grappler/mutable_graph_view.cc
@@ -117,10 +117,11 @@ bool CanDedupControlWithRegularInput(const MutableGraphView& graph,
 }  // namespace
 
 void MutableGraphView::AddAndDedupFanouts(NodeDef* node) {
-  // TODO(lyandy): Checks for self loops, Switch control dependencies and if
-  // fanins exist.
+  // TODO(lyandy): Checks for self loops, Switch control dependencies, fanins
+  // exist, and all regular fanins come before controlling fanins.
   absl::flat_hash_set<absl::string_view> fanins;
   absl::flat_hash_set<absl::string_view> controlling_fanins;
+  int max_input_port = -1;
   int pos = 0;
   const int last_idx = node->input_size() - 1;
   int last_pos = last_idx;
@@ -144,6 +145,7 @@ void MutableGraphView::AddAndDedupFanouts(NodeDef* node) {
       if (is_control_input) {
         fanouts()[output].emplace(node, Graph::kControlSlot);
       } else {
+        max_input_port = pos;
         max_regular_output_port()[output.node] =
             std::max(max_regular_output_port()[output.node], output.port_id);
         fanouts()[output].emplace(node, pos);
@@ -158,6 +160,10 @@ void MutableGraphView::AddAndDedupFanouts(NodeDef* node) {
   if (last_pos < last_idx) {
     node->mutable_input()->DeleteSubrange(last_pos + 1, last_idx - last_pos);
   }
+
+  if (max_input_port > -1) {
+    max_regular_input_port()[node] = max_input_port;
+  }
 }
 
 void MutableGraphView::UpdateMaxRegularOutputPortForRemovedFanin(
@@ -303,9 +309,7 @@ Status MutableGraphView::UpdateFanoutsInternal(NodeDef* from_node,
     // Update input at destination node.
     input_port.node->set_input(
         input_port.port_id,
-        output_port.port_id == 0
-            ? to_node->name()
-            : absl::StrCat(to_node->name(), ":", output_port.port_id));
+        TensorIdToString({to_node->name(), output_port.port_id}));
 
     // Remove old edge between the `from_node` and the fanout node.
     remove_edge(output_port, input_port);
@@ -333,7 +337,7 @@ Status MutableGraphView::UpdateFanoutsInternal(NodeDef* from_node,
 
 bool MutableGraphView::AddFaninInternal(NodeDef* node,
                                         const OutputPort& fanin) {
-  int num_non_controlling_fanins =
+  int num_regular_fanins =
       NumFanins(*node, /*include_controlling_nodes=*/false);
   bool input_is_control = IsOutputPortControlling(fanin);
   bool can_dedup_control_with_regular_input =
@@ -341,7 +345,7 @@ bool MutableGraphView::AddFaninInternal(NodeDef* node,
   // Don't add duplicate control dependencies.
   if (input_is_control) {
     const int start =
-        can_dedup_control_with_regular_input ? 0 : num_non_controlling_fanins;
+        can_dedup_control_with_regular_input ? 0 : num_regular_fanins;
     for (int i = start; i < node->input_size(); ++i) {
       if (ParseTensorName(node->input(i)).node() == fanin.node->name()) {
         return false;
@@ -351,17 +355,15 @@ bool MutableGraphView::AddFaninInternal(NodeDef* node,
 
   InputPort input;
   input.node = node;
-  input.port_id =
-      input_is_control ? Graph::kControlSlot : num_non_controlling_fanins;
+  input.port_id = input_is_control ? Graph::kControlSlot : num_regular_fanins;
 
   node->add_input(TensorIdToString({fanin.node->name(), fanin.port_id}));
-  if (IsOutputPortRegular(fanin)) {
+  if (!input_is_control) {
     int last_node_input = node->input_size() - 1;
     // If there are control dependencies in node, move newly inserted fanin to
     // be before such control dependencies.
-    if (num_non_controlling_fanins < last_node_input) {
-      node->mutable_input()->SwapElements(last_node_input,
-                                          num_non_controlling_fanins);
+    if (num_regular_fanins < last_node_input) {
+      node->mutable_input()->SwapElements(last_node_input, num_regular_fanins);
     }
   }
 
@@ -370,9 +372,12 @@ bool MutableGraphView::AddFaninInternal(NodeDef* node,
     max_regular_output_port()[fanin.node] = fanin.port_id;
   }
 
-  // Dedup control dependencies.
-  if (!input_is_control && can_dedup_control_with_regular_input) {
-    RemoveControllingFaninInternal(node, fanin.node);
+  // Update max input port and dedup control dependencies.
+  if (!input_is_control) {
+    max_regular_input_port()[node] = num_regular_fanins;
+    if (can_dedup_control_with_regular_input) {
+      RemoveControllingFaninInternal(node, fanin.node);
+    }
   }
 
   return true;
@@ -491,14 +496,12 @@ bool MutableGraphView::RemoveRegularFaninInternal(NodeDef* node,
 
   auto mutable_inputs = node->mutable_input();
   bool modified = false;
-  const int num_inputs = node->input_size();
+  const int num_regular_fanins =
+      NumFanins(*node, /*include_controlling_nodes=*/false);
   int i;
   int curr_pos = 0;
-  for (i = 0; i < num_inputs; ++i) {
+  for (i = 0; i < num_regular_fanins; ++i) {
     TensorId tensor_id = ParseTensorName(node->input(i));
-    if (IsTensorIdControlling(tensor_id)) {
-      break;
-    }
     if (tensor_id.node() == fanin.node->name() &&
         tensor_id.index() == fanin.port_id) {
       remove_input(fanin, i, /*update_max_port=*/true);
@@ -517,9 +520,17 @@ bool MutableGraphView::RemoveRegularFaninInternal(NodeDef* node,
     }
   }
 
-  if (modified && curr_pos < i) {
-    // Remove fanins from node inputs.
-    mutable_inputs->DeleteSubrange(curr_pos, i - curr_pos);
+  if (modified) {
+    const int last_regular_input_port = curr_pos - 1;
+    if (last_regular_input_port < 0) {
+      max_regular_input_port().erase(node);
+    } else {
+      max_regular_input_port()[node] = last_regular_input_port;
+    }
+    if (curr_pos < i) {
+      // Remove fanins from node inputs.
+      mutable_inputs->DeleteSubrange(curr_pos, i - curr_pos);
+    }
   }
 
   return modified;
@@ -601,14 +612,14 @@ Status MutableGraphView::RemoveAllFanins(absl::string_view node_name,
     return Status::OK();
   }
 
+  const int num_regular_fanins =
+      NumFanins(*node, /*include_controlling_nodes=*/false);
   RemoveFaninsInternal(node, keep_controlling_fanins);
   if (keep_controlling_fanins) {
-    int num_non_controlling_fanins =
-        NumFanins(*node, /*include_controlling_nodes=*/false);
-    if (num_non_controlling_fanins == 0) {
+    if (num_regular_fanins == 0) {
       return Status::OK();
-    } else if (num_non_controlling_fanins < node->input_size()) {
-      node->mutable_input()->DeleteSubrange(0, num_non_controlling_fanins);
+    } else if (num_regular_fanins < node->input_size()) {
+      node->mutable_input()->DeleteSubrange(0, num_regular_fanins);
     } else {
       node->clear_input();
     }
@@ -673,33 +684,27 @@ Status MutableGraphView::UpdateFanin(absl::string_view node_name,
     return Status::OK();
   }
 
-  // In place mutation, requires no shifting of ports.
+  // In place mutation of regular fanins, requires no shifting of ports.
   string to_fanin_string = TensorIdToString(to_fanin);
-  int num_inputs = node->input_size();
+  const int num_regular_fanins =
+      NumFanins(*node, /*include_controlling_nodes=*/false);
   bool modified = false;
   absl::flat_hash_set<InputPort>* from_fanin_port_fanouts = nullptr;
   absl::flat_hash_set<InputPort>* to_fanin_port_fanouts = nullptr;
-  for (int i = 0; i < num_inputs; ++i) {
+  for (int i = 0; i < num_regular_fanins; ++i) {
     if (ParseTensorName(node->input(i)) == from_fanin) {
-      InputPort old_input;
-      old_input.node = node;
-      old_input.port_id =
-          IsTensorIdControlling(from_fanin) ? Graph::kControlSlot : i;
+      InputPort input(node, i);
       if (from_fanin_port_fanouts == nullptr) {
         OutputPort from_fanin_port(from_fanin_node, from_fanin.index());
         from_fanin_port_fanouts = &fanouts()[from_fanin_port];
       }
-      from_fanin_port_fanouts->erase(old_input);
+      from_fanin_port_fanouts->erase(input);
 
-      InputPort new_input;
-      new_input.node = node;
-      new_input.port_id =
-          IsTensorIdControlling(to_fanin) ? Graph::kControlSlot : i;
       if (to_fanin_port_fanouts == nullptr) {
         OutputPort to_fanin_port(to_fanin_node, to_fanin.index());
         to_fanin_port_fanouts = &fanouts()[to_fanin_port];
       }
-      to_fanin_port_fanouts->insert(new_input);
+      to_fanin_port_fanouts->insert(input);
 
       node->set_input(i, to_fanin_string);
       modified = true;
@@ -824,14 +829,15 @@ void MutableGraphView::RemoveFaninsInternal(NodeDef* deleted_node,
                                             bool keep_controlling_fanins) {
   for (int i = 0; i < deleted_node->input_size(); ++i) {
     TensorId tensor_id = ParseTensorName(deleted_node->input(i));
-    if (keep_controlling_fanins && IsTensorIdControlling(tensor_id)) {
+    bool is_control = IsTensorIdControlling(tensor_id);
+    if (keep_controlling_fanins && is_control) {
       break;
     }
     OutputPort fanin(nodes()[tensor_id.node()], tensor_id.index());
 
     InputPort input;
     input.node = deleted_node;
-    input.port_id = IsTensorIdControlling(tensor_id) ? Graph::kControlSlot : i;
+    input.port_id = is_control ? Graph::kControlSlot : i;
 
     auto it = fanouts().find(fanin);
     if (it != fanouts().end()) {
@@ -840,11 +846,12 @@ void MutableGraphView::RemoveFaninsInternal(NodeDef* deleted_node,
       UpdateMaxRegularOutputPortForRemovedFanin(fanin, *fanouts_set);
     }
   }
+  max_regular_input_port().erase(deleted_node);
 }
 
 void MutableGraphView::RemoveFanoutsInternal(NodeDef* deleted_node) {
-  const int max_port = gtl::FindWithDefault(max_regular_output_port(),
-                                            deleted_node, Graph::kControlSlot);
+  const int max_port =
+      gtl::FindWithDefault(max_regular_output_port(), deleted_node, -1);
   for (int i = Graph::kControlSlot; i <= max_port; ++i) {
     fanouts().erase({deleted_node, i});
   }
-- 
GitLab


From cb40852a6ee804b12ab71aadb12babf96b090991 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 18 Jan 2019 13:09:17 -0800
Subject: [PATCH 0999/1095] Adds a layer normalization layer to core (So users
 can switch away from tf.contrib.layers.layer_norm)

PiperOrigin-RevId: 229980536
---
 tensorflow/python/keras/layers/__init__.py    |   1 +
 .../python/keras/layers/normalization.py      | 224 ++++++++++++++++-
 .../python/keras/layers/normalization_test.py | 229 ++++++++++++++++++
 .../tools/api/generator/api_init_files.bzl    |   1 +
 .../tools/api/generator/api_init_files_v1.bzl |   1 +
 ...rs.experimental.-layer-normalization.pbtxt | 183 ++++++++++++++
 ...tensorflow.keras.layers.experimental.pbtxt |   7 +
 .../golden/v1/tensorflow.keras.layers.pbtxt   |   4 +
 ...rs.experimental.-layer-normalization.pbtxt | 183 ++++++++++++++
 ...tensorflow.keras.layers.experimental.pbtxt |   7 +
 .../golden/v2/tensorflow.keras.layers.pbtxt   |   4 +
 11 files changed, 842 insertions(+), 2 deletions(-)
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.layers.experimental.-layer-normalization.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.layers.experimental.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.layers.experimental.-layer-normalization.pbtxt
 create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.layers.experimental.pbtxt

diff --git a/tensorflow/python/keras/layers/__init__.py b/tensorflow/python/keras/layers/__init__.py
index 285388f340..dec9673b5b 100644
--- a/tensorflow/python/keras/layers/__init__.py
+++ b/tensorflow/python/keras/layers/__init__.py
@@ -110,6 +110,7 @@ from tensorflow.python.keras.layers.noise import GaussianDropout
 
 # Normalization layers.
 from tensorflow.python.keras.layers.normalization import BatchNormalization
+from tensorflow.python.keras.layers.normalization import LayerNormalization
 
 # Pooling layers.
 from tensorflow.python.keras.layers.pooling import MaxPooling1D
diff --git a/tensorflow/python/keras/layers/normalization.py b/tensorflow/python/keras/layers/normalization.py
index 171e1e231e..5008bd7714 100644
--- a/tensorflow/python/keras/layers/normalization.py
+++ b/tensorflow/python/keras/layers/normalization.py
@@ -90,8 +90,7 @@ class BatchNormalizationV2(Layer):
       if the fused implementation cannot be used. If `None`, use the faster
       implementation if possible. If False, do not used the fused
       implementation.
-    trainable: Boolean, if `True` also add variables to the graph collection
-      `GraphKeys.TRAINABLE_VARIABLES` (see tf.Variable).
+    trainable: Boolean, if `True` the variables will be marked as trainable.
     virtual_batch_size: An `int`. By default, `virtual_batch_size` is `None`,
       which means batch normalization is performed across the whole batch. When
       `virtual_batch_size` is not `None`, instead perform "Ghost Batch
@@ -798,3 +797,224 @@ if tf2.enabled():
   enable_v2_batch_normalization()
 else:
   disable_v2_batch_normalization()
+
+
+@keras_export('keras.layers.experimental.LayerNormalization')
+class LayerNormalization(Layer):
+  """Layer normalization layer (Ba et al., 2016).
+
+  Normalize the activations of the previous layer for each given example in a
+  batch independently, rather than across a batch like Batch Normalization.
+  i.e. applies a transformation that maintains the mean activation within each
+  example close to 0 and the activation standard deviation close to 1.
+
+  Given a tensor `inputs` of rank `R`, moments are calculated and normalization
+  is performed over all axes in norm_axis.  Scaling and centering,
+  if requested, is performed over all axes in params_axis.
+
+  By default, normalization is performed over all but the first axis
+  (the `HWC` if `inputs` is `NHWC`), while the `beta` and `gamma` trainable
+  parameters are calculated for the rightmost axis (the `C` if `inputs` is
+  `NHWC`).  Scaling and recentering is performed via broadcast of the
+  `beta` and `gamma` parameters with the normalized tensor.
+
+  The shapes of `beta` and `gamma` are
+  `[inputs.shape[i] for i in (param axes)]`,
+  and this part of the inputs' shape must be fully defined.
+
+  Arguments:
+    norm_axis: Integer or List. normalization will be
+      performed along these dimensions. If unspecified (None), it will default
+      to the dimensions `begin_norm_axis : rank(inputs)`
+    params_axis: Integer or List. The (beta, gamma) dimensions: scale
+      and centering parameters will have take their shapes from these axes and
+      will be broadcast with the normalized inputs accordingly. If unspecified
+      (None), it will default to the last dimension
+    epsilon: Small float added to variance to avoid dividing by zero.
+    center: If True, add offset of `beta` to normalized tensor.
+        If False, `beta` is ignored.
+    scale: If True, multiply by `gamma`.
+        If False, `gamma` is not used.
+        When the next layer is linear (also e.g. `nn.relu`),
+        this can be disabled since the scaling
+        will be done by the next layer.
+    beta_initializer: Initializer for the beta weight.
+    gamma_initializer: Initializer for the gamma weight.
+    beta_regularizer: Optional regularizer for the beta weight.
+    gamma_regularizer: Optional regularizer for the gamma weight.
+    beta_constraint: Optional constraint for the beta weight.
+    gamma_constraint: Optional constraint for the gamma weight.
+    trainable: Boolean, if `True` the variables will be marked as trainable.
+
+  Input shape:
+      Arbitrary. Use the keyword argument `input_shape`
+      (tuple of integers, does not include the samples axis)
+      when using this layer as the first layer in a model.
+
+  Output shape:
+      Same shape as input.
+
+  References:
+      - [Layer Normalization](https://arxiv.org/abs/1607.06450)
+  """
+
+  def __init__(self,
+               norm_axis=None,
+               params_axis=-1,
+               epsilon=1e-12,
+               center=True,
+               scale=True,
+               beta_initializer='zeros',
+               gamma_initializer='ones',
+               beta_regularizer=None,
+               gamma_regularizer=None,
+               beta_constraint=None,
+               gamma_constraint=None,
+               trainable=True,
+               name=None,
+               **kwargs):
+    super(LayerNormalization, self).__init__(
+        name=name, trainable=trainable, **kwargs)
+    if isinstance(norm_axis, list):
+      self.norm_axis = norm_axis[:]
+    elif isinstance(norm_axis, int):
+      self.norm_axis = norm_axis
+    elif norm_axis is None:
+      self.norm_axis = None
+    else:
+      raise TypeError('norm_axis must be int or list or None, type given: %s'
+                      % type(norm_axis))
+
+    if isinstance(params_axis, list):
+      self.params_axis = params_axis[:]
+    elif isinstance(params_axis, int):
+      self.params_axis = params_axis
+    else:
+      raise TypeError('params_axis must be int or list, type given: %s'
+                      % type(params_axis))
+
+    self.epsilon = epsilon
+    self.center = center
+    self.scale = scale
+    self.beta_initializer = initializers.get(beta_initializer)
+    self.gamma_initializer = initializers.get(gamma_initializer)
+    self.beta_regularizer = regularizers.get(beta_regularizer)
+    self.gamma_regularizer = regularizers.get(gamma_regularizer)
+    self.beta_constraint = constraints.get(beta_constraint)
+    self.gamma_constraint = constraints.get(gamma_constraint)
+
+    self.supports_masking = True
+
+  def build(self, input_shape):
+    ndims = len(input_shape)
+    if ndims is None:
+      raise ValueError('Input shape %s has undefined rank.' % input_shape)
+
+    # Handle an unspecified norm_axis
+    if self.norm_axis is None:
+      self.norm_axis = list(range(1, ndims))
+
+    # Convert axes to lists and resolve negatives
+    if isinstance(self.norm_axis, int):
+      self.norm_axis = [self.norm_axis]
+    for idx, x in enumerate(self.norm_axis):
+      if x < 0:
+        self.norm_axis[idx] = ndims + x
+
+    if isinstance(self.params_axis, int):
+      self.params_axis = [self.params_axis]
+    for idx, x in enumerate(self.params_axis):
+      if x < 0:
+        self.params_axis[idx] = ndims + x
+
+    # Validate axes
+    for x in self.norm_axis:
+      if x < 0 or x >= ndims:
+        raise ValueError('Invalid axis: %d' % x)
+    if len(self.norm_axis) != len(set(self.norm_axis)):
+      raise ValueError('Duplicate axis: %s' % self.norm_axis)
+
+    for x in self.params_axis:
+      if x < 0 or x >= ndims:
+        raise ValueError('Invalid axis: %d' % x)
+    if len(self.params_axis) != len(set(self.params_axis)):
+      raise ValueError('Duplicate axis: %s' % self.params_axis)
+
+    param_shape = [input_shape[dim] for dim in self.params_axis]
+
+    if self.scale:
+      self.gamma = self.add_weight(
+          name='gamma',
+          shape=param_shape,
+          initializer=self.gamma_initializer,
+          regularizer=self.gamma_regularizer,
+          constraint=self.gamma_constraint,
+          trainable=True)
+    else:
+      self.gamma = None
+
+    if self.center:
+      self.beta = self.add_weight(
+          name='beta',
+          shape=param_shape,
+          initializer=self.beta_initializer,
+          regularizer=self.beta_regularizer,
+          constraint=self.beta_constraint,
+          trainable=True)
+    else:
+      self.beta = None
+
+  def call(self, inputs):
+    # Compute the axes along which to reduce the mean / variance
+    input_shape = inputs.get_shape()
+    ndims = len(input_shape)
+
+    # Calculate the moments on the last axis (layer activations).
+    mean, variance = nn.moments(inputs, self.norm_axis, keep_dims=True)
+
+    # Broadcasting only necessary for norm where the params axes aren't just
+    # the last dimension
+    broadcast_shape = [1] * ndims
+    for dim in self.params_axis:
+      broadcast_shape[dim] = input_shape.dims[dim].value
+    def _broadcast(v):
+      if (v is not None and
+          len(v.get_shape()) != ndims and
+          self.params_axis != [ndims - 1]):
+        return array_ops.reshape(v, broadcast_shape)
+      return v
+    scale, offset = _broadcast(self.gamma), _broadcast(self.beta)
+
+    # Compute layer normalization using the batch_normalization function.
+    outputs = nn.batch_normalization(
+        inputs,
+        mean,
+        variance,
+        offset=offset,
+        scale=scale,
+        variance_epsilon=self.epsilon)
+
+    # If some components of the shape got lost due to adjustments, fix that.
+    outputs.set_shape(input_shape)
+
+    return outputs
+
+  def compute_output_shape(self, input_shape):
+    return input_shape
+
+  def get_config(self):
+    config = {
+        'norm_axis': self.norm_axis,
+        'params_axis': self.params_axis,
+        'epsilon': self.epsilon,
+        'center': self.center,
+        'scale': self.scale,
+        'beta_initializer': initializers.serialize(self.beta_initializer),
+        'gamma_initializer': initializers.serialize(self.gamma_initializer),
+        'beta_regularizer': regularizers.serialize(self.beta_regularizer),
+        'gamma_regularizer': regularizers.serialize(self.gamma_regularizer),
+        'beta_constraint': constraints.serialize(self.beta_constraint),
+        'gamma_constraint': constraints.serialize(self.gamma_constraint)
+    }
+    base_config = super(LayerNormalization, self).get_config()
+    return dict(list(base_config.items()) + list(config.items()))
diff --git a/tensorflow/python/keras/layers/normalization_test.py b/tensorflow/python/keras/layers/normalization_test.py
index f81ddcecb4..4308e0f724 100644
--- a/tensorflow/python/keras/layers/normalization_test.py
+++ b/tensorflow/python/keras/layers/normalization_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python import keras
+from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import test_util as tf_test_util
 from tensorflow.python.keras import keras_parameterized
 from tensorflow.python.keras import testing_utils
@@ -363,5 +364,233 @@ class NormalizationLayersGraphModeOnlyTest(test.TestCase):
           (val_a - np.mean(val_a)) / np.std(val_a), out, atol=1e-3)
 
 
+def _run_layernorm_correctness_test(layer, dtype='float32'):
+  model = keras.models.Sequential()
+  norm = layer(input_shape=(2, 2, 2))
+  model.add(norm)
+  model.compile(loss='mse',
+                optimizer=gradient_descent.GradientDescentOptimizer(0.01),
+                run_eagerly=testing_utils.should_run_eagerly())
+
+  # centered on 5.0, variance 10.0
+  x = (np.random.normal(loc=5.0, scale=10.0, size=(1000, 2, 2, 2))
+       .astype(dtype))
+  model.fit(x, x, epochs=4, verbose=0)
+  out = model.predict(x)
+  out -= keras.backend.eval(norm.beta)
+  out /= keras.backend.eval(norm.gamma)
+
+  np.testing.assert_allclose(out.mean(), 0.0, atol=1e-1)
+  np.testing.assert_allclose(out.std(), 1.0, atol=1e-1)
+
+
+class LayerNormalizationTest(keras_parameterized.TestCase):
+
+  @keras_parameterized.run_all_keras_modes
+  def test_basic_layernorm(self):
+    testing_utils.layer_test(
+        keras.layers.LayerNormalization,
+        kwargs={
+            'gamma_regularizer': keras.regularizers.l2(0.01),
+            'beta_regularizer': keras.regularizers.l2(0.01)
+        },
+        input_shape=(3, 4, 2))
+    testing_utils.layer_test(
+        keras.layers.LayerNormalization,
+        kwargs={
+            'gamma_initializer': 'ones',
+            'beta_initializer': 'ones',
+        },
+        input_shape=(3, 4, 2))
+    testing_utils.layer_test(
+        keras.layers.LayerNormalization,
+        kwargs={'scale': False,
+                'center': False},
+        input_shape=(3, 3))
+
+  @tf_test_util.run_in_graph_and_eager_modes
+  def test_layernorm_weights(self):
+    layer = keras.layers.LayerNormalization(scale=False, center=False)
+    layer.build((None, 3, 4))
+    self.assertEqual(len(layer.trainable_weights), 0)
+    self.assertEqual(len(layer.weights), 0)
+
+    layer = keras.layers.LayerNormalization()
+    layer.build((None, 3, 4))
+    self.assertEqual(len(layer.trainable_weights), 2)
+    self.assertEqual(len(layer.weights), 2)
+
+  @tf_test_util.run_in_graph_and_eager_modes
+  def test_layernorm_regularization(self):
+    layer = keras.layers.LayerNormalization(
+        gamma_regularizer='l1', beta_regularizer='l1')
+    layer.build((None, 3, 4))
+    self.assertEqual(len(layer.losses), 2)
+    max_norm = keras.constraints.max_norm
+    layer = keras.layers.LayerNormalization(
+        gamma_constraint=max_norm, beta_constraint=max_norm)
+    layer.build((None, 3, 4))
+    self.assertEqual(layer.gamma.constraint, max_norm)
+    self.assertEqual(layer.beta.constraint, max_norm)
+
+  @keras_parameterized.run_all_keras_modes
+  def test_layernorm_convnet(self):
+    if test.is_gpu_available(cuda_only=True):
+      with self.session(use_gpu=True):
+        model = keras.models.Sequential()
+        norm = keras.layers.LayerNormalization(input_shape=(3, 4, 4))
+        model.add(norm)
+        model.compile(loss='mse',
+                      optimizer=gradient_descent.GradientDescentOptimizer(0.01),
+                      run_eagerly=testing_utils.should_run_eagerly())
+
+        # centered on 5.0, variance 10.0
+        x = np.random.normal(loc=5.0, scale=10.0, size=(1000, 3, 4, 4))
+        model.fit(x, x, epochs=4, verbose=0)
+        out = model.predict(x)
+        out -= np.reshape(keras.backend.eval(norm.beta), (1, 3, 1, 1))
+        out /= np.reshape(keras.backend.eval(norm.gamma), (1, 3, 1, 1))
+
+        np.testing.assert_allclose(np.mean(out, axis=(0, 2, 3)), 0.0, atol=1e-1)
+        np.testing.assert_allclose(np.std(out, axis=(0, 2, 3)), 1.0, atol=1e-1)
+
+  @keras_parameterized.run_all_keras_modes
+  def test_layernorm_convnet_channel_last(self):
+    model = keras.models.Sequential()
+    norm = keras.layers.LayerNormalization(input_shape=(4, 4, 3))
+    model.add(norm)
+    model.compile(loss='mse',
+                  optimizer=gradient_descent.GradientDescentOptimizer(0.01),
+                  run_eagerly=testing_utils.should_run_eagerly())
+
+    # centered on 5.0, variance 10.0
+    x = np.random.normal(loc=5.0, scale=10.0, size=(1000, 4, 4, 3))
+    model.fit(x, x, epochs=4, verbose=0)
+    out = model.predict(x)
+    out -= np.reshape(keras.backend.eval(norm.beta), (1, 1, 1, 3))
+    out /= np.reshape(keras.backend.eval(norm.gamma), (1, 1, 1, 3))
+
+    np.testing.assert_allclose(np.mean(out, axis=(0, 1, 2)), 0.0, atol=1e-1)
+    np.testing.assert_allclose(np.std(out, axis=(0, 1, 2)), 1.0, atol=1e-1)
+
+  @keras_parameterized.run_all_keras_modes
+  def test_layernorm_correctness(self):
+    _run_layernorm_correctness_test(
+        normalization.LayerNormalization, dtype='float32')
+
+  @keras_parameterized.run_all_keras_modes
+  def test_layernorm_mixed_precision(self):
+    _run_layernorm_correctness_test(
+        normalization.LayerNormalization, dtype='float16')
+
+  def doOutputTest(self,
+                   input_shape,
+                   tol=1e-5,
+                   norm_axis=None,
+                   params_axis=-1,
+                   dtype=None):
+    ndim = len(input_shape)
+    if norm_axis is None:
+      moments_axis = range(1, ndim)
+    elif isinstance(norm_axis, int):
+      if norm_axis < 0:
+        moments_axis = [norm_axis + ndim]
+      else:
+        moments_axis = [norm_axis]
+    else:
+      moments_axis = []
+      for dim in norm_axis:
+        if dim < 0:
+          dim = dim + ndim
+        moments_axis.append(dim)
+
+    moments_axis = tuple(moments_axis)
+    expected_shape = []
+    for i in range(ndim):
+      if i not in moments_axis:
+        expected_shape.append(input_shape[i])
+
+    expected_mean = np.zeros(expected_shape)
+    expected_var = np.ones(expected_shape)
+    for mu in [0.0, 1e2]:
+      for sigma in [1.0, 0.1]:
+        inputs = np.random.randn(*input_shape) * sigma + mu
+        inputs_t = constant_op.constant(inputs, shape=input_shape)
+        layer = normalization.LayerNormalization(
+            norm_axis=norm_axis, params_axis=params_axis, dtype=dtype)
+        outputs = layer(inputs_t)
+        beta = layer.beta
+        gamma = layer.gamma
+        for weight in layer.weights:
+          self.evaluate(weight.initializer)
+        outputs = self.evaluate(outputs)
+        beta = self.evaluate(beta)
+        gamma = self.evaluate(gamma)
+
+        # The mean and variance of the output should be close to 0 and 1
+        # respectively.
+
+        # Make sure that there are no NaNs
+        self.assertFalse(np.isnan(outputs).any())
+        mean = np.mean(outputs, axis=moments_axis)
+        var = np.var(outputs, axis=moments_axis)
+        # Layer-norm implemented in numpy
+        eps = 1e-12
+        expected_out = (
+            (gamma * (inputs - np.mean(
+                inputs, axis=moments_axis, keepdims=True)) /
+             np.sqrt(eps + np.var(
+                 inputs, axis=moments_axis, keepdims=True))) + beta)
+        self.assertAllClose(expected_mean, mean, atol=tol, rtol=tol)
+        self.assertAllClose(expected_var, var, atol=tol)
+        # The full computation gets a bigger tolerance
+        self.assertAllClose(expected_out, outputs, atol=5 * tol)
+
+  @tf_test_util.run_in_graph_and_eager_modes
+  def testOutput2DInput(self):
+    self.doOutputTest((10, 300))
+    self.doOutputTest((10, 300), norm_axis=[0])
+    self.doOutputTest((10, 300), params_axis=[0, 1])
+
+  @tf_test_util.run_in_graph_and_eager_modes
+  def testOutput2DInputDegenerateNormAxis(self):
+    with self.assertRaisesRegexp(ValueError, r'Invalid axis: 2'):
+      self.doOutputTest((10, 300), norm_axis=2)
+
+  @tf_test_util.run_in_graph_and_eager_modes
+  def testOutput4DInput(self):
+    self.doOutputTest((100, 10, 10, 3))
+
+  @tf_test_util.run_in_graph_and_eager_modes
+  def testOutput4DInputNormOnInnermostAxis(self):
+    # Equivalent tests
+    shape = (100, 10, 10, 3)
+    self.doOutputTest(
+        shape, norm_axis=list(range(3, len(shape))), tol=1e-4, dtype='float64')
+    self.doOutputTest(shape, norm_axis=-1, tol=1e-4, dtype='float64')
+
+  @tf_test_util.run_in_graph_and_eager_modes
+  def testOutputSmallInput(self):
+    self.doOutputTest((10, 10, 10, 30))
+
+  @tf_test_util.run_in_graph_and_eager_modes
+  def testOutputSmallInputNormOnInnermostAxis(self):
+    self.doOutputTest((10, 10, 10, 30), norm_axis=3)
+
+  @tf_test_util.run_in_graph_and_eager_modes
+  def testOutputSmallInputNormOnMixedAxes(self):
+    self.doOutputTest((10, 10, 10, 30), norm_axis=[0, 3])
+    self.doOutputTest((10, 10, 10, 30), params_axis=[-2, -1])
+    self.doOutputTest((10, 10, 10, 30), norm_axis=[0, 3],
+                      params_axis=[-3, -2, -1])
+
+  @tf_test_util.run_in_graph_and_eager_modes
+  def testOutputBigInput(self):
+    self.doOutputTest((1, 100, 100, 1))
+    self.doOutputTest((1, 100, 100, 1), norm_axis=[1, 2])
+    self.doOutputTest((1, 100, 100, 1), norm_axis=[1, 2],
+                      params_axis=[-2, -1])
+
+
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/tools/api/generator/api_init_files.bzl b/tensorflow/python/tools/api/generator/api_init_files.bzl
index 5182f53beb..7637cbd584 100644
--- a/tensorflow/python/tools/api/generator/api_init_files.bzl
+++ b/tensorflow/python/tools/api/generator/api_init_files.bzl
@@ -77,6 +77,7 @@ KERAS_API_INIT_FILES = [
     "keras/experimental/__init__.py",
     "keras/initializers/__init__.py",
     "keras/layers/__init__.py",
+    "keras/layers/experimental/__init__.py",
     "keras/losses/__init__.py",
     "keras/metrics/__init__.py",
     "keras/models/__init__.py",
diff --git a/tensorflow/python/tools/api/generator/api_init_files_v1.bzl b/tensorflow/python/tools/api/generator/api_init_files_v1.bzl
index 3e2c00c87f..5db84b1d0b 100644
--- a/tensorflow/python/tools/api/generator/api_init_files_v1.bzl
+++ b/tensorflow/python/tools/api/generator/api_init_files_v1.bzl
@@ -100,6 +100,7 @@ KERAS_API_INIT_FILES_V1 = [
     "keras/experimental/__init__.py",
     "keras/initializers/__init__.py",
     "keras/layers/__init__.py",
+    "keras/layers/experimental/__init__.py",
     "keras/losses/__init__.py",
     "keras/metrics/__init__.py",
     "keras/models/__init__.py",
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.experimental.-layer-normalization.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.experimental.-layer-normalization.pbtxt
new file mode 100644
index 0000000000..476c597fe6
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.experimental.-layer-normalization.pbtxt
@@ -0,0 +1,183 @@
+path: "tensorflow.keras.layers.experimental.LayerNormalization"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.layers.normalization.LayerNormalization\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'norm_axis\', \'params_axis\', \'epsilon\', \'center\', \'scale\', \'beta_initializer\', \'gamma_initializer\', \'beta_regularizer\', \'gamma_regularizer\', \'beta_constraint\', \'gamma_constraint\', \'trainable\', \'name\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'-1\', \'1e-12\', \'True\', \'True\', \'zeros\', \'ones\', \'None\', \'None\', \'None\', \'None\', \'True\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.experimental.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.experimental.pbtxt
new file mode 100644
index 0000000000..0f22961546
--- /dev/null
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.experimental.pbtxt
@@ -0,0 +1,7 @@
+path: "tensorflow.keras.layers.experimental"
+tf_module {
+  member {
+    name: "LayerNormalization"
+    mtype: "<type \'type\'>"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.pbtxt
index 9d7e5bb8c7..2c5fd644d4 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.pbtxt
@@ -396,6 +396,10 @@ tf_module {
     name: "ZeroPadding3D"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "experimental"
+    mtype: "<type \'module\'>"
+  }
   member_method {
     name: "Input"
     argspec: "args=[\'shape\', \'batch_size\', \'name\', \'dtype\', \'sparse\', \'tensor\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'False\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.experimental.-layer-normalization.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.experimental.-layer-normalization.pbtxt
new file mode 100644
index 0000000000..476c597fe6
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.experimental.-layer-normalization.pbtxt
@@ -0,0 +1,183 @@
+path: "tensorflow.keras.layers.experimental.LayerNormalization"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras.layers.normalization.LayerNormalization\'>"
+  is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
+  is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "activity_regularizer"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dynamic"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "inbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "losses"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "non_trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "outbound_nodes"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_mask"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "output_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "trainable_weights"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "updates"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "variables"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "weights"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'norm_axis\', \'params_axis\', \'epsilon\', \'center\', \'scale\', \'beta_initializer\', \'gamma_initializer\', \'beta_regularizer\', \'gamma_regularizer\', \'beta_constraint\', \'gamma_constraint\', \'trainable\', \'name\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'-1\', \'1e-12\', \'True\', \'True\', \'zeros\', \'ones\', \'None\', \'None\', \'None\', \'None\', \'True\', \'None\'], "
+  }
+  member_method {
+    name: "add_loss"
+    argspec: "args=[\'self\', \'losses\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_metric"
+    argspec: "args=[\'self\', \'value\', \'aggregation\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_update"
+    argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "add_variable"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "add_weight"
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
+  }
+  member_method {
+    name: "apply"
+    argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+  member_method {
+    name: "build"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "call"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "compute_mask"
+    argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "compute_output_shape"
+    argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "count_params"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "from_config"
+    argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_config"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_input_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_losses_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_mask_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_output_shape_at"
+    argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_updates_for"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_weights"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "set_weights"
+    argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.experimental.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.experimental.pbtxt
new file mode 100644
index 0000000000..0f22961546
--- /dev/null
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.experimental.pbtxt
@@ -0,0 +1,7 @@
+path: "tensorflow.keras.layers.experimental"
+tf_module {
+  member {
+    name: "LayerNormalization"
+    mtype: "<type \'type\'>"
+  }
+}
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.pbtxt
index e84c9a2a8f..5cf80b97ae 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.pbtxt
@@ -396,6 +396,10 @@ tf_module {
     name: "ZeroPadding3D"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "experimental"
+    mtype: "<type \'module\'>"
+  }
   member_method {
     name: "Input"
     argspec: "args=[\'shape\', \'batch_size\', \'name\', \'dtype\', \'sparse\', \'tensor\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'False\', \'None\'], "
-- 
GitLab


From 07f552d779abdb4b90ed66e13e2eeef465f2c47a Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Fri, 18 Jan 2019 13:09:34 -0800
Subject: [PATCH 1000/1095] Add a way to check if a function exists in the
 eager context

Simplifies SavedModel loading slightly

PiperOrigin-RevId: 229980590
---
 tensorflow/c/eager/c_api.cc                    |  4 ++++
 tensorflow/c/eager/c_api.h                     |  4 ++++
 tensorflow/python/eager/context.py             |  4 ++++
 tensorflow/python/eager/pywrap_tfe_src.cc      |  8 ++++----
 tensorflow/python/eager/pywrap_tfe_test.py     | 11 +++++++++++
 .../python/framework/function_def_to_graph.py  | 13 +++++++++++--
 tensorflow/python/pywrap_tfe.i                 |  8 ++++++++
 .../saved_model/function_deserialization.py    | 18 +++++-------------
 8 files changed, 51 insertions(+), 19 deletions(-)

diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc
index f0408677d4..e2574d5311 100755
--- a/tensorflow/c/eager/c_api.cc
+++ b/tensorflow/c/eager/c_api.cc
@@ -754,6 +754,10 @@ void TFE_ContextAddFunction(TFE_Context* ctx, TF_Function* function,
   status->status = ctx->context.AddFunctionDef(function->fdef);
 }
 
+unsigned char TFE_ContextHasFunction(TFE_Context* ctx, const char* name) {
+  return ctx->context.FindFunctionDef(name) != nullptr;
+}
+
 void TFE_ContextEnableRunMetadata(TFE_Context* ctx) {
   ctx->context.SetShouldStoreMetadata(true);
 }
diff --git a/tensorflow/c/eager/c_api.h b/tensorflow/c/eager/c_api.h
index 120748ab76..044dfb7415 100755
--- a/tensorflow/c/eager/c_api.h
+++ b/tensorflow/c/eager/c_api.h
@@ -393,6 +393,10 @@ TF_CAPI_EXPORT extern void TFE_ContextAddFunction(TFE_Context* ctx,
                                                   TF_Function* function,
                                                   TF_Status* status);
 
+// Checks whether a function is registered under `name`.
+TF_CAPI_EXPORT unsigned char TFE_ContextHasFunction(TFE_Context* ctx,
+                                                    const char* name);
+
 // Enables tracing of RunMetadata on the ops executed from this context.
 TF_CAPI_EXPORT extern void TFE_ContextEnableRunMetadata(TFE_Context* ctx);
 
diff --git a/tensorflow/python/eager/context.py b/tensorflow/python/eager/context.py
index 6c5b191cf9..33551fccd5 100644
--- a/tensorflow/python/eager/context.py
+++ b/tensorflow/python/eager/context.py
@@ -643,6 +643,10 @@ class Context(object):
     pywrap_tensorflow.TFE_ContextAddFunctionDef(
         self._handle, fdef_string, len(fdef_string))
 
+  def has_function(self, name):
+    """Check if a function `name` is registered."""
+    return bool(pywrap_tensorflow.TFE_ContextHasFunction(self._handle, name))
+
   def add_post_execution_callback(self, callback):
     """Add a post-execution callback to the context.
 
diff --git a/tensorflow/python/eager/pywrap_tfe_src.cc b/tensorflow/python/eager/pywrap_tfe_src.cc
index 9ce500bc08..da1bb24fc7 100644
--- a/tensorflow/python/eager/pywrap_tfe_src.cc
+++ b/tensorflow/python/eager/pywrap_tfe_src.cc
@@ -835,15 +835,15 @@ int MaybeRaiseExceptionFromStatus(const tensorflow::Status& status,
 }
 
 const char* TFE_GetPythonString(PyObject* o) {
+#if PY_MAJOR_VERSION >= 3
   if (PyBytes_Check(o)) {
     return PyBytes_AsString(o);
-  }
-#if PY_MAJOR_VERSION >= 3
-  if (PyUnicode_Check(o)) {
+  } else {
     return PyUnicode_AsUTF8(o);
   }
+#else
+  return PyBytes_AsString(o);
 #endif
-  return nullptr;
 }
 
 int64_t get_uid() {
diff --git a/tensorflow/python/eager/pywrap_tfe_test.py b/tensorflow/python/eager/pywrap_tfe_test.py
index 2abd4678af..445ffbc532 100644
--- a/tensorflow/python/eager/pywrap_tfe_test.py
+++ b/tensorflow/python/eager/pywrap_tfe_test.py
@@ -22,6 +22,7 @@ from tensorflow.python import pywrap_tensorflow
 from tensorflow.python.eager import backprop
 from tensorflow.python.eager import context
 from tensorflow.python.eager import core
+from tensorflow.python.eager import def_function
 from tensorflow.python.eager import test
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -259,6 +260,16 @@ class Tests(test.TestCase):
         "Value for attr 'num_split' of 0 must be at least minimum 1"):
       array_ops.split(value=[1, 2, 3], num_or_size_splits=0)
 
+  def testIsFunction(self):
+    ctx = context.context()
+    self.assertFalse(ctx.has_function("not_a_function"))
+
+    @def_function.function
+    def f():
+      return 1.
+
+    self.assertTrue(ctx.has_function(f.get_concrete_function().name))
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/framework/function_def_to_graph.py b/tensorflow/python/framework/function_def_to_graph.py
index 10ad7ad541..aa670f1e37 100644
--- a/tensorflow/python/framework/function_def_to_graph.py
+++ b/tensorflow/python/framework/function_def_to_graph.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 from tensorflow.core.framework import graph_pb2
 from tensorflow.core.framework import types_pb2
 from tensorflow.core.framework import versions_pb2
+from tensorflow.python.eager import context
 from tensorflow.python.framework import importer
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import versions
@@ -76,6 +77,14 @@ def function_def_to_graph(fdef, input_shapes=None):
   return func_graph
 
 
+def _is_function(fname):
+  """Checks for a function definition with `fname` in the current context."""
+  if context.executing_eagerly():
+    return context.context().has_function(fname)
+  else:
+    return ops.get_default_graph()._is_function(fname)  # pylint: disable=protected-access
+
+
 def function_def_to_graph_def(fdef, input_shapes=None):
   """Convert a FunctionDef to a GraphDef.
 
@@ -147,12 +156,12 @@ def function_def_to_graph_def(fdef, input_shapes=None):
     for attr in op_def.attr:
       if attr.type == "func":
         fname = node_def.attr[attr.name].func.name
-        if not ops.get_default_graph()._is_function(fname):  # pylint: disable=protected-access
+        if not _is_function(fname):
           raise ValueError("%s function not found." % fname)
       elif attr.type == "list(func)":
         for fn in node_def.attr[attr.name].list.func:
           fname = fn.name
-          if not ops.get_default_graph()._is_function(fname):  # pylint: disable=protected-access
+          if not _is_function(fname):
             raise ValueError("%s function not found." % fname)
 
     # Iterate over output_args in op_def to build the map.
diff --git a/tensorflow/python/pywrap_tfe.i b/tensorflow/python/pywrap_tfe.i
index d0ba244853..5f6fbdcb5e 100755
--- a/tensorflow/python/pywrap_tfe.i
+++ b/tensorflow/python/pywrap_tfe.i
@@ -22,6 +22,7 @@ limitations under the License.
 %rename("%s") TFE_ContextListDevices;
 %rename("%s") TFE_ContextAddFunction;
 %rename("%s") TFE_ContextAddFunctionDef;
+%rename("%s") TFE_ContextHasFunction;
 %rename("%s") TFE_ContextEnableRunMetadata;
 %rename("%s") TFE_ContextDisableRunMetadata;
 %rename("%s") TFE_ContextExportRunMetadata;
@@ -140,6 +141,13 @@ limitations under the License.
   $1 = const_cast<char*>(TFE_GetPythonString($input));
 }
 
+// For const parameters in a function, SWIG pretty much ignores the const.
+// See: http://www.swig.org/Doc2.0/SWIG.html#SWIG_nn13
+// Hence the 'const_cast'.
+%typemap(in) const char* name {
+  $1 = const_cast<char*>(TFE_GetPythonString($input));
+}
+
 %typemap(in) (TFE_Context*) {
   $1 = (TFE_Context*)PyCapsule_GetPointer($input, nullptr);
 
diff --git a/tensorflow/python/saved_model/function_deserialization.py b/tensorflow/python/saved_model/function_deserialization.py
index f6427864bb..934e189f78 100644
--- a/tensorflow/python/saved_model/function_deserialization.py
+++ b/tensorflow/python/saved_model/function_deserialization.py
@@ -164,22 +164,14 @@ def load_function_def_library(library):
   """
   functions = {}
 
-  # Note: Use a new graph to allow function_def_to_graph to help validating
-  # that the functions are loaded correctly. This is not possible to do
-  # just in eager mode as there is no python API to find if a function has
-  # been registered in eager. Note also that despite this the created
-  # func_graphs can still be used in eager or in other graphs.
-  import_graph = ops.Graph()
-
   for fdef in _sort_function_defs(library):
-    with import_graph.as_default():
-      copy = _fix_fdef(fdef, functions)
+    copy = _fix_fdef(fdef, functions)
 
-      func_graph = function_def_lib.function_def_to_graph(copy)
-      func = function_lib.ConcreteFunction(func_graph)
-      func.add_to_graph(import_graph)
+    func_graph = function_def_lib.function_def_to_graph(copy)
+    func = function_lib.ConcreteFunction(func_graph)
+    func.add_to_graph()
 
-      functions[fdef.signature.name] = func
+    functions[fdef.signature.name] = func
 
     # Also register the gradients in the current root context.
     with ops.init_scope():
-- 
GitLab


From 7a283b835b80808a9cdc6043998f29c8188342a5 Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Fri, 18 Jan 2019 13:12:18 -0800
Subject: [PATCH 1001/1095] [XLA] Fix invalid return type for Real() of C128 in
 HLO evaluator. Add a test.

PiperOrigin-RevId: 229981105
---
 tensorflow/compiler/xla/service/hlo_evaluator.cc    |  2 +-
 .../compiler/xla/service/hlo_evaluator_test.cc      | 13 +++++++++++++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc
index 1d9af45745..a40ded76cf 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc
@@ -542,7 +542,7 @@ Status HloEvaluator::HandleReal(HloInstruction* real) {
       break;
     }
     case C128: {
-      auto result_or = ElementWiseUnaryOpImpl<float, complex128>(
+      auto result_or = ElementWiseUnaryOpImpl<double, complex128>(
           real, [](complex128 elem_operand) { return std::real(elem_operand); },
           GetEvaluatedLiteralFor(operand));
       TF_ASSIGN_OR_RETURN(evaluated_[real], std::move(result_or));
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_test.cc b/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
index d34fa48efb..590f76f472 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
@@ -308,6 +308,19 @@ TEST_F(HloEvaluatorTest, DoesNotR2) {
                                     {0, std::numeric_limits<int>::min()}});
   TestUnaryOp(HloOpcode::kNot, std::move(expected), std::move(operand));
 }
+
+TEST_F(HloEvaluatorTest, DoesRealC128) {
+  auto x = LiteralUtil::CreateR1<complex128>({{1, 0}, {-100, 4}});
+  auto expected_real = LiteralUtil::CreateR1<double>({1, -100});
+  TestUnaryOp(HloOpcode::kReal, std::move(expected_real), std::move(x));
+}
+
+TEST_F(HloEvaluatorTest, DoesImagC128) {
+  auto x = LiteralUtil::CreateR1<complex128>({{1, 0}, {-100, 4}});
+  auto expected_imag = LiteralUtil::CreateR1<double>({0, 4});
+  TestUnaryOp(HloOpcode::kImag, std::move(expected_imag), std::move(x));
+}
+
 // Verifies that HloEvaluator evaluates a HLO Computation with non-parameter nor
 // constant operands.
 TEST_F(HloEvaluatorTest, DoesTraverseInstructions) {
-- 
GitLab


From 5ee41c266b389c54fddb73a94db22c9f5259374e Mon Sep 17 00:00:00 2001
From: Shashi Shekhar <shashishekhar@google.com>
Date: Fri, 18 Jan 2019 13:36:30 -0800
Subject: [PATCH 1002/1095] Fix check for loggable buffers.

PiperOrigin-RevId: 229985133
---
 tensorflow/lite/tools/optimize/calibrator.cc | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/tools/optimize/calibrator.cc b/tensorflow/lite/tools/optimize/calibrator.cc
index 0e817f9346..ccb55c3081 100644
--- a/tensorflow/lite/tools/optimize/calibrator.cc
+++ b/tensorflow/lite/tools/optimize/calibrator.cc
@@ -199,8 +199,10 @@ std::vector<int> GetLoggableTensorIndices(
   for (auto tensor_index : tensor_indices) {
     auto tensor = tensors->Get(tensor_index);
     auto buffer_index = tensor->buffer();
-    bool has_no_buffer =
-        buffer_index == 0 || (tensor_buffers->Get(buffer_index) == nullptr);
+    const bool has_no_buffer =
+        (tensor_buffers->Get(buffer_index) == nullptr) ||
+        (tensor_buffers->Get(buffer_index)->data() == nullptr) ||
+        (tensor_buffers->Get(buffer_index)->data()->size() == 0);
     if (has_no_buffer && tensor->type() == tflite::TensorType_FLOAT32) {
       loggable.push_back(tensor_index);
     }
-- 
GitLab


From 8c9dfb62e40bbee2dfeab6c3c9ca633c95002ffd Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 18 Jan 2019 13:45:43 -0800
Subject: [PATCH 1003/1095] Update sequence input layer to new FeatureColumn
 API.

PiperOrigin-RevId: 229986930
---
 tensorflow/contrib/feature_column/BUILD       |  18 +-
 .../sequence_feature_column_v2.py             | 232 +++++++-------
 ...ence_feature_column_v2_integration_test.py | 283 ++++++++++++++++++
 .../sequence_feature_column_v2_test.py        | 201 +++++++------
 tensorflow/python/feature_column/BUILD        |   1 +
 .../feature_column/feature_column_v2.py       | 163 ++++++----
 .../feature_column/feature_column_v2_test.py  |   4 +-
 ...sorflow.keras.layers.-dense-features.pbtxt |   1 +
 8 files changed, 645 insertions(+), 258 deletions(-)
 create mode 100644 tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2_integration_test.py

diff --git a/tensorflow/contrib/feature_column/BUILD b/tensorflow/contrib/feature_column/BUILD
index 4e29e25599..8fc5f1cfe7 100644
--- a/tensorflow/contrib/feature_column/BUILD
+++ b/tensorflow/contrib/feature_column/BUILD
@@ -96,7 +96,6 @@ tf_py_test(
     name = "sequence_feature_column_v2_test",
     srcs = ["python/feature_column/sequence_feature_column_v2_test.py"],
     additional_deps = [
-        ":sequence_feature_column",
         ":sequence_feature_column_v2",
         "@absl_py//absl/testing:parameterized",
         "//third_party/py/numpy",
@@ -113,3 +112,20 @@ tf_py_test(
     ],
     tags = ["no_pip"],
 )
+
+py_test(
+    name = "sequence_feature_column_v2_integration_test",
+    srcs = ["python/feature_column/sequence_feature_column_v2_integration_test.py"],
+    srcs_version = "PY2AND3",
+    tags = ["no_pip"],
+    deps = [
+        ":sequence_feature_column_v2",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:parsing_ops",
+        "//tensorflow/python:training",
+        "//tensorflow/python:util",
+        "//tensorflow/python/feature_column:feature_column_py",
+        "//tensorflow/python/keras:layers",
+    ],
+)
diff --git a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2.py b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2.py
index 83b93ec332..2f4bda194a 100644
--- a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2.py
+++ b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2.py
@@ -27,6 +27,7 @@ import collections
 
 from tensorflow.python.feature_column import feature_column as fc_old
 from tensorflow.python.feature_column import feature_column_lib as fc
+from tensorflow.python.feature_column import feature_column_v2 as fc_v2
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
@@ -34,107 +35,115 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import check_ops
 from tensorflow.python.ops import parsing_ops
 from tensorflow.python.ops import sparse_ops
-from tensorflow.python.ops import variable_scope
 
 # pylint: disable=protected-access
 
 
-def sequence_input_layer(
-    features,
-    feature_columns,
-    weight_collections=None,
-    trainable=True):
-  """"Builds input layer for sequence input.
+class SequenceFeatures(fc_v2._BaseFeaturesLayer):
+  """A layer for sequence input.
 
-  All `feature_columns` must be sequence dense columns with the same
-  `sequence_length`. The output of this method can be fed into sequence
-  networks, such as RNN.
+    All `feature_columns` must be sequence dense columns with the same
+    `sequence_length`. The output of this method can be fed into sequence
+    networks, such as RNN.
 
-  The output of this method is a 3D `Tensor` of shape `[batch_size, T, D]`.
-  `T` is the maximum sequence length for this batch, which could differ from
-  batch to batch.
+    The output of this method is a 3D `Tensor` of shape `[batch_size, T, D]`.
+    `T` is the maximum sequence length for this batch, which could differ from
+    batch to batch.
 
-  If multiple `feature_columns` are given with `Di` `num_elements` each, their
-  outputs are concatenated. So, the final `Tensor` has shape
-  `[batch_size, T, D0 + D1 + ... + Dn]`.
+    If multiple `feature_columns` are given with `Di` `num_elements` each, their
+    outputs are concatenated. So, the final `Tensor` has shape
+    `[batch_size, T, D0 + D1 + ... + Dn]`.
 
-  Example:
+    Example:
 
-  ```python
-  rating = sequence_numeric_column('rating')
-  watches = sequence_categorical_column_with_identity(
-      'watches', num_buckets=1000)
-  watches_embedding = embedding_column(watches, dimension=10)
-  columns = [rating, watches]
+    ```python
+    rating = sequence_numeric_column('rating')
+    watches = sequence_categorical_column_with_identity(
+        'watches', num_buckets=1000)
+    watches_embedding = embedding_column(watches, dimension=10)
+    columns = [rating, watches]
 
-  features = tf.parse_example(..., features=make_parse_example_spec(columns))
-  input_layer, sequence_length = sequence_input_layer(features, columns)
+    features = tf.parse_example(..., features=make_parse_example_spec(columns))
+    sequence_input_layer = SequenceFeatures(columns)
+    sequence_input, sequence_length = sequence_input_layer(features)
+    sequence_length_mask = tf.sequence_mask(sequence_length)
 
-  rnn_cell = tf.nn.rnn_cell.BasicRNNCell(hidden_size)
-  outputs, state = tf.nn.dynamic_rnn(
-      rnn_cell, inputs=input_layer, sequence_length=sequence_length)
-  ```
+    rnn_cell = tf.keras.layers.SimpleRNNCell(hidden_size)
+    rnn_layer = tf.keras.layers.RNN(rnn_cell)
+    outputs, state = rnn_layer(sequence_input, mask=sequence_length_mask)
+    ```
+  """
 
-  Args:
-    features: A dict mapping keys to tensors.
-    feature_columns: An iterable of dense sequence columns. Valid columns are
-      - `embedding_column` that wraps a `sequence_categorical_column_with_*`
-      - `sequence_numeric_column`.
-    weight_collections: A list of collection names to which the Variable will be
-      added. Note that variables will also be added to collections
-      `tf.GraphKeys.GLOBAL_VARIABLES` and `ops.GraphKeys.MODEL_VARIABLES`.
-    trainable: If `True` also add the variable to the graph collection
-      `GraphKeys.TRAINABLE_VARIABLES`.
+  def __init__(
+      self,
+      feature_columns,
+      trainable=True,
+      name=None,
+      **kwargs):
+    """"Constructs a SequenceFeatures layer.
 
-  Returns:
-    An `(input_layer, sequence_length)` tuple where:
-    - input_layer: A float `Tensor` of shape `[batch_size, T, D]`.
-        `T` is the maximum sequence length for this batch, which could differ
-        from batch to batch. `D` is the sum of `num_elements` for all
-        `feature_columns`.
-    - sequence_length: An int `Tensor` of shape `[batch_size]`. The sequence
-        length for each example.
+    Args:
+      feature_columns: An iterable of dense sequence columns. Valid columns are
+        - `embedding_column` that wraps a `sequence_categorical_column_with_*`
+        - `sequence_numeric_column`.
+      trainable: Boolean, whether the layer's variables will be updated via
+        gradient descent during training.
+      name: Name to give to the SequenceFeatures.
+      **kwargs: Keyword arguments to construct a layer.
+
+    Raises:
+      ValueError: If any of the `feature_columns` is not a
+        `SequenceDenseColumn`.
+    """
+    super(SequenceFeatures, self).__init__(
+        feature_columns=feature_columns,
+        trainable=trainable,
+        name=name,
+        expected_column_type=fc_v2.SequenceDenseColumn,
+        **kwargs)
 
-  Raises:
-    ValueError: If any of the `feature_columns` is the wrong type.
-  """
-  feature_columns = fc_old._normalize_feature_columns(feature_columns)
-  for c in feature_columns:
-    if not isinstance(c, fc_old._SequenceDenseColumn):
-      raise ValueError(
-          'All feature_columns must be of type _SequenceDenseColumn. '
-          'You can wrap a sequence_categorical_column with an embedding_column '
-          'or indicator_column. '
-          'Given (type {}): {}'.format(type(c), c))
-
-  with variable_scope.variable_scope(
-      None, default_name='sequence_input_layer', values=features.values()):
-    builder = fc_old._LazyBuilder(features)
+  def _target_shape(self, input_shape, total_elements):
+    return (input_shape[0], input_shape[1], total_elements)
+
+  def call(self, features):
+    """Returns sequence input corresponding to the `feature_columns`.
+
+    Args:
+      features: A dict mapping keys to tensors.
+
+    Returns:
+      An `(input_layer, sequence_length)` tuple where:
+      - input_layer: A float `Tensor` of shape `[batch_size, T, D]`.
+          `T` is the maximum sequence length for this batch, which could differ
+          from batch to batch. `D` is the sum of `num_elements` for all
+          `feature_columns`.
+      - sequence_length: An int `Tensor` of shape `[batch_size]`. The sequence
+          length for each example.
+
+    Raises:
+      ValueError: If features are not a dictionary.
+    """
+    if not isinstance(features, dict):
+      raise ValueError('We expected a dictionary here. Instead we got: ',
+                       features)
+    transformation_cache = fc.FeatureTransformationCache(features)
     output_tensors = []
     sequence_lengths = []
-    ordered_columns = []
-
-    for column in sorted(feature_columns, key=lambda x: x.name):
-      ordered_columns.append(column)
-      with variable_scope.variable_scope(
-          None, default_name=column._var_scope_name):
-        dense_tensor, sequence_length = column._get_sequence_dense_tensor(
-            builder,
-            weight_collections=weight_collections,
-            trainable=trainable)
+
+    for column in self._feature_columns:
+      with ops.name_scope(column.name):
+        dense_tensor, sequence_length = column.get_sequence_dense_tensor(
+            transformation_cache, self._state_manager)
         # Flattens the final dimension to produce a 3D Tensor.
-        num_elements = column._variable_shape.num_elements()
-        shape = array_ops.shape(dense_tensor)
-        target_shape = [shape[0], shape[1], num_elements]
-        output_tensors.append(
-            array_ops.reshape(dense_tensor, shape=target_shape))
+        output_tensors.append(self._process_dense_tensor(column, dense_tensor))
         sequence_lengths.append(sequence_length)
 
-    fc_old._verify_static_batch_size_equality(output_tensors, ordered_columns)
-    fc_old._verify_static_batch_size_equality(sequence_lengths, ordered_columns)
+    # Check and process sequence lengths.
+    fc_v2._verify_static_batch_size_equality(sequence_lengths,
+                                             self._feature_columns)
     sequence_length = _assert_all_equal_and_return(sequence_lengths)
 
-    return array_ops.concat(output_tensors, -1), sequence_length
+    return self._verify_and_concat_tensors(output_tensors), sequence_length
 
 
 def concatenate_context_input(context_input, sequence_input):
@@ -203,12 +212,13 @@ def sequence_categorical_column_with_identity(
   columns = [watches_embedding]
 
   features = tf.parse_example(..., features=make_parse_example_spec(columns))
-  sequence_feature_layer = SequenceFeatureLayer(columns)
-  input_layer, sequence_length = sequence_feature_layer(features)
+  sequence_feature_layer = SequenceFeatures(columns)
+  sequence_input, sequence_length = sequence_feature_layer(features)
+  sequence_length_mask = tf.sequence_mask(sequence_length)
 
-  rnn_cell = tf.nn.rnn_cell.BasicRNNCell(hidden_size)
-  outputs, state = tf.nn.dynamic_rnn(
-      rnn_cell, inputs=input_layer, sequence_length=sequence_length)
+  rnn_cell = tf.keras.layers.SimpleRNNCell(hidden_size)
+  rnn_layer = tf.keras.layers.RNN(rnn_cell)
+  outputs, state = rnn_layer(sequence_input, mask=sequence_length_mask)
   ```
 
   Args:
@@ -250,12 +260,13 @@ def sequence_categorical_column_with_hash_bucket(
   columns = [tokens_embedding]
 
   features = tf.parse_example(..., features=make_parse_example_spec(columns))
-  sequence_feature_layer = SequenceFeatureLayer(columns)
-  input_layer, sequence_length = sequence_feature_layer(features)
+  sequence_feature_layer = SequenceFeatures(columns)
+  sequence_input, sequence_length = sequence_feature_layer(features)
+  sequence_length_mask = tf.sequence_mask(sequence_length)
 
-  rnn_cell = tf.nn.rnn_cell.BasicRNNCell(hidden_size)
-  outputs, state = tf.nn.dynamic_rnn(
-      rnn_cell, inputs=input_layer, sequence_length=sequence_length)
+  rnn_cell = tf.keras.layers.SimpleRNNCell(hidden_size)
+  rnn_layer = tf.keras.layers.RNN(rnn_cell)
+  outputs, state = rnn_layer(sequence_input, mask=sequence_length_mask)
   ```
 
   Args:
@@ -296,12 +307,13 @@ def sequence_categorical_column_with_vocabulary_file(
   columns = [states_embedding]
 
   features = tf.parse_example(..., features=make_parse_example_spec(columns))
-  sequence_feature_layer = SequenceFeatureLayer(columns)
-  input_layer, sequence_length = sequence_feature_layer(features)
+  sequence_feature_layer = SequenceFeatures(columns)
+  sequence_input, sequence_length = sequence_feature_layer(features)
+  sequence_length_mask = tf.sequence_mask(sequence_length)
 
-  rnn_cell = tf.nn.rnn_cell.BasicRNNCell(hidden_size)
-  outputs, state = tf.nn.dynamic_rnn(
-      rnn_cell, inputs=input_layer, sequence_length=sequence_length)
+  rnn_cell = tf.keras.layers.SimpleRNNCell(hidden_size)
+  rnn_layer = tf.keras.layers.RNN(rnn_cell)
+  outputs, state = rnn_layer(sequence_input, mask=sequence_length_mask)
   ```
 
   Args:
@@ -358,12 +370,13 @@ def sequence_categorical_column_with_vocabulary_list(
   columns = [colors_embedding]
 
   features = tf.parse_example(..., features=make_parse_example_spec(columns))
-  sequence_feature_layer = SequenceFeatureLayer(columns)
-  input_layer, sequence_length = sequence_feature_layer(features)
+  sequence_feature_layer = SequenceFeatures(columns)
+  sequence_input, sequence_length = sequence_feature_layer(features)
+  sequence_length_mask = tf.sequence_mask(sequence_length)
 
-  rnn_cell = tf.nn.rnn_cell.BasicRNNCell(hidden_size)
-  outputs, state = tf.nn.dynamic_rnn(
-      rnn_cell, inputs=input_layer, sequence_length=sequence_length)
+  rnn_cell = tf.keras.layers.SimpleRNNCell(hidden_size)
+  rnn_layer = tf.keras.layers.RNN(rnn_cell)
+  outputs, state = rnn_layer(sequence_input, mask=sequence_length_mask)
   ```
 
   Args:
@@ -415,12 +428,13 @@ def sequence_numeric_column(
   columns = [temperature]
 
   features = tf.parse_example(..., features=make_parse_example_spec(columns))
-  sequence_feature_layer = SequenceFeatureLayer(columns)
-  input_layer, sequence_length = sequence_feature_layer(features)
+  sequence_feature_layer = SequenceFeatures(columns)
+  sequence_input, sequence_length = sequence_feature_layer(features)
+  sequence_length_mask = tf.sequence_mask(sequence_length)
 
-  rnn_cell = tf.nn.rnn_cell.BasicRNNCell(hidden_size)
-  outputs, state = tf.nn.dynamic_rnn(
-      rnn_cell, inputs=input_layer, sequence_length=sequence_length)
+  rnn_cell = tf.keras.layers.SimpleRNNCell(hidden_size)
+  rnn_layer = tf.keras.layers.RNN(rnn_cell)
+  outputs, state = rnn_layer(sequence_input, mask=sequence_length_mask)
   ```
 
   Args:
@@ -445,7 +459,7 @@ def sequence_numeric_column(
     ValueError: if any dimension in shape is not a positive integer.
     ValueError: if `dtype` is not convertible to `tf.float32`.
   """
-  shape = fc_old._check_shape(shape=shape, key=key)
+  shape = fc_v2._check_shape(shape=shape, key=key)
   if not (dtype.is_integer or dtype.is_floating):
     raise ValueError('dtype must be convertible to float. '
                      'dtype: {}, key: {}'.format(dtype, key))
@@ -540,8 +554,10 @@ class SequenceNumericColumn(
     # For the 2D case, the raw values are grouped according to num_elements;
     # for the 3D case, the grouping happens in the third dimension, and
     # sequence length is not affected.
-    num_elements = (self.variable_shape.num_elements()
-                    if sp_tensor.shape.ndims == 2 else 1)
+    if sp_tensor.shape.ndims == 2:
+      num_elements = self.variable_shape.num_elements()
+    else:
+      num_elements = 1
     seq_length = fc_old._sequence_length_from_sparse_tensor(
         sp_tensor, num_elements=num_elements)
 
diff --git a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2_integration_test.py b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2_integration_test.py
new file mode 100644
index 0000000000..1b165a620a
--- /dev/null
+++ b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2_integration_test.py
@@ -0,0 +1,283 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Integration test for sequence feature columns with SequenceExamples."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import string
+import tempfile
+
+from google.protobuf import text_format
+
+from tensorflow.contrib.feature_column.python.feature_column import sequence_feature_column_v2 as sfc
+from tensorflow.core.example import example_pb2
+from tensorflow.core.example import feature_pb2
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.feature_column import feature_column_v2 as fc
+from tensorflow.python.keras.layers import recurrent
+from tensorflow.python.ops import parsing_ops
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import test
+from tensorflow.python.util import compat
+
+
+class SequenceFeatureColumnIntegrationTest(test.TestCase):
+
+  def _make_sequence_example(self):
+    example = example_pb2.SequenceExample()
+    example.context.feature['int_ctx'].int64_list.value.extend([5])
+    example.context.feature['float_ctx'].float_list.value.extend([123.6])
+    for val in range(0, 10, 2):
+      feat = feature_pb2.Feature()
+      feat.int64_list.value.extend([val] * val)
+      example.feature_lists.feature_list['int_list'].feature.extend([feat])
+    for val in range(1, 11, 2):
+      feat = feature_pb2.Feature()
+      feat.bytes_list.value.extend([compat.as_bytes(str(val))] * val)
+      example.feature_lists.feature_list['str_list'].feature.extend([feat])
+
+    return example
+
+  def _build_feature_columns(self):
+    col = fc.categorical_column_with_identity('int_ctx', num_buckets=100)
+    ctx_cols = [
+        fc.embedding_column(col, dimension=10),
+        fc.numeric_column('float_ctx')
+    ]
+
+    identity_col = sfc.sequence_categorical_column_with_identity(
+        'int_list', num_buckets=10)
+    bucket_col = sfc.sequence_categorical_column_with_hash_bucket(
+        'bytes_list', hash_bucket_size=100)
+    seq_cols = [
+        fc.embedding_column(identity_col, dimension=10),
+        fc.embedding_column(bucket_col, dimension=20)
+    ]
+
+    return ctx_cols, seq_cols
+
+  def test_sequence_example_into_input_layer(self):
+    examples = [_make_sequence_example().SerializeToString()] * 100
+    ctx_cols, seq_cols = self._build_feature_columns()
+
+    def _parse_example(example):
+      ctx, seq = parsing_ops.parse_single_sequence_example(
+          example,
+          context_features=fc.make_parse_example_spec_v2(ctx_cols),
+          sequence_features=fc.make_parse_example_spec_v2(seq_cols))
+      ctx.update(seq)
+      return ctx
+
+    ds = dataset_ops.Dataset.from_tensor_slices(examples)
+    ds = ds.map(_parse_example)
+    ds = ds.batch(20)
+
+    # Test on a single batch
+    features = ds.make_one_shot_iterator().get_next()
+
+    # Tile the context features across the sequence features
+    sequence_input_layer = sfc.SequenceFeatures(seq_cols)
+    seq_layer, _ = sequence_input_layer(features)
+    input_layer = fc.DenseFeatures(ctx_cols)
+    ctx_layer = input_layer(features)
+    input_layer = sfc.concatenate_context_input(ctx_layer, seq_layer)
+
+    rnn_layer = recurrent.RNN(recurrent.SimpleRNNCell(10))
+    output = rnn_layer(input_layer)
+
+    with self.cached_session() as sess:
+      sess.run(variables.global_variables_initializer())
+      features_r = sess.run(features)
+      self.assertAllEqual(features_r['int_list'].dense_shape, [20, 3, 6])
+
+      output_r = sess.run(output)
+      self.assertAllEqual(output_r.shape, [20, 10])
+
+
+class SequenceExampleParsingTest(test.TestCase):
+
+  def test_seq_ex_in_sequence_categorical_column_with_identity(self):
+    self._test_parsed_sequence_example(
+        'int_list', sfc.sequence_categorical_column_with_identity,
+        10, [3, 6], [2, 4, 6])
+
+  def test_seq_ex_in_sequence_categorical_column_with_hash_bucket(self):
+    self._test_parsed_sequence_example(
+        'bytes_list', sfc.sequence_categorical_column_with_hash_bucket,
+        10, [3, 4], [compat.as_bytes(x) for x in 'acg'])
+
+  def test_seq_ex_in_sequence_categorical_column_with_vocabulary_list(self):
+    self._test_parsed_sequence_example(
+        'bytes_list', sfc.sequence_categorical_column_with_vocabulary_list,
+        list(string.ascii_lowercase), [3, 4],
+        [compat.as_bytes(x) for x in 'acg'])
+
+  def test_seq_ex_in_sequence_categorical_column_with_vocabulary_file(self):
+    _, fname = tempfile.mkstemp()
+    with open(fname, 'w') as f:
+      f.write(string.ascii_lowercase)
+    self._test_parsed_sequence_example(
+        'bytes_list', sfc.sequence_categorical_column_with_vocabulary_file,
+        fname, [3, 4], [compat.as_bytes(x) for x in 'acg'])
+
+  def _test_parsed_sequence_example(
+      self, col_name, col_fn, col_arg, shape, values):
+    """Helper function to check that each FeatureColumn parses correctly.
+
+    Args:
+      col_name: string, name to give to the feature column. Should match
+        the name that the column will parse out of the features dict.
+      col_fn: function used to create the feature column. For example,
+        sequence_numeric_column.
+      col_arg: second arg that the target feature column is expecting.
+      shape: the expected dense_shape of the feature after parsing into
+        a SparseTensor.
+      values: the expected values at index [0, 2, 6] of the feature
+        after parsing into a SparseTensor.
+    """
+    example = _make_sequence_example()
+    columns = [
+        fc.categorical_column_with_identity('int_ctx', num_buckets=100),
+        fc.numeric_column('float_ctx'),
+        col_fn(col_name, col_arg)
+    ]
+    context, seq_features = parsing_ops.parse_single_sequence_example(
+        example.SerializeToString(),
+        context_features=fc.make_parse_example_spec_v2(columns[:2]),
+        sequence_features=fc.make_parse_example_spec_v2(columns[2:]))
+
+    with self.cached_session() as sess:
+      ctx_result, seq_result = sess.run([context, seq_features])
+      self.assertEqual(list(seq_result[col_name].dense_shape), shape)
+      self.assertEqual(
+          list(seq_result[col_name].values[[0, 2, 6]]), values)
+      self.assertEqual(list(ctx_result['int_ctx'].dense_shape), [1])
+      self.assertEqual(ctx_result['int_ctx'].values[0], 5)
+      self.assertEqual(list(ctx_result['float_ctx'].shape), [1])
+      self.assertAlmostEqual(ctx_result['float_ctx'][0], 123.6, places=1)
+
+
+_SEQ_EX_PROTO = """
+context {
+  feature {
+    key: "float_ctx"
+    value {
+      float_list {
+        value: 123.6
+      }
+    }
+  }
+  feature {
+    key: "int_ctx"
+    value {
+      int64_list {
+        value: 5
+      }
+    }
+  }
+}
+feature_lists {
+  feature_list {
+    key: "bytes_list"
+    value {
+      feature {
+        bytes_list {
+          value: "a"
+        }
+      }
+      feature {
+        bytes_list {
+          value: "b"
+          value: "c"
+        }
+      }
+      feature {
+        bytes_list {
+          value: "d"
+          value: "e"
+          value: "f"
+          value: "g"
+        }
+      }
+    }
+  }
+  feature_list {
+    key: "float_list"
+    value {
+      feature {
+        float_list {
+          value: 1.0
+        }
+      }
+      feature {
+        float_list {
+          value: 3.0
+          value: 3.0
+          value: 3.0
+        }
+      }
+      feature {
+        float_list {
+          value: 5.0
+          value: 5.0
+          value: 5.0
+          value: 5.0
+          value: 5.0
+        }
+      }
+    }
+  }
+  feature_list {
+    key: "int_list"
+    value {
+      feature {
+        int64_list {
+          value: 2
+          value: 2
+        }
+      }
+      feature {
+        int64_list {
+          value: 4
+          value: 4
+          value: 4
+          value: 4
+        }
+      }
+      feature {
+        int64_list {
+          value: 6
+          value: 6
+          value: 6
+          value: 6
+          value: 6
+          value: 6
+        }
+      }
+    }
+  }
+}
+"""
+
+
+def _make_sequence_example():
+  example = example_pb2.SequenceExample()
+  return text_format.Parse(_SEQ_EX_PROTO, example)
+
+
+if __name__ == '__main__':
+  test.main()
diff --git a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2_test.py b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2_test.py
index be012a8769..661eba4741 100644
--- a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2_test.py
+++ b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2_test.py
@@ -22,9 +22,7 @@ import os
 from absl.testing import parameterized
 import numpy as np
 
-from tensorflow.contrib.feature_column.python.feature_column import sequence_feature_column as sfc_old
 from tensorflow.contrib.feature_column.python.feature_column import sequence_feature_column_v2 as sfc
-from tensorflow.python.feature_column import feature_column as fc_old
 from tensorflow.python.feature_column import feature_column_lib as fc
 from tensorflow.python.feature_column.feature_column_v2_test import _TestStateManager
 from tensorflow.python.framework import dtypes
@@ -32,13 +30,15 @@ from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import lookup_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import sparse_ops
+from tensorflow.python.ops import variables as variables_lib
 from tensorflow.python.platform import test
 from tensorflow.python.training import monitored_session
 
 
-class SequenceInputLayerTest(test.TestCase, parameterized.TestCase):
+class SequenceFeaturesTest(test.TestCase, parameterized.TestCase):
 
   @parameterized.named_parameters(
       {'testcase_name': '2D',
@@ -111,29 +111,27 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase):
 
     categorical_column_a = sfc.sequence_categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column_a = fc_old._embedding_column(
+    embedding_column_a = fc.embedding_column(
         categorical_column_a,
         dimension=embedding_dimension_a,
         initializer=_get_initializer(embedding_dimension_a, embedding_values_a))
     categorical_column_b = sfc.sequence_categorical_column_with_identity(
         key='bbb', num_buckets=vocabulary_size)
-    embedding_column_b = fc_old._embedding_column(
+    embedding_column_b = fc.embedding_column(
         categorical_column_b,
         dimension=embedding_dimension_b,
         initializer=_get_initializer(embedding_dimension_b, embedding_values_b))
 
-    input_layer, sequence_length = sfc.sequence_input_layer(
-        features={
-            'aaa': sparse_input_a,
-            'bbb': sparse_input_b,
-        },
-        # Test that columns are reordered alphabetically.
-        feature_columns=[embedding_column_b, embedding_column_a])
+    # Test that columns are reordered alphabetically.
+    sequence_input_layer = sfc.SequenceFeatures(
+        [embedding_column_b, embedding_column_a])
+    input_layer, sequence_length = sequence_input_layer({
+        'aaa': sparse_input_a, 'bbb': sparse_input_b,})
 
     global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)
     self.assertCountEqual(
-        ('sequence_input_layer/aaa_embedding/embedding_weights:0',
-         'sequence_input_layer/bbb_embedding/embedding_weights:0'),
+        ('sequence_features/aaa_embedding/embedding_weights:0',
+         'sequence_features/bbb_embedding/embedding_weights:0'),
         tuple([v.name for v in global_vars]))
     with monitored_session.MonitoredSession() as sess:
       self.assertAllEqual(embedding_values_a, global_vars[0].eval(session=sess))
@@ -152,18 +150,17 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase):
         values=(2, 0, 1),
         dense_shape=(2, 2))
 
-    categorical_column_a = fc_old._categorical_column_with_identity(
+    categorical_column_a = fc.categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column_a = fc_old._embedding_column(
+    embedding_column_a = fc.embedding_column(
         categorical_column_a, dimension=2)
 
     with self.assertRaisesRegexp(
         ValueError,
         r'In embedding_column: aaa_embedding\. categorical_column must be of '
-        r'type _SequenceCategoricalColumn to use sequence_input_layer\.'):
-      _, _ = sfc.sequence_input_layer(
-          features={'aaa': sparse_input},
-          feature_columns=[embedding_column_a])
+        r'type SequenceCategoricalColumn to use SequenceFeatures\.'):
+      sequence_input_layer = sfc.SequenceFeatures([embedding_column_a])
+      _, _ = sequence_input_layer({'aaa': sparse_input})
 
   def test_shared_embedding_column(self):
     vocabulary_size = 3
@@ -210,21 +207,18 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase):
     categorical_column_b = sfc.sequence_categorical_column_with_identity(
         key='bbb', num_buckets=vocabulary_size)
     # Test that columns are reordered alphabetically.
-    shared_embedding_columns = fc.shared_embedding_columns(
+    shared_embedding_columns = fc.shared_embedding_columns_v2(
         [categorical_column_b, categorical_column_a],
         dimension=embedding_dimension,
         initializer=_get_initializer(embedding_dimension, embedding_values))
 
-    input_layer, sequence_length = sfc.sequence_input_layer(
-        features={
-            'aaa': sparse_input_a,
-            'bbb': sparse_input_b,
-        },
-        feature_columns=shared_embedding_columns)
+    sequence_input_layer = sfc.SequenceFeatures(shared_embedding_columns)
+    input_layer, sequence_length = sequence_input_layer({
+        'aaa': sparse_input_a, 'bbb': sparse_input_b})
 
     global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)
     self.assertCountEqual(
-        ('sequence_input_layer/aaa_bbb_shared_embedding/embedding_weights:0',),
+        ('aaa_bbb_shared_embedding:0',),
         tuple([v.name for v in global_vars]))
     with monitored_session.MonitoredSession() as sess:
       self.assertAllEqual(embedding_values, global_vars[0].eval(session=sess))
@@ -248,23 +242,20 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase):
         values=(2, 0, 1),
         dense_shape=(2, 2))
 
-    categorical_column_a = fc_old._categorical_column_with_identity(
+    categorical_column_a = fc.categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    categorical_column_b = fc_old._categorical_column_with_identity(
+    categorical_column_b = fc.categorical_column_with_identity(
         key='bbb', num_buckets=vocabulary_size)
-    shared_embedding_columns = fc.shared_embedding_columns(
+    shared_embedding_columns = fc.shared_embedding_columns_v2(
         [categorical_column_a, categorical_column_b], dimension=2)
 
     with self.assertRaisesRegexp(
         ValueError,
         r'In embedding_column: aaa_shared_embedding\. categorical_column must '
-        r'be of type _SequenceCategoricalColumn to use sequence_input_layer\.'):
-      _, _ = sfc.sequence_input_layer(
-          features={
-              'aaa': sparse_input_a,
-              'bbb': sparse_input_b
-          },
-          feature_columns=shared_embedding_columns)
+        r'be of type SequenceCategoricalColumn to use SequenceFeatures\.'):
+      sequence_input_layer = sfc.SequenceFeatures(shared_embedding_columns)
+      _, _ = sequence_input_layer({'aaa': sparse_input_a,
+                                   'bbb': sparse_input_b})
 
   @parameterized.named_parameters(
       {'testcase_name': '2D',
@@ -319,17 +310,15 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase):
 
     categorical_column_a = sfc.sequence_categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size_a)
-    indicator_column_a = fc_old._indicator_column(categorical_column_a)
+    indicator_column_a = fc.indicator_column(categorical_column_a)
     categorical_column_b = sfc.sequence_categorical_column_with_identity(
         key='bbb', num_buckets=vocabulary_size_b)
-    indicator_column_b = fc_old._indicator_column(categorical_column_b)
-    input_layer, sequence_length = sfc.sequence_input_layer(
-        features={
-            'aaa': sparse_input_a,
-            'bbb': sparse_input_b,
-        },
-        # Test that columns are reordered alphabetically.
-        feature_columns=[indicator_column_b, indicator_column_a])
+    indicator_column_b = fc.indicator_column(categorical_column_b)
+    # Test that columns are reordered alphabetically.
+    sequence_input_layer = sfc.SequenceFeatures(
+        [indicator_column_b, indicator_column_a])
+    input_layer, sequence_length = sequence_input_layer({
+        'aaa': sparse_input_a, 'bbb': sparse_input_b})
 
     with monitored_session.MonitoredSession() as sess:
       self.assertAllEqual(expected_input_layer, input_layer.eval(session=sess))
@@ -346,17 +335,16 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase):
         values=(2, 0, 1),
         dense_shape=(2, 2))
 
-    categorical_column_a = fc_old._categorical_column_with_identity(
+    categorical_column_a = fc.categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    indicator_column_a = fc_old._indicator_column(categorical_column_a)
+    indicator_column_a = fc.indicator_column(categorical_column_a)
 
     with self.assertRaisesRegexp(
         ValueError,
         r'In indicator_column: aaa_indicator\. categorical_column must be of '
-        r'type _SequenceCategoricalColumn to use sequence_input_layer\.'):
-      _, _ = sfc.sequence_input_layer(
-          features={'aaa': sparse_input},
-          feature_columns=[indicator_column_a])
+        r'type SequenceCategoricalColumn to use SequenceFeatures\.'):
+      sequence_input_layer = sfc.SequenceFeatures([indicator_column_a])
+      _, _ = sequence_input_layer({'aaa': sparse_input})
 
   @parameterized.named_parameters(
       {'testcase_name': '2D',
@@ -386,11 +374,10 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase):
       self, sparse_input_args, expected_input_layer, expected_sequence_length):
     sparse_input = sparse_tensor.SparseTensorValue(**sparse_input_args)
 
-    numeric_column = sfc_old.sequence_numeric_column('aaa')
+    numeric_column = sfc.sequence_numeric_column('aaa')
 
-    input_layer, sequence_length = sfc.sequence_input_layer(
-        features={'aaa': sparse_input},
-        feature_columns=[numeric_column])
+    sequence_input_layer = sfc.SequenceFeatures([numeric_column])
+    input_layer, sequence_length = sequence_input_layer({'aaa': sparse_input})
 
     with monitored_session.MonitoredSession() as sess:
       self.assertAllEqual(expected_input_layer, input_layer.eval(session=sess))
@@ -428,14 +415,13 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase):
       )
   def test_numeric_column_multi_dim(
       self, sparse_input_args, expected_input_layer, expected_sequence_length):
-    """Tests sequence_input_layer for multi-dimensional numeric_column."""
+    """Tests SequenceFeatures for multi-dimensional numeric_column."""
     sparse_input = sparse_tensor.SparseTensorValue(**sparse_input_args)
 
-    numeric_column = sfc_old.sequence_numeric_column('aaa', shape=(2, 2))
+    numeric_column = sfc.sequence_numeric_column('aaa', shape=(2, 2))
 
-    input_layer, sequence_length = sfc.sequence_input_layer(
-        features={'aaa': sparse_input},
-        feature_columns=[numeric_column])
+    sequence_input_layer = sfc.SequenceFeatures([numeric_column])
+    input_layer, sequence_length = sequence_input_layer({'aaa': sparse_input})
 
     with monitored_session.MonitoredSession() as sess:
       self.assertAllEqual(expected_input_layer, input_layer.eval(session=sess))
@@ -454,22 +440,20 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase):
         indices=((0, 0), (1, 0)),
         values=(1., 10.),
         dense_shape=(2, 2))
-    numeric_column_a = sfc_old.sequence_numeric_column('aaa')
-    numeric_column_b = sfc_old.sequence_numeric_column('bbb')
+    numeric_column_a = sfc.sequence_numeric_column('aaa')
+    numeric_column_b = sfc.sequence_numeric_column('bbb')
 
-    _, sequence_length = sfc.sequence_input_layer(
-        features={
-            'aaa': sparse_input_a,
-            'bbb': sparse_input_b,
-        },
-        feature_columns=[numeric_column_a, numeric_column_b])
+    sequence_input_layer = sfc.SequenceFeatures(
+        [numeric_column_a, numeric_column_b])
+    _, sequence_length = sequence_input_layer({
+        'aaa': sparse_input_a, 'bbb': sparse_input_b})
 
     with monitored_session.MonitoredSession() as sess:
       with self.assertRaisesRegexp(
           errors.InvalidArgumentError,
           r'\[Condition x == y did not hold element-wise:\] '
-          r'\[x \(sequence_input_layer/aaa/sequence_length:0\) = \] \[2 1\] '
-          r'\[y \(sequence_input_layer/bbb/sequence_length:0\) = \] \[1 1\]'):
+          r'\[x \(sequence_features/aaa/sequence_length:0\) = \] \[2 1\] '
+          r'\[y \(sequence_features/bbb/sequence_length:0\) = \] \[1 1\]'):
         sess.run(sequence_length)
 
   @parameterized.named_parameters(
@@ -497,11 +481,10 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase):
       self, sparse_input_args, expected_shape):
     """Tests that we return a known static shape when we have one."""
     sparse_input = sparse_tensor.SparseTensorValue(**sparse_input_args)
-    numeric_column = sfc_old.sequence_numeric_column('aaa', shape=(2, 2))
+    numeric_column = sfc.sequence_numeric_column('aaa', shape=(2, 2))
 
-    input_layer, _ = sfc.sequence_input_layer(
-        features={'aaa': sparse_input},
-        feature_columns=[numeric_column])
+    sequence_input_layer = sfc.SequenceFeatures([numeric_column])
+    input_layer, _ = sequence_input_layer({'aaa': sparse_input})
     shape = input_layer.get_shape()
     self.assertEqual(shape, expected_shape)
 
@@ -534,13 +517,49 @@ class SequenceInputLayerTest(test.TestCase, parameterized.TestCase):
     sparse_input = sparse_tensor.SparseTensorValue(**sparse_input_args)
     categorical_column = sfc.sequence_categorical_column_with_identity(
         key='aaa', num_buckets=3)
-    indicator_column = fc_old._indicator_column(categorical_column)
+    indicator_column = fc.indicator_column(categorical_column)
 
-    input_layer, _ = sfc.sequence_input_layer(
-        features={'aaa': sparse_input}, feature_columns=[indicator_column])
+    sequence_input_layer = sfc.SequenceFeatures([indicator_column])
+    input_layer, _ = sequence_input_layer({'aaa': sparse_input})
     shape = input_layer.get_shape()
     self.assertEqual(shape, expected_shape)
 
+  def test_compute_output_shape(self):
+    price1 = sfc.sequence_numeric_column('price1', shape=2)
+    price2 = sfc.sequence_numeric_column('price2')
+    with ops.Graph().as_default():
+      features = {
+          'price1': sparse_tensor.SparseTensor(
+              indices=[[0, 0, 0], [0, 0, 1],
+                       [0, 1, 0], [0, 1, 1],
+                       [1, 0, 0], [1, 0, 1],
+                       [2, 0, 0], [2, 0, 1],
+                       [3, 0, 0], [3, 0, 1]],
+              values=[0., 1., 10., 11., 100., 101., 200., 201., 300., 301.],
+              dense_shape=(4, 3, 2)),
+          'price2': sparse_tensor.SparseTensor(
+              indices=[[0, 0],
+                       [0, 1],
+                       [1, 0],
+                       [2, 0],
+                       [3, 0]],
+              values=[10., 11., 20., 30., 40.],
+              dense_shape=(4, 3))}
+      sequence_features = sfc.SequenceFeatures([price1, price2])
+      seq_input, seq_len = sequence_features(features)
+      self.assertEqual(
+          sequence_features.compute_output_shape((None, None)),
+          (None, None, 3))
+      self.evaluate(variables_lib.global_variables_initializer())
+      self.evaluate(lookup_ops.tables_initializer())
+
+      self.assertAllClose([[[0., 1., 10.], [10., 11., 11.], [0., 0., 0.]],
+                           [[100., 101., 20.], [0., 0., 0.], [0., 0., 0.]],
+                           [[200., 201., 30.], [0., 0., 0.], [0., 0., 0.]],
+                           [[300., 301., 40.], [0., 0., 0.], [0., 0., 0.]]],
+                          self.evaluate(seq_input))
+      self.assertAllClose([2, 1, 1, 1], self.evaluate(seq_len))
+
 
 class ConcatenateContextInputTest(test.TestCase, parameterized.TestCase):
   """Tests the utility fn concatenate_context_input."""
@@ -605,8 +624,8 @@ class ConcatenateContextInputTest(test.TestCase, parameterized.TestCase):
       sfc.concatenate_context_input(context_input, seq_input)
 
 
-class InputLayerTest(test.TestCase):
-  """Tests input_layer with sequence feature columns."""
+class DenseFeaturesTest(test.TestCase):
+  """Tests DenseFeatures with sequence feature columns."""
 
   def test_embedding_column(self):
     """Tests that error is raised for sequence embedding column."""
@@ -620,16 +639,15 @@ class InputLayerTest(test.TestCase):
 
     categorical_column_a = sfc.sequence_categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    embedding_column_a = fc_old._embedding_column(
+    embedding_column_a = fc.embedding_column(
         categorical_column_a, dimension=2)
 
     with self.assertRaisesRegexp(
         ValueError,
         r'In embedding_column: aaa_embedding\. categorical_column must not be '
-        r'of type _SequenceCategoricalColumn\.'):
-      _ = fc_old.input_layer(
-          features={'aaa': sparse_input},
-          feature_columns=[embedding_column_a])
+        r'of type SequenceCategoricalColumn\.'):
+      input_layer = fc.DenseFeatures([embedding_column_a])
+      _ = input_layer({'aaa': sparse_input})
 
   def test_indicator_column(self):
     """Tests that error is raised for sequence indicator column."""
@@ -643,15 +661,14 @@ class InputLayerTest(test.TestCase):
 
     categorical_column_a = sfc.sequence_categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
-    indicator_column_a = fc_old._indicator_column(categorical_column_a)
+    indicator_column_a = fc.indicator_column(categorical_column_a)
 
     with self.assertRaisesRegexp(
         ValueError,
         r'In indicator_column: aaa_indicator\. categorical_column must not be '
-        r'of type _SequenceCategoricalColumn\.'):
-      _ = fc_old.input_layer(
-          features={'aaa': sparse_input},
-          feature_columns=[indicator_column_a])
+        r'of type SequenceCategoricalColumn\.'):
+      input_layer = fc.DenseFeatures([indicator_column_a])
+      _ = input_layer({'aaa': sparse_input})
 
 
 def _assert_sparse_tensor_value(test_case, expected, actual):
@@ -946,7 +963,7 @@ class SequenceEmbeddingColumnTest(
         embedding_column, {'aaa': inputs})
 
     global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)
-    self.assertItemsEqual(
+    self.assertCountEqual(
         ('embedding_weights:0',), tuple([v.name for v in global_vars]))
     with monitored_session.MonitoredSession() as sess:
       self.assertAllEqual(embedding_values, global_vars[0].eval(session=sess))
diff --git a/tensorflow/python/feature_column/BUILD b/tensorflow/python/feature_column/BUILD
index 2b986348b7..789887e537 100644
--- a/tensorflow/python/feature_column/BUILD
+++ b/tensorflow/python/feature_column/BUILD
@@ -29,6 +29,7 @@ py_library(
         "//tensorflow/python:embedding_ops",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:init_ops",
+        "//tensorflow/python:layers_base",
         "//tensorflow/python:lookup_ops",
         "//tensorflow/python:math_ops",
         "//tensorflow/python:nn_ops",
diff --git a/tensorflow/python/feature_column/feature_column_v2.py b/tensorflow/python/feature_column/feature_column_v2.py
index 5976f647af..7c28fe2467 100644
--- a/tensorflow/python/feature_column/feature_column_v2.py
+++ b/tensorflow/python/feature_column/feature_column_v2.py
@@ -304,8 +304,84 @@ class _StateManagerImpl(StateManager):
     raise ValueError('Variable does not exist.')
 
 
+class _BaseFeaturesLayer(Layer):
+  """Base class for DenseFeatures and SequenceFeatures.
+
+  Defines common methods and helpers.
+
+  Args:
+    feature_columns: An iterable containing the FeatureColumns to use as
+      inputs to your model.
+    expected_column_type: Expected class for provided feature columns.
+    trainable:  Boolean, whether the layer's variables will be updated via
+      gradient descent during training.
+    name: Name to give to the DenseFeatures.
+    **kwargs: Keyword arguments to construct a layer.
+
+  Raises:
+    ValueError: if an item in `feature_columns` doesn't match
+      `expected_column_type`.
+  """
+  def __init__(self, feature_columns, expected_column_type, trainable, name,
+               **kwargs):
+    super(_BaseFeaturesLayer, self).__init__(
+        name=name, trainable=trainable, **kwargs)
+    self._feature_columns = _normalize_feature_columns(feature_columns)
+    self._state_manager = _StateManagerImpl(self, self.trainable)
+    for column in self._feature_columns:
+      if not isinstance(column, expected_column_type):
+        raise ValueError(
+            'Items of feature_columns must be a {}. '
+            'You can wrap a categorical column with an '
+            'embedding_column or indicator_column. Given: {}'.format(
+                expected_column_type, column))
+
+  def build(self, _):
+    for column in self._feature_columns:
+      with variable_scope._pure_variable_scope(self.name):  # pylint: disable=protected-access
+        with variable_scope._pure_variable_scope(column.name):  # pylint: disable=protected-access
+          column.create_state(self._state_manager)
+    super(_BaseFeaturesLayer, self).build(None)
+
+  def _output_shape(self, input_shape, num_elements):
+    """Computes expected output shape of the layer or a column's dense tensor.
+
+    Args:
+      input_shape: Tensor or array with batch shape.
+      num_elements: Size of the last dimension of the output.
+
+    Returns:
+      Tuple with output shape.
+    """
+    raise NotImplementedError('Calling an abstract method.')
+
+  def compute_output_shape(self, input_shape):
+    total_elements = 0
+    for column in self._feature_columns:
+      total_elements += column.variable_shape.num_elements()
+    return self._target_shape(input_shape, total_elements)
+
+  def _process_dense_tensor(self, column, tensor):
+    """Reshapes the dense tensor output of a column based on expected shape.
+
+    Args:
+      column: A DenseColumn or SequenceDenseColumn object.
+      tensor: A dense tensor obtained from the same column.
+
+    Returns:
+      Reshaped dense tensor."""
+    num_elements = column.variable_shape.num_elements()
+    target_shape = self._target_shape(array_ops.shape(tensor), num_elements)
+    return array_ops.reshape(tensor, shape=target_shape)
+
+  def _verify_and_concat_tensors(self, output_tensors):
+    """Verifies and concatenates the dense output of several columns."""
+    _verify_static_batch_size_equality(output_tensors, self._feature_columns)
+    return array_ops.concat(output_tensors, -1)
+
+
 @keras_export('keras.layers.DenseFeatures', v1=[])
-class DenseFeatures(Layer):
+class DenseFeatures(_BaseFeaturesLayer):
   """A layer that produces a dense `Tensor` based on given `feature_columns`.
 
   Generally a single example in training data is described with FeatureColumns.
@@ -345,8 +421,8 @@ class DenseFeatures(Layer):
         `bucketized_column`, `indicator_column`. If you have categorical
         features, you can wrap them with an `embedding_column` or
         `indicator_column`.
-      trainable: If `True` also add the variable to the graph collection
-        `GraphKeys.TRAINABLE_VARIABLES` (see `tf.Variable`).
+      trainable:  Boolean, whether the layer's variables will be updated via
+        gradient descent during training.
       name: Name to give to the DenseFeatures.
       **kwargs: Keyword arguments to construct a layer.
 
@@ -354,28 +430,18 @@ class DenseFeatures(Layer):
       ValueError: if an item in `feature_columns` is not a `DenseColumn`.
     """
     super(DenseFeatures, self).__init__(
-        name=name, trainable=trainable, **kwargs)
-
-    self._feature_columns = _normalize_feature_columns(feature_columns)
-    self._feature_columns = sorted(self._feature_columns, key=lambda x: x.name)
-    self._state_manager = _StateManagerImpl(self, self.trainable)
-    for column in self._feature_columns:
-      if not isinstance(column, DenseColumn):
-        raise ValueError(
-            'Items of feature_columns must be a DenseColumn. '
-            'You can wrap a categorical column with an '
-            'embedding_column or indicator_column. Given: {}'.format(column))
+        feature_columns=feature_columns,
+        trainable=trainable,
+        name=name,
+        expected_column_type=DenseColumn,
+        **kwargs)
 
   @property
   def _is_feature_layer(self):
     return True
 
-  def build(self, _):
-    for column in self._feature_columns:
-      with variable_scope._pure_variable_scope(self.name):  # pylint: disable=protected-access
-        with variable_scope._pure_variable_scope(column.name):  # pylint: disable=protected-access
-          column.create_state(self._state_manager)
-      super(DenseFeatures, self).build(None)
+  def _target_shape(self, input_shape, total_elements):
+    return (input_shape[0], total_elements)
 
   def call(self, features, cols_to_output_tensors=None):
     """Returns a dense tensor corresponding to the `feature_columns`.
@@ -401,27 +467,15 @@ class DenseFeatures(Layer):
                        features)
     transformation_cache = FeatureTransformationCache(features)
     output_tensors = []
-    ordered_columns = []
     for column in self._feature_columns:
       with ops.name_scope(column.name):
-        ordered_columns.append(column)
         tensor = column.get_dense_tensor(transformation_cache,
                                          self._state_manager)
-        num_elements = column.variable_shape.num_elements()
-        batch_size = array_ops.shape(tensor)[0]
-        tensor = array_ops.reshape(tensor, shape=(batch_size, num_elements))
-        output_tensors.append(tensor)
+        processed_tensors = self._process_dense_tensor(column, tensor)
         if cols_to_output_tensors is not None:
-          cols_to_output_tensors[column] = tensor
-
-    _verify_static_batch_size_equality(output_tensors, ordered_columns)
-    return array_ops.concat(output_tensors, 1)
-
-  def compute_output_shape(self, input_shape):
-    total_elements = 0
-    for column in self._feature_columns:
-      total_elements += column.variable_shape.num_elements()
-    return (input_shape[0], total_elements)
+          cols_to_output_tensors[column] = processed_tensors
+        output_tensors.append(processed_tensors)
+    return self._verify_and_concat_tensors(output_tensors)
 
 
 class _LinearModelLayer(Layer):
@@ -438,7 +492,6 @@ class _LinearModelLayer(Layer):
         name=name, trainable=trainable, **kwargs)
 
     self._feature_columns = _normalize_feature_columns(feature_columns)
-    self._feature_columns = sorted(self._feature_columns, key=lambda x: x.name)
     for column in self._feature_columns:
       if not isinstance(column, (DenseColumn, CategoricalColumn)):
         raise ValueError(
@@ -694,7 +747,7 @@ def _transform_features_v2(features, feature_columns, state_manager):
   with ops.name_scope(
       None, default_name='transform_features', values=features.values()):
     transformation_cache = FeatureTransformationCache(features)
-    for column in sorted(feature_columns, key=lambda x: x.name):
+    for column in feature_columns:
       with ops.name_scope(None, default_name=column.name):
         outputs[column] = transformation_cache.get(column, state_manager)
   return outputs
@@ -2660,7 +2713,7 @@ def _normalize_feature_columns(feature_columns):
                                                name_to_column[column.name]))
     name_to_column[column.name] = column
 
-  return feature_columns
+  return sorted(feature_columns, key=lambda x: x.name)
 
 
 class NumericColumn(
@@ -3068,10 +3121,10 @@ class EmbeddingColumn(
       raise ValueError(
           'In embedding_column: {}. '
           'categorical_column must not be of type SequenceCategoricalColumn. '
-          'Suggested fix A: If you wish to use input_layer, use a '
+          'Suggested fix A: If you wish to use DenseFeatures, use a '
           'non-sequence categorical_column_with_*. '
           'Suggested fix B: If you wish to create sequence input, use '
-          'sequence_input_layer instead of input_layer. '
+          'SequenceFeatures instead of DenseFeatures. '
           'Given (type {}): {}'.format(self.name, type(self.categorical_column),
                                        self.categorical_column))
     # Get sparse IDs and weights.
@@ -3088,10 +3141,10 @@ class EmbeddingColumn(
       raise ValueError(
           'In embedding_column: {}. '
           'categorical_column must not be of type _SequenceCategoricalColumn. '
-          'Suggested fix A: If you wish to use input_layer, use a '
+          'Suggested fix A: If you wish to use DenseFeatures, use a '
           'non-sequence categorical_column_with_*. '
           'Suggested fix B: If you wish to create sequence input, use '
-          'sequence_input_layer instead of input_layer. '
+          'SequenceFeatures instead of DenseFeatures. '
           'Given (type {}): {}'.format(self.name, type(self.categorical_column),
                                        self.categorical_column))
     sparse_tensors = self.categorical_column._get_sparse_tensors(  # pylint: disable=protected-access
@@ -3105,7 +3158,7 @@ class EmbeddingColumn(
       raise ValueError(
           'In embedding_column: {}. '
           'categorical_column must be of type SequenceCategoricalColumn '
-          'to use sequence_input_layer. '
+          'to use SequenceFeatures. '
           'Suggested fix: Use one of sequence_categorical_column_with_*. '
           'Given (type {}): {}'.format(self.name, type(self.categorical_column),
                                        self.categorical_column))
@@ -3129,8 +3182,8 @@ class EmbeddingColumn(
         (SequenceCategoricalColumn, fc_old._SequenceCategoricalColumn)):  # pylint: disable=protected-access
       raise ValueError(
           'In embedding_column: {}. '
-          'categorical_column must be of type _SequenceCategoricalColumn '
-          'to use sequence_input_layer. '
+          'categorical_column must be of type SequenceCategoricalColumn '
+          'to use SequenceFeatures. '
           'Suggested fix: Use one of sequence_categorical_column_with_*. '
           'Given (type {}): {}'.format(self.name, type(self.categorical_column),
                                        self.categorical_column))
@@ -3298,10 +3351,10 @@ class SharedEmbeddingColumn(
       raise ValueError(
           'In embedding_column: {}. '
           'categorical_column must not be of type SequenceCategoricalColumn. '
-          'Suggested fix A: If you wish to use input_layer, use a '
+          'Suggested fix A: If you wish to use DenseFeatures, use a '
           'non-sequence categorical_column_with_*. '
           'Suggested fix B: If you wish to create sequence input, use '
-          'SequenceFeatureLayer instead of FeatureLayer. '
+          'SequenceFeatures instead of DenseFeatures. '
           'Given (type {}): {}'.format(self.name, type(self.categorical_column),
                                        self.categorical_column))
     return self._get_dense_tensor_internal(transformation_cache, state_manager)
@@ -3315,7 +3368,7 @@ class SharedEmbeddingColumn(
       raise ValueError(
           'In embedding_column: {}. '
           'categorical_column must be of type SequenceCategoricalColumn '
-          'to use SequenceFeatureLayer. '
+          'to use SequenceFeatures. '
           'Suggested fix: Use one of sequence_categorical_column_with_*. '
           'Given (type {}): {}'.format(self.name, type(self.categorical_column),
                                        self.categorical_column))
@@ -4333,10 +4386,10 @@ class IndicatorColumn(
       raise ValueError(
           'In indicator_column: {}. '
           'categorical_column must not be of type SequenceCategoricalColumn. '
-          'Suggested fix A: If you wish to use input_layer, use a '
+          'Suggested fix A: If you wish to use DenseFeatures, use a '
           'non-sequence categorical_column_with_*. '
           'Suggested fix B: If you wish to create sequence input, use '
-          'sequence_input_layer instead of input_layer. '
+          'SequenceFeatures instead of DenseFeatures. '
           'Given (type {}): {}'.format(self.name, type(self.categorical_column),
                                        self.categorical_column))
     # Feature has been already transformed. Return the intermediate
@@ -4354,10 +4407,10 @@ class IndicatorColumn(
       raise ValueError(
           'In indicator_column: {}. '
           'categorical_column must not be of type _SequenceCategoricalColumn. '
-          'Suggested fix A: If you wish to use input_layer, use a '
+          'Suggested fix A: If you wish to use DenseFeatures, use a '
           'non-sequence categorical_column_with_*. '
           'Suggested fix B: If you wish to create sequence input, use '
-          'sequence_input_layer instead of input_layer. '
+          'SequenceFeatures instead of DenseFeatures. '
           'Given (type {}): {}'.format(self.name, type(self.categorical_column),
                                        self.categorical_column))
     # Feature has been already transformed. Return the intermediate
@@ -4370,7 +4423,7 @@ class IndicatorColumn(
       raise ValueError(
           'In indicator_column: {}. '
           'categorical_column must be of type SequenceCategoricalColumn '
-          'to use sequence_input_layer. '
+          'to use SequenceFeatures. '
           'Suggested fix: Use one of sequence_categorical_column_with_*. '
           'Given (type {}): {}'.format(self.name, type(self.categorical_column),
                                        self.categorical_column))
@@ -4400,7 +4453,7 @@ class IndicatorColumn(
       raise ValueError(
           'In indicator_column: {}. '
           'categorical_column must be of type _SequenceCategoricalColumn '
-          'to use sequence_input_layer. '
+          'to use SequenceFeatures. '
           'Suggested fix: Use one of sequence_categorical_column_with_*. '
           'Given (type {}): {}'.format(self.name, type(self.categorical_column),
                                        self.categorical_column))
diff --git a/tensorflow/python/feature_column/feature_column_v2_test.py b/tensorflow/python/feature_column/feature_column_v2_test.py
index 137d5e0a8c..2b150790c1 100644
--- a/tensorflow/python/feature_column/feature_column_v2_test.py
+++ b/tensorflow/python/feature_column/feature_column_v2_test.py
@@ -3262,7 +3262,7 @@ class DenseFeaturesTest(test.TestCase):
       fc.DenseFeatures(feature_columns=[])(features={})
 
   def test_should_be_dense_column(self):
-    with self.assertRaisesRegexp(ValueError, 'must be a DenseColumn'):
+    with self.assertRaisesRegexp(ValueError, 'must be a .*DenseColumn'):
       fc.DenseFeatures(feature_columns=[
           fc.categorical_column_with_hash_bucket('wire_cast', 4)
       ])(
@@ -3423,7 +3423,7 @@ class DenseFeaturesTest(test.TestCase):
               sparse_tensor.SparseTensor(
                   indices=[[0, 0], [0, 1]], values=[1, 2], dense_shape=[1, 2])
       }
-      with self.assertRaisesRegexp(Exception, 'must be a DenseColumn'):
+      with self.assertRaisesRegexp(Exception, 'must be a .*DenseColumn'):
         fc.DenseFeatures([animal])(features)
 
   def test_static_batch_size_mismatch(self):
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense-features.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense-features.pbtxt
index 28c1926bd7..3caa3ff4ed 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense-features.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-dense-features.pbtxt
@@ -1,6 +1,7 @@
 path: "tensorflow.keras.layers.DenseFeatures"
 tf_class {
   is_instance: "<class \'tensorflow.python.feature_column.feature_column_v2.DenseFeatures\'>"
+  is_instance: "<class \'tensorflow.python.feature_column.feature_column_v2._BaseFeaturesLayer\'>"
   is_instance: "<class \'tensorflow.python.keras.engine.base_layer.Layer\'>"
   is_instance: "<class \'tensorflow.python.training.checkpointable.base.Checkpointable\'>"
   is_instance: "<type \'object\'>"
-- 
GitLab


From 36d3a359562f89819013465e736e351a9782c9be Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Fri, 18 Jan 2019 13:52:59 -0800
Subject: [PATCH 1004/1095] [TF:XLA] Bump open source llvm revision to r351568

PiperOrigin-RevId: 229988232
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 3d8ea6d9e1..466262ab3f 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -512,11 +512,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "llvm",
         build_file = clean_dep("//third_party/llvm:llvm.autogenerated.BUILD"),
-        sha256 = "986122dee6053db98d4cf8a60efedd6d94e20c557cc97299a0c7d33b9efae12d",
-        strip_prefix = "llvm-329f768b5fc380a4bfa327396f108a8d8f33e77b",
+        sha256 = "d78389159aee7c3465b49843b46a8ac1b37db28db09dcf7522f6d31f32e5c97c",
+        strip_prefix = "llvm-316e6ae08e262dee261d1ca53a685b5ca6757a7f",
         urls = [
-            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/329f768b5fc380a4bfa327396f108a8d8f33e77b.tar.gz",
-            "https://github.com/llvm-mirror/llvm/archive/329f768b5fc380a4bfa327396f108a8d8f33e77b.tar.gz",
+            "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/316e6ae08e262dee261d1ca53a685b5ca6757a7f.tar.gz",
+            "https://github.com/llvm-mirror/llvm/archive/316e6ae08e262dee261d1ca53a685b5ca6757a7f.tar.gz",
         ],
     )
 
-- 
GitLab


From 04b0afd11dc4ab680793ae865917ac8c8f0b910c Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Fri, 18 Jan 2019 13:53:02 -0800
Subject: [PATCH 1005/1095] New Equality Check API

The new equality check API allows users to configure different options when comparing two shapes. Eventually this will subsume other equality check functions (Compatible, CompatibleIgnoringElementType, CompatibleIgnoringFpPrecision, etc).

PiperOrigin-RevId: 229988246
---
 tensorflow/compiler/xla/shape.cc      |  73 ++++++++++++++++++
 tensorflow/compiler/xla/shape.h       |  39 ++++++++++
 tensorflow/compiler/xla/shape_util.cc | 105 ++------------------------
 tensorflow/compiler/xla/shape_util.h  |   8 +-
 4 files changed, 121 insertions(+), 104 deletions(-)

diff --git a/tensorflow/compiler/xla/shape.cc b/tensorflow/compiler/xla/shape.cc
index 1a029efe85..a36d3547a0 100644
--- a/tensorflow/compiler/xla/shape.cc
+++ b/tensorflow/compiler/xla/shape.cc
@@ -112,6 +112,79 @@ void Shape::DeleteDimension(int64 dim_to_delete) {
   }
 }
 
+bool Shape::Equal::operator()(const Shape& lhs, const Shape& rhs) {
+  if (lhs.IsTuple()) {
+    return rhs.IsTuple() &&
+           absl::c_equal(
+               lhs.tuple_shapes(), rhs.tuple_shapes(),
+               [=](const Shape& l, const Shape& r) { return (*this)(l, r); });
+  } else if (!lhs.IsArray()) {
+    // Non-tuple, non-array tupes such as opaque and token types are trivially
+    // the same.
+    return lhs.element_type() == rhs.element_type();
+  }
+
+  if (!rhs.IsArray()) {
+    return false;
+  }
+
+  if (!ignore_element_type_) {
+    if ((ignore_fp_precision_ &&
+         !ShapeUtil::SameElementTypeIgnoringFpPrecision(lhs, rhs)) ||
+        (!ignore_fp_precision_ && !ShapeUtil::SameElementType(lhs, rhs))) {
+      VLOG(3) << "CompareShapes: lhs element type != rhs element type";
+      return false;
+    }
+  }
+
+  if (!ignore_layout_) {
+    if (lhs.layout().format() != rhs.layout().format()) {
+      VLOG(3) << "CompareShapes: lhs layout format != rhs layout format";
+      return false;
+    }
+    if (LayoutUtil::IsDenseArray(lhs)) {
+      if (!absl::c_equal(LayoutUtil::MinorToMajor(lhs),
+                         LayoutUtil::MinorToMajor(rhs))) {
+        VLOG(3) << "CompareShapes: lhs layout != rhs layout";
+        return false;
+      }
+
+      const auto& lhs_tiles = lhs.layout().tiles();
+      const auto& rhs_tiles = rhs.layout().tiles();
+      if (lhs_tiles.size() != rhs_tiles.size()) {
+        return false;
+      }
+      for (int64 i = 0; i < lhs_tiles.size(); i++) {
+        if (!absl::c_equal(lhs_tiles[i].dimensions(),
+                           rhs_tiles[i].dimensions())) {
+          return false;
+        }
+      }
+
+      if (lhs.layout().element_size_in_bits() !=
+          rhs.layout().element_size_in_bits()) {
+        return false;
+      }
+    }
+  }
+
+  if (!ShapeUtil::SameDimensions(lhs, rhs)) {
+    VLOG(3) << "CompareShapes: lhs dimensions != rhs dimensions";
+    return false;
+  }
+
+  if (!ignore_dynamic_dimension_) {
+    for (int i = 0; i < lhs.rank(); ++i) {
+      if (lhs.is_dynamic_dimension(i) != rhs.is_dynamic_dimension(i)) {
+        VLOG(3)
+            << "CompareShapes: lhs and rhs have different dynamic dimensions.";
+        return false;
+      }
+    }
+  }
+  return true;
+}
+
 std::ostream& operator<<(std::ostream& out, const Shape& shape) {
   out << shape.ToString(/*print_layout=*/true);
   return out;
diff --git a/tensorflow/compiler/xla/shape.h b/tensorflow/compiler/xla/shape.h
index dc4cdc31a7..e40e5ca37a 100644
--- a/tensorflow/compiler/xla/shape.h
+++ b/tensorflow/compiler/xla/shape.h
@@ -138,6 +138,45 @@ class Shape {
   string ShortDebugString() const { return ToProto().ShortDebugString(); }
   string DebugString() const { return ToProto().DebugString(); }
 
+  // Equal is a configurable functor to check the equality of two shapes.
+  //
+  // Examples:
+  //
+  // - Comparing two shapes ignoring they layout difference:
+  //   Equal().IgnoreLayout()(shape1, shape2);
+  //
+  // - Comparing two shapes ignoring they layout and element type difference:
+  //   Equal().IgnoreLayout().IgnoreElementType()(shape1, shape2);
+  class Equal {
+   public:
+    Equal() = default;
+
+    bool operator()(const Shape& lhs, const Shape& rhs);
+
+    Equal& IgnoreLayout() {
+      ignore_layout_ = true;
+      return *this;
+    }
+    Equal& IgnoreElementType() {
+      ignore_element_type_ = true;
+      return *this;
+    }
+    Equal& IgnoreFpPrecision() {
+      ignore_fp_precision_ = true;
+      return *this;
+    }
+    Equal& IgnoreDynamicDimension() {
+      ignore_dynamic_dimension_ = true;
+      return *this;
+    }
+
+   public:
+    bool ignore_layout_ = false;
+    bool ignore_element_type_ = false;
+    bool ignore_fp_precision_ = false;
+    bool ignore_dynamic_dimension_ = false;
+  };
+
  private:
   // The element type of this shape (tuple, array, etc).
   PrimitiveType element_type_ = PRIMITIVE_TYPE_INVALID;
diff --git a/tensorflow/compiler/xla/shape_util.cc b/tensorflow/compiler/xla/shape_util.cc
index f1ef202da0..799a4483df 100644
--- a/tensorflow/compiler/xla/shape_util.cc
+++ b/tensorflow/compiler/xla/shape_util.cc
@@ -85,77 +85,6 @@ bool ShapeIndexView::StartsWith(ShapeIndexView prefix) const {
 }
 
 namespace {
-
-// Recursive helper for comparing the equality of two shapes. Returns true if
-// the shapes are the same. If compare_layouts is true, then layouts must also
-// match.
-bool CompareShapes(const Shape& lhs, const Shape& rhs, bool compare_layouts,
-                   bool ignore_fp_precision) {
-  if ((ignore_fp_precision &&
-       !ShapeUtil::SameElementTypeIgnoringFpPrecision(lhs, rhs)) ||
-      (!ignore_fp_precision && !ShapeUtil::SameElementType(lhs, rhs))) {
-    VLOG(3) << "CompareShapes: lhs element type != rhs element type";
-    return false;
-  }
-
-  if (lhs.IsTuple()) {
-    return absl::c_equal(lhs.tuple_shapes(), rhs.tuple_shapes(),
-                         [=](const Shape& l, const Shape& r) {
-                           return CompareShapes(l, r, compare_layouts,
-                                                ignore_fp_precision);
-                         });
-  } else if (!lhs.IsArray()) {
-    // Non-tuple, non-array tupes such as opaque and token types are trivially
-    // the same.
-    return true;
-  }
-
-  if (compare_layouts) {
-    if (lhs.layout().format() != rhs.layout().format()) {
-      VLOG(3) << "CompareShapes: lhs layout format != rhs layout format";
-      return false;
-    }
-    if (LayoutUtil::IsDenseArray(lhs)) {
-      if (!absl::c_equal(LayoutUtil::MinorToMajor(lhs),
-                         LayoutUtil::MinorToMajor(rhs))) {
-        VLOG(3) << "CompareShapes: lhs layout != rhs layout";
-        return false;
-      }
-
-      const auto& lhs_tiles = lhs.layout().tiles();
-      const auto& rhs_tiles = rhs.layout().tiles();
-      if (lhs_tiles.size() != rhs_tiles.size()) {
-        return false;
-      }
-      for (int64 i = 0; i < lhs_tiles.size(); i++) {
-        if (!absl::c_equal(lhs_tiles[i].dimensions(),
-                           rhs_tiles[i].dimensions())) {
-          return false;
-        }
-      }
-
-      if (lhs.layout().element_size_in_bits() !=
-          rhs.layout().element_size_in_bits()) {
-        return false;
-      }
-    }
-  }
-
-  if (!ShapeUtil::SameDimensions(lhs, rhs)) {
-    VLOG(3) << "CompareShapes: lhs dimensions != rhs dimensions";
-    return false;
-  }
-
-  for (int i = 0; i < lhs.rank(); ++i) {
-    if (lhs.is_dynamic_dimension(i) != rhs.is_dynamic_dimension(i)) {
-      VLOG(3)
-          << "CompareShapes: lhs and rhs have different dynamic dimensions.";
-      return false;
-    }
-  }
-  return true;
-}
-
 // Constructs and returns the new shape with the given minor_to_major order in
 // its Layout.
 StatusOr<Shape> MakeShapeWithLayoutInternal(
@@ -182,12 +111,11 @@ StatusOr<Shape> MakeShapeWithLayoutInternal(
   TF_RETURN_IF_ERROR(ShapeUtil::ValidateShape(shape));
   return shape;
 }
-
 }  // namespace
 
 /* static */ bool ShapeUtil::Equal(const Shape& lhs, const Shape& rhs) {
-  bool equal = CompareShapes(lhs, rhs, /*compare_layouts=*/true,
-                             /*ignore_fp_precision=*/false);
+  bool equal = Shape::Equal()(lhs, rhs);
+
   if (!equal && VLOG_IS_ON(3)) {
     VLOG(3) << "ShapeUtil::Equal differ: lhs = " << lhs.ShortDebugString()
             << ", rhs = " << rhs.ShortDebugString();
@@ -198,8 +126,7 @@ StatusOr<Shape> MakeShapeWithLayoutInternal(
 
 /* static */ bool ShapeUtil::EqualIgnoringFpPrecision(const Shape& lhs,
                                                       const Shape& rhs) {
-  bool equal = CompareShapes(lhs, rhs, /*compare_layouts=*/true,
-                             /*ignore_fp_precision=*/true);
+  bool equal = Shape::Equal().IgnoreFpPrecision()(lhs, rhs);
   if (!equal && VLOG_IS_ON(3)) {
     VLOG(3) << "ShapeUtil::EqualIgnoringFpPrecision differ: lhs = "
             << lhs.ShortDebugString() << ", rhs = " << rhs.ShortDebugString();
@@ -563,37 +490,17 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout(
 }
 
 /* static */ bool ShapeUtil::Compatible(const Shape& lhs, const Shape& rhs) {
-  return CompareShapes(lhs, rhs, /*compare_layouts=*/false,
-                       /*ignore_fp_precision=*/false);
+  return Shape::Equal().IgnoreLayout()(lhs, rhs);
 }
 
 /* static */ bool ShapeUtil::CompatibleIgnoringElementType(const Shape& lhs,
                                                            const Shape& rhs) {
-  if (lhs.IsArray()) {
-    return rhs.IsArray() && SameDimensions(lhs, rhs);
-  } else if (lhs.element_type() == TUPLE) {
-    return rhs.element_type() == TUPLE &&
-           absl::c_equal(lhs.tuple_shapes(), rhs.tuple_shapes(),
-                         CompatibleIgnoringElementType);
-  } else {
-    // Opaque, token, etc types are vacuously compatible.
-    return lhs.element_type() == rhs.element_type();
-  }
+  return Shape::Equal().IgnoreElementType().IgnoreLayout()(lhs, rhs);
 }
 
 /* static */ bool ShapeUtil::CompatibleIgnoringFpPrecision(const Shape& lhs,
                                                            const Shape& rhs) {
-  if (lhs.IsArray()) {
-    return rhs.IsArray() && SameElementTypeIgnoringFpPrecision(lhs, rhs) &&
-           CompatibleIgnoringElementType(lhs, rhs);
-  } else if (lhs.element_type() == TUPLE) {
-    return rhs.element_type() == TUPLE &&
-           absl::c_equal(lhs.tuple_shapes(), rhs.tuple_shapes(),
-                         CompatibleIgnoringFpPrecision);
-  } else {
-    // Opaque, token, etc types are vacuously compatible.
-    return lhs.element_type() == rhs.element_type();
-  }
+  return Shape::Equal().IgnoreFpPrecision().IgnoreLayout()(lhs, rhs);
 }
 
 /* static */ int64 ShapeUtil::GetDimension(const Shape& shape,
diff --git a/tensorflow/compiler/xla/shape_util.h b/tensorflow/compiler/xla/shape_util.h
index e98c6e024b..fb6da7460e 100644
--- a/tensorflow/compiler/xla/shape_util.h
+++ b/tensorflow/compiler/xla/shape_util.h
@@ -675,11 +675,9 @@ class ShapeUtil {
 
   template <typename FnType>
   static void ForEachIndex(const Shape& shape, const FnType& visitor_function) {
-    ForEachIndexWithStatus(shape,
-                           [&](absl::Span<const int64> indices) {
-                             return StatusOr<bool>(visitor_function(indices));
-                           })
-        .IgnoreError();
+    ForEachIndexWithStatus(shape, [&](absl::Span<const int64> indices) {
+      return StatusOr<bool>(visitor_function(indices));
+    }).IgnoreError();
   }
 
   // A parallel version of ForEachIndex(WithStatus). This can only be used if
-- 
GitLab


From 916df403fbfc799ac5b6af49a0c1cdf3846e66aa Mon Sep 17 00:00:00 2001
From: Francois Chollet <fchollet@google.com>
Date: Fri, 18 Jan 2019 13:55:00 -0800
Subject: [PATCH 1006/1095] Extend tensor casting to graph mode in Layer's
 __call__ method.

PiperOrigin-RevId: 229988566
---
 tensorflow/python/keras/engine/base_layer.py      |  9 ++++-----
 tensorflow/python/keras/engine/base_layer_test.py | 15 ++++++++++++++-
 2 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/keras/engine/base_layer.py b/tensorflow/python/keras/engine/base_layer.py
index f63fb4e694..93849e771b 100644
--- a/tensorflow/python/keras/engine/base_layer.py
+++ b/tensorflow/python/keras/engine/base_layer.py
@@ -513,11 +513,10 @@ class Layer(checkpointable.Checkpointable):
       ValueError: if the layer's `call` method returns None (an invalid value).
     """
     input_list = nest.flatten(inputs)
-    if context.executing_eagerly():
-      # Accept NumPy inputs by converting to Tensors when executing eagerly.
-      if all(isinstance(x, (np.ndarray, float, int)) for x in input_list):
-        inputs = nest.map_structure(ops.convert_to_tensor, inputs)
-        input_list = nest.flatten(inputs)
+    # Accept NumPy inputs by converting to Tensors.
+    if all(isinstance(x, (np.ndarray, float, int)) for x in input_list):
+      inputs = nest.map_structure(ops.convert_to_tensor, inputs)
+      input_list = nest.flatten(inputs)
 
     # We will attempt to build a TF graph if & only if all inputs are symbolic.
     # This is always the case in graph mode. It can also be the case in eager
diff --git a/tensorflow/python/keras/engine/base_layer_test.py b/tensorflow/python/keras/engine/base_layer_test.py
index ebee4a3043..c925a8ca8a 100644
--- a/tensorflow/python/keras/engine/base_layer_test.py
+++ b/tensorflow/python/keras/engine/base_layer_test.py
@@ -26,6 +26,8 @@ from tensorflow.python.eager import context
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import test_util
+from tensorflow.python.keras import keras_parameterized
+from tensorflow.python.keras import testing_utils
 from tensorflow.python.keras.engine import base_layer
 from tensorflow.python.keras.optimizer_v2 import rmsprop
 from tensorflow.python.ops import array_ops
@@ -71,7 +73,7 @@ class InvalidLayer(base_layer.Layer):
     raise ValueError('You did something wrong!')
 
 
-class BaseLayerTest(test.TestCase, parameterized.TestCase):
+class BaseLayerTest(keras_parameterized.TestCase):
 
   @parameterized.parameters(DynamicLayer1, DynamicLayer2)
   def test_dynamic_layer_in_functional_model_in_graph_mode(self, layer_class):
@@ -210,6 +212,17 @@ class BaseLayerTest(test.TestCase, parameterized.TestCase):
     with self.assertRaisesRegexp(ValueError, 'You did something wrong!'):
       _ = InvalidLayer()(inputs)
 
+  @keras_parameterized.run_with_all_model_types
+  @test_util.run_in_graph_and_eager_modes
+  def test_build_with_numpy_data(self):
+    model_layers = [
+        keras.layers.Dense(3, activation='relu', kernel_initializer='ones'),
+        keras.layers.Dense(1, activation='sigmoid', kernel_initializer='ones')
+    ]
+    model = testing_utils.get_model_from_layers(model_layers, input_shape=(4,))
+    model(np.zeros((2, 4), dtype='float32'))
+    self.assertTrue(model.built)
+
   def test_using_symbolic_tensors_with_tf_ops(self):
     # Single-input.
     x = keras.Input((3,))
-- 
GitLab


From 5c5102a3c925748b8919c85a515955443a7594cd Mon Sep 17 00:00:00 2001
From: David Majnemer <majnemer@google.com>
Date: Fri, 18 Jan 2019 14:34:28 -0800
Subject: [PATCH 1007/1095] [XLA] Don't rely on ctype for classifying
 characters

Instead, use the ascii routines in ABSL. This makes the code do the same thing
regardless of how the locale is configured.

PiperOrigin-RevId: 229995801
---
 tensorflow/compiler/xla/metric_table_report.cc           | 4 ++--
 tensorflow/compiler/xla/parse_flags_from_env.cc          | 6 ++++--
 tensorflow/compiler/xla/service/BUILD                    | 3 ++-
 .../service/cpu/tests/cpu_eigen_dot_operation_test.cc    | 1 -
 .../compiler/xla/service/cpu/tests/cpu_intrinsic_test.cc | 7 ++++---
 tensorflow/compiler/xla/service/hlo_lexer.cc             | 9 +++++----
 .../compiler/xla/service/indexed_array_analysis_test.cc  | 5 ++---
 tensorflow/compiler/xla/service/name_uniquer.cc          | 5 +++--
 8 files changed, 22 insertions(+), 18 deletions(-)

diff --git a/tensorflow/compiler/xla/metric_table_report.cc b/tensorflow/compiler/xla/metric_table_report.cc
index ad1699a1ae..bad65ac320 100644
--- a/tensorflow/compiler/xla/metric_table_report.cc
+++ b/tensorflow/compiler/xla/metric_table_report.cc
@@ -15,9 +15,9 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/metric_table_report.h"
 
-#include <cctype>
 #include <unordered_map>
 
+#include "absl/strings/ascii.h"
 #include "absl/strings/str_cat.h"
 #include "absl/strings/str_format.h"
 #include "tensorflow/core/platform/logging.h"
@@ -249,7 +249,7 @@ string MetricTableReport::MetricString(double metric) {
   string output;
   // Copy leading non-digit characters unconditionally.
   // This picks up the leading sign.
-  while (!sp1.empty() && !isdigit(sp1[0])) {
+  while (!sp1.empty() && !absl::ascii_isdigit(sp1[0])) {
     output.push_back(sp1[0]);
     sp1.remove_prefix(1);
   }
diff --git a/tensorflow/compiler/xla/parse_flags_from_env.cc b/tensorflow/compiler/xla/parse_flags_from_env.cc
index 5b568888d1..91e71f5d1d 100644
--- a/tensorflow/compiler/xla/parse_flags_from_env.cc
+++ b/tensorflow/compiler/xla/parse_flags_from_env.cc
@@ -24,6 +24,7 @@ limitations under the License.
 #include <unordered_map>
 #include <vector>
 
+#include "absl/strings/ascii.h"
 #include "absl/strings/str_format.h"
 #include "absl/strings/str_join.h"
 #include "absl/types/span.h"
@@ -37,7 +38,7 @@ limitations under the License.
 
 namespace xla {
 
-static const char kWS[] = " \t\r\n";           // whitespace
+static const char kWS[] = " \t\r\n";  // whitespace
 
 // The following struct represents an argv[]-style array, parsed
 // from data gleaned from the environment.
@@ -104,7 +105,8 @@ static void ParseArgvFromString(const string& flag_str, EnvArgv* a) {
     // Set e to the index just past the end of the flag.
     size_t e = b;
     while (e != flag_str.size() && isascii(flag_str[e]) &&
-           (strchr("-_", flag_str[e]) != nullptr || isalnum(flag_str[e]))) {
+           (strchr("-_", flag_str[e]) != nullptr ||
+            absl::ascii_isalnum(flag_str[e]))) {
       e++;
     }
     if (e != flag_str.size() && flag_str[e] == '=' &&
diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index 2b2c32a681..3c123fd5c6 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -3656,12 +3656,13 @@ tf_cc_test(
     extra_copts = ["-Wno-string-plus-int"],
     deps = [
         ":hlo_matchers",
+        ":hlo_parser",
         ":indexed_array_analysis",
         "//tensorflow/compiler/xla:test",
-        "//tensorflow/compiler/xla/service:hlo_parser",
         "//tensorflow/compiler/xla/tests:hlo_test_base",
         "//tensorflow/compiler/xla/tests:test_utils",
         "//tensorflow/core:test",
+        "@com_google_absl//absl/strings",
     ],
 )
 
diff --git a/tensorflow/compiler/xla/service/cpu/tests/cpu_eigen_dot_operation_test.cc b/tensorflow/compiler/xla/service/cpu/tests/cpu_eigen_dot_operation_test.cc
index 0b4ac9dc29..8b7f843582 100644
--- a/tensorflow/compiler/xla/service/cpu/tests/cpu_eigen_dot_operation_test.cc
+++ b/tensorflow/compiler/xla/service/cpu/tests/cpu_eigen_dot_operation_test.cc
@@ -16,7 +16,6 @@ limitations under the License.
 // Tests that we call into Eigen for dot operations as needed.
 
 #include <algorithm>
-#include <cctype>
 #include <string>
 
 #include "absl/strings/str_cat.h"
diff --git a/tensorflow/compiler/xla/service/cpu/tests/cpu_intrinsic_test.cc b/tensorflow/compiler/xla/service/cpu/tests/cpu_intrinsic_test.cc
index 3fb0e3cd91..9078b8fd1f 100644
--- a/tensorflow/compiler/xla/service/cpu/tests/cpu_intrinsic_test.cc
+++ b/tensorflow/compiler/xla/service/cpu/tests/cpu_intrinsic_test.cc
@@ -14,9 +14,9 @@ limitations under the License.
 ==============================================================================*/
 
 #include <algorithm>
-#include <cctype>
 #include <string>
 
+#include "absl/strings/ascii.h"
 #include "absl/strings/str_cat.h"
 #include "tensorflow/compiler/xla/service/cpu/cpu_compiler.h"
 #include "tensorflow/compiler/xla/service/cpu/tests/cpu_codegen_test.h"
@@ -59,8 +59,9 @@ class CpuUnaryIntrinsicTest
 
     string features{spec.features.data(), spec.features.size()};
     if (!features.empty()) {
-      std::replace_if(features.begin(), features.end(),
-                      [](char c) { return c != '_' && !isalnum(c); }, '_');
+      std::replace_if(
+          features.begin(), features.end(),
+          [](char c) { return c != '_' && !absl::ascii_isalnum(c); }, '_');
     } else {
       features = "";
     }
diff --git a/tensorflow/compiler/xla/service/hlo_lexer.cc b/tensorflow/compiler/xla/service/hlo_lexer.cc
index 5e81515134..798760885d 100644
--- a/tensorflow/compiler/xla/service/hlo_lexer.cc
+++ b/tensorflow/compiler/xla/service/hlo_lexer.cc
@@ -17,6 +17,7 @@ limitations under the License.
 
 #include <unordered_map>
 
+#include "absl/strings/ascii.h"
 #include "absl/strings/escaping.h"
 #include "absl/strings/numbers.h"
 #include "absl/strings/str_split.h"
@@ -37,8 +38,8 @@ constexpr int kError = -2;
 
 // [a-zA-Z0-9_.-]
 bool IsIdentifierChar(char c) {
-  return isalnum(static_cast<unsigned char>(c)) || c == '-' || c == '.' ||
-         c == '_';
+  return absl::ascii_isalnum(static_cast<unsigned char>(c)) || c == '-' ||
+         c == '.' || c == '_';
 }
 
 }  // namespace
@@ -105,7 +106,7 @@ TokKind HloLexer::LexToken() {
     switch (current_char) {
       default:
         // [a-zA-Z_]
-        if (isalpha(static_cast<unsigned char>(current_char)) ||
+        if (absl::ascii_isalpha(static_cast<unsigned char>(current_char)) ||
             current_char == '_') {
           return LexIdentifier();
         }
@@ -300,7 +301,7 @@ TokKind HloLexer::LexIdentifier() {
 // name ::= [a-zA-Z_][a-zA-Z0-9_.-]*
 TokKind HloLexer::LexPercent() {
   const char* name_start = current_ptr_;
-  if (isalpha(static_cast<unsigned char>(PeekCurrentChar())) ||
+  if (absl::ascii_isalpha(static_cast<unsigned char>(PeekCurrentChar())) ||
       PeekCurrentChar() == '_') {
     current_ptr_++;
     while (IsIdentifierChar(PeekCurrentChar())) {
diff --git a/tensorflow/compiler/xla/service/indexed_array_analysis_test.cc b/tensorflow/compiler/xla/service/indexed_array_analysis_test.cc
index 295465c848..62107b5a88 100644
--- a/tensorflow/compiler/xla/service/indexed_array_analysis_test.cc
+++ b/tensorflow/compiler/xla/service/indexed_array_analysis_test.cc
@@ -13,9 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include <ctype.h>
-
 #include "tensorflow/compiler/xla/service/indexed_array_analysis.h"
+#include "absl/strings/ascii.h"
 #include "tensorflow/compiler/xla/tests/hlo_test_base.h"
 #include "tensorflow/compiler/xla/tests/test_utils.h"
 
@@ -43,7 +42,7 @@ class IndexedArrayAnalysisTest : public HloTestBase {
     string result;
 
     for (char c : text) {
-      if (!isspace(c)) {
+      if (!absl::ascii_isspace(c)) {
         result.push_back(c);
       } else if (!result.empty() && result.back() != ' ') {
         result.push_back(' ');
diff --git a/tensorflow/compiler/xla/service/name_uniquer.cc b/tensorflow/compiler/xla/service/name_uniquer.cc
index f6feed2993..e55b83d17e 100644
--- a/tensorflow/compiler/xla/service/name_uniquer.cc
+++ b/tensorflow/compiler/xla/service/name_uniquer.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/service/name_uniquer.h"
 
+#include "absl/strings/ascii.h"
 #include "absl/strings/numbers.h"
 #include "absl/strings/str_cat.h"
 #include "tensorflow/compiler/xla/primitive_util.h"
@@ -28,7 +29,7 @@ namespace {
 
 bool IsAllowed(char character) {
   auto c = static_cast<unsigned char>(character);
-  return (isalnum(c) != 0) || c == '_' || c == '.' || c == '-';
+  return (absl::ascii_isalnum(c) != 0) || c == '_' || c == '.' || c == '-';
 }
 
 }  // namespace
@@ -46,7 +47,7 @@ NameUniquer::NameUniquer(const string& separator) {
 
   string result = name;
   char c = static_cast<unsigned char>(result[0]);
-  if (!isalpha(c) && c != '_') {
+  if (!absl::ascii_isalpha(c) && c != '_') {
     result[0] = '_';
   }
   for (int i = 1; i < result.length(); i++) {
-- 
GitLab


From f6fde51408b1a474ee32f7c1ce48b9358775babe Mon Sep 17 00:00:00 2001
From: Francois Chollet <fchollet@google.com>
Date: Fri, 18 Jan 2019 14:38:26 -0800
Subject: [PATCH 1008/1095] Small cleanup of backend tests.

PiperOrigin-RevId: 229996461
---
 tensorflow/python/keras/backend_test.py | 709 +++++++++++-------------
 1 file changed, 337 insertions(+), 372 deletions(-)

diff --git a/tensorflow/python/keras/backend_test.py b/tensorflow/python/keras/backend_test.py
index e3f86ee2c0..d25c214622 100644
--- a/tensorflow/python/keras/backend_test.py
+++ b/tensorflow/python/keras/backend_test.py
@@ -122,20 +122,19 @@ class BackendUtilsTest(test.TestCase):
         sess.run(y, feed_dict={x: np.random.random((2, 3))})
 
   def test_learning_phase_scope(self):
-    with self.cached_session():
-      initial_learning_phase = keras.backend.learning_phase()
-      with keras.backend.learning_phase_scope(1) as lp:
-        self.assertEqual(lp, 1)
-        self.assertEqual(keras.backend.learning_phase(), 1)
-      self.assertEqual(keras.backend.learning_phase(), initial_learning_phase)
-      with keras.backend.learning_phase_scope(0) as lp:
-        self.assertEqual(lp, 0)
-        self.assertEqual(keras.backend.learning_phase(), 0)
-      self.assertEqual(keras.backend.learning_phase(), initial_learning_phase)
-      with self.assertRaises(ValueError):
-        with keras.backend.learning_phase_scope(None):
-          pass
-      self.assertEqual(keras.backend.learning_phase(), initial_learning_phase)
+    initial_learning_phase = keras.backend.learning_phase()
+    with keras.backend.learning_phase_scope(1) as lp:
+      self.assertEqual(lp, 1)
+      self.assertEqual(keras.backend.learning_phase(), 1)
+    self.assertEqual(keras.backend.learning_phase(), initial_learning_phase)
+    with keras.backend.learning_phase_scope(0) as lp:
+      self.assertEqual(lp, 0)
+      self.assertEqual(keras.backend.learning_phase(), 0)
+    self.assertEqual(keras.backend.learning_phase(), initial_learning_phase)
+    with self.assertRaises(ValueError):
+      with keras.backend.learning_phase_scope(None):
+        pass
+    self.assertEqual(keras.backend.learning_phase(), initial_learning_phase)
 
   def test_int_shape(self):
     x = keras.backend.ones(shape=(3, 4))
@@ -146,21 +145,20 @@ class BackendUtilsTest(test.TestCase):
       self.assertEqual(keras.backend.int_shape(x), (None, 4))
 
   def test_in_train_phase(self):
-    with self.cached_session():
-      y1 = keras.backend.variable(1)
-      y2 = keras.backend.variable(2)
-      if context.executing_eagerly():
-        with keras.backend.learning_phase_scope(0):
-          y_val_test = keras.backend.in_train_phase(y1, y2).numpy()
-        with keras.backend.learning_phase_scope(1):
-          y_val_train = keras.backend.in_train_phase(y1, y2).numpy()
-      else:
-        y = keras.backend.in_train_phase(y1, y2)
-        f = keras.backend.function([keras.backend.learning_phase()], [y])
-        y_val_test = f([0])[0]
-        y_val_train = f([1])[0]
-      self.assertAllClose(y_val_test, 2)
-      self.assertAllClose(y_val_train, 1)
+    y1 = keras.backend.variable(1)
+    y2 = keras.backend.variable(2)
+    if context.executing_eagerly():
+      with keras.backend.learning_phase_scope(0):
+        y_val_test = keras.backend.in_train_phase(y1, y2).numpy()
+      with keras.backend.learning_phase_scope(1):
+        y_val_train = keras.backend.in_train_phase(y1, y2).numpy()
+    else:
+      y = keras.backend.in_train_phase(y1, y2)
+      f = keras.backend.function([keras.backend.learning_phase()], [y])
+      y_val_test = f([0])[0]
+      y_val_train = f([1])[0]
+    self.assertAllClose(y_val_test, 2)
+    self.assertAllClose(y_val_train, 1)
 
   def test_is_keras_tensor(self):
     x = keras.backend.variable(1)
@@ -187,74 +185,63 @@ class BackendUtilsTest(test.TestCase):
 class BackendVariableTest(test.TestCase):
 
   def test_zeros(self):
-    with self.cached_session():
-      x = keras.backend.zeros((3, 4))
-      val = keras.backend.eval(x)
-      self.assertAllClose(val, np.zeros((3, 4)))
+    x = keras.backend.zeros((3, 4))
+    val = keras.backend.eval(x)
+    self.assertAllClose(val, np.zeros((3, 4)))
 
   def test_ones(self):
-    with self.cached_session():
-      x = keras.backend.ones((3, 4))
-      val = keras.backend.eval(x)
-      self.assertAllClose(val, np.ones((3, 4)))
+    x = keras.backend.ones((3, 4))
+    val = keras.backend.eval(x)
+    self.assertAllClose(val, np.ones((3, 4)))
 
   def test_eye(self):
-    with self.cached_session():
-      x = keras.backend.eye(4)
-      val = keras.backend.eval(x)
-      self.assertAllClose(val, np.eye(4))
+    x = keras.backend.eye(4)
+    val = keras.backend.eval(x)
+    self.assertAllClose(val, np.eye(4))
 
   def test_zeros_like(self):
-    with self.cached_session():
-      x = keras.backend.zeros((3, 4))
-      y = keras.backend.zeros_like(x)
-      val = keras.backend.eval(y)
-      self.assertAllClose(val, np.zeros((3, 4)))
+    x = keras.backend.zeros((3, 4))
+    y = keras.backend.zeros_like(x)
+    val = keras.backend.eval(y)
+    self.assertAllClose(val, np.zeros((3, 4)))
 
   def test_ones_like(self):
-    with self.cached_session():
-      x = keras.backend.zeros((3, 4))
-      y = keras.backend.ones_like(x)
-      val = keras.backend.eval(y)
-      self.assertAllClose(val, np.ones((3, 4)))
+    x = keras.backend.zeros((3, 4))
+    y = keras.backend.ones_like(x)
+    val = keras.backend.eval(y)
+    self.assertAllClose(val, np.ones((3, 4)))
 
   def test_random_uniform_variable(self):
-    with self.cached_session():
-      x = keras.backend.random_uniform_variable((30, 20), low=1, high=2, seed=0)
-      val = keras.backend.eval(x)
-      self.assertAllClose(val.mean(), 1.5, atol=1e-1)
-      self.assertAllClose(val.max(), 2., atol=1e-1)
-      self.assertAllClose(val.min(), 1., atol=1e-1)
+    x = keras.backend.random_uniform_variable((30, 20), low=1, high=2, seed=0)
+    val = keras.backend.eval(x)
+    self.assertAllClose(val.mean(), 1.5, atol=1e-1)
+    self.assertAllClose(val.max(), 2., atol=1e-1)
+    self.assertAllClose(val.min(), 1., atol=1e-1)
 
   def test_random_normal_variable(self):
-    with self.cached_session():
-      x = keras.backend.random_normal_variable((30, 20), 1., 0.5,
-                                               seed=0)
-      val = keras.backend.eval(x)
-      self.assertAllClose(val.mean(), 1., atol=1e-1)
-      self.assertAllClose(val.std(), 0.5, atol=1e-1)
+    x = keras.backend.random_normal_variable((30, 20), 1., 0.5, seed=0)
+    val = keras.backend.eval(x)
+    self.assertAllClose(val.mean(), 1., atol=1e-1)
+    self.assertAllClose(val.std(), 0.5, atol=1e-1)
 
   def test_count_params(self):
-    with self.cached_session():
-      x = keras.backend.zeros((4, 5))
-      val = keras.backend.count_params(x)
-      self.assertAllClose(val, 20)
+    x = keras.backend.zeros((4, 5))
+    val = keras.backend.count_params(x)
+    self.assertAllClose(val, 20)
 
   def test_constant(self):
-    with self.cached_session():
-      ref_val = np.random.random((3, 4)).astype('float32')
-      x = keras.backend.constant(ref_val)
-      val = keras.backend.eval(x)
-      self.assertAllClose(val, ref_val)
+    ref_val = np.random.random((3, 4)).astype('float32')
+    x = keras.backend.constant(ref_val)
+    val = keras.backend.eval(x)
+    self.assertAllClose(val, ref_val)
 
   def test_sparse_variable(self):
-    with self.cached_session():
-      val = scipy.sparse.eye(10)
-      x = keras.backend.variable(val)
-      self.assertTrue(isinstance(x, sparse_tensor.SparseTensor))
+    val = scipy.sparse.eye(10)
+    x = keras.backend.variable(val)
+    self.assertTrue(isinstance(x, sparse_tensor.SparseTensor))
 
-      y = keras.backend.to_dense(x)
-      self.assertFalse(keras.backend.is_sparse(y))
+    y = keras.backend.to_dense(x)
+    self.assertFalse(keras.backend.is_sparse(y))
 
 
 @test_util.run_all_in_graph_and_eager_modes
@@ -292,20 +279,19 @@ class BackendLinearAlgebraTest(test.TestCase):
         (keras.backend.argmax, np.argmax),
     ]
     for keras_op, np_op in ops_to_test:
-      with self.cached_session():
-        compare_single_input_op_to_numpy(keras_op, np_op, input_shape=(4, 7, 5),
-                                         keras_kwargs={'axis': 1},
-                                         np_kwargs={'axis': 1})
-        compare_single_input_op_to_numpy(keras_op, np_op, input_shape=(4, 7, 5),
-                                         keras_kwargs={'axis': -1},
-                                         np_kwargs={'axis': -1})
-        if 'keepdims' in tf_inspect.getargspec(keras_op).args:
-          compare_single_input_op_to_numpy(keras_op, np_op,
-                                           input_shape=(4, 7, 5),
-                                           keras_kwargs={'axis': 1,
-                                                         'keepdims': True},
-                                           np_kwargs={'axis': 1,
-                                                      'keepdims': True})
+      compare_single_input_op_to_numpy(keras_op, np_op, input_shape=(4, 7, 5),
+                                       keras_kwargs={'axis': 1},
+                                       np_kwargs={'axis': 1})
+      compare_single_input_op_to_numpy(keras_op, np_op, input_shape=(4, 7, 5),
+                                       keras_kwargs={'axis': -1},
+                                       np_kwargs={'axis': -1})
+      if 'keepdims' in tf_inspect.getargspec(keras_op).args:
+        compare_single_input_op_to_numpy(keras_op, np_op,
+                                         input_shape=(4, 7, 5),
+                                         keras_kwargs={'axis': 1,
+                                                       'keepdims': True},
+                                         np_kwargs={'axis': 1,
+                                                    'keepdims': True})
 
   def test_elementwise_ops(self):
     ops_to_test = [
@@ -318,32 +304,28 @@ class BackendLinearAlgebraTest(test.TestCase):
         (keras.backend.exp, np.exp),
     ]
     for keras_op, np_op in ops_to_test:
-      with self.cached_session():
-        compare_single_input_op_to_numpy(keras_op, np_op, input_shape=(4, 7))
+      compare_single_input_op_to_numpy(keras_op, np_op, input_shape=(4, 7))
 
     ops_to_test = [
         (keras.backend.sqrt, np.sqrt),
         (keras.backend.log, np.log),
     ]
     for keras_op, np_op in ops_to_test:
-      with self.cached_session():
-        compare_single_input_op_to_numpy(keras_op, np_op,
-                                         input_shape=(4, 7),
-                                         negative_values=False)
+      compare_single_input_op_to_numpy(keras_op, np_op,
+                                       input_shape=(4, 7),
+                                       negative_values=False)
 
-    with self.cached_session():
-      compare_single_input_op_to_numpy(
-          keras.backend.clip, np.clip,
-          input_shape=(6, 4),
-          keras_kwargs={'min_value': 0.1, 'max_value': 2.4},
-          np_kwargs={'a_min': 0.1, 'a_max': 1.4})
+    compare_single_input_op_to_numpy(
+        keras.backend.clip, np.clip,
+        input_shape=(6, 4),
+        keras_kwargs={'min_value': 0.1, 'max_value': 2.4},
+        np_kwargs={'a_min': 0.1, 'a_max': 1.4})
 
-    with self.cached_session():
-      compare_single_input_op_to_numpy(
-          keras.backend.pow, np.power,
-          input_shape=(6, 4),
-          keras_args=[3],
-          np_args=[3])
+    compare_single_input_op_to_numpy(
+        keras.backend.pow, np.power,
+        input_shape=(6, 4),
+        keras_args=[3],
+        np_args=[3])
 
   def test_two_tensor_ops(self):
     ops_to_test = [
@@ -357,84 +339,82 @@ class BackendLinearAlgebraTest(test.TestCase):
         (keras.backend.minimum, np.minimum),
     ]
     for keras_op, np_op in ops_to_test:
-      with self.cached_session():
-        compare_two_inputs_op_to_numpy(keras_op, np_op,
-                                       input_shape_a=(4, 7),
-                                       input_shape_b=(4, 7))
+      compare_two_inputs_op_to_numpy(keras_op, np_op,
+                                     input_shape_a=(4, 7),
+                                     input_shape_b=(4, 7))
 
   def test_relu(self):
     x = ops.convert_to_tensor([[-4, 0], [2, 7]], 'float32')
-    with self.cached_session():
-      # standard relu
-      relu_op = keras.backend.relu(x)
-      self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [2, 7]])
 
-      # alpha (leaky relu used)
-      relu_op = keras.backend.relu(x, alpha=0.5)
-      if not context.executing_eagerly():
-        self.assertTrue('LeakyRelu' in relu_op.name)
-      self.assertAllClose(keras.backend.eval(relu_op), [[-2, 0], [2, 7]])
+    # standard relu
+    relu_op = keras.backend.relu(x)
+    self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [2, 7]])
 
-      # max_value < some elements
-      relu_op = keras.backend.relu(x, max_value=5)
-      self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [2, 5]])
+    # alpha (leaky relu used)
+    relu_op = keras.backend.relu(x, alpha=0.5)
+    if not context.executing_eagerly():
+      self.assertTrue('LeakyRelu' in relu_op.name)
+    self.assertAllClose(keras.backend.eval(relu_op), [[-2, 0], [2, 7]])
 
-      # nn.relu6 used
-      relu_op = keras.backend.relu(x, max_value=6)
-      if not context.executing_eagerly():
-        self.assertTrue('Relu6' in relu_op.name)  # uses tf.nn.relu6
-      self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [2, 6]])
+    # max_value < some elements
+    relu_op = keras.backend.relu(x, max_value=5)
+    self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [2, 5]])
+
+    # nn.relu6 used
+    relu_op = keras.backend.relu(x, max_value=6)
+    if not context.executing_eagerly():
+      self.assertTrue('Relu6' in relu_op.name)  # uses tf.nn.relu6
+    self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [2, 6]])
 
-      # max value > 6
-      relu_op = keras.backend.relu(x, max_value=10)
-      self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [2, 7]])
+    # max value > 6
+    relu_op = keras.backend.relu(x, max_value=10)
+    self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [2, 7]])
 
-      # max value is float
-      relu_op = keras.backend.relu(x, max_value=4.3)
-      self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [2, 4.3]])
+    # max value is float
+    relu_op = keras.backend.relu(x, max_value=4.3)
+    self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [2, 4.3]])
 
-      # max value == 0
-      relu_op = keras.backend.relu(x, max_value=0)
-      self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [0, 0]])
+    # max value == 0
+    relu_op = keras.backend.relu(x, max_value=0)
+    self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [0, 0]])
 
-      # alpha and max_value
-      relu_op = keras.backend.relu(x, alpha=0.25, max_value=3)
-      self.assertAllClose(keras.backend.eval(relu_op), [[-1, 0], [2, 3]])
+    # alpha and max_value
+    relu_op = keras.backend.relu(x, alpha=0.25, max_value=3)
+    self.assertAllClose(keras.backend.eval(relu_op), [[-1, 0], [2, 3]])
 
-      # threshold
-      relu_op = keras.backend.relu(x, threshold=3)
-      self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [0, 7]])
+    # threshold
+    relu_op = keras.backend.relu(x, threshold=3)
+    self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [0, 7]])
 
-      # threshold is float
-      relu_op = keras.backend.relu(x, threshold=1.5)
-      self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [2, 7]])
+    # threshold is float
+    relu_op = keras.backend.relu(x, threshold=1.5)
+    self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [2, 7]])
 
-      # threshold is negative
-      relu_op = keras.backend.relu(x, threshold=-5)
-      self.assertAllClose(keras.backend.eval(relu_op), [[-4, 0], [2, 7]])
+    # threshold is negative
+    relu_op = keras.backend.relu(x, threshold=-5)
+    self.assertAllClose(keras.backend.eval(relu_op), [[-4, 0], [2, 7]])
 
-      # threshold and max_value
-      relu_op = keras.backend.relu(x, threshold=3, max_value=5)
-      self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [0, 5]])
+    # threshold and max_value
+    relu_op = keras.backend.relu(x, threshold=3, max_value=5)
+    self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [0, 5]])
 
-      # threshold and alpha
-      relu_op = keras.backend.relu(x, alpha=0.25, threshold=4)
-      self.assertAllClose(keras.backend.eval(relu_op), [[-2, -1], [-0.5, 7]])
+    # threshold and alpha
+    relu_op = keras.backend.relu(x, alpha=0.25, threshold=4)
+    self.assertAllClose(keras.backend.eval(relu_op), [[-2, -1], [-0.5, 7]])
 
-      # threshold, alpha, and max_value
-      relu_op = keras.backend.relu(x, alpha=0.25, threshold=4, max_value=5)
-      self.assertAllClose(keras.backend.eval(relu_op), [[-2, -1], [-0.5, 5]])
+    # threshold, alpha, and max_value
+    relu_op = keras.backend.relu(x, alpha=0.25, threshold=4, max_value=5)
+    self.assertAllClose(keras.backend.eval(relu_op), [[-2, -1], [-0.5, 5]])
 
 
 @test_util.run_all_in_graph_and_eager_modes
 class BackendShapeOpsTest(test.TestCase):
 
   def test_reshape(self):
-    with self.cached_session():
-      compare_single_input_op_to_numpy(keras.backend.reshape, np.reshape,
-                                       input_shape=(4, 7),
-                                       keras_args=[(2, 14)],
-                                       np_args=[(2, 14)])
+    compare_single_input_op_to_numpy(keras.backend.reshape, np.reshape,
+                                     input_shape=(4, 7),
+                                     keras_args=[(2, 14)],
+                                     np_args=[(2, 14)])
 
   def test_concatenate(self):
     a = keras.backend.variable(np.ones((1, 2, 3)))
@@ -443,12 +423,11 @@ class BackendShapeOpsTest(test.TestCase):
     self.assertEqual(y.get_shape().as_list(), [1, 2, 5])
 
   def test_permute_dimensions(self):
-    with self.cached_session():
-      compare_single_input_op_to_numpy(keras.backend.permute_dimensions,
-                                       np.transpose,
-                                       input_shape=(4, 7),
-                                       keras_args=[(1, 0)],
-                                       np_args=[(1, 0)])
+    compare_single_input_op_to_numpy(keras.backend.permute_dimensions,
+                                     np.transpose,
+                                     input_shape=(4, 7),
+                                     keras_args=[(1, 0)],
+                                     np_args=[(1, 0)])
 
   def test_resize_images(self):
     height_factor = 2
@@ -523,18 +502,16 @@ class BackendShapeOpsTest(test.TestCase):
     self.assertEqual(y.get_shape().as_list(), [1, 2, 3])
 
   def test_flatten(self):
-    with self.cached_session():
-      compare_single_input_op_to_numpy(keras.backend.flatten,
-                                       np.reshape,
-                                       input_shape=(4, 7, 6),
-                                       np_args=[(4 * 7 * 6,)])
+    compare_single_input_op_to_numpy(keras.backend.flatten,
+                                     np.reshape,
+                                     input_shape=(4, 7, 6),
+                                     np_args=[(4 * 7 * 6,)])
 
   def test_batch_flatten(self):
-    with self.cached_session():
-      compare_single_input_op_to_numpy(keras.backend.batch_flatten,
-                                       np.reshape,
-                                       input_shape=(4, 7, 6),
-                                       np_args=[(4, 7 * 6)])
+    compare_single_input_op_to_numpy(keras.backend.batch_flatten,
+                                     np.reshape,
+                                     input_shape=(4, 7, 6),
+                                     np_args=[(4, 7 * 6)])
 
   def test_temporal_padding(self):
 
@@ -545,12 +522,11 @@ class BackendShapeOpsTest(test.TestCase):
       y[:, padding[0]:-padding[1], :] = x
       return y
 
-    with self.cached_session():
-      compare_single_input_op_to_numpy(keras.backend.temporal_padding,
-                                       ref_op,
-                                       input_shape=(4, 7, 6),
-                                       keras_args=[(2, 3)],
-                                       np_args=[(2, 3)])
+    compare_single_input_op_to_numpy(keras.backend.temporal_padding,
+                                     ref_op,
+                                     input_shape=(4, 7, 6),
+                                     keras_args=[(2, 3)],
+                                     np_args=[(2, 3)])
 
   def test_spatial_2d_padding(self):
 
@@ -568,23 +544,22 @@ class BackendShapeOpsTest(test.TestCase):
         y[:, :, padding[0][0]:-padding[0][1], padding[1][0]:-padding[1][1]] = x
       return y
 
-    with self.cached_session():
-      compare_single_input_op_to_numpy(
-          keras.backend.spatial_2d_padding,
-          ref_op,
-          input_shape=(2, 3, 2, 3),
-          keras_args=[((2, 3), (1, 2))],
-          keras_kwargs={'data_format': 'channels_last'},
-          np_args=[((2, 3), (1, 2))],
-          np_kwargs={'data_format': 'channels_last'})
-      compare_single_input_op_to_numpy(
-          keras.backend.spatial_2d_padding,
-          ref_op,
-          input_shape=(2, 3, 2, 3),
-          keras_args=[((2, 3), (1, 2))],
-          keras_kwargs={'data_format': 'channels_first'},
-          np_args=[((2, 3), (1, 2))],
-          np_kwargs={'data_format': 'channels_first'})
+    compare_single_input_op_to_numpy(
+        keras.backend.spatial_2d_padding,
+        ref_op,
+        input_shape=(2, 3, 2, 3),
+        keras_args=[((2, 3), (1, 2))],
+        keras_kwargs={'data_format': 'channels_last'},
+        np_args=[((2, 3), (1, 2))],
+        np_kwargs={'data_format': 'channels_last'})
+    compare_single_input_op_to_numpy(
+        keras.backend.spatial_2d_padding,
+        ref_op,
+        input_shape=(2, 3, 2, 3),
+        keras_args=[((2, 3), (1, 2))],
+        keras_kwargs={'data_format': 'channels_first'},
+        np_args=[((2, 3), (1, 2))],
+        np_kwargs={'data_format': 'channels_first'})
 
   def test_spatial_3d_padding(self):
 
@@ -611,73 +586,70 @@ class BackendShapeOpsTest(test.TestCase):
           padding[2][0]:-padding[2][1]] = x
       return y
 
-    with self.cached_session():
-      compare_single_input_op_to_numpy(
-          keras.backend.spatial_3d_padding,
-          ref_op,
-          input_shape=(2, 3, 2, 3, 2),
-          keras_args=[((2, 3), (1, 2), (2, 3))],
-          keras_kwargs={'data_format': 'channels_last'},
-          np_args=[((2, 3), (1, 2), (2, 3))],
-          np_kwargs={'data_format': 'channels_last'})
-      compare_single_input_op_to_numpy(
-          keras.backend.spatial_3d_padding,
-          ref_op,
-          input_shape=(2, 3, 2, 3, 2),
-          keras_args=[((2, 3), (1, 2), (2, 3))],
-          keras_kwargs={'data_format': 'channels_first'},
-          np_args=[((2, 3), (1, 2), (2, 3))],
-          np_kwargs={'data_format': 'channels_first'})
+    compare_single_input_op_to_numpy(
+        keras.backend.spatial_3d_padding,
+        ref_op,
+        input_shape=(2, 3, 2, 3, 2),
+        keras_args=[((2, 3), (1, 2), (2, 3))],
+        keras_kwargs={'data_format': 'channels_last'},
+        np_args=[((2, 3), (1, 2), (2, 3))],
+        np_kwargs={'data_format': 'channels_last'})
+    compare_single_input_op_to_numpy(
+        keras.backend.spatial_3d_padding,
+        ref_op,
+        input_shape=(2, 3, 2, 3, 2),
+        keras_args=[((2, 3), (1, 2), (2, 3))],
+        keras_kwargs={'data_format': 'channels_first'},
+        np_args=[((2, 3), (1, 2), (2, 3))],
+        np_kwargs={'data_format': 'channels_first'})
 
 
 @test_util.run_all_in_graph_and_eager_modes
 class BackendNNOpsTest(test.TestCase, parameterized.TestCase):
 
   def test_bias_add(self):
-    with self.cached_session():
-      keras_op = keras.backend.bias_add
-      np_op = np.add
-      compare_two_inputs_op_to_numpy(keras_op, np_op,
-                                     input_shape_a=(4, 7),
-                                     input_shape_b=(7,))
-      compare_two_inputs_op_to_numpy(keras_op, np_op,
-                                     input_shape_a=(4, 3, 7),
-                                     input_shape_b=(7,))
-      compare_two_inputs_op_to_numpy(keras_op, np_op,
-                                     input_shape_a=(4, 3, 5, 7),
-                                     input_shape_b=(7,))
-      compare_two_inputs_op_to_numpy(keras_op, np_op,
-                                     input_shape_a=(4, 3, 5, 2, 7),
-                                     input_shape_b=(7,))
-
-      with self.assertRaises((ValueError, errors_impl.InvalidArgumentError)):
-        x = keras.backend.variable((3, 4))
-        b = keras.backend.variable((3, 4))
-        keras.backend.bias_add(x, b)
-      with self.assertRaises(ValueError):
-        x = keras.backend.variable((3, 4))
-        b = keras.backend.variable((4,))
-        keras.backend.bias_add(x, b, data_format='unknown')
+    keras_op = keras.backend.bias_add
+    np_op = np.add
+    compare_two_inputs_op_to_numpy(keras_op, np_op,
+                                   input_shape_a=(4, 7),
+                                   input_shape_b=(7,))
+    compare_two_inputs_op_to_numpy(keras_op, np_op,
+                                   input_shape_a=(4, 3, 7),
+                                   input_shape_b=(7,))
+    compare_two_inputs_op_to_numpy(keras_op, np_op,
+                                   input_shape_a=(4, 3, 5, 7),
+                                   input_shape_b=(7,))
+    compare_two_inputs_op_to_numpy(keras_op, np_op,
+                                   input_shape_a=(4, 3, 5, 2, 7),
+                                   input_shape_b=(7,))
+
+    with self.assertRaises((ValueError, errors_impl.InvalidArgumentError)):
+      x = keras.backend.variable((3, 4))
+      b = keras.backend.variable((3, 4))
+      keras.backend.bias_add(x, b)
+    with self.assertRaises(ValueError):
+      x = keras.backend.variable((3, 4))
+      b = keras.backend.variable((4,))
+      keras.backend.bias_add(x, b, data_format='unknown')
 
   def test_bias_add_channels_first(self):
-    with self.cached_session():
 
-      def keras_op(x, b):
-        return keras.backend.bias_add(x, b, data_format='channels_first')
+    def keras_op(x, b):
+      return keras.backend.bias_add(x, b, data_format='channels_first')
 
-      def np_op(x, b):
-        if x.ndim == 3:
-          b = b.reshape((1, b.shape[0], 1))
-        if x.ndim == 4:
-          b = b.reshape((1, b.shape[0], 1, 1))
-        return x + b
+    def np_op(x, b):
+      if x.ndim == 3:
+        b = b.reshape((1, b.shape[0], 1))
+      if x.ndim == 4:
+        b = b.reshape((1, b.shape[0], 1, 1))
+      return x + b
 
-      compare_two_inputs_op_to_numpy(keras_op, np_op,
-                                     input_shape_a=(4, 3, 7),
-                                     input_shape_b=(3,))
-      compare_two_inputs_op_to_numpy(keras_op, np_op,
-                                     input_shape_a=(4, 3, 5, 7),
-                                     input_shape_b=(3,))
+    compare_two_inputs_op_to_numpy(keras_op, np_op,
+                                   input_shape_a=(4, 3, 7),
+                                   input_shape_b=(3,))
+    compare_two_inputs_op_to_numpy(keras_op, np_op,
+                                   input_shape_a=(4, 3, 5, 7),
+                                   input_shape_b=(3,))
 
   def test_pool2d(self):
     val = np.random.random((10, 3, 10, 10))
@@ -837,9 +809,9 @@ class BackendNNOpsTest(test.TestCase, parameterized.TestCase):
                                              strides,
                                              output_shape,
                                              'channels_last')
-          with self.cached_session():
-            conv_cf = keras.backend.eval(conv_cf)
-            conv_cl = keras.backend.eval(conv_cl)
+
+          conv_cf = keras.backend.eval(conv_cf)
+          conv_cl = keras.backend.eval(conv_cl)
 
           self.assertAllCloseAccordingToType(
               conv_cf,
@@ -887,9 +859,8 @@ class BackendNNOpsTest(test.TestCase, parameterized.TestCase):
                                                   output_shape,
                                                   'channels_last')
 
-    with self.cached_session():
-      local_conv = keras.backend.eval(local_conv)
-      local_conv_dim = keras.backend.eval(local_conv_dim)
+    local_conv = keras.backend.eval(local_conv)
+    local_conv_dim = keras.backend.eval(local_conv_dim)
 
     self.assertAllCloseAccordingToType(local_conv, local_conv_dim)
 
@@ -1045,24 +1016,23 @@ class BackendNNOpsTest(test.TestCase, parameterized.TestCase):
         {'go_backwards': False, 'mask': mask},
         {'go_backwards': False, 'mask': mask, 'unroll': True},
     ]
-    with self.cached_session():
-      for i, kwargs in enumerate(kwargs_list):
-        last_output, outputs, new_states = keras.backend.rnn(rnn_fn, inputs,
-                                                             initial_states,
-                                                             **kwargs)
-        # check static shape inference
-        self.assertEqual(last_output.get_shape().as_list(),
+    for i, kwargs in enumerate(kwargs_list):
+      last_output, outputs, new_states = keras.backend.rnn(rnn_fn, inputs,
+                                                           initial_states,
+                                                           **kwargs)
+      # check static shape inference
+      self.assertEqual(last_output.get_shape().as_list(),
+                       [num_samples, output_dim])
+      self.assertEqual(outputs.get_shape().as_list(),
+                       [num_samples, timesteps, output_dim])
+      for state in new_states:
+        self.assertEqual(state.get_shape().as_list(),
                          [num_samples, output_dim])
-        self.assertEqual(outputs.get_shape().as_list(),
-                         [num_samples, timesteps, output_dim])
-        for state in new_states:
-          self.assertEqual(state.get_shape().as_list(),
-                           [num_samples, output_dim])
 
-        last_output_list[i].append(keras.backend.eval(last_output))
-        outputs_list[i].append(keras.backend.eval(outputs))
-        self.assertEqual(len(new_states), 1)
-        state_list[i].append(keras.backend.eval(new_states[0]))
+      last_output_list[i].append(keras.backend.eval(last_output))
+      outputs_list[i].append(keras.backend.eval(outputs))
+      self.assertLen(new_states, 1)
+      state_list[i].append(keras.backend.eval(new_states[0]))
 
       def assert_list_pairwise(z_list, atol=1e-05):
         for (z1, z2) in zip(z_list[1:], z_list[:-1]):
@@ -1144,29 +1114,28 @@ class BackendNNOpsTest(test.TestCase, parameterized.TestCase):
         {'go_backwards': False, 'mask': mask},
         {'go_backwards': False, 'mask': mask, 'unroll': True},
     ]
-    with self.cached_session():
-      for i, kwargs in enumerate(kwargs_list):
-        last_output, outputs, new_states = keras.backend.rnn(rnn_fn, inputs,
-                                                             initial_states,
-                                                             **kwargs)
-        # check static shape inference
-        self.assertEqual(last_output.get_shape().as_list(),
-                         [num_samples, output_dim])
-        self.assertEqual(outputs.get_shape().as_list(),
-                         [num_samples, timesteps, output_dim])
-        # for state in new_states:
-        #   self.assertEqual(state.get_shape().as_list(),
-        #                     [num_samples, output_dim])
-        self.assertEqual(new_states[0].get_shape().as_list(),
-                         [num_samples, output_dim])
-        self.assertEqual(new_states[1].get_shape().as_list(),
-                         [num_samples, 2 * output_dim])
-
-        last_output_list[i].append(keras.backend.eval(last_output))
-        outputs_list[i].append(keras.backend.eval(outputs))
-        self.assertEqual(len(new_states), 2)
-        state_list[i].append(keras.backend.eval(new_states[0]))
-        additional_state_list[i].append(keras.backend.eval(new_states[1]))
+    for i, kwargs in enumerate(kwargs_list):
+      last_output, outputs, new_states = keras.backend.rnn(rnn_fn, inputs,
+                                                           initial_states,
+                                                           **kwargs)
+      # check static shape inference
+      self.assertEqual(last_output.get_shape().as_list(),
+                       [num_samples, output_dim])
+      self.assertEqual(outputs.get_shape().as_list(),
+                       [num_samples, timesteps, output_dim])
+      # for state in new_states:
+      #   self.assertEqual(state.get_shape().as_list(),
+      #                     [num_samples, output_dim])
+      self.assertEqual(new_states[0].get_shape().as_list(),
+                       [num_samples, output_dim])
+      self.assertEqual(new_states[1].get_shape().as_list(),
+                       [num_samples, 2 * output_dim])
+
+      last_output_list[i].append(keras.backend.eval(last_output))
+      outputs_list[i].append(keras.backend.eval(outputs))
+      self.assertLen(new_states, 2)
+      state_list[i].append(keras.backend.eval(new_states[0]))
+      additional_state_list[i].append(keras.backend.eval(new_states[1]))
 
       def assert_list_pairwise(z_list, atol=1e-05):
         for (z1, z2) in zip(z_list[1:], z_list[:-1]):
@@ -1356,53 +1325,52 @@ class BackendNNOpsTest(test.TestCase, parameterized.TestCase):
 class TestCTC(test.TestCase):
 
   def test_ctc_decode(self):
-    with self.cached_session():
-      depth = 6
-      seq_len_0 = 5
-      input_prob_matrix_0 = np.asarray(
-          [[0.30999, 0.309938, 0.0679938, 0.0673362, 0.0708352, 0.173908],
-           [0.215136, 0.439699, 0.0370931, 0.0393967, 0.0381581, 0.230517],
-           [0.199959, 0.489485, 0.0233221, 0.0251417, 0.0233289, 0.238763],
-           [0.279611, 0.452966, 0.0204795, 0.0209126, 0.0194803, 0.20655],
-           [0.51286, 0.288951, 0.0243026, 0.0220788, 0.0219297, 0.129878],
-           # Random entry added in at time=5
-           [0.155251, 0.164444, 0.173517, 0.176138, 0.169979, 0.160671]],
-          dtype=np.float32)
-
-      # len max_time_steps array of batch_size x depth matrices
-      inputs = ([input_prob_matrix_0[t, :][np.newaxis, :]
-                 for t in range(seq_len_0)] +  # Pad to max_time_steps = 8
-                2 * [np.zeros((1, depth), dtype=np.float32)])
-
-      inputs = keras.backend.variable(np.asarray(inputs).transpose((1, 0, 2)))
-
-      # batch_size length vector of sequence_lengths
-      input_length = keras.backend.variable(
-          np.array([seq_len_0], dtype=np.int32))
-      # batch_size length vector of negative log probabilities
-      log_prob_truth = np.array([
-          -3.5821197,  # output beam 0
-          -3.777835    # output beam 1
-      ], np.float32)[np.newaxis, :]
-
-      decode_truth = [np.array([1, 0]), np.array([0, 1, 0])]
-      beam_width = 2
-      top_paths = 2
-
-      decode_pred_tf, log_prob_pred_tf = keras.backend.ctc_decode(
-          inputs,
-          input_length,
-          greedy=False,
-          beam_width=beam_width,
-          top_paths=top_paths)
-
-      self.assertEqual(len(decode_pred_tf), top_paths)
-      log_prob_pred = keras.backend.eval(log_prob_pred_tf)
-      for i in range(top_paths):
-        self.assertTrue(
-            np.alltrue(
-                decode_truth[i] == keras.backend.eval(decode_pred_tf[i])))
-      self.assertAllClose(log_prob_truth, log_prob_pred)
+    depth = 6
+    seq_len_0 = 5
+    input_prob_matrix_0 = np.asarray(
+        [[0.30999, 0.309938, 0.0679938, 0.0673362, 0.0708352, 0.173908],
+         [0.215136, 0.439699, 0.0370931, 0.0393967, 0.0381581, 0.230517],
+         [0.199959, 0.489485, 0.0233221, 0.0251417, 0.0233289, 0.238763],
+         [0.279611, 0.452966, 0.0204795, 0.0209126, 0.0194803, 0.20655],
+         [0.51286, 0.288951, 0.0243026, 0.0220788, 0.0219297, 0.129878],
+         # Random entry added in at time=5
+         [0.155251, 0.164444, 0.173517, 0.176138, 0.169979, 0.160671]],
+        dtype=np.float32)
+
+    # len max_time_steps array of batch_size x depth matrices
+    inputs = ([input_prob_matrix_0[t, :][np.newaxis, :]
+               for t in range(seq_len_0)] +  # Pad to max_time_steps = 8
+              2 * [np.zeros((1, depth), dtype=np.float32)])
+
+    inputs = keras.backend.variable(np.asarray(inputs).transpose((1, 0, 2)))
+
+    # batch_size length vector of sequence_lengths
+    input_length = keras.backend.variable(
+        np.array([seq_len_0], dtype=np.int32))
+    # batch_size length vector of negative log probabilities
+    log_prob_truth = np.array([
+        -3.5821197,  # output beam 0
+        -3.777835    # output beam 1
+    ], np.float32)[np.newaxis, :]
+
+    decode_truth = [np.array([1, 0]), np.array([0, 1, 0])]
+    beam_width = 2
+    top_paths = 2
+
+    decode_pred_tf, log_prob_pred_tf = keras.backend.ctc_decode(
+        inputs,
+        input_length,
+        greedy=False,
+        beam_width=beam_width,
+        top_paths=top_paths)
+
+    self.assertEqual(len(decode_pred_tf), top_paths)
+    log_prob_pred = keras.backend.eval(log_prob_pred_tf)
+    for i in range(top_paths):
+      self.assertTrue(
+          np.alltrue(
+              decode_truth[i] == keras.backend.eval(decode_pred_tf[i])))
+    self.assertAllClose(log_prob_truth, log_prob_pred)
 
   @test_util.run_v1_only('b/120545219')
   def test_ctc_batch_cost(self):
@@ -1463,29 +1431,26 @@ class TestCTC(test.TestCase):
 class TestRandomOps(test.TestCase):
 
   def test_random_binomial(self):
-    with self.cached_session():
-      np.random.seed(123)
-      x = keras.backend.random_binomial((1000, 1000), p=0.5)
-      self.assertAllClose(np.mean(keras.backend.eval(x)), 0.5, atol=0.1)
+    np.random.seed(123)
+    x = keras.backend.random_binomial((1000, 1000), p=0.5)
+    self.assertAllClose(np.mean(keras.backend.eval(x)), 0.5, atol=0.1)
 
   def test_truncated_normal(self):
-    with self.cached_session():
-      np.random.seed(123)
-      x = keras.backend.truncated_normal((1000, 1000), mean=0.0, stddev=1.0)
-      y = keras.backend.eval(x)
-      self.assertAllClose(np.mean(y), 0., atol=0.1)
-      self.assertAllClose(np.std(y), 0.88, atol=0.1)
-      self.assertAllClose(np.max(y), 2., atol=0.1)
-      self.assertAllClose(np.min(y), -2., atol=0.1)
+    np.random.seed(123)
+    x = keras.backend.truncated_normal((1000, 1000), mean=0.0, stddev=1.0)
+    y = keras.backend.eval(x)
+    self.assertAllClose(np.mean(y), 0., atol=0.1)
+    self.assertAllClose(np.std(y), 0.88, atol=0.1)
+    self.assertAllClose(np.max(y), 2., atol=0.1)
+    self.assertAllClose(np.min(y), -2., atol=0.1)
 
   def test_string_input(self):
-    with self.cached_session():
-      seq = keras.Sequential([
-          keras.layers.InputLayer(input_shape=(1,), dtype=dtypes.string),
-          keras.layers.Lambda(lambda x: x[0])
-      ])
-      preds = seq.predict([['tensorflow eager']])
-      self.assertEqual(preds.shape, (1,))
+    seq = keras.Sequential([
+        keras.layers.InputLayer(input_shape=(1,), dtype=dtypes.string),
+        keras.layers.Lambda(lambda x: x[0])
+    ])
+    preds = seq.predict([['tensorflow eager']])
+    self.assertEqual(preds.shape, (1,))
 
 
 class BackendGraphTests(test.TestCase):
-- 
GitLab


From 84edc28bcf8cb73f17acd1d267c17d76c417ccfe Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 18 Jan 2019 14:39:56 -0800
Subject: [PATCH 1009/1095] Properly handle errors while parsing op data.

PiperOrigin-RevId: 229996694
---
 tensorflow/lite/core/api/BUILD                |  1 +
 .../lite/core/api/flatbuffer_conversions.cc   | 25 +++--
 .../core/api/flatbuffer_conversions_test.cc   | 99 ++++++++++++-------
 tensorflow/lite/kernels/reshape_test.cc       | 10 --
 4 files changed, 79 insertions(+), 56 deletions(-)

diff --git a/tensorflow/lite/core/api/BUILD b/tensorflow/lite/core/api/BUILD
index 6a43b0322d..db6b4a2d18 100644
--- a/tensorflow/lite/core/api/BUILD
+++ b/tensorflow/lite/core/api/BUILD
@@ -51,6 +51,7 @@ cc_test(
     srcs = ["flatbuffer_conversions_test.cc"],
     deps = [
         ":api",
+        "//tensorflow/lite:string",
         "//tensorflow/lite/c:c_api_internal",
         "@com_google_googletest//:gtest",
     ],
diff --git a/tensorflow/lite/core/api/flatbuffer_conversions.cc b/tensorflow/lite/core/api/flatbuffer_conversions.cc
index 7f27f1f503..bcf7680290 100644
--- a/tensorflow/lite/core/api/flatbuffer_conversions.cc
+++ b/tensorflow/lite/core/api/flatbuffer_conversions.cc
@@ -28,22 +28,27 @@ namespace {
 // Copies the contents from the flatbuffer int vector `flatbuffer` into the
 // int array `buffer`. `flat_vector` and `buffer` represent the same
 // configuration operation for a given operation.
-void FlatBufferIntVectorToArray(int max_size_of_buffer,
-                                const flatbuffers::Vector<int32_t>* flat_vector,
-                                int* buffer, ErrorReporter* error_reporter) {
+TfLiteStatus FlatBufferIntVectorToArray(
+    int max_size_of_buffer, const flatbuffers::Vector<int32_t>* flat_vector,
+    int* buffer, ErrorReporter* error_reporter, const char* op_name) {
   if (!flat_vector) {
-    error_reporter->Report("Input array not provided for operation.\n");
+    error_reporter->Report("Input array not provided for operation '%s'.\n",
+                           op_name);
+    return kTfLiteError;
   } else {
     int num_dimensions = flat_vector->Length();
     if (num_dimensions > max_size_of_buffer / sizeof(int)) {
       error_reporter->Report(
-          "Found too many dimensions in the operation's input array.\n");
+          "Found too many dimensions in the input array of operation '%s'.\n",
+          op_name);
+      return kTfLiteError;
     } else {
       for (int i = 0; i < num_dimensions; ++i) {
         buffer[i] = flat_vector->Get(i);
       }
     }
   }
+  return kTfLiteOk;
 }
 
 }  // namespace
@@ -452,8 +457,9 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type,
       auto* params = allocator->AllocatePOD<TfLiteReshapeParams>();
       if (auto* schema_params = op->builtin_options_as_ReshapeOptions()) {
         auto* new_shape = schema_params->new_shape();
-        FlatBufferIntVectorToArray(sizeof(params->shape), new_shape,
-                                   params->shape, error_reporter);
+        TF_LITE_ENSURE_STATUS(FlatBufferIntVectorToArray(
+            sizeof(params->shape), new_shape, params->shape, error_reporter,
+            "reshape"));
         params->num_dimensions = new_shape->Length();
       }
       *builtin_data = reinterpret_cast<void*>(params);
@@ -521,8 +527,9 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type,
       auto* params = allocator->AllocatePOD<TfLiteSqueezeParams>();
       if (auto* schema_params = op->builtin_options_as_SqueezeOptions()) {
         const auto& squeeze_dims = schema_params->squeeze_dims();
-        FlatBufferIntVectorToArray(sizeof(params->squeeze_dims), squeeze_dims,
-                                   params->squeeze_dims, error_reporter);
+        TF_LITE_ENSURE_STATUS(FlatBufferIntVectorToArray(
+            sizeof(params->squeeze_dims), squeeze_dims, params->squeeze_dims,
+            error_reporter, "squeeze"));
         params->num_squeeze_dims = squeeze_dims->Length();
       }
       *builtin_data = reinterpret_cast<void*>(params);
diff --git a/tensorflow/lite/core/api/flatbuffer_conversions_test.cc b/tensorflow/lite/core/api/flatbuffer_conversions_test.cc
index 4d1d1b21fd..4a5de48302 100644
--- a/tensorflow/lite/core/api/flatbuffer_conversions_test.cc
+++ b/tensorflow/lite/core/api/flatbuffer_conversions_test.cc
@@ -17,8 +17,10 @@ limitations under the License.
 
 #include <cstring>
 
+#include <gmock/gmock.h>
 #include <gtest/gtest.h>
 #include "tensorflow/lite/c/builtin_op_data.h"
+#include "tensorflow/lite/string.h"
 
 namespace tflite {
 namespace {
@@ -33,6 +35,8 @@ class MockErrorReporter : public ErrorReporter {
   char* GetBuffer() { return buffer_; }
   int GetBufferSize() { return buffer_size_; }
 
+  string GetAsString() const { return string(buffer_, buffer_size_); }
+
  private:
   static constexpr int kBufferSize = 256;
   char buffer_[kBufferSize];
@@ -60,25 +64,56 @@ class MockDataAllocator : public BuiltinDataAllocator {
 
 }  // namespace
 
-TEST(FlatbufferConversions, TestParseOpDataConv) {
-  MockErrorReporter mock_reporter;
-  ErrorReporter* reporter = &mock_reporter;
-  MockDataAllocator mock_allocator;
-
-  flatbuffers::FlatBufferBuilder builder;
-  flatbuffers::Offset<void> conv_options =
-      CreateConv2DOptions(builder, Padding_SAME, 1, 2,
-                          ActivationFunctionType_RELU, 3, 4)
-          .Union();
-  flatbuffers::Offset<Operator> conv_offset = CreateOperatorDirect(
-      builder, 0, nullptr, nullptr, BuiltinOptions_Conv2DOptions, conv_options,
-      nullptr, CustomOptionsFormat_FLEXBUFFERS, nullptr);
-  builder.Finish(conv_offset);
-  void* conv_pointer = builder.GetBufferPointer();
-  const Operator* conv_op = flatbuffers::GetRoot<Operator>(conv_pointer);
+class FlatbufferConversionsTest : public ::testing::Test {
+ public:
+  const Operator* BuildTestOperator(BuiltinOptions op_type,
+                                    flatbuffers::Offset<void> options) {
+    flatbuffers::Offset<Operator> offset =
+        CreateOperatorDirect(builder_, 0, nullptr, nullptr, op_type, options,
+                             nullptr, CustomOptionsFormat_FLEXBUFFERS, nullptr);
+    builder_.Finish(offset);
+    void* pointer = builder_.GetBufferPointer();
+    return flatbuffers::GetRoot<Operator>(pointer);
+  }
+
+ protected:
+  MockErrorReporter mock_reporter_;
+  MockDataAllocator mock_allocator_;
+  flatbuffers::FlatBufferBuilder builder_;
+};
+
+TEST_F(FlatbufferConversionsTest, ParseBadSqueeze) {
+  const Operator* op = BuildTestOperator(
+      BuiltinOptions_SqueezeOptions, CreateSqueezeOptions(builder_).Union());
+  void* output_data = nullptr;
+  EXPECT_NE(kTfLiteOk, ParseOpData(op, BuiltinOperator_SQUEEZE, &mock_reporter_,
+                                   &mock_allocator_, &output_data));
+  EXPECT_THAT(mock_reporter_.GetAsString(),
+              ::testing::ContainsRegex(
+                  "Input array not provided for operation 'squeeze'"));
+}
+
+TEST_F(FlatbufferConversionsTest, ParseBadReshape) {
+  const Operator* op = BuildTestOperator(
+      BuiltinOptions_ReshapeOptions, CreateSqueezeOptions(builder_).Union());
+  void* output_data = nullptr;
+  EXPECT_NE(kTfLiteOk, ParseOpData(op, BuiltinOperator_RESHAPE, &mock_reporter_,
+                                   &mock_allocator_, &output_data));
+  EXPECT_THAT(mock_reporter_.GetAsString(),
+              ::testing::ContainsRegex(
+                  "Input array not provided for operation 'reshape'"));
+}
+
+TEST_F(FlatbufferConversionsTest, TestParseOpDataConv) {
+  const Operator* conv_op =
+      BuildTestOperator(BuiltinOptions_Conv2DOptions,
+                        CreateConv2DOptions(builder_, Padding_SAME, 1, 2,
+                                            ActivationFunctionType_RELU, 3, 4)
+                            .Union());
   void* output_data = nullptr;
-  EXPECT_EQ(kTfLiteOk, ParseOpData(conv_op, BuiltinOperator_CONV_2D, reporter,
-                                   &mock_allocator, &output_data));
+  EXPECT_EQ(kTfLiteOk,
+            ParseOpData(conv_op, BuiltinOperator_CONV_2D, &mock_reporter_,
+                        &mock_allocator_, &output_data));
   EXPECT_NE(nullptr, output_data);
   TfLiteConvParams* params = reinterpret_cast<TfLiteConvParams*>(output_data);
   EXPECT_EQ(kTfLitePaddingSame, params->padding);
@@ -89,30 +124,20 @@ TEST(FlatbufferConversions, TestParseOpDataConv) {
   EXPECT_EQ(4, params->dilation_height_factor);
 }
 
-TEST(FlatbufferConversions, TestParseOpDataCustom) {
-  MockErrorReporter mock_reporter;
-  ErrorReporter* reporter = &mock_reporter;
-  MockDataAllocator mock_allocator;
-
-  flatbuffers::FlatBufferBuilder builder;
-  flatbuffers::Offset<void> null_options;
-  flatbuffers::Offset<Operator> custom_offset = CreateOperatorDirect(
-      builder, 0, nullptr, nullptr, BuiltinOptions_NONE, null_options, nullptr,
-      CustomOptionsFormat_FLEXBUFFERS, nullptr);
-  builder.Finish(custom_offset);
-  void* custom_pointer = builder.GetBufferPointer();
-  const Operator* custom_op = flatbuffers::GetRoot<Operator>(custom_pointer);
+TEST_F(FlatbufferConversionsTest, TestParseOpDataCustom) {
+  const Operator* custom_op =
+      BuildTestOperator(BuiltinOptions_NONE, flatbuffers::Offset<void>());
   void* output_data = nullptr;
-  EXPECT_EQ(kTfLiteOk, ParseOpData(custom_op, BuiltinOperator_CUSTOM, reporter,
-                                   &mock_allocator, &output_data));
+  EXPECT_EQ(kTfLiteOk,
+            ParseOpData(custom_op, BuiltinOperator_CUSTOM, &mock_reporter_,
+                        &mock_allocator_, &output_data));
   EXPECT_EQ(nullptr, output_data);
 }
 
-TEST(FlatbufferConversions, TestConvertTensorType) {
-  MockErrorReporter mock_reporter;
-  ErrorReporter* reporter = &mock_reporter;
+TEST_F(FlatbufferConversionsTest, TestConvertTensorType) {
   TfLiteType type;
-  EXPECT_EQ(kTfLiteOk, ConvertTensorType(TensorType_FLOAT32, &type, reporter));
+  EXPECT_EQ(kTfLiteOk,
+            ConvertTensorType(TensorType_FLOAT32, &type, &mock_reporter_));
   EXPECT_EQ(kTfLiteFloat32, type);
 }
 
diff --git a/tensorflow/lite/kernels/reshape_test.cc b/tensorflow/lite/kernels/reshape_test.cc
index e2210aeaf0..e9d12a9def 100644
--- a/tensorflow/lite/kernels/reshape_test.cc
+++ b/tensorflow/lite/kernels/reshape_test.cc
@@ -137,21 +137,11 @@ TEST_P(ReshapeOpTest, MismatchedDimensions) {
 #endif
 
 TEST_P(ReshapeOpTest, TooManyDimensions) {
-  if (GetParam() == kAsReshapeOption) {
 #ifdef GTEST_HAS_DEATH_TEST
     EXPECT_DEATH(ReshapeOpModel<float>({1, 1, 2, 1, 1, 1, 1, 1, 1}, {9},
                                        {1, 1, 1, 1, 1, 1, 1, 1, 2}, GetParam()),
                  "Found too many dimensions");
 #endif
-  } else {
-    ReshapeOpModel<float> m({1, 1, 2, 1, 1, 1, 1, 1, 1}, {9},
-                            {1, 1, 1, 1, 1, 1, 1, 1, 2}, GetParam());
-    m.SetInput({3, 4});
-    m.Invoke();
-    EXPECT_THAT(m.GetOutput(), ElementsAreArray({3, 4}));
-    EXPECT_THAT(m.GetOutputShape(),
-                ElementsAreArray({1, 1, 1, 1, 1, 1, 1, 1, 2}));
-  }
 }
 
 #ifdef GTEST_HAS_DEATH_TEST
-- 
GitLab


From fe7fb35c9cafc37e731074c7ca5b9f7e04649c3c Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Fri, 18 Jan 2019 14:45:01 -0800
Subject: [PATCH 1010/1095] Best-effort lift initializers out of tf.functions
 to avoid racy initialization.

PiperOrigin-RevId: 229997649
---
 tensorflow/python/eager/def_function.py | 58 ++++++++++++++++++++++++-
 1 file changed, 56 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/eager/def_function.py b/tensorflow/python/eager/def_function.py
index e5950f5454..2e8b82c68d 100644
--- a/tensorflow/python/eager/def_function.py
+++ b/tensorflow/python/eager/def_function.py
@@ -25,6 +25,7 @@ import weakref
 from tensorflow.python.eager import context
 from tensorflow.python.eager import function as function_lib
 from tensorflow.python.eager import lift_to_graph
+from tensorflow.python.eager import tape
 from tensorflow.python.framework import func_graph as func_graph_module
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import control_flow_ops
@@ -56,6 +57,9 @@ class UnliftedInitializerVariable(resource_variable_ops.ResourceVariable):
                dtype=None,
                constraint=None,
                add_initializers_to=None,
+               lifted_initializer_graph=None,
+               lifted_all_initializers=None,
+               lifted_placeholders=None,
                **unused_kwargs):
     """Creates a variable.
 
@@ -89,6 +93,11 @@ class UnliftedInitializerVariable(resource_variable_ops.ResourceVariable):
       add_initializers_to: if not None and not in legacy graph mode, the
         initializer tensor will be added to this map instead of adding the
         assignment to the function.
+      lifted_initializer_graph: FuncGraph to try to lift initializers to.
+      lifted_all_initializers: list with one boolean element, which will be
+        set to False if we cannot lift this initializer to the above graph.
+      lifted_placeholders: placeholders for resource handles lifted out of
+        this graph.
 
     Raises:
       ValueError: If the initial value is not specified, or does not have a
@@ -167,6 +176,7 @@ class UnliftedInitializerVariable(resource_variable_ops.ResourceVariable):
             with ops.name_scope("Assign") as n, ops.colocate_with(self._handle):
               self._initializer_op = resource_variable_ops.assign_variable_op(
                   self._handle, lifted_initializer, name=n)
+              assign = self._initializer_op
           with ops.name_scope("Read"), ops.colocate_with(self._handle):
             # Manually assign reads to the handle's device to avoid log
             # messages.
@@ -177,6 +187,7 @@ class UnliftedInitializerVariable(resource_variable_ops.ResourceVariable):
       else:
         if add_initializers_to is not None:
           add_initializers_to[self] = initial_value
+          assign = None
         else:
           def assign_fn():
             with ops.name_scope("Assign") as n, ops.colocate_with(self._handle):
@@ -190,9 +201,18 @@ class UnliftedInitializerVariable(resource_variable_ops.ResourceVariable):
             return ops.convert_to_tensor(0)
           # Note: this cond is always guaranteed to run because we're inside a
           # defun which will insert automatic control dependencies.
-          control_flow_ops.cond(
+          assign = control_flow_ops.cond(
               resource_variable_ops.var_is_initialized_op(self._handle),
               not_assign_fn, assign_fn)
+      if lifted_initializer_graph is not None and assign is not None:
+        try:
+          handle_placeholder = ops.convert_to_tensor(self._handle)
+          op_map = lift_to_graph.lift_to_graph(
+              assign, lifted_initializer_graph,
+              sources=[handle_placeholder])
+          lifted_placeholders.append((self._handle, op_map[handle_placeholder]))
+        except ValueError:
+          lifted_all_initializers[0] = False
 
     # After the handle has been created, set up a way to clean it up when
     # executing eagerly. We'll hold the only reference to the deleter, so that
@@ -205,6 +225,19 @@ class UnliftedInitializerVariable(resource_variable_ops.ResourceVariable):
     self._cached_shape_as_list = None
 
 
+class FunctionDeleter(object):
+
+  def __init__(self, func_graph):
+    self.func_graph = func_graph
+
+  def __del__(self):
+    try:
+      func_graph_module.dismantle_func_graph(self.func_graph)
+    except:  # pylint: disable=bare-except
+      # Note: bare except here because this can be noisy at shutdown time.
+      pass
+
+
 class Function(object):
   """Wrapper class for the graph functions defined for a Python function.
 
@@ -296,11 +329,17 @@ class Function(object):
     """
 
     created_variables = []
+    lifted_initializer_graph = func_graph_module.FuncGraph("initializer")
+    lifted_all_initializers = [True]
+    lifted_placeholders = []
 
     def variable_capturing_scope(unused_next_creator, **kwds):
       """Creates UnliftedInitializerVariables and saves references to them."""
       v = UnliftedInitializerVariable(
-          add_initializers_to=add_initializers_to, **kwds)
+          add_initializers_to=add_initializers_to,
+          lifted_initializer_graph=lifted_initializer_graph,
+          lifted_all_initializers=lifted_all_initializers,
+          lifted_placeholders=lifted_placeholders, **kwds)
       created_variables.append(weakref.ref(v))
       return v
 
@@ -308,9 +347,13 @@ class Function(object):
     self._stateful_fn = self._defun_with_scope(variable_capturing_scope)
     self._stateful_fn._name = self._name  # pylint: disable=protected-access
     # Force the definition of the function for these arguments
+    self._lifted_initializer_graph = lifted_initializer_graph
+    self._graph_deleter = FunctionDeleter(self._lifted_initializer_graph)
+    self._lifted_placeholders = lifted_placeholders
     self._concrete_stateful_fn = (
         self._stateful_fn._get_concrete_function_internal_garbage_collected(  # pylint: disable=protected-access
             *args, **kwds))
+    self._lifted_all_initializers = lifted_all_initializers[0]
 
     def invalid_creator_scope(*unused_args, **unused_kwds):
       """Disables variable creation."""
@@ -338,6 +381,17 @@ class Function(object):
 
     # This is the first call of __call__, so we have to initialize.
     self._initialize(args, kwds)
+    if self._lifted_all_initializers and self._lifted_placeholders:
+      with ops.init_scope():
+        handles, placeholders = zip(*self._lifted_placeholders)
+        if context.executing_eagerly():
+          lifted_fn = function_lib._EagerDefinedFunction(  # pylint: disable=protected-access
+              "initializer" + str(ops.uid()),
+              self._lifted_initializer_graph,
+              placeholders, [], {})
+          with tape.stop_recording():
+            lifted_fn.call(context.context(), list(handles))
+      return self._stateless_fn(*args, **kwds)
     canon_args, canon_kwds = self._canonicalize_function_inputs(args, kwds)
 
     if not self._created_variables:
-- 
GitLab


From a8e0f21a3770b5fb16330cf8617e0c2bb1558300 Mon Sep 17 00:00:00 2001
From: Mingsheng Hong <hongm@google.com>
Date: Fri, 18 Jan 2019 14:48:30 -0800
Subject: [PATCH 1011/1095] Added various debugging support to C++ eager layer:
 tensor handle and eager operation. Added experimental C APIs for debug
 prints.

Added some VLOG statements for to help memory/pointer debugging.

PiperOrigin-RevId: 229998268
---
 tensorflow/c/c_api_experimental.cc            | 13 ++++++++++++
 tensorflow/c/c_api_experimental.h             | 12 +++++++++++
 tensorflow/c/c_api_experimental_test.cc       | 20 +++++++++++++++++++
 tensorflow/c/eager/c_api.cc                   |  1 +
 .../common_runtime/eager/eager_operation.cc   | 19 ++++++++++++++++++
 .../common_runtime/eager/eager_operation.h    |  2 ++
 .../common_runtime/eager/tensor_handle.cc     | 11 ++++++++++
 .../core/common_runtime/eager/tensor_handle.h |  3 +++
 8 files changed, 81 insertions(+)

diff --git a/tensorflow/c/c_api_experimental.cc b/tensorflow/c/c_api_experimental.cc
index f04b285037..92f7da7de0 100644
--- a/tensorflow/c/c_api_experimental.cc
+++ b/tensorflow/c/c_api_experimental.cc
@@ -128,6 +128,14 @@ const char* TF_GraphDebugString(TF_Graph* graph, size_t* len) {
   return ret;
 }
 
+char* TF_FunctionDebugString(TF_Function* func, size_t* len) {
+  const auto& debug_str = func->fdef.DebugString();
+  *len = debug_str.size();
+  char* ret = static_cast<char*>(malloc(*len + 1));
+  memcpy(ret, debug_str.c_str(), *len + 1);
+  return ret;
+}
+
 // On success, returns a set of TF_Function instances from `text_proto` of
 // GraphDef type. These functions must be deleted by calling TF_DeleteFunction.
 //
@@ -8748,6 +8756,11 @@ void TFE_TensorHandlePrintDebugString(TFE_TensorHandle* handle) {
   TF_DeleteStatus(status);
 }
 
+void TFE_OpPrintDebugString(TFE_Op* op) {
+  VLOG(1) << "TFE_OpPrintDebugString() over " << op;
+  LOG(INFO) << op->operation.DebugString();
+}
+
 struct TFE_ExecuteOpNotification {
   TFE_ExecuteOpNotification() : status(TF_NewStatus(), TF_DeleteStatus) {}
   tensorflow::Notification n;
diff --git a/tensorflow/c/c_api_experimental.h b/tensorflow/c/c_api_experimental.h
index e6d04d0c2b..e8da4099b8 100644
--- a/tensorflow/c/c_api_experimental.h
+++ b/tensorflow/c/c_api_experimental.h
@@ -84,6 +84,15 @@ TF_CAPI_EXPORT extern TF_Buffer* TF_CreateRunOptions(
 TF_CAPI_EXPORT extern const char* TF_GraphDebugString(TF_Graph* graph,
                                                       size_t* len);
 
+// Returns the function content in a human-readable format, with length set in
+// `len`. The format is subject to change in the future.
+// The returned string is heap-allocated, and caller should call free() on it.
+//
+// Do not return const char*, because some foreign language binding
+// (e.g. swift) cannot then call free() on the returned pointer.
+TF_CAPI_EXPORT extern char* TF_FunctionDebugString(TF_Function* func,
+                                                   size_t* len);
+
 // Creates a stack of data set + iterator nodes, currently hard-coded to return
 // a sequence of 3 float values <42.0, 43.0, 44.0> over 3 calls. On success,
 // returns the IteratorGetNext node, which caller can run or feed into an node.
@@ -181,6 +190,8 @@ TF_CAPI_EXPORT extern TFE_TensorHandle* TFE_DequeueVariantTensor(
 TF_CAPI_EXPORT extern void TFE_TensorHandlePrintDebugString(
     TFE_TensorHandle* handle);
 
+TF_CAPI_EXPORT extern void TFE_OpPrintDebugString(TFE_Op* op);
+
 typedef struct TFE_ExecuteOpNotification TFE_ExecuteOpNotification;
 
 // Allows invoking a kernel asynchronously, and explicitly returns a
@@ -255,6 +266,7 @@ TF_CAPI_EXPORT extern void TFE_EnableCollectiveOps(TFE_Context* ctx,
                                                    const void* proto,
                                                    size_t proto_len,
                                                    TF_Status* status);
+
 #ifdef __cplusplus
 } /* end extern "C" */
 #endif
diff --git a/tensorflow/c/c_api_experimental_test.cc b/tensorflow/c/c_api_experimental_test.cc
index daa7701b7f..65a4e10b26 100644
--- a/tensorflow/c/c_api_experimental_test.cc
+++ b/tensorflow/c/c_api_experimental_test.cc
@@ -296,5 +296,25 @@ TEST(CAPI_EXPERIMENTAL, TFE_ExecuteOpInNewThreadTest_Blocking) {
   TF_DeleteStatus(status);
 }
 
+TEST(CAPI_EXPERIMENTAL, DebugPrint) {
+  TF_Status* status = TF_NewStatus();
+  TFE_ContextOptions* opts = TFE_NewContextOptions();
+  CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
+  TFE_Context* ctx = TFE_NewContext(opts, status);
+  CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
+  TFE_DeleteContextOptions(opts);
+
+  TFE_TensorHandle* m = TestMatrixTensorHandle();
+  TFE_Op* op = MatMulOp(ctx, m, m);
+
+  CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
+  TFE_OpPrintDebugString(op);
+
+  TFE_DeleteTensorHandle(m);
+  TFE_DeleteOp(op);
+  TFE_DeleteContext(ctx);
+  TF_DeleteStatus(status);
+}
+
 }  // namespace
 }  // namespace tensorflow
diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc
index e2574d5311..a92ba1f0f3 100755
--- a/tensorflow/c/eager/c_api.cc
+++ b/tensorflow/c/eager/c_api.cc
@@ -712,6 +712,7 @@ void TFE_OpSetAttrFunctionList(TFE_Op* op, const char* attr_name,
 
 void TFE_Execute(TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals,
                  TF_Status* status) {
+  VLOG(1) << "Calling TFE_Execute() on op " << op;
   tensorflow::gtl::InlinedVector<tensorflow::TensorHandle*, 2> handle_retvals(
       *num_retvals);
   status->status =
diff --git a/tensorflow/core/common_runtime/eager/eager_operation.cc b/tensorflow/core/common_runtime/eager/eager_operation.cc
index 0718e68932..b10320ca30 100644
--- a/tensorflow/core/common_runtime/eager/eager_operation.cc
+++ b/tensorflow/core/common_runtime/eager/eager_operation.cc
@@ -35,4 +35,23 @@ void EagerOperation::ConsumeInput(tensorflow::TensorHandle* h) {
   inputs_.push_back(h);
   attrs_.NumInputs(static_cast<int>(inputs_.size()));
 }
+
+string EagerOperation::DebugString() const {
+  string out;
+  VLOG(1) << "EagerOperation::DebugString() over " << this;
+
+  strings::StrAppend(&out, "Name: ", name_, "\n");
+  strings::StrAppend(
+      &out, "Device: ", Device() ? Device()->DebugString() : "[]", "\n");
+  for (const auto& input : inputs_) {
+    VLOG(1) << "Input ptr: " << input;
+    strings::StrAppend(&out, "Input: ", input->DebugString(), "\n");
+  }
+
+  NodeDef ndef;
+  Attrs().FillAttrValueMap(ndef.mutable_attr());
+  strings::StrAppend(&out, "Attrs: ", ndef.DebugString(), "\n");
+  return out;
+}
+
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/eager/eager_operation.h b/tensorflow/core/common_runtime/eager/eager_operation.h
index 5a9e1f0292..23a2d1bf98 100644
--- a/tensorflow/core/common_runtime/eager/eager_operation.h
+++ b/tensorflow/core/common_runtime/eager/eager_operation.h
@@ -64,6 +64,8 @@ class EagerOperation {
 
   void SetUseXla(bool use_xla) { use_xla_ = use_xla; }
 
+  string DebugString() const;
+
  private:
   tensorflow::EagerContext* ctx_;  // Must outlive the EagerOperation.
   const tensorflow::string name_;
diff --git a/tensorflow/core/common_runtime/eager/tensor_handle.cc b/tensorflow/core/common_runtime/eager/tensor_handle.cc
index eaca6d7b32..5e5936f1cf 100644
--- a/tensorflow/core/common_runtime/eager/tensor_handle.cc
+++ b/tensorflow/core/common_runtime/eager/tensor_handle.cc
@@ -312,4 +312,15 @@ Device* GetResourceDevice(const Tensor& t, EagerContext* ctx) {
   return it->second;
 }
 
+string TensorHandle::DebugString() const {
+  VLOG(1) << "Calling TensorHandle::DebugString() on " << this;
+
+  string out;
+  strings::StrAppend(&out, "Device: ", device_ ? device_->DebugString() : "[]");
+  // Consider supporting non-CPU tensors (when device_ is non-NULL) if needed.
+  strings::StrAppend(&out, ", Tensor: ", device_ ? "?" : tensor_.DebugString(),
+                     "\n");
+  return out;
+}
+
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/eager/tensor_handle.h b/tensorflow/core/common_runtime/eager/tensor_handle.h
index a4727d6f76..0ecd9e6ddc 100644
--- a/tensorflow/core/common_runtime/eager/tensor_handle.h
+++ b/tensorflow/core/common_runtime/eager/tensor_handle.h
@@ -59,6 +59,7 @@ class TensorHandle : public core::RefCounted {
                Device* op_device, Device* resource_device, EagerContext* ctx);
 
   ~TensorHandle() override {
+    VLOG(1) << "Deleting TensorHandle " << this;
     if (call_on_destroy_) {
       call_on_destroy_();
     }
@@ -114,6 +115,8 @@ class TensorHandle : public core::RefCounted {
 
   bool IsRemote();
 
+  string DebugString() const;
+
  private:
   // If the contents of the Tensor pointed to by this handle is yet to be
   // computed by a EagerNode, this function will block till that compuatation is
-- 
GitLab


From d319d8d716b58d71ee0794025a7679e47ed79622 Mon Sep 17 00:00:00 2001
From: Jacques Pienaar <jpienaar@google.com>
Date: Fri, 18 Jan 2019 15:11:35 -0800
Subject: [PATCH 1012/1095] Differentiate calls to colocate_with from private
 vs public API.

colocate_with has been deprecated from public API. There are still TF internal uses of colocate_with but previously calls to private and public were treated the same and resulted in deprecation warnings for internal uses.

PiperOrigin-RevId: 230002109
---
 tensorflow/python/framework/ops.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index e0f61622e4..63b7108f66 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -5145,12 +5145,19 @@ def _colocate_with_for_gradient(op, gradient_uid, ignore_existing=False):
         op, gradient_uid=gradient_uid, ignore_existing=ignore_existing)
 
 
+# Internal interface to colocate_with. colocate_with has been deprecated from
+# public API. There are still a few internal uses of colocate_with. Add internal
+# only API for those uses to avoid deprecation warning.
+def colocate_with(op, ignore_existing=False):
+  return _colocate_with_for_gradient(op, None, ignore_existing=ignore_existing)
+
+
 @deprecation.deprecated(
     date=None,
     instructions="Colocations handled automatically by placer.")
 @tf_export(v1=["colocate_with"])
-def colocate_with(op, ignore_existing=False):
-  return _colocate_with_for_gradient(op, None, ignore_existing=ignore_existing)
+def _colocate_with(op, ignore_existing=False):
+  return colocate_with(op, ignore_existing)
 
 
 @tf_export("control_dependencies")
-- 
GitLab


From fb4381d0b58ff000db720356d3fa62b2a6ba25d3 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 18 Jan 2019 15:16:37 -0800
Subject: [PATCH 1013/1095] Automated rollback of commit
 916df403fbfc799ac5b6af49a0c1cdf3846e66aa

PiperOrigin-RevId: 230002892
---
 tensorflow/python/keras/engine/base_layer.py      |  9 +++++----
 tensorflow/python/keras/engine/base_layer_test.py | 15 +--------------
 2 files changed, 6 insertions(+), 18 deletions(-)

diff --git a/tensorflow/python/keras/engine/base_layer.py b/tensorflow/python/keras/engine/base_layer.py
index 93849e771b..f63fb4e694 100644
--- a/tensorflow/python/keras/engine/base_layer.py
+++ b/tensorflow/python/keras/engine/base_layer.py
@@ -513,10 +513,11 @@ class Layer(checkpointable.Checkpointable):
       ValueError: if the layer's `call` method returns None (an invalid value).
     """
     input_list = nest.flatten(inputs)
-    # Accept NumPy inputs by converting to Tensors.
-    if all(isinstance(x, (np.ndarray, float, int)) for x in input_list):
-      inputs = nest.map_structure(ops.convert_to_tensor, inputs)
-      input_list = nest.flatten(inputs)
+    if context.executing_eagerly():
+      # Accept NumPy inputs by converting to Tensors when executing eagerly.
+      if all(isinstance(x, (np.ndarray, float, int)) for x in input_list):
+        inputs = nest.map_structure(ops.convert_to_tensor, inputs)
+        input_list = nest.flatten(inputs)
 
     # We will attempt to build a TF graph if & only if all inputs are symbolic.
     # This is always the case in graph mode. It can also be the case in eager
diff --git a/tensorflow/python/keras/engine/base_layer_test.py b/tensorflow/python/keras/engine/base_layer_test.py
index c925a8ca8a..ebee4a3043 100644
--- a/tensorflow/python/keras/engine/base_layer_test.py
+++ b/tensorflow/python/keras/engine/base_layer_test.py
@@ -26,8 +26,6 @@ from tensorflow.python.eager import context
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import test_util
-from tensorflow.python.keras import keras_parameterized
-from tensorflow.python.keras import testing_utils
 from tensorflow.python.keras.engine import base_layer
 from tensorflow.python.keras.optimizer_v2 import rmsprop
 from tensorflow.python.ops import array_ops
@@ -73,7 +71,7 @@ class InvalidLayer(base_layer.Layer):
     raise ValueError('You did something wrong!')
 
 
-class BaseLayerTest(keras_parameterized.TestCase):
+class BaseLayerTest(test.TestCase, parameterized.TestCase):
 
   @parameterized.parameters(DynamicLayer1, DynamicLayer2)
   def test_dynamic_layer_in_functional_model_in_graph_mode(self, layer_class):
@@ -212,17 +210,6 @@ class BaseLayerTest(keras_parameterized.TestCase):
     with self.assertRaisesRegexp(ValueError, 'You did something wrong!'):
       _ = InvalidLayer()(inputs)
 
-  @keras_parameterized.run_with_all_model_types
-  @test_util.run_in_graph_and_eager_modes
-  def test_build_with_numpy_data(self):
-    model_layers = [
-        keras.layers.Dense(3, activation='relu', kernel_initializer='ones'),
-        keras.layers.Dense(1, activation='sigmoid', kernel_initializer='ones')
-    ]
-    model = testing_utils.get_model_from_layers(model_layers, input_shape=(4,))
-    model(np.zeros((2, 4), dtype='float32'))
-    self.assertTrue(model.built)
-
   def test_using_symbolic_tensors_with_tf_ops(self):
     # Single-input.
     x = keras.Input((3,))
-- 
GitLab


From eefbb37190dc4634fa651a47840bce7a7ea16ba4 Mon Sep 17 00:00:00 2001
From: Zhenyu Tan <tanzheny@google.com>
Date: Fri, 18 Jan 2019 15:23:08 -0800
Subject: [PATCH 1014/1095] Copy clustering ops to core.

PiperOrigin-RevId: 230003911
---
 tensorflow/python/BUILD                      |  25 +
 tensorflow/python/ops/clustering_ops.py      | 770 +++++++++++++++++++
 tensorflow/python/ops/clustering_ops_test.py | 212 +++++
 3 files changed, 1007 insertions(+)
 create mode 100644 tensorflow/python/ops/clustering_ops.py
 create mode 100644 tensorflow/python/ops/clustering_ops_test.py

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 30de1fa440..4690bed62b 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -118,6 +118,7 @@ py_library(
         ":check_ops",
         ":client",
         ":client_testlib",
+        ":clustering_ops",
         ":collective_ops",
         ":cond_v2",
         ":confusion_matrix",
@@ -2112,6 +2113,30 @@ tf_py_test(
     ],
 )
 
+py_library(
+    name = "clustering_ops",
+    srcs = ["ops/clustering_ops.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":clustering_ops_gen",
+        ":framework",
+        ":ops",
+        ":training",
+    ],
+)
+
+tf_py_test(
+    name = "clustering_ops_test",
+    size = "medium",
+    srcs = ["ops/clustering_ops_test.py"],
+    additional_deps = [
+        ":client_testlib",
+        ":clustering_ops",
+        ":framework_for_generated_wrappers",
+        "//third_party/py/numpy",
+    ],
+)
+
 py_library(
     name = "collective_ops",
     srcs = ["ops/collective_ops.py"],
diff --git a/tensorflow/python/ops/clustering_ops.py b/tensorflow/python/ops/clustering_ops.py
new file mode 100644
index 0000000000..d48b89cbac
--- /dev/null
+++ b/tensorflow/python/ops/clustering_ops.py
@@ -0,0 +1,770 @@
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Clustering Operations."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import check_ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import gen_clustering_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import nn_impl
+from tensorflow.python.ops import random_ops
+from tensorflow.python.ops import state_ops
+from tensorflow.python.ops import variable_scope
+from tensorflow.python.ops.embedding_ops import embedding_lookup
+# go/tf-wildcard-import
+# pylint: disable=wildcard-import
+from tensorflow.python.ops.gen_clustering_ops import *
+# pylint: enable=wildcard-import
+
+# Euclidean distance between vectors U and V is defined as \\(||U - V||_F\\)
+# which is the square root of the sum of the absolute squares of the elements
+# difference.
+SQUARED_EUCLIDEAN_DISTANCE = 'squared_euclidean'
+# Cosine distance between vectors U and V is defined as
+# \\(1 - (U \dot V) / (||U||_F ||V||_F)\\)
+COSINE_DISTANCE = 'cosine'
+
+RANDOM_INIT = 'random'
+KMEANS_PLUS_PLUS_INIT = 'kmeans_plus_plus'
+KMC2_INIT = 'kmc2'
+
+# The name of the variable holding the cluster centers. Used by the Estimator.
+CLUSTERS_VAR_NAME = 'clusters'
+
+
+class KMeans(object):
+  """Creates the graph for k-means clustering."""
+
+  def __init__(self,
+               inputs,
+               num_clusters,
+               initial_clusters=RANDOM_INIT,
+               distance_metric=SQUARED_EUCLIDEAN_DISTANCE,
+               use_mini_batch=False,
+               mini_batch_steps_per_iteration=1,
+               random_seed=0,
+               kmeans_plus_plus_num_retries=2,
+               kmc2_chain_length=200):
+    """Creates an object for generating KMeans clustering graph.
+
+    This class implements the following variants of K-means algorithm:
+
+    If use_mini_batch is False, it runs standard full batch K-means. Each step
+    runs a single iteration of K-Means. This step can be run sharded across
+    multiple workers by passing a list of sharded inputs to this class. Note
+    however that a single step needs to process the full input at once.
+
+    If use_mini_batch is True, it runs a generalization of the mini-batch
+    K-means algorithm. It runs multiple iterations, where each iteration is
+    composed of mini_batch_steps_per_iteration steps. Two copies of cluster
+    centers are maintained: one that is updated at the end of each iteration,
+    and one that is updated every step. The first copy is used to compute
+    cluster allocations for each step, and for inference, while the second copy
+    is the one updated each step using the mini-batch update rule. After each
+    iteration is complete, this second copy is copied back the first copy.
+
+    Note that for use_mini_batch=True, when mini_batch_steps_per_iteration=1,
+    the algorithm reduces to the standard mini-batch algorithm. Also by setting
+    mini_batch_steps_per_iteration = num_inputs / batch_size, the algorithm
+    becomes an asynchronous version of the full-batch algorithm. Note however
+    that there is no guarantee by this implementation that each input is seen
+    exactly once per iteration. Also, different updates are applied
+    asynchronously without locking. So this asynchronous version may not behave
+    exactly like a full-batch version.
+
+    Args:
+      inputs: An input tensor or list of input tensors. It is assumed that the
+        data points have been previously randomly permuted.
+      num_clusters: An integer tensor specifying the number of clusters. This
+        argument is ignored if initial_clusters is a tensor or numpy array.
+      initial_clusters: Specifies the clusters used during initialization. One
+        of the following:
+        - a tensor or numpy array with the initial cluster centers.
+        - a function f(inputs, k) that returns up to k centers from `inputs`.
+        - "random": Choose centers randomly from `inputs`.
+        - "kmeans_plus_plus": Use kmeans++ to choose centers from `inputs`.
+        - "kmc2": Use the fast k-MC2 algorithm to choose centers from `inputs`.
+        In the last three cases, one batch of `inputs` may not yield
+        `num_clusters` centers, in which case initialization will require
+        multiple batches until enough centers are chosen. In the case of
+        "random" or "kmeans_plus_plus", if the input size is <= `num_clusters`
+        then the entire batch is chosen to be cluster centers.
+      distance_metric: Distance metric used for clustering. Supported options:
+        "squared_euclidean", "cosine".
+      use_mini_batch: If true, use the mini-batch k-means algorithm. Else assume
+        full batch.
+      mini_batch_steps_per_iteration: Number of steps after which the updated
+        cluster centers are synced back to a master copy.
+      random_seed: Seed for PRNG used to initialize seeds.
+      kmeans_plus_plus_num_retries: For each point that is sampled during
+        kmeans++ initialization, this parameter specifies the number of
+        additional points to draw from the current distribution before selecting
+        the best. If a negative value is specified, a heuristic is used to
+        sample O(log(num_to_sample)) additional points.
+      kmc2_chain_length: Determines how many candidate points are used by the
+        k-MC2 algorithm to produce one new cluster centers. If a (mini-)batch
+        contains less points, one new cluster center is generated from the
+        (mini-)batch.
+
+    Raises:
+      ValueError: An invalid argument was passed to initial_clusters or
+        distance_metric.
+    """
+    if isinstance(initial_clusters, str) and initial_clusters not in [
+        RANDOM_INIT, KMEANS_PLUS_PLUS_INIT, KMC2_INIT
+    ]:
+      raise ValueError(
+          "Unsupported initialization algorithm '%s'" % initial_clusters)
+    if distance_metric not in [SQUARED_EUCLIDEAN_DISTANCE, COSINE_DISTANCE]:
+      raise ValueError("Unsupported distance metric '%s'" % distance_metric)
+    self._inputs = inputs if isinstance(inputs, list) else [inputs]
+    self._num_clusters = num_clusters
+    self._initial_clusters = initial_clusters
+    self._distance_metric = distance_metric
+    self._use_mini_batch = use_mini_batch
+    self._mini_batch_steps_per_iteration = int(mini_batch_steps_per_iteration)
+    self._random_seed = random_seed
+    self._kmeans_plus_plus_num_retries = kmeans_plus_plus_num_retries
+    self._kmc2_chain_length = kmc2_chain_length
+
+  @classmethod
+  def _distance_graph(cls, inputs, clusters, distance_metric):
+    """Computes distance between each input and each cluster center.
+
+    Args:
+      inputs: list of input Tensors.
+      clusters: cluster Tensor.
+      distance_metric: distance metric used for clustering
+
+    Returns:
+      list of Tensors, where each element corresponds to each element in inputs.
+      The value is the distance of each row to all the cluster centers.
+      Currently only Euclidean distance and cosine distance are supported.
+    """
+    assert isinstance(inputs, list)
+    if distance_metric == SQUARED_EUCLIDEAN_DISTANCE:
+      return cls._compute_euclidean_distance(inputs, clusters)
+    elif distance_metric == COSINE_DISTANCE:
+      return cls._compute_cosine_distance(
+          inputs, clusters, inputs_normalized=True)
+    else:
+      assert False, str(distance_metric)
+
+  @classmethod
+  def _compute_euclidean_distance(cls, inputs, clusters):
+    """Computes Euclidean distance between each input and each cluster center.
+
+    Args:
+      inputs: list of input Tensors.
+      clusters: cluster Tensor.
+
+    Returns:
+      list of Tensors, where each element corresponds to each element in inputs.
+      The value is the distance of each row to all the cluster centers.
+    """
+    output = []
+    for inp in inputs:
+      with ops.colocate_with(inp, ignore_existing=True):
+        # Computes Euclidean distance. Note the first and third terms are
+        # broadcast additions.
+        squared_distance = (
+            math_ops.reduce_sum(math_ops.square(inp), 1, keepdims=True) -
+            2 * math_ops.matmul(inp, clusters, transpose_b=True) +
+            array_ops.transpose(
+                math_ops.reduce_sum(
+                    math_ops.square(clusters), 1, keepdims=True)))
+        output.append(squared_distance)
+
+    return output
+
+  @classmethod
+  def _compute_cosine_distance(cls, inputs, clusters, inputs_normalized=True):
+    """Computes cosine distance between each input and each cluster center.
+
+    Args:
+      inputs: list of input Tensor.
+      clusters: cluster Tensor
+      inputs_normalized: if True, it assumes that inp and clusters are
+      normalized and computes the dot product which is equivalent to the cosine
+      distance. Else it L2 normalizes the inputs first.
+
+    Returns:
+      list of Tensors, where each element corresponds to each element in inp.
+      The value is the distance of each row to all the cluster centers.
+    """
+    output = []
+    if not inputs_normalized:
+      with ops.colocate_with(clusters, ignore_existing=True):
+        clusters = nn_impl.l2_normalize(clusters, dim=1)
+    for inp in inputs:
+      with ops.colocate_with(inp, ignore_existing=True):
+        if not inputs_normalized:
+          inp = nn_impl.l2_normalize(inp, dim=1)
+        output.append(1 - math_ops.matmul(inp, clusters, transpose_b=True))
+    return output
+
+  def _infer_graph(self, inputs, clusters):
+    """Maps input to closest cluster and the score.
+
+    Args:
+      inputs: list of input Tensors.
+      clusters: Tensor of cluster centers.
+
+    Returns:
+      List of tuple, where each value in tuple corresponds to a value in inp.
+      The tuple has following three elements:
+      all_scores: distance of each input to each cluster center.
+      score: distance of each input to closest cluster center.
+      cluster_idx: index of cluster center closest to the corresponding input.
+    """
+    assert isinstance(inputs, list)
+    # Pairwise distances are used only by transform(). In all other cases, this
+    # sub-graph is not evaluated.
+    scores = self._distance_graph(inputs, clusters, self._distance_metric)
+    output = []
+    if (self._distance_metric == COSINE_DISTANCE and
+        not self._clusters_l2_normalized()):
+      # The cosine distance between normalized vectors x and y is the same as
+      # 2 * squared_euclidean_distance. We are using this fact and reusing the
+      # nearest_neighbors op.
+      # TODO(ands): Support COSINE distance in nearest_neighbors and remove
+      # this.
+      with ops.colocate_with(clusters, ignore_existing=True):
+        clusters = nn_impl.l2_normalize(clusters, dim=1)
+    for inp, score in zip(inputs, scores):
+      with ops.colocate_with(inp, ignore_existing=True):
+        (indices, distances) = gen_clustering_ops.nearest_neighbors(
+            inp, clusters, 1)
+        if self._distance_metric == COSINE_DISTANCE:
+          distances *= 0.5
+        output.append((score, array_ops.squeeze(distances, [-1]),
+                       array_ops.squeeze(indices, [-1])))
+    return zip(*output)
+
+  def _clusters_l2_normalized(self):
+    """Returns True if clusters centers are kept normalized."""
+    return (self._distance_metric == COSINE_DISTANCE and
+            (not self._use_mini_batch or
+             self._mini_batch_steps_per_iteration > 1))
+
+  def _create_variables(self, num_clusters):
+    """Creates variables.
+
+    Args:
+      num_clusters: an integer Tensor providing the number of clusters.
+
+    Returns:
+      Tuple with following elements:
+      - cluster_centers: a Tensor for storing cluster centers
+      - cluster_centers_initialized: bool Variable indicating whether clusters
+            are initialized.
+      - cluster_counts: a Tensor for storing counts of points assigned to this
+            cluster. This is used by mini-batch training.
+      - cluster_centers_updated: Tensor representing copy of cluster centers
+            that are updated every step.
+      - update_in_steps: numbers of steps left before we sync
+            cluster_centers_updated back to cluster_centers.
+    """
+    init_value = array_ops.constant([], dtype=dtypes.float32)
+    cluster_centers = variable_scope.variable(
+        init_value, name=CLUSTERS_VAR_NAME, validate_shape=False)
+    cluster_centers_initialized = variable_scope.variable(
+        False, dtype=dtypes.bool, name='initialized')
+
+    if self._use_mini_batch and self._mini_batch_steps_per_iteration > 1:
+      # Copy of cluster centers actively updated each step according to
+      # mini-batch update rule.
+      cluster_centers_updated = variable_scope.variable(
+          init_value, name='clusters_updated', validate_shape=False)
+      # How many steps till we copy the updated clusters to cluster_centers.
+      update_in_steps = variable_scope.variable(
+          self._mini_batch_steps_per_iteration,
+          dtype=dtypes.int64,
+          name='update_in_steps')
+      # Count of points assigned to cluster_centers_updated.
+      cluster_counts = variable_scope.variable(
+          array_ops.zeros([num_clusters], dtype=dtypes.int64))
+    else:
+      cluster_centers_updated = cluster_centers
+      update_in_steps = None
+      cluster_counts = (
+          variable_scope.variable(
+              array_ops.ones([num_clusters], dtype=dtypes.int64))
+          if self._use_mini_batch else None)
+    return (cluster_centers, cluster_centers_initialized, cluster_counts,
+            cluster_centers_updated, update_in_steps)
+
+  @classmethod
+  def _l2_normalize_data(cls, inputs):
+    """Normalized the input data."""
+    output = []
+    for inp in inputs:
+      with ops.colocate_with(inp, ignore_existing=True):
+        output.append(nn_impl.l2_normalize(inp, dim=1))
+    return output
+
+  def training_graph(self):
+    """Generate a training graph for kmeans algorithm.
+
+    This returns, among other things, an op that chooses initial centers
+    (init_op), a boolean variable that is set to True when the initial centers
+    are chosen (cluster_centers_initialized), and an op to perform either an
+    entire Lloyd iteration or a mini-batch of a Lloyd iteration (training_op).
+    The caller should use these components as follows. A single worker should
+    execute init_op multiple times until cluster_centers_initialized becomes
+    True. Then multiple workers may execute training_op any number of times.
+
+    Returns:
+      A tuple consisting of:
+      all_scores: A matrix (or list of matrices) of dimensions (num_input,
+        num_clusters) where the value is the distance of an input vector and a
+        cluster center.
+      cluster_idx: A vector (or list of vectors). Each element in the vector
+        corresponds to an input row in 'inp' and specifies the cluster id
+        corresponding to the input.
+      scores: Similar to cluster_idx but specifies the distance to the
+        assigned cluster instead.
+      cluster_centers_initialized: scalar indicating whether clusters have been
+        initialized.
+      init_op: an op to initialize the clusters.
+      training_op: an op that runs an iteration of training.
+    """
+    # Implementation of kmeans.
+    if (isinstance(self._initial_clusters, str) or
+        callable(self._initial_clusters)):
+      initial_clusters = self._initial_clusters
+      num_clusters = ops.convert_to_tensor(self._num_clusters)
+    else:
+      initial_clusters = ops.convert_to_tensor(self._initial_clusters)
+      num_clusters = array_ops.shape(initial_clusters)[0]
+
+    inputs = self._inputs
+    (cluster_centers_var, cluster_centers_initialized, total_counts,
+     cluster_centers_updated,
+     update_in_steps) = self._create_variables(num_clusters)
+    init_op = _InitializeClustersOpFactory(
+        self._inputs, num_clusters, initial_clusters, self._distance_metric,
+        self._random_seed, self._kmeans_plus_plus_num_retries,
+        self._kmc2_chain_length, cluster_centers_var, cluster_centers_updated,
+        cluster_centers_initialized).op()
+    cluster_centers = cluster_centers_var
+
+    if self._distance_metric == COSINE_DISTANCE:
+      inputs = self._l2_normalize_data(inputs)
+      if not self._clusters_l2_normalized():
+        cluster_centers = nn_impl.l2_normalize(cluster_centers, dim=1)
+
+    all_scores, scores, cluster_idx = self._infer_graph(inputs, cluster_centers)
+    if self._use_mini_batch:
+      sync_updates_op = self._mini_batch_sync_updates_op(
+          update_in_steps, cluster_centers_var, cluster_centers_updated,
+          total_counts)
+      assert sync_updates_op is not None
+      with ops.control_dependencies([sync_updates_op]):
+        training_op = self._mini_batch_training_op(
+            inputs, cluster_idx, cluster_centers_updated, total_counts)
+    else:
+      assert cluster_centers == cluster_centers_var
+      training_op = self._full_batch_training_op(
+          inputs, num_clusters, cluster_idx, cluster_centers_var)
+
+    return (all_scores, cluster_idx, scores, cluster_centers_initialized,
+            init_op, training_op)
+
+  def _mini_batch_sync_updates_op(self, update_in_steps, cluster_centers_var,
+                                  cluster_centers_updated, total_counts):
+    if self._use_mini_batch and self._mini_batch_steps_per_iteration > 1:
+      assert update_in_steps is not None
+      with ops.colocate_with(update_in_steps, ignore_existing=True):
+
+        def _f():
+          # Note that there is a race condition here, so we do a best effort
+          # updates here. We reset update_in_steps first so that other workers
+          # don't duplicate the updates. Also we update cluster_center_vars
+          # before resetting total_counts to avoid large updates to
+          # cluster_centers_updated based on partially updated
+          # cluster_center_vars.
+          with ops.control_dependencies([
+              state_ops.assign(update_in_steps,
+                               self._mini_batch_steps_per_iteration - 1)
+          ]):
+            with ops.colocate_with(
+                cluster_centers_updated, ignore_existing=True):
+              if self._distance_metric == COSINE_DISTANCE:
+                cluster_centers = nn_impl.l2_normalize(
+                    cluster_centers_updated, dim=1)
+              else:
+                cluster_centers = cluster_centers_updated
+            with ops.colocate_with(cluster_centers_var, ignore_existing=True):
+              with ops.control_dependencies(
+                  [state_ops.assign(cluster_centers_var, cluster_centers)]):
+                with ops.colocate_with(None, ignore_existing=True):
+                  with ops.control_dependencies([
+                      state_ops.assign(total_counts,
+                                       array_ops.zeros_like(total_counts))
+                  ]):
+                    return array_ops.identity(update_in_steps)
+
+        return control_flow_ops.cond(
+            update_in_steps <= 0, _f,
+            lambda: state_ops.assign_sub(update_in_steps, 1))
+    else:
+      return control_flow_ops.no_op()
+
+  def _mini_batch_training_op(self, inputs, cluster_idx_list, cluster_centers,
+                              total_counts):
+    """Creates an op for training for mini batch case.
+
+    Args:
+      inputs: list of input Tensors.
+      cluster_idx_list: A vector (or list of vectors). Each element in the
+        vector corresponds to an input row in 'inp' and specifies the cluster id
+        corresponding to the input.
+      cluster_centers: Tensor Ref of cluster centers.
+      total_counts: Tensor Ref of cluster counts.
+
+    Returns:
+      An op for doing an update of mini-batch k-means.
+    """
+    update_ops = []
+    for inp, cluster_idx in zip(inputs, cluster_idx_list):
+      with ops.colocate_with(inp, ignore_existing=True):
+        assert total_counts is not None
+        cluster_idx = array_ops.reshape(cluster_idx, [-1])
+        # Dedupe the unique ids of cluster_centers being updated so that updates
+        # can be locally aggregated.
+        unique_ids, unique_idx = array_ops.unique(cluster_idx)
+        num_unique_cluster_idx = array_ops.size(unique_ids)
+        # Fetch the old values of counts and cluster_centers.
+        with ops.colocate_with(total_counts, ignore_existing=True):
+          old_counts = array_ops.gather(total_counts, unique_ids)
+        # TODO(agarwal): This colocation seems to run into problems. Fix it.
+        with ops.colocate_with(cluster_centers, ignore_existing=True):
+          old_cluster_centers = array_ops.gather(cluster_centers, unique_ids)
+        # Locally aggregate the increment to counts.
+        count_updates = math_ops.unsorted_segment_sum(
+            array_ops.ones_like(unique_idx, dtype=total_counts.dtype),
+            unique_idx, num_unique_cluster_idx)
+        # Locally compute the sum of inputs mapped to each id.
+        # For a cluster with old cluster value x, old count n, and with data
+        # d_1,...d_k newly assigned to it, we recompute the new value as
+        # \\(x += (sum_i(d_i) - k * x) / (n + k)\\).
+        # Compute \\(sum_i(d_i)\\), see comment above.
+        cluster_center_updates = math_ops.unsorted_segment_sum(
+            inp, unique_idx, num_unique_cluster_idx)
+        # Shape to enable broadcasting count_updates and learning_rate to inp.
+        # It extends the shape with 1's to match the rank of inp.
+        broadcast_shape = array_ops.concat([
+            array_ops.reshape(num_unique_cluster_idx, [1]),
+            array_ops.ones(
+                array_ops.reshape(array_ops.rank(inp) - 1, [1]),
+                dtype=dtypes.int32)
+        ], 0)
+        # Subtract k * x, see comment above.
+        cluster_center_updates -= math_ops.cast(
+            array_ops.reshape(count_updates, broadcast_shape),
+            inp.dtype) * old_cluster_centers
+        learning_rate = math_ops.reciprocal(
+            math_ops.cast(old_counts + count_updates, inp.dtype))
+        learning_rate = array_ops.reshape(learning_rate, broadcast_shape)
+        # scale by 1 / (n + k), see comment above.
+        cluster_center_updates *= learning_rate
+        # Apply the updates.
+      update_counts = state_ops.scatter_add(total_counts, unique_ids,
+                                            count_updates)
+      update_cluster_centers = state_ops.scatter_add(
+          cluster_centers, unique_ids, cluster_center_updates)
+      update_ops.extend([update_counts, update_cluster_centers])
+    return control_flow_ops.group(*update_ops)
+
+  def _full_batch_training_op(self, inputs, num_clusters, cluster_idx_list,
+                              cluster_centers):
+    """Creates an op for training for full batch case.
+
+    Args:
+      inputs: list of input Tensors.
+      num_clusters: an integer Tensor providing the number of clusters.
+      cluster_idx_list: A vector (or list of vectors). Each element in the
+        vector corresponds to an input row in 'inp' and specifies the cluster id
+        corresponding to the input.
+      cluster_centers: Tensor Ref of cluster centers.
+
+    Returns:
+      An op for doing an update of mini-batch k-means.
+    """
+    cluster_sums = []
+    cluster_counts = []
+    epsilon = constant_op.constant(1e-6, dtype=inputs[0].dtype)
+    for inp, cluster_idx in zip(inputs, cluster_idx_list):
+      with ops.colocate_with(inp, ignore_existing=True):
+        cluster_sums.append(
+            math_ops.unsorted_segment_sum(inp, cluster_idx, num_clusters))
+        cluster_counts.append(
+            math_ops.unsorted_segment_sum(
+                array_ops.reshape(
+                    array_ops.ones(
+                        array_ops.reshape(array_ops.shape(inp)[0], [-1])),
+                    [-1, 1]), cluster_idx, num_clusters))
+    with ops.colocate_with(cluster_centers, ignore_existing=True):
+      new_clusters_centers = math_ops.add_n(cluster_sums) / (
+          math_ops.cast(math_ops.add_n(cluster_counts), cluster_sums[0].dtype) +
+          epsilon)
+      if self._clusters_l2_normalized():
+        new_clusters_centers = nn_impl.l2_normalize(new_clusters_centers, dim=1)
+    return state_ops.assign(cluster_centers, new_clusters_centers)
+
+
+class _InitializeClustersOpFactory(object):
+  """Internal class to create the op to initialize the clusters.
+
+    The op performs this algorithm (see constructor args):
+
+    num_remaining = num_clusters - length(cluster_centers)
+    if num_remaining == 0:
+      assert that cluster_centers_initialized is true
+    else:
+      assert that num_remaining > 0
+      new_centers = choose up to num_remaining initial centers
+      l2-normalize new_centers if using cosine distance
+      all_centers = concat(cluster_centers, new_centers)
+      cluster_centers := all_centers
+      if there is a cluster_centers_updated variable:
+        cluster_centers_updated := cluster_centers
+      num_now_remaining = num_clusters - length(cluster_centers)
+      if num_now_remaining == 0:
+        cluster_centers_initialized := true
+  """
+
+  # TODO(ccolby): Refactor this class so that kmc2 isn't so much a special case.
+
+  def __init__(self, inputs, num_clusters, initial_clusters, distance_metric,
+               random_seed, kmeans_plus_plus_num_retries, kmc2_chain_length,
+               cluster_centers, cluster_centers_updated,
+               cluster_centers_initialized):
+    """Creates an op factory.
+
+    Args:
+      inputs: See KMeans constructor.
+      num_clusters: An integer Tensor providing the number of clusters.
+      initial_clusters: See KMeans constructor.
+      distance_metric: See KMeans constructor.
+      random_seed: See KMeans constructor.
+      kmeans_plus_plus_num_retries: See KMeans constructor.
+      kmc2_chain_length: See KMeans constructor.
+      cluster_centers: The TF variable holding the initial centers. It may
+          already contain some centers when the op is executed.
+      cluster_centers_updated: A second TF variable to hold a copy of the
+          initial centers, used for full-batch mode. In mini-batch mode,
+          cluster_centers_updated is the same variable as cluster_centers.
+      cluster_centers_initialized: A boolean TF variable that will be set
+          to true when all the initial centers have been chosen.
+    """
+    # All of these instance variables are constants.
+    self._inputs = inputs
+    self._num_clusters = num_clusters
+    self._initial_clusters = initial_clusters
+    self._distance_metric = distance_metric
+    self._random_seed = random_seed
+    self._kmeans_plus_plus_num_retries = kmeans_plus_plus_num_retries
+    self._kmc2_chain_length = kmc2_chain_length
+    self._cluster_centers = cluster_centers
+    self._cluster_centers_updated = cluster_centers_updated
+    self._cluster_centers_initialized = cluster_centers_initialized
+
+    self._num_selected = array_ops.shape(self._cluster_centers)[0]
+    self._num_remaining = self._num_clusters - self._num_selected
+    self._num_data = math_ops.add_n(
+        [array_ops.shape(i)[0] for i in self._inputs])
+
+  def _random(self):
+    indices = random_ops.random_uniform(
+        array_ops.reshape(self._num_remaining, [-1]),
+        minval=0,
+        maxval=math_ops.cast(self._num_data, dtypes.int64),
+        seed=self._random_seed,
+        dtype=dtypes.int64)
+    return embedding_lookup(self._inputs, indices, partition_strategy='div')
+
+  def _kmeans_plus_plus(self):
+    # Points from only the first shard are used for initializing centers.
+    # TODO(ands): Use all points.
+    inp = self._inputs[0]
+    if self._distance_metric == COSINE_DISTANCE:
+      inp = nn_impl.l2_normalize(inp, dim=1)
+    return gen_clustering_ops.kmeans_plus_plus_initialization(
+        inp,
+        math_ops.to_int64(self._num_remaining), self._random_seed,
+        self._kmeans_plus_plus_num_retries)
+
+  def _kmc2_multiple_centers(self):
+    """Adds new initial cluster centers using the k-MC2 algorithm.
+
+    In each call to the op, the provided batch is split into subsets based on
+    the specified `kmc2_chain_length`. On each subset, a single Markov chain of
+    the k-MC2 algorithm is used to add *one* new center cluster center. If there
+    are less than `kmc2_chain_length` points in the subset, a single center is
+    added using one Markov chain on the full input. It is assumed that the
+    provided batch has previously been randomly permuted. Otherwise, k-MC2 may
+    return suboptimal centers.
+
+    Returns:
+      An op that adds new cluster centers.
+    """
+    # The op only operates on the first shard of data.
+    first_shard = self._inputs[0]
+    # Number of points in the input that can be used.
+    batch_size = array_ops.shape(first_shard)[0]
+    # Maximum number of subsets such that the size of each subset is at least
+    # `kmc2_chain_length`. Final subsets may be larger.
+    max_to_sample = math_ops.cast(
+        batch_size / self._kmc2_chain_length, dtype=dtypes.int32)
+    # We sample at least one new center and at most all remaining centers.
+    num_to_sample = math_ops.maximum(
+        math_ops.minimum(self._num_remaining, max_to_sample), 1)
+
+    def _cond(i, _):
+      """Stopping condition for the while loop."""
+      return math_ops.less(i, num_to_sample)
+
+    def _body(i, _):
+      """Body that adds a single new center based on a subset."""
+
+      def _sample_random():
+        """Returns a random point as a cluster center."""
+        # By assumption the batch is reshuffled and _sample_random is always
+        # called for i=0. Hence, we simply return the first point.
+        new_center = array_ops.reshape(first_shard[0], [1, -1])
+        if self._distance_metric == COSINE_DISTANCE:
+          new_center = nn_impl.l2_normalize(new_center, dim=1)
+        return new_center
+
+      def _sample_kmc2_chain():
+        """Returns previous centers as well as a new center sampled using k-MC2.
+        """
+        # Extract the subset from the underlying batch.
+        start = i * self._kmc2_chain_length
+        end = start + self._kmc2_chain_length
+        subset = first_shard[start:end]
+        # Compute the distances from points in the subset to previous centers.
+        _, distances = gen_clustering_ops.nearest_neighbors(
+            subset, self._cluster_centers, 1)
+        # Sample index of new center using k-MC2 Markov chain.
+        new_center_index = gen_clustering_ops.kmc2_chain_initialization(
+            array_ops.squeeze(distances), self._random_seed)
+        # Extract actual new center.
+        newly_sampled_center = array_ops.reshape(subset[new_center_index],
+                                                 [1, -1])
+        # Return concatenation with previously sampled centers.
+        if self._distance_metric == COSINE_DISTANCE:
+          newly_sampled_center = nn_impl.l2_normalize(
+              newly_sampled_center, dim=1)
+        return array_ops.concat([self._cluster_centers, newly_sampled_center],
+                                0)
+
+      # Obtain a random point if there are no previously sampled centers.
+      # Otherwise, construct a k-MC2 Markov chain.
+      new_centers = control_flow_ops.cond(
+          math_ops.equal(self._num_selected, 0), _sample_random,
+          _sample_kmc2_chain)
+      # Assign new cluster centers to underlying variable.
+      assigned_centers = state_ops.assign(
+          self._cluster_centers, new_centers, validate_shape=False)
+      if self._cluster_centers_updated is not self._cluster_centers:
+        assigned_centers = state_ops.assign(
+            self._cluster_centers_updated,
+            assigned_centers,
+            validate_shape=False)
+      return i + 1, self._num_clusters - array_ops.shape(assigned_centers)[0]
+
+    # Add num_to_sample new data points.
+    _, num_remaining = control_flow_ops.while_loop(_cond, _body, [0, 0])
+    return num_remaining
+
+  def _greedy_batch_sampler(self, sampler):
+    # If the input dataset size is smaller than the number of centers
+    # remaining, choose the entire input dataset as centers. This can happen
+    # with mini-batch. Otherwise, sample the batch according to the provided
+    # sampler.
+    return control_flow_ops.cond(self._num_data <= self._num_remaining,
+                                 lambda: array_ops.concat(self._inputs, 0),
+                                 sampler)
+
+  def _single_batch_sampler(self, sampler):
+    # Enforce that there are at least as many data points as centers
+    # remaining. This gives the provided sampler the chance to select all
+    # remaining centers from a single batch.
+    with ops.control_dependencies(
+        [check_ops.assert_greater_equal(self._num_data, self._num_remaining)]):
+      return sampler()
+
+  def _choose_initial_centers(self):
+    if isinstance(self._initial_clusters, str):
+      if self._initial_clusters == RANDOM_INIT:
+        return self._greedy_batch_sampler(self._random)
+      else:  # self._initial_clusters == KMEANS_PLUS_PLUS_INIT
+        return self._single_batch_sampler(self._kmeans_plus_plus)
+    elif callable(self._initial_clusters):
+      return self._initial_clusters(self._inputs, self._num_remaining)
+    else:
+      with ops.control_dependencies([
+          check_ops.assert_equal(self._num_remaining,
+                                 array_ops.shape(self._initial_clusters)[0])
+      ]):
+        return self._initial_clusters
+
+  def _add_new_centers(self):
+    """Adds some centers and returns the number of centers remaining."""
+    new_centers = self._choose_initial_centers()
+    if self._distance_metric == COSINE_DISTANCE:
+      new_centers = nn_impl.l2_normalize(new_centers, dim=1)
+    # If cluster_centers is empty, it doesn't have the right shape for concat.
+    all_centers = control_flow_ops.cond(
+        math_ops.equal(self._num_selected, 0), lambda: new_centers,
+        lambda: array_ops.concat([self._cluster_centers, new_centers], 0))
+    # TODO(ccolby): De-dupe all_centers?
+    a = state_ops.assign(
+        self._cluster_centers, all_centers, validate_shape=False)
+    if self._cluster_centers_updated is not self._cluster_centers:
+      a = state_ops.assign(
+          self._cluster_centers_updated, a, validate_shape=False)
+    return self._num_clusters - array_ops.shape(a)[0]
+
+  def _initialize(self):
+    with ops.control_dependencies([
+        check_ops.assert_positive(self._num_remaining),
+    ]):
+      if self._initial_clusters == KMC2_INIT:
+        num_now_remaining = self._kmc2_multiple_centers()
+      else:
+        num_now_remaining = self._add_new_centers()
+      return control_flow_ops.cond(
+          math_ops.equal(num_now_remaining, 0),
+          lambda: state_ops.assign(self._cluster_centers_initialized, True),
+          control_flow_ops.no_op)
+
+  def op(self):
+    """Returns the cluster initializer op."""
+    return control_flow_ops.cond(
+        math_ops.equal(self._num_remaining, 0),
+        lambda: check_ops.assert_equal(self._cluster_centers_initialized, True),
+        self._initialize)
diff --git a/tensorflow/python/ops/clustering_ops_test.py b/tensorflow/python/ops/clustering_ops_test.py
new file mode 100644
index 0000000000..5804c660e6
--- /dev/null
+++ b/tensorflow/python/ops/clustering_ops_test.py
@@ -0,0 +1,212 @@
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License.  You may obtain a copy of
+# the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+# License for the specific language governing permissions and limitations under
+# the License.
+# ==============================================================================
+"""Tests for clustering_ops."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import clustering_ops
+from tensorflow.python.platform import test
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class KmeansPlusPlusInitializationTest(test.TestCase):
+
+  # All but one input point are close to (101, 1). With uniform random sampling,
+  # it is highly improbable for (-1, -1) to be selected.
+  def setUp(self):
+    self._points = np.array([[100., 0.],
+                             [101., 2.],
+                             [102., 0.],
+                             [100., 1.],
+                             [100., 2.],
+                             [101., 0.],
+                             [101., 0.],
+                             [101., 1.],
+                             [102., 0.],
+                             [-1., -1.]]).astype(np.float32)
+
+  def runTestWithSeed(self, seed):
+    with self.cached_session():
+      sampled_points = clustering_ops.kmeans_plus_plus_initialization(
+          self._points, 3, seed, (seed % 5) - 1)
+      self.assertAllClose(
+          sorted(self.evaluate(sampled_points).tolist()),
+          [[-1., -1.], [101., 1.], [101., 1.]],
+          atol=1.0)
+
+  def testBasic(self):
+    for seed in range(100):
+      self.runTestWithSeed(seed)
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class KMC2InitializationTest(test.TestCase):
+
+  def runTestWithSeed(self, seed):
+    with self.cached_session():
+      distances = np.zeros(1000).astype(np.float32)
+      distances[6] = 10e7
+      distances[4] = 10e3
+
+      sampled_point = clustering_ops.kmc2_chain_initialization(distances, seed)
+      self.assertAllEqual(sampled_point, 6)
+      distances[6] = 0.0
+      sampled_point = clustering_ops.kmc2_chain_initialization(distances, seed)
+      self.assertAllEqual(sampled_point, 4)
+
+  def testBasic(self):
+    for seed in range(100):
+      self.runTestWithSeed(seed)
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class KMC2InitializationLargeTest(test.TestCase):
+
+  def setUp(self):
+    self._distances = np.zeros(1001)
+    self._distances[500] = 100.0
+    self._distances[1000] = 50.0
+
+  def testBasic(self):
+    with self.cached_session():
+      counts = {}
+      seed = 0
+      for i in range(50):
+        sample = self.evaluate(
+            clustering_ops.kmc2_chain_initialization(self._distances, seed + i))
+        counts[sample] = counts.get(sample, 0) + 1
+      self.assertEquals(len(counts), 2)
+      self.assertTrue(500 in counts)
+      self.assertTrue(1000 in counts)
+      self.assertGreaterEqual(counts[500], 5)
+      self.assertGreaterEqual(counts[1000], 5)
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class KMC2InitializationCornercaseTest(test.TestCase):
+
+  def setUp(self):
+    self._distances = np.zeros(10)
+
+  def runTestWithSeed(self, seed):
+    with self.cached_session():
+      sampled_point = clustering_ops.kmc2_chain_initialization(
+          self._distances, seed)
+      self.assertAllEqual(sampled_point, 0)
+
+  def testBasic(self):
+    for seed in range(100):
+      self.runTestWithSeed(seed)
+
+
+@test_util.run_all_in_graph_and_eager_modes
+# A simple test that can be verified by hand.
+class NearestCentersTest(test.TestCase):
+
+  def setUp(self):
+    self._points = np.array([[100., 0.],
+                             [101., 2.],
+                             [99., 2.],
+                             [1., 1.]]).astype(np.float32)
+
+    self._centers = np.array([[100., 0.],
+                              [99., 1.],
+                              [50., 50.],
+                              [0., 0.],
+                              [1., 1.]]).astype(np.float32)
+
+  def testNearest1(self):
+    with self.cached_session():
+      [indices, distances] = clustering_ops.nearest_neighbors(self._points,
+                                                              self._centers, 1)
+      self.assertAllClose(indices, [[0], [0], [1], [4]])
+      self.assertAllClose(distances, [[0.], [5.], [1.], [0.]])
+
+  def testNearest2(self):
+    with self.cached_session():
+      [indices, distances] = clustering_ops.nearest_neighbors(self._points,
+                                                              self._centers, 2)
+      self.assertAllClose(indices, [[0, 1], [0, 1], [1, 0], [4, 3]])
+      self.assertAllClose(distances, [[0., 2.], [5., 5.], [1., 5.], [0., 2.]])
+
+
+@test_util.run_all_in_graph_and_eager_modes
+# A test with large inputs.
+class NearestCentersLargeTest(test.TestCase):
+
+  def setUp(self):
+    num_points = 1000
+    num_centers = 2000
+    num_dim = 100
+    max_k = 5
+    # Construct a small number of random points and later tile them.
+    points_per_tile = 10
+    assert num_points % points_per_tile == 0
+    points = np.random.standard_normal(
+        [points_per_tile, num_dim]).astype(np.float32)
+    # Construct random centers.
+    self._centers = np.random.standard_normal(
+        [num_centers, num_dim]).astype(np.float32)
+
+    # Exhaustively compute expected nearest neighbors.
+    def squared_distance(x, y):
+      return np.linalg.norm(x - y, ord=2)**2
+
+    nearest_neighbors = [
+        sorted([(squared_distance(point, self._centers[j]), j)
+                for j in range(num_centers)])[:max_k] for point in points
+    ]
+    expected_nearest_neighbor_indices = np.array(
+        [[i for _, i in nn] for nn in nearest_neighbors])
+    expected_nearest_neighbor_squared_distances = np.array(
+        [[dist for dist, _ in nn] for nn in nearest_neighbors])
+    # Tile points and expected results to reach requested size (num_points)
+    (self._points, self._expected_nearest_neighbor_indices,
+     self._expected_nearest_neighbor_squared_distances) = (
+         np.tile(x, (int(num_points / points_per_tile), 1))
+         for x in (points, expected_nearest_neighbor_indices,
+                   expected_nearest_neighbor_squared_distances))
+
+  def testNearest1(self):
+    with self.cached_session():
+      [indices, distances] = clustering_ops.nearest_neighbors(self._points,
+                                                              self._centers, 1)
+      self.assertAllClose(
+          indices,
+          self._expected_nearest_neighbor_indices[:, [0]])
+      self.assertAllClose(
+          distances,
+          self._expected_nearest_neighbor_squared_distances[:, [0]])
+
+  def testNearest5(self):
+    with self.cached_session():
+      [indices, distances] = clustering_ops.nearest_neighbors(self._points,
+                                                              self._centers, 5)
+      self.assertAllClose(
+          indices,
+          self._expected_nearest_neighbor_indices[:, 0:5])
+      self.assertAllClose(
+          distances,
+          self._expected_nearest_neighbor_squared_distances[:, 0:5])
+
+
+if __name__ == "__main__":
+  np.random.seed(0)
+  test.main()
-- 
GitLab


From d86e3ae60853df703baa664c6ccfb872fbd67893 Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Fri, 18 Jan 2019 15:39:23 -0800
Subject: [PATCH 1015/1095] [tf.data] Add logging to the performance modeling
 framework.

PiperOrigin-RevId: 230006654
---
 tensorflow/core/framework/model.cc | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/tensorflow/core/framework/model.cc b/tensorflow/core/framework/model.cc
index 47c8dbe61b..a1c87a3f42 100644
--- a/tensorflow/core/framework/model.cc
+++ b/tensorflow/core/framework/model.cc
@@ -380,7 +380,12 @@ std::shared_ptr<Node> Model::AddNode(Node::Factory factory, const string& name,
     output_ = node;
   }
   if (output) {
+    VLOG(3) << "Adding " << node->name() << "(id:" << node->id()
+            << ") as input for " << output->name() << "(id:" << output->id()
+            << ")";
     output->add_input(node);
+  } else {
+    VLOG(3) << "Adding " << node->name() << "(id:" << node->id() << ")";
   }
   collect_resource_usage_ =
       collect_resource_usage_ || node->has_tunable_parameters();
@@ -493,10 +498,13 @@ void Model::RecordStop(const string& name, bool start_output) {
 void Model::RemoveNode(const string& name) {
   mutex_lock l(mu_);
   auto node = gtl::FindOrNull(lookup_table_, name);
-  if (node && (*node)->output()) {
-    (*node)->output()->remove_input(*node);
+  if (node) {
+    if ((*node)->output()) {
+      (*node)->output()->remove_input(*node);
+    }
+    VLOG(3) << "Removing " << (*node)->name() << "(id:" << (*node)->id() << ")";
+    remove_node_hook_(*node);
   }
-  remove_node_hook_(*node);
   lookup_table_.erase(name);
 }
 
-- 
GitLab


From 708090d48995456bfa66615398d8c56dadebe018 Mon Sep 17 00:00:00 2001
From: James Ring <sjr@google.com>
Date: Fri, 18 Jan 2019 15:59:16 -0800
Subject: [PATCH 1016/1095] Make TF_DeleteKernelBuilder not crash on nullptr.

After this change, TF_DeleteKernelBuilder will comply with the
conventions established in c_api.h, namely that *Delete* functions are
safe with nullptr parameters.

PiperOrigin-RevId: 230009727
---
 tensorflow/c/kernels.cc      | 7 ++++---
 tensorflow/c/kernels_test.cc | 4 ++++
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/tensorflow/c/kernels.cc b/tensorflow/c/kernels.cc
index c45efd7fae..9505bf9dda 100644
--- a/tensorflow/c/kernels.cc
+++ b/tensorflow/c/kernels.cc
@@ -48,9 +48,10 @@ TF_KernelBuilder* TF_NewKernelBuilder(
 }
 
 void TF_DeleteKernelBuilder(TF_KernelBuilder* builder) {
-  DCHECK_NE(builder, nullptr);
-  delete builder->cc_builder;
-  delete builder;
+  if (builder != nullptr) {
+    delete builder->cc_builder;
+    delete builder;
+  }
 }
 
 namespace tensorflow {
diff --git a/tensorflow/c/kernels_test.cc b/tensorflow/c/kernels_test.cc
index 531eb598fa..0d2954717e 100644
--- a/tensorflow/c/kernels_test.cc
+++ b/tensorflow/c/kernels_test.cc
@@ -224,4 +224,8 @@ TEST(TestKernel, TestInputAndOutputCount) {
   }
 }
 
+TEST(TestKernel, DeleteKernelBuilderIsOkOnNull) {
+  TF_DeleteKernelBuilder(nullptr);
+}
+
 }  // namespace tensorflow
-- 
GitLab


From f21f60fb993b89d7ea3a136c78a2491b1f1c4783 Mon Sep 17 00:00:00 2001
From: Mark Daoust <markdaoust@google.com>
Date: Fri, 18 Jan 2019 16:02:12 -0800
Subject: [PATCH 1017/1095] Make tf2 doc-generator aware of tf_export,
 tf_inspect. Fix tf.__all__ for doc generation. Add site_path flag.

PiperOrigin-RevId: 230010213
---
 tensorflow/tools/docs/BUILD        |  5 +++-
 tensorflow/tools/docs/generate2.py | 39 ++++++++++++++++++++++++++++--
 2 files changed, 41 insertions(+), 3 deletions(-)

diff --git a/tensorflow/tools/docs/BUILD b/tensorflow/tools/docs/BUILD
index b072853a4e..4bb452f256 100644
--- a/tensorflow/tools/docs/BUILD
+++ b/tensorflow/tools/docs/BUILD
@@ -163,7 +163,10 @@ py_binary(
     name = "generate2",
     srcs = ["generate2.py"],
     srcs_version = "PY2AND3",
-    deps = ["//tensorflow:tensorflow_py"],
+    deps = [
+        "//tensorflow:tensorflow_py",
+        "//tensorflow/python:util",
+    ],
 )
 
 py_library(
diff --git a/tensorflow/tools/docs/generate2.py b/tensorflow/tools/docs/generate2.py
index fba909d26d..0a50eb6c23 100644
--- a/tensorflow/tools/docs/generate2.py
+++ b/tensorflow/tools/docs/generate2.py
@@ -31,10 +31,22 @@ from os import path
 
 from absl import app
 from absl import flags
-
 import tensorflow as tf
 
+from tensorflow_docs.api_generator import doc_generator_visitor
 from tensorflow_docs.api_generator import generate_lib
+from tensorflow_docs.api_generator import parser
+
+from tensorflow.python.util import tf_export
+from tensorflow.python.util import tf_inspect
+
+# Use tensorflow's `tf_inspect`, which is aware of `tf_decorator`.
+parser.tf_inspect = tf_inspect
+
+# `tf` has an `__all__` that doesn't list important things like `keras`.
+# The doc generator recognizes `__all__` as the list of public symbols.
+# So patch `tf.__all__` to list everything.
+tf.__all__ = [item_name for item_name, value in tf_inspect.getmembers(tf)]
 
 FLAGS = flags.FLAGS
 
@@ -50,6 +62,28 @@ flags.DEFINE_string(
 flags.DEFINE_bool("search_hints", True,
                   "Include meta-data search hints at the top of each file.")
 
+flags.DEFINE_string("site_path", "",
+                    "The prefix ({site-path}/api_docs/python/...) used in the "
+                    "`_toc.yaml` and `_redirects.yaml` files")
+
+
+# The doc generator isn't aware of tf_export.
+# So prefix the score tuples with -1 when this is the canonical name, +1
+# otherwise. The generator chooses the name with the lowest score.
+class TfExportAwareDocGeneratorVisitor(
+    doc_generator_visitor.DocGeneratorVisitor):
+  """A `tf_export` aware doc_visitor."""
+
+  def _score_name(self, name):
+    canonical = tf_export.get_canonical_name_for_symbol(self._index[name])
+
+    canonical_score = 1
+    if canonical is not None and name == "tf." + canonical:
+      canonical_score = -1
+
+    scores = super(TfExportAwareDocGeneratorVisitor, self)._score_name(name)
+    return (canonical_score,) + scores
+
 
 def build_docs(output_dir, code_url_prefix, search_hints=True):
   """Build api docs for tensorflow v2.
@@ -66,7 +100,8 @@ def build_docs(output_dir, code_url_prefix, search_hints=True):
       base_dir=base_dir,
       search_hints=search_hints,
       code_url_prefix=code_url_prefix,
-      site_path="api_docs/")
+      site_path=FLAGS.site_path,
+      visitor_cls=TfExportAwareDocGeneratorVisitor)
 
   doc_generator.build(output_dir)
 
-- 
GitLab


From e38ca6bfaf6ef77ca97ab76d71ec8b17ad61da29 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 18 Jan 2019 16:09:38 -0800
Subject: [PATCH 1018/1095] Automated rollback of commit
 2e2a3ca8e827138a02da1cf129ab28057b9f9ddd

PiperOrigin-RevId: 230011598
---
 tensorflow/core/kernels/cwise_ops.h | 34 ++++++++++++++++++++++++++---
 tensorflow/workspace.bzl            |  8 +++----
 2 files changed, 35 insertions(+), 7 deletions(-)

diff --git a/tensorflow/core/kernels/cwise_ops.h b/tensorflow/core/kernels/cwise_ops.h
index 15bd533be0..a22d76717a 100644
--- a/tensorflow/core/kernels/cwise_ops.h
+++ b/tensorflow/core/kernels/cwise_ops.h
@@ -51,12 +51,15 @@ struct scalar_arg_op<std::complex<double>> {
 };
 #endif
 
-#if EIGEN_HAS_CXX11_MATH == 0
 template <typename T>
 struct scalar_asinh_op {
   EIGEN_EMPTY_STRUCT_CTOR(scalar_asinh_op)
   EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T operator()(const T& a) const {
+#if EIGEN_HAS_CXX11_MATH
+    return numext::asinh(a);
+#else
     return std::asinh(a);
+#endif  // EIGEN_HAS_CXX11_MATH
   }
 };
 template <typename T>
@@ -68,7 +71,11 @@ template <typename T>
 struct scalar_acosh_op {
   EIGEN_EMPTY_STRUCT_CTOR(scalar_acosh_op)
   EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T operator()(const T& a) const {
+#if EIGEN_HAS_CXX11_MATH
+    return numext::acosh(a);
+#else
     return std::acosh(a);
+#endif  // EIGEN_HAS_CXX11_MATH
   }
 };
 template <typename T>
@@ -80,14 +87,35 @@ template <typename T>
 struct scalar_atanh_op {
   EIGEN_EMPTY_STRUCT_CTOR(scalar_atanh_op)
   EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T operator()(const T& a) const {
+#if EIGEN_HAS_CXX11_MATH
+    return numext::atanh(a);
+#else
     return std::atanh(a);
+#endif  // EIGEN_HAS_CXX11_MATH
   }
 };
 template <typename T>
 struct functor_traits<scalar_atanh_op<T>> {
   enum { Cost = 5 * NumTraits<T>::MulCost, PacketAccess = false };
 };
-#endif
+
+// TODO(rmlarsen): This is a workaround for upstream change
+// https://bitbucket.org/eigen/eigen/commits/f339468d04d0f87caeb6cab9aef568627e9f6ea9
+// that renamed scalar_binary_pow_op to scalar_pow_op and deleted the unary
+// version of the latter. Remove once we upgrade to Eigen 3.3.
+template <typename Scalar, typename Exponent>
+struct scalar_binary_pow_op_google {
+  EIGEN_EMPTY_STRUCT_CTOR(scalar_binary_pow_op_google)
+  EIGEN_DEVICE_FUNC inline Scalar operator()(const Scalar& a,
+                                             const Exponent& b) const {
+    return numext::pow(a, b);
+  }
+};
+
+template <typename Scalar, typename Exponent>
+struct functor_traits<scalar_binary_pow_op_google<Scalar, Exponent>> {
+  enum { Cost = 5 * NumTraits<Scalar>::MulCost, PacketAccess = false };
+};
 
 template <typename Scalar, typename Exponent>
 struct safe_scalar_binary_pow_op {
@@ -837,7 +865,7 @@ template <typename T>
 struct floor_div_real : base<T, Eigen::internal::google_floor_div_real<T>> {};
 
 template <typename T>
-struct pow : base<T, Eigen::internal::scalar_pow_op<T, T>> {};
+struct pow : base<T, Eigen::internal::scalar_binary_pow_op_google<T, T>> {};
 
 template <typename T>
 struct safe_pow : base<T, Eigen::internal::safe_scalar_binary_pow_op<T, T>> {
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 466262ab3f..49c68b6ec0 100755
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -140,11 +140,11 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "eigen_archive",
         build_file = clean_dep("//third_party:eigen.BUILD"),
-        sha256 = "48678550a32665331d729be87076e576f2502fff325f5b6c2c78ebf7b1b22c7b",
-        strip_prefix = "eigen-eigen-bcc817c0ba98",
+        sha256 = "df54d805573871d03aaef6d27d45439c18ff6e8db215a3b922b2daefd6914147",
+        strip_prefix = "eigen-eigen-ad3bcd81cc49",
         urls = [
-            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/bcc817c0ba98.tar.gz",
-            "https://bitbucket.org/eigen/eigen/get/bcc817c0ba98.tar.gz",
+            "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/ad3bcd81cc49.tar.gz",
+            "https://bitbucket.org/eigen/eigen/get/ad3bcd81cc49.tar.gz",
         ],
     )
 
-- 
GitLab


From 6108a72eafbbec6a161f70691b9dcef1798dbbbe Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Fri, 18 Jan 2019 16:16:17 -0800
Subject: [PATCH 1019/1095] Put the calibration resource into the device's
 resource manager as well. This helps to make the calibration process work in
 both the old workflow and the new one.

PiperOrigin-RevId: 230012540
---
 .../contrib/tensorrt/convert/convert_graph.cc |  6 ---
 .../contrib/tensorrt/kernels/trt_engine_op.cc | 43 +++++++++++++------
 .../contrib/tensorrt/kernels/trt_engine_op.h  |  4 +-
 3 files changed, 33 insertions(+), 20 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index e2350f69a6..15cf97a2ee 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -666,14 +666,8 @@ tensorflow::Status CreateTRTNode(const std::vector<EngineInfo>& infos, int pos,
     segment_string = info.segment_graph_def.SerializeAsString();
   }
 
-  // TODO(aaroey): use enum instead, and add a helper method to do the
-  // conversion.
   string prec_string;
   TF_RETURN_IF_ERROR(GetPrecisionModeName(info.precision_mode, &prec_string));
-  if (info.precision_mode == INT8MODE && calibrate_int8 &&
-      !TRTResourceManager::instance()->getManager("TRTCalibration")) {
-    LOG(ERROR) << "Failed to construct calibration storage";
-  }
   tensorflow::NodeDefBuilder node_builder(info.engine_name, "TRTEngineOp");
   if (!info.device.empty()) node_builder.Device(info.device);
   if (VLOG_IS_ON(1)) {
diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
index 7548c8ccda..d28d88fa3a 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
+++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
@@ -195,18 +195,37 @@ void TRTEngineOp::ExecuteCalibration(OpKernelContext* ctx,
   VLOG(1) << "Executing TRT calibration: " << name();
   helper->Ref();
   tensorflow::core::ScopedUnref sc(helper);
-  // TODO(aaroey): remove the ResourceMgr singleton.
-  auto trt_rm = TRTResourceManager::instance();
-  auto res_mgr = trt_rm->getManager("TRTCalibration");
+  auto res_mgr = ctx->resource_manager();
   TRTCalibrationResource* calib_res = nullptr;
-  auto status = res_mgr->LookupOrCreate(
-      funcdef_name_, "Calibrator", &calib_res,
-      {[ctx, this](TRTCalibrationResource** cr) -> tensorflow::Status {
-        return this->AllocateCalibrationResources(ctx, cr);
-      }});
-  if (!status.ok()) {
-    ctx->SetStatus(status);
-    return;
+  OP_REQUIRES_OK(
+      ctx,
+      res_mgr->LookupOrCreate(
+          "TF_TRT_Calibration", name(),
+          reinterpret_cast<SerializableResourceBase**>(&calib_res),
+          {[ctx, this](SerializableResourceBase** cr) -> tensorflow::Status {
+            return this->AllocateCalibrationResources(ctx, cr);
+          }}));
+  tensorflow::core::ScopedUnref calib_sc(calib_res);
+  // TODO(aaroey): here we also add the resource to the ResourceMgr singleton.
+  // This is needed before we migrate all uses of calib_graph_to_infer_graph()
+  // to the new calibration workflow. After that we'll remove this block.
+  {
+    auto deprecated_rm =
+        TRTResourceManager::instance()->getManager("TRTCalibration");
+    TRTCalibrationResource* copied_resource = nullptr;
+    // Check whether the resource exists, and create it if not.
+    if (deprecated_rm->Lookup(funcdef_name_, "Calibrator", &copied_resource)
+            .ok()) {
+      // Do nothing if the resource exists.
+      copied_resource->Unref();
+    } else {
+      copied_resource = calib_res;
+      // Increase the refcount by 1 then transfer the ownership of that refcount
+      // to the ResourceMgr singleton.
+      copied_resource->Ref();
+      OP_REQUIRES_OK(ctx, deprecated_rm->Create(funcdef_name_, "Calibrator",
+                                                copied_resource));
+    }
   }
   int num_inputs = ctx->num_inputs();
   // Pass input data to calibrator
@@ -542,7 +561,7 @@ EngineContext* TRTEngineOp::GetEngine(
 }
 
 tensorflow::Status TRTEngineOp::AllocateCalibrationResources(
-    OpKernelContext* ctx, TRTCalibrationResource** cr) {
+    OpKernelContext* ctx, SerializableResourceBase** cr) {
   auto cres = new TRTCalibrationResource();
   *cr = cres;
   // Get the allocator.
diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
index 7f0f05aa0a..76c6020078 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
+++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
@@ -23,6 +23,7 @@ limitations under the License.
 #include "tensorflow/contrib/tensorrt/log/trt_logger.h"
 #include "tensorflow/contrib/tensorrt/resources/trt_allocator.h"
 #include "tensorflow/contrib/tensorrt/resources/trt_lru_cache.h"
+#include "tensorflow/contrib/tensorrt/resources/trt_resources.h"
 #include "tensorflow/core/framework/function.h"
 #include "tensorflow/core/framework/graph.pb.h"
 #include "tensorflow/core/framework/op.h"
@@ -38,7 +39,6 @@ limitations under the License.
 namespace tensorflow {
 namespace tensorrt {
 struct TRTInt8Calibrator;
-class TRTCalibrationResource;
 class AsyncHelper;
 //  TODO(Sami): Remove this file?
 
@@ -69,7 +69,7 @@ class TRTEngineOp : public AsyncOpKernel {
 
   // Allocate necessary resources for calibration
   Status AllocateCalibrationResources(OpKernelContext* ctx,
-                                      TRTCalibrationResource** cr);
+                                      SerializableResourceBase** cr);
 
   // Get engine for the input shape
   EngineContext* GetEngine(const std::vector<TensorShape>& input_shapes,
-- 
GitLab


From fe77e94dbf3d5633be38aacee5f90e1c1302506e Mon Sep 17 00:00:00 2001
From: Scott Zhu <scottzhu@google.com>
Date: Fri, 18 Jan 2019 16:25:12 -0800
Subject: [PATCH 1020/1095] Add unit test and fix bugs in AttentionMechanismV2.

PiperOrigin-RevId: 230013805
---
 tensorflow/contrib/seq2seq/BUILD              | 15 +++
 .../kernel_tests/attention_wrapper_v2_test.py | 94 +++++++++++++++++++
 .../seq2seq/python/ops/attention_wrapper.py   | 71 +++++++++-----
 3 files changed, 159 insertions(+), 21 deletions(-)
 create mode 100644 tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_v2_test.py

diff --git a/tensorflow/contrib/seq2seq/BUILD b/tensorflow/contrib/seq2seq/BUILD
index 89176180ae..7d5ba90ded 100644
--- a/tensorflow/contrib/seq2seq/BUILD
+++ b/tensorflow/contrib/seq2seq/BUILD
@@ -213,3 +213,18 @@ cuda_py_test(
         "//tensorflow/python:variables",
     ],
 )
+
+cuda_py_test(
+    name = "attention_wrapper_v2_test",
+    size = "medium",
+    srcs = ["python/kernel_tests/attention_wrapper_v2_test.py"],
+    additional_deps = [
+        ":seq2seq_py",
+        "@absl_py//absl/testing:parameterized",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:init_ops",
+        "//tensorflow/python:platform_test",
+        "//tensorflow/python:variables",
+    ],
+)
diff --git a/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_v2_test.py b/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_v2_test.py
new file mode 100644
index 0000000000..7ff04e1780
--- /dev/null
+++ b/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_v2_test.py
@@ -0,0 +1,94 @@
+# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for contrib.seq2seq.python.ops.attention_wrapper."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from absl.testing import parameterized
+import numpy as np
+
+from tensorflow.contrib.seq2seq.python.ops import attention_wrapper as wrapper
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import test
+
+
+@test_util.run_all_in_graph_and_eager_modes
+class AttentionMechanismTest(test.TestCase, parameterized.TestCase):
+
+  def setUp(self):
+    super(AttentionMechanismTest, self).setUp()
+    self.batch = 10
+    self.timestep = 5
+    self.memory_size = 6
+    self.units = 8
+
+    self.memory = ops.convert_to_tensor(
+        np.random.random((self.batch, self.timestep, self.memory_size)),
+        dtype=np.float32)
+    self.query = ops.convert_to_tensor(
+        np.random.random((self.batch, self.units)), dtype=np.float32)
+    self.state = ops.convert_to_tensor(
+        np.random.random((self.batch, self.timestep)), dtype=np.float32)
+
+  @parameterized.named_parameters(
+      ("luong", wrapper.LuongAttentionV2),
+      ("luong_monotonic", wrapper.LuongMonotonicAttentionV2),
+      ("bahdanau", wrapper.BahdanauAttentionV2),
+      ("bahdanau_monotonic", wrapper.BahdanauMonotonicAttentionV2),
+  )
+  def test_attention_shape_inference(self, attention_cls):
+    attention = attention_cls(self.units)
+    attention_score = attention([self.query, self.state, self.memory])
+    self.assertLen(attention_score, 2)
+    self.assertEqual(attention_score[0].shape, (self.batch, self.timestep))
+    self.assertEqual(attention_score[1].shape, (self.batch, self.timestep))
+
+  @parameterized.named_parameters(
+      ("luong", wrapper.LuongAttentionV2),
+      ("luong_monotonic", wrapper.LuongMonotonicAttentionV2),
+      ("bahdanau", wrapper.BahdanauAttentionV2),
+      ("bahdanau_monotonic", wrapper.BahdanauMonotonicAttentionV2),
+  )
+  def test_get_config(self, attention_cls):
+    attention = attention_cls(self.units)
+    config = attention.get_config()
+
+    attention_from_config = attention_cls.from_config(config)
+    config_from_clone = attention_from_config.get_config()
+
+    self.assertDictEqual(config, config_from_clone)
+
+  @parameterized.named_parameters(
+      ("luong", wrapper.LuongAttentionV2),
+      ("luong_monotonic", wrapper.LuongMonotonicAttentionV2),
+      ("bahdanau", wrapper.BahdanauAttentionV2),
+      ("bahdanau_monotonic", wrapper.BahdanauMonotonicAttentionV2),
+  )
+  def test_layer_output(self, attention_cls):
+    attention = attention_cls(self.units)
+
+    score = attention([self.query, self.state, self.memory])
+    self.evaluate(variables.variables_initializer(attention.variables))
+
+    score_val = self.evaluate(score)
+    self.assertLen(score_val, 2)
+    self.assertEqual(score_val[0].shape, (self.batch, self.timestep))
+    self.assertEqual(score_val[1].shape, (self.batch, self.timestep))
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py b/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
index 31c62d5849..ae3e7f1b5d 100644
--- a/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
+++ b/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
@@ -28,7 +28,7 @@ from tensorflow.contrib.framework.python.framework import tensor_util
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
-from tensorflow.python.keras.engine.base_layer import Layer
+from tensorflow.python.keras import layers
 from tensorflow.python.layers import base as layers_base
 from tensorflow.python.layers import core as layers_core
 from tensorflow.python.ops import array_ops
@@ -218,7 +218,7 @@ class _BaseAttentionMechanism(AttentionMechanism):
     return self.initial_alignments(batch_size, dtype)
 
 
-class _BaseAttentionMechanismV2(AttentionMechanism, Layer):
+class _BaseAttentionMechanismV2(AttentionMechanism, layers.Layer):
   """A base AttentionMechanism class providing common functionality.
 
   Common functionality includes:
@@ -251,11 +251,11 @@ class _BaseAttentionMechanismV2(AttentionMechanism, Layer):
         creation.
     """
     if (query_layer is not None
-        and not isinstance(query_layer, layers_base.Layer)):
+        and not isinstance(query_layer, layers.Layer)):
       raise TypeError(
           "query_layer is not a Layer: %s" % type(query_layer).__name__)
     if (memory_layer is not None
-        and not isinstance(memory_layer, layers_base.Layer)):
+        and not isinstance(memory_layer, layers.Layer)):
       raise TypeError(
           "memory_layer is not a Layer: %s" % type(memory_layer).__name__)
     self.query_layer = query_layer
@@ -275,10 +275,12 @@ class _BaseAttentionMechanismV2(AttentionMechanism, Layer):
     self._check_inner_dims_defined = True
 
   def build(self, input_shape):
+    # The layer suppose to take 3 inputs, [query, state, memory].
+    query_input_shape, _, memory_input_shape = input_shape
     if self.query_layer is not None:
-      self.query_layer.build(input_shape)
+      self.query_layer.build(query_input_shape)
     if self.memory_layer is not None:
-      self.memory_layer.build(input_shape)
+      self.memory_layer.build(memory_input_shape)
     # dtype of the layer is known at this moment, create the score_mask_value if
     # needed.
     self.score_mask_value = dtypes.as_dtype(self.dtype).as_numpy_dtype(-np.inf)
@@ -310,8 +312,8 @@ class _BaseAttentionMechanismV2(AttentionMechanism, Layer):
       self.batch_size = (
           tensor_shape.dimension_value(self.keys.shape[0]) or
           array_ops.shape(self.keys)[0])
-      self.alignments_size = (tensor_shape.dimension_value(self.keys.shape[1])
-                              or array_ops.shape(self.keys)[1])
+      self._alignments_size = (tensor_shape.dimension_value(self.keys.shape[1])
+                               or array_ops.shape(self.keys)[1])
       if memory_mask is not None:
         self.probability_fn = lambda score, prev: (  # pylint:disable=g-long-lambda
             self.probability_fn(_maybe_mask_score(
@@ -398,6 +400,8 @@ class _BaseAttentionMechanismV2(AttentionMechanism, Layer):
     """
     # Reconstruct the query and memory layer for parent class.
     from tensorflow.python.keras.layers import deserialize as deserialize_layer  # pylint: disable=g-import-not-at-top
+    # Instead of updating the input, create a copy and use that.
+    config = config.copy()
     query_layer_config = config.pop("query_layer", None)
     if query_layer_config:
       query_layer = deserialize_layer(query_layer_config,
@@ -410,6 +414,10 @@ class _BaseAttentionMechanismV2(AttentionMechanism, Layer):
       config["memory_layer"] = memory_layer
     return config
 
+  @property
+  def alignments_size(self):
+    return self._alignments_size
+
 
 def _luong_score(query, keys, scale):
   """Implements Luong-style (multiplicative) scoring function.
@@ -606,10 +614,13 @@ class LuongAttentionV2(_BaseAttentionMechanismV2):
     wrapped_probability_fn = lambda score, _: probability_fn(score)
     if dtype is None:
       dtype = dtypes.float32
+    memory_layer = kwargs.pop("memory_layer", None)
+    if not memory_layer:
+      memory_layer = layers.Dense(
+          units, name="memory_layer", use_bias=False, dtype=dtype)
     super(LuongAttentionV2, self).__init__(
         query_layer=None,
-        memory_layer=layers_core.Dense(
-            units, name="memory_layer", use_bias=False, dtype=dtype),
+        memory_layer=memory_layer,
         probability_fn=wrapped_probability_fn,
         name=name,
         dtype=dtype,
@@ -640,6 +651,7 @@ class LuongAttentionV2(_BaseAttentionMechanismV2):
       alignments: Tensor of dtype matching `self.values` and shape
         `[batch_size, alignments_size]` (`alignments_size` is memory's
         `max_time`).
+      next_state: Same as the alignments.
     """
     score = _luong_score(query, self.keys, self.scale_weight)
     alignments = self.probability_fn(score, state)
@@ -865,11 +877,17 @@ class BahdanauAttentionV2(_BaseAttentionMechanismV2):
     wrapped_probability_fn = lambda score, _: probability_fn(score)
     if dtype is None:
       dtype = dtypes.float32
+    query_layer = kwargs.pop("query_layer", None)
+    if not query_layer:
+      query_layer = layers.Dense(
+          units, name="query_layer", use_bias=False, dtype=dtype)
+    memory_layer = kwargs.pop("memory_layer", None)
+    if not memory_layer:
+      memory_layer = layers.Dense(
+          units, name="memory_layer", use_bias=False, dtype=dtype)
     super(BahdanauAttentionV2, self).__init__(
-        query_layer=layers_core.Dense(
-            units, name="query_layer", use_bias=False, dtype=dtype),
-        memory_layer=layers_core.Dense(
-            units, name="memory_layer", use_bias=False, dtype=dtype),
+        query_layer=query_layer,
+        memory_layer=memory_layer,
         probability_fn=wrapped_probability_fn,
         name=name,
         dtype=dtype,
@@ -907,6 +925,7 @@ class BahdanauAttentionV2(_BaseAttentionMechanismV2):
       alignments: Tensor of dtype matching `self.values` and shape
         `[batch_size, alignments_size]` (`alignments_size` is memory's
         `max_time`).
+      next_state: same as alignments.
     """
     processed_query = self.query_layer(query) if self.query_layer else query
     score = _bahdanau_score(processed_query, self.keys, self.attention_v,
@@ -1314,11 +1333,17 @@ class BahdanauMonotonicAttentionV2(_BaseMonotonicAttentionMechanismV2):
     wrapped_probability_fn = functools.partial(
         _monotonic_probability_fn, sigmoid_noise=sigmoid_noise, mode=mode,
         seed=sigmoid_noise_seed)
+    query_layer = kwargs.pop("query_layer", None)
+    if not query_layer:
+      query_layer = layers.Dense(
+          units, name="query_layer", use_bias=False, dtype=dtype)
+    memory_layer = kwargs.pop("memory_layer", None)
+    if not memory_layer:
+      memory_layer = layers.Dense(
+          units, name="memory_layer", use_bias=False, dtype=dtype)
     super(BahdanauMonotonicAttentionV2, self).__init__(
-        query_layer=layers_core.Dense(
-            units, name="query_layer", use_bias=False, dtype=dtype),
-        memory_layer=layers_core.Dense(
-            units, name="memory_layer", use_bias=False, dtype=dtype),
+        query_layer=query_layer,
+        memory_layer=memory_layer,
         probability_fn=wrapped_probability_fn,
         name=name,
         dtype=dtype,
@@ -1551,10 +1576,13 @@ class LuongMonotonicAttentionV2(_BaseMonotonicAttentionMechanismV2):
     wrapped_probability_fn = functools.partial(
         _monotonic_probability_fn, sigmoid_noise=sigmoid_noise, mode=mode,
         seed=sigmoid_noise_seed)
+    memory_layer = kwargs.pop("memory_layer", None)
+    if not memory_layer:
+      memory_layer = layers.Dense(
+          units, name="memory_layer", use_bias=False, dtype=dtype)
     super(LuongMonotonicAttentionV2, self).__init__(
         query_layer=None,
-        memory_layer=layers_core.Dense(
-            units, name="memory_layer", use_bias=False, dtype=dtype),
+        memory_layer=memory_layer,
         probability_fn=wrapped_probability_fn,
         name=name,
         dtype=dtype,
@@ -1593,6 +1621,7 @@ class LuongMonotonicAttentionV2(_BaseMonotonicAttentionMechanismV2):
       alignments: Tensor of dtype matching `self.values` and shape
         `[batch_size, alignments_size]` (`alignments_size` is memory's
         `max_time`).
+      next_state: Same as alignments
     """
     score = _luong_score(query, self.keys, self.attention_g)
     score += self.attention_score_bias
@@ -1603,7 +1632,7 @@ class LuongMonotonicAttentionV2(_BaseMonotonicAttentionMechanismV2):
   def get_config(self):
     config = {
         "units": self.units,
-        "normalize": self.normalize,
+        "scale": self.scale,
         "sigmoid_noise": self.sigmoid_noise,
         "sigmoid_noise_seed": self.sigmoid_noise_seed,
         "score_bias_init": self.score_bias_init,
-- 
GitLab


From c2e7173f8f61c74529cfd879becb762b48db37ad Mon Sep 17 00:00:00 2001
From: Pete Warden <petewarden@google.com>
Date: Fri, 18 Jan 2019 16:38:54 -0800
Subject: [PATCH 1021/1095] Updated Micro documentation on porting to a new
 platform

PiperOrigin-RevId: 230015624
---
 tensorflow/lite/experimental/micro/README.md | 175 ++++++++++++++++++-
 1 file changed, 173 insertions(+), 2 deletions(-)

diff --git a/tensorflow/lite/experimental/micro/README.md b/tensorflow/lite/experimental/micro/README.md
index b47e0feb79..88f3cd71f5 100644
--- a/tensorflow/lite/experimental/micro/README.md
+++ b/tensorflow/lite/experimental/micro/README.md
@@ -537,7 +537,9 @@ critical parts of the code with versions specifically tailored to your
 architecture. The framework has been designed with this in mind, and we hope the
 combination of small modules and many tests makes it as straightforward as
 possible to swap in your own code a piece at a time, ensuring you have a working
-version at every step.
+version at every step. To write specialized implementations for a platform, it's
+useful to understand how optional components are handled inside the build
+system.
 
 ### Code Module Organization
 
@@ -628,7 +630,7 @@ kernel implementations, but with some specific conventions:
     latest in the ordered list will be chosen. This allows us to express “I’d
     like generically-optimized fixed point if it’s available, but I’d prefer
     something using the CMSIS library” using the list 'fixed_point cmsis'. These
-    tags are passed in as `TAGS="<foo"` on the command line when you use the
+    tags are passed in as `TAGS="<foo>"` on the command line when you use the
     main Makefile to build.
 -   There is an implicit “reference” tag at the start of every list, so that
     it’s possible to support directory structures like the current
@@ -640,3 +642,172 @@ kernel implementations, but with some specific conventions:
     top level.
 -   Tests should be at the parent level, with no platform-specific code.
 -   No platform-specific macros or #ifdef’s should be used in any portable code.
+
+The implementation of these rules is handled inside the Makefile, with a
+[`specialize` function](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/experimental/micro/tools/make/helper_functions.inc#L42)
+that takes a list of reference source file paths as an input, and returns the
+equivalent list with specialized versions of those files swapped in if they
+exist.
+
+### Working with Generated Projects
+
+So far, I've recommended that you use the standalone generated projects for your
+system. You might be wondering why you're not just checking out the full
+[TensorFlow codebase from Github](https://github.com/tensorflow/tensorflow/)?
+The main reason is that there is a lot more diversity of architectures, IDEs,
+support libraries, and operating systems in the embedded world. Many of the
+toolchains require their own copy of source files, or a list of sources to be
+written to a project file. When a developer working on TensorFlow adds a new
+source file or changes its location, we can't expect her to update multiple
+different project files, many of which she may not have the right software to
+verify the change was correct. That means we have to rely on a central listing
+of source files (which in our case is held in the makefile), and then call a
+tool to generate other project files from those. We could ask embedded
+developers to do this process themselves after downloading the main source, but
+running the makefile requires a Linux system which may not be available, takes
+time, and involves downloading a lot of dependencies. That is why we've opted to
+make regular snapshots of the results of generating these projects for popular
+IDEs and platforms, so that embedded developers have a fast and friendly way to
+start using TensorFlow Lite for Microcontrollers.
+
+This does have the disadvantage that you're no longer working directly on the
+main repository, instead you have a copy that's outside of source control. We've
+tried to make the copy as similar to the main repo as possible, for example by
+keeping the paths of all source files the same, and ensuring that there are no
+changes between the copied files and the originals, but it still makes it
+tougher to sync as the main repository is updated. There are also multiple
+copies of the source tree, one for each target, so any change you make to one
+copy has to be manually propagated across all the other projects you care about.
+This doesn't matter so much if you're just using the projects as they are to
+build products, but if you want to support a new platform and have the changes
+reflected in the main code base, you'll have to do some extra work.
+
+As an example, think about the `DebugLog()` implementation we discussed adding
+for a new platform earlier. At this point, you have a new version of
+`debug_log.cc` that does what's required, but how can you share that with the
+wider community? The first step is to pick a tag name for your platform. This
+can either be the operating system (for example 'mbed'), the name of a device
+('bluepill'), or some other text that describes it. This should be a short
+string with no spaces or special characters. Log in or create an account on
+Github, fork the full
+[TensorFlow codebase](https://github.com/tensorflow/tensorflow/) using the
+'Fork' button on the top left, and then grab your fork by using a command like
+`git clone https://github.com/<your user name>/tensorflow`.
+
+You'll either need Linux, MacOS, or Windows with something like CygWin installed
+to run the next steps, since they involve building a makefile. Run the following
+commands from a terminal, inside the root of the source folder:
+
+```
+tensorflow/lite/experimental/micro/tools/make/download_dependencies.sh
+make -f tensorflow/lite/experimental/micro/tools/make/Makefile generate_projects
+```
+
+This will take a few minutes, since it has to download some large toolchains for
+the dependencies. Once it has finished, you should see some folders created
+inside a path like
+`tensorflow/lite/experimental/micro/tools/make/gen/linux_x86_64/prj/`. The exact
+path depends on your host operating system, but you should be able to figure it
+out from all the copy commands. These folders contain the generated project and
+source files, with
+`tensorflow/lite/experimental/micro/tools/make/gen/linux_x86_64/prj/keil`
+containing the Keil uVision targets,
+`tensorflow/lite/experimental/micro/tools/make/gen/linux_x86_64/prj/mbed` with
+the Mbed versions, and so on.
+
+If you've got this far, you've successfully set up the project generation flow.
+Now you need to add your specialized implementation of `DebugLog()`. Start by
+creating a folder inside `tensorflow/lite/experimental/micro/` named after the
+tag you picked earlier. Put your `debug_log.cc` file inside this folder, and
+then run this command, with '<your tag>' replaced by the actual folder name:
+
+```
+make -f tensorflow/lite/experimental/micro/tools/make/Makefile TAGS="<your tag>" generate_projects
+```
+
+If your tag name actually refers to a whole target architecture, then you'll use
+TARGET or TARGET_ARCH instead. For example, here's how a simple RISC-V set of
+projects is generated:
+
+```
+make -f tensorflow/lite/experimental/micro/tools/make/Makefile TARGET="riscv32_mcu" generate_projects
+```
+
+The way it works is the same as TAGS though, it just looks for specialized
+implementations with the same containing folder name.
+
+If you look inside the projects that have been created, you should see that the
+default `DebugLog()` implementation is no longer present at
+`tensorflow/lite/experimental/micro/debug_log.cc`, and instead
+`tensorflow/lite/experimental/micro/<your tag>/debug_log.cc` is being used. Copy
+over the generated project files and try building them in your own IDE. If
+everything works, then you're ready to submit your change.
+
+To do this, run something like:
+
+```
+git add tensorflow/lite/experimental/micro/<your tag>/debug_log.cc
+git commit -a -m "Added DebugLog() support for <your platform>"
+git push origin master
+```
+
+Then go back to https://github.com/<your account>/tensorflow, and choose "New
+Pull Request" near the top. You should then be able to go through the standard
+TensorFlow PR process to get your change added to the main repository, and
+available to the rest of the community!
+
+### Supporting a Platform with Makefiles
+
+The changes you've made so far will enable other developers using the generated
+projects to use your platform, but TensorFlow's continuous integration process
+uses makefiles to build frequently and ensure changes haven't broken the build
+process for different systems. If you are able to convert your build procedure
+into something that can be expressed by a makefile, then we can integrate your
+platform into our CI builds and make sure it continues to work.
+
+Fully describing how to do this is beyond the scope of this documentation, but
+the biggest needs are:
+
+-   A command-line compiler that can be called for every source file.
+-   A list of the arguments to pass into the compiler to build and link all
+    files.
+-   The correct linker map files and startup assembler to ensure `main()` gets
+    called.
+
+### Supporting a Platform with Emulation Testing
+
+Integrating your platform into the makefile process should help us make sure
+that it continues to build, but it doesn't guarantee that the results of the
+build process will run correctly. Running tests is something we require to be
+able to say that TensorFlow officially supports a platform, since otherwise we
+can't guarantee that users will have a good experience when they try using it.
+Since physically maintaining a full set of all supported hardware devices isn't
+feasible, we rely on software emulation to run these tests. A good example is
+our
+[STM32F4 'Bluepill' support](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/experimental/micro/testing/test_bluepill_binary.sh),
+which uses [Docker](https://www.docker.com/) and [Renode](https://renode.io/) to
+run built binaries in an emulator. You can use whatever technologies you want,
+the only requirements are that they capture the debug log output of the tests
+being run in the emulator, and parse them for the string that indicates the test
+was successful. These scripts need to run on Ubuntu 18.04, in a bash
+environment, though Docker is available if you need to install extra software or
+have other dependencies.
+
+### Optimizing for your Platform
+
+Clearly, getting debug logging support is only the beginning of the work you'll
+need to do on a particular platform. It's very likely that you'll want to
+optimize the core deep learning operations that take up the most time when
+running models you care about. The good news is that the process for providing
+optimized implementations is the same as the one you just went through to
+provide your own logging. You'll need to identify parts of the code that are
+bottlenecks, and then add specialized implementations in their own folders.
+These don't need to be platform specific, they can also be broken out by which
+library they rely on for example. [Here's where we do that for the CMSIS
+implementation of integer fast-fourier
+transforms](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/preprocessor.cc).
+This more complex case shows that you can also add helper source files alongside
+the main implementation, as long as you
+[mention them in the platform-specific makefile](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/experimental/micro/examples/micro_speech/CMSIS/Makefile.inc).
+You can also do things like update the list of libraries that need to be linked
+in, or add include paths to required headers.
-- 
GitLab


From a6924e6affd935f537cdaf8977094df0e15a7957 Mon Sep 17 00:00:00 2001
From: Mingsheng Hong <hongm@google.com>
Date: Fri, 18 Jan 2019 17:00:33 -0800
Subject: [PATCH 1022/1095] Added tracing support to the eager C++ layer, where
 we can create a graph by symbolically executing a sequence of eager ops. This
 involves the following changes:

1. Extended the eager C++ layer with symbolic tensor support, where a
tensorflow::TensorHandle represents a graph node, whose execution yields the
"concrete" tensor value. In contrast, the existing tensor handles are conrete
tensors.

Symbolic tensors can be created from a graph node (represented as TF_Output) via
TFE_NewTensorHandleFromTFOutput(). The associated graph node can be retrieved
from a symbolic tensor via TFE_GetTFOutputFromTensorHandle().

2. Added another experimental C API TFE_AddEagerOpToGraph() for symbolic
execution of a TFE_Op, which converts the op to a graph node, and the graph is
associated with a newly added TFE_TraceContext object to be passed into the C
API call.

3. If an eager op OP takes any concrete tensors as input,
TFE_AddEagerOpToGraph() will create symbolic tensors for them, to serve as input
to the converted graph node from OP. When executing that graph, those concrete
tensors will be fed as input.

Here's an example tracing use case illustrated by Swift code:
let x = Tensor<Float>(1.0)
let y = x + x

The first statement is executed on the host, yielding a concrete tensor value
for x.

The second statement is first symbolically executed to create a
graph (containing the "Add" node). Next, the graph is executed to compute y.

During this symbolic execution, TFE_AddEagerOpToGraph() is called on the "Add"
eager op for x + x, to insert an "Add" graph node. The node takes as input a
symbolic tensor (a PlaceHolder graph node) corresponding to x. When executing
the graph, the concrete tensor of x is then fed as the input.

TFE_AddEagerOpToGraph() does not yet support TF op attributes.

An example call-site code of this feature is Swift for TensorFlow at
https://github.com/apple/swift/pull/21589 (where some of the above C APIs are
subject to name changes).

PiperOrigin-RevId: 230018312
---
 tensorflow/c/BUILD                            |   2 +
 tensorflow/c/c_api_experimental.cc            | 165 ++++++++++++++++++
 tensorflow/c/c_api_experimental.h             |  48 +++++
 tensorflow/c/c_api_experimental_test.cc       |  51 +++++-
 tensorflow/c/c_api_internal.h                 |   2 +
 tensorflow/c/eager/c_api.cc                   |   2 +
 tensorflow/c/eager/c_api_internal.h           |  26 +++
 tensorflow/core/common_runtime/eager/BUILD    |   1 +
 .../common_runtime/eager/tensor_handle.cc     |  19 ++
 .../core/common_runtime/eager/tensor_handle.h |  21 ++-
 10 files changed, 335 insertions(+), 2 deletions(-)

diff --git a/tensorflow/c/BUILD b/tensorflow/c/BUILD
index 4e2fe34d28..a48778d612 100644
--- a/tensorflow/c/BUILD
+++ b/tensorflow/c/BUILD
@@ -129,6 +129,7 @@ tf_cuda_library(
         "//tensorflow/core:lib_platform",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core/common_runtime/eager:attr_builder",
+        "@com_google_absl//absl/strings",
     ],
 )
 
@@ -318,6 +319,7 @@ tf_cc_test(
     deps = [
         ":c_api",
         ":c_api_experimental",
+        ":c_api_internal",
         ":c_test_util",
         "//tensorflow/c/eager:c_api",
         "//tensorflow/c/eager:c_api_test_util",
diff --git a/tensorflow/c/c_api_experimental.cc b/tensorflow/c/c_api_experimental.cc
index 92f7da7de0..6cc74cfb32 100644
--- a/tensorflow/c/c_api_experimental.cc
+++ b/tensorflow/c/c_api_experimental.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/c/c_api_experimental.h"
 
+#include "absl/strings/substitute.h"
 #include "tensorflow/c/c_api.h"
 #include "tensorflow/c/c_api_internal.h"
 #include "tensorflow/c/eager/c_api.h"
@@ -8745,6 +8746,12 @@ static void CheckOk(TF_Status* status) {
 
 void TFE_TensorHandlePrintDebugString(TFE_TensorHandle* handle) {
   auto* status = TF_NewStatus();
+  if (!TFE_TensorHandleIsConcrete(handle)) {
+    VLOG(1) << "Symbolic tensor: " << handle;
+    TF_DeleteStatus(status);
+    return;
+  }
+
   TF_Tensor* t = TFE_TensorHandleResolve(handle, status);
   CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
 
@@ -8954,3 +8961,161 @@ TF_CAPI_EXPORT extern void TFE_EnableCollectiveOps(TFE_Context* ctx,
   }
   status->status = EnableCollectiveOps(server_def, ctx);
 }
+
+std::string tensorflow::getTF_OutputDebugString(TF_Output node) {
+  return absl::Substitute("TF_Output($0, $1)", node.oper, node.index);
+}
+
+using tensorflow::getTF_OutputDebugString;
+
+TFE_TensorHandle* TFE_NewTensorHandleFromTFOutput(TF_Output t,
+                                                  TF_DataType dtype) {
+  auto ret = new TFE_TensorHandle(t, dtype);
+  VLOG(1) << "Storing TFOutput " << getTF_OutputDebugString(t)
+          << " into tensor handle " << ret << " with internal handle "
+          << ret->handle;
+  return ret;
+}
+
+unsigned char TFE_TensorHandleIsConcrete(TFE_TensorHandle* handle) {
+  assert(handle->handle != nullptr);
+  return handle->handle->getSymbolicTensor() == nullptr;
+}
+
+TF_Output TFE_GetTFOutputFromTensorHandle(TFE_TensorHandle* handle,
+                                          TF_Status* status) {
+  if (TFE_TensorHandleIsConcrete(handle)) {
+    status->status =
+        tensorflow::errors::Internal("Not a symbolic tensor: ", handle);
+    return TF_Output{nullptr, -1};
+  }
+
+  auto* sym_tensor = handle->handle->getSymbolicTensor();
+  CHECK(sym_tensor != nullptr);
+  auto ret = TF_Output{sym_tensor->oper, sym_tensor->index};
+  VLOG(1) << "Retrieving " << getTF_OutputDebugString(ret)
+          << " from tensor handle " << handle;
+  CHECK_GE(sym_tensor->index, 0);
+  return ret;
+}
+
+TFE_TraceContext* TFE_NewTraceContext(TF_Graph* graph) {
+  return new TFE_TraceContext(graph);
+}
+
+void TFE_DeleteTraceContext(TFE_TraceContext* trace_ctx) { delete trace_ctx; }
+
+// If `handle` is already symbolic, return it. Otherwise map it to a new
+// symbolic tensor (a PlaceHolder op) and return that.
+static TF_Output getOrCreateSymbolicTensor(TFE_TraceContext* trace_ctx,
+                                           tensorflow::TensorHandle* handle,
+                                           TF_Status* status) {
+  VLOG(1) << "Getting symbolic tensor for input tensor handle " << handle
+          << ": " << handle->DebugString();
+
+  auto* sym_tensor = handle->getSymbolicTensor();
+  if (sym_tensor != nullptr) {
+    auto ret = TF_Output{sym_tensor->oper, sym_tensor->index};
+    VLOG(1) << "This handle is a symbolic tensor " << sym_tensor << ": "
+            << getTF_OutputDebugString(ret);
+    return ret;
+  }
+
+  auto find_it = trace_ctx->input_tensor_map.find(handle);
+  if (find_it != trace_ctx->input_tensor_map.end()) {
+    VLOG(1) << "There exists a map entry from this concrete tensor to: "
+            << getTF_OutputDebugString(find_it->second);
+    return find_it->second;
+  }
+
+  auto node_name = tensorflow::strings::StrCat("additional_input_",
+                                               trace_ctx->node_counter++);
+  VLOG(1) << "Adding a place holder node named " << node_name;
+  auto* desc =
+      TF_NewOperation(trace_ctx->graph, "Placeholder", node_name.c_str());
+  TF_SetAttrType(desc, "dtype",
+                 static_cast<TF_DataType>(handle->dtype) /*TF_FLOAT*/);
+  auto* result = TF_FinishOperation(desc, status);
+  if (!status->status.ok()) {
+    return TF_Output{nullptr, -1};
+  }
+
+  auto ret = TF_Output{result, 0};
+  VLOG(1) << "Creating a new map entry to map to: "
+          << getTF_OutputDebugString(ret);
+  trace_ctx->input_tensor_map[handle] = ret;
+  // `handle` could be destroyed before it's read from `input_tensor_map` (say
+  // during a subsequent TFE_FinalizeInputTensorsFromTraceContext() call), so we
+  // increment its ref count to extend its life span to that of `trace_ctx`.
+  handle->Ref();
+  VLOG(1) << "Ref count for handle " << handle
+          << " is 1?: " << handle->RefCountIsOne();
+  return ret;
+}
+
+void TFE_AddEagerOpToGraph(TFE_Op* op, TFE_TraceContext* trace_ctx,
+                           TFE_TensorHandle** retvals, int* num_retvals,
+                           TF_Status* status) {
+  VLOG(1) << "Calling TFE_AddEagerOpToGraph() with op " << op << ": "
+          << op->operation.DebugString();
+
+  const auto& op_type = op->operation.Name();
+  auto op_name =
+      tensorflow::strings::StrCat(op_type, "_", trace_ctx->node_counter++);
+  auto* desc =
+      TF_NewOperation(trace_ctx->graph, op_type.c_str(), op_name.c_str());
+  for (auto* input : op->operation.Inputs()) {
+    auto symbolic_input = getOrCreateSymbolicTensor(trace_ctx, input, status);
+    if (!status->status.ok()) return;
+    TF_AddInput(desc, symbolic_input);
+  }
+
+  VLOG(1) << "Adding attrs.";
+  // TODO(hongm): add attrs
+
+  auto* graph_op = TF_FinishOperation(desc, status);
+  if (!status->status.ok()) return;
+
+  VLOG(1) << "Op finalized; setting return tensors.";
+  *num_retvals = TF_OperationNumOutputs(graph_op);
+  VLOG(1) << "This op has " << *num_retvals << " outputs.";
+  for (int i = 0; i < *num_retvals; ++i) {
+    auto output = TF_Output{graph_op, i};
+    auto dtype = TF_OperationOutputType(output);
+    retvals[i] = TFE_NewTensorHandleFromTFOutput(output, dtype);
+  }
+}
+
+int TFE_FinalizeInputTensorsFromTraceContext(TFE_TraceContext* trace_ctx) {
+  if (trace_ctx->input_tensors == nullptr) {
+    trace_ctx->input_tensors =
+        new std::vector<std::pair<tensorflow::TensorHandle*, TF_Output>>();
+    trace_ctx->input_tensors->reserve(trace_ctx->input_tensor_map.size());
+
+    for (auto input : trace_ctx->input_tensor_map) {
+      trace_ctx->input_tensors->emplace_back(input.first, input.second);
+    }
+  }
+  return trace_ctx->input_tensor_map.size();
+}
+
+TF_Output TFE_GetInputGraphNodeFromTraceContext(TFE_TraceContext* trace_ctx,
+                                                unsigned int idx) {
+  CHECK(trace_ctx->input_tensors != nullptr);
+  CHECK(trace_ctx->input_tensors->size() > idx);
+  return trace_ctx->input_tensors->at(idx).second;
+}
+
+TFE_TensorHandle* TFE_ConsumeInputConcreteTensorFromTraceContext(
+    TFE_TraceContext* trace_ctx, unsigned int idx) {
+  CHECK(trace_ctx->input_tensors != nullptr);
+  CHECK(trace_ctx->input_tensors->size() > idx);
+  auto* handle = trace_ctx->input_tensors->at(idx).first;
+  VLOG(1) << "Ref count for internal handle " << handle
+          << " is 1?: " << handle->RefCountIsOne();
+  handle->Ref();
+  auto* ret = new TFE_TensorHandle(handle);
+  VLOG(1) << "Returning a new tensor handle " << ret << ": "
+          << handle->DebugString();
+  return ret;
+}
diff --git a/tensorflow/c/c_api_experimental.h b/tensorflow/c/c_api_experimental.h
index e8da4099b8..48ea0ec1ed 100644
--- a/tensorflow/c/c_api_experimental.h
+++ b/tensorflow/c/c_api_experimental.h
@@ -267,6 +267,54 @@ TF_CAPI_EXPORT extern void TFE_EnableCollectiveOps(TFE_Context* ctx,
                                                    size_t proto_len,
                                                    TF_Status* status);
 
+// Create a symbolic tensor from the input graph node.
+TF_CAPI_EXPORT extern TFE_TensorHandle* TFE_NewTensorHandleFromTFOutput(
+    TF_Output t, TF_DataType data_type);
+
+// Returns 0 if the input tensor handle represents a symbolic tensor (i.e., a
+// graph node). Otherwise returns non-0.
+TF_CAPI_EXPORT extern unsigned char TFE_TensorHandleIsConcrete(
+    TFE_TensorHandle* handle);
+
+// If `handle` is a symbolic tensor, return the corresponding graph node
+// represented by TF_Output. Otherwise, return an error status.
+TF_CAPI_EXPORT extern TF_Output TFE_GetTFOutputFromTensorHandle(
+    TFE_TensorHandle* handle, TF_Status* status);
+
+typedef struct TFE_TraceContext TFE_TraceContext;
+
+// A trace context contains a trace graph, to which TFE_AddEagerOpToGraph()
+// calls add graph nodes as a way to symbolically execute the eager ops.
+//
+// It also contains a hash map from concrete input tensors to symbolic
+// tensors. That map will be used to create input tensors to the trace graph.
+TF_CAPI_EXPORT extern TFE_TraceContext* TFE_NewTraceContext(TF_Graph* graph);
+
+TF_CAPI_EXPORT extern void TFE_DeleteTraceContext(TFE_TraceContext* trace_ctx);
+
+// Symbolically executes `op`, by adding a corresponding node to the graph
+// associated with `trace_ctx`. This graph node outputs a set of symbolic
+// tensors in `retvals` and `num_retvals`.
+TF_CAPI_EXPORT extern void TFE_AddEagerOpToGraph(TFE_Op* op,
+                                                 TFE_TraceContext* trace_ctx,
+                                                 TFE_TensorHandle** retvals,
+                                                 int* num_retvals,
+                                                 TF_Status* status);
+
+// Finalizes the trace graph and its inputs, and returns the number of inputs.
+// After this call, the next two APIs can be called to iterate over the input
+// tensors.
+TF_CAPI_EXPORT extern int TFE_FinalizeInputTensorsFromTraceContext(
+    TFE_TraceContext* trace_ctx);
+
+TF_CAPI_EXPORT extern TF_Output TFE_GetInputGraphNodeFromTraceContext(
+    TFE_TraceContext* trace_ctx, unsigned int idx);
+
+// Each input tensor should be consumed at most once.
+TF_CAPI_EXPORT extern TFE_TensorHandle*
+TFE_ConsumeInputConcreteTensorFromTraceContext(TFE_TraceContext* trace_ctx,
+                                               unsigned int idx);
+
 #ifdef __cplusplus
 } /* end extern "C" */
 #endif
diff --git a/tensorflow/c/c_api_experimental_test.cc b/tensorflow/c/c_api_experimental_test.cc
index 65a4e10b26..4cfcf2ef3b 100644
--- a/tensorflow/c/c_api_experimental_test.cc
+++ b/tensorflow/c/c_api_experimental_test.cc
@@ -14,6 +14,7 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/c/c_api_experimental.h"
+#include "tensorflow/c/c_api_internal.h"
 #include "tensorflow/c/c_test_util.h"
 #include "tensorflow/c/eager/c_api.h"
 #include "tensorflow/c/eager/c_api_test_util.h"
@@ -296,7 +297,29 @@ TEST(CAPI_EXPERIMENTAL, TFE_ExecuteOpInNewThreadTest_Blocking) {
   TF_DeleteStatus(status);
 }
 
-TEST(CAPI_EXPERIMENTAL, DebugPrint) {
+TEST(CAPI_EXPERIMENTAL, SymbolicTensor) {
+  TF_Status* status = TF_NewStatus();
+  auto node = TF_Output{nullptr, 1};
+  auto* sym_handle = TFE_NewTensorHandleFromTFOutput(node, TF_FLOAT);
+  TFE_TensorHandlePrintDebugString(sym_handle);
+  CHECK_EQ(TFE_TensorHandleDataType(sym_handle), TF_FLOAT);
+  ASSERT_FALSE(TFE_TensorHandleIsConcrete(sym_handle));
+  auto same_node = TFE_GetTFOutputFromTensorHandle(sym_handle, status);
+  CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
+  ASSERT_EQ(same_node.oper, node.oper);
+  ASSERT_EQ(same_node.index, node.index);
+  TFE_DeleteTensorHandle(sym_handle);
+
+  TFE_TensorHandle* m = TestMatrixTensorHandle();
+  ASSERT_TRUE(TFE_TensorHandleIsConcrete(m));
+  (void)TFE_GetTFOutputFromTensorHandle(m, status);
+  CHECK_EQ(TF_INTERNAL, TF_GetCode(status)) << TF_Message(status);
+  TFE_DeleteTensorHandle(m);
+
+  TF_DeleteStatus(status);
+}
+
+TEST(CAPI_EXPERIMENTAL, DebugPrintAndSymbolicExecution) {
   TF_Status* status = TF_NewStatus();
   TFE_ContextOptions* opts = TFE_NewContextOptions();
   CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
@@ -310,6 +333,32 @@ TEST(CAPI_EXPERIMENTAL, DebugPrint) {
   CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
   TFE_OpPrintDebugString(op);
 
+  auto* graph = TF_NewGraph();
+  auto* trace_ctx = TFE_NewTraceContext(graph);
+  TFE_TensorHandle* retvals[5];
+  int num_retvals = 5;
+  // Symbolically execute this op, which adds a graph node to `trace_ctx`.
+  TFE_AddEagerOpToGraph(op, trace_ctx, retvals, &num_retvals, status);
+  CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
+
+  int num_inputs = TFE_FinalizeInputTensorsFromTraceContext(trace_ctx);
+  CHECK_EQ(num_inputs, 1);
+  auto input_sym_tensor = TFE_GetInputGraphNodeFromTraceContext(trace_ctx,
+                                                                /*idx*/ 0);
+
+  LOG(INFO) << tensorflow::getTF_OutputDebugString(input_sym_tensor);
+  auto handle = TFE_ConsumeInputConcreteTensorFromTraceContext(trace_ctx,
+                                                               /*idx*/ 0);
+  TFE_TensorHandlePrintDebugString(handle);
+  TFE_DeleteTensorHandle(handle);
+
+  CHECK_EQ(num_retvals, 1);
+  CHECK_EQ(TFE_TensorHandleDataType(retvals[0]), TF_FLOAT);
+  TFE_DeleteTensorHandle(retvals[0]);
+
+  TFE_DeleteTraceContext(trace_ctx);
+  TF_DeleteGraph(graph);
+
   TFE_DeleteTensorHandle(m);
   TFE_DeleteOp(op);
   TFE_DeleteContext(ctx);
diff --git a/tensorflow/c/c_api_internal.h b/tensorflow/c/c_api_internal.h
index 5ba26d3c58..73283d7756 100644
--- a/tensorflow/c/c_api_internal.h
+++ b/tensorflow/c/c_api_internal.h
@@ -228,6 +228,8 @@ void RecordMutation(TF_Graph* graph, const TF_Operation& op,
 bool ExtendSessionGraphHelper(TF_Session* session, TF_Status* status)
     LOCKS_EXCLUDED(session->graph->mu, session->mu);
 
+std::string getTF_OutputDebugString(TF_Output node);
+
 }  // end namespace tensorflow
 
 #endif  // TENSORFLOW_C_C_API_INTERNAL_H_
diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc
index a92ba1f0f3..af13f487af 100755
--- a/tensorflow/c/eager/c_api.cc
+++ b/tensorflow/c/eager/c_api.cc
@@ -356,6 +356,8 @@ TFE_TensorHandle* TFE_NewTensorHandle(TF_Tensor* t, TF_Status* status) {
 
 void TFE_DeleteTensorHandle(TFE_TensorHandle* h) {
   if (h == nullptr) return;
+  VLOG(1) << "Deleting tensor handle " << h << " with internal handle "
+          << h->handle;
   if (h->handle) {
     h->handle->Unref();
   }
diff --git a/tensorflow/c/eager/c_api_internal.h b/tensorflow/c/eager/c_api_internal.h
index d330aa4290..3b9e681194 100644
--- a/tensorflow/c/eager/c_api_internal.h
+++ b/tensorflow/c/eager/c_api_internal.h
@@ -83,6 +83,12 @@ struct TFE_TensorHandle {
   TFE_TensorHandle(tensorflow::TensorHandle* handle) : handle(handle) {}
 
   tensorflow::TensorHandle* handle;
+
+  // Create a symbolic tensor.
+  TFE_TensorHandle(TF_Output t, TF_DataType dtype)
+      : handle(new tensorflow::TensorHandle(
+            tensorflow::OutputGraphNode{t.oper, t.index},
+            static_cast<tensorflow::DataType>(dtype))) {}
 };
 
 struct TFE_TensorDebugInfo {
@@ -115,4 +121,24 @@ void SetOpAttrValueScalar(TFE_Context* ctx, TFE_Op* op,
                           const char* attr_name, TF_Status* status);
 }  // namespace tensorflow
 
+struct TFE_TraceContext {
+  TF_Graph* const graph;
+
+  unsigned int node_counter = 0;
+  // Each tensor handle will have its ref count incremented when it's added as a
+  // map key, and decremented when this object is destroyed.
+  std::map<tensorflow::TensorHandle*, TF_Output> input_tensor_map;
+  std::vector<std::pair<tensorflow::TensorHandle*, TF_Output>>* input_tensors =
+      nullptr;
+
+  TFE_TraceContext(TF_Graph* graph) : graph(graph) {}
+
+  ~TFE_TraceContext() {
+    delete input_tensors;
+    for (auto input : input_tensor_map) {
+      input.first->Unref();
+    }
+  }
+};
+
 #endif  // TENSORFLOW_C_EAGER_C_API_INTERNAL_H_
diff --git a/tensorflow/core/common_runtime/eager/BUILD b/tensorflow/core/common_runtime/eager/BUILD
index 06cbe8049f..0a16f80c8f 100644
--- a/tensorflow/core/common_runtime/eager/BUILD
+++ b/tensorflow/core/common_runtime/eager/BUILD
@@ -106,6 +106,7 @@ tf_cuda_library(
             "//tensorflow/core:android_tensorflow_lib_lite",
         ],
         "//conditions:default": [
+            "@com_google_absl//absl/strings",
             "//tensorflow/core:core_cpu_lib",
             "//tensorflow/core:framework",
             "//tensorflow/core:framework_internal",
diff --git a/tensorflow/core/common_runtime/eager/tensor_handle.cc b/tensorflow/core/common_runtime/eager/tensor_handle.cc
index 5e5936f1cf..e44a97b265 100644
--- a/tensorflow/core/common_runtime/eager/tensor_handle.cc
+++ b/tensorflow/core/common_runtime/eager/tensor_handle.cc
@@ -22,6 +22,7 @@ limitations under the License.
 #include <string>
 #include <vector>
 
+#include "absl/strings/substitute.h"
 #include "tensorflow/core/common_runtime/copy_tensor.h"
 #include "tensorflow/core/common_runtime/device.h"
 #include "tensorflow/core/common_runtime/device_factory.h"
@@ -99,6 +100,19 @@ TensorHandle::TensorHandle(int64 op_id, int32 output_num,
                               : resource_device_ == nullptr);
 }
 
+TensorHandle::TensorHandle(OutputGraphNode symbolic_tensor, DataType dtype)
+    : dtype(dtype),
+      node_id_(0),
+      device_(nullptr),
+      op_device_(nullptr),
+      resource_device_(nullptr),
+      remote_op_id_(-1),
+      remote_output_num_(-1),
+      remote_shape_node_id_(-1),
+      ctx_(nullptr),
+      is_ready_(true),
+      symbolic_tensor(new OutputGraphNode(symbolic_tensor)) {}
+
 bool TensorHandle::IsReady() {
   if (node_id_ == 0) return true;
   mutex_lock l(ctx_mutex_);
@@ -315,6 +329,11 @@ Device* GetResourceDevice(const Tensor& t, EagerContext* ctx) {
 string TensorHandle::DebugString() const {
   VLOG(1) << "Calling TensorHandle::DebugString() on " << this;
 
+  if (symbolic_tensor) {
+    return absl::Substitute("TF_Output($0, $1)", symbolic_tensor->oper,
+                            symbolic_tensor->index);
+  }
+
   string out;
   strings::StrAppend(&out, "Device: ", device_ ? device_->DebugString() : "[]");
   // Consider supporting non-CPU tensors (when device_ is non-NULL) if needed.
diff --git a/tensorflow/core/common_runtime/eager/tensor_handle.h b/tensorflow/core/common_runtime/eager/tensor_handle.h
index 0ecd9e6ddc..e4ccb11dba 100644
--- a/tensorflow/core/common_runtime/eager/tensor_handle.h
+++ b/tensorflow/core/common_runtime/eager/tensor_handle.h
@@ -41,8 +41,17 @@ limitations under the License.
 #include "tensorflow/core/public/session_options.h"
 #include "tensorflow/core/public/version.h"
 
+struct TF_Operation;
+
 namespace tensorflow {
 
+// This struct is isomorphic to TF_Output, but we cannot use the latter here due
+// to layering concerns (TF_Output is defined at the C API layer).
+struct OutputGraphNode {
+  TF_Operation* oper;
+  int index;  // The index of the output within oper.
+};
+
 // Associates a Tensor and a Device, used in the eager runtime. Internal version
 // of the TFE_TensorHandle struct and the python EagerTensor class
 // (unrelated to python TensorHandle).
@@ -58,8 +67,11 @@ class TensorHandle : public core::RefCounted {
                DataType dtype, std::function<void()> call_on_destroy, Device* d,
                Device* op_device, Device* resource_device, EagerContext* ctx);
 
+  // Symbolic tensor constructor.
+  TensorHandle(OutputGraphNode symbolic_tensor, DataType dtype);
+
   ~TensorHandle() override {
-    VLOG(1) << "Deleting TensorHandle " << this;
+    VLOG(1) << "Deleting internal TensorHandle " << this;
     if (call_on_destroy_) {
       call_on_destroy_();
     }
@@ -115,6 +127,8 @@ class TensorHandle : public core::RefCounted {
 
   bool IsRemote();
 
+  OutputGraphNode* getSymbolicTensor() const { return symbolic_tensor.get(); }
+
   string DebugString() const;
 
  private:
@@ -168,6 +182,11 @@ class TensorHandle : public core::RefCounted {
   // `ctx` object is not owned and should outlive this handle.
   EagerContext* ctx_ GUARDED_BY(ctx_mutex_);
   bool is_ready_ GUARDED_BY(ctx_mutex_);
+
+  // When non-NULL, this tensor handle instance represents a symbolic tensor
+  // (corresponding to a graph node), whose concrete value is to be produced by
+  // executing that graph node.
+  std::unique_ptr<OutputGraphNode> symbolic_tensor;
 };
 
 // If tensor's dtype is DT_RESOURCE, returns the device backing the resource.
-- 
GitLab


From 502ca3eed27ea4b096a707dd88b56d339482a4f6 Mon Sep 17 00:00:00 2001
From: Yu-Cheng Ling <ycling@google.com>
Date: Fri, 18 Jan 2019 17:35:42 -0800
Subject: [PATCH 1023/1095] Fix a typo in op_hint.py comment

PiperOrigin-RevId: 230022859
---
 tensorflow/lite/python/op_hint.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/tensorflow/lite/python/op_hint.py b/tensorflow/lite/python/op_hint.py
index 6ec050171f..8df37c111b 100644
--- a/tensorflow/lite/python/op_hint.py
+++ b/tensorflow/lite/python/op_hint.py
@@ -36,9 +36,7 @@ Example:
   session = tf.Session()
 
   graphdef_to_convert = tf.lite.convert_op_hints_to_stubs(session)
-  tflite_graph = tf.lite.toco_convert(graphdef_to_convert,
-                                              [image], [output])
-                                              [image], [output])
+  tflite_graph = tf.lite.toco_convert(graphdef_to_convert, [image], [output])
   with open("/tmp/graph.fb", "wb") as fp:
     fp.write(tflite_graph)
 
-- 
GitLab


From 698d685ca1217dbb83b16599c44f65c0e937beba Mon Sep 17 00:00:00 2001
From: Pete Warden <petewarden@google.com>
Date: Fri, 18 Jan 2019 17:47:55 -0800
Subject: [PATCH 1024/1095] Added table of contents and getting started guide
 to TFL micro documentation

PiperOrigin-RevId: 230023991
---
 tensorflow/lite/experimental/micro/README.md | 290 +++++++++++--------
 1 file changed, 175 insertions(+), 115 deletions(-)

diff --git a/tensorflow/lite/experimental/micro/README.md b/tensorflow/lite/experimental/micro/README.md
index 88f3cd71f5..e85b9660e3 100644
--- a/tensorflow/lite/experimental/micro/README.md
+++ b/tensorflow/lite/experimental/micro/README.md
@@ -7,118 +7,66 @@ so it's designed to be portable even to 'bare metal' systems. The core runtime
 fits in 16KB on a Cortex M3, and with enough operators to run a speech keyword
 detection model, takes up a total of 22KB.
 
-The design goals are for the framework to be:
-
--   **Readable**: We want embedded software engineers to be able to understand
-    what's required to run ML inference without having to study research papers.
-    We've tried to keep the code base small, modular, and have reference
-    implementations of all operations to help with this.
-
--   **Easy to modify**: We know that there are a lot of different platforms and
-    requirements in the embedded world, and we don't expect to cover all of them
-    in one framework. Instead, we're hoping that it can be a good starting point
-    for developers to build on top of to meet their own needs. For example, we
-    tried to make it easy to replace the implementations of key computational
-    operators that are often crucial for performance, without having to touch
-    the data flow and other runtime code. We want it to make more sense to use
-    our workflow to handle things like model import and less-important
-    operations, and customize the parts that matter, rather than having to
-    reimplement everything in your own engine.
-
--   **Well-tested**: If you're modifying code, you need to know if your changes
-    are correct. Having an easy way to test lets you develop much faster. To
-    help there, we've written tests for all the components, and we've made sure
-    that the tests can be run on almost any platform, with no dependencies apart
-    from the ability to log text to a debug console somewhere. We also provide
-    an easy way to run all the tests on-device as part of an automated test
-    framework, and we use qemu/Renode emulation so that tests can be run even
-    without physical devices present.
-
--   **Easy to integrate**: We want to be as open a system as possible, and use
-    the best code available for each platform. To do that, we're going to rely
-    on projects like
-    [CMSIS-NN](https://www.keil.com/pack/doc/CMSIS/NN/html/index.html),
-    [uTensor](https://github.com/uTensor/uTensor), and other vendor libraries to
-    handle as much performance-critical code as possible. We know that there are
-    an increasing number of options to accelerate neural networks on
-    microcontrollers, so we're aiming to be a good host for deploying those
-    hardware technologies too.
-
--   **Compatible**: We're using the same file schema, interpreter API, and
-    kernel interface as regular TensorFlow Lite, so we leverage the large
-    existing set of tools, documentation, and examples for the project. The
-    biggest barrier to deploying ML models is getting them from a training
-    environment into a form that's easy to run inference on, so we see reusing
-    this rich ecosystem as being crucial to being easily usable. We also hope to
-    integrate this experimental work back into the main codebase in the future.
-
-To meet those goals, we've made some tradeoffs:
-
--   **Simple C++**: To help with readability, our code is written in a modern
-    version of C++, but we generally treat it as a "better C", rather relying on
-    more complex features such as template meta-programming. As mentioned
-    earlier, we avoid any use of dynamic memory allocation (new/delete) or the
-    standard C/C++ libraries, so we believe this should still be fairly
-    portable. It does mean that some older devices with C-only toolchains won't
-    be supported, but we're hoping that the reference operator implementations
-    (which are simple C-like functions) can still be useful in those cases. The
-    interfaces are also designed to be C-only, so it should be possible to
-    integrate the resulting library with pure C projects.
-
--   **Interpreted**: Code generation is a popular pattern for embedded code,
-    because it gives standalone code that's easy to modify and step through, but
-    we've chosen to go with an interpreted approach. In our internal
-    microcontroller work we've found that using an extremely stripped-down
-    interpreter with almost no dependencies gives us a lot of the same
-    advantages, but is easier to maintain. For example, when new updates come
-    out for the underlying library, you can just merge your local modifications
-    in a single step, rather than having to regenerate new code and then patch
-    in any changes you subsequently made. The coarse granularity of the
-    interpreted primitives means that each operation call typically takes
-    hundreds of thousands of instruction cycles at least, so we don't see
-    noticeable performance gains from avoiding what's essentially a single
-    switch statement at the interpreter level to call each operation. We're
-    still working on improving the packaging though, for example we're
-    considering having the ability to snapshot all the source files and headers
-    used for a particular model, being able to compile the code and data
-    together as a library, and then access it through a minimal set of C
-    interface calls which hide the underlying complexity.
-
--   **Flatbuffers**: We represent our models using
-    [the standard flatbuffer schema used by the rest of TensorFlow Lite](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/schema/schema.fbs),
-    with the difference that we always keep it in read-only program memory
-    (typically flash) rather than relying on having a file system to read it
-    from. This is a good fit because flatbuffer's serialized format is designed
-    to be mapped into memory without requiring any extra memory allocations or
-    modifications to access it. All of the functions to read model values work
-    directly on the serialized bytes, and large sections of data like weights
-    are directly accessible as sequential C-style arrays of their data type,
-    with no strides or unpacking needed. We do get a lot of value from using
-    flatbuffers, but there is a cost in complexity. The flat buffer library code
-    is all inline
-    [inside the main headers](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/schema/schema_generated.h),
-    but it isn't straightforward to inspect their implementations, and the model
-    data structures aren't easy to comprehend from the debugger. The header for
-    the schema itself also has to be periodically updated when new information
-    is added to the file format, though we try to handle that transparently for
-    most developers by checking in a pre-generated version.
-
--   **Code Duplication**: Some of the code in this prototype largely duplicates
-    the logic in other parts of the TensorFlow Lite code base, for example the
-    operator wrappers. We've tried to keep share as much as we can between the
-    two interpreters, but there are some assumptions built into the original
-    runtime that make this difficult. We'll be working on modularizing the main
-    interpreter so that we can move to an entirely shared system.
-
-This initial preview release is designed to get early feedback, and is not
-intended to be a final product. It only includes enough operations to run a
-simple keyword recognition model, and the implementations are not optimized.
-We're hoping this will be a good way to get feedback and collaborate to improve
-the framework.
-
-## Getting Started with Make
-
-Building requires a Linux or OS X machine.
+## Table of Contents
+
+-   [Getting Started](#getting-started)
+    *   [Getting Started with Portable Reference Code](#getting-started-with-portable-reference-code)
+    *   [Building Portable Reference Code using Make](#building-portable-reference-code-using-make)
+    *   [Building for the "Blue Pill" STM32F103 using Make](#building-for-the-blue-pill-stm32f103-using-make)
+    *   [Building for "Hifive1" SiFive FE310 development board using Make](#building-for-hifive1-sifive-fe310-development-board-using-make)
+    *   [Building for Ambiq Micro Apollo3Blue EVB using Make](#building-for-ambiq-micro-apollo3blue-evb-using-make)
+        *   [Additional Apollo3 Instructions](#additional-apollo3-instructions)
+-   [Goals](#goals)
+-   [Generating Project Files](#generating-project-#files)
+-   [How to Port TensorFlow Lite Micro to a New Platform](#how-to-port-tensorflow-lite-micro-to-a-new-platform)
+    *   [Requirements](#requirements)
+    *   [Getting Started](getting-started)
+    *   [Troubleshooting](#troubleshooting)
+    *   [Optimizing for your Platform](#optimizing-for-your-platform)
+    *   [Code Module Organization](#code-module-organization)
+    *   [Working with Generated Projects](#working-with-generated-projects)
+    *   [Supporting a Platform with Makefiles](#supporting-a-platform-with-makefiles)
+    *   [Supporting a Platform with Emulation Testing](#supporting-a-platform-with-emulation-testing)
+    *   [Implementing More Optimizations](#implementing-more-optimizations)
+
+# Getting Started
+
+One of the challenges of embedded software development is that there are a lot
+of different architectures, devices, operating systems, and build systems. We
+aim to support as many of the popular combinations as we can, and make it as
+easy as possible to add support for others.
+
+If you're a product developer, we have build instructions or pre-generated
+project files that you can download for the following platforms:
+
+| Device | Mbed | Keil | Make/GCC
+-------- | ---- | ---- | --------
+[STM32F746G Discovery Board](https://www.st.com/en/evaluation-tools/32f746gdiscovery.html) | [Download](https://drive.google.com/open?id=1OtgVkytQBrEYIpJPsE8F6GUKHPBS3Xeb) | - | [Download](https://drive.google.com/open?id=1u46mTtAMZ7Y1aD-He1u3R8AE4ZyEpnOl) | -
+["Blue Pill" STM32F103-compatible development board](https://github.com/google/stm32_bare_lib) | - | - | [Instructions](#building-for-the-blue-pill-stm32f103-using-make)
+[Ambiq Micro Apollo3Blue EVB using Make](https://ambiqmicro.com/apollo-ultra-low-power-mcus/) | - | - | [Instructions](#building-for-ambiq-micro-apollo3blue-evb-using-make)
+[Generic Keil uVision Projects](http://www2.keil.com/mdk5/uvision/) | - | [Download](https://drive.google.com/open?id=1Lw9rsdquNKObozClLPoE5CTJLuhfh5mV) | -
+
+If your device is not yet supported, it may not be too hard to add support. You
+can learn about that process
+[here](#how-to-port-tensorflow-lite-micro-to-a-new-platform). We're looking
+forward to getting your help expanding this table!
+
+## Getting Started with Portable Reference Code
+
+If you don't have a particular microcontroller platform in mind yet, or just
+want to try out the code before beginning porting, the easiest way to begin is
+by
+[downloading the platform-agnostic reference code](https://drive.google.com/open?id=1cawEQAkqquK_SO4crReDYqf_v7yAwOY8).
+You'll see a series of folders inside the archive, with each one containing just
+the source files you need to build one binary. There is a simple Makefile for
+each folder, but you should be able to load the files into almost any IDE and
+build them. There's also a [Visual Studio Code](https://code.visualstudio.com/) project file already set up, so
+you can easily explore the code in a cross-platform IDE.
+
+## Building Portable Reference Code using Make
+
+It's easy to build portable reference code directly from Github using make if
+you're on a Linux or OS X machine.
 
 -   Open a terminal
 -   Download the TensorFlow source with `git clone
@@ -177,7 +125,7 @@ building binaries that run locally on the Mac OS or Linux machine you're
 building on, but this approach becomes important when we're targeting simple
 micro controller devices.
 
-## Building for the "Blue Pill" STM32F103
+## Building for the "Blue Pill" STM32F103 using Make
 
 The goal of this library is to enable machine learning on resource-constrained
 micro controllers and DSPs, and as part of that we've targeted the
@@ -365,6 +313,117 @@ file =
 tensorflow/lite/experimental/micro/tools/make/gen/apollo3evb_cortex-m4/bin/pushbutton_cmsis_speech_test.bin
 5. Prog Addr = 0x0000C000
 
+## Goals
+
+The design goals are for the framework to be:
+
+-   **Readable**: We want embedded software engineers to be able to understand
+    what's required to run ML inference without having to study research papers.
+    We've tried to keep the code base small, modular, and have reference
+    implementations of all operations to help with this.
+
+-   **Easy to modify**: We know that there are a lot of different platforms and
+    requirements in the embedded world, and we don't expect to cover all of them
+    in one framework. Instead, we're hoping that it can be a good starting point
+    for developers to build on top of to meet their own needs. For example, we
+    tried to make it easy to replace the implementations of key computational
+    operators that are often crucial for performance, without having to touch
+    the data flow and other runtime code. We want it to make more sense to use
+    our workflow to handle things like model import and less-important
+    operations, and customize the parts that matter, rather than having to
+    reimplement everything in your own engine.
+
+-   **Well-tested**: If you're modifying code, you need to know if your changes
+    are correct. Having an easy way to test lets you develop much faster. To
+    help there, we've written tests for all the components, and we've made sure
+    that the tests can be run on almost any platform, with no dependencies apart
+    from the ability to log text to a debug console somewhere. We also provide
+    an easy way to run all the tests on-device as part of an automated test
+    framework, and we use qemu/Renode emulation so that tests can be run even
+    without physical devices present.
+
+-   **Easy to integrate**: We want to be as open a system as possible, and use
+    the best code available for each platform. To do that, we're going to rely
+    on projects like
+    [CMSIS-NN](https://www.keil.com/pack/doc/CMSIS/NN/html/index.html),
+    [uTensor](https://github.com/uTensor/uTensor), and other vendor libraries to
+    handle as much performance-critical code as possible. We know that there are
+    an increasing number of options to accelerate neural networks on
+    microcontrollers, so we're aiming to be a good host for deploying those
+    hardware technologies too.
+
+-   **Compatible**: We're using the same file schema, interpreter API, and
+    kernel interface as regular TensorFlow Lite, so we leverage the large
+    existing set of tools, documentation, and examples for the project. The
+    biggest barrier to deploying ML models is getting them from a training
+    environment into a form that's easy to run inference on, so we see reusing
+    this rich ecosystem as being crucial to being easily usable. We also hope to
+    integrate this experimental work back into the main codebase in the future.
+
+To meet those goals, we've made some tradeoffs:
+
+-   **Simple C++**: To help with readability, our code is written in a modern
+    version of C++, but we generally treat it as a "better C", rather relying on
+    more complex features such as template meta-programming. As mentioned
+    earlier, we avoid any use of dynamic memory allocation (new/delete) or the
+    standard C/C++ libraries, so we believe this should still be fairly
+    portable. It does mean that some older devices with C-only toolchains won't
+    be supported, but we're hoping that the reference operator implementations
+    (which are simple C-like functions) can still be useful in those cases. The
+    interfaces are also designed to be C-only, so it should be possible to
+    integrate the resulting library with pure C projects.
+
+-   **Interpreted**: Code generation is a popular pattern for embedded code,
+    because it gives standalone code that's easy to modify and step through, but
+    we've chosen to go with an interpreted approach. In our internal
+    microcontroller work we've found that using an extremely stripped-down
+    interpreter with almost no dependencies gives us a lot of the same
+    advantages, but is easier to maintain. For example, when new updates come
+    out for the underlying library, you can just merge your local modifications
+    in a single step, rather than having to regenerate new code and then patch
+    in any changes you subsequently made. The coarse granularity of the
+    interpreted primitives means that each operation call typically takes
+    hundreds of thousands of instruction cycles at least, so we don't see
+    noticeable performance gains from avoiding what's essentially a single
+    switch statement at the interpreter level to call each operation. We're
+    still working on improving the packaging though, for example we're
+    considering having the ability to snapshot all the source files and headers
+    used for a particular model, being able to compile the code and data
+    together as a library, and then access it through a minimal set of C
+    interface calls which hide the underlying complexity.
+
+-   **Flatbuffers**: We represent our models using
+    [the standard flatbuffer schema used by the rest of TensorFlow Lite](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/schema/schema.fbs),
+    with the difference that we always keep it in read-only program memory
+    (typically flash) rather than relying on having a file system to read it
+    from. This is a good fit because flatbuffer's serialized format is designed
+    to be mapped into memory without requiring any extra memory allocations or
+    modifications to access it. All of the functions to read model values work
+    directly on the serialized bytes, and large sections of data like weights
+    are directly accessible as sequential C-style arrays of their data type,
+    with no strides or unpacking needed. We do get a lot of value from using
+    flatbuffers, but there is a cost in complexity. The flat buffer library code
+    is all inline
+    [inside the main headers](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/schema/schema_generated.h),
+    but it isn't straightforward to inspect their implementations, and the model
+    data structures aren't easy to comprehend from the debugger. The header for
+    the schema itself also has to be periodically updated when new information
+    is added to the file format, though we try to handle that transparently for
+    most developers by checking in a pre-generated version.
+
+-   **Code Duplication**: Some of the code in this prototype largely duplicates
+    the logic in other parts of the TensorFlow Lite code base, for example the
+    operator wrappers. We've tried to keep share as much as we can between the
+    two interpreters, but there are some assumptions built into the original
+    runtime that make this difficult. We'll be working on modularizing the main
+    interpreter so that we can move to an entirely shared system.
+
+This initial preview release is designed to get early feedback, and is not
+intended to be a final product. It only includes enough operations to run a
+simple keyword recognition model, and the implementations are not optimized.
+We're hoping this will be a good way to get feedback and collaborate to improve
+the framework.
+
 ## Generating Project Files
 
 It's not always easy or convenient to use a makefile-based build process,
@@ -384,7 +443,8 @@ This will create a folder in
 `tensorflow/lite/experimental/micro/tools/make/gen/mbed_cortex-m4/prj/micro_speech_main_test/mbed`
 that contains the source and header files, some Mbed configuration files, and a
 README. You should then be able to copy this directory to another machine, and
-use it just like any other Mbed project.
+use it just like any other Mbed project. There's more information about project
+files [below](#working-with-generated-projects).
 
 ## How to Port TensorFlow Lite Micro to a New Platform
 
@@ -793,7 +853,7 @@ was successful. These scripts need to run on Ubuntu 18.04, in a bash
 environment, though Docker is available if you need to install extra software or
 have other dependencies.
 
-### Optimizing for your Platform
+### Implementing More Optimizations
 
 Clearly, getting debug logging support is only the beginning of the work you'll
 need to do on a particular platform. It's very likely that you'll want to
-- 
GitLab


From 1790a016fc49e1760078682e17c42168f1fbfa77 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 18 Jan 2019 18:04:14 -0800
Subject: [PATCH 1025/1095] Recreate preconfigs with nvcc binaries.

PiperOrigin-RevId: 230025587
---
 .../ubuntu14.04/cuda10.0-cudnn7/cuda/BUILD    | 44 +++++++++++-------
 .../ubuntu14.04/cuda9.0-cudnn7/cuda/BUILD     | 45 ++++++++++++-------
 .../preconfig/ubuntu14.04/tensorrt5/BUILD     |  8 +---
 .../ubuntu14.04/tensorrt5/build_defs.bzl      |  4 +-
 4 files changed, 62 insertions(+), 39 deletions(-)

diff --git a/third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/cuda/BUILD b/third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/cuda/BUILD
index c813efccf9..13429d7b88 100755
--- a/third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/cuda/BUILD
+++ b/third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/cuda/BUILD
@@ -1192,9 +1192,7 @@ genrule(
         "cuda/include/vector_functions.hpp",
         "cuda/include/vector_types.h",
     ],
-    cmd = """
-if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp -f "/usr/local/cuda-10.0/include/CL/cl.h" "$(@D)/cuda/include/CL/cl.h" && cp -f "/usr/local/cuda-10.0/include/CL/cl.hpp" "$(@D)/cuda/include/CL/cl.hpp" && cp -f "/usr/local/cuda-10.0/include/CL/cl_egl.h" "$(@D)/cuda/include/CL/cl_egl.h" && cp -f "/usr/local/cuda-10.0/include/CL/cl_ext.h" "$(@D)/cuda/include/CL/cl_ext.h" && cp -f "/usr/local/cuda-10.0/include/CL/cl_gl.h" "$(@D)/cuda/include/CL/cl_gl.h" && cp -f "/usr/local/cuda-10.0/include/CL/cl_gl_ext.h" "$(@D)/cuda/include/CL/cl_gl_ext.h" && cp -f "/usr/local/cuda-10.0/include/CL/cl_platform.h" "$(@D)/cuda/include/CL/cl_platform.h" && cp -f "/usr/local/cuda-10.0/include/CL/opencl.h" "$(@D)/cuda/include/CL/opencl.h" && cp -f "/usr/local/cuda-10.0/include/builtin_types.h" "$(@D)/cuda/include/builtin_types.h" && cp -f "/usr/local/cuda-10.0/include/channel_descriptor.h" "$(@D)/cuda/include/channel_descriptor.h" && cp -f "/usr/local/cuda-10.0/include/common_functions.h" "$(@D)/cuda/include/common_functions.h" && cp -f "/usr/local/cuda-10.0/include/cooperative_groups.h" "$(@D)/cuda/include/cooperative_groups.h" && cp -f "/usr/local/cuda-10.0/include/cooperative_groups_helpers.h" "$(@D)/cuda/include/cooperative_groups_helpers.h" && cp -f "/usr/local/cuda-10.0/include/crt/common_functions.h" "$(@D)/cuda/include/crt/common_functions.h" && cp -f "/usr/local/cuda-10.0/include/crt/device_double_functions.h" "$(@D)/cuda/include/crt/device_double_functions.h" && cp -f "/usr/local/cuda-10.0/include/crt/device_double_functions.hpp" "$(@D)/cuda/include/crt/device_double_functions.hpp" && cp -f "/usr/local/cuda-10.0/include/crt/device_functions.h" "$(@D)/cuda/include/crt/device_functions.h" && cp -f "/usr/local/cuda-10.0/include/crt/device_functions.hpp" "$(@D)/cuda/include/crt/device_functions.hpp" && cp -f "/usr/local/cuda-10.0/include/crt/func_macro.h" "$(@D)/cuda/include/crt/func_macro.h" && cp -f "/usr/local/cuda-10.0/include/crt/host_config.h" "$(@D)/cuda/include/crt/host_config.h" && cp -f "/usr/local/cuda-10.0/include/crt/host_defines.h" "$(@D)/cuda/include/crt/host_defines.h" && cp -f "/usr/local/cuda-10.0/include/crt/host_runtime.h" "$(@D)/cuda/include/crt/host_runtime.h" && cp -f "/usr/local/cuda-10.0/include/crt/math_functions.h" "$(@D)/cuda/include/crt/math_functions.h" && cp -f "/usr/local/cuda-10.0/include/crt/math_functions.hpp" "$(@D)/cuda/include/crt/math_functions.hpp" && cp -f "/usr/local/cuda-10.0/include/crt/mma.h" "$(@D)/cuda/include/crt/mma.h" && cp -f "/usr/local/cuda-10.0/include/crt/mma.hpp" "$(@D)/cuda/include/crt/mma.hpp" && cp -f "/usr/local/cuda-10.0/include/crt/nvfunctional" "$(@D)/cuda/include/crt/nvfunctional" && cp -f "/usr/local/cuda-10.0/include/crt/sm_70_rt.h" "$(@D)/cuda/include/crt/sm_70_rt.h" && cp -f "/usr/local/cuda-10.0/include/crt/sm_70_rt.hpp" "$(@D)/cuda/include/crt/sm_70_rt.hpp" && cp -f "/usr/local/cuda-10.0/include/crt/storage_class.h" "$(@D)/cuda/include/crt/storage_class.h" && cp -f "/usr/local/cuda-10.0/include/cuComplex.h" "$(@D)/cuda/include/cuComplex.h" && cp -f "/usr/local/cuda-10.0/include/cublas.h" "$(@D)/cuda/include/cublas.h" && cp -f "/usr/local/cuda-10.0/include/cublasXt.h" "$(@D)/cuda/include/cublasXt.h" && cp -f "/usr/local/cuda-10.0/include/cublas_api.h" "$(@D)/cuda/include/cublas_api.h" && cp -f "/usr/local/cuda-10.0/include/cublas_v2.h" "$(@D)/cuda/include/cublas_v2.h" && cp -f "/usr/local/cuda-10.0/include/cuda.h" "$(@D)/cuda/include/cuda.h" && cp -f "/usr/local/cuda-10.0/include/cudaEGL.h" "$(@D)/cuda/include/cudaEGL.h" && cp -f "/usr/local/cuda-10.0/include/cudaGL.h" "$(@D)/cuda/include/cudaGL.h" && cp -f "/usr/local/cuda-10.0/include/cudaProfiler.h" "$(@D)/cuda/include/cudaProfiler.h" && cp -f "/usr/local/cuda-10.0/include/cudaVDPAU.h" "$(@D)/cuda/include/cudaVDPAU.h" && cp -f "/usr/local/cuda-10.0/include/cuda_device_runtime_api.h" "$(@D)/cuda/include/cuda_device_runtime_api.h" && cp -f "/usr/local/cuda-10.0/include/cuda_egl_interop.h" "$(@D)/cuda/include/cuda_egl_interop.h" && cp -f "/usr/local/cuda-10.0/include/cuda_fp16.h" "$(@D)/cuda/include/cuda_fp16.h" && cp -f "/usr/local/cuda-10.0/include/cuda_fp16.hpp" "$(@D)/cuda/include/cuda_fp16.hpp" && cp -f "/usr/local/cuda-10.0/include/cuda_gl_interop.h" "$(@D)/cuda/include/cuda_gl_interop.h" && cp -f "/usr/local/cuda-10.0/include/cuda_occupancy.h" "$(@D)/cuda/include/cuda_occupancy.h" && cp -f "/usr/local/cuda-10.0/include/cuda_profiler_api.h" "$(@D)/cuda/include/cuda_profiler_api.h" && cp -f "/usr/local/cuda-10.0/include/cuda_runtime.h" "$(@D)/cuda/include/cuda_runtime.h" && cp -f "/usr/local/cuda-10.0/include/cuda_runtime_api.h" "$(@D)/cuda/include/cuda_runtime_api.h" && cp -f "/usr/local/cuda-10.0/include/cuda_surface_types.h" "$(@D)/cuda/include/cuda_surface_types.h" && cp -f "/usr/local/cuda-10.0/include/cuda_texture_types.h" "$(@D)/cuda/include/cuda_texture_types.h" && cp -f "/usr/local/cuda-10.0/include/cuda_vdpau_interop.h" "$(@D)/cuda/include/cuda_vdpau_interop.h" && cp -f "/usr/local/cuda-10.0/include/cudalibxt.h" "$(@D)/cuda/include/cudalibxt.h" && cp -f "/usr/local/cuda-10.0/include/cudart_platform.h" "$(@D)/cuda/include/cudart_platform.h" && cp -f "/usr/local/cuda-10.0/include/cufft.h" "$(@D)/cuda/include/cufft.h" && cp -f "/usr/local/cuda-10.0/include/cufftXt.h" "$(@D)/cuda/include/cufftXt.h" && cp -f "/usr/local/cuda-10.0/include/cufftw.h" "$(@D)/cuda/include/cufftw.h" && cp -f "/usr/local/cuda-10.0/include/curand.h" "$(@D)/cuda/include/curand.h" && cp -f "/usr/local/cuda-10.0/include/curand_discrete.h" "$(@D)/cuda/include/curand_discrete.h" && cp -f "/usr/local/cuda-10.0/include/curand_discrete2.h" "$(@D)/cuda/include/curand_discrete2.h" && cp -f "/usr/local/cuda-10.0/include/curand_globals.h" "$(@D)/cuda/include/curand_globals.h" && cp -f "/usr/local/cuda-10.0/include/curand_kernel.h" "$(@D)/cuda/include/curand_kernel.h" && cp -f "/usr/local/cuda-10.0/include/curand_lognormal.h" "$(@D)/cuda/include/curand_lognormal.h" && cp -f "/usr/local/cuda-10.0/include/curand_mrg32k3a.h" "$(@D)/cuda/include/curand_mrg32k3a.h" && cp -f "/usr/local/cuda-10.0/include/curand_mtgp32.h" "$(@D)/cuda/include/curand_mtgp32.h" && cp -f "/usr/local/cuda-10.0/include/curand_mtgp32_host.h" "$(@D)/cuda/include/curand_mtgp32_host.h" && cp -f "/usr/local/cuda-10.0/include/curand_mtgp32_kernel.h" "$(@D)/cuda/include/curand_mtgp32_kernel.h" && cp -f "/usr/local/cuda-10.0/include/curand_mtgp32dc_p_11213.h" "$(@D)/cuda/include/curand_mtgp32dc_p_11213.h" && cp -f "/usr/local/cuda-10.0/include/curand_normal.h" "$(@D)/cuda/include/curand_normal.h" && cp -f "/usr/local/cuda-10.0/include/curand_normal_static.h" "$(@D)/cuda/include/curand_normal_static.h" && cp -f "/usr/local/cuda-10.0/include/curand_philox4x32_x.h" "$(@D)/cuda/include/curand_philox4x32_x.h" && cp -f "/usr/local/cuda-10.0/include/curand_poisson.h" "$(@D)/cuda/include/curand_poisson.h" && cp -f "/usr/local/cuda-10.0/include/curand_precalc.h" "$(@D)/cuda/include/curand_precalc.h" && cp -f "/usr/local/cuda-10.0/include/curand_uniform.h" "$(@D)/cuda/include/curand_uniform.h" && cp -f "/usr/local/cuda-10.0/include/cusolverDn.h" "$(@D)/cuda/include/cusolverDn.h" && cp -f "/usr/local/cuda-10.0/include/cusolverRf.h" "$(@D)/cuda/include/cusolverRf.h" && cp -f "/usr/local/cuda-10.0/include/cusolverSp.h" "$(@D)/cuda/include/cusolverSp.h" && cp -f "/usr/local/cuda-10.0/include/cusolverSp_LOWLEVEL_PREVIEW.h" "$(@D)/cuda/include/cusolverSp_LOWLEVEL_PREVIEW.h" && cp -f "/usr/local/cuda-10.0/include/cusolver_common.h" "$(@D)/cuda/include/cusolver_common.h" && cp -f "/usr/local/cuda-10.0/include/cusparse.h" "$(@D)/cuda/include/cusparse.h" && cp -f "/usr/local/cuda-10.0/include/cusparse_v2.h" "$(@D)/cuda/include/cusparse_v2.h" && cp -f "/usr/local/cuda-10.0/include/device_atomic_functions.h" "$(@D)/cuda/include/device_atomic_functions.h" && cp -f "/usr/local/cuda-10.0/include/device_atomic_functions.hpp" "$(@D)/cuda/include/device_atomic_functions.hpp" && cp -f "/usr/local/cuda-10.0/include/device_double_functions.h" "$(@D)/cuda/include/device_double_functions.h" && cp -f "/usr/local/cuda-10.0/include/device_functions.h" "$(@D)/cuda/include/device_functions.h" && cp -f "/usr/local/cuda-10.0/include/device_launch_parameters.h" "$(@D)/cuda/include/device_launch_parameters.h" && cp -f "/usr/local/cuda-10.0/include/device_types.h" "$(@D)/cuda/include/device_types.h" && cp -f "/usr/local/cuda-10.0/include/driver_functions.h" "$(@D)/cuda/include/driver_functions.h" && cp -f "/usr/local/cuda-10.0/include/driver_types.h" "$(@D)/cuda/include/driver_types.h" && cp -f "/usr/local/cuda-10.0/include/fatBinaryCtl.h" "$(@D)/cuda/include/fatBinaryCtl.h" && cp -f "/usr/local/cuda-10.0/include/fatbinary.h" "$(@D)/cuda/include/fatbinary.h" && cp -f "/usr/local/cuda-10.0/include/host_config.h" "$(@D)/cuda/include/host_config.h" && cp -f "/usr/local/cuda-10.0/include/host_defines.h" "$(@D)/cuda/include/host_defines.h" && cp -f "/usr/local/cuda-10.0/include/library_types.h" "$(@D)/cuda/include/library_types.h" && cp -f "/usr/local/cuda-10.0/include/math_constants.h" "$(@D)/cuda/include/math_constants.h" && cp -f "/usr/local/cuda-10.0/include/math_functions.h" "$(@D)/cuda/include/math_functions.h" && cp -f "/usr/local/cuda-10.0/include/mma.h" "$(@D)/cuda/include/mma.h" && cp -f "/usr/local/cuda-10.0/include/npp.h" "$(@D)/cuda/include/npp.h" && cp -f "/usr/local/cuda-10.0/include/nppcore.h" "$(@D)/cuda/include/nppcore.h" && cp -f "/usr/local/cuda-10.0/include/nppdefs.h" "$(@D)/cuda/include/nppdefs.h" && cp -f "/usr/local/cuda-10.0/include/nppi.h" "$(@D)/cuda/include/nppi.h" && cp -f "/usr/local/cuda-10.0/include/nppi_arithmetic_and_logical_operations.h" "$(@D)/cuda/include/nppi_arithmetic_and_logical_operations.h" && cp -f "/usr/local/cuda-10.0/include/nppi_color_conversion.h" "$(@D)/cuda/include/nppi_color_conversion.h" && cp -f "/usr/local/cuda-10.0/include/nppi_compression_functions.h" "$(@D)/cuda/include/nppi_compression_functions.h" && cp -f "/usr/local/cuda-10.0/include/nppi_computer_vision.h" "$(@D)/cuda/include/nppi_computer_vision.h" && cp -f "/usr/local/cuda-10.0/include/nppi_data_exchange_and_initialization.h" "$(@D)/cuda/include/nppi_data_exchange_and_initialization.h" && cp -f "/usr/local/cuda-10.0/include/nppi_filtering_functions.h" "$(@D)/cuda/include/nppi_filtering_functions.h" && cp -f "/usr/local/cuda-10.0/include/nppi_geometry_transforms.h" "$(@D)/cuda/include/nppi_geometry_transforms.h" && cp -f "/usr/local/cuda-10.0/include/nppi_linear_transforms.h" "$(@D)/cuda/include/nppi_linear_transforms.h" && cp -f "/usr/local/cuda-10.0/include/nppi_morphological_operations.h" "$(@D)/cuda/include/nppi_morphological_operations.h" && cp -f "/usr/local/cuda-10.0/include/nppi_statistics_functions.h" "$(@D)/cuda/include/nppi_statistics_functions.h" && cp -f "/usr/local/cuda-10.0/include/nppi_support_functions.h" "$(@D)/cuda/include/nppi_support_functions.h" && cp -f "/usr/local/cuda-10.0/include/nppi_threshold_and_compare_operations.h" "$(@D)/cuda/include/nppi_threshold_and_compare_operations.h" && cp -f "/usr/local/cuda-10.0/include/npps.h" "$(@D)/cuda/include/npps.h" && cp -f "/usr/local/cuda-10.0/include/npps_arithmetic_and_logical_operations.h" "$(@D)/cuda/include/npps_arithmetic_and_logical_operations.h" && cp -f "/usr/local/cuda-10.0/include/npps_conversion_functions.h" "$(@D)/cuda/include/npps_conversion_functions.h" && cp -f "/usr/local/cuda-10.0/include/npps_filtering_functions.h" "$(@D)/cuda/include/npps_filtering_functions.h" && cp -f "/usr/local/cuda-10.0/include/npps_initialization.h" "$(@D)/cuda/include/npps_initialization.h" && cp -f "/usr/local/cuda-10.0/include/npps_statistics_functions.h" "$(@D)/cuda/include/npps_statistics_functions.h" && cp -f "/usr/local/cuda-10.0/include/npps_support_functions.h" "$(@D)/cuda/include/npps_support_functions.h" && cp -f "/usr/local/cuda-10.0/include/nppversion.h" "$(@D)/cuda/include/nppversion.h" && cp -f "/usr/local/cuda-10.0/include/nvToolsExt.h" "$(@D)/cuda/include/nvToolsExt.h" && cp -f "/usr/local/cuda-10.0/include/nvToolsExtCuda.h" "$(@D)/cuda/include/nvToolsExtCuda.h" && cp -f "/usr/local/cuda-10.0/include/nvToolsExtCudaRt.h" "$(@D)/cuda/include/nvToolsExtCudaRt.h" && cp -f "/usr/local/cuda-10.0/include/nvToolsExtMeta.h" "$(@D)/cuda/include/nvToolsExtMeta.h" && cp -f "/usr/local/cuda-10.0/include/nvToolsExtSync.h" "$(@D)/cuda/include/nvToolsExtSync.h" && cp -f "/usr/local/cuda-10.0/include/nvblas.h" "$(@D)/cuda/include/nvblas.h" && cp -f "/usr/local/cuda-10.0/include/nvfunctional" "$(@D)/cuda/include/nvfunctional" && cp -f "/usr/local/cuda-10.0/include/nvgraph.h" "$(@D)/cuda/include/nvgraph.h" && cp -f "/usr/local/cuda-10.0/include/nvjpeg.h" "$(@D)/cuda/include/nvjpeg.h" && cp -f "/usr/local/cuda-10.0/include/nvml.h" "$(@D)/cuda/include/nvml.h" && cp -f "/usr/local/cuda-10.0/include/nvrtc.h" "$(@D)/cuda/include/nvrtc.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvToolsExt.h" "$(@D)/cuda/include/nvtx3/nvToolsExt.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvToolsExtCuda.h" "$(@D)/cuda/include/nvtx3/nvToolsExtCuda.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvToolsExtCudaRt.h" "$(@D)/cuda/include/nvtx3/nvToolsExtCudaRt.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvToolsExtOpenCL.h" "$(@D)/cuda/include/nvtx3/nvToolsExtOpenCL.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvToolsExtSync.h" "$(@D)/cuda/include/nvtx3/nvToolsExtSync.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvtxDetail/nvtxImpl.h" "$(@D)/cuda/include/nvtx3/nvtxDetail/nvtxImpl.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvtxDetail/nvtxImplCore.h" "$(@D)/cuda/include/nvtx3/nvtxDetail/nvtxImplCore.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvtxDetail/nvtxImplCudaRt_v3.h" "$(@D)/cuda/include/nvtx3/nvtxDetail/nvtxImplCudaRt_v3.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvtxDetail/nvtxImplCuda_v3.h" "$(@D)/cuda/include/nvtx3/nvtxDetail/nvtxImplCuda_v3.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvtxDetail/nvtxImplOpenCL_v3.h" "$(@D)/cuda/include/nvtx3/nvtxDetail/nvtxImplOpenCL_v3.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvtxDetail/nvtxImplSync_v3.h" "$(@D)/cuda/include/nvtx3/nvtxDetail/nvtxImplSync_v3.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvtxDetail/nvtxInit.h" "$(@D)/cuda/include/nvtx3/nvtxDetail/nvtxInit.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvtxDetail/nvtxInitDecls.h" "$(@D)/cuda/include/nvtx3/nvtxDetail/nvtxInitDecls.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvtxDetail/nvtxInitDefs.h" "$(@D)/cuda/include/nvtx3/nvtxDetail/nvtxInitDefs.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvtxDetail/nvtxLinkOnce.h" "$(@D)/cuda/include/nvtx3/nvtxDetail/nvtxLinkOnce.h" && cp -f "/usr/local/cuda-10.0/include/nvtx3/nvtxDetail/nvtxTypes.h" "$(@D)/cuda/include/nvtx3/nvtxDetail/nvtxTypes.h" && cp -f "/usr/local/cuda-10.0/include/sm_20_atomic_functions.h" "$(@D)/cuda/include/sm_20_atomic_functions.h" && cp -f "/usr/local/cuda-10.0/include/sm_20_atomic_functions.hpp" "$(@D)/cuda/include/sm_20_atomic_functions.hpp" && cp -f "/usr/local/cuda-10.0/include/sm_20_intrinsics.h" "$(@D)/cuda/include/sm_20_intrinsics.h" && cp -f "/usr/local/cuda-10.0/include/sm_20_intrinsics.hpp" "$(@D)/cuda/include/sm_20_intrinsics.hpp" && cp -f "/usr/local/cuda-10.0/include/sm_30_intrinsics.h" "$(@D)/cuda/include/sm_30_intrinsics.h" && cp -f "/usr/local/cuda-10.0/include/sm_30_intrinsics.hpp" "$(@D)/cuda/include/sm_30_intrinsics.hpp" && cp -f "/usr/local/cuda-10.0/include/sm_32_atomic_functions.h" "$(@D)/cuda/include/sm_32_atomic_functions.h" && cp -f "/usr/local/cuda-10.0/include/sm_32_atomic_functions.hpp" "$(@D)/cuda/include/sm_32_atomic_functions.hpp" && cp -f "/usr/local/cuda-10.0/include/sm_32_intrinsics.h" "$(@D)/cuda/include/sm_32_intrinsics.h" && cp -f "/usr/local/cuda-10.0/include/sm_32_intrinsics.hpp" "$(@D)/cuda/include/sm_32_intrinsics.hpp" && cp -f "/usr/local/cuda-10.0/include/sm_35_atomic_functions.h" "$(@D)/cuda/include/sm_35_atomic_functions.h" && cp -f "/usr/local/cuda-10.0/include/sm_35_intrinsics.h" "$(@D)/cuda/include/sm_35_intrinsics.h" && cp -f "/usr/local/cuda-10.0/include/sm_60_atomic_functions.h" "$(@D)/cuda/include/sm_60_atomic_functions.h" && cp -f "/usr/local/cuda-10.0/include/sm_60_atomic_functions.hpp" "$(@D)/cuda/include/sm_60_atomic_functions.hpp" && cp -f "/usr/local/cuda-10.0/include/sm_61_intrinsics.h" "$(@D)/cuda/include/sm_61_intrinsics.h" && cp -f "/usr/local/cuda-10.0/include/sm_61_intrinsics.hpp" "$(@D)/cuda/include/sm_61_intrinsics.hpp" && cp -f "/usr/local/cuda-10.0/include/sobol_direction_vectors.h" "$(@D)/cuda/include/sobol_direction_vectors.h" && cp -f "/usr/local/cuda-10.0/include/surface_functions.h" "$(@D)/cuda/include/surface_functions.h" && cp -f "/usr/local/cuda-10.0/include/surface_functions.hpp" "$(@D)/cuda/include/surface_functions.hpp" && cp -f "/usr/local/cuda-10.0/include/surface_indirect_functions.h" "$(@D)/cuda/include/surface_indirect_functions.h" && cp -f "/usr/local/cuda-10.0/include/surface_indirect_functions.hpp" "$(@D)/cuda/include/surface_indirect_functions.hpp" && cp -f "/usr/local/cuda-10.0/include/surface_types.h" "$(@D)/cuda/include/surface_types.h" && cp -f "/usr/local/cuda-10.0/include/texture_fetch_functions.h" "$(@D)/cuda/include/texture_fetch_functions.h" && cp -f "/usr/local/cuda-10.0/include/texture_fetch_functions.hpp" "$(@D)/cuda/include/texture_fetch_functions.hpp" && cp -f "/usr/local/cuda-10.0/include/texture_indirect_functions.h" "$(@D)/cuda/include/texture_indirect_functions.h" && cp -f "/usr/local/cuda-10.0/include/texture_indirect_functions.hpp" "$(@D)/cuda/include/texture_indirect_functions.hpp" && cp -f "/usr/local/cuda-10.0/include/texture_types.h" "$(@D)/cuda/include/texture_types.h" && cp -f "/usr/local/cuda-10.0/include/thrust/adjacent_difference.h" "$(@D)/cuda/include/thrust/adjacent_difference.h" && cp -f "/usr/local/cuda-10.0/include/thrust/advance.h" "$(@D)/cuda/include/thrust/advance.h" && cp -f "/usr/local/cuda-10.0/include/thrust/binary_search.h" "$(@D)/cuda/include/thrust/binary_search.h" && cp -f "/usr/local/cuda-10.0/include/thrust/complex.h" "$(@D)/cuda/include/thrust/complex.h" && cp -f "/usr/local/cuda-10.0/include/thrust/copy.h" "$(@D)/cuda/include/thrust/copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/count.h" "$(@D)/cuda/include/thrust/count.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/adjacent_difference.inl" "$(@D)/cuda/include/thrust/detail/adjacent_difference.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/advance.inl" "$(@D)/cuda/include/thrust/detail/advance.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/alignment.h" "$(@D)/cuda/include/thrust/detail/alignment.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/allocator_traits.h" "$(@D)/cuda/include/thrust/detail/allocator/allocator_traits.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/allocator_traits.inl" "$(@D)/cuda/include/thrust/detail/allocator/allocator_traits.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/copy_construct_range.h" "$(@D)/cuda/include/thrust/detail/allocator/copy_construct_range.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/copy_construct_range.inl" "$(@D)/cuda/include/thrust/detail/allocator/copy_construct_range.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/default_construct_range.h" "$(@D)/cuda/include/thrust/detail/allocator/default_construct_range.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/default_construct_range.inl" "$(@D)/cuda/include/thrust/detail/allocator/default_construct_range.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/destroy_range.h" "$(@D)/cuda/include/thrust/detail/allocator/destroy_range.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/destroy_range.inl" "$(@D)/cuda/include/thrust/detail/allocator/destroy_range.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/fill_construct_range.h" "$(@D)/cuda/include/thrust/detail/allocator/fill_construct_range.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/fill_construct_range.inl" "$(@D)/cuda/include/thrust/detail/allocator/fill_construct_range.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/malloc_allocator.h" "$(@D)/cuda/include/thrust/detail/allocator/malloc_allocator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/malloc_allocator.inl" "$(@D)/cuda/include/thrust/detail/allocator/malloc_allocator.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/no_throw_allocator.h" "$(@D)/cuda/include/thrust/detail/allocator/no_throw_allocator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/tagged_allocator.h" "$(@D)/cuda/include/thrust/detail/allocator/tagged_allocator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/tagged_allocator.inl" "$(@D)/cuda/include/thrust/detail/allocator/tagged_allocator.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/temporary_allocator.h" "$(@D)/cuda/include/thrust/detail/allocator/temporary_allocator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/allocator/temporary_allocator.inl" "$(@D)/cuda/include/thrust/detail/allocator/temporary_allocator.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/binary_search.inl" "$(@D)/cuda/include/thrust/detail/binary_search.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/arithmetic.h" "$(@D)/cuda/include/thrust/detail/complex/arithmetic.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/c99math.h" "$(@D)/cuda/include/thrust/detail/complex/c99math.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/catrig.h" "$(@D)/cuda/include/thrust/detail/complex/catrig.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/catrigf.h" "$(@D)/cuda/include/thrust/detail/complex/catrigf.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/ccosh.h" "$(@D)/cuda/include/thrust/detail/complex/ccosh.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/ccoshf.h" "$(@D)/cuda/include/thrust/detail/complex/ccoshf.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/cexp.h" "$(@D)/cuda/include/thrust/detail/complex/cexp.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/cexpf.h" "$(@D)/cuda/include/thrust/detail/complex/cexpf.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/clog.h" "$(@D)/cuda/include/thrust/detail/complex/clog.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/clogf.h" "$(@D)/cuda/include/thrust/detail/complex/clogf.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/complex.inl" "$(@D)/cuda/include/thrust/detail/complex/complex.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/cpow.h" "$(@D)/cuda/include/thrust/detail/complex/cpow.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/cproj.h" "$(@D)/cuda/include/thrust/detail/complex/cproj.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/csinh.h" "$(@D)/cuda/include/thrust/detail/complex/csinh.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/csinhf.h" "$(@D)/cuda/include/thrust/detail/complex/csinhf.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/csqrt.h" "$(@D)/cuda/include/thrust/detail/complex/csqrt.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/csqrtf.h" "$(@D)/cuda/include/thrust/detail/complex/csqrtf.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/ctanh.h" "$(@D)/cuda/include/thrust/detail/complex/ctanh.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/ctanhf.h" "$(@D)/cuda/include/thrust/detail/complex/ctanhf.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/math_private.h" "$(@D)/cuda/include/thrust/detail/complex/math_private.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/complex/stream.h" "$(@D)/cuda/include/thrust/detail/complex/stream.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/config.h" "$(@D)/cuda/include/thrust/detail/config.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/config/compiler.h" "$(@D)/cuda/include/thrust/detail/config/compiler.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/config/compiler_fence.h" "$(@D)/cuda/include/thrust/detail/config/compiler_fence.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/config/config.h" "$(@D)/cuda/include/thrust/detail/config/config.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/config/debug.h" "$(@D)/cuda/include/thrust/detail/config/debug.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/config/device_system.h" "$(@D)/cuda/include/thrust/detail/config/device_system.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/config/exec_check_disable.h" "$(@D)/cuda/include/thrust/detail/config/exec_check_disable.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/config/forceinline.h" "$(@D)/cuda/include/thrust/detail/config/forceinline.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/config/global_workarounds.h" "$(@D)/cuda/include/thrust/detail/config/global_workarounds.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/config/host_device.h" "$(@D)/cuda/include/thrust/detail/config/host_device.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/config/host_system.h" "$(@D)/cuda/include/thrust/detail/config/host_system.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/config/simple_defines.h" "$(@D)/cuda/include/thrust/detail/config/simple_defines.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/contiguous_storage.h" "$(@D)/cuda/include/thrust/detail/contiguous_storage.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/contiguous_storage.inl" "$(@D)/cuda/include/thrust/detail/contiguous_storage.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/copy.h" "$(@D)/cuda/include/thrust/detail/copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/copy.inl" "$(@D)/cuda/include/thrust/detail/copy.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/copy_if.h" "$(@D)/cuda/include/thrust/detail/copy_if.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/copy_if.inl" "$(@D)/cuda/include/thrust/detail/copy_if.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/count.inl" "$(@D)/cuda/include/thrust/detail/count.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/cstdint.h" "$(@D)/cuda/include/thrust/detail/cstdint.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/device_delete.inl" "$(@D)/cuda/include/thrust/detail/device_delete.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/device_free.inl" "$(@D)/cuda/include/thrust/detail/device_free.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/device_malloc.inl" "$(@D)/cuda/include/thrust/detail/device_malloc.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/device_new.inl" "$(@D)/cuda/include/thrust/detail/device_new.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/device_ptr.inl" "$(@D)/cuda/include/thrust/detail/device_ptr.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/device_reference.inl" "$(@D)/cuda/include/thrust/detail/device_reference.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/device_vector.inl" "$(@D)/cuda/include/thrust/detail/device_vector.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/dispatch/is_trivial_copy.h" "$(@D)/cuda/include/thrust/detail/dispatch/is_trivial_copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/distance.inl" "$(@D)/cuda/include/thrust/detail/distance.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/equal.inl" "$(@D)/cuda/include/thrust/detail/equal.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/execute_with_allocator.h" "$(@D)/cuda/include/thrust/detail/execute_with_allocator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/execution_policy.h" "$(@D)/cuda/include/thrust/detail/execution_policy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/extrema.inl" "$(@D)/cuda/include/thrust/detail/extrema.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/fill.inl" "$(@D)/cuda/include/thrust/detail/fill.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/find.inl" "$(@D)/cuda/include/thrust/detail/find.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/for_each.inl" "$(@D)/cuda/include/thrust/detail/for_each.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/function.h" "$(@D)/cuda/include/thrust/detail/function.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/functional.inl" "$(@D)/cuda/include/thrust/detail/functional.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/functional/actor.h" "$(@D)/cuda/include/thrust/detail/functional/actor.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/functional/actor.inl" "$(@D)/cuda/include/thrust/detail/functional/actor.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/functional/argument.h" "$(@D)/cuda/include/thrust/detail/functional/argument.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/functional/composite.h" "$(@D)/cuda/include/thrust/detail/functional/composite.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/functional/operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/functional/operators/arithmetic_operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators/arithmetic_operators.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/functional/operators/assignment_operator.h" "$(@D)/cuda/include/thrust/detail/functional/operators/assignment_operator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/functional/operators/bitwise_operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators/bitwise_operators.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/functional/operators/compound_assignment_operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators/compound_assignment_operators.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/functional/operators/logical_operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators/logical_operators.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/functional/operators/operator_adaptors.h" "$(@D)/cuda/include/thrust/detail/functional/operators/operator_adaptors.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/functional/operators/relational_operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators/relational_operators.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/functional/placeholder.h" "$(@D)/cuda/include/thrust/detail/functional/placeholder.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/functional/value.h" "$(@D)/cuda/include/thrust/detail/functional/value.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/gather.inl" "$(@D)/cuda/include/thrust/detail/gather.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/generate.inl" "$(@D)/cuda/include/thrust/detail/generate.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/get_iterator_value.h" "$(@D)/cuda/include/thrust/detail/get_iterator_value.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/host_vector.inl" "$(@D)/cuda/include/thrust/detail/host_vector.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/inner_product.inl" "$(@D)/cuda/include/thrust/detail/inner_product.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/integer_math.h" "$(@D)/cuda/include/thrust/detail/integer_math.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/integer_traits.h" "$(@D)/cuda/include/thrust/detail/integer_traits.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/internal_functional.h" "$(@D)/cuda/include/thrust/detail/internal_functional.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/logical.inl" "$(@D)/cuda/include/thrust/detail/logical.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/malloc_and_free.h" "$(@D)/cuda/include/thrust/detail/malloc_and_free.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/merge.inl" "$(@D)/cuda/include/thrust/detail/merge.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/minmax.h" "$(@D)/cuda/include/thrust/detail/minmax.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/mismatch.inl" "$(@D)/cuda/include/thrust/detail/mismatch.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/mpl/math.h" "$(@D)/cuda/include/thrust/detail/mpl/math.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/numeric_traits.h" "$(@D)/cuda/include/thrust/detail/numeric_traits.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/overlapped_copy.h" "$(@D)/cuda/include/thrust/detail/overlapped_copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/pair.inl" "$(@D)/cuda/include/thrust/detail/pair.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/partition.inl" "$(@D)/cuda/include/thrust/detail/partition.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/pointer.h" "$(@D)/cuda/include/thrust/detail/pointer.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/pointer.inl" "$(@D)/cuda/include/thrust/detail/pointer.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/preprocessor.h" "$(@D)/cuda/include/thrust/detail/preprocessor.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/range/head_flags.h" "$(@D)/cuda/include/thrust/detail/range/head_flags.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/range/tail_flags.h" "$(@D)/cuda/include/thrust/detail/range/tail_flags.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/raw_pointer_cast.h" "$(@D)/cuda/include/thrust/detail/raw_pointer_cast.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/raw_reference_cast.h" "$(@D)/cuda/include/thrust/detail/raw_reference_cast.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/reduce.inl" "$(@D)/cuda/include/thrust/detail/reduce.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/reference.h" "$(@D)/cuda/include/thrust/detail/reference.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/reference.inl" "$(@D)/cuda/include/thrust/detail/reference.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/reference_forward_declaration.h" "$(@D)/cuda/include/thrust/detail/reference_forward_declaration.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/remove.inl" "$(@D)/cuda/include/thrust/detail/remove.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/replace.inl" "$(@D)/cuda/include/thrust/detail/replace.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/reverse.inl" "$(@D)/cuda/include/thrust/detail/reverse.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/scan.inl" "$(@D)/cuda/include/thrust/detail/scan.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/scatter.inl" "$(@D)/cuda/include/thrust/detail/scatter.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/seq.h" "$(@D)/cuda/include/thrust/detail/seq.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/sequence.inl" "$(@D)/cuda/include/thrust/detail/sequence.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/set_operations.inl" "$(@D)/cuda/include/thrust/detail/set_operations.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/sort.inl" "$(@D)/cuda/include/thrust/detail/sort.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/static_assert.h" "$(@D)/cuda/include/thrust/detail/static_assert.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/static_map.h" "$(@D)/cuda/include/thrust/detail/static_map.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/swap.h" "$(@D)/cuda/include/thrust/detail/swap.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/swap.inl" "$(@D)/cuda/include/thrust/detail/swap.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/swap_ranges.inl" "$(@D)/cuda/include/thrust/detail/swap_ranges.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/tabulate.inl" "$(@D)/cuda/include/thrust/detail/tabulate.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/temporary_array.h" "$(@D)/cuda/include/thrust/detail/temporary_array.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/temporary_array.inl" "$(@D)/cuda/include/thrust/detail/temporary_array.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/temporary_buffer.h" "$(@D)/cuda/include/thrust/detail/temporary_buffer.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/transform.inl" "$(@D)/cuda/include/thrust/detail/transform.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/transform_reduce.inl" "$(@D)/cuda/include/thrust/detail/transform_reduce.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/transform_scan.inl" "$(@D)/cuda/include/thrust/detail/transform_scan.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/trivial_sequence.h" "$(@D)/cuda/include/thrust/detail/trivial_sequence.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/tuple.inl" "$(@D)/cuda/include/thrust/detail/tuple.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/tuple_meta_transform.h" "$(@D)/cuda/include/thrust/detail/tuple_meta_transform.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/tuple_transform.h" "$(@D)/cuda/include/thrust/detail/tuple_transform.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/type_traits.h" "$(@D)/cuda/include/thrust/detail/type_traits.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/type_traits/algorithm/intermediate_type_from_function_and_iterators.h" "$(@D)/cuda/include/thrust/detail/type_traits/algorithm/intermediate_type_from_function_and_iterators.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/type_traits/function_traits.h" "$(@D)/cuda/include/thrust/detail/type_traits/function_traits.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/type_traits/has_member_function.h" "$(@D)/cuda/include/thrust/detail/type_traits/has_member_function.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/type_traits/has_nested_type.h" "$(@D)/cuda/include/thrust/detail/type_traits/has_nested_type.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/type_traits/has_trivial_assign.h" "$(@D)/cuda/include/thrust/detail/type_traits/has_trivial_assign.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/type_traits/is_call_possible.h" "$(@D)/cuda/include/thrust/detail/type_traits/is_call_possible.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/type_traits/is_metafunction_defined.h" "$(@D)/cuda/include/thrust/detail/type_traits/is_metafunction_defined.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/type_traits/iterator/is_discard_iterator.h" "$(@D)/cuda/include/thrust/detail/type_traits/iterator/is_discard_iterator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/type_traits/iterator/is_output_iterator.h" "$(@D)/cuda/include/thrust/detail/type_traits/iterator/is_output_iterator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/type_traits/minimum_type.h" "$(@D)/cuda/include/thrust/detail/type_traits/minimum_type.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/type_traits/pointer_traits.h" "$(@D)/cuda/include/thrust/detail/type_traits/pointer_traits.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/type_traits/result_of_adaptable_function.h" "$(@D)/cuda/include/thrust/detail/type_traits/result_of_adaptable_function.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/uninitialized_copy.inl" "$(@D)/cuda/include/thrust/detail/uninitialized_copy.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/uninitialized_fill.inl" "$(@D)/cuda/include/thrust/detail/uninitialized_fill.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/unique.inl" "$(@D)/cuda/include/thrust/detail/unique.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/use_default.h" "$(@D)/cuda/include/thrust/detail/use_default.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/util/align.h" "$(@D)/cuda/include/thrust/detail/util/align.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/util/blocking.h" "$(@D)/cuda/include/thrust/detail/util/blocking.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/vector_base.h" "$(@D)/cuda/include/thrust/detail/vector_base.h" && cp -f "/usr/local/cuda-10.0/include/thrust/detail/vector_base.inl" "$(@D)/cuda/include/thrust/detail/vector_base.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/device_allocator.h" "$(@D)/cuda/include/thrust/device_allocator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/device_delete.h" "$(@D)/cuda/include/thrust/device_delete.h" && cp -f "/usr/local/cuda-10.0/include/thrust/device_free.h" "$(@D)/cuda/include/thrust/device_free.h" && cp -f "/usr/local/cuda-10.0/include/thrust/device_malloc.h" "$(@D)/cuda/include/thrust/device_malloc.h" && cp -f "/usr/local/cuda-10.0/include/thrust/device_malloc_allocator.h" "$(@D)/cuda/include/thrust/device_malloc_allocator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/device_new.h" "$(@D)/cuda/include/thrust/device_new.h" && cp -f "/usr/local/cuda-10.0/include/thrust/device_new_allocator.h" "$(@D)/cuda/include/thrust/device_new_allocator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/device_ptr.h" "$(@D)/cuda/include/thrust/device_ptr.h" && cp -f "/usr/local/cuda-10.0/include/thrust/device_reference.h" "$(@D)/cuda/include/thrust/device_reference.h" && cp -f "/usr/local/cuda-10.0/include/thrust/device_vector.h" "$(@D)/cuda/include/thrust/device_vector.h" && cp -f "/usr/local/cuda-10.0/include/thrust/distance.h" "$(@D)/cuda/include/thrust/distance.h" && cp -f "/usr/local/cuda-10.0/include/thrust/equal.h" "$(@D)/cuda/include/thrust/equal.h" && cp -f "/usr/local/cuda-10.0/include/thrust/execution_policy.h" "$(@D)/cuda/include/thrust/execution_policy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/extrema.h" "$(@D)/cuda/include/thrust/extrema.h" && cp -f "/usr/local/cuda-10.0/include/thrust/fill.h" "$(@D)/cuda/include/thrust/fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/find.h" "$(@D)/cuda/include/thrust/find.h" && cp -f "/usr/local/cuda-10.0/include/thrust/for_each.h" "$(@D)/cuda/include/thrust/for_each.h" && cp -f "/usr/local/cuda-10.0/include/thrust/functional.h" "$(@D)/cuda/include/thrust/functional.h" && cp -f "/usr/local/cuda-10.0/include/thrust/gather.h" "$(@D)/cuda/include/thrust/gather.h" && cp -f "/usr/local/cuda-10.0/include/thrust/generate.h" "$(@D)/cuda/include/thrust/generate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/host_vector.h" "$(@D)/cuda/include/thrust/host_vector.h" && cp -f "/usr/local/cuda-10.0/include/thrust/inner_product.h" "$(@D)/cuda/include/thrust/inner_product.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/constant_iterator.h" "$(@D)/cuda/include/thrust/iterator/constant_iterator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/counting_iterator.h" "$(@D)/cuda/include/thrust/iterator/counting_iterator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/any_assign.h" "$(@D)/cuda/include/thrust/iterator/detail/any_assign.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/any_system_tag.h" "$(@D)/cuda/include/thrust/iterator/detail/any_system_tag.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/constant_iterator_base.h" "$(@D)/cuda/include/thrust/iterator/detail/constant_iterator_base.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/counting_iterator.inl" "$(@D)/cuda/include/thrust/iterator/detail/counting_iterator.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/device_system_tag.h" "$(@D)/cuda/include/thrust/iterator/detail/device_system_tag.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/discard_iterator_base.h" "$(@D)/cuda/include/thrust/iterator/detail/discard_iterator_base.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/distance_from_result.h" "$(@D)/cuda/include/thrust/iterator/detail/distance_from_result.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/host_system_tag.h" "$(@D)/cuda/include/thrust/iterator/detail/host_system_tag.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/is_iterator_category.h" "$(@D)/cuda/include/thrust/iterator/detail/is_iterator_category.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/is_trivial_iterator.h" "$(@D)/cuda/include/thrust/iterator/detail/is_trivial_iterator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/iterator_adaptor_base.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_adaptor_base.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/iterator_category_to_system.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_category_to_system.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/iterator_category_to_traversal.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_category_to_traversal.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/iterator_category_with_system_and_traversal.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_category_with_system_and_traversal.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/iterator_facade_category.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_facade_category.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/iterator_traits.inl" "$(@D)/cuda/include/thrust/iterator/detail/iterator_traits.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/iterator_traversal_tags.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_traversal_tags.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/join_iterator.h" "$(@D)/cuda/include/thrust/iterator/detail/join_iterator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/minimum_category.h" "$(@D)/cuda/include/thrust/iterator/detail/minimum_category.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/minimum_system.h" "$(@D)/cuda/include/thrust/iterator/detail/minimum_system.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/normal_iterator.h" "$(@D)/cuda/include/thrust/iterator/detail/normal_iterator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/permutation_iterator_base.h" "$(@D)/cuda/include/thrust/iterator/detail/permutation_iterator_base.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/retag.h" "$(@D)/cuda/include/thrust/iterator/detail/retag.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/reverse_iterator.inl" "$(@D)/cuda/include/thrust/iterator/detail/reverse_iterator.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/reverse_iterator_base.h" "$(@D)/cuda/include/thrust/iterator/detail/reverse_iterator_base.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/tagged_iterator.h" "$(@D)/cuda/include/thrust/iterator/detail/tagged_iterator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/transform_iterator.inl" "$(@D)/cuda/include/thrust/iterator/detail/transform_iterator.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/transform_output_iterator.inl" "$(@D)/cuda/include/thrust/iterator/detail/transform_output_iterator.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/tuple_of_iterator_references.h" "$(@D)/cuda/include/thrust/iterator/detail/tuple_of_iterator_references.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/universal_categories.h" "$(@D)/cuda/include/thrust/iterator/detail/universal_categories.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/zip_iterator.inl" "$(@D)/cuda/include/thrust/iterator/detail/zip_iterator.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/detail/zip_iterator_base.h" "$(@D)/cuda/include/thrust/iterator/detail/zip_iterator_base.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/discard_iterator.h" "$(@D)/cuda/include/thrust/iterator/discard_iterator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/iterator_adaptor.h" "$(@D)/cuda/include/thrust/iterator/iterator_adaptor.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/iterator_categories.h" "$(@D)/cuda/include/thrust/iterator/iterator_categories.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/iterator_facade.h" "$(@D)/cuda/include/thrust/iterator/iterator_facade.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/iterator_traits.h" "$(@D)/cuda/include/thrust/iterator/iterator_traits.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/permutation_iterator.h" "$(@D)/cuda/include/thrust/iterator/permutation_iterator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/retag.h" "$(@D)/cuda/include/thrust/iterator/retag.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/reverse_iterator.h" "$(@D)/cuda/include/thrust/iterator/reverse_iterator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/transform_iterator.h" "$(@D)/cuda/include/thrust/iterator/transform_iterator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/transform_output_iterator.h" "$(@D)/cuda/include/thrust/iterator/transform_output_iterator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/iterator/zip_iterator.h" "$(@D)/cuda/include/thrust/iterator/zip_iterator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/logical.h" "$(@D)/cuda/include/thrust/logical.h" && cp -f "/usr/local/cuda-10.0/include/thrust/memory.h" "$(@D)/cuda/include/thrust/memory.h" && cp -f "/usr/local/cuda-10.0/include/thrust/merge.h" "$(@D)/cuda/include/thrust/merge.h" && cp -f "/usr/local/cuda-10.0/include/thrust/mismatch.h" "$(@D)/cuda/include/thrust/mismatch.h" && cp -f "/usr/local/cuda-10.0/include/thrust/pair.h" "$(@D)/cuda/include/thrust/pair.h" && cp -f "/usr/local/cuda-10.0/include/thrust/partition.h" "$(@D)/cuda/include/thrust/partition.h" && cp -f "/usr/local/cuda-10.0/include/thrust/random.h" "$(@D)/cuda/include/thrust/random.h" && cp -f "/usr/local/cuda-10.0/include/thrust/random/detail/discard_block_engine.inl" "$(@D)/cuda/include/thrust/random/detail/discard_block_engine.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/random/detail/linear_congruential_engine.inl" "$(@D)/cuda/include/thrust/random/detail/linear_congruential_engine.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/random/detail/linear_congruential_engine_discard.h" "$(@D)/cuda/include/thrust/random/detail/linear_congruential_engine_discard.h" && cp -f "/usr/local/cuda-10.0/include/thrust/random/detail/linear_feedback_shift_engine.inl" "$(@D)/cuda/include/thrust/random/detail/linear_feedback_shift_engine.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/random/detail/linear_feedback_shift_engine_wordmask.h" "$(@D)/cuda/include/thrust/random/detail/linear_feedback_shift_engine_wordmask.h" && cp -f "/usr/local/cuda-10.0/include/thrust/random/detail/mod.h" "$(@D)/cuda/include/thrust/random/detail/mod.h" && cp -f "/usr/local/cuda-10.0/include/thrust/random/detail/normal_distribution.inl" "$(@D)/cuda/include/thrust/random/detail/normal_distribution.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/random/detail/normal_distribution_base.h" "$(@D)/cuda/include/thrust/random/detail/normal_distribution_base.h" && cp -f "/usr/local/cuda-10.0/include/thrust/random/detail/random_core_access.h" "$(@D)/cuda/include/thrust/random/detail/random_core_access.h" && cp -f "/usr/local/cuda-10.0/include/thrust/random/detail/subtract_with_carry_engine.inl" "$(@D)/cuda/include/thrust/random/detail/subtract_with_carry_engine.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/random/detail/uniform_int_distribution.inl" "$(@D)/cuda/include/thrust/random/detail/uniform_int_distribution.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/random/detail/uniform_real_distribution.inl" "$(@D)/cuda/include/thrust/random/detail/uniform_real_distribution.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/random/detail/xor_combine_engine.inl" "$(@D)/cuda/include/thrust/random/detail/xor_combine_engine.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/random/detail/xor_combine_engine_max.h" "$(@D)/cuda/include/thrust/random/detail/xor_combine_engine_max.h" && cp -f "/usr/local/cuda-10.0/include/thrust/random/discard_block_engine.h" "$(@D)/cuda/include/thrust/random/discard_block_engine.h" && cp -f "/usr/local/cuda-10.0/include/thrust/random/linear_congruential_engine.h" "$(@D)/cuda/include/thrust/random/linear_congruential_engine.h" && cp -f "/usr/local/cuda-10.0/include/thrust/random/linear_feedback_shift_engine.h" "$(@D)/cuda/include/thrust/random/linear_feedback_shift_engine.h" && cp -f "/usr/local/cuda-10.0/include/thrust/random/normal_distribution.h" "$(@D)/cuda/include/thrust/random/normal_distribution.h" && cp -f "/usr/local/cuda-10.0/include/thrust/random/subtract_with_carry_engine.h" "$(@D)/cuda/include/thrust/random/subtract_with_carry_engine.h" && cp -f "/usr/local/cuda-10.0/include/thrust/random/uniform_int_distribution.h" "$(@D)/cuda/include/thrust/random/uniform_int_distribution.h" && cp -f "/usr/local/cuda-10.0/include/thrust/random/uniform_real_distribution.h" "$(@D)/cuda/include/thrust/random/uniform_real_distribution.h" && cp -f "/usr/local/cuda-10.0/include/thrust/random/xor_combine_engine.h" "$(@D)/cuda/include/thrust/random/xor_combine_engine.h" && cp -f "/usr/local/cuda-10.0/include/thrust/reduce.h" "$(@D)/cuda/include/thrust/reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/remove.h" "$(@D)/cuda/include/thrust/remove.h" && cp -f "/usr/local/cuda-10.0/include/thrust/replace.h" "$(@D)/cuda/include/thrust/replace.h" && cp -f "/usr/local/cuda-10.0/include/thrust/reverse.h" "$(@D)/cuda/include/thrust/reverse.h" && cp -f "/usr/local/cuda-10.0/include/thrust/scan.h" "$(@D)/cuda/include/thrust/scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/scatter.h" "$(@D)/cuda/include/thrust/scatter.h" && cp -f "/usr/local/cuda-10.0/include/thrust/sequence.h" "$(@D)/cuda/include/thrust/sequence.h" && cp -f "/usr/local/cuda-10.0/include/thrust/set_operations.h" "$(@D)/cuda/include/thrust/set_operations.h" && cp -f "/usr/local/cuda-10.0/include/thrust/sort.h" "$(@D)/cuda/include/thrust/sort.h" && cp -f "/usr/local/cuda-10.0/include/thrust/swap.h" "$(@D)/cuda/include/thrust/swap.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/cpp/detail/adjacent_difference.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/assign_value.h" "$(@D)/cuda/include/thrust/system/cpp/detail/assign_value.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/binary_search.h" "$(@D)/cuda/include/thrust/system/cpp/detail/binary_search.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/copy.h" "$(@D)/cuda/include/thrust/system/cpp/detail/copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/copy_if.h" "$(@D)/cuda/include/thrust/system/cpp/detail/copy_if.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/count.h" "$(@D)/cuda/include/thrust/system/cpp/detail/count.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/equal.h" "$(@D)/cuda/include/thrust/system/cpp/detail/equal.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/execution_policy.h" "$(@D)/cuda/include/thrust/system/cpp/detail/execution_policy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/extrema.h" "$(@D)/cuda/include/thrust/system/cpp/detail/extrema.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/fill.h" "$(@D)/cuda/include/thrust/system/cpp/detail/fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/find.h" "$(@D)/cuda/include/thrust/system/cpp/detail/find.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/for_each.h" "$(@D)/cuda/include/thrust/system/cpp/detail/for_each.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/gather.h" "$(@D)/cuda/include/thrust/system/cpp/detail/gather.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/generate.h" "$(@D)/cuda/include/thrust/system/cpp/detail/generate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/get_value.h" "$(@D)/cuda/include/thrust/system/cpp/detail/get_value.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/inner_product.h" "$(@D)/cuda/include/thrust/system/cpp/detail/inner_product.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/iter_swap.h" "$(@D)/cuda/include/thrust/system/cpp/detail/iter_swap.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/logical.h" "$(@D)/cuda/include/thrust/system/cpp/detail/logical.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/cpp/detail/malloc_and_free.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/memory.inl" "$(@D)/cuda/include/thrust/system/cpp/detail/memory.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/merge.h" "$(@D)/cuda/include/thrust/system/cpp/detail/merge.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/mismatch.h" "$(@D)/cuda/include/thrust/system/cpp/detail/mismatch.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/par.h" "$(@D)/cuda/include/thrust/system/cpp/detail/par.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/partition.h" "$(@D)/cuda/include/thrust/system/cpp/detail/partition.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/reduce.h" "$(@D)/cuda/include/thrust/system/cpp/detail/reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/cpp/detail/reduce_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/remove.h" "$(@D)/cuda/include/thrust/system/cpp/detail/remove.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/replace.h" "$(@D)/cuda/include/thrust/system/cpp/detail/replace.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/reverse.h" "$(@D)/cuda/include/thrust/system/cpp/detail/reverse.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/scan.h" "$(@D)/cuda/include/thrust/system/cpp/detail/scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/scan_by_key.h" "$(@D)/cuda/include/thrust/system/cpp/detail/scan_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/scatter.h" "$(@D)/cuda/include/thrust/system/cpp/detail/scatter.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/sequence.h" "$(@D)/cuda/include/thrust/system/cpp/detail/sequence.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/set_operations.h" "$(@D)/cuda/include/thrust/system/cpp/detail/set_operations.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/sort.h" "$(@D)/cuda/include/thrust/system/cpp/detail/sort.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/swap_ranges.h" "$(@D)/cuda/include/thrust/system/cpp/detail/swap_ranges.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/tabulate.h" "$(@D)/cuda/include/thrust/system/cpp/detail/tabulate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/cpp/detail/temporary_buffer.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/transform.h" "$(@D)/cuda/include/thrust/system/cpp/detail/transform.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/transform_reduce.h" "$(@D)/cuda/include/thrust/system/cpp/detail/transform_reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/transform_scan.h" "$(@D)/cuda/include/thrust/system/cpp/detail/transform_scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/cpp/detail/uninitialized_copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/cpp/detail/uninitialized_fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/unique.h" "$(@D)/cuda/include/thrust/system/cpp/detail/unique.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/unique_by_key.h" "$(@D)/cuda/include/thrust/system/cpp/detail/unique_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/detail/vector.inl" "$(@D)/cuda/include/thrust/system/cpp/detail/vector.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/execution_policy.h" "$(@D)/cuda/include/thrust/system/cpp/execution_policy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/memory.h" "$(@D)/cuda/include/thrust/system/cpp/memory.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cpp/vector.h" "$(@D)/cuda/include/thrust/system/cpp/vector.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/config.h" "$(@D)/cuda/include/thrust/system/cuda/config.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/cuda/detail/adjacent_difference.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/assign_value.h" "$(@D)/cuda/include/thrust/system/cuda/detail/assign_value.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/binary_search.h" "$(@D)/cuda/include/thrust/system/cuda/detail/binary_search.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/copy.h" "$(@D)/cuda/include/thrust/system/cuda/detail/copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/copy_if.h" "$(@D)/cuda/include/thrust/system/cuda/detail/copy_if.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/core/agent_launcher.h" "$(@D)/cuda/include/thrust/system/cuda/detail/core/agent_launcher.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/core/alignment.h" "$(@D)/cuda/include/thrust/system/cuda/detail/core/alignment.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/core/triple_chevron_launch.h" "$(@D)/cuda/include/thrust/system/cuda/detail/core/triple_chevron_launch.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/core/util.h" "$(@D)/cuda/include/thrust/system/cuda/detail/core/util.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/count.h" "$(@D)/cuda/include/thrust/system/cuda/detail/count.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cross_system.h" "$(@D)/cuda/include/thrust/system/cuda/detail/cross_system.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/agent_histogram.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_histogram.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/agent_radix_sort_downsweep.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_radix_sort_downsweep.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/agent_radix_sort_upsweep.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_radix_sort_upsweep.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/agent_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_reduce.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/agent_reduce_by_key.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_reduce_by_key.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/agent_rle.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_rle.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/agent_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_scan.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/agent_segment_fixup.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_segment_fixup.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/agent_select_if.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_select_if.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/agent_spmv_orig.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_spmv_orig.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/agent/single_pass_scan_operators.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/single_pass_scan_operators.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_adjacent_difference.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_adjacent_difference.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_discontinuity.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_discontinuity.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_exchange.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_exchange.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_histogram.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_histogram.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_load.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_load.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_radix_rank.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_radix_rank.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_radix_sort.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_radix_sort.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_raking_layout.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_raking_layout.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_reduce.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_scan.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_shuffle.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_shuffle.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/block_store.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_store.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/specializations/block_histogram_atomic.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_histogram_atomic.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/specializations/block_histogram_sort.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_histogram_sort.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_raking.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_raking.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_raking_commutative_only.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_raking_commutative_only.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_warp_reductions.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_warp_reductions.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_raking.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_raking.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans2.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans2.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans3.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans3.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/cub.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/cub.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/device_histogram.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_histogram.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/device_partition.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_partition.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/device_radix_sort.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_radix_sort.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/device_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_reduce.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/device_run_length_encode.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_run_length_encode.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/device_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_scan.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/device_segmented_radix_sort.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_segmented_radix_sort.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/device_segmented_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_segmented_reduce.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/device_select.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_select.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/device_spmv.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_spmv.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_histogram.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_histogram.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_radix_sort.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_radix_sort.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_reduce.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_reduce_by_key.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_reduce_by_key.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_rle.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_rle.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_scan.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_select_if.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_select_if.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_spmv_orig.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_spmv_orig.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/grid/grid_barrier.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/grid/grid_barrier.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/grid/grid_even_share.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/grid/grid_even_share.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/grid/grid_mapping.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/grid/grid_mapping.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/grid/grid_queue.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/grid/grid_queue.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/host/mutex.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/host/mutex.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/iterator/arg_index_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/arg_index_input_iterator.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/iterator/cache_modified_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/cache_modified_input_iterator.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/iterator/cache_modified_output_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/cache_modified_output_iterator.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/iterator/constant_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/constant_input_iterator.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/iterator/counting_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/counting_input_iterator.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/iterator/discard_output_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/discard_output_iterator.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/iterator/tex_obj_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/tex_obj_input_iterator.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/iterator/tex_ref_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/tex_ref_input_iterator.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/iterator/transform_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/transform_input_iterator.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/thread/thread_load.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_load.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/thread/thread_operators.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_operators.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/thread/thread_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_reduce.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/thread/thread_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_scan.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/thread/thread_search.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_search.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/thread/thread_store.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_store.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/util_allocator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_allocator.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/util_arch.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_arch.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/util_debug.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_debug.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/util_device.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_device.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/util_macro.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_macro.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/util_namespace.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_namespace.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/util_ptx.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_ptx.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/util_type.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_type.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/warp/specializations/warp_reduce_shfl.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/specializations/warp_reduce_shfl.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/warp/specializations/warp_reduce_smem.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/specializations/warp_reduce_smem.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/warp/specializations/warp_scan_shfl.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/specializations/warp_scan_shfl.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/warp/specializations/warp_scan_smem.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/specializations/warp_scan_smem.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/warp/warp_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/warp_reduce.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/cub/warp/warp_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/warp_scan.cuh" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/equal.h" "$(@D)/cuda/include/thrust/system/cuda/detail/equal.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/error.inl" "$(@D)/cuda/include/thrust/system/cuda/detail/error.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/execution_policy.h" "$(@D)/cuda/include/thrust/system/cuda/detail/execution_policy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/extrema.h" "$(@D)/cuda/include/thrust/system/cuda/detail/extrema.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/fill.h" "$(@D)/cuda/include/thrust/system/cuda/detail/fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/find.h" "$(@D)/cuda/include/thrust/system/cuda/detail/find.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/for_each.h" "$(@D)/cuda/include/thrust/system/cuda/detail/for_each.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/gather.h" "$(@D)/cuda/include/thrust/system/cuda/detail/gather.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/generate.h" "$(@D)/cuda/include/thrust/system/cuda/detail/generate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/get_value.h" "$(@D)/cuda/include/thrust/system/cuda/detail/get_value.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/guarded_cuda_runtime_api.h" "$(@D)/cuda/include/thrust/system/cuda/detail/guarded_cuda_runtime_api.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/guarded_driver_types.h" "$(@D)/cuda/include/thrust/system/cuda/detail/guarded_driver_types.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/inner_product.h" "$(@D)/cuda/include/thrust/system/cuda/detail/inner_product.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/internal/copy_cross_system.h" "$(@D)/cuda/include/thrust/system/cuda/detail/internal/copy_cross_system.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/internal/copy_device_to_device.h" "$(@D)/cuda/include/thrust/system/cuda/detail/internal/copy_device_to_device.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/iter_swap.h" "$(@D)/cuda/include/thrust/system/cuda/detail/iter_swap.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/logical.h" "$(@D)/cuda/include/thrust/system/cuda/detail/logical.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/cuda/detail/malloc_and_free.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/memory.inl" "$(@D)/cuda/include/thrust/system/cuda/detail/memory.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/merge.h" "$(@D)/cuda/include/thrust/system/cuda/detail/merge.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/mismatch.h" "$(@D)/cuda/include/thrust/system/cuda/detail/mismatch.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/par.h" "$(@D)/cuda/include/thrust/system/cuda/detail/par.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/par_to_seq.h" "$(@D)/cuda/include/thrust/system/cuda/detail/par_to_seq.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/parallel_for.h" "$(@D)/cuda/include/thrust/system/cuda/detail/parallel_for.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/partition.h" "$(@D)/cuda/include/thrust/system/cuda/detail/partition.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/reduce.h" "$(@D)/cuda/include/thrust/system/cuda/detail/reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/cuda/detail/reduce_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/remove.h" "$(@D)/cuda/include/thrust/system/cuda/detail/remove.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/replace.h" "$(@D)/cuda/include/thrust/system/cuda/detail/replace.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/reverse.h" "$(@D)/cuda/include/thrust/system/cuda/detail/reverse.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/scan.h" "$(@D)/cuda/include/thrust/system/cuda/detail/scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/scan_by_key.h" "$(@D)/cuda/include/thrust/system/cuda/detail/scan_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/scatter.h" "$(@D)/cuda/include/thrust/system/cuda/detail/scatter.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/sequence.h" "$(@D)/cuda/include/thrust/system/cuda/detail/sequence.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/set_operations.h" "$(@D)/cuda/include/thrust/system/cuda/detail/set_operations.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/sort.h" "$(@D)/cuda/include/thrust/system/cuda/detail/sort.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/swap_ranges.h" "$(@D)/cuda/include/thrust/system/cuda/detail/swap_ranges.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/tabulate.h" "$(@D)/cuda/include/thrust/system/cuda/detail/tabulate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/cuda/detail/temporary_buffer.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/terminate.h" "$(@D)/cuda/include/thrust/system/cuda/detail/terminate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/transform.h" "$(@D)/cuda/include/thrust/system/cuda/detail/transform.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/transform_reduce.h" "$(@D)/cuda/include/thrust/system/cuda/detail/transform_reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/transform_scan.h" "$(@D)/cuda/include/thrust/system/cuda/detail/transform_scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/cuda/detail/uninitialized_copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/cuda/detail/uninitialized_fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/unique.h" "$(@D)/cuda/include/thrust/system/cuda/detail/unique.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/unique_by_key.h" "$(@D)/cuda/include/thrust/system/cuda/detail/unique_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/util.h" "$(@D)/cuda/include/thrust/system/cuda/detail/util.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/detail/vector.inl" "$(@D)/cuda/include/thrust/system/cuda/detail/vector.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/error.h" "$(@D)/cuda/include/thrust/system/cuda/error.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/execution_policy.h" "$(@D)/cuda/include/thrust/system/cuda/execution_policy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/experimental/pinned_allocator.h" "$(@D)/cuda/include/thrust/system/cuda/experimental/pinned_allocator.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/memory.h" "$(@D)/cuda/include/thrust/system/cuda/memory.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/cuda/vector.h" "$(@D)/cuda/include/thrust/system/cuda/vector.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/detail/adl/adjacent_difference.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/assign_value.h" "$(@D)/cuda/include/thrust/system/detail/adl/assign_value.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/binary_search.h" "$(@D)/cuda/include/thrust/system/detail/adl/binary_search.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/copy.h" "$(@D)/cuda/include/thrust/system/detail/adl/copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/copy_if.h" "$(@D)/cuda/include/thrust/system/detail/adl/copy_if.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/count.h" "$(@D)/cuda/include/thrust/system/detail/adl/count.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/equal.h" "$(@D)/cuda/include/thrust/system/detail/adl/equal.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/extrema.h" "$(@D)/cuda/include/thrust/system/detail/adl/extrema.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/fill.h" "$(@D)/cuda/include/thrust/system/detail/adl/fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/find.h" "$(@D)/cuda/include/thrust/system/detail/adl/find.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/for_each.h" "$(@D)/cuda/include/thrust/system/detail/adl/for_each.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/gather.h" "$(@D)/cuda/include/thrust/system/detail/adl/gather.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/generate.h" "$(@D)/cuda/include/thrust/system/detail/adl/generate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/get_value.h" "$(@D)/cuda/include/thrust/system/detail/adl/get_value.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/inner_product.h" "$(@D)/cuda/include/thrust/system/detail/adl/inner_product.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/iter_swap.h" "$(@D)/cuda/include/thrust/system/detail/adl/iter_swap.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/logical.h" "$(@D)/cuda/include/thrust/system/detail/adl/logical.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/detail/adl/malloc_and_free.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/merge.h" "$(@D)/cuda/include/thrust/system/detail/adl/merge.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/mismatch.h" "$(@D)/cuda/include/thrust/system/detail/adl/mismatch.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/partition.h" "$(@D)/cuda/include/thrust/system/detail/adl/partition.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/reduce.h" "$(@D)/cuda/include/thrust/system/detail/adl/reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/detail/adl/reduce_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/remove.h" "$(@D)/cuda/include/thrust/system/detail/adl/remove.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/replace.h" "$(@D)/cuda/include/thrust/system/detail/adl/replace.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/reverse.h" "$(@D)/cuda/include/thrust/system/detail/adl/reverse.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/scan.h" "$(@D)/cuda/include/thrust/system/detail/adl/scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/scan_by_key.h" "$(@D)/cuda/include/thrust/system/detail/adl/scan_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/scatter.h" "$(@D)/cuda/include/thrust/system/detail/adl/scatter.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/sequence.h" "$(@D)/cuda/include/thrust/system/detail/adl/sequence.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/set_operations.h" "$(@D)/cuda/include/thrust/system/detail/adl/set_operations.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/sort.h" "$(@D)/cuda/include/thrust/system/detail/adl/sort.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/swap_ranges.h" "$(@D)/cuda/include/thrust/system/detail/adl/swap_ranges.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/tabulate.h" "$(@D)/cuda/include/thrust/system/detail/adl/tabulate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/detail/adl/temporary_buffer.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/transform.h" "$(@D)/cuda/include/thrust/system/detail/adl/transform.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/transform_reduce.h" "$(@D)/cuda/include/thrust/system/detail/adl/transform_reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/transform_scan.h" "$(@D)/cuda/include/thrust/system/detail/adl/transform_scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/detail/adl/uninitialized_copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/detail/adl/uninitialized_fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/unique.h" "$(@D)/cuda/include/thrust/system/detail/adl/unique.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/adl/unique_by_key.h" "$(@D)/cuda/include/thrust/system/detail/adl/unique_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/bad_alloc.h" "$(@D)/cuda/include/thrust/system/detail/bad_alloc.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/errno.h" "$(@D)/cuda/include/thrust/system/detail/errno.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/error_category.inl" "$(@D)/cuda/include/thrust/system/detail/error_category.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/error_code.inl" "$(@D)/cuda/include/thrust/system/detail/error_code.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/error_condition.inl" "$(@D)/cuda/include/thrust/system/detail/error_condition.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/detail/generic/adjacent_difference.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/adjacent_difference.inl" "$(@D)/cuda/include/thrust/system/detail/generic/adjacent_difference.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/advance.h" "$(@D)/cuda/include/thrust/system/detail/generic/advance.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/advance.inl" "$(@D)/cuda/include/thrust/system/detail/generic/advance.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/binary_search.h" "$(@D)/cuda/include/thrust/system/detail/generic/binary_search.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/binary_search.inl" "$(@D)/cuda/include/thrust/system/detail/generic/binary_search.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/copy.h" "$(@D)/cuda/include/thrust/system/detail/generic/copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/copy.inl" "$(@D)/cuda/include/thrust/system/detail/generic/copy.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/copy_if.h" "$(@D)/cuda/include/thrust/system/detail/generic/copy_if.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/copy_if.inl" "$(@D)/cuda/include/thrust/system/detail/generic/copy_if.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/count.h" "$(@D)/cuda/include/thrust/system/detail/generic/count.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/count.inl" "$(@D)/cuda/include/thrust/system/detail/generic/count.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/distance.h" "$(@D)/cuda/include/thrust/system/detail/generic/distance.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/distance.inl" "$(@D)/cuda/include/thrust/system/detail/generic/distance.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/equal.h" "$(@D)/cuda/include/thrust/system/detail/generic/equal.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/equal.inl" "$(@D)/cuda/include/thrust/system/detail/generic/equal.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/extrema.h" "$(@D)/cuda/include/thrust/system/detail/generic/extrema.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/extrema.inl" "$(@D)/cuda/include/thrust/system/detail/generic/extrema.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/fill.h" "$(@D)/cuda/include/thrust/system/detail/generic/fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/find.h" "$(@D)/cuda/include/thrust/system/detail/generic/find.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/find.inl" "$(@D)/cuda/include/thrust/system/detail/generic/find.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/for_each.h" "$(@D)/cuda/include/thrust/system/detail/generic/for_each.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/gather.h" "$(@D)/cuda/include/thrust/system/detail/generic/gather.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/gather.inl" "$(@D)/cuda/include/thrust/system/detail/generic/gather.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/generate.h" "$(@D)/cuda/include/thrust/system/detail/generic/generate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/generate.inl" "$(@D)/cuda/include/thrust/system/detail/generic/generate.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/inner_product.h" "$(@D)/cuda/include/thrust/system/detail/generic/inner_product.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/inner_product.inl" "$(@D)/cuda/include/thrust/system/detail/generic/inner_product.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/logical.h" "$(@D)/cuda/include/thrust/system/detail/generic/logical.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/memory.h" "$(@D)/cuda/include/thrust/system/detail/generic/memory.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/memory.inl" "$(@D)/cuda/include/thrust/system/detail/generic/memory.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/merge.h" "$(@D)/cuda/include/thrust/system/detail/generic/merge.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/merge.inl" "$(@D)/cuda/include/thrust/system/detail/generic/merge.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/mismatch.h" "$(@D)/cuda/include/thrust/system/detail/generic/mismatch.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/mismatch.inl" "$(@D)/cuda/include/thrust/system/detail/generic/mismatch.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/partition.h" "$(@D)/cuda/include/thrust/system/detail/generic/partition.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/partition.inl" "$(@D)/cuda/include/thrust/system/detail/generic/partition.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/reduce.h" "$(@D)/cuda/include/thrust/system/detail/generic/reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/reduce.inl" "$(@D)/cuda/include/thrust/system/detail/generic/reduce.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/detail/generic/reduce_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/reduce_by_key.inl" "$(@D)/cuda/include/thrust/system/detail/generic/reduce_by_key.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/remove.h" "$(@D)/cuda/include/thrust/system/detail/generic/remove.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/remove.inl" "$(@D)/cuda/include/thrust/system/detail/generic/remove.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/replace.h" "$(@D)/cuda/include/thrust/system/detail/generic/replace.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/replace.inl" "$(@D)/cuda/include/thrust/system/detail/generic/replace.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/reverse.h" "$(@D)/cuda/include/thrust/system/detail/generic/reverse.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/reverse.inl" "$(@D)/cuda/include/thrust/system/detail/generic/reverse.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/scalar/binary_search.h" "$(@D)/cuda/include/thrust/system/detail/generic/scalar/binary_search.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/scalar/binary_search.inl" "$(@D)/cuda/include/thrust/system/detail/generic/scalar/binary_search.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/scan.h" "$(@D)/cuda/include/thrust/system/detail/generic/scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/scan.inl" "$(@D)/cuda/include/thrust/system/detail/generic/scan.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/scan_by_key.h" "$(@D)/cuda/include/thrust/system/detail/generic/scan_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/scan_by_key.inl" "$(@D)/cuda/include/thrust/system/detail/generic/scan_by_key.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/scatter.h" "$(@D)/cuda/include/thrust/system/detail/generic/scatter.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/scatter.inl" "$(@D)/cuda/include/thrust/system/detail/generic/scatter.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/select_system.h" "$(@D)/cuda/include/thrust/system/detail/generic/select_system.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/sequence.h" "$(@D)/cuda/include/thrust/system/detail/generic/sequence.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/sequence.inl" "$(@D)/cuda/include/thrust/system/detail/generic/sequence.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/set_operations.h" "$(@D)/cuda/include/thrust/system/detail/generic/set_operations.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/set_operations.inl" "$(@D)/cuda/include/thrust/system/detail/generic/set_operations.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/sort.h" "$(@D)/cuda/include/thrust/system/detail/generic/sort.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/sort.inl" "$(@D)/cuda/include/thrust/system/detail/generic/sort.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/swap_ranges.h" "$(@D)/cuda/include/thrust/system/detail/generic/swap_ranges.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/swap_ranges.inl" "$(@D)/cuda/include/thrust/system/detail/generic/swap_ranges.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/tabulate.h" "$(@D)/cuda/include/thrust/system/detail/generic/tabulate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/tabulate.inl" "$(@D)/cuda/include/thrust/system/detail/generic/tabulate.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/tag.h" "$(@D)/cuda/include/thrust/system/detail/generic/tag.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/detail/generic/temporary_buffer.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/temporary_buffer.inl" "$(@D)/cuda/include/thrust/system/detail/generic/temporary_buffer.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/transform.h" "$(@D)/cuda/include/thrust/system/detail/generic/transform.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/transform.inl" "$(@D)/cuda/include/thrust/system/detail/generic/transform.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/transform_reduce.h" "$(@D)/cuda/include/thrust/system/detail/generic/transform_reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/transform_reduce.inl" "$(@D)/cuda/include/thrust/system/detail/generic/transform_reduce.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/transform_scan.h" "$(@D)/cuda/include/thrust/system/detail/generic/transform_scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/transform_scan.inl" "$(@D)/cuda/include/thrust/system/detail/generic/transform_scan.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/type_traits.h" "$(@D)/cuda/include/thrust/system/detail/generic/type_traits.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/detail/generic/uninitialized_copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/uninitialized_copy.inl" "$(@D)/cuda/include/thrust/system/detail/generic/uninitialized_copy.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/detail/generic/uninitialized_fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/uninitialized_fill.inl" "$(@D)/cuda/include/thrust/system/detail/generic/uninitialized_fill.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/unique.h" "$(@D)/cuda/include/thrust/system/detail/generic/unique.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/unique.inl" "$(@D)/cuda/include/thrust/system/detail/generic/unique.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/unique_by_key.h" "$(@D)/cuda/include/thrust/system/detail/generic/unique_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/generic/unique_by_key.inl" "$(@D)/cuda/include/thrust/system/detail/generic/unique_by_key.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/internal/decompose.h" "$(@D)/cuda/include/thrust/system/detail/internal/decompose.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/detail/sequential/adjacent_difference.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/assign_value.h" "$(@D)/cuda/include/thrust/system/detail/sequential/assign_value.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/binary_search.h" "$(@D)/cuda/include/thrust/system/detail/sequential/binary_search.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/copy.h" "$(@D)/cuda/include/thrust/system/detail/sequential/copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/copy.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/copy.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/copy_backward.h" "$(@D)/cuda/include/thrust/system/detail/sequential/copy_backward.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/copy_if.h" "$(@D)/cuda/include/thrust/system/detail/sequential/copy_if.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/count.h" "$(@D)/cuda/include/thrust/system/detail/sequential/count.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/equal.h" "$(@D)/cuda/include/thrust/system/detail/sequential/equal.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/execution_policy.h" "$(@D)/cuda/include/thrust/system/detail/sequential/execution_policy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/extrema.h" "$(@D)/cuda/include/thrust/system/detail/sequential/extrema.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/fill.h" "$(@D)/cuda/include/thrust/system/detail/sequential/fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/find.h" "$(@D)/cuda/include/thrust/system/detail/sequential/find.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/for_each.h" "$(@D)/cuda/include/thrust/system/detail/sequential/for_each.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/gather.h" "$(@D)/cuda/include/thrust/system/detail/sequential/gather.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/general_copy.h" "$(@D)/cuda/include/thrust/system/detail/sequential/general_copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/generate.h" "$(@D)/cuda/include/thrust/system/detail/sequential/generate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/get_value.h" "$(@D)/cuda/include/thrust/system/detail/sequential/get_value.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/inner_product.h" "$(@D)/cuda/include/thrust/system/detail/sequential/inner_product.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/insertion_sort.h" "$(@D)/cuda/include/thrust/system/detail/sequential/insertion_sort.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/iter_swap.h" "$(@D)/cuda/include/thrust/system/detail/sequential/iter_swap.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/logical.h" "$(@D)/cuda/include/thrust/system/detail/sequential/logical.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/detail/sequential/malloc_and_free.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/merge.h" "$(@D)/cuda/include/thrust/system/detail/sequential/merge.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/merge.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/merge.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/mismatch.h" "$(@D)/cuda/include/thrust/system/detail/sequential/mismatch.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/partition.h" "$(@D)/cuda/include/thrust/system/detail/sequential/partition.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/reduce.h" "$(@D)/cuda/include/thrust/system/detail/sequential/reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/detail/sequential/reduce_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/remove.h" "$(@D)/cuda/include/thrust/system/detail/sequential/remove.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/replace.h" "$(@D)/cuda/include/thrust/system/detail/sequential/replace.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/reverse.h" "$(@D)/cuda/include/thrust/system/detail/sequential/reverse.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/scan.h" "$(@D)/cuda/include/thrust/system/detail/sequential/scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/scan_by_key.h" "$(@D)/cuda/include/thrust/system/detail/sequential/scan_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/scatter.h" "$(@D)/cuda/include/thrust/system/detail/sequential/scatter.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/sequence.h" "$(@D)/cuda/include/thrust/system/detail/sequential/sequence.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/set_operations.h" "$(@D)/cuda/include/thrust/system/detail/sequential/set_operations.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/sort.h" "$(@D)/cuda/include/thrust/system/detail/sequential/sort.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/sort.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/sort.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/stable_merge_sort.h" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_merge_sort.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/stable_merge_sort.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_merge_sort.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/stable_primitive_sort.h" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_primitive_sort.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/stable_primitive_sort.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_primitive_sort.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/stable_radix_sort.h" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_radix_sort.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/stable_radix_sort.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_radix_sort.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/swap_ranges.h" "$(@D)/cuda/include/thrust/system/detail/sequential/swap_ranges.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/tabulate.h" "$(@D)/cuda/include/thrust/system/detail/sequential/tabulate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/detail/sequential/temporary_buffer.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/transform.h" "$(@D)/cuda/include/thrust/system/detail/sequential/transform.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/transform_reduce.h" "$(@D)/cuda/include/thrust/system/detail/sequential/transform_reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/transform_scan.h" "$(@D)/cuda/include/thrust/system/detail/sequential/transform_scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/trivial_copy.h" "$(@D)/cuda/include/thrust/system/detail/sequential/trivial_copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/detail/sequential/uninitialized_copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/detail/sequential/uninitialized_fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/unique.h" "$(@D)/cuda/include/thrust/system/detail/sequential/unique.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/sequential/unique_by_key.h" "$(@D)/cuda/include/thrust/system/detail/sequential/unique_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/detail/system_error.inl" "$(@D)/cuda/include/thrust/system/detail/system_error.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/error_code.h" "$(@D)/cuda/include/thrust/system/error_code.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/omp/detail/adjacent_difference.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/assign_value.h" "$(@D)/cuda/include/thrust/system/omp/detail/assign_value.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/binary_search.h" "$(@D)/cuda/include/thrust/system/omp/detail/binary_search.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/copy.h" "$(@D)/cuda/include/thrust/system/omp/detail/copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/copy.inl" "$(@D)/cuda/include/thrust/system/omp/detail/copy.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/copy_if.h" "$(@D)/cuda/include/thrust/system/omp/detail/copy_if.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/copy_if.inl" "$(@D)/cuda/include/thrust/system/omp/detail/copy_if.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/count.h" "$(@D)/cuda/include/thrust/system/omp/detail/count.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/default_decomposition.h" "$(@D)/cuda/include/thrust/system/omp/detail/default_decomposition.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/default_decomposition.inl" "$(@D)/cuda/include/thrust/system/omp/detail/default_decomposition.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/equal.h" "$(@D)/cuda/include/thrust/system/omp/detail/equal.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/execution_policy.h" "$(@D)/cuda/include/thrust/system/omp/detail/execution_policy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/extrema.h" "$(@D)/cuda/include/thrust/system/omp/detail/extrema.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/fill.h" "$(@D)/cuda/include/thrust/system/omp/detail/fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/find.h" "$(@D)/cuda/include/thrust/system/omp/detail/find.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/for_each.h" "$(@D)/cuda/include/thrust/system/omp/detail/for_each.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/for_each.inl" "$(@D)/cuda/include/thrust/system/omp/detail/for_each.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/gather.h" "$(@D)/cuda/include/thrust/system/omp/detail/gather.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/generate.h" "$(@D)/cuda/include/thrust/system/omp/detail/generate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/get_value.h" "$(@D)/cuda/include/thrust/system/omp/detail/get_value.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/inner_product.h" "$(@D)/cuda/include/thrust/system/omp/detail/inner_product.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/iter_swap.h" "$(@D)/cuda/include/thrust/system/omp/detail/iter_swap.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/logical.h" "$(@D)/cuda/include/thrust/system/omp/detail/logical.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/omp/detail/malloc_and_free.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/memory.inl" "$(@D)/cuda/include/thrust/system/omp/detail/memory.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/merge.h" "$(@D)/cuda/include/thrust/system/omp/detail/merge.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/mismatch.h" "$(@D)/cuda/include/thrust/system/omp/detail/mismatch.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/par.h" "$(@D)/cuda/include/thrust/system/omp/detail/par.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/partition.h" "$(@D)/cuda/include/thrust/system/omp/detail/partition.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/partition.inl" "$(@D)/cuda/include/thrust/system/omp/detail/partition.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/reduce.h" "$(@D)/cuda/include/thrust/system/omp/detail/reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/reduce.inl" "$(@D)/cuda/include/thrust/system/omp/detail/reduce.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/omp/detail/reduce_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/reduce_by_key.inl" "$(@D)/cuda/include/thrust/system/omp/detail/reduce_by_key.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/reduce_intervals.h" "$(@D)/cuda/include/thrust/system/omp/detail/reduce_intervals.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/reduce_intervals.inl" "$(@D)/cuda/include/thrust/system/omp/detail/reduce_intervals.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/remove.h" "$(@D)/cuda/include/thrust/system/omp/detail/remove.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/remove.inl" "$(@D)/cuda/include/thrust/system/omp/detail/remove.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/replace.h" "$(@D)/cuda/include/thrust/system/omp/detail/replace.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/reverse.h" "$(@D)/cuda/include/thrust/system/omp/detail/reverse.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/scan.h" "$(@D)/cuda/include/thrust/system/omp/detail/scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/scan_by_key.h" "$(@D)/cuda/include/thrust/system/omp/detail/scan_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/scatter.h" "$(@D)/cuda/include/thrust/system/omp/detail/scatter.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/sequence.h" "$(@D)/cuda/include/thrust/system/omp/detail/sequence.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/set_operations.h" "$(@D)/cuda/include/thrust/system/omp/detail/set_operations.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/sort.h" "$(@D)/cuda/include/thrust/system/omp/detail/sort.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/sort.inl" "$(@D)/cuda/include/thrust/system/omp/detail/sort.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/swap_ranges.h" "$(@D)/cuda/include/thrust/system/omp/detail/swap_ranges.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/tabulate.h" "$(@D)/cuda/include/thrust/system/omp/detail/tabulate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/omp/detail/temporary_buffer.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/transform.h" "$(@D)/cuda/include/thrust/system/omp/detail/transform.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/transform_reduce.h" "$(@D)/cuda/include/thrust/system/omp/detail/transform_reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/transform_scan.h" "$(@D)/cuda/include/thrust/system/omp/detail/transform_scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/omp/detail/uninitialized_copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/omp/detail/uninitialized_fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/unique.h" "$(@D)/cuda/include/thrust/system/omp/detail/unique.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/unique.inl" "$(@D)/cuda/include/thrust/system/omp/detail/unique.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/unique_by_key.h" "$(@D)/cuda/include/thrust/system/omp/detail/unique_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/unique_by_key.inl" "$(@D)/cuda/include/thrust/system/omp/detail/unique_by_key.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/detail/vector.inl" "$(@D)/cuda/include/thrust/system/omp/detail/vector.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/execution_policy.h" "$(@D)/cuda/include/thrust/system/omp/execution_policy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/memory.h" "$(@D)/cuda/include/thrust/system/omp/memory.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/omp/vector.h" "$(@D)/cuda/include/thrust/system/omp/vector.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/system_error.h" "$(@D)/cuda/include/thrust/system/system_error.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/tbb/detail/adjacent_difference.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/assign_value.h" "$(@D)/cuda/include/thrust/system/tbb/detail/assign_value.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/binary_search.h" "$(@D)/cuda/include/thrust/system/tbb/detail/binary_search.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/copy.h" "$(@D)/cuda/include/thrust/system/tbb/detail/copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/copy.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/copy.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/copy_if.h" "$(@D)/cuda/include/thrust/system/tbb/detail/copy_if.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/copy_if.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/copy_if.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/count.h" "$(@D)/cuda/include/thrust/system/tbb/detail/count.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/equal.h" "$(@D)/cuda/include/thrust/system/tbb/detail/equal.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/execution_policy.h" "$(@D)/cuda/include/thrust/system/tbb/detail/execution_policy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/extrema.h" "$(@D)/cuda/include/thrust/system/tbb/detail/extrema.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/fill.h" "$(@D)/cuda/include/thrust/system/tbb/detail/fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/find.h" "$(@D)/cuda/include/thrust/system/tbb/detail/find.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/for_each.h" "$(@D)/cuda/include/thrust/system/tbb/detail/for_each.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/for_each.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/for_each.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/gather.h" "$(@D)/cuda/include/thrust/system/tbb/detail/gather.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/generate.h" "$(@D)/cuda/include/thrust/system/tbb/detail/generate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/get_value.h" "$(@D)/cuda/include/thrust/system/tbb/detail/get_value.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/inner_product.h" "$(@D)/cuda/include/thrust/system/tbb/detail/inner_product.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/iter_swap.h" "$(@D)/cuda/include/thrust/system/tbb/detail/iter_swap.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/logical.h" "$(@D)/cuda/include/thrust/system/tbb/detail/logical.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/tbb/detail/malloc_and_free.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/memory.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/memory.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/merge.h" "$(@D)/cuda/include/thrust/system/tbb/detail/merge.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/merge.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/merge.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/mismatch.h" "$(@D)/cuda/include/thrust/system/tbb/detail/mismatch.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/par.h" "$(@D)/cuda/include/thrust/system/tbb/detail/par.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/partition.h" "$(@D)/cuda/include/thrust/system/tbb/detail/partition.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/partition.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/partition.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/reduce.h" "$(@D)/cuda/include/thrust/system/tbb/detail/reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/reduce.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/reduce.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/tbb/detail/reduce_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/reduce_by_key.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/reduce_by_key.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/reduce_intervals.h" "$(@D)/cuda/include/thrust/system/tbb/detail/reduce_intervals.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/remove.h" "$(@D)/cuda/include/thrust/system/tbb/detail/remove.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/remove.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/remove.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/replace.h" "$(@D)/cuda/include/thrust/system/tbb/detail/replace.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/reverse.h" "$(@D)/cuda/include/thrust/system/tbb/detail/reverse.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/scan.h" "$(@D)/cuda/include/thrust/system/tbb/detail/scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/scan.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/scan.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/scan_by_key.h" "$(@D)/cuda/include/thrust/system/tbb/detail/scan_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/scatter.h" "$(@D)/cuda/include/thrust/system/tbb/detail/scatter.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/sequence.h" "$(@D)/cuda/include/thrust/system/tbb/detail/sequence.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/set_operations.h" "$(@D)/cuda/include/thrust/system/tbb/detail/set_operations.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/sort.h" "$(@D)/cuda/include/thrust/system/tbb/detail/sort.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/sort.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/sort.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/swap_ranges.h" "$(@D)/cuda/include/thrust/system/tbb/detail/swap_ranges.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/tabulate.h" "$(@D)/cuda/include/thrust/system/tbb/detail/tabulate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/tbb/detail/temporary_buffer.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/transform.h" "$(@D)/cuda/include/thrust/system/tbb/detail/transform.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/transform_reduce.h" "$(@D)/cuda/include/thrust/system/tbb/detail/transform_reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/transform_scan.h" "$(@D)/cuda/include/thrust/system/tbb/detail/transform_scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/tbb/detail/uninitialized_copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/tbb/detail/uninitialized_fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/unique.h" "$(@D)/cuda/include/thrust/system/tbb/detail/unique.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/unique.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/unique.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/unique_by_key.h" "$(@D)/cuda/include/thrust/system/tbb/detail/unique_by_key.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/unique_by_key.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/unique_by_key.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/detail/vector.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/vector.inl" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/execution_policy.h" "$(@D)/cuda/include/thrust/system/tbb/execution_policy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/memory.h" "$(@D)/cuda/include/thrust/system/tbb/memory.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system/tbb/vector.h" "$(@D)/cuda/include/thrust/system/tbb/vector.h" && cp -f "/usr/local/cuda-10.0/include/thrust/system_error.h" "$(@D)/cuda/include/thrust/system_error.h" && cp -f "/usr/local/cuda-10.0/include/thrust/tabulate.h" "$(@D)/cuda/include/thrust/tabulate.h" && cp -f "/usr/local/cuda-10.0/include/thrust/transform.h" "$(@D)/cuda/include/thrust/transform.h" && cp -f "/usr/local/cuda-10.0/include/thrust/transform_reduce.h" "$(@D)/cuda/include/thrust/transform_reduce.h" && cp -f "/usr/local/cuda-10.0/include/thrust/transform_scan.h" "$(@D)/cuda/include/thrust/transform_scan.h" && cp -f "/usr/local/cuda-10.0/include/thrust/tuple.h" "$(@D)/cuda/include/thrust/tuple.h" && cp -f "/usr/local/cuda-10.0/include/thrust/uninitialized_copy.h" "$(@D)/cuda/include/thrust/uninitialized_copy.h" && cp -f "/usr/local/cuda-10.0/include/thrust/uninitialized_fill.h" "$(@D)/cuda/include/thrust/uninitialized_fill.h" && cp -f "/usr/local/cuda-10.0/include/thrust/unique.h" "$(@D)/cuda/include/thrust/unique.h" && cp -f "/usr/local/cuda-10.0/include/thrust/version.h" "$(@D)/cuda/include/thrust/version.h" && cp -f "/usr/local/cuda-10.0/include/vector_functions.h" "$(@D)/cuda/include/vector_functions.h" && cp -f "/usr/local/cuda-10.0/include/vector_functions.hpp" "$(@D)/cuda/include/vector_functions.hpp" && cp -f "/usr/local/cuda-10.0/include/vector_types.h" "$(@D)/cuda/include/vector_types.h"
-   """,
+    cmd = """cp -rf "/usr/local/cuda-10.0/include/." "$(@D)/cuda/include/" """,
 )
 
 genrule(
@@ -1202,9 +1200,7 @@ genrule(
     outs = [
         "cuda/nvvm/libdevice/libdevice.10.bc",
     ],
-    cmd = """
-if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp -f "/usr/local/cuda-10.0/nvvm/libdevice/libdevice.10.bc" "$(@D)//libdevice.10.bc"
-   """,
+    cmd = """cp -rf "/usr/local/cuda-10.0/nvvm/libdevice/." "$(@D)/" """,
 )
 
 genrule(
@@ -1241,9 +1237,7 @@ genrule(
         "cuda/extras/CUPTI/include/openmp/cupti_openmp.h",
         "cuda/extras/CUPTI/include/openmp/ompt.h",
     ],
-    cmd = """
-if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/GL/gl.h" "$(@D)/cuda/extras/CUPTI/include/GL/gl.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/GL/glew.h" "$(@D)/cuda/extras/CUPTI/include/GL/glew.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/GL/glext.h" "$(@D)/cuda/extras/CUPTI/include/GL/glext.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/GL/glu.h" "$(@D)/cuda/extras/CUPTI/include/GL/glu.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/GL/glut.h" "$(@D)/cuda/extras/CUPTI/include/GL/glut.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/GL/glx.h" "$(@D)/cuda/extras/CUPTI/include/GL/glx.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/GL/glxext.h" "$(@D)/cuda/extras/CUPTI/include/GL/glxext.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/GL/wglew.h" "$(@D)/cuda/extras/CUPTI/include/GL/wglew.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/GL/wglext.h" "$(@D)/cuda/extras/CUPTI/include/GL/wglext.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/cuda_stdint.h" "$(@D)/cuda/extras/CUPTI/include/cuda_stdint.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/cupti.h" "$(@D)/cuda/extras/CUPTI/include/cupti.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/cupti_activity.h" "$(@D)/cuda/extras/CUPTI/include/cupti_activity.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/cupti_callbacks.h" "$(@D)/cuda/extras/CUPTI/include/cupti_callbacks.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/cupti_driver_cbid.h" "$(@D)/cuda/extras/CUPTI/include/cupti_driver_cbid.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/cupti_events.h" "$(@D)/cuda/extras/CUPTI/include/cupti_events.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/cupti_metrics.h" "$(@D)/cuda/extras/CUPTI/include/cupti_metrics.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/cupti_nvtx_cbid.h" "$(@D)/cuda/extras/CUPTI/include/cupti_nvtx_cbid.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/cupti_result.h" "$(@D)/cuda/extras/CUPTI/include/cupti_result.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/cupti_runtime_cbid.h" "$(@D)/cuda/extras/CUPTI/include/cupti_runtime_cbid.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/cupti_version.h" "$(@D)/cuda/extras/CUPTI/include/cupti_version.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/generated_cudaGL_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cudaGL_meta.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/generated_cudaVDPAU_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cudaVDPAU_meta.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/generated_cuda_gl_interop_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cuda_gl_interop_meta.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/generated_cuda_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cuda_meta.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/generated_cuda_runtime_api_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cuda_runtime_api_meta.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/generated_cuda_vdpau_interop_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cuda_vdpau_interop_meta.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/generated_nvtx_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_nvtx_meta.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/openacc/cupti_openacc.h" "$(@D)/cuda/extras/CUPTI/include/openacc/cupti_openacc.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/openmp/cupti_openmp.h" "$(@D)/cuda/extras/CUPTI/include/openmp/cupti_openmp.h" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/include/openmp/ompt.h" "$(@D)/cuda/extras/CUPTI/include/openmp/ompt.h"
-   """,
+    cmd = """cp -rf "/usr/local/cuda-10.0/extras/CUPTI/include/." "$(@D)/cuda/extras/CUPTI/include/" """,
 )
 
 genrule(
@@ -1259,9 +1253,31 @@ genrule(
         "cuda/lib/libcudnn.so.7",
         "cuda/lib/libcupti.so.10.0",
     ],
-    cmd = """
-if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp -f "/usr/local/cuda-10.0/targets/x86_64-linux/lib/stubs/libcuda.so" "$(@D)/cuda/lib/libcuda.so" && cp -f "/usr/local/cuda-10.0/targets/x86_64-linux/lib/libcudart.so.10.0.130" "$(@D)/cuda/lib/libcudart.so.10.0" && cp -f "/usr/local/cuda-10.0/targets/x86_64-linux/lib/libcudart_static.a" "$(@D)/cuda/lib/libcudart_static.a" && cp -f "/usr/local/cuda-10.0/targets/x86_64-linux/lib/libcublas.so.10.0.130" "$(@D)/cuda/lib/libcublas.so.10.0" && cp -f "/usr/local/cuda-10.0/targets/x86_64-linux/lib/libcusolver.so.10.0.130" "$(@D)/cuda/lib/libcusolver.so.10.0" && cp -f "/usr/local/cuda-10.0/targets/x86_64-linux/lib/libcurand.so.10.0.130" "$(@D)/cuda/lib/libcurand.so.10.0" && cp -f "/usr/local/cuda-10.0/targets/x86_64-linux/lib/libcufft.so.10.0.145" "$(@D)/cuda/lib/libcufft.so.10.0" && cp -f "/usr/lib/x86_64-linux-gnu/libcudnn.so.7.3.1" "$(@D)/cuda/lib/libcudnn.so.7" && cp -f "/usr/local/cuda-10.0/extras/CUPTI/lib64/libcupti.so.10.0.130" "$(@D)/cuda/lib/libcupti.so.10.0"
-   """,
+    cmd = """cp -f "/usr/local/cuda-10.0/lib64/stubs/libcuda.so" $(location cuda/lib/libcuda.so) && cp -f "/usr/local/cuda-10.0/lib64/libcudart.so.10.0" $(location cuda/lib/libcudart.so.10.0) && cp -f "/usr/local/cuda-10.0/lib64/libcudart_static.a" $(location cuda/lib/libcudart_static.a) && cp -f "/usr/local/cuda-10.0/lib64/libcublas.so.10.0" $(location cuda/lib/libcublas.so.10.0) && cp -f "/usr/local/cuda-10.0/lib64/libcusolver.so.10.0" $(location cuda/lib/libcusolver.so.10.0) && cp -f "/usr/local/cuda-10.0/lib64/libcurand.so.10.0" $(location cuda/lib/libcurand.so.10.0) && cp -f "/usr/local/cuda-10.0/lib64/libcufft.so.10.0" $(location cuda/lib/libcufft.so.10.0) && cp -f "/usr/lib/x86_64-linux-gnu/libcudnn.so.7" $(location cuda/lib/libcudnn.so.7) && cp -f "/usr/local/cuda-10.0/extras/CUPTI/lib64/libcupti.so.10.0" $(location cuda/lib/libcupti.so.10.0) """,
+)
+
+genrule(
+    name = "cuda-bin",
+    outs = [
+        "cuda/bin/bin2c",
+        "cuda/bin/crt/link.stub",
+        "cuda/bin/crt/prelink.stub",
+        "cuda/bin/cuda-gdb",
+        "cuda/bin/cuda-gdbserver",
+        "cuda/bin/cuda-memcheck",
+        "cuda/bin/cudafe++",
+        "cuda/bin/cuobjdump",
+        "cuda/bin/fatbinary",
+        "cuda/bin/gpu-library-advisor",
+        "cuda/bin/nvcc",
+        "cuda/bin/nvcc.profile",
+        "cuda/bin/nvdisasm",
+        "cuda/bin/nvlink",
+        "cuda/bin/nvprof",
+        "cuda/bin/nvprune",
+        "cuda/bin/ptxas",
+    ],
+    cmd = """cp -rf "/usr/local/cuda-10.0/bin/." "$(@D)/cuda/bin/" """,
 )
 
 genrule(
@@ -1269,7 +1285,5 @@ genrule(
     outs = [
         "cuda/include/cudnn.h",
     ],
-    cmd = """
-if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp -f "/usr/include/cudnn.h" "$(@D)/cudnn.h"
-   """,
+    cmd = """cp -f "/usr/include/cudnn.h" $(location cuda/include/cudnn.h) """,
 )
diff --git a/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/BUILD b/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/BUILD
index c6930904b5..cfd0a08e93 100755
--- a/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/BUILD
+++ b/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/BUILD
@@ -1187,9 +1187,7 @@ genrule(
         "cuda/include/vector_functions.hpp",
         "cuda/include/vector_types.h",
     ],
-    cmd = """
-if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp -f "/usr/local/cuda-9.0/include/CL/cl.h" "$(@D)/cuda/include/CL/cl.h" && cp -f "/usr/local/cuda-9.0/include/CL/cl.hpp" "$(@D)/cuda/include/CL/cl.hpp" && cp -f "/usr/local/cuda-9.0/include/CL/cl_egl.h" "$(@D)/cuda/include/CL/cl_egl.h" && cp -f "/usr/local/cuda-9.0/include/CL/cl_ext.h" "$(@D)/cuda/include/CL/cl_ext.h" && cp -f "/usr/local/cuda-9.0/include/CL/cl_gl.h" "$(@D)/cuda/include/CL/cl_gl.h" && cp -f "/usr/local/cuda-9.0/include/CL/cl_gl_ext.h" "$(@D)/cuda/include/CL/cl_gl_ext.h" && cp -f "/usr/local/cuda-9.0/include/CL/cl_platform.h" "$(@D)/cuda/include/CL/cl_platform.h" && cp -f "/usr/local/cuda-9.0/include/CL/opencl.h" "$(@D)/cuda/include/CL/opencl.h" && cp -f "/usr/local/cuda-9.0/include/builtin_types.h" "$(@D)/cuda/include/builtin_types.h" && cp -f "/usr/local/cuda-9.0/include/channel_descriptor.h" "$(@D)/cuda/include/channel_descriptor.h" && cp -f "/usr/local/cuda-9.0/include/common_functions.h" "$(@D)/cuda/include/common_functions.h" && cp -f "/usr/local/cuda-9.0/include/cooperative_groups.h" "$(@D)/cuda/include/cooperative_groups.h" && cp -f "/usr/local/cuda-9.0/include/cooperative_groups_helpers.h" "$(@D)/cuda/include/cooperative_groups_helpers.h" && cp -f "/usr/local/cuda-9.0/include/crt/common_functions.h" "$(@D)/cuda/include/crt/common_functions.h" && cp -f "/usr/local/cuda-9.0/include/crt/device_double_functions.h" "$(@D)/cuda/include/crt/device_double_functions.h" && cp -f "/usr/local/cuda-9.0/include/crt/device_double_functions.hpp" "$(@D)/cuda/include/crt/device_double_functions.hpp" && cp -f "/usr/local/cuda-9.0/include/crt/device_functions.h" "$(@D)/cuda/include/crt/device_functions.h" && cp -f "/usr/local/cuda-9.0/include/crt/device_functions.hpp" "$(@D)/cuda/include/crt/device_functions.hpp" && cp -f "/usr/local/cuda-9.0/include/crt/func_macro.h" "$(@D)/cuda/include/crt/func_macro.h" && cp -f "/usr/local/cuda-9.0/include/crt/host_config.h" "$(@D)/cuda/include/crt/host_config.h" && cp -f "/usr/local/cuda-9.0/include/crt/host_defines.h" "$(@D)/cuda/include/crt/host_defines.h" && cp -f "/usr/local/cuda-9.0/include/crt/host_runtime.h" "$(@D)/cuda/include/crt/host_runtime.h" && cp -f "/usr/local/cuda-9.0/include/crt/math_functions.h" "$(@D)/cuda/include/crt/math_functions.h" && cp -f "/usr/local/cuda-9.0/include/crt/math_functions.hpp" "$(@D)/cuda/include/crt/math_functions.hpp" && cp -f "/usr/local/cuda-9.0/include/crt/mma.h" "$(@D)/cuda/include/crt/mma.h" && cp -f "/usr/local/cuda-9.0/include/crt/mma.hpp" "$(@D)/cuda/include/crt/mma.hpp" && cp -f "/usr/local/cuda-9.0/include/crt/nvfunctional" "$(@D)/cuda/include/crt/nvfunctional" && cp -f "/usr/local/cuda-9.0/include/crt/sm_70_rt.h" "$(@D)/cuda/include/crt/sm_70_rt.h" && cp -f "/usr/local/cuda-9.0/include/crt/sm_70_rt.hpp" "$(@D)/cuda/include/crt/sm_70_rt.hpp" && cp -f "/usr/local/cuda-9.0/include/crt/storage_class.h" "$(@D)/cuda/include/crt/storage_class.h" && cp -f "/usr/local/cuda-9.0/include/cuComplex.h" "$(@D)/cuda/include/cuComplex.h" && cp -f "/usr/local/cuda-9.0/include/cublas.h" "$(@D)/cuda/include/cublas.h" && cp -f "/usr/local/cuda-9.0/include/cublasXt.h" "$(@D)/cuda/include/cublasXt.h" && cp -f "/usr/local/cuda-9.0/include/cublas_api.h" "$(@D)/cuda/include/cublas_api.h" && cp -f "/usr/local/cuda-9.0/include/cublas_v2.h" "$(@D)/cuda/include/cublas_v2.h" && cp -f "/usr/local/cuda-9.0/include/cuda.h" "$(@D)/cuda/include/cuda.h" && cp -f "/usr/local/cuda-9.0/include/cudaEGL.h" "$(@D)/cuda/include/cudaEGL.h" && cp -f "/usr/local/cuda-9.0/include/cudaGL.h" "$(@D)/cuda/include/cudaGL.h" && cp -f "/usr/local/cuda-9.0/include/cudaProfiler.h" "$(@D)/cuda/include/cudaProfiler.h" && cp -f "/usr/local/cuda-9.0/include/cudaVDPAU.h" "$(@D)/cuda/include/cudaVDPAU.h" && cp -f "/usr/local/cuda-9.0/include/cuda_device_runtime_api.h" "$(@D)/cuda/include/cuda_device_runtime_api.h" && cp -f "/usr/local/cuda-9.0/include/cuda_fp16.h" "$(@D)/cuda/include/cuda_fp16.h" && cp -f "/usr/local/cuda-9.0/include/cuda_fp16.hpp" "$(@D)/cuda/include/cuda_fp16.hpp" && cp -f "/usr/local/cuda-9.0/include/cuda_gl_interop.h" "$(@D)/cuda/include/cuda_gl_interop.h" && cp -f "/usr/local/cuda-9.0/include/cuda_occupancy.h" "$(@D)/cuda/include/cuda_occupancy.h" && cp -f "/usr/local/cuda-9.0/include/cuda_profiler_api.h" "$(@D)/cuda/include/cuda_profiler_api.h" && cp -f "/usr/local/cuda-9.0/include/cuda_runtime.h" "$(@D)/cuda/include/cuda_runtime.h" && cp -f "/usr/local/cuda-9.0/include/cuda_runtime_api.h" "$(@D)/cuda/include/cuda_runtime_api.h" && cp -f "/usr/local/cuda-9.0/include/cuda_surface_types.h" "$(@D)/cuda/include/cuda_surface_types.h" && cp -f "/usr/local/cuda-9.0/include/cuda_texture_types.h" "$(@D)/cuda/include/cuda_texture_types.h" && cp -f "/usr/local/cuda-9.0/include/cuda_vdpau_interop.h" "$(@D)/cuda/include/cuda_vdpau_interop.h" && cp -f "/usr/local/cuda-9.0/include/cudalibxt.h" "$(@D)/cuda/include/cudalibxt.h" && cp -f "/usr/local/cuda-9.0/include/cufft.h" "$(@D)/cuda/include/cufft.h" && cp -f "/usr/local/cuda-9.0/include/cufftXt.h" "$(@D)/cuda/include/cufftXt.h" && cp -f "/usr/local/cuda-9.0/include/cufftw.h" "$(@D)/cuda/include/cufftw.h" && cp -f "/usr/local/cuda-9.0/include/curand.h" "$(@D)/cuda/include/curand.h" && cp -f "/usr/local/cuda-9.0/include/curand_discrete.h" "$(@D)/cuda/include/curand_discrete.h" && cp -f "/usr/local/cuda-9.0/include/curand_discrete2.h" "$(@D)/cuda/include/curand_discrete2.h" && cp -f "/usr/local/cuda-9.0/include/curand_globals.h" "$(@D)/cuda/include/curand_globals.h" && cp -f "/usr/local/cuda-9.0/include/curand_kernel.h" "$(@D)/cuda/include/curand_kernel.h" && cp -f "/usr/local/cuda-9.0/include/curand_lognormal.h" "$(@D)/cuda/include/curand_lognormal.h" && cp -f "/usr/local/cuda-9.0/include/curand_mrg32k3a.h" "$(@D)/cuda/include/curand_mrg32k3a.h" && cp -f "/usr/local/cuda-9.0/include/curand_mtgp32.h" "$(@D)/cuda/include/curand_mtgp32.h" && cp -f "/usr/local/cuda-9.0/include/curand_mtgp32_host.h" "$(@D)/cuda/include/curand_mtgp32_host.h" && cp -f "/usr/local/cuda-9.0/include/curand_mtgp32_kernel.h" "$(@D)/cuda/include/curand_mtgp32_kernel.h" && cp -f "/usr/local/cuda-9.0/include/curand_mtgp32dc_p_11213.h" "$(@D)/cuda/include/curand_mtgp32dc_p_11213.h" && cp -f "/usr/local/cuda-9.0/include/curand_normal.h" "$(@D)/cuda/include/curand_normal.h" && cp -f "/usr/local/cuda-9.0/include/curand_normal_static.h" "$(@D)/cuda/include/curand_normal_static.h" && cp -f "/usr/local/cuda-9.0/include/curand_philox4x32_x.h" "$(@D)/cuda/include/curand_philox4x32_x.h" && cp -f "/usr/local/cuda-9.0/include/curand_poisson.h" "$(@D)/cuda/include/curand_poisson.h" && cp -f "/usr/local/cuda-9.0/include/curand_precalc.h" "$(@D)/cuda/include/curand_precalc.h" && cp -f "/usr/local/cuda-9.0/include/curand_uniform.h" "$(@D)/cuda/include/curand_uniform.h" && cp -f "/usr/local/cuda-9.0/include/cusolverDn.h" "$(@D)/cuda/include/cusolverDn.h" && cp -f "/usr/local/cuda-9.0/include/cusolverRf.h" "$(@D)/cuda/include/cusolverRf.h" && cp -f "/usr/local/cuda-9.0/include/cusolverSp.h" "$(@D)/cuda/include/cusolverSp.h" && cp -f "/usr/local/cuda-9.0/include/cusolverSp_LOWLEVEL_PREVIEW.h" "$(@D)/cuda/include/cusolverSp_LOWLEVEL_PREVIEW.h" && cp -f "/usr/local/cuda-9.0/include/cusolver_common.h" "$(@D)/cuda/include/cusolver_common.h" && cp -f "/usr/local/cuda-9.0/include/cusparse.h" "$(@D)/cuda/include/cusparse.h" && cp -f "/usr/local/cuda-9.0/include/cusparse_v2.h" "$(@D)/cuda/include/cusparse_v2.h" && cp -f "/usr/local/cuda-9.0/include/device_atomic_functions.h" "$(@D)/cuda/include/device_atomic_functions.h" && cp -f "/usr/local/cuda-9.0/include/device_atomic_functions.hpp" "$(@D)/cuda/include/device_atomic_functions.hpp" && cp -f "/usr/local/cuda-9.0/include/device_double_functions.h" "$(@D)/cuda/include/device_double_functions.h" && cp -f "/usr/local/cuda-9.0/include/device_double_functions.hpp" "$(@D)/cuda/include/device_double_functions.hpp" && cp -f "/usr/local/cuda-9.0/include/device_functions.h" "$(@D)/cuda/include/device_functions.h" && cp -f "/usr/local/cuda-9.0/include/device_functions.hpp" "$(@D)/cuda/include/device_functions.hpp" && cp -f "/usr/local/cuda-9.0/include/device_functions_decls.h" "$(@D)/cuda/include/device_functions_decls.h" && cp -f "/usr/local/cuda-9.0/include/device_launch_parameters.h" "$(@D)/cuda/include/device_launch_parameters.h" && cp -f "/usr/local/cuda-9.0/include/device_types.h" "$(@D)/cuda/include/device_types.h" && cp -f "/usr/local/cuda-9.0/include/driver_functions.h" "$(@D)/cuda/include/driver_functions.h" && cp -f "/usr/local/cuda-9.0/include/driver_types.h" "$(@D)/cuda/include/driver_types.h" && cp -f "/usr/local/cuda-9.0/include/dynlink_cuda.h" "$(@D)/cuda/include/dynlink_cuda.h" && cp -f "/usr/local/cuda-9.0/include/dynlink_cuda_cuda.h" "$(@D)/cuda/include/dynlink_cuda_cuda.h" && cp -f "/usr/local/cuda-9.0/include/dynlink_cuviddec.h" "$(@D)/cuda/include/dynlink_cuviddec.h" && cp -f "/usr/local/cuda-9.0/include/dynlink_nvcuvid.h" "$(@D)/cuda/include/dynlink_nvcuvid.h" && cp -f "/usr/local/cuda-9.0/include/fatBinaryCtl.h" "$(@D)/cuda/include/fatBinaryCtl.h" && cp -f "/usr/local/cuda-9.0/include/fatbinary.h" "$(@D)/cuda/include/fatbinary.h" && cp -f "/usr/local/cuda-9.0/include/host_config.h" "$(@D)/cuda/include/host_config.h" && cp -f "/usr/local/cuda-9.0/include/host_defines.h" "$(@D)/cuda/include/host_defines.h" && cp -f "/usr/local/cuda-9.0/include/library_types.h" "$(@D)/cuda/include/library_types.h" && cp -f "/usr/local/cuda-9.0/include/math_constants.h" "$(@D)/cuda/include/math_constants.h" && cp -f "/usr/local/cuda-9.0/include/math_functions.h" "$(@D)/cuda/include/math_functions.h" && cp -f "/usr/local/cuda-9.0/include/math_functions.hpp" "$(@D)/cuda/include/math_functions.hpp" && cp -f "/usr/local/cuda-9.0/include/math_functions_dbl_ptx3.h" "$(@D)/cuda/include/math_functions_dbl_ptx3.h" && cp -f "/usr/local/cuda-9.0/include/math_functions_dbl_ptx3.hpp" "$(@D)/cuda/include/math_functions_dbl_ptx3.hpp" && cp -f "/usr/local/cuda-9.0/include/mma.h" "$(@D)/cuda/include/mma.h" && cp -f "/usr/local/cuda-9.0/include/npp.h" "$(@D)/cuda/include/npp.h" && cp -f "/usr/local/cuda-9.0/include/nppcore.h" "$(@D)/cuda/include/nppcore.h" && cp -f "/usr/local/cuda-9.0/include/nppdefs.h" "$(@D)/cuda/include/nppdefs.h" && cp -f "/usr/local/cuda-9.0/include/nppi.h" "$(@D)/cuda/include/nppi.h" && cp -f "/usr/local/cuda-9.0/include/nppi_arithmetic_and_logical_operations.h" "$(@D)/cuda/include/nppi_arithmetic_and_logical_operations.h" && cp -f "/usr/local/cuda-9.0/include/nppi_color_conversion.h" "$(@D)/cuda/include/nppi_color_conversion.h" && cp -f "/usr/local/cuda-9.0/include/nppi_compression_functions.h" "$(@D)/cuda/include/nppi_compression_functions.h" && cp -f "/usr/local/cuda-9.0/include/nppi_computer_vision.h" "$(@D)/cuda/include/nppi_computer_vision.h" && cp -f "/usr/local/cuda-9.0/include/nppi_data_exchange_and_initialization.h" "$(@D)/cuda/include/nppi_data_exchange_and_initialization.h" && cp -f "/usr/local/cuda-9.0/include/nppi_filtering_functions.h" "$(@D)/cuda/include/nppi_filtering_functions.h" && cp -f "/usr/local/cuda-9.0/include/nppi_geometry_transforms.h" "$(@D)/cuda/include/nppi_geometry_transforms.h" && cp -f "/usr/local/cuda-9.0/include/nppi_linear_transforms.h" "$(@D)/cuda/include/nppi_linear_transforms.h" && cp -f "/usr/local/cuda-9.0/include/nppi_morphological_operations.h" "$(@D)/cuda/include/nppi_morphological_operations.h" && cp -f "/usr/local/cuda-9.0/include/nppi_statistics_functions.h" "$(@D)/cuda/include/nppi_statistics_functions.h" && cp -f "/usr/local/cuda-9.0/include/nppi_support_functions.h" "$(@D)/cuda/include/nppi_support_functions.h" && cp -f "/usr/local/cuda-9.0/include/nppi_threshold_and_compare_operations.h" "$(@D)/cuda/include/nppi_threshold_and_compare_operations.h" && cp -f "/usr/local/cuda-9.0/include/npps.h" "$(@D)/cuda/include/npps.h" && cp -f "/usr/local/cuda-9.0/include/npps_arithmetic_and_logical_operations.h" "$(@D)/cuda/include/npps_arithmetic_and_logical_operations.h" && cp -f "/usr/local/cuda-9.0/include/npps_conversion_functions.h" "$(@D)/cuda/include/npps_conversion_functions.h" && cp -f "/usr/local/cuda-9.0/include/npps_filtering_functions.h" "$(@D)/cuda/include/npps_filtering_functions.h" && cp -f "/usr/local/cuda-9.0/include/npps_initialization.h" "$(@D)/cuda/include/npps_initialization.h" && cp -f "/usr/local/cuda-9.0/include/npps_statistics_functions.h" "$(@D)/cuda/include/npps_statistics_functions.h" && cp -f "/usr/local/cuda-9.0/include/npps_support_functions.h" "$(@D)/cuda/include/npps_support_functions.h" && cp -f "/usr/local/cuda-9.0/include/nppversion.h" "$(@D)/cuda/include/nppversion.h" && cp -f "/usr/local/cuda-9.0/include/nvToolsExt.h" "$(@D)/cuda/include/nvToolsExt.h" && cp -f "/usr/local/cuda-9.0/include/nvToolsExtCuda.h" "$(@D)/cuda/include/nvToolsExtCuda.h" && cp -f "/usr/local/cuda-9.0/include/nvToolsExtCudaRt.h" "$(@D)/cuda/include/nvToolsExtCudaRt.h" && cp -f "/usr/local/cuda-9.0/include/nvToolsExtMeta.h" "$(@D)/cuda/include/nvToolsExtMeta.h" && cp -f "/usr/local/cuda-9.0/include/nvToolsExtSync.h" "$(@D)/cuda/include/nvToolsExtSync.h" && cp -f "/usr/local/cuda-9.0/include/nvblas.h" "$(@D)/cuda/include/nvblas.h" && cp -f "/usr/local/cuda-9.0/include/nvfunctional" "$(@D)/cuda/include/nvfunctional" && cp -f "/usr/local/cuda-9.0/include/nvgraph.h" "$(@D)/cuda/include/nvgraph.h" && cp -f "/usr/local/cuda-9.0/include/nvml.h" "$(@D)/cuda/include/nvml.h" && cp -f "/usr/local/cuda-9.0/include/nvrtc.h" "$(@D)/cuda/include/nvrtc.h" && cp -f "/usr/local/cuda-9.0/include/sm_20_atomic_functions.h" "$(@D)/cuda/include/sm_20_atomic_functions.h" && cp -f "/usr/local/cuda-9.0/include/sm_20_atomic_functions.hpp" "$(@D)/cuda/include/sm_20_atomic_functions.hpp" && cp -f "/usr/local/cuda-9.0/include/sm_20_intrinsics.h" "$(@D)/cuda/include/sm_20_intrinsics.h" && cp -f "/usr/local/cuda-9.0/include/sm_20_intrinsics.hpp" "$(@D)/cuda/include/sm_20_intrinsics.hpp" && cp -f "/usr/local/cuda-9.0/include/sm_30_intrinsics.h" "$(@D)/cuda/include/sm_30_intrinsics.h" && cp -f "/usr/local/cuda-9.0/include/sm_30_intrinsics.hpp" "$(@D)/cuda/include/sm_30_intrinsics.hpp" && cp -f "/usr/local/cuda-9.0/include/sm_32_atomic_functions.h" "$(@D)/cuda/include/sm_32_atomic_functions.h" && cp -f "/usr/local/cuda-9.0/include/sm_32_atomic_functions.hpp" "$(@D)/cuda/include/sm_32_atomic_functions.hpp" && cp -f "/usr/local/cuda-9.0/include/sm_32_intrinsics.h" "$(@D)/cuda/include/sm_32_intrinsics.h" && cp -f "/usr/local/cuda-9.0/include/sm_32_intrinsics.hpp" "$(@D)/cuda/include/sm_32_intrinsics.hpp" && cp -f "/usr/local/cuda-9.0/include/sm_35_atomic_functions.h" "$(@D)/cuda/include/sm_35_atomic_functions.h" && cp -f "/usr/local/cuda-9.0/include/sm_35_intrinsics.h" "$(@D)/cuda/include/sm_35_intrinsics.h" && cp -f "/usr/local/cuda-9.0/include/sm_60_atomic_functions.h" "$(@D)/cuda/include/sm_60_atomic_functions.h" && cp -f "/usr/local/cuda-9.0/include/sm_60_atomic_functions.hpp" "$(@D)/cuda/include/sm_60_atomic_functions.hpp" && cp -f "/usr/local/cuda-9.0/include/sm_61_intrinsics.h" "$(@D)/cuda/include/sm_61_intrinsics.h" && cp -f "/usr/local/cuda-9.0/include/sm_61_intrinsics.hpp" "$(@D)/cuda/include/sm_61_intrinsics.hpp" && cp -f "/usr/local/cuda-9.0/include/sobol_direction_vectors.h" "$(@D)/cuda/include/sobol_direction_vectors.h" && cp -f "/usr/local/cuda-9.0/include/surface_functions.h" "$(@D)/cuda/include/surface_functions.h" && cp -f "/usr/local/cuda-9.0/include/surface_functions.hpp" "$(@D)/cuda/include/surface_functions.hpp" && cp -f "/usr/local/cuda-9.0/include/surface_indirect_functions.h" "$(@D)/cuda/include/surface_indirect_functions.h" && cp -f "/usr/local/cuda-9.0/include/surface_indirect_functions.hpp" "$(@D)/cuda/include/surface_indirect_functions.hpp" && cp -f "/usr/local/cuda-9.0/include/surface_types.h" "$(@D)/cuda/include/surface_types.h" && cp -f "/usr/local/cuda-9.0/include/texture_fetch_functions.h" "$(@D)/cuda/include/texture_fetch_functions.h" && cp -f "/usr/local/cuda-9.0/include/texture_fetch_functions.hpp" "$(@D)/cuda/include/texture_fetch_functions.hpp" && cp -f "/usr/local/cuda-9.0/include/texture_indirect_functions.h" "$(@D)/cuda/include/texture_indirect_functions.h" && cp -f "/usr/local/cuda-9.0/include/texture_indirect_functions.hpp" "$(@D)/cuda/include/texture_indirect_functions.hpp" && cp -f "/usr/local/cuda-9.0/include/texture_types.h" "$(@D)/cuda/include/texture_types.h" && cp -f "/usr/local/cuda-9.0/include/thrust/adjacent_difference.h" "$(@D)/cuda/include/thrust/adjacent_difference.h" && cp -f "/usr/local/cuda-9.0/include/thrust/advance.h" "$(@D)/cuda/include/thrust/advance.h" && cp -f "/usr/local/cuda-9.0/include/thrust/binary_search.h" "$(@D)/cuda/include/thrust/binary_search.h" && cp -f "/usr/local/cuda-9.0/include/thrust/complex.h" "$(@D)/cuda/include/thrust/complex.h" && cp -f "/usr/local/cuda-9.0/include/thrust/copy.h" "$(@D)/cuda/include/thrust/copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/count.h" "$(@D)/cuda/include/thrust/count.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/adjacent_difference.inl" "$(@D)/cuda/include/thrust/detail/adjacent_difference.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/advance.inl" "$(@D)/cuda/include/thrust/detail/advance.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/allocator_traits.h" "$(@D)/cuda/include/thrust/detail/allocator/allocator_traits.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/allocator_traits.inl" "$(@D)/cuda/include/thrust/detail/allocator/allocator_traits.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/copy_construct_range.h" "$(@D)/cuda/include/thrust/detail/allocator/copy_construct_range.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/copy_construct_range.inl" "$(@D)/cuda/include/thrust/detail/allocator/copy_construct_range.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/default_construct_range.h" "$(@D)/cuda/include/thrust/detail/allocator/default_construct_range.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/default_construct_range.inl" "$(@D)/cuda/include/thrust/detail/allocator/default_construct_range.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/destroy_range.h" "$(@D)/cuda/include/thrust/detail/allocator/destroy_range.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/destroy_range.inl" "$(@D)/cuda/include/thrust/detail/allocator/destroy_range.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/fill_construct_range.h" "$(@D)/cuda/include/thrust/detail/allocator/fill_construct_range.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/fill_construct_range.inl" "$(@D)/cuda/include/thrust/detail/allocator/fill_construct_range.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/malloc_allocator.h" "$(@D)/cuda/include/thrust/detail/allocator/malloc_allocator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/malloc_allocator.inl" "$(@D)/cuda/include/thrust/detail/allocator/malloc_allocator.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/no_throw_allocator.h" "$(@D)/cuda/include/thrust/detail/allocator/no_throw_allocator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/tagged_allocator.h" "$(@D)/cuda/include/thrust/detail/allocator/tagged_allocator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/tagged_allocator.inl" "$(@D)/cuda/include/thrust/detail/allocator/tagged_allocator.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/temporary_allocator.h" "$(@D)/cuda/include/thrust/detail/allocator/temporary_allocator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/allocator/temporary_allocator.inl" "$(@D)/cuda/include/thrust/detail/allocator/temporary_allocator.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/binary_search.inl" "$(@D)/cuda/include/thrust/detail/binary_search.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/arithmetic.h" "$(@D)/cuda/include/thrust/detail/complex/arithmetic.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/c99math.h" "$(@D)/cuda/include/thrust/detail/complex/c99math.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/catrig.h" "$(@D)/cuda/include/thrust/detail/complex/catrig.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/catrigf.h" "$(@D)/cuda/include/thrust/detail/complex/catrigf.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/ccosh.h" "$(@D)/cuda/include/thrust/detail/complex/ccosh.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/ccoshf.h" "$(@D)/cuda/include/thrust/detail/complex/ccoshf.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/cexp.h" "$(@D)/cuda/include/thrust/detail/complex/cexp.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/cexpf.h" "$(@D)/cuda/include/thrust/detail/complex/cexpf.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/clog.h" "$(@D)/cuda/include/thrust/detail/complex/clog.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/clogf.h" "$(@D)/cuda/include/thrust/detail/complex/clogf.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/complex.inl" "$(@D)/cuda/include/thrust/detail/complex/complex.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/cpow.h" "$(@D)/cuda/include/thrust/detail/complex/cpow.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/cpowf.h" "$(@D)/cuda/include/thrust/detail/complex/cpowf.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/cproj.h" "$(@D)/cuda/include/thrust/detail/complex/cproj.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/csinh.h" "$(@D)/cuda/include/thrust/detail/complex/csinh.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/csinhf.h" "$(@D)/cuda/include/thrust/detail/complex/csinhf.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/csqrt.h" "$(@D)/cuda/include/thrust/detail/complex/csqrt.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/csqrtf.h" "$(@D)/cuda/include/thrust/detail/complex/csqrtf.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/ctanh.h" "$(@D)/cuda/include/thrust/detail/complex/ctanh.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/ctanhf.h" "$(@D)/cuda/include/thrust/detail/complex/ctanhf.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/math_private.h" "$(@D)/cuda/include/thrust/detail/complex/math_private.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/complex/stream.h" "$(@D)/cuda/include/thrust/detail/complex/stream.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/config.h" "$(@D)/cuda/include/thrust/detail/config.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/config/compiler.h" "$(@D)/cuda/include/thrust/detail/config/compiler.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/config/compiler_fence.h" "$(@D)/cuda/include/thrust/detail/config/compiler_fence.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/config/config.h" "$(@D)/cuda/include/thrust/detail/config/config.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/config/debug.h" "$(@D)/cuda/include/thrust/detail/config/debug.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/config/device_system.h" "$(@D)/cuda/include/thrust/detail/config/device_system.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/config/exec_check_disable.h" "$(@D)/cuda/include/thrust/detail/config/exec_check_disable.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/config/forceinline.h" "$(@D)/cuda/include/thrust/detail/config/forceinline.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/config/global_workarounds.h" "$(@D)/cuda/include/thrust/detail/config/global_workarounds.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/config/host_device.h" "$(@D)/cuda/include/thrust/detail/config/host_device.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/config/host_system.h" "$(@D)/cuda/include/thrust/detail/config/host_system.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/config/simple_defines.h" "$(@D)/cuda/include/thrust/detail/config/simple_defines.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/contiguous_storage.h" "$(@D)/cuda/include/thrust/detail/contiguous_storage.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/contiguous_storage.inl" "$(@D)/cuda/include/thrust/detail/contiguous_storage.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/copy.h" "$(@D)/cuda/include/thrust/detail/copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/copy.inl" "$(@D)/cuda/include/thrust/detail/copy.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/copy_if.h" "$(@D)/cuda/include/thrust/detail/copy_if.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/copy_if.inl" "$(@D)/cuda/include/thrust/detail/copy_if.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/count.inl" "$(@D)/cuda/include/thrust/detail/count.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/cstdint.h" "$(@D)/cuda/include/thrust/detail/cstdint.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/device_delete.inl" "$(@D)/cuda/include/thrust/detail/device_delete.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/device_free.inl" "$(@D)/cuda/include/thrust/detail/device_free.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/device_malloc.inl" "$(@D)/cuda/include/thrust/detail/device_malloc.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/device_new.inl" "$(@D)/cuda/include/thrust/detail/device_new.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/device_ptr.inl" "$(@D)/cuda/include/thrust/detail/device_ptr.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/device_reference.inl" "$(@D)/cuda/include/thrust/detail/device_reference.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/device_vector.inl" "$(@D)/cuda/include/thrust/detail/device_vector.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/dispatch/is_trivial_copy.h" "$(@D)/cuda/include/thrust/detail/dispatch/is_trivial_copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/distance.inl" "$(@D)/cuda/include/thrust/detail/distance.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/equal.inl" "$(@D)/cuda/include/thrust/detail/equal.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/execute_with_allocator.h" "$(@D)/cuda/include/thrust/detail/execute_with_allocator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/execution_policy.h" "$(@D)/cuda/include/thrust/detail/execution_policy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/extrema.inl" "$(@D)/cuda/include/thrust/detail/extrema.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/fill.inl" "$(@D)/cuda/include/thrust/detail/fill.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/find.inl" "$(@D)/cuda/include/thrust/detail/find.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/for_each.inl" "$(@D)/cuda/include/thrust/detail/for_each.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/function.h" "$(@D)/cuda/include/thrust/detail/function.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/functional.inl" "$(@D)/cuda/include/thrust/detail/functional.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/functional/actor.h" "$(@D)/cuda/include/thrust/detail/functional/actor.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/functional/actor.inl" "$(@D)/cuda/include/thrust/detail/functional/actor.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/functional/argument.h" "$(@D)/cuda/include/thrust/detail/functional/argument.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/functional/composite.h" "$(@D)/cuda/include/thrust/detail/functional/composite.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/functional/operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/functional/operators/arithmetic_operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators/arithmetic_operators.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/functional/operators/assignment_operator.h" "$(@D)/cuda/include/thrust/detail/functional/operators/assignment_operator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/functional/operators/bitwise_operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators/bitwise_operators.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/functional/operators/compound_assignment_operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators/compound_assignment_operators.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/functional/operators/logical_operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators/logical_operators.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/functional/operators/operator_adaptors.h" "$(@D)/cuda/include/thrust/detail/functional/operators/operator_adaptors.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/functional/operators/relational_operators.h" "$(@D)/cuda/include/thrust/detail/functional/operators/relational_operators.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/functional/placeholder.h" "$(@D)/cuda/include/thrust/detail/functional/placeholder.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/functional/value.h" "$(@D)/cuda/include/thrust/detail/functional/value.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/gather.inl" "$(@D)/cuda/include/thrust/detail/gather.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/generate.inl" "$(@D)/cuda/include/thrust/detail/generate.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/get_iterator_value.h" "$(@D)/cuda/include/thrust/detail/get_iterator_value.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/host_vector.inl" "$(@D)/cuda/include/thrust/detail/host_vector.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/inner_product.inl" "$(@D)/cuda/include/thrust/detail/inner_product.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/integer_math.h" "$(@D)/cuda/include/thrust/detail/integer_math.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/integer_traits.h" "$(@D)/cuda/include/thrust/detail/integer_traits.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/internal_functional.h" "$(@D)/cuda/include/thrust/detail/internal_functional.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/logical.inl" "$(@D)/cuda/include/thrust/detail/logical.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/malloc_and_free.h" "$(@D)/cuda/include/thrust/detail/malloc_and_free.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/merge.inl" "$(@D)/cuda/include/thrust/detail/merge.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/minmax.h" "$(@D)/cuda/include/thrust/detail/minmax.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/mismatch.inl" "$(@D)/cuda/include/thrust/detail/mismatch.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/mpl/math.h" "$(@D)/cuda/include/thrust/detail/mpl/math.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/numeric_traits.h" "$(@D)/cuda/include/thrust/detail/numeric_traits.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/overlapped_copy.h" "$(@D)/cuda/include/thrust/detail/overlapped_copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/pair.inl" "$(@D)/cuda/include/thrust/detail/pair.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/partition.inl" "$(@D)/cuda/include/thrust/detail/partition.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/pointer.h" "$(@D)/cuda/include/thrust/detail/pointer.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/pointer.inl" "$(@D)/cuda/include/thrust/detail/pointer.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/range/head_flags.h" "$(@D)/cuda/include/thrust/detail/range/head_flags.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/range/tail_flags.h" "$(@D)/cuda/include/thrust/detail/range/tail_flags.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/raw_pointer_cast.h" "$(@D)/cuda/include/thrust/detail/raw_pointer_cast.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/raw_reference_cast.h" "$(@D)/cuda/include/thrust/detail/raw_reference_cast.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/reduce.inl" "$(@D)/cuda/include/thrust/detail/reduce.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/reference.h" "$(@D)/cuda/include/thrust/detail/reference.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/reference.inl" "$(@D)/cuda/include/thrust/detail/reference.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/reference_forward_declaration.h" "$(@D)/cuda/include/thrust/detail/reference_forward_declaration.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/remove.inl" "$(@D)/cuda/include/thrust/detail/remove.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/replace.inl" "$(@D)/cuda/include/thrust/detail/replace.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/reverse.inl" "$(@D)/cuda/include/thrust/detail/reverse.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/scan.inl" "$(@D)/cuda/include/thrust/detail/scan.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/scatter.inl" "$(@D)/cuda/include/thrust/detail/scatter.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/seq.h" "$(@D)/cuda/include/thrust/detail/seq.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/sequence.inl" "$(@D)/cuda/include/thrust/detail/sequence.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/set_operations.inl" "$(@D)/cuda/include/thrust/detail/set_operations.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/sort.inl" "$(@D)/cuda/include/thrust/detail/sort.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/static_assert.h" "$(@D)/cuda/include/thrust/detail/static_assert.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/static_map.h" "$(@D)/cuda/include/thrust/detail/static_map.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/swap.h" "$(@D)/cuda/include/thrust/detail/swap.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/swap.inl" "$(@D)/cuda/include/thrust/detail/swap.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/swap_ranges.inl" "$(@D)/cuda/include/thrust/detail/swap_ranges.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/tabulate.inl" "$(@D)/cuda/include/thrust/detail/tabulate.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/temporary_array.h" "$(@D)/cuda/include/thrust/detail/temporary_array.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/temporary_array.inl" "$(@D)/cuda/include/thrust/detail/temporary_array.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/temporary_buffer.h" "$(@D)/cuda/include/thrust/detail/temporary_buffer.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/transform.inl" "$(@D)/cuda/include/thrust/detail/transform.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/transform_reduce.inl" "$(@D)/cuda/include/thrust/detail/transform_reduce.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/transform_scan.inl" "$(@D)/cuda/include/thrust/detail/transform_scan.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/trivial_sequence.h" "$(@D)/cuda/include/thrust/detail/trivial_sequence.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/tuple.inl" "$(@D)/cuda/include/thrust/detail/tuple.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/tuple_meta_transform.h" "$(@D)/cuda/include/thrust/detail/tuple_meta_transform.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/tuple_transform.h" "$(@D)/cuda/include/thrust/detail/tuple_transform.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/type_traits.h" "$(@D)/cuda/include/thrust/detail/type_traits.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/type_traits/algorithm/intermediate_type_from_function_and_iterators.h" "$(@D)/cuda/include/thrust/detail/type_traits/algorithm/intermediate_type_from_function_and_iterators.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/type_traits/function_traits.h" "$(@D)/cuda/include/thrust/detail/type_traits/function_traits.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/type_traits/has_member_function.h" "$(@D)/cuda/include/thrust/detail/type_traits/has_member_function.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/type_traits/has_nested_type.h" "$(@D)/cuda/include/thrust/detail/type_traits/has_nested_type.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/type_traits/has_trivial_assign.h" "$(@D)/cuda/include/thrust/detail/type_traits/has_trivial_assign.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/type_traits/is_call_possible.h" "$(@D)/cuda/include/thrust/detail/type_traits/is_call_possible.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/type_traits/is_metafunction_defined.h" "$(@D)/cuda/include/thrust/detail/type_traits/is_metafunction_defined.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/type_traits/iterator/is_discard_iterator.h" "$(@D)/cuda/include/thrust/detail/type_traits/iterator/is_discard_iterator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/type_traits/iterator/is_output_iterator.h" "$(@D)/cuda/include/thrust/detail/type_traits/iterator/is_output_iterator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/type_traits/minimum_type.h" "$(@D)/cuda/include/thrust/detail/type_traits/minimum_type.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/type_traits/pointer_traits.h" "$(@D)/cuda/include/thrust/detail/type_traits/pointer_traits.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/type_traits/result_of_adaptable_function.h" "$(@D)/cuda/include/thrust/detail/type_traits/result_of_adaptable_function.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/uninitialized_copy.inl" "$(@D)/cuda/include/thrust/detail/uninitialized_copy.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/uninitialized_fill.inl" "$(@D)/cuda/include/thrust/detail/uninitialized_fill.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/unique.inl" "$(@D)/cuda/include/thrust/detail/unique.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/use_default.h" "$(@D)/cuda/include/thrust/detail/use_default.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/util/align.h" "$(@D)/cuda/include/thrust/detail/util/align.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/util/blocking.h" "$(@D)/cuda/include/thrust/detail/util/blocking.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/vector_base.h" "$(@D)/cuda/include/thrust/detail/vector_base.h" && cp -f "/usr/local/cuda-9.0/include/thrust/detail/vector_base.inl" "$(@D)/cuda/include/thrust/detail/vector_base.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/device_allocator.h" "$(@D)/cuda/include/thrust/device_allocator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/device_delete.h" "$(@D)/cuda/include/thrust/device_delete.h" && cp -f "/usr/local/cuda-9.0/include/thrust/device_free.h" "$(@D)/cuda/include/thrust/device_free.h" && cp -f "/usr/local/cuda-9.0/include/thrust/device_malloc.h" "$(@D)/cuda/include/thrust/device_malloc.h" && cp -f "/usr/local/cuda-9.0/include/thrust/device_malloc_allocator.h" "$(@D)/cuda/include/thrust/device_malloc_allocator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/device_new.h" "$(@D)/cuda/include/thrust/device_new.h" && cp -f "/usr/local/cuda-9.0/include/thrust/device_new_allocator.h" "$(@D)/cuda/include/thrust/device_new_allocator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/device_ptr.h" "$(@D)/cuda/include/thrust/device_ptr.h" && cp -f "/usr/local/cuda-9.0/include/thrust/device_reference.h" "$(@D)/cuda/include/thrust/device_reference.h" && cp -f "/usr/local/cuda-9.0/include/thrust/device_vector.h" "$(@D)/cuda/include/thrust/device_vector.h" && cp -f "/usr/local/cuda-9.0/include/thrust/distance.h" "$(@D)/cuda/include/thrust/distance.h" && cp -f "/usr/local/cuda-9.0/include/thrust/equal.h" "$(@D)/cuda/include/thrust/equal.h" && cp -f "/usr/local/cuda-9.0/include/thrust/execution_policy.h" "$(@D)/cuda/include/thrust/execution_policy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/extrema.h" "$(@D)/cuda/include/thrust/extrema.h" && cp -f "/usr/local/cuda-9.0/include/thrust/fill.h" "$(@D)/cuda/include/thrust/fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/find.h" "$(@D)/cuda/include/thrust/find.h" && cp -f "/usr/local/cuda-9.0/include/thrust/for_each.h" "$(@D)/cuda/include/thrust/for_each.h" && cp -f "/usr/local/cuda-9.0/include/thrust/functional.h" "$(@D)/cuda/include/thrust/functional.h" && cp -f "/usr/local/cuda-9.0/include/thrust/gather.h" "$(@D)/cuda/include/thrust/gather.h" && cp -f "/usr/local/cuda-9.0/include/thrust/generate.h" "$(@D)/cuda/include/thrust/generate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/host_vector.h" "$(@D)/cuda/include/thrust/host_vector.h" && cp -f "/usr/local/cuda-9.0/include/thrust/inner_product.h" "$(@D)/cuda/include/thrust/inner_product.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/constant_iterator.h" "$(@D)/cuda/include/thrust/iterator/constant_iterator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/counting_iterator.h" "$(@D)/cuda/include/thrust/iterator/counting_iterator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/any_assign.h" "$(@D)/cuda/include/thrust/iterator/detail/any_assign.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/any_system_tag.h" "$(@D)/cuda/include/thrust/iterator/detail/any_system_tag.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/constant_iterator_base.h" "$(@D)/cuda/include/thrust/iterator/detail/constant_iterator_base.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/counting_iterator.inl" "$(@D)/cuda/include/thrust/iterator/detail/counting_iterator.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/device_system_tag.h" "$(@D)/cuda/include/thrust/iterator/detail/device_system_tag.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/discard_iterator_base.h" "$(@D)/cuda/include/thrust/iterator/detail/discard_iterator_base.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/distance_from_result.h" "$(@D)/cuda/include/thrust/iterator/detail/distance_from_result.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/host_system_tag.h" "$(@D)/cuda/include/thrust/iterator/detail/host_system_tag.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/is_iterator_category.h" "$(@D)/cuda/include/thrust/iterator/detail/is_iterator_category.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/is_trivial_iterator.h" "$(@D)/cuda/include/thrust/iterator/detail/is_trivial_iterator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/iterator_adaptor_base.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_adaptor_base.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/iterator_category_to_system.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_category_to_system.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/iterator_category_to_traversal.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_category_to_traversal.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/iterator_category_with_system_and_traversal.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_category_with_system_and_traversal.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/iterator_facade_category.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_facade_category.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/iterator_traits.inl" "$(@D)/cuda/include/thrust/iterator/detail/iterator_traits.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/iterator_traversal_tags.h" "$(@D)/cuda/include/thrust/iterator/detail/iterator_traversal_tags.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/join_iterator.h" "$(@D)/cuda/include/thrust/iterator/detail/join_iterator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/minimum_category.h" "$(@D)/cuda/include/thrust/iterator/detail/minimum_category.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/minimum_system.h" "$(@D)/cuda/include/thrust/iterator/detail/minimum_system.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/normal_iterator.h" "$(@D)/cuda/include/thrust/iterator/detail/normal_iterator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/permutation_iterator_base.h" "$(@D)/cuda/include/thrust/iterator/detail/permutation_iterator_base.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/retag.h" "$(@D)/cuda/include/thrust/iterator/detail/retag.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/reverse_iterator.inl" "$(@D)/cuda/include/thrust/iterator/detail/reverse_iterator.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/reverse_iterator_base.h" "$(@D)/cuda/include/thrust/iterator/detail/reverse_iterator_base.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/tagged_iterator.h" "$(@D)/cuda/include/thrust/iterator/detail/tagged_iterator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/transform_iterator.inl" "$(@D)/cuda/include/thrust/iterator/detail/transform_iterator.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/transform_output_iterator.inl" "$(@D)/cuda/include/thrust/iterator/detail/transform_output_iterator.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/tuple_of_iterator_references.h" "$(@D)/cuda/include/thrust/iterator/detail/tuple_of_iterator_references.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/universal_categories.h" "$(@D)/cuda/include/thrust/iterator/detail/universal_categories.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/zip_iterator.inl" "$(@D)/cuda/include/thrust/iterator/detail/zip_iterator.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/detail/zip_iterator_base.h" "$(@D)/cuda/include/thrust/iterator/detail/zip_iterator_base.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/discard_iterator.h" "$(@D)/cuda/include/thrust/iterator/discard_iterator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/iterator_adaptor.h" "$(@D)/cuda/include/thrust/iterator/iterator_adaptor.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/iterator_categories.h" "$(@D)/cuda/include/thrust/iterator/iterator_categories.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/iterator_facade.h" "$(@D)/cuda/include/thrust/iterator/iterator_facade.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/iterator_traits.h" "$(@D)/cuda/include/thrust/iterator/iterator_traits.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/permutation_iterator.h" "$(@D)/cuda/include/thrust/iterator/permutation_iterator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/retag.h" "$(@D)/cuda/include/thrust/iterator/retag.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/reverse_iterator.h" "$(@D)/cuda/include/thrust/iterator/reverse_iterator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/transform_iterator.h" "$(@D)/cuda/include/thrust/iterator/transform_iterator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/transform_output_iterator.h" "$(@D)/cuda/include/thrust/iterator/transform_output_iterator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/iterator/zip_iterator.h" "$(@D)/cuda/include/thrust/iterator/zip_iterator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/logical.h" "$(@D)/cuda/include/thrust/logical.h" && cp -f "/usr/local/cuda-9.0/include/thrust/memory.h" "$(@D)/cuda/include/thrust/memory.h" && cp -f "/usr/local/cuda-9.0/include/thrust/merge.h" "$(@D)/cuda/include/thrust/merge.h" && cp -f "/usr/local/cuda-9.0/include/thrust/mismatch.h" "$(@D)/cuda/include/thrust/mismatch.h" && cp -f "/usr/local/cuda-9.0/include/thrust/pair.h" "$(@D)/cuda/include/thrust/pair.h" && cp -f "/usr/local/cuda-9.0/include/thrust/partition.h" "$(@D)/cuda/include/thrust/partition.h" && cp -f "/usr/local/cuda-9.0/include/thrust/random.h" "$(@D)/cuda/include/thrust/random.h" && cp -f "/usr/local/cuda-9.0/include/thrust/random/detail/discard_block_engine.inl" "$(@D)/cuda/include/thrust/random/detail/discard_block_engine.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/random/detail/linear_congruential_engine.inl" "$(@D)/cuda/include/thrust/random/detail/linear_congruential_engine.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/random/detail/linear_congruential_engine_discard.h" "$(@D)/cuda/include/thrust/random/detail/linear_congruential_engine_discard.h" && cp -f "/usr/local/cuda-9.0/include/thrust/random/detail/linear_feedback_shift_engine.inl" "$(@D)/cuda/include/thrust/random/detail/linear_feedback_shift_engine.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/random/detail/linear_feedback_shift_engine_wordmask.h" "$(@D)/cuda/include/thrust/random/detail/linear_feedback_shift_engine_wordmask.h" && cp -f "/usr/local/cuda-9.0/include/thrust/random/detail/mod.h" "$(@D)/cuda/include/thrust/random/detail/mod.h" && cp -f "/usr/local/cuda-9.0/include/thrust/random/detail/normal_distribution.inl" "$(@D)/cuda/include/thrust/random/detail/normal_distribution.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/random/detail/normal_distribution_base.h" "$(@D)/cuda/include/thrust/random/detail/normal_distribution_base.h" && cp -f "/usr/local/cuda-9.0/include/thrust/random/detail/random_core_access.h" "$(@D)/cuda/include/thrust/random/detail/random_core_access.h" && cp -f "/usr/local/cuda-9.0/include/thrust/random/detail/subtract_with_carry_engine.inl" "$(@D)/cuda/include/thrust/random/detail/subtract_with_carry_engine.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/random/detail/uniform_int_distribution.inl" "$(@D)/cuda/include/thrust/random/detail/uniform_int_distribution.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/random/detail/uniform_real_distribution.inl" "$(@D)/cuda/include/thrust/random/detail/uniform_real_distribution.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/random/detail/xor_combine_engine.inl" "$(@D)/cuda/include/thrust/random/detail/xor_combine_engine.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/random/detail/xor_combine_engine_max.h" "$(@D)/cuda/include/thrust/random/detail/xor_combine_engine_max.h" && cp -f "/usr/local/cuda-9.0/include/thrust/random/discard_block_engine.h" "$(@D)/cuda/include/thrust/random/discard_block_engine.h" && cp -f "/usr/local/cuda-9.0/include/thrust/random/linear_congruential_engine.h" "$(@D)/cuda/include/thrust/random/linear_congruential_engine.h" && cp -f "/usr/local/cuda-9.0/include/thrust/random/linear_feedback_shift_engine.h" "$(@D)/cuda/include/thrust/random/linear_feedback_shift_engine.h" && cp -f "/usr/local/cuda-9.0/include/thrust/random/normal_distribution.h" "$(@D)/cuda/include/thrust/random/normal_distribution.h" && cp -f "/usr/local/cuda-9.0/include/thrust/random/subtract_with_carry_engine.h" "$(@D)/cuda/include/thrust/random/subtract_with_carry_engine.h" && cp -f "/usr/local/cuda-9.0/include/thrust/random/uniform_int_distribution.h" "$(@D)/cuda/include/thrust/random/uniform_int_distribution.h" && cp -f "/usr/local/cuda-9.0/include/thrust/random/uniform_real_distribution.h" "$(@D)/cuda/include/thrust/random/uniform_real_distribution.h" && cp -f "/usr/local/cuda-9.0/include/thrust/random/xor_combine_engine.h" "$(@D)/cuda/include/thrust/random/xor_combine_engine.h" && cp -f "/usr/local/cuda-9.0/include/thrust/reduce.h" "$(@D)/cuda/include/thrust/reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/remove.h" "$(@D)/cuda/include/thrust/remove.h" && cp -f "/usr/local/cuda-9.0/include/thrust/replace.h" "$(@D)/cuda/include/thrust/replace.h" && cp -f "/usr/local/cuda-9.0/include/thrust/reverse.h" "$(@D)/cuda/include/thrust/reverse.h" && cp -f "/usr/local/cuda-9.0/include/thrust/scan.h" "$(@D)/cuda/include/thrust/scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/scatter.h" "$(@D)/cuda/include/thrust/scatter.h" && cp -f "/usr/local/cuda-9.0/include/thrust/sequence.h" "$(@D)/cuda/include/thrust/sequence.h" && cp -f "/usr/local/cuda-9.0/include/thrust/set_operations.h" "$(@D)/cuda/include/thrust/set_operations.h" && cp -f "/usr/local/cuda-9.0/include/thrust/sort.h" "$(@D)/cuda/include/thrust/sort.h" && cp -f "/usr/local/cuda-9.0/include/thrust/swap.h" "$(@D)/cuda/include/thrust/swap.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/cpp/detail/adjacent_difference.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/assign_value.h" "$(@D)/cuda/include/thrust/system/cpp/detail/assign_value.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/binary_search.h" "$(@D)/cuda/include/thrust/system/cpp/detail/binary_search.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/copy.h" "$(@D)/cuda/include/thrust/system/cpp/detail/copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/copy_if.h" "$(@D)/cuda/include/thrust/system/cpp/detail/copy_if.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/count.h" "$(@D)/cuda/include/thrust/system/cpp/detail/count.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/equal.h" "$(@D)/cuda/include/thrust/system/cpp/detail/equal.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/execution_policy.h" "$(@D)/cuda/include/thrust/system/cpp/detail/execution_policy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/extrema.h" "$(@D)/cuda/include/thrust/system/cpp/detail/extrema.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/fill.h" "$(@D)/cuda/include/thrust/system/cpp/detail/fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/find.h" "$(@D)/cuda/include/thrust/system/cpp/detail/find.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/for_each.h" "$(@D)/cuda/include/thrust/system/cpp/detail/for_each.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/gather.h" "$(@D)/cuda/include/thrust/system/cpp/detail/gather.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/generate.h" "$(@D)/cuda/include/thrust/system/cpp/detail/generate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/get_value.h" "$(@D)/cuda/include/thrust/system/cpp/detail/get_value.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/inner_product.h" "$(@D)/cuda/include/thrust/system/cpp/detail/inner_product.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/iter_swap.h" "$(@D)/cuda/include/thrust/system/cpp/detail/iter_swap.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/logical.h" "$(@D)/cuda/include/thrust/system/cpp/detail/logical.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/cpp/detail/malloc_and_free.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/memory.inl" "$(@D)/cuda/include/thrust/system/cpp/detail/memory.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/merge.h" "$(@D)/cuda/include/thrust/system/cpp/detail/merge.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/mismatch.h" "$(@D)/cuda/include/thrust/system/cpp/detail/mismatch.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/par.h" "$(@D)/cuda/include/thrust/system/cpp/detail/par.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/partition.h" "$(@D)/cuda/include/thrust/system/cpp/detail/partition.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/reduce.h" "$(@D)/cuda/include/thrust/system/cpp/detail/reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/cpp/detail/reduce_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/remove.h" "$(@D)/cuda/include/thrust/system/cpp/detail/remove.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/replace.h" "$(@D)/cuda/include/thrust/system/cpp/detail/replace.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/reverse.h" "$(@D)/cuda/include/thrust/system/cpp/detail/reverse.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/scan.h" "$(@D)/cuda/include/thrust/system/cpp/detail/scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/scan_by_key.h" "$(@D)/cuda/include/thrust/system/cpp/detail/scan_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/scatter.h" "$(@D)/cuda/include/thrust/system/cpp/detail/scatter.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/sequence.h" "$(@D)/cuda/include/thrust/system/cpp/detail/sequence.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/set_operations.h" "$(@D)/cuda/include/thrust/system/cpp/detail/set_operations.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/sort.h" "$(@D)/cuda/include/thrust/system/cpp/detail/sort.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/swap_ranges.h" "$(@D)/cuda/include/thrust/system/cpp/detail/swap_ranges.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/tabulate.h" "$(@D)/cuda/include/thrust/system/cpp/detail/tabulate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/cpp/detail/temporary_buffer.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/transform.h" "$(@D)/cuda/include/thrust/system/cpp/detail/transform.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/transform_reduce.h" "$(@D)/cuda/include/thrust/system/cpp/detail/transform_reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/transform_scan.h" "$(@D)/cuda/include/thrust/system/cpp/detail/transform_scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/cpp/detail/uninitialized_copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/cpp/detail/uninitialized_fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/unique.h" "$(@D)/cuda/include/thrust/system/cpp/detail/unique.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/unique_by_key.h" "$(@D)/cuda/include/thrust/system/cpp/detail/unique_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/detail/vector.inl" "$(@D)/cuda/include/thrust/system/cpp/detail/vector.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/execution_policy.h" "$(@D)/cuda/include/thrust/system/cpp/execution_policy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/memory.h" "$(@D)/cuda/include/thrust/system/cpp/memory.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cpp/vector.h" "$(@D)/cuda/include/thrust/system/cpp/vector.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/config.h" "$(@D)/cuda/include/thrust/system/cuda/config.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/cuda/detail/adjacent_difference.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/assign_value.h" "$(@D)/cuda/include/thrust/system/cuda/detail/assign_value.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/binary_search.h" "$(@D)/cuda/include/thrust/system/cuda/detail/binary_search.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/copy.h" "$(@D)/cuda/include/thrust/system/cuda/detail/copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/copy_if.h" "$(@D)/cuda/include/thrust/system/cuda/detail/copy_if.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/core/agent_launcher.h" "$(@D)/cuda/include/thrust/system/cuda/detail/core/agent_launcher.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/core/alignment.h" "$(@D)/cuda/include/thrust/system/cuda/detail/core/alignment.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/core/triple_chevron_launch.h" "$(@D)/cuda/include/thrust/system/cuda/detail/core/triple_chevron_launch.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/core/util.h" "$(@D)/cuda/include/thrust/system/cuda/detail/core/util.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/count.h" "$(@D)/cuda/include/thrust/system/cuda/detail/count.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cross_system.h" "$(@D)/cuda/include/thrust/system/cuda/detail/cross_system.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_histogram.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_histogram.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_radix_sort_downsweep.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_radix_sort_downsweep.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_radix_sort_upsweep.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_radix_sort_upsweep.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_reduce.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_reduce_by_key.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_reduce_by_key.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_rle.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_rle.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_scan.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_segment_fixup.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_segment_fixup.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_select_if.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_select_if.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_spmv_csrt.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_spmv_csrt.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_spmv_orig.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_spmv_orig.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/agent_spmv_row_based.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/agent_spmv_row_based.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/agent/single_pass_scan_operators.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/agent/single_pass_scan_operators.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_adjacent_difference.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_adjacent_difference.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_discontinuity.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_discontinuity.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_exchange.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_exchange.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_histogram.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_histogram.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_load.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_load.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_radix_rank.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_radix_rank.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_radix_sort.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_radix_sort.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_raking_layout.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_raking_layout.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_reduce.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_scan.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_shuffle.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_shuffle.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/block_store.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/block_store.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/specializations/block_histogram_atomic.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_histogram_atomic.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/specializations/block_histogram_sort.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_histogram_sort.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_raking.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_raking.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_raking_commutative_only.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_raking_commutative_only.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_warp_reductions.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_reduce_warp_reductions.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_raking.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_raking.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans2.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans2.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans3.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/block/specializations/block_scan_warp_scans3.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/cub.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/cub.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_histogram.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_histogram.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_partition.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_partition.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_radix_sort.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_radix_sort.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_reduce.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_run_length_encode.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_run_length_encode.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_scan.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_segmented_radix_sort.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_segmented_radix_sort.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_segmented_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_segmented_reduce.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_select.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_select.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/device_spmv.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/device_spmv.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_histogram.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_histogram.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_radix_sort.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_radix_sort.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_reduce.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_reduce_by_key.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_reduce_by_key.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_rle.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_rle.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_scan.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_select_if.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_select_if.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_spmv_csrt.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_spmv_csrt.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_spmv_orig.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_spmv_orig.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_spmv_row_based.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/device/dispatch/dispatch_spmv_row_based.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/grid/grid_barrier.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/grid/grid_barrier.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/grid/grid_even_share.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/grid/grid_even_share.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/grid/grid_mapping.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/grid/grid_mapping.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/grid/grid_queue.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/grid/grid_queue.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/host/mutex.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/host/mutex.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/iterator/arg_index_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/arg_index_input_iterator.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/iterator/cache_modified_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/cache_modified_input_iterator.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/iterator/cache_modified_output_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/cache_modified_output_iterator.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/iterator/constant_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/constant_input_iterator.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/iterator/counting_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/counting_input_iterator.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/iterator/discard_output_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/discard_output_iterator.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/iterator/tex_obj_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/tex_obj_input_iterator.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/iterator/tex_ref_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/tex_ref_input_iterator.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/iterator/transform_input_iterator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/iterator/transform_input_iterator.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/thread/thread_load.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_load.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/thread/thread_operators.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_operators.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/thread/thread_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_reduce.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/thread/thread_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_scan.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/thread/thread_search.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_search.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/thread/thread_store.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/thread/thread_store.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/util_allocator.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_allocator.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/util_arch.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_arch.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/util_debug.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_debug.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/util_device.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_device.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/util_macro.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_macro.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/util_namespace.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_namespace.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/util_ptx.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_ptx.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/util_type.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/util_type.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/warp/specializations/warp_reduce_shfl.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/specializations/warp_reduce_shfl.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/warp/specializations/warp_reduce_smem.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/specializations/warp_reduce_smem.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/warp/specializations/warp_scan_shfl.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/specializations/warp_scan_shfl.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/warp/specializations/warp_scan_smem.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/specializations/warp_scan_smem.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/warp/warp_reduce.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/warp_reduce.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/cub/warp/warp_scan.cuh" "$(@D)/cuda/include/thrust/system/cuda/detail/cub/warp/warp_scan.cuh" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/equal.h" "$(@D)/cuda/include/thrust/system/cuda/detail/equal.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/error.inl" "$(@D)/cuda/include/thrust/system/cuda/detail/error.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/execution_policy.h" "$(@D)/cuda/include/thrust/system/cuda/detail/execution_policy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/extrema.h" "$(@D)/cuda/include/thrust/system/cuda/detail/extrema.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/fill.h" "$(@D)/cuda/include/thrust/system/cuda/detail/fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/find.h" "$(@D)/cuda/include/thrust/system/cuda/detail/find.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/for_each.h" "$(@D)/cuda/include/thrust/system/cuda/detail/for_each.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/gather.h" "$(@D)/cuda/include/thrust/system/cuda/detail/gather.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/generate.h" "$(@D)/cuda/include/thrust/system/cuda/detail/generate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/get_value.h" "$(@D)/cuda/include/thrust/system/cuda/detail/get_value.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/guarded_cuda_runtime_api.h" "$(@D)/cuda/include/thrust/system/cuda/detail/guarded_cuda_runtime_api.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/guarded_driver_types.h" "$(@D)/cuda/include/thrust/system/cuda/detail/guarded_driver_types.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/inner_product.h" "$(@D)/cuda/include/thrust/system/cuda/detail/inner_product.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/internal/copy_cross_system.h" "$(@D)/cuda/include/thrust/system/cuda/detail/internal/copy_cross_system.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/internal/copy_device_to_device.h" "$(@D)/cuda/include/thrust/system/cuda/detail/internal/copy_device_to_device.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/iter_swap.h" "$(@D)/cuda/include/thrust/system/cuda/detail/iter_swap.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/logical.h" "$(@D)/cuda/include/thrust/system/cuda/detail/logical.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/cuda/detail/malloc_and_free.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/memory.inl" "$(@D)/cuda/include/thrust/system/cuda/detail/memory.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/memory_buffer.h" "$(@D)/cuda/include/thrust/system/cuda/detail/memory_buffer.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/merge.h" "$(@D)/cuda/include/thrust/system/cuda/detail/merge.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/mismatch.h" "$(@D)/cuda/include/thrust/system/cuda/detail/mismatch.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/par.h" "$(@D)/cuda/include/thrust/system/cuda/detail/par.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/par_to_seq.h" "$(@D)/cuda/include/thrust/system/cuda/detail/par_to_seq.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/parallel_for.h" "$(@D)/cuda/include/thrust/system/cuda/detail/parallel_for.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/partition.h" "$(@D)/cuda/include/thrust/system/cuda/detail/partition.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/reduce.h" "$(@D)/cuda/include/thrust/system/cuda/detail/reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/cuda/detail/reduce_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/remove.h" "$(@D)/cuda/include/thrust/system/cuda/detail/remove.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/replace.h" "$(@D)/cuda/include/thrust/system/cuda/detail/replace.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/reverse.h" "$(@D)/cuda/include/thrust/system/cuda/detail/reverse.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/scan.h" "$(@D)/cuda/include/thrust/system/cuda/detail/scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/scan_by_key.h" "$(@D)/cuda/include/thrust/system/cuda/detail/scan_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/scatter.h" "$(@D)/cuda/include/thrust/system/cuda/detail/scatter.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/sequence.h" "$(@D)/cuda/include/thrust/system/cuda/detail/sequence.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/set_operations.h" "$(@D)/cuda/include/thrust/system/cuda/detail/set_operations.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/sort.h" "$(@D)/cuda/include/thrust/system/cuda/detail/sort.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/swap_ranges.h" "$(@D)/cuda/include/thrust/system/cuda/detail/swap_ranges.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/tabulate.h" "$(@D)/cuda/include/thrust/system/cuda/detail/tabulate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/cuda/detail/temporary_buffer.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/terminate.h" "$(@D)/cuda/include/thrust/system/cuda/detail/terminate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/transform.h" "$(@D)/cuda/include/thrust/system/cuda/detail/transform.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/transform_reduce.h" "$(@D)/cuda/include/thrust/system/cuda/detail/transform_reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/transform_scan.h" "$(@D)/cuda/include/thrust/system/cuda/detail/transform_scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/cuda/detail/uninitialized_copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/cuda/detail/uninitialized_fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/unique.h" "$(@D)/cuda/include/thrust/system/cuda/detail/unique.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/unique_by_key.h" "$(@D)/cuda/include/thrust/system/cuda/detail/unique_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/util.h" "$(@D)/cuda/include/thrust/system/cuda/detail/util.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/detail/vector.inl" "$(@D)/cuda/include/thrust/system/cuda/detail/vector.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/error.h" "$(@D)/cuda/include/thrust/system/cuda/error.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/execution_policy.h" "$(@D)/cuda/include/thrust/system/cuda/execution_policy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/experimental/pinned_allocator.h" "$(@D)/cuda/include/thrust/system/cuda/experimental/pinned_allocator.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/memory.h" "$(@D)/cuda/include/thrust/system/cuda/memory.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/cuda/vector.h" "$(@D)/cuda/include/thrust/system/cuda/vector.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/detail/adl/adjacent_difference.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/assign_value.h" "$(@D)/cuda/include/thrust/system/detail/adl/assign_value.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/binary_search.h" "$(@D)/cuda/include/thrust/system/detail/adl/binary_search.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/copy.h" "$(@D)/cuda/include/thrust/system/detail/adl/copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/copy_if.h" "$(@D)/cuda/include/thrust/system/detail/adl/copy_if.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/count.h" "$(@D)/cuda/include/thrust/system/detail/adl/count.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/equal.h" "$(@D)/cuda/include/thrust/system/detail/adl/equal.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/extrema.h" "$(@D)/cuda/include/thrust/system/detail/adl/extrema.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/fill.h" "$(@D)/cuda/include/thrust/system/detail/adl/fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/find.h" "$(@D)/cuda/include/thrust/system/detail/adl/find.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/for_each.h" "$(@D)/cuda/include/thrust/system/detail/adl/for_each.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/gather.h" "$(@D)/cuda/include/thrust/system/detail/adl/gather.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/generate.h" "$(@D)/cuda/include/thrust/system/detail/adl/generate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/get_value.h" "$(@D)/cuda/include/thrust/system/detail/adl/get_value.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/inner_product.h" "$(@D)/cuda/include/thrust/system/detail/adl/inner_product.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/iter_swap.h" "$(@D)/cuda/include/thrust/system/detail/adl/iter_swap.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/logical.h" "$(@D)/cuda/include/thrust/system/detail/adl/logical.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/detail/adl/malloc_and_free.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/merge.h" "$(@D)/cuda/include/thrust/system/detail/adl/merge.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/mismatch.h" "$(@D)/cuda/include/thrust/system/detail/adl/mismatch.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/partition.h" "$(@D)/cuda/include/thrust/system/detail/adl/partition.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/reduce.h" "$(@D)/cuda/include/thrust/system/detail/adl/reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/detail/adl/reduce_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/remove.h" "$(@D)/cuda/include/thrust/system/detail/adl/remove.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/replace.h" "$(@D)/cuda/include/thrust/system/detail/adl/replace.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/reverse.h" "$(@D)/cuda/include/thrust/system/detail/adl/reverse.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/scan.h" "$(@D)/cuda/include/thrust/system/detail/adl/scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/scan_by_key.h" "$(@D)/cuda/include/thrust/system/detail/adl/scan_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/scatter.h" "$(@D)/cuda/include/thrust/system/detail/adl/scatter.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/sequence.h" "$(@D)/cuda/include/thrust/system/detail/adl/sequence.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/set_operations.h" "$(@D)/cuda/include/thrust/system/detail/adl/set_operations.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/sort.h" "$(@D)/cuda/include/thrust/system/detail/adl/sort.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/swap_ranges.h" "$(@D)/cuda/include/thrust/system/detail/adl/swap_ranges.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/tabulate.h" "$(@D)/cuda/include/thrust/system/detail/adl/tabulate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/detail/adl/temporary_buffer.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/transform.h" "$(@D)/cuda/include/thrust/system/detail/adl/transform.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/transform_reduce.h" "$(@D)/cuda/include/thrust/system/detail/adl/transform_reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/transform_scan.h" "$(@D)/cuda/include/thrust/system/detail/adl/transform_scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/detail/adl/uninitialized_copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/detail/adl/uninitialized_fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/unique.h" "$(@D)/cuda/include/thrust/system/detail/adl/unique.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/adl/unique_by_key.h" "$(@D)/cuda/include/thrust/system/detail/adl/unique_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/bad_alloc.h" "$(@D)/cuda/include/thrust/system/detail/bad_alloc.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/errno.h" "$(@D)/cuda/include/thrust/system/detail/errno.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/error_category.inl" "$(@D)/cuda/include/thrust/system/detail/error_category.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/error_code.inl" "$(@D)/cuda/include/thrust/system/detail/error_code.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/error_condition.inl" "$(@D)/cuda/include/thrust/system/detail/error_condition.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/detail/generic/adjacent_difference.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/adjacent_difference.inl" "$(@D)/cuda/include/thrust/system/detail/generic/adjacent_difference.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/advance.h" "$(@D)/cuda/include/thrust/system/detail/generic/advance.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/advance.inl" "$(@D)/cuda/include/thrust/system/detail/generic/advance.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/binary_search.h" "$(@D)/cuda/include/thrust/system/detail/generic/binary_search.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/binary_search.inl" "$(@D)/cuda/include/thrust/system/detail/generic/binary_search.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/copy.h" "$(@D)/cuda/include/thrust/system/detail/generic/copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/copy.inl" "$(@D)/cuda/include/thrust/system/detail/generic/copy.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/copy_if.h" "$(@D)/cuda/include/thrust/system/detail/generic/copy_if.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/copy_if.inl" "$(@D)/cuda/include/thrust/system/detail/generic/copy_if.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/count.h" "$(@D)/cuda/include/thrust/system/detail/generic/count.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/count.inl" "$(@D)/cuda/include/thrust/system/detail/generic/count.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/distance.h" "$(@D)/cuda/include/thrust/system/detail/generic/distance.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/distance.inl" "$(@D)/cuda/include/thrust/system/detail/generic/distance.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/equal.h" "$(@D)/cuda/include/thrust/system/detail/generic/equal.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/equal.inl" "$(@D)/cuda/include/thrust/system/detail/generic/equal.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/extrema.h" "$(@D)/cuda/include/thrust/system/detail/generic/extrema.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/extrema.inl" "$(@D)/cuda/include/thrust/system/detail/generic/extrema.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/fill.h" "$(@D)/cuda/include/thrust/system/detail/generic/fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/find.h" "$(@D)/cuda/include/thrust/system/detail/generic/find.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/find.inl" "$(@D)/cuda/include/thrust/system/detail/generic/find.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/for_each.h" "$(@D)/cuda/include/thrust/system/detail/generic/for_each.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/gather.h" "$(@D)/cuda/include/thrust/system/detail/generic/gather.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/gather.inl" "$(@D)/cuda/include/thrust/system/detail/generic/gather.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/generate.h" "$(@D)/cuda/include/thrust/system/detail/generic/generate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/generate.inl" "$(@D)/cuda/include/thrust/system/detail/generic/generate.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/inner_product.h" "$(@D)/cuda/include/thrust/system/detail/generic/inner_product.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/inner_product.inl" "$(@D)/cuda/include/thrust/system/detail/generic/inner_product.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/logical.h" "$(@D)/cuda/include/thrust/system/detail/generic/logical.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/memory.h" "$(@D)/cuda/include/thrust/system/detail/generic/memory.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/memory.inl" "$(@D)/cuda/include/thrust/system/detail/generic/memory.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/merge.h" "$(@D)/cuda/include/thrust/system/detail/generic/merge.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/merge.inl" "$(@D)/cuda/include/thrust/system/detail/generic/merge.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/mismatch.h" "$(@D)/cuda/include/thrust/system/detail/generic/mismatch.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/mismatch.inl" "$(@D)/cuda/include/thrust/system/detail/generic/mismatch.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/partition.h" "$(@D)/cuda/include/thrust/system/detail/generic/partition.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/partition.inl" "$(@D)/cuda/include/thrust/system/detail/generic/partition.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/reduce.h" "$(@D)/cuda/include/thrust/system/detail/generic/reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/reduce.inl" "$(@D)/cuda/include/thrust/system/detail/generic/reduce.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/detail/generic/reduce_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/reduce_by_key.inl" "$(@D)/cuda/include/thrust/system/detail/generic/reduce_by_key.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/remove.h" "$(@D)/cuda/include/thrust/system/detail/generic/remove.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/remove.inl" "$(@D)/cuda/include/thrust/system/detail/generic/remove.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/replace.h" "$(@D)/cuda/include/thrust/system/detail/generic/replace.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/replace.inl" "$(@D)/cuda/include/thrust/system/detail/generic/replace.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/reverse.h" "$(@D)/cuda/include/thrust/system/detail/generic/reverse.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/reverse.inl" "$(@D)/cuda/include/thrust/system/detail/generic/reverse.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/scalar/binary_search.h" "$(@D)/cuda/include/thrust/system/detail/generic/scalar/binary_search.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/scalar/binary_search.inl" "$(@D)/cuda/include/thrust/system/detail/generic/scalar/binary_search.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/scan.h" "$(@D)/cuda/include/thrust/system/detail/generic/scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/scan.inl" "$(@D)/cuda/include/thrust/system/detail/generic/scan.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/scan_by_key.h" "$(@D)/cuda/include/thrust/system/detail/generic/scan_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/scan_by_key.inl" "$(@D)/cuda/include/thrust/system/detail/generic/scan_by_key.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/scatter.h" "$(@D)/cuda/include/thrust/system/detail/generic/scatter.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/scatter.inl" "$(@D)/cuda/include/thrust/system/detail/generic/scatter.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/select_system.h" "$(@D)/cuda/include/thrust/system/detail/generic/select_system.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/sequence.h" "$(@D)/cuda/include/thrust/system/detail/generic/sequence.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/sequence.inl" "$(@D)/cuda/include/thrust/system/detail/generic/sequence.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/set_operations.h" "$(@D)/cuda/include/thrust/system/detail/generic/set_operations.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/set_operations.inl" "$(@D)/cuda/include/thrust/system/detail/generic/set_operations.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/sort.h" "$(@D)/cuda/include/thrust/system/detail/generic/sort.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/sort.inl" "$(@D)/cuda/include/thrust/system/detail/generic/sort.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/swap_ranges.h" "$(@D)/cuda/include/thrust/system/detail/generic/swap_ranges.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/swap_ranges.inl" "$(@D)/cuda/include/thrust/system/detail/generic/swap_ranges.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/tabulate.h" "$(@D)/cuda/include/thrust/system/detail/generic/tabulate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/tabulate.inl" "$(@D)/cuda/include/thrust/system/detail/generic/tabulate.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/tag.h" "$(@D)/cuda/include/thrust/system/detail/generic/tag.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/detail/generic/temporary_buffer.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/temporary_buffer.inl" "$(@D)/cuda/include/thrust/system/detail/generic/temporary_buffer.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/transform.h" "$(@D)/cuda/include/thrust/system/detail/generic/transform.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/transform.inl" "$(@D)/cuda/include/thrust/system/detail/generic/transform.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/transform_reduce.h" "$(@D)/cuda/include/thrust/system/detail/generic/transform_reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/transform_reduce.inl" "$(@D)/cuda/include/thrust/system/detail/generic/transform_reduce.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/transform_scan.h" "$(@D)/cuda/include/thrust/system/detail/generic/transform_scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/transform_scan.inl" "$(@D)/cuda/include/thrust/system/detail/generic/transform_scan.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/type_traits.h" "$(@D)/cuda/include/thrust/system/detail/generic/type_traits.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/detail/generic/uninitialized_copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/uninitialized_copy.inl" "$(@D)/cuda/include/thrust/system/detail/generic/uninitialized_copy.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/detail/generic/uninitialized_fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/uninitialized_fill.inl" "$(@D)/cuda/include/thrust/system/detail/generic/uninitialized_fill.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/unique.h" "$(@D)/cuda/include/thrust/system/detail/generic/unique.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/unique.inl" "$(@D)/cuda/include/thrust/system/detail/generic/unique.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/unique_by_key.h" "$(@D)/cuda/include/thrust/system/detail/generic/unique_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/generic/unique_by_key.inl" "$(@D)/cuda/include/thrust/system/detail/generic/unique_by_key.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/internal/decompose.h" "$(@D)/cuda/include/thrust/system/detail/internal/decompose.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/detail/sequential/adjacent_difference.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/assign_value.h" "$(@D)/cuda/include/thrust/system/detail/sequential/assign_value.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/binary_search.h" "$(@D)/cuda/include/thrust/system/detail/sequential/binary_search.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/copy.h" "$(@D)/cuda/include/thrust/system/detail/sequential/copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/copy.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/copy.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/copy_backward.h" "$(@D)/cuda/include/thrust/system/detail/sequential/copy_backward.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/copy_if.h" "$(@D)/cuda/include/thrust/system/detail/sequential/copy_if.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/count.h" "$(@D)/cuda/include/thrust/system/detail/sequential/count.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/equal.h" "$(@D)/cuda/include/thrust/system/detail/sequential/equal.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/execution_policy.h" "$(@D)/cuda/include/thrust/system/detail/sequential/execution_policy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/extrema.h" "$(@D)/cuda/include/thrust/system/detail/sequential/extrema.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/fill.h" "$(@D)/cuda/include/thrust/system/detail/sequential/fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/find.h" "$(@D)/cuda/include/thrust/system/detail/sequential/find.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/for_each.h" "$(@D)/cuda/include/thrust/system/detail/sequential/for_each.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/gather.h" "$(@D)/cuda/include/thrust/system/detail/sequential/gather.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/general_copy.h" "$(@D)/cuda/include/thrust/system/detail/sequential/general_copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/generate.h" "$(@D)/cuda/include/thrust/system/detail/sequential/generate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/get_value.h" "$(@D)/cuda/include/thrust/system/detail/sequential/get_value.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/inner_product.h" "$(@D)/cuda/include/thrust/system/detail/sequential/inner_product.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/insertion_sort.h" "$(@D)/cuda/include/thrust/system/detail/sequential/insertion_sort.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/iter_swap.h" "$(@D)/cuda/include/thrust/system/detail/sequential/iter_swap.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/logical.h" "$(@D)/cuda/include/thrust/system/detail/sequential/logical.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/detail/sequential/malloc_and_free.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/merge.h" "$(@D)/cuda/include/thrust/system/detail/sequential/merge.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/merge.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/merge.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/mismatch.h" "$(@D)/cuda/include/thrust/system/detail/sequential/mismatch.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/partition.h" "$(@D)/cuda/include/thrust/system/detail/sequential/partition.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/reduce.h" "$(@D)/cuda/include/thrust/system/detail/sequential/reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/detail/sequential/reduce_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/remove.h" "$(@D)/cuda/include/thrust/system/detail/sequential/remove.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/replace.h" "$(@D)/cuda/include/thrust/system/detail/sequential/replace.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/reverse.h" "$(@D)/cuda/include/thrust/system/detail/sequential/reverse.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/scan.h" "$(@D)/cuda/include/thrust/system/detail/sequential/scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/scan_by_key.h" "$(@D)/cuda/include/thrust/system/detail/sequential/scan_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/scatter.h" "$(@D)/cuda/include/thrust/system/detail/sequential/scatter.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/sequence.h" "$(@D)/cuda/include/thrust/system/detail/sequential/sequence.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/set_operations.h" "$(@D)/cuda/include/thrust/system/detail/sequential/set_operations.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/sort.h" "$(@D)/cuda/include/thrust/system/detail/sequential/sort.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/sort.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/sort.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/stable_merge_sort.h" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_merge_sort.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/stable_merge_sort.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_merge_sort.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/stable_primitive_sort.h" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_primitive_sort.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/stable_primitive_sort.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_primitive_sort.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/stable_radix_sort.h" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_radix_sort.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/stable_radix_sort.inl" "$(@D)/cuda/include/thrust/system/detail/sequential/stable_radix_sort.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/swap_ranges.h" "$(@D)/cuda/include/thrust/system/detail/sequential/swap_ranges.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/tabulate.h" "$(@D)/cuda/include/thrust/system/detail/sequential/tabulate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/detail/sequential/temporary_buffer.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/transform.h" "$(@D)/cuda/include/thrust/system/detail/sequential/transform.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/transform_reduce.h" "$(@D)/cuda/include/thrust/system/detail/sequential/transform_reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/transform_scan.h" "$(@D)/cuda/include/thrust/system/detail/sequential/transform_scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/trivial_copy.h" "$(@D)/cuda/include/thrust/system/detail/sequential/trivial_copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/detail/sequential/uninitialized_copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/detail/sequential/uninitialized_fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/unique.h" "$(@D)/cuda/include/thrust/system/detail/sequential/unique.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/sequential/unique_by_key.h" "$(@D)/cuda/include/thrust/system/detail/sequential/unique_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/detail/system_error.inl" "$(@D)/cuda/include/thrust/system/detail/system_error.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/error_code.h" "$(@D)/cuda/include/thrust/system/error_code.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/omp/detail/adjacent_difference.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/assign_value.h" "$(@D)/cuda/include/thrust/system/omp/detail/assign_value.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/binary_search.h" "$(@D)/cuda/include/thrust/system/omp/detail/binary_search.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/copy.h" "$(@D)/cuda/include/thrust/system/omp/detail/copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/copy.inl" "$(@D)/cuda/include/thrust/system/omp/detail/copy.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/copy_if.h" "$(@D)/cuda/include/thrust/system/omp/detail/copy_if.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/copy_if.inl" "$(@D)/cuda/include/thrust/system/omp/detail/copy_if.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/count.h" "$(@D)/cuda/include/thrust/system/omp/detail/count.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/default_decomposition.h" "$(@D)/cuda/include/thrust/system/omp/detail/default_decomposition.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/default_decomposition.inl" "$(@D)/cuda/include/thrust/system/omp/detail/default_decomposition.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/equal.h" "$(@D)/cuda/include/thrust/system/omp/detail/equal.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/execution_policy.h" "$(@D)/cuda/include/thrust/system/omp/detail/execution_policy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/extrema.h" "$(@D)/cuda/include/thrust/system/omp/detail/extrema.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/fill.h" "$(@D)/cuda/include/thrust/system/omp/detail/fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/find.h" "$(@D)/cuda/include/thrust/system/omp/detail/find.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/for_each.h" "$(@D)/cuda/include/thrust/system/omp/detail/for_each.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/for_each.inl" "$(@D)/cuda/include/thrust/system/omp/detail/for_each.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/gather.h" "$(@D)/cuda/include/thrust/system/omp/detail/gather.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/generate.h" "$(@D)/cuda/include/thrust/system/omp/detail/generate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/get_value.h" "$(@D)/cuda/include/thrust/system/omp/detail/get_value.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/inner_product.h" "$(@D)/cuda/include/thrust/system/omp/detail/inner_product.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/iter_swap.h" "$(@D)/cuda/include/thrust/system/omp/detail/iter_swap.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/logical.h" "$(@D)/cuda/include/thrust/system/omp/detail/logical.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/omp/detail/malloc_and_free.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/memory.inl" "$(@D)/cuda/include/thrust/system/omp/detail/memory.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/merge.h" "$(@D)/cuda/include/thrust/system/omp/detail/merge.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/mismatch.h" "$(@D)/cuda/include/thrust/system/omp/detail/mismatch.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/par.h" "$(@D)/cuda/include/thrust/system/omp/detail/par.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/partition.h" "$(@D)/cuda/include/thrust/system/omp/detail/partition.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/partition.inl" "$(@D)/cuda/include/thrust/system/omp/detail/partition.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/reduce.h" "$(@D)/cuda/include/thrust/system/omp/detail/reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/reduce.inl" "$(@D)/cuda/include/thrust/system/omp/detail/reduce.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/omp/detail/reduce_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/reduce_by_key.inl" "$(@D)/cuda/include/thrust/system/omp/detail/reduce_by_key.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/reduce_intervals.h" "$(@D)/cuda/include/thrust/system/omp/detail/reduce_intervals.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/reduce_intervals.inl" "$(@D)/cuda/include/thrust/system/omp/detail/reduce_intervals.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/remove.h" "$(@D)/cuda/include/thrust/system/omp/detail/remove.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/remove.inl" "$(@D)/cuda/include/thrust/system/omp/detail/remove.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/replace.h" "$(@D)/cuda/include/thrust/system/omp/detail/replace.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/reverse.h" "$(@D)/cuda/include/thrust/system/omp/detail/reverse.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/scan.h" "$(@D)/cuda/include/thrust/system/omp/detail/scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/scan_by_key.h" "$(@D)/cuda/include/thrust/system/omp/detail/scan_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/scatter.h" "$(@D)/cuda/include/thrust/system/omp/detail/scatter.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/sequence.h" "$(@D)/cuda/include/thrust/system/omp/detail/sequence.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/set_operations.h" "$(@D)/cuda/include/thrust/system/omp/detail/set_operations.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/sort.h" "$(@D)/cuda/include/thrust/system/omp/detail/sort.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/sort.inl" "$(@D)/cuda/include/thrust/system/omp/detail/sort.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/swap_ranges.h" "$(@D)/cuda/include/thrust/system/omp/detail/swap_ranges.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/tabulate.h" "$(@D)/cuda/include/thrust/system/omp/detail/tabulate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/omp/detail/temporary_buffer.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/transform.h" "$(@D)/cuda/include/thrust/system/omp/detail/transform.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/transform_reduce.h" "$(@D)/cuda/include/thrust/system/omp/detail/transform_reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/transform_scan.h" "$(@D)/cuda/include/thrust/system/omp/detail/transform_scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/omp/detail/uninitialized_copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/omp/detail/uninitialized_fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/unique.h" "$(@D)/cuda/include/thrust/system/omp/detail/unique.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/unique.inl" "$(@D)/cuda/include/thrust/system/omp/detail/unique.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/unique_by_key.h" "$(@D)/cuda/include/thrust/system/omp/detail/unique_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/unique_by_key.inl" "$(@D)/cuda/include/thrust/system/omp/detail/unique_by_key.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/detail/vector.inl" "$(@D)/cuda/include/thrust/system/omp/detail/vector.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/execution_policy.h" "$(@D)/cuda/include/thrust/system/omp/execution_policy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/memory.h" "$(@D)/cuda/include/thrust/system/omp/memory.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/omp/vector.h" "$(@D)/cuda/include/thrust/system/omp/vector.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/system_error.h" "$(@D)/cuda/include/thrust/system/system_error.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/adjacent_difference.h" "$(@D)/cuda/include/thrust/system/tbb/detail/adjacent_difference.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/assign_value.h" "$(@D)/cuda/include/thrust/system/tbb/detail/assign_value.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/binary_search.h" "$(@D)/cuda/include/thrust/system/tbb/detail/binary_search.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/copy.h" "$(@D)/cuda/include/thrust/system/tbb/detail/copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/copy.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/copy.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/copy_if.h" "$(@D)/cuda/include/thrust/system/tbb/detail/copy_if.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/copy_if.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/copy_if.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/count.h" "$(@D)/cuda/include/thrust/system/tbb/detail/count.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/equal.h" "$(@D)/cuda/include/thrust/system/tbb/detail/equal.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/execution_policy.h" "$(@D)/cuda/include/thrust/system/tbb/detail/execution_policy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/extrema.h" "$(@D)/cuda/include/thrust/system/tbb/detail/extrema.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/fill.h" "$(@D)/cuda/include/thrust/system/tbb/detail/fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/find.h" "$(@D)/cuda/include/thrust/system/tbb/detail/find.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/for_each.h" "$(@D)/cuda/include/thrust/system/tbb/detail/for_each.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/for_each.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/for_each.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/gather.h" "$(@D)/cuda/include/thrust/system/tbb/detail/gather.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/generate.h" "$(@D)/cuda/include/thrust/system/tbb/detail/generate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/get_value.h" "$(@D)/cuda/include/thrust/system/tbb/detail/get_value.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/inner_product.h" "$(@D)/cuda/include/thrust/system/tbb/detail/inner_product.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/iter_swap.h" "$(@D)/cuda/include/thrust/system/tbb/detail/iter_swap.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/logical.h" "$(@D)/cuda/include/thrust/system/tbb/detail/logical.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/malloc_and_free.h" "$(@D)/cuda/include/thrust/system/tbb/detail/malloc_and_free.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/memory.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/memory.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/merge.h" "$(@D)/cuda/include/thrust/system/tbb/detail/merge.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/merge.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/merge.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/mismatch.h" "$(@D)/cuda/include/thrust/system/tbb/detail/mismatch.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/par.h" "$(@D)/cuda/include/thrust/system/tbb/detail/par.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/partition.h" "$(@D)/cuda/include/thrust/system/tbb/detail/partition.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/partition.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/partition.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/reduce.h" "$(@D)/cuda/include/thrust/system/tbb/detail/reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/reduce.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/reduce.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/reduce_by_key.h" "$(@D)/cuda/include/thrust/system/tbb/detail/reduce_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/reduce_by_key.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/reduce_by_key.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/reduce_intervals.h" "$(@D)/cuda/include/thrust/system/tbb/detail/reduce_intervals.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/remove.h" "$(@D)/cuda/include/thrust/system/tbb/detail/remove.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/remove.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/remove.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/replace.h" "$(@D)/cuda/include/thrust/system/tbb/detail/replace.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/reverse.h" "$(@D)/cuda/include/thrust/system/tbb/detail/reverse.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/scan.h" "$(@D)/cuda/include/thrust/system/tbb/detail/scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/scan.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/scan.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/scan_by_key.h" "$(@D)/cuda/include/thrust/system/tbb/detail/scan_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/scatter.h" "$(@D)/cuda/include/thrust/system/tbb/detail/scatter.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/sequence.h" "$(@D)/cuda/include/thrust/system/tbb/detail/sequence.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/set_operations.h" "$(@D)/cuda/include/thrust/system/tbb/detail/set_operations.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/sort.h" "$(@D)/cuda/include/thrust/system/tbb/detail/sort.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/sort.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/sort.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/swap_ranges.h" "$(@D)/cuda/include/thrust/system/tbb/detail/swap_ranges.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/tabulate.h" "$(@D)/cuda/include/thrust/system/tbb/detail/tabulate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/temporary_buffer.h" "$(@D)/cuda/include/thrust/system/tbb/detail/temporary_buffer.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/transform.h" "$(@D)/cuda/include/thrust/system/tbb/detail/transform.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/transform_reduce.h" "$(@D)/cuda/include/thrust/system/tbb/detail/transform_reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/transform_scan.h" "$(@D)/cuda/include/thrust/system/tbb/detail/transform_scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/uninitialized_copy.h" "$(@D)/cuda/include/thrust/system/tbb/detail/uninitialized_copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/uninitialized_fill.h" "$(@D)/cuda/include/thrust/system/tbb/detail/uninitialized_fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/unique.h" "$(@D)/cuda/include/thrust/system/tbb/detail/unique.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/unique.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/unique.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/unique_by_key.h" "$(@D)/cuda/include/thrust/system/tbb/detail/unique_by_key.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/unique_by_key.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/unique_by_key.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/detail/vector.inl" "$(@D)/cuda/include/thrust/system/tbb/detail/vector.inl" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/execution_policy.h" "$(@D)/cuda/include/thrust/system/tbb/execution_policy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/memory.h" "$(@D)/cuda/include/thrust/system/tbb/memory.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system/tbb/vector.h" "$(@D)/cuda/include/thrust/system/tbb/vector.h" && cp -f "/usr/local/cuda-9.0/include/thrust/system_error.h" "$(@D)/cuda/include/thrust/system_error.h" && cp -f "/usr/local/cuda-9.0/include/thrust/tabulate.h" "$(@D)/cuda/include/thrust/tabulate.h" && cp -f "/usr/local/cuda-9.0/include/thrust/transform.h" "$(@D)/cuda/include/thrust/transform.h" && cp -f "/usr/local/cuda-9.0/include/thrust/transform_reduce.h" "$(@D)/cuda/include/thrust/transform_reduce.h" && cp -f "/usr/local/cuda-9.0/include/thrust/transform_scan.h" "$(@D)/cuda/include/thrust/transform_scan.h" && cp -f "/usr/local/cuda-9.0/include/thrust/tuple.h" "$(@D)/cuda/include/thrust/tuple.h" && cp -f "/usr/local/cuda-9.0/include/thrust/uninitialized_copy.h" "$(@D)/cuda/include/thrust/uninitialized_copy.h" && cp -f "/usr/local/cuda-9.0/include/thrust/uninitialized_fill.h" "$(@D)/cuda/include/thrust/uninitialized_fill.h" && cp -f "/usr/local/cuda-9.0/include/thrust/unique.h" "$(@D)/cuda/include/thrust/unique.h" && cp -f "/usr/local/cuda-9.0/include/thrust/version.h" "$(@D)/cuda/include/thrust/version.h" && cp -f "/usr/local/cuda-9.0/include/vector_functions.h" "$(@D)/cuda/include/vector_functions.h" && cp -f "/usr/local/cuda-9.0/include/vector_functions.hpp" "$(@D)/cuda/include/vector_functions.hpp" && cp -f "/usr/local/cuda-9.0/include/vector_types.h" "$(@D)/cuda/include/vector_types.h"
-   """,
+    cmd = """cp -rf "/usr/local/cuda-9.0/include/." "$(@D)/cuda/include/" """,
 )
 
 genrule(
@@ -1197,9 +1195,7 @@ genrule(
     outs = [
         "cuda/nvvm/libdevice/libdevice.10.bc",
     ],
-    cmd = """
-if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp -f "/usr/local/cuda-9.0/nvvm/libdevice/libdevice.10.bc" "$(@D)//libdevice.10.bc"
-   """,
+    cmd = """cp -rf "/usr/local/cuda-9.0/nvvm/libdevice/." "$(@D)/" """,
 )
 
 genrule(
@@ -1234,9 +1230,7 @@ genrule(
         "cuda/extras/CUPTI/include/generated_nvtx_meta.h",
         "cuda/extras/CUPTI/include/openacc/cupti_openacc.h",
     ],
-    cmd = """
-if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/GL/gl.h" "$(@D)/cuda/extras/CUPTI/include/GL/gl.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/GL/glew.h" "$(@D)/cuda/extras/CUPTI/include/GL/glew.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/GL/glext.h" "$(@D)/cuda/extras/CUPTI/include/GL/glext.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/GL/glu.h" "$(@D)/cuda/extras/CUPTI/include/GL/glu.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/GL/glut.h" "$(@D)/cuda/extras/CUPTI/include/GL/glut.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/GL/glx.h" "$(@D)/cuda/extras/CUPTI/include/GL/glx.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/GL/glxext.h" "$(@D)/cuda/extras/CUPTI/include/GL/glxext.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/GL/wglew.h" "$(@D)/cuda/extras/CUPTI/include/GL/wglew.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/GL/wglext.h" "$(@D)/cuda/extras/CUPTI/include/GL/wglext.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/cuda_stdint.h" "$(@D)/cuda/extras/CUPTI/include/cuda_stdint.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/cupti.h" "$(@D)/cuda/extras/CUPTI/include/cupti.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/cupti_activity.h" "$(@D)/cuda/extras/CUPTI/include/cupti_activity.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/cupti_callbacks.h" "$(@D)/cuda/extras/CUPTI/include/cupti_callbacks.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/cupti_driver_cbid.h" "$(@D)/cuda/extras/CUPTI/include/cupti_driver_cbid.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/cupti_events.h" "$(@D)/cuda/extras/CUPTI/include/cupti_events.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/cupti_metrics.h" "$(@D)/cuda/extras/CUPTI/include/cupti_metrics.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/cupti_nvtx_cbid.h" "$(@D)/cuda/extras/CUPTI/include/cupti_nvtx_cbid.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/cupti_result.h" "$(@D)/cuda/extras/CUPTI/include/cupti_result.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/cupti_runtime_cbid.h" "$(@D)/cuda/extras/CUPTI/include/cupti_runtime_cbid.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/cupti_version.h" "$(@D)/cuda/extras/CUPTI/include/cupti_version.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/generated_cudaGL_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cudaGL_meta.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/generated_cudaVDPAU_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cudaVDPAU_meta.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/generated_cuda_gl_interop_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cuda_gl_interop_meta.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/generated_cuda_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cuda_meta.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/generated_cuda_runtime_api_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cuda_runtime_api_meta.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/generated_cuda_vdpau_interop_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_cuda_vdpau_interop_meta.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/generated_nvtx_meta.h" "$(@D)/cuda/extras/CUPTI/include/generated_nvtx_meta.h" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/include/openacc/cupti_openacc.h" "$(@D)/cuda/extras/CUPTI/include/openacc/cupti_openacc.h"
-   """,
+    cmd = """cp -rf "/usr/local/cuda-9.0/extras/CUPTI/include/." "$(@D)/cuda/extras/CUPTI/include/" """,
 )
 
 genrule(
@@ -1252,9 +1246,32 @@ genrule(
         "cuda/lib/libcudnn.so.7",
         "cuda/lib/libcupti.so.9.0",
     ],
-    cmd = """
-if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp -f "/usr/local/cuda-9.0/targets/x86_64-linux/lib/stubs/libcuda.so" "$(@D)/cuda/lib/libcuda.so" && cp -f "/usr/local/cuda-9.0/targets/x86_64-linux/lib/libcudart.so.9.0.176" "$(@D)/cuda/lib/libcudart.so.9.0" && cp -f "/usr/local/cuda-9.0/targets/x86_64-linux/lib/libcudart_static.a" "$(@D)/cuda/lib/libcudart_static.a" && cp -f "/usr/local/cuda-9.0/targets/x86_64-linux/lib/libcublas.so.9.0.480" "$(@D)/cuda/lib/libcublas.so.9.0" && cp -f "/usr/local/cuda-9.0/targets/x86_64-linux/lib/libcusolver.so.9.0.176" "$(@D)/cuda/lib/libcusolver.so.9.0" && cp -f "/usr/local/cuda-9.0/targets/x86_64-linux/lib/libcurand.so.9.0.176" "$(@D)/cuda/lib/libcurand.so.9.0" && cp -f "/usr/local/cuda-9.0/targets/x86_64-linux/lib/libcufft.so.9.0.176" "$(@D)/cuda/lib/libcufft.so.9.0" && cp -f "/usr/lib/x86_64-linux-gnu/libcudnn.so.7.1.4" "$(@D)/cuda/lib/libcudnn.so.7" && cp -f "/usr/local/cuda-9.0/extras/CUPTI/lib64/libcupti.so.9.0.176" "$(@D)/cuda/lib/libcupti.so.9.0"
-   """,
+    cmd = """cp -f "/usr/local/cuda-9.0/lib64/stubs/libcuda.so" $(location cuda/lib/libcuda.so) && cp -f "/usr/local/cuda-9.0/lib64/libcudart.so.9.0" $(location cuda/lib/libcudart.so.9.0) && cp -f "/usr/local/cuda-9.0/lib64/libcudart_static.a" $(location cuda/lib/libcudart_static.a) && cp -f "/usr/local/cuda-9.0/lib64/libcublas.so.9.0" $(location cuda/lib/libcublas.so.9.0) && cp -f "/usr/local/cuda-9.0/lib64/libcusolver.so.9.0" $(location cuda/lib/libcusolver.so.9.0) && cp -f "/usr/local/cuda-9.0/lib64/libcurand.so.9.0" $(location cuda/lib/libcurand.so.9.0) && cp -f "/usr/local/cuda-9.0/lib64/libcufft.so.9.0" $(location cuda/lib/libcufft.so.9.0) && cp -f "/usr/lib/x86_64-linux-gnu/libcudnn.so.7" $(location cuda/lib/libcudnn.so.7) && cp -f "/usr/local/cuda-9.0/extras/CUPTI/lib64/libcupti.so.9.0" $(location cuda/lib/libcupti.so.9.0) """,
+)
+
+genrule(
+    name = "cuda-bin",
+    outs = [
+        "cuda/bin/bin2c",
+        "cuda/bin/crt/link.stub",
+        "cuda/bin/crt/prelink.stub",
+        "cuda/bin/cuda-gdb",
+        "cuda/bin/cuda-gdbserver",
+        "cuda/bin/cuda-memcheck",
+        "cuda/bin/cudafe",
+        "cuda/bin/cudafe++",
+        "cuda/bin/cuobjdump",
+        "cuda/bin/fatbinary",
+        "cuda/bin/gpu-library-advisor",
+        "cuda/bin/nvcc",
+        "cuda/bin/nvcc.profile",
+        "cuda/bin/nvdisasm",
+        "cuda/bin/nvlink",
+        "cuda/bin/nvprof",
+        "cuda/bin/nvprune",
+        "cuda/bin/ptxas",
+    ],
+    cmd = """cp -rf "/usr/local/cuda-9.0/bin/." "$(@D)/cuda/bin/" """,
 )
 
 genrule(
@@ -1262,7 +1279,5 @@ genrule(
     outs = [
         "cuda/include/cudnn.h",
     ],
-    cmd = """
-if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp -f "/usr/include/cudnn.h" "$(@D)/cudnn.h"
-   """,
+    cmd = """cp -f "/usr/include/cudnn.h" $(location cuda/include/cudnn.h) """,
 )
diff --git a/third_party/toolchains/preconfig/ubuntu14.04/tensorrt5/BUILD b/third_party/toolchains/preconfig/ubuntu14.04/tensorrt5/BUILD
index 04442d9c51..75e2b7fd33 100755
--- a/third_party/toolchains/preconfig/ubuntu14.04/tensorrt5/BUILD
+++ b/third_party/toolchains/preconfig/ubuntu14.04/tensorrt5/BUILD
@@ -35,9 +35,7 @@ genrule(
     outs = [
         "tensorrt/lib/libnvinfer.so.5",
     ],
-    cmd = """
-if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp -f "/usr/lib/x86_64-linux-gnu/libnvinfer.so.5.0.2" "$(@D)/libnvinfer.so.5"
-   """,
+    cmd = """cp -f "/usr/lib/x86_64-linux-gnu/libnvinfer.so.5" $(location tensorrt/lib/libnvinfer.so.5) """,
 )
 
 genrule(
@@ -46,7 +44,5 @@ genrule(
         "tensorrt/include/NvInfer.h",
         "tensorrt/include/NvUtils.h",
     ],
-    cmd = """
-if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp -f "/usr/include/x86_64-linux-gnu/NvInfer.h" "$(@D)/tensorrt/include/NvInfer.h" && cp -f "/usr/include/x86_64-linux-gnu/NvUtils.h" "$(@D)/tensorrt/include/NvUtils.h"
-   """,
+    cmd = """cp -f "/usr/include/x86_64-linux-gnu/NvInfer.h" $(location tensorrt/include/NvInfer.h) && cp -f "/usr/include/x86_64-linux-gnu/NvUtils.h" $(location tensorrt/include/NvUtils.h) """,
 )
diff --git a/third_party/toolchains/preconfig/ubuntu14.04/tensorrt5/build_defs.bzl b/third_party/toolchains/preconfig/ubuntu14.04/tensorrt5/build_defs.bzl
index 5c1c40361d..527be93834 100755
--- a/third_party/toolchains/preconfig/ubuntu14.04/tensorrt5/build_defs.bzl
+++ b/third_party/toolchains/preconfig/ubuntu14.04/tensorrt5/build_defs.bzl
@@ -2,6 +2,4 @@
 
 def if_tensorrt(if_true, if_false = []):
     """Tests whether TensorRT was enabled during the configure process."""
-    if True:
-        return if_true
-    return if_false
+    return if_true
-- 
GitLab


From e2968144a0a1d0344166a7996afc3cedd4381a9d Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Fri, 18 Jan 2019 18:08:42 -0800
Subject: [PATCH 1026/1095] [Grappler] AddSubgraph in MutableGraphView

PiperOrigin-RevId: 230026030
---
 tensorflow/core/grappler/graph_view.h         | 15 ++++--
 .../core/grappler/mutable_graph_view.cc       | 25 +++++++++
 tensorflow/core/grappler/mutable_graph_view.h | 10 ++++
 .../core/grappler/mutable_graph_view_test.cc  | 52 +++++++++++++++++++
 4 files changed, 98 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/grappler/graph_view.h b/tensorflow/core/grappler/graph_view.h
index 88b1e975c1..63c58a0aed 100644
--- a/tensorflow/core/grappler/graph_view.h
+++ b/tensorflow/core/grappler/graph_view.h
@@ -318,11 +318,18 @@ class GraphViewInternal {
  protected:
   explicit GraphViewInternal(GraphDefT* graph) : graph_(graph) {}
 
+  Status AddUniqueNode(NodeDefT* node) {
+    auto inserted = nodes_.emplace(node->name(), node);
+    return inserted.second
+               ? Status::OK()
+               : errors::InvalidArgument("Non unique node name detected: ",
+                                         node->name());
+  }
+
+  // TODO(ezhulenev): Remove this function.
   void AddUniqueNodeOrDie(NodeDefT* node) {
-    auto result = nodes_.emplace(node->name(), node);
-    // TODO(ezhulenev): Replace CHECK with factory method returning
-    // absl::StatusOr (when available).
-    CHECK(result.second) << "Non unique node name detected: " << node->name();
+    Status st = AddUniqueNode(node);
+    CHECK(st.ok()) << st.error_message();
   }
 
   // TODO(lyandy): Checks for self loops, Switch control dependencies, fanins
diff --git a/tensorflow/core/grappler/mutable_graph_view.cc b/tensorflow/core/grappler/mutable_graph_view.cc
index 43dec07c1d..0f171f7656 100644
--- a/tensorflow/core/grappler/mutable_graph_view.cc
+++ b/tensorflow/core/grappler/mutable_graph_view.cc
@@ -215,6 +215,31 @@ NodeDef* MutableGraphView::AddNode(NodeDef&& node) {
   return node_in_graph;
 }
 
+Status MutableGraphView::AddSubgraph(GraphDef&& subgraph) {
+  if (subgraph.library().function_size() != 0) {
+    return errors::InvalidArgument(
+        "Can't add a subgraph with non-empty function library");
+  }
+
+  int node_size_before = graph()->node_size();
+
+  for (NodeDef& node : *subgraph.mutable_node()) {
+    auto* node_in_graph = graph()->add_node();
+    *node_in_graph = std::move(node);
+    TF_RETURN_IF_ERROR(AddUniqueNode(node_in_graph));
+  }
+
+  // TODO(ezhulenev, lyandy): Right now AddAndDedupFanouts do not check that
+  // fanins actually exists in the graph, and there is already TODO for that.
+
+  for (int i = node_size_before; i < graph()->node_size(); ++i) {
+    NodeDef* node = graph()->mutable_node(i);
+    AddAndDedupFanouts(node);
+  }
+
+  return Status::OK();
+}
+
 Status MutableGraphView::UpdateFanouts(absl::string_view from_node,
                                        absl::string_view to_node) {
   NodeDef* from_node_ptr = GetNode(from_node);
diff --git a/tensorflow/core/grappler/mutable_graph_view.h b/tensorflow/core/grappler/mutable_graph_view.h
index c62129bcad..f08b5dde08 100644
--- a/tensorflow/core/grappler/mutable_graph_view.h
+++ b/tensorflow/core/grappler/mutable_graph_view.h
@@ -63,6 +63,16 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   // node in graph.
   NodeDef* AddNode(NodeDef&& node);
 
+  // Adds all nodes from the `subgraph` to the underlying graph and updates the
+  // view. `subgraph` doesn't have to be a valid graph definition on it's own,
+  // it can have edges to the nodes that are not in it, however after adding
+  // it to the underlying graph, final graph must be valid.
+  //
+  // TODO(ezhulenev): Currently it will fail if subgraph has non-empty function
+  // library. Add support for adding new functions from the subgraph function
+  // library into the underlying graph.
+  Status AddSubgraph(GraphDef&& subgraph);
+
   // Updates all fanouts (input ports fetching output tensors) from `from_node`
   // to the `to_node`, including control dependencies.
   //
diff --git a/tensorflow/core/grappler/mutable_graph_view_test.cc b/tensorflow/core/grappler/mutable_graph_view_test.cc
index acfaba5ddd..ea88359e2e 100644
--- a/tensorflow/core/grappler/mutable_graph_view_test.cc
+++ b/tensorflow/core/grappler/mutable_graph_view_test.cc
@@ -141,6 +141,58 @@ void CheckGraph(const MutableGraphView& mutable_graph) {
   }
 }
 
+TEST(MutableGraphViewTest, AddSubgraph) {
+  GraphDef graph_def = test::function::GDef(
+      {
+          NDef("foo", "NotImportant", {}, {}),
+          NDef("bar", "NotImportant", {}, {}),
+          NDef("baz", "NotImportant", {"foo", "bar"}),
+      },
+      /*funcs=*/{});
+  MutableGraphView graph(&graph_def);
+
+  // `s/bar` node has inputs that are valid only if we add subgraph into the
+  // original graph.
+  GraphDef subgraph = test::function::GDef(
+      {
+          NDef("s/n0", "NotImportant", {}, {}),
+          NDef("s/n1", "NotImportant", {"bar", "s/n0"}, {}),
+      },
+      /*funcs=*/{});
+
+  TF_EXPECT_OK(graph.AddSubgraph(std::move(subgraph)));
+
+  // Fanins and fanouts must be updated for the nodes of the original graph, and
+  // added subgraph.
+  CheckNode(graph, "bar", "NotImportant", "", {}, {}, {"baz:1", "s/n1"});
+  CheckNode(graph, "s/n1", "NotImportant", "", {}, {"bar", "s/n0"}, {});
+  CheckGraph(graph);
+}
+
+// TODO(ezhulenev): Add support for adding a subgraph and merging function
+// libraries.
+TEST(MutableGraphViewTest, AddSubgraphWithFunctionLibrary) {
+  GraphDef graph_def = test::function::GDef(
+      {
+          NDef("foo", "NotImportant", {}, {}),
+          NDef("bar", "NotImportant", {}, {}),
+          NDef("baz", "NotImportant", {"foo", "bar"}),
+      },
+      /*funcs=*/{});
+  MutableGraphView graph(&graph_def);
+
+  FunctionDef x_times_two = test::function::XTimesTwo();
+  GraphDef subgraph = test::function::GDef(
+      {
+          NDef("s/n0", "NotImportant", {}, {}),
+          NDef("s/n1", "NotImportant", {"bar", "s/n0"}, {}),
+      },
+      /*funcs=*/{x_times_two});
+
+  Status status = graph.AddSubgraph(std::move(subgraph));
+  EXPECT_FALSE(status.ok());
+}
+
 TEST(MutableGraphViewTest, AddAndUpdateFanouts) {
   // Actual node.op() is not important in this test.
   GraphDef graph_def = test::function::GDef(
-- 
GitLab


From fdd4979d7be91c8b17bdc0afa56cfe012d8a7913 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Fri, 18 Jan 2019 19:15:05 -0800
Subject: [PATCH 1027/1095] Add direct dependencies to tf op libraries.

PiperOrigin-RevId: 230030481
---
 tensorflow/contrib/batching/BUILD                   |  1 +
 .../contrib/hvx/hvx_ops_support_checker/BUILD       | 13 +++++++++++++
 tensorflow/contrib/util/BUILD                       |  3 +++
 3 files changed, 17 insertions(+)

diff --git a/tensorflow/contrib/batching/BUILD b/tensorflow/contrib/batching/BUILD
index 648f3ebb05..5174afe0a6 100644
--- a/tensorflow/contrib/batching/BUILD
+++ b/tensorflow/contrib/batching/BUILD
@@ -37,6 +37,7 @@ py_library(
 cc_library(
     name = "batch_ops_kernels",
     deps = [
+        "//tensorflow/core:batch_ops_op_lib",
         "//tensorflow/core/kernels:batch_kernels",
     ],
     alwayslink = 1,
diff --git a/tensorflow/contrib/hvx/hvx_ops_support_checker/BUILD b/tensorflow/contrib/hvx/hvx_ops_support_checker/BUILD
index 0081fb6177..d319aa7986 100644
--- a/tensorflow/contrib/hvx/hvx_ops_support_checker/BUILD
+++ b/tensorflow/contrib/hvx/hvx_ops_support_checker/BUILD
@@ -16,9 +16,22 @@ tf_cc_binary(
     srcs = ["hvx_ops_support_checker_main.cc"],
     visibility = ["//visibility:public"],
     deps = [
+        "//tensorflow/core:array_ops_op_lib",
+        "//tensorflow/core:candidate_sampling_ops_op_lib",
+        "//tensorflow/core:control_flow_ops_op_lib",
         "//tensorflow/core:framework_internal",
+        "//tensorflow/core:functional_ops_op_lib",
         "//tensorflow/core:lib",
+        "//tensorflow/core:list_ops_op_lib",
+        "//tensorflow/core:manip_ops_op_lib",
+        "//tensorflow/core:math_ops_op_lib",
+        "//tensorflow/core:nn_ops_op_lib",
         "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:random_ops_op_lib",
+        "//tensorflow/core:remote_fused_graph_ops_op_lib",
+        "//tensorflow/core:string_ops_op_lib",
+        "//tensorflow/core:training_ops_op_lib",
+        "//tensorflow/core:user_ops_op_lib",
         "//tensorflow/core/kernels:remote_fused_graph_execute_utils",
         "//tensorflow/core/kernels/hexagon:graph_transferer",
         "//tensorflow/tools/graph_transforms:file_utils",
diff --git a/tensorflow/contrib/util/BUILD b/tensorflow/contrib/util/BUILD
index d9ccda8e89..07dbd5ca8d 100644
--- a/tensorflow/contrib/util/BUILD
+++ b/tensorflow/contrib/util/BUILD
@@ -16,9 +16,12 @@ cc_library(
     srcs = ["convert_graphdef_memmapped_format_lib.cc"],
     hdrs = ["convert_graphdef_memmapped_format_lib.h"],
     deps = [
+        "//tensorflow/core:array_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:framework_internal",
+        "//tensorflow/core:functional_ops_op_lib",
         "//tensorflow/core:lib",
+        "//tensorflow/core:nn_ops_op_lib",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:tensorflow",
         "//tensorflow/core/kernels:immutable_constant_op",
-- 
GitLab


From 467c1c2e215c5481372ba4e685869ae504b9e74f Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Fri, 18 Jan 2019 20:17:09 -0800
Subject: [PATCH 1028/1095] Add direct dependencies to tf op libraries.

PiperOrigin-RevId: 230034058
---
 tensorflow/c/BUILD                              |  7 +++++++
 tensorflow/compiler/jit/BUILD                   |  6 ++++++
 tensorflow/compiler/tf2xla/kernels/BUILD        |  9 +++++++++
 tensorflow/core/common_runtime/eager/BUILD      |  4 ++++
 tensorflow/core/debug/BUILD                     |  4 ++++
 tensorflow/core/distributed_runtime/BUILD       |  6 ++++++
 tensorflow/core/distributed_runtime/rpc/BUILD   |  5 +++++
 tensorflow/core/kernels/data/BUILD              |  7 +++++++
 tensorflow/core/kernels/data/experimental/BUILD |  5 +++++
 tensorflow/core/kernels/hexagon/BUILD           |  7 +++++++
 tensorflow/tools/graph_transforms/BUILD         | 14 ++++++++++++++
 11 files changed, 74 insertions(+)

diff --git a/tensorflow/c/BUILD b/tensorflow/c/BUILD
index a48778d612..6e50a09bfc 100644
--- a/tensorflow/c/BUILD
+++ b/tensorflow/c/BUILD
@@ -289,13 +289,20 @@ tf_cuda_cc_test(
         "//tensorflow/cc/saved_model:signature_constants",
         "//tensorflow/cc/saved_model:tag_constants",
         "//tensorflow/compiler/jit",
+        "//tensorflow/core:array_ops_op_lib",
+        "//tensorflow/core:bitwise_ops_op_lib",
+        "//tensorflow/core:control_flow_ops_op_lib",
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:direct_session",
         "//tensorflow/core:framework",
         "//tensorflow/core:framework_internal",
+        "//tensorflow/core:functional_ops_op_lib",
         "//tensorflow/core:lib",
+        "//tensorflow/core:math_ops_op_lib",
+        "//tensorflow/core:nn_ops_op_lib",
         "//tensorflow/core:proto_text",
         "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:spectral_ops_op_lib",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core/kernels:array",
diff --git a/tensorflow/compiler/jit/BUILD b/tensorflow/compiler/jit/BUILD
index 1d72d8c2d8..55e2e6d11f 100644
--- a/tensorflow/compiler/jit/BUILD
+++ b/tensorflow/compiler/jit/BUILD
@@ -175,12 +175,18 @@ cc_library(
         "//tensorflow/compiler/xla/client:global_data",
         "//tensorflow/compiler/xla/client:local_client",
         "//tensorflow/compiler/xla/service:stream_pool",
+        "//tensorflow/core:array_ops_op_lib",
+        "//tensorflow/core:control_flow_ops_op_lib",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:framework",
+        "//tensorflow/core:functional_ops_op_lib",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:math_ops_op_lib",
+        "//tensorflow/core:nn_ops_op_lib",
         "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:resource_variable_ops_op_lib",
         "//tensorflow/core:stream_executor_no_cuda",
         "//tensorflow/core/kernels:cast_op",
         "//tensorflow/core/kernels:constant_op",
diff --git a/tensorflow/compiler/tf2xla/kernels/BUILD b/tensorflow/compiler/tf2xla/kernels/BUILD
index 47209d285f..52d2901e73 100644
--- a/tensorflow/compiler/tf2xla/kernels/BUILD
+++ b/tensorflow/compiler/tf2xla/kernels/BUILD
@@ -144,13 +144,22 @@ tf_kernel_library(
         "//tensorflow/compiler/xla/client/lib:quantize",
         "//tensorflow/compiler/xla/client/lib:sorting",
         "//tensorflow/compiler/xla/client/lib:triangular_solve",
+        "//tensorflow/core:bitwise_ops_op_lib",
+        "//tensorflow/core:control_flow_ops_op_lib",
         "//tensorflow/core:framework",
+        "//tensorflow/core:functional_ops_op_lib",
         "//tensorflow/core:image_ops_op_lib",
         "//tensorflow/core:lib",
         "//tensorflow/core:linalg_ops_op_lib",
+        "//tensorflow/core:list_ops_op_lib",
+        "//tensorflow/core:math_ops_op_lib",
+        "//tensorflow/core:nn_ops_op_lib",
         "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:random_ops_op_lib",
+        "//tensorflow/core:resource_variable_ops_op_lib",
         "//tensorflow/core:spectral_ops_op_lib",
         "//tensorflow/core:stateless_random_ops_op_lib",
+        "//tensorflow/core:training_ops_op_lib",
         "//tensorflow/core/kernels:bounds_check",
         "//tensorflow/core/kernels:concat_lib",
         "//tensorflow/core/kernels:constant_op",
diff --git a/tensorflow/core/common_runtime/eager/BUILD b/tensorflow/core/common_runtime/eager/BUILD
index 0a16f80c8f..370ec529be 100644
--- a/tensorflow/core/common_runtime/eager/BUILD
+++ b/tensorflow/core/common_runtime/eager/BUILD
@@ -181,11 +181,15 @@ tf_cc_test(
         "//tensorflow/cc:client_session",
         "//tensorflow/cc:ops",
         "//tensorflow/cc:scope",
+        "//tensorflow/core:array_ops_op_lib",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:direct_session",
         "//tensorflow/core:framework",
+        "//tensorflow/core:functional_ops_op_lib",
         "//tensorflow/core:lib",
+        "//tensorflow/core:math_ops_op_lib",
+        "//tensorflow/core:nn_ops_op_lib",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
diff --git a/tensorflow/core/debug/BUILD b/tensorflow/core/debug/BUILD
index 591c22b8f6..e1961b822b 100644
--- a/tensorflow/core/debug/BUILD
+++ b/tensorflow/core/debug/BUILD
@@ -221,10 +221,14 @@ tf_cc_test(
     deps = [
         ":debug_grpc_testlib",
         ":debug_io_utils",
+        "//tensorflow/core:array_ops_op_lib",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:framework",
+        "//tensorflow/core:functional_ops_op_lib",
         "//tensorflow/core:lib",
         "//tensorflow/core:master_proto_cc",
+        "//tensorflow/core:math_ops_op_lib",
+        "//tensorflow/core:nn_ops_op_lib",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
diff --git a/tensorflow/core/distributed_runtime/BUILD b/tensorflow/core/distributed_runtime/BUILD
index 197a3f24d4..6f09bef5ea 100644
--- a/tensorflow/core/distributed_runtime/BUILD
+++ b/tensorflow/core/distributed_runtime/BUILD
@@ -648,6 +648,7 @@ tf_cuda_cc_test(
         ":remote_device",
         ":worker_interface",
         "//tensorflow:grpc++",
+        "//tensorflow/core:control_flow_ops_op_lib",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:framework",
@@ -713,9 +714,14 @@ tf_cuda_cc_test(
     tags = tf_cuda_tests_tags(),
     deps = [
         "//tensorflow/cc:cc_ops",
+        "//tensorflow/core:array_ops_op_lib",
+        "//tensorflow/core:bitwise_ops_op_lib",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:framework",
+        "//tensorflow/core:functional_ops_op_lib",
         "//tensorflow/core:lib",
+        "//tensorflow/core:math_ops_op_lib",
+        "//tensorflow/core:nn_ops_op_lib",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:tensorflow",
         "//tensorflow/core:test",
diff --git a/tensorflow/core/distributed_runtime/rpc/BUILD b/tensorflow/core/distributed_runtime/rpc/BUILD
index 273709a01f..a081ec7b67 100644
--- a/tensorflow/core/distributed_runtime/rpc/BUILD
+++ b/tensorflow/core/distributed_runtime/rpc/BUILD
@@ -314,7 +314,9 @@ tf_cc_binary(
         "//tensorflow:grpc++",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:framework_internal",
+        "//tensorflow/core:functional_ops_op_lib",
         "//tensorflow/core:lib",
+        "//tensorflow/core:math_ops_op_lib",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core/distributed_runtime:server_lib",
         "//tensorflow/core/kernels:data_flow",
@@ -330,9 +332,12 @@ tf_cc_binary(
     deps = [
         ":grpc_server_lib",
         "//tensorflow:grpc++",
+        "//tensorflow/core:array_ops_op_lib",
+        "//tensorflow/core:bitwise_ops_op_lib",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:framework_internal",
         "//tensorflow/core:lib",
+        "//tensorflow/core:nn_ops_op_lib",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:testlib",
         "//tensorflow/core/distributed_runtime:server_lib",
diff --git a/tensorflow/core/kernels/data/BUILD b/tensorflow/core/kernels/data/BUILD
index dcbdb74691..3cadb55a21 100644
--- a/tensorflow/core/kernels/data/BUILD
+++ b/tensorflow/core/kernels/data/BUILD
@@ -76,13 +76,18 @@ tf_cc_test(
     srcs = ["single_threaded_executor_test.cc"],
     deps = [
         ":single_threaded_executor",
+        "//tensorflow/core:bitwise_ops_op_lib",
+        "//tensorflow/core:control_flow_ops_op_lib",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:framework",
         "//tensorflow/core:framework_internal",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:math_ops_op_lib",
         "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:random_ops_op_lib",
+        "//tensorflow/core:spectral_ops_op_lib",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
@@ -566,6 +571,8 @@ tf_kernel_library(
         ":tensor_slice_dataset_op",
         ":window_dataset_op",
         ":zip_dataset_op",
+        "//tensorflow/core:array_ops_op_lib",
+        "//tensorflow/core:nn_ops_op_lib",
         "//tensorflow/core/kernels/data/experimental:dataset_kernels",
     ],
 )
diff --git a/tensorflow/core/kernels/data/experimental/BUILD b/tensorflow/core/kernels/data/experimental/BUILD
index 2c3ffad375..4f7c8f156c 100644
--- a/tensorflow/core/kernels/data/experimental/BUILD
+++ b/tensorflow/core/kernels/data/experimental/BUILD
@@ -119,12 +119,14 @@ tf_kernel_library(
     name = "map_and_batch_dataset_op",
     srcs = ["map_and_batch_dataset_op.cc"],
     deps = [
+        "//tensorflow/core:array_ops_op_lib",
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:dataset_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:metrics",
+        "//tensorflow/core:nn_ops_op_lib",
         "//tensorflow/core/kernels:inplace_ops",
         "//tensorflow/core/kernels/data:captured_function",
         "//tensorflow/core/kernels/data:dataset_utils",
@@ -169,11 +171,13 @@ tf_kernel_library(
     name = "numa_map_and_batch_dataset_op",
     srcs = ["numa_map_and_batch_dataset_op.cc"],
     deps = [
+        "//tensorflow/core:array_ops_op_lib",
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:experimental_dataset_ops_op_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:nn_ops_op_lib",
         "//tensorflow/core/kernels:inplace_ops",
         "//tensorflow/core/kernels/data:captured_function",
         "@com_google_absl//absl/memory",
@@ -200,6 +204,7 @@ tf_kernel_library(
     deps = [
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:framework",
+        "//tensorflow/core:functional_ops_op_lib",
         "//tensorflow/core/kernels/data:parallel_map_iterator",
     ],
 )
diff --git a/tensorflow/core/kernels/hexagon/BUILD b/tensorflow/core/kernels/hexagon/BUILD
index 87d36f22d7..a85de34ac2 100644
--- a/tensorflow/core/kernels/hexagon/BUILD
+++ b/tensorflow/core/kernels/hexagon/BUILD
@@ -24,11 +24,18 @@ tf_cc_test(
     deps = [
         ":graph_transferer",
         "//tensorflow/cc:cc_ops",
+        "//tensorflow/core:array_ops_op_lib",
+        "//tensorflow/core:bitwise_ops_op_lib",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:direct_session",
+        "//tensorflow/core:functional_ops_op_lib",
         "//tensorflow/core:lib",
+        "//tensorflow/core:math_ops_op_lib",
+        "//tensorflow/core:mkl_nn_ops_op_lib",
+        "//tensorflow/core:nn_ops_op_lib",
         "//tensorflow/core:ops",
         "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:remote_fused_graph_ops_op_lib",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
diff --git a/tensorflow/tools/graph_transforms/BUILD b/tensorflow/tools/graph_transforms/BUILD
index f229099e49..41ed31a5c1 100644
--- a/tensorflow/tools/graph_transforms/BUILD
+++ b/tensorflow/tools/graph_transforms/BUILD
@@ -136,6 +136,19 @@ cc_library(
     ] + if_not_windows([
         "//tensorflow/core/kernels:remote_fused_graph_rewriter_transform",
         "//tensorflow/core/kernels/hexagon:hexagon_rewriter_transform",
+        "//tensorflow/core:user_ops_op_lib",
+        "//tensorflow/core:training_ops_op_lib",
+        "//tensorflow/core:string_ops_op_lib",
+        "//tensorflow/core:remote_fused_graph_ops_op_lib",
+        "//tensorflow/core:random_ops_op_lib",
+        "//tensorflow/core:nn_ops_op_lib",
+        "//tensorflow/core:math_ops_op_lib",
+        "//tensorflow/core:manip_ops_op_lib",
+        "//tensorflow/core:list_ops_op_lib",
+        "//tensorflow/core:functional_ops_op_lib",
+        "//tensorflow/core:control_flow_ops_op_lib",
+        "//tensorflow/core:candidate_sampling_ops_op_lib",
+        "//tensorflow/core:array_ops_op_lib",
     ]) + if_not_v2([
         "//tensorflow/contrib/rnn:gru_ops_op_lib",
         "//tensorflow/contrib/rnn:lstm_ops_op_lib",
@@ -175,6 +188,7 @@ tf_cc_test(
         ":transforms_lib",
         "//tensorflow/cc:cc_ops",
         "//tensorflow/cc:sendrecv_ops",
+        "//tensorflow/core:bitwise_ops_op_lib",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:lib",
         "//tensorflow/core:protos_all_cc",
-- 
GitLab


From f52351444551016d7dd949a5aa599da489a97045 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sat, 19 Jan 2019 01:03:16 -0800
Subject: [PATCH 1029/1095] compat: Update forward compatibility horizon to
 2019-01-19

PiperOrigin-RevId: 230049974
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index ccc0735b6e..14b9a562cd 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 18)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 19)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From d28aabb83228524df1b906c66b35bdb5657a59b5 Mon Sep 17 00:00:00 2001
From: David Majnemer <majnemer@google.com>
Date: Sat, 19 Jan 2019 14:18:03 -0800
Subject: [PATCH 1030/1095] [XLA] Optimize division/remainder by a power of 2

X / Y => X >> log_2(y)
X % Y => X & (Y-1)

PiperOrigin-RevId: 230081985
---
 .../xla/service/algebraic_simplifier.cc       | 262 ++++++++++++++++++
 .../xla/service/algebraic_simplifier_test.cc  |  80 ++++++
 .../compiler/xla/service/gather_expander.cc   |  17 +-
 .../xla/service/hlo_creation_utils.cc         |  12 +-
 .../compiler/xla/service/hlo_creation_utils.h |  12 +-
 .../xla/service/hlo_creation_utils_test.cc    |  10 +-
 .../compiler/xla/service/scatter_expander.cc  |  10 +-
 7 files changed, 369 insertions(+), 34 deletions(-)

diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
index 8a02c48146..da15ff7d7a 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
@@ -36,6 +36,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/layout_util.h"
 #include "tensorflow/compiler/xla/literal.h"
 #include "tensorflow/compiler/xla/literal_util.h"
+#include "tensorflow/compiler/xla/primitive_util.h"
 #include "tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h"
 #include "tensorflow/compiler/xla/service/hlo_casting_utils.h"
 #include "tensorflow/compiler/xla/service/hlo_computation.h"
@@ -52,6 +53,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/util.h"
 #include "tensorflow/compiler/xla/window_util.h"
 #include "tensorflow/compiler/xla/xla_data.pb.h"
+#include "tensorflow/core/lib/core/bits.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/logging.h"
@@ -219,6 +221,8 @@ class AlgebraicSimplifierVisitor : public DfsHloVisitorWithDefault {
 
   Status HandlePower(HloInstruction* power) override;
 
+  Status HandleRemainder(HloInstruction* remainder) override;
+
   Status HandleReshape(HloInstruction* reshape) override;
 
   Status HandleReduce(HloInstruction* reduce) override;
@@ -812,6 +816,79 @@ Status InvertConstant(const HloInstruction& constant, Literal* result) {
     return T{1.0} / constant.literal().Get<T>(indices);
   });
 }
+
+template <typename T>
+std::unique_ptr<HloInstruction> TryDivideToShift(HloInstruction* divide,
+                                                 HloComputation* computation) {
+  HloInstruction *a, *b, *c;
+  CHECK(Match(divide, m::Divide(m::Op(&a), m::Op(&b))));
+
+  if (ShapeUtil::ElementIsIntegral(divide->shape()) &&
+      !Match(b, m::ConstantEffectiveScalar(&c)) &&
+      !Match(b, m::Broadcast(m::ConstantEffectiveScalar(&c)))) {
+    return nullptr;
+  }
+
+  if (ShapeUtil::ElementIsSigned(divide->shape())) {
+    int64 b_value = c->literal().GetFirstElement<T>();
+    if (b_value > 0 && IsPowerOfTwo(static_cast<uint64>(b_value))) {
+      // Handle negative dividends by negating the result of the division.
+      HloInstruction* zero_like_a = BroadcastZeros(
+          computation, a->shape().element_type(), a->shape().dimensions());
+
+      auto* dividend_is_negative =
+          computation->AddInstruction(HloInstruction::CreateBinary(
+              ShapeUtil::ChangeElementType(a->shape(), PRED), HloOpcode::kLt, a,
+              zero_like_a));
+
+      auto* negated_dividend = computation->AddInstruction(
+          HloInstruction::CreateUnary(a->shape(), HloOpcode::kNegate, a));
+
+      auto* abs_dividend =
+          computation->AddInstruction(HloInstruction::CreateTernary(
+              a->shape(), HloOpcode::kSelect, dividend_is_negative,
+              negated_dividend, a));
+
+      int log2_abs_b_value = tensorflow::Log2Floor64(b_value);
+
+      auto* shift_amount =
+          computation->AddInstruction(HloInstruction::CreateConstant(
+              LiteralUtil::CreateR0<T>(log2_abs_b_value)));
+      if (!ShapeUtil::IsScalar(b->shape())) {
+        shift_amount = computation->AddInstruction(
+            HloInstruction::CreateBroadcast(b->shape(), shift_amount, {}));
+      }
+
+      auto* quotient = computation->AddInstruction(HloInstruction::CreateBinary(
+          divide->shape(), HloOpcode::kShiftRightLogical, abs_dividend,
+          shift_amount));
+
+      auto* neqated_quotient =
+          computation->AddInstruction(HloInstruction::CreateUnary(
+              quotient->shape(), HloOpcode::kNegate, quotient));
+
+      return HloInstruction::CreateTernary(divide->shape(), HloOpcode::kSelect,
+                                           dividend_is_negative,
+                                           neqated_quotient, quotient);
+    }
+  } else {
+    uint64 b_value = c->literal().GetFirstElement<T>();
+    if (IsPowerOfTwo(b_value)) {
+      int log2_abs_b_value = tensorflow::Log2Floor64(b_value);
+      HloInstruction* shift_amount =
+          computation->AddInstruction(HloInstruction::CreateConstant(
+              LiteralUtil::CreateR0<T>(log2_abs_b_value)));
+      if (!ShapeUtil::IsScalar(b->shape())) {
+        shift_amount = computation->AddInstruction(
+            HloInstruction::CreateBroadcast(b->shape(), shift_amount, {}));
+      }
+      return HloInstruction::CreateBinary(
+          divide->shape(), HloOpcode::kShiftRightLogical, a, shift_amount);
+    }
+  }
+
+  return nullptr;
+}
 }  // namespace
 
 Status AlgebraicSimplifierVisitor::HandleDivide(HloInstruction* divide) {
@@ -824,6 +901,60 @@ Status AlgebraicSimplifierVisitor::HandleDivide(HloInstruction* divide) {
     return Status::OK();
   }
 
+  // A / B => A >> log2(B) if B is a power of 2.
+  switch (divide->shape().element_type()) {
+    case S8:
+      if (std::unique_ptr<HloInstruction> shift =
+              TryDivideToShift<int8>(divide, computation_)) {
+        return ReplaceWithNewInstruction(divide, std::move(shift));
+      }
+      break;
+    case S16:
+      if (std::unique_ptr<HloInstruction> shift =
+              TryDivideToShift<int16>(divide, computation_)) {
+        return ReplaceWithNewInstruction(divide, std::move(shift));
+      }
+      break;
+    case S32:
+      if (std::unique_ptr<HloInstruction> shift =
+              TryDivideToShift<int32>(divide, computation_)) {
+        return ReplaceWithNewInstruction(divide, std::move(shift));
+      }
+      break;
+    case S64:
+      if (std::unique_ptr<HloInstruction> shift =
+              TryDivideToShift<int64>(divide, computation_)) {
+        return ReplaceWithNewInstruction(divide, std::move(shift));
+      }
+      break;
+    case U8:
+      if (std::unique_ptr<HloInstruction> shift =
+              TryDivideToShift<uint8>(divide, computation_)) {
+        return ReplaceWithNewInstruction(divide, std::move(shift));
+      }
+      break;
+    case U16:
+      if (std::unique_ptr<HloInstruction> shift =
+              TryDivideToShift<uint16>(divide, computation_)) {
+        return ReplaceWithNewInstruction(divide, std::move(shift));
+      }
+      break;
+    case U32:
+      if (std::unique_ptr<HloInstruction> shift =
+              TryDivideToShift<uint32>(divide, computation_)) {
+        return ReplaceWithNewInstruction(divide, std::move(shift));
+      }
+      break;
+    case U64:
+      if (std::unique_ptr<HloInstruction> shift =
+              TryDivideToShift<uint64>(divide, computation_)) {
+        return ReplaceWithNewInstruction(divide, std::move(shift));
+      }
+      break;
+    default:
+      break;
+  }
+
   // exp(A)/exp(B) => exp(A-B)
   if (Match(divide, m::Divide(m::Exp(m::Op(&a)), m::Exp(m::Op(&b)))
                         .WithShape(m::Shape(&shape)))) {
@@ -2159,6 +2290,137 @@ AlgebraicSimplifierVisitor::TryToSinkBroadcastAfterOpWithUniqueNonScalarOperand(
   return changed;
 }
 
+namespace {
+template <typename T>
+std::unique_ptr<HloInstruction> TryRemainderToAnd(HloInstruction* remainder,
+                                                  HloComputation* computation) {
+  HloInstruction *a, *b, *c;
+  CHECK(Match(remainder, m::Remainder(m::Op(&a), m::Op(&b))));
+
+  if (ShapeUtil::ElementIsIntegral(remainder->shape()) &&
+      !Match(b, m::ConstantEffectiveScalar(&c)) &&
+      !Match(b, m::Broadcast(m::ConstantEffectiveScalar(&c)))) {
+    return nullptr;
+  }
+
+  if (ShapeUtil::ElementIsSigned(remainder->shape())) {
+    int64 b_value = c->literal().GetFirstElement<T>();
+    if (b_value > 0 && IsPowerOfTwo(static_cast<uint64>(b_value))) {
+      // Handle negative dividends by negating the result of the division.
+      HloInstruction* zero_like_a = BroadcastZeros(
+          computation, a->shape().element_type(), a->shape().dimensions());
+
+      auto* dividend_is_negative =
+          computation->AddInstruction(HloInstruction::CreateBinary(
+              ShapeUtil::ChangeElementType(a->shape(), PRED), HloOpcode::kLt, a,
+              zero_like_a));
+
+      auto* negated_dividend = computation->AddInstruction(
+          HloInstruction::CreateUnary(a->shape(), HloOpcode::kNegate, a));
+
+      auto* abs_dividend =
+          computation->AddInstruction(HloInstruction::CreateTernary(
+              a->shape(), HloOpcode::kSelect, dividend_is_negative,
+              negated_dividend, a));
+
+      auto* mask_amount =
+          computation->AddInstruction(HloInstruction::CreateConstant(
+              LiteralUtil::CreateR0<T>(b_value - 1)));
+      if (!ShapeUtil::IsScalar(b->shape())) {
+        mask_amount = computation->AddInstruction(
+            HloInstruction::CreateBroadcast(b->shape(), mask_amount, {}));
+      }
+
+      auto* quotient = computation->AddInstruction(HloInstruction::CreateBinary(
+          remainder->shape(), HloOpcode::kAnd, abs_dividend, mask_amount));
+
+      auto* neqated_quotient =
+          computation->AddInstruction(HloInstruction::CreateUnary(
+              quotient->shape(), HloOpcode::kNegate, quotient));
+
+      return HloInstruction::CreateTernary(
+          remainder->shape(), HloOpcode::kSelect, dividend_is_negative,
+          neqated_quotient, quotient);
+    }
+  } else {
+    uint64 b_value = c->literal().GetFirstElement<T>();
+    if (IsPowerOfTwo(b_value)) {
+      HloInstruction* mask_amount =
+          computation->AddInstruction(HloInstruction::CreateConstant(
+              LiteralUtil::CreateR0<T>(b_value - 1)));
+      if (!ShapeUtil::IsScalar(b->shape())) {
+        mask_amount = computation->AddInstruction(
+            HloInstruction::CreateBroadcast(b->shape(), mask_amount, {}));
+      }
+      return HloInstruction::CreateBinary(remainder->shape(), HloOpcode::kAnd,
+                                          a, mask_amount);
+    }
+  }
+  return nullptr;
+}
+}  // namespace
+
+Status AlgebraicSimplifierVisitor::HandleRemainder(HloInstruction* remainder) {
+  HloInstruction *a, *b;
+  CHECK(Match(remainder, m::Remainder(m::Op(&a), m::Op(&b))));
+
+  // A % B => A & (B - 1) if B is a power of 2.
+  switch (remainder->shape().element_type()) {
+    case S8:
+      if (std::unique_ptr<HloInstruction> shift =
+              TryRemainderToAnd<int8>(remainder, computation_)) {
+        return ReplaceWithNewInstruction(remainder, std::move(shift));
+      }
+      break;
+    case S16:
+      if (std::unique_ptr<HloInstruction> shift =
+              TryRemainderToAnd<int16>(remainder, computation_)) {
+        return ReplaceWithNewInstruction(remainder, std::move(shift));
+      }
+      break;
+    case S32:
+      if (std::unique_ptr<HloInstruction> shift =
+              TryRemainderToAnd<int32>(remainder, computation_)) {
+        return ReplaceWithNewInstruction(remainder, std::move(shift));
+      }
+      break;
+    case S64:
+      if (std::unique_ptr<HloInstruction> shift =
+              TryRemainderToAnd<int64>(remainder, computation_)) {
+        return ReplaceWithNewInstruction(remainder, std::move(shift));
+      }
+      break;
+    case U8:
+      if (std::unique_ptr<HloInstruction> shift =
+              TryRemainderToAnd<uint8>(remainder, computation_)) {
+        return ReplaceWithNewInstruction(remainder, std::move(shift));
+      }
+      break;
+    case U16:
+      if (std::unique_ptr<HloInstruction> shift =
+              TryRemainderToAnd<uint16>(remainder, computation_)) {
+        return ReplaceWithNewInstruction(remainder, std::move(shift));
+      }
+      break;
+    case U32:
+      if (std::unique_ptr<HloInstruction> shift =
+              TryRemainderToAnd<uint32>(remainder, computation_)) {
+        return ReplaceWithNewInstruction(remainder, std::move(shift));
+      }
+      break;
+    case U64:
+      if (std::unique_ptr<HloInstruction> shift =
+              TryRemainderToAnd<uint64>(remainder, computation_)) {
+        return ReplaceWithNewInstruction(remainder, std::move(shift));
+      }
+      break;
+    default:
+      break;
+  }
+
+  return Status::OK();
+}
+
 Status AlgebraicSimplifierVisitor::HandleReshape(HloInstruction* reshape) {
   auto operand = reshape->mutable_operand(0);
 
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
index 0dea498456..96354e6f6e 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
@@ -194,6 +194,86 @@ TEST_F(AlgebraicSimplifierTest, FactorFpAdditionBfloat16) {
                   m::Broadcast(m::ConstantScalar(0.125)))));
 }
 
+TEST_F(AlgebraicSimplifierTest, UnsignedDivideByPowerOf2) {
+  const char* kModuleStr = R"(
+    HloModule m
+    test {
+      p = u32[4] parameter(0)
+      c = u32[] constant(8)
+      b = u32[4] broadcast(c), dimensions={}
+      ROOT d = u32[4] divide(p, b)
+    }
+  )";
+  TF_ASSERT_OK_AND_ASSIGN(auto m, ParseAndReturnVerifiedModule(kModuleStr));
+  ASSERT_TRUE(AlgebraicSimplifier(default_options_).Run(m.get()).ValueOrDie());
+  EXPECT_THAT(m->entry_computation()->root_instruction(),
+              GmockMatch(m::ShiftRightLogical(
+                  m::Parameter(0), m::Broadcast(m::ConstantScalar(3)))));
+}
+
+TEST_F(AlgebraicSimplifierTest, SignedDivideByPowerOf2) {
+  const char* kModuleStr = R"(
+    HloModule m
+    test {
+      p = s32[4] parameter(0)
+      c = s32[] constant(8)
+      b = s32[4] broadcast(c), dimensions={}
+      ROOT d = s32[4] divide(p, b)
+    }
+  )";
+  TF_ASSERT_OK_AND_ASSIGN(auto m, ParseAndReturnVerifiedModule(kModuleStr));
+  ASSERT_TRUE(AlgebraicSimplifier(default_options_).Run(m.get()).ValueOrDie());
+  auto match_dividend_is_negative =
+      m::Lt(m::Parameter(0), m::Broadcast(m::ConstantScalar(0)));
+  auto match_abs = m::Select(match_dividend_is_negative,
+                             m::Negate(m::Parameter(0)), m::Parameter(0));
+  auto match_shift =
+      m::ShiftRightLogical(match_abs, m::Broadcast(m::ConstantScalar(3)));
+  EXPECT_THAT(m->entry_computation()->root_instruction(),
+              GmockMatch(m::Select(match_dividend_is_negative,
+                                   m::Negate(match_shift), match_shift)));
+}
+
+TEST_F(AlgebraicSimplifierTest, UnsignedRemainderByPowerOf2) {
+  const char* kModuleStr = R"(
+    HloModule m
+    test {
+      p = u32[4] parameter(0)
+      c = u32[] constant(8)
+      b = u32[4] broadcast(c), dimensions={}
+      ROOT r = u32[4] remainder(p, b)
+    }
+  )";
+  TF_ASSERT_OK_AND_ASSIGN(auto m, ParseAndReturnVerifiedModule(kModuleStr));
+  ASSERT_TRUE(AlgebraicSimplifier(default_options_).Run(m.get()).ValueOrDie());
+  EXPECT_THAT(m->entry_computation()->root_instruction(),
+              GmockMatch(m::AndAnyOrder(m::Parameter(0),
+                                        m::Broadcast(m::ConstantScalar(7)))));
+}
+
+TEST_F(AlgebraicSimplifierTest, SignedRemainderByPowerOf2) {
+  const char* kModuleStr = R"(
+    HloModule m
+    test {
+      p = s32[4] parameter(0)
+      c = s32[] constant(8)
+      b = s32[4] broadcast(c), dimensions={}
+      ROOT r = s32[4] remainder(p, b)
+    }
+  )";
+  TF_ASSERT_OK_AND_ASSIGN(auto m, ParseAndReturnVerifiedModule(kModuleStr));
+  ASSERT_TRUE(AlgebraicSimplifier(default_options_).Run(m.get()).ValueOrDie());
+  auto match_dividend_is_negative =
+      m::Lt(m::Parameter(0), m::Broadcast(m::ConstantScalar(0)));
+  auto match_abs = m::Select(match_dividend_is_negative,
+                             m::Negate(m::Parameter(0)), m::Parameter(0));
+  auto match_and =
+      m::AndAnyOrder(match_abs, m::Broadcast(m::ConstantScalar(7)));
+  EXPECT_THAT(m->entry_computation()->root_instruction(),
+              GmockMatch(m::Select(match_dividend_is_negative,
+                                   m::Negate(match_and), match_and)));
+}
+
 // Test that A * 0 is simplified to 0
 TEST_F(AlgebraicSimplifierTest, MulZero) {
   auto m = CreateNewVerifiedModule();
diff --git a/tensorflow/compiler/xla/service/gather_expander.cc b/tensorflow/compiler/xla/service/gather_expander.cc
index 01cef49966..590942cddc 100644
--- a/tensorflow/compiler/xla/service/gather_expander.cc
+++ b/tensorflow/compiler/xla/service/gather_expander.cc
@@ -153,10 +153,9 @@ static StatusOr<std::vector<HloInstruction*>> GatherLoopBody(
            dim_numbers.index_vector_dim() ==
                gather.operand(1)->shape().dimensions_size());
 
-  TF_ASSIGN_OR_RETURN(
-      HloInstruction * induction_var_as_vector,
+  HloInstruction* induction_var_as_vector =
       MakeBroadcastHlo(induction_var, /*broadcast_dimensions=*/{},
-                       /*result_shape_bounds=*/{1}));
+                       /*result_shape_bounds=*/{1});
 
   HloInstruction* index_vector;
 
@@ -222,7 +221,7 @@ static StatusOr<std::vector<HloInstruction*>> GatherLoopBody(
       {operand, start_indices, updated_accumulator}};
 }
 
-static StatusOr<HloInstruction*> CreateGatherLoopAccumulatorInitValue(
+static HloInstruction* CreateGatherLoopAccumulatorInitValue(
     HloComputation* computation, PrimitiveType element_type,
     absl::Span<const int64> slice_sizes, int64 gather_loop_trip_count,
     const GatherDimensionNumbers& dim_numbers) {
@@ -332,12 +331,10 @@ StatusOr<HloInstruction*> GatherExpander::ExpandGather(
   CHECK_EQ(gather_loop_trip_count,
            canonical_start_indices->shape().dimensions(0));
 
-  TF_ASSIGN_OR_RETURN(
-      HloInstruction * accumulator_init,
-      CreateGatherLoopAccumulatorInitValue(
-          computation, output_shape.element_type(),
-          gather_instr->gather_slice_sizes(), gather_loop_trip_count,
-          gather_instr->gather_dimension_numbers()));
+  HloInstruction* accumulator_init = CreateGatherLoopAccumulatorInitValue(
+      computation, output_shape.element_type(),
+      gather_instr->gather_slice_sizes(), gather_loop_trip_count,
+      gather_instr->gather_dimension_numbers());
 
   StatusOr<std::vector<HloInstruction*>> gather_loop_result_or_error =
       WhileUtil::MakeCountedLoop(
diff --git a/tensorflow/compiler/xla/service/hlo_creation_utils.cc b/tensorflow/compiler/xla/service/hlo_creation_utils.cc
index bb5d21c654..d56f673455 100644
--- a/tensorflow/compiler/xla/service/hlo_creation_utils.cc
+++ b/tensorflow/compiler/xla/service/hlo_creation_utils.cc
@@ -156,9 +156,9 @@ StatusOr<HloInstruction*> MakeDynamicUpdateSliceHlo(
       dynamic_update_slice_shape, operand, update, scalar_start_indices));
 }
 
-StatusOr<HloInstruction*> MakeBroadcastHlo(
-    HloInstruction* operand, absl::Span<const int64> broadcast_dimensions,
-    absl::Span<const int64> result_shape_bounds) {
+HloInstruction* MakeBroadcastHlo(HloInstruction* operand,
+                                 absl::Span<const int64> broadcast_dimensions,
+                                 absl::Span<const int64> result_shape_bounds) {
   HloComputation* computation = operand->parent();
   Shape broadcast_shape = ShapeUtil::MakeShape(operand->shape().element_type(),
                                                result_shape_bounds);
@@ -394,9 +394,9 @@ StatusOr<HloInstruction*> PadVectorWithZeros(HloInstruction* operand,
   return MakePadHlo(operand, zero, padding_config);
 }
 
-StatusOr<HloInstruction*> BroadcastZeros(
-    HloComputation* computation, PrimitiveType element_type,
-    absl::Span<const int64> broadcast_dimensions) {
+HloInstruction* BroadcastZeros(HloComputation* computation,
+                               PrimitiveType element_type,
+                               absl::Span<const int64> broadcast_dimensions) {
   HloInstruction* zero = computation->AddInstruction(
       HloInstruction::CreateConstant(LiteralUtil::Zero(element_type)));
   return MakeBroadcastHlo(zero, /*broadcast_dimensions=*/{},
diff --git a/tensorflow/compiler/xla/service/hlo_creation_utils.h b/tensorflow/compiler/xla/service/hlo_creation_utils.h
index 8e5ddbbd50..1c3174e9c8 100644
--- a/tensorflow/compiler/xla/service/hlo_creation_utils.h
+++ b/tensorflow/compiler/xla/service/hlo_creation_utils.h
@@ -82,9 +82,9 @@ StatusOr<HloInstruction*> MakeDynamicUpdateSliceHlo(
 
 // Creates a broadcast HLO instruction and adds it to the computation containing
 // `operand`.
-StatusOr<HloInstruction*> MakeBroadcastHlo(
-    HloInstruction* operand, absl::Span<const int64> broadcast_dimensions,
-    absl::Span<const int64> result_shape_bounds);
+HloInstruction* MakeBroadcastHlo(HloInstruction* operand,
+                                 absl::Span<const int64> broadcast_dimensions,
+                                 absl::Span<const int64> result_shape_bounds);
 
 // Creates a GetTupleElement HLO instruction and adds it to the computation
 // containing `operand`.
@@ -198,9 +198,9 @@ StatusOr<HloInstruction*> PadVectorWithZeros(HloInstruction* operand,
 // Broadcasts a zero value of type `element_type` into a tensor with element
 // type `element_type` and dimension bounds `broadcast_dimensions`.  The
 // broadcast instruction is emitted into `computation`.
-StatusOr<HloInstruction*> BroadcastZeros(
-    HloComputation* computation, PrimitiveType element_type,
-    absl::Span<const int64> broadcast_dimensions);
+HloInstruction* BroadcastZeros(HloComputation* computation,
+                               PrimitiveType element_type,
+                               absl::Span<const int64> broadcast_dimensions);
 
 // Creates a HLO computation that takes arguments of type `domain` and produces
 // a value of type `range`.
diff --git a/tensorflow/compiler/xla/service/hlo_creation_utils_test.cc b/tensorflow/compiler/xla/service/hlo_creation_utils_test.cc
index 3715e12b4e..6025e6a779 100644
--- a/tensorflow/compiler/xla/service/hlo_creation_utils_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_creation_utils_test.cc
@@ -191,9 +191,8 @@ TEST_F(HloCreationUtilsTest, BroadcastZeros_S32) {
                                              /*output_shape_dims=*/{2, 2},
                                              &param, &entry_computation);
 
-  TF_ASSERT_OK_AND_ASSIGN(
-      HloInstruction * zeros,
-      BroadcastZeros(module->entry_computation(), S32, {2, 2}));
+  HloInstruction* zeros =
+      BroadcastZeros(module->entry_computation(), S32, {2, 2});
   entry_computation->set_root_instruction(zeros);
 
   HloEvaluator evaluator;
@@ -211,9 +210,8 @@ TEST_F(HloCreationUtilsTest, BroadcastZeros_F32) {
                                              /*output_shape_dims=*/{2, 2},
                                              &param, &entry_computation);
 
-  TF_ASSERT_OK_AND_ASSIGN(
-      HloInstruction * zeros,
-      BroadcastZeros(module->entry_computation(), F32, {2, 2}));
+  HloInstruction* zeros =
+      BroadcastZeros(module->entry_computation(), F32, {2, 2});
   entry_computation->set_root_instruction(zeros);
 
   HloEvaluator evaluator;
diff --git a/tensorflow/compiler/xla/service/scatter_expander.cc b/tensorflow/compiler/xla/service/scatter_expander.cc
index 036c3c36f6..acad871c4d 100644
--- a/tensorflow/compiler/xla/service/scatter_expander.cc
+++ b/tensorflow/compiler/xla/service/scatter_expander.cc
@@ -171,10 +171,9 @@ static StatusOr<HloInstruction*> CheckIndexValidity(
   // Valid range for the index: [0, operand_dims - window_sizes]
 
   // Check if the index has any negative values.
-  TF_ASSIGN_OR_RETURN(
-      HloInstruction * zero_index,
+  HloInstruction* zero_index =
       BroadcastZeros(computation, index->shape().element_type(),
-                     AsInt64Slice(index->shape().dimensions())));
+                     AsInt64Slice(index->shape().dimensions()));
   TF_ASSIGN_OR_RETURN(HloInstruction * negative_index_check,
                       MakeBinaryHlo(HloOpcode::kLe, zero_index, index));
 
@@ -222,10 +221,9 @@ static StatusOr<std::vector<HloInstruction*>> ScatterLoopBody(
   bool has_scalar_indices = scatter_indices->shape().dimensions_size() == 1;
 
   // Build a vector form of the induction variable of the while loop.
-  TF_ASSIGN_OR_RETURN(
-      HloInstruction * induction_var_as_vector,
+  HloInstruction* induction_var_as_vector =
       MakeBroadcastHlo(induction_var, /*broadcast_dimensions=*/{},
-                       /*result_shape_bounds=*/{1}));
+                       /*result_shape_bounds=*/{1});
 
   // Pick the index to scatter from scatter_indices based on the induction_var
   // and transform that to an index into the `operand` space.
-- 
GitLab


From 12e86468e2a9b57e636c1d0afcdf3f657f6df0b6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sun, 20 Jan 2019 01:03:06 -0800
Subject: [PATCH 1031/1095] compat: Update forward compatibility horizon to
 2019-01-20

PiperOrigin-RevId: 230110846
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 14b9a562cd..16fe3db26e 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 19)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 20)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 329bd829d234165d235d2a7d47c3b2d8c8ca114f Mon Sep 17 00:00:00 2001
From: Pete Warden <petewarden@google.com>
Date: Sun, 20 Jan 2019 14:48:27 -0800
Subject: [PATCH 1032/1095] Fix typos in TFL Micro documentation

PiperOrigin-RevId: 230148789
---
 tensorflow/lite/experimental/micro/README.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/lite/experimental/micro/README.md b/tensorflow/lite/experimental/micro/README.md
index e85b9660e3..6568ecd0fa 100644
--- a/tensorflow/lite/experimental/micro/README.md
+++ b/tensorflow/lite/experimental/micro/README.md
@@ -65,7 +65,7 @@ you can easily explore the code in a cross-platform IDE.
 
 ## Building Portable Reference Code using Make
 
-It's easy to build portable reference code directly from Github using make if
+It's easy to build portable reference code directly from GitHub using make if
 you're on a Linux or OS X machine.
 
 -   Open a terminal
@@ -261,7 +261,7 @@ You should see the following log with the magic string `~~~ALL TEST PASSED~~~`:
 02:25:22.4253 [DEBUG] uart0: [+0.16ms host +0s virt 0.28s virt from start]   Progam has exited with code:0x00000000
 ```
 
-## Building for Ambiq Micro Apollo3Blue EVB
+## Building for Ambiq Micro Apollo3Blue EVB using Make
 
 Follow these steps to get the pushbutton yes/no example working on Apollo 3:
 
@@ -713,7 +713,7 @@ exist.
 
 So far, I've recommended that you use the standalone generated projects for your
 system. You might be wondering why you're not just checking out the full
-[TensorFlow codebase from Github](https://github.com/tensorflow/tensorflow/)?
+[TensorFlow codebase from GitHub](https://github.com/tensorflow/tensorflow/)?
 The main reason is that there is a lot more diversity of architectures, IDEs,
 support libraries, and operating systems in the embedded world. Many of the
 toolchains require their own copy of source files, or a list of sources to be
@@ -749,7 +749,7 @@ wider community? The first step is to pick a tag name for your platform. This
 can either be the operating system (for example 'mbed'), the name of a device
 ('bluepill'), or some other text that describes it. This should be a short
 string with no spaces or special characters. Log in or create an account on
-Github, fork the full
+GitHub, fork the full
 [TensorFlow codebase](https://github.com/tensorflow/tensorflow/) using the
 'Fork' button on the top left, and then grab your fork by using a command like
 `git clone https://github.com/<your user name>/tensorflow`.
-- 
GitLab


From eb17145a7e8f8d50418d0238e8dbd445ea2bd1d7 Mon Sep 17 00:00:00 2001
From: Alan Chiao <alanchiao@google.com>
Date: Sun, 20 Jan 2019 14:53:09 -0800
Subject: [PATCH 1033/1095] Refresh open-source perturbed test acoustic model
 to be compatible with today's kernels.

Add equivalent model with post-training quantization.

PiperOrigin-RevId: 230148964
---
 tensorflow/lite/models/speech_test.cc         | 15 +++-
 .../lite/models/testdata/g3doc/README.md      | 90 ++++++++-----------
 2 files changed, 50 insertions(+), 55 deletions(-)

diff --git a/tensorflow/lite/models/speech_test.cc b/tensorflow/lite/models/speech_test.cc
index f3509d1ece..a3713c5531 100644
--- a/tensorflow/lite/models/speech_test.cc
+++ b/tensorflow/lite/models/speech_test.cc
@@ -139,7 +139,7 @@ TEST_P(SpeechTest, DISABLED_SpeakerIdOkGoogleTest) {
       << test_driver.GetErrorMessage();
 }
 
-TEST_P(SpeechTest, DISABLED_AsrAmTest) {
+TEST_P(SpeechTest, AsrAmTest) {
   std::stringstream os;
   ASSERT_TRUE(
       ConvertCsvData("speech_asr_am_model.tflite", "speech_asr_am_model_in.csv",
@@ -152,6 +152,19 @@ TEST_P(SpeechTest, DISABLED_AsrAmTest) {
       << test_driver.GetErrorMessage();
 }
 
+TEST_P(SpeechTest, AsrAmQuantizedTest) {
+  std::stringstream os;
+  ASSERT_TRUE(ConvertCsvData(
+      "speech_asr_am_model_int8.tflite", "speech_asr_am_model_in.csv",
+      "speech_asr_am_model_int8_out.csv", /*input_tensor=*/"0",
+      /*output_tensor=*/"104",
+      /*persistent_tensors=*/"18,19,38,39,58,59,78,79,98,99",
+      /*sequence_size=*/320, &os));
+  testing::TfLiteDriver test_driver(/*use_nnapi=*/false);
+  ASSERT_TRUE(testing::ParseAndRunTests(&os, &test_driver, GetMaxInvocations()))
+      << test_driver.GetErrorMessage();
+}
+
 // The original version of speech_asr_lm_model_test.cc ran a few sequences
 // through the interpreter and stored the sum of all the output, which was them
 // compared for correctness. In this test we are comparing all the intermediate
diff --git a/tensorflow/lite/models/testdata/g3doc/README.md b/tensorflow/lite/models/testdata/g3doc/README.md
index 2a4f1c143a..afe5f16b38 100644
--- a/tensorflow/lite/models/testdata/g3doc/README.md
+++ b/tensorflow/lite/models/testdata/g3doc/README.md
@@ -3,6 +3,42 @@
 Sample test data has been provided for speech related models in Tensorflow Lite
 to help users working with speech models to verify and test their models.
 
+### Models and Inputs and Outputs:
+
+[ASR AM model](https://storage.googleapis.com/download.tensorflow.org/models/tflite/speech_asr_am_model.tflite)
+
+[ASR AM quantized model](https://storage.googleapis.com/download.tensorflow.org/models/tflite/speech_asr_am_model_int8.tflite)
+
+[ASR AM test inputs](https://storage.googleapis.com/download.tensorflow.org/models/tflite/speech_asr_am_model_in.csv)
+
+[ASR AM test outputs](https://storage.googleapis.com/download.tensorflow.org/models/tflite/speech_asr_am_model_out.csv)
+
+[ASR AM int8 test outputs](https://storage.googleapis.com/download.tensorflow.org/models/tflite/speech_asr_am_model_int8_out.csv)
+
+The models below are not maintained.
+
+[Speech hotword model (Svdf
+rank=1)](https://storage.googleapis.com/download.tensorflow.org/models/tflite/speech_hotword_model_rank1_2017_11_14.tflite)
+
+[Speech hotword model (Svdf
+rank=2)](https://storage.googleapis.com/download.tensorflow.org/models/tflite/speech_hotword_model_rank2_2017_11_14.tflite)
+
+[Speaker-id
+model](https://storage.googleapis.com/download.tensorflow.org/models/tflite/speech_speakerid_model_2017_11_14.tflite)
+
+[TTS
+model](https://storage.googleapis.com/download.tensorflow.org/models/tflite/speech_tts_model_2017_11_14.tflite)
+
+### Test Bench
+
+[Model tests](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/models/speech_test.cc)
+
+Download the ASR AM test models and inputs and output files to the
+models/testdata directory to run the tests.
+
+
+## Speech Model Architectures
+
 For the hotword, speaker-id and automatic speech recognition sample models, the
 architecture assumes that the models receive their input from a speech
 pre-processing module. The speech pre-processing module receives the audio
@@ -87,57 +123,3 @@ The model consists of a convolutional layer, followed by a fully-connected
 layer, two LSTM layers, and two additional fully-connected layers.
 The corresponding parameters as shown in the figure.
 ![endpointer_model](endpointer.svg "Endpointer model")
-
-
-## Speech models test input/output generation
-
-As mentioned above the input to models are generated from a pre-processing
-module (output of a log-mel filterbank, or linguistic features), and the outputs
-are generated by running the equivalent TensorFlow model by feeding them the
-same input.
-
-## Link to the open source code
-
-### Models:
-
-[Speech hotword model (Svdf
-rank=1)](https://storage.googleapis.com/download.tensorflow.org/models/tflite/speech_hotword_model_rank1_2017_11_14.tflite)
-
-[Speech hotword model (Svdf
-rank=2)](https://storage.googleapis.com/download.tensorflow.org/models/tflite/speech_hotword_model_rank2_2017_11_14.tflite)
-
-[Speaker-id
-model](https://storage.googleapis.com/download.tensorflow.org/models/tflite/speech_speakerid_model_2017_11_14.tflite)
-
-[TTS
-model](https://storage.googleapis.com/download.tensorflow.org/models/tflite/speech_tts_model_2017_11_14.tflite)
-
-[ASR AM
-model](https://storage.googleapis.com/download.tensorflow.org/models/tflite/speech_terse_am_model_2017_11_14.tflite)
-
-### Test benches
-
-[Speech hotword model
-test](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/models/speech_hotword_model_test.cc)
-
-[Speaker-id model
-test](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/models/speech_speakerid_model_test.cc)
-
-[TTS model
-test](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/models/speech_tts_model_test.cc)
-
-[ASR AM model
-test](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/models/speech_asr_am_model_test.cc)
-
-[ASR LM model
-test](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/models/speech_asr_lm_model_test.cc)
-
-[Endpointer model
-test](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/models/speech_endpointer_model_test.cc)
-
-## Android Support
-The models have been tested on Android phones, using the following tests:
-
-[Hotword] (https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/android/BUILD?rcl=172930882&l=25)
-
-[Speaker-id] (https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/android/BUILD?rcl=172930882&l=36)
-- 
GitLab


From 4f1b0ee65a652743ede11d95a33513588ba7f772 Mon Sep 17 00:00:00 2001
From: Haoyu Zhang <haoyuzhang@google.com>
Date: Sun, 20 Jan 2019 16:37:03 -0800
Subject: [PATCH 1034/1095] Prevent repeatedly calling iterator.get_next() in
 Keras training loop.

Calling iterator.get_next() repeatedly causes execution slowdown and eventually leads to OOM. This change avoids this call on (1) training dataset iterator when the dataset needs to be reset after an epoch; (2) validation dataset iterator when multiple validation passes need to be invoked during training.

PiperOrigin-RevId: 230154198
---
 .../python/keras/engine/training_arrays.py    | 38 +++++++++++++------
 1 file changed, 26 insertions(+), 12 deletions(-)

diff --git a/tensorflow/python/keras/engine/training_arrays.py b/tensorflow/python/keras/engine/training_arrays.py
index d0881c11a5..cefd4d57fd 100644
--- a/tensorflow/python/keras/engine/training_arrays.py
+++ b/tensorflow/python/keras/engine/training_arrays.py
@@ -60,6 +60,7 @@ def model_iteration(model,
                     validation_freq=1,
                     mode=ModeKeys.TRAIN,
                     validation_in_fit=False,
+                    prepared_feed_values_from_dataset=False,
                     steps_name='steps',
                     **kwargs):
   """Loop function for arrays of data with modes TRAIN/TEST/PREDICT.
@@ -94,11 +95,14 @@ def model_iteration(model,
         which to run validation, e.g. `validation_freq=[1, 2, 10]` runs
         validation at the end of the 1st, 2nd, and 10th epochs.
       mode: One of ModeKeys.TRAIN/ModeKeys.TEST/ModeKeys.PREDICT.
-      validation_in_fit: DEPRECATED: if true, then this method is invoked from
-        within training iteration (for validation). In this case, do not copy
-        weights when using a tf.distribute.Strategy. The input is deprecated as
-        it is not required if the user creates a distributed model under the
-        distribution strategy scope rather than passing it to compile.
+      validation_in_fit: if true, then this method is invoked from within
+        training iteration (for validation). In the case where `val_inputs` is a
+        dataset, this flag indicates that its iterator and feed values are
+        already created so should properly reuse resources.
+      prepared_feed_values_from_dataset: if True, `inputs` is a list of feed
+        tensors returned from `_prepare_feed_values` call on the validation
+        dataset, so do not call it again on `inputs`. Should only be used for
+        inline validation (i.e., only if `validation_in_fit` is also True).
       steps_name: The string name of the steps argument, either `steps`,
         `validation_steps`, or `steps_per_epoch`. Only used for error message
         formatting.
@@ -133,10 +137,6 @@ def model_iteration(model,
           inputs, steps_per_epoch, epochs=epochs, steps_name=steps_name)
     input_iterator = _get_iterator(inputs, model._distribution_strategy)
 
-  val_iterator = None
-  if isinstance(val_inputs, (dataset_ops.DatasetV1, dataset_ops.DatasetV2)):
-    val_iterator = _get_iterator(val_inputs, model._distribution_strategy)
-
   if mode == ModeKeys.TRAIN:
     _print_train_info(inputs, val_inputs, steps_per_epoch, verbose)
 
@@ -156,13 +156,28 @@ def model_iteration(model,
 
   # Prepare input data.
   inputs = input_iterator or inputs
-  ins = _prepare_feed_values(model, inputs, targets, sample_weights, mode)
+  if validation_in_fit and prepared_feed_values_from_dataset:
+    # When invoking validation in training loop, avoid creating iterator and
+    # list of feed values for the same validation dataset multiple times (which
+    # essentially would call `iterator.get_next()` that slows down execution and
+    # leads to OOM errors eventually.
+    ins = inputs
+  else:
+    ins = _prepare_feed_values(model, inputs, targets, sample_weights, mode)
   if not is_dataset:
     num_samples_or_steps = _get_num_samples_or_steps(ins, batch_size,
                                                      steps_per_epoch)
   else:
     num_samples_or_steps = steps_per_epoch
 
+  # Prepare validation data. Hold references to the iterator and the input list
+  # to properly reinitialize and reuse in multiple validation passes.
+  val_iterator = None
+  if isinstance(val_inputs, (dataset_ops.DatasetV1, dataset_ops.DatasetV2)):
+    val_iterator = _get_iterator(val_inputs, model._distribution_strategy)
+    val_inputs = _prepare_feed_values(
+        model, val_iterator, val_targets, val_sample_weights, ModeKeys.TEST)
+
   # Configure callbacks.
   count_mode = 'steps' if use_steps else 'samples'
   callbacks = cbks.configure_callbacks(
@@ -338,7 +353,6 @@ def model_iteration(model,
     if (do_validation and
         training_utils.should_run_validation(validation_freq, epoch) and
         not callbacks.model.stop_training):
-      val_inputs = val_iterator or val_inputs
       val_results = model_iteration(
           model,
           val_inputs,
@@ -350,6 +364,7 @@ def model_iteration(model,
           verbose=0,
           mode=ModeKeys.TEST,
           validation_in_fit=True,
+          prepared_feed_values_from_dataset=(val_iterator is not None),
           steps_name='validation_steps')
       if not isinstance(val_results, list):
         val_results = [val_results]
@@ -366,7 +381,6 @@ def model_iteration(model,
     # Reinitialize dataset iterator for the next epoch.
     if reset_dataset_after_each_epoch and epoch < epochs - 1:
       _reinitialize_iterator(input_iterator, model._distribution_strategy)
-      ins = _prepare_feed_values(model, input_iterator, None, None, mode)
 
   callbacks._call_end_hook(mode)
 
-- 
GitLab


From 48998029e313b9206daa2d49f55aeb810242361e Mon Sep 17 00:00:00 2001
From: Pete Warden <petewarden@google.com>
Date: Sun, 20 Jan 2019 17:03:28 -0800
Subject: [PATCH 1035/1095] Fix formatting on Apollo3 instructions

PiperOrigin-RevId: 230155453
---
 tensorflow/lite/experimental/micro/README.md | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/tensorflow/lite/experimental/micro/README.md b/tensorflow/lite/experimental/micro/README.md
index 6568ecd0fa..30070bf008 100644
--- a/tensorflow/lite/experimental/micro/README.md
+++ b/tensorflow/lite/experimental/micro/README.md
@@ -307,10 +307,12 @@ Follow these steps to get the pushbutton yes/no example working on Apollo 3:
 
 ### Additional Apollo3 Instructions
 
-To flash a part with JFlash Lite, do the following: 1. At the command line:
-JFlashLiteExe 2. Device = AMA3B1KK-KBR 3. Interface = SWD at 1000 kHz 4. Data
-file =
-tensorflow/lite/experimental/micro/tools/make/gen/apollo3evb_cortex-m4/bin/pushbutton_cmsis_speech_test.bin
+To flash a part with JFlash Lite, do the following: 
+
+1. At the command line: JFlashLiteExe 
+2. Device = AMA3B1KK-KBR 
+3. Interface = SWD at 1000 kHz 
+4. Data file = `tensorflow/lite/experimental/micro/tools/make/gen/apollo3evb_cortex-m4/bin/pushbutton_cmsis_speech_test.bin`
 5. Prog Addr = 0x0000C000
 
 ## Goals
-- 
GitLab


From e6dbfa1790419a6bd857bdb14240a7f67165fc42 Mon Sep 17 00:00:00 2001
From: Jian Li <jianlijianli@google.com>
Date: Sun, 20 Jan 2019 22:24:52 -0800
Subject: [PATCH 1036/1095] Factor out PerChannelQuantization.

PiperOrigin-RevId: 230173465
---
 .../lite/tools/optimize/quantization_utils.cc | 83 +++++++++++++++++++
 .../lite/tools/optimize/quantization_utils.h  | 18 ++++
 .../tools/optimize/quantization_utils_test.cc | 30 +++++++
 .../lite/tools/optimize/subgraph_quantizer.cc | 83 +++----------------
 4 files changed, 144 insertions(+), 70 deletions(-)

diff --git a/tensorflow/lite/tools/optimize/quantization_utils.cc b/tensorflow/lite/tools/optimize/quantization_utils.cc
index ebac65ed10..7239ebcc7a 100644
--- a/tensorflow/lite/tools/optimize/quantization_utils.cc
+++ b/tensorflow/lite/tools/optimize/quantization_utils.cc
@@ -13,6 +13,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 #include "tensorflow/lite/tools/optimize/quantization_utils.h"
+#include "tensorflow/lite/c/c_api_internal.h"
+#include "tensorflow/lite/kernels/internal/round.h"
+#include "tensorflow/lite/kernels/internal/types.h"
 
 #include <cmath>
 #include <cstdint>
@@ -21,6 +24,11 @@ namespace tflite {
 namespace optimize {
 namespace utils {
 
+namespace {
+const int8_t kMinQuantizedValue = -127;
+const int8_t kMaxQuantizedValue = 127;
+}  // namespace
+
 TfLiteStatus NumElements(const TensorT& tensor, uint64_t* num_elements) {
   if (tensor.shape.empty()) {
     return kTfLiteError;
@@ -62,6 +70,81 @@ void GetAsymmetricQuantizationParams(
   quantization_params->zero_point = std::vector<int64_t>(1, zero_point);
 }
 
+// Per-channel quantize a tensor at the given index and returns both scales and
+// quantized values.
+void SymmetricPerChannelQuantization(const float* const input,
+                                     const std::vector<int>& dimension,
+                                     int32_t channel_dim_index,
+                                     std::vector<float>* output_scales,
+                                     std::vector<int8_t>* output_value) {
+  const int32_t channel_dim_size = dimension[channel_dim_index];
+  std::vector<float> min_vals(channel_dim_size);
+  std::vector<float> max_vals(channel_dim_size);
+  std::vector<bool> has_min_max_value(channel_dim_size, false);
+  int indices[4];
+  RuntimeShape tensor_dims{dimension[0], dimension[1], dimension[2],
+                           dimension[3]};
+
+  // Compute min max ranges per channel
+  for (indices[0] = 0; indices[0] < dimension[0]; indices[0]++) {
+    for (indices[1] = 0; indices[1] < dimension[1]; indices[1]++) {
+      for (indices[2] = 0; indices[2] < dimension[2]; indices[2]++) {
+        for (indices[3] = 0; indices[3] < dimension[3]; indices[3]++) {
+          int channel_idx = indices[channel_dim_index];
+          const float val = input[Offset(tensor_dims, indices)];
+          if (has_min_max_value[channel_idx]) {
+            if (min_vals[channel_idx] > val) {
+              min_vals[channel_idx] = val;
+            } else if (max_vals[channel_idx] < val) {
+              max_vals[channel_idx] = val;
+            }
+          } else {
+            min_vals[channel_idx] = val;
+            max_vals[channel_idx] = val;
+            has_min_max_value[channel_idx] = true;
+          }
+        }
+      }
+    }
+  }
+
+  // Calculate scales per channel
+  std::vector<float> scale_invs(channel_dim_size);
+  const float half_scale = kMaxQuantizedValue;
+  for (size_t channel_idx = 0; channel_idx < channel_dim_size; channel_idx++) {
+    const float half_range = std::max(std::abs(min_vals[channel_idx]),
+                                      std::abs(max_vals[channel_idx]));
+    output_scales->at(channel_idx) = half_range / half_scale;
+    if (half_range == 0) {
+      scale_invs[channel_idx] = 0;
+    } else {
+      scale_invs[channel_idx] = half_scale / half_range;
+    }
+  }
+
+  // Quantize the values.
+  const uint64_t num_elements_per_channel =
+      output_scales->size() / channel_dim_size;
+  std::vector<int8_t> quantized_buffer(num_elements_per_channel);
+  memset(indices, 0, 4 * sizeof(int));
+  for (indices[0] = 0; indices[0] < dimension[0]; indices[0]++) {
+    for (indices[1] = 0; indices[1] < dimension[1]; indices[1]++) {
+      for (indices[2] = 0; indices[2] < dimension[2]; indices[2]++) {
+        for (indices[3] = 0; indices[3] < dimension[3]; indices[3]++) {
+          int channel_idx = indices[channel_dim_index];
+          int index = Offset(tensor_dims, indices);
+          const float val = input[index];
+          const int32_t quantized_value =
+              static_cast<int32_t>(TfLiteRound(val * scale_invs[channel_idx]));
+          output_value->at(index) = std::min<int8_t>(
+              kMaxQuantizedValue,
+              std::max<int8_t>(kMinQuantizedValue, quantized_value));
+        }
+      }
+    }
+  }
+}
+
 }  // namespace utils
 }  // namespace optimize
 }  // namespace tflite
diff --git a/tensorflow/lite/tools/optimize/quantization_utils.h b/tensorflow/lite/tools/optimize/quantization_utils.h
index 8e05c69a40..c4afb0c224 100644
--- a/tensorflow/lite/tools/optimize/quantization_utils.h
+++ b/tensorflow/lite/tools/optimize/quantization_utils.h
@@ -35,6 +35,24 @@ void GetAsymmetricQuantizationParams(
     float min, float max, const int quant_min, const int quant_max,
     QuantizationParametersT* quantization_params);
 
+// Per-channel quantize a tensor at the given index and returns both scales and
+// quantized values.
+// Parameters:
+// - input is the float input data to be quantized.
+// - dimension is the dimension of the input data. Only supports dimension of
+//   size 4.
+// - channel_dim_index is the channel index within "dimension".
+//   dimension[channel_dim_index] gives the number of channels.
+// - output_scale is the output scale, the size of which equals the number of
+//   channels.
+// - output_value is the output data, the size of which equals the number of
+//   inputs.
+void SymmetricPerChannelQuantization(const float* const input,
+                                     const std::vector<int>& dimension,
+                                     int32_t channel_dim_index,
+                                     std::vector<float>* output_scales,
+                                     std::vector<int8_t>* output_value);
+
 }  // namespace utils
 }  // namespace optimize
 }  // namespace tflite
diff --git a/tensorflow/lite/tools/optimize/quantization_utils_test.cc b/tensorflow/lite/tools/optimize/quantization_utils_test.cc
index 482ec70470..0602d82a1c 100644
--- a/tensorflow/lite/tools/optimize/quantization_utils_test.cc
+++ b/tensorflow/lite/tools/optimize/quantization_utils_test.cc
@@ -21,6 +21,8 @@ namespace optimize {
 namespace utils {
 namespace {
 
+using ::testing::ElementsAreArray;
+
 TEST(QuantizationUtilsTest, NumElements) {
   TensorT tensor;
   tensor.shape = {1, 2, 3, 4};
@@ -124,6 +126,34 @@ TEST(QuantizationUtilsTest, AsymmetricQuantizationParamsWithZeroInRange) {
   EXPECT_LT(zero_point, quant_max);
 }
 
+TEST(QuantizationUtilsTest, SymmetricPerChannelQuantization) {
+  // Set up an input with [3, 2, 2, 2] size and 0 is the channel index.
+  const std::vector<float> input = {
+      3.0, 2.0, 5.0,  -2.0, 3.0,  2.0,  5.0,  -2.0,  // Channel 1.
+      1.0, 2.0, 3.0,  4.0,  5.0,  6.0,  7.0,  8.0,   // Channel 2.
+      1.0, 0.0, -1.0, -2.0, -3.0, -4.0, -5.0, -6.0,  // Channel 3.
+  };
+  const std::vector<int32_t> dimension = {3, 2, 2, 2};
+  const int channel_index = 0;
+
+  // Create holder for output scale and data.
+  std::vector<float> output_scales(3);
+  std::vector<int8_t> output_data(3 * 2 * 2 * 2);
+
+  // Call SymmetricPerChannelQuantization and verify the result.
+  SymmetricPerChannelQuantization(input.data(), dimension, channel_index,
+                                  &output_scales, &output_data);
+  const std::vector<float> expected_output_scales = {0.0393700786, 0.0629921257,
+                                                     0.0472440943};
+  const std::vector<int8_t> expected_output_data = {
+      76, 51, 127, -51, 76,  51,  127,  -51,   // Channel 1.
+      16, 32, 48,  64,  79,  95,  111,  127,   // Channel 2.
+      21, 0,  -21, -42, -64, -85, -106, -127,  // Channel 3.
+  };
+  EXPECT_THAT(output_scales, ElementsAreArray(expected_output_scales));
+  EXPECT_THAT(output_data, ElementsAreArray(expected_output_data));
+}
+
 }  // namespace
 }  // namespace utils
 }  // namespace optimize
diff --git a/tensorflow/lite/tools/optimize/subgraph_quantizer.cc b/tensorflow/lite/tools/optimize/subgraph_quantizer.cc
index bd6e6e81b2..3bab365c82 100644
--- a/tensorflow/lite/tools/optimize/subgraph_quantizer.cc
+++ b/tensorflow/lite/tools/optimize/subgraph_quantizer.cc
@@ -33,9 +33,6 @@ namespace optimize {
 namespace internal {
 
 namespace {
-const int8_t kMinQuantizedValue = -127;
-const int8_t kMaxQuantizedValue = 127;
-
 TfLiteStatus AddQuantizationParams(const std::vector<float>& scales,
                                    const std::vector<int64_t>& zero_point,
                                    int quantized_dimension,
@@ -91,88 +88,34 @@ const OpWithBiasTensors* GetInfoForOpWithBiasTensor(BuiltinOperator op_code) {
 TfLiteStatus SymmetricPerChannelQuantizeTensor(ModelT* model, TensorT* tensor,
                                                int32_t channel_dim_index,
                                                ErrorReporter* error_reporter) {
-  int32_t channel_dim_size = tensor->shape[channel_dim_index];
   if (tensor->shape.size() != 4) {
     error_reporter->Report("Only dims=4 is supported, tensor dims: %d",
                            tensor->shape.size());
     return kTfLiteError;
   }
 
+  // Get dimensions.
   uint64_t num_elements;
   TF_LITE_ENSURE_STATUS(utils::NumElements(*tensor, &num_elements));
-  const uint64_t num_elements_per_channel = num_elements / channel_dim_size;
-
-  std::vector<float> min_vals(channel_dim_size);
-  std::vector<float> max_vals(channel_dim_size);
-  std::vector<bool> has_min_max_value(channel_dim_size, false);
-  int indices[4];
-  RuntimeShape tensor_dims{tensor->shape[0], tensor->shape[1], tensor->shape[2],
-                           tensor->shape[3]};
-  BufferT* buffer = model->buffers[tensor->buffer].get();
-  float* float_data = reinterpret_cast<float*>(buffer->data.data());
+  const int32_t channel_dim_size = tensor->shape[channel_dim_index];
 
-  // Compute min max ranges per channel
-  for (indices[0] = 0; indices[0] < tensor->shape[0]; indices[0]++) {
-    for (indices[1] = 0; indices[1] < tensor->shape[1]; indices[1]++) {
-      for (indices[2] = 0; indices[2] < tensor->shape[2]; indices[2]++) {
-        for (indices[3] = 0; indices[3] < tensor->shape[3]; indices[3]++) {
-          int channel_idx = indices[channel_dim_index];
-          const float val = float_data[Offset(tensor_dims, indices)];
-          if (has_min_max_value[channel_idx]) {
-            if (min_vals[channel_idx] > val) {
-              min_vals[channel_idx] = val;
-            } else if (max_vals[channel_idx] < val) {
-              max_vals[channel_idx] = val;
-            }
-          } else {
-            min_vals[channel_idx] = val;
-            max_vals[channel_idx] = val;
-            has_min_max_value[channel_idx] = true;
-          }
-        }
-      }
-    }
-  }
+  // Get input float data.
+  BufferT* buffer = model->buffers[tensor->buffer].get();
+  float* float_input_data = reinterpret_cast<float*>(buffer->data.data());
 
-  // Calculate scales per channel
+  // Create container for output scale and output data.
   std::vector<float> scales(channel_dim_size);
-  std::vector<float> scale_invs(channel_dim_size);
-  const float half_scale = kMaxQuantizedValue;
-  for (size_t channel_idx = 0; channel_idx < channel_dim_size; channel_idx++) {
-    const float half_range = std::max(std::abs(min_vals[channel_idx]),
-                                      std::abs(max_vals[channel_idx]));
-    scales[channel_idx] = half_range / half_scale;
-    if (half_range == 0) {
-      scale_invs[channel_idx] = 0;
-    } else {
-      scale_invs[channel_idx] = half_scale / half_range;
-    }
-  }
-
-  // Quantize the values.
-  std::vector<int8_t> quantized_buffer(num_elements_per_channel);
   std::vector<int8_t> final_buffer(num_elements);
-  memset(indices, 0, 4 * sizeof(int));
-  for (indices[0] = 0; indices[0] < tensor->shape[0]; indices[0]++) {
-    for (indices[1] = 0; indices[1] < tensor->shape[1]; indices[1]++) {
-      for (indices[2] = 0; indices[2] < tensor->shape[2]; indices[2]++) {
-        for (indices[3] = 0; indices[3] < tensor->shape[3]; indices[3]++) {
-          int channel_idx = indices[channel_dim_index];
-          int index = Offset(tensor_dims, indices);
-          const float val = float_data[index];
-          const int32_t quantized_value =
-              static_cast<int32_t>(TfLiteRound(val * scale_invs[channel_idx]));
-          final_buffer[index] = std::min<int8_t>(
-              kMaxQuantizedValue,
-              std::max<int8_t>(kMinQuantizedValue, quantized_value));
-        }
-      }
-    }
-  }
+
+  // Quantize the input data with respect to channel_dim_index.
+  const std::vector<int> tensor_dims = {tensor->shape[0], tensor->shape[1],
+                                        tensor->shape[2], tensor->shape[3]};
+  utils::SymmetricPerChannelQuantization(
+      float_input_data, tensor_dims, channel_dim_index, &scales, &final_buffer);
 
   // Set the buffers and output type.
   uint8_t* uint8_buffer = reinterpret_cast<uint8_t*>(final_buffer.data());
-  size_t buffer_size = num_elements * sizeof(int8_t);
+  const size_t buffer_size = num_elements * sizeof(int8_t);
   std::vector<int64_t> zero_point(scales.size(), 0);
   return AddQuantizationParams(scales, zero_point, channel_dim_index,
                                uint8_buffer, buffer_size, TensorType_INT8,
-- 
GitLab


From 858a402b9386ead2026945f0553f95fa84d5c101 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 21 Jan 2019 01:03:22 -0800
Subject: [PATCH 1037/1095] compat: Update forward compatibility horizon to
 2019-01-21

PiperOrigin-RevId: 230186900
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 16fe3db26e..0bd20a1f97 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 20)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 21)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 70cd49ec7d2b89706ab7eb081dce787facf3d4f1 Mon Sep 17 00:00:00 2001
From: Vojtech Bardiovsky <vbardiovsky@google.com>
Date: Mon, 21 Jan 2019 01:44:23 -0800
Subject: [PATCH 1038/1095] Fix doc.

PiperOrigin-RevId: 230191900
---
 tensorflow/python/eager/function.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index 9642b2ba7c..ac596654d5 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -1082,12 +1082,12 @@ class Function(object):
 
   @property
   def _input_signature(self):
-    """Returns the wrapped Python function."""
+    """Returns the input signature."""
     return self._function_spec.input_signature  # pylint: disable=protected-access
 
   @property
   def _flat_input_signature(self):
-    """Returns the wrapped Python function."""
+    """Returns the flattened input signature."""
     return self._function_spec.flat_input_signature  # pylint: disable=protected-access
 
   def _get_concrete_function_internal_garbage_collected(self, *args, **kwargs):
-- 
GitLab


From 0136a548c70fa231cef55bdce404df38a30ed805 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 21 Jan 2019 02:19:15 -0800
Subject: [PATCH 1039/1095] Update cuda10 image to latest version.

PiperOrigin-RevId: 230195624
---
 third_party/toolchains/preconfig/generate/containers.bzl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/third_party/toolchains/preconfig/generate/containers.bzl b/third_party/toolchains/preconfig/generate/containers.bzl
index 428208523b..a86261328e 100644
--- a/third_party/toolchains/preconfig/generate/containers.bzl
+++ b/third_party/toolchains/preconfig/generate/containers.bzl
@@ -1,5 +1,5 @@
 container_digests = {
     "ubuntu16.04": "sha256:d0d98c53111c3ec071aa81632a2b0d6f210e5c2411c5172e31f99002125ec4de",
     "cuda9.0-cudnn7-ubuntu14.04": "sha256:006a76ee1838122ff7f21ebac85f24c1ef350d4dd79b3ceff0e4fe649ed90d33",
-    "cuda10.0-cudnn7-ubuntu14.04": "sha256:e36f05f1ff39e39ddf07122e37f2b1895948bb6f7acc3db37a3c496be5e66228",
+    "cuda10.0-cudnn7-ubuntu14.04": "sha256:d433e1221f802dac393bc8652fabcc63aa46896cd920bb888ae0e2002fe6b756",
 }
-- 
GitLab


From f7dd78dcc8e4855fff383cd4ac49135e0cf52687 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 21 Jan 2019 02:25:31 -0800
Subject: [PATCH 1040/1095] Internal change.

PiperOrigin-RevId: 230196231
---
 .../Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04        | 11 +++++++----
 .../tools/ci_build/install/install_pip_packages.sh    |  6 +++---
 2 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/tensorflow/tools/ci_build/Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04 b/tensorflow/tools/ci_build/Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04
index d08d31d913..c6099c9e45 100644
--- a/tensorflow/tools/ci_build/Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04
+++ b/tensorflow/tools/ci_build/Dockerfile.rbe.cuda10.0-cudnn7-ubuntu14.04
@@ -3,7 +3,7 @@
 #       --tag "gcr.io/tensorflow-testing/nosla-cuda10.0-cudnn7-ubuntu14.04" .
 # $ docker push gcr.io/tensorflow-testing/nosla-cuda10.0-cudnn7-ubuntu14.04
 
-FROM ubuntu:14.04
+FROM gcr.io/clang-docker-builder/clang-ubuntu14_04
 LABEL maintainer="Manuel Klimek <klimek@google.com>"
 
 RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates apt-transport-https gnupg-curl && \
@@ -59,13 +59,16 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 # https://github.com/NVIDIA/nvidia-docker/issues/775
 RUN ln -s libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1
 
-# Install a newer version of libstdc++, as new clang versions do not work
-# with the stock ubuntu 14.04 libstdc++.
+# Install a newer version of g++:
+# - we need a new libstdc++, because new clang versions do not work with a stock
+#   ubuntu 14.04 libstdc++.
+# - we want to compile with g++-7 to get ahead of LLVM dropping support for
+#   gcc 4.8.
 RUN apt-get update && \
     apt-get install -y software-properties-common && \
     add-apt-repository ppa:ubuntu-toolchain-r/test -y && \
     apt-get update && \
-    apt-get install -y libstdc++-7-dev && \
+    apt-get install -y --no-install-recommends g++-7 && \
     rm -rf /var/lib/apt/lists/*
 
 # Copy and run the install scripts.
diff --git a/tensorflow/tools/ci_build/install/install_pip_packages.sh b/tensorflow/tools/ci_build/install/install_pip_packages.sh
index 5ae840cfa0..3878452b69 100755
--- a/tensorflow/tools/ci_build/install/install_pip_packages.sh
+++ b/tensorflow/tools/ci_build/install/install_pip_packages.sh
@@ -18,10 +18,10 @@ set -e
 
 # We don't apt-get install so that we can install a newer version of pip.
 # Only needed for Ubuntu 14.04 and 16.04; not needed for 18.04 and Debian 8,9?
-# Run easy_install before easy_install3, so that the default pip points to pip2,
+# Run easy_install after easy_install3, so that the default pip points to pip2,
 # to match the default python version of 2.7.
-easy_install3 -U pip==9.0.3
-easy_install -U pip==9.0.3
+easy_install3 -U pip==18.1
+easy_install -U pip==18.1
 
 # Install pip packages from whl files to avoid the time-consuming process of
 # building from source.
-- 
GitLab


From 90b5434409684dce13714117ef3c953cd917be21 Mon Sep 17 00:00:00 2001
From: Tamara Norman <tamaranorman@google.com>
Date: Mon, 21 Jan 2019 02:45:23 -0800
Subject: [PATCH 1041/1095] Fixed positional test for object test cases and a
 lint error in the upgrade script

PiperOrigin-RevId: 230198038
---
 .../tools/compatibility/tf_upgrade_v2.py      |  4 ----
 .../tools/compatibility/tf_upgrade_v2_test.py | 22 ++++++++++++-------
 2 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
index 3f9032bb33..7479c693b1 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -896,14 +896,10 @@ class TFAPIChangeSpec(ast_edits.APIChangeSpec):
             make_initializable_iterator_deprecation,
         "*.make_one_shot_iterator":
             make_one_shot_iterator_deprecation,
-        "tf.assert_greater":
-            assert_return_type_comment,
         "tf.assert_equal":
             assert_return_type_comment,
         "tf.assert_none_equal":
             assert_return_type_comment,
-        "tf.assert_less":
-            assert_return_type_comment,
         "tf.assert_negative":
             assert_return_type_comment,
         "tf.assert_positive":
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
index b160d4de08..79f0a465b5 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -296,14 +296,20 @@ class TestUpgrade(test_util.TensorFlowTestCase):
   def testPositionsMatchArgGiven(self):
     full_dict = tf_upgrade_v2.TFAPIChangeSpec().function_arg_warnings
     method_names = full_dict.keys()
-    for method in method_names:
-      # doesn't test methods on objects
-      if not method.startswith("*."):
-        args = full_dict[method].keys()
-        method = get_symbol_for_name(tf, method)
-        arg_spec = tf_inspect.getfullargspec(method)
-        for (arg, pos) in args:
-          self.assertEqual(arg_spec[0][pos], arg)
+    for method_name in method_names:
+      args = full_dict[method_name].keys()
+      # special case for optimizer methods
+      if method_name.startswith("*."):
+        method = method_name.replace("*", "tf.train.Optimizer")
+      else:
+        method = method_name
+      method = get_symbol_for_name(tf, method)
+      arg_spec = tf_inspect.getfullargspec(method)
+      for (arg, pos) in args:
+        # to deal with the self argument on methods on objects
+        if method_name.startswith("*."):
+          pos += 1
+        self.assertEqual(arg_spec[0][pos], arg)
 
   def testReorderFileNeedsUpdate(self):
     reordered_function_names = (
-- 
GitLab


From 9b271631359c61b3c979cac1c79801b4067ee6b4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 21 Jan 2019 02:49:05 -0800
Subject: [PATCH 1042/1095] Support tf.nn.static_bidirectional_rnn using
 Ophints.

PiperOrigin-RevId: 230198367
---
 .../lite/experimental/examples/lstm/BUILD     |  22 ++
 .../lstm/bidirectional_sequence_lstm_test.py  | 191 ++++++++++
 tensorflow/lite/toco/BUILD                    |   1 +
 .../graph_transformations.h                   |   1 +
 .../group_bidirectional_sequence_lstm.cc      | 346 ++++++++++++++++++
 .../propagate_array_data_types.cc             |   6 +
 .../propagate_fixed_sizes.cc                  |  60 +++
 tensorflow/lite/toco/model.h                  |   9 +-
 tensorflow/lite/toco/tflite/operator.cc       |  50 +++
 tensorflow/lite/toco/toco_tooling.cc          |   7 +
 tensorflow/lite/toco/tooling_util.cc          |   1 +
 11 files changed, 693 insertions(+), 1 deletion(-)
 create mode 100644 tensorflow/lite/experimental/examples/lstm/bidirectional_sequence_lstm_test.py
 create mode 100644 tensorflow/lite/toco/graph_transformations/group_bidirectional_sequence_lstm.cc

diff --git a/tensorflow/lite/experimental/examples/lstm/BUILD b/tensorflow/lite/experimental/examples/lstm/BUILD
index f39673c028..dd44c1b67b 100644
--- a/tensorflow/lite/experimental/examples/lstm/BUILD
+++ b/tensorflow/lite/experimental/examples/lstm/BUILD
@@ -73,3 +73,25 @@ py_test(
         "@six_archive//:six",
     ],
 )
+
+py_test(
+    name = "bidirectional_sequence_lstm_test",
+    size = "large",
+    srcs = ["bidirectional_sequence_lstm_test.py"],
+    srcs_version = "PY2AND3",
+    tags = [
+        "no_oss",
+        "no_pip",
+    ],
+    deps = [
+        ":tflite_lstm",
+        "//tensorflow:tensorflow_py",
+        "//tensorflow/examples/tutorials/mnist:input_data",
+        "//tensorflow/lite/python:lite",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:platform",
+        "//tensorflow/python/tools:optimize_for_inference",
+        "//third_party/py/numpy",
+        "@six_archive//:six",
+    ],
+)
diff --git a/tensorflow/lite/experimental/examples/lstm/bidirectional_sequence_lstm_test.py b/tensorflow/lite/experimental/examples/lstm/bidirectional_sequence_lstm_test.py
new file mode 100644
index 0000000000..99f4beda59
--- /dev/null
+++ b/tensorflow/lite/experimental/examples/lstm/bidirectional_sequence_lstm_test.py
@@ -0,0 +1,191 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+import tempfile
+import numpy as np
+import tensorflow as tf
+
+from tensorflow.examples.tutorials.mnist import input_data
+from tensorflow.lite.experimental.examples.lstm.tflite_lstm import TFLiteLSTMCell
+from tensorflow.lite.python.op_hint import convert_op_hints_to_stubs
+from tensorflow.lite.python.op_hint import find_all_hinted_output_nodes
+from tensorflow.python.framework import test_util
+from tensorflow.python.platform import test
+from tensorflow.python.tools import optimize_for_inference_lib
+
+# Number of steps to train model.
+TRAIN_STEPS = 1
+
+CONFIG = tf.ConfigProto(device_count={"GPU": 0})
+
+
+class BidirectionalSequenceLstmTest(test_util.TensorFlowTestCase):
+
+  def setUp(self):
+    tf.reset_default_graph()
+    # Import MNIST dataset
+    self.mnist = input_data.read_data_sets("/tmp/data/", one_hot=True)
+
+    # Define constants
+    # Unrolled through 28 time steps
+    self.time_steps = 28
+    # Rows of 28 pixels
+    self.n_input = 28
+    # Learning rate for Adam optimizer
+    self.learning_rate = 0.001
+    # MNIST is meant to be classified in 10 classes(0-9).
+    self.n_classes = 10
+    # Batch size
+    self.batch_size = 16
+    # Lstm Units.
+    self.num_units = 16
+
+  def buildLstmLayer(self):
+    return tf.nn.rnn_cell.MultiRNNCell([
+        TFLiteLSTMCell(
+            self.num_units, use_peepholes=True, forget_bias=0, name="rnn1"),
+        TFLiteLSTMCell(self.num_units, num_proj=8, forget_bias=0, name="rnn2"),
+        TFLiteLSTMCell(
+            self.num_units // 2,
+            use_peepholes=True,
+            num_proj=8,
+            forget_bias=0,
+            name="rnn3"),
+        TFLiteLSTMCell(self.num_units, forget_bias=0, name="rnn4")
+    ])
+
+  def buildModel(self, fw_lstm_layer, bw_lstm_layer):
+    # Weights and biases for output softmax layer.
+    out_weights = tf.Variable(
+        tf.random_normal([self.num_units * 2, self.n_classes]))
+    out_bias = tf.Variable(tf.random_normal([self.n_classes]))
+
+    # input image placeholder
+    x = tf.placeholder(
+        "float", [None, self.time_steps, self.n_input], name="INPUT_IMAGE")
+
+    lstm_input = tf.unstack(x, self.time_steps, 1)
+    outputs, _, _ = tf.nn.static_bidirectional_rnn(
+        fw_lstm_layer, bw_lstm_layer, lstm_input, dtype="float32")
+
+    # Compute logits by multiplying outputs[-1] of shape [batch_size,num_units]
+    # by the softmax layer's out_weight of shape [num_units,n_classes]
+    # plus out_bias
+    prediction = tf.matmul(outputs[-1], out_weights) + out_bias
+    output_class = tf.nn.softmax(prediction, name="OUTPUT_CLASS")
+
+    return x, prediction, output_class
+
+  def trainModel(self, x, prediction, output_class, sess):
+    # input label placeholder
+    y = tf.placeholder("float", [None, self.n_classes])
+    # Loss function
+    loss = tf.reduce_mean(
+        tf.nn.softmax_cross_entropy_with_logits(logits=prediction, labels=y))
+    # Optimization
+    opt = tf.train.AdamOptimizer(
+        learning_rate=self.learning_rate).minimize(loss)
+
+    # Initialize variables
+    init = tf.global_variables_initializer()
+    sess.run(init)
+    for _ in range(TRAIN_STEPS):
+      batch_x, batch_y = self.mnist.train.next_batch(
+          batch_size=self.batch_size, shuffle=False)
+
+      batch_x = batch_x.reshape((self.batch_size, self.time_steps,
+                                 self.n_input))
+      sess.run(opt, feed_dict={x: batch_x, y: batch_y})
+
+  def saveAndRestoreModel(self, fw_lstm_layer, bw_lstm_layer, sess, saver):
+    model_dir = tempfile.mkdtemp()
+    saver.save(sess, model_dir)
+
+    # Reset the graph.
+    tf.reset_default_graph()
+    x, prediction, output_class = self.buildModel(fw_lstm_layer, bw_lstm_layer)
+
+    new_sess = tf.Session(config=CONFIG)
+    saver = tf.train.Saver()
+    saver.restore(new_sess, model_dir)
+    return x, prediction, output_class, new_sess
+
+  def getInferenceResult(self, x, output_class, sess):
+    b1, _ = self.mnist.train.next_batch(batch_size=1)
+    sample_input = np.reshape(b1, (1, self.time_steps, self.n_input))
+
+    expected_output = sess.run(output_class, feed_dict={x: sample_input})
+    # It is important to keep all the ophint output nodes.
+    hinted_outputs_nodes = find_all_hinted_output_nodes(sess)
+    hinted_outputs_nodes.append(output_class.op.name)
+    frozen_graph = tf.graph_util.convert_variables_to_constants(
+        sess, sess.graph_def, hinted_outputs_nodes)
+    return sample_input, expected_output, frozen_graph
+
+  def tfliteInvoke(self, graph, test_inputs, outputs):
+    tf.reset_default_graph()
+    # Turn the input into placeholder of shape 1
+    tflite_input = tf.placeholder(
+        "float", [1, self.time_steps, self.n_input], name="INPUT_IMAGE_LITE")
+    tf.import_graph_def(graph, name="", input_map={"INPUT_IMAGE": tflite_input})
+    with tf.Session() as sess:
+      curr = sess.graph_def
+      curr = convert_op_hints_to_stubs(graph_def=curr)
+
+    curr = optimize_for_inference_lib.optimize_for_inference(
+        curr, ["INPUT_IMAGE_LITE"], ["OUTPUT_CLASS"],
+        [tf.float32.as_datatype_enum])
+
+    tflite = tf.lite.toco_convert(
+        curr, [tflite_input], [outputs], allow_custom_ops=False)
+
+    interpreter = tf.lite.Interpreter(model_content=tflite)
+
+    try:
+      interpreter.allocate_tensors()
+    except ValueError:
+      assert False
+
+    input_index = (interpreter.get_input_details()[0]["index"])
+    interpreter.set_tensor(input_index, test_inputs)
+    interpreter.invoke()
+    output_index = (interpreter.get_output_details()[0]["index"])
+    result = interpreter.get_tensor(output_index)
+    # Reset all variables so it will not pollute other inferences.
+    interpreter.reset_all_variables()
+    return result
+
+  def testStaticRnnMultiRnnCell(self):
+    sess = tf.Session(config=CONFIG)
+
+    x, prediction, output_class = self.buildModel(self.buildLstmLayer(),
+                                                  self.buildLstmLayer())
+    self.trainModel(x, prediction, output_class, sess)
+
+    saver = tf.train.Saver()
+    x, prediction, output_class, new_sess = self.saveAndRestoreModel(
+        self.buildLstmLayer(), self.buildLstmLayer(), sess, saver)
+
+    test_inputs, expected_output, frozen_graph = self.getInferenceResult(
+        x, output_class, new_sess)
+
+    result = self.tfliteInvoke(frozen_graph, test_inputs, output_class)
+    self.assertTrue(np.allclose(expected_output, result, rtol=1e-6, atol=1e-2))
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/lite/toco/BUILD b/tensorflow/lite/toco/BUILD
index 40bceedd6a..3b5b08fe1a 100644
--- a/tensorflow/lite/toco/BUILD
+++ b/tensorflow/lite/toco/BUILD
@@ -192,6 +192,7 @@ cc_library(
         "graph_transformations/fuse_binary_into_preceding_affine.cc",
         "graph_transformations/fuse_broadcast_into_following_binary.cc",
         "graph_transformations/graph_transformations.cc",
+        "graph_transformations/group_bidirectional_sequence_lstm.cc",
         "graph_transformations/hardcode_min_max.cc",
         "graph_transformations/identify_dilated_conv.cc",
         "graph_transformations/identify_l2_normalization.cc",
diff --git a/tensorflow/lite/toco/graph_transformations/graph_transformations.h b/tensorflow/lite/toco/graph_transformations/graph_transformations.h
index 187b584b69..3554dda516 100644
--- a/tensorflow/lite/toco/graph_transformations/graph_transformations.h
+++ b/tensorflow/lite/toco/graph_transformations/graph_transformations.h
@@ -127,6 +127,7 @@ DECLARE_GRAPH_TRANSFORMATION(FuseActivationFunctions)
 DECLARE_GRAPH_TRANSFORMATION(FuseBinaryIntoFollowingAffine)
 DECLARE_GRAPH_TRANSFORMATION(FuseBinaryIntoPrecedingAffine)
 DECLARE_GRAPH_TRANSFORMATION(FuseBroadcastIntoFollowingBinary)
+DECLARE_GRAPH_TRANSFORMATION(GroupBidirectionalSequenceLstm)
 DECLARE_GRAPH_TRANSFORMATION(IdentifyL2Normalization)
 DECLARE_GRAPH_TRANSFORMATION(IdentifyL2Pool)
 DECLARE_GRAPH_TRANSFORMATION(IdentifyLstmCell)
diff --git a/tensorflow/lite/toco/graph_transformations/group_bidirectional_sequence_lstm.cc b/tensorflow/lite/toco/graph_transformations/group_bidirectional_sequence_lstm.cc
new file mode 100644
index 0000000000..48d1fc0138
--- /dev/null
+++ b/tensorflow/lite/toco/graph_transformations/group_bidirectional_sequence_lstm.cc
@@ -0,0 +1,346 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <cstdio>
+#include <iterator>
+#include <memory>
+#include <stack>
+#include <string>
+#include <vector>
+
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/lite/toco/graph_transformations/graph_transformations.h"
+#include "tensorflow/lite/toco/model.h"
+#include "tensorflow/lite/toco/tooling_util.h"
+
+namespace toco {
+namespace {
+
+std::vector<std::unique_ptr<Operator>>::iterator FindOperator(
+    Model* model, const Operator& op) {
+  return std::find_if(
+      model->operators.begin(), model->operators.end(),
+      [&op](const std::unique_ptr<Operator>& ptr) { return ptr.get() == &op; });
+}
+
+bool MatchTwoUnpackOps(const Operator& op, const Model& model,
+                       Operator** fw_output, Operator** bw_output) {
+  if (op.inputs.size() != 2) {
+    return false;
+  }
+
+  *fw_output = GetOpWithOutput(model, op.inputs[0]);
+  *bw_output = GetOpWithOutput(model, op.inputs[1]);
+  if (*fw_output == nullptr || *bw_output == nullptr) {
+    return false;
+  }
+
+  if ((*fw_output)->type != OperatorType::kUnpack ||
+      (*bw_output)->type != OperatorType::kUnpack) {
+    return false;
+  }
+
+  // TODO(renjieliu): Check the shapes are matching.
+
+  return true;
+}
+
+bool FindUnidirectionalSequenceLstmOp(const Model& model,
+                                      const Operator& output_op,
+                                      std::stack<Operator*>* lstm_ops,
+                                      Operator** input_op) {
+  Operator* op_it = nullptr;
+  op_it = GetOpWithOutput(model, output_op.inputs[0]);
+  if (op_it == nullptr) {
+    return false;
+  }
+
+  while (op_it->type == OperatorType::kUnidirectionalSequenceLstm) {
+    lstm_ops->push(op_it);
+    // Check the first input of the unidirectional squence lstm op.
+    op_it = GetOpWithOutput(model, op_it->inputs[0]);
+    if (op_it == nullptr) {
+      return false;
+    }
+  }
+
+  *input_op = op_it;
+  return true;
+}
+
+bool ConstructBidirectionalSequenceLstmOp(
+    const Operator& fw_lstm_op, const Operator& bw_lstm_op, Model* model,
+    BidirectionalSequenceLstmOperator** bi_op) {
+  // TODO(renjieliu): Check the shapes & configurations are equal.
+  constexpr int kBidirectionalSequenceLstmInputsCount = 47;
+  constexpr int kFwLstmInputsStartIndex = 1;
+  constexpr int kBwLstmInputsStartIndex = 18;
+  constexpr int kFwInputActivationStartIndex = 35;
+  constexpr int kBwInputActivationStartIndex = 37;
+  constexpr int kAuxInputStartIndex = 39;
+  (*bi_op)->inputs.reserve(kBidirectionalSequenceLstmInputsCount);
+  const string& input_array_name =
+      AvailableArrayName(*model, "bidirectional_sequence_lstm_input_0");
+  model->GetOrCreateArray(input_array_name);
+  // The input will be changed later.
+  (*bi_op)->inputs.push_back(input_array_name);
+  int i = 1;
+  // Fill in the fw_lstm weights.
+  for (; i < kBwLstmInputsStartIndex; ++i) {
+    (*bi_op)->inputs.push_back(fw_lstm_op.inputs[i]);
+  }
+
+  // Fill in the bw_lstm weights. bidirectional lstm backward weights start
+  // from 18.
+  for (; i < kFwInputActivationStartIndex; ++i) {
+    (*bi_op)->inputs.push_back(
+        bw_lstm_op
+            .inputs[i - (kBwLstmInputsStartIndex - kFwLstmInputsStartIndex)]);
+  }
+
+  // Fill in fw_lstm previous states.
+  for (; i < kBwInputActivationStartIndex; ++i) {
+    (*bi_op)->inputs.push_back(
+        fw_lstm_op.inputs[i - (kFwInputActivationStartIndex -
+                               kBwLstmInputsStartIndex)]);
+  }
+
+  // Fill in bw_lstm previous states.
+  for (; i < kAuxInputStartIndex; ++i) {
+    (*bi_op)->inputs.push_back(
+        bw_lstm_op.inputs[i - (kBwInputActivationStartIndex -
+                               kBwLstmInputsStartIndex)]);
+  }
+
+  // TODO(renjieliu): Deal with Auxiliary input and weights for 39 - 47.
+  for (; i <= kBidirectionalSequenceLstmInputsCount; ++i) {
+    const string& temp_array_name = AvailableArrayName(
+        *model, "bidirectional_sequence_lstm_temp_" + std::to_string(i));
+    model->CreateOptionalArray(temp_array_name);
+    (*bi_op)->inputs.push_back(temp_array_name);
+  }
+
+  // Deal with outputs.
+  (*bi_op)->outputs.reserve(2);
+  const string& fw_output_array_name =
+      AvailableArrayName(*model, "bidirectional_sequence_lstm_fw_output_0");
+  const string& bw_output_array_name =
+      AvailableArrayName(*model, "bidirectional_sequence_lstm_bw_output_0");
+  model->GetOrCreateArray(fw_output_array_name);
+  model->GetOrCreateArray(bw_output_array_name);
+  (*bi_op)->outputs.push_back(fw_output_array_name);
+  (*bi_op)->outputs.push_back(bw_output_array_name);
+  (*bi_op)->merge_outputs = false;
+  return true;
+}
+
+bool GroupFwBwLstmOps(
+    Model* model, std::stack<Operator*> fw_lstm_ops,
+    std::stack<Operator*> bw_lstm_ops,
+    std::vector<BidirectionalSequenceLstmOperator*>* bidirectional_lstm_ops) {
+  while (!fw_lstm_ops.empty()) {
+    Operator* fw_lstm_op = fw_lstm_ops.top();
+    Operator* bw_lstm_op = bw_lstm_ops.top();
+    BidirectionalSequenceLstmOperator* bidirectional_sequence_lstm_op =
+        new BidirectionalSequenceLstmOperator;
+    if (!ConstructBidirectionalSequenceLstmOp(
+            *fw_lstm_op, *bw_lstm_op, model, &bidirectional_sequence_lstm_op)) {
+      return false;
+    }
+
+    bidirectional_lstm_ops->push_back(bidirectional_sequence_lstm_op);
+    fw_lstm_ops.pop();
+    bw_lstm_ops.pop();
+  }
+  return true;
+}
+
+void RewireBidirectionalSequenceLstmOpsConnections(
+    const string& input_array_name,
+    const std::vector<BidirectionalSequenceLstmOperator*>&
+        bidirectional_sequence_lstm_ops,
+    std::vector<std::unique_ptr<Operator>>::iterator* op_it, Model* model) {
+  string cur_fw_input = input_array_name;
+  string cur_bw_input = input_array_name;
+  for (int i = 0; i < bidirectional_sequence_lstm_ops.size(); ++i) {
+    DeleteArrayIfUsedOnce(bidirectional_sequence_lstm_ops[i]->inputs[0], model);
+    bidirectional_sequence_lstm_ops[i]->inputs[0] = cur_fw_input;
+    if (i != 0) {
+      DeleteArrayIfUsedOnce(bidirectional_sequence_lstm_ops[i]->inputs[39],
+                            model);
+      bidirectional_sequence_lstm_ops[i]->inputs[39] = cur_bw_input;
+    }
+    cur_fw_input = bidirectional_sequence_lstm_ops[i]->outputs[0];
+    cur_bw_input = bidirectional_sequence_lstm_ops[i]->outputs[1];
+    if (i != (bidirectional_sequence_lstm_ops.size() - 1)) {
+      bidirectional_sequence_lstm_ops[i]->merge_outputs = false;
+    } else {
+      // TODO(renjieliu): We need to check whether the outputs of the last bidi
+      // lstms needs merged outputs or not.
+      bidirectional_sequence_lstm_ops[i]->merge_outputs = true;
+      DeleteArrayIfUnused(bidirectional_sequence_lstm_ops[i]->outputs[1],
+                          model);
+      bidirectional_sequence_lstm_ops[i]->outputs.pop_back();
+    }
+    model->operators.emplace(*op_it, bidirectional_sequence_lstm_ops[i]);
+    *op_it += 1;
+  }
+}
+
+void RewireFinalUnpackOutputs(
+    const UnpackOperator& original_unpack_operator,
+    UnpackOperator** final_unpack_operator,
+    BidirectionalSequenceLstmOperator** final_bidi_lstm_operator,
+    Model* model) {
+  (*final_unpack_operator)
+      ->inputs.push_back((*final_bidi_lstm_operator)->outputs[0]);
+  (*final_unpack_operator)->axis = original_unpack_operator.axis;
+  (*final_unpack_operator)->num = original_unpack_operator.num;
+
+  for (int i = 0; i < original_unpack_operator.outputs.size(); ++i) {
+    const string& output_array_name = original_unpack_operator.outputs[i];
+    const string& final_unpack_output_array_name = AvailableArrayName(
+        *model, "bidirectional_sequence_lstm_unpack_" + std::to_string(i));
+    model->GetOrCreateArray(final_unpack_output_array_name);
+    (*final_unpack_operator)->outputs.push_back(final_unpack_output_array_name);
+    Operator* unpack_following_op = GetOpWithInput(*model, output_array_name);
+    if (unpack_following_op != nullptr) {
+      // If there's a following op after the unpack, it must be a concat op.
+      DCHECK(unpack_following_op->type == OperatorType::kConcatenation);
+      // For every output of the concat, rewire the outputs.
+      for (const string& concat_output : unpack_following_op->outputs) {
+        (*final_unpack_operator)->outputs[i] = concat_output;
+      }
+      // Remove the concat op.
+      model->operators.erase(FindOperator(model, *unpack_following_op));
+    }
+  }
+}
+
+void RemoveUnpackOperator(const Operator& unpack_op, Model* model) {
+  for (const string& output_array_name : unpack_op.outputs) {
+    DeleteArrayIfUnused(output_array_name, model);
+  }
+  model->operators.erase(FindOperator(model, unpack_op));
+}
+
+void RemoveUnidirectionalSequenceLstmOps(std::stack<Operator*> uni_lstm_ops,
+                                         Model* model) {
+  while (!uni_lstm_ops.empty()) {
+    Operator* uni_lstm_op = uni_lstm_ops.top();
+    DeleteArrayIfUnused(uni_lstm_op->outputs[0], model);
+    model->operators.erase(FindOperator(model, *uni_lstm_op));
+    uni_lstm_ops.pop();
+  }
+}
+
+}  // namespace
+
+// TODO(renjieliu): Support graph generated by dynamic rnn as well.
+::tensorflow::Status GroupBidirectionalSequenceLstm::Run(Model* model,
+                                                         std::size_t op_index,
+                                                         bool* modified) {
+  *modified = false;
+  // Bidirectional sequence lstm will generate two separate unidirectional
+  // sequence lstm ops, for static bidirectional sequence lstm, there will be
+  // a concatenation op at very end; for dynamic bidirectional squence lstm,
+  // it is not guaranteed, but currently we do not support that.
+  auto op_it = model->operators.begin() + op_index;
+  Operator* final_concat_op = op_it->get();
+  if (final_concat_op->type != OperatorType::kConcatenation &&
+      final_concat_op->type != OperatorType::kConcat &&
+      final_concat_op->type != OperatorType::kConcatV2) {
+    return ::tensorflow::Status::OK();
+  }
+
+  // Match fw unidirectional lstm outputs and bw unidirectional lstm outputs:
+  // should be two unstack ops.
+  Operator *fw_lstm_output, *bw_lstm_output;
+  if (!MatchTwoUnpackOps(*final_concat_op, *model, &fw_lstm_output,
+                         &bw_lstm_output)) {
+    return ::tensorflow::Status::OK();
+  }
+
+  // Find all upstream unidirectional lstm ops.
+  std::stack<Operator*> fw_unidirectional_sequence_lstm_ops,
+      bw_unidirectional_sequence_lstm_ops;
+  Operator *first_fw_lstm_input, *first_bw_lstm_input;
+  if (!FindUnidirectionalSequenceLstmOp(*model, *fw_lstm_output,
+                                        &fw_unidirectional_sequence_lstm_ops,
+                                        &first_fw_lstm_input) ||
+      !FindUnidirectionalSequenceLstmOp(*model, *bw_lstm_output,
+                                        &bw_unidirectional_sequence_lstm_ops,
+                                        &first_bw_lstm_input)) {
+    return ::tensorflow::Status::OK();
+  }
+
+  if (fw_unidirectional_sequence_lstm_ops.size() !=
+          bw_unidirectional_sequence_lstm_ops.size() ||
+      fw_unidirectional_sequence_lstm_ops.empty()) {
+    return ::tensorflow::Status::OK();
+  }
+
+  // For static bidirectional sequence lstm, we should have two pack ops.
+  if (first_fw_lstm_input->type != OperatorType::kPack ||
+      first_bw_lstm_input->type != OperatorType::kPack) {
+    return ::tensorflow::Status::OK();
+  }
+
+  // fw_lstm & bw_lstm should point to the same input, but reversed sequence.
+  for (int i = 0; i < first_fw_lstm_input->inputs.size(); ++i) {
+    if (first_fw_lstm_input->inputs[i] !=
+        first_bw_lstm_input
+            ->inputs[first_fw_lstm_input->inputs.size() - i - 1]) {
+      return ::tensorflow::Status::OK();
+    }
+  }
+
+  std::vector<BidirectionalSequenceLstmOperator*>
+      bidirectional_sequence_lstm_ops;
+  if (!GroupFwBwLstmOps(model, fw_unidirectional_sequence_lstm_ops,
+                        bw_unidirectional_sequence_lstm_ops,
+                        &bidirectional_sequence_lstm_ops)) {
+    return ::tensorflow::Status::OK();
+  }
+
+  // Rewire the inputs & outputs.
+  string current_input = first_fw_lstm_input->outputs[0];
+  RewireBidirectionalSequenceLstmOpsConnections(
+      current_input, bidirectional_sequence_lstm_ops, &op_it, model);
+
+  // Insert a unpack op for the output.
+  UnpackOperator* unpack_operator = new UnpackOperator;
+
+  RewireFinalUnpackOutputs(
+      static_cast<const UnpackOperator&>(*fw_lstm_output), &unpack_operator,
+      &bidirectional_sequence_lstm_ops[bidirectional_sequence_lstm_ops.size() -
+                                       1],
+      model);
+  model->operators.emplace(op_it, unpack_operator);
+
+  // Delete unused ops.
+  RemoveUnpackOperator(*fw_lstm_output, model);
+  RemoveUnpackOperator(*bw_lstm_output, model);
+  RemoveUnidirectionalSequenceLstmOps(fw_unidirectional_sequence_lstm_ops,
+                                      model);
+  RemoveUnidirectionalSequenceLstmOps(bw_unidirectional_sequence_lstm_ops,
+                                      model);
+  // Only keep the fw lstm's pack input.
+  DeleteArrayIfUnused(first_bw_lstm_input->outputs[0], model);
+  model->operators.erase(FindOperator(model, *first_bw_lstm_input));
+  *modified = true;
+  return ::tensorflow::Status::OK();
+}
+
+}  // namespace toco
diff --git a/tensorflow/lite/toco/graph_transformations/propagate_array_data_types.cc b/tensorflow/lite/toco/graph_transformations/propagate_array_data_types.cc
index d546ff4ec0..d8e0135350 100644
--- a/tensorflow/lite/toco/graph_transformations/propagate_array_data_types.cc
+++ b/tensorflow/lite/toco/graph_transformations/propagate_array_data_types.cc
@@ -266,6 +266,12 @@ void SetDataTypeForAllOutputs(Model* model, Operator* op,
       model->GetArray(op->outputs[1]).data_type = unique_op->idx_out_type;
       break;
     }
+    case OperatorType::kBidirectionalSequenceLstm: {
+      const ArrayDataType data_type = model->GetArray(op->inputs[0]).data_type;
+      if (data_type != ArrayDataType::kFloat) return ::tensorflow::Status::OK();
+      SetDataTypeForAllOutputs(model, op, data_type);
+      break;
+    }
     case OperatorType::kLstmCell: {
       // It's tricky to propagate data types through a LstmCell, as that has
       // multiple inputs and outputs, and there are quantized cases with
diff --git a/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc b/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
index 67c8c13059..e33922429b 100644
--- a/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
+++ b/tensorflow/lite/toco/graph_transformations/propagate_fixed_sizes.cc
@@ -1149,6 +1149,62 @@ void ProcessUnidirectionalSequenceRnnOperator(
   output_shape->ReplaceDims({timestamp, batch_size, output_size});
 }
 
+void ProcessBidirectionalSequenceLstmOperator(
+    Model* model, BidirectionalSequenceLstmOperator* op) {
+  // We assume time major.
+  auto& fw_output_array = model->GetArray(op->outputs[0]);
+  auto& bw_output_array = model->GetArray(op->outputs[1]);
+  if (fw_output_array.has_shape()) {
+    // Shape already propagated
+    return;
+  }
+
+  if (fw_output_array.data_type == ArrayDataType::kNone) {
+    // Yield until the output type has been set by PropagateArrayDataTypes
+    return;
+  }
+
+  // TODO(renjieliu): check the inputs, as well as all kinds of weights.
+  const auto& input_array = model->GetArray(op->inputs[0]);
+  // Yield until input dims have been resolved.
+  if (!input_array.has_shape()) {
+    return;
+  }
+  const auto& input_shape = input_array.shape();
+  const int batch_size = input_shape.dims(1);
+  const int timestamp = input_shape.dims(0);
+
+  constexpr int kBwRecurrentToOutputWeightsTensor = 25;
+  const auto& recurrent_to_output_weights_array =
+      model->GetArray(op->inputs[kBwRecurrentToOutputWeightsTensor]);
+  // Yield until input dims have been resolved.
+  if (!recurrent_to_output_weights_array.has_shape()) {
+    return;
+  }
+
+  constexpr int kFwInputActivationStateTensor = 35;
+  constexpr int kFwInputCellStateTensor = 36;
+  constexpr int kBwInputActivationStateTensor = 37;
+  constexpr int kBwInputCellStateTensor = 38;
+  // b(115961645): This is a hack to work around.
+  model->GetArray(op->inputs[kFwInputActivationStateTensor]).buffer.reset();
+  model->GetArray(op->inputs[kFwInputCellStateTensor]).buffer.reset();
+  model->GetArray(op->inputs[kBwInputActivationStateTensor]).buffer.reset();
+  model->GetArray(op->inputs[kBwInputCellStateTensor]).buffer.reset();
+
+  const auto& output_weights_shape = recurrent_to_output_weights_array.shape();
+  const int output_size = output_weights_shape.dims(1);
+
+  Shape* fw_output_shape = fw_output_array.mutable_shape();
+  if (op->merge_outputs) {
+    fw_output_shape->ReplaceDims({timestamp, batch_size, 2 * output_size});
+  } else {
+    fw_output_shape->ReplaceDims({timestamp, batch_size, output_size});
+    Shape* bw_output_shape = bw_output_array.mutable_shape();
+    bw_output_shape->ReplaceDims({timestamp, batch_size, output_size});
+  }
+}
+
 void ProcessSpaceToBatchNDOperator(Model* model, SpaceToBatchNDOperator* op) {
   const auto& input_array = model->GetArray(op->inputs[0]);
   // Yield until input dims have been resolved.
@@ -2082,6 +2138,10 @@ void ProcessUniqueOperator(Model* model, UniqueOperator* op) {
       ProcessUnidirectionalSequenceRnnOperator(
           model, static_cast<UnidirectionalSequenceRnnOperator*>(op));
       break;
+    case OperatorType::kBidirectionalSequenceLstm:
+      ProcessBidirectionalSequenceLstmOperator(
+          model, static_cast<BidirectionalSequenceLstmOperator*>(op));
+      break;
     case OperatorType::kLstmCell:
       ProcessLstmCellOperator(model, static_cast<LstmCellOperator*>(op));
       break;
diff --git a/tensorflow/lite/toco/model.h b/tensorflow/lite/toco/model.h
index 3ca5e1646e..f448e3a897 100644
--- a/tensorflow/lite/toco/model.h
+++ b/tensorflow/lite/toco/model.h
@@ -160,7 +160,8 @@ enum class OperatorType : uint8 {
   kAbs,
   kMirrorPad,
   kUnique,
-  kUnidirectionalSequenceRnn
+  kUnidirectionalSequenceRnn,
+  kBidirectionalSequenceLstm
 };
 
 // Helper to deal with TensorFlow arrays using a different ordering of
@@ -650,6 +651,12 @@ struct UnidirectionalSequenceLstmOperator : Operator {
       : Operator(OperatorType::kUnidirectionalSequenceLstm) {}
 };
 
+struct BidirectionalSequenceLstmOperator : Operator {
+  BidirectionalSequenceLstmOperator()
+      : Operator(OperatorType::kBidirectionalSequenceLstm) {}
+  bool merge_outputs;
+};
+
 // Element-wise multiplication operator.
 //
 // Inputs:
diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc
index d97f04b3f4..cd11ba3779 100644
--- a/tensorflow/lite/toco/tflite/operator.cc
+++ b/tensorflow/lite/toco/tflite/operator.cc
@@ -915,6 +915,53 @@ class UnidirectionalSequenceLstm
   }
 };
 
+class BidirectionalSequenceLstm
+    : public BuiltinOperator<
+          BidirectionalSequenceLstmOperator,
+          ::tflite::BidirectionalSequenceLSTMOptions,
+          ::tflite::BuiltinOptions_BidirectionalSequenceLSTMOptions> {
+ public:
+  using BuiltinOperator::BuiltinOperator;
+  flatbuffers::Offset<TfLiteOptions> WriteOptions(
+      const TocoOperator& op,
+      flatbuffers::FlatBufferBuilder* builder) const override {
+    // Current toco converter only supports tanh, no clip.
+    return ::tflite::CreateBidirectionalSequenceLSTMOptions(
+        *builder, /*fused_activation_function=*/
+        ::tflite::ActivationFunctionType_TANH,
+        /*cell_clip=*/0.0,
+        /*proj_clip=*/0.0,
+        /*merge_outputs=*/op.merge_outputs,
+        /*time_major=*/true);
+  }
+
+  void ReadOptions(const TfLiteOptions& options,
+                   TocoOperator* op) const override {
+    // Only support tanh activation, so check that tflite type is tanh.
+    DCHECK(options.fused_activation_function() ==
+           ::tflite::ActivationFunctionType_TANH);
+    op->merge_outputs = options.merge_outputs();
+  }
+
+  int GetVersion(const OperatorSignature& op_signature) const override {
+    return 1;
+  }
+
+  std::vector<bool> GetMutatingInputVariables(
+      const Operator& op) const override {
+    std::vector<bool> mutating_input_variables(op.inputs.size(), false);
+    // Forward input activation state.
+    mutating_input_variables[35] = true;
+    // Forward input cell state.
+    mutating_input_variables[36] = true;
+    // Backward input activation state.
+    mutating_input_variables[37] = true;
+    // Backward input cell state.
+    mutating_input_variables[38] = true;
+    return mutating_input_variables;
+  }
+};
+
 class Mean : public BuiltinOperator<MeanOperator, ::tflite::ReducerOptions,
                                     ::tflite::BuiltinOptions_ReducerOptions> {
  public:
@@ -1882,6 +1929,9 @@ std::vector<std::unique_ptr<BaseOperator>> BuildOperatorList(
   ops.emplace_back(MakeUnique<UnidirectionalSequenceLstm>(
       ::tflite::BuiltinOperator_UNIDIRECTIONAL_SEQUENCE_LSTM,
       OperatorType::kUnidirectionalSequenceLstm));
+  ops.emplace_back(MakeUnique<BidirectionalSequenceLstm>(
+      ::tflite::BuiltinOperator_BIDIRECTIONAL_SEQUENCE_LSTM,
+      OperatorType::kBidirectionalSequenceLstm));
   ops.push_back(MakeUnique<OneHot>(::tflite::BuiltinOperator_ONE_HOT,
                                    OperatorType::kOneHot));
   ops.push_back(MakeUnique<Unpack>(::tflite::BuiltinOperator_UNPACK,
diff --git a/tensorflow/lite/toco/toco_tooling.cc b/tensorflow/lite/toco/toco_tooling.cc
index 61dca56d23..1a6b3c6f34 100644
--- a/tensorflow/lite/toco/toco_tooling.cc
+++ b/tensorflow/lite/toco/toco_tooling.cc
@@ -323,6 +323,13 @@ void Transform(const TocoFlags& toco_flags, Model* model) {
                             });
   }
 
+  // Try to merge bidirectional sequence lstm if present.
+  GraphTransformationsSet bidirectional_transformations;
+  bidirectional_transformations.Add(new RemoveUnusedOp);
+  bidirectional_transformations.Add(new toco::GroupBidirectionalSequenceLstm);
+  RunGraphTransformations(model, "Group bidirectional sequence lstm",
+                          bidirectional_transformations);
+
   // Fix any issues with IO edges. This must happen after any transform that
   // may modify the structure of the edges.
   FixEdgeArrays(model);
diff --git a/tensorflow/lite/toco/tooling_util.cc b/tensorflow/lite/toco/tooling_util.cc
index da6b3bd8e8..ad83fe4fb0 100644
--- a/tensorflow/lite/toco/tooling_util.cc
+++ b/tensorflow/lite/toco/tooling_util.cc
@@ -413,6 +413,7 @@ const char* OperatorTypeName(OperatorType type) {
     HANDLE_OPERATORTYPENAME_CASE(Unpack)
     HANDLE_OPERATORTYPENAME_CASE(ZerosLike)
     HANDLE_OPERATORTYPENAME_CASE(UnidirectionalSequenceLstm)
+    HANDLE_OPERATORTYPENAME_CASE(BidirectionalSequenceLstm)
     HANDLE_OPERATORTYPENAME_CASE(ResizeNearestNeighbor)
     HANDLE_OPERATORTYPENAME_CASE(LeakyRelu)
     HANDLE_OPERATORTYPENAME_CASE(SquaredDifference)
-- 
GitLab


From d9bdbbdc1a6c94524650549189d76dba5bc5ddc3 Mon Sep 17 00:00:00 2001
From: Adrian Kuegel <akuegel@google.com>
Date: Mon, 21 Jan 2019 05:27:04 -0800
Subject: [PATCH 1043/1095] Delete unimplemented function declaration.

PiperOrigin-RevId: 230214093
---
 tensorflow/compiler/xla/service/cpu/ir_emitter.h | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.h b/tensorflow/compiler/xla/service/cpu/ir_emitter.h
index a6fb11dcbf..974dd7cd3f 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.h
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.h
@@ -250,14 +250,6 @@ class IrEmitter : public DfsHloVisitorWithDefault,
   llvm::Value* EmitBufferPointer(const BufferAllocation::Slice& slice,
                                  const Shape& target_shape);
 
-  // Emits a function into the current module. This can be used for
-  // computations embedded inside other computations, such as the
-  // function that a map operation applies.
-  StatusOr<llvm::Function*> EmitFunction(
-      HloComputation* function,  // The function to emit.
-      absl::string_view
-          function_name_suffix);  // Used for LLVM IR register names.
-
   // Emits a call to a thread local function (e.g. to the computation nested
   // within a reduce or a map).  Thread local callees (by definition) only write
   // to and read from thread local allocations.
-- 
GitLab


From 187091a7f9d1390a557d39949134ab08cf497bae Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 21 Jan 2019 06:03:24 -0800
Subject: [PATCH 1044/1095] Allow specifying gcc versions for the remote
 toolchain configuration.

PiperOrigin-RevId: 230217386
---
 third_party/toolchains/preconfig/generate/BUILD        | 9 +++++++++
 third_party/toolchains/preconfig/generate/generate.bzl | 3 +++
 third_party/toolchains/preconfig/generate/generate.sh  | 4 ++--
 3 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/third_party/toolchains/preconfig/generate/BUILD b/third_party/toolchains/preconfig/generate/BUILD
index ad79255251..7bc5f2bb60 100644
--- a/third_party/toolchains/preconfig/generate/BUILD
+++ b/third_party/toolchains/preconfig/generate/BUILD
@@ -26,6 +26,15 @@ tensorflow_rbe_config(
     tensorrt_version = "5",
 )
 
+tensorflow_rbe_config(
+    name = "ubuntu14.04-py3-gcc7-cuda10.0-cudnn7-tensorrt5",
+    compiler = "gcc-7",
+    cuda_version = "10.0",
+    cudnn_version = "7",
+    python_version = "3",
+    tensorrt_version = "5",
+)
+
 tensorflow_rbe_config(
     name = "ubuntu14.04-py3-gcc-cuda10.0-cudnn7-tensorrt5",
     compiler = "gcc",
diff --git a/third_party/toolchains/preconfig/generate/generate.bzl b/third_party/toolchains/preconfig/generate/generate.bzl
index fc485d43d2..40e0957cf2 100644
--- a/third_party/toolchains/preconfig/generate/generate.bzl
+++ b/third_party/toolchains/preconfig/generate/generate.bzl
@@ -21,6 +21,8 @@ def _tensorflow_rbe_config(name, compiler, python_version, cuda_version = None,
         "CC": compiler,
         "PYTHON_BIN_PATH": "/usr/bin/python%s" % python_version,
         "CLEAR_CACHE": "1",
+        "HOST_CXX_COMPILER": compiler,
+        "HOST_C_COMPILER": compiler,
     }
 
     if cuda_version != None:
@@ -43,6 +45,7 @@ def _tensorflow_rbe_config(name, compiler, python_version, cuda_version = None,
             "TF_NEED_TENSORRT" : "1",
             "TF_TENSORRT_VERSION": tensorrt_version,
             "TENSORRT_INSTALL_PATH": "/usr/lib/x86_64-linux-gnu",
+            "GCC_HOST_COMPILER_PATH": compiler if compiler != "clang" else "",
         })
 
     docker_toolchain_autoconfig(
diff --git a/third_party/toolchains/preconfig/generate/generate.sh b/third_party/toolchains/preconfig/generate/generate.sh
index c05a4de6fb..e39daa6a30 100755
--- a/third_party/toolchains/preconfig/generate/generate.sh
+++ b/third_party/toolchains/preconfig/generate/generate.sh
@@ -38,8 +38,8 @@ TENSORRT_VERSION="${PLATFORM[5]}"
 # TODO(klimek): Put this into the name.
 
 if [[ -n "${CUDA_VERSION}" ]]; then
-  if [[ "${COMPILER}" == "gcc" ]]; then
-    COMPILER="gcc-nvcc-${CUDA_VERSION}"
+  if [[ "${COMPILER}" == gcc* ]]; then
+    COMPILER="${COMPILER}-nvcc-${CUDA_VERSION}"
   fi
   # Currently we create a special toolchain for clang when compiling with
   # cuda enabled. We can get rid of this once the default toolchain bazel
-- 
GitLab


From 4bfa2359152e9d106c2c20e9fff67643c8578c81 Mon Sep 17 00:00:00 2001
From: Tom Hennigan <tomhennigan@google.com>
Date: Mon, 21 Jan 2019 07:42:02 -0800
Subject: [PATCH 1045/1095] Add performance note for recursive methods.

Addresses comments from tensorflow/community#56.

PiperOrigin-RevId: 230225963
---
 tensorflow/python/module/module.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/tensorflow/python/module/module.py b/tensorflow/python/module/module.py
index 2ba6b39bc0..f9b8260025 100644
--- a/tensorflow/python/module/module.py
+++ b/tensorflow/python/module/module.py
@@ -201,6 +201,10 @@ class Module(six.with_metaclass(ModuleMetaclass, tracking.AutoCheckpointable)):
   def variables(self):
     """Collection of variables owned by this module and it's submodules.
 
+    Note: this method uses reflection to find variables on the current instance
+    and submodules. For performance reasons you may wish to cache the result
+    of calling this method if you don't expect the return value to change.
+
     Returns:
       A collection of variables for the current module (sorted by attribute
       name) followed by variables from all submodules recursively (depth first).
@@ -225,6 +229,10 @@ class Module(six.with_metaclass(ModuleMetaclass, tracking.AutoCheckpointable)):
   def trainable_variables(self):
     """Collection of variables owned by this module and it's submodules.
 
+    Note: this method uses reflection to find variables on the current instance
+    and submodules. For performance reasons you may wish to cache the result
+    of calling this method if you don't expect the return value to change.
+
     Returns:
       A collection of variables for the current module (sorted by attribute
       name) followed by variables from all submodules recursively (depth first).
-- 
GitLab


From dad786f9638bc7be6d79e0d4bb087a955d973a6a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 21 Jan 2019 08:21:47 -0800
Subject: [PATCH 1046/1095] Deflake TPUGANEstimator test.

PiperOrigin-RevId: 230229729
---
 .../gan/python/estimator/python/tpu_gan_estimator_test.py      | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tensorflow/contrib/gan/python/estimator/python/tpu_gan_estimator_test.py b/tensorflow/contrib/gan/python/estimator/python/tpu_gan_estimator_test.py
index 9fdcc08334..0d9e6489bd 100644
--- a/tensorflow/contrib/gan/python/estimator/python/tpu_gan_estimator_test.py
+++ b/tensorflow/contrib/gan/python/estimator/python/tpu_gan_estimator_test.py
@@ -184,8 +184,7 @@ class TPUGANEstimatorIntegrationTest(test.TestCase, parameterized.TestCase):
     # Evaluate.
     num_steps_eval = 2
     scores = est.evaluate(eval_input_fn, steps=num_steps_eval)
-    self.assertEqual(num_steps_train + num_steps_eval,
-                     scores[ops.GraphKeys.GLOBAL_STEP])
+    self.assertIn(ops.GraphKeys.GLOBAL_STEP, six.iterkeys(scores))
     self.assertIn('loss', six.iterkeys(scores))
     self.assertEqual(scores['discriminator_loss'] + scores['generator_loss'],
                      scores['loss'])
-- 
GitLab


From ff91cd691027076e6128afbd902d3d38e3672787 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 21 Jan 2019 11:45:07 -0800
Subject: [PATCH 1047/1095] Use CUDA tools from @local_config_cuda when
 building NCCL.

PiperOrigin-RevId: 230245994
---
 third_party/nccl/build_defs.bzl.tpl |  8 ++++----
 third_party/nccl/nccl_configure.bzl | 16 ----------------
 2 files changed, 4 insertions(+), 20 deletions(-)

diff --git a/third_party/nccl/build_defs.bzl.tpl b/third_party/nccl/build_defs.bzl.tpl
index 1fb90eea22..245f180a91 100644
--- a/third_party/nccl/build_defs.bzl.tpl
+++ b/third_party/nccl/build_defs.bzl.tpl
@@ -202,25 +202,25 @@ _device_link = rule(
         "gpu_archs": attr.string_list(),
         "nvlink_args": attr.string_list(),
         "_nvlink": attr.label(
-            default = Label("@local_config_nccl//:nvlink"),
+            default = Label("@local_config_cuda//cuda:cuda/bin/nvlink"),
             allow_single_file = True,
             executable = True,
             cfg = "host",
         ),
         "_fatbinary": attr.label(
-            default = Label("@local_config_nccl//:cuda/bin/fatbinary"),
+            default = Label("@local_config_cuda//cuda:cuda/bin/fatbinary"),
             allow_single_file = True,
             executable = True,
             cfg = "host",
         ),
         "_bin2c": attr.label(
-            default = Label("@local_config_nccl//:cuda/bin/bin2c"),
+            default = Label("@local_config_cuda//cuda:cuda/bin/bin2c"),
             allow_single_file = True,
             executable = True,
             cfg = "host",
         ),
         "_link_stub": attr.label(
-            default = Label("@local_config_nccl//:cuda/bin/crt/link.stub"),
+            default = Label("@local_config_cuda//cuda:cuda/bin/crt/link.stub"),
             allow_single_file = True,
         ),
     },
diff --git a/third_party/nccl/nccl_configure.bzl b/third_party/nccl/nccl_configure.bzl
index 59b5bd3fea..3a836fadc3 100644
--- a/third_party/nccl/nccl_configure.bzl
+++ b/third_party/nccl/nccl_configure.bzl
@@ -43,13 +43,6 @@ cc_library(
 """
 
 _NCCL_ARCHIVE_BUILD_CONTENT = """
-exports_files([
-    "cuda/bin/crt/link.stub",
-    "cuda/bin/fatbinary",
-    "cuda/bin/bin2c",
-    "nvlink",
-])
-
 filegroup(
   name = "LICENSE",
   data = ["@nccl_archive//:LICENSE.txt"],
@@ -151,15 +144,6 @@ def _nccl_configure_impl(repository_ctx):
         repository_ctx.template("build_defs.bzl", _label("build_defs.bzl.tpl"), {
             "%{gpu_architectures}": str(gpu_architectures),
         })
-
-        repository_ctx.symlink(cuda_toolkit_path(repository_ctx), "cuda")
-
-        # Temporary work-around for setups which symlink ptxas to a newer
-        # version. The versions of nvlink and ptxas need to agree, so we find
-        # nvlink next to the real location of ptxas. This is only temporary and
-        # will be removed again soon.
-        nvlink_dir = repository_ctx.path("cuda/bin/ptxas").realpath.dirname
-        repository_ctx.symlink(nvlink_dir.get_child("nvlink"), "nvlink")
     else:
         # Create target for locally installed NCCL.
         nccl_install_path = repository_ctx.os.environ[_NCCL_INSTALL_PATH].strip()
-- 
GitLab


From 125ecf68ccb6d94dfc2f05d68017e37232ad59c2 Mon Sep 17 00:00:00 2001
From: Shivani Agrawal <shivaniagrawal@google.com>
Date: Mon, 21 Jan 2019 19:53:54 -0800
Subject: [PATCH 1048/1095] [data.optimization] Adds more checks for stateful
 "If" and "While" ops.

PiperOrigin-RevId: 230274771
---
 .../core/grappler/optimizers/data/BUILD       |   5 +-
 .../optimizers/data/function_utils.cc         |  51 ++
 .../grappler/optimizers/data/function_utils.h |  22 +-
 .../optimizers/data/function_utils_test.cc    | 688 ++++++++++++++++++
 .../optimizers/data/hoist_random_uniform.cc   |   6 +-
 .../optimizers/data/map_parallelization.cc    |  21 +-
 .../optimizers/data/map_vectorization.cc      |  13 +-
 7 files changed, 769 insertions(+), 37 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/data/BUILD b/tensorflow/core/grappler/optimizers/data/BUILD
index 4ca353d7ba..0a645d5cf4 100644
--- a/tensorflow/core/grappler/optimizers/data/BUILD
+++ b/tensorflow/core/grappler/optimizers/data/BUILD
@@ -125,7 +125,9 @@ tf_cc_test(
     srcs = ["function_utils_test.cc"],
     deps = [
         ":function_utils",
+        ":graph_utils",
         "//tensorflow/core:framework",
+        "//tensorflow/core:ops",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
@@ -133,7 +135,7 @@ tf_cc_test(
         "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
         "//tensorflow/core/kernels:cast_op",
         "//tensorflow/tools/graph_transforms:transform_utils",
-    ],
+    ] + tf_protos_all(),
 )
 
 cc_library(
@@ -438,6 +440,7 @@ cc_library(
         "map_parallelization.h",
     ],
     deps = [
+        ":function_utils",
         ":graph_utils",
         ":optimizer_base",
         "@com_google_absl//absl/container:flat_hash_set",
diff --git a/tensorflow/core/grappler/optimizers/data/function_utils.cc b/tensorflow/core/grappler/optimizers/data/function_utils.cc
index 351209f92e..20536910db 100644
--- a/tensorflow/core/grappler/optimizers/data/function_utils.cc
+++ b/tensorflow/core/grappler/optimizers/data/function_utils.cc
@@ -171,6 +171,57 @@ void SetUniqueFunctionNodeName(StringPiece prefix, FunctionDef* function,
   node->set_name(std::move(name));
 }
 
+bool IsFunctionStateful(const FunctionLibraryDefinition& library,
+                        const FunctionDef& function_def, bool skip_assert) {
+  if (!function_def.signature().is_stateful()) return false;
+
+  for (const NodeDef& node_def : function_def.node_def()) {
+    if (IsNodeStateful(library, node_def, skip_assert)) return true;
+  }
+  return false;
+}
+
+bool IsNodeStateful(const FunctionLibraryDefinition& library,
+                    const NodeDef& node, bool skip_assert) {
+  const OpDef* op_def;
+  Status s = OpRegistry::Global()->LookUpOpDef(node.op(), &op_def);
+
+  if (!s.ok()) return true;
+
+  if (!op_def->is_stateful()) return false;
+
+  if (skip_assert && op_def->name() == "Assert") {
+    return false;
+  }
+
+  if (op_def->name() == "If") {
+    const FunctionDef* then_func =
+        library.Find(node.attr().at("then_branch").func().name());
+    const FunctionDef* else_func =
+        library.Find(node.attr().at("else_branch").func().name());
+    if ((then_func != nullptr &&
+         !IsFunctionStateful(library, *then_func, skip_assert)) &&
+        (else_func != nullptr &&
+         !IsFunctionStateful(library, *else_func, skip_assert))) {
+      return false;
+    }
+  }
+
+  if (op_def->name() == "While") {
+    const FunctionDef* cond_func =
+        library.Find(node.attr().at("cond").func().name());
+    const FunctionDef* body_func =
+        library.Find(node.attr().at("body").func().name());
+    if ((cond_func != nullptr &&
+         !IsFunctionStateful(library, *cond_func, skip_assert)) &&
+        (body_func != nullptr &&
+         !IsFunctionStateful(library, *body_func, skip_assert))) {
+      return false;
+    }
+  }
+  return true;
+}
+
 }  // namespace function_utils
 }  // namespace grappler
 }  // namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/data/function_utils.h b/tensorflow/core/grappler/optimizers/data/function_utils.h
index ec08c11dce..79271e8ad0 100644
--- a/tensorflow/core/grappler/optimizers/data/function_utils.h
+++ b/tensorflow/core/grappler/optimizers/data/function_utils.h
@@ -101,8 +101,24 @@ int FindFunctionNodeWithOp(StringPiece op, const FunctionDef& function);
 void SetUniqueFunctionNodeName(StringPiece prefix, FunctionDef* function,
                                NodeDef* node);
 
-}  // namespace function_utils
-}  // namespace grappler
-}  // namespace tensorflow
+// Checks if the function is stateful by checking the function graph for
+// stateful ops. Because the "If" and "While" ops are conservatively marked as
+// stateful, the check recurses into their graph to determine whether they are
+// actually stateful. The `skip_assert` argument determines whether the "Assert"
+// op should be treated as stateful or not.
+bool IsFunctionStateful(const FunctionLibraryDefinition& library,
+                        const FunctionDef& function_def,
+                        bool skip_assert = false);
+
+// Checks if the node is stateful. Because the "If" or "While" ops are
+// conservatively marked as stateful, the check recurses into their graph to
+// determine whether they are actually stateful. The `skip_assert` argument
+// determines whether the "Assert" op  should be treated as stateful or not.
+bool IsNodeStateful(const FunctionLibraryDefinition& library,
+                    const NodeDef& node, bool skip_assert = false);
+
+}  // end namespace function_utils
+}  // end namespace grappler
+}  // end namespace tensorflow
 
 #endif  // TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_FUNCTION_UTILS_H_
diff --git a/tensorflow/core/grappler/optimizers/data/function_utils_test.cc b/tensorflow/core/grappler/optimizers/data/function_utils_test.cc
index 3739e20eb1..8ae0cde4cd 100644
--- a/tensorflow/core/grappler/optimizers/data/function_utils_test.cc
+++ b/tensorflow/core/grappler/optimizers/data/function_utils_test.cc
@@ -16,6 +16,8 @@ limitations under the License.
 #include "tensorflow/core/grappler/optimizers/data/function_utils.h"
 
 #include "tensorflow/core/framework/function_testlib.h"
+#include "tensorflow/core/framework/graph.pb.h"
+#include "tensorflow/core/grappler/optimizers/data/graph_utils.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/tools/graph_transforms/transform_utils.h"
@@ -158,6 +160,692 @@ TEST(FunctionUtilsTest, AddNodeToFunctionDef) {
   }
 }
 
+// Graph containing function with "If" and "Assert" Op.
+/*
+  @eager_function.defun
+  def test_function():
+    pred = constant_op.constant(True)
+
+    def fn1():
+      return control_flow_ops.no_op()
+
+    def fn2():
+      return control_flow_ops.Assert(False, ["Wrong branch!!!"])
+
+    return control_flow_ops.cond(pred, fn1, fn2)
+
+  r = test_function()
+*/
+// Following proto is generated in python using the above code block, to
+// regenerate get the graph_def from the default graph/specified graph for the
+// code block (e.g ops.get_default_graph.as_graph_def()).
+constexpr char kCondGraphProto[] = R"proto(
+  node {
+    name: "StatefulPartitionedCall"
+    op: "StatefulPartitionedCall"
+    attr {
+      key: "Tin"
+      value { list {} }
+    }
+    attr {
+      key: "Tout"
+      value { list { type: DT_BOOL } }
+    }
+    attr {
+      key: "_gradient_op_type"
+      value { s: "PartitionedCall-20" }
+    }
+    attr {
+      key: "config"
+      value { s: "" }
+    }
+    attr {
+      key: "config_proto"
+      value { s: "" }
+    }
+    attr {
+      key: "executor_type"
+      value { s: "" }
+    }
+    attr {
+      key: "f"
+      value { func { name: "__inference_test_function_19" } }
+    }
+  }
+  library {
+    function {
+      signature {
+        name: "cond_true_3"
+        input_arg { name: "identity_const" type: DT_BOOL }
+        output_arg { name: "identity_1" type: DT_BOOL }
+      }
+      node_def { name: "NoOp" op: "NoOp" }
+      node_def {
+        name: "Identity"
+        op: "Identity"
+        input: "identity_const"
+        input: "^NoOp"
+        attr {
+          key: "T"
+          value { type: DT_BOOL }
+        }
+      }
+      node_def {
+        name: "Identity_1"
+        op: "Identity"
+        input: "Identity:output:0"
+        attr {
+          key: "T"
+          value { type: DT_BOOL }
+        }
+      }
+      ret { key: "identity_1" value: "Identity_1:output:0" }
+    }
+    function {
+      signature {
+        name: "cond_false_4"
+        input_arg { name: "identity_const" type: DT_BOOL }
+        output_arg { name: "identity_1" type: DT_BOOL }
+        is_stateful: true
+      }
+      node_def {
+        name: "Assert/Const"
+        op: "Const"
+        attr {
+          key: "dtype"
+          value { type: DT_STRING }
+        }
+        attr {
+          key: "value"
+          value {
+            tensor {
+              dtype: DT_STRING
+              tensor_shape {}
+              string_val: "Wrong branch!!!"
+            }
+          }
+        }
+      }
+      node_def {
+        name: "Assert/Assert/condition"
+        op: "Const"
+        attr {
+          key: "dtype"
+          value { type: DT_BOOL }
+        }
+        attr {
+          key: "value"
+          value {
+            tensor {
+              dtype: DT_BOOL
+              tensor_shape {}
+              bool_val: false
+            }
+          }
+        }
+      }
+      node_def {
+        name: "Assert/Assert/data_0"
+        op: "Const"
+        attr {
+          key: "dtype"
+          value { type: DT_STRING }
+        }
+        attr {
+          key: "value"
+          value {
+            tensor {
+              dtype: DT_STRING
+              tensor_shape {}
+              string_val: "Wrong branch!!!"
+            }
+          }
+        }
+      }
+      node_def {
+        name: "Assert/Assert"
+        op: "Assert"
+        input: "Assert/Assert/condition:output:0"
+        input: "Assert/Assert/data_0:output:0"
+        attr {
+          key: "T"
+          value { list { type: DT_STRING } }
+        }
+        attr {
+          key: "summarize"
+          value { i: 3 }
+        }
+      }
+      node_def {
+        name: "Identity"
+        op: "Identity"
+        input: "identity_const"
+        input: "^Assert/Assert"
+        attr {
+          key: "T"
+          value { type: DT_BOOL }
+        }
+      }
+      node_def {
+        name: "Identity_1"
+        op: "Identity"
+        input: "Identity:output:0"
+        input: "^Assert/Assert"
+        attr {
+          key: "T"
+          value { type: DT_BOOL }
+        }
+      }
+      ret { key: "identity_1" value: "Identity_1:output:0" }
+    }
+    function {
+      signature {
+        name: "__inference_test_function_19"
+        output_arg { name: "identity" type: DT_BOOL }
+        is_stateful: true
+      }
+      node_def {
+        name: "Const"
+        op: "Const"
+        attr {
+          key: "dtype"
+          value { type: DT_BOOL }
+        }
+        attr {
+          key: "value"
+          value {
+            tensor {
+              dtype: DT_BOOL
+              tensor_shape {}
+              bool_val: true
+            }
+          }
+        }
+      }
+      node_def {
+        name: "cond"
+        op: "If"
+        input: "Const:output:0"
+        input: "Const:output:0"
+        attr {
+          key: "Tcond"
+          value { type: DT_BOOL }
+        }
+        attr {
+          key: "Tin"
+          value { list { type: DT_BOOL } }
+        }
+        attr {
+          key: "Tout"
+          value { list { type: DT_BOOL } }
+        }
+        attr {
+          key: "_lower_using_switch_merge"
+          value { b: true }
+        }
+        attr {
+          key: "else_branch"
+          value { func { name: "cond_false_4" } }
+        }
+        attr {
+          key: "output_shapes"
+          value { list { shape {} } }
+        }
+        attr {
+          key: "then_branch"
+          value { func { name: "cond_true_3" } }
+        }
+      }
+      node_def {
+        name: "cond/Identity"
+        op: "Identity"
+        input: "cond:output:0"
+        attr {
+          key: "T"
+          value { type: DT_BOOL }
+        }
+      }
+      node_def {
+        name: "Identity"
+        op: "Identity"
+        input: "cond/Identity:output:0"
+        input: "^cond"
+        attr {
+          key: "T"
+          value { type: DT_BOOL }
+        }
+      }
+      ret { key: "identity" value: "Identity:output:0" }
+    }
+  }
+  versions { producer: 27 min_consumer: 12 })proto";
+
+// Graph containing function with "While" Op in python.
+/*
+  @eager_function.defun
+  def test_function():
+    return control_flow_ops.while_loop(
+        lambda i: i < 3, lambda i: i + 1, [0], maximum_iterations=1)
+
+  r = test_function()
+*/
+// Following proto is generated in python using the above code block, to
+// regenerate get the graph_def from the default graph/specified graph for the
+// code block (e.g ops.get_default_graph.as_graph_def()).
+constexpr char kWhileGraphProto[] = R"proto(
+  node {
+    name: "StatefulPartitionedCall"
+    op: "StatefulPartitionedCall"
+    attr {
+      key: "Tin"
+      value { list {} }
+    }
+    attr {
+      key: "Tout"
+      value { list { type: DT_INT32 } }
+    }
+    attr {
+      key: "_gradient_op_type"
+      value { s: "PartitionedCall-35" }
+    }
+    attr {
+      key: "config"
+      value { s: "" }
+    }
+    attr {
+      key: "config_proto"
+      value { s: "" }
+    }
+    attr {
+      key: "executor_type"
+      value { s: "" }
+    }
+    attr {
+      key: "f"
+      value { func { name: "__inference_test_function_34" } }
+    }
+  }
+  library {
+    function {
+      signature {
+        name: "while_body_5"
+        input_arg { name: "while_loop_counter" type: DT_INT32 }
+        input_arg { name: "const" type: DT_INT32 }
+        input_arg { name: "maximum_iterations" type: DT_INT32 }
+        output_arg { name: "identity" type: DT_INT32 }
+        output_arg { name: "identity_1" type: DT_INT32 }
+        output_arg { name: "identity_2" type: DT_INT32 }
+      }
+      node_def {
+        name: "add/y"
+        op: "Const"
+        attr {
+          key: "dtype"
+          value { type: DT_INT32 }
+        }
+        attr {
+          key: "value"
+          value {
+            tensor {
+              dtype: DT_INT32
+              tensor_shape {}
+              int_val: 1
+            }
+          }
+        }
+      }
+      node_def {
+        name: "add"
+        op: "Add"
+        input: "const"
+        input: "add/y:output:0"
+        attr {
+          key: "T"
+          value { type: DT_INT32 }
+        }
+      }
+      node_def {
+        name: "add_1/y"
+        op: "Const"
+        attr {
+          key: "dtype"
+          value { type: DT_INT32 }
+        }
+        attr {
+          key: "value"
+          value {
+            tensor {
+              dtype: DT_INT32
+              tensor_shape {}
+              int_val: 1
+            }
+          }
+        }
+      }
+      node_def {
+        name: "add_1"
+        op: "Add"
+        input: "while_loop_counter"
+        input: "add_1/y:output:0"
+        attr {
+          key: "T"
+          value { type: DT_INT32 }
+        }
+      }
+      node_def {
+        name: "Identity"
+        op: "Identity"
+        input: "add_1:z:0"
+        attr {
+          key: "T"
+          value { type: DT_INT32 }
+        }
+      }
+      node_def {
+        name: "Identity_1"
+        op: "Identity"
+        input: "add:z:0"
+        attr {
+          key: "T"
+          value { type: DT_INT32 }
+        }
+      }
+      node_def {
+        name: "Identity_2"
+        op: "Identity"
+        input: "maximum_iterations"
+        attr {
+          key: "T"
+          value { type: DT_INT32 }
+        }
+      }
+      ret { key: "identity" value: "Identity:output:0" }
+      ret { key: "identity_1" value: "Identity_1:output:0" }
+      ret { key: "identity_2" value: "Identity_2:output:0" }
+    }
+    function {
+      signature {
+        name: "__inference_test_function_34"
+        output_arg { name: "identity" type: DT_INT32 }
+        is_stateful: true
+      }
+      node_def {
+        name: "maximum_iterations"
+        op: "Const"
+        attr {
+          key: "dtype"
+          value { type: DT_INT32 }
+        }
+        attr {
+          key: "value"
+          value {
+            tensor {
+              dtype: DT_INT32
+              tensor_shape {}
+              int_val: 1
+            }
+          }
+        }
+      }
+      node_def {
+        name: "Const"
+        op: "Const"
+        attr {
+          key: "dtype"
+          value { type: DT_INT32 }
+        }
+        attr {
+          key: "value"
+          value {
+            tensor {
+              dtype: DT_INT32
+              tensor_shape {}
+              int_val: 0
+            }
+          }
+        }
+      }
+      node_def {
+        name: "while/loop_counter"
+        op: "Const"
+        attr {
+          key: "dtype"
+          value { type: DT_INT32 }
+        }
+        attr {
+          key: "value"
+          value {
+            tensor {
+              dtype: DT_INT32
+              tensor_shape {}
+              int_val: 0
+            }
+          }
+        }
+      }
+      node_def {
+        name: "while"
+        op: "While"
+        input: "while/loop_counter:output:0"
+        input: "Const:output:0"
+        input: "maximum_iterations:output:0"
+        attr {
+          key: "T"
+          value { list { type: DT_INT32 type: DT_INT32 type: DT_INT32 } }
+        }
+        attr {
+          key: "_lower_using_switch_merge"
+          value { b: true }
+        }
+        attr {
+          key: "body"
+          value { func { name: "while_body_5" } }
+        }
+        attr {
+          key: "cond"
+          value { func { name: "while_cond_4" } }
+        }
+        attr {
+          key: "output_shapes"
+          value {
+            list {
+              shape {}
+              shape {}
+              shape {}
+            }
+          }
+        }
+      }
+      node_def {
+        name: "while/Identity"
+        op: "Identity"
+        input: "while:output:0"
+        attr {
+          key: "T"
+          value { type: DT_INT32 }
+        }
+      }
+      node_def {
+        name: "while/Identity_1"
+        op: "Identity"
+        input: "while:output:1"
+        attr {
+          key: "T"
+          value { type: DT_INT32 }
+        }
+      }
+      node_def {
+        name: "while/Identity_2"
+        op: "Identity"
+        input: "while:output:2"
+        attr {
+          key: "T"
+          value { type: DT_INT32 }
+        }
+      }
+      node_def {
+        name: "Identity"
+        op: "Identity"
+        input: "while/Identity_1:output:0"
+        input: "^while"
+        attr {
+          key: "T"
+          value { type: DT_INT32 }
+        }
+      }
+      ret { key: "identity" value: "Identity:output:0" }
+    }
+    function {
+      signature {
+        name: "while_cond_4"
+        input_arg { name: "while_loop_counter" type: DT_INT32 }
+        input_arg { name: "const" type: DT_INT32 }
+        input_arg { name: "less_maximum_iterations" type: DT_INT32 }
+        output_arg { name: "identity" type: DT_BOOL }
+      }
+      node_def {
+        name: "Less"
+        op: "Less"
+        input: "while_loop_counter"
+        input: "less_maximum_iterations"
+        attr {
+          key: "T"
+          value { type: DT_INT32 }
+        }
+      }
+      node_def {
+        name: "Less_1/y"
+        op: "Const"
+        attr {
+          key: "dtype"
+          value { type: DT_INT32 }
+        }
+        attr {
+          key: "value"
+          value {
+            tensor {
+              dtype: DT_INT32
+              tensor_shape {}
+              int_val: 3
+            }
+          }
+        }
+      }
+      node_def {
+        name: "Less_1"
+        op: "Less"
+        input: "const"
+        input: "Less_1/y:output:0"
+        attr {
+          key: "T"
+          value { type: DT_INT32 }
+        }
+      }
+      node_def {
+        name: "LogicalAnd"
+        op: "LogicalAnd"
+        input: "Less:z:0"
+        input: "Less_1:z:0"
+      }
+      node_def {
+        name: "Identity"
+        op: "Identity"
+        input: "LogicalAnd:z:0"
+        attr {
+          key: "T"
+          value { type: DT_BOOL }
+        }
+      }
+      ret { key: "identity" value: "Identity:output:0" }
+    }
+  }
+  versions { producer: 27 min_consumer: 12 })proto";
+
+// TODO(shivaniagrawal): split the test into multiple tests for better
+// readability and add full coverage i.e. add/separate out the tests for all
+// branches of IsNodeStateful and IsFunctionStateful:
+// - test for IsNodeStateful for Cond that has a stateful branch
+// - test for IsNodeStateful for Cond that does not have a stateful branches
+// - test for IsNodeStateful for While that has a stateful branch
+// - test for IsNodeStateful for While that does not have a stateful branches
+// - test for IsNodeStateful for Assert
+// - test for IsNodeStateful for a stateful op
+// - test for IsNodeStateful for a stateless op
+//
+// - test for IsFunctionStateful for a function that contains a Cond
+// - test for IsFunctionStateful for a function that contains a While
+// - test for IsFunctionStateful for a function that contains an Assert (and no
+//   other stateful op)
+// - test for IsFunctionStateful for a function that contains a stateful op
+//   other than Assert
+// - test for IsFunctionStateful for a function that does not contain a stateful
+//   op
+
+TEST(FunctionUtilsTest, IsFunctionStateful) {
+  GraphDef graph_def;
+  MutableGraphView graph(&graph_def);
+
+  NodeDef* nodeA = graph_utils::AddNode("", "A", {}, {}, &graph);
+  FunctionDef* function = graph_def.mutable_library()->add_function();
+  *function = test::function::XTimesTwo();
+
+  FunctionLibraryDefinition lib_def(OpRegistry::Global(),
+                                    *graph_def.mutable_library());
+
+  EXPECT_FALSE(IsFunctionStateful(lib_def, *function));
+
+  // Op "A" is not a registered Op.
+  EXPECT_TRUE(IsNodeStateful(lib_def, *nodeA));
+
+  // Get graph_def for the graph `kCondGraphProto`, graph with function
+  // containing "If" and "Assert" Op.
+
+  GraphDef graph_def_cond;
+  protobuf::TextFormat::ParseFromString(kCondGraphProto, &graph_def_cond);
+  FunctionLibraryDefinition cond_lib(OpRegistry::Global(),
+                                     graph_def_cond.library());
+
+  const FunctionDef* no_op_fnc = cond_lib.Find("cond_true_3");
+
+  EXPECT_FALSE(IsFunctionStateful(cond_lib, *no_op_fnc));
+  EXPECT_FALSE(IsFunctionStateful(cond_lib, *no_op_fnc, true));
+
+  const FunctionDef* assert_func = cond_lib.Find("cond_false_4");
+
+  EXPECT_TRUE(IsFunctionStateful(cond_lib, *assert_func));
+  EXPECT_FALSE(IsFunctionStateful(cond_lib, *assert_func, true));
+
+  EXPECT_TRUE(ContainsFunctionNodeWithOp("Const", *assert_func));
+  EXPECT_TRUE(ContainsFunctionNodeWithOp("Assert", *assert_func));
+
+  for (auto node : assert_func->node_def()) {
+    if (node.op() == "Const") {
+      EXPECT_FALSE(IsNodeStateful(lib_def, node));
+    }
+    if (node.op() == "Assert") {
+      EXPECT_TRUE(IsNodeStateful(lib_def, node));
+      EXPECT_FALSE(IsNodeStateful(lib_def, node, true));
+    }
+  }
+
+  const FunctionDef* cond_func = cond_lib.Find("__inference_test_function_19");
+
+  EXPECT_TRUE(IsFunctionStateful(cond_lib, *cond_func));
+  EXPECT_FALSE(IsFunctionStateful(cond_lib, *cond_func, true));
+
+  // Get graph def for the graph `kWhileGraphProto`, graph with function
+  // containing "While" Op.
+
+  GraphDef graph_def_while;
+  protobuf::TextFormat::ParseFromString(kWhileGraphProto, &graph_def_while);
+
+  FunctionLibraryDefinition while_lib(OpRegistry::Global(),
+                                      graph_def_while.library());
+  const FunctionDef* while_function =
+      while_lib.Find("__inference_test_function_34");
+  EXPECT_FALSE(IsFunctionStateful(while_lib, *while_function));
+  EXPECT_FALSE(IsFunctionStateful(while_lib, *while_function, true));
+}
 }  // namespace
 }  // namespace function_utils
 }  // namespace grappler
diff --git a/tensorflow/core/grappler/optimizers/data/hoist_random_uniform.cc b/tensorflow/core/grappler/optimizers/data/hoist_random_uniform.cc
index 496277780f..e29b620140 100644
--- a/tensorflow/core/grappler/optimizers/data/hoist_random_uniform.cc
+++ b/tensorflow/core/grappler/optimizers/data/hoist_random_uniform.cc
@@ -174,7 +174,7 @@ const FunctionDef* MakeLessStatefulFunction(const FunctionDef& map_function,
   return stateless_function;
 }
 // This function returns true if function is stateful and has single
-// RandomUniform op and no other stateful ops except Assert.
+// RandomUniform op and no other stateful ops except Assert and If/While.
 // `is_stateful_after_hoisting` is set to true if RandomUniform is the only
 // stateful op and hoisting can be performed.
 bool CanHoistRandomUniform(const FunctionDef& map_function,
@@ -189,10 +189,10 @@ bool CanHoistRandomUniform(const FunctionDef& map_function,
   for (const auto& node : map_function.node_def()) {
     const OpDef* op_def;
     TF_CHECK_OK(library.LookUpOpDef(node.op(), &op_def));
-    // Skip stateless nodes and assert, as it does not actually have a state.
     if (!op_def->is_stateful()) continue;
 
-    if (op_def->name() == "Assert") {
+    if (!function_utils::IsNodeStateful(library, node, true)) {
+      // Skip ops that are marked stateful but are in fact not stateful.
       have_other_stateful_ops = true;
       continue;
     }
diff --git a/tensorflow/core/grappler/optimizers/data/map_parallelization.cc b/tensorflow/core/grappler/optimizers/data/map_parallelization.cc
index 57e541cde6..90dd885c7f 100644
--- a/tensorflow/core/grappler/optimizers/data/map_parallelization.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_parallelization.cc
@@ -22,6 +22,7 @@ limitations under the License.
 #include "tensorflow/core/grappler/mutable_graph_view.h"
 #include "tensorflow/core/grappler/op_types.h"
 #include "tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.h"
+#include "tensorflow/core/grappler/optimizers/data/function_utils.h"
 #include "tensorflow/core/grappler/optimizers/data/graph_utils.h"
 #include "tensorflow/core/grappler/utils.h"
 
@@ -33,23 +34,6 @@ constexpr char kMapDataset[] = "MapDataset";
 constexpr char kParallelMapDataset[] = "ParallelMapDataset";
 constexpr int kAutotune = -1;
 
-bool CanParallelize(const FunctionDef& function,
-                    const FunctionLibraryDefinition& library) {
-  if (!function.signature().is_stateful()) return true;
-
-  for (const auto& node : function.node_def()) {
-    const OpDef* op_def;
-    TF_CHECK_OK(library.LookUpOpDef(node.op(), &op_def));
-    // Assert is marked as stateful, but it does not have any state (except
-    // changing io).  Similarly to CUDA, we do not give guarantee that the
-    // assert operation that would fail would be the first one, so that we can
-    // parallelize it.
-    if (op_def->is_stateful() && op_def->name() != "Assert") return false;
-  }
-
-  return true;
-}
-
 NodeDef MakeParallelMap(const string& name, MutableGraphView* graph) {
   // The inputs of the node to be parallelized could be changed by the
   // optimization pass, so we need to look it up in the modified graph.
@@ -88,7 +72,8 @@ Status MapParallelization::OptimizeAndCollectStats(Cluster* cluster,
 
     auto* function =
         function_library.Find(map_node->attr().at("f").func().name());
-    if (!CanParallelize(*function, function_library)) continue;
+    if (function_utils::IsFunctionStateful(function_library, *function, true))
+      continue;
 
     auto* parallel_map =
         graph.AddNode(MakeParallelMap(map_node->name(), &graph));
diff --git a/tensorflow/core/grappler/optimizers/data/map_vectorization.cc b/tensorflow/core/grappler/optimizers/data/map_vectorization.cc
index 2dbd6e4698..db64cfff02 100644
--- a/tensorflow/core/grappler/optimizers/data/map_vectorization.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_vectorization.cc
@@ -131,17 +131,6 @@ bool IsOutputShapesFullyDefined(const NodeDef& node) {
   return true;
 }
 
-bool IsStatefulFn(const FunctionLibraryDefinition& library,
-                  const FunctionDef& function_def) {
-  for (const NodeDef& node_def : function_def.node_def()) {
-    const OpDef* op_def;
-    Status s = library.LookUpOpDef(node_def.op(), &op_def);
-    if (!s.ok() || op_def->is_stateful()) {
-      return true;
-    }
-  }
-  return false;
-}
 
 NodeDef MakeNewBatchNode(const NodeDef& old_batch_node,
                          const NodeDef& input_node,
@@ -250,7 +239,7 @@ Status MapVectorization::OptimizeAndCollectStats(Cluster* cluster,
     // Check that this is a valid optimization.
     if (!IsOutputShapesFullyDefined(*input_node) ||
         !IsOutputShapesFullyDefined(*map_node) ||
-        IsStatefulFn(function_library, *orig_func)) {
+        function_utils::IsFunctionStateful(function_library, *orig_func)) {
       // 1. If any of the inputs have an unknown shape, don't optimize, since
       // inputs might not be batchable.
       // 2. If any of the map func outputs have an unknown shape, don't
-- 
GitLab


From 58d76bb520d0f10e7008a96789252de52dac18e7 Mon Sep 17 00:00:00 2001
From: Haoyu Zhang <haoyuzhang@google.com>
Date: Mon, 21 Jan 2019 22:26:53 -0800
Subject: [PATCH 1049/1095] Pass the `validation_freq` argument to enable
 custom validation frequency in distribution strategy cases.

PiperOrigin-RevId: 230284959
---
 tensorflow/python/keras/engine/training.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 366a96e7a6..41abd3c4ad 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -812,7 +812,8 @@ class Model(Network):
               sample_weight=sample_weight,
               initial_epoch=initial_epoch,
               steps_per_epoch=steps_per_epoch,
-              validation_steps=validation_steps)
+              validation_steps=validation_steps,
+              validation_freq=validation_freq)
 
         # Independent worker only for now.
         return dc.run_distribute_coordinator(
@@ -836,7 +837,8 @@ class Model(Network):
             sample_weight=sample_weight,
             initial_epoch=initial_epoch,
             steps_per_epoch=steps_per_epoch,
-            validation_steps=validation_steps)
+            validation_steps=validation_steps,
+            validation_freq=validation_freq)
 
     batch_size = self._validate_or_infer_batch_size(
         batch_size, steps_per_epoch, x)
-- 
GitLab


From f44c86305911b1066d916d2c289a7a429b6f2a69 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 22 Jan 2019 01:02:44 -0800
Subject: [PATCH 1050/1095] compat: Update forward compatibility horizon to
 2019-01-22

PiperOrigin-RevId: 230297751
---
 tensorflow/python/compat/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 0bd20a1f97..95940550b4 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -27,7 +27,7 @@ import datetime
 from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import tf_export
 
-_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 21)
+_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 1, 22)
 
 
 @tf_export("compat.forward_compatible")
-- 
GitLab


From 4b45dcc63dd043acbe6bb90475d356d2f3149b0c Mon Sep 17 00:00:00 2001
From: Vojtech Bardiovsky <vbardiovsky@google.com>
Date: Tue, 22 Jan 2019 05:15:47 -0800
Subject: [PATCH 1051/1095] Add more debug information when deserializing.

PiperOrigin-RevId: 230322487
---
 .../saved_model/function_deserialization.py   | 21 ++++++++++++-------
 tensorflow/python/saved_model/load_test.py    |  2 +-
 2 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/tensorflow/python/saved_model/function_deserialization.py b/tensorflow/python/saved_model/function_deserialization.py
index 934e189f78..22d5ff4cf4 100644
--- a/tensorflow/python/saved_model/function_deserialization.py
+++ b/tensorflow/python/saved_model/function_deserialization.py
@@ -122,16 +122,20 @@ def recreate_function(saved_function, concrete_functions):
     """Calls a restored function."""
     # TODO(allenl): Functions saved with input_signatures should revive with
     # input_signatures.
+    try:
+      canonicalized_inputs = function_spec.canonicalize_function_inputs(
+          *args, **kwargs)
+    except ValueError as e:
+      raise ValueError(
+          "Cannot canonicalize input args %r and kwargs %r. Error: %r." %
+          (args, kwargs, e))
+
+    debug_considered_signatures = []
     for concrete_function in saved_function.concrete_function:
       function_obj = concrete_functions[concrete_function.name]
       canonicalized_original_inputs = coder.decode_proto(
           concrete_function.canonicalized_input_signature)
-
-      try:
-        canonicalized_inputs = function_spec.canonicalize_function_inputs(
-            *args, **kwargs)
-      except ValueError:
-        continue
+      debug_considered_signatures.append(canonicalized_original_inputs)
 
       if _inputs_compatible(canonicalized_inputs,
                             canonicalized_original_inputs):
@@ -140,8 +144,9 @@ def recreate_function(saved_function, concrete_functions):
         return function_obj._call_flat(filtered_inputs)  # pylint: disable=protected-access
 
     raise AssertionError(
-        "Could not find matching function to call for args %r and kwargs %r" %
-        (args, kwargs))
+        "Could not find matching function to call for canonicalized inputs %r. "
+        "Only existing signatures are %r."
+        % (canonicalized_inputs, debug_considered_signatures))
 
   return restored_function
 
diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py
index 1c999a625d..b1dc0bd128 100644
--- a/tensorflow/python/saved_model/load_test.py
+++ b/tensorflow/python/saved_model/load_test.py
@@ -469,7 +469,7 @@ class LoadTest(test.TestCase):
 
     imported = self.cycle(root)
 
-    with self.assertRaises(AssertionError):
+    with self.assertRaisesRegexp(ValueError, "Cannot canonicalize"):
       # We cannot call the function with a constant of shape ().
       self.assertEqual(7, imported.f(constant_op.constant(2)).numpy())
 
-- 
GitLab


From c38ed2f020920176fbb412e4419ffe4d68885853 Mon Sep 17 00:00:00 2001
From: Shashi Shekhar <shashishekhar@google.com>
Date: Tue, 22 Jan 2019 08:57:02 -0800
Subject: [PATCH 1052/1095] Add back computed bias scales to the quantization
 params.

PiperOrigin-RevId: 230348592
---
 .../lite/tools/optimize/subgraph_quantizer.cc | 12 +++-----
 .../tools/optimize/subgraph_quantizer_test.cc | 30 +++++++++++--------
 2 files changed, 22 insertions(+), 20 deletions(-)

diff --git a/tensorflow/lite/tools/optimize/subgraph_quantizer.cc b/tensorflow/lite/tools/optimize/subgraph_quantizer.cc
index 3bab365c82..99cc8584c6 100644
--- a/tensorflow/lite/tools/optimize/subgraph_quantizer.cc
+++ b/tensorflow/lite/tools/optimize/subgraph_quantizer.cc
@@ -186,14 +186,10 @@ TfLiteStatus SymmetricPerChannelBiasQuantize(const TensorT* input_tensor,
   // Set the buffers and output type.
   uint8_t* uint8_buffer = reinterpret_cast<uint8_t*>(final_buffer.data());
   size_t buffer_size = num_elements * sizeof(int32_t);
-  // For Bias we only set the quantized values, the scale and quantized
-  // dimension is implicit.
-  tensor->quantization = nullptr;
-  model->buffers[tensor->buffer]->data.assign(uint8_buffer,
-                                              uint8_buffer + buffer_size);
-
-  tensor->type = TensorType_INT32;
-  return kTfLiteOk;
+  std::vector<int64_t> zero_point(scales.size(), 0);
+  return AddQuantizationParams(scales, zero_point, channel_dim_index,
+                               uint8_buffer, buffer_size, TensorType_INT32,
+                               model, tensor);
 }
 }  // namespace
 
diff --git a/tensorflow/lite/tools/optimize/subgraph_quantizer_test.cc b/tensorflow/lite/tools/optimize/subgraph_quantizer_test.cc
index 1ea49da899..580ed5eb26 100644
--- a/tensorflow/lite/tools/optimize/subgraph_quantizer_test.cc
+++ b/tensorflow/lite/tools/optimize/subgraph_quantizer_test.cc
@@ -90,16 +90,16 @@ TEST(SubgraphQuantizerTest, VerifyConvQuantizationWithUnitScale) {
 
   ASSERT_TRUE(weights_tensor->quantization);
   const int out_channel_size = weights_tensor->shape[0];
-
-  // Bias tensor doesn't contain quantization info.
-  ASSERT_FALSE(bias_tensor->quantization);
-
+  ASSERT_TRUE(bias_tensor->quantization);
+  ASSERT_TRUE(weights_tensor->quantization);
+  const std::vector<float>& bias_scales = bias_tensor->quantization->scale;
   const std::vector<float>& weights_scales =
       weights_tensor->quantization->scale;
 
   const std::vector<int64_t>& weights_zero_points =
       weights_tensor->quantization->zero_point;
 
+  ASSERT_EQ(bias_scales.size(), out_channel_size);
   ASSERT_EQ(weights_scales.size(), out_channel_size);
   ASSERT_EQ(weights_zero_points.size(), out_channel_size);
   ASSERT_EQ(input_tensor->quantization->scale.size(), 1);
@@ -108,6 +108,7 @@ TEST(SubgraphQuantizerTest, VerifyConvQuantizationWithUnitScale) {
 
   for (size_t i = 0; i < out_channel_size; i++) {
     EXPECT_EQ(weights_scales[i], 1);
+    EXPECT_EQ(bias_scales[i], 1);
     EXPECT_EQ(weights_zero_points[i], 0);
   }
 
@@ -188,21 +189,28 @@ TEST(SubgraphQuantizerTest, VerifyConvQuantization) {
 
   ASSERT_TRUE(weights_tensor->quantization);
   const int out_channel_size = weights_tensor->shape[0];
-
-  // Bias tensor doesn't contain quantization info.
-  ASSERT_FALSE(bias_tensor->quantization);
-
+  ASSERT_TRUE(bias_tensor->quantization);
   ASSERT_TRUE(weights_tensor->quantization);
+  const std::vector<float>& bias_scales = bias_tensor->quantization->scale;
   const std::vector<float>& weights_scales =
       weights_tensor->quantization->scale;
   const std::vector<int64_t>& weights_zero_points =
       weights_tensor->quantization->zero_point;
 
+  ASSERT_EQ(bias_scales.size(), out_channel_size);
   ASSERT_EQ(weights_scales.size(), out_channel_size);
   ASSERT_EQ(weights_zero_points.size(), out_channel_size);
   ASSERT_EQ(input_tensor->quantization->scale.size(), 1);
   ASSERT_EQ(output_tensor->quantization->scale.size(), 1);
 
+  const float eps = 1e-7;
+
+  // Bias scale should be input * per_channel_weight_scale.
+  for (size_t i = 0; i < out_channel_size; i++) {
+    EXPECT_NEAR(bias_scales[i],
+                input_tensor->quantization->scale[0] * weights_scales[i], eps);
+  }
+
   const auto bias_buffer = model.buffers[bias_tensor->buffer].get();
   ASSERT_EQ(bias_buffer->data.size(), sizeof(int32_t) * bias_tensor->shape[0]);
   const int32_t* bias_values =
@@ -213,10 +221,8 @@ TEST(SubgraphQuantizerTest, VerifyConvQuantization) {
       reinterpret_cast<const float*>(original_bias_buffer->data()->data());
 
   for (size_t i = 0; i < out_channel_size; i++) {
-    const float bias_scale =
-        input_tensor->quantization->scale[0] * weights_scales[i];
-    auto dequantized_value = bias_values[i] * bias_scale;
-    EXPECT_NEAR(dequantized_value, bias_float_buffer[i], bias_scale / 2);
+    auto dequantized_value = bias_values[i] * bias_scales[i];
+    EXPECT_NEAR(dequantized_value, bias_float_buffer[i], bias_scales[i] / 2);
   }
 
   const auto weights_buffer = model.buffers[weights_tensor->buffer].get();
-- 
GitLab


From 05677b7c9127967c685af444a6f5ac7291fb3b86 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 22 Jan 2019 09:00:42 -0800
Subject: [PATCH 1053/1095] Delete deprecated and no-longer-used
 tf.Strategy.distribute_dataset().

PiperOrigin-RevId: 230349129
---
 tensorflow/contrib/distribute/python/BUILD    |   2 -
 .../python/collective_all_reduce_strategy.py  |  11 +-
 .../distribute/python/input_lib_test.py       | 234 ------------------
 .../distribute/python/mirrored_strategy.py    |  12 -
 .../distribute/python/one_device_strategy.py  |   4 -
 .../contrib/distribute/python/tpu_strategy.py |   9 +-
 tensorflow/python/distribute/BUILD            |   3 -
 .../python/distribute/distribute_lib.py       |  51 +---
 tensorflow/python/distribute/input_lib.py     | 234 ------------------
 .../python/distribute/mirrored_strategy.py    |  15 +-
 .../distribute/parameter_server_strategy.py   |  11 +-
 ...orflow.distribute.-mirrored-strategy.pbtxt |   4 -
 .../v1/tensorflow.distribute.-strategy.pbtxt  |   4 -
 ...orflow.distribute.-mirrored-strategy.pbtxt |   4 -
 .../v2/tensorflow.distribute.-strategy.pbtxt  |   4 -
 15 files changed, 5 insertions(+), 597 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index 7a77dbe29c..fb8732d591 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -51,9 +51,7 @@ cuda_py_test(
         ":mirrored_strategy",
         ":multi_worker_test_base",
         "@absl_py//absl/testing:parameterized",
-        "//tensorflow/core:protos_all_py",
         "//tensorflow/python:errors",
-        "//tensorflow/python:framework_test_lib",
         "//tensorflow/python/data/ops:dataset_ops",
         "//tensorflow/python/distribute:input_lib",
         "//tensorflow/python/distribute:values",
diff --git a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
index eee0754325..aa4d82b4d0 100644
--- a/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
+++ b/tensorflow/contrib/distribute/python/collective_all_reduce_strategy.py
@@ -251,14 +251,6 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
         self._container_strategy(), device_map, logical_device,
         _real_mirrored_creator, *args, **kwargs)
 
-  def _distribute_dataset(self, dataset_fn):
-    """Distributes the dataset to each local GPU."""
-    # TODO(yuefengz): shard the dataset.
-    worker_index = 0
-    return input_lib.PerReplicaDataset(
-        self._call_dataset_fn(dataset_fn), self._input_workers, worker_index,
-        prefetch_on_device=True)
-
   def _make_dataset_iterator(self, dataset):
     return input_lib.DatasetIterator(dataset, self._input_workers,
                                      self._num_replicas_in_sync)
@@ -374,8 +366,7 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended):
   def _global_batch_size(self):
     """`make_dataset_iterator` and `make_numpy_iterator` use global batch size.
 
-    `distribute_dataset` and `make_input_fn_iterator` assume per-replica
-    batching.
+    `make_input_fn_iterator` assumes per-replica batching.
 
     Returns:
       Boolean.
diff --git a/tensorflow/contrib/distribute/python/input_lib_test.py b/tensorflow/contrib/distribute/python/input_lib_test.py
index f589cd6ad5..10a58316ec 100644
--- a/tensorflow/contrib/distribute/python/input_lib_test.py
+++ b/tensorflow/contrib/distribute/python/input_lib_test.py
@@ -22,7 +22,6 @@ from absl.testing import parameterized
 
 from tensorflow.contrib.distribute.python import combinations
 from tensorflow.contrib.distribute.python import multi_worker_test_base
-from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.data.experimental.ops import batching
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.distribute import distribute_lib
@@ -31,243 +30,10 @@ from tensorflow.python.distribute import values
 from tensorflow.python.eager import context
 from tensorflow.python.eager import test
 from tensorflow.python.framework import errors
-from tensorflow.python.framework import test_util
 from tensorflow.python.ops import control_flow_ops
-from tensorflow.python.ops import random_ops
 from tensorflow.python.util import nest
 
 
-class PerReplicaDatasetTest(test.TestCase):
-
-  config = config_pb2.ConfigProto()
-  config.allow_soft_placement = True
-
-  def _test_iterator(self, devices, dataset, expected_values):
-    device_map = values.ReplicaDeviceMap(devices)
-    input_workers = input_lib.InputWorkers(device_map)
-    per_replica_dataset = input_lib.PerReplicaDataset(dataset, input_workers, 0)
-    if context.executing_eagerly():
-      iterator = per_replica_dataset.make_one_shot_iterator()
-    else:
-      iterator = per_replica_dataset.make_initializable_iterator()
-      self.evaluate([iterator.initializer])
-
-    for expected_value in expected_values:
-      next_element = iterator.get_next_as_list()
-      computed_value = self.evaluate(next_element)
-      self.assertEqual(expected_value, computed_value)
-
-    with self.assertRaises(errors.OutOfRangeError):
-      next_element = iterator.get_next_as_list()
-      self.evaluate(next_element)
-
-  @test_util.run_in_graph_and_eager_modes
-  def testOneDevice(self):
-    devices = ["/device:CPU:0"]
-    dataset = dataset_ops.Dataset.range(10)
-
-    expected_values = [[i] for i in range(10)]
-
-    self._test_iterator(devices, dataset, expected_values)
-
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testMultipleDevices(self):
-    if context.num_gpus() < 1 and context.executing_eagerly():
-      self.skipTest("A GPU is not available for this test in eager mode.")
-
-    devices = ["/device:CPU:0", "/device:GPU:0"]
-    dataset = dataset_ops.Dataset.range(10)
-
-    expected_values = [[i, i+1] for i in range(0, 10, 2)]
-
-    self._test_iterator(devices, dataset, expected_values)
-
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testTupleDataset(self):
-    if context.num_gpus() < 1 and context.executing_eagerly():
-      self.skipTest("A GPU is not available for this test in eager mode.")
-
-    devices = ["/device:CPU:0", "/device:GPU:0"]
-    dataset1 = dataset_ops.Dataset.range(10)
-    dataset2 = dataset_ops.Dataset.range(10).map(lambda x: x**2)
-    dataset = dataset_ops.Dataset.zip((dataset1, dataset2))
-
-    expected_values = [[(i, i**2), (i+1, (i+1)**2)] for i in range(0, 10, 2)]
-
-    self._test_iterator(devices, dataset, expected_values)
-
-  @test_util.run_in_graph_and_eager_modes(config=config)
-  def testUnevenDatasetBatches(self):
-    if context.num_gpus() < 1 and context.executing_eagerly():
-      self.skipTest("A GPU is not available for this test in eager mode.")
-
-    devices = ["/device:CPU:0", "/device:GPU:0"]
-    dataset = dataset_ops.Dataset.range(11)
-
-    expected_values = [[i, i+1] for i in range(0, 10, 2)]
-    self._test_iterator(devices, dataset, expected_values)
-
-  def testInitializableIterator(self):
-    with context.graph_mode():
-      devices = ["/device:CPU:0"]
-      # Using random input since that is only allowed with initializable
-      # iterator.
-      dataset = dataset_ops.Dataset.from_tensor_slices(
-          random_ops.random_uniform((10,)))
-
-      device_map = values.ReplicaDeviceMap(devices)
-      input_workers = input_lib.InputWorkers(device_map)
-      per_replica_dataset = input_lib.PerReplicaDataset(
-          dataset, input_workers, 0)
-      iterator = per_replica_dataset.make_initializable_iterator()
-
-      self.evaluate(iterator.initializer)
-      next_element = iterator.get_next_as_list()
-      for _ in range(10):
-        self.evaluate(next_element)
-
-      # Should fail after the input is finished.
-      with self.assertRaises(errors.OutOfRangeError):
-        self.evaluate(next_element)
-
-      # After re-initializing the iterator, should be able to iterate again.
-      self.evaluate(iterator.initializer)
-      for _ in range(10):
-        self.evaluate(next_element)
-
-
-class MultiWorkerDatasetTest(multi_worker_test_base.MultiWorkerTestBase):
-
-  def _test_iterator(self, sess, iterator, devices, expected_values):
-    next_element = iterator.get_next()
-    for r, device in enumerate(devices):
-      v = values.select_replica(r, next_element)
-      # The `v` here can be a tuple.
-      for element in nest.flatten(v):
-        self.assertTrue(element.device in device)
-
-    for expected_value in expected_values:
-      t = [values.select_replica(r, next_element) for r in range(len(devices))]
-      actual = sess.run(t)
-      self.assertEqual(expected_value, actual)
-
-    with self.assertRaises(errors.OutOfRangeError):
-      sess.run([values.select_replica(r, next_element)
-                for r in range(len(devices))])
-
-  def _test_dataset(self, dataset_fn, worker_devices, devices,
-                    expected_values):
-    device_map = values.ReplicaDeviceMap(devices)
-    input_workers = input_lib.InputWorkers(device_map, worker_devices)
-    multi_worker_dataset = input_lib.MultiWorkerDataset(
-        dataset_fn, input_workers)
-    multi_worker_iterator = multi_worker_dataset.make_initializable_iterator()
-    with self.cached_session() as sess:
-      sess.run(multi_worker_iterator.initializer)
-      self._test_iterator(sess, multi_worker_iterator, devices, expected_values)
-
-  def _cpu_devices(self):
-    worker_devices = (
-        ("/job:worker/replica:0/task:0",
-         ["/job:worker/replica:0/task:0/device:CPU:0"]),
-        ("/job:worker/replica:0/task:1",
-         ["/job:worker/replica:0/task:1/device:CPU:0"])
-    )
-    devices = [
-        "/job:worker/replica:0/task:0/device:CPU:0",
-        "/job:worker/replica:0/task:1/device:CPU:0"
-    ]
-    return worker_devices, devices
-
-  def _cpu_and_one_gpu_devices(self):
-    worker_devices = (
-        ("/job:worker/replica:0/task:0", (
-            "/job:worker/replica:0/task:0/device:GPU:0",
-            "/job:worker/replica:0/task:0/device:CPU:0"
-        )),
-        ("/job:worker/replica:0/task:1", (
-            "/job:worker/replica:0/task:1/device:GPU:0",
-            "/job:worker/replica:0/task:1/device:CPU:0"
-        ))
-    )
-    devices = [
-        "/job:worker/replica:0/task:0/device:GPU:0",
-        "/job:worker/replica:0/task:0/device:CPU:0",
-        "/job:worker/replica:0/task:1/device:GPU:0",
-        "/job:worker/replica:0/task:1/device:CPU:0"
-    ]
-    return worker_devices, devices
-
-  def testDataDistributionOneDevicePerWorker(self):
-    worker_devices, devices = self._cpu_devices()
-    with context.graph_mode():
-      dataset_fn = lambda: dataset_ops.Dataset.range(8)
-      self._test_dataset(
-          dataset_fn, worker_devices, devices,
-          [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4], [5, 5], [6, 6], [7, 7]])
-
-  def testDataDistributionTwoDevicePerWorker(self):
-    if context.num_gpus() < 1:
-      self.skipTest("A GPU is not available for this test.")
-    worker_devices, devices = self._cpu_and_one_gpu_devices()
-    with context.graph_mode():
-      dataset_fn = lambda: dataset_ops.Dataset.range(8)
-      self._test_dataset(
-          dataset_fn, worker_devices, devices,
-          [[0, 1, 0, 1], [2, 3, 2, 3], [4, 5, 4, 5], [6, 7, 6, 7]])
-
-  def testTupleDataset(self):
-    worker_devices, devices = self._cpu_devices()
-
-    with context.graph_mode():
-
-      def dataset_fn():
-        dataset1 = dataset_ops.Dataset.range(8)
-        dataset2 = dataset_ops.Dataset.range(8).map(lambda x: x**2)
-        return dataset_ops.Dataset.zip((dataset1, dataset2))
-
-      expected_values = [[(i, i**2), (i, i**2)] for i in range(8)]
-      self._test_dataset(dataset_fn, worker_devices, devices,
-                         expected_values)
-
-  def testInitializableIterator(self):
-    worker_devices, devices = self._cpu_devices()
-    with context.graph_mode(), self.cached_session() as sess:
-      dataset_fn = lambda: dataset_ops.Dataset.range(8)
-      device_map = values.ReplicaDeviceMap(devices)
-      input_workers = input_lib.InputWorkers(device_map, worker_devices)
-      multi_worker_dataset = input_lib.MultiWorkerDataset(
-          dataset_fn, input_workers)
-      multi_worker_iterator = multi_worker_dataset.make_initializable_iterator()
-
-      sess.run(multi_worker_iterator.initializer)
-      self._test_iterator(
-          sess, multi_worker_iterator, devices,
-          [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4], [5, 5], [6, 6], [7, 7]])
-
-      # After re-initializing the iterator, should be able to iterate again.
-      sess.run(multi_worker_iterator.initializer)
-      self._test_iterator(
-          sess, multi_worker_iterator, devices,
-          [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4], [5, 5], [6, 6], [7, 7]])
-
-  def testValueErrorForIterator(self):
-    # Incompatiable arguments.
-    d1 = "/device:GPU:0"
-    d2 = "/device:GPU:1"
-    device_map = values.ReplicaDeviceMap([d1, d2])
-    input_workers = input_lib.InputWorkers(
-        device_map, (("w1", (d1,)), ("w2", (d2,))))
-    with self.assertRaises(ValueError):
-      input_lib.MultiWorkerDataIterator([("w1", None)], input_workers)
-
-  def testDuplicateDevices(self):
-    _, devices = self._cpu_devices()
-    devices.append("/job:worker/replica:0/task:0/device:CPU:0")
-    with self.assertRaises(ValueError):
-      _ = values.ReplicaDeviceMap(devices)
-
-
 class InputIteratorTestBase(test.TestCase):
 
   def _test_iterator(self, input_type, dataset_fn, worker_device_pairs,
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy.py b/tensorflow/contrib/distribute/python/mirrored_strategy.py
index 2e23a51ee5..5391e083fc 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy.py
@@ -18,8 +18,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import functools
-
 from tensorflow.python.distribute import distribute_lib
 from tensorflow.python.distribute import input_lib
 from tensorflow.python.distribute import mirrored_strategy
@@ -192,16 +190,6 @@ class MirroredExtended(CoreMirroredExtended):
     """
     return input_lib.DatasetIterator(dataset, self._input_workers)
 
-  def _distribute_dataset(self, dataset_fn):
-    if self._local_mode:
-      return input_lib.PerReplicaDataset(
-          self._call_dataset_fn(dataset_fn), self._input_workers, 0)
-    else:
-      return input_lib.MultiWorkerDataset(
-          functools.partial(self._call_dataset_fn, dataset_fn),
-          self._input_workers,
-          auto_shard=self._auto_shard_dataset)
-
   # TODO(priyag): Delete this once all strategies use global batch size.
   @property
   def _global_batch_size(self):
diff --git a/tensorflow/contrib/distribute/python/one_device_strategy.py b/tensorflow/contrib/distribute/python/one_device_strategy.py
index 836cb7cc41..24d6a443fe 100644
--- a/tensorflow/contrib/distribute/python/one_device_strategy.py
+++ b/tensorflow/contrib/distribute/python/one_device_strategy.py
@@ -72,10 +72,6 @@ class OneDeviceExtended(distribute_lib.DistributionStrategyExtended):
     """Make iterator from dataset without splitting the batch."""
     return input_lib.DatasetIterator(dataset, self._input_workers)
 
-  def _distribute_dataset(self, dataset_fn):
-    return input_lib.PerReplicaDataset(
-        self._call_dataset_fn(dataset_fn), self._input_workers, 0)
-
   def _make_input_fn_iterator(
       self,
       input_fn,
diff --git a/tensorflow/contrib/distribute/python/tpu_strategy.py b/tensorflow/contrib/distribute/python/tpu_strategy.py
index f55e6f0b20..5a494a6ae7 100644
--- a/tensorflow/contrib/distribute/python/tpu_strategy.py
+++ b/tensorflow/contrib/distribute/python/tpu_strategy.py
@@ -22,7 +22,6 @@ from __future__ import division
 from __future__ import print_function
 
 import copy
-import functools
 
 from tensorflow.contrib.tpu.python.ops import tpu_ops
 from tensorflow.contrib.tpu.python.tpu import device_assignment as device_assignment_lib
@@ -323,11 +322,6 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
     return input_lib.DatasetIterator(dataset, self._input_workers,
                                      self._num_replicas_in_sync)
 
-  def _distribute_dataset(self, dataset_fn):
-    return input_lib.MultiWorkerDataset(
-        functools.partial(self._call_dataset_fn, dataset_fn),
-        self._input_workers)
-
   def _make_input_fn_iterator(
       self,
       input_fn,
@@ -726,8 +720,7 @@ class TPUExtended(distribute_lib.DistributionStrategyExtended):
   def _global_batch_size(self):
     """`make_dataset_iterator` and `make_numpy_iterator` use global batch size.
 
-    `distribute_dataset` and `make_input_fn_iterator` assume per-replica
-    batching.
+    `make_input_fn_iterator` assumes per-replica batching.
 
     Returns:
       Boolean.
diff --git a/tensorflow/python/distribute/BUILD b/tensorflow/python/distribute/BUILD
index a6a1c470b4..24a2a84b7f 100644
--- a/tensorflow/python/distribute/BUILD
+++ b/tensorflow/python/distribute/BUILD
@@ -316,10 +316,7 @@ py_library(
         ":distribute_lib",
         ":input_ops",
         ":values",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:control_flow_ops",
         "//tensorflow/python:framework_ops",
-        "//tensorflow/python:util",
         "//tensorflow/python/data/ops:multi_device_iterator_ops",
         "//tensorflow/python/eager:context",
     ],
diff --git a/tensorflow/python/distribute/distribute_lib.py b/tensorflow/python/distribute/distribute_lib.py
index e7d52c6c98..2cc99b3db4 100644
--- a/tensorflow/python/distribute/distribute_lib.py
+++ b/tensorflow/python/distribute/distribute_lib.py
@@ -333,36 +333,6 @@ class DistributionStrategy(object):
     """DEPRECATED: use extended.colocate_vars_with() instead."""
     return self._extended.colocate_vars_with(colocate_with_variable)
 
-  @doc_controls.do_not_generate_docs  # DEPRECATED
-  def distribute_dataset(self, dataset_fn):
-    """Return a `dataset` split across all replicas.  DEPRECATED.
-
-    DEPRECATED: Please use `make_dataset_iterator` or
-    `make_input_fn_iterator` instead.
-
-    Suitable for providing input to `extended.call_for_each_replica()` by
-    creating an iterator:
-
-    ```
-    def dataset_fn():
-      return tf.data.Dataset.from_tensors([[1.]]).repeat()
-
-    with strategy.scope():
-      distributed_dataset = strategy.distribute_dataset(dataset_fn)
-      iterator = distributed_dataset.make_initializable_iterator()
-      replica_results = strategy.extended.call_for_each_replica(
-          replica_fn, args=(iterator.get_next(),))
-    ```
-
-    Args:
-      dataset_fn: A function that returns a `tf.data.Dataset` with per-replica
-        batching.
-
-    Returns:
-      A `PerReplicaDataset` that will produce data for each replica.
-    """
-    return self._extended._distribute_dataset(dataset_fn)  # pylint: disable=protected-access
-
   def make_dataset_iterator(self, dataset):
     """Makes an iterator for input provided via `dataset`.
 
@@ -756,8 +726,7 @@ class DistributionStrategyExtended(object):
     a variable (which by definition will have locality V(`v`), though
     will match another locality if inside a `colocate_vars_with`
     scope).
-  * `d.make_dataset_iterator(dataset)` (or the deprecated
-    `d.distribute_dataset(dataset).make_one_shot_iterator()`): in cross-replica
+  * `d.make_dataset_iterator(dataset)`: in cross-replica
     context, produces an iterator with locality T
   * `d.broadcast(t)`: in cross-replica context, produces a value with locality M
   * `d.extended.broadcast_to(t, v)`: in cross-replica context, produces a value
@@ -965,21 +934,6 @@ class DistributionStrategyExtended(object):
     """Validate `colocate_with_variable` argument to `colocate_vars_with`."""
     pass
 
-  def _call_dataset_fn(self, dataset_fn):
-    """Call the `dataset_fn` with `input_context` as argument."""
-    result = dataset_fn()
-    if not isinstance(result, dataset_ops.DatasetV2):
-      raise ValueError(
-          "dataset_fn() must return a tf.data.Dataset when using a "
-          "tf.distribute.Strategy.")
-    return result
-
-  # TODO(josh11b): `PerReplicaDataset` currently only implements a few methods of
-  # Dataset API such as make_one_shot_iterator and make_initializable_iterator.
-  # Extend to implement more functionality of datasets.
-  def _distribute_dataset(self, dataset_fn):
-    raise NotImplementedError("must be implemented in descendants")
-
   def _make_dataset_iterator(self, dataset):
     raise NotImplementedError("must be implemented in descendants")
 
@@ -1549,9 +1503,6 @@ class _DefaultDistributionExtended(DistributionStrategyExtended):
   def variable_created_in_scope(self, v):
     return v._distribute_strategy is None  # pylint: disable=protected-access
 
-  def _distribute_dataset(self, dataset_fn):
-    return self._call_dataset_fn(dataset_fn)
-
   def _make_dataset_iterator(self, dataset):
     return _DefaultDistributionExtended.DefaultInputIterator(dataset)
 
diff --git a/tensorflow/python/distribute/input_lib.py b/tensorflow/python/distribute/input_lib.py
index 3ba80c9287..43915917d6 100644
--- a/tensorflow/python/distribute/input_lib.py
+++ b/tensorflow/python/distribute/input_lib.py
@@ -29,9 +29,6 @@ from tensorflow.python.eager import context
 from tensorflow.python.framework import device as tf_device
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_util
-from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import control_flow_ops
-from tensorflow.python.util import nest
 
 
 class InputWorkers(object):
@@ -91,237 +88,6 @@ class InputWorkers(object):
         self.__class__.__name__, debug_repr, self._device_map)
 
 
-class PerReplicaDataIterator(object):
-  """An iterator (like `tf.data.Iterator`) into a `PerReplicaDataset`."""
-
-  def __init__(self, iterator, input_workers, worker_index, prefetch_on_device):
-    assert isinstance(input_workers, InputWorkers)
-    self._iterator = iterator
-    self._input_workers = input_workers
-    self._worker_index = worker_index
-    self._prefetch_on_device = prefetch_on_device
-
-  @property
-  def initializer(self):
-    return self._iterator.initializer
-
-  def get_next_as_list(self, name=None):
-    """Scatter the input across devices."""
-    if self._prefetch_on_device:
-      data_list = self._iterator.get_next()
-    else:
-      batch = self._iterator.get_next(name=name)
-      data_list = []
-      def get_ith(i):
-        return lambda x: x[i]
-
-      devices = self._input_workers.compute_devices_for_worker(
-          self._worker_index)
-      for i, d in enumerate(devices):
-        v = nest.map_structure(get_ith(i), batch)
-        if context.executing_eagerly():
-          with ops.device(d):
-            v = nest.map_structure(array_ops.identity, v)
-        data_list.append(v)
-
-    return data_list
-
-  def get_next(self, name=None):
-    assert self._input_workers.num_workers == 1
-    data_list = self.get_next_as_list(name)
-    return values.regroup(self._input_workers.device_map, data_list)
-
-  @property
-  def output_classes(self):
-    return self._iterator.output_classes
-
-  @property
-  def output_shapes(self):
-    return self._iterator.output_shapes
-
-  @property
-  def output_types(self):
-    return self._iterator.output_types
-
-
-class PerReplicaDataset(object):
-  """Like `tf.data.Dataset` split devices, producing `PerReplica` data."""
-
-  def __init__(self, dataset, input_workers, worker_index,
-               prefetch_on_device=None):
-    assert isinstance(input_workers, InputWorkers)
-    assert worker_index is not None
-    assert worker_index is not True  # pylint: disable=g-bool-id-comparison
-    assert worker_index is not False  # pylint: disable=g-bool-id-comparison
-    self._input_workers = input_workers
-    self._worker_index = worker_index
-
-    # Default to using prefetching, unless specified.
-    self._prefetch_on_device = prefetch_on_device
-    if self._prefetch_on_device is None:
-      self._prefetch_on_device = True
-
-    self._dataset = dataset
-    if not self._prefetch_on_device:
-      # TODO(priyag): If dropping remainder is not appropriate, find another
-      # approach to distributing the dataset when not possible to divide evenly.
-      # Possibly not an issue when we start using PartitionedDataset.
-      num_replicas = len(
-          self._input_workers.compute_devices_for_worker(self._worker_index))
-      self._dataset = self._dataset.batch(num_replicas, drop_remainder=True)
-    else:
-      self._replica_devices = self._input_workers.compute_devices_for_worker(
-          self._worker_index)
-
-  def make_one_shot_iterator(self):
-    """Get a one time use iterator for the distributed PerReplicaDataset."""
-    # Graph mode with one shot iterator is disabled.
-    if not context.executing_eagerly():
-      raise ValueError("Cannot create a one shot iterator. Please use "
-                       "`make_initializable_iterator()` instead.")
-    if self._prefetch_on_device:
-      dataset_iterator = multi_device_iterator_ops.MultiDeviceIterator(
-          self._dataset, self._replica_devices)
-    else:
-      dataset_iterator = dataset_ops.make_one_shot_iterator(self._dataset)
-    return PerReplicaDataIterator(
-        dataset_iterator,
-        self._input_workers,
-        self._worker_index,
-        prefetch_on_device=self._prefetch_on_device)
-
-  def make_initializable_iterator(self):
-    """Get an initializable iterator for the distributed PerReplicaDataset."""
-    # Eager mode generates already initialized iterators. Hence we cannot create
-    # an initializable iterator.
-    if context.executing_eagerly():
-      raise ValueError("Cannot create initializable iterator in Eager mode. "
-                       "Please use `make_one_shot_iterator` instead.")
-    if self._prefetch_on_device:
-      dataset_iterator = multi_device_iterator_ops.MultiDeviceIterator(
-          self._dataset, self._replica_devices)
-    else:
-      dataset_iterator = dataset_ops.make_initializable_iterator(self._dataset)
-    return PerReplicaDataIterator(
-        dataset_iterator, self._input_workers, self._worker_index,
-        prefetch_on_device=self._prefetch_on_device)
-
-
-class MultiWorkerDataIterator(object):
-  """An iterator (like `tf.data.Iterator`) into a `MultiWorkerDataset`."""
-
-  def __init__(self, iterators, input_workers):
-    """Initialize the `MultiWorkerDataIterator` object.
-
-    Args:
-      iterators: a list of worker, iterator pairs.
-      input_workers: an `InputWorkers` object.
-
-    Raises:
-      ValueError: if iterators and input_workers are not compatible.
-    """
-    assert isinstance(input_workers, InputWorkers)
-    workers = tuple(d for d, _ in iterators)
-    if workers != input_workers.worker_devices:
-      raise ValueError("iterators and input_workers are not compatible. "
-                       "iterator workers: %r input_workers devices: %r" %
-                       (workers, input_workers.worker_devices))
-    self._iterators = tuple(i for _, i in iterators)
-    self._input_workers = input_workers
-
-  @property
-  def initializer(self):
-    return control_flow_ops.group(
-        tuple(iterator.initializer for iterator in self._iterators))
-
-  def get_iterator(self, worker):
-    for i, w in enumerate(self._input_workers.worker_devices):
-      if worker == w:
-        return self._iterators[i]
-    return None
-
-  @property
-  def output_shapes(self):
-    return self._iterators[0].output_shapes
-
-  @property
-  def output_types(self):
-    return self._iterators[0].output_types
-
-  def get_next(self, name=None):
-    """Scatter the input across hosts and devices."""
-    replicas = []
-    for worker, iterator in zip(self._input_workers.worker_devices,
-                                self._iterators):
-      if name is not None:
-        d = tf_device.DeviceSpec.from_string(worker)
-        new_name = "%s_%s_%d" % (name, d.job, d.task)
-      else:
-        new_name = None
-      with ops.device(worker):
-        data_per_worker = iterator.get_next_as_list(name=new_name)
-        # Append to replicas to get a flat list of values indexed by replica.
-        replicas.extend(data_per_worker)
-
-    return values.regroup(self._input_workers.device_map, replicas)
-
-
-class MultiWorkerDataset(object):
-  """Like a `tf.data.Dataset` that distributes data to different workers.
-
-  Each worker gets one shard of the input dataset. This currently does not work
-  in eager mode.
-  """
-
-  def __init__(self, dataset_fn, input_workers, prefetch_on_device=None,
-               auto_shard=False):
-    """Initialize the MultiWorkerDataset object.
-
-    Args:
-      dataset_fn: a function or a list of functions that returns a
-        `tf.data.Dataset`.
-      input_workers: an `InputWorkers` object.
-      prefetch_on_device: whether to prefetch to devices.
-      auto_shard: whether to auto-shard the dataset.
-    """
-    assert isinstance(input_workers, InputWorkers)
-    if isinstance(dataset_fn, (list, tuple)):
-      if len(dataset_fn) != input_workers.num_workers:
-        raise ValueError("If `dataset_fn` is a list, it must have one entry "
-                         "per worker")
-    # TODO(rohanj): b/120673685 to track re-enabling auto sharding.
-    if auto_shard:
-      raise ValueError("Currently autosharding is not supported.")
-    self._input_workers = input_workers
-    self._datasets = []
-    # TODO(yuefengz, priyag): support different set of jobs for input
-    # processing.
-    for i, worker in enumerate(input_workers.worker_devices):
-      with ops.device(worker):
-        if isinstance(dataset_fn, (list, tuple)):
-          worker_input = dataset_fn[i]()
-        else:
-          worker_input = dataset_fn()
-        dataset = PerReplicaDataset(worker_input, input_workers, i,
-                                    prefetch_on_device=prefetch_on_device)
-        self._datasets.append((worker, dataset))
-
-  def make_one_shot_iterator(self):
-    iterators = []
-    for worker, dataset in self._datasets:
-      with ops.device(worker):
-        iterators.append((worker, dataset_ops.make_one_shot_iterator(dataset)))
-    return MultiWorkerDataIterator(iterators, self._input_workers)
-
-  def make_initializable_iterator(self):
-    iterators = []
-    for worker, dataset in self._datasets:
-      with ops.device(worker):
-        iterators.append(
-            (worker, dataset_ops.make_initializable_iterator(dataset)))
-    return MultiWorkerDataIterator(iterators, self._input_workers)
-
-
 class InputIterator(object):
   """An input iterator, intended to be passed to `DistributionStrategy.run`."""
 
diff --git a/tensorflow/python/distribute/mirrored_strategy.py b/tensorflow/python/distribute/mirrored_strategy.py
index f279c3ba1b..c5314e386e 100644
--- a/tensorflow/python/distribute/mirrored_strategy.py
+++ b/tensorflow/python/distribute/mirrored_strategy.py
@@ -20,7 +20,6 @@ from __future__ import print_function
 
 import contextlib
 import copy
-import functools
 import threading
 
 from tensorflow.python import pywrap_tensorflow
@@ -548,17 +547,6 @@ class MirroredExtended(distribute_lib.DistributionStrategyExtended):
   def _validate_colocate_with_variable(self, colocate_with_variable):
     values.validate_colocate_distributed_variable(colocate_with_variable, self)
 
-  def _distribute_dataset(self, dataset_fn):
-    if self._local_mode:
-      worker_index = 0
-      return input_lib.PerReplicaDataset(
-          self._call_dataset_fn(dataset_fn), self._input_workers, worker_index)
-    else:
-      return input_lib.MultiWorkerDataset(
-          functools.partial(self._call_dataset_fn, dataset_fn),
-          self._input_workers,
-          auto_shard=False)
-
   def _make_dataset_iterator(self, dataset):
     return input_lib.DatasetIterator(
         dataset, self._input_workers, self._num_replicas_in_sync)
@@ -784,8 +772,7 @@ class MirroredExtended(distribute_lib.DistributionStrategyExtended):
   def _global_batch_size(self):
     """`make_dataset_iterator` and `make_numpy_iterator` use global batch size.
 
-    `distribute_dataset` and `make_input_fn_iterator` assume per-replica
-    batching.
+    `make_input_fn_iterator` assumes per-replica batching.
 
     Returns:
       Boolean.
diff --git a/tensorflow/python/distribute/parameter_server_strategy.py b/tensorflow/python/distribute/parameter_server_strategy.py
index 8093f8ccb0..6cc5865eed 100644
--- a/tensorflow/python/distribute/parameter_server_strategy.py
+++ b/tensorflow/python/distribute/parameter_server_strategy.py
@@ -233,14 +233,6 @@ class ParameterServerStrategyExtended(
   def _validate_colocate_with_variable(self, colocate_with_variable):
     values.validate_colocate(colocate_with_variable, self)
 
-  def _distribute_dataset(self, dataset_fn):
-    """Distributes the dataset to each local GPU."""
-    return input_lib.PerReplicaDataset(
-        self._call_dataset_fn(dataset_fn),
-        self._input_workers,
-        0,
-        prefetch_on_device=True)
-
   def _make_dataset_iterator(self, dataset):
     return input_lib.DatasetIterator(dataset, self._input_workers,
                                      self._num_replicas_in_sync)
@@ -538,8 +530,7 @@ class ParameterServerStrategyExtended(
   def _global_batch_size(self):
     """`make_dataset_iterator` and `make_numpy_iterator` use global batch size.
 
-    `distribute_dataset` and `make_input_fn_iterator` assume per-replica
-    batching.
+    `make_input_fn_iterator` assumes per-replica batching.
 
     Returns:
       Boolean.
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-mirrored-strategy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-mirrored-strategy.pbtxt
index 4cb78b08f8..89748f7713 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-mirrored-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-mirrored-strategy.pbtxt
@@ -27,10 +27,6 @@ tf_class {
     name: "configure"
     argspec: "args=[\'self\', \'session_config\', \'cluster_spec\', \'task_type\', \'task_id\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
-  member_method {
-    name: "distribute_dataset"
-    argspec: "args=[\'self\', \'dataset_fn\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "experimental_make_numpy_iterator"
     argspec: "args=[\'self\', \'numpy_input\', \'batch_size\', \'num_epochs\', \'shuffle\', \'session\'], varargs=None, keywords=None, defaults=[\'1\', \'1024\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt
index 11c1479b5b..6ed49d339d 100644
--- a/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v1/tensorflow.distribute.-strategy.pbtxt
@@ -26,10 +26,6 @@ tf_class {
     name: "configure"
     argspec: "args=[\'self\', \'session_config\', \'cluster_spec\', \'task_type\', \'task_id\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
-  member_method {
-    name: "distribute_dataset"
-    argspec: "args=[\'self\', \'dataset_fn\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "experimental_make_numpy_iterator"
     argspec: "args=[\'self\', \'numpy_input\', \'batch_size\', \'num_epochs\', \'shuffle\', \'session\'], varargs=None, keywords=None, defaults=[\'1\', \'1024\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-mirrored-strategy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-mirrored-strategy.pbtxt
index 4cb78b08f8..89748f7713 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-mirrored-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-mirrored-strategy.pbtxt
@@ -27,10 +27,6 @@ tf_class {
     name: "configure"
     argspec: "args=[\'self\', \'session_config\', \'cluster_spec\', \'task_type\', \'task_id\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
-  member_method {
-    name: "distribute_dataset"
-    argspec: "args=[\'self\', \'dataset_fn\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "experimental_make_numpy_iterator"
     argspec: "args=[\'self\', \'numpy_input\', \'batch_size\', \'num_epochs\', \'shuffle\', \'session\'], varargs=None, keywords=None, defaults=[\'1\', \'1024\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt
index 11c1479b5b..6ed49d339d 100644
--- a/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt
+++ b/tensorflow/tools/api/golden/v2/tensorflow.distribute.-strategy.pbtxt
@@ -26,10 +26,6 @@ tf_class {
     name: "configure"
     argspec: "args=[\'self\', \'session_config\', \'cluster_spec\', \'task_type\', \'task_id\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
-  member_method {
-    name: "distribute_dataset"
-    argspec: "args=[\'self\', \'dataset_fn\'], varargs=None, keywords=None, defaults=None"
-  }
   member_method {
     name: "experimental_make_numpy_iterator"
     argspec: "args=[\'self\', \'numpy_input\', \'batch_size\', \'num_epochs\', \'shuffle\', \'session\'], varargs=None, keywords=None, defaults=[\'1\', \'1024\', \'None\'], "
-- 
GitLab


From 0b65aa379fc46b40f963edcf58545d99b4cf2f63 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 22 Jan 2019 09:03:17 -0800
Subject: [PATCH 1054/1095] Adding transformer.TransformerContext to 1. wrap
 the immutable EntityContext and 2. track the current OriginInfo, so that we
 can move error AutoGraphParseError handling to api.to_graph().

Removing the finally block in visit() in transformer.Base error handling.

PiperOrigin-RevId: 230349773
---
 .../pyct/static_analysis/live_values.py       | 11 ++---
 .../pyct/static_analysis/type_info.py         |  5 ++-
 .../python/autograph/pyct/transformer.py      | 43 +++++++++++++------
 3 files changed, 39 insertions(+), 20 deletions(-)

diff --git a/tensorflow/python/autograph/pyct/static_analysis/live_values.py b/tensorflow/python/autograph/pyct/static_analysis/live_values.py
index e8e3d229be..14504fbe1b 100644
--- a/tensorflow/python/autograph/pyct/static_analysis/live_values.py
+++ b/tensorflow/python/autograph/pyct/static_analysis/live_values.py
@@ -39,7 +39,8 @@ class LiveValueResolver(transformer.Base):
 
   def visit_ClassDef(self, node):
     self.generic_visit(node)
-    anno.setanno(node, 'live_val', self.entity_info.namespace[node.name])
+    anno.setanno(
+        node, 'live_val', self.transformer_ctx.entity_info.namespace[node.name])
     return node
 
   def visit_Name(self, node):
@@ -53,8 +54,8 @@ class LiveValueResolver(transformer.Base):
       if not is_defined:
         if node.id in self.literals:
           anno.setanno(node, 'live_val', self.literals[node.id])
-        elif node.id in self.entity_info.namespace:
-          obj = self.entity_info.namespace[node.id]
+        elif node.id in self.transformer_ctx.entity_info.namespace:
+          obj = self.transformer_ctx.entity_info.namespace[node.id]
           anno.setanno(node, 'live_val', obj)
           if hasattr(obj, '__name__'):
             anno.setanno(node, 'fqn', (obj.__name__,))
@@ -86,8 +87,8 @@ class LiveValueResolver(transformer.Base):
         def_, = defs
         # Note: param_of is a weakref.
         if def_.param_of and def_.param_of() is self.enclosing_entities[0]:
-          if node.id in self.entity_info.arg_values:
-            obj = self.entity_info.arg_values[node.id]
+          if node.id in self.transformer_ctx.entity_info.arg_values:
+            obj = self.transformer_ctx.entity_info.arg_values[node.id]
             anno.setanno(node, 'live_val', obj)
             anno.setanno(node, 'fqn', (obj.__class__.__name__,))
     return node
diff --git a/tensorflow/python/autograph/pyct/static_analysis/type_info.py b/tensorflow/python/autograph/pyct/static_analysis/type_info.py
index edb2ef0e27..9230457b2e 100644
--- a/tensorflow/python/autograph/pyct/static_analysis/type_info.py
+++ b/tensorflow/python/autograph/pyct/static_analysis/type_info.py
@@ -141,10 +141,11 @@ class TypeInfoResolver(transformer.Base):
     arg_name = str(qn)
     self.scope.setval(qn, arg_node)
     if (len(self.enclosing_entities) == 1 and
-        arg_name in self.entity_info.arg_types):
+        arg_name in self.transformer_ctx.entity_info.arg_types):
       # Forge a node to hold the type information, so that method calls on
       # it can resolve the type.
-      type_string, type_obj = self.entity_info.arg_types[arg_name]
+      type_string, type_obj = self.transformer_ctx.entity_info.arg_types[
+          arg_name]
       anno.setanno(arg_node, 'type', type_obj)
       anno.setanno(arg_node, 'type_fqn', tuple(type_string.split('.')))
 
diff --git a/tensorflow/python/autograph/pyct/transformer.py b/tensorflow/python/autograph/pyct/transformer.py
index 3578f38c65..8c019dd8f6 100644
--- a/tensorflow/python/autograph/pyct/transformer.py
+++ b/tensorflow/python/autograph/pyct/transformer.py
@@ -33,6 +33,23 @@ class AutoGraphParseError(SyntaxError):
   pass
 
 
+# TODO(znado): Use namedtuple.
+class Context(object):
+  """Contains information about a source code transformation.
+
+  This object is mutable, and is updated during conversion. Not thread safe.
+
+  Attributes:
+    entity_info: EntityInfo, immutable.
+    current_origin: origin_info.OriginInfo, holds the OriginInfo of the last
+      AST node to be processed successfully. Useful for error handling.
+  """
+
+  def __init__(self, entity_info):
+    self.entity_info = entity_info
+    self.current_origin = None
+
+
 # TODO(mdan): Use namedtuple.
 class EntityInfo(object):
   """Contains information about a Python entity.
@@ -208,11 +225,11 @@ class Base(gast.NodeTransformer):
     Args:
       entity_info: An EntityInfo object.
     """
-    self._current_origin = None
     self._lineno = 0
     self._col_offset = 0
-    # TODO(znado): remove this from the constructor of all Transformers.
-    self.entity_info = entity_info
+    # TODO(znado): make the constructor take a Context instead of an EntityInfo.
+    # TODO(mdan): Rename to self.ctx.
+    self.transformer_ctx = Context(entity_info)
     self._enclosing_entities = []
 
     # A stack that allows keeping mutable, scope-local state where scopes may be
@@ -446,8 +463,8 @@ class Base(gast.NodeTransformer):
     local_scope_size_at_entry = len(self._local_scope_state)
     processing_expr_node = False
 
+    parent_origin = self.transformer_ctx.current_origin
     try:
-      parent_origin = self._current_origin
       if isinstance(node, (gast.FunctionDef, gast.ClassDef, gast.Lambda)):
         did_enter_function = True
       elif isinstance(node, gast.Expr):
@@ -457,14 +474,15 @@ class Base(gast.NodeTransformer):
         self._enclosing_entities.append(node)
 
       if anno.hasanno(node, anno.Basic.ORIGIN):
-        self._current_origin = anno.getanno(node, anno.Basic.ORIGIN)
+        self.transformer_ctx.current_origin = anno.getanno(node,
+                                                           anno.Basic.ORIGIN)
 
       if processing_expr_node:
         entry_expr_value = node.value
 
       if not anno.hasanno(node, anno.Basic.SKIP_PROCESSING):
         result = super(Base, self).visit(node)
-      self._current_origin = parent_origin
+      self.transformer_ctx.current_origin = parent_origin
 
       # Adjust for consistency: replacing the value of an Expr with
       # an Assign node removes the need for the Expr node.
@@ -491,12 +509,13 @@ class Base(gast.NodeTransformer):
       return result
 
     except (ValueError, AttributeError, KeyError, NotImplementedError) as e:
-      if not self._current_origin:
+      if not self.transformer_ctx.current_origin:
         raise e
-      original_file_path = self._current_origin.loc.filename
-      original_line_number = self._current_origin.loc.lineno
-      original_col_offset = self._current_origin.loc.col_offset
-      original_source_line = self._current_origin.source_code_line
+      original_file_path = self.transformer_ctx.current_origin.loc.filename
+      original_line_number = self.transformer_ctx.current_origin.loc.lineno
+      original_col_offset = self.transformer_ctx.current_origin.loc.col_offset
+      original_source_line = (
+          self.transformer_ctx.current_origin.source_code_line)
       msg = '%s: %s.' % (e.__class__.__name__, str(e))
 
       # TODO(mdan): Avoid the printing of the original exception.
@@ -507,5 +526,3 @@ class Base(gast.NodeTransformer):
           AutoGraphParseError(msg, (original_file_path, original_line_number,
                                     original_col_offset, original_source_line)),
           sys.exc_info()[2])
-    finally:
-      self._current_origin = parent_origin
-- 
GitLab


From 80f47caee4bca375b8285e3b3185c7d29113b9e8 Mon Sep 17 00:00:00 2001
From: Benjamin Kramer <kramerb@google.com>
Date: Tue, 22 Jan 2019 09:19:46 -0800
Subject: [PATCH 1055/1095] [TF:XLA] Drop unused types.h include from
 executable_run_options.h

This is part of the tfcompile deps, and we don't want to have a dependency on
half of TensorFlow for AOT users, even though it's header-only.

PiperOrigin-RevId: 230352690
---
 tensorflow/compiler/xla/executable_run_options.h | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/tensorflow/compiler/xla/executable_run_options.h b/tensorflow/compiler/xla/executable_run_options.h
index ba3217f31b..6f36d11dfb 100644
--- a/tensorflow/compiler/xla/executable_run_options.h
+++ b/tensorflow/compiler/xla/executable_run_options.h
@@ -16,9 +16,6 @@ limitations under the License.
 #ifndef TENSORFLOW_COMPILER_XLA_EXECUTABLE_RUN_OPTIONS_H_
 #define TENSORFLOW_COMPILER_XLA_EXECUTABLE_RUN_OPTIONS_H_
 
-// Pulls in the ::stream_executor -> ::xla::se namespace alias.
-#include "tensorflow/compiler/xla/types.h"
-
 // These classes are forward declared so that ExecutableRunOptions can be linked
 // into an XLA-compiled binary without having to link all of the pointed-to
 // objects (e.g., for an ahead-of-time compiled CPU binary, the gpu tools don't
@@ -28,12 +25,6 @@ class Stream;
 class Platform;
 }  // namespace stream_executor
 
-namespace tensorflow {
-namespace thread {
-class ThreadPool;
-}  // namespace thread
-}  // namespace tensorflow
-
 namespace Eigen {
 struct ThreadPoolDevice;
 }  // namespace Eigen
-- 
GitLab


From 5d99fe9b657f34f22abf696e3d20d01f491af143 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Tue, 22 Jan 2019 09:44:08 -0800
Subject: [PATCH 1056/1095] Canonicalize Not(...) more aggressively in deadness
 analysis

This change makes deadness analysis canonicalize Not expressions more
aggressively using DeMorgan's law.  It uses this added power to more
aggressively detect And and Or expressions whose operands include a predicate
and its negation.

The main motivation for doing this is to simplify ~#true to #false and ~#false
to #true, but I can't test that directly before another related change lands.

PiperOrigin-RevId: 230356857
---
 tensorflow/compiler/jit/deadness_analysis.cc  | 131 ++++++++++++++++--
 .../compiler/jit/deadness_analysis_test.cc    |  39 ++++++
 2 files changed, 155 insertions(+), 15 deletions(-)

diff --git a/tensorflow/compiler/jit/deadness_analysis.cc b/tensorflow/compiler/jit/deadness_analysis.cc
index 0562838f62..9a2ada5b42 100644
--- a/tensorflow/compiler/jit/deadness_analysis.cc
+++ b/tensorflow/compiler/jit/deadness_analysis.cc
@@ -333,16 +333,19 @@ class PredicateFactory {
   }
 
   Predicate* MakeNotPredicate(Predicate* pred) {
-    SignatureForNot signature = pred;
-    auto it = interned_not_instances_.find(signature);
-    if (it == interned_not_instances_.end()) {
-      std::unique_ptr<Predicate> new_pred = Make<NotPredicate>(pred);
-      Predicate* new_pred_ptr = new_pred.get();
-      interned_not_instances_.emplace(signature, std::move(new_pred));
-      return new_pred_ptr;
-    } else {
-      return it->second.get();
+    auto it = make_not_predicate_cache_.find(pred);
+    if (it != make_not_predicate_cache_.end()) {
+      return it->second;
     }
+
+    Predicate* result = MakeNotPredicateImpl(pred);
+
+    bool insert_successful =
+        make_not_predicate_cache_.insert({pred, result}).second;
+    (void)insert_successful;
+    DCHECK(insert_successful);
+
+    return result;
   }
 
   Predicate* MakeAndRecurrencePredicate(Predicate* start, Predicate* step) {
@@ -378,7 +381,52 @@ class PredicateFactory {
   Predicate* MakeTrue() { return MakeAndPredicate({}); }
   Predicate* MakeFalse() { return MakeOrPredicate({}); }
 
+  ~PredicateFactory() {
+    DCHECK_EQ(stack_depth_, 0) << "Unnested IncrementStackDepth?";
+  }
+
  private:
+  Predicate* MakeNotPredicateImpl(Predicate* pred) {
+    IncrementStackDepth stack_frame(this);
+    if (!stack_frame.HasOverflowed()) {
+      if (Predicate* simplified = SimplifyUsingDeMorgan(pred)) {
+        return simplified;
+      }
+
+      // ~~A => A
+      if (auto* not_pred = dynamic_cast<NotPredicate*>(pred)) {
+        return not_pred->operand();
+      }
+    }
+
+    SignatureForNot signature = pred;
+    auto it = interned_not_instances_.find(signature);
+    if (it == interned_not_instances_.end()) {
+      std::unique_ptr<Predicate> new_pred = Make<NotPredicate>(pred);
+      Predicate* new_pred_ptr = new_pred.get();
+      interned_not_instances_.emplace(signature, std::move(new_pred));
+      return new_pred_ptr;
+    } else {
+      return it->second.get();
+    }
+  }
+
+  Predicate* SimplifyUsingDeMorgan(Predicate* pred) {
+    // ~(A & B & C & ...) => ~A | ~B | ~C | ~...
+    // ~(A | B | C | ...) -> ~A & ~B & ~C & ~...
+    Predicate::Kind kind = pred->kind();
+
+    if (kind == Predicate::Kind::kAnd || kind == Predicate::Kind::kOr) {
+      std::vector<Predicate*> new_operands;
+      absl::c_transform(pred->GetOperands(), std::back_inserter(new_operands),
+                        [&](Predicate* p) { return MakeNotPredicate(p); });
+      return kind == Predicate::Kind::kOr ? MakeAndPredicate(new_operands)
+                                          : MakeOrPredicate(new_operands);
+    }
+
+    return nullptr;
+  }
+
   template <typename PredicateT, typename... Args>
   std::unique_ptr<Predicate> Make(Args&&... args) {
     return std::unique_ptr<PredicateT>(
@@ -422,6 +470,36 @@ class PredicateFactory {
     }
   };
 
+  // Used to limit recursion to avoid blowing up the stack and cap compile time.
+  class IncrementStackDepth {
+   public:
+    explicit IncrementStackDepth(PredicateFactory* parent) : parent_(parent) {
+      parent_->stack_depth_++;
+    }
+
+    bool HasOverflowed() const {
+      const int kMaxStackDepth = 8;
+      return parent_->stack_depth_ >= kMaxStackDepth;
+    }
+
+    ~IncrementStackDepth() { parent_->stack_depth_--; }
+
+   private:
+    PredicateFactory* parent_;
+  };
+
+  // A cache for the MakeNotPredicate function.
+  //
+  // NB! This is *not* the same as `interned_not_instances_`.
+  // `interned_not_instances_` maps ensures pointer identity for `NotPredicate`
+  // instances, i.e., it ensures there at most one instance of Not(predicate)
+  // for any given predicate whereas `make_not_predicate_cache_` simply caches
+  // the result of the `MakeNotPredicate` function.  The values in
+  // `interned_not_instances_` are always instance of `NotPredicate` whereas the
+  // values in `make_not_predicate_cache_` may not be (for instance it will map
+  // Not(Not(A)) to A).
+  absl::flat_hash_map<Predicate*, Predicate*> make_not_predicate_cache_;
+
   absl::flat_hash_map<SignatureForAndOr, std::unique_ptr<Predicate>,
                       HashSignatureForAndOr>
       interned_and_or_instances_;
@@ -432,6 +510,7 @@ class PredicateFactory {
   absl::flat_hash_map<SignatureForSymbol, std::unique_ptr<Predicate>,
                       HashSignatureForSymbol>
       interned_symbol_instances_;
+  int stack_depth_ = 0;
 };
 
 Predicate* PredicateFactory::MakeInternedAndOr(
@@ -466,6 +545,13 @@ Predicate* PredicateFactory::MakeAndOrImpl(
     absl::Span<Predicate* const> operands, bool is_and) {
   Predicate::Kind pred_kind =
       is_and ? Predicate::Kind::kAnd : Predicate::Kind::kOr;
+
+  IncrementStackDepth stack_frame(this);
+  if (stack_frame.HasOverflowed()) {
+    return MakeInternedAndOr(
+        std::vector<Predicate*>(operands.begin(), operands.end()), pred_kind);
+  }
+
   Predicate::Kind other_pred_kind =
       is_and ? Predicate::Kind::kOr : Predicate::Kind::kAnd;
   absl::flat_hash_set<Predicate*> simplified_ops_set;
@@ -494,16 +580,31 @@ Predicate* PredicateFactory::MakeAndOrImpl(
 
   // Simplify "A&~A=>False" and "A|~A=>True".
   absl::flat_hash_set<Predicate*> negated_ops;
-  for (Predicate* op : simplified_ops) {
-    if (op->kind() == Predicate::Kind::kNot) {
-      negated_ops.insert(dynamic_cast<NotPredicate&>(*op).operand());
-    }
-  }
-
   for (Predicate* op : simplified_ops) {
     if (negated_ops.count(op)) {
+      // Simple case:
+      //
+      //   A & ~A & ... == False
+      //   A | ~A | ... == True
       return is_and ? MakeFalse() : MakeTrue();
     }
+
+    Predicate* negated_op = MakeNotPredicate(op);
+    if (negated_op->kind() == pred_kind) {
+      // Slightly more complicated case:
+      //
+      //   (~A | ~B | ~C) & A & B & C & ... ==
+      //   ~(A & B & C) & (A & B & C) & ... == False
+      //
+      //   (~A & ~B & ~C) | A | B | C | ... ==
+      //   ~(A | B | C) | (A | B | C) | ... == True
+      if (absl::c_all_of(negated_op->GetOperands(), [&](Predicate* p) {
+            return simplified_ops_set.contains(p);
+          })) {
+        return is_and ? MakeFalse() : MakeTrue();
+      }
+    }
+    negated_ops.insert(negated_op);
   }
 
   // If all ops contain the same subop, then factor it out thanks to the
diff --git a/tensorflow/compiler/jit/deadness_analysis_test.cc b/tensorflow/compiler/jit/deadness_analysis_test.cc
index 8a73101c18..11ee00ac6b 100644
--- a/tensorflow/compiler/jit/deadness_analysis_test.cc
+++ b/tensorflow/compiler/jit/deadness_analysis_test.cc
@@ -818,5 +818,44 @@ TEST(DeadnessAnalysisTest, RecvVsSwitchText) {
   EXPECT_EQ(predicate_map[logical_and_output_0], "(recv:0 & *recv:0)");
 }
 
+TEST(DeadnessAnalysisTest, DeMorgan) {
+  Scope root = Scope::NewRootScope().ExitOnError();
+
+  Output cond_0 = ops::Placeholder(root.WithOpName("cond_0"), DT_BOOL);
+  Output cond_1 = ops::Placeholder(root.WithOpName("cond_1"), DT_BOOL);
+  Output value = ops::Placeholder(root.WithOpName("value"), DT_FLOAT);
+
+  ops::Switch sw_0(root.WithOpName("switch_0"), value, cond_0);
+  ops::Switch sw_1(root.WithOpName("switch_1"), value, cond_1);
+
+  Output and_0_1 =
+      ops::Add(root.WithOpName("and_0_1"), sw_0.output_true, sw_1.output_true);
+
+  Output or_not0_not1 = ops::Merge(root.WithOpName("or_not0_not1"),
+                                   {sw_0.output_false, sw_1.output_false})
+                            .output;
+
+  // Predicate(should_always_be_dead) =
+  // (A & B) & (~A | ~B) = (A & B) & ~(A & B) = False
+  Output should_always_be_dead =
+      ops::Add(root.WithOpName("should_always_be_dead"), and_0_1, or_not0_not1);
+
+  // Predicate(should_always_be_dead) =
+  // (A & B) | (~A | ~B) = (A & B) | ~(A & B) = True
+  Output should_always_be_alive =
+      ops::Merge(root.WithOpName("should_always_be_alive"),
+                 {and_0_1, or_not0_not1})
+          .output;
+
+  std::unique_ptr<DeadnessAnalysis> result;
+  TF_ASSERT_OK(AnalyzeDeadness(root.graph(), &result));
+
+  PredicateMapTy predicate_map;
+  TF_ASSERT_OK(ComputePredicates(*root.graph(), &predicate_map));
+
+  EXPECT_EQ(predicate_map[ControlOutputFor(should_always_be_dead)], "#false");
+  EXPECT_EQ(predicate_map[ControlOutputFor(should_always_be_alive)], "#true");
+}
+
 }  // namespace
 }  // namespace tensorflow
-- 
GitLab


From 5731073b59c56e4d7bfc81b153153d4665f76bf5 Mon Sep 17 00:00:00 2001
From: Shanqing Cai <cais@google.com>
Date: Tue, 22 Jan 2019 09:44:23 -0800
Subject: [PATCH 1057/1095] Eager SPINN example: Replace py_binary with
 py_library

PiperOrigin-RevId: 230356890
---
 third_party/examples/eager/spinn/BUILD | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/third_party/examples/eager/spinn/BUILD b/third_party/examples/eager/spinn/BUILD
index 0e39d4696f..640bcb230c 100644
--- a/third_party/examples/eager/spinn/BUILD
+++ b/third_party/examples/eager/spinn/BUILD
@@ -1,6 +1,6 @@
 licenses(["notice"])  # 3-clause BSD.
 
-py_binary(
+py_library(
     name = "spinn",
     srcs = ["spinn.py"],
     srcs_version = "PY2AND3",
-- 
GitLab


From 4aaca394a3c86ac2c0db0e749a569c1395376fd2 Mon Sep 17 00:00:00 2001
From: Karim Nosir <karimnosseir@google.com>
Date: Tue, 22 Jan 2019 09:51:03 -0800
Subject: [PATCH 1058/1095] Remove unused dependencies.

PiperOrigin-RevId: 230357984
---
 tensorflow/lite/experimental/c/BUILD                      | 2 --
 tensorflow/lite/experimental/kernels/BUILD                | 8 --------
 tensorflow/lite/experimental/micro/BUILD                  | 1 -
 .../lite/experimental/micro/examples/micro_speech/BUILD   | 3 ---
 tensorflow/lite/experimental/micro/kernels/BUILD          | 2 --
 tensorflow/lite/experimental/writer/BUILD                 | 1 -
 tensorflow/lite/kernels/BUILD                             | 2 --
 tensorflow/lite/kernels/internal/BUILD                    | 3 ---
 tensorflow/lite/profiling/BUILD                           | 1 -
 tensorflow/lite/tools/benchmark/BUILD                     | 4 ----
 10 files changed, 27 deletions(-)

diff --git a/tensorflow/lite/experimental/c/BUILD b/tensorflow/lite/experimental/c/BUILD
index cde53e2838..2f0f4327b7 100644
--- a/tensorflow/lite/experimental/c/BUILD
+++ b/tensorflow/lite/experimental/c/BUILD
@@ -66,7 +66,6 @@ cc_library(
         ":c_api_internal",
         "//tensorflow/lite:context",
         "//tensorflow/lite:framework",
-        "//tensorflow/lite:schema_fbs_version",
         "//tensorflow/lite/kernels:builtin_ops",
     ],
 )
@@ -94,7 +93,6 @@ cc_test(
     deps = [
         ":c_api",
         "//tensorflow/lite:context",
-        "//tensorflow/lite:kernel_api",
         "//tensorflow/lite/testing:util",
         "@com_google_googletest//:gtest",
     ],
diff --git a/tensorflow/lite/experimental/kernels/BUILD b/tensorflow/lite/experimental/kernels/BUILD
index dd314545cb..5d2337f2e2 100644
--- a/tensorflow/lite/experimental/kernels/BUILD
+++ b/tensorflow/lite/experimental/kernels/BUILD
@@ -50,21 +50,13 @@ cc_library(
     }),
     deps = [
         ":ctc_utils",
-        "//tensorflow/lite:builtin_op_data",
         "//tensorflow/lite:framework",
-        "//tensorflow/lite:string_util",
         "//tensorflow/lite/c:c_api_internal",
-        "//tensorflow/lite/kernels:builtin_ops",
-        "//tensorflow/lite/kernels:gemm_support",
         "//tensorflow/lite/kernels:kernel_util",
         "//tensorflow/lite/kernels:op_macros",
-        "//tensorflow/lite/kernels/internal:kernel_utils",
         "//tensorflow/lite/kernels/internal:optimized",
         "//tensorflow/lite/kernels/internal:optimized_base",
-        "//tensorflow/lite/kernels/internal:quantization_util",
-        "//tensorflow/lite/kernels/internal:reference_base",
         "//tensorflow/lite/kernels/internal:tensor",
-        "//tensorflow/lite/kernels/internal:tensor_utils",
         "@flatbuffers",
     ],
 )
diff --git a/tensorflow/lite/experimental/micro/BUILD b/tensorflow/lite/experimental/micro/BUILD
index 2d00ef76f4..b16b8b49f8 100644
--- a/tensorflow/lite/experimental/micro/BUILD
+++ b/tensorflow/lite/experimental/micro/BUILD
@@ -29,7 +29,6 @@ cc_library(
         "simple_tensor_allocator.h",
     ],
     deps = [
-        "//tensorflow/lite:schema_fbs_version",
         "//tensorflow/lite/c:c_api_internal",
         "//tensorflow/lite/core/api",
         "//tensorflow/lite/schema:schema_fbs",
diff --git a/tensorflow/lite/experimental/micro/examples/micro_speech/BUILD b/tensorflow/lite/experimental/micro/examples/micro_speech/BUILD
index 171d07cf56..51ba2976a2 100644
--- a/tensorflow/lite/experimental/micro/examples/micro_speech/BUILD
+++ b/tensorflow/lite/experimental/micro/examples/micro_speech/BUILD
@@ -235,15 +235,12 @@ cc_binary(
     deps = [
         ":audio_provider",
         ":feature_provider",
-        ":features_test_data",
         ":model_settings",
-        ":preprocessor_reference",
         ":recognize_commands",
         ":tiny_conv_model_data",
         "//tensorflow/lite:schema_fbs_version",
         "//tensorflow/lite/experimental/micro:micro_framework",
         "//tensorflow/lite/experimental/micro/kernels:all_ops_resolver",
-        "//tensorflow/lite/experimental/micro/kernels:micro_ops",
         "//tensorflow/lite/schema:schema_fbs",
     ],
 )
diff --git a/tensorflow/lite/experimental/micro/kernels/BUILD b/tensorflow/lite/experimental/micro/kernels/BUILD
index 47ac85c605..e2d3164d4c 100644
--- a/tensorflow/lite/experimental/micro/kernels/BUILD
+++ b/tensorflow/lite/experimental/micro/kernels/BUILD
@@ -22,7 +22,6 @@ cc_library(
     copts = tflite_copts(),
     deps = [
         "//tensorflow/lite/c:c_api_internal",
-        "//tensorflow/lite/experimental/micro:micro_framework",
         "//tensorflow/lite/kernels:kernel_util",
         "//tensorflow/lite/kernels:op_macros",
         "//tensorflow/lite/kernels:padding",
@@ -43,7 +42,6 @@ cc_library(
     copts = tflite_copts(),
     deps = [
         ":micro_ops",
-        "//tensorflow/lite/c:c_api_internal",
         "//tensorflow/lite/experimental/micro:micro_framework",
     ],
 )
diff --git a/tensorflow/lite/experimental/writer/BUILD b/tensorflow/lite/experimental/writer/BUILD
index 57ce636367..6aa81ff173 100644
--- a/tensorflow/lite/experimental/writer/BUILD
+++ b/tensorflow/lite/experimental/writer/BUILD
@@ -33,7 +33,6 @@ cc_library(
         "//tensorflow/lite:builtin_op_data",
         "//tensorflow/lite:framework",
         "//tensorflow/lite:schema_fbs_version",
-        "//tensorflow/lite/kernels:builtin_ops",
         "//tensorflow/lite/schema:schema_fbs_with_reflection",
     ],
 )
diff --git a/tensorflow/lite/kernels/BUILD b/tensorflow/lite/kernels/BUILD
index be6e5d9ba5..765d7a6731 100644
--- a/tensorflow/lite/kernels/BUILD
+++ b/tensorflow/lite/kernels/BUILD
@@ -111,7 +111,6 @@ cc_library(
     deps = [
         "//tensorflow/lite/c:c_api_internal",
         "//tensorflow/lite/kernels/internal:round",
-        "//tensorflow/lite/kernels/internal:types",
     ],
 )
 
@@ -286,7 +285,6 @@ cc_library(
     srcs = ["register_ref.cc"],
     hdrs = ["register_ref.h"],
     deps = [
-        ":builtin_op_kernels",
         "//tensorflow/lite:framework",
         "//tensorflow/lite:util",
         "//tensorflow/lite/c:c_api_internal",
diff --git a/tensorflow/lite/kernels/internal/BUILD b/tensorflow/lite/kernels/internal/BUILD
index 1ff5164d2c..c736e5d640 100644
--- a/tensorflow/lite/kernels/internal/BUILD
+++ b/tensorflow/lite/kernels/internal/BUILD
@@ -46,7 +46,6 @@ cc_library(
     ],
     deps = [
         "//tensorflow/lite/kernels:op_macros",
-        "@com_google_absl//absl/base:core_headers",
     ],
 )
 
@@ -60,7 +59,6 @@ cc_library(
     ],
     deps = [
         "//tensorflow/lite/kernels:op_macros",
-        "@com_google_absl//absl/base:core_headers",
     ],
 )
 
@@ -546,7 +544,6 @@ cc_library(
     }),
     deps = [
         ":types",
-        "//tensorflow/lite:string",
     ],
 )
 
diff --git a/tensorflow/lite/profiling/BUILD b/tensorflow/lite/profiling/BUILD
index 64723ba385..bbc252045b 100644
--- a/tensorflow/lite/profiling/BUILD
+++ b/tensorflow/lite/profiling/BUILD
@@ -74,7 +74,6 @@ tf_cc_test(
         ":profile_summarizer",
         "//tensorflow/lite:framework",
         "//tensorflow/lite:schema_fbs_version",
-        "//tensorflow/lite/kernels:builtin_ops",
         "//tensorflow/lite/kernels:kernel_util",
         "//tensorflow/lite/kernels:test_util",
         "//tensorflow/lite/testing:util",
diff --git a/tensorflow/lite/tools/benchmark/BUILD b/tensorflow/lite/tools/benchmark/BUILD
index e68997a00b..ce31eaf42f 100644
--- a/tensorflow/lite/tools/benchmark/BUILD
+++ b/tensorflow/lite/tools/benchmark/BUILD
@@ -141,10 +141,6 @@ cc_library(
         ":logging",
         "//tensorflow/core:stats_calculator_portable",
         "//tensorflow/lite:framework",
-        "//tensorflow/lite:string_util",
-        "//tensorflow/lite/kernels:builtin_ops",
-        "//tensorflow/lite/profiling:profile_summarizer",
-        "//tensorflow/lite/profiling:profiler",
         "//tensorflow/lite/profiling:time",
     ],
 )
-- 
GitLab


From 7d11353dbc2d7f83f154d01eeb2f127eeeb785bf Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 22 Jan 2019 09:59:01 -0800
Subject: [PATCH 1059/1095]  Update documentation for
 GraphOptimizer::Optimize's return value.

PiperOrigin-RevId: 230359499
---
 tensorflow/core/grappler/optimizers/graph_optimizer.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/grappler/optimizers/graph_optimizer.h b/tensorflow/core/grappler/optimizers/graph_optimizer.h
index e587a2b2af..44dfe0de78 100644
--- a/tensorflow/core/grappler/optimizers/graph_optimizer.h
+++ b/tensorflow/core/grappler/optimizers/graph_optimizer.h
@@ -39,7 +39,7 @@ class GraphOptimizer {
   // Routine called to allow an algorithm to propose a rewritten graph
   // for the graph, feeds and fetches in "item" to run more efficiently
   // on "cluster".
-  // Returns true iff it managed to generate a solution, false otherwise.
+  // Returns an error status if it failed to generate a solution.
   virtual Status Optimize(Cluster* cluster, const GrapplerItem& item,
                           GraphDef* optimized_graph) = 0;
 
-- 
GitLab


From 7df1985f651268cd02ba9011b4295412e7d635cf Mon Sep 17 00:00:00 2001
From: Saurabh Saxena <srbs@google.com>
Date: Tue, 22 Jan 2019 10:13:12 -0800
Subject: [PATCH 1060/1095] Handle fetching uninitialized tensors from
 TensorLists.

This is needed to support use-cases of the form:

c = constant([1., 2.])
l = tensor_list_from_tensor(c, [])
t = tensor_list_get_item(l, 0)
dt = gradients(t, c)

In the above example gradient of the TensorListFromTensor op, a TensorListStack op, would attempt to stack a list with DT_INVALID type tensors. Since we only read index 0, the gradient list has only index 0 set.

This change replaces the DT_INVALID tensors with zeros of the appropriate shape and type. This change adds an `element_shape` input to TensorListGetItem, TensorListPopBack, TensorListStack and TensorListGather ops. This is the shape used to build a zeros tensor for a DT_INVALID tensor in the list. Returning zeros for DT_VARIANT (nested lists) type tensors is not supported yet.

I will add support for TensorListConcat in a follow-up change.

PiperOrigin-RevId: 230362520
---
 tensorflow/core/kernels/fill_functor.cu.cc    |   6 +
 tensorflow/core/kernels/list_kernels.cc       | 194 +++---------
 tensorflow/core/kernels/list_kernels.cu.cc    |  65 ++--
 tensorflow/core/kernels/list_kernels.h        | 289 ++++++++++++------
 .../core/ops/compat/ops_history.v1.pbtxt      |  16 +
 tensorflow/core/ops/list_ops.cc               |   4 +
 tensorflow/core/ops/ops.pbtxt                 |  16 +
 .../python/kernel_tests/list_ops_test.py      | 178 ++++++++++-
 .../kernel_tests/tensor_array_ops_test.py     |  47 ++-
 tensorflow/python/ops/list_ops.py             |  74 ++++-
 10 files changed, 590 insertions(+), 299 deletions(-)

diff --git a/tensorflow/core/kernels/fill_functor.cu.cc b/tensorflow/core/kernels/fill_functor.cu.cc
index 050c95cf40..946d5c9b02 100644
--- a/tensorflow/core/kernels/fill_functor.cu.cc
+++ b/tensorflow/core/kernels/fill_functor.cu.cc
@@ -88,6 +88,12 @@ struct SetZeroFunctor<GPUDevice, T> {
   }
 };
 
+template <>
+void SetZeroFunctor<GPUDevice, Variant>::operator()(
+    const GPUDevice& d, typename TTypes<Variant>::Flat out) {
+  // TODO(b/123028789): Implement this.
+}
+
 #define DEFINE_SETZERO_GPU(T) template struct SetZeroFunctor<GPUDevice, T>;
 TF_CALL_NUMBER_TYPES(DEFINE_SETZERO_GPU);
 TF_CALL_bool(DEFINE_SETZERO_GPU);
diff --git a/tensorflow/core/kernels/list_kernels.cc b/tensorflow/core/kernels/list_kernels.cc
index 5859c20d89..7c0b70d651 100644
--- a/tensorflow/core/kernels/list_kernels.cc
+++ b/tensorflow/core/kernels/list_kernels.cc
@@ -173,6 +173,17 @@ Status TensorShapeFromTensor(const Tensor& t, PartialTensorShape* out) {
       DataTypeString(t.dtype()));
 }
 
+Status GetElementShapeFromInput(OpKernelContext* c,
+                                const TensorList& tensor_list, int index,
+                                PartialTensorShape* element_shape) {
+  TF_RETURN_IF_ERROR(TensorShapeFromTensor(c->input(index), element_shape));
+  // Check that `element_shape` and `tensor_list.element_shape` are
+  // compatible and store the merged shape in `element_shape`.
+  PartialTensorShape tmp = *element_shape;
+  TF_RETURN_IF_ERROR(tmp.MergeWith(tensor_list.element_shape, element_shape));
+  return Status::OK();
+}
+
 class EmptyTensorList : public OpKernel {
  public:
   explicit EmptyTensorList(OpKernelConstruction* ctx) : OpKernel(ctx) {
@@ -358,69 +369,6 @@ class TensorListElementShape : public OpKernel {
 REGISTER_KERNEL_BUILDER(Name("TensorListElementShape").Device(DEVICE_CPU),
                         TensorListElementShape);
 
-#if GOOGLE_CUDA
-
-REGISTER_KERNEL_BUILDER(Name("TensorListElementShape")
-                            .Device(DEVICE_GPU)
-                            .HostMemory("element_shape"),
-                        TensorListElementShape);
-
-#endif  // GOOGLE_CUDA
-
-class TensorListPopBack : public OpKernel {
- public:
-  explicit TensorListPopBack(OpKernelConstruction* c) : OpKernel(c) {
-    OP_REQUIRES_OK(c, c->GetAttr("element_dtype", &element_dtype_));
-  }
-
-  ~TensorListPopBack() override {}
-
-  void Compute(OpKernelContext* c) override {
-    const TensorList* l = c->input(0).scalar<Variant>()().get<TensorList>();
-    OP_REQUIRES(c, l != nullptr,
-                errors::InvalidArgument(
-                    "Input handle is not a list. Saw: '",
-                    c->input(0).scalar<Variant>()().DebugString(), "'"));
-    OP_REQUIRES(c, element_dtype_ == l->element_dtype,
-                errors::InvalidArgument("Invalid data types; op elements ",
-                                        DataTypeString(element_dtype_),
-                                        " but list elements ",
-                                        DataTypeString(l->element_dtype)));
-
-    OP_REQUIRES(c, !l->tensors.empty(),
-                errors::InvalidArgument("Trying to pop from an empty list."));
-
-    c->set_output(1, l->tensors.back());
-    AllocatorAttributes attr;
-    attr.set_on_host(true);
-    std::unique_ptr<Tensor> maybe_result = c->forward_input(
-        0, 0, DT_VARIANT, TensorShape{}, c->input_memory_type(0), attr);
-    if (maybe_result != nullptr) {
-      maybe_result->scalar<Variant>()().get<TensorList>()->tensors.pop_back();
-    } else {
-      TensorList output;
-      output = *l;
-      output.tensors.pop_back();
-      Tensor* result;
-      OP_REQUIRES_OK(c, c->allocate_output(0, TensorShape{}, &result, attr));
-      result->scalar<Variant>()() = std::move(output);
-    }
-  }
-
- private:
-  DataType element_dtype_;
-};
-
-REGISTER_KERNEL_BUILDER(Name("TensorListPopBack").Device(DEVICE_CPU),
-                        TensorListPopBack);
-
-#if GOOGLE_CUDA
-
-REGISTER_KERNEL_BUILDER(Name("TensorListPopBack").Device(DEVICE_GPU),
-                        TensorListPopBack);
-
-#endif  // GOOGLE_CUDA
-
 class TensorListReserve : public OpKernel {
  public:
   explicit TensorListReserve(OpKernelConstruction* c) : OpKernel(c) {
@@ -458,61 +406,6 @@ REGISTER_KERNEL_BUILDER(Name("TensorListReserve")
                         TensorListReserve);
 
 #endif  // GOOGLE_CUDA
-
-class TensorListGetItem : public OpKernel {
- public:
-  explicit TensorListGetItem(OpKernelConstruction* c) : OpKernel(c) {
-    OP_REQUIRES_OK(c, c->GetAttr("element_dtype", &element_dtype_));
-  }
-
-  void Compute(OpKernelContext* c) override {
-    OP_REQUIRES(
-        c, c->input(0).shape().num_elements() == 1,
-        errors::InvalidArgument("List tensors are supposed to be scalars."));
-    const TensorList* l = c->input(0).scalar<Variant>()().get<TensorList>();
-    OP_REQUIRES(c, l != nullptr,
-                errors::InvalidArgument(
-                    "Input handle is not a list. Saw: '",
-                    c->input(0).scalar<Variant>()().DebugString(), "'"));
-    OP_REQUIRES(c, element_dtype_ == l->element_dtype,
-                errors::InvalidArgument("Invalid data types; op elements ",
-                                        DataTypeString(element_dtype_),
-                                        " but list elements ",
-                                        DataTypeString(l->element_dtype)));
-    int32 index = c->input(1).scalar<int32>()();
-    OP_REQUIRES(c, index < l->tensors.size(),
-                errors::InvalidArgument("Trying to access element ", index,
-                                        " in a list with ", l->tensors.size(),
-                                        " elements."));
-    c->set_output(0, l->tensors[index]);
-  }
-
- private:
-  DataType element_dtype_;
-};
-
-REGISTER_KERNEL_BUILDER(Name("TensorListGetItem").Device(DEVICE_CPU),
-                        TensorListGetItem);
-
-#if GOOGLE_CUDA
-
-#define REGISTER_TENSOR_LIST_GET_ITEM_GPU(T)                      \
-  REGISTER_KERNEL_BUILDER(Name("TensorListGetItem")               \
-                              .TypeConstraint<T>("element_dtype") \
-                              .Device(DEVICE_GPU)                 \
-                              .HostMemory("index"),               \
-                          TensorListGetItem);
-
-TF_CALL_GPU_NUMBER_TYPES(REGISTER_TENSOR_LIST_GET_ITEM_GPU);
-TF_CALL_complex64(REGISTER_TENSOR_LIST_GET_ITEM_GPU);
-TF_CALL_complex128(REGISTER_TENSOR_LIST_GET_ITEM_GPU);
-TF_CALL_int64(REGISTER_TENSOR_LIST_GET_ITEM_GPU);
-REGISTER_TENSOR_LIST_GET_ITEM_GPU(bfloat16)
-REGISTER_TENSOR_LIST_GET_ITEM_GPU(bool)
-#undef REGISTER_TENSOR_LIST_GET_ITEM_GPU
-
-#endif  // GOOGLE_CUDA
-
 class TensorListResize : public OpKernel {
  public:
   explicit TensorListResize(OpKernelConstruction* c) : OpKernel(c) {}
@@ -736,23 +629,7 @@ REGISTER_KERNEL_BUILDER(Name("TensorListConcatLists").Device(DEVICE_GPU),
 
 #endif  // GOOGLE_CUDA
 
-#define REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU(T)               \
-  REGISTER_KERNEL_BUILDER(Name("TensorListPushBackBatch")         \
-                              .TypeConstraint<T>("element_dtype") \
-                              .Device(DEVICE_CPU),                \
-                          TensorListPushBackBatch<CPUDevice, T>)
-
-TF_CALL_ALL_TYPES(REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU);
-REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU(quint8);
-REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU(qint8);
-REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU(quint16);
-REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU(qint16);
-REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU(qint32);
-REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU(bfloat16);
-
-#undef REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU
-
-#define REGISTER_TENSOR_LIST_STACK_CPU(T)                         \
+#define REGISTER_TENSOR_LIST_OPS_CPU(T)                           \
   REGISTER_KERNEL_BUILDER(Name("TensorListStack")                 \
                               .TypeConstraint<T>("element_dtype") \
                               .Device(DEVICE_CPU),                \
@@ -764,19 +641,15 @@ REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_CPU(bfloat16);
   REGISTER_KERNEL_BUILDER(Name("TensorListConcat")                \
                               .TypeConstraint<T>("element_dtype") \
                               .Device(DEVICE_CPU),                \
-                          TensorListConcat<CPUDevice, T>)
-
-TF_CALL_POD_STRING_TYPES(REGISTER_TENSOR_LIST_STACK_CPU);
-REGISTER_TENSOR_LIST_STACK_CPU(quint8);
-REGISTER_TENSOR_LIST_STACK_CPU(qint8);
-REGISTER_TENSOR_LIST_STACK_CPU(quint16);
-REGISTER_TENSOR_LIST_STACK_CPU(qint16);
-REGISTER_TENSOR_LIST_STACK_CPU(qint32);
-REGISTER_TENSOR_LIST_STACK_CPU(bfloat16);
-
-#undef REGISTER_TENSOR_LIST_STACK_CPU
-
-#define REGISTER_TENSOR_LIST_FROM_TENSOR_CPU(T)                   \
+                          TensorListConcat<CPUDevice, T>)         \
+  REGISTER_KERNEL_BUILDER(Name("TensorListGetItem")               \
+                              .TypeConstraint<T>("element_dtype") \
+                              .Device(DEVICE_CPU),                \
+                          TensorListGetItem<CPUDevice, T>)        \
+  REGISTER_KERNEL_BUILDER(Name("TensorListPopBack")               \
+                              .TypeConstraint<T>("element_dtype") \
+                              .Device(DEVICE_CPU),                \
+                          TensorListPopBack<CPUDevice, T>)        \
   REGISTER_KERNEL_BUILDER(Name("TensorListFromTensor")            \
                               .TypeConstraint<T>("element_dtype") \
                               .Device(DEVICE_CPU),                \
@@ -788,17 +661,24 @@ REGISTER_TENSOR_LIST_STACK_CPU(bfloat16);
   REGISTER_KERNEL_BUILDER(Name("TensorListSplit")                 \
                               .TypeConstraint<T>("element_dtype") \
                               .Device(DEVICE_CPU),                \
-                          TensorListSplit<CPUDevice, T>)
+                          TensorListSplit<CPUDevice, T>)          \
+  REGISTER_KERNEL_BUILDER(Name("TensorListPushBackBatch")         \
+                              .TypeConstraint<T>("element_dtype") \
+                              .Device(DEVICE_CPU),                \
+                          TensorListPushBackBatch<CPUDevice, T>)
+
+TF_CALL_POD_STRING_TYPES(REGISTER_TENSOR_LIST_OPS_CPU);
+REGISTER_TENSOR_LIST_OPS_CPU(quint8);
+REGISTER_TENSOR_LIST_OPS_CPU(qint8);
+REGISTER_TENSOR_LIST_OPS_CPU(quint16);
+REGISTER_TENSOR_LIST_OPS_CPU(qint16);
+REGISTER_TENSOR_LIST_OPS_CPU(qint32);
+REGISTER_TENSOR_LIST_OPS_CPU(bfloat16);
+REGISTER_TENSOR_LIST_OPS_CPU(Variant);
 
-TF_CALL_POD_STRING_TYPES(REGISTER_TENSOR_LIST_FROM_TENSOR_CPU);
-REGISTER_TENSOR_LIST_FROM_TENSOR_CPU(quint8);
-REGISTER_TENSOR_LIST_FROM_TENSOR_CPU(qint8);
-REGISTER_TENSOR_LIST_FROM_TENSOR_CPU(quint16);
-REGISTER_TENSOR_LIST_FROM_TENSOR_CPU(qint16);
-REGISTER_TENSOR_LIST_FROM_TENSOR_CPU(qint32);
-REGISTER_TENSOR_LIST_FROM_TENSOR_CPU(bfloat16);
+#undef REGISTER_TENSOR_LIST_OPS_CPU
 
-#undef REGISTER_TENSOR_LIST_FROM_TENSOR_CPU
+#define REGISTER_TENSOR_LIST_OPS_CPU(T)
 
 REGISTER_UNARY_VARIANT_BINARY_OP_FUNCTION(ADD_VARIANT_BINARY_OP, DEVICE_CPU,
                                           TensorList,
diff --git a/tensorflow/core/kernels/list_kernels.cu.cc b/tensorflow/core/kernels/list_kernels.cu.cc
index 23f552642c..c3a5041b00 100644
--- a/tensorflow/core/kernels/list_kernels.cu.cc
+++ b/tensorflow/core/kernels/list_kernels.cu.cc
@@ -36,47 +36,38 @@ namespace tensorflow {
 
 typedef Eigen::GpuDevice GPUDevice;
 
-#define REGISTER_TENSOR_LIST_STACK_GPU(T)                         \
+#define REGISTER_TENSOR_LIST_OPS_GPU(T)                           \
   REGISTER_KERNEL_BUILDER(Name("TensorListStack")                 \
                               .TypeConstraint<T>("element_dtype") \
-                              .Device(DEVICE_GPU),                \
+                              .Device(DEVICE_GPU)                 \
+                              .HostMemory("element_shape"),       \
                           TensorListStack<GPUDevice, T>)          \
   REGISTER_KERNEL_BUILDER(Name("TensorListGather")                \
                               .TypeConstraint<T>("element_dtype") \
                               .Device(DEVICE_GPU)                 \
-                              .HostMemory("indices"),             \
+                              .HostMemory("indices")              \
+                              .HostMemory("element_shape"),       \
                           TensorListGather<GPUDevice, T>)         \
+  REGISTER_KERNEL_BUILDER(Name("TensorListGetItem")               \
+                              .TypeConstraint<T>("element_dtype") \
+                              .Device(DEVICE_GPU)                 \
+                              .HostMemory("index")                \
+                              .HostMemory("element_shape"),       \
+                          TensorListGetItem<GPUDevice, T>)        \
+  REGISTER_KERNEL_BUILDER(Name("TensorListPopBack")               \
+                              .TypeConstraint<T>("element_dtype") \
+                              .Device(DEVICE_GPU)                 \
+                              .HostMemory("element_shape"),       \
+                          TensorListPopBack<GPUDevice, T>)        \
   REGISTER_KERNEL_BUILDER(Name("TensorListConcat")                \
                               .TypeConstraint<T>("element_dtype") \
                               .Device(DEVICE_GPU)                 \
                               .HostMemory("lengths"),             \
-                          TensorListConcat<GPUDevice, T>)
-
-TF_CALL_GPU_NUMBER_TYPES(REGISTER_TENSOR_LIST_STACK_GPU);
-REGISTER_TENSOR_LIST_STACK_GPU(bfloat16);
-TF_CALL_complex64(REGISTER_TENSOR_LIST_STACK_GPU);
-TF_CALL_complex128(REGISTER_TENSOR_LIST_STACK_GPU);
-TF_CALL_int64(REGISTER_TENSOR_LIST_STACK_GPU);
-REGISTER_TENSOR_LIST_STACK_GPU(bool);
-
-#undef REGISTER_TENSOR_LIST_STACK_GPU
-
-#define REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_GPU(T)               \
+                          TensorListConcat<GPUDevice, T>)         \
   REGISTER_KERNEL_BUILDER(Name("TensorListPushBackBatch")         \
                               .TypeConstraint<T>("element_dtype") \
                               .Device(DEVICE_GPU),                \
-                          TensorListPushBackBatch<GPUDevice, T>)
-
-TF_CALL_GPU_NUMBER_TYPES(REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_GPU);
-REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_GPU(bfloat16);
-TF_CALL_complex64(REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_GPU);
-TF_CALL_complex128(REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_GPU);
-TF_CALL_int64(REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_GPU);
-REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_GPU(bool);
-
-#undef REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_GPU
-
-#define REGISTER_TENSOR_LIST_FROM_TENSOR_GPU(T)                   \
+                          TensorListPushBackBatch<GPUDevice, T>)  \
   REGISTER_KERNEL_BUILDER(Name("TensorListFromTensor")            \
                               .TypeConstraint<T>("element_dtype") \
                               .Device(DEVICE_GPU)                 \
@@ -95,14 +86,20 @@ REGISTER_TENSOR_LIST_PUSH_BACK_BATCH_GPU(bool);
                               .HostMemory("lengths"),             \
                           TensorListSplit<GPUDevice, T>)
 
-TF_CALL_GPU_NUMBER_TYPES(REGISTER_TENSOR_LIST_FROM_TENSOR_GPU);
-REGISTER_TENSOR_LIST_FROM_TENSOR_GPU(bfloat16);
-TF_CALL_complex64(REGISTER_TENSOR_LIST_FROM_TENSOR_GPU);
-TF_CALL_complex128(REGISTER_TENSOR_LIST_FROM_TENSOR_GPU);
-TF_CALL_int64(REGISTER_TENSOR_LIST_FROM_TENSOR_GPU);
-REGISTER_TENSOR_LIST_FROM_TENSOR_GPU(bool);
+TF_CALL_GPU_NUMBER_TYPES(REGISTER_TENSOR_LIST_OPS_GPU);
+REGISTER_TENSOR_LIST_OPS_GPU(bfloat16);
+TF_CALL_complex64(REGISTER_TENSOR_LIST_OPS_GPU);
+TF_CALL_complex128(REGISTER_TENSOR_LIST_OPS_GPU);
+TF_CALL_int64(REGISTER_TENSOR_LIST_OPS_GPU);
+REGISTER_TENSOR_LIST_OPS_GPU(bool);
+
+#undef REGISTER_TENSOR_LIST_OPS_GPU
 
-#undef REGISTER_TENSOR_LIST_FROM_TENSOR_GPU
+REGISTER_KERNEL_BUILDER(Name("TensorListPopBack")
+                            .TypeConstraint<Variant>("element_dtype")
+                            .Device(DEVICE_GPU)
+                            .HostMemory("element_shape"),
+                        TensorListPopBack<GPUDevice, Variant>)
 
 REGISTER_UNARY_VARIANT_BINARY_OP_FUNCTION(ADD_VARIANT_BINARY_OP, DEVICE_GPU,
                                           TensorList,
diff --git a/tensorflow/core/kernels/list_kernels.h b/tensorflow/core/kernels/list_kernels.h
index fd1be80f11..f74a88a768 100644
--- a/tensorflow/core/kernels/list_kernels.h
+++ b/tensorflow/core/kernels/list_kernels.h
@@ -66,14 +66,9 @@ struct TensorList {
 
 Status TensorShapeFromTensor(const Tensor& t, PartialTensorShape* out);
 
-// Allocates a Tensor of requested shape and dtype and fills it with zeros.
-template <typename Device, typename T>
-void BuildZerosTensor(OpKernelContext* c, DataType dtype,
-                      const TensorShape& shape, Tensor* zeros) {
-  OP_REQUIRES_OK(c, c->allocate_temp(dtype, shape, zeros));
-  functor::SetZeroFunctor<Device, T> f;
-  f(c->eigen_device<Device>(), zeros->flat<T>());
-}
+Status GetElementShapeFromInput(OpKernelContext* c,
+                                const TensorList& tensor_list, int index,
+                                PartialTensorShape* element_shape);
 
 template <typename Device, typename T>
 class TensorListStack : public OpKernel {
@@ -85,8 +80,6 @@ class TensorListStack : public OpKernel {
     OP_REQUIRES_OK(c, c->GetAttr("num_elements", &num_elements_));
   }
 
-  ~TensorListStack() {}
-
   void Compute(OpKernelContext* c) override {
     const TensorList* tensor_list =
         c->input(0).scalar<Variant>()().get<TensorList>();
@@ -99,13 +92,6 @@ class TensorListStack : public OpKernel {
         errors::InvalidArgument(
             "Invalid data types; op elements ", DataTypeString(element_dtype_),
             " but list elements ", DataTypeString(tensor_list->element_dtype)));
-    OP_REQUIRES(
-        c,
-        !tensor_list->tensors.empty() ||
-            tensor_list->element_shape.IsFullyDefined(),
-        errors::InvalidArgument("Tried to stack elements of a empty ",
-                                "list with non-fully-defined element_shape: ",
-                                tensor_list->element_shape.DebugString()));
     if (num_elements_ != -1) {
       OP_REQUIRES(c, tensor_list->tensors.size() == num_elements_,
                   errors::InvalidArgument(
@@ -113,44 +99,40 @@ class TensorListStack : public OpKernel {
                       " elements but got a list with ",
                       tensor_list->tensors.size(), " elements."));
     }
-    // Compute the shape of the output tensor.
-    // If `element_shape` is fully-defined it gets used. It is assumed that all
-    // element tensors have the same shape.
-    // If `element_shape` is not fully-defined the shape of the first
-    // initialized element tensor is used and it is checked that all other
-    // initialized tensors have the same shape. An error is thrown if the list
-    // only contains DT_INVALID type tensors.
-    TensorShape resulting_element_shape;
-    if (!tensor_list->element_shape.AsTensorShape(&resulting_element_shape)) {
-      bool resulting_element_shape_initialized = false;
+    PartialTensorShape partial_element_shape;
+    OP_REQUIRES_OK(c, GetElementShapeFromInput(c, *tensor_list, 1,
+                                               &partial_element_shape));
+    OP_REQUIRES(
+        c,
+        partial_element_shape.IsFullyDefined() || !tensor_list->tensors.empty(),
+        errors::InvalidArgument("Tried to stack elements of an empty ",
+                                "list with non-fully-defined element_shape: ",
+                                partial_element_shape.DebugString()));
+
+    // Check that `element_shape` input tensor is compatible with the shapes of
+    // element tensors.
+    if (!tensor_list->element_shape.IsFullyDefined()) {
       for (int i = 0; i < tensor_list->tensors.size(); ++i) {
         const Tensor& t = tensor_list->tensors[i];
-        if (!resulting_element_shape_initialized) {
-          if (t.dtype() == DT_INVALID) {
-            continue;
-          }
-          resulting_element_shape = t.shape();
-          resulting_element_shape_initialized = true;
-          continue;
+        if (t.dtype() != DT_INVALID) {
+          PartialTensorShape tmp = partial_element_shape;
+          OP_REQUIRES_OK(c, tmp.MergeWith(t.shape(), &partial_element_shape));
         }
-        OP_REQUIRES(
-            c, t.dtype() == DT_INVALID || t.shape() == resulting_element_shape,
-            errors::InvalidArgument(
-                "Tried to stack tensors with unequal shapes: ",
-                resulting_element_shape.DebugString(), " vs ",
-                t.shape().DebugString()));
       }
-      OP_REQUIRES(
-          c, resulting_element_shape_initialized,
-          errors::InvalidArgument("Tried to stack list which only contains ",
-                                  "uninitialized tensors and has a ",
-                                  "non-fully-defined element_shape: ",
-                                  tensor_list->element_shape.DebugString()));
     }
-    TensorShape output_tensor_shape = resulting_element_shape;
-    output_tensor_shape.InsertDim(0, tensor_list->tensors.size());
+
+    // Compute the shape of the output tensor by pre-pending the leading dim to
+    // the element_shape.
+    TensorShape element_shape;
+    OP_REQUIRES(c, partial_element_shape.AsTensorShape(&element_shape),
+                errors::InvalidArgument(
+                    "Tried to stack list which only contains uninitialized ",
+                    "tensors and has a non-fully-defined element_shape: ",
+                    partial_element_shape.DebugString()));
+    TensorShape output_shape = element_shape;
+    output_shape.InsertDim(0, tensor_list->tensors.size());
     Tensor* output;
-    OP_REQUIRES_OK(c, c->allocate_output(0, output_tensor_shape, &output));
+    OP_REQUIRES_OK(c, c->allocate_output(0, output_shape, &output));
     if (output->NumElements() == 0) {
       return;
     }
@@ -158,13 +140,21 @@ class TensorListStack : public OpKernel {
     ConstMatrixVector inputs_flat;
     inputs_flat.reserve(tensor_list->tensors.size());
     Tensor zeros;
-    BuildZerosTensor<Device, T>(c, element_dtype_, resulting_element_shape,
-                                &zeros);
     for (const auto& t : tensor_list->tensors) {
       if (t.dtype() != DT_INVALID) {
         inputs_flat.emplace_back(new typename TTypes<T, 2>::ConstMatrix(
             t.shaped<T, 2>({1, t.NumElements()})));
       } else {
+        if (!zeros.NumElements()) {
+          AllocatorAttributes attr;
+          if (element_dtype_ == DT_VARIANT) {
+            attr.set_on_host(true);
+          }
+          OP_REQUIRES_OK(
+              c, c->allocate_temp(element_dtype_, element_shape, &zeros, attr));
+          functor::SetZeroFunctor<Device, T>()(c->eigen_device<Device>(),
+                                               zeros.flat<T>());
+        }
         inputs_flat.emplace_back(new typename TTypes<T, 2>::ConstMatrix(
             const_cast<const Tensor&>(zeros).shaped<T, 2>(
                 {1, zeros.NumElements()})));
@@ -186,6 +176,123 @@ class TensorListStack : public OpKernel {
   DataType element_dtype_;
 };
 
+template <typename Device, typename T>
+class TensorListGetItem : public OpKernel {
+ public:
+  explicit TensorListGetItem(OpKernelConstruction* c) : OpKernel(c) {
+    OP_REQUIRES_OK(c, c->GetAttr("element_dtype", &element_dtype_));
+  }
+
+  void Compute(OpKernelContext* c) override {
+    OP_REQUIRES(
+        c, c->input(0).shape().num_elements() == 1,
+        errors::InvalidArgument("List tensors are supposed to be scalars."));
+    const TensorList* l = c->input(0).scalar<Variant>()().get<TensorList>();
+    OP_REQUIRES(c, l != nullptr,
+                errors::InvalidArgument(
+                    "Input handle is not a list. Saw: '",
+                    c->input(0).scalar<Variant>()().DebugString(), "'"));
+    OP_REQUIRES(c, element_dtype_ == l->element_dtype,
+                errors::InvalidArgument("Invalid data types; op elements ",
+                                        DataTypeString(element_dtype_),
+                                        " but list elements ",
+                                        DataTypeString(l->element_dtype)));
+    int32 index = c->input(1).scalar<int32>()();
+    OP_REQUIRES(c, index < l->tensors.size(),
+                errors::InvalidArgument("Trying to access element ", index,
+                                        " in a list with ", l->tensors.size(),
+                                        " elements."));
+    if (l->tensors[index].dtype() != DT_INVALID) {
+      c->set_output(0, l->tensors[index]);
+    } else {
+      PartialTensorShape partial_element_shape;
+      OP_REQUIRES_OK(
+          c, GetElementShapeFromInput(c, *l, 2, &partial_element_shape));
+      TensorShape element_shape;
+      OP_REQUIRES(
+          c, partial_element_shape.AsTensorShape(&element_shape),
+          errors::InvalidArgument("Trying to read an uninitialized tensor but ",
+                                  "element_shape is not fully defined.",
+                                  partial_element_shape.DebugString()));
+      Tensor* result;
+      AllocatorAttributes attr;
+      if (element_dtype_ == DT_VARIANT) {
+        attr.set_on_host(true);
+      }
+      OP_REQUIRES_OK(c, c->allocate_output(0, element_shape, &result, attr));
+      functor::SetZeroFunctor<Device, T>()(c->eigen_device<Device>(),
+                                           result->flat<T>());
+    }
+  }
+
+ private:
+  DataType element_dtype_;
+};
+
+template <typename Device, typename T>
+class TensorListPopBack : public OpKernel {
+ public:
+  explicit TensorListPopBack(OpKernelConstruction* c) : OpKernel(c) {
+    OP_REQUIRES_OK(c, c->GetAttr("element_dtype", &element_dtype_));
+  }
+
+  void Compute(OpKernelContext* c) override {
+    const TensorList* l = c->input(0).scalar<Variant>()().get<TensorList>();
+    OP_REQUIRES(c, l != nullptr,
+                errors::InvalidArgument(
+                    "Input handle is not a list. Saw: '",
+                    c->input(0).scalar<Variant>()().DebugString(), "'"));
+    OP_REQUIRES(c, element_dtype_ == l->element_dtype,
+                errors::InvalidArgument("Invalid data types; op elements ",
+                                        DataTypeString(element_dtype_),
+                                        " but list elements ",
+                                        DataTypeString(l->element_dtype)));
+
+    OP_REQUIRES(c, !l->tensors.empty(),
+                errors::InvalidArgument("Trying to pop from an empty list."));
+
+    const Tensor& t = l->tensors.back();
+    if (t.dtype() != DT_INVALID) {
+      c->set_output(1, t);
+    } else {
+      PartialTensorShape partial_element_shape;
+      OP_REQUIRES_OK(
+          c, GetElementShapeFromInput(c, *l, 1, &partial_element_shape));
+      TensorShape element_shape;
+      OP_REQUIRES(
+          c, partial_element_shape.AsTensorShape(&element_shape),
+          errors::InvalidArgument("Trying to read an uninitialized tensor but ",
+                                  "element_shape is not fully defined.",
+                                  partial_element_shape.DebugString()));
+      Tensor* result;
+      AllocatorAttributes attr;
+      if (element_dtype_ == DT_VARIANT) {
+        attr.set_on_host(true);
+      }
+      OP_REQUIRES_OK(c, c->allocate_output(1, element_shape, &result, attr));
+      functor::SetZeroFunctor<Device, T>()(c->eigen_device<Device>(),
+                                           result->flat<T>());
+    }
+    AllocatorAttributes attr;
+    attr.set_on_host(true);
+    std::unique_ptr<Tensor> maybe_result = c->forward_input(
+        0, 0, DT_VARIANT, TensorShape{}, c->input_memory_type(0), attr);
+    if (maybe_result != nullptr) {
+      maybe_result->scalar<Variant>()().get<TensorList>()->tensors.pop_back();
+    } else {
+      TensorList output;
+      output = *l;
+      output.tensors.pop_back();
+      Tensor* result;
+      OP_REQUIRES_OK(c, c->allocate_output(0, TensorShape{}, &result, attr));
+      result->scalar<Variant>()() = std::move(output);
+    }
+  }
+
+ private:
+  DataType element_dtype_;
+};
+
 template <typename Device, typename T>
 class TensorListConcat : public OpKernel {
  public:
@@ -206,8 +313,6 @@ class TensorListConcat : public OpKernel {
     }
   }
 
-  ~TensorListConcat() {}
-
   void Compute(OpKernelContext* c) override {
     // Check that the input Variant tensor is indeed a TensorList and has the
     // correct element type.
@@ -418,47 +523,47 @@ class TensorListGather : public OpKernel {
             "Invalid data types; op elements ", DataTypeString(element_dtype_),
             " but list elements ", DataTypeString(tensor_list->element_dtype)));
     Tensor indices = c->input(1);
+    PartialTensorShape partial_element_shape;
+    OP_REQUIRES_OK(c, GetElementShapeFromInput(c, *tensor_list, 2,
+                                               &partial_element_shape));
     OP_REQUIRES(
-        c,
-        indices.NumElements() > 0 ||
-            tensor_list->element_shape.IsFullyDefined(),
+        c, partial_element_shape.IsFullyDefined() || indices.NumElements() > 0,
         errors::InvalidArgument("Tried to gather 0-elements from "
                                 "a list with non-fully-defined shape: ",
-                                tensor_list->element_shape.DebugString()));
-    // Compute the shape of the output tensor.
-    // If `element_shape` is fully-defined it gets used. It is assumed that all
-    // requested tensors have the same shape.
-    // If `element_shape` is not fully-defined the shape of the first requested
-    // tensor is used and it is checked that all other tensors have the same
-    // shape.
-    TensorShape resulting_shape;
-    if (!tensor_list->element_shape.AsTensorShape(&resulting_shape)) {
-      const int i = indices.flat<int32>()(0);
-      OP_REQUIRES(
-          c, i < tensor_list->tensors.size(),
-          errors::InvalidArgument("Index ", i, " out o range; list only has ",
-                                  tensor_list->tensors.size(), " elements."));
-      const Tensor& t = tensor_list->tensors[i];
-      resulting_shape = t.shape();
-      for (int index = 1; index < indices.NumElements(); ++index) {
+                                partial_element_shape.DebugString()));
+
+    // Check that `element_shape` input tensor is compatible with the shapes of
+    // element tensors.
+    if (!tensor_list->element_shape.IsFullyDefined()) {
+      for (int index = 0; index < indices.NumElements(); ++index) {
         const int i = indices.flat<int32>()(index);
         const Tensor& t = tensor_list->tensors[i];
-        OP_REQUIRES(c, t.shape() == resulting_shape,
-                    errors::InvalidArgument(
-                        "Tried to gather elements with unequal shapes: ",
-                        resulting_shape.DebugString(), " vs ",
-                        t.shape().DebugString()));
+        if (t.dtype() != DT_INVALID) {
+          PartialTensorShape tmp = partial_element_shape;
+          OP_REQUIRES_OK(c, tmp.MergeWith(t.shape(), &partial_element_shape));
+        }
       }
     }
-    resulting_shape.InsertDim(0, indices.NumElements());
+
+    // Compute the shape of the output tensor by pre-pending the leading dim to
+    // the element_shape.
+    TensorShape element_shape;
+    OP_REQUIRES(
+        c, partial_element_shape.AsTensorShape(&element_shape),
+        errors::InvalidArgument("Tried to gather uninitialized tensors from a ",
+                                "list with non-fully-defined element_shape: ",
+                                partial_element_shape.DebugString()));
+    TensorShape output_shape = element_shape;
+    output_shape.InsertDim(0, indices.NumElements());
     Tensor* output;
-    OP_REQUIRES_OK(c, c->allocate_output(0, resulting_shape, &output));
+    OP_REQUIRES_OK(c, c->allocate_output(0, output_shape, &output));
     if (output->NumElements() == 0) {
       return;
     }
 
     ConstMatrixVector inputs_flat;
-    inputs_flat.reserve(tensor_list->tensors.size());
+    inputs_flat.reserve(indices.NumElements());
+    Tensor zeros;
     for (int index = 0; index < indices.NumElements(); ++index) {
       const int i = indices.flat<int32>()(index);
       OP_REQUIRES(
@@ -466,8 +571,24 @@ class TensorListGather : public OpKernel {
           errors::InvalidArgument("Index ", i, " out o range; list only has ",
                                   tensor_list->tensors.size(), " elements."));
       const Tensor& t = tensor_list->tensors[i];
-      inputs_flat.emplace_back(new typename TTypes<T, 2>::ConstMatrix(
-          t.shaped<T, 2>({1, t.NumElements()})));
+      if (t.dtype() != DT_INVALID) {
+        inputs_flat.emplace_back(new typename TTypes<T, 2>::ConstMatrix(
+            t.shaped<T, 2>({1, t.NumElements()})));
+      } else {
+        if (!zeros.NumElements()) {
+          AllocatorAttributes attr;
+          if (element_dtype_ == DT_VARIANT) {
+            attr.set_on_host(true);
+          }
+          OP_REQUIRES_OK(
+              c, c->allocate_temp(element_dtype_, element_shape, &zeros, attr));
+          functor::SetZeroFunctor<Device, T>()(c->eigen_device<Device>(),
+                                               zeros.flat<T>());
+        }
+        inputs_flat.emplace_back(new typename TTypes<T, 2>::ConstMatrix(
+            const_cast<const Tensor&>(zeros).shaped<T, 2>(
+                {1, zeros.NumElements()})));
+      }
     }
     auto output_flat = output->shaped<T, 2>({1, output->NumElements()});
 
@@ -657,8 +778,6 @@ class TensorListPushBackBatch : public OpKernel {
     OP_REQUIRES_OK(c, c->GetAttr("element_dtype", &element_dtype_));
   }
 
-  ~TensorListPushBackBatch() override {}
-
   void Compute(OpKernelContext* c) override {
     const Tensor& input = c->input(1);
     OP_REQUIRES(c, element_dtype_ == input.dtype(),
diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 3b7a7b812a..eb251fc517 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -77304,6 +77304,10 @@ op {
     name: "indices"
     type: DT_INT32
   }
+  input_arg {
+    name: "element_shape"
+    type: DT_INT32
+  }
   output_arg {
     name: "values"
     type_attr: "element_dtype"
@@ -77323,6 +77327,10 @@ op {
     name: "index"
     type: DT_INT32
   }
+  input_arg {
+    name: "element_shape"
+    type: DT_INT32
+  }
   output_arg {
     name: "item"
     type_attr: "element_dtype"
@@ -77349,6 +77357,10 @@ op {
     name: "input_handle"
     type: DT_VARIANT
   }
+  input_arg {
+    name: "element_shape"
+    type: DT_INT32
+  }
   output_arg {
     name: "output_handle"
     type: DT_VARIANT
@@ -77540,6 +77552,10 @@ op {
     name: "input_handle"
     type: DT_VARIANT
   }
+  input_arg {
+    name: "element_shape"
+    type: DT_INT32
+  }
   output_arg {
     name: "tensor"
     type_attr: "element_dtype"
diff --git a/tensorflow/core/ops/list_ops.cc b/tensorflow/core/ops/list_ops.cc
index cbc9c7a2f4..a16fabe2ab 100644
--- a/tensorflow/core/ops/list_ops.cc
+++ b/tensorflow/core/ops/list_ops.cc
@@ -130,6 +130,7 @@ REGISTER_OP("TensorListLength")
 
 REGISTER_OP("TensorListPopBack")
     .Input("input_handle: variant")
+    .Input("element_shape: int32")
     .Output("output_handle: variant")
     .Output("tensor: element_dtype")
     .Attr("element_dtype: type")
@@ -166,6 +167,7 @@ REGISTER_OP("TensorListPopBack")
 
 REGISTER_OP("TensorListStack")
     .Input("input_handle: variant")
+    .Input("element_shape: int32")
     .Output("tensor: element_dtype")
     .Attr("element_dtype: type")
     .Attr("num_elements: int = -1")
@@ -351,6 +353,7 @@ REGISTER_OP("TensorListReserve")
 REGISTER_OP("TensorListGetItem")
     .Input("input_handle: variant")
     .Input("index: int32")
+    .Input("element_shape: int32")
     .Output("item: element_dtype")
     .Attr("element_dtype: type")
     .SetShapeFn([](shape_inference::InferenceContext* c) {
@@ -418,6 +421,7 @@ REGISTER_OP("TensorListSetItem")
 REGISTER_OP("TensorListGather")
     .Input("input_handle: variant")
     .Input("indices: int32")
+    .Input("element_shape: int32")
     .Output("values: element_dtype")
     .Attr("element_dtype: type")
     .SetShapeFn([](shape_inference::InferenceContext* c) {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 92a7bb9a34..f43a8c4744 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -37082,6 +37082,10 @@ op {
     name: "indices"
     type: DT_INT32
   }
+  input_arg {
+    name: "element_shape"
+    type: DT_INT32
+  }
   output_arg {
     name: "values"
     type_attr: "element_dtype"
@@ -37101,6 +37105,10 @@ op {
     name: "index"
     type: DT_INT32
   }
+  input_arg {
+    name: "element_shape"
+    type: DT_INT32
+  }
   output_arg {
     name: "item"
     type_attr: "element_dtype"
@@ -37127,6 +37135,10 @@ op {
     name: "input_handle"
     type: DT_VARIANT
   }
+  input_arg {
+    name: "element_shape"
+    type: DT_INT32
+  }
   output_arg {
     name: "output_handle"
     type: DT_VARIANT
@@ -37318,6 +37330,10 @@ op {
     name: "input_handle"
     type: DT_VARIANT
   }
+  input_arg {
+    name: "element_shape"
+    type: DT_INT32
+  }
   output_arg {
     name: "tensor"
     type_attr: "element_dtype"
diff --git a/tensorflow/python/kernel_tests/list_ops_test.py b/tensorflow/python/kernel_tests/list_ops_test.py
index ec6906f20c..3f340aa07d 100644
--- a/tensorflow/python/kernel_tests/list_ops_test.py
+++ b/tensorflow/python/kernel_tests/list_ops_test.py
@@ -33,6 +33,7 @@ from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import gen_list_ops
 from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import list_ops
 from tensorflow.python.ops import math_ops
@@ -89,6 +90,58 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       l = list_ops.tensor_list_pop_back(l, element_dtype=dtypes.float32)
       self.evaluate(l)
 
+  def testPopUninitializedTensorUseListElementShape(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=[2, 3], num_elements=3)
+    _, e = list_ops.tensor_list_pop_back(l, element_dtype=dtypes.float32)
+    self.assertAllEqual(e, np.zeros((2, 3)))
+
+  def testPopUninitializedTensorUseSpecifiedElementShape(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=[None, 3], num_elements=3)
+    _, e = gen_list_ops.tensor_list_pop_back(
+        l, element_dtype=dtypes.float32, element_shape=[4, 3])
+    self.assertAllEqual(e, np.zeros((4, 3)))
+
+  def testPopUninitializedTensorWithInvalidElementShapeFails(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=None, num_elements=3)
+    with self.assertRaisesRegexp(
+        errors.InvalidArgumentError,
+        "Trying to read an uninitialized tensor but "
+        "element_shape is not fully defined"):
+      _, e = list_ops.tensor_list_pop_back(l, element_dtype=dtypes.float32)
+      self.evaluate(e)
+
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=[None, 2], num_elements=3)
+    with self.assertRaisesRegexp(
+        errors.InvalidArgumentError,
+        r"Incompatible shapes during merge: \[1,3\] vs. \[\?,2\]"):
+      _, e = gen_list_ops.tensor_list_pop_back(
+          l, element_dtype=dtypes.float32, element_shape=[1, 3])
+      self.evaluate(e)
+
+  def testPushGetGrad(self):
+    with backprop.GradientTape() as tape:
+      l = list_ops.empty_tensor_list(
+          element_dtype=dtypes.float32, element_shape=None)
+      c0 = constant_op.constant(5.0)
+      c1 = constant_op.constant([10.0, 20.0])
+      tape.watch(c0)
+      tape.watch(c1)
+      l = list_ops.tensor_list_push_back(l, c0)
+      l = list_ops.tensor_list_push_back(l, c1)
+      t1 = list_ops.tensor_list_get_item(l, 1, element_dtype=dtypes.float32)
+      self.assertAllEqual(self.evaluate(t1), [10.0, 20.0])
+      # t1 == c1 so the gradient should be [0., [1., 1.]]
+      # This tests that the gradient of push_back correctly converts DT_INVALID
+      # tensors to zeros. The list returned by the gradient of GetItem will
+      # have only have tensor at index 1 set and others set to DT_INVALID.
+      dt0, dt1 = tape.gradient(t1, [c0, c1])
+      self.assertAllEqual(self.evaluate(dt1), [1.0, 1.0])
+      self.assertEqual(self.evaluate(dt0), 0.0)
+
   def _testStack(self, max_num_elements):
     l = list_ops.empty_tensor_list(
         element_dtype=dtypes.float32,
@@ -130,7 +183,8 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
     # Should raise an error when the element tensors do not all have the same
     # shape.
-    with self.assertRaisesRegexp(errors.InvalidArgumentError, "unequal shapes"):
+    with self.assertRaisesRegexp(errors.InvalidArgumentError,
+                                 "Incompatible ranks during merge: 0 vs. 1"):
       l = list_ops.tensor_list_push_back(l, constant_op.constant([3.0, 4.0]))
       t = list_ops.tensor_list_stack(l, element_dtype=dtypes.float32)
       self.evaluate(t)
@@ -151,7 +205,9 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
     # Should raise an error when the element tensors do not all have the same
     # shape.
-    with self.assertRaisesRegexp(errors.InvalidArgumentError, "unequal shapes"):
+    with self.assertRaisesRegexp(
+        errors.InvalidArgumentError,
+        r"Incompatible shapes during merge: \[1\] vs. \[2\]"):
       l = list_ops.tensor_list_push_back(l, constant_op.constant([2.0, 3.0]))
       t = list_ops.tensor_list_stack(l, element_dtype=dtypes.float32)
       self.evaluate(t)
@@ -230,6 +286,13 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       t = list_ops.tensor_list_stack(l, element_dtype=dtypes.float32)
       self.evaluate(t)
 
+  def testStackUsingSpecifiedElementShape(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=None, num_elements=3)
+    t = gen_list_ops.tensor_list_stack(
+        l, element_dtype=dtypes.float32, element_shape=[])
+    self.assertAllEqual(self.evaluate(t), np.zeros((3,)))
+
   @parameterized.named_parameters(("NoMaxNumElements", None),
                                   ("WithMaxNumElements", 2))
   def testGatherGrad(self, max_num_elements):
@@ -268,7 +331,8 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
     # Should raise an error when the requested tensors do not all have the same
     # shape.
-    with self.assertRaisesRegexp(errors.InvalidArgumentError, "unequal shapes"):
+    with self.assertRaisesRegexp(errors.InvalidArgumentError,
+                                 "Incompatible ranks during merge: 0 vs. 1"):
       t = list_ops.tensor_list_gather(l, [0, 2], element_dtype=dtypes.float32)
       self.evaluate(t)
 
@@ -292,7 +356,9 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
 
     # Should raise an error when the requested tensors do not all have the same
     # shape.
-    with self.assertRaisesRegexp(errors.InvalidArgumentError, "unequal shapes"):
+    with self.assertRaisesRegexp(
+        errors.InvalidArgumentError,
+        r"Incompatible shapes during merge: \[1\] vs. \[2\]"):
       t = list_ops.tensor_list_gather(l, [0, 2], element_dtype=dtypes.float32)
       self.evaluate(t)
 
@@ -347,6 +413,54 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     dl_length = list_ops.tensor_list_length(dl)
     self.assertAllEqual(self.evaluate(dl_length), 3)
 
+  def _testGatherWithUninitializedTensors(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=[], num_elements=3)
+    t = list_ops.tensor_list_gather(l, [0, 2], element_dtype=dtypes.float32)
+    self.assertAllEqual(self.evaluate(t), [0., 0.])
+
+  def testGatherWithUninitializedTensors(self):
+    self._testGatherWithUninitializedTensors()
+
+  def testGatherWithUninitializedTensorsGpu(self):
+    if not context.num_gpus():
+      return
+    with context.device("gpu:0"):
+      self._testGatherWithUninitializedTensors()
+
+  def _testGatherWithUninitializedTensorsInferShape(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=None, num_elements=3)
+    l = list_ops.tensor_list_set_item(l, 1, [1., 2.])
+    t = list_ops.tensor_list_gather(l, [1, 2], element_dtype=dtypes.float32)
+    self.assertAllEqual(self.evaluate(t), [[1., 2.], [0., 0.]])
+
+  def testGatherWithUninitializedTensorsInferShape(self):
+    self._testGatherWithUninitializedTensorsInferShape()
+
+  def testGatherWithUninitializedTensorsInferShapeGpu(self):
+    if not context.num_gpus():
+      return
+    with context.device("gpu:0"):
+      self._testGatherWithUninitializedTensorsInferShape()
+
+  def testGatherReservedListWithNoElementsAndPartialElementShapeFails(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=None, num_elements=3)
+    with self.assertRaisesRegexp(
+        errors.InvalidArgumentError,
+        "Tried to gather uninitialized tensors from a"
+        " list with non-fully-defined element_shape"):
+      t = list_ops.tensor_list_gather(l, [0], element_dtype=dtypes.float32)
+      self.evaluate(t)
+
+  def testGatherUsingSpecifiedElementShape(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=None, num_elements=3)
+    t = gen_list_ops.tensor_list_gather(
+        l, [0, 1, 2], element_dtype=dtypes.float32, element_shape=[])
+    self.assertAllEqual(self.evaluate(t), np.zeros((3,)))
+
   def testScatterOutputListSize(self):
     c0 = constant_op.constant([1.0, 2.0])
     l = list_ops.tensor_list_scatter(
@@ -386,6 +500,18 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
     dt = tape.gradient(loss, c0)
     self.assertAllEqual(self.evaluate(dt), [2., 4.])
 
+  def testScatterWithPartialReadGrad(self):
+    with backprop.GradientTape() as tape:
+      c0 = constant_op.constant([1.0, 2.0])
+      tape.watch(c0)
+      l = list_ops.tensor_list_scatter(
+          c0, [1, 0], ops.convert_to_tensor([], dtype=dtypes.int32))
+      t0 = list_ops.tensor_list_get_item(l, 0, element_dtype=dtypes.float32)
+      self.assertAllEqual(self.evaluate(t0), 2.0)
+      loss = t0 * t0
+    dt = tape.gradient(loss, c0)
+    self.assertAllEqual(self.evaluate(dt), [0., 4.])
+
   def testTensorListFromTensor(self):
     t = constant_op.constant([1.0, 2.0])
     l = list_ops.tensor_list_from_tensor(t, element_shape=[])
@@ -427,6 +553,50 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase):
       self.assertAllEqual(self.evaluate(e), 10.0)
     self.assertAllEqual(self.evaluate(tape.gradient(e, t)), 2.0)
 
+  def testGetUninitializedTensorUseListElementShape(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=[], num_elements=3)
+    l = list_ops.tensor_list_set_item(l, 0, 5.)
+    e1 = list_ops.tensor_list_get_item(l, 1, element_dtype=dtypes.float32)
+    e2 = list_ops.tensor_list_get_item(l, 2, element_dtype=dtypes.float32)
+    self.assertEqual(self.evaluate(e1), 0.)
+    self.assertEqual(self.evaluate(e2), 0.)
+
+  def testGetUninitializedTensorUseSpecifiedElementShape(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=None, num_elements=3)
+    e0 = gen_list_ops.tensor_list_get_item(
+        l, 0, element_shape=[], element_dtype=dtypes.float32)
+    e1 = gen_list_ops.tensor_list_get_item(
+        l, 1, element_shape=[2, 3], element_dtype=dtypes.float32)
+    self.assertEqual(self.evaluate(e0), 0.)
+    self.assertAllEqual(self.evaluate(e1), np.zeros((2, 3)))
+
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=[None, 3], num_elements=3)
+    e1 = gen_list_ops.tensor_list_get_item(
+        l, 1, element_shape=[2, 3], element_dtype=dtypes.float32)
+    self.assertAllEqual(self.evaluate(e1), np.zeros((2, 3)))
+
+  def testGetUninitializedTensorWithInvalidElementShapeFails(self):
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=None, num_elements=3)
+    with self.assertRaisesRegexp(
+        errors.InvalidArgumentError,
+        "Trying to read an uninitialized tensor but "
+        "element_shape is not fully defined"):
+      e0 = list_ops.tensor_list_get_item(l, 0, element_dtype=dtypes.float32)
+      self.evaluate(e0)
+
+    l = list_ops.tensor_list_reserve(
+        element_dtype=dtypes.float32, element_shape=[None, 2], num_elements=3)
+    with self.assertRaisesRegexp(
+        errors.InvalidArgumentError,
+        r"Incompatible shapes during merge: \[1,3\] vs. \[\?,2\]"):
+      e0 = gen_list_ops.tensor_list_get_item(
+          l, 0, element_dtype=dtypes.float32, element_shape=[1, 3])
+      self.evaluate(e0)
+
   @test_util.run_deprecated_v1
   @test_util.enable_control_flow_v2
   def testSkipEagerSetItemIndexOutOfBounds(self):
diff --git a/tensorflow/python/kernel_tests/tensor_array_ops_test.py b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
index e8af998a70..d0efb47a38 100644
--- a/tensorflow/python/kernel_tests/tensor_array_ops_test.py
+++ b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
@@ -1281,13 +1281,23 @@ class TensorArrayTest(test.TestCase):
       grad_r0_vals = session.run(grad_r0)[0]
       self.assertAllEqual(grad_r0_vals, [1.0, 0.0])
 
-  # TODO(srbs): Figure out how to enable this. This is probably failing
-  # because we are trying to stack a TensorList with invalid tensors.
-  # That is because we do not receive gradients for all list indices.
-  # Figure out how TensorArray handles this.
-  def disabletestGradientWhenNotAllComponentsRead(self):
+  @test_util.run_deprecated_v1
+  def testSkipEagerGradientWhenNotAllComponentsRead(self):
     self._testGradientWhenNotAllComponentsRead()
 
+  @test_util.run_deprecated_v1
+  def testSkipEagerWriteButNotAllComponentsReadGrad(self):
+    with self.cached_session(use_gpu=True) as session:
+      x0 = constant_op.constant(5.0)
+      x1 = constant_op.constant(10.0)
+      ta = tensor_array_ops.TensorArray(
+          dtype=dtypes.float32, size=2).write(0, x0).write(1, x1)
+      r0 = ta.read(0)
+      # calculate (dr0/dx0, dr0/dx1).  since r0 = x0, gradients are (1, 0).
+      grad_r0_x1 = gradients_impl.gradients(ys=[r0], xs=[x0, x1], grad_ys=[1.0])
+      grad_r0_x1_vals = session.run(grad_r0_x1)
+      self.assertAllEqual(grad_r0_x1_vals, [1.0, 0.0])
+
   def _testTensorArrayUnpackDynamic(self):
     with self.cached_session(use_gpu=True) as sess:
       ta = tensor_array_ops.TensorArray(
@@ -1321,8 +1331,8 @@ class TensorArrayTest(test.TestCase):
     with self.cached_session(use_gpu=True):
       ta = tensor_array_ops.TensorArray(
           dtype=dtypes.float32, size=0, dynamic_size=False, infer_shape=False)
-      v2_msg = ("Tried to stack elements of a empty list with non-fully-defined"
-                " element_shape")
+      v2_msg = ("Tried to stack elements of an empty list with "
+                "non-fully-defined element_shape")
       v1_msg = (
           "TensorArray has size zero, but element shape <unknown> is not "
           "fully defined. Currently only static shapes are supported when "
@@ -1385,6 +1395,29 @@ class TensorArrayTest(test.TestCase):
       self.assertAllEqual([10.0, -10.0], read_vals[1])
       self.assertAllEqual([[2.0, 3.0], [4.0, 5.0]], grad_vals[0])
 
+  @test_util.run_v1_only("b/117943489")
+  def testSkipEagerTensorArrayScatterPartialReadAndGradients(self):
+    with self.session(use_gpu=True) as session:
+      ta = tensor_array_ops.TensorArray(
+          dtype=dtypes.float32,
+          tensor_array_name="foo",
+          size=0,
+          dynamic_size=True)
+
+      indices = constant_op.constant([1, 8])
+      value = constant_op.constant([[1.0, -1.0], [10.0, -10.0]])
+
+      w = ta.scatter(indices, value)
+      r0 = w.read(1)
+
+      # Test combined gradients + aggregation of read(0)
+      grad = gradients_impl.gradients(
+          ys=[r0], xs=[value], grad_ys=[[2.0, 3.0]])[0]
+      read_val, grad_val = session.run([r0, grad])
+
+      self.assertAllEqual([1.0, -1.0], read_val)
+      self.assertAllEqual([[2.0, 3.0], [0.0, 0.0]], grad_val)
+
   @test_util.run_v1_only("b/118890905")
   def testTensorArrayWriteGatherAndGradients(self):
     with self.session(use_gpu=True) as session:
diff --git a/tensorflow/python/ops/list_ops.py b/tensorflow/python/ops/list_ops.py
index df928ea85d..af059a541e 100644
--- a/tensorflow/python/ops/list_ops.py
+++ b/tensorflow/python/ops/list_ops.py
@@ -71,6 +71,41 @@ def tensor_list_from_tensor(tensor, element_shape, name=None):
       name=name)
 
 
+def tensor_list_get_item(input_handle, index, element_dtype, name=None):
+  return gen_list_ops.tensor_list_get_item(
+      input_handle=input_handle,
+      index=index,
+      element_shape=-1,
+      element_dtype=element_dtype,
+      name=name)
+
+
+def tensor_list_pop_back(input_handle, element_dtype, name=None):
+  return gen_list_ops.tensor_list_pop_back(
+      input_handle=input_handle,
+      element_shape=-1,
+      element_dtype=element_dtype,
+      name=name)
+
+
+def tensor_list_gather(input_handle, indices, element_dtype, name=None):
+  return gen_list_ops.tensor_list_gather(
+      input_handle=input_handle,
+      indices=indices,
+      element_shape=-1,
+      element_dtype=element_dtype,
+      name=name)
+
+
+def tensor_list_stack(input_handle, element_dtype, num_elements=-1, name=None):
+  return gen_list_ops.tensor_list_stack(
+      input_handle=input_handle,
+      element_shape=-1,
+      element_dtype=element_dtype,
+      num_elements=num_elements,
+      name=name)
+
+
 def tensor_list_concat(input_handle, element_dtype, element_shape=None,
                        name=None):
   # Ignore the lengths output of TensorListConcat. It is only used during
@@ -110,7 +145,9 @@ def tensor_list_set_item(input_handle,
 @ops.RegisterGradient("TensorListPushBack")
 def _PushBackGrad(op, dresult):
   return gen_list_ops.tensor_list_pop_back(
-      dresult, element_dtype=op.get_attr("element_dtype"))
+      dresult,
+      element_shape=array_ops.shape(op.inputs[1]),
+      element_dtype=op.get_attr("element_dtype"))
 
 
 @ops.RegisterGradient("TensorListPopBack")
@@ -120,12 +157,12 @@ def _PopBackGrad(op, dlist, delement):
         element_dtype=delement.dtype,
         element_shape=gen_list_ops.tensor_list_element_shape(
             op.outputs[0], shape_type=dtypes.int32))
-  return gen_list_ops.tensor_list_push_back(dlist, delement)
+  return gen_list_ops.tensor_list_push_back(dlist, delement), None
 
 
 @ops.RegisterGradient("TensorListStack")
 def _TensorListStackGrad(unused_op, dtensor):
-  return tensor_list_from_tensor(dtensor, element_shape=dtensor.shape[1:])
+  return tensor_list_from_tensor(dtensor, element_shape=dtensor.shape[1:]), None
 
 
 @ops.RegisterGradient("TensorListConcat")
@@ -150,17 +187,21 @@ def _TensorListSplitGrad(op, dlist):
 @ops.RegisterGradient("TensorListFromTensor")
 def _TensorListFromTensorGrad(op, dlist):
   """Gradient for TensorListFromTensor."""
-  if op.inputs[0].shape.dims and op.inputs[0].shape.dims[0].value is not None:
-    num_elements = op.inputs[0].shape.dims[0].value
+  t = op.inputs[0]
+  if t.shape.dims and t.shape.dims[0].value is not None:
+    num_elements = t.shape.dims[0].value
   else:
     num_elements = None
   if dlist is None:
     dlist = empty_tensor_list(
-        element_dtype=op.inputs[0].dtype,
+        element_dtype=t.dtype,
         element_shape=gen_list_ops.tensor_list_element_shape(
             op.outputs[0], shape_type=dtypes.int32))
   tensor_grad = gen_list_ops.tensor_list_stack(
-      dlist, element_dtype=op.inputs[0].dtype, num_elements=num_elements)
+      dlist,
+      element_shape=array_ops.slice(array_ops.shape(t), [1], [-1]),
+      element_dtype=t.dtype,
+      num_elements=num_elements)
   shape_grad = None
   return tensor_grad, shape_grad
 
@@ -177,17 +218,22 @@ def _TensorListGetItemGrad(op, ditem):
       index=op.inputs[1],
       item=ditem)
   index_grad = None
-  return list_grad, index_grad
+  element_shape_grad = None
+  return list_grad, index_grad, element_shape_grad
 
 
 @ops.RegisterGradient("TensorListSetItem")
 def _TensorListSetItemGrad(op, dlist):
+  """Gradient function for TensorListSetItem."""
   _, index, item = op.inputs
   list_grad = gen_list_ops.tensor_list_set_item(
       dlist, index=index, item=array_ops.zeros_like(item))
   index_grad = None
   element_grad = gen_list_ops.tensor_list_get_item(
-      dlist, index, element_dtype=item.dtype)
+      dlist,
+      index,
+      element_shape=array_ops.shape(item),
+      element_dtype=item.dtype)
   return list_grad, index_grad, element_grad
 
 
@@ -200,7 +246,8 @@ def _TensorListResizeGrad(op, dlist):
 
 @ops.RegisterGradient("TensorListGather")
 def _TensorListGatherGrad(op, dtensor):
-  input_list, indices = op.inputs
+  """Gradient function for TensorListGather."""
+  input_list, indices, _ = op.inputs
   dlist = gen_list_ops.tensor_list_scatter(
       tensor=dtensor,
       indices=indices,
@@ -209,14 +256,17 @@ def _TensorListGatherGrad(op, dtensor):
   # so we manually resize it to match the size of the input list.
   input_list_size = gen_list_ops.tensor_list_length(input_list)
   dlist = gen_list_ops.tensor_list_resize(dlist, input_list_size)
-  return dlist, None
+  return dlist, None, None
 
 
 @ops.RegisterGradient("TensorListScatter")
 def _TensorListScatterGrad(op, dlist):
   t, indices, _ = op.inputs
   return gen_list_ops.tensor_list_gather(
-      dlist, indices, element_dtype=t.dtype), None, None
+      dlist,
+      indices,
+      element_shape=array_ops.slice(array_ops.shape(t), [1], [-1]),
+      element_dtype=t.dtype), None, None
 
 
 def _build_element_shape(shape):
-- 
GitLab


From 0404ca1401ef04904ae4cf5be4619491dca32adf Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 22 Jan 2019 10:14:17 -0800
Subject: [PATCH 1061/1095] Automated rollback of commit
 80f47caee4bca375b8285e3b3185c7d29113b9e8

PiperOrigin-RevId: 230362704
---
 tensorflow/compiler/xla/executable_run_options.h | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/tensorflow/compiler/xla/executable_run_options.h b/tensorflow/compiler/xla/executable_run_options.h
index 6f36d11dfb..ba3217f31b 100644
--- a/tensorflow/compiler/xla/executable_run_options.h
+++ b/tensorflow/compiler/xla/executable_run_options.h
@@ -16,6 +16,9 @@ limitations under the License.
 #ifndef TENSORFLOW_COMPILER_XLA_EXECUTABLE_RUN_OPTIONS_H_
 #define TENSORFLOW_COMPILER_XLA_EXECUTABLE_RUN_OPTIONS_H_
 
+// Pulls in the ::stream_executor -> ::xla::se namespace alias.
+#include "tensorflow/compiler/xla/types.h"
+
 // These classes are forward declared so that ExecutableRunOptions can be linked
 // into an XLA-compiled binary without having to link all of the pointed-to
 // objects (e.g., for an ahead-of-time compiled CPU binary, the gpu tools don't
@@ -25,6 +28,12 @@ class Stream;
 class Platform;
 }  // namespace stream_executor
 
+namespace tensorflow {
+namespace thread {
+class ThreadPool;
+}  // namespace thread
+}  // namespace tensorflow
+
 namespace Eigen {
 struct ThreadPoolDevice;
 }  // namespace Eigen
-- 
GitLab


From 6624e34a94088a96a1ac5a6fafeb1055c96580b6 Mon Sep 17 00:00:00 2001
From: Peter Ma <pcma@google.com>
Date: Tue, 22 Jan 2019 10:25:42 -0800
Subject: [PATCH 1062/1095] Modification for internal usage.

PiperOrigin-RevId: 230364779
---
 tensorflow/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index 29d71c323a..0be7920d11 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -385,6 +385,7 @@ package_group(
         "-//third_party/tensorflow/python/estimator",
         "//learning/deepmind/...",
         "//learning/meta_rank/...",
+        "//platforms/performance/autograppler/...",
         "//tensorflow/...",
         "//tensorflow_estimator/contrib/...",
         "//tensorflow_fold/llgtm/...",
-- 
GitLab


From 951a5f1531bb65e90aaa76cec2e56230c11ccdab Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 22 Jan 2019 10:27:52 -0800
Subject: [PATCH 1063/1095] Trivial wording change for tf.Variable
 documentation.

PiperOrigin-RevId: 230365183
---
 tensorflow/python/ops/variables.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py
index 3392b1aad9..236a1cd537 100644
--- a/tensorflow/python/ops/variables.py
+++ b/tensorflow/python/ops/variables.py
@@ -304,7 +304,7 @@ class Variable(six.with_metaclass(VariableMetaclass,
   tf.cond(v, lambda: v.assign(False), my_false_fn)  # Note: this is broken.
   ```
 
-  Here replacing adding `use_resource=True` when constructing the variable will
+  Here, adding `use_resource=True` when constructing the variable will
   fix any nondeterminism issues:
 
   ```
@@ -1239,7 +1239,7 @@ class VariableV1(Variable):
   tf.cond(v, lambda: v.assign(False), my_false_fn)  # Note: this is broken.
   ```
 
-  Here replacing adding `use_resource=True` when constructing the variable will
+  Here, adding `use_resource=True` when constructing the variable will
   fix any nondeterminism issues:
   ```
   v = tf.Variable(True, use_resource=True)
-- 
GitLab


From 94ea59c903e00a0ffde07d541aea4a1aee3c563d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 22 Jan 2019 10:30:50 -0800
Subject: [PATCH 1064/1095] Automated rollback of commit
 80f47caee4bca375b8285e3b3185c7d29113b9e8

PiperOrigin-RevId: 230365804
---
 tensorflow/compiler/xla/executable_run_options.h | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/tensorflow/compiler/xla/executable_run_options.h b/tensorflow/compiler/xla/executable_run_options.h
index ba3217f31b..6f36d11dfb 100644
--- a/tensorflow/compiler/xla/executable_run_options.h
+++ b/tensorflow/compiler/xla/executable_run_options.h
@@ -16,9 +16,6 @@ limitations under the License.
 #ifndef TENSORFLOW_COMPILER_XLA_EXECUTABLE_RUN_OPTIONS_H_
 #define TENSORFLOW_COMPILER_XLA_EXECUTABLE_RUN_OPTIONS_H_
 
-// Pulls in the ::stream_executor -> ::xla::se namespace alias.
-#include "tensorflow/compiler/xla/types.h"
-
 // These classes are forward declared so that ExecutableRunOptions can be linked
 // into an XLA-compiled binary without having to link all of the pointed-to
 // objects (e.g., for an ahead-of-time compiled CPU binary, the gpu tools don't
@@ -28,12 +25,6 @@ class Stream;
 class Platform;
 }  // namespace stream_executor
 
-namespace tensorflow {
-namespace thread {
-class ThreadPool;
-}  // namespace thread
-}  // namespace tensorflow
-
 namespace Eigen {
 struct ThreadPoolDevice;
 }  // namespace Eigen
-- 
GitLab


From 13ab1b065de437efc3c1ba1d7c5c5bfe3edb7f99 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 22 Jan 2019 10:43:22 -0800
Subject: [PATCH 1065/1095] Fixed typo; see also changes to
 tag_python/saved_model/tag_constants.py and
 tools/api/golden/v2/tensorflow.saved_model.pbtxt

PiperOrigin-RevId: 230368577
---
 tensorflow/tools/compatibility/renames_v2.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
index 9d8d25016a..9c02c20f0f 100644
--- a/tensorflow/tools/compatibility/renames_v2.py
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -497,7 +497,7 @@ renames = {
     'tf.saved_model.Builder': 'tf.compat.v1.saved_model.Builder',
     'tf.saved_model.LEGACY_INIT_OP_KEY': 'tf.compat.v1.saved_model.LEGACY_INIT_OP_KEY',
     'tf.saved_model.MAIN_OP_KEY': 'tf.compat.v1.saved_model.MAIN_OP_KEY',
-    'tf.saved_model.TRAINING': 'tf.saved_model.TRANING',
+    'tf.saved_model.TRAINING': 'tf.saved_model.TRAINING',
     'tf.saved_model.build_tensor_info': 'tf.compat.v1.saved_model.build_tensor_info',
     'tf.saved_model.builder.SavedModelBuilder': 'tf.compat.v1.saved_model.builder.SavedModelBuilder',
     'tf.saved_model.constants.ASSETS_DIRECTORY': 'tf.saved_model.ASSETS_DIRECTORY',
@@ -538,7 +538,7 @@ renames = {
     'tf.saved_model.tag_constants.GPU': 'tf.saved_model.GPU',
     'tf.saved_model.tag_constants.SERVING': 'tf.saved_model.SERVING',
     'tf.saved_model.tag_constants.TPU': 'tf.saved_model.TPU',
-    'tf.saved_model.tag_constants.TRAINING': 'tf.saved_model.TRANING',
+    'tf.saved_model.tag_constants.TRAINING': 'tf.saved_model.TRAINING',
     'tf.saved_model.utils.build_tensor_info': 'tf.compat.v1.saved_model.utils.build_tensor_info',
     'tf.saved_model.utils.get_tensor_from_tensor_info': 'tf.compat.v1.saved_model.utils.get_tensor_from_tensor_info',
     'tf.scatter_add': 'tf.compat.v1.scatter_add',
-- 
GitLab


From 73cfd7b0c3f88deaf0a5e84ec84206dbcdd98e5e Mon Sep 17 00:00:00 2001
From: Bixia Zheng <bixia@google.com>
Date: Tue, 22 Jan 2019 10:56:24 -0800
Subject: [PATCH 1066/1095] [XLA] Fix zero sized HLO elimination to handle
 instructions without layouts.

The zero sized HLO elimination pass is run before layout assignment and may
encounter instructions without layouts. Fix the pass to use a default layout
to create the resulting literal when the instruction doesn't have a layout.

Add a test case.

PiperOrigin-RevId: 230371248
---
 .../xla/service/zero_sized_hlo_elimination.cc       | 10 ++++++++--
 .../xla/service/zero_sized_hlo_elimination_test.cc  | 13 +++++++++++++
 2 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/xla/service/zero_sized_hlo_elimination.cc b/tensorflow/compiler/xla/service/zero_sized_hlo_elimination.cc
index a76fbf3f66..661b7aa7d9 100644
--- a/tensorflow/compiler/xla/service/zero_sized_hlo_elimination.cc
+++ b/tensorflow/compiler/xla/service/zero_sized_hlo_elimination.cc
@@ -37,9 +37,15 @@ StatusOr<bool> ZeroSizedHloElimination::Run(HloModule* module) {
       }
       if (comp->IsRemovable(instruction) &&
           ShapeUtil::IsZeroElementArray(instruction->shape())) {
+        // If the instruction doesn't have a layout, use a default layout for
+        // the literal.
+        Shape shape = instruction->shape();
+        if (!LayoutUtil::HasLayout(shape)) {
+          LayoutUtil::SetToDefaultLayout(&shape);
+        }
         TF_RETURN_IF_ERROR(comp->ReplaceWithNewInstruction(
-            instruction, HloInstruction::CreateConstant(
-                             Literal::CreateFromShape(instruction->shape()))));
+            instruction,
+            HloInstruction::CreateConstant(Literal::CreateFromShape(shape))));
         changed = true;
       }
     }
diff --git a/tensorflow/compiler/xla/service/zero_sized_hlo_elimination_test.cc b/tensorflow/compiler/xla/service/zero_sized_hlo_elimination_test.cc
index a546a6d39c..572a79609e 100644
--- a/tensorflow/compiler/xla/service/zero_sized_hlo_elimination_test.cc
+++ b/tensorflow/compiler/xla/service/zero_sized_hlo_elimination_test.cc
@@ -82,5 +82,18 @@ TEST_F(ZeroSizedHloEliminationTest, DoesNotEliminateConstant) {
   EXPECT_FALSE(changed);
 }
 
+TEST_F(ZeroSizedHloEliminationTest, ZeroSizedInstructionWithoutLayoutFolded) {
+  Shape op_shape = ShapeUtil::MakeShape(F32, {4, 0});
+  op_shape.clear_layout();
+  HloInstruction* param1 = builder_.AddInstruction(
+      HloInstruction::CreateParameter(1, op_shape, "zero sized param 1"));
+  HloInstruction* param2 = builder_.AddInstruction(
+      HloInstruction::CreateParameter(2, op_shape, "zero sized param 2"));
+  builder_.AddInstruction(
+      HloInstruction::CreateBinary(op_shape, HloOpcode::kAdd, param1, param2));
+  TF_ASSERT_OK_AND_ASSIGN(bool changed, RunZeroSizedElimination());
+  EXPECT_TRUE(changed);
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From e1e77eacb92e6bfcfc3f554dcdfed2de59798d41 Mon Sep 17 00:00:00 2001
From: Andy Ly <lyandy@google.com>
Date: Tue, 22 Jan 2019 11:03:20 -0800
Subject: [PATCH 1067/1095] [Grappler] Improve error messages in
 MutableGraphView to make them easily distinguishable from one another.

PiperOrigin-RevId: 230372780
---
 .../core/grappler/mutable_graph_view.cc       | 299 ++++++++++--------
 tensorflow/core/grappler/mutable_graph_view.h |   7 +-
 .../core/grappler/mutable_graph_view_test.cc  | 205 ++++++++----
 3 files changed, 314 insertions(+), 197 deletions(-)

diff --git a/tensorflow/core/grappler/mutable_graph_view.cc b/tensorflow/core/grappler/mutable_graph_view.cc
index 0f171f7656..dc9be9ebe7 100644
--- a/tensorflow/core/grappler/mutable_graph_view.cc
+++ b/tensorflow/core/grappler/mutable_graph_view.cc
@@ -38,26 +38,6 @@ namespace grappler {
 
 namespace {
 
-const char kMissingMsg[] = "missing";
-const char kInvalidMsg[] = "invalid";
-const char kNoErrMsg[] = "";
-
-string FaninError(bool tensor_id_valid, bool node_missing) {
-  string s;
-  if (!tensor_id_valid && node_missing) {
-    s = absl::StrCat(" ", kInvalidMsg, "/", kMissingMsg);
-  } else if (!tensor_id_valid) {
-    s = absl::StrCat(" ", kInvalidMsg);
-  } else if (node_missing) {
-    s = absl::StrCat(" ", kMissingMsg);
-  }
-  return s;
-}
-
-string NodeError(bool node_missing) {
-  return node_missing ? absl::StrCat(" ", kMissingMsg) : kNoErrMsg;
-}
-
 bool IsTensorIdPortValid(const TensorId& tensor_id) {
   return tensor_id.index() >= Graph::kControlSlot;
 }
@@ -114,6 +94,32 @@ bool CanDedupControlWithRegularInput(const MutableGraphView& graph,
   return CanDedupControlWithRegularInput(graph, *control_node);
 }
 
+Status MutationError(absl::string_view function_name, absl::string_view params,
+                     absl::string_view msg) {
+  return errors::InvalidArgument(absl::Substitute(
+      "MutableGraphView::$0($1) error: $2.", function_name, params, msg));
+}
+
+string NodeMissingErrorMsg(absl::string_view node_name) {
+  return absl::Substitute("node '$0' was not found", node_name);
+}
+
+string InvalidFaninErrorMsg(absl::string_view fanin) {
+  return absl::Substitute("fanin '$0' must be a valid tensor id", fanin);
+}
+
+string RegularFaninErrorMsg(absl::string_view fanin) {
+  return absl::Substitute("fanin '$0' must be a regular tensor id", fanin);
+}
+
+Status UpdateFanoutsError(absl::string_view from_node_name,
+                          absl::string_view to_node_name,
+                          absl::string_view msg) {
+  string params = absl::Substitute("from_node_name='$0', to_node_name='$1'",
+                                   from_node_name, to_node_name);
+  return MutationError("UpdateFanouts", params, msg);
+}
+
 }  // namespace
 
 void MutableGraphView::AddAndDedupFanouts(NodeDef* node) {
@@ -217,8 +223,20 @@ NodeDef* MutableGraphView::AddNode(NodeDef&& node) {
 
 Status MutableGraphView::AddSubgraph(GraphDef&& subgraph) {
   if (subgraph.library().function_size() != 0) {
-    return errors::InvalidArgument(
-        "Can't add a subgraph with non-empty function library");
+    constexpr int kMaxNumNodesForDebugStr = 100;
+    string subgraph_str = "too large to display";
+    if (subgraph.node_size() < kMaxNumNodesForDebugStr) {
+      constexpr int kMaxLenSubgraphStr = 1000;
+      subgraph_str = subgraph.ShortDebugString();
+      if (subgraph_str.length() > kMaxLenSubgraphStr) {
+        subgraph_str =
+            absl::StrCat(subgraph_str.substr(0, kMaxLenSubgraphStr), "...");
+      }
+    }
+    string params = absl::Substitute("subgraph='$0'", subgraph_str);
+    return MutationError(
+        "AddSubgraph", params,
+        "can't add a subgraph with non-empty function library");
   }
 
   int node_size_before = graph()->node_size();
@@ -240,27 +258,20 @@ Status MutableGraphView::AddSubgraph(GraphDef&& subgraph) {
   return Status::OK();
 }
 
-Status MutableGraphView::UpdateFanouts(absl::string_view from_node,
-                                       absl::string_view to_node) {
-  NodeDef* from_node_ptr = GetNode(from_node);
-  NodeDef* to_node_ptr = GetNode(to_node);
-  if (from_node_ptr && to_node_ptr) {
-    return UpdateFanoutsInternal(from_node_ptr, to_node_ptr);
-  } else if (!from_node_ptr) {
-    return errors::Internal(absl::Substitute(
-        "Can't update fanouts from '$0' to '$1', from node was not found.",
-        from_node, to_node));
-  } else if (!to_node_ptr) {
-    return errors::Internal(absl::Substitute(
-        "Can't update fanouts from '$0' to '$1', to node was not found.",
-        from_node, to_node));
-  } else {
-    return errors::Internal(
-        absl::Substitute("Can't update fanouts from '$0' to '$1', from and to "
-                         "nodes were not found.",
-                         from_node, to_node));
+Status MutableGraphView::UpdateFanouts(absl::string_view from_node_name,
+                                       absl::string_view to_node_name) {
+  NodeDef* from_node = GetNode(from_node_name);
+  if (from_node == nullptr) {
+    return UpdateFanoutsError(from_node_name, to_node_name,
+                              NodeMissingErrorMsg(from_node_name));
   }
-  return Status::OK();
+  NodeDef* to_node = GetNode(to_node_name);
+  if (to_node == nullptr) {
+    return UpdateFanoutsError(from_node_name, to_node_name,
+                              NodeMissingErrorMsg(to_node_name));
+  }
+
+  return UpdateFanoutsInternal(from_node, to_node);
 }
 
 Status MutableGraphView::UpdateFanoutsInternal(NodeDef* from_node,
@@ -298,10 +309,11 @@ Status MutableGraphView::UpdateFanoutsInternal(NodeDef* from_node,
     if (to_node_is_switch) {
       // Trying to add a Switch as a control dependency, which if allowed will
       // make the graph invalid.
-      return errors::Internal(
-          absl::Substitute("Can't update fanouts from '$0' to '$1', to node is "
-                           "being added as a Switch control dependency.",
-                           from_node->name(), to_node->name()));
+      return UpdateFanoutsError(
+          from_node->name(), to_node->name(),
+          absl::Substitute("can't update fanouts to node '$0' as it will "
+                           "become a Switch control dependency",
+                           to_node->name()));
     }
 
     NodeDef* node = control_port.node;
@@ -410,20 +422,26 @@ bool MutableGraphView::AddFaninInternal(NodeDef* node,
 
 Status MutableGraphView::AddRegularFanin(absl::string_view node_name,
                                          const TensorId& fanin) {
-  NodeDef* node = GetNode(node_name);
-  NodeDef* fanin_node = GetNode(fanin.node());
+  auto error_status = [node_name, fanin](absl::string_view msg) {
+    string params = absl::Substitute("node_name='$0', fanin='$1'", node_name,
+                                     fanin.ToString());
+    return MutationError("AddRegularFanin", params, msg);
+  };
 
-  string node_err = NodeError(/*node_missing=*/node == nullptr);
-  string fanin_err = FaninError(IsTensorIdRegular(fanin),
-                                /*node_missing=*/fanin_node == nullptr);
-  if (!node_err.empty() || !fanin_err.empty()) {
-    return errors::Internal(absl::Substitute(
-        "Can't add$0 fanin '$1' as regular fanin to$2 node '$3'.", fanin_err,
-        fanin.ToString(), node_err, node_name));
+  if (!IsTensorIdRegular(fanin)) {
+    return error_status(RegularFaninErrorMsg(fanin.ToString()));
   }
   if (node_name == fanin.node()) {
-    return errors::Internal(absl::Substitute(
-        "Can't add fanin '$0' as regular fanin to self.", fanin.ToString()));
+    return error_status(absl::Substitute("can't add regular fanin '$0' to self",
+                                         fanin.ToString()));
+  }
+  NodeDef* node = GetNode(node_name);
+  if (node == nullptr) {
+    return error_status(NodeMissingErrorMsg(node_name));
+  }
+  NodeDef* fanin_node = GetNode(fanin.node());
+  if (fanin_node == nullptr) {
+    return error_status(NodeMissingErrorMsg(fanin.node()));
   }
 
   AddFaninInternal(node, {fanin_node, fanin.index()});
@@ -432,20 +450,26 @@ Status MutableGraphView::AddRegularFanin(absl::string_view node_name,
 
 Status MutableGraphView::AddControllingFanin(absl::string_view node_name,
                                              const TensorId& fanin) {
-  NodeDef* node = GetNode(node_name);
-  NodeDef* fanin_node = GetNode(fanin.node());
+  auto error_status = [node_name, fanin](absl::string_view msg) {
+    string params = absl::Substitute("node_name='$0', fanin='$1'", node_name,
+                                     fanin.ToString());
+    return MutationError("AddControllingFanin", params, msg);
+  };
 
-  string node_err = NodeError(/*node_missing=*/node == nullptr);
-  string fanin_err = FaninError(IsTensorIdPortValid(fanin),
-                                /*node_missing=*/fanin_node == nullptr);
-  if (!node_err.empty() || !fanin_err.empty()) {
-    return errors::Internal(
-        absl::Substitute("Can't add$0 controlling fanin '$1' to$2 node '$3'.",
-                         fanin_err, fanin.ToString(), node_err, node_name));
+  if (!IsTensorIdPortValid(fanin)) {
+    return error_status(InvalidFaninErrorMsg(fanin.ToString()));
   }
   if (node_name == fanin.node()) {
-    return errors::Internal(absl::Substitute(
-        "Can't add controlling fanin '$0' to self.", fanin.ToString()));
+    return error_status(absl::Substitute(
+        "can't add controlling fanin '$0' to self", fanin.ToString()));
+  }
+  NodeDef* node = GetNode(node_name);
+  if (node == nullptr) {
+    return error_status(NodeMissingErrorMsg(node_name));
+  }
+  NodeDef* fanin_node = GetNode(fanin.node());
+  if (fanin_node == nullptr) {
+    return error_status(NodeMissingErrorMsg(fanin.node()));
   }
 
   if (!IsSwitch(*fanin_node)) {
@@ -453,9 +477,10 @@ Status MutableGraphView::AddControllingFanin(absl::string_view node_name,
   } else {
     if (IsTensorIdControlling(fanin)) {
       // Can't add a Switch node control dependency.
-      return errors::Internal(absl::Substitute(
-          "Can't add Switch as controlling fanin '$0' to node '$1'.",
-          fanin.ToString(), node_name));
+      return error_status(
+          absl::Substitute("can't add controlling fanin '$0' as it will become "
+                           "a Switch control dependency",
+                           fanin.ToString()));
     }
     // We can't anchor control dependencies directly on the switch node: unlike
     // other nodes only one of the outputs of the switch node will be generated
@@ -468,10 +493,9 @@ Status MutableGraphView::AddControllingFanin(absl::string_view node_name,
       if (IsIdentity(*fanout.node) || IsIdentityNSingleInput(*fanout.node)) {
         if (ParseTensorName(fanout.node->input(0)) == fanin) {
           if (fanout.node->name() == node_name) {
-            return errors::Internal(absl::Substitute(
-                "Can't add found controlling fanin '$0' from fanin '$1' to "
-                "self.",
-                AsControlDependency(fanout.node->name()), fanin.ToString()));
+            return error_status(absl::Substitute(
+                "can't add found controlling fanin '$0' to self",
+                AsControlDependency(fanout.node->name())));
           }
           AddFaninInternal(node, {fanout.node, Graph::kControlSlot});
           return Status::OK();
@@ -483,9 +507,9 @@ Status MutableGraphView::AddControllingFanin(absl::string_view node_name,
     string ctrl_dep_name = AddPrefixToNodeName(
         absl::StrCat(fanin.node(), "_", fanin.index()), kMutableGraphViewCtrl);
     if (node_name == ctrl_dep_name) {
-      return errors::Internal(absl::Substitute(
-          "Can't add generated controlling fanin '$0' from fanin '$1' to self.",
-          AsControlDependency(ctrl_dep_name), fanin.ToString()));
+      return error_status(
+          absl::Substitute("can't add generated controlling fanin '$0' to self",
+                           AsControlDependency(ctrl_dep_name)));
     }
 
     // Reuse a previously created node, if possible.
@@ -563,21 +587,26 @@ bool MutableGraphView::RemoveRegularFaninInternal(NodeDef* node,
 
 Status MutableGraphView::RemoveRegularFanin(absl::string_view node_name,
                                             const TensorId& fanin) {
-  NodeDef* node = GetNode(node_name);
-  NodeDef* fanin_node = GetNode(fanin.node());
+  auto error_status = [node_name, fanin](absl::string_view msg) {
+    string params = absl::Substitute("node_name='$0', fanin='$1'", node_name,
+                                     fanin.ToString());
+    return MutationError("RemoveRegularFanin", params, msg);
+  };
 
-  string node_err = NodeError(/*node_missing=*/node == nullptr);
-  string fanin_err = FaninError(IsTensorIdRegular(fanin),
-                                /*node_missing=*/fanin_node == nullptr);
-  if (!node_err.empty() || !fanin_err.empty()) {
-    return errors::Internal(absl::Substitute(
-        "Can't remove$0 fanin '$1' as regular fanin from$2 node '$3'.",
-        fanin_err, fanin.ToString(), node_err, node_name));
+  if (!IsTensorIdRegular(fanin)) {
+    return error_status(RegularFaninErrorMsg(fanin.ToString()));
   }
   if (node_name == fanin.node()) {
-    return errors::Internal(
-        absl::Substitute("Can't remove fanin '$0' as regular fanin from self.",
-                         fanin.ToString()));
+    return error_status(absl::Substitute(
+        "can't remove regular fanin '$0' from self", fanin.ToString()));
+  }
+  NodeDef* node = GetNode(node_name);
+  if (node == nullptr) {
+    return error_status(NodeMissingErrorMsg(node_name));
+  }
+  NodeDef* fanin_node = GetNode(fanin.node());
+  if (fanin_node == nullptr) {
+    return error_status(NodeMissingErrorMsg(fanin.node()));
   }
 
   RemoveRegularFaninInternal(node, {fanin_node, fanin.index()});
@@ -604,21 +633,25 @@ bool MutableGraphView::RemoveControllingFaninInternal(NodeDef* node,
 
 Status MutableGraphView::RemoveControllingFanin(
     absl::string_view node_name, absl::string_view fanin_node_name) {
-  NodeDef* node = GetNode(node_name);
-  NodeDef* fanin_node = GetNode(fanin_node_name);
+  auto error_status = [node_name, fanin_node_name](absl::string_view msg) {
+    string params = absl::Substitute("node_name='$0', fanin_node_name='$1'",
+                                     node_name, fanin_node_name);
+    return MutationError("RemoveControllingFanin", params, msg);
+  };
 
-  string node_err = NodeError(/*node_missing=*/node == nullptr);
-  string fanin_err = NodeError(/*node_missing=*/fanin_node == nullptr);
-  if (!node_err.empty() || !fanin_err.empty()) {
-    return errors::Internal(absl::Substitute(
-        "Can't remove$0 controlling fanin '$1' from$2 node '$3'.", fanin_err,
-        AsControlDependency(string(fanin_node_name)), node_err, node_name));
-  }
   if (node_name == fanin_node_name) {
-    return errors::Internal(
-        absl::Substitute("Can't remove controlling fanin '$0' from self.",
+    return error_status(
+        absl::Substitute("can't remove controlling fanin '$0' from self",
                          AsControlDependency(string(fanin_node_name))));
   }
+  NodeDef* node = GetNode(node_name);
+  if (node == nullptr) {
+    return error_status(NodeMissingErrorMsg(node_name));
+  }
+  NodeDef* fanin_node = GetNode(fanin_node_name);
+  if (fanin_node == nullptr) {
+    return error_status(NodeMissingErrorMsg(fanin_node_name));
+  }
 
   RemoveControllingFaninInternal(node, fanin_node);
   return Status::OK();
@@ -629,8 +662,11 @@ Status MutableGraphView::RemoveAllFanins(absl::string_view node_name,
   NodeDef* node = GetNode(node_name);
 
   if (node == nullptr) {
-    return errors::Internal(absl::Substitute(
-        "Can't remove all fanins from missing node '$0'.", node_name));
+    string params =
+        absl::Substitute("node_name='$0', keep_controlling_fanins=$1",
+                         node_name, keep_controlling_fanins);
+    return MutationError("RemoveAllFanins", params,
+                         NodeMissingErrorMsg(node_name));
   }
 
   if (node->input().empty()) {
@@ -657,21 +693,30 @@ Status MutableGraphView::RemoveAllFanins(absl::string_view node_name,
 Status MutableGraphView::UpdateFanin(absl::string_view node_name,
                                      const TensorId& from_fanin,
                                      const TensorId& to_fanin) {
+  auto error_status = [node_name, from_fanin, to_fanin](absl::string_view msg) {
+    string params =
+        absl::Substitute("node_name='$0', from_fanin='$1', to_fanin='$2'",
+                         node_name, from_fanin.ToString(), to_fanin.ToString());
+    return MutationError("UpdateFanin", params, msg);
+  };
+
+  if (!IsTensorIdPortValid(from_fanin)) {
+    return error_status(InvalidFaninErrorMsg(from_fanin.ToString()));
+  }
+  if (!IsTensorIdPortValid(to_fanin)) {
+    return error_status(InvalidFaninErrorMsg(to_fanin.ToString()));
+  }
   NodeDef* node = GetNode(node_name);
+  if (node == nullptr) {
+    return error_status(NodeMissingErrorMsg(node_name));
+  }
   NodeDef* from_fanin_node = GetNode(from_fanin.node());
+  if (from_fanin_node == nullptr) {
+    return error_status(NodeMissingErrorMsg(from_fanin.node()));
+  }
   NodeDef* to_fanin_node = GetNode(to_fanin.node());
-
-  string node_err = NodeError(/*node_missing=*/node == nullptr);
-  string from_fanin_err =
-      FaninError(IsTensorIdPortValid(from_fanin),
-                 /*node_missing=*/from_fanin_node == nullptr);
-  string to_fanin_err = FaninError(IsTensorIdPortValid(to_fanin),
-                                   /*node_missing=*/to_fanin_node == nullptr);
-  if (!node_err.empty() || !from_fanin_err.empty() || !to_fanin_err.empty()) {
-    return errors::Internal(absl::Substitute(
-        "Can't update$0 fanin '$1' to$2 fanin '$3' in$4 node '$5'.",
-        from_fanin_err, from_fanin.ToString(), to_fanin_err,
-        to_fanin.ToString(), node_err, node_name));
+  if (to_fanin_node == nullptr) {
+    return error_status(NodeMissingErrorMsg(from_fanin.node()));
   }
 
   // When replacing a non control dependency fanin with a control dependency, or
@@ -679,15 +724,13 @@ Status MutableGraphView::UpdateFanin(absl::string_view node_name,
   bool to_fanin_is_control = IsTensorIdControlling(to_fanin);
   if (to_fanin_is_control && IsSwitch(*to_fanin_node)) {
     // Can't add Switch node as a control dependency.
-    return errors::Internal(absl::Substitute(
-        "Can't update fanin '$0' to fanin '$1' in node '$2', to fanin is a "
-        "Switch control dependency.",
-        from_fanin.ToString(), to_fanin.ToString(), node_name));
+    return error_status(
+        absl::Substitute("can't update to fanin '$0' as it will become a "
+                         "Switch control dependency",
+                         to_fanin.ToString()));
   }
   if (node_name == from_fanin.node() || node_name == to_fanin.node()) {
-    return errors::Internal(absl::Substitute(
-        "Can't update fanin '$0' to fanin '$1' in self '$2'.",
-        from_fanin.ToString(), to_fanin.ToString(), node_name));
+    return error_status("can't update fanin to or from self");
   }
 
   if (from_fanin == to_fanin) {
@@ -797,16 +840,18 @@ Status MutableGraphView::CheckNodesCanBeDeleted(
   };
 
   if (!missing_nodes.empty()) {
-    VLOG(1) << absl::Substitute("Attempting to delete missing node(s) [$0]",
+    VLOG(2) << absl::Substitute("Attempting to delete missing node(s) [$0].",
                                 sort_and_sample(&missing_nodes));
   }
   if (!nodes_with_fanouts.empty()) {
     std::vector<string> input_node_names(nodes_to_delete.begin(),
                                          nodes_to_delete.end());
-    return errors::Internal(absl::Substitute(
-        "Can't delete node(s) with retained fanout(s) [$0] from node(s) [$1].",
-        sort_and_sample(&nodes_with_fanouts),
-        sort_and_sample(&input_node_names)));
+    string params = absl::Substitute("nodes_to_delete={$0}",
+                                     sort_and_sample(&input_node_names));
+    string error_msg =
+        absl::Substitute("can't delete node(s) with retained fanouts(s) [$0]",
+                         sort_and_sample(&nodes_with_fanouts));
+    return MutationError("DeleteNodes", params, error_msg);
   }
 
   return Status::OK();
diff --git a/tensorflow/core/grappler/mutable_graph_view.h b/tensorflow/core/grappler/mutable_graph_view.h
index f08b5dde08..73c32f4e55 100644
--- a/tensorflow/core/grappler/mutable_graph_view.h
+++ b/tensorflow/core/grappler/mutable_graph_view.h
@@ -73,8 +73,8 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   // library into the underlying graph.
   Status AddSubgraph(GraphDef&& subgraph);
 
-  // Updates all fanouts (input ports fetching output tensors) from `from_node`
-  // to the `to_node`, including control dependencies.
+  // Updates all fanouts (input ports fetching output tensors) from
+  // `from_node_name` to the `to_node_name`, including control dependencies.
   //
   // Example: We have 3 nodes that use `bar` node output tensors as inputs:
   //   1. foo1(bar:0, bar:1, other:0)
@@ -85,7 +85,8 @@ class MutableGraphView : public internal::GraphViewInternal<GraphDef, NodeDef> {
   //   1. foo1(new_bar:0, new_bar:1, other:0)
   //   2. foo2(new_bar:1, other:1)
   //   3. foo3(other:2, ^new_bar)
-  Status UpdateFanouts(absl::string_view from_node, absl::string_view to_node);
+  Status UpdateFanouts(absl::string_view from_node_name,
+                       absl::string_view to_node_name);
 
   // Adds regular fanin `fanin` to node `node_name`. If the node or fanin do not
   // exist in the graph, nothing will be modified in the graph. Otherwise fanin
diff --git a/tensorflow/core/grappler/mutable_graph_view_test.cc b/tensorflow/core/grappler/mutable_graph_view_test.cc
index ea88359e2e..8b94364d54 100644
--- a/tensorflow/core/grappler/mutable_graph_view_test.cc
+++ b/tensorflow/core/grappler/mutable_graph_view_test.cc
@@ -189,8 +189,14 @@ TEST(MutableGraphViewTest, AddSubgraphWithFunctionLibrary) {
       },
       /*funcs=*/{x_times_two});
 
+  string subgraph_str = subgraph.ShortDebugString();
   Status status = graph.AddSubgraph(std::move(subgraph));
   EXPECT_FALSE(status.ok());
+  string expected_msg = absl::Substitute(
+      "MutableGraphView::AddSubgraph(subgraph='$0') error: can't add a "
+      "subgraph with non-empty function library.",
+      subgraph_str);
+  EXPECT_EQ(status.error_message(), expected_msg);
 }
 
 TEST(MutableGraphViewTest, AddAndUpdateFanouts) {
@@ -289,14 +295,16 @@ TEST(MutableGraphViewTest, UpdateFanoutsToSwitchWithControlFromSwitch) {
   Status s = graph.UpdateFanouts("a", "b");
   EXPECT_FALSE(s.ok());
   string expected_msg =
-      "Can't update fanouts from 'a' to 'b', to node is being added as a "
-      "Switch control dependency.";
+      "MutableGraphView::UpdateFanouts(from_node_name='a', to_node_name='b') "
+      "error: can't update fanouts to node 'b' as it will become a Switch "
+      "control dependency.";
   EXPECT_EQ(s.error_message(), expected_msg);
   s = graph.UpdateFanouts("d", "b");
   EXPECT_FALSE(s.ok());
   expected_msg =
-      "Can't update fanouts from 'd' to 'b', to node is being added as a "
-      "Switch control dependency.";
+      "MutableGraphView::UpdateFanouts(from_node_name='d', to_node_name='b') "
+      "error: can't update fanouts to node 'b' as it will become a Switch "
+      "control dependency.";
   EXPECT_EQ(s.error_message(), expected_msg);
 
   EXPECT_EQ(graph.graph()->node_size(), 5);
@@ -431,62 +439,80 @@ TEST(MutableGraphViewTest, AddRegularFanin) {
                       error_msg, {"c:1", "^b", "^a"});
 
   // Add control to node with 1 input 0 controls.
-  error_msg = "Can't add invalid fanin '^b' as regular fanin to node 'foo_1'.";
+  error_msg =
+      "MutableGraphView::AddRegularFanin(node_name='foo_1', fanin='^b') error: "
+      "fanin '^b' must be a regular tensor id.";
   TestAddRegularFanin("foo_1", /*node_exists=*/true, {"b", Graph::kControlSlot},
                       /*success=*/false, error_msg, {"a"});
   // Add control to node with multiple inputs and 0 controls.
-  error_msg = "Can't add invalid fanin '^c' as regular fanin to node 'foo_3'.";
+  error_msg =
+      "MutableGraphView::AddRegularFanin(node_name='foo_3', fanin='^c') error: "
+      "fanin '^c' must be a regular tensor id.";
   TestAddRegularFanin("foo_3", /*node_exists=*/true, {"c", Graph::kControlSlot},
                       /*success=*/false, error_msg, {"b", "a:1", "a:1"});
   // Add control to node with 1 input multiple controls.
-  error_msg = "Can't add invalid fanin '^d' as regular fanin to node 'foo_2'.";
+  error_msg =
+      "MutableGraphView::AddRegularFanin(node_name='foo_2', fanin='^d') error: "
+      "fanin '^d' must be a regular tensor id.";
   TestAddRegularFanin("foo_2", /*node_exists=*/true, {"d", Graph::kControlSlot},
                       /*success=*/false, error_msg, {"b", "^a", "^c"});
   // Add control to node with multiple input multiple controls.
-  error_msg = "Can't add invalid fanin '^a' as regular fanin to node 'foo_4'.";
+  error_msg =
+      "MutableGraphView::AddRegularFanin(node_name='foo_4', fanin='^a') error: "
+      "fanin '^a' must be a regular tensor id.";
   TestAddRegularFanin("foo_4", /*node_exists=*/true, {"a", Graph::kControlSlot},
                       /*success=*/false, error_msg,
                       {"a", "b:2", "b:2", "^c", "^d"});
   // Add control to node with 0 inputs 0 controls.
-  error_msg = "Can't add invalid fanin '^a' as regular fanin to node 'foo_5'.";
+  error_msg =
+      "MutableGraphView::AddRegularFanin(node_name='foo_5', fanin='^a') error: "
+      "fanin '^a' must be a regular tensor id.";
   TestAddRegularFanin("foo_5", /*node_exists=*/true, {"a", Graph::kControlSlot},
                       /*success=*/false, error_msg, {});
   // Add control to node with 0 inputs multiple controls.
-  error_msg = "Can't add invalid fanin '^c' as regular fanin to node 'foo_6'.";
+  error_msg =
+      "MutableGraphView::AddRegularFanin(node_name='foo_6', fanin='^c') error: "
+      "fanin '^c' must be a regular tensor id.";
   TestAddRegularFanin("foo_6", /*node_exists=*/true, {"c", Graph::kControlSlot},
                       /*success=*/false, error_msg, {"^a", "^b"});
   // Add control to node with control that already exists.
-  error_msg = "Can't add invalid fanin '^a' as regular fanin to node 'foo_2'.";
+  error_msg =
+      "MutableGraphView::AddRegularFanin(node_name='foo_2', fanin='^a') error: "
+      "fanin '^a' must be a regular tensor id.";
   TestAddRegularFanin("foo_2", /*node_exists=*/true, {"a", Graph::kControlSlot},
                       /*success=*/false, error_msg, {"b", "^a", "^c"});
 
   // Add fanin to node where node is missing.
   error_msg =
-      "Can't add fanin 'a:0' as regular fanin to missing node 'foo_missing'.";
+      "MutableGraphView::AddRegularFanin(node_name='foo_missing', fanin='a:0') "
+      "error: node 'foo_missing' was not found.";
   TestAddRegularFanin("foo_missing", /*node_exists=*/false, {"a", 0},
                       /*success=*/false, error_msg, {});
   // Add fanin to node where fanin is missing.
   error_msg =
-      "Can't add missing fanin 'bar_missing:0' as regular fanin to node "
-      "'foo_1'.";
+      "MutableGraphView::AddRegularFanin(node_name='foo_1', "
+      "fanin='bar_missing:0') error: node 'bar_missing' was not found.";
   TestAddRegularFanin("foo_1", /*node_exists=*/true, {"bar_missing", 0},
                       /*success=*/false, error_msg, {"a"});
   // Add fanin to node where node and fanin are missing.
   error_msg =
-      "Can't add missing fanin 'bar_missing:0' as regular fanin to missing "
-      "node 'foo_missing'.";
+      "MutableGraphView::AddRegularFanin(node_name='foo_missing', "
+      "fanin='bar_missing:0') error: node 'foo_missing' was not found.";
   TestAddRegularFanin("foo_missing", /*node_exists=*/false, {"bar_missing", 0},
                       /*success=*/false, error_msg, {});
   // Add control fanin to node where node and fanin are missing.
   error_msg =
-      "Can't add invalid/missing fanin '^bar_missing' as regular fanin to "
-      "missing node 'foo_missing'.";
+      "MutableGraphView::AddRegularFanin(node_name='foo_missing', "
+      "fanin='^bar_missing') error: fanin '^bar_missing' must be a regular "
+      "tensor id.";
   TestAddRegularFanin("foo_missing", /*node_exists=*/false,
                       {"bar_missing", Graph::kControlSlot},
                       /*success=*/false, error_msg, {});
 
   // Add self to create cycle.
-  error_msg = "Can't add fanin 'foo_6:2' as regular fanin to self.";
+  error_msg =
+      "MutableGraphView::AddRegularFanin(node_name='foo_6', fanin='foo_6:2') "
+      "error: can't add regular fanin 'foo_6:2' to self.";
   TestAddRegularFanin("foo_6", /*node_exists=*/true, {"foo_6", 2},
                       /*success=*/false, error_msg, {"^a", "^b"});
 }
@@ -558,19 +584,22 @@ TEST(MutableGraphViewTest, RemoveRegularFanin) {
 
   // Remove control from node with 1 input multiple controls.
   error_msg =
-      "Can't remove invalid fanin '^a' as regular fanin from node 'foo_2'.";
+      "MutableGraphView::RemoveRegularFanin(node_name='foo_2', fanin='^a') "
+      "error: fanin '^a' must be a regular tensor id.";
   TestRemoveRegularFanin("foo_2", /*node_exists=*/true,
                          {"a", Graph::kControlSlot},
                          /*success=*/false, error_msg, {"b", "^a", "^c"});
   // Remove control from node with multiple input multiple controls.
   error_msg =
-      "Can't remove invalid fanin '^d' as regular fanin from node 'foo_4'.";
+      "MutableGraphView::RemoveRegularFanin(node_name='foo_4', fanin='^d') "
+      "error: fanin '^d' must be a regular tensor id.";
   TestRemoveRegularFanin(
       "foo_4", /*node_exists=*/true, {"d", Graph::kControlSlot},
       /*success=*/false, error_msg, {"a", "b:2", "b:2", "^c", "^d"});
   // Remove control from node with 0 inputs multiple controls.
   error_msg =
-      "Can't remove invalid fanin '^a' as regular fanin from node 'foo_6'.";
+      "MutableGraphView::RemoveRegularFanin(node_name='foo_6', fanin='^a') "
+      "error: fanin '^a' must be a regular tensor id.";
   TestRemoveRegularFanin("foo_6", /*node_exists=*/true,
                          {"a", Graph::kControlSlot},
                          /*success=*/false, error_msg, {"^a", "^b"});
@@ -585,51 +614,57 @@ TEST(MutableGraphViewTest, RemoveRegularFanin) {
 
   // Remove control from node with 1 input 0 controls.
   error_msg =
-      "Can't remove invalid fanin '^b' as regular fanin from node 'foo_1'.";
+      "MutableGraphView::RemoveRegularFanin(node_name='foo_1', fanin='^b') "
+      "error: fanin '^b' must be a regular tensor id.";
   TestRemoveRegularFanin("foo_1", /*node_exists=*/true,
                          {"b", Graph::kControlSlot},
                          /*success=*/false, error_msg, {"a"});
   // Remove control from node with multiple inputs and 0 controls.
   error_msg =
-      "Can't remove invalid fanin '^c' as regular fanin from node 'foo_3'.";
+      "MutableGraphView::RemoveRegularFanin(node_name='foo_3', fanin='^c') "
+      "error: fanin '^c' must be a regular tensor id.";
   TestRemoveRegularFanin("foo_3", /*node_exists=*/true,
                          {"c", Graph::kControlSlot},
                          /*success=*/false, error_msg, {"b", "a:1", "a:1"});
   // Remove control from node with 0 inputs 0 controls.
   error_msg =
-      "Can't remove invalid fanin '^a' as regular fanin from node 'foo_5'.";
+      "MutableGraphView::RemoveRegularFanin(node_name='foo_5', fanin='^a') "
+      "error: fanin '^a' must be a regular tensor id.";
   TestRemoveRegularFanin("foo_5", /*node_exists=*/true,
                          {"a", Graph::kControlSlot},
                          /*success=*/false, error_msg, {});
 
   // Remove fanin from node where node is missing.
   error_msg =
-      "Can't remove fanin 'a:0' as regular fanin from missing node "
-      "'foo_missing'.";
+      "MutableGraphView::RemoveRegularFanin(node_name='foo_missing', "
+      "fanin='a:0') error: node 'foo_missing' was not found.";
   TestRemoveRegularFanin("foo_missing", /*node_exists=*/false, {"a", 0},
                          /*success=*/false, error_msg, {});
   // Remove fanin from node where fanin is missing.
   error_msg =
-      "Can't remove missing fanin 'bar_missing:0' as regular fanin from node "
-      "'foo_1'.";
+      "MutableGraphView::RemoveRegularFanin(node_name='foo_1', "
+      "fanin='bar_missing:0') error: node 'bar_missing' was not found.";
   TestRemoveRegularFanin("foo_1", /*node_exists=*/true, {"bar_missing", 0},
                          /*success=*/false, error_msg, {"a"});
   // Remove fanin from node where node and fanin are missing.
   error_msg =
-      "Can't remove missing fanin 'bar_missing:0' as regular fanin from "
-      "missing node 'foo_missing'.";
+      "MutableGraphView::RemoveRegularFanin(node_name='foo_missing', "
+      "fanin='bar_missing:0') error: node 'foo_missing' was not found.";
   TestRemoveRegularFanin("foo_missing", /*node_exists=*/false,
                          {"bar_missing", 0}, /*success=*/false, error_msg, {});
   // Remove control from node where node and fanin are missing.
   error_msg =
-      "Can't remove invalid/missing fanin '^bar_missing' as regular fanin from "
-      "missing node 'foo_missing'.";
+      "MutableGraphView::RemoveRegularFanin(node_name='foo_missing', "
+      "fanin='^bar_missing') error: fanin '^bar_missing' must be a regular "
+      "tensor id.";
   TestRemoveRegularFanin("foo_missing", /*node_exists=*/false,
                          {"bar_missing", Graph::kControlSlot},
                          /*success=*/false, error_msg, {});
 
   // Remove self.
-  error_msg = "Can't remove fanin 'foo_6:2' as regular fanin from self.";
+  error_msg =
+      "MutableGraphView::RemoveRegularFanin(node_name='foo_6', "
+      "fanin='foo_6:2') error: can't remove regular fanin 'foo_6:2' from self.";
   TestRemoveRegularFanin("foo_6", /*node_exists=*/true, {"foo_6", 2},
                          /*success=*/false, error_msg, {"^a", "^b"});
 }
@@ -718,10 +753,15 @@ TEST(MutableGraphViewTest, RemoveAllFanins) {
                       /*success=*/true, error_msg, {"^a", "^b"});
 
   // Remove all fanins from node where node is missing.
-  error_msg = "Can't remove all fanins from missing node 'foo_missing'.";
+  error_msg =
+      "MutableGraphView::RemoveAllFanins(node_name='foo_missing', "
+      "keep_controlling_fanins=false) error: node 'foo_missing' was not found.";
   TestRemoveAllFanins("foo_missing", /*node_exists=*/false,
                       /*keep_controlling_nodes=*/false,
                       /*success=*/false, error_msg, {});
+  error_msg =
+      "MutableGraphView::RemoveAllFanins(node_name='foo_missing', "
+      "keep_controlling_fanins=true) error: node 'foo_missing' was not found.";
   TestRemoveAllFanins("foo_missing", /*node_exists=*/false,
                       /*keep_controlling_nodes=*/true,
                       /*success=*/false, error_msg, {});
@@ -794,63 +834,77 @@ TEST(MutableGraphViewTest, UpdateFanin) {
 
   // Update fanin of node where node is missing.
   error_msg =
-      "Can't update fanin 'a:0' to fanin 'a:1' in missing node 'foo_missing'.";
+      "MutableGraphView::UpdateFanin(node_name='foo_missing', "
+      "from_fanin='a:0', to_fanin='a:1') error: node 'foo_missing' was not "
+      "found.";
   TestUpdateFanin("foo_missing", /*node_exists=*/false, {"a", 0}, {"a", 1},
                   /*success=*/false, error_msg, {});
   // Update fanin of node where from fanin is missing.
   error_msg =
-      "Can't update missing fanin 'from_bar_missing:0' to fanin 'a:1' in node "
-      "'foo_1'.";
+      "MutableGraphView::UpdateFanin(node_name='foo_1', "
+      "from_fanin='from_bar_missing:0', to_fanin='a:1') error: node "
+      "'from_bar_missing' was not found.";
   TestUpdateFanin("foo_1", /*node_exists=*/true, {"from_bar_missing", 0},
                   {"a", 1},
                   /*success=*/false, error_msg, {"a"});
   // Update fanin of node where to fanin is missing.
   error_msg =
-      "Can't update fanin 'a:0' to missing fanin 'to_bar_missing:1' in node "
-      "'foo_1'.";
+      "MutableGraphView::UpdateFanin(node_name='foo_1', from_fanin='a:0', "
+      "to_fanin='to_bar_missing:1') error: node 'a' was not found.";
   TestUpdateFanin("foo_1", /*node_exists=*/true, {"a", 0},
                   {"to_bar_missing", 1}, /*success=*/false, error_msg, {"a"});
   // Update fanin of node where from/to fanins and node are missing.
   error_msg =
-      "Can't update missing fanin 'from_bar_missing:0' to missing fanin "
-      "'to_bar_missing:1' in missing node 'foo_missing'.";
+      "MutableGraphView::UpdateFanin(node_name='foo_missing', "
+      "from_fanin='from_bar_missing:0', to_fanin='to_bar_missing:1') error: "
+      "node 'foo_missing' was not found.";
   TestUpdateFanin("foo_missing", /*node_exists=*/false, {"from_bar_missing", 0},
                   {"to_bar_missing", 1},
                   /*success=*/false, error_msg, {});
   // Update fanin of node where from fanin is invalid.
   error_msg =
-      "Can't update invalid fanin 'a:-2' to fanin 'a:0' in node 'foo_1'.";
+      "MutableGraphView::UpdateFanin(node_name='foo_1', from_fanin='a:-2', "
+      "to_fanin='a:0') error: fanin 'a:-2' must be a valid tensor id.";
   TestUpdateFanin("foo_1", /*node_exists=*/true, {"a", -2}, {"a", 0},
                   /*success=*/false, error_msg, {"a"});
   // Update fanin of node where to fanin is invalid.
   error_msg =
-      "Can't update fanin 'a:0' to invalid fanin 'a:-2' in node 'foo_1'.";
+      "MutableGraphView::UpdateFanin(node_name='foo_1', from_fanin='a:0', "
+      "to_fanin='a:-2') error: fanin 'a:-2' must be a valid tensor id.";
   TestUpdateFanin("foo_1", /*node_exists=*/true, {"a", 0}, {"a", -2},
                   /*success=*/false, error_msg, {"a"});
   // Update fanin of node where from/to fanins are invalid and missing and node
   // is missing.
   error_msg =
-      "Can't update invalid/missing fanin 'from_bar_missing:-2' to "
-      "invalid/missing fanin 'to_bar_missing:-3' in missing node "
-      "'foo_missing'.";
+      "MutableGraphView::UpdateFanin(node_name='foo_missing', "
+      "from_fanin='from_bar_missing:-2', to_fanin='to_bar_missing:-3') error: "
+      "fanin 'from_bar_missing:-2' must be a valid tensor id.";
   TestUpdateFanin("foo_missing", /*node_exists=*/false,
                   {"from_bar_missing", -2}, {"to_bar_missing", -3},
                   /*success=*/false, error_msg, {});
 
   // Update to self to create cycle.
-  error_msg = "Can't update fanin 'b:2' to fanin 'foo_4:3' in self 'foo_4'.";
+  error_msg =
+      "MutableGraphView::UpdateFanin(node_name='foo_4', from_fanin='b:2', "
+      "to_fanin='foo_4:3') error: can't update fanin to or from self.";
   TestUpdateFanin("foo_4", /*node_exists=*/true, {"b", 2}, {"foo_4", 3},
                   /*success=*/false, error_msg,
                   {"a", "b:2", "b:2", "^c", "^d"});
-  error_msg = "Can't update fanin 'b:2' to fanin '^foo_4' in self 'foo_4'.";
+  error_msg =
+      "MutableGraphView::UpdateFanin(node_name='foo_4', from_fanin='b:2', "
+      "to_fanin='^foo_4') error: can't update fanin to or from self.";
   TestUpdateFanin(
       "foo_4", /*node_exists=*/true, {"b", 2}, {"foo_4", Graph::kControlSlot},
       /*success=*/false, error_msg, {"a", "b:2", "b:2", "^c", "^d"});
-  error_msg = "Can't update fanin '^c' to fanin 'foo_4:4' in self 'foo_4'.";
+  error_msg =
+      "MutableGraphView::UpdateFanin(node_name='foo_4', from_fanin='^c', "
+      "to_fanin='foo_4:4') error: can't update fanin to or from self.";
   TestUpdateFanin(
       "foo_4", /*node_exists=*/true, {"c", Graph::kControlSlot}, {"foo_4", 4},
       /*success=*/false, error_msg, {"a", "b:2", "b:2", "^c", "^d"});
-  error_msg = "Can't update fanin '^c' to fanin '^foo_4' in self 'foo_4'.";
+  error_msg =
+      "MutableGraphView::UpdateFanin(node_name='foo_4', from_fanin='^c', "
+      "to_fanin='^foo_4') error: can't update fanin to or from self.";
   TestUpdateFanin("foo_4", /*node_exists=*/true, {"c", Graph::kControlSlot},
                   {"foo_4", Graph::kControlSlot}, /*success=*/false, error_msg,
                   {"a", "b:2", "b:2", "^c", "^d"});
@@ -868,8 +922,9 @@ void TestUpdateFaninFromFaninToNodeAsSwitchControl(const TensorId& fanin) {
   Status s = graph.UpdateFanin("c", fanin, {"b", Graph::kControlSlot});
   EXPECT_FALSE(s.ok());
   string expected_msg = absl::Substitute(
-      "Can't update fanin '$0' to fanin '^b' in node 'c', to fanin is a Switch "
-      "control dependency.",
+      "MutableGraphView::UpdateFanin(node_name='c', from_fanin='$0', "
+      "to_fanin='^b') error: can't update to fanin '^b' as it will become a "
+      "Switch control dependency.",
       fanin.ToString());
   EXPECT_EQ(s.error_message(), expected_msg);
 
@@ -1090,18 +1145,23 @@ TEST(MutableGraphViewTest, AddControllingFaninMissing) {
   // Missing fanin.
   Status s = graph.AddControllingFanin("a", {"c", Graph::kControlSlot});
   EXPECT_FALSE(s.ok());
-  string expected_msg = "Can't add missing controlling fanin '^c' to node 'a'.";
+  string expected_msg =
+      "MutableGraphView::AddControllingFanin(node_name='a', fanin='^c') error: "
+      "node 'c' was not found.";
   EXPECT_EQ(s.error_message(), expected_msg);
   // Missing node.
   s = graph.AddControllingFanin("d", {"a", Graph::kControlSlot});
   EXPECT_FALSE(s.ok());
-  expected_msg = "Can't add controlling fanin '^a' to missing node 'd'.";
+  expected_msg =
+      "MutableGraphView::AddControllingFanin(node_name='d', fanin='^a') error: "
+      "node 'd' was not found.";
   EXPECT_EQ(s.error_message(), expected_msg);
   // Missing node and fanin.
   s = graph.AddControllingFanin("c", {"d", Graph::kControlSlot});
   EXPECT_FALSE(s.ok());
   expected_msg =
-      "Can't add missing controlling fanin '^d' to missing node 'c'.";
+      "MutableGraphView::AddControllingFanin(node_name='c', fanin='^d') error: "
+      "node 'c' was not found.";
   EXPECT_EQ(s.error_message(), expected_msg);
 
   ASSERT_EQ(graph.graph()->node_size(), 2);
@@ -1158,7 +1218,9 @@ TEST(MutableGraphViewTest, AddControllingFaninSwitch) {
   Status s = graph.AddControllingFanin("a", {"b", Graph::kControlSlot});
   EXPECT_FALSE(s.ok());
   string expected_msg =
-      "Can't add Switch as controlling fanin '^b' to node 'a'.";
+      "MutableGraphView::AddControllingFanin(node_name='a', fanin='^b') error: "
+      "can't add controlling fanin '^b' as it will become a Switch control "
+      "dependency.";
   EXPECT_EQ(s.error_message(), expected_msg);
 
   ASSERT_EQ(graph.graph()->node_size(), 2);
@@ -1264,7 +1326,9 @@ void TestAddControllingFaninSelfLoops(absl::string_view node_name,
 }
 
 TEST(MutableGraphViewTest, AddControllingFaninSelfLoops) {
-  string error_msg = "Can't add controlling fanin '^a' to self.";
+  string error_msg =
+      "MutableGraphView::AddControllingFanin(node_name='a', fanin='^a') error: "
+      "can't add controlling fanin '^a' to self.";
   TestAddControllingFaninSelfLoops("a", {"a", Graph::kControlSlot}, error_msg);
 
   // Adding Switch control dependency to Identity consumer. Node `c` is
@@ -1273,7 +1337,8 @@ TEST(MutableGraphViewTest, AddControllingFaninSelfLoops) {
   // Identity, this will introduce a self loop, so no control dependency should
   // be added.
   error_msg =
-      "Can't add found controlling fanin '^c' from fanin 'b:0' to self.";
+      "MutableGraphView::AddControllingFanin(node_name='c', fanin='b:0') "
+      "error: can't add found controlling fanin '^c' to self.";
   TestAddControllingFaninSelfLoops("c", {"b", 0}, error_msg);
 
   // Adding Switch control dependency to Identity consumer. Node `d` is
@@ -1282,7 +1347,8 @@ TEST(MutableGraphViewTest, AddControllingFaninSelfLoops) {
   // Identity, this will introduce a self loop, so no control dependency should
   // be added.
   error_msg =
-      "Can't add found controlling fanin '^d' from fanin 'b:1' to self.";
+      "MutableGraphView::AddControllingFanin(node_name='d', fanin='b:1') "
+      "error: can't add found controlling fanin '^d' to self.";
   TestAddControllingFaninSelfLoops("d", {"b", 1}, error_msg);
 }
 
@@ -1305,8 +1371,9 @@ TEST(MutableGraphViewTest, AddControllingFaninSelfLoopsGeneratedIdentity) {
   Status s = graph.AddControllingFanin("ConstantFoldingCtrl/b_1", {"b", 1});
   EXPECT_FALSE(s.ok());
   string expected_msg =
-      "Can't add generated controlling fanin '^ConstantFoldingCtrl/b_1' from "
-      "fanin 'b:1' to self.";
+      "MutableGraphView::AddControllingFanin(node_name='ConstantFoldingCtrl/"
+      "b_1', fanin='b:1') error: can't add generated controlling fanin "
+      "'^ConstantFoldingCtrl/b_1' to self.";
   EXPECT_EQ(s.error_message(), expected_msg);
 
   EXPECT_EQ(graph.graph()->node_size(), 4);
@@ -1396,7 +1463,10 @@ TEST(MutableGraphViewTest, RemoveControllingFaninSelfLoop) {
 
   Status s = graph.RemoveControllingFanin("c", "c");
   EXPECT_FALSE(s.ok());
-  string expected_msg = "Can't remove controlling fanin '^c' from self.";
+  string expected_msg =
+      "MutableGraphView::RemoveControllingFanin(node_name='c', "
+      "fanin_node_name='c') error: can't remove controlling fanin '^c' from "
+      "self.";
   EXPECT_EQ(s.error_message(), expected_msg);
 
   ASSERT_EQ(graph.graph()->node_size(), 3);
@@ -1517,8 +1587,8 @@ TEST(MutableGraphViewTest, DeleteNodesWithError) {
   Status s = graph.DeleteNodes({"b", "a"});
   EXPECT_FALSE(s.ok());
   string error_msg =
-      "Can't delete node(s) with retained fanout(s) [a, b] from node(s) [a, "
-      "b].";
+      "MutableGraphView::DeleteNodes(nodes_to_delete={a, b}) error: can't "
+      "delete node(s) with retained fanouts(s) [a, b].";
   EXPECT_EQ(s.error_message(), error_msg);
 
   EXPECT_EQ(graph.graph()->node_size(), 6);
@@ -1551,8 +1621,9 @@ TEST(MutableGraphViewTest, DeleteNodesWithLargeError) {
   Status s = graph.DeleteNodes({"a", "b", "c", "d", "e", "f"});
   EXPECT_FALSE(s.ok());
   string error_msg =
-      "Can't delete node(s) with retained fanout(s) [a, b, c, d, e, ...] from "
-      "node(s) [a, b, c, d, e, ...].";
+      "MutableGraphView::DeleteNodes(nodes_to_delete={a, b, c, d, e, ...}) "
+      "error: can't delete node(s) with retained fanouts(s) [a, b, c, d, e, "
+      "...].";
   EXPECT_EQ(s.error_message(), error_msg);
 
   EXPECT_EQ(graph.graph()->node_size(), 13);
-- 
GitLab


From 6da1788b4ad78b714f7918042381e2aba36a3c67 Mon Sep 17 00:00:00 2001
From: Yunlu Li <yunluli@google.com>
Date: Tue, 22 Jan 2019 11:23:10 -0800
Subject: [PATCH 1068/1095] Add support for joining data with default
 precision.

PiperOrigin-RevId: 230376960
---
 tensorflow/lite/testing/join.h       | 16 +++++++++++++++-
 tensorflow/lite/testing/join_test.cc |  5 +++++
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/tensorflow/lite/testing/join.h b/tensorflow/lite/testing/join.h
index d1c3146086..d10d2909b5 100644
--- a/tensorflow/lite/testing/join.h
+++ b/tensorflow/lite/testing/join.h
@@ -24,7 +24,21 @@ limitations under the License.
 namespace tflite {
 namespace testing {
 
-// Join a list of data separated by delimiter.
+// Join a list of data with default precision separated by delimiter.
+template <typename T>
+string JoinDefault(T* data, size_t len, const string& delimiter) {
+  if (len == 0 || data == nullptr) {
+    return "";
+  }
+  std::stringstream result;
+  result << data[0];
+  for (int i = 1; i < len; i++) {
+    result << delimiter << data[i];
+  }
+  return result.str();
+}
+
+// Join a list of data with fixed precision separated by delimiter.
 template <typename T>
 string Join(T* data, size_t len, const string& delimiter) {
   if (len == 0 || data == nullptr) {
diff --git a/tensorflow/lite/testing/join_test.cc b/tensorflow/lite/testing/join_test.cc
index 0b3c07f37e..476a7f2059 100644
--- a/tensorflow/lite/testing/join_test.cc
+++ b/tensorflow/lite/testing/join_test.cc
@@ -26,6 +26,11 @@ TEST(JoinTest, JoinInt) {
   EXPECT_EQ(Join(data.data(), data.size(), ","), "1,2,3");
 }
 
+TEST(JoinDefaultTest, JoinFloat) {
+  float data[] = {1.0, -3, 2.3, 1e-5};
+  EXPECT_EQ(JoinDefault(data, 4, " "), "1 -3 2.3 1e-05");
+}
+
 TEST(JoinTest, JoinFloat) {
   float data[] = {1.0, -3, 2.3, 1e-5};
   EXPECT_EQ(Join(data, 4, " "), "1 -3 2.29999995 9.99999975e-06");
-- 
GitLab


From 79e040fd8a7e09bc4754f4635aed7a3e1a91ce40 Mon Sep 17 00:00:00 2001
From: Cong Liu <congliu@google.com>
Date: Tue, 22 Jan 2019 11:54:24 -0800
Subject: [PATCH 1069/1095] [XLA] Propagate is_dynamic_dimensions for some
 operations.

PiperOrigin-RevId: 230382946
---
 tensorflow/compiler/xla/client/BUILD          |   1 +
 tensorflow/compiler/xla/client/xla_builder.cc |  60 ++-
 tensorflow/compiler/xla/client/xla_builder.h  |  14 +-
 .../compiler/xla/client/xla_builder_test.cc   | 406 +++++++++++++++++-
 .../compiler/xla/service/shape_inference.cc   | 125 +++++-
 tensorflow/compiler/xla/shape.h               |   4 +
 tensorflow/compiler/xla/shape_util.cc         |   4 +
 tensorflow/compiler/xla/shape_util_test.cc    |  20 +
 8 files changed, 586 insertions(+), 48 deletions(-)

diff --git a/tensorflow/compiler/xla/client/BUILD b/tensorflow/compiler/xla/client/BUILD
index 27c075e8f1..f5d56e8a9e 100644
--- a/tensorflow/compiler/xla/client/BUILD
+++ b/tensorflow/compiler/xla/client/BUILD
@@ -246,6 +246,7 @@ tf_cc_test(
         "//tensorflow/compiler/xla:status_macros",
         "//tensorflow/compiler/xla:test",
         "//tensorflow/compiler/xla:test_helpers",
+        "//tensorflow/compiler/xla:util",
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/service:hlo",
         "//tensorflow/compiler/xla/service:hlo_matchers",
diff --git a/tensorflow/compiler/xla/client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_builder.cc
index 20298d175d..5c9f9f7088 100644
--- a/tensorflow/compiler/xla/client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_builder.cc
@@ -299,23 +299,25 @@ XlaComputation XlaBuilder::BuildAndNoteError() {
   return build_status.ConsumeValueOrDie();
 }
 
-StatusOr<XlaComputation> XlaBuilder::Build() {
+StatusOr<XlaComputation> XlaBuilder::Build(bool remove_dynamic_dimensions) {
   if (!first_error_.ok()) {
     string backtrace;
     first_error_backtrace_.Dump(tensorflow::DebugWriteToString, &backtrace);
     return AppendStatus(first_error_, backtrace);
   }
-  return Build(instructions_.back().id());
+  return Build(instructions_.back().id(), remove_dynamic_dimensions);
 }
 
-StatusOr<XlaComputation> XlaBuilder::Build(XlaOp root) {
+StatusOr<XlaComputation> XlaBuilder::Build(XlaOp root,
+                                           bool remove_dynamic_dimensions) {
   if (root.builder_ != this) {
     return InvalidArgument("Given root operation is not in this computation.");
   }
-  return Build(root.handle());
+  return Build(root.handle(), remove_dynamic_dimensions);
 }
 
-StatusOr<XlaComputation> XlaBuilder::Build(int64 root_id) {
+StatusOr<XlaComputation> XlaBuilder::Build(int64 root_id,
+                                           bool remove_dynamic_dimensions) {
   if (!first_error_.ok()) {
     string backtrace;
     first_error_backtrace_.Dump(tensorflow::DebugWriteToString, &backtrace);
@@ -325,20 +327,22 @@ StatusOr<XlaComputation> XlaBuilder::Build(int64 root_id) {
   // TODO(b/121223198): XLA backend cannot handle dynamic dimensions yet, remove
   // all dynamic dimensions before building xla program until we have support in
   // the backend.
-  std::function<void(ShapeProto*)> remove_dynamic_dimension =
-      [&](ShapeProto* shape) {
-        if (shape->tuple_shapes_size() != 0) {
-          for (int64 i = 0; i < shape->tuple_shapes_size(); ++i) {
-            remove_dynamic_dimension(shape->mutable_tuple_shapes(i));
+  if (remove_dynamic_dimensions) {
+    std::function<void(ShapeProto*)> remove_dynamic_dimension =
+        [&](ShapeProto* shape) {
+          if (shape->tuple_shapes_size() != 0) {
+            for (int64 i = 0; i < shape->tuple_shapes_size(); ++i) {
+              remove_dynamic_dimension(shape->mutable_tuple_shapes(i));
+            }
           }
-        }
-        for (int64 i = 0; i < shape->dimensions_size(); ++i) {
-          shape->set_is_dynamic_dimension(i, false);
-        }
-      };
+          for (int64 i = 0; i < shape->dimensions_size(); ++i) {
+            shape->set_is_dynamic_dimension(i, false);
+          }
+        };
 
-  for (auto& instruction : instructions_) {
-    remove_dynamic_dimension(instruction.mutable_shape());
+    for (auto& instruction : instructions_) {
+      remove_dynamic_dimension(instruction.mutable_shape());
+    }
   }
 
   HloComputationProto entry;
@@ -497,16 +501,19 @@ XlaOp XlaBuilder::BinaryOp(HloOpcode binop, const XlaOp& lhs, const XlaOp& rhs,
       const Shape& from_shape = should_broadcast_lhs ? lhs_shape : rhs_shape;
 
       std::vector<int64> to_size;
-      for (int64 size : shape.dimensions()) {
-        to_size.push_back(size);
+      std::vector<bool> to_size_is_dynamic;
+      for (int i = 0; i < shape.rank(); i++) {
+        to_size.push_back(shape.dimensions(i));
+        to_size_is_dynamic.push_back(shape.is_dynamic_dimension(i));
       }
       for (int64 from_dim = 0; from_dim < from_shape.rank(); from_dim++) {
         int64 to_dim = broadcast_dimensions[from_dim];
         to_size[to_dim] = from_shape.dimensions(from_dim);
+        to_size_is_dynamic[to_dim] = from_shape.is_dynamic_dimension(from_dim);
       }
 
-      const Shape& broadcasted_shape =
-          ShapeUtil::MakeShape(from_shape.element_type(), to_size);
+      const Shape& broadcasted_shape = ShapeUtil::MakeShape(
+          from_shape.element_type(), to_size, to_size_is_dynamic);
       TF_ASSIGN_OR_RETURN(
           XlaOp broadcasted_operand,
           InDimBroadcast(broadcasted_shape, from, broadcast_dimensions));
@@ -665,8 +672,17 @@ XlaOp XlaBuilder::BroadcastInDim(
     TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand));
     // Output shape, in the case of degenerate broadcast, the out_dim_size is
     // not necessarily the same as the dimension sizes of the output shape.
-    const auto& output_shape =
+    auto output_shape =
         ShapeUtil::MakeShape(operand_shape.element_type(), out_dim_size);
+    for (int i = 0; i < broadcast_dimensions.size(); i++) {
+      if (broadcast_dimensions[i] < 0 ||
+          broadcast_dimensions[i] > out_dim_size.size()) {
+        return InvalidArgument("Broadcast dimension %lld is out of bound",
+                               broadcast_dimensions[i]);
+      }
+      output_shape.set_dynamic_dimension(broadcast_dimensions[i],
+                                         operand_shape.is_dynamic_dimension(i));
+    }
 
     TF_RETURN_IF_ERROR(ShapeInference::InferBroadcastShape(
                            operand_shape, output_shape, broadcast_dimensions)
diff --git a/tensorflow/compiler/xla/client/xla_builder.h b/tensorflow/compiler/xla/client/xla_builder.h
index 8908d172fa..7df1efe6b8 100644
--- a/tensorflow/compiler/xla/client/xla_builder.h
+++ b/tensorflow/compiler/xla/client/xla_builder.h
@@ -197,11 +197,19 @@ class XlaBuilder {
   // status. Note that all ops that have been enqueued will be moved to the
   // computation being returned. The root of the computation will be the last
   // added operation.
-  StatusOr<XlaComputation> Build();
+  //
+  // `remove_dynamic_dimensions` tells the builder whether to remove the
+  // dyanmic dimensions information in all ops.
+  //
+  // TODO(b/121223198): Delete `remove_dynamic_dimensions` and keeps the
+  // dynamic dimensions information when XLA backend can handle dynamic
+  // dimensions.
+  StatusOr<XlaComputation> Build(bool remove_dynamic_dimensions = true);
 
   // Overload of Build which specifies a particular root instruction for the
   // computation.
-  StatusOr<XlaComputation> Build(XlaOp root);
+  StatusOr<XlaComputation> Build(XlaOp root,
+                                 bool remove_dynamic_dimensions = true);
 
   // Builds the computation with the requested operations, or notes an error in
   // the parent XlaBuilder and returns an empty computation if building failed.
@@ -293,7 +301,7 @@ class XlaBuilder {
   };
 
   // Build helper which takes the id of the root operation..
-  StatusOr<XlaComputation> Build(int64 root_id);
+  StatusOr<XlaComputation> Build(int64 root_id, bool remove_dynamic_dimensions);
 
   // Description for the methods below can be found in the corresponding public
   // functions section in this file.
diff --git a/tensorflow/compiler/xla/client/xla_builder_test.cc b/tensorflow/compiler/xla/client/xla_builder_test.cc
index feee8187c7..098165000a 100644
--- a/tensorflow/compiler/xla/client/xla_builder_test.cc
+++ b/tensorflow/compiler/xla/client/xla_builder_test.cc
@@ -25,6 +25,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/status_macros.h"
 #include "tensorflow/compiler/xla/test.h"
 #include "tensorflow/compiler/xla/test_helpers.h"
+#include "tensorflow/compiler/xla/util.h"
 #include "tensorflow/compiler/xla/xla_data.pb.h"
 
 namespace xla {
@@ -39,7 +40,8 @@ using ::testing::HasSubstr;
 class XlaBuilderTest : public ::testing::Test {
  protected:
   StatusOr<std::unique_ptr<HloModule>> BuildHloModule(XlaBuilder* b) {
-    TF_ASSIGN_OR_RETURN(XlaComputation computation, b->Build());
+    TF_ASSIGN_OR_RETURN(XlaComputation computation,
+                        b->Build(/*remove_dynamic_dimensions=*/false));
     const HloModuleProto& proto = computation.proto();
     TF_ASSIGN_OR_RETURN(const auto& config,
                         HloModule::CreateModuleConfigFromProto(
@@ -50,7 +52,8 @@ class XlaBuilderTest : public ::testing::Test {
   // Overload which explicitly specifies the root instruction.
   StatusOr<std::unique_ptr<HloModule>> BuildHloModule(XlaBuilder* b,
                                                       XlaOp root) {
-    TF_ASSIGN_OR_RETURN(XlaComputation computation, b->Build(root));
+    TF_ASSIGN_OR_RETURN(XlaComputation computation,
+                        b->Build(root, /*remove_dynamic_dimensions=*/false));
     const HloModuleProto& proto = computation.proto();
     TF_ASSIGN_OR_RETURN(const auto& config,
                         HloModule::CreateModuleConfigFromProto(
@@ -447,8 +450,7 @@ TEST_F(XlaBuilderTest, ProtoMatches) {
 }
 
 TEST_F(XlaBuilderTest, DynamicParameter) {
-  std::vector<XlaComputation> computations;
-  XlaBuilder b("builder");
+  XlaBuilder b(TestName());
   Shape tuple_param_shape = ShapeUtil::MakeTupleShape(
       {ShapeUtil::MakeShape(F32, {5}), ShapeUtil::MakeShape(F32, {6})});
   auto p0 = Parameter(&b, 0, tuple_param_shape, "p0");
@@ -463,9 +465,399 @@ TEST_F(XlaBuilderTest, DynamicParameter) {
                                  ->parameter_instruction(0)
                                  ->shape()
                                  .tuple_shapes(1);
-  // TODO(b/121223198): The dynamic dimension should be set once we enable
-  // dynamic dimensions in xla builder.
-  EXPECT_FALSE(param_shape.is_dynamic_dimension(0));
+  EXPECT_TRUE(param_shape.is_dynamic_dimension(0));
+}
+
+TEST_F(XlaBuilderTest, DynamicUnary) {
+  XlaBuilder b(TestName());
+  Shape tuple_param_shape = ShapeUtil::MakeTupleShape(
+      {ShapeUtil::MakeShape(F32, {5}), ShapeUtil::MakeShape(U32, {})});
+  auto p0 = Parameter(&b, 0, tuple_param_shape, "p0");
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{1},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{0},
+                                   /*target_dim_num=*/0));
+  auto gte = GetTupleElement(p0, 0);
+  Neg(gte);
+  TF_ASSERT_OK_AND_ASSIGN(auto module, BuildHloModule(&b));
+  const Shape& result_shape =
+      module->entry_computation()->root_instruction()->shape();
+  EXPECT_TRUE(result_shape.is_dynamic_dimension(0));
+}
+
+TEST_F(XlaBuilderTest, DynamicBinary) {
+  XlaBuilder b(TestName());
+  Shape tuple_param_shape = ShapeUtil::MakeTupleShape(
+      {ShapeUtil::MakeShape(F32, {5}), ShapeUtil::MakeShape(F32, {5}),
+       ShapeUtil::MakeShape(U32, {})});
+  auto p0 = Parameter(&b, 0, tuple_param_shape, "p0");
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{2},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{0},
+                                   /*target_dim_num=*/0));
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{2},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{1},
+                                   /*target_dim_num=*/0));
+  auto gte0 = GetTupleElement(p0, 0);
+  auto gte1 = GetTupleElement(p0, 1);
+  Add(gte0, gte1);
+  TF_ASSERT_OK_AND_ASSIGN(auto module, BuildHloModule(&b));
+  const Shape& result_shape =
+      module->entry_computation()->root_instruction()->shape();
+  EXPECT_TRUE(result_shape.is_dynamic_dimension(0));
+}
+
+TEST_F(XlaBuilderTest, DynamicBinaryHasBroadcast) {
+  XlaBuilder b(TestName());
+  Shape tuple_param_shape = ShapeUtil::MakeTupleShape(
+      {ShapeUtil::MakeShape(F32, {5, 4}), ShapeUtil::MakeShape(F32, {5}),
+       ShapeUtil::MakeShape(U32, {})});
+  auto p0 = Parameter(&b, 0, tuple_param_shape, "p0");
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{2},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{0},
+                                   /*target_dim_num=*/0));
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{2},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{1},
+                                   /*target_dim_num=*/0));
+  auto gte0 = GetTupleElement(p0, 0);
+  auto gte1 = GetTupleElement(p0, 1);
+  Add(gte0, gte1, {0});
+  TF_ASSERT_OK_AND_ASSIGN(auto module, BuildHloModule(&b));
+  const Shape& result_shape =
+      module->entry_computation()->root_instruction()->shape();
+  EXPECT_TRUE(ContainersEqual(result_shape.dynamic_dimensions(), {true, false}))
+      << result_shape;
+}
+
+TEST_F(XlaBuilderTest, DynamicBroadcast) {
+  XlaBuilder b(TestName());
+  Shape tuple_param_shape = ShapeUtil::MakeTupleShape(
+      {ShapeUtil::MakeShape(F32, {5, 4}), ShapeUtil::MakeShape(U32, {})});
+  auto p0 = Parameter(&b, 0, tuple_param_shape, "p0");
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{1},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{0},
+                                   /*target_dim_num=*/0));
+  auto gte = GetTupleElement(p0, 0);
+  BroadcastInDim(gte, /*out_dim_size=*/{3, 5, 4},
+                 /*broadcast_dimensions=*/{1, 2});
+  TF_ASSERT_OK_AND_ASSIGN(auto module, BuildHloModule(&b));
+  const Shape& result_shape =
+      module->entry_computation()->root_instruction()->shape();
+  EXPECT_TRUE(
+      ContainersEqual(result_shape.dynamic_dimensions(), {false, true, false}))
+      << result_shape;
+}
+
+TEST_F(XlaBuilderTest, DynamicPad) {
+  XlaBuilder b(TestName());
+  Shape tuple_param_shape = ShapeUtil::MakeTupleShape(
+      {ShapeUtil::MakeShape(F32, {5, 4}), ShapeUtil::MakeShape(U32, {})});
+  auto p0 = Parameter(&b, 0, tuple_param_shape, "p0");
+  auto pad_val = ConstantR0<float>(&b, -1);
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{1},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{0},
+                                   /*target_dim_num=*/0));
+  auto gte = GetTupleElement(p0, 0);
+  PaddingConfig padding_config;
+  for (int i = 0; i < 2; i++) {
+    auto dimension = padding_config.add_dimensions();
+    dimension->set_edge_padding_low(0);
+    dimension->set_edge_padding_high(0);
+    dimension->set_interior_padding(0);
+  }
+  Pad(gte, pad_val, padding_config);
+  TF_ASSERT_OK_AND_ASSIGN(auto module, BuildHloModule(&b));
+  const Shape& result_shape =
+      module->entry_computation()->root_instruction()->shape();
+  EXPECT_TRUE(ContainersEqual(result_shape.dynamic_dimensions(), {true, false}))
+      << result_shape;
+}
+
+TEST_F(XlaBuilderTest, DynamicConvolution) {
+  XlaBuilder b(TestName());
+  Shape tuple_param_shape = ShapeUtil::MakeTupleShape(
+      {ShapeUtil::MakeShape(F32, {1, 2, 2, 128}),
+       ShapeUtil::MakeShape(F32, {2, 2, 128, 8}), ShapeUtil::MakeShape(U32, {}),
+       ShapeUtil::MakeShape(U32, {})});
+  auto p0 = Parameter(&b, 0, tuple_param_shape, "p0");
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{2},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{0},
+                                   /*target_dim_num=*/0));
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{3},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{1},
+                                   /*target_dim_num=*/2));
+  auto input = GetTupleElement(p0, 0);
+  auto filter = GetTupleElement(p0, 1);
+  ConvolutionDimensionNumbers dnums;
+  dnums.set_input_batch_dimension(0);
+  dnums.set_output_batch_dimension(0);
+  dnums.add_input_spatial_dimensions(1);
+  dnums.add_output_spatial_dimensions(1);
+  dnums.add_input_spatial_dimensions(2);
+  dnums.add_output_spatial_dimensions(2);
+  dnums.set_input_feature_dimension(3);
+  dnums.set_output_feature_dimension(3);
+  dnums.add_kernel_spatial_dimensions(0);
+  dnums.add_kernel_spatial_dimensions(1);
+  dnums.set_kernel_input_feature_dimension(2);
+  dnums.set_kernel_output_feature_dimension(3);
+  ConvWithGeneralDimensions(input, filter, {1, 1}, Padding::kValid, dnums,
+                            /*feature_group_count=*/1);
+  TF_ASSERT_OK_AND_ASSIGN(auto module, BuildHloModule(&b));
+  const Shape& result_shape =
+      module->entry_computation()->root_instruction()->shape();
+  EXPECT_TRUE(ContainersEqual(result_shape.dynamic_dimensions(),
+                              {true, false, false, false}))
+      << result_shape;
+}
+
+TEST_F(XlaBuilderTest, DynamicDot) {
+  XlaBuilder b(TestName());
+  Shape tuple_param_shape = ShapeUtil::MakeTupleShape(
+      {ShapeUtil::MakeShape(F32, {2, 3, 4}),
+       ShapeUtil::MakeShape(F32, {2, 4, 5}), ShapeUtil::MakeShape(U32, {}),
+       ShapeUtil::MakeShape(U32, {})});
+  auto p0 = Parameter(&b, 0, tuple_param_shape, "p0");
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{2},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{0},
+                                   /*target_dim_num=*/0));
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{2},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{1},
+                                   /*target_dim_num=*/0));
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{3},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{0},
+                                   /*target_dim_num=*/1));
+
+  auto lhs = GetTupleElement(p0, 0);
+  auto rhs = GetTupleElement(p0, 1);
+  DotDimensionNumbers dnums;
+  dnums.add_lhs_contracting_dimensions(2);
+  dnums.add_rhs_contracting_dimensions(1);
+  dnums.add_lhs_batch_dimensions(0);
+  dnums.add_rhs_batch_dimensions(0);
+  DotGeneral(lhs, rhs, dnums);
+  TF_ASSERT_OK_AND_ASSIGN(auto module, BuildHloModule(&b));
+  const Shape& result_shape =
+      module->entry_computation()->root_instruction()->shape();
+  EXPECT_TRUE(
+      ContainersEqual(result_shape.dynamic_dimensions(), {true, true, false}))
+      << result_shape;
+}
+
+TEST_F(XlaBuilderTest, DynamicReduce) {
+  XlaBuilder b(TestName());
+  Shape tuple_param_shape = ShapeUtil::MakeTupleShape(
+      {ShapeUtil::MakeShape(F32, {5, 4, 3}), ShapeUtil::MakeShape(U32, {})});
+  auto p0 = Parameter(&b, 0, tuple_param_shape, "p0");
+  auto init = ConstantR0<float>(&b, 0);
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{1},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{0},
+                                   /*target_dim_num=*/1));
+  auto gte = GetTupleElement(p0, 0);
+  XlaBuilder bsum(TestName());
+  Add(Parameter(&bsum, 0, ShapeUtil::MakeShape(F32, {}), "x"),
+      Parameter(&bsum, 1, ShapeUtil::MakeShape(F32, {}), "y"));
+  TF_ASSERT_OK_AND_ASSIGN(auto sum, bsum.Build());
+  Reduce(gte, init, sum, {0});
+  TF_ASSERT_OK_AND_ASSIGN(auto module, BuildHloModule(&b));
+  const Shape& result_shape =
+      module->entry_computation()->root_instruction()->shape();
+  EXPECT_TRUE(ContainersEqual(result_shape.dynamic_dimensions(), {true, false}))
+      << result_shape;
+}
+
+TEST_F(XlaBuilderTest, DynamicReduceWindow) {
+  XlaBuilder b(TestName());
+  Shape tuple_param_shape = ShapeUtil::MakeTupleShape(
+      {ShapeUtil::MakeShape(F32, {2, 4, 8}), ShapeUtil::MakeShape(U32, {})});
+  auto p0 = Parameter(&b, 0, tuple_param_shape, "p0");
+  auto init = ConstantR0<float>(&b, 0.f);
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{1},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{0},
+                                   /*target_dim_num=*/0));
+  auto gte = GetTupleElement(p0, 0);
+  XlaBuilder bsum(TestName());
+  Add(Parameter(&bsum, 0, ShapeUtil::MakeShape(F32, {}), "x"),
+      Parameter(&bsum, 1, ShapeUtil::MakeShape(F32, {}), "y"));
+  TF_ASSERT_OK_AND_ASSIGN(auto sum, bsum.Build());
+  ReduceWindow(gte, init, sum, /*window_dimensions=*/{1, 2, 4},
+               /*window_strides=*/{1, 1, 1}, Padding::kValid);
+  TF_ASSERT_OK_AND_ASSIGN(auto module, BuildHloModule(&b));
+  const Shape& result_shape =
+      module->entry_computation()->root_instruction()->shape();
+  EXPECT_TRUE(
+      ContainersEqual(result_shape.dynamic_dimensions(), {true, false, false}))
+      << result_shape;
+}
+
+TEST_F(XlaBuilderTest, DynamicSelectAndScatter) {
+  XlaBuilder b(TestName());
+  Shape tuple_param_shape = ShapeUtil::MakeTupleShape(
+      {ShapeUtil::MakeShape(F32, {2, 4, 8}),
+       ShapeUtil::MakeShape(F32, {2, 2, 2}), ShapeUtil::MakeShape(U32, {})});
+  auto p0 = Parameter(&b, 0, tuple_param_shape, "p0");
+  auto init = ConstantR0<float>(&b, 0.f);
+  XlaBuilder bsum(TestName());
+  Add(Parameter(&bsum, 0, ShapeUtil::MakeShape(F32, {}), "x"),
+      Parameter(&bsum, 1, ShapeUtil::MakeShape(F32, {}), "y"));
+  TF_ASSERT_OK_AND_ASSIGN(auto sum, bsum.Build());
+  XlaBuilder bge(TestName());
+  Ge(Parameter(&bge, 0, ShapeUtil::MakeShape(F32, {}), "x"),
+     Parameter(&bge, 1, ShapeUtil::MakeShape(F32, {}), "y"));
+  TF_ASSERT_OK_AND_ASSIGN(auto ge, bge.Build());
+
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{2},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{0},
+                                   /*target_dim_num=*/0));
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{2},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{1},
+                                   /*target_dim_num=*/0));
+  auto gte0 = GetTupleElement(p0, 0);
+  auto source = GetTupleElement(p0, 1);
+  SelectAndScatter(gte0, ge, {1, 2, 4}, {1, 2, 4}, Padding::kValid, source,
+                   init, sum);
+  TF_ASSERT_OK_AND_ASSIGN(auto module, BuildHloModule(&b));
+  const Shape& result_shape =
+      module->entry_computation()->root_instruction()->shape();
+  EXPECT_TRUE(
+      ContainersEqual(result_shape.dynamic_dimensions(), {true, false, false}))
+      << result_shape;
+}
+
+TEST_F(XlaBuilderTest, DynamicReshape) {
+  XlaBuilder b(TestName());
+  Shape tuple_param_shape = ShapeUtil::MakeTupleShape(
+      {ShapeUtil::MakeShape(F32, {2, 3, 4, 5, 6}),
+       ShapeUtil::MakeShape(U32, {}), ShapeUtil::MakeShape(U32, {})});
+  auto p0 = Parameter(&b, 0, tuple_param_shape, "p0");
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{1},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{0},
+                                   /*target_dim_num=*/2));
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{2},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{0},
+                                   /*target_dim_num=*/3));
+  auto gte = GetTupleElement(p0, 0);  // f32[2, 3, <=4, <=5, 6]
+  Reshape(gte, /*new_sizes=*/{6, 4, 1, 5, 2, 3});
+  TF_ASSERT_OK_AND_ASSIGN(auto module, BuildHloModule(&b));
+  const Shape& result_shape =
+      module->entry_computation()->root_instruction()->shape();
+  EXPECT_TRUE(result_shape.is_dynamic_dimension(1));
+  EXPECT_TRUE(result_shape.is_dynamic_dimension(3));
+  EXPECT_TRUE(ContainersEqual(result_shape.dynamic_dimensions(),
+                              {false, true, false, true, false, false}))
+      << result_shape;
+}
+
+TEST_F(XlaBuilderTest, DynamicSelect) {
+  XlaBuilder b(TestName());
+  Shape tuple_param_shape = ShapeUtil::MakeTupleShape(
+      {ShapeUtil::MakeShape(F32, {4, 5, 6}),
+       ShapeUtil::MakeShape(F32, {4, 5, 6}), ShapeUtil::MakeShape(U32, {}),
+       ShapeUtil::MakeShape(U32, {})});
+  auto p0 = Parameter(&b, 0, tuple_param_shape, "p0");
+  auto pred = Parameter(&b, 1, ShapeUtil::MakeShape(PRED, {}), "pred");
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{2},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{0},
+                                   /*target_dim_num=*/1));
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{3},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{1},
+                                   /*target_dim_num=*/1));
+  auto gte0 = GetTupleElement(p0, 0);
+  auto gte1 = GetTupleElement(p0, 1);
+  Select(pred, gte0, gte1);
+  TF_ASSERT_OK_AND_ASSIGN(auto module, BuildHloModule(&b));
+  const Shape& result_shape =
+      module->entry_computation()->root_instruction()->shape();
+  EXPECT_TRUE(result_shape.is_dynamic_dimension(1));
+  EXPECT_FALSE(result_shape.is_dynamic_dimension(2));
+  EXPECT_TRUE(
+      ContainersEqual(result_shape.dynamic_dimensions(), {false, true, false}))
+      << result_shape;
+}
+
+TEST_F(XlaBuilderTest, DynamicSelectNotCompatible) {
+  XlaBuilder b(TestName());
+  Shape tuple_param_shape = ShapeUtil::MakeTupleShape(
+      {ShapeUtil::MakeShape(F32, {4, 5, 6}),
+       ShapeUtil::MakeShape(F32, {4, 5, 6}), ShapeUtil::MakeShape(U32, {}),
+       ShapeUtil::MakeShape(U32, {})});
+  auto p0 = Parameter(&b, 0, tuple_param_shape, "p0");
+  auto pred = Parameter(&b, 1, ShapeUtil::MakeShape(PRED, {}), "pred");
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{2},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{0},
+                                   /*target_dim_num=*/1));
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{3},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{1},
+                                   /*target_dim_num=*/2));
+  auto gte0 = GetTupleElement(p0, 0);  // f32[4,<=5,6]
+  auto gte1 = GetTupleElement(p0, 1);  // f32[4,5,<=6]
+  Select(pred, gte0, gte1);
+  Status status = BuildHloModule(&b).status();
+  ASSERT_IS_NOT_OK(status);
+  EXPECT_THAT(status.error_message(),
+              ::testing::HasSubstr("Operands to select must be the same shape; "
+                                   "got f32[4,<=5,6] and f32[4,5,<=6]"));
+}
+
+TEST_F(XlaBuilderTest, DynamicTranspose) {
+  XlaBuilder b(TestName());
+  Shape tuple_param_shape = ShapeUtil::MakeTupleShape(
+      {ShapeUtil::MakeShape(F32, {3, 5}), ShapeUtil::MakeShape(U32, {})});
+  auto p0 = Parameter(&b, 0, tuple_param_shape, "p0");
+  ASSERT_IS_OK(b.SetDynamicBinding(/*dynamic_size_param_num=*/0,
+                                   /*dynamic_size_param_index=*/{1},
+                                   /*target_param_num=*/0,
+                                   /*target_param_index=*/{0},
+                                   /*target_dim_num=*/0));
+  auto gte = GetTupleElement(p0, 0);
+  Transpose(gte, /*permutation=*/{1, 0});
+  TF_ASSERT_OK_AND_ASSIGN(auto module, BuildHloModule(&b));
+  const Shape& result_shape =
+      module->entry_computation()->root_instruction()->shape();
+  EXPECT_TRUE(ContainersEqual(result_shape.dynamic_dimensions(), {false, true}))
+      << result_shape;
 }
 
 TEST_F(XlaBuilderTest, AfterAllWithNonTokenOperands) {
diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc
index fad7afefaf..4200f09abd 100644
--- a/tensorflow/compiler/xla/service/shape_inference.cc
+++ b/tensorflow/compiler/xla/service/shape_inference.cc
@@ -156,6 +156,14 @@ Status VerifyReducerShape(const ProgramShape& reducer_shape,
   return Status::OK();
 }
 
+bool IsTrivialWindowDimension(const WindowDimension& window_dimension) {
+  return window_dimension.size() == 1 && window_dimension.stride() == 1 &&
+         window_dimension.padding_low() == 0 &&
+         window_dimension.padding_high() == 0 &&
+         window_dimension.window_dilation() == 1 &&
+         window_dimension.base_dilation() == 1;
+}
+
 StatusOr<Shape> InferWindowOutputShape(const Shape& base_shape,
                                        const Window& window,
                                        PrimitiveType element_type,
@@ -167,6 +175,7 @@ StatusOr<Shape> InferWindowOutputShape(const Shape& base_shape,
   }
 
   std::vector<int64> output_dimensions(window.dimensions_size());
+  std::vector<bool> output_is_dynamic(window.dimensions_size());
   for (int64 i = 0; i < window.dimensions_size(); ++i) {
     const auto& dim = window.dimensions(i);
     if (dim.size() <= 0) {
@@ -196,6 +205,12 @@ StatusOr<Shape> InferWindowOutputShape(const Shape& base_shape,
           window.DebugString());
     }
 
+    if (base_shape.is_dynamic_dimension(i) && !IsTrivialWindowDimension(dim)) {
+      return Unimplemented(
+          "Dynamic shape is not supported for non trivial window: %s",
+          window_util::ToString(window));
+    }
+
     const int64 dilated_base = window_util::DilatedBound(
         ShapeUtil::GetDimension(base_shape, i), dim.base_dilation());
     const int64 padded_dilated_base =
@@ -205,9 +220,11 @@ StatusOr<Shape> InferWindowOutputShape(const Shape& base_shape,
 
     output_dimensions[i] = window_util::StridedBound(
         padded_dilated_base, dilated_window, dim.stride());
+    output_is_dynamic[i] = base_shape.is_dynamic_dimension(i);
   }
 
-  return ShapeUtil::MakeValidatedShape(element_type, output_dimensions);
+  return ShapeUtil::MakeValidatedShape(element_type, output_dimensions,
+                                       output_is_dynamic);
 }
 
 }  // namespace
@@ -500,17 +517,29 @@ StatusOr<Shape> InferWindowOutputShape(const Shape& base_shape,
                            padding_config.ShortDebugString());
   }
 
+  if (!padding_value_shape.is_static()) {
+    return InvalidArgument("Dynamic padding value is not supported");
+  }
+
   std::vector<int64> dimensions(operand_shape.rank());
+  std::vector<bool> is_dynamic(operand_shape.rank());
   for (int64 i = 0; i < operand_shape.dimensions_size(); ++i) {
     const auto& p = padding_config.dimensions(i);
+    if (operand_shape.is_dynamic_dimension(i) && p.edge_padding_high() != 0 &&
+        p.edge_padding_low() != 0 && p.interior_padding() != 0) {
+      return InvalidArgument(
+          "Dynamic dimension on padding dimension is not supported.");
+    }
     dimensions[i] = operand_shape.dimensions(i) + p.edge_padding_low() +
                     p.edge_padding_high() +
                     std::max<int64>(operand_shape.dimensions(i) - 1, 0LL) *
                         p.interior_padding();
+    is_dynamic[i] = operand_shape.is_dynamic_dimension(i);
   }
+
   return ShapeUtil::MakeShape(
       ShapeUtil::HigherPrecisionElementType(operand_shape, padding_value_shape),
-      dimensions);
+      dimensions, is_dynamic);
 }
 
 // Current DotDimensionNumbers Requirements:
@@ -620,7 +649,9 @@ Status ValidateDotDimensionNumbers(
     const int64 rhs_contracting_dimension =
         dimension_numbers.rhs_contracting_dimensions(i);
     if (lhs.dimensions(lhs_contracting_dimension) !=
-        rhs.dimensions(rhs_contracting_dimension)) {
+            rhs.dimensions(rhs_contracting_dimension) ||
+        lhs.is_dynamic_dimension(lhs_contracting_dimension) !=
+            rhs.is_dynamic_dimension(rhs_contracting_dimension)) {
       return fail("Contracting dimension sizes do not match.");
     }
   }
@@ -634,7 +665,10 @@ Status ValidateDotDimensionNumbers(
   // Check that batch dimension numbers and sizes match.
   for (int64 i = 0; i < dimension_numbers.lhs_batch_dimensions_size(); ++i) {
     if (lhs.dimensions(dimension_numbers.lhs_batch_dimensions(i)) !=
-        rhs.dimensions(dimension_numbers.rhs_batch_dimensions(i))) {
+            rhs.dimensions(dimension_numbers.rhs_batch_dimensions(i)) ||
+        lhs.is_dynamic_dimension(dimension_numbers.lhs_batch_dimensions(i)) !=
+            rhs.is_dynamic_dimension(
+                dimension_numbers.rhs_batch_dimensions(i))) {
       return fail("Batch dimension sizes must match for lhs/rhs.");
     }
   }
@@ -645,14 +679,17 @@ Status ValidateDotDimensionNumbers(
   // Generate the result dimensions in order, rhs dimensions followed by lhs
   // dimensions except the contracted and batch dimensions.
   std::vector<int64> dimensions;
+  std::vector<bool> is_dynamic;
   for (int64 lhs_dim : dimension_numbers.lhs_batch_dimensions()) {
     dimensions.push_back(lhs.dimensions(lhs_dim));
+    is_dynamic.push_back(lhs.is_dynamic_dimension(lhs_dim));
   }
   for (int64 i = 0; i < lhs.rank(); i++) {
     if (!absl::c_linear_search(dimension_numbers.lhs_contracting_dimensions(),
                                i) &&
         !absl::c_linear_search(dimension_numbers.lhs_batch_dimensions(), i)) {
       dimensions.push_back(lhs.dimensions(i));
+      is_dynamic.push_back(lhs.is_dynamic_dimension(i));
     }
   }
   for (int64 i = 0; i < rhs.rank(); i++) {
@@ -660,10 +697,11 @@ Status ValidateDotDimensionNumbers(
                                i) &&
         !absl::c_linear_search(dimension_numbers.rhs_batch_dimensions(), i)) {
       dimensions.push_back(rhs.dimensions(i));
+      is_dynamic.push_back(rhs.is_dynamic_dimension(i));
     }
   }
   Shape result = ShapeUtil::MakeShape(
-      ShapeUtil::HigherPrecisionElementType(lhs, rhs), dimensions);
+      ShapeUtil::HigherPrecisionElementType(lhs, rhs), dimensions, is_dynamic);
 
   TF_DCHECK_OK(ShapeUtil::ValidateShapeWithOptionalLayout(result));
   VLOG(2) << "inferred dot shape: " << ShapeUtil::HumanString(result);
@@ -681,13 +719,17 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
   // dimension). In that case, the output shape has the non-1 dimension size
   // from the lhs/rhs pair in every index.
   std::vector<int64> output_dimensions(lhs.rank());
+  std::vector<bool> output_dimensions_is_dynamic(lhs.rank());
   for (int64 i = 0; i < lhs.rank(); ++i) {
     if (lhs.dimensions(i) == rhs.dimensions(i)) {
       output_dimensions[i] = lhs.dimensions(i);
+      output_dimensions_is_dynamic[i] = lhs.is_dynamic_dimension(i);
     } else if (lhs.dimensions(i) == 1) {
       output_dimensions[i] = rhs.dimensions(i);
+      output_dimensions_is_dynamic[i] = rhs.is_dynamic_dimension(i);
     } else if (rhs.dimensions(i) == 1) {
       output_dimensions[i] = lhs.dimensions(i);
+      output_dimensions_is_dynamic[i] = lhs.is_dynamic_dimension(i);
     } else {
       return InvalidArgument(
           "Binary op %s with incompatible shapes: %s and %s.",
@@ -696,7 +738,7 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
     }
   }
   return ShapeUtil::MakeShape(ShapeUtil::HigherPrecisionElementType(lhs, rhs),
-                              output_dimensions);
+                              output_dimensions, output_dimensions_is_dynamic);
 }
 
 /* static */ StatusOr<Shape> ShapeInference::InferInDimBroadcastShape(
@@ -775,6 +817,9 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
     }
     int64 small_dimension_size = smaller_shape.dimensions(i);
     int64 large_dimension_size = larger_shape.dimensions(dimension_to_match);
+    bool small_is_dynamic = smaller_shape.is_dynamic_dimension(i);
+    bool large_is_dynamic =
+        larger_shape.is_dynamic_dimension(dimension_to_match);
     // Dimension sizes must be compatible: match or be degenerate (degenerate
     // case is handled by degenerate dimension broadcasting which occurs after
     // InDim broadcasting).
@@ -786,6 +831,12 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
           ShapeUtil::HumanString(smaller_shape),
           ShapeUtil::HumanString(larger_shape));
     }
+    if (small_is_dynamic != large_is_dynamic) {
+      return InvalidArgument(
+          "Broadcast dimension %d dynamism mismatch: %s and %s.", i,
+          ShapeUtil::HumanString(smaller_shape),
+          ShapeUtil::HumanString(larger_shape));
+    }
     // Make sure the broadcast dimensions are listed in a strictly increasing
     // order.
     if (i > 0 && broadcast_dimensions.at(i - 1) >= dimension_to_match) {
@@ -795,6 +846,7 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
     }
 
     output_shape.set_dimensions(dimension_to_match, small_dimension_size);
+    output_shape.set_dynamic_dimension(dimension_to_match, small_is_dynamic);
   }
 
   return output_shape;
@@ -1731,8 +1783,33 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
     dimensions[dnums.output_spatial_dimensions(i)] =
         window_output_shape.dimensions(i);
   }
+  std::vector<bool> is_dynamic(num_dims);
+  for (int i = 0; i < num_dims; i++) {
+    if (lhs.is_dynamic_dimension(i)) {
+      if (i == dnums.input_batch_dimension()) {
+        is_dynamic[dnums.output_batch_dimension()] = true;
+      } else if (i == dnums.input_feature_dimension()) {
+        // Input feature dimension is a contracting dimension, which does not
+        // affect the output dimension size. So we need to do nothing.
+      } else {
+        return InvalidArgument(
+            "Dynamic Spatial Convolution is not supported: lhs shape is %s ",
+            lhs.ToString());
+      }
+    }
+    if (rhs.is_dynamic_dimension(i)) {
+      if (i == dnums.kernel_input_feature_dimension()) {
+        // Kernel feature dimension does not affect the output dimension size.
+        // So we need to do nothing.
+      } else {
+        return InvalidArgument(
+            "Dynamic Spatial Convolution is not supported: rhs shape is %s ",
+            rhs.ToString());
+      }
+    }
+  }
   return ShapeUtil::MakeShape(ShapeUtil::HigherPrecisionElementType(lhs, rhs),
-                              dimensions);
+                              dimensions, is_dynamic);
 }
 
 /* static */ StatusOr<Shape> ShapeInference::InferFftShape(
@@ -1904,7 +1981,7 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
   for (int64 i = 1; i < num_reduced_args; ++i) {
     if (!ShapeUtil::SameDimensions(*reduced_args[0], *reduced_args[i])) {
       return InvalidArgument(
-          "All reduced tensors must have the sime dimension. Tensor 0 has "
+          "All reduced tensors must have the same dimension. Tensor 0 has "
           "shape %s, Tensor %d has shape %s",
           ShapeUtil::HumanString(*reduced_args[0]), i,
           ShapeUtil::HumanString(*reduced_args[i]));
@@ -1933,20 +2010,22 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
   std::set<int64> dimensions_to_reduce_set(dimensions_to_reduce.begin(),
                                            dimensions_to_reduce.end());
   std::vector<int64> new_dimensions;
+  std::vector<bool> new_is_dynamic;
   for (int i = 0; i < arg.rank(); ++i) {
     if (dimensions_to_reduce_set.find(i) == dimensions_to_reduce_set.end()) {
       new_dimensions.push_back(arg.dimensions(i));
+      new_is_dynamic.push_back(arg.is_dynamic_dimension(i));
     }
   }
 
   if (ShapeUtil::IsScalar(to_apply.result())) {
     return ShapeUtil::MakeShape(to_apply.result().element_type(),
-                                new_dimensions);
+                                new_dimensions, new_is_dynamic);
   } else {
     std::vector<Shape> result_subshapes;
     for (const Shape& subshape : to_apply.result().tuple_shapes()) {
-      result_subshapes.push_back(
-          ShapeUtil::MakeShape(subshape.element_type(), new_dimensions));
+      result_subshapes.push_back(ShapeUtil::MakeShape(
+          subshape.element_type(), new_dimensions, new_is_dynamic));
     }
     return ShapeUtil::MakeTupleShape(result_subshapes);
   }
@@ -2020,6 +2099,7 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
         ShapeUtil::HumanString(source_shape),
         ShapeUtil::HumanString(window_result_shape));
   }
+
   return operand_shape;
 }
 
@@ -2502,6 +2582,12 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
           i, operand_shape.dimensions(i), broadcast_dimensions[i],
           output_shape.dimensions(broadcast_dimensions[i]));
     }
+    if (operand_shape.is_dynamic_dimension(i) !=
+        output_shape.is_dynamic_dimension(broadcast_dimensions[i])) {
+      return InvalidArgument(
+          "Broadcast input and output dynamism mismatch: %s and %s",
+          operand_shape.ToString(), output_shape.ToString());
+    }
     // Make sure the broadcast dimensions are listed in a strictly increasing
     // order.
     if (i > 0 && broadcast_dimensions[i - 1] >= broadcast_dimensions[i]) {
@@ -2544,6 +2630,14 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
         StrJoin(dimensions, ","), ShapeUtil::HumanString(operand));
   }
 
+  std::vector<std::pair<int64, int64>> unmodified_dims =
+      ShapeUtil::DimensionsUnmodifiedByReshape(operand, inferred_shape);
+  for (auto& unmodified : unmodified_dims) {
+    if (operand.is_dynamic_dimension(unmodified.first)) {
+      inferred_shape.set_dynamic_dimension(unmodified.second, true);
+    }
+  }
+
   return inferred_shape;
 }
 
@@ -2624,12 +2718,11 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
     // dimensions as on_true and on_false.
     return ShapeUtil::ChangeElementType(
         on_true, ShapeUtil::HigherPrecisionElementType(on_true, on_false));
-  } else {
-    return InvalidArgument(
-        "Select operation with non-scalar predicate with dimensionality "
-        " different from the other operands: %s.",
-        ShapeUtil::HumanString(pred));
   }
+  return InvalidArgument(
+      "Select operation with non-scalar predicate with dimensionality "
+      " different from the other operands: %s.",
+      ShapeUtil::HumanString(pred));
 }
 
 /* static */ StatusOr<Shape> ShapeInference::InferTupleSelectShape(
diff --git a/tensorflow/compiler/xla/shape.h b/tensorflow/compiler/xla/shape.h
index e40e5ca37a..e6b4e872f6 100644
--- a/tensorflow/compiler/xla/shape.h
+++ b/tensorflow/compiler/xla/shape.h
@@ -72,6 +72,10 @@ class Shape {
     dynamic_dimensions_[dimension] = is_dynamic;
   }
 
+  const std::vector<bool>& dynamic_dimensions() const {
+    return dynamic_dimensions_;
+  }
+
   // Add dimension_upper_bound().
 
   // Removes the given dimension form the shape. Layout, if it exists, is
diff --git a/tensorflow/compiler/xla/shape_util.cc b/tensorflow/compiler/xla/shape_util.cc
index 799a4483df..1ada4bc036 100644
--- a/tensorflow/compiler/xla/shape_util.cc
+++ b/tensorflow/compiler/xla/shape_util.cc
@@ -929,6 +929,10 @@ Status ForEachMutableSubshapeHelper(
   for (auto dim : Permute(permutation, shape.dimensions())) {
     new_shape.add_dimensions(dim);
   }
+  for (int64 i = 0; i < shape.rank(); i++) {
+    new_shape.set_dynamic_dimension(permutation[i],
+                                    shape.is_dynamic_dimension(i));
+  }
 
   // If `shape` has a layout, by contract we choose a new layout such that the
   // transpose defined by this permutation is a bitcast.
diff --git a/tensorflow/compiler/xla/shape_util_test.cc b/tensorflow/compiler/xla/shape_util_test.cc
index 61b4e73e06..126ae58293 100644
--- a/tensorflow/compiler/xla/shape_util_test.cc
+++ b/tensorflow/compiler/xla/shape_util_test.cc
@@ -710,6 +710,26 @@ TEST(ShapeUtilTest, PermuteDimensionsLayout) {
   } while (std::next_permutation(layout.begin(), layout.end()));
 }
 
+TEST(ShapeUtilTest, PermuteDynamicDimensions) {
+  Shape shape =
+      ShapeUtil::MakeShape(F32, {10, 100, 1000},
+                           /*dynamic_dimensions*/ {false, true, true});
+  SCOPED_TRACE(absl::StrCat("shape=", shape.ToString()));
+
+  std::vector<int64> permutation(3);
+  std::iota(permutation.begin(), permutation.end(), 0);
+  do {
+    SCOPED_TRACE(absl::StrCat("permutation=", absl::StrJoin(permutation, ",")));
+
+    auto permuted = ShapeUtil::PermuteDimensions(permutation, shape);
+    for (int i = 0; i < shape.rank(); i++) {
+      EXPECT_EQ(permuted.dimensions(permutation[i]), shape.dimensions(i));
+      EXPECT_EQ(permuted.is_dynamic_dimension(permutation[i]),
+                shape.is_dynamic_dimension(i));
+    }
+  } while (std::next_permutation(permutation.begin(), permutation.end()));
+}
+
 TEST(AlgebraicSimplifierTest, ReshapeIsBitcast_3x2x2_6x2_Dim0IsMostMinor) {
   EXPECT_FALSE(ShapeUtil::ReshapeIsBitcast(
       ShapeUtil::MakeShapeWithLayout(F32, {3, 2, 2}, {0, 1, 2}),
-- 
GitLab


From 0b8ac6e1b9e85b3d16c8180cb2291c38ff66a1ba Mon Sep 17 00:00:00 2001
From: Eugene Zhulenev <ezhulenev@google.com>
Date: Tue, 22 Jan 2019 12:15:34 -0800
Subject: [PATCH 1070/1095] [Grappler] Cleanup grappler function inlining rules
 wrt stateful and dataset ops.

Grappler doesn't try to be the "automatic control dependency tracking framework", if side-effects after inlining are not guaranteed to have well defined execution order, we just do not inline it, and don't try to fix it. This is a graph-construction problem that should be fixed in tf-eager auto-control dependencies tracking.

In Agressive mode function optimizer skips this check and always inline.

PiperOrigin-RevId: 230386480
---
 .../core/grappler/graph_topology_view.cc      |  60 ++-
 .../core/grappler/graph_topology_view.h       |   6 +
 tensorflow/core/grappler/op_types.cc          |  16 +
 tensorflow/core/grappler/op_types.h           |   5 +
 .../grappler/optimizers/function_optimizer.cc | 450 ++++++++++++------
 .../optimizers/function_optimizer_test.cc     |  38 +-
 6 files changed, 395 insertions(+), 180 deletions(-)

diff --git a/tensorflow/core/grappler/graph_topology_view.cc b/tensorflow/core/grappler/graph_topology_view.cc
index 38ccfbaeb8..79e2f9a92f 100644
--- a/tensorflow/core/grappler/graph_topology_view.cc
+++ b/tensorflow/core/grappler/graph_topology_view.cc
@@ -62,19 +62,37 @@ Status GraphTopologyView::InitializeFromGraph(
   // 1. Add ephemeral edges to the adjacency lists.
   for (const GraphView::Edge& edge : ephemeral_edges) {
     const auto src = node_name_to_index_.find(edge.src.node->name());
-    if (src == node_name_to_index_.end()) {
-      return errors::InvalidArgument("Non-existent src node: ",
-                                     edge.src.node->name());
+    const bool valid_src = src != node_name_to_index_.end();
+
+    if (!valid_src) {
+      const string error_message =
+          absl::StrCat("Non-existent src node: ", edge.src.node->name());
+      if (skip_invalid_edges_) {
+        VLOG(0) << "Skip error: " << error_message;
+      } else {
+        return errors::InvalidArgument(error_message);
+      }
     }
+
     const auto dst = node_name_to_index_.find(edge.dst.node->name());
-    if (dst == node_name_to_index_.end()) {
-      return errors::InvalidArgument("Non-existent dst node: ",
-                                     edge.dst.node->name());
+    const bool valid_dst = dst != node_name_to_index_.end();
+
+    if (!valid_dst) {
+      const string error_message =
+          absl::StrCat("Non-existent dst node: ", edge.dst.node->name());
+      if (skip_invalid_edges_) {
+        VLOG(0) << "Skip error: " << error_message;
+      } else {
+        return errors::InvalidArgument(error_message);
+      }
+    }
+
+    if (valid_dst && valid_src) {
+      const int src_idx = src->second;
+      const int dst_idx = dst->second;
+      fanins_[dst_idx].push_back(src_idx);
+      fanouts_[src_idx].push_back(dst_idx);
     }
-    const int src_idx = src->second;
-    const int dst_idx = dst->second;
-    fanins_[dst_idx].push_back(src_idx);
-    fanouts_[src_idx].push_back(dst_idx);
   }
 
   // 2. Add graph edges to the adjacency lists.
@@ -85,13 +103,23 @@ Status GraphTopologyView::InitializeFromGraph(
     for (const string& input : node.input()) {
       TensorId tensor = ParseTensorName(input);
       const auto it = node_name_to_index_.find(tensor.node());
-      if (it == node_name_to_index_.end()) {
-        return errors::InvalidArgument("Non-existent input ", input,
-                                       " for node ", node.name());
+      const bool valid_input = it != node_name_to_index_.end();
+
+      if (!valid_input) {
+        const string error_message = absl::StrCat("Non-existent input ", input,
+                                                  " in node ", node.name());
+        if (skip_invalid_edges_) {
+          VLOG(3) << "Skip error: " << error_message;
+        } else {
+          return errors::InvalidArgument(error_message);
+        }
+      }
+
+      if (valid_input) {
+        const int input_idx = it->second;
+        fanins_[node_idx].push_back(input_idx);
+        fanouts_[input_idx].push_back(node_idx);
       }
-      const int input_idx = it->second;
-      fanins_[node_idx].push_back(input_idx);
-      fanouts_[input_idx].push_back(node_idx);
     }
 
     // Dedup the input list while it's still hot in cache.
diff --git a/tensorflow/core/grappler/graph_topology_view.h b/tensorflow/core/grappler/graph_topology_view.h
index 1c222df4b6..c40d0093b9 100644
--- a/tensorflow/core/grappler/graph_topology_view.h
+++ b/tensorflow/core/grappler/graph_topology_view.h
@@ -47,6 +47,8 @@ namespace grappler {
 class GraphTopologyView {
  public:
   GraphTopologyView() = default;
+  explicit GraphTopologyView(bool skip_invalid_edges)
+      : skip_invalid_edges_(skip_invalid_edges) {}
 
   // Initialize graph topology view from the graph. It's possible to pass
   // additional edges that do not exist in a graph, but must be respected when
@@ -84,6 +86,10 @@ class GraphTopologyView {
   const absl::InlinedVector<int, 2>& GetFanout(int node_idx) const;
 
  private:
+  // If true, all invalid edges and inputs (srd, dst or input node not found in
+  // a graph) will be skipped, otherwise initialization will fail with error.
+  bool skip_invalid_edges_ = false;
+
   // WARN: `graph_` must outlive this object and graph nodes must not be
   // destructed, because node names captured with absl::string_view.
   const GraphDef* graph_ = nullptr;  // do not own
diff --git a/tensorflow/core/grappler/op_types.cc b/tensorflow/core/grappler/op_types.cc
index f7931c615c..e9bf36ab29 100644
--- a/tensorflow/core/grappler/op_types.cc
+++ b/tensorflow/core/grappler/op_types.cc
@@ -568,6 +568,22 @@ bool IsDataset(const NodeDef& node) {
          op == "DatasetToSingleElement" || op == "ReduceDataset";
 }
 
+bool IsStateful(const NodeDef node, const OpRegistryInterface* op_registry) {
+  const OpDef* op_def = nullptr;
+  const string& op_name = node.op();
+  Status status = op_registry->LookUpOpDef(op_name, &op_def);
+  if (!status.ok()) {
+    LOG(WARNING) << "Failed to lookup OpDef for " << op_name
+                 << ". Error: " << status.error_message();
+    return false;
+  }
+  return op_def->is_stateful();
+}
+
+bool IsStateful(const NodeDef node) {
+  return IsStateful(node, OpRegistry::Global());
+}
+
 bool IsFreeOfSideEffect(const NodeDef& node,
                         const OpRegistryInterface* op_registry) {
   // Placeholders must be preserved to keep the graph feedable.
diff --git a/tensorflow/core/grappler/op_types.h b/tensorflow/core/grappler/op_types.h
index 092748f4ff..4f48ec4767 100644
--- a/tensorflow/core/grappler/op_types.h
+++ b/tensorflow/core/grappler/op_types.h
@@ -186,6 +186,11 @@ bool IsPersistent(const NodeDef& node);
 // Returns true if the node belongs to the NC_DATASET class (see graph/graph.h).
 bool IsDataset(const NodeDef& node);
 
+// Returns true if the node op is marked as stateful, or if it was not found in
+// op_registry.
+bool IsStateful(const NodeDef node, const OpRegistryInterface* op_registry);
+bool IsStateful(const NodeDef node);  // use OpRegistry::Global()
+
 bool IsFreeOfSideEffect(const NodeDef& node,
                         const OpRegistryInterface* op_registry);
 bool IsFreeOfSideEffect(const NodeDef& node);  // use OpRegistry::Global()
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.cc b/tensorflow/core/grappler/optimizers/function_optimizer.cc
index bd69405b1a..da8ad4a778 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer.cc
@@ -68,6 +68,11 @@ constexpr char kFuncAttrName[] = "f";
 
 constexpr char kNoInlineAttr[] = "_noinline";
 
+// Names of the nodes that used to anchor incoming/outgoing control edges for
+// inlined function calls (see InlineIndirectFunctionCall).
+constexpr char kControlInputNodeName[] = "control_input";
+constexpr char kControlOutputNodeName[] = "control_output";
+
 bool AttrIsTrue(const FunctionDef& func, const string& attr) {
   return func.attr().count(attr) != 0 && func.attr().at(attr).b();
 }
@@ -1237,12 +1242,26 @@ Status InlineSymbolicGradient(const NodeDef& node,
 // dependency tracking via input/output control edges, and we relax some of the
 // constraints that we have for direct function call inlining.
 //
-// "When a `PartitionedCallOp` function has a resource (DT_RESOURCE data type)
-// input argument it "captures" the mutable resource.  This is implemented by
-// automatically adding a incoming control edge from the previous side-effectful
-// op touching that resource, and an outgoing control edge to the next
-// side-effectful op using the same resource. This serializes the mutations of
-// the resource to make graph execution deterministic.
+// Automatic control dependency rules:
+//
+// 1) "When a `PartitionedCallOp` function has a resource (DT_RESOURCE data
+//    type) input argument it "captures" the mutable resource.  This is
+//    implemented by automatically adding a incoming control edge from the
+//    previous side-effectful op touching that resource, and an outgoing control
+//    edge to the next side-effectful op using the same resource. This
+//    serializes the mutations of the resource to make graph execution
+//    deterministic.
+//
+// 2) All stateful ops inside a function body are guaranteed to execute in
+//    program order, this is achieved by adding control edges between stateful
+//    ops at graph construction time.
+//
+// 3) Furthermore, all ops accepting the same resource as an input are
+//    guaranteed to run in program order. This is also done by adding control
+//    edges at graph construction time. The last op touching the resource
+//    will have an outgoing control edge to all function return nodes, which
+//    will guarantee that all side effects to the resource will happen before
+//    function completion.
 //
 // Function call inlining must preserve side effect visibility:
 //
@@ -1251,17 +1270,31 @@ Status InlineSymbolicGradient(const NodeDef& node,
 // 2) All side effects to the captured resources, that happened inside function
 //    body, must be visible to every op/function using that resource after the
 //    function call completed.
-
-// To guarantee that these properties are preserved after inlining we do:
 //
-// 1) Forward all input control dependencies from the function call node to the
-//    inlined function inputs (Identity nodes).
-// 2) Each side-effectful op inside function body adds itself as a control
-//    dependency to all the nodes in output control set of function call node.
+// To guarantee that these properties are preserved after inlining we:
+//
+// 1) Create "input_control" NoOp. Function call node incoming control edges
+//    will be forwarded *to* this node. Function inputs (Identity nodes) will
+//    have a control edge *from* this node. If function has no inputs, by
+//    construction it must have nodes without inputs in the function body, and
+//    in this case these nodes will have a control edge *from* this node.
+
+// 2) Create "output_control" NoOp. All nodes that have incoming control edge
+//    *from* the function call node, will be forwarded to this node. Function
+//    outputs (Identity nodes) will have a control edge *to* this node. This
+//    will guarantee that nodes that have control dependency on the function
+//    call, will observe all side-effects (guaranteed by graph construction with
+//    automatic control dependencies tracking).
 //
-// We do not add any other control dependencies to/from function body nodes,
-// because they are pure functions of input tensors, and can be freely
-// reordered.
+// If after function instantiation we find a stateful or a dataset op inside
+// the function body, that is not reachable from any of the function outputs (or
+// if the function has no outputs), we do not inline it, because we can't
+// guarantee that these nodes will be executed in correct order (or executed at
+// all) after inlining.
+//
+// We do not try to add any extra control edges to make sure that all
+// side-effectful nodes will be executed, that should be handled at graph
+// construction time.
 
 struct MaybeDeadOutput {
   const NodeDef* dead_tensor_src;
@@ -1382,6 +1415,131 @@ Status IsInlinableIndirectFunctionCall(const FunctionOptimizerContext& ctx,
   return Status::OK();
 }
 
+// Checks that all side-effects will be executed in well defined order. We do it
+// by checking if there is a path from stateful/dataset ops to one of the output
+// nodes.
+Status CheckThatSideEffectsWillExecute(
+    const FunctionOptimizerContext& ctx,
+    const GraphTopologyView& graph_topo_view,
+    const absl::flat_hash_set<string> output_nodes) {
+  // We ignore side-effects safety check in aggressive mode.
+  const bool aggressive = ctx.opt_level() == RewriterConfig::AGGRESSIVE;
+
+  for (const NodeDef& func_body_node : graph_topo_view.graph()->node()) {
+    const bool node_must_execute =
+        IsDataset(func_body_node) ||
+        IsStateful(func_body_node, &ctx.function_library());
+    if (!node_must_execute) continue;
+
+    VLOG(3) << "Check that node " << func_body_node.name()
+            << " will execute after inlining.";
+    bool will_execute = false;
+
+    // Check if we reached one of the output nodes.
+    const auto callbacks = DfsCallbacks::PreOrder([&](const NodeDef* node) {
+      if (output_nodes.count(node->name())) will_execute = true;
+    });
+
+    // Stop if we already proved that node will execute.
+    const auto predicates = DfsPredicates::Enter(
+        [&](const NodeDef* node) { return !will_execute; });
+
+    DfsTraversal(graph_topo_view, {&func_body_node},
+                 TraversalDirection::kFollowOutputs, predicates, callbacks);
+
+    if (!will_execute && !aggressive) {
+      return errors::Internal(
+          "Can't guarantee execution of a side-effectful node, that is not "
+          "reachable from function outputs. Function body node: ",
+          SummarizeNodeDef(func_body_node));
+    }
+
+    if (!will_execute && aggressive) {
+      LOG(WARNING)
+          << "Can't guarantee execution of a side-effectful node, that is not "
+             "reachable from function outputs. Function body node: "
+          << SummarizeNodeDef(func_body_node);
+    }
+  }
+
+  return Status::OK();
+}
+
+Status PlaceInlinedFunctionBody(
+    const FunctionOptimizerContext& ctx, const NodeDef& func_node,
+    const GrapplerFunctionItem& item,
+    const absl::flat_hash_map<absl::string_view, int>& input_placeholders_idx,
+    GraphDef* placed_graph_def) {
+  // Control flow lowering and Placer works with a Graph object.
+  std::unique_ptr<Graph> func_body_graph =
+      absl::make_unique<Graph>(ctx.function_library());
+
+  GraphConstructorOptions opts;
+  TF_RETURN_IF_ERROR(
+      ConvertGraphDefToGraph(opts, item.graph, func_body_graph.get()));
+
+  // TODO(ezhulenev): Lower If/While ops.
+
+  // ------------------------------------------------------------------------ //
+  // Before placing the function body nodes we pin input placeholders to the
+  // same device as their corresponding input nodes.
+
+  for (Node* func_body_node : func_body_graph->nodes()) {
+    const auto input_placeholder_idx =
+        input_placeholders_idx.find(func_body_node->name());
+
+    if (input_placeholder_idx != input_placeholders_idx.end()) {
+      const int input_idx = input_placeholder_idx->second;
+      const GraphView::OutputPort output_port =
+          ctx.graph_view().GetRegularFanin({&func_node, input_idx});
+
+      VLOG(3) << "Pin inlined function input node '" << func_body_node->name()
+              << "' to the '" << output_port.node->device() << "' device.";
+      func_body_node->set_requested_device(output_port.node->device());
+    }
+  }
+
+  // ------------------------------------------------------------------------ //
+  // After placing nodes corresponding to the function inputs, we need to assign
+  // device placements to all other function body nodes.
+
+  const DeviceSet* devices = ctx.devices();
+
+  if (devices->devices().empty()) {
+    // If there are no devices available for placer, we just put all nodes to
+    // the same device as a function caller node. This can happen if Grappler is
+    // running "offline", without active runtime session, for example as a part
+    // of a batch job for graph analysis/optimization.
+    VLOG(3) << "Assign function call node device to all function body nodes. "
+            << "Device: " << func_node.device();
+    for (Node* func_body_node : func_body_graph->nodes()) {
+      func_body_node->set_requested_device(func_node.device());
+    }
+  } else {
+    // If we are running in an active runtime session, Grappler will get the
+    // graph after initial placing is done, and we should have devices for the
+    // placer.
+    VLOG(3) << "Run placer for instantiated function body. Devices: ["
+            << absl::StrJoin(
+                   devices->devices(), ", ",
+                   [](string* out, const Device* d) { out->append(d->name()); })
+            << "]";
+
+    // Use function caller node device as a default for placer.
+    const Device* default_device =
+        devices->FindDeviceByName(func_node.device());
+
+    Placer placer(func_body_graph.get(), devices,
+                  nullptr /* No session options */, default_device);
+    TF_RETURN_IF_ERROR(placer.Run());
+  }
+
+  // Convert Graph back to the placed GraphDef.
+  func_body_graph->ToGraphDef(placed_graph_def);
+
+  return Status::OK();
+}
+
 Status InlineIndirectFunctionCall(const NodeDef& func_node,
                                   const FunctionDef& func,
                                   const int graph_def_version,
@@ -1457,26 +1615,6 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
     inputs.push_back(tensor_id);
   }
 
-  // If we have a node inside the function body without inputs (e.g. Const), we
-  // must attach a control dependency to it, to make sure that if a function
-  // call happens inside a loop, the node will be evaluated in correct frame.
-  //
-  // If the function call node has no inputs and no control dependencies, it
-  // means that it can't be a function call inside a loop, and we can safely
-  // insert that node without inputs into the main graph.
-  //
-  // TODO(ezhulenev): Use FrameMap (see grappler/utils/frame.h) to find out if
-  // the function is called inside a loop.
-  std::vector<string> empty_inputs_hook;
-  if (!item.inputs().empty()) {
-    const InputArgExpansion& arg0 = item.inputs()[0];
-    DCHECK(!arg0.placeholders.empty());
-    empty_inputs_hook.push_back(AsControlDependency(AddPrefixToNodeName(
-        arg0.placeholders[0], /*prefix=*/func_node.name())));
-  } else if (!happens_before.empty()) {
-    empty_inputs_hook.push_back(AsControlDependency(happens_before[0]));
-  }
-
   // Mapping from input placeholder name to function input position.
   absl::flat_hash_map<absl::string_view, int> input_placeholders_idx;
   for (const InputArgExpansion& input_arg : item.inputs()) {
@@ -1489,70 +1627,88 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
   const string prefix = strings::StrCat(func_node.name(), "/");
 
   // ------------------------------------------------------------------------ //
-  // Before placing the function body nodes we pin input placeholders to the
-  // same device as their corresponding input nodes.
+  // For each function output value we added an identity node that reads the
+  // tensor from one of the function body nodes. When we inline function into
+  // the main graph we want to bypass these nodes, so we keep a mapping from
+  // 'output node name' -> 'output tensor name'.
+  absl::flat_hash_map<string, string> output_tensors;
 
-  for (NodeDef& func_body_node : *item.graph.mutable_node()) {
-    const auto input_placeholder_idx =
-        input_placeholders_idx.find(func_body_node.name());
+  // Unique names of nodes producing tensors in `output_tensors`.
+  absl::flat_hash_set<string> output_tensors_nodes;
 
-    if (input_placeholder_idx != input_placeholders_idx.end()) {
-      const int input_idx = input_placeholder_idx->second;
-      const GraphView::OutputPort output_port =
-          ctx->graph_view().GetRegularFanin({&func_node, input_idx});
+  // Identity nodes added to the function body in place of function outputs.
+  absl::flat_hash_set<string> output_nodes;
+  for (const OutputArgExpansion& output_arg : item.outputs()) {
+    for (const string& output_node : output_arg.output_nodes) {
+      output_nodes.insert(output_node);
+    }
+  }
 
-      VLOG(3) << "Pin inlined function input node '" << func_body_node.name()
-              << "' to the '" << output_port.node->device() << "' device.";
-      func_body_node.set_device(output_port.node->device());
+  for (const NodeDef& func_body_node : item.graph.node()) {
+    const string& node_name = func_body_node.name();
+
+    if (IsIdentity(func_body_node) && output_nodes.count(node_name)) {
+      const string& output_tensor = func_body_node.input(0);
+      output_tensors.emplace(node_name, output_tensor);
+
+      SafeTensorId tensor_id = ParseTensorName(output_tensor);
+      output_tensors_nodes.insert(tensor_id.node());
     }
   }
 
   // ------------------------------------------------------------------------ //
-  // After placing nodes corresponding to the function inputs, we need to assign
-  // device placements to all other function body nodes.
+  // To guarantee side-effects execution order we add NoOp control_input and
+  // control_output nodes:
+  // 1) 'control_input' node will have incoming control edges from all nodes in
+  //    'happens_before' set.
+  // 2) 'control_output' node will have outgoing control edges to all nodes in
+  //    'happens_after' set.
 
-  GraphDef placed_graph_def;
+  NodeDef* control_input = nullptr;
+  NodeDef* control_output = nullptr;
 
-  const DeviceSet* devices = ctx->devices();
+  // IMPORTANT: Actual control inputs will be added to these nodes at the very
+  // last stage, because we don't want to have invalid edges in a function body
+  // graph (control edges depend on the nodes in the "outer" optimized graph).
 
-  if (devices->devices().empty()) {
-    // If there are no devices available for placer, we just put all nodes to
-    // the same device as a function caller node. This can happen if Grappler is
-    // running "offline", without active runtime session, for example as a part
-    // of a batch job for graph analysis/optimization.
-    VLOG(3) << "Assign function call node device to all function body nodes. "
-            << "Device: " << func_node.device();
-    placed_graph_def = item.mutable_function_body();
-    for (NodeDef& node : *placed_graph_def.mutable_node()) {
-      node.set_device(func_node.device());
-    }
-  } else {
-    // If we are running in an active runtime session, Grappler will get the
-    // graph after initial placing is done, and we should have devices for the
-    // placer.
-    VLOG(3) << "Run placer for instantiated function body. Devices: ["
-            << absl::StrJoin(
-                   devices->devices(), ", ",
-                   [](string* out, const Device* d) { out->append(d->name()); })
-            << "]";
+  if (!happens_before.empty()) {
+    control_input = item.graph.add_node();
+    control_input->set_op("NoOp");
+    control_input->set_name(kControlInputNodeName);
+  }
 
-    // Construct a Graph object from the instantiated function body.
-    GraphConstructorOptions opts;
-    Graph graph(ctx->function_library());
-    TF_RETURN_IF_ERROR(
-        ConvertGraphDefToGraph(opts, item.function_body(), &graph));
+  if (!happens_after.empty()) {
+    control_output = item.graph.add_node();
+    control_output->set_op("NoOp");
+    control_output->set_name(kControlOutputNodeName);
+  }
 
-    // Use function caller node device as a default for placer.
-    const Device* default_device =
-        devices->FindDeviceByName(func_node.device());
+  // ------------------------------------------------------------------------ //
+  // If we have a node inside the function body without inputs (e.g. Const), we
+  // must attach a control dependency to it, to make sure that if a function
+  // call happens inside a loop, the node will be evaluated in correct frame.
+  //
+  // If the function call node has no inputs and no control dependencies, it
+  // means that it can't be a function call inside a loop, and we can safely
+  // insert that node without inputs into the main graph.
+  //
+  // TODO(ezhulenev): Use FrameMap (see grappler/utils/frame.h) to find out if
+  // the function is called inside a loop.
+  std::vector<string> empty_inputs_hook;
+  if (!item.inputs().empty()) {
+    const InputArgExpansion& arg0 = item.inputs()[0];
+    empty_inputs_hook.push_back(arg0.placeholders[0]);
+  } else if (control_input != nullptr) {
+    empty_inputs_hook.push_back(control_input->name());
+  }
 
-    Placer placer(&graph, devices, nullptr, /* No session options */
-                  default_device);
-    TF_RETURN_IF_ERROR(placer.Run());
+  // ------------------------------------------------------------------------ //
+  // Grappler called after PRE_PLACEMENT and PLACEMENT passes, so we have to
+  // make sure that after inlining all nodes will have valid device assignment.
 
-    // Convert Graph back to the GraphDef.
-    graph.ToGraphDef(&placed_graph_def);
-  }
+  GraphDef placed_graph_def;
+  TF_RETURN_IF_ERROR(PlaceInlinedFunctionBody(
+      *ctx, func_node, item, input_placeholders_idx, &placed_graph_def));
 
   // ------------------------------------------------------------------------ //
   // After all nodes placed we need to prepare them for inlining into the
@@ -1578,17 +1734,21 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
       func_body_node.add_input(inputs[input_idx].ToString());
 
       // All side effects must happen before inputs can start executing.
-      for (const string& hb_node : happens_before) {
-        func_body_node.add_input(AsControlDependency(hb_node));
+      if (control_input) {
+        func_body_node.add_input(
+            AsControlDependency(inlined_node_name(control_input->name())));
       }
-
     } else {
       // Update inputs of the regular function body nodes.
       for (string& input : *func_body_node.mutable_input()) {
         input = inlined_node_name(input);
       }
-      if (func_body_node.input_size() == 0 && !empty_inputs_hook.empty()) {
-        *func_body_node.add_input() = empty_inputs_hook[0];
+      // Add control input to ensure node executed in correct frame.
+      if (func_body_node.input_size() == 0 && !empty_inputs_hook.empty() &&
+          func_body_node.name() != kControlInputNodeName &&
+          func_body_node.name() != kControlOutputNodeName) {
+        *func_body_node.add_input() =
+            AsControlDependency(inlined_node_name(empty_inputs_hook[0]));
       }
     }
 
@@ -1606,97 +1766,85 @@ Status InlineIndirectFunctionCall(const NodeDef& func_node,
     AddDefaultsToNodeDef(*op_def, &func_body_node);
   }
 
-  // Construct a graph view for the preprocessed function body graph.
-  GraphView placed_graph_view(&placed_graph_def);
+  // ------------------------------------------------------------------------ //
+  // Check that after inlining all side-effects will be executed in well defined
+  // order. We do it by checking if there is a path from stateful/dataset ops to
+  // one of the output nodes.
+
+  // Because we rename all the nodes before inlining, we need a copy of
+  // output_nodes with a new names.
+  absl::flat_hash_set<string> inlined_output_nodes;
+  for (const string& output_node : output_nodes) {
+    inlined_output_nodes.insert(inlined_node_name(output_node));
+  }
+  const auto is_inlined_output_node = [&](const NodeDef& node) -> bool {
+    return inlined_output_nodes.find(node.name()) != inlined_output_nodes.end();
+  };
 
-  // Keep track of side-effectful ops inside function body. Each outgoing
-  // control edge from the function call node, must be replaced with control
-  // edges from inlined side-effectful ops.
-  std::vector<string> side_effectful_nodes;
+  // Construct a graph topology view for DFS traversals (skip invalid edges for
+  // input nodes connected to nodes in the optimized graph).
+  GraphTopologyView placed_topo_view(/*skip_invalid_edges=*/true);
+  TF_RETURN_IF_ERROR(placed_topo_view.InitializeFromGraph(placed_graph_def));
+  TF_RETURN_IF_ERROR(CheckThatSideEffectsWillExecute(*ctx, placed_topo_view,
+                                                     inlined_output_nodes));
 
-  // We have to make sure that all side-effectful and dataset-output nodes
-  // inside a function body will be executed after function inlining.
-  for (NodeDef& func_body_node : *placed_graph_def.mutable_node()) {
-    const bool node_must_execute =
-        !IsFreeOfSideEffect(func_body_node, &ctx->function_library()) ||
-        IsDataset(func_body_node);
-
-    if (node_must_execute) {
-      int num_fanouts = placed_graph_view.NumFanouts(
-          func_body_node, /*include_controlled_nodes=*/true);
-
-      // If the node doesn't have any outgoing edges and we do not have any
-      // nodes in the `happens_after` set, we can't inline a function and
-      // guarantee that it will be executed. The only exception if we do
-      // function library optimization, and the GrapplerItem was instantiated
-      // for the function body, because functions do not prune these ops.
-
-      if (num_fanouts == 0 && happens_after.empty() &&
-          !ctx->optimization_options().is_function_instantiation) {
-        return errors::Internal(
-            "Can't inline a function with a side-effectful op with empty "
-            "fanouts and empty output control edge set. Function body node: ",
-            SummarizeNodeDef(func_body_node));
-      }
+  // ------------------------------------------------------------------------ //
+  // Move all the nodes to the optimized graph after successful preprocessing.
 
-      side_effectful_nodes.push_back(func_body_node.name());
+  if (control_input != nullptr) {
+    string inlined_node = inlined_node_name(control_input->name());
+    absl::optional<int> node_idx = placed_topo_view.GetNodeIndex(inlined_node);
+
+    for (const string& node_name : happens_before) {
+      placed_graph_def.mutable_node(*node_idx)->add_input(
+          AsControlDependency(node_name));
     }
   }
 
-  // Identity nodes added to the function body in place of function outputs.
-  absl::flat_hash_set<string> output_nodes;
-  for (const OutputArgExpansion& output_arg : item.outputs()) {
-    for (const string& output_node : output_arg.output_nodes) {
-      output_nodes.insert(inlined_node_name(output_node));
+  if (control_output != nullptr) {
+    string inlined_node = inlined_node_name(control_output->name());
+    absl::optional<int> node_idx = placed_topo_view.GetNodeIndex(inlined_node);
+
+    // Add control edges from all nodes producing output tensors.
+    for (const string& node_name : output_tensors_nodes) {
+      placed_graph_def.mutable_node(*node_idx)->add_input(
+          AsControlDependency(inlined_node_name(node_name)));
     }
-  }
 
-  // For each function output value we added an identity node that reads the
-  // tensor from one of the function body nodes. When we inline function into
-  // the main graph we want to bypass these nodes, so we keep a mapping from
-  // 'output node name' -> 'output tensor name'.
-  absl::flat_hash_map<string, string> output_tensors;
+    // Forward all control dependencies in the optimized graph to the new node.
+    ctx->AddControlOverrides(func_node, {inlined_node});
+  }
 
-  // Move all the nodes to the optimized graph after successful preprocessing.
   for (NodeDef& func_body_node : *placed_graph_def.mutable_node()) {
-    const string& node_name = func_body_node.name();
-
-    // Skip output identity node, and add a mapping to the output tensor.
-    if (IsIdentity(func_body_node) && output_nodes.count(node_name)) {
-      output_tensors.emplace(node_name, func_body_node.input(0));
+    // Skip output identity nodes.
+    if (IsIdentity(func_body_node) && is_inlined_output_node(func_body_node))
       continue;
-    }
 
     optimized_graph->add_node()->Swap(&func_body_node);
   }
 
-  DCHECK(output_tensors.size() == item.output_size())
-      << "Each function output must be mapped to an output tensor";
-
-  // TODO(ezhulenev): Inline nested indirect function calls.
-
   // Indirect function call is fully inlined into the optimized graph, and we do
   // not copy the original function call node, so we have to setup tensor
   // mapping from old output tensors, to the outputs of inlined nodes.
   int output_idx = 0;
   for (const OutputArgExpansion& output : item.outputs()) {
     for (const string& output_node : output.output_nodes) {
-      const string inlined_output = inlined_node_name(output_node);
-      const string& output_tensor = output_tensors.at(inlined_output);
+      const string& output_tensor = output_tensors.at(output_node);
 
       const SafeTensorId from_tensor(func_node.name(), output_idx++);
       const SafeTensorId to_tensor = ParseTensorName(output_tensor);
 
-      ctx->AddTensorMapping(from_tensor, to_tensor);
+      const SafeTensorId inlined_to_tensor =
+          SafeTensorId(absl::StrCat(func_node.name(), "/", to_tensor.node()),
+                       to_tensor.index());
+
+      ctx->AddTensorMapping(from_tensor, inlined_to_tensor);
     }
   }
 
-  // After inlining we'll have to forward all control dependencies from function
-  // call node to all side-effectful ops inside function body.
-  ctx->AddControlOverrides(func_node, side_effectful_nodes);
-
   VLOG(3) << "Successfully inlined indirect function call: "
           << SummarizeNodeDef(func_node);
+
   return Status::OK();
 }
 
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
index 827b0658c5..a67acffd03 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
@@ -831,36 +831,43 @@ TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithControlDependencies) {
             kDevice),
 
        // Function body of a first function call inlined into the graph.
-       NDef("f1/x", "Identity", {"a:0", "^init_v"}, {{"T", DT_FLOAT}}, kDevice),
-       NDef("f1/y", "Identity", {"b:0", "^init_v"}, {{"T", DT_FLOAT}}, kDevice),
-       NDef("f1/v", "Identity", {"v:0", "^init_v"}, {{"T", DT_RESOURCE}},
+       NDef("f1/control_input", "NoOp", {"^init_v"}, {}, kDevice),
+       NDef("f1/x", "Identity", {"a:0", "^f1/control_input"}, {{"T", DT_FLOAT}},
             kDevice),
+       NDef("f1/y", "Identity", {"b:0", "^f1/control_input"}, {{"T", DT_FLOAT}},
+            kDevice),
+       NDef("f1/v", "Identity", {"v:0", "^f1/control_input"},
+            {{"T", DT_RESOURCE}}, kDevice),
        NDef("f1/one", "Const", {"^f1/x"},
             {{"dtype", DT_FLOAT}, {"value", kOne}}, kDevice),
        NDef("f1/add", "AssignAddVariableOp", {"f1/v", "f1/one"},
             {{"dtype", DT_FLOAT}}, kDevice),
        NDef("f1/mul", "Mul", {"f1/x", "f1/y", "^f1/add"}, {{"T", DT_FLOAT}},
             kDevice),
+       NDef("f1/control_output", "NoOp", {"^f1/mul"}, {}, kDevice),
 
        // Function body of a second function call also inlined into the graph,
        // and input nodes read directly from the inlined nodes of the first
        // function call.
-       NDef("f2/x", "Identity", {"f1/mul:0", "^f1/add"}, {{"T", DT_FLOAT}},
-            kDevice),
-       NDef("f2/y", "Identity", {"f1/mul:0", "^f1/add"}, {{"T", DT_FLOAT}},
-            kDevice),
-       NDef("f2/v", "Identity", {"v:0", "^f1/add"}, {{"T", DT_RESOURCE}},
-            kDevice),
+       NDef("f2/control_input", "NoOp", {"^f1/control_output"}, {}, kDevice),
+       NDef("f2/x", "Identity", {"f1/mul:0", "^f2/control_input"},
+            {{"T", DT_FLOAT}}, kDevice),
+       NDef("f2/y", "Identity", {"f1/mul:0", "^f2/control_input"},
+            {{"T", DT_FLOAT}}, kDevice),
+       NDef("f2/v", "Identity", {"v:0", "^f2/control_input"},
+            {{"T", DT_RESOURCE}}, kDevice),
        NDef("f2/one", "Const", {"^f2/x"},
             {{"dtype", DT_FLOAT}, {"value", kOne}}, kDevice),
        NDef("f2/add", "AssignAddVariableOp", {"f2/v", "f2/one"},
             {{"dtype", DT_FLOAT}}, kDevice),
        NDef("f2/mul", "Mul", {"f2/x", "f2/y", "^f2/add"}, {{"T", DT_FLOAT}},
             kDevice),
+       NDef("f2/control_output", "NoOp", {"^f2/mul"}, {}, kDevice),
 
        // Return values read directly from inlined nodes.
        NDef("out_1", "Identity", {"f2/mul:0"}, {{"T", DT_FLOAT}}, kDevice),
-       NDef("out_2", "ReadVariableOp", {"v", "^f1/add", "^f2/add"},
+       NDef("out_2", "ReadVariableOp",
+            {"v", "^f1/control_output", "^f2/control_output"},
             {{"dtype", DT_FLOAT}}, kDevice)},
 
       // Function library.
@@ -937,7 +944,8 @@ TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithDevicePlacement) {
   CompareGraphs(expected, optimized_graph);
 }
 
-TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithoutSideEffects) {
+TEST_F(FunctionOptimizerTest,
+       InlineIndirectFunctionWithControlDependencyAndNoSideEffects) {
   using test::function::NDef;
   using FDH = FunctionDefHelper;
 
@@ -998,12 +1006,16 @@ TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithoutSideEffects) {
        NDef("f1/x", "Identity", {"a:0"}, {{"T", DT_FLOAT}}, kDevice),
        NDef("f1/y", "Identity", {"b:0"}, {{"T", DT_FLOAT}}, kDevice),
        NDef("f1/mul", "Mul", {"f1/x", "f1/y"}, {{"T", DT_FLOAT}}, kDevice),
+       NDef("f1/control_output", "NoOp", {"^f1/mul"}, {}, kDevice),
 
        // Function body of a second function call also inlined into the graph,
        // and input nodes read directly from the inlined nodes of the first
        // function call, and control dependency edge removed.
-       NDef("f2/x", "Identity", {"f1/mul:0"}, {{"T", DT_FLOAT}}, kDevice),
-       NDef("f2/y", "Identity", {"f1/mul:0"}, {{"T", DT_FLOAT}}, kDevice),
+       NDef("f2/control_input", "NoOp", {"^f1/control_output"}, {}, kDevice),
+       NDef("f2/x", "Identity", {"f1/mul:0", "^f2/control_input"},
+            {{"T", DT_FLOAT}}, kDevice),
+       NDef("f2/y", "Identity", {"f1/mul:0", "^f2/control_input"},
+            {{"T", DT_FLOAT}}, kDevice),
        NDef("f2/mul", "Mul", {"f2/x", "f2/y"}, {{"T", DT_FLOAT}}, kDevice),
 
        // Return directly from inlined node of f2.
-- 
GitLab


From bfcddf733e85c790921afccec4ca80849e9eb9ab Mon Sep 17 00:00:00 2001
From: Shashi Shekhar <shashishekhar@google.com>
Date: Tue, 22 Jan 2019 12:22:53 -0800
Subject: [PATCH 1071/1095] Refactor Python utility functions from Python
 wrapper. - Moves ErrorReporter to a separate class. - Moves some utility
 functions to a separate class.

PiperOrigin-RevId: 230387612
---
 .../lite/python/interpreter_wrapper/BUILD     | 23 +++++
 .../interpreter_wrapper.cc                    | 95 ++-----------------
 .../interpreter_wrapper/interpreter_wrapper.i |  1 +
 .../python_error_reporter.cc                  | 43 +++++++++
 .../python_error_reporter.h                   | 49 ++++++++++
 .../interpreter_wrapper/python_utils.cc       | 77 +++++++++++++++
 .../python/interpreter_wrapper/python_utils.h | 38 ++++++++
 7 files changed, 237 insertions(+), 89 deletions(-)
 create mode 100644 tensorflow/lite/python/interpreter_wrapper/python_error_reporter.cc
 create mode 100644 tensorflow/lite/python/interpreter_wrapper/python_error_reporter.h
 create mode 100644 tensorflow/lite/python/interpreter_wrapper/python_utils.cc
 create mode 100644 tensorflow/lite/python/interpreter_wrapper/python_utils.h

diff --git a/tensorflow/lite/python/interpreter_wrapper/BUILD b/tensorflow/lite/python/interpreter_wrapper/BUILD
index 767a9fc476..6de6fb48f7 100644
--- a/tensorflow/lite/python/interpreter_wrapper/BUILD
+++ b/tensorflow/lite/python/interpreter_wrapper/BUILD
@@ -11,6 +11,8 @@ cc_library(
     srcs = ["interpreter_wrapper.cc"],
     hdrs = ["interpreter_wrapper.h"],
     deps = [
+        ":python_error_reporter",
+        ":python_utils",
         "//tensorflow/lite:framework",
         "//tensorflow/lite/kernels:builtin_ops",
         "//third_party/py/numpy:headers",
@@ -19,6 +21,27 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "python_error_reporter",
+    srcs = ["python_error_reporter.cc"],
+    hdrs = ["python_error_reporter.h"],
+    deps = [
+        "//tensorflow/lite/core/api",
+        "//third_party/python_runtime:headers",
+    ],
+)
+
+cc_library(
+    name = "python_utils",
+    srcs = ["python_utils.cc"],
+    hdrs = ["python_utils.h"],
+    deps = [
+        "//tensorflow/lite:framework",
+        "//third_party/py/numpy:headers",
+        "//third_party/python_runtime:headers",
+    ],
+)
+
 tf_py_wrap_cc(
     name = "tensorflow_wrap_interpreter_wrapper",
     srcs = [
diff --git a/tensorflow/lite/python/interpreter_wrapper/interpreter_wrapper.cc b/tensorflow/lite/python/interpreter_wrapper/interpreter_wrapper.cc
index d14af439ec..9ccaabbfe9 100644
--- a/tensorflow/lite/python/interpreter_wrapper/interpreter_wrapper.cc
+++ b/tensorflow/lite/python/interpreter_wrapper/interpreter_wrapper.cc
@@ -21,6 +21,8 @@ limitations under the License.
 #include "tensorflow/lite/interpreter.h"
 #include "tensorflow/lite/kernels/register.h"
 #include "tensorflow/lite/model.h"
+#include "tensorflow/lite/python/interpreter_wrapper/python_error_reporter.h"
+#include "tensorflow/lite/python/interpreter_wrapper/python_utils.h"
 
 // Disallow Numpy 1.7 deprecated symbols.
 #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
@@ -60,36 +62,6 @@ limitations under the License.
 namespace tflite {
 namespace interpreter_wrapper {
 
-class PythonErrorReporter : public tflite::ErrorReporter {
- public:
-  PythonErrorReporter() {}
-
-  // Report an error message
-  int Report(const char* format, va_list args) override {
-    char buf[1024];
-    int formatted = vsnprintf(buf, sizeof(buf), format, args);
-    buffer_ << buf;
-    return formatted;
-  }
-
-  // Set's a Python runtime exception with the last error.
-  PyObject* exception() {
-    std::string last_message = message();
-    PyErr_SetString(PyExc_RuntimeError, last_message.c_str());
-    return nullptr;
-  }
-
-  // Gets the last error message and clears the buffer.
-  std::string message() {
-    std::string value = buffer_.str();
-    buffer_.clear();
-    return value;
-  }
-
- private:
-  std::stringstream buffer_;
-};
-
 namespace {
 
 // Calls PyArray's initialization to initialize all the API pointers. Note that
@@ -114,61 +86,6 @@ std::unique_ptr<tflite::Interpreter> CreateInterpreter(
   return interpreter;
 }
 
-int TfLiteTypeToPyArrayType(TfLiteType tf_lite_type) {
-  switch (tf_lite_type) {
-    case kTfLiteFloat32:
-      return NPY_FLOAT32;
-    case kTfLiteInt32:
-      return NPY_INT32;
-    case kTfLiteInt16:
-      return NPY_INT16;
-    case kTfLiteUInt8:
-      return NPY_UINT8;
-    case kTfLiteInt8:
-      return NPY_INT8;
-    case kTfLiteInt64:
-      return NPY_INT64;
-    case kTfLiteString:
-      return NPY_OBJECT;
-    case kTfLiteBool:
-      return NPY_BOOL;
-    case kTfLiteComplex64:
-      return NPY_COMPLEX64;
-    case kTfLiteNoType:
-      return NPY_NOTYPE;
-      // Avoid default so compiler errors created when new types are made.
-  }
-  return NPY_NOTYPE;
-}
-
-TfLiteType TfLiteTypeFromPyArray(PyArrayObject* array) {
-  int pyarray_type = PyArray_TYPE(array);
-  switch (pyarray_type) {
-    case NPY_FLOAT32:
-      return kTfLiteFloat32;
-    case NPY_INT32:
-      return kTfLiteInt32;
-    case NPY_INT16:
-      return kTfLiteInt16;
-    case NPY_UINT8:
-      return kTfLiteUInt8;
-    case NPY_INT8:
-      return kTfLiteInt8;
-    case NPY_INT64:
-      return kTfLiteInt64;
-    case NPY_BOOL:
-      return kTfLiteBool;
-    case NPY_OBJECT:
-    case NPY_STRING:
-    case NPY_UNICODE:
-      return kTfLiteString;
-    case NPY_COMPLEX64:
-      return kTfLiteComplex64;
-      // Avoid default so compiler errors created when new types are made.
-  }
-  return kTfLiteNoType;
-}
-
 struct PyDecrefDeleter {
   void operator()(PyObject* p) const { Py_DECREF(p); }
 };
@@ -307,7 +224,7 @@ PyObject* InterpreterWrapper::TensorType(int i) const {
     return nullptr;
   }
 
-  int code = TfLiteTypeToPyArrayType(tensor->type);
+  int code = python_utils::TfLiteTypeToPyArrayType(tensor->type);
   if (code == -1) {
     PyErr_Format(PyExc_ValueError, "Invalid tflite type code %d", code);
     return nullptr;
@@ -352,12 +269,12 @@ PyObject* InterpreterWrapper::SetTensor(int i, PyObject* value) {
   PyArrayObject* array = reinterpret_cast<PyArrayObject*>(array_safe.get());
   const TfLiteTensor* tensor = interpreter_->tensor(i);
 
-  if (TfLiteTypeFromPyArray(array) != tensor->type) {
+  if (python_utils::TfLiteTypeFromPyArray(array) != tensor->type) {
     PyErr_Format(PyExc_ValueError,
                  "Cannot set tensor:"
                  " Got tensor of type %d"
                  " but expected type %d for input %d ",
-                 TfLiteTypeFromPyArray(array), tensor->type, i);
+                 python_utils::TfLiteTypeFromPyArray(array), tensor->type, i);
     return nullptr;
   }
 
@@ -400,7 +317,7 @@ PyObject* CheckGetTensorArgs(Interpreter* interpreter_, int tensor_index,
     return nullptr;
   }
 
-  *type_num = TfLiteTypeToPyArrayType((*tensor)->type);
+  *type_num = python_utils::TfLiteTypeToPyArrayType((*tensor)->type);
   if (*type_num == -1) {
     PyErr_SetString(PyExc_ValueError, "Unknown tensor type.");
     return nullptr;
diff --git a/tensorflow/lite/python/interpreter_wrapper/interpreter_wrapper.i b/tensorflow/lite/python/interpreter_wrapper/interpreter_wrapper.i
index f52ef1eeca..ef4b28f047 100644
--- a/tensorflow/lite/python/interpreter_wrapper/interpreter_wrapper.i
+++ b/tensorflow/lite/python/interpreter_wrapper/interpreter_wrapper.i
@@ -21,6 +21,7 @@ limitations under the License.
 #include "tensorflow/lite/interpreter.h"
 #include "tensorflow/lite/model.h"
 #include "tensorflow/lite/python/interpreter_wrapper/interpreter_wrapper.h"
+#include "tensorflow/lite/python/interpreter_wrapper/python_error_reporter.h"
 %}
 
 
diff --git a/tensorflow/lite/python/interpreter_wrapper/python_error_reporter.cc b/tensorflow/lite/python/interpreter_wrapper/python_error_reporter.cc
new file mode 100644
index 0000000000..803a4c2934
--- /dev/null
+++ b/tensorflow/lite/python/interpreter_wrapper/python_error_reporter.cc
@@ -0,0 +1,43 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/python/interpreter_wrapper/python_error_reporter.h"
+
+namespace tflite {
+namespace interpreter_wrapper {
+
+// Report an error message
+int PythonErrorReporter::Report(const char* format, va_list args) {
+  char buf[1024];
+  int formatted = vsnprintf(buf, sizeof(buf), format, args);
+  buffer_ << buf;
+  return formatted;
+}
+
+// Set's a Python runtime exception with the last error.
+PyObject* PythonErrorReporter::exception() {
+  std::string last_message = message();
+  PyErr_SetString(PyExc_RuntimeError, last_message.c_str());
+  return nullptr;
+}
+
+// Gets the last error message and clears the buffer.
+std::string PythonErrorReporter::message() {
+  std::string value = buffer_.str();
+  buffer_.clear();
+  return value;
+}
+}  // namespace interpreter_wrapper
+}  // namespace tflite
diff --git a/tensorflow/lite/python/interpreter_wrapper/python_error_reporter.h b/tensorflow/lite/python/interpreter_wrapper/python_error_reporter.h
new file mode 100644
index 0000000000..7d4e308834
--- /dev/null
+++ b/tensorflow/lite/python/interpreter_wrapper/python_error_reporter.h
@@ -0,0 +1,49 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_LITE_PYTHON_INTERPRETER_WRAPPER_PYTHON_ERROR_REPORTER_H_
+#define TENSORFLOW_LITE_PYTHON_INTERPRETER_WRAPPER_PYTHON_ERROR_REPORTER_H_
+
+#include <Python.h>
+
+#include <sstream>
+#include <string>
+
+#include "tensorflow/lite/core/api/error_reporter.h"
+
+namespace tflite {
+namespace interpreter_wrapper {
+
+class PythonErrorReporter : public tflite::ErrorReporter {
+ public:
+  PythonErrorReporter() {}
+
+  // Report an error message
+  int Report(const char* format, va_list args) override;
+
+  // Sets a Python runtime exception with the last error and
+  // clears the error message buffer.
+  PyObject* exception();
+
+  // Gets the last error message and clears the buffer.
+  std::string message();
+
+ private:
+  std::stringstream buffer_;
+};
+
+}  // namespace interpreter_wrapper
+}  // namespace tflite
+#endif  // TENSORFLOW_LITE_PYTHON_INTERPRETER_WRAPPER_PYTHON_ERROR_REPORTER_H_
diff --git a/tensorflow/lite/python/interpreter_wrapper/python_utils.cc b/tensorflow/lite/python/interpreter_wrapper/python_utils.cc
new file mode 100644
index 0000000000..2dc604356a
--- /dev/null
+++ b/tensorflow/lite/python/interpreter_wrapper/python_utils.cc
@@ -0,0 +1,77 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/python/interpreter_wrapper/python_utils.h"
+
+namespace tflite {
+namespace python_utils {
+
+int TfLiteTypeToPyArrayType(TfLiteType tf_lite_type) {
+  switch (tf_lite_type) {
+    case kTfLiteFloat32:
+      return NPY_FLOAT32;
+    case kTfLiteInt32:
+      return NPY_INT32;
+    case kTfLiteInt16:
+      return NPY_INT16;
+    case kTfLiteUInt8:
+      return NPY_UINT8;
+    case kTfLiteInt8:
+      return NPY_INT8;
+    case kTfLiteInt64:
+      return NPY_INT64;
+    case kTfLiteString:
+      return NPY_OBJECT;
+    case kTfLiteBool:
+      return NPY_BOOL;
+    case kTfLiteComplex64:
+      return NPY_COMPLEX64;
+    case kTfLiteNoType:
+      return NPY_NOTYPE;
+      // Avoid default so compiler errors created when new types are made.
+  }
+  return NPY_NOTYPE;
+}
+
+TfLiteType TfLiteTypeFromPyArray(PyArrayObject* array) {
+  int pyarray_type = PyArray_TYPE(array);
+  switch (pyarray_type) {
+    case NPY_FLOAT32:
+      return kTfLiteFloat32;
+    case NPY_INT32:
+      return kTfLiteInt32;
+    case NPY_INT16:
+      return kTfLiteInt16;
+    case NPY_UINT8:
+      return kTfLiteUInt8;
+    case NPY_INT8:
+      return kTfLiteInt8;
+    case NPY_INT64:
+      return kTfLiteInt64;
+    case NPY_BOOL:
+      return kTfLiteBool;
+    case NPY_OBJECT:
+    case NPY_STRING:
+    case NPY_UNICODE:
+      return kTfLiteString;
+    case NPY_COMPLEX64:
+      return kTfLiteComplex64;
+      // Avoid default so compiler errors created when new types are made.
+  }
+  return kTfLiteNoType;
+}
+
+}  // namespace python_utils
+}  // namespace tflite
diff --git a/tensorflow/lite/python/interpreter_wrapper/python_utils.h b/tensorflow/lite/python/interpreter_wrapper/python_utils.h
new file mode 100644
index 0000000000..30a44226b8
--- /dev/null
+++ b/tensorflow/lite/python/interpreter_wrapper/python_utils.h
@@ -0,0 +1,38 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_LITE_PYTHON_INTERPRETER_WRAPPER_PYTHON_UTILS_H_
+#define TENSORFLOW_LITE_PYTHON_INTERPRETER_WRAPPER_PYTHON_UTILS_H_
+
+#include "tensorflow/lite/context.h"
+
+// Disallow Numpy 1.7 deprecated symbols.
+#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
+
+#include <Python.h>
+
+#include "numpy/arrayobject.h"
+#include "numpy/ufuncobject.h"
+
+namespace tflite {
+namespace python_utils {
+
+int TfLiteTypeToPyArrayType(TfLiteType tf_lite_type);
+
+TfLiteType TfLiteTypeFromPyArray(PyArrayObject* array);
+
+}  // namespace python_utils
+}  // namespace tflite
+#endif  // TENSORFLOW_LITE_PYTHON_INTERPRETER_WRAPPER_PYTHON_UTILS_H_
-- 
GitLab


From 27c2f0e7e52d0919ad9c526447f036c49a4a601e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 22 Jan 2019 12:41:06 -0800
Subject: [PATCH 1072/1095] Remove NCCL2 remote config repo, it is no longer
 being used because NCCL is built from source.

PiperOrigin-RevId: 230390673
---
 tensorflow/opensource_only.files              |  1 -
 .../preconfig/ubuntu14.04/nccl2/BUILD         | 25 -------------------
 .../preconfig/ubuntu14.04/nccl2/WORKSPACE     |  2 --
 3 files changed, 28 deletions(-)
 delete mode 100755 third_party/toolchains/preconfig/ubuntu14.04/nccl2/BUILD
 delete mode 100644 third_party/toolchains/preconfig/ubuntu14.04/nccl2/WORKSPACE

diff --git a/tensorflow/opensource_only.files b/tensorflow/opensource_only.files
index 9c4dd2f4b8..c546a34cf8 100644
--- a/tensorflow/opensource_only.files
+++ b/tensorflow/opensource_only.files
@@ -47,7 +47,6 @@ tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda10.0-cudnn7/cuda/BUI
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/gcc-nvcc-cuda10.0/BUILD
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/build_defs.bzl
 tensorflow/third_party/toolchains/preconfig/ubuntu14.04/cuda9.0-cudnn7/cuda/BUILD
-tensorflow/third_party/toolchains/preconfig/ubuntu14.04/nccl2/BUILD
 tensorflow/third_party/toolchains/preconfig/generate/workspace.bzl
 tensorflow/third_party/toolchains/preconfig/generate/containers.bzl
 tensorflow/third_party/toolchains/preconfig/generate/generate.bzl
diff --git a/third_party/toolchains/preconfig/ubuntu14.04/nccl2/BUILD b/third_party/toolchains/preconfig/ubuntu14.04/nccl2/BUILD
deleted file mode 100755
index 96ed60d3cf..0000000000
--- a/third_party/toolchains/preconfig/ubuntu14.04/nccl2/BUILD
+++ /dev/null
@@ -1,25 +0,0 @@
-filegroup(
-    name = "LICENSE",
-    visibility = ["//visibility:public"],
-)
-
-cc_library(
-    name = "nccl",
-    srcs = ["libnccl.so.2"],
-    hdrs = ["nccl.h"],
-    include_prefix = "third_party/nccl",
-    visibility = ["//visibility:public"],
-    deps = [
-        "@local_config_cuda//cuda:cuda_headers",
-    ],
-)
-
-genrule(
-    name = "nccl-files",
-    outs = [
-        "libnccl.so.2",
-        "nccl.h",
-    ],
-    cmd = """cp "/usr/include/nccl.h" "$(@D)/nccl.h" &&
-           cp "/usr/lib/libnccl.so.2" "$(@D)/libnccl.so.2" """,
-)
diff --git a/third_party/toolchains/preconfig/ubuntu14.04/nccl2/WORKSPACE b/third_party/toolchains/preconfig/ubuntu14.04/nccl2/WORKSPACE
deleted file mode 100644
index 1e6662ac91..0000000000
--- a/third_party/toolchains/preconfig/ubuntu14.04/nccl2/WORKSPACE
+++ /dev/null
@@ -1,2 +0,0 @@
-# DO NOT EDIT: automatically generated WORKSPACE file for nccl_configure rule
-workspace(name = "local_config_nccl")
-- 
GitLab


From e9106d123c6d75f76a86d406d036e3650371cf97 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Tue, 22 Jan 2019 12:59:02 -0800
Subject: [PATCH 1073/1095] [TF:XLA] Include the frame name in the identity of
 and recurrences

This is a correctness fix that is important for users outside the loop
(i.e. users that use that "last value" of the and recurrence).  The last value
of the and recurrence depends on how many times the loop executes, which isn't
_just_ a function of the init and step deadness predicate.

For instance if we have a loop like:

  Init = ... // Could be live or dead
  Step = ... // Could be live or dead
  V = Init
  for (int i = 0; i < N; i++) {
    V = V * Step
  }
  X = V

Then if N is 0 then X (the "loop exit" value of V) is alive iff Init was alive
(i.e. Predicate(X) == Predicate(Init)).  But if N is 1 then X is alive iff both
Init and Step are alive (i.e. Predicate(X) == Predicate(Init) &
Predicate(Step)).  We can't infer this dependence of Predicate(X) on N if
Predicate(X) == {Predicate(Init),&,Predicate(Step)} since that does not denote
the dependence on the loop trip count.  So we make the frame name (i.e. the
loop) part of the and recurrence identity.

Once I added the control flow inference logic I also noticed that quite a few of
the tests were violating frame invariants.  I've fixed those tests.

Also worth pointing out: I've not yet seen this bug cause any problems in the
wild.

PiperOrigin-RevId: 230393971
---
 tensorflow/compiler/jit/deadness_analysis.cc  | 112 +++++++--
 .../compiler/jit/deadness_analysis_test.cc    | 222 +++++++++++-------
 2 files changed, 232 insertions(+), 102 deletions(-)

diff --git a/tensorflow/compiler/jit/deadness_analysis.cc b/tensorflow/compiler/jit/deadness_analysis.cc
index 9a2ada5b42..22c265adcb 100644
--- a/tensorflow/compiler/jit/deadness_analysis.cc
+++ b/tensorflow/compiler/jit/deadness_analysis.cc
@@ -21,6 +21,7 @@ limitations under the License.
 #include "tensorflow/compiler/jit/deadness_analysis_internal.h"
 #include "tensorflow/compiler/jit/xla_cluster_util.h"
 #include "tensorflow/core/graph/algorithm.h"
+#include "tensorflow/core/graph/control_flow.h"
 #include "tensorflow/core/graph/tensor_id.h"
 #include "tensorflow/core/lib/hash/hash.h"
 
@@ -222,29 +223,40 @@ class NotPredicate : public Predicate {
   std::array<Predicate*, 1> operands_;
 };
 
-// Represents an infinite list of predicates.
+// Represents the liveness of an induction variable.  For users inside the loop
+// this represents the "current" liveness of the induction variable.  For users
+// outside the loop it represents the "last" liveness of the induction variable.
 //
-// An AndRecurrence with start = S and step = X is printed as {S,&,X} and stands
-// for the list of predicates:
+// More concretely, an and recurrence {S,&,X}<loop> represents the liveness of V
+// in the following graph:
 //
-//   S, S & GenSym(X,1), S & GenSym(X,1) & GenSym(X,2), ...
+//   V = Merge(S', V_NextIt)
+//   V = Op(V, X')
+//   V_NextIt = NextIteration(V)
 //
-// where GenSym(<expression>, <id>) renames every SymbolPredicate in
-// <expression> by appending <id> to it, in effect creating a "fresh" symbol.
-// This means {P,&,Q} is not equal to "P on the first iteration; P&Q on
-// subsequent iterations".
+// where Predicate(S') = S and Predicate(X') = X.
+//
+// `X` may contain symbolic predicates and the operations corresponding to these
+// symbolic predicates are either in frame `loop` or outside it.  The symbols
+// that are inside frame `loop` are loop variant (i.e. can have different
+// liveness in each loop iteration) and the symbols that are outside frame
+// `loop` are loop invariant (i.e. have the same liveness across all
+// iterations).
 class AndRecurrencePredicate : public Predicate {
  public:
-  explicit AndRecurrencePredicate(Predicate* start, Predicate* step)
-      : Predicate(HashPredicateSequence(Kind::kAndRecurrence, {start, step})),
-        operands_({start, step}) {}
+  explicit AndRecurrencePredicate(Predicate* start, Predicate* step,
+                                  string frame)
+      : Predicate(Hash(start, step, frame)),
+        operands_({start, step}),
+        frame_(std::move(frame)) {}
 
   Predicate* start() const { return operands_[0]; }
   Predicate* step() const { return operands_[1]; }
+  absl::string_view frame() const { return frame_; }
 
   string ToString() const override {
     return absl::StrCat("{", start()->ToString(), ",&,", step()->ToString(),
-                        "}");
+                        "}<", frame(), ">");
   }
 
   Kind kind() const override { return Kind::kAndRecurrence; }
@@ -255,6 +267,13 @@ class AndRecurrencePredicate : public Predicate {
 
  private:
   std::array<Predicate*, 2> operands_;
+  string frame_;
+
+  static int64 Hash(Predicate* start, Predicate* step, const string& frame) {
+    return Hash64Combine(
+        HashPredicateSequence(Kind::kAndRecurrence, {start, step}),
+        Hash64(frame));
+  }
 };
 
 // Represents an uninterpreted symbol in a logical predicate.
@@ -348,18 +367,22 @@ class PredicateFactory {
     return result;
   }
 
-  Predicate* MakeAndRecurrencePredicate(Predicate* start, Predicate* step) {
-    auto it = interned_and_rec_instances_.find({start, step});
+  Predicate* MakeAndRecurrencePredicate(Predicate* start, Predicate* step,
+                                        string frame) {
+    SignatureForAndRec signature(start, step, std::move(frame));
+    auto it = interned_and_rec_instances_.find(signature);
     if (it != interned_and_rec_instances_.end()) {
       return it->second.get();
     }
 
-    std::unique_ptr<Predicate> new_pred =
-        Make<AndRecurrencePredicate>(start, step);
+    std::unique_ptr<Predicate> new_pred = Make<AndRecurrencePredicate>(
+        std::get<0>(signature), std::get<1>(signature), std::get<2>(signature));
     Predicate* new_pred_ptr = new_pred.get();
-    CHECK(interned_and_rec_instances_
-              .emplace(SignatureForAndRec(start, step), std::move(new_pred))
-              .second);
+    bool inserted =
+        interned_and_rec_instances_.emplace(signature, std::move(new_pred))
+            .second;
+    (void)inserted;
+    DCHECK(inserted);
     return new_pred_ptr;
   }
 
@@ -450,7 +473,7 @@ class PredicateFactory {
   using SignatureForAndOr =
       std::pair<Predicate::Kind, absl::Span<Predicate* const>>;
   using SignatureForNot = Predicate*;
-  using SignatureForAndRec = std::pair<Predicate*, Predicate*>;
+  using SignatureForAndRec = std::tuple<Predicate*, Predicate*, string>;
   using SignatureForSymbol = std::pair<SafeTensorId, bool>;
 
   struct HashSignatureForAndOr {
@@ -720,6 +743,7 @@ class DeadnessAnalysisImpl : public DeadnessAnalysis {
   const Graph& graph_;
   absl::flat_hash_map<TensorId, Predicate*, TensorId::Hasher> predicate_map_;
   PredicateFactory predicate_factory_;
+  std::vector<ControlFlowInfo> control_flow_info_;
   bool vlog_;
 };
 
@@ -862,6 +886,28 @@ Predicate* DeduceStepPredicate(PredicateFactory* predicate_factory,
 
   return found_sym ? predicate_factory->MakeAndPredicate(and_ops) : nullptr;
 }
+
+Status GetFullFrameName(const Node* n,
+                        absl::Span<const ControlFlowInfo> cfi_infos,
+                        string* full_frame_name) {
+  int depth = 0;
+  for (const ControlFlowInfo* cfi_iter = &cfi_infos[n->id()]; !n->IsSource();
+       n = cfi_iter->parent_frame, cfi_iter = &cfi_infos[n->id()]) {
+    if (depth > 0) {
+      absl::StrAppend(full_frame_name, ";");
+    }
+
+    absl::StrAppend(full_frame_name, cfi_iter->frame_name);
+
+    if (depth++ > 5000) {
+      return errors::Internal(
+          "Frame of depth > 5000:  Probably malformed graph or a bug in "
+          "BuildControlFlowInfo");
+    }
+  }
+
+  return Status::OK();
+}
 }  // namespace
 
 Status DeadnessAnalysisImpl::HandleMerge(Node* n,
@@ -926,8 +972,11 @@ Status DeadnessAnalysisImpl::HandleMerge(Node* n,
 
         Predicate* start =
             predicate_factory_.MakeOrPredicate(non_recurrent_inputs);
-        Predicate* and_rec =
-            predicate_factory_.MakeAndRecurrencePredicate(start, step);
+        string frame_name;
+        TF_RETURN_IF_ERROR(
+            GetFullFrameName(n, control_flow_info_, &frame_name));
+        Predicate* and_rec = predicate_factory_.MakeAndRecurrencePredicate(
+            start, step, std::move(frame_name));
         SetPredicate(n, {0, 1, Graph::kControlSlot}, and_rec, should_revisit);
         return Status::OK();
       }
@@ -993,6 +1042,25 @@ Status DeadnessAnalysisImpl::Populate() {
 
 Status DeadnessAnalysisImpl::PopulateWithReversePostOrder(
     absl::Span<Node* const> rpo) {
+  std::vector<string> unreachable_nodes;
+  // Compute the loop structure of the graph.
+  std::vector<ControlFlowInfo> control_flow_info;
+  TF_RETURN_IF_ERROR(
+      BuildControlFlowInfo(&graph_, &control_flow_info_, &unreachable_nodes));
+
+  // Do some opportunistic error checking:
+  if (!unreachable_nodes.empty()) {
+    if (unreachable_nodes.size() > 5) {
+      unreachable_nodes.erase(unreachable_nodes.begin() + 5,
+                              unreachable_nodes.end());
+    }
+
+    return errors::InvalidArgument(
+        "Found unreachable nodes, most likely source and sink nodes not "
+        "connected: ",
+        absl::StrJoin(unreachable_nodes, ", "));
+  }
+
   // This an abstract interpretation over the deadness propagation semantics of
   // the graph executor.
   //
diff --git a/tensorflow/compiler/jit/deadness_analysis_test.cc b/tensorflow/compiler/jit/deadness_analysis_test.cc
index 11ee00ac6b..1adc32cf67 100644
--- a/tensorflow/compiler/jit/deadness_analysis_test.cc
+++ b/tensorflow/compiler/jit/deadness_analysis_test.cc
@@ -123,10 +123,9 @@ InductionVarInfo CreateInductionVariable(const Scope& root,
   Output increment_by = ops::Const(root.WithOpName(prefix + "/incr"), 1);
   Output final_value = ops::Const(root.WithOpName(prefix + "/final"), 10);
   Output loop_cond_expr =
-      ops::Less(root.WithOpName(prefix + "/less"), iv.output, final_value);
-  Output loop_cond =
-      ops::LoopCond(root.WithOpName(prefix + "/cond"), loop_cond_expr);
-  ops::Switch latch(root.WithOpName(prefix + "/latch"), iv.output, loop_cond);
+      ops::Less(root.WithOpName(prefix + "/cond"), iv.output, final_value);
+  ops::Switch latch(root.WithOpName(prefix + "/latch"), iv.output,
+                    loop_cond_expr);
   ops::internal::Exit exit(root.WithOpName(prefix + "/exit"),
                            latch.output_false);
   Output iv_next = ops::Add(root.WithOpName(prefix + "/ivnext"),
@@ -140,7 +139,7 @@ InductionVarInfo CreateInductionVariable(const Scope& root,
   root.graph()->AddControlEdge(iv.output.node(), increment_by.node());
   root.graph()->AddControlEdge(iv.output.node(), final_value.node());
 
-  return {iv.output, loop_cond};
+  return {iv.output, loop_cond_expr};
 }
 
 InductionVarInfo CreateInductionVariable(const Scope& root,
@@ -515,24 +514,27 @@ TEST(DeadnessAnalysisTest, Loop) {
 
     // In theory we should be able to tell that iv0/cond:0 and iv1/cond:0
     // produce the same deadness.  But we're not that smart today.
-    EXPECT_EQ(predicate_map[ControlOutputFor(iv0)], "{#true,&,*iv0/cond:0}");
-    EXPECT_EQ(predicate_map[ControlOutputFor(iv1)], "{#true,&,*iv1/cond:0}");
-    EXPECT_EQ(predicate_map[ControlOutputFor(iv2)], "{#true,&,*iv2/cond:0}");
+    EXPECT_EQ(predicate_map[ControlOutputFor(iv0)],
+              "{#true,&,*iv0/cond:0}<fr0>");
+    EXPECT_EQ(predicate_map[ControlOutputFor(iv1)],
+              "{#true,&,*iv1/cond:0}<fr0>");
+    EXPECT_EQ(predicate_map[ControlOutputFor(iv2)],
+              "{#true,&,*iv2/cond:0}<fr0>");
     EXPECT_EQ(predicate_map[ControlOutputFor(add0)],
-              "({#true,&,*iv1/cond:0} & {#true,&,*iv0/cond:0})");
+              "({#true,&,*iv1/cond:0}<fr0> & {#true,&,*iv0/cond:0}<fr0>)");
     EXPECT_EQ(predicate_map[ControlOutputFor(add1)],
-              "({#true,&,*iv1/cond:0} & {#true,&,*iv2/cond:0})");
+              "({#true,&,*iv2/cond:0}<fr0> & {#true,&,*iv1/cond:0}<fr0>)");
   }
 }
 
 TEST(DeadnessAnalysisTest, ControlEquivalentLoopBodies) {
   Scope root = Scope::NewRootScope().ExitOnError();
-  InductionVarInfo iv = CreateInductionVariable(root, "iv0", "frame", 0);
+  InductionVarInfo iv = CreateInductionVariable(root, "iv0", "loop", 0);
   Output dependent_iv0 =
-      CreateDependentLoopInvariantValue(root, "div0", "frame", iv.loop_cond, 0)
+      CreateDependentLoopInvariantValue(root, "div0", "loop", iv.loop_cond, 0)
           .induction_var;
   Output dependent_iv1 =
-      CreateDependentLoopInvariantValue(root, "div1", "frame", iv.loop_cond, 0)
+      CreateDependentLoopInvariantValue(root, "div1", "loop", iv.loop_cond, 0)
           .induction_var;
   Output add0 = ops::Add(root.WithOpName("add0"), dependent_iv0, dependent_iv1);
 
@@ -549,13 +551,13 @@ TEST(DeadnessAnalysisTest, ControlEquivalentLoopBodies) {
     TF_ASSERT_OK(ComputePredicates(*root.graph(), &predicate_map));
 
     EXPECT_EQ(predicate_map[ControlOutputFor(iv.induction_var)],
-              "{#true,&,*iv0/cond:0}");
+              "{#true,&,*iv0/cond:0}<loop>");
     EXPECT_EQ(predicate_map[ControlOutputFor(dependent_iv0)],
-              "{#true,&,(*iv0/cond:0 & iv0/iv:0)}");
+              "{#true,&,(*iv0/cond:0 & iv0/iv:0)}<loop>");
     EXPECT_EQ(predicate_map[ControlOutputFor(dependent_iv1)],
-              "{#true,&,(*iv0/cond:0 & iv0/iv:0)}");
+              "{#true,&,(*iv0/cond:0 & iv0/iv:0)}<loop>");
     EXPECT_EQ(predicate_map[ControlOutputFor(add0)],
-              "{#true,&,(*iv0/cond:0 & iv0/iv:0)}");
+              "{#true,&,(*iv0/cond:0 & iv0/iv:0)}<loop>");
   }
 }
 
@@ -595,32 +597,33 @@ TEST(DeadnessAnalysisTest, LoopInvariantPredicateOnBackedge) {
 TEST(DeadnessAnalysisTest, ControlEquivalentNestedLoopBodies) {
   Scope root = Scope::NewRootScope().ExitOnError();
   InductionVarInfo iv_outer =
-      CreateInductionVariable(root, "iv_outer", "frame", 0);
+      CreateInductionVariable(root, "iv_outer", "outer_loop", 0);
+  Output enter_constant_outer_loop = ops::internal::Enter(
+      root.WithOpName("constant_enter_outer_loop"),
+      ops::Const(root.WithOpName("constant"), 5), "outer_loop",
+      ops::internal::Enter::Attrs().IsConstant(true));
   ops::Switch inner_value(root.WithOpName("outer_is_live"),
-                          ops::Const(root.WithOpName("constant"), 5),
-                          iv_outer.loop_cond);
+                          enter_constant_outer_loop, iv_outer.loop_cond);
   InductionVarInfo iv_inner = CreateInductionVariable(
-      root, "iv_inner", "frame",
-      ops::internal::Enter(root.WithOpName("iv_inner/enter"),
-                           inner_value.output_true, "frame_inner"));
+      root, "iv_inner", "inner_loop", inner_value.output_true);
 
   Output dependent_outer_iv0 =
-      CreateDependentLoopInvariantValue(root, "dependent_outer_iv0", "frame",
-                                        iv_outer.loop_cond, 0)
+      CreateDependentLoopInvariantValue(root, "dependent_outer_iv0",
+                                        "outer_loop", iv_outer.loop_cond, 0)
           .induction_var;
   Output dependent_outer_iv1 =
-      CreateDependentLoopInvariantValue(root, "dependent_outer_iv1", "frame",
-                                        iv_outer.loop_cond, 0)
+      CreateDependentLoopInvariantValue(root, "dependent_outer_iv1",
+                                        "outer_loop", iv_outer.loop_cond, 0)
           .induction_var;
 
-  Output dependent_inner_iv0 =
-      CreateDependentLoopInvariantValue(root, "dependent_inner_iv0", "frame",
-                                        iv_inner.loop_cond, dependent_outer_iv0)
-          .induction_var;
-  Output dependent_inner_iv1 =
-      CreateDependentLoopInvariantValue(root, "dependent_inner_iv1", "frame",
-                                        iv_inner.loop_cond, dependent_outer_iv1)
-          .induction_var;
+  Output dependent_inner_iv0 = CreateDependentLoopInvariantValue(
+                                   root, "dependent_inner_iv0", "inner_loop",
+                                   iv_inner.loop_cond, dependent_outer_iv0)
+                                   .induction_var;
+  Output dependent_inner_iv1 = CreateDependentLoopInvariantValue(
+                                   root, "dependent_inner_iv1", "inner_loop",
+                                   iv_inner.loop_cond, dependent_outer_iv1)
+                                   .induction_var;
 
   Output add0 = ops::Add(root.WithOpName("add0"), dependent_inner_iv0,
                          dependent_inner_iv1);
@@ -638,46 +641,50 @@ TEST(DeadnessAnalysisTest, ControlEquivalentNestedLoopBodies) {
     TF_ASSERT_OK(ComputePredicates(*root.graph(), &predicate_map));
 
     EXPECT_EQ(predicate_map[ControlOutputFor(iv_outer.induction_var)],
-              "{#true,&,*iv_outer/cond:0}");
+              "{#true,&,*iv_outer/cond:0}<outer_loop>");
     EXPECT_EQ(predicate_map[ControlOutputFor(iv_inner.induction_var)],
-              "{(*iv_outer/cond:0 & {#true,&,*iv_outer/cond:0}),&,"
-              "*iv_inner/cond:0}");
+              "{({#true,&,*iv_outer/cond:0}<outer_loop> & "
+              "*iv_outer/cond:0),&,*iv_inner/cond:0}<inner_loop;outer_loop>");
 
     EXPECT_EQ(predicate_map[ControlOutputFor(dependent_inner_iv0)],
-              "{{#true,&,(iv_outer/iv:0 & *iv_outer/cond:0)},&,"
-              "(*iv_inner/cond:0 & iv_inner/iv:0)}");
+              "{{#true,&,(iv_outer/iv:0 & "
+              "*iv_outer/cond:0)}<outer_loop>,&,(*iv_inner/cond:0 & "
+              "iv_inner/iv:0)}<inner_loop;outer_loop>");
+
     EXPECT_EQ(predicate_map[ControlOutputFor(dependent_inner_iv1)],
-              "{{#true,&,(iv_outer/iv:0 & *iv_outer/cond:0)},&,"
-              "(*iv_inner/cond:0 & iv_inner/iv:0)}");
+              "{{#true,&,(iv_outer/iv:0 & "
+              "*iv_outer/cond:0)}<outer_loop>,&,(*iv_inner/cond:0 & "
+              "iv_inner/iv:0)}<inner_loop;outer_loop>");
     EXPECT_EQ(predicate_map[ControlOutputFor(add0)],
-              "{{#true,&,(iv_outer/iv:0 & *iv_outer/cond:0)},&,"
-              "(*iv_inner/cond:0 & iv_inner/iv:0)}");
+              "{{#true,&,(iv_outer/iv:0 & "
+              "*iv_outer/cond:0)}<outer_loop>,&,(*iv_inner/cond:0 & "
+              "iv_inner/iv:0)}<inner_loop;outer_loop>");
   }
 }
 
 TEST(DeadnessAnalysisTest, ControlNonEquivalentNestedLoopBodies) {
   Scope root = Scope::NewRootScope().ExitOnError();
-  InductionVarInfo iv_outer_0 =
-      CreateInductionVariable(root, "iv_outer_0", "frame", 0);
-  ops::Switch inner_value_0(root.WithOpName("outer_0_is_live"),
-                            ops::Const(root.WithOpName("constant"), 5),
-                            iv_outer_0.loop_cond);
-  InductionVarInfo iv_inner_0 = CreateInductionVariable(
-      root, "iv_inner_0", "frame",
-      ops::internal::Enter(root.WithOpName("iv_inner_0/enter"),
-                           inner_value_0.output_true, "frame_inner"));
-
-  InductionVarInfo iv_outer_1 =
-      CreateInductionVariable(root, "iv_outer_1", "frame", 1);
-  ops::Switch inner_init_value_1(root.WithOpName("outer_1_is_live"),
-                                 ops::Const(root.WithOpName("constant"), 5),
-                                 iv_outer_1.loop_cond);
-  InductionVarInfo iv_inner_1 = CreateInductionVariable(
-      root, "iv_inner_1", "frame",
-      ops::internal::Enter(root.WithOpName("iv_inner_1/enter"),
-                           inner_init_value_1.output_true, "frame_inner"));
-  Output add0 = ops::Add(root.WithOpName("add0"), iv_inner_0.induction_var,
-                         iv_inner_1.induction_var);
+
+  std::array<Output, 2> outer_iv;
+  std::array<Output, 2> inner_iv;
+
+  for (int i : {0, 1}) {
+    InductionVarInfo iv_outer =
+        CreateInductionVariable(root, "iv_outer", "outer_loop", 0);
+    Output enter_constant_outer_loop = ops::internal::Enter(
+        root.WithOpName("constant_enter_outer_loop"),
+        ops::Const(root.WithOpName("constant"), 5), "outer_loop",
+        ops::internal::Enter::Attrs().IsConstant(true));
+    ops::Switch inner_value(root.WithOpName("outer_is_live"),
+                            enter_constant_outer_loop, iv_outer.loop_cond);
+    InductionVarInfo iv_inner = CreateInductionVariable(
+        root, "iv_inner", "inner_loop", inner_value.output_true);
+
+    outer_iv[i] = iv_outer.induction_var;
+    inner_iv[i] = iv_inner.induction_var;
+  }
+
+  Output add0 = ops::Add(root.WithOpName("add0"), inner_iv[0], inner_iv[1]);
 
   VLogGraphIfAsked(*root.graph());
 
@@ -692,21 +699,76 @@ TEST(DeadnessAnalysisTest, ControlNonEquivalentNestedLoopBodies) {
     PredicateMapTy predicate_map;
     TF_ASSERT_OK(ComputePredicates(*root.graph(), &predicate_map));
 
-    EXPECT_EQ(predicate_map[ControlOutputFor(iv_outer_0.induction_var)],
-              "{#true,&,*iv_outer_0/cond:0}");
-    EXPECT_EQ(predicate_map[ControlOutputFor(iv_inner_0.induction_var)],
-              "{(*iv_outer_0/cond:0 & {#true,&,*iv_outer_0/cond:0}),&,"
-              "*iv_inner_0/cond:0}");
-    EXPECT_EQ(predicate_map[ControlOutputFor(iv_outer_1.induction_var)],
-              "{#true,&,*iv_outer_1/cond:0}");
-    EXPECT_EQ(predicate_map[ControlOutputFor(iv_inner_1.induction_var)],
-              "{(*iv_outer_1/cond:0 & {#true,&,*iv_outer_1/cond:0}),&,"
-              "*iv_inner_1/cond:0}");
-    EXPECT_EQ(predicate_map[ControlOutputFor(add0)],
-              "({(*iv_outer_1/cond:0 & {#true,&,*iv_outer_1/cond:0}),&,"
-              "*iv_inner_1/cond:0} & "
-              "{(*iv_outer_0/cond:0 & {#true,&,*iv_outer_0/cond:0}),&,"
-              "*iv_inner_0/cond:0})");
+    EXPECT_EQ(predicate_map[ControlOutputFor(outer_iv[0])],
+              "{#true,&,*iv_outer/cond:0}<outer_loop>");
+    EXPECT_EQ(predicate_map[ControlOutputFor(inner_iv[0])],
+              "{({#true,&,*iv_outer/cond:0}<outer_loop> & "
+              "*iv_outer/cond:0),&,*iv_inner/cond:0}<inner_loop;outer_loop>");
+    EXPECT_EQ(predicate_map[ControlOutputFor(outer_iv[1])],
+              "{#true,&,*iv_outer/cond_1:0}<outer_loop>");
+    EXPECT_EQ(
+        predicate_map[ControlOutputFor(inner_iv[1])],
+        "{({#true,&,*iv_outer/cond_1:0}<outer_loop> & "
+        "*iv_outer/cond_1:0),&,*iv_inner/cond_1:0}<inner_loop;outer_loop>");
+    EXPECT_EQ(
+        predicate_map[ControlOutputFor(add0)],
+        "({({#true,&,*iv_outer/cond_1:0}<outer_loop> & "
+        "*iv_outer/cond_1:0),&,*iv_inner/cond_1:0}<inner_loop;outer_loop> & "
+        "{({#true,&,*iv_outer/cond:0}<outer_loop> & "
+        "*iv_outer/cond:0),&,*iv_inner/cond:0}<inner_loop;outer_loop>)");
+  }
+}
+
+TEST(DeadnessAnalysisTest, AndRecurrenceNeedsFrameName) {
+  Scope root = Scope::NewRootScope().ExitOnError();
+  InductionVarInfo iv_0 = CreateInductionVariable(root, "iv_0", "frame_0", 10);
+  InductionVarInfo iv_1 = CreateInductionVariable(root, "iv_1", "frame_1", 9);
+
+  Output init = CreateSwitch(root, "init").output_true;
+  Output step = CreateSwitch(root, "step").output_true;
+
+  std::array<Output, 2> exits;
+  std::array<Output, 2> next_iterations;
+
+  for (int i : {0, 1}) {
+    Output init_enter = ops::internal::Enter(
+        root.WithOpName(absl::StrCat("init_enter_frame_", i)), init,
+        absl::StrCat("frame_", i),
+        ops::internal::Enter::Attrs().IsConstant(true));
+    Output step_enter = ops::internal::Enter(
+        root.WithOpName(absl::StrCat("step_enter_frame_", i)), step,
+        absl::StrCat("frame_", i),
+        ops::internal::Enter::Attrs().IsConstant(true));
+
+    ops::Merge iv(root.WithOpName(absl::StrCat("expr_", i)),
+                  {init_enter, init_enter});
+    Output add = ops::Add(root.WithOpName(absl::StrCat("add_", i)), iv.output,
+                          step_enter);
+    next_iterations[i] = ops::NextIteration(
+        root.WithOpName(absl::StrCat("expr_", i, "_next_iteration")), add);
+    EXPECT_TRUE(
+        root.graph()
+            ->UpdateEdge(next_iterations[i].node(), 0, iv.output.node(), 1)
+            .ok());
+    exits[i] = ops::internal::Exit(root.WithOpName(absl::StrCat("exit_", i)),
+                                   iv.output);
+  }
+
+  FixupSourceAndSinkEdges(root.graph());
+
+  {
+    PredicateMapTy predicate_map;
+    TF_ASSERT_OK(ComputePredicates(*root.graph(), &predicate_map));
+
+    EXPECT_NE(predicate_map[ControlOutputFor(exits[0])],
+              predicate_map[ControlOutputFor(exits[1])]);
+    EXPECT_NE(predicate_map[ControlOutputFor(exits[0])], "");
+    EXPECT_NE(predicate_map[ControlOutputFor(exits[1])], "");
+
+    EXPECT_NE(predicate_map[ControlOutputFor(next_iterations[0])],
+              predicate_map[ControlOutputFor(next_iterations[1])]);
+    EXPECT_NE(predicate_map[ControlOutputFor(next_iterations[0])], "");
+    EXPECT_NE(predicate_map[ControlOutputFor(next_iterations[1])], "");
   }
 }
 
-- 
GitLab


From 794b7c6713c7e576063ad10c34048933f92f4085 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Tue, 22 Jan 2019 13:13:33 -0800
Subject: [PATCH 1074/1095] Internal change.

PiperOrigin-RevId: 230396943
---
 tensorflow/cc/profiler/BUILD                | 2 +-
 tensorflow/contrib/distribute/python/BUILD  | 2 ++
 tensorflow/contrib/fused_conv/BUILD         | 1 +
 tensorflow/python/kernel_tests/BUILD        | 4 ----
 tensorflow/python/kernel_tests/linalg/BUILD | 1 -
 tensorflow/python/kernel_tests/signal/BUILD | 2 --
 tensorflow/python/ops/parallel_for/BUILD    | 1 -
 7 files changed, 4 insertions(+), 9 deletions(-)

diff --git a/tensorflow/cc/profiler/BUILD b/tensorflow/cc/profiler/BUILD
index cf65fe1ab9..e9838d9aba 100644
--- a/tensorflow/cc/profiler/BUILD
+++ b/tensorflow/cc/profiler/BUILD
@@ -10,7 +10,7 @@ tf_cuda_cc_test(
     name = "profiler_test",
     srcs = ["profiler_test.cc"],
     tags = [
-        "noguitar",  # b/77649654
+        "nogpu",  # b/77649654
     ],
     deps = [
         ":profiler",
diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index fb8732d591..c663e32026 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -679,6 +679,7 @@ distribute_py_test(
         "no_oss",  # TODO(b/117919883): Fix python error.
         "no_pip",
         "no_windows_gpu",
+        "notap",  # TODO(b/123097760): Fix and re-enable the test.
         "notsan",
     ],
     deps = [
@@ -763,6 +764,7 @@ distribute_py_test(
         "no_oss",  # TODO(b/117919883): Fix python error.
         "no_pip",
         "no_windows_gpu",
+        "notap",  # TODO(b/123097760): Fix and re-enable the test.
         "notsan",
     ],
     deps = [
diff --git a/tensorflow/contrib/fused_conv/BUILD b/tensorflow/contrib/fused_conv/BUILD
index 57a5bfbf43..f65f450eba 100644
--- a/tensorflow/contrib/fused_conv/BUILD
+++ b/tensorflow/contrib/fused_conv/BUILD
@@ -171,6 +171,7 @@ cuda_py_test(
     main = "python/ops/fused_conv2d_bias_activation_benchmark.py",
     tags = [
         "manual",  # TODO(b/117128481): re-enable after fixing OSS build
+        "nogpu",
         "requires-gpu-sm70",
     ],
 )
diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index aef4354ee0..2271d1f9f7 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -1435,7 +1435,6 @@ cuda_py_test(
         "noasan",  # times out
         "optonly",  # times out
     ],
-    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -1525,7 +1524,6 @@ cuda_py_test(
         "//tensorflow/python:nn_grad",
         "//tensorflow/python:nn_ops",
     ],
-    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2756,7 +2754,6 @@ cuda_py_test(
     tags = [
         "optonly",  # times out
     ],
-    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
@@ -2774,7 +2771,6 @@ cuda_py_test(
     ],
     # TODO(b/118842098): Re-enable this test in Kokoro.
     tags = ["no_oss"],
-    xla_enable_strict_auto_jit = True,
 )
 
 tf_py_test(
diff --git a/tensorflow/python/kernel_tests/linalg/BUILD b/tensorflow/python/kernel_tests/linalg/BUILD
index c1aa99cc37..b4608aa38c 100644
--- a/tensorflow/python/kernel_tests/linalg/BUILD
+++ b/tensorflow/python/kernel_tests/linalg/BUILD
@@ -150,7 +150,6 @@ cuda_py_test(
         "noasan",  # times out, b/63678675
         "optonly",  # times out, b/79171797
     ],
-    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
diff --git a/tensorflow/python/kernel_tests/signal/BUILD b/tensorflow/python/kernel_tests/signal/BUILD
index 554bf38029..3e37d230ef 100644
--- a/tensorflow/python/kernel_tests/signal/BUILD
+++ b/tensorflow/python/kernel_tests/signal/BUILD
@@ -29,7 +29,6 @@ cuda_py_tests(
         "//tensorflow/python:spectral_ops_test_util",
         "//tensorflow/python/ops/signal",
     ],
-    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_tests(
@@ -46,7 +45,6 @@ cuda_py_tests(
     ],
     shard_count = 4,
     tags = ["optonly"],
-    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_tests(
diff --git a/tensorflow/python/ops/parallel_for/BUILD b/tensorflow/python/ops/parallel_for/BUILD
index f54ca558f4..fd89d8ad45 100644
--- a/tensorflow/python/ops/parallel_for/BUILD
+++ b/tensorflow/python/ops/parallel_for/BUILD
@@ -114,7 +114,6 @@ cuda_py_test(
         "//tensorflow/python:random_ops",
         "//tensorflow/python:util",
     ],
-    xla_enable_strict_auto_jit = True,
 )
 
 cuda_py_test(
-- 
GitLab


From da5ba1b58296512efe961dbeb76120cd933f203b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 22 Jan 2019 13:27:05 -0800
Subject: [PATCH 1075/1095] Fix spelling/typo.

PiperOrigin-RevId: 230399257
---
 tensorflow/tools/api/tests/api_compatibility_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/api/tests/api_compatibility_test.py b/tensorflow/tools/api/tests/api_compatibility_test.py
index 723fceef41..6b9fc7ddce 100644
--- a/tensorflow/tools/api/tests/api_compatibility_test.py
+++ b/tensorflow/tools/api/tests/api_compatibility_test.py
@@ -59,7 +59,7 @@ _UPDATE_GOLDENS_HELP = """
 # DEFINE_boolean, only_test_core_api, default False:
 _ONLY_TEST_CORE_API_HELP = """
     Some TF APIs are being moved outside of the tensorflow/ directory. There is
-    no garuntee which versions of these APIs will be present when running this
+    no guarantee which versions of these APIs will be present when running this
     test. Therefore, do not error out on API changes in non-core TF code
     if this flag is set.
 """
-- 
GitLab


From a4d4392677f58eb26eed7e7e8ec7604d3d21335b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 22 Jan 2019 13:49:06 -0800
Subject: [PATCH 1076/1095] Minor fix in comments

PiperOrigin-RevId: 230403343
---
 tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc b/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc
index 5b4f863f74..61a8933208 100644
--- a/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc
+++ b/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc
@@ -441,12 +441,6 @@ xla::StatusOr<xla::XlaOp> MakeXlaBackpropFilterConvOp(
   std::vector<int64> window_strides(attrs.num_spatial_dims);
   std::vector<int64> ones(attrs.num_spatial_dims, 1);
 
-  // The activations (inputs) form the LHS of the convolution.
-  // Activations have shape: [batch, in_rows, in_cols, ..., in_depth]
-  // For the gradient computation, we flip the roles of the batch and
-  // feature dimensions.
-  // Each spatial entry has size in_depth * batch
-
   // Swap n_dim and c_dim in the activations.
   dnums.set_input_batch_dimension(c_dim);
   dnums.set_input_feature_dimension(n_dim);
@@ -480,7 +474,7 @@ xla::StatusOr<xla::XlaOp> MakeXlaBackpropFilterConvOp(
     // convolution, we get the right size for the filter.
     // The padded_in_rows should be such that when we convolve this with the
     // expanded_out_rows as a filter, we should get filter_rows back.
-    //
+
     const int64 padded_in_size =
         dims.spatial_dims[i].expanded_output_size +
         (dims.spatial_dims[i].filter_size - 1) * attrs.dilations[dim];
-- 
GitLab


From ea1d491e6702a191e748160be30ebadcdffc7fdd Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Tue, 22 Jan 2019 14:11:24 -0800
Subject: [PATCH 1077/1095] Move TF-TRT C++ implementations to
 compiler/tf2tensorrt.

PiperOrigin-RevId: 230407626
---
 tensorflow/compiler/tf2tensorrt/BUILD         | 440 +++++++++++++++++
 .../tf2tensorrt}/convert/convert_graph.cc     |  23 +-
 .../tf2tensorrt}/convert/convert_graph.h      |   8 +-
 .../convert/convert_graph_test.cc             |   4 +-
 .../tf2tensorrt}/convert/convert_nodes.cc     |  22 +-
 .../tf2tensorrt}/convert/convert_nodes.h      |  16 +-
 .../convert/convert_nodes_test.cc             |  11 +-
 .../convert/trt_optimization_pass.cc          |  14 +-
 .../convert/trt_optimization_pass.h           |   6 +-
 .../tf2tensorrt}/convert/utils.cc             |   2 +-
 .../tf2tensorrt}/convert/utils.h              |   6 +-
 .../kernels/get_serialized_resource_op.cc     |   8 +-
 .../get_serialized_resource_op_test.cc        |   2 +-
 .../tf2tensorrt}/kernels/trt_engine_op.cc     |  21 +-
 .../tf2tensorrt}/kernels/trt_engine_op.h      |  16 +-
 .../ops/get_serialized_resource_op.cc         |   0
 .../tf2tensorrt}/ops/trt_engine_op.cc         |   0
 .../tf2tensorrt}/plugin/trt_plugin.cc         |   6 +-
 .../tf2tensorrt}/plugin/trt_plugin.h          |   6 +-
 .../tf2tensorrt}/plugin/trt_plugin_factory.cc |   2 +-
 .../tf2tensorrt}/plugin/trt_plugin_factory.h  |  10 +-
 .../plugin/trt_plugin_factory_test.cc         |   4 +-
 .../tf2tensorrt}/plugin/trt_plugin_utils.cc   |   2 +-
 .../tf2tensorrt}/plugin/trt_plugin_utils.h    |   8 +-
 .../tf2tensorrt}/python/ops/trt_ops.py        |   6 +-
 .../tf2tensorrt}/segment/segment.cc           |   9 +-
 .../tf2tensorrt}/segment/segment.h            |   6 +-
 .../tf2tensorrt}/segment/segment_test.cc      |   2 +-
 .../tf2tensorrt}/segment/union_find.h         |   6 +-
 .../tf2tensorrt}/tensorrt_test.cc             |   0
 .../tf2tensorrt/utils/test_utils.cc}          |   2 +-
 .../tf2tensorrt/utils/test_utils.h}           |   6 +-
 .../tf2tensorrt/utils}/trt_allocator.cc       |   2 +-
 .../tf2tensorrt/utils}/trt_allocator.h        |   6 +-
 .../tf2tensorrt/utils}/trt_allocator_test.cc  |   2 +-
 .../tf2tensorrt/utils}/trt_int8_calibrator.cc |   2 +-
 .../tf2tensorrt/utils}/trt_int8_calibrator.h  |   6 +-
 .../tf2tensorrt/utils}/trt_logger.cc          |   2 +-
 .../tf2tensorrt/utils}/trt_logger.h           |   6 +-
 .../tf2tensorrt/utils}/trt_lru_cache.h        |  10 +-
 .../tf2tensorrt/utils}/trt_lru_cache_test.cc  |   2 +-
 .../utils}/trt_resource_manager.cc            |   2 +-
 .../tf2tensorrt/utils}/trt_resource_manager.h |   6 +-
 .../tf2tensorrt/utils}/trt_resources.cc       |   2 +-
 .../tf2tensorrt/utils}/trt_resources.h        |  14 +-
 tensorflow/contrib/tensorrt/BUILD             | 441 +-----------------
 .../tensorrt/custom_plugin_examples/BUILD     |   4 +-
 .../custom_plugin_examples/inc_op_plugin.cc   |   2 +-
 .../custom_plugin_examples/inc_op_plugin.h    |   2 +-
 .../contrib/tensorrt/python/__init__.py       |   2 +-
 .../tensorrt/python/trt_convert_test.py       |   4 +-
 .../contrib/tensorrt/shape_fn/trt_shfn.cc     |   4 +-
 .../tensorrt/test/quantization_mnist_test.py  |   4 +-
 .../test/tf_trt_integration_test_base.py      |   4 +-
 tensorflow/contrib/tensorrt/trt_conversion.i  |   6 +-
 55 files changed, 630 insertions(+), 579 deletions(-)
 create mode 100644 tensorflow/compiler/tf2tensorrt/BUILD
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/convert/convert_graph.cc (98%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/convert/convert_graph.h (94%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/convert/convert_graph_test.cc (98%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/convert/convert_nodes.cc (99%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/convert/convert_nodes.h (97%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/convert/convert_nodes_test.cc (99%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/convert/trt_optimization_pass.cc (96%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/convert/trt_optimization_pass.h (91%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/convert/utils.cc (97%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/convert/utils.h (88%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/kernels/get_serialized_resource_op.cc (88%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/kernels/get_serialized_resource_op_test.cc (97%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/kernels/trt_engine_op.cc (97%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/kernels/trt_engine_op.h (88%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/ops/get_serialized_resource_op.cc (100%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/ops/trt_engine_op.cc (100%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/plugin/trt_plugin.cc (96%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/plugin/trt_plugin.h (92%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/plugin/trt_plugin_factory.cc (96%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/plugin/trt_plugin_factory.h (91%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/plugin/trt_plugin_factory_test.cc (96%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/plugin/trt_plugin_utils.cc (94%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/plugin/trt_plugin_utils.h (82%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/python/ops/trt_ops.py (88%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/segment/segment.cc (99%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/segment/segment.h (92%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/segment/segment_test.cc (99%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/segment/union_find.h (92%)
 rename tensorflow/{contrib/tensorrt => compiler/tf2tensorrt}/tensorrt_test.cc (100%)
 rename tensorflow/{contrib/tensorrt/test/utils.cc => compiler/tf2tensorrt/utils/test_utils.cc} (97%)
 rename tensorflow/{contrib/tensorrt/test/utils.h => compiler/tf2tensorrt/utils/test_utils.h} (89%)
 rename tensorflow/{contrib/tensorrt/resources => compiler/tf2tensorrt/utils}/trt_allocator.cc (98%)
 rename tensorflow/{contrib/tensorrt/resources => compiler/tf2tensorrt/utils}/trt_allocator.h (93%)
 rename tensorflow/{contrib/tensorrt/resources => compiler/tf2tensorrt/utils}/trt_allocator_test.cc (98%)
 rename tensorflow/{contrib/tensorrt/resources => compiler/tf2tensorrt/utils}/trt_int8_calibrator.cc (98%)
 rename tensorflow/{contrib/tensorrt/resources => compiler/tf2tensorrt/utils}/trt_int8_calibrator.h (93%)
 rename tensorflow/{contrib/tensorrt/log => compiler/tf2tensorrt/utils}/trt_logger.cc (96%)
 rename tensorflow/{contrib/tensorrt/log => compiler/tf2tensorrt/utils}/trt_logger.h (86%)
 rename tensorflow/{contrib/tensorrt/resources => compiler/tf2tensorrt/utils}/trt_lru_cache.h (94%)
 rename tensorflow/{contrib/tensorrt/resources => compiler/tf2tensorrt/utils}/trt_lru_cache_test.cc (96%)
 rename tensorflow/{contrib/tensorrt/resources => compiler/tf2tensorrt/utils}/trt_resource_manager.cc (96%)
 rename tensorflow/{contrib/tensorrt/resources => compiler/tf2tensorrt/utils}/trt_resource_manager.h (87%)
 rename tensorflow/{contrib/tensorrt/resources => compiler/tf2tensorrt/utils}/trt_resources.cc (96%)
 rename tensorflow/{contrib/tensorrt/resources => compiler/tf2tensorrt/utils}/trt_resources.h (82%)

diff --git a/tensorflow/compiler/tf2tensorrt/BUILD b/tensorflow/compiler/tf2tensorrt/BUILD
new file mode 100644
index 0000000000..a67e511826
--- /dev/null
+++ b/tensorflow/compiler/tf2tensorrt/BUILD
@@ -0,0 +1,440 @@
+# Description:
+#   Wrap NVIDIA TensorRT (http://developer.nvidia.com/tensorrt) with tensorflow
+#   and provide TensorRT operators and converter package.
+#   APIs are meant to change over time.
+
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])  # Apache 2.0
+
+exports_files(["LICENSE"])
+
+load(
+    "//tensorflow:tensorflow.bzl",
+    "tf_cc_test",
+    "tf_copts",
+    "tf_cuda_library",
+    "tf_custom_op_library",
+    "tf_custom_op_library_additional_deps",
+    "tf_gen_op_libs",
+    "tf_gen_op_wrapper_py",
+)
+load("//tensorflow:tensorflow.bzl", "tf_cuda_cc_test")
+load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library")
+load(
+    "@local_config_tensorrt//:build_defs.bzl",
+    "if_tensorrt",
+)
+
+tf_cuda_cc_test(
+    name = "tensorrt_test_cc",
+    size = "small",
+    srcs = ["tensorrt_test.cc"],
+    tags = [
+        "no_windows",
+        "nomac",
+    ],
+    deps = [
+        "//tensorflow/core:gpu_init",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:stream_executor",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ] + if_tensorrt([
+        "@local_config_cuda//cuda:cuda_headers",
+        "@local_config_tensorrt//:tensorrt",
+    ]),
+)
+
+tf_custom_op_library(
+    name = "python/ops/_trt_ops.so",
+    srcs = [
+        "ops/get_serialized_resource_op.cc",
+        "ops/trt_engine_op.cc",
+    ],
+    deps = [
+        "//tensorflow/core:lib_proto_parsing",
+    ] + if_tensorrt([
+        "@local_config_tensorrt//:tensorrt",
+    ]),
+)
+
+cc_library(
+    name = "trt_op_kernels",
+    srcs = [
+        "kernels/get_serialized_resource_op.cc",
+        "kernels/trt_engine_op.cc",
+    ],
+    hdrs = [
+        "kernels/trt_engine_op.h",
+    ],
+    copts = tf_copts(),
+    visibility = ["//visibility:public"],
+    deps = [
+        ":test_utils",
+        ":trt_allocator",
+        ":trt_conversion",
+        ":trt_logging",
+        ":trt_plugins",
+        ":trt_resources",
+        ":utils",
+        "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/strings",
+        "//tensorflow/core:gpu_headers_lib",
+        "//tensorflow/core:lib_proto_parsing",
+        "//tensorflow/core:stream_executor_headers_lib",
+        "//tensorflow/core/grappler/costs:graph_properties",
+    ] + if_tensorrt([
+        "@local_config_tensorrt//:tensorrt",
+    ]) + tf_custom_op_library_additional_deps(),
+    # TODO(laigd): fix this by merging header file in cc file.
+    alwayslink = 1,  # buildozer: disable=alwayslink-with-hdrs
+)
+
+tf_cuda_cc_test(
+    name = "get_serialized_resource_op_test",
+    size = "small",
+    srcs = ["kernels/get_serialized_resource_op_test.cc"],
+    tags = [
+        "no_cuda_on_cpu_tap",
+        "no_windows",
+        "nomac",
+    ],
+    deps = [
+        ":get_serialized_resource_op_op_lib",
+        ":trt_op_kernels",
+        ":trt_resources",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
+        "//tensorflow/core/kernels:ops_testutil",
+    ],
+)
+
+tf_gen_op_libs(
+    op_lib_names = [
+        "trt_engine_op",
+        "get_serialized_resource_op",
+    ],
+)
+
+tf_cuda_library(
+    name = "trt_logging",
+    srcs = ["utils/trt_logger.cc"],
+    hdrs = ["utils/trt_logger.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//tensorflow/core:lib_proto_parsing",
+    ] + if_tensorrt([
+        "@local_config_tensorrt//:tensorrt",
+    ]),
+)
+
+tf_gen_op_wrapper_py(
+    name = "trt_ops",
+    deps = [
+        ":get_serialized_resource_op_op_lib",
+        ":trt_engine_op_op_lib",
+        ":trt_logging",
+    ],
+)
+
+tf_custom_op_py_library(
+    name = "trt_ops_loader",
+    srcs = ["python/ops/trt_ops.py"],
+    dso = [
+        "python/ops/_trt_ops.so",
+    ] + if_tensorrt([
+        "@local_config_tensorrt//:tensorrt",
+    ]),
+    kernels = [
+        ":trt_op_kernels",
+        ":trt_engine_op_op_lib",
+        ":get_serialized_resource_op_op_lib",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:platform",
+        "//tensorflow/python:resources",
+    ],
+)
+
+tf_cuda_library(
+    name = "trt_resources",
+    srcs = [
+        "utils/trt_int8_calibrator.cc",
+        "utils/trt_resource_manager.cc",
+        "utils/trt_resources.cc",
+    ],
+    hdrs = [
+        "utils/trt_int8_calibrator.h",
+        "utils/trt_lru_cache.h",
+        "utils/trt_resource_manager.h",
+        "utils/trt_resources.h",
+    ],
+    deps = [
+        ":trt_allocator",
+        ":trt_logging",
+        ":utils",
+        "//tensorflow/core:framework_headers_lib",
+        "//tensorflow/core:framework_lite",
+        "//tensorflow/core:lib_proto_parsing",
+    ] + if_tensorrt([
+        "@local_config_tensorrt//:tensorrt",
+    ]),
+)
+
+tf_cuda_library(
+    name = "trt_allocator",
+    srcs = ["utils/trt_allocator.cc"],
+    hdrs = ["utils/trt_allocator.h"],
+    deps = [
+        "//tensorflow/core:framework_headers_lib",
+        "//tensorflow/core:framework_lite",
+        "//tensorflow/core:lib_proto_parsing",
+    ] + if_tensorrt([
+        "@local_config_tensorrt//:tensorrt",
+    ]),
+)
+
+tf_cc_test(
+    name = "trt_allocator_test",
+    size = "small",
+    srcs = ["utils/trt_allocator_test.cc"],
+    tags = [
+        "no_windows",
+        "nomac",
+    ],
+    deps = [
+        ":trt_allocator",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ],
+)
+
+tf_cc_test(
+    name = "trt_lru_cache_test",
+    size = "small",
+    srcs = ["utils/trt_lru_cache_test.cc"],
+    tags = [
+        "no_windows",
+        "nomac",
+    ],
+    deps = [
+        ":trt_resources",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ],
+)
+
+# Library for the node-level conversion portion of TensorRT operation creation
+tf_cuda_library(
+    name = "trt_conversion",
+    srcs = [
+        "convert/convert_graph.cc",
+        "convert/convert_nodes.cc",
+        "convert/trt_optimization_pass.cc",
+    ],
+    hdrs = [
+        "convert/convert_graph.h",
+        "convert/convert_nodes.h",
+        "convert/trt_optimization_pass.h",
+    ],
+    deps = [
+        ":segment",
+        ":test_utils",
+        ":trt_allocator",
+        ":trt_plugins",
+        ":trt_logging",
+        ":trt_resources",
+        ":utils",
+        "@com_google_absl//absl/strings",
+        "//tensorflow/core/grappler/clusters:cluster",
+        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
+        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
+        "//tensorflow/core/grappler:grappler_item",
+        "//tensorflow/core/grappler:utils",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:framework_lite",
+        "//tensorflow/core:gpu_runtime",
+        "//tensorflow/core:graph",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core/grappler:devices",
+        "//tensorflow/core/grappler/clusters:virtual_cluster",
+        "//tensorflow/core/grappler/costs:graph_properties",
+        "//tensorflow/core/grappler/optimizers:meta_optimizer",
+    ] + if_tensorrt([
+        "@local_config_tensorrt//:tensorrt",
+    ]) + tf_custom_op_library_additional_deps(),
+)
+
+tf_cuda_cc_test(
+    name = "convert_graph_test",
+    size = "medium",
+    srcs = ["convert/convert_graph_test.cc"],
+    tags = [
+        "no_cuda_on_cpu_tap",
+        "no_windows",
+        "nomac",
+    ],
+    deps = [
+        ":trt_conversion",
+        "@com_google_googletest//:gtest",
+        "@com_google_absl//absl/strings",
+        "//tensorflow/cc:cc_ops",
+        "//tensorflow/cc:scope",
+        "//tensorflow/core/grappler:grappler_item",
+        "//tensorflow/core/grappler/clusters:cluster",
+        "//tensorflow/core:core_cpu",
+        "//tensorflow/core:core_cpu_base",
+        "//tensorflow/core:direct_session",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
+    ] + if_tensorrt([
+        "@local_config_tensorrt//:tensorrt",
+    ]),
+)
+
+tf_cuda_cc_test(
+    name = "convert_nodes_test",
+    size = "medium",
+    srcs = ["convert/convert_nodes_test.cc"],
+    tags = [
+        "no_cuda_on_cpu_tap",
+        "no_windows",
+        "nomac",
+    ],
+    deps = [
+        ":trt_logging",
+        ":trt_conversion",
+        ":trt_plugins",
+        "@com_google_googletest//:gtest",
+        "@com_google_absl//absl/strings",
+        "//tensorflow/cc:cc_ops",
+        "//tensorflow/cc:ops",
+        "//tensorflow/cc:scope",
+        "//tensorflow/core/grappler/costs:graph_properties",
+        "//tensorflow/core:core_cpu",
+        "//tensorflow/core:core_cpu_base",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
+    ] + if_tensorrt([
+        "@local_config_cuda//cuda:cuda_headers",
+        "@local_config_tensorrt//:tensorrt",
+    ]),
+)
+
+# Library for the segmenting portion of TensorRT operation creation
+cc_library(
+    name = "segment",
+    srcs = ["segment/segment.cc"],
+    hdrs = [
+        "segment/segment.h",
+        "segment/union_find.h",
+    ],
+    deps = [
+        "//tensorflow/core:graph",
+        "//tensorflow/core:lib_proto_parsing",
+        "//tensorflow/core:protos_all_cc",
+        "@com_google_absl//absl/strings",
+        "@protobuf_archive//:protobuf_headers",
+    ],
+)
+
+tf_cc_test(
+    name = "segment_test",
+    size = "small",
+    srcs = ["segment/segment_test.cc"],
+    tags = [
+        "no_windows",
+        "nomac",
+    ],
+    deps = [
+        ":segment",
+        "//tensorflow/cc:cc_ops",
+        "//tensorflow/cc:scope",
+        "//tensorflow/core:core_cpu",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:ops",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
+    ],
+)
+
+# Library for the plugin factory
+tf_cuda_library(
+    name = "trt_plugins",
+    srcs = [
+        "plugin/trt_plugin.cc",
+        "plugin/trt_plugin_factory.cc",
+        "plugin/trt_plugin_utils.cc",
+    ],
+    hdrs = [
+        "plugin/trt_plugin.h",
+        "plugin/trt_plugin_factory.h",
+        "plugin/trt_plugin_utils.h",
+    ],
+    deps = [
+        "//tensorflow/core:framework_lite",
+        "//tensorflow/core:lib_proto_parsing",
+    ] + if_tensorrt([
+        "@local_config_tensorrt//:tensorrt",
+    ]),
+)
+
+tf_cuda_cc_test(
+    name = "trt_plugin_factory_test",
+    size = "small",
+    srcs = ["plugin/trt_plugin_factory_test.cc"],
+    tags = [
+        "no_cuda_on_cpu_tap",
+        "no_windows",
+        "nomac",
+    ],
+    deps = [
+        ":trt_plugins",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ] + if_tensorrt([
+        "@local_config_cuda//cuda:cuda_headers",
+        "@local_config_tensorrt//:tensorrt",
+    ]),
+)
+
+cc_library(
+    name = "utils",
+    srcs = ["convert/utils.cc"],
+    hdrs = ["convert/utils.h"],
+    copts = tf_copts(),
+    deps = [
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+    ],
+)
+
+cc_library(
+    name = "test_utils",
+    srcs = ["utils/test_utils.cc"],
+    hdrs = ["utils/test_utils.h"],
+    deps = [
+        "//tensorflow/core:lib",
+        "@com_googlesource_code_re2//:re2",
+    ],
+)
diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_graph.cc
similarity index 98%
rename from tensorflow/contrib/tensorrt/convert/convert_graph.cc
rename to tensorflow/compiler/tf2tensorrt/convert/convert_graph.cc
index 15cf97a2ee..1fdb099cc1 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_graph.cc
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/contrib/tensorrt/convert/convert_graph.h"
+#include "tensorflow/compiler/tf2tensorrt/convert/convert_graph.h"
 
 #include <fstream>
 #include <list>
@@ -24,13 +24,14 @@ limitations under the License.
 #include <utility>
 #include <vector>
 
-#include "tensorflow/contrib/tensorrt/convert/convert_nodes.h"
-#include "tensorflow/contrib/tensorrt/convert/utils.h"
-#include "tensorflow/contrib/tensorrt/plugin/trt_plugin_factory.h"
-#include "tensorflow/contrib/tensorrt/resources/trt_resource_manager.h"
-#include "tensorflow/contrib/tensorrt/resources/trt_resources.h"
-#include "tensorflow/contrib/tensorrt/segment/segment.h"
-#include "tensorflow/contrib/tensorrt/test/utils.h"
+#include "absl/strings/str_cat.h"
+#include "tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h"
+#include "tensorflow/compiler/tf2tensorrt/convert/utils.h"
+#include "tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_factory.h"
+#include "tensorflow/compiler/tf2tensorrt/segment/segment.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/test_utils.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_resource_manager.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_resources.h"
 #include "tensorflow/core/common_runtime/gpu/gpu_id.h"
 #include "tensorflow/core/common_runtime/gpu/gpu_id_manager.h"
 #include "tensorflow/core/common_runtime/gpu/gpu_process_state.h"
@@ -63,8 +64,8 @@ limitations under the License.
 namespace tensorflow {
 namespace tensorrt {
 namespace convert {
-using ::tensorflow::strings::StrAppend;
-using ::tensorflow::strings::StrCat;
+using absl::StrAppend;
+using absl::StrCat;
 
 // Returns compiled TRT version information {Maj, Min, Patch}
 std::vector<int> GetLinkedTensorRTVersion() {
@@ -151,7 +152,7 @@ Status TrtCandidateSelector::IsTensorRTCandidate(const tensorflow::Node* node) {
   if (precision_mode_ == INT8MODE && quantize_ops.count(node->type_string())) {
     is_supported_op_type = true;
   }
-  // LINT.ThenChange(//tensorflow/contrib/tensorrt/convert/convert_nodes.cc)
+  // LINT.ThenChange(//tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc)
   if (!is_supported_op_type) {
     return errors::Unimplemented("Op type ", node->type_string(),
                                  " is not supported");
diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.h b/tensorflow/compiler/tf2tensorrt/convert/convert_graph.h
similarity index 94%
rename from tensorflow/contrib/tensorrt/convert/convert_graph.h
rename to tensorflow/compiler/tf2tensorrt/convert/convert_graph.h
index 1f39f56f63..fb82a430c6 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.h
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_graph.h
@@ -12,12 +12,12 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#ifndef TENSORFLOW_CONTRIB_TENSORRT_CONVERT_CONVERT_GRAPH_H_
-#define TENSORFLOW_CONTRIB_TENSORRT_CONVERT_CONVERT_GRAPH_H_
+#ifndef TENSORFLOW_COMPILER_TF2TENSORRT_CONVERT_CONVERT_GRAPH_H_
+#define TENSORFLOW_COMPILER_TF2TENSORRT_CONVERT_CONVERT_GRAPH_H_
 
 #include <vector>
 
-#include "tensorflow/contrib/tensorrt/convert/convert_nodes.h"
+#include "tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h"
 #include "tensorflow/core/framework/graph.pb.h"
 #include "tensorflow/core/grappler/clusters/cluster.h"
 #include "tensorflow/core/grappler/costs/graph_properties.h"
@@ -123,4 +123,4 @@ std::pair<int, tensorflow::Allocator*> GetDeviceAndAllocator(
 #endif  // GOOGLE_TENSORRT
 #endif  // GOOGLE_CUDA
 
-#endif  // TENSORFLOW_CONTRIB_TENSORRT_CONVERT_CONVERT_GRAPH_H_
+#endif  // TENSORFLOW_COMPILER_TF2TENSORRT_CONVERT_CONVERT_GRAPH_H_
diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph_test.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_graph_test.cc
similarity index 98%
rename from tensorflow/contrib/tensorrt/convert/convert_graph_test.cc
rename to tensorflow/compiler/tf2tensorrt/convert/convert_graph_test.cc
index 2d2bfeb192..a3c3a8ac65 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph_test.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_graph_test.cc
@@ -13,13 +13,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/contrib/tensorrt/convert/convert_graph.h"
+#include "tensorflow/compiler/tf2tensorrt/convert/convert_graph.h"
 
 #include <gmock/gmock.h>
 #include <gtest/gtest.h>
 #include "tensorflow/cc/framework/scope.h"
 #include "tensorflow/cc/ops/standard_ops.h"
-#include "tensorflow/contrib/tensorrt/convert/convert_nodes.h"
+#include "tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h"
 #include "tensorflow/core/common_runtime/device_mgr.h"
 #include "tensorflow/core/common_runtime/device_set.h"
 #include "tensorflow/core/framework/tensor_shape.h"
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
similarity index 99%
rename from tensorflow/contrib/tensorrt/convert/convert_nodes.cc
rename to tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
index 59d295f2e1..c08582a42e 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/contrib/tensorrt/convert/convert_nodes.h"
+#include "tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h"
 
 #include <algorithm>
 #include <cstring>
@@ -24,11 +24,12 @@ limitations under the License.
 #include <utility>
 #include <vector>
 
-#include "tensorflow/contrib/tensorrt/convert/utils.h"
-#include "tensorflow/contrib/tensorrt/log/trt_logger.h"
-#include "tensorflow/contrib/tensorrt/plugin/trt_plugin_factory.h"
-#include "tensorflow/contrib/tensorrt/resources/trt_resource_manager.h"
-#include "tensorflow/contrib/tensorrt/resources/trt_resources.h"
+#include "absl/strings/str_cat.h"
+#include "tensorflow/compiler/tf2tensorrt/convert/utils.h"
+#include "tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_factory.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_logger.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_resource_manager.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_resources.h"
 #include "tensorflow/core/framework/node_def.pb.h"  // NOLINT
 #include "tensorflow/core/framework/node_def_builder.h"
 #include "tensorflow/core/framework/tensor.pb.h"        // NOLINT
@@ -43,6 +44,7 @@ limitations under the License.
 #include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/logging.h"
+#include "tensorflow/core/platform/protobuf.h"
 #include "tensorflow/core/platform/tensor_coding.h"
 #include "tensorflow/core/platform/types.h"
 
@@ -81,9 +83,9 @@ const char* const kInputPHName = "TensorRTInputPH_";
 const char* const kOutputPHName = "TensorRTOutputPH_";
 
 namespace convert {
+using absl::StrAppend;
+using absl::StrCat;
 using ::tensorflow::str_util::Split;
-using ::tensorflow::strings::StrAppend;
-using ::tensorflow::strings::StrCat;
 
 inline tensorflow::Status ConvertDType(tensorflow::DataType tf_dtype,
                                        nvinfer1::DataType* trt_dtype) {
@@ -3692,7 +3694,7 @@ tensorflow::Status ConvertGraphDefToEngine(
     if (tensorflow::str_util::StartsWith(node_name, kInputPHName) &&
         (node_def.op() == "Placeholder")) {
       int32 slot_number = -1;
-      if (!tensorflow::strings::safe_strto32(
+      if (!tensorflow::strings::safe_strto32(  // non-absl ok
               node_name.c_str() + strlen(kInputPHName), &slot_number)) {
         return tensorflow::errors::InvalidArgument(
             "Failed to parse slot number from ", node_name);
@@ -3721,7 +3723,7 @@ tensorflow::Status ConvertGraphDefToEngine(
     } else if (tensorflow::str_util::StartsWith(node_name, kOutputPHName) &&
                (node_def.op() == "Identity")) {
       int32 slot_number = -1;
-      if (!tensorflow::strings::safe_strto32(
+      if (!tensorflow::strings::safe_strto32(  // non-absl ok
               node_name.c_str() + strlen(kOutputPHName), &slot_number)) {
         return tensorflow::errors::InvalidArgument(
             "Failed to parse slot number from ", node_name);
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.h b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h
similarity index 97%
rename from tensorflow/contrib/tensorrt/convert/convert_nodes.h
rename to tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h
index 8495a1a19a..aebc0ca38d 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.h
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h
@@ -13,8 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_CONTRIB_TENSORRT_CONVERT_CONVERT_NODES_H_
-#define TENSORFLOW_CONTRIB_TENSORRT_CONVERT_CONVERT_NODES_H_
+#ifndef TENSORFLOW_COMPILER_TF2TENSORRT_CONVERT_CONVERT_NODES_H_
+#define TENSORFLOW_COMPILER_TF2TENSORRT_CONVERT_CONVERT_NODES_H_
 
 #include <set>
 #include <string>
@@ -22,11 +22,11 @@ limitations under the License.
 #include <utility>
 #include <vector>
 
-#include "tensorflow/contrib/tensorrt/convert/utils.h"
-#include "tensorflow/contrib/tensorrt/log/trt_logger.h"
-#include "tensorflow/contrib/tensorrt/resources/trt_allocator.h"
-#include "tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.h"
-#include "tensorflow/contrib/tensorrt/resources/trt_resources.h"
+#include "tensorflow/compiler/tf2tensorrt/convert/utils.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_allocator.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_int8_calibrator.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_logger.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_resources.h"
 #include "tensorflow/core/framework/graph.pb.h"
 #include "tensorflow/core/graph/graph.h"
 #include "tensorflow/core/grappler/costs/graph_properties.h"
@@ -555,4 +555,4 @@ class Converter {
 #endif  // GOOGLE_TENSORRT
 #endif  // GOOGLE_CUDA
 
-#endif  // TENSORFLOW_CONTRIB_TENSORRT_CONVERT_CONVERT_NODES_H_
+#endif  // TENSORFLOW_COMPILER_TF2TENSORRT_CONVERT_CONVERT_NODES_H_
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
similarity index 99%
rename from tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
rename to tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
index 3bd4ff94d1..3a70423d12 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes_test.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/contrib/tensorrt/convert/convert_nodes.h"
+#include "tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h"
 
 #include <memory>
 #include <unordered_map>
@@ -21,11 +21,12 @@ limitations under the License.
 
 #include <gmock/gmock.h>
 #include <gtest/gtest.h>
+#include "absl/strings/str_cat.h"
 #include "tensorflow/cc/framework/ops.h"
 #include "tensorflow/cc/framework/scope.h"
 #include "tensorflow/cc/ops/standard_ops.h"
-#include "tensorflow/contrib/tensorrt/log/trt_logger.h"
-#include "tensorflow/contrib/tensorrt/plugin/trt_plugin_factory.h"
+#include "tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_factory.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_logger.h"
 #include "tensorflow/core/framework/node_def.pb.h"  // NOLINT
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor.pb.h"  // NOLINT
@@ -51,7 +52,7 @@ namespace tensorflow {
 namespace tensorrt {
 namespace convert {
 
-using ::tensorflow::strings::StrCat;
+using absl::StrCat;
 using ::testing::ElementsAre;
 using ::testing::ElementsAreArray;
 
@@ -2316,7 +2317,7 @@ TEST_F(OpConverterTest, ConvertSqueeze) {
     Scope s = Scope::NewRootScope();
     auto input = ops::Placeholder(s.WithOpName("input"), DT_FLOAT);
     ops::Squeeze::Attrs squeeze_attrs;
-    squeeze_attrs.axis_ = gtl::ArraySlice<int>(axis);
+    squeeze_attrs.axis_ = gtl::ArraySlice<int>(axis);  // non-absl ok
     auto squeeze =
         ops::Squeeze(s.WithOpName("my_squeeze"), input, squeeze_attrs);
     return squeeze.operation.node()->def();
diff --git a/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc b/tensorflow/compiler/tf2tensorrt/convert/trt_optimization_pass.cc
similarity index 96%
rename from tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc
rename to tensorflow/compiler/tf2tensorrt/convert/trt_optimization_pass.cc
index d57f2300f8..ebf8df1349 100644
--- a/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/trt_optimization_pass.cc
@@ -12,9 +12,11 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/contrib/tensorrt/convert/trt_optimization_pass.h"
-#include "tensorflow/contrib/tensorrt/convert/convert_graph.h"
-#include "tensorflow/contrib/tensorrt/convert/utils.h"
+#include "tensorflow/compiler/tf2tensorrt/convert/trt_optimization_pass.h"
+
+#include "absl/strings/str_cat.h"
+#include "tensorflow/compiler/tf2tensorrt/convert/convert_graph.h"
+#include "tensorflow/compiler/tf2tensorrt/convert/utils.h"
 #include "tensorflow/core/grappler/clusters/cluster.h"
 #include "tensorflow/core/grappler/grappler_item.h"
 #include "tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.h"
@@ -30,9 +32,9 @@ namespace tensorflow {
 namespace tensorrt {
 namespace convert {
 // TODO(sami): Remove VLOG messages once the code matures
+using absl::StrAppend;
+using absl::StrCat;
 using tensorflow::str_util::Uppercase;
-using tensorflow::strings::StrAppend;
-using tensorflow::strings::StrCat;
 
 tensorflow::Status TRTOptimizationPass::Init(
     const tensorflow::RewriterConfig_CustomGraphOptimizer* config) {
@@ -243,7 +245,7 @@ tensorflow::Status TRTOptimizationPass::Optimize(
     // If the last token is not an integer, it must be part of the name.
     // Otherwise it is port number.
     if (tokens.size() > 1 &&
-        !strings::safe_strto32(tokens.back(), &dumm_port)) {
+        !strings::safe_strto32(tokens.back(), &dumm_port)) {  // non-absl ok
       StrAppend(&s, ":", tokens.back());
     }
     nodes_to_preserve.push_back(s);
diff --git a/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.h b/tensorflow/compiler/tf2tensorrt/convert/trt_optimization_pass.h
similarity index 91%
rename from tensorflow/contrib/tensorrt/convert/trt_optimization_pass.h
rename to tensorflow/compiler/tf2tensorrt/convert/trt_optimization_pass.h
index 3e8dc0978e..bd6c6dbce1 100644
--- a/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.h
+++ b/tensorflow/compiler/tf2tensorrt/convert/trt_optimization_pass.h
@@ -13,8 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_CONTRIB_TENSORRT_CONVERT_TRT_OPTIMIZATION_PASS_H_
-#define TENSORFLOW_CONTRIB_TENSORRT_CONVERT_TRT_OPTIMIZATION_PASS_H_
+#ifndef TENSORFLOW_COMPILER_TF2TENSORRT_CONVERT_TRT_OPTIMIZATION_PASS_H_
+#define TENSORFLOW_COMPILER_TF2TENSORRT_CONVERT_TRT_OPTIMIZATION_PASS_H_
 
 #include <string>
 
@@ -77,4 +77,4 @@ class TRTOptimizationPass : public tensorflow::grappler::CustomGraphOptimizer {
 
 #endif  // GOOGLE_CUDA
 #endif  // GOOGLE_TENSORRT
-#endif  // TENSORFLOW_CONTRIB_TENSORRT_CONVERT_TRT_OPTIMIZATION_PASS_H_
+#endif  // TENSORFLOW_COMPILER_TF2TENSORRT_CONVERT_TRT_OPTIMIZATION_PASS_H_
diff --git a/tensorflow/contrib/tensorrt/convert/utils.cc b/tensorflow/compiler/tf2tensorrt/convert/utils.cc
similarity index 97%
rename from tensorflow/contrib/tensorrt/convert/utils.cc
rename to tensorflow/compiler/tf2tensorrt/convert/utils.cc
index e7a1febb8c..62a0f62ad6 100644
--- a/tensorflow/contrib/tensorrt/convert/utils.cc
+++ b/tensorflow/compiler/tf2tensorrt/convert/utils.cc
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/contrib/tensorrt/convert/utils.h"
+#include "tensorflow/compiler/tf2tensorrt/convert/utils.h"
 
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status.h"
diff --git a/tensorflow/contrib/tensorrt/convert/utils.h b/tensorflow/compiler/tf2tensorrt/convert/utils.h
similarity index 88%
rename from tensorflow/contrib/tensorrt/convert/utils.h
rename to tensorflow/compiler/tf2tensorrt/convert/utils.h
index 0592f31462..9f9ee59087 100644
--- a/tensorflow/contrib/tensorrt/convert/utils.h
+++ b/tensorflow/compiler/tf2tensorrt/convert/utils.h
@@ -13,8 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_CONTRIB_TENSORRT_CONVERT_UTILS_H_
-#define TENSORFLOW_CONTRIB_TENSORRT_CONVERT_UTILS_H_
+#ifndef TENSORFLOW_COMPILER_TF2TENSORRT_CONVERT_UTILS_H_
+#define TENSORFLOW_COMPILER_TF2TENSORRT_CONVERT_UTILS_H_
 
 #include <memory>
 
@@ -47,4 +47,4 @@ Status GetPrecisionMode(const string& name, int* precision_mode);
 }  // namespace tensorrt
 }  // namespace tensorflow
 
-#endif  // TENSORFLOW_CONTRIB_TENSORRT_CONVERT_UTILS_H_
+#endif  // TENSORFLOW_COMPILER_TF2TENSORRT_CONVERT_UTILS_H_
diff --git a/tensorflow/contrib/tensorrt/kernels/get_serialized_resource_op.cc b/tensorflow/compiler/tf2tensorrt/kernels/get_serialized_resource_op.cc
similarity index 88%
rename from tensorflow/contrib/tensorrt/kernels/get_serialized_resource_op.cc
rename to tensorflow/compiler/tf2tensorrt/kernels/get_serialized_resource_op.cc
index f68bc2b485..eae1f8e752 100644
--- a/tensorflow/contrib/tensorrt/kernels/get_serialized_resource_op.cc
+++ b/tensorflow/compiler/tf2tensorrt/kernels/get_serialized_resource_op.cc
@@ -13,13 +13,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_CONTRIB_TENSORRT_KERNELS_GET_SERIALIZED_RESOURCE_OP_H_
-#define TENSORFLOW_CONTRIB_TENSORRT_KERNELS_GET_SERIALIZED_RESOURCE_OP_H_
+#ifndef TENSORFLOW_COMPILER_TF2TENSORRT_KERNELS_GET_SERIALIZED_RESOURCE_OP_H_
+#define TENSORFLOW_COMPILER_TF2TENSORRT_KERNELS_GET_SERIALIZED_RESOURCE_OP_H_
 
 #include <memory>
 #include <vector>
 
-#include "tensorflow/contrib/tensorrt/resources/trt_resources.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_resources.h"
 #include "tensorflow/core/framework/op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/resource_mgr.h"
@@ -70,4 +70,4 @@ REGISTER_KERNEL_BUILDER(Name("GetSerializedResourceOp").Device(DEVICE_GPU),
 
 #endif  // GOOGLE_TENSORRT
 #endif  // GOOGLE_CUDA
-#endif  // TENSORFLOW_CONTRIB_TENSORRT_KERNELS_GET_SERIALIZED_RESOURCE_OP_H_
+#endif  // TENSORFLOW_COMPILER_TF2TENSORRT_KERNELS_GET_SERIALIZED_RESOURCE_OP_H_
diff --git a/tensorflow/contrib/tensorrt/kernels/get_serialized_resource_op_test.cc b/tensorflow/compiler/tf2tensorrt/kernels/get_serialized_resource_op_test.cc
similarity index 97%
rename from tensorflow/contrib/tensorrt/kernels/get_serialized_resource_op_test.cc
rename to tensorflow/compiler/tf2tensorrt/kernels/get_serialized_resource_op_test.cc
index a91228e4c4..ec038ebda0 100644
--- a/tensorflow/contrib/tensorrt/kernels/get_serialized_resource_op_test.cc
+++ b/tensorflow/compiler/tf2tensorrt/kernels/get_serialized_resource_op_test.cc
@@ -18,7 +18,7 @@ limitations under the License.
 #include <fstream>
 #include <vector>
 
-#include "tensorflow/contrib/tensorrt/resources/trt_resources.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_resources.h"
 #include "tensorflow/core/framework/fake_input.h"
 #include "tensorflow/core/framework/node_def_builder.h"
 #include "tensorflow/core/framework/tensor.h"
diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc b/tensorflow/compiler/tf2tensorrt/kernels/trt_engine_op.cc
similarity index 97%
rename from tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
rename to tensorflow/compiler/tf2tensorrt/kernels/trt_engine_op.cc
index d28d88fa3a..198d68b609 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
+++ b/tensorflow/compiler/tf2tensorrt/kernels/trt_engine_op.cc
@@ -12,18 +12,19 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#include "tensorflow/contrib/tensorrt/kernels/trt_engine_op.h"
+#include "tensorflow/compiler/tf2tensorrt/kernels/trt_engine_op.h"
 
 #include <algorithm>
 
 #include "absl/memory/memory.h"
-#include "tensorflow/contrib/tensorrt/convert/convert_nodes.h"
-#include "tensorflow/contrib/tensorrt/convert/utils.h"
-#include "tensorflow/contrib/tensorrt/log/trt_logger.h"
-#include "tensorflow/contrib/tensorrt/plugin/trt_plugin_factory.h"
-#include "tensorflow/contrib/tensorrt/resources/trt_resource_manager.h"
-#include "tensorflow/contrib/tensorrt/resources/trt_resources.h"
-#include "tensorflow/contrib/tensorrt/test/utils.h"
+#include "absl/strings/str_cat.h"
+#include "tensorflow/compiler/tf2tensorrt/convert/convert_nodes.h"
+#include "tensorflow/compiler/tf2tensorrt/convert/utils.h"
+#include "tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_factory.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/test_utils.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_logger.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_resource_manager.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_resources.h"
 #include "tensorflow/core/framework/graph_to_functiondef.h"
 #include "tensorflow/core/lib/core/refcount.h"
 #include "tensorflow/core/lib/strings/str_util.h"
@@ -39,9 +40,9 @@ limitations under the License.
 namespace tensorflow {
 namespace tensorrt {
 static Logger logger;
+using absl::StrAppend;
+using absl::StrCat;
 using ::nvinfer1::IRuntime;
-using ::tensorflow::strings::StrAppend;
-using ::tensorflow::strings::StrCat;
 
 // A helper class to call done() when destructed for asynchronous execution.
 // Helps simultaneous execution of native and TRT engines.
diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h b/tensorflow/compiler/tf2tensorrt/kernels/trt_engine_op.h
similarity index 88%
rename from tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
rename to tensorflow/compiler/tf2tensorrt/kernels/trt_engine_op.h
index 76c6020078..64f8c97a74 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
+++ b/tensorflow/compiler/tf2tensorrt/kernels/trt_engine_op.h
@@ -13,17 +13,17 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_CONTRIB_TENSORRT_KERNELS_TRT_ENGINE_OP_H_
-#define TENSORFLOW_CONTRIB_TENSORRT_KERNELS_TRT_ENGINE_OP_H_
+#ifndef TENSORFLOW_COMPILER_TF2TENSORRT_KERNELS_TRT_ENGINE_OP_H_
+#define TENSORFLOW_COMPILER_TF2TENSORRT_KERNELS_TRT_ENGINE_OP_H_
 
 #include <memory>
 #include <vector>
 
-#include "tensorflow/contrib/tensorrt/convert/utils.h"
-#include "tensorflow/contrib/tensorrt/log/trt_logger.h"
-#include "tensorflow/contrib/tensorrt/resources/trt_allocator.h"
-#include "tensorflow/contrib/tensorrt/resources/trt_lru_cache.h"
-#include "tensorflow/contrib/tensorrt/resources/trt_resources.h"
+#include "tensorflow/compiler/tf2tensorrt/convert/utils.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_allocator.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_logger.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_lru_cache.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_resources.h"
 #include "tensorflow/core/framework/function.h"
 #include "tensorflow/core/framework/graph.pb.h"
 #include "tensorflow/core/framework/op.h"
@@ -127,4 +127,4 @@ class TRTEngineOp : public AsyncOpKernel {
 #endif  // GOOGLE_TENSORRT
 #endif  // GOOGLE_CUDA
 
-#endif  // TENSORFLOW_CONTRIB_TENSORRT_KERNELS_TRT_ENGINE_OP_H_
+#endif  // TENSORFLOW_COMPILER_TF2TENSORRT_KERNELS_TRT_ENGINE_OP_H_
diff --git a/tensorflow/contrib/tensorrt/ops/get_serialized_resource_op.cc b/tensorflow/compiler/tf2tensorrt/ops/get_serialized_resource_op.cc
similarity index 100%
rename from tensorflow/contrib/tensorrt/ops/get_serialized_resource_op.cc
rename to tensorflow/compiler/tf2tensorrt/ops/get_serialized_resource_op.cc
diff --git a/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc b/tensorflow/compiler/tf2tensorrt/ops/trt_engine_op.cc
similarity index 100%
rename from tensorflow/contrib/tensorrt/ops/trt_engine_op.cc
rename to tensorflow/compiler/tf2tensorrt/ops/trt_engine_op.cc
diff --git a/tensorflow/contrib/tensorrt/plugin/trt_plugin.cc b/tensorflow/compiler/tf2tensorrt/plugin/trt_plugin.cc
similarity index 96%
rename from tensorflow/contrib/tensorrt/plugin/trt_plugin.cc
rename to tensorflow/compiler/tf2tensorrt/plugin/trt_plugin.cc
index 062f86e8bb..a4341c530f 100644
--- a/tensorflow/contrib/tensorrt/plugin/trt_plugin.cc
+++ b/tensorflow/compiler/tf2tensorrt/plugin/trt_plugin.cc
@@ -13,10 +13,12 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/contrib/tensorrt/plugin/trt_plugin.h"
+#include "tensorflow/compiler/tf2tensorrt/plugin/trt_plugin.h"
+
 #include <cassert>
 #include <cstring>
-#include "tensorflow/contrib/tensorrt/plugin/trt_plugin_utils.h"
+
+#include "tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_utils.h"
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
diff --git a/tensorflow/contrib/tensorrt/plugin/trt_plugin.h b/tensorflow/compiler/tf2tensorrt/plugin/trt_plugin.h
similarity index 92%
rename from tensorflow/contrib/tensorrt/plugin/trt_plugin.h
rename to tensorflow/compiler/tf2tensorrt/plugin/trt_plugin.h
index 754920b60c..f495d85703 100644
--- a/tensorflow/contrib/tensorrt/plugin/trt_plugin.h
+++ b/tensorflow/compiler/tf2tensorrt/plugin/trt_plugin.h
@@ -13,8 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_CONTRIB_TENSORRT_PLUGIN_TRT_PLUGIN_H_
-#define TENSORFLOW_CONTRIB_TENSORRT_PLUGIN_TRT_PLUGIN_H_
+#ifndef TENSORFLOW_COMPILER_TF2TENSORRT_PLUGIN_TRT_PLUGIN_H_
+#define TENSORFLOW_COMPILER_TF2TENSORRT_PLUGIN_TRT_PLUGIN_H_
 
 #include <iostream>
 #include <unordered_map>
@@ -71,4 +71,4 @@ class PluginTensorRT : public nvinfer1::IPlugin {
 #endif  // GOOGLE_TENSORRT
 #endif  // GOOGLE_CUDA
 
-#endif  // TENSORFLOW_CONTRIB_TENSORRT_PLUGIN_TRT_PLUGIN_H_
+#endif  // TENSORFLOW_COMPILER_TF2TENSORRT_PLUGIN_TRT_PLUGIN_H_
diff --git a/tensorflow/contrib/tensorrt/plugin/trt_plugin_factory.cc b/tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_factory.cc
similarity index 96%
rename from tensorflow/contrib/tensorrt/plugin/trt_plugin_factory.cc
rename to tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_factory.cc
index cccc912262..871fb1210b 100644
--- a/tensorflow/contrib/tensorrt/plugin/trt_plugin_factory.cc
+++ b/tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_factory.cc
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/contrib/tensorrt/plugin/trt_plugin_factory.h"
+#include "tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_factory.h"
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
diff --git a/tensorflow/contrib/tensorrt/plugin/trt_plugin_factory.h b/tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_factory.h
similarity index 91%
rename from tensorflow/contrib/tensorrt/plugin/trt_plugin_factory.h
rename to tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_factory.h
index bbae9fb65c..9aa99a40b8 100644
--- a/tensorflow/contrib/tensorrt/plugin/trt_plugin_factory.h
+++ b/tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_factory.h
@@ -13,14 +13,14 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_CONTRIB_TENSORRT_PLUGIN_TRT_PLUGIN_FACTORY_H_
-#define TENSORFLOW_CONTRIB_TENSORRT_PLUGIN_TRT_PLUGIN_FACTORY_H_
+#ifndef TENSORFLOW_COMPILER_TF2TENSORRT_PLUGIN_TRT_PLUGIN_FACTORY_H_
+#define TENSORFLOW_COMPILER_TF2TENSORRT_PLUGIN_TRT_PLUGIN_FACTORY_H_
 
 #include <memory>
 #include <unordered_map>
 
-#include "tensorflow/contrib/tensorrt/plugin/trt_plugin.h"
-#include "tensorflow/contrib/tensorrt/plugin/trt_plugin_utils.h"
+#include "tensorflow/compiler/tf2tensorrt/plugin/trt_plugin.h"
+#include "tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_utils.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/macros.h"
 #include "tensorflow/core/platform/mutex.h"
@@ -99,4 +99,4 @@ class TrtPluginRegistrar {
 #endif  // GOOGLE_TENSORRT
 #endif  // GOOGLE_CUDA
 
-#endif  // TENSORFLOW_CONTRIB_TENSORRT_PLUGIN_TRT_PLUGIN_FACTORY_H_
+#endif  // TENSORFLOW_COMPILER_TF2TENSORRT_PLUGIN_TRT_PLUGIN_FACTORY_H_
diff --git a/tensorflow/contrib/tensorrt/plugin/trt_plugin_factory_test.cc b/tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_factory_test.cc
similarity index 96%
rename from tensorflow/contrib/tensorrt/plugin/trt_plugin_factory_test.cc
rename to tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_factory_test.cc
index 129bdcdbc2..7d9c465c22 100644
--- a/tensorflow/contrib/tensorrt/plugin/trt_plugin_factory_test.cc
+++ b/tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_factory_test.cc
@@ -13,9 +13,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/contrib/tensorrt/plugin/trt_plugin_factory.h"
+#include "tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_factory.h"
 
-#include "tensorflow/contrib/tensorrt/plugin/trt_plugin.h"
+#include "tensorflow/compiler/tf2tensorrt/plugin/trt_plugin.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/test.h"
diff --git a/tensorflow/contrib/tensorrt/plugin/trt_plugin_utils.cc b/tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_utils.cc
similarity index 94%
rename from tensorflow/contrib/tensorrt/plugin/trt_plugin_utils.cc
rename to tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_utils.cc
index a8f60886c0..f3d6b4ff47 100644
--- a/tensorflow/contrib/tensorrt/plugin/trt_plugin_utils.cc
+++ b/tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_utils.cc
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/contrib/tensorrt/plugin/trt_plugin_utils.h"
+#include "tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_utils.h"
 #include <cassert>
 
 #if GOOGLE_CUDA
diff --git a/tensorflow/contrib/tensorrt/plugin/trt_plugin_utils.h b/tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_utils.h
similarity index 82%
rename from tensorflow/contrib/tensorrt/plugin/trt_plugin_utils.h
rename to tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_utils.h
index 274ce42fec..e5eff15c19 100644
--- a/tensorflow/contrib/tensorrt/plugin/trt_plugin_utils.h
+++ b/tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_utils.h
@@ -13,12 +13,12 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_CONTRIB_TENSORRT_PLUGIN_TRT_PLUGIN_UTILS_H_
-#define TENSORFLOW_CONTRIB_TENSORRT_PLUGIN_TRT_PLUGIN_UTILS_H_
+#ifndef TENSORFLOW_COMPILER_TF2TENSORRT_PLUGIN_TRT_PLUGIN_UTILS_H_
+#define TENSORFLOW_COMPILER_TF2TENSORRT_PLUGIN_TRT_PLUGIN_UTILS_H_
 
 #include <functional>
 
-#include "tensorflow/contrib/tensorrt/plugin/trt_plugin.h"
+#include "tensorflow/compiler/tf2tensorrt/plugin/trt_plugin.h"
 #include "tensorflow/core/platform/types.h"
 
 #if GOOGLE_CUDA
@@ -43,4 +43,4 @@ string ExtractOpName(const void* serial_data, size_t serial_length,
 #endif  // GOOGLE_TENSORRT
 #endif  // GOOGLE_CUDA
 
-#endif  // TENSORFLOW_CONTRIB_TENSORRT_PLUGIN_TRT_PLUGIN_UTILS_H_
+#endif  // TENSORFLOW_COMPILER_TF2TENSORRT_PLUGIN_TRT_PLUGIN_UTILS_H_
diff --git a/tensorflow/contrib/tensorrt/python/ops/trt_ops.py b/tensorflow/compiler/tf2tensorrt/python/ops/trt_ops.py
similarity index 88%
rename from tensorflow/contrib/tensorrt/python/ops/trt_ops.py
rename to tensorflow/compiler/tf2tensorrt/python/ops/trt_ops.py
index 1fee06854f..86bfabf99e 100644
--- a/tensorflow/contrib/tensorrt/python/ops/trt_ops.py
+++ b/tensorflow/compiler/tf2tensorrt/python/ops/trt_ops.py
@@ -22,13 +22,13 @@ import platform
 
 if platform.system() != "Windows":
   # pylint: disable=wildcard-import,unused-import,g-import-not-at-top
-  from tensorflow.contrib.tensorrt.ops.gen_trt_ops import *
+  from tensorflow.compiler.tf2tensorrt.ops.gen_trt_ops import *
 
-  from tensorflow.contrib.util import loader
+  from tensorflow.python.framework import load_library
   from tensorflow.python.platform import resource_loader
   # pylint: enable=wildcard-import,unused-import,g-import-not-at-top
 
-  _trt_ops = loader.load_op_library(
+  _trt_ops = load_library.load_op_library(
       resource_loader.get_path_to_datafile("_trt_ops.so"))
 else:
   raise RuntimeError("Windows platforms are not supported")
diff --git a/tensorflow/contrib/tensorrt/segment/segment.cc b/tensorflow/compiler/tf2tensorrt/segment/segment.cc
similarity index 99%
rename from tensorflow/contrib/tensorrt/segment/segment.cc
rename to tensorflow/compiler/tf2tensorrt/segment/segment.cc
index ecaffa3023..4a8a4ac758 100644
--- a/tensorflow/contrib/tensorrt/segment/segment.cc
+++ b/tensorflow/compiler/tf2tensorrt/segment/segment.cc
@@ -13,14 +13,15 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/contrib/tensorrt/segment/segment.h"
+#include "tensorflow/compiler/tf2tensorrt/segment/segment.h"
 
 #include <queue>
 #include <set>
 #include <unordered_map>
 #include <vector>
 
-#include "tensorflow/contrib/tensorrt/segment/union_find.h"
+#include "absl/strings/str_cat.h"
+#include "tensorflow/compiler/tf2tensorrt/segment/union_find.h"
 #include "tensorflow/core/graph/algorithm.h"
 #include "tensorflow/core/graph/graph.h"
 #include "tensorflow/core/graph/graph_constructor.h"
@@ -32,8 +33,8 @@ limitations under the License.
 namespace tensorflow {
 namespace tensorrt {
 namespace segment {
-using ::tensorflow::strings::StrAppend;
-using ::tensorflow::strings::StrCat;
+using absl::StrAppend;
+using absl::StrCat;
 
 // A simple graph representation to mirror tensorflow::Graph. This structure
 // helps saving memory since segmenter modifies the graph in place, preventing
diff --git a/tensorflow/contrib/tensorrt/segment/segment.h b/tensorflow/compiler/tf2tensorrt/segment/segment.h
similarity index 92%
rename from tensorflow/contrib/tensorrt/segment/segment.h
rename to tensorflow/compiler/tf2tensorrt/segment/segment.h
index 6cc92cdb5d..9a0ccc9aef 100644
--- a/tensorflow/contrib/tensorrt/segment/segment.h
+++ b/tensorflow/compiler/tf2tensorrt/segment/segment.h
@@ -13,8 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_CONTRIB_TENSORRT_SEGMENT_SEGMENT_H_
-#define TENSORFLOW_CONTRIB_TENSORRT_SEGMENT_SEGMENT_H_
+#ifndef TENSORFLOW_COMPILER_TF2TENSORRT_SEGMENT_SEGMENT_H_
+#define TENSORFLOW_COMPILER_TF2TENSORRT_SEGMENT_SEGMENT_H_
 
 #include <set>
 #include <vector>
@@ -60,4 +60,4 @@ tensorflow::Status SegmentGraph(
 }  // namespace tensorrt
 }  // namespace tensorflow
 
-#endif  // TENSORFLOW_CONTRIB_TENSORRT_SEGMENT_SEGMENT_H_
+#endif  // TENSORFLOW_COMPILER_TF2TENSORRT_SEGMENT_SEGMENT_H_
diff --git a/tensorflow/contrib/tensorrt/segment/segment_test.cc b/tensorflow/compiler/tf2tensorrt/segment/segment_test.cc
similarity index 99%
rename from tensorflow/contrib/tensorrt/segment/segment_test.cc
rename to tensorflow/compiler/tf2tensorrt/segment/segment_test.cc
index 4ac02327ae..58512d3b09 100644
--- a/tensorflow/contrib/tensorrt/segment/segment_test.cc
+++ b/tensorflow/compiler/tf2tensorrt/segment/segment_test.cc
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/contrib/tensorrt/segment/segment.h"
+#include "tensorflow/compiler/tf2tensorrt/segment/segment.h"
 
 #include "tensorflow/cc/framework/scope.h"
 #include "tensorflow/cc/ops/standard_ops.h"
diff --git a/tensorflow/contrib/tensorrt/segment/union_find.h b/tensorflow/compiler/tf2tensorrt/segment/union_find.h
similarity index 92%
rename from tensorflow/contrib/tensorrt/segment/union_find.h
rename to tensorflow/compiler/tf2tensorrt/segment/union_find.h
index 1c64ebbb0a..6458ae692f 100644
--- a/tensorflow/contrib/tensorrt/segment/union_find.h
+++ b/tensorflow/compiler/tf2tensorrt/segment/union_find.h
@@ -13,8 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_CONTRIB_TENSORRT_SEGMENT_UNION_FIND_H_
-#define TENSORFLOW_CONTRIB_TENSORRT_SEGMENT_UNION_FIND_H_
+#ifndef TENSORFLOW_COMPILER_TF2TENSORRT_SEGMENT_UNION_FIND_H_
+#define TENSORFLOW_COMPILER_TF2TENSORRT_SEGMENT_UNION_FIND_H_
 
 namespace tensorflow {
 namespace tensorrt {
@@ -76,4 +76,4 @@ UnionFind<T>* UnionFind<T>::FindRoot() {
 }  // namespace tensorrt
 }  // namespace tensorflow
 
-#endif  // TENSORFLOW_CONTRIB_TENSORRT_SEGMENT_UNION_FIND_H_
+#endif  // TENSORFLOW_COMPILER_TF2TENSORRT_SEGMENT_UNION_FIND_H_
diff --git a/tensorflow/contrib/tensorrt/tensorrt_test.cc b/tensorflow/compiler/tf2tensorrt/tensorrt_test.cc
similarity index 100%
rename from tensorflow/contrib/tensorrt/tensorrt_test.cc
rename to tensorflow/compiler/tf2tensorrt/tensorrt_test.cc
diff --git a/tensorflow/contrib/tensorrt/test/utils.cc b/tensorflow/compiler/tf2tensorrt/utils/test_utils.cc
similarity index 97%
rename from tensorflow/contrib/tensorrt/test/utils.cc
rename to tensorflow/compiler/tf2tensorrt/utils/test_utils.cc
index 276308b3a0..3bcca99afb 100644
--- a/tensorflow/contrib/tensorrt/test/utils.cc
+++ b/tensorflow/compiler/tf2tensorrt/utils/test_utils.cc
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/contrib/tensorrt/test/utils.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/test_utils.h"
 
 #include <unordered_map>
 #include <vector>
diff --git a/tensorflow/contrib/tensorrt/test/utils.h b/tensorflow/compiler/tf2tensorrt/utils/test_utils.h
similarity index 89%
rename from tensorflow/contrib/tensorrt/test/utils.h
rename to tensorflow/compiler/tf2tensorrt/utils/test_utils.h
index 4bb4120206..bcd628b62f 100644
--- a/tensorflow/contrib/tensorrt/test/utils.h
+++ b/tensorflow/compiler/tf2tensorrt/utils/test_utils.h
@@ -13,8 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_CONTRIB_TENSORRT_TEST_UTILS_H_
-#define TENSORFLOW_CONTRIB_TENSORRT_TEST_UTILS_H_
+#ifndef TENSORFLOW_COMPILER_TF2TENSORRT_UTILS_TEST_UTILS_H_
+#define TENSORFLOW_COMPILER_TF2TENSORRT_UTILS_TEST_UTILS_H_
 
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status.h"
@@ -41,4 +41,4 @@ string GetTestValue(const string& label);
 }  // namespace tensorrt
 }  // namespace tensorflow
 
-#endif  // TENSORFLOW_CONTRIB_TENSORRT_TEST_UTILS_H_
+#endif  // TENSORFLOW_COMPILER_TF2TENSORRT_UTILS_TEST_UTILS_H_
diff --git a/tensorflow/contrib/tensorrt/resources/trt_allocator.cc b/tensorflow/compiler/tf2tensorrt/utils/trt_allocator.cc
similarity index 98%
rename from tensorflow/contrib/tensorrt/resources/trt_allocator.cc
rename to tensorflow/compiler/tf2tensorrt/utils/trt_allocator.cc
index 7a2e93414a..1636cdc30c 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_allocator.cc
+++ b/tensorflow/compiler/tf2tensorrt/utils/trt_allocator.cc
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/contrib/tensorrt/resources/trt_allocator.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_allocator.h"
 
 #include "tensorflow/core/platform/logging.h"
 
diff --git a/tensorflow/contrib/tensorrt/resources/trt_allocator.h b/tensorflow/compiler/tf2tensorrt/utils/trt_allocator.h
similarity index 93%
rename from tensorflow/contrib/tensorrt/resources/trt_allocator.h
rename to tensorflow/compiler/tf2tensorrt/utils/trt_allocator.h
index f857a9de05..59ffb42bad 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_allocator.h
+++ b/tensorflow/compiler/tf2tensorrt/utils/trt_allocator.h
@@ -13,8 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_ALLOCATOR_H_
-#define TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_ALLOCATOR_H_
+#ifndef TENSORFLOW_COMPILER_TF2TENSORRT_UTILS_TRT_ALLOCATOR_H_
+#define TENSORFLOW_COMPILER_TF2TENSORRT_UTILS_TRT_ALLOCATOR_H_
 
 #include <unordered_map>
 
@@ -81,4 +81,4 @@ class TRTDeviceAllocator : public TRTBaseAllocator {
 
 #endif  // GOOGLE_TENSORRT
 #endif  // GOOGLE_CUDA
-#endif  // TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_ALLOCATOR_H_
+#endif  // TENSORFLOW_COMPILER_TF2TENSORRT_UTILS_TRT_ALLOCATOR_H_
diff --git a/tensorflow/contrib/tensorrt/resources/trt_allocator_test.cc b/tensorflow/compiler/tf2tensorrt/utils/trt_allocator_test.cc
similarity index 98%
rename from tensorflow/contrib/tensorrt/resources/trt_allocator_test.cc
rename to tensorflow/compiler/tf2tensorrt/utils/trt_allocator_test.cc
index beb1284208..e457c64928 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_allocator_test.cc
+++ b/tensorflow/compiler/tf2tensorrt/utils/trt_allocator_test.cc
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/contrib/tensorrt/resources/trt_allocator.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_allocator.h"
 
 #include "tensorflow/core/platform/test.h"
 
diff --git a/tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.cc b/tensorflow/compiler/tf2tensorrt/utils/trt_int8_calibrator.cc
similarity index 98%
rename from tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.cc
rename to tensorflow/compiler/tf2tensorrt/utils/trt_int8_calibrator.cc
index dab1dd9343..bf111d3a2e 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.cc
+++ b/tensorflow/compiler/tf2tensorrt/utils/trt_int8_calibrator.cc
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_int8_calibrator.h"
 
 #include <atomic>
 #include <unordered_map>
diff --git a/tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.h b/tensorflow/compiler/tf2tensorrt/utils/trt_int8_calibrator.h
similarity index 93%
rename from tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.h
rename to tensorflow/compiler/tf2tensorrt/utils/trt_int8_calibrator.h
index 65466c9741..10587e9962 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.h
+++ b/tensorflow/compiler/tf2tensorrt/utils/trt_int8_calibrator.h
@@ -13,8 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_INT8_CALIBRATOR_H_
-#define TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_INT8_CALIBRATOR_H_
+#ifndef TENSORFLOW_COMPILER_TF2TENSORRT_UTILS_TRT_INT8_CALIBRATOR_H_
+#define TENSORFLOW_COMPILER_TF2TENSORRT_UTILS_TRT_INT8_CALIBRATOR_H_
 
 #include <atomic>
 #include <string>
@@ -96,4 +96,4 @@ struct TRTInt8Calibrator : public nvinfer1::IInt8EntropyCalibrator {
 
 #endif
 #endif
-#endif  // TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_INT8_CALIBRATOR_H_
+#endif  // TENSORFLOW_COMPILER_TF2TENSORRT_UTILS_TRT_INT8_CALIBRATOR_H_
diff --git a/tensorflow/contrib/tensorrt/log/trt_logger.cc b/tensorflow/compiler/tf2tensorrt/utils/trt_logger.cc
similarity index 96%
rename from tensorflow/contrib/tensorrt/log/trt_logger.cc
rename to tensorflow/compiler/tf2tensorrt/utils/trt_logger.cc
index dda0dc9e71..c48bd6bf77 100644
--- a/tensorflow/contrib/tensorrt/log/trt_logger.cc
+++ b/tensorflow/compiler/tf2tensorrt/utils/trt_logger.cc
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/contrib/tensorrt/log/trt_logger.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_logger.h"
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
diff --git a/tensorflow/contrib/tensorrt/log/trt_logger.h b/tensorflow/compiler/tf2tensorrt/utils/trt_logger.h
similarity index 86%
rename from tensorflow/contrib/tensorrt/log/trt_logger.h
rename to tensorflow/compiler/tf2tensorrt/utils/trt_logger.h
index 96ccacb791..22f4de970a 100644
--- a/tensorflow/contrib/tensorrt/log/trt_logger.h
+++ b/tensorflow/compiler/tf2tensorrt/utils/trt_logger.h
@@ -13,8 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_CONTRIB_TENSORRT_LOG_TRT_LOGGER_H_
-#define TENSORFLOW_CONTRIB_TENSORRT_LOG_TRT_LOGGER_H_
+#ifndef TENSORFLOW_COMPILER_TF2TENSORRT_UTILS_TRT_LOGGER_H_
+#define TENSORFLOW_COMPILER_TF2TENSORRT_UTILS_TRT_LOGGER_H_
 
 #include "tensorflow/core/platform/types.h"
 
@@ -41,4 +41,4 @@ class Logger : public nvinfer1::ILogger {
 #endif  // GOOGLE_TENSORRT
 #endif  // GOOGLE_CUDA
 
-#endif  // TENSORFLOW_CONTRIB_TENSORRT_LOG_TRT_LOGGER_H_
+#endif  // TENSORFLOW_COMPILER_TF2TENSORRT_UTILS_TRT_LOGGER_H_
diff --git a/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h b/tensorflow/compiler/tf2tensorrt/utils/trt_lru_cache.h
similarity index 94%
rename from tensorflow/contrib/tensorrt/resources/trt_lru_cache.h
rename to tensorflow/compiler/tf2tensorrt/utils/trt_lru_cache.h
index afd1b83e74..09c47b36b0 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_lru_cache.h
+++ b/tensorflow/compiler/tf2tensorrt/utils/trt_lru_cache.h
@@ -13,14 +13,14 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_LRU_CACHE_H_
-#define TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_LRU_CACHE_H_
+#ifndef TENSORFLOW_COMPILER_TF2TENSORRT_UTILS_TRT_LRU_CACHE_H_
+#define TENSORFLOW_COMPILER_TF2TENSORRT_UTILS_TRT_LRU_CACHE_H_
 
 #include <list>
 #include <unordered_map>
 
-#include "tensorflow/contrib/tensorrt/convert/utils.h"
-#include "tensorflow/contrib/tensorrt/resources/trt_allocator.h"
+#include "tensorflow/compiler/tf2tensorrt/convert/utils.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_allocator.h"
 #include "tensorflow/core/framework/resource_mgr.h"
 #include "tensorflow/core/lib/core/errors.h"
 
@@ -189,4 +189,4 @@ class TRTEngineCacheResource : public tensorflow::ResourceBase {
 }  // namespace tensorrt
 }  // namespace tensorflow
 
-#endif  // TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_LRU_CACHE_H_
+#endif  // TENSORFLOW_COMPILER_TF2TENSORRT_UTILS_TRT_LRU_CACHE_H_
diff --git a/tensorflow/contrib/tensorrt/resources/trt_lru_cache_test.cc b/tensorflow/compiler/tf2tensorrt/utils/trt_lru_cache_test.cc
similarity index 96%
rename from tensorflow/contrib/tensorrt/resources/trt_lru_cache_test.cc
rename to tensorflow/compiler/tf2tensorrt/utils/trt_lru_cache_test.cc
index a0959479ce..0aa5eb8f7d 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_lru_cache_test.cc
+++ b/tensorflow/compiler/tf2tensorrt/utils/trt_lru_cache_test.cc
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/contrib/tensorrt/resources/trt_lru_cache.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_lru_cache.h"
 
 #include "tensorflow/core/platform/test.h"
 
diff --git a/tensorflow/contrib/tensorrt/resources/trt_resource_manager.cc b/tensorflow/compiler/tf2tensorrt/utils/trt_resource_manager.cc
similarity index 96%
rename from tensorflow/contrib/tensorrt/resources/trt_resource_manager.cc
rename to tensorflow/compiler/tf2tensorrt/utils/trt_resource_manager.cc
index 9c3698e5d1..0a72a88bc7 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_resource_manager.cc
+++ b/tensorflow/compiler/tf2tensorrt/utils/trt_resource_manager.cc
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/contrib/tensorrt/resources/trt_resource_manager.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_resource_manager.h"
 #include "tensorflow/core/platform/logging.h"
 
 namespace tensorflow {
diff --git a/tensorflow/contrib/tensorrt/resources/trt_resource_manager.h b/tensorflow/compiler/tf2tensorrt/utils/trt_resource_manager.h
similarity index 87%
rename from tensorflow/contrib/tensorrt/resources/trt_resource_manager.h
rename to tensorflow/compiler/tf2tensorrt/utils/trt_resource_manager.h
index 19f39e6d3d..03879ffff2 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_resource_manager.h
+++ b/tensorflow/compiler/tf2tensorrt/utils/trt_resource_manager.h
@@ -13,8 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_RESOURCE_MANAGER_H_
-#define TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_RESOURCE_MANAGER_H_
+#ifndef TENSORFLOW_COMPILER_TF2TENSORRT_UTILS_TRT_RESOURCE_MANAGER_H_
+#define TENSORFLOW_COMPILER_TF2TENSORRT_UTILS_TRT_RESOURCE_MANAGER_H_
 #include <memory>
 
 #include <string>
@@ -42,4 +42,4 @@ class TRTResourceManager {
 }  // namespace tensorrt
 }  // namespace tensorflow
 
-#endif  // TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_RESOURCE_MANAGER_H_
+#endif  // TENSORFLOW_COMPILER_TF2TENSORRT_UTILS_TRT_RESOURCE_MANAGER_H_
diff --git a/tensorflow/contrib/tensorrt/resources/trt_resources.cc b/tensorflow/compiler/tf2tensorrt/utils/trt_resources.cc
similarity index 96%
rename from tensorflow/contrib/tensorrt/resources/trt_resources.cc
rename to tensorflow/compiler/tf2tensorrt/utils/trt_resources.cc
index c19eb34dab..37f7fe99fb 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_resources.cc
+++ b/tensorflow/compiler/tf2tensorrt/utils/trt_resources.cc
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/contrib/tensorrt/resources/trt_resources.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_resources.h"
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
diff --git a/tensorflow/contrib/tensorrt/resources/trt_resources.h b/tensorflow/compiler/tf2tensorrt/utils/trt_resources.h
similarity index 82%
rename from tensorflow/contrib/tensorrt/resources/trt_resources.h
rename to tensorflow/compiler/tf2tensorrt/utils/trt_resources.h
index d0a87f2c31..5e8d4b3b73 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_resources.h
+++ b/tensorflow/compiler/tf2tensorrt/utils/trt_resources.h
@@ -13,8 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_RESOURCES_H_
-#define TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_RESOURCES_H_
+#ifndef TENSORFLOW_COMPILER_TF2TENSORRT_UTILS_TRT_RESOURCES_H_
+#define TENSORFLOW_COMPILER_TF2TENSORRT_UTILS_TRT_RESOURCES_H_
 
 #include <list>
 #include <sstream>
@@ -23,10 +23,10 @@ limitations under the License.
 #include <unordered_map>
 #include <vector>
 
-#include "tensorflow/contrib/tensorrt/convert/utils.h"
-#include "tensorflow/contrib/tensorrt/log/trt_logger.h"
-#include "tensorflow/contrib/tensorrt/resources/trt_allocator.h"
-#include "tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.h"
+#include "tensorflow/compiler/tf2tensorrt/convert/utils.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_allocator.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_int8_calibrator.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_logger.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/resource_mgr.h"
 
@@ -70,4 +70,4 @@ class TRTCalibrationResource : public SerializableResourceBase {
 
 #endif  // GOOGLE_TENSORRT
 #endif  // GOOGLE_CUDA
-#endif  // TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_RESOURCES_H_
+#endif  // TENSORFLOW_COMPILER_TF2TENSORRT_UTILS_TRT_RESOURCES_H_
diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index 3f10906768..67461450f8 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -11,18 +11,12 @@ exports_files(["LICENSE"])
 
 load(
     "//tensorflow:tensorflow.bzl",
-    "tf_cc_test",
     "tf_copts",
     "tf_cuda_library",
-    "tf_custom_op_library",
     "tf_custom_op_library_additional_deps",
-    "tf_gen_op_libs",
-    "tf_gen_op_wrapper_py",
 )
 load("//tensorflow:tensorflow.bzl", "cuda_py_test")
 load("//tensorflow:tensorflow.bzl", "cuda_py_tests")
-load("//tensorflow:tensorflow.bzl", "tf_cuda_cc_test")
-load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library")
 load("//tensorflow:tensorflow.bzl", "tf_py_wrap_cc")
 load(
     "@local_config_tensorrt//:build_defs.bzl",
@@ -33,158 +27,19 @@ exports_files(glob([
     "test/testdata/*",
 ]))
 
-tf_cuda_cc_test(
-    name = "tensorrt_test_cc",
-    size = "small",
-    srcs = ["tensorrt_test.cc"],
-    tags = [
-        "no_windows",
-        "nomac",
-    ],
-    deps = [
-        "//tensorflow/core:gpu_init",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:stream_executor",
-        "//tensorflow/core:test",
-        "//tensorflow/core:test_main",
-    ] + if_tensorrt([
-        "@local_config_cuda//cuda:cuda_headers",
-        "@local_config_tensorrt//:tensorrt",
-    ]),
-)
-
-tf_custom_op_library(
-    name = "python/ops/_trt_ops.so",
-    srcs = [
-        "ops/get_serialized_resource_op.cc",
-        "ops/trt_engine_op.cc",
-    ],
-    deps = [
-        ":trt_shape_function",
-        "//tensorflow/core:lib_proto_parsing",
-    ] + if_tensorrt([
-        "@local_config_tensorrt//:tensorrt",
-    ]),
-)
-
 tf_cuda_library(
     name = "trt_shape_function",
     srcs = ["shape_fn/trt_shfn.cc"],
     hdrs = ["shape_fn/trt_shfn.h"],
     visibility = ["//visibility:public"],
     deps = [
-        ":trt_logging",
-        ":trt_plugins",
+        "//tensorflow/compiler/tf2tensorrt:trt_logging",
+        "//tensorflow/compiler/tf2tensorrt:trt_plugins",
     ] + if_tensorrt([
         "@local_config_tensorrt//:tensorrt",
     ]) + tf_custom_op_library_additional_deps(),
 )
 
-cc_library(
-    name = "trt_op_kernels",
-    srcs = [
-        "kernels/get_serialized_resource_op.cc",
-        "kernels/trt_engine_op.cc",
-    ],
-    hdrs = [
-        "kernels/trt_engine_op.h",
-    ],
-    copts = tf_copts(),
-    visibility = ["//visibility:public"],
-    deps = [
-        ":test_utils",
-        ":trt_allocator",
-        ":trt_conversion",
-        ":trt_logging",
-        ":trt_plugins",
-        ":trt_resources",
-        ":utils",
-        "@com_google_absl//absl/memory",
-        "//tensorflow/core:gpu_headers_lib",
-        "//tensorflow/core:lib_proto_parsing",
-        "//tensorflow/core:stream_executor_headers_lib",
-        "//tensorflow/core/grappler/costs:graph_properties",
-    ] + if_tensorrt([
-        "@local_config_tensorrt//:tensorrt",
-    ]) + tf_custom_op_library_additional_deps(),
-    # TODO(laigd): fix this by merging header file in cc file.
-    alwayslink = 1,  # buildozer: disable=alwayslink-with-hdrs
-)
-
-tf_cuda_cc_test(
-    name = "get_serialized_resource_op_test",
-    size = "small",
-    srcs = ["kernels/get_serialized_resource_op_test.cc"],
-    tags = [
-        "no_cuda_on_cpu_tap",
-        "no_windows",
-        "nomac",
-    ],
-    deps = [
-        ":get_serialized_resource_op_op_lib",
-        ":trt_op_kernels",
-        ":trt_resources",
-        "//tensorflow/core:framework",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:protos_all_cc",
-        "//tensorflow/core:test",
-        "//tensorflow/core:test_main",
-        "//tensorflow/core:testlib",
-        "//tensorflow/core/kernels:ops_testutil",
-    ],
-)
-
-tf_gen_op_libs(
-    op_lib_names = [
-        "trt_engine_op",
-        "get_serialized_resource_op",
-    ],
-)
-
-tf_cuda_library(
-    name = "trt_logging",
-    srcs = ["log/trt_logger.cc"],
-    hdrs = ["log/trt_logger.h"],
-    visibility = ["//visibility:public"],
-    deps = [
-        "//tensorflow/core:lib_proto_parsing",
-    ] + if_tensorrt([
-        "@local_config_tensorrt//:tensorrt",
-    ]),
-)
-
-tf_gen_op_wrapper_py(
-    name = "trt_ops",
-    deps = [
-        ":get_serialized_resource_op_op_lib",
-        ":trt_engine_op_op_lib",
-        ":trt_logging",
-        ":trt_shape_function",
-    ],
-)
-
-tf_custom_op_py_library(
-    name = "trt_ops_loader",
-    srcs = ["python/ops/trt_ops.py"],
-    dso = [
-        ":python/ops/_trt_ops.so",
-    ] + if_tensorrt([
-        "@local_config_tensorrt//:tensorrt",
-    ]),
-    kernels = [
-        ":trt_op_kernels",
-        ":trt_engine_op_op_lib",
-        ":get_serialized_resource_op_op_lib",
-        ":trt_shape_function",
-    ],
-    srcs_version = "PY2AND3",
-    deps = [
-        "//tensorflow/contrib/util:util_py",
-        "//tensorflow/python:framework_for_generated_wrappers",
-        "//tensorflow/python:resources",
-    ],
-)
-
 py_library(
     name = "init_py",
     srcs = [
@@ -204,8 +59,8 @@ py_library(
     name = "trt_ops_py",
     srcs_version = "PY2AND3",
     deps = [
-        ":trt_ops",
-        ":trt_ops_loader",
+        "//tensorflow/compiler/tf2tensorrt:trt_ops",
+        "//tensorflow/compiler/tf2tensorrt:trt_ops_loader",
     ],
 )
 
@@ -234,264 +89,13 @@ tf_py_wrap_cc(
         "//tensorflow/python:platform/base.i",
     ],
     deps = [
-        ":test_utils",
-        ":trt_conversion",
-        ":trt_op_kernels",
+        "//tensorflow/compiler/tf2tensorrt:test_utils",
+        "//tensorflow/compiler/tf2tensorrt:trt_conversion",
+        "//tensorflow/compiler/tf2tensorrt:trt_op_kernels",
         "//third_party/python_runtime:headers",
     ],
 )
 
-tf_cuda_library(
-    name = "trt_resources",
-    srcs = [
-        "resources/trt_int8_calibrator.cc",
-        "resources/trt_resource_manager.cc",
-        "resources/trt_resources.cc",
-    ],
-    hdrs = [
-        "resources/trt_int8_calibrator.h",
-        "resources/trt_lru_cache.h",
-        "resources/trt_resource_manager.h",
-        "resources/trt_resources.h",
-    ],
-    deps = [
-        ":trt_allocator",
-        ":trt_logging",
-        ":utils",
-        "//tensorflow/core:framework_headers_lib",
-        "//tensorflow/core:framework_lite",
-        "//tensorflow/core:lib_proto_parsing",
-    ] + if_tensorrt([
-        "@local_config_tensorrt//:tensorrt",
-    ]),
-)
-
-tf_cuda_library(
-    name = "trt_allocator",
-    srcs = ["resources/trt_allocator.cc"],
-    hdrs = ["resources/trt_allocator.h"],
-    deps = [
-        "//tensorflow/core:framework_headers_lib",
-        "//tensorflow/core:framework_lite",
-        "//tensorflow/core:lib_proto_parsing",
-    ] + if_tensorrt([
-        "@local_config_tensorrt//:tensorrt",
-    ]),
-)
-
-tf_cc_test(
-    name = "trt_allocator_test",
-    size = "small",
-    srcs = ["resources/trt_allocator_test.cc"],
-    tags = [
-        "no_windows",
-        "nomac",
-    ],
-    deps = [
-        ":trt_allocator",
-        "//tensorflow/core:test",
-        "//tensorflow/core:test_main",
-    ],
-)
-
-tf_cc_test(
-    name = "trt_lru_cache_test",
-    size = "small",
-    srcs = ["resources/trt_lru_cache_test.cc"],
-    tags = [
-        "no_windows",
-        "nomac",
-    ],
-    deps = [
-        ":trt_resources",
-        "//tensorflow/core:test",
-        "//tensorflow/core:test_main",
-    ],
-)
-
-# Library for the node-level conversion portion of TensorRT operation creation
-tf_cuda_library(
-    name = "trt_conversion",
-    srcs = [
-        "convert/convert_graph.cc",
-        "convert/convert_nodes.cc",
-        "convert/trt_optimization_pass.cc",
-    ],
-    hdrs = [
-        "convert/convert_graph.h",
-        "convert/convert_nodes.h",
-        "convert/trt_optimization_pass.h",
-    ],
-    deps = [
-        ":segment",
-        ":test_utils",
-        ":trt_allocator",
-        ":trt_plugins",
-        ":trt_logging",
-        ":trt_resources",
-        ":utils",
-        "//tensorflow/core/grappler/clusters:cluster",
-        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer",
-        "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry",
-        "//tensorflow/core/grappler:grappler_item",
-        "//tensorflow/core/grappler:utils",
-        "//tensorflow/core:framework",
-        "//tensorflow/core:framework_lite",
-        "//tensorflow/core:gpu_runtime",
-        "//tensorflow/core:graph",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:lib_internal",
-        "//tensorflow/core:protos_all_cc",
-        "//tensorflow/core/grappler:devices",
-        "//tensorflow/core/grappler/clusters:virtual_cluster",
-        "//tensorflow/core/grappler/costs:graph_properties",
-        "//tensorflow/core/grappler/optimizers:meta_optimizer",
-    ] + if_tensorrt([
-        "@local_config_tensorrt//:tensorrt",
-    ]) + tf_custom_op_library_additional_deps(),
-)
-
-tf_cuda_cc_test(
-    name = "convert_graph_test",
-    size = "medium",
-    srcs = ["convert/convert_graph_test.cc"],
-    tags = [
-        "no_cuda_on_cpu_tap",
-        "no_windows",
-        "nomac",
-    ],
-    deps = [
-        ":trt_conversion",
-        "@com_google_googletest//:gtest",
-        "//tensorflow/cc:cc_ops",
-        "//tensorflow/cc:scope",
-        "//tensorflow/core/grappler:grappler_item",
-        "//tensorflow/core/grappler/clusters:cluster",
-        "//tensorflow/core:core_cpu",
-        "//tensorflow/core:core_cpu_base",
-        "//tensorflow/core:direct_session",
-        "//tensorflow/core:framework",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:protos_all_cc",
-        "//tensorflow/core:test",
-        "//tensorflow/core:test_main",
-        "//tensorflow/core:testlib",
-    ] + if_tensorrt([
-        "@local_config_tensorrt//:tensorrt",
-    ]),
-)
-
-tf_cuda_cc_test(
-    name = "convert_nodes_test",
-    size = "medium",
-    srcs = ["convert/convert_nodes_test.cc"],
-    tags = [
-        "no_cuda_on_cpu_tap",
-        "no_windows",
-        "nomac",
-    ],
-    deps = [
-        ":trt_logging",
-        ":trt_conversion",
-        ":trt_plugins",
-        "@com_google_googletest//:gtest",
-        "//tensorflow/cc:cc_ops",
-        "//tensorflow/cc:ops",
-        "//tensorflow/cc:scope",
-        "//tensorflow/core/grappler/costs:graph_properties",
-        "//tensorflow/core:core_cpu",
-        "//tensorflow/core:core_cpu_base",
-        "//tensorflow/core:framework",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:protos_all_cc",
-        "//tensorflow/core:test",
-        "//tensorflow/core:test_main",
-        "//tensorflow/core:testlib",
-    ] + if_tensorrt([
-        "@local_config_cuda//cuda:cuda_headers",
-        "@local_config_tensorrt//:tensorrt",
-    ]),
-)
-
-# Library for the segmenting portion of TensorRT operation creation
-cc_library(
-    name = "segment",
-    srcs = ["segment/segment.cc"],
-    hdrs = [
-        "segment/segment.h",
-        "segment/union_find.h",
-    ],
-    deps = [
-        "//tensorflow/core:graph",
-        "//tensorflow/core:lib_proto_parsing",
-        "//tensorflow/core:protos_all_cc",
-        "@protobuf_archive//:protobuf_headers",
-    ],
-)
-
-tf_cc_test(
-    name = "segment_test",
-    size = "small",
-    srcs = ["segment/segment_test.cc"],
-    tags = [
-        "no_windows",
-        "nomac",
-    ],
-    deps = [
-        ":segment",
-        "//tensorflow/cc:cc_ops",
-        "//tensorflow/cc:scope",
-        "//tensorflow/core:core_cpu",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:ops",
-        "//tensorflow/core:protos_all_cc",
-        "//tensorflow/core:test",
-        "//tensorflow/core:test_main",
-        "//tensorflow/core:testlib",
-    ],
-)
-
-# Library for the plugin factory
-tf_cuda_library(
-    name = "trt_plugins",
-    srcs = [
-        "plugin/trt_plugin.cc",
-        "plugin/trt_plugin_factory.cc",
-        "plugin/trt_plugin_utils.cc",
-    ],
-    hdrs = [
-        "plugin/trt_plugin.h",
-        "plugin/trt_plugin_factory.h",
-        "plugin/trt_plugin_utils.h",
-    ],
-    deps = [
-        "//tensorflow/core:framework_lite",
-        "//tensorflow/core:lib_proto_parsing",
-    ] + if_tensorrt([
-        "@local_config_tensorrt//:tensorrt",
-    ]),
-)
-
-tf_cuda_cc_test(
-    name = "trt_plugin_factory_test",
-    size = "small",
-    srcs = ["plugin/trt_plugin_factory_test.cc"],
-    tags = [
-        "no_cuda_on_cpu_tap",
-        "no_windows",
-        "nomac",
-    ],
-    deps = [
-        ":trt_plugins",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:test",
-        "//tensorflow/core:test_main",
-    ] + if_tensorrt([
-        "@local_config_cuda//cuda:cuda_headers",
-        "@local_config_tensorrt//:tensorrt",
-    ]),
-)
-
 py_library(
     name = "tf_trt_integration_test_base",
     srcs = ["test/tf_trt_integration_test_base.py"],
@@ -590,23 +194,20 @@ cuda_py_test(
     ],
 )
 
-cc_library(
-    name = "utils",
-    srcs = ["convert/utils.cc"],
-    hdrs = ["convert/utils.h"],
-    copts = tf_copts(),
-    deps = [
-        "//tensorflow/core:framework",
-        "//tensorflow/core:lib",
-    ],
+# The following rules forward the libraries that were moved in order to not
+# break other internal targets.
+
+alias(
+    name = "trt_conversion",
+    actual = "//tensorflow/compiler/tf2tensorrt:trt_conversion",
 )
 
-cc_library(
-    name = "test_utils",
-    srcs = ["test/utils.cc"],
-    hdrs = ["test/utils.h"],
-    deps = [
-        "//tensorflow/core:lib",
-        "@com_googlesource_code_re2//:re2",
-    ],
+alias(
+    name = "trt_op_kernels",
+    actual = "//tensorflow/compiler/tf2tensorrt:trt_op_kernels",
+)
+
+alias(
+    name = "trt_engine_op_op_lib",
+    actual = "//tensorflow/compiler/tf2tensorrt:trt_engine_op_op_lib",
 )
diff --git a/tensorflow/contrib/tensorrt/custom_plugin_examples/BUILD b/tensorflow/contrib/tensorrt/custom_plugin_examples/BUILD
index 09990fb5e3..0a2cf105ba 100644
--- a/tensorflow/contrib/tensorrt/custom_plugin_examples/BUILD
+++ b/tensorflow/contrib/tensorrt/custom_plugin_examples/BUILD
@@ -45,7 +45,7 @@ tf_custom_op_library(
         "inc_op_kernel.cu.cc",
     ],
     deps = [
-        "//tensorflow/contrib/tensorrt:trt_plugins",
+        "//tensorflow/compiler/tf2tensorrt:trt_plugins",
         "//tensorflow/core:framework_lite",
     ] + if_tensorrt([
         "@local_config_tensorrt//:tensorrt",
@@ -64,7 +64,7 @@ tf_kernel_library(
         "inc_op_kernel.cu.cc",
     ],
     deps = [
-        "//tensorflow/contrib/tensorrt:trt_plugins",
+        "//tensorflow/compiler/tf2tensorrt:trt_plugins",
         "//tensorflow/core:stream_executor_headers_lib",
     ] + if_tensorrt([
         "@local_config_tensorrt//:tensorrt",
diff --git a/tensorflow/contrib/tensorrt/custom_plugin_examples/inc_op_plugin.cc b/tensorflow/contrib/tensorrt/custom_plugin_examples/inc_op_plugin.cc
index 8d4c893af5..7c9075142a 100644
--- a/tensorflow/contrib/tensorrt/custom_plugin_examples/inc_op_plugin.cc
+++ b/tensorflow/contrib/tensorrt/custom_plugin_examples/inc_op_plugin.cc
@@ -15,8 +15,8 @@ limitations under the License.
 
 #include "tensorflow/contrib/tensorrt/custom_plugin_examples/inc_op_plugin.h"
 
+#include "tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_factory.h"
 #include "tensorflow/contrib/tensorrt/custom_plugin_examples/inc_op_kernel.h"
-#include "tensorflow/contrib/tensorrt/plugin/trt_plugin_factory.h"
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
diff --git a/tensorflow/contrib/tensorrt/custom_plugin_examples/inc_op_plugin.h b/tensorflow/contrib/tensorrt/custom_plugin_examples/inc_op_plugin.h
index 189e9c939b..fb048d7b19 100644
--- a/tensorflow/contrib/tensorrt/custom_plugin_examples/inc_op_plugin.h
+++ b/tensorflow/contrib/tensorrt/custom_plugin_examples/inc_op_plugin.h
@@ -19,7 +19,7 @@ limitations under the License.
 #include <cassert>
 #include <cstring>
 
-#include "tensorflow/contrib/tensorrt/plugin/trt_plugin.h"
+#include "tensorflow/compiler/tf2tensorrt/plugin/trt_plugin.h"
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
diff --git a/tensorflow/contrib/tensorrt/python/__init__.py b/tensorflow/contrib/tensorrt/python/__init__.py
index e2cf253ca0..75490aecfb 100644
--- a/tensorflow/contrib/tensorrt/python/__init__.py
+++ b/tensorflow/contrib/tensorrt/python/__init__.py
@@ -19,7 +19,7 @@ from __future__ import division
 from __future__ import print_function
 
 # pylint: disable=unused-import,line-too-long
-from tensorflow.contrib.tensorrt.python.ops import trt_ops
+from tensorflow.compiler.tf2tensorrt.python.ops import trt_ops
 from tensorflow.contrib.tensorrt.python.trt_convert import add_test_value
 from tensorflow.contrib.tensorrt.python.trt_convert import calib_graph_to_infer_graph
 from tensorflow.contrib.tensorrt.python.trt_convert import clear_test_values
diff --git a/tensorflow/contrib/tensorrt/python/trt_convert_test.py b/tensorflow/contrib/tensorrt/python/trt_convert_test.py
index 3ef18e3e15..abd822c7b7 100644
--- a/tensorflow/contrib/tensorrt/python/trt_convert_test.py
+++ b/tensorflow/contrib/tensorrt/python/trt_convert_test.py
@@ -20,10 +20,10 @@ from __future__ import print_function
 
 import os
 
-from tensorflow.contrib.tensorrt.python import trt_convert
 # pylint: disable=unused-import
-from tensorflow.contrib.tensorrt.python.ops import trt_ops
+from tensorflow.compiler.tf2tensorrt.python.ops import trt_ops
 # pylint: enable=unused-import
+from tensorflow.contrib.tensorrt.python import trt_convert
 from tensorflow.core.framework import graph_pb2
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.core.protobuf import rewriter_config_pb2
diff --git a/tensorflow/contrib/tensorrt/shape_fn/trt_shfn.cc b/tensorflow/contrib/tensorrt/shape_fn/trt_shfn.cc
index f30dba59ad..5c60d6b589 100644
--- a/tensorflow/contrib/tensorrt/shape_fn/trt_shfn.cc
+++ b/tensorflow/contrib/tensorrt/shape_fn/trt_shfn.cc
@@ -14,14 +14,14 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/contrib/tensorrt/shape_fn/trt_shfn.h"
-#include "tensorflow/contrib/tensorrt/plugin/trt_plugin_factory.h"
 
 #include <string>
 #include <vector>
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
-#include "tensorflow/contrib/tensorrt/log/trt_logger.h"
+#include "tensorflow/compiler/tf2tensorrt/plugin/trt_plugin_factory.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/trt_logger.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorrt/include/NvInfer.h"
 
diff --git a/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py b/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py
index 753b47f4d9..d68211a7ee 100644
--- a/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py
+++ b/tensorflow/contrib/tensorrt/test/quantization_mnist_test.py
@@ -18,10 +18,10 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.tensorrt.python import trt_convert
 # pylint: disable=unused-import
-from tensorflow.contrib.tensorrt.python.ops import trt_ops
+from tensorflow.compiler.tf2tensorrt.python.ops import trt_ops
 # pylint: enable=unused-import
+from tensorflow.contrib.tensorrt.python import trt_convert
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python import data
 from tensorflow.python import keras
diff --git a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
index db43581404..9a00cdb11a 100644
--- a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
+++ b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
@@ -25,10 +25,10 @@ import warnings
 import numpy as np
 import six
 
-from tensorflow.contrib.tensorrt.python import trt_convert
 # pylint: disable=unused-import
-from tensorflow.contrib.tensorrt.python.ops import trt_ops
+from tensorflow.compiler.tf2tensorrt.python.ops import trt_ops
 # pylint: enable=unused-import
+from tensorflow.contrib.tensorrt.python import trt_convert
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.core.protobuf import rewriter_config_pb2
 from tensorflow.python.framework import dtypes
diff --git a/tensorflow/contrib/tensorrt/trt_conversion.i b/tensorflow/contrib/tensorrt/trt_conversion.i
index 6ea15fb8ef..c12895c730 100644
--- a/tensorflow/contrib/tensorrt/trt_conversion.i
+++ b/tensorflow/contrib/tensorrt/trt_conversion.i
@@ -99,9 +99,9 @@ _LIST_OUTPUT_TYPEMAP(int, PyLong_FromLong);
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/util/stat_summarizer.h"
-#include "tensorflow/contrib/tensorrt/convert/convert_graph.h"
-#include "tensorflow/contrib/tensorrt/convert/utils.h"
-#include "tensorflow/contrib/tensorrt/test/utils.h"
+#include "tensorflow/compiler/tf2tensorrt/convert/convert_graph.h"
+#include "tensorflow/compiler/tf2tensorrt/convert/utils.h"
+#include "tensorflow/compiler/tf2tensorrt/utils/test_utils.h"
 %}
 
 %ignoreall
-- 
GitLab


From f5fc7af1b0584672ca29bd4c584481e3dedaacd6 Mon Sep 17 00:00:00 2001
From: Blake Hechtman <blakehechtman@google.com>
Date: Tue, 22 Jan 2019 14:11:54 -0800
Subject: [PATCH 1078/1095] [XLA:CLIENT] Make a simple einsum interface to dot
 general.

PiperOrigin-RevId: 230407712
---
 tensorflow/compiler/xla/client/lib/BUILD      |  21 +-
 tensorflow/compiler/xla/client/lib/matrix.cc  | 259 ++++++++++++++----
 tensorflow/compiler/xla/client/lib/matrix.h   |  38 +++
 .../compiler/xla/client/lib/matrix_test.cc    |  76 +++++
 tensorflow/compiler/xla/tests/BUILD           |   2 +
 .../compiler/xla/tests/dot_operation_test.cc  |  38 +++
 6 files changed, 369 insertions(+), 65 deletions(-)

diff --git a/tensorflow/compiler/xla/client/lib/BUILD b/tensorflow/compiler/xla/client/lib/BUILD
index 27f9c61848..ded072faf7 100644
--- a/tensorflow/compiler/xla/client/lib/BUILD
+++ b/tensorflow/compiler/xla/client/lib/BUILD
@@ -214,12 +214,16 @@ cc_library(
         ":arithmetic",
         ":constants",
         "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla:status",
         "//tensorflow/compiler/xla:status_macros",
         "//tensorflow/compiler/xla:statusor",
         "//tensorflow/compiler/xla:types",
         "//tensorflow/compiler/xla:util",
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/client:xla_builder",
+        "@com_google_absl//absl/algorithm:container",
+        "@com_google_absl//absl/container:flat_hash_set",
+        "@com_google_absl//absl/strings",
         "@com_google_absl//absl/types:span",
     ],
 )
@@ -230,12 +234,16 @@ xla_test(
     deps = [
         ":matrix",
         ":slicing",
+        "//tensorflow/compiler/xla:status",
+        "//tensorflow/compiler/xla:status_macros",
+        "//tensorflow/compiler/xla:statusor",
         "//tensorflow/compiler/xla:test",
         "//tensorflow/compiler/xla:types",
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/client:xla_builder",
         "//tensorflow/compiler/xla/tests:client_library_test_base",
         "//tensorflow/compiler/xla/tests:xla_internal_test_main",
+        "@com_google_absl//absl/strings",
     ],
 )
 
@@ -274,7 +282,6 @@ cc_library(
         "//tensorflow/compiler/xla:util",
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/client:xla_builder",
-        "//tensorflow/core:lib",
         "@com_google_absl//absl/base",
     ],
 )
@@ -326,12 +333,7 @@ cc_library(
     srcs = ["slicing.cc"],
     hdrs = ["slicing.h"],
     deps = [
-        "//tensorflow/compiler/xla:shape_util",
-        "//tensorflow/compiler/xla:status_macros",
-        "//tensorflow/compiler/xla:statusor",
         "//tensorflow/compiler/xla:types",
-        "//tensorflow/compiler/xla:util",
-        "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/client:xla_builder",
         "@com_google_absl//absl/types:span",
     ],
@@ -345,7 +347,6 @@ xla_test(
         "//tensorflow/compiler/xla:literal_util",
         "//tensorflow/compiler/xla:test",
         "//tensorflow/compiler/xla:types",
-        "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/client:xla_builder",
         "//tensorflow/compiler/xla/tests:client_library_test_base",
         "//tensorflow/compiler/xla/tests:xla_internal_test_main",
@@ -372,7 +373,6 @@ xla_test(
         ":sorting",
         "//tensorflow/compiler/xla:test",
         "//tensorflow/compiler/xla:types",
-        "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/client:xla_builder",
         "//tensorflow/compiler/xla/tests:client_library_test_base",
         "//tensorflow/compiler/xla/tests:xla_internal_test_main",
@@ -466,18 +466,13 @@ xla_test(
         ":triangular_solve",
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:literal",
-        "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:statusor",
         "//tensorflow/compiler/xla:test",
         "//tensorflow/compiler/xla:types",
-        "//tensorflow/compiler/xla:xla_data_proto",
-        "//tensorflow/compiler/xla/client:global_data",
-        "//tensorflow/compiler/xla/client:local_client",
         "//tensorflow/compiler/xla/client:xla_builder",
         "//tensorflow/compiler/xla/tests:client_library_test_base",
         "//tensorflow/compiler/xla/tests:literal_test_util",
         "//tensorflow/compiler/xla/tests:xla_internal_test_main",
-        "//tensorflow/core:lib",
         "//tensorflow/core:test",
     ],
 )
diff --git a/tensorflow/compiler/xla/client/lib/matrix.cc b/tensorflow/compiler/xla/client/lib/matrix.cc
index dcec2139e4..a5aea96090 100644
--- a/tensorflow/compiler/xla/client/lib/matrix.cc
+++ b/tensorflow/compiler/xla/client/lib/matrix.cc
@@ -15,17 +15,25 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/client/lib/matrix.h"
 
+#include <array>
 #include <numeric>
 #include <vector>
 
+#include "absl/algorithm/container.h"
+#include "absl/container/flat_hash_set.h"
+#include "absl/strings/ascii.h"
+#include "absl/strings/str_split.h"
+#include "absl/strings/string_view.h"
 #include "absl/types/span.h"
 #include "tensorflow/compiler/xla/client/lib/arithmetic.h"
 #include "tensorflow/compiler/xla/client/lib/constants.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/shape_util.h"
+#include "tensorflow/compiler/xla/status.h"
 #include "tensorflow/compiler/xla/status_macros.h"
 #include "tensorflow/compiler/xla/statusor.h"
 #include "tensorflow/compiler/xla/util.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
 
 namespace xla {
 
@@ -91,77 +99,224 @@ XlaOp UpperTriangle(XlaOp x) { return Triangle(x, false); }
 
 XlaOp LowerTriangle(XlaOp x) { return Triangle(x, true); }
 
-XlaOp BatchDot(XlaOp x, XlaOp y, PrecisionConfig::Precision precision) {
+Status ValidateEinsumNumericDimensions(absl::Span<const int64> x_config,
+                                       absl::Span<const int64> y_config,
+                                       absl::Span<const int64> output_config) {
+  for (auto dim : output_config) {
+    if (absl::c_linear_search(x_config, dim) ||
+        absl::c_linear_search(y_config, dim)) {
+      if (absl::c_count(output_config, dim) > 1) {
+        return InvalidArgument("Einsum has repeated output dimension.");
+      }
+      continue;
+    }
+    return InvalidArgument(
+        "Einsum has output dimension without corresponding input dimension.");
+  }
+  for (auto dim : x_config) {
+    if (absl::c_linear_search(y_config, dim) ||
+        absl::c_linear_search(output_config, dim)) {
+      if (absl::c_count(x_config, dim) > 1) {
+        return InvalidArgument("Einsum has repeated lhs dimension.");
+      }
+      continue;
+    }
+    return InvalidArgument(
+        "Einsum has lhs dimension without corresponding rhs or output "
+        "dimension.");
+  }
+  for (auto dim : y_config) {
+    if (absl::c_linear_search(x_config, dim) ||
+        absl::c_linear_search(output_config, dim)) {
+      if (absl::c_count(y_config, dim) > 1) {
+        return InvalidArgument("Einsum has repeated rhs dimension.");
+      }
+      continue;
+    }
+    return InvalidArgument(
+        "Einsum has rhs dimension without corresponding lhs or output "
+        "dimension.");
+  }
+  return Status::OK();
+}
+
+xla::XlaOp Einsum(xla::XlaOp x, absl::Span<const int64> x_config, xla::XlaOp y,
+                  absl::Span<const int64> y_config,
+                  absl::Span<const int64> output_config,
+                  xla::PrecisionConfig::Precision precision) {
   XlaBuilder* builder = x.builder();
   return builder->ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
-    TF_ASSIGN_OR_RETURN(Shape x_shape, builder->GetShape(x));
-    TF_ASSIGN_OR_RETURN(Shape y_shape, builder->GetShape(y));
+    TF_RETURN_IF_ERROR(
+        ValidateEinsumNumericDimensions(x_config, y_config, output_config));
+    const int64 x_rank = x_config.size();
+    const int64 y_rank = y_config.size();
+    const int64 output_rank = output_config.size();
+    absl::flat_hash_set<int64> x_map;
+    absl::flat_hash_set<int64> y_map;
+    absl::flat_hash_set<int64> output_map;
 
-    // Check that both tensors have the same number of dimensions. There must be
-    // at least two (the batch dimensions can be empty).
-    if (x_shape.rank() != y_shape.rank()) {
-      return InvalidArgument(
-          "Arguments to BatchDot have different ranks: %s vs. %s",
-          ShapeUtil::HumanString(x_shape), ShapeUtil::HumanString(y_shape));
+    auto find = [&](const absl::flat_hash_set<int64>& map, int64 d) {
+      return map.count(d) != 0;
+    };
+
+    auto insert = [&](absl::flat_hash_set<int64>& map, char d) {
+      CHECK(!find(map, d));
+      map.insert(d);
+    };
+
+    for (auto d : x_config) {
+      insert(x_map, d);
     }
-    const int ndims = x_shape.rank();
-    if (ndims < 2) {
-      return InvalidArgument(
-          "Arguments to BatchDot must have rank >= 2: got %d", ndims);
+
+    for (auto d : y_config) {
+      insert(y_map, d);
     }
 
-    // The batch dimensions must be equal and the matrix dimensions must be
-    // valid.
-    std::vector<int64> batch_dimension_numbers;
-    for (int i = 0; i < ndims - 2; ++i) {
-      if (x_shape.dimensions(i) != y_shape.dimensions(i)) {
-        return InvalidArgument(
-            "Dimension %d of inputs to BatchDot must be equal: shapes %s vs %s",
-            i, ShapeUtil::HumanString(x_shape),
-            ShapeUtil::HumanString(y_shape));
-      }
-      batch_dimension_numbers.push_back(i);
+    for (auto d : output_config) {
+      insert(output_map, d);
     }
 
-    int x_inner_dim = ndims - 1;
-    int y_inner_dim = ndims - 2;
-    if (x_shape.dimensions(x_inner_dim) != y_shape.dimensions(y_inner_dim)) {
-      return InvalidArgument(
-          "Dimensions %d and %d of arguments to BatchDot must be equal: "
-          "shapes %s vs %s",
-          x_inner_dim, y_inner_dim, ShapeUtil::HumanString(x_shape),
-          ShapeUtil::HumanString(y_shape));
+    DotDimensionNumbers dnums;
+    std::vector<int64> lhs_outer_dims;
+    auto is_batch_dim = [&](int64 d) {
+      return find(x_map, d) && find(y_map, d) && find(output_map, d);
+    };
+    auto is_contracting = [&](int64 d) {
+      return find(x_map, d) && find(y_map, d);
+    };
+    auto rhs_dimension_number = [&](int64 d) {
+      return absl::c_find(y_config, d) - y_config.begin();
+    };
+    for (int64 i = 0; i < x_rank; ++i) {
+      auto dim_name = x_config[i];
+      if (is_batch_dim(dim_name)) {
+        dnums.add_lhs_batch_dimensions(i);
+        dnums.add_rhs_batch_dimensions(rhs_dimension_number(dim_name));
+      } else if (is_contracting(dim_name)) {
+        dnums.add_lhs_contracting_dimensions(i);
+        dnums.add_rhs_contracting_dimensions(rhs_dimension_number(dim_name));
+      } else {
+        lhs_outer_dims.push_back(i);
+      }
     }
 
-    // Check for zero lhs/rhs dim size.
-    if (ShapeUtil::IsZeroElementArray(x_shape) ||
-        ShapeUtil::IsZeroElementArray(y_shape)) {
-      std::vector<int64> dimensions(batch_dimension_numbers.size());
-      for (int i = 0; i < batch_dimension_numbers.size(); ++i) {
-        dimensions[i] = x_shape.dimensions(batch_dimension_numbers[i]);
+    std::vector<int64> rhs_outer_dims;
+    for (int64 i = 0; i < y_rank; ++i) {
+      auto dim_name = y_config[i];
+      if (!is_batch_dim(dim_name) && !is_contracting(dim_name)) {
+        rhs_outer_dims.push_back(i);
       }
-      int x_outer_dim = ndims - 2;
-      int y_outer_dim = ndims - 1;
-      dimensions.push_back(x_shape.dimensions(x_outer_dim));
-      dimensions.push_back(y_shape.dimensions(y_outer_dim));
-      return Broadcast(
-          ConstantLiteral(builder, LiteralUtil::Zero(x_shape.element_type())),
-          dimensions);
+    }
+
+    auto output_dimension_number = [&](char d) {
+      return absl::c_find(output_config, d) - output_config.begin();
+    };
+
+    std::vector<int64> output_dims;
+    output_dims.reserve(output_rank);
+    for (auto d : dnums.lhs_batch_dimensions()) {
+      output_dims.push_back(output_dimension_number(x_config[d]));
+    }
+    for (auto d : lhs_outer_dims) {
+      output_dims.push_back(output_dimension_number(x_config[d]));
+    }
+    for (auto d : rhs_outer_dims) {
+      output_dims.push_back(output_dimension_number(y_config[d]));
+    }
+
+    std::vector<int64> transpose_dims(output_rank);
+    for (int64 i = 0; i < output_rank; ++i) {
+      transpose_dims[output_dims[i]] = i;
     }
 
     PrecisionConfig precision_proto;
     precision_proto.add_operand_precision(precision);
     precision_proto.add_operand_precision(precision);
+    return Transpose(DotGeneral(x, y, dnums, &precision_proto), transpose_dims);
+  });
+}
+
+XlaOp BatchDot(XlaOp x, XlaOp y, PrecisionConfig::Precision precision) {
+  XlaBuilder* builder = x.builder();
+  return builder->ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    TF_ASSIGN_OR_RETURN(Shape x_shape, builder->GetShape(x));
+    TF_ASSIGN_OR_RETURN(Shape y_shape, builder->GetShape(y));
 
-    DotDimensionNumbers dot_dnums;
-    dot_dnums.add_lhs_contracting_dimensions(x_inner_dim);
-    dot_dnums.add_rhs_contracting_dimensions(y_inner_dim);
-    for (auto batch_dimension_number : batch_dimension_numbers) {
-      dot_dnums.add_lhs_batch_dimensions(batch_dimension_number);
-      dot_dnums.add_rhs_batch_dimensions(batch_dimension_number);
+    // The batch dimensions must be equal and the matrix dimensions must be
+    // valid.
+    std::vector<int64> batch_dimension_numbers;
+    const int ndims = x_shape.rank();
+    batch_dimension_numbers.reserve(ndims - 2);
+    for (int i = 0; i < ndims - 2; ++i) {
+      batch_dimension_numbers.push_back(i);
     }
+    std::vector<int64> x_config = batch_dimension_numbers;
+    x_config.push_back(ndims - 2);
+    x_config.push_back(ndims);
+    std::vector<int64> y_config = batch_dimension_numbers;
+    y_config.push_back(ndims);
+    y_config.push_back(ndims - 1);
+    std::vector<int64> output_config = batch_dimension_numbers;
+    output_config.push_back(ndims - 2);
+    output_config.push_back(ndims - 1);
+    return Einsum(x, x_config, y, y_config, output_config, precision);
+  });
+}
 
-    return DotGeneral(x, y, dot_dnums, &precision_proto);
+StatusOr<std::array<std::vector<int64>, 3>> ParseEinsumString(
+    absl::string_view einsum_config) {
+  std::array<std::vector<int64>, 3> einsum_config_numeric;
+  std::vector<absl::string_view> main_split =
+      absl::StrSplit(einsum_config, ',');
+
+  if (main_split.size() != 2) {
+    return InvalidArgument("Expected one \",\" in einsum_config.");
+  }
+
+  auto maybe_invalid_character = [](char d) {
+    if (absl::ascii_isalpha(d)) {
+      return Status::OK();
+    }
+    if (d == '.') {
+      return InvalidArgument("Unsupported \"...\" or \".\" in einsum config.");
+    }
+    return InvalidArgument("Unexpected character in einsum config.");
+  };
+
+  auto& x_config = einsum_config_numeric[0];
+  x_config.reserve(main_split[0].size());
+  for (auto d : main_split[0]) {
+    TF_RETURN_IF_ERROR(maybe_invalid_character(d));
+    x_config.push_back(static_cast<int64>(d));
+  }
+  std::vector<absl::string_view> y_output_split =
+      absl::StrSplit(main_split[1], "->");
+  if (y_output_split.size() != 2) {
+    return InvalidArgument("Expected one \"->\" in einsum_config.");
+  }
+  auto& y_config = einsum_config_numeric[1];
+  y_config.reserve(y_output_split[0].size());
+  for (auto d : y_output_split[0]) {
+    TF_RETURN_IF_ERROR(maybe_invalid_character(d));
+    y_config.push_back(static_cast<int64>(d));
+  }
+  auto& output_config = einsum_config_numeric[2];
+  output_config.reserve(y_output_split[1].size());
+  for (auto d : y_output_split[1]) {
+    TF_RETURN_IF_ERROR(maybe_invalid_character(d));
+    output_config.push_back(static_cast<int64>(d));
+  }
+  return einsum_config_numeric;
+}
+
+XlaOp Einsum(XlaOp x, XlaOp y, absl::string_view einsum_config,
+             PrecisionConfig::Precision precision) {
+  XlaBuilder* builder = x.builder();
+  return builder->ReportErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    TF_ASSIGN_OR_RETURN(auto einsum_config_numeric,
+                        ParseEinsumString(einsum_config));
+    return Einsum(x, einsum_config_numeric[0], y, einsum_config_numeric[1],
+                  einsum_config_numeric[2], precision);
   });
 }
 
diff --git a/tensorflow/compiler/xla/client/lib/matrix.h b/tensorflow/compiler/xla/client/lib/matrix.h
index 916cd83748..491f1eab4c 100644
--- a/tensorflow/compiler/xla/client/lib/matrix.h
+++ b/tensorflow/compiler/xla/client/lib/matrix.h
@@ -16,7 +16,11 @@ limitations under the License.
 #ifndef TENSORFLOW_COMPILER_XLA_CLIENT_LIB_MATRIX_H_
 #define TENSORFLOW_COMPILER_XLA_CLIENT_LIB_MATRIX_H_
 
+#include <array>
+#include "absl/strings/string_view.h"
+#include "absl/types/span.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
+#include "tensorflow/compiler/xla/statusor.h"
 #include "tensorflow/compiler/xla/types.h"
 #include "tensorflow/compiler/xla/xla_data.pb.h"
 
@@ -65,6 +69,40 @@ xla::XlaOp BatchDot(
     xla::XlaOp x, xla::XlaOp y,
     xla::PrecisionConfig::Precision precision = xla::PrecisionConfig::DEFAULT);
 
+// Parse an einsum string into dimension numbers:
+//   "ab,cb->ac"
+// becomes:
+//   {{0, 1},{2, 1},{0, 2}}
+//
+// NOTE: This function is meant for testing, there is no need to call it
+// directly.
+
+StatusOr<std::array<std::vector<int64>, 3>> ParseEinsumString(
+    absl::string_view einsum_config);
+
+// Determine if each dimension label is in at least two inputs.
+//
+// NOTE: This function is meant for testing, there is no need to call it
+// directly.
+Status ValidateEinsumNumericDimensions(absl::Span<const int64> x_config,
+                                       absl::Span<const int64> y_config,
+                                       absl::Span<const int64> output_config);
+
+// Supports two operand einsum notation like "ab,cb->ac".
+xla::XlaOp Einsum(
+    xla::XlaOp x, xla::XlaOp y, absl::string_view einsum_config,
+    xla::PrecisionConfig::Precision precision = xla::PrecisionConfig::DEFAULT);
+
+// Same as above but supporting numeric labels on dimensins. So "ab,cb->ac"
+// becomes:
+//   x_config = {0, 1}
+//   y_config = {2, 1}
+//   output_config = {0, 2}
+xla::XlaOp Einsum(
+    xla::XlaOp x, absl::Span<const int64> x_config, xla::XlaOp y,
+    absl::Span<const int64> y_config, absl::Span<const int64> output_config,
+    xla::PrecisionConfig::Precision precision = xla::PrecisionConfig::DEFAULT);
+
 // Transposes a stack of matrices `x` by swapping the last two dimensions.
 xla::XlaOp TransposeInMinorDims(xla::XlaOp x);
 
diff --git a/tensorflow/compiler/xla/client/lib/matrix_test.cc b/tensorflow/compiler/xla/client/lib/matrix_test.cc
index 0593a7517a..79cf529ee9 100644
--- a/tensorflow/compiler/xla/client/lib/matrix_test.cc
+++ b/tensorflow/compiler/xla/client/lib/matrix_test.cc
@@ -15,8 +15,11 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/client/lib/matrix.h"
 
+#include "absl/strings/string_view.h"
 #include "tensorflow/compiler/xla/client/lib/slicing.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
+#include "tensorflow/compiler/xla/status.h"
+#include "tensorflow/compiler/xla/statusor.h"
 #include "tensorflow/compiler/xla/test.h"
 #include "tensorflow/compiler/xla/tests/client_library_test_base.h"
 #include "tensorflow/compiler/xla/tests/test_macros.h"
@@ -101,5 +104,78 @@ XLA_TEST_F(MatrixTest, RowBatchDot) {
   ComputeAndCompareR3<float>(&builder, {{{33}}, {{292}}},
                              {a_data.get(), row_data.get(), index_data.get()});
 }
+
+XLA_TEST_F(MatrixTest, Einsum) {
+  XlaBuilder builder(TestName());
+
+  int n = 4;
+
+  XlaOp a, row, index;
+  auto a_data =
+      CreateR3Parameter<float>(BatchedAValsFull(), 0, "a", &builder, &a);
+  auto row_data = CreateR3Parameter<float>({{{9, 1, 0, 0}}, {{2, 4, 0, 0}}}, 1,
+                                           "row", &builder, &row);
+  // Select {{3, 6, 0, 1}, {24, 61,  82,  48}} out of BatchedAValsFull().
+  auto index_data = CreateR0Parameter<int>(1, 2, "index", &builder, &index);
+
+  auto l_index = DynamicSliceInMinorDims(
+      a, {index, ConstantR0<int32>(&builder, 0)}, {1, n});
+  Einsum(l_index, row, "abc,adc->abd");
+
+  ComputeAndCompareR3<float>(&builder, {{{33}}, {{292}}},
+                             {a_data.get(), row_data.get(), index_data.get()});
+}
+
+XLA_TEST_F(MatrixTest, ParseEinsumString) {
+  auto to_vec = [](absl::string_view s) {
+    std::vector<int64> v;
+    v.reserve(s.size());
+    for (auto c : s) {
+      v.push_back(int64{c});
+    }
+    return v;
+  };
+
+  auto to_string = [&](absl::string_view x, absl::string_view y,
+                       absl::string_view o) {
+    return absl::StrCat(x, ",", y, "->", o);
+  };
+
+  std::vector<std::vector<string>> good_test_cases = {{"ab", "bc", "ac"},
+                                                      {"Bab", "Bbc", "Bac"},
+                                                      {"ab", "cd", "dcba"},
+                                                      {"abc", "abd", "cbd"}};
+  for (auto test_case : good_test_cases) {
+    auto parse_result_or_status =
+        ParseEinsumString(to_string(test_case[0], test_case[1], test_case[2]));
+    EXPECT_TRUE(parse_result_or_status.status().ok());
+    auto parse_result = parse_result_or_status.ValueOrDie();
+    for (int i = 0; i < 3; ++i) {
+      EXPECT_EQ(parse_result[i], to_vec(test_case[i]));
+    }
+    EXPECT_TRUE(ValidateEinsumNumericDimensions(
+                    parse_result[0], parse_result[1], parse_result[2])
+                    .ok());
+  }
+
+  std::vector<string> einsum_strings_that_fail_parsing = {
+      "", "a", "ab->ba", "ab,bc,cd->ad", "a...b,bc->a...c"};
+  for (auto test_case : einsum_strings_that_fail_parsing) {
+    auto parse_result_or_status = ParseEinsumString(test_case);
+    EXPECT_FALSE(parse_result_or_status.status().ok());
+  }
+
+  std::vector<string> einsum_strings_that_fail_numeric_validation = {
+      "a,b->c", "ab,bc->acd", "abz,bc->ac", "ab,bcz->ac"};
+  for (auto test_case : einsum_strings_that_fail_numeric_validation) {
+    auto parse_result_or_status = ParseEinsumString(test_case);
+    EXPECT_TRUE(parse_result_or_status.status().ok());
+    auto parse_result = parse_result_or_status.ValueOrDie();
+    EXPECT_FALSE(ValidateEinsumNumericDimensions(
+                     parse_result[0], parse_result[1], parse_result[2])
+                     .ok());
+  }
+}
+
 }  // namespace
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index 4e7480c9af..7e05a7956e 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -730,6 +730,7 @@ xla_test(
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla/client:local_client",
         "//tensorflow/compiler/xla/client:xla_builder",
+        "//tensorflow/compiler/xla/client/lib:matrix",
         "//tensorflow/compiler/xla/tests:client_library_test_base",
         "//tensorflow/compiler/xla/tests:hlo_test_base",
         "//tensorflow/compiler/xla/tests:literal_test_util",
@@ -788,6 +789,7 @@ xla_test(
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla/client:local_client",
         "//tensorflow/compiler/xla/client:xla_builder",
+        "//tensorflow/compiler/xla/client/lib:matrix",
         "//tensorflow/compiler/xla/tests:client_library_test_base",
         "//tensorflow/compiler/xla/tests:hlo_test_base",
         "//tensorflow/compiler/xla/tests:literal_test_util",
diff --git a/tensorflow/compiler/xla/tests/dot_operation_test.cc b/tensorflow/compiler/xla/tests/dot_operation_test.cc
index 4576a04ec3..f740f48158 100644
--- a/tensorflow/compiler/xla/tests/dot_operation_test.cc
+++ b/tensorflow/compiler/xla/tests/dot_operation_test.cc
@@ -19,6 +19,7 @@ limitations under the License.
 #include "absl/strings/str_cat.h"
 #include "tensorflow/compiler/xla/array2d.h"
 #include "tensorflow/compiler/xla/array3d.h"
+#include "tensorflow/compiler/xla/client/lib/matrix.h"
 #include "tensorflow/compiler/xla/client/local_client.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/primitive_util.h"
@@ -1157,6 +1158,43 @@ XLA_TEST_F(DotOperationTest, DotRank2AndRank2NonDefaultContractionDims) {
   ComputeAndCompareR2<float>(&builder, expected, {}, error_spec_);
 }
 
+using EinsumParamType =
+    std::tuple<std::vector<int64>, std::vector<int64>, string>;
+class EinsumTest : public DotOperationTest,
+                   public ::testing::WithParamInterface<EinsumParamType> {};
+XLA_TEST_P(EinsumTest, SimpleEinsumTest) {
+  XlaBuilder builder(TestName());
+  auto x = AddParam(
+      MakeFakeLiteral(ShapeUtil::MakeShape(F32, std::get<0>(GetParam())))
+          .ValueOrDie(),
+      &builder);
+  auto y = AddParam(
+      MakeFakeLiteral(ShapeUtil::MakeShape(F32, std::get<1>(GetParam())))
+          .ValueOrDie(),
+      &builder);
+  Einsum(x, y, std::get<2>(GetParam()));
+  ComputeAndCompare(&builder, {}, ErrorSpec{1e-3, 1e-3});
+}
+
+std::vector<EinsumParamType> GetEinsumTestCases() {
+  using v = std::vector<int64>;
+  using p = EinsumParamType;
+  std::vector<p> test_cases = {
+      p{v{5, 6}, v{6, 7}, "mk,kn->mn"},
+      p{v{5, 6}, v{6, 7}, "mk,kn->nm"},
+      p{v{5, 6, 11}, v{6, 11, 7}, "mkB,kBn->nmB"},
+      p{v{31, 55, 11}, v{55, 11, 29}, "mkB,kBn->nmB"},
+      p{v{31, 55, 11}, v{55, 11, 29}, "mkB,kBn->Bnm"},
+      p{v{8, 55, 11, 3}, v{55, 11, 3, 29}, "mkBC,kBCn->BCnm"},
+      p{v{5, 6}, v{6, 7}, "ab,cd->dcba"},
+      p{v{6}, v{6, 7}, "b,bc->c"},
+  };
+  return test_cases;
+}
+
+INSTANTIATE_TEST_CASE_P(Einsum, EinsumTest,
+                        ::testing::ValuesIn(GetEinsumTestCases()));
+
 class DotOperationTextTest : public HloTestBase {};
 
 XLA_TEST_F(DotOperationTextTest, DotReorderedDotDims) {
-- 
GitLab


From 4e15557ac4c92f219f51ed94883d1e47dca88417 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 22 Jan 2019 14:16:37 -0800
Subject: [PATCH 1079/1095] Update references to bounds_checks.h

PiperOrigin-RevId: 230408522
---
 .../compiler/jit/mark_for_compilation_pass.cc |  2 +-
 tensorflow/compiler/jit/xla_cluster_util.cc   |  2 +-
 .../compiler/tf2xla/kernels/concat_op.cc      |  2 +-
 .../tf2xla/kernels/conv_op_helpers.cc         |  2 +-
 .../compiler/tf2xla/kernels/conv_ops.cc       |  2 +-
 .../tf2xla/kernels/dynamic_stitch_op.cc       |  2 +-
 tensorflow/compiler/tf2xla/kernels/fft_ops.cc |  2 +-
 .../compiler/tf2xla/kernels/index_ops.cc      |  2 +-
 .../compiler/tf2xla/kernels/index_ops_cpu.cc  |  2 +-
 tensorflow/compiler/tf2xla/kernels/pack_op.cc |  2 +-
 .../compiler/tf2xla/kernels/pooling_ops.cc    |  2 +-
 .../compiler/tf2xla/kernels/scan_ops.cc       |  2 +-
 .../compiler/tf2xla/kernels/select_op.cc      |  2 +-
 .../compiler/tf2xla/kernels/shape_op.cc       |  2 +-
 .../compiler/tf2xla/kernels/shape_util.cc     |  2 +-
 .../compiler/tf2xla/kernels/stack_ops.cc      |  2 +-
 .../tf2xla/kernels/tensor_array_ops.cc        |  2 +-
 .../tf2xla/kernels/tensor_list_ops.cc         |  2 +-
 .../compiler/tf2xla/kernels/transpose_op.cc   |  2 +-
 .../compiler/tf2xla/kernels/unpack_op.cc      |  2 +-
 .../kernels/masked_matmul_ops.cc              |  2 +-
 .../fused_conv2d_bias_activation_op.cc        |  2 +-
 .../tensor_forest/kernels/tree_utils.h        |  2 +-
 .../core/common_runtime/eval_const_tensor.cc  |  2 +-
 .../core/common_runtime/shape_refiner.cc      |  2 +-
 tensorflow/core/framework/shape_inference.cc  |  2 +-
 tensorflow/core/framework/tensor_shape.cc     |  2 +-
 tensorflow/core/kernels/BUILD                 |  9 ++------
 tensorflow/core/kernels/argmax_op.cc          |  2 +-
 tensorflow/core/kernels/bias_op.cc            |  2 +-
 tensorflow/core/kernels/bounds_check.h        | 21 -------------------
 tensorflow/core/kernels/concat_op.cc          |  4 ++--
 tensorflow/core/kernels/constant_op.cc        |  2 +-
 tensorflow/core/kernels/conv_ops.cc           |  2 +-
 tensorflow/core/kernels/conv_ops_fused.cc     |  2 +-
 .../kernels/conv_ops_fused_image_transform.cc |  2 +-
 .../core/kernels/conv_ops_using_gemm.cc       |  2 +-
 tensorflow/core/kernels/crop_and_resize_op.cc |  2 +-
 tensorflow/core/kernels/ctc_loss_op.cc        |  2 +-
 tensorflow/core/kernels/cwise_op_select.cc    |  2 +-
 tensorflow/core/kernels/cwise_ops.h           |  2 +-
 tensorflow/core/kernels/decode_bmp_op.cc      |  2 +-
 .../core/kernels/depthwise_conv_grad_op.cc    |  2 +-
 tensorflow/core/kernels/depthwise_conv_op.cc  |  2 +-
 .../core/kernels/dynamic_partition_op.cc      |  2 +-
 .../kernels/dynamic_partition_op_gpu.cu.cc    |  2 +-
 tensorflow/core/kernels/dynamic_stitch_op.cc  |  2 +-
 tensorflow/core/kernels/encode_jpeg_op.cc     |  2 +-
 tensorflow/core/kernels/encode_png_op.cc      |  2 +-
 tensorflow/core/kernels/encode_wav_op.cc      |  2 +-
 .../core/kernels/extract_image_patches_op.cc  |  2 +-
 .../core/kernels/extract_jpeg_shape_op.cc     |  2 +-
 .../core/kernels/extract_volume_patches_op.cc |  2 +-
 tensorflow/core/kernels/gather_functor.h      |  2 +-
 tensorflow/core/kernels/gather_nd_op.cc       |  2 +-
 tensorflow/core/kernels/gather_nd_op.h        |  2 +-
 .../core/kernels/gather_nd_op_cpu_impl.h      |  2 +-
 tensorflow/core/kernels/gather_op.cc          |  2 +-
 tensorflow/core/kernels/image_resizer_state.h |  2 +-
 tensorflow/core/kernels/in_topk_op.cc         |  2 +-
 tensorflow/core/kernels/lookup_table_op.h     |  2 +-
 tensorflow/core/kernels/lrn_op.cc             |  2 +-
 tensorflow/core/kernels/maxpooling_op.cc      |  2 +-
 tensorflow/core/kernels/mkl_concat_op.cc      |  2 +-
 tensorflow/core/kernels/mkl_conv_ops.cc       |  2 +-
 tensorflow/core/kernels/mkl_conv_ops.h        |  2 +-
 tensorflow/core/kernels/mkl_lrn_op.cc         |  2 +-
 .../core/kernels/mkl_pooling_ops_common.cc    |  2 +-
 .../kernels/neon/neon_depthwise_conv_op.cc    |  2 +-
 .../core/kernels/non_max_suppression_op.cc    |  2 +-
 .../core/kernels/resource_variable_ops.cc     |  2 +-
 tensorflow/core/kernels/reverse_op.cc         |  2 +-
 tensorflow/core/kernels/roll_op.cc            |  2 +-
 .../sample_distorted_bounding_box_op.cc       |  2 +-
 .../core/kernels/save_restore_tensor.cc       |  2 +-
 .../core/kernels/save_restore_v2_ops.cc       |  2 +-
 .../core/kernels/scale_and_translate_op.cc    |  2 +-
 tensorflow/core/kernels/scan_ops.cc           |  2 +-
 tensorflow/core/kernels/scatter_functor.h     |  2 +-
 tensorflow/core/kernels/scatter_nd_op.cc      |  2 +-
 tensorflow/core/kernels/scatter_nd_op.h       |  2 +-
 .../core/kernels/scatter_nd_op_cpu_impl.h     |  2 +-
 tensorflow/core/kernels/searchsorted_op.cc    |  2 +-
 .../core/kernels/segment_reduction_ops.cc     |  4 +++-
 tensorflow/core/kernels/shape_ops.h           |  2 +-
 .../core/kernels/spacetobatch_functor.h       |  2 +-
 .../kernels/sparse_tensor_dense_matmul_op.cc  |  2 +-
 .../sparse_tensor_dense_matmul_op_gpu.cu.cc   |  2 +-
 tensorflow/core/kernels/sparse_xent_op.h      |  2 +-
 tensorflow/core/kernels/split_op.cc           |  2 +-
 tensorflow/core/kernels/split_v_op.cc         |  2 +-
 .../core/kernels/stateful_random_ops.cc       |  2 +-
 .../core/kernels/stateless_random_ops.cc      |  2 +-
 tensorflow/core/kernels/strided_slice_op.cc   |  2 +-
 .../core/kernels/strided_slice_op_impl.h      |  2 +-
 tensorflow/core/kernels/substr_op.cc          |  2 +-
 tensorflow/core/kernels/tensor_array_ops.cc   |  2 +-
 tensorflow/core/kernels/training_ops.cc       |  2 +-
 tensorflow/core/kernels/transpose_op.cc       |  2 +-
 tensorflow/core/kernels/unicode_ops.cc        |  2 +-
 tensorflow/core/kernels/unique_op.cc          |  2 +-
 tensorflow/core/kernels/unpack_op.cc          |  2 +-
 tensorflow/core/kernels/where_op.cc           |  2 +-
 tensorflow/core/kernels/where_op_gpu.cu.h     |  2 +-
 tensorflow/core/util/sparse/dim_comparator.h  |  2 +-
 tensorflow/core/util/sparse/sparse_tensor.h   |  2 +-
 tensorflow/core/util/strided_slice_op.cc      |  2 +-
 107 files changed, 110 insertions(+), 134 deletions(-)
 delete mode 100644 tensorflow/core/kernels/bounds_check.h

diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass.cc b/tensorflow/compiler/jit/mark_for_compilation_pass.cc
index cb01690845..234e8052d1 100644
--- a/tensorflow/compiler/jit/mark_for_compilation_pass.cc
+++ b/tensorflow/compiler/jit/mark_for_compilation_pass.cc
@@ -34,6 +34,7 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/util.h"
 #include "tensorflow/core/common_runtime/function.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/graph_def_util.h"
 #include "tensorflow/core/framework/memory_types.h"
 #include "tensorflow/core/framework/node_def.pb.h"
@@ -42,7 +43,6 @@ limitations under the License.
 #include "tensorflow/core/graph/algorithm.h"
 #include "tensorflow/core/graph/control_flow.h"
 #include "tensorflow/core/graph/graph_constructor.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/gtl/cleanup.h"
 #include "tensorflow/core/lib/strings/stringprintf.h"
 #include "tensorflow/core/public/version.h"
diff --git a/tensorflow/compiler/jit/xla_cluster_util.cc b/tensorflow/compiler/jit/xla_cluster_util.cc
index fef28fc810..80993861ab 100644
--- a/tensorflow/compiler/jit/xla_cluster_util.cc
+++ b/tensorflow/compiler/jit/xla_cluster_util.cc
@@ -19,9 +19,9 @@ limitations under the License.
 
 #include "absl/strings/str_cat.h"
 #include "tensorflow/compiler/jit/resource_operation_safety_analysis.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/graph/control_flow.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/util/device_name_utils.h"
 
 namespace tensorflow {
diff --git a/tensorflow/compiler/tf2xla/kernels/concat_op.cc b/tensorflow/compiler/tf2xla/kernels/concat_op.cc
index cd7c7f4a82..91e4d9cea7 100644
--- a/tensorflow/compiler/tf2xla/kernels/concat_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/concat_op.cc
@@ -24,13 +24,13 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/literal_util.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/concat_lib.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/types.h"
diff --git a/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc b/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc
index 61a8933208..5f99b24e22 100644
--- a/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc
+++ b/tensorflow/compiler/tf2xla/kernels/conv_op_helpers.cc
@@ -26,13 +26,13 @@ limitations under the License.
 #include "tensorflow/compiler/xla/client/lib/constants.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/literal_util.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/node_def_util.h"
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_slice.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/conv_grad_ops.h"
 #include "tensorflow/core/kernels/ops_util.h"
 #include "tensorflow/core/util/padding.h"
diff --git a/tensorflow/compiler/tf2xla/kernels/conv_ops.cc b/tensorflow/compiler/tf2xla/kernels/conv_ops.cc
index eafdba876a..52c3c2c4a9 100644
--- a/tensorflow/compiler/tf2xla/kernels/conv_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/conv_ops.cc
@@ -25,13 +25,13 @@ limitations under the License.
 #include "tensorflow/compiler/xla/client/lib/matrix.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/literal_util.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/node_def_util.h"
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_slice.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/conv_grad_ops.h"
 #include "tensorflow/core/kernels/ops_util.h"
 #include "tensorflow/core/util/padding.h"
diff --git a/tensorflow/compiler/tf2xla/kernels/dynamic_stitch_op.cc b/tensorflow/compiler/tf2xla/kernels/dynamic_stitch_op.cc
index 6e6ba21daf..b119997cf3 100644
--- a/tensorflow/compiler/tf2xla/kernels/dynamic_stitch_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/dynamic_stitch_op.cc
@@ -22,10 +22,10 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/literal_util.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 
 namespace tensorflow {
 namespace {
diff --git a/tensorflow/compiler/tf2xla/kernels/fft_ops.cc b/tensorflow/compiler/tf2xla/kernels/fft_ops.cc
index 6df8b5367d..a623585aad 100644
--- a/tensorflow/compiler/tf2xla/kernels/fft_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/fft_ops.cc
@@ -21,12 +21,12 @@ limitations under the License.
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/literal_util.h"
 #include "tensorflow/compiler/xla/util.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_slice.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/conv_grad_ops.h"
 #include "tensorflow/core/kernels/ops_util.h"
 #include "tensorflow/core/util/padding.h"
diff --git a/tensorflow/compiler/tf2xla/kernels/index_ops.cc b/tensorflow/compiler/tf2xla/kernels/index_ops.cc
index 978e9480ea..c1539f48d4 100644
--- a/tensorflow/compiler/tf2xla/kernels/index_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/index_ops.cc
@@ -22,12 +22,12 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/client/lib/arithmetic.h"
 #include "tensorflow/compiler/xla/shape_util.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/kernel_def_builder.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 
 namespace tensorflow {
 XlaArgMinMaxOp::XlaArgMinMaxOp(OpKernelConstruction* ctx, bool is_min)
diff --git a/tensorflow/compiler/tf2xla/kernels/index_ops_cpu.cc b/tensorflow/compiler/tf2xla/kernels/index_ops_cpu.cc
index 30b993045c..e4bbdef648 100644
--- a/tensorflow/compiler/tf2xla/kernels/index_ops_cpu.cc
+++ b/tensorflow/compiler/tf2xla/kernels/index_ops_cpu.cc
@@ -20,12 +20,12 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/client/lib/arithmetic.h"
 #include "tensorflow/compiler/xla/literal_util.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/kernel_def_builder.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 
 namespace tensorflow {
 namespace {
diff --git a/tensorflow/compiler/tf2xla/kernels/pack_op.cc b/tensorflow/compiler/tf2xla/kernels/pack_op.cc
index a9b519d892..426a0941df 100644
--- a/tensorflow/compiler/tf2xla/kernels/pack_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/pack_op.cc
@@ -24,12 +24,12 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/literal_util.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/concat_lib.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/types.h"
diff --git a/tensorflow/compiler/tf2xla/kernels/pooling_ops.cc b/tensorflow/compiler/tf2xla/kernels/pooling_ops.cc
index 06c6cc37ec..23bb050a34 100644
--- a/tensorflow/compiler/tf2xla/kernels/pooling_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/pooling_ops.cc
@@ -26,10 +26,10 @@ limitations under the License.
 #include "tensorflow/compiler/xla/client/xla_computation.h"
 #include "tensorflow/compiler/xla/literal.h"
 #include "tensorflow/compiler/xla/util.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/conv_grad_ops.h"
 #include "tensorflow/core/kernels/pooling_ops_common.h"
 
diff --git a/tensorflow/compiler/tf2xla/kernels/scan_ops.cc b/tensorflow/compiler/tf2xla/kernels/scan_ops.cc
index 4b9e1a578b..daefdfc58a 100644
--- a/tensorflow/compiler/tf2xla/kernels/scan_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/scan_ops.cc
@@ -23,13 +23,13 @@ limitations under the License.
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/client/xla_computation.h"
 #include "tensorflow/compiler/xla/literal.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/partial_tensor_shape.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/concat_lib.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/types.h"
diff --git a/tensorflow/compiler/tf2xla/kernels/select_op.cc b/tensorflow/compiler/tf2xla/kernels/select_op.cc
index 9e4c57c9bf..aaf8c6075d 100644
--- a/tensorflow/compiler/tf2xla/kernels/select_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/select_op.cc
@@ -20,8 +20,8 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/kernel_def_builder.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 
 namespace tensorflow {
 namespace {
diff --git a/tensorflow/compiler/tf2xla/kernels/shape_op.cc b/tensorflow/compiler/tf2xla/kernels/shape_op.cc
index 85b0367f73..31d4cc1316 100644
--- a/tensorflow/compiler/tf2xla/kernels/shape_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/shape_op.cc
@@ -22,9 +22,9 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/client/lib/constants.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/kernel_def_builder.h"
 #include "tensorflow/core/framework/tensor_shape.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 
 namespace tensorflow {
 namespace {
diff --git a/tensorflow/compiler/tf2xla/kernels/shape_util.cc b/tensorflow/compiler/tf2xla/kernels/shape_util.cc
index 76ea5f5255..b18e3f965c 100644
--- a/tensorflow/compiler/tf2xla/kernels/shape_util.cc
+++ b/tensorflow/compiler/tf2xla/kernels/shape_util.cc
@@ -17,7 +17,7 @@ limitations under the License.
 
 #include <limits>
 
-#include "tensorflow/core/kernels/bounds_check.h"
+#include "tensorflow/core/framework/bounds_check.h"
 
 namespace tensorflow {
 
diff --git a/tensorflow/compiler/tf2xla/kernels/stack_ops.cc b/tensorflow/compiler/tf2xla/kernels/stack_ops.cc
index d0c5231e84..b6c96b1f58 100644
--- a/tensorflow/compiler/tf2xla/kernels/stack_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/stack_ops.cc
@@ -24,13 +24,13 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/literal.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/partial_tensor_shape.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/concat_lib.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/types.h"
diff --git a/tensorflow/compiler/tf2xla/kernels/tensor_array_ops.cc b/tensorflow/compiler/tf2xla/kernels/tensor_array_ops.cc
index 22e64789a3..77a3e5c001 100644
--- a/tensorflow/compiler/tf2xla/kernels/tensor_array_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/tensor_array_ops.cc
@@ -27,13 +27,13 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_resource.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/literal.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/partial_tensor_shape.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/concat_lib.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/types.h"
diff --git a/tensorflow/compiler/tf2xla/kernels/tensor_list_ops.cc b/tensorflow/compiler/tf2xla/kernels/tensor_list_ops.cc
index 5dda62fd49..6757a21190 100644
--- a/tensorflow/compiler/tf2xla/kernels/tensor_list_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/tensor_list_ops.cc
@@ -24,13 +24,13 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/literal.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/partial_tensor_shape.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/concat_lib.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/types.h"
diff --git a/tensorflow/compiler/tf2xla/kernels/transpose_op.cc b/tensorflow/compiler/tf2xla/kernels/transpose_op.cc
index c9b324a243..76793d677b 100644
--- a/tensorflow/compiler/tf2xla/kernels/transpose_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/transpose_op.cc
@@ -24,9 +24,9 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/kernel_def_builder.h"
 #include "tensorflow/core/framework/register_types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 
 namespace tensorflow {
 namespace {
diff --git a/tensorflow/compiler/tf2xla/kernels/unpack_op.cc b/tensorflow/compiler/tf2xla/kernels/unpack_op.cc
index 8671632976..2fc5619de7 100644
--- a/tensorflow/compiler/tf2xla/kernels/unpack_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/unpack_op.cc
@@ -24,12 +24,12 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/client/xla_builder.h"
 #include "tensorflow/compiler/xla/literal.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/concat_lib.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/types.h"
diff --git a/tensorflow/contrib/factorization/kernels/masked_matmul_ops.cc b/tensorflow/contrib/factorization/kernels/masked_matmul_ops.cc
index a8c5d0763c..68078ba8bb 100644
--- a/tensorflow/contrib/factorization/kernels/masked_matmul_ops.cc
+++ b/tensorflow/contrib/factorization/kernels/masked_matmul_ops.cc
@@ -19,12 +19,12 @@
 #include <numeric>
 #include <vector>
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/threadpool.h"
 
diff --git a/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc b/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc
index c541c71f99..b6b75ffa24 100644
--- a/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc
+++ b/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc
@@ -19,13 +19,13 @@ limitations under the License.
 
 #include "tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.h"
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_slice.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/conv_2d.h"
 #include "tensorflow/core/kernels/ops_util.h"
 #include "tensorflow/core/lib/core/errors.h"
diff --git a/tensorflow/contrib/tensor_forest/kernels/tree_utils.h b/tensorflow/contrib/tensor_forest/kernels/tree_utils.h
index e04eb60f9b..774da472f1 100644
--- a/tensorflow/contrib/tensor_forest/kernels/tree_utils.h
+++ b/tensorflow/contrib/tensor_forest/kernels/tree_utils.h
@@ -18,10 +18,10 @@
 #include <limits>
 
 #include "tensorflow/contrib/tensor_forest/kernels/data_spec.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/random/distribution_sampler.h"
 #include "tensorflow/core/lib/random/simple_philox.h"
 #include "tensorflow/core/lib/strings/strcat.h"
diff --git a/tensorflow/core/common_runtime/eval_const_tensor.cc b/tensorflow/core/common_runtime/eval_const_tensor.cc
index 87749da7af..fb51e2dec3 100644
--- a/tensorflow/core/common_runtime/eval_const_tensor.cc
+++ b/tensorflow/core/common_runtime/eval_const_tensor.cc
@@ -19,11 +19,11 @@ limitations under the License.
 
 #include "tensorflow/core/common_runtime/graph_runner.h"
 #include "tensorflow/core/common_runtime/shape_refiner.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/versions.pb.h"
 #include "tensorflow/core/graph/graph.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 
 namespace tensorflow {
 
diff --git a/tensorflow/core/common_runtime/shape_refiner.cc b/tensorflow/core/common_runtime/shape_refiner.cc
index 9488a44778..8f28d27903 100644
--- a/tensorflow/core/common_runtime/shape_refiner.cc
+++ b/tensorflow/core/common_runtime/shape_refiner.cc
@@ -20,6 +20,7 @@ limitations under the License.
 #include <vector>
 
 #include "tensorflow/core/common_runtime/eval_const_tensor.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/common_shape_fns.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/framework/tensor.h"
@@ -27,7 +28,6 @@ limitations under the License.
 #include "tensorflow/core/framework/versions.pb.h"
 #include "tensorflow/core/graph/algorithm.h"
 #include "tensorflow/core/graph/graph_constructor.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/gtl/stl_util.h"
 #include "tensorflow/core/public/session.h"
diff --git a/tensorflow/core/framework/shape_inference.cc b/tensorflow/core/framework/shape_inference.cc
index 4dcc80680f..bffbee50b5 100644
--- a/tensorflow/core/framework/shape_inference.cc
+++ b/tensorflow/core/framework/shape_inference.cc
@@ -14,10 +14,10 @@ limitations under the License.
 ==============================================================================*/
 #include "tensorflow/core/framework/shape_inference.h"
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/node_def.pb_text.h"
 #include "tensorflow/core/framework/partial_tensor_shape.h"
 #include "tensorflow/core/framework/tensor_shape.pb.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/strings/numbers.h"
 #include "tensorflow/core/lib/strings/scanner.h"
diff --git a/tensorflow/core/framework/tensor_shape.cc b/tensorflow/core/framework/tensor_shape.cc
index 5e0b976e17..be451d1e31 100644
--- a/tensorflow/core/framework/tensor_shape.cc
+++ b/tensorflow/core/framework/tensor_shape.cc
@@ -15,8 +15,8 @@ limitations under the License.
 
 #include "tensorflow/core/framework/tensor_shape.h"
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/tensor_shape.pb.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 9fcd4106fa..049d429d36 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -649,13 +649,10 @@ cc_library(
     ],
 )
 
-cc_library(
+alias(
     name = "bounds_check",
-    hdrs = ["bounds_check.h"],
+    actual = "//tensorflow/core:framework_bounds_check",
     visibility = [":friends"],
-    deps = [
-        "//tensorflow/core:framework_bounds_check",
-    ],
 )
 
 # Private support libraries ---------------------------------------------------
@@ -5439,7 +5436,6 @@ filegroup(
     srcs = [
         "avgpooling_op.h",
         "batch_util.h",
-        "bounds_check.h",
         "cwise_ops.h",
         "cwise_ops_common.h",
         "cwise_ops_gradients.h",
@@ -5480,7 +5476,6 @@ filegroup(
         "assign_op.h",
         "bias_op.cc",
         "bias_op.h",
-        "bounds_check.h",
         "cast_op.cc",
         "cast_op.h",
         "cast_op_impl.h",
diff --git a/tensorflow/core/kernels/argmax_op.cc b/tensorflow/core/kernels/argmax_op.cc
index c731b64993..778f818a61 100644
--- a/tensorflow/core/kernels/argmax_op.cc
+++ b/tensorflow/core/kernels/argmax_op.cc
@@ -25,13 +25,13 @@ limitations under the License.
 
 #include <memory>
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/macros.h"
 
diff --git a/tensorflow/core/kernels/bias_op.cc b/tensorflow/core/kernels/bias_op.cc
index 9006fb46fd..074f64a634 100644
--- a/tensorflow/core/kernels/bias_op.cc
+++ b/tensorflow/core/kernels/bias_op.cc
@@ -19,11 +19,11 @@ limitations under the License.
 
 #include "tensorflow/core/kernels/bias_op.h"
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/util/tensor_format.h"
 
 #if GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/bounds_check.h b/tensorflow/core/kernels/bounds_check.h
deleted file mode 100644
index ce6ec1012d..0000000000
--- a/tensorflow/core/kernels/bounds_check.h
+++ /dev/null
@@ -1,21 +0,0 @@
-/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#ifndef TENSORFLOW_CORE_KERNELS_BOUNDS_CHECK_H_
-#define TENSORFLOW_CORE_KERNELS_BOUNDS_CHECK_H_
-
-#include "tensorflow/core/framework/bounds_check.h"
-
-#endif  // TENSORFLOW_CORE_KERNELS_BOUNDS_CHECK_H_
diff --git a/tensorflow/core/kernels/concat_op.cc b/tensorflow/core/kernels/concat_op.cc
index ff62983517..72d8b45dd9 100644
--- a/tensorflow/core/kernels/concat_op.cc
+++ b/tensorflow/core/kernels/concat_op.cc
@@ -18,16 +18,16 @@ limitations under the License.
 #include <limits>
 #include <vector>
 
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/concat_lib.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/types.h"
-#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 
 namespace tensorflow {
 
diff --git a/tensorflow/core/kernels/constant_op.cc b/tensorflow/core/kernels/constant_op.cc
index 75ca77fad5..5ff428dd31 100644
--- a/tensorflow/core/kernels/constant_op.cc
+++ b/tensorflow/core/kernels/constant_op.cc
@@ -24,6 +24,7 @@ limitations under the License.
 #include "tensorflow/core/kernels/constant_op.h"
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
@@ -32,7 +33,6 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/framework/variant_op_registry.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/fill_functor.h"
 #include "tensorflow/core/platform/macros.h"
 
diff --git a/tensorflow/core/kernels/conv_ops.cc b/tensorflow/core/kernels/conv_ops.cc
index a8138fd0a7..979c76dc3c 100644
--- a/tensorflow/core/kernels/conv_ops.cc
+++ b/tensorflow/core/kernels/conv_ops.cc
@@ -28,13 +28,13 @@ limitations under the License.
 #include <map>
 #include <vector>
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_slice.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/conv_2d.h"
 #include "tensorflow/core/kernels/deep_conv2d.h"
 #include "tensorflow/core/kernels/ops_util.h"
diff --git a/tensorflow/core/kernels/conv_ops_fused.cc b/tensorflow/core/kernels/conv_ops_fused.cc
index 551782c488..9c807c3375 100644
--- a/tensorflow/core/kernels/conv_ops_fused.cc
+++ b/tensorflow/core/kernels/conv_ops_fused.cc
@@ -41,11 +41,11 @@ limitations under the License.
 #include "absl/strings/str_cat.h"
 #include "absl/strings/str_join.h"
 #include "absl/strings/substitute.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/conv_2d.h"
 #include "tensorflow/core/kernels/conv_ops.h"
 #include "tensorflow/core/kernels/ops_util.h"
diff --git a/tensorflow/core/kernels/conv_ops_fused_image_transform.cc b/tensorflow/core/kernels/conv_ops_fused_image_transform.cc
index 7be1de29c9..0542216a23 100644
--- a/tensorflow/core/kernels/conv_ops_fused_image_transform.cc
+++ b/tensorflow/core/kernels/conv_ops_fused_image_transform.cc
@@ -21,6 +21,7 @@ limitations under the License.
 
 #include <string>
 #include <vector>
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/common_shape_fns.h"
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
@@ -29,7 +30,6 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_slice.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/conv_2d.h"
 #include "tensorflow/core/kernels/conv_ops.h"
 #include "tensorflow/core/kernels/gemm_functors.h"
diff --git a/tensorflow/core/kernels/conv_ops_using_gemm.cc b/tensorflow/core/kernels/conv_ops_using_gemm.cc
index af0a9fa82e..05df9e0207 100644
--- a/tensorflow/core/kernels/conv_ops_using_gemm.cc
+++ b/tensorflow/core/kernels/conv_ops_using_gemm.cc
@@ -49,6 +49,7 @@ limitations under the License.
 #include <string.h>
 #include <map>
 #include <vector>
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/common_shape_fns.h"
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
@@ -57,7 +58,6 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_slice.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/conv_ops.h"
 #include "tensorflow/core/kernels/gemm_functors.h"
 #include "tensorflow/core/kernels/image_resizer_state.h"
diff --git a/tensorflow/core/kernels/crop_and_resize_op.cc b/tensorflow/core/kernels/crop_and_resize_op.cc
index 99d01b4db6..838cedd7a4 100644
--- a/tensorflow/core/kernels/crop_and_resize_op.cc
+++ b/tensorflow/core/kernels/crop_and_resize_op.cc
@@ -23,11 +23,11 @@ limitations under the License.
 #include <string>
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/logging.h"
diff --git a/tensorflow/core/kernels/ctc_loss_op.cc b/tensorflow/core/kernels/ctc_loss_op.cc
index fb375ee4b3..aa68e105ad 100644
--- a/tensorflow/core/kernels/ctc_loss_op.cc
+++ b/tensorflow/core/kernels/ctc_loss_op.cc
@@ -15,10 +15,10 @@ limitations under the License.
 
 // See docs in ../ops/ctc_ops.cc.
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/macros.h"
 #include "tensorflow/core/util/ctc/ctc_loss_calculator.h"
diff --git a/tensorflow/core/kernels/cwise_op_select.cc b/tensorflow/core/kernels/cwise_op_select.cc
index dd4e4ea547..3b51563ca2 100644
--- a/tensorflow/core/kernels/cwise_op_select.cc
+++ b/tensorflow/core/kernels/cwise_op_select.cc
@@ -19,8 +19,8 @@ limitations under the License.
 #define EIGEN_USE_GPU
 #endif  // GOOGLE_CUDA
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/register_types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/cwise_ops_common.h"
 #include "tensorflow/core/platform/prefetch.h"
 
diff --git a/tensorflow/core/kernels/cwise_ops.h b/tensorflow/core/kernels/cwise_ops.h
index a22d76717a..e7bc7f86cc 100644
--- a/tensorflow/core/kernels/cwise_ops.h
+++ b/tensorflow/core/kernels/cwise_ops.h
@@ -22,9 +22,9 @@ limitations under the License.
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/numeric_types.h"
 #include "tensorflow/core/framework/tensor_types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 
 namespace Eigen {
 namespace internal {
diff --git a/tensorflow/core/kernels/decode_bmp_op.cc b/tensorflow/core/kernels/decode_bmp_op.cc
index ae451be7e2..c75fc94bc1 100644
--- a/tensorflow/core/kernels/decode_bmp_op.cc
+++ b/tensorflow/core/kernels/decode_bmp_op.cc
@@ -16,13 +16,13 @@ limitations under the License.
 // See docs in ../ops/image_ops.cc
 
 #include <memory>
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/framework/types.pb.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/logging.h"
 
diff --git a/tensorflow/core/kernels/depthwise_conv_grad_op.cc b/tensorflow/core/kernels/depthwise_conv_grad_op.cc
index c152f2b7e4..ab98cacd1a 100644
--- a/tensorflow/core/kernels/depthwise_conv_grad_op.cc
+++ b/tensorflow/core/kernels/depthwise_conv_grad_op.cc
@@ -18,6 +18,7 @@ limitations under the License.
 #include <algorithm>
 #include <cmath>
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
@@ -25,7 +26,6 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/conv_grad_ops.h"
 #include "tensorflow/core/kernels/depthwise_conv_op.h"
 #include "tensorflow/core/kernels/ops_util.h"
diff --git a/tensorflow/core/kernels/depthwise_conv_op.cc b/tensorflow/core/kernels/depthwise_conv_op.cc
index dacd3cfea8..11c2b31633 100644
--- a/tensorflow/core/kernels/depthwise_conv_op.cc
+++ b/tensorflow/core/kernels/depthwise_conv_op.cc
@@ -19,6 +19,7 @@ limitations under the License.
 #include <cmath>
 #include <type_traits>
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
@@ -26,7 +27,6 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/conv_ops.h"
 #include "tensorflow/core/kernels/depthwise_conv_op.h"
 #include "tensorflow/core/kernels/ops_util.h"
diff --git a/tensorflow/core/kernels/dynamic_partition_op.cc b/tensorflow/core/kernels/dynamic_partition_op.cc
index 572d04ae2c..95af19c4c4 100644
--- a/tensorflow/core/kernels/dynamic_partition_op.cc
+++ b/tensorflow/core/kernels/dynamic_partition_op.cc
@@ -16,11 +16,11 @@ limitations under the License.
 // See docs in ../ops/data_flow_ops.cc.
 
 #include <vector>
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/gtl/inlined_vector.h"
 #include "tensorflow/core/util/util.h"
 
diff --git a/tensorflow/core/kernels/dynamic_partition_op_gpu.cu.cc b/tensorflow/core/kernels/dynamic_partition_op_gpu.cu.cc
index e7882acc80..59f687bf9c 100644
--- a/tensorflow/core/kernels/dynamic_partition_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/dynamic_partition_op_gpu.cu.cc
@@ -40,11 +40,11 @@ limitations under the License.
 #include "third_party/cub/iterator/constant_input_iterator.cuh"
 #include "third_party/cub/thread/thread_operators.cuh"
 #include "tensorflow/core/common_runtime/gpu/gpu_event_mgr.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/fill_functor.h"
 #include "tensorflow/core/kernels/gather_functor_gpu.cu.h"
 #include "tensorflow/core/util/cuda_kernel_helper.h"
diff --git a/tensorflow/core/kernels/dynamic_stitch_op.cc b/tensorflow/core/kernels/dynamic_stitch_op.cc
index f21f2acf26..5b8845b675 100644
--- a/tensorflow/core/kernels/dynamic_stitch_op.cc
+++ b/tensorflow/core/kernels/dynamic_stitch_op.cc
@@ -15,10 +15,10 @@ limitations under the License.
 
 // See docs in ../ops/data_flow_ops.cc.
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/core/threadpool.h"
 
 #ifdef GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/encode_jpeg_op.cc b/tensorflow/core/kernels/encode_jpeg_op.cc
index 1a5b0f2b67..e80404a437 100644
--- a/tensorflow/core/kernels/encode_jpeg_op.cc
+++ b/tensorflow/core/kernels/encode_jpeg_op.cc
@@ -16,12 +16,12 @@ limitations under the License.
 // See docs in ../ops/image_ops.cc
 
 #include <memory>
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/jpeg/jpeg_mem.h"
 #include "tensorflow/core/platform/logging.h"
diff --git a/tensorflow/core/kernels/encode_png_op.cc b/tensorflow/core/kernels/encode_png_op.cc
index 8fcda25e69..cb9a1660a7 100644
--- a/tensorflow/core/kernels/encode_png_op.cc
+++ b/tensorflow/core/kernels/encode_png_op.cc
@@ -16,12 +16,12 @@ limitations under the License.
 // See docs in ../ops/image_ops.cc
 
 #include <memory>
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/png/png_io.h"
 #include "tensorflow/core/platform/logging.h"
diff --git a/tensorflow/core/kernels/encode_wav_op.cc b/tensorflow/core/kernels/encode_wav_op.cc
index aed095076b..082f9a74ae 100644
--- a/tensorflow/core/kernels/encode_wav_op.cc
+++ b/tensorflow/core/kernels/encode_wav_op.cc
@@ -15,12 +15,12 @@ limitations under the License.
 
 // See docs in ../ops/audio_ops.cc
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/wav/wav_io.h"
 
diff --git a/tensorflow/core/kernels/extract_image_patches_op.cc b/tensorflow/core/kernels/extract_image_patches_op.cc
index 68631d14db..9306eccf9f 100644
--- a/tensorflow/core/kernels/extract_image_patches_op.cc
+++ b/tensorflow/core/kernels/extract_image_patches_op.cc
@@ -20,11 +20,11 @@ limitations under the License.
 
 #include "tensorflow/core/kernels/extract_image_patches_op.h"
 #include <vector>
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/ops_util.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/platform/logging.h"
diff --git a/tensorflow/core/kernels/extract_jpeg_shape_op.cc b/tensorflow/core/kernels/extract_jpeg_shape_op.cc
index 60d798af56..ab424595c1 100644
--- a/tensorflow/core/kernels/extract_jpeg_shape_op.cc
+++ b/tensorflow/core/kernels/extract_jpeg_shape_op.cc
@@ -16,12 +16,12 @@ limitations under the License.
 // See docs in ../ops/image_ops.cc
 
 #include <memory>
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/jpeg/jpeg_mem.h"
 #include "tensorflow/core/platform/logging.h"
diff --git a/tensorflow/core/kernels/extract_volume_patches_op.cc b/tensorflow/core/kernels/extract_volume_patches_op.cc
index 52cd078a35..8107bca7d1 100644
--- a/tensorflow/core/kernels/extract_volume_patches_op.cc
+++ b/tensorflow/core/kernels/extract_volume_patches_op.cc
@@ -26,11 +26,11 @@ when rates are to be added.
 
 #include "tensorflow/core/kernels/extract_volume_patches_op.h"
 #include <vector>
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/ops_util.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/platform/logging.h"
diff --git a/tensorflow/core/kernels/gather_functor.h b/tensorflow/core/kernels/gather_functor.h
index 7710cf93d6..93bdebc00e 100644
--- a/tensorflow/core/kernels/gather_functor.h
+++ b/tensorflow/core/kernels/gather_functor.h
@@ -18,11 +18,11 @@ limitations under the License.
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/type_traits.h"
 #include "tensorflow/core/framework/variant.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/platform/prefetch.h"
 #include "tensorflow/core/platform/types.h"
 #include "tensorflow/core/util/work_sharder.h"
diff --git a/tensorflow/core/kernels/gather_nd_op.cc b/tensorflow/core/kernels/gather_nd_op.cc
index e50b7fe3bf..58867a34bc 100644
--- a/tensorflow/core/kernels/gather_nd_op.cc
+++ b/tensorflow/core/kernels/gather_nd_op.cc
@@ -17,10 +17,10 @@ limitations under the License.
 #define EIGEN_USE_THREADS
 
 #include "tensorflow/core/kernels/gather_nd_op.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/mem.h"
diff --git a/tensorflow/core/kernels/gather_nd_op.h b/tensorflow/core/kernels/gather_nd_op.h
index 003badb74d..77c0d7717e 100644
--- a/tensorflow/core/kernels/gather_nd_op.h
+++ b/tensorflow/core/kernels/gather_nd_op.h
@@ -18,8 +18,8 @@ limitations under the License.
 // Functor definition for GatherOp, must be compilable by nvcc.
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/tensor_types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/platform/types.h"
 
 namespace tensorflow {
diff --git a/tensorflow/core/kernels/gather_nd_op_cpu_impl.h b/tensorflow/core/kernels/gather_nd_op_cpu_impl.h
index 1c78de253e..cf9817dc30 100644
--- a/tensorflow/core/kernels/gather_nd_op_cpu_impl.h
+++ b/tensorflow/core/kernels/gather_nd_op_cpu_impl.h
@@ -22,10 +22,10 @@ limitations under the License.
 
 #include <atomic>
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/gather_nd_op.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/mem.h"
diff --git a/tensorflow/core/kernels/gather_op.cc b/tensorflow/core/kernels/gather_op.cc
index 5795f68889..b26f0a7528 100644
--- a/tensorflow/core/kernels/gather_op.cc
+++ b/tensorflow/core/kernels/gather_op.cc
@@ -15,12 +15,12 @@ limitations under the License.
 
 // See docs in ../ops/array_ops.cc.
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/variant.h"
 #include "tensorflow/core/framework/variant_encode_decode.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/gather_functor.h"
 #include "tensorflow/core/platform/mem.h"
 #include "tensorflow/core/platform/types.h"
diff --git a/tensorflow/core/kernels/image_resizer_state.h b/tensorflow/core/kernels/image_resizer_state.h
index 1d4fa1a7db..8078c7036a 100644
--- a/tensorflow/core/kernels/image_resizer_state.h
+++ b/tensorflow/core/kernels/image_resizer_state.h
@@ -28,12 +28,12 @@ limitations under the License.
 #include <array>
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 
 namespace tensorflow {
 
diff --git a/tensorflow/core/kernels/in_topk_op.cc b/tensorflow/core/kernels/in_topk_op.cc
index c37055239c..506091f76e 100644
--- a/tensorflow/core/kernels/in_topk_op.cc
+++ b/tensorflow/core/kernels/in_topk_op.cc
@@ -18,10 +18,10 @@ limitations under the License.
 #define EIGEN_USE_THREADS
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 
 namespace tensorflow {
 
diff --git a/tensorflow/core/kernels/lookup_table_op.h b/tensorflow/core/kernels/lookup_table_op.h
index 9451247f26..b046401c0a 100644
--- a/tensorflow/core/kernels/lookup_table_op.h
+++ b/tensorflow/core/kernels/lookup_table_op.h
@@ -16,12 +16,12 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_KERNELS_LOOKUP_TABLE_OP_H_
 #define TENSORFLOW_CORE_KERNELS_LOOKUP_TABLE_OP_H_
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/lookup_interface.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/resource_mgr.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/lookup_util.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status.h"
diff --git a/tensorflow/core/kernels/lrn_op.cc b/tensorflow/core/kernels/lrn_op.cc
index f405ca3c58..ba30432e21 100644
--- a/tensorflow/core/kernels/lrn_op.cc
+++ b/tensorflow/core/kernels/lrn_op.cc
@@ -19,10 +19,10 @@ limitations under the License.
 #define EIGEN_USE_THREADS
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/ops_util.h"
 #include "tensorflow/core/lib/core/errors.h"
 
diff --git a/tensorflow/core/kernels/maxpooling_op.cc b/tensorflow/core/kernels/maxpooling_op.cc
index ab235843f7..d24cb1cc92 100644
--- a/tensorflow/core/kernels/maxpooling_op.cc
+++ b/tensorflow/core/kernels/maxpooling_op.cc
@@ -22,13 +22,13 @@ limitations under the License.
 #include <vector>
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/common_runtime/device.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_slice.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/conv_2d.h"
 #include "tensorflow/core/kernels/eigen_pooling.h"
 #include "tensorflow/core/kernels/ops_util.h"
diff --git a/tensorflow/core/kernels/mkl_concat_op.cc b/tensorflow/core/kernels/mkl_concat_op.cc
index f0278caee6..3a5c87485c 100644
--- a/tensorflow/core/kernels/mkl_concat_op.cc
+++ b/tensorflow/core/kernels/mkl_concat_op.cc
@@ -18,12 +18,12 @@ limitations under the License.
 
 #include "mkldnn.hpp"
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/concat_lib.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/types.h"
diff --git a/tensorflow/core/kernels/mkl_conv_ops.cc b/tensorflow/core/kernels/mkl_conv_ops.cc
index 8c585fb48c..6578f97ef4 100644
--- a/tensorflow/core/kernels/mkl_conv_ops.cc
+++ b/tensorflow/core/kernels/mkl_conv_ops.cc
@@ -21,13 +21,13 @@ limitations under the License.
 #include <map>
 #include <vector>
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_slice.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/mkl_conv_ops.h"
 #include "tensorflow/core/kernels/mkl_quantized_conv_ops.h"
 #include "tensorflow/core/kernels/no_op.h"
diff --git a/tensorflow/core/kernels/mkl_conv_ops.h b/tensorflow/core/kernels/mkl_conv_ops.h
index fd279ead9d..0881e8ebcd 100644
--- a/tensorflow/core/kernels/mkl_conv_ops.h
+++ b/tensorflow/core/kernels/mkl_conv_ops.h
@@ -21,13 +21,13 @@ limitations under the License.
 #include <vector>
 
 #include "mkldnn.hpp"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_slice.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/conv_grad_ops.h"
 #include "tensorflow/core/kernels/ops_util.h"
 #include "tensorflow/core/lib/core/errors.h"
diff --git a/tensorflow/core/kernels/mkl_lrn_op.cc b/tensorflow/core/kernels/mkl_lrn_op.cc
index 4d46abb0a4..bc52127b94 100644
--- a/tensorflow/core/kernels/mkl_lrn_op.cc
+++ b/tensorflow/core/kernels/mkl_lrn_op.cc
@@ -24,10 +24,10 @@ limitations under the License.
 #include <vector>
 #include "mkldnn.hpp"
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/ops_util.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/util/mkl_util.h"
diff --git a/tensorflow/core/kernels/mkl_pooling_ops_common.cc b/tensorflow/core/kernels/mkl_pooling_ops_common.cc
index dc84d3941e..a8d1dffd4e 100644
--- a/tensorflow/core/kernels/mkl_pooling_ops_common.cc
+++ b/tensorflow/core/kernels/mkl_pooling_ops_common.cc
@@ -19,8 +19,8 @@ limitations under the License.
 #include <limits>
 #include <vector>
 #include "tensorflow/core/common_runtime/device.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/common_shape_fns.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 
 namespace tensorflow {
 
diff --git a/tensorflow/core/kernels/neon/neon_depthwise_conv_op.cc b/tensorflow/core/kernels/neon/neon_depthwise_conv_op.cc
index 0e820bbb62..b218f62ddd 100644
--- a/tensorflow/core/kernels/neon/neon_depthwise_conv_op.cc
+++ b/tensorflow/core/kernels/neon/neon_depthwise_conv_op.cc
@@ -19,6 +19,7 @@ limitations under the License.
 
 #define GEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK
 #include "public/gemmlowp.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
@@ -26,7 +27,6 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/neon/depthwiseconv_float.h"
 #include "tensorflow/core/kernels/ops_util.h"
 #include "tensorflow/core/lib/core/status.h"
diff --git a/tensorflow/core/kernels/non_max_suppression_op.cc b/tensorflow/core/kernels/non_max_suppression_op.cc
index 37f615abd9..2f4e51272f 100644
--- a/tensorflow/core/kernels/non_max_suppression_op.cc
+++ b/tensorflow/core/kernels/non_max_suppression_op.cc
@@ -24,12 +24,12 @@ limitations under the License.
 #include <vector>
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/gtl/stl_util.h"
 #include "tensorflow/core/platform/logging.h"
diff --git a/tensorflow/core/kernels/resource_variable_ops.cc b/tensorflow/core/kernels/resource_variable_ops.cc
index 4167b60051..8e3c52ba5b 100644
--- a/tensorflow/core/kernels/resource_variable_ops.cc
+++ b/tensorflow/core/kernels/resource_variable_ops.cc
@@ -56,12 +56,12 @@ limitations under the License.
 
 #include "absl/strings/str_join.h"
 #include "tensorflow/core/common_runtime/device.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/resource_mgr.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/variant_op_registry.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/dense_update_functor.h"
 #include "tensorflow/core/kernels/gather_functor.h"
 #include "tensorflow/core/kernels/resource_variable_ops.h"
diff --git a/tensorflow/core/kernels/reverse_op.cc b/tensorflow/core/kernels/reverse_op.cc
index 1c4d0bc1ae..aa2434da03 100644
--- a/tensorflow/core/kernels/reverse_op.cc
+++ b/tensorflow/core/kernels/reverse_op.cc
@@ -19,13 +19,13 @@ limitations under the License.
 #include "tensorflow/core/kernels/reverse_op.h"
 #include <memory>
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/type_traits.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/util/work_sharder.h"
diff --git a/tensorflow/core/kernels/roll_op.cc b/tensorflow/core/kernels/roll_op.cc
index efa30438d9..494a846ff5 100644
--- a/tensorflow/core/kernels/roll_op.cc
+++ b/tensorflow/core/kernels/roll_op.cc
@@ -13,13 +13,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/common_shape_fns.h"
 #include "tensorflow/core/framework/op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/register_types_traits.h"
 #include "tensorflow/core/framework/shape_inference.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/gtl/array_slice.h"
 #include "tensorflow/core/platform/types.h"
 #include "tensorflow/core/util/work_sharder.h"
diff --git a/tensorflow/core/kernels/sample_distorted_bounding_box_op.cc b/tensorflow/core/kernels/sample_distorted_bounding_box_op.cc
index c0fde8042e..0e68af867b 100644
--- a/tensorflow/core/kernels/sample_distorted_bounding_box_op.cc
+++ b/tensorflow/core/kernels/sample_distorted_bounding_box_op.cc
@@ -14,11 +14,11 @@ limitations under the License.
 ==============================================================================*/
 // See docs in ../ops/image_ops.cc.
 #include <math.h>
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/random/simple_philox.h"
 #include "tensorflow/core/util/guarded_philox_random.h"
 
diff --git a/tensorflow/core/kernels/save_restore_tensor.cc b/tensorflow/core/kernels/save_restore_tensor.cc
index 82546d581a..8580891fc0 100644
--- a/tensorflow/core/kernels/save_restore_tensor.cc
+++ b/tensorflow/core/kernels/save_restore_tensor.cc
@@ -19,10 +19,10 @@ limitations under the License.
 #include <utility>
 #include <vector>
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/core/threadpool.h"
 #include "tensorflow/core/lib/gtl/array_slice.h"
 #include "tensorflow/core/lib/strings/str_util.h"
diff --git a/tensorflow/core/kernels/save_restore_v2_ops.cc b/tensorflow/core/kernels/save_restore_v2_ops.cc
index 180eb3ca34..ed1195c053 100644
--- a/tensorflow/core/kernels/save_restore_v2_ops.cc
+++ b/tensorflow/core/kernels/save_restore_v2_ops.cc
@@ -18,11 +18,11 @@ limitations under the License.
 #include <string>
 #include <vector>
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/framework/types.pb.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/save_restore_tensor.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/io/path.h"
diff --git a/tensorflow/core/kernels/scale_and_translate_op.cc b/tensorflow/core/kernels/scale_and_translate_op.cc
index 149c5526ae..34fef536df 100644
--- a/tensorflow/core/kernels/scale_and_translate_op.cc
+++ b/tensorflow/core/kernels/scale_and_translate_op.cc
@@ -20,13 +20,13 @@ limitations under the License.
 
 #include <memory>
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/framework/types.pb.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/sampling_kernels.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/strings/stringprintf.h"
diff --git a/tensorflow/core/kernels/scan_ops.cc b/tensorflow/core/kernels/scan_ops.cc
index 0a6848361a..ea42fdefb4 100644
--- a/tensorflow/core/kernels/scan_ops.cc
+++ b/tensorflow/core/kernels/scan_ops.cc
@@ -18,12 +18,12 @@ limitations under the License.
 #define EIGEN_USE_GPU
 #endif  // GOOGLE_CUDA
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 
 #include "third_party/eigen3/Eigen/Core"
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
diff --git a/tensorflow/core/kernels/scatter_functor.h b/tensorflow/core/kernels/scatter_functor.h
index 2d43bde23f..755f8f8dc5 100644
--- a/tensorflow/core/kernels/scatter_functor.h
+++ b/tensorflow/core/kernels/scatter_functor.h
@@ -20,10 +20,10 @@ limitations under the License.
 
 #include "third_party/eigen3/Eigen/Core"
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/variant_op_registry.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/dense_update_functor.h"
 #include "tensorflow/core/platform/types.h"
 
diff --git a/tensorflow/core/kernels/scatter_nd_op.cc b/tensorflow/core/kernels/scatter_nd_op.cc
index 50e4c66b7e..9c51d4e3a7 100644
--- a/tensorflow/core/kernels/scatter_nd_op.cc
+++ b/tensorflow/core/kernels/scatter_nd_op.cc
@@ -22,11 +22,11 @@ limitations under the License.
 
 #include "tensorflow/core/kernels/scatter_nd_op.h"
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/dense_update_functor.h"
 #include "tensorflow/core/kernels/fill_functor.h"
 #include "tensorflow/core/kernels/inplace_ops_functor.h"
diff --git a/tensorflow/core/kernels/scatter_nd_op.h b/tensorflow/core/kernels/scatter_nd_op.h
index 8d04731aae..eec70ba69e 100644
--- a/tensorflow/core/kernels/scatter_nd_op.h
+++ b/tensorflow/core/kernels/scatter_nd_op.h
@@ -18,11 +18,11 @@ limitations under the License.
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/fill_functor.h"
 #include "tensorflow/core/kernels/scatter_nd_op.h"
 #include "tensorflow/core/platform/mutex.h"
diff --git a/tensorflow/core/kernels/scatter_nd_op_cpu_impl.h b/tensorflow/core/kernels/scatter_nd_op_cpu_impl.h
index 472f5a3547..01e4656eab 100644
--- a/tensorflow/core/kernels/scatter_nd_op_cpu_impl.h
+++ b/tensorflow/core/kernels/scatter_nd_op_cpu_impl.h
@@ -24,11 +24,11 @@ limitations under the License.
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/fill_functor.h"
 #include "tensorflow/core/kernels/scatter_nd_op.h"
 #include "tensorflow/core/platform/mutex.h"
diff --git a/tensorflow/core/kernels/searchsorted_op.cc b/tensorflow/core/kernels/searchsorted_op.cc
index dc627ac77a..06b2d81837 100644
--- a/tensorflow/core/kernels/searchsorted_op.cc
+++ b/tensorflow/core/kernels/searchsorted_op.cc
@@ -17,11 +17,11 @@ limitations under the License.
 
 #include "tensorflow/core/kernels/searchsorted_op.h"
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/types.h"
 
diff --git a/tensorflow/core/kernels/segment_reduction_ops.cc b/tensorflow/core/kernels/segment_reduction_ops.cc
index 2328fc6afd..6e1a0d57a1 100644
--- a/tensorflow/core/kernels/segment_reduction_ops.cc
+++ b/tensorflow/core/kernels/segment_reduction_ops.cc
@@ -22,15 +22,17 @@ limitations under the License.
 
 #include "third_party/eigen3/Eigen/Core"
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+
 #include "tensorflow/core/kernels/segment_reduction_ops.h"
 #include <vector>
+
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/util/util.h"
diff --git a/tensorflow/core/kernels/shape_ops.h b/tensorflow/core/kernels/shape_ops.h
index 7a50f158af..dbd82d2fe7 100644
--- a/tensorflow/core/kernels/shape_ops.h
+++ b/tensorflow/core/kernels/shape_ops.h
@@ -20,11 +20,11 @@ limitations under the License.
 #include <unordered_set>
 #include <vector>
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/variant_op_registry.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 
 namespace tensorflow {
 
diff --git a/tensorflow/core/kernels/spacetobatch_functor.h b/tensorflow/core/kernels/spacetobatch_functor.h
index f46a84da1e..459f20b0ae 100644
--- a/tensorflow/core/kernels/spacetobatch_functor.h
+++ b/tensorflow/core/kernels/spacetobatch_functor.h
@@ -19,9 +19,9 @@ limitations under the License.
 #include <type_traits>
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/platform/types.h"
 
 namespace tensorflow {
diff --git a/tensorflow/core/kernels/sparse_tensor_dense_matmul_op.cc b/tensorflow/core/kernels/sparse_tensor_dense_matmul_op.cc
index 30c57ef287..0a97c6b6a5 100644
--- a/tensorflow/core/kernels/sparse_tensor_dense_matmul_op.cc
+++ b/tensorflow/core/kernels/sparse_tensor_dense_matmul_op.cc
@@ -19,9 +19,9 @@ limitations under the License.
 
 #include "tensorflow/core/kernels/sparse_tensor_dense_matmul_op.h"
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op.h"
 #include "tensorflow/core/framework/op_kernel.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/fill_functor.h"
 
 namespace tensorflow {
diff --git a/tensorflow/core/kernels/sparse_tensor_dense_matmul_op_gpu.cu.cc b/tensorflow/core/kernels/sparse_tensor_dense_matmul_op_gpu.cu.cc
index e261e42e0d..ea95a882b1 100644
--- a/tensorflow/core/kernels/sparse_tensor_dense_matmul_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/sparse_tensor_dense_matmul_op_gpu.cu.cc
@@ -19,8 +19,8 @@ limitations under the License.
 
 #include "tensorflow/core/kernels/sparse_tensor_dense_matmul_op.h"
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/register_types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/util/cuda_kernel_helper.h"
 
 namespace tensorflow {
diff --git a/tensorflow/core/kernels/sparse_xent_op.h b/tensorflow/core/kernels/sparse_xent_op.h
index 5e462424ed..c94597f297 100644
--- a/tensorflow/core/kernels/sparse_xent_op.h
+++ b/tensorflow/core/kernels/sparse_xent_op.h
@@ -18,9 +18,9 @@ limitations under the License.
 // Functor definition for SparseXentOp, must be compilable by nvcc.
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/tensor_types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/platform/macros.h"
 #include "tensorflow/core/platform/types.h"
 
diff --git a/tensorflow/core/kernels/split_op.cc b/tensorflow/core/kernels/split_op.cc
index 11db72bfa3..ed3429ff5c 100644
--- a/tensorflow/core/kernels/split_op.cc
+++ b/tensorflow/core/kernels/split_op.cc
@@ -18,10 +18,10 @@ limitations under the License.
 #define EIGEN_USE_THREADS
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/ops_util.h"
 #include "tensorflow/core/kernels/split_lib.h"
 #include "tensorflow/core/lib/core/status.h"
diff --git a/tensorflow/core/kernels/split_v_op.cc b/tensorflow/core/kernels/split_v_op.cc
index 5c19a45fb1..0324ce9bab 100644
--- a/tensorflow/core/kernels/split_v_op.cc
+++ b/tensorflow/core/kernels/split_v_op.cc
@@ -24,10 +24,10 @@ limitations under the License.
 #include <numeric>
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/ops_util.h"
 #include "tensorflow/core/kernels/split_lib.h"
 #include "tensorflow/core/lib/core/status.h"
diff --git a/tensorflow/core/kernels/stateful_random_ops.cc b/tensorflow/core/kernels/stateful_random_ops.cc
index e5c1c25864..7cbbfd490e 100644
--- a/tensorflow/core/kernels/stateful_random_ops.cc
+++ b/tensorflow/core/kernels/stateful_random_ops.cc
@@ -22,13 +22,13 @@ limitations under the License.
 #include "absl/strings/str_join.h"
 #include "absl/types/variant.h"
 #include "tensorflow/core/common_runtime/device.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/resource_mgr.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/variant_op_registry.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/dense_update_functor.h"
 #include "tensorflow/core/kernels/gather_functor.h"
 #include "tensorflow/core/kernels/random_op.h"
diff --git a/tensorflow/core/kernels/stateless_random_ops.cc b/tensorflow/core/kernels/stateless_random_ops.cc
index 925f5291a6..959334abc8 100644
--- a/tensorflow/core/kernels/stateless_random_ops.cc
+++ b/tensorflow/core/kernels/stateless_random_ops.cc
@@ -13,11 +13,11 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/random_op.h"
 #include "tensorflow/core/lib/random/random_distributions.h"
 #include "tensorflow/core/platform/logging.h"
diff --git a/tensorflow/core/kernels/strided_slice_op.cc b/tensorflow/core/kernels/strided_slice_op.cc
index 6db68f937d..20bf42ccaa 100644
--- a/tensorflow/core/kernels/strided_slice_op.cc
+++ b/tensorflow/core/kernels/strided_slice_op.cc
@@ -27,10 +27,10 @@ limitations under the License.
 #include "tensorflow/core/kernels/strided_slice_op_impl.h"
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/ops_util.h"
 #include "tensorflow/core/kernels/training_op_helpers.h"
 #include "tensorflow/core/kernels/variable_ops.h"
diff --git a/tensorflow/core/kernels/strided_slice_op_impl.h b/tensorflow/core/kernels/strided_slice_op_impl.h
index c4205159c3..d9b62d4c75 100644
--- a/tensorflow/core/kernels/strided_slice_op_impl.h
+++ b/tensorflow/core/kernels/strided_slice_op_impl.h
@@ -22,13 +22,13 @@ limitations under the License.
 #include "tensorflow/core/kernels/strided_slice_op.h"
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/register_types_traits.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/variant.h"
 #include "tensorflow/core/framework/variant_encode_decode.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/dense_update_functor.h"
 #include "tensorflow/core/kernels/ops_util.h"
 #include "tensorflow/core/lib/core/status.h"
diff --git a/tensorflow/core/kernels/substr_op.cc b/tensorflow/core/kernels/substr_op.cc
index 93c427039d..77b16b9384 100644
--- a/tensorflow/core/kernels/substr_op.cc
+++ b/tensorflow/core/kernels/substr_op.cc
@@ -18,6 +18,7 @@ limitations under the License.
 #include <string>
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/kernel_def_builder.h"
 #include "tensorflow/core/framework/op.h"
 #include "tensorflow/core/framework/op_kernel.h"
@@ -25,7 +26,6 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/string_util.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/stringpiece.h"
diff --git a/tensorflow/core/kernels/tensor_array_ops.cc b/tensorflow/core/kernels/tensor_array_ops.cc
index aa85f546a8..129035638a 100644
--- a/tensorflow/core/kernels/tensor_array_ops.cc
+++ b/tensorflow/core/kernels/tensor_array_ops.cc
@@ -23,13 +23,13 @@ limitations under the License.
 #include <numeric>  // clang-format off
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/resource_mgr.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/concat_lib.h"
 #include "tensorflow/core/kernels/split_lib.h"
 #include "tensorflow/core/kernels/tensor_array.h"
diff --git a/tensorflow/core/kernels/training_ops.cc b/tensorflow/core/kernels/training_ops.cc
index b2239ab5c3..5594c998dd 100644
--- a/tensorflow/core/kernels/training_ops.cc
+++ b/tensorflow/core/kernels/training_ops.cc
@@ -18,9 +18,9 @@ limitations under the License.
 
 #include <algorithm>
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/training_op_helpers.h"
 #include "tensorflow/core/kernels/training_ops.h"
 #include "tensorflow/core/kernels/variable_ops.h"
diff --git a/tensorflow/core/kernels/transpose_op.cc b/tensorflow/core/kernels/transpose_op.cc
index 48e392c070..1c0d70c333 100644
--- a/tensorflow/core/kernels/transpose_op.cc
+++ b/tensorflow/core/kernels/transpose_op.cc
@@ -19,11 +19,11 @@ limitations under the License.
 
 #include "tensorflow/core/kernels/transpose_op.h"
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/transpose_functor.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/strings/str_util.h"
diff --git a/tensorflow/core/kernels/unicode_ops.cc b/tensorflow/core/kernels/unicode_ops.cc
index c9c2ac1e69..c071db6064 100644
--- a/tensorflow/core/kernels/unicode_ops.cc
+++ b/tensorflow/core/kernels/unicode_ops.cc
@@ -31,6 +31,7 @@ limitations under the License.
 #include "unicode/unistr.h"  // TF:icu
 #include "unicode/uset.h"  // TF:icu
 #include "unicode/utypes.h"  // TF:icu
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/kernel_def_builder.h"
 #include "tensorflow/core/framework/op.h"
 #include "tensorflow/core/framework/op_kernel.h"
@@ -39,7 +40,6 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/string_util.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status.h"
diff --git a/tensorflow/core/kernels/unique_op.cc b/tensorflow/core/kernels/unique_op.cc
index 3bdcfc90b8..adf84bae49 100644
--- a/tensorflow/core/kernels/unique_op.cc
+++ b/tensorflow/core/kernels/unique_op.cc
@@ -17,11 +17,11 @@ limitations under the License.
 #include <unordered_map>
 #include <utility>
 
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/hash/hash.h"
 
diff --git a/tensorflow/core/kernels/unpack_op.cc b/tensorflow/core/kernels/unpack_op.cc
index 1e1647db5c..8577ce7bf7 100644
--- a/tensorflow/core/kernels/unpack_op.cc
+++ b/tensorflow/core/kernels/unpack_op.cc
@@ -18,10 +18,10 @@ limitations under the License.
 #define EIGEN_USE_THREADS
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/ops_util.h"
 #include "tensorflow/core/kernels/split_lib.h"
 #include "tensorflow/core/lib/core/status.h"
diff --git a/tensorflow/core/kernels/where_op.cc b/tensorflow/core/kernels/where_op.cc
index aa65fa6b63..374257d176 100644
--- a/tensorflow/core/kernels/where_op.cc
+++ b/tensorflow/core/kernels/where_op.cc
@@ -26,13 +26,13 @@ limitations under the License.
 #include <memory>
 #include <numeric>
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/macros.h"
 #include "tensorflow/core/platform/types.h"
diff --git a/tensorflow/core/kernels/where_op_gpu.cu.h b/tensorflow/core/kernels/where_op_gpu.cu.h
index 2255597651..54b22d230a 100644
--- a/tensorflow/core/kernels/where_op_gpu.cu.h
+++ b/tensorflow/core/kernels/where_op_gpu.cu.h
@@ -25,9 +25,9 @@ limitations under the License.
 #include "third_party/cub/device/device_select.cuh"
 #include "third_party/cub/iterator/counting_input_iterator.cuh"
 #include "third_party/cub/iterator/transform_input_iterator.cuh"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor_types.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/where_op.h"
 #include "tensorflow/core/platform/macros.h"
 #include "tensorflow/core/platform/types.h"
diff --git a/tensorflow/core/util/sparse/dim_comparator.h b/tensorflow/core/util/sparse/dim_comparator.h
index 0782e7e1a8..498df7a021 100644
--- a/tensorflow/core/util/sparse/dim_comparator.h
+++ b/tensorflow/core/util/sparse/dim_comparator.h
@@ -17,7 +17,7 @@ limitations under the License.
 #define TENSORFLOW_CORE_UTIL_SPARSE_DIM_COMPARATOR_H_
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
-#include "tensorflow/core/kernels/bounds_check.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/lib/gtl/array_slice.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/types.h"
diff --git a/tensorflow/core/util/sparse/sparse_tensor.h b/tensorflow/core/util/sparse/sparse_tensor.h
index 89c163aa51..cd168f6597 100644
--- a/tensorflow/core/util/sparse/sparse_tensor.h
+++ b/tensorflow/core/util/sparse/sparse_tensor.h
@@ -22,11 +22,11 @@ limitations under the License.
 
 #include "absl/base/macros.h"
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/framework/types.pb.h"
-#include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/strings/str_util.h"
diff --git a/tensorflow/core/util/strided_slice_op.cc b/tensorflow/core/util/strided_slice_op.cc
index 55688e5808..9db35da6a2 100644
--- a/tensorflow/core/util/strided_slice_op.cc
+++ b/tensorflow/core/util/strided_slice_op.cc
@@ -16,7 +16,7 @@ limitations under the License.
 #include "tensorflow/core/util/strided_slice_op.h"
 
 #include <array>
-#include "tensorflow/core/kernels/bounds_check.h"
+#include "tensorflow/core/framework/bounds_check.h"
 #include "tensorflow/core/lib/core/status.h"
 
 namespace tensorflow {
-- 
GitLab


From 89d7b76d3a58364e3021a70e88c58896584f44ee Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Tue, 22 Jan 2019 14:53:12 -0800
Subject: [PATCH 1080/1095] Automated rollback of commit
 74a6cca5d867d37e79ec9d780f2c57b926f07a80

PiperOrigin-RevId: 230415975
---
 tensorflow/python/framework/dtypes.py | 110 ++++++++++++--------------
 1 file changed, 52 insertions(+), 58 deletions(-)

diff --git a/tensorflow/python/framework/dtypes.py b/tensorflow/python/framework/dtypes.py
index 465ea4229b..574d82ea93 100644
--- a/tensorflow/python/framework/dtypes.py
+++ b/tensorflow/python/framework/dtypes.py
@@ -535,42 +535,29 @@ _np_qint32 = np.dtype([("qint32", np.int32, 1)])
 np_resource = np.dtype([("resource", np.ubyte, 1)])
 
 # Standard mappings between types_pb2.DataType values and numpy.dtypes.
-_NP_TO_TF = {
-    np.float16: float16,
-    np.float32: float32,
-    np.float64: float64,
-    np.int32: int32,
-    np.int64: int64,
-    np.uint8: uint8,
-    np.uint16: uint16,
-    np.uint32: uint32,
-    np.uint64: uint64,
-    np.int16: int16,
-    np.int8: int8,
-    np.complex64: complex64,
-    np.complex128: complex128,
-    np.object_: string,
-    np.string_: string,
-    np.unicode_: string,
-    np.bool_: bool,
-    _np_qint8: qint8,
-    _np_quint8: quint8,
-    _np_qint16: qint16,
-    _np_quint16: quint16,
-    _np_qint32: qint32,
-    _np_bfloat16: bfloat16,
-}
-
-# On Python 2.X `np.longlong` could be a distinct type used for long
-# integers e.g. 42L. See numpy/numpy#9799.
-for pdt in [
-    np.longlong,
-    np.ulonglong,
-]:
-  if pdt not in _NP_TO_TF:
-    _NP_TO_TF[pdt] = next(
-        _NP_TO_TF[dt] for dt in _NP_TO_TF if dt == pdt().dtype)
-
+_NP_TO_TF = frozenset([
+    (np.float16, float16),
+    (np.float32, float32),
+    (np.float64, float64),
+    (np.int32, int32),
+    (np.int64, int64),
+    (np.uint8, uint8),
+    (np.uint16, uint16),
+    (np.uint32, uint32),
+    (np.uint64, uint64),
+    (np.int16, int16),
+    (np.int8, int8),
+    (np.complex64, complex64),
+    (np.complex128, complex128),
+    (np.object_, string),
+    (np.bool_, bool),
+    (_np_qint8, qint8),
+    (_np_quint8, quint8),
+    (_np_qint16, qint16),
+    (_np_quint16, quint16),
+    (_np_qint32, qint32),
+    (_np_bfloat16, bfloat16),
+])
 _TF_TO_NP = {
     types_pb2.DT_HALF:
         np.float16,
@@ -677,20 +664,6 @@ _PYTHON_TO_TF = {
     builtins.object: string
 }
 
-_ANY_TO_TF = {}
-_ANY_TO_TF.update(_INTERN_TABLE)
-_ANY_TO_TF.update(_STRING_TO_TF)
-_ANY_TO_TF.update(_PYTHON_TO_TF)
-_ANY_TO_TF.update(_NP_TO_TF)
-
-# Ensure no collisions.
-assert len(_ANY_TO_TF) == sum(len(d) for d in [
-    _INTERN_TABLE,
-    _STRING_TO_TF,
-    _PYTHON_TO_TF,
-    _NP_TO_TF
-])
-
 
 @tf_export("dtypes.as_dtype", "as_dtype")
 def as_dtype(type_value):
@@ -711,16 +684,37 @@ def as_dtype(type_value):
   if isinstance(type_value, DType):
     return type_value
 
-  if isinstance(type_value, np.dtype):
-    try:
-      return _NP_TO_TF[type_value.type]
-    except KeyError:
-      pass
+  try:
+    return _INTERN_TABLE[type_value]
+  except KeyError:
+    pass
+
+  try:
+    return _STRING_TO_TF[type_value]
+  except KeyError:
+    pass
 
   try:
-    return _ANY_TO_TF[type_value]
+    return _PYTHON_TO_TF[type_value]
   except KeyError:
     pass
 
-  raise TypeError(
-      "Cannot convert value %r to a TensorFlow DType." % type_value)
+  if isinstance(type_value, np.dtype):
+    # The numpy dtype for strings is variable length. We can not compare
+    # dtype with a single constant (np.string does not exist) to decide
+    # dtype is a "string" type. We need to compare the dtype.type to be
+    # sure it's a string type.
+    if type_value.type == np.string_ or type_value.type == np.unicode_:
+      return string
+
+  if isinstance(type_value, (type, np.dtype)):
+    for key, val in _NP_TO_TF:
+      try:
+        if key == type_value:
+          return val
+      except TypeError as e:
+        raise TypeError("Cannot convert {} to a dtype. {}".format(
+            type_value, e))
+
+  raise TypeError("Cannot convert value %r to a TensorFlow DType." % type_value)
+
-- 
GitLab


From 46ddf3105bfd2a8eebabfff50c0cbf3e16d911dd Mon Sep 17 00:00:00 2001
From: Michael Kuperstein <mkuper@google.com>
Date: Tue, 22 Jan 2019 14:56:56 -0800
Subject: [PATCH 1081/1095] [XLA] HloPassFix should implement RunOnModuleGroup

The default implementation just forwards to the underlying pass' RunOnModuleGroup, which means we silently run the underlying pass once, instead of running it to a fixed point.

PiperOrigin-RevId: 230416670
---
 .../compiler/xla/service/hlo_pass_fix.h       | 28 +++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/tensorflow/compiler/xla/service/hlo_pass_fix.h b/tensorflow/compiler/xla/service/hlo_pass_fix.h
index 791b1a97b0..35dc9c0029 100644
--- a/tensorflow/compiler/xla/service/hlo_pass_fix.h
+++ b/tensorflow/compiler/xla/service/hlo_pass_fix.h
@@ -19,6 +19,7 @@ limitations under the License.
 #include <algorithm>
 
 #include "tensorflow/compiler/xla/service/hlo_module.h"
+#include "tensorflow/compiler/xla/service/hlo_module_group.h"
 #include "tensorflow/compiler/xla/status_macros.h"
 #include "tensorflow/compiler/xla/statusor.h"
 #include "tensorflow/compiler/xla/types.h"
@@ -39,9 +40,36 @@ class HloPassFix : public Pass {
     int64 iteration_count = 0;
     int64 limit =
         std::max(static_cast<int64>(1000), module->instruction_count());
+    VLOG(3) << "Running HloPassFix.";
     while (changed_this_iteration) {
       TF_ASSIGN_OR_RETURN(changed_this_iteration, Pass::Run(module));
       changed |= changed_this_iteration;
+      VLOG(3) << "changed_this_iteration: " << changed_this_iteration;
+      ++iteration_count;
+      if (iteration_count == limit) {
+        LOG(ERROR)
+            << "Unexpectedly high number of iterations in HLO passes ("
+            << iteration_count
+            << ")\nIf compilation hangs here, please file a bug with XLA.";
+      }
+    }
+    return changed;
+  }
+
+  StatusOr<bool> RunOnModuleGroup(HloModuleGroup* module_group) override {
+    bool changed = false;
+    bool changed_this_iteration = true;
+    int64 iteration_count = 0;
+    int64 limit = 1000;
+    for (const HloModule* module : module_group->modules()) {
+      limit = std::max<int64>(limit, module->instruction_count());
+    }
+    VLOG(3) << "Running HloPassFix.";
+    while (changed_this_iteration) {
+      TF_ASSIGN_OR_RETURN(changed_this_iteration,
+                          Pass::RunOnModuleGroup(module_group));
+      changed |= changed_this_iteration;
+      VLOG(3) << "changed_this_iteration: " << changed_this_iteration;
       ++iteration_count;
       if (iteration_count == limit) {
         LOG(ERROR)
-- 
GitLab


From f7d933a71fa022f1f3cddc3fec2a9db7b520473f Mon Sep 17 00:00:00 2001
From: Francois Chollet <fchollet@google.com>
Date: Tue, 22 Jan 2019 14:58:31 -0800
Subject: [PATCH 1082/1095] Reapply change "cast inputs to Layer's __call__ to
 tensors" after fixing the failing unit test.

PiperOrigin-RevId: 230416943
---
 .../sequence_feature_column_v2_test.py            |  2 +-
 tensorflow/python/keras/engine/base_layer.py      |  9 ++++-----
 tensorflow/python/keras/engine/base_layer_test.py | 15 ++++++++++++++-
 3 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2_test.py b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2_test.py
index 661eba4741..95d8ee278c 100644
--- a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2_test.py
+++ b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_v2_test.py
@@ -370,7 +370,7 @@ class SequenceFeaturesTest(test.TestCase, parameterized.TestCase):
            [[3.], [0.], [8.], [0.]]],
        'expected_sequence_length': [2, 2]},
       )
-  def test_numeric_column(
+  def DISABLED_test_numeric_column(  # See b/123240957
       self, sparse_input_args, expected_input_layer, expected_sequence_length):
     sparse_input = sparse_tensor.SparseTensorValue(**sparse_input_args)
 
diff --git a/tensorflow/python/keras/engine/base_layer.py b/tensorflow/python/keras/engine/base_layer.py
index f63fb4e694..93849e771b 100644
--- a/tensorflow/python/keras/engine/base_layer.py
+++ b/tensorflow/python/keras/engine/base_layer.py
@@ -513,11 +513,10 @@ class Layer(checkpointable.Checkpointable):
       ValueError: if the layer's `call` method returns None (an invalid value).
     """
     input_list = nest.flatten(inputs)
-    if context.executing_eagerly():
-      # Accept NumPy inputs by converting to Tensors when executing eagerly.
-      if all(isinstance(x, (np.ndarray, float, int)) for x in input_list):
-        inputs = nest.map_structure(ops.convert_to_tensor, inputs)
-        input_list = nest.flatten(inputs)
+    # Accept NumPy inputs by converting to Tensors.
+    if all(isinstance(x, (np.ndarray, float, int)) for x in input_list):
+      inputs = nest.map_structure(ops.convert_to_tensor, inputs)
+      input_list = nest.flatten(inputs)
 
     # We will attempt to build a TF graph if & only if all inputs are symbolic.
     # This is always the case in graph mode. It can also be the case in eager
diff --git a/tensorflow/python/keras/engine/base_layer_test.py b/tensorflow/python/keras/engine/base_layer_test.py
index ebee4a3043..c925a8ca8a 100644
--- a/tensorflow/python/keras/engine/base_layer_test.py
+++ b/tensorflow/python/keras/engine/base_layer_test.py
@@ -26,6 +26,8 @@ from tensorflow.python.eager import context
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import test_util
+from tensorflow.python.keras import keras_parameterized
+from tensorflow.python.keras import testing_utils
 from tensorflow.python.keras.engine import base_layer
 from tensorflow.python.keras.optimizer_v2 import rmsprop
 from tensorflow.python.ops import array_ops
@@ -71,7 +73,7 @@ class InvalidLayer(base_layer.Layer):
     raise ValueError('You did something wrong!')
 
 
-class BaseLayerTest(test.TestCase, parameterized.TestCase):
+class BaseLayerTest(keras_parameterized.TestCase):
 
   @parameterized.parameters(DynamicLayer1, DynamicLayer2)
   def test_dynamic_layer_in_functional_model_in_graph_mode(self, layer_class):
@@ -210,6 +212,17 @@ class BaseLayerTest(test.TestCase, parameterized.TestCase):
     with self.assertRaisesRegexp(ValueError, 'You did something wrong!'):
       _ = InvalidLayer()(inputs)
 
+  @keras_parameterized.run_with_all_model_types
+  @test_util.run_in_graph_and_eager_modes
+  def test_build_with_numpy_data(self):
+    model_layers = [
+        keras.layers.Dense(3, activation='relu', kernel_initializer='ones'),
+        keras.layers.Dense(1, activation='sigmoid', kernel_initializer='ones')
+    ]
+    model = testing_utils.get_model_from_layers(model_layers, input_shape=(4,))
+    model(np.zeros((2, 4), dtype='float32'))
+    self.assertTrue(model.built)
+
   def test_using_symbolic_tensors_with_tf_ops(self):
     # Single-input.
     x = keras.Input((3,))
-- 
GitLab


From 29970780c7e2aaf55040f09e10e5c1f6649cfd32 Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Tue, 22 Jan 2019 15:06:11 -0800
Subject: [PATCH 1083/1095] Add tests for dynamic parameter binding's
 serialization and deserialization behaviors.

PiperOrigin-RevId: 230418554
---
 .../xla/service/dynamic_parameter_binding.cc  |   3 +-
 .../xla/service/dynamic_parameter_binding.h   |   2 +-
 .../service/dynamic_parameter_binding_test.cc | 107 +++++++++++-------
 3 files changed, 70 insertions(+), 42 deletions(-)

diff --git a/tensorflow/compiler/xla/service/dynamic_parameter_binding.cc b/tensorflow/compiler/xla/service/dynamic_parameter_binding.cc
index 5549cccfa8..e9c8aa03e2 100644
--- a/tensorflow/compiler/xla/service/dynamic_parameter_binding.cc
+++ b/tensorflow/compiler/xla/service/dynamic_parameter_binding.cc
@@ -29,7 +29,8 @@ Status DynamicParameterBinding::Bind(
 }
 
 absl::optional<DynamicParameterBinding::DynamicParameter>
-DynamicParameterBinding::GetBinding(const DynamicDimension& dynamic_dimension) {
+DynamicParameterBinding::GetBinding(
+    const DynamicDimension& dynamic_dimension) const {
   auto param_iter = bindings_.find(dynamic_dimension);
   if (param_iter == bindings_.end()) {
     return absl::nullopt;
diff --git a/tensorflow/compiler/xla/service/dynamic_parameter_binding.h b/tensorflow/compiler/xla/service/dynamic_parameter_binding.h
index dd474d8eed..57af2c43d3 100644
--- a/tensorflow/compiler/xla/service/dynamic_parameter_binding.h
+++ b/tensorflow/compiler/xla/service/dynamic_parameter_binding.h
@@ -89,7 +89,7 @@ class DynamicParameterBinding {
   //
   // Returns nullopt if the binding is not set.
   absl::optional<DynamicParameter> GetBinding(
-      const DynamicDimension& dynamic_dimension);
+      const DynamicDimension& dynamic_dimension) const;
 
   using BindingFn =
       std::function<Status(const DynamicParameter& dynamic_parameter,
diff --git a/tensorflow/compiler/xla/service/dynamic_parameter_binding_test.cc b/tensorflow/compiler/xla/service/dynamic_parameter_binding_test.cc
index 83a6d83dff..b5d57cda4f 100644
--- a/tensorflow/compiler/xla/service/dynamic_parameter_binding_test.cc
+++ b/tensorflow/compiler/xla/service/dynamic_parameter_binding_test.cc
@@ -33,7 +33,15 @@ limitations under the License.
 
 namespace xla {
 namespace {
-class DynamicParameterBindingTest : public HloTestBase {};
+class DynamicParameterBindingTest : public HloTestBase {
+ protected:
+  // Serialize and then deserialize a binding.
+  void SerializeAndDeserialize(DynamicParameterBinding* binding) {
+    DynamicParameterBindingProto proto = binding->ToProto();
+    TF_ASSERT_OK_AND_ASSIGN(*binding,
+                            DynamicParameterBinding::CreateFromProto(proto));
+  }
+};
 
 TEST_F(DynamicParameterBindingTest, SimpleBinding) {
   // 'b' is a dynamic shape; 'a' represents the real size of b's first
@@ -56,15 +64,20 @@ ENTRY main {
       binding.Bind(DynamicParameterBinding::DynamicParameter{0, {}},
                    DynamicParameterBinding::DynamicDimension{1, {}, 0}));
 
-  absl::optional<DynamicParameterBinding::DynamicParameter> param =
-      binding.GetBinding(
-          DynamicParameterBinding::DynamicDimension{/*parameter_num=*/1,
-                                                    /*parameter_index=*/{},
-                                                    /*dimension=*/0});
-  EXPECT_TRUE(param);
-  EXPECT_EQ(param->parameter_num, 0);
-  EXPECT_EQ(param->parameter_index, ShapeIndex({}));
-  TF_EXPECT_OK(binding.Verify(*module));
+  auto test = [&](const DynamicParameterBinding& binding) {
+    absl::optional<DynamicParameterBinding::DynamicParameter> param =
+        binding.GetBinding(
+            DynamicParameterBinding::DynamicDimension{/*parameter_num=*/1,
+                                                      /*parameter_index=*/{},
+                                                      /*dimension=*/0});
+    EXPECT_TRUE(param);
+    EXPECT_EQ(param->parameter_num, 0);
+    EXPECT_EQ(param->parameter_index, ShapeIndex({}));
+    TF_EXPECT_OK(binding.Verify(*module));
+  };
+  test(binding);
+  SerializeAndDeserialize(&binding);
+  test(binding);
 }
 
 TEST_F(DynamicParameterBindingTest, TupleBinding) {
@@ -89,16 +102,21 @@ ENTRY main {
       binding.Bind(DynamicParameterBinding::DynamicParameter{0, {0}},
                    DynamicParameterBinding::DynamicDimension{0, {1}, 0}));
 
-  absl::optional<DynamicParameterBinding::DynamicParameter> param =
-      binding.GetBinding(
-          DynamicParameterBinding::DynamicDimension{/*parameter_num=*/0,
-                                                    /*parameter_index=*/{1},
-                                                    /*dimension=*/0});
-
-  EXPECT_TRUE(param);
-  EXPECT_EQ(param->parameter_num, 0);
-  EXPECT_EQ(param->parameter_index, ShapeIndex({0}));
-  TF_EXPECT_OK(binding.Verify(*module));
+  auto test = [&](const DynamicParameterBinding& binding) {
+    absl::optional<DynamicParameterBinding::DynamicParameter> param =
+        binding.GetBinding(
+            DynamicParameterBinding::DynamicDimension{/*parameter_num=*/0,
+                                                      /*parameter_index=*/{1},
+                                                      /*dimension=*/0});
+
+    EXPECT_TRUE(param);
+    EXPECT_EQ(param->parameter_num, 0);
+    EXPECT_EQ(param->parameter_index, ShapeIndex({0}));
+    TF_EXPECT_OK(binding.Verify(*module));
+  };
+  test(binding);
+  SerializeAndDeserialize(&binding);
+  test(binding);
 }
 
 TEST_F(DynamicParameterBindingTest, TupleBindingWithMultiDimension) {
@@ -127,26 +145,35 @@ ENTRY main {
       binding.Bind(DynamicParameterBinding::DynamicParameter{0, {0}},
                    DynamicParameterBinding::DynamicDimension{0, {1}, 1}));
 
-  absl::optional<DynamicParameterBinding::DynamicParameter> param =
-      binding.GetBinding(
-          DynamicParameterBinding::DynamicDimension{/*parameter_num=*/0,
-                                                    /*parameter_index=*/{1},
-                                                    /*dimension=*/0});
-
-  EXPECT_TRUE(param);
-  EXPECT_EQ(param->parameter_num, 0);
-  EXPECT_EQ(param->parameter_index, ShapeIndex({0}));
-
-  absl::optional<DynamicParameterBinding::DynamicParameter> param2 =
-      binding.GetBinding(
-          DynamicParameterBinding::DynamicDimension{/*parameter_num=*/0,
-                                                    /*parameter_index=*/{1},
-                                                    /*dimension=*/0});
-  EXPECT_TRUE(param2);
-  EXPECT_EQ(param2->parameter_num, 0);
-  EXPECT_EQ(param2->parameter_index, ShapeIndex({0}));
-
-  TF_EXPECT_OK(binding.Verify(*module));
+  auto test = [&](const DynamicParameterBinding& binding) {
+    absl::optional<DynamicParameterBinding::DynamicParameter> param =
+        binding.GetBinding(
+            DynamicParameterBinding::DynamicDimension{/*parameter_num=*/0,
+                                                      /*parameter_index=*/{1},
+                                                      /*dimension=*/0});
+
+    EXPECT_TRUE(param);
+    EXPECT_EQ(param->parameter_num, 0);
+    EXPECT_EQ(param->parameter_index, ShapeIndex({0}));
+
+    absl::optional<DynamicParameterBinding::DynamicParameter> param2 =
+
+        binding.GetBinding(
+            DynamicParameterBinding::DynamicDimension{/*parameter_num=*/0,
+                                                      /*parameter_index=*/{1},
+                                                      /*dimension=*/0});
+    EXPECT_TRUE(param2);
+    EXPECT_EQ(param2->parameter_num, 0);
+    EXPECT_EQ(param2->parameter_index, ShapeIndex({0}));
+    TF_EXPECT_OK(binding.Verify(*module));
+  };
+
+  test(binding);
+
+  SerializeAndDeserialize(&binding);
+
+  // Test the binding again after deserialization.
+  test(binding);
 }
 
 }  // namespace
-- 
GitLab


From 522066c826bbfb73cba6c07adcb62e00f4542017 Mon Sep 17 00:00:00 2001
From: Yu-Cheng Ling <ycling@google.com>
Date: Tue, 22 Jan 2019 15:13:33 -0800
Subject: [PATCH 1084/1095] Fix TFLite Makefile

Do not build with NNAPI for microcontroller or iOS

PiperOrigin-RevId: 230420008
---
 tensorflow/lite/tools/make/Makefile | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/tensorflow/lite/tools/make/Makefile b/tensorflow/lite/tools/make/Makefile
index 994f660dba..e98ba9b2e0 100644
--- a/tensorflow/lite/tools/make/Makefile
+++ b/tensorflow/lite/tools/make/Makefile
@@ -111,15 +111,27 @@ $(wildcard tensorflow/lite/*/*/*test.cc) \
 $(wildcard tensorflow/lite/*/*/*/*test.cc) \
 $(wildcard tensorflow/lite/kernels/test_util.cc) \
 $(MINIMAL_SRCS)
+
 ifeq ($(BUILD_TYPE),micro)
-CORE_CC_EXCLUDE_SRCS += \
-tensorflow/lite/mmap_allocation.cc \
-tensorflow/lite/nnapi_delegate.cc
+	CORE_CC_EXCLUDE_SRCS += tensorflow/lite/mmap_allocation.cc
 else
-CORE_CC_EXCLUDE_SRCS += \
-tensorflow/lite/mmap_allocation_disabled.cc \
-tensorflow/lite/nnapi_delegate_disabled.cc
+	CORE_CC_EXCLUDE_SRCS += tensorflow/lite/mmap_allocation_disabled.cc
+endif
+
+BUILD_WITH_NNAPI=true
+ifeq ($(BUILD_TYPE),micro)
+	BUILD_WITH_NNAPI=false
+endif
+ifeq ($(TARGET),ios)
+	BUILD_WITH_NNAPI=false
 endif
+ifeq ($(BUILD_WITH_NNAPI),true)
+	CORE_CC_EXCLUDE_SRCS += tensorflow/lite/nnapi_delegate_disabled.cc
+else
+	CORE_CC_EXCLUDE_SRCS += tensorflow/lite/nnapi_delegate.cc
+endif
+
+
 # Filter out all the excluded files.
 TF_LITE_CC_SRCS := $(filter-out $(CORE_CC_EXCLUDE_SRCS), $(CORE_CC_ALL_SRCS))
 
-- 
GitLab


From 93d5185f1668c390224224972f166fd9afb786ee Mon Sep 17 00:00:00 2001
From: Rachel Lim <rachelim@google.com>
Date: Tue, 22 Jan 2019 15:26:48 -0800
Subject: [PATCH 1085/1095] [tf.data] Make map_vectorization optimization work
 with ExperimentalMapAndBatchDataset as well. i.e.
 dataset.apply(tf.data.experimental.map_and_batch(map_fn, ...)) ->
 dataset.batch(...).map(vectorized_map_fn)

PiperOrigin-RevId: 230422450
---
 .../core/grappler/optimizers/data/BUILD       |   2 +
 .../optimizers/data/map_vectorization.cc      | 231 ++++++++++++-----
 .../optimizers/data/map_vectorization.h       |   6 +
 .../optimizers/data/map_vectorization_test.cc | 235 ++++++++++++++----
 .../kernel_tests/optimization/BUILD           |   3 +-
 .../optimization/map_vectorization_test.py    |  81 +++++-
 6 files changed, 443 insertions(+), 115 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/data/BUILD b/tensorflow/core/grappler/optimizers/data/BUILD
index 0a645d5cf4..93e8405d18 100644
--- a/tensorflow/core/grappler/optimizers/data/BUILD
+++ b/tensorflow/core/grappler/optimizers/data/BUILD
@@ -481,6 +481,7 @@ cc_library(
         ":optimizer_base",
         ":vectorization_utils",
         "@com_google_absl//absl/container:flat_hash_set",
+        "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core/grappler:mutable_graph_view",
         "//tensorflow/core/grappler:grappler_item",
@@ -505,6 +506,7 @@ tf_cc_test(
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
         "//tensorflow/core/grappler:grappler_item",
+        "//tensorflow/core/grappler/utils:topological_sort",
         "//tensorflow/core/kernels/data",
     ],
 )
diff --git a/tensorflow/core/grappler/optimizers/data/map_vectorization.cc b/tensorflow/core/grappler/optimizers/data/map_vectorization.cc
index db64cfff02..bc935bbd0a 100644
--- a/tensorflow/core/grappler/optimizers/data/map_vectorization.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_vectorization.cc
@@ -19,6 +19,7 @@ limitations under the License.
 #include "absl/container/flat_hash_set.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/node_def.pb.h"
+#include "tensorflow/core/framework/node_def_util.h"
 #include "tensorflow/core/framework/tensor.pb.h"  // NOLINT
 #include "tensorflow/core/framework/tensor_shape.pb.h"
 #include "tensorflow/core/grappler/clusters/cluster.h"
@@ -29,6 +30,7 @@ limitations under the License.
 #include "tensorflow/core/grappler/optimizers/data/function_utils.h"
 #include "tensorflow/core/grappler/optimizers/data/graph_utils.h"
 #include "tensorflow/core/grappler/utils.h"
+#include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/gtl/map_util.h"
 #include "tensorflow/core/platform/protobuf.h"
 
@@ -36,6 +38,12 @@ namespace tensorflow {
 namespace grappler {
 namespace {
 
+constexpr char kBatchOp[] = "BatchDataset";
+constexpr char kBatchV2Op[] = "BatchDatasetV2";
+constexpr char kExperimentalMapAndBatchOp[] = "ExperimentalMapAndBatchDataset";
+constexpr char kMapOp[] = "MapDataset";
+constexpr char kParallelMapOp[] = "ParallelMapDataset";
+
 // Returns a FunctionDef containing a MapDefun op that wraps the original
 // function.
 FunctionDef* CreateMapDefunWrapper(const NodeDef& map_node,
@@ -101,7 +109,6 @@ FunctionDef* AddVectorizedFunction(const NodeDef& map_node,
   const NodeDef& map_defun_node = vectorized_func->node_def(0);
   DCHECK_EQ(map_defun_node.op(), "MapDefun");
 
-  // TODO(b/116285210): Unreferenced functions should get cleaned up later
   FunctionDef* result;
   Status s = vectorization_utils::VectorizeMapDefun(
       *vectorized_func, map_defun_node, library, &result);
@@ -131,23 +138,53 @@ bool IsOutputShapesFullyDefined(const NodeDef& node) {
   return true;
 }
 
+// Returns a mapping from input names to the [start, end) indices of the input
+// in the node's input list.
+Status GetInputMap(const NodeDef& node, NameRangeMap* result) {
+  const OpRegistrationData* op_reg_data;  // Owned by global op registry
+  TF_RETURN_IF_ERROR(OpRegistry::Global()->LookUp(node.op(), &op_reg_data));
+
+  return NameRangesForNode(node, op_reg_data->op_def, result,
+                           /*outputs=*/nullptr);
+}
+
+Status CopyInputs(StringPiece input_name, const NameRangeMap& input_map,
+                  const NodeDef& from, NodeDef* to) {
+  const auto* range = gtl::FindOrNull(input_map, input_name);
+  if (range == nullptr) {
+    return errors::Internal(
+        "Failed to copy inputs: did not find inputs with name: ", input_name,
+        ", in node with name: ", from.name());
+  }
+  for (int i = range->first; i < range->second; ++i) {
+    to->add_input(from.input(i));
+  }
+
+  return Status::OK();
+}
 
-NodeDef MakeNewBatchNode(const NodeDef& old_batch_node,
-                         const NodeDef& input_node,
-                         const FunctionDef& vectorized_func,
-                         MutableGraphView* graph) {
+Status AddNewBatchNode(const NodeDef& old_batch_node, const NodeDef& input_node,
+                       const FunctionDef& vectorized_func,
+                       MutableGraphView* graph, NodeDef** new_batch_node) {
   NodeDef batch_node;
-  batch_node.set_op(old_batch_node.op());
+  batch_node.set_op(old_batch_node.op() == kBatchOp ? kBatchOp : kBatchV2Op);
   graph_utils::SetUniqueGraphNodeName(batch_node.op(), graph->graph(),
                                       &batch_node);
 
   // Set the `input_dataset` input argument
   batch_node.add_input(input_node.name());
-  // Set the `batch_size` input_argument
-  batch_node.add_input(old_batch_node.input(1));
-  if (batch_node.op() == "BatchDatasetV2") {
-    // Set the `drop_remainder` input argument
-    batch_node.add_input(old_batch_node.input(2));
+
+  NameRangeMap input_map;
+  TF_RETURN_IF_ERROR(GetInputMap(old_batch_node, &input_map));
+
+  // Set the `batch_size` input argument
+  TF_RETURN_IF_ERROR(
+      CopyInputs("batch_size", input_map, old_batch_node, &batch_node));
+
+  // Set the `drop_remainder` input argument
+  if (batch_node.op() != kBatchOp) {
+    TF_RETURN_IF_ERROR(
+        CopyInputs("drop_remainder", input_map, old_batch_node, &batch_node));
   }
 
   // Set attrs
@@ -157,6 +194,9 @@ NodeDef MakeNewBatchNode(const NodeDef& old_batch_node,
   }
   (*batch_node.mutable_attr())["output_types"] = output_types;
 
+  // It is safe to assume that input_node has the "output_shapes" attr here,
+  // because earlier we checked that the input node has fully defined output
+  // shapes.
   auto& output_shapes_attr = (*batch_node.mutable_attr())["output_shapes"];
   const auto& input_shapes =
       input_node.attr().at("output_shapes").list().shape();
@@ -168,23 +208,54 @@ NodeDef MakeNewBatchNode(const NodeDef& old_batch_node,
     dim->set_size(batch_size);
     shape->MergeFrom(input_shapes.Get(i));
   }
-  return batch_node;
+
+  *new_batch_node = graph->AddNode(std::move(batch_node));
+  return Status::OK();
 }
 
-NodeDef MakeNewMapNode(const NodeDef& old_map_node,
-                       const NodeDef& old_batch_node,
-                       const NodeDef& new_batch_node,
-                       const FunctionDef& vectorized_func,
-                       MutableGraphView* graph) {
+Status AddNewMapNode(const NodeDef& old_map_node, const NodeDef& old_batch_node,
+                     const NodeDef& new_batch_node,
+                     const FunctionDef& vectorized_func,
+                     MutableGraphView* graph, NodeDef** new_map_node) {
   NodeDef map_node;
-  map_node.set_op(old_map_node.op());
+  map_node.set_op(old_map_node.op() == kMapOp ? kMapOp : kParallelMapOp);
   graph_utils::SetUniqueGraphNodeName(map_node.op(), graph->graph(), &map_node);
 
   // Set the `input_dataset` input argument
   map_node.add_input(new_batch_node.name());
-  for (int i = 1; i < old_map_node.input_size(); i++) {
-    // Set the `other_arguments` and `num_parallel_calls` input arguments
-    map_node.add_input(old_map_node.input(i));
+
+  NameRangeMap input_map;
+  TF_RETURN_IF_ERROR(GetInputMap(old_map_node, &input_map));
+
+  // Set the `other_arguments` input argument
+  TF_RETURN_IF_ERROR(
+      CopyInputs("other_arguments", input_map, old_map_node, &map_node));
+
+  // Set the `num_parallel_calls` input argument
+  // TODO(rachelim): Evaluate the performance of potential transformations to
+  // the new `num_parallel_calls`:
+  //   1) dividing by the `batch_size`, since the new map will be operating
+  //      over larger elements
+  //   2) use the autotune value (i.e. -1)
+  //   3) use the original value
+  if (old_map_node.op() == kExperimentalMapAndBatchOp) {
+    // This `Cast` op is necessary because the `num_parallel_calls` input for
+    // ExperimentalMapAndBatch has type DT_INT64, but the input for ParallelMap
+    // expects type DT_INT32
+    NodeDef cast_node;
+    cast_node.set_op("Cast");
+    graph_utils::SetUniqueGraphNodeName(cast_node.op(), graph->graph(),
+                                        &cast_node);
+    AddNodeAttr("SrcT", DT_INT64, &cast_node);
+    AddNodeAttr("DstT", DT_INT32, &cast_node);
+    TF_RETURN_IF_ERROR(
+        CopyInputs("num_parallel_calls", input_map, old_map_node, &cast_node));
+    auto added_cast_node = graph->AddNode(std::move(cast_node));
+    map_node.add_input(added_cast_node->name());
+
+  } else if (old_map_node.op() == kParallelMapOp) {
+    TF_RETURN_IF_ERROR(
+        CopyInputs("num_parallel_calls", input_map, old_map_node, &map_node));
   }
 
   // Set attrs
@@ -197,8 +268,65 @@ NodeDef MakeNewMapNode(const NodeDef& old_map_node,
   }
 
   (*map_node.mutable_attr())["use_inter_op_parallelism"].set_b(true);
+  *new_map_node = graph->AddNode(std::move(map_node));
+  return Status::OK();
+}
 
-  return map_node;
+// Given an input pipeline graph and a query node, tries to the node to the
+// 'batch' node in a input_dataset->map->batch pattern, or the 'map_and_batch'
+// node in an input_dataset->map_and_batch pattern.
+bool FindMapAndBatchPattern(const MutableGraphView& graph, const NodeDef& node,
+                            const FunctionLibraryDefinition& function_library,
+                            const NodeDef** batch_node_output,
+                            const NodeDef** map_node_output,
+                            const NodeDef** input_node_output,
+                            const FunctionDef** map_fn_output) {
+  const FunctionDef*& map_fn = *map_fn_output;
+  const NodeDef*& batch_node = *batch_node_output;
+  const NodeDef*& map_node = *map_node_output;
+  const NodeDef*& input_node = *input_node_output;
+
+  if (node.op() == kExperimentalMapAndBatchOp) {
+    batch_node = &node;
+    map_node = &node;
+  } else if (node.op() == kBatchOp || node.op() == kBatchV2Op) {
+    batch_node = &node;
+    map_node = graph_utils::GetInputNode(*batch_node, graph);
+    if (map_node->op() != kMapOp && map_node->op() != kParallelMapOp) {
+      return false;
+    }
+    if (!IsOutputShapesFullyDefined(*map_node)) {
+      // If any of the map func outputs have an unknown shape, don't
+      // optimize, so that batching errors surface as before.
+      VLOG(1) << "Cannot vectorize dataset.map().batch() because the map "
+                 "dataset does not have fully defined output shapes.";
+      return false;
+    }
+  } else {
+    return false;
+  }
+
+  // Input to the map node
+  input_node = graph_utils::GetInputNode(*map_node, graph);
+  DCHECK_NE(input_node, nullptr);
+
+  if (!IsOutputShapesFullyDefined(*input_node)) {
+    // If any of the inputs have an unknown shape, don't optimize, since
+    // inputs might not be batchable.
+    VLOG(1) << "Cannot vectorize dataset.map().batch() because the input "
+               "dataset does not have fully defined output shapes.";
+    return false;
+  }
+
+  map_fn = function_library.Find(map_node->attr().at("f").func().name());
+
+  if (function_utils::IsFunctionStateful(function_library, *map_fn)) {
+    VLOG(1) << "Cannot vectorize dataset.map().batch() because the map "
+               "function is stateful.";
+    return false;
+  }
+
+  return true;
 }
 
 }  // namespace
@@ -211,58 +339,37 @@ Status MapVectorization::OptimizeAndCollectStats(Cluster* cluster,
   MutableGraphView graph(output);
   absl::flat_hash_set<string> nodes_to_delete;
 
-  for (const NodeDef& node : item.graph.node()) {
-    // Find Map->Batch nodes.
-    // TODO(rachelim): Optimize MapAndBatchDataset[V2] as well.
-    if (node.op() != "BatchDataset" && node.op() != "BatchDatasetV2") {
-      continue;
-    }
-
-    const NodeDef& batch_node(node);
-    NodeDef* node2 = graph_utils::GetInputNode(batch_node, graph);
-    if (node2->op() != "MapDataset" && node2->op() != "ParallelMapDataset") {
-      continue;
-    }
-
-    // Use a more descriptive variable name now that we know the node type.
-    NodeDef* map_node = node2;
-    // Input to the map node
-    NodeDef* input_node = graph_utils::GetInputNode(*map_node, graph);
-    CHECK_NOTNULL(input_node);
-
-    FunctionDefLibrary* library = output->mutable_library();
+  FunctionDefLibrary* library = output->mutable_library();
 
+  for (const NodeDef& node : item.graph.node()) {
     FunctionLibraryDefinition function_library(OpRegistry::Global(), *library);
-    const FunctionDef* orig_func =
-        function_library.Find(map_node->attr().at("f").func().name());
-
-    // Check that this is a valid optimization.
-    if (!IsOutputShapesFullyDefined(*input_node) ||
-        !IsOutputShapesFullyDefined(*map_node) ||
-        function_utils::IsFunctionStateful(function_library, *orig_func)) {
-      // 1. If any of the inputs have an unknown shape, don't optimize, since
-      // inputs might not be batchable.
-      // 2. If any of the map func outputs have an unknown shape, don't
-      // optimize, so that batching errors surface as before.
-      // 3. If the function is stateful, don't vectorize it.
+    const NodeDef* map_node;
+    const NodeDef* batch_node;
+    const NodeDef* input_node;
+    const FunctionDef* map_func;
+    if (!FindMapAndBatchPattern(graph, node, function_library, &batch_node,
+                                &map_node, &input_node, &map_func)) {
       continue;
     }
 
     FunctionDef* vectorized_func =
-        AddVectorizedFunction(*map_node, *orig_func, library);
+        AddVectorizedFunction(*map_node, *map_func, library);
     CHECK_NOTNULL(vectorized_func);
 
-    auto* new_batch_node = graph.AddNode(
-        MakeNewBatchNode(batch_node, *input_node, *vectorized_func, &graph));
+    NodeDef* new_batch_node;
+    TF_RETURN_IF_ERROR(AddNewBatchNode(
+        *batch_node, *input_node, *vectorized_func, &graph, &new_batch_node));
 
-    auto* new_map_node = graph.AddNode(MakeNewMapNode(
-        *map_node, batch_node, *new_batch_node, *vectorized_func, &graph));
-    TF_RETURN_IF_ERROR(
-        graph.UpdateFanouts(batch_node.name(), new_map_node->name()));
+    NodeDef* new_map_node;
+    TF_RETURN_IF_ERROR(AddNewMapNode(*map_node, *batch_node, *new_batch_node,
+                                     *vectorized_func, &graph, &new_map_node));
 
+    // Make output of Batch point to Map instead.
+    TF_RETURN_IF_ERROR(
+        graph.UpdateFanouts(batch_node->name(), new_map_node->name()));
     // Mark the `Map` and `Batch` nodes for removal.
     nodes_to_delete.insert(map_node->name());
-    nodes_to_delete.insert(batch_node.name());
+    nodes_to_delete.insert(batch_node->name());
     stats->num_changes++;
   }
   TF_RETURN_IF_ERROR(graph.DeleteNodes(nodes_to_delete));
diff --git a/tensorflow/core/grappler/optimizers/data/map_vectorization.h b/tensorflow/core/grappler/optimizers/data/map_vectorization.h
index cde6ef6839..3e170aae27 100644
--- a/tensorflow/core/grappler/optimizers/data/map_vectorization.h
+++ b/tensorflow/core/grappler/optimizers/data/map_vectorization.h
@@ -21,6 +21,12 @@ limitations under the License.
 namespace tensorflow {
 namespace grappler {
 
+// This optimizer rewrites dataset.map(map_fn, ...).batch(...) and
+// dataset.apply(tf.data.experimental.map_and_batch(map_fn, ...)) patterns in an
+// input pipeline. It vectorizes the map_fn, such that this segment can be
+// rewritten as dataset.batch().map(vectorized_map_fn). This is more performant
+// when the map_fn is cheap, because it amortizes the cost of running a map
+// function over a larger batch.
 class MapVectorization : public TFDataOptimizerBase {
  public:
   MapVectorization() = default;
diff --git a/tensorflow/core/grappler/optimizers/data/map_vectorization_test.cc b/tensorflow/core/grappler/optimizers/data/map_vectorization_test.cc
index 890041f57c..8a6df49758 100644
--- a/tensorflow/core/grappler/optimizers/data/map_vectorization_test.cc
+++ b/tensorflow/core/grappler/optimizers/data/map_vectorization_test.cc
@@ -20,6 +20,7 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor_testutil.h"
 #include "tensorflow/core/grappler/grappler_item.h"
 #include "tensorflow/core/grappler/optimizers/data/graph_utils.h"
+#include "tensorflow/core/grappler/utils/topological_sort.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/lib/gtl/array_slice.h"
 #include "tensorflow/core/platform/test.h"
@@ -28,28 +29,45 @@ namespace tensorflow {
 namespace grappler {
 namespace {
 
+constexpr char kConstOp[] = "Const";
+constexpr char kRangeOp[] = "RangeDataset";
+constexpr char kBatchOp[] = "BatchDataset";
+constexpr char kBatchV2Op[] = "BatchDatasetV2";
+constexpr char kExperimentalMapAndBatchOp[] = "ExperimentalMapAndBatchDataset";
+constexpr char kMapOp[] = "MapDataset";
+constexpr char kParallelMapOp[] = "ParallelMapDataset";
+constexpr char kAttrNameF[] = "f";
+constexpr char kAttrNameTarguments[] = "Targuments";
+constexpr char kAttrNameOutputTypes[] = "output_types";
+constexpr char kAttrNameOutputShapes[] = "output_shapes";
+constexpr char kAttrNameInterOpParallelism[] = "use_inter_op_parallelism";
+constexpr char kAttrNamePreserveCardinality[] = "preserve_cardinality";
+constexpr char kAttrNameSloppy[] = "sloppy";
+constexpr char kAttrNameValue[] = "value";
+constexpr char kAttrNameDtype[] = "dtype";
+
 using test::function::NDef;
 
 NodeDef* AddConstBoolNode(GraphDef* graph, StringPiece name, bool value) {
   NodeDef* node = graph->add_node();
-  *node = NDef(/*name=*/name, /*op=*/"Const", /*inputs=*/{},
-               /*attrs=*/{{"value", value}, {"dtype", DT_BOOL}});
+  *node = NDef(/*name=*/name, /*op=*/kConstOp, /*inputs=*/{},
+               /*attrs=*/{{kAttrNameValue, value}, {kAttrNameDtype, DT_BOOL}});
 
   return node;
 }
 
 NodeDef* AddConstInt32Node(GraphDef* graph, StringPiece name, int value) {
   NodeDef* node = graph->add_node();
-  *node = NDef(/*name=*/name, /*op=*/"Const", /*inputs=*/{},
-               /*attrs=*/{{"value", value}, {"dtype", DT_INT32}});
+  *node = NDef(/*name=*/name, /*op=*/kConstOp, /*inputs=*/{},
+               /*attrs=*/{{kAttrNameValue, value}, {kAttrNameDtype, DT_INT32}});
 
   return node;
 }
 
 NodeDef* AddConstInt64Node(GraphDef* graph, StringPiece name, int64 value) {
   NodeDef* node = graph->add_node();
-  *node = NDef(/*name=*/name, /*op=*/"Const", /*inputs=*/{},
-               /*attrs=*/{{"value", value}, {"dtype", DT_INT64}});
+  *node = NDef(/*name=*/name, /*op=*/kConstOp, /*inputs=*/{},
+               /*attrs=*/{{kAttrNameValue, value}, {kAttrNameDtype, DT_INT64}});
 
   return node;
 }
@@ -70,60 +88,64 @@ FunctionDef* AddMapFn(GraphDef* graph) {
 }
 
 NodeDef* AddMapNode(GraphDef* graph, const string& input_dataset,
-                    bool parallel = false) {
+                    const string& map_fn, bool parallel = false,
+                    int suffix = 0) {
   NodeDef* map_node = graph->add_node();
-  auto map_fn = AddMapFn(graph);
 
   if (parallel) {
-    auto num_parallel_calls = AddConstInt32Node(graph, "num_parallel_calls", 8);
-    *map_node =
-        NDef(/*name=*/"map", /*op=*/"ParallelMapDataset",
-             /*inputs=*/{input_dataset, num_parallel_calls->name()},
-             /*attrs=*/
-             {{"f", FunctionDefHelper::FunctionRef(map_fn->signature().name())},
-              {"Targuments", gtl::ArraySlice<DataType>({})},
-              {"output_types", gtl::ArraySlice<DataType>({DT_INT64})},
-              {"output_shapes", gtl::ArraySlice<TensorShape>({{}})},
-              {"use_inter_op_parallelism", false},
-              {"sloppy", true},
-              {"preserve_cardinality", true}});
+    auto num_parallel_calls = AddConstInt32Node(
+        graph, strings::StrCat("num_parallel_calls", suffix), 8);
+    *map_node = NDef(
+        /*name=*/strings::StrCat("map", suffix), /*op=*/kParallelMapOp,
+        /*inputs=*/{input_dataset, num_parallel_calls->name()},
+        /*attrs=*/
+        {{kAttrNameF, FunctionDefHelper::FunctionRef(map_fn)},
+         {kAttrNameTarguments, gtl::ArraySlice<DataType>({})},
+         {kAttrNameOutputTypes, gtl::ArraySlice<DataType>({DT_INT64})},
+         {kAttrNameOutputShapes, gtl::ArraySlice<TensorShape>({{}})},
+         {kAttrNameInterOpParallelism, false},
+         {kAttrNameSloppy, true},
+         {kAttrNamePreserveCardinality, true}});
   } else {
     *map_node =
-        NDef(/*name=*/"map", /*op=*/"MapDataset",
+        NDef(/*name=*/strings::StrCat("map", suffix), /*op=*/kMapOp,
              /*inputs=*/{input_dataset},
              /*attrs=*/
-             {{"f", FunctionDefHelper::FunctionRef(map_fn->signature().name())},
-              {"Targuments", gtl::ArraySlice<DataType>({})},
-              {"output_types", gtl::ArraySlice<DataType>({DT_INT64})},
-              {"output_shapes", gtl::ArraySlice<TensorShape>({{}})},
-              {"use_inter_op_parallelism", false},
-              {"preserve_cardinality", true}});
+             {{kAttrNameF, FunctionDefHelper::FunctionRef(map_fn)},
+              {kAttrNameTarguments, gtl::ArraySlice<DataType>({})},
+              {kAttrNameOutputTypes, gtl::ArraySlice<DataType>({DT_INT64})},
+              {kAttrNameOutputShapes, gtl::ArraySlice<TensorShape>({{}})},
+              {kAttrNameInterOpParallelism, false},
+              {kAttrNamePreserveCardinality, true}});
   }
 
   return map_node;
 }
 
 NodeDef* AddBatchNode(GraphDef* graph, const string& input_dataset,
-                      bool v2 = false) {
+                      bool v2 = false, int suffix = 0) {
   NodeDef* batch_node = graph->add_node();
 
-  auto batch_size = AddConstInt64Node(graph, "batch_size", 10);
+  auto batch_size =
+      AddConstInt64Node(graph, strings::StrCat("batch_size", suffix), 10);
 
   if (v2) {
-    auto drop_remainder = AddConstBoolNode(graph, "drop_remainder", true);
+    auto drop_remainder = AddConstBoolNode(
+        graph, strings::StrCat("drop_remainder", suffix), true);
     *batch_node = NDef(
-        /*name=*/"batch", /*op=*/"BatchDatasetV2",
+        /*name=*/strings::StrCat("batch", suffix), /*op=*/kBatchV2Op,
         /*inputs=*/{input_dataset, batch_size->name(), drop_remainder->name()},
         /*attrs=*/
-        {{"output_types", gtl::ArraySlice<DataType>({DT_INT64})},
-         {"output_shapes", gtl::ArraySlice<TensorShape>({{10, 1}})}});
+        {{kAttrNameOutputTypes, gtl::ArraySlice<DataType>({DT_INT64})},
+         {kAttrNameOutputShapes, gtl::ArraySlice<TensorShape>({{10, 1}})}});
   } else {
     *batch_node = NDef(
-        /*name=*/"batch", /*op=*/"BatchDataset",
+        /*name=*/strings::StrCat("batch", suffix), /*op=*/kBatchOp,
         /*inputs=*/{input_dataset, batch_size->name()},
         /*attrs=*/
-        {{"output_types", gtl::ArraySlice<DataType>({DT_INT64})},
-         {"output_shapes", gtl::ArraySlice<PartialTensorShape>({{-1, 1}})}});
+        {{kAttrNameOutputTypes, gtl::ArraySlice<DataType>({DT_INT64})},
+         {kAttrNameOutputShapes,
+          gtl::ArraySlice<PartialTensorShape>({{v2 ? 10 : -1, 1}})}});
   }
   return batch_node;
 }
@@ -133,11 +155,12 @@ NodeDef* AddRangeNode(GraphDef* graph) {
   auto stop = AddConstInt64Node(graph, "stop", 10);
   auto step = AddConstInt64Node(graph, "step", 1);
   NodeDef* range_node = graph->add_node();
-  *range_node = NDef(/*name=*/"range", /*op=*/"RangeDataset",
-                     /*inputs=*/{start->name(), stop->name(), step->name()},
-                     /*attrs=*/
-                     {{"output_shapes", gtl::ArraySlice<TensorShape>({{}})},
-                      {"output_types", gtl::ArraySlice<DataType>({DT_INT64})}});
+  *range_node =
+      NDef(/*name=*/"range", /*op=*/kRangeOp,
+           /*inputs=*/{start->name(), stop->name(), step->name()},
+           /*attrs=*/
+           {{kAttrNameOutputShapes, gtl::ArraySlice<TensorShape>({{}})},
+            {kAttrNameOutputTypes, gtl::ArraySlice<DataType>({DT_INT64})}});
 
   return range_node;
 }
@@ -167,7 +190,7 @@ void CheckVectorized(const GraphDef& output, const string& map_op,
 
   // Check that the function is actually vectorized.
   // The vectorization of the identity function is itself.
-  string function_name = map_node.attr().at("f").func().name();
+  string function_name = map_node.attr().at(kAttrNameF).func().name();
   int found =
       graph_utils::FindGraphFunctionWithName(function_name, output.library());
   ASSERT_NE(found, -1);
@@ -183,7 +206,9 @@ TEST_P(MapThenBatchTest, IsVectorized) {
   bool use_batch_v2 = std::get<1>(GetParam());
   GrapplerItem item;
   auto range_node = AddRangeNode(&item.graph);
-  auto map_node = AddMapNode(&item.graph, range_node->name(), use_parallel_map);
+  auto map_fn = AddMapFn(&item.graph);
+  auto map_node = AddMapNode(&item.graph, range_node->name(),
+                             map_fn->signature().name(), use_parallel_map);
   auto batch_node = AddBatchNode(&item.graph, map_node->name(), use_batch_v2);
   MapVectorization optimizer;
   GraphDef output;
@@ -195,18 +220,118 @@ INSTANTIATE_TEST_SUITE_P(MapThenBatchTest, MapThenBatchTest,
                          ::testing::Combine(::testing::Bool(),
                                             ::testing::Bool()));
 
-// Not all dataset types have "output_shapes" and "output_types" attrs defined.
-// Add a generic input node which may not have these attrs defined.
+NodeDef* AddMapAndBatchNode(GraphDef* graph, const string& input_dataset,
+                            const string& map_fn, int suffix = 0) {
+  auto batch_size =
+      AddConstInt64Node(graph, strings::StrCat("batch_size", suffix), 10);
+  auto num_parallel_calls = AddConstInt64Node(
+      graph, strings::StrCat("num_parallel_calls", suffix), 8);
+  auto drop_remainder =
+      AddConstBoolNode(graph, strings::StrCat("drop_remainder", suffix), true);
+
+  auto map_and_batch_node = graph->add_node();
+  *map_and_batch_node =
+      NDef(/*name=*/strings::StrCat("map_and_batch", suffix),
+           /*op=*/kExperimentalMapAndBatchOp,
+           /*inputs=*/
+           {input_dataset, batch_size->name(), num_parallel_calls->name(),
+            drop_remainder->name()},
+           /*attrs=*/
+           {{kAttrNameF, FunctionDefHelper::FunctionRef(map_fn)},
+            {kAttrNameTarguments, gtl::ArraySlice<DataType>({})},
+            {kAttrNameOutputTypes, gtl::ArraySlice<DataType>({DT_INT64})},
+            {kAttrNameOutputShapes,
+             gtl::ArraySlice<PartialTensorShape>({{10, 1}})}});
+
+  return map_and_batch_node;
+}
+
+TEST(MapVectorizationTest, VectorizeExperimentalMapAndBatch) {
+  GrapplerItem item;
+  auto range_node = AddRangeNode(&item.graph);
+  auto map_fn = AddMapFn(&item.graph);
+  auto map_and_batch_node = AddMapAndBatchNode(&item.graph, range_node->name(),
+                                               map_fn->signature().name());
+  ASSERT_NE(map_and_batch_node, nullptr);
+
+  MapVectorization optimizer;
+  GraphDef output;
+  TF_ASSERT_OK(optimizer.Optimize(nullptr, item, &output));
+  CheckVectorized(output, kParallelMapOp, kBatchV2Op, "range");
+}
+
+class ChainedMapAndBatchTest
+    : public ::testing::TestWithParam<std::tuple<bool, bool>> {};
+
+// Tests:
+// 1) map.batch.map.batch
+// 2) map.batch.map_and_batch
+// 3) map_and_batch.map.batch
+// 4) map_and_batch.map_and_batch
+TEST_P(ChainedMapAndBatchTest, IsVectorized) {
+  GrapplerItem item;
+  auto input_node = AddRangeNode(&item.graph);
+
+  auto map_fn = AddMapFn(&item.graph);
+
+  auto make_map_and_batch = [&item, map_fn](NodeDef* input, bool fuse,
+                                            int suffix) {
+    if (fuse) {
+      return AddMapAndBatchNode(&item.graph, input->name(),
+                                map_fn->signature().name(), suffix);
+    }
+    auto map_node = AddMapNode(&item.graph, input->name(),
+                               map_fn->signature().name(), true, suffix);
+    auto batch_node = AddBatchNode(&item.graph, map_node->name(), true, suffix);
+    return batch_node;
+  };
+
+  auto map_and_batch_0 =
+      make_map_and_batch(input_node, std::get<0>(GetParam()), 0);
+  auto map_and_batch_1 =
+      make_map_and_batch(map_and_batch_0, std::get<1>(GetParam()), 1);
+  ASSERT_NE(map_and_batch_1, nullptr);
+
+  MapVectorization optimizer;
+  GraphDef output;
+  TF_ASSERT_OK(optimizer.Optimize(nullptr, item, &output));
+  TF_ASSERT_OK(TopologicalSort(&output));
+  std::vector<int> map_nodes =
+      graph_utils::FindAllGraphNodesWithOp(kParallelMapOp, output);
+  std::vector<int> batch_nodes =
+      graph_utils::FindAllGraphNodesWithOp(kBatchV2Op, output);
+  ASSERT_EQ(map_nodes.size(), 2);
+  ASSERT_EQ(batch_nodes.size(), 2);
+  const NodeDef& range_node =
+      output.node(graph_utils::FindGraphNodeWithOp(kRangeOp, output));
+
+  const NodeDef& batch_node_0 = output.node(batch_nodes[0]);
+  EXPECT_EQ(batch_node_0.input(0), range_node.name());
+  const NodeDef& map_node_0 = output.node(map_nodes[0]);
+  EXPECT_EQ(map_node_0.input(0), batch_node_0.name());
+  const NodeDef& batch_node_1 = output.node(batch_nodes[1]);
+  EXPECT_EQ(batch_node_1.input(0), map_node_0.name());
+  const NodeDef& map_node_1 = output.node(map_nodes[1]);
+  EXPECT_EQ(map_node_1.input(0), batch_node_1.name());
+}
+
+INSTANTIATE_TEST_SUITE_P(ChainedMapAndBatchTest, ChainedMapAndBatchTest,
+                         ::testing::Combine(::testing::Bool(),
+                                            ::testing::Bool()));
+
+// Not all dataset types have kAttrNameOutputShapes and kAttrNameOutputTypes
+// attrs defined. Add a generic input node which may not have these attrs
+// defined.
 NodeDef* AddArbitraryInputNode(GraphDef* graph,
                                std::vector<PartialTensorShape>* output_shapes,
                                std::vector<DataType>* output_types) {
   auto input_node = graph->add_node();
   std::vector<std::pair<string, FunctionDefHelper::AttrValueWrapper>> attrs;
   if (output_shapes) {
-    attrs.push_back({"output_shapes", *output_shapes});
+    attrs.push_back({kAttrNameOutputShapes, *output_shapes});
   }
   if (output_types) {
-    attrs.push_back({"output_types", *output_types});
+    attrs.push_back({kAttrNameOutputTypes, *output_types});
   }
 
   *input_node = NDef(/*name=*/"input", /*op=*/"InputDataset", /*inputs=*/{},
@@ -222,7 +347,9 @@ TEST(MapVectorizationTest, VectorizeWithUndefinedOutputShapes) {
   GrapplerItem item;
   std::vector<DataType> input_types({DT_INT64});
   auto input_node = AddArbitraryInputNode(&item.graph, nullptr, &input_types);
-  auto map_node = AddMapNode(&item.graph, input_node->name());
+  auto map_fn = AddMapFn(&item.graph);
+  auto map_node =
+      AddMapNode(&item.graph, input_node->name(), map_fn->signature().name());
   auto batch_node = AddBatchNode(&item.graph, map_node->name());
   MapVectorization optimizer;
   GraphDef output;
@@ -240,7 +367,9 @@ TEST(MapVectorizationTest, VectorizeWithUnknownRank) {
   std::vector<DataType> input_types({DT_INT64});
   auto input_node =
       AddArbitraryInputNode(&item.graph, &input_shapes, &input_types);
-  auto map_node = AddMapNode(&item.graph, input_node->name());
+  auto map_fn = AddMapFn(&item.graph);
+  auto map_node =
+      AddMapNode(&item.graph, input_node->name(), map_fn->signature().name());
   auto batch_node = AddBatchNode(&item.graph, map_node->name());
   MapVectorization optimizer;
   GraphDef output;
@@ -258,7 +387,9 @@ TEST(MapVectorizationTest, VectorizeWithUnknownDim) {
   std::vector<DataType> input_types({DT_INT64});
   auto input_node =
       AddArbitraryInputNode(&item.graph, &input_shapes, &input_types);
-  auto map_node = AddMapNode(&item.graph, input_node->name());
+  auto map_fn = AddMapFn(&item.graph);
+  auto map_node =
+      AddMapNode(&item.graph, input_node->name(), map_fn->signature().name());
   auto batch_node = AddBatchNode(&item.graph, map_node->name());
   MapVectorization optimizer;
   GraphDef output;
@@ -274,7 +405,9 @@ TEST(MapVectorizationTest, VectorizeWithUndefinedOutputTypes) {
   GrapplerItem item;
   std::vector<PartialTensorShape> input_shapes({{1}});
   auto input_node = AddArbitraryInputNode(&item.graph, &input_shapes, nullptr);
-  auto map_node = AddMapNode(&item.graph, input_node->name());
+  auto map_fn = AddMapFn(&item.graph);
+  auto map_node =
+      AddMapNode(&item.graph, input_node->name(), map_fn->signature().name());
   auto batch_node = AddBatchNode(&item.graph, map_node->name());
   MapVectorization optimizer;
   GraphDef output;
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD b/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD
index 703d3350db..3bfe55244e 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD
@@ -207,7 +207,7 @@ py_test(
 
 py_test(
     name = "map_vectorization_test",
-    size = "medium",
+    size = "small",
     srcs = ["map_vectorization_test.py"],
     shard_count = 8,
     srcs_version = "PY2AND3",
@@ -233,6 +233,7 @@ py_test(
         "//tensorflow/python:nn",
         "//tensorflow/python:parsing_ops",
         "//tensorflow/python:sparse_tensor",
+        "//tensorflow/python/data/experimental/ops:batching",
         "//tensorflow/python/data/experimental/ops:optimization",
         "//tensorflow/python/data/experimental/ops:optimization_options",
         "//tensorflow/python/data/kernel_tests:test_base",
diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
index 9ffb8a1825..5e748a78ef 100644
--- a/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
+++ b/tensorflow/python/data/experimental/kernel_tests/optimization/map_vectorization_test.py
@@ -22,6 +22,7 @@ import numpy as np
 
 from tensorflow.core.example import example_pb2
 from tensorflow.core.example import feature_pb2
+from tensorflow.python.data.experimental.ops import batching
 from tensorflow.python.data.experimental.ops import optimization
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
@@ -384,7 +385,7 @@ class MapVectorizationTest(test_base.DatasetTestBase, parameterized.TestCase):
       self.evaluate(nxt)
 
   def testOptimizationWithCapturedInputs(self):
-    # Tests that vectorization works with captured inputs
+    # Tests that vectorization works with captured inputs.
     y = constant_op.constant(1, shape=(2,))
     z = constant_op.constant(2, shape=(2,))
 
@@ -397,6 +398,84 @@ class MapVectorizationTest(test_base.DatasetTestBase, parameterized.TestCase):
         base_dataset, map_fn, expect_optimized=True)
     self.assertDatasetsEqual(optimized, unoptimized)
 
+  def testOptimizationWithMapAndBatchFusion(self):
+    # Tests that vectorization works on fused map and batch.
+    y = constant_op.constant(1, shape=(2,))
+    z = constant_op.constant(2, shape=(2,))
+
+    def map_fn(x):
+      return x, y, z
+
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    base_dataset = dataset_ops.Dataset.from_tensor_slices([[1, 2],
+                                                           [3, 4]]).repeat(5)
+    base_dataset = base_dataset.with_options(options)
+
+    def _make_dataset(node_names):
+      dataset = base_dataset.apply(optimization.assert_next(node_names))
+      dataset = dataset.apply(batching.map_and_batch(map_fn, 100))
+      return dataset
+
+    unoptimized = _make_dataset(["MapAndBatch"])
+    optimized = _make_dataset(["Batch", "ParallelMap"])
+    options = dataset_ops.Options()
+    options.experimental_optimization.map_vectorization = True
+    optimized = optimized.with_options(options)
+    self.assertDatasetsEqual(optimized, unoptimized)
+
+  @parameterized.named_parameters(
+      ("1", True, True),
+      ("2", True, False),
+      ("3", False, True),
+      ("4", False, False),
+  )
+  def testOptimizationWithChainedMapAndBatch(self, fuse_first, fuse_second):
+    # Tests that vectorization works on chained map and batch functions.
+    def map_fn(x):
+      return x * 2
+
+    unoptimized_seq = []
+
+    def make_apply_fn(is_fused):
+      if is_fused:
+        unoptimized_seq.append("MapAndBatch")
+
+        def apply_fn(dataset):
+          return dataset.apply(
+              batching.map_and_batch(map_fn, 2, 12, drop_remainder=True))
+
+        return apply_fn
+      else:
+        unoptimized_seq.extend(["ParallelMap", "Batch"])
+
+        def apply_fn(dataset):
+          return dataset.map(map_fn, 12).batch(2, drop_remainder=True)
+
+        return apply_fn
+
+    base_dataset = dataset_ops.Dataset.range(1000)
+    options = dataset_ops.Options()
+    options.experimental_optimization.apply_default_optimizations = False
+    base_dataset = base_dataset.with_options(options)
+
+    apply_fn_1 = make_apply_fn(fuse_first)
+    apply_fn_2 = make_apply_fn(fuse_second)
+
+    def make_dataset(node_names):
+      dataset = base_dataset.apply(optimization.assert_next(node_names))
+      dataset = apply_fn_1(dataset)
+      dataset = apply_fn_2(dataset)
+      return dataset
+
+    unoptimized = make_dataset(unoptimized_seq)
+    optimized = make_dataset(["Batch", "ParallelMap", "Batch", "ParallelMap"])
+    options = dataset_ops.Options()
+    options.experimental_optimization.map_vectorization = True
+    optimized = optimized.with_options(options)
+
+    self.assertDatasetsEqual(optimized, unoptimized)
+
   # TODO(b/117581999): Add eager coverage for the following tests.
   def testSkipEagerOptimizationIgnoreStateful(self):
 
-- 
GitLab


From 96a2610cc56910f27219c64d037245a96f07ef89 Mon Sep 17 00:00:00 2001
From: Pete Warden <petewarden@google.com>
Date: Tue, 22 Jan 2019 15:37:10 -0800
Subject: [PATCH 1086/1095] Remove binary from micro tests

PiperOrigin-RevId: 230424415
---
 tensorflow/lite/experimental/micro/tools/make/Makefile       | 5 +----
 .../lite/experimental/micro/tools/make/helper_functions.inc  | 4 +++-
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/tensorflow/lite/experimental/micro/tools/make/Makefile b/tensorflow/lite/experimental/micro/tools/make/Makefile
index fde195118b..8be57cc18d 100644
--- a/tensorflow/lite/experimental/micro/tools/make/Makefile
+++ b/tensorflow/lite/experimental/micro/tools/make/Makefile
@@ -156,9 +156,6 @@ include $(wildcard tensorflow/lite/experimental/micro/examples/*/Makefile.inc)
 MICROLITE_LIB_OBJS := $(addprefix $(OBJDIR), \
 $(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(MICROLITE_CC_SRCS))))
 
-MICROLITE_TEST_TARGETS := $(addprefix $(BINDIR), \
-$(patsubst %_test.cc,%.test_target,$(MICROLITE_TEST_SRCS)))
-
 # For normal manually-created TensorFlow C++ source files.
 $(OBJDIR)%.o: %.cc
 	@mkdir -p $(dir $@)
@@ -201,7 +198,7 @@ $(BINDIR)%.test_target: $(BINDIR)%_test
 $(foreach TEST_TARGET,$(MICROLITE_TEST_SRCS),\
 $(eval $(call microlite_test,$(notdir $(basename $(TEST_TARGET))),$(TEST_TARGET))))
 
-test: test_micro_speech $(MICROLITE_TEST_TARGETS)
+test: $(MICROLITE_TEST_TARGETS)
 
 # Gets rid of all generated files.
 clean:
diff --git a/tensorflow/lite/experimental/micro/tools/make/helper_functions.inc b/tensorflow/lite/experimental/micro/tools/make/helper_functions.inc
index 87c002635f..c20cc5cfb3 100644
--- a/tensorflow/lite/experimental/micro/tools/make/helper_functions.inc
+++ b/tensorflow/lite/experimental/micro/tools/make/helper_functions.inc
@@ -103,7 +103,6 @@ $(1)_LOCAL_HDRS := $(3)
 $(1)_LOCAL_OBJS := $$(addprefix $$(OBJDIR), \
 $$(patsubst %.cc,%.o,$$(patsubst %.c,%.o,$$($(1)_LOCAL_SRCS))))
 $(1)_BINARY := $$(BINDIR)$(1)
-ALL_BINARIES += $$($(1)_BINARY)
 $$($(1)_BINARY): $$($(1)_LOCAL_OBJS) $$(MICROLITE_LIB_PATH)
 	@mkdir -p $$(dir $$@)
 	$$(CXX) $$(CXXFLAGS) $$(INCLUDES) \
@@ -113,5 +112,8 @@ $(1): $$($(1)_BINARY)
 $(1)_bin: $$($(1)_BINARY).bin
 test_$(1): $$($(1)_BINARY)
 	$$(TEST_SCRIPT) $$($(1)_BINARY) '~~~ALL TESTS PASSED~~~'
+ifneq (,$(findstring _test,$(1)))
+  MICROLITE_TEST_TARGETS += test_$(1)
+endif
 $(eval $(call generate_microlite_projects,$(1),$(call specialize,$(2)),$(3)))
 endef
-- 
GitLab


From 824c73a1533fed37222ed5d071e04bae1fc0c3b1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 22 Jan 2019 15:40:52 -0800
Subject: [PATCH 1087/1095] Fix test size to enable running during presubmits.

PiperOrigin-RevId: 230425144
---
 tensorflow/contrib/eager/python/examples/densenet/BUILD | 6 ++++--
 tensorflow/contrib/eager/python/examples/l2hmc/BUILD    | 3 ++-
 tensorflow/contrib/eager/python/examples/resnet50/BUILD | 6 ++++--
 tensorflow/contrib/eager/python/examples/revnet/BUILD   | 9 ++++++---
 4 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/tensorflow/contrib/eager/python/examples/densenet/BUILD b/tensorflow/contrib/eager/python/examples/densenet/BUILD
index 56a682ec55..6e10d25acf 100644
--- a/tensorflow/contrib/eager/python/examples/densenet/BUILD
+++ b/tensorflow/contrib/eager/python/examples/densenet/BUILD
@@ -17,13 +17,14 @@ py_binary(
 
 cuda_py_test(
     name = "densenet_test",
-    size = "large",
+    size = "medium",
     srcs = ["densenet_test.py"],
     additional_deps = [
         ":densenet",
         "//tensorflow/contrib/eager/python:tfe",
         "//tensorflow:tensorflow_py",
     ],
+    shard_count = 4,
     tags = [
         "no_pip",
         "optonly",
@@ -33,13 +34,14 @@ cuda_py_test(
 
 cuda_py_test(
     name = "densenet_graph_test",
-    size = "large",
+    size = "medium",
     srcs = ["densenet_graph_test.py"],
     additional_deps = [
         ":densenet",
         "//third_party/py/numpy",
         "//tensorflow:tensorflow_py",
     ],
+    shard_count = 4,
     tags = [
         "no_pip",
         "noasan",
diff --git a/tensorflow/contrib/eager/python/examples/l2hmc/BUILD b/tensorflow/contrib/eager/python/examples/l2hmc/BUILD
index 78548c51c9..35d5099042 100644
--- a/tensorflow/contrib/eager/python/examples/l2hmc/BUILD
+++ b/tensorflow/contrib/eager/python/examples/l2hmc/BUILD
@@ -28,7 +28,7 @@ py_library(
 
 cuda_py_test(
     name = "l2hmc_test",
-    size = "large",
+    size = "medium",
     srcs = ["l2hmc_test.py"],
     additional_deps = [
         ":l2hmc",
@@ -36,6 +36,7 @@ cuda_py_test(
         "//tensorflow/contrib/eager/python:tfe",
         "//third_party/py/numpy",
     ],
+    shard_count = 4,
     tags = [
         "oss_serial",
     ],
diff --git a/tensorflow/contrib/eager/python/examples/resnet50/BUILD b/tensorflow/contrib/eager/python/examples/resnet50/BUILD
index 04ac78a2d3..f2851d9722 100644
--- a/tensorflow/contrib/eager/python/examples/resnet50/BUILD
+++ b/tensorflow/contrib/eager/python/examples/resnet50/BUILD
@@ -27,7 +27,7 @@ py_library(
 
 cuda_py_test(
     name = "resnet50_test",
-    size = "large",
+    size = "medium",
     srcs = ["resnet50_test.py"],
     additional_deps = [
         ":resnet50",
@@ -35,6 +35,7 @@ cuda_py_test(
         "//tensorflow/contrib/eager/python:tfe",
         "//tensorflow:tensorflow_py",
     ],
+    shard_count = 4,
     tags = [
         "noasan",  # Fix b/118130911
         "nomsan",  # Fix b/118130911
@@ -46,7 +47,7 @@ cuda_py_test(
 
 cuda_py_test(
     name = "resnet50_graph_test",
-    size = "large",
+    size = "medium",
     srcs = ["resnet50_graph_test.py"],
     additional_deps = [
         ":resnet50",
@@ -54,6 +55,7 @@ cuda_py_test(
         "//third_party/py/numpy",
         "//tensorflow:tensorflow_py",
     ],
+    shard_count = 4,
     tags = [
         "noasan",
         "nomsan",
diff --git a/tensorflow/contrib/eager/python/examples/revnet/BUILD b/tensorflow/contrib/eager/python/examples/revnet/BUILD
index d05270089f..1948c1d8ff 100644
--- a/tensorflow/contrib/eager/python/examples/revnet/BUILD
+++ b/tensorflow/contrib/eager/python/examples/revnet/BUILD
@@ -67,12 +67,13 @@ py_library(
 # Tests
 cuda_py_test(
     name = "ops_test",
-    size = "large",
+    size = "medium",
     srcs = ["ops_test.py"],
     additional_deps = [
         ":ops",
         "//tensorflow:tensorflow_py",
     ],
+    shard_count = 4,
     tags = [
         "oss_serial",
     ],
@@ -80,12 +81,13 @@ cuda_py_test(
 
 cuda_py_test(
     name = "blocks_test",
-    size = "large",
+    size = "medium",
     srcs = ["blocks_test.py"],
     additional_deps = [
         ":blocks",
         "//tensorflow:tensorflow_py",
     ],
+    shard_count = 4,
     tags = [
         "no_oss",  # b/123045964
         "optonly",
@@ -94,7 +96,7 @@ cuda_py_test(
 
 cuda_py_test(
     name = "revnet_test",
-    size = "large",
+    size = "medium",
     srcs = ["revnet_test.py"],
     additional_deps = [
         ":blocks_test",
@@ -102,6 +104,7 @@ cuda_py_test(
         ":revnet",
         "//tensorflow:tensorflow_py",
     ],
+    shard_count = 4,
     tags = [
         "no_pip",  # depends on blocks_test, which is not available in pip package
         "optonly",
-- 
GitLab


From fea5ff8e3dc49feef809d28ef0ac1fd48f6a7009 Mon Sep 17 00:00:00 2001
From: Jian Li <jianlijianli@google.com>
Date: Tue, 22 Jan 2019 15:53:15 -0800
Subject: [PATCH 1088/1095] Remove unused vector.

PiperOrigin-RevId: 230427369
---
 tensorflow/lite/tools/optimize/quantization_utils.cc | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tensorflow/lite/tools/optimize/quantization_utils.cc b/tensorflow/lite/tools/optimize/quantization_utils.cc
index 7239ebcc7a..c2405ca389 100644
--- a/tensorflow/lite/tools/optimize/quantization_utils.cc
+++ b/tensorflow/lite/tools/optimize/quantization_utils.cc
@@ -123,9 +123,6 @@ void SymmetricPerChannelQuantization(const float* const input,
   }
 
   // Quantize the values.
-  const uint64_t num_elements_per_channel =
-      output_scales->size() / channel_dim_size;
-  std::vector<int8_t> quantized_buffer(num_elements_per_channel);
   memset(indices, 0, 4 * sizeof(int));
   for (indices[0] = 0; indices[0] < dimension[0]; indices[0]++) {
     for (indices[1] = 0; indices[1] < dimension[1]; indices[1]++) {
-- 
GitLab


From 632658950c5f3acb1e05bc375bbcff90479934f0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 22 Jan 2019 15:54:34 -0800
Subject: [PATCH 1089/1095] Print the model name as part of
 layer_utils.print_summary (aka model.summary)

CURRENT:
_________________________________________________________________
Layer (type)                 Output Shape              Param #
=================================================================
model_1_input (InputLayer)   [(None, 128)]             0

PROPOSED
_________________________________________________________________
Model: "part_of_speech_tagger"
Layer (type)                 Output Shape              Param #
=================================================================
model_1_input (InputLayer)   [(None, 128)]             0
PiperOrigin-RevId: 230427583
---
 tensorflow/python/keras/utils/layer_utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/python/keras/utils/layer_utils.py b/tensorflow/python/keras/utils/layer_utils.py
index 1d85e8a25f..640462d5c6 100644
--- a/tensorflow/python/keras/utils/layer_utils.py
+++ b/tensorflow/python/keras/utils/layer_utils.py
@@ -158,6 +158,7 @@ def print_summary(model, line_length=None, positions=None, print_fn=None):
       line += ' ' * (positions[i] - len(line))
     print_fn(line)
 
+  print_fn('Model: "{}"'.format(model.name))
   print_fn('_' * line_length)
   print_row(to_display, positions)
   print_fn('=' * line_length)
-- 
GitLab


From 24baec1b8c2556c8c1ffe1fdf6f67b0324f4389c Mon Sep 17 00:00:00 2001
From: Youlong Cheng <ylc@google.com>
Date: Tue, 22 Jan 2019 16:00:46 -0800
Subject: [PATCH 1090/1095]   Internal change.

PiperOrigin-RevId: 230428660
---
 tensorflow/contrib/tpu/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/contrib/tpu/BUILD b/tensorflow/contrib/tpu/BUILD
index d898f05405..b3d5c41e7b 100644
--- a/tensorflow/contrib/tpu/BUILD
+++ b/tensorflow/contrib/tpu/BUILD
@@ -171,6 +171,7 @@ tf_custom_op_py_library(
         ":tpu_ordinal_selector_op_op_lib",
     ],
     srcs_version = "PY2AND3",
+    visibility = ["//visibility:public"],
     deps = [
         ":tpu_ordinal_selector_op",
     ],
-- 
GitLab


From b084482d79d28bb79f26c9b86f7ba654fcf25238 Mon Sep 17 00:00:00 2001
From: James Keeling <jtkeeling@google.com>
Date: Tue, 22 Jan 2019 16:12:11 -0800
Subject: [PATCH 1091/1095] Add bounds checking when looking at the stack in
 deprecation messages.

This was breaking integration with R, fixes #25067.

PiperOrigin-RevId: 230431304
---
 tensorflow/python/util/deprecation.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/util/deprecation.py b/tensorflow/python/util/deprecation.py
index 9aaf0c2de9..f8e8d4c28a 100644
--- a/tensorflow/python/util/deprecation.py
+++ b/tensorflow/python/util/deprecation.py
@@ -100,7 +100,13 @@ def _validate_deprecation_args(date, instructions):
 def _call_location(outer=False):
   """Returns call location given level up from current call."""
   stack = tf_stack.extract_stack()
-  frame = stack[-4 if outer else -3]
+  length = len(stack)
+  if length == 0:  # should never happen as we're in a function
+    return 'UNKNOWN'
+  index = length-4 if outer else length-3
+  if index < 0:
+    index = 0
+  frame = stack[index]
   return '{filename}:{lineno}'.format(filename=frame[0], lineno=frame[1])
 
 
-- 
GitLab


From d09e5afc188343492877ab815d4f7cf46c81ff4f Mon Sep 17 00:00:00 2001
From: James Keeling <jtkeeling@google.com>
Date: Tue, 22 Jan 2019 16:13:25 -0800
Subject: [PATCH 1092/1095] Split strided slice GPU code into multiple files

This file was a bottleneck during compilation, often taking many minutes to compile. In local testing this change reduces the wall-clock build time for the affected kernels from >3 mins to approx 1 min.

PiperOrigin-RevId: 230431569
---
 tensorflow/core/kernels/BUILD                 |  6 +++-
 .../kernels/strided_slice_op_gpu_bool.cu.cc   | 27 +++++++++++++++++
 .../strided_slice_op_gpu_complex.cu.cc        | 28 ++++++++++++++++++
 ..._gpu.cu.cc => strided_slice_op_gpu_impl.h} | 13 +++------
 .../kernels/strided_slice_op_gpu_int.cu.cc    | 29 +++++++++++++++++++
 .../strided_slice_op_gpu_number_types.cu.cc   | 27 +++++++++++++++++
 6 files changed, 120 insertions(+), 10 deletions(-)
 create mode 100644 tensorflow/core/kernels/strided_slice_op_gpu_bool.cu.cc
 create mode 100644 tensorflow/core/kernels/strided_slice_op_gpu_complex.cu.cc
 rename tensorflow/core/kernels/{strided_slice_op_gpu.cu.cc => strided_slice_op_gpu_impl.h} (90%)
 create mode 100644 tensorflow/core/kernels/strided_slice_op_gpu_int.cu.cc
 create mode 100644 tensorflow/core/kernels/strided_slice_op_gpu_number_types.cu.cc

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 049d429d36..ce6504e0b6 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -138,7 +138,11 @@ tf_kernel_library(
         "slice_op.h",
         "strided_slice_op.h",
         "strided_slice_op_impl.h",
-        "strided_slice_op_gpu.cu.cc",
+        "strided_slice_op_gpu_impl.h",
+        "strided_slice_op_gpu_int.cu.cc",
+        "strided_slice_op_gpu_complex.cu.cc",
+        "strided_slice_op_gpu_bool.cu.cc",
+        "strided_slice_op_gpu_number_types.cu.cc",
     ],
     deps = [
         ":bounds_check",
diff --git a/tensorflow/core/kernels/strided_slice_op_gpu_bool.cu.cc b/tensorflow/core/kernels/strided_slice_op_gpu_bool.cu.cc
new file mode 100644
index 0000000000..8c3f8f2ad3
--- /dev/null
+++ b/tensorflow/core/kernels/strided_slice_op_gpu_bool.cu.cc
@@ -0,0 +1,27 @@
+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/strided_slice_op.h"
+#include "tensorflow/core/kernels/strided_slice_op_gpu_impl.h"
+
+namespace tensorflow {
+TF_CALL_bool(DEFINE_GPU_KERNELS);
+}  // end namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/strided_slice_op_gpu_complex.cu.cc b/tensorflow/core/kernels/strided_slice_op_gpu_complex.cu.cc
new file mode 100644
index 0000000000..f695192465
--- /dev/null
+++ b/tensorflow/core/kernels/strided_slice_op_gpu_complex.cu.cc
@@ -0,0 +1,28 @@
+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/strided_slice_op.h"
+#include "tensorflow/core/kernels/strided_slice_op_gpu_impl.h"
+
+namespace tensorflow {
+TF_CALL_complex64(DEFINE_GPU_KERNELS);
+TF_CALL_complex128(DEFINE_GPU_KERNELS);
+}  // end namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/strided_slice_op_gpu.cu.cc b/tensorflow/core/kernels/strided_slice_op_gpu_impl.h
similarity index 90%
rename from tensorflow/core/kernels/strided_slice_op_gpu.cu.cc
rename to tensorflow/core/kernels/strided_slice_op_gpu_impl.h
index cce1d2fddd..d70f369ac0 100644
--- a/tensorflow/core/kernels/strided_slice_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/strided_slice_op_gpu_impl.h
@@ -13,6 +13,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#ifndef TENSORFLOW_CORE_KERNELS_STRIDED_SLICE_OP_GPU_IMPL_H_
+#define TENSORFLOW_CORE_KERNELS_STRIDED_SLICE_OP_GPU_IMPL_H_
+
 #if GOOGLE_CUDA
 
 #define EIGEN_USE_GPU
@@ -50,16 +53,8 @@ typedef Eigen::GpuDevice GPUDevice;
   template struct functor::StridedSliceAssign<GPUDevice, T, 6>; \
   template struct functor::StridedSliceAssign<GPUDevice, T, 7>; \
   template struct functor::StridedSliceAssignScalar<GPUDevice, T>;
-TF_CALL_GPU_NUMBER_TYPES(DEFINE_GPU_KERNELS);
-TF_CALL_complex64(DEFINE_GPU_KERNELS);
-TF_CALL_complex128(DEFINE_GPU_KERNELS);
-TF_CALL_int64(DEFINE_GPU_KERNELS);
-TF_CALL_bool(DEFINE_GPU_KERNELS);
-TF_CALL_int8(DEFINE_GPU_KERNELS);
-DEFINE_GPU_KERNELS(int32);
-
-#undef DEFINE_GPU_KERNELS
 
 }  // end namespace tensorflow
 
 #endif  // GOOGLE_CUDA
+#endif  // TENSORFLOW_CORE_KERNELS_STRIDED_SLICE_OP_GPU_IMPL_H_
diff --git a/tensorflow/core/kernels/strided_slice_op_gpu_int.cu.cc b/tensorflow/core/kernels/strided_slice_op_gpu_int.cu.cc
new file mode 100644
index 0000000000..07dd0130ad
--- /dev/null
+++ b/tensorflow/core/kernels/strided_slice_op_gpu_int.cu.cc
@@ -0,0 +1,29 @@
+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/strided_slice_op.h"
+#include "tensorflow/core/kernels/strided_slice_op_gpu_impl.h"
+
+namespace tensorflow {
+TF_CALL_int8(DEFINE_GPU_KERNELS);
+TF_CALL_int32(DEFINE_GPU_KERNELS);
+TF_CALL_int64(DEFINE_GPU_KERNELS);
+}  // end namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/strided_slice_op_gpu_number_types.cu.cc b/tensorflow/core/kernels/strided_slice_op_gpu_number_types.cu.cc
new file mode 100644
index 0000000000..149886308c
--- /dev/null
+++ b/tensorflow/core/kernels/strided_slice_op_gpu_number_types.cu.cc
@@ -0,0 +1,27 @@
+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/strided_slice_op.h"
+#include "tensorflow/core/kernels/strided_slice_op_gpu_impl.h"
+
+namespace tensorflow {
+TF_CALL_GPU_NUMBER_TYPES(DEFINE_GPU_KERNELS);
+}  // end namespace tensorflow
+
+#endif  // GOOGLE_CUDA
-- 
GitLab


From a6fe0dfd38a7b5f61e6259471baf2a44cc590570 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 22 Jan 2019 16:20:46 -0800
Subject: [PATCH 1093/1095] Fix the test size

PiperOrigin-RevId: 230433114
---
 tensorflow/python/keras/BUILD | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index c4c403119c..7705491c85 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -770,7 +770,7 @@ tf_py_test(
 
 tf_py_test(
     name = "data_utils_test",
-    size = "large",
+    size = "medium",
     srcs = ["utils/data_utils_test.py"],
     additional_deps = [
         ":keras",
@@ -778,6 +778,7 @@ tf_py_test(
         "//third_party/py/numpy",
         "//tensorflow/python:client_testlib",
     ],
+    shard_count = 6,
     tags = [
         "no_oss",
         "no_windows",
@@ -967,7 +968,7 @@ tf_py_test(
 
 tf_py_test(
     name = "training_generator_test",
-    size = "large",
+    size = "medium",
     srcs = ["engine/training_generator_test.py"],
     additional_deps = [
         ":keras",
@@ -975,7 +976,7 @@ tf_py_test(
         "//third_party/py/numpy",
         "//tensorflow/python:client_testlib",
     ],
-    shard_count = 3,
+    shard_count = 6,
     tags = [
         "no_oss",
         "notsan",
-- 
GitLab


From 6fa83a643f148c78a9bf4e50a8a7572accde5e09 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 22 Jan 2019 16:40:27 -0800
Subject: [PATCH 1094/1095] Fix the test size

PiperOrigin-RevId: 230436567
---
 tensorflow/python/BUILD | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 4690bed62b..bc109c09bc 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -5451,7 +5451,7 @@ tf_py_test(
 
 cuda_py_test(
     name = "accumulate_n_benchmark",
-    size = "large",
+    size = "medium",
     srcs = ["ops/accumulate_n_benchmark.py"],
     additional_deps = [
         ":array_ops",
@@ -5466,6 +5466,7 @@ cuda_py_test(
         ":state_ops_gen",
     ],
     main = "ops/accumulate_n_benchmark.py",
+    shard_count = 6,
 )
 
 cuda_py_test(
-- 
GitLab


From 09bdeffbcd2c4f8f9c0da8d101eaf9edc1faf54d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 22 Jan 2019 16:50:38 -0800
Subject: [PATCH 1095/1095] n/a

PiperOrigin-RevId: 230438132
---
 tensorflow/contrib/distribute/python/examples/BUILD |  6 ++++++
 tensorflow/contrib/eager/python/examples/BUILD      | 10 +++++-----
 .../contrib/eager/python/examples/densenet/BUILD    |  7 +++++++
 tensorflow/contrib/eager/python/examples/gan/BUILD  |  7 +++++++
 .../eager/python/examples/linear_regression/BUILD   |  7 +++++++
 .../contrib/eager/python/examples/revnet/BUILD      | 13 ++++++++++---
 .../eager/python/examples/rnn_colorbot/BUILD        |  7 +++++++
 .../contrib/eager/python/examples/rnn_ptb/BUILD     |  7 +++++++
 tensorflow/contrib/rnn/BUILD                        |  9 ++++++++-
 9 files changed, 64 insertions(+), 9 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/examples/BUILD b/tensorflow/contrib/distribute/python/examples/BUILD
index 84b106545e..75d9da6a96 100644
--- a/tensorflow/contrib/distribute/python/examples/BUILD
+++ b/tensorflow/contrib/distribute/python/examples/BUILD
@@ -31,6 +31,12 @@ py_binary(
 
 py_binary(
     name = "keras_mnist",
+    srcs = ["keras_mnist.py"],
+    deps = [":keras_mnist_lib"],
+)
+
+py_library(
+    name = "keras_mnist_lib",
     srcs = [
         "keras_mnist.py",
     ],
diff --git a/tensorflow/contrib/eager/python/examples/BUILD b/tensorflow/contrib/eager/python/examples/BUILD
index 97c299a911..3e0881754c 100644
--- a/tensorflow/contrib/eager/python/examples/BUILD
+++ b/tensorflow/contrib/eager/python/examples/BUILD
@@ -6,16 +6,16 @@ package(default_visibility = ["//tensorflow:internal"])
 py_library(
     name = "examples_pip",
     deps = [
-        "//tensorflow/contrib/eager/python/examples/densenet",
-        "//tensorflow/contrib/eager/python/examples/gan:mnist",
+        "//tensorflow/contrib/eager/python/examples/densenet:densenet_lib",
+        "//tensorflow/contrib/eager/python/examples/gan:mnist_lib",
         "//tensorflow/contrib/eager/python/examples/l2hmc",
         "//tensorflow/contrib/eager/python/examples/l2hmc:neural_nets",
-        "//tensorflow/contrib/eager/python/examples/linear_regression",
+        "//tensorflow/contrib/eager/python/examples/linear_regression:linear_regression_lib",
         "//tensorflow/contrib/eager/python/examples/resnet50",
         "//tensorflow/contrib/eager/python/examples/revnet",
         "//tensorflow/contrib/eager/python/examples/revnet:config",
-        "//tensorflow/contrib/eager/python/examples/rnn_colorbot",
-        "//tensorflow/contrib/eager/python/examples/rnn_ptb",
+        "//tensorflow/contrib/eager/python/examples/rnn_colorbot:rnn_colorbot_lib",
+        "//tensorflow/contrib/eager/python/examples/rnn_ptb:rnn_ptb_lib",
         "//tensorflow/contrib/eager/python/examples/spinn:data",
     ],
 )
diff --git a/tensorflow/contrib/eager/python/examples/densenet/BUILD b/tensorflow/contrib/eager/python/examples/densenet/BUILD
index 6e10d25acf..fbb5daf230 100644
--- a/tensorflow/contrib/eager/python/examples/densenet/BUILD
+++ b/tensorflow/contrib/eager/python/examples/densenet/BUILD
@@ -9,6 +9,13 @@ py_binary(
     name = "densenet",
     srcs = ["densenet.py"],
     srcs_version = "PY2AND3",
+    deps = [":densenet_lib"],
+)
+
+py_library(
+    name = "densenet_lib",
+    srcs = ["densenet.py"],
+    srcs_version = "PY2AND3",
     deps = [
         "//tensorflow:tensorflow_py",
         "//tensorflow/contrib/eager/python:tfe",
diff --git a/tensorflow/contrib/eager/python/examples/gan/BUILD b/tensorflow/contrib/eager/python/examples/gan/BUILD
index d64c8eb9ce..74b742515f 100644
--- a/tensorflow/contrib/eager/python/examples/gan/BUILD
+++ b/tensorflow/contrib/eager/python/examples/gan/BUILD
@@ -9,6 +9,13 @@ py_binary(
     name = "mnist",
     srcs = ["mnist.py"],
     srcs_version = "PY2AND3",
+    deps = [":mnist_lib"],
+)
+
+py_library(
+    name = "mnist_lib",
+    srcs = ["mnist.py"],
+    srcs_version = "PY2AND3",
     deps = [
         "//tensorflow:tensorflow_py",
         "//tensorflow/contrib/eager/python:tfe",
diff --git a/tensorflow/contrib/eager/python/examples/linear_regression/BUILD b/tensorflow/contrib/eager/python/examples/linear_regression/BUILD
index 6a178ddcec..2e7b8a013b 100644
--- a/tensorflow/contrib/eager/python/examples/linear_regression/BUILD
+++ b/tensorflow/contrib/eager/python/examples/linear_regression/BUILD
@@ -9,6 +9,13 @@ py_binary(
     name = "linear_regression",
     srcs = ["linear_regression.py"],
     srcs_version = "PY2AND3",
+    deps = [":linear_regression_lib"],
+)
+
+py_library(
+    name = "linear_regression_lib",
+    srcs = ["linear_regression.py"],
+    srcs_version = "PY2AND3",
     deps = [
         "//tensorflow:tensorflow_py",
         "//tensorflow/contrib/eager/python:tfe",
diff --git a/tensorflow/contrib/eager/python/examples/revnet/BUILD b/tensorflow/contrib/eager/python/examples/revnet/BUILD
index 1948c1d8ff..cb207b8ddf 100644
--- a/tensorflow/contrib/eager/python/examples/revnet/BUILD
+++ b/tensorflow/contrib/eager/python/examples/revnet/BUILD
@@ -135,6 +135,13 @@ py_binary(
     name = "main",
     srcs = ["main.py"],
     srcs_version = "PY2AND3",
+    deps = [":main_lib"],
+)
+
+py_library(
+    name = "main_lib",
+    srcs = ["main.py"],
+    srcs_version = "PY2AND3",
     deps = [
         ":cifar_input",
         ":config",
@@ -149,7 +156,7 @@ py_binary(
     srcs_version = "PY2AND3",
     deps = [
         ":cifar_input",
-        ":main",
+        ":main_lib",
         ":revnet",
         "//tensorflow:tensorflow_py",
     ],
@@ -161,7 +168,7 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":cifar_input",
-        ":main",
+        ":main_lib",
         ":revnet",
         "//tensorflow:tensorflow_py",
     ],
@@ -173,7 +180,7 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":cifar_input",
-        ":main",
+        ":main_lib",
         ":revnet",
         "//tensorflow:tensorflow_py",
     ],
diff --git a/tensorflow/contrib/eager/python/examples/rnn_colorbot/BUILD b/tensorflow/contrib/eager/python/examples/rnn_colorbot/BUILD
index 576f60396e..d4a854fb53 100644
--- a/tensorflow/contrib/eager/python/examples/rnn_colorbot/BUILD
+++ b/tensorflow/contrib/eager/python/examples/rnn_colorbot/BUILD
@@ -9,6 +9,13 @@ py_binary(
     name = "rnn_colorbot",
     srcs = ["rnn_colorbot.py"],
     srcs_version = "PY2AND3",
+    deps = [":rnn_colorbot_lib"],
+)
+
+py_library(
+    name = "rnn_colorbot_lib",
+    srcs = ["rnn_colorbot.py"],
+    srcs_version = "PY2AND3",
     deps = [
         "//tensorflow:tensorflow_py",
         "//tensorflow/contrib/eager/python:tfe",
diff --git a/tensorflow/contrib/eager/python/examples/rnn_ptb/BUILD b/tensorflow/contrib/eager/python/examples/rnn_ptb/BUILD
index 5d48ee4a5f..1c5a3842d5 100644
--- a/tensorflow/contrib/eager/python/examples/rnn_ptb/BUILD
+++ b/tensorflow/contrib/eager/python/examples/rnn_ptb/BUILD
@@ -9,6 +9,13 @@ py_binary(
     name = "rnn_ptb",
     srcs = ["rnn_ptb.py"],
     srcs_version = "PY2AND3",
+    deps = [":rnn_ptb_lib"],
+)
+
+py_library(
+    name = "rnn_ptb_lib",
+    srcs = ["rnn_ptb.py"],
+    srcs_version = "PY2AND3",
     deps = [
         "//tensorflow:tensorflow_py",
         "//tensorflow/contrib/cudnn_rnn:cudnn_rnn_py",
diff --git a/tensorflow/contrib/rnn/BUILD b/tensorflow/contrib/rnn/BUILD
index 39b6885968..d65d80df80 100644
--- a/tensorflow/contrib/rnn/BUILD
+++ b/tensorflow/contrib/rnn/BUILD
@@ -388,6 +388,13 @@ py_binary(
     name = "checkpoint_convert",
     srcs = ["python/tools/checkpoint_convert.py"],
     srcs_version = "PY2AND3",
+    deps = [":checkpoint_convert_lib"],
+)
+
+py_library(
+    name = "checkpoint_convert_lib",
+    srcs = ["python/tools/checkpoint_convert.py"],
+    srcs_version = "PY2AND3",
     deps = [
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python:framework_ops",
@@ -406,7 +413,7 @@ py_test(
     srcs_version = "PY2AND3",
     tags = ["no_pip"],
     deps = [
-        ":checkpoint_convert",
+        ":checkpoint_convert_lib",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:session",
-- 
GitLab